From 4035b1bfb1e5843a539a8b624d21952b756974d1 Mon Sep 17 00:00:00 2001
From: Daniel Baumann <daniel.baumann@progress-linux.org>
Date: Sat, 27 Apr 2024 16:19:18 +0200
Subject: Adding upstream version 6.1.22-dfsg.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
---
 src/VBox/VMM/.scm-settings                         |    36 +
 src/VBox/VMM/Config.kmk                            |    91 +
 src/VBox/VMM/Docs-CodingGuidelines.cpp             |    93 +
 src/VBox/VMM/Docs-RawMode.cpp                      |   148 +
 src/VBox/VMM/Makefile.kmk                          |   775 +
 src/VBox/VMM/VBoxVMM.d                             |   397 +
 src/VBox/VMM/VMMAll/APICAll.cpp                    |  3610 ++++
 src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp             |   101 +
 src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp                |  6414 +++++++
 src/VBox/VMM/VMMAll/CPUMAllRegs.cpp                |  3033 ++++
 src/VBox/VMM/VMMAll/DBGFAll.cpp                    |   410 +
 src/VBox/VMM/VMMAll/EMAll.cpp                      |  1237 ++
 src/VBox/VMM/VMMAll/GIMAll.cpp                     |   492 +
 src/VBox/VMM/VMMAll/GIMAllHv.cpp                   |  1487 ++
 src/VBox/VMM/VMMAll/GIMAllKvm.cpp                  |   441 +
 src/VBox/VMM/VMMAll/HMAll.cpp                      |   905 +
 src/VBox/VMM/VMMAll/HMSVMAll.cpp                   |   521 +
 src/VBox/VMM/VMMAll/HMVMXAll.cpp                   |  1315 ++
 src/VBox/VMM/VMMAll/IEMAll.cpp                     | 16422 ++++++++++++++++++
 src/VBox/VMM/VMMAll/IEMAllAImpl.asm                |  3024 ++++
 src/VBox/VMM/VMMAll/IEMAllAImplC.cpp               |  1450 ++
 src/VBox/VMM/VMMAll/IEMAllCImpl.cpp.h              |  9046 ++++++++++
 src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h      |  1762 ++
 src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp.h      |  1430 ++
 src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp.h      |  8953 ++++++++++
 src/VBox/VMM/VMMAll/IEMAllInstructions.cpp.h       |   776 +
 src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h  |   133 +
 .../VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h     | 11818 +++++++++++++
 src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py    |  3568 ++++
 .../VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h   |   905 +
 .../VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h   |   502 +
 .../VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h   |  9779 +++++++++++
 .../VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h     |  4054 +++++
 .../VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h     |   940 +
 .../VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h     |   557 +
 src/VBox/VMM/VMMAll/IOMAll.cpp                     |   596 +
 src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp              |  1274 ++
 src/VBox/VMM/VMMAll/MMAll.cpp                      |   666 +
 src/VBox/VMM/VMMAll/MMAllHyper.cpp                 |  1337 ++
 src/VBox/VMM/VMMAll/Makefile.kup                   |     0
 src/VBox/VMM/VMMAll/NEMAll.cpp                     |   153 +
 src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h |  4961 ++++++
 src/VBox/VMM/VMMAll/PDMAll.cpp                     |   343 +
 src/VBox/VMM/VMMAll/PDMAllCritSect.cpp             |   830 +
 src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp         |    97 +
 src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp           |  1445 ++
 src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp            |    77 +
 src/VBox/VMM/VMMAll/PDMAllQueue.cpp                |   208 +
 src/VBox/VMM/VMMAll/PDMAllTask.cpp                 |   114 +
 src/VBox/VMM/VMMAll/PGMAll.cpp                     |  3779 ++++
 src/VBox/VMM/VMMAll/PGMAllBth.h                    |  4611 +++++
 src/VBox/VMM/VMMAll/PGMAllGst.h                    |   521 +
 src/VBox/VMM/VMMAll/PGMAllHandler.cpp              |  1768 ++
 src/VBox/VMM/VMMAll/PGMAllMap.cpp                  |   930 +
 src/VBox/VMM/VMMAll/PGMAllPhys.cpp                 |  4729 +++++
 src/VBox/VMM/VMMAll/PGMAllPool.cpp                 |  5552 ++++++
 src/VBox/VMM/VMMAll/PGMAllShw.h                    |   617 +
 src/VBox/VMM/VMMAll/SELMAll.cpp                    |   388 +
 src/VBox/VMM/VMMAll/TMAll.cpp                      |  2617 +++
 src/VBox/VMM/VMMAll/TMAllCpu.cpp                   |   605 +
 src/VBox/VMM/VMMAll/TMAllReal.cpp                  |    53 +
 src/VBox/VMM/VMMAll/TMAllVirtual.cpp               |   998 ++
 src/VBox/VMM/VMMAll/TRPMAll.cpp                    |   429 +
 src/VBox/VMM/VMMAll/VMAll.cpp                      |   429 +
 src/VBox/VMM/VMMAll/VMMAll.cpp                     |   370 +
 src/VBox/VMM/VMMAll/VMMAllA.asm                    |    83 +
 src/VBox/VMM/VMMR0/CPUMR0.cpp                      |   954 +
 src/VBox/VMM/VMMR0/CPUMR0A.asm                     |   358 +
 src/VBox/VMM/VMMR0/EMR0.cpp                        |    61 +
 src/VBox/VMM/VMMR0/GIMR0.cpp                       |   111 +
 src/VBox/VMM/VMMR0/GIMR0Hv.cpp                     |   182 +
 src/VBox/VMM/VMMR0/GMMR0.cpp                       |  5746 ++++++
 src/VBox/VMM/VMMR0/GMMR0Internal.h                 |   116 +
 src/VBox/VMM/VMMR0/GVMMR0.cpp                      |  3029 ++++
 src/VBox/VMM/VMMR0/GVMMR0Internal.h                |    73 +
 src/VBox/VMM/VMMR0/HMR0.cpp                        |  1862 ++
 src/VBox/VMM/VMMR0/HMR0A.asm                       |  1705 ++
 src/VBox/VMM/VMMR0/HMSVMR0.cpp                     |  7847 +++++++++
 src/VBox/VMM/VMMR0/HMSVMR0.h                       |    82 +
 src/VBox/VMM/VMMR0/HMVMXR0.cpp                     | 17380 +++++++++++++++++++
 src/VBox/VMM/VMMR0/HMVMXR0.h                       |    56 +
 src/VBox/VMM/VMMR0/IOMR0.cpp                       |    57 +
 src/VBox/VMM/VMMR0/IOMR0IoPort.cpp                 |   382 +
 src/VBox/VMM/VMMR0/IOMR0Mmio.cpp                   |   378 +
 src/VBox/VMM/VMMR0/Makefile.kup                    |     0
 src/VBox/VMM/VMMR0/NEMR0Native-win.cpp             |  2616 +++
 src/VBox/VMM/VMMR0/PDMR0DevHlp.cpp                 |  1558 ++
 src/VBox/VMM/VMMR0/PDMR0Device.cpp                 |   803 +
 src/VBox/VMM/VMMR0/PDMR0Driver.cpp                 |   163 +
 src/VBox/VMM/VMMR0/PGMR0.cpp                       |   807 +
 src/VBox/VMM/VMMR0/PGMR0Bth.h                      |    25 +
 src/VBox/VMM/VMMR0/PGMR0Pool.cpp                   |   153 +
 src/VBox/VMM/VMMR0/PGMR0SharedPage.cpp             |   171 +
 src/VBox/VMM/VMMR0/VMMR0.cpp                       |  2753 +++
 src/VBox/VMM/VMMR0/VMMR0.def                       |   120 +
 src/VBox/VMM/VMMR0/VMMR0JmpA-amd64.asm             |   491 +
 src/VBox/VMM/VMMR0/VMMR0JmpA-x86.asm               |   401 +
 src/VBox/VMM/VMMR0/VMMR0TripleFaultHack.cpp        |   209 +
 src/VBox/VMM/VMMR0/VMMR0TripleFaultHackA.asm       |   264 +
 src/VBox/VMM/VMMR3/APIC.cpp                        |  1564 ++
 src/VBox/VMM/VMMR3/CFGM.cpp                        |  3273 ++++
 src/VBox/VMM/VMMR3/CPUM.cpp                        |  4627 +++++
 src/VBox/VMM/VMMR3/CPUMDbg.cpp                     |  1260 ++
 src/VBox/VMM/VMMR3/CPUMR3CpuId.cpp                 |  7232 ++++++++
 src/VBox/VMM/VMMR3/CPUMR3Db.cpp                    |  1162 ++
 src/VBox/VMM/VMMR3/DBGF.cpp                        |  2106 +++
 src/VBox/VMM/VMMR3/DBGFAddr.cpp                    |   485 +
 src/VBox/VMM/VMMR3/DBGFAddrSpace.cpp               |  1351 ++
 src/VBox/VMM/VMMR3/DBGFBp.cpp                      |  1426 ++
 src/VBox/VMM/VMMR3/DBGFCoreWrite.cpp               |   664 +
 src/VBox/VMM/VMMR3/DBGFCpu.cpp                     |   163 +
 src/VBox/VMM/VMMR3/DBGFDisas.cpp                   |   790 +
 src/VBox/VMM/VMMR3/DBGFInfo.cpp                    |  1464 ++
 src/VBox/VMM/VMMR3/DBGFLog.cpp                     |   187 +
 src/VBox/VMM/VMMR3/DBGFMem.cpp                     |   643 +
 src/VBox/VMM/VMMR3/DBGFModule.cpp                  |   290 +
 src/VBox/VMM/VMMR3/DBGFOS.cpp                      |   661 +
 src/VBox/VMM/VMMR3/DBGFR3BugCheck.cpp              |   920 +
 src/VBox/VMM/VMMR3/DBGFR3Flow.cpp                  |  2260 +++
 src/VBox/VMM/VMMR3/DBGFR3ModInMem.cpp              |  1098 ++
 src/VBox/VMM/VMMR3/DBGFR3PlugIn.cpp                |   616 +
 src/VBox/VMM/VMMR3/DBGFR3Trace.cpp                 |   447 +
 src/VBox/VMM/VMMR3/DBGFR3Type.cpp                  |  1278 ++
 src/VBox/VMM/VMMR3/DBGFReg.cpp                     |  2719 +++
 src/VBox/VMM/VMMR3/DBGFStack.cpp                   |  1153 ++
 src/VBox/VMM/VMMR3/EM.cpp                          |  2869 +++
 src/VBox/VMM/VMMR3/EMHM.cpp                        |   477 +
 src/VBox/VMM/VMMR3/EMR3Dbg.cpp                     |   338 +
 src/VBox/VMM/VMMR3/EMR3Nem.cpp                     |   475 +
 src/VBox/VMM/VMMR3/GIM.cpp                         |   696 +
 src/VBox/VMM/VMMR3/GIMHv.cpp                       |  2287 +++
 src/VBox/VMM/VMMR3/GIMKvm.cpp                      |   620 +
 src/VBox/VMM/VMMR3/GIMMinimal.cpp                  |   131 +
 src/VBox/VMM/VMMR3/GMM.cpp                         |   451 +
 src/VBox/VMM/VMMR3/HM.cpp                          |  3446 ++++
 src/VBox/VMM/VMMR3/IEMR3.cpp                       |   211 +
 src/VBox/VMM/VMMR3/IOM.cpp                         |   460 +
 src/VBox/VMM/VMMR3/IOMR3IoPort.cpp                 |   629 +
 src/VBox/VMM/VMMR3/IOMR3Mmio.cpp                   |   553 +
 src/VBox/VMM/VMMR3/MM.cpp                          |   849 +
 src/VBox/VMM/VMMR3/MMHeap.cpp                      |   751 +
 src/VBox/VMM/VMMR3/MMHyper.cpp                     |  1516 ++
 src/VBox/VMM/VMMR3/MMPagePool.cpp                  |    72 +
 src/VBox/VMM/VMMR3/MMUkHeap.cpp                    |   427 +
 src/VBox/VMM/VMMR3/Makefile.kup                    |     0
 src/VBox/VMM/VMMR3/NEMR3.cpp                       |   531 +
 src/VBox/VMM/VMMR3/NEMR3Native-win.cpp             |  2790 +++
 src/VBox/VMM/VMMR3/PDM.cpp                         |  3079 ++++
 src/VBox/VMM/VMMR3/PDMAsyncCompletion.cpp          |  1805 ++
 src/VBox/VMM/VMMR3/PDMAsyncCompletionFile.cpp      |  1293 ++
 .../VMM/VMMR3/PDMAsyncCompletionFileFailsafe.cpp   |   270 +
 .../VMM/VMMR3/PDMAsyncCompletionFileNormal.cpp     |  1736 ++
 src/VBox/VMM/VMMR3/PDMBlkCache.cpp                 |  2809 +++
 src/VBox/VMM/VMMR3/PDMCritSect.cpp                 |  1084 ++
 src/VBox/VMM/VMMR3/PDMDevHlp.cpp                   |  4717 +++++
 src/VBox/VMM/VMMR3/PDMDevMiscHlp.cpp               |   410 +
 src/VBox/VMM/VMMR3/PDMDevice.cpp                   |  1211 ++
 src/VBox/VMM/VMMR3/PDMDriver.cpp                   |  1877 ++
 src/VBox/VMM/VMMR3/PDMLdr.cpp                      |  1769 ++
 src/VBox/VMM/VMMR3/PDMNetShaper.cpp                |   551 +
 src/VBox/VMM/VMMR3/PDMQueue.cpp                    |   877 +
 src/VBox/VMM/VMMR3/PDMR3Task.cpp                   |   628 +
 src/VBox/VMM/VMMR3/PDMThread.cpp                   |  1093 ++
 src/VBox/VMM/VMMR3/PDMUsb.cpp                      |  2004 +++
 src/VBox/VMM/VMMR3/PGM.cpp                         |  2782 +++
 src/VBox/VMM/VMMR3/PGMDbg.cpp                      |  2863 +++
 src/VBox/VMM/VMMR3/PGMHandler.cpp                  |   395 +
 src/VBox/VMM/VMMR3/PGMMap.cpp                      |  1472 ++
 src/VBox/VMM/VMMR3/PGMPhys.cpp                     |  5486 ++++++
 src/VBox/VMM/VMMR3/PGMPhysRWTmpl.h                 |    61 +
 src/VBox/VMM/VMMR3/PGMPool.cpp                     |   929 +
 src/VBox/VMM/VMMR3/PGMR3DbgA.asm                   |   475 +
 src/VBox/VMM/VMMR3/PGMSavedState.cpp               |  3303 ++++
 src/VBox/VMM/VMMR3/PGMSharedPage.cpp               |   442 +
 src/VBox/VMM/VMMR3/SELM.cpp                        |   671 +
 src/VBox/VMM/VMMR3/SSM.cpp                         |  9923 +++++++++++
 src/VBox/VMM/VMMR3/STAM.cpp                        |  3090 ++++
 src/VBox/VMM/VMMR3/TM.cpp                          |  4071 +++++
 src/VBox/VMM/VMMR3/TRPM.cpp                        |   461 +
 src/VBox/VMM/VMMR3/VM.cpp                          |  4428 +++++
 src/VBox/VMM/VMMR3/VMEmt.cpp                       |  1437 ++
 src/VBox/VMM/VMMR3/VMM.cpp                         |  2650 +++
 src/VBox/VMM/VMMR3/VMMGuruMeditation.cpp           |   685 +
 src/VBox/VMM/VMMR3/VMMR3.def                       |   457 +
 src/VBox/VMM/VMMR3/VMMTests.cpp                    |   188 +
 src/VBox/VMM/VMMR3/VMReq.cpp                       |  1333 ++
 src/VBox/VMM/VMMR3/cpus/AMD_Athlon_64_3200.h       |   224 +
 .../VMMR3/cpus/AMD_Athlon_64_X2_Dual_Core_4200.h   |   232 +
 src/VBox/VMM/VMMR3/cpus/AMD_FX_8150_Eight_Core.h   |   383 +
 src/VBox/VMM/VMMR3/cpus/AMD_Phenom_II_X6_1100T.h   |   272 +
 src/VBox/VMM/VMMR3/cpus/Hygon_C86_7185_32_core.h   |  5222 ++++++
 src/VBox/VMM/VMMR3/cpus/Intel_80186.h              |    75 +
 src/VBox/VMM/VMMR3/cpus/Intel_80286.h              |    75 +
 src/VBox/VMM/VMMR3/cpus/Intel_80386.h              |    75 +
 src/VBox/VMM/VMMR3/cpus/Intel_80486.h              |    73 +
 src/VBox/VMM/VMMR3/cpus/Intel_8086.h               |    75 +
 src/VBox/VMM/VMMR3/cpus/Intel_Atom_330_1_60GHz.h   |   210 +
 .../VMM/VMMR3/cpus/Intel_Core2_T7600_2_33GHz.h     |   195 +
 .../VMM/VMMR3/cpus/Intel_Core2_X6800_2_93GHz.h     |   260 +
 .../VMM/VMMR3/cpus/Intel_Core_Duo_T2600_2_16GHz.h  |   225 +
 src/VBox/VMM/VMMR3/cpus/Intel_Core_i5_3570.h       |   339 +
 src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_2635QM.h     |   332 +
 src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_3820QM.h     |   386 +
 src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_3960X.h      |   369 +
 src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_5600U.h      |   368 +
 src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_6700K.h      |   510 +
 src/VBox/VMM/VMMR3/cpus/Intel_Pentium_4_3_00GHz.h  |   277 +
 .../VMMR3/cpus/Intel_Pentium_M_processor_2_00GHz.h |   216 +
 .../VMM/VMMR3/cpus/Intel_Pentium_N3530_2_16GHz.h   |   265 +
 src/VBox/VMM/VMMR3/cpus/Intel_Xeon_X5482_3_20GHz.h |   248 +
 src/VBox/VMM/VMMR3/cpus/Makefile.kup               |     0
 .../VMM/VMMR3/cpus/Quad_Core_AMD_Opteron_2384.h    |   270 +
 .../VMM/VMMR3/cpus/VIA_QuadCore_L4700_1_2_GHz.h    |   404 +
 .../VMMR3/cpus/ZHAOXIN_KaiXian_KX_U5581_1_8GHz.h   |   417 +
 src/VBox/VMM/VMMRZ/CPUMRZ.cpp                      |   144 +
 src/VBox/VMM/VMMRZ/CPUMRZA.asm                     |   384 +
 src/VBox/VMM/VMMRZ/DBGFRZ.cpp                      |   240 +
 src/VBox/VMM/VMMRZ/Makefile.kup                    |     0
 src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp                 |  2692 +++
 src/VBox/VMM/VMMRZ/VMMRZ.cpp                       |   253 +
 src/VBox/VMM/dtrace/int-1.d                        |   125 +
 src/VBox/VMM/dtrace/lib/amd64/vbox-arch-types.d    |    39 +
 src/VBox/VMM/dtrace/lib/vbox-types.d               |    58 +
 src/VBox/VMM/dtrace/lib/x86/vbox-arch-types.d      |    22 +
 .../VMM/dtrace/return-to-ring-3-aggregation-1.d    |    36 +
 src/VBox/VMM/dtrace/vmexit-reason-aggregation-1.d  |    36 +
 src/VBox/VMM/dtrace/vmexit-rip-aggregation-1.d     |    32 +
 src/VBox/VMM/include/APICInternal.h                |  1426 ++
 src/VBox/VMM/include/CFGMInternal.h                |   134 +
 src/VBox/VMM/include/CPUMInternal.h                |   534 +
 src/VBox/VMM/include/CPUMInternal.mac              |   709 +
 src/VBox/VMM/include/DBGFInternal.h                |   605 +
 src/VBox/VMM/include/EMHandleRCTmpl.h              |   261 +
 src/VBox/VMM/include/EMInternal.h                  |   368 +
 src/VBox/VMM/include/GIMHvInternal.h               |  1375 ++
 src/VBox/VMM/include/GIMInternal.h                 |   123 +
 src/VBox/VMM/include/GIMKvmInternal.h              |   272 +
 src/VBox/VMM/include/GIMMinimalInternal.h          |    38 +
 src/VBox/VMM/include/HMInternal.h                  |  1239 ++
 src/VBox/VMM/include/HMInternal.mac                |    45 +
 src/VBox/VMM/include/IEMInternal.h                 |  1902 ++
 src/VBox/VMM/include/IOMInline.h                   |   260 +
 src/VBox/VMM/include/IOMInternal.h                 |   610 +
 src/VBox/VMM/include/MMInternal.h                  |   661 +
 src/VBox/VMM/include/NEMInternal.h                 |   453 +
 .../VMM/include/PDMAsyncCompletionFileInternal.h   |   566 +
 src/VBox/VMM/include/PDMAsyncCompletionInternal.h  |   281 +
 src/VBox/VMM/include/PDMBlkCacheInternal.h         |   334 +
 src/VBox/VMM/include/PDMInline.h                   |    42 +
 src/VBox/VMM/include/PDMInternal.h                 |  1538 ++
 src/VBox/VMM/include/PDMNetShaperInternal.h        |    54 +
 src/VBox/VMM/include/PGMGstDefs.h                  |   231 +
 src/VBox/VMM/include/PGMInline.h                   |  1435 ++
 src/VBox/VMM/include/PGMInternal.h                 |  4073 +++++
 src/VBox/VMM/include/SELMInternal.h                |    62 +
 src/VBox/VMM/include/SSMInternal.h                 |   331 +
 src/VBox/VMM/include/STAMInternal.h                |   177 +
 src/VBox/VMM/include/TMInline.h                    |    59 +
 src/VBox/VMM/include/TMInternal.h                  |   843 +
 src/VBox/VMM/include/TRPMInternal.h                |    96 +
 src/VBox/VMM/include/VMInternal.h                  |   485 +
 src/VBox/VMM/include/VMMInternal.h                 |   589 +
 src/VBox/VMM/include/VMMInternal.mac               |   141 +
 src/VBox/VMM/include/VMMTracing.h                  |   126 +
 src/VBox/VMM/pure_test.sh                          |    84 +
 .../testcase/Instructions/InstructionTestGen.py    |  2239 +++
 src/VBox/VMM/testcase/Instructions/Makefile.kmk    |    69 +
 .../testcase/Instructions/env-bs2-r0-32-big.mac    |    35 +
 .../testcase/Instructions/env-bs2-r0-64-big.mac    |    35 +
 .../VMM/testcase/Instructions/env-bs2-r0-64.mac    |    35 +
 .../VMM/testcase/Instructions/env-bs2-r0-big.mac   |    57 +
 .../testcase/Instructions/env-bs2-r0-common.mac    |   115 +
 src/VBox/VMM/testcase/Instructions/env-bs2-r0.mac  |    53 +
 src/VBox/VMM/testcase/Instructions/env-common.mac  |   346 +
 .../VMM/testcase/Instructions/env-iprt-r3-32.mac   |    19 +
 .../VMM/testcase/Instructions/env-iprt-r3-64.mac   |    19 +
 src/VBox/VMM/testcase/Instructions/env-iprt-r3.mac |    99 +
 src/VBox/VMM/testcase/Instructions/itgTableDaa.py  |  1105 ++
 src/VBox/VMM/testcase/Instructions/itgTableDas.py  |  1105 ++
 .../VMM/testcase/Instructions/tstVBInsTstR3.cpp    |   120 +
 src/VBox/VMM/testcase/Makefile.kmk                 |   653 +
 src/VBox/VMM/testcase/NemRawBench-1.cpp            |  1346 ++
 src/VBox/VMM/testcase/dev.tar.gz                   |   Bin 0 -> 732 bytes
 src/VBox/VMM/testcase/mkdsk.sh                     |    76 +
 src/VBox/VMM/testcase/tstAnimate.cpp               |   943 +
 src/VBox/VMM/testcase/tstAsmStructs.cpp            |    54 +
 src/VBox/VMM/testcase/tstAsmStructsAsm-lst.sed     |   105 +
 src/VBox/VMM/testcase/tstAsmStructsAsm.asm         |    39 +
 src/VBox/VMM/testcase/tstCFGM.cpp                  |   171 +
 src/VBox/VMM/testcase/tstCompressionBenchmark.cpp  |   642 +
 src/VBox/VMM/testcase/tstGlobalConfig.cpp          |   138 +
 src/VBox/VMM/testcase/tstHelp.h                    |   169 +
 src/VBox/VMM/testcase/tstIEMCheckMc.cpp            |   769 +
 src/VBox/VMM/testcase/tstMMHyperHeap.cpp           |   284 +
 src/VBox/VMM/testcase/tstMicro.h                   |   146 +
 src/VBox/VMM/testcase/tstMicro.mac                 |    40 +
 src/VBox/VMM/testcase/tstMicroRC.cpp               |   258 +
 src/VBox/VMM/testcase/tstMicroRC.def               |    28 +
 src/VBox/VMM/testcase/tstMicroRCA.asm              |   558 +
 src/VBox/VMM/testcase/tstPDMAsyncCompletion.cpp    |   274 +
 .../VMM/testcase/tstPDMAsyncCompletionStress.cpp   |   648 +
 src/VBox/VMM/testcase/tstSSM-2.cpp                 |    86 +
 src/VBox/VMM/testcase/tstSSM.cpp                   |   935 +
 src/VBox/VMM/testcase/tstVMM-HM.cpp                |   121 +
 src/VBox/VMM/testcase/tstVMMFork.cpp               |   170 +
 src/VBox/VMM/testcase/tstVMMR0CallHost-1.cpp       |   181 +
 src/VBox/VMM/testcase/tstVMREQ.cpp                 |   344 +
 src/VBox/VMM/testcase/tstVMStruct.h                |  1494 ++
 src/VBox/VMM/testcase/tstVMStructDTrace.cpp        |   144 +
 src/VBox/VMM/testcase/tstVMStructRC.cpp            |    99 +
 src/VBox/VMM/testcase/tstVMStructSize.cpp          |   464 +
 src/VBox/VMM/testcase/tstX86-1.cpp                 |   270 +
 src/VBox/VMM/testcase/tstX86-1A.asm                |  3443 ++++
 src/VBox/VMM/testcase/tstX86-FpuSaveRestore.cpp    |   116 +
 src/VBox/VMM/testcase/tstX86-FpuSaveRestoreA.asm   |   117 +
 src/VBox/VMM/tools/Makefile.kmk                    |    76 +
 src/VBox/VMM/tools/VBoxCpuReport.cpp               |  4989 ++++++
 src/VBox/VMM/tools/VBoxCpuReport.h                 |    53 +
 src/VBox/VMM/tools/VBoxCpuReportMsrLinux.cpp       |   170 +
 src/VBox/VMM/tools/VBoxCpuReportMsrSup.cpp         |    54 +
 src/VBox/VMM/tools/VBoxVMMPreload.cpp              |   224 +
 src/VBox/VMM/tools/VBoxVMMPreloadHardened.cpp      |    25 +
 322 files changed, 388338 insertions(+)
 create mode 100644 src/VBox/VMM/.scm-settings
 create mode 100644 src/VBox/VMM/Config.kmk
 create mode 100644 src/VBox/VMM/Docs-CodingGuidelines.cpp
 create mode 100644 src/VBox/VMM/Docs-RawMode.cpp
 create mode 100644 src/VBox/VMM/Makefile.kmk
 create mode 100644 src/VBox/VMM/VBoxVMM.d
 create mode 100644 src/VBox/VMM/VMMAll/APICAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp
 create mode 100644 src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp
 create mode 100644 src/VBox/VMM/VMMAll/CPUMAllRegs.cpp
 create mode 100644 src/VBox/VMM/VMMAll/DBGFAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/EMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/GIMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/GIMAllHv.cpp
 create mode 100644 src/VBox/VMM/VMMAll/GIMAllKvm.cpp
 create mode 100644 src/VBox/VMM/VMMAll/HMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/HMSVMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/HMVMXAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/IEMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllAImpl.asm
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllAImplC.cpp
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllCImpl.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructions.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h
 create mode 100755 src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/IOMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp
 create mode 100644 src/VBox/VMM/VMMAll/MMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/MMAllHyper.cpp
 create mode 100644 src/VBox/VMM/VMMAll/Makefile.kup
 create mode 100644 src/VBox/VMM/VMMAll/NEMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h
 create mode 100644 src/VBox/VMM/VMMAll/PDMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/PDMAllCritSect.cpp
 create mode 100644 src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp
 create mode 100644 src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp
 create mode 100644 src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp
 create mode 100644 src/VBox/VMM/VMMAll/PDMAllQueue.cpp
 create mode 100644 src/VBox/VMM/VMMAll/PDMAllTask.cpp
 create mode 100644 src/VBox/VMM/VMMAll/PGMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/PGMAllBth.h
 create mode 100644 src/VBox/VMM/VMMAll/PGMAllGst.h
 create mode 100644 src/VBox/VMM/VMMAll/PGMAllHandler.cpp
 create mode 100644 src/VBox/VMM/VMMAll/PGMAllMap.cpp
 create mode 100644 src/VBox/VMM/VMMAll/PGMAllPhys.cpp
 create mode 100644 src/VBox/VMM/VMMAll/PGMAllPool.cpp
 create mode 100644 src/VBox/VMM/VMMAll/PGMAllShw.h
 create mode 100644 src/VBox/VMM/VMMAll/SELMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/TMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/TMAllCpu.cpp
 create mode 100644 src/VBox/VMM/VMMAll/TMAllReal.cpp
 create mode 100644 src/VBox/VMM/VMMAll/TMAllVirtual.cpp
 create mode 100644 src/VBox/VMM/VMMAll/TRPMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/VMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/VMMAll.cpp
 create mode 100644 src/VBox/VMM/VMMAll/VMMAllA.asm
 create mode 100644 src/VBox/VMM/VMMR0/CPUMR0.cpp
 create mode 100644 src/VBox/VMM/VMMR0/CPUMR0A.asm
 create mode 100644 src/VBox/VMM/VMMR0/EMR0.cpp
 create mode 100644 src/VBox/VMM/VMMR0/GIMR0.cpp
 create mode 100644 src/VBox/VMM/VMMR0/GIMR0Hv.cpp
 create mode 100644 src/VBox/VMM/VMMR0/GMMR0.cpp
 create mode 100644 src/VBox/VMM/VMMR0/GMMR0Internal.h
 create mode 100644 src/VBox/VMM/VMMR0/GVMMR0.cpp
 create mode 100644 src/VBox/VMM/VMMR0/GVMMR0Internal.h
 create mode 100644 src/VBox/VMM/VMMR0/HMR0.cpp
 create mode 100644 src/VBox/VMM/VMMR0/HMR0A.asm
 create mode 100644 src/VBox/VMM/VMMR0/HMSVMR0.cpp
 create mode 100644 src/VBox/VMM/VMMR0/HMSVMR0.h
 create mode 100644 src/VBox/VMM/VMMR0/HMVMXR0.cpp
 create mode 100644 src/VBox/VMM/VMMR0/HMVMXR0.h
 create mode 100644 src/VBox/VMM/VMMR0/IOMR0.cpp
 create mode 100644 src/VBox/VMM/VMMR0/IOMR0IoPort.cpp
 create mode 100644 src/VBox/VMM/VMMR0/IOMR0Mmio.cpp
 create mode 100644 src/VBox/VMM/VMMR0/Makefile.kup
 create mode 100644 src/VBox/VMM/VMMR0/NEMR0Native-win.cpp
 create mode 100644 src/VBox/VMM/VMMR0/PDMR0DevHlp.cpp
 create mode 100644 src/VBox/VMM/VMMR0/PDMR0Device.cpp
 create mode 100644 src/VBox/VMM/VMMR0/PDMR0Driver.cpp
 create mode 100644 src/VBox/VMM/VMMR0/PGMR0.cpp
 create mode 100644 src/VBox/VMM/VMMR0/PGMR0Bth.h
 create mode 100644 src/VBox/VMM/VMMR0/PGMR0Pool.cpp
 create mode 100644 src/VBox/VMM/VMMR0/PGMR0SharedPage.cpp
 create mode 100644 src/VBox/VMM/VMMR0/VMMR0.cpp
 create mode 100644 src/VBox/VMM/VMMR0/VMMR0.def
 create mode 100644 src/VBox/VMM/VMMR0/VMMR0JmpA-amd64.asm
 create mode 100644 src/VBox/VMM/VMMR0/VMMR0JmpA-x86.asm
 create mode 100644 src/VBox/VMM/VMMR0/VMMR0TripleFaultHack.cpp
 create mode 100644 src/VBox/VMM/VMMR0/VMMR0TripleFaultHackA.asm
 create mode 100644 src/VBox/VMM/VMMR3/APIC.cpp
 create mode 100644 src/VBox/VMM/VMMR3/CFGM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/CPUM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/CPUMDbg.cpp
 create mode 100644 src/VBox/VMM/VMMR3/CPUMR3CpuId.cpp
 create mode 100644 src/VBox/VMM/VMMR3/CPUMR3Db.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGF.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFAddr.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFAddrSpace.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFBp.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFCoreWrite.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFCpu.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFDisas.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFInfo.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFLog.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFMem.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFModule.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFOS.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFR3BugCheck.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFR3Flow.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFR3ModInMem.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFR3PlugIn.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFR3Trace.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFR3Type.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFReg.cpp
 create mode 100644 src/VBox/VMM/VMMR3/DBGFStack.cpp
 create mode 100644 src/VBox/VMM/VMMR3/EM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/EMHM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/EMR3Dbg.cpp
 create mode 100644 src/VBox/VMM/VMMR3/EMR3Nem.cpp
 create mode 100644 src/VBox/VMM/VMMR3/GIM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/GIMHv.cpp
 create mode 100644 src/VBox/VMM/VMMR3/GIMKvm.cpp
 create mode 100644 src/VBox/VMM/VMMR3/GIMMinimal.cpp
 create mode 100644 src/VBox/VMM/VMMR3/GMM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/HM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/IEMR3.cpp
 create mode 100644 src/VBox/VMM/VMMR3/IOM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/IOMR3IoPort.cpp
 create mode 100644 src/VBox/VMM/VMMR3/IOMR3Mmio.cpp
 create mode 100644 src/VBox/VMM/VMMR3/MM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/MMHeap.cpp
 create mode 100644 src/VBox/VMM/VMMR3/MMHyper.cpp
 create mode 100644 src/VBox/VMM/VMMR3/MMPagePool.cpp
 create mode 100644 src/VBox/VMM/VMMR3/MMUkHeap.cpp
 create mode 100644 src/VBox/VMM/VMMR3/Makefile.kup
 create mode 100644 src/VBox/VMM/VMMR3/NEMR3.cpp
 create mode 100644 src/VBox/VMM/VMMR3/NEMR3Native-win.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMAsyncCompletion.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMAsyncCompletionFile.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMAsyncCompletionFileFailsafe.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMAsyncCompletionFileNormal.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMBlkCache.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMCritSect.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMDevHlp.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMDevMiscHlp.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMDevice.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMDriver.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMLdr.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMNetShaper.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMQueue.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMR3Task.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMThread.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PDMUsb.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PGM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PGMDbg.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PGMHandler.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PGMMap.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PGMPhys.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PGMPhysRWTmpl.h
 create mode 100644 src/VBox/VMM/VMMR3/PGMPool.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PGMR3DbgA.asm
 create mode 100644 src/VBox/VMM/VMMR3/PGMSavedState.cpp
 create mode 100644 src/VBox/VMM/VMMR3/PGMSharedPage.cpp
 create mode 100644 src/VBox/VMM/VMMR3/SELM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/SSM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/STAM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/TM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/TRPM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/VM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/VMEmt.cpp
 create mode 100644 src/VBox/VMM/VMMR3/VMM.cpp
 create mode 100644 src/VBox/VMM/VMMR3/VMMGuruMeditation.cpp
 create mode 100644 src/VBox/VMM/VMMR3/VMMR3.def
 create mode 100644 src/VBox/VMM/VMMR3/VMMTests.cpp
 create mode 100644 src/VBox/VMM/VMMR3/VMReq.cpp
 create mode 100644 src/VBox/VMM/VMMR3/cpus/AMD_Athlon_64_3200.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/AMD_Athlon_64_X2_Dual_Core_4200.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/AMD_FX_8150_Eight_Core.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/AMD_Phenom_II_X6_1100T.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Hygon_C86_7185_32_core.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_80186.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_80286.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_80386.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_80486.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_8086.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Atom_330_1_60GHz.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Core2_T7600_2_33GHz.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Core2_X6800_2_93GHz.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Core_Duo_T2600_2_16GHz.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Core_i5_3570.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_2635QM.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_3820QM.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_3960X.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_5600U.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_6700K.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Pentium_4_3_00GHz.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Pentium_M_processor_2_00GHz.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Pentium_N3530_2_16GHz.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Intel_Xeon_X5482_3_20GHz.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Makefile.kup
 create mode 100644 src/VBox/VMM/VMMR3/cpus/Quad_Core_AMD_Opteron_2384.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/VIA_QuadCore_L4700_1_2_GHz.h
 create mode 100644 src/VBox/VMM/VMMR3/cpus/ZHAOXIN_KaiXian_KX_U5581_1_8GHz.h
 create mode 100644 src/VBox/VMM/VMMRZ/CPUMRZ.cpp
 create mode 100644 src/VBox/VMM/VMMRZ/CPUMRZA.asm
 create mode 100644 src/VBox/VMM/VMMRZ/DBGFRZ.cpp
 create mode 100644 src/VBox/VMM/VMMRZ/Makefile.kup
 create mode 100644 src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp
 create mode 100644 src/VBox/VMM/VMMRZ/VMMRZ.cpp
 create mode 100644 src/VBox/VMM/dtrace/int-1.d
 create mode 100644 src/VBox/VMM/dtrace/lib/amd64/vbox-arch-types.d
 create mode 100644 src/VBox/VMM/dtrace/lib/vbox-types.d
 create mode 100644 src/VBox/VMM/dtrace/lib/x86/vbox-arch-types.d
 create mode 100644 src/VBox/VMM/dtrace/return-to-ring-3-aggregation-1.d
 create mode 100644 src/VBox/VMM/dtrace/vmexit-reason-aggregation-1.d
 create mode 100644 src/VBox/VMM/dtrace/vmexit-rip-aggregation-1.d
 create mode 100644 src/VBox/VMM/include/APICInternal.h
 create mode 100644 src/VBox/VMM/include/CFGMInternal.h
 create mode 100644 src/VBox/VMM/include/CPUMInternal.h
 create mode 100644 src/VBox/VMM/include/CPUMInternal.mac
 create mode 100644 src/VBox/VMM/include/DBGFInternal.h
 create mode 100644 src/VBox/VMM/include/EMHandleRCTmpl.h
 create mode 100644 src/VBox/VMM/include/EMInternal.h
 create mode 100644 src/VBox/VMM/include/GIMHvInternal.h
 create mode 100644 src/VBox/VMM/include/GIMInternal.h
 create mode 100644 src/VBox/VMM/include/GIMKvmInternal.h
 create mode 100644 src/VBox/VMM/include/GIMMinimalInternal.h
 create mode 100644 src/VBox/VMM/include/HMInternal.h
 create mode 100644 src/VBox/VMM/include/HMInternal.mac
 create mode 100644 src/VBox/VMM/include/IEMInternal.h
 create mode 100644 src/VBox/VMM/include/IOMInline.h
 create mode 100644 src/VBox/VMM/include/IOMInternal.h
 create mode 100644 src/VBox/VMM/include/MMInternal.h
 create mode 100644 src/VBox/VMM/include/NEMInternal.h
 create mode 100644 src/VBox/VMM/include/PDMAsyncCompletionFileInternal.h
 create mode 100644 src/VBox/VMM/include/PDMAsyncCompletionInternal.h
 create mode 100644 src/VBox/VMM/include/PDMBlkCacheInternal.h
 create mode 100644 src/VBox/VMM/include/PDMInline.h
 create mode 100644 src/VBox/VMM/include/PDMInternal.h
 create mode 100644 src/VBox/VMM/include/PDMNetShaperInternal.h
 create mode 100644 src/VBox/VMM/include/PGMGstDefs.h
 create mode 100644 src/VBox/VMM/include/PGMInline.h
 create mode 100644 src/VBox/VMM/include/PGMInternal.h
 create mode 100644 src/VBox/VMM/include/SELMInternal.h
 create mode 100644 src/VBox/VMM/include/SSMInternal.h
 create mode 100644 src/VBox/VMM/include/STAMInternal.h
 create mode 100644 src/VBox/VMM/include/TMInline.h
 create mode 100644 src/VBox/VMM/include/TMInternal.h
 create mode 100644 src/VBox/VMM/include/TRPMInternal.h
 create mode 100644 src/VBox/VMM/include/VMInternal.h
 create mode 100644 src/VBox/VMM/include/VMMInternal.h
 create mode 100644 src/VBox/VMM/include/VMMInternal.mac
 create mode 100644 src/VBox/VMM/include/VMMTracing.h
 create mode 100755 src/VBox/VMM/pure_test.sh
 create mode 100755 src/VBox/VMM/testcase/Instructions/InstructionTestGen.py
 create mode 100644 src/VBox/VMM/testcase/Instructions/Makefile.kmk
 create mode 100644 src/VBox/VMM/testcase/Instructions/env-bs2-r0-32-big.mac
 create mode 100644 src/VBox/VMM/testcase/Instructions/env-bs2-r0-64-big.mac
 create mode 100644 src/VBox/VMM/testcase/Instructions/env-bs2-r0-64.mac
 create mode 100644 src/VBox/VMM/testcase/Instructions/env-bs2-r0-big.mac
 create mode 100644 src/VBox/VMM/testcase/Instructions/env-bs2-r0-common.mac
 create mode 100644 src/VBox/VMM/testcase/Instructions/env-bs2-r0.mac
 create mode 100644 src/VBox/VMM/testcase/Instructions/env-common.mac
 create mode 100644 src/VBox/VMM/testcase/Instructions/env-iprt-r3-32.mac
 create mode 100644 src/VBox/VMM/testcase/Instructions/env-iprt-r3-64.mac
 create mode 100644 src/VBox/VMM/testcase/Instructions/env-iprt-r3.mac
 create mode 100644 src/VBox/VMM/testcase/Instructions/itgTableDaa.py
 create mode 100644 src/VBox/VMM/testcase/Instructions/itgTableDas.py
 create mode 100644 src/VBox/VMM/testcase/Instructions/tstVBInsTstR3.cpp
 create mode 100644 src/VBox/VMM/testcase/Makefile.kmk
 create mode 100644 src/VBox/VMM/testcase/NemRawBench-1.cpp
 create mode 100644 src/VBox/VMM/testcase/dev.tar.gz
 create mode 100755 src/VBox/VMM/testcase/mkdsk.sh
 create mode 100644 src/VBox/VMM/testcase/tstAnimate.cpp
 create mode 100644 src/VBox/VMM/testcase/tstAsmStructs.cpp
 create mode 100644 src/VBox/VMM/testcase/tstAsmStructsAsm-lst.sed
 create mode 100644 src/VBox/VMM/testcase/tstAsmStructsAsm.asm
 create mode 100644 src/VBox/VMM/testcase/tstCFGM.cpp
 create mode 100644 src/VBox/VMM/testcase/tstCompressionBenchmark.cpp
 create mode 100644 src/VBox/VMM/testcase/tstGlobalConfig.cpp
 create mode 100644 src/VBox/VMM/testcase/tstHelp.h
 create mode 100644 src/VBox/VMM/testcase/tstIEMCheckMc.cpp
 create mode 100644 src/VBox/VMM/testcase/tstMMHyperHeap.cpp
 create mode 100644 src/VBox/VMM/testcase/tstMicro.h
 create mode 100644 src/VBox/VMM/testcase/tstMicro.mac
 create mode 100644 src/VBox/VMM/testcase/tstMicroRC.cpp
 create mode 100644 src/VBox/VMM/testcase/tstMicroRC.def
 create mode 100644 src/VBox/VMM/testcase/tstMicroRCA.asm
 create mode 100644 src/VBox/VMM/testcase/tstPDMAsyncCompletion.cpp
 create mode 100644 src/VBox/VMM/testcase/tstPDMAsyncCompletionStress.cpp
 create mode 100644 src/VBox/VMM/testcase/tstSSM-2.cpp
 create mode 100644 src/VBox/VMM/testcase/tstSSM.cpp
 create mode 100644 src/VBox/VMM/testcase/tstVMM-HM.cpp
 create mode 100644 src/VBox/VMM/testcase/tstVMMFork.cpp
 create mode 100644 src/VBox/VMM/testcase/tstVMMR0CallHost-1.cpp
 create mode 100644 src/VBox/VMM/testcase/tstVMREQ.cpp
 create mode 100644 src/VBox/VMM/testcase/tstVMStruct.h
 create mode 100644 src/VBox/VMM/testcase/tstVMStructDTrace.cpp
 create mode 100644 src/VBox/VMM/testcase/tstVMStructRC.cpp
 create mode 100644 src/VBox/VMM/testcase/tstVMStructSize.cpp
 create mode 100644 src/VBox/VMM/testcase/tstX86-1.cpp
 create mode 100644 src/VBox/VMM/testcase/tstX86-1A.asm
 create mode 100644 src/VBox/VMM/testcase/tstX86-FpuSaveRestore.cpp
 create mode 100644 src/VBox/VMM/testcase/tstX86-FpuSaveRestoreA.asm
 create mode 100644 src/VBox/VMM/tools/Makefile.kmk
 create mode 100644 src/VBox/VMM/tools/VBoxCpuReport.cpp
 create mode 100644 src/VBox/VMM/tools/VBoxCpuReport.h
 create mode 100644 src/VBox/VMM/tools/VBoxCpuReportMsrLinux.cpp
 create mode 100644 src/VBox/VMM/tools/VBoxCpuReportMsrSup.cpp
 create mode 100644 src/VBox/VMM/tools/VBoxVMMPreload.cpp
 create mode 100644 src/VBox/VMM/tools/VBoxVMMPreloadHardened.cpp

(limited to 'src/VBox/VMM')

diff --git a/src/VBox/VMM/.scm-settings b/src/VBox/VMM/.scm-settings
new file mode 100644
index 00000000..f27cfac8
--- /dev/null
+++ b/src/VBox/VMM/.scm-settings
@@ -0,0 +1,36 @@
+# $Id: .scm-settings $
+## @file
+# Source code massager settings for the VMM.
+#
+
+#
+# Copyright (C) 2017-2020 Oracle Corporation
+#
+# This file is part of VirtualBox Open Source Edition (OSE), as
+# available from http://www.virtualbox.org. This file is free software;
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License (GPL) as published by the Free Software
+# Foundation, in version 2 as it comes in the "COPYING" file of the
+# VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+# hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+#
+
+
+/VMMAll/IEMAllInstructionsPython.py: --license-ose-dual
+
+--filter-out-files /testcase/dev.tar.gz
+
+/include/PGMGstDefs.h: --no-fix-header-guards
+/VMMR0/PGMR0Bth.h: --no-fix-header-guards
+/VMMR3/PGMPhysRWTmpl.h: --no-fix-header-guards
+/VMMRC/PGMRCBth.h: --no-fix-header-guards
+/VMMRC/PGMRCShw.h: --no-fix-header-guards
+/VMMRC/PGMRCGst.h: --no-fix-header-guards
+/VMMAll/PGMAllBth.h: --no-fix-header-guards
+/VMMAll/PGMAllGst.h: --no-fix-header-guards
+/VMMAll/PGMAllShw.h: --no-fix-header-guards
+/testcase/tstVMStruct.h: --no-fix-header-guards
+
+/*.h: --guard-relative-to-dir . --guard-prefix VMM_INCLUDED_SRC_
+/VMMR3/cpus/*.h: --guard-relative-to-dir VMMR3/cpus --guard-prefix VBOX_CPUDB_
+
diff --git a/src/VBox/VMM/Config.kmk b/src/VBox/VMM/Config.kmk
new file mode 100644
index 00000000..d7ab7021
--- /dev/null
+++ b/src/VBox/VMM/Config.kmk
@@ -0,0 +1,91 @@
+# $Id: Config.kmk $
+## @file
+# kBuild Configuration file for the VMM.
+#
+
+#
+# Copyright (C) 2006-2020 Oracle Corporation
+#
+# This file is part of VirtualBox Open Source Edition (OSE), as
+# available from http://www.virtualbox.org. This file is free software;
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License (GPL) as published by the Free Software
+# Foundation, in version 2 as it comes in the "COPYING" file of the
+# VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+# hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+#
+
+VBOX_VMM_CONFIG_KMK_INCLUDED = 1
+
+# Include the top-level configure file.
+ifndef VBOX_ROOT_CONFIG_KMK_INCLUDED
+ include $(PATH_ROOT)/Config.kmk
+endif
+
+#
+# Globals
+#
+
+## DEFS variable that is picked up by all three VMM targets (R0, R3, RC).
+# Can be prepended to by setting it in LocalConfig.kmk
+VMM_COMMON_DEFS := USING_VMM_COMMON_DEFS
+ifdef VBOX_WITH_3RD_IEM_STEP
+ VMM_COMMON_DEFS += VBOX_WITH_3RD_IEM_STEP
+endif
+ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+ VMM_COMMON_DEFS += VBOX_WITH_NESTED_HWVIRT_SVM
+endif
+ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+ VMM_COMMON_DEFS += VBOX_WITH_NESTED_HWVIRT_VMX
+endif
+ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
+ VMM_COMMON_DEFS += VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
+endif
+#ifdef VBOX_WITH_IEM
+# VMM_COMMON_DEFS += VBOX_WITH_IEM
+#endif
+# part of global DEFS
+ifdef VBOX_WITH_MULTI_CORE
+ VMM_COMMON_DEFS += VBOX_WITH_MULTI_CORE
+endif
+ifdef VBOX_WITH_R0_LOGGING
+ VMM_COMMON_DEFS += VBOX_WITH_R0_LOGGING
+endif
+ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+ VMM_COMMON_DEFS += VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+endif
+ifdef VBOX_WITH_PCI_PASSTHROUGH
+ VMM_COMMON_DEFS += VBOX_WITH_PCI_PASSTHROUGH
+endif
+ifdef VBOX_WITH_DTRACE_RC
+ VMM_COMMON_DEFS += VBOX_WITH_DTRACE_RC
+endif
+ifdef VBOX_WITH_SAFE_STR
+ VMM_COMMON_DEFS += VBOX_WITH_SAFE_STR
+endif
+ifdef VBOX_WITH_64ON32_IDT
+ VMM_COMMON_DEFS += VBOX_WITH_64ON32_IDT
+endif
+ifdef VBOX_WITH_64ON32_CMOS_DEBUG
+ VMM_COMMON_DEFS += VBOX_WITH_64ON32_CMOS_DEBUG
+endif
+ifdef VBOXSTRICTRC_STRICT_ENABLED
+ VMM_COMMON_DEFS += VBOXSTRICTRC_STRICT_ENABLED
+endif
+ifeq ($(KBUILD_TARGET),amd64)
+ VMM_COMMON_DEFS += VBOX_WITH_MORE_RING0_MEM_MAPPINGS
+endif
+ifdef VBOX_WITH_NATIVE_NEM
+ if1of ($(KBUILD_TARGET).$(KBUILD_TARGET_ARCH), win.amd64)
+  VMM_COMMON_DEFS += VBOX_WITH_NATIVE_NEM VBOX_WITH_NEM_R0
+ endif
+endif
+ifdef VBOX_WITH_RAM_IN_KERNEL
+ VMM_COMMON_DEFS += VBOX_WITH_RAM_IN_KERNEL
+ if1of ($(KBUILD_TARGET), linux solaris) # Hosts that implements SUPR0HCPhysToVirt.
+  VMM_COMMON_DEFS += VBOX_WITH_LINEAR_HOST_PHYS_MEM
+ endif
+endif
+
+# VMM_COMMON_DEFS += VBOX_WITH_NS_ACCOUNTING_STATS
+
diff --git a/src/VBox/VMM/Docs-CodingGuidelines.cpp b/src/VBox/VMM/Docs-CodingGuidelines.cpp
new file mode 100644
index 00000000..7894893d
--- /dev/null
+++ b/src/VBox/VMM/Docs-CodingGuidelines.cpp
@@ -0,0 +1,93 @@
+/* $Id: Docs-CodingGuidelines.cpp $ */
+/** @file
+ * VMM - Coding Guidelines.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_vmm_guideline                  VMM Coding Guidelines
+ *
+ * The guidelines extends the VBox coding guidelines (@ref pg_vbox_guideline)
+ * and consists of a compulsory part and an optional part. It is very important
+ * that the rules of the compulsory part is followed. That will prevent obvious
+ * bugs, and it will ease porting the code to 32/64 and 64/32 bits setups.
+ *
+ *
+ *
+ * @section sec_vmm_guideline_compulsory        Compulsory
+ *
+ * It is of vital importance is to distinguish between addresses - both virtual
+ * and physical - applying to Guest Context and Host Context. To assist the
+ * coder in this, a set of types and macros have been created. Another vital
+ * thing is that structures shared between the two contexts ends up with the
+ * same size and member offsets in both places. There are types and macros
+ * for that too.
+ *
+ *
+ * The rules:
+ *
+ *      - When declaring pointers in shared structures use the RCPTRTYPE(),
+ *        R0PTRTYPE() and R3PTRTYPE() macros.
+ *
+ *      - Use RTGCPTR and RTHCPTR when dealing with the other context in
+ *        none shared structures, parameter lists, stack variables and such.
+ *
+ *      - Following the above rules, pointers will in a context other than the
+ *        one a pointer was defined for, appear as unsigned integers.
+ *
+ *      - It is NOT permitted to subject a pointer from the other context to pointer
+ *        types of the current context by direct cast or by definition.
+ *
+ *      - When doing pointer arithmetic cast using uintptr_t, intptr_t or char *.
+ *        Never cast a pointer to anything else for this purpose, that will not
+ *        work everywhere! (1)
+ *
+ *      - Physical addresses are also specific to their context. Use RTGCPHYS
+ *        and RTHCPHYS when dealing when them. Both types are unsigned integers.
+ *
+ *      - Integers in shared structures should be using a RT integer type or
+ *        any of the [u]int[0-9]+_t types. (2)
+ *
+ *      - If code is shared between the contexts, GCTYPE() can be used to declare
+ *        things differently. If GCTYPE() usage is extensive, don't share the code.
+ *
+ *      - The context is part of all public symbols which are specific to a single
+ *        context.
+ *
+ *
+ * (1) Talking about porting between 32-bit and 64-bit architectures and even
+ *     between 64-bit platforms. On 64-bit linux int is 32-bit, long is 64-bit.
+ *     However on 64-bit windows both int and long are 32-bit - there is no
+ *     standard 64 bit type (_int64 is not a standard type, it's an stupid
+ *     extension).
+ *
+ * (2) The VBox integer types are RTINT, RTUINT, RTGCINT, RTGCUINT,
+ *     RTGCINTPTR, RTGCUINTPTR, RTHCINT, RTHCUINT, RTHCINTPTR and
+ *     RTHCUINTPTR.
+ *
+ *
+ *
+ * @section sec_vmm_guideline_optional          Optional
+ *
+ * There are the general VBox guidelines, see @ref sec_vbox_guideline_optional.
+ * In addition to these for the following rules applies to the VMM:
+ *
+ *      - Prefixes GCPtr and HCPtr are preferred over suffixes HC and GC of
+ *        pointers.
+ *
+ *      - Prefixes GCPhys and HCPhys are generally used for physical addresses,
+ *        types RTGCPHYS and RTHCPHYS respectively.
+ *
+ */
+
diff --git a/src/VBox/VMM/Docs-RawMode.cpp b/src/VBox/VMM/Docs-RawMode.cpp
new file mode 100644
index 00000000..ed203c18
--- /dev/null
+++ b/src/VBox/VMM/Docs-RawMode.cpp
@@ -0,0 +1,148 @@
+/* $Id: Docs-RawMode.cpp $ */
+/** @file
+ * This file contains the documentation of the raw-mode execution.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+
+/** @page pg_raw                Raw-mode Code Execution
+ *
+ * VirtualBox 0.0 thru 6.0 implemented a mode of guest code execution that
+ * allowed executing mostly raw guest code directly the host CPU but without any
+ * support from VT-x or AMD-V.  It was implemented for AMD64, AMD-V and VT-x
+ * were available (former) or even specified (latter two).  This mode was
+ * removed in 6.1 (code ripped out) as it was mostly unused by that point and
+ * not worth the effort of maintaining.
+ *
+ * A future VirtualBox version may reintroduce a new kind of raw-mode for
+ * emulating non-x86 architectures, making use of the host MMU to efficiently
+ * emulate the target MMU.  This is just a wild idea at this point.
+ *
+ *
+ * @section sec_old_rawmode     Old Raw-mode
+ *
+ * Running guest code unmodified on the host CPU is reasonably unproblematic for
+ * ring-3 code when it runs without IOPL=3.  There will be some information
+ * leaks thru CPUID, a bunch of 286 area unprivileged instructions revealing
+ * privileged information (like SGDT, SIDT, SLDT, STR, SMSW), and hypervisor
+ * selectors can probably be identified using VERR, VERW and such instructions.
+ * However, it generally works fine for half friendly software when the CPUID
+ * difference between the target and host isn't too big.
+ *
+ * Kernel code can be executed on the host CPU too, however it needs to be
+ * pushed up a ring (guest ring-0 to ring-1, guest ring-1 to ring2) to let the
+ * hypervisor (VMMRC.rc) be in charge of ring-0.   Ring compression causes
+ * issues when CS or SS are pushed and inspected by the guest, since the values
+ * will have bit 0 set whereas the guest expects that bit to be cleared.  In
+ * addition there are problematic instructions like POPF and IRET that the guest
+ * code uses to restore/modify EFLAGS.IF state, however the CPU just silently
+ * ignores EFLAGS.IF when it isn't running in ring-0 (or with an appropriate
+ * IOPL), which causes major headache.  The SIDT, SGDT, STR, SLDT and SMSW
+ * instructions also causes problems since they will return information about
+ * the hypervisor rather than the guest state and cannot be trapped.
+ *
+ * So, guest kernel code needed to be scanned (by CSAM) and problematic
+ * instructions or sequences patched or recompiled (by PATM).
+ *
+ * The raw-mode execution operates in a slightly modified guest memory context,
+ * so memory accesses can be done directly without any checking or masking.  The
+ * modification was to insert the hypervisor in an unused portion of the the
+ * page tables, making it float around and require it to be relocated when the
+ * guest mapped code into the area it was occupying.
+ *
+ * The old raw-mode code was 32-bit only because its inception predates the
+ * availability of the AMD64 architecture and the promise of AMD-V and VT-x made
+ * it unnecessary to do a 64-bit version of the mode.  (A long-mode port of the
+ * raw-mode execution hypvisor could in theory have been used for both 32-bit
+ * and 64-bit guest, making the relocating unnecessary for 32-bit guests,
+ * however v8086 mode does not work when the CPU is operating in long-mode made
+ * it a little less attractive.)
+ *
+ *
+ * @section sec_rawmode_v2      Raw-mode v2
+ *
+ * The vision for the reinvention of raw-mode execution is to put it inside
+ * VT-x/AMD-V and run non-native instruction sets via a recompiler.
+ *
+ * The main motivation is TLB emulation using the host MMU.  An added benefit is
+ * would be that the non-native instruction sets would be add-ons put on top of
+ * the existing x86/AMD64 virtualization product and therefore not require a
+ * complete separate product build.
+ *
+ *
+ * Outline:
+ *
+ *  - Plug-in based, so the target architecture specific stuff is mostly in
+ *    separate modules (ring-3, ring-0 (optional) and raw-mode images).
+ *
+ *  - Only 64-bit mode code (no problem since VirtualBox requires a 64-bit host
+ *    since 6.0).  So, not reintroducing structure alignment pain from old RC.
+ *
+ *  - Map the RC-hypervisor modules as ROM, using the shadowing feature for the
+ *    data sections.
+ *
+ *  - Use MMIO2-like regions for all the memory that the RC-hypervisor needs,
+ *    all shared with the associated host side plug-in components.
+ *
+ *  - The ROM and MMIO2 regions does not directly end up in the saved state, the
+ *    state is instead saved by the ring-3 architecture module.
+ *
+ *  - Device access thru MMIO mappings could be done transparently thru to the
+ *    x86/AMD64 core VMM.  It would however be possible to reintroduce the RC
+ *    side device handling, as that will not be removed in the old-RC cleanup.
+ *
+ *  - Virtual memory managed by the RC-hypervisor, optionally with help of the
+ *    ring-3 and/or ring-0 architecture modules.
+ *
+ *  - The mapping of the RC modules and memory will probably have to runtime
+ *    relocatable again, like it was in the old RC.  Though initially and for
+ *    32-bit target architectures, we will probably use a fixed mapping.
+ *
+ *  - Memory accesses must unfortunately be range checked before being issued,
+ *    in order to prevent the guest code from accessing the hypervisor.  The
+ *    recompiled code must be able to run, modify state, call ROM code, update
+ *    statistics and such, so we cannot use page table stuff protect the
+ *    hypervisor code & data.  (If long mode implement segment limits, we
+ *    could've used that, but it doesn't.)
+ *
+ *  - The RC-hypervisor will make hypercalls to communicate with the ring-0 and
+ *    ring-3 host code.
+ *
+ *  - The host side should be able to dig out the current guest state from
+ *    information (think AMD64 unwinding) stored in translation blocks.
+ *
+ *  - Non-atomic state updates outside TBs could be flagged so the host know
+ *    how to roll the back.
+ *
+ *  - SMP must be taken into account early on.
+ *
+ *  - As must existing IEM-based recompiler ideas, preferrably sharing code
+ *    (basically compiling IEM targetting the other architecture).
+ *
+ * The actual implementation will depend a lot on which architectures are
+ * targeted and how they can be mapped onto AMD64/x86.  It is possible that
+ * there are some significan roadblocks preventing us from using the host MMU
+ * efficiently even.  AMD64 is for instance rather low on virtual address space
+ * compared to several other 64-bit architectures, which means we'll generate a
+ * lot of \#GPs when the guest tries to access spaced reserved on AMD64.  The
+ * proposed 5-level page tables will help with this, of course, but that need to
+ * get into silicon and into user computers for it to be really helpful.
+ *
+ * One thing that helps a lot is that we don't have to consider 32-bit x86 any
+ * more, meaning that the recompiler only need to generate 64-bit code and can
+ * assume having 15-16 GPRs at its disposal.
+ *
+ */
+
diff --git a/src/VBox/VMM/Makefile.kmk b/src/VBox/VMM/Makefile.kmk
new file mode 100644
index 00000000..e4a8abd0
--- /dev/null
+++ b/src/VBox/VMM/Makefile.kmk
@@ -0,0 +1,775 @@
+# $Id: Makefile.kmk $
+## @file
+# Top-level makefile for the VMM.
+#
+
+#
+# Copyright (C) 2006-2020 Oracle Corporation
+#
+# This file is part of VirtualBox Open Source Edition (OSE), as
+# available from http://www.virtualbox.org. This file is free software;
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License (GPL) as published by the Free Software
+# Foundation, in version 2 as it comes in the "COPYING" file of the
+# VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+# hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+#
+
+SUB_DEPTH = ../../..
+include $(KBUILD_PATH)/subheader.kmk
+
+# Include our Config.kmk if kmk is invoked from a parent directory.
+ifndef VBOX_VMM_CONFIG_KMK_INCLUDED
+ include $(PATH_SUB_CURRENT)/Config.kmk
+endif
+
+# Include sub-makefiles.
+ifndef VBOX_ONLY_EXTPACKS
+ include $(PATH_SUB_CURRENT)/tools/Makefile.kmk
+ include $(PATH_SUB_CURRENT)/testcase/Makefile.kmk
+endif
+
+
+# Fail on unsupported hosts.
+ifeq ($(KBUILD_TARGET_ARCH),x86)
+ ifeq ($(KBUILD_TARGET),darwin)
+$(error 32-bit darwin is no longer a supported VirtualBox host. Go back to 4.3 or older for 32-bit host support.)
+ else ifeq ($(KBUILD_TARGET),solaris)
+$(error 32-bit solaris is no longer a supported VirtualBox host. Go back to 4.2 or older for 32-bit host support.)
+ else ifn1of ($(KBUILD_TARGET_ARCH), $(VBOX_SUPPORTED_HOST_ARCHS))
+$(error 32-bit builds of the VirtualBox host are no longer supported. Go back to 6.0 or older for 32-bit host support.)
+ endif
+endif
+
+
+#
+# The VMM DLL.
+#
+ifndef VBOX_ONLY_EXTPACKS_USE_IMPLIBS
+ DLLS += VBoxVMM
+endif
+VBoxVMM_TEMPLATE = VBoxR3DllNoPic
+VBoxVMM_SONAME.linux = VBoxVMM.so
+
+VBoxVMM_DEFS     = VBOX_IN_VMM IN_VMM_R3 IN_DIS IN_GMM_R3 IN_DBG $(VMM_COMMON_DEFS)
+## @todo eliminate IN_GMM_R3
+ifdef VBOX_WITH_PREALLOC_RAM_BY_DEFAULT
+VBoxVMM_DEFS    += VBOX_WITH_PREALLOC_RAM_BY_DEFAULT
+endif
+ifdef VBOX_WITH_VUSB
+VBoxVMM_DEFS    += VBOX_WITH_USB
+endif
+ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+VBoxVMM_DEFS    += VBOX_WITH_PDM_ASYNC_COMPLETION
+endif
+ifdef VBOX_WITH_NETSHAPER
+VBoxVMM_DEFS    += VBOX_WITH_NETSHAPER
+endif
+ifdef VBOX_WITH_RAW_MODE
+VBoxVMM_DEFS    += VBOX_WITH_RAW_MODE VBOX_WITH_RAW_MODE_NOT_R0
+endif
+ifdef VBOX_WITH_VMM_R0_SWITCH_STACK
+VBoxVMM_DEFS    += VMM_R0_SWITCH_STACK
+endif
+if "$(KBUILD_TYPE)" == "debug" && "$(USERNAME)" == "bird"
+VBoxVMM_DEFS    += RTMEM_WRAP_TO_EF_APIS
+endif
+VBoxVMM_DEFS.darwin = VMM_R0_SWITCH_STACK
+
+VBoxVMM_INCS     = \
+	include \
+	$(if-expr defined(VBOX_WITH_RAW_MODE),PATM,) \
+	$(VBoxVMM_0_OUTDIR)/CommonGenIncs
+VBoxVMM_ASINCS   = .
+
+VBoxVMM_SOURCES  = \
+	VBoxVMM.d \
+	VMMR3/APIC.cpp \
+	VMMR3/CFGM.cpp \
+	VMMR3/CPUM.cpp \
+	VMMR3/CPUMR3CpuId.cpp \
+	VMMR3/CPUMR3Db.cpp \
+	VMMR3/CPUMDbg.cpp \
+	VMMR3/DBGF.cpp \
+	VMMR3/DBGFAddr.cpp \
+	VMMR3/DBGFAddrSpace.cpp \
+	VMMR3/DBGFBp.cpp \
+	VMMR3/DBGFR3BugCheck.cpp \
+	VMMR3/DBGFCoreWrite.cpp \
+	VMMR3/DBGFCpu.cpp \
+	VMMR3/DBGFDisas.cpp \
+	VMMR3/DBGFInfo.cpp \
+	VMMR3/DBGFLog.cpp \
+	VMMR3/DBGFMem.cpp \
+	VMMR3/DBGFR3ModInMem.cpp \
+	VMMR3/DBGFOS.cpp \
+	VMMR3/DBGFR3PlugIn.cpp \
+	VMMR3/DBGFReg.cpp \
+	VMMR3/DBGFStack.cpp \
+	VMMR3/DBGFR3Flow.cpp \
+	VMMR3/DBGFR3Trace.cpp \
+	VMMR3/DBGFR3Type.cpp \
+	VMMR3/EM.cpp \
+	VMMR3/EMR3Dbg.cpp \
+	VMMR3/EMHM.cpp \
+	VMMR3/EMR3Nem.cpp \
+	VMMR3/GIM.cpp \
+	VMMR3/GIMHv.cpp \
+	VMMR3/GIMKvm.cpp \
+	VMMR3/GIMMinimal.cpp \
+	VMMR3/IEMR3.cpp \
+	VMMR3/IOM.cpp \
+	VMMR3/IOMR3IoPort.cpp \
+	VMMR3/IOMR3Mmio.cpp \
+	VMMR3/GMM.cpp \
+	VMMR3/MM.cpp \
+	VMMR3/MMHeap.cpp \
+	VMMR3/MMHyper.cpp \
+	VMMR3/MMPagePool.cpp \
+	VMMR3/MMUkHeap.cpp \
+	VMMR3/NEMR3.cpp \
+	VMMR3/PDM.cpp \
+	VMMR3/PDMBlkCache.cpp \
+	VMMR3/PDMDevice.cpp \
+	VMMR3/PDMDevHlp.cpp \
+	VMMR3/PDMDevMiscHlp.cpp \
+	VMMR3/PDMDriver.cpp \
+	VMMR3/PDMLdr.cpp \
+	VMMR3/PDMCritSect.cpp \
+	VMMR3/PDMQueue.cpp \
+	VMMR3/PDMR3Task.cpp \
+	VMMR3/PDMThread.cpp \
+	VMMR3/PGM.cpp \
+	VMMR3/PGMDbg.cpp \
+	VMMR3/PGMR3DbgA.asm \
+	VMMR3/PGMHandler.cpp \
+	VMMR3/PGMMap.cpp \
+	VMMR3/PGMPhys.cpp \
+	VMMR3/PGMPool.cpp \
+	VMMR3/PGMSavedState.cpp \
+	VMMR3/PGMSharedPage.cpp \
+	VMMR3/SELM.cpp \
+	VMMR3/SSM.cpp \
+	VMMR3/STAM.cpp \
+	VMMR3/TM.cpp \
+	VMMR3/TRPM.cpp \
+	VMMR3/VM.cpp \
+	VMMR3/VMEmt.cpp \
+	VMMR3/VMReq.cpp \
+	VMMR3/VMM.cpp \
+	VMMR3/VMMGuruMeditation.cpp \
+	VMMR3/VMMTests.cpp \
+	VMMR3/HM.cpp \
+	$(if-expr defined(VBOX_WITH_RAW_MODE), \
+	VMMR3/CSAM.cpp \
+	VMMR3/PATM.cpp \
+	VMMR3/PATMPatch.cpp \
+	VMMR3/PATMGuest.cpp \
+	VMMR3/PATMA.asm \
+	VMMR3/PATMSSM.cpp \
+	VMMR3/PATMR3Dbg.cpp \
+       ,) \
+	VMMAll/APICAll.cpp \
+	VMMAll/CPUMAllRegs.cpp \
+	VMMAll/CPUMAllMsrs.cpp \
+	VMMAll/DBGFAll.cpp \
+	VMMAll/HMAll.cpp \
+	VMMAll/HMSVMAll.cpp \
+	VMMAll/HMVMXAll.cpp \
+	VMMAll/IEMAll.cpp \
+	VMMAll/IEMAllAImpl.asm \
+	VMMAll/IEMAllAImplC.cpp \
+	VMMAll/IOMAll.cpp \
+	VMMAll/IOMAllMmioNew.cpp \
+	VMMAll/MMAll.cpp \
+	VMMAll/MMAllHyper.cpp \
+	VMMAll/NEMAll.cpp \
+	VMMAll/PDMAll.cpp \
+	VMMAll/PDMAllCritSect.cpp \
+	VMMAll/PDMAllCritSectRw.cpp \
+	VMMAll/PDMAllCritSectBoth.cpp \
+	VMMAll/PDMAllQueue.cpp \
+	VMMAll/PDMAllTask.cpp \
+	VMMAll/PGMAll.cpp \
+	VMMAll/PGMAllHandler.cpp \
+	VMMAll/PGMAllMap.cpp \
+	VMMAll/PGMAllPhys.cpp \
+	VMMAll/PGMAllPool.cpp \
+	VMMAll/SELMAll.cpp \
+	VMMAll/EMAll.cpp \
+	VMMAll/GIMAll.cpp \
+	VMMAll/GIMAllHv.cpp \
+	VMMAll/GIMAllKvm.cpp \
+	VMMAll/TMAll.cpp \
+	VMMAll/TMAllCpu.cpp \
+	VMMAll/TMAllReal.cpp \
+	VMMAll/TMAllVirtual.cpp \
+	VMMAll/TRPMAll.cpp \
+	VMMAll/VMAll.cpp \
+	VMMAll/VMMAll.cpp \
+	VMMAll/VMMAllA.asm \
+	$(if-expr defined(VBOX_WITH_RAW_MODE), \
+	VMMAll/CSAMAll.cpp \
+	VMMAll/PATMAll.cpp \
+	,)
+ifdef VBOX_WITH_VUSB
+VBoxVMM_SOURCES += VMMR3/PDMUsb.cpp
+endif
+ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+VBoxVMM_SOURCES += \
+	VMMR3/PDMAsyncCompletion.cpp \
+	VMMR3/PDMAsyncCompletionFile.cpp \
+	VMMR3/PDMAsyncCompletionFileFailsafe.cpp \
+	VMMR3/PDMAsyncCompletionFileNormal.cpp
+endif
+ifdef VBOX_WITH_NETSHAPER
+VBoxVMM_SOURCES += \
+	VMMR3/PDMNetShaper.cpp \
+	VMMAll/PDMAllNetShaper.cpp
+endif
+
+ifdef VBOX_WITH_NATIVE_NEM
+VBoxVMM_SOURCES.win.amd64 += VMMR3/NEMR3Native-win.cpp
+VBoxVMM_DEFS.win.amd64    += VBOX_WITH_NATIVE_NEM
+VBoxVMM_SDKS.win          += VBOX_NTDLL
+VMMR3/NEMR3Native-win.cpp_DEFS.amd64 = _AMD64_
+VMMR3/NEMR3Native-win.cpp_INCS = \
+	$(KBUILD_DEVTOOLS)/win.x86/sdk/v10.0.17134.0/include/10.0.17134.0/um \
+	$(KBUILD_DEVTOOLS)/win.x86/sdk/v10.0.17134.0/include/10.0.17134.0/shared
+endif
+
+VBoxVMM_LIBS = \
+	$(PATH_STAGE_LIB)/DisasmR3$(VBOX_SUFF_LIB)
+ifdef VBOX_WITH_DEBUGGER
+VBoxVMM_LIBS += \
+	$(PATH_STAGE_LIB)/Debugger$(VBOX_SUFF_LIB)
+endif
+VBoxVMM_LIBS += \
+	$(LIB_REM) \
+	$(LIB_RUNTIME)
+
+VBoxVMM_LIBS.win = $(PATH_TOOL_$(VBOX_VCC_TOOL)_LIB)/delayimp.lib
+VBoxVMM_LDFLAGS.linux = $(VBOX_GCC_NO_UNDEFINED)
+VBoxVMM_LDFLAGS.darwin = -install_name $(VBOX_DYLD_EXECUTABLE_PATH)/VBoxVMM.dylib
+VBoxVMM_LDFLAGS.solaris = -mimpure-text
+
+# SSM wish to know the build type, host os and arch.
+VMMR3/SSM.cpp_DEFS +=	\
+	KBUILD_TYPE=\"$(KBUILD_TYPE)\" \
+	KBUILD_TARGET=\"$(KBUILD_TARGET)\" \
+	KBUILD_TARGET_ARCH=\"$(KBUILD_TARGET_ARCH)\"
+
+ifdef VBOX_WITH_GCC_SANITIZER
+VMMR3/PGMPool.cpp_CXXFLAGS.linux += -fno-sanitize=address
+endif
+
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+# ifeq ($(KBUILD_HOST), linux)
+#VBoxVMM_LIBS += aio
+# endif
+#endif
+
+if "$(USERNAME)" == "bird" && "$(KBUILD_TARGET)" == "win"
+VBoxVMM_VMMAll/IEMAll.cpp_CXXFLAGS         = /FAcs /Fa$(subst /,\\,$(outbase).cod)
+VBoxVMM_VMMAll/IEMAllAImplC.cpp_CXXFLAGS   = /FAcs /Fa$(subst /,\\,$(outbase).cod)
+VBoxVMM_VMMAll/PGMAll.cpp_CXXFLAGS         = /FAcs /Fa$(subst /,\\,$(outbase).cod)
+VBoxVMM_VMMAll/PDMAllCritSect.cpp_CXXFLAGS = /FAcs /Fa$(subst /,\\,$(outbase).cod)
+VBoxVMM_CLEAN += $(addprefix $(VBoxVMM_0_OUTDIR)/VMMAll/, IEMAll.cod IEMAllAImplC.cod PGMAll.cod PDMAllCritSect.cod)
+endif
+
+$(call VBOX_SET_VER_INFO_DLL,VBoxVMM,VirtualBox VMM) # Version info / description.
+
+
+#
+# Generate macro template for IEM instruction statistics.
+#
+$(call KB_FN_DO_PASS0_ON_TARGET,VBoxVMM) # Set VBoxVMM_0_OUTDIR
+VBoxVMM_INTERMEDIATES += $(VBoxVMM_0_OUTDIR)/CommonGenIncs/IEMInstructionStatisticsTmpl.h
+VBoxVMM_CLEAN         += \
+	$(VBoxVMM_0_OUTDIR)/CommonGenIncs/IEMInstructionStatisticsTmpl.h.ts \
+	$(VBoxVMM_0_OUTDIR)/CommonGenIncs/IEMInstructionStatisticsTmpl.h
+$(call KB_FN_AUTO_CMD_DEPS,$(VBoxVMM_0_OUTDIR)/CommonGenIncs/IEMInstructionStatisticsTmpl.h.ts)
+$(VBoxVMM_0_OUTDIR)/CommonGenIncs/IEMInstructionStatisticsTmpl.h.ts \
++| $(VBoxVMM_0_OUTDIR)/CommonGenIncs/IEMInstructionStatisticsTmpl.h: \
+		$(PATH_SUB_CURRENT)/VMMAll/IEMAllInstructions.cpp.h \
+		$(PATH_SUB_CURRENT)/VMMAll/IEMAllInstructionsOneByte.cpp.h \
+		$(PATH_SUB_CURRENT)/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h \
+		$(PATH_SUB_CURRENT)/VMMAll/IEMAllInstructionsThree0f38.cpp.h \
+		$(PATH_SUB_CURRENT)/VMMAll/IEMAllInstructionsThree0f3a.cpp.h \
+		$(PATH_SUB_CURRENT)/VMMAll/IEMAllInstructionsVexMap1.cpp.h \
+		$(PATH_SUB_CURRENT)/VMMAll/IEMAllInstructionsVexMap2.cpp.h \
+		$(PATH_SUB_CURRENT)/VMMAll/IEMAllInstructionsVexMap3.cpp.h \
+		$(PATH_SUB_CURRENT)/VMMAll/IEMAllInstructions3DNow.cpp.h
+	$(QUIET)$(call MSG_GENERATE,VBoxVMM,$@,VMMAll/IEMAllInstructions*.cpp.h)
+	$(QUIET)$(RM) -f -- "$@.tmp" "$@.tmp" "$@.sorted"
+	$(QUIET)$(MKDIR) -p -- "$(dir $@)"
+	$(call KB_FN_AUTO_CMD_DEPS_COMMANDS)
+	$(QUIET)$(SED) \
+		-e '/IEMOP_MNEMONIC\(\|[01234]\|[01234]EX\)(/!d' \
+		-e ':look-for-end-of-invocation' \
+		-e '/)/bend-of-invocation' \
+		-e 'N' \
+		-e 'blook-for-end-of-invocation' \
+		-e ':end-of-invocation' \
+		-e 's/\n/ /g' \
+		-e 's/  */ /g' \
+		-e 's/^.*IEMOP_MNEMONIC\(\|[01234]\|[01234]EX\)(/IEM_DO_INSTR_STAT\1(/' \
+		-e 's/;.*$(DOLLAR)//' \
+		--output "$@.tmp" $(filter %.cpp.h,$^)
+# Windows sort does some kind of seeking. So, we must use a temporary file and kmk_cat to define and undefine our macros.
+	$(QUIET)$(REDIRECT) -wto "$@.sorted" -- sort "$@.tmp"
+	$(QUIET)$(APPEND) -nt "$@" \
+		'/* Warning autogenerated by VMM/Makefile.kmk. */ ' \
+		'#define IEM_DO_INSTR_STAT0(f,u,l,fd,fi)                   IEM_DO_INSTR_STAT(l,  #l)' \
+		'#define IEM_DO_INSTR_STAT1(f,u,l,o1,fd,fi)                IEM_DO_INSTR_STAT(l ## _ ## o1,                                   #l " " #o1)' \
+		'#define IEM_DO_INSTR_STAT2(f,u,l,o1,o2,fd,fi)             IEM_DO_INSTR_STAT(l ## _ ## o1 ## _ ## o2,                        #l " " #o1 "," #o2)' \
+		'#define IEM_DO_INSTR_STAT3(f,u,l,o1,o2,o3,fd,fi)          IEM_DO_INSTR_STAT(l ## _ ## o1 ## _ ## o2 ## _ ## o3,             #l " " #o1 "," #o2 "," #o3)' \
+		'#define IEM_DO_INSTR_STAT4(f,u,l,o1,o2,o3,o4,fd,fi)       IEM_DO_INSTR_STAT(l ## _ ## o1 ## _ ## o2 ## _ ## o3 ## _ ## o4,  #l " " #o1 "," #o2 "," #o3 "," #o4)' \
+		'#define IEM_DO_INSTR_STAT0EX(s,m,f,u,l,fd,fi)             IEM_DO_INSTR_STAT(s,m)' \
+		'#define IEM_DO_INSTR_STAT1EX(s,m,f,u,l,o1,fd,fi)          IEM_DO_INSTR_STAT(s,m)' \
+		'#define IEM_DO_INSTR_STAT2EX(s,m,f,u,l,o1,o2,fd,fi)       IEM_DO_INSTR_STAT(s,m)' \
+		'#define IEM_DO_INSTR_STAT3EX(s,m,f,u,l,o1,o2,o3,fd,fi)    IEM_DO_INSTR_STAT(s,m)' \
+		'#define IEM_DO_INSTR_STAT4EX(s,m,f,u,l,o1,o2,o3,o4,fd,fi) IEM_DO_INSTR_STAT(s,m)' \
+               ''
+	$(QUIET)$(REDIRECT) -ato "$@" -- $(CAT_EXT) "$@.sorted"
+	$(QUIET)$(APPEND) -n "$@" \
+		'' \
+		'#undef IEM_DO_INSTR_STAT0' \
+		'#undef IEM_DO_INSTR_STAT1' \
+		'#undef IEM_DO_INSTR_STAT2' \
+		'#undef IEM_DO_INSTR_STAT3' \
+		'#undef IEM_DO_INSTR_STAT4' \
+		'#undef IEM_DO_INSTR_STAT0EX' \
+		'#undef IEM_DO_INSTR_STAT1EX' \
+		'#undef IEM_DO_INSTR_STAT2EX' \
+		'#undef IEM_DO_INSTR_STAT3EX' \
+		'#undef IEM_DO_INSTR_STAT4EX' \
+		''
+	$(QUIET)$(RM) -f -- "$@.tmp" "$@.sorted"
+	$(QUIET)$(CP) -v -f --changed -- "$@" "$(patsubst %.ts,%,$@)"
+
+foobar: $(VBoxVMM_0_OUTDIR)/CommonGenIncs/IEMInstructionStatisticsTmpl.h
+
+if "$(KBUILD_TARGET)" == "win" && !defined(VBOX_ONLY_EXTPACKS_USE_IMPLIBS)
+#
+# Debug type info hack for VMCPU, VM and similar.
+#
+# The microsoft linker seems to be using the last occurence of the structures
+# when writing the module PDB file.  So, we put the fully complete structures
+# in a library which is at the end of the library list.
+#
+VBoxVMM_LIBS    += $(VBoxVMMPdbTypeHack_1_TARGET)
+VBoxVMM_LDFLAGS += /Export:PdbTypeHack
+
+LIBRARIES += VBoxVMMPdbTypeHack
+VBoxVMMPdbTypeHack_TEMPLATE       = $(VBoxVMM_TEMPLATE)
+VBoxVMMPdbTypeHack_SOURCES        = VMMAll/AllPdbTypeHack.cpp
+VBoxVMMPdbTypeHack_DEFS           = $(VBoxVMM_DEFS)
+VBoxVMMPdbTypeHack_DEFS.win       = $(VBoxVMM_DEFS.win)
+VBoxVMMPdbTypeHack_DEFS.win.x86   = $(VBoxVMM_DEFS.win.x86)
+VBoxVMMPdbTypeHack_DEFS.win.amd64 = $(VBoxVMM_DEFS.win.amd64)
+VBoxVMMPdbTypeHack_INCS           = $(VBoxVMM_INCS)
+VBoxVMMPdbTypeHack_INTERMEDIATES  = $(VBoxVMM_INTERMEDIATES)
+endif
+
+
+if1of ($(VBOX_LDR_FMT), pe lx)
+#
+# VMMR0Imp.lib
+#
+LIBRARIES += VMMR0Imp
+VMMR0Imp_TEMPLATE = VBoxR0
+VMMR0Imp_SOURCES  = $(VMMR0Imp_0_OUTDIR)/VMMR0.def
+VMMR0Imp_CLEAN    = $(VMMR0Imp_0_OUTDIR)/VMMR0.def
+ ifeq ($(KBUILD_TARGET),win) # Experiment: Let's see how blunt the ones messing our NULL_THUNK_DATA entries on W10 are.
+VMMR0Imp_POST_CMDS = $(KLIBTWEAKER_EXT) --clear-timestamps --fill-null_thunk_data $(out)
+ endif
+$(call KB_FN_DO_PASS0_ON_TARGET,VMMR0Imp)
+
+$(call KB_FN_AUTO_CMD_DEPS,$(VMMR0Imp_0_OUTDIR)/VMMR0.def)
+$(VMMR0Imp_0_OUTDIR)/VMMR0.def: $(VMMR0Imp_DEFPATH)/VMMR0/VMMR0.def | $$(dir $$@)
+	$(call KB_FN_AUTO_CMD_DEPS_COMMANDS)
+ ifeq ($(VBOX_LDR_FMT),lx)
+	$(SED) \
+		-e '/not-os2/d' \
+		-e '/not-amd64/d' \
+		-e 's/^[ \t][ \t]*\([a-zA-Z]\)/    _\1/' \
+		-e 's/[ \t]DATA[ \t]*/ /'  \
+		--output $@ $(VMMR0Imp_DEFPATH)/VMMR0/VMMR0.def
+	$(APPEND) "$@" ""
+	$(APPEND) "$@" "    ___ehInit"
+ else
+	$(SED) \
+		-e '/not-win/d' \
+		-e '/not-$(KBUILD_TARGET_ARCH)/d' \
+		--output $@ $(VMMR0Imp_DEFPATH)/VMMR0/VMMR0.def
+ endif
+endif # R0: pe + lx
+
+
+#
+# VMMR3Imp.lib
+#
+IMPORT_LIBS += VMMR3Imp
+$(call VBOX_GENERATE_IMPORT_TARGET_FN,VMMR3Imp,VBoxVMM,VMMR3/VMMR3.def)
+
+ifneq ($(VBOX_LIB_VMM_LAZY),$(LIB_VMM))
+#
+# VMMR3LazyImp.lib (experimental)
+#
+LIBRARIES += VMMR3LazyImp
+VMMR3LazyImp_TEMPLATE = VBoxR3Dll
+VMMR3LazyImp_INST     = $(INST_LIB)
+VMMR3LazyImp_SOURCES  = $(VMMR3LazyImp_0_OUTDIR)/VMMR3LazyLoad.asm
+VMMR3LazyImp_CLEAN    = $(VMMR3LazyImp_0_OUTDIR)/VMMR3LazyLoad.asm
+$(call KB_FN_DO_PASS0_ON_TARGET,VMMR3LazyImp)
+
+$(call KB_FN_AUTO_CMD_DEPS,$(VMMR3LazyImp_0_OUTDIR)/VMMR3LazyLoad.asm)
+$(VMMR3LazyImp_0_OUTDIR)/VMMR3LazyLoad.asm: $(VMMR3LazyImp_DEFPATH)/VMMR3/VMMR3.def $(VBOX_DEF_2_LAZY_LOAD) | $$(dir $$@)
+	$(call KB_FN_AUTO_CMD_DEPS_COMMANDS)
+	$(VBOX_DEF_2_LAZY_LOAD) --output $@ --library VBoxVMM $(VMMR3LazyImp_DEFPATH)/VMMR3/VMMR3.def
+endif
+
+
+ifndef VBOX_ONLY_EXTPACKS
+#
+# VMMR0.r0
+#
+$(if-expr defined(VBOX_WITH_VBOXR0_AS_DLL),DLLS,SYSMODS) += VMMR0
+VMMR0_TEMPLATE  = VBoxR0
+VMMR0_SYSSUFF   = .r0
+
+VMMR0_DEFS      = VBOX_IN_VMM IN_VMM_R0 IN_RT_R0 IN_DIS DIS_CORE_ONLY IN_GVMM_R0 IN_GMM_R0 IN_INTNET_R0 \
+	$(VMM_COMMON_DEFS) RTASSERT_HAVE_SHOULD_PANIC
+## @todo eliminate IN_GVMM_R0 IN_GMM_R0
+ ifdef VBOX_WITH_PCI_PASSTHROUGH
+VMMR0_DEFS     += IN_PCIRAW_R0
+ endif
+ ifdef VBOX_WITH_TRIPLE_FAULT_HACK
+VMMR0_DEFS     += VBOX_WITH_TRIPLE_FAULT_HACK
+ endif
+ ifdef VBOX_WITH_RAW_MODE
+VMMR0_DEFS     += VBOX_WITH_RAW_MODE
+ endif
+ ifdef VBOX_WITH_VMM_R0_SWITCH_STACK
+VMMR0_DEFS     += VMM_R0_SWITCH_STACK
+ endif
+ if1of ($(KBUILD_TARGET), darwin linux win)
+VMMR0_DEFS     += VMM_R0_TOUCH_FPU
+ endif
+VMMR0_DEFS.darwin = VMM_R0_SWITCH_STACK
+VMMR0_DEFS.win.amd64  = VBOX_WITH_KERNEL_USING_XMM
+
+ ifeq ($(VBOX_LDR_FMT),elf)
+VMMR0_CXXFLAGS += -Wunused -Wunused-variable -Wno-unused-parameter
+ endif
+
+VMMR0_INCS      = \
+	include \
+	$(if-expr defined(VBOX_WITH_RAW_MODE),PATM,) \
+	$(VBoxVMM_0_OUTDIR)/CommonGenIncs
+
+VMMR0_SOURCES   = \
+	VBoxVMM.d \
+	VMMR0/CPUMR0.cpp \
+	VMMR0/CPUMR0A.asm \
+	VMMR0/GIMR0.cpp \
+	VMMR0/GIMR0Hv.cpp \
+	VMMR0/GMMR0.cpp \
+	VMMR0/GVMMR0.cpp \
+	VMMR0/EMR0.cpp \
+	VMMR0/HMR0.cpp \
+	VMMR0/HMR0A.asm \
+	VMMR0/HMVMXR0.cpp \
+	VMMR0/HMSVMR0.cpp \
+	VMMR0/IOMR0.cpp \
+	VMMR0/IOMR0IoPort.cpp \
+	VMMR0/IOMR0Mmio.cpp \
+	VMMR0/PDMR0Device.cpp \
+	VMMR0/PDMR0DevHlp.cpp \
+	VMMR0/PDMR0Driver.cpp \
+	VMMR0/PGMR0.cpp \
+	VMMR0/PGMR0Pool.cpp \
+	VMMR0/PGMR0SharedPage.cpp \
+	VMMR0/VMMR0.cpp \
+	VMMRZ/CPUMRZ.cpp \
+	VMMRZ/CPUMRZA.asm \
+	VMMRZ/DBGFRZ.cpp \
+	VMMRZ/VMMRZ.cpp \
+	VMMAll/APICAll.cpp \
+	VMMAll/CPUMAllRegs.cpp \
+	VMMAll/CPUMAllMsrs.cpp \
+	VMMAll/DBGFAll.cpp \
+	VMMAll/EMAll.cpp \
+	VMMAll/GIMAll.cpp \
+	VMMAll/GIMAllHv.cpp \
+	VMMAll/GIMAllKvm.cpp \
+	VMMAll/HMAll.cpp \
+	VMMAll/HMSVMAll.cpp \
+	VMMAll/HMVMXAll.cpp \
+	VMMAll/IEMAll.cpp \
+	VMMAll/IEMAllAImpl.asm \
+	VMMAll/IEMAllAImplC.cpp \
+	VMMAll/IOMAll.cpp \
+	VMMAll/IOMAllMmioNew.cpp \
+	VMMAll/MMAll.cpp \
+	VMMAll/MMAllHyper.cpp \
+	VMMAll/NEMAll.cpp \
+	VMMAll/PDMAll.cpp \
+	VMMAll/PDMAllCritSect.cpp \
+	VMMAll/PDMAllCritSectRw.cpp \
+	VMMAll/PDMAllCritSectBoth.cpp \
+	VMMAll/PDMAllQueue.cpp \
+	VMMAll/PDMAllTask.cpp \
+	VMMAll/PGMAll.cpp \
+	VMMAll/PGMAllHandler.cpp \
+	VMMAll/PGMAllMap.cpp \
+	VMMAll/PGMAllPhys.cpp \
+	VMMAll/PGMAllPool.cpp \
+	VMMAll/SELMAll.cpp \
+	VMMAll/TMAll.cpp \
+	VMMAll/TMAllCpu.cpp \
+	VMMAll/TMAllReal.cpp \
+	VMMAll/TMAllVirtual.cpp \
+	VMMAll/TRPMAll.cpp \
+	VMMAll/VMAll.cpp \
+	VMMAll/VMMAll.cpp \
+	VMMAll/VMMAllA.asm
+ if1of ($(VBOX_LDR_FMT), pe lx)
+VMMR0_SOURCES += $(VMMR0Imp_0_OUTDIR)/VMMR0.def
+ endif
+ ifdef VBOX_WITH_TRIPLE_FAULT_HACK
+VMMR0_SOURCES += \
+	VMMR0/VMMR0TripleFaultHack.cpp \
+	VMMR0/VMMR0TripleFaultHackA.asm
+ endif
+ ifdef VBOX_WITH_NETSHAPER
+VMMR0_SOURCES += \
+	VMMAll/PDMAllNetShaper.cpp
+ endif
+VMMR0_SOURCES.amd64 = \
+	VMMR0/VMMR0JmpA-amd64.asm
+VMMR0_SOURCES.x86 = \
+	VMMR0/VMMR0JmpA-x86.asm
+
+VMMR0_LIBS = \
+	$(PATH_STAGE_LIB)/ServicesR0$(VBOX_SUFF_LIB) \
+	$(PATH_STAGE_LIB)/RuntimeR0$(VBOX_SUFF_LIB) \
+	$(PATH_STAGE_LIB)/DisasmR0$(VBOX_SUFF_LIB)
+ ifneq ($(filter pe lx,$(VBOX_LDR_FMT)),)
+VMMR0_LIBS += \
+	$(PATH_STAGE_LIB)/SUPR0$(VBOX_SUFF_LIB)
+ endif
+ ifdef VBOX_WITH_NATIVE_NEM
+VMMR0_SOURCES.win.amd64 += VMMR0/NEMR0Native-win.cpp
+VMMR0_DEFS.win.amd64    += VBOX_WITH_NATIVE_NEM VBOX_WITH_NEM_R0
+VMMR0/NEMR0Native-win.cpp_SDKS.win = ReorderCompilerIncs $(VBOX_WINDDK) $(VBOX_WINPSDK)INCS
+ endif
+
+$(call VBOX_SET_VER_INFO_R0,VMMR0,VirtualBox VMM - ring-0 context parts) # Version info / description.
+
+ if "$(USERNAME)" == "bird" && "$(KBUILD_TARGET)" == "win"
+VMMR0_VMMAll/IEMAll.cpp_CXXFLAGS       = /FAcs /Fa$(subst /,\\,$(outbase).cod)
+VMMR0_VMMAll/IEMAllAImplC.cpp_CXXFLAGS = /FAcs /Fa$(subst /,\\,$(outbase).cod)
+VMMR0_VMMAll/PGMAll.cpp_CXXFLAGS       = /FAcs /Fa$(subst /,\\,$(outbase).cod)
+VMMR0_CLEAN += $(addprefix $(VMMR0_0_OUTDIR)/VMMAll/, IEMAll.cod IEMAllAImplC.cod PGMAll.cod)
+ endif
+
+VMMR0_INTERMEDIATES += $(VBoxVMM_0_OUTDIR)/CommonGenIncs/IEMInstructionStatisticsTmpl.h
+
+ if "$(KBUILD_TARGET)" == "win"
+# Debug type info hack for VMCPU, VM and similar.  See VBoxVMM for details.
+VMMR0_LIBS    += $(VMMR0PdbTypeHack_1_TARGET)
+VMMR0_LDFLAGS += /Export:PdbTypeHack
+
+LIBRARIES += VMMR0PdbTypeHack
+VMMR0PdbTypeHack_TEMPLATE       = $(VMMR0_TEMPLATE)
+VMMR0PdbTypeHack_SOURCES        = VMMAll/AllPdbTypeHack.cpp
+VMMR0PdbTypeHack_DEFS           = $(VMMR0_DEFS)
+VMMR0PdbTypeHack_DEFS.win       = $(VMMR0_DEFS.win)
+VMMR0PdbTypeHack_DEFS.win.x86   = $(VMMR0_DEFS.win.x86)
+VMMR0PdbTypeHack_DEFS.win.amd64 = $(VMMR0_DEFS.win.amd64)
+VMMR0PdbTypeHack_INCS           = $(VMMR0_INCS)
+VMMR0PdbTypeHack_INTERMEDIATES  = $(VMMR0_INTERMEDIATES)
+ endif
+
+endif # !VBOX_ONLY_EXTPACKS
+
+
+
+ifndef VBOX_ONLY_EXTPACKS
+#
+# SSMStandalone.lib/a for linking with VBoxSVC and other executables.
+#
+LIBRARIES += SSMStandalone
+SSMStandalone_TEMPLATE = VBOXR3EXE
+SSMStandalone_DEFS     = VBOX_IN_VMM IN_VMM_R3 IN_VMM_STATIC SSM_STANDALONE CPUM_DB_STANDALONE $(VMM_COMMON_DEFS)
+SSMStandalone_INCS     = include
+SSMStandalone_SOURCES  = \
+	VMMR3/SSM.cpp \
+	VMMR3/CPUMR3Db.cpp
+endif # !VBOX_ONLY_EXTPACKS
+
+
+if !defined(VBOX_ONLY_EXTPACKS) \
+ && (   defined(VBOX_WITH_DTRACE_R3) \
+     || defined(VBOX_WITH_DTRACE_R0) \
+     || defined(VBOX_WITH_DTRACE_RC))
+#
+# Install the dtrace library files.
+#
+INSTALLS += VMMLibDTrace
+VMMLibDTrace_INST    = $(VBOX_INST_DTRACE_LIB)$(KBUILD_TARGET_ARCH)/
+VMMLibDTrace_SOURCES = \
+	dtrace/lib/vbox-types.d \
+	dtrace/lib/$(KBUILD_TARGET_ARCH)/vbox-arch-types.d \
+	$(VMMLibDTrace_0_OUTDIR)/vm.d \
+	$(VMMLibDTrace_0_OUTDIR)/cpumctx.d \
+	$(VMMLibDTrace_0_OUTDIR)/cpum.d \
+	$(VMMLibDTrace_0_OUTDIR)/CPUMInternal.d \
+	$(VMMLibDTrace_0_OUTDIR)/x86.d
+$(call KB_FN_DO_PASS0_ON_TARGET,VMMLibDTrace)
+
+
+##
+# Turn the header $2 into the DTrace library script $1.
+#
+ define def_vmm_lib_dtrace_preprocess
+$$(call KB_FN_AUTO_CMD_DEPS,$$(VMMLibDTrace_0_OUTDIR)/$1)
+$$(VMMLibDTrace_0_OUTDIR)/$1: $2 $$(VBOX_VBOXCPP) | $$$$(dir $$$$@)
+	$$(call KB_FN_AUTO_CMD_DEPS_COMMANDS)
+	$$(QUIET)$$(call MSG_GENERATE,VMMLibDTrace,$$@,$2)
+	$$(QUIET)$(VBOX_VBOXCPP) -d \
+		-D VBOX_FOR_DTRACE_LIB \
+		-D VBOX_FOR_DTRACE_LIB_$(toupper $(KBUILD_TARGET_ARCH)) \
+		-D IN_RING0 \
+		-D RT_C_DECLS_BEGIN= \
+		-D RT_C_DECLS_END= \
+		-D RT_GCC_EXTENSION= \
+		-D 'RCPTRTYPE(a_Type)=RTRCPTR' \
+		-D 'R3PTRTYPE(a_Type)=RTR3PTR' \
+		-D 'R0PTRTYPE(a_Type)=a_Type' \
+		-D 'AssertCompile(a_Expr)=' \
+		-D 'AssertCompileSize(a_Stuct, a_Size)=' \
+		-D 'bool=uint8_t' \
+		$$(foreach def,\
+			$$(DEFS) \
+			$$(DEFS.$$(KBUILD_TARGET)) \
+			$$(DEFS.$(KBUILD_TARGET_ARCH)) \
+			$$(VMM_COMMON_DEFS) \
+			$$(ARCH_BITS_DEFS)\
+		   ,-D '$$(def)') \
+		$2 \
+		$$@
+	$$(QUIET)$$(CHMOD) 644 $$@
+
+ VMMLibDTrace_CLEAN += $(VMMLibDTrace_0_OUTDIR)/$1
+ endef
+$(evalcall2 def_vmm_lib_dtrace_preprocess,vm.d,$(PATH_ROOT)/include/VBox/vmm/vm.h)
+$(evalcall2 def_vmm_lib_dtrace_preprocess,cpumctx.d,$(PATH_ROOT)/include/VBox/vmm/cpumctx.h)
+$(evalcall2 def_vmm_lib_dtrace_preprocess,cpum.d,$(PATH_ROOT)/include/VBox/vmm/cpum.h)
+$(evalcall2 def_vmm_lib_dtrace_preprocess,CPUMInternal.d,$(PATH_SUB_CURRENT)/include/CPUMInternal.h)
+$(evalcall2 def_vmm_lib_dtrace_preprocess,x86.d,$(PATH_ROOT)/include/iprt/x86.h)
+
+endif
+
+
+
+#
+# For vmmGetSvnRev.
+#
+VMMAll/VMMAll.cpp_DEFS = VBOX_SVN_REV=$(VBOX_SVN_REV)
+
+#
+# Disable annoying warnings about array subscript above array bounds in aPages[]
+#
+ifneq ($(KBUILD_TARGET),win)
+VMMR3/PGMPool.cpp_CXXFLAGS     = $(VBOX_GCC_Wno-array_bounds)
+VMMAll/PGMAllPool.cpp_CXXFLAGS = $(VBOX_GCC_Wno-array_bounds)
+VMMAll/PGMAll.cpp_CXXFLAGS     = -Wno-unused-function
+VMMAll/IEMAll.cpp_CXXFLAGS     = -Wno-unused-function
+VMMR0/GMMR0.cpp_CXXFLAGS       = -Wno-unused-value
+endif
+
+#
+# Always optimize the interpreter.
+#
+if $(USERNAME) != "bird" || "$(KBUILD_TYPE)" == "release" #|| "$(KBUILD_TARGET).$(KBUILD_TARGET_ARCH)" == "win.amd64"
+ if1of ($(KBUILD_TARGET), win)
+# -noover is recognized despite the statement saying otherwise. It silences these warnings:
+# cl : Command line warning D9025 : overriding '/Od' with '/O2'
+# cl : Command line warning D9025 : overriding '/Oy-' with '/Oy'
+VMMAll/IEMAll.cpp_CXXFLAGS += -noover -O2xy
+ else
+VMMAll/IEMAll.cpp_CXXFLAGS += -O2
+#VMMAll/IEMAll.cpp_CXXFLAGS += -fno-align-functions -fno-align-jumps -fno-align-loops # Saves a few of percents, not worth it.
+#VMMAll/IEMAll.cpp_CXXFLAGS += -fno-reorder-blocks    # Saves one or two percent ... never mind.
+VMMAll/IEMAll.cpp_CXXFLAGS += -fomit-frame-pointer # Omitting the frame pointer results in larger code, but it might be worth it. (esp addressing vs ebp?)
+ endif
+endif # bird wants good stacks
+
+
+# Alias the CPU database entries.
+$(foreach base,$(notdir $(basename $(wildcard $(PATH_SUB_CURRENT)/VMMR3/cpus/*.h))), $(eval $(base).o $(base).obj: CPUMR3Db.o))
+
+
+#
+# Process python source(s).
+#
+BLDDIRS += $(PATH_TARGET)/pylint
+
+define def_vbox_vmm_py_check
+$(eval name:=$(basename $(notdir $(py))))
+
+pylint:    $(name)-py-phony.o
+$(name).o: $(name)-py-phony.o
+$(PATH_TARGET)/pylint/$(name).o $(name)-py-phony.o:: $(py) | $(PATH_TARGET)/pylint/
+ifdef VBOX_WITH_PYLINT
+	$(QUIET2)$(call MSG_L1,Subjecting $(py) to pylint...)
+	$(QUIET)$(REDIRECT) -C "$(dir $(py))" -E LC_ALL=C -- \
+		$(VBOX_PYLINT) --rcfile=$(PATH_TARGET)/no-such-pylintrc \
+			$$(VBOX_PYLINT_FLAGS) $$($(py)_VBOX_PYLINT_FLAGS) ./$(notdir $(py))
+endif
+	$(QUIET)$(APPEND) -t "$(PATH_TARGET)/pylint/$(name).o"
+
+TESTING += $(name)-py-phony.o
+endef # def_vbox_vmm_py_check
+
+$(foreach py, $(addprefix $(PATH_SUB_CURRENT)/VMMAll/, IEMAllInstructionsPython.py ) , $(eval $(def_vbox_vmm_py_check)))
+
+
+include $(FILE_KBUILD_SUB_FOOTER)
+
+
+# Alias the PGM templates to the object in which they are defined.
+PGMInternal.o: 	PGM.o
+
+PGMAllBth.o   	PGMAllGst.o   	PGMAllShw.o \
+PGMAllBth.obj 	PGMAllGst.obj 	PGMAllShw.obj: 	PGMAll.o
+
+PGMRCBth.o   	PGMRCGst.o   	PGMRCShw.o \
+PGMRCBth.obj 	PGMRCGst.obj 	PGMRCShw.obj: 	PGMRC.o
+
+PGMPhysRWTmpl.o PGMPhysRWTmpl.obj: 		PGMPhys.o
+
+PGMInline.o 	PGMInline.obj: 			PGMDbg.o
+
+# Alias the IEM templates to the object in which they are instantiated.
+IEMInternal.o \
+IEMAllInstructions.cpp.o           IEMAllInstructions.cpp.obj \
+IEMAllInstructionsOneByte.cpp.o    IEMAllInstructionsOneByte.cpp.obj \
+IEMAllInstructionsTwoByte0f.cpp.o  IEMAllInstructionsTwoByte0f.cpp.obj \
+IEMAllInstructionsThree0f38.cpp.o  IEMAllInstructionsThree0f38.cpp.obj \
+IEMAllInstructionsThree0f3a.cpp.o  IEMAllInstructionsThree0f3a.cpp.obj \
+IEMAllInstructionsVexMap1.cpp.o    IEMAllInstructionsVexMap1.cpp.obj \
+IEMAllInstructionsVexMap2.cpp.o    IEMAllInstructionsVexMap2.cpp.obj \
+IEMAllInstructionsVexMap3.cpp.o    IEMAllInstructionsVexMap3.cpp.obj \
+IEMAllInstructions3DNow.cpp.o      IEMAllInstructions3DNow.cpp.obj \
+IEMAllCImpl.cpp.o                  IEMAllCImpl.cpp.obj \
+IEMAllCImplStrInstr.cpp.o          IEMAllCImplStrInstr.cpp.obj \
+IEMAllCImplSvmInstr.cpp.o          IEMAllCImplSvmInstr.cpp.obj \
+IEMAllCImplVmxInstr.cpp.o          IEMAllCImplVmxInstr.cpp.obj: IEMAll.o
+
+# Alias the NEM template to the objects where it is used:
+NEMAllNativeTemplate-win.cpp.o:    NEMR3Native-win.o NEMR0Native-win.o
+
diff --git a/src/VBox/VMM/VBoxVMM.d b/src/VBox/VMM/VBoxVMM.d
new file mode 100644
index 00000000..a3dc129b
--- /dev/null
+++ b/src/VBox/VMM/VBoxVMM.d
@@ -0,0 +1,397 @@
+/* $Id: VBoxVMM.d $ */
+/** @file
+ * VBoxVMM - Static dtrace probes.
+ */
+
+/*
+ * Copyright (C) 2009-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+provider vboxvmm
+{
+    probe em__state__changed(struct VMCPU *a_pVCpu, int a_enmOldState, int a_enmNewState, int a_rc);
+    /*^^VMM-ALT-TP: "%d -> %d (rc=%d)", a_enmOldState, a_enmNewState, a_rc */
+
+    probe em__state__unchanged(struct VMCPU *a_pVCpu, int a_enmState, int a_rc);
+    /*^^VMM-ALT-TP: "%d (rc=%d)", a_enmState, a_rc */
+
+    probe em__raw__run__pre(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /*^^VMM-ALT-TP: "%04x:%08llx", (a_pCtx)->cs, (a_pCtx)->rip */
+
+    probe em__raw__run__ret(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, int a_rc);
+    /*^^VMM-ALT-TP: "%04x:%08llx rc=%d", (a_pCtx)->cs, (a_pCtx)->rip, (a_rc) */
+
+    probe em__ff__high(struct VMCPU *a_pVCpu, uint32_t a_fGlobal, uint64_t a_fLocal, int a_rc);
+    /*^^VMM-ALT-TP: "vm=%#x cpu=%#x rc=%d", (a_fGlobal), (a_fLocal), (a_rc) */
+
+    probe em__ff__all(struct VMCPU *a_pVCpu, uint32_t a_fGlobal, uint64_t a_fLocal, int a_rc);
+    /*^^VMM-ALT-TP: "vm=%#x cpu=%#x rc=%d", (a_fGlobal), (a_fLocal), (a_rc) */
+
+    probe em__ff__all__ret(struct VMCPU *a_pVCpu, int a_rc);
+    /*^^VMM-ALT-TP: "%d", (a_rc) */
+
+    probe em__ff__raw(struct VMCPU *a_pVCpu, uint32_t a_fGlobal, uint64_t a_fLocal);
+    /*^^VMM-ALT-TP: "vm=%#x cpu=%#x", (a_fGlobal), (a_fLocal) */
+
+    probe em__ff__raw_ret(struct VMCPU *a_pVCpu, int a_rc);
+    /*^^VMM-ALT-TP: "%d", (a_rc) */
+
+    probe pdm__irq__get( struct VMCPU *a_pVCpu, uint32_t a_uTag, uint32_t a_idSource, uint32_t a_iIrq);
+    probe pdm__irq__high(struct VMCPU *a_pVCpu, uint32_t a_uTag, uint32_t a_idSource);
+    probe pdm__irq__low( struct VMCPU *a_pVCpu, uint32_t a_uTag, uint32_t a_idSource);
+    probe pdm__irq__hilo(struct VMCPU *a_pVCpu, uint32_t a_uTag, uint32_t a_idSource);
+
+
+    probe r0__gvmm__vm__created(void *a_pGVM, void *a_pVM, uint32_t a_Pid, void *a_hEMT0, uint32_t a_cCpus);
+    probe r0__hmsvm__vmexit(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint64_t a_ExitCode, struct SVMVMCB *a_pVmcb);
+    probe r0__hmvmx__vmexit(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint64_t a_ExitReason, uint64_t a_ExitQualification);
+    probe r0__hmvmx__vmexit__noctx(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pIncompleteCtx, uint64_t a_ExitReason);
+
+    probe r0__vmm__return__to__ring3__rc(struct VMCPU *a_pVCpu, struct CPUMCTX *p_Ctx, int a_rc);
+    probe r0__vmm__return__to__ring3__hm(struct VMCPU *a_pVCpu, struct CPUMCTX *p_Ctx, int a_rc);
+    probe r0__vmm__return__to__ring3__nem(struct VMCPU *a_pVCpu, struct CPUMCTX *p_Ctx, int a_rc);
+
+
+    /** @name CPU Exception probes
+     * These probes will intercept guest CPU exceptions as best we
+     * can.  In some execution modes some of these probes may also
+     * see non-guest exceptions as we don't try distiguish between
+     * virtualization and guest exceptions before firing the probes.
+     *
+     * Using these probes may have a performance impact on guest
+     * activities involving lots of exceptions.
+     * @{
+     */
+    /** \#DE - integer divide error.  */
+    probe xcpt__de(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** \#DB - debug fault / trap.  */
+    probe xcpt__db(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint64_t a_dr6);
+    /** \#BP - breakpoint (INT3).  */
+    probe xcpt__bp(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** \#OF - overflow (INTO).  */
+    probe xcpt__of(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** \#BR - bound range exceeded (BOUND).  */
+    probe xcpt__br(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** \#UD - undefined opcode.  */
+    probe xcpt__ud(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** \#NM - FPU not avaible and more.  */
+    probe xcpt__nm(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** \#DF - double fault.  */
+    probe xcpt__df(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** \#TS - TSS related fault.  */
+    probe xcpt__ts(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint32_t a_uErr);
+    /** \#NP - segment not present.  */
+    probe xcpt__np(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint32_t a_uErr);
+    /** \#SS - stack segment fault.  */
+    probe xcpt__ss(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint32_t a_uErr);
+    /** \#GP - general protection fault.  */
+    probe xcpt__gp(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint32_t a_uErr);
+    /** \#PF - page fault.  */
+    probe xcpt__pf(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint32_t a_uErr, uint64_t a_cr2);
+    /** \#MF - math fault (FPU).  */
+    probe xcpt__mf(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** \#AC - alignment check.  */
+    probe xcpt__ac(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** \#XF - SIMD floating point exception.  */
+    probe xcpt__xf(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** \#VE - virtualization exception.  */
+    probe xcpt__ve(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** \#SX - security exception.  */
+    probe xcpt__sx(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint32_t a_uErr);
+    /** @} */
+
+
+    /** Software interrupt (INT XXh).
+     * It may be very difficult to implement this probe when using hardware
+     * virtualization, so maybe we have to drop it... */
+    probe int__software(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint8_t a_iInterrupt);
+    /** Hardware interrupt being dispatched.
+     *
+     * Relates to pdm__irq__get ...
+     */
+    probe int__hardware(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint8_t a_iInterrupt, uint32_t a_uTag, uint32_t a_idSource);
+
+    /** @name Instruction probes
+     * These are instructions normally related to VM exits.  These
+     * probes differs from the exit probes in that we will try make
+     * these instructions cause exits and fire the probe whenever
+     * they are executed by the guest.  This means some of these
+     * probes will have a noticable performance impact (like
+     * instr__pause).
+     * @{ */
+    /** Instruction: HALT */
+    probe instr__halt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: MWAIT */
+    probe instr__mwait(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: MONITOR */
+    probe instr__monitor(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: CPUID instruction (missing stuff in raw-mode). */
+    probe instr__cpuid(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint32_t uLeaf, uint32_t uSubLeaf);
+    /** Instruction: INVD  */
+    probe instr__invd(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: WBINVD */
+    probe instr__wbinvd(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: INVLPG */
+    probe instr__invlpg(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: RDTSC  */
+    probe instr__rdtsc(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: RDTSCP */
+    probe instr__rdtscp(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: RDPMC  */
+    probe instr__rdpmc(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: RDMSR  */
+    probe instr__rdmsr(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint32_t a_idMsr);
+    /** Instruction: WRMSR  */
+    probe instr__wrmsr(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint32_t a_idMsr, uint64_t a_uValue);
+    /** Instruction: CRx read instruction (missing smsw in raw-mode,
+     *  and reads in general in VT-x). */
+    probe instr__crx__read(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint8_t a_iReg);
+    /** Instruction: CRx write instruction. */
+    probe instr__crx__write(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint8_t a_iReg);
+    /** Instruction: DRx read instruction. */
+    probe instr__drx__read(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint8_t a_iReg);
+    /** Instruction: DRx write instruction. */
+    probe instr__drx__write(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint8_t a_iReg);
+    /** Instruction: PAUSE instruction (not in raw-mode). */
+    probe instr__pause(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: XSETBV */
+    probe instr__xsetbv(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: SIDT  */
+    probe instr__sidt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: LIDT */
+    probe instr__lidt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: SGDT */
+    probe instr__sgdt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: LGDT */
+    probe instr__lgdt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: SLDT */
+    probe instr__sldt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: LLDT */
+    probe instr__lldt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: STR */
+    probe instr__str(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: LTR */
+    probe instr__ltr(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: GETSEC */
+    probe instr__getsec(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: RSM */
+    probe instr__rsm(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: RDRAND */
+    probe instr__rdrand(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: RDSEED */
+    probe instr__rdseed(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: XSAVES */
+    probe instr__xsaves(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: XRSTORS  */
+    probe instr__xrstors(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VMCALL (intel) or VMMCALL (AMD) instruction. */
+    probe instr__vmm__call(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+
+    /** Instruction: VT-x VMCLEAR instruction. */
+    probe instr__vmx__vmclear(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VT-x VMLAUNCH */
+    probe instr__vmx__vmlaunch(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VT-x VMPTRLD */
+    probe instr__vmx__vmptrld(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VT-x VMPTRST */
+    probe instr__vmx__vmptrst(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VT-x VMREAD */
+    probe instr__vmx__vmread(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VT-x VMRESUME */
+    probe instr__vmx__vmresume(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VT-x VMWRITE */
+    probe instr__vmx__vmwrite(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VT-x VMXOFF */
+    probe instr__vmx__vmxoff(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VT-x VMXON */
+    probe instr__vmx__vmxon(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VT-x VMFUNC */
+    probe instr__vmx__vmfunc(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VT-x INVEPT */
+    probe instr__vmx__invept(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VT-x INVVPID */
+    probe instr__vmx__invvpid(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: VT-x INVPCID */
+    probe instr__vmx__invpcid(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+
+    /** Instruction: AMD-V VMRUN */
+    probe instr__svm__vmrun(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: AMD-V VMLOAD */
+    probe instr__svm__vmload(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: AMD-V VMSAVE */
+    probe instr__svm__vmsave(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: AMD-V STGI */
+    probe instr__svm__stgi(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** Instruction: AMD-V CLGI */
+    probe instr__svm__clgi(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** @} */
+
+
+    /** @name VM exit probes
+     * These are named exits with (in some cases at least) useful
+     * information as arguments.  Unlike the instruction probes,
+     * these will not change the number of VM exits and have much
+     * less of an impact on VM performance.
+     * @{ */
+    /** VM Exit: Task switch. */
+    probe exit__task__switch(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: HALT instruction.
+     * @todo not yet implemented. */
+    probe exit__halt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: MWAIT instruction. */
+    probe exit__mwait(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: MONITOR instruction. */
+    probe exit__monitor(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: CPUID instruction (missing stuff in raw-mode). */
+    probe exit__cpuid(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint32_t uLeaf, uint32_t uSubLeaf);
+    /** VM Exit: INVD instruction.  */
+    probe exit__invd(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: WBINVD instruction. */
+    probe exit__wbinvd(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: INVLPG instruction. */
+    probe exit__invlpg(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: RDTSC instruction.  */
+    probe exit__rdtsc(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: RDTSCP instruction. */
+    probe exit__rdtscp(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: RDPMC instruction.  */
+    probe exit__rdpmc(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: RDMSR instruction.  */
+    probe exit__rdmsr(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint32_t a_idMsr);
+    /** VM Exit: WRMSR instruction.  */
+    probe exit__wrmsr(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint32_t a_idMsr, uint64_t a_uValue);
+    /** VM Exit: CRx read instruction (missing smsw in raw-mode,
+     *  and reads in general in VT-x). */
+    probe exit__crx__read(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint8_t a_iReg);
+    /** VM Exit: CRx write instruction. */
+    probe exit__crx__write(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint8_t a_iReg);
+    /** VM Exit: DRx read instruction. */
+    probe exit__drx__read(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint8_t a_iReg);
+    /** VM Exit: DRx write instruction. */
+    probe exit__drx__write(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx, uint8_t a_iReg);
+    /** VM Exit: PAUSE instruction (not in raw-mode). */
+    probe exit__pause(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: XSETBV instruction. */
+    probe exit__xsetbv(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: SIDT instruction.  */
+    probe exit__sidt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: LIDT instruction. */
+    probe exit__lidt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: SGDT instruction. */
+    probe exit__sgdt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: LGDT instruction. */
+    probe exit__lgdt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: SLDT instruction. */
+    probe exit__sldt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: LLDT instruction. */
+    probe exit__lldt(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: STR instruction. */
+    probe exit__str(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: LTR instruction. */
+    probe exit__ltr(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: GETSEC instruction. */
+    probe exit__getsec(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: RSM instruction. */
+    probe exit__rsm(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: RDRAND instruction. */
+    probe exit__rdrand(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: RDSEED instruction. */
+    probe exit__rdseed(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: XSAVES instruction. */
+    probe exit__xsaves(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: XRSTORS instruction.  */
+    probe exit__xrstors(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VMCALL (intel) or VMMCALL (AMD) instruction. */
+    probe exit__vmm__call(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+
+    /** VM Exit: VT-x VMCLEAR instruction. */
+    probe exit__vmx__vmclear(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x VMLAUNCH instruction. */
+    probe exit__vmx__vmlaunch(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x VMPTRLD instruction. */
+    probe exit__vmx__vmptrld(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x VMPTRST instruction. */
+    probe exit__vmx__vmptrst(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x VMREAD instruction. */
+    probe exit__vmx__vmread(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x VMRESUME instruction. */
+    probe exit__vmx__vmresume(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x VMWRITE instruction. */
+    probe exit__vmx__vmwrite(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x VMXOFF instruction. */
+    probe exit__vmx__vmxoff(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x VMXON instruction. */
+    probe exit__vmx__vmxon(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x VMFUNC instruction. */
+    probe exit__vmx__vmfunc(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x INVEPT instruction. */
+    probe exit__vmx__invept(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x INVVPID instruction. */
+    probe exit__vmx__invvpid(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x INVPCID instruction. */
+    probe exit__vmx__invpcid(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x EPT violation. */
+    probe exit__vmx__ept__violation(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x EPT misconfiguration. */
+    probe exit__vmx__ept__misconfig(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x Virtual APIC page access. */
+    probe exit__vmx__vapic__access(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: VT-x Virtual APIC page write needing virtualizing. */
+    probe exit__vmx__vapic__write(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+
+    /** VM Exit: AMD-V VMRUN instruction. */
+    probe exit__svm__vmrun(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: AMD-V VMLOAD instruction. */
+    probe exit__svm__vmload(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: AMD-V VMSAVE instruction. */
+    probe exit__svm__vmsave(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: AMD-V STGI instruction. */
+    probe exit__svm__stgi(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** VM Exit: AMD-V CLGI instruction. */
+    probe exit__svm__clgi(struct VMCPU *a_pVCpu, struct CPUMCTX *a_pCtx);
+    /** @} */
+
+
+    /** @name IPRT tracepoints we link in.
+     * @{ */
+    probe iprt__critsect__entered(void *a_pvCritSect, const char *a_pszLaterNm, int32_t a_cLockers, uint32_t a_cNestings);
+    probe iprt__critsect__leaving(void *a_pvCritSect, const char *a_pszLaterNm, int32_t a_cLockers, uint32_t a_cNestings);
+    probe iprt__critsect__waiting(void *a_pvCritSect, const char *a_pszLaterNm, int32_t a_cLockers, void *a_pvNativeThreadOwner);
+    probe iprt__critsect__busy(   void *a_pvCritSect, const char *a_pszLaterNm, int32_t a_cLockers, void *a_pvNativeThreadOwner);
+
+    probe iprt__critsectrw__excl_entered(void *a_pvCritSect, const char *a_pszLaterNm, uint32_t a_cNestings,
+                                         uint32_t a_cWaitingReaders, uint32_t cWriters);
+    probe iprt__critsectrw__excl_leaving(void *a_pvCritSect, const char *a_pszLaterNm, uint32_t a_cNestings,
+                                         uint32_t a_cWaitingReaders, uint32_t cWriters);
+    probe iprt__critsectrw__excl_waiting(void *a_pvCritSect, const char *a_pszLaterNm, uint8_t a_fWriteMode, uint32_t a_cWaitingReaders,
+                                         uint32_t a_cReaders, uint32_t a_cWriters, void *a_pvNativeOwnerThread);
+    probe iprt__critsectrw__excl_busy(   void *a_pvCritSect, const char *a_pszLaterNm, uint8_t a_fWriteMode, uint32_t a_cWaitingReaders,
+                                         uint32_t a_cReaders, uint32_t a_cWriters, void *a_pvNativeOwnerThread);
+    probe iprt__critsectrw__excl_entered_shared(void *a_pvCritSect, const char *a_pszLaterNm, uint32_t a_cNestings,
+                                                uint32_t a_cWaitingReaders, uint32_t a_cWriters);
+    probe iprt__critsectrw__excl_leaving_shared(void *a_pvCritSect, const char *a_pszLaterNm, uint32_t a_cNestings,
+                                                uint32_t a_cWaitingReaders, uint32_t a_cWriters);
+    probe iprt__critsectrw__shared_entered(void *a_pvCritSect, const char *a_pszLaterNm, uint32_t a_cReaders, uint32_t a_cNestings);
+    probe iprt__critsectrw__shared_leaving(void *a_pvCritSect, const char *a_pszLaterNm, uint32_t a_cReaders, uint32_t a_cNestings);
+    probe iprt__critsectrw__shared_waiting(void *a_pvCritSect, const char *a_pszLaterNm, void *a_pvNativeThreadOwner,
+                                           uint32_t cWaitingReaders, uint32_t cWriters);
+    probe iprt__critsectrw__shared_busy(   void *a_pvCritSect, const char *a_pszLaterNm, void *a_pvNativeThreadOwner,
+                                           uint32_t a_cWaitingReaders, uint32_t a_cWriters);
+
+    /** @} */
+};
+
+#pragma D attributes Evolving/Evolving/Common provider vboxvmm provider
+#pragma D attributes Private/Private/Unknown  provider vboxvmm module
+#pragma D attributes Private/Private/Unknown  provider vboxvmm function
+#pragma D attributes Evolving/Evolving/Common provider vboxvmm name
+#pragma D attributes Evolving/Evolving/Common provider vboxvmm args
+
diff --git a/src/VBox/VMM/VMMAll/APICAll.cpp b/src/VBox/VMM/VMMAll/APICAll.cpp
new file mode 100644
index 00000000..14209e84
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/APICAll.cpp
@@ -0,0 +1,3610 @@
+/* $Id: APICAll.cpp $ */
+/** @file
+ * APIC - Advanced Programmable Interrupt Controller - All Contexts.
+ */
+
+/*
+ * Copyright (C) 2016-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DEV_APIC
+#include "APICInternal.h"
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/vmcpuset.h>
+#ifdef IN_RING0
+# include <VBox/vmm/gvmm.h>
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static void apicSetInterruptFF(PVMCPUCC pVCpu, PDMAPICIRQ enmType);
+static void apicStopTimer(PVMCPUCC pVCpu);
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+/** An ordered array of valid LVT masks. */
+static const uint32_t g_au32LvtValidMasks[] =
+{
+    XAPIC_LVT_TIMER_VALID,
+    XAPIC_LVT_THERMAL_VALID,
+    XAPIC_LVT_PERF_VALID,
+    XAPIC_LVT_LINT_VALID,   /* LINT0 */
+    XAPIC_LVT_LINT_VALID,   /* LINT1 */
+    XAPIC_LVT_ERROR_VALID
+};
+#endif
+
+#if 0
+/** @todo CMCI */
+static const uint32_t g_au32LvtExtValidMask[] =
+{
+    XAPIC_LVT_CMCI_VALID
+};
+#endif
+
+
+/**
+ * Checks if a vector is set in an APIC 256-bit sparse register.
+ *
+ * @returns true if the specified vector is set, false otherwise.
+ * @param   pApicReg        The APIC 256-bit spare register.
+ * @param   uVector         The vector to check if set.
+ */
+DECLINLINE(bool) apicTestVectorInReg(const volatile XAPIC256BITREG *pApicReg, uint8_t uVector)
+{
+    const volatile uint8_t *pbBitmap = (const volatile uint8_t *)&pApicReg->u[0];
+    return ASMBitTest(pbBitmap + XAPIC_REG256_VECTOR_OFF(uVector), XAPIC_REG256_VECTOR_BIT(uVector));
+}
+
+
+/**
+ * Sets the vector in an APIC 256-bit sparse register.
+ *
+ * @param   pApicReg        The APIC 256-bit spare register.
+ * @param   uVector         The vector to set.
+ */
+DECLINLINE(void) apicSetVectorInReg(volatile XAPIC256BITREG *pApicReg, uint8_t uVector)
+{
+    volatile uint8_t *pbBitmap = (volatile uint8_t *)&pApicReg->u[0];
+    ASMAtomicBitSet(pbBitmap + XAPIC_REG256_VECTOR_OFF(uVector), XAPIC_REG256_VECTOR_BIT(uVector));
+}
+
+
+/**
+ * Clears the vector in an APIC 256-bit sparse register.
+ *
+ * @param   pApicReg        The APIC 256-bit spare register.
+ * @param   uVector         The vector to clear.
+ */
+DECLINLINE(void) apicClearVectorInReg(volatile XAPIC256BITREG *pApicReg, uint8_t uVector)
+{
+    volatile uint8_t *pbBitmap = (volatile uint8_t *)&pApicReg->u[0];
+    ASMAtomicBitClear(pbBitmap + XAPIC_REG256_VECTOR_OFF(uVector), XAPIC_REG256_VECTOR_BIT(uVector));
+}
+
+
+#if 0 /* unused */
+/**
+ * Checks if a vector is set in an APIC Pending-Interrupt Bitmap (PIB).
+ *
+ * @returns true if the specified vector is set, false otherwise.
+ * @param   pvPib           Opaque pointer to the PIB.
+ * @param   uVector         The vector to check if set.
+ */
+DECLINLINE(bool) apicTestVectorInPib(volatile void *pvPib, uint8_t uVector)
+{
+    return ASMBitTest(pvPib, uVector);
+}
+#endif /* unused */
+
+
+/**
+ * Atomically sets the PIB notification bit.
+ *
+ * @returns non-zero if the bit was already set, 0 otherwise.
+ * @param   pApicPib        Pointer to the PIB.
+ */
+DECLINLINE(uint32_t) apicSetNotificationBitInPib(PAPICPIB pApicPib)
+{
+    return ASMAtomicXchgU32(&pApicPib->fOutstandingNotification, RT_BIT_32(31));
+}
+
+
+/**
+ * Atomically tests and clears the PIB notification bit.
+ *
+ * @returns non-zero if the bit was already set, 0 otherwise.
+ * @param   pApicPib        Pointer to the PIB.
+ */
+DECLINLINE(uint32_t) apicClearNotificationBitInPib(PAPICPIB pApicPib)
+{
+    return ASMAtomicXchgU32(&pApicPib->fOutstandingNotification, UINT32_C(0));
+}
+
+
+/**
+ * Sets the vector in an APIC Pending-Interrupt Bitmap (PIB).
+ *
+ * @param   pvPib           Opaque pointer to the PIB.
+ * @param   uVector         The vector to set.
+ */
+DECLINLINE(void) apicSetVectorInPib(volatile void *pvPib, uint8_t uVector)
+{
+    ASMAtomicBitSet(pvPib, uVector);
+}
+
+#if 0 /* unused */
+/**
+ * Clears the vector in an APIC Pending-Interrupt Bitmap (PIB).
+ *
+ * @param   pvPib           Opaque pointer to the PIB.
+ * @param   uVector         The vector to clear.
+ */
+DECLINLINE(void) apicClearVectorInPib(volatile void *pvPib, uint8_t uVector)
+{
+    ASMAtomicBitClear(pvPib, uVector);
+}
+#endif /* unused */
+
+#if 0 /* unused */
+/**
+ * Atomically OR's a fragment (32 vectors) into an APIC 256-bit sparse
+ * register.
+ *
+ * @param   pApicReg        The APIC 256-bit spare register.
+ * @param   idxFragment     The index of the 32-bit fragment in @a
+ *                          pApicReg.
+ * @param   u32Fragment     The 32-bit vector fragment to OR.
+ */
+DECLINLINE(void) apicOrVectorsToReg(volatile XAPIC256BITREG *pApicReg, size_t idxFragment, uint32_t u32Fragment)
+{
+    Assert(idxFragment < RT_ELEMENTS(pApicReg->u));
+    ASMAtomicOrU32(&pApicReg->u[idxFragment].u32Reg, u32Fragment);
+}
+#endif /* unused */
+
+
+#if 0 /* unused */
+/**
+ * Atomically AND's a fragment (32 vectors) into an APIC
+ * 256-bit sparse register.
+ *
+ * @param   pApicReg        The APIC 256-bit spare register.
+ * @param   idxFragment     The index of the 32-bit fragment in @a
+ *                          pApicReg.
+ * @param   u32Fragment     The 32-bit vector fragment to AND.
+ */
+DECLINLINE(void) apicAndVectorsToReg(volatile XAPIC256BITREG *pApicReg, size_t idxFragment, uint32_t u32Fragment)
+{
+    Assert(idxFragment < RT_ELEMENTS(pApicReg->u));
+    ASMAtomicAndU32(&pApicReg->u[idxFragment].u32Reg, u32Fragment);
+}
+#endif /* unused */
+
+
+/**
+ * Reports and returns appropriate error code for invalid MSR accesses.
+ *
+ * @returns VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u32Reg          The MSR being accessed.
+ * @param   enmAccess       The invalid-access type.
+ */
+static int apicMsrAccessError(PVMCPUCC pVCpu, uint32_t u32Reg, APICMSRACCESS enmAccess)
+{
+    static struct
+    {
+        const char *pszBefore;   /* The error message before printing the MSR index */
+        const char *pszAfter;    /* The error message after printing the MSR index */
+    } const s_aAccess[] =
+    {
+        /* enmAccess  pszBefore                        pszAfter */
+        /* 0 */     { "read MSR",                      " while not in x2APIC mode"    },
+        /* 1 */     { "write MSR",                     " while not in x2APIC mode"    },
+        /* 2 */     { "read reserved/unknown MSR",     ""                             },
+        /* 3 */     { "write reserved/unknown MSR",    ""                             },
+        /* 4 */     { "read write-only MSR",           ""                             },
+        /* 5 */     { "write read-only MSR",           ""                             },
+        /* 6 */     { "read reserved bits of MSR",     ""                             },
+        /* 7 */     { "write reserved bits of MSR",    ""                             },
+        /* 8 */     { "write an invalid value to MSR", ""                             },
+        /* 9 */     { "write MSR",                     " disallowed by configuration" },
+        /* 10 */    { "read MSR",                      " disallowed by configuration" },
+    };
+    AssertCompile(RT_ELEMENTS(s_aAccess) == APICMSRACCESS_COUNT);
+
+    size_t const i = enmAccess;
+    Assert(i < RT_ELEMENTS(s_aAccess));
+    if (pVCpu->apic.s.cLogMaxAccessError++ < 5)
+        LogRel(("APIC%u: Attempt to %s (%#x)%s -> #GP(0)\n", pVCpu->idCpu, s_aAccess[i].pszBefore, u32Reg, s_aAccess[i].pszAfter));
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * Gets the descriptive APIC mode.
+ *
+ * @returns The name.
+ * @param   enmMode     The xAPIC mode.
+ */
+const char *apicGetModeName(APICMODE enmMode)
+{
+    switch (enmMode)
+    {
+        case APICMODE_DISABLED:  return "Disabled";
+        case APICMODE_XAPIC:     return "xAPIC";
+        case APICMODE_X2APIC:    return "x2APIC";
+        default:                 break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the descriptive destination format name.
+ *
+ * @returns The destination format name.
+ * @param   enmDestFormat       The destination format.
+ */
+const char *apicGetDestFormatName(XAPICDESTFORMAT enmDestFormat)
+{
+    switch (enmDestFormat)
+    {
+        case XAPICDESTFORMAT_FLAT:      return "Flat";
+        case XAPICDESTFORMAT_CLUSTER:   return "Cluster";
+        default:                        break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the descriptive delivery mode name.
+ *
+ * @returns The delivery mode name.
+ * @param   enmDeliveryMode     The delivery mode.
+ */
+const char *apicGetDeliveryModeName(XAPICDELIVERYMODE enmDeliveryMode)
+{
+    switch (enmDeliveryMode)
+    {
+        case XAPICDELIVERYMODE_FIXED:        return "Fixed";
+        case XAPICDELIVERYMODE_LOWEST_PRIO:  return "Lowest-priority";
+        case XAPICDELIVERYMODE_SMI:          return "SMI";
+        case XAPICDELIVERYMODE_NMI:          return "NMI";
+        case XAPICDELIVERYMODE_INIT:         return "INIT";
+        case XAPICDELIVERYMODE_STARTUP:      return "SIPI";
+        case XAPICDELIVERYMODE_EXTINT:       return "ExtINT";
+        default:                             break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the descriptive destination mode name.
+ *
+ * @returns The destination mode name.
+ * @param   enmDestMode     The destination mode.
+ */
+const char *apicGetDestModeName(XAPICDESTMODE enmDestMode)
+{
+    switch (enmDestMode)
+    {
+        case XAPICDESTMODE_PHYSICAL:  return "Physical";
+        case XAPICDESTMODE_LOGICAL:   return "Logical";
+        default:                      break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the descriptive trigger mode name.
+ *
+ * @returns The trigger mode name.
+ * @param   enmTriggerMode     The trigger mode.
+ */
+const char *apicGetTriggerModeName(XAPICTRIGGERMODE enmTriggerMode)
+{
+    switch (enmTriggerMode)
+    {
+        case XAPICTRIGGERMODE_EDGE:     return "Edge";
+        case XAPICTRIGGERMODE_LEVEL:    return "Level";
+        default:                        break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the destination shorthand name.
+ *
+ * @returns The destination shorthand name.
+ * @param   enmDestShorthand     The destination shorthand.
+ */
+const char *apicGetDestShorthandName(XAPICDESTSHORTHAND enmDestShorthand)
+{
+    switch (enmDestShorthand)
+    {
+        case XAPICDESTSHORTHAND_NONE:           return "None";
+        case XAPICDESTSHORTHAND_SELF:           return "Self";
+        case XAPIDDESTSHORTHAND_ALL_INCL_SELF:  return "All including self";
+        case XAPICDESTSHORTHAND_ALL_EXCL_SELF:  return "All excluding self";
+        default:                                break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the timer mode name.
+ *
+ * @returns The timer mode name.
+ * @param   enmTimerMode         The timer mode.
+ */
+const char *apicGetTimerModeName(XAPICTIMERMODE enmTimerMode)
+{
+    switch (enmTimerMode)
+    {
+        case XAPICTIMERMODE_ONESHOT:        return "One-shot";
+        case XAPICTIMERMODE_PERIODIC:       return "Periodic";
+        case XAPICTIMERMODE_TSC_DEADLINE:   return "TSC deadline";
+        default:                            break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the APIC mode given the base MSR value.
+ *
+ * @returns The APIC mode.
+ * @param   uApicBaseMsr        The APIC Base MSR value.
+ */
+APICMODE apicGetMode(uint64_t uApicBaseMsr)
+{
+    uint32_t const uMode   = (uApicBaseMsr >> 10) & UINT64_C(3);
+    APICMODE const enmMode = (APICMODE)uMode;
+#ifdef VBOX_STRICT
+    /* Paranoia. */
+    switch (uMode)
+    {
+        case APICMODE_DISABLED:
+        case APICMODE_INVALID:
+        case APICMODE_XAPIC:
+        case APICMODE_X2APIC:
+            break;
+        default:
+            AssertMsgFailed(("Invalid mode"));
+    }
+#endif
+    return enmMode;
+}
+
+
+/**
+ * Returns whether the APIC is hardware enabled or not.
+ *
+ * @returns true if enabled, false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) APICIsEnabled(PCVMCPUCC pVCpu)
+{
+    PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    return RT_BOOL(pApicCpu->uApicBaseMsr & MSR_IA32_APICBASE_EN);
+}
+
+
+/**
+ * Finds the most significant set bit in an APIC 256-bit sparse register.
+ *
+ * @returns @a rcNotFound if no bit was set, 0-255 otherwise.
+ * @param   pReg            The APIC 256-bit sparse register.
+ * @param   rcNotFound      What to return when no bit is set.
+ */
+static int apicGetHighestSetBitInReg(volatile const XAPIC256BITREG *pReg, int rcNotFound)
+{
+    ssize_t const  cFragments     = RT_ELEMENTS(pReg->u);
+    unsigned const uFragmentShift = 5;
+    AssertCompile(1 << uFragmentShift == sizeof(pReg->u[0].u32Reg) * 8);
+    for (ssize_t i = cFragments - 1; i >= 0; i--)
+    {
+        uint32_t const uFragment = pReg->u[i].u32Reg;
+        if (uFragment)
+        {
+            unsigned idxSetBit = ASMBitLastSetU32(uFragment);
+            --idxSetBit;
+            idxSetBit |= i << uFragmentShift;
+            return idxSetBit;
+        }
+    }
+    return rcNotFound;
+}
+
+
+/**
+ * Reads a 32-bit register at a specified offset.
+ *
+ * @returns The value at the specified offset.
+ * @param   pXApicPage      The xAPIC page.
+ * @param   offReg          The offset of the register being read.
+ */
+DECLINLINE(uint32_t) apicReadRaw32(PCXAPICPAGE pXApicPage, uint16_t offReg)
+{
+    Assert(offReg < sizeof(*pXApicPage) - sizeof(uint32_t));
+    uint8_t  const *pbXApic =  (const uint8_t *)pXApicPage;
+    uint32_t const  uValue  = *(const uint32_t *)(pbXApic + offReg);
+    return uValue;
+}
+
+
+/**
+ * Writes a 32-bit register at a specified offset.
+ *
+ * @param   pXApicPage      The xAPIC page.
+ * @param   offReg          The offset of the register being written.
+ * @param   uReg            The value of the register.
+ */
+DECLINLINE(void) apicWriteRaw32(PXAPICPAGE pXApicPage, uint16_t offReg, uint32_t uReg)
+{
+    Assert(offReg < sizeof(*pXApicPage) - sizeof(uint32_t));
+    uint8_t *pbXApic = (uint8_t *)pXApicPage;
+    *(uint32_t *)(pbXApic + offReg) = uReg;
+}
+
+
+/**
+ * Sets an error in the internal ESR of the specified APIC.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uError          The error.
+ * @thread  Any.
+ */
+DECLINLINE(void) apicSetError(PVMCPUCC pVCpu, uint32_t uError)
+{
+    PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    ASMAtomicOrU32(&pApicCpu->uEsrInternal, uError);
+}
+
+
+/**
+ * Clears all errors in the internal ESR.
+ *
+ * @returns The value of the internal ESR before clearing.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+DECLINLINE(uint32_t) apicClearAllErrors(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    return ASMAtomicXchgU32(&pApicCpu->uEsrInternal, 0);
+}
+
+
+/**
+ * Signals the guest if a pending interrupt is ready to be serviced.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+static void apicSignalNextPendingIntr(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    if (pXApicPage->svr.u.fApicSoftwareEnable)
+    {
+        int const irrv = apicGetHighestSetBitInReg(&pXApicPage->irr, -1 /* rcNotFound */);
+        if (irrv >= 0)
+        {
+            Assert(irrv <= (int)UINT8_MAX);
+            uint8_t const uVector = irrv;
+            int const isrv        = apicGetHighestSetBitInReg(&pXApicPage->isr, 0 /* rcNotFound */);
+            Assert(isrv <= (int)UINT8_MAX);
+            uint8_t const uIsrVec = isrv;
+
+            /* uIsrVect reflects the highest interrupt vector currently serviced (i.e. in ISR),
+             * or zero if there's none. We want to report a pending interrupt only if IRR > ISR but
+             * regardless of TPR. Hence we can't look at the PPR value, since that also reflects TPR.
+             * NB: The APIC emulation will know when ISR changes, but not necessarily when TPR does.
+             */
+            if (XAPIC_PPR_GET_PP(uVector) > XAPIC_PPR_GET_PP(uIsrVec))
+            {
+                Log2(("APIC%u: apicSignalNextPendingIntr: Signalling pending interrupt. uVector=%#x\n", pVCpu->idCpu, uVector));
+                apicSetInterruptFF(pVCpu, PDMAPICIRQ_HARDWARE);
+            }
+            else
+            {
+                Log2(("APIC%u: apicSignalNextPendingIntr: Nothing to signal yet. uVector=%#x uIsrVec=%#x\n", pVCpu->idCpu, uVector, uIsrVec));
+            }
+        }
+    }
+    else
+    {
+        Log2(("APIC%u: apicSignalNextPendingIntr: APIC software-disabled, clearing pending interrupt\n", pVCpu->idCpu));
+        apicClearInterruptFF(pVCpu, PDMAPICIRQ_HARDWARE);
+    }
+}
+
+
+/**
+ * Sets the Spurious-Interrupt Vector Register (SVR).
+ *
+ * @returns VINF_SUCCESS or VERR_CPUM_RAISE_GP_0.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uSvr            The SVR value.
+ */
+static int apicSetSvr(PVMCPUCC pVCpu, uint32_t uSvr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    uint32_t   uValidMask = XAPIC_SVR_VALID;
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    if (pXApicPage->version.u.fEoiBroadcastSupression)
+        uValidMask |= XAPIC_SVR_SUPRESS_EOI_BROADCAST;
+
+    if (   XAPIC_IN_X2APIC_MODE(pVCpu)
+        && (uSvr & ~uValidMask))
+        return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_SVR, APICMSRACCESS_WRITE_RSVD_BITS);
+
+    Log2(("APIC%u: apicSetSvr: uSvr=%#RX32\n", pVCpu->idCpu, uSvr));
+    apicWriteRaw32(pXApicPage, XAPIC_OFF_SVR, uSvr);
+    if (!pXApicPage->svr.u.fApicSoftwareEnable)
+    {
+        /** @todo CMCI. */
+        pXApicPage->lvt_timer.u.u1Mask   = 1;
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+        pXApicPage->lvt_thermal.u.u1Mask = 1;
+#endif
+        pXApicPage->lvt_perf.u.u1Mask    = 1;
+        pXApicPage->lvt_lint0.u.u1Mask   = 1;
+        pXApicPage->lvt_lint1.u.u1Mask   = 1;
+        pXApicPage->lvt_error.u.u1Mask   = 1;
+    }
+
+    apicSignalNextPendingIntr(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sends an interrupt to one or more APICs.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure, can be
+ *                              NULL if the source of the interrupt is not an
+ *                              APIC (for e.g. a bus).
+ * @param   uVector             The interrupt vector.
+ * @param   enmTriggerMode      The trigger mode.
+ * @param   enmDeliveryMode     The delivery mode.
+ * @param   pDestCpuSet         The destination CPU set.
+ * @param   pfIntrAccepted      Where to store whether this interrupt was
+ *                              accepted by the target APIC(s) or not.
+ *                              Optional, can be NULL.
+ * @param   uSrcTag             The interrupt source tag (debugging).
+ * @param   rcRZ                The return code if the operation cannot be
+ *                              performed in the current context.
+ */
+static VBOXSTRICTRC apicSendIntr(PVMCC pVM, PVMCPUCC pVCpu, uint8_t uVector, XAPICTRIGGERMODE enmTriggerMode,
+                                 XAPICDELIVERYMODE enmDeliveryMode, PCVMCPUSET pDestCpuSet, bool *pfIntrAccepted,
+                                 uint32_t uSrcTag, int rcRZ)
+{
+    VBOXSTRICTRC  rcStrict  = VINF_SUCCESS;
+    VMCPUID const cCpus     = pVM->cCpus;
+    bool          fAccepted = false;
+    switch (enmDeliveryMode)
+    {
+        case XAPICDELIVERYMODE_FIXED:
+        {
+            for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+                if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu))
+                {
+                    PVMCPUCC pItVCpu = pVM->CTX_SUFF(apCpus)[idCpu];
+                    if (APICIsEnabled(pItVCpu))
+                        fAccepted = apicPostInterrupt(pItVCpu, uVector, enmTriggerMode, uSrcTag);
+                }
+            break;
+        }
+
+        case XAPICDELIVERYMODE_LOWEST_PRIO:
+        {
+            VMCPUID const idCpu = VMCPUSET_FIND_FIRST_PRESENT(pDestCpuSet);
+            AssertMsgBreak(idCpu < pVM->cCpus, ("APIC: apicSendIntr: No CPU found for lowest-priority delivery mode! idCpu=%u\n", idCpu));
+            PVMCPUCC pVCpuDst = pVM->CTX_SUFF(apCpus)[idCpu];
+            if (APICIsEnabled(pVCpuDst))
+                fAccepted = apicPostInterrupt(pVCpuDst, uVector, enmTriggerMode, uSrcTag);
+            else
+                AssertMsgFailed(("APIC: apicSendIntr: Target APIC not enabled in lowest-priority delivery mode! idCpu=%u\n", idCpu));
+            break;
+        }
+
+        case XAPICDELIVERYMODE_SMI:
+        {
+            for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+                if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu))
+                {
+                    Log2(("APIC: apicSendIntr: Raising SMI on VCPU%u\n", idCpu));
+                    apicSetInterruptFF(pVM->CTX_SUFF(apCpus)[idCpu], PDMAPICIRQ_SMI);
+                    fAccepted = true;
+                }
+            break;
+        }
+
+        case XAPICDELIVERYMODE_NMI:
+        {
+            for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+                if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu))
+                {
+                    PVMCPUCC pItVCpu = pVM->CTX_SUFF(apCpus)[idCpu];
+                    if (APICIsEnabled(pItVCpu))
+                    {
+                        Log2(("APIC: apicSendIntr: Raising NMI on VCPU%u\n", idCpu));
+                        apicSetInterruptFF(pItVCpu, PDMAPICIRQ_NMI);
+                        fAccepted = true;
+                    }
+                }
+            break;
+        }
+
+        case XAPICDELIVERYMODE_INIT:
+        {
+#ifdef IN_RING3
+            for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+                if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu))
+                {
+                    Log2(("APIC: apicSendIntr: Issuing INIT to VCPU%u\n", idCpu));
+                    VMMR3SendInitIpi(pVM, idCpu);
+                    fAccepted = true;
+                }
+#else
+            /* We need to return to ring-3 to deliver the INIT. */
+            rcStrict = rcRZ;
+            fAccepted = true;
+#endif
+            break;
+        }
+
+        case XAPICDELIVERYMODE_STARTUP:
+        {
+#ifdef IN_RING3
+            for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+                if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu))
+                {
+                    Log2(("APIC: apicSendIntr: Issuing SIPI to VCPU%u\n", idCpu));
+                    VMMR3SendStartupIpi(pVM, idCpu, uVector);
+                    fAccepted = true;
+                }
+#else
+            /* We need to return to ring-3 to deliver the SIPI. */
+            rcStrict = rcRZ;
+            fAccepted = true;
+            Log2(("APIC: apicSendIntr: SIPI issued, returning to RZ. rc=%Rrc\n", rcRZ));
+#endif
+            break;
+        }
+
+        case XAPICDELIVERYMODE_EXTINT:
+        {
+            for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+                if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu))
+                {
+                    Log2(("APIC: apicSendIntr: Raising EXTINT on VCPU%u\n", idCpu));
+                    apicSetInterruptFF(pVM->CTX_SUFF(apCpus)[idCpu], PDMAPICIRQ_EXTINT);
+                    fAccepted = true;
+                }
+            break;
+        }
+
+        default:
+        {
+            AssertMsgFailed(("APIC: apicSendIntr: Unsupported delivery mode %#x (%s)\n", enmDeliveryMode,
+                             apicGetDeliveryModeName(enmDeliveryMode)));
+            break;
+        }
+    }
+
+    /*
+     * If an illegal vector is programmed, set the 'send illegal vector' error here if the
+     * interrupt is being sent by an APIC.
+     *
+     * The 'receive illegal vector' will be set on the target APIC when the interrupt
+     * gets generated, see apicPostInterrupt().
+     *
+     * See Intel spec. 10.5.3 "Error Handling".
+     */
+    if (   rcStrict != rcRZ
+        && pVCpu)
+    {
+        /*
+         * Flag only errors when the delivery mode is fixed and not others.
+         *
+         * Ubuntu 10.04-3 amd64 live CD with 2 VCPUs gets upset as it sends an SIPI to the
+         * 2nd VCPU with vector 6 and checks the ESR for no errors, see @bugref{8245#c86}.
+         */
+        /** @todo The spec says this for LVT, but not explcitly for ICR-lo
+         *        but it probably is true. */
+        if (enmDeliveryMode == XAPICDELIVERYMODE_FIXED)
+        {
+            if (RT_UNLIKELY(uVector <= XAPIC_ILLEGAL_VECTOR_END))
+                apicSetError(pVCpu, XAPIC_ESR_SEND_ILLEGAL_VECTOR);
+        }
+    }
+
+    if (pfIntrAccepted)
+        *pfIntrAccepted = fAccepted;
+
+    return rcStrict;
+}
+
+
+/**
+ * Checks if this APIC belongs to a logical destination.
+ *
+ * @returns true if the APIC belongs to the logical
+ *          destination, false otherwise.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   fDest                   The destination mask.
+ *
+ * @thread  Any.
+ */
+static bool apicIsLogicalDest(PVMCPUCC pVCpu, uint32_t fDest)
+{
+    if (XAPIC_IN_X2APIC_MODE(pVCpu))
+    {
+        /*
+         * Flat logical mode is not supported in x2APIC mode.
+         * In clustered logical mode, the 32-bit logical ID in the LDR is interpreted as follows:
+         *    - High 16 bits is the cluster ID.
+         *    - Low 16 bits: each bit represents a unique APIC within the cluster.
+         */
+        PCX2APICPAGE pX2ApicPage = VMCPU_TO_CX2APICPAGE(pVCpu);
+        uint32_t const u32Ldr    = pX2ApicPage->ldr.u32LogicalApicId;
+        if (X2APIC_LDR_GET_CLUSTER_ID(u32Ldr) == (fDest & X2APIC_LDR_CLUSTER_ID))
+            return RT_BOOL(u32Ldr & fDest & X2APIC_LDR_LOGICAL_ID);
+        return false;
+    }
+
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+    /*
+     * In both flat and clustered logical mode, a destination mask of all set bits indicates a broadcast.
+     * See AMD spec. 16.6.1 "Receiving System and IPI Interrupts".
+     */
+    Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+    if ((fDest & XAPIC_LDR_FLAT_LOGICAL_ID) == XAPIC_LDR_FLAT_LOGICAL_ID)
+        return true;
+
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    XAPICDESTFORMAT enmDestFormat = (XAPICDESTFORMAT)pXApicPage->dfr.u.u4Model;
+    if (enmDestFormat == XAPICDESTFORMAT_FLAT)
+    {
+        /* The destination mask is interpreted as a bitmap of 8 unique logical APIC IDs. */
+        uint8_t const u8Ldr = pXApicPage->ldr.u.u8LogicalApicId;
+        return RT_BOOL(u8Ldr & fDest & XAPIC_LDR_FLAT_LOGICAL_ID);
+    }
+
+    /*
+     * In clustered logical mode, the 8-bit logical ID in the LDR is interpreted as follows:
+     *    - High 4 bits is the cluster ID.
+     *    - Low 4 bits: each bit represents a unique APIC within the cluster.
+     */
+    Assert(enmDestFormat == XAPICDESTFORMAT_CLUSTER);
+    uint8_t const u8Ldr = pXApicPage->ldr.u.u8LogicalApicId;
+    if (XAPIC_LDR_CLUSTERED_GET_CLUSTER_ID(u8Ldr) == (fDest & XAPIC_LDR_CLUSTERED_CLUSTER_ID))
+        return RT_BOOL(u8Ldr & fDest & XAPIC_LDR_CLUSTERED_LOGICAL_ID);
+    return false;
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+}
+
+
+/**
+ * Figures out the set of destination CPUs for a given destination mode, format
+ * and delivery mode setting.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fDestMask       The destination mask.
+ * @param   fBroadcastMask  The broadcast mask.
+ * @param   enmDestMode     The destination mode.
+ * @param   enmDeliveryMode The delivery mode.
+ * @param   pDestCpuSet     The destination CPU set to update.
+ */
+static void apicGetDestCpuSet(PVMCC pVM, uint32_t fDestMask, uint32_t fBroadcastMask, XAPICDESTMODE enmDestMode,
+                              XAPICDELIVERYMODE enmDeliveryMode, PVMCPUSET pDestCpuSet)
+{
+    VMCPUSET_EMPTY(pDestCpuSet);
+
+    /*
+     * Physical destination mode only supports either a broadcast or a single target.
+     *    - Broadcast with lowest-priority delivery mode is not supported[1], we deliver it
+     *      as a regular broadcast like in fixed delivery mode.
+     *    - For a single target, lowest-priority delivery mode makes no sense. We deliver
+     *      to the target like in fixed delivery mode.
+     *
+     * [1] See Intel spec. 10.6.2.1 "Physical Destination Mode".
+     */
+    if (   enmDestMode == XAPICDESTMODE_PHYSICAL
+        && enmDeliveryMode == XAPICDELIVERYMODE_LOWEST_PRIO)
+    {
+        AssertMsgFailed(("APIC: Lowest-priority delivery using physical destination mode!"));
+        enmDeliveryMode = XAPICDELIVERYMODE_FIXED;
+    }
+
+    uint32_t const cCpus = pVM->cCpus;
+    if (enmDeliveryMode == XAPICDELIVERYMODE_LOWEST_PRIO)
+    {
+        Assert(enmDestMode == XAPICDESTMODE_LOGICAL);
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+        VMCPUID idCpuLowestTpr = NIL_VMCPUID;
+        uint8_t u8LowestTpr    = UINT8_C(0xff);
+        for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+        {
+            PVMCPUCC pVCpuDst = pVM->CTX_SUFF(apCpus)[idCpu];
+            if (apicIsLogicalDest(pVCpuDst, fDestMask))
+            {
+                PCXAPICPAGE   pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpuDst);
+                uint8_t const u8Tpr      = pXApicPage->tpr.u8Tpr;         /* PAV */
+
+                /*
+                 * If there is a tie for lowest priority, the local APIC with the highest ID is chosen.
+                 * Hence the use of "<=" in the check below.
+                 * See AMD spec. 16.6.2 "Lowest Priority Messages and Arbitration".
+                 */
+                if (u8Tpr <= u8LowestTpr)
+                {
+                    u8LowestTpr    = u8Tpr;
+                    idCpuLowestTpr = idCpu;
+                }
+            }
+        }
+        if (idCpuLowestTpr != NIL_VMCPUID)
+            VMCPUSET_ADD(pDestCpuSet, idCpuLowestTpr);
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+        return;
+    }
+
+    /*
+     * x2APIC:
+     *    - In both physical and logical destination mode, a destination mask of 0xffffffff implies a broadcast[1].
+     * xAPIC:
+     *    - In physical destination mode, a destination mask of 0xff implies a broadcast[2].
+     *    - In both flat and clustered logical mode, a destination mask of 0xff implies a broadcast[3].
+     *
+     * [1] See Intel spec. 10.12.9 "ICR Operation in x2APIC Mode".
+     * [2] See Intel spec. 10.6.2.1 "Physical Destination Mode".
+     * [2] See AMD spec. 16.6.1 "Receiving System and IPI Interrupts".
+     */
+    if ((fDestMask & fBroadcastMask) == fBroadcastMask)
+    {
+        VMCPUSET_FILL(pDestCpuSet);
+        return;
+    }
+
+    if (enmDestMode == XAPICDESTMODE_PHYSICAL)
+    {
+        /* The destination mask is interpreted as the physical APIC ID of a single target. */
+#if 1
+        /* Since our physical APIC ID is read-only to software, set the corresponding bit in the CPU set. */
+        if (RT_LIKELY(fDestMask < cCpus))
+            VMCPUSET_ADD(pDestCpuSet, fDestMask);
+#else
+        /* The physical APIC ID may not match our VCPU ID, search through the list of targets. */
+        for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+        {
+            PVMCPUCC pVCpuDst = &pVM->aCpus[idCpu];
+            if (XAPIC_IN_X2APIC_MODE(pVCpuDst))
+            {
+                PCX2APICPAGE pX2ApicPage = VMCPU_TO_CX2APICPAGE(pVCpuDst);
+                if (pX2ApicPage->id.u32ApicId == fDestMask)
+                    VMCPUSET_ADD(pDestCpuSet, pVCpuDst->idCpu);
+            }
+            else
+            {
+                PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpuDst);
+                if (pXApicPage->id.u8ApicId == (uint8_t)fDestMask)
+                    VMCPUSET_ADD(pDestCpuSet, pVCpuDst->idCpu);
+            }
+        }
+#endif
+    }
+    else
+    {
+        Assert(enmDestMode == XAPICDESTMODE_LOGICAL);
+
+        /* A destination mask of all 0's implies no target APICs (since it's interpreted as a bitmap or partial bitmap). */
+        if (RT_UNLIKELY(!fDestMask))
+            return;
+
+        /* The destination mask is interpreted as a bitmap of software-programmable logical APIC ID of the target APICs. */
+        for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+        {
+            PVMCPUCC pVCpuDst = pVM->CTX_SUFF(apCpus)[idCpu];
+            if (apicIsLogicalDest(pVCpuDst, fDestMask))
+                VMCPUSET_ADD(pDestCpuSet, pVCpuDst->idCpu);
+        }
+    }
+}
+
+
+/**
+ * Sends an Interprocessor Interrupt (IPI) using values from the Interrupt
+ * Command Register (ICR).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   rcRZ            The return code if the operation cannot be
+ *                          performed in the current context.
+ */
+DECLINLINE(VBOXSTRICTRC) apicSendIpi(PVMCPUCC pVCpu, int rcRZ)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    XAPICDELIVERYMODE const  enmDeliveryMode  = (XAPICDELIVERYMODE)pXApicPage->icr_lo.u.u3DeliveryMode;
+    XAPICDESTMODE const      enmDestMode      = (XAPICDESTMODE)pXApicPage->icr_lo.u.u1DestMode;
+    XAPICINITLEVEL const     enmInitLevel     = (XAPICINITLEVEL)pXApicPage->icr_lo.u.u1Level;
+    XAPICTRIGGERMODE const   enmTriggerMode   = (XAPICTRIGGERMODE)pXApicPage->icr_lo.u.u1TriggerMode;
+    XAPICDESTSHORTHAND const enmDestShorthand = (XAPICDESTSHORTHAND)pXApicPage->icr_lo.u.u2DestShorthand;
+    uint8_t const            uVector          = pXApicPage->icr_lo.u.u8Vector;
+
+    PX2APICPAGE pX2ApicPage = VMCPU_TO_X2APICPAGE(pVCpu);
+    uint32_t const fDest    = XAPIC_IN_X2APIC_MODE(pVCpu) ? pX2ApicPage->icr_hi.u32IcrHi : pXApicPage->icr_hi.u.u8Dest;
+
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+    /*
+     * INIT Level De-assert is not support on Pentium 4 and Xeon processors.
+     * Apparently, this also applies to NMI, SMI, lowest-priority and fixed delivery modes,
+     * see @bugref{8245#c116}.
+     *
+     * See AMD spec. 16.5 "Interprocessor Interrupts (IPI)" for a table of valid ICR combinations.
+     */
+    if (   enmTriggerMode  == XAPICTRIGGERMODE_LEVEL
+        && enmInitLevel    == XAPICINITLEVEL_DEASSERT
+        && (   enmDeliveryMode == XAPICDELIVERYMODE_FIXED
+            || enmDeliveryMode == XAPICDELIVERYMODE_LOWEST_PRIO
+            || enmDeliveryMode == XAPICDELIVERYMODE_SMI
+            || enmDeliveryMode == XAPICDELIVERYMODE_NMI
+            || enmDeliveryMode == XAPICDELIVERYMODE_INIT))
+    {
+        Log2(("APIC%u: %s level de-assert unsupported, ignoring!\n", pVCpu->idCpu, apicGetDeliveryModeName(enmDeliveryMode)));
+        return VINF_SUCCESS;
+    }
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+
+    /*
+     * The destination and delivery modes are ignored/by-passed when a destination shorthand is specified.
+     * See Intel spec. 10.6.2.3 "Broadcast/Self Delivery Mode".
+     */
+    VMCPUSET DestCpuSet;
+    switch (enmDestShorthand)
+    {
+        case XAPICDESTSHORTHAND_NONE:
+        {
+            PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+            uint32_t const fBroadcastMask = XAPIC_IN_X2APIC_MODE(pVCpu) ? X2APIC_ID_BROADCAST_MASK : XAPIC_ID_BROADCAST_MASK;
+            apicGetDestCpuSet(pVM, fDest, fBroadcastMask, enmDestMode, enmDeliveryMode, &DestCpuSet);
+            break;
+        }
+
+        case XAPICDESTSHORTHAND_SELF:
+        {
+            VMCPUSET_EMPTY(&DestCpuSet);
+            VMCPUSET_ADD(&DestCpuSet, pVCpu->idCpu);
+            break;
+        }
+
+        case XAPIDDESTSHORTHAND_ALL_INCL_SELF:
+        {
+            VMCPUSET_FILL(&DestCpuSet);
+            break;
+        }
+
+        case XAPICDESTSHORTHAND_ALL_EXCL_SELF:
+        {
+            VMCPUSET_FILL(&DestCpuSet);
+            VMCPUSET_DEL(&DestCpuSet, pVCpu->idCpu);
+            break;
+        }
+    }
+
+    return apicSendIntr(pVCpu->CTX_SUFF(pVM), pVCpu, uVector, enmTriggerMode, enmDeliveryMode, &DestCpuSet,
+                        NULL /* pfIntrAccepted */, 0 /* uSrcTag */, rcRZ);
+}
+
+
+/**
+ * Sets the Interrupt Command Register (ICR) high dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uIcrHi          The ICR high dword.
+ */
+static VBOXSTRICTRC apicSetIcrHi(PVMCPUCC pVCpu, uint32_t uIcrHi)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    pXApicPage->icr_hi.all.u32IcrHi = uIcrHi & XAPIC_ICR_HI_DEST;
+    STAM_COUNTER_INC(&pVCpu->apic.s.StatIcrHiWrite);
+    Log2(("APIC%u: apicSetIcrHi: uIcrHi=%#RX32\n", pVCpu->idCpu, pXApicPage->icr_hi.all.u32IcrHi));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the Interrupt Command Register (ICR) low dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uIcrLo          The ICR low dword.
+ * @param   rcRZ            The return code if the operation cannot be performed
+ *                          in the current context.
+ * @param   fUpdateStat     Whether to update the ICR low write statistics
+ *                          counter.
+ */
+static VBOXSTRICTRC apicSetIcrLo(PVMCPUCC pVCpu, uint32_t uIcrLo, int rcRZ, bool fUpdateStat)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PXAPICPAGE pXApicPage  = VMCPU_TO_XAPICPAGE(pVCpu);
+    pXApicPage->icr_lo.all.u32IcrLo = uIcrLo & XAPIC_ICR_LO_WR_VALID;
+    Log2(("APIC%u: apicSetIcrLo: uIcrLo=%#RX32\n", pVCpu->idCpu, pXApicPage->icr_lo.all.u32IcrLo));
+
+    if (fUpdateStat)
+        STAM_COUNTER_INC(&pVCpu->apic.s.StatIcrLoWrite);
+    RT_NOREF(fUpdateStat);
+
+    return apicSendIpi(pVCpu, rcRZ);
+}
+
+
+/**
+ * Sets the Interrupt Command Register (ICR).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u64Icr          The ICR (High and Low combined).
+ * @param   rcRZ            The return code if the operation cannot be performed
+ *                          in the current context.
+ *
+ * @remarks This function is used by both x2APIC interface and the Hyper-V
+ *          interface, see APICHvSetIcr. The Hyper-V spec isn't clear what
+ *          happens when invalid bits are set. For the time being, it will
+ *          \#GP like a regular x2APIC access.
+ */
+static VBOXSTRICTRC apicSetIcr(PVMCPUCC pVCpu, uint64_t u64Icr, int rcRZ)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /* Validate. */
+    uint32_t const uLo = RT_LO_U32(u64Icr);
+    if (RT_LIKELY(!(uLo & ~XAPIC_ICR_LO_WR_VALID)))
+    {
+        /* Update high dword first, then update the low dword which sends the IPI. */
+        PX2APICPAGE pX2ApicPage = VMCPU_TO_X2APICPAGE(pVCpu);
+        pX2ApicPage->icr_hi.u32IcrHi = RT_HI_U32(u64Icr);
+        STAM_COUNTER_INC(&pVCpu->apic.s.StatIcrFullWrite);
+        return apicSetIcrLo(pVCpu, uLo, rcRZ, false /* fUpdateStat */);
+    }
+    return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_ICR, APICMSRACCESS_WRITE_RSVD_BITS);
+}
+
+
+/**
+ * Sets the Error Status Register (ESR).
+ *
+ * @returns VINF_SUCCESS or VERR_CPUM_RAISE_GP_0.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uEsr            The ESR value.
+ */
+static int apicSetEsr(PVMCPUCC pVCpu, uint32_t uEsr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    Log2(("APIC%u: apicSetEsr: uEsr=%#RX32\n", pVCpu->idCpu, uEsr));
+
+    if (   XAPIC_IN_X2APIC_MODE(pVCpu)
+        && (uEsr & ~XAPIC_ESR_WO_VALID))
+        return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_ESR, APICMSRACCESS_WRITE_RSVD_BITS);
+
+    /*
+     * Writes to the ESR causes the internal state to be updated in the register,
+     * clearing the original state. See AMD spec. 16.4.6 "APIC Error Interrupts".
+     */
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    pXApicPage->esr.all.u32Errors = apicClearAllErrors(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Updates the Processor Priority Register (PPR).
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+static void apicUpdatePpr(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /* See Intel spec 10.8.3.1 "Task and Processor Priorities". */
+    PXAPICPAGE    pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    uint8_t const uIsrv      = apicGetHighestSetBitInReg(&pXApicPage->isr, 0 /* rcNotFound */);
+    uint8_t       uPpr;
+    if (XAPIC_TPR_GET_TP(pXApicPage->tpr.u8Tpr) >= XAPIC_PPR_GET_PP(uIsrv))
+        uPpr = pXApicPage->tpr.u8Tpr;
+    else
+        uPpr = XAPIC_PPR_GET_PP(uIsrv);
+    pXApicPage->ppr.u8Ppr = uPpr;
+}
+
+
+/**
+ * Gets the Processor Priority Register (PPR).
+ *
+ * @returns The PPR value.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+static uint8_t apicGetPpr(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    STAM_COUNTER_INC(&pVCpu->apic.s.StatTprRead);
+
+    /*
+     * With virtualized APIC registers or with TPR virtualization, the hardware may
+     * update ISR/TPR transparently. We thus re-calculate the PPR which may be out of sync.
+     * See Intel spec. 29.2.2 "Virtual-Interrupt Delivery".
+     *
+     * In all other instances, whenever the TPR or ISR changes, we need to update the PPR
+     * as well (e.g. like we do manually in apicR3InitIpi and by calling apicUpdatePpr).
+     */
+    PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    if (pApic->fVirtApicRegsEnabled)        /** @todo re-think this */
+        apicUpdatePpr(pVCpu);
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    return pXApicPage->ppr.u8Ppr;
+}
+
+
+/**
+ * Sets the Task Priority Register (TPR).
+ *
+ * @returns VINF_SUCCESS or VERR_CPUM_RAISE_GP_0.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   uTpr                    The TPR value.
+ * @param   fForceX2ApicBehaviour   Pretend the APIC is in x2APIC mode during
+ *                                  this write.
+ */
+static int apicSetTprEx(PVMCPUCC pVCpu, uint32_t uTpr, bool fForceX2ApicBehaviour)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    Log2(("APIC%u: apicSetTprEx: uTpr=%#RX32\n", pVCpu->idCpu, uTpr));
+    STAM_COUNTER_INC(&pVCpu->apic.s.StatTprWrite);
+
+    bool const fX2ApicMode = XAPIC_IN_X2APIC_MODE(pVCpu) || fForceX2ApicBehaviour;
+    if (   fX2ApicMode
+        && (uTpr & ~XAPIC_TPR_VALID))
+        return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_TPR, APICMSRACCESS_WRITE_RSVD_BITS);
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    pXApicPage->tpr.u8Tpr = uTpr;
+    apicUpdatePpr(pVCpu);
+    apicSignalNextPendingIntr(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the End-Of-Interrupt (EOI) register.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   uEoi                    The EOI value.
+ * @param   rcBusy                  The busy return code when the write cannot
+ *                                  be completed successfully in this context.
+ * @param   fForceX2ApicBehaviour   Pretend the APIC is in x2APIC mode during
+ *                                  this write.
+ */
+static VBOXSTRICTRC apicSetEoi(PVMCPUCC pVCpu, uint32_t uEoi, int rcBusy, bool fForceX2ApicBehaviour)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    Log2(("APIC%u: apicSetEoi: uEoi=%#RX32\n", pVCpu->idCpu, uEoi));
+    STAM_COUNTER_INC(&pVCpu->apic.s.StatEoiWrite);
+
+    bool const fX2ApicMode = XAPIC_IN_X2APIC_MODE(pVCpu) || fForceX2ApicBehaviour;
+    if (   fX2ApicMode
+        && (uEoi & ~XAPIC_EOI_WO_VALID))
+        return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_EOI, APICMSRACCESS_WRITE_RSVD_BITS);
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    int isrv = apicGetHighestSetBitInReg(&pXApicPage->isr, -1 /* rcNotFound */);
+    if (isrv >= 0)
+    {
+        /*
+         * Broadcast the EOI to the I/O APIC(s).
+         *
+         * We'll handle the EOI broadcast first as there is tiny chance we get rescheduled to
+         * ring-3 due to contention on the I/O APIC lock. This way we don't mess with the rest
+         * of the APIC state and simply restart the EOI write operation from ring-3.
+         */
+        Assert(isrv <= (int)UINT8_MAX);
+        uint8_t const uVector      = isrv;
+        bool const fLevelTriggered = apicTestVectorInReg(&pXApicPage->tmr, uVector);
+        if (fLevelTriggered)
+        {
+            VBOXSTRICTRC rc = PDMIoApicBroadcastEoi(pVCpu->CTX_SUFF(pVM), uVector);
+            if (rc == VINF_SUCCESS)
+            { /* likely */ }
+            else
+                return rcBusy;
+
+            /*
+             * Clear the vector from the TMR.
+             *
+             * The broadcast to I/O APIC can re-trigger new interrupts to arrive via the bus. However,
+             * APICUpdatePendingInterrupts() which updates TMR can only be done from EMT which we
+             * currently are on, so no possibility of concurrent updates.
+             */
+            apicClearVectorInReg(&pXApicPage->tmr, uVector);
+
+            /*
+             * Clear the remote IRR bit for level-triggered, fixed mode LINT0 interrupt.
+             * The LINT1 pin does not support level-triggered interrupts.
+             * See Intel spec. 10.5.1 "Local Vector Table".
+             */
+            uint32_t const uLvtLint0 = pXApicPage->lvt_lint0.all.u32LvtLint0;
+            if (   XAPIC_LVT_GET_REMOTE_IRR(uLvtLint0)
+                && XAPIC_LVT_GET_VECTOR(uLvtLint0) == uVector
+                && XAPIC_LVT_GET_DELIVERY_MODE(uLvtLint0) == XAPICDELIVERYMODE_FIXED)
+            {
+                ASMAtomicAndU32((volatile uint32_t *)&pXApicPage->lvt_lint0.all.u32LvtLint0, ~XAPIC_LVT_REMOTE_IRR);
+                Log2(("APIC%u: apicSetEoi: Cleared remote-IRR for LINT0. uVector=%#x\n", pVCpu->idCpu, uVector));
+            }
+
+            Log2(("APIC%u: apicSetEoi: Cleared level triggered interrupt from TMR. uVector=%#x\n", pVCpu->idCpu, uVector));
+        }
+
+        /*
+         * Mark interrupt as serviced, update the PPR and signal pending interrupts.
+         */
+        Log2(("APIC%u: apicSetEoi: Clearing interrupt from ISR. uVector=%#x\n", pVCpu->idCpu, uVector));
+        apicClearVectorInReg(&pXApicPage->isr, uVector);
+        apicUpdatePpr(pVCpu);
+        apicSignalNextPendingIntr(pVCpu);
+    }
+    else
+    {
+#ifdef DEBUG_ramshankar
+        /** @todo Figure out if this is done intentionally by guests or is a bug
+         *   in our emulation. Happened with Win10 SMP VM during reboot after
+         *   installation of guest additions with 3D support. */
+        AssertMsgFailed(("APIC%u: apicSetEoi: Failed to find any ISR bit\n", pVCpu->idCpu));
+#endif
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the Logical Destination Register (LDR).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uLdr            The LDR value.
+ *
+ * @remarks LDR is read-only in x2APIC mode.
+ */
+static VBOXSTRICTRC apicSetLdr(PVMCPUCC pVCpu, uint32_t uLdr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    Assert(!XAPIC_IN_X2APIC_MODE(pVCpu) || pApic->fHyperVCompatMode); RT_NOREF_PV(pApic);
+
+    Log2(("APIC%u: apicSetLdr: uLdr=%#RX32\n", pVCpu->idCpu, uLdr));
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    apicWriteRaw32(pXApicPage, XAPIC_OFF_LDR, uLdr & XAPIC_LDR_VALID);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the Destination Format Register (DFR).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uDfr            The DFR value.
+ *
+ * @remarks DFR is not available in x2APIC mode.
+ */
+static VBOXSTRICTRC apicSetDfr(PVMCPUCC pVCpu, uint32_t uDfr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+
+    uDfr &= XAPIC_DFR_VALID;
+    uDfr |= XAPIC_DFR_RSVD_MB1;
+
+    Log2(("APIC%u: apicSetDfr: uDfr=%#RX32\n", pVCpu->idCpu, uDfr));
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    apicWriteRaw32(pXApicPage, XAPIC_OFF_DFR, uDfr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the Timer Divide Configuration Register (DCR).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uTimerDcr       The timer DCR value.
+ */
+static VBOXSTRICTRC apicSetTimerDcr(PVMCPUCC pVCpu, uint32_t uTimerDcr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    if (   XAPIC_IN_X2APIC_MODE(pVCpu)
+        && (uTimerDcr & ~XAPIC_TIMER_DCR_VALID))
+        return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_TIMER_DCR, APICMSRACCESS_WRITE_RSVD_BITS);
+
+    Log2(("APIC%u: apicSetTimerDcr: uTimerDcr=%#RX32\n", pVCpu->idCpu, uTimerDcr));
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    apicWriteRaw32(pXApicPage, XAPIC_OFF_TIMER_DCR, uTimerDcr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the timer's Current Count Register (CCR).
+ *
+ * @returns VBox status code.
+ * @param   pDevIns         The device instance.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   rcBusy          The busy return code for the timer critical section.
+ * @param   puValue         Where to store the LVT timer CCR.
+ */
+static VBOXSTRICTRC apicGetTimerCcr(PPDMDEVINS pDevIns, PVMCPUCC pVCpu, int rcBusy, uint32_t *puValue)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(puValue);
+
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    *puValue = 0;
+
+    /* In TSC-deadline mode, CCR returns 0, see Intel spec. 10.5.4.1 "TSC-Deadline Mode". */
+    if (pXApicPage->lvt_timer.u.u2TimerMode == XAPIC_TIMER_MODE_TSC_DEADLINE)
+        return VINF_SUCCESS;
+
+    /* If the initial-count register is 0, CCR returns 0 as it cannot exceed the ICR. */
+    uint32_t const uInitialCount = pXApicPage->timer_icr.u32InitialCount;
+    if (!uInitialCount)
+        return VINF_SUCCESS;
+
+    /*
+     * Reading the virtual-sync clock requires locking its timer because it's not
+     * a simple atomic operation, see tmVirtualSyncGetEx().
+     *
+     * We also need to lock before reading the timer CCR, see apicR3TimerCallback().
+     */
+    PCAPICCPU       pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    TMTIMERHANDLE   hTimer   = pApicCpu->hTimer;
+
+    VBOXSTRICTRC rc = PDMDevHlpTimerLockClock(pDevIns, hTimer, rcBusy);
+    if (rc == VINF_SUCCESS)
+    {
+        /* If the current-count register is 0, it implies the timer expired. */
+        uint32_t const uCurrentCount = pXApicPage->timer_ccr.u32CurrentCount;
+        if (uCurrentCount)
+        {
+            uint64_t const cTicksElapsed = PDMDevHlpTimerGet(pDevIns, hTimer) - pApicCpu->u64TimerInitial;
+            PDMDevHlpTimerUnlockClock(pDevIns, hTimer);
+            uint8_t  const uTimerShift   = apicGetTimerShift(pXApicPage);
+            uint64_t const uDelta        = cTicksElapsed >> uTimerShift;
+            if (uInitialCount > uDelta)
+                *puValue = uInitialCount - uDelta;
+        }
+        else
+            PDMDevHlpTimerUnlockClock(pDevIns, hTimer);
+    }
+    return rc;
+}
+
+
+/**
+ * Sets the timer's Initial-Count Register (ICR).
+ *
+ * @returns Strict VBox status code.
+ * @param   pDevIns         The device instance.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   rcBusy          The busy return code for the timer critical section.
+ * @param   uInitialCount   The timer ICR.
+ */
+static VBOXSTRICTRC apicSetTimerIcr(PPDMDEVINS pDevIns, PVMCPUCC pVCpu, int rcBusy, uint32_t uInitialCount)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PAPIC      pApic      = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    PAPICCPU   pApicCpu   = VMCPU_TO_APICCPU(pVCpu);
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+
+    Log2(("APIC%u: apicSetTimerIcr: uInitialCount=%#RX32\n", pVCpu->idCpu, uInitialCount));
+    STAM_COUNTER_INC(&pApicCpu->StatTimerIcrWrite);
+
+    /* In TSC-deadline mode, timer ICR writes are ignored, see Intel spec. 10.5.4.1 "TSC-Deadline Mode". */
+    if (   pApic->fSupportsTscDeadline
+        && pXApicPage->lvt_timer.u.u2TimerMode == XAPIC_TIMER_MODE_TSC_DEADLINE)
+        return VINF_SUCCESS;
+
+    /*
+     * The timer CCR may be modified by apicR3TimerCallback() in parallel,
+     * so obtain the lock -before- updating it here to be consistent with the
+     * timer ICR. We rely on CCR being consistent in apicGetTimerCcr().
+     */
+    TMTIMERHANDLE hTimer = pApicCpu->hTimer;
+    VBOXSTRICTRC rc = PDMDevHlpTimerLockClock(pDevIns, hTimer, rcBusy);
+    if (rc == VINF_SUCCESS)
+    {
+        pXApicPage->timer_icr.u32InitialCount = uInitialCount;
+        pXApicPage->timer_ccr.u32CurrentCount = uInitialCount;
+        if (uInitialCount)
+            apicStartTimer(pVCpu, uInitialCount);
+        else
+            apicStopTimer(pVCpu);
+        PDMDevHlpTimerUnlockClock(pDevIns, hTimer);
+    }
+    return rc;
+}
+
+
+/**
+ * Sets an LVT entry.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   offLvt          The LVT entry offset in the xAPIC page.
+ * @param   uLvt            The LVT value to set.
+ */
+static VBOXSTRICTRC apicSetLvtEntry(PVMCPUCC pVCpu, uint16_t offLvt, uint32_t uLvt)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+    AssertMsg(   offLvt == XAPIC_OFF_LVT_TIMER
+              || offLvt == XAPIC_OFF_LVT_THERMAL
+              || offLvt == XAPIC_OFF_LVT_PERF
+              || offLvt == XAPIC_OFF_LVT_LINT0
+              || offLvt == XAPIC_OFF_LVT_LINT1
+              || offLvt == XAPIC_OFF_LVT_ERROR,
+             ("APIC%u: apicSetLvtEntry: invalid offset, offLvt=%#RX16, uLvt=%#RX32\n", pVCpu->idCpu, offLvt, uLvt));
+
+    /*
+     * If TSC-deadline mode isn't support, ignore the bit in xAPIC mode
+     * and raise #GP(0) in x2APIC mode.
+     */
+    PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    if (offLvt == XAPIC_OFF_LVT_TIMER)
+    {
+        if (   !pApic->fSupportsTscDeadline
+            && (uLvt & XAPIC_LVT_TIMER_TSCDEADLINE))
+        {
+            if (XAPIC_IN_X2APIC_MODE(pVCpu))
+                return apicMsrAccessError(pVCpu, XAPIC_GET_X2APIC_MSR(offLvt), APICMSRACCESS_WRITE_RSVD_BITS);
+            uLvt &= ~XAPIC_LVT_TIMER_TSCDEADLINE;
+            /** @todo TSC-deadline timer mode transition */
+        }
+    }
+
+    /*
+     * Validate rest of the LVT bits.
+     */
+    uint16_t const idxLvt = (offLvt - XAPIC_OFF_LVT_START) >> 4;
+    AssertReturn(idxLvt < RT_ELEMENTS(g_au32LvtValidMasks), VERR_OUT_OF_RANGE);
+
+    /*
+     * For x2APIC, disallow setting of invalid/reserved bits.
+     * For xAPIC, mask out invalid/reserved bits (i.e. ignore them).
+     */
+    if (   XAPIC_IN_X2APIC_MODE(pVCpu)
+        && (uLvt & ~g_au32LvtValidMasks[idxLvt]))
+        return apicMsrAccessError(pVCpu, XAPIC_GET_X2APIC_MSR(offLvt), APICMSRACCESS_WRITE_RSVD_BITS);
+
+    uLvt &= g_au32LvtValidMasks[idxLvt];
+
+    /*
+     * In the software-disabled state, LVT mask-bit must remain set and attempts to clear the mask
+     * bit must be ignored. See Intel spec. 10.4.7.2 "Local APIC State After It Has Been Software Disabled".
+     */
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    if (!pXApicPage->svr.u.fApicSoftwareEnable)
+        uLvt |= XAPIC_LVT_MASK;
+
+    /*
+     * It is unclear whether we should signal a 'send illegal vector' error here and ignore updating
+     * the LVT entry when the delivery mode is 'fixed'[1] or update it in addition to signalling the
+     * error or not signal the error at all. For now, we'll allow setting illegal vectors into the LVT
+     * but set the 'send illegal vector' error here. The 'receive illegal vector' error will be set if
+     * the interrupt for the vector happens to be generated, see apicPostInterrupt().
+     *
+     * [1] See Intel spec. 10.5.2 "Valid Interrupt Vectors".
+     */
+    if (RT_UNLIKELY(   XAPIC_LVT_GET_VECTOR(uLvt) <= XAPIC_ILLEGAL_VECTOR_END
+                    && XAPIC_LVT_GET_DELIVERY_MODE(uLvt) == XAPICDELIVERYMODE_FIXED))
+        apicSetError(pVCpu, XAPIC_ESR_SEND_ILLEGAL_VECTOR);
+
+    Log2(("APIC%u: apicSetLvtEntry: offLvt=%#RX16 uLvt=%#RX32\n", pVCpu->idCpu, offLvt, uLvt));
+
+    apicWriteRaw32(pXApicPage, offLvt, uLvt);
+    return VINF_SUCCESS;
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif  /* XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 */
+}
+
+
+#if 0
+/**
+ * Sets an LVT entry in the extended LVT range.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   offLvt          The LVT entry offset in the xAPIC page.
+ * @param   uValue          The LVT value to set.
+ */
+static int apicSetLvtExtEntry(PVMCPUCC pVCpu, uint16_t offLvt, uint32_t uLvt)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    AssertMsg(offLvt == XAPIC_OFF_CMCI, ("APIC%u: apicSetLvt1Entry: invalid offset %#RX16\n", pVCpu->idCpu, offLvt));
+
+    /** @todo support CMCI. */
+    return VERR_NOT_IMPLEMENTED;
+}
+#endif
+
+
+/**
+ * Hints TM about the APIC timer frequency.
+ *
+ * @param   pDevIns         The device instance.
+ * @param   pApicCpu        The APIC CPU state.
+ * @param   uInitialCount   The new initial count.
+ * @param   uTimerShift     The new timer shift.
+ * @thread  Any.
+ */
+void apicHintTimerFreq(PPDMDEVINS pDevIns, PAPICCPU pApicCpu, uint32_t uInitialCount, uint8_t uTimerShift)
+{
+    Assert(pApicCpu);
+
+    if (   pApicCpu->uHintedTimerInitialCount != uInitialCount
+        || pApicCpu->uHintedTimerShift        != uTimerShift)
+    {
+        uint32_t uHz;
+        if (uInitialCount)
+        {
+            uint64_t cTicksPerPeriod = (uint64_t)uInitialCount << uTimerShift;
+            uHz = PDMDevHlpTimerGetFreq(pDevIns, pApicCpu->hTimer) / cTicksPerPeriod;
+        }
+        else
+            uHz = 0;
+
+        PDMDevHlpTimerSetFrequencyHint(pDevIns, pApicCpu->hTimer, uHz);
+        pApicCpu->uHintedTimerInitialCount = uInitialCount;
+        pApicCpu->uHintedTimerShift = uTimerShift;
+    }
+}
+
+
+/**
+ * Gets the Interrupt Command Register (ICR), without performing any interface
+ * checks.
+ *
+ * @returns The ICR value.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+DECLINLINE(uint64_t) apicGetIcrNoCheck(PVMCPUCC pVCpu)
+{
+    PCX2APICPAGE pX2ApicPage = VMCPU_TO_CX2APICPAGE(pVCpu);
+    uint64_t const uHi  = pX2ApicPage->icr_hi.u32IcrHi;
+    uint64_t const uLo  = pX2ApicPage->icr_lo.all.u32IcrLo;
+    uint64_t const uIcr = RT_MAKE_U64(uLo, uHi);
+    return uIcr;
+}
+
+
+/**
+ * Reads an APIC register.
+ *
+ * @returns VBox status code.
+ * @param   pDevIns         The device instance.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   offReg          The offset of the register being read.
+ * @param   puValue         Where to store the register value.
+ */
+DECLINLINE(VBOXSTRICTRC) apicReadRegister(PPDMDEVINS pDevIns, PVMCPUCC pVCpu, uint16_t offReg, uint32_t *puValue)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(offReg <= XAPIC_OFF_MAX_VALID);
+
+    PXAPICPAGE   pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    uint32_t     uValue = 0;
+    VBOXSTRICTRC rc = VINF_SUCCESS;
+    switch (offReg)
+    {
+        case XAPIC_OFF_ID:
+        case XAPIC_OFF_VERSION:
+        case XAPIC_OFF_TPR:
+        case XAPIC_OFF_EOI:
+        case XAPIC_OFF_RRD:
+        case XAPIC_OFF_LDR:
+        case XAPIC_OFF_DFR:
+        case XAPIC_OFF_SVR:
+        case XAPIC_OFF_ISR0:    case XAPIC_OFF_ISR1:    case XAPIC_OFF_ISR2:    case XAPIC_OFF_ISR3:
+        case XAPIC_OFF_ISR4:    case XAPIC_OFF_ISR5:    case XAPIC_OFF_ISR6:    case XAPIC_OFF_ISR7:
+        case XAPIC_OFF_TMR0:    case XAPIC_OFF_TMR1:    case XAPIC_OFF_TMR2:    case XAPIC_OFF_TMR3:
+        case XAPIC_OFF_TMR4:    case XAPIC_OFF_TMR5:    case XAPIC_OFF_TMR6:    case XAPIC_OFF_TMR7:
+        case XAPIC_OFF_IRR0:    case XAPIC_OFF_IRR1:    case XAPIC_OFF_IRR2:    case XAPIC_OFF_IRR3:
+        case XAPIC_OFF_IRR4:    case XAPIC_OFF_IRR5:    case XAPIC_OFF_IRR6:    case XAPIC_OFF_IRR7:
+        case XAPIC_OFF_ESR:
+        case XAPIC_OFF_ICR_LO:
+        case XAPIC_OFF_ICR_HI:
+        case XAPIC_OFF_LVT_TIMER:
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+        case XAPIC_OFF_LVT_THERMAL:
+#endif
+        case XAPIC_OFF_LVT_PERF:
+        case XAPIC_OFF_LVT_LINT0:
+        case XAPIC_OFF_LVT_LINT1:
+        case XAPIC_OFF_LVT_ERROR:
+        case XAPIC_OFF_TIMER_ICR:
+        case XAPIC_OFF_TIMER_DCR:
+        {
+            Assert(   !XAPIC_IN_X2APIC_MODE(pVCpu)
+                   || (   offReg != XAPIC_OFF_DFR
+                       && offReg != XAPIC_OFF_ICR_HI
+                       && offReg != XAPIC_OFF_EOI));
+            uValue = apicReadRaw32(pXApicPage, offReg);
+            Log2(("APIC%u: apicReadRegister: offReg=%#x uValue=%#x\n", pVCpu->idCpu, offReg, uValue));
+            break;
+        }
+
+        case XAPIC_OFF_PPR:
+        {
+            uValue = apicGetPpr(pVCpu);
+            break;
+        }
+
+        case XAPIC_OFF_TIMER_CCR:
+        {
+            Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+            rc = apicGetTimerCcr(pDevIns, pVCpu, VINF_IOM_R3_MMIO_READ, &uValue);
+            break;
+        }
+
+        case XAPIC_OFF_APR:
+        {
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+            /* Unsupported on Pentium 4 and Xeon CPUs, invalid in x2APIC mode. */
+            Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+            break;
+        }
+
+        default:
+        {
+            Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+            rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS, "VCPU[%u]: offReg=%#RX16\n", pVCpu->idCpu, offReg);
+            apicSetError(pVCpu, XAPIC_ESR_ILLEGAL_REG_ADDRESS);
+            break;
+        }
+    }
+
+    *puValue = uValue;
+    return rc;
+}
+
+
+/**
+ * Writes an APIC register.
+ *
+ * @returns Strict VBox status code.
+ * @param   pDevIns         The device instance.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   offReg          The offset of the register being written.
+ * @param   uValue          The register value.
+ */
+DECLINLINE(VBOXSTRICTRC) apicWriteRegister(PPDMDEVINS pDevIns, PVMCPUCC pVCpu, uint16_t offReg, uint32_t uValue)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(offReg <= XAPIC_OFF_MAX_VALID);
+    Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    switch (offReg)
+    {
+        case XAPIC_OFF_TPR:
+        {
+            rcStrict = apicSetTprEx(pVCpu, uValue, false /* fForceX2ApicBehaviour */);
+            break;
+        }
+
+        case XAPIC_OFF_LVT_TIMER:
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+        case XAPIC_OFF_LVT_THERMAL:
+#endif
+        case XAPIC_OFF_LVT_PERF:
+        case XAPIC_OFF_LVT_LINT0:
+        case XAPIC_OFF_LVT_LINT1:
+        case XAPIC_OFF_LVT_ERROR:
+        {
+            rcStrict = apicSetLvtEntry(pVCpu, offReg, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_TIMER_ICR:
+        {
+            rcStrict = apicSetTimerIcr(pDevIns, pVCpu, VINF_IOM_R3_MMIO_WRITE, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_EOI:
+        {
+            rcStrict = apicSetEoi(pVCpu, uValue, VINF_IOM_R3_MMIO_WRITE, false /* fForceX2ApicBehaviour */);
+            break;
+        }
+
+        case XAPIC_OFF_LDR:
+        {
+            rcStrict = apicSetLdr(pVCpu, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_DFR:
+        {
+            rcStrict = apicSetDfr(pVCpu, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_SVR:
+        {
+            rcStrict = apicSetSvr(pVCpu, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_ICR_LO:
+        {
+            rcStrict = apicSetIcrLo(pVCpu, uValue, VINF_IOM_R3_MMIO_WRITE, true /* fUpdateStat */);
+            break;
+        }
+
+        case XAPIC_OFF_ICR_HI:
+        {
+            rcStrict = apicSetIcrHi(pVCpu, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_TIMER_DCR:
+        {
+            rcStrict = apicSetTimerDcr(pVCpu, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_ESR:
+        {
+            rcStrict = apicSetEsr(pVCpu, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_APR:
+        case XAPIC_OFF_RRD:
+        {
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+            /* Unsupported on Pentium 4 and Xeon CPUs but writes do -not- set an illegal register access error. */
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+            break;
+        }
+
+        /* Read-only, write ignored: */
+        case XAPIC_OFF_VERSION:
+        case XAPIC_OFF_ID:
+            break;
+
+        /* Unavailable/reserved in xAPIC mode: */
+        case X2APIC_OFF_SELF_IPI:
+        /* Read-only registers: */
+        case XAPIC_OFF_PPR:
+        case XAPIC_OFF_ISR0:    case XAPIC_OFF_ISR1:    case XAPIC_OFF_ISR2:    case XAPIC_OFF_ISR3:
+        case XAPIC_OFF_ISR4:    case XAPIC_OFF_ISR5:    case XAPIC_OFF_ISR6:    case XAPIC_OFF_ISR7:
+        case XAPIC_OFF_TMR0:    case XAPIC_OFF_TMR1:    case XAPIC_OFF_TMR2:    case XAPIC_OFF_TMR3:
+        case XAPIC_OFF_TMR4:    case XAPIC_OFF_TMR5:    case XAPIC_OFF_TMR6:    case XAPIC_OFF_TMR7:
+        case XAPIC_OFF_IRR0:    case XAPIC_OFF_IRR1:    case XAPIC_OFF_IRR2:    case XAPIC_OFF_IRR3:
+        case XAPIC_OFF_IRR4:    case XAPIC_OFF_IRR5:    case XAPIC_OFF_IRR6:    case XAPIC_OFF_IRR7:
+        case XAPIC_OFF_TIMER_CCR:
+        default:
+        {
+            rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS, "APIC%u: offReg=%#RX16\n", pVCpu->idCpu, offReg);
+            apicSetError(pVCpu, XAPIC_ESR_ILLEGAL_REG_ADDRESS);
+            break;
+        }
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Reads an APIC MSR.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u32Reg          The MSR being read.
+ * @param   pu64Value       Where to store the read value.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICReadMsr(PVMCPUCC pVCpu, uint32_t u32Reg, uint64_t *pu64Value)
+{
+    /*
+     * Validate.
+     */
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(u32Reg >= MSR_IA32_X2APIC_ID && u32Reg <= MSR_IA32_X2APIC_SELF_IPI);
+    Assert(pu64Value);
+
+    /*
+     * Is the APIC enabled?
+     */
+    PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    if (APICIsEnabled(pVCpu))
+    { /* likely */ }
+    else
+        return apicMsrAccessError(pVCpu, u32Reg, pApic->enmMaxMode == PDMAPICMODE_NONE ?
+                                                 APICMSRACCESS_READ_DISALLOWED_CONFIG : APICMSRACCESS_READ_RSVD_OR_UNKNOWN);
+
+#ifndef IN_RING3
+    if (pApic->CTXALLMID(f,Enabled))
+    { /* likely */}
+    else
+        return VINF_CPUM_R3_MSR_READ;
+#endif
+
+    STAM_COUNTER_INC(&pVCpu->apic.s.CTX_SUFF_Z(StatMsrRead));
+
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    if (RT_LIKELY(   XAPIC_IN_X2APIC_MODE(pVCpu)
+                  || pApic->fHyperVCompatMode))
+    {
+        switch (u32Reg)
+        {
+            /* Special handling for x2APIC: */
+            case MSR_IA32_X2APIC_ICR:
+            {
+                *pu64Value = apicGetIcrNoCheck(pVCpu);
+                break;
+            }
+
+            /* Special handling, compatible with xAPIC: */
+            case MSR_IA32_X2APIC_TIMER_CCR:
+            {
+                uint32_t uValue;
+                rcStrict = apicGetTimerCcr(VMCPU_TO_DEVINS(pVCpu), pVCpu, VINF_CPUM_R3_MSR_READ, &uValue);
+                *pu64Value = uValue;
+                break;
+            }
+
+            /* Special handling, compatible with xAPIC: */
+            case MSR_IA32_X2APIC_PPR:
+            {
+                *pu64Value = apicGetPpr(pVCpu);
+                break;
+            }
+
+            /* Raw read, compatible with xAPIC: */
+            case MSR_IA32_X2APIC_ID:
+            {
+                STAM_COUNTER_INC(&pVCpu->apic.s.StatIdMsrRead);
+                RT_FALL_THRU();
+            }
+            case MSR_IA32_X2APIC_VERSION:
+            case MSR_IA32_X2APIC_TPR:
+            case MSR_IA32_X2APIC_LDR:
+            case MSR_IA32_X2APIC_SVR:
+            case MSR_IA32_X2APIC_ISR0:  case MSR_IA32_X2APIC_ISR1:  case MSR_IA32_X2APIC_ISR2:  case MSR_IA32_X2APIC_ISR3:
+            case MSR_IA32_X2APIC_ISR4:  case MSR_IA32_X2APIC_ISR5:  case MSR_IA32_X2APIC_ISR6:  case MSR_IA32_X2APIC_ISR7:
+            case MSR_IA32_X2APIC_TMR0:  case MSR_IA32_X2APIC_TMR1:  case MSR_IA32_X2APIC_TMR2:  case MSR_IA32_X2APIC_TMR3:
+            case MSR_IA32_X2APIC_TMR4:  case MSR_IA32_X2APIC_TMR5:  case MSR_IA32_X2APIC_TMR6:  case MSR_IA32_X2APIC_TMR7:
+            case MSR_IA32_X2APIC_IRR0:  case MSR_IA32_X2APIC_IRR1:  case MSR_IA32_X2APIC_IRR2:  case MSR_IA32_X2APIC_IRR3:
+            case MSR_IA32_X2APIC_IRR4:  case MSR_IA32_X2APIC_IRR5:  case MSR_IA32_X2APIC_IRR6:  case MSR_IA32_X2APIC_IRR7:
+            case MSR_IA32_X2APIC_ESR:
+            case MSR_IA32_X2APIC_LVT_TIMER:
+            case MSR_IA32_X2APIC_LVT_THERMAL:
+            case MSR_IA32_X2APIC_LVT_PERF:
+            case MSR_IA32_X2APIC_LVT_LINT0:
+            case MSR_IA32_X2APIC_LVT_LINT1:
+            case MSR_IA32_X2APIC_LVT_ERROR:
+            case MSR_IA32_X2APIC_TIMER_ICR:
+            case MSR_IA32_X2APIC_TIMER_DCR:
+            {
+                PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+                uint16_t const offReg = X2APIC_GET_XAPIC_OFF(u32Reg);
+                *pu64Value = apicReadRaw32(pXApicPage, offReg);
+                break;
+            }
+
+            /* Write-only MSRs: */
+            case MSR_IA32_X2APIC_SELF_IPI:
+            case MSR_IA32_X2APIC_EOI:
+            {
+                rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_READ_WRITE_ONLY);
+                break;
+            }
+
+            /*
+             * Windows guest using Hyper-V x2APIC MSR compatibility mode tries to read the "high"
+             * LDR bits, which is quite absurd (as it's a 32-bit register) using this invalid MSR
+             * index (0x80E), see @bugref{8382#c175}.
+             */
+            case MSR_IA32_X2APIC_LDR + 1:
+            {
+                if (pApic->fHyperVCompatMode)
+                    *pu64Value = 0;
+                else
+                    rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_READ_RSVD_OR_UNKNOWN);
+                break;
+            }
+
+            /* Reserved MSRs: */
+            case MSR_IA32_X2APIC_LVT_CMCI:
+            default:
+            {
+                rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_READ_RSVD_OR_UNKNOWN);
+                break;
+            }
+        }
+    }
+    else
+        rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_INVALID_READ_MODE);
+
+    return rcStrict;
+}
+
+
+/**
+ * Writes an APIC MSR.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u32Reg          The MSR being written.
+ * @param   u64Value        The value to write.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICWriteMsr(PVMCPUCC pVCpu, uint32_t u32Reg, uint64_t u64Value)
+{
+    /*
+     * Validate.
+     */
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(u32Reg >= MSR_IA32_X2APIC_ID && u32Reg <= MSR_IA32_X2APIC_SELF_IPI);
+
+    /*
+     * Is the APIC enabled?
+     */
+    PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    if (APICIsEnabled(pVCpu))
+    { /* likely */ }
+    else
+        return apicMsrAccessError(pVCpu, u32Reg, pApic->enmMaxMode == PDMAPICMODE_NONE ?
+                                  APICMSRACCESS_WRITE_DISALLOWED_CONFIG : APICMSRACCESS_WRITE_RSVD_OR_UNKNOWN);
+
+#ifndef IN_RING3
+    if (pApic->CTXALLMID(f,Enabled))
+    { /* likely */ }
+    else
+        return VINF_CPUM_R3_MSR_WRITE;
+#endif
+
+    STAM_COUNTER_INC(&pVCpu->apic.s.CTX_SUFF_Z(StatMsrWrite));
+
+    /*
+     * In x2APIC mode, we need to raise #GP(0) for writes to reserved bits, unlike MMIO
+     * accesses where they are ignored. Hence, we need to validate each register before
+     * invoking the generic/xAPIC write functions.
+     *
+     * Bits 63:32 of all registers except the ICR are reserved, we'll handle this common
+     * case first and handle validating the remaining bits on a per-register basis.
+     * See Intel spec. 10.12.1.2 "x2APIC Register Address Space".
+     */
+    if (   u32Reg != MSR_IA32_X2APIC_ICR
+        && RT_HI_U32(u64Value))
+        return apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_WRITE_RSVD_BITS);
+
+    uint32_t     u32Value = RT_LO_U32(u64Value);
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    if (RT_LIKELY(   XAPIC_IN_X2APIC_MODE(pVCpu)
+                  || pApic->fHyperVCompatMode))
+    {
+        switch (u32Reg)
+        {
+            case MSR_IA32_X2APIC_TPR:
+            {
+                rcStrict = apicSetTprEx(pVCpu, u32Value, false /* fForceX2ApicBehaviour */);
+                break;
+            }
+
+            case MSR_IA32_X2APIC_ICR:
+            {
+                rcStrict = apicSetIcr(pVCpu, u64Value, VINF_CPUM_R3_MSR_WRITE);
+                break;
+            }
+
+            case MSR_IA32_X2APIC_SVR:
+            {
+                rcStrict = apicSetSvr(pVCpu, u32Value);
+                break;
+            }
+
+            case MSR_IA32_X2APIC_ESR:
+            {
+                rcStrict = apicSetEsr(pVCpu, u32Value);
+                break;
+            }
+
+            case MSR_IA32_X2APIC_TIMER_DCR:
+            {
+                rcStrict = apicSetTimerDcr(pVCpu, u32Value);
+                break;
+            }
+
+            case MSR_IA32_X2APIC_LVT_TIMER:
+            case MSR_IA32_X2APIC_LVT_THERMAL:
+            case MSR_IA32_X2APIC_LVT_PERF:
+            case MSR_IA32_X2APIC_LVT_LINT0:
+            case MSR_IA32_X2APIC_LVT_LINT1:
+            case MSR_IA32_X2APIC_LVT_ERROR:
+            {
+                rcStrict = apicSetLvtEntry(pVCpu, X2APIC_GET_XAPIC_OFF(u32Reg), u32Value);
+                break;
+            }
+
+            case MSR_IA32_X2APIC_TIMER_ICR:
+            {
+                rcStrict = apicSetTimerIcr(VMCPU_TO_DEVINS(pVCpu), pVCpu, VINF_CPUM_R3_MSR_WRITE, u32Value);
+                break;
+            }
+
+            /* Write-only MSRs: */
+            case MSR_IA32_X2APIC_SELF_IPI:
+            {
+                uint8_t const uVector = XAPIC_SELF_IPI_GET_VECTOR(u32Value);
+                apicPostInterrupt(pVCpu, uVector, XAPICTRIGGERMODE_EDGE, 0 /* uSrcTag */);
+                rcStrict = VINF_SUCCESS;
+                break;
+            }
+
+            case MSR_IA32_X2APIC_EOI:
+            {
+                rcStrict = apicSetEoi(pVCpu, u32Value, VINF_CPUM_R3_MSR_WRITE, false /* fForceX2ApicBehaviour */);
+                break;
+            }
+
+            /*
+             * Windows guest using Hyper-V x2APIC MSR compatibility mode tries to write the "high"
+             * LDR bits, which is quite absurd (as it's a 32-bit register) using this invalid MSR
+             * index (0x80E). The write value was 0xffffffff on a Windows 8.1 64-bit guest. We can
+             * safely ignore this nonsense, See @bugref{8382#c7}.
+             */
+            case MSR_IA32_X2APIC_LDR + 1:
+            {
+                if (pApic->fHyperVCompatMode)
+                    rcStrict = VINF_SUCCESS;
+                else
+                    rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_WRITE_RSVD_OR_UNKNOWN);
+                break;
+            }
+
+            /* Special-treament (read-only normally, but not with Hyper-V) */
+            case MSR_IA32_X2APIC_LDR:
+            {
+                if (pApic->fHyperVCompatMode)
+                {
+                    rcStrict = apicSetLdr(pVCpu, u32Value);
+                    break;
+                }
+            }
+            RT_FALL_THRU();
+            /* Read-only MSRs: */
+            case MSR_IA32_X2APIC_ID:
+            case MSR_IA32_X2APIC_VERSION:
+            case MSR_IA32_X2APIC_PPR:
+            case MSR_IA32_X2APIC_ISR0:  case MSR_IA32_X2APIC_ISR1:  case MSR_IA32_X2APIC_ISR2:  case MSR_IA32_X2APIC_ISR3:
+            case MSR_IA32_X2APIC_ISR4:  case MSR_IA32_X2APIC_ISR5:  case MSR_IA32_X2APIC_ISR6:  case MSR_IA32_X2APIC_ISR7:
+            case MSR_IA32_X2APIC_TMR0:  case MSR_IA32_X2APIC_TMR1:  case MSR_IA32_X2APIC_TMR2:  case MSR_IA32_X2APIC_TMR3:
+            case MSR_IA32_X2APIC_TMR4:  case MSR_IA32_X2APIC_TMR5:  case MSR_IA32_X2APIC_TMR6:  case MSR_IA32_X2APIC_TMR7:
+            case MSR_IA32_X2APIC_IRR0:  case MSR_IA32_X2APIC_IRR1:  case MSR_IA32_X2APIC_IRR2:  case MSR_IA32_X2APIC_IRR3:
+            case MSR_IA32_X2APIC_IRR4:  case MSR_IA32_X2APIC_IRR5:  case MSR_IA32_X2APIC_IRR6:  case MSR_IA32_X2APIC_IRR7:
+            case MSR_IA32_X2APIC_TIMER_CCR:
+            {
+                rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_WRITE_READ_ONLY);
+                break;
+            }
+
+            /* Reserved MSRs: */
+            case MSR_IA32_X2APIC_LVT_CMCI:
+            default:
+            {
+                rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_WRITE_RSVD_OR_UNKNOWN);
+                break;
+            }
+        }
+    }
+    else
+        rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_INVALID_WRITE_MODE);
+
+    return rcStrict;
+}
+
+
+/**
+ * Resets the APIC base MSR.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+static void apicResetBaseMsr(PVMCPUCC pVCpu)
+{
+    /*
+     * Initialize the APIC base MSR. The APIC enable-bit is set upon power-up or reset[1].
+     *
+     * A Reset (in xAPIC and x2APIC mode) brings up the local APIC in xAPIC mode.
+     * An INIT IPI does -not- cause a transition between xAPIC and x2APIC mode[2].
+     *
+     * [1] See AMD spec. 14.1.3 "Processor Initialization State"
+     * [2] See Intel spec. 10.12.5.1 "x2APIC States".
+     */
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+
+    /* Construct. */
+    PAPICCPU pApicCpu     = VMCPU_TO_APICCPU(pVCpu);
+    PAPIC    pApic        = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    uint64_t uApicBaseMsr = MSR_IA32_APICBASE_ADDR;
+    if (pVCpu->idCpu == 0)
+        uApicBaseMsr |= MSR_IA32_APICBASE_BSP;
+
+    /* If the VM was configured with no APIC, don't enable xAPIC mode, obviously. */
+    if (pApic->enmMaxMode != PDMAPICMODE_NONE)
+    {
+        uApicBaseMsr |= MSR_IA32_APICBASE_EN;
+
+        /*
+         * While coming out of a reset the APIC is enabled and in xAPIC mode. If software had previously
+         * disabled the APIC (which results in the CPUID bit being cleared as well) we re-enable it here.
+         * See Intel spec. 10.12.5.1 "x2APIC States".
+         */
+        if (CPUMSetGuestCpuIdPerCpuApicFeature(pVCpu, true /*fVisible*/) == false)
+            LogRel(("APIC%u: Resetting mode to xAPIC\n", pVCpu->idCpu));
+    }
+
+    /* Commit. */
+    ASMAtomicWriteU64(&pApicCpu->uApicBaseMsr, uApicBaseMsr);
+}
+
+
+/**
+ * Initializes per-VCPU APIC to the state following an INIT reset
+ * ("Wait-for-SIPI" state).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+void apicInitIpi(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+
+    /*
+     * See Intel spec. 10.4.7.3 "Local APIC State After an INIT Reset (Wait-for-SIPI State)"
+     * and AMD spec 16.3.2 "APIC Registers".
+     *
+     * The reason we don't simply zero out the entire APIC page and only set the non-zero members
+     * is because there are some registers that are not touched by the INIT IPI (e.g. version)
+     * operation and this function is only a subset of the reset operation.
+     */
+    RT_ZERO(pXApicPage->irr);
+    RT_ZERO(pXApicPage->irr);
+    RT_ZERO(pXApicPage->isr);
+    RT_ZERO(pXApicPage->tmr);
+    RT_ZERO(pXApicPage->icr_hi);
+    RT_ZERO(pXApicPage->icr_lo);
+    RT_ZERO(pXApicPage->ldr);
+    RT_ZERO(pXApicPage->tpr);
+    RT_ZERO(pXApicPage->ppr);
+    RT_ZERO(pXApicPage->timer_icr);
+    RT_ZERO(pXApicPage->timer_ccr);
+    RT_ZERO(pXApicPage->timer_dcr);
+
+    pXApicPage->dfr.u.u4Model        = XAPICDESTFORMAT_FLAT;
+    pXApicPage->dfr.u.u28ReservedMb1 = UINT32_C(0xfffffff);
+
+    /** @todo CMCI. */
+
+    RT_ZERO(pXApicPage->lvt_timer);
+    pXApicPage->lvt_timer.u.u1Mask = 1;
+
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+    RT_ZERO(pXApicPage->lvt_thermal);
+    pXApicPage->lvt_thermal.u.u1Mask = 1;
+#endif
+
+    RT_ZERO(pXApicPage->lvt_perf);
+    pXApicPage->lvt_perf.u.u1Mask = 1;
+
+    RT_ZERO(pXApicPage->lvt_lint0);
+    pXApicPage->lvt_lint0.u.u1Mask = 1;
+
+    RT_ZERO(pXApicPage->lvt_lint1);
+    pXApicPage->lvt_lint1.u.u1Mask = 1;
+
+    RT_ZERO(pXApicPage->lvt_error);
+    pXApicPage->lvt_error.u.u1Mask = 1;
+
+    RT_ZERO(pXApicPage->svr);
+    pXApicPage->svr.u.u8SpuriousVector = 0xff;
+
+    /* The self-IPI register is reset to 0. See Intel spec. 10.12.5.1 "x2APIC States" */
+    PX2APICPAGE pX2ApicPage = VMCPU_TO_X2APICPAGE(pVCpu);
+    RT_ZERO(pX2ApicPage->self_ipi);
+
+    /* Clear the pending-interrupt bitmaps. */
+    PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    RT_BZERO(&pApicCpu->ApicPibLevel, sizeof(APICPIB));
+    RT_BZERO(pApicCpu->CTX_SUFF(pvApicPib), sizeof(APICPIB));
+
+    /* Clear the interrupt line states for LINT0 and LINT1 pins. */
+    pApicCpu->fActiveLint0 = false;
+    pApicCpu->fActiveLint1 = false;
+}
+
+
+/**
+ * Initializes per-VCPU APIC to the state following a power-up or hardware
+ * reset.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   fResetApicBaseMsr   Whether to reset the APIC base MSR.
+ */
+void apicResetCpu(PVMCPUCC pVCpu, bool fResetApicBaseMsr)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+
+    LogFlow(("APIC%u: apicR3ResetCpu: fResetApicBaseMsr=%RTbool\n", pVCpu->idCpu, fResetApicBaseMsr));
+
+#ifdef VBOX_STRICT
+    /* Verify that the initial APIC ID reported via CPUID matches our VMCPU ID assumption. */
+    uint32_t uEax, uEbx, uEcx, uEdx;
+    uEax = uEbx = uEcx = uEdx = UINT32_MAX;
+    CPUMGetGuestCpuId(pVCpu, 1, 0, &uEax, &uEbx, &uEcx, &uEdx);
+    Assert(((uEbx >> 24) & 0xff) == pVCpu->idCpu);
+#endif
+
+    /*
+     * The state following a power-up or reset is a superset of the INIT state.
+     * See Intel spec. 10.4.7.3 "Local APIC State After an INIT Reset ('Wait-for-SIPI' State)"
+     */
+    apicInitIpi(pVCpu);
+
+    /*
+     * The APIC version register is read-only, so just initialize it here.
+     * It is not clear from the specs, where exactly it is initialized.
+     * The version determines the number of LVT entries and size of the APIC ID (8 bits for P4).
+     */
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+    pXApicPage->version.u.u8MaxLvtEntry = XAPIC_MAX_LVT_ENTRIES_P4 - 1;
+    pXApicPage->version.u.u8Version     = XAPIC_HARDWARE_VERSION_P4;
+    AssertCompile(sizeof(pXApicPage->id.u8ApicId) >= XAPIC_APIC_ID_BIT_COUNT_P4 / 8);
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+
+    /** @todo It isn't clear in the spec. where exactly the default base address
+     *        is (re)initialized, atm we do it here in Reset. */
+    if (fResetApicBaseMsr)
+        apicResetBaseMsr(pVCpu);
+
+    /*
+     * Initialize the APIC ID register to xAPIC format.
+     */
+    ASMMemZero32(&pXApicPage->id, sizeof(pXApicPage->id));
+    pXApicPage->id.u8ApicId = pVCpu->idCpu;
+}
+
+
+/**
+ * Sets the APIC base MSR.
+ *
+ * @returns VBox status code - no informational ones, esp. not
+ *          VINF_CPUM_R3_MSR_WRITE.  Only the following two:
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   u64BaseMsr  The value to set.
+ */
+VMM_INT_DECL(int) APICSetBaseMsr(PVMCPUCC pVCpu, uint64_t u64BaseMsr)
+{
+    Assert(pVCpu);
+
+    PAPICCPU pApicCpu   = VMCPU_TO_APICCPU(pVCpu);
+    PAPIC    pApic      = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    APICMODE enmOldMode = apicGetMode(pApicCpu->uApicBaseMsr);
+    APICMODE enmNewMode = apicGetMode(u64BaseMsr);
+    uint64_t uBaseMsr   = pApicCpu->uApicBaseMsr;
+
+    Log2(("APIC%u: ApicSetBaseMsr: u64BaseMsr=%#RX64 enmNewMode=%s enmOldMode=%s\n", pVCpu->idCpu, u64BaseMsr,
+          apicGetModeName(enmNewMode), apicGetModeName(enmOldMode)));
+
+    /*
+     * We do not support re-mapping the APIC base address because:
+     *    - We'll have to manage all the mappings ourselves in the APIC (reference counting based unmapping etc.)
+     *      i.e. we can only unmap the MMIO region if no other APIC is mapped on that location.
+     *    - It's unclear how/if IOM can fallback to handling regions as regular memory (if the MMIO
+     *      region remains mapped but doesn't belong to the called VCPU's APIC).
+     */
+    /** @todo Handle per-VCPU APIC base relocation. */
+    if (MSR_IA32_APICBASE_GET_ADDR(uBaseMsr) != MSR_IA32_APICBASE_ADDR)
+    {
+        if (pVCpu->apic.s.cLogMaxSetApicBaseAddr++ < 5)
+            LogRel(("APIC%u: Attempt to relocate base to %#RGp, unsupported -> #GP(0)\n", pVCpu->idCpu,
+                    MSR_IA32_APICBASE_GET_ADDR(uBaseMsr)));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    /* Don't allow enabling xAPIC/x2APIC if the VM is configured with the APIC disabled. */
+    if (pApic->enmMaxMode == PDMAPICMODE_NONE)
+    {
+        LogRel(("APIC%u: Disallowing APIC base MSR write as the VM is configured with APIC disabled!\n", pVCpu->idCpu));
+        return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_DISALLOWED_CONFIG);
+    }
+
+    /*
+     * Act on state transition.
+     */
+    if (enmNewMode != enmOldMode)
+    {
+        switch (enmNewMode)
+        {
+            case APICMODE_DISABLED:
+            {
+                /*
+                 * The APIC state needs to be reset (especially the APIC ID as x2APIC APIC ID bit layout
+                 * is different). We can start with a clean slate identical to the state after a power-up/reset.
+                 *
+                 * See Intel spec. 10.4.3 "Enabling or Disabling the Local APIC".
+                 *
+                 * We'll also manually manage the APIC base MSR here. We want a single-point of commit
+                 * at the end of this function rather than updating it in apicR3ResetCpu. This means we also
+                 * need to update the CPUID leaf ourselves.
+                 */
+                apicResetCpu(pVCpu, false /* fResetApicBaseMsr */);
+                uBaseMsr &= ~(MSR_IA32_APICBASE_EN | MSR_IA32_APICBASE_EXTD);
+                CPUMSetGuestCpuIdPerCpuApicFeature(pVCpu, false /*fVisible*/);
+                LogRel(("APIC%u: Switched mode to disabled\n", pVCpu->idCpu));
+                break;
+            }
+
+            case APICMODE_XAPIC:
+            {
+                if (enmOldMode != APICMODE_DISABLED)
+                {
+                    LogRel(("APIC%u: Can only transition to xAPIC state from disabled state\n", pVCpu->idCpu));
+                    return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_INVALID);
+                }
+
+                uBaseMsr |= MSR_IA32_APICBASE_EN;
+                CPUMSetGuestCpuIdPerCpuApicFeature(pVCpu, true /*fVisible*/);
+                LogRel(("APIC%u: Switched mode to xAPIC\n", pVCpu->idCpu));
+                break;
+            }
+
+            case APICMODE_X2APIC:
+            {
+                if (pApic->enmMaxMode != PDMAPICMODE_X2APIC)
+                {
+                    LogRel(("APIC%u: Disallowing transition to x2APIC mode as the VM is configured with the x2APIC disabled!\n",
+                            pVCpu->idCpu));
+                    return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_INVALID);
+                }
+
+                if (enmOldMode != APICMODE_XAPIC)
+                {
+                    LogRel(("APIC%u: Can only transition to x2APIC state from xAPIC state\n", pVCpu->idCpu));
+                    return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_INVALID);
+                }
+
+                uBaseMsr |= MSR_IA32_APICBASE_EN | MSR_IA32_APICBASE_EXTD;
+
+                /*
+                 * The APIC ID needs updating when entering x2APIC mode.
+                 * Software written APIC ID in xAPIC mode isn't preserved.
+                 * The APIC ID becomes read-only to software in x2APIC mode.
+                 *
+                 * See Intel spec. 10.12.5.1 "x2APIC States".
+                 */
+                PX2APICPAGE pX2ApicPage = VMCPU_TO_X2APICPAGE(pVCpu);
+                ASMMemZero32(&pX2ApicPage->id, sizeof(pX2ApicPage->id));
+                pX2ApicPage->id.u32ApicId = pVCpu->idCpu;
+
+                /*
+                 * LDR initialization occurs when entering x2APIC mode.
+                 * See Intel spec. 10.12.10.2 "Deriving Logical x2APIC ID from the Local x2APIC ID".
+                 */
+                pX2ApicPage->ldr.u32LogicalApicId = ((pX2ApicPage->id.u32ApicId & UINT32_C(0xffff0)) << 16)
+                                                  | (UINT32_C(1) << pX2ApicPage->id.u32ApicId & UINT32_C(0xf));
+
+                LogRel(("APIC%u: Switched mode to x2APIC\n", pVCpu->idCpu));
+                break;
+            }
+
+            case APICMODE_INVALID:
+            default:
+            {
+                Log(("APIC%u: Invalid state transition attempted\n", pVCpu->idCpu));
+                return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_INVALID);
+            }
+        }
+    }
+
+    ASMAtomicWriteU64(&pApicCpu->uApicBaseMsr, uBaseMsr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the APIC base MSR (no checks are performed wrt APIC hardware or its
+ * state).
+ *
+ * @returns The base MSR value.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(uint64_t) APICGetBaseMsrNoCheck(PCVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+    PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    return pApicCpu->uApicBaseMsr;
+}
+
+
+/**
+ * Gets the APIC base MSR.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pu64Value   Where to store the MSR value.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICGetBaseMsr(PVMCPUCC pVCpu, uint64_t *pu64Value)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+
+    PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    if (pApic->enmMaxMode != PDMAPICMODE_NONE)
+    {
+        *pu64Value = APICGetBaseMsrNoCheck(pVCpu);
+        return VINF_SUCCESS;
+    }
+
+    if (pVCpu->apic.s.cLogMaxGetApicBaseAddr++ < 5)
+        LogRel(("APIC%u: Reading APIC base MSR (%#x) when there is no APIC -> #GP(0)\n", pVCpu->idCpu, MSR_IA32_APICBASE));
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * Sets the TPR (Task Priority Register).
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_CPUM_RAISE_GP_0
+ * @retval  VERR_PDM_NO_APIC_INSTANCE
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   u8Tpr       The TPR value to set.
+ */
+VMMDECL(int) APICSetTpr(PVMCPUCC pVCpu, uint8_t u8Tpr)
+{
+    if (APICIsEnabled(pVCpu))
+        return apicSetTprEx(pVCpu, u8Tpr, false /* fForceX2ApicBehaviour */);
+    return VERR_PDM_NO_APIC_INSTANCE;
+}
+
+
+/**
+ * Gets the highest priority pending interrupt.
+ *
+ * @returns true if any interrupt is pending, false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   pu8PendingIntr      Where to store the interrupt vector if the
+ *                              interrupt is pending (optional, can be NULL).
+ */
+static bool apicGetHighestPendingInterrupt(PCVMCPUCC pVCpu, uint8_t *pu8PendingIntr)
+{
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    int const irrv = apicGetHighestSetBitInReg(&pXApicPage->irr, -1);
+    if (irrv >= 0)
+    {
+        Assert(irrv <= (int)UINT8_MAX);
+        if (pu8PendingIntr)
+            *pu8PendingIntr = (uint8_t)irrv;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the APIC TPR (Task Priority Register).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pu8Tpr          Where to store the TPR.
+ * @param   pfPending       Where to store whether there is a pending interrupt
+ *                          (optional, can be NULL).
+ * @param   pu8PendingIntr  Where to store the highest-priority pending
+ *                          interrupt (optional, can be NULL).
+ */
+VMMDECL(int) APICGetTpr(PCVMCPUCC pVCpu, uint8_t *pu8Tpr, bool *pfPending, uint8_t *pu8PendingIntr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    if (APICIsEnabled(pVCpu))
+    {
+        PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+        if (pfPending)
+        {
+            /*
+             * Just return whatever the highest pending interrupt is in the IRR.
+             * The caller is responsible for figuring out if it's masked by the TPR etc.
+             */
+            *pfPending = apicGetHighestPendingInterrupt(pVCpu, pu8PendingIntr);
+        }
+
+        *pu8Tpr = pXApicPage->tpr.u8Tpr;
+        return VINF_SUCCESS;
+    }
+
+    *pu8Tpr = 0;
+    return VERR_PDM_NO_APIC_INSTANCE;
+}
+
+
+/**
+ * Gets the APIC timer frequency.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pu64Value       Where to store the timer frequency.
+ */
+VMM_INT_DECL(int) APICGetTimerFreq(PVMCC pVM, uint64_t *pu64Value)
+{
+    /*
+     * Validate.
+     */
+    Assert(pVM);
+    AssertPtrReturn(pu64Value, VERR_INVALID_PARAMETER);
+
+    PVMCPUCC pVCpu = pVM->CTX_SUFF(apCpus)[0];
+    if (APICIsEnabled(pVCpu))
+    {
+        PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+        *pu64Value = PDMDevHlpTimerGetFreq(VMCPU_TO_DEVINS(pVCpu), pApicCpu->hTimer);
+        return VINF_SUCCESS;
+    }
+    return VERR_PDM_NO_APIC_INSTANCE;
+}
+
+
+/**
+ * Delivers an interrupt message via the system bus.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   uDest           The destination mask.
+ * @param   uDestMode       The destination mode.
+ * @param   uDeliveryMode   The delivery mode.
+ * @param   uVector         The interrupt vector.
+ * @param   uPolarity       The interrupt line polarity.
+ * @param   uTriggerMode    The trigger mode.
+ * @param   uSrcTag         The interrupt source tag (debugging).
+ */
+VMM_INT_DECL(int) APICBusDeliver(PVMCC pVM, uint8_t uDest, uint8_t uDestMode, uint8_t uDeliveryMode, uint8_t uVector,
+                                 uint8_t uPolarity, uint8_t uTriggerMode, uint32_t uSrcTag)
+{
+    NOREF(uPolarity);
+
+    /*
+     * If the APIC isn't enabled, do nothing and pretend success.
+     */
+    if (APICIsEnabled(pVM->CTX_SUFF(apCpus)[0]))
+    { /* likely */ }
+    else
+        return VINF_SUCCESS;
+
+    /*
+     * The destination field (mask) in the IO APIC redirectable table entry is 8-bits.
+     * Hence, the broadcast mask is 0xff.
+     * See IO APIC spec. 3.2.4. "IOREDTBL[23:0] - I/O Redirectable Table Registers".
+     */
+    XAPICTRIGGERMODE  enmTriggerMode  = (XAPICTRIGGERMODE)uTriggerMode;
+    XAPICDELIVERYMODE enmDeliveryMode = (XAPICDELIVERYMODE)uDeliveryMode;
+    XAPICDESTMODE     enmDestMode     = (XAPICDESTMODE)uDestMode;
+    uint32_t          fDestMask       = uDest;
+    uint32_t          fBroadcastMask  = UINT32_C(0xff);
+
+    Log2(("APIC: apicBusDeliver: fDestMask=%#x enmDestMode=%s enmTriggerMode=%s enmDeliveryMode=%s uVector=%#x\n", fDestMask,
+          apicGetDestModeName(enmDestMode), apicGetTriggerModeName(enmTriggerMode), apicGetDeliveryModeName(enmDeliveryMode),
+          uVector));
+
+    bool     fIntrAccepted;
+    VMCPUSET DestCpuSet;
+    apicGetDestCpuSet(pVM, fDestMask, fBroadcastMask, enmDestMode, enmDeliveryMode, &DestCpuSet);
+    VBOXSTRICTRC rcStrict = apicSendIntr(pVM, NULL /* pVCpu */, uVector, enmTriggerMode, enmDeliveryMode, &DestCpuSet,
+                                         &fIntrAccepted, uSrcTag, VINF_SUCCESS /* rcRZ */);
+    if (fIntrAccepted)
+        return VBOXSTRICTRC_VAL(rcStrict);
+    return VERR_APIC_INTR_DISCARDED;
+}
+
+
+/**
+ * Assert/de-assert the local APIC's LINT0/LINT1 interrupt pins.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   u8Pin       The interrupt pin (0 for LINT0 or 1 for LINT1).
+ * @param   u8Level     The level (0 for low or 1 for high).
+ * @param   rcRZ        The return code if the operation cannot be performed in
+ *                      the current context.
+ *
+ * @note    All callers totally ignores the status code!
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICLocalInterrupt(PVMCPUCC pVCpu, uint8_t u8Pin, uint8_t u8Level, int rcRZ)
+{
+    AssertReturn(u8Pin <= 1, VERR_INVALID_PARAMETER);
+    AssertReturn(u8Level <= 1, VERR_INVALID_PARAMETER);
+
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+
+    /* If the APIC is enabled, the interrupt is subject to LVT programming. */
+    if (APICIsEnabled(pVCpu))
+    {
+        PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+
+        /* Pick the LVT entry corresponding to the interrupt pin. */
+        static const uint16_t s_au16LvtOffsets[] =
+        {
+            XAPIC_OFF_LVT_LINT0,
+            XAPIC_OFF_LVT_LINT1
+        };
+        Assert(u8Pin < RT_ELEMENTS(s_au16LvtOffsets));
+        uint16_t const offLvt = s_au16LvtOffsets[u8Pin];
+        uint32_t const uLvt   = apicReadRaw32(pXApicPage, offLvt);
+
+        /* If software hasn't masked the interrupt in the LVT entry, proceed interrupt processing. */
+        if (!XAPIC_LVT_IS_MASKED(uLvt))
+        {
+            XAPICDELIVERYMODE const enmDeliveryMode = XAPIC_LVT_GET_DELIVERY_MODE(uLvt);
+            XAPICTRIGGERMODE        enmTriggerMode  = XAPIC_LVT_GET_TRIGGER_MODE(uLvt);
+
+            switch (enmDeliveryMode)
+            {
+                case XAPICDELIVERYMODE_INIT:
+                {
+                    /** @todo won't work in R0/RC because callers don't care about rcRZ. */
+                    AssertMsgFailed(("INIT through LINT0/LINT1 is not yet supported\n"));
+                }
+                RT_FALL_THRU();
+                case XAPICDELIVERYMODE_FIXED:
+                {
+                    PAPICCPU       pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+                    uint8_t const  uVector  = XAPIC_LVT_GET_VECTOR(uLvt);
+                    bool           fActive  = RT_BOOL(u8Level & 1);
+                    bool volatile *pfActiveLine = u8Pin == 0 ? &pApicCpu->fActiveLint0 : &pApicCpu->fActiveLint1;
+                    /** @todo Polarity is busted elsewhere, we need to fix that
+                     *        first. See @bugref{8386#c7}. */
+#if 0
+                    uint8_t const u8Polarity = XAPIC_LVT_GET_POLARITY(uLvt);
+                    fActive ^= u8Polarity; */
+#endif
+                    if (!fActive)
+                    {
+                        ASMAtomicCmpXchgBool(pfActiveLine, false, true);
+                        break;
+                    }
+
+                    /* Level-sensitive interrupts are not supported for LINT1. See Intel spec. 10.5.1 "Local Vector Table". */
+                    if (offLvt == XAPIC_OFF_LVT_LINT1)
+                        enmTriggerMode = XAPICTRIGGERMODE_EDGE;
+                    /** @todo figure out what "If the local APIC is not used in conjunction with an I/O APIC and fixed
+                              delivery mode is selected; the Pentium 4, Intel Xeon, and P6 family processors will always
+                              use level-sensitive triggering, regardless if edge-sensitive triggering is selected."
+                              means. */
+
+                    bool fSendIntr;
+                    if (enmTriggerMode == XAPICTRIGGERMODE_EDGE)
+                    {
+                        /* Recognize and send the interrupt only on an edge transition. */
+                        fSendIntr = ASMAtomicCmpXchgBool(pfActiveLine, true, false);
+                    }
+                    else
+                    {
+                        /* For level-triggered interrupts, redundant interrupts are not a problem. */
+                        Assert(enmTriggerMode == XAPICTRIGGERMODE_LEVEL);
+                        ASMAtomicCmpXchgBool(pfActiveLine, true, false);
+
+                        /* Only when the remote IRR isn't set, set it and send the interrupt. */
+                        if (!(pXApicPage->lvt_lint0.all.u32LvtLint0 & XAPIC_LVT_REMOTE_IRR))
+                        {
+                            Assert(offLvt == XAPIC_OFF_LVT_LINT0);
+                            ASMAtomicOrU32((volatile uint32_t *)&pXApicPage->lvt_lint0.all.u32LvtLint0, XAPIC_LVT_REMOTE_IRR);
+                            fSendIntr = true;
+                        }
+                        else
+                            fSendIntr = false;
+                    }
+
+                    if (fSendIntr)
+                    {
+                        VMCPUSET DestCpuSet;
+                        VMCPUSET_EMPTY(&DestCpuSet);
+                        VMCPUSET_ADD(&DestCpuSet, pVCpu->idCpu);
+                        rcStrict = apicSendIntr(pVCpu->CTX_SUFF(pVM), pVCpu, uVector, enmTriggerMode, enmDeliveryMode,
+                                                &DestCpuSet, NULL /* pfIntrAccepted */, 0 /* uSrcTag */, rcRZ);
+                    }
+                    break;
+                }
+
+                case XAPICDELIVERYMODE_SMI:
+                case XAPICDELIVERYMODE_NMI:
+                {
+                    VMCPUSET DestCpuSet;
+                    VMCPUSET_EMPTY(&DestCpuSet);
+                    VMCPUSET_ADD(&DestCpuSet, pVCpu->idCpu);
+                    uint8_t const uVector = XAPIC_LVT_GET_VECTOR(uLvt);
+                    rcStrict = apicSendIntr(pVCpu->CTX_SUFF(pVM), pVCpu, uVector, enmTriggerMode, enmDeliveryMode, &DestCpuSet,
+                                            NULL /* pfIntrAccepted */, 0 /* uSrcTag */, rcRZ);
+                    break;
+                }
+
+                case XAPICDELIVERYMODE_EXTINT:
+                {
+                    Log2(("APIC%u: apicLocalInterrupt: %s ExtINT through LINT%u\n", pVCpu->idCpu,
+                          u8Level ? "Raising" : "Lowering", u8Pin));
+                    if (u8Level)
+                        apicSetInterruptFF(pVCpu, PDMAPICIRQ_EXTINT);
+                    else
+                        apicClearInterruptFF(pVCpu, PDMAPICIRQ_EXTINT);
+                    break;
+                }
+
+                /* Reserved/unknown delivery modes: */
+                case XAPICDELIVERYMODE_LOWEST_PRIO:
+                case XAPICDELIVERYMODE_STARTUP:
+                default:
+                {
+                    AssertMsgFailed(("APIC%u: LocalInterrupt: Invalid delivery mode %#x (%s) on LINT%d\n", pVCpu->idCpu,
+                                     enmDeliveryMode, apicGetDeliveryModeName(enmDeliveryMode), u8Pin));
+                    rcStrict = VERR_INTERNAL_ERROR_3;
+                    break;
+                }
+            }
+        }
+    }
+    else
+    {
+        /* The APIC is hardware disabled. The CPU behaves as though there is no on-chip APIC. */
+        if (u8Pin == 0)
+        {
+            /* LINT0 behaves as an external interrupt pin. */
+            Log2(("APIC%u: apicLocalInterrupt: APIC hardware-disabled, %s INTR\n", pVCpu->idCpu,
+                  u8Level ? "raising" : "lowering"));
+            if (u8Level)
+                apicSetInterruptFF(pVCpu, PDMAPICIRQ_EXTINT);
+            else
+                apicClearInterruptFF(pVCpu, PDMAPICIRQ_EXTINT);
+        }
+        else
+        {
+            /* LINT1 behaves as NMI. */
+            Log2(("APIC%u: apicLocalInterrupt: APIC hardware-disabled, raising NMI\n", pVCpu->idCpu));
+            apicSetInterruptFF(pVCpu, PDMAPICIRQ_NMI);
+        }
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Gets the next highest-priority interrupt from the APIC, marking it as an
+ * "in-service" interrupt.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pu8Vector   Where to store the vector.
+ * @param   puSrcTag    Where to store the interrupt source tag (debugging).
+ */
+VMM_INT_DECL(int) APICGetInterrupt(PVMCPUCC pVCpu, uint8_t *pu8Vector, uint32_t *puSrcTag)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(pu8Vector);
+
+    LogFlow(("APIC%u: apicGetInterrupt:\n", pVCpu->idCpu));
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    bool const fApicHwEnabled = APICIsEnabled(pVCpu);
+    if (   fApicHwEnabled
+        && pXApicPage->svr.u.fApicSoftwareEnable)
+    {
+        int const irrv = apicGetHighestSetBitInReg(&pXApicPage->irr, -1);
+        if (RT_LIKELY(irrv >= 0))
+        {
+            Assert(irrv <= (int)UINT8_MAX);
+            uint8_t const uVector = irrv;
+
+            /*
+             * This can happen if the APIC receives an interrupt when the CPU has interrupts
+             * disabled but the TPR is raised by the guest before re-enabling interrupts.
+             */
+            uint8_t const uTpr = pXApicPage->tpr.u8Tpr;
+            if (   uTpr > 0
+                && XAPIC_TPR_GET_TP(uVector) <= XAPIC_TPR_GET_TP(uTpr))
+            {
+                Log2(("APIC%u: apicGetInterrupt: Interrupt masked. uVector=%#x uTpr=%#x SpuriousVector=%#x\n", pVCpu->idCpu,
+                      uVector, uTpr, pXApicPage->svr.u.u8SpuriousVector));
+                *pu8Vector = uVector;
+                *puSrcTag  = 0;
+                STAM_COUNTER_INC(&pVCpu->apic.s.StatMaskedByTpr);
+                return VERR_APIC_INTR_MASKED_BY_TPR;
+            }
+
+            /*
+             * The PPR should be up-to-date at this point through apicSetEoi().
+             * We're on EMT so no parallel updates possible.
+             * Subject the pending vector to PPR prioritization.
+             */
+            uint8_t const uPpr = pXApicPage->ppr.u8Ppr;
+            if (   !uPpr
+                || XAPIC_PPR_GET_PP(uVector) > XAPIC_PPR_GET_PP(uPpr))
+            {
+                apicClearVectorInReg(&pXApicPage->irr, uVector);
+                apicSetVectorInReg(&pXApicPage->isr, uVector);
+                apicUpdatePpr(pVCpu);
+                apicSignalNextPendingIntr(pVCpu);
+
+                /* Retrieve the interrupt source tag associated with this interrupt. */
+                PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+                AssertCompile(RT_ELEMENTS(pApicCpu->auSrcTags) > UINT8_MAX);
+                *puSrcTag = pApicCpu->auSrcTags[uVector];
+                pApicCpu->auSrcTags[uVector] = 0;
+
+                Log2(("APIC%u: apicGetInterrupt: Valid Interrupt. uVector=%#x\n", pVCpu->idCpu, uVector));
+                *pu8Vector = uVector;
+                return VINF_SUCCESS;
+            }
+            else
+            {
+                STAM_COUNTER_INC(&pVCpu->apic.s.StatMaskedByPpr);
+                Log2(("APIC%u: apicGetInterrupt: Interrupt's priority is not higher than the PPR. uVector=%#x PPR=%#x\n",
+                      pVCpu->idCpu, uVector, uPpr));
+            }
+        }
+        else
+            Log2(("APIC%u: apicGetInterrupt: No pending bits in IRR\n", pVCpu->idCpu));
+    }
+    else
+        Log2(("APIC%u: apicGetInterrupt: APIC %s disabled\n", pVCpu->idCpu, !fApicHwEnabled ? "hardware" : "software"));
+
+    *pu8Vector = 0;
+    *puSrcTag  = 0;
+    return VERR_APIC_INTR_NOT_PENDING;
+}
+
+
+/**
+ * @callback_method_impl{FNIOMMMIONEWREAD}
+ */
+DECLCALLBACK(VBOXSTRICTRC) apicReadMmio(PPDMDEVINS pDevIns, void *pvUser, RTGCPHYS off, void *pv, unsigned cb)
+{
+    NOREF(pvUser);
+    Assert(!(off & 0xf));
+    Assert(cb == 4); RT_NOREF_PV(cb);
+
+    PVMCPUCC pVCpu    = PDMDevHlpGetVMCPU(pDevIns);
+    uint16_t offReg   = off & 0xff0;
+    uint32_t uValue   = 0;
+
+    STAM_COUNTER_INC(&pVCpu->apic.s.CTX_SUFF_Z(StatMmioRead));
+
+    VBOXSTRICTRC rc = VBOXSTRICTRC_VAL(apicReadRegister(pDevIns, pVCpu, offReg, &uValue));
+    *(uint32_t *)pv = uValue;
+
+    Log2(("APIC%u: apicReadMmio: offReg=%#RX16 uValue=%#RX32\n", pVCpu->idCpu, offReg, uValue));
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNIOMMMIONEWWRITE}
+ */
+DECLCALLBACK(VBOXSTRICTRC) apicWriteMmio(PPDMDEVINS pDevIns, void *pvUser, RTGCPHYS off, void const *pv, unsigned cb)
+{
+    NOREF(pvUser);
+    Assert(!(off & 0xf));
+    Assert(cb == 4); RT_NOREF_PV(cb);
+
+    PVMCPUCC pVCpu    = PDMDevHlpGetVMCPU(pDevIns);
+    uint16_t offReg   = off & 0xff0;
+    uint32_t uValue   = *(uint32_t *)pv;
+
+    STAM_COUNTER_INC(&pVCpu->apic.s.CTX_SUFF_Z(StatMmioWrite));
+
+    Log2(("APIC%u: apicWriteMmio: offReg=%#RX16 uValue=%#RX32\n", pVCpu->idCpu, offReg, uValue));
+
+    return apicWriteRegister(pDevIns, pVCpu, offReg, uValue);
+}
+
+
+/**
+ * Sets the interrupt pending force-flag and pokes the EMT if required.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmType         The IRQ type.
+ */
+static void apicSetInterruptFF(PVMCPUCC pVCpu, PDMAPICIRQ enmType)
+{
+#ifdef IN_RING3
+    /* IRQ state should be loaded as-is by "LoadExec". Changes can be made from LoadDone. */
+    Assert(pVCpu->pVMR3->enmVMState != VMSTATE_LOADING || PDMR3HasLoadedState(pVCpu->pVMR3));
+#endif
+
+    switch (enmType)
+    {
+        case PDMAPICIRQ_HARDWARE:
+            VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC);
+            break;
+        case PDMAPICIRQ_UPDATE_PENDING: VMCPU_FF_SET(pVCpu, VMCPU_FF_UPDATE_APIC);    break;
+        case PDMAPICIRQ_NMI:            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI);  break;
+        case PDMAPICIRQ_SMI:            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI);  break;
+        case PDMAPICIRQ_EXTINT:         VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC);  break;
+        default:
+            AssertMsgFailed(("enmType=%d\n", enmType));
+            break;
+    }
+
+    /*
+     * We need to wake up the target CPU if we're not on EMT.
+     */
+#if defined(IN_RING0)
+    PVMCC   pVM   = pVCpu->CTX_SUFF(pVM);
+    VMCPUID idCpu = pVCpu->idCpu;
+    if (   enmType != PDMAPICIRQ_HARDWARE
+        && VMMGetCpuId(pVM) != idCpu)
+    {
+        switch (VMCPU_GET_STATE(pVCpu))
+        {
+            case VMCPUSTATE_STARTED_EXEC:
+                GVMMR0SchedPokeNoGVMNoLock(pVM, idCpu);
+                break;
+
+            case VMCPUSTATE_STARTED_HALTED:
+                GVMMR0SchedWakeUpNoGVMNoLock(pVM, idCpu);
+                break;
+
+            default:
+                break; /* nothing to do in other states. */
+        }
+    }
+#elif defined(IN_RING3)
+    if (enmType != PDMAPICIRQ_HARDWARE)
+        VMR3NotifyCpuFFU(pVCpu->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM | VMNOTIFYFF_FLAGS_POKE);
+#endif
+}
+
+
+/**
+ * Clears the interrupt pending force-flag.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmType         The IRQ type.
+ */
+void apicClearInterruptFF(PVMCPUCC pVCpu, PDMAPICIRQ enmType)
+{
+#ifdef IN_RING3
+    /* IRQ state should be loaded as-is by "LoadExec". Changes can be made from LoadDone. */
+    Assert(pVCpu->pVMR3->enmVMState != VMSTATE_LOADING || PDMR3HasLoadedState(pVCpu->pVMR3));
+#endif
+
+    /* NMI/SMI can't be cleared. */
+    switch (enmType)
+    {
+        case PDMAPICIRQ_HARDWARE:   VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC); break;
+        case PDMAPICIRQ_EXTINT:     VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);  break;
+        default:
+            AssertMsgFailed(("enmType=%d\n", enmType));
+            break;
+    }
+}
+
+
+/**
+ * Posts an interrupt to a target APIC.
+ *
+ * This function handles interrupts received from the system bus or
+ * interrupts generated locally from the LVT or via a self IPI.
+ *
+ * Don't use this function to try and deliver ExtINT style interrupts.
+ *
+ * @returns true if the interrupt was accepted, false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uVector             The vector of the interrupt to be posted.
+ * @param   enmTriggerMode      The trigger mode of the interrupt.
+ * @param   uSrcTag             The interrupt source tag (debugging).
+ *
+ * @thread  Any.
+ */
+bool apicPostInterrupt(PVMCPUCC pVCpu, uint8_t uVector, XAPICTRIGGERMODE enmTriggerMode, uint32_t uSrcTag)
+{
+    Assert(pVCpu);
+    Assert(uVector > XAPIC_ILLEGAL_VECTOR_END);
+
+    PVMCC    pVM       = pVCpu->CTX_SUFF(pVM);
+    PCAPIC   pApic     = VM_TO_APIC(pVM);
+    PAPICCPU pApicCpu  = VMCPU_TO_APICCPU(pVCpu);
+    bool     fAccepted = true;
+
+    STAM_PROFILE_START(&pApicCpu->StatPostIntr, a);
+
+    /*
+     * Only post valid interrupt vectors.
+     * See Intel spec. 10.5.2 "Valid Interrupt Vectors".
+     */
+    if (RT_LIKELY(uVector > XAPIC_ILLEGAL_VECTOR_END))
+    {
+        /*
+         * If the interrupt is already pending in the IRR we can skip the
+         * potential expensive operation of poking the guest EMT out of execution.
+         */
+        PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+        if (!apicTestVectorInReg(&pXApicPage->irr, uVector))     /* PAV */
+        {
+            /* Update the interrupt source tag (debugging). */
+            if (!pApicCpu->auSrcTags[uVector])
+                pApicCpu->auSrcTags[uVector]  = uSrcTag;
+            else
+                pApicCpu->auSrcTags[uVector] |= RT_BIT_32(31);
+
+            Log2(("APIC: apicPostInterrupt: SrcCpu=%u TargetCpu=%u uVector=%#x\n", VMMGetCpuId(pVM), pVCpu->idCpu, uVector));
+            if (enmTriggerMode == XAPICTRIGGERMODE_EDGE)
+            {
+                if (pApic->fPostedIntrsEnabled)
+                { /** @todo posted-interrupt call to hardware */ }
+                else
+                {
+                    apicSetVectorInPib(pApicCpu->CTX_SUFF(pvApicPib), uVector);
+                    uint32_t const fAlreadySet = apicSetNotificationBitInPib((PAPICPIB)pApicCpu->CTX_SUFF(pvApicPib));
+                    if (!fAlreadySet)
+                    {
+                        Log2(("APIC: apicPostInterrupt: Setting UPDATE_APIC FF for edge-triggered intr. uVector=%#x\n", uVector));
+                        apicSetInterruptFF(pVCpu, PDMAPICIRQ_UPDATE_PENDING);
+                    }
+                }
+            }
+            else
+            {
+                /*
+                 * Level-triggered interrupts requires updating of the TMR and thus cannot be
+                 * delivered asynchronously.
+                 */
+                apicSetVectorInPib(&pApicCpu->ApicPibLevel, uVector);
+                uint32_t const fAlreadySet = apicSetNotificationBitInPib(&pApicCpu->ApicPibLevel);
+                if (!fAlreadySet)
+                {
+                    Log2(("APIC: apicPostInterrupt: Setting UPDATE_APIC FF for level-triggered intr. uVector=%#x\n", uVector));
+                    apicSetInterruptFF(pVCpu, PDMAPICIRQ_UPDATE_PENDING);
+                }
+            }
+        }
+        else
+        {
+            Log2(("APIC: apicPostInterrupt: SrcCpu=%u TargetCpu=%u. Vector %#x Already in IRR, skipping\n", VMMGetCpuId(pVM),
+                  pVCpu->idCpu, uVector));
+            STAM_COUNTER_INC(&pApicCpu->StatPostIntrAlreadyPending);
+        }
+    }
+    else
+    {
+        fAccepted = false;
+        apicSetError(pVCpu, XAPIC_ESR_RECV_ILLEGAL_VECTOR);
+    }
+
+    STAM_PROFILE_STOP(&pApicCpu->StatPostIntr, a);
+    return fAccepted;
+}
+
+
+/**
+ * Starts the APIC timer.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uInitialCount   The timer's Initial-Count Register (ICR), must be >
+ *                          0.
+ * @thread  Any.
+ */
+void apicStartTimer(PVMCPUCC pVCpu, uint32_t uInitialCount)
+{
+    Assert(pVCpu);
+    PAPICCPU   pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    PPDMDEVINS pDevIns  = VMCPU_TO_DEVINS(pVCpu);
+    Assert(PDMDevHlpTimerIsLockOwner(pDevIns, pApicCpu->hTimer));
+    Assert(uInitialCount > 0);
+
+    PCXAPICPAGE    pXApicPage   = APICCPU_TO_CXAPICPAGE(pApicCpu);
+    uint8_t  const uTimerShift  = apicGetTimerShift(pXApicPage);
+    uint64_t const cTicksToNext = (uint64_t)uInitialCount << uTimerShift;
+
+    Log2(("APIC%u: apicStartTimer: uInitialCount=%#RX32 uTimerShift=%u cTicksToNext=%RU64\n", pVCpu->idCpu, uInitialCount,
+          uTimerShift, cTicksToNext));
+
+    /*
+     * The assumption here is that the timer doesn't tick during this call
+     * and thus setting a relative time to fire next is accurate. The advantage
+     * however is updating u64TimerInitial 'atomically' while setting the next
+     * tick.
+     */
+    PDMDevHlpTimerSetRelative(pDevIns, pApicCpu->hTimer, cTicksToNext, &pApicCpu->u64TimerInitial);
+    apicHintTimerFreq(pDevIns, pApicCpu, uInitialCount, uTimerShift);
+}
+
+
+/**
+ * Stops the APIC timer.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @thread  Any.
+ */
+static void apicStopTimer(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu);
+    PAPICCPU   pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    PPDMDEVINS pDevIns  = VMCPU_TO_DEVINS(pVCpu);
+    Assert(PDMDevHlpTimerIsLockOwner(pDevIns, pApicCpu->hTimer));
+
+    Log2(("APIC%u: apicStopTimer\n", pVCpu->idCpu));
+
+    PDMDevHlpTimerStop(pDevIns, pApicCpu->hTimer); /* This will reset the hint, no need to explicitly call TMTimerSetFrequencyHint(). */
+    pApicCpu->uHintedTimerInitialCount = 0;
+    pApicCpu->uHintedTimerShift = 0;
+}
+
+
+/**
+ * Queues a pending interrupt as in-service.
+ *
+ * This function should only be needed without virtualized APIC
+ * registers. With virtualized APIC registers, it's sufficient to keep
+ * the interrupts pending in the IRR as the hardware takes care of
+ * virtual interrupt delivery.
+ *
+ * @returns true if the interrupt was queued to in-service interrupts,
+ *          false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   u8PendingIntr       The pending interrupt to queue as
+ *                              in-service.
+ *
+ * @remarks This assumes the caller has done the necessary checks and
+ *          is ready to take actually service the interrupt (TPR,
+ *          interrupt shadow etc.)
+ */
+VMM_INT_DECL(bool) APICQueueInterruptToService(PVMCPUCC pVCpu, uint8_t u8PendingIntr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PVMCC pVM   = pVCpu->CTX_SUFF(pVM);
+    PAPIC pApic = VM_TO_APIC(pVM);
+    Assert(!pApic->fVirtApicRegsEnabled);
+    NOREF(pApic);
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    bool const fIsPending = apicTestVectorInReg(&pXApicPage->irr, u8PendingIntr);
+    if (fIsPending)
+    {
+        apicClearVectorInReg(&pXApicPage->irr, u8PendingIntr);
+        apicSetVectorInReg(&pXApicPage->isr, u8PendingIntr);
+        apicUpdatePpr(pVCpu);
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * De-queues a pending interrupt from in-service.
+ *
+ * This undoes APICQueueInterruptToService() for premature VM-exits before event
+ * injection.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   u8PendingIntr       The pending interrupt to de-queue from
+ *                              in-service.
+ */
+VMM_INT_DECL(void) APICDequeueInterruptFromService(PVMCPUCC pVCpu, uint8_t u8PendingIntr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PVMCC pVM   = pVCpu->CTX_SUFF(pVM);
+    PAPIC pApic = VM_TO_APIC(pVM);
+    Assert(!pApic->fVirtApicRegsEnabled);
+    NOREF(pApic);
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    bool const fInService = apicTestVectorInReg(&pXApicPage->isr, u8PendingIntr);
+    if (fInService)
+    {
+        apicClearVectorInReg(&pXApicPage->isr, u8PendingIntr);
+        apicSetVectorInReg(&pXApicPage->irr, u8PendingIntr);
+        apicUpdatePpr(pVCpu);
+    }
+}
+
+
+/**
+ * Updates pending interrupts from the pending-interrupt bitmaps to the IRR.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ *
+ * @note    NEM/win is ASSUMING the an up to date TPR is not required here.
+ */
+VMMDECL(void) APICUpdatePendingInterrupts(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+
+    PAPICCPU   pApicCpu         = VMCPU_TO_APICCPU(pVCpu);
+    PXAPICPAGE pXApicPage       = VMCPU_TO_XAPICPAGE(pVCpu);
+    bool       fHasPendingIntrs = false;
+
+    Log3(("APIC%u: APICUpdatePendingInterrupts:\n", pVCpu->idCpu));
+    STAM_PROFILE_START(&pApicCpu->StatUpdatePendingIntrs, a);
+
+    /* Update edge-triggered pending interrupts. */
+    PAPICPIB pPib = (PAPICPIB)pApicCpu->CTX_SUFF(pvApicPib);
+    for (;;)
+    {
+        uint32_t const fAlreadySet = apicClearNotificationBitInPib((PAPICPIB)pApicCpu->CTX_SUFF(pvApicPib));
+        if (!fAlreadySet)
+            break;
+
+        AssertCompile(RT_ELEMENTS(pXApicPage->irr.u) == 2 * RT_ELEMENTS(pPib->au64VectorBitmap));
+        for (size_t idxPib = 0, idxReg = 0; idxPib < RT_ELEMENTS(pPib->au64VectorBitmap); idxPib++, idxReg += 2)
+        {
+            uint64_t const u64Fragment = ASMAtomicXchgU64(&pPib->au64VectorBitmap[idxPib], 0);
+            if (u64Fragment)
+            {
+                uint32_t const u32FragmentLo = RT_LO_U32(u64Fragment);
+                uint32_t const u32FragmentHi = RT_HI_U32(u64Fragment);
+
+                pXApicPage->irr.u[idxReg].u32Reg     |=  u32FragmentLo;
+                pXApicPage->irr.u[idxReg + 1].u32Reg |=  u32FragmentHi;
+
+                pXApicPage->tmr.u[idxReg].u32Reg     &= ~u32FragmentLo;
+                pXApicPage->tmr.u[idxReg + 1].u32Reg &= ~u32FragmentHi;
+                fHasPendingIntrs = true;
+            }
+        }
+    }
+
+    /* Update level-triggered pending interrupts. */
+    pPib = (PAPICPIB)&pApicCpu->ApicPibLevel;
+    for (;;)
+    {
+        uint32_t const fAlreadySet = apicClearNotificationBitInPib((PAPICPIB)&pApicCpu->ApicPibLevel);
+        if (!fAlreadySet)
+            break;
+
+        AssertCompile(RT_ELEMENTS(pXApicPage->irr.u) == 2 * RT_ELEMENTS(pPib->au64VectorBitmap));
+        for (size_t idxPib = 0, idxReg = 0; idxPib < RT_ELEMENTS(pPib->au64VectorBitmap); idxPib++, idxReg += 2)
+        {
+            uint64_t const u64Fragment = ASMAtomicXchgU64(&pPib->au64VectorBitmap[idxPib], 0);
+            if (u64Fragment)
+            {
+                uint32_t const u32FragmentLo = RT_LO_U32(u64Fragment);
+                uint32_t const u32FragmentHi = RT_HI_U32(u64Fragment);
+
+                pXApicPage->irr.u[idxReg].u32Reg     |= u32FragmentLo;
+                pXApicPage->irr.u[idxReg + 1].u32Reg |= u32FragmentHi;
+
+                pXApicPage->tmr.u[idxReg].u32Reg     |= u32FragmentLo;
+                pXApicPage->tmr.u[idxReg + 1].u32Reg |= u32FragmentHi;
+                fHasPendingIntrs = true;
+            }
+        }
+    }
+
+    STAM_PROFILE_STOP(&pApicCpu->StatUpdatePendingIntrs, a);
+    Log3(("APIC%u: APICUpdatePendingInterrupts: fHasPendingIntrs=%RTbool\n", pVCpu->idCpu, fHasPendingIntrs));
+
+    if (   fHasPendingIntrs
+        && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC))
+        apicSignalNextPendingIntr(pVCpu);
+}
+
+
+/**
+ * Gets the highest priority pending interrupt.
+ *
+ * @returns true if any interrupt is pending, false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   pu8PendingIntr      Where to store the interrupt vector if the
+ *                              interrupt is pending.
+ */
+VMM_INT_DECL(bool) APICGetHighestPendingInterrupt(PVMCPUCC pVCpu, uint8_t *pu8PendingIntr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    return apicGetHighestPendingInterrupt(pVCpu, pu8PendingIntr);
+}
+
+
+/**
+ * Posts an interrupt to a target APIC, Hyper-V interface.
+ *
+ * @returns true if the interrupt was accepted, false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uVector             The vector of the interrupt to be posted.
+ * @param   fAutoEoi            Whether this interrupt has automatic EOI
+ *                              treatment.
+ * @param   enmTriggerMode      The trigger mode of the interrupt.
+ *
+ * @thread  Any.
+ */
+VMM_INT_DECL(void) APICHvSendInterrupt(PVMCPUCC pVCpu, uint8_t uVector, bool fAutoEoi, XAPICTRIGGERMODE enmTriggerMode)
+{
+    Assert(pVCpu);
+    Assert(!fAutoEoi);    /** @todo AutoEOI.  */
+    RT_NOREF(fAutoEoi);
+    apicPostInterrupt(pVCpu, uVector, enmTriggerMode, 0 /* uSrcTag */);
+}
+
+
+/**
+ * Sets the Task Priority Register (TPR), Hyper-V interface.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uTpr        The TPR value to set.
+ *
+ * @remarks Validates like in x2APIC mode.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICHvSetTpr(PVMCPUCC pVCpu, uint8_t uTpr)
+{
+    Assert(pVCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+    return apicSetTprEx(pVCpu, uTpr, true /* fForceX2ApicBehaviour */);
+}
+
+
+/**
+ * Gets the Task Priority Register (TPR), Hyper-V interface.
+ *
+ * @returns The TPR value.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(uint8_t) APICHvGetTpr(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * The APIC could be operating in xAPIC mode and thus we should not use the apicReadMsr()
+     * interface which validates the APIC mode and will throw a #GP(0) if not in x2APIC mode.
+     * We could use the apicReadRegister() MMIO interface, but why bother getting the PDMDEVINS
+     * pointer, so just directly read the APIC page.
+     */
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    return apicReadRaw32(pXApicPage, XAPIC_OFF_TPR);
+}
+
+
+/**
+ * Sets the Interrupt Command Register (ICR), Hyper-V interface.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uIcr        The ICR value to set.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICHvSetIcr(PVMCPUCC pVCpu, uint64_t uIcr)
+{
+    Assert(pVCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+    return apicSetIcr(pVCpu, uIcr, VINF_CPUM_R3_MSR_WRITE);
+}
+
+
+/**
+ * Gets the Interrupt Command Register (ICR), Hyper-V interface.
+ *
+ * @returns The ICR value.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(uint64_t) APICHvGetIcr(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+    return apicGetIcrNoCheck(pVCpu);
+}
+
+
+/**
+ * Sets the End-Of-Interrupt (EOI) register, Hyper-V interface.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uEoi            The EOI value.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICHvSetEoi(PVMCPUCC pVCpu, uint32_t uEoi)
+{
+    Assert(pVCpu);
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+    return apicSetEoi(pVCpu, uEoi, VINF_CPUM_R3_MSR_WRITE, true /* fForceX2ApicBehaviour */);
+}
+
+
+/**
+ * Gets the APIC page pointers for the specified VCPU.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pHCPhys         Where to store the host-context physical address.
+ * @param   pR0Ptr          Where to store the ring-0 address.
+ * @param   pR3Ptr          Where to store the ring-3 address (optional).
+ */
+VMM_INT_DECL(int) APICGetApicPageForCpu(PCVMCPUCC pVCpu, PRTHCPHYS pHCPhys, PRTR0PTR pR0Ptr, PRTR3PTR pR3Ptr)
+{
+    AssertReturn(pVCpu,   VERR_INVALID_PARAMETER);
+    AssertReturn(pHCPhys, VERR_INVALID_PARAMETER);
+    AssertReturn(pR0Ptr,  VERR_INVALID_PARAMETER);
+
+    Assert(PDMHasApic(pVCpu->CTX_SUFF(pVM)));
+
+    PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    *pHCPhys = pApicCpu->HCPhysApicPage;
+    *pR0Ptr  = pApicCpu->pvApicPageR0;
+    if (pR3Ptr)
+        *pR3Ptr  = pApicCpu->pvApicPageR3;
+    return VINF_SUCCESS;
+}
+
+#ifndef IN_RING3
+
+/**
+ * @callback_method_impl{PDMDEVREGR0,pfnConstruct}
+ */
+static DECLCALLBACK(int) apicRZConstruct(PPDMDEVINS pDevIns)
+{
+    PDMDEV_CHECK_VERSIONS_RETURN(pDevIns);
+    PAPICDEV pThis = PDMDEVINS_2_DATA(pDevIns, PAPICDEV);
+    PVMCC    pVM   = PDMDevHlpGetVM(pDevIns);
+
+    pVM->apicr0.s.pDevInsR0 = pDevIns;
+
+    int rc = PDMDevHlpSetDeviceCritSect(pDevIns, PDMDevHlpCritSectGetNop(pDevIns));
+    AssertRCReturn(rc, rc);
+
+    rc = PDMDevHlpApicSetUpContext(pDevIns);
+    AssertRCReturn(rc, rc);
+
+    rc = PDMDevHlpMmioSetUpContext(pDevIns, pThis->hMmio, apicWriteMmio, apicReadMmio, NULL /*pvUser*/);
+    AssertRCReturn(rc, rc);
+
+    return VINF_SUCCESS;
+}
+#endif /* !IN_RING3 */
+
+/**
+ * APIC device registration structure.
+ */
+const PDMDEVREG g_DeviceAPIC =
+{
+    /* .u32Version = */             PDM_DEVREG_VERSION,
+    /* .uReserved0 = */             0,
+    /* .szName = */                 "apic",
+    /* .fFlags = */                 PDM_DEVREG_FLAGS_DEFAULT_BITS | PDM_DEVREG_FLAGS_RZ | PDM_DEVREG_FLAGS_NEW_STYLE
+                                    | PDM_DEVREG_FLAGS_REQUIRE_R0 | PDM_DEVREG_FLAGS_REQUIRE_RC,
+    /* .fClass = */                 PDM_DEVREG_CLASS_PIC,
+    /* .cMaxInstances = */          1,
+    /* .uSharedVersion = */         42,
+    /* .cbInstanceShared = */       sizeof(APICDEV),
+    /* .cbInstanceCC = */           0,
+    /* .cbInstanceRC = */           0,
+    /* .cMaxPciDevices = */         0,
+    /* .cMaxMsixVectors = */        0,
+    /* .pszDescription = */         "Advanced Programmable Interrupt Controller",
+#if defined(IN_RING3)
+    /* .szRCMod = */                "VMMRC.rc",
+    /* .szR0Mod = */                "VMMR0.r0",
+    /* .pfnConstruct = */           apicR3Construct,
+    /* .pfnDestruct = */            apicR3Destruct,
+    /* .pfnRelocate = */            apicR3Relocate,
+    /* .pfnMemSetup = */            NULL,
+    /* .pfnPowerOn = */             NULL,
+    /* .pfnReset = */               apicR3Reset,
+    /* .pfnSuspend = */             NULL,
+    /* .pfnResume = */              NULL,
+    /* .pfnAttach = */              NULL,
+    /* .pfnDetach = */              NULL,
+    /* .pfnQueryInterface = */      NULL,
+    /* .pfnInitComplete = */        apicR3InitComplete,
+    /* .pfnPowerOff = */            NULL,
+    /* .pfnSoftReset = */           NULL,
+    /* .pfnReserved0 = */           NULL,
+    /* .pfnReserved1 = */           NULL,
+    /* .pfnReserved2 = */           NULL,
+    /* .pfnReserved3 = */           NULL,
+    /* .pfnReserved4 = */           NULL,
+    /* .pfnReserved5 = */           NULL,
+    /* .pfnReserved6 = */           NULL,
+    /* .pfnReserved7 = */           NULL,
+#elif defined(IN_RING0)
+    /* .pfnEarlyConstruct = */      NULL,
+    /* .pfnConstruct = */           apicRZConstruct,
+    /* .pfnDestruct = */            NULL,
+    /* .pfnFinalDestruct = */       NULL,
+    /* .pfnRequest = */             NULL,
+    /* .pfnReserved0 = */           NULL,
+    /* .pfnReserved1 = */           NULL,
+    /* .pfnReserved2 = */           NULL,
+    /* .pfnReserved3 = */           NULL,
+    /* .pfnReserved4 = */           NULL,
+    /* .pfnReserved5 = */           NULL,
+    /* .pfnReserved6 = */           NULL,
+    /* .pfnReserved7 = */           NULL,
+#elif defined(IN_RC)
+    /* .pfnConstruct = */           apicRZConstruct,
+    /* .pfnReserved0 = */           NULL,
+    /* .pfnReserved1 = */           NULL,
+    /* .pfnReserved2 = */           NULL,
+    /* .pfnReserved3 = */           NULL,
+    /* .pfnReserved4 = */           NULL,
+    /* .pfnReserved5 = */           NULL,
+    /* .pfnReserved6 = */           NULL,
+    /* .pfnReserved7 = */           NULL,
+#else
+# error "Not in IN_RING3, IN_RING0 or IN_RC!"
+#endif
+    /* .u32VersionEnd = */          PDM_DEVREG_VERSION
+};
+
diff --git a/src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp b/src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp
new file mode 100644
index 00000000..11445efb
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp
@@ -0,0 +1,101 @@
+/* $Id: AllPdbTypeHack.cpp $ */
+/** @file
+ * Debug info hack for the VM and VMCPU structures.
+ */
+
+/*
+ * Copyright (C) 2016-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/stam.h>
+#include "../include/PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include "../include/CFGMInternal.h"
+#include "../include/CPUMInternal.h"
+#include "../include/MMInternal.h"
+#include "../include/PGMInternal.h"
+#include "../include/SELMInternal.h"
+#include "../include/TRPMInternal.h"
+#include "../include/TMInternal.h"
+#include "../include/IOMInternal.h"
+#ifdef IN_RING3
+# include "../include/SSMInternal.h"
+#endif
+#include "../include/HMInternal.h"
+#include "../include/VMMInternal.h"
+#include "../include/DBGFInternal.h"
+#include "../include/GIMInternal.h"
+#include "../include/APICInternal.h"
+#include "../include/STAMInternal.h"
+#include "../include/VMInternal.h"
+#include "../include/EMInternal.h"
+#include "../include/IEMInternal.h"
+#include "../include/NEMInternal.h"
+#include "../VMMR0/GMMR0Internal.h"
+#include "../VMMR0/GVMMR0Internal.h"
+#include <VBox/vmm/vmcc.h>
+#ifdef IN_RING3
+# include <VBox/vmm/uvm.h>
+#endif
+#include <VBox/vmm/gvm.h>
+
+
+extern "C" {
+
+/* Global pointer variables as an alternative to the parameter list.  Just to ensure the precense of the types. */
+PVM                 g_PdbTypeHack1 = NULL;
+PVMCPU              g_PdbTypeHack2 = NULL;
+PPDMCRITSECT        g_PdbTypeHack3 = NULL;
+PPDMCRITSECTRW      g_PdbTypeHack4 = NULL;
+PPDMDEVINS          g_PdbTypeHack5 = NULL;
+PPDMDRVINS          g_PdbTypeHack6 = NULL;
+PPDMUSBINS          g_PdbTypeHack7 = NULL;
+PCVMCPU             g_PdbTypeHack8 = NULL;
+CTX_SUFF(PVM)       g_PdbTypeHack9 = NULL;
+CTX_SUFF(PVMCPU)    g_PdbTypeHack10 = NULL;
+
+DECLEXPORT(uint32_t) PdbTypeHack(PVM pVM, PVMCPU pVCpu, PPDMCRITSECT pCs1, PPDMCRITSECTRW pCs2);
+}
+
+DECLEXPORT(uint32_t) PdbTypeHack(PVM pVM, PVMCPU pVCpu, PPDMCRITSECT pCs1, PPDMCRITSECTRW pCs2)
+{
+    /* Just some dummy operations accessing each type. Probably not necessary, but
+       helps making sure we've included all we need to get at the internal stuff.. */
+    return pVM->fGlobalForcedActions
+         | (pVM == g_PdbTypeHack1)
+         | (pVM == g_PdbTypeHack9)
+         | pVCpu->fLocalForcedActions
+         | (pVCpu == g_PdbTypeHack2)
+         | (pVCpu == g_PdbTypeHack8)
+         | (pVCpu == g_PdbTypeHack10)
+         | pCs1->s.Core.fFlags
+         | (pCs1 == g_PdbTypeHack3)
+         | pCs2->s.Core.fFlags
+         | (pCs2 == g_PdbTypeHack4)
+         | g_PdbTypeHack5->Internal.s.idxR0Device
+         | (g_PdbTypeHack5 != NULL)
+         | (uint32_t)g_PdbTypeHack6->Internal.s.fDetaching
+         | (g_PdbTypeHack6 != NULL)
+         | (uint32_t)g_PdbTypeHack7->Internal.s.fVMSuspended
+         | (g_PdbTypeHack7 != NULL);
+}
+
diff --git a/src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp b/src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp
new file mode 100644
index 00000000..34bc5d4e
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp
@@ -0,0 +1,6414 @@
+/* $Id: CPUMAllMsrs.cpp $ */
+/** @file
+ * CPUM - CPU MSR Registers.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_CPUM
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/hm_vmx.h>
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+# include <VBox/vmm/iem.h>
+#endif
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/gim.h>
+#include "CPUMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/**
+ * Validates the CPUMMSRRANGE::offCpumCpu value and declares a local variable
+ * pointing to it.
+ *
+ * ASSUMES sizeof(a_Type) is a power of two and that the member is aligned
+ * correctly.
+ */
+#define CPUM_MSR_ASSERT_CPUMCPU_OFFSET_RETURN(a_pVCpu, a_pRange, a_Type, a_VarName) \
+    AssertMsgReturn(   (a_pRange)->offCpumCpu >= 8 \
+                    && (a_pRange)->offCpumCpu < sizeof(CPUMCPU) \
+                    && !((a_pRange)->offCpumCpu & (RT_MIN(sizeof(a_Type), 8) - 1)) \
+                    , ("offCpumCpu=%#x %s\n", (a_pRange)->offCpumCpu, (a_pRange)->szName), \
+                    VERR_CPUM_MSR_BAD_CPUMCPU_OFFSET); \
+    a_Type *a_VarName = (a_Type *)((uintptr_t)&(a_pVCpu)->cpum.s + (a_pRange)->offCpumCpu)
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+
+/**
+ * Implements reading one or more MSRs.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_CPUM_R3_MSR_READ if the MSR read could not be serviced in the
+ *          current context (raw-mode or ring-0).
+ * @retval  VERR_CPUM_RAISE_GP_0 on failure (invalid MSR).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR we're reading.
+ * @param   pRange      The MSR range descriptor.
+ * @param   puValue     Where to return the value.
+ */
+typedef DECLCALLBACK(VBOXSTRICTRC) FNCPUMRDMSR(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue);
+/** Pointer to a RDMSR worker for a specific MSR or range of MSRs.  */
+typedef FNCPUMRDMSR *PFNCPUMRDMSR;
+
+
+/**
+ * Implements writing one or more MSRs.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_CPUM_R3_MSR_WRITE if the MSR write could not be serviced in the
+ *          current context (raw-mode or ring-0).
+ * @retval  VERR_CPUM_RAISE_GP_0 on failure.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR we're writing.
+ * @param   pRange      The MSR range descriptor.
+ * @param   uValue      The value to set, ignored bits masked.
+ * @param   uRawValue   The raw value with the ignored bits not masked.
+ */
+typedef DECLCALLBACK(VBOXSTRICTRC) FNCPUMWRMSR(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue);
+/** Pointer to a WRMSR worker for a specific MSR or range of MSRs.  */
+typedef FNCPUMWRMSR *PFNCPUMWRMSR;
+
+
+
+/*
+ * Generic functions.
+ * Generic functions.
+ * Generic functions.
+ */
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_FixedValue(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IgnoreWrite(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    Log(("CPUM: Ignoring WRMSR %#x (%s), %#llx\n", idMsr, pRange->szName, uValue));
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_WriteOnly(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(puValue);
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_ReadOnly(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    Assert(pRange->fWrGpMask == UINT64_MAX);
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+
+
+/*
+ * IA32
+ * IA32
+ * IA32
+ */
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32P5McAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0; /** @todo implement machine check injection. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32P5McAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement machine check injection. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32P5McType(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0; /** @todo implement machine check injection. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32P5McType(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement machine check injection. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32TimestampCounter(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = TMCpuTickGet(pVCpu);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    *puValue = CPUMApplyNestedGuestTscOffset(pVCpu, *puValue);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32TimestampCounter(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    TMCpuTickSet(pVCpu->CTX_SUFF(pVM), pVCpu, uValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PlatformId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    uint64_t uValue = pRange->uValue;
+    if (uValue & 0x1f00)
+    {
+        /* Max allowed bus ratio present. */
+        /** @todo Implement scaled BUS frequency. */
+    }
+
+    *puValue = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ApicBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    return APICGetBaseMsr(pVCpu, puValue);
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32ApicBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return APICSetBaseMsr(pVCpu, uValue);
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32FeatureControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64FeatCtrl;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32FeatureControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32BiosSignId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo fake microcode update. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32BiosSignId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* Normally, zero is written to Ia32BiosSignId before reading it in order
+       to select the signature instead of the BBL_CR_D3 behaviour.  The GP mask
+       of the database entry should take care of most illegal writes for now, so
+       just ignore all writes atm. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32BiosUpdateTrigger(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+
+    /* Microcode updates cannot be loaded in VMX non-root mode. */
+    if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest))
+        return VINF_SUCCESS;
+
+    /** @todo Fake bios update trigger better.  The value is the address to an
+     *        update package, I think.  We should probably GP if it's invalid. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get MSR_IA32_SMM_MONITOR_CTL value for IEM and cpumMsrRd_Ia32SmmMonitorCtl.
+ *
+ * @returns The MSR_IA32_SMM_MONITOR_CTL value.
+ * @param   pVCpu           The cross context per CPU structure.
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestIa32SmmMonitorCtl(PCVMCPU pVCpu)
+{
+    /* We do not support dual-monitor treatment for SMI and SMM. */
+    /** @todo SMM. */
+    RT_NOREF(pVCpu);
+    return 0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SmmMonitorCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = CPUMGetGuestIa32SmmMonitorCtl(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SmmMonitorCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo SMM. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PmcN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo check CPUID leaf 0ah. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PmcN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo check CPUID leaf 0ah. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MonitorFilterLineSize(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo return 0x1000 if we try emulate mwait 100% correctly. */
+    *puValue = 0x40; /** @todo Change to CPU cache line size. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MonitorFilterLineSize(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo should remember writes, though it's supposedly something only a BIOS
+     * would write so, it's not extremely important. */
+    return VINF_SUCCESS;
+}
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MPerf(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Read MPERF: Adjust against previously written MPERF value.  Is TSC
+     *        what we want? */
+    *puValue = TMCpuTickGet(pVCpu);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    *puValue = CPUMApplyNestedGuestTscOffset(pVCpu, *puValue);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MPerf(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Write MPERF: Calc adjustment. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32APerf(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Read APERF: Adjust against previously written MPERF value.  Is TSC
+     *        what we want? */
+    *puValue = TMCpuTickGet(pVCpu);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    *puValue = CPUMApplyNestedGuestTscOffset(pVCpu, *puValue);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32APerf(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Write APERF: Calc adjustment. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get fixed IA32_MTRR_CAP value for NEM and cpumMsrRd_Ia32MtrrCap.
+ *
+ * @returns Fixed IA32_MTRR_CAP value.
+ * @param   pVCpu           The cross context per CPU structure.
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestIa32MtrrCap(PCVMCPU pVCpu)
+{
+    RT_NOREF_PV(pVCpu);
+
+    /* This is currently a bit weird. :-) */
+    uint8_t const   cVariableRangeRegs              = 0;
+    bool const      fSystemManagementRangeRegisters = false;
+    bool const      fFixedRangeRegisters            = false;
+    bool const      fWriteCombiningType             = false;
+    return cVariableRangeRegs
+         | (fFixedRangeRegisters            ? RT_BIT_64(8)  : 0)
+         | (fWriteCombiningType             ? RT_BIT_64(10) : 0)
+         | (fSystemManagementRangeRegisters ? RT_BIT_64(11) : 0);
+}
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = CPUMGetGuestIa32MtrrCap(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrPhysBaseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Implement variable MTRR storage. */
+    Assert(pRange->uValue == (idMsr - 0x200) / 2);
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MtrrPhysBaseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    /*
+     * Validate the value.
+     */
+    Assert(pRange->uValue == (idMsr - 0x200) / 2);
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(uRawValue); RT_NOREF_PV(pRange);
+
+    uint8_t uType = uValue & 0xff;
+    if ((uType >= 7) || (uType == 2) || (uType == 3))
+    {
+        Log(("CPUM: Invalid type set writing MTRR PhysBase MSR %#x: %#llx (%#llx)\n", idMsr, uValue, uType));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    uint64_t fInvPhysMask = ~(RT_BIT_64(pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.cMaxPhysAddrWidth) - 1U);
+    if (fInvPhysMask & uValue)
+    {
+        Log(("CPUM: Invalid physical address bits set writing MTRR PhysBase MSR %#x: %#llx (%#llx)\n",
+             idMsr, uValue, uValue & fInvPhysMask));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    /*
+     * Store it.
+     */
+    /** @todo Implement variable MTRR storage. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrPhysMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Implement variable MTRR storage. */
+    Assert(pRange->uValue == (idMsr - 0x200) / 2);
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MtrrPhysMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    /*
+     * Validate the value.
+     */
+    Assert(pRange->uValue == (idMsr - 0x200) / 2);
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(uRawValue); RT_NOREF_PV(pRange);
+
+    uint64_t fInvPhysMask = ~(RT_BIT_64(pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.cMaxPhysAddrWidth) - 1U);
+    if (fInvPhysMask & uValue)
+    {
+        Log(("CPUM: Invalid physical address bits set writing MTRR PhysMask MSR %#x: %#llx (%#llx)\n",
+             idMsr, uValue, uValue & fInvPhysMask));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    /*
+     * Store it.
+     */
+    /** @todo Implement variable MTRR storage. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrFixed(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    CPUM_MSR_ASSERT_CPUMCPU_OFFSET_RETURN(pVCpu, pRange, uint64_t, puFixedMtrr);
+    *puValue = *puFixedMtrr;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MtrrFixed(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    CPUM_MSR_ASSERT_CPUMCPU_OFFSET_RETURN(pVCpu, pRange, uint64_t, puFixedMtrr);
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(uRawValue);
+
+    for (uint32_t cShift = 0; cShift < 63; cShift += 8)
+    {
+        uint8_t uType = (uint8_t)(uValue >> cShift);
+        if ((uType >= 7) || (uType == 2) || (uType == 3))
+        {
+            Log(("CPUM: Invalid MTRR type at %u:%u in fixed range (%#x/%s): %#llx (%#llx)\n",
+                 cShift + 7, cShift, idMsr, pRange->szName, uValue, uType));
+            return VERR_CPUM_RAISE_GP_0;
+        }
+    }
+    *puFixedMtrr = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrDefType(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.GuestMsrs.msr.MtrrDefType;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MtrrDefType(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+
+    uint8_t uType = uValue & 0xff;
+    if ((uType >= 7) || (uType == 2) || (uType == 3))
+    {
+        Log(("CPUM: Invalid MTRR default type value on %s: %#llx (%#llx)\n", pRange->szName, uValue, uType));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    pVCpu->cpum.s.GuestMsrs.msr.MtrrDefType = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32Pat(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrPAT;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32Pat(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    if (CPUMIsPatMsrValid(uValue))
+    {
+        pVCpu->cpum.s.Guest.msrPAT = uValue;
+        return VINF_SUCCESS;
+    }
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SysEnterCs(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.SysEnter.cs;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SysEnterCs(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+
+    /* Note! We used to mask this by 0xffff, but turns out real HW doesn't and
+             there are generally 32-bit working bits backing this register. */
+    pVCpu->cpum.s.Guest.SysEnter.cs = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SysEnterEsp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.SysEnter.esp;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SysEnterEsp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    if (X86_IS_CANONICAL(uValue))
+    {
+        pVCpu->cpum.s.Guest.SysEnter.esp = uValue;
+        return VINF_SUCCESS;
+    }
+    Log(("CPUM: IA32_SYSENTER_ESP not canonical! %#llx\n", uValue));
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SysEnterEip(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.SysEnter.eip;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SysEnterEip(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    if (X86_IS_CANONICAL(uValue))
+    {
+        pVCpu->cpum.s.Guest.SysEnter.eip = uValue;
+        return VINF_SUCCESS;
+    }
+    LogRel(("CPUM: IA32_SYSENTER_EIP not canonical! %#llx\n", uValue));
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McgCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+#if 0 /** @todo implement machine checks. */
+    *puValue = pRange->uValue & (RT_BIT_64(8) | 0);
+#else
+    *puValue = 0;
+#endif
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McgStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement machine checks. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32McgStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement machine checks. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McgCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement machine checks. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32McgCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement machine checks. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32DebugCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_DEBUGCTL. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32DebugCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_DEBUGCTL. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SmrrPhysBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement intel SMM. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SmrrPhysBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement intel SMM. */
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SmrrPhysMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement intel SMM. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SmrrPhysMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement intel SMM. */
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PlatformDcaCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement intel direct cache access (DCA)?? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PlatformDcaCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement intel direct cache access (DCA)?? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32CpuDcaCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement intel direct cache access (DCA)?? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32Dca0Cap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement intel direct cache access (DCA)?? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32Dca0Cap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement intel direct cache access (DCA)?? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfEvtSelN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_PERFEVTSEL0+. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfEvtSelN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_PERFEVTSEL0+. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    uint64_t uValue = pRange->uValue;
+
+    /* Always provide the max bus ratio for now.  XNU expects it. */
+    uValue &= ~((UINT64_C(0x1f) << 40) | RT_BIT_64(46));
+
+    PVMCC    pVM            = pVCpu->CTX_SUFF(pVM);
+    uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVM);
+    uint64_t uTscHz         = TMCpuTicksPerSecond(pVM);
+    uint8_t  uTscRatio      = (uint8_t)((uTscHz + uScalableBusHz / 2) / uScalableBusHz);
+    if (uTscRatio > 0x1f)
+        uTscRatio = 0x1f;
+    uValue |= (uint64_t)uTscRatio << 40;
+
+    *puValue = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* Pentium4 allows writing, but all bits are ignored. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_PERFCTL. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_PERFCTL. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32FixedCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_FIXED_CTRn (fixed performance counters). */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32FixedCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_FIXED_CTRn (fixed performance counters). */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfCapabilities(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfCapabilities(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32FixedCtrCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32FixedCtrCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfGlobalStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfGlobalStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfGlobalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfGlobalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfGlobalOvfCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfGlobalOvfCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PebsEnable(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PebsEnable(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ClockModulation(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_CLOCK_MODULATION. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32ClockModulation(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_CLOCK_MODULATION. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ThermInterrupt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_THERM_INTERRUPT. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32ThermInterrupt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_THERM_STATUS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ThermStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_THERM_STATUS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32ThermStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_THERM_INTERRUPT. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32Therm2Ctl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_THERM2_CTL. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32Therm2Ctl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_THERM2_CTL. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MiscEnable(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.GuestMsrs.msr.MiscEnable;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MiscEnable(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+#ifdef LOG_ENABLED
+    uint64_t const uOld = pVCpu->cpum.s.GuestMsrs.msr.MiscEnable;
+#endif
+
+    /* Unsupported bits are generally ignored and stripped by the MSR range
+       entry that got us here. So, we just need to preserve fixed bits. */
+    pVCpu->cpum.s.GuestMsrs.msr.MiscEnable = uValue
+                                           | MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL
+                                           | MSR_IA32_MISC_ENABLE_BTS_UNAVAIL;
+
+    Log(("CPUM: IA32_MISC_ENABLE; old=%#llx written=%#llx => %#llx\n",
+         uOld, uValue,  pVCpu->cpum.s.GuestMsrs.msr.MiscEnable));
+
+    /** @todo Wire IA32_MISC_ENABLE bit 22 to our NT 4 CPUID trick. */
+    /** @todo Wire up MSR_IA32_MISC_ENABLE_XD_DISABLE. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McCtlStatusAddrMiscN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(pRange);
+
+    /** @todo Implement machine check exception injection. */
+    switch (idMsr & 3)
+    {
+        case 0:
+        case 1:
+            *puValue = 0;
+            break;
+
+        /* The ADDR and MISC registers aren't accessible since the
+           corresponding STATUS bits are zero. */
+        case 2:
+            Log(("CPUM: Reading IA32_MCi_ADDR %#x -> #GP\n", idMsr));
+            return VERR_CPUM_RAISE_GP_0;
+        case 3:
+            Log(("CPUM: Reading IA32_MCi_MISC %#x -> #GP\n", idMsr));
+            return VERR_CPUM_RAISE_GP_0;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32McCtlStatusAddrMiscN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    switch (idMsr & 3)
+    {
+        case 0:
+            /* Ignore writes to the CTL register. */
+            break;
+
+        case 1:
+            /* According to specs, the STATUS register can only be written to
+               with the value 0.  VBoxCpuReport thinks different for a
+               Pentium M Dothan, but implementing according to specs now. */
+            if (uValue != 0)
+            {
+                Log(("CPUM: Writing non-zero value (%#llx) to IA32_MCi_STATUS %#x -> #GP\n", uValue, idMsr));
+                return VERR_CPUM_RAISE_GP_0;
+            }
+            break;
+
+        /* Specs states that ADDR and MISC can be cleared by writing zeros.
+           Writing 1s will GP.  Need to figure out how this relates to the
+           ADDRV and MISCV status flags.  If writing is independent of those
+           bits, we need to know whether the CPU really implements them since
+           that is exposed by writing 0 to them.
+           Implementing the solution with the fewer GPs for now. */
+        case 2:
+            if (uValue != 0)
+            {
+                Log(("CPUM: Writing non-zero value (%#llx) to IA32_MCi_ADDR %#x -> #GP\n", uValue, idMsr));
+                return VERR_CPUM_RAISE_GP_0;
+            }
+            break;
+        case 3:
+            if (uValue != 0)
+            {
+                Log(("CPUM: Writing non-zero value (%#llx) to IA32_MCi_MISC %#x -> #GP\n", uValue, idMsr));
+                return VERR_CPUM_RAISE_GP_0;
+            }
+            break;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McNCtl2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Implement machine check exception injection. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32McNCtl2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Implement machine check exception injection. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32DsArea(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_DS_AREA. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32DsArea(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32TscDeadline(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement TSC deadline timer. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32TscDeadline(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement TSC deadline timer. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32X2ApicN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pRange);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (   CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest)
+        && CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.s.Guest, VMX_PROC_CTLS2_VIRT_X2APIC_MODE))
+    {
+        VBOXSTRICTRC rcStrict = IEMExecVmxVirtApicAccessMsr(pVCpu, idMsr, puValue, false /* fWrite */);
+        if (rcStrict == VINF_VMX_MODIFIES_BEHAVIOR)
+            return VINF_SUCCESS;
+        if (rcStrict == VERR_OUT_OF_RANGE)
+            return VERR_CPUM_RAISE_GP_0;
+        Assert(rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE);
+    }
+#endif
+    return APICReadMsr(pVCpu, idMsr, puValue);
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32X2ApicN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (   CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest)
+        && CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.s.Guest, VMX_PROC_CTLS2_VIRT_X2APIC_MODE))
+    {
+        VBOXSTRICTRC rcStrict = IEMExecVmxVirtApicAccessMsr(pVCpu, idMsr, &uValue, true /* fWrite */);
+        if (rcStrict == VINF_VMX_MODIFIES_BEHAVIOR)
+            return VINF_SUCCESS;
+        if (rcStrict == VERR_OUT_OF_RANGE)
+            return VERR_CPUM_RAISE_GP_0;
+        Assert(rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE);
+    }
+#endif
+    return APICWriteMsr(pVCpu, idMsr, uValue);
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32DebugInterface(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo IA32_DEBUG_INTERFACE (no docs)  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32DebugInterface(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo IA32_DEBUG_INTERFACE (no docs)  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxBasic(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Basic;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxPinbasedCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.PinCtls.u;
+    return VINF_SUCCESS;
+}
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxProcbasedCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.ProcCtls.u;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxExitCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.ExitCtls.u;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxEntryCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.EntryCtls.u;
+    return VINF_SUCCESS;
+}
+
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxMisc(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Misc;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxCr0Fixed0(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Cr0Fixed0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxCr0Fixed1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Cr0Fixed1;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxCr4Fixed0(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Cr4Fixed0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxCr4Fixed1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Cr4Fixed1;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxVmcsEnum(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64VmcsEnum;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxProcBasedCtls2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.ProcCtls2.u;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxEptVpidCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64EptVpidCaps;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxTruePinbasedCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxTrueProcbasedCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxTrueExitCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxTrueEntryCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxVmFunc(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64VmFunc;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SpecCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.GuestMsrs.msr.SpecCtrl;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SpecCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+
+    /* NB: The STIBP bit can be set even when IBRS is present, regardless of whether STIBP is actually implemented. */
+    if (uValue & ~(MSR_IA32_SPEC_CTRL_F_IBRS | MSR_IA32_SPEC_CTRL_F_STIBP))
+    {
+        Log(("CPUM: Invalid IA32_SPEC_CTRL bits (trying to write %#llx)\n", uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    pVCpu->cpum.s.GuestMsrs.msr.SpecCtrl = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PredCmd(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ArchCapabilities(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.GuestMsrs.msr.ArchCaps;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32FlushCmd(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    if ((uValue & ~MSR_IA32_FLUSH_CMD_F_L1D) == 0)
+        return VINF_SUCCESS;
+    Log(("CPUM: Invalid MSR_IA32_FLUSH_CMD_ bits (trying to write %#llx)\n", uValue));
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+
+/*
+ * AMD64
+ * AMD64
+ * AMD64
+ */
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64Efer(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrEFER;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64Efer(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    uint64_t uValidatedEfer;
+    uint64_t const uOldEfer = pVCpu->cpum.s.Guest.msrEFER;
+    int rc = CPUMIsGuestEferMsrWriteValid(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.s.Guest.cr0, uOldEfer, uValue, &uValidatedEfer);
+    if (RT_FAILURE(rc))
+        return VERR_CPUM_RAISE_GP_0;
+
+    CPUMSetGuestEferMsrNoChecks(pVCpu, uOldEfer, uValidatedEfer);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64SyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrSTAR;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64SyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    pVCpu->cpum.s.Guest.msrSTAR = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64LongSyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrLSTAR;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64LongSyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    pVCpu->cpum.s.Guest.msrLSTAR = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64CompSyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrCSTAR;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64CompSyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    pVCpu->cpum.s.Guest.msrCSTAR = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64SyscallFlagMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrSFMASK;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64SyscallFlagMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    pVCpu->cpum.s.Guest.msrSFMASK = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64FsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.fs.u64Base;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64FsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    pVCpu->cpum.s.Guest.fs.u64Base = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64GsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.gs.u64Base;
+    return VINF_SUCCESS;
+}
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64GsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    pVCpu->cpum.s.Guest.gs.u64Base = uValue;
+    return VINF_SUCCESS;
+}
+
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64KernelGsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrKERNELGSBASE;
+    return VINF_SUCCESS;
+}
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64KernelGsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    pVCpu->cpum.s.Guest.msrKERNELGSBASE = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64TscAux(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.GuestMsrs.msr.TscAux;
+    return VINF_SUCCESS;
+}
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64TscAux(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    pVCpu->cpum.s.GuestMsrs.msr.TscAux = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/*
+ * Intel specific
+ * Intel specific
+ * Intel specific
+ */
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelEblCrPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo recalc clock frequency ratio? */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelEblCrPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Write EBL_CR_POWERON: Remember written bits. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7CoreThreadCount(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+
+    /* Note! According to cpuid_set_info in XNU (10.7.0), Westmere CPU only
+             have a 4-bit core count. */
+    uint16_t cCores   = pVCpu->CTX_SUFF(pVM)->cCpus;
+    uint16_t cThreads = cCores; /** @todo hyper-threading. */
+    *puValue = RT_MAKE_U32(cThreads, cCores);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP4EbcHardPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo P4 hard power on config */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelP4EbcHardPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo P4 hard power on config */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP4EbcSoftPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo P4 soft power on config  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelP4EbcSoftPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo P4 soft power on config */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP4EbcFrequencyId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+
+    uint64_t uValue;
+    PVMCC    pVM            = pVCpu->CTX_SUFF(pVM);
+    uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVM);
+    if (pVM->cpum.s.GuestFeatures.uModel >= 2)
+    {
+        if (uScalableBusHz <= CPUM_SBUSFREQ_100MHZ && pVM->cpum.s.GuestFeatures.uModel <= 2)
+        {
+            uScalableBusHz = CPUM_SBUSFREQ_100MHZ;
+            uValue = 0;
+        }
+        else if (uScalableBusHz <= CPUM_SBUSFREQ_133MHZ)
+        {
+            uScalableBusHz = CPUM_SBUSFREQ_133MHZ;
+            uValue = 1;
+        }
+        else if (uScalableBusHz <= CPUM_SBUSFREQ_167MHZ)
+        {
+            uScalableBusHz = CPUM_SBUSFREQ_167MHZ;
+            uValue = 3;
+        }
+        else if (uScalableBusHz <= CPUM_SBUSFREQ_200MHZ)
+        {
+            uScalableBusHz = CPUM_SBUSFREQ_200MHZ;
+            uValue = 2;
+        }
+        else if (uScalableBusHz <= CPUM_SBUSFREQ_267MHZ && pVM->cpum.s.GuestFeatures.uModel > 2)
+        {
+            uScalableBusHz = CPUM_SBUSFREQ_267MHZ;
+            uValue = 0;
+        }
+        else
+        {
+            uScalableBusHz = CPUM_SBUSFREQ_333MHZ;
+            uValue = 6;
+        }
+        uValue <<= 16;
+
+        uint64_t uTscHz    = TMCpuTicksPerSecond(pVM);
+        uint8_t  uTscRatio = (uint8_t)((uTscHz + uScalableBusHz / 2) / uScalableBusHz);
+        uValue |= (uint32_t)uTscRatio << 24;
+
+        uValue |= pRange->uValue & ~UINT64_C(0xff0f0000);
+    }
+    else
+    {
+        /* Probably more stuff here, but intel doesn't want to tell us. */
+        uValue = pRange->uValue;
+        uValue &= ~(RT_BIT_64(21) | RT_BIT_64(22) | RT_BIT_64(23)); /* 100 MHz is only documented value */
+    }
+
+    *puValue = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelP4EbcFrequencyId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo P4 bus frequency config  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP6FsbFrequency(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+
+    /* Convert the scalable bus frequency to the encoding in the intel manual (for core+). */
+    uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVCpu->CTX_SUFF(pVM));
+    if (uScalableBusHz <= CPUM_SBUSFREQ_100MHZ)
+        *puValue = 5;
+    else if (uScalableBusHz <= CPUM_SBUSFREQ_133MHZ)
+        *puValue = 1;
+    else if (uScalableBusHz <= CPUM_SBUSFREQ_167MHZ)
+        *puValue = 3;
+    else if (uScalableBusHz <= CPUM_SBUSFREQ_200MHZ)
+        *puValue = 2;
+    else if (uScalableBusHz <= CPUM_SBUSFREQ_267MHZ)
+        *puValue = 0;
+    else if (uScalableBusHz <= CPUM_SBUSFREQ_333MHZ)
+        *puValue = 4;
+    else /*if (uScalableBusHz <= CPUM_SBUSFREQ_400MHZ)*/
+        *puValue = 6;
+
+    *puValue |= pRange->uValue & ~UINT64_C(0x7);
+
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelPlatformInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+
+    /* Just indicate a fixed TSC, no turbo boost, no programmable anything. */
+    PVMCC    pVM            = pVCpu->CTX_SUFF(pVM);
+    uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVM);
+    uint64_t uTscHz         = TMCpuTicksPerSecond(pVM);
+    uint8_t  uTscRatio      = (uint8_t)((uTscHz + uScalableBusHz / 2) / uScalableBusHz);
+    uint64_t uValue         = ((uint32_t)uTscRatio << 8)   /* TSC invariant frequency. */
+                            | ((uint64_t)uTscRatio << 40); /* The max turbo frequency. */
+
+    /* Ivy bridge has a minimum operating ratio as well. */
+    if (true) /** @todo detect sandy bridge. */
+        uValue |= (uint64_t)uTscRatio << 48;
+
+    *puValue = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelFlexRatio(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+
+    uint64_t uValue = pRange->uValue & ~UINT64_C(0x1ff00);
+
+    PVMCC    pVM            = pVCpu->CTX_SUFF(pVM);
+    uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVM);
+    uint64_t uTscHz         = TMCpuTicksPerSecond(pVM);
+    uint8_t  uTscRatio      = (uint8_t)((uTscHz + uScalableBusHz / 2) / uScalableBusHz);
+    uValue |= (uint32_t)uTscRatio << 8;
+
+    *puValue = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelFlexRatio(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement writing MSR_FLEX_RATIO. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelPkgCStConfigControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.GuestMsrs.msr.PkgCStateCfgCtrl;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelPkgCStConfigControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+
+    if (pVCpu->cpum.s.GuestMsrs.msr.PkgCStateCfgCtrl & RT_BIT_64(15))
+    {
+        Log(("CPUM: WRMSR %#x (%s), %#llx: Write protected -> #GP\n", idMsr, pRange->szName, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+#if 0 /** @todo check what real (old) hardware does. */
+    if ((uValue & 7) >= 5)
+    {
+        Log(("CPUM: WRMSR %#x (%s), %#llx: Invalid limit (%d) -> #GP\n", idMsr, pRange->szName, uValue, (uint32_t)(uValue & 7)));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+#endif
+    pVCpu->cpum.s.GuestMsrs.msr.PkgCStateCfgCtrl = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelPmgIoCaptureBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement I/O mwait wakeup. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelPmgIoCaptureBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement I/O mwait wakeup. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelLastBranchFromToN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last branch records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelLastBranchFromToN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement last branch records. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelLastBranchFromN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last branch records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelLastBranchFromN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    /** @todo implement last branch records. */
+    /** @todo Probing indicates that bit 63 is settable on SandyBridge, at least
+     *        if the rest of the bits are zero.  Automatic sign extending?
+     *        Investigate! */
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelLastBranchToN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last branch records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelLastBranchToN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement last branch records. */
+    /** @todo Probing indicates that bit 63 is settable on SandyBridge, at least
+     *        if the rest of the bits are zero.  Automatic sign extending?
+     *        Investigate! */
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelLastBranchTos(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last branch records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelLastBranchTos(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement last branch records. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelBblCrCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelBblCrCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelBblCrCtl3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelBblCrCtl3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7TemperatureTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7TemperatureTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7MsrOffCoreResponseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo machine check. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7MsrOffCoreResponseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo machine check. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7MiscPwrMgmt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7MiscPwrMgmt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP6CrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr);
+    int rc = CPUMGetGuestCRx(pVCpu, pRange->uValue, puValue);
+    AssertRC(rc);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelP6CrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* This CRx interface differs from the MOV CRx, GReg interface in that
+       #GP(0) isn't raised if unsupported bits are written to.  Instead they
+       are simply ignored and masked off. (Pentium M Dothan)  */
+    /** @todo Implement MSR_P6_CRx writing.  Too much effort for very little, if
+     *        any, gain. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCpuId1FeatureMaskEcdx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement CPUID masking.  */
+    *puValue = UINT64_MAX;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCpuId1FeatureMaskEcdx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement CPUID masking.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCpuId1FeatureMaskEax(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement CPUID masking.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCpuId1FeatureMaskEax(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement CPUID masking.  */
+    return VINF_SUCCESS;
+}
+
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCpuId80000001FeatureMaskEcdx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement CPUID masking.  */
+    *puValue = UINT64_MAX;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCpuId80000001FeatureMaskEcdx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement CPUID masking.  */
+    return VINF_SUCCESS;
+}
+
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyAesNiCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement AES-NI.  */
+    *puValue = 3;  /* Bit 0 is lock bit, bit 1 disables AES-NI. That's what they say. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyAesNiCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement AES-NI.  */
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7TurboRatioLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement intel C states.  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7TurboRatioLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement intel C states.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7LbrSelect(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last-branch-records.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7LbrSelect(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement last-branch-records.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyErrorControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement memory error injection (MSR_ERROR_CONTROL).  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyErrorControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement memory error injection (MSR_ERROR_CONTROL).  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7VirtualLegacyWireCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement memory VLW?  */
+    *puValue = pRange->uValue;
+    /* Note: A20M is known to be bit 1 as this was disclosed in spec update
+       AAJ49/AAK51/????, which documents the inversion of this bit.  The
+       Sandy bridge CPU here has value 0x74, so it probably doesn't have a BIOS
+       that correct things.  Some guesses at the other bits:
+                 bit 2 = INTR
+                 bit 4 = SMI
+                 bit 5 = INIT
+                 bit 6 = NMI */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7PowerCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7PowerCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel power management  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyPebsNumAlt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel performance counters.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyPebsNumAlt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel performance counters.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7PebsLdLat(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel performance counters.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7PebsLdLat(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel performance counters.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7PkgCnResidencyN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7CoreCnResidencyN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyVrCurrentConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Figure out what MSR_VR_CURRENT_CONFIG & MSR_VR_MISC_CONFIG are.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyVrCurrentConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Figure out what MSR_VR_CURRENT_CONFIG & MSR_VR_MISC_CONFIG are.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyVrMiscConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Figure out what MSR_VR_CURRENT_CONFIG & MSR_VR_MISC_CONFIG are.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyVrMiscConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Figure out what MSR_VR_CURRENT_CONFIG & MSR_VR_MISC_CONFIG are.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyRaplPowerUnit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyRaplPowerUnit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* Note! This is documented as read only and except for a Silvermont sample has
+             always been classified as read only.  This is just here to make it compile. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyPkgCnIrtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyPkgCnIrtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel power management.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyPkgC2Residency(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyPkgC2Residency(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* Note! This is documented as read only and except for a Silvermont sample has
+             always been classified as read only.  This is just here to make it compile. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPkgPowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPkgPowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel RAPL.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPkgEnergyStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPkgPerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPkgPowerInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplDramPowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplDramPowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel RAPL.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplDramEnergyStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplDramPerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplDramPowerInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp0PowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPp0PowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel RAPL.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp0EnergyStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp0Policy(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPp0Policy(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel RAPL.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp0PerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp1PowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPp1PowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel RAPL.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp1EnergyStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp1Policy(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPp1Policy(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel RAPL.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyConfigTdpNominal(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyConfigTdpLevel1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyConfigTdpLevel2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyConfigTdpControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7IvyConfigTdpControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel power management.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyTurboActivationRatio(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7IvyTurboActivationRatio(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel power management.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfGlobalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfGlobalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfGlobalStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfGlobalStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfGlobalOvfCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfGlobalOvfCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfFixedCtrCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfFixedCtrCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfFixedCtr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfFixedCtr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncCBoxConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncArbPerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncArbPerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncArbPerfEvtSelN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncArbPerfEvtSelN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SmiCount(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+
+    /*
+     * 31:0 is SMI count (read only), 63:32 reserved.
+     * Since we don't do SMI, the count is always zero.
+     */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore2EmttmCrTablesN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement enhanced multi thread termal monitoring? */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore2EmttmCrTablesN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement enhanced multi thread termal monitoring? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore2SmmCStMiscInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo SMM & C-states? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore2SmmCStMiscInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo SMM & C-states? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore1ExtConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Core1&2 EXT_CONFIG (whatever that is)? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore1ExtConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Core1&2 EXT_CONFIG (whatever that is)? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore1DtsCalControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Core1&2(?) DTS_CAL_CTRL (whatever that is)? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore1DtsCalControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Core1&2(?) DTS_CAL_CTRL (whatever that is)? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore2PeciControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Core2+ platform environment control interface control register? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore2PeciControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Core2+ platform environment control interface control register? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelAtSilvCoreC1Recidency(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/*
+ * Multiple vendor P6 MSRs.
+ * Multiple vendor P6 MSRs.
+ * Multiple vendor P6 MSRs.
+ *
+ * These MSRs were introduced with the P6 but not elevated to architectural
+ * MSRs, despite other vendors implementing them.
+ */
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_P6LastBranchFromIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /* AMD seems to just record RIP, while intel claims to record RIP+CS.BASE
+       if I read the docs correctly, thus the need for separate functions. */
+    /** @todo implement last branch records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_P6LastBranchToIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last branch records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_P6LastIntFromIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last exception records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_P6LastIntFromIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement last exception records. */
+    /* Note! On many CPUs, the high bit of the 0x000001dd register is always writable, even when the result is
+             a non-cannonical address. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_P6LastIntToIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last exception records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_P6LastIntToIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement last exception records. */
+    return VINF_SUCCESS;
+}
+
+
+
+/*
+ * AMD specific
+ * AMD specific
+ * AMD specific
+ */
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hTscRate(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Implement TscRateMsr */
+    *puValue = RT_MAKE_U64(0, 1); /* 1.0 = reset value. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hTscRate(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Implement TscRateMsr */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hLwpCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Implement AMD LWP? (Instructions: LWPINS, LWPVAL, LLWPCB, SLWPCB) */
+    /* Note: Only listes in BKDG for Family 15H. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hLwpCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Implement AMD LWP? (Instructions: LWPINS, LWPVAL, LLWPCB, SLWPCB) */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hLwpCbAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Implement AMD LWP? (Instructions: LWPINS, LWPVAL, LLWPCB, SLWPCB) */
+    /* Note: Only listes in BKDG for Family 15H. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hLwpCbAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Implement AMD LWP? (Instructions: LWPINS, LWPVAL, LLWPCB, SLWPCB) */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hMc4MiscN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo machine check. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hMc4MiscN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo machine check. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8PerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD performance events. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8PerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD performance events. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8PerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD performance events. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8PerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD performance events. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SysCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SYS_CFG */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SysCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SYS_CFG */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8HwCr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD HW_CFG */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8HwCr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD HW_CFG */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8IorrBaseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IorrMask/IorrBase */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8IorrBaseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IorrMask/IorrBase */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8IorrMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IorrMask/IorrBase */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8IorrMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IorrMask/IorrBase */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8TopOfMemN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0;
+    /** @todo return 4GB - RamHoleSize here for TOPMEM. Figure out what to return
+     *        for TOPMEM2. */
+    //if (pRange->uValue == 0)
+    //    *puValue = _4G - RamHoleSize;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8TopOfMemN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD TOPMEM and TOPMEM2/TOM2. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8NbCfg1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD NB_CFG1 */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8NbCfg1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD NB_CFG1 */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8McXcptRedir(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo machine check. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8McXcptRedir(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo machine check. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuNameN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr);
+    PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeaf(pVCpu->CTX_SUFF(pVM), pRange->uValue / 2 + 0x80000001);
+    if (pLeaf)
+    {
+        if (!(pRange->uValue & 1))
+            *puValue = RT_MAKE_U64(pLeaf->uEax, pLeaf->uEbx);
+        else
+            *puValue = RT_MAKE_U64(pLeaf->uEcx, pLeaf->uEdx);
+    }
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuNameN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Remember guest programmed CPU name. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8HwThermalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD HTC. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8HwThermalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD HTC. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SwThermalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD STC. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SwThermalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD STC. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8FidVidControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD FIDVID_CTL. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8FidVidControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD FIDVID_CTL. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8FidVidStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD FIDVID_STATUS. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8McCtlMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD MC. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8McCtlMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD MC. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmiOnIoTrapN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM/SMI and I/O trap. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmiOnIoTrapN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM/SMI and I/O trap. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmiOnIoTrapCtlSts(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM/SMI and I/O trap. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmiOnIoTrapCtlSts(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM/SMI and I/O trap. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8IntPendingMessage(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Interrupt pending message. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8IntPendingMessage(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Interrupt pending message. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmiTriggerIoCycle(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM/SMI and trigger I/O cycle. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmiTriggerIoCycle(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM/SMI and trigger I/O cycle. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hMmioCfgBaseAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD MMIO Configuration base address. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hMmioCfgBaseAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD MMIO Configuration base address. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hTrapCtlMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD 0xc0010059. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hTrapCtlMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD 0xc0010059. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hPStateCurLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD P-states. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hPStateControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD P-states. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hPStateControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD P-states. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hPStateStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD P-states. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hPStateStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD P-states. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hPStateN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD P-states. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hPStateN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD P-states. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hCofVidControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD P-states. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hCofVidControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD P-states. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hCofVidStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD P-states. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hCofVidStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* Note! Writing 0 seems to not GP, not sure if it does anything to the value... */
+    /** @todo AMD P-states. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hCStateIoBaseAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD C-states. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hCStateIoBaseAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD C-states. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hCpuWatchdogTimer(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD machine checks. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hCpuWatchdogTimer(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD machine checks. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmmBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmmBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmmAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmmAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM. */
+    return VINF_SUCCESS;
+}
+
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmmMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmmMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8VmCr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    if (pVM->cpum.s.GuestFeatures.fSvm)
+        *puValue = MSR_K8_VM_CR_LOCK;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8VmCr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    if (pVM->cpum.s.GuestFeatures.fSvm)
+    {
+        /* Silently ignore writes to LOCK and SVM_DISABLE bit when the LOCK bit is set (see cpumMsrRd_AmdK8VmCr). */
+        if (uValue & (MSR_K8_VM_CR_DPD | MSR_K8_VM_CR_R_INIT | MSR_K8_VM_CR_DIS_A20M))
+            return VERR_CPUM_RAISE_GP_0;
+        return VINF_SUCCESS;
+    }
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8IgnNe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IGNNE\# control. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8IgnNe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IGNNE\# control. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmmCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmmCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8VmHSavePa(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.svm.uMsrHSavePa;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8VmHSavePa(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    if (uValue & UINT64_C(0xfff))
+    {
+        Log(("CPUM: Invalid setting of low 12 bits set writing host-state save area MSR %#x: %#llx\n", idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    uint64_t fInvPhysMask = ~(RT_BIT_64(pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.cMaxPhysAddrWidth) - 1U);
+    if (fInvPhysMask & uValue)
+    {
+        Log(("CPUM: Invalid physical address bits set writing host-state save area MSR %#x: %#llx (%#llx)\n",
+             idMsr, uValue, uValue & fInvPhysMask));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    pVCpu->cpum.s.Guest.hwvirt.svm.uMsrHSavePa = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hVmLockKey(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SVM. */
+    *puValue = 0; /* RAZ */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hVmLockKey(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SVM. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hSmmLockKey(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM. */
+    *puValue = 0; /* RAZ */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hSmmLockKey(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hLocalSmiStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM/SMI. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hLocalSmiStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM/SMI. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hOsVisWrkIdLength(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo AMD OS visible workaround. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hOsVisWrkIdLength(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD OS visible workaround. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hOsVisWrkStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD OS visible workaround. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hOsVisWrkStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD OS visible workaround. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam16hL2IPerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD L2I performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam16hL2IPerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD L2I performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam16hL2IPerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD L2I performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam16hL2IPerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD L2I performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hNorthbridgePerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD Northbridge performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hNorthbridgePerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD Northbridge performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hNorthbridgePerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD Northbridge performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hNorthbridgePerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD Northbridge performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7MicrocodeCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus. Need to be explored and verify K7 presence. */
+    /** @todo Undocumented register only seen mentioned in fam15h erratum \#608. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7MicrocodeCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo Undocumented register only seen mentioned in fam15h erratum \#608. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7ClusterIdMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus. Need to be explored and verify K7 presence. */
+    /** @todo Undocumented register only seen mentioned in fam16h BKDG r3.00 when
+     *        describing EBL_CR_POWERON. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7ClusterIdMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo Undocumented register only seen mentioned in fam16h BKDG r3.00 when
+     *        describing EBL_CR_POWERON. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuIdCtlStd07hEbax(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    bool           fIgnored;
+    PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeafEx(pVCpu->CTX_SUFF(pVM), 0x00000007, 0, &fIgnored);
+    if (pLeaf)
+        *puValue = RT_MAKE_U64(pLeaf->uEbx, pLeaf->uEax);
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuIdCtlStd07hEbax(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Changing CPUID leaf 7/0. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuIdCtlStd06hEcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeaf(pVCpu->CTX_SUFF(pVM), 0x00000006);
+    if (pLeaf)
+        *puValue = pLeaf->uEcx;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuIdCtlStd06hEcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Changing CPUID leaf 6. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuIdCtlStd01hEdcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeaf(pVCpu->CTX_SUFF(pVM), 0x00000001);
+    if (pLeaf)
+        *puValue = RT_MAKE_U64(pLeaf->uEdx, pLeaf->uEcx);
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuIdCtlStd01hEdcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Changing CPUID leaf 0x80000001. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuIdCtlExt01hEdcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeaf(pVCpu->CTX_SUFF(pVM), 0x80000001);
+    if (pLeaf)
+        *puValue = RT_MAKE_U64(pLeaf->uEdx, pLeaf->uEcx);
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuIdCtlExt01hEdcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Changing CPUID leaf 0x80000001. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8PatchLevel(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Fake AMD microcode patching.  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8PatchLoader(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Fake AMD microcode patching.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7DebugStatusMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7DebugStatusMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7BHTraceBaseMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7BHTraceBaseMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7BHTracePtrMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7BHTracePtrMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7BHTraceLimitMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7BHTraceLimitMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7HardwareDebugToolCfgMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7HardwareDebugToolCfgMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7FastFlushCountMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7FastFlushCountMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7NodeId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD node ID and bios scratch. */
+    *puValue = 0; /* nodeid = 0; nodes-per-cpu = 1 */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7NodeId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD node ID and bios scratch. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7DrXAddrMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD DRx address masking (range breakpoints). */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7DrXAddrMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD DRx address masking (range breakpoints). */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7Dr0DataMatchMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD undocument debugging features. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7Dr0DataMatchMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD undocument debugging features. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7Dr0DataMaskMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD undocument debugging features. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7Dr0DataMaskMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD undocument debugging features. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7LoadStoreCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD load-store config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7LoadStoreCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD load-store config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7InstrCacheCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD instruction cache config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7InstrCacheCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD instruction cache config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7DataCacheCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD data cache config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7DataCacheCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD data cache config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7BusUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD bus unit config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7BusUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD bus unit config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7DebugCtl2Maybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo Undocument AMD debug control register \#2. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7DebugCtl2Maybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo Undocument AMD debug control register \#2. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hFpuCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD FPU config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hFpuCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD FPU config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hDecoderCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD decoder config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hDecoderCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD decoder config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hBusUnitCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /* Note! 10h and 16h */
+    /** @todo AMD bus unit  config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hBusUnitCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* Note! 10h and 16h */
+    /** @todo AMD bus unit config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hCombUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD unit config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hCombUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD unit config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hCombUnitCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD unit config 2. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hCombUnitCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD unit config 2. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hCombUnitCfg3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD combined unit config 3. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hCombUnitCfg3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD combined unit config 3. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hExecUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD execution unit config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hExecUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD execution unit config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hLoadStoreCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD load-store config 2. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hLoadStoreCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD load-store config 2. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsFetchCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsFetchCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsFetchLinAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsFetchLinAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsFetchPhysAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsFetchPhysAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpExecCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpExecCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpRip(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpRip(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpData(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpData(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpData2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpData2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpData3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpData3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsDcLinAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsDcLinAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsDcPhysAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsDcPhysAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam14hIbsBrTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam14hIbsBrTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    return VINF_SUCCESS;
+}
+
+
+
+/*
+ * GIM MSRs.
+ * GIM MSRs.
+ * GIM MSRs.
+ */
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Gim(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+    /* Raise #GP(0) like a physical CPU would since the nested-hypervisor hasn't intercept these MSRs. */
+    if (   CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.s.Guest)
+        || CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest))
+        return VERR_CPUM_RAISE_GP_0;
+#endif
+    return GIMReadMsr(pVCpu, idMsr, pRange, puValue);
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Gim(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+    /* Raise #GP(0) like a physical CPU would since the nested-hypervisor hasn't intercept these MSRs. */
+    if (   CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.s.Guest)
+        || CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest))
+        return VERR_CPUM_RAISE_GP_0;
+#endif
+    return GIMWriteMsr(pVCpu, idMsr, pRange, uValue, uRawValue);
+}
+
+
+/**
+ * MSR read function table.
+ */
+static const PFNCPUMRDMSR g_aCpumRdMsrFns[kCpumMsrRdFn_End] =
+{
+    NULL, /* Invalid */
+    cpumMsrRd_FixedValue,
+    NULL, /* Alias */
+    cpumMsrRd_WriteOnly,
+    cpumMsrRd_Ia32P5McAddr,
+    cpumMsrRd_Ia32P5McType,
+    cpumMsrRd_Ia32TimestampCounter,
+    cpumMsrRd_Ia32PlatformId,
+    cpumMsrRd_Ia32ApicBase,
+    cpumMsrRd_Ia32FeatureControl,
+    cpumMsrRd_Ia32BiosSignId,
+    cpumMsrRd_Ia32SmmMonitorCtl,
+    cpumMsrRd_Ia32PmcN,
+    cpumMsrRd_Ia32MonitorFilterLineSize,
+    cpumMsrRd_Ia32MPerf,
+    cpumMsrRd_Ia32APerf,
+    cpumMsrRd_Ia32MtrrCap,
+    cpumMsrRd_Ia32MtrrPhysBaseN,
+    cpumMsrRd_Ia32MtrrPhysMaskN,
+    cpumMsrRd_Ia32MtrrFixed,
+    cpumMsrRd_Ia32MtrrDefType,
+    cpumMsrRd_Ia32Pat,
+    cpumMsrRd_Ia32SysEnterCs,
+    cpumMsrRd_Ia32SysEnterEsp,
+    cpumMsrRd_Ia32SysEnterEip,
+    cpumMsrRd_Ia32McgCap,
+    cpumMsrRd_Ia32McgStatus,
+    cpumMsrRd_Ia32McgCtl,
+    cpumMsrRd_Ia32DebugCtl,
+    cpumMsrRd_Ia32SmrrPhysBase,
+    cpumMsrRd_Ia32SmrrPhysMask,
+    cpumMsrRd_Ia32PlatformDcaCap,
+    cpumMsrRd_Ia32CpuDcaCap,
+    cpumMsrRd_Ia32Dca0Cap,
+    cpumMsrRd_Ia32PerfEvtSelN,
+    cpumMsrRd_Ia32PerfStatus,
+    cpumMsrRd_Ia32PerfCtl,
+    cpumMsrRd_Ia32FixedCtrN,
+    cpumMsrRd_Ia32PerfCapabilities,
+    cpumMsrRd_Ia32FixedCtrCtrl,
+    cpumMsrRd_Ia32PerfGlobalStatus,
+    cpumMsrRd_Ia32PerfGlobalCtrl,
+    cpumMsrRd_Ia32PerfGlobalOvfCtrl,
+    cpumMsrRd_Ia32PebsEnable,
+    cpumMsrRd_Ia32ClockModulation,
+    cpumMsrRd_Ia32ThermInterrupt,
+    cpumMsrRd_Ia32ThermStatus,
+    cpumMsrRd_Ia32Therm2Ctl,
+    cpumMsrRd_Ia32MiscEnable,
+    cpumMsrRd_Ia32McCtlStatusAddrMiscN,
+    cpumMsrRd_Ia32McNCtl2,
+    cpumMsrRd_Ia32DsArea,
+    cpumMsrRd_Ia32TscDeadline,
+    cpumMsrRd_Ia32X2ApicN,
+    cpumMsrRd_Ia32DebugInterface,
+    cpumMsrRd_Ia32VmxBasic,
+    cpumMsrRd_Ia32VmxPinbasedCtls,
+    cpumMsrRd_Ia32VmxProcbasedCtls,
+    cpumMsrRd_Ia32VmxExitCtls,
+    cpumMsrRd_Ia32VmxEntryCtls,
+    cpumMsrRd_Ia32VmxMisc,
+    cpumMsrRd_Ia32VmxCr0Fixed0,
+    cpumMsrRd_Ia32VmxCr0Fixed1,
+    cpumMsrRd_Ia32VmxCr4Fixed0,
+    cpumMsrRd_Ia32VmxCr4Fixed1,
+    cpumMsrRd_Ia32VmxVmcsEnum,
+    cpumMsrRd_Ia32VmxProcBasedCtls2,
+    cpumMsrRd_Ia32VmxEptVpidCap,
+    cpumMsrRd_Ia32VmxTruePinbasedCtls,
+    cpumMsrRd_Ia32VmxTrueProcbasedCtls,
+    cpumMsrRd_Ia32VmxTrueExitCtls,
+    cpumMsrRd_Ia32VmxTrueEntryCtls,
+    cpumMsrRd_Ia32VmxVmFunc,
+    cpumMsrRd_Ia32SpecCtrl,
+    cpumMsrRd_Ia32ArchCapabilities,
+
+    cpumMsrRd_Amd64Efer,
+    cpumMsrRd_Amd64SyscallTarget,
+    cpumMsrRd_Amd64LongSyscallTarget,
+    cpumMsrRd_Amd64CompSyscallTarget,
+    cpumMsrRd_Amd64SyscallFlagMask,
+    cpumMsrRd_Amd64FsBase,
+    cpumMsrRd_Amd64GsBase,
+    cpumMsrRd_Amd64KernelGsBase,
+    cpumMsrRd_Amd64TscAux,
+
+    cpumMsrRd_IntelEblCrPowerOn,
+    cpumMsrRd_IntelI7CoreThreadCount,
+    cpumMsrRd_IntelP4EbcHardPowerOn,
+    cpumMsrRd_IntelP4EbcSoftPowerOn,
+    cpumMsrRd_IntelP4EbcFrequencyId,
+    cpumMsrRd_IntelP6FsbFrequency,
+    cpumMsrRd_IntelPlatformInfo,
+    cpumMsrRd_IntelFlexRatio,
+    cpumMsrRd_IntelPkgCStConfigControl,
+    cpumMsrRd_IntelPmgIoCaptureBase,
+    cpumMsrRd_IntelLastBranchFromToN,
+    cpumMsrRd_IntelLastBranchFromN,
+    cpumMsrRd_IntelLastBranchToN,
+    cpumMsrRd_IntelLastBranchTos,
+    cpumMsrRd_IntelBblCrCtl,
+    cpumMsrRd_IntelBblCrCtl3,
+    cpumMsrRd_IntelI7TemperatureTarget,
+    cpumMsrRd_IntelI7MsrOffCoreResponseN,
+    cpumMsrRd_IntelI7MiscPwrMgmt,
+    cpumMsrRd_IntelP6CrN,
+    cpumMsrRd_IntelCpuId1FeatureMaskEcdx,
+    cpumMsrRd_IntelCpuId1FeatureMaskEax,
+    cpumMsrRd_IntelCpuId80000001FeatureMaskEcdx,
+    cpumMsrRd_IntelI7SandyAesNiCtl,
+    cpumMsrRd_IntelI7TurboRatioLimit,
+    cpumMsrRd_IntelI7LbrSelect,
+    cpumMsrRd_IntelI7SandyErrorControl,
+    cpumMsrRd_IntelI7VirtualLegacyWireCap,
+    cpumMsrRd_IntelI7PowerCtl,
+    cpumMsrRd_IntelI7SandyPebsNumAlt,
+    cpumMsrRd_IntelI7PebsLdLat,
+    cpumMsrRd_IntelI7PkgCnResidencyN,
+    cpumMsrRd_IntelI7CoreCnResidencyN,
+    cpumMsrRd_IntelI7SandyVrCurrentConfig,
+    cpumMsrRd_IntelI7SandyVrMiscConfig,
+    cpumMsrRd_IntelI7SandyRaplPowerUnit,
+    cpumMsrRd_IntelI7SandyPkgCnIrtlN,
+    cpumMsrRd_IntelI7SandyPkgC2Residency,
+    cpumMsrRd_IntelI7RaplPkgPowerLimit,
+    cpumMsrRd_IntelI7RaplPkgEnergyStatus,
+    cpumMsrRd_IntelI7RaplPkgPerfStatus,
+    cpumMsrRd_IntelI7RaplPkgPowerInfo,
+    cpumMsrRd_IntelI7RaplDramPowerLimit,
+    cpumMsrRd_IntelI7RaplDramEnergyStatus,
+    cpumMsrRd_IntelI7RaplDramPerfStatus,
+    cpumMsrRd_IntelI7RaplDramPowerInfo,
+    cpumMsrRd_IntelI7RaplPp0PowerLimit,
+    cpumMsrRd_IntelI7RaplPp0EnergyStatus,
+    cpumMsrRd_IntelI7RaplPp0Policy,
+    cpumMsrRd_IntelI7RaplPp0PerfStatus,
+    cpumMsrRd_IntelI7RaplPp1PowerLimit,
+    cpumMsrRd_IntelI7RaplPp1EnergyStatus,
+    cpumMsrRd_IntelI7RaplPp1Policy,
+    cpumMsrRd_IntelI7IvyConfigTdpNominal,
+    cpumMsrRd_IntelI7IvyConfigTdpLevel1,
+    cpumMsrRd_IntelI7IvyConfigTdpLevel2,
+    cpumMsrRd_IntelI7IvyConfigTdpControl,
+    cpumMsrRd_IntelI7IvyTurboActivationRatio,
+    cpumMsrRd_IntelI7UncPerfGlobalCtrl,
+    cpumMsrRd_IntelI7UncPerfGlobalStatus,
+    cpumMsrRd_IntelI7UncPerfGlobalOvfCtrl,
+    cpumMsrRd_IntelI7UncPerfFixedCtrCtrl,
+    cpumMsrRd_IntelI7UncPerfFixedCtr,
+    cpumMsrRd_IntelI7UncCBoxConfig,
+    cpumMsrRd_IntelI7UncArbPerfCtrN,
+    cpumMsrRd_IntelI7UncArbPerfEvtSelN,
+    cpumMsrRd_IntelI7SmiCount,
+    cpumMsrRd_IntelCore2EmttmCrTablesN,
+    cpumMsrRd_IntelCore2SmmCStMiscInfo,
+    cpumMsrRd_IntelCore1ExtConfig,
+    cpumMsrRd_IntelCore1DtsCalControl,
+    cpumMsrRd_IntelCore2PeciControl,
+    cpumMsrRd_IntelAtSilvCoreC1Recidency,
+
+    cpumMsrRd_P6LastBranchFromIp,
+    cpumMsrRd_P6LastBranchToIp,
+    cpumMsrRd_P6LastIntFromIp,
+    cpumMsrRd_P6LastIntToIp,
+
+    cpumMsrRd_AmdFam15hTscRate,
+    cpumMsrRd_AmdFam15hLwpCfg,
+    cpumMsrRd_AmdFam15hLwpCbAddr,
+    cpumMsrRd_AmdFam10hMc4MiscN,
+    cpumMsrRd_AmdK8PerfCtlN,
+    cpumMsrRd_AmdK8PerfCtrN,
+    cpumMsrRd_AmdK8SysCfg,
+    cpumMsrRd_AmdK8HwCr,
+    cpumMsrRd_AmdK8IorrBaseN,
+    cpumMsrRd_AmdK8IorrMaskN,
+    cpumMsrRd_AmdK8TopOfMemN,
+    cpumMsrRd_AmdK8NbCfg1,
+    cpumMsrRd_AmdK8McXcptRedir,
+    cpumMsrRd_AmdK8CpuNameN,
+    cpumMsrRd_AmdK8HwThermalCtrl,
+    cpumMsrRd_AmdK8SwThermalCtrl,
+    cpumMsrRd_AmdK8FidVidControl,
+    cpumMsrRd_AmdK8FidVidStatus,
+    cpumMsrRd_AmdK8McCtlMaskN,
+    cpumMsrRd_AmdK8SmiOnIoTrapN,
+    cpumMsrRd_AmdK8SmiOnIoTrapCtlSts,
+    cpumMsrRd_AmdK8IntPendingMessage,
+    cpumMsrRd_AmdK8SmiTriggerIoCycle,
+    cpumMsrRd_AmdFam10hMmioCfgBaseAddr,
+    cpumMsrRd_AmdFam10hTrapCtlMaybe,
+    cpumMsrRd_AmdFam10hPStateCurLimit,
+    cpumMsrRd_AmdFam10hPStateControl,
+    cpumMsrRd_AmdFam10hPStateStatus,
+    cpumMsrRd_AmdFam10hPStateN,
+    cpumMsrRd_AmdFam10hCofVidControl,
+    cpumMsrRd_AmdFam10hCofVidStatus,
+    cpumMsrRd_AmdFam10hCStateIoBaseAddr,
+    cpumMsrRd_AmdFam10hCpuWatchdogTimer,
+    cpumMsrRd_AmdK8SmmBase,
+    cpumMsrRd_AmdK8SmmAddr,
+    cpumMsrRd_AmdK8SmmMask,
+    cpumMsrRd_AmdK8VmCr,
+    cpumMsrRd_AmdK8IgnNe,
+    cpumMsrRd_AmdK8SmmCtl,
+    cpumMsrRd_AmdK8VmHSavePa,
+    cpumMsrRd_AmdFam10hVmLockKey,
+    cpumMsrRd_AmdFam10hSmmLockKey,
+    cpumMsrRd_AmdFam10hLocalSmiStatus,
+    cpumMsrRd_AmdFam10hOsVisWrkIdLength,
+    cpumMsrRd_AmdFam10hOsVisWrkStatus,
+    cpumMsrRd_AmdFam16hL2IPerfCtlN,
+    cpumMsrRd_AmdFam16hL2IPerfCtrN,
+    cpumMsrRd_AmdFam15hNorthbridgePerfCtlN,
+    cpumMsrRd_AmdFam15hNorthbridgePerfCtrN,
+    cpumMsrRd_AmdK7MicrocodeCtl,
+    cpumMsrRd_AmdK7ClusterIdMaybe,
+    cpumMsrRd_AmdK8CpuIdCtlStd07hEbax,
+    cpumMsrRd_AmdK8CpuIdCtlStd06hEcx,
+    cpumMsrRd_AmdK8CpuIdCtlStd01hEdcx,
+    cpumMsrRd_AmdK8CpuIdCtlExt01hEdcx,
+    cpumMsrRd_AmdK8PatchLevel,
+    cpumMsrRd_AmdK7DebugStatusMaybe,
+    cpumMsrRd_AmdK7BHTraceBaseMaybe,
+    cpumMsrRd_AmdK7BHTracePtrMaybe,
+    cpumMsrRd_AmdK7BHTraceLimitMaybe,
+    cpumMsrRd_AmdK7HardwareDebugToolCfgMaybe,
+    cpumMsrRd_AmdK7FastFlushCountMaybe,
+    cpumMsrRd_AmdK7NodeId,
+    cpumMsrRd_AmdK7DrXAddrMaskN,
+    cpumMsrRd_AmdK7Dr0DataMatchMaybe,
+    cpumMsrRd_AmdK7Dr0DataMaskMaybe,
+    cpumMsrRd_AmdK7LoadStoreCfg,
+    cpumMsrRd_AmdK7InstrCacheCfg,
+    cpumMsrRd_AmdK7DataCacheCfg,
+    cpumMsrRd_AmdK7BusUnitCfg,
+    cpumMsrRd_AmdK7DebugCtl2Maybe,
+    cpumMsrRd_AmdFam15hFpuCfg,
+    cpumMsrRd_AmdFam15hDecoderCfg,
+    cpumMsrRd_AmdFam10hBusUnitCfg2,
+    cpumMsrRd_AmdFam15hCombUnitCfg,
+    cpumMsrRd_AmdFam15hCombUnitCfg2,
+    cpumMsrRd_AmdFam15hCombUnitCfg3,
+    cpumMsrRd_AmdFam15hExecUnitCfg,
+    cpumMsrRd_AmdFam15hLoadStoreCfg2,
+    cpumMsrRd_AmdFam10hIbsFetchCtl,
+    cpumMsrRd_AmdFam10hIbsFetchLinAddr,
+    cpumMsrRd_AmdFam10hIbsFetchPhysAddr,
+    cpumMsrRd_AmdFam10hIbsOpExecCtl,
+    cpumMsrRd_AmdFam10hIbsOpRip,
+    cpumMsrRd_AmdFam10hIbsOpData,
+    cpumMsrRd_AmdFam10hIbsOpData2,
+    cpumMsrRd_AmdFam10hIbsOpData3,
+    cpumMsrRd_AmdFam10hIbsDcLinAddr,
+    cpumMsrRd_AmdFam10hIbsDcPhysAddr,
+    cpumMsrRd_AmdFam10hIbsCtl,
+    cpumMsrRd_AmdFam14hIbsBrTarget,
+
+    cpumMsrRd_Gim
+};
+
+
+/**
+ * MSR write function table.
+ */
+static const PFNCPUMWRMSR g_aCpumWrMsrFns[kCpumMsrWrFn_End] =
+{
+    NULL, /* Invalid */
+    cpumMsrWr_IgnoreWrite,
+    cpumMsrWr_ReadOnly,
+    NULL, /* Alias */
+    cpumMsrWr_Ia32P5McAddr,
+    cpumMsrWr_Ia32P5McType,
+    cpumMsrWr_Ia32TimestampCounter,
+    cpumMsrWr_Ia32ApicBase,
+    cpumMsrWr_Ia32FeatureControl,
+    cpumMsrWr_Ia32BiosSignId,
+    cpumMsrWr_Ia32BiosUpdateTrigger,
+    cpumMsrWr_Ia32SmmMonitorCtl,
+    cpumMsrWr_Ia32PmcN,
+    cpumMsrWr_Ia32MonitorFilterLineSize,
+    cpumMsrWr_Ia32MPerf,
+    cpumMsrWr_Ia32APerf,
+    cpumMsrWr_Ia32MtrrPhysBaseN,
+    cpumMsrWr_Ia32MtrrPhysMaskN,
+    cpumMsrWr_Ia32MtrrFixed,
+    cpumMsrWr_Ia32MtrrDefType,
+    cpumMsrWr_Ia32Pat,
+    cpumMsrWr_Ia32SysEnterCs,
+    cpumMsrWr_Ia32SysEnterEsp,
+    cpumMsrWr_Ia32SysEnterEip,
+    cpumMsrWr_Ia32McgStatus,
+    cpumMsrWr_Ia32McgCtl,
+    cpumMsrWr_Ia32DebugCtl,
+    cpumMsrWr_Ia32SmrrPhysBase,
+    cpumMsrWr_Ia32SmrrPhysMask,
+    cpumMsrWr_Ia32PlatformDcaCap,
+    cpumMsrWr_Ia32Dca0Cap,
+    cpumMsrWr_Ia32PerfEvtSelN,
+    cpumMsrWr_Ia32PerfStatus,
+    cpumMsrWr_Ia32PerfCtl,
+    cpumMsrWr_Ia32FixedCtrN,
+    cpumMsrWr_Ia32PerfCapabilities,
+    cpumMsrWr_Ia32FixedCtrCtrl,
+    cpumMsrWr_Ia32PerfGlobalStatus,
+    cpumMsrWr_Ia32PerfGlobalCtrl,
+    cpumMsrWr_Ia32PerfGlobalOvfCtrl,
+    cpumMsrWr_Ia32PebsEnable,
+    cpumMsrWr_Ia32ClockModulation,
+    cpumMsrWr_Ia32ThermInterrupt,
+    cpumMsrWr_Ia32ThermStatus,
+    cpumMsrWr_Ia32Therm2Ctl,
+    cpumMsrWr_Ia32MiscEnable,
+    cpumMsrWr_Ia32McCtlStatusAddrMiscN,
+    cpumMsrWr_Ia32McNCtl2,
+    cpumMsrWr_Ia32DsArea,
+    cpumMsrWr_Ia32TscDeadline,
+    cpumMsrWr_Ia32X2ApicN,
+    cpumMsrWr_Ia32DebugInterface,
+    cpumMsrWr_Ia32SpecCtrl,
+    cpumMsrWr_Ia32PredCmd,
+    cpumMsrWr_Ia32FlushCmd,
+
+    cpumMsrWr_Amd64Efer,
+    cpumMsrWr_Amd64SyscallTarget,
+    cpumMsrWr_Amd64LongSyscallTarget,
+    cpumMsrWr_Amd64CompSyscallTarget,
+    cpumMsrWr_Amd64SyscallFlagMask,
+    cpumMsrWr_Amd64FsBase,
+    cpumMsrWr_Amd64GsBase,
+    cpumMsrWr_Amd64KernelGsBase,
+    cpumMsrWr_Amd64TscAux,
+
+    cpumMsrWr_IntelEblCrPowerOn,
+    cpumMsrWr_IntelP4EbcHardPowerOn,
+    cpumMsrWr_IntelP4EbcSoftPowerOn,
+    cpumMsrWr_IntelP4EbcFrequencyId,
+    cpumMsrWr_IntelFlexRatio,
+    cpumMsrWr_IntelPkgCStConfigControl,
+    cpumMsrWr_IntelPmgIoCaptureBase,
+    cpumMsrWr_IntelLastBranchFromToN,
+    cpumMsrWr_IntelLastBranchFromN,
+    cpumMsrWr_IntelLastBranchToN,
+    cpumMsrWr_IntelLastBranchTos,
+    cpumMsrWr_IntelBblCrCtl,
+    cpumMsrWr_IntelBblCrCtl3,
+    cpumMsrWr_IntelI7TemperatureTarget,
+    cpumMsrWr_IntelI7MsrOffCoreResponseN,
+    cpumMsrWr_IntelI7MiscPwrMgmt,
+    cpumMsrWr_IntelP6CrN,
+    cpumMsrWr_IntelCpuId1FeatureMaskEcdx,
+    cpumMsrWr_IntelCpuId1FeatureMaskEax,
+    cpumMsrWr_IntelCpuId80000001FeatureMaskEcdx,
+    cpumMsrWr_IntelI7SandyAesNiCtl,
+    cpumMsrWr_IntelI7TurboRatioLimit,
+    cpumMsrWr_IntelI7LbrSelect,
+    cpumMsrWr_IntelI7SandyErrorControl,
+    cpumMsrWr_IntelI7PowerCtl,
+    cpumMsrWr_IntelI7SandyPebsNumAlt,
+    cpumMsrWr_IntelI7PebsLdLat,
+    cpumMsrWr_IntelI7SandyVrCurrentConfig,
+    cpumMsrWr_IntelI7SandyVrMiscConfig,
+    cpumMsrWr_IntelI7SandyRaplPowerUnit,
+    cpumMsrWr_IntelI7SandyPkgCnIrtlN,
+    cpumMsrWr_IntelI7SandyPkgC2Residency,
+    cpumMsrWr_IntelI7RaplPkgPowerLimit,
+    cpumMsrWr_IntelI7RaplDramPowerLimit,
+    cpumMsrWr_IntelI7RaplPp0PowerLimit,
+    cpumMsrWr_IntelI7RaplPp0Policy,
+    cpumMsrWr_IntelI7RaplPp1PowerLimit,
+    cpumMsrWr_IntelI7RaplPp1Policy,
+    cpumMsrWr_IntelI7IvyConfigTdpControl,
+    cpumMsrWr_IntelI7IvyTurboActivationRatio,
+    cpumMsrWr_IntelI7UncPerfGlobalCtrl,
+    cpumMsrWr_IntelI7UncPerfGlobalStatus,
+    cpumMsrWr_IntelI7UncPerfGlobalOvfCtrl,
+    cpumMsrWr_IntelI7UncPerfFixedCtrCtrl,
+    cpumMsrWr_IntelI7UncPerfFixedCtr,
+    cpumMsrWr_IntelI7UncArbPerfCtrN,
+    cpumMsrWr_IntelI7UncArbPerfEvtSelN,
+    cpumMsrWr_IntelCore2EmttmCrTablesN,
+    cpumMsrWr_IntelCore2SmmCStMiscInfo,
+    cpumMsrWr_IntelCore1ExtConfig,
+    cpumMsrWr_IntelCore1DtsCalControl,
+    cpumMsrWr_IntelCore2PeciControl,
+
+    cpumMsrWr_P6LastIntFromIp,
+    cpumMsrWr_P6LastIntToIp,
+
+    cpumMsrWr_AmdFam15hTscRate,
+    cpumMsrWr_AmdFam15hLwpCfg,
+    cpumMsrWr_AmdFam15hLwpCbAddr,
+    cpumMsrWr_AmdFam10hMc4MiscN,
+    cpumMsrWr_AmdK8PerfCtlN,
+    cpumMsrWr_AmdK8PerfCtrN,
+    cpumMsrWr_AmdK8SysCfg,
+    cpumMsrWr_AmdK8HwCr,
+    cpumMsrWr_AmdK8IorrBaseN,
+    cpumMsrWr_AmdK8IorrMaskN,
+    cpumMsrWr_AmdK8TopOfMemN,
+    cpumMsrWr_AmdK8NbCfg1,
+    cpumMsrWr_AmdK8McXcptRedir,
+    cpumMsrWr_AmdK8CpuNameN,
+    cpumMsrWr_AmdK8HwThermalCtrl,
+    cpumMsrWr_AmdK8SwThermalCtrl,
+    cpumMsrWr_AmdK8FidVidControl,
+    cpumMsrWr_AmdK8McCtlMaskN,
+    cpumMsrWr_AmdK8SmiOnIoTrapN,
+    cpumMsrWr_AmdK8SmiOnIoTrapCtlSts,
+    cpumMsrWr_AmdK8IntPendingMessage,
+    cpumMsrWr_AmdK8SmiTriggerIoCycle,
+    cpumMsrWr_AmdFam10hMmioCfgBaseAddr,
+    cpumMsrWr_AmdFam10hTrapCtlMaybe,
+    cpumMsrWr_AmdFam10hPStateControl,
+    cpumMsrWr_AmdFam10hPStateStatus,
+    cpumMsrWr_AmdFam10hPStateN,
+    cpumMsrWr_AmdFam10hCofVidControl,
+    cpumMsrWr_AmdFam10hCofVidStatus,
+    cpumMsrWr_AmdFam10hCStateIoBaseAddr,
+    cpumMsrWr_AmdFam10hCpuWatchdogTimer,
+    cpumMsrWr_AmdK8SmmBase,
+    cpumMsrWr_AmdK8SmmAddr,
+    cpumMsrWr_AmdK8SmmMask,
+    cpumMsrWr_AmdK8VmCr,
+    cpumMsrWr_AmdK8IgnNe,
+    cpumMsrWr_AmdK8SmmCtl,
+    cpumMsrWr_AmdK8VmHSavePa,
+    cpumMsrWr_AmdFam10hVmLockKey,
+    cpumMsrWr_AmdFam10hSmmLockKey,
+    cpumMsrWr_AmdFam10hLocalSmiStatus,
+    cpumMsrWr_AmdFam10hOsVisWrkIdLength,
+    cpumMsrWr_AmdFam10hOsVisWrkStatus,
+    cpumMsrWr_AmdFam16hL2IPerfCtlN,
+    cpumMsrWr_AmdFam16hL2IPerfCtrN,
+    cpumMsrWr_AmdFam15hNorthbridgePerfCtlN,
+    cpumMsrWr_AmdFam15hNorthbridgePerfCtrN,
+    cpumMsrWr_AmdK7MicrocodeCtl,
+    cpumMsrWr_AmdK7ClusterIdMaybe,
+    cpumMsrWr_AmdK8CpuIdCtlStd07hEbax,
+    cpumMsrWr_AmdK8CpuIdCtlStd06hEcx,
+    cpumMsrWr_AmdK8CpuIdCtlStd01hEdcx,
+    cpumMsrWr_AmdK8CpuIdCtlExt01hEdcx,
+    cpumMsrWr_AmdK8PatchLoader,
+    cpumMsrWr_AmdK7DebugStatusMaybe,
+    cpumMsrWr_AmdK7BHTraceBaseMaybe,
+    cpumMsrWr_AmdK7BHTracePtrMaybe,
+    cpumMsrWr_AmdK7BHTraceLimitMaybe,
+    cpumMsrWr_AmdK7HardwareDebugToolCfgMaybe,
+    cpumMsrWr_AmdK7FastFlushCountMaybe,
+    cpumMsrWr_AmdK7NodeId,
+    cpumMsrWr_AmdK7DrXAddrMaskN,
+    cpumMsrWr_AmdK7Dr0DataMatchMaybe,
+    cpumMsrWr_AmdK7Dr0DataMaskMaybe,
+    cpumMsrWr_AmdK7LoadStoreCfg,
+    cpumMsrWr_AmdK7InstrCacheCfg,
+    cpumMsrWr_AmdK7DataCacheCfg,
+    cpumMsrWr_AmdK7BusUnitCfg,
+    cpumMsrWr_AmdK7DebugCtl2Maybe,
+    cpumMsrWr_AmdFam15hFpuCfg,
+    cpumMsrWr_AmdFam15hDecoderCfg,
+    cpumMsrWr_AmdFam10hBusUnitCfg2,
+    cpumMsrWr_AmdFam15hCombUnitCfg,
+    cpumMsrWr_AmdFam15hCombUnitCfg2,
+    cpumMsrWr_AmdFam15hCombUnitCfg3,
+    cpumMsrWr_AmdFam15hExecUnitCfg,
+    cpumMsrWr_AmdFam15hLoadStoreCfg2,
+    cpumMsrWr_AmdFam10hIbsFetchCtl,
+    cpumMsrWr_AmdFam10hIbsFetchLinAddr,
+    cpumMsrWr_AmdFam10hIbsFetchPhysAddr,
+    cpumMsrWr_AmdFam10hIbsOpExecCtl,
+    cpumMsrWr_AmdFam10hIbsOpRip,
+    cpumMsrWr_AmdFam10hIbsOpData,
+    cpumMsrWr_AmdFam10hIbsOpData2,
+    cpumMsrWr_AmdFam10hIbsOpData3,
+    cpumMsrWr_AmdFam10hIbsDcLinAddr,
+    cpumMsrWr_AmdFam10hIbsDcPhysAddr,
+    cpumMsrWr_AmdFam10hIbsCtl,
+    cpumMsrWr_AmdFam14hIbsBrTarget,
+
+    cpumMsrWr_Gim
+};
+
+
+/**
+ * Looks up the range for the given MSR.
+ *
+ * @returns Pointer to the range if found, NULL if not.
+ * @param   pVM                The cross context VM structure.
+ * @param   idMsr              The MSR to look up.
+ */
+# ifndef IN_RING3
+static
+# endif
+PCPUMMSRRANGE cpumLookupMsrRange(PVM pVM, uint32_t idMsr)
+{
+    /*
+     * Binary lookup.
+     */
+    uint32_t        cRanges   = pVM->cpum.s.GuestInfo.cMsrRanges;
+    if (!cRanges)
+        return NULL;
+    PCPUMMSRRANGE   paRanges  = pVM->cpum.s.GuestInfo.CTX_SUFF(paMsrRanges);
+    for (;;)
+    {
+        uint32_t i = cRanges / 2;
+        if (idMsr < paRanges[i].uFirst)
+        {
+            if (i == 0)
+                break;
+            cRanges = i;
+        }
+        else if (idMsr > paRanges[i].uLast)
+        {
+            i++;
+            if (i >= cRanges)
+                break;
+            cRanges -= i;
+            paRanges = &paRanges[i];
+        }
+        else
+        {
+            if (paRanges[i].enmRdFn == kCpumMsrRdFn_MsrAlias)
+                return cpumLookupMsrRange(pVM, paRanges[i].uValue);
+            return &paRanges[i];
+        }
+    }
+
+# ifdef VBOX_STRICT
+    /*
+     * Linear lookup to verify the above binary search.
+     */
+    uint32_t        cLeft = pVM->cpum.s.GuestInfo.cMsrRanges;
+    PCPUMMSRRANGE   pCur  = pVM->cpum.s.GuestInfo.CTX_SUFF(paMsrRanges);
+    while (cLeft-- > 0)
+    {
+        if (idMsr >= pCur->uFirst && idMsr <= pCur->uLast)
+        {
+            AssertFailed();
+            if (pCur->enmRdFn == kCpumMsrRdFn_MsrAlias)
+                return cpumLookupMsrRange(pVM, pCur->uValue);
+            return pCur;
+        }
+        pCur++;
+    }
+# endif
+    return NULL;
+}
+
+
+/**
+ * Query a guest MSR.
+ *
+ * The caller is responsible for checking privilege if the call is the result of
+ * a RDMSR instruction.  We'll do the rest.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_CPUM_R3_MSR_READ if the MSR read could not be serviced in the
+ *          current context (raw-mode or ring-0).
+ * @retval  VERR_CPUM_RAISE_GP_0 on failure (invalid MSR), the caller is
+ *          expected to take the appropriate actions. @a *puValue is set to 0.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   idMsr               The MSR.
+ * @param   puValue             Where to return the value.
+ *
+ * @remarks This will always return the right values, even when we're in the
+ *          recompiler.
+ */
+VMMDECL(VBOXSTRICTRC) CPUMQueryGuestMsr(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *puValue)
+{
+    *puValue = 0;
+
+    VBOXSTRICTRC    rcStrict;
+    PVM             pVM    = pVCpu->CTX_SUFF(pVM);
+    PCPUMMSRRANGE   pRange = cpumLookupMsrRange(pVM, idMsr);
+    if (pRange)
+    {
+        CPUMMSRRDFN  enmRdFn = (CPUMMSRRDFN)pRange->enmRdFn;
+        AssertReturn(enmRdFn > kCpumMsrRdFn_Invalid && enmRdFn < kCpumMsrRdFn_End, VERR_CPUM_IPE_1);
+
+        PFNCPUMRDMSR pfnRdMsr = g_aCpumRdMsrFns[enmRdFn];
+        AssertReturn(pfnRdMsr, VERR_CPUM_IPE_2);
+
+        STAM_COUNTER_INC(&pRange->cReads);
+        STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrReads);
+
+        rcStrict = pfnRdMsr(pVCpu, idMsr, pRange, puValue);
+        if (rcStrict == VINF_SUCCESS)
+            Log2(("CPUM: RDMSR %#x (%s) -> %#llx\n", idMsr, pRange->szName, *puValue));
+        else if (rcStrict == VERR_CPUM_RAISE_GP_0)
+        {
+            Log(("CPUM: RDMSR %#x (%s) -> #GP(0)\n", idMsr, pRange->szName));
+            STAM_COUNTER_INC(&pRange->cGps);
+            STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrReadsRaiseGp);
+        }
+#ifndef IN_RING3
+        else if (rcStrict == VINF_CPUM_R3_MSR_READ)
+            Log(("CPUM: RDMSR %#x (%s) -> ring-3\n", idMsr, pRange->szName));
+#endif
+        else
+        {
+            Log(("CPUM: RDMSR %#x (%s) -> rcStrict=%Rrc\n", idMsr, pRange->szName, VBOXSTRICTRC_VAL(rcStrict)));
+            AssertMsgStmt(RT_FAILURE_NP(rcStrict), ("%Rrc idMsr=%#x\n", VBOXSTRICTRC_VAL(rcStrict), idMsr),
+                          rcStrict = VERR_IPE_UNEXPECTED_INFO_STATUS);
+            Assert(rcStrict != VERR_EM_INTERPRETER);
+        }
+    }
+    else
+    {
+        Log(("CPUM: Unknown RDMSR %#x -> #GP(0)\n", idMsr));
+        STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrReads);
+        STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrReadsUnknown);
+        rcStrict = VERR_CPUM_RAISE_GP_0;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Writes to a guest MSR.
+ *
+ * The caller is responsible for checking privilege if the call is the result of
+ * a WRMSR instruction.  We'll do the rest.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_CPUM_R3_MSR_WRITE if the MSR write could not be serviced in the
+ *          current context (raw-mode or ring-0).
+ * @retval  VERR_CPUM_RAISE_GP_0 on failure, the caller is expected to take the
+ *          appropriate actions.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR id.
+ * @param   uValue      The value to set.
+ *
+ * @remarks Everyone changing MSR values, including the recompiler, shall do it
+ *          by calling this method.  This makes sure we have current values and
+ *          that we trigger all the right actions when something changes.
+ *
+ *          For performance reasons, this actually isn't entirely true for some
+ *          MSRs when in HM mode.  The code here and in HM must be aware of
+ *          this.
+ */
+VMMDECL(VBOXSTRICTRC) CPUMSetGuestMsr(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t uValue)
+{
+    VBOXSTRICTRC    rcStrict;
+    PVM             pVM    = pVCpu->CTX_SUFF(pVM);
+    PCPUMMSRRANGE   pRange = cpumLookupMsrRange(pVM, idMsr);
+    if (pRange)
+    {
+        STAM_COUNTER_INC(&pRange->cWrites);
+        STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWrites);
+
+        if (!(uValue & pRange->fWrGpMask))
+        {
+            CPUMMSRWRFN  enmWrFn = (CPUMMSRWRFN)pRange->enmWrFn;
+            AssertReturn(enmWrFn > kCpumMsrWrFn_Invalid && enmWrFn < kCpumMsrWrFn_End, VERR_CPUM_IPE_1);
+
+            PFNCPUMWRMSR pfnWrMsr = g_aCpumWrMsrFns[enmWrFn];
+            AssertReturn(pfnWrMsr, VERR_CPUM_IPE_2);
+
+            uint64_t uValueAdjusted = uValue & ~pRange->fWrIgnMask;
+            if (uValueAdjusted != uValue)
+            {
+                STAM_COUNTER_INC(&pRange->cIgnoredBits);
+                STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWritesToIgnoredBits);
+            }
+
+            rcStrict = pfnWrMsr(pVCpu, idMsr, pRange, uValueAdjusted, uValue);
+            if (rcStrict == VINF_SUCCESS)
+                Log2(("CPUM: WRMSR %#x (%s), %#llx [%#llx]\n", idMsr, pRange->szName, uValueAdjusted, uValue));
+            else if (rcStrict == VERR_CPUM_RAISE_GP_0)
+            {
+                Log(("CPUM: WRMSR %#x (%s), %#llx [%#llx] -> #GP(0)\n", idMsr, pRange->szName, uValueAdjusted, uValue));
+                STAM_COUNTER_INC(&pRange->cGps);
+                STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWritesRaiseGp);
+            }
+#ifndef IN_RING3
+            else if (rcStrict == VINF_CPUM_R3_MSR_WRITE)
+                Log(("CPUM: WRMSR %#x (%s), %#llx [%#llx] -> ring-3\n", idMsr, pRange->szName, uValueAdjusted, uValue));
+#endif
+            else
+            {
+                Log(("CPUM: WRMSR %#x (%s), %#llx [%#llx] -> rcStrict=%Rrc\n",
+                     idMsr, pRange->szName, uValueAdjusted, uValue, VBOXSTRICTRC_VAL(rcStrict)));
+                AssertMsgStmt(RT_FAILURE_NP(rcStrict), ("%Rrc idMsr=%#x\n", VBOXSTRICTRC_VAL(rcStrict), idMsr),
+                              rcStrict = VERR_IPE_UNEXPECTED_INFO_STATUS);
+                Assert(rcStrict != VERR_EM_INTERPRETER);
+            }
+        }
+        else
+        {
+            Log(("CPUM: WRMSR %#x (%s), %#llx -> #GP(0) - invalid bits %#llx\n",
+                 idMsr, pRange->szName, uValue, uValue & pRange->fWrGpMask));
+            STAM_COUNTER_INC(&pRange->cGps);
+            STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWritesRaiseGp);
+            rcStrict = VERR_CPUM_RAISE_GP_0;
+        }
+    }
+    else
+    {
+        Log(("CPUM: Unknown WRMSR %#x, %#llx -> #GP(0)\n", idMsr, uValue));
+        STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWrites);
+        STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWritesUnknown);
+        rcStrict = VERR_CPUM_RAISE_GP_0;
+    }
+    return rcStrict;
+}
+
+
+#if defined(VBOX_STRICT) && defined(IN_RING3)
+/**
+ * Performs some checks on the static data related to MSRs.
+ *
+ * @returns VINF_SUCCESS on success, error on failure.
+ */
+int cpumR3MsrStrictInitChecks(void)
+{
+#define CPUM_ASSERT_RD_MSR_FN(a_Register) \
+        AssertReturn(g_aCpumRdMsrFns[kCpumMsrRdFn_##a_Register] == cpumMsrRd_##a_Register, VERR_CPUM_IPE_2);
+#define CPUM_ASSERT_WR_MSR_FN(a_Register) \
+        AssertReturn(g_aCpumWrMsrFns[kCpumMsrWrFn_##a_Register] == cpumMsrWr_##a_Register, VERR_CPUM_IPE_2);
+
+    AssertReturn(g_aCpumRdMsrFns[kCpumMsrRdFn_Invalid] == NULL, VERR_CPUM_IPE_2);
+    CPUM_ASSERT_RD_MSR_FN(FixedValue);
+    CPUM_ASSERT_RD_MSR_FN(WriteOnly);
+    CPUM_ASSERT_RD_MSR_FN(Ia32P5McAddr);
+    CPUM_ASSERT_RD_MSR_FN(Ia32P5McType);
+    CPUM_ASSERT_RD_MSR_FN(Ia32TimestampCounter);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PlatformId);
+    CPUM_ASSERT_RD_MSR_FN(Ia32ApicBase);
+    CPUM_ASSERT_RD_MSR_FN(Ia32FeatureControl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32BiosSignId);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SmmMonitorCtl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PmcN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MonitorFilterLineSize);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MPerf);
+    CPUM_ASSERT_RD_MSR_FN(Ia32APerf);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MtrrCap);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MtrrPhysBaseN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MtrrPhysMaskN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MtrrFixed);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MtrrDefType);
+    CPUM_ASSERT_RD_MSR_FN(Ia32Pat);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SysEnterCs);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SysEnterEsp);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SysEnterEip);
+    CPUM_ASSERT_RD_MSR_FN(Ia32McgCap);
+    CPUM_ASSERT_RD_MSR_FN(Ia32McgStatus);
+    CPUM_ASSERT_RD_MSR_FN(Ia32McgCtl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32DebugCtl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SmrrPhysBase);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SmrrPhysMask);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PlatformDcaCap);
+    CPUM_ASSERT_RD_MSR_FN(Ia32CpuDcaCap);
+    CPUM_ASSERT_RD_MSR_FN(Ia32Dca0Cap);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfEvtSelN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfStatus);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfCtl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32FixedCtrN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfCapabilities);
+    CPUM_ASSERT_RD_MSR_FN(Ia32FixedCtrCtrl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfGlobalStatus);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfGlobalCtrl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfGlobalOvfCtrl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PebsEnable);
+    CPUM_ASSERT_RD_MSR_FN(Ia32ClockModulation);
+    CPUM_ASSERT_RD_MSR_FN(Ia32ThermInterrupt);
+    CPUM_ASSERT_RD_MSR_FN(Ia32ThermStatus);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MiscEnable);
+    CPUM_ASSERT_RD_MSR_FN(Ia32McCtlStatusAddrMiscN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32McNCtl2);
+    CPUM_ASSERT_RD_MSR_FN(Ia32DsArea);
+    CPUM_ASSERT_RD_MSR_FN(Ia32TscDeadline);
+    CPUM_ASSERT_RD_MSR_FN(Ia32X2ApicN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32DebugInterface);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxBasic);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxPinbasedCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxProcbasedCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxExitCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxEntryCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxMisc);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxCr0Fixed0);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxCr0Fixed1);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxCr4Fixed0);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxCr4Fixed1);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxVmcsEnum);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxProcBasedCtls2);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxEptVpidCap);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxTruePinbasedCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxTrueProcbasedCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxTrueExitCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxTrueEntryCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxVmFunc);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SpecCtrl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32ArchCapabilities);
+
+    CPUM_ASSERT_RD_MSR_FN(Amd64Efer);
+    CPUM_ASSERT_RD_MSR_FN(Amd64SyscallTarget);
+    CPUM_ASSERT_RD_MSR_FN(Amd64LongSyscallTarget);
+    CPUM_ASSERT_RD_MSR_FN(Amd64CompSyscallTarget);
+    CPUM_ASSERT_RD_MSR_FN(Amd64SyscallFlagMask);
+    CPUM_ASSERT_RD_MSR_FN(Amd64FsBase);
+    CPUM_ASSERT_RD_MSR_FN(Amd64GsBase);
+    CPUM_ASSERT_RD_MSR_FN(Amd64KernelGsBase);
+    CPUM_ASSERT_RD_MSR_FN(Amd64TscAux);
+
+    CPUM_ASSERT_RD_MSR_FN(IntelEblCrPowerOn);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7CoreThreadCount);
+    CPUM_ASSERT_RD_MSR_FN(IntelP4EbcHardPowerOn);
+    CPUM_ASSERT_RD_MSR_FN(IntelP4EbcSoftPowerOn);
+    CPUM_ASSERT_RD_MSR_FN(IntelP4EbcFrequencyId);
+    CPUM_ASSERT_RD_MSR_FN(IntelP6FsbFrequency);
+    CPUM_ASSERT_RD_MSR_FN(IntelPlatformInfo);
+    CPUM_ASSERT_RD_MSR_FN(IntelFlexRatio);
+    CPUM_ASSERT_RD_MSR_FN(IntelPkgCStConfigControl);
+    CPUM_ASSERT_RD_MSR_FN(IntelPmgIoCaptureBase);
+    CPUM_ASSERT_RD_MSR_FN(IntelLastBranchFromToN);
+    CPUM_ASSERT_RD_MSR_FN(IntelLastBranchFromN);
+    CPUM_ASSERT_RD_MSR_FN(IntelLastBranchToN);
+    CPUM_ASSERT_RD_MSR_FN(IntelLastBranchTos);
+    CPUM_ASSERT_RD_MSR_FN(IntelBblCrCtl);
+    CPUM_ASSERT_RD_MSR_FN(IntelBblCrCtl3);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7TemperatureTarget);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7MsrOffCoreResponseN);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7MiscPwrMgmt);
+    CPUM_ASSERT_RD_MSR_FN(IntelP6CrN);
+    CPUM_ASSERT_RD_MSR_FN(IntelCpuId1FeatureMaskEcdx);
+    CPUM_ASSERT_RD_MSR_FN(IntelCpuId1FeatureMaskEax);
+    CPUM_ASSERT_RD_MSR_FN(IntelCpuId80000001FeatureMaskEcdx);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyAesNiCtl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7TurboRatioLimit);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7LbrSelect);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyErrorControl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7VirtualLegacyWireCap);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7PowerCtl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyPebsNumAlt);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7PebsLdLat);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7PkgCnResidencyN);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7CoreCnResidencyN);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyVrCurrentConfig);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyVrMiscConfig);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyRaplPowerUnit);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyPkgCnIrtlN);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyPkgC2Residency);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPkgPowerLimit);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPkgEnergyStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPkgPerfStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPkgPowerInfo);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplDramPowerLimit);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplDramEnergyStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplDramPerfStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplDramPowerInfo);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp0PowerLimit);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp0EnergyStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp0Policy);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp0PerfStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp1PowerLimit);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp1EnergyStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp1Policy);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7IvyConfigTdpNominal);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7IvyConfigTdpLevel1);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7IvyConfigTdpLevel2);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7IvyConfigTdpControl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7IvyTurboActivationRatio);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfGlobalCtrl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfGlobalStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfGlobalOvfCtrl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfFixedCtrCtrl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfFixedCtr);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncCBoxConfig);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncArbPerfCtrN);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncArbPerfEvtSelN);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SmiCount);
+    CPUM_ASSERT_RD_MSR_FN(IntelCore2EmttmCrTablesN);
+    CPUM_ASSERT_RD_MSR_FN(IntelCore2SmmCStMiscInfo);
+    CPUM_ASSERT_RD_MSR_FN(IntelCore1ExtConfig);
+    CPUM_ASSERT_RD_MSR_FN(IntelCore1DtsCalControl);
+    CPUM_ASSERT_RD_MSR_FN(IntelCore2PeciControl);
+    CPUM_ASSERT_RD_MSR_FN(IntelAtSilvCoreC1Recidency);
+
+    CPUM_ASSERT_RD_MSR_FN(P6LastBranchFromIp);
+    CPUM_ASSERT_RD_MSR_FN(P6LastBranchToIp);
+    CPUM_ASSERT_RD_MSR_FN(P6LastIntFromIp);
+    CPUM_ASSERT_RD_MSR_FN(P6LastIntToIp);
+
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hTscRate);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hLwpCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hLwpCbAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hMc4MiscN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8PerfCtlN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8PerfCtrN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SysCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8HwCr);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8IorrBaseN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8IorrMaskN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8TopOfMemN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8NbCfg1);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8McXcptRedir);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8CpuNameN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8HwThermalCtrl);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SwThermalCtrl);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8FidVidControl);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8FidVidStatus);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8McCtlMaskN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmiOnIoTrapN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmiOnIoTrapCtlSts);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8IntPendingMessage);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmiTriggerIoCycle);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hMmioCfgBaseAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hTrapCtlMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hPStateCurLimit);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hPStateControl);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hPStateStatus);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hPStateN);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hCofVidControl);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hCofVidStatus);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hCStateIoBaseAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hCpuWatchdogTimer);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmmBase);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmmAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmmMask);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8VmCr);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8IgnNe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmmCtl);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8VmHSavePa);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hVmLockKey);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hSmmLockKey);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hLocalSmiStatus);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hOsVisWrkIdLength);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hOsVisWrkStatus);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam16hL2IPerfCtlN);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam16hL2IPerfCtrN);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hNorthbridgePerfCtlN);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hNorthbridgePerfCtrN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7MicrocodeCtl);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7ClusterIdMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8CpuIdCtlStd07hEbax);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8CpuIdCtlStd06hEcx);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8CpuIdCtlStd01hEdcx);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8CpuIdCtlExt01hEdcx);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8PatchLevel);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7DebugStatusMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7BHTraceBaseMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7BHTracePtrMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7BHTraceLimitMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7HardwareDebugToolCfgMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7FastFlushCountMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7NodeId);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7DrXAddrMaskN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7Dr0DataMatchMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7Dr0DataMaskMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7LoadStoreCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7InstrCacheCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7DataCacheCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7BusUnitCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7DebugCtl2Maybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hFpuCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hDecoderCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hBusUnitCfg2);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hCombUnitCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hCombUnitCfg2);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hCombUnitCfg3);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hExecUnitCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hLoadStoreCfg2);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsFetchCtl);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsFetchLinAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsFetchPhysAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpExecCtl);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpRip);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpData);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpData2);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpData3);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsDcLinAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsDcPhysAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsCtl);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam14hIbsBrTarget);
+
+    CPUM_ASSERT_RD_MSR_FN(Gim)
+
+    AssertReturn(g_aCpumWrMsrFns[kCpumMsrWrFn_Invalid] == NULL, VERR_CPUM_IPE_2);
+    CPUM_ASSERT_WR_MSR_FN(Ia32P5McAddr);
+    CPUM_ASSERT_WR_MSR_FN(Ia32P5McType);
+    CPUM_ASSERT_WR_MSR_FN(Ia32TimestampCounter);
+    CPUM_ASSERT_WR_MSR_FN(Ia32ApicBase);
+    CPUM_ASSERT_WR_MSR_FN(Ia32FeatureControl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32BiosSignId);
+    CPUM_ASSERT_WR_MSR_FN(Ia32BiosUpdateTrigger);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SmmMonitorCtl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PmcN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MonitorFilterLineSize);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MPerf);
+    CPUM_ASSERT_WR_MSR_FN(Ia32APerf);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MtrrPhysBaseN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MtrrPhysMaskN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MtrrFixed);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MtrrDefType);
+    CPUM_ASSERT_WR_MSR_FN(Ia32Pat);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SysEnterCs);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SysEnterEsp);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SysEnterEip);
+    CPUM_ASSERT_WR_MSR_FN(Ia32McgStatus);
+    CPUM_ASSERT_WR_MSR_FN(Ia32McgCtl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32DebugCtl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SmrrPhysBase);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SmrrPhysMask);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PlatformDcaCap);
+    CPUM_ASSERT_WR_MSR_FN(Ia32Dca0Cap);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfEvtSelN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfStatus);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfCtl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32FixedCtrN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfCapabilities);
+    CPUM_ASSERT_WR_MSR_FN(Ia32FixedCtrCtrl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfGlobalStatus);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfGlobalCtrl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfGlobalOvfCtrl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PebsEnable);
+    CPUM_ASSERT_WR_MSR_FN(Ia32ClockModulation);
+    CPUM_ASSERT_WR_MSR_FN(Ia32ThermInterrupt);
+    CPUM_ASSERT_WR_MSR_FN(Ia32ThermStatus);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MiscEnable);
+    CPUM_ASSERT_WR_MSR_FN(Ia32McCtlStatusAddrMiscN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32McNCtl2);
+    CPUM_ASSERT_WR_MSR_FN(Ia32DsArea);
+    CPUM_ASSERT_WR_MSR_FN(Ia32TscDeadline);
+    CPUM_ASSERT_WR_MSR_FN(Ia32X2ApicN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32DebugInterface);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SpecCtrl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PredCmd);
+    CPUM_ASSERT_WR_MSR_FN(Ia32FlushCmd);
+
+    CPUM_ASSERT_WR_MSR_FN(Amd64Efer);
+    CPUM_ASSERT_WR_MSR_FN(Amd64SyscallTarget);
+    CPUM_ASSERT_WR_MSR_FN(Amd64LongSyscallTarget);
+    CPUM_ASSERT_WR_MSR_FN(Amd64CompSyscallTarget);
+    CPUM_ASSERT_WR_MSR_FN(Amd64SyscallFlagMask);
+    CPUM_ASSERT_WR_MSR_FN(Amd64FsBase);
+    CPUM_ASSERT_WR_MSR_FN(Amd64GsBase);
+    CPUM_ASSERT_WR_MSR_FN(Amd64KernelGsBase);
+    CPUM_ASSERT_WR_MSR_FN(Amd64TscAux);
+
+    CPUM_ASSERT_WR_MSR_FN(IntelEblCrPowerOn);
+    CPUM_ASSERT_WR_MSR_FN(IntelP4EbcHardPowerOn);
+    CPUM_ASSERT_WR_MSR_FN(IntelP4EbcSoftPowerOn);
+    CPUM_ASSERT_WR_MSR_FN(IntelP4EbcFrequencyId);
+    CPUM_ASSERT_WR_MSR_FN(IntelFlexRatio);
+    CPUM_ASSERT_WR_MSR_FN(IntelPkgCStConfigControl);
+    CPUM_ASSERT_WR_MSR_FN(IntelPmgIoCaptureBase);
+    CPUM_ASSERT_WR_MSR_FN(IntelLastBranchFromToN);
+    CPUM_ASSERT_WR_MSR_FN(IntelLastBranchFromN);
+    CPUM_ASSERT_WR_MSR_FN(IntelLastBranchToN);
+    CPUM_ASSERT_WR_MSR_FN(IntelLastBranchTos);
+    CPUM_ASSERT_WR_MSR_FN(IntelBblCrCtl);
+    CPUM_ASSERT_WR_MSR_FN(IntelBblCrCtl3);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7TemperatureTarget);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7MsrOffCoreResponseN);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7MiscPwrMgmt);
+    CPUM_ASSERT_WR_MSR_FN(IntelP6CrN);
+    CPUM_ASSERT_WR_MSR_FN(IntelCpuId1FeatureMaskEcdx);
+    CPUM_ASSERT_WR_MSR_FN(IntelCpuId1FeatureMaskEax);
+    CPUM_ASSERT_WR_MSR_FN(IntelCpuId80000001FeatureMaskEcdx);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyAesNiCtl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7TurboRatioLimit);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7LbrSelect);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyErrorControl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7PowerCtl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyPebsNumAlt);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7PebsLdLat);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyVrCurrentConfig);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyVrMiscConfig);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyPkgCnIrtlN);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyPkgC2Residency);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPkgPowerLimit);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7RaplDramPowerLimit);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPp0PowerLimit);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPp0Policy);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPp1PowerLimit);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPp1Policy);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7IvyConfigTdpControl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7IvyTurboActivationRatio);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfGlobalCtrl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfGlobalStatus);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfGlobalOvfCtrl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfFixedCtrCtrl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfFixedCtr);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncArbPerfCtrN);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncArbPerfEvtSelN);
+    CPUM_ASSERT_WR_MSR_FN(IntelCore2EmttmCrTablesN);
+    CPUM_ASSERT_WR_MSR_FN(IntelCore2SmmCStMiscInfo);
+    CPUM_ASSERT_WR_MSR_FN(IntelCore1ExtConfig);
+    CPUM_ASSERT_WR_MSR_FN(IntelCore1DtsCalControl);
+    CPUM_ASSERT_WR_MSR_FN(IntelCore2PeciControl);
+
+    CPUM_ASSERT_WR_MSR_FN(P6LastIntFromIp);
+    CPUM_ASSERT_WR_MSR_FN(P6LastIntToIp);
+
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hTscRate);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hLwpCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hLwpCbAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hMc4MiscN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8PerfCtlN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8PerfCtrN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SysCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8HwCr);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8IorrBaseN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8IorrMaskN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8TopOfMemN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8NbCfg1);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8McXcptRedir);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8CpuNameN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8HwThermalCtrl);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SwThermalCtrl);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8FidVidControl);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8McCtlMaskN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmiOnIoTrapN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmiOnIoTrapCtlSts);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8IntPendingMessage);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmiTriggerIoCycle);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hMmioCfgBaseAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hTrapCtlMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hPStateControl);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hPStateStatus);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hPStateN);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hCofVidControl);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hCofVidStatus);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hCStateIoBaseAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hCpuWatchdogTimer);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmmBase);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmmAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmmMask);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8VmCr);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8IgnNe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmmCtl);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8VmHSavePa);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hVmLockKey);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hSmmLockKey);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hLocalSmiStatus);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hOsVisWrkIdLength);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hOsVisWrkStatus);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam16hL2IPerfCtlN);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam16hL2IPerfCtrN);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hNorthbridgePerfCtlN);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hNorthbridgePerfCtrN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7MicrocodeCtl);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7ClusterIdMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8CpuIdCtlStd07hEbax);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8CpuIdCtlStd06hEcx);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8CpuIdCtlStd01hEdcx);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8CpuIdCtlExt01hEdcx);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8PatchLoader);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7DebugStatusMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7BHTraceBaseMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7BHTracePtrMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7BHTraceLimitMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7HardwareDebugToolCfgMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7FastFlushCountMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7NodeId);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7DrXAddrMaskN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7Dr0DataMatchMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7Dr0DataMaskMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7LoadStoreCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7InstrCacheCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7DataCacheCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7BusUnitCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7DebugCtl2Maybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hFpuCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hDecoderCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hBusUnitCfg2);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hCombUnitCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hCombUnitCfg2);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hCombUnitCfg3);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hExecUnitCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hLoadStoreCfg2);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsFetchCtl);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsFetchLinAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsFetchPhysAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpExecCtl);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpRip);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpData);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpData2);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpData3);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsDcLinAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsDcPhysAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsCtl);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam14hIbsBrTarget);
+
+    CPUM_ASSERT_WR_MSR_FN(Gim);
+
+    return VINF_SUCCESS;
+}
+#endif /* VBOX_STRICT && IN_RING3 */
+
+
+/**
+ * Gets the scalable bus frequency.
+ *
+ * The bus frequency is used as a base in several MSRs that gives the CPU and
+ * other frequency ratios.
+ *
+ * @returns Scalable bus frequency in Hz. Will not return CPUM_SBUSFREQ_UNKNOWN.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMDECL(uint64_t) CPUMGetGuestScalableBusFrequency(PVM pVM)
+{
+    uint64_t uFreq = pVM->cpum.s.GuestInfo.uScalableBusFreq;
+    if (uFreq == CPUM_SBUSFREQ_UNKNOWN)
+        uFreq = CPUM_SBUSFREQ_100MHZ;
+    return uFreq;
+}
+
+
+/**
+ * Sets the guest EFER MSR without performing any additional checks.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uOldEfer    The previous EFER MSR value.
+ * @param   uValidEfer  The new, validated EFER MSR value.
+ *
+ * @remarks One would normally call CPUMIsGuestEferMsrWriteValid() before calling
+ *          this function to change the EFER in order to perform an EFER transition.
+ */
+VMMDECL(void) CPUMSetGuestEferMsrNoChecks(PVMCPUCC pVCpu, uint64_t uOldEfer, uint64_t uValidEfer)
+{
+    pVCpu->cpum.s.Guest.msrEFER = uValidEfer;
+
+    /* AMD64 Architecture Programmer's Manual: 15.15 TLB Control; flush the TLB
+       if MSR_K6_EFER_NXE, MSR_K6_EFER_LME or MSR_K6_EFER_LMA are changed. */
+    if (   (uOldEfer                    & (MSR_K6_EFER_NXE | MSR_K6_EFER_LME | MSR_K6_EFER_LMA))
+        != (pVCpu->cpum.s.Guest.msrEFER & (MSR_K6_EFER_NXE | MSR_K6_EFER_LME | MSR_K6_EFER_LMA)))
+    {
+        /// @todo PGMFlushTLB(pVCpu, cr3, true /*fGlobal*/);
+        HMFlushTlb(pVCpu);
+
+        /* Notify PGM about NXE changes. */
+        if (   (uOldEfer                    & MSR_K6_EFER_NXE)
+            != (pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_NXE))
+            PGMNotifyNxeChanged(pVCpu, !(uOldEfer & MSR_K6_EFER_NXE));
+    }
+}
+
+
+/**
+ * Checks if a guest PAT MSR write is valid.
+ *
+ * @returns @c true if the PAT bit combination is valid, @c false otherwise.
+ * @param   uValue      The PAT MSR value.
+ */
+VMMDECL(bool) CPUMIsPatMsrValid(uint64_t uValue)
+{
+    for (uint32_t cShift = 0; cShift < 63; cShift += 8)
+    {
+        /* Check all eight bits because the top 5 bits of each byte are reserved. */
+        uint8_t uType = (uint8_t)(uValue >> cShift);
+        if ((uType >= 8) || (uType == 2) || (uType == 3))
+        {
+            Log(("CPUM: Invalid PAT type at %u:%u in IA32_PAT: %#llx (%#llx)\n", cShift + 7, cShift, uValue, uType));
+            return false;
+        }
+    }
+    return true;
+}
+
+
+/**
+ * Validates an EFER MSR write and provides the new, validated EFER MSR.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   uCr0            The CR0 of the CPU corresponding to the EFER MSR.
+ * @param   uOldEfer        Value of the previous EFER MSR on the CPU if any.
+ * @param   uNewEfer        The new EFER MSR value being written.
+ * @param   puValidEfer     Where to store the validated EFER (only updated if
+ *                          this function returns VINF_SUCCESS).
+ */
+VMMDECL(int) CPUMIsGuestEferMsrWriteValid(PVM pVM, uint64_t uCr0, uint64_t uOldEfer, uint64_t uNewEfer, uint64_t *puValidEfer)
+{
+    /* #GP(0) If anything outside the allowed bits is set. */
+    uint64_t fMask = CPUMGetGuestEferMsrValidMask(pVM);
+    if (uNewEfer & ~fMask)
+    {
+        Log(("CPUM: Settings disallowed EFER bit. uNewEfer=%#RX64 fAllowed=%#RX64 -> #GP(0)\n", uNewEfer, fMask));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    /* Check for illegal MSR_K6_EFER_LME transitions: not allowed to change LME if
+       paging is enabled. (AMD Arch. Programmer's Manual Volume 2: Table 14-5) */
+    if (   (uOldEfer & MSR_K6_EFER_LME) != (uNewEfer & MSR_K6_EFER_LME)
+        && (uCr0 & X86_CR0_PG))
+    {
+        Log(("CPUM: Illegal MSR_K6_EFER_LME change: paging is enabled!!\n"));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    /* There are a few more: e.g. MSR_K6_EFER_LMSLE. */
+    AssertMsg(!(uNewEfer & ~(  MSR_K6_EFER_NXE
+                             | MSR_K6_EFER_LME
+                             | MSR_K6_EFER_LMA /* ignored anyway */
+                             | MSR_K6_EFER_SCE
+                             | MSR_K6_EFER_FFXSR
+                             | MSR_K6_EFER_SVME)),
+              ("Unexpected value %#RX64\n", uNewEfer));
+
+    /* Ignore EFER.LMA, it's updated when setting CR0. */
+    fMask &= ~MSR_K6_EFER_LMA;
+
+    *puValidEfer = (uOldEfer & ~fMask) | (uNewEfer & fMask);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the mask of valid EFER bits depending on supported guest-CPU features.
+ *
+ * @returns Mask of valid EFER bits.
+ * @param   pVM     The cross context VM structure.
+ *
+ * @remarks EFER.LMA is included as part of the valid mask. It's not invalid but
+ *          rather a read-only bit.
+ */
+VMMDECL(uint64_t) CPUMGetGuestEferMsrValidMask(PVM pVM)
+{
+    uint32_t const  fExtFeatures = pVM->cpum.s.aGuestCpuIdPatmExt[0].uEax >= 0x80000001
+                                 ? pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx
+                                 : 0;
+    uint64_t        fMask        = 0;
+    uint64_t const  fIgnoreMask  = MSR_K6_EFER_LMA;
+
+    /* Filter out those bits the guest is allowed to change. (e.g. LMA is read-only) */
+    if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_NX)
+        fMask |= MSR_K6_EFER_NXE;
+    if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
+        fMask |= MSR_K6_EFER_LME;
+    if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_SYSCALL)
+        fMask |= MSR_K6_EFER_SCE;
+    if (fExtFeatures & X86_CPUID_AMD_FEATURE_EDX_FFXSR)
+        fMask |= MSR_K6_EFER_FFXSR;
+    if (pVM->cpum.s.GuestFeatures.fSvm)
+        fMask |= MSR_K6_EFER_SVME;
+
+    return (fIgnoreMask | fMask);
+}
+
+
+/**
+ * Fast way for HM to access the MSR_K8_TSC_AUX register.
+ *
+ * @returns The register value.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestTscAux(PVMCPUCC pVCpu)
+{
+    Assert(!(pVCpu->cpum.s.Guest.fExtrn & CPUMCTX_EXTRN_TSC_AUX));
+    return pVCpu->cpum.s.GuestMsrs.msr.TscAux;
+}
+
+
+/**
+ * Fast way for HM to access the MSR_K8_TSC_AUX register.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   uValue  The new value.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(void) CPUMSetGuestTscAux(PVMCPUCC pVCpu, uint64_t uValue)
+{
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_TSC_AUX;
+    pVCpu->cpum.s.GuestMsrs.msr.TscAux = uValue;
+}
+
+
+/**
+ * Fast way for HM to access the IA32_SPEC_CTRL register.
+ *
+ * @returns The register value.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestSpecCtrl(PVMCPUCC pVCpu)
+{
+    return pVCpu->cpum.s.GuestMsrs.msr.SpecCtrl;
+}
+
+
+/**
+ * Fast way for HM to access the IA32_SPEC_CTRL register.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   uValue  The new value.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(void) CPUMSetGuestSpecCtrl(PVMCPUCC pVCpu, uint64_t uValue)
+{
+    pVCpu->cpum.s.GuestMsrs.msr.SpecCtrl = uValue;
+}
+
diff --git a/src/VBox/VMM/VMMAll/CPUMAllRegs.cpp b/src/VBox/VMM/VMMAll/CPUMAllRegs.cpp
new file mode 100644
index 00000000..6f0e0e4c
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/CPUMAllRegs.cpp
@@ -0,0 +1,3033 @@
+/* $Id: CPUMAllRegs.cpp $ */
+/** @file
+ * CPUM - CPU Monitor(/Manager) - Getters and Setters.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_CPUM
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/hm.h>
+#include "CPUMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/dis.h>
+#include <VBox/log.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/tm.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#ifdef IN_RING3
+# include <iprt/thread.h>
+#endif
+
+/** Disable stack frame pointer generation here. */
+#if defined(_MSC_VER) && !defined(DEBUG) && defined(RT_ARCH_X86)
+# pragma optimize("y", off)
+#endif
+
+AssertCompile2MemberOffsets(VM, cpum.s.HostFeatures,  cpum.ro.HostFeatures);
+AssertCompile2MemberOffsets(VM, cpum.s.GuestFeatures, cpum.ro.GuestFeatures);
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/**
+ * Converts a CPUMCPU::Guest pointer into a VMCPU pointer.
+ *
+ * @returns Pointer to the Virtual CPU.
+ * @param   a_pGuestCtx     Pointer to the guest context.
+ */
+#define CPUM_GUEST_CTX_TO_VMCPU(a_pGuestCtx) RT_FROM_MEMBER(a_pGuestCtx, VMCPU, cpum.s.Guest)
+
+/**
+ * Lazily loads the hidden parts of a selector register when using raw-mode.
+ */
+#define CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(a_pVCpu, a_pSReg) \
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(a_pVCpu, a_pSReg))
+
+/** @def CPUM_INT_ASSERT_NOT_EXTRN
+ * Macro for asserting that @a a_fNotExtrn are present.
+ *
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling EMT.
+ * @param   a_fNotExtrn     Mask of CPUMCTX_EXTRN_XXX bits to check.
+ */
+#define CPUM_INT_ASSERT_NOT_EXTRN(a_pVCpu, a_fNotExtrn) \
+    AssertMsg(!((a_pVCpu)->cpum.s.Guest.fExtrn & (a_fNotExtrn)), \
+              ("%#RX64; a_fNotExtrn=%#RX64\n", (a_pVCpu)->cpum.s.Guest.fExtrn, (a_fNotExtrn)))
+
+
+VMMDECL(void) CPUMSetHyperCR3(PVMCPU pVCpu, uint32_t cr3)
+{
+    pVCpu->cpum.s.Hyper.cr3 = cr3;
+}
+
+VMMDECL(uint32_t) CPUMGetHyperCR3(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.cr3;
+}
+
+
+/** @def MAYBE_LOAD_DRx
+ * Macro for updating DRx values in raw-mode and ring-0 contexts.
+ */
+#ifdef IN_RING0
+# define MAYBE_LOAD_DRx(a_pVCpu, a_fnLoad, a_uValue) do { a_fnLoad(a_uValue); } while (0)
+#else
+# define MAYBE_LOAD_DRx(a_pVCpu, a_fnLoad, a_uValue) do { } while (0)
+#endif
+
+VMMDECL(void) CPUMSetHyperDR0(PVMCPU pVCpu, RTGCUINTREG uDr0)
+{
+    pVCpu->cpum.s.Hyper.dr[0] = uDr0;
+    MAYBE_LOAD_DRx(pVCpu, ASMSetDR0, uDr0);
+}
+
+
+VMMDECL(void) CPUMSetHyperDR1(PVMCPU pVCpu, RTGCUINTREG uDr1)
+{
+    pVCpu->cpum.s.Hyper.dr[1] = uDr1;
+    MAYBE_LOAD_DRx(pVCpu, ASMSetDR1, uDr1);
+}
+
+
+VMMDECL(void) CPUMSetHyperDR2(PVMCPU pVCpu, RTGCUINTREG uDr2)
+{
+    pVCpu->cpum.s.Hyper.dr[2] = uDr2;
+    MAYBE_LOAD_DRx(pVCpu, ASMSetDR2, uDr2);
+}
+
+
+VMMDECL(void) CPUMSetHyperDR3(PVMCPU pVCpu, RTGCUINTREG uDr3)
+{
+    pVCpu->cpum.s.Hyper.dr[3] = uDr3;
+    MAYBE_LOAD_DRx(pVCpu, ASMSetDR3, uDr3);
+}
+
+
+VMMDECL(void) CPUMSetHyperDR6(PVMCPU pVCpu, RTGCUINTREG uDr6)
+{
+    pVCpu->cpum.s.Hyper.dr[6] = uDr6;
+}
+
+
+VMMDECL(void) CPUMSetHyperDR7(PVMCPU pVCpu, RTGCUINTREG uDr7)
+{
+    pVCpu->cpum.s.Hyper.dr[7] = uDr7;
+}
+
+
+VMMDECL(RTGCUINTREG) CPUMGetHyperDR0(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.dr[0];
+}
+
+
+VMMDECL(RTGCUINTREG) CPUMGetHyperDR1(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.dr[1];
+}
+
+
+VMMDECL(RTGCUINTREG) CPUMGetHyperDR2(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.dr[2];
+}
+
+
+VMMDECL(RTGCUINTREG) CPUMGetHyperDR3(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.dr[3];
+}
+
+
+VMMDECL(RTGCUINTREG) CPUMGetHyperDR6(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.dr[6];
+}
+
+
+VMMDECL(RTGCUINTREG) CPUMGetHyperDR7(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.dr[7];
+}
+
+
+/**
+ * Gets the pointer to the internal CPUMCTXCORE structure.
+ * This is only for reading in order to save a few calls.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(PCCPUMCTXCORE) CPUMGetGuestCtxCore(PVMCPU pVCpu)
+{
+    return CPUMCTX2CORE(&pVCpu->cpum.s.Guest);
+}
+
+
+/**
+ * Queries the pointer to the internal CPUMCTX structure.
+ *
+ * @returns The CPUMCTX pointer.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(PCPUMCTX) CPUMQueryGuestCtxPtr(PVMCPU pVCpu)
+{
+    return &pVCpu->cpum.s.Guest;
+}
+
+
+/**
+ * Queries the pointer to the internal CPUMCTXMSRS structure.
+ *
+ * This is for NEM only.
+ *
+ * @returns The CPUMCTX pointer.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(PCPUMCTXMSRS) CPUMQueryGuestCtxMsrsPtr(PVMCPU pVCpu)
+{
+    return &pVCpu->cpum.s.GuestMsrs;
+}
+
+
+VMMDECL(int) CPUMSetGuestGDTR(PVMCPU pVCpu, uint64_t GCPtrBase, uint16_t cbLimit)
+{
+    pVCpu->cpum.s.Guest.gdtr.cbGdt = cbLimit;
+    pVCpu->cpum.s.Guest.gdtr.pGdt  = GCPtrBase;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_GDTR;
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_GDTR;
+    return VINF_SUCCESS; /* formality, consider it void. */
+}
+
+
+VMMDECL(int) CPUMSetGuestIDTR(PVMCPU pVCpu, uint64_t GCPtrBase, uint16_t cbLimit)
+{
+    pVCpu->cpum.s.Guest.idtr.cbIdt = cbLimit;
+    pVCpu->cpum.s.Guest.idtr.pIdt  = GCPtrBase;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_IDTR;
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_IDTR;
+    return VINF_SUCCESS; /* formality, consider it void. */
+}
+
+
+VMMDECL(int) CPUMSetGuestTR(PVMCPU pVCpu, uint16_t tr)
+{
+    pVCpu->cpum.s.Guest.tr.Sel  = tr;
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_TR;
+    return VINF_SUCCESS; /* formality, consider it void. */
+}
+
+
+VMMDECL(int) CPUMSetGuestLDTR(PVMCPU pVCpu, uint16_t ldtr)
+{
+    pVCpu->cpum.s.Guest.ldtr.Sel      = ldtr;
+    /* The caller will set more hidden bits if it has them. */
+    pVCpu->cpum.s.Guest.ldtr.ValidSel = 0;
+    pVCpu->cpum.s.Guest.ldtr.fFlags   = 0;
+    pVCpu->cpum.s.fChanged  |= CPUM_CHANGED_LDTR;
+    return VINF_SUCCESS; /* formality, consider it void. */
+}
+
+
+/**
+ * Set the guest CR0.
+ *
+ * When called in GC, the hyper CR0 may be updated if that is
+ * required. The caller only has to take special action if AM,
+ * WP, PG or PE changes.
+ *
+ * @returns VINF_SUCCESS (consider it void).
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   cr0     The new CR0 value.
+ */
+VMMDECL(int) CPUMSetGuestCR0(PVMCPUCC pVCpu, uint64_t cr0)
+{
+    /*
+     * Check for changes causing TLB flushes (for REM).
+     * The caller is responsible for calling PGM when appropriate.
+     */
+    if (    (cr0                     & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE))
+        !=  (pVCpu->cpum.s.Guest.cr0 & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE)))
+        pVCpu->cpum.s.fChanged |= CPUM_CHANGED_GLOBAL_TLB_FLUSH;
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_CR0;
+
+    /*
+     * Let PGM know if the WP goes from 0 to 1 (netware WP0+RO+US hack)
+     */
+    if (((cr0 ^ pVCpu->cpum.s.Guest.cr0) & X86_CR0_WP) && (cr0 & X86_CR0_WP))
+        PGMCr0WpEnabled(pVCpu);
+
+    /* The ET flag is settable on a 386 and hardwired on 486+. */
+    if (   !(cr0 & X86_CR0_ET)
+        && pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386)
+        cr0 |= X86_CR0_ET;
+
+    pVCpu->cpum.s.Guest.cr0 = cr0;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR0;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestCR2(PVMCPU pVCpu, uint64_t cr2)
+{
+    pVCpu->cpum.s.Guest.cr2 = cr2;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR2;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestCR3(PVMCPU pVCpu, uint64_t cr3)
+{
+    pVCpu->cpum.s.Guest.cr3 = cr3;
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_CR3;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR3;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestCR4(PVMCPU pVCpu, uint64_t cr4)
+{
+    /* Note! We don't bother with OSXSAVE and legacy CPUID patches. */
+
+    if (   (cr4                     & (X86_CR4_PGE | X86_CR4_PAE | X86_CR4_PSE))
+        != (pVCpu->cpum.s.Guest.cr4 & (X86_CR4_PGE | X86_CR4_PAE | X86_CR4_PSE)))
+        pVCpu->cpum.s.fChanged |= CPUM_CHANGED_GLOBAL_TLB_FLUSH;
+
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_CR4;
+    pVCpu->cpum.s.Guest.cr4 = cr4;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR4;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEFlags(PVMCPU pVCpu, uint32_t eflags)
+{
+    pVCpu->cpum.s.Guest.eflags.u32 = eflags;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEIP(PVMCPU pVCpu, uint32_t eip)
+{
+    pVCpu->cpum.s.Guest.eip = eip;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEAX(PVMCPU pVCpu, uint32_t eax)
+{
+    pVCpu->cpum.s.Guest.eax = eax;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEBX(PVMCPU pVCpu, uint32_t ebx)
+{
+    pVCpu->cpum.s.Guest.ebx = ebx;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestECX(PVMCPU pVCpu, uint32_t ecx)
+{
+    pVCpu->cpum.s.Guest.ecx = ecx;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEDX(PVMCPU pVCpu, uint32_t edx)
+{
+    pVCpu->cpum.s.Guest.edx = edx;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestESP(PVMCPU pVCpu, uint32_t esp)
+{
+    pVCpu->cpum.s.Guest.esp = esp;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEBP(PVMCPU pVCpu, uint32_t ebp)
+{
+    pVCpu->cpum.s.Guest.ebp = ebp;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestESI(PVMCPU pVCpu, uint32_t esi)
+{
+    pVCpu->cpum.s.Guest.esi = esi;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEDI(PVMCPU pVCpu, uint32_t edi)
+{
+    pVCpu->cpum.s.Guest.edi = edi;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestSS(PVMCPU pVCpu, uint16_t ss)
+{
+    pVCpu->cpum.s.Guest.ss.Sel = ss;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestCS(PVMCPU pVCpu, uint16_t cs)
+{
+    pVCpu->cpum.s.Guest.cs.Sel = cs;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestDS(PVMCPU pVCpu, uint16_t ds)
+{
+    pVCpu->cpum.s.Guest.ds.Sel = ds;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestES(PVMCPU pVCpu, uint16_t es)
+{
+    pVCpu->cpum.s.Guest.es.Sel = es;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestFS(PVMCPU pVCpu, uint16_t fs)
+{
+    pVCpu->cpum.s.Guest.fs.Sel = fs;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestGS(PVMCPU pVCpu, uint16_t gs)
+{
+    pVCpu->cpum.s.Guest.gs.Sel = gs;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(void) CPUMSetGuestEFER(PVMCPU pVCpu, uint64_t val)
+{
+    pVCpu->cpum.s.Guest.msrEFER = val;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_EFER;
+}
+
+
+VMMDECL(RTGCPTR) CPUMGetGuestIDTR(PCVMCPU pVCpu, uint16_t *pcbLimit)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_IDTR);
+    if (pcbLimit)
+        *pcbLimit = pVCpu->cpum.s.Guest.idtr.cbIdt;
+    return pVCpu->cpum.s.Guest.idtr.pIdt;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestTR(PCVMCPU pVCpu, PCPUMSELREGHID pHidden)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_TR);
+    if (pHidden)
+        *pHidden = pVCpu->cpum.s.Guest.tr;
+    return pVCpu->cpum.s.Guest.tr.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestCS(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CS);
+    return pVCpu->cpum.s.Guest.cs.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestDS(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DS);
+    return pVCpu->cpum.s.Guest.ds.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestES(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_ES);
+    return pVCpu->cpum.s.Guest.es.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestFS(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_FS);
+    return pVCpu->cpum.s.Guest.fs.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestGS(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_GS);
+    return pVCpu->cpum.s.Guest.gs.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestSS(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_SS);
+    return pVCpu->cpum.s.Guest.ss.Sel;
+}
+
+
+VMMDECL(uint64_t)   CPUMGetGuestFlatPC(PVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
+    CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.cs);
+    if (   !CPUMIsGuestInLongMode(pVCpu)
+        || !pVCpu->cpum.s.Guest.cs.Attr.n.u1Long)
+        return pVCpu->cpum.s.Guest.eip + (uint32_t)pVCpu->cpum.s.Guest.cs.u64Base;
+    return pVCpu->cpum.s.Guest.rip + pVCpu->cpum.s.Guest.cs.u64Base;
+}
+
+
+VMMDECL(uint64_t)   CPUMGetGuestFlatSP(PVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
+    CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.ss);
+    if (   !CPUMIsGuestInLongMode(pVCpu)
+        || !pVCpu->cpum.s.Guest.cs.Attr.n.u1Long)
+        return pVCpu->cpum.s.Guest.eip + (uint32_t)pVCpu->cpum.s.Guest.ss.u64Base;
+    return pVCpu->cpum.s.Guest.rip + pVCpu->cpum.s.Guest.ss.u64Base;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestLDTR(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_LDTR);
+    return pVCpu->cpum.s.Guest.ldtr.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestLdtrEx(PCVMCPU pVCpu, uint64_t *pGCPtrBase, uint32_t *pcbLimit)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_LDTR);
+    *pGCPtrBase = pVCpu->cpum.s.Guest.ldtr.u64Base;
+    *pcbLimit   = pVCpu->cpum.s.Guest.ldtr.u32Limit;
+    return pVCpu->cpum.s.Guest.ldtr.Sel;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestCR0(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+    return pVCpu->cpum.s.Guest.cr0;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestCR2(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR2);
+    return pVCpu->cpum.s.Guest.cr2;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestCR3(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR3);
+    return pVCpu->cpum.s.Guest.cr3;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestCR4(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR4);
+    return pVCpu->cpum.s.Guest.cr4;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestCR8(PCVMCPUCC pVCpu)
+{
+    uint64_t u64;
+    int rc = CPUMGetGuestCRx(pVCpu, DISCREG_CR8, &u64);
+    if (RT_FAILURE(rc))
+        u64 = 0;
+    return u64;
+}
+
+
+VMMDECL(void) CPUMGetGuestGDTR(PCVMCPU pVCpu, PVBOXGDTR pGDTR)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_GDTR);
+    *pGDTR = pVCpu->cpum.s.Guest.gdtr;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEIP(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP);
+    return pVCpu->cpum.s.Guest.eip;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestRIP(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP);
+    return pVCpu->cpum.s.Guest.rip;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEAX(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RAX);
+    return pVCpu->cpum.s.Guest.eax;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEBX(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RBX);
+    return pVCpu->cpum.s.Guest.ebx;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestECX(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RCX);
+    return pVCpu->cpum.s.Guest.ecx;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEDX(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RDX);
+    return pVCpu->cpum.s.Guest.edx;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestESI(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RSI);
+    return pVCpu->cpum.s.Guest.esi;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEDI(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RDI);
+    return pVCpu->cpum.s.Guest.edi;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestESP(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RSP);
+    return pVCpu->cpum.s.Guest.esp;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEBP(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RBP);
+    return pVCpu->cpum.s.Guest.ebp;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEFlags(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RFLAGS);
+    return pVCpu->cpum.s.Guest.eflags.u32;
+}
+
+
+VMMDECL(int) CPUMGetGuestCRx(PCVMCPUCC pVCpu, unsigned iReg, uint64_t *pValue)
+{
+    switch (iReg)
+    {
+        case DISCREG_CR0:
+            CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+            *pValue = pVCpu->cpum.s.Guest.cr0;
+            break;
+
+        case DISCREG_CR2:
+            CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR2);
+            *pValue = pVCpu->cpum.s.Guest.cr2;
+            break;
+
+        case DISCREG_CR3:
+            CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR3);
+            *pValue = pVCpu->cpum.s.Guest.cr3;
+            break;
+
+        case DISCREG_CR4:
+            CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR4);
+            *pValue = pVCpu->cpum.s.Guest.cr4;
+            break;
+
+        case DISCREG_CR8:
+        {
+            CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
+            uint8_t u8Tpr;
+            int rc = APICGetTpr(pVCpu, &u8Tpr, NULL /* pfPending */, NULL /* pu8PendingIrq */);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsg(rc == VERR_PDM_NO_APIC_INSTANCE, ("%Rrc\n", rc));
+                *pValue = 0;
+                return rc;
+            }
+            *pValue = u8Tpr >> 4; /* bits 7-4 contain the task priority that go in cr8, bits 3-0 */
+            break;
+        }
+
+        default:
+            return VERR_INVALID_PARAMETER;
+    }
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestDR0(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+    return pVCpu->cpum.s.Guest.dr[0];
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestDR1(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+    return pVCpu->cpum.s.Guest.dr[1];
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestDR2(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+    return pVCpu->cpum.s.Guest.dr[2];
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestDR3(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+    return pVCpu->cpum.s.Guest.dr[3];
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestDR6(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR6);
+    return pVCpu->cpum.s.Guest.dr[6];
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestDR7(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR7);
+    return pVCpu->cpum.s.Guest.dr[7];
+}
+
+
+VMMDECL(int) CPUMGetGuestDRx(PCVMCPU pVCpu, uint32_t iReg, uint64_t *pValue)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR_MASK);
+    AssertReturn(iReg <= DISDREG_DR7, VERR_INVALID_PARAMETER);
+    /* DR4 is an alias for DR6, and DR5 is an alias for DR7. */
+    if (iReg == 4 || iReg == 5)
+        iReg += 2;
+    *pValue = pVCpu->cpum.s.Guest.dr[iReg];
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestEFER(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_EFER);
+    return pVCpu->cpum.s.Guest.msrEFER;
+}
+
+
+/**
+ * Looks up a CPUID leaf in the CPUID leaf array, no subleaf.
+ *
+ * @returns Pointer to the leaf if found, NULL if not.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   uLeaf               The leaf to get.
+ */
+PCPUMCPUIDLEAF cpumCpuIdGetLeaf(PVM pVM, uint32_t uLeaf)
+{
+    unsigned            iEnd     = pVM->cpum.s.GuestInfo.cCpuIdLeaves;
+    if (iEnd)
+    {
+        unsigned        iStart   = 0;
+        PCPUMCPUIDLEAF  paLeaves = pVM->cpum.s.GuestInfo.CTX_SUFF(paCpuIdLeaves);
+        for (;;)
+        {
+            unsigned i = iStart + (iEnd - iStart) / 2U;
+            if (uLeaf < paLeaves[i].uLeaf)
+            {
+                if (i <= iStart)
+                    return NULL;
+                iEnd = i;
+            }
+            else if (uLeaf > paLeaves[i].uLeaf)
+            {
+                i += 1;
+                if (i >= iEnd)
+                    return NULL;
+                iStart = i;
+            }
+            else
+            {
+                if (RT_LIKELY(paLeaves[i].fSubLeafMask == 0 && paLeaves[i].uSubLeaf == 0))
+                    return &paLeaves[i];
+
+                /* This shouldn't normally happen. But in case the it does due
+                   to user configuration overrids or something, just return the
+                   first sub-leaf. */
+                AssertMsgFailed(("uLeaf=%#x fSubLeafMask=%#x uSubLeaf=%#x\n",
+                                 uLeaf, paLeaves[i].fSubLeafMask, paLeaves[i].uSubLeaf));
+                while (   paLeaves[i].uSubLeaf != 0
+                       && i > 0
+                       && uLeaf == paLeaves[i - 1].uLeaf)
+                    i--;
+                return &paLeaves[i];
+            }
+        }
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Looks up a CPUID leaf in the CPUID leaf array.
+ *
+ * @returns Pointer to the leaf if found, NULL if not.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   uLeaf               The leaf to get.
+ * @param   uSubLeaf            The subleaf, if applicable.  Just pass 0 if it
+ *                              isn't.
+ * @param   pfExactSubLeafHit   Whether we've got an exact subleaf hit or not.
+ */
+PCPUMCPUIDLEAF cpumCpuIdGetLeafEx(PVM pVM, uint32_t uLeaf, uint32_t uSubLeaf, bool *pfExactSubLeafHit)
+{
+    unsigned            iEnd     = pVM->cpum.s.GuestInfo.cCpuIdLeaves;
+    if (iEnd)
+    {
+        unsigned        iStart   = 0;
+        PCPUMCPUIDLEAF  paLeaves = pVM->cpum.s.GuestInfo.CTX_SUFF(paCpuIdLeaves);
+        for (;;)
+        {
+            unsigned i = iStart + (iEnd - iStart) / 2U;
+            if (uLeaf < paLeaves[i].uLeaf)
+            {
+                if (i <= iStart)
+                    return NULL;
+                iEnd = i;
+            }
+            else if (uLeaf > paLeaves[i].uLeaf)
+            {
+                i += 1;
+                if (i >= iEnd)
+                    return NULL;
+                iStart = i;
+            }
+            else
+            {
+                uSubLeaf &= paLeaves[i].fSubLeafMask;
+                if (uSubLeaf == paLeaves[i].uSubLeaf)
+                    *pfExactSubLeafHit = true;
+                else
+                {
+                    /* Find the right subleaf.  We return the last one before
+                       uSubLeaf if we don't find an exact match. */
+                    if (uSubLeaf < paLeaves[i].uSubLeaf)
+                        while (   i > 0
+                               && uLeaf    == paLeaves[i - 1].uLeaf
+                               && uSubLeaf <= paLeaves[i - 1].uSubLeaf)
+                            i--;
+                    else
+                        while (   i + 1 < pVM->cpum.s.GuestInfo.cCpuIdLeaves
+                               && uLeaf    == paLeaves[i + 1].uLeaf
+                               && uSubLeaf >= paLeaves[i + 1].uSubLeaf)
+                            i++;
+                    *pfExactSubLeafHit = uSubLeaf == paLeaves[i].uSubLeaf;
+                }
+                return &paLeaves[i];
+            }
+        }
+    }
+
+    *pfExactSubLeafHit = false;
+    return NULL;
+}
+
+
+/**
+ * Gets a CPUID leaf.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uLeaf       The CPUID leaf to get.
+ * @param   uSubLeaf    The CPUID sub-leaf to get, if applicable.
+ * @param   pEax        Where to store the EAX value.
+ * @param   pEbx        Where to store the EBX value.
+ * @param   pEcx        Where to store the ECX value.
+ * @param   pEdx        Where to store the EDX value.
+ */
+VMMDECL(void) CPUMGetGuestCpuId(PVMCPUCC pVCpu, uint32_t uLeaf, uint32_t uSubLeaf,
+                                uint32_t *pEax, uint32_t *pEbx, uint32_t *pEcx, uint32_t *pEdx)
+{
+    bool            fExactSubLeafHit;
+    PVM             pVM   = pVCpu->CTX_SUFF(pVM);
+    PCCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeafEx(pVM, uLeaf, uSubLeaf, &fExactSubLeafHit);
+    if (pLeaf)
+    {
+        AssertMsg(pLeaf->uLeaf == uLeaf, ("%#x %#x\n", pLeaf->uLeaf, uLeaf));
+        if (fExactSubLeafHit)
+        {
+            *pEax = pLeaf->uEax;
+            *pEbx = pLeaf->uEbx;
+            *pEcx = pLeaf->uEcx;
+            *pEdx = pLeaf->uEdx;
+
+            /*
+             * Deal with CPU specific information.
+             */
+            if (pLeaf->fFlags & (  CPUMCPUIDLEAF_F_CONTAINS_APIC_ID
+                                 | CPUMCPUIDLEAF_F_CONTAINS_OSXSAVE
+                                 | CPUMCPUIDLEAF_F_CONTAINS_APIC ))
+            {
+                if (uLeaf == 1)
+                {
+                    /* EBX: Bits 31-24: Initial APIC ID. */
+                    Assert(pVCpu->idCpu <= 255);
+                    AssertMsg((pLeaf->uEbx >> 24) == 0, ("%#x\n", pLeaf->uEbx)); /* raw-mode assumption */
+                    *pEbx = (pLeaf->uEbx & UINT32_C(0x00ffffff)) | (pVCpu->idCpu << 24);
+
+                    /* EDX: Bit 9: AND with APICBASE.EN. */
+                    if (!pVCpu->cpum.s.fCpuIdApicFeatureVisible && (pLeaf->fFlags & CPUMCPUIDLEAF_F_CONTAINS_APIC))
+                        *pEdx &= ~X86_CPUID_FEATURE_EDX_APIC;
+
+                    /* ECX: Bit 27: CR4.OSXSAVE mirror. */
+                    *pEcx = (pLeaf->uEcx & ~X86_CPUID_FEATURE_ECX_OSXSAVE)
+                          | (pVCpu->cpum.s.Guest.cr4 & X86_CR4_OSXSAVE ? X86_CPUID_FEATURE_ECX_OSXSAVE : 0);
+                }
+                else if (uLeaf == 0xb)
+                {
+                    /* EDX: Initial extended APIC ID. */
+                    AssertMsg(pLeaf->uEdx == 0, ("%#x\n", pLeaf->uEdx)); /* raw-mode assumption */
+                    *pEdx = pVCpu->idCpu;
+                    Assert(!(pLeaf->fFlags & ~(CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES)));
+                }
+                else if (uLeaf == UINT32_C(0x8000001e))
+                {
+                    /* EAX: Initial extended APIC ID. */
+                    AssertMsg(pLeaf->uEax == 0, ("%#x\n", pLeaf->uEax)); /* raw-mode assumption */
+                    *pEax = pVCpu->idCpu;
+                    Assert(!(pLeaf->fFlags & ~CPUMCPUIDLEAF_F_CONTAINS_APIC_ID));
+                }
+                else if (uLeaf == UINT32_C(0x80000001))
+                {
+                    /* EDX: Bit 9: AND with APICBASE.EN. */
+                    if (!pVCpu->cpum.s.fCpuIdApicFeatureVisible)
+                        *pEdx &= ~X86_CPUID_AMD_FEATURE_EDX_APIC;
+                    Assert(!(pLeaf->fFlags & ~CPUMCPUIDLEAF_F_CONTAINS_APIC));
+                }
+                else
+                    AssertMsgFailed(("uLeaf=%#x\n", uLeaf));
+            }
+        }
+        /*
+         * Out of range sub-leaves aren't quite as easy and pretty as we emulate
+         * them here, but we do the best we can here...
+         */
+        else
+        {
+            *pEax = *pEbx = *pEcx = *pEdx = 0;
+            if (pLeaf->fFlags & CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES)
+            {
+                *pEcx = uSubLeaf & 0xff;
+                *pEdx = pVCpu->idCpu;
+            }
+        }
+    }
+    else
+    {
+        /*
+         * Different CPUs have different ways of dealing with unknown CPUID leaves.
+         */
+        switch (pVM->cpum.s.GuestInfo.enmUnknownCpuIdMethod)
+        {
+            default:
+                AssertFailed();
+                RT_FALL_THRU();
+            case CPUMUNKNOWNCPUID_DEFAULTS:
+            case CPUMUNKNOWNCPUID_LAST_STD_LEAF: /* ASSUME this is executed */
+            case CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX: /** @todo Implement CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX */
+                *pEax = pVM->cpum.s.GuestInfo.DefCpuId.uEax;
+                *pEbx = pVM->cpum.s.GuestInfo.DefCpuId.uEbx;
+                *pEcx = pVM->cpum.s.GuestInfo.DefCpuId.uEcx;
+                *pEdx = pVM->cpum.s.GuestInfo.DefCpuId.uEdx;
+                break;
+            case CPUMUNKNOWNCPUID_PASSTHRU:
+                *pEax = uLeaf;
+                *pEbx = 0;
+                *pEcx = uSubLeaf;
+                *pEdx = 0;
+                break;
+        }
+    }
+    Log2(("CPUMGetGuestCpuId: uLeaf=%#010x/%#010x %RX32 %RX32 %RX32 %RX32\n", uLeaf, uSubLeaf, *pEax, *pEbx, *pEcx, *pEdx));
+}
+
+
+/**
+ * Sets the visibility of the X86_CPUID_FEATURE_EDX_APIC and
+ * X86_CPUID_AMD_FEATURE_EDX_APIC CPUID bits.
+ *
+ * @returns Previous value.
+ * @param   pVCpu       The cross context virtual CPU structure to make the
+ *                      change on.  Usually the calling EMT.
+ * @param   fVisible    Whether to make it visible (true) or hide it (false).
+ *
+ * @remarks This is "VMMDECL" so that it still links with
+ *          the old APIC code which is in VBoxDD2 and not in
+ *          the VMM module.
+ */
+VMMDECL(bool) CPUMSetGuestCpuIdPerCpuApicFeature(PVMCPU pVCpu, bool fVisible)
+{
+    bool fOld = pVCpu->cpum.s.fCpuIdApicFeatureVisible;
+    pVCpu->cpum.s.fCpuIdApicFeatureVisible = fVisible;
+    return fOld;
+}
+
+
+/**
+ * Gets the host CPU vendor.
+ *
+ * @returns CPU vendor.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(CPUMCPUVENDOR) CPUMGetHostCpuVendor(PVM pVM)
+{
+    return (CPUMCPUVENDOR)pVM->cpum.s.HostFeatures.enmCpuVendor;
+}
+
+
+/**
+ * Gets the host CPU microarchitecture.
+ *
+ * @returns CPU microarchitecture.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(CPUMMICROARCH) CPUMGetHostMicroarch(PCVM pVM)
+{
+    return pVM->cpum.s.HostFeatures.enmMicroarch;
+}
+
+
+/**
+ * Gets the guest CPU vendor.
+ *
+ * @returns CPU vendor.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(CPUMCPUVENDOR) CPUMGetGuestCpuVendor(PVM pVM)
+{
+    return (CPUMCPUVENDOR)pVM->cpum.s.GuestFeatures.enmCpuVendor;
+}
+
+
+/**
+ * Gets the guest CPU microarchitecture.
+ *
+ * @returns CPU microarchitecture.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(CPUMMICROARCH) CPUMGetGuestMicroarch(PCVM pVM)
+{
+    return pVM->cpum.s.GuestFeatures.enmMicroarch;
+}
+
+
+VMMDECL(int) CPUMSetGuestDR0(PVMCPUCC pVCpu, uint64_t uDr0)
+{
+    pVCpu->cpum.s.Guest.dr[0] = uDr0;
+    return CPUMRecalcHyperDRx(pVCpu, 0, false);
+}
+
+
+VMMDECL(int) CPUMSetGuestDR1(PVMCPUCC pVCpu, uint64_t uDr1)
+{
+    pVCpu->cpum.s.Guest.dr[1] = uDr1;
+    return CPUMRecalcHyperDRx(pVCpu, 1, false);
+}
+
+
+VMMDECL(int) CPUMSetGuestDR2(PVMCPUCC pVCpu, uint64_t uDr2)
+{
+    pVCpu->cpum.s.Guest.dr[2] = uDr2;
+    return CPUMRecalcHyperDRx(pVCpu, 2, false);
+}
+
+
+VMMDECL(int) CPUMSetGuestDR3(PVMCPUCC pVCpu, uint64_t uDr3)
+{
+    pVCpu->cpum.s.Guest.dr[3] = uDr3;
+    return CPUMRecalcHyperDRx(pVCpu, 3, false);
+}
+
+
+VMMDECL(int) CPUMSetGuestDR6(PVMCPU pVCpu, uint64_t uDr6)
+{
+    pVCpu->cpum.s.Guest.dr[6] = uDr6;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_DR6;
+    return VINF_SUCCESS; /* No need to recalc. */
+}
+
+
+VMMDECL(int) CPUMSetGuestDR7(PVMCPUCC pVCpu, uint64_t uDr7)
+{
+    pVCpu->cpum.s.Guest.dr[7] = uDr7;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_DR7;
+    return CPUMRecalcHyperDRx(pVCpu, 7, false);
+}
+
+
+VMMDECL(int) CPUMSetGuestDRx(PVMCPUCC pVCpu, uint32_t iReg, uint64_t Value)
+{
+    AssertReturn(iReg <= DISDREG_DR7, VERR_INVALID_PARAMETER);
+    /* DR4 is an alias for DR6, and DR5 is an alias for DR7. */
+    if (iReg == 4 || iReg == 5)
+        iReg += 2;
+    pVCpu->cpum.s.Guest.dr[iReg] = Value;
+    return CPUMRecalcHyperDRx(pVCpu, iReg, false);
+}
+
+
+/**
+ * Recalculates the hypervisor DRx register values based on current guest
+ * registers and DBGF breakpoints, updating changed registers depending on the
+ * context.
+ *
+ * This is called whenever a guest DRx register is modified (any context) and
+ * when DBGF sets a hardware breakpoint (ring-3 only, rendezvous).
+ *
+ * In raw-mode context this function will reload any (hyper) DRx registers which
+ * comes out with a different value.  It may also have to save the host debug
+ * registers if that haven't been done already.  In this context though, we'll
+ * be intercepting and emulating all DRx accesses, so the hypervisor DRx values
+ * are only important when breakpoints are actually enabled.
+ *
+ * In ring-0 (HM) context DR0-3 will be relocated by us, while DR7 will be
+ * reloaded by the HM code if it changes.  Further more, we will only use the
+ * combined register set when the VBox debugger is actually using hardware BPs,
+ * when it isn't we'll keep the guest DR0-3 + (maybe) DR6 loaded (DR6 doesn't
+ * concern us here).
+ *
+ * In ring-3 we won't be loading anything, so well calculate hypervisor values
+ * all the time.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iGstReg     The guest debug register number that was modified.
+ *                      UINT8_MAX if not guest register.
+ * @param   fForceHyper Used in HM to force hyper registers because of single
+ *                      stepping.
+ */
+VMMDECL(int) CPUMRecalcHyperDRx(PVMCPUCC pVCpu, uint8_t iGstReg, bool fForceHyper)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+#ifndef IN_RING0
+    RT_NOREF_PV(iGstReg);
+#endif
+
+    /*
+     * Compare the DR7s first.
+     *
+     * We only care about the enabled flags.  GD is virtualized when we
+     * dispatch the #DB, we never enable it.  The DBGF DR7 value is will
+     * always have the LE and GE bits set, so no need to check and disable
+     * stuff if they're cleared like we have to for the guest DR7.
+     */
+    RTGCUINTREG uGstDr7 = CPUMGetGuestDR7(pVCpu);
+    /** @todo This isn't correct. BPs work without setting LE and GE under AMD-V.  They are also documented as unsupported by P6+. */
+    if (!(uGstDr7 & (X86_DR7_LE | X86_DR7_GE)))
+        uGstDr7 = 0;
+    else if (!(uGstDr7 & X86_DR7_LE))
+        uGstDr7 &= ~X86_DR7_LE_ALL;
+    else if (!(uGstDr7 & X86_DR7_GE))
+        uGstDr7 &= ~X86_DR7_GE_ALL;
+
+    const RTGCUINTREG uDbgfDr7 = DBGFBpGetDR7(pVM);
+
+    /** @todo r=bird: I'm totally confused by fForceHyper! */
+#ifdef IN_RING0
+    if (!fForceHyper && (pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_HYPER))
+        fForceHyper = true;
+#endif
+    if ((!fForceHyper ? uDbgfDr7 : (uGstDr7 | uDbgfDr7)) & X86_DR7_ENABLED_MASK)
+    {
+        Assert(!CPUMIsGuestDebugStateActive(pVCpu));
+
+        /*
+         * Ok, something is enabled.  Recalc each of the breakpoints, taking
+         * the VM debugger ones of the guest ones.  In raw-mode context we will
+         * not allow breakpoints with values inside the hypervisor area.
+         */
+        RTGCUINTREG uNewDr7 = X86_DR7_GE | X86_DR7_LE | X86_DR7_RA1_MASK;
+
+        /* bp 0 */
+        RTGCUINTREG uNewDr0;
+        if (uDbgfDr7 & (X86_DR7_L0 | X86_DR7_G0))
+        {
+            uNewDr7 |= uDbgfDr7 & (X86_DR7_L0 | X86_DR7_G0 | X86_DR7_RW0_MASK | X86_DR7_LEN0_MASK);
+            uNewDr0 = DBGFBpGetDR0(pVM);
+        }
+        else if (uGstDr7 & (X86_DR7_L0 | X86_DR7_G0))
+        {
+            uNewDr0 = CPUMGetGuestDR0(pVCpu);
+            uNewDr7 |= uGstDr7 & (X86_DR7_L0 | X86_DR7_G0 | X86_DR7_RW0_MASK | X86_DR7_LEN0_MASK);
+        }
+        else
+            uNewDr0 = 0;
+
+        /* bp 1 */
+        RTGCUINTREG uNewDr1;
+        if (uDbgfDr7 & (X86_DR7_L1 | X86_DR7_G1))
+        {
+            uNewDr7 |= uDbgfDr7 & (X86_DR7_L1 | X86_DR7_G1 | X86_DR7_RW1_MASK | X86_DR7_LEN1_MASK);
+            uNewDr1 = DBGFBpGetDR1(pVM);
+        }
+        else if (uGstDr7 & (X86_DR7_L1 | X86_DR7_G1))
+        {
+            uNewDr1 = CPUMGetGuestDR1(pVCpu);
+            uNewDr7 |= uGstDr7 & (X86_DR7_L1 | X86_DR7_G1 | X86_DR7_RW1_MASK | X86_DR7_LEN1_MASK);
+        }
+        else
+            uNewDr1 = 0;
+
+        /* bp 2 */
+        RTGCUINTREG uNewDr2;
+        if (uDbgfDr7 & (X86_DR7_L2 | X86_DR7_G2))
+        {
+            uNewDr7 |= uDbgfDr7 & (X86_DR7_L2 | X86_DR7_G2 | X86_DR7_RW2_MASK | X86_DR7_LEN2_MASK);
+            uNewDr2 = DBGFBpGetDR2(pVM);
+        }
+        else if (uGstDr7 & (X86_DR7_L2 | X86_DR7_G2))
+        {
+            uNewDr2 = CPUMGetGuestDR2(pVCpu);
+            uNewDr7 |= uGstDr7 & (X86_DR7_L2 | X86_DR7_G2 | X86_DR7_RW2_MASK | X86_DR7_LEN2_MASK);
+        }
+        else
+            uNewDr2 = 0;
+
+        /* bp 3 */
+        RTGCUINTREG uNewDr3;
+        if (uDbgfDr7 & (X86_DR7_L3 | X86_DR7_G3))
+        {
+            uNewDr7 |= uDbgfDr7 & (X86_DR7_L3 | X86_DR7_G3 | X86_DR7_RW3_MASK | X86_DR7_LEN3_MASK);
+            uNewDr3 = DBGFBpGetDR3(pVM);
+        }
+        else if (uGstDr7 & (X86_DR7_L3 | X86_DR7_G3))
+        {
+            uNewDr3 = CPUMGetGuestDR3(pVCpu);
+            uNewDr7 |= uGstDr7 & (X86_DR7_L3 | X86_DR7_G3 | X86_DR7_RW3_MASK | X86_DR7_LEN3_MASK);
+        }
+        else
+            uNewDr3 = 0;
+
+        /*
+         * Apply the updates.
+         */
+        pVCpu->cpum.s.fUseFlags |= CPUM_USE_DEBUG_REGS_HYPER;
+        if (uNewDr3 != pVCpu->cpum.s.Hyper.dr[3])
+            CPUMSetHyperDR3(pVCpu, uNewDr3);
+        if (uNewDr2 != pVCpu->cpum.s.Hyper.dr[2])
+            CPUMSetHyperDR2(pVCpu, uNewDr2);
+        if (uNewDr1 != pVCpu->cpum.s.Hyper.dr[1])
+            CPUMSetHyperDR1(pVCpu, uNewDr1);
+        if (uNewDr0 != pVCpu->cpum.s.Hyper.dr[0])
+            CPUMSetHyperDR0(pVCpu, uNewDr0);
+        if (uNewDr7 != pVCpu->cpum.s.Hyper.dr[7])
+            CPUMSetHyperDR7(pVCpu, uNewDr7);
+    }
+#ifdef IN_RING0
+    else if (CPUMIsGuestDebugStateActive(pVCpu))
+    {
+        /*
+         * Reload the register that was modified.  Normally this won't happen
+         * as we won't intercept DRx writes when not having the hyper debug
+         * state loaded, but in case we do for some reason we'll simply deal
+         * with it.
+         */
+        switch (iGstReg)
+        {
+            case 0: ASMSetDR0(CPUMGetGuestDR0(pVCpu)); break;
+            case 1: ASMSetDR1(CPUMGetGuestDR1(pVCpu)); break;
+            case 2: ASMSetDR2(CPUMGetGuestDR2(pVCpu)); break;
+            case 3: ASMSetDR3(CPUMGetGuestDR3(pVCpu)); break;
+            default:
+                AssertReturn(iGstReg != UINT8_MAX, VERR_INTERNAL_ERROR_3);
+        }
+    }
+#endif
+    else
+    {
+        /*
+         * No active debug state any more.  In raw-mode this means we have to
+         * make sure DR7 has everything disabled now, if we armed it already.
+         * In ring-0 we might end up here when just single stepping.
+         */
+#ifdef IN_RING0
+        if (pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_HYPER)
+        {
+            if (pVCpu->cpum.s.Hyper.dr[0])
+                ASMSetDR0(0);
+            if (pVCpu->cpum.s.Hyper.dr[1])
+                ASMSetDR1(0);
+            if (pVCpu->cpum.s.Hyper.dr[2])
+                ASMSetDR2(0);
+            if (pVCpu->cpum.s.Hyper.dr[3])
+                ASMSetDR3(0);
+            pVCpu->cpum.s.fUseFlags &= ~CPUM_USED_DEBUG_REGS_HYPER;
+        }
+#endif
+        pVCpu->cpum.s.fUseFlags &= ~CPUM_USE_DEBUG_REGS_HYPER;
+
+        /* Clear all the registers. */
+        pVCpu->cpum.s.Hyper.dr[7] = X86_DR7_RA1_MASK;
+        pVCpu->cpum.s.Hyper.dr[3] = 0;
+        pVCpu->cpum.s.Hyper.dr[2] = 0;
+        pVCpu->cpum.s.Hyper.dr[1] = 0;
+        pVCpu->cpum.s.Hyper.dr[0] = 0;
+
+    }
+    Log2(("CPUMRecalcHyperDRx: fUseFlags=%#x %RGr %RGr %RGr %RGr  %RGr %RGr\n",
+          pVCpu->cpum.s.fUseFlags, pVCpu->cpum.s.Hyper.dr[0], pVCpu->cpum.s.Hyper.dr[1],
+          pVCpu->cpum.s.Hyper.dr[2], pVCpu->cpum.s.Hyper.dr[3], pVCpu->cpum.s.Hyper.dr[6],
+          pVCpu->cpum.s.Hyper.dr[7]));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Set the guest XCR0 register.
+ *
+ * Will load additional state if the FPU state is already loaded (in ring-0 &
+ * raw-mode context).
+ *
+ * @returns VINF_SUCCESS on success, VERR_CPUM_RAISE_GP_0 on invalid input
+ *          value.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uNewValue   The new value.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(int)   CPUMSetGuestXcr0(PVMCPUCC pVCpu, uint64_t uNewValue)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_XCRx);
+    if (   (uNewValue & ~pVCpu->CTX_SUFF(pVM)->cpum.s.fXStateGuestMask) == 0
+        /* The X87 bit cannot be cleared. */
+        && (uNewValue & XSAVE_C_X87)
+        /* AVX requires SSE. */
+        && (uNewValue & (XSAVE_C_SSE | XSAVE_C_YMM)) != XSAVE_C_YMM
+        /* AVX-512 requires YMM, SSE and all of its three components to be enabled. */
+        && (   (uNewValue & (XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI)) == 0
+            ||    (uNewValue & (XSAVE_C_SSE | XSAVE_C_YMM | XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI))
+               ==              (XSAVE_C_SSE | XSAVE_C_YMM | XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI) )
+       )
+    {
+        pVCpu->cpum.s.Guest.aXcr[0] = uNewValue;
+
+        /* If more state components are enabled, we need to take care to load
+           them if the FPU/SSE state is already loaded.  May otherwise leak
+           host state to the guest. */
+        uint64_t fNewComponents = ~pVCpu->cpum.s.Guest.fXStateMask & uNewValue;
+        if (fNewComponents)
+        {
+#ifdef IN_RING0
+            if (pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST)
+            {
+                if (pVCpu->cpum.s.Guest.fXStateMask != 0)
+                    /* Adding more components. */
+                    ASMXRstor(pVCpu->cpum.s.Guest.CTX_SUFF(pXState), fNewComponents);
+                else
+                {
+                    /* We're switching from FXSAVE/FXRSTOR to XSAVE/XRSTOR. */
+                    pVCpu->cpum.s.Guest.fXStateMask |= XSAVE_C_X87 | XSAVE_C_SSE;
+                    if (uNewValue & ~(XSAVE_C_X87 | XSAVE_C_SSE))
+                        ASMXRstor(pVCpu->cpum.s.Guest.CTX_SUFF(pXState), uNewValue & ~(XSAVE_C_X87 | XSAVE_C_SSE));
+                }
+            }
+#endif
+            pVCpu->cpum.s.Guest.fXStateMask |= uNewValue;
+        }
+        return VINF_SUCCESS;
+    }
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * Tests if the guest has No-Execute Page Protection Enabled (NXE).
+ *
+ * @returns true if in real mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestNXEnabled(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_EFER);
+    return !!(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_NXE);
+}
+
+
+/**
+ * Tests if the guest has the Page Size Extension enabled (PSE).
+ *
+ * @returns true if in real mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestPageSizeExtEnabled(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR4);
+    /* PAE or AMD64 implies support for big pages regardless of CR4.PSE */
+    return !!(pVCpu->cpum.s.Guest.cr4 & (X86_CR4_PSE | X86_CR4_PAE));
+}
+
+
+/**
+ * Tests if the guest has the paging enabled (PG).
+ *
+ * @returns true if in real mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestPagingEnabled(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+    return !!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PG);
+}
+
+
+/**
+ * Tests if the guest has the paging enabled (PG).
+ *
+ * @returns true if in real mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestR0WriteProtEnabled(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+    return !!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_WP);
+}
+
+
+/**
+ * Tests if the guest is running in real mode or not.
+ *
+ * @returns true if in real mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestInRealMode(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+    return !(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE);
+}
+
+
+/**
+ * Tests if the guest is running in real or virtual 8086 mode.
+ *
+ * @returns @c true if it is, @c false if not.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestInRealOrV86Mode(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS);
+    return !(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE)
+        || pVCpu->cpum.s.Guest.eflags.Bits.u1VM; /** @todo verify that this cannot be set in long mode. */
+}
+
+
+/**
+ * Tests if the guest is running in protected or not.
+ *
+ * @returns true if in protected mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestInProtectedMode(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+    return !!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE);
+}
+
+
+/**
+ * Tests if the guest is running in paged protected or not.
+ *
+ * @returns true if in paged protected mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestInPagedProtectedMode(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+    return (pVCpu->cpum.s.Guest.cr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG);
+}
+
+
+/**
+ * Tests if the guest is running in long mode or not.
+ *
+ * @returns true if in long mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestInLongMode(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_EFER);
+    return (pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA) == MSR_K6_EFER_LMA;
+}
+
+
+/**
+ * Tests if the guest is running in PAE mode or not.
+ *
+ * @returns true if in PAE mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestInPAEMode(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER);
+    /* Intel mentions EFER.LMA and EFER.LME in different parts of their spec. We shall use EFER.LMA rather
+       than EFER.LME as it reflects if the CPU has entered paging with EFER.LME set.  */
+    return (pVCpu->cpum.s.Guest.cr4 & X86_CR4_PAE)
+        && (pVCpu->cpum.s.Guest.cr0 & X86_CR0_PG)
+        && !(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA);
+}
+
+
+/**
+ * Tests if the guest is running in 64 bits mode or not.
+ *
+ * @returns true if in 64 bits protected mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(bool) CPUMIsGuestIn64BitCode(PVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
+    if (!CPUMIsGuestInLongMode(pVCpu))
+        return false;
+    CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.cs);
+    return pVCpu->cpum.s.Guest.cs.Attr.n.u1Long;
+}
+
+
+/**
+ * Helper for CPUMIsGuestIn64BitCodeEx that handles lazy resolving of hidden CS
+ * registers.
+ *
+ * @returns true if in 64 bits protected mode, otherwise false.
+ * @param   pCtx        Pointer to the current guest CPU context.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestIn64BitCodeSlow(PCPUMCTX pCtx)
+{
+    return CPUMIsGuestIn64BitCode(CPUM_GUEST_CTX_TO_VMCPU(pCtx));
+}
+
+
+/**
+ * Sets the specified changed flags (CPUM_CHANGED_*).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   fChangedAdd The changed flags to add.
+ */
+VMMDECL(void) CPUMSetChangedFlags(PVMCPU pVCpu, uint32_t fChangedAdd)
+{
+    pVCpu->cpum.s.fChanged |= fChangedAdd;
+}
+
+
+/**
+ * Checks if the CPU supports the XSAVE and XRSTOR instruction.
+ *
+ * @returns true if supported.
+ * @returns false if not supported.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(bool) CPUMSupportsXSave(PVM pVM)
+{
+    return pVM->cpum.s.HostFeatures.fXSaveRstor != 0;
+}
+
+
+/**
+ * Checks if the host OS uses the SYSENTER / SYSEXIT instructions.
+ * @returns true if used.
+ * @returns false if not used.
+ * @param   pVM       The cross context VM structure.
+ */
+VMMDECL(bool) CPUMIsHostUsingSysEnter(PVM pVM)
+{
+    return RT_BOOL(pVM->cpum.s.fHostUseFlags & CPUM_USE_SYSENTER);
+}
+
+
+/**
+ * Checks if the host OS uses the SYSCALL / SYSRET instructions.
+ * @returns true if used.
+ * @returns false if not used.
+ * @param   pVM       The cross context VM structure.
+ */
+VMMDECL(bool) CPUMIsHostUsingSysCall(PVM pVM)
+{
+    return RT_BOOL(pVM->cpum.s.fHostUseFlags & CPUM_USE_SYSCALL);
+}
+
+
+/**
+ * Checks if we activated the FPU/XMM state of the guest OS.
+ *
+ * This differs from CPUMIsGuestFPUStateLoaded() in that it refers to the next
+ * time we'll be executing guest code, so it may return true for 64-on-32 when
+ * we still haven't actually loaded the FPU status, just scheduled it to be
+ * loaded the next time we go thru the world switcher (CPUM_SYNC_FPU_STATE).
+ *
+ * @returns true / false.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestFPUStateActive(PVMCPU pVCpu)
+{
+    return RT_BOOL(pVCpu->cpum.s.fUseFlags & (CPUM_USED_FPU_GUEST | CPUM_SYNC_FPU_STATE));
+}
+
+
+/**
+ * Checks if we've really loaded the FPU/XMM state of the guest OS.
+ *
+ * @returns true / false.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestFPUStateLoaded(PVMCPU pVCpu)
+{
+    return RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST);
+}
+
+
+/**
+ * Checks if we saved the FPU/XMM state of the host OS.
+ *
+ * @returns true / false.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsHostFPUStateSaved(PVMCPU pVCpu)
+{
+    return RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_HOST);
+}
+
+
+/**
+ * Checks if the guest debug state is active.
+ *
+ * @returns boolean
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(bool) CPUMIsGuestDebugStateActive(PVMCPU pVCpu)
+{
+    return RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_GUEST);
+}
+
+
+/**
+ * Checks if the guest debug state is to be made active during the world-switch
+ * (currently only used for the 32->64 switcher case).
+ *
+ * @returns boolean
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(bool) CPUMIsGuestDebugStateActivePending(PVMCPU pVCpu)
+{
+    return RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_SYNC_DEBUG_REGS_GUEST);
+}
+
+
+/**
+ * Checks if the hyper debug state is active.
+ *
+ * @returns boolean
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(bool) CPUMIsHyperDebugStateActive(PVMCPU pVCpu)
+{
+    return RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_HYPER);
+}
+
+
+/**
+ * Checks if the hyper debug state is to be made active during the world-switch
+ * (currently only used for the 32->64 switcher case).
+ *
+ * @returns boolean
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(bool) CPUMIsHyperDebugStateActivePending(PVMCPU pVCpu)
+{
+    return RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_SYNC_DEBUG_REGS_HYPER);
+}
+
+
+/**
+ * Mark the guest's debug state as inactive.
+ *
+ * @returns boolean
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @todo    This API doesn't make sense any more.
+ */
+VMMDECL(void) CPUMDeactivateGuestDebugState(PVMCPU pVCpu)
+{
+    Assert(!(pVCpu->cpum.s.fUseFlags & (CPUM_USED_DEBUG_REGS_GUEST | CPUM_USED_DEBUG_REGS_HYPER | CPUM_USED_DEBUG_REGS_HOST)));
+    NOREF(pVCpu);
+}
+
+
+/**
+ * Get the current privilege level of the guest.
+ *
+ * @returns CPL
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(uint32_t) CPUMGetGuestCPL(PVMCPU pVCpu)
+{
+    /*
+     * CPL can reliably be found in SS.DPL (hidden regs valid) or SS if not.
+     *
+     * Note! We used to check CS.DPL here, assuming it was always equal to
+     * CPL even if a conforming segment was loaded.  But this turned out to
+     * only apply to older AMD-V.  With VT-x we had an ACP2 regression
+     * during install after a far call to ring 2 with VT-x.  Then on newer
+     * AMD-V CPUs we have to move the VMCB.guest.u8CPL into cs.Attr.n.u2Dpl
+     * as well as ss.Attr.n.u2Dpl to make this (and other) code work right.
+     *
+     * So, forget CS.DPL, always use SS.DPL.
+     *
+     * Note! The SS RPL is always equal to the CPL, while the CS RPL
+     * isn't necessarily equal if the segment is conforming.
+     * See section 4.11.1 in the AMD manual.
+     *
+     * Update: Where the heck does it say CS.RPL can differ from CPL other than
+     *         right after real->prot mode switch and when in V8086 mode?  That
+     *         section says the RPL specified in a direct transfere (call, jmp,
+     *         ret) is not the one loaded into CS. Besides, if CS.RPL != CPL
+     *         it would be impossible for an exception handle or the iret
+     *         instruction to figure out whether SS:ESP are part of the frame
+     *         or not.  VBox or qemu bug must've lead to this misconception.
+     *
+     * Update2: On an AMD bulldozer system here, I've no trouble loading a null
+     *         selector into SS with an RPL other than the CPL when CPL != 3 and
+     *         we're in 64-bit mode.  The intel dev box doesn't allow this, on
+     *         RPL = CPL.  Weird.
+     */
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
+    uint32_t uCpl;
+    if (pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE)
+    {
+        if (!pVCpu->cpum.s.Guest.eflags.Bits.u1VM)
+        {
+            if (CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.s.Guest.ss))
+                uCpl = pVCpu->cpum.s.Guest.ss.Attr.n.u2Dpl;
+            else
+                uCpl = (pVCpu->cpum.s.Guest.ss.Sel & X86_SEL_RPL);
+        }
+        else
+            uCpl = 3; /* V86 has CPL=3; REM doesn't set DPL=3 in V8086 mode. See @bugref{5130}. */
+    }
+    else
+        uCpl = 0;     /* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */
+    return uCpl;
+}
+
+
+/**
+ * Gets the current guest CPU mode.
+ *
+ * If paging mode is what you need, check out PGMGetGuestMode().
+ *
+ * @returns The CPU mode.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(CPUMMODE) CPUMGetGuestMode(PVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER);
+    CPUMMODE enmMode;
+    if (!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE))
+        enmMode = CPUMMODE_REAL;
+    else if (!(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA))
+        enmMode = CPUMMODE_PROTECTED;
+    else
+        enmMode = CPUMMODE_LONG;
+
+    return enmMode;
+}
+
+
+/**
+ * Figure whether the CPU is currently executing 16, 32 or 64 bit code.
+ *
+ * @returns 16, 32 or 64.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(uint32_t)       CPUMGetGuestCodeBits(PVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS);
+
+    if (!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE))
+        return 16;
+
+    if (pVCpu->cpum.s.Guest.eflags.Bits.u1VM)
+    {
+        Assert(!(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA));
+        return 16;
+    }
+
+    CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.cs);
+    if (   pVCpu->cpum.s.Guest.cs.Attr.n.u1Long
+        && (pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA))
+        return 64;
+
+    if (pVCpu->cpum.s.Guest.cs.Attr.n.u1DefBig)
+        return 32;
+
+    return 16;
+}
+
+
+VMMDECL(DISCPUMODE)     CPUMGetGuestDisMode(PVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS);
+
+    if (!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE))
+        return DISCPUMODE_16BIT;
+
+    if (pVCpu->cpum.s.Guest.eflags.Bits.u1VM)
+    {
+        Assert(!(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA));
+        return DISCPUMODE_16BIT;
+    }
+
+    CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.cs);
+    if (   pVCpu->cpum.s.Guest.cs.Attr.n.u1Long
+        && (pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA))
+        return DISCPUMODE_64BIT;
+
+    if (pVCpu->cpum.s.Guest.cs.Attr.n.u1DefBig)
+        return DISCPUMODE_32BIT;
+
+    return DISCPUMODE_16BIT;
+}
+
+
+/**
+ * Gets the guest MXCSR_MASK value.
+ *
+ * This does not access the x87 state, but the value we determined at VM
+ * initialization.
+ *
+ * @returns MXCSR mask.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMDECL(uint32_t) CPUMGetGuestMxCsrMask(PVM pVM)
+{
+    return pVM->cpum.s.GuestInfo.fMxCsrMask;
+}
+
+
+/**
+ * Returns whether the guest has physical interrupts enabled.
+ *
+ * @returns @c true if interrupts are enabled, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remarks Warning! This function does -not- take into account the global-interrupt
+ *          flag (GIF).
+ */
+VMM_INT_DECL(bool) CPUMIsGuestPhysIntrEnabled(PVMCPU pVCpu)
+{
+    if (!CPUMIsGuestInNestedHwvirtMode(&pVCpu->cpum.s.Guest))
+    {
+        uint32_t const fEFlags = pVCpu->cpum.s.Guest.eflags.u;
+        return RT_BOOL(fEFlags & X86_EFL_IF);
+    }
+
+    if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest))
+        return CPUMIsGuestVmxPhysIntrEnabled(&pVCpu->cpum.s.Guest);
+
+    Assert(CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.s.Guest));
+    return CPUMIsGuestSvmPhysIntrEnabled(pVCpu, &pVCpu->cpum.s.Guest);
+}
+
+
+/**
+ * Returns whether the nested-guest has virtual interrupts enabled.
+ *
+ * @returns @c true if interrupts are enabled, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remarks Warning! This function does -not- take into account the global-interrupt
+ *          flag (GIF).
+ */
+VMM_INT_DECL(bool) CPUMIsGuestVirtIntrEnabled(PVMCPU pVCpu)
+{
+    PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    Assert(CPUMIsGuestInNestedHwvirtMode(pCtx));
+
+    if (CPUMIsGuestInVmxNonRootMode(pCtx))
+        return CPUMIsGuestVmxVirtIntrEnabled(pCtx);
+
+    Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
+    return CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
+}
+
+
+/**
+ * Calculates the interruptiblity of the guest.
+ *
+ * @returns Interruptibility level.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(CPUMINTERRUPTIBILITY) CPUMGetGuestInterruptibility(PVMCPU pVCpu)
+{
+#if 1
+    /* Global-interrupt flag blocks pretty much everything we care about here. */
+    if (CPUMGetGuestGif(&pVCpu->cpum.s.Guest))
+    {
+        /*
+         * Physical interrupts are primarily blocked using EFLAGS. However, we cannot access
+         * it directly here. If and how EFLAGS are used depends on the context (nested-guest
+         * or raw-mode). Hence we use the function below which handles the details.
+         */
+        if (    CPUMIsGuestPhysIntrEnabled(pVCpu)
+            && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_BLOCK_NMIS | VMCPU_FF_INHIBIT_INTERRUPTS))
+        {
+            if (   !CPUMIsGuestInNestedHwvirtMode(&pVCpu->cpum.s.Guest)
+                ||  CPUMIsGuestVirtIntrEnabled(pVCpu))
+                return CPUMINTERRUPTIBILITY_UNRESTRAINED;
+
+            /* Physical interrupts are enabled, but nested-guest virtual interrupts are disabled. */
+            return CPUMINTERRUPTIBILITY_VIRT_INT_DISABLED;
+        }
+
+        /*
+         * Blocking the delivery of NMIs during an interrupt shadow is CPU implementation
+         * specific. Therefore, in practice, we can't deliver an NMI in an interrupt shadow.
+         * However, there is some uncertainity regarding the converse, i.e. whether
+         * NMI-blocking until IRET blocks delivery of physical interrupts.
+         *
+         * See Intel spec. 25.4.1 "Event Blocking".
+         */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+            return CPUMINTERRUPTIBILITY_NMI_INHIBIT;
+
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+            return CPUMINTERRUPTIBILITY_INT_INHIBITED;
+
+        return CPUMINTERRUPTIBILITY_INT_DISABLED;
+    }
+    return CPUMINTERRUPTIBILITY_GLOBAL_INHIBIT;
+#else
+    if (pVCpu->cpum.s.Guest.rflags.Bits.u1IF)
+    {
+        if (pVCpu->cpum.s.Guest.hwvirt.fGif)
+        {
+            if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_BLOCK_NMIS | VMCPU_FF_INHIBIT_INTERRUPTS))
+                return CPUMINTERRUPTIBILITY_UNRESTRAINED;
+
+            /** @todo does blocking NMIs mean interrupts are also inhibited? */
+            if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+            {
+                if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+                    return CPUMINTERRUPTIBILITY_INT_INHIBITED;
+                return CPUMINTERRUPTIBILITY_NMI_INHIBIT;
+            }
+            AssertFailed();
+            return CPUMINTERRUPTIBILITY_NMI_INHIBIT;
+        }
+        return CPUMINTERRUPTIBILITY_GLOBAL_INHIBIT;
+    }
+    else
+    {
+        if (pVCpu->cpum.s.Guest.hwvirt.fGif)
+        {
+            if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+                return CPUMINTERRUPTIBILITY_NMI_INHIBIT;
+            return CPUMINTERRUPTIBILITY_INT_DISABLED;
+        }
+        return CPUMINTERRUPTIBILITY_GLOBAL_INHIBIT;
+    }
+#endif
+}
+
+
+/**
+ * Gets whether the guest (or nested-guest) is currently blocking delivery of NMIs.
+ *
+ * @returns @c true if NMIs are blocked, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestNmiBlocking(PCVMCPU pVCpu)
+{
+    /*
+     * Return the state of guest-NMI blocking in any of the following cases:
+     *   - We're not executing a nested-guest.
+     *   - We're executing an SVM nested-guest[1].
+     *   - We're executing a VMX nested-guest without virtual-NMIs enabled.
+     *
+     * [1] -- SVM does not support virtual-NMIs or virtual-NMI blocking.
+     *        SVM hypervisors must track NMI blocking themselves by intercepting
+     *        the IRET instruction after injection of an NMI.
+     */
+    PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    if (   !CPUMIsGuestInNestedHwvirtMode(pCtx)
+        ||  CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
+        || !CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_VIRT_NMI))
+        return VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
+
+    /*
+     * Return the state of virtual-NMI blocking, if we are executing a
+     * VMX nested-guest with virtual-NMIs enabled.
+     */
+    return CPUMIsGuestVmxVirtNmiBlocking(pCtx);
+}
+
+
+/**
+ * Sets blocking delivery of NMIs to the guest.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   fBlock  Whether NMIs are blocked or not.
+ */
+VMM_INT_DECL(void) CPUMSetGuestNmiBlocking(PVMCPU pVCpu, bool fBlock)
+{
+    /*
+     * Set the state of guest-NMI blocking in any of the following cases:
+     *   - We're not executing a nested-guest.
+     *   - We're executing an SVM nested-guest[1].
+     *   - We're executing a VMX nested-guest without virtual-NMIs enabled.
+     *
+     * [1] -- SVM does not support virtual-NMIs or virtual-NMI blocking.
+     *        SVM hypervisors must track NMI blocking themselves by intercepting
+     *        the IRET instruction after injection of an NMI.
+     */
+    PCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    if (   !CPUMIsGuestInNestedHwvirtMode(pCtx)
+        ||  CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
+        || !CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_VIRT_NMI))
+    {
+        if (fBlock)
+        {
+            if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
+        }
+        else
+        {
+            if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
+        }
+        return;
+    }
+
+    /*
+     * Set the state of virtual-NMI blocking, if we are executing a
+     * VMX nested-guest with virtual-NMIs enabled.
+     */
+    return CPUMSetGuestVmxVirtNmiBlocking(pCtx, fBlock);
+}
+
+
+/**
+ * Checks whether the SVM nested-guest has physical interrupts enabled.
+ *
+ * @returns true if interrupts are enabled, false otherwise.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   pCtx    The guest-CPU context.
+ *
+ * @remarks This does -not- take into account the global-interrupt flag.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestSvmPhysIntrEnabled(PCVMCPU pVCpu, PCCPUMCTX pCtx)
+{
+    /** @todo Optimization: Avoid this function call and use a pointer to the
+     *        relevant eflags instead (setup during VMRUN instruction emulation). */
+    Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
+
+    X86EFLAGS fEFlags;
+    if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, pCtx))
+        fEFlags.u = pCtx->hwvirt.svm.HostState.rflags.u;
+    else
+        fEFlags.u = pCtx->eflags.u;
+
+    return fEFlags.Bits.u1IF;
+}
+
+
+/**
+ * Checks whether the SVM nested-guest is in a state to receive virtual (setup
+ * for injection by VMRUN instruction) interrupts.
+ *
+ * @returns VBox status code.
+ * @retval  true if it's ready, false otherwise.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   pCtx    The guest-CPU context.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestSvmVirtIntrEnabled(PCVMCPU pVCpu, PCCPUMCTX pCtx)
+{
+    RT_NOREF(pVCpu);
+    Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
+
+    PCSVMVMCBCTRL pVmcbCtrl    = &pCtx->hwvirt.svm.CTX_SUFF(pVmcb)->ctrl;
+    PCSVMINTCTRL  pVmcbIntCtrl = &pVmcbCtrl->IntCtrl;
+    Assert(!pVmcbIntCtrl->n.u1VGifEnable);      /* We don't support passing virtual-GIF feature to the guest yet. */
+    if (   !pVmcbIntCtrl->n.u1IgnoreTPR
+        &&  pVmcbIntCtrl->n.u4VIntrPrio <= pVmcbIntCtrl->n.u8VTPR)
+        return false;
+
+    return RT_BOOL(pCtx->eflags.u & X86_EFL_IF);
+}
+
+
+/**
+ * Gets the pending SVM nested-guest interruptvector.
+ *
+ * @returns The nested-guest interrupt to inject.
+ * @param   pCtx            The guest-CPU context.
+ */
+VMM_INT_DECL(uint8_t) CPUMGetGuestSvmVirtIntrVector(PCCPUMCTX pCtx)
+{
+    PCSVMVMCBCTRL pVmcbCtrl = &pCtx->hwvirt.svm.CTX_SUFF(pVmcb)->ctrl;
+    return pVmcbCtrl->IntCtrl.n.u8VIntrVector;
+}
+
+
+/**
+ * Restores the host-state from the host-state save area as part of a \#VMEXIT.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pCtx        The guest-CPU context.
+ */
+VMM_INT_DECL(void) CPUMSvmVmExitRestoreHostState(PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+    /*
+     * Reload the guest's "host state".
+     */
+    PSVMHOSTSTATE pHostState = &pCtx->hwvirt.svm.HostState;
+    pCtx->es         = pHostState->es;
+    pCtx->cs         = pHostState->cs;
+    pCtx->ss         = pHostState->ss;
+    pCtx->ds         = pHostState->ds;
+    pCtx->gdtr       = pHostState->gdtr;
+    pCtx->idtr       = pHostState->idtr;
+    CPUMSetGuestEferMsrNoChecks(pVCpu, pCtx->msrEFER, pHostState->uEferMsr);
+    CPUMSetGuestCR0(pVCpu, pHostState->uCr0 | X86_CR0_PE);
+    pCtx->cr3        = pHostState->uCr3;
+    CPUMSetGuestCR4(pVCpu, pHostState->uCr4);
+    pCtx->rflags     = pHostState->rflags;
+    pCtx->rflags.Bits.u1VM = 0;
+    pCtx->rip        = pHostState->uRip;
+    pCtx->rsp        = pHostState->uRsp;
+    pCtx->rax        = pHostState->uRax;
+    pCtx->dr[7]     &= ~(X86_DR7_ENABLED_MASK | X86_DR7_RAZ_MASK | X86_DR7_MBZ_MASK);
+    pCtx->dr[7]     |= X86_DR7_RA1_MASK;
+    Assert(pCtx->ss.Attr.n.u2Dpl == 0);
+
+    /** @todo if RIP is not canonical or outside the CS segment limit, we need to
+     *        raise \#GP(0) in the guest. */
+
+    /** @todo check the loaded host-state for consistency. Figure out what
+     *        exactly this involves? */
+}
+
+
+/**
+ * Saves the host-state to the host-state save area as part of a VMRUN.
+ *
+ * @param   pCtx        The guest-CPU context.
+ * @param   cbInstr     The length of the VMRUN instruction in bytes.
+ */
+VMM_INT_DECL(void) CPUMSvmVmRunSaveHostState(PCPUMCTX pCtx, uint8_t cbInstr)
+{
+    PSVMHOSTSTATE pHostState = &pCtx->hwvirt.svm.HostState;
+    pHostState->es       = pCtx->es;
+    pHostState->cs       = pCtx->cs;
+    pHostState->ss       = pCtx->ss;
+    pHostState->ds       = pCtx->ds;
+    pHostState->gdtr     = pCtx->gdtr;
+    pHostState->idtr     = pCtx->idtr;
+    pHostState->uEferMsr = pCtx->msrEFER;
+    pHostState->uCr0     = pCtx->cr0;
+    pHostState->uCr3     = pCtx->cr3;
+    pHostState->uCr4     = pCtx->cr4;
+    pHostState->rflags   = pCtx->rflags;
+    pHostState->uRip     = pCtx->rip + cbInstr;
+    pHostState->uRsp     = pCtx->rsp;
+    pHostState->uRax     = pCtx->rax;
+}
+
+
+/**
+ * Applies the TSC offset of a nested-guest if any and returns the TSC value for the
+ * nested-guest.
+ *
+ * @returns The TSC offset after applying any nested-guest TSC offset.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uTscValue   The guest TSC.
+ *
+ * @sa      CPUMRemoveNestedGuestTscOffset.
+ */
+VMM_INT_DECL(uint64_t) CPUMApplyNestedGuestTscOffset(PCVMCPU pVCpu, uint64_t uTscValue)
+{
+    PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    if (CPUMIsGuestInVmxNonRootMode(pCtx))
+    {
+        PCVMXVVMCS pVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
+        Assert(pVmcs);
+        if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_USE_TSC_OFFSETTING))
+            return uTscValue + pVmcs->u64TscOffset.u;
+        return uTscValue;
+    }
+
+    if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+    {
+        uint64_t offTsc;
+        if (!HMGetGuestSvmTscOffset(pVCpu, &offTsc))
+        {
+            PCSVMVMCB pVmcb = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
+            Assert(pVmcb);
+            offTsc = pVmcb->ctrl.u64TSCOffset;
+        }
+        return uTscValue + offTsc;
+    }
+    return uTscValue;
+}
+
+
+/**
+ * Removes the TSC offset of a nested-guest if any and returns the TSC value for the
+ * guest.
+ *
+ * @returns The TSC offset after removing any nested-guest TSC offset.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uTscValue   The nested-guest TSC.
+ *
+ * @sa      CPUMApplyNestedGuestTscOffset.
+ */
+VMM_INT_DECL(uint64_t) CPUMRemoveNestedGuestTscOffset(PCVMCPU pVCpu, uint64_t uTscValue)
+{
+    PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    if (CPUMIsGuestInVmxNonRootMode(pCtx))
+    {
+        if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_USE_TSC_OFFSETTING))
+        {
+            PCVMXVVMCS pVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
+            Assert(pVmcs);
+            return uTscValue - pVmcs->u64TscOffset.u;
+        }
+        return uTscValue;
+    }
+
+    if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+    {
+        uint64_t offTsc;
+        if (!HMGetGuestSvmTscOffset(pVCpu, &offTsc))
+        {
+            PCSVMVMCB pVmcb = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
+            Assert(pVmcb);
+            offTsc = pVmcb->ctrl.u64TSCOffset;
+        }
+        return uTscValue - offTsc;
+    }
+    return uTscValue;
+}
+
+
+/**
+ * Used to dynamically imports state residing in NEM or HM.
+ *
+ * This is a worker for the CPUM_IMPORT_EXTRN_RET() macro and various IEM ones.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   fExtrnImport    The fields to import.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(int) CPUMImportGuestStateOnDemand(PVMCPUCC pVCpu, uint64_t fExtrnImport)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    if (pVCpu->cpum.s.Guest.fExtrn & fExtrnImport)
+    {
+        switch (pVCpu->cpum.s.Guest.fExtrn & CPUMCTX_EXTRN_KEEPER_MASK)
+        {
+            case CPUMCTX_EXTRN_KEEPER_NEM:
+            {
+                int rc = NEMImportStateOnDemand(pVCpu, fExtrnImport);
+                Assert(rc == VINF_SUCCESS || RT_FAILURE_NP(rc));
+                return rc;
+            }
+
+            case CPUMCTX_EXTRN_KEEPER_HM:
+            {
+#ifdef IN_RING0
+                int rc = HMR0ImportStateOnDemand(pVCpu, fExtrnImport);
+                Assert(rc == VINF_SUCCESS || RT_FAILURE_NP(rc));
+                return rc;
+#else
+                AssertLogRelMsgFailed(("TODO Fetch HM state: %#RX64 vs %#RX64\n", pVCpu->cpum.s.Guest.fExtrn, fExtrnImport));
+                return VINF_SUCCESS;
+#endif
+            }
+            default:
+                AssertLogRelMsgFailedReturn(("%#RX64 vs %#RX64\n", pVCpu->cpum.s.Guest.fExtrn, fExtrnImport), VERR_CPUM_IPE_2);
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets valid CR4 bits for the guest.
+ *
+ * @returns Valid CR4 bits.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestCR4ValidMask(PVM pVM)
+{
+    PCCPUMFEATURES pGuestFeatures = &pVM->cpum.s.GuestFeatures;
+    uint64_t fMask = X86_CR4_VME | X86_CR4_PVI
+                   | X86_CR4_TSD | X86_CR4_DE
+                   | X86_CR4_PSE | X86_CR4_PAE
+                   | X86_CR4_MCE | X86_CR4_PGE
+                   | X86_CR4_PCE
+                   | X86_CR4_OSXMMEEXCPT;  /** @todo r=ramshankar: Introduced in Pentium III along with SSE. Check fSse here? */
+    if (pGuestFeatures->fFxSaveRstor)
+        fMask |= X86_CR4_OSFXSR;
+    if (pGuestFeatures->fVmx)
+        fMask |= X86_CR4_VMXE;
+    if (pGuestFeatures->fXSaveRstor)
+        fMask |= X86_CR4_OSXSAVE;
+    if (pGuestFeatures->fPcid)
+        fMask |= X86_CR4_PCIDE;
+    if (pGuestFeatures->fFsGsBase)
+        fMask |= X86_CR4_FSGSBASE;
+    return fMask;
+}
+
+
+/**
+ * Starts a VMX-preemption timer to expire as specified by the nested hypervisor.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   uTimer          The VMCS preemption timer value.
+ * @param   cShift          The VMX-preemption timer shift (usually based on guest
+ *                          VMX MSR rate).
+ * @param   pu64EntryTick   Where to store the current tick when the timer is
+ *                          programmed.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(int) CPUMStartGuestVmxPremptTimer(PVMCPUCC pVCpu, uint32_t uTimer, uint8_t cShift, uint64_t *pu64EntryTick)
+{
+    Assert(uTimer);
+    Assert(cShift <= 31);
+    Assert(pu64EntryTick);
+    VMCPU_ASSERT_EMT(pVCpu);
+    uint64_t const cTicksToNext = uTimer << cShift;
+    return TMTimerSetRelative(pVCpu->cpum.s.CTX_SUFF(pNestedVmxPreemptTimer), cTicksToNext, pu64EntryTick);
+}
+
+
+/**
+ * Stops the VMX-preemption timer from firing.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling thread.
+ * @thread  EMT.
+ *
+ * @remarks This can be called during VM reset, so we cannot assume it will be on
+ *          the EMT corresponding to @c pVCpu.
+ */
+VMM_INT_DECL(int) CPUMStopGuestVmxPremptTimer(PVMCPUCC pVCpu)
+{
+    /*
+     * CPUM gets initialized before TM, so we defer creation of timers till CPUMR3InitCompleted().
+     * However, we still get called during CPUMR3Init() and hence we need to check if we  have
+     * a valid timer object before trying to stop it.
+     */
+    PTMTIMER pTimer = pVCpu->cpum.s.CTX_SUFF(pNestedVmxPreemptTimer);
+    if (!pTimer)
+        return VERR_NOT_FOUND;
+
+    int rc = TMTimerLock(pTimer, VERR_IGNORED);
+    if (rc == VINF_SUCCESS)
+    {
+        if (TMTimerIsActive(pTimer))
+            TMTimerStop(pTimer);
+        TMTimerUnlock(pTimer);
+    }
+    return rc;
+}
+
+
+/**
+ * Gets the read and write permission bits for an MSR in an MSR bitmap.
+ *
+ * @returns VMXMSRPM_XXX - the MSR permission.
+ * @param   pvMsrBitmap     Pointer to the MSR bitmap.
+ * @param   idMsr           The MSR to get permissions for.
+ *
+ * @sa      hmR0VmxSetMsrPermission.
+ */
+VMM_INT_DECL(uint32_t) CPUMGetVmxMsrPermission(void const *pvMsrBitmap, uint32_t idMsr)
+{
+    AssertPtrReturn(pvMsrBitmap, VMXMSRPM_EXIT_RD | VMXMSRPM_EXIT_WR);
+
+    uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
+
+    /*
+     * MSR Layout:
+     *   Byte index            MSR range            Interpreted as
+     * 0x000 - 0x3ff    0x00000000 - 0x00001fff    Low MSR read bits.
+     * 0x400 - 0x7ff    0xc0000000 - 0xc0001fff    High MSR read bits.
+     * 0x800 - 0xbff    0x00000000 - 0x00001fff    Low MSR write bits.
+     * 0xc00 - 0xfff    0xc0000000 - 0xc0001fff    High MSR write bits.
+     *
+     * A bit corresponding to an MSR within the above range causes a VM-exit
+     * if the bit is 1 on executions of RDMSR/WRMSR.  If an MSR falls out of
+     * the MSR range, it always cause a VM-exit.
+     *
+     * See Intel spec. 24.6.9 "MSR-Bitmap Address".
+     */
+    uint32_t const offBitmapRead  = 0;
+    uint32_t const offBitmapWrite = 0x800;
+    uint32_t       offMsr;
+    uint32_t       iBit;
+    if (idMsr <= UINT32_C(0x00001fff))
+    {
+        offMsr = 0;
+        iBit   = idMsr;
+    }
+    else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
+    {
+        offMsr = 0x400;
+        iBit   = idMsr - UINT32_C(0xc0000000);
+    }
+    else
+    {
+        LogFunc(("Warning! Out of range MSR %#RX32\n", idMsr));
+        return VMXMSRPM_EXIT_RD | VMXMSRPM_EXIT_WR;
+    }
+
+    /*
+     * Get the MSR read permissions.
+     */
+    uint32_t fRet;
+    uint32_t const offMsrRead = offBitmapRead + offMsr;
+    Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
+    if (ASMBitTest(pbMsrBitmap + offMsrRead, iBit))
+        fRet = VMXMSRPM_EXIT_RD;
+    else
+        fRet = VMXMSRPM_ALLOW_RD;
+
+    /*
+     * Get the MSR write permissions.
+     */
+    uint32_t const offMsrWrite = offBitmapWrite + offMsr;
+    Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
+    if (ASMBitTest(pbMsrBitmap + offMsrWrite, iBit))
+        fRet |= VMXMSRPM_EXIT_WR;
+    else
+        fRet |= VMXMSRPM_ALLOW_WR;
+
+    Assert(VMXMSRPM_IS_FLAG_VALID(fRet));
+    return fRet;
+}
+
+
+/**
+ * Checks the permission bits for the specified I/O port from the given I/O bitmap
+ * to see if causes a VM-exit.
+ *
+ * @returns @c true if the I/O port access must cause a VM-exit, @c false otherwise.
+ * @param   pvIoBitmap  Pointer to I/O bitmap.
+ * @param   uPort       The I/O port being accessed.
+ * @param   cbAccess    e size of the I/O access in bytes (1, 2 or 4 bytes).
+ */
+static bool cpumGetVmxIoBitmapPermission(void const *pvIoBitmap, uint16_t uPort, uint8_t cbAccess)
+{
+    Assert(cbAccess == 1 || cbAccess == 2 || cbAccess == 4);
+
+    /*
+     * If the I/O port access wraps around the 16-bit port I/O space, we must cause a
+     * VM-exit.
+     *
+     * Reading 1, 2, 4 bytes at ports 0xffff, 0xfffe and 0xfffc are valid and do not
+     * constitute a wrap around. However, reading 2 bytes at port 0xffff or 4 bytes
+     * from port 0xffff/0xfffe/0xfffd constitute a wrap around. In other words, any
+     * access to -both- ports 0xffff and port 0 is a wrap around.
+     *
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    uint32_t const uPortLast = uPort + cbAccess;
+    if (uPortLast > 0x10000)
+        return true;
+
+    /*
+     * If any bit corresponding to the I/O access is set, we must cause a VM-exit.
+     */
+    uint8_t const *pbIoBitmap = (uint8_t const *)pvIoBitmap;
+    uint16_t const offPerm    = uPort >> 3;                         /* Byte offset of the port. */
+    uint16_t const idxPermBit = uPort - (offPerm << 3);             /* Bit offset within byte. */
+    Assert(idxPermBit < 8);
+    static const uint8_t s_afMask[] = { 0x0, 0x1, 0x3, 0x7, 0xf };  /* Bit-mask for all access sizes. */
+    uint16_t const fMask = s_afMask[cbAccess] << idxPermBit;        /* Bit-mask of the access. */
+
+    /* Fetch 8 or 16-bits depending on whether the access spans 8-bit boundary. */
+    RTUINT16U uPerm;
+    uPerm.s.Lo = *(pbIoBitmap + offPerm);
+    if (idxPermBit + cbAccess > 8)
+        uPerm.s.Hi = *(pbIoBitmap + 1 + offPerm);
+    else
+        uPerm.s.Hi = 0;
+
+    /* If any bit for the access is 1, we must cause a VM-exit. */
+    if (uPerm.u & fMask)
+        return true;
+
+    return false;
+}
+
+
+/**
+ * Returns whether the given VMCS field is valid and supported for the guest.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   u64VmcsField    The VMCS field.
+ *
+ * @remarks This takes into account the CPU features exposed to the guest.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestVmxVmcsFieldValid(PVMCC pVM, uint64_t u64VmcsField)
+{
+    uint32_t const uFieldEncHi = RT_HI_U32(u64VmcsField);
+    uint32_t const uFieldEncLo = RT_LO_U32(u64VmcsField);
+    if (!uFieldEncHi)
+    { /* likely */ }
+    else
+        return false;
+
+    PCCPUMFEATURES pFeat = &pVM->cpum.s.GuestFeatures;
+    switch (uFieldEncLo)
+    {
+        /*
+         * 16-bit fields.
+         */
+        /* Control fields. */
+        case VMX_VMCS16_VPID:                             return pFeat->fVmxVpid;
+        case VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR:         return pFeat->fVmxPostedInt;
+        case VMX_VMCS16_EPTP_INDEX:                       return pFeat->fVmxEptXcptVe;
+
+        /* Guest-state fields. */
+        case VMX_VMCS16_GUEST_ES_SEL:
+        case VMX_VMCS16_GUEST_CS_SEL:
+        case VMX_VMCS16_GUEST_SS_SEL:
+        case VMX_VMCS16_GUEST_DS_SEL:
+        case VMX_VMCS16_GUEST_FS_SEL:
+        case VMX_VMCS16_GUEST_GS_SEL:
+        case VMX_VMCS16_GUEST_LDTR_SEL:
+        case VMX_VMCS16_GUEST_TR_SEL:                     return true;
+        case VMX_VMCS16_GUEST_INTR_STATUS:                return pFeat->fVmxVirtIntDelivery;
+        case VMX_VMCS16_GUEST_PML_INDEX:                  return pFeat->fVmxPml;
+
+        /* Host-state fields. */
+        case VMX_VMCS16_HOST_ES_SEL:
+        case VMX_VMCS16_HOST_CS_SEL:
+        case VMX_VMCS16_HOST_SS_SEL:
+        case VMX_VMCS16_HOST_DS_SEL:
+        case VMX_VMCS16_HOST_FS_SEL:
+        case VMX_VMCS16_HOST_GS_SEL:
+        case VMX_VMCS16_HOST_TR_SEL:                      return true;
+
+        /*
+         * 64-bit fields.
+         */
+        /* Control fields. */
+        case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
+        case VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH:
+        case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
+        case VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH:            return pFeat->fVmxUseIoBitmaps;
+        case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
+        case VMX_VMCS64_CTRL_MSR_BITMAP_HIGH:             return pFeat->fVmxUseMsrBitmaps;
+        case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
+        case VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH:
+        case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
+        case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH:
+        case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
+        case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH:
+        case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
+        case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH:          return true;
+        case VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL:
+        case VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH:          return pFeat->fVmxPml;
+        case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
+        case VMX_VMCS64_CTRL_TSC_OFFSET_HIGH:             return true;
+        case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
+        case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH:     return pFeat->fVmxUseTprShadow;
+        case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
+        case VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH:        return pFeat->fVmxVirtApicAccess;
+        case VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL:
+        case VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH:       return pFeat->fVmxPostedInt;
+        case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
+        case VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH:           return pFeat->fVmxVmFunc;
+        case VMX_VMCS64_CTRL_EPTP_FULL:
+        case VMX_VMCS64_CTRL_EPTP_HIGH:                   return pFeat->fVmxEpt;
+        case VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH:           return pFeat->fVmxVirtIntDelivery;
+        case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
+        case VMX_VMCS64_CTRL_EPTP_LIST_HIGH:
+        {
+            PCVMCPU pVCpu = pVM->CTX_SUFF(apCpus)[0];
+            uint64_t const uVmFuncMsr = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64VmFunc;
+            return RT_BOOL(RT_BF_GET(uVmFuncMsr, VMX_BF_VMFUNC_EPTP_SWITCHING));
+        }
+        case VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL:
+        case VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH:
+        case VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL:
+        case VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH:         return pFeat->fVmxVmcsShadowing;
+        case VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL:
+        case VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH:     return pFeat->fVmxEptXcptVe;
+        case VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL:
+        case VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH:     return pFeat->fVmxXsavesXrstors;
+        case VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL:
+        case VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH:   return false;
+        case VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL:
+        case VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH:         return pFeat->fVmxUseTscScaling;
+
+        /* Read-only data fields. */
+        case VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL:
+        case VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH:          return pFeat->fVmxEpt;
+
+        /* Guest-state fields. */
+        case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
+        case VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH:
+        case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
+        case VMX_VMCS64_GUEST_DEBUGCTL_HIGH:              return true;
+        case VMX_VMCS64_GUEST_PAT_FULL:
+        case VMX_VMCS64_GUEST_PAT_HIGH:                   return pFeat->fVmxEntryLoadPatMsr || pFeat->fVmxExitSavePatMsr;
+        case VMX_VMCS64_GUEST_EFER_FULL:
+        case VMX_VMCS64_GUEST_EFER_HIGH:                  return pFeat->fVmxEntryLoadEferMsr || pFeat->fVmxExitSaveEferMsr;
+        case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
+        case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH:      return false;
+        case VMX_VMCS64_GUEST_PDPTE0_FULL:
+        case VMX_VMCS64_GUEST_PDPTE0_HIGH:
+        case VMX_VMCS64_GUEST_PDPTE1_FULL:
+        case VMX_VMCS64_GUEST_PDPTE1_HIGH:
+        case VMX_VMCS64_GUEST_PDPTE2_FULL:
+        case VMX_VMCS64_GUEST_PDPTE2_HIGH:
+        case VMX_VMCS64_GUEST_PDPTE3_FULL:
+        case VMX_VMCS64_GUEST_PDPTE3_HIGH:                return pFeat->fVmxEpt;
+        case VMX_VMCS64_GUEST_BNDCFGS_FULL:
+        case VMX_VMCS64_GUEST_BNDCFGS_HIGH:               return false;
+
+        /* Host-state fields. */
+        case VMX_VMCS64_HOST_PAT_FULL:
+        case VMX_VMCS64_HOST_PAT_HIGH:                    return pFeat->fVmxExitLoadPatMsr;
+        case VMX_VMCS64_HOST_EFER_FULL:
+        case VMX_VMCS64_HOST_EFER_HIGH:                   return pFeat->fVmxExitLoadEferMsr;
+        case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
+        case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH:       return false;
+
+        /*
+         * 32-bit fields.
+         */
+        /* Control fields. */
+        case VMX_VMCS32_CTRL_PIN_EXEC:
+        case VMX_VMCS32_CTRL_PROC_EXEC:
+        case VMX_VMCS32_CTRL_EXCEPTION_BITMAP:
+        case VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK:
+        case VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH:
+        case VMX_VMCS32_CTRL_CR3_TARGET_COUNT:
+        case VMX_VMCS32_CTRL_EXIT:
+        case VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT:
+        case VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT:
+        case VMX_VMCS32_CTRL_ENTRY:
+        case VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT:
+        case VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO:
+        case VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE:
+        case VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH:          return true;
+        case VMX_VMCS32_CTRL_TPR_THRESHOLD:               return pFeat->fVmxUseTprShadow;
+        case VMX_VMCS32_CTRL_PROC_EXEC2:                  return pFeat->fVmxSecondaryExecCtls;
+        case VMX_VMCS32_CTRL_PLE_GAP:
+        case VMX_VMCS32_CTRL_PLE_WINDOW:                  return pFeat->fVmxPauseLoopExit;
+
+        /* Read-only data fields. */
+        case VMX_VMCS32_RO_VM_INSTR_ERROR:
+        case VMX_VMCS32_RO_EXIT_REASON:
+        case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
+        case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
+        case VMX_VMCS32_RO_IDT_VECTORING_INFO:
+        case VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE:
+        case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
+        case VMX_VMCS32_RO_EXIT_INSTR_INFO:               return true;
+
+        /* Guest-state fields. */
+        case VMX_VMCS32_GUEST_ES_LIMIT:
+        case VMX_VMCS32_GUEST_CS_LIMIT:
+        case VMX_VMCS32_GUEST_SS_LIMIT:
+        case VMX_VMCS32_GUEST_DS_LIMIT:
+        case VMX_VMCS32_GUEST_FS_LIMIT:
+        case VMX_VMCS32_GUEST_GS_LIMIT:
+        case VMX_VMCS32_GUEST_LDTR_LIMIT:
+        case VMX_VMCS32_GUEST_TR_LIMIT:
+        case VMX_VMCS32_GUEST_GDTR_LIMIT:
+        case VMX_VMCS32_GUEST_IDTR_LIMIT:
+        case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_INT_STATE:
+        case VMX_VMCS32_GUEST_ACTIVITY_STATE:
+        case VMX_VMCS32_GUEST_SMBASE:
+        case VMX_VMCS32_GUEST_SYSENTER_CS:                return true;
+        case VMX_VMCS32_PREEMPT_TIMER_VALUE:              return pFeat->fVmxPreemptTimer;
+
+        /* Host-state fields. */
+        case VMX_VMCS32_HOST_SYSENTER_CS:                 return true;
+
+        /*
+         * Natural-width fields.
+         */
+        /* Control fields. */
+        case VMX_VMCS_CTRL_CR0_MASK:
+        case VMX_VMCS_CTRL_CR4_MASK:
+        case VMX_VMCS_CTRL_CR0_READ_SHADOW:
+        case VMX_VMCS_CTRL_CR4_READ_SHADOW:
+        case VMX_VMCS_CTRL_CR3_TARGET_VAL0:
+        case VMX_VMCS_CTRL_CR3_TARGET_VAL1:
+        case VMX_VMCS_CTRL_CR3_TARGET_VAL2:
+        case VMX_VMCS_CTRL_CR3_TARGET_VAL3:               return true;
+
+        /* Read-only data fields. */
+        case VMX_VMCS_RO_EXIT_QUALIFICATION:
+        case VMX_VMCS_RO_IO_RCX:
+        case VMX_VMCS_RO_IO_RSI:
+        case VMX_VMCS_RO_IO_RDI:
+        case VMX_VMCS_RO_IO_RIP:
+        case VMX_VMCS_RO_GUEST_LINEAR_ADDR:               return true;
+
+        /* Guest-state fields. */
+        case VMX_VMCS_GUEST_CR0:
+        case VMX_VMCS_GUEST_CR3:
+        case VMX_VMCS_GUEST_CR4:
+        case VMX_VMCS_GUEST_ES_BASE:
+        case VMX_VMCS_GUEST_CS_BASE:
+        case VMX_VMCS_GUEST_SS_BASE:
+        case VMX_VMCS_GUEST_DS_BASE:
+        case VMX_VMCS_GUEST_FS_BASE:
+        case VMX_VMCS_GUEST_GS_BASE:
+        case VMX_VMCS_GUEST_LDTR_BASE:
+        case VMX_VMCS_GUEST_TR_BASE:
+        case VMX_VMCS_GUEST_GDTR_BASE:
+        case VMX_VMCS_GUEST_IDTR_BASE:
+        case VMX_VMCS_GUEST_DR7:
+        case VMX_VMCS_GUEST_RSP:
+        case VMX_VMCS_GUEST_RIP:
+        case VMX_VMCS_GUEST_RFLAGS:
+        case VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS:
+        case VMX_VMCS_GUEST_SYSENTER_ESP:
+        case VMX_VMCS_GUEST_SYSENTER_EIP:                 return true;
+
+        /* Host-state fields. */
+        case VMX_VMCS_HOST_CR0:
+        case VMX_VMCS_HOST_CR3:
+        case VMX_VMCS_HOST_CR4:
+        case VMX_VMCS_HOST_FS_BASE:
+        case VMX_VMCS_HOST_GS_BASE:
+        case VMX_VMCS_HOST_TR_BASE:
+        case VMX_VMCS_HOST_GDTR_BASE:
+        case VMX_VMCS_HOST_IDTR_BASE:
+        case VMX_VMCS_HOST_SYSENTER_ESP:
+        case VMX_VMCS_HOST_SYSENTER_EIP:
+        case VMX_VMCS_HOST_RSP:
+        case VMX_VMCS_HOST_RIP:                           return true;
+    }
+
+    return false;
+}
+
+
+/**
+ * Checks whether the given I/O access should cause a nested-guest VM-exit.
+ *
+ * @returns @c true if it causes a VM-exit, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   u16Port     The I/O port being accessed.
+ * @param   cbAccess    The size of the I/O access in bytes (1, 2 or 4 bytes).
+ */
+VMM_INT_DECL(bool) CPUMIsGuestVmxIoInterceptSet(PCVMCPU pVCpu, uint16_t u16Port, uint8_t cbAccess)
+{
+    PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_UNCOND_IO_EXIT))
+        return true;
+
+    if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_USE_IO_BITMAPS))
+    {
+        uint8_t const *pbIoBitmap = (uint8_t const *)pCtx->hwvirt.vmx.CTX_SUFF(pvIoBitmap);
+        Assert(pbIoBitmap);
+        return cpumGetVmxIoBitmapPermission(pbIoBitmap, u16Port, cbAccess);
+    }
+
+    return false;
+}
+
+
+/**
+ * Checks whether the Mov-to-CR3 instruction causes a nested-guest VM-exit.
+ *
+ * @returns @c true if it causes a VM-exit, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uNewCr3     The CR3 value being written.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestVmxMovToCr3InterceptSet(PVMCPU pVCpu, uint64_t uNewCr3)
+{
+    /*
+     * If the CR3-load exiting control is set and the new CR3 value does not
+     * match any of the CR3-target values in the VMCS, we must cause a VM-exit.
+     *
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    PCCPUMCTX  pCtx  = &pVCpu->cpum.s.Guest;
+    PCVMXVVMCS pVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
+    if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_CR3_LOAD_EXIT))
+    {
+        uint32_t const uCr3TargetCount = pVmcs->u32Cr3TargetCount;
+        Assert(uCr3TargetCount <= VMX_V_CR3_TARGET_COUNT);
+
+        /* If the CR3-target count is 0, cause a VM-exit. */
+        if (uCr3TargetCount == 0)
+            return true;
+
+        /* If the CR3 being written doesn't match any of the target values, cause a VM-exit. */
+        AssertCompile(VMX_V_CR3_TARGET_COUNT == 4);
+        if (   uNewCr3 != pVmcs->u64Cr3Target0.u
+            && uNewCr3 != pVmcs->u64Cr3Target1.u
+            && uNewCr3 != pVmcs->u64Cr3Target2.u
+            && uNewCr3 != pVmcs->u64Cr3Target3.u)
+            return true;
+    }
+    return false;
+}
+
+
+/**
+ * Checks whether a VMREAD or VMWRITE instruction for the given VMCS field causes a
+ * VM-exit or not.
+ *
+ * @returns @c true if the VMREAD/VMWRITE is intercepted, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason (VMX_EXIT_VMREAD or
+ *                          VMX_EXIT_VMREAD).
+ * @param   u64VmcsField    The VMCS field.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestVmxVmreadVmwriteInterceptSet(PCVMCPU pVCpu, uint32_t uExitReason, uint64_t u64VmcsField)
+{
+    Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest));
+    Assert(   uExitReason == VMX_EXIT_VMREAD
+           || uExitReason == VMX_EXIT_VMWRITE);
+
+    /*
+     * Without VMCS shadowing, all VMREAD and VMWRITE instructions are intercepted.
+     */
+    if (!CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.s.Guest, VMX_PROC_CTLS2_VMCS_SHADOWING))
+        return true;
+
+    /*
+     * If any reserved bit in the 64-bit VMCS field encoding is set, the VMREAD/VMWRITE
+     * is intercepted. This excludes any reserved bits in the valid parts of the field
+     * encoding (i.e. bit 12).
+     */
+    if (u64VmcsField & VMX_VMCSFIELD_RSVD_MASK)
+        return true;
+
+    /*
+     * Finally, consult the VMREAD/VMWRITE bitmap whether to intercept the instruction or not.
+     */
+    uint32_t const u32VmcsField = RT_LO_U32(u64VmcsField);
+    uint8_t const *pbBitmap     = uExitReason == VMX_EXIT_VMREAD
+                                ? (uint8_t const *)pVCpu->cpum.s.Guest.hwvirt.vmx.CTX_SUFF(pvVmreadBitmap)
+                                : (uint8_t const *)pVCpu->cpum.s.Guest.hwvirt.vmx.CTX_SUFF(pvVmwriteBitmap);
+    Assert(pbBitmap);
+    Assert(u32VmcsField >> 3 < VMX_V_VMREAD_VMWRITE_BITMAP_SIZE);
+    return ASMBitTest(pbBitmap + (u32VmcsField >> 3), u32VmcsField & 7);
+}
+
+
+
+/**
+ * Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
+ *
+ * @param   pvIoBitmap      Pointer to the nested-guest IO bitmap.
+ * @param   u16Port         The IO port being accessed.
+ * @param   enmIoType       The type of IO access.
+ * @param   cbReg           The IO operand size in bytes.
+ * @param   cAddrSizeBits   The address size bits (for 16, 32 or 64).
+ * @param   iEffSeg         The effective segment number.
+ * @param   fRep            Whether this is a repeating IO instruction (REP prefix).
+ * @param   fStrIo          Whether this is a string IO instruction.
+ * @param   pIoExitInfo     Pointer to the SVMIOIOEXITINFO struct to be filled.
+ *                          Optional, can be NULL.
+ */
+VMM_INT_DECL(bool) CPUMIsSvmIoInterceptSet(void *pvIoBitmap, uint16_t u16Port, SVMIOIOTYPE enmIoType, uint8_t cbReg,
+                                           uint8_t cAddrSizeBits, uint8_t iEffSeg, bool fRep, bool fStrIo,
+                                           PSVMIOIOEXITINFO pIoExitInfo)
+{
+    Assert(cAddrSizeBits == 16 || cAddrSizeBits == 32 || cAddrSizeBits == 64);
+    Assert(cbReg == 1 || cbReg == 2 || cbReg == 4 || cbReg == 8);
+
+    /*
+     * The IOPM layout:
+     * Each bit represents one 8-bit port. That makes a total of 0..65535 bits or
+     * two 4K pages.
+     *
+     * For IO instructions that access more than a single byte, the permission bits
+     * for all bytes are checked; if any bit is set to 1, the IO access is intercepted.
+     *
+     * Since it's possible to do a 32-bit IO access at port 65534 (accessing 4 bytes),
+     * we need 3 extra bits beyond the second 4K page.
+     */
+    static const uint16_t s_auSizeMasks[] = { 0, 1, 3, 0, 0xf, 0, 0, 0 };
+
+    uint16_t const offIopm   = u16Port >> 3;
+    uint16_t const fSizeMask = s_auSizeMasks[(cAddrSizeBits >> SVM_IOIO_OP_SIZE_SHIFT) & 7];
+    uint8_t  const cShift    = u16Port - (offIopm << 3);
+    uint16_t const fIopmMask = (1 << cShift) | (fSizeMask << cShift);
+
+    uint8_t const *pbIopm = (uint8_t *)pvIoBitmap;
+    Assert(pbIopm);
+    pbIopm += offIopm;
+    uint16_t const u16Iopm = *(uint16_t *)pbIopm;
+    if (u16Iopm & fIopmMask)
+    {
+        if (pIoExitInfo)
+        {
+            static const uint32_t s_auIoOpSize[] =
+            { SVM_IOIO_32_BIT_OP, SVM_IOIO_8_BIT_OP, SVM_IOIO_16_BIT_OP, 0, SVM_IOIO_32_BIT_OP, 0, 0, 0 };
+
+            static const uint32_t s_auIoAddrSize[] =
+            { 0, SVM_IOIO_16_BIT_ADDR, SVM_IOIO_32_BIT_ADDR, 0, SVM_IOIO_64_BIT_ADDR, 0, 0, 0 };
+
+            pIoExitInfo->u         = s_auIoOpSize[cbReg & 7];
+            pIoExitInfo->u        |= s_auIoAddrSize[(cAddrSizeBits >> 4) & 7];
+            pIoExitInfo->n.u1Str   = fStrIo;
+            pIoExitInfo->n.u1Rep   = fRep;
+            pIoExitInfo->n.u3Seg   = iEffSeg & 7;
+            pIoExitInfo->n.u1Type  = enmIoType;
+            pIoExitInfo->n.u16Port = u16Port;
+        }
+        return true;
+    }
+
+    /** @todo remove later (for debugging as VirtualBox always traps all IO
+     *        intercepts). */
+    AssertMsgFailed(("CPUMSvmIsIOInterceptActive: We expect an IO intercept here!\n"));
+    return false;
+}
+
+
+/**
+ * Gets the MSR permission bitmap byte and bit offset for the specified MSR.
+ *
+ * @returns VBox status code.
+ * @param   idMsr       The MSR being requested.
+ * @param   pbOffMsrpm  Where to store the byte offset in the MSR permission
+ *                      bitmap for @a idMsr.
+ * @param   puMsrpmBit  Where to store the bit offset starting at the byte
+ *                      returned in @a pbOffMsrpm.
+ */
+VMM_INT_DECL(int) CPUMGetSvmMsrpmOffsetAndBit(uint32_t idMsr, uint16_t *pbOffMsrpm, uint8_t *puMsrpmBit)
+{
+    Assert(pbOffMsrpm);
+    Assert(puMsrpmBit);
+
+    /*
+     * MSRPM Layout:
+     * Byte offset          MSR range
+     * 0x000  - 0x7ff       0x00000000 - 0x00001fff
+     * 0x800  - 0xfff       0xc0000000 - 0xc0001fff
+     * 0x1000 - 0x17ff      0xc0010000 - 0xc0011fff
+     * 0x1800 - 0x1fff              Reserved
+     *
+     * Each MSR is represented by 2 permission bits (read and write).
+     */
+    if (idMsr <= 0x00001fff)
+    {
+        /* Pentium-compatible MSRs. */
+        uint32_t const bitoffMsr = idMsr << 1;
+        *pbOffMsrpm = bitoffMsr >> 3;
+        *puMsrpmBit = bitoffMsr & 7;
+        return VINF_SUCCESS;
+    }
+
+    if (   idMsr >= 0xc0000000
+        && idMsr <= 0xc0001fff)
+    {
+        /* AMD Sixth Generation x86 Processor MSRs. */
+        uint32_t const bitoffMsr = (idMsr - 0xc0000000) << 1;
+        *pbOffMsrpm = 0x800 + (bitoffMsr >> 3);
+        *puMsrpmBit = bitoffMsr & 7;
+        return VINF_SUCCESS;
+    }
+
+    if (   idMsr >= 0xc0010000
+        && idMsr <= 0xc0011fff)
+    {
+        /* AMD Seventh and Eighth Generation Processor MSRs. */
+        uint32_t const bitoffMsr = (idMsr - 0xc0010000) << 1;
+        *pbOffMsrpm = 0x1000 + (bitoffMsr >> 3);
+        *puMsrpmBit = bitoffMsr & 7;
+        return VINF_SUCCESS;
+    }
+
+    *pbOffMsrpm = 0;
+    *puMsrpmBit = 0;
+    return VERR_OUT_OF_RANGE;
+}
+
diff --git a/src/VBox/VMM/VMMAll/DBGFAll.cpp b/src/VBox/VMM/VMMAll/DBGFAll.cpp
new file mode 100644
index 00000000..a538b21b
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/DBGFAll.cpp
@@ -0,0 +1,410 @@
+/* $Id: DBGFAll.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, All Context Code.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/stdarg.h>
+
+
+/*
+ * Check the read-only VM members.
+ */
+AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.bmSoftIntBreakpoints,  VM, dbgf.ro.bmSoftIntBreakpoints);
+AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.bmHardIntBreakpoints,  VM, dbgf.ro.bmHardIntBreakpoints);
+AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.bmSelectedEvents,      VM, dbgf.ro.bmSelectedEvents);
+AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.cHardIntBreakpoints,   VM, dbgf.ro.cHardIntBreakpoints);
+AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.cSoftIntBreakpoints,   VM, dbgf.ro.cSoftIntBreakpoints);
+AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.cSelectedEvents,       VM, dbgf.ro.cSelectedEvents);
+
+
+/**
+ * Gets the hardware breakpoint configuration as DR7.
+ *
+ * @returns DR7 from the DBGF point of view.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR7(PVM pVM)
+{
+    RTGCUINTREG uDr7 = X86_DR7_GD | X86_DR7_GE | X86_DR7_LE | X86_DR7_RA1_MASK;
+    PDBGFBP     pBp = &pVM->dbgf.s.aHwBreakpoints[0];
+    unsigned    cLeft = RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
+    while (cLeft-- > 0)
+    {
+        if (    pBp->enmType == DBGFBPTYPE_REG
+            &&  pBp->fEnabled)
+        {
+            static const uint8_t s_au8Sizes[8] =
+            {
+                X86_DR7_LEN_BYTE, X86_DR7_LEN_BYTE, X86_DR7_LEN_WORD, X86_DR7_LEN_BYTE,
+                X86_DR7_LEN_DWORD,X86_DR7_LEN_BYTE, X86_DR7_LEN_BYTE, X86_DR7_LEN_QWORD
+            };
+            uDr7 |= X86_DR7_G(pBp->u.Reg.iReg)
+                 |  X86_DR7_RW(pBp->u.Reg.iReg, pBp->u.Reg.fType)
+                 |  X86_DR7_LEN(pBp->u.Reg.iReg, s_au8Sizes[pBp->u.Reg.cb]);
+        }
+        pBp++;
+    }
+    return uDr7;
+}
+
+
+/**
+ * Gets the address of the hardware breakpoint number 0.
+ *
+ * @returns DR0 from the DBGF point of view.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR0(PVM pVM)
+{
+    PCDBGFBP    pBp = &pVM->dbgf.s.aHwBreakpoints[0];
+    Assert(pBp->u.Reg.iReg == 0);
+    return pBp->u.Reg.GCPtr;
+}
+
+
+/**
+ * Gets the address of the hardware breakpoint number 1.
+ *
+ * @returns DR1 from the DBGF point of view.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR1(PVM pVM)
+{
+    PCDBGFBP    pBp = &pVM->dbgf.s.aHwBreakpoints[1];
+    Assert(pBp->u.Reg.iReg == 1);
+    return pBp->u.Reg.GCPtr;
+}
+
+
+/**
+ * Gets the address of the hardware breakpoint number 2.
+ *
+ * @returns DR2 from the DBGF point of view.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR2(PVM pVM)
+{
+    PCDBGFBP    pBp = &pVM->dbgf.s.aHwBreakpoints[2];
+    Assert(pBp->u.Reg.iReg == 2);
+    return pBp->u.Reg.GCPtr;
+}
+
+
+/**
+ * Gets the address of the hardware breakpoint number 3.
+ *
+ * @returns DR3 from the DBGF point of view.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR3(PVM pVM)
+{
+    PCDBGFBP    pBp = &pVM->dbgf.s.aHwBreakpoints[3];
+    Assert(pBp->u.Reg.iReg == 3);
+    return pBp->u.Reg.GCPtr;
+}
+
+
+/**
+ * Checks if any of the hardware breakpoints are armed.
+ *
+ * @returns true if armed, false if not.
+ * @param   pVM        The cross context VM structure.
+ * @remarks Don't call this from CPUMRecalcHyperDRx!
+ */
+VMM_INT_DECL(bool) DBGFBpIsHwArmed(PVM pVM)
+{
+    return pVM->dbgf.s.cEnabledHwBreakpoints > 0;
+}
+
+
+/**
+ * Checks if any of the hardware I/O breakpoints are armed.
+ *
+ * @returns true if armed, false if not.
+ * @param   pVM        The cross context VM structure.
+ * @remarks Don't call this from CPUMRecalcHyperDRx!
+ */
+VMM_INT_DECL(bool) DBGFBpIsHwIoArmed(PVM pVM)
+{
+    return pVM->dbgf.s.cEnabledHwIoBreakpoints > 0;
+}
+
+
+/**
+ * Checks if any INT3 breakpoints are armed.
+ *
+ * @returns true if armed, false if not.
+ * @param   pVM        The cross context VM structure.
+ * @remarks Don't call this from CPUMRecalcHyperDRx!
+ */
+VMM_INT_DECL(bool) DBGFBpIsInt3Armed(PVM pVM)
+{
+    return pVM->dbgf.s.cEnabledInt3Breakpoints > 0;
+}
+
+
+/**
+ * Checks I/O access for guest or hypervisor breakpoints.
+ *
+ * @returns Strict VBox status code
+ * @retval  VINF_SUCCESS no breakpoint.
+ * @retval  VINF_EM_DBG_BREAKPOINT hypervisor breakpoint triggered.
+ * @retval  VINF_EM_RAW_GUEST_TRAP guest breakpoint triggered, DR6 and DR7 have
+ *          been updated appropriately.
+ *
+ * @param   pVM        The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pCtx        The CPU context for the calling EMT.
+ * @param   uIoPort     The I/O port being accessed.
+ * @param   cbValue     The size/width of the access, in bytes.
+ */
+VMM_INT_DECL(VBOXSTRICTRC)  DBGFBpCheckIo(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTIOPORT uIoPort, uint8_t cbValue)
+{
+    uint32_t const uIoPortFirst = uIoPort;
+    uint32_t const uIoPortLast  = uIoPortFirst + cbValue - 1;
+
+
+    /*
+     * Check hyper breakpoints first as the VMM debugger has priority over
+     * the guest.
+     */
+    if (pVM->dbgf.s.cEnabledHwIoBreakpoints > 0)
+    {
+        for (unsigned iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); iBp++)
+        {
+            if (   pVM->dbgf.s.aHwBreakpoints[iBp].u.Reg.fType == X86_DR7_RW_IO
+                && pVM->dbgf.s.aHwBreakpoints[iBp].fEnabled
+                && pVM->dbgf.s.aHwBreakpoints[iBp].enmType     == DBGFBPTYPE_REG )
+            {
+                uint8_t  cbReg      = pVM->dbgf.s.aHwBreakpoints[iBp].u.Reg.cb; Assert(RT_IS_POWER_OF_TWO(cbReg));
+                uint64_t uDrXFirst  = pVM->dbgf.s.aHwBreakpoints[iBp].u.Reg.GCPtr & ~(uint64_t)(cbReg - 1);
+                uint64_t uDrXLast   = uDrXFirst + cbReg - 1;
+                if (uDrXFirst <= uIoPortLast && uDrXLast >= uIoPortFirst)
+                {
+                    /* (See also DBGFRZTrap01Handler.) */
+                    pVCpu->dbgf.s.iActiveBp = pVM->dbgf.s.aHwBreakpoints[iBp].iBp;
+                    pVCpu->dbgf.s.fSingleSteppingRaw = false;
+
+                    LogFlow(("DBGFBpCheckIo: hit hw breakpoint %d at %04x:%RGv (iop %#x)\n",
+                             pVM->dbgf.s.aHwBreakpoints[iBp].iBp, pCtx->cs.Sel, pCtx->rip, uIoPort));
+                    return VINF_EM_DBG_BREAKPOINT;
+                }
+            }
+        }
+    }
+
+    /*
+     * Check the guest.
+     */
+    uint32_t const uDr7 = pCtx->dr[7];
+    if (   (uDr7 & X86_DR7_ENABLED_MASK)
+        && X86_DR7_ANY_RW_IO(uDr7)
+        && (pCtx->cr4 & X86_CR4_DE) )
+    {
+        for (unsigned iBp = 0; iBp < 4; iBp++)
+        {
+            if (   (uDr7 & X86_DR7_L_G(iBp))
+                && X86_DR7_GET_RW(uDr7, iBp) == X86_DR7_RW_IO)
+            {
+                /* ASSUME the breakpoint and the I/O width qualifier uses the same encoding (1 2 x 4). */
+                static uint8_t const s_abInvAlign[4] = { 0, 1, 7, 3 };
+                uint8_t  cbInvAlign = s_abInvAlign[X86_DR7_GET_LEN(uDr7, iBp)];
+                uint64_t uDrXFirst  = pCtx->dr[iBp] & ~(uint64_t)cbInvAlign;
+                uint64_t uDrXLast   = uDrXFirst + cbInvAlign;
+
+                if (uDrXFirst <= uIoPortLast && uDrXLast >= uIoPortFirst)
+                {
+                    /*
+                     * Update DR6 and DR7.
+                     *
+                     * See "AMD64 Architecture Programmer's Manual Volume 2",
+                     * chapter 13.1.1.3 for details on DR6 bits.  The basics is
+                     * that the B0..B3 bits are always cleared while the others
+                     * must be cleared by software.
+                     *
+                     * The following sub chapters says the GD bit is always
+                     * cleared when generating a #DB so the handler can safely
+                     * access the debug registers.
+                     */
+                    pCtx->dr[6] &= ~X86_DR6_B_MASK;
+                    pCtx->dr[6] |= X86_DR6_B(iBp);
+                    pCtx->dr[7] &= ~X86_DR7_GD;
+                    LogFlow(("DBGFBpCheckIo: hit hw breakpoint %d at %04x:%RGv (iop %#x)\n",
+                             pVM->dbgf.s.aHwBreakpoints[iBp].iBp, pCtx->cs.Sel, pCtx->rip, uIoPort));
+                    return VINF_EM_RAW_GUEST_TRAP;
+                }
+            }
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Returns the single stepping state for a virtual CPU.
+ *
+ * @returns stepping (true) or not (false).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) DBGFIsStepping(PVMCPU pVCpu)
+{
+    return pVCpu->dbgf.s.fSingleSteppingRaw;
+}
+
+
+/**
+ * Checks if the specified generic event is enabled or not.
+ *
+ * @returns true / false.
+ * @param   pVM                 The cross context VM structure.
+ * @param   enmEvent            The generic event being raised.
+ * @param   uEventArg           The argument of that event.
+ */
+DECLINLINE(bool) dbgfEventIsGenericWithArgEnabled(PVM pVM, DBGFEVENTTYPE enmEvent, uint64_t uEventArg)
+{
+    if (DBGF_IS_EVENT_ENABLED(pVM, enmEvent))
+    {
+        switch (enmEvent)
+        {
+            case DBGFEVENT_INTERRUPT_HARDWARE:
+                AssertReturn(uEventArg < 256, false);
+                return ASMBitTest(pVM->dbgf.s.bmHardIntBreakpoints, (uint32_t)uEventArg);
+
+            case DBGFEVENT_INTERRUPT_SOFTWARE:
+                AssertReturn(uEventArg < 256, false);
+                return ASMBitTest(pVM->dbgf.s.bmSoftIntBreakpoints, (uint32_t)uEventArg);
+
+            default:
+                return true;
+
+        }
+    }
+    return false;
+}
+
+
+/**
+ * Raises a generic debug event if enabled and not being ignored.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_EM_DBG_EVENT if the event was raised and the caller should
+ *          return ASAP to the debugger (via EM).  We set VMCPU_FF_DBGF so, it
+ *          is okay not to pass this along in some situations.
+ * @retval  VINF_SUCCESS if the event was disabled or ignored.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   enmEvent            The generic event being raised.
+ * @param   enmCtx              The context in which this event is being raised.
+ * @param   cArgs               Number of arguments (0 - 6).
+ * @param   ...                 Event arguments.
+ *
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) DBGFEventGenericWithArgs(PVM pVM, PVMCPU pVCpu, DBGFEVENTTYPE enmEvent, DBGFEVENTCTX enmCtx,
+                                                    unsigned cArgs, ...)
+{
+    Assert(cArgs < RT_ELEMENTS(pVCpu->dbgf.s.aEvents[0].Event.u.Generic.auArgs));
+
+    /*
+     * Is it enabled.
+     */
+    va_list va;
+    va_start(va, cArgs);
+    uint64_t uEventArg0 = cArgs ? va_arg(va, uint64_t) : 0;
+    if (dbgfEventIsGenericWithArgEnabled(pVM, enmEvent, uEventArg0))
+    {
+        /*
+         * Any events on the stack. Should the incoming event be ignored?
+         */
+        uint64_t const rip = CPUMGetGuestRIP(pVCpu);
+        uint32_t i = pVCpu->dbgf.s.cEvents;
+        if (i > 0)
+        {
+            while (i-- > 0)
+            {
+                if (   pVCpu->dbgf.s.aEvents[i].Event.enmType   == enmEvent
+                    && pVCpu->dbgf.s.aEvents[i].enmState        == DBGFEVENTSTATE_IGNORE
+                    && pVCpu->dbgf.s.aEvents[i].rip             == rip)
+                {
+                    pVCpu->dbgf.s.aEvents[i].enmState = DBGFEVENTSTATE_RESTORABLE;
+                    va_end(va);
+                    return VINF_SUCCESS;
+                }
+                Assert(pVCpu->dbgf.s.aEvents[i].enmState != DBGFEVENTSTATE_CURRENT);
+            }
+
+            /*
+             * Trim the event stack.
+             */
+            i = pVCpu->dbgf.s.cEvents;
+            while (i-- > 0)
+            {
+                if (   pVCpu->dbgf.s.aEvents[i].rip == rip
+                    && (   pVCpu->dbgf.s.aEvents[i].enmState == DBGFEVENTSTATE_RESTORABLE
+                        || pVCpu->dbgf.s.aEvents[i].enmState == DBGFEVENTSTATE_IGNORE) )
+                    pVCpu->dbgf.s.aEvents[i].enmState = DBGFEVENTSTATE_IGNORE;
+                else
+                {
+                    if (i + 1 != pVCpu->dbgf.s.cEvents)
+                        memmove(&pVCpu->dbgf.s.aEvents[i], &pVCpu->dbgf.s.aEvents[i + 1],
+                                (pVCpu->dbgf.s.cEvents - i) * sizeof(pVCpu->dbgf.s.aEvents));
+                    pVCpu->dbgf.s.cEvents--;
+                }
+            }
+
+            i = pVCpu->dbgf.s.cEvents;
+            AssertStmt(i < RT_ELEMENTS(pVCpu->dbgf.s.aEvents), i = RT_ELEMENTS(pVCpu->dbgf.s.aEvents) - 1);
+        }
+
+        /*
+         * Push the event.
+         */
+        pVCpu->dbgf.s.aEvents[i].enmState               = DBGFEVENTSTATE_CURRENT;
+        pVCpu->dbgf.s.aEvents[i].rip                    = rip;
+        pVCpu->dbgf.s.aEvents[i].Event.enmType          = enmEvent;
+        pVCpu->dbgf.s.aEvents[i].Event.enmCtx           = enmCtx;
+        pVCpu->dbgf.s.aEvents[i].Event.u.Generic.cArgs  = cArgs;
+        pVCpu->dbgf.s.aEvents[i].Event.u.Generic.auArgs[0] = uEventArg0;
+        if (cArgs > 1)
+        {
+            AssertStmt(cArgs < RT_ELEMENTS(pVCpu->dbgf.s.aEvents[i].Event.u.Generic.auArgs),
+                       cArgs = RT_ELEMENTS(pVCpu->dbgf.s.aEvents[i].Event.u.Generic.auArgs));
+            for (unsigned iArg = 1; iArg < cArgs; iArg++)
+                pVCpu->dbgf.s.aEvents[i].Event.u.Generic.auArgs[iArg] = va_arg(va, uint64_t);
+        }
+        pVCpu->dbgf.s.cEvents = i + 1;
+
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_DBGF);
+        va_end(va);
+        return VINF_EM_DBG_EVENT;
+    }
+
+    va_end(va);
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMAll/EMAll.cpp b/src/VBox/VMM/VMMAll/EMAll.cpp
new file mode 100644
index 00000000..8f63f928
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/EMAll.cpp
@@ -0,0 +1,1237 @@
+/* $Id: EMAll.cpp $ */
+/** @file
+ * EM - Execution Monitor(/Manager) - All contexts
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_EM
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/stam.h>
+#include "EMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+
+
+/**
+ * Get the current execution manager status.
+ *
+ * @returns Current status.
+ * @param   pVCpu         The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(EMSTATE) EMGetState(PVMCPU pVCpu)
+{
+    return pVCpu->em.s.enmState;
+}
+
+
+/**
+ * Sets the current execution manager status. (use only when you know what you're doing!)
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   enmNewState The new state, EMSTATE_WAIT_SIPI or EMSTATE_HALTED.
+ */
+VMM_INT_DECL(void)    EMSetState(PVMCPU pVCpu, EMSTATE enmNewState)
+{
+    /* Only allowed combination: */
+    Assert(pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI && enmNewState == EMSTATE_HALTED);
+    pVCpu->em.s.enmState = enmNewState;
+}
+
+
+/**
+ * Sets the PC for which interrupts should be inhibited.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   PC          The PC.
+ */
+VMMDECL(void) EMSetInhibitInterruptsPC(PVMCPU pVCpu, RTGCUINTPTR PC)
+{
+    pVCpu->em.s.GCPtrInhibitInterrupts = PC;
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+}
+
+
+/**
+ * Gets the PC for which interrupts should be inhibited.
+ *
+ * There are a few instructions which inhibits or delays interrupts
+ * for the instruction following them. These instructions are:
+ *      - STI
+ *      - MOV SS, r/m16
+ *      - POP SS
+ *
+ * @returns The PC for which interrupts should be inhibited.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ */
+VMMDECL(RTGCUINTPTR) EMGetInhibitInterruptsPC(PVMCPU pVCpu)
+{
+    return pVCpu->em.s.GCPtrInhibitInterrupts;
+}
+
+
+/**
+ * Checks if interrupt inhibiting is enabled for the current instruction.
+ *
+ * @returns true if interrupts are inhibited, false if not.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) EMIsInhibitInterruptsActive(PVMCPU pVCpu)
+{
+    if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+        return false;
+    if (pVCpu->em.s.GCPtrInhibitInterrupts == CPUMGetGuestRIP(pVCpu))
+        return true;
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+    return false;
+}
+
+
+/**
+ * Enables / disable hypercall instructions.
+ *
+ * This interface is used by GIM to tell the execution monitors whether the
+ * hypercall instruction (VMMCALL & VMCALL) are allowed or should \#UD.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure this applies to.
+ * @param   fEnabled    Whether hypercall instructions are enabled (true) or not.
+ */
+VMMDECL(void) EMSetHypercallInstructionsEnabled(PVMCPU pVCpu, bool fEnabled)
+{
+    pVCpu->em.s.fHypercallEnabled = fEnabled;
+}
+
+
+/**
+ * Checks if hypercall instructions (VMMCALL & VMCALL) are enabled or not.
+ *
+ * @returns true if enabled, false if not.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @note    If this call becomes a performance factor, we can make the data
+ *          field available thru a read-only view in VMCPU.  See VM::cpum.ro.
+ */
+VMMDECL(bool) EMAreHypercallInstructionsEnabled(PVMCPU pVCpu)
+{
+    return pVCpu->em.s.fHypercallEnabled;
+}
+
+
+/**
+ * Prepare an MWAIT - essentials of the MONITOR instruction.
+ *
+ * @returns VINF_SUCCESS
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   rax                 The content of RAX.
+ * @param   rcx                 The content of RCX.
+ * @param   rdx                 The content of RDX.
+ * @param   GCPhys              The physical address corresponding to rax.
+ */
+VMM_INT_DECL(int) EMMonitorWaitPrepare(PVMCPU pVCpu, uint64_t rax, uint64_t rcx, uint64_t rdx, RTGCPHYS GCPhys)
+{
+    pVCpu->em.s.MWait.uMonitorRAX = rax;
+    pVCpu->em.s.MWait.uMonitorRCX = rcx;
+    pVCpu->em.s.MWait.uMonitorRDX = rdx;
+    pVCpu->em.s.MWait.fWait |= EMMWAIT_FLAG_MONITOR_ACTIVE;
+    /** @todo Make use of GCPhys. */
+    NOREF(GCPhys);
+    /** @todo Complete MONITOR implementation.  */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the monitor hardware is armed / active.
+ *
+ * @returns true if armed, false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ */
+VMM_INT_DECL(bool) EMMonitorIsArmed(PVMCPU pVCpu)
+{
+    return RT_BOOL(pVCpu->em.s.MWait.fWait & EMMWAIT_FLAG_MONITOR_ACTIVE);
+}
+
+
+/**
+ * Checks if we're in a MWAIT.
+ *
+ * @retval  1 if regular,
+ * @retval  > 1 if MWAIT with EMMWAIT_FLAG_BREAKIRQIF0
+ * @retval  0 if not armed
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ */
+VMM_INT_DECL(unsigned) EMMonitorWaitIsActive(PVMCPU pVCpu)
+{
+    uint32_t fWait = pVCpu->em.s.MWait.fWait;
+    AssertCompile(EMMWAIT_FLAG_ACTIVE == 1);
+    AssertCompile(EMMWAIT_FLAG_BREAKIRQIF0 == 2);
+    AssertCompile((EMMWAIT_FLAG_ACTIVE << 1) == EMMWAIT_FLAG_BREAKIRQIF0);
+    return fWait & (EMMWAIT_FLAG_ACTIVE | ((fWait & EMMWAIT_FLAG_ACTIVE) << 1));
+}
+
+
+/**
+ * Performs an MWAIT.
+ *
+ * @returns VINF_SUCCESS
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   rax                 The content of RAX.
+ * @param   rcx                 The content of RCX.
+ */
+VMM_INT_DECL(int) EMMonitorWaitPerform(PVMCPU pVCpu, uint64_t rax, uint64_t rcx)
+{
+    pVCpu->em.s.MWait.uMWaitRAX = rax;
+    pVCpu->em.s.MWait.uMWaitRCX = rcx;
+    pVCpu->em.s.MWait.fWait |= EMMWAIT_FLAG_ACTIVE;
+    if (rcx)
+        pVCpu->em.s.MWait.fWait |= EMMWAIT_FLAG_BREAKIRQIF0;
+    else
+        pVCpu->em.s.MWait.fWait &= ~EMMWAIT_FLAG_BREAKIRQIF0;
+    /** @todo not completely correct?? */
+    return VINF_EM_HALT;
+}
+
+
+/**
+ * Clears any address-range monitoring that is active.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ */
+VMM_INT_DECL(void) EMMonitorWaitClear(PVMCPU pVCpu)
+{
+    LogFlowFunc(("Clearing MWAIT\n"));
+    pVCpu->em.s.MWait.fWait &= ~(EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0);
+}
+
+
+/**
+ * Determine if we should continue execution in HM after encountering an mwait
+ * instruction.
+ *
+ * Clears MWAIT flags if returning @c true.
+ *
+ * @returns true if we should continue, false if we should halt.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pCtx            Current CPU context.
+ */
+VMM_INT_DECL(bool) EMMonitorWaitShouldContinue(PVMCPU pVCpu, PCPUMCTX pCtx)
+{
+    if (CPUMGetGuestGif(pCtx))
+    {
+        if (   CPUMIsGuestPhysIntrEnabled(pVCpu)
+            || (   CPUMIsGuestInNestedHwvirtMode(pCtx)
+                && CPUMIsGuestVirtIntrEnabled(pVCpu))
+            || (   (pVCpu->em.s.MWait.fWait & (EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0))
+                ==                            (EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0)) )
+        {
+            if (VMCPU_FF_IS_ANY_SET(pVCpu, (  VMCPU_FF_UPDATE_APIC | VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC
+                                            | VMCPU_FF_INTERRUPT_NESTED_GUEST)))
+            {
+                pVCpu->em.s.MWait.fWait &= ~(EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0);
+                return true;
+            }
+        }
+    }
+
+    return false;
+}
+
+
+/**
+ * Determine if we should continue execution in HM after encountering a hlt
+ * instruction.
+ *
+ * @returns true if we should continue, false if we should halt.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pCtx            Current CPU context.
+ */
+VMM_INT_DECL(bool) EMShouldContinueAfterHalt(PVMCPU pVCpu, PCPUMCTX pCtx)
+{
+    if (CPUMGetGuestGif(pCtx))
+    {
+        if (CPUMIsGuestPhysIntrEnabled(pVCpu))
+            return VMCPU_FF_IS_ANY_SET(pVCpu, (VMCPU_FF_UPDATE_APIC | VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC));
+
+        if (   CPUMIsGuestInNestedHwvirtMode(pCtx)
+            && CPUMIsGuestVirtIntrEnabled(pVCpu))
+            return VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
+    }
+    return false;
+}
+
+
+/**
+ * Unhalts and wakes up the given CPU.
+ *
+ * This is an API for assisting the KVM hypercall API in implementing KICK_CPU.
+ * It sets VMCPU_FF_UNHALT for @a pVCpuDst and makes sure it is woken up.   If
+ * the CPU isn't currently in a halt, the next HLT instruction it executes will
+ * be affected.
+ *
+ * @returns GVMMR0SchedWakeUpEx result or VINF_SUCCESS depending on context.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpuDst        The cross context virtual CPU structure of the
+ *                          CPU to unhalt and wake up.  This is usually not the
+ *                          same as the caller.
+ * @thread  EMT
+ */
+VMM_INT_DECL(int) EMUnhaltAndWakeUp(PVMCC pVM, PVMCPUCC pVCpuDst)
+{
+    /*
+     * Flag the current(/next) HLT to unhalt immediately.
+     */
+    VMCPU_FF_SET(pVCpuDst, VMCPU_FF_UNHALT);
+
+    /*
+     * Wake up the EMT (technically should be abstracted by VMM/VMEmt, but
+     * just do it here for now).
+     */
+#ifdef IN_RING0
+    /* We might be here with preemption disabled or enabled (i.e. depending on
+       thread-context hooks being used), so don't try obtaining the GVMMR0 used
+       lock here. See @bugref{7270#c148}. */
+    int rc = GVMMR0SchedWakeUpNoGVMNoLock(pVM, pVCpuDst->idCpu);
+    AssertRC(rc);
+
+#elif defined(IN_RING3)
+    int rc = SUPR3CallVMMR0(VMCC_GET_VMR0_FOR_CALL(pVM), pVCpuDst->idCpu, VMMR0_DO_GVMM_SCHED_WAKE_UP, NULL /* pvArg */);
+    AssertRC(rc);
+
+#else
+    /* Nothing to do for raw-mode, shouldn't really be used by raw-mode guests anyway. */
+    Assert(pVM->cCpus == 1); NOREF(pVM);
+    int rc = VINF_SUCCESS;
+#endif
+    return rc;
+}
+
+#ifndef IN_RING3
+
+/**
+ * Makes an I/O port write pending for ring-3 processing.
+ *
+ * @returns VINF_EM_PENDING_R3_IOPORT_READ
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uPort           The I/O port.
+ * @param   cbInstr         The instruction length (for RIP updating).
+ * @param   cbValue         The write size.
+ * @param   uValue          The value being written.
+ * @sa      emR3ExecutePendingIoPortWrite
+ *
+ * @note    Must not be used when I/O port breakpoints are pending or when single stepping.
+ */
+VMMRZ_INT_DECL(VBOXSTRICTRC)
+EMRZSetPendingIoPortWrite(PVMCPU pVCpu, RTIOPORT uPort, uint8_t cbInstr, uint8_t cbValue, uint32_t uValue)
+{
+    Assert(pVCpu->em.s.PendingIoPortAccess.cbValue == 0);
+    pVCpu->em.s.PendingIoPortAccess.uPort     = uPort;
+    pVCpu->em.s.PendingIoPortAccess.cbValue   = cbValue;
+    pVCpu->em.s.PendingIoPortAccess.cbInstr   = cbInstr;
+    pVCpu->em.s.PendingIoPortAccess.uValue    = uValue;
+    return VINF_EM_PENDING_R3_IOPORT_WRITE;
+}
+
+
+/**
+ * Makes an I/O port read pending for ring-3 processing.
+ *
+ * @returns VINF_EM_PENDING_R3_IOPORT_READ
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uPort           The I/O port.
+ * @param   cbInstr         The instruction length (for RIP updating).
+ * @param   cbValue         The read size.
+ * @sa      emR3ExecutePendingIoPortRead
+ *
+ * @note    Must not be used when I/O port breakpoints are pending or when single stepping.
+ */
+VMMRZ_INT_DECL(VBOXSTRICTRC)
+EMRZSetPendingIoPortRead(PVMCPU pVCpu, RTIOPORT uPort, uint8_t cbInstr, uint8_t cbValue)
+{
+    Assert(pVCpu->em.s.PendingIoPortAccess.cbValue == 0);
+    pVCpu->em.s.PendingIoPortAccess.uPort     = uPort;
+    pVCpu->em.s.PendingIoPortAccess.cbValue   = cbValue;
+    pVCpu->em.s.PendingIoPortAccess.cbInstr   = cbInstr;
+    pVCpu->em.s.PendingIoPortAccess.uValue    = UINT32_C(0x52454144); /* 'READ' */
+    return VINF_EM_PENDING_R3_IOPORT_READ;
+}
+
+#endif /* IN_RING3 */
+
+
+/**
+ * Worker for EMHistoryExec that checks for ring-3 returns and flags
+ * continuation of the EMHistoryExec run there.
+ */
+DECL_FORCE_INLINE(void) emHistoryExecSetContinueExitRecIdx(PVMCPU pVCpu, VBOXSTRICTRC rcStrict, PCEMEXITREC pExitRec)
+{
+    pVCpu->em.s.idxContinueExitRec = UINT16_MAX;
+#ifdef IN_RING3
+    RT_NOREF_PV(rcStrict); RT_NOREF_PV(pExitRec);
+#else
+    switch (VBOXSTRICTRC_VAL(rcStrict))
+    {
+        case VINF_SUCCESS:
+        default:
+            break;
+
+        /*
+         * Only status codes that EMHandleRCTmpl.h will resume EMHistoryExec with.
+         */
+        case VINF_IOM_R3_IOPORT_READ:           /* -> emR3ExecuteIOInstruction */
+        case VINF_IOM_R3_IOPORT_WRITE:          /* -> emR3ExecuteIOInstruction */
+        case VINF_IOM_R3_IOPORT_COMMIT_WRITE:   /* -> VMCPU_FF_IOM -> VINF_EM_RESUME_R3_HISTORY_EXEC -> emR3ExecuteIOInstruction */
+        case VINF_IOM_R3_MMIO_READ:             /* -> emR3ExecuteInstruction */
+        case VINF_IOM_R3_MMIO_WRITE:            /* -> emR3ExecuteInstruction */
+        case VINF_IOM_R3_MMIO_READ_WRITE:       /* -> emR3ExecuteInstruction */
+        case VINF_IOM_R3_MMIO_COMMIT_WRITE:     /* -> VMCPU_FF_IOM -> VINF_EM_RESUME_R3_HISTORY_EXEC -> emR3ExecuteIOInstruction */
+        case VINF_CPUM_R3_MSR_READ:             /* -> emR3ExecuteInstruction */
+        case VINF_CPUM_R3_MSR_WRITE:            /* -> emR3ExecuteInstruction */
+        case VINF_GIM_R3_HYPERCALL:             /* -> emR3ExecuteInstruction */
+            pVCpu->em.s.idxContinueExitRec = (uint16_t)(pExitRec - &pVCpu->em.s.aExitRecords[0]);
+            break;
+    }
+#endif /* !IN_RING3 */
+}
+
+
+/**
+ * Execute using history.
+ *
+ * This function will be called when EMHistoryAddExit() and friends returns a
+ * non-NULL result.  This happens in response to probing or when probing has
+ * uncovered adjacent exits which can more effectively be reached by using IEM
+ * than restarting execution using the main execution engine and fielding an
+ * regular exit.
+ *
+ * @returns VBox strict status code, see IEMExecForExits.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitRec        The exit record return by a previous history add
+ *                          or update call.
+ * @param   fWillExit       Flags indicating to IEM what will cause exits, TBD.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) EMHistoryExec(PVMCPUCC pVCpu, PCEMEXITREC pExitRec, uint32_t fWillExit)
+{
+    Assert(pExitRec);
+    VMCPU_ASSERT_EMT(pVCpu);
+    IEMEXECFOREXITSTATS ExecStats;
+    switch (pExitRec->enmAction)
+    {
+        /*
+         * Executes multiple instruction stopping only when we've gone a given
+         * number without perceived exits.
+         */
+        case EMEXITACTION_EXEC_WITH_MAX:
+        {
+            STAM_REL_PROFILE_START(&pVCpu->em.s.StatHistoryExec, a);
+            LogFlow(("EMHistoryExec/EXEC_WITH_MAX: %RX64, max %u\n", pExitRec->uFlatPC, pExitRec->cMaxInstructionsWithoutExit));
+            VBOXSTRICTRC rcStrict = IEMExecForExits(pVCpu, fWillExit,
+                                                    pExitRec->cMaxInstructionsWithoutExit /* cMinInstructions*/,
+                                                    pVCpu->em.s.cHistoryExecMaxInstructions,
+                                                    pExitRec->cMaxInstructionsWithoutExit,
+                                                    &ExecStats);
+            LogFlow(("EMHistoryExec/EXEC_WITH_MAX: %Rrc cExits=%u cMaxExitDistance=%u cInstructions=%u\n",
+                     VBOXSTRICTRC_VAL(rcStrict), ExecStats.cExits, ExecStats.cMaxExitDistance, ExecStats.cInstructions));
+            emHistoryExecSetContinueExitRecIdx(pVCpu, rcStrict, pExitRec);
+
+            /* Ignore instructions IEM doesn't know about. */
+            if (   (   rcStrict != VERR_IEM_INSTR_NOT_IMPLEMENTED
+                    && rcStrict != VERR_IEM_ASPECT_NOT_IMPLEMENTED)
+                || ExecStats.cInstructions == 0)
+            { /* likely */ }
+            else
+                rcStrict = VINF_SUCCESS;
+
+            if (ExecStats.cExits > 1)
+                STAM_REL_COUNTER_ADD(&pVCpu->em.s.StatHistoryExecSavedExits, ExecStats.cExits - 1);
+            STAM_REL_COUNTER_ADD(&pVCpu->em.s.StatHistoryExecInstructions, ExecStats.cInstructions);
+            STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatHistoryExec, a);
+            return rcStrict;
+        }
+
+        /*
+         * Probe a exit for close by exits.
+         */
+        case EMEXITACTION_EXEC_PROBE:
+        {
+            STAM_REL_PROFILE_START(&pVCpu->em.s.StatHistoryProbe, b);
+            LogFlow(("EMHistoryExec/EXEC_PROBE: %RX64\n", pExitRec->uFlatPC));
+            PEMEXITREC   pExitRecUnconst = (PEMEXITREC)pExitRec;
+            VBOXSTRICTRC rcStrict = IEMExecForExits(pVCpu, fWillExit,
+                                                    pVCpu->em.s.cHistoryProbeMinInstructions,
+                                                    pVCpu->em.s.cHistoryExecMaxInstructions,
+                                                    pVCpu->em.s.cHistoryProbeMaxInstructionsWithoutExit,
+                                                    &ExecStats);
+            LogFlow(("EMHistoryExec/EXEC_PROBE: %Rrc cExits=%u cMaxExitDistance=%u cInstructions=%u\n",
+                     VBOXSTRICTRC_VAL(rcStrict), ExecStats.cExits, ExecStats.cMaxExitDistance, ExecStats.cInstructions));
+            emHistoryExecSetContinueExitRecIdx(pVCpu, rcStrict, pExitRecUnconst);
+            if (   ExecStats.cExits >= 2
+                && RT_SUCCESS(rcStrict))
+            {
+                Assert(ExecStats.cMaxExitDistance > 0 && ExecStats.cMaxExitDistance <= 32);
+                pExitRecUnconst->cMaxInstructionsWithoutExit = ExecStats.cMaxExitDistance;
+                pExitRecUnconst->enmAction = EMEXITACTION_EXEC_WITH_MAX;
+                LogFlow(("EMHistoryExec/EXEC_PROBE: -> EXEC_WITH_MAX %u\n", ExecStats.cMaxExitDistance));
+                STAM_REL_COUNTER_INC(&pVCpu->em.s.StatHistoryProbedExecWithMax);
+            }
+#ifndef IN_RING3
+            else if (   pVCpu->em.s.idxContinueExitRec != UINT16_MAX
+                     && RT_SUCCESS(rcStrict))
+            {
+                STAM_REL_COUNTER_INC(&pVCpu->em.s.StatHistoryProbedToRing3);
+                LogFlow(("EMHistoryExec/EXEC_PROBE: -> ring-3\n"));
+            }
+#endif
+            else
+            {
+                pExitRecUnconst->enmAction = EMEXITACTION_NORMAL_PROBED;
+                pVCpu->em.s.idxContinueExitRec = UINT16_MAX;
+                LogFlow(("EMHistoryExec/EXEC_PROBE: -> PROBED\n"));
+                STAM_REL_COUNTER_INC(&pVCpu->em.s.StatHistoryProbedNormal);
+                if (   rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED
+                    || rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
+                    rcStrict = VINF_SUCCESS;
+            }
+            STAM_REL_COUNTER_ADD(&pVCpu->em.s.StatHistoryProbeInstructions, ExecStats.cInstructions);
+            STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatHistoryProbe, b);
+            return rcStrict;
+        }
+
+        /* We shouldn't ever see these here! */
+        case EMEXITACTION_FREE_RECORD:
+        case EMEXITACTION_NORMAL:
+        case EMEXITACTION_NORMAL_PROBED:
+            break;
+
+        /* No default case, want compiler warnings. */
+    }
+    AssertLogRelFailedReturn(VERR_EM_INTERNAL_ERROR);
+}
+
+
+/**
+ * Worker for emHistoryAddOrUpdateRecord.
+ */
+DECL_FORCE_INLINE(PCEMEXITREC) emHistoryRecordInit(PEMEXITREC pExitRec, uint64_t uFlatPC, uint32_t uFlagsAndType, uint64_t uExitNo)
+{
+    pExitRec->uFlatPC                     = uFlatPC;
+    pExitRec->uFlagsAndType               = uFlagsAndType;
+    pExitRec->enmAction                   = EMEXITACTION_NORMAL;
+    pExitRec->bUnused                     = 0;
+    pExitRec->cMaxInstructionsWithoutExit = 64;
+    pExitRec->uLastExitNo                 = uExitNo;
+    pExitRec->cHits                       = 1;
+    return NULL;
+}
+
+
+/**
+ * Worker for emHistoryAddOrUpdateRecord.
+ */
+DECL_FORCE_INLINE(PCEMEXITREC) emHistoryRecordInitNew(PVMCPU pVCpu, PEMEXITENTRY pHistEntry, uintptr_t idxSlot,
+                                                      PEMEXITREC pExitRec, uint64_t uFlatPC,
+                                                      uint32_t uFlagsAndType, uint64_t uExitNo)
+{
+    pHistEntry->idxSlot = (uint32_t)idxSlot;
+    pVCpu->em.s.cExitRecordUsed++;
+    LogFlow(("emHistoryRecordInitNew: [%#x] = %#07x %016RX64; (%u of %u used)\n", idxSlot, uFlagsAndType, uFlatPC,
+             pVCpu->em.s.cExitRecordUsed, RT_ELEMENTS(pVCpu->em.s.aExitRecords) ));
+    return emHistoryRecordInit(pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+}
+
+
+/**
+ * Worker for emHistoryAddOrUpdateRecord.
+ */
+DECL_FORCE_INLINE(PCEMEXITREC) emHistoryRecordInitReplacement(PEMEXITENTRY pHistEntry, uintptr_t idxSlot,
+                                                              PEMEXITREC pExitRec, uint64_t uFlatPC,
+                                                              uint32_t uFlagsAndType, uint64_t uExitNo)
+{
+    pHistEntry->idxSlot = (uint32_t)idxSlot;
+    LogFlow(("emHistoryRecordInitReplacement: [%#x] = %#07x %016RX64 replacing %#07x %016RX64 with %u hits, %u exits old\n",
+             idxSlot, uFlagsAndType, uFlatPC, pExitRec->uFlagsAndType, pExitRec->uFlatPC, pExitRec->cHits,
+             uExitNo - pExitRec->uLastExitNo));
+    return emHistoryRecordInit(pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+}
+
+
+/**
+ * Adds or updates the EMEXITREC for this PC/type and decide on an action.
+ *
+ * @returns Pointer to an exit record if special action should be taken using
+ *          EMHistoryExec().  Take normal exit action when NULL.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uFlagsAndType   Combined flags and type, EMEXIT_F_KIND_EM set and
+ *                          both EMEXIT_F_CS_EIP and EMEXIT_F_UNFLATTENED_PC are clear.
+ * @param   uFlatPC         The flattened program counter.
+ * @param   pHistEntry      The exit history entry.
+ * @param   uExitNo         The current exit number.
+ */
+static PCEMEXITREC emHistoryAddOrUpdateRecord(PVMCPU pVCpu, uint64_t uFlagsAndType, uint64_t uFlatPC,
+                                              PEMEXITENTRY pHistEntry, uint64_t uExitNo)
+{
+# ifdef IN_RING0
+    /* Disregard the hm flag. */
+    uFlagsAndType &= ~EMEXIT_F_HM;
+# endif
+
+    /*
+     * Work the hash table.
+     */
+    AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitRecords) == 1024);
+# define EM_EXIT_RECORDS_IDX_MASK 0x3ff
+    uintptr_t  idxSlot  = ((uintptr_t)uFlatPC >> 1) & EM_EXIT_RECORDS_IDX_MASK;
+    PEMEXITREC pExitRec = &pVCpu->em.s.aExitRecords[idxSlot];
+    if (pExitRec->uFlatPC == uFlatPC)
+    {
+        Assert(pExitRec->enmAction != EMEXITACTION_FREE_RECORD);
+        pHistEntry->idxSlot = (uint32_t)idxSlot;
+        if (pExitRec->uFlagsAndType == uFlagsAndType)
+        {
+            pExitRec->uLastExitNo = uExitNo;
+            STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecHits[0]);
+        }
+        else
+        {
+            STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecTypeChanged[0]);
+            return emHistoryRecordInit(pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+        }
+    }
+    else if (pExitRec->enmAction == EMEXITACTION_FREE_RECORD)
+    {
+        STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecNew[0]);
+        return emHistoryRecordInitNew(pVCpu, pHistEntry, idxSlot, pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+    }
+    else
+    {
+        /*
+         * Collision.  We calculate a new hash for stepping away from the first,
+         * doing up to 8 steps away before replacing the least recently used record.
+         */
+        uintptr_t idxOldest     = idxSlot;
+        uint64_t  uOldestExitNo = pExitRec->uLastExitNo;
+        unsigned  iOldestStep   = 0;
+        unsigned  iStep         = 1;
+        uintptr_t const idxAdd  = (uintptr_t)(uFlatPC >> 11) & (EM_EXIT_RECORDS_IDX_MASK / 4);
+        for (;;)
+        {
+            Assert(iStep < RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits));
+            AssertCompile(RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecNew)         == RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits));
+            AssertCompile(RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecReplaced)    == RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits));
+            AssertCompile(RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecTypeChanged) == RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits));
+
+            /* Step to the next slot. */
+            idxSlot += idxAdd;
+            idxSlot &= EM_EXIT_RECORDS_IDX_MASK;
+            pExitRec = &pVCpu->em.s.aExitRecords[idxSlot];
+
+            /* Does it match? */
+            if (pExitRec->uFlatPC == uFlatPC)
+            {
+                Assert(pExitRec->enmAction != EMEXITACTION_FREE_RECORD);
+                pHistEntry->idxSlot = (uint32_t)idxSlot;
+                if (pExitRec->uFlagsAndType == uFlagsAndType)
+                {
+                    pExitRec->uLastExitNo = uExitNo;
+                    STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecHits[iStep]);
+                    break;
+                }
+                STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecTypeChanged[iStep]);
+                return emHistoryRecordInit(pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+            }
+
+            /* Is it free? */
+            if (pExitRec->enmAction == EMEXITACTION_FREE_RECORD)
+            {
+                STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecNew[iStep]);
+                return emHistoryRecordInitNew(pVCpu, pHistEntry, idxSlot, pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+            }
+
+            /* Is it the least recently used one? */
+            if (pExitRec->uLastExitNo < uOldestExitNo)
+            {
+                uOldestExitNo = pExitRec->uLastExitNo;
+                idxOldest     = idxSlot;
+                iOldestStep   = iStep;
+            }
+
+            /* Next iteration? */
+            iStep++;
+            Assert(iStep < RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecReplaced));
+            if (RT_LIKELY(iStep < 8 + 1))
+            { /* likely */ }
+            else
+            {
+                /* Replace the least recently used slot. */
+                STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecReplaced[iOldestStep]);
+                pExitRec = &pVCpu->em.s.aExitRecords[idxOldest];
+                return emHistoryRecordInitReplacement(pHistEntry, idxOldest, pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+            }
+        }
+    }
+
+    /*
+     * Found an existing record.
+     */
+    switch (pExitRec->enmAction)
+    {
+        case EMEXITACTION_NORMAL:
+        {
+            uint64_t const cHits = ++pExitRec->cHits;
+            if (cHits < 256)
+                return NULL;
+            LogFlow(("emHistoryAddOrUpdateRecord: [%#x] %#07x %16RX64: -> EXEC_PROBE\n", idxSlot, uFlagsAndType, uFlatPC));
+            pExitRec->enmAction = EMEXITACTION_EXEC_PROBE;
+            return pExitRec;
+        }
+
+        case EMEXITACTION_NORMAL_PROBED:
+            pExitRec->cHits += 1;
+            return NULL;
+
+        default:
+            pExitRec->cHits += 1;
+            return pExitRec;
+
+        /* This will happen if the caller ignores or cannot serve the probe
+           request (forced to ring-3, whatever).  We retry this 256 times. */
+        case EMEXITACTION_EXEC_PROBE:
+        {
+            uint64_t const cHits = ++pExitRec->cHits;
+            if (cHits < 512)
+                return pExitRec;
+            pExitRec->enmAction = EMEXITACTION_NORMAL_PROBED;
+            LogFlow(("emHistoryAddOrUpdateRecord: [%#x] %#07x %16RX64: -> PROBED\n", idxSlot, uFlagsAndType, uFlatPC));
+            return NULL;
+        }
+    }
+}
+
+
+/**
+ * Adds an exit to the history for this CPU.
+ *
+ * @returns Pointer to an exit record if special action should be taken using
+ *          EMHistoryExec().  Take normal exit action when NULL.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uFlagsAndType   Combined flags and type (see EMEXIT_MAKE_FLAGS_AND_TYPE).
+ * @param   uFlatPC         The flattened program counter (RIP).  UINT64_MAX if not available.
+ * @param   uTimestamp      The TSC value for the exit, 0 if not available.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(PCEMEXITREC) EMHistoryAddExit(PVMCPUCC pVCpu, uint32_t uFlagsAndType, uint64_t uFlatPC, uint64_t uTimestamp)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Add the exit history entry.
+     */
+    AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitHistory) == 256);
+    uint64_t uExitNo = pVCpu->em.s.iNextExit++;
+    PEMEXITENTRY pHistEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)uExitNo & 0xff];
+    pHistEntry->uFlatPC       = uFlatPC;
+    pHistEntry->uTimestamp    = uTimestamp;
+    pHistEntry->uFlagsAndType = uFlagsAndType;
+    pHistEntry->idxSlot       = UINT32_MAX;
+
+    /*
+     * If common exit type, we will insert/update the exit into the exit record hash table.
+     */
+    if (   (uFlagsAndType & (EMEXIT_F_KIND_MASK | EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC)) == EMEXIT_F_KIND_EM
+#ifdef IN_RING0
+        && pVCpu->em.s.fExitOptimizationEnabledR0
+        && ( !(uFlagsAndType & EMEXIT_F_HM) || pVCpu->em.s.fExitOptimizationEnabledR0PreemptDisabled)
+#else
+        && pVCpu->em.s.fExitOptimizationEnabled
+#endif
+        && uFlatPC != UINT64_MAX
+       )
+        return emHistoryAddOrUpdateRecord(pVCpu, uFlagsAndType, uFlatPC, pHistEntry, uExitNo);
+    return NULL;
+}
+
+
+#ifdef IN_RING0
+/**
+ * Interface that VT-x uses to supply the PC of an exit when CS:RIP is being read.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uFlatPC         The flattened program counter (RIP).
+ * @param   fFlattened      Set if RIP was subjected to CS.BASE, clear if not.
+ */
+VMMR0_INT_DECL(void) EMR0HistoryUpdatePC(PVMCPU pVCpu, uint64_t uFlatPC, bool fFlattened)
+{
+    AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitHistory) == 256);
+    uint64_t     uExitNo    = pVCpu->em.s.iNextExit - 1;
+    PEMEXITENTRY pHistEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)uExitNo & 0xff];
+    pHistEntry->uFlatPC = uFlatPC;
+    if (fFlattened)
+        pHistEntry->uFlagsAndType &= ~EMEXIT_F_UNFLATTENED_PC;
+    else
+        pHistEntry->uFlagsAndType |= EMEXIT_F_UNFLATTENED_PC;
+}
+#endif
+
+
+/**
+ * Interface for convering a engine specific exit to a generic one and get guidance.
+ *
+ * @returns Pointer to an exit record if special action should be taken using
+ *          EMHistoryExec().  Take normal exit action when NULL.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uFlagsAndType   Combined flags and type (see EMEXIT_MAKE_FLAGS_AND_TYPE).
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(PCEMEXITREC) EMHistoryUpdateFlagsAndType(PVMCPUCC pVCpu, uint32_t uFlagsAndType)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Do the updating.
+     */
+    AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitHistory) == 256);
+    uint64_t     uExitNo    = pVCpu->em.s.iNextExit - 1;
+    PEMEXITENTRY pHistEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)uExitNo & 0xff];
+    pHistEntry->uFlagsAndType = uFlagsAndType | (pHistEntry->uFlagsAndType & (EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC));
+
+    /*
+     * If common exit type, we will insert/update the exit into the exit record hash table.
+     */
+    if (   (uFlagsAndType & (EMEXIT_F_KIND_MASK | EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC)) == EMEXIT_F_KIND_EM
+#ifdef IN_RING0
+        && pVCpu->em.s.fExitOptimizationEnabledR0
+        && ( !(uFlagsAndType & EMEXIT_F_HM) || pVCpu->em.s.fExitOptimizationEnabledR0PreemptDisabled)
+#else
+        && pVCpu->em.s.fExitOptimizationEnabled
+#endif
+        && pHistEntry->uFlatPC != UINT64_MAX
+       )
+        return emHistoryAddOrUpdateRecord(pVCpu, uFlagsAndType, pHistEntry->uFlatPC, pHistEntry, uExitNo);
+    return NULL;
+}
+
+
+/**
+ * Interface for convering a engine specific exit to a generic one and get
+ * guidance, supplying flattened PC too.
+ *
+ * @returns Pointer to an exit record if special action should be taken using
+ *          EMHistoryExec().  Take normal exit action when NULL.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uFlagsAndType   Combined flags and type (see EMEXIT_MAKE_FLAGS_AND_TYPE).
+ * @param   uFlatPC         The flattened program counter (RIP).
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(PCEMEXITREC) EMHistoryUpdateFlagsAndTypeAndPC(PVMCPUCC pVCpu, uint32_t uFlagsAndType, uint64_t uFlatPC)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(uFlatPC != UINT64_MAX);
+
+    /*
+     * Do the updating.
+     */
+    AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitHistory) == 256);
+    uint64_t     uExitNo    = pVCpu->em.s.iNextExit - 1;
+    PEMEXITENTRY pHistEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)uExitNo & 0xff];
+    pHistEntry->uFlagsAndType = uFlagsAndType;
+    pHistEntry->uFlatPC       = uFlatPC;
+
+    /*
+     * If common exit type, we will insert/update the exit into the exit record hash table.
+     */
+    if (   (uFlagsAndType & (EMEXIT_F_KIND_MASK | EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC)) == EMEXIT_F_KIND_EM
+#ifdef IN_RING0
+        && pVCpu->em.s.fExitOptimizationEnabledR0
+        && ( !(uFlagsAndType & EMEXIT_F_HM) || pVCpu->em.s.fExitOptimizationEnabledR0PreemptDisabled)
+#else
+        && pVCpu->em.s.fExitOptimizationEnabled
+#endif
+       )
+        return emHistoryAddOrUpdateRecord(pVCpu, uFlagsAndType, uFlatPC, pHistEntry, uExitNo);
+    return NULL;
+}
+
+
+/**
+ * @callback_method_impl{FNDISREADBYTES}
+ */
+static DECLCALLBACK(int) emReadBytes(PDISCPUSTATE pDis, uint8_t offInstr, uint8_t cbMinRead, uint8_t cbMaxRead)
+{
+    PVMCPUCC    pVCpu    = (PVMCPUCC)pDis->pvUser;
+    RTUINTPTR   uSrcAddr = pDis->uInstrAddr + offInstr;
+
+    /*
+     * Figure how much we can or must read.
+     */
+    size_t      cbToRead = PAGE_SIZE - (uSrcAddr & PAGE_OFFSET_MASK);
+    if (cbToRead > cbMaxRead)
+        cbToRead = cbMaxRead;
+    else if (cbToRead < cbMinRead)
+        cbToRead = cbMinRead;
+
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, &pDis->abInstr[offInstr], uSrcAddr, cbToRead);
+    if (RT_FAILURE(rc))
+    {
+        if (cbToRead > cbMinRead)
+        {
+            cbToRead = cbMinRead;
+            rc = PGMPhysSimpleReadGCPtr(pVCpu, &pDis->abInstr[offInstr], uSrcAddr, cbToRead);
+        }
+        if (RT_FAILURE(rc))
+        {
+            /*
+             * If we fail to find the page via the guest's page tables
+             * we invalidate the page in the host TLB (pertaining to
+             * the guest in the NestedPaging case). See @bugref{6043}.
+             */
+            if (rc == VERR_PAGE_TABLE_NOT_PRESENT || rc == VERR_PAGE_NOT_PRESENT)
+            {
+                HMInvalidatePage(pVCpu, uSrcAddr);
+                if (((uSrcAddr + cbToRead - 1) >> PAGE_SHIFT) !=  (uSrcAddr >> PAGE_SHIFT))
+                    HMInvalidatePage(pVCpu, uSrcAddr + cbToRead - 1);
+            }
+        }
+    }
+
+    pDis->cbCachedInstr = offInstr + (uint8_t)cbToRead;
+    return rc;
+}
+
+
+/**
+ * Disassembles the current instruction.
+ *
+ * @returns VBox status code, see SELMToFlatEx and EMInterpretDisasOneEx for
+ *          details.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pDis            Where to return the parsed instruction info.
+ * @param   pcbInstr        Where to return the instruction size. (optional)
+ */
+VMM_INT_DECL(int) EMInterpretDisasCurrent(PVMCC pVM, PVMCPUCC pVCpu, PDISCPUSTATE pDis, unsigned *pcbInstr)
+{
+    PCPUMCTXCORE pCtxCore = CPUMCTX2CORE(CPUMQueryGuestCtxPtr(pVCpu));
+    RTGCPTR GCPtrInstr;
+#if 0
+    int rc = SELMToFlatEx(pVCpu, DISSELREG_CS, pCtxCore, pCtxCore->rip, 0, &GCPtrInstr);
+#else
+/** @todo Get the CPU mode as well while we're at it! */
+    int rc = SELMValidateAndConvertCSAddr(pVCpu, pCtxCore->eflags, pCtxCore->ss.Sel, pCtxCore->cs.Sel, &pCtxCore->cs,
+                                          pCtxCore->rip, &GCPtrInstr);
+#endif
+    if (RT_FAILURE(rc))
+    {
+        Log(("EMInterpretDisasOne: Failed to convert %RTsel:%RGv (cpl=%d) - rc=%Rrc !!\n",
+             pCtxCore->cs.Sel, (RTGCPTR)pCtxCore->rip, pCtxCore->ss.Sel & X86_SEL_RPL, rc));
+        return rc;
+    }
+    return EMInterpretDisasOneEx(pVM, pVCpu, (RTGCUINTPTR)GCPtrInstr, pCtxCore, pDis, pcbInstr);
+}
+
+
+/**
+ * Disassembles one instruction.
+ *
+ * This is used by internally by the interpreter and by trap/access handlers.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPtrInstr      The flat address of the instruction.
+ * @param   pCtxCore        The context core (used to determine the cpu mode).
+ * @param   pDis            Where to return the parsed instruction info.
+ * @param   pcbInstr        Where to return the instruction size. (optional)
+ */
+VMM_INT_DECL(int) EMInterpretDisasOneEx(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINTPTR GCPtrInstr, PCCPUMCTXCORE pCtxCore,
+                                        PDISCPUSTATE pDis, unsigned *pcbInstr)
+{
+    NOREF(pVM);
+    Assert(pCtxCore == CPUMGetGuestCtxCore(pVCpu)); NOREF(pCtxCore);
+    DISCPUMODE enmCpuMode = CPUMGetGuestDisMode(pVCpu);
+    /** @todo Deal with too long instruction (=> \#GP), opcode read errors (=>
+     *        \#PF, \#GP, \#??), undefined opcodes (=> \#UD), and such. */
+    int rc = DISInstrWithReader(GCPtrInstr, enmCpuMode, emReadBytes, pVCpu, pDis, pcbInstr);
+    if (RT_SUCCESS(rc))
+        return VINF_SUCCESS;
+    AssertMsg(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT, ("DISCoreOne failed to GCPtrInstr=%RGv rc=%Rrc\n", GCPtrInstr, rc));
+    return rc;
+}
+
+
+/**
+ * Interprets the current instruction.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_*                  Scheduling instructions.
+ * @retval  VERR_EM_INTERPRETER     Something we can't cope with.
+ * @retval  VERR_*                  Fatal errors.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pRegFrame   The register frame.
+ *                      Updates the EIP if an instruction was executed successfully.
+ * @param   pvFault     The fault address (CR2).
+ *
+ * @remark  Invalid opcode exceptions have a higher priority than GP (see Intel
+ *          Architecture System Developers Manual, Vol 3, 5.5) so we don't need
+ *          to worry about e.g. invalid modrm combinations (!)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) EMInterpretInstruction(PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
+{
+    Assert(pRegFrame == CPUMGetGuestCtxCore(pVCpu));
+    LogFlow(("EMInterpretInstruction %RGv fault %RGv\n", (RTGCPTR)pRegFrame->rip, pvFault));
+    NOREF(pvFault);
+
+    VBOXSTRICTRC rc = IEMExecOneBypassEx(pVCpu, pRegFrame, NULL);
+    if (RT_UNLIKELY(   rc == VERR_IEM_ASPECT_NOT_IMPLEMENTED
+                    || rc == VERR_IEM_INSTR_NOT_IMPLEMENTED))
+        rc = VERR_EM_INTERPRETER;
+    if (rc != VINF_SUCCESS)
+        Log(("EMInterpretInstruction: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
+
+    return rc;
+}
+
+
+/**
+ * Interprets the current instruction.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_*                  Scheduling instructions.
+ * @retval  VERR_EM_INTERPRETER     Something we can't cope with.
+ * @retval  VERR_*                  Fatal errors.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pRegFrame   The register frame.
+ *                      Updates the EIP if an instruction was executed successfully.
+ * @param   pvFault     The fault address (CR2).
+ * @param   pcbWritten  Size of the write (if applicable).
+ *
+ * @remark  Invalid opcode exceptions have a higher priority than GP (see Intel
+ *          Architecture System Developers Manual, Vol 3, 5.5) so we don't need
+ *          to worry about e.g. invalid modrm combinations (!)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) EMInterpretInstructionEx(PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, uint32_t *pcbWritten)
+{
+    LogFlow(("EMInterpretInstructionEx %RGv fault %RGv\n", (RTGCPTR)pRegFrame->rip, pvFault));
+    Assert(pRegFrame == CPUMGetGuestCtxCore(pVCpu));
+    NOREF(pvFault);
+
+    VBOXSTRICTRC rc = IEMExecOneBypassEx(pVCpu, pRegFrame, pcbWritten);
+    if (RT_UNLIKELY(   rc == VERR_IEM_ASPECT_NOT_IMPLEMENTED
+                    || rc == VERR_IEM_INSTR_NOT_IMPLEMENTED))
+        rc = VERR_EM_INTERPRETER;
+    if (rc != VINF_SUCCESS)
+        Log(("EMInterpretInstructionEx: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
+
+    return rc;
+}
+
+
+/**
+ * Interprets the current instruction using the supplied DISCPUSTATE structure.
+ *
+ * IP/EIP/RIP *IS* updated!
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_*                  Scheduling instructions. When these are returned, it
+ *                                  starts to get a bit tricky to know whether code was
+ *                                  executed or not... We'll address this when it becomes a problem.
+ * @retval  VERR_EM_INTERPRETER     Something we can't cope with.
+ * @retval  VERR_*                  Fatal errors.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pDis        The disassembler cpu state for the instruction to be
+ *                      interpreted.
+ * @param   pRegFrame   The register frame. IP/EIP/RIP *IS* changed!
+ * @param   pvFault     The fault address (CR2).
+ * @param   enmCodeType Code type (user/supervisor)
+ *
+ * @remark  Invalid opcode exceptions have a higher priority than GP (see Intel
+ *          Architecture System Developers Manual, Vol 3, 5.5) so we don't need
+ *          to worry about e.g. invalid modrm combinations (!)
+ *
+ * @todo    At this time we do NOT check if the instruction overwrites vital information.
+ *          Make sure this can't happen!! (will add some assertions/checks later)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) EMInterpretInstructionDisasState(PVMCPUCC pVCpu, PDISCPUSTATE pDis, PCPUMCTXCORE pRegFrame,
+                                                            RTGCPTR pvFault, EMCODETYPE enmCodeType)
+{
+    LogFlow(("EMInterpretInstructionDisasState %RGv fault %RGv\n", (RTGCPTR)pRegFrame->rip, pvFault));
+    Assert(pRegFrame == CPUMGetGuestCtxCore(pVCpu));
+    NOREF(pDis); NOREF(pvFault); NOREF(enmCodeType);
+
+    VBOXSTRICTRC rc = IEMExecOneBypassWithPrefetchedByPC(pVCpu, pRegFrame, pRegFrame->rip, pDis->abInstr, pDis->cbCachedInstr);
+    if (RT_UNLIKELY(   rc == VERR_IEM_ASPECT_NOT_IMPLEMENTED
+                    || rc == VERR_IEM_INSTR_NOT_IMPLEMENTED))
+        rc = VERR_EM_INTERPRETER;
+
+    if (rc != VINF_SUCCESS)
+        Log(("EMInterpretInstructionDisasState: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
+
+    return rc;
+}
+
+
+
+
+/*
+ *
+ * Old interpreter primitives used by HM, move/eliminate later.
+ * Old interpreter primitives used by HM, move/eliminate later.
+ * Old interpreter primitives used by HM, move/eliminate later.
+ * Old interpreter primitives used by HM, move/eliminate later.
+ * Old interpreter primitives used by HM, move/eliminate later.
+ *
+ */
+
+
+/**
+ * Interpret RDPMC.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pRegFrame   The register frame.
+ *
+ */
+VMM_INT_DECL(int) EMInterpretRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame)
+{
+    Assert(pRegFrame == CPUMGetGuestCtxCore(pVCpu));
+    uint32_t uCR4 = CPUMGetGuestCR4(pVCpu);
+
+    /* If X86_CR4_PCE is not set, then CPL must be zero. */
+    if (    !(uCR4 & X86_CR4_PCE)
+        &&  CPUMGetGuestCPL(pVCpu) != 0)
+    {
+        Assert(CPUMGetGuestCR0(pVCpu) & X86_CR0_PE);
+        return VERR_EM_INTERPRETER; /* genuine #GP */
+    }
+
+    /* Just return zero here; rather tricky to properly emulate this, especially as the specs are a mess. */
+    pRegFrame->rax = 0;
+    pRegFrame->rdx = 0;
+    /** @todo We should trigger a \#GP here if the CPU doesn't support the index in
+     *        ecx but see @bugref{3472}! */
+
+    NOREF(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/* VT-x only: */
+
+/**
+ * Interpret DRx write.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pRegFrame   The register frame.
+ * @param   DestRegDrx  DRx register index (USE_REG_DR*)
+ * @param   SrcRegGen   General purpose register index (USE_REG_E**))
+ *
+ */
+VMM_INT_DECL(int) EMInterpretDRxWrite(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, uint32_t DestRegDrx, uint32_t SrcRegGen)
+{
+    Assert(pRegFrame == CPUMGetGuestCtxCore(pVCpu));
+    uint64_t uNewDrX;
+    int      rc;
+    NOREF(pVM);
+
+    if (CPUMIsGuestIn64BitCode(pVCpu))
+        rc = DISFetchReg64(pRegFrame, SrcRegGen, &uNewDrX);
+    else
+    {
+        uint32_t val32;
+        rc = DISFetchReg32(pRegFrame, SrcRegGen, &val32);
+        uNewDrX = val32;
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        if (DestRegDrx == 6)
+        {
+            uNewDrX |= X86_DR6_RA1_MASK;
+            uNewDrX &= ~X86_DR6_RAZ_MASK;
+        }
+        else if (DestRegDrx == 7)
+        {
+            uNewDrX |= X86_DR7_RA1_MASK;
+            uNewDrX &= ~X86_DR7_RAZ_MASK;
+        }
+
+        /** @todo we don't fail if illegal bits are set/cleared for e.g. dr7 */
+        rc = CPUMSetGuestDRx(pVCpu, DestRegDrx, uNewDrX);
+        if (RT_SUCCESS(rc))
+            return rc;
+        AssertMsgFailed(("CPUMSetGuestDRx %d failed\n", DestRegDrx));
+    }
+    return VERR_EM_INTERPRETER;
+}
+
+
+/**
+ * Interpret DRx read.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pRegFrame   The register frame.
+ * @param   DestRegGen  General purpose register index (USE_REG_E**))
+ * @param   SrcRegDrx   DRx register index (USE_REG_DR*)
+ */
+VMM_INT_DECL(int) EMInterpretDRxRead(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, uint32_t DestRegGen, uint32_t SrcRegDrx)
+{
+    uint64_t val64;
+    Assert(pRegFrame == CPUMGetGuestCtxCore(pVCpu));
+    NOREF(pVM);
+
+    int rc = CPUMGetGuestDRx(pVCpu, SrcRegDrx, &val64);
+    AssertMsgRCReturn(rc, ("CPUMGetGuestDRx %d failed\n", SrcRegDrx), VERR_EM_INTERPRETER);
+    if (CPUMIsGuestIn64BitCode(pVCpu))
+        rc = DISWriteReg64(pRegFrame, DestRegGen, val64);
+    else
+        rc = DISWriteReg32(pRegFrame, DestRegGen, (uint32_t)val64);
+
+    if (RT_SUCCESS(rc))
+        return VINF_SUCCESS;
+
+    return VERR_EM_INTERPRETER;
+}
+
diff --git a/src/VBox/VMM/VMMAll/GIMAll.cpp b/src/VBox/VMM/VMMAll/GIMAll.cpp
new file mode 100644
index 00000000..83716f9c
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/GIMAll.cpp
@@ -0,0 +1,492 @@
+/* $Id: GIMAll.cpp $ */
+/** @file
+ * GIM - Guest Interface Manager - All Contexts.
+ */
+
+/*
+ * Copyright (C) 2014-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/em.h>    /* For EMInterpretDisasCurrent */
+#include "GIMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/dis.h>       /* For DISCPUSTATE */
+#include <VBox/err.h>
+#include <iprt/string.h>
+
+/* Include all the providers. */
+#include "GIMHvInternal.h"
+#include "GIMMinimalInternal.h"
+
+
+/**
+ * Checks whether GIM is being used by this VM.
+ *
+ * @retval  true if used.
+ * @retval  false if no GIM provider ("none") is used.
+ *
+ * @param   pVM       The cross context VM structure.
+ */
+VMMDECL(bool) GIMIsEnabled(PVM pVM)
+{
+    return pVM->gim.s.enmProviderId != GIMPROVIDERID_NONE;
+}
+
+
+/**
+ * Gets the GIM provider configured for this VM.
+ *
+ * @returns The GIM provider Id.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(GIMPROVIDERID) GIMGetProvider(PVM pVM)
+{
+    return pVM->gim.s.enmProviderId;
+}
+
+
+/**
+ * Returns the array of MMIO2 regions that are expected to be registered and
+ * later mapped into the guest-physical address space for the GIM provider
+ * configured for the VM.
+ *
+ * @returns Pointer to an array of GIM MMIO2 regions, may return NULL.
+ * @param   pVM         The cross context VM structure.
+ * @param   pcRegions   Where to store the number of items in the array.
+ *
+ * @remarks The caller does not own and therefore must -NOT- try to free the
+ *          returned pointer.
+ */
+VMMDECL(PGIMMMIO2REGION) GIMGetMmio2Regions(PVMCC pVM, uint32_t *pcRegions)
+{
+    Assert(pVM);
+    Assert(pcRegions);
+
+    *pcRegions = 0;
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvGetMmio2Regions(pVM, pcRegions);
+
+        default:
+            break;
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Returns whether the guest has configured and enabled calls to the hypervisor.
+ *
+ * @returns true if hypercalls are enabled and usable, false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) GIMAreHypercallsEnabled(PVMCPUCC pVCpu)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    if (!GIMIsEnabled(pVM))
+        return false;
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvAreHypercallsEnabled(pVM);
+
+        case GIMPROVIDERID_KVM:
+            return gimKvmAreHypercallsEnabled(pVCpu);
+
+        default:
+            return false;
+    }
+}
+
+
+/**
+ * Implements a GIM hypercall with the provider configured for the VM.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating
+ *          RIP.
+ * @retval  VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ * @retval  VERR_GIM_HYPERCALLS_NOT_AVAILABLE hypercalls unavailable.
+ * @retval  VERR_GIM_NOT_ENABLED GIM is not enabled (shouldn't really happen)
+ * @retval  VERR_GIM_HYPERCALL_MEMORY_READ_FAILED hypercall failed while reading
+ *          memory.
+ * @retval  VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED hypercall failed while
+ *          writing memory.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ *
+ * @remarks The caller of this function needs to advance RIP as required.
+ * @thread  EMT.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GIMHypercall(PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    if (RT_UNLIKELY(!GIMIsEnabled(pVM)))
+        return VERR_GIM_NOT_ENABLED;
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvHypercall(pVCpu, pCtx);
+
+        case GIMPROVIDERID_KVM:
+            return gimKvmHypercall(pVCpu, pCtx);
+
+        default:
+            AssertMsgFailed(("GIMHypercall: for provider %u not available/implemented\n", pVM->gim.s.enmProviderId));
+            return VERR_GIM_HYPERCALLS_NOT_AVAILABLE;
+    }
+}
+
+
+/**
+ * Same as GIMHypercall, except with disassembler opcode and instruction length.
+ *
+ * This is the interface used by IEM.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating
+ *          RIP.
+ * @retval  VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ * @retval  VERR_GIM_HYPERCALLS_NOT_AVAILABLE hypercalls unavailable.
+ * @retval  VERR_GIM_NOT_ENABLED GIM is not enabled (shouldn't really happen)
+ * @retval  VERR_GIM_HYPERCALL_MEMORY_READ_FAILED hypercall failed while reading
+ *          memory.
+ * @retval  VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED hypercall failed while
+ *          writing memory.
+ * @retval  VERR_GIM_INVALID_HYPERCALL_INSTR if uDisOpcode is the wrong one; raise \#UD.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   uDisOpcode  The disassembler opcode.
+ * @param   cbInstr     The instruction length.
+ *
+ * @remarks The caller of this function needs to advance RIP as required.
+ * @thread  EMT.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GIMHypercallEx(PVMCPUCC pVCpu, PCPUMCTX pCtx, unsigned uDisOpcode, uint8_t cbInstr)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    if (RT_UNLIKELY(!GIMIsEnabled(pVM)))
+        return VERR_GIM_NOT_ENABLED;
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvHypercallEx(pVCpu, pCtx, uDisOpcode, cbInstr);
+
+        case GIMPROVIDERID_KVM:
+            return gimKvmHypercallEx(pVCpu, pCtx, uDisOpcode, cbInstr);
+
+        default:
+            AssertMsgFailedReturn(("enmProviderId=%u\n", pVM->gim.s.enmProviderId), VERR_GIM_HYPERCALLS_NOT_AVAILABLE);
+    }
+}
+
+
+/**
+ * Disassembles the instruction at RIP and if it's a hypercall
+ * instruction, performs the hypercall.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   pcbInstr    Where to store the disassembled instruction length.
+ *                      Optional, can be NULL.
+ *
+ * @todo    This interface should disappear when IEM/REM execution engines
+ *          handle VMCALL/VMMCALL instructions to call into GIM when
+ *          required. See @bugref{7270#c168}.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GIMExecHypercallInstr(PVMCPUCC pVCpu, PCPUMCTX pCtx, uint8_t *pcbInstr)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    if (RT_UNLIKELY(!GIMIsEnabled(pVM)))
+        return VERR_GIM_NOT_ENABLED;
+
+    unsigned    cbInstr;
+    DISCPUSTATE Dis;
+    int rc = EMInterpretDisasCurrent(pVM, pVCpu, &Dis, &cbInstr);
+    if (RT_SUCCESS(rc))
+    {
+        if (pcbInstr)
+            *pcbInstr = (uint8_t)cbInstr;
+        switch (pVM->gim.s.enmProviderId)
+        {
+            case GIMPROVIDERID_HYPERV:
+                return gimHvHypercallEx(pVCpu, pCtx, Dis.pCurInstr->uOpcode, Dis.cbInstr);
+
+            case GIMPROVIDERID_KVM:
+                return gimKvmHypercallEx(pVCpu, pCtx, Dis.pCurInstr->uOpcode, Dis.cbInstr);
+
+            default:
+                AssertMsgFailed(("GIMExecHypercallInstr: for provider %u not available/implemented\n", pVM->gim.s.enmProviderId));
+                return VERR_GIM_HYPERCALLS_NOT_AVAILABLE;
+        }
+    }
+
+    Log(("GIM: GIMExecHypercallInstr: Failed to disassemble CS:RIP=%04x:%08RX64. rc=%Rrc\n", pCtx->cs.Sel, pCtx->rip, rc));
+    return rc;
+}
+
+
+/**
+ * Returns whether the guest has configured and setup the use of paravirtualized
+ * TSC.
+ *
+ * Paravirtualized TSCs are per-VM and the rest of the execution engine logic
+ * relies on that.
+ *
+ * @returns true if enabled and usable, false otherwise.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(bool) GIMIsParavirtTscEnabled(PVMCC pVM)
+{
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvIsParavirtTscEnabled(pVM);
+
+        case GIMPROVIDERID_KVM:
+            return gimKvmIsParavirtTscEnabled(pVM);
+
+        default:
+            break;
+    }
+    return false;
+}
+
+
+/**
+ * Whether \#UD exceptions in the guest needs to be intercepted by the GIM
+ * provider.
+ *
+ * At the moment, the reason why this isn't a more generic interface wrt to
+ * exceptions is because of performance (each VM-exit would have to manually
+ * check whether or not GIM needs to be notified). Left as a todo for later if
+ * really required.
+ *
+ * @returns true if needed, false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) GIMShouldTrapXcptUD(PVMCPUCC pVCpu)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    if (!GIMIsEnabled(pVM))
+        return false;
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_KVM:
+            return gimKvmShouldTrapXcptUD(pVM);
+
+        case GIMPROVIDERID_HYPERV:
+            return gimHvShouldTrapXcptUD(pVCpu);
+
+        default:
+            return false;
+    }
+}
+
+
+/**
+ * Exception handler for \#UD when requested by the GIM provider.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_R3_HYPERCALL restart the hypercall from ring-3.
+ * @retval  VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating
+ *          RIP.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ * @retval  VERR_GIM_INVALID_HYPERCALL_INSTR instruction at RIP is not a valid
+ *          hypercall instruction.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   pDis        Pointer to the disassembled instruction state at RIP.
+ *                      If NULL is passed, it implies the disassembly of the
+ *                      the instruction at RIP is the responsibility of the
+ *                      GIM provider.
+ * @param   pcbInstr    Where to store the instruction length of the hypercall
+ *                      instruction. Optional, can be NULL.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GIMXcptUD(PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    Assert(GIMIsEnabled(pVM));
+    Assert(pDis || pcbInstr);
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_KVM:
+            return gimKvmXcptUD(pVM, pVCpu, pCtx, pDis, pcbInstr);
+
+        case GIMPROVIDERID_HYPERV:
+            return gimHvXcptUD(pVCpu, pCtx, pDis, pcbInstr);
+
+        default:
+            return VERR_GIM_OPERATION_FAILED;
+    }
+}
+
+
+/**
+ * Invokes the read-MSR handler for the GIM provider configured for the VM.
+ *
+ * @returns Strict VBox status code like CPUMQueryGuestMsr.
+ * @retval  VINF_CPUM_R3_MSR_READ
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR to read.
+ * @param   pRange      The range this MSR belongs to.
+ * @param   puValue     Where to store the MSR value read.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GIMReadMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    Assert(pVCpu);
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    Assert(GIMIsEnabled(pVM));
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvReadMsr(pVCpu, idMsr, pRange, puValue);
+
+        case GIMPROVIDERID_KVM:
+            return gimKvmReadMsr(pVCpu, idMsr, pRange, puValue);
+
+        default:
+            AssertMsgFailed(("GIMReadMsr: for unknown provider %u idMsr=%#RX32 -> #GP(0)", pVM->gim.s.enmProviderId, idMsr));
+            return VERR_CPUM_RAISE_GP_0;
+    }
+}
+
+
+/**
+ * Invokes the write-MSR handler for the GIM provider configured for the VM.
+ *
+ * @returns Strict VBox status code like CPUMSetGuestMsr.
+ * @retval  VINF_CPUM_R3_MSR_WRITE
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR to write.
+ * @param   pRange      The range this MSR belongs to.
+ * @param   uValue      The value to set, ignored bits masked.
+ * @param   uRawValue   The raw value with the ignored bits not masked.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GIMWriteMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    AssertPtr(pVCpu);
+    NOREF(uValue);
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    Assert(GIMIsEnabled(pVM));
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvWriteMsr(pVCpu, idMsr, pRange, uRawValue);
+
+        case GIMPROVIDERID_KVM:
+            return gimKvmWriteMsr(pVCpu, idMsr, pRange, uRawValue);
+
+        default:
+            AssertMsgFailed(("GIMWriteMsr: for unknown provider %u idMsr=%#RX32 -> #GP(0)", pVM->gim.s.enmProviderId, idMsr));
+            return VERR_CPUM_RAISE_GP_0;
+    }
+}
+
+
+/**
+ * Queries the opcode bytes for a native hypercall.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvBuf       The destination buffer.
+ * @param   cbBuf       The size of the buffer.
+ * @param   pcbWritten  Where to return the number of bytes written.  This is
+ *                      reliably updated only on successful return.  Optional.
+ * @param   puDisOpcode Where to return the disassembler opcode.  Optional.
+ */
+VMM_INT_DECL(int) GIMQueryHypercallOpcodeBytes(PVM pVM, void *pvBuf, size_t cbBuf, size_t *pcbWritten, uint16_t *puDisOpcode)
+{
+    AssertPtrReturn(pvBuf, VERR_INVALID_POINTER);
+
+    CPUMCPUVENDOR  enmHostCpu = CPUMGetHostCpuVendor(pVM);
+    uint8_t const *pbSrc;
+    size_t         cbSrc;
+    switch (enmHostCpu)
+    {
+        case CPUMCPUVENDOR_AMD:
+        case CPUMCPUVENDOR_HYGON:
+        {
+            if (puDisOpcode)
+                *puDisOpcode = OP_VMMCALL;
+            static uint8_t const s_abHypercall[] = { 0x0F, 0x01, 0xD9 };   /* VMMCALL */
+            pbSrc = s_abHypercall;
+            cbSrc = sizeof(s_abHypercall);
+            break;
+        }
+
+        case CPUMCPUVENDOR_INTEL:
+        case CPUMCPUVENDOR_VIA:
+        case CPUMCPUVENDOR_SHANGHAI:
+        {
+            if (puDisOpcode)
+                *puDisOpcode = OP_VMCALL;
+            static uint8_t const s_abHypercall[] = { 0x0F, 0x01, 0xC1 };   /* VMCALL */
+            pbSrc = s_abHypercall;
+            cbSrc = sizeof(s_abHypercall);
+            break;
+        }
+
+        default:
+            AssertMsgFailedReturn(("%d\n", enmHostCpu), VERR_UNSUPPORTED_CPU);
+    }
+    if (RT_LIKELY(cbBuf >= cbSrc))
+    {
+        memcpy(pvBuf, pbSrc, cbSrc);
+        if (pcbWritten)
+            *pcbWritten = cbSrc;
+        return VINF_SUCCESS;
+    }
+    return VERR_BUFFER_OVERFLOW;
+}
+
diff --git a/src/VBox/VMM/VMMAll/GIMAllHv.cpp b/src/VBox/VMM/VMMAll/GIMAllHv.cpp
new file mode 100644
index 00000000..983c32b7
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/GIMAllHv.cpp
@@ -0,0 +1,1487 @@
+/* $Id: GIMAllHv.cpp $ */
+/** @file
+ * GIM - Guest Interface Manager, Microsoft Hyper-V, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2014-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/em.h>
+#include "GIMHvInternal.h"
+#include "GIMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/err.h>
+
+#include <iprt/asm-amd64-x86.h>
+#ifdef IN_RING3
+# include <iprt/mem.h>
+#endif
+
+
+#ifdef IN_RING3
+/**
+ * Read and validate slow hypercall parameters.
+ *
+ * @returns VBox status code.
+ * @param   pVM               The cross context VM structure.
+ * @param   pCtx              Pointer to the guest-CPU context.
+ * @param   fIs64BitMode      Whether the guest is currently in 64-bit mode or not.
+ * @param   enmParam          The hypercall parameter type.
+ * @param   prcHv             Where to store the Hyper-V status code. Only valid
+ *                            to the caller when this function returns
+ *                            VINF_SUCCESS.
+ */
+static int gimHvReadSlowHypercallParam(PVM pVM, PCPUMCTX pCtx, bool fIs64BitMode, GIMHVHYPERCALLPARAM enmParam, int *prcHv)
+{
+    int       rc = VINF_SUCCESS;
+    PGIMHV    pHv = &pVM->gim.s.u.Hv;
+    RTGCPHYS  GCPhysParam;
+    void     *pvDst;
+    if (enmParam == GIMHVHYPERCALLPARAM_IN)
+    {
+        GCPhysParam = fIs64BitMode ? pCtx->rdx : (pCtx->rbx << 32) | pCtx->ecx;
+        pvDst = pHv->pbHypercallIn;
+        pHv->GCPhysHypercallIn = GCPhysParam;
+    }
+    else
+    {
+        GCPhysParam = fIs64BitMode ? pCtx->r8 : (pCtx->rdi << 32) | pCtx->esi;
+        pvDst = pHv->pbHypercallOut;
+        pHv->GCPhysHypercallOut = GCPhysParam;
+        Assert(enmParam == GIMHVHYPERCALLPARAM_OUT);
+    }
+
+    const char *pcszParam = enmParam == GIMHVHYPERCALLPARAM_IN ? "input" : "output";  NOREF(pcszParam);
+    if (RT_ALIGN_64(GCPhysParam, 8) == GCPhysParam)
+    {
+        if (PGMPhysIsGCPhysNormal(pVM, GCPhysParam))
+        {
+            rc = PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysParam, GIM_HV_PAGE_SIZE);
+            if (RT_SUCCESS(rc))
+            {
+                *prcHv = GIM_HV_STATUS_SUCCESS;
+                return VINF_SUCCESS;
+            }
+            LogRel(("GIM: HyperV: Failed reading %s param at %#RGp. rc=%Rrc\n", pcszParam, GCPhysParam, rc));
+            rc = VERR_GIM_HYPERCALL_MEMORY_READ_FAILED;
+        }
+        else
+        {
+            Log(("GIM: HyperV: Invalid %s param address %#RGp\n", pcszParam, GCPhysParam));
+            *prcHv = GIM_HV_STATUS_INVALID_PARAMETER;
+        }
+    }
+    else
+    {
+        Log(("GIM: HyperV: Misaligned %s param address %#RGp\n", pcszParam, GCPhysParam));
+        *prcHv = GIM_HV_STATUS_INVALID_ALIGNMENT;
+    }
+    return rc;
+}
+
+
+/**
+ * Helper for reading and validating slow hypercall input and output parameters.
+ *
+ * @returns VBox status code.
+ * @param   pVM               The cross context VM structure.
+ * @param   pCtx              Pointer to the guest-CPU context.
+ * @param   fIs64BitMode      Whether the guest is currently in 64-bit mode or not.
+ * @param   prcHv             Where to store the Hyper-V status code. Only valid
+ *                            to the caller when this function returns
+ *                            VINF_SUCCESS.
+ */
+static int gimHvReadSlowHypercallParamsInOut(PVM pVM, PCPUMCTX pCtx, bool fIs64BitMode, int *prcHv)
+{
+    int rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_IN, prcHv);
+    if (   RT_SUCCESS(rc)
+        && *prcHv == GIM_HV_STATUS_SUCCESS)
+        rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_OUT, prcHv);
+    return rc;
+}
+#endif
+
+
+/**
+ * Handles all Hyper-V hypercalls.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
+ * @retval  VERR_GIM_HYPERCALLS_NOT_ENABLED hypercalls are disabled by the
+ *          guest.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ * @retval  VERR_GIM_HYPERCALL_MEMORY_READ_FAILED hypercall failed while reading
+ *          memory.
+ * @retval  VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED hypercall failed while
+ *          writing memory.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pCtx            Pointer to the guest-CPU context.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimHvHypercall(PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+#ifndef IN_RING3
+    RT_NOREF_PV(pVCpu);
+    RT_NOREF_PV(pCtx);
+    return VINF_GIM_R3_HYPERCALL;
+#else
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    STAM_REL_COUNTER_INC(&pVM->gim.s.StatHypercalls);
+
+    /*
+     * Verify that hypercalls are enabled by the guest.
+     */
+    if (!gimHvAreHypercallsEnabled(pVM))
+        return VERR_GIM_HYPERCALLS_NOT_ENABLED;
+
+    /*
+     * Verify guest is in ring-0 protected mode.
+     */
+    uint32_t uCpl = CPUMGetGuestCPL(pVCpu);
+    if (   uCpl
+        || CPUMIsGuestInRealModeEx(pCtx))
+    {
+        return VERR_GIM_HYPERCALL_ACCESS_DENIED;
+    }
+
+    /*
+     * Get the hypercall operation code and modes.
+     * Fast hypercalls have only two or fewer inputs but no output parameters.
+     */
+    const bool       fIs64BitMode     = CPUMIsGuestIn64BitCodeEx(pCtx);
+    const uint64_t   uHyperIn         = fIs64BitMode ? pCtx->rcx : (pCtx->rdx << 32) | pCtx->eax;
+    const uint16_t   uHyperOp         = GIM_HV_HYPERCALL_IN_CALL_CODE(uHyperIn);
+    const bool       fHyperFast       = GIM_HV_HYPERCALL_IN_IS_FAST(uHyperIn);
+    const uint16_t   cHyperReps       = GIM_HV_HYPERCALL_IN_REP_COUNT(uHyperIn);
+    const uint16_t   idxHyperRepStart = GIM_HV_HYPERCALL_IN_REP_START_IDX(uHyperIn);
+    uint64_t         cHyperRepsDone   = 0;
+
+    /* Currently no repeating hypercalls are supported. */
+    RT_NOREF2(cHyperReps, idxHyperRepStart);
+
+    int rc     = VINF_SUCCESS;
+    int rcHv   = GIM_HV_STATUS_OPERATION_DENIED;
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+
+    /*
+     * Validate common hypercall input parameters.
+     */
+    if (   !GIM_HV_HYPERCALL_IN_RSVD_1(uHyperIn)
+        && !GIM_HV_HYPERCALL_IN_RSVD_2(uHyperIn)
+        && !GIM_HV_HYPERCALL_IN_RSVD_3(uHyperIn))
+    {
+        /*
+         * Perform the hypercall.
+         */
+        switch (uHyperOp)
+        {
+            case GIM_HV_HYPERCALL_OP_RETREIVE_DEBUG_DATA:   /* Non-rep, memory IO. */
+            {
+                if (pHv->uPartFlags & GIM_HV_PART_FLAGS_DEBUGGING)
+                {
+                    rc  = gimHvReadSlowHypercallParamsInOut(pVM, pCtx, fIs64BitMode, &rcHv);
+                    if (   RT_SUCCESS(rc)
+                        && rcHv == GIM_HV_STATUS_SUCCESS)
+                    {
+                        LogRelMax(1, ("GIM: HyperV: Initiated debug data reception via hypercall\n"));
+                        rc = gimR3HvHypercallRetrieveDebugData(pVM, &rcHv);
+                        if (RT_FAILURE(rc))
+                            LogRelMax(10, ("GIM: HyperV: gimR3HvHypercallRetrieveDebugData failed. rc=%Rrc\n", rc));
+                    }
+                }
+                else
+                    rcHv = GIM_HV_STATUS_ACCESS_DENIED;
+                break;
+            }
+
+            case GIM_HV_HYPERCALL_OP_POST_DEBUG_DATA:   /* Non-rep, memory IO. */
+            {
+                if (pHv->uPartFlags & GIM_HV_PART_FLAGS_DEBUGGING)
+                {
+                    rc = gimHvReadSlowHypercallParamsInOut(pVM, pCtx, fIs64BitMode, &rcHv);
+                    if (   RT_SUCCESS(rc)
+                        && rcHv == GIM_HV_STATUS_SUCCESS)
+                    {
+                        LogRelMax(1, ("GIM: HyperV: Initiated debug data transmission via hypercall\n"));
+                        rc = gimR3HvHypercallPostDebugData(pVM, &rcHv);
+                        if (RT_FAILURE(rc))
+                            LogRelMax(10, ("GIM: HyperV: gimR3HvHypercallPostDebugData failed. rc=%Rrc\n", rc));
+                    }
+                }
+                else
+                    rcHv = GIM_HV_STATUS_ACCESS_DENIED;
+                break;
+            }
+
+            case GIM_HV_HYPERCALL_OP_RESET_DEBUG_SESSION:   /* Non-rep, fast (register IO). */
+            {
+                if (pHv->uPartFlags & GIM_HV_PART_FLAGS_DEBUGGING)
+                {
+                    uint32_t fFlags = 0;
+                    if (!fHyperFast)
+                    {
+                        rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_IN, &rcHv);
+                        if (   RT_SUCCESS(rc)
+                            && rcHv == GIM_HV_STATUS_SUCCESS)
+                        {
+                            PGIMHVDEBUGRESETIN pIn = (PGIMHVDEBUGRESETIN)pHv->pbHypercallIn;
+                            fFlags = pIn->fFlags;
+                        }
+                    }
+                    else
+                    {
+                        rcHv = GIM_HV_STATUS_SUCCESS;
+                        fFlags = fIs64BitMode ? pCtx->rdx : pCtx->ebx;
+                    }
+
+                    /*
+                     * Nothing to flush on the sending side as we don't maintain our own buffers.
+                     */
+                    /** @todo We should probably ask the debug receive thread to flush it's buffer. */
+                    if (rcHv == GIM_HV_STATUS_SUCCESS)
+                    {
+                        if (fFlags)
+                            LogRel(("GIM: HyperV: Resetting debug session via hypercall\n"));
+                        else
+                            rcHv = GIM_HV_STATUS_INVALID_PARAMETER;
+                    }
+                }
+                else
+                    rcHv = GIM_HV_STATUS_ACCESS_DENIED;
+                break;
+            }
+
+            case GIM_HV_HYPERCALL_OP_POST_MESSAGE:      /* Non-rep, memory IO. */
+            {
+                if (pHv->fIsInterfaceVs)
+                {
+                    rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_IN, &rcHv);
+                    if (   RT_SUCCESS(rc)
+                        && rcHv == GIM_HV_STATUS_SUCCESS)
+                    {
+                        PGIMHVPOSTMESSAGEIN pMsgIn = (PGIMHVPOSTMESSAGEIN)pHv->pbHypercallIn;
+                        PCGIMHVCPU          pHvCpu = &pVCpu->gim.s.u.HvCpu;
+                        if (    pMsgIn->uConnectionId  == GIM_HV_VMBUS_MSG_CONNECTION_ID
+                            &&  pMsgIn->enmMessageType == GIMHVMSGTYPE_VMBUS
+                            && !MSR_GIM_HV_SINT_IS_MASKED(pHvCpu->auSintMsrs[GIM_HV_VMBUS_MSG_SINT])
+                            &&  MSR_GIM_HV_SIMP_IS_ENABLED(pHvCpu->uSimpMsr))
+                        {
+                            RTGCPHYS GCPhysSimp = MSR_GIM_HV_SIMP_GPA(pHvCpu->uSimpMsr);
+                            if (PGMPhysIsGCPhysNormal(pVM, GCPhysSimp))
+                            {
+                                /*
+                                 * The VMBus client (guest) expects to see 0xf at offsets 4 and 16 and 1 at offset 0.
+                                 */
+                                GIMHVMSG HvMsg;
+                                RT_ZERO(HvMsg);
+                                HvMsg.MsgHdr.enmMessageType = GIMHVMSGTYPE_VMBUS;
+                                HvMsg.MsgHdr.cbPayload = 0xf;
+                                HvMsg.aPayload[0]      = 0xf;
+                                uint16_t const offMsg = GIM_HV_VMBUS_MSG_SINT * sizeof(GIMHVMSG);
+                                int rc2 = PGMPhysSimpleWriteGCPhys(pVM, GCPhysSimp + offMsg, &HvMsg, sizeof(HvMsg));
+                                if (RT_SUCCESS(rc2))
+                                    LogRel(("GIM: HyperV: SIMP hypercall faking message at %#RGp:%u\n", GCPhysSimp, offMsg));
+                                else
+                                {
+                                    LogRel(("GIM: HyperV: Failed to write SIMP message at %#RGp:%u, rc=%Rrc\n", GCPhysSimp,
+                                            offMsg, rc));
+                                }
+                            }
+                        }
+
+                        /*
+                         * Make the call fail after updating the SIMP, so the guest can go back to using
+                         * the Hyper-V debug MSR interface. Any error code below GIM_HV_STATUS_NOT_ACKNOWLEDGED
+                         * and the guest tries to proceed with initializing VMBus which is totally unnecessary
+                         * for what we're trying to accomplish, i.e. convince guest to use Hyper-V debugging. Also,
+                         * we don't implement other VMBus/SynIC functionality so the guest would #GP and die.
+                         */
+                        rcHv = GIM_HV_STATUS_NOT_ACKNOWLEDGED;
+                    }
+                    else
+                        rcHv = GIM_HV_STATUS_INVALID_PARAMETER;
+                }
+                else
+                    rcHv = GIM_HV_STATUS_ACCESS_DENIED;
+                break;
+            }
+
+            case GIM_HV_EXT_HYPERCALL_OP_QUERY_CAP:              /* Non-rep, extended hypercall. */
+            {
+                if (pHv->uPartFlags & GIM_HV_PART_FLAGS_EXTENDED_HYPERCALLS)
+                {
+                    rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_OUT, &rcHv);
+                    if (   RT_SUCCESS(rc)
+                        && rcHv == GIM_HV_STATUS_SUCCESS)
+                    {
+                        rc = gimR3HvHypercallExtQueryCap(pVM, &rcHv);
+                    }
+                }
+                else
+                {
+                    LogRel(("GIM: HyperV: Denied HvExtCallQueryCapabilities when the feature is not exposed\n"));
+                    rcHv = GIM_HV_STATUS_ACCESS_DENIED;
+                }
+                break;
+            }
+
+            case GIM_HV_EXT_HYPERCALL_OP_GET_BOOT_ZEROED_MEM:    /* Non-rep, extended hypercall. */
+            {
+                if (pHv->uPartFlags & GIM_HV_PART_FLAGS_EXTENDED_HYPERCALLS)
+                {
+                    rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_OUT, &rcHv);
+                    if (   RT_SUCCESS(rc)
+                        && rcHv == GIM_HV_STATUS_SUCCESS)
+                    {
+                        rc = gimR3HvHypercallExtGetBootZeroedMem(pVM, &rcHv);
+                    }
+                }
+                else
+                {
+                    LogRel(("GIM: HyperV: Denied HvExtCallGetBootZeroedMemory when the feature is not exposed\n"));
+                    rcHv = GIM_HV_STATUS_ACCESS_DENIED;
+                }
+                break;
+            }
+
+            default:
+            {
+                LogRel(("GIM: HyperV: Unknown/invalid hypercall opcode %#x (%u)\n", uHyperOp, uHyperOp));
+                rcHv = GIM_HV_STATUS_INVALID_HYPERCALL_CODE;
+                break;
+            }
+        }
+    }
+    else
+        rcHv = GIM_HV_STATUS_INVALID_HYPERCALL_INPUT;
+
+    /*
+     * Update the guest with results of the hypercall.
+     */
+    if (RT_SUCCESS(rc))
+    {
+        if (fIs64BitMode)
+            pCtx->rax = (cHyperRepsDone << 32) | rcHv;
+        else
+        {
+            pCtx->edx = cHyperRepsDone;
+            pCtx->eax = rcHv;
+        }
+    }
+
+    return rc;
+#endif
+}
+
+
+/**
+ * Returns a pointer to the MMIO2 regions supported by Hyper-V.
+ *
+ * @returns Pointer to an array of MMIO2 regions.
+ * @param   pVM         The cross context VM structure.
+ * @param   pcRegions   Where to store the number of regions in the array.
+ */
+VMM_INT_DECL(PGIMMMIO2REGION) gimHvGetMmio2Regions(PVM pVM, uint32_t *pcRegions)
+{
+    Assert(GIMIsEnabled(pVM));
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+
+    AssertCompile(RT_ELEMENTS(pHv->aMmio2Regions) <= 8);
+    *pcRegions = RT_ELEMENTS(pHv->aMmio2Regions);
+    return pHv->aMmio2Regions;
+}
+
+
+/**
+ * Returns whether the guest has configured and enabled the use of Hyper-V's
+ * hypercall interface.
+ *
+ * @returns true if hypercalls are enabled, false otherwise.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(bool) gimHvAreHypercallsEnabled(PCVM pVM)
+{
+    return RT_BOOL(pVM->gim.s.u.Hv.u64GuestOsIdMsr != 0);
+}
+
+
+/**
+ * Returns whether the guest has configured and enabled the use of Hyper-V's
+ * paravirtualized TSC.
+ *
+ * @returns true if paravirt. TSC is enabled, false otherwise.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(bool) gimHvIsParavirtTscEnabled(PVM pVM)
+{
+    return MSR_GIM_HV_REF_TSC_IS_ENABLED(pVM->gim.s.u.Hv.u64TscPageMsr);
+}
+
+
+#ifdef IN_RING3
+/**
+ * Gets the descriptive OS ID variant as identified via the
+ * MSR_GIM_HV_GUEST_OS_ID MSR.
+ *
+ * @returns The name.
+ * @param   uGuestOsIdMsr     The MSR_GIM_HV_GUEST_OS_ID MSR.
+ */
+static const char *gimHvGetGuestOsIdVariantName(uint64_t uGuestOsIdMsr)
+{
+    /* Refer the Hyper-V spec, section 3.6 "Reporting the Guest OS Identity". */
+    uint32_t uVendor = MSR_GIM_HV_GUEST_OS_ID_VENDOR(uGuestOsIdMsr);
+    if (uVendor == 1 /* Microsoft */)
+    {
+        uint32_t uOsVariant = MSR_GIM_HV_GUEST_OS_ID_OS_VARIANT(uGuestOsIdMsr);
+        switch (uOsVariant)
+        {
+            case 0:  return "Undefined";
+            case 1:  return "MS-DOS";
+            case 2:  return "Windows 3.x";
+            case 3:  return "Windows 9x";
+            case 4:  return "Windows NT or derivative";
+            case 5:  return "Windows CE";
+            default: return "Unknown";
+        }
+    }
+    return "Unknown";
+}
+#endif
+
+/**
+ * Gets the time reference count for the current VM.
+ *
+ * @returns The time reference count.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(uint64_t) gimHvGetTimeRefCount(PVMCPUCC pVCpu)
+{
+    /* Hyper-V reports the time in 100 ns units (10 MHz). */
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+    PCGIMHV pHv = &pVCpu->CTX_SUFF(pVM)->gim.s.u.Hv;
+    uint64_t const u64Tsc        = TMCpuTickGet(pVCpu);     /** @todo should we be passing VCPU0 always? */
+    uint64_t const u64TscHz      = pHv->cTscTicksPerSecond;
+    uint64_t const u64Tsc100NS   = u64TscHz / UINT64_C(10000000); /* 100 ns */
+    uint64_t const uTimeRefCount = (u64Tsc / u64Tsc100NS);
+    return uTimeRefCount;
+}
+
+
+/**
+ * Starts the synthetic timer.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pHvStimer   Pointer to the Hyper-V synthetic timer.
+ *
+ * @remarks Caller needs to hold the timer critical section.
+ * @thread  Any.
+ */
+VMM_INT_DECL(void) gimHvStartStimer(PVMCPUCC pVCpu, PCGIMHVSTIMER pHvStimer)
+{
+    PTMTIMER pTimer = pHvStimer->CTX_SUFF(pTimer);
+    Assert(TMTimerIsLockOwner(pTimer));
+
+    uint64_t const uTimerCount = pHvStimer->uStimerCountMsr;
+    if (uTimerCount)
+    {
+        uint64_t const uTimerCountNS = uTimerCount * 100;
+
+        /* For periodic timers, 'uTimerCountNS' represents the relative interval. */
+        if (MSR_GIM_HV_STIMER_IS_PERIODIC(pHvStimer->uStimerConfigMsr))
+        {
+            TMTimerSetNano(pTimer, uTimerCountNS);
+            LogFlow(("GIM%u: HyperV: Started relative periodic STIMER%u with uTimerCountNS=%RU64\n", pVCpu->idCpu,
+                     pHvStimer->idxStimer, uTimerCountNS));
+        }
+        else
+        {
+            /* For one-shot timers, 'uTimerCountNS' represents an absolute expiration wrt to Hyper-V reference time,
+               we convert it to a relative time and program the timer. */
+            uint64_t const uCurRefTimeNS = gimHvGetTimeRefCount(pVCpu) * 100;
+            if (uTimerCountNS > uCurRefTimeNS)
+            {
+                uint64_t const uRelativeNS = uTimerCountNS - uCurRefTimeNS;
+                TMTimerSetNano(pTimer, uRelativeNS);
+                LogFlow(("GIM%u: HyperV: Started one-shot relative STIMER%u with uRelativeNS=%RU64\n", pVCpu->idCpu,
+                         pHvStimer->idxStimer, uRelativeNS));
+            }
+        }
+        /** @todo frequency hinting? */
+    }
+}
+
+
+/**
+ * Stops the synthetic timer for the given VCPU.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pHvStimer   Pointer to the Hyper-V synthetic timer.
+ *
+ * @remarks Caller needs to the hold the timer critical section.
+ * @thread  EMT(pVCpu).
+ */
+static void gimHvStopStimer(PVMCPUCC pVCpu, PGIMHVSTIMER pHvStimer)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+    RT_NOREF(pVCpu);
+
+    PTMTIMER pTimer = pHvStimer->CTX_SUFF(pTimer);
+    Assert(TMTimerIsLockOwner(pTimer));
+    RT_NOREF(pTimer);
+
+    if (TMTimerIsActive(pHvStimer->CTX_SUFF(pTimer)))
+        TMTimerStop(pHvStimer->CTX_SUFF(pTimer));
+}
+
+
+/**
+ * MSR read handler for Hyper-V.
+ *
+ * @returns Strict VBox status code like CPUMQueryGuestMsr().
+ * @retval  VINF_CPUM_R3_MSR_READ
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR being read.
+ * @param   pRange      The range this MSR belongs to.
+ * @param   puValue     Where to store the MSR value read.
+ *
+ * @thread  EMT.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimHvReadMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    NOREF(pRange);
+    PVMCC   pVM = pVCpu->CTX_SUFF(pVM);
+    PCGIMHV pHv = &pVM->gim.s.u.Hv;
+
+    switch (idMsr)
+    {
+        case MSR_GIM_HV_TIME_REF_COUNT:
+            *puValue = gimHvGetTimeRefCount(pVCpu);
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_VP_INDEX:
+            *puValue = pVCpu->idCpu;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_TPR:
+            *puValue = APICHvGetTpr(pVCpu);
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_ICR:
+            *puValue = APICHvGetIcr(pVCpu);
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_GUEST_OS_ID:
+            *puValue = pHv->u64GuestOsIdMsr;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_HYPERCALL:
+            *puValue = pHv->u64HypercallMsr;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_REF_TSC:
+            *puValue = pHv->u64TscPageMsr;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_TSC_FREQ:
+            *puValue = TMCpuTicksPerSecond(pVM);
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_APIC_FREQ:
+        {
+            int rc = APICGetTimerFreq(pVM, puValue);
+            if (RT_FAILURE(rc))
+                return VERR_CPUM_RAISE_GP_0;
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_SYNTH_DEBUG_STATUS:
+            *puValue = pHv->uDbgStatusMsr;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_SINT0:   case MSR_GIM_HV_SINT1:   case MSR_GIM_HV_SINT2:   case MSR_GIM_HV_SINT3:
+        case MSR_GIM_HV_SINT4:   case MSR_GIM_HV_SINT5:   case MSR_GIM_HV_SINT6:   case MSR_GIM_HV_SINT7:
+        case MSR_GIM_HV_SINT8:   case MSR_GIM_HV_SINT9:   case MSR_GIM_HV_SINT10:  case MSR_GIM_HV_SINT11:
+        case MSR_GIM_HV_SINT12:  case MSR_GIM_HV_SINT13:  case MSR_GIM_HV_SINT14:  case MSR_GIM_HV_SINT15:
+        {
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            *puValue = pHvCpu->auSintMsrs[idMsr - MSR_GIM_HV_SINT0];
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_STIMER0_CONFIG:
+        case MSR_GIM_HV_STIMER1_CONFIG:
+        case MSR_GIM_HV_STIMER2_CONFIG:
+        case MSR_GIM_HV_STIMER3_CONFIG:
+        {
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            uint8_t const idxStimer  = (idMsr - MSR_GIM_HV_STIMER0_CONFIG) >> 1;
+            PCGIMHVSTIMER pcHvStimer = &pHvCpu->aStimers[idxStimer];
+            *puValue = pcHvStimer->uStimerConfigMsr;
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_STIMER0_COUNT:
+        case MSR_GIM_HV_STIMER1_COUNT:
+        case MSR_GIM_HV_STIMER2_COUNT:
+        case MSR_GIM_HV_STIMER3_COUNT:
+        {
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            uint8_t const idxStimer  = (idMsr - MSR_GIM_HV_STIMER0_COUNT) >> 1;
+            PCGIMHVSTIMER pcHvStimer = &pHvCpu->aStimers[idxStimer];
+            *puValue = pcHvStimer->uStimerCountMsr;
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_EOM:
+        {
+            *puValue = 0;
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_SCONTROL:
+        {
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            *puValue = pHvCpu->uSControlMsr;
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_SIMP:
+        {
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            *puValue = pHvCpu->uSimpMsr;
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_SVERSION:
+            *puValue = GIM_HV_SVERSION;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_RESET:
+            *puValue = 0;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_CRASH_CTL:
+            *puValue = pHv->uCrashCtlMsr;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_CRASH_P0: *puValue = pHv->uCrashP0Msr;   return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P1: *puValue = pHv->uCrashP1Msr;   return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P2: *puValue = pHv->uCrashP2Msr;   return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P3: *puValue = pHv->uCrashP3Msr;   return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P4: *puValue = pHv->uCrashP4Msr;   return VINF_SUCCESS;
+
+        case MSR_GIM_HV_DEBUG_OPTIONS_MSR:
+        {
+            if (pHv->fIsVendorMsHv)
+            {
+#ifndef IN_RING3
+                return VINF_CPUM_R3_MSR_READ;
+#else
+                LogRelMax(1, ("GIM: HyperV: Guest querying debug options, suggesting %s interface\n",
+                              pHv->fDbgHypercallInterface ? "hypercall" : "MSR"));
+                *puValue = pHv->fDbgHypercallInterface ? GIM_HV_DEBUG_OPTIONS_USE_HYPERCALLS : 0;
+                return VINF_SUCCESS;
+#endif
+            }
+            break;
+        }
+
+        /* Write-only MSRs: */
+        case MSR_GIM_HV_EOI:
+        /* Reserved/unknown MSRs: */
+        default:
+        {
+#ifdef IN_RING3
+            static uint32_t s_cTimes = 0;
+            if (s_cTimes++ < 20)
+                LogRel(("GIM: HyperV: Unknown/invalid RdMsr (%#x) -> #GP(0)\n", idMsr));
+            LogFunc(("Unknown/invalid RdMsr (%#RX32) -> #GP(0)\n", idMsr));
+            break;
+#else
+            return VINF_CPUM_R3_MSR_READ;
+#endif
+        }
+    }
+
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * MSR write handler for Hyper-V.
+ *
+ * @returns Strict VBox status code like CPUMSetGuestMsr().
+ * @retval  VINF_CPUM_R3_MSR_WRITE
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR being written.
+ * @param   pRange      The range this MSR belongs to.
+ * @param   uRawValue   The raw value with the ignored bits not masked.
+ *
+ * @thread  EMT.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimHvWriteMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uRawValue)
+{
+    NOREF(pRange);
+    PVM    pVM = pVCpu->CTX_SUFF(pVM);
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+
+    switch (idMsr)
+    {
+        case MSR_GIM_HV_TPR:
+            return APICHvSetTpr(pVCpu, uRawValue);
+
+        case MSR_GIM_HV_EOI:
+            return APICHvSetEoi(pVCpu, uRawValue);
+
+        case MSR_GIM_HV_ICR:
+            return APICHvSetIcr(pVCpu, uRawValue);
+
+        case MSR_GIM_HV_GUEST_OS_ID:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            /* Disable the hypercall-page and hypercalls if 0 is written to this MSR. */
+            if (!uRawValue)
+            {
+                if (MSR_GIM_HV_HYPERCALL_PAGE_IS_ENABLED(pHv->u64HypercallMsr))
+                {
+                    gimR3HvDisableHypercallPage(pVM);
+                    pHv->u64HypercallMsr &= ~MSR_GIM_HV_HYPERCALL_PAGE_ENABLE;
+                    LogRel(("GIM: HyperV: Hypercall page disabled via Guest OS ID MSR\n"));
+                }
+            }
+            else
+            {
+                LogRel(("GIM: HyperV: Guest OS reported ID %#RX64\n", uRawValue));
+                LogRel(("GIM: HyperV: Open-source=%RTbool Vendor=%#x OS=%#x (%s) Major=%u Minor=%u ServicePack=%u Build=%u\n",
+                        MSR_GIM_HV_GUEST_OS_ID_IS_OPENSOURCE(uRawValue),   MSR_GIM_HV_GUEST_OS_ID_VENDOR(uRawValue),
+                        MSR_GIM_HV_GUEST_OS_ID_OS_VARIANT(uRawValue),      gimHvGetGuestOsIdVariantName(uRawValue),
+                        MSR_GIM_HV_GUEST_OS_ID_MAJOR_VERSION(uRawValue),   MSR_GIM_HV_GUEST_OS_ID_MINOR_VERSION(uRawValue),
+                        MSR_GIM_HV_GUEST_OS_ID_SERVICE_VERSION(uRawValue), MSR_GIM_HV_GUEST_OS_ID_BUILD(uRawValue)));
+
+                /* Update the CPUID leaf, see Hyper-V spec. "Microsoft Hypervisor CPUID Leaves". */
+                CPUMCPUIDLEAF HyperLeaf;
+                RT_ZERO(HyperLeaf);
+                HyperLeaf.uLeaf = UINT32_C(0x40000002);
+                HyperLeaf.uEax  = MSR_GIM_HV_GUEST_OS_ID_BUILD(uRawValue);
+                HyperLeaf.uEbx  =  MSR_GIM_HV_GUEST_OS_ID_MINOR_VERSION(uRawValue)
+                                | (MSR_GIM_HV_GUEST_OS_ID_MAJOR_VERSION(uRawValue) << 16);
+                HyperLeaf.uEcx  = MSR_GIM_HV_GUEST_OS_ID_SERVICE_VERSION(uRawValue);
+                HyperLeaf.uEdx  =  MSR_GIM_HV_GUEST_OS_ID_SERVICE_VERSION(uRawValue)
+                                | (MSR_GIM_HV_GUEST_OS_ID_BUILD(uRawValue) << 24);
+                int rc2 = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+                AssertRC(rc2);
+            }
+
+            pHv->u64GuestOsIdMsr = uRawValue;
+
+            /*
+             * Update EM on hypercall instruction enabled state.
+             */
+            if (uRawValue)
+                for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                    EMSetHypercallInstructionsEnabled(pVM->CTX_SUFF(apCpus)[idCpu], true);
+            else
+                for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                    EMSetHypercallInstructionsEnabled(pVM->CTX_SUFF(apCpus)[idCpu], false);
+
+            return VINF_SUCCESS;
+#endif /* IN_RING3 */
+        }
+
+        case MSR_GIM_HV_HYPERCALL:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            /** @todo There is/was a problem with hypercalls for FreeBSD 10.1 guests,
+             *  see @bugref{7270#c116}. */
+            /* First, update all but the hypercall page enable bit. */
+            pHv->u64HypercallMsr = (uRawValue & ~MSR_GIM_HV_HYPERCALL_PAGE_ENABLE);
+
+            /* Hypercall page can only be enabled when the guest has enabled hypercalls. */
+            bool fEnable = MSR_GIM_HV_HYPERCALL_PAGE_IS_ENABLED(uRawValue);
+            if (   fEnable
+                && !gimHvAreHypercallsEnabled(pVM))
+            {
+                return VINF_SUCCESS;
+            }
+
+            /* Is the guest disabling the hypercall-page? Allow it regardless of the Guest-OS Id Msr. */
+            if (!fEnable)
+            {
+                gimR3HvDisableHypercallPage(pVM);
+                pHv->u64HypercallMsr = uRawValue;
+                return VINF_SUCCESS;
+            }
+
+            /* Enable the hypercall-page. */
+            RTGCPHYS GCPhysHypercallPage = MSR_GIM_HV_HYPERCALL_GUEST_PFN(uRawValue) << PAGE_SHIFT;
+            int rc = gimR3HvEnableHypercallPage(pVM, GCPhysHypercallPage);
+            if (RT_SUCCESS(rc))
+            {
+                pHv->u64HypercallMsr = uRawValue;
+                return VINF_SUCCESS;
+            }
+
+            return VERR_CPUM_RAISE_GP_0;
+#endif
+        }
+
+        case MSR_GIM_HV_REF_TSC:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else  /* IN_RING3 */
+            /* First, update all but the TSC page enable bit. */
+            pHv->u64TscPageMsr = (uRawValue & ~MSR_GIM_HV_REF_TSC_ENABLE);
+
+            /* Is the guest disabling the TSC page? */
+            bool fEnable = MSR_GIM_HV_REF_TSC_IS_ENABLED(uRawValue);
+            if (!fEnable)
+            {
+                gimR3HvDisableTscPage(pVM);
+                pHv->u64TscPageMsr = uRawValue;
+                return VINF_SUCCESS;
+            }
+
+            /* Enable the TSC page. */
+            RTGCPHYS GCPhysTscPage = MSR_GIM_HV_REF_TSC_GUEST_PFN(uRawValue) << PAGE_SHIFT;
+            int rc = gimR3HvEnableTscPage(pVM, GCPhysTscPage, false /* fUseThisTscSequence */, 0 /* uTscSequence */);
+            if (RT_SUCCESS(rc))
+            {
+                pHv->u64TscPageMsr = uRawValue;
+                return VINF_SUCCESS;
+            }
+
+            return VERR_CPUM_RAISE_GP_0;
+#endif /* IN_RING3 */
+        }
+
+        case MSR_GIM_HV_APIC_ASSIST_PAGE:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else  /* IN_RING3 */
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            pHvCpu->uApicAssistPageMsr = uRawValue;
+
+            if (MSR_GIM_HV_APICASSIST_PAGE_IS_ENABLED(uRawValue))
+            {
+                RTGCPHYS GCPhysApicAssistPage = MSR_GIM_HV_APICASSIST_GUEST_PFN(uRawValue) << PAGE_SHIFT;
+                if (PGMPhysIsGCPhysNormal(pVM, GCPhysApicAssistPage))
+                {
+                    int rc = gimR3HvEnableApicAssistPage(pVCpu, GCPhysApicAssistPage);
+                    if (RT_SUCCESS(rc))
+                    {
+                        pHvCpu->uApicAssistPageMsr = uRawValue;
+                        return VINF_SUCCESS;
+                    }
+                }
+                else
+                {
+                    LogRelMax(5, ("GIM%u: HyperV: APIC-assist page address %#RGp invalid!\n", pVCpu->idCpu,
+                                  GCPhysApicAssistPage));
+                }
+            }
+            else
+                gimR3HvDisableApicAssistPage(pVCpu);
+
+            return VERR_CPUM_RAISE_GP_0;
+#endif /* IN_RING3 */
+        }
+
+        case MSR_GIM_HV_RESET:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            if (MSR_GIM_HV_RESET_IS_ENABLED(uRawValue))
+            {
+                LogRel(("GIM: HyperV: Reset initiated through MSR\n"));
+                int rc = PDMDevHlpVMReset(pVM->gim.s.pDevInsR3, PDMVMRESET_F_GIM);
+                AssertRC(rc); /* Note! Not allowed to return VINF_EM_RESET / VINF_EM_HALT here, so ignore them. */
+            }
+            /* else: Ignore writes to other bits. */
+            return VINF_SUCCESS;
+#endif /* IN_RING3 */
+        }
+
+        case MSR_GIM_HV_CRASH_CTL:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            if (uRawValue & MSR_GIM_HV_CRASH_CTL_NOTIFY)
+            {
+                LogRel(("GIM: HyperV: Guest indicates a fatal condition! P0=%#RX64 P1=%#RX64 P2=%#RX64 P3=%#RX64 P4=%#RX64\n",
+                        pHv->uCrashP0Msr, pHv->uCrashP1Msr, pHv->uCrashP2Msr, pHv->uCrashP3Msr, pHv->uCrashP4Msr));
+                DBGFR3ReportBugCheck(pVM, pVCpu, DBGFEVENT_BSOD_MSR, pHv->uCrashP0Msr, pHv->uCrashP1Msr,
+                                     pHv->uCrashP2Msr, pHv->uCrashP3Msr, pHv->uCrashP4Msr);
+                /* (Do not try pass VINF_EM_DBG_EVENT, doesn't work from here!) */
+            }
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_SYNTH_DEBUG_SEND_BUFFER:
+        {
+            if (!pHv->fDbgEnabled)
+                return VERR_CPUM_RAISE_GP_0;
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            RTGCPHYS GCPhysBuffer    = (RTGCPHYS)uRawValue;
+            pHv->uDbgSendBufferMsr = GCPhysBuffer;
+            if (PGMPhysIsGCPhysNormal(pVM, GCPhysBuffer))
+                LogRel(("GIM: HyperV: Set up debug send buffer at %#RGp\n", GCPhysBuffer));
+            else
+                LogRel(("GIM: HyperV: Destroyed debug send buffer\n"));
+            pHv->uDbgSendBufferMsr = uRawValue;
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_SYNTH_DEBUG_RECEIVE_BUFFER:
+        {
+            if (!pHv->fDbgEnabled)
+                return VERR_CPUM_RAISE_GP_0;
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            RTGCPHYS GCPhysBuffer  = (RTGCPHYS)uRawValue;
+            pHv->uDbgRecvBufferMsr = GCPhysBuffer;
+            if (PGMPhysIsGCPhysNormal(pVM, GCPhysBuffer))
+                LogRel(("GIM: HyperV: Set up debug receive buffer at %#RGp\n", GCPhysBuffer));
+            else
+                LogRel(("GIM: HyperV: Destroyed debug receive buffer\n"));
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_SYNTH_DEBUG_PENDING_BUFFER:
+        {
+            if (!pHv->fDbgEnabled)
+                return VERR_CPUM_RAISE_GP_0;
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            RTGCPHYS GCPhysBuffer      = (RTGCPHYS)uRawValue;
+            pHv->uDbgPendingBufferMsr  = GCPhysBuffer;
+            if (PGMPhysIsGCPhysNormal(pVM, GCPhysBuffer))
+                LogRel(("GIM: HyperV: Set up debug pending buffer at %#RGp\n", uRawValue));
+            else
+                LogRel(("GIM: HyperV: Destroyed debug pending buffer\n"));
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_SYNTH_DEBUG_CONTROL:
+        {
+            if (!pHv->fDbgEnabled)
+                return VERR_CPUM_RAISE_GP_0;
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            if (   MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_WRITE(uRawValue)
+                && MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_READ(uRawValue))
+            {
+                LogRel(("GIM: HyperV: Requesting both read and write through debug control MSR -> #GP(0)\n"));
+                return VERR_CPUM_RAISE_GP_0;
+            }
+
+            if (MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_WRITE(uRawValue))
+            {
+                uint32_t cbWrite = MSR_GIM_HV_SYNTH_DEBUG_CONTROL_W_LEN(uRawValue);
+                if (   cbWrite > 0
+                    && cbWrite < GIM_HV_PAGE_SIZE)
+                {
+                    if (PGMPhysIsGCPhysNormal(pVM, (RTGCPHYS)pHv->uDbgSendBufferMsr))
+                    {
+                        Assert(pHv->pvDbgBuffer);
+                        int rc = PGMPhysSimpleReadGCPhys(pVM, pHv->pvDbgBuffer, (RTGCPHYS)pHv->uDbgSendBufferMsr, cbWrite);
+                        if (RT_SUCCESS(rc))
+                        {
+                            LogRelMax(1, ("GIM: HyperV: Initiated debug data transmission via MSR\n"));
+                            uint32_t cbWritten = 0;
+                            rc = gimR3HvDebugWrite(pVM, pHv->pvDbgBuffer, cbWrite, &cbWritten, false /*fUdpPkt*/);
+                            if (   RT_SUCCESS(rc)
+                                && cbWrite == cbWritten)
+                                pHv->uDbgStatusMsr = MSR_GIM_HV_SYNTH_DEBUG_STATUS_W_SUCCESS;
+                            else
+                                pHv->uDbgStatusMsr = 0;
+                        }
+                        else
+                            LogRelMax(5, ("GIM: HyperV: Failed to read debug send buffer at %#RGp, rc=%Rrc\n",
+                                          (RTGCPHYS)pHv->uDbgSendBufferMsr, rc));
+                    }
+                    else
+                        LogRelMax(5, ("GIM: HyperV: Debug send buffer address %#RGp invalid! Ignoring debug write!\n",
+                                      (RTGCPHYS)pHv->uDbgSendBufferMsr));
+                }
+                else
+                    LogRelMax(5, ("GIM: HyperV: Invalid write size %u specified in MSR, ignoring debug write!\n",
+                                  MSR_GIM_HV_SYNTH_DEBUG_CONTROL_W_LEN(uRawValue)));
+            }
+            else if (MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_READ(uRawValue))
+            {
+                if (PGMPhysIsGCPhysNormal(pVM, (RTGCPHYS)pHv->uDbgRecvBufferMsr))
+                {
+                    LogRelMax(1, ("GIM: HyperV: Initiated debug data reception via MSR\n"));
+                    uint32_t cbReallyRead;
+                    Assert(pHv->pvDbgBuffer);
+                    int rc = gimR3HvDebugRead(pVM, pHv->pvDbgBuffer, PAGE_SIZE, PAGE_SIZE, &cbReallyRead, 0, false /*fUdpPkt*/);
+                    if (   RT_SUCCESS(rc)
+                        && cbReallyRead > 0)
+                    {
+                        rc = PGMPhysSimpleWriteGCPhys(pVM, (RTGCPHYS)pHv->uDbgRecvBufferMsr, pHv->pvDbgBuffer, cbReallyRead);
+                        if (RT_SUCCESS(rc))
+                        {
+                            pHv->uDbgStatusMsr  = ((uint16_t)cbReallyRead) << 16;
+                            pHv->uDbgStatusMsr |= MSR_GIM_HV_SYNTH_DEBUG_STATUS_R_SUCCESS;
+                        }
+                        else
+                        {
+                            pHv->uDbgStatusMsr = 0;
+                            LogRelMax(5, ("GIM: HyperV: PGMPhysSimpleWriteGCPhys failed. rc=%Rrc\n", rc));
+                        }
+                    }
+                    else
+                        pHv->uDbgStatusMsr = 0;
+                }
+                else
+                {
+                    LogRelMax(5, ("GIM: HyperV: Debug receive buffer address %#RGp invalid! Ignoring debug read!\n",
+                                  (RTGCPHYS)pHv->uDbgRecvBufferMsr));
+                }
+            }
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_SINT0:    case MSR_GIM_HV_SINT1:    case MSR_GIM_HV_SINT2:    case MSR_GIM_HV_SINT3:
+        case MSR_GIM_HV_SINT4:    case MSR_GIM_HV_SINT5:    case MSR_GIM_HV_SINT6:    case MSR_GIM_HV_SINT7:
+        case MSR_GIM_HV_SINT8:    case MSR_GIM_HV_SINT9:    case MSR_GIM_HV_SINT10:   case MSR_GIM_HV_SINT11:
+        case MSR_GIM_HV_SINT12:   case MSR_GIM_HV_SINT13:   case MSR_GIM_HV_SINT14:   case MSR_GIM_HV_SINT15:
+        {
+            uint8_t      uVector    = MSR_GIM_HV_SINT_GET_VECTOR(uRawValue);
+            bool const   fVMBusMsg  = RT_BOOL(idMsr == GIM_HV_VMBUS_MSG_SINT);
+            size_t const idxSintMsr = idMsr - MSR_GIM_HV_SINT0;
+            const char  *pszDesc    = fVMBusMsg ? "VMBus Message" : "Generic";
+            if (uVector < GIM_HV_SINT_VECTOR_VALID_MIN)
+            {
+                LogRel(("GIM%u: HyperV: Programmed an invalid vector in SINT%u (%s), uVector=%u -> #GP(0)\n", pVCpu->idCpu,
+                        idxSintMsr, pszDesc, uVector));
+                return VERR_CPUM_RAISE_GP_0;
+            }
+
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            pHvCpu->auSintMsrs[idxSintMsr] = uRawValue;
+            if (fVMBusMsg)
+            {
+                if (MSR_GIM_HV_SINT_IS_MASKED(uRawValue))
+                    Log(("GIM%u: HyperV: Masked SINT%u (%s)\n", pVCpu->idCpu, idxSintMsr, pszDesc));
+                else
+                    Log(("GIM%u: HyperV: Unmasked SINT%u (%s), uVector=%u\n", pVCpu->idCpu, idxSintMsr, pszDesc, uVector));
+            }
+            Log(("GIM%u: HyperV: Written SINT%u=%#RX64\n", pVCpu->idCpu, idxSintMsr, uRawValue));
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_SCONTROL:
+        {
+#ifndef IN_RING3
+            /** @todo make this RZ later? */
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            pHvCpu->uSControlMsr = uRawValue;
+            if (MSR_GIM_HV_SCONTROL_IS_ENABLED(uRawValue))
+                LogRel(("GIM%u: HyperV: Synthetic interrupt control enabled\n", pVCpu->idCpu));
+            else
+                LogRel(("GIM%u: HyperV: Synthetic interrupt control disabled\n", pVCpu->idCpu));
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_STIMER0_CONFIG:
+        case MSR_GIM_HV_STIMER1_CONFIG:
+        case MSR_GIM_HV_STIMER2_CONFIG:
+        case MSR_GIM_HV_STIMER3_CONFIG:
+        {
+            PGIMHVCPU     pHvCpu    = &pVCpu->gim.s.u.HvCpu;
+            uint8_t const idxStimer = (idMsr - MSR_GIM_HV_STIMER0_CONFIG) >> 1;
+
+            /* Validate the writable bits. */
+            if (RT_LIKELY(!(uRawValue & ~MSR_GIM_HV_STIMER_RW_VALID)))
+            {
+                Assert(idxStimer < RT_ELEMENTS(pHvCpu->aStimers));
+                PGIMHVSTIMER pHvStimer = &pHvCpu->aStimers[idxStimer];
+                PTMTIMER     pTimer    = pHvStimer->CTX_SUFF(pTimer);
+
+                /* Lock to prevent concurrent access from the timer callback. */
+                int rc = TMTimerLock(pTimer, VERR_IGNORED);
+                if (rc == VINF_SUCCESS)
+                {
+                    /* Update the MSR value. */
+                    pHvStimer->uStimerConfigMsr = uRawValue;
+                    Log(("GIM%u: HyperV: Set STIMER_CONFIG%u=%#RX64\n", pVCpu->idCpu, idxStimer, uRawValue));
+
+                    /* Process the MSR bits. */
+                    if (   !MSR_GIM_HV_STIMER_GET_SINTX(uRawValue)   /* Writing SINTx as 0 causes the timer to be disabled. */
+                        || !MSR_GIM_HV_STIMER_IS_ENABLED(uRawValue))
+                    {
+                        pHvStimer->uStimerConfigMsr &= ~MSR_GIM_HV_STIMER_ENABLE;
+                        gimHvStopStimer(pVCpu, pHvStimer);
+                        Log(("GIM%u: HyperV: Disabled STIMER_CONFIG%u\n", pVCpu->idCpu, idxStimer));
+                    }
+                    else if (MSR_GIM_HV_STIMER_IS_ENABLED(uRawValue))
+                    {
+                        /* Auto-enable implies writing to the STIMERx_COUNT MSR is what starts the timer. */
+                        if (!MSR_GIM_HV_STIMER_IS_AUTO_ENABLED(uRawValue))
+                        {
+                            if (!TMTimerIsActive(pHvStimer->CTX_SUFF(pTimer)))
+                            {
+                                gimHvStartStimer(pVCpu, pHvStimer);
+                                Log(("GIM%u: HyperV: Started STIMER%u\n", pVCpu->idCpu, idxStimer));
+                            }
+                            else
+                            {
+                                /*
+                                 * Enabling a timer that's already enabled is undefined behaviour,
+                                 * see Hyper-V spec. 15.3.1 "Synthetic Timer Configuration Register".
+                                 *
+                                 * Our implementation just re-starts the timer. Guests that comform to
+                                 * the Hyper-V specs. should not be doing this anyway.
+                                 */
+                                AssertFailed();
+                                gimHvStopStimer(pVCpu, pHvStimer);
+                                gimHvStartStimer(pVCpu, pHvStimer);
+                            }
+                        }
+                    }
+
+                    TMTimerUnlock(pTimer);
+                }
+                return rc;
+            }
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            LogRel(("GIM%u: HyperV: Setting reserved bits of STIMER%u MSR (uRawValue=%#RX64) -> #GP(0)\n", pVCpu->idCpu,
+                    idxStimer, uRawValue));
+            return VERR_CPUM_RAISE_GP_0;
+#endif
+        }
+
+        case MSR_GIM_HV_STIMER0_COUNT:
+        case MSR_GIM_HV_STIMER1_COUNT:
+        case MSR_GIM_HV_STIMER2_COUNT:
+        case MSR_GIM_HV_STIMER3_COUNT:
+        {
+            PGIMHVCPU     pHvCpu    = &pVCpu->gim.s.u.HvCpu;
+            uint8_t const idxStimer = (idMsr - MSR_GIM_HV_STIMER0_CONFIG) >> 1;
+            Assert(idxStimer < RT_ELEMENTS(pHvCpu->aStimers));
+            PGIMHVSTIMER  pHvStimer = &pHvCpu->aStimers[idxStimer];
+            int const     rcBusy    = VINF_CPUM_R3_MSR_WRITE;
+
+            /*
+             * Writing zero to this MSR disables the timer regardless of whether the auto-enable
+             * flag is set in the config MSR corresponding to the timer.
+             */
+            if (!uRawValue)
+            {
+                gimHvStopStimer(pVCpu, pHvStimer);
+                pHvStimer->uStimerCountMsr = 0;
+                Log(("GIM%u: HyperV: Set STIMER_COUNT%u=%RU64, stopped timer\n", pVCpu->idCpu, idxStimer, uRawValue));
+                return VINF_SUCCESS;
+            }
+
+            /*
+             * Concurrent writes to the config. MSR can't happen as it's serialized by way
+             * of being done on the same EMT as this.
+             */
+            if (MSR_GIM_HV_STIMER_IS_AUTO_ENABLED(pHvStimer->uStimerConfigMsr))
+            {
+                PTMTIMER pTimer = pHvStimer->CTX_SUFF(pTimer);
+                int rc = TMTimerLock(pTimer, rcBusy);
+                if (rc == VINF_SUCCESS)
+                {
+                    pHvStimer->uStimerCountMsr = uRawValue;
+                    gimHvStartStimer(pVCpu, pHvStimer);
+                    TMTimerUnlock(pTimer);
+                    Log(("GIM%u: HyperV: Set STIMER_COUNT%u=%RU64 %RU64 msec, auto-started timer\n", pVCpu->idCpu, idxStimer,
+                         uRawValue, (uRawValue * 100) / RT_NS_1MS_64));
+                }
+                return rc;
+            }
+
+            /* Simple update of the counter without any timer start/stop side-effects. */
+            pHvStimer->uStimerCountMsr = uRawValue;
+            Log(("GIM%u: HyperV: Set STIMER_COUNT%u=%RU64\n", pVCpu->idCpu, idxStimer, uRawValue));
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_EOM:
+        {
+            /** @todo implement EOM. */
+            Log(("GIM%u: HyperV: EOM\n", pVCpu->idCpu));
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_SIEFP:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            pHvCpu->uSiefpMsr = uRawValue;
+            if (MSR_GIM_HV_SIEF_PAGE_IS_ENABLED(uRawValue))
+            {
+                RTGCPHYS GCPhysSiefPage = MSR_GIM_HV_SIEF_GUEST_PFN(uRawValue) << PAGE_SHIFT;
+                if (PGMPhysIsGCPhysNormal(pVM, GCPhysSiefPage))
+                {
+                    int rc = gimR3HvEnableSiefPage(pVCpu, GCPhysSiefPage);
+                    if (RT_SUCCESS(rc))
+                    {
+                        LogRel(("GIM%u: HyperV: Enabled synthetic interrupt event flags page at %#RGp\n", pVCpu->idCpu,
+                                GCPhysSiefPage));
+                        /** @todo SIEF setup. */
+                        return VINF_SUCCESS;
+                    }
+                }
+                else
+                    LogRelMax(5, ("GIM%u: HyperV: SIEF page address %#RGp invalid!\n", pVCpu->idCpu, GCPhysSiefPage));
+            }
+            else
+                gimR3HvDisableSiefPage(pVCpu);
+
+            return VERR_CPUM_RAISE_GP_0;
+#endif
+            break;
+        }
+
+        case MSR_GIM_HV_SIMP:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            pHvCpu->uSimpMsr = uRawValue;
+            if (MSR_GIM_HV_SIMP_IS_ENABLED(uRawValue))
+            {
+                RTGCPHYS GCPhysSimp = MSR_GIM_HV_SIMP_GPA(uRawValue);
+                if (PGMPhysIsGCPhysNormal(pVM, GCPhysSimp))
+                {
+                    uint8_t abSimp[PAGE_SIZE];
+                    RT_ZERO(abSimp);
+                    int rc2 = PGMPhysSimpleWriteGCPhys(pVM, GCPhysSimp, &abSimp[0], sizeof(abSimp));
+                    if (RT_SUCCESS(rc2))
+                        LogRel(("GIM%u: HyperV: Enabled synthetic interrupt message page at %#RGp\n", pVCpu->idCpu, GCPhysSimp));
+                    else
+                    {
+                        LogRel(("GIM%u: HyperV: Failed to update synthetic interrupt message page at %#RGp. uSimpMsr=%#RX64 rc=%Rrc\n",
+                                pVCpu->idCpu, pHvCpu->uSimpMsr, GCPhysSimp, rc2));
+                        return VERR_CPUM_RAISE_GP_0;
+                    }
+                }
+                else
+                {
+                    LogRel(("GIM%u: HyperV: Enabled synthetic interrupt message page at invalid address %#RGp\n", pVCpu->idCpu,
+                            GCPhysSimp));
+                }
+            }
+            else
+                LogRel(("GIM%u: HyperV: Disabled synthetic interrupt message page\n", pVCpu->idCpu));
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_CRASH_P0:  pHv->uCrashP0Msr = uRawValue;  return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P1:  pHv->uCrashP1Msr = uRawValue;  return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P2:  pHv->uCrashP2Msr = uRawValue;  return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P3:  pHv->uCrashP3Msr = uRawValue;  return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P4:  pHv->uCrashP4Msr = uRawValue;  return VINF_SUCCESS;
+
+        case MSR_GIM_HV_TIME_REF_COUNT:     /* Read-only MSRs. */
+        case MSR_GIM_HV_VP_INDEX:
+        case MSR_GIM_HV_TSC_FREQ:
+        case MSR_GIM_HV_APIC_FREQ:
+            LogFunc(("WrMsr on read-only MSR %#RX32 -> #GP(0)\n", idMsr));
+            break;
+
+        case MSR_GIM_HV_DEBUG_OPTIONS_MSR:
+        {
+            if (pHv->fIsVendorMsHv)
+            {
+#ifndef IN_RING3
+                return VINF_CPUM_R3_MSR_WRITE;
+#else
+                LogRelMax(5, ("GIM: HyperV: Write debug options MSR with %#RX64 ignored\n", uRawValue));
+                return VINF_SUCCESS;
+#endif
+            }
+            return VERR_CPUM_RAISE_GP_0;
+        }
+
+        default:
+        {
+#ifdef IN_RING3
+            static uint32_t s_cTimes = 0;
+            if (s_cTimes++ < 20)
+                LogRel(("GIM: HyperV: Unknown/invalid WrMsr (%#x,%#x`%08x) -> #GP(0)\n", idMsr,
+                        uRawValue & UINT64_C(0xffffffff00000000), uRawValue & UINT64_C(0xffffffff)));
+            LogFunc(("Unknown/invalid WrMsr (%#RX32,%#RX64) -> #GP(0)\n", idMsr, uRawValue));
+            break;
+#else
+            return VINF_CPUM_R3_MSR_WRITE;
+#endif
+        }
+    }
+
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * Whether we need to trap \#UD exceptions in the guest.
+ *
+ * We only needed to trap \#UD exceptions for the old raw-mode guests when
+ * hypercalls are enabled. For HM VMs, the hypercall would be handled via the
+ * VMCALL/VMMCALL VM-exit.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) gimHvShouldTrapXcptUD(PVMCPU pVCpu)
+{
+    RT_NOREF(pVCpu);
+    return false;
+}
+
+
+/**
+ * Checks the instruction and executes the hypercall if it's a valid hypercall
+ * instruction.
+ *
+ * This interface is used by \#UD handlers and IEM.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   uDisOpcode  The disassembler opcode.
+ * @param   cbInstr     The instruction length.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimHvHypercallEx(PVMCPUCC pVCpu, PCPUMCTX pCtx, unsigned uDisOpcode, uint8_t cbInstr)
+{
+    Assert(pVCpu);
+    Assert(pCtx);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    CPUMCPUVENDOR const enmGuestCpuVendor = (CPUMCPUVENDOR)pVM->cpum.ro.GuestFeatures.enmCpuVendor;
+    if (   (   uDisOpcode == OP_VMCALL
+            && (   enmGuestCpuVendor == CPUMCPUVENDOR_INTEL
+                || enmGuestCpuVendor == CPUMCPUVENDOR_VIA
+                || enmGuestCpuVendor == CPUMCPUVENDOR_SHANGHAI))
+        || (   uDisOpcode == OP_VMMCALL
+            && (   enmGuestCpuVendor == CPUMCPUVENDOR_AMD
+                || enmGuestCpuVendor == CPUMCPUVENDOR_HYGON)) )
+        return gimHvHypercall(pVCpu, pCtx);
+
+    RT_NOREF_PV(cbInstr);
+    return VERR_GIM_INVALID_HYPERCALL_INSTR;
+}
+
+
+/**
+ * Exception handler for \#UD.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
+ * @retval  VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating
+ *          RIP.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ * @retval  VERR_GIM_INVALID_HYPERCALL_INSTR instruction at RIP is not a valid
+ *          hypercall instruction.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   pDis        Pointer to the disassembled instruction state at RIP.
+ *                      Optional, can be NULL.
+ * @param   pcbInstr    Where to store the instruction length of the hypercall
+ *                      instruction. Optional, can be NULL.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimHvXcptUD(PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * If we didn't ask for #UD to be trapped, bail.
+     */
+    if (!gimHvShouldTrapXcptUD(pVCpu))
+        return VERR_GIM_IPE_1;
+
+    if (!pDis)
+    {
+        /*
+         * Disassemble the instruction at RIP to figure out if it's the Intel VMCALL instruction
+         * or the AMD VMMCALL instruction and if so, handle it as a hypercall.
+         */
+        unsigned    cbInstr;
+        DISCPUSTATE Dis;
+        int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, &Dis, &cbInstr);
+        if (RT_SUCCESS(rc))
+        {
+            if (pcbInstr)
+                *pcbInstr = (uint8_t)cbInstr;
+            return gimHvHypercallEx(pVCpu, pCtx, Dis.pCurInstr->uOpcode, Dis.cbInstr);
+        }
+
+        Log(("GIM: HyperV: Failed to disassemble instruction at CS:RIP=%04x:%08RX64. rc=%Rrc\n", pCtx->cs.Sel, pCtx->rip, rc));
+        return rc;
+    }
+
+    return gimHvHypercallEx(pVCpu, pCtx, pDis->pCurInstr->uOpcode, pDis->cbInstr);
+}
+
diff --git a/src/VBox/VMM/VMMAll/GIMAllKvm.cpp b/src/VBox/VMM/VMMAll/GIMAllKvm.cpp
new file mode 100644
index 00000000..8a755b47
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/GIMAllKvm.cpp
@@ -0,0 +1,441 @@
+/* $Id: GIMAllKvm.cpp $ */
+/** @file
+ * GIM - Guest Interface Manager, KVM, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2015-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/pdmapi.h>
+#include "GIMKvmInternal.h"
+#include "GIMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/dis.h>
+#include <VBox/err.h>
+#include <VBox/sup.h>
+
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/time.h>
+
+
+/**
+ * Handles the KVM hypercall.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pCtx            Pointer to the guest-CPU context.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimKvmHypercall(PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    STAM_REL_COUNTER_INC(&pVM->gim.s.StatHypercalls);
+
+    /*
+     * Get the hypercall operation and arguments.
+     */
+    bool const fIs64BitMode = CPUMIsGuestIn64BitCodeEx(pCtx);
+    uint64_t uHyperOp       = pCtx->rax;
+    uint64_t uHyperArg0     = pCtx->rbx;
+    uint64_t uHyperArg1     = pCtx->rcx;
+    uint64_t uHyperArg2     = pCtx->rdi;
+    uint64_t uHyperArg3     = pCtx->rsi;
+    uint64_t uHyperRet      = KVM_HYPERCALL_RET_ENOSYS;
+    uint64_t uAndMask       = UINT64_C(0xffffffffffffffff);
+    if (!fIs64BitMode)
+    {
+        uAndMask    = UINT64_C(0xffffffff);
+        uHyperOp   &= UINT64_C(0xffffffff);
+        uHyperArg0 &= UINT64_C(0xffffffff);
+        uHyperArg1 &= UINT64_C(0xffffffff);
+        uHyperArg2 &= UINT64_C(0xffffffff);
+        uHyperArg3 &= UINT64_C(0xffffffff);
+        uHyperRet  &= UINT64_C(0xffffffff);
+    }
+
+    /*
+     * Verify that guest ring-0 is the one making the hypercall.
+     */
+    uint32_t uCpl = CPUMGetGuestCPL(pVCpu);
+    if (RT_UNLIKELY(uCpl))
+    {
+        pCtx->rax = KVM_HYPERCALL_RET_EPERM & uAndMask;
+        return VERR_GIM_HYPERCALL_ACCESS_DENIED;
+    }
+
+    /*
+     * Do the work.
+     */
+    int rc = VINF_SUCCESS;
+    switch (uHyperOp)
+    {
+        case KVM_HYPERCALL_OP_KICK_CPU:
+        {
+            if (uHyperArg1 < pVM->cCpus)
+            {
+                PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, uHyperArg1); /* ASSUMES pVCpu index == ApicId of the VCPU. */
+                EMUnhaltAndWakeUp(pVM, pVCpuDst);
+                uHyperRet = KVM_HYPERCALL_RET_SUCCESS;
+            }
+            else
+            {
+                /* Shouldn't ever happen! If it does, throw a guru, as otherwise it'll lead to deadlocks in the guest anyway! */
+                rc = VERR_GIM_HYPERCALL_FAILED;
+            }
+            break;
+        }
+
+        case KVM_HYPERCALL_OP_VAPIC_POLL_IRQ:
+            uHyperRet = KVM_HYPERCALL_RET_SUCCESS;
+            break;
+
+        default:
+            break;
+    }
+
+    /*
+     * Place the result in rax/eax.
+     */
+    pCtx->rax = uHyperRet & uAndMask;
+    return rc;
+}
+
+
+/**
+ * Returns whether the guest has configured and enabled the use of KVM's
+ * hypercall interface.
+ *
+ * @returns true if hypercalls are enabled, false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) gimKvmAreHypercallsEnabled(PVMCPU pVCpu)
+{
+    NOREF(pVCpu);
+    /* KVM paravirt interface doesn't have hypercall control bits (like Hyper-V does)
+       that guests can control, i.e. hypercalls are always enabled. */
+    return true;
+}
+
+
+/**
+ * Returns whether the guest has configured and enabled the use of KVM's
+ * paravirtualized TSC.
+ *
+ * @returns true if paravirt. TSC is enabled, false otherwise.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(bool) gimKvmIsParavirtTscEnabled(PVMCC pVM)
+{
+    uint32_t const cCpus = pVM->cCpus;
+    for (uint32_t idCpu = 0; idCpu < cCpus; idCpu++)
+    {
+        PVMCPUCC   pVCpu      = pVM->CTX_SUFF(apCpus)[idCpu];
+        PGIMKVMCPU pGimKvmCpu = &pVCpu->gim.s.u.KvmCpu;
+        if (MSR_GIM_KVM_SYSTEM_TIME_IS_ENABLED(pGimKvmCpu->u64SystemTimeMsr))
+            return true;
+    }
+    return false;
+}
+
+
+/**
+ * MSR read handler for KVM.
+ *
+ * @returns Strict VBox status code like CPUMQueryGuestMsr().
+ * @retval  VINF_CPUM_R3_MSR_READ
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR being read.
+ * @param   pRange      The range this MSR belongs to.
+ * @param   puValue     Where to store the MSR value read.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimKvmReadMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    NOREF(pRange);
+    PVM     pVM        = pVCpu->CTX_SUFF(pVM);
+    PGIMKVM pKvm       = &pVM->gim.s.u.Kvm;
+    PGIMKVMCPU pKvmCpu = &pVCpu->gim.s.u.KvmCpu;
+
+    switch (idMsr)
+    {
+        case MSR_GIM_KVM_SYSTEM_TIME:
+        case MSR_GIM_KVM_SYSTEM_TIME_OLD:
+            *puValue = pKvmCpu->u64SystemTimeMsr;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_KVM_WALL_CLOCK:
+        case MSR_GIM_KVM_WALL_CLOCK_OLD:
+            *puValue = pKvm->u64WallClockMsr;
+            return VINF_SUCCESS;
+
+        default:
+        {
+#ifdef IN_RING3
+            static uint32_t s_cTimes = 0;
+            if (s_cTimes++ < 20)
+                LogRel(("GIM: KVM: Unknown/invalid RdMsr (%#x) -> #GP(0)\n", idMsr));
+#endif
+            LogFunc(("Unknown/invalid RdMsr (%#RX32) -> #GP(0)\n", idMsr));
+            break;
+        }
+    }
+
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * MSR write handler for KVM.
+ *
+ * @returns Strict VBox status code like CPUMSetGuestMsr().
+ * @retval  VINF_CPUM_R3_MSR_WRITE
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR being written.
+ * @param   pRange      The range this MSR belongs to.
+ * @param   uRawValue   The raw value with the ignored bits not masked.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimKvmWriteMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uRawValue)
+{
+    NOREF(pRange);
+    switch (idMsr)
+    {
+        case MSR_GIM_KVM_SYSTEM_TIME:
+        case MSR_GIM_KVM_SYSTEM_TIME_OLD:
+        {
+#ifndef IN_RING3
+            RT_NOREF2(pVCpu, uRawValue);
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            PVMCC      pVM = pVCpu->CTX_SUFF(pVM);
+            PGIMKVMCPU pKvmCpu = &pVCpu->gim.s.u.KvmCpu;
+            if (uRawValue & MSR_GIM_KVM_SYSTEM_TIME_ENABLE_BIT)
+                gimR3KvmEnableSystemTime(pVM, pVCpu, uRawValue);
+            else
+                gimR3KvmDisableSystemTime(pVM);
+
+            pKvmCpu->u64SystemTimeMsr = uRawValue;
+            return VINF_SUCCESS;
+#endif /* IN_RING3 */
+        }
+
+        case MSR_GIM_KVM_WALL_CLOCK:
+        case MSR_GIM_KVM_WALL_CLOCK_OLD:
+        {
+#ifndef IN_RING3
+            RT_NOREF2(pVCpu, uRawValue);
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            /* Enable the wall-clock struct. */
+            RTGCPHYS GCPhysWallClock = MSR_GIM_KVM_WALL_CLOCK_GUEST_GPA(uRawValue);
+            if (RT_LIKELY(RT_ALIGN_64(GCPhysWallClock, 4) == GCPhysWallClock))
+            {
+                PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+                int rc = gimR3KvmEnableWallClock(pVM, GCPhysWallClock);
+                if (RT_SUCCESS(rc))
+                {
+                    PGIMKVM pKvm = &pVM->gim.s.u.Kvm;
+                    pKvm->u64WallClockMsr = uRawValue;
+                    return VINF_SUCCESS;
+                }
+            }
+            return VERR_CPUM_RAISE_GP_0;
+#endif /* IN_RING3 */
+        }
+
+        default:
+        {
+#ifdef IN_RING3
+            static uint32_t s_cTimes = 0;
+            if (s_cTimes++ < 20)
+                LogRel(("GIM: KVM: Unknown/invalid WrMsr (%#x,%#x`%08x) -> #GP(0)\n", idMsr,
+                        uRawValue & UINT64_C(0xffffffff00000000), uRawValue & UINT64_C(0xffffffff)));
+#endif
+            LogFunc(("Unknown/invalid WrMsr (%#RX32,%#RX64) -> #GP(0)\n", idMsr, uRawValue));
+            break;
+        }
+    }
+
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * Whether we need to trap \#UD exceptions in the guest.
+ *
+ * On AMD-V we need to trap them because paravirtualized Linux/KVM guests use
+ * the Intel VMCALL instruction to make hypercalls and we need to trap and
+ * optionally patch them to the AMD-V VMMCALL instruction and handle the
+ * hypercall.
+ *
+ * I guess this was done so that guest teleporation between an AMD and an Intel
+ * machine would working without any changes at the time of teleporation.
+ * However, this also means we -always- need to intercept \#UD exceptions on one
+ * of the two CPU models (Intel or AMD). Hyper-V solves this problem more
+ * elegantly by letting the hypervisor supply an opaque hypercall page.
+ *
+ * For raw-mode VMs, this function will always return true. See gimR3KvmInit().
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(bool) gimKvmShouldTrapXcptUD(PVM pVM)
+{
+    return pVM->gim.s.u.Kvm.fTrapXcptUD;
+}
+
+
+/**
+ * Checks the instruction and executes the hypercall if it's a valid hypercall
+ * instruction.
+ *
+ * This interface is used by \#UD handlers and IEM.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   uDisOpcode  The disassembler opcode.
+ * @param   cbInstr     The instruction length.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimKvmHypercallEx(PVMCPUCC pVCpu, PCPUMCTX pCtx, unsigned uDisOpcode, uint8_t cbInstr)
+{
+    Assert(pVCpu);
+    Assert(pCtx);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * If the instruction at RIP is the Intel VMCALL instruction or
+     * the AMD VMMCALL instruction handle it as a hypercall.
+     *
+     * Linux/KVM guests always uses the Intel VMCALL instruction but we patch
+     * it to the host-native one whenever we encounter it so subsequent calls
+     * will not require disassembly (when coming from HM).
+     */
+    if (   uDisOpcode == OP_VMCALL
+        || uDisOpcode == OP_VMMCALL)
+    {
+        /*
+         * Perform the hypercall.
+         *
+         * For HM, we can simply resume guest execution without performing the hypercall now and
+         * do it on the next VMCALL/VMMCALL exit handler on the patched instruction.
+         *
+         * For raw-mode we need to do this now anyway. So we do it here regardless with an added
+         * advantage is that it saves one world-switch for the HM case.
+         */
+        VBOXSTRICTRC rcStrict = gimKvmHypercall(pVCpu, pCtx);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            /*
+             * Patch the instruction to so we don't have to spend time disassembling it each time.
+             * Makes sense only for HM as with raw-mode we will be getting a #UD regardless.
+             */
+            PVM      pVM  = pVCpu->CTX_SUFF(pVM);
+            PCGIMKVM pKvm = &pVM->gim.s.u.Kvm;
+            if (   uDisOpcode != pKvm->uOpcodeNative
+                && cbInstr == sizeof(pKvm->abOpcodeNative) )
+            {
+                /** @todo r=ramshankar: we probably should be doing this in an
+                 *        EMT rendezvous. */
+                /** @todo Add stats for patching. */
+                int rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, pKvm->abOpcodeNative, sizeof(pKvm->abOpcodeNative));
+                AssertRC(rc);
+            }
+        }
+        else
+        {
+            /* The KVM provider doesn't have any concept of continuing hypercalls. */
+            Assert(rcStrict != VINF_GIM_HYPERCALL_CONTINUING);
+#ifdef IN_RING3
+            Assert(rcStrict != VINF_GIM_R3_HYPERCALL);
+#endif
+        }
+        return rcStrict;
+    }
+
+    return VERR_GIM_INVALID_HYPERCALL_INSTR;
+}
+
+
+/**
+ * Exception handler for \#UD.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ * @retval  VERR_GIM_INVALID_HYPERCALL_INSTR instruction at RIP is not a valid
+ *          hypercall instruction.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   pDis        Pointer to the disassembled instruction state at RIP.
+ *                      Optional, can be NULL.
+ * @param   pcbInstr    Where to store the instruction length of the hypercall
+ *                      instruction. Optional, can be NULL.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimKvmXcptUD(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * If we didn't ask for #UD to be trapped, bail.
+     */
+    if (RT_UNLIKELY(!pVM->gim.s.u.Kvm.fTrapXcptUD))
+        return VERR_GIM_IPE_3;
+
+    if (!pDis)
+    {
+        unsigned    cbInstr;
+        DISCPUSTATE Dis;
+        int rc = EMInterpretDisasCurrent(pVM, pVCpu, &Dis, &cbInstr);
+        if (RT_SUCCESS(rc))
+        {
+            if (pcbInstr)
+                *pcbInstr = (uint8_t)cbInstr;
+            return gimKvmHypercallEx(pVCpu, pCtx, Dis.pCurInstr->uOpcode, Dis.cbInstr);
+        }
+
+        Log(("GIM: KVM: Failed to disassemble instruction at CS:RIP=%04x:%08RX64. rc=%Rrc\n", pCtx->cs.Sel, pCtx->rip, rc));
+        return rc;
+    }
+
+    return gimKvmHypercallEx(pVCpu, pCtx, pDis->pCurInstr->uOpcode, pDis->cbInstr);
+}
+
diff --git a/src/VBox/VMM/VMMAll/HMAll.cpp b/src/VBox/VMM/VMMAll/HMAll.cpp
new file mode 100644
index 00000000..5ad8c343
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/HMAll.cpp
@@ -0,0 +1,905 @@
+/* $Id: HMAll.cpp $ */
+/** @file
+ * HM - All contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_HM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pgm.h>
+#include "HMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/hm_vmx.h>
+#include <VBox/vmm/hm_svm.h>
+#include <iprt/errcore.h>
+#include <VBox/log.h>
+#include <iprt/param.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/x86.h>
+#include <iprt/asm-amd64-x86.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#define EXIT_REASON(a_Def, a_Val, a_Str)      #a_Def " - " #a_Val " - " a_Str
+#define EXIT_REASON_NIL()                     NULL
+
+/** Exit reason descriptions for VT-x, used to describe statistics and exit
+ *  history. */
+static const char * const g_apszVmxExitReasons[MAX_EXITREASON_STAT] =
+{
+    EXIT_REASON(VMX_EXIT_XCPT_OR_NMI            ,   0, "Exception or non-maskable interrupt (NMI)."),
+    EXIT_REASON(VMX_EXIT_EXT_INT                ,   1, "External interrupt."),
+    EXIT_REASON(VMX_EXIT_TRIPLE_FAULT           ,   2, "Triple fault."),
+    EXIT_REASON(VMX_EXIT_INIT_SIGNAL            ,   3, "INIT signal."),
+    EXIT_REASON(VMX_EXIT_SIPI                   ,   4, "Start-up IPI (SIPI)."),
+    EXIT_REASON(VMX_EXIT_IO_SMI_IRQ             ,   5, "I/O system-management interrupt (SMI)."),
+    EXIT_REASON(VMX_EXIT_SMI_IRQ                ,   6, "Other SMI."),
+    EXIT_REASON(VMX_EXIT_INT_WINDOW             ,   7, "Interrupt window."),
+    EXIT_REASON(VMX_EXIT_NMI_WINDOW             ,   8, "NMI window."),
+    EXIT_REASON(VMX_EXIT_TASK_SWITCH            ,   9, "Task switch."),
+    EXIT_REASON(VMX_EXIT_CPUID                  ,  10, "CPUID instruction."),
+    EXIT_REASON(VMX_EXIT_GETSEC                 ,  11, "GETSEC instruction."),
+    EXIT_REASON(VMX_EXIT_HLT                    ,  12, "HLT instruction."),
+    EXIT_REASON(VMX_EXIT_INVD                   ,  13, "INVD instruction."),
+    EXIT_REASON(VMX_EXIT_INVLPG                 ,  14, "INVLPG instruction."),
+    EXIT_REASON(VMX_EXIT_RDPMC                  ,  15, "RDPMC instruction."),
+    EXIT_REASON(VMX_EXIT_RDTSC                  ,  16, "RDTSC instruction."),
+    EXIT_REASON(VMX_EXIT_RSM                    ,  17, "RSM instruction in SMM."),
+    EXIT_REASON(VMX_EXIT_VMCALL                 ,  18, "VMCALL instruction."),
+    EXIT_REASON(VMX_EXIT_VMCLEAR                ,  19, "VMCLEAR instruction."),
+    EXIT_REASON(VMX_EXIT_VMLAUNCH               ,  20, "VMLAUNCH instruction."),
+    EXIT_REASON(VMX_EXIT_VMPTRLD                ,  21, "VMPTRLD instruction."),
+    EXIT_REASON(VMX_EXIT_VMPTRST                ,  22, "VMPTRST instruction."),
+    EXIT_REASON(VMX_EXIT_VMREAD                 ,  23, "VMREAD instruction."),
+    EXIT_REASON(VMX_EXIT_VMRESUME               ,  24, "VMRESUME instruction."),
+    EXIT_REASON(VMX_EXIT_VMWRITE                ,  25, "VMWRITE instruction."),
+    EXIT_REASON(VMX_EXIT_VMXOFF                 ,  26, "VMXOFF instruction."),
+    EXIT_REASON(VMX_EXIT_VMXON                  ,  27, "VMXON instruction."),
+    EXIT_REASON(VMX_EXIT_MOV_CRX                ,  28, "Control-register accesses."),
+    EXIT_REASON(VMX_EXIT_MOV_DRX                ,  29, "Debug-register accesses."),
+    EXIT_REASON(VMX_EXIT_PORT_IO                ,  30, "I/O instruction."),
+    EXIT_REASON(VMX_EXIT_RDMSR                  ,  31, "RDMSR instruction."),
+    EXIT_REASON(VMX_EXIT_WRMSR                  ,  32, "WRMSR instruction."),
+    EXIT_REASON(VMX_EXIT_ERR_INVALID_GUEST_STATE,  33, "VM-entry failure due to invalid guest state."),
+    EXIT_REASON(VMX_EXIT_ERR_MSR_LOAD           ,  34, "VM-entry failure due to MSR loading."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_MWAIT                  ,  36, "MWAIT instruction."),
+    EXIT_REASON(VMX_EXIT_MTF                    ,  37, "Monitor Trap Flag."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_MONITOR                ,  39, "MONITOR instruction."),
+    EXIT_REASON(VMX_EXIT_PAUSE                  ,  40, "PAUSE instruction."),
+    EXIT_REASON(VMX_EXIT_ERR_MACHINE_CHECK      ,  41, "VM-entry failure due to machine-check."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_TPR_BELOW_THRESHOLD    ,  43, "TPR below threshold (MOV to CR8)."),
+    EXIT_REASON(VMX_EXIT_APIC_ACCESS            ,  44, "APIC access."),
+    EXIT_REASON(VMX_EXIT_VIRTUALIZED_EOI        ,  45, "Virtualized EOI."),
+    EXIT_REASON(VMX_EXIT_GDTR_IDTR_ACCESS       ,  46, "GDTR/IDTR access using LGDT/SGDT/LIDT/SIDT."),
+    EXIT_REASON(VMX_EXIT_LDTR_TR_ACCESS         ,  47, "LDTR/TR access using LLDT/SLDT/LTR/STR."),
+    EXIT_REASON(VMX_EXIT_EPT_VIOLATION          ,  48, "EPT violation."),
+    EXIT_REASON(VMX_EXIT_EPT_MISCONFIG          ,  49, "EPT misconfiguration."),
+    EXIT_REASON(VMX_EXIT_INVEPT                 ,  50, "INVEPT instruction."),
+    EXIT_REASON(VMX_EXIT_RDTSCP                 ,  51, "RDTSCP instruction."),
+    EXIT_REASON(VMX_EXIT_PREEMPT_TIMER          ,  52, "VMX-preemption timer expired."),
+    EXIT_REASON(VMX_EXIT_INVVPID                ,  53, "INVVPID instruction."),
+    EXIT_REASON(VMX_EXIT_WBINVD                 ,  54, "WBINVD instruction."),
+    EXIT_REASON(VMX_EXIT_XSETBV                 ,  55, "XSETBV instruction."),
+    EXIT_REASON(VMX_EXIT_APIC_WRITE             ,  56, "APIC write completed to virtual-APIC page."),
+    EXIT_REASON(VMX_EXIT_RDRAND                 ,  57, "RDRAND instruction."),
+    EXIT_REASON(VMX_EXIT_INVPCID                ,  58, "INVPCID instruction."),
+    EXIT_REASON(VMX_EXIT_VMFUNC                 ,  59, "VMFUNC instruction."),
+    EXIT_REASON(VMX_EXIT_ENCLS                  ,  60, "ENCLS instruction."),
+    EXIT_REASON(VMX_EXIT_RDSEED                 ,  61, "RDSEED instruction."),
+    EXIT_REASON(VMX_EXIT_PML_FULL               ,  62, "Page-modification log full."),
+    EXIT_REASON(VMX_EXIT_XSAVES                 ,  63, "XSAVES instruction."),
+    EXIT_REASON(VMX_EXIT_XRSTORS                ,  64, "XRSTORS instruction."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_SPP_EVENT              ,  66, "SPP-related event."),
+    EXIT_REASON(VMX_EXIT_UMWAIT                 ,  67, "UMWAIT instruction."),
+    EXIT_REASON(VMX_EXIT_TPAUSE                 ,  68, "TPAUSE instruction.")
+};
+/** Array index of the last valid VT-x exit reason. */
+#define MAX_EXITREASON_VTX                         68
+
+/** A partial list of \#EXIT reason descriptions for AMD-V, used to describe
+ *  statistics and exit history.
+ *
+ *  @note AMD-V have annoyingly large gaps (e.g. \#NPF VMEXIT comes at 1024),
+ *        this array doesn't contain the entire set of exit reasons, we
+ *        handle them via hmSvmGetSpecialExitReasonDesc(). */
+static const char * const g_apszSvmExitReasons[MAX_EXITREASON_STAT] =
+{
+    EXIT_REASON(SVM_EXIT_READ_CR0       ,    0, "Read CR0."),
+    EXIT_REASON(SVM_EXIT_READ_CR1       ,    1, "Read CR1."),
+    EXIT_REASON(SVM_EXIT_READ_CR2       ,    2, "Read CR2."),
+    EXIT_REASON(SVM_EXIT_READ_CR3       ,    3, "Read CR3."),
+    EXIT_REASON(SVM_EXIT_READ_CR4       ,    4, "Read CR4."),
+    EXIT_REASON(SVM_EXIT_READ_CR5       ,    5, "Read CR5."),
+    EXIT_REASON(SVM_EXIT_READ_CR6       ,    6, "Read CR6."),
+    EXIT_REASON(SVM_EXIT_READ_CR7       ,    7, "Read CR7."),
+    EXIT_REASON(SVM_EXIT_READ_CR8       ,    8, "Read CR8."),
+    EXIT_REASON(SVM_EXIT_READ_CR9       ,    9, "Read CR9."),
+    EXIT_REASON(SVM_EXIT_READ_CR10      ,   10, "Read CR10."),
+    EXIT_REASON(SVM_EXIT_READ_CR11      ,   11, "Read CR11."),
+    EXIT_REASON(SVM_EXIT_READ_CR12      ,   12, "Read CR12."),
+    EXIT_REASON(SVM_EXIT_READ_CR13      ,   13, "Read CR13."),
+    EXIT_REASON(SVM_EXIT_READ_CR14      ,   14, "Read CR14."),
+    EXIT_REASON(SVM_EXIT_READ_CR15      ,   15, "Read CR15."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR0      ,   16, "Write CR0."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR1      ,   17, "Write CR1."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR2      ,   18, "Write CR2."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR3      ,   19, "Write CR3."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR4      ,   20, "Write CR4."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR5      ,   21, "Write CR5."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR6      ,   22, "Write CR6."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR7      ,   23, "Write CR7."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR8      ,   24, "Write CR8."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR9      ,   25, "Write CR9."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR10     ,   26, "Write CR10."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR11     ,   27, "Write CR11."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR12     ,   28, "Write CR12."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR13     ,   29, "Write CR13."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR14     ,   30, "Write CR14."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR15     ,   31, "Write CR15."),
+    EXIT_REASON(SVM_EXIT_READ_DR0       ,   32, "Read DR0."),
+    EXIT_REASON(SVM_EXIT_READ_DR1       ,   33, "Read DR1."),
+    EXIT_REASON(SVM_EXIT_READ_DR2       ,   34, "Read DR2."),
+    EXIT_REASON(SVM_EXIT_READ_DR3       ,   35, "Read DR3."),
+    EXIT_REASON(SVM_EXIT_READ_DR4       ,   36, "Read DR4."),
+    EXIT_REASON(SVM_EXIT_READ_DR5       ,   37, "Read DR5."),
+    EXIT_REASON(SVM_EXIT_READ_DR6       ,   38, "Read DR6."),
+    EXIT_REASON(SVM_EXIT_READ_DR7       ,   39, "Read DR7."),
+    EXIT_REASON(SVM_EXIT_READ_DR8       ,   40, "Read DR8."),
+    EXIT_REASON(SVM_EXIT_READ_DR9       ,   41, "Read DR9."),
+    EXIT_REASON(SVM_EXIT_READ_DR10      ,   42, "Read DR10."),
+    EXIT_REASON(SVM_EXIT_READ_DR11      ,   43, "Read DR11"),
+    EXIT_REASON(SVM_EXIT_READ_DR12      ,   44, "Read DR12."),
+    EXIT_REASON(SVM_EXIT_READ_DR13      ,   45, "Read DR13."),
+    EXIT_REASON(SVM_EXIT_READ_DR14      ,   46, "Read DR14."),
+    EXIT_REASON(SVM_EXIT_READ_DR15      ,   47, "Read DR15."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR0      ,   48, "Write DR0."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR1      ,   49, "Write DR1."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR2      ,   50, "Write DR2."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR3      ,   51, "Write DR3."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR4      ,   52, "Write DR4."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR5      ,   53, "Write DR5."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR6      ,   54, "Write DR6."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR7      ,   55, "Write DR7."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR8      ,   56, "Write DR8."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR9      ,   57, "Write DR9."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR10     ,   58, "Write DR10."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR11     ,   59, "Write DR11."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR12     ,   60, "Write DR12."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR13     ,   61, "Write DR13."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR14     ,   62, "Write DR14."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR15     ,   63, "Write DR15."),
+    EXIT_REASON(SVM_EXIT_XCPT_0         ,   64, "Exception 0  (#DE)."),
+    EXIT_REASON(SVM_EXIT_XCPT_1         ,   65, "Exception 1  (#DB)."),
+    EXIT_REASON(SVM_EXIT_XCPT_2         ,   66, "Exception 2  (#NMI)."),
+    EXIT_REASON(SVM_EXIT_XCPT_3         ,   67, "Exception 3  (#BP)."),
+    EXIT_REASON(SVM_EXIT_XCPT_4         ,   68, "Exception 4  (#OF)."),
+    EXIT_REASON(SVM_EXIT_XCPT_5         ,   69, "Exception 5  (#BR)."),
+    EXIT_REASON(SVM_EXIT_XCPT_6         ,   70, "Exception 6  (#UD)."),
+    EXIT_REASON(SVM_EXIT_XCPT_7         ,   71, "Exception 7  (#NM)."),
+    EXIT_REASON(SVM_EXIT_XCPT_8         ,   72, "Exception 8  (#DF)."),
+    EXIT_REASON(SVM_EXIT_XCPT_9         ,   73, "Exception 9  (#CO_SEG_OVERRUN)."),
+    EXIT_REASON(SVM_EXIT_XCPT_10        ,   74, "Exception 10 (#TS)."),
+    EXIT_REASON(SVM_EXIT_XCPT_11        ,   75, "Exception 11 (#NP)."),
+    EXIT_REASON(SVM_EXIT_XCPT_12        ,   76, "Exception 12 (#SS)."),
+    EXIT_REASON(SVM_EXIT_XCPT_13        ,   77, "Exception 13 (#GP)."),
+    EXIT_REASON(SVM_EXIT_XCPT_14        ,   78, "Exception 14 (#PF)."),
+    EXIT_REASON(SVM_EXIT_XCPT_15        ,   79, "Exception 15 (0x0f)."),
+    EXIT_REASON(SVM_EXIT_XCPT_16        ,   80, "Exception 16 (#MF)."),
+    EXIT_REASON(SVM_EXIT_XCPT_17        ,   81, "Exception 17 (#AC)."),
+    EXIT_REASON(SVM_EXIT_XCPT_18        ,   82, "Exception 18 (#MC)."),
+    EXIT_REASON(SVM_EXIT_XCPT_19        ,   83, "Exception 19 (#XF)."),
+    EXIT_REASON(SVM_EXIT_XCPT_20        ,   84, "Exception 20 (#VE)."),
+    EXIT_REASON(SVM_EXIT_XCPT_21        ,   85, "Exception 22 (0x15)."),
+    EXIT_REASON(SVM_EXIT_XCPT_22        ,   86, "Exception 22 (0x16)."),
+    EXIT_REASON(SVM_EXIT_XCPT_23        ,   87, "Exception 23 (0x17)."),
+    EXIT_REASON(SVM_EXIT_XCPT_24        ,   88, "Exception 24 (0x18)."),
+    EXIT_REASON(SVM_EXIT_XCPT_25        ,   89, "Exception 25 (0x19)."),
+    EXIT_REASON(SVM_EXIT_XCPT_26        ,   90, "Exception 26 (0x1a)."),
+    EXIT_REASON(SVM_EXIT_XCPT_27        ,   91, "Exception 27 (0x1b)."),
+    EXIT_REASON(SVM_EXIT_XCPT_28        ,   92, "Exception 28 (0x1c)."),
+    EXIT_REASON(SVM_EXIT_XCPT_29        ,   93, "Exception 29 (0x1d)."),
+    EXIT_REASON(SVM_EXIT_XCPT_30        ,   94, "Exception 30 (#SX)."),
+    EXIT_REASON(SVM_EXIT_XCPT_31        ,   95, "Exception 31 (0x1F)."),
+    EXIT_REASON(SVM_EXIT_INTR           ,   96, "Physical maskable interrupt (host)."),
+    EXIT_REASON(SVM_EXIT_NMI            ,   97, "Physical non-maskable interrupt (host)."),
+    EXIT_REASON(SVM_EXIT_SMI            ,   98, "System management interrupt (host)."),
+    EXIT_REASON(SVM_EXIT_INIT           ,   99, "Physical INIT signal (host)."),
+    EXIT_REASON(SVM_EXIT_VINTR          ,  100, "Virtual interrupt-window exit."),
+    EXIT_REASON(SVM_EXIT_CR0_SEL_WRITE  ,  101, "Selective CR0 Write (to bits other than CR0.TS and CR0.MP)."),
+    EXIT_REASON(SVM_EXIT_IDTR_READ      ,  102, "Read IDTR."),
+    EXIT_REASON(SVM_EXIT_GDTR_READ      ,  103, "Read GDTR."),
+    EXIT_REASON(SVM_EXIT_LDTR_READ      ,  104, "Read LDTR."),
+    EXIT_REASON(SVM_EXIT_TR_READ        ,  105, "Read TR."),
+    EXIT_REASON(SVM_EXIT_IDTR_WRITE     ,  106, "Write IDTR."),
+    EXIT_REASON(SVM_EXIT_GDTR_WRITE     ,  107, "Write GDTR."),
+    EXIT_REASON(SVM_EXIT_LDTR_WRITE     ,  108, "Write LDTR."),
+    EXIT_REASON(SVM_EXIT_TR_WRITE       ,  109, "Write TR."),
+    EXIT_REASON(SVM_EXIT_RDTSC          ,  110, "RDTSC instruction."),
+    EXIT_REASON(SVM_EXIT_RDPMC          ,  111, "RDPMC instruction."),
+    EXIT_REASON(SVM_EXIT_PUSHF          ,  112, "PUSHF instruction."),
+    EXIT_REASON(SVM_EXIT_POPF           ,  113, "POPF instruction."),
+    EXIT_REASON(SVM_EXIT_CPUID          ,  114, "CPUID instruction."),
+    EXIT_REASON(SVM_EXIT_RSM            ,  115, "RSM instruction."),
+    EXIT_REASON(SVM_EXIT_IRET           ,  116, "IRET instruction."),
+    EXIT_REASON(SVM_EXIT_SWINT          ,  117, "Software interrupt (INTn instructions)."),
+    EXIT_REASON(SVM_EXIT_INVD           ,  118, "INVD instruction."),
+    EXIT_REASON(SVM_EXIT_PAUSE          ,  119, "PAUSE instruction."),
+    EXIT_REASON(SVM_EXIT_HLT            ,  120, "HLT instruction."),
+    EXIT_REASON(SVM_EXIT_INVLPG         ,  121, "INVLPG instruction."),
+    EXIT_REASON(SVM_EXIT_INVLPGA        ,  122, "INVLPGA instruction."),
+    EXIT_REASON(SVM_EXIT_IOIO           ,  123, "IN/OUT/INS/OUTS instruction."),
+    EXIT_REASON(SVM_EXIT_MSR            ,  124, "RDMSR or WRMSR access to protected MSR."),
+    EXIT_REASON(SVM_EXIT_TASK_SWITCH    ,  125, "Task switch."),
+    EXIT_REASON(SVM_EXIT_FERR_FREEZE    ,  126, "FERR Freeze; CPU frozen in an x87/mmx instruction waiting for interrupt."),
+    EXIT_REASON(SVM_EXIT_SHUTDOWN       ,  127, "Shutdown."),
+    EXIT_REASON(SVM_EXIT_VMRUN          ,  128, "VMRUN instruction."),
+    EXIT_REASON(SVM_EXIT_VMMCALL        ,  129, "VMCALL instruction."),
+    EXIT_REASON(SVM_EXIT_VMLOAD         ,  130, "VMLOAD instruction."),
+    EXIT_REASON(SVM_EXIT_VMSAVE         ,  131, "VMSAVE instruction."),
+    EXIT_REASON(SVM_EXIT_STGI           ,  132, "STGI instruction."),
+    EXIT_REASON(SVM_EXIT_CLGI           ,  133, "CLGI instruction."),
+    EXIT_REASON(SVM_EXIT_SKINIT         ,  134, "SKINIT instruction."),
+    EXIT_REASON(SVM_EXIT_RDTSCP         ,  135, "RDTSCP instruction."),
+    EXIT_REASON(SVM_EXIT_ICEBP          ,  136, "ICEBP instruction."),
+    EXIT_REASON(SVM_EXIT_WBINVD         ,  137, "WBINVD instruction."),
+    EXIT_REASON(SVM_EXIT_MONITOR        ,  138, "MONITOR instruction."),
+    EXIT_REASON(SVM_EXIT_MWAIT          ,  139, "MWAIT instruction."),
+    EXIT_REASON(SVM_EXIT_MWAIT_ARMED    ,  140, "MWAIT instruction when armed."),
+    EXIT_REASON(SVM_EXIT_XSETBV         ,  141, "XSETBV instruction."),
+    EXIT_REASON(SVM_EXIT_RDPRU          ,  142, "RDPRU instruction."),
+    EXIT_REASON(SVM_EXIT_WRITE_EFER_TRAP,  143, "Write EFER (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR0_TRAP ,  144, "Write CR0 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR1_TRAP ,  145, "Write CR1 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR2_TRAP ,  146, "Write CR2 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR3_TRAP ,  147, "Write CR3 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR4_TRAP ,  148, "Write CR4 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR5_TRAP ,  149, "Write CR5 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR6_TRAP ,  150, "Write CR6 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR7_TRAP ,  151, "Write CR7 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR8_TRAP ,  152, "Write CR8 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR9_TRAP ,  153, "Write CR9 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR10_TRAP,  154, "Write CR10 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR11_TRAP,  155, "Write CR11 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR12_TRAP,  156, "Write CR12 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR13_TRAP,  157, "Write CR13 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR14_TRAP,  158, "Write CR14 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR15_TRAP,  159, "Write CR15 (trap-like)."),
+    EXIT_REASON_NIL()                   ,
+    EXIT_REASON_NIL()                   ,
+    EXIT_REASON_NIL()                   ,
+    EXIT_REASON(SVM_EXIT_MCOMMIT        ,  163, "MCOMMIT instruction."),
+};
+/** Array index of the last valid AMD-V exit reason. */
+#define MAX_EXITREASON_AMDV              163
+
+/** Special exit reasons not covered in the array above. */
+#define SVM_EXIT_REASON_NPF                  EXIT_REASON(SVM_EXIT_NPF                , 1024, "Nested Page Fault.")
+#define SVM_EXIT_REASON_AVIC_INCOMPLETE_IPI  EXIT_REASON(SVM_EXIT_AVIC_INCOMPLETE_IPI, 1025, "AVIC - Incomplete IPI delivery.")
+#define SVM_EXIT_REASON_AVIC_NOACCEL         EXIT_REASON(SVM_EXIT_AVIC_NOACCEL       , 1026, "AVIC - Unhandled register.")
+
+/**
+ * Gets the SVM exit reason if it's one of the reasons not present in the @c
+ * g_apszSvmExitReasons array.
+ *
+ * @returns The exit reason or NULL if unknown.
+ * @param   uExit       The exit.
+ */
+DECLINLINE(const char *) hmSvmGetSpecialExitReasonDesc(uint16_t uExit)
+{
+    switch (uExit)
+    {
+        case SVM_EXIT_NPF:                 return SVM_EXIT_REASON_NPF;
+        case SVM_EXIT_AVIC_INCOMPLETE_IPI: return SVM_EXIT_REASON_AVIC_INCOMPLETE_IPI;
+        case SVM_EXIT_AVIC_NOACCEL:        return SVM_EXIT_REASON_AVIC_NOACCEL;
+    }
+    return EXIT_REASON_NIL();
+}
+#undef EXIT_REASON_NIL
+#undef EXIT_REASON
+
+
+/**
+ * Checks whether HM (VT-x/AMD-V) is being used by this VM.
+ *
+ * @retval  true if used.
+ * @retval  false if software virtualization (raw-mode) is used.
+ * @param   pVM        The cross context VM structure.
+ * @sa      HMIsEnabled, HMR3IsEnabled
+ * @internal
+ */
+VMMDECL(bool) HMIsEnabledNotMacro(PVM pVM)
+{
+    Assert(pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NOT_SET);
+    return pVM->fHMEnabled;
+}
+
+
+/**
+ * Checks if the guest is in a suitable state for hardware-assisted execution.
+ *
+ * @returns @c true if it is suitable, @c false otherwise.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   pCtx    Pointer to the guest CPU context.
+ *
+ * @remarks @a pCtx can be a partial context created and not necessarily the same as
+ *          pVCpu->cpum.GstCtx.
+ */
+VMMDECL(bool) HMCanExecuteGuest(PVMCC pVM, PVMCPUCC pVCpu, PCCPUMCTX pCtx)
+{
+    Assert(HMIsEnabled(pVM));
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
+    if (   CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
+        || CPUMIsGuestInVmxNonRootMode(pCtx))
+    {
+        LogFunc(("In nested-guest mode - returning false"));
+        return false;
+    }
+#endif
+
+    /* AMD-V supports real & protected mode with or without paging. */
+    if (pVM->hm.s.svm.fEnabled)
+    {
+        pVCpu->hm.s.fActive = true;
+        return true;
+    }
+
+    bool rc = HMCanExecuteVmxGuest(pVM, pVCpu, pCtx);
+    LogFlowFunc(("returning %RTbool\n", rc));
+    return rc;
+}
+
+
+/**
+ * Queues a guest page for invalidation.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCVirt      Page to invalidate.
+ */
+static void hmQueueInvlPage(PVMCPU pVCpu, RTGCPTR GCVirt)
+{
+    /* Nothing to do if a TLB flush is already pending */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
+        return;
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+    NOREF(GCVirt);
+}
+
+
+/**
+ * Invalidates a guest page.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCVirt      Page to invalidate.
+ */
+VMM_INT_DECL(int) HMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
+{
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushPageManual);
+#ifdef IN_RING0
+    return HMR0InvalidatePage(pVCpu, GCVirt);
+#else
+    hmQueueInvlPage(pVCpu, GCVirt);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+#ifdef IN_RING0
+
+/**
+ * Dummy RTMpOnSpecific handler since RTMpPokeCpu couldn't be used.
+ *
+ */
+static DECLCALLBACK(void) hmFlushHandler(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    NOREF(idCpu); NOREF(pvUser1); NOREF(pvUser2);
+    return;
+}
+
+
+/**
+ * Wrapper for RTMpPokeCpu to deal with VERR_NOT_SUPPORTED.
+ */
+static void hmR0PokeCpu(PVMCPU pVCpu, RTCPUID idHostCpu)
+{
+    uint32_t cWorldSwitchExits = ASMAtomicUoReadU32(&pVCpu->hm.s.cWorldSwitchExits);
+
+    STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatPoke, x);
+    int rc = RTMpPokeCpu(idHostCpu);
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPoke, x);
+
+    /* Not implemented on some platforms (Darwin, Linux kernel < 2.6.19); fall
+       back to a less efficient implementation (broadcast). */
+    if (rc == VERR_NOT_SUPPORTED)
+    {
+        STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatSpinPoke, z);
+        /* synchronous. */
+        RTMpOnSpecific(idHostCpu, hmFlushHandler, 0, 0);
+        STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatSpinPoke, z);
+    }
+    else
+    {
+        if (rc == VINF_SUCCESS)
+            STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatSpinPoke, z);
+        else
+            STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatSpinPokeFailed, z);
+
+/** @todo If more than one CPU is going to be poked, we could optimize this
+ *        operation by poking them first and wait afterwards.  Would require
+ *        recording who to poke and their current cWorldSwitchExits values,
+ *        that's something not suitable for stack... So, pVCpu->hm.s.something
+ *        then. */
+        /* Spin until the VCPU has switched back (poking is async). */
+        while (   ASMAtomicUoReadBool(&pVCpu->hm.s.fCheckedTLBFlush)
+               && cWorldSwitchExits == ASMAtomicUoReadU32(&pVCpu->hm.s.cWorldSwitchExits))
+            ASMNopPause();
+
+        if (rc == VINF_SUCCESS)
+            STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatSpinPoke, z);
+        else
+            STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatSpinPokeFailed, z);
+    }
+}
+
+#endif /* IN_RING0 */
+
+/**
+ * Flushes the guest TLB.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(int) HMFlushTlb(PVMCPU pVCpu)
+{
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbManual);
+    return VINF_SUCCESS;
+}
+
+/**
+ * Poke an EMT so it can perform the appropriate TLB shootdowns.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              EMT poke.
+ * @param   fAccountFlushStat   Whether to account the call to
+ *                              StatTlbShootdownFlush or StatTlbShootdown.
+ */
+static void hmPokeCpuForTlbFlush(PVMCPU pVCpu, bool fAccountFlushStat)
+{
+    if (ASMAtomicUoReadBool(&pVCpu->hm.s.fCheckedTLBFlush))
+    {
+        if (fAccountFlushStat)
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdownFlush);
+        else
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
+#ifdef IN_RING0
+        RTCPUID idHostCpu = pVCpu->hm.s.idEnteredCpu;
+        if (idHostCpu != NIL_RTCPUID)
+            hmR0PokeCpu(pVCpu, idHostCpu);
+#else
+        VMR3NotifyCpuFFU(pVCpu->pUVCpu, VMNOTIFYFF_FLAGS_POKE);
+#endif
+    }
+    else
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushPageManual);
+}
+
+
+/**
+ * Invalidates a guest page on all VCPUs.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCVirt      Page to invalidate.
+ */
+VMM_INT_DECL(int) HMInvalidatePageOnAllVCpus(PVMCC pVM, RTGCPTR GCVirt)
+{
+    /*
+     * The VT-x/AMD-V code will be flushing TLB each time a VCPU migrates to a different
+     * host CPU, see hmR0VmxFlushTaggedTlbBoth() and hmR0SvmFlushTaggedTlb().
+     *
+     * This is the reason why we do not care about thread preemption here and just
+     * execute HMInvalidatePage() assuming it might be the 'right' CPU.
+     */
+    VMCPUID const idCurCpu = VMMGetCpuId(pVM);
+    STAM_COUNTER_INC(&VMCC_GET_CPU(pVM, idCurCpu)->hm.s.StatFlushPage);
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+
+        /* Nothing to do if a TLB flush is already pending; the VCPU should
+           have already been poked if it were active. */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
+            continue;
+
+        if (pVCpu->idCpu == idCurCpu)
+            HMInvalidatePage(pVCpu, GCVirt);
+        else
+        {
+            hmQueueInvlPage(pVCpu, GCVirt);
+            hmPokeCpuForTlbFlush(pVCpu, false /* fAccountFlushStat */);
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Flush the TLBs of all VCPUs.
+ *
+ * @returns VBox status code.
+ * @param   pVM       The cross context VM structure.
+ */
+VMM_INT_DECL(int) HMFlushTlbOnAllVCpus(PVMCC pVM)
+{
+    if (pVM->cCpus == 1)
+        return HMFlushTlb(VMCC_GET_CPU_0(pVM));
+
+    VMCPUID const idThisCpu = VMMGetCpuId(pVM);
+
+    STAM_COUNTER_INC(&VMCC_GET_CPU(pVM, idThisCpu)->hm.s.StatFlushTlb);
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+
+        /* Nothing to do if a TLB flush is already pending; the VCPU should
+           have already been poked if it were active. */
+        if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
+        {
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+            if (idThisCpu != idCpu)
+                hmPokeCpuForTlbFlush(pVCpu, true /* fAccountFlushStat */);
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Invalidates a guest page by physical address.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      Page to invalidate.
+ *
+ * @remarks Assumes the current instruction references this physical page
+ *          though a virtual address!
+ */
+VMM_INT_DECL(int) HMInvalidatePhysPage(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    if (!HMIsNestedPagingActive(pVM))
+        return VINF_SUCCESS;
+
+    /*
+     * AMD-V: Doesn't support invalidation with guest physical addresses.
+     *
+     * VT-x: Doesn't support invalidation with guest physical addresses.
+     * INVVPID instruction takes only a linear address while invept only flushes by EPT
+     * not individual addresses.
+     *
+     * We update the force flag and flush before the next VM-entry, see @bugref{6568}.
+     */
+    RT_NOREF(GCPhys);
+    /** @todo Remove or figure out to way to update the Phys STAT counter.  */
+    /* STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys); */
+    return HMFlushTlbOnAllVCpus(pVM);
+}
+
+
+/**
+ * Checks if nested paging is enabled.
+ *
+ * @returns true if nested paging is active, false otherwise.
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks Works before hmR3InitFinalizeR0.
+ */
+VMM_INT_DECL(bool) HMIsNestedPagingActive(PVM pVM)
+{
+    return HMIsEnabled(pVM) && pVM->hm.s.fNestedPaging;
+}
+
+
+/**
+ * Checks if both nested paging and unhampered guest execution are enabled.
+ *
+ * The almost complete guest execution in hardware is only applicable to VT-x.
+ *
+ * @returns true if we have both enabled, otherwise false.
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks Works before hmR3InitFinalizeR0.
+ */
+VMM_INT_DECL(bool) HMAreNestedPagingAndFullGuestExecEnabled(PVM pVM)
+{
+    return HMIsEnabled(pVM)
+        && pVM->hm.s.fNestedPaging
+        && (   pVM->hm.s.vmx.fUnrestrictedGuest
+            || pVM->hm.s.svm.fSupported);
+}
+
+
+/**
+ * Checks if this VM is using HM and is long-mode capable.
+ *
+ * Use VMR3IsLongModeAllowed() instead of this, when possible.
+ *
+ * @returns true if long mode is allowed, false otherwise.
+ * @param   pVM         The cross context VM structure.
+ * @sa      VMR3IsLongModeAllowed, NEMHCIsLongModeAllowed
+ */
+VMM_INT_DECL(bool) HMIsLongModeAllowed(PVM pVM)
+{
+    return HMIsEnabled(pVM) && pVM->hm.s.fAllow64BitGuests;
+}
+
+
+/**
+ * Checks if MSR bitmaps are active. It is assumed that when it's available
+ * it will be used as well.
+ *
+ * @returns true if MSR bitmaps are available, false otherwise.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(bool) HMIsMsrBitmapActive(PVM pVM)
+{
+    if (HMIsEnabled(pVM))
+    {
+        if (pVM->hm.s.svm.fSupported)
+            return true;
+
+        if (   pVM->hm.s.vmx.fSupported
+            && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS))
+            return true;
+    }
+    return false;
+}
+
+
+/**
+ * Checks if AMD-V is active.
+ *
+ * @returns true if AMD-V is active.
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks Works before hmR3InitFinalizeR0.
+ */
+VMM_INT_DECL(bool) HMIsSvmActive(PVM pVM)
+{
+    return pVM->hm.s.svm.fSupported && HMIsEnabled(pVM);
+}
+
+
+/**
+ * Checks if VT-x is active.
+ *
+ * @returns true if VT-x is active.
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks Works before hmR3InitFinalizeR0.
+ */
+VMM_INT_DECL(bool) HMIsVmxActive(PVM pVM)
+{
+    return pVM->hm.s.vmx.fSupported && HMIsEnabled(pVM);
+}
+
+
+/**
+ * Checks if an interrupt event is currently pending.
+ *
+ * @returns Interrupt event pending state.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(bool) HMHasPendingIrq(PVMCC pVM)
+{
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    return !!pVCpu->hm.s.Event.fPending;
+}
+
+
+/**
+ * Return the PAE PDPE entries.
+ *
+ * @returns Pointer to the PAE PDPE array.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(PX86PDPE) HMGetPaePdpes(PVMCPU pVCpu)
+{
+    return &pVCpu->hm.s.aPdpes[0];
+}
+
+
+/**
+ * Sets or clears the single instruction flag.
+ *
+ * When set, HM will try its best to return to ring-3 after executing a single
+ * instruction.  This can be used for debugging.  See also
+ * EMR3HmSingleInstruction.
+ *
+ * @returns The old flag state.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   fEnable The new flag state.
+ */
+VMM_INT_DECL(bool) HMSetSingleInstruction(PVMCC pVM, PVMCPUCC pVCpu, bool fEnable)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    bool fOld = pVCpu->hm.s.fSingleInstruction;
+    pVCpu->hm.s.fSingleInstruction = fEnable;
+    pVCpu->hm.s.fUseDebugLoop = fEnable || pVM->hm.s.fUseDebugLoop;
+    return fOld;
+}
+
+
+/**
+ * Notification callback which is called whenever there is a chance that a CR3
+ * value might have changed.
+ *
+ * This is called by PGM.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmShadowMode   New shadow paging mode.
+ * @param   enmGuestMode    New guest paging mode.
+ */
+VMM_INT_DECL(void) HMHCChangedPagingMode(PVM pVM, PVMCPU pVCpu, PGMMODE enmShadowMode, PGMMODE enmGuestMode)
+{
+#ifdef IN_RING3
+    /* Ignore page mode changes during state loading. */
+    if (VMR3GetState(pVM) == VMSTATE_LOADING)
+        return;
+#endif
+
+    pVCpu->hm.s.enmShadowMode = enmShadowMode;
+
+    /*
+     * If the guest left protected mode VMX execution, we'll have to be
+     * extra careful if/when the guest switches back to protected mode.
+     */
+    if (enmGuestMode == PGMMODE_REAL)
+    {
+        PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+        pVmcsInfo->fWasInRealMode = true;
+    }
+
+#ifdef IN_RING0
+    /*
+     * We need to tickle SVM and VT-x state updates.
+     *
+     * Note! We could probably reduce this depending on what exactly changed.
+     */
+    if (VM_IS_HM_ENABLED(pVM))
+    {
+        CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER); /* No recursion! */
+        uint64_t fChanged = HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_CR3 | HM_CHANGED_GUEST_CR4 | HM_CHANGED_GUEST_EFER_MSR;
+        if (pVM->hm.s.svm.fSupported)
+            fChanged |= HM_CHANGED_SVM_XCPT_INTERCEPTS;
+        else
+            fChanged |= HM_CHANGED_VMX_XCPT_INTERCEPTS | HM_CHANGED_VMX_ENTRY_EXIT_CTLS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fChanged);
+    }
+#endif
+
+    Log4(("HMHCChangedPagingMode: Guest paging mode '%s', shadow paging mode '%s'\n", PGMGetModeName(enmGuestMode),
+          PGMGetModeName(enmShadowMode)));
+}
+
+
+/**
+ * Gets VMX MSRs from the provided hardware-virtualization MSRs struct.
+ *
+ * This abstraction exists to insulate the support driver from including VMX
+ * structures from HM headers.
+ *
+ * @param   pHwvirtMsrs     The hardware-virtualization MSRs.
+ * @param   pVmxMsrs        Where to store the VMX MSRs.
+ */
+VMM_INT_DECL(void) HMGetVmxMsrsFromHwvirtMsrs(PCSUPHWVIRTMSRS pHwvirtMsrs, PVMXMSRS pVmxMsrs)
+{
+    AssertReturnVoid(pHwvirtMsrs);
+    AssertReturnVoid(pVmxMsrs);
+    pVmxMsrs->u64FeatCtrl      = pHwvirtMsrs->u.vmx.u64FeatCtrl;
+    pVmxMsrs->u64Basic         = pHwvirtMsrs->u.vmx.u64Basic;
+    pVmxMsrs->PinCtls.u        = pHwvirtMsrs->u.vmx.u64PinCtls;
+    pVmxMsrs->ProcCtls.u       = pHwvirtMsrs->u.vmx.u64ProcCtls;
+    pVmxMsrs->ProcCtls2.u      = pHwvirtMsrs->u.vmx.u64ProcCtls2;
+    pVmxMsrs->ExitCtls.u       = pHwvirtMsrs->u.vmx.u64ExitCtls;
+    pVmxMsrs->EntryCtls.u      = pHwvirtMsrs->u.vmx.u64EntryCtls;
+    pVmxMsrs->TruePinCtls.u    = pHwvirtMsrs->u.vmx.u64TruePinCtls;
+    pVmxMsrs->TrueProcCtls.u   = pHwvirtMsrs->u.vmx.u64TrueProcCtls;
+    pVmxMsrs->TrueEntryCtls.u  = pHwvirtMsrs->u.vmx.u64TrueEntryCtls;
+    pVmxMsrs->TrueExitCtls.u   = pHwvirtMsrs->u.vmx.u64TrueExitCtls;
+    pVmxMsrs->u64Misc          = pHwvirtMsrs->u.vmx.u64Misc;
+    pVmxMsrs->u64Cr0Fixed0     = pHwvirtMsrs->u.vmx.u64Cr0Fixed0;
+    pVmxMsrs->u64Cr0Fixed1     = pHwvirtMsrs->u.vmx.u64Cr0Fixed1;
+    pVmxMsrs->u64Cr4Fixed0     = pHwvirtMsrs->u.vmx.u64Cr4Fixed0;
+    pVmxMsrs->u64Cr4Fixed1     = pHwvirtMsrs->u.vmx.u64Cr4Fixed1;
+    pVmxMsrs->u64VmcsEnum      = pHwvirtMsrs->u.vmx.u64VmcsEnum;
+    pVmxMsrs->u64VmFunc        = pHwvirtMsrs->u.vmx.u64VmFunc;
+    pVmxMsrs->u64EptVpidCaps   = pHwvirtMsrs->u.vmx.u64EptVpidCaps;
+}
+
+
+/**
+ * Gets SVM MSRs from the provided hardware-virtualization MSRs struct.
+ *
+ * This abstraction exists to insulate the support driver from including SVM
+ * structures from HM headers.
+ *
+ * @param   pHwvirtMsrs     The hardware-virtualization MSRs.
+ * @param   pSvmMsrs        Where to store the SVM MSRs.
+ */
+VMM_INT_DECL(void) HMGetSvmMsrsFromHwvirtMsrs(PCSUPHWVIRTMSRS pHwvirtMsrs, PSVMMSRS pSvmMsrs)
+{
+    AssertReturnVoid(pHwvirtMsrs);
+    AssertReturnVoid(pSvmMsrs);
+    pSvmMsrs->u64MsrHwcr = pHwvirtMsrs->u.svm.u64MsrHwcr;
+}
+
+
+/**
+ * Gets the name of a VT-x exit code.
+ *
+ * @returns Pointer to read only string if @a uExit is known, otherwise NULL.
+ * @param   uExit               The VT-x exit to name.
+ */
+VMM_INT_DECL(const char *) HMGetVmxExitName(uint32_t uExit)
+{
+    uint16_t const uReason = VMX_EXIT_REASON_BASIC(uExit);
+    if (uReason <= MAX_EXITREASON_VTX)
+    {
+        Assert(uReason < RT_ELEMENTS(g_apszVmxExitReasons));
+        return g_apszVmxExitReasons[uReason];
+    }
+    return NULL;
+}
+
+
+/**
+ * Gets the name of an AMD-V exit code.
+ *
+ * @returns Pointer to read only string if @a uExit is known, otherwise NULL.
+ * @param   uExit               The AMD-V exit to name.
+ */
+VMM_INT_DECL(const char *) HMGetSvmExitName(uint32_t uExit)
+{
+    if (uExit <= MAX_EXITREASON_AMDV)
+    {
+        Assert(uExit < RT_ELEMENTS(g_apszSvmExitReasons));
+        return g_apszSvmExitReasons[uExit];
+    }
+    return hmSvmGetSpecialExitReasonDesc(uExit);
+}
+
diff --git a/src/VBox/VMM/VMMAll/HMSVMAll.cpp b/src/VBox/VMM/VMMAll/HMSVMAll.cpp
new file mode 100644
index 00000000..3f168408
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/HMSVMAll.cpp
@@ -0,0 +1,521 @@
+/* $Id: HMSVMAll.cpp $ */
+/** @file
+ * HM SVM (AMD-V) - All contexts.
+ */
+
+/*
+ * Copyright (C) 2017-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_HM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include "HMInternal.h"
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/err.h>
+
+
+
+/**
+ * Emulates a simple MOV TPR (CR8) instruction.
+ *
+ * Used for TPR patching on 32-bit guests. This simply looks up the patch record
+ * at EIP and does the required.
+ *
+ * This VMMCALL is used a fallback mechanism when mov to/from cr8 isn't exactly
+ * like how we want it to be (e.g. not followed by shr 4 as is usually done for
+ * TPR). See hmR3ReplaceTprInstr() for the details.
+ *
+ * @returns VBox status code.
+ * @retval VINF_SUCCESS if the access was handled successfully, RIP + RFLAGS updated.
+ * @retval VERR_NOT_FOUND if no patch record for this RIP could be found.
+ * @retval VERR_SVM_UNEXPECTED_PATCH_TYPE if the found patch type is invalid.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(int) hmEmulateSvmMovTpr(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    Log4(("Emulated VMMCall TPR access replacement at RIP=%RGv\n", pCtx->rip));
+
+    /*
+     * We do this in a loop as we increment the RIP after a successful emulation
+     * and the new RIP may be a patched instruction which needs emulation as well.
+     */
+    bool fPatchFound = false;
+    for (;;)
+    {
+        PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
+        if (!pPatch)
+            break;
+        fPatchFound = true;
+
+        uint8_t u8Tpr;
+        switch (pPatch->enmType)
+        {
+            case HMTPRINSTR_READ:
+            {
+                bool fPending;
+                int  rc = APICGetTpr(pVCpu, &u8Tpr, &fPending, NULL /* pu8PendingIrq */);
+                AssertRC(rc);
+
+                rc = DISWriteReg32(CPUMCTX2CORE(pCtx), pPatch->uDstOperand, u8Tpr);
+                AssertRC(rc);
+                pCtx->rip += pPatch->cbOp;
+                pCtx->eflags.Bits.u1RF = 0;
+                break;
+            }
+
+            case HMTPRINSTR_WRITE_REG:
+            case HMTPRINSTR_WRITE_IMM:
+            {
+                if (pPatch->enmType == HMTPRINSTR_WRITE_REG)
+                {
+                    uint32_t u32Val;
+                    int rc = DISFetchReg32(CPUMCTX2CORE(pCtx), pPatch->uSrcOperand, &u32Val);
+                    AssertRC(rc);
+                    u8Tpr = u32Val;
+                }
+                else
+                    u8Tpr = (uint8_t)pPatch->uSrcOperand;
+
+                int rc2 = APICSetTpr(pVCpu, u8Tpr);
+                AssertRC(rc2);
+                pCtx->rip += pPatch->cbOp;
+                pCtx->eflags.Bits.u1RF = 0;
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR
+                                                         | HM_CHANGED_GUEST_RIP
+                                                         | HM_CHANGED_GUEST_RFLAGS);
+                break;
+            }
+
+            default:
+            {
+                AssertMsgFailed(("Unexpected patch type %d\n", pPatch->enmType));
+                pVCpu->hm.s.u32HMError = pPatch->enmType;
+                return VERR_SVM_UNEXPECTED_PATCH_TYPE;
+            }
+        }
+    }
+
+    return fPatchFound ? VINF_SUCCESS : VERR_NOT_FOUND;
+}
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+/**
+ * Notification callback for when a \#VMEXIT happens outside SVM R0 code (e.g.
+ * in IEM).
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pCtx            Pointer to the guest-CPU context.
+ *
+ * @sa      hmR0SvmVmRunCacheVmcb.
+ */
+VMM_INT_DECL(void) HMNotifySvmNstGstVmexit(PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+    PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        /*
+         * Restore fields as our own code might look at the VMCB controls as part
+         * of the #VMEXIT handling in IEM. Otherwise, strictly speaking we don't need to
+         * restore these fields because currently none of them are written back to memory
+         * by a physical CPU on #VMEXIT.
+         */
+        PSVMVMCBCTRL pVmcbNstGstCtrl = &pCtx->hwvirt.svm.CTX_SUFF(pVmcb)->ctrl;
+        pVmcbNstGstCtrl->u16InterceptRdCRx                 = pVmcbNstGstCache->u16InterceptRdCRx;
+        pVmcbNstGstCtrl->u16InterceptWrCRx                 = pVmcbNstGstCache->u16InterceptWrCRx;
+        pVmcbNstGstCtrl->u16InterceptRdDRx                 = pVmcbNstGstCache->u16InterceptRdDRx;
+        pVmcbNstGstCtrl->u16InterceptWrDRx                 = pVmcbNstGstCache->u16InterceptWrDRx;
+        pVmcbNstGstCtrl->u16PauseFilterThreshold           = pVmcbNstGstCache->u16PauseFilterThreshold;
+        pVmcbNstGstCtrl->u16PauseFilterCount               = pVmcbNstGstCache->u16PauseFilterCount;
+        pVmcbNstGstCtrl->u32InterceptXcpt                  = pVmcbNstGstCache->u32InterceptXcpt;
+        pVmcbNstGstCtrl->u64InterceptCtrl                  = pVmcbNstGstCache->u64InterceptCtrl;
+        pVmcbNstGstCtrl->u64TSCOffset                      = pVmcbNstGstCache->u64TSCOffset;
+        pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking          = pVmcbNstGstCache->fVIntrMasking;
+        pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVmcbNstGstCache->fNestedPaging;
+        pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt               = pVmcbNstGstCache->fLbrVirt;
+        pVmcbNstGstCache->fCacheValid = false;
+    }
+
+    /*
+     * Transitions to ring-3 flag a full CPU-state change except if we transition to ring-3
+     * in response to a physical CPU interrupt as no changes to the guest-CPU state are
+     * expected (see VINF_EM_RAW_INTERRUPT handling in hmR0SvmExitToRing3).
+     *
+     * However, with nested-guests, the state -can- change on trips to ring-3 for we might
+     * try to inject a nested-guest physical interrupt and cause a SVM_EXIT_INTR #VMEXIT for
+     * the nested-guest from ring-3. Import the complete state here as we will be swapping
+     * to the guest VMCB after the #VMEXIT.
+     */
+    CPUMImportGuestStateOnDemand(pVCpu, CPUMCTX_EXTRN_ALL);
+    CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_ALL);
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+}
+#endif
+
+/**
+ * Checks if the Virtual GIF (Global Interrupt Flag) feature is supported and
+ * enabled for the VM.
+ *
+ * @returns @c true if VGIF is enabled, @c false otherwise.
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks This value returned by this functions is expected by the callers not
+ *          to change throughout the lifetime of the VM.
+ */
+VMM_INT_DECL(bool) HMIsSvmVGifActive(PCVM pVM)
+{
+    bool const fVGif    = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VGIF);
+    bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
+    return fVGif && fUseVGif;
+}
+
+
+/**
+ * Interface used by IEM to handle patched TPR accesses.
+ *
+ * @returns VBox status code
+ * @retval  VINF_SUCCESS if hypercall was handled, RIP + RFLAGS all dealt with.
+ * @retval  VERR_NOT_FOUND if hypercall was _not_ handled.
+ * @retval  VERR_SVM_UNEXPECTED_PATCH_TYPE on IPE.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(int) HMHCMaybeMovTprSvmHypercall(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    if (pVM->hm.s.fTprPatchingAllowed)
+    {
+        int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
+        if (RT_SUCCESS(rc))
+            return VINF_SUCCESS;
+        return rc;
+    }
+    return VERR_NOT_FOUND;
+}
+
+
+/**
+ * Checks if the current AMD CPU is subject to erratum 170 "In SVM mode,
+ * incorrect code bytes may be fetched after a world-switch".
+ *
+ * @param   pu32Family      Where to store the CPU family (can be NULL).
+ * @param   pu32Model       Where to store the CPU model (can be NULL).
+ * @param   pu32Stepping    Where to store the CPU stepping (can be NULL).
+ * @returns true if the erratum applies, false otherwise.
+ */
+VMM_INT_DECL(int) HMIsSubjectToSvmErratum170(uint32_t *pu32Family, uint32_t *pu32Model, uint32_t *pu32Stepping)
+{
+    /*
+     * Erratum 170 which requires a forced TLB flush for each world switch:
+     * See AMD spec. "Revision Guide for AMD NPT Family 0Fh Processors".
+     *
+     * All BH-G1/2 and DH-G1/2 models include a fix:
+     * Athlon X2:   0x6b 1/2
+     *              0x68 1/2
+     * Athlon 64:   0x7f 1
+     *              0x6f 2
+     * Sempron:     0x7f 1/2
+     *              0x6f 2
+     *              0x6c 2
+     *              0x7c 2
+     * Turion 64:   0x68 2
+     */
+    uint32_t u32Dummy;
+    uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
+    ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
+    u32BaseFamily = (u32Version >> 8) & 0xf;
+    u32Family     = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
+    u32Model      = ((u32Version >> 4) & 0xf);
+    u32Model      = u32Model | ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
+    u32Stepping   = u32Version & 0xf;
+
+    bool fErratumApplies = false;
+    if (   u32Family == 0xf
+        && !((u32Model == 0x68 || u32Model == 0x6b || u32Model == 0x7f) && u32Stepping >= 1)
+        && !((u32Model == 0x6f || u32Model == 0x6c || u32Model == 0x7c) && u32Stepping >= 2))
+    {
+        fErratumApplies = true;
+    }
+
+    if (pu32Family)
+        *pu32Family   = u32Family;
+    if (pu32Model)
+        *pu32Model    = u32Model;
+    if (pu32Stepping)
+        *pu32Stepping = u32Stepping;
+
+    return fErratumApplies;
+}
+
+
+
+/**
+ * Converts an SVM event type to a TRPM event type.
+ *
+ * @returns The TRPM event type.
+ * @retval  TRPM_32BIT_HACK if the specified type of event isn't among the set
+ *          of recognized trap types.
+ *
+ * @param   pEvent       Pointer to the SVM event.
+ * @param   uVector      The vector associated with the event.
+ */
+VMM_INT_DECL(TRPMEVENT) HMSvmEventToTrpmEventType(PCSVMEVENT pEvent, uint8_t uVector)
+{
+    uint8_t const uType = pEvent->n.u3Type;
+    switch (uType)
+    {
+        case SVM_EVENT_EXTERNAL_IRQ:    return TRPM_HARDWARE_INT;
+        case SVM_EVENT_SOFTWARE_INT:    return TRPM_SOFTWARE_INT;
+        case SVM_EVENT_NMI:             return TRPM_TRAP;
+        case SVM_EVENT_EXCEPTION:
+        {
+            if (   uVector == X86_XCPT_BP
+                || uVector == X86_XCPT_OF)
+                return TRPM_SOFTWARE_INT;
+            return TRPM_TRAP;
+        }
+        default:
+            break;
+    }
+    AssertMsgFailed(("HMSvmEventToTrpmEvent: Invalid pending-event type %#x\n", uType));
+    return TRPM_32BIT_HACK;
+}
+
+
+/**
+ * Gets the SVM nested-guest control intercepts if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu64Intercepts  Where to store the control intercepts. Only updated when
+ *                          @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmCtrlIntercepts(PCVMCPU pVCpu, uint64_t *pu64Intercepts)
+{
+    Assert(pu64Intercepts);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu64Intercepts = pVmcbNstGstCache->u64InterceptCtrl;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the SVM nested-guest CRx-read intercepts if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu16Intercepts  Where to store the CRx-read intercepts. Only updated
+ *                          when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmReadCRxIntercepts(PCVMCPU pVCpu, uint16_t *pu16Intercepts)
+{
+    Assert(pu16Intercepts);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu16Intercepts = pVmcbNstGstCache->u16InterceptRdCRx;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the SVM nested-guest CRx-write intercepts if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu16Intercepts  Where to store the CRx-write intercepts. Only updated
+ *                          when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmWriteCRxIntercepts(PCVMCPU pVCpu, uint16_t *pu16Intercepts)
+{
+    Assert(pu16Intercepts);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu16Intercepts = pVmcbNstGstCache->u16InterceptWrCRx;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the SVM nested-guest DRx-read intercepts if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu16Intercepts  Where to store the DRx-read intercepts. Only updated
+ *                          when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmReadDRxIntercepts(PCVMCPU pVCpu, uint16_t *pu16Intercepts)
+{
+    Assert(pu16Intercepts);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu16Intercepts = pVmcbNstGstCache->u16InterceptRdDRx;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the SVM nested-guest DRx-write intercepts if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu16Intercepts  Where to store the DRx-write intercepts. Only updated
+ *                          when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmWriteDRxIntercepts(PCVMCPU pVCpu, uint16_t *pu16Intercepts)
+{
+    Assert(pu16Intercepts);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu16Intercepts = pVmcbNstGstCache->u16InterceptWrDRx;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the SVM nested-guest exception intercepts if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu32Intercepts  Where to store the exception intercepts. Only updated
+ *                          when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmXcptIntercepts(PCVMCPU pVCpu, uint32_t *pu32Intercepts)
+{
+    Assert(pu32Intercepts);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu32Intercepts = pVmcbNstGstCache->u32InterceptXcpt;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Checks if the nested-guest VMCB has virtual-interrupts masking enabled.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pfVIntrMasking  Where to store the virtual-interrupt masking bit.
+ *                          Updated only when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmVirtIntrMasking(PCVMCPU pVCpu, bool *pfVIntrMasking)
+{
+    Assert(pfVIntrMasking);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pfVIntrMasking = pVmcbNstGstCache->fVIntrMasking;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the SVM nested-guest nested-paging bit if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling EMT.
+ * @param   pfNestedPaging      Where to store the nested-paging bit. Updated only
+ *                              when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmNestedPaging(PCVMCPU pVCpu, bool *pfNestedPaging)
+{
+    Assert(pfNestedPaging);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pfNestedPaging = pVmcbNstGstCache->fNestedPaging;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Returns the nested-guest VMCB pause-filter count.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu                   The cross context virtual CPU structure of the
+ *                                  calling EMT.
+ * @param   pu16PauseFilterCount    Where to store the pause-filter count. Only
+ *                                  updated @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmPauseFilterCount(PCVMCPU pVCpu, uint16_t *pu16PauseFilterCount)
+{
+    Assert(pu16PauseFilterCount);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu16PauseFilterCount = pVmcbNstGstCache->u16PauseFilterCount;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Returns the SVM nested-guest TSC offset if cached by HM.
+ *
+ * @returns The TSC offset after applying any nested-guest TSC offset.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu64TscOffset   Where to store the TSC offset. Only updated when @c
+ *                          true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmTscOffset(PCVMCPU pVCpu, uint64_t *pu64TscOffset)
+{
+    Assert(pu64TscOffset);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu64TscOffset = pVmcbNstGstCache->u64TSCOffset;
+        return true;
+    }
+    return false;
+}
+
diff --git a/src/VBox/VMM/VMMAll/HMVMXAll.cpp b/src/VBox/VMM/VMMAll/HMVMXAll.cpp
new file mode 100644
index 00000000..e82a503b
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/HMVMXAll.cpp
@@ -0,0 +1,1315 @@
+/* $Id: HMVMXAll.cpp $ */
+/** @file
+ * HM VMX (VT-x) - All contexts.
+ */
+
+/*
+ * Copyright (C) 2018-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_HM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include "HMInternal.h"
+#include <VBox/vmm/hmvmxinline.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/pdmapi.h>
+#include <iprt/errcore.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#define VMXV_DIAG_DESC(a_Def, a_Desc)      #a_Def " - " #a_Desc
+/** VMX virtual-instructions and VM-exit diagnostics. */
+static const char * const g_apszVmxVDiagDesc[] =
+{
+    /* Internal processing errors. */
+    VMXV_DIAG_DESC(kVmxVDiag_None                             , "None"                      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_1                            , "Ipe_1"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_2                            , "Ipe_2"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_3                            , "Ipe_3"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_4                            , "Ipe_4"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_5                            , "Ipe_5"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_6                            , "Ipe_6"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_7                            , "Ipe_7"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_8                            , "Ipe_8"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_9                            , "Ipe_9"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_10                           , "Ipe_10"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_11                           , "Ipe_11"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_12                           , "Ipe_12"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_13                           , "Ipe_13"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_14                           , "Ipe_14"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_15                           , "Ipe_15"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_16                           , "Ipe_16"                    ),
+    /* VMXON. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_A20M                       , "A20M"                      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cpl                        , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cr0Fixed0                  , "Cr0Fixed0"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cr0Fixed1                  , "Cr0Fixed1"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cr4Fixed0                  , "Cr4Fixed0"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cr4Fixed1                  , "Cr4Fixed1"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Intercept                  , "Intercept"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_LongModeCS                 , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_MsrFeatCtl                 , "MsrFeatCtl"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrAbnormal                , "PtrAbnormal"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrAlign                   , "PtrAlign"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrMap                     , "PtrMap"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrReadPhys                , "PtrReadPhys"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrWidth                   , "PtrWidth"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_RealOrV86Mode              , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_ShadowVmcs                 , "ShadowVmcs"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_VmxAlreadyRoot             , "VmxAlreadyRoot"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Vmxe                       , "Vmxe"                      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_VmcsRevId                  , "VmcsRevId"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_VmxRootCpl                 , "VmxRootCpl"                ),
+    /* VMXOFF. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_Cpl                       , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_Intercept                 , "Intercept"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_LongModeCS                , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_RealOrV86Mode             , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_Vmxe                      , "Vmxe"                      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_VmxRoot                   , "VmxRoot"                   ),
+    /* VMPTRLD. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_Cpl                      , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_LongModeCS               , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrAbnormal              , "PtrAbnormal"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrAlign                 , "PtrAlign"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrMap                   , "PtrMap"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrReadPhys              , "PtrReadPhys"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrVmxon                 , "PtrVmxon"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrWidth                 , "PtrWidth"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_RealOrV86Mode            , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_RevPtrReadPhys           , "RevPtrReadPhys"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_ShadowVmcs               , "ShadowVmcs"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_VmcsRevId                , "VmcsRevId"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_VmxRoot                  , "VmxRoot"                   ),
+    /* VMPTRST. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_Cpl                      , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_LongModeCS               , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_PtrMap                   , "PtrMap"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_RealOrV86Mode            , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_VmxRoot                  , "VmxRoot"                   ),
+    /* VMCLEAR. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_Cpl                      , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_LongModeCS               , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrAbnormal              , "PtrAbnormal"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrAlign                 , "PtrAlign"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrMap                   , "PtrMap"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrReadPhys              , "PtrReadPhys"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrVmxon                 , "PtrVmxon"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrWidth                 , "PtrWidth"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_RealOrV86Mode            , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_VmxRoot                  , "VmxRoot"                   ),
+    /* VMWRITE. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_Cpl                      , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_FieldInvalid             , "FieldInvalid"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_FieldRo                  , "FieldRo"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_LinkPtrInvalid           , "LinkPtrInvalid"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_LongModeCS               , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_PtrInvalid               , "PtrInvalid"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_PtrMap                   , "PtrMap"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_RealOrV86Mode            , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_VmxRoot                  , "VmxRoot"                   ),
+    /* VMREAD. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_Cpl                       , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_FieldInvalid              , "FieldInvalid"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_LinkPtrInvalid            , "LinkPtrInvalid"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_LongModeCS                , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_PtrInvalid                , "PtrInvalid"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_PtrMap                    , "PtrMap"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_RealOrV86Mode             , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_VmxRoot                   , "VmxRoot"                   ),
+    /* INVVPID. */
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Cpl                      , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_DescRsvd                 , "DescRsvd"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_LongModeCS               , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_RealOrV86Mode            , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_TypeInvalid              , "TypeInvalid"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Type0InvalidAddr         , "Type0InvalidAddr"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Type0InvalidVpid         , "Type0InvalidVpid"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Type1InvalidVpid         , "Type1InvalidVpid"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Type3InvalidVpid         , "Type3InvalidVpid"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_VmxRoot                  , "VmxRoot"                   ),
+    /* VMLAUNCH/VMRESUME. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrApicAccess           , "AddrApicAccess"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrApicAccessEqVirtApic , "AddrApicAccessEqVirtApic"  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrApicAccessHandlerReg , "AddrApicAccessHandlerReg"  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrEntryMsrLoad         , "AddrEntryMsrLoad"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrExitMsrLoad          , "AddrExitMsrLoad"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrExitMsrStore         , "AddrExitMsrStore"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrIoBitmapA            , "AddrIoBitmapA"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrIoBitmapB            , "AddrIoBitmapB"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrMsrBitmap            , "AddrMsrBitmap"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrVirtApicPage         , "AddrVirtApicPage"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrVmcsLinkPtr          , "AddrVmcsLinkPtr"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrVmreadBitmap         , "AddrVmreadBitmap"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrVmwriteBitmap        , "AddrVmwriteBitmap"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ApicRegVirt              , "ApicRegVirt"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_BlocKMovSS               , "BlockMovSS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_Cpl                      , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_Cr3TargetCount           , "Cr3TargetCount"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryCtlsAllowed1        , "EntryCtlsAllowed1"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryCtlsDisallowed0     , "EntryCtlsDisallowed0"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryInstrLen            , "EntryInstrLen"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryInstrLenZero        , "EntryInstrLenZero"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryIntInfoErrCodePe    , "EntryIntInfoErrCodePe"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryIntInfoErrCodeVec   , "EntryIntInfoErrCodeVec"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryIntInfoTypeVecRsvd  , "EntryIntInfoTypeVecRsvd"   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryXcptErrCodeRsvd     , "EntryXcptErrCodeRsvd"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ExitCtlsAllowed1         , "ExitCtlsAllowed1"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ExitCtlsDisallowed0      , "ExitCtlsDisallowed0"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateHlt         , "GuestActStateHlt"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateRsvd        , "GuestActStateRsvd"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateShutdown    , "GuestActStateShutdown"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateSsDpl       , "GuestActStateSsDpl"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateStiMovSs    , "GuestActStateStiMovSs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr0Fixed0           , "GuestCr0Fixed0"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr0Fixed1           , "GuestCr0Fixed1"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr0PgPe             , "GuestCr0PgPe"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr3                 , "GuestCr3"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr4Fixed0           , "GuestCr4Fixed0"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr4Fixed1           , "GuestCr4Fixed1"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestDebugCtl            , "GuestDebugCtl"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestDr7                 , "GuestDr7"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestEferMsr             , "GuestEferMsr"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestEferMsrRsvd         , "GuestEferMsrRsvd"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestGdtrBase            , "GuestGdtrBase"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestGdtrLimit           , "GuestGdtrLimit"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIdtrBase            , "GuestIdtrBase"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIdtrLimit           , "GuestIdtrLimit"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateEnclave     , "GuestIntStateEnclave"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateExtInt      , "GuestIntStateExtInt"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateNmi         , "GuestIntStateNmi"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateRFlagsSti   , "GuestIntStateRFlagsSti"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateRsvd        , "GuestIntStateRsvd"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateSmi         , "GuestIntStateSmi"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateStiMovSs    , "GuestIntStateStiMovSs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateVirtNmi     , "GuestIntStateVirtNmi"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPae                 , "GuestPae"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPatMsr              , "GuestPatMsr"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPcide               , "GuestPcide"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPdpteCr3ReadPhys    , "GuestPdpteCr3ReadPhys"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPdpte0Rsvd          , "GuestPdpte0Rsvd"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPdpte1Rsvd          , "GuestPdpte1Rsvd"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPdpte2Rsvd          , "GuestPdpte2Rsvd"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPdpte3Rsvd          , "GuestPdpte3Rsvd"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPndDbgXcptBsNoTf    , "GuestPndDbgXcptBsNoTf"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPndDbgXcptBsTf      , "GuestPndDbgXcptBsTf"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPndDbgXcptRsvd      , "GuestPndDbgXcptRsvd"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPndDbgXcptRtm       , "GuestPndDbgXcptRtm"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRip                 , "GuestRip"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRipRsvd             , "GuestRipRsvd"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRFlagsIf            , "GuestRFlagsIf"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRFlagsRsvd          , "GuestRFlagsRsvd"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRFlagsVm            , "GuestRFlagsVm"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsDefBig     , "GuestSegAttrCsDefBig"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsDplEqSs    , "GuestSegAttrCsDplEqSs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsDplLtSs    , "GuestSegAttrCsDplLtSs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsDplZero    , "GuestSegAttrCsDplZero"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsType       , "GuestSegAttrCsType"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsTypeRead   , "GuestSegAttrCsTypeRead"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeCs   , "GuestSegAttrDescTypeCs"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeDs   , "GuestSegAttrDescTypeDs"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeEs   , "GuestSegAttrDescTypeEs"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeFs   , "GuestSegAttrDescTypeFs"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeGs   , "GuestSegAttrDescTypeGs"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeSs   , "GuestSegAttrDescTypeSs"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplCs     , "GuestSegAttrDplRplCs"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplDs     , "GuestSegAttrDplRplDs"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplEs     , "GuestSegAttrDplRplEs"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplFs     , "GuestSegAttrDplRplFs"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplGs     , "GuestSegAttrDplRplGs"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplSs     , "GuestSegAttrDplRplSs"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranCs       , "GuestSegAttrGranCs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranDs       , "GuestSegAttrGranDs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranEs       , "GuestSegAttrGranEs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranFs       , "GuestSegAttrGranFs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranGs       , "GuestSegAttrGranGs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranSs       , "GuestSegAttrGranSs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrDescType , "GuestSegAttrLdtrDescType"  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrGran     , "GuestSegAttrLdtrGran"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrPresent  , "GuestSegAttrLdtrPresent"   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrRsvd     , "GuestSegAttrLdtrRsvd"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrType     , "GuestSegAttrLdtrType"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentCs    , "GuestSegAttrPresentCs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentDs    , "GuestSegAttrPresentDs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentEs    , "GuestSegAttrPresentEs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentFs    , "GuestSegAttrPresentFs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentGs    , "GuestSegAttrPresentGs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentSs    , "GuestSegAttrPresentSs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdCs       , "GuestSegAttrRsvdCs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdDs       , "GuestSegAttrRsvdDs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdEs       , "GuestSegAttrRsvdEs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdFs       , "GuestSegAttrRsvdFs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdGs       , "GuestSegAttrRsvdGs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdSs       , "GuestSegAttrRsvdSs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrSsDplEqRpl   , "GuestSegAttrSsDplEqRpl"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrSsDplZero    , "GuestSegAttrSsDplZero "    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrSsType       , "GuestSegAttrSsType"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrDescType   , "GuestSegAttrTrDescType"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrGran       , "GuestSegAttrTrGran"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrPresent    , "GuestSegAttrTrPresent"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrRsvd       , "GuestSegAttrTrRsvd"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrType       , "GuestSegAttrTrType"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrUnusable   , "GuestSegAttrTrUnusable"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccCs    , "GuestSegAttrTypeAccCs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccDs    , "GuestSegAttrTypeAccDs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccEs    , "GuestSegAttrTypeAccEs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccFs    , "GuestSegAttrTypeAccFs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccGs    , "GuestSegAttrTypeAccGs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccSs    , "GuestSegAttrTypeAccSs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Cs        , "GuestSegAttrV86Cs"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Ds        , "GuestSegAttrV86Ds"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Es        , "GuestSegAttrV86Es"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Fs        , "GuestSegAttrV86Fs"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Gs        , "GuestSegAttrV86Gs"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Ss        , "GuestSegAttrV86Ss"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseCs           , "GuestSegBaseCs"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseDs           , "GuestSegBaseDs"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseEs           , "GuestSegBaseEs"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseFs           , "GuestSegBaseFs"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseGs           , "GuestSegBaseGs"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseLdtr         , "GuestSegBaseLdtr"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseSs           , "GuestSegBaseSs"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseTr           , "GuestSegBaseTr"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Cs        , "GuestSegBaseV86Cs"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Ds        , "GuestSegBaseV86Ds"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Es        , "GuestSegBaseV86Es"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Fs        , "GuestSegBaseV86Fs"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Gs        , "GuestSegBaseV86Gs"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Ss        , "GuestSegBaseV86Ss"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Cs       , "GuestSegLimitV86Cs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Ds       , "GuestSegLimitV86Ds"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Es       , "GuestSegLimitV86Es"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Fs       , "GuestSegLimitV86Fs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Gs       , "GuestSegLimitV86Gs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Ss       , "GuestSegLimitV86Ss"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegSelCsSsRpl       , "GuestSegSelCsSsRpl"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegSelLdtr          , "GuestSegSelLdtr"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegSelTr            , "GuestSegSelTr"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSysenterEspEip      , "GuestSysenterEspEip"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLinkPtrCurVmcs       , "VmcsLinkPtrCurVmcs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLinkPtrReadPhys      , "VmcsLinkPtrReadPhys"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLinkPtrRevId         , "VmcsLinkPtrRevId"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLinkPtrShadow        , "VmcsLinkPtrShadow"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr0Fixed0            , "HostCr0Fixed0"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr0Fixed1            , "HostCr0Fixed1"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr3                  , "HostCr3"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr4Fixed0            , "HostCr4Fixed0"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr4Fixed1            , "HostCr4Fixed1"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr4Pae               , "HostCr4Pae"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr4Pcide             , "HostCr4Pcide"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCsTr                 , "HostCsTr"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostEferMsr              , "HostEferMsr"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostEferMsrRsvd          , "HostEferMsrRsvd"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostGuestLongMode        , "HostGuestLongMode"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostGuestLongModeNoCpu   , "HostGuestLongModeNoCpu"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostLongMode             , "HostLongMode"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostPatMsr               , "HostPatMsr"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostRip                  , "HostRip"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostRipRsvd              , "HostRipRsvd"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostSel                  , "HostSel"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostSegBase              , "HostSegBase"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostSs                   , "HostSs"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostSysenterEspEip       , "HostSysenterEspEip"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_IoBitmapAPtrReadPhys     , "IoBitmapAPtrReadPhys"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_IoBitmapBPtrReadPhys     , "IoBitmapBPtrReadPhys"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_LongModeCS               , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrBitmapPtrReadPhys     , "MsrBitmapPtrReadPhys"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoad                  , "MsrLoad"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoadCount             , "MsrLoadCount"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoadPtrReadPhys       , "MsrLoadPtrReadPhys"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoadRing3             , "MsrLoadRing3"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoadRsvd              , "MsrLoadRsvd"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_NmiWindowExit            , "NmiWindowExit"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_PinCtlsAllowed1          , "PinCtlsAllowed1"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_PinCtlsDisallowed0       , "PinCtlsDisallowed0"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ProcCtlsAllowed1         , "ProcCtlsAllowed1"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ProcCtlsDisallowed0      , "ProcCtlsDisallowed0"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ProcCtls2Allowed1        , "ProcCtls2Allowed1"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ProcCtls2Disallowed0     , "ProcCtls2Disallowed0"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_PtrInvalid               , "PtrInvalid"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_PtrShadowVmcs            , "PtrShadowVmcs"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_RealOrV86Mode            , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_SavePreemptTimer         , "SavePreemptTimer"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_TprThresholdRsvd         , "TprThresholdRsvd"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_TprThresholdVTpr         , "TprThresholdVTpr"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtApicPagePtrReadPhys  , "VirtApicPageReadPhys"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtIntDelivery          , "VirtIntDelivery"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtNmi                  , "VirtNmi"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtX2ApicTprShadow      , "VirtX2ApicTprShadow"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtX2ApicVirtApic       , "VirtX2ApicVirtApic"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsClear                , "VmcsClear"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLaunch               , "VmcsLaunch"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmreadBitmapPtrReadPhys  , "VmreadBitmapPtrReadPhys"   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmwriteBitmapPtrReadPhys , "VmwriteBitmapPtrReadPhys"  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmxRoot                  , "VmxRoot"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_Vpid                     , "Vpid"                      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_HostPdpteCr3ReadPhys      , "HostPdpteCr3ReadPhys"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_HostPdpte0Rsvd            , "HostPdpte0Rsvd"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_HostPdpte1Rsvd            , "HostPdpte1Rsvd"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_HostPdpte2Rsvd            , "HostPdpte2Rsvd"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_HostPdpte3Rsvd            , "HostPdpte3Rsvd"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoad                   , "MsrLoad"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoadCount              , "MsrLoadCount"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoadPtrReadPhys        , "MsrLoadPtrReadPhys"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoadRing3              , "MsrLoadRing3"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoadRsvd               , "MsrLoadRsvd"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStore                  , "MsrStore"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStoreCount             , "MsrStoreCount"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStorePtrReadPhys       , "MsrStorePtrReadPhys"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStorePtrWritePhys      , "MsrStorePtrWritePhys"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStoreRing3             , "MsrStoreRing3"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStoreRsvd              , "MsrStoreRsvd"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_VirtApicPagePtrWritePhys  , "VirtApicPagePtrWritePhys"  )
+    /* kVmxVDiag_End */
+};
+AssertCompile(RT_ELEMENTS(g_apszVmxVDiagDesc) == kVmxVDiag_End);
+#undef VMXV_DIAG_DESC
+
+
+/**
+ * Gets the descriptive name of a VMX instruction/VM-exit diagnostic code.
+ *
+ * @returns The descriptive string.
+ * @param   enmDiag    The VMX diagnostic.
+ */
+VMM_INT_DECL(const char *) HMGetVmxDiagDesc(VMXVDIAG enmDiag)
+{
+    if (RT_LIKELY((unsigned)enmDiag < RT_ELEMENTS(g_apszVmxVDiagDesc)))
+        return g_apszVmxVDiagDesc[enmDiag];
+    return "Unknown/invalid";
+}
+
+
+/**
+ * Checks if a code selector (CS) is suitable for execution using hardware-assisted
+ * VMX when unrestricted execution isn't available.
+ *
+ * @returns true if selector is suitable for VMX, otherwise
+ *        false.
+ * @param   pSel        Pointer to the selector to check (CS).
+ * @param   uStackDpl   The CPL, aka the DPL of the stack segment.
+ */
+static bool hmVmxIsCodeSelectorOk(PCCPUMSELREG pSel, unsigned uStackDpl)
+{
+    /*
+     * Segment must be an accessed code segment, it must be present and it must
+     * be usable.
+     * Note! These are all standard requirements and if CS holds anything else
+     *       we've got buggy code somewhere!
+     */
+    AssertCompile(X86DESCATTR_TYPE == 0xf);
+    AssertMsgReturn(   (pSel->Attr.u & (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_CODE | X86DESCATTR_DT | X86DESCATTR_P | X86DESCATTR_UNUSABLE))
+                    ==                 (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_CODE | X86DESCATTR_DT | X86DESCATTR_P),
+                    ("%#x\n", pSel->Attr.u),
+                    false);
+
+    /*
+     * For conforming segments, CS.DPL must be <= SS.DPL, while CS.DPL must equal
+     * SS.DPL for non-confroming segments.
+     * Note! This is also a hard requirement like above.
+     */
+    AssertMsgReturn(  pSel->Attr.n.u4Type & X86_SEL_TYPE_CONF
+                    ? pSel->Attr.n.u2Dpl <= uStackDpl
+                    : pSel->Attr.n.u2Dpl == uStackDpl,
+                    ("u4Type=%#x u2Dpl=%u uStackDpl=%u\n", pSel->Attr.n.u4Type, pSel->Attr.n.u2Dpl, uStackDpl),
+                    false);
+
+    /*
+     * The following two requirements are VT-x specific:
+     *  - G bit must be set if any high limit bits are set.
+     *  - G bit must be clear if any low limit bits are clear.
+     */
+    if (   ((pSel->u32Limit & 0xfff00000) == 0x00000000 ||  pSel->Attr.n.u1Granularity)
+        && ((pSel->u32Limit & 0x00000fff) == 0x00000fff || !pSel->Attr.n.u1Granularity))
+        return true;
+    return false;
+}
+
+
+/**
+ * Checks if a data selector (DS/ES/FS/GS) is suitable for execution using
+ * hardware-assisted VMX when unrestricted execution isn't available.
+ *
+ * @returns true if selector is suitable for VMX, otherwise
+ *        false.
+ * @param   pSel        Pointer to the selector to check
+ *                      (DS/ES/FS/GS).
+ */
+static bool hmVmxIsDataSelectorOk(PCCPUMSELREG pSel)
+{
+    /*
+     * Unusable segments are OK.  These days they should be marked as such, as
+     * but as an alternative we for old saved states and AMD<->VT-x migration
+     * we also treat segments with all the attributes cleared as unusable.
+     */
+    if (pSel->Attr.n.u1Unusable || !pSel->Attr.u)
+        return true;
+
+    /** @todo tighten these checks. Will require CPUM load adjusting. */
+
+    /* Segment must be accessed. */
+    if (pSel->Attr.u & X86_SEL_TYPE_ACCESSED)
+    {
+        /* Code segments must also be readable. */
+        if (  !(pSel->Attr.u & X86_SEL_TYPE_CODE)
+            || (pSel->Attr.u & X86_SEL_TYPE_READ))
+        {
+            /* The S bit must be set. */
+            if (pSel->Attr.n.u1DescType)
+            {
+                /* Except for conforming segments, DPL >= RPL. */
+                if (   pSel->Attr.n.u2Dpl  >= (pSel->Sel & X86_SEL_RPL)
+                    || pSel->Attr.n.u4Type >= X86_SEL_TYPE_ER_ACC)
+                {
+                    /* Segment must be present. */
+                    if (pSel->Attr.n.u1Present)
+                    {
+                        /*
+                         * The following two requirements are VT-x specific:
+                         *   - G bit must be set if any high limit bits are set.
+                         *   - G bit must be clear if any low limit bits are clear.
+                         */
+                        if (   ((pSel->u32Limit & 0xfff00000) == 0x00000000 ||  pSel->Attr.n.u1Granularity)
+                            && ((pSel->u32Limit & 0x00000fff) == 0x00000fff || !pSel->Attr.n.u1Granularity))
+                            return true;
+                    }
+                }
+            }
+        }
+    }
+
+    return false;
+}
+
+
+/**
+ * Checks if the stack selector (SS) is suitable for execution using
+ * hardware-assisted VMX when unrestricted execution isn't available.
+ *
+ * @returns true if selector is suitable for VMX, otherwise
+ *        false.
+ * @param   pSel        Pointer to the selector to check (SS).
+ */
+static bool hmVmxIsStackSelectorOk(PCCPUMSELREG pSel)
+{
+    /*
+     * Unusable segments are OK.  These days they should be marked as such, as
+     * but as an alternative we for old saved states and AMD<->VT-x migration
+     * we also treat segments with all the attributes cleared as unusable.
+     */
+    /** @todo r=bird: actually all zeroes isn't gonna cut it... SS.DPL == CPL. */
+    if (pSel->Attr.n.u1Unusable || !pSel->Attr.u)
+        return true;
+
+    /*
+     * Segment must be an accessed writable segment, it must be present.
+     * Note! These are all standard requirements and if SS holds anything else
+     *       we've got buggy code somewhere!
+     */
+    AssertCompile(X86DESCATTR_TYPE == 0xf);
+    AssertMsgReturn(   (pSel->Attr.u & (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_WRITE | X86DESCATTR_DT | X86DESCATTR_P | X86_SEL_TYPE_CODE))
+                    ==                 (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_WRITE | X86DESCATTR_DT | X86DESCATTR_P),
+                    ("%#x\n", pSel->Attr.u), false);
+
+    /*
+     * DPL must equal RPL. But in real mode or soon after enabling protected
+     * mode, it might not be.
+     */
+    if (pSel->Attr.n.u2Dpl == (pSel->Sel & X86_SEL_RPL))
+    {
+        /*
+         * The following two requirements are VT-x specific:
+         *   - G bit must be set if any high limit bits are set.
+         *   - G bit must be clear if any low limit bits are clear.
+         */
+        if (   ((pSel->u32Limit & 0xfff00000) == 0x00000000 ||  pSel->Attr.n.u1Granularity)
+            && ((pSel->u32Limit & 0x00000fff) == 0x00000fff || !pSel->Attr.n.u1Granularity))
+            return true;
+    }
+    return false;
+}
+
+
+/**
+ * Checks if the CPU is subject to the "VMX-Preemption Timer Does Not Count Down at
+ * the Rate Specified" erratum.
+ *
+ * Errata names and related steppings:
+ *      - BA86   - D0.
+ *      - AAX65  - C2.
+ *      - AAU65  - C2, K0.
+ *      - AAO95  - B1.
+ *      - AAT59  - C2.
+ *      - AAK139 - D0.
+ *      - AAM126 - C0, C1, D0.
+ *      - AAN92  - B1.
+ *      - AAJ124 - C0, D0.
+ *      - AAP86  - B1.
+ *
+ * Steppings: B1, C0, C1, C2, D0, K0.
+ *
+ * @returns @c true if subject to it, @c false if not.
+ */
+VMM_INT_DECL(bool) HMIsSubjectToVmxPreemptTimerErratum(void)
+{
+    uint32_t u = ASMCpuId_EAX(1);
+    u &= ~(RT_BIT_32(14) | RT_BIT_32(15) | RT_BIT_32(28) | RT_BIT_32(29) | RT_BIT_32(30) | RT_BIT_32(31));
+    if (   u == 0x000206E6 /* 323344.pdf - BA86   - D0 - Xeon Processor 7500 Series */
+        || u == 0x00020652 /* 323056.pdf - AAX65  - C2 - Xeon Processor L3406 */
+                           /* 322814.pdf - AAT59  - C2 - CoreTM i7-600, i5-500, i5-400 and i3-300 Mobile Processor Series */
+                           /* 322911.pdf - AAU65  - C2 - CoreTM i5-600, i3-500 Desktop Processor Series and Intel Pentium Processor G6950 */
+        || u == 0x00020655 /* 322911.pdf - AAU65  - K0 - CoreTM i5-600, i3-500 Desktop Processor Series and Intel Pentium Processor G6950 */
+        || u == 0x000106E5 /* 322373.pdf - AAO95  - B1 - Xeon Processor 3400 Series */
+                           /* 322166.pdf - AAN92  - B1 - CoreTM i7-800 and i5-700 Desktop Processor Series */
+                           /* 320767.pdf - AAP86  - B1 - Core i7-900 Mobile Processor Extreme Edition Series, Intel Core i7-800 and i7-700 Mobile Processor Series */
+        || u == 0x000106A0 /* 321333.pdf - AAM126 - C0 - Xeon Processor 3500 Series Specification */
+        || u == 0x000106A1 /* 321333.pdf - AAM126 - C1 - Xeon Processor 3500 Series Specification */
+        || u == 0x000106A4 /* 320836.pdf - AAJ124 - C0 - Core i7-900 Desktop Processor Extreme Edition Series and Intel Core i7-900 Desktop Processor Series */
+        || u == 0x000106A5 /* 321333.pdf - AAM126 - D0 - Xeon Processor 3500 Series Specification */
+                           /* 321324.pdf - AAK139 - D0 - Xeon Processor 5500 Series Specification */
+                           /* 320836.pdf - AAJ124 - D0 - Core i7-900 Desktop Processor Extreme Edition Series and Intel Core i7-900 Desktop Processor Series */
+        )
+        return true;
+    return false;
+}
+
+
+/**
+ * Checks if the guest is in a suitable state for hardware-assisted VMX execution.
+ *
+ * @returns @c true if it is suitable, @c false otherwise.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   pCtx    Pointer to the guest CPU context.
+ *
+ * @remarks @a pCtx can be a partial context and thus may not be necessarily the
+ *          same as pVCpu->cpum.GstCtx! Thus don't eliminate the @a pCtx parameter.
+ *          Secondly, if additional checks are added that require more of the CPU
+ *          state, make sure REM (which supplies a partial state) is updated.
+ */
+VMM_INT_DECL(bool) HMCanExecuteVmxGuest(PVMCC pVM, PVMCPUCC pVCpu, PCCPUMCTX pCtx)
+{
+    Assert(HMIsEnabled(pVM));
+    Assert(   ( pVM->hm.s.vmx.fUnrestrictedGuest && !pVM->hm.s.vmx.pRealModeTSS)
+           || (!pVM->hm.s.vmx.fUnrestrictedGuest && pVM->hm.s.vmx.pRealModeTSS));
+
+    pVCpu->hm.s.fActive = false;
+
+    bool const fSupportsRealMode = pVM->hm.s.vmx.fUnrestrictedGuest || PDMVmmDevHeapIsEnabled(pVM);
+    if (!pVM->hm.s.vmx.fUnrestrictedGuest)
+    {
+        /*
+         * The VMM device heap is a requirement for emulating real mode or protected mode without paging with the unrestricted
+         * guest execution feature is missing (VT-x only).
+         */
+        if (fSupportsRealMode)
+        {
+            if (CPUMIsGuestInRealModeEx(pCtx))
+            {
+                /*
+                 * In V86 mode (VT-x or not), the CPU enforces real-mode compatible selector
+                 * bases, limits, and attributes, i.e. limit must be 64K, base must be selector * 16,
+                 * and attributes must be 0x9b for code and 0x93 for code segments.
+                 * If this is not true, we cannot execute real mode as V86 and have to fall
+                 * back to emulation.
+                 */
+                if (   pCtx->cs.Sel != (pCtx->cs.u64Base >> 4)
+                    || pCtx->ds.Sel != (pCtx->ds.u64Base >> 4)
+                    || pCtx->es.Sel != (pCtx->es.u64Base >> 4)
+                    || pCtx->ss.Sel != (pCtx->ss.u64Base >> 4)
+                    || pCtx->fs.Sel != (pCtx->fs.u64Base >> 4)
+                    || pCtx->gs.Sel != (pCtx->gs.u64Base >> 4))
+                {
+                    STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadRmSelBase);
+                    return false;
+                }
+                if (   (pCtx->cs.u32Limit != 0xffff)
+                    || (pCtx->ds.u32Limit != 0xffff)
+                    || (pCtx->es.u32Limit != 0xffff)
+                    || (pCtx->ss.u32Limit != 0xffff)
+                    || (pCtx->fs.u32Limit != 0xffff)
+                    || (pCtx->gs.u32Limit != 0xffff))
+                {
+                    STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadRmSelLimit);
+                    return false;
+                }
+                if (   (pCtx->cs.Attr.u != 0x9b)
+                    || (pCtx->ds.Attr.u != 0x93)
+                    || (pCtx->es.Attr.u != 0x93)
+                    || (pCtx->ss.Attr.u != 0x93)
+                    || (pCtx->fs.Attr.u != 0x93)
+                    || (pCtx->gs.Attr.u != 0x93))
+                {
+                    STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadRmSelAttr);
+                    return false;
+                }
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckRmOk);
+            }
+            else
+            {
+                /*
+                 * Verify the requirements for executing code in protected mode. VT-x can't
+                 * handle the CPU state right after a switch from real to protected mode
+                 * (all sorts of RPL & DPL assumptions).
+                 */
+                PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+                if (pVmcsInfo->fWasInRealMode)
+                {
+                    if (!CPUMIsGuestInV86ModeEx(pCtx))
+                    {
+                        /* The guest switched to protected mode, check if the state is suitable for VT-x. */
+                        if ((pCtx->cs.Sel & X86_SEL_RPL) != (pCtx->ss.Sel & X86_SEL_RPL))
+                        {
+                            STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadRpl);
+                            return false;
+                        }
+                        if (   !hmVmxIsCodeSelectorOk(&pCtx->cs, pCtx->ss.Attr.n.u2Dpl)
+                            || !hmVmxIsDataSelectorOk(&pCtx->ds)
+                            || !hmVmxIsDataSelectorOk(&pCtx->es)
+                            || !hmVmxIsDataSelectorOk(&pCtx->fs)
+                            || !hmVmxIsDataSelectorOk(&pCtx->gs)
+                            || !hmVmxIsStackSelectorOk(&pCtx->ss))
+                        {
+                            STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadSel);
+                            return false;
+                        }
+                    }
+                    else
+                    {
+                        /* The guest switched to V86 mode, check if the state is suitable for VT-x. */
+                        if (   pCtx->cs.Sel != (pCtx->cs.u64Base >> 4)
+                            || pCtx->ds.Sel != (pCtx->ds.u64Base >> 4)
+                            || pCtx->es.Sel != (pCtx->es.u64Base >> 4)
+                            || pCtx->ss.Sel != (pCtx->ss.u64Base >> 4)
+                            || pCtx->fs.Sel != (pCtx->fs.u64Base >> 4)
+                            || pCtx->gs.Sel != (pCtx->gs.u64Base >> 4))
+                        {
+                            STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadV86SelBase);
+                            return false;
+                        }
+                        if (   pCtx->cs.u32Limit != 0xffff
+                            || pCtx->ds.u32Limit != 0xffff
+                            || pCtx->es.u32Limit != 0xffff
+                            || pCtx->ss.u32Limit != 0xffff
+                            || pCtx->fs.u32Limit != 0xffff
+                            || pCtx->gs.u32Limit != 0xffff)
+                        {
+                            STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadV86SelLimit);
+                            return false;
+                        }
+                        if (   pCtx->cs.Attr.u != 0xf3
+                            || pCtx->ds.Attr.u != 0xf3
+                            || pCtx->es.Attr.u != 0xf3
+                            || pCtx->ss.Attr.u != 0xf3
+                            || pCtx->fs.Attr.u != 0xf3
+                            || pCtx->gs.Attr.u != 0xf3)
+                        {
+                            STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadV86SelAttr);
+                            return false;
+                        }
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (!CPUMIsGuestInLongModeEx(pCtx))
+            {
+                if (   !pVM->hm.s.fNestedPaging        /* Requires a fake PD for real *and* protected mode without paging - stored in the VMM device heap */
+                    ||  CPUMIsGuestInRealModeEx(pCtx)) /* Requires a fake TSS for real mode - stored in the VMM device heap */
+                    return false;
+
+                /* Too early for VT-x; Solaris guests will fail with a guru meditation otherwise; same for XP. */
+                if (pCtx->idtr.pIdt == 0 || pCtx->idtr.cbIdt == 0 || pCtx->tr.Sel == 0)
+                    return false;
+
+                /*
+                 * The guest is about to complete the switch to protected mode. Wait a bit longer.
+                 * Windows XP; switch to protected mode; all selectors are marked not present
+                 * in the hidden registers (possible recompiler bug; see load_seg_vm).
+                 */
+                /** @todo Is this supposed recompiler bug still relevant with IEM? */
+                if (pCtx->cs.Attr.n.u1Present == 0)
+                    return false;
+                if (pCtx->ss.Attr.n.u1Present == 0)
+                    return false;
+
+                /*
+                 * Windows XP: possible same as above, but new recompiler requires new
+                 * heuristics? VT-x doesn't seem to like something about the guest state and
+                 * this stuff avoids it.
+                 */
+                /** @todo This check is actually wrong, it doesn't take the direction of the
+                 *        stack segment into account. But, it does the job for now. */
+                if (pCtx->rsp >= pCtx->ss.u32Limit)
+                    return false;
+            }
+        }
+    }
+
+    if (pVM->hm.s.vmx.fEnabled)
+    {
+        uint32_t uCr0Mask;
+
+        /* If bit N is set in cr0_fixed0, then it must be set in the guest's cr0. */
+        uCr0Mask = (uint32_t)pVM->hm.s.vmx.Msrs.u64Cr0Fixed0;
+
+        /* We ignore the NE bit here on purpose; see HMR0.cpp for details. */
+        uCr0Mask &= ~X86_CR0_NE;
+
+        if (fSupportsRealMode)
+        {
+            /* We ignore the PE & PG bits here on purpose; we emulate real and protected mode without paging. */
+            uCr0Mask &= ~(X86_CR0_PG | X86_CR0_PE);
+        }
+        else
+        {
+            /* We support protected mode without paging using identity mapping. */
+            uCr0Mask &= ~X86_CR0_PG;
+        }
+        if ((pCtx->cr0 & uCr0Mask) != uCr0Mask)
+            return false;
+
+        /* If bit N is cleared in cr0_fixed1, then it must be zero in the guest's cr0. */
+        uCr0Mask = (uint32_t)~pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
+        if ((pCtx->cr0 & uCr0Mask) != 0)
+            return false;
+
+        /* If bit N is set in cr4_fixed0, then it must be set in the guest's cr4. */
+        uCr0Mask  = (uint32_t)pVM->hm.s.vmx.Msrs.u64Cr4Fixed0;
+        uCr0Mask &= ~X86_CR4_VMXE;
+        if ((pCtx->cr4 & uCr0Mask) != uCr0Mask)
+            return false;
+
+        /* If bit N is cleared in cr4_fixed1, then it must be zero in the guest's cr4. */
+        uCr0Mask = (uint32_t)~pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
+        if ((pCtx->cr4 & uCr0Mask) != 0)
+            return false;
+
+        pVCpu->hm.s.fActive = true;
+        return true;
+    }
+
+    return false;
+}
+
+
+/**
+ * Dumps the virtual VMCS state to the release log.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(void) HMDumpHwvirtVmxState(PVMCPU pVCpu)
+{
+    /* The string width of -4 used in the macros below to cover 'LDTR', 'GDTR', 'IDTR. */
+#define HMVMX_DUMP_HOST_XDTR(a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \
+    do { \
+        LogRel(("  %s%-4s                       = {base=%016RX64}\n", \
+            (a_pszPrefix), (a_SegName), (a_pVmcs)->u64Host##a_Seg##Base.u)); \
+    } while (0)
+#define HMVMX_DUMP_HOST_FS_GS_TR(a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \
+    do { \
+        LogRel(("  %s%-4s                       = {%04x base=%016RX64}\n", \
+                (a_pszPrefix), (a_SegName), (a_pVmcs)->Host##a_Seg, (a_pVmcs)->u64Host##a_Seg##Base.u)); \
+    } while (0)
+#define HMVMX_DUMP_GUEST_SEGREG(a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \
+    do { \
+        LogRel(("  %s%-4s                       = {%04x base=%016RX64 limit=%08x flags=%04x}\n", \
+                (a_pszPrefix), (a_SegName), (a_pVmcs)->Guest##a_Seg, (a_pVmcs)->u64Guest##a_Seg##Base.u, \
+                (a_pVmcs)->u32Guest##a_Seg##Limit, (a_pVmcs)->u32Guest##a_Seg##Attr)); \
+    } while (0)
+#define HMVMX_DUMP_GUEST_XDTR(a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \
+    do { \
+        LogRel(("  %s%-4s                       = {base=%016RX64 limit=%08x}\n", \
+                (a_pszPrefix), (a_SegName), (a_pVmcs)->u64Guest##a_Seg##Base.u, (a_pVmcs)->u32Guest##a_Seg##Limit)); \
+    } while (0)
+
+    PCCPUMCTX  pCtx  = &pVCpu->cpum.GstCtx;
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    if (!pVmcs)
+    {
+        LogRel(("Virtual VMCS not allocated\n"));
+        return;
+    }
+    LogRel(("GCPhysVmxon                = %#RGp\n",     pCtx->hwvirt.vmx.GCPhysVmxon));
+    LogRel(("GCPhysVmcs                 = %#RGp\n",     pCtx->hwvirt.vmx.GCPhysVmcs));
+    LogRel(("GCPhysShadowVmcs           = %#RGp\n",     pCtx->hwvirt.vmx.GCPhysShadowVmcs));
+    LogRel(("enmDiag                    = %u (%s)\n",   pCtx->hwvirt.vmx.enmDiag, HMGetVmxDiagDesc(pCtx->hwvirt.vmx.enmDiag)));
+    LogRel(("uDiagAux                   = %#RX64\n",    pCtx->hwvirt.vmx.uDiagAux));
+    LogRel(("enmAbort                   = %u (%s)\n",   pCtx->hwvirt.vmx.enmAbort, VMXGetAbortDesc(pCtx->hwvirt.vmx.enmAbort)));
+    LogRel(("uAbortAux                  = %u (%#x)\n",  pCtx->hwvirt.vmx.uAbortAux, pCtx->hwvirt.vmx.uAbortAux));
+    LogRel(("fInVmxRootMode             = %RTbool\n",   pCtx->hwvirt.vmx.fInVmxRootMode));
+    LogRel(("fInVmxNonRootMode          = %RTbool\n",   pCtx->hwvirt.vmx.fInVmxNonRootMode));
+    LogRel(("fInterceptEvents           = %RTbool\n",   pCtx->hwvirt.vmx.fInterceptEvents));
+    LogRel(("fNmiUnblockingIret         = %RTbool\n",   pCtx->hwvirt.vmx.fNmiUnblockingIret));
+    LogRel(("uFirstPauseLoopTick        = %RX64\n",     pCtx->hwvirt.vmx.uFirstPauseLoopTick));
+    LogRel(("uPrevPauseTick             = %RX64\n",     pCtx->hwvirt.vmx.uPrevPauseTick));
+    LogRel(("uEntryTick                 = %RX64\n",     pCtx->hwvirt.vmx.uEntryTick));
+    LogRel(("offVirtApicWrite           = %#RX16\n",    pCtx->hwvirt.vmx.offVirtApicWrite));
+    LogRel(("fVirtNmiBlocking           = %RTbool\n",   pCtx->hwvirt.vmx.fVirtNmiBlocking));
+    LogRel(("VMCS cache:\n"));
+
+    const char *pszPrefix = "  ";
+    /* Header. */
+    {
+        LogRel(("%sHeader:\n", pszPrefix));
+        LogRel(("  %sVMCS revision id           = %#RX32\n",      pszPrefix, pVmcs->u32VmcsRevId));
+        LogRel(("  %sVMX-abort id               = %#RX32 (%s)\n", pszPrefix, pVmcs->enmVmxAbort, VMXGetAbortDesc(pVmcs->enmVmxAbort)));
+        LogRel(("  %sVMCS state                 = %#x (%s)\n",    pszPrefix, pVmcs->fVmcsState, VMXGetVmcsStateDesc(pVmcs->fVmcsState)));
+    }
+
+    /* Control fields. */
+    {
+        /* 16-bit. */
+        LogRel(("%sControl:\n", pszPrefix));
+        LogRel(("  %sVPID                       = %#RX16\n",   pszPrefix, pVmcs->u16Vpid));
+        LogRel(("  %sPosted intr notify vector  = %#RX16\n",   pszPrefix, pVmcs->u16PostIntNotifyVector));
+        LogRel(("  %sEPTP index                 = %#RX16\n",   pszPrefix, pVmcs->u16EptpIndex));
+
+        /* 32-bit. */
+        LogRel(("  %sPin ctls                   = %#RX32\n",   pszPrefix, pVmcs->u32PinCtls));
+        LogRel(("  %sProcessor ctls             = %#RX32\n",   pszPrefix, pVmcs->u32ProcCtls));
+        LogRel(("  %sSecondary processor ctls   = %#RX32\n",   pszPrefix, pVmcs->u32ProcCtls2));
+        LogRel(("  %sVM-exit ctls               = %#RX32\n",   pszPrefix, pVmcs->u32ExitCtls));
+        LogRel(("  %sVM-entry ctls              = %#RX32\n",   pszPrefix, pVmcs->u32EntryCtls));
+        LogRel(("  %sException bitmap           = %#RX32\n",   pszPrefix, pVmcs->u32XcptBitmap));
+        LogRel(("  %sPage-fault mask            = %#RX32\n",   pszPrefix, pVmcs->u32XcptPFMask));
+        LogRel(("  %sPage-fault match           = %#RX32\n",   pszPrefix, pVmcs->u32XcptPFMatch));
+        LogRel(("  %sCR3-target count           = %RU32\n",    pszPrefix, pVmcs->u32Cr3TargetCount));
+        LogRel(("  %sVM-exit MSR store count    = %RU32\n",    pszPrefix, pVmcs->u32ExitMsrStoreCount));
+        LogRel(("  %sVM-exit MSR load count     = %RU32\n",    pszPrefix, pVmcs->u32ExitMsrLoadCount));
+        LogRel(("  %sVM-entry MSR load count    = %RU32\n",    pszPrefix, pVmcs->u32EntryMsrLoadCount));
+        LogRel(("  %sVM-entry interruption info = %#RX32\n",   pszPrefix, pVmcs->u32EntryIntInfo));
+        {
+            uint32_t const fInfo = pVmcs->u32EntryIntInfo;
+            uint8_t  const uType = VMX_ENTRY_INT_INFO_TYPE(fInfo);
+            LogRel(("    %sValid                      = %RTbool\n",  pszPrefix, VMX_ENTRY_INT_INFO_IS_VALID(fInfo)));
+            LogRel(("    %sType                       = %#x (%s)\n", pszPrefix, uType, VMXGetEntryIntInfoTypeDesc(uType)));
+            LogRel(("    %sVector                     = %#x\n",      pszPrefix, VMX_ENTRY_INT_INFO_VECTOR(fInfo)));
+            LogRel(("    %sNMI-unblocking-IRET        = %RTbool\n",  pszPrefix, VMX_ENTRY_INT_INFO_IS_NMI_UNBLOCK_IRET(fInfo)));
+            LogRel(("    %sError-code valid           = %RTbool\n",  pszPrefix, VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(fInfo)));
+        }
+        LogRel(("  %sVM-entry xcpt error-code   = %#RX32\n",   pszPrefix, pVmcs->u32EntryXcptErrCode));
+        LogRel(("  %sVM-entry instr length      = %u byte(s)\n", pszPrefix, pVmcs->u32EntryInstrLen));
+        LogRel(("  %sTPR threshold              = %#RX32\n",   pszPrefix, pVmcs->u32TprThreshold));
+        LogRel(("  %sPLE gap                    = %#RX32\n",   pszPrefix, pVmcs->u32PleGap));
+        LogRel(("  %sPLE window                 = %#RX32\n",   pszPrefix, pVmcs->u32PleWindow));
+
+        /* 64-bit. */
+        LogRel(("  %sIO-bitmap A addr           = %#RX64\n",   pszPrefix, pVmcs->u64AddrIoBitmapA.u));
+        LogRel(("  %sIO-bitmap B addr           = %#RX64\n",   pszPrefix, pVmcs->u64AddrIoBitmapB.u));
+        LogRel(("  %sMSR-bitmap addr            = %#RX64\n",   pszPrefix, pVmcs->u64AddrMsrBitmap.u));
+        LogRel(("  %sVM-exit MSR store addr     = %#RX64\n",   pszPrefix, pVmcs->u64AddrExitMsrStore.u));
+        LogRel(("  %sVM-exit MSR load addr      = %#RX64\n",   pszPrefix, pVmcs->u64AddrExitMsrLoad.u));
+        LogRel(("  %sVM-entry MSR load addr     = %#RX64\n",   pszPrefix, pVmcs->u64AddrEntryMsrLoad.u));
+        LogRel(("  %sExecutive VMCS ptr         = %#RX64\n",   pszPrefix, pVmcs->u64ExecVmcsPtr.u));
+        LogRel(("  %sPML addr                   = %#RX64\n",   pszPrefix, pVmcs->u64AddrPml.u));
+        LogRel(("  %sTSC offset                 = %#RX64\n",   pszPrefix, pVmcs->u64TscOffset.u));
+        LogRel(("  %sVirtual-APIC addr          = %#RX64\n",   pszPrefix, pVmcs->u64AddrVirtApic.u));
+        LogRel(("  %sAPIC-access addr           = %#RX64\n",   pszPrefix, pVmcs->u64AddrApicAccess.u));
+        LogRel(("  %sPosted-intr desc addr      = %#RX64\n",   pszPrefix, pVmcs->u64AddrPostedIntDesc.u));
+        LogRel(("  %sVM-functions control       = %#RX64\n",   pszPrefix, pVmcs->u64VmFuncCtls.u));
+        LogRel(("  %sEPTP ptr                   = %#RX64\n",   pszPrefix, pVmcs->u64EptpPtr.u));
+        LogRel(("  %sEOI-exit bitmap 0 addr     = %#RX64\n",   pszPrefix, pVmcs->u64EoiExitBitmap0.u));
+        LogRel(("  %sEOI-exit bitmap 1 addr     = %#RX64\n",   pszPrefix, pVmcs->u64EoiExitBitmap1.u));
+        LogRel(("  %sEOI-exit bitmap 2 addr     = %#RX64\n",   pszPrefix, pVmcs->u64EoiExitBitmap2.u));
+        LogRel(("  %sEOI-exit bitmap 3 addr     = %#RX64\n",   pszPrefix, pVmcs->u64EoiExitBitmap3.u));
+        LogRel(("  %sEPTP-list addr             = %#RX64\n",   pszPrefix, pVmcs->u64AddrEptpList.u));
+        LogRel(("  %sVMREAD-bitmap addr         = %#RX64\n",   pszPrefix, pVmcs->u64AddrVmreadBitmap.u));
+        LogRel(("  %sVMWRITE-bitmap addr        = %#RX64\n",   pszPrefix, pVmcs->u64AddrVmwriteBitmap.u));
+        LogRel(("  %sVirt-Xcpt info addr        = %#RX64\n",   pszPrefix, pVmcs->u64AddrXcptVeInfo.u));
+        LogRel(("  %sXSS-bitmap                 = %#RX64\n",   pszPrefix, pVmcs->u64XssBitmap.u));
+        LogRel(("  %sENCLS-exiting bitmap       = %#RX64\n",   pszPrefix, pVmcs->u64EnclsBitmap.u));
+        LogRel(("  %sSPPT pointer               = %#RX64\n",   pszPrefix, pVmcs->u64SpptPtr.u));
+        LogRel(("  %sTSC multiplier             = %#RX64\n",   pszPrefix, pVmcs->u64TscMultiplier.u));
+
+        /* Natural width. */
+        LogRel(("  %sCR0 guest/host mask        = %#RX64\n",   pszPrefix, pVmcs->u64Cr0Mask.u));
+        LogRel(("  %sCR4 guest/host mask        = %#RX64\n",   pszPrefix, pVmcs->u64Cr4Mask.u));
+        LogRel(("  %sCR0 read shadow            = %#RX64\n",   pszPrefix, pVmcs->u64Cr0ReadShadow.u));
+        LogRel(("  %sCR4 read shadow            = %#RX64\n",   pszPrefix, pVmcs->u64Cr4ReadShadow.u));
+        LogRel(("  %sCR3-target 0               = %#RX64\n",   pszPrefix, pVmcs->u64Cr3Target0.u));
+        LogRel(("  %sCR3-target 1               = %#RX64\n",   pszPrefix, pVmcs->u64Cr3Target1.u));
+        LogRel(("  %sCR3-target 2               = %#RX64\n",   pszPrefix, pVmcs->u64Cr3Target2.u));
+        LogRel(("  %sCR3-target 3               = %#RX64\n",   pszPrefix, pVmcs->u64Cr3Target3.u));
+    }
+
+    /* Guest state. */
+    {
+        LogRel(("%sGuest state:\n", pszPrefix));
+
+        /* 16-bit. */
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Cs,   "cs",   pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Ss,   "ss",   pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Es,   "es",   pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Ds,   "ds",   pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Fs,   "fs",   pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Gs,   "gs",   pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Ldtr, "ldtr", pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Tr,   "tr",   pszPrefix);
+        HMVMX_DUMP_GUEST_XDTR(  pVmcs, Gdtr, "gdtr", pszPrefix);
+        HMVMX_DUMP_GUEST_XDTR(  pVmcs, Idtr, "idtr", pszPrefix);
+        LogRel(("  %sInterrupt status           = %#RX16\n",   pszPrefix, pVmcs->u16GuestIntStatus));
+        LogRel(("  %sPML index                  = %#RX16\n",   pszPrefix, pVmcs->u16PmlIndex));
+
+        /* 32-bit. */
+        LogRel(("  %sInterruptibility state     = %#RX32\n",   pszPrefix, pVmcs->u32GuestIntrState));
+        LogRel(("  %sActivity state             = %#RX32\n",   pszPrefix, pVmcs->u32GuestActivityState));
+        LogRel(("  %sSMBASE                     = %#RX32\n",   pszPrefix, pVmcs->u32GuestSmBase));
+        LogRel(("  %sSysEnter CS                = %#RX32\n",   pszPrefix, pVmcs->u32GuestSysenterCS));
+        LogRel(("  %sVMX-preemption timer value = %#RX32\n",   pszPrefix, pVmcs->u32PreemptTimer));
+
+        /* 64-bit. */
+        LogRel(("  %sVMCS link ptr              = %#RX64\n",   pszPrefix, pVmcs->u64VmcsLinkPtr.u));
+        LogRel(("  %sDBGCTL                     = %#RX64\n",   pszPrefix, pVmcs->u64GuestDebugCtlMsr.u));
+        LogRel(("  %sPAT                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestPatMsr.u));
+        LogRel(("  %sEFER                       = %#RX64\n",   pszPrefix, pVmcs->u64GuestEferMsr.u));
+        LogRel(("  %sPERFGLOBALCTRL             = %#RX64\n",   pszPrefix, pVmcs->u64GuestPerfGlobalCtlMsr.u));
+        LogRel(("  %sPDPTE 0                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestPdpte0.u));
+        LogRel(("  %sPDPTE 1                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestPdpte1.u));
+        LogRel(("  %sPDPTE 2                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestPdpte2.u));
+        LogRel(("  %sPDPTE 3                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestPdpte3.u));
+        LogRel(("  %sBNDCFGS                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestBndcfgsMsr.u));
+        LogRel(("  %sRTIT_CTL                   = %#RX64\n",   pszPrefix, pVmcs->u64GuestRtitCtlMsr.u));
+
+        /* Natural width. */
+        LogRel(("  %scr0                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestCr0.u));
+        LogRel(("  %scr3                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestCr3.u));
+        LogRel(("  %scr4                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestCr4.u));
+        LogRel(("  %sdr7                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestDr7.u));
+        LogRel(("  %srsp                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestRsp.u));
+        LogRel(("  %srip                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestRip.u));
+        LogRel(("  %srflags                     = %#RX64\n",   pszPrefix, pVmcs->u64GuestRFlags.u));
+        LogRel(("  %sPending debug xcpts        = %#RX64\n",   pszPrefix, pVmcs->u64GuestPendingDbgXcpts.u));
+        LogRel(("  %sSysEnter ESP               = %#RX64\n",   pszPrefix, pVmcs->u64GuestSysenterEsp.u));
+        LogRel(("  %sSysEnter EIP               = %#RX64\n",   pszPrefix, pVmcs->u64GuestSysenterEip.u));
+    }
+
+    /* Host state. */
+    {
+        LogRel(("%sHost state:\n", pszPrefix));
+
+        /* 16-bit. */
+        LogRel(("  %scs                         = %#RX16\n",   pszPrefix, pVmcs->HostCs));
+        LogRel(("  %sss                         = %#RX16\n",   pszPrefix, pVmcs->HostSs));
+        LogRel(("  %sds                         = %#RX16\n",   pszPrefix, pVmcs->HostDs));
+        LogRel(("  %ses                         = %#RX16\n",   pszPrefix, pVmcs->HostEs));
+        HMVMX_DUMP_HOST_FS_GS_TR(pVmcs, Fs, "fs", pszPrefix);
+        HMVMX_DUMP_HOST_FS_GS_TR(pVmcs, Gs, "gs", pszPrefix);
+        HMVMX_DUMP_HOST_FS_GS_TR(pVmcs, Tr, "tr", pszPrefix);
+        HMVMX_DUMP_HOST_XDTR(pVmcs, Gdtr, "gdtr", pszPrefix);
+        HMVMX_DUMP_HOST_XDTR(pVmcs, Idtr, "idtr", pszPrefix);
+
+        /* 32-bit. */
+        LogRel(("  %sSysEnter CS                = %#RX32\n",   pszPrefix, pVmcs->u32HostSysenterCs));
+
+        /* 64-bit. */
+        LogRel(("  %sEFER                       = %#RX64\n",   pszPrefix, pVmcs->u64HostEferMsr.u));
+        LogRel(("  %sPAT                        = %#RX64\n",   pszPrefix, pVmcs->u64HostPatMsr.u));
+        LogRel(("  %sPERFGLOBALCTRL             = %#RX64\n",   pszPrefix, pVmcs->u64HostPerfGlobalCtlMsr.u));
+
+        /* Natural width. */
+        LogRel(("  %scr0                        = %#RX64\n",   pszPrefix, pVmcs->u64HostCr0.u));
+        LogRel(("  %scr3                        = %#RX64\n",   pszPrefix, pVmcs->u64HostCr3.u));
+        LogRel(("  %scr4                        = %#RX64\n",   pszPrefix, pVmcs->u64HostCr4.u));
+        LogRel(("  %sSysEnter ESP               = %#RX64\n",   pszPrefix, pVmcs->u64HostSysenterEsp.u));
+        LogRel(("  %sSysEnter EIP               = %#RX64\n",   pszPrefix, pVmcs->u64HostSysenterEip.u));
+        LogRel(("  %srsp                        = %#RX64\n",   pszPrefix, pVmcs->u64HostRsp.u));
+        LogRel(("  %srip                        = %#RX64\n",   pszPrefix, pVmcs->u64HostRip.u));
+    }
+
+    /* Read-only fields. */
+    {
+        LogRel(("%sRead-only data fields:\n", pszPrefix));
+
+        /* 16-bit (none currently). */
+
+        /* 32-bit. */
+        uint32_t const uExitReason = pVmcs->u32RoExitReason;
+        LogRel(("  %sExit reason                = %u (%s)\n",  pszPrefix, uExitReason, HMGetVmxExitName(uExitReason)));
+        LogRel(("  %sExit qualification         = %#RX64\n",   pszPrefix, pVmcs->u64RoExitQual.u));
+        LogRel(("  %sVM-instruction error       = %#RX32\n",   pszPrefix, pVmcs->u32RoVmInstrError));
+        LogRel(("  %sVM-exit intr info          = %#RX32\n",   pszPrefix, pVmcs->u32RoExitIntInfo));
+        {
+            uint32_t const fInfo = pVmcs->u32RoExitIntInfo;
+            uint8_t  const uType = VMX_EXIT_INT_INFO_TYPE(fInfo);
+            LogRel(("    %sValid                      = %RTbool\n",  pszPrefix, VMX_EXIT_INT_INFO_IS_VALID(fInfo)));
+            LogRel(("    %sType                       = %#x (%s)\n", pszPrefix, uType, VMXGetExitIntInfoTypeDesc(uType)));
+            LogRel(("    %sVector                     = %#x\n",      pszPrefix, VMX_EXIT_INT_INFO_VECTOR(fInfo)));
+            LogRel(("    %sNMI-unblocking-IRET        = %RTbool\n",  pszPrefix, VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(fInfo)));
+            LogRel(("    %sError-code valid           = %RTbool\n",  pszPrefix, VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(fInfo)));
+        }
+        LogRel(("  %sVM-exit intr error-code    = %#RX32\n",   pszPrefix, pVmcs->u32RoExitIntErrCode));
+        LogRel(("  %sIDT-vectoring info         = %#RX32\n",   pszPrefix, pVmcs->u32RoIdtVectoringInfo));
+        {
+            uint32_t const fInfo = pVmcs->u32RoIdtVectoringInfo;
+            uint8_t  const uType = VMX_IDT_VECTORING_INFO_TYPE(fInfo);
+            LogRel(("    %sValid                      = %RTbool\n",  pszPrefix, VMX_IDT_VECTORING_INFO_IS_VALID(fInfo)));
+            LogRel(("    %sType                       = %#x (%s)\n", pszPrefix, uType, VMXGetIdtVectoringInfoTypeDesc(uType)));
+            LogRel(("    %sVector                     = %#x\n",      pszPrefix, VMX_IDT_VECTORING_INFO_VECTOR(fInfo)));
+            LogRel(("    %sError-code valid           = %RTbool\n",  pszPrefix, VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(fInfo)));
+        }
+        LogRel(("  %sIDT-vectoring error-code   = %#RX32\n",   pszPrefix, pVmcs->u32RoIdtVectoringErrCode));
+        LogRel(("  %sVM-exit instruction length = %u bytes\n", pszPrefix, pVmcs->u32RoExitInstrLen));
+        LogRel(("  %sVM-exit instruction info   = %#RX64\n",   pszPrefix, pVmcs->u32RoExitInstrInfo));
+
+        /* 64-bit. */
+        LogRel(("  %sGuest-physical addr        = %#RX64\n",   pszPrefix, pVmcs->u64RoGuestPhysAddr.u));
+
+        /* Natural width. */
+        LogRel(("  %sI/O RCX                    = %#RX64\n",   pszPrefix, pVmcs->u64RoIoRcx.u));
+        LogRel(("  %sI/O RSI                    = %#RX64\n",   pszPrefix, pVmcs->u64RoIoRsi.u));
+        LogRel(("  %sI/O RDI                    = %#RX64\n",   pszPrefix, pVmcs->u64RoIoRdi.u));
+        LogRel(("  %sI/O RIP                    = %#RX64\n",   pszPrefix, pVmcs->u64RoIoRip.u));
+        LogRel(("  %sGuest-linear addr          = %#RX64\n",   pszPrefix, pVmcs->u64RoGuestLinearAddr.u));
+    }
+
+#undef HMVMX_DUMP_HOST_XDTR
+#undef HMVMX_DUMP_HOST_FS_GS_TR
+#undef HMVMX_DUMP_GUEST_SEGREG
+#undef HMVMX_DUMP_GUEST_XDTR
+}
+
+
+/**
+ * Gets the active (in use) VMCS info. object for the specified VCPU.
+ *
+ * This is either the guest or nested-guest VMCS info. and need not necessarily
+ * pertain to the "current" VMCS (in the VMX definition of the term). For instance,
+ * if the VM-entry failed due to an invalid-guest state, we may have "cleared" the
+ * current VMCS while returning to ring-3. However, the VMCS info. object for that
+ * VMCS would still be active and returned here so that we could dump the VMCS
+ * fields to ring-3 for diagnostics. This function is thus only used to
+ * distinguish between the nested-guest or guest VMCS.
+ *
+ * @returns The active VMCS information.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @thread  EMT.
+ * @remarks This function may be called with preemption or interrupts disabled!
+ */
+VMM_INT_DECL(PVMXVMCSINFO) hmGetVmxActiveVmcsInfo(PVMCPU pVCpu)
+{
+    if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
+        return &pVCpu->hm.s.vmx.VmcsInfo;
+    return &pVCpu->hm.s.vmx.VmcsInfoNstGst;
+}
+
+
+/**
+ * Converts a VMX event type into an appropriate TRPM event type.
+ *
+ * @returns TRPM event.
+ * @param   uIntInfo    The VMX event.
+ */
+VMM_INT_DECL(TRPMEVENT) HMVmxEventTypeToTrpmEventType(uint32_t uIntInfo)
+{
+    Assert(VMX_IDT_VECTORING_INFO_IS_VALID(uIntInfo));
+
+    TRPMEVENT enmTrapType;
+    uint8_t const uType   = VMX_IDT_VECTORING_INFO_TYPE(uIntInfo);
+    uint8_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(uIntInfo);
+
+    switch (uType)
+    {
+        case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
+           enmTrapType = TRPM_HARDWARE_INT;
+           break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_NMI:
+        case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
+            enmTrapType = TRPM_TRAP;
+            break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:  /* INT1 (ICEBP). */
+            Assert(uVector == X86_XCPT_DB); NOREF(uVector);
+            enmTrapType = TRPM_SOFTWARE_INT;
+            break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:       /* INT3 (#BP) and INTO (#OF) */
+            Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF); NOREF(uVector);
+            enmTrapType = TRPM_SOFTWARE_INT;
+            break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
+            enmTrapType = TRPM_SOFTWARE_INT;
+            break;
+
+        default:
+            AssertMsgFailed(("Invalid trap type %#x\n", uType));
+            enmTrapType = TRPM_32BIT_HACK;
+            break;
+    }
+
+    return enmTrapType;
+}
+
+
+/**
+ * Converts a TRPM event type into an appropriate VMX event type.
+ *
+ * @returns VMX event type mask.
+ * @param   uVector         The event vector.
+ * @param   enmTrpmEvent    The TRPM event.
+ * @param   fIcebp          Whether the \#DB vector is caused by an INT1/ICEBP
+ *                          instruction.
+ */
+VMM_INT_DECL(uint32_t) HMTrpmEventTypeToVmxEventType(uint8_t uVector, TRPMEVENT enmTrpmEvent, bool fIcebp)
+{
+    uint32_t uIntInfoType = 0;
+    if (enmTrpmEvent == TRPM_TRAP)
+    {
+        Assert(!fIcebp);
+        switch (uVector)
+        {
+            case X86_XCPT_NMI:
+                uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_NMI << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+                break;
+
+            case X86_XCPT_BP:
+            case X86_XCPT_OF:
+                uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+                break;
+
+            case X86_XCPT_PF:
+            case X86_XCPT_DF:
+            case X86_XCPT_TS:
+            case X86_XCPT_NP:
+            case X86_XCPT_SS:
+            case X86_XCPT_GP:
+            case X86_XCPT_AC:
+                uIntInfoType |= VMX_IDT_VECTORING_INFO_ERROR_CODE_VALID;
+                RT_FALL_THRU();
+            default:
+                uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+                break;
+        }
+    }
+    else if (enmTrpmEvent == TRPM_HARDWARE_INT)
+    {
+        Assert(!fIcebp);
+        uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_EXT_INT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+    }
+    else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
+    {
+        switch (uVector)
+        {
+            case X86_XCPT_BP:
+            case X86_XCPT_OF:
+                uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+                break;
+
+            case X86_XCPT_DB:
+            {
+                if (fIcebp)
+                {
+                    uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+                    break;
+                }
+                RT_FALL_THRU();
+            }
+            default:
+                uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_SW_INT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+                break;
+        }
+    }
+    else
+        AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
+    return uIntInfoType;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Notification callback for when a VM-exit happens outside VMX R0 code (e.g. in
+ * IEM).
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks Can be called from ring-0 as well as ring-3.
+ */
+VMM_INT_DECL(void) HMNotifyVmxNstGstVmexit(PVMCPU pVCpu)
+{
+    LogFlowFunc(("\n"));
+
+    /*
+     * Transitions to ring-3 flag a full CPU-state change except if we transition to ring-3
+     * in response to a physical CPU interrupt as no changes to the guest-CPU state are
+     * expected (see VINF_EM_RAW_INTERRUPT handling in hmR0VmxExitToRing3).
+     *
+     * However, with nested-guests, the state -can- change on trips to ring-3 for we might
+     * try to inject a nested-guest physical interrupt and cause a VMX_EXIT_EXT_INT VM-exit
+     * for the nested-guest from ring-3.
+     *
+     * Signalling reload of just the guest-CPU state that changed with the VM-exit is -not-
+     * sufficient since HM also needs to reload state related to VM-entry/VM-exit controls
+     * etc. So signal reloading of the entire state. It does not seem worth making this any
+     * more fine grained at the moment.
+     */
+    CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_ALL);
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+
+    /*
+     * Make sure we need to merge the guest VMCS controls with the nested-guest
+     * VMCS controls on the next nested-guest VM-entry.
+     */
+    pVCpu->hm.s.vmx.fMergedNstGstCtls = false;
+
+    /*
+     * Flush the TLB before entering the outer guest execution (mainly required since the
+     * APIC-access guest-physical address would have changed and probably more things in
+     * the future).
+     */
+    pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
+
+    /** @todo Handle releasing of the page-mapping lock later. */
+#if 0
+    if (pVCpu->hm.s.vmx.fVirtApicPageLocked)
+    {
+        PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->hm.s.vmx.PgMapLockVirtApic);
+        pVCpu->hm.s.vmx.fVirtApicPageLocked = false;
+    }
+#endif
+}
+
+
+/**
+ * Notification callback for when the nested hypervisor's current VMCS is loaded or
+ * changed outside VMX R0 code (e.g. in IEM).
+ *
+ * This need -not- be called for modifications to the nested hypervisor's current
+ * VMCS when the guest is in VMX non-root mode as VMCS shadowing is not applicable
+ * there.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks Can be called from ring-0 as well as ring-3.
+ */
+VMM_INT_DECL(void) HMNotifyVmxNstGstCurrentVmcsChanged(PVMCPU pVCpu)
+{
+    CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, CPUMCTX_EXTRN_HWVIRT);
+
+    /*
+     * Make sure we need to copy the nested hypervisor's current VMCS into the shadow VMCS
+     * on the next guest VM-entry.
+     */
+    pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = false;
+}
+
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAll.cpp b/src/VBox/VMM/VMMAll/IEMAll.cpp
new file mode 100644
index 00000000..a7a1bf8f
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAll.cpp
@@ -0,0 +1,16422 @@
+/* $Id: IEMAll.cpp $ */
+/** @file
+ * IEM - Interpreted Execution Manager - All Contexts.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_iem    IEM - Interpreted Execution Manager
+ *
+ * The interpreted exeuction manager (IEM) is for executing short guest code
+ * sequences that are causing too many exits / virtualization traps.  It will
+ * also be used to interpret single instructions, thus replacing the selective
+ * interpreters in EM and IOM.
+ *
+ * Design goals:
+ *      - Relatively small footprint, although we favour speed and correctness
+ *        over size.
+ *      - Reasonably fast.
+ *      - Correctly handle lock prefixed instructions.
+ *      - Complete instruction set - eventually.
+ *      - Refactorable into a recompiler, maybe.
+ *      - Replace EMInterpret*.
+ *
+ * Using the existing disassembler has been considered, however this is thought
+ * to conflict with speed as the disassembler chews things a bit too much while
+ * leaving us with a somewhat complicated state to interpret afterwards.
+ *
+ *
+ * The current code is very much work in progress. You've been warned!
+ *
+ *
+ * @section sec_iem_fpu_instr   FPU Instructions
+ *
+ * On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
+ * same or equivalent instructions on the host FPU.  To make life easy, we also
+ * let the FPU prioritize the unmasked exceptions for us.  This however, only
+ * works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
+ * for FPU exception delivery, because with CR0.NE=0 there is a window where we
+ * can trigger spurious FPU exceptions.
+ *
+ * The guest FPU state is not loaded into the host CPU and kept there till we
+ * leave IEM because the calling conventions have declared an all year open
+ * season on much of the FPU state.  For instance an innocent looking call to
+ * memcpy might end up using a whole bunch of XMM or MM registers if the
+ * particular implementation finds it worthwhile.
+ *
+ *
+ * @section sec_iem_logging     Logging
+ *
+ * The IEM code uses the \"IEM\" log group for the main logging. The different
+ * logging levels/flags are generally used for the following purposes:
+ *      - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
+ *      - Flow (LogFlow): Basic enter/exit IEM state info.
+ *      - Level 2 (Log2): ?
+ *      - Level 3 (Log3): More detailed enter/exit IEM state info.
+ *      - Level 4 (Log4): Decoding mnemonics w/ EIP.
+ *      - Level 5 (Log5): Decoding details.
+ *      - Level 6 (Log6): Enables/disables the lockstep comparison with REM.
+ *      - Level 7 (Log7): iret++ execution logging.
+ *      - Level 8 (Log8): Memory writes.
+ *      - Level 9 (Log9): Memory reads.
+ *
+ */
+
+//#define IEM_LOG_MEMORY_WRITES
+#define IEM_IMPLEMENTS_TASKSWITCH
+
+/* Disabled warning C4505: 'iemRaisePageFaultJmp' : unreferenced local function has been removed */
+#ifdef _MSC_VER
+# pragma warning(disable:4505)
+#endif
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP   LOG_GROUP_IEM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/gim.h>
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+# include <VBox/vmm/em.h>
+# include <VBox/vmm/hm_svm.h>
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+# include <VBox/vmm/hmvmxinline.h>
+#endif
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/dbgftrace.h>
+#include "IEMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <iprt/asm-math.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/x86.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/** @typedef PFNIEMOP
+ * Pointer to an opcode decoder function.
+ */
+
+/** @def FNIEMOP_DEF
+ * Define an opcode decoder function.
+ *
+ * We're using macors for this so that adding and removing parameters as well as
+ * tweaking compiler specific attributes becomes easier.  See FNIEMOP_CALL
+ *
+ * @param   a_Name      The function name.
+ */
+
+/** @typedef PFNIEMOPRM
+ * Pointer to an opcode decoder function with RM byte.
+ */
+
+/** @def FNIEMOPRM_DEF
+ * Define an opcode decoder function with RM byte.
+ *
+ * We're using macors for this so that adding and removing parameters as well as
+ * tweaking compiler specific attributes becomes easier.  See FNIEMOP_CALL_1
+ *
+ * @param   a_Name      The function name.
+ */
+
+#if defined(__GNUC__) && defined(RT_ARCH_X86)
+typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PVMCPUCC pVCpu);
+typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
+# define FNIEMOP_DEF(a_Name) \
+    IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu)
+# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
+    IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
+# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
+    IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
+
+#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
+typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PVMCPUCC pVCpu);
+typedef VBOXSTRICTRC (__fastcall * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
+# define FNIEMOP_DEF(a_Name) \
+    IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
+# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
+    IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
+# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
+    IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
+
+#elif defined(__GNUC__)
+typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
+typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
+# define FNIEMOP_DEF(a_Name) \
+    IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu)
+# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
+    IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
+# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
+    IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
+
+#else
+typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
+typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
+# define FNIEMOP_DEF(a_Name) \
+    IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
+# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
+    IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
+# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
+    IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
+
+#endif
+#define FNIEMOPRM_DEF(a_Name) FNIEMOP_DEF_1(a_Name, uint8_t, bRm)
+
+
+/**
+ * Selector descriptor table entry as fetched by iemMemFetchSelDesc.
+ */
+typedef union IEMSELDESC
+{
+    /** The legacy view. */
+    X86DESC     Legacy;
+    /** The long mode view. */
+    X86DESC64   Long;
+} IEMSELDESC;
+/** Pointer to a selector descriptor table entry. */
+typedef IEMSELDESC *PIEMSELDESC;
+
+/**
+ * CPU exception classes.
+ */
+typedef enum IEMXCPTCLASS
+{
+    IEMXCPTCLASS_BENIGN,
+    IEMXCPTCLASS_CONTRIBUTORY,
+    IEMXCPTCLASS_PAGE_FAULT,
+    IEMXCPTCLASS_DOUBLE_FAULT
+} IEMXCPTCLASS;
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** @def IEM_WITH_SETJMP
+ * Enables alternative status code handling using setjmps.
+ *
+ * This adds a bit of expense via the setjmp() call since it saves all the
+ * non-volatile registers.  However, it eliminates return code checks and allows
+ * for more optimal return value passing (return regs instead of stack buffer).
+ */
+#if defined(DOXYGEN_RUNNING) || defined(RT_OS_WINDOWS) || 1
+# define IEM_WITH_SETJMP
+#endif
+
+/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
+ * due to GCC lacking knowledge about the value range of a switch. */
+#define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
+
+/** Variant of IEM_NOT_REACHED_DEFAULT_CASE_RET that returns a custom value. */
+#define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: AssertFailedReturn(a_RetValue)
+
+/**
+ * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
+ * occation.
+ */
+#ifdef LOG_ENABLED
+# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
+    do { \
+        /*Log*/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
+        return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
+    } while (0)
+#else
+# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
+    return VERR_IEM_ASPECT_NOT_IMPLEMENTED
+#endif
+
+/**
+ * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
+ * occation using the supplied logger statement.
+ *
+ * @param   a_LoggerArgs    What to log on failure.
+ */
+#ifdef LOG_ENABLED
+# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
+    do { \
+        LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
+        /*LogFunc(a_LoggerArgs);*/ \
+        return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
+    } while (0)
+#else
+# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
+    return VERR_IEM_ASPECT_NOT_IMPLEMENTED
+#endif
+
+/**
+ * Call an opcode decoder function.
+ *
+ * We're using macors for this so that adding and removing parameters can be
+ * done as we please.  See FNIEMOP_DEF.
+ */
+#define FNIEMOP_CALL(a_pfn) (a_pfn)(pVCpu)
+
+/**
+ * Call a common opcode decoder function taking one extra argument.
+ *
+ * We're using macors for this so that adding and removing parameters can be
+ * done as we please.  See FNIEMOP_DEF_1.
+ */
+#define FNIEMOP_CALL_1(a_pfn, a0)           (a_pfn)(pVCpu, a0)
+
+/**
+ * Call a common opcode decoder function taking one extra argument.
+ *
+ * We're using macors for this so that adding and removing parameters can be
+ * done as we please.  See FNIEMOP_DEF_1.
+ */
+#define FNIEMOP_CALL_2(a_pfn, a0, a1)       (a_pfn)(pVCpu, a0, a1)
+
+/**
+ * Check if we're currently executing in real or virtual 8086 mode.
+ *
+ * @returns @c true if it is, @c false if not.
+ * @param   a_pVCpu         The IEM state of the current CPU.
+ */
+#define IEM_IS_REAL_OR_V86_MODE(a_pVCpu)    (CPUMIsGuestInRealOrV86ModeEx(IEM_GET_CTX(a_pVCpu)))
+
+/**
+ * Check if we're currently executing in virtual 8086 mode.
+ *
+ * @returns @c true if it is, @c false if not.
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling thread.
+ */
+#define IEM_IS_V86_MODE(a_pVCpu)            (CPUMIsGuestInV86ModeEx(IEM_GET_CTX(a_pVCpu)))
+
+/**
+ * Check if we're currently executing in long mode.
+ *
+ * @returns @c true if it is, @c false if not.
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling thread.
+ */
+#define IEM_IS_LONG_MODE(a_pVCpu)           (CPUMIsGuestInLongModeEx(IEM_GET_CTX(a_pVCpu)))
+
+/**
+ * Check if we're currently executing in a 64-bit code segment.
+ *
+ * @returns @c true if it is, @c false if not.
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling thread.
+ */
+#define IEM_IS_64BIT_CODE(a_pVCpu)          (CPUMIsGuestIn64BitCodeEx(IEM_GET_CTX(a_pVCpu)))
+
+/**
+ * Check if we're currently executing in real mode.
+ *
+ * @returns @c true if it is, @c false if not.
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling thread.
+ */
+#define IEM_IS_REAL_MODE(a_pVCpu)           (CPUMIsGuestInRealModeEx(IEM_GET_CTX(a_pVCpu)))
+
+/**
+ * Returns a (const) pointer to the CPUMFEATURES for the guest CPU.
+ * @returns PCCPUMFEATURES
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling thread.
+ */
+#define IEM_GET_GUEST_CPU_FEATURES(a_pVCpu) (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.GuestFeatures))
+
+/**
+ * Returns a (const) pointer to the CPUMFEATURES for the host CPU.
+ * @returns PCCPUMFEATURES
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling thread.
+ */
+#define IEM_GET_HOST_CPU_FEATURES(a_pVCpu)  (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.HostFeatures))
+
+/**
+ * Evaluates to true if we're presenting an Intel CPU to the guest.
+ */
+#define IEM_IS_GUEST_CPU_INTEL(a_pVCpu)     ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL )
+
+/**
+ * Evaluates to true if we're presenting an AMD CPU to the guest.
+ */
+#define IEM_IS_GUEST_CPU_AMD(a_pVCpu)       ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_AMD || (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_HYGON )
+
+/**
+ * Check if the address is canonical.
+ */
+#define IEM_IS_CANONICAL(a_u64Addr)         X86_IS_CANONICAL(a_u64Addr)
+
+/**
+ * Gets the effective VEX.VVVV value.
+ *
+ * The 4th bit is ignored if not 64-bit code.
+ * @returns effective V-register value.
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling thread.
+ */
+#define IEM_GET_EFFECTIVE_VVVV(a_pVCpu) \
+    ((a_pVCpu)->iem.s.enmCpuMode == IEMMODE_64BIT ? (a_pVCpu)->iem.s.uVex3rdReg : (a_pVCpu)->iem.s.uVex3rdReg & 7)
+
+/** @def IEM_USE_UNALIGNED_DATA_ACCESS
+ * Use unaligned accesses instead of elaborate byte assembly. */
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) || defined(DOXYGEN_RUNNING)
+# define IEM_USE_UNALIGNED_DATA_ACCESS
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+
+/**
+ * Check if the guest has entered VMX root operation.
+ */
+# define IEM_VMX_IS_ROOT_MODE(a_pVCpu)      (CPUMIsGuestInVmxRootMode(IEM_GET_CTX(a_pVCpu)))
+
+/**
+ * Check if the guest has entered VMX non-root operation.
+ */
+# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu)  (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(a_pVCpu)))
+
+/**
+ * Check if the nested-guest has the given Pin-based VM-execution control set.
+ */
+# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_PinCtl) \
+    (CPUMIsGuestVmxPinCtlsSet(IEM_GET_CTX(a_pVCpu), (a_PinCtl)))
+
+/**
+ * Check if the nested-guest has the given Processor-based VM-execution control set.
+ */
+#define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_ProcCtl) \
+    (CPUMIsGuestVmxProcCtlsSet(IEM_GET_CTX(a_pVCpu), (a_ProcCtl)))
+
+/**
+ * Check if the nested-guest has the given Secondary Processor-based VM-execution
+ * control set.
+ */
+#define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_ProcCtl2) \
+    (CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(a_pVCpu), (a_ProcCtl2)))
+
+/**
+ * Invokes the VMX VM-exit handler for an instruction intercept.
+ */
+# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) \
+    do { return iemVmxVmexitInstr((a_pVCpu), (a_uExitReason), (a_cbInstr)); } while (0)
+
+/**
+ * Invokes the VMX VM-exit handler for an instruction intercept where the
+ * instruction provides additional VM-exit information.
+ */
+# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) \
+    do { return iemVmxVmexitInstrNeedsInfo((a_pVCpu), (a_uExitReason), (a_uInstrId), (a_cbInstr)); } while (0)
+
+/**
+ * Invokes the VMX VM-exit handler for a task switch.
+ */
+# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) \
+    do { return iemVmxVmexitTaskSwitch((a_pVCpu), (a_enmTaskSwitch), (a_SelNewTss), (a_cbInstr)); } while (0)
+
+/**
+ * Invokes the VMX VM-exit handler for MWAIT.
+ */
+# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) \
+    do { return iemVmxVmexitInstrMwait((a_pVCpu), (a_fMonitorArmed), (a_cbInstr)); } while (0)
+
+/**
+ * Invokes the VMX VM-exit handler.
+ */
+# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) \
+    do { return iemVmxVmexit((a_pVCpu), (a_uExitReason), (a_uExitQual)); } while (0)
+
+#else
+# define IEM_VMX_IS_ROOT_MODE(a_pVCpu)                                          (false)
+# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu)                                      (false)
+# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_cbInstr)                             (false)
+# define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_cbInstr)                            (false)
+# define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_cbInstr)                           (false)
+# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr)            do { return VERR_VMX_IPE_1; } while (0)
+# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr)  do { return VERR_VMX_IPE_1; } while (0)
+# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr)    do { return VERR_VMX_IPE_1; } while (0)
+# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr)          do { return VERR_VMX_IPE_1; } while (0)
+# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual)   do { return VERR_VMX_IPE_1; } while (0)
+
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+/**
+ * Check if an SVM control/instruction intercept is set.
+ */
+# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) \
+    (CPUMIsGuestSvmCtrlInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_Intercept)))
+
+/**
+ * Check if an SVM read CRx intercept is set.
+ */
+# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
+    (CPUMIsGuestSvmReadCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
+
+/**
+ * Check if an SVM write CRx intercept is set.
+ */
+# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
+    (CPUMIsGuestSvmWriteCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
+
+/**
+ * Check if an SVM read DRx intercept is set.
+ */
+# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
+    (CPUMIsGuestSvmReadDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
+
+/**
+ * Check if an SVM write DRx intercept is set.
+ */
+# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
+    (CPUMIsGuestSvmWriteDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
+
+/**
+ * Check if an SVM exception intercept is set.
+ */
+# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) \
+    (CPUMIsGuestSvmXcptInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uVector)))
+
+/**
+ * Invokes the SVM \#VMEXIT handler for the nested-guest.
+ */
+# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
+    do { return iemSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); } while (0)
+
+/**
+ * Invokes the 'MOV CRx' SVM \#VMEXIT handler after constructing the
+ * corresponding decode assist information.
+ */
+# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) \
+    do \
+    { \
+        uint64_t uExitInfo1; \
+        if (   IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmDecodeAssists \
+            && (a_enmAccessCrX) == IEMACCESSCRX_MOV_CRX) \
+            uExitInfo1 = SVM_EXIT1_MOV_CRX_MASK | ((a_iGReg) & 7); \
+        else \
+            uExitInfo1 = 0; \
+        IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, uExitInfo1, 0); \
+    } while (0)
+
+/** Check and handles SVM nested-guest instruction intercept and updates
+ *  NRIP if needed.
+ */
+# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
+    do \
+    { \
+        if (IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept)) \
+        { \
+            IEM_SVM_UPDATE_NRIP(a_pVCpu); \
+            IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2); \
+        } \
+    } while (0)
+
+/** Checks and handles SVM nested-guest CR0 read intercept. */
+# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) \
+    do \
+    { \
+        if (!IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, 0)) \
+        { /* probably likely */ } \
+        else \
+        { \
+            IEM_SVM_UPDATE_NRIP(a_pVCpu); \
+            IEM_SVM_VMEXIT_RET(a_pVCpu, SVM_EXIT_READ_CR0, a_uExitInfo1, a_uExitInfo2); \
+        } \
+    } while (0)
+
+/**
+ * Updates the NextRIP (NRI) field in the nested-guest VMCB.
+ */
+# define IEM_SVM_UPDATE_NRIP(a_pVCpu) \
+    do { \
+        if (IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmNextRipSave) \
+            CPUMGuestSvmUpdateNRip(a_pVCpu, IEM_GET_CTX(a_pVCpu), IEM_GET_INSTR_LEN(a_pVCpu)); \
+    } while (0)
+
+#else
+# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept)                              (false)
+# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr)                                 (false)
+# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr)                                (false)
+# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr)                                 (false)
+# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr)                                (false)
+# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector)                                (false)
+# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2)             do { return VERR_SVM_IPE_1; } while (0)
+# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg)            do { return VERR_SVM_IPE_1; } while (0)
+# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2)   do { } while (0)
+# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2)                          do { } while (0)
+# define IEM_SVM_UPDATE_NRIP(a_pVCpu)                                                     do { } while (0)
+
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+extern const PFNIEMOP g_apfnOneByteMap[256]; /* not static since we need to forward declare it. */
+
+
+/** Function table for the ADD instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_add =
+{
+    iemAImpl_add_u8,  iemAImpl_add_u8_locked,
+    iemAImpl_add_u16, iemAImpl_add_u16_locked,
+    iemAImpl_add_u32, iemAImpl_add_u32_locked,
+    iemAImpl_add_u64, iemAImpl_add_u64_locked
+};
+
+/** Function table for the ADC instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_adc =
+{
+    iemAImpl_adc_u8,  iemAImpl_adc_u8_locked,
+    iemAImpl_adc_u16, iemAImpl_adc_u16_locked,
+    iemAImpl_adc_u32, iemAImpl_adc_u32_locked,
+    iemAImpl_adc_u64, iemAImpl_adc_u64_locked
+};
+
+/** Function table for the SUB instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_sub =
+{
+    iemAImpl_sub_u8,  iemAImpl_sub_u8_locked,
+    iemAImpl_sub_u16, iemAImpl_sub_u16_locked,
+    iemAImpl_sub_u32, iemAImpl_sub_u32_locked,
+    iemAImpl_sub_u64, iemAImpl_sub_u64_locked
+};
+
+/** Function table for the SBB instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_sbb =
+{
+    iemAImpl_sbb_u8,  iemAImpl_sbb_u8_locked,
+    iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked,
+    iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked,
+    iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked
+};
+
+/** Function table for the OR instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_or =
+{
+    iemAImpl_or_u8,  iemAImpl_or_u8_locked,
+    iemAImpl_or_u16, iemAImpl_or_u16_locked,
+    iemAImpl_or_u32, iemAImpl_or_u32_locked,
+    iemAImpl_or_u64, iemAImpl_or_u64_locked
+};
+
+/** Function table for the XOR instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_xor =
+{
+    iemAImpl_xor_u8,  iemAImpl_xor_u8_locked,
+    iemAImpl_xor_u16, iemAImpl_xor_u16_locked,
+    iemAImpl_xor_u32, iemAImpl_xor_u32_locked,
+    iemAImpl_xor_u64, iemAImpl_xor_u64_locked
+};
+
+/** Function table for the AND instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_and =
+{
+    iemAImpl_and_u8,  iemAImpl_and_u8_locked,
+    iemAImpl_and_u16, iemAImpl_and_u16_locked,
+    iemAImpl_and_u32, iemAImpl_and_u32_locked,
+    iemAImpl_and_u64, iemAImpl_and_u64_locked
+};
+
+/** Function table for the CMP instruction.
+ * @remarks Making operand order ASSUMPTIONS.
+ */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_cmp =
+{
+    iemAImpl_cmp_u8,  NULL,
+    iemAImpl_cmp_u16, NULL,
+    iemAImpl_cmp_u32, NULL,
+    iemAImpl_cmp_u64, NULL
+};
+
+/** Function table for the TEST instruction.
+ * @remarks Making operand order ASSUMPTIONS.
+ */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_test =
+{
+    iemAImpl_test_u8,  NULL,
+    iemAImpl_test_u16, NULL,
+    iemAImpl_test_u32, NULL,
+    iemAImpl_test_u64, NULL
+};
+
+/** Function table for the BT instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bt =
+{
+    NULL,  NULL,
+    iemAImpl_bt_u16, NULL,
+    iemAImpl_bt_u32, NULL,
+    iemAImpl_bt_u64, NULL
+};
+
+/** Function table for the BTC instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_btc =
+{
+    NULL,  NULL,
+    iemAImpl_btc_u16, iemAImpl_btc_u16_locked,
+    iemAImpl_btc_u32, iemAImpl_btc_u32_locked,
+    iemAImpl_btc_u64, iemAImpl_btc_u64_locked
+};
+
+/** Function table for the BTR instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_btr =
+{
+    NULL,  NULL,
+    iemAImpl_btr_u16, iemAImpl_btr_u16_locked,
+    iemAImpl_btr_u32, iemAImpl_btr_u32_locked,
+    iemAImpl_btr_u64, iemAImpl_btr_u64_locked
+};
+
+/** Function table for the BTS instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bts =
+{
+    NULL,  NULL,
+    iemAImpl_bts_u16, iemAImpl_bts_u16_locked,
+    iemAImpl_bts_u32, iemAImpl_bts_u32_locked,
+    iemAImpl_bts_u64, iemAImpl_bts_u64_locked
+};
+
+/** Function table for the BSF instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsf =
+{
+    NULL,  NULL,
+    iemAImpl_bsf_u16, NULL,
+    iemAImpl_bsf_u32, NULL,
+    iemAImpl_bsf_u64, NULL
+};
+
+/** Function table for the BSR instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsr =
+{
+    NULL,  NULL,
+    iemAImpl_bsr_u16, NULL,
+    iemAImpl_bsr_u32, NULL,
+    iemAImpl_bsr_u64, NULL
+};
+
+/** Function table for the IMUL instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_imul_two =
+{
+    NULL,  NULL,
+    iemAImpl_imul_two_u16, NULL,
+    iemAImpl_imul_two_u32, NULL,
+    iemAImpl_imul_two_u64, NULL
+};
+
+/** Group 1 /r lookup table. */
+IEM_STATIC const PCIEMOPBINSIZES g_apIemImplGrp1[8] =
+{
+    &g_iemAImpl_add,
+    &g_iemAImpl_or,
+    &g_iemAImpl_adc,
+    &g_iemAImpl_sbb,
+    &g_iemAImpl_and,
+    &g_iemAImpl_sub,
+    &g_iemAImpl_xor,
+    &g_iemAImpl_cmp
+};
+
+/** Function table for the INC instruction. */
+IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_inc =
+{
+    iemAImpl_inc_u8,  iemAImpl_inc_u8_locked,
+    iemAImpl_inc_u16, iemAImpl_inc_u16_locked,
+    iemAImpl_inc_u32, iemAImpl_inc_u32_locked,
+    iemAImpl_inc_u64, iemAImpl_inc_u64_locked
+};
+
+/** Function table for the DEC instruction. */
+IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_dec =
+{
+    iemAImpl_dec_u8,  iemAImpl_dec_u8_locked,
+    iemAImpl_dec_u16, iemAImpl_dec_u16_locked,
+    iemAImpl_dec_u32, iemAImpl_dec_u32_locked,
+    iemAImpl_dec_u64, iemAImpl_dec_u64_locked
+};
+
+/** Function table for the NEG instruction. */
+IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_neg =
+{
+    iemAImpl_neg_u8,  iemAImpl_neg_u8_locked,
+    iemAImpl_neg_u16, iemAImpl_neg_u16_locked,
+    iemAImpl_neg_u32, iemAImpl_neg_u32_locked,
+    iemAImpl_neg_u64, iemAImpl_neg_u64_locked
+};
+
+/** Function table for the NOT instruction. */
+IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_not =
+{
+    iemAImpl_not_u8,  iemAImpl_not_u8_locked,
+    iemAImpl_not_u16, iemAImpl_not_u16_locked,
+    iemAImpl_not_u32, iemAImpl_not_u32_locked,
+    iemAImpl_not_u64, iemAImpl_not_u64_locked
+};
+
+
+/** Function table for the ROL instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rol =
+{
+    iemAImpl_rol_u8,
+    iemAImpl_rol_u16,
+    iemAImpl_rol_u32,
+    iemAImpl_rol_u64
+};
+
+/** Function table for the ROR instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_ror =
+{
+    iemAImpl_ror_u8,
+    iemAImpl_ror_u16,
+    iemAImpl_ror_u32,
+    iemAImpl_ror_u64
+};
+
+/** Function table for the RCL instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcl =
+{
+    iemAImpl_rcl_u8,
+    iemAImpl_rcl_u16,
+    iemAImpl_rcl_u32,
+    iemAImpl_rcl_u64
+};
+
+/** Function table for the RCR instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcr =
+{
+    iemAImpl_rcr_u8,
+    iemAImpl_rcr_u16,
+    iemAImpl_rcr_u32,
+    iemAImpl_rcr_u64
+};
+
+/** Function table for the SHL instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shl =
+{
+    iemAImpl_shl_u8,
+    iemAImpl_shl_u16,
+    iemAImpl_shl_u32,
+    iemAImpl_shl_u64
+};
+
+/** Function table for the SHR instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shr =
+{
+    iemAImpl_shr_u8,
+    iemAImpl_shr_u16,
+    iemAImpl_shr_u32,
+    iemAImpl_shr_u64
+};
+
+/** Function table for the SAR instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_sar =
+{
+    iemAImpl_sar_u8,
+    iemAImpl_sar_u16,
+    iemAImpl_sar_u32,
+    iemAImpl_sar_u64
+};
+
+
+/** Function table for the MUL instruction. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_mul =
+{
+    iemAImpl_mul_u8,
+    iemAImpl_mul_u16,
+    iemAImpl_mul_u32,
+    iemAImpl_mul_u64
+};
+
+/** Function table for the IMUL instruction working implicitly on rAX. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_imul =
+{
+    iemAImpl_imul_u8,
+    iemAImpl_imul_u16,
+    iemAImpl_imul_u32,
+    iemAImpl_imul_u64
+};
+
+/** Function table for the DIV instruction. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_div =
+{
+    iemAImpl_div_u8,
+    iemAImpl_div_u16,
+    iemAImpl_div_u32,
+    iemAImpl_div_u64
+};
+
+/** Function table for the MUL instruction. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_idiv =
+{
+    iemAImpl_idiv_u8,
+    iemAImpl_idiv_u16,
+    iemAImpl_idiv_u32,
+    iemAImpl_idiv_u64
+};
+
+/** Function table for the SHLD instruction */
+IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shld =
+{
+    iemAImpl_shld_u16,
+    iemAImpl_shld_u32,
+    iemAImpl_shld_u64,
+};
+
+/** Function table for the SHRD instruction */
+IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd =
+{
+    iemAImpl_shrd_u16,
+    iemAImpl_shrd_u32,
+    iemAImpl_shrd_u64,
+};
+
+
+/** Function table for the PUNPCKLBW instruction */
+IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpcklbw  = { iemAImpl_punpcklbw_u64,  iemAImpl_punpcklbw_u128 };
+/** Function table for the PUNPCKLBD instruction */
+IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpcklwd  = { iemAImpl_punpcklwd_u64,  iemAImpl_punpcklwd_u128 };
+/** Function table for the PUNPCKLDQ instruction */
+IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpckldq  = { iemAImpl_punpckldq_u64,  iemAImpl_punpckldq_u128 };
+/** Function table for the PUNPCKLQDQ instruction */
+IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpcklqdq = { NULL, iemAImpl_punpcklqdq_u128 };
+
+/** Function table for the PUNPCKHBW instruction */
+IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhbw  = { iemAImpl_punpckhbw_u64,  iemAImpl_punpckhbw_u128 };
+/** Function table for the PUNPCKHBD instruction */
+IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhwd  = { iemAImpl_punpckhwd_u64,  iemAImpl_punpckhwd_u128 };
+/** Function table for the PUNPCKHDQ instruction */
+IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhdq  = { iemAImpl_punpckhdq_u64,  iemAImpl_punpckhdq_u128 };
+/** Function table for the PUNPCKHQDQ instruction */
+IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhqdq = { NULL, iemAImpl_punpckhqdq_u128 };
+
+/** Function table for the PXOR instruction */
+IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pxor         = { iemAImpl_pxor_u64,       iemAImpl_pxor_u128 };
+/** Function table for the PCMPEQB instruction */
+IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pcmpeqb      = { iemAImpl_pcmpeqb_u64,    iemAImpl_pcmpeqb_u128 };
+/** Function table for the PCMPEQW instruction */
+IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pcmpeqw      = { iemAImpl_pcmpeqw_u64,    iemAImpl_pcmpeqw_u128 };
+/** Function table for the PCMPEQD instruction */
+IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pcmpeqd      = { iemAImpl_pcmpeqd_u64,    iemAImpl_pcmpeqd_u128 };
+
+
+#if defined(IEM_LOG_MEMORY_WRITES)
+/** What IEM just wrote. */
+uint8_t g_abIemWrote[256];
+/** How much IEM just wrote. */
+size_t g_cbIemWrote;
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+IEM_STATIC VBOXSTRICTRC     iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr);
+IEM_STATIC VBOXSTRICTRC     iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu);
+IEM_STATIC VBOXSTRICTRC     iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu);
+IEM_STATIC VBOXSTRICTRC     iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel);
+/*IEM_STATIC VBOXSTRICTRC     iemRaiseSelectorNotPresent(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);*/
+IEM_STATIC VBOXSTRICTRC     iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel);
+IEM_STATIC VBOXSTRICTRC     iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr);
+IEM_STATIC VBOXSTRICTRC     iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel);
+IEM_STATIC VBOXSTRICTRC     iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr);
+IEM_STATIC VBOXSTRICTRC     iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr);
+IEM_STATIC VBOXSTRICTRC     iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu);
+IEM_STATIC VBOXSTRICTRC     iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL uSel);
+IEM_STATIC VBOXSTRICTRC     iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);
+IEM_STATIC VBOXSTRICTRC     iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel);
+IEM_STATIC VBOXSTRICTRC     iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);
+IEM_STATIC VBOXSTRICTRC     iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc);
+IEM_STATIC VBOXSTRICTRC     iemRaiseAlignmentCheckException(PVMCPUCC pVCpu);
+#ifdef IEM_WITH_SETJMP
+DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc);
+DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu);
+DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);
+DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel);
+DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);
+#endif
+
+IEM_STATIC VBOXSTRICTRC     iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess);
+IEM_STATIC VBOXSTRICTRC     iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess);
+IEM_STATIC VBOXSTRICTRC     iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
+IEM_STATIC VBOXSTRICTRC     iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
+IEM_STATIC VBOXSTRICTRC     iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
+IEM_STATIC VBOXSTRICTRC     iemMemFetchSysU8(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
+IEM_STATIC VBOXSTRICTRC     iemMemFetchSysU16(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
+IEM_STATIC VBOXSTRICTRC     iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
+IEM_STATIC VBOXSTRICTRC     iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
+IEM_STATIC VBOXSTRICTRC     iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt, uint16_t uErrorCode);
+IEM_STATIC VBOXSTRICTRC     iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt);
+IEM_STATIC VBOXSTRICTRC     iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp);
+IEM_STATIC VBOXSTRICTRC     iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, void **ppvMem, uint64_t *puNewRsp);
+IEM_STATIC VBOXSTRICTRC     iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value);
+IEM_STATIC VBOXSTRICTRC     iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value);
+IEM_STATIC VBOXSTRICTRC     iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel);
+IEM_STATIC uint16_t         iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg);
+IEM_STATIC uint64_t         iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+IEM_STATIC VBOXSTRICTRC     iemVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual);
+IEM_STATIC VBOXSTRICTRC     iemVmxVmexitTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, RTSEL SelNewTss, uint8_t cbInstr);
+IEM_STATIC VBOXSTRICTRC     iemVmxVmexitEvent(PVMCPUCC pVCpu, uint8_t uVector, uint32_t fFlags, uint32_t uErrCode, uint64_t uCr2, uint8_t cbInstr);
+IEM_STATIC VBOXSTRICTRC     iemVmxVmexitEventDoubleFault(PVMCPUCC pVCpu);
+IEM_STATIC VBOXSTRICTRC     iemVmxVirtApicAccessMem(PVMCPUCC pVCpu, uint16_t offAccess, size_t cbAccess, void *pvData, uint32_t fAccess);
+IEM_STATIC VBOXSTRICTRC     iemVmxVirtApicAccessMsrRead(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Value);
+IEM_STATIC VBOXSTRICTRC     iemVmxVirtApicAccessMsrWrite(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t u64Value);
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+IEM_STATIC VBOXSTRICTRC     iemSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2);
+IEM_STATIC VBOXSTRICTRC     iemHandleSvmEventIntercept(PVMCPUCC pVCpu, uint8_t u8Vector, uint32_t fFlags, uint32_t uErr, uint64_t uCr2);
+#endif
+
+
+/**
+ * Sets the pass up status.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   rcPassUp            The pass up status.  Must be informational.
+ *                              VINF_SUCCESS is not allowed.
+ */
+IEM_STATIC int iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp)
+{
+    AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
+
+    int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
+    if (rcOldPassUp == VINF_SUCCESS)
+        pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
+    /* If both are EM scheduling codes, use EM priority rules. */
+    else if (   rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
+             && rcPassUp    >= VINF_EM_FIRST && rcPassUp    <= VINF_EM_LAST)
+    {
+        if (rcPassUp < rcOldPassUp)
+        {
+            Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
+            pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
+        }
+        else
+            Log(("IEM: rcPassUp=%Rrc  rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
+    }
+    /* Override EM scheduling with specific status code. */
+    else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
+    {
+        Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
+        pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
+    }
+    /* Don't override specific status code, first come first served. */
+    else
+        Log(("IEM: rcPassUp=%Rrc  rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Calculates the CPU mode.
+ *
+ * This is mainly for updating IEMCPU::enmCpuMode.
+ *
+ * @returns CPU mode.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ */
+DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPUCC pVCpu)
+{
+    if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
+        return IEMMODE_64BIT;
+    if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */
+        return IEMMODE_32BIT;
+    return IEMMODE_16BIT;
+}
+
+
+/**
+ * Initializes the execution state.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   fBypassHandlers     Whether to bypass access handlers.
+ *
+ * @remarks Callers of this must call iemUninitExec() to undo potentially fatal
+ *          side-effects in strict builds.
+ */
+DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, bool fBypassHandlers)
+{
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
+    Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
+
+    pVCpu->iem.s.uCpl               = CPUMGetGuestCPL(pVCpu);
+    pVCpu->iem.s.enmCpuMode         = iemCalcCpuMode(pVCpu);
+#ifdef VBOX_STRICT
+    pVCpu->iem.s.enmDefAddrMode     = (IEMMODE)0xfe;
+    pVCpu->iem.s.enmEffAddrMode     = (IEMMODE)0xfe;
+    pVCpu->iem.s.enmDefOpSize       = (IEMMODE)0xfe;
+    pVCpu->iem.s.enmEffOpSize       = (IEMMODE)0xfe;
+    pVCpu->iem.s.fPrefixes          = 0xfeedbeef;
+    pVCpu->iem.s.uRexReg            = 127;
+    pVCpu->iem.s.uRexB              = 127;
+    pVCpu->iem.s.offModRm           = 127;
+    pVCpu->iem.s.uRexIndex          = 127;
+    pVCpu->iem.s.iEffSeg            = 127;
+    pVCpu->iem.s.idxPrefix          = 127;
+    pVCpu->iem.s.uVex3rdReg         = 127;
+    pVCpu->iem.s.uVexLength         = 127;
+    pVCpu->iem.s.fEvexStuff         = 127;
+    pVCpu->iem.s.uFpuOpcode         = UINT16_MAX;
+# ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.offInstrNextByte   = UINT16_MAX;
+    pVCpu->iem.s.pbInstrBuf         = NULL;
+    pVCpu->iem.s.cbInstrBuf         = UINT16_MAX;
+    pVCpu->iem.s.cbInstrBufTotal    = UINT16_MAX;
+    pVCpu->iem.s.offCurInstrStart   = INT16_MAX;
+    pVCpu->iem.s.uInstrBufPc        = UINT64_C(0xc0ffc0ffcff0c0ff);
+# else
+    pVCpu->iem.s.offOpcode          = 127;
+    pVCpu->iem.s.cbOpcode           = 127;
+# endif
+#endif
+
+    pVCpu->iem.s.cActiveMappings    = 0;
+    pVCpu->iem.s.iNextMapping       = 0;
+    pVCpu->iem.s.rcPassUp           = VINF_SUCCESS;
+    pVCpu->iem.s.fBypassHandlers    = fBypassHandlers;
+#if 0
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (    CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
+        &&  CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
+    {
+        PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+        Assert(pVmcs);
+        RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u;
+        if (!PGMHandlerPhysicalIsRegistered(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess))
+        {
+           int rc = PGMHandlerPhysicalRegister(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess, GCPhysApicAccess + X86_PAGE_4K_SIZE - 1,
+                                               pVCpu->iem.s.hVmxApicAccessPage, NIL_RTR3PTR /* pvUserR3 */,
+                                               NIL_RTR0PTR /* pvUserR0 */,  NIL_RTRCPTR /* pvUserRC */, NULL /* pszDesc */);
+           AssertRC(rc);
+        }
+    }
+#endif
+#endif
+}
+
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+/**
+ * Performs a minimal reinitialization of the execution state.
+ *
+ * This is intended to be used by VM-exits, SMM, LOADALL and other similar
+ * 'world-switch' types operations on the CPU. Currently only nested
+ * hardware-virtualization uses it.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ */
+IEM_STATIC void iemReInitExec(PVMCPUCC pVCpu)
+{
+    IEMMODE const enmMode = iemCalcCpuMode(pVCpu);
+    uint8_t const uCpl    = CPUMGetGuestCPL(pVCpu);
+
+    pVCpu->iem.s.uCpl             = uCpl;
+    pVCpu->iem.s.enmCpuMode       = enmMode;
+    pVCpu->iem.s.enmDefAddrMode   = enmMode;  /** @todo check if this is correct... */
+    pVCpu->iem.s.enmEffAddrMode   = enmMode;
+    if (enmMode != IEMMODE_64BIT)
+    {
+        pVCpu->iem.s.enmDefOpSize = enmMode;  /** @todo check if this is correct... */
+        pVCpu->iem.s.enmEffOpSize = enmMode;
+    }
+    else
+    {
+        pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
+        pVCpu->iem.s.enmEffOpSize = enmMode;
+    }
+    pVCpu->iem.s.iEffSeg          = X86_SREG_DS;
+#ifndef IEM_WITH_CODE_TLB
+    /** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */
+    pVCpu->iem.s.offOpcode        = 0;
+    pVCpu->iem.s.cbOpcode         = 0;
+#endif
+    pVCpu->iem.s.rcPassUp         = VINF_SUCCESS;
+}
+#endif
+
+/**
+ * Counterpart to #iemInitExec that undoes evil strict-build stuff.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ */
+DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu)
+{
+    /* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
+#ifdef VBOX_STRICT
+# ifdef IEM_WITH_CODE_TLB
+    NOREF(pVCpu);
+# else
+    pVCpu->iem.s.cbOpcode = 0;
+# endif
+#else
+    NOREF(pVCpu);
+#endif
+}
+
+
+/**
+ * Initializes the decoder state.
+ *
+ * iemReInitDecoder is mostly a copy of this function.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   fBypassHandlers     Whether to bypass access handlers.
+ */
+DECLINLINE(void) iemInitDecoder(PVMCPUCC pVCpu, bool fBypassHandlers)
+{
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+    Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
+
+    pVCpu->iem.s.uCpl               = CPUMGetGuestCPL(pVCpu);
+    IEMMODE enmMode = iemCalcCpuMode(pVCpu);
+    pVCpu->iem.s.enmCpuMode         = enmMode;
+    pVCpu->iem.s.enmDefAddrMode     = enmMode;  /** @todo check if this is correct... */
+    pVCpu->iem.s.enmEffAddrMode     = enmMode;
+    if (enmMode != IEMMODE_64BIT)
+    {
+        pVCpu->iem.s.enmDefOpSize   = enmMode;  /** @todo check if this is correct... */
+        pVCpu->iem.s.enmEffOpSize   = enmMode;
+    }
+    else
+    {
+        pVCpu->iem.s.enmDefOpSize   = IEMMODE_32BIT;
+        pVCpu->iem.s.enmEffOpSize   = IEMMODE_32BIT;
+    }
+    pVCpu->iem.s.fPrefixes          = 0;
+    pVCpu->iem.s.uRexReg            = 0;
+    pVCpu->iem.s.uRexB              = 0;
+    pVCpu->iem.s.uRexIndex          = 0;
+    pVCpu->iem.s.idxPrefix          = 0;
+    pVCpu->iem.s.uVex3rdReg         = 0;
+    pVCpu->iem.s.uVexLength         = 0;
+    pVCpu->iem.s.fEvexStuff         = 0;
+    pVCpu->iem.s.iEffSeg            = X86_SREG_DS;
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf         = NULL;
+    pVCpu->iem.s.offInstrNextByte   = 0;
+    pVCpu->iem.s.offCurInstrStart   = 0;
+# ifdef VBOX_STRICT
+    pVCpu->iem.s.cbInstrBuf         = UINT16_MAX;
+    pVCpu->iem.s.cbInstrBufTotal    = UINT16_MAX;
+    pVCpu->iem.s.uInstrBufPc        = UINT64_C(0xc0ffc0ffcff0c0ff);
+# endif
+#else
+    pVCpu->iem.s.offOpcode          = 0;
+    pVCpu->iem.s.cbOpcode           = 0;
+#endif
+    pVCpu->iem.s.offModRm           = 0;
+    pVCpu->iem.s.cActiveMappings    = 0;
+    pVCpu->iem.s.iNextMapping       = 0;
+    pVCpu->iem.s.rcPassUp           = VINF_SUCCESS;
+    pVCpu->iem.s.fBypassHandlers    = fBypassHandlers;
+
+#ifdef DBGFTRACE_ENABLED
+    switch (enmMode)
+    {
+        case IEMMODE_64BIT:
+            RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
+            break;
+        case IEMMODE_32BIT:
+            RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
+            break;
+        case IEMMODE_16BIT:
+            RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
+            break;
+    }
+#endif
+}
+
+
+/**
+ * Reinitializes the decoder state 2nd+ loop of IEMExecLots.
+ *
+ * This is mostly a copy of iemInitDecoder.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ */
+DECLINLINE(void) iemReInitDecoder(PVMCPUCC pVCpu)
+{
+    Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
+
+    pVCpu->iem.s.uCpl               = CPUMGetGuestCPL(pVCpu);   /** @todo this should be updated during execution! */
+    IEMMODE enmMode = iemCalcCpuMode(pVCpu);
+    pVCpu->iem.s.enmCpuMode         = enmMode;                  /** @todo this should be updated during execution! */
+    pVCpu->iem.s.enmDefAddrMode     = enmMode;  /** @todo check if this is correct... */
+    pVCpu->iem.s.enmEffAddrMode     = enmMode;
+    if (enmMode != IEMMODE_64BIT)
+    {
+        pVCpu->iem.s.enmDefOpSize   = enmMode;  /** @todo check if this is correct... */
+        pVCpu->iem.s.enmEffOpSize   = enmMode;
+    }
+    else
+    {
+        pVCpu->iem.s.enmDefOpSize   = IEMMODE_32BIT;
+        pVCpu->iem.s.enmEffOpSize   = IEMMODE_32BIT;
+    }
+    pVCpu->iem.s.fPrefixes          = 0;
+    pVCpu->iem.s.uRexReg            = 0;
+    pVCpu->iem.s.uRexB              = 0;
+    pVCpu->iem.s.uRexIndex          = 0;
+    pVCpu->iem.s.idxPrefix          = 0;
+    pVCpu->iem.s.uVex3rdReg         = 0;
+    pVCpu->iem.s.uVexLength         = 0;
+    pVCpu->iem.s.fEvexStuff         = 0;
+    pVCpu->iem.s.iEffSeg            = X86_SREG_DS;
+#ifdef IEM_WITH_CODE_TLB
+    if (pVCpu->iem.s.pbInstrBuf)
+    {
+        uint64_t off = (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rip : pVCpu->cpum.GstCtx.eip + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base)
+                     - pVCpu->iem.s.uInstrBufPc;
+        if (off < pVCpu->iem.s.cbInstrBufTotal)
+        {
+            pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
+            pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
+            if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
+                pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
+            else
+                pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
+        }
+        else
+        {
+            pVCpu->iem.s.pbInstrBuf       = NULL;
+            pVCpu->iem.s.offInstrNextByte = 0;
+            pVCpu->iem.s.offCurInstrStart = 0;
+            pVCpu->iem.s.cbInstrBuf       = 0;
+            pVCpu->iem.s.cbInstrBufTotal  = 0;
+        }
+    }
+    else
+    {
+        pVCpu->iem.s.offInstrNextByte = 0;
+        pVCpu->iem.s.offCurInstrStart = 0;
+        pVCpu->iem.s.cbInstrBuf       = 0;
+        pVCpu->iem.s.cbInstrBufTotal  = 0;
+    }
+#else
+    pVCpu->iem.s.cbOpcode           = 0;
+    pVCpu->iem.s.offOpcode          = 0;
+#endif
+    pVCpu->iem.s.offModRm           = 0;
+    Assert(pVCpu->iem.s.cActiveMappings == 0);
+    pVCpu->iem.s.iNextMapping       = 0;
+    Assert(pVCpu->iem.s.rcPassUp   == VINF_SUCCESS);
+    Assert(pVCpu->iem.s.fBypassHandlers == false);
+
+#ifdef DBGFTRACE_ENABLED
+    switch (enmMode)
+    {
+        case IEMMODE_64BIT:
+            RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
+            break;
+        case IEMMODE_32BIT:
+            RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
+            break;
+        case IEMMODE_16BIT:
+            RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
+            break;
+    }
+#endif
+}
+
+
+
+/**
+ * Prefetch opcodes the first time when starting executing.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   fBypassHandlers     Whether to bypass access handlers.
+ */
+IEM_STATIC VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PVMCPUCC pVCpu, bool fBypassHandlers)
+{
+    iemInitDecoder(pVCpu, fBypassHandlers);
+
+#ifdef IEM_WITH_CODE_TLB
+    /** @todo Do ITLB lookup here. */
+
+#else /* !IEM_WITH_CODE_TLB */
+
+    /*
+     * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
+     *
+     * First translate CS:rIP to a physical address.
+     */
+    uint32_t    cbToTryRead;
+    RTGCPTR     GCPtrPC;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        cbToTryRead = PAGE_SIZE;
+        GCPtrPC     = pVCpu->cpum.GstCtx.rip;
+        if (IEM_IS_CANONICAL(GCPtrPC))
+            cbToTryRead = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    else
+    {
+        uint32_t GCPtrPC32 = pVCpu->cpum.GstCtx.eip;
+        AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT, ("%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+        if (GCPtrPC32 <= pVCpu->cpum.GstCtx.cs.u32Limit)
+            cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrPC32 + 1;
+        else
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+        if (cbToTryRead) { /* likely */ }
+        else /* overflowed */
+        {
+            Assert(GCPtrPC32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
+            cbToTryRead = UINT32_MAX;
+        }
+        GCPtrPC = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrPC32;
+        Assert(GCPtrPC <= UINT32_MAX);
+    }
+
+    RTGCPHYS    GCPhys;
+    uint64_t    fFlags;
+    int rc = PGMGstGetPage(pVCpu, GCPtrPC, &fFlags, &GCPhys);
+    if (RT_SUCCESS(rc)) { /* probable */ }
+    else
+    {
+        Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
+        return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, rc);
+    }
+    if ((fFlags & X86_PTE_US) || pVCpu->iem.s.uCpl != 3) { /* likely */ }
+    else
+    {
+        Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
+        return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+    }
+    if (!(fFlags & X86_PTE_PAE_NX) || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) { /* likely */ }
+    else
+    {
+        Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
+        return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+    }
+    GCPhys |= GCPtrPC & PAGE_OFFSET_MASK;
+    /** @todo Check reserved bits and such stuff. PGM is better at doing
+     *        that, so do it when implementing the guest virtual address
+     *        TLB... */
+
+    /*
+     * Read the bytes at this address.
+     */
+    uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
+    if (cbToTryRead > cbLeftOnPage)
+        cbToTryRead = cbLeftOnPage;
+    if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode))
+        cbToTryRead = sizeof(pVCpu->iem.s.abOpcode);
+
+    if (!pVCpu->iem.s.fBypassHandlers)
+    {
+        VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, pVCpu->iem.s.abOpcode, cbToTryRead, PGMACCESSORIGIN_IEM);
+        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        { /* likely */ }
+        else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+        {
+            Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status -  rcStrict=%Rrc\n",
+                 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+        }
+        else
+        {
+            Log((RT_SUCCESS(rcStrict)
+                 ? "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
+                 : "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
+                 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+            return rcStrict;
+        }
+    }
+    else
+    {
+        rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->iem.s.abOpcode, GCPhys, cbToTryRead);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n",
+                 GCPtrPC, GCPhys, rc, cbToTryRead));
+            return rc;
+        }
+    }
+    pVCpu->iem.s.cbOpcode = cbToTryRead;
+#endif /* !IEM_WITH_CODE_TLB */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Invalidates the IEM TLBs.
+ *
+ * This is called internally as well as by PGM when moving GC mappings.
+ *
+ * @returns
+ * @param   pVCpu       The cross context virtual CPU structure of the calling
+ *                      thread.
+ * @param   fVmm        Set when PGM calls us with a remapping.
+ */
+VMM_INT_DECL(void) IEMTlbInvalidateAll(PVMCPUCC pVCpu, bool fVmm)
+{
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.cbInstrBufTotal = 0;
+    pVCpu->iem.s.CodeTlb.uTlbRevision += IEMTLB_REVISION_INCR;
+    if (pVCpu->iem.s.CodeTlb.uTlbRevision != 0)
+    { /* very likely */ }
+    else
+    {
+        pVCpu->iem.s.CodeTlb.uTlbRevision = IEMTLB_REVISION_INCR;
+        unsigned i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
+        while (i-- > 0)
+            pVCpu->iem.s.CodeTlb.aEntries[i].uTag = 0;
+    }
+#endif
+
+#ifdef IEM_WITH_DATA_TLB
+    pVCpu->iem.s.DataTlb.uTlbRevision += IEMTLB_REVISION_INCR;
+    if (pVCpu->iem.s.DataTlb.uTlbRevision != 0)
+    { /* very likely */ }
+    else
+    {
+        pVCpu->iem.s.DataTlb.uTlbRevision = IEMTLB_REVISION_INCR;
+        unsigned i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
+        while (i-- > 0)
+            pVCpu->iem.s.DataTlb.aEntries[i].uTag = 0;
+    }
+#endif
+    NOREF(pVCpu); NOREF(fVmm);
+}
+
+
+/**
+ * Invalidates a page in the TLBs.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling
+ *                      thread.
+ * @param   GCPtr       The address of the page to invalidate
+ */
+VMM_INT_DECL(void) IEMTlbInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtr)
+{
+#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
+    GCPtr = GCPtr >> X86_PAGE_SHIFT;
+    AssertCompile(RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries) == 256);
+    AssertCompile(RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries) == 256);
+    uintptr_t idx = (uint8_t)GCPtr;
+
+# ifdef IEM_WITH_CODE_TLB
+    if (pVCpu->iem.s.CodeTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.CodeTlb.uTlbRevision))
+    {
+        pVCpu->iem.s.CodeTlb.aEntries[idx].uTag = 0;
+        if (GCPtr == (pVCpu->iem.s.uInstrBufPc >> X86_PAGE_SHIFT))
+            pVCpu->iem.s.cbInstrBufTotal = 0;
+    }
+# endif
+
+# ifdef IEM_WITH_DATA_TLB
+    if (pVCpu->iem.s.DataTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.DataTlb.uTlbRevision))
+        pVCpu->iem.s.DataTlb.aEntries[idx].uTag = 0;
+# endif
+#else
+    NOREF(pVCpu); NOREF(GCPtr);
+#endif
+}
+
+
+/**
+ * Invalidates the host physical aspects of the IEM TLBs.
+ *
+ * This is called internally as well as by PGM when moving GC mappings.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling
+ *                      thread.
+ */
+VMM_INT_DECL(void) IEMTlbInvalidateAllPhysical(PVMCPUCC pVCpu)
+{
+#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
+    /* Note! This probably won't end up looking exactly like this, but it give an idea... */
+
+# ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.cbInstrBufTotal = 0;
+# endif
+    uint64_t uTlbPhysRev = pVCpu->iem.s.CodeTlb.uTlbPhysRev + IEMTLB_PHYS_REV_INCR;
+    if (uTlbPhysRev != 0)
+    {
+        pVCpu->iem.s.CodeTlb.uTlbPhysRev = uTlbPhysRev;
+        pVCpu->iem.s.DataTlb.uTlbPhysRev = uTlbPhysRev;
+    }
+    else
+    {
+        pVCpu->iem.s.CodeTlb.uTlbPhysRev = IEMTLB_PHYS_REV_INCR;
+        pVCpu->iem.s.DataTlb.uTlbPhysRev = IEMTLB_PHYS_REV_INCR;
+
+        unsigned i;
+# ifdef IEM_WITH_CODE_TLB
+        i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
+        while (i-- > 0)
+        {
+            pVCpu->iem.s.CodeTlb.aEntries[i].pbMappingR3       = NULL;
+            pVCpu->iem.s.CodeTlb.aEntries[i].fFlagsAndPhysRev &= ~(IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PHYS_REV);
+        }
+# endif
+# ifdef IEM_WITH_DATA_TLB
+        i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
+        while (i-- > 0)
+        {
+            pVCpu->iem.s.DataTlb.aEntries[i].pbMappingR3       = NULL;
+            pVCpu->iem.s.DataTlb.aEntries[i].fFlagsAndPhysRev &= ~(IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PHYS_REV);
+        }
+# endif
+    }
+#else
+    NOREF(pVCpu);
+#endif
+}
+
+
+/**
+ * Invalidates the host physical aspects of the IEM TLBs.
+ *
+ * This is called internally as well as by PGM when moving GC mappings.
+ *
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks Caller holds the PGM lock.
+ */
+VMM_INT_DECL(void) IEMTlbInvalidateAllPhysicalAllCpus(PVM pVM)
+{
+    RT_NOREF_PV(pVM);
+}
+
+#ifdef IEM_WITH_CODE_TLB
+
+/**
+ * Tries to fetches @a cbDst opcode bytes, raise the appropriate exception on
+ * failure and jumps.
+ *
+ * We end up here for a number of reasons:
+ *      - pbInstrBuf isn't yet initialized.
+ *      - Advancing beyond the buffer boundrary (e.g. cross page).
+ *      - Advancing beyond the CS segment limit.
+ *      - Fetching from non-mappable page (e.g. MMIO).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   pvDst               Where to return the bytes.
+ * @param   cbDst               Number of bytes to read.
+ *
+ * @todo    Make cbDst = 0 a way of initializing pbInstrBuf?
+ */
+IEM_STATIC void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst)
+{
+#ifdef IN_RING3
+    for (;;)
+    {
+        Assert(cbDst <= 8);
+        uint32_t offBuf = pVCpu->iem.s.offInstrNextByte;
+
+        /*
+         * We might have a partial buffer match, deal with that first to make the
+         * rest simpler.  This is the first part of the cross page/buffer case.
+         */
+        if (pVCpu->iem.s.pbInstrBuf != NULL)
+        {
+            if (offBuf < pVCpu->iem.s.cbInstrBuf)
+            {
+                Assert(offBuf + cbDst > pVCpu->iem.s.cbInstrBuf);
+                uint32_t const cbCopy = pVCpu->iem.s.cbInstrBuf - pVCpu->iem.s.offInstrNextByte;
+                memcpy(pvDst, &pVCpu->iem.s.pbInstrBuf[offBuf], cbCopy);
+
+                cbDst  -= cbCopy;
+                pvDst   = (uint8_t *)pvDst + cbCopy;
+                offBuf += cbCopy;
+                pVCpu->iem.s.offInstrNextByte += offBuf;
+            }
+        }
+
+        /*
+         * Check segment limit, figuring how much we're allowed to access at this point.
+         *
+         * We will fault immediately if RIP is past the segment limit / in non-canonical
+         * territory.  If we do continue, there are one or more bytes to read before we
+         * end up in trouble and we need to do that first before faulting.
+         */
+        RTGCPTR  GCPtrFirst;
+        uint32_t cbMaxRead;
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        {
+            GCPtrFirst = pVCpu->cpum.GstCtx.rip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
+            if (RT_LIKELY(IEM_IS_CANONICAL(GCPtrFirst)))
+            { /* likely */ }
+            else
+                iemRaiseGeneralProtectionFault0Jmp(pVCpu);
+            cbMaxRead = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
+        }
+        else
+        {
+            GCPtrFirst = pVCpu->cpum.GstCtx.eip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
+            Assert(!(GCPtrFirst & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
+            if (RT_LIKELY((uint32_t)GCPtrFirst <= pVCpu->cpum.GstCtx.cs.u32Limit))
+            { /* likely */ }
+            else
+                iemRaiseSelectorBoundsJmp(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+            cbMaxRead = pVCpu->cpum.GstCtx.cs.u32Limit - (uint32_t)GCPtrFirst + 1;
+            if (cbMaxRead != 0)
+            { /* likely */ }
+            else
+            {
+                /* Overflowed because address is 0 and limit is max. */
+                Assert(GCPtrFirst == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
+                cbMaxRead = X86_PAGE_SIZE;
+            }
+            GCPtrFirst = (uint32_t)GCPtrFirst + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base;
+            uint32_t cbMaxRead2 = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
+            if (cbMaxRead2 < cbMaxRead)
+                cbMaxRead = cbMaxRead2;
+            /** @todo testcase: unreal modes, both huge 16-bit and 32-bit. */
+        }
+
+        /*
+         * Get the TLB entry for this piece of code.
+         */
+        uint64_t     uTag  = (GCPtrFirst >> X86_PAGE_SHIFT) | pVCpu->iem.s.CodeTlb.uTlbRevision;
+        AssertCompile(RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries) == 256);
+        PIEMTLBENTRY pTlbe = &pVCpu->iem.s.CodeTlb.aEntries[(uint8_t)uTag];
+        if (pTlbe->uTag == uTag)
+        {
+            /* likely when executing lots of code, otherwise unlikely */
+# ifdef VBOX_WITH_STATISTICS
+            pVCpu->iem.s.CodeTlb.cTlbHits++;
+# endif
+        }
+        else
+        {
+            pVCpu->iem.s.CodeTlb.cTlbMisses++;
+            RTGCPHYS    GCPhys;
+            uint64_t    fFlags;
+            int rc = PGMGstGetPage(pVCpu, GCPtrFirst, &fFlags, &GCPhys);
+            if (RT_FAILURE(rc))
+            {
+                Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc));
+                iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, rc);
+            }
+
+            AssertCompile(IEMTLBE_F_PT_NO_EXEC == 1);
+            pTlbe->uTag             = uTag;
+            pTlbe->fFlagsAndPhysRev = (~fFlags & (X86_PTE_US | X86_PTE_RW | X86_PTE_D)) | (fFlags >> X86_PTE_PAE_BIT_NX);
+            pTlbe->GCPhys           = GCPhys;
+            pTlbe->pbMappingR3      = NULL;
+        }
+
+        /*
+         * Check TLB page table level access flags.
+         */
+        if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER | IEMTLBE_F_PT_NO_EXEC))
+        {
+            if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) && pVCpu->iem.s.uCpl == 3)
+            {
+                Log(("iemOpcodeFetchBytesJmp: %RGv - supervisor page\n", GCPtrFirst));
+                iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+            }
+            if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
+            {
+                Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrFirst));
+                iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+            }
+        }
+
+        /*
+         * Look up the physical page info if necessary.
+         */
+        if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
+        { /* not necessary */ }
+        else
+        {
+            AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE     == IEMTLBE_F_PG_NO_WRITE);
+            AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ      == IEMTLBE_F_PG_NO_READ);
+            AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
+            pTlbe->fFlagsAndPhysRev &= ~(  IEMTLBE_F_PHYS_REV
+                                         | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE);
+            int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.CodeTlb.uTlbPhysRev,
+                                                &pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev);
+            AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
+        }
+
+# if defined(IN_RING3) || (defined(IN_RING0) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE))
+        /*
+         * Try do a direct read using the pbMappingR3 pointer.
+         */
+        if (    (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ))
+             == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
+        {
+            uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
+            pVCpu->iem.s.cbInstrBufTotal  = offPg + cbMaxRead;
+            if (offBuf == (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart)
+            {
+                pVCpu->iem.s.cbInstrBuf       = offPg + RT_MIN(15, cbMaxRead);
+                pVCpu->iem.s.offCurInstrStart = (int16_t)offPg;
+            }
+            else
+            {
+                uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
+                Assert(cbInstr < cbMaxRead);
+                pVCpu->iem.s.cbInstrBuf       = offPg + RT_MIN(cbMaxRead + cbInstr, 15) - cbInstr;
+                pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
+            }
+            if (cbDst <= cbMaxRead)
+            {
+                pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst;
+                pVCpu->iem.s.uInstrBufPc      = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
+                pVCpu->iem.s.pbInstrBuf       = pTlbe->pbMappingR3;
+                memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbDst);
+                return;
+            }
+            pVCpu->iem.s.pbInstrBuf = NULL;
+
+            memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbMaxRead);
+            pVCpu->iem.s.offInstrNextByte = offPg + cbMaxRead;
+        }
+        else
+# endif
+#if 0
+        /*
+         * If there is no special read handling, so we can read a bit more and
+         * put it in the prefetch buffer.
+         */
+        if (   cbDst < cbMaxRead
+            && (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_PG_NO_READ)) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
+        {
+            VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys,
+                                                &pVCpu->iem.s.abOpcode[0], cbToTryRead, PGMACCESSORIGIN_IEM);
+            if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+            { /* likely */ }
+            else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+            {
+                Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status -  rcStrict=%Rrc\n",
+                     GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+                rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICRC_VAL(rcStrict)));
+            }
+            else
+            {
+                Log((RT_SUCCESS(rcStrict)
+                     ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
+                     : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
+                     GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+                longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+            }
+        }
+        /*
+         * Special read handling, so only read exactly what's needed.
+         * This is a highly unlikely scenario.
+         */
+        else
+#endif
+        {
+            pVCpu->iem.s.CodeTlb.cTlbSlowReadPath++;
+            uint32_t const cbToRead = RT_MIN((uint32_t)cbDst, cbMaxRead);
+            VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK),
+                                                pvDst, cbToRead, PGMACCESSORIGIN_IEM);
+            if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+            { /* likely */ }
+            else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+            {
+                Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status -  rcStrict=%Rrc\n",
+                     GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
+                rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict)));
+            }
+            else
+            {
+                Log((RT_SUCCESS(rcStrict)
+                     ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
+                     : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
+                     GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
+                longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+            }
+            pVCpu->iem.s.offInstrNextByte = offBuf + cbToRead;
+            if (cbToRead == cbDst)
+                return;
+        }
+
+        /*
+         * More to read, loop.
+         */
+        cbDst -= cbMaxRead;
+        pvDst  = (uint8_t *)pvDst + cbMaxRead;
+    }
+#else
+    RT_NOREF(pvDst, cbDst);
+    longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VERR_INTERNAL_ERROR);
+#endif
+}
+
+#else
+
+/**
+ * Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
+ * exception if it fails.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   cbMin               The minimum number of bytes relative offOpcode
+ *                              that must be read.
+ */
+IEM_STATIC VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin)
+{
+    /*
+     * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
+     *
+     * First translate CS:rIP to a physical address.
+     */
+    uint8_t     cbLeft = pVCpu->iem.s.cbOpcode - pVCpu->iem.s.offOpcode; Assert(cbLeft < cbMin);
+    uint32_t    cbToTryRead;
+    RTGCPTR     GCPtrNext;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        cbToTryRead = PAGE_SIZE;
+        GCPtrNext   = pVCpu->cpum.GstCtx.rip + pVCpu->iem.s.cbOpcode;
+        if (!IEM_IS_CANONICAL(GCPtrNext))
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    else
+    {
+        uint32_t GCPtrNext32 = pVCpu->cpum.GstCtx.eip;
+        Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
+        GCPtrNext32 += pVCpu->iem.s.cbOpcode;
+        if (GCPtrNext32 > pVCpu->cpum.GstCtx.cs.u32Limit)
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+        cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrNext32 + 1;
+        if (!cbToTryRead) /* overflowed */
+        {
+            Assert(GCPtrNext32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
+            cbToTryRead = UINT32_MAX;
+            /** @todo check out wrapping around the code segment.  */
+        }
+        if (cbToTryRead < cbMin - cbLeft)
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+        GCPtrNext = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrNext32;
+    }
+
+    /* Only read up to the end of the page, and make sure we don't read more
+       than the opcode buffer can hold. */
+    uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrNext & PAGE_OFFSET_MASK);
+    if (cbToTryRead > cbLeftOnPage)
+        cbToTryRead = cbLeftOnPage;
+    if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode)
+        cbToTryRead = sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode;
+/** @todo r=bird: Convert assertion into undefined opcode exception? */
+    Assert(cbToTryRead >= cbMin - cbLeft); /* ASSUMPTION based on iemInitDecoderAndPrefetchOpcodes. */
+
+    RTGCPHYS    GCPhys;
+    uint64_t    fFlags;
+    int rc = PGMGstGetPage(pVCpu, GCPtrNext, &fFlags, &GCPhys);
+    if (RT_FAILURE(rc))
+    {
+        Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
+        return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, rc);
+    }
+    if (!(fFlags & X86_PTE_US) && pVCpu->iem.s.uCpl == 3)
+    {
+        Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
+        return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+    }
+    if ((fFlags & X86_PTE_PAE_NX) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
+    {
+        Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
+        return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+    }
+    GCPhys |= GCPtrNext & PAGE_OFFSET_MASK;
+    Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n",  GCPtrNext,  GCPhys,  pVCpu->iem.s.cbOpcode));
+    /** @todo Check reserved bits and such stuff. PGM is better at doing
+     *        that, so do it when implementing the guest virtual address
+     *        TLB... */
+
+    /*
+     * Read the bytes at this address.
+     *
+     * We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already,
+     * and since PATM should only patch the start of an instruction there
+     * should be no need to check again here.
+     */
+    if (!pVCpu->iem.s.fBypassHandlers)
+    {
+        VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode],
+                                            cbToTryRead, PGMACCESSORIGIN_IEM);
+        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        { /* likely */ }
+        else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+        {
+            Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status -  rcStrict=%Rrc\n",
+                 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+        }
+        else
+        {
+            Log((RT_SUCCESS(rcStrict)
+                 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
+                 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
+                 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+            return rcStrict;
+        }
+    }
+    else
+    {
+        rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode], GCPhys, cbToTryRead);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc));
+            return rc;
+        }
+    }
+    pVCpu->iem.s.cbOpcode += cbToTryRead;
+    Log5(("%.*Rhxs\n", pVCpu->iem.s.cbOpcode, pVCpu->iem.s.abOpcode));
+
+    return VINF_SUCCESS;
+}
+
+#endif /* !IEM_WITH_CODE_TLB */
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   pb                  Where to return the opcode byte.
+ */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb)
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pb = pVCpu->iem.s.abOpcode[offOpcode];
+        pVCpu->iem.s.offOpcode = offOpcode + 1;
+    }
+    else
+        *pb = 0;
+    return rcStrict;
+}
+
+
+/**
+ * Fetches the next opcode byte.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   pu8                 Where to return the opcode byte.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8)
+{
+    uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
+    {
+        pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
+        *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
+        return VINF_SUCCESS;
+    }
+    return iemOpcodeGetNextU8Slow(pVCpu, pu8);
+}
+
+#else  /* IEM_WITH_SETJMP */
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU8Jmp doesn't like, longjmp on error.
+ *
+ * @returns The opcode byte.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECL_NO_INLINE(IEM_STATIC, uint8_t) iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu)
+{
+# ifdef IEM_WITH_CODE_TLB
+    uint8_t u8;
+    iemOpcodeFetchBytesJmp(pVCpu, sizeof(u8), &u8);
+    return u8;
+# else
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
+    if (rcStrict == VINF_SUCCESS)
+        return pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++];
+    longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+# endif
+}
+
+
+/**
+ * Fetches the next opcode byte, longjmp on error.
+ *
+ * @returns The opcode byte.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu)
+{
+# ifdef IEM_WITH_CODE_TLB
+    uintptr_t       offBuf = pVCpu->iem.s.offInstrNextByte;
+    uint8_t const  *pbBuf  = pVCpu->iem.s.pbInstrBuf;
+    if (RT_LIKELY(   pbBuf != NULL
+                  && offBuf < pVCpu->iem.s.cbInstrBuf))
+    {
+        pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
+        return pbBuf[offBuf];
+    }
+# else
+    uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
+    {
+        pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
+        return pVCpu->iem.s.abOpcode[offOpcode];
+    }
+# endif
+    return iemOpcodeGetNextU8SlowJmp(pVCpu);
+}
+
+#endif /* IEM_WITH_SETJMP */
+
+/**
+ * Fetches the next opcode byte, returns automatically on failure.
+ *
+ * @param   a_pu8               Where to return the opcode byte.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
+        if (rcStrict2 == VINF_SUCCESS) \
+        { /* likely */ } \
+        else \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
+#endif /* IEM_WITH_SETJMP */
+
+
+#ifndef IEM_WITH_SETJMP
+/**
+ * Fetches the next signed byte from the opcode stream.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pi8                 Where to return the signed byte.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8)
+{
+    return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
+}
+#endif /* !IEM_WITH_SETJMP */
+
+
+/**
+ * Fetches the next signed byte from the opcode stream, returning automatically
+ * on failure.
+ *
+ * @param   a_pi8               Where to return the signed byte.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else /* IEM_WITH_SETJMP */
+# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
+
+#endif /* IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16                Where to return the opcode dword.
+ */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16)
+{
+    uint8_t      u8;
+    VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
+    if (rcStrict == VINF_SUCCESS)
+        *pu16 = (int8_t)u8;
+    return rcStrict;
+}
+
+
+/**
+ * Fetches the next signed byte from the opcode stream, extending it to
+ * unsigned 16-bit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16                Where to return the unsigned word.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16)
+{
+    uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
+        return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
+
+    *pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
+    pVCpu->iem.s.offOpcode = offOpcode + 1;
+    return VINF_SUCCESS;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+/**
+ * Fetches the next signed byte from the opcode stream and sign-extending it to
+ * a word, returning automatically on failure.
+ *
+ * @param   a_pu16              Where to return the word.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32                Where to return the opcode dword.
+ */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32)
+{
+    uint8_t      u8;
+    VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
+    if (rcStrict == VINF_SUCCESS)
+        *pu32 = (int8_t)u8;
+    return rcStrict;
+}
+
+
+/**
+ * Fetches the next signed byte from the opcode stream, extending it to
+ * unsigned 32-bit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32                Where to return the unsigned dword.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32)
+{
+    uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
+        return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
+
+    *pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
+    pVCpu->iem.s.offOpcode = offOpcode + 1;
+    return VINF_SUCCESS;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+/**
+ * Fetches the next signed byte from the opcode stream and sign-extending it to
+ * a word, returning automatically on failure.
+ *
+ * @param   a_pu32              Where to return the word.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+#define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode qword.
+ */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
+{
+    uint8_t      u8;
+    VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
+    if (rcStrict == VINF_SUCCESS)
+        *pu64 = (int8_t)u8;
+    return rcStrict;
+}
+
+
+/**
+ * Fetches the next signed byte from the opcode stream, extending it to
+ * unsigned 64-bit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the unsigned qword.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
+{
+    uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
+        return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
+
+    *pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
+    pVCpu->iem.s.offOpcode = offOpcode + 1;
+    return VINF_SUCCESS;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+
+/**
+ * Fetches the next signed byte from the opcode stream and sign-extending it to
+ * a word, returning automatically on failure.
+ *
+ * @param   a_pu64              Where to return the word.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
+#endif
+
+
+#ifndef IEM_WITH_SETJMP
+/**
+ * Fetches the next opcode byte.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   pu8                 Where to return the opcode byte.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8)
+{
+    uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
+    pVCpu->iem.s.offModRm = offOpcode;
+    if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
+    {
+        pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
+        *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
+        return VINF_SUCCESS;
+    }
+    return iemOpcodeGetNextU8Slow(pVCpu, pu8);
+}
+#else  /* IEM_WITH_SETJMP */
+/**
+ * Fetches the next opcode byte, longjmp on error.
+ *
+ * @returns The opcode byte.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu)
+{
+# ifdef IEM_WITH_CODE_TLB
+    uintptr_t       offBuf = pVCpu->iem.s.offInstrNextByte;
+    pVCpu->iem.s.offModRm  = offBuf;
+    uint8_t const  *pbBuf  = pVCpu->iem.s.pbInstrBuf;
+    if (RT_LIKELY(   pbBuf != NULL
+                  && offBuf < pVCpu->iem.s.cbInstrBuf))
+    {
+        pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
+        return pbBuf[offBuf];
+    }
+# else
+    uintptr_t offOpcode   = pVCpu->iem.s.offOpcode;
+    pVCpu->iem.s.offModRm = offOpcode;
+    if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
+    {
+        pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
+        return pVCpu->iem.s.abOpcode[offOpcode];
+    }
+# endif
+    return iemOpcodeGetNextU8SlowJmp(pVCpu);
+}
+#endif /* IEM_WITH_SETJMP */
+
+/**
+ * Fetches the next opcode byte, which is a ModR/M byte, returns automatically
+ * on failure.
+ *
+ * Will note down the position of the ModR/M byte for VT-x exits.
+ *
+ * @param   a_pbRm              Where to return the RM opcode byte.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
+        if (rcStrict2 == VINF_SUCCESS) \
+        { /* likely */ } \
+        else \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
+#endif /* IEM_WITH_SETJMP */
+
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16                Where to return the opcode word.
+ */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16)
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+# else
+        *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+# endif
+        pVCpu->iem.s.offOpcode = offOpcode + 2;
+    }
+    else
+        *pu16 = 0;
+    return rcStrict;
+}
+
+
+/**
+ * Fetches the next opcode word.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16                Where to return the opcode word.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16)
+{
+    uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
+    {
+        pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
+# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+# else
+        *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+# endif
+        return VINF_SUCCESS;
+    }
+    return iemOpcodeGetNextU16Slow(pVCpu, pu16);
+}
+
+#else  /* IEM_WITH_SETJMP */
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU16Jmp doesn't like, longjmp on error
+ *
+ * @returns The opcode word.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECL_NO_INLINE(IEM_STATIC, uint16_t) iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu)
+{
+# ifdef IEM_WITH_CODE_TLB
+    uint16_t u16;
+    iemOpcodeFetchBytesJmp(pVCpu, sizeof(u16), &u16);
+    return u16;
+# else
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        pVCpu->iem.s.offOpcode += 2;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+#  else
+        return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+#  endif
+    }
+    longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+# endif
+}
+
+
+/**
+ * Fetches the next opcode word, longjmp on error.
+ *
+ * @returns The opcode word.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu)
+{
+# ifdef IEM_WITH_CODE_TLB
+    uintptr_t       offBuf = pVCpu->iem.s.offInstrNextByte;
+    uint8_t const  *pbBuf  = pVCpu->iem.s.pbInstrBuf;
+    if (RT_LIKELY(   pbBuf != NULL
+                  && offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
+    {
+        pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint16_t const *)&pbBuf[offBuf];
+#  else
+        return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
+#  endif
+    }
+# else
+    uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
+    {
+        pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+#  else
+        return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+#  endif
+    }
+# endif
+    return iemOpcodeGetNextU16SlowJmp(pVCpu);
+}
+
+#endif /* IEM_WITH_SETJMP */
+
+
+/**
+ * Fetches the next opcode word, returns automatically on failure.
+ *
+ * @param   a_pu16              Where to return the opcode word.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32                Where to return the opcode double word.
+ */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32)
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+        pVCpu->iem.s.offOpcode = offOpcode + 2;
+    }
+    else
+        *pu32 = 0;
+    return rcStrict;
+}
+
+
+/**
+ * Fetches the next opcode word, zero extending it to a double word.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32                Where to return the opcode double word.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32)
+{
+    uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
+        return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
+
+    *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+    pVCpu->iem.s.offOpcode = offOpcode + 2;
+    return VINF_SUCCESS;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+
+/**
+ * Fetches the next opcode word and zero extends it to a double word, returns
+ * automatically on failure.
+ *
+ * @param   a_pu32              Where to return the opcode double word.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode quad word.
+ */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+        pVCpu->iem.s.offOpcode = offOpcode + 2;
+    }
+    else
+        *pu64 = 0;
+    return rcStrict;
+}
+
+
+/**
+ * Fetches the next opcode word, zero extending it to a quad word.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode quad word.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
+{
+    uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
+        return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
+
+    *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+    pVCpu->iem.s.offOpcode = offOpcode + 2;
+    return VINF_SUCCESS;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+/**
+ * Fetches the next opcode word and zero extends it to a quad word, returns
+ * automatically on failure.
+ *
+ * @param   a_pu64              Where to return the opcode quad word.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64)  (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
+#endif
+
+
+#ifndef IEM_WITH_SETJMP
+/**
+ * Fetches the next signed word from the opcode stream.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pi16                Where to return the signed word.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16)
+{
+    return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
+}
+#endif /* !IEM_WITH_SETJMP */
+
+
+/**
+ * Fetches the next signed word from the opcode stream, returning automatically
+ * on failure.
+ *
+ * @param   a_pi16              Where to return the signed word.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32                Where to return the opcode dword.
+ */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32)
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+# else
+        *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 3]);
+# endif
+        pVCpu->iem.s.offOpcode = offOpcode + 4;
+    }
+    else
+        *pu32 = 0;
+    return rcStrict;
+}
+
+
+/**
+ * Fetches the next opcode dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32                Where to return the opcode double word.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32)
+{
+    uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
+    {
+        pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
+# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+# else
+        *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 3]);
+# endif
+        return VINF_SUCCESS;
+    }
+    return iemOpcodeGetNextU32Slow(pVCpu, pu32);
+}
+
+#else  /* !IEM_WITH_SETJMP */
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU32Jmp doesn't like, longjmp on error.
+ *
+ * @returns The opcode dword.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECL_NO_INLINE(IEM_STATIC, uint32_t) iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu)
+{
+# ifdef IEM_WITH_CODE_TLB
+    uint32_t u32;
+    iemOpcodeFetchBytesJmp(pVCpu, sizeof(u32), &u32);
+    return u32;
+# else
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        pVCpu->iem.s.offOpcode = offOpcode + 4;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+#  else
+        return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 3]);
+#  endif
+    }
+    longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+# endif
+}
+
+
+/**
+ * Fetches the next opcode dword, longjmp on error.
+ *
+ * @returns The opcode dword.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu)
+{
+# ifdef IEM_WITH_CODE_TLB
+    uintptr_t       offBuf = pVCpu->iem.s.offInstrNextByte;
+    uint8_t const  *pbBuf  = pVCpu->iem.s.pbInstrBuf;
+    if (RT_LIKELY(   pbBuf != NULL
+                  && offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
+    {
+        pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint32_t const *)&pbBuf[offBuf];
+#  else
+        return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
+                                   pbBuf[offBuf + 1],
+                                   pbBuf[offBuf + 2],
+                                   pbBuf[offBuf + 3]);
+#  endif
+    }
+# else
+    uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
+    {
+        pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+#  else
+        return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 3]);
+#  endif
+    }
+# endif
+    return iemOpcodeGetNextU32SlowJmp(pVCpu);
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+
+/**
+ * Fetches the next opcode dword, returns automatically on failure.
+ *
+ * @param   a_pu32              Where to return the opcode dword.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode dword.
+ */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 3]);
+        pVCpu->iem.s.offOpcode = offOpcode + 4;
+    }
+    else
+        *pu64 = 0;
+    return rcStrict;
+}
+
+
+/**
+ * Fetches the next opcode dword, zero extending it to a quad word.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode quad word.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
+{
+    uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
+        return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
+
+    *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                pVCpu->iem.s.abOpcode[offOpcode + 3]);
+    pVCpu->iem.s.offOpcode = offOpcode + 4;
+    return VINF_SUCCESS;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+
+/**
+ * Fetches the next opcode dword and zero extends it to a quad word, returns
+ * automatically on failure.
+ *
+ * @param   a_pu64              Where to return the opcode quad word.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
+#endif
+
+
+#ifndef IEM_WITH_SETJMP
+/**
+ * Fetches the next signed double word from the opcode stream.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pi32                Where to return the signed double word.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32)
+{
+    return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
+}
+#endif
+
+/**
+ * Fetches the next signed double word from the opcode stream, returning
+ * automatically on failure.
+ *
+ * @param   a_pi32              Where to return the signed double word.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_S32(a_pi32)    (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode qword.
+ */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                             pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                             pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                             pVCpu->iem.s.abOpcode[offOpcode + 3]);
+        pVCpu->iem.s.offOpcode = offOpcode + 4;
+    }
+    else
+        *pu64 = 0;
+    return rcStrict;
+}
+
+
+/**
+ * Fetches the next opcode dword, sign extending it into a quad word.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode quad word.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
+{
+    uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
+        return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
+
+    int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                      pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                      pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                      pVCpu->iem.s.abOpcode[offOpcode + 3]);
+    *pu64 = i32;
+    pVCpu->iem.s.offOpcode = offOpcode + 4;
+    return VINF_SUCCESS;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+
+/**
+ * Fetches the next opcode double word and sign extends it to a quad word,
+ * returns automatically on failure.
+ *
+ * @param   a_pu64              Where to return the opcode quad word.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode qword.
+ */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+# else
+        *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 3],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 4],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 5],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 6],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 7]);
+# endif
+        pVCpu->iem.s.offOpcode = offOpcode + 8;
+    }
+    else
+        *pu64 = 0;
+    return rcStrict;
+}
+
+
+/**
+ * Fetches the next opcode qword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode qword.
+ */
+DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64)
+{
+    uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
+    {
+# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+# else
+        *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 3],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 4],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 5],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 6],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 7]);
+# endif
+        pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
+        return VINF_SUCCESS;
+    }
+    return iemOpcodeGetNextU64Slow(pVCpu, pu64);
+}
+
+#else  /* IEM_WITH_SETJMP */
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU64Jmp doesn't like, longjmp on error.
+ *
+ * @returns The opcode qword.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECL_NO_INLINE(IEM_STATIC, uint64_t) iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu)
+{
+# ifdef IEM_WITH_CODE_TLB
+    uint64_t u64;
+    iemOpcodeFetchBytesJmp(pVCpu, sizeof(u64), &u64);
+    return u64;
+# else
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        pVCpu->iem.s.offOpcode = offOpcode + 8;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+#  else
+        return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 3],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 4],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 5],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 6],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 7]);
+#  endif
+    }
+    longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+# endif
+}
+
+
+/**
+ * Fetches the next opcode qword, longjmp on error.
+ *
+ * @returns The opcode qword.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu)
+{
+# ifdef IEM_WITH_CODE_TLB
+    uintptr_t       offBuf = pVCpu->iem.s.offInstrNextByte;
+    uint8_t const  *pbBuf  = pVCpu->iem.s.pbInstrBuf;
+    if (RT_LIKELY(   pbBuf != NULL
+                  && offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
+    {
+        pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint64_t const *)&pbBuf[offBuf];
+#  else
+        return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
+                                   pbBuf[offBuf + 1],
+                                   pbBuf[offBuf + 2],
+                                   pbBuf[offBuf + 3],
+                                   pbBuf[offBuf + 4],
+                                   pbBuf[offBuf + 5],
+                                   pbBuf[offBuf + 6],
+                                   pbBuf[offBuf + 7]);
+#  endif
+    }
+# else
+    uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
+    if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
+    {
+        pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+#  else
+        return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 3],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 4],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 5],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 6],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 7]);
+#  endif
+    }
+# endif
+    return iemOpcodeGetNextU64SlowJmp(pVCpu);
+}
+
+#endif /* IEM_WITH_SETJMP */
+
+/**
+ * Fetches the next opcode quad word, returns automatically on failure.
+ *
+ * @param   a_pu64              Where to return the opcode quad word.
+ * @remark Implicitly references pVCpu.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+#else
+# define IEM_OPCODE_GET_NEXT_U64(a_pu64)    ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
+#endif
+
+
+/** @name  Misc Worker Functions.
+ * @{
+ */
+
+/**
+ * Gets the exception class for the specified exception vector.
+ *
+ * @returns The class of the specified exception.
+ * @param   uVector       The exception vector.
+ */
+IEM_STATIC IEMXCPTCLASS iemGetXcptClass(uint8_t uVector)
+{
+    Assert(uVector <= X86_XCPT_LAST);
+    switch (uVector)
+    {
+        case X86_XCPT_DE:
+        case X86_XCPT_TS:
+        case X86_XCPT_NP:
+        case X86_XCPT_SS:
+        case X86_XCPT_GP:
+        case X86_XCPT_SX:   /* AMD only */
+            return IEMXCPTCLASS_CONTRIBUTORY;
+
+        case X86_XCPT_PF:
+        case X86_XCPT_VE:   /* Intel only */
+            return IEMXCPTCLASS_PAGE_FAULT;
+
+        case X86_XCPT_DF:
+            return IEMXCPTCLASS_DOUBLE_FAULT;
+    }
+    return IEMXCPTCLASS_BENIGN;
+}
+
+
+/**
+ * Evaluates how to handle an exception caused during delivery of another event
+ * (exception / interrupt).
+ *
+ * @returns How to handle the recursive exception.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   fPrevFlags          The flags of the previous event.
+ * @param   uPrevVector         The vector of the previous event.
+ * @param   fCurFlags           The flags of the current exception.
+ * @param   uCurVector          The vector of the current exception.
+ * @param   pfXcptRaiseInfo     Where to store additional information about the
+ *                              exception condition. Optional.
+ */
+VMM_INT_DECL(IEMXCPTRAISE) IEMEvaluateRecursiveXcpt(PVMCPUCC pVCpu, uint32_t fPrevFlags, uint8_t uPrevVector, uint32_t fCurFlags,
+                                                    uint8_t uCurVector, PIEMXCPTRAISEINFO pfXcptRaiseInfo)
+{
+    /*
+     * Only CPU exceptions can be raised while delivering other events, software interrupt
+     * (INTn/INT3/INTO/ICEBP) generated exceptions cannot occur as the current (second) exception.
+     */
+    AssertReturn(fCurFlags & IEM_XCPT_FLAGS_T_CPU_XCPT, IEMXCPTRAISE_INVALID);
+    Assert(pVCpu); RT_NOREF(pVCpu);
+    Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x\n", uPrevVector, uCurVector));
+
+    IEMXCPTRAISE     enmRaise   = IEMXCPTRAISE_CURRENT_XCPT;
+    IEMXCPTRAISEINFO fRaiseInfo = IEMXCPTRAISEINFO_NONE;
+    if (fPrevFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+    {
+        IEMXCPTCLASS enmPrevXcptClass = iemGetXcptClass(uPrevVector);
+        if (enmPrevXcptClass != IEMXCPTCLASS_BENIGN)
+        {
+            IEMXCPTCLASS enmCurXcptClass = iemGetXcptClass(uCurVector);
+            if (   enmPrevXcptClass == IEMXCPTCLASS_PAGE_FAULT
+                && (   enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT
+                    || enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY))
+            {
+                enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
+                fRaiseInfo = enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT ? IEMXCPTRAISEINFO_PF_PF
+                                                                        : IEMXCPTRAISEINFO_PF_CONTRIBUTORY_XCPT;
+                Log2(("IEMEvaluateRecursiveXcpt: Vectoring page fault. uPrevVector=%#x uCurVector=%#x uCr2=%#RX64\n", uPrevVector,
+                      uCurVector, pVCpu->cpum.GstCtx.cr2));
+            }
+            else if (   enmPrevXcptClass == IEMXCPTCLASS_CONTRIBUTORY
+                     && enmCurXcptClass  == IEMXCPTCLASS_CONTRIBUTORY)
+            {
+                enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
+                Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x -> #DF\n", uPrevVector, uCurVector));
+            }
+            else if (   enmPrevXcptClass == IEMXCPTCLASS_DOUBLE_FAULT
+                     && (   enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY
+                         || enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT))
+            {
+                enmRaise = IEMXCPTRAISE_TRIPLE_FAULT;
+                Log2(("IEMEvaluateRecursiveXcpt: #DF handler raised a %#x exception -> triple fault\n", uCurVector));
+            }
+        }
+        else
+        {
+            if (uPrevVector == X86_XCPT_NMI)
+            {
+                fRaiseInfo = IEMXCPTRAISEINFO_NMI_XCPT;
+                if (uCurVector == X86_XCPT_PF)
+                {
+                    fRaiseInfo |= IEMXCPTRAISEINFO_NMI_PF;
+                    Log2(("IEMEvaluateRecursiveXcpt: NMI delivery caused a page fault\n"));
+                }
+            }
+            else if (   uPrevVector == X86_XCPT_AC
+                     && uCurVector  == X86_XCPT_AC)
+            {
+                enmRaise   = IEMXCPTRAISE_CPU_HANG;
+                fRaiseInfo = IEMXCPTRAISEINFO_AC_AC;
+                Log2(("IEMEvaluateRecursiveXcpt: Recursive #AC - Bad guest\n"));
+            }
+        }
+    }
+    else if (fPrevFlags & IEM_XCPT_FLAGS_T_EXT_INT)
+    {
+        fRaiseInfo = IEMXCPTRAISEINFO_EXT_INT_XCPT;
+        if (uCurVector == X86_XCPT_PF)
+            fRaiseInfo |= IEMXCPTRAISEINFO_EXT_INT_PF;
+    }
+    else
+    {
+        Assert(fPrevFlags & IEM_XCPT_FLAGS_T_SOFT_INT);
+        fRaiseInfo = IEMXCPTRAISEINFO_SOFT_INT_XCPT;
+    }
+
+    if (pfXcptRaiseInfo)
+        *pfXcptRaiseInfo = fRaiseInfo;
+    return enmRaise;
+}
+
+
+/**
+ * Enters the CPU shutdown state initiated by a triple fault or other
+ * unrecoverable conditions.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling thread.
+ */
+IEM_STATIC VBOXSTRICTRC iemInitiateCpuShutdown(PVMCPUCC pVCpu)
+{
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN))
+    {
+        Log2(("shutdown: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SHUTDOWN, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    RT_NOREF(pVCpu);
+    return VINF_EM_TRIPLE_FAULT;
+}
+
+
+/**
+ * Validates a new SS segment.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling thread.
+ * @param   NewSS           The new SS selctor.
+ * @param   uCpl            The CPL to load the stack for.
+ * @param   pDesc           Where to return the descriptor.
+ */
+IEM_STATIC VBOXSTRICTRC iemMiscValidateNewSS(PVMCPUCC pVCpu, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc)
+{
+    /* Null selectors are not allowed (we're not called for dispatching
+       interrupts with SS=0 in long mode). */
+    if (!(NewSS & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("iemMiscValidateNewSSandRsp: %#x - null selector -> #TS(0)\n", NewSS));
+        return iemRaiseTaskSwitchFault0(pVCpu);
+    }
+
+    /** @todo testcase: check that the TSS.ssX RPL is checked.  Also check when. */
+    if ((NewSS & X86_SEL_RPL) != uCpl)
+    {
+        Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #TS\n", NewSS, uCpl));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
+    }
+
+    /*
+     * Read the descriptor.
+     */
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, pDesc, NewSS, X86_XCPT_TS);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Perform the descriptor validation documented for LSS, POP SS and MOV SS.
+     */
+    if (!pDesc->Legacy.Gen.u1DescType)
+    {
+        Log(("iemMiscValidateNewSSandRsp: %#x - system selector (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
+    }
+
+    if (    (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
+        || !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
+    {
+        Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
+    }
+    if (pDesc->Legacy.Gen.u2Dpl != uCpl)
+    {
+        Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #TS\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
+    }
+
+    /* Is it there? */
+    /** @todo testcase: Is this checked before the canonical / limit check below? */
+    if (!pDesc->Legacy.Gen.u1Present)
+    {
+        Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, NewSS);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the correct EFLAGS regardless of whether PATM stores parts of them or
+ * not (kind of obsolete now).
+ *
+ * @param   a_pVCpu The cross context virtual CPU structure of the calling thread.
+ */
+#define IEMMISC_GET_EFL(a_pVCpu)            ( (a_pVCpu)->cpum.GstCtx.eflags.u  )
+
+/**
+ * Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
+ *
+ * @param   a_pVCpu The cross context virtual CPU structure of the calling thread.
+ * @param   a_fEfl  The new EFLAGS.
+ */
+#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl)    do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
+
+/** @} */
+
+
+/** @name  Raising Exceptions.
+ *
+ * @{
+ */
+
+
+/**
+ * Loads the specified stack far pointer from the TSS.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   uCpl            The CPL to load the stack for.
+ * @param   pSelSS          Where to return the new stack segment.
+ * @param   puEsp           Where to return the new stack pointer.
+ */
+IEM_STATIC VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PVMCPUCC pVCpu, uint8_t uCpl, PRTSEL pSelSS, uint32_t *puEsp)
+{
+    VBOXSTRICTRC rcStrict;
+    Assert(uCpl < 4);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
+    switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
+    {
+        /*
+         * 16-bit TSS (X86TSS16).
+         */
+        case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
+        case X86_SEL_TYPE_SYS_286_TSS_BUSY:
+        {
+            uint32_t off = uCpl * 4 + 2;
+            if (off + 4 <= pVCpu->cpum.GstCtx.tr.u32Limit)
+            {
+                /** @todo check actual access pattern here. */
+                uint32_t u32Tmp = 0; /* gcc maybe... */
+                rcStrict = iemMemFetchSysU32(pVCpu, &u32Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
+                if (rcStrict == VINF_SUCCESS)
+                {
+                    *puEsp  = RT_LOWORD(u32Tmp);
+                    *pSelSS = RT_HIWORD(u32Tmp);
+                    return VINF_SUCCESS;
+                }
+            }
+            else
+            {
+                Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
+                rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
+            }
+            break;
+        }
+
+        /*
+         * 32-bit TSS (X86TSS32).
+         */
+        case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
+        case X86_SEL_TYPE_SYS_386_TSS_BUSY:
+        {
+            uint32_t off = uCpl * 8 + 4;
+            if (off + 7 <= pVCpu->cpum.GstCtx.tr.u32Limit)
+            {
+/** @todo check actual access pattern here. */
+                uint64_t u64Tmp;
+                rcStrict = iemMemFetchSysU64(pVCpu, &u64Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
+                if (rcStrict == VINF_SUCCESS)
+                {
+                    *puEsp  = u64Tmp & UINT32_MAX;
+                    *pSelSS = (RTSEL)(u64Tmp >> 32);
+                    return VINF_SUCCESS;
+                }
+            }
+            else
+            {
+                Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
+                rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
+            }
+            break;
+        }
+
+        default:
+            AssertFailed();
+            rcStrict = VERR_IEM_IPE_4;
+            break;
+    }
+
+    *puEsp  = 0; /* make gcc happy */
+    *pSelSS = 0; /* make gcc happy */
+    return rcStrict;
+}
+
+
+/**
+ * Loads the specified stack pointer from the 64-bit TSS.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   uCpl            The CPL to load the stack for.
+ * @param   uIst            The interrupt stack table index, 0 if to use uCpl.
+ * @param   puRsp           Where to return the new stack pointer.
+ */
+IEM_STATIC VBOXSTRICTRC iemRaiseLoadStackFromTss64(PVMCPUCC pVCpu, uint8_t uCpl, uint8_t uIst, uint64_t *puRsp)
+{
+    Assert(uCpl < 4);
+    Assert(uIst < 8);
+    *puRsp  = 0; /* make gcc happy */
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
+    AssertReturn(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY, VERR_IEM_IPE_5);
+
+    uint32_t off;
+    if (uIst)
+        off = (uIst - 1) * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, ist1);
+    else
+        off = uCpl * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, rsp0);
+    if (off + sizeof(uint64_t) > pVCpu->cpum.GstCtx.tr.u32Limit)
+    {
+        Log(("iemRaiseLoadStackFromTss64: out of bounds! uCpl=%d uIst=%d, u32Limit=%#x\n", uCpl, uIst, pVCpu->cpum.GstCtx.tr.u32Limit));
+        return iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
+    }
+
+    return iemMemFetchSysU64(pVCpu, puRsp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
+}
+
+
+/**
+ * Adjust the CPU state according to the exception being raised.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   u8Vector        The exception that has been raised.
+ */
+DECLINLINE(void) iemRaiseXcptAdjustState(PVMCPUCC pVCpu, uint8_t u8Vector)
+{
+    switch (u8Vector)
+    {
+        case X86_XCPT_DB:
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
+            pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
+            break;
+        /** @todo Read the AMD and Intel exception reference... */
+    }
+}
+
+
+/**
+ * Implements exceptions and interrupts for real mode.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr         The number of bytes to offset rIP by in the return
+ *                          address.
+ * @param   u8Vector        The interrupt / exception vector number.
+ * @param   fFlags          The flags.
+ * @param   uErr            The error value if IEM_XCPT_FLAGS_ERR is set.
+ * @param   uCr2            The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
+ */
+IEM_STATIC VBOXSTRICTRC
+iemRaiseXcptOrIntInRealMode(PVMCPUCC      pVCpu,
+                            uint8_t     cbInstr,
+                            uint8_t     u8Vector,
+                            uint32_t    fFlags,
+                            uint16_t    uErr,
+                            uint64_t    uCr2)
+{
+    NOREF(uErr); NOREF(uCr2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    /*
+     * Read the IDT entry.
+     */
+    if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
+    {
+        Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
+        return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+    RTFAR16 Idte;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, (uint32_t *)&Idte, UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(4) * u8Vector);
+    if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
+    {
+        Log(("iemRaiseXcptOrIntInRealMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /*
+     * Push the stack frame.
+     */
+    uint16_t *pu16Frame;
+    uint64_t  uNewRsp;
+    rcStrict = iemMemStackPushBeginSpecial(pVCpu, 6, (void **)&pu16Frame, &uNewRsp);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
+#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
+    AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
+    if (pVCpu->iem.s.uTargetCpu <= IEMTARGETCPU_186)
+        fEfl |= UINT16_C(0xf000);
+#endif
+    pu16Frame[2] = (uint16_t)fEfl;
+    pu16Frame[1] = (uint16_t)pVCpu->cpum.GstCtx.cs.Sel;
+    pu16Frame[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
+    rcStrict = iemMemStackPushCommitSpecial(pVCpu, pu16Frame, uNewRsp);
+    if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
+        return rcStrict;
+
+    /*
+     * Load the vector address into cs:ip and make exception specific state
+     * adjustments.
+     */
+    pVCpu->cpum.GstCtx.cs.Sel           = Idte.sel;
+    pVCpu->cpum.GstCtx.cs.ValidSel      = Idte.sel;
+    pVCpu->cpum.GstCtx.cs.fFlags        = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.u64Base       = (uint32_t)Idte.sel << 4;
+    /** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
+    pVCpu->cpum.GstCtx.rip              = Idte.off;
+    fEfl &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_AC);
+    IEMMISC_SET_EFL(pVCpu, fEfl);
+
+    /** @todo do we actually do this in real mode? */
+    if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+        iemRaiseXcptAdjustState(pVCpu, u8Vector);
+
+    return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
+}
+
+
+/**
+ * Loads a NULL data selector into when coming from V8086 mode.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   pSReg           Pointer to the segment register.
+ */
+IEM_STATIC void iemHlpLoadNullDataSelectorOnV86Xcpt(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
+{
+    pSReg->Sel      = 0;
+    pSReg->ValidSel = 0;
+    if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
+    {
+        /* VT-x (Intel 3960x) doesn't change the base and limit, clears and sets the following attributes */
+        pSReg->Attr.u &= X86DESCATTR_DT | X86DESCATTR_TYPE | X86DESCATTR_DPL | X86DESCATTR_G | X86DESCATTR_D;
+        pSReg->Attr.u |= X86DESCATTR_UNUSABLE;
+    }
+    else
+    {
+        pSReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+        /** @todo check this on AMD-V */
+        pSReg->u64Base  = 0;
+        pSReg->u32Limit = 0;
+    }
+}
+
+
+/**
+ * Loads a segment selector during a task switch in V8086 mode.
+ *
+ * @param   pSReg           Pointer to the segment register.
+ * @param   uSel            The selector value to load.
+ */
+IEM_STATIC void iemHlpLoadSelectorInV86Mode(PCPUMSELREG pSReg, uint16_t uSel)
+{
+    /* See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
+    pSReg->Sel      = uSel;
+    pSReg->ValidSel = uSel;
+    pSReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+    pSReg->u64Base  = uSel << 4;
+    pSReg->u32Limit = 0xffff;
+    pSReg->Attr.u   = 0xf3;
+}
+
+
+/**
+ * Loads a NULL data selector into a selector register, both the hidden and
+ * visible parts, in protected mode.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pSReg               Pointer to the segment register.
+ * @param   uRpl                The RPL.
+ */
+IEM_STATIC void iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl)
+{
+    /** @todo Testcase: write a testcase checking what happends when loading a NULL
+     *        data selector in protected mode. */
+    pSReg->Sel      = uRpl;
+    pSReg->ValidSel = uRpl;
+    pSReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+    if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
+    {
+        /* VT-x (Intel 3960x) observed doing something like this. */
+        pSReg->Attr.u   = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT);
+        pSReg->u32Limit = UINT32_MAX;
+        pSReg->u64Base  = 0;
+    }
+    else
+    {
+        pSReg->Attr.u   = X86DESCATTR_UNUSABLE;
+        pSReg->u32Limit = 0;
+        pSReg->u64Base  = 0;
+    }
+}
+
+
+/**
+ * Loads a segment selector during a task switch in protected mode.
+ *
+ * In this task switch scenario, we would throw \#TS exceptions rather than
+ * \#GPs.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   pSReg           Pointer to the segment register.
+ * @param   uSel            The new selector value.
+ *
+ * @remarks This does _not_ handle CS or SS.
+ * @remarks This expects pVCpu->iem.s.uCpl to be up to date.
+ */
+IEM_STATIC VBOXSTRICTRC iemHlpTaskSwitchLoadDataSelectorInProtMode(PVMCPUCC pVCpu, PCPUMSELREG pSReg, uint16_t uSel)
+{
+    Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
+
+    /* Null data selector. */
+    if (!(uSel & X86_SEL_MASK_OFF_RPL))
+    {
+        iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, uSel);
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
+        CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
+        return VINF_SUCCESS;
+    }
+
+    /* Fetch the descriptor. */
+    IEMSELDESC Desc;
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_TS);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: failed to fetch selector. uSel=%u rc=%Rrc\n", uSel,
+             VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /* Must be a data segment or readable code segment. */
+    if (   !Desc.Legacy.Gen.u1DescType
+        || (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
+    {
+        Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: invalid segment type. uSel=%u Desc.u4Type=%#x\n", uSel,
+             Desc.Legacy.Gen.u4Type));
+        return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /* Check privileges for data segments and non-conforming code segments. */
+    if (   (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+        != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+    {
+        /* The RPL and the new CPL must be less than or equal to the DPL. */
+        if (   (unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
+            || (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl))
+        {
+            Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Invalid priv. uSel=%u uSel.RPL=%u DPL=%u CPL=%u\n",
+                 uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+        }
+    }
+
+    /* Is it there? */
+    if (!Desc.Legacy.Gen.u1Present)
+    {
+        Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Segment not present. uSel=%u\n", uSel));
+        return iemRaiseSelectorNotPresentWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /* The base and limit. */
+    uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
+    uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
+
+    /*
+     * Ok, everything checked out fine. Now set the accessed bit before
+     * committing the result into the registers.
+     */
+    if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+
+    /* Commit */
+    pSReg->Sel      = uSel;
+    pSReg->Attr.u   = X86DESC_GET_HID_ATTR(&Desc.Legacy);
+    pSReg->u32Limit = cbLimit;
+    pSReg->u64Base  = u64Base;  /** @todo testcase/investigate: seen claims that the upper half of the base remains unchanged... */
+    pSReg->ValidSel = uSel;
+    pSReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+    if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
+        pSReg->Attr.u &= ~X86DESCATTR_UNUSABLE;
+
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Performs a task switch.
+ *
+ * If the task switch is the result of a JMP, CALL or IRET instruction, the
+ * caller is responsible for performing the necessary checks (like DPL, TSS
+ * present etc.) which are specific to JMP/CALL/IRET. See Intel Instruction
+ * reference for JMP, CALL, IRET.
+ *
+ * If the task switch is the due to a software interrupt or hardware exception,
+ * the caller is responsible for validating the TSS selector and descriptor. See
+ * Intel Instruction reference for INT n.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   enmTaskSwitch   The cause of the task switch.
+ * @param   uNextEip        The EIP effective after the task switch.
+ * @param   fFlags          The flags, see IEM_XCPT_FLAGS_XXX.
+ * @param   uErr            The error value if IEM_XCPT_FLAGS_ERR is set.
+ * @param   uCr2            The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
+ * @param   SelTSS          The TSS selector of the new task.
+ * @param   pNewDescTSS     Pointer to the new TSS descriptor.
+ */
+IEM_STATIC VBOXSTRICTRC
+iemTaskSwitch(PVMCPUCC        pVCpu,
+              IEMTASKSWITCH   enmTaskSwitch,
+              uint32_t        uNextEip,
+              uint32_t        fFlags,
+              uint16_t        uErr,
+              uint64_t        uCr2,
+              RTSEL           SelTSS,
+              PIEMSELDESC     pNewDescTSS)
+{
+    Assert(!IEM_IS_REAL_MODE(pVCpu));
+    Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    uint32_t const uNewTSSType = pNewDescTSS->Legacy.Gate.u4Type;
+    Assert(   uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_AVAIL
+           || uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
+           || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
+           || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
+
+    bool const fIsNewTSS386 = (   uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
+                               || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
+
+    Log(("iemTaskSwitch: enmTaskSwitch=%u NewTSS=%#x fIsNewTSS386=%RTbool EIP=%#RX32 uNextEip=%#RX32\n", enmTaskSwitch, SelTSS,
+         fIsNewTSS386, pVCpu->cpum.GstCtx.eip, uNextEip));
+
+    /* Update CR2 in case it's a page-fault. */
+    /** @todo This should probably be done much earlier in IEM/PGM. See
+     *        @bugref{5653#c49}. */
+    if (fFlags & IEM_XCPT_FLAGS_CR2)
+        pVCpu->cpum.GstCtx.cr2 = uCr2;
+
+    /*
+     * Check the new TSS limit. See Intel spec. 6.15 "Exception and Interrupt Reference"
+     * subsection "Interrupt 10 - Invalid TSS Exception (#TS)".
+     */
+    uint32_t const uNewTSSLimit    = pNewDescTSS->Legacy.Gen.u16LimitLow | (pNewDescTSS->Legacy.Gen.u4LimitHigh << 16);
+    uint32_t const uNewTSSLimitMin = fIsNewTSS386 ? X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN : X86_SEL_TYPE_SYS_286_TSS_LIMIT_MIN;
+    if (uNewTSSLimit < uNewTSSLimitMin)
+    {
+        Log(("iemTaskSwitch: Invalid new TSS limit. enmTaskSwitch=%u uNewTSSLimit=%#x uNewTSSLimitMin=%#x -> #TS\n",
+             enmTaskSwitch, uNewTSSLimit, uNewTSSLimitMin));
+        return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /*
+     * Task switches in VMX non-root mode always cause task switches.
+     * The new TSS must have been read and validated (DPL, limits etc.) before a
+     * task-switch VM-exit commences.
+     *
+     * See Intel spec. 25.4.2 "Treatment of Task Switches".
+     */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        Log(("iemTaskSwitch: Guest intercept (source=%u, sel=%#x) -> VM-exit.\n", enmTaskSwitch, SelTSS));
+        IEM_VMX_VMEXIT_TASK_SWITCH_RET(pVCpu, enmTaskSwitch, SelTSS, uNextEip - pVCpu->cpum.GstCtx.eip);
+    }
+
+    /*
+     * The SVM nested-guest intercept for task-switch takes priority over all exceptions
+     * after validating the incoming (new) TSS, see AMD spec. 15.14.1 "Task Switch Intercept".
+     */
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH))
+    {
+        uint32_t const uExitInfo1 = SelTSS;
+        uint32_t       uExitInfo2 = uErr;
+        switch (enmTaskSwitch)
+        {
+            case IEMTASKSWITCH_JUMP: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_JUMP; break;
+            case IEMTASKSWITCH_IRET: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_IRET; break;
+            default: break;
+        }
+        if (fFlags & IEM_XCPT_FLAGS_ERR)
+            uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_HAS_ERROR_CODE;
+        if (pVCpu->cpum.GstCtx.eflags.Bits.u1RF)
+            uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_EFLAGS_RF;
+
+        Log(("iemTaskSwitch: Guest intercept -> #VMEXIT. uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitInfo1, uExitInfo2));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TASK_SWITCH, uExitInfo1, uExitInfo2);
+        RT_NOREF2(uExitInfo1, uExitInfo2);
+    }
+
+    /*
+     * Check the current TSS limit. The last written byte to the current TSS during the
+     * task switch will be 2 bytes at offset 0x5C (32-bit) and 1 byte at offset 0x28 (16-bit).
+     * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
+     *
+     * The AMD docs doesn't mention anything about limit checks with LTR which suggests you can
+     * end up with smaller than "legal" TSS limits.
+     */
+    uint32_t const uCurTSSLimit    = pVCpu->cpum.GstCtx.tr.u32Limit;
+    uint32_t const uCurTSSLimitMin = fIsNewTSS386 ? 0x5F : 0x29;
+    if (uCurTSSLimit < uCurTSSLimitMin)
+    {
+        Log(("iemTaskSwitch: Invalid current TSS limit. enmTaskSwitch=%u uCurTSSLimit=%#x uCurTSSLimitMin=%#x -> #TS\n",
+             enmTaskSwitch, uCurTSSLimit, uCurTSSLimitMin));
+        return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /*
+     * Verify that the new TSS can be accessed and map it. Map only the required contents
+     * and not the entire TSS.
+     */
+    void           *pvNewTSS;
+    uint32_t  const cbNewTSS    = uNewTSSLimitMin + 1;
+    RTGCPTR   const GCPtrNewTSS = X86DESC_BASE(&pNewDescTSS->Legacy);
+    AssertCompile(sizeof(X86TSS32) == X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN + 1);
+    /** @todo Handle if the TSS crosses a page boundary. Intel specifies that it may
+     *        not perform correct translation if this happens. See Intel spec. 7.2.1
+     *        "Task-State Segment". */
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvNewTSS, cbNewTSS, UINT8_MAX, GCPtrNewTSS, IEM_ACCESS_SYS_RW);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iemTaskSwitch: Failed to read new TSS. enmTaskSwitch=%u cbNewTSS=%u uNewTSSLimit=%u rc=%Rrc\n", enmTaskSwitch,
+             cbNewTSS, uNewTSSLimit, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /*
+     * Clear the busy bit in current task's TSS descriptor if it's a task switch due to JMP/IRET.
+     */
+    uint32_t u32EFlags = pVCpu->cpum.GstCtx.eflags.u32;
+    if (   enmTaskSwitch == IEMTASKSWITCH_JUMP
+        || enmTaskSwitch == IEMTASKSWITCH_IRET)
+    {
+        PX86DESC pDescCurTSS;
+        rcStrict = iemMemMap(pVCpu, (void **)&pDescCurTSS, sizeof(*pDescCurTSS), UINT8_MAX,
+                             pVCpu->cpum.GstCtx.gdtr.pGdt + (pVCpu->cpum.GstCtx.tr.Sel & X86_SEL_MASK), IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
+                 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        pDescCurTSS->Gate.u4Type &= ~X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pDescCurTSS, IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
+                 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* Clear EFLAGS.NT (Nested Task) in the eflags memory image, if it's a task switch due to an IRET. */
+        if (enmTaskSwitch == IEMTASKSWITCH_IRET)
+        {
+            Assert(   uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
+                   || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
+            u32EFlags &= ~X86_EFL_NT;
+        }
+    }
+
+    /*
+     * Save the CPU state into the current TSS.
+     */
+    RTGCPTR const GCPtrCurTSS = pVCpu->cpum.GstCtx.tr.u64Base;
+    if (GCPtrNewTSS == GCPtrCurTSS)
+    {
+        Log(("iemTaskSwitch: Switching to the same TSS! enmTaskSwitch=%u GCPtr[Cur|New]TSS=%#RGv\n", enmTaskSwitch, GCPtrCurTSS));
+        Log(("uCurCr3=%#x uCurEip=%#x uCurEflags=%#x uCurEax=%#x uCurEsp=%#x uCurEbp=%#x uCurCS=%#04x uCurSS=%#04x uCurLdt=%#x\n",
+             pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.eax,
+             pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel,
+             pVCpu->cpum.GstCtx.ldtr.Sel));
+    }
+    if (fIsNewTSS386)
+    {
+        /*
+         * Verify that the current TSS (32-bit) can be accessed, only the minimum required size.
+         * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
+         */
+        void          *pvCurTSS32;
+        uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS32, eip);
+        uint32_t const cbCurTSS  = RT_UOFFSETOF(X86TSS32, selLdt) - RT_UOFFSETOF(X86TSS32, eip);
+        AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, selLdt) - RTASSERT_OFFSET_OF(X86TSS32, eip) == 64);
+        rcStrict = iemMemMap(pVCpu, &pvCurTSS32, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to read current 32-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
+                 enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
+        PX86TSS32 pCurTSS32 = (PX86TSS32)((uintptr_t)pvCurTSS32 - offCurTSS);
+        pCurTSS32->eip    = uNextEip;
+        pCurTSS32->eflags = u32EFlags;
+        pCurTSS32->eax    = pVCpu->cpum.GstCtx.eax;
+        pCurTSS32->ecx    = pVCpu->cpum.GstCtx.ecx;
+        pCurTSS32->edx    = pVCpu->cpum.GstCtx.edx;
+        pCurTSS32->ebx    = pVCpu->cpum.GstCtx.ebx;
+        pCurTSS32->esp    = pVCpu->cpum.GstCtx.esp;
+        pCurTSS32->ebp    = pVCpu->cpum.GstCtx.ebp;
+        pCurTSS32->esi    = pVCpu->cpum.GstCtx.esi;
+        pCurTSS32->edi    = pVCpu->cpum.GstCtx.edi;
+        pCurTSS32->es     = pVCpu->cpum.GstCtx.es.Sel;
+        pCurTSS32->cs     = pVCpu->cpum.GstCtx.cs.Sel;
+        pCurTSS32->ss     = pVCpu->cpum.GstCtx.ss.Sel;
+        pCurTSS32->ds     = pVCpu->cpum.GstCtx.ds.Sel;
+        pCurTSS32->fs     = pVCpu->cpum.GstCtx.fs.Sel;
+        pCurTSS32->gs     = pVCpu->cpum.GstCtx.gs.Sel;
+
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS32, IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to commit current 32-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+    else
+    {
+        /*
+         * Verify that the current TSS (16-bit) can be accessed. Again, only the minimum required size.
+         */
+        void          *pvCurTSS16;
+        uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS16, ip);
+        uint32_t const cbCurTSS  = RT_UOFFSETOF(X86TSS16, selLdt) - RT_UOFFSETOF(X86TSS16, ip);
+        AssertCompile(RTASSERT_OFFSET_OF(X86TSS16, selLdt) - RTASSERT_OFFSET_OF(X86TSS16, ip) == 28);
+        rcStrict = iemMemMap(pVCpu, &pvCurTSS16, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to read current 16-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
+                 enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
+        PX86TSS16 pCurTSS16 = (PX86TSS16)((uintptr_t)pvCurTSS16 - offCurTSS);
+        pCurTSS16->ip    = uNextEip;
+        pCurTSS16->flags = u32EFlags;
+        pCurTSS16->ax    = pVCpu->cpum.GstCtx.ax;
+        pCurTSS16->cx    = pVCpu->cpum.GstCtx.cx;
+        pCurTSS16->dx    = pVCpu->cpum.GstCtx.dx;
+        pCurTSS16->bx    = pVCpu->cpum.GstCtx.bx;
+        pCurTSS16->sp    = pVCpu->cpum.GstCtx.sp;
+        pCurTSS16->bp    = pVCpu->cpum.GstCtx.bp;
+        pCurTSS16->si    = pVCpu->cpum.GstCtx.si;
+        pCurTSS16->di    = pVCpu->cpum.GstCtx.di;
+        pCurTSS16->es    = pVCpu->cpum.GstCtx.es.Sel;
+        pCurTSS16->cs    = pVCpu->cpum.GstCtx.cs.Sel;
+        pCurTSS16->ss    = pVCpu->cpum.GstCtx.ss.Sel;
+        pCurTSS16->ds    = pVCpu->cpum.GstCtx.ds.Sel;
+
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS16, IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to commit current 16-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+
+    /*
+     * Update the previous task link field for the new TSS, if the task switch is due to a CALL/INT_XCPT.
+     */
+    if (   enmTaskSwitch == IEMTASKSWITCH_CALL
+        || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
+    {
+        /* 16 or 32-bit TSS doesn't matter, we only access the first, common 16-bit field (selPrev) here. */
+        PX86TSS32 pNewTSS = (PX86TSS32)pvNewTSS;
+        pNewTSS->selPrev  = pVCpu->cpum.GstCtx.tr.Sel;
+    }
+
+    /*
+     * Read the state from the new TSS into temporaries. Setting it immediately as the new CPU state is tricky,
+     * it's done further below with error handling (e.g. CR3 changes will go through PGM).
+     */
+    uint32_t uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEcx, uNewEdx, uNewEbx, uNewEsp, uNewEbp, uNewEsi, uNewEdi;
+    uint16_t uNewES,  uNewCS, uNewSS, uNewDS, uNewFS, uNewGS, uNewLdt;
+    bool     fNewDebugTrap;
+    if (fIsNewTSS386)
+    {
+        PCX86TSS32 pNewTSS32 = (PCX86TSS32)pvNewTSS;
+        uNewCr3       = (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG) ? pNewTSS32->cr3 : 0;
+        uNewEip       = pNewTSS32->eip;
+        uNewEflags    = pNewTSS32->eflags;
+        uNewEax       = pNewTSS32->eax;
+        uNewEcx       = pNewTSS32->ecx;
+        uNewEdx       = pNewTSS32->edx;
+        uNewEbx       = pNewTSS32->ebx;
+        uNewEsp       = pNewTSS32->esp;
+        uNewEbp       = pNewTSS32->ebp;
+        uNewEsi       = pNewTSS32->esi;
+        uNewEdi       = pNewTSS32->edi;
+        uNewES        = pNewTSS32->es;
+        uNewCS        = pNewTSS32->cs;
+        uNewSS        = pNewTSS32->ss;
+        uNewDS        = pNewTSS32->ds;
+        uNewFS        = pNewTSS32->fs;
+        uNewGS        = pNewTSS32->gs;
+        uNewLdt       = pNewTSS32->selLdt;
+        fNewDebugTrap = RT_BOOL(pNewTSS32->fDebugTrap);
+    }
+    else
+    {
+        PCX86TSS16 pNewTSS16 = (PCX86TSS16)pvNewTSS;
+        uNewCr3       = 0;
+        uNewEip       = pNewTSS16->ip;
+        uNewEflags    = pNewTSS16->flags;
+        uNewEax       = UINT32_C(0xffff0000) | pNewTSS16->ax;
+        uNewEcx       = UINT32_C(0xffff0000) | pNewTSS16->cx;
+        uNewEdx       = UINT32_C(0xffff0000) | pNewTSS16->dx;
+        uNewEbx       = UINT32_C(0xffff0000) | pNewTSS16->bx;
+        uNewEsp       = UINT32_C(0xffff0000) | pNewTSS16->sp;
+        uNewEbp       = UINT32_C(0xffff0000) | pNewTSS16->bp;
+        uNewEsi       = UINT32_C(0xffff0000) | pNewTSS16->si;
+        uNewEdi       = UINT32_C(0xffff0000) | pNewTSS16->di;
+        uNewES        = pNewTSS16->es;
+        uNewCS        = pNewTSS16->cs;
+        uNewSS        = pNewTSS16->ss;
+        uNewDS        = pNewTSS16->ds;
+        uNewFS        = 0;
+        uNewGS        = 0;
+        uNewLdt       = pNewTSS16->selLdt;
+        fNewDebugTrap = false;
+    }
+
+    if (GCPtrNewTSS == GCPtrCurTSS)
+        Log(("uNewCr3=%#x uNewEip=%#x uNewEflags=%#x uNewEax=%#x uNewEsp=%#x uNewEbp=%#x uNewCS=%#04x uNewSS=%#04x uNewLdt=%#x\n",
+             uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEsp, uNewEbp, uNewCS, uNewSS, uNewLdt));
+
+    /*
+     * We're done accessing the new TSS.
+     */
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewTSS, IEM_ACCESS_SYS_RW);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iemTaskSwitch: Failed to commit new TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /*
+     * Set the busy bit in the new TSS descriptor, if the task switch is a JMP/CALL/INT_XCPT.
+     */
+    if (enmTaskSwitch != IEMTASKSWITCH_IRET)
+    {
+        rcStrict = iemMemMap(pVCpu, (void **)&pNewDescTSS, sizeof(*pNewDescTSS), UINT8_MAX,
+                             pVCpu->cpum.GstCtx.gdtr.pGdt + (SelTSS & X86_SEL_MASK), IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
+                 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* Check that the descriptor indicates the new TSS is available (not busy). */
+        AssertMsg(   pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
+                  || pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL,
+                     ("Invalid TSS descriptor type=%#x", pNewDescTSS->Legacy.Gate.u4Type));
+
+        pNewDescTSS->Legacy.Gate.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pNewDescTSS, IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
+                 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+
+    /*
+     * From this point on, we're technically in the new task. We will defer exceptions
+     * until the completion of the task switch but before executing any instructions in the new task.
+     */
+    pVCpu->cpum.GstCtx.tr.Sel      = SelTSS;
+    pVCpu->cpum.GstCtx.tr.ValidSel = SelTSS;
+    pVCpu->cpum.GstCtx.tr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.tr.Attr.u   = X86DESC_GET_HID_ATTR(&pNewDescTSS->Legacy);
+    pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&pNewDescTSS->Legacy);
+    pVCpu->cpum.GstCtx.tr.u64Base  = X86DESC_BASE(&pNewDescTSS->Legacy);
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_TR);
+
+    /* Set the busy bit in TR. */
+    pVCpu->cpum.GstCtx.tr.Attr.n.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
+
+    /* Set EFLAGS.NT (Nested Task) in the eflags loaded from the new TSS, if it's a task switch due to a CALL/INT_XCPT. */
+    if (   enmTaskSwitch == IEMTASKSWITCH_CALL
+        || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
+    {
+        uNewEflags |= X86_EFL_NT;
+    }
+
+    pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_LE_ALL;     /** @todo Should we clear DR7.LE bit too? */
+    pVCpu->cpum.GstCtx.cr0   |= X86_CR0_TS;
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR0);
+
+    pVCpu->cpum.GstCtx.eip    = uNewEip;
+    pVCpu->cpum.GstCtx.eax    = uNewEax;
+    pVCpu->cpum.GstCtx.ecx    = uNewEcx;
+    pVCpu->cpum.GstCtx.edx    = uNewEdx;
+    pVCpu->cpum.GstCtx.ebx    = uNewEbx;
+    pVCpu->cpum.GstCtx.esp    = uNewEsp;
+    pVCpu->cpum.GstCtx.ebp    = uNewEbp;
+    pVCpu->cpum.GstCtx.esi    = uNewEsi;
+    pVCpu->cpum.GstCtx.edi    = uNewEdi;
+
+    uNewEflags &= X86_EFL_LIVE_MASK;
+    uNewEflags |= X86_EFL_RA1_MASK;
+    IEMMISC_SET_EFL(pVCpu, uNewEflags);
+
+    /*
+     * Switch the selectors here and do the segment checks later. If we throw exceptions, the selectors
+     * will be valid in the exception handler. We cannot update the hidden parts until we've switched CR3
+     * due to the hidden part data originating from the guest LDT/GDT which is accessed through paging.
+     */
+    pVCpu->cpum.GstCtx.es.Sel       = uNewES;
+    pVCpu->cpum.GstCtx.es.Attr.u   &= ~X86DESCATTR_P;
+
+    pVCpu->cpum.GstCtx.cs.Sel       = uNewCS;
+    pVCpu->cpum.GstCtx.cs.Attr.u   &= ~X86DESCATTR_P;
+
+    pVCpu->cpum.GstCtx.ss.Sel       = uNewSS;
+    pVCpu->cpum.GstCtx.ss.Attr.u   &= ~X86DESCATTR_P;
+
+    pVCpu->cpum.GstCtx.ds.Sel       = uNewDS;
+    pVCpu->cpum.GstCtx.ds.Attr.u   &= ~X86DESCATTR_P;
+
+    pVCpu->cpum.GstCtx.fs.Sel       = uNewFS;
+    pVCpu->cpum.GstCtx.fs.Attr.u   &= ~X86DESCATTR_P;
+
+    pVCpu->cpum.GstCtx.gs.Sel       = uNewGS;
+    pVCpu->cpum.GstCtx.gs.Attr.u   &= ~X86DESCATTR_P;
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
+
+    pVCpu->cpum.GstCtx.ldtr.Sel     = uNewLdt;
+    pVCpu->cpum.GstCtx.ldtr.fFlags  = CPUMSELREG_FLAGS_STALE;
+    pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_P;
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_LDTR);
+
+    if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
+    {
+        pVCpu->cpum.GstCtx.es.Attr.u   |= X86DESCATTR_UNUSABLE;
+        pVCpu->cpum.GstCtx.cs.Attr.u   |= X86DESCATTR_UNUSABLE;
+        pVCpu->cpum.GstCtx.ss.Attr.u   |= X86DESCATTR_UNUSABLE;
+        pVCpu->cpum.GstCtx.ds.Attr.u   |= X86DESCATTR_UNUSABLE;
+        pVCpu->cpum.GstCtx.fs.Attr.u   |= X86DESCATTR_UNUSABLE;
+        pVCpu->cpum.GstCtx.gs.Attr.u   |= X86DESCATTR_UNUSABLE;
+        pVCpu->cpum.GstCtx.ldtr.Attr.u |= X86DESCATTR_UNUSABLE;
+    }
+
+    /*
+     * Switch CR3 for the new task.
+     */
+    if (   fIsNewTSS386
+        && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG))
+    {
+        /** @todo Should we update and flush TLBs only if CR3 value actually changes? */
+        int rc = CPUMSetGuestCR3(pVCpu, uNewCr3);
+        AssertRCSuccessReturn(rc, rc);
+
+        /* Inform PGM. */
+        rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
+        AssertRCReturn(rc, rc);
+        /* ignore informational status codes */
+
+        CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR3);
+    }
+
+    /*
+     * Switch LDTR for the new task.
+     */
+    if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
+        iemHlpLoadNullDataSelectorProt(pVCpu, &pVCpu->cpum.GstCtx.ldtr, uNewLdt);
+    else
+    {
+        Assert(!pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present);   /* Ensures that LDT.TI check passes in iemMemFetchSelDesc() below. */
+
+        IEMSELDESC DescNewLdt;
+        rcStrict = iemMemFetchSelDesc(pVCpu, &DescNewLdt, uNewLdt, X86_XCPT_TS);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: fetching LDT failed. enmTaskSwitch=%u uNewLdt=%u cbGdt=%u rc=%Rrc\n", enmTaskSwitch,
+                 uNewLdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+        if (   !DescNewLdt.Legacy.Gen.u1Present
+            ||  DescNewLdt.Legacy.Gen.u1DescType
+            ||  DescNewLdt.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
+        {
+            Log(("iemTaskSwitch: Invalid LDT. enmTaskSwitch=%u uNewLdt=%u DescNewLdt.Legacy.u=%#RX64 -> #TS\n", enmTaskSwitch,
+                 uNewLdt, DescNewLdt.Legacy.u));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
+        }
+
+        pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
+        pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ldtr.u64Base  = X86DESC_BASE(&DescNewLdt.Legacy);
+        pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&DescNewLdt.Legacy);
+        pVCpu->cpum.GstCtx.ldtr.Attr.u   = X86DESC_GET_HID_ATTR(&DescNewLdt.Legacy);
+        if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
+            pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_UNUSABLE;
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
+    }
+
+    IEMSELDESC DescSS;
+    if (IEM_IS_V86_MODE(pVCpu))
+    {
+        pVCpu->iem.s.uCpl = 3;
+        iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.es, uNewES);
+        iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.cs, uNewCS);
+        iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ss, uNewSS);
+        iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ds, uNewDS);
+        iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.fs, uNewFS);
+        iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.gs, uNewGS);
+
+        /* Quick fix: fake DescSS. */ /** @todo fix the code further down? */
+        DescSS.Legacy.u = 0;
+        DescSS.Legacy.Gen.u16LimitLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u32Limit;
+        DescSS.Legacy.Gen.u4LimitHigh = pVCpu->cpum.GstCtx.ss.u32Limit >> 16;
+        DescSS.Legacy.Gen.u16BaseLow  = (uint16_t)pVCpu->cpum.GstCtx.ss.u64Base;
+        DescSS.Legacy.Gen.u8BaseHigh1 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 16);
+        DescSS.Legacy.Gen.u8BaseHigh2 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 24);
+        DescSS.Legacy.Gen.u4Type      = X86_SEL_TYPE_RW_ACC;
+        DescSS.Legacy.Gen.u2Dpl       = 3;
+    }
+    else
+    {
+        uint8_t const uNewCpl = (uNewCS & X86_SEL_RPL);
+
+        /*
+         * Load the stack segment for the new task.
+         */
+        if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
+        {
+            Log(("iemTaskSwitch: Null stack segment. enmTaskSwitch=%u uNewSS=%#x -> #TS\n", enmTaskSwitch, uNewSS));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* Fetch the descriptor. */
+        rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_TS);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: failed to fetch SS. uNewSS=%#x rc=%Rrc\n", uNewSS,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* SS must be a data segment and writable. */
+        if (    !DescSS.Legacy.Gen.u1DescType
+            ||  (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
+            || !(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE))
+        {
+            Log(("iemTaskSwitch: SS invalid descriptor type. uNewSS=%#x u1DescType=%u u4Type=%#x\n",
+                 uNewSS, DescSS.Legacy.Gen.u1DescType, DescSS.Legacy.Gen.u4Type));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* The SS.RPL, SS.DPL, CS.RPL (CPL) must be equal. */
+        if (   (uNewSS & X86_SEL_RPL) != uNewCpl
+            || DescSS.Legacy.Gen.u2Dpl != uNewCpl)
+        {
+            Log(("iemTaskSwitch: Invalid priv. for SS. uNewSS=%#x SS.DPL=%u uNewCpl=%u -> #TS\n", uNewSS, DescSS.Legacy.Gen.u2Dpl,
+                 uNewCpl));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* Is it there? */
+        if (!DescSS.Legacy.Gen.u1Present)
+        {
+            Log(("iemTaskSwitch: SS not present. uNewSS=%#x -> #NP\n", uNewSS));
+            return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        uint32_t cbLimit = X86DESC_LIMIT_G(&DescSS.Legacy);
+        uint64_t u64Base = X86DESC_BASE(&DescSS.Legacy);
+
+        /* Set the accessed bit before committing the result into SS. */
+        if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /* Commit SS. */
+        pVCpu->cpum.GstCtx.ss.Sel      = uNewSS;
+        pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
+        pVCpu->cpum.GstCtx.ss.Attr.u   = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
+        pVCpu->cpum.GstCtx.ss.u32Limit = cbLimit;
+        pVCpu->cpum.GstCtx.ss.u64Base  = u64Base;
+        pVCpu->cpum.GstCtx.ss.fFlags   = CPUMSELREG_FLAGS_VALID;
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+
+        /* CPL has changed, update IEM before loading rest of segments. */
+        pVCpu->iem.s.uCpl = uNewCpl;
+
+        /*
+         * Load the data segments for the new task.
+         */
+        rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.es, uNewES);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.ds, uNewDS);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.fs, uNewFS);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.gs, uNewGS);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        /*
+         * Load the code segment for the new task.
+         */
+        if (!(uNewCS & X86_SEL_MASK_OFF_RPL))
+        {
+            Log(("iemTaskSwitch #TS: Null code segment. enmTaskSwitch=%u uNewCS=%#x\n", enmTaskSwitch, uNewCS));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* Fetch the descriptor. */
+        IEMSELDESC DescCS;
+        rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_TS);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: failed to fetch CS. uNewCS=%u rc=%Rrc\n", uNewCS, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* CS must be a code segment. */
+        if (   !DescCS.Legacy.Gen.u1DescType
+            || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
+        {
+            Log(("iemTaskSwitch: CS invalid descriptor type. uNewCS=%#x u1DescType=%u u4Type=%#x -> #TS\n", uNewCS,
+                 DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* For conforming CS, DPL must be less than or equal to the RPL. */
+        if (   (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
+            && DescCS.Legacy.Gen.u2Dpl > (uNewCS & X86_SEL_RPL))
+        {
+            Log(("iemTaskSwitch: confirming CS DPL > RPL. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, DescCS.Legacy.Gen.u4Type,
+                 DescCS.Legacy.Gen.u2Dpl));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* For non-conforming CS, DPL must match RPL. */
+        if (   !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
+            && DescCS.Legacy.Gen.u2Dpl != (uNewCS & X86_SEL_RPL))
+        {
+            Log(("iemTaskSwitch: non-confirming CS DPL RPL mismatch. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS,
+                 DescCS.Legacy.Gen.u4Type, DescCS.Legacy.Gen.u2Dpl));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* Is it there? */
+        if (!DescCS.Legacy.Gen.u1Present)
+        {
+            Log(("iemTaskSwitch: CS not present. uNewCS=%#x -> #NP\n", uNewCS));
+            return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
+        u64Base = X86DESC_BASE(&DescCS.Legacy);
+
+        /* Set the accessed bit before committing the result into CS. */
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /* Commit CS. */
+        pVCpu->cpum.GstCtx.cs.Sel      = uNewCS;
+        pVCpu->cpum.GstCtx.cs.ValidSel = uNewCS;
+        pVCpu->cpum.GstCtx.cs.Attr.u   = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+        pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
+        pVCpu->cpum.GstCtx.cs.u64Base  = u64Base;
+        pVCpu->cpum.GstCtx.cs.fFlags   = CPUMSELREG_FLAGS_VALID;
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+    }
+
+    /** @todo Debug trap. */
+    if (fIsNewTSS386 && fNewDebugTrap)
+        Log(("iemTaskSwitch: Debug Trap set in new TSS. Not implemented!\n"));
+
+    /*
+     * Construct the error code masks based on what caused this task switch.
+     * See Intel Instruction reference for INT.
+     */
+    uint16_t uExt;
+    if (   enmTaskSwitch == IEMTASKSWITCH_INT_XCPT
+        && (   !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
+            ||  (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)))
+    {
+        uExt = 1;
+    }
+    else
+        uExt = 0;
+
+    /*
+     * Push any error code on to the new stack.
+     */
+    if (fFlags & IEM_XCPT_FLAGS_ERR)
+    {
+        Assert(enmTaskSwitch == IEMTASKSWITCH_INT_XCPT);
+        uint32_t      cbLimitSS    = X86DESC_LIMIT_G(&DescSS.Legacy);
+        uint8_t const cbStackFrame = fIsNewTSS386 ? 4 : 2;
+
+        /* Check that there is sufficient space on the stack. */
+        /** @todo Factor out segment limit checking for normal/expand down segments
+         *        into a separate function. */
+        if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
+        {
+            if (   pVCpu->cpum.GstCtx.esp - 1 > cbLimitSS
+                || pVCpu->cpum.GstCtx.esp < cbStackFrame)
+            {
+                /** @todo Intel says \#SS(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
+                Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #SS\n",
+                     pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
+                return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
+            }
+        }
+        else
+        {
+            if (   pVCpu->cpum.GstCtx.esp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
+                || pVCpu->cpum.GstCtx.esp - cbStackFrame < cbLimitSS + UINT32_C(1))
+            {
+                Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #SS\n",
+                     pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
+                return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
+            }
+        }
+
+
+        if (fIsNewTSS386)
+            rcStrict = iemMemStackPushU32(pVCpu, uErr);
+        else
+            rcStrict = iemMemStackPushU16(pVCpu, uErr);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Can't push error code to new task's stack. %s-bit TSS. rc=%Rrc\n",
+                 fIsNewTSS386 ? "32" : "16", VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+
+    /* Check the new EIP against the new CS limit. */
+    if (pVCpu->cpum.GstCtx.eip > pVCpu->cpum.GstCtx.cs.u32Limit)
+    {
+        Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: New EIP exceeds CS limit. uNewEIP=%#RX32 CS limit=%u -> #GP(0)\n",
+             pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.cs.u32Limit));
+        /** @todo Intel says \#GP(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
+        return iemRaiseGeneralProtectionFault(pVCpu, uExt);
+    }
+
+    Log(("iemTaskSwitch: Success! New CS:EIP=%#04x:%#x SS=%#04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
+         pVCpu->cpum.GstCtx.ss.Sel));
+    return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
+}
+
+
+/**
+ * Implements exceptions and interrupts for protected mode.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr         The number of bytes to offset rIP by in the return
+ *                          address.
+ * @param   u8Vector        The interrupt / exception vector number.
+ * @param   fFlags          The flags.
+ * @param   uErr            The error value if IEM_XCPT_FLAGS_ERR is set.
+ * @param   uCr2            The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
+ */
+IEM_STATIC VBOXSTRICTRC
+iemRaiseXcptOrIntInProtMode(PVMCPUCC      pVCpu,
+                            uint8_t     cbInstr,
+                            uint8_t     u8Vector,
+                            uint32_t    fFlags,
+                            uint16_t    uErr,
+                            uint64_t    uCr2)
+{
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    /*
+     * Read the IDT entry.
+     */
+    if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
+    {
+        Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
+        return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+    X86DESC Idte;
+    VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.u, UINT8_MAX,
+                                              pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(8) * u8Vector);
+    if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
+    {
+        Log(("iemRaiseXcptOrIntInProtMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+    Log(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
+         u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
+         Idte.Gate.u5ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
+
+    /*
+     * Check the descriptor type, DPL and such.
+     * ASSUMES this is done in the same order as described for call-gate calls.
+     */
+    if (Idte.Gate.u1DescType)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+    bool     fTaskGate   = false;
+    uint8_t  f32BitGate  = true;
+    uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM;
+    switch (Idte.Gate.u4Type)
+    {
+        case X86_SEL_TYPE_SYS_UNDEFINED:
+        case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
+        case X86_SEL_TYPE_SYS_LDT:
+        case X86_SEL_TYPE_SYS_286_TSS_BUSY:
+        case X86_SEL_TYPE_SYS_286_CALL_GATE:
+        case X86_SEL_TYPE_SYS_UNDEFINED2:
+        case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
+        case X86_SEL_TYPE_SYS_UNDEFINED3:
+        case X86_SEL_TYPE_SYS_386_TSS_BUSY:
+        case X86_SEL_TYPE_SYS_386_CALL_GATE:
+        case X86_SEL_TYPE_SYS_UNDEFINED4:
+        {
+            /** @todo check what actually happens when the type is wrong...
+             *        esp. call gates. */
+            Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
+            return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+        }
+
+        case X86_SEL_TYPE_SYS_286_INT_GATE:
+            f32BitGate = false;
+            RT_FALL_THRU();
+        case X86_SEL_TYPE_SYS_386_INT_GATE:
+            fEflToClear |= X86_EFL_IF;
+            break;
+
+        case X86_SEL_TYPE_SYS_TASK_GATE:
+            fTaskGate = true;
+#ifndef IEM_IMPLEMENTS_TASKSWITCH
+            IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Task gates\n"));
+#endif
+            break;
+
+        case X86_SEL_TYPE_SYS_286_TRAP_GATE:
+            f32BitGate = false;
+        case X86_SEL_TYPE_SYS_386_TRAP_GATE:
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    /* Check DPL against CPL if applicable. */
+    if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
+    {
+        if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
+        {
+            Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
+            return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+        }
+    }
+
+    /* Is it there? */
+    if (!Idte.Gate.u1Present)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
+        return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+
+    /* Is it a task-gate? */
+    if (fTaskGate)
+    {
+        /*
+         * Construct the error code masks based on what caused this task switch.
+         * See Intel Instruction reference for INT.
+         */
+        uint16_t const uExt     = (    (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
+                                   && !(fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)) ? 0 : 1;
+        uint16_t const uSelMask = X86_SEL_MASK_OFF_RPL;
+        RTSEL          SelTSS   = Idte.Gate.u16Sel;
+
+        /*
+         * Fetch the TSS descriptor in the GDT.
+         */
+        IEMSELDESC DescTSS;
+        rcStrict = iemMemFetchSelDescWithErr(pVCpu, &DescTSS, SelTSS, X86_XCPT_GP, (SelTSS & uSelMask) | uExt);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("RaiseXcptOrIntInProtMode %#x - failed to fetch TSS selector %#x, rc=%Rrc\n", u8Vector, SelTSS,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* The TSS descriptor must be a system segment and be available (not busy). */
+        if (   DescTSS.Legacy.Gen.u1DescType
+            || (   DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
+                && DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL))
+        {
+            Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x of task gate not a system descriptor or not available %#RX64\n",
+                 u8Vector, SelTSS, DescTSS.Legacy.au64));
+            return iemRaiseGeneralProtectionFault(pVCpu, (SelTSS & uSelMask) | uExt);
+        }
+
+        /* The TSS must be present. */
+        if (!DescTSS.Legacy.Gen.u1Present)
+        {
+            Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x not present %#RX64\n", u8Vector, SelTSS, DescTSS.Legacy.au64));
+            return iemRaiseSelectorNotPresentWithErr(pVCpu, (SelTSS & uSelMask) | uExt);
+        }
+
+        /* Do the actual task switch. */
+        return iemTaskSwitch(pVCpu, IEMTASKSWITCH_INT_XCPT,
+                             (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip,
+                             fFlags, uErr, uCr2, SelTSS, &DescTSS);
+    }
+
+    /* A null CS is bad. */
+    RTSEL NewCS = Idte.Gate.u16Sel;
+    if (!(NewCS & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Fetch the descriptor for the new CS. */
+    IEMSELDESC DescCS;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP); /** @todo correct exception? */
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /* Must be a code segment. */
+    if (!DescCS.Legacy.Gen.u1DescType)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
+    }
+    if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /* Don't allow lowering the privilege level. */
+    /** @todo Does the lowering of privileges apply to software interrupts
+     *        only?  This has bearings on the more-privileged or
+     *        same-privilege stack behavior further down.  A testcase would
+     *        be nice. */
+    if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
+             u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /* Make sure the selector is present. */
+    if (!DescCS.Legacy.Gen.u1Present)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
+    }
+
+    /* Check the new EIP against the new CS limit. */
+    uint32_t const uNewEip =    Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
+                             || Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
+                           ? Idte.Gate.u16OffsetLow
+                           : Idte.Gate.u16OffsetLow | ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
+    uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
+    if (uNewEip > cbLimitCS)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - EIP=%#x > cbLimitCS=%#x (CS=%#x) -> #GP(0)\n",
+             u8Vector, uNewEip, cbLimitCS, NewCS));
+        return iemRaiseGeneralProtectionFault(pVCpu, 0);
+    }
+    Log7(("iemRaiseXcptOrIntInProtMode: new EIP=%#x CS=%#x\n", uNewEip, NewCS));
+
+    /* Calc the flag image to push. */
+    uint32_t        fEfl    = IEMMISC_GET_EFL(pVCpu);
+    if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT))
+        fEfl &= ~X86_EFL_RF;
+    else
+        fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */
+
+    /* From V8086 mode only go to CPL 0. */
+    uint8_t const   uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
+                            ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
+    if ((fEfl & X86_EFL_VM) && uNewCpl != 0) /** @todo When exactly is this raised? */
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - New CPL (%d) != 0 w/ VM=1 -> #GP\n", u8Vector, NewCS, uNewCpl));
+        return iemRaiseGeneralProtectionFault(pVCpu, 0);
+    }
+
+    /*
+     * If the privilege level changes, we need to get a new stack from the TSS.
+     * This in turns means validating the new SS and ESP...
+     */
+    if (uNewCpl != pVCpu->iem.s.uCpl)
+    {
+        RTSEL    NewSS;
+        uint32_t uNewEsp;
+        rcStrict = iemRaiseLoadStackFromTss32Or16(pVCpu, uNewCpl, &NewSS, &uNewEsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        IEMSELDESC DescSS;
+        rcStrict = iemMiscValidateNewSS(pVCpu, NewSS, uNewCpl, &DescSS);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        /* If the new SS is 16-bit, we are only going to use SP, not ESP. */
+        if (!DescSS.Legacy.Gen.u1DefBig)
+        {
+            Log(("iemRaiseXcptOrIntInProtMode: Forcing ESP=%#x to 16 bits\n", uNewEsp));
+            uNewEsp = (uint16_t)uNewEsp;
+        }
+
+        Log7(("iemRaiseXcptOrIntInProtMode: New SS=%#x ESP=%#x (from TSS); current SS=%#x ESP=%#x\n", NewSS, uNewEsp, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
+
+        /* Check that there is sufficient space for the stack frame. */
+        uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
+        uint8_t const cbStackFrame = !(fEfl & X86_EFL_VM)
+                                   ? (fFlags & IEM_XCPT_FLAGS_ERR ? 12 : 10) << f32BitGate
+                                   : (fFlags & IEM_XCPT_FLAGS_ERR ? 20 : 18) << f32BitGate;
+
+        if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
+        {
+            if (   uNewEsp - 1 > cbLimitSS
+                || uNewEsp < cbStackFrame)
+            {
+                Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
+                     u8Vector, NewSS, uNewEsp, cbStackFrame));
+                return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
+            }
+        }
+        else
+        {
+            if (   uNewEsp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT16_MAX)
+                || uNewEsp - cbStackFrame < cbLimitSS + UINT32_C(1))
+            {
+                Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #GP\n",
+                     u8Vector, NewSS, uNewEsp, cbStackFrame));
+                return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
+            }
+        }
+
+        /*
+         * Start making changes.
+         */
+
+        /* Set the new CPL so that stack accesses use it. */
+        uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
+        pVCpu->iem.s.uCpl = uNewCpl;
+
+        /* Create the stack frame. */
+        RTPTRUNION uStackFrame;
+        rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
+                             uNewEsp - cbStackFrame + X86DESC_BASE(&DescSS.Legacy), IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        void * const pvStackFrame = uStackFrame.pv;
+        if (f32BitGate)
+        {
+            if (fFlags & IEM_XCPT_FLAGS_ERR)
+                *uStackFrame.pu32++ = uErr;
+            uStackFrame.pu32[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
+            uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl;
+            uStackFrame.pu32[2] = fEfl;
+            uStackFrame.pu32[3] = pVCpu->cpum.GstCtx.esp;
+            uStackFrame.pu32[4] = pVCpu->cpum.GstCtx.ss.Sel;
+            Log7(("iemRaiseXcptOrIntInProtMode: 32-bit push SS=%#x ESP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
+            if (fEfl & X86_EFL_VM)
+            {
+                uStackFrame.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel;
+                uStackFrame.pu32[5] = pVCpu->cpum.GstCtx.es.Sel;
+                uStackFrame.pu32[6] = pVCpu->cpum.GstCtx.ds.Sel;
+                uStackFrame.pu32[7] = pVCpu->cpum.GstCtx.fs.Sel;
+                uStackFrame.pu32[8] = pVCpu->cpum.GstCtx.gs.Sel;
+            }
+        }
+        else
+        {
+            if (fFlags & IEM_XCPT_FLAGS_ERR)
+                *uStackFrame.pu16++ = uErr;
+            uStackFrame.pu16[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
+            uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl;
+            uStackFrame.pu16[2] = fEfl;
+            uStackFrame.pu16[3] = pVCpu->cpum.GstCtx.sp;
+            uStackFrame.pu16[4] = pVCpu->cpum.GstCtx.ss.Sel;
+            Log7(("iemRaiseXcptOrIntInProtMode: 16-bit push SS=%#x SP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.sp));
+            if (fEfl & X86_EFL_VM)
+            {
+                uStackFrame.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
+                uStackFrame.pu16[5] = pVCpu->cpum.GstCtx.es.Sel;
+                uStackFrame.pu16[6] = pVCpu->cpum.GstCtx.ds.Sel;
+                uStackFrame.pu16[7] = pVCpu->cpum.GstCtx.fs.Sel;
+                uStackFrame.pu16[8] = pVCpu->cpum.GstCtx.gs.Sel;
+            }
+        }
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        /* Mark the selectors 'accessed' (hope this is the correct time). */
+        /** @todo testcase: excatly _when_ are the accessed bits set - before or
+         *        after pushing the stack frame? (Write protect the gdt + stack to
+         *        find out.) */
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewSS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /*
+         * Start comitting the register changes (joins with the DPL=CPL branch).
+         */
+        pVCpu->cpum.GstCtx.ss.Sel            = NewSS;
+        pVCpu->cpum.GstCtx.ss.ValidSel       = NewSS;
+        pVCpu->cpum.GstCtx.ss.fFlags         = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ss.u32Limit       = cbLimitSS;
+        pVCpu->cpum.GstCtx.ss.u64Base        = X86DESC_BASE(&DescSS.Legacy);
+        pVCpu->cpum.GstCtx.ss.Attr.u         = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
+        /** @todo When coming from 32-bit code and operating with a 16-bit TSS and
+         *        16-bit handler, the high word of ESP remains unchanged (i.e. only
+         *        SP is loaded).
+         *  Need to check the other combinations too:
+         *      - 16-bit TSS, 32-bit handler
+         *      - 32-bit TSS, 16-bit handler */
+        if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+            pVCpu->cpum.GstCtx.sp            = (uint16_t)(uNewEsp - cbStackFrame);
+        else
+            pVCpu->cpum.GstCtx.rsp           = uNewEsp - cbStackFrame;
+
+        if (fEfl & X86_EFL_VM)
+        {
+            iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.gs);
+            iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.fs);
+            iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.es);
+            iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.ds);
+        }
+    }
+    /*
+     * Same privilege, no stack change and smaller stack frame.
+     */
+    else
+    {
+        uint64_t        uNewRsp;
+        RTPTRUNION      uStackFrame;
+        uint8_t const   cbStackFrame = (fFlags & IEM_XCPT_FLAGS_ERR ? 8 : 6) << f32BitGate;
+        rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbStackFrame, &uStackFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        void * const pvStackFrame = uStackFrame.pv;
+
+        if (f32BitGate)
+        {
+            if (fFlags & IEM_XCPT_FLAGS_ERR)
+                *uStackFrame.pu32++ = uErr;
+            uStackFrame.pu32[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
+            uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl;
+            uStackFrame.pu32[2] = fEfl;
+        }
+        else
+        {
+            if (fFlags & IEM_XCPT_FLAGS_ERR)
+                *uStackFrame.pu16++ = uErr;
+            uStackFrame.pu16[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
+            uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl;
+            uStackFrame.pu16[2] = fEfl;
+        }
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        /* Mark the CS selector as 'accessed'. */
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /*
+         * Start committing the register changes (joins with the other branch).
+         */
+        pVCpu->cpum.GstCtx.rsp = uNewRsp;
+    }
+
+    /* ... register committing continues. */
+    pVCpu->cpum.GstCtx.cs.Sel            = (NewCS & ~X86_SEL_RPL) | uNewCpl;
+    pVCpu->cpum.GstCtx.cs.ValidSel       = (NewCS & ~X86_SEL_RPL) | uNewCpl;
+    pVCpu->cpum.GstCtx.cs.fFlags         = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.u32Limit       = cbLimitCS;
+    pVCpu->cpum.GstCtx.cs.u64Base        = X86DESC_BASE(&DescCS.Legacy);
+    pVCpu->cpum.GstCtx.cs.Attr.u         = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+
+    pVCpu->cpum.GstCtx.rip               = uNewEip;  /* (The entire register is modified, see pe16_32 bs3kit tests.) */
+    fEfl &= ~fEflToClear;
+    IEMMISC_SET_EFL(pVCpu, fEfl);
+
+    if (fFlags & IEM_XCPT_FLAGS_CR2)
+        pVCpu->cpum.GstCtx.cr2 = uCr2;
+
+    if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+        iemRaiseXcptAdjustState(pVCpu, u8Vector);
+
+    return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
+}
+
+
+/**
+ * Implements exceptions and interrupts for long mode.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr         The number of bytes to offset rIP by in the return
+ *                          address.
+ * @param   u8Vector        The interrupt / exception vector number.
+ * @param   fFlags          The flags.
+ * @param   uErr            The error value if IEM_XCPT_FLAGS_ERR is set.
+ * @param   uCr2            The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
+ */
+IEM_STATIC VBOXSTRICTRC
+iemRaiseXcptOrIntInLongMode(PVMCPUCC      pVCpu,
+                            uint8_t     cbInstr,
+                            uint8_t     u8Vector,
+                            uint32_t    fFlags,
+                            uint16_t    uErr,
+                            uint64_t    uCr2)
+{
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    /*
+     * Read the IDT entry.
+     */
+    uint16_t offIdt = (uint16_t)u8Vector << 4;
+    if (pVCpu->cpum.GstCtx.idtr.cbIdt < offIdt + 7)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
+        return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+    X86DESC64 Idte;
+    VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[0], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[1], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt + 8);
+    if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
+    {
+        Log(("iemRaiseXcptOrIntInLongMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+    Log(("iemRaiseXcptOrIntInLongMode: vec=%#x P=%u DPL=%u DT=%u:%u IST=%u %04x:%08x%04x%04x\n",
+         u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
+         Idte.Gate.u3IST, Idte.Gate.u16Sel, Idte.Gate.u32OffsetTop, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
+
+    /*
+     * Check the descriptor type, DPL and such.
+     * ASSUMES this is done in the same order as described for call-gate calls.
+     */
+    if (Idte.Gate.u1DescType)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+    uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM;
+    switch (Idte.Gate.u4Type)
+    {
+        case AMD64_SEL_TYPE_SYS_INT_GATE:
+            fEflToClear |= X86_EFL_IF;
+            break;
+        case AMD64_SEL_TYPE_SYS_TRAP_GATE:
+            break;
+
+        default:
+            Log(("iemRaiseXcptOrIntInLongMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
+            return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+
+    /* Check DPL against CPL if applicable. */
+    if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
+    {
+        if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
+        {
+            Log(("iemRaiseXcptOrIntInLongMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
+            return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+        }
+    }
+
+    /* Is it there? */
+    if (!Idte.Gate.u1Present)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - not present -> #NP\n", u8Vector));
+        return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+
+    /* A null CS is bad. */
+    RTSEL NewCS = Idte.Gate.u16Sel;
+    if (!(NewCS & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Fetch the descriptor for the new CS. */
+    IEMSELDESC DescCS;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /* Must be a 64-bit code segment. */
+    if (!DescCS.Long.Gen.u1DescType)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
+    }
+    if (   !DescCS.Long.Gen.u1Long
+        || DescCS.Long.Gen.u1DefBig
+        || !(DescCS.Long.Gen.u4Type & X86_SEL_TYPE_CODE) )
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - not 64-bit code selector (%#x, L=%u, D=%u) -> #GP\n",
+             u8Vector, NewCS, DescCS.Legacy.Gen.u4Type, DescCS.Long.Gen.u1Long, DescCS.Long.Gen.u1DefBig));
+        return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /* Don't allow lowering the privilege level.  For non-conforming CS
+       selectors, the CS.DPL sets the privilege level the trap/interrupt
+       handler runs at.  For conforming CS selectors, the CPL remains
+       unchanged, but the CS.DPL must be <= CPL. */
+    /** @todo Testcase: Interrupt handler with CS.DPL=1, interrupt dispatched
+     *        when CPU in Ring-0. Result \#GP?  */
+    if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
+             u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
+    }
+
+
+    /* Make sure the selector is present. */
+    if (!DescCS.Legacy.Gen.u1Present)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
+    }
+
+    /* Check that the new RIP is canonical. */
+    uint64_t const uNewRip = Idte.Gate.u16OffsetLow
+                           | ((uint32_t)Idte.Gate.u16OffsetHigh << 16)
+                           | ((uint64_t)Idte.Gate.u32OffsetTop  << 32);
+    if (!IEM_IS_CANONICAL(uNewRip))
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - RIP=%#RX64 - Not canonical -> #GP(0)\n", u8Vector, uNewRip));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * If the privilege level changes or if the IST isn't zero, we need to get
+     * a new stack from the TSS.
+     */
+    uint64_t        uNewRsp;
+    uint8_t const   uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
+                            ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
+    if (   uNewCpl != pVCpu->iem.s.uCpl
+        || Idte.Gate.u3IST != 0)
+    {
+        rcStrict = iemRaiseLoadStackFromTss64(pVCpu, uNewCpl, Idte.Gate.u3IST, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+    else
+        uNewRsp = pVCpu->cpum.GstCtx.rsp;
+    uNewRsp &= ~(uint64_t)0xf;
+
+    /*
+     * Calc the flag image to push.
+     */
+    uint32_t        fEfl    = IEMMISC_GET_EFL(pVCpu);
+    if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT))
+        fEfl &= ~X86_EFL_RF;
+    else
+        fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */
+
+    /*
+     * Start making changes.
+     */
+    /* Set the new CPL so that stack accesses use it. */
+    uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
+    pVCpu->iem.s.uCpl = uNewCpl;
+
+    /* Create the stack frame. */
+    uint32_t   cbStackFrame = sizeof(uint64_t) * (5 + !!(fFlags & IEM_XCPT_FLAGS_ERR));
+    RTPTRUNION uStackFrame;
+    rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
+                         uNewRsp - cbStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    void * const pvStackFrame = uStackFrame.pv;
+
+    if (fFlags & IEM_XCPT_FLAGS_ERR)
+        *uStackFrame.pu64++ = uErr;
+    uStackFrame.pu64[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.rip + cbInstr : pVCpu->cpum.GstCtx.rip;
+    uStackFrame.pu64[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl; /* CPL paranoia */
+    uStackFrame.pu64[2] = fEfl;
+    uStackFrame.pu64[3] = pVCpu->cpum.GstCtx.rsp;
+    uStackFrame.pu64[4] = pVCpu->cpum.GstCtx.ss.Sel;
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Mark the CS selectors 'accessed' (hope this is the correct time). */
+    /** @todo testcase: excatly _when_ are the accessed bits set - before or
+     *        after pushing the stack frame? (Write protect the gdt + stack to
+     *        find out.) */
+    if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+
+    /*
+     * Start comitting the register changes.
+     */
+    /** @todo research/testcase: Figure out what VT-x and AMD-V loads into the
+     *        hidden registers when interrupting 32-bit or 16-bit code! */
+    if (uNewCpl != uOldCpl)
+    {
+        pVCpu->cpum.GstCtx.ss.Sel        = 0 | uNewCpl;
+        pVCpu->cpum.GstCtx.ss.ValidSel   = 0 | uNewCpl;
+        pVCpu->cpum.GstCtx.ss.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ss.u32Limit   = UINT32_MAX;
+        pVCpu->cpum.GstCtx.ss.u64Base    = 0;
+        pVCpu->cpum.GstCtx.ss.Attr.u     = (uNewCpl << X86DESCATTR_DPL_SHIFT) | X86DESCATTR_UNUSABLE;
+    }
+    pVCpu->cpum.GstCtx.rsp           = uNewRsp - cbStackFrame;
+    pVCpu->cpum.GstCtx.cs.Sel        = (NewCS & ~X86_SEL_RPL) | uNewCpl;
+    pVCpu->cpum.GstCtx.cs.ValidSel   = (NewCS & ~X86_SEL_RPL) | uNewCpl;
+    pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.u32Limit   = X86DESC_LIMIT_G(&DescCS.Legacy);
+    pVCpu->cpum.GstCtx.cs.u64Base    = X86DESC_BASE(&DescCS.Legacy);
+    pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+    pVCpu->cpum.GstCtx.rip           = uNewRip;
+
+    fEfl &= ~fEflToClear;
+    IEMMISC_SET_EFL(pVCpu, fEfl);
+
+    if (fFlags & IEM_XCPT_FLAGS_CR2)
+        pVCpu->cpum.GstCtx.cr2 = uCr2;
+
+    if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+        iemRaiseXcptAdjustState(pVCpu, u8Vector);
+
+    return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
+}
+
+
+/**
+ * Implements exceptions and interrupts.
+ *
+ * All exceptions and interrupts goes thru this function!
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr         The number of bytes to offset rIP by in the return
+ *                          address.
+ * @param   u8Vector        The interrupt / exception vector number.
+ * @param   fFlags          The flags.
+ * @param   uErr            The error value if IEM_XCPT_FLAGS_ERR is set.
+ * @param   uCr2            The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
+ */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC)
+iemRaiseXcptOrInt(PVMCPUCC    pVCpu,
+                  uint8_t     cbInstr,
+                  uint8_t     u8Vector,
+                  uint32_t    fFlags,
+                  uint16_t    uErr,
+                  uint64_t    uCr2)
+{
+    /*
+     * Get all the state that we might need here.
+     */
+    IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+#ifndef IEM_WITH_CODE_TLB /** @todo we're doing it afterwards too, that should suffice... */
+    /*
+     * Flush prefetch buffer
+     */
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+
+    /*
+     * Perform the V8086 IOPL check and upgrade the fault without nesting.
+     */
+    if (   pVCpu->cpum.GstCtx.eflags.Bits.u1VM
+        && pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL != 3
+        && (fFlags & (  IEM_XCPT_FLAGS_T_SOFT_INT
+                      | IEM_XCPT_FLAGS_BP_INSTR
+                      | IEM_XCPT_FLAGS_ICEBP_INSTR
+                      | IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
+        && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
+    {
+        Log(("iemRaiseXcptOrInt: V8086 IOPL check failed for int %#x -> #GP(0)\n", u8Vector));
+        fFlags   = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR;
+        u8Vector = X86_XCPT_GP;
+        uErr     = 0;
+    }
+#ifdef DBGFTRACE_ENABLED
+    RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "Xcpt/%u: %02x %u %x %x %llx %04x:%04llx %04x:%04llx",
+                      pVCpu->iem.s.cXcptRecursions, u8Vector, cbInstr, fFlags, uErr, uCr2,
+                      pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp);
+#endif
+
+    /*
+     * Evaluate whether NMI blocking should be in effect.
+     * Normally, NMI blocking is in effect whenever we inject an NMI.
+     */
+    bool fBlockNmi;
+    if (   u8Vector == X86_XCPT_NMI
+        && (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
+        fBlockNmi = true;
+    else
+        fBlockNmi = false;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict0 = iemVmxVmexitEvent(pVCpu, u8Vector, fFlags, uErr, uCr2, cbInstr);
+        if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict0;
+
+        /* If virtual-NMI blocking is in effect for the nested-guest, guest NMIs are not blocked. */
+        if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking)
+        {
+            Assert(CPUMIsGuestVmxPinCtlsSet(&pVCpu->cpum.GstCtx, VMX_PIN_CTLS_VIRT_NMI));
+            fBlockNmi = false;
+        }
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
+    {
+        /*
+         * If the event is being injected as part of VMRUN, it isn't subject to event
+         * intercepts in the nested-guest. However, secondary exceptions that occur
+         * during injection of any event -are- subject to exception intercepts.
+         *
+         * See AMD spec. 15.20 "Event Injection".
+         */
+        if (!pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents)
+            pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = true;
+        else
+        {
+            /*
+             * Check and handle if the event being raised is intercepted.
+             */
+            VBOXSTRICTRC rcStrict0 = iemHandleSvmEventIntercept(pVCpu, u8Vector, fFlags, uErr, uCr2);
+            if (rcStrict0 != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+                return rcStrict0;
+        }
+    }
+#endif
+
+    /*
+     * Set NMI blocking if necessary.
+     */
+    if (   fBlockNmi
+        && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
+
+    /*
+     * Do recursion accounting.
+     */
+    uint8_t const  uPrevXcpt = pVCpu->iem.s.uCurXcpt;
+    uint32_t const fPrevXcpt = pVCpu->iem.s.fCurXcpt;
+    if (pVCpu->iem.s.cXcptRecursions == 0)
+        Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
+             u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2));
+    else
+    {
+        Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
+             u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2, pVCpu->iem.s.uCurXcpt,
+             pVCpu->iem.s.cXcptRecursions + 1, fPrevXcpt));
+
+        if (pVCpu->iem.s.cXcptRecursions >= 4)
+        {
+#ifdef DEBUG_bird
+            AssertFailed();
+#endif
+            IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Too many fault nestings.\n"));
+        }
+
+        /*
+         * Evaluate the sequence of recurring events.
+         */
+        IEMXCPTRAISE enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fPrevXcpt, uPrevXcpt, fFlags, u8Vector,
+                                                         NULL /* pXcptRaiseInfo */);
+        if (enmRaise == IEMXCPTRAISE_CURRENT_XCPT)
+        { /* likely */ }
+        else if (enmRaise == IEMXCPTRAISE_DOUBLE_FAULT)
+        {
+            Log2(("iemRaiseXcptOrInt: Raising double fault. uPrevXcpt=%#x\n", uPrevXcpt));
+            fFlags   = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR;
+            u8Vector = X86_XCPT_DF;
+            uErr     = 0;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            /* VMX nested-guest #DF intercept needs to be checked here. */
+            if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+            {
+                VBOXSTRICTRC rcStrict0 = iemVmxVmexitEventDoubleFault(pVCpu);
+                if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+                    return rcStrict0;
+            }
+#endif
+            /* SVM nested-guest #DF intercepts need to be checked now. See AMD spec. 15.12 "Exception Intercepts". */
+            if (IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, X86_XCPT_DF))
+                IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_DF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+        else if (enmRaise == IEMXCPTRAISE_TRIPLE_FAULT)
+        {
+            Log2(("iemRaiseXcptOrInt: Raising triple fault. uPrevXcpt=%#x\n", uPrevXcpt));
+            return iemInitiateCpuShutdown(pVCpu);
+        }
+        else if (enmRaise == IEMXCPTRAISE_CPU_HANG)
+        {
+            /* If a nested-guest enters an endless CPU loop condition, we'll emulate it; otherwise guru. */
+            Log2(("iemRaiseXcptOrInt: CPU hang condition detected\n"));
+            if (   !CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))
+                && !CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
+                return VERR_EM_GUEST_CPU_HANG;
+        }
+        else
+        {
+            AssertMsgFailed(("Unexpected condition! enmRaise=%#x uPrevXcpt=%#x fPrevXcpt=%#x, u8Vector=%#x fFlags=%#x\n",
+                             enmRaise, uPrevXcpt, fPrevXcpt, u8Vector, fFlags));
+            return VERR_IEM_IPE_9;
+        }
+
+        /*
+         * The 'EXT' bit is set when an exception occurs during deliver of an external
+         * event (such as an interrupt or earlier exception)[1]. Privileged software
+         * exception (INT1) also sets the EXT bit[2]. Exceptions generated by software
+         * interrupts and INTO, INT3 instructions, the 'EXT' bit will not be set.
+         *
+         * [1] - Intel spec. 6.13 "Error Code"
+         * [2] - Intel spec. 26.5.1.1 "Details of Vectored-Event Injection".
+         * [3] - Intel Instruction reference for INT n.
+         */
+        if (   (fPrevXcpt & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_T_EXT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR))
+            && (fFlags & IEM_XCPT_FLAGS_ERR)
+            && u8Vector != X86_XCPT_PF
+            && u8Vector != X86_XCPT_DF)
+        {
+            uErr |= X86_TRAP_ERR_EXTERNAL;
+        }
+    }
+
+    pVCpu->iem.s.cXcptRecursions++;
+    pVCpu->iem.s.uCurXcpt    = u8Vector;
+    pVCpu->iem.s.fCurXcpt    = fFlags;
+    pVCpu->iem.s.uCurXcptErr = uErr;
+    pVCpu->iem.s.uCurXcptCr2 = uCr2;
+
+    /*
+     * Extensive logging.
+     */
+#if defined(LOG_ENABLED) && defined(IN_RING3)
+    if (LogIs3Enabled())
+    {
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR_MASK);
+        PVM     pVM = pVCpu->CTX_SUFF(pVM);
+        char    szRegs[4096];
+        DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
+                        "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
+                        "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
+                        "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
+                        "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
+                        "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
+                        "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
+                        "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
+                        "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
+                        "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
+                        "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
+                        "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
+                        "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
+                        "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
+                        "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim}  idtr=%016VR{idtr_base}:%04VR{idtr_lim}  rflags=%08VR{rflags}\n"
+                        "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
+                        "tr  ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
+                        "    sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
+                        "        efer=%016VR{efer}\n"
+                        "         pat=%016VR{pat}\n"
+                        "     sf_mask=%016VR{sf_mask}\n"
+                        "krnl_gs_base=%016VR{krnl_gs_base}\n"
+                        "       lstar=%016VR{lstar}\n"
+                        "        star=%016VR{star} cstar=%016VR{cstar}\n"
+                        "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
+                        );
+
+        char szInstr[256];
+        DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
+                           DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                           szInstr, sizeof(szInstr), NULL);
+        Log3(("%s%s\n", szRegs, szInstr));
+    }
+#endif /* LOG_ENABLED */
+
+    /*
+     * Call the mode specific worker function.
+     */
+    VBOXSTRICTRC    rcStrict;
+    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
+        rcStrict = iemRaiseXcptOrIntInRealMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
+    else if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)
+        rcStrict = iemRaiseXcptOrIntInLongMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
+    else
+        rcStrict = iemRaiseXcptOrIntInProtMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
+
+    /* Flush the prefetch buffer. */
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+#else
+    pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
+#endif
+
+    /*
+     * Unwind.
+     */
+    pVCpu->iem.s.cXcptRecursions--;
+    pVCpu->iem.s.uCurXcpt = uPrevXcpt;
+    pVCpu->iem.s.fCurXcpt = fPrevXcpt;
+    Log(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv cpl=%u depth=%d\n",
+         VBOXSTRICTRC_VAL(rcStrict), u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, pVCpu->iem.s.uCpl,
+         pVCpu->iem.s.cXcptRecursions + 1));
+    return rcStrict;
+}
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * See iemRaiseXcptOrInt.  Will not return.
+ */
+IEM_STATIC DECL_NO_RETURN(void)
+iemRaiseXcptOrIntJmp(PVMCPUCC      pVCpu,
+                     uint8_t     cbInstr,
+                     uint8_t     u8Vector,
+                     uint32_t    fFlags,
+                     uint16_t    uErr,
+                     uint64_t    uCr2)
+{
+    VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
+    longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+}
+#endif
+
+
+/** \#DE - 00.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseDivideError(PVMCPUCC pVCpu)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** \#DB - 01.
+ * @note This automatically clear DR7.GD.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseDebugException(PVMCPUCC pVCpu)
+{
+    /** @todo set/clear RF. */
+    pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** \#BR - 05.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_BR, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** \#UD - 06.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseUndefinedOpcode(PVMCPUCC pVCpu)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** \#NM - 07.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** \#TS(err) - 0a.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
+}
+
+
+/** \#TS(tr) - 0a.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                             pVCpu->cpum.GstCtx.tr.Sel, 0);
+}
+
+
+/** \#TS(0) - 0a.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                             0, 0);
+}
+
+
+/** \#TS(err) - 0a.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                             uSel & X86_SEL_MASK_OFF_RPL, 0);
+}
+
+
+/** \#NP(err) - 0b.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
+}
+
+
+/** \#NP(sel) - 0b.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                             uSel & ~X86_SEL_RPL, 0);
+}
+
+
+/** \#SS(seg) - 0c.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                             uSel & ~X86_SEL_RPL, 0);
+}
+
+
+/** \#SS(err) - 0c.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
+}
+
+
+/** \#GP(n) - 0d.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
+}
+
+
+/** \#GP(0) - 0d.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+
+#ifdef IEM_WITH_SETJMP
+/** \#GP(0) - 0d.  */
+DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu)
+{
+    iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+#endif
+
+
+/** \#GP(sel) - 0d.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                             Sel & ~X86_SEL_RPL, 0);
+}
+
+
+/** \#GP(0) - 0d.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseNotCanonical(PVMCPUCC pVCpu)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+
+
+/** \#GP(sel) - 0d.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess)
+{
+    NOREF(iSegReg); NOREF(fAccess);
+    return iemRaiseXcptOrInt(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
+                             IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+
+#ifdef IEM_WITH_SETJMP
+/** \#GP(sel) - 0d, longjmp.  */
+DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess)
+{
+    NOREF(iSegReg); NOREF(fAccess);
+    iemRaiseXcptOrIntJmp(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
+                         IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+#endif
+
+/** \#GP(sel) - 0d.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel)
+{
+    NOREF(Sel);
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+
+#ifdef IEM_WITH_SETJMP
+/** \#GP(sel) - 0d, longjmp.  */
+DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel)
+{
+    NOREF(Sel);
+    iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+#endif
+
+
+/** \#GP(sel) - 0d.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess)
+{
+    NOREF(iSegReg); NOREF(fAccess);
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+
+#ifdef IEM_WITH_SETJMP
+/** \#GP(sel) - 0d, longjmp.  */
+DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg,
+                                                                                  uint32_t fAccess)
+{
+    NOREF(iSegReg); NOREF(fAccess);
+    iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+#endif
+
+
+/** \#PF(n) - 0e.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc)
+{
+    uint16_t uErr;
+    switch (rc)
+    {
+        case VERR_PAGE_NOT_PRESENT:
+        case VERR_PAGE_TABLE_NOT_PRESENT:
+        case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
+        case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
+            uErr = 0;
+            break;
+
+        default:
+            AssertMsgFailed(("%Rrc\n", rc));
+            RT_FALL_THRU();
+        case VERR_ACCESS_DENIED:
+            uErr = X86_TRAP_PF_P;
+            break;
+
+        /** @todo reserved  */
+    }
+
+    if (pVCpu->iem.s.uCpl == 3)
+        uErr |= X86_TRAP_PF_US;
+
+    if (   (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
+        && (   (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
+            && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) ) )
+        uErr |= X86_TRAP_PF_ID;
+
+#if 0 /* This is so much non-sense, really.  Why was it done like that? */
+    /* Note! RW access callers reporting a WRITE protection fault, will clear
+             the READ flag before calling.  So, read-modify-write accesses (RW)
+             can safely be reported as READ faults. */
+    if ((fAccess & (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
+        uErr |= X86_TRAP_PF_RW;
+#else
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+    {
+        if (!(fAccess & IEM_ACCESS_TYPE_READ))
+            uErr |= X86_TRAP_PF_RW;
+    }
+#endif
+
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR | IEM_XCPT_FLAGS_CR2,
+                             uErr, GCPtrWhere);
+}
+
+#ifdef IEM_WITH_SETJMP
+/** \#PF(n) - 0e, longjmp.  */
+IEM_STATIC DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc)
+{
+    longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(iemRaisePageFault(pVCpu, GCPtrWhere, fAccess, rc)));
+}
+#endif
+
+
+/** \#MF(0) - 10.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseMathFault(PVMCPUCC pVCpu)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** \#AC(0) - 11.  */
+DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseAlignmentCheckException(PVMCPUCC pVCpu)
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/**
+ * Macro for calling iemCImplRaiseDivideError().
+ *
+ * This enables us to add/remove arguments and force different levels of
+ * inlining as we wish.
+ *
+ * @return  Strict VBox status code.
+ */
+#define IEMOP_RAISE_DIVIDE_ERROR()          IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseDivideError)
+IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
+{
+    NOREF(cbInstr);
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/**
+ * Macro for calling iemCImplRaiseInvalidLockPrefix().
+ *
+ * This enables us to add/remove arguments and force different levels of
+ * inlining as we wish.
+ *
+ * @return  Strict VBox status code.
+ */
+#define IEMOP_RAISE_INVALID_LOCK_PREFIX()   IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidLockPrefix)
+IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
+{
+    NOREF(cbInstr);
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/**
+ * Macro for calling iemCImplRaiseInvalidOpcode().
+ *
+ * This enables us to add/remove arguments and force different levels of
+ * inlining as we wish.
+ *
+ * @return  Strict VBox status code.
+ */
+#define IEMOP_RAISE_INVALID_OPCODE()        IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidOpcode)
+IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
+{
+    NOREF(cbInstr);
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** @}  */
+
+
+/*
+ *
+ * Helpers routines.
+ * Helpers routines.
+ * Helpers routines.
+ *
+ */
+
+/**
+ * Recalculates the effective operand size.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+IEM_STATIC void iemRecalEffOpSize(PVMCPUCC pVCpu)
+{
+    switch (pVCpu->iem.s.enmCpuMode)
+    {
+        case IEMMODE_16BIT:
+            pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
+            break;
+        case IEMMODE_32BIT:
+            pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
+            break;
+        case IEMMODE_64BIT:
+            switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP))
+            {
+                case 0:
+                    pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
+                    break;
+                case IEM_OP_PRF_SIZE_OP:
+                    pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
+                    break;
+                case IEM_OP_PRF_SIZE_REX_W:
+                case IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP:
+                    pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
+                    break;
+            }
+            break;
+        default:
+            AssertFailed();
+    }
+}
+
+
+/**
+ * Sets the default operand size to 64-bit and recalculates the effective
+ * operand size.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+IEM_STATIC void iemRecalEffOpSize64Default(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
+    pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
+    if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
+        pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
+    else
+        pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
+}
+
+
+/*
+ *
+ * Common opcode decoders.
+ * Common opcode decoders.
+ * Common opcode decoders.
+ *
+ */
+//#include <iprt/mem.h>
+
+/**
+ * Used to add extra details about a stub case.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ */
+IEM_STATIC void iemOpStubMsg2(PVMCPUCC pVCpu)
+{
+#if defined(LOG_ENABLED) && defined(IN_RING3)
+    PVM  pVM = pVCpu->CTX_SUFF(pVM);
+    char szRegs[4096];
+    DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
+                    "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
+                    "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
+                    "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
+                    "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
+                    "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
+                    "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
+                    "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
+                    "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
+                    "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
+                    "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
+                    "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
+                    "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
+                    "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
+                    "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim}  idtr=%016VR{idtr_base}:%04VR{idtr_lim}  rflags=%08VR{rflags}\n"
+                    "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
+                    "tr  ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
+                    "    sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
+                    "        efer=%016VR{efer}\n"
+                    "         pat=%016VR{pat}\n"
+                    "     sf_mask=%016VR{sf_mask}\n"
+                    "krnl_gs_base=%016VR{krnl_gs_base}\n"
+                    "       lstar=%016VR{lstar}\n"
+                    "        star=%016VR{star} cstar=%016VR{cstar}\n"
+                    "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
+                    );
+
+    char szInstr[256];
+    DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
+                       DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                       szInstr, sizeof(szInstr), NULL);
+
+    RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
+#else
+    RTAssertMsg2Weak("cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip);
+#endif
+}
+
+/**
+ * Complains about a stub.
+ *
+ * Providing two versions of this macro, one for daily use and one for use when
+ * working on IEM.
+ */
+#if 0
+# define IEMOP_BITCH_ABOUT_STUB() \
+    do { \
+        RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__); \
+        iemOpStubMsg2(pVCpu); \
+        RTAssertPanic(); \
+    } while (0)
+#else
+# define IEMOP_BITCH_ABOUT_STUB() Log(("Stub: %s (line %d)\n", __FUNCTION__, __LINE__));
+#endif
+
+/** Stubs an opcode. */
+#define FNIEMOP_STUB(a_Name) \
+    FNIEMOP_DEF(a_Name) \
+    { \
+        RT_NOREF_PV(pVCpu); \
+        IEMOP_BITCH_ABOUT_STUB(); \
+        return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
+    } \
+    typedef int ignore_semicolon
+
+/** Stubs an opcode. */
+#define FNIEMOP_STUB_1(a_Name, a_Type0, a_Name0) \
+    FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
+    { \
+        RT_NOREF_PV(pVCpu); \
+        RT_NOREF_PV(a_Name0); \
+        IEMOP_BITCH_ABOUT_STUB(); \
+        return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
+    } \
+    typedef int ignore_semicolon
+
+/** Stubs an opcode which currently should raise \#UD. */
+#define FNIEMOP_UD_STUB(a_Name) \
+    FNIEMOP_DEF(a_Name) \
+    { \
+        Log(("Unsupported instruction %Rfn\n", __FUNCTION__)); \
+        return IEMOP_RAISE_INVALID_OPCODE(); \
+    } \
+    typedef int ignore_semicolon
+
+/** Stubs an opcode which currently should raise \#UD. */
+#define FNIEMOP_UD_STUB_1(a_Name, a_Type0, a_Name0) \
+    FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
+    { \
+        RT_NOREF_PV(pVCpu); \
+        RT_NOREF_PV(a_Name0); \
+        Log(("Unsupported instruction %Rfn\n", __FUNCTION__)); \
+        return IEMOP_RAISE_INVALID_OPCODE(); \
+    } \
+    typedef int ignore_semicolon
+
+
+
+/** @name   Register Access.
+ * @{
+ */
+
+/**
+ * Gets a reference (pointer) to the specified hidden segment register.
+ *
+ * @returns Hidden register reference.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The segment register.
+ */
+IEM_STATIC PCPUMSELREG iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg)
+{
+    Assert(iSegReg < X86_SREG_COUNT);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+    PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
+
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
+    return pSReg;
+}
+
+
+/**
+ * Ensures that the given hidden segment register is up to date.
+ *
+ * @returns Hidden register reference.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pSReg               The segment register.
+ */
+IEM_STATIC PCPUMSELREG iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
+{
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
+    NOREF(pVCpu);
+    return pSReg;
+}
+
+
+/**
+ * Gets a reference (pointer) to the specified segment register (the selector
+ * value).
+ *
+ * @returns Pointer to the selector variable.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The segment register.
+ */
+DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg)
+{
+    Assert(iSegReg < X86_SREG_COUNT);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+    return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
+}
+
+
+/**
+ * Fetches the selector value of a segment register.
+ *
+ * @returns The selector value.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The segment register.
+ */
+DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg)
+{
+    Assert(iSegReg < X86_SREG_COUNT);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+    return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
+}
+
+
+/**
+ * Fetches the base address value of a segment register.
+ *
+ * @returns The selector value.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The segment register.
+ */
+DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg)
+{
+    Assert(iSegReg < X86_SREG_COUNT);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+    return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
+}
+
+
+/**
+ * Gets a reference (pointer) to the specified general purpose register.
+ *
+ * @returns Register reference.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iReg                The general purpose register.
+ */
+DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg)
+{
+    Assert(iReg < 16);
+    return &pVCpu->cpum.GstCtx.aGRegs[iReg];
+}
+
+
+/**
+ * Gets a reference (pointer) to the specified 8-bit general purpose register.
+ *
+ * Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
+ *
+ * @returns Register reference.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iReg                The register.
+ */
+DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg)
+{
+    if (iReg < 4 || (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
+    {
+        Assert(iReg < 16);
+        return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
+    }
+    /* high 8-bit register. */
+    Assert(iReg < 8);
+    return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
+}
+
+
+/**
+ * Gets a reference (pointer) to the specified 16-bit general purpose register.
+ *
+ * @returns Register reference.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iReg                The register.
+ */
+DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg)
+{
+    Assert(iReg < 16);
+    return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
+}
+
+
+/**
+ * Gets a reference (pointer) to the specified 32-bit general purpose register.
+ *
+ * @returns Register reference.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iReg                The register.
+ */
+DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg)
+{
+    Assert(iReg < 16);
+    return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
+}
+
+
+/**
+ * Gets a reference (pointer) to the specified 64-bit general purpose register.
+ *
+ * @returns Register reference.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iReg                The register.
+ */
+DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg)
+{
+    Assert(iReg < 64);
+    return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
+}
+
+
+/**
+ * Gets a reference (pointer) to the specified segment register's base address.
+ *
+ * @returns Segment register base address reference.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The segment selector.
+ */
+DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg)
+{
+    Assert(iSegReg < X86_SREG_COUNT);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+    return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
+}
+
+
+/**
+ * Fetches the value of a 8-bit general purpose register.
+ *
+ * @returns The register value.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iReg                The register.
+ */
+DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg)
+{
+    return *iemGRegRefU8(pVCpu, iReg);
+}
+
+
+/**
+ * Fetches the value of a 16-bit general purpose register.
+ *
+ * @returns The register value.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iReg                The register.
+ */
+DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg)
+{
+    Assert(iReg < 16);
+    return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
+}
+
+
+/**
+ * Fetches the value of a 32-bit general purpose register.
+ *
+ * @returns The register value.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iReg                The register.
+ */
+DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg)
+{
+    Assert(iReg < 16);
+    return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
+}
+
+
+/**
+ * Fetches the value of a 64-bit general purpose register.
+ *
+ * @returns The register value.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iReg                The register.
+ */
+DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg)
+{
+    Assert(iReg < 16);
+    return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
+}
+
+
+/**
+ * Adds a 8-bit signed jump offset to RIP/EIP/IP.
+ *
+ * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
+ * segment limit.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   offNextInstr        The offset of the next instruction.
+ */
+IEM_STATIC VBOXSTRICTRC iemRegRipRelativeJumpS8(PVMCPUCC pVCpu, int8_t offNextInstr)
+{
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
+            if (   uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
+                && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            pVCpu->cpum.GstCtx.rip = uNewIp;
+            break;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
+            Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
+
+            uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
+            if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            pVCpu->cpum.GstCtx.rip = uNewEip;
+            break;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
+
+            uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
+            if (!IEM_IS_CANONICAL(uNewRip))
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            pVCpu->cpum.GstCtx.rip = uNewRip;
+            break;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Adds a 16-bit signed jump offset to RIP/EIP/IP.
+ *
+ * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
+ * segment limit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   offNextInstr        The offset of the next instruction.
+ */
+IEM_STATIC VBOXSTRICTRC iemRegRipRelativeJumpS16(PVMCPUCC pVCpu, int16_t offNextInstr)
+{
+    Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT);
+
+    uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
+    if (   uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
+        && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    /** @todo Test 16-bit jump in 64-bit mode. possible?  */
+    pVCpu->cpum.GstCtx.rip = uNewIp;
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Adds a 32-bit signed jump offset to RIP/EIP/IP.
+ *
+ * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
+ * segment limit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   offNextInstr        The offset of the next instruction.
+ */
+IEM_STATIC VBOXSTRICTRC iemRegRipRelativeJumpS32(PVMCPUCC pVCpu, int32_t offNextInstr)
+{
+    Assert(pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT);
+
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT)
+    {
+        Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
+
+        uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
+        if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        pVCpu->cpum.GstCtx.rip = uNewEip;
+    }
+    else
+    {
+        Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
+
+        uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
+        if (!IEM_IS_CANONICAL(uNewRip))
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        pVCpu->cpum.GstCtx.rip = uNewRip;
+    }
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Performs a near jump to the specified address.
+ *
+ * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
+ * segment limit.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   uNewRip             The new RIP value.
+ */
+IEM_STATIC VBOXSTRICTRC iemRegRipJump(PVMCPUCC pVCpu, uint64_t uNewRip)
+{
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            Assert(uNewRip <= UINT16_MAX);
+            if (   uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit
+                && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            /** @todo Test 16-bit jump in 64-bit mode.  */
+            pVCpu->cpum.GstCtx.rip = uNewRip;
+            break;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            Assert(uNewRip <= UINT32_MAX);
+            Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
+            Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
+
+            if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit)
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            pVCpu->cpum.GstCtx.rip = uNewRip;
+            break;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
+
+            if (!IEM_IS_CANONICAL(uNewRip))
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            pVCpu->cpum.GstCtx.rip = uNewRip;
+            break;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get the address of the top of the stack.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu)
+{
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        return pVCpu->cpum.GstCtx.rsp;
+    if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+        return pVCpu->cpum.GstCtx.esp;
+    return pVCpu->cpum.GstCtx.sp;
+}
+
+
+/**
+ * Updates the RIP/EIP/IP to point to the next instruction.
+ *
+ * This function leaves the EFLAGS.RF flag alone.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr             The number of bytes to add.
+ */
+IEM_STATIC void iemRegAddToRipKeepRF(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    switch (pVCpu->iem.s.enmCpuMode)
+    {
+        case IEMMODE_16BIT:
+            Assert(pVCpu->cpum.GstCtx.rip <= UINT16_MAX);
+            pVCpu->cpum.GstCtx.eip += cbInstr;
+            pVCpu->cpum.GstCtx.eip &= UINT32_C(0xffff);
+            break;
+
+        case IEMMODE_32BIT:
+            pVCpu->cpum.GstCtx.eip += cbInstr;
+            Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
+            break;
+
+        case IEMMODE_64BIT:
+            pVCpu->cpum.GstCtx.rip += cbInstr;
+            break;
+        default: AssertFailed();
+    }
+}
+
+
+#if 0
+/**
+ * Updates the RIP/EIP/IP to point to the next instruction.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+IEM_STATIC void iemRegUpdateRipKeepRF(PVMCPUCC pVCpu)
+{
+    return iemRegAddToRipKeepRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
+}
+#endif
+
+
+
+/**
+ * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr             The number of bytes to add.
+ */
+IEM_STATIC void iemRegAddToRipAndClearRF(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+    AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
+#if ARCH_BITS >= 64
+    static uint64_t const s_aRipMasks[] = { UINT64_C(0xffffffff), UINT64_C(0xffffffff), UINT64_MAX };
+    Assert(pVCpu->cpum.GstCtx.rip <= s_aRipMasks[(unsigned)pVCpu->iem.s.enmCpuMode]);
+    pVCpu->cpum.GstCtx.rip = (pVCpu->cpum.GstCtx.rip + cbInstr) & s_aRipMasks[(unsigned)pVCpu->iem.s.enmCpuMode];
+#else
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        pVCpu->cpum.GstCtx.rip += cbInstr;
+    else
+        pVCpu->cpum.GstCtx.eip += cbInstr;
+#endif
+}
+
+
+/**
+ * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+IEM_STATIC void iemRegUpdateRipAndClearRF(PVMCPUCC pVCpu)
+{
+    return iemRegAddToRipAndClearRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
+}
+
+
+/**
+ * Adds to the stack pointer.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbToAdd             The number of bytes to add (8-bit!).
+ */
+DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd)
+{
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        pVCpu->cpum.GstCtx.rsp += cbToAdd;
+    else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+        pVCpu->cpum.GstCtx.esp += cbToAdd;
+    else
+        pVCpu->cpum.GstCtx.sp  += cbToAdd;
+}
+
+
+/**
+ * Subtracts from the stack pointer.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbToSub             The number of bytes to subtract (8-bit!).
+ */
+DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub)
+{
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        pVCpu->cpum.GstCtx.rsp -= cbToSub;
+    else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+        pVCpu->cpum.GstCtx.esp -= cbToSub;
+    else
+        pVCpu->cpum.GstCtx.sp  -= cbToSub;
+}
+
+
+/**
+ * Adds to the temporary stack pointer.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pTmpRsp             The temporary SP/ESP/RSP to update.
+ * @param   cbToAdd             The number of bytes to add (16-bit).
+ */
+DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd)
+{
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        pTmpRsp->u           += cbToAdd;
+    else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+        pTmpRsp->DWords.dw0  += cbToAdd;
+    else
+        pTmpRsp->Words.w0    += cbToAdd;
+}
+
+
+/**
+ * Subtracts from the temporary stack pointer.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pTmpRsp             The temporary SP/ESP/RSP to update.
+ * @param   cbToSub             The number of bytes to subtract.
+ * @remarks The @a cbToSub argument *MUST* be 16-bit, iemCImpl_enter is
+ *          expecting that.
+ */
+DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub)
+{
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        pTmpRsp->u          -= cbToSub;
+    else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+        pTmpRsp->DWords.dw0 -= cbToSub;
+    else
+        pTmpRsp->Words.w0   -= cbToSub;
+}
+
+
+/**
+ * Calculates the effective stack address for a push of the specified size as
+ * well as the new RSP value (upper bits may be masked).
+ *
+ * @returns Effective stack addressf for the push.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbItem              The size of the stack item to pop.
+ * @param   puNewRsp            Where to return the new RSP value.
+ */
+DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
+{
+    RTUINT64U   uTmpRsp;
+    RTGCPTR     GCPtrTop;
+    uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        GCPtrTop = uTmpRsp.u            -= cbItem;
+    else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+        GCPtrTop = uTmpRsp.DWords.dw0   -= cbItem;
+    else
+        GCPtrTop = uTmpRsp.Words.w0     -= cbItem;
+    *puNewRsp = uTmpRsp.u;
+    return GCPtrTop;
+}
+
+
+/**
+ * Gets the current stack pointer and calculates the value after a pop of the
+ * specified size.
+ *
+ * @returns Current stack pointer.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbItem              The size of the stack item to pop.
+ * @param   puNewRsp            Where to return the new RSP value.
+ */
+DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
+{
+    RTUINT64U   uTmpRsp;
+    RTGCPTR     GCPtrTop;
+    uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        GCPtrTop = uTmpRsp.u;
+        uTmpRsp.u += cbItem;
+    }
+    else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+    {
+        GCPtrTop = uTmpRsp.DWords.dw0;
+        uTmpRsp.DWords.dw0 += cbItem;
+    }
+    else
+    {
+        GCPtrTop = uTmpRsp.Words.w0;
+        uTmpRsp.Words.w0 += cbItem;
+    }
+    *puNewRsp = uTmpRsp.u;
+    return GCPtrTop;
+}
+
+
+/**
+ * Calculates the effective stack address for a push of the specified size as
+ * well as the new temporary RSP value (upper bits may be masked).
+ *
+ * @returns Effective stack addressf for the push.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pTmpRsp             The temporary stack pointer.  This is updated.
+ * @param   cbItem              The size of the stack item to pop.
+ */
+DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
+{
+    RTGCPTR GCPtrTop;
+
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        GCPtrTop = pTmpRsp->u          -= cbItem;
+    else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+        GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
+    else
+        GCPtrTop = pTmpRsp->Words.w0   -= cbItem;
+    return GCPtrTop;
+}
+
+
+/**
+ * Gets the effective stack address for a pop of the specified size and
+ * calculates and updates the temporary RSP.
+ *
+ * @returns Current stack pointer.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pTmpRsp             The temporary stack pointer.  This is updated.
+ * @param   cbItem              The size of the stack item to pop.
+ */
+DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
+{
+    RTGCPTR GCPtrTop;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        GCPtrTop = pTmpRsp->u;
+        pTmpRsp->u          += cbItem;
+    }
+    else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+    {
+        GCPtrTop = pTmpRsp->DWords.dw0;
+        pTmpRsp->DWords.dw0 += cbItem;
+    }
+    else
+    {
+        GCPtrTop = pTmpRsp->Words.w0;
+        pTmpRsp->Words.w0   += cbItem;
+    }
+    return GCPtrTop;
+}
+
+/** @}  */
+
+
+/** @name   FPU access and helpers.
+ *
+ * @{
+ */
+
+
+/**
+ * Hook for preparing to use the host FPU.
+ *
+ * This is necessary in ring-0 and raw-mode context (nop in ring-3).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu)
+{
+#ifdef IN_RING3
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
+#else
+    CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
+#endif
+    IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
+}
+
+
+/**
+ * Hook for preparing to use the host FPU for SSE.
+ *
+ * This is necessary in ring-0 and raw-mode context (nop in ring-3).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu)
+{
+    iemFpuPrepareUsage(pVCpu);
+}
+
+
+/**
+ * Hook for preparing to use the host FPU for AVX.
+ *
+ * This is necessary in ring-0 and raw-mode context (nop in ring-3).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu)
+{
+    iemFpuPrepareUsage(pVCpu);
+}
+
+
+/**
+ * Hook for actualizing the guest FPU state before the interpreter reads it.
+ *
+ * This is necessary in ring-0 and raw-mode context (nop in ring-3).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu)
+{
+#ifdef IN_RING3
+    NOREF(pVCpu);
+#else
+    CPUMRZFpuStateActualizeForRead(pVCpu);
+#endif
+    IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
+}
+
+
+/**
+ * Hook for actualizing the guest FPU state before the interpreter changes it.
+ *
+ * This is necessary in ring-0 and raw-mode context (nop in ring-3).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu)
+{
+#ifdef IN_RING3
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
+#else
+    CPUMRZFpuStateActualizeForChange(pVCpu);
+#endif
+    IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
+}
+
+
+/**
+ * Hook for actualizing the guest XMM0..15 and MXCSR register state for read
+ * only.
+ *
+ * This is necessary in ring-0 and raw-mode context (nop in ring-3).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu)
+{
+#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
+    NOREF(pVCpu);
+#else
+    CPUMRZFpuStateActualizeSseForRead(pVCpu);
+#endif
+    IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
+}
+
+
+/**
+ * Hook for actualizing the guest XMM0..15 and MXCSR register state for
+ * read+write.
+ *
+ * This is necessary in ring-0 and raw-mode context (nop in ring-3).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu)
+{
+#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
+#else
+    CPUMRZFpuStateActualizeForChange(pVCpu);
+#endif
+    IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
+
+    /* Make sure any changes are loaded the next time around. */
+    pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->Hdr.bmXState |= XSAVE_C_SSE;
+}
+
+
+/**
+ * Hook for actualizing the guest YMM0..15 and MXCSR register state for read
+ * only.
+ *
+ * This is necessary in ring-0 and raw-mode context (nop in ring-3).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu)
+{
+#ifdef IN_RING3
+    NOREF(pVCpu);
+#else
+    CPUMRZFpuStateActualizeAvxForRead(pVCpu);
+#endif
+    IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
+}
+
+
+/**
+ * Hook for actualizing the guest YMM0..15 and MXCSR register state for
+ * read+write.
+ *
+ * This is necessary in ring-0 and raw-mode context (nop in ring-3).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu)
+{
+#ifdef IN_RING3
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
+#else
+    CPUMRZFpuStateActualizeForChange(pVCpu);
+#endif
+    IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
+
+    /* Just assume we're going to make changes to the SSE and YMM_HI parts. */
+    pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->Hdr.bmXState |= XSAVE_C_YMM | XSAVE_C_SSE;
+}
+
+
+/**
+ * Stores a QNaN value into a FPU register.
+ *
+ * @param   pReg                Pointer to the register.
+ */
+DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
+{
+    pReg->au32[0] = UINT32_C(0x00000000);
+    pReg->au32[1] = UINT32_C(0xc0000000);
+    pReg->au16[4] = UINT16_C(0xffff);
+}
+
+
+/**
+ * Updates the FOP, FPU.CS and FPUIP registers.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pFpuCtx             The FPU context.
+ */
+DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx)
+{
+    Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
+    pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode;
+    /** @todo x87.CS and FPUIP needs to be kept seperately. */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        /** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
+         *        happens in real mode here based on the fnsave and fnstenv images. */
+        pFpuCtx->CS    = 0;
+        pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip | ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
+    }
+    else
+    {
+        pFpuCtx->CS    = pVCpu->cpum.GstCtx.cs.Sel;
+        pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
+    }
+}
+
+
+/**
+ * Updates the x87.DS and FPUDP registers.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pFpuCtx             The FPU context.
+ * @param   iEffSeg             The effective segment register.
+ * @param   GCPtrEff            The effective address relative to @a iEffSeg.
+ */
+DECLINLINE(void) iemFpuUpdateDP(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
+{
+    RTSEL sel;
+    switch (iEffSeg)
+    {
+        case X86_SREG_DS: sel = pVCpu->cpum.GstCtx.ds.Sel; break;
+        case X86_SREG_SS: sel = pVCpu->cpum.GstCtx.ss.Sel; break;
+        case X86_SREG_CS: sel = pVCpu->cpum.GstCtx.cs.Sel; break;
+        case X86_SREG_ES: sel = pVCpu->cpum.GstCtx.es.Sel; break;
+        case X86_SREG_FS: sel = pVCpu->cpum.GstCtx.fs.Sel; break;
+        case X86_SREG_GS: sel = pVCpu->cpum.GstCtx.gs.Sel; break;
+        default:
+            AssertMsgFailed(("%d\n", iEffSeg));
+            sel = pVCpu->cpum.GstCtx.ds.Sel;
+    }
+    /** @todo pFpuCtx->DS and FPUDP needs to be kept seperately. */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        pFpuCtx->DS    = 0;
+        pFpuCtx->FPUDP = (uint32_t)GCPtrEff + ((uint32_t)sel << 4);
+    }
+    else
+    {
+        pFpuCtx->DS    = sel;
+        pFpuCtx->FPUDP = GCPtrEff;
+    }
+}
+
+
+/**
+ * Rotates the stack registers in the push direction.
+ *
+ * @param   pFpuCtx             The FPU context.
+ * @remarks This is a complete waste of time, but fxsave stores the registers in
+ *          stack order.
+ */
+DECLINLINE(void) iemFpuRotateStackPush(PX86FXSTATE pFpuCtx)
+{
+    RTFLOAT80U r80Tmp = pFpuCtx->aRegs[7].r80;
+    pFpuCtx->aRegs[7].r80 = pFpuCtx->aRegs[6].r80;
+    pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[5].r80;
+    pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[4].r80;
+    pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[3].r80;
+    pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[2].r80;
+    pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[1].r80;
+    pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[0].r80;
+    pFpuCtx->aRegs[0].r80 = r80Tmp;
+}
+
+
+/**
+ * Rotates the stack registers in the pop direction.
+ *
+ * @param   pFpuCtx             The FPU context.
+ * @remarks This is a complete waste of time, but fxsave stores the registers in
+ *          stack order.
+ */
+DECLINLINE(void) iemFpuRotateStackPop(PX86FXSTATE pFpuCtx)
+{
+    RTFLOAT80U r80Tmp = pFpuCtx->aRegs[0].r80;
+    pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[1].r80;
+    pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[2].r80;
+    pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[3].r80;
+    pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[4].r80;
+    pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[5].r80;
+    pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[6].r80;
+    pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[7].r80;
+    pFpuCtx->aRegs[7].r80 = r80Tmp;
+}
+
+
+/**
+ * Updates FSW and pushes a FPU result onto the FPU stack if no pending
+ * exception prevents it.
+ *
+ * @param   pResult             The FPU operation result to push.
+ * @param   pFpuCtx             The FPU context.
+ */
+IEM_STATIC void iemFpuMaybePushResult(PIEMFPURESULT pResult, PX86FXSTATE pFpuCtx)
+{
+    /* Update FSW and bail if there are pending exceptions afterwards. */
+    uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
+    fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
+    if (   (fFsw             & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
+        & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
+    {
+        pFpuCtx->FSW = fFsw;
+        return;
+    }
+
+    uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
+    if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
+    {
+        /* All is fine, push the actual value. */
+        pFpuCtx->FTW |= RT_BIT(iNewTop);
+        pFpuCtx->aRegs[7].r80 = pResult->r80Result;
+    }
+    else if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked stack overflow, push QNaN. */
+        fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1;
+        iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
+    }
+    else
+    {
+        /* Raise stack overflow, don't push anything. */
+        pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES;
+        return;
+    }
+
+    fFsw &= ~X86_FSW_TOP_MASK;
+    fFsw |= iNewTop << X86_FSW_TOP_SHIFT;
+    pFpuCtx->FSW = fFsw;
+
+    iemFpuRotateStackPush(pFpuCtx);
+}
+
+
+/**
+ * Stores a result in a FPU register and updates the FSW and FTW.
+ *
+ * @param   pFpuCtx             The FPU context.
+ * @param   pResult             The result to store.
+ * @param   iStReg              Which FPU register to store it in.
+ */
+IEM_STATIC void iemFpuStoreResultOnly(PX86FXSTATE pFpuCtx, PIEMFPURESULT pResult, uint8_t iStReg)
+{
+    Assert(iStReg < 8);
+    uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
+    pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+    pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_TOP_MASK;
+    pFpuCtx->FTW |= RT_BIT(iReg);
+    pFpuCtx->aRegs[iStReg].r80 = pResult->r80Result;
+}
+
+
+/**
+ * Only updates the FPU status word (FSW) with the result of the current
+ * instruction.
+ *
+ * @param   pFpuCtx             The FPU context.
+ * @param   u16FSW              The FSW output of the current instruction.
+ */
+IEM_STATIC void iemFpuUpdateFSWOnly(PX86FXSTATE pFpuCtx, uint16_t u16FSW)
+{
+    pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+    pFpuCtx->FSW |= u16FSW & ~X86_FSW_TOP_MASK;
+}
+
+
+/**
+ * Pops one item off the FPU stack if no pending exception prevents it.
+ *
+ * @param   pFpuCtx             The FPU context.
+ */
+IEM_STATIC void iemFpuMaybePopOne(PX86FXSTATE pFpuCtx)
+{
+    /* Check pending exceptions. */
+    uint16_t uFSW = pFpuCtx->FSW;
+    if (   (pFpuCtx->FSW & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
+        & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
+        return;
+
+    /* TOP--. */
+    uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
+    uFSW &= ~X86_FSW_TOP_MASK;
+    uFSW |= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
+    pFpuCtx->FSW = uFSW;
+
+    /* Mark the previous ST0 as empty. */
+    iOldTop >>= X86_FSW_TOP_SHIFT;
+    pFpuCtx->FTW &= ~RT_BIT(iOldTop);
+
+    /* Rotate the registers. */
+    iemFpuRotateStackPop(pFpuCtx);
+}
+
+
+/**
+ * Pushes a FPU result onto the FPU stack if no pending exception prevents it.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The FPU operation result to push.
+ */
+IEM_STATIC void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuMaybePushResult(pResult, pFpuCtx);
+}
+
+
+/**
+ * Pushes a FPU result onto the FPU stack if no pending exception prevents it,
+ * and sets FPUDP and FPUDS.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The FPU operation result to push.
+ * @param   iEffSeg             The effective segment register.
+ * @param   GCPtrEff            The effective address relative to @a iEffSeg.
+ */
+IEM_STATIC void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuMaybePushResult(pResult, pFpuCtx);
+}
+
+
+/**
+ * Replace ST0 with the first value and push the second onto the FPU stack,
+ * unless a pending exception prevents it.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The FPU operation result to store and push.
+ */
+IEM_STATIC void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+
+    /* Update FSW and bail if there are pending exceptions afterwards. */
+    uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
+    fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
+    if (   (fFsw             & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
+        & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
+    {
+        pFpuCtx->FSW = fFsw;
+        return;
+    }
+
+    uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
+    if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
+    {
+        /* All is fine, push the actual value. */
+        pFpuCtx->FTW |= RT_BIT(iNewTop);
+        pFpuCtx->aRegs[0].r80 = pResult->r80Result1;
+        pFpuCtx->aRegs[7].r80 = pResult->r80Result2;
+    }
+    else if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked stack overflow, push QNaN. */
+        fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1;
+        iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
+        iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
+    }
+    else
+    {
+        /* Raise stack overflow, don't push anything. */
+        pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES;
+        return;
+    }
+
+    fFsw &= ~X86_FSW_TOP_MASK;
+    fFsw |= iNewTop << X86_FSW_TOP_SHIFT;
+    pFpuCtx->FSW = fFsw;
+
+    iemFpuRotateStackPush(pFpuCtx);
+}
+
+
+/**
+ * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
+ * FOP.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The result to store.
+ * @param   iStReg              Which FPU register to store it in.
+ */
+IEM_STATIC void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
+}
+
+
+/**
+ * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
+ * FOP, and then pops the stack.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The result to store.
+ * @param   iStReg              Which FPU register to store it in.
+ */
+IEM_STATIC void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+/**
+ * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
+ * FPUDP, and FPUDS.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The result to store.
+ * @param   iStReg              Which FPU register to store it in.
+ * @param   iEffSeg             The effective memory operand selector register.
+ * @param   GCPtrEff            The effective memory operand offset.
+ */
+IEM_STATIC void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
+                                           uint8_t iEffSeg, RTGCPTR GCPtrEff)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
+}
+
+
+/**
+ * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
+ * FPUDP, and FPUDS, and then pops the stack.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The result to store.
+ * @param   iStReg              Which FPU register to store it in.
+ * @param   iEffSeg             The effective memory operand selector register.
+ * @param   GCPtrEff            The effective memory operand offset.
+ */
+IEM_STATIC void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult,
+                                                  uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+/**
+ * Updates the FOP, FPUIP, and FPUCS.  For FNOP.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+IEM_STATIC void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+}
+
+
+/**
+ * Marks the specified stack register as free (for FFREE).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iStReg              The register to free.
+ */
+IEM_STATIC void iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg)
+{
+    Assert(iStReg < 8);
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    uint8_t     iReg    = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
+    pFpuCtx->FTW &= ~RT_BIT(iReg);
+}
+
+
+/**
+ * Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+IEM_STATIC void iemFpuStackIncTop(PVMCPUCC pVCpu)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    uint16_t    uFsw    = pFpuCtx->FSW;
+    uint16_t    uTop    = uFsw & X86_FSW_TOP_MASK;
+    uTop  = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
+    uFsw &= ~X86_FSW_TOP_MASK;
+    uFsw |= uTop;
+    pFpuCtx->FSW = uFsw;
+}
+
+
+/**
+ * Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+IEM_STATIC void iemFpuStackDecTop(PVMCPUCC pVCpu)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    uint16_t    uFsw    = pFpuCtx->FSW;
+    uint16_t    uTop    = uFsw & X86_FSW_TOP_MASK;
+    uTop  = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
+    uFsw &= ~X86_FSW_TOP_MASK;
+    uFsw |= uTop;
+    pFpuCtx->FSW = uFsw;
+}
+
+
+/**
+ * Updates the FSW, FOP, FPUIP, and FPUCS.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16FSW              The FSW from the current instruction.
+ */
+IEM_STATIC void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
+}
+
+
+/**
+ * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16FSW              The FSW from the current instruction.
+ */
+IEM_STATIC void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+/**
+ * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16FSW              The FSW from the current instruction.
+ * @param   iEffSeg             The effective memory operand selector register.
+ * @param   GCPtrEff            The effective memory operand offset.
+ */
+IEM_STATIC void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
+}
+
+
+/**
+ * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16FSW              The FSW from the current instruction.
+ */
+IEM_STATIC void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
+    iemFpuMaybePopOne(pFpuCtx);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+/**
+ * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16FSW              The FSW from the current instruction.
+ * @param   iEffSeg             The effective memory operand selector register.
+ * @param   GCPtrEff            The effective memory operand offset.
+ */
+IEM_STATIC void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+/**
+ * Worker routine for raising an FPU stack underflow exception.
+ *
+ * @param   pFpuCtx             The FPU context.
+ * @param   iStReg              The stack register being accessed.
+ */
+IEM_STATIC void iemFpuStackUnderflowOnly(PX86FXSTATE pFpuCtx, uint8_t iStReg)
+{
+    Assert(iStReg < 8 || iStReg == UINT8_MAX);
+    if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked underflow. */
+        pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
+        uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
+        if (iStReg != UINT8_MAX)
+        {
+            pFpuCtx->FTW |= RT_BIT(iReg);
+            iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
+        }
+    }
+    else
+    {
+        pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
+    }
+}
+
+
+/**
+ * Raises a FPU stack underflow exception.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iStReg              The destination register that should be loaded
+ *                              with QNaN if \#IS is not masked. Specify
+ *                              UINT8_MAX if none (like for fcom).
+ */
+DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
+}
+
+
+DECL_NO_INLINE(IEM_STATIC, void)
+iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
+}
+
+
+DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+DECL_NO_INLINE(IEM_STATIC, void)
+iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackUnderflowOnly(pFpuCtx, UINT8_MAX);
+    iemFpuMaybePopOne(pFpuCtx);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+DECL_NO_INLINE(IEM_STATIC, void)
+iemFpuStackPushUnderflow(PVMCPUCC pVCpu)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+
+    if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked overflow - Push QNaN. */
+        uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
+        pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
+        pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
+        pFpuCtx->FTW |= RT_BIT(iNewTop);
+        iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
+        iemFpuRotateStackPush(pFpuCtx);
+    }
+    else
+    {
+        /* Exception pending - don't change TOP or the register stack. */
+        pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
+    }
+}
+
+
+DECL_NO_INLINE(IEM_STATIC, void)
+iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+
+    if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked overflow - Push QNaN. */
+        uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
+        pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
+        pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
+        pFpuCtx->FTW |= RT_BIT(iNewTop);
+        iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
+        iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
+        iemFpuRotateStackPush(pFpuCtx);
+    }
+    else
+    {
+        /* Exception pending - don't change TOP or the register stack. */
+        pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
+    }
+}
+
+
+/**
+ * Worker routine for raising an FPU stack overflow exception on a push.
+ *
+ * @param   pFpuCtx             The FPU context.
+ */
+IEM_STATIC void iemFpuStackPushOverflowOnly(PX86FXSTATE pFpuCtx)
+{
+    if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked overflow. */
+        uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
+        pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
+        pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF;
+        pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
+        pFpuCtx->FTW |= RT_BIT(iNewTop);
+        iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
+        iemFpuRotateStackPush(pFpuCtx);
+    }
+    else
+    {
+        /* Exception pending - don't change TOP or the register stack. */
+        pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
+    }
+}
+
+
+/**
+ * Raises a FPU stack overflow exception on a push.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackPushOverflow(PVMCPUCC pVCpu)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackPushOverflowOnly(pFpuCtx);
+}
+
+
+/**
+ * Raises a FPU stack overflow exception on a push with a memory operand.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iEffSeg             The effective memory operand selector register.
+ * @param   GCPtrEff            The effective memory operand offset.
+ */
+DECL_NO_INLINE(IEM_STATIC, void)
+iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackPushOverflowOnly(pFpuCtx);
+}
+
+
+IEM_STATIC int iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    uint16_t    iReg    = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
+    if (pFpuCtx->FTW & RT_BIT(iReg))
+        return VINF_SUCCESS;
+    return VERR_NOT_FOUND;
+}
+
+
+IEM_STATIC int iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    uint16_t    iReg    = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
+    if (pFpuCtx->FTW & RT_BIT(iReg))
+    {
+        *ppRef = &pFpuCtx->aRegs[iStReg].r80;
+        return VINF_SUCCESS;
+    }
+    return VERR_NOT_FOUND;
+}
+
+
+IEM_STATIC int iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
+                                        uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    uint16_t    iTop    = X86_FSW_TOP_GET(pFpuCtx->FSW);
+    uint16_t    iReg0   = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
+    uint16_t    iReg1   = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
+    if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
+    {
+        *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
+        *ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
+        return VINF_SUCCESS;
+    }
+    return VERR_NOT_FOUND;
+}
+
+
+IEM_STATIC int iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1)
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    uint16_t    iTop    = X86_FSW_TOP_GET(pFpuCtx->FSW);
+    uint16_t    iReg0   = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
+    uint16_t    iReg1   = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
+    if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
+    {
+        *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
+        return VINF_SUCCESS;
+    }
+    return VERR_NOT_FOUND;
+}
+
+
+/**
+ * Updates the FPU exception status after FCW is changed.
+ *
+ * @param   pFpuCtx             The FPU context.
+ */
+IEM_STATIC void iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx)
+{
+    uint16_t u16Fsw = pFpuCtx->FSW;
+    if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
+        u16Fsw |= X86_FSW_ES | X86_FSW_B;
+    else
+        u16Fsw &= ~(X86_FSW_ES | X86_FSW_B);
+    pFpuCtx->FSW = u16Fsw;
+}
+
+
+/**
+ * Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
+ *
+ * @returns The full FTW.
+ * @param   pFpuCtx             The FPU context.
+ */
+IEM_STATIC uint16_t iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx)
+{
+    uint8_t const   u8Ftw  = (uint8_t)pFpuCtx->FTW;
+    uint16_t        u16Ftw = 0;
+    unsigned const  iTop   = X86_FSW_TOP_GET(pFpuCtx->FSW);
+    for (unsigned iSt = 0; iSt < 8; iSt++)
+    {
+        unsigned const iReg = (iSt + iTop) & 7;
+        if (!(u8Ftw & RT_BIT(iReg)))
+            u16Ftw |= 3 << (iReg * 2); /* empty */
+        else
+        {
+            uint16_t uTag;
+            PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
+            if (pr80Reg->s.uExponent == 0x7fff)
+                uTag = 2; /* Exponent is all 1's => Special. */
+            else if (pr80Reg->s.uExponent == 0x0000)
+            {
+                if (pr80Reg->s.u64Mantissa == 0x0000)
+                    uTag = 1; /* All bits are zero => Zero. */
+                else
+                    uTag = 2; /* Must be special. */
+            }
+            else if (pr80Reg->s.u64Mantissa & RT_BIT_64(63)) /* The J bit. */
+                uTag = 0; /* Valid. */
+            else
+                uTag = 2; /* Must be special. */
+
+            u16Ftw |= uTag << (iReg * 2); /* empty */
+        }
+    }
+
+    return u16Ftw;
+}
+
+
+/**
+ * Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
+ *
+ * @returns The compressed FTW.
+ * @param   u16FullFtw      The full FTW to convert.
+ */
+IEM_STATIC uint16_t iemFpuCompressFtw(uint16_t u16FullFtw)
+{
+    uint8_t u8Ftw = 0;
+    for (unsigned i = 0; i < 8; i++)
+    {
+        if ((u16FullFtw & 3) != 3 /*empty*/)
+            u8Ftw |= RT_BIT(i);
+        u16FullFtw >>= 2;
+    }
+
+    return u8Ftw;
+}
+
+/** @}  */
+
+
+/** @name   Memory access.
+ *
+ * @{
+ */
+
+
+/**
+ * Updates the IEMCPU::cbWritten counter if applicable.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   fAccess             The access being accounted for.
+ * @param   cbMem               The access size.
+ */
+DECL_FORCE_INLINE(void) iemMemUpdateWrittenCounter(PVMCPUCC pVCpu, uint32_t fAccess, size_t cbMem)
+{
+    if (   (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_STACK | IEM_ACCESS_TYPE_WRITE)
+        || (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_DATA | IEM_ACCESS_TYPE_WRITE) )
+        pVCpu->iem.s.cbWritten += (uint32_t)cbMem;
+}
+
+
+/**
+ * Checks if the given segment can be written to, raise the appropriate
+ * exception if not.
+ *
+ * @returns VBox strict status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pHid                Pointer to the hidden register.
+ * @param   iSegReg             The register number.
+ * @param   pu64BaseAddr        Where to return the base address to use for the
+ *                              segment. (In 64-bit code it may differ from the
+ *                              base in the hidden segment.)
+ */
+IEM_STATIC VBOXSTRICTRC
+iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
+    else
+    {
+        if (!pHid->Attr.n.u1Present)
+        {
+            uint16_t    uSel = iemSRegFetchU16(pVCpu, iSegReg);
+            AssertRelease(uSel == 0);
+            Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        if (   (   (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                || !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
+            &&  pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT )
+            return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
+        *pu64BaseAddr = pHid->u64Base;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the given segment can be read from, raise the appropriate
+ * exception if not.
+ *
+ * @returns VBox strict status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pHid                Pointer to the hidden register.
+ * @param   iSegReg             The register number.
+ * @param   pu64BaseAddr        Where to return the base address to use for the
+ *                              segment. (In 64-bit code it may differ from the
+ *                              base in the hidden segment.)
+ */
+IEM_STATIC VBOXSTRICTRC
+iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
+    else
+    {
+        if (!pHid->Attr.n.u1Present)
+        {
+            uint16_t    uSel = iemSRegFetchU16(pVCpu, iSegReg);
+            AssertRelease(uSel == 0);
+            Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
+            return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
+        *pu64BaseAddr = pHid->u64Base;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Applies the segment limit, base and attributes.
+ *
+ * This may raise a \#GP or \#SS.
+ *
+ * @returns VBox strict status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   fAccess             The kind of access which is being performed.
+ * @param   iSegReg             The index of the segment register to apply.
+ *                              This is UINT8_MAX if none (for IDT, GDT, LDT,
+ *                              TSS, ++).
+ * @param   cbMem               The access size.
+ * @param   pGCPtrMem           Pointer to the guest memory address to apply
+ *                              segmentation to.  Input and output parameter.
+ */
+IEM_STATIC VBOXSTRICTRC
+iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem)
+{
+    if (iSegReg == UINT8_MAX)
+        return VINF_SUCCESS;
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+    PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
+    switch (pVCpu->iem.s.enmCpuMode)
+    {
+        case IEMMODE_16BIT:
+        case IEMMODE_32BIT:
+        {
+            RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
+            RTGCPTR32 GCPtrLast32  = GCPtrFirst32 + (uint32_t)cbMem - 1;
+
+            if (   pSel->Attr.n.u1Present
+                && !pSel->Attr.n.u1Unusable)
+            {
+                Assert(pSel->Attr.n.u1DescType);
+                if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
+                {
+                    if (   (fAccess & IEM_ACCESS_TYPE_WRITE)
+                        && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
+                        return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
+
+                    if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
+                    {
+                        /** @todo CPL check. */
+                    }
+
+                    /*
+                     * There are two kinds of data selectors, normal and expand down.
+                     */
+                    if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
+                    {
+                        if (   GCPtrFirst32 > pSel->u32Limit
+                            || GCPtrLast32  > pSel->u32Limit) /* yes, in real mode too (since 80286). */
+                            return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
+                    }
+                    else
+                    {
+                       /*
+                        * The upper boundary is defined by the B bit, not the G bit!
+                        */
+                       if (   GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
+                           || GCPtrLast32  > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
+                          return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
+                    }
+                    *pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
+                }
+                else
+                {
+
+                    /*
+                     * Code selector and usually be used to read thru, writing is
+                     * only permitted in real and V8086 mode.
+                     */
+                    if (   (   (fAccess & IEM_ACCESS_TYPE_WRITE)
+                            || (   (fAccess & IEM_ACCESS_TYPE_READ)
+                               && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
+                        && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )
+                        return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
+
+                    if (   GCPtrFirst32 > pSel->u32Limit
+                        || GCPtrLast32  > pSel->u32Limit) /* yes, in real mode too (since 80286). */
+                        return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
+
+                    if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
+                    {
+                        /** @todo CPL check. */
+                    }
+
+                    *pGCPtrMem  = GCPtrFirst32 += (uint32_t)pSel->u64Base;
+                }
+            }
+            else
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            RTGCPTR GCPtrMem = *pGCPtrMem;
+            if (iSegReg == X86_SREG_GS || iSegReg == X86_SREG_FS)
+                *pGCPtrMem = GCPtrMem + pSel->u64Base;
+
+            Assert(cbMem >= 1);
+            if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
+                return VINF_SUCCESS;
+            /** @todo We should probably raise \#SS(0) here if segment is SS; see AMD spec.
+             *        4.12.2 "Data Limit Checks in 64-bit Mode". */
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        default:
+            AssertFailedReturn(VERR_IEM_IPE_7);
+    }
+}
+
+
+/**
+ * Translates a virtual address to a physical physical address and checks if we
+ * can access the page as specified.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   GCPtrMem            The virtual address.
+ * @param   fAccess             The intended access.
+ * @param   pGCPhysMem          Where to return the physical address.
+ */
+IEM_STATIC VBOXSTRICTRC
+iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t fAccess, PRTGCPHYS pGCPhysMem)
+{
+    /** @todo Need a different PGM interface here.  We're currently using
+     *        generic / REM interfaces. this won't cut it for R0. */
+    /** @todo If/when PGM handles paged real-mode, we can remove the hack in
+     *        iemSvmWorldSwitch/iemVmxWorldSwitch to work around raising a page-fault
+     *        here. */
+    RTGCPHYS    GCPhys;
+    uint64_t    fFlags;
+    int rc = PGMGstGetPage(pVCpu, GCPtrMem, &fFlags, &GCPhys);
+    if (RT_FAILURE(rc))
+    {
+        Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
+        /** @todo Check unassigned memory in unpaged mode. */
+        /** @todo Reserved bits in page tables. Requires new PGM interface. */
+        *pGCPhysMem = NIL_RTGCPHYS;
+        return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, rc);
+    }
+
+    /* If the page is writable and does not have the no-exec bit set, all
+       access is allowed.  Otherwise we'll have to check more carefully... */
+    if ((fFlags & (X86_PTE_RW | X86_PTE_US | X86_PTE_PAE_NX)) != (X86_PTE_RW | X86_PTE_US))
+    {
+        /* Write to read only memory? */
+        if (   (fAccess & IEM_ACCESS_TYPE_WRITE)
+            && !(fFlags & X86_PTE_RW)
+            && (       (pVCpu->iem.s.uCpl == 3
+                    && !(fAccess & IEM_ACCESS_WHAT_SYS))
+                || (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
+        {
+            Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
+            *pGCPhysMem = NIL_RTGCPHYS;
+            return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
+        }
+
+        /* Kernel memory accessed by userland? */
+        if (   !(fFlags & X86_PTE_US)
+            && pVCpu->iem.s.uCpl == 3
+            && !(fAccess & IEM_ACCESS_WHAT_SYS))
+        {
+            Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
+            *pGCPhysMem = NIL_RTGCPHYS;
+            return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
+        }
+
+        /* Executing non-executable memory? */
+        if (   (fAccess & IEM_ACCESS_TYPE_EXEC)
+            && (fFlags & X86_PTE_PAE_NX)
+            && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) )
+        {
+            Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
+            *pGCPhysMem = NIL_RTGCPHYS;
+            return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~(IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE),
+                                     VERR_ACCESS_DENIED);
+        }
+    }
+
+    /*
+     * Set the dirty / access flags.
+     * ASSUMES this is set when the address is translated rather than on committ...
+     */
+    /** @todo testcase: check when A and D bits are actually set by the CPU.  */
+    uint32_t fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A;
+    if ((fFlags & fAccessedDirty) != fAccessedDirty)
+    {
+        int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
+        AssertRC(rc2);
+    }
+
+    GCPhys |= GCPtrMem & PAGE_OFFSET_MASK;
+    *pGCPhysMem = GCPhys;
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Maps a physical page.
+ *
+ * @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   GCPhysMem           The physical address.
+ * @param   fAccess             The intended access.
+ * @param   ppvMem              Where to return the mapping address.
+ * @param   pLock               The PGM lock.
+ */
+IEM_STATIC int iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem, PPGMPAGEMAPLOCK pLock)
+{
+#ifdef IEM_LOG_MEMORY_WRITES
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+        return VERR_PGM_PHYS_TLB_CATCH_ALL;
+#endif
+
+    /** @todo This API may require some improving later.  A private deal with PGM
+     *        regarding locking and unlocking needs to be struct.  A couple of TLBs
+     *        living in PGM, but with publicly accessible inlined access methods
+     *        could perhaps be an even better solution. */
+    int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
+                                  GCPhysMem,
+                                  RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
+                                  pVCpu->iem.s.fBypassHandlers,
+                                  ppvMem,
+                                  pLock);
+    /*Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.*Rhxs\n", rc, sizeof(*pLock), pLock));*/
+    AssertMsg(rc == VINF_SUCCESS || RT_FAILURE_NP(rc), ("%Rrc\n", rc));
+
+    return rc;
+}
+
+
+/**
+ * Unmap a page previously mapped by iemMemPageMap.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   GCPhysMem           The physical address.
+ * @param   fAccess             The intended access.
+ * @param   pvMem               What iemMemPageMap returned.
+ * @param   pLock               The PGM lock.
+ */
+DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem, PPGMPAGEMAPLOCK pLock)
+{
+    NOREF(pVCpu);
+    NOREF(GCPhysMem);
+    NOREF(fAccess);
+    NOREF(pvMem);
+    PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
+}
+
+
+/**
+ * Looks up a memory mapping entry.
+ *
+ * @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   pvMem           The memory address.
+ * @param   fAccess         The access to.
+ */
+DECLINLINE(int) iemMapLookup(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
+{
+    Assert(pVCpu->iem.s.cActiveMappings <= RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
+    fAccess &= IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK;
+    if (   pVCpu->iem.s.aMemMappings[0].pv == pvMem
+        && (pVCpu->iem.s.aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
+        return 0;
+    if (   pVCpu->iem.s.aMemMappings[1].pv == pvMem
+        && (pVCpu->iem.s.aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
+        return 1;
+    if (   pVCpu->iem.s.aMemMappings[2].pv == pvMem
+        && (pVCpu->iem.s.aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
+        return 2;
+    return VERR_NOT_FOUND;
+}
+
+
+/**
+ * Finds a free memmap entry when using iNextMapping doesn't work.
+ *
+ * @returns Memory mapping index, 1024 on failure.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+IEM_STATIC unsigned iemMemMapFindFree(PVMCPUCC pVCpu)
+{
+    /*
+     * The easy case.
+     */
+    if (pVCpu->iem.s.cActiveMappings == 0)
+    {
+        pVCpu->iem.s.iNextMapping = 1;
+        return 0;
+    }
+
+    /* There should be enough mappings for all instructions. */
+    AssertReturn(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), 1024);
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aMemMappings); i++)
+        if (pVCpu->iem.s.aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
+            return i;
+
+    AssertFailedReturn(1024);
+}
+
+
+/**
+ * Commits a bounce buffer that needs writing back and unmaps it.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   iMemMap         The index of the buffer to commit.
+ * @param   fPostponeFail   Whether we can postpone writer failures to ring-3.
+ *                          Always false in ring-3, obviously.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PVMCPUCC pVCpu, unsigned iMemMap, bool fPostponeFail)
+{
+    Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
+    Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
+#ifdef IN_RING3
+    Assert(!fPostponeFail);
+    RT_NOREF_PV(fPostponeFail);
+#endif
+
+    /*
+     * Do the writing.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned)
+    {
+        uint16_t const  cbFirst  = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
+        uint16_t const  cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
+        uint8_t const  *pbBuf    = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
+        if (!pVCpu->iem.s.fBypassHandlers)
+        {
+            /*
+             * Carefully and efficiently dealing with access handler return
+             * codes make this a little bloated.
+             */
+            VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM,
+                                                 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
+                                                 pbBuf,
+                                                 cbFirst,
+                                                 PGMACCESSORIGIN_IEM);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                if (cbSecond)
+                {
+                    rcStrict = PGMPhysWrite(pVM,
+                                            pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
+                                            pbBuf + cbFirst,
+                                            cbSecond,
+                                            PGMACCESSORIGIN_IEM);
+                    if (rcStrict == VINF_SUCCESS)
+                    { /* nothing */ }
+                    else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
+                        rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                    }
+#ifndef IN_RING3
+                    else if (fPostponeFail)
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
+                        pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;
+                        VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
+                        return iemSetPassUpStatus(pVCpu, rcStrict);
+                    }
+#endif
+                    else
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
+                        return rcStrict;
+                    }
+                }
+            }
+            else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+            {
+                if (!cbSecond)
+                {
+                    Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc\n",
+                         pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict) ));
+                    rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                }
+                else
+                {
+                    VBOXSTRICTRC rcStrict2 = PGMPhysWrite(pVM,
+                                                          pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
+                                                          pbBuf + cbFirst,
+                                                          cbSecond,
+                                                          PGMACCESSORIGIN_IEM);
+                    if (rcStrict2 == VINF_SUCCESS)
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
+                        rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                    }
+                    else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
+                        PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
+                        rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                    }
+#ifndef IN_RING3
+                    else if (fPostponeFail)
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
+                        pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;
+                        VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
+                        return iemSetPassUpStatus(pVCpu, rcStrict);
+                    }
+#endif
+                    else
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
+                        return rcStrict2;
+                    }
+                }
+            }
+#ifndef IN_RING3
+            else if (fPostponeFail)
+            {
+                Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
+                     pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                     pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
+                if (!cbSecond)
+                    pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST;
+                else
+                    pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND;
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
+                return iemSetPassUpStatus(pVCpu, rcStrict);
+            }
+#endif
+            else
+            {
+                Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
+                     pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
+                     pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
+                return rcStrict;
+            }
+        }
+        else
+        {
+            /*
+             * No access handlers, much simpler.
+             */
+            int rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pbBuf, cbFirst);
+            if (RT_SUCCESS(rc))
+            {
+                if (cbSecond)
+                {
+                    rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pbBuf + cbFirst, cbSecond);
+                    if (RT_SUCCESS(rc))
+                    { /* likely */ }
+                    else
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc));
+                        return rc;
+                    }
+                }
+            }
+            else
+            {
+                Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
+                     pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, rc,
+                     pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
+                return rc;
+            }
+        }
+    }
+
+#if defined(IEM_LOG_MEMORY_WRITES)
+    Log(("IEM Wrote %RGp: %.*Rhxs\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
+         RT_MAX(RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, 64), 1), &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]));
+    if (pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond)
+        Log(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
+             RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond, 64),
+             &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst]));
+
+    size_t cbWrote = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
+    g_cbIemWrote = cbWrote;
+    memcpy(g_abIemWrote, &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote)));
+#endif
+
+    /*
+     * Free the mapping entry.
+     */
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+    Assert(pVCpu->iem.s.cActiveMappings != 0);
+    pVCpu->iem.s.cActiveMappings--;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * iemMemMap worker that deals with a request crossing pages.
+ */
+IEM_STATIC VBOXSTRICTRC
+iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void **ppvMem, size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
+{
+    /*
+     * Do the address translations.
+     */
+    RTGCPHYS GCPhysFirst;
+    VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrFirst, fAccess, &GCPhysFirst);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    RTGCPHYS GCPhysSecond;
+    rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)PAGE_OFFSET_MASK,
+                                                 fAccess, &GCPhysSecond);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    GCPhysSecond &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Read in the current memory content if it's a read, execute or partial
+     * write access.
+     */
+    uint8_t        *pbBuf        = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
+    uint32_t const  cbFirstPage  = PAGE_SIZE - (GCPhysFirst & PAGE_OFFSET_MASK);
+    uint32_t const  cbSecondPage = (uint32_t)(cbMem - cbFirstPage);
+
+    if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))
+    {
+        if (!pVCpu->iem.s.fBypassHandlers)
+        {
+            /*
+             * Must carefully deal with access handler status codes here,
+             * makes the code a bit bloated.
+             */
+            rcStrict = PGMPhysRead(pVM, GCPhysFirst, pbBuf, cbFirstPage, PGMACCESSORIGIN_IEM);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                rcStrict = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
+                if (rcStrict == VINF_SUCCESS)
+                { /*likely */ }
+                else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+                    rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                else
+                {
+                    Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (!!)\n",
+                         GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict) ));
+                    return rcStrict;
+                }
+            }
+            else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+            {
+                VBOXSTRICTRC rcStrict2 = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
+                if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
+                {
+                    PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
+                    rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                }
+                else
+                {
+                    Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (rcStrict=%Rrc) (!!)\n",
+                         GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict2) ));
+                    return rcStrict2;
+                }
+            }
+            else
+            {
+                Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
+                     GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
+                return rcStrict;
+            }
+        }
+        else
+        {
+            /*
+             * No informational status codes here, much more straight forward.
+             */
+            int rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf, GCPhysFirst, cbFirstPage);
+            if (RT_SUCCESS(rc))
+            {
+                Assert(rc == VINF_SUCCESS);
+                rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
+                if (RT_SUCCESS(rc))
+                    Assert(rc == VINF_SUCCESS);
+                else
+                {
+                    Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));
+                    return rc;
+                }
+            }
+            else
+            {
+                Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));
+                return rc;
+            }
+        }
+    }
+#ifdef VBOX_STRICT
+    else
+        memset(pbBuf, 0xcc, cbMem);
+    if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
+        memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
+#endif
+
+    /*
+     * Commit the bounce buffer entry.
+     */
+    pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst    = GCPhysFirst;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond   = GCPhysSecond;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst        = (uint16_t)cbFirstPage;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond       = (uint16_t)cbSecondPage;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned    = false;
+    pVCpu->iem.s.aMemMappings[iMemMap].pv               = pbBuf;
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess          = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;
+    pVCpu->iem.s.iNextMapping = iMemMap + 1;
+    pVCpu->iem.s.cActiveMappings++;
+
+    iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
+    *ppvMem = pbBuf;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * iemMemMap woker that deals with iemMemPageMap failures.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
+                                                  RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
+{
+    /*
+     * Filter out conditions we can handle and the ones which shouldn't happen.
+     */
+    if (   rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE
+        && rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
+        && rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
+    {
+        AssertReturn(RT_FAILURE_NP(rcMap), VERR_IEM_IPE_8);
+        return rcMap;
+    }
+    pVCpu->iem.s.cPotentialExits++;
+
+    /*
+     * Read in the current memory content if it's a read, execute or partial
+     * write access.
+     */
+    uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
+    if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))
+    {
+        if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
+            memset(pbBuf, 0xff, cbMem);
+        else
+        {
+            int rc;
+            if (!pVCpu->iem.s.fBypassHandlers)
+            {
+                VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhysFirst, pbBuf, cbMem, PGMACCESSORIGIN_IEM);
+                if (rcStrict == VINF_SUCCESS)
+                { /* nothing */ }
+                else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+                    rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                else
+                {
+                    Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
+                         GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
+                    return rcStrict;
+                }
+            }
+            else
+            {
+                rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbBuf, GCPhysFirst, cbMem);
+                if (RT_SUCCESS(rc))
+                { /* likely */ }
+                else
+                {
+                    Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
+                         GCPhysFirst, rc));
+                    return rc;
+                }
+            }
+        }
+    }
+#ifdef VBOX_STRICT
+    else
+        memset(pbBuf, 0xcc, cbMem);
+#endif
+#ifdef VBOX_STRICT
+    if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
+        memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
+#endif
+
+    /*
+     * Commit the bounce buffer entry.
+     */
+    pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst    = GCPhysFirst;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond   = NIL_RTGCPHYS;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst        = (uint16_t)cbMem;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond       = 0;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned    = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
+    pVCpu->iem.s.aMemMappings[iMemMap].pv               = pbBuf;
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess          = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;
+    pVCpu->iem.s.iNextMapping = iMemMap + 1;
+    pVCpu->iem.s.cActiveMappings++;
+
+    iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
+    *ppvMem = pbBuf;
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Maps the specified guest memory for the given kind of access.
+ *
+ * This may be using bounce buffering of the memory if it's crossing a page
+ * boundary or if there is an access handler installed for any of it.  Because
+ * of lock prefix guarantees, we're in for some extra clutter when this
+ * happens.
+ *
+ * This may raise a \#GP, \#SS, \#PF or \#AC.
+ *
+ * @returns VBox strict status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   ppvMem              Where to return the pointer to the mapped
+ *                              memory.
+ * @param   cbMem               The number of bytes to map.  This is usually 1,
+ *                              2, 4, 6, 8, 12, 16, 32 or 512.  When used by
+ *                              string operations it can be up to a page.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ *                              Use UINT8_MAX to indicate that no segmentation
+ *                              is required (for IDT, GDT and LDT accesses).
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   fAccess             How the memory is being accessed.  The
+ *                              IEM_ACCESS_TYPE_XXX bit is used to figure out
+ *                              how to map the memory, while the
+ *                              IEM_ACCESS_WHAT_XXX bit is used when raising
+ *                              exceptions.
+ */
+IEM_STATIC VBOXSTRICTRC
+iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess)
+{
+    /*
+     * Check the input and figure out which mapping entry to use.
+     */
+    Assert(cbMem <= 64 || cbMem == 512 || cbMem == 256 || cbMem == 108 || cbMem == 104 || cbMem == 102 || cbMem == 94); /* 512 is the max! */
+    Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
+    Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
+
+    unsigned iMemMap = pVCpu->iem.s.iNextMapping;
+    if (   iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
+        || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
+    {
+        iMemMap = iemMemMapFindFree(pVCpu);
+        AssertLogRelMsgReturn(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
+                              ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
+                               pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
+                               pVCpu->iem.s.aMemMappings[2].fAccess),
+                              VERR_IEM_IPE_9);
+    }
+
+    /*
+     * Map the memory, checking that we can actually access it.  If something
+     * slightly complicated happens, fall back on bounce buffering.
+     */
+    VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    if ((GCPtrMem & PAGE_OFFSET_MASK) + cbMem > PAGE_SIZE) /* Crossing a page boundary? */
+        return iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
+
+    RTGCPHYS GCPhysFirst;
+    rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+        Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
+    if (fAccess & IEM_ACCESS_TYPE_READ)
+        Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
+
+    void *pvMem;
+    rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+    if (rcStrict != VINF_SUCCESS)
+        return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
+
+    /*
+     * Fill in the mapping table entry.
+     */
+    pVCpu->iem.s.aMemMappings[iMemMap].pv      = pvMem;
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
+    pVCpu->iem.s.iNextMapping = iMemMap + 1;
+    pVCpu->iem.s.cActiveMappings++;
+
+    iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
+    *ppvMem = pvMem;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Commits the guest memory if bounce buffered and unmaps it.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pvMem               The mapping.
+ * @param   fAccess             The kind of access.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
+{
+    int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
+    AssertReturn(iMemMap >= 0, iMemMap);
+
+    /* If it's bounce buffered, we may need to write back the buffer. */
+    if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
+    {
+        if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
+            return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);
+    }
+    /* Otherwise unlock it. */
+    else
+        PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+
+    /* Free the entry. */
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+    Assert(pVCpu->iem.s.cActiveMappings != 0);
+    pVCpu->iem.s.cActiveMappings--;
+    return VINF_SUCCESS;
+}
+
+#ifdef IEM_WITH_SETJMP
+
+/**
+ * Maps the specified guest memory for the given kind of access, longjmp on
+ * error.
+ *
+ * This may be using bounce buffering of the memory if it's crossing a page
+ * boundary or if there is an access handler installed for any of it.  Because
+ * of lock prefix guarantees, we're in for some extra clutter when this
+ * happens.
+ *
+ * This may raise a \#GP, \#SS, \#PF or \#AC.
+ *
+ * @returns Pointer to the mapped memory.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbMem               The number of bytes to map.  This is usually 1,
+ *                              2, 4, 6, 8, 12, 16, 32 or 512.  When used by
+ *                              string operations it can be up to a page.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ *                              Use UINT8_MAX to indicate that no segmentation
+ *                              is required (for IDT, GDT and LDT accesses).
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   fAccess             How the memory is being accessed.  The
+ *                              IEM_ACCESS_TYPE_XXX bit is used to figure out
+ *                              how to map the memory, while the
+ *                              IEM_ACCESS_WHAT_XXX bit is used when raising
+ *                              exceptions.
+ */
+IEM_STATIC void *iemMemMapJmp(PVMCPUCC pVCpu, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess)
+{
+    /*
+     * Check the input and figure out which mapping entry to use.
+     */
+    Assert(cbMem <= 64 || cbMem == 512 || cbMem == 108 || cbMem == 104 || cbMem == 94); /* 512 is the max! */
+    Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
+    Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
+
+    unsigned iMemMap = pVCpu->iem.s.iNextMapping;
+    if (   iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
+        || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
+    {
+        iMemMap = iemMemMapFindFree(pVCpu);
+        AssertLogRelMsgStmt(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
+                            ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
+                             pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
+                             pVCpu->iem.s.aMemMappings[2].fAccess),
+                            longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_9));
+    }
+
+    /*
+     * Map the memory, checking that we can actually access it.  If something
+     * slightly complicated happens, fall back on bounce buffering.
+     */
+    VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
+    if (rcStrict == VINF_SUCCESS) { /*likely*/ }
+    else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+
+    /* Crossing a page boundary? */
+    if ((GCPtrMem & PAGE_OFFSET_MASK) + cbMem <= PAGE_SIZE)
+    { /* No (likely). */ }
+    else
+    {
+        void *pvMem;
+        rcStrict = iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, &pvMem, cbMem, GCPtrMem, fAccess);
+        if (rcStrict == VINF_SUCCESS)
+            return pvMem;
+        longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+    }
+
+    RTGCPHYS GCPhysFirst;
+    rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
+    if (rcStrict == VINF_SUCCESS) { /*likely*/ }
+    else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+        Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
+    if (fAccess & IEM_ACCESS_TYPE_READ)
+        Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
+
+    void *pvMem;
+    rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+    if (rcStrict == VINF_SUCCESS)
+    { /* likely */ }
+    else
+    {
+        rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, &pvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
+        if (rcStrict == VINF_SUCCESS)
+            return pvMem;
+        longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+    }
+
+    /*
+     * Fill in the mapping table entry.
+     */
+    pVCpu->iem.s.aMemMappings[iMemMap].pv      = pvMem;
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
+    pVCpu->iem.s.iNextMapping = iMemMap + 1;
+    pVCpu->iem.s.cActiveMappings++;
+
+    iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
+    return pvMem;
+}
+
+
+/**
+ * Commits the guest memory if bounce buffered and unmaps it, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pvMem               The mapping.
+ * @param   fAccess             The kind of access.
+ */
+IEM_STATIC void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
+{
+    int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
+    AssertStmt(iMemMap >= 0, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), iMemMap));
+
+    /* If it's bounce buffered, we may need to write back the buffer. */
+    if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
+    {
+        if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
+        {
+            VBOXSTRICTRC rcStrict = iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);
+            if (rcStrict == VINF_SUCCESS)
+                return;
+            longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+        }
+    }
+    /* Otherwise unlock it. */
+    else
+        PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+
+    /* Free the entry. */
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+    Assert(pVCpu->iem.s.cActiveMappings != 0);
+    pVCpu->iem.s.cActiveMappings--;
+}
+
+#endif /* IEM_WITH_SETJMP */
+
+#ifndef IN_RING3
+/**
+ * Commits the guest memory if bounce buffered and unmaps it, if any bounce
+ * buffer part shows trouble it will be postponed to ring-3 (sets FF and stuff).
+ *
+ * Allows the instruction to be completed and retired, while the IEM user will
+ * return to ring-3 immediately afterwards and do the postponed writes there.
+ *
+ * @returns VBox status code (no strict statuses).  Caller must check
+ *          VMCPU_FF_IEM before repeating string instructions and similar stuff.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pvMem               The mapping.
+ * @param   fAccess             The kind of access.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
+{
+    int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
+    AssertReturn(iMemMap >= 0, iMemMap);
+
+    /* If it's bounce buffered, we may need to write back the buffer. */
+    if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
+    {
+        if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
+            return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, true /*fPostponeFail*/);
+    }
+    /* Otherwise unlock it. */
+    else
+        PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+
+    /* Free the entry. */
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+    Assert(pVCpu->iem.s.cActiveMappings != 0);
+    pVCpu->iem.s.cActiveMappings--;
+    return VINF_SUCCESS;
+}
+#endif
+
+
+/**
+ * Rollbacks mappings, releasing page locks and such.
+ *
+ * The caller shall only call this after checking cActiveMappings.
+ *
+ * @returns Strict VBox status code to pass up.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ */
+IEM_STATIC void iemMemRollback(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu->iem.s.cActiveMappings > 0);
+
+    uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
+    while (iMemMap-- > 0)
+    {
+        uint32_t const fAccess = pVCpu->iem.s.aMemMappings[iMemMap].fAccess;
+        if (fAccess != IEM_ACCESS_INVALID)
+        {
+            AssertMsg(!(fAccess & ~IEM_ACCESS_VALID_MASK) && fAccess != 0, ("%#x\n", fAccess));
+            pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+            if (!(fAccess & IEM_ACCESS_BOUNCE_BUFFERED))
+                PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+            AssertMsg(pVCpu->iem.s.cActiveMappings > 0,
+                      ("iMemMap=%u fAccess=%#x pv=%p GCPhysFirst=%RGp GCPhysSecond=%RGp\n",
+                       iMemMap, fAccess, pVCpu->iem.s.aMemMappings[iMemMap].pv,
+                       pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond));
+            pVCpu->iem.s.cActiveMappings--;
+        }
+    }
+}
+
+
+/**
+ * Fetches a data byte.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu8Dst              Where to return the byte.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    uint8_t const *pu8Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Src, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu8Dst = *pu8Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data byte, longjmp on error.
+ *
+ * @returns The byte.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+DECL_NO_INLINE(IEM_STATIC, uint8_t) iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    uint8_t const *pu8Src = (uint8_t const *)iemMemMapJmp(pVCpu, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    uint8_t const  bRet   = *pu8Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
+    return bRet;
+}
+#endif /* IEM_WITH_SETJMP */
+
+
+/**
+ * Fetches a data word.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16Dst             Where to return the word.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    uint16_t const *pu16Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Dst = *pu16Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data word, longjmp on error.
+ *
+ * @returns The word
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+DECL_NO_INLINE(IEM_STATIC, uint16_t) iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    uint16_t const *pu16Src = (uint16_t const *)iemMemMapJmp(pVCpu, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    uint16_t const u16Ret = *pu16Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
+    return u16Ret;
+}
+#endif
+
+
+/**
+ * Fetches a data dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32Dst             Where to return the dword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    uint32_t const *pu32Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Dst = *pu32Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+/**
+ * Fetches a data dword and zero extends it to a qword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Dst             Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    uint32_t const *pu32Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = *pu32Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+
+IEM_STATIC RTGCPTR iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
+{
+    Assert(cbMem >= 1);
+    Assert(iSegReg < X86_SREG_COUNT);
+
+    /*
+     * 64-bit mode is simpler.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        if (iSegReg >= X86_SREG_FS)
+        {
+            IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+            PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
+            GCPtrMem += pSel->u64Base;
+        }
+
+        if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
+            return GCPtrMem;
+    }
+    /*
+     * 16-bit and 32-bit segmentation.
+     */
+    else
+    {
+        IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+        PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
+        if (      (pSel->Attr.u & (X86DESCATTR_P | X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN))
+               == X86DESCATTR_P /* data, expand up */
+            ||    (pSel->Attr.u & (X86DESCATTR_P | X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ))
+               == (X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ) /* code, read-only */ )
+        {
+            /* expand up */
+            uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
+            if (RT_LIKELY(   GCPtrLast32 > pSel->u32Limit
+                          && GCPtrLast32 > (uint32_t)GCPtrMem))
+                return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
+        }
+        else if (   (pSel->Attr.u & (X86DESCATTR_P | X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN))
+                 == (X86DESCATTR_P | X86_SEL_TYPE_DOWN) /* data, expand down */ )
+        {
+            /* expand down */
+            uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
+            if (RT_LIKELY(   (uint32_t)GCPtrMem >  pSel->u32Limit
+                          && GCPtrLast32        <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
+                          && GCPtrLast32 > (uint32_t)GCPtrMem))
+                return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
+        }
+        else
+            iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
+        iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
+    }
+    iemRaiseGeneralProtectionFault0Jmp(pVCpu);
+}
+
+
+IEM_STATIC RTGCPTR iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
+{
+    Assert(cbMem >= 1);
+    Assert(iSegReg < X86_SREG_COUNT);
+
+    /*
+     * 64-bit mode is simpler.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        if (iSegReg >= X86_SREG_FS)
+        {
+            IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+            PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
+            GCPtrMem += pSel->u64Base;
+        }
+
+        if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
+            return GCPtrMem;
+    }
+    /*
+     * 16-bit and 32-bit segmentation.
+     */
+    else
+    {
+        IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+        PCPUMSELREGHID pSel           = iemSRegGetHid(pVCpu, iSegReg);
+        uint32_t const fRelevantAttrs = pSel->Attr.u & (  X86DESCATTR_P     | X86DESCATTR_UNUSABLE
+                                                        | X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN);
+        if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE)) /* data, expand up */
+        {
+            /* expand up */
+            uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
+            if (RT_LIKELY(   GCPtrLast32 > pSel->u32Limit
+                          && GCPtrLast32 > (uint32_t)GCPtrMem))
+                return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
+        }
+        else if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN)) /* data, expand up */
+        {
+            /* expand down */
+            uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
+            if (RT_LIKELY(   (uint32_t)GCPtrMem >  pSel->u32Limit
+                          && GCPtrLast32        <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
+                          && GCPtrLast32 > (uint32_t)GCPtrMem))
+                return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
+        }
+        else
+            iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
+        iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
+    }
+    iemRaiseGeneralProtectionFault0Jmp(pVCpu);
+}
+
+
+/**
+ * Fetches a data dword, longjmp on error, fallback/safe version.
+ *
+ * @returns The dword
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC uint32_t iemMemFetchDataU32SafeJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    uint32_t const  u32Ret  = *pu32Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
+    return u32Ret;
+}
+
+
+/**
+ * Fetches a data dword, longjmp on error.
+ *
+ * @returns The dword
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+DECL_NO_INLINE(IEM_STATIC, uint32_t) iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+# ifdef IEM_WITH_DATA_TLB
+    RTGCPTR GCPtrEff = iemMemApplySegmentToReadJmp(pVCpu, iSegReg, sizeof(uint32_t), GCPtrMem);
+    if (RT_LIKELY((GCPtrEff & X86_PAGE_OFFSET_MASK) <= X86_PAGE_SIZE - sizeof(uint32_t)))
+    {
+        /// @todo more later.
+    }
+
+    return iemMemFetchDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem);
+# else
+    /* The lazy approach. */
+    uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    uint32_t const  u32Ret  = *pu32Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
+    return u32Ret;
+# endif
+}
+#endif
+
+
+#ifdef SOME_UNUSED_FUNCTION
+/**
+ * Fetches a data dword and sign extends it to a qword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Dst             Where to return the sign extended value.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    int32_t const *pi32Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pi32Src, sizeof(*pi32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = *pi32Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
+    }
+#ifdef __GNUC__ /* warning: GCC may be a royal pain */
+    else
+        *pu64Dst = 0;
+#endif
+    return rc;
+}
+#endif
+
+
+/**
+ * Fetches a data qword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Dst             Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    uint64_t const *pu64Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = *pu64Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data qword, longjmp on error.
+ *
+ * @returns The qword.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+DECL_NO_INLINE(IEM_STATIC, uint64_t) iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    uint64_t const u64Ret = *pu64Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
+    return u64Ret;
+}
+#endif
+
+
+/**
+ * Fetches a data qword, aligned at a 16 byte boundrary (for SSE).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Dst             Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
+    if (RT_UNLIKELY(GCPtrMem & 15))
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    uint64_t const *pu64Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = *pu64Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data qword, longjmp on error.
+ *
+ * @returns The qword.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+DECL_NO_INLINE(IEM_STATIC, uint64_t) iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
+    if (RT_LIKELY(!(GCPtrMem & 15)))
+    {
+        uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+        uint64_t const u64Ret = *pu64Src;
+        iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
+        return u64Ret;
+    }
+
+    VBOXSTRICTRC rc = iemRaiseGeneralProtectionFault0(pVCpu);
+    longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rc));
+}
+#endif
+
+
+/**
+ * Fetches a data tword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pr80Dst             Where to return the tword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    PCRTFLOAT80U pr80Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pr80Src, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pr80Dst = *pr80Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data tword, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pr80Dst             Where to return the tword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+DECL_NO_INLINE(IEM_STATIC, void) iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    PCRTFLOAT80U pr80Src = (PCRTFLOAT80U)iemMemMapJmp(pVCpu, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    *pr80Dst = *pr80Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
+}
+#endif
+
+
+/**
+ * Fetches a data dqword (double qword), generally SSE related.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu128Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    PCRTUINT128U pu128Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    if (rc == VINF_SUCCESS)
+    {
+        pu128Dst->au64[0] = pu128Src->au64[0];
+        pu128Dst->au64[1] = pu128Src->au64[1];
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data dqword (double qword), generally SSE related.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu128Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    pu128Dst->au64[0] = pu128Src->au64[0];
+    pu128Dst->au64[1] = pu128Src->au64[1];
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
+}
+#endif
+
+
+/**
+ * Fetches a data dqword (double qword) at an aligned address, generally SSE
+ * related.
+ *
+ * Raises \#GP(0) if not aligned.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu128Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
+    if (   (GCPtrMem & 15)
+        && !(pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    PCRTUINT128U pu128Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    if (rc == VINF_SUCCESS)
+    {
+        pu128Dst->au64[0] = pu128Src->au64[0];
+        pu128Dst->au64[1] = pu128Src->au64[1];
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data dqword (double qword) at an aligned address, generally SSE
+ * related, longjmp on error.
+ *
+ * Raises \#GP(0) if not aligned.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu128Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+DECL_NO_INLINE(IEM_STATIC, void) iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
+    if (   (GCPtrMem & 15) == 0
+        || (pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
+    {
+        PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+        pu128Dst->au64[0] = pu128Src->au64[0];
+        pu128Dst->au64[1] = pu128Src->au64[1];
+        iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
+        return;
+    }
+
+    VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
+    longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+}
+#endif
+
+
+/**
+ * Fetches a data oword (octo word), generally AVX related.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu256Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    PCRTUINT256U pu256Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    if (rc == VINF_SUCCESS)
+    {
+        pu256Dst->au64[0] = pu256Src->au64[0];
+        pu256Dst->au64[1] = pu256Src->au64[1];
+        pu256Dst->au64[2] = pu256Src->au64[2];
+        pu256Dst->au64[3] = pu256Src->au64[3];
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data oword (octo word), generally AVX related.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu256Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    pu256Dst->au64[0] = pu256Src->au64[0];
+    pu256Dst->au64[1] = pu256Src->au64[1];
+    pu256Dst->au64[2] = pu256Src->au64[2];
+    pu256Dst->au64[3] = pu256Src->au64[3];
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
+}
+#endif
+
+
+/**
+ * Fetches a data oword (octo word) at an aligned address, generally AVX
+ * related.
+ *
+ * Raises \#GP(0) if not aligned.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu256Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on AVX stuff. */
+    if (GCPtrMem & 31)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    PCRTUINT256U pu256Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+    if (rc == VINF_SUCCESS)
+    {
+        pu256Dst->au64[0] = pu256Src->au64[0];
+        pu256Dst->au64[1] = pu256Src->au64[1];
+        pu256Dst->au64[2] = pu256Src->au64[2];
+        pu256Dst->au64[3] = pu256Src->au64[3];
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data oword (octo word) at an aligned address, generally AVX
+ * related, longjmp on error.
+ *
+ * Raises \#GP(0) if not aligned.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu256Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+DECL_NO_INLINE(IEM_STATIC, void) iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on AVX stuff. */
+    if ((GCPtrMem & 31) == 0)
+    {
+        PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
+        pu256Dst->au64[0] = pu256Src->au64[0];
+        pu256Dst->au64[1] = pu256Src->au64[1];
+        pu256Dst->au64[2] = pu256Src->au64[2];
+        pu256Dst->au64[3] = pu256Src->au64[3];
+        iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
+        return;
+    }
+
+    VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
+    longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+}
+#endif
+
+
+
+/**
+ * Fetches a descriptor register (lgdt, lidt).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pcbLimit            Where to return the limit.
+ * @param   pGCPtrBase          Where to return the base.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   enmOpSize           The effective operand size.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
+                                            RTGCPTR GCPtrMem, IEMMODE enmOpSize)
+{
+    /*
+     * Just like SIDT and SGDT, the LIDT and LGDT instructions are a
+     * little special:
+     *      - The two reads are done separately.
+     *      - Operand size override works in 16-bit and 32-bit code, but 64-bit.
+     *      - We suspect the 386 to actually commit the limit before the base in
+     *        some cases (search for 386 in  bs3CpuBasic2_lidt_lgdt_One).  We
+     *        don't try emulate this eccentric behavior, because it's not well
+     *        enough understood and rather hard to trigger.
+     *      - The 486 seems to do a dword limit read when the operand size is 32-bit.
+     */
+    VBOXSTRICTRC rcStrict;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemFetchDataU64(pVCpu, pGCPtrBase, iSegReg, GCPtrMem + 2);
+    }
+    else
+    {
+        uint32_t uTmp = 0; /* (Visual C++ maybe used uninitialized) */
+        if (enmOpSize == IEMMODE_32BIT)
+        {
+            if (IEM_GET_TARGET_CPU(pVCpu) != IEMTARGETCPU_486)
+            {
+                rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
+                if (rcStrict == VINF_SUCCESS)
+                    rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
+            }
+            else
+            {
+                rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem);
+                if (rcStrict == VINF_SUCCESS)
+                {
+                    *pcbLimit = (uint16_t)uTmp;
+                    rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
+                }
+            }
+            if (rcStrict == VINF_SUCCESS)
+                *pGCPtrBase = uTmp;
+        }
+        else
+        {
+            rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
+                if (rcStrict == VINF_SUCCESS)
+                    *pGCPtrBase = uTmp & UINT32_C(0x00ffffff);
+            }
+        }
+    }
+    return rcStrict;
+}
+
+
+
+/**
+ * Stores a data byte.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u8Value             The value to store.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value)
+{
+    /* The lazy approach for now... */
+    uint8_t *pu8Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Dst, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu8Dst = u8Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data byte, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u8Value             The value to store.
+ */
+IEM_STATIC void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value)
+{
+    /* The lazy approach for now... */
+    uint8_t *pu8Dst = (uint8_t *)iemMemMapJmp(pVCpu, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    *pu8Dst = u8Value;
+    iemMemCommitAndUnmapJmp(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data word.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u16Value            The value to store.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value)
+{
+    /* The lazy approach for now... */
+    uint16_t *pu16Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Dst = u16Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data word, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u16Value            The value to store.
+ */
+IEM_STATIC void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value)
+{
+    /* The lazy approach for now... */
+    uint16_t *pu16Dst = (uint16_t *)iemMemMapJmp(pVCpu, sizeof(*pu16Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    *pu16Dst = u16Value;
+    iemMemCommitAndUnmapJmp(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u32Value            The value to store.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value)
+{
+    /* The lazy approach for now... */
+    uint32_t *pu32Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Dst = u32Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u32Value            The value to store.
+ */
+IEM_STATIC void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value)
+{
+    /* The lazy approach for now... */
+    uint32_t *pu32Dst = (uint32_t *)iemMemMapJmp(pVCpu, sizeof(*pu32Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    *pu32Dst = u32Value;
+    iemMemCommitAndUnmapJmp(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data qword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u64Value            The value to store.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value)
+{
+    /* The lazy approach for now... */
+    uint64_t *pu64Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = u64Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data qword, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u64Value            The value to store.
+ */
+IEM_STATIC void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value)
+{
+    /* The lazy approach for now... */
+    uint64_t *pu64Dst = (uint64_t *)iemMemMapJmp(pVCpu, sizeof(*pu64Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    *pu64Dst = u64Value;
+    iemMemCommitAndUnmapJmp(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data dqword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u128Value            The value to store.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
+{
+    /* The lazy approach for now... */
+    PRTUINT128U pu128Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    if (rc == VINF_SUCCESS)
+    {
+        pu128Dst->au64[0] = u128Value.au64[0];
+        pu128Dst->au64[1] = u128Value.au64[1];
+        rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data dqword, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u128Value            The value to store.
+ */
+IEM_STATIC void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
+{
+    /* The lazy approach for now... */
+    PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    pu128Dst->au64[0] = u128Value.au64[0];
+    pu128Dst->au64[1] = u128Value.au64[1];
+    iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data dqword, SSE aligned.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u128Value           The value to store.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
+{
+    /* The lazy approach for now... */
+    if (   (GCPtrMem & 15)
+        && !(pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    PRTUINT128U pu128Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    if (rc == VINF_SUCCESS)
+    {
+        pu128Dst->au64[0] = u128Value.au64[0];
+        pu128Dst->au64[1] = u128Value.au64[1];
+        rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data dqword, SSE aligned.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u128Value           The value to store.
+ */
+DECL_NO_INLINE(IEM_STATIC, void)
+iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
+{
+    /* The lazy approach for now... */
+    if (   (GCPtrMem & 15) == 0
+        || (pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
+    {
+        PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+        pu128Dst->au64[0] = u128Value.au64[0];
+        pu128Dst->au64[1] = u128Value.au64[1];
+        iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
+        return;
+    }
+
+    VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
+    longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+}
+#endif
+
+
+/**
+ * Stores a data dqword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   pu256Value          Pointer to the value to store.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
+{
+    /* The lazy approach for now... */
+    PRTUINT256U pu256Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    if (rc == VINF_SUCCESS)
+    {
+        pu256Dst->au64[0] = pu256Value->au64[0];
+        pu256Dst->au64[1] = pu256Value->au64[1];
+        pu256Dst->au64[2] = pu256Value->au64[2];
+        pu256Dst->au64[3] = pu256Value->au64[3];
+        rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data dqword, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   pu256Value          Pointer to the value to store.
+ */
+IEM_STATIC void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
+{
+    /* The lazy approach for now... */
+    PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    pu256Dst->au64[0] = pu256Value->au64[0];
+    pu256Dst->au64[1] = pu256Value->au64[1];
+    pu256Dst->au64[2] = pu256Value->au64[2];
+    pu256Dst->au64[3] = pu256Value->au64[3];
+    iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data dqword, AVX aligned.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   pu256Value          Pointer to the value to store.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
+{
+    /* The lazy approach for now... */
+    if (GCPtrMem & 31)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    PRTUINT256U pu256Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+    if (rc == VINF_SUCCESS)
+    {
+        pu256Dst->au64[0] = pu256Value->au64[0];
+        pu256Dst->au64[1] = pu256Value->au64[1];
+        pu256Dst->au64[2] = pu256Value->au64[2];
+        pu256Dst->au64[3] = pu256Value->au64[3];
+        rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data dqword, AVX aligned.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   pu256Value          Pointer to the value to store.
+ */
+DECL_NO_INLINE(IEM_STATIC, void)
+iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
+{
+    /* The lazy approach for now... */
+    if ((GCPtrMem & 31) == 0)
+    {
+        PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
+        pu256Dst->au64[0] = pu256Value->au64[0];
+        pu256Dst->au64[1] = pu256Value->au64[1];
+        pu256Dst->au64[2] = pu256Value->au64[2];
+        pu256Dst->au64[3] = pu256Value->au64[3];
+        iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
+        return;
+    }
+
+    VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
+    longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
+}
+#endif
+
+
+/**
+ * Stores a descriptor register (sgdt, sidt).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbLimit             The limit.
+ * @param   GCPtrBase           The base address.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC
+iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /*
+     * The SIDT and SGDT instructions actually stores the data using two
+     * independent writes.  The instructions does not respond to opsize prefixes.
+     */
+    VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iSegReg, GCPtrMem, cbLimit);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_16BIT)
+            rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2,
+                                          IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_286
+                                          ? (uint32_t)GCPtrBase | UINT32_C(0xff000000) : (uint32_t)GCPtrBase);
+        else if (pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT)
+            rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2, (uint32_t)GCPtrBase);
+        else
+            rcStrict = iemMemStoreDataU64(pVCpu, iSegReg, GCPtrMem + 2, GCPtrBase);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Pushes a word onto the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16Value            The value to push.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value)
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPush(pVCpu, 2, &uNewRsp);
+
+    /* Write the word the lazy way. */
+    uint16_t *pu16Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Dst = u16Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        pVCpu->cpum.GstCtx.rsp = uNewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pushes a dword onto the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u32Value            The value to push.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value)
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
+
+    /* Write the dword the lazy way. */
+    uint32_t *pu32Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Dst = u32Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        pVCpu->cpum.GstCtx.rsp = uNewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pushes a dword segment register value onto the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u32Value            The value to push.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value)
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
+
+    /* The intel docs talks about zero extending the selector register
+       value.  My actual intel CPU here might be zero extending the value
+       but it still only writes the lower word... */
+    /** @todo Test this on new HW and on AMD and in 64-bit mode.  Also test what
+     * happens when crossing an electric page boundrary, is the high word checked
+     * for write accessibility or not? Probably it is.  What about segment limits?
+     * It appears this behavior is also shared with trap error codes.
+     *
+     * Docs indicate the behavior changed maybe in Pentium or Pentium Pro. Check
+     * ancient hardware when it actually did change. */
+    uint16_t *pu16Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(uint32_t), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_RW);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Dst = (uint16_t)u32Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_RW);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        pVCpu->cpum.GstCtx.rsp = uNewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pushes a qword onto the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u64Value            The value to push.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value)
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPush(pVCpu, 8, &uNewRsp);
+
+    /* Write the word the lazy way. */
+    uint64_t *pu64Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = u64Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        pVCpu->cpum.GstCtx.rsp = uNewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pops a word from the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16Value           Where to store the popped value.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value)
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPop(pVCpu, 2, &uNewRsp);
+
+    /* Write the word the lazy way. */
+    uint16_t const *pu16Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Value = *pu16Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
+
+        /* Commit the new RSP value. */
+        if (rc == VINF_SUCCESS)
+            pVCpu->cpum.GstCtx.rsp = uNewRsp;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Pops a dword from the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32Value           Where to store the popped value.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value)
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPop(pVCpu, 4, &uNewRsp);
+
+    /* Write the word the lazy way. */
+    uint32_t const *pu32Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Value = *pu32Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
+
+        /* Commit the new RSP value. */
+        if (rc == VINF_SUCCESS)
+            pVCpu->cpum.GstCtx.rsp = uNewRsp;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Pops a qword from the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Value           Where to store the popped value.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value)
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPop(pVCpu, 8, &uNewRsp);
+
+    /* Write the word the lazy way. */
+    uint64_t const *pu64Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Value = *pu64Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
+
+        /* Commit the new RSP value. */
+        if (rc == VINF_SUCCESS)
+            pVCpu->cpum.GstCtx.rsp = uNewRsp;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Pushes a word onto the stack, using a temporary stack pointer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16Value            The value to push.
+ * @param   pTmpRsp             Pointer to the temporary stack pointer.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp)
+{
+    /* Increment the stack pointer. */
+    RTUINT64U   NewRsp = *pTmpRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 2);
+
+    /* Write the word the lazy way. */
+    uint16_t *pu16Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Dst = u16Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        *pTmpRsp = NewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pushes a dword onto the stack, using a temporary stack pointer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u32Value            The value to push.
+ * @param   pTmpRsp             Pointer to the temporary stack pointer.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp)
+{
+    /* Increment the stack pointer. */
+    RTUINT64U   NewRsp = *pTmpRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 4);
+
+    /* Write the word the lazy way. */
+    uint32_t *pu32Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Dst = u32Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        *pTmpRsp = NewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pushes a dword onto the stack, using a temporary stack pointer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u64Value            The value to push.
+ * @param   pTmpRsp             Pointer to the temporary stack pointer.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp)
+{
+    /* Increment the stack pointer. */
+    RTUINT64U   NewRsp = *pTmpRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 8);
+
+    /* Write the word the lazy way. */
+    uint64_t *pu64Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = u64Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        *pTmpRsp = NewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pops a word from the stack, using a temporary stack pointer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16Value           Where to store the popped value.
+ * @param   pTmpRsp             Pointer to the temporary stack pointer.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp)
+{
+    /* Increment the stack pointer. */
+    RTUINT64U   NewRsp = *pTmpRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 2);
+
+    /* Write the word the lazy way. */
+    uint16_t const *pu16Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Value = *pu16Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
+
+        /* Commit the new RSP value. */
+        if (rc == VINF_SUCCESS)
+            *pTmpRsp = NewRsp;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Pops a dword from the stack, using a temporary stack pointer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32Value           Where to store the popped value.
+ * @param   pTmpRsp             Pointer to the temporary stack pointer.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp)
+{
+    /* Increment the stack pointer. */
+    RTUINT64U   NewRsp = *pTmpRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 4);
+
+    /* Write the word the lazy way. */
+    uint32_t const *pu32Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Value = *pu32Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
+
+        /* Commit the new RSP value. */
+        if (rc == VINF_SUCCESS)
+            *pTmpRsp = NewRsp;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Pops a qword from the stack, using a temporary stack pointer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Value           Where to store the popped value.
+ * @param   pTmpRsp             Pointer to the temporary stack pointer.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp)
+{
+    /* Increment the stack pointer. */
+    RTUINT64U   NewRsp = *pTmpRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
+
+    /* Write the word the lazy way. */
+    uint64_t const *pu64Src;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        *pu64Value = *pu64Src;
+        rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
+
+        /* Commit the new RSP value. */
+        if (rcStrict == VINF_SUCCESS)
+            *pTmpRsp = NewRsp;
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Begin a special stack push (used by interrupt, exceptions and such).
+ *
+ * This will raise \#SS or \#PF if appropriate.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbMem               The number of bytes to push onto the stack.
+ * @param   ppvMem              Where to return the pointer to the stack memory.
+ *                              As with the other memory functions this could be
+ *                              direct access or bounce buffered access, so
+ *                              don't commit register until the commit call
+ *                              succeeds.
+ * @param   puNewRsp            Where to return the new RSP value.  This must be
+ *                              passed unchanged to
+ *                              iemMemStackPushCommitSpecial().
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, void **ppvMem, uint64_t *puNewRsp)
+{
+    Assert(cbMem < UINT8_MAX);
+    RTGCPTR     GCPtrTop = iemRegGetRspForPush(pVCpu, (uint8_t)cbMem, puNewRsp);
+    return iemMemMap(pVCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
+}
+
+
+/**
+ * Commits a special stack push (started by iemMemStackPushBeginSpecial).
+ *
+ * This will update the rSP.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pvMem               The pointer returned by
+ *                              iemMemStackPushBeginSpecial().
+ * @param   uNewRsp             The new RSP value returned by
+ *                              iemMemStackPushBeginSpecial().
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp)
+{
+    VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem, IEM_ACCESS_STACK_W);
+    if (rcStrict == VINF_SUCCESS)
+        pVCpu->cpum.GstCtx.rsp = uNewRsp;
+    return rcStrict;
+}
+
+
+/**
+ * Begin a special stack pop (used by iret, retf and such).
+ *
+ * This will raise \#SS or \#PF if appropriate.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbMem               The number of bytes to pop from the stack.
+ * @param   ppvMem              Where to return the pointer to the stack memory.
+ * @param   puNewRsp            Where to return the new RSP value.  This must be
+ *                              assigned to CPUMCTX::rsp manually some time
+ *                              after iemMemStackPopDoneSpecial() has been
+ *                              called.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, void const **ppvMem, uint64_t *puNewRsp)
+{
+    Assert(cbMem < UINT8_MAX);
+    RTGCPTR     GCPtrTop = iemRegGetRspForPop(pVCpu, (uint8_t)cbMem, puNewRsp);
+    return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
+}
+
+
+/**
+ * Continue a special stack pop (used by iret and retf).
+ *
+ * This will raise \#SS or \#PF if appropriate.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbMem               The number of bytes to pop from the stack.
+ * @param   ppvMem              Where to return the pointer to the stack memory.
+ * @param   puNewRsp            Where to return the new RSP value.  This must be
+ *                              assigned to CPUMCTX::rsp manually some time
+ *                              after iemMemStackPopDoneSpecial() has been
+ *                              called.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t cbMem, void const **ppvMem, uint64_t *puNewRsp)
+{
+    Assert(cbMem < UINT8_MAX);
+    RTUINT64U   NewRsp;
+    NewRsp.u = *puNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
+    *puNewRsp = NewRsp.u;
+    return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
+}
+
+
+/**
+ * Done with a special stack pop (started by iemMemStackPopBeginSpecial or
+ * iemMemStackPopContinueSpecial).
+ *
+ * The caller will manually commit the rSP.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pvMem               The pointer returned by
+ *                              iemMemStackPopBeginSpecial() or
+ *                              iemMemStackPopContinueSpecial().
+ */
+IEM_STATIC VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem)
+{
+    return iemMemCommitAndUnmap(pVCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
+}
+
+
+/**
+ * Fetches a system table byte.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pbDst               Where to return the byte.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pbDst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    uint8_t const *pbSrc;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pbSrc, sizeof(*pbSrc), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pbDst = *pbSrc;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pbSrc, IEM_ACCESS_SYS_R);
+    }
+    return rc;
+}
+
+
+/**
+ * Fetches a system table word.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16Dst             Where to return the word.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    uint16_t const *pu16Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Dst = *pu16Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_SYS_R);
+    }
+    return rc;
+}
+
+
+/**
+ * Fetches a system table dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32Dst             Where to return the dword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    uint32_t const *pu32Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Dst = *pu32Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
+    }
+    return rc;
+}
+
+
+/**
+ * Fetches a system table qword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Dst             Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
+{
+    /* The lazy approach for now... */
+    uint64_t const *pu64Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = *pu64Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
+    }
+    return rc;
+}
+
+
+/**
+ * Fetches a descriptor table entry with caller specified error code.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pDesc               Where to return the descriptor table entry.
+ * @param   uSel                The selector which table entry to fetch.
+ * @param   uXcpt               The exception to raise on table lookup error.
+ * @param   uErrorCode          The error code associated with the exception.
+ */
+IEM_STATIC VBOXSTRICTRC
+iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt, uint16_t uErrorCode)
+{
+    AssertPtr(pDesc);
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
+
+    /** @todo did the 286 require all 8 bytes to be accessible? */
+    /*
+     * Get the selector table base and check bounds.
+     */
+    RTGCPTR GCPtrBase;
+    if (uSel & X86_SEL_LDT)
+    {
+        if (   !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
+            || (uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
+        {
+            Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
+                 uSel, pVCpu->cpum.GstCtx.ldtr.u32Limit, pVCpu->cpum.GstCtx.ldtr.Sel));
+            return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                                     uErrorCode, 0);
+        }
+
+        Assert(pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present);
+        GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
+    }
+    else
+    {
+        if ((uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
+        {
+            Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pVCpu->cpum.GstCtx.gdtr.cbGdt));
+            return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                                     uErrorCode, 0);
+        }
+        GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
+    }
+
+    /*
+     * Read the legacy descriptor and maybe the long mode extensions if
+     * required.
+     */
+    VBOXSTRICTRC rcStrict;
+    if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_286)
+        rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
+    else
+    {
+        rcStrict     = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[0], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 0);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 2);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[2], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 4);
+        if (rcStrict == VINF_SUCCESS)
+            pDesc->Legacy.au16[3] = 0;
+        else
+            return rcStrict;
+    }
+
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (   !IEM_IS_LONG_MODE(pVCpu)
+            || pDesc->Legacy.Gen.u1DescType)
+            pDesc->Long.au64[1] = 0;
+        else if ((uint32_t)(uSel | X86_SEL_RPL_LDT) + 8 <= (uSel & X86_SEL_LDT ? pVCpu->cpum.GstCtx.ldtr.u32Limit : pVCpu->cpum.GstCtx.gdtr.cbGdt))
+            rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel | X86_SEL_RPL_LDT) + 1);
+        else
+        {
+            Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
+            /** @todo is this the right exception? */
+            return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErrorCode, 0);
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Fetches a descriptor table entry.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pDesc               Where to return the descriptor table entry.
+ * @param   uSel                The selector which table entry to fetch.
+ * @param   uXcpt               The exception to raise on table lookup error.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt)
+{
+    return iemMemFetchSelDescWithErr(pVCpu, pDesc, uSel, uXcpt, uSel & X86_SEL_MASK_OFF_RPL);
+}
+
+
+/**
+ * Fakes a long mode stack selector for SS = 0.
+ *
+ * @param   pDescSs             Where to return the fake stack descriptor.
+ * @param   uDpl                The DPL we want.
+ */
+IEM_STATIC void iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
+{
+    pDescSs->Long.au64[0] = 0;
+    pDescSs->Long.au64[1] = 0;
+    pDescSs->Long.Gen.u4Type     = X86_SEL_TYPE_RW_ACC;
+    pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
+    pDescSs->Long.Gen.u2Dpl      = uDpl;
+    pDescSs->Long.Gen.u1Present  = 1;
+    pDescSs->Long.Gen.u1Long     = 1;
+}
+
+
+/**
+ * Marks the selector descriptor as accessed (only non-system descriptors).
+ *
+ * This function ASSUMES that iemMemFetchSelDesc has be called previously and
+ * will therefore skip the limit checks.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   uSel                The selector.
+ */
+IEM_STATIC VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel)
+{
+    /*
+     * Get the selector table base and calculate the entry address.
+     */
+    RTGCPTR GCPtr = uSel & X86_SEL_LDT
+                  ? pVCpu->cpum.GstCtx.ldtr.u64Base
+                  : pVCpu->cpum.GstCtx.gdtr.pGdt;
+    GCPtr += uSel & X86_SEL_MASK;
+
+    /*
+     * ASMAtomicBitSet will assert if the address is misaligned, so do some
+     * ugly stuff to avoid this.  This will make sure it's an atomic access
+     * as well more or less remove any question about 8-bit or 32-bit accesss.
+     */
+    VBOXSTRICTRC        rcStrict;
+    uint32_t volatile  *pu32;
+    if ((GCPtr & 3) == 0)
+    {
+        /* The normal case, map the 32-bit bits around the accessed bit (40). */
+        GCPtr += 2 + 2;
+        rcStrict = iemMemMap(pVCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
+    }
+    else
+    {
+        /* The misaligned GDT/LDT case, map the whole thing. */
+        rcStrict = iemMemMap(pVCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        switch ((uintptr_t)pu32 & 3)
+        {
+            case 0: ASMAtomicBitSet(pu32,                         40 + 0 -  0); break;
+            case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
+            case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
+            case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 -  8); break;
+        }
+    }
+
+    return iemMemCommitAndUnmap(pVCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
+}
+
+/** @} */
+
+
+/*
+ * Include the C/C++ implementation of instruction.
+ */
+#include "IEMAllCImpl.cpp.h"
+
+
+
+/** @name   "Microcode" macros.
+ *
+ * The idea is that we should be able to use the same code to interpret
+ * instructions as well as recompiler instructions.  Thus this obfuscation.
+ *
+ * @{
+ */
+#define IEM_MC_BEGIN(a_cArgs, a_cLocals)                {
+#define IEM_MC_END()                                    }
+#define IEM_MC_PAUSE()                                  do {} while (0)
+#define IEM_MC_CONTINUE()                               do {} while (0)
+
+/** Internal macro. */
+#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrict2 = a_Expr; \
+        if (rcStrict2 != VINF_SUCCESS) \
+            return rcStrict2; \
+    } while (0)
+
+
+#define IEM_MC_ADVANCE_RIP()                            iemRegUpdateRipAndClearRF(pVCpu)
+#define IEM_MC_REL_JMP_S8(a_i8)                         IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS8(pVCpu, a_i8))
+#define IEM_MC_REL_JMP_S16(a_i16)                       IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS16(pVCpu, a_i16))
+#define IEM_MC_REL_JMP_S32(a_i32)                       IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS32(pVCpu, a_i32))
+#define IEM_MC_SET_RIP_U16(a_u16NewIP)                  IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u16NewIP)))
+#define IEM_MC_SET_RIP_U32(a_u32NewIP)                  IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u32NewIP)))
+#define IEM_MC_SET_RIP_U64(a_u64NewIP)                  IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u64NewIP)))
+#define IEM_MC_RAISE_DIVIDE_ERROR()                     return iemRaiseDivideError(pVCpu)
+#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE()       \
+    do { \
+        if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS)) \
+            return iemRaiseDeviceNotAvailable(pVCpu); \
+    } while (0)
+#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE()  \
+    do { \
+        if ((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP | X86_CR0_TS)) == (X86_CR0_MP | X86_CR0_TS)) \
+            return iemRaiseDeviceNotAvailable(pVCpu); \
+    } while (0)
+#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
+    do { \
+        if (pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FSW & X86_FSW_ES) \
+            return iemRaiseMathFault(pVCpu); \
+    } while (0)
+#define IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT() \
+    do { \
+        if (   (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) != (XSAVE_C_YMM | XSAVE_C_SSE) \
+            || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
+            || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx2) \
+            return iemRaiseUndefinedOpcode(pVCpu); \
+        if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
+            return iemRaiseDeviceNotAvailable(pVCpu); \
+    } while (0)
+#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
+    do { \
+        if (   (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) != (XSAVE_C_YMM | XSAVE_C_SSE) \
+            || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
+            || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx) \
+            return iemRaiseUndefinedOpcode(pVCpu); \
+        if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
+            return iemRaiseDeviceNotAvailable(pVCpu); \
+    } while (0)
+#define IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT() \
+    do { \
+        if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
+            || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
+            || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse41) \
+            return iemRaiseUndefinedOpcode(pVCpu); \
+        if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
+            return iemRaiseDeviceNotAvailable(pVCpu); \
+    } while (0)
+#define IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT() \
+    do { \
+        if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
+            || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
+            || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse3) \
+            return iemRaiseUndefinedOpcode(pVCpu); \
+        if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
+            return iemRaiseDeviceNotAvailable(pVCpu); \
+    } while (0)
+#define IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT() \
+    do { \
+        if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
+            || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
+            || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2) \
+            return iemRaiseUndefinedOpcode(pVCpu); \
+        if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
+            return iemRaiseDeviceNotAvailable(pVCpu); \
+    } while (0)
+#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
+    do { \
+        if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
+            || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
+            || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse) \
+            return iemRaiseUndefinedOpcode(pVCpu); \
+        if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
+            return iemRaiseDeviceNotAvailable(pVCpu); \
+    } while (0)
+#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
+    do { \
+        if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
+            || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMmx) \
+            return iemRaiseUndefinedOpcode(pVCpu); \
+        if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
+            return iemRaiseDeviceNotAvailable(pVCpu); \
+    } while (0)
+#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT() \
+    do { \
+        if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
+            || (   !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse \
+                && !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAmdMmxExts) ) \
+            return iemRaiseUndefinedOpcode(pVCpu); \
+        if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
+            return iemRaiseDeviceNotAvailable(pVCpu); \
+    } while (0)
+#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
+    do { \
+        if (pVCpu->iem.s.uCpl != 0) \
+            return iemRaiseGeneralProtectionFault0(pVCpu); \
+    } while (0)
+#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
+    do { \
+        if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
+        else return iemRaiseGeneralProtectionFault0(pVCpu); \
+    } while (0)
+#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
+    do { \
+        if (   pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT \
+            || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fFsGsBase \
+            || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE)) \
+            return iemRaiseUndefinedOpcode(pVCpu); \
+    } while (0)
+#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
+    do { \
+        if (!IEM_IS_CANONICAL(a_u64Addr)) \
+            return iemRaiseGeneralProtectionFault0(pVCpu); \
+    } while (0)
+
+
+#define IEM_MC_LOCAL(a_Type, a_Name)                    a_Type a_Name
+#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value)     a_Type const a_Name = (a_Value)
+#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local)            (a_pRefArg) = &(a_Local)
+#define IEM_MC_ARG(a_Type, a_Name, a_iArg)              a_Type a_Name
+#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg)       a_Type const a_Name = (a_Value)
+#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg)   a_Type const a_Name = &(a_Local)
+#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
+    uint32_t a_Name; \
+    uint32_t *a_pName = &a_Name
+#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
+   do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
+
+#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst)   (a_VarOrArg) = (a_CVariableOrConst)
+#define IEM_MC_ASSIGN_TO_SMALLER                        IEM_MC_ASSIGN
+
+#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg)          (a_u8Dst)  = iemGRegFetchU8(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg)  (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg)  (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg)  (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg)  (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg)  (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg)  (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg)        (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg)        (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg)        (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
+#define IEM_MC_FETCH_GREG_U64_ZX_U64                    IEM_MC_FETCH_GREG_U64
+#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
+        IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
+        (a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
+    } while (0)
+#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
+        IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
+        (a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
+    } while (0)
+#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
+        IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
+        (a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
+    } while (0)
+/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
+#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
+        IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
+        (a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
+    } while (0)
+#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
+        IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
+        (a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
+    } while (0)
+/** @note Not for IOPL or IF testing or modification. */
+#define IEM_MC_FETCH_EFLAGS(a_EFlags)                   (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
+#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags)                (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u
+#define IEM_MC_FETCH_FSW(a_u16Fsw)                      (a_u16Fsw) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FSW
+#define IEM_MC_FETCH_FCW(a_u16Fcw)                      (a_u16Fcw) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FCW
+
+#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value)        *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
+#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value)      *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
+#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value)      *iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) /* clear high bits. */
+#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value)      *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
+#define IEM_MC_STORE_GREG_U8_CONST                      IEM_MC_STORE_GREG_U8
+#define IEM_MC_STORE_GREG_U16_CONST                     IEM_MC_STORE_GREG_U16
+#define IEM_MC_STORE_GREG_U32_CONST                     IEM_MC_STORE_GREG_U32
+#define IEM_MC_STORE_GREG_U64_CONST                     IEM_MC_STORE_GREG_U64
+#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg)             *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
+#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst)    do { (a_pu32Dst)[1] = 0; } while (0)
+/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
+#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
+        IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
+        *iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
+    } while (0)
+#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
+        IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
+        *iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); /* clear high bits. */ \
+    } while (0)
+#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
+    do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
+
+
+#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg)           (a_pu8Dst)  = iemGRegRefU8( pVCpu, (a_iGReg))
+#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg)         (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
+/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
+ *        Use IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF! */
+#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg)         (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
+#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg)         (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
+/** @note Not for IOPL or IF testing or modification. */
+#define IEM_MC_REF_EFLAGS(a_pEFlags)                    (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.u
+
+#define IEM_MC_ADD_GREG_U8(a_iGReg, a_u8Value)          *iemGRegRefU8( pVCpu, (a_iGReg)) += (a_u8Value)
+#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value)        *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
+#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
+    do { \
+        uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
+        *pu32Reg += (a_u32Value); \
+        pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
+    } while (0)
+#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value)        *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
+
+#define IEM_MC_SUB_GREG_U8(a_iGReg,  a_u8Value)         *iemGRegRefU8( pVCpu, (a_iGReg)) -= (a_u8Value)
+#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u16Value)        *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u16Value)
+#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u32Value) \
+    do { \
+        uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
+        *pu32Reg -= (a_u32Value); \
+        pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
+    } while (0)
+#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u64Value)        *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u64Value)
+#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const)   do { (a_u16Value) -= a_u16Const; } while (0)
+
+#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg)    do { (a_u8Value)  += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
+#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg)  do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
+#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg)  do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
+#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg)  do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
+#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
+#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
+#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
+
+#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask)        do { (a_u8Local)  &= (a_u8Mask);  } while (0)
+#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask)     do { (a_u16Local) &= (a_u16Mask); } while (0)
+#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask)     do { (a_u32Local) &= (a_u32Mask); } while (0)
+#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask)     do { (a_u64Local) &= (a_u64Mask); } while (0)
+
+#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask)         do { (a_u16Arg) &= (a_u16Mask); } while (0)
+#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask)         do { (a_u32Arg) &= (a_u32Mask); } while (0)
+#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask)         do { (a_u64Arg) &= (a_u64Mask); } while (0)
+
+#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask)         do { (a_u8Local)  |= (a_u8Mask);  } while (0)
+#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask)      do { (a_u16Local) |= (a_u16Mask); } while (0)
+#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask)      do { (a_u32Local) |= (a_u32Mask); } while (0)
+
+#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift)      do { (a_i16Local) >>= (a_cShift);  } while (0)
+#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift)      do { (a_i32Local) >>= (a_cShift);  } while (0)
+#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift)      do { (a_i64Local) >>= (a_cShift);  } while (0)
+
+#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift)      do { (a_i16Local) <<= (a_cShift);  } while (0)
+#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift)      do { (a_i32Local) <<= (a_cShift);  } while (0)
+#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift)      do { (a_i64Local) <<= (a_cShift);  } while (0)
+
+#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask)     do { (a_u32Local) &= (a_u32Mask); } while (0)
+
+#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask)      do { (a_u32Local) |= (a_u32Mask); } while (0)
+
+#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value)          *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
+#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value)        *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
+#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
+    do { \
+        uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
+        *pu32Reg &= (a_u32Value); \
+        pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
+    } while (0)
+#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value)        *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
+
+#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value)           *iemGRegRefU8( pVCpu, (a_iGReg)) |= (a_u8Value)
+#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value)         *iemGRegRefU16(pVCpu, (a_iGReg)) |= (a_u16Value)
+#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
+    do { \
+        uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
+        *pu32Reg |= (a_u32Value); \
+        pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
+    } while (0)
+#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value)         *iemGRegRefU64(pVCpu, (a_iGReg)) |= (a_u64Value)
+
+
+/** @note Not for IOPL or IF modification. */
+#define IEM_MC_SET_EFL_BIT(a_fBit)                      do { pVCpu->cpum.GstCtx.eflags.u |= (a_fBit); } while (0)
+/** @note Not for IOPL or IF modification. */
+#define IEM_MC_CLEAR_EFL_BIT(a_fBit)                    do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
+/** @note Not for IOPL or IF modification. */
+#define IEM_MC_FLIP_EFL_BIT(a_fBit)                     do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
+
+#define IEM_MC_CLEAR_FSW_EX()   do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FSW &= X86_FSW_C_MASK | X86_FSW_TOP_MASK; } while (0)
+
+/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
+#define IEM_MC_FPU_TO_MMX_MODE() do { \
+        pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FSW &= ~X86_FSW_TOP_MASK; \
+        pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FTW  = 0xff; \
+    } while (0)
+
+/** Switches the FPU state from MMX mode (FTW=0xffff). */
+#define IEM_MC_FPU_FROM_MMX_MODE() do { \
+        pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FTW  = 0; \
+    } while (0)
+
+#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
+    do { (a_u64Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx; } while (0)
+#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg) \
+    do { (a_u32Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].au32[0]; } while (0)
+#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) do { \
+        pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
+        pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
+    } while (0)
+#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) do { \
+        pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
+        pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
+    } while (0)
+#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
+        (a_pu64Dst) = (&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx)
+#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
+        (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx)
+#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
+        (a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx)
+
+#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
+    do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0]; \
+         (a_u128Value).au64[1] = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1]; \
+    } while (0)
+#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg) \
+    do { (a_u64Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0]; } while (0)
+#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg) \
+    do { (a_u32Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au32[0]; } while (0)
+#define IEM_MC_FETCH_XREG_HI_U64(a_u64Value, a_iXReg) \
+    do { (a_u64Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1]; } while (0)
+#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
+    do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
+         pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
+    } while (0)
+#define IEM_MC_STORE_XREG_U64(a_iXReg, a_u64Value) \
+    do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); } while (0)
+#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
+    do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
+         pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1] = 0; \
+    } while (0)
+#define IEM_MC_STORE_XREG_U32(a_iXReg, a_u32Value) \
+    do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au32[0] = (a_u32Value); } while (0)
+#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
+    do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
+         pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1] = 0; \
+    } while (0)
+#define IEM_MC_STORE_XREG_HI_U64(a_iXReg, a_u64Value) \
+    do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1] = (a_u64Value); } while (0)
+#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg)       \
+    (a_pu128Dst) = (&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].uXmm)
+#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
+    (a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].uXmm)
+#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
+    (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0])
+#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
+    do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXRegDst)].au64[0] \
+            = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXRegSrc)].au64[0]; \
+         pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXRegDst)].au64[1] \
+            = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXRegSrc)].au64[1]; \
+    } while (0)
+
+#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegSrcTmp    = (a_iYRegSrc); \
+         (a_u32Dst) = pXStateTmp->x87.aXMM[iYRegSrcTmp].au32[0]; \
+    } while (0)
+#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegSrcTmp    = (a_iYRegSrc); \
+         (a_u64Dst) = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
+    } while (0)
+#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegSrcTmp    = (a_iYRegSrc); \
+         (a_u128Dst).au64[0] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
+         (a_u128Dst).au64[1] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[1]; \
+    } while (0)
+#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegSrcTmp    = (a_iYRegSrc); \
+         (a_u256Dst).au64[0] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
+         (a_u256Dst).au64[1] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[1]; \
+         (a_u256Dst).au64[2] = pXStateTmp->u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
+         (a_u256Dst).au64[3] = pXStateTmp->u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
+    } while (0)
+
+#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_pXState, a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
+#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegDstTmp    = (a_iYRegDst); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au32[0]       = (a_u32Src); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au32[1]       = 0; \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1]       = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
+         IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
+    } while (0)
+#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegDstTmp    = (a_iYRegDst); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0]       = (a_u64Src); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1]       = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
+         IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
+    } while (0)
+#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegDstTmp    = (a_iYRegDst); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0]       = (a_u128Src).au64[0]; \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1]       = (a_u128Src).au64[1]; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
+         IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
+    } while (0)
+#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegDstTmp    = (a_iYRegDst); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0]       = (a_u256Src).au64[0]; \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1]       = (a_u256Src).au64[1]; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
+         IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
+    } while (0)
+
+#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg)       \
+    (a_pu128Dst) = (&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aYMM[(a_iYReg)].uXmm)
+#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
+    (a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aYMM[(a_iYReg)].uXmm)
+#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
+    (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aYMM[(a_iYReg)].au64[0])
+#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
+    do { PX86XSAVEAREA   pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegTmp   = (a_iYReg); \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
+         IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegTmp); \
+    } while (0)
+
+#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegDstTmp    = (a_iYRegDst); \
+         uintptr_t const iYRegSrcTmp    = (a_iYRegSrc); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0]       = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1]       = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[1]; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pXStateTmp->u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pXStateTmp->u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
+         IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
+    } while (0)
+#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegDstTmp    = (a_iYRegDst); \
+         uintptr_t const iYRegSrcTmp    = (a_iYRegSrc); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0]       = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1]       = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[1]; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
+         IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
+    } while (0)
+#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegDstTmp    = (a_iYRegDst); \
+         uintptr_t const iYRegSrcTmp    = (a_iYRegSrc); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0]       = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1]       = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
+         IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
+    } while (0)
+
+#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegDstTmp    = (a_iYRegDst); \
+         uintptr_t const iYRegSrc32Tmp  = (a_iYRegSrc32); \
+         uintptr_t const iYRegSrcHxTmp  = (a_iYRegSrcHx); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au32[0]       = pXStateTmp->x87.aXMM[iYRegSrc32Tmp].au32[0]; \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au32[1]       = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au32[1]; \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1]       = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au64[1]; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
+         IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
+    } while (0)
+#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegDstTmp    = (a_iYRegDst); \
+         uintptr_t const iYRegSrc64Tmp  = (a_iYRegSrc64); \
+         uintptr_t const iYRegSrcHxTmp  = (a_iYRegSrcHx); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0]       = pXStateTmp->x87.aXMM[iYRegSrc64Tmp].au64[0]; \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1]       = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au64[1]; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
+         IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
+    } while (0)
+#define IEM_MC_MERGE_YREG_U64HI_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegDstTmp    = (a_iYRegDst); \
+         uintptr_t const iYRegSrc64Tmp  = (a_iYRegSrc64); \
+         uintptr_t const iYRegSrcHxTmp  = (a_iYRegSrcHx); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0]       = pXStateTmp->x87.aXMM[iYRegSrc64Tmp].au64[1]; \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1]       = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au64[1]; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
+         IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
+    } while (0)
+#define IEM_MC_MERGE_YREG_U64LOCAL_U64_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
+    do { PX86XSAVEAREA   pXStateTmp     = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
+         uintptr_t const iYRegDstTmp    = (a_iYRegDst); \
+         uintptr_t const iYRegSrcHxTmp  = (a_iYRegSrcHx); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0]       = (a_u64Local); \
+         pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1]       = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au64[1]; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
+         pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
+         IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
+    } while (0)
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
+# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
+#else
+# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
+    ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
+# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
+    ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
+# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
+#else
+# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
+    ((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
+# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
+    ((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
+# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
+#else
+# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
+    ((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
+# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
+    ((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+#endif
+
+#ifdef SOME_UNUSED_FUNCTION
+# define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
+# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
+#else
+# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
+    ((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
+# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
+    ((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u32, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).au64[0], (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
+#else
+# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
+    ((a_r32Dst).u32 = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
+    ((a_r64Dst).au64[0] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
+    iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
+#else
+# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
+    iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
+# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
+    iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
+#else
+# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
+    iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
+# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
+    iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
+#endif
+
+
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
+    do { \
+        uint8_t u8Tmp; \
+        IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
+        (a_u16Dst) = u8Tmp; \
+    } while (0)
+# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
+    do { \
+        uint8_t u8Tmp; \
+        IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
+        (a_u32Dst) = u8Tmp; \
+    } while (0)
+# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    do { \
+        uint8_t u8Tmp; \
+        IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
+        (a_u64Dst) = u8Tmp; \
+    } while (0)
+# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
+    do { \
+        uint16_t u16Tmp; \
+        IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
+        (a_u32Dst) = u16Tmp; \
+    } while (0)
+# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    do { \
+        uint16_t u16Tmp; \
+        IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
+        (a_u64Dst) = u16Tmp; \
+    } while (0)
+# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    do { \
+        uint32_t u32Tmp; \
+        IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
+        (a_u64Dst) = u32Tmp; \
+    } while (0)
+#else  /* IEM_WITH_SETJMP */
+# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+#endif /* IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
+    do { \
+        uint8_t u8Tmp; \
+        IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
+        (a_u16Dst) = (int8_t)u8Tmp; \
+    } while (0)
+# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
+    do { \
+        uint8_t u8Tmp; \
+        IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
+        (a_u32Dst) = (int8_t)u8Tmp; \
+    } while (0)
+# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    do { \
+        uint8_t u8Tmp; \
+        IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
+        (a_u64Dst) = (int8_t)u8Tmp; \
+    } while (0)
+# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
+    do { \
+        uint16_t u16Tmp; \
+        IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
+        (a_u32Dst) = (int16_t)u16Tmp; \
+    } while (0)
+# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    do { \
+        uint16_t u16Tmp; \
+        IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
+        (a_u64Dst) = (int16_t)u16Tmp; \
+    } while (0)
+# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    do { \
+        uint32_t u32Tmp; \
+        IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
+        (a_u64Dst) = (int32_t)u32Tmp; \
+    } while (0)
+#else  /* IEM_WITH_SETJMP */
+# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
+    ((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
+#endif /* IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
+# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
+# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
+# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
+#else
+# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
+    iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
+# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
+    iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
+# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
+    iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
+# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
+    iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
+# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
+# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
+# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
+#else
+# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
+    iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
+# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
+    iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
+# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
+    iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
+# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
+    iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
+#endif
+
+#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst,  a_i8C)     *(a_pi8Dst)  = (a_i8C)
+#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C)    *(a_pi16Dst) = (a_i16C)
+#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C)    *(a_pi32Dst) = (a_i32C)
+#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C)    *(a_pi64Dst) = (a_i64C)
+#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst)         (a_pr32Dst)->u32     = UINT32_C(0xffc00000)
+#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst)         (a_pr64Dst)->au64[0] = UINT64_C(0xfff8000000000000)
+#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
+    do { \
+        (a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
+        (a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
+    } while (0)
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
+# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
+#else
+# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
+    iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
+# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
+    iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
+#endif
+
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
+# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
+#else
+# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
+    iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
+# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
+    iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
+#endif
+
+
+#define IEM_MC_PUSH_U16(a_u16Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
+#define IEM_MC_PUSH_U32(a_u32Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
+#define IEM_MC_PUSH_U32_SREG(a_u32Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_u32Value)))
+#define IEM_MC_PUSH_U64(a_u64Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
+
+#define IEM_MC_POP_U16(a_pu16Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
+#define IEM_MC_POP_U32(a_pu32Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
+#define IEM_MC_POP_U64(a_pu64Value) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
+
+/** Maps guest memory for direct or bounce buffered access.
+ * The purpose is to pass it to an operand implementation, thus the a_iArg.
+ * @remarks     May return.
+ */
+#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pMem), sizeof(*(a_pMem)), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
+
+/** Maps guest memory for direct or bounce buffered access.
+ * The purpose is to pass it to an operand implementation, thus the a_iArg.
+ * @remarks     May return.
+ */
+#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_iArg) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
+
+/** Commits the memory and unmaps the guest memory.
+ * @remarks     May return.
+ */
+#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess)))
+
+/** Commits the memory and unmaps the guest memory unless the FPU status word
+ * indicates (@a a_u16FSW) and FPU control word indicates a pending exception
+ * that would cause FLD not to store.
+ *
+ * The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
+ * store, while \#P will not.
+ *
+ * @remarks     May in theory return - for now.
+ */
+#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW) \
+    do { \
+        if (   !(a_u16FSW & X86_FSW_ES) \
+            || !(  (a_u16FSW & (X86_FSW_UE | X86_FSW_OE | X86_FSW_IE)) \
+                 & ~(pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FCW & X86_FCW_MASK_ALL) ) ) \
+            IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess))); \
+    } while (0)
+
+/** Calculate efficient address from R/M. */
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm, cbImm) \
+    IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (bRm), (cbImm), &(a_GCPtrEff)))
+#else
+# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm, cbImm) \
+    ((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (bRm), (cbImm)))
+#endif
+
+#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn)                   (a_pfn)()
+#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0)               (a_pfn)((a0))
+#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1)           (a_pfn)((a0), (a1))
+#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2)       (a_pfn)((a0), (a1), (a2))
+#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3)   (a_pfn)((a0), (a1), (a2), (a3))
+#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2)      (a_rc) = (a_pfn)((a0), (a1), (a2))
+#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3)  (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
+
+/**
+ * Defers the rest of the instruction emulation to a C implementation routine
+ * and returns, only taking the standard parameters.
+ *
+ * @param   a_pfnCImpl      The pointer to the C routine.
+ * @sa      IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
+ */
+#define IEM_MC_CALL_CIMPL_0(a_pfnCImpl)                 return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
+
+/**
+ * Defers the rest of instruction emulation to a C implementation routine and
+ * returns, taking one argument in addition to the standard ones.
+ *
+ * @param   a_pfnCImpl      The pointer to the C routine.
+ * @param   a0              The argument.
+ */
+#define IEM_MC_CALL_CIMPL_1(a_pfnCImpl, a0)             return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0)
+
+/**
+ * Defers the rest of the instruction emulation to a C implementation routine
+ * and returns, taking two arguments in addition to the standard ones.
+ *
+ * @param   a_pfnCImpl      The pointer to the C routine.
+ * @param   a0              The first extra argument.
+ * @param   a1              The second extra argument.
+ */
+#define IEM_MC_CALL_CIMPL_2(a_pfnCImpl, a0, a1)         return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1)
+
+/**
+ * Defers the rest of the instruction emulation to a C implementation routine
+ * and returns, taking three arguments in addition to the standard ones.
+ *
+ * @param   a_pfnCImpl      The pointer to the C routine.
+ * @param   a0              The first extra argument.
+ * @param   a1              The second extra argument.
+ * @param   a2              The third extra argument.
+ */
+#define IEM_MC_CALL_CIMPL_3(a_pfnCImpl, a0, a1, a2)     return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2)
+
+/**
+ * Defers the rest of the instruction emulation to a C implementation routine
+ * and returns, taking four arguments in addition to the standard ones.
+ *
+ * @param   a_pfnCImpl      The pointer to the C routine.
+ * @param   a0              The first extra argument.
+ * @param   a1              The second extra argument.
+ * @param   a2              The third extra argument.
+ * @param   a3              The fourth extra argument.
+ */
+#define IEM_MC_CALL_CIMPL_4(a_pfnCImpl, a0, a1, a2, a3)     return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3)
+
+/**
+ * Defers the rest of the instruction emulation to a C implementation routine
+ * and returns, taking two arguments in addition to the standard ones.
+ *
+ * @param   a_pfnCImpl      The pointer to the C routine.
+ * @param   a0              The first extra argument.
+ * @param   a1              The second extra argument.
+ * @param   a2              The third extra argument.
+ * @param   a3              The fourth extra argument.
+ * @param   a4              The fifth extra argument.
+ */
+#define IEM_MC_CALL_CIMPL_5(a_pfnCImpl, a0, a1, a2, a3, a4) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4)
+
+/**
+ * Defers the entire instruction emulation to a C implementation routine and
+ * returns, only taking the standard parameters.
+ *
+ * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
+ *
+ * @param   a_pfnCImpl      The pointer to the C routine.
+ * @sa      IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
+ */
+#define IEM_MC_DEFER_TO_CIMPL_0(a_pfnCImpl)             (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
+
+/**
+ * Defers the entire instruction emulation to a C implementation routine and
+ * returns, taking one argument in addition to the standard ones.
+ *
+ * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
+ *
+ * @param   a_pfnCImpl      The pointer to the C routine.
+ * @param   a0              The argument.
+ */
+#define IEM_MC_DEFER_TO_CIMPL_1(a_pfnCImpl, a0)         (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0)
+
+/**
+ * Defers the entire instruction emulation to a C implementation routine and
+ * returns, taking two arguments in addition to the standard ones.
+ *
+ * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
+ *
+ * @param   a_pfnCImpl      The pointer to the C routine.
+ * @param   a0              The first extra argument.
+ * @param   a1              The second extra argument.
+ */
+#define IEM_MC_DEFER_TO_CIMPL_2(a_pfnCImpl, a0, a1)     (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1)
+
+/**
+ * Defers the entire instruction emulation to a C implementation routine and
+ * returns, taking three arguments in addition to the standard ones.
+ *
+ * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
+ *
+ * @param   a_pfnCImpl      The pointer to the C routine.
+ * @param   a0              The first extra argument.
+ * @param   a1              The second extra argument.
+ * @param   a2              The third extra argument.
+ */
+#define IEM_MC_DEFER_TO_CIMPL_3(a_pfnCImpl, a0, a1, a2) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2)
+
+/**
+ * Calls a FPU assembly implementation taking one visible argument.
+ *
+ * @param   a_pfnAImpl      Pointer to the assembly FPU routine.
+ * @param   a0              The first extra argument.
+ */
+#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
+    do { \
+        a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0)); \
+    } while (0)
+
+/**
+ * Calls a FPU assembly implementation taking two visible arguments.
+ *
+ * @param   a_pfnAImpl      Pointer to the assembly FPU routine.
+ * @param   a0              The first extra argument.
+ * @param   a1              The second extra argument.
+ */
+#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
+    do { \
+        a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1)); \
+    } while (0)
+
+/**
+ * Calls a FPU assembly implementation taking three visible arguments.
+ *
+ * @param   a_pfnAImpl      Pointer to the assembly FPU routine.
+ * @param   a0              The first extra argument.
+ * @param   a1              The second extra argument.
+ * @param   a2              The third extra argument.
+ */
+#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
+    do { \
+        a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1), (a2)); \
+    } while (0)
+
+#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
+    do { \
+        (a_FpuData).FSW       = (a_FSW); \
+        (a_FpuData).r80Result = *(a_pr80Value); \
+    } while (0)
+
+/** Pushes FPU result onto the stack. */
+#define IEM_MC_PUSH_FPU_RESULT(a_FpuData) \
+    iemFpuPushResult(pVCpu, &a_FpuData)
+/** Pushes FPU result onto the stack and sets the FPUDP. */
+#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff) \
+    iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff)
+
+/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
+#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo) \
+    iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo)
+
+/** Stores FPU result in a stack register. */
+#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg) \
+    iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg)
+/** Stores FPU result in a stack register and pops the stack. */
+#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg) \
+    iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg)
+/** Stores FPU result in a stack register and sets the FPUDP. */
+#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
+    iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
+/** Stores FPU result in a stack register, sets the FPUDP, and pops the
+ *  stack. */
+#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
+    iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
+
+/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
+#define IEM_MC_UPDATE_FPU_OPCODE_IP() \
+    iemFpuUpdateOpcodeAndIp(pVCpu)
+/** Free a stack register (for FFREE and FFREEP). */
+#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
+    iemFpuStackFree(pVCpu, a_iStReg)
+/** Increment the FPU stack pointer. */
+#define IEM_MC_FPU_STACK_INC_TOP() \
+    iemFpuStackIncTop(pVCpu)
+/** Decrement the FPU stack pointer. */
+#define IEM_MC_FPU_STACK_DEC_TOP() \
+    iemFpuStackDecTop(pVCpu)
+
+/** Updates the FSW, FOP, FPUIP, and FPUCS. */
+#define IEM_MC_UPDATE_FSW(a_u16FSW) \
+    iemFpuUpdateFSW(pVCpu, a_u16FSW)
+/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
+#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW) \
+    iemFpuUpdateFSW(pVCpu, a_u16FSW)
+/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
+#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
+    iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
+/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
+#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW) \
+    iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW)
+/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
+ *  stack. */
+#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
+    iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
+/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
+#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW) \
+    iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW)
+
+/** Raises a FPU stack underflow exception.  Sets FPUIP, FPUCS and FOP. */
+#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst) \
+    iemFpuStackUnderflow(pVCpu, a_iStDst)
+/** Raises a FPU stack underflow exception.  Sets FPUIP, FPUCS and FOP. Pops
+ *  stack. */
+#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst) \
+    iemFpuStackUnderflowThenPop(pVCpu, a_iStDst)
+/** Raises a FPU stack underflow exception.  Sets FPUIP, FPUCS, FOP, FPUDP and
+ *  FPUDS. */
+#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
+    iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
+/** Raises a FPU stack underflow exception.  Sets FPUIP, FPUCS, FOP, FPUDP and
+ *  FPUDS. Pops stack. */
+#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
+    iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
+/** Raises a FPU stack underflow exception.  Sets FPUIP, FPUCS and FOP. Pops
+ *  stack twice. */
+#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP() \
+    iemFpuStackUnderflowThenPopPop(pVCpu)
+/** Raises a FPU stack underflow exception for an instruction pushing a result
+ *  value onto the stack. Sets FPUIP, FPUCS and FOP. */
+#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW() \
+    iemFpuStackPushUnderflow(pVCpu)
+/** Raises a FPU stack underflow exception for an instruction pushing a result
+ *  value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
+#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO() \
+    iemFpuStackPushUnderflowTwo(pVCpu)
+
+/** Raises a FPU stack overflow exception as part of a push attempt.  Sets
+ *  FPUIP, FPUCS and FOP. */
+#define IEM_MC_FPU_STACK_PUSH_OVERFLOW() \
+    iemFpuStackPushOverflow(pVCpu)
+/** Raises a FPU stack overflow exception as part of a push attempt.  Sets
+ *  FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
+#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff) \
+    iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff)
+/** Prepares for using the FPU state.
+ * Ensures that we can use the host FPU in the current context (RC+R0.
+ * Ensures the guest FPU state in the CPUMCTX is up to date. */
+#define IEM_MC_PREPARE_FPU_USAGE()              iemFpuPrepareUsage(pVCpu)
+/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
+#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ()   iemFpuActualizeStateForRead(pVCpu)
+/** Actualizes the guest FPU state so it can be accessed and modified. */
+#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
+
+/** Prepares for using the SSE state.
+ * Ensures that we can use the host SSE/FPU in the current context (RC+R0.
+ * Ensures the guest SSE state in the CPUMCTX is up to date. */
+#define IEM_MC_PREPARE_SSE_USAGE()              iemFpuPrepareUsageSse(pVCpu)
+/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
+#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ()   iemFpuActualizeSseStateForRead(pVCpu)
+/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
+#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
+
+/** Prepares for using the AVX state.
+ * Ensures that we can use the host AVX/FPU in the current context (RC+R0.
+ * Ensures the guest AVX state in the CPUMCTX is up to date.
+ * @note This will include the AVX512 state too when support for it is added
+ *       due to the zero extending feature of VEX instruction. */
+#define IEM_MC_PREPARE_AVX_USAGE()              iemFpuPrepareUsageAvx(pVCpu)
+/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
+#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ()   iemFpuActualizeAvxStateForRead(pVCpu)
+/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
+#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
+
+/**
+ * Calls a MMX assembly implementation taking two visible arguments.
+ *
+ * @param   a_pfnAImpl      Pointer to the assembly MMX routine.
+ * @param   a0              The first extra argument.
+ * @param   a1              The second extra argument.
+ */
+#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
+    do { \
+        IEM_MC_PREPARE_FPU_USAGE(); \
+        a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1)); \
+    } while (0)
+
+/**
+ * Calls a MMX assembly implementation taking three visible arguments.
+ *
+ * @param   a_pfnAImpl      Pointer to the assembly MMX routine.
+ * @param   a0              The first extra argument.
+ * @param   a1              The second extra argument.
+ * @param   a2              The third extra argument.
+ */
+#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
+    do { \
+        IEM_MC_PREPARE_FPU_USAGE(); \
+        a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1), (a2)); \
+    } while (0)
+
+
+/**
+ * Calls a SSE assembly implementation taking two visible arguments.
+ *
+ * @param   a_pfnAImpl      Pointer to the assembly SSE routine.
+ * @param   a0              The first extra argument.
+ * @param   a1              The second extra argument.
+ */
+#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
+    do { \
+        IEM_MC_PREPARE_SSE_USAGE(); \
+        a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1)); \
+    } while (0)
+
+/**
+ * Calls a SSE assembly implementation taking three visible arguments.
+ *
+ * @param   a_pfnAImpl      Pointer to the assembly SSE routine.
+ * @param   a0              The first extra argument.
+ * @param   a1              The second extra argument.
+ * @param   a2              The third extra argument.
+ */
+#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
+    do { \
+        IEM_MC_PREPARE_SSE_USAGE(); \
+        a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1), (a2)); \
+    } while (0)
+
+
+/** Declares implicit arguments for IEM_MC_CALL_AVX_AIMPL_2,
+ *  IEM_MC_CALL_AVX_AIMPL_3, IEM_MC_CALL_AVX_AIMPL_4, ... */
+#define IEM_MC_IMPLICIT_AVX_AIMPL_ARGS() \
+    IEM_MC_ARG_CONST(PX86XSAVEAREA, pXState, pVCpu->cpum.GstCtx.CTX_SUFF(pXState), 0)
+
+/**
+ * Calls a AVX assembly implementation taking two visible arguments.
+ *
+ * There is one implicit zero'th argument, a pointer to the extended state.
+ *
+ * @param   a_pfnAImpl      Pointer to the assembly AVX routine.
+ * @param   a1              The first extra argument.
+ * @param   a2              The second extra argument.
+ */
+#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a1, a2) \
+    do { \
+        IEM_MC_PREPARE_AVX_USAGE(); \
+        a_pfnAImpl(pXState, (a1), (a2)); \
+    } while (0)
+
+/**
+ * Calls a AVX assembly implementation taking three visible arguments.
+ *
+ * There is one implicit zero'th argument, a pointer to the extended state.
+ *
+ * @param   a_pfnAImpl      Pointer to the assembly AVX routine.
+ * @param   a1              The first extra argument.
+ * @param   a2              The second extra argument.
+ * @param   a3              The third extra argument.
+ */
+#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a1, a2, a3) \
+    do { \
+        IEM_MC_PREPARE_AVX_USAGE(); \
+        a_pfnAImpl(pXState, (a1), (a2), (a3)); \
+    } while (0)
+
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_EFL_BIT_SET(a_fBit)                   if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit)               if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits)             if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits)              if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2)         \
+    if (   !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
+        != !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2)         \
+    if (   !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
+        == !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
+    if (   (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
+        ||    !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
+           != !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
+    if (   !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
+        &&    !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
+           == !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
+#define IEM_MC_IF_CX_IS_NZ()                            if (pVCpu->cpum.GstCtx.cx != 0) {
+#define IEM_MC_IF_ECX_IS_NZ()                           if (pVCpu->cpum.GstCtx.ecx != 0) {
+#define IEM_MC_IF_RCX_IS_NZ()                           if (pVCpu->cpum.GstCtx.rcx != 0) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
+        if (   pVCpu->cpum.GstCtx.cx != 0 \
+            && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
+        if (   pVCpu->cpum.GstCtx.ecx != 0 \
+            && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
+        if (   pVCpu->cpum.GstCtx.rcx != 0 \
+            && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
+        if (   pVCpu->cpum.GstCtx.cx != 0 \
+            && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
+        if (   pVCpu->cpum.GstCtx.ecx != 0 \
+            && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
+/** @note Not for IOPL or IF testing. */
+#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
+        if (   pVCpu->cpum.GstCtx.rcx != 0 \
+            && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
+#define IEM_MC_IF_LOCAL_IS_Z(a_Local)                   if ((a_Local) == 0) {
+#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo)       if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
+
+#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
+    if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
+#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
+    if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
+#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
+    if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
+#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
+    if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
+#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
+    if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
+#define IEM_MC_IF_FCW_IM() \
+    if (pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FCW & X86_FCW_IM) {
+
+#define IEM_MC_ELSE()                                   } else {
+#define IEM_MC_ENDIF()                                  } do {} while (0)
+
+/** @}  */
+
+
+/** @name   Opcode Debug Helpers.
+ * @{
+ */
+#ifdef VBOX_WITH_STATISTICS
+# define IEMOP_INC_STATS(a_Stats) do { pVCpu->iem.s.CTX_SUFF(pStats)->a_Stats += 1; } while (0)
+#else
+# define IEMOP_INC_STATS(a_Stats) do { } while (0)
+#endif
+
+#ifdef DEBUG
+# define IEMOP_MNEMONIC(a_Stats, a_szMnemonic) \
+    do { \
+        IEMOP_INC_STATS(a_Stats); \
+        Log4(("decode - %04x:%RGv %s%s [#%u]\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, \
+              pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK ? "lock " : "", a_szMnemonic, pVCpu->iem.s.cInstructions)); \
+    } while (0)
+
+# define IEMOP_MNEMONIC0EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) \
+    do { \
+        IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
+        (void)RT_CONCAT(IEMOPFORM_, a_Form); \
+        (void)RT_CONCAT(OP_,a_Upper); \
+        (void)(a_fDisHints); \
+        (void)(a_fIemHints); \
+    } while (0)
+
+# define IEMOP_MNEMONIC1EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) \
+    do { \
+        IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
+        (void)RT_CONCAT(IEMOPFORM_, a_Form); \
+        (void)RT_CONCAT(OP_,a_Upper); \
+        (void)RT_CONCAT(OP_PARM_,a_Op1); \
+        (void)(a_fDisHints); \
+        (void)(a_fIemHints); \
+    } while (0)
+
+# define IEMOP_MNEMONIC2EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) \
+    do { \
+        IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
+        (void)RT_CONCAT(IEMOPFORM_, a_Form); \
+        (void)RT_CONCAT(OP_,a_Upper); \
+        (void)RT_CONCAT(OP_PARM_,a_Op1); \
+        (void)RT_CONCAT(OP_PARM_,a_Op2); \
+        (void)(a_fDisHints); \
+        (void)(a_fIemHints); \
+    } while (0)
+
+# define IEMOP_MNEMONIC3EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) \
+    do { \
+        IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
+        (void)RT_CONCAT(IEMOPFORM_, a_Form); \
+        (void)RT_CONCAT(OP_,a_Upper); \
+        (void)RT_CONCAT(OP_PARM_,a_Op1); \
+        (void)RT_CONCAT(OP_PARM_,a_Op2); \
+        (void)RT_CONCAT(OP_PARM_,a_Op3); \
+        (void)(a_fDisHints); \
+        (void)(a_fIemHints); \
+    } while (0)
+
+# define IEMOP_MNEMONIC4EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) \
+    do { \
+        IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
+        (void)RT_CONCAT(IEMOPFORM_, a_Form); \
+        (void)RT_CONCAT(OP_,a_Upper); \
+        (void)RT_CONCAT(OP_PARM_,a_Op1); \
+        (void)RT_CONCAT(OP_PARM_,a_Op2); \
+        (void)RT_CONCAT(OP_PARM_,a_Op3); \
+        (void)RT_CONCAT(OP_PARM_,a_Op4); \
+        (void)(a_fDisHints); \
+        (void)(a_fIemHints); \
+    } while (0)
+
+#else
+# define IEMOP_MNEMONIC(a_Stats, a_szMnemonic) IEMOP_INC_STATS(a_Stats)
+
+# define IEMOP_MNEMONIC0EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) \
+         IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
+# define IEMOP_MNEMONIC1EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) \
+         IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
+# define IEMOP_MNEMONIC2EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) \
+         IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
+# define IEMOP_MNEMONIC3EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) \
+         IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
+# define IEMOP_MNEMONIC4EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) \
+         IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
+
+#endif
+
+#define IEMOP_MNEMONIC0(a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) \
+    IEMOP_MNEMONIC0EX(a_Lower, \
+                      #a_Lower, \
+                      a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints)
+#define IEMOP_MNEMONIC1(a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) \
+    IEMOP_MNEMONIC1EX(RT_CONCAT3(a_Lower,_,a_Op1), \
+                      #a_Lower " " #a_Op1, \
+                      a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints)
+#define IEMOP_MNEMONIC2(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) \
+    IEMOP_MNEMONIC2EX(RT_CONCAT5(a_Lower,_,a_Op1,_,a_Op2), \
+                      #a_Lower " " #a_Op1 "," #a_Op2, \
+                      a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints)
+#define IEMOP_MNEMONIC3(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) \
+    IEMOP_MNEMONIC3EX(RT_CONCAT7(a_Lower,_,a_Op1,_,a_Op2,_,a_Op3), \
+                      #a_Lower " " #a_Op1 "," #a_Op2 "," #a_Op3, \
+                      a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints)
+#define IEMOP_MNEMONIC4(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) \
+    IEMOP_MNEMONIC4EX(RT_CONCAT9(a_Lower,_,a_Op1,_,a_Op2,_,a_Op3,_,a_Op4), \
+                      #a_Lower " " #a_Op1 "," #a_Op2 "," #a_Op3 "," #a_Op4, \
+                      a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints)
+
+/** @} */
+
+
+/** @name   Opcode Helpers.
+ * @{
+ */
+
+#ifdef IN_RING3
+# define IEMOP_HLP_MIN_CPU(a_uMinCpu, a_fOnlyIf) \
+    do { \
+        if (IEM_GET_TARGET_CPU(pVCpu) >= (a_uMinCpu) || !(a_fOnlyIf)) { } \
+        else \
+        { \
+            (void)DBGFSTOP(pVCpu->CTX_SUFF(pVM)); \
+            return IEMOP_RAISE_INVALID_OPCODE(); \
+        } \
+    } while (0)
+#else
+# define IEMOP_HLP_MIN_CPU(a_uMinCpu, a_fOnlyIf) \
+    do { \
+        if (IEM_GET_TARGET_CPU(pVCpu) >= (a_uMinCpu) || !(a_fOnlyIf)) { } \
+        else return IEMOP_RAISE_INVALID_OPCODE(); \
+    } while (0)
+#endif
+
+/** The instruction requires a 186 or later. */
+#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_186
+# define IEMOP_HLP_MIN_186() do { } while (0)
+#else
+# define IEMOP_HLP_MIN_186() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_186, true)
+#endif
+
+/** The instruction requires a 286 or later. */
+#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_286
+# define IEMOP_HLP_MIN_286() do { } while (0)
+#else
+# define IEMOP_HLP_MIN_286() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_286, true)
+#endif
+
+/** The instruction requires a 386 or later. */
+#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_386
+# define IEMOP_HLP_MIN_386() do { } while (0)
+#else
+# define IEMOP_HLP_MIN_386() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_386, true)
+#endif
+
+/** The instruction requires a 386 or later if the given expression is true. */
+#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_386
+# define IEMOP_HLP_MIN_386_EX(a_fOnlyIf) do { } while (0)
+#else
+# define IEMOP_HLP_MIN_386_EX(a_fOnlyIf) IEMOP_HLP_MIN_CPU(IEMTARGETCPU_386, a_fOnlyIf)
+#endif
+
+/** The instruction requires a 486 or later. */
+#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_486
+# define IEMOP_HLP_MIN_486() do { } while (0)
+#else
+# define IEMOP_HLP_MIN_486() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_486, true)
+#endif
+
+/** The instruction requires a Pentium (586) or later. */
+#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_PENTIUM
+# define IEMOP_HLP_MIN_586() do { } while (0)
+#else
+# define IEMOP_HLP_MIN_586() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_PENTIUM, true)
+#endif
+
+/** The instruction requires a PentiumPro (686) or later. */
+#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_PPRO
+# define IEMOP_HLP_MIN_686() do { } while (0)
+#else
+# define IEMOP_HLP_MIN_686() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_PPRO, true)
+#endif
+
+
+/** The instruction raises an \#UD in real and V8086 mode. */
+#define IEMOP_HLP_NO_REAL_OR_V86_MODE() \
+    do \
+    { \
+        if (!IEM_IS_REAL_OR_V86_MODE(pVCpu)) { /* likely */ } \
+        else return IEMOP_RAISE_INVALID_OPCODE(); \
+    } while (0)
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/** This instruction raises an \#UD in real and V8086 mode or when not using a
+ *  64-bit code segment when in long mode (applicable to all VMX instructions
+ *  except VMCALL).
+ */
+#define IEMOP_HLP_VMX_INSTR(a_szInstr, a_InsDiagPrefix) \
+    do \
+    { \
+        if (   !IEM_IS_REAL_OR_V86_MODE(pVCpu) \
+            && (  !IEM_IS_LONG_MODE(pVCpu) \
+                || IEM_IS_64BIT_CODE(pVCpu))) \
+        { /* likely */ } \
+        else \
+        { \
+            if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) \
+            { \
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = a_InsDiagPrefix##_RealOrV86Mode; \
+                Log5((a_szInstr ": Real or v8086 mode -> #UD\n")); \
+                return IEMOP_RAISE_INVALID_OPCODE(); \
+            } \
+            if (IEM_IS_LONG_MODE(pVCpu) && !IEM_IS_64BIT_CODE(pVCpu)) \
+            { \
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = a_InsDiagPrefix##_LongModeCS; \
+                Log5((a_szInstr ": Long mode without 64-bit code segment -> #UD\n")); \
+                return IEMOP_RAISE_INVALID_OPCODE(); \
+            } \
+        } \
+    } while (0)
+
+/** The instruction can only be executed in VMX operation (VMX root mode and
+ * non-root mode).
+ *
+ *  @note Update IEM_VMX_IN_VMX_OPERATION if changes are made here.
+ */
+# define IEMOP_HLP_IN_VMX_OPERATION(a_szInstr, a_InsDiagPrefix) \
+    do \
+    { \
+        if (IEM_VMX_IS_ROOT_MODE(pVCpu)) { /* likely */ } \
+        else \
+        { \
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = a_InsDiagPrefix##_VmxRoot; \
+            Log5((a_szInstr ": Not in VMX operation (root mode) -> #UD\n")); \
+            return IEMOP_RAISE_INVALID_OPCODE(); \
+        } \
+    } while (0)
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+/** The instruction is not available in 64-bit mode, throw \#UD if we're in
+ * 64-bit mode. */
+#define IEMOP_HLP_NO_64BIT() \
+    do \
+    { \
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) \
+            return IEMOP_RAISE_INVALID_OPCODE(); \
+    } while (0)
+
+/** The instruction is only available in 64-bit mode, throw \#UD if we're not in
+ * 64-bit mode. */
+#define IEMOP_HLP_ONLY_64BIT() \
+    do \
+    { \
+        if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) \
+            return IEMOP_RAISE_INVALID_OPCODE(); \
+    } while (0)
+
+/** The instruction defaults to 64-bit operand size if 64-bit mode. */
+#define IEMOP_HLP_DEFAULT_64BIT_OP_SIZE() \
+    do \
+    { \
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) \
+            iemRecalEffOpSize64Default(pVCpu); \
+    } while (0)
+
+/** The instruction has 64-bit operand size if 64-bit mode. */
+#define IEMOP_HLP_64BIT_OP_SIZE() \
+    do \
+    { \
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) \
+            pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT; \
+    } while (0)
+
+/** Only a REX prefix immediately preceeding the first opcode byte takes
+ * effect. This macro helps ensuring this as well as logging bad guest code.  */
+#define IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE(a_szPrf) \
+    do \
+    { \
+        if (RT_UNLIKELY(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX)) \
+        { \
+            Log5((a_szPrf ": Overriding REX prefix at %RX16! fPrefixes=%#x\n", pVCpu->cpum.GstCtx.rip, pVCpu->iem.s.fPrefixes)); \
+            pVCpu->iem.s.fPrefixes &= ~IEM_OP_PRF_REX_MASK; \
+            pVCpu->iem.s.uRexB     = 0; \
+            pVCpu->iem.s.uRexIndex = 0; \
+            pVCpu->iem.s.uRexReg   = 0; \
+            iemRecalEffOpSize(pVCpu); \
+        } \
+    } while (0)
+
+/**
+ * Done decoding.
+ */
+#define IEMOP_HLP_DONE_DECODING() \
+    do \
+    { \
+        /*nothing for now, maybe later... */ \
+    } while (0)
+
+/**
+ * Done decoding, raise \#UD exception if lock prefix present.
+ */
+#define IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX() \
+    do \
+    { \
+        if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))) \
+        { /* likely */ } \
+        else \
+            return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
+    } while (0)
+
+
+/**
+ * Done decoding VEX instruction, raise \#UD exception if any lock, rex, repz,
+ * repnz or size prefixes are present, or if in real or v8086 mode.
+ */
+#define IEMOP_HLP_DONE_VEX_DECODING() \
+    do \
+    { \
+        if (RT_LIKELY(   !(  pVCpu->iem.s.fPrefixes \
+                           & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REX)) \
+                      && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )) \
+        { /* likely */ } \
+        else \
+            return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
+    } while (0)
+
+/**
+ * Done decoding VEX instruction, raise \#UD exception if any lock, rex, repz,
+ * repnz or size prefixes are present, or if in real or v8086 mode.
+ */
+#define IEMOP_HLP_DONE_VEX_DECODING_L0() \
+    do \
+    { \
+        if (RT_LIKELY(   !(  pVCpu->iem.s.fPrefixes \
+                           & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REX)) \
+                      && !IEM_IS_REAL_OR_V86_MODE(pVCpu) \
+                      && pVCpu->iem.s.uVexLength == 0)) \
+        { /* likely */ } \
+        else \
+            return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
+    } while (0)
+
+
+/**
+ * Done decoding VEX instruction, raise \#UD exception if any lock, rex, repz,
+ * repnz or size prefixes are present, or if the VEX.VVVV field doesn't indicate
+ * register 0, or if in real or v8086 mode.
+ */
+#define IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV() \
+    do \
+    { \
+        if (RT_LIKELY(   !(  pVCpu->iem.s.fPrefixes \
+                           & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REX)) \
+                      && !pVCpu->iem.s.uVex3rdReg \
+                      && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )) \
+        { /* likely */ } \
+        else \
+            return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
+    } while (0)
+
+/**
+ * Done decoding VEX, no V, L=0.
+ * Raises \#UD exception if rex, rep, opsize or lock prefixes are present, if
+ * we're in real or v8086 mode, if VEX.V!=0xf, or if VEX.L!=0.
+ */
+#define IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV() \
+    do \
+    { \
+        if (RT_LIKELY(   !(  pVCpu->iem.s.fPrefixes \
+                           & (IEM_OP_PRF_LOCK | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_REX)) \
+                      && pVCpu->iem.s.uVexLength == 0 \
+                      && pVCpu->iem.s.uVex3rdReg == 0 \
+                      && !IEM_IS_REAL_OR_V86_MODE(pVCpu))) \
+        { /* likely */ } \
+        else \
+            return IEMOP_RAISE_INVALID_OPCODE(); \
+    } while (0)
+
+#define IEMOP_HLP_DECODED_NL_1(a_uDisOpNo, a_fIemOpFlags, a_uDisParam0, a_fDisOpType) \
+    do \
+    { \
+        if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))) \
+        { /* likely */ } \
+        else \
+        { \
+            NOREF(a_uDisOpNo); NOREF(a_fIemOpFlags); NOREF(a_uDisParam0); NOREF(a_fDisOpType); \
+            return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
+        } \
+    } while (0)
+#define IEMOP_HLP_DECODED_NL_2(a_uDisOpNo, a_fIemOpFlags, a_uDisParam0, a_uDisParam1, a_fDisOpType) \
+    do \
+    { \
+        if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))) \
+        { /* likely */ } \
+        else \
+        { \
+            NOREF(a_uDisOpNo); NOREF(a_fIemOpFlags); NOREF(a_uDisParam0); NOREF(a_uDisParam1); NOREF(a_fDisOpType); \
+            return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
+        } \
+    } while (0)
+
+/**
+ * Done decoding, raise \#UD exception if any lock, repz or repnz prefixes
+ * are present.
+ */
+#define IEMOP_HLP_DONE_DECODING_NO_LOCK_REPZ_OR_REPNZ_PREFIXES() \
+    do \
+    { \
+        if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)))) \
+        { /* likely */ } \
+        else \
+            return IEMOP_RAISE_INVALID_OPCODE(); \
+    } while (0)
+
+/**
+ * Done decoding, raise \#UD exception if any operand-size override, repz or repnz
+ * prefixes are present.
+ */
+#define IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES() \
+    do \
+    { \
+        if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)))) \
+        { /* likely */ } \
+        else \
+            return IEMOP_RAISE_INVALID_OPCODE(); \
+    } while (0)
+
+
+/**
+ * Calculates the effective address of a ModR/M memory operand.
+ *
+ * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
+ *
+ * @return  Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   bRm                 The ModRM byte.
+ * @param   cbImm               The size of any immediate following the
+ *                              effective address opcode bytes. Important for
+ *                              RIP relative addressing.
+ * @param   pGCPtrEff           Where to return the effective address.
+ */
+IEM_STATIC VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff)
+{
+    Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
+# define SET_SS_DEF() \
+    do \
+    { \
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
+            pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
+    } while (0)
+
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
+    {
+/** @todo Check the effective address size crap! */
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
+        {
+            uint16_t u16EffAddr;
+
+            /* Handle the disp16 form with no registers first. */
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
+                IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
+            else
+            {
+                /* Get the displacment. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0:  u16EffAddr = 0;                             break;
+                    case 1:  IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
+                    case 2:  IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);       break;
+                    default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
+                }
+
+                /* Add the base and index registers to the disp. */
+                switch (bRm & X86_MODRM_RM_MASK)
+                {
+                    case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
+                    case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
+                    case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
+                    case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
+                    case 4: u16EffAddr += pVCpu->cpum.GstCtx.si;            break;
+                    case 5: u16EffAddr += pVCpu->cpum.GstCtx.di;            break;
+                    case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp;            SET_SS_DEF(); break;
+                    case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx;            break;
+                }
+            }
+
+            *pGCPtrEff = u16EffAddr;
+        }
+        else
+        {
+            Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
+            uint32_t u32EffAddr;
+
+            /* Handle the disp32 form with no registers first. */
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+                IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
+            else
+            {
+                /* Get the register (or SIB) value. */
+                switch ((bRm & X86_MODRM_RM_MASK))
+                {
+                    case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
+                    case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
+                    case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
+                    case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
+                    case 4: /* SIB */
+                    {
+                        uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
+
+                        /* Get the index and scale it. */
+                        switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
+                        {
+                            case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
+                            case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
+                            case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
+                            case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
+                            case 4: u32EffAddr = 0; /*none */ break;
+                            case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
+                            case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
+                            case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
+                            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                        }
+                        u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                        /* add base */
+                        switch (bSib & X86_SIB_BASE_MASK)
+                        {
+                            case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
+                            case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
+                            case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
+                            case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
+                            case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
+                            case 5:
+                                if ((bRm & X86_MODRM_MOD_MASK) != 0)
+                                {
+                                    u32EffAddr += pVCpu->cpum.GstCtx.ebp;
+                                    SET_SS_DEF();
+                                }
+                                else
+                                {
+                                    uint32_t u32Disp;
+                                    IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                                    u32EffAddr += u32Disp;
+                                }
+                                break;
+                            case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
+                            case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
+                            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                        }
+                        break;
+                    }
+                    case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
+                    case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
+                    case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+
+                /* Get and add the displacement. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0:
+                        break;
+                    case 1:
+                    {
+                        int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
+                        u32EffAddr += i8Disp;
+                        break;
+                    }
+                    case 2:
+                    {
+                        uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                        u32EffAddr += u32Disp;
+                        break;
+                    }
+                    default:
+                        AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
+                }
+
+            }
+            if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
+                *pGCPtrEff = u32EffAddr;
+            else
+            {
+                Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
+                *pGCPtrEff = u32EffAddr & UINT16_MAX;
+            }
+        }
+    }
+    else
+    {
+        uint64_t u64EffAddr;
+
+        /* Handle the rip+disp32 form with no registers first. */
+        if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+        {
+            IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
+            u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
+        }
+        else
+        {
+            /* Get the register (or SIB) value. */
+            switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
+            {
+                case  0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
+                case  1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
+                case  2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
+                case  3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
+                case  5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
+                case  6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
+                case  7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
+                case  8: u64EffAddr = pVCpu->cpum.GstCtx.r8;  break;
+                case  9: u64EffAddr = pVCpu->cpum.GstCtx.r9;  break;
+                case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
+                case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
+                case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
+                case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
+                case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
+                /* SIB */
+                case 4:
+                case 12:
+                {
+                    uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
+
+                    /* Get the index and scale it. */
+                    switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
+                    {
+                        case  0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
+                        case  1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
+                        case  2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
+                        case  3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
+                        case  4: u64EffAddr = 0; /*none */ break;
+                        case  5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
+                        case  6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
+                        case  7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
+                        case  8: u64EffAddr = pVCpu->cpum.GstCtx.r8;  break;
+                        case  9: u64EffAddr = pVCpu->cpum.GstCtx.r9;  break;
+                        case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
+                        case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
+                        case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
+                        case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
+                        case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
+                        case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
+                        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                    }
+                    u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                    /* add base */
+                    switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
+                    {
+                        case  0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
+                        case  1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
+                        case  2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
+                        case  3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
+                        case  4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
+                        case  6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
+                        case  7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
+                        case  8: u64EffAddr += pVCpu->cpum.GstCtx.r8;  break;
+                        case  9: u64EffAddr += pVCpu->cpum.GstCtx.r9;  break;
+                        case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
+                        case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
+                        case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
+                        case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
+                        case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
+                        /* complicated encodings */
+                        case 5:
+                        case 13:
+                            if ((bRm & X86_MODRM_MOD_MASK) != 0)
+                            {
+                                if (!pVCpu->iem.s.uRexB)
+                                {
+                                    u64EffAddr += pVCpu->cpum.GstCtx.rbp;
+                                    SET_SS_DEF();
+                                }
+                                else
+                                    u64EffAddr += pVCpu->cpum.GstCtx.r13;
+                            }
+                            else
+                            {
+                                uint32_t u32Disp;
+                                IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                                u64EffAddr += (int32_t)u32Disp;
+                            }
+                            break;
+                        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                    }
+                    break;
+                }
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+
+            /* Get and add the displacement. */
+            switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+            {
+                case 0:
+                    break;
+                case 1:
+                {
+                    int8_t i8Disp;
+                    IEM_OPCODE_GET_NEXT_S8(&i8Disp);
+                    u64EffAddr += i8Disp;
+                    break;
+                }
+                case 2:
+                {
+                    uint32_t u32Disp;
+                    IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                    u64EffAddr += (int32_t)u32Disp;
+                    break;
+                }
+                IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
+            }
+
+        }
+
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
+            *pGCPtrEff = u64EffAddr;
+        else
+        {
+            Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
+            *pGCPtrEff = u64EffAddr & UINT32_MAX;
+        }
+    }
+
+    Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Calculates the effective address of a ModR/M memory operand.
+ *
+ * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
+ *
+ * @return  Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   bRm                 The ModRM byte.
+ * @param   cbImm               The size of any immediate following the
+ *                              effective address opcode bytes. Important for
+ *                              RIP relative addressing.
+ * @param   pGCPtrEff           Where to return the effective address.
+ * @param   offRsp              RSP displacement.
+ */
+IEM_STATIC VBOXSTRICTRC iemOpHlpCalcRmEffAddrEx(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff, int8_t offRsp)
+{
+    Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
+# define SET_SS_DEF() \
+    do \
+    { \
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
+            pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
+    } while (0)
+
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
+    {
+/** @todo Check the effective address size crap! */
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
+        {
+            uint16_t u16EffAddr;
+
+            /* Handle the disp16 form with no registers first. */
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
+                IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
+            else
+            {
+                /* Get the displacment. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0:  u16EffAddr = 0;                             break;
+                    case 1:  IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
+                    case 2:  IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);       break;
+                    default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
+                }
+
+                /* Add the base and index registers to the disp. */
+                switch (bRm & X86_MODRM_RM_MASK)
+                {
+                    case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
+                    case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
+                    case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
+                    case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
+                    case 4: u16EffAddr += pVCpu->cpum.GstCtx.si;            break;
+                    case 5: u16EffAddr += pVCpu->cpum.GstCtx.di;            break;
+                    case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp;            SET_SS_DEF(); break;
+                    case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx;            break;
+                }
+            }
+
+            *pGCPtrEff = u16EffAddr;
+        }
+        else
+        {
+            Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
+            uint32_t u32EffAddr;
+
+            /* Handle the disp32 form with no registers first. */
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+                IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
+            else
+            {
+                /* Get the register (or SIB) value. */
+                switch ((bRm & X86_MODRM_RM_MASK))
+                {
+                    case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
+                    case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
+                    case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
+                    case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
+                    case 4: /* SIB */
+                    {
+                        uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
+
+                        /* Get the index and scale it. */
+                        switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
+                        {
+                            case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
+                            case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
+                            case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
+                            case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
+                            case 4: u32EffAddr = 0; /*none */ break;
+                            case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
+                            case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
+                            case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
+                            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                        }
+                        u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                        /* add base */
+                        switch (bSib & X86_SIB_BASE_MASK)
+                        {
+                            case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
+                            case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
+                            case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
+                            case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
+                            case 4:
+                                u32EffAddr += pVCpu->cpum.GstCtx.esp + offRsp;
+                                SET_SS_DEF();
+                                break;
+                            case 5:
+                                if ((bRm & X86_MODRM_MOD_MASK) != 0)
+                                {
+                                    u32EffAddr += pVCpu->cpum.GstCtx.ebp;
+                                    SET_SS_DEF();
+                                }
+                                else
+                                {
+                                    uint32_t u32Disp;
+                                    IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                                    u32EffAddr += u32Disp;
+                                }
+                                break;
+                            case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
+                            case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
+                            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                        }
+                        break;
+                    }
+                    case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
+                    case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
+                    case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+
+                /* Get and add the displacement. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0:
+                        break;
+                    case 1:
+                    {
+                        int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
+                        u32EffAddr += i8Disp;
+                        break;
+                    }
+                    case 2:
+                    {
+                        uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                        u32EffAddr += u32Disp;
+                        break;
+                    }
+                    default:
+                        AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
+                }
+
+            }
+            if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
+                *pGCPtrEff = u32EffAddr;
+            else
+            {
+                Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
+                *pGCPtrEff = u32EffAddr & UINT16_MAX;
+            }
+        }
+    }
+    else
+    {
+        uint64_t u64EffAddr;
+
+        /* Handle the rip+disp32 form with no registers first. */
+        if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+        {
+            IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
+            u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
+        }
+        else
+        {
+            /* Get the register (or SIB) value. */
+            switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
+            {
+                case  0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
+                case  1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
+                case  2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
+                case  3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
+                case  5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
+                case  6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
+                case  7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
+                case  8: u64EffAddr = pVCpu->cpum.GstCtx.r8;  break;
+                case  9: u64EffAddr = pVCpu->cpum.GstCtx.r9;  break;
+                case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
+                case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
+                case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
+                case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
+                case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
+                /* SIB */
+                case 4:
+                case 12:
+                {
+                    uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
+
+                    /* Get the index and scale it. */
+                    switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
+                    {
+                        case  0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
+                        case  1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
+                        case  2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
+                        case  3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
+                        case  4: u64EffAddr = 0; /*none */ break;
+                        case  5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
+                        case  6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
+                        case  7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
+                        case  8: u64EffAddr = pVCpu->cpum.GstCtx.r8;  break;
+                        case  9: u64EffAddr = pVCpu->cpum.GstCtx.r9;  break;
+                        case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
+                        case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
+                        case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
+                        case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
+                        case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
+                        case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
+                        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                    }
+                    u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                    /* add base */
+                    switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
+                    {
+                        case  0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
+                        case  1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
+                        case  2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
+                        case  3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
+                        case  4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + offRsp; SET_SS_DEF(); break;
+                        case  6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
+                        case  7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
+                        case  8: u64EffAddr += pVCpu->cpum.GstCtx.r8;  break;
+                        case  9: u64EffAddr += pVCpu->cpum.GstCtx.r9;  break;
+                        case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
+                        case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
+                        case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
+                        case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
+                        case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
+                        /* complicated encodings */
+                        case 5:
+                        case 13:
+                            if ((bRm & X86_MODRM_MOD_MASK) != 0)
+                            {
+                                if (!pVCpu->iem.s.uRexB)
+                                {
+                                    u64EffAddr += pVCpu->cpum.GstCtx.rbp;
+                                    SET_SS_DEF();
+                                }
+                                else
+                                    u64EffAddr += pVCpu->cpum.GstCtx.r13;
+                            }
+                            else
+                            {
+                                uint32_t u32Disp;
+                                IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                                u64EffAddr += (int32_t)u32Disp;
+                            }
+                            break;
+                        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                    }
+                    break;
+                }
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+
+            /* Get and add the displacement. */
+            switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+            {
+                case 0:
+                    break;
+                case 1:
+                {
+                    int8_t i8Disp;
+                    IEM_OPCODE_GET_NEXT_S8(&i8Disp);
+                    u64EffAddr += i8Disp;
+                    break;
+                }
+                case 2:
+                {
+                    uint32_t u32Disp;
+                    IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                    u64EffAddr += (int32_t)u32Disp;
+                    break;
+                }
+                IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
+            }
+
+        }
+
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
+            *pGCPtrEff = u64EffAddr;
+        else
+        {
+            Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
+            *pGCPtrEff = u64EffAddr & UINT32_MAX;
+        }
+    }
+
+    Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
+    return VINF_SUCCESS;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Calculates the effective address of a ModR/M memory operand.
+ *
+ * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
+ *
+ * May longjmp on internal error.
+ *
+ * @return  The effective address.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   bRm                 The ModRM byte.
+ * @param   cbImm               The size of any immediate following the
+ *                              effective address opcode bytes. Important for
+ *                              RIP relative addressing.
+ */
+IEM_STATIC RTGCPTR iemOpHlpCalcRmEffAddrJmp(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm)
+{
+    Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
+# define SET_SS_DEF() \
+    do \
+    { \
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
+            pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
+    } while (0)
+
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
+    {
+/** @todo Check the effective address size crap! */
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
+        {
+            uint16_t u16EffAddr;
+
+            /* Handle the disp16 form with no registers first. */
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
+                IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
+            else
+            {
+                /* Get the displacment. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0:  u16EffAddr = 0;                             break;
+                    case 1:  IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
+                    case 2:  IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);       break;
+                    default: AssertFailedStmt(longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_1)); /* (caller checked for these) */
+                }
+
+                /* Add the base and index registers to the disp. */
+                switch (bRm & X86_MODRM_RM_MASK)
+                {
+                    case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
+                    case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
+                    case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
+                    case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
+                    case 4: u16EffAddr += pVCpu->cpum.GstCtx.si;            break;
+                    case 5: u16EffAddr += pVCpu->cpum.GstCtx.di;            break;
+                    case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp;            SET_SS_DEF(); break;
+                    case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx;            break;
+                }
+            }
+
+            Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16\n", u16EffAddr));
+            return u16EffAddr;
+        }
+
+        Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
+        uint32_t u32EffAddr;
+
+        /* Handle the disp32 form with no registers first. */
+        if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+            IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
+        else
+        {
+            /* Get the register (or SIB) value. */
+            switch ((bRm & X86_MODRM_RM_MASK))
+            {
+                case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
+                case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
+                case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
+                case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
+                case 4: /* SIB */
+                {
+                    uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
+
+                    /* Get the index and scale it. */
+                    switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
+                    {
+                        case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
+                        case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
+                        case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
+                        case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
+                        case 4: u32EffAddr = 0; /*none */ break;
+                        case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
+                        case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
+                        case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
+                        IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
+                    }
+                    u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                    /* add base */
+                    switch (bSib & X86_SIB_BASE_MASK)
+                    {
+                        case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
+                        case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
+                        case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
+                        case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
+                        case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
+                        case 5:
+                            if ((bRm & X86_MODRM_MOD_MASK) != 0)
+                            {
+                                u32EffAddr += pVCpu->cpum.GstCtx.ebp;
+                                SET_SS_DEF();
+                            }
+                            else
+                            {
+                                uint32_t u32Disp;
+                                IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                                u32EffAddr += u32Disp;
+                            }
+                            break;
+                        case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
+                        case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
+                        IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
+                    }
+                    break;
+                }
+                case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
+                case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
+                case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
+            }
+
+            /* Get and add the displacement. */
+            switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+            {
+                case 0:
+                    break;
+                case 1:
+                {
+                    int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
+                    u32EffAddr += i8Disp;
+                    break;
+                }
+                case 2:
+                {
+                    uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                    u32EffAddr += u32Disp;
+                    break;
+                }
+                default:
+                    AssertFailedStmt(longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_2)); /* (caller checked for these) */
+            }
+        }
+
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
+        {
+            Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32\n", u32EffAddr));
+            return u32EffAddr;
+        }
+        Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
+        Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX32\n", u32EffAddr & UINT16_MAX));
+        return u32EffAddr & UINT16_MAX;
+    }
+
+    uint64_t u64EffAddr;
+
+    /* Handle the rip+disp32 form with no registers first. */
+    if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+    {
+        IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
+        u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
+    }
+    else
+    {
+        /* Get the register (or SIB) value. */
+        switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
+        {
+            case  0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
+            case  1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
+            case  2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
+            case  3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
+            case  5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
+            case  6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
+            case  7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
+            case  8: u64EffAddr = pVCpu->cpum.GstCtx.r8;  break;
+            case  9: u64EffAddr = pVCpu->cpum.GstCtx.r9;  break;
+            case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
+            case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
+            case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
+            case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
+            case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
+            /* SIB */
+            case 4:
+            case 12:
+            {
+                uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
+
+                /* Get the index and scale it. */
+                switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
+                {
+                    case  0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
+                    case  1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
+                    case  2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
+                    case  3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
+                    case  4: u64EffAddr = 0; /*none */ break;
+                    case  5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
+                    case  6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
+                    case  7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
+                    case  8: u64EffAddr = pVCpu->cpum.GstCtx.r8;  break;
+                    case  9: u64EffAddr = pVCpu->cpum.GstCtx.r9;  break;
+                    case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
+                    case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
+                    case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
+                    case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
+                    case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
+                    case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
+                }
+                u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                /* add base */
+                switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
+                {
+                    case  0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
+                    case  1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
+                    case  2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
+                    case  3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
+                    case  4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
+                    case  6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
+                    case  7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
+                    case  8: u64EffAddr += pVCpu->cpum.GstCtx.r8;  break;
+                    case  9: u64EffAddr += pVCpu->cpum.GstCtx.r9;  break;
+                    case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
+                    case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
+                    case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
+                    case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
+                    case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
+                    /* complicated encodings */
+                    case 5:
+                    case 13:
+                        if ((bRm & X86_MODRM_MOD_MASK) != 0)
+                        {
+                            if (!pVCpu->iem.s.uRexB)
+                            {
+                                u64EffAddr += pVCpu->cpum.GstCtx.rbp;
+                                SET_SS_DEF();
+                            }
+                            else
+                                u64EffAddr += pVCpu->cpum.GstCtx.r13;
+                        }
+                        else
+                        {
+                            uint32_t u32Disp;
+                            IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                            u64EffAddr += (int32_t)u32Disp;
+                        }
+                        break;
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
+                }
+                break;
+            }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
+        }
+
+        /* Get and add the displacement. */
+        switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+        {
+            case 0:
+                break;
+            case 1:
+            {
+                int8_t i8Disp;
+                IEM_OPCODE_GET_NEXT_S8(&i8Disp);
+                u64EffAddr += i8Disp;
+                break;
+            }
+            case 2:
+            {
+                uint32_t u32Disp;
+                IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                u64EffAddr += (int32_t)u32Disp;
+                break;
+            }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
+        }
+
+    }
+
+    if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
+    {
+        Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr));
+        return u64EffAddr;
+    }
+    Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
+    Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr & UINT32_MAX));
+    return u64EffAddr & UINT32_MAX;
+}
+#endif /* IEM_WITH_SETJMP */
+
+/** @}  */
+
+
+
+/*
+ * Include the instructions
+ */
+#include "IEMAllInstructions.cpp.h"
+
+
+
+#ifdef LOG_ENABLED
+/**
+ * Logs the current instruction.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   fSameCtx    Set if we have the same context information as the VMM,
+ *                      clear if we may have already executed an instruction in
+ *                      our debug context. When clear, we assume IEMCPU holds
+ *                      valid CPU mode info.
+ *
+ *                      The @a fSameCtx parameter is now misleading and obsolete.
+ * @param   pszFunction The IEM function doing the execution.
+ */
+IEM_STATIC void iemLogCurInstr(PVMCPUCC pVCpu, bool fSameCtx, const char *pszFunction)
+{
+# ifdef IN_RING3
+    if (LogIs2Enabled())
+    {
+        char     szInstr[256];
+        uint32_t cbInstr = 0;
+        if (fSameCtx)
+            DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
+                               DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                               szInstr, sizeof(szInstr), &cbInstr);
+        else
+        {
+            uint32_t fFlags = 0;
+            switch (pVCpu->iem.s.enmCpuMode)
+            {
+                case IEMMODE_64BIT: fFlags |= DBGF_DISAS_FLAGS_64BIT_MODE; break;
+                case IEMMODE_32BIT: fFlags |= DBGF_DISAS_FLAGS_32BIT_MODE; break;
+                case IEMMODE_16BIT:
+                    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) || pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
+                        fFlags |= DBGF_DISAS_FLAGS_16BIT_REAL_MODE;
+                    else
+                        fFlags |= DBGF_DISAS_FLAGS_16BIT_MODE;
+                    break;
+            }
+            DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fFlags,
+                               szInstr, sizeof(szInstr), &cbInstr);
+        }
+
+        PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+        Log2(("**** %s\n"
+              " eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
+              " eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
+              " cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
+              " fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
+              " %s\n"
+              , pszFunction,
+              pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
+              pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
+              pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
+              pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
+              pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
+              szInstr));
+
+        if (LogIs3Enabled())
+            DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
+    }
+    else
+# endif
+        LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
+                 pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(fSameCtx);
+}
+#endif /* LOG_ENABLED */
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Deals with VMCPU_FF_VMX_APIC_WRITE, VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_NMI_WINDOW,
+ * VMCPU_FF_VMX_PREEMPT_TIMER and VMCPU_FF_VMX_INT_WINDOW.
+ *
+ * @returns Modified rcStrict.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ * @param   rcStrict    The instruction execution status.
+ */
+static VBOXSTRICTRC iemHandleNestedInstructionBoundraryFFs(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
+{
+    Assert(CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)));
+    if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF))
+    {
+        /* VMX preemption timer takes priority over NMI-window exits. */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
+        {
+            rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
+            Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
+        }
+        /*
+         * Check remaining intercepts.
+         *
+         * NMI-window and Interrupt-window VM-exits.
+         * Interrupt shadow (block-by-STI and Mov SS) inhibits interrupts and may also block NMIs.
+         * Event injection during VM-entry takes priority over NMI-window and interrupt-window VM-exits.
+         *
+         * See Intel spec. 26.7.6 "NMI-Window Exiting".
+         * See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
+         */
+        else if (   VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW | VMCPU_FF_VMX_INT_WINDOW)
+                 && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+                 && !TRPMHasTrap(pVCpu))
+        {
+            Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
+            if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)
+                && CPUMIsGuestVmxVirtNmiBlocking(&pVCpu->cpum.GstCtx))
+            {
+                rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */);
+                Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW));
+            }
+            else if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)
+                     && CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx))
+            {
+                rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */);
+                Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW));
+            }
+        }
+    }
+    /* TPR-below threshold/APIC write has the highest priority. */
+    else  if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
+    {
+        rcStrict = iemVmxApicWriteEmulation(pVCpu);
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE));
+    }
+    /* MTF takes priority over VMX-preemption timer. */
+    else
+    {
+        rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* u64ExitQual */);
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
+    }
+    return rcStrict;
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/**
+ * Makes status code addjustments (pass up from I/O and access handler)
+ * as well as maintaining statistics.
+ *
+ * @returns Strict VBox status code to pass up.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ * @param   rcStrict    The status from executing an instruction.
+ */
+DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
+{
+    if (rcStrict != VINF_SUCCESS)
+    {
+        if (RT_SUCCESS(rcStrict))
+        {
+            AssertMsg(   (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
+                      || rcStrict == VINF_IOM_R3_IOPORT_READ
+                      || rcStrict == VINF_IOM_R3_IOPORT_WRITE
+                      || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
+                      || rcStrict == VINF_IOM_R3_MMIO_READ
+                      || rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
+                      || rcStrict == VINF_IOM_R3_MMIO_WRITE
+                      || rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
+                      || rcStrict == VINF_CPUM_R3_MSR_READ
+                      || rcStrict == VINF_CPUM_R3_MSR_WRITE
+                      || rcStrict == VINF_EM_RAW_EMULATE_INSTR
+                      || rcStrict == VINF_EM_RAW_TO_R3
+                      || rcStrict == VINF_EM_TRIPLE_FAULT
+                      || rcStrict == VINF_GIM_R3_HYPERCALL
+                      /* raw-mode / virt handlers only: */
+                      || rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
+                      || rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
+                      || rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
+                      || rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
+                      || rcStrict == VINF_SELM_SYNC_GDT
+                      || rcStrict == VINF_CSAM_PENDING_ACTION
+                      || rcStrict == VINF_PATM_CHECK_PATCH_PAGE
+                      /* nested hw.virt codes: */
+                      || rcStrict == VINF_VMX_VMEXIT
+                      || rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
+                      || rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
+                      || rcStrict == VINF_SVM_VMEXIT
+                      , ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
+            int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            if (   rcStrict == VINF_VMX_VMEXIT
+                && rcPassUp == VINF_SUCCESS)
+                rcStrict = VINF_SUCCESS;
+            else
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+            if (   rcStrict == VINF_SVM_VMEXIT
+                && rcPassUp == VINF_SUCCESS)
+                rcStrict = VINF_SUCCESS;
+            else
+#endif
+            if (rcPassUp == VINF_SUCCESS)
+                pVCpu->iem.s.cRetInfStatuses++;
+            else if (   rcPassUp < VINF_EM_FIRST
+                     || rcPassUp > VINF_EM_LAST
+                     || rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
+            {
+                Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
+                pVCpu->iem.s.cRetPassUpStatus++;
+                rcStrict = rcPassUp;
+            }
+            else
+            {
+                Log(("IEM: rcPassUp=%Rrc  rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
+                pVCpu->iem.s.cRetInfStatuses++;
+            }
+        }
+        else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
+            pVCpu->iem.s.cRetAspectNotImplemented++;
+        else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
+            pVCpu->iem.s.cRetInstrNotImplemented++;
+        else
+            pVCpu->iem.s.cRetErrStatuses++;
+    }
+    else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
+    {
+        pVCpu->iem.s.cRetPassUpStatus++;
+        rcStrict = pVCpu->iem.s.rcPassUp;
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * The actual code execution bits of IEMExecOne, IEMExecOneEx, and
+ * IEMExecOneWithPrefetchedByPC.
+ *
+ * Similar code is found in IEMExecLots.
+ *
+ * @return  Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   fExecuteInhibit     If set, execute the instruction following CLI,
+ *                      POP SS and MOV SS,GR.
+ * @param   pszFunction The calling function name.
+ */
+DECLINLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPUCC pVCpu, bool fExecuteInhibit, const char *pszFunction)
+{
+    AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
+    AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
+    AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
+    RT_NOREF_PV(pszFunction);
+
+#ifdef IEM_WITH_SETJMP
+    VBOXSTRICTRC rcStrict;
+    jmp_buf      JmpBuf;
+    jmp_buf     *pSavedJmpBuf  = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
+    pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
+    if ((rcStrict = setjmp(JmpBuf)) == 0)
+    {
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+    else
+        pVCpu->iem.s.cLongJumps++;
+    pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
+#else
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
+#endif
+    if (rcStrict == VINF_SUCCESS)
+        pVCpu->iem.s.cInstructions++;
+    if (pVCpu->iem.s.cActiveMappings > 0)
+    {
+        Assert(rcStrict != VINF_SUCCESS);
+        iemMemRollback(pVCpu);
+    }
+    AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
+    AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
+    AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
+
+//#ifdef DEBUG
+//    AssertMsg(IEM_GET_INSTR_LEN(pVCpu) == cbInstr || rcStrict != VINF_SUCCESS, ("%u %u\n", IEM_GET_INSTR_LEN(pVCpu), cbInstr));
+//#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * Perform any VMX nested-guest instruction boundary actions.
+     *
+     * If any of these causes a VM-exit, we must skip executing the next
+     * instruction (would run into stale page tables). A VM-exit makes sure
+     * there is no interrupt-inhibition, so that should ensure we don't go
+     * to try execute the next instruction. Clearing fExecuteInhibit is
+     * problematic because of the setjmp/longjmp clobbering above.
+     */
+    if (   rcStrict == VINF_SUCCESS
+        && VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER
+                                    | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW))
+        rcStrict = iemHandleNestedInstructionBoundraryFFs(pVCpu, rcStrict);
+#endif
+
+    /* Execute the next instruction as well if a cli, pop ss or
+       mov ss, Gr has just completed successfully. */
+    if (   fExecuteInhibit
+        && rcStrict == VINF_SUCCESS
+        && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+        && EMIsInhibitInterruptsActive(pVCpu))
+    {
+        rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, pVCpu->iem.s.fBypassHandlers);
+        if (rcStrict == VINF_SUCCESS)
+        {
+#ifdef LOG_ENABLED
+            iemLogCurInstr(pVCpu, false, pszFunction);
+#endif
+#ifdef IEM_WITH_SETJMP
+            pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
+            if ((rcStrict = setjmp(JmpBuf)) == 0)
+            {
+                uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+                rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
+            }
+            else
+                pVCpu->iem.s.cLongJumps++;
+            pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
+#else
+            IEM_OPCODE_GET_NEXT_U8(&b);
+            rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
+#endif
+            if (rcStrict == VINF_SUCCESS)
+                pVCpu->iem.s.cInstructions++;
+            if (pVCpu->iem.s.cActiveMappings > 0)
+            {
+                Assert(rcStrict != VINF_SUCCESS);
+                iemMemRollback(pVCpu);
+            }
+            AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
+            AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
+            AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
+        }
+        else if (pVCpu->iem.s.cActiveMappings > 0)
+            iemMemRollback(pVCpu);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS); /* hope this is correct for all exceptional cases... */
+    }
+
+    /*
+     * Return value fiddling, statistics and sanity assertions.
+     */
+    rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+    return rcStrict;
+}
+
+
+/**
+ * Execute one instruction.
+ *
+ * @return  Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPUCC pVCpu)
+{
+#ifdef LOG_ENABLED
+    iemLogCurInstr(pVCpu, true, "IEMExecOne");
+#endif
+
+    /*
+     * Do the decoding and emulation.
+     */
+    VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOne");
+    else if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    if (rcStrict != VINF_SUCCESS)
+        LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
+                 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+VMMDECL(VBOXSTRICTRC) IEMExecOneEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
+{
+    AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
+
+    uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
+    VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneEx");
+        if (pcbWritten)
+            *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
+    }
+    else if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    return rcStrict;
+}
+
+
+VMMDECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
+                                                   const void *pvOpcodeBytes, size_t cbOpcodeBytes)
+{
+    AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
+
+    VBOXSTRICTRC rcStrict;
+    if (   cbOpcodeBytes
+        && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
+    {
+        iemInitDecoder(pVCpu, false);
+#ifdef IEM_WITH_CODE_TLB
+        pVCpu->iem.s.uInstrBufPc      = OpcodeBytesPC;
+        pVCpu->iem.s.pbInstrBuf       = (uint8_t const *)pvOpcodeBytes;
+        pVCpu->iem.s.cbInstrBufTotal  = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
+        pVCpu->iem.s.offCurInstrStart = 0;
+        pVCpu->iem.s.offInstrNextByte = 0;
+#else
+        pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
+        memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
+#endif
+        rcStrict = VINF_SUCCESS;
+    }
+    else
+        rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneWithPrefetchedByPC");
+    else if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    return rcStrict;
+}
+
+
+VMMDECL(VBOXSTRICTRC) IEMExecOneBypassEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
+{
+    AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
+
+    uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
+    VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassEx");
+        if (pcbWritten)
+            *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
+    }
+    else if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    return rcStrict;
+}
+
+
+VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
+                                                         const void *pvOpcodeBytes, size_t cbOpcodeBytes)
+{
+    AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
+
+    VBOXSTRICTRC rcStrict;
+    if (   cbOpcodeBytes
+        && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
+    {
+        iemInitDecoder(pVCpu, true);
+#ifdef IEM_WITH_CODE_TLB
+        pVCpu->iem.s.uInstrBufPc      = OpcodeBytesPC;
+        pVCpu->iem.s.pbInstrBuf       = (uint8_t const *)pvOpcodeBytes;
+        pVCpu->iem.s.cbInstrBufTotal  = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
+        pVCpu->iem.s.offCurInstrStart = 0;
+        pVCpu->iem.s.offInstrNextByte = 0;
+#else
+        pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
+        memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
+#endif
+        rcStrict = VINF_SUCCESS;
+    }
+    else
+        rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPC");
+    else if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    return rcStrict;
+}
+
+
+/**
+ * For debugging DISGetParamSize, may come in handy.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.
+ * @param   pCtxCore        The context core structure.
+ * @param   OpcodeBytesPC   The PC of the opcode bytes.
+ * @param   pvOpcodeBytes   Prefeched opcode bytes.
+ * @param   cbOpcodeBytes   Number of prefetched bytes.
+ * @param   pcbWritten      Where to return the number of bytes written.
+ *                          Optional.
+ */
+VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPCWritten(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
+                                                                const void *pvOpcodeBytes, size_t cbOpcodeBytes,
+                                                                uint32_t *pcbWritten)
+{
+    AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
+
+    uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
+    VBOXSTRICTRC rcStrict;
+    if (   cbOpcodeBytes
+        && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
+    {
+        iemInitDecoder(pVCpu, true);
+#ifdef IEM_WITH_CODE_TLB
+        pVCpu->iem.s.uInstrBufPc      = OpcodeBytesPC;
+        pVCpu->iem.s.pbInstrBuf       = (uint8_t const *)pvOpcodeBytes;
+        pVCpu->iem.s.cbInstrBufTotal  = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
+        pVCpu->iem.s.offCurInstrStart = 0;
+        pVCpu->iem.s.offInstrNextByte = 0;
+#else
+        pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
+        memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
+#endif
+        rcStrict = VINF_SUCCESS;
+    }
+    else
+        rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPCWritten");
+        if (pcbWritten)
+            *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
+    }
+    else if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    return rcStrict;
+}
+
+
+VMMDECL(VBOXSTRICTRC) IEMExecLots(PVMCPUCC pVCpu, uint32_t cMaxInstructions, uint32_t cPollRate, uint32_t *pcInstructions)
+{
+    uint32_t const cInstructionsAtStart = pVCpu->iem.s.cInstructions;
+    AssertMsg(RT_IS_POWER_OF_TWO(cPollRate + 1), ("%#x\n", cPollRate));
+
+    /*
+     * See if there is an interrupt pending in TRPM, inject it if we can.
+     */
+    /** @todo Can we centralize this under CPUMCanInjectInterrupt()? */
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+    bool fIntrEnabled = CPUMGetGuestGif(&pVCpu->cpum.GstCtx);
+    if (fIntrEnabled)
+    {
+        if (!CPUMIsGuestInNestedHwvirtMode(IEM_GET_CTX(pVCpu)))
+            fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
+        else if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
+            fIntrEnabled = CPUMIsGuestVmxPhysIntrEnabled(IEM_GET_CTX(pVCpu));
+        else
+        {
+            Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
+            fIntrEnabled = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, IEM_GET_CTX(pVCpu));
+        }
+    }
+#else
+    bool fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
+#endif
+
+    /** @todo What if we are injecting an exception and not an interrupt? Is that
+     *        possible here? For now we assert it is indeed only an interrupt. */
+    if (   fIntrEnabled
+        && TRPMHasTrap(pVCpu)
+        && EMGetInhibitInterruptsPC(pVCpu) != pVCpu->cpum.GstCtx.rip)
+    {
+        uint8_t     u8TrapNo;
+        TRPMEVENT   enmType;
+        uint32_t    uErrCode;
+        RTGCPTR     uCr2;
+        int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /* pu8InstLen */, NULL /* fIcebp */);
+        AssertRC(rc2);
+        Assert(enmType == TRPM_HARDWARE_INT);
+        VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /* cbInstr */);
+        TRPMResetTrap(pVCpu);
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+        /* Injecting an event may cause a VM-exit. */
+        if (   rcStrict != VINF_SUCCESS
+            && rcStrict != VINF_IEM_RAISED_XCPT)
+            return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+#else
+        NOREF(rcStrict);
+#endif
+    }
+
+    /*
+     * Initial decoder init w/ prefetch, then setup setjmp.
+     */
+    VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
+    if (rcStrict == VINF_SUCCESS)
+    {
+#ifdef IEM_WITH_SETJMP
+        jmp_buf         JmpBuf;
+        jmp_buf        *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
+        pVCpu->iem.s.CTX_SUFF(pJmpBuf)   = &JmpBuf;
+        pVCpu->iem.s.cActiveMappings     = 0;
+        if ((rcStrict = setjmp(JmpBuf)) == 0)
+#endif
+        {
+            /*
+             * The run loop.  We limit ourselves to 4096 instructions right now.
+             */
+            uint32_t cMaxInstructionsGccStupidity = cMaxInstructions;
+            PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+            for (;;)
+            {
+                /*
+                 * Log the state.
+                 */
+#ifdef LOG_ENABLED
+                iemLogCurInstr(pVCpu, true, "IEMExecLots");
+#endif
+
+                /*
+                 * Do the decoding and emulation.
+                 */
+                uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+                rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
+                if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+                {
+                    Assert(pVCpu->iem.s.cActiveMappings == 0);
+                    pVCpu->iem.s.cInstructions++;
+                    if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
+                    {
+                        uint64_t fCpu = pVCpu->fLocalForcedActions
+                                      & ( VMCPU_FF_ALL_MASK & ~(  VMCPU_FF_PGM_SYNC_CR3
+                                                                | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
+                                                                | VMCPU_FF_TLB_FLUSH
+                                                                | VMCPU_FF_INHIBIT_INTERRUPTS
+                                                                | VMCPU_FF_BLOCK_NMIS
+                                                                | VMCPU_FF_UNHALT ));
+
+                        if (RT_LIKELY(   (   !fCpu
+                                          || (   !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
+                                              && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
+                                      && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
+                        {
+                            if (cMaxInstructionsGccStupidity-- > 0)
+                            {
+                                /* Poll timers every now an then according to the caller's specs. */
+                                if (   (cMaxInstructionsGccStupidity & cPollRate) != 0
+                                    || !TMTimerPollBool(pVM, pVCpu))
+                                {
+                                    Assert(pVCpu->iem.s.cActiveMappings == 0);
+                                    iemReInitDecoder(pVCpu);
+                                    continue;
+                                }
+                            }
+                        }
+                    }
+                    Assert(pVCpu->iem.s.cActiveMappings == 0);
+                }
+                else if (pVCpu->iem.s.cActiveMappings > 0)
+                        iemMemRollback(pVCpu);
+                rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+                break;
+            }
+        }
+#ifdef IEM_WITH_SETJMP
+        else
+        {
+            if (pVCpu->iem.s.cActiveMappings > 0)
+                iemMemRollback(pVCpu);
+# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+            rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+# endif
+            pVCpu->iem.s.cLongJumps++;
+        }
+        pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
+#endif
+
+        /*
+         * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
+         */
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+    }
+    else
+    {
+        if (pVCpu->iem.s.cActiveMappings > 0)
+            iemMemRollback(pVCpu);
+
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+        /*
+         * When a nested-guest causes an exception intercept (e.g. #PF) when fetching
+         * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
+         */
+        rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+#endif
+    }
+
+    /*
+     * Maybe re-enter raw-mode and log.
+     */
+    if (rcStrict != VINF_SUCCESS)
+        LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
+                 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
+    if (pcInstructions)
+        *pcInstructions = pVCpu->iem.s.cInstructions - cInstructionsAtStart;
+    return rcStrict;
+}
+
+
+/**
+ * Interface used by EMExecuteExec, does exit statistics and limits.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   fWillExit           To be defined.
+ * @param   cMinInstructions    Minimum number of instructions to execute before checking for FFs.
+ * @param   cMaxInstructions    Maximum number of instructions to execute.
+ * @param   cMaxInstructionsWithoutExits
+ *                              The max number of instructions without exits.
+ * @param   pStats              Where to return statistics.
+ */
+VMMDECL(VBOXSTRICTRC) IEMExecForExits(PVMCPUCC pVCpu, uint32_t fWillExit, uint32_t cMinInstructions, uint32_t cMaxInstructions,
+                                      uint32_t cMaxInstructionsWithoutExits, PIEMEXECFOREXITSTATS pStats)
+{
+    NOREF(fWillExit); /** @todo define flexible exit crits */
+
+    /*
+     * Initialize return stats.
+     */
+    pStats->cInstructions    = 0;
+    pStats->cExits           = 0;
+    pStats->cMaxExitDistance = 0;
+    pStats->cReserved        = 0;
+
+    /*
+     * Initial decoder init w/ prefetch, then setup setjmp.
+     */
+    VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
+    if (rcStrict == VINF_SUCCESS)
+    {
+#ifdef IEM_WITH_SETJMP
+        jmp_buf         JmpBuf;
+        jmp_buf        *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
+        pVCpu->iem.s.CTX_SUFF(pJmpBuf)   = &JmpBuf;
+        pVCpu->iem.s.cActiveMappings     = 0;
+        if ((rcStrict = setjmp(JmpBuf)) == 0)
+#endif
+        {
+#ifdef IN_RING0
+            bool const fCheckPreemptionPending   = !RTThreadPreemptIsPossible() || !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
+#endif
+            uint32_t   cInstructionSinceLastExit = 0;
+
+            /*
+             * The run loop.  We limit ourselves to 4096 instructions right now.
+             */
+            PVM pVM = pVCpu->CTX_SUFF(pVM);
+            for (;;)
+            {
+                /*
+                 * Log the state.
+                 */
+#ifdef LOG_ENABLED
+                iemLogCurInstr(pVCpu, true, "IEMExecForExits");
+#endif
+
+                /*
+                 * Do the decoding and emulation.
+                 */
+                uint32_t const cPotentialExits = pVCpu->iem.s.cPotentialExits;
+
+                uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+                rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
+
+                if (   cPotentialExits != pVCpu->iem.s.cPotentialExits
+                    && cInstructionSinceLastExit > 0 /* don't count the first */ )
+                {
+                    pStats->cExits += 1;
+                    if (cInstructionSinceLastExit > pStats->cMaxExitDistance)
+                        pStats->cMaxExitDistance = cInstructionSinceLastExit;
+                    cInstructionSinceLastExit = 0;
+                }
+
+                if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+                {
+                    Assert(pVCpu->iem.s.cActiveMappings == 0);
+                    pVCpu->iem.s.cInstructions++;
+                    pStats->cInstructions++;
+                    cInstructionSinceLastExit++;
+                    if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
+                    {
+                        uint64_t fCpu = pVCpu->fLocalForcedActions
+                                      & ( VMCPU_FF_ALL_MASK & ~(  VMCPU_FF_PGM_SYNC_CR3
+                                                                | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
+                                                                | VMCPU_FF_TLB_FLUSH
+                                                                | VMCPU_FF_INHIBIT_INTERRUPTS
+                                                                | VMCPU_FF_BLOCK_NMIS
+                                                                | VMCPU_FF_UNHALT ));
+
+                        if (RT_LIKELY(   (   (   !fCpu
+                                              || (   !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
+                                                  && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF))
+                                          && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) )
+                                      || pStats->cInstructions < cMinInstructions))
+                        {
+                            if (pStats->cInstructions < cMaxInstructions)
+                            {
+                                if (cInstructionSinceLastExit <= cMaxInstructionsWithoutExits)
+                                {
+#ifdef IN_RING0
+                                    if (   !fCheckPreemptionPending
+                                        || !RTThreadPreemptIsPending(NIL_RTTHREAD))
+#endif
+                                    {
+                                        Assert(pVCpu->iem.s.cActiveMappings == 0);
+                                        iemReInitDecoder(pVCpu);
+                                        continue;
+                                    }
+#ifdef IN_RING0
+                                    rcStrict = VINF_EM_RAW_INTERRUPT;
+                                    break;
+#endif
+                                }
+                            }
+                        }
+                        Assert(!(fCpu & VMCPU_FF_IEM));
+                    }
+                    Assert(pVCpu->iem.s.cActiveMappings == 0);
+                }
+                else if (pVCpu->iem.s.cActiveMappings > 0)
+                        iemMemRollback(pVCpu);
+                rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+                break;
+            }
+        }
+#ifdef IEM_WITH_SETJMP
+        else
+        {
+            if (pVCpu->iem.s.cActiveMappings > 0)
+                iemMemRollback(pVCpu);
+            pVCpu->iem.s.cLongJumps++;
+        }
+        pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
+#endif
+
+        /*
+         * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
+         */
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+    }
+    else
+    {
+        if (pVCpu->iem.s.cActiveMappings > 0)
+            iemMemRollback(pVCpu);
+
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+        /*
+         * When a nested-guest causes an exception intercept (e.g. #PF) when fetching
+         * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
+         */
+        rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+#endif
+    }
+
+    /*
+     * Maybe re-enter raw-mode and log.
+     */
+    if (rcStrict != VINF_SUCCESS)
+        LogFlow(("IEMExecForExits: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc; ins=%u exits=%u maxdist=%u\n",
+                 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp,
+                 pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict), pStats->cInstructions, pStats->cExits, pStats->cMaxExitDistance));
+    return rcStrict;
+}
+
+
+/**
+ * Injects a trap, fault, abort, software interrupt or external interrupt.
+ *
+ * The parameter list matches TRPMQueryTrapAll pretty closely.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   u8TrapNo            The trap number.
+ * @param   enmType             What type is it (trap/fault/abort), software
+ *                              interrupt or hardware interrupt.
+ * @param   uErrCode            The error code if applicable.
+ * @param   uCr2                The CR2 value if applicable.
+ * @param   cbInstr             The instruction length (only relevant for
+ *                              software interrupts).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2,
+                                         uint8_t cbInstr)
+{
+    iemInitDecoder(pVCpu, false);
+#ifdef DBGFTRACE_ENABLED
+    RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx",
+                      u8TrapNo, enmType, uErrCode, uCr2);
+#endif
+
+    uint32_t fFlags;
+    switch (enmType)
+    {
+        case TRPM_HARDWARE_INT:
+            Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
+            fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
+            uErrCode = uCr2 = 0;
+            break;
+
+        case TRPM_SOFTWARE_INT:
+            Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
+            fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
+            uErrCode = uCr2 = 0;
+            break;
+
+        case TRPM_TRAP:
+            Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
+            fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
+            if (u8TrapNo == X86_XCPT_PF)
+                fFlags |= IEM_XCPT_FLAGS_CR2;
+            switch (u8TrapNo)
+            {
+                case X86_XCPT_DF:
+                case X86_XCPT_TS:
+                case X86_XCPT_NP:
+                case X86_XCPT_SS:
+                case X86_XCPT_PF:
+                case X86_XCPT_AC:
+                case X86_XCPT_GP:
+                    fFlags |= IEM_XCPT_FLAGS_ERR;
+                    break;
+            }
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2);
+
+    if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    return rcStrict;
+}
+
+
+/**
+ * Injects the active TRPM event.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMDECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPUCC pVCpu)
+{
+#ifndef IEM_IMPLEMENTS_TASKSWITCH
+    IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n"));
+#else
+    uint8_t     u8TrapNo;
+    TRPMEVENT   enmType;
+    uint32_t    uErrCode;
+    RTGCUINTPTR uCr2;
+    uint8_t     cbInstr;
+    int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr, NULL /* fIcebp */);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /** @todo r=ramshankar: Pass ICEBP info. to IEMInjectTrap() below and handle
+     *        ICEBP \#DB injection as a special case. */
+    VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (rcStrict == VINF_SVM_VMEXIT)
+        rcStrict = VINF_SUCCESS;
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (rcStrict == VINF_VMX_VMEXIT)
+        rcStrict = VINF_SUCCESS;
+#endif
+    /** @todo Are there any other codes that imply the event was successfully
+     *        delivered to the guest? See @bugref{6607}.  */
+    if (   rcStrict == VINF_SUCCESS
+        || rcStrict == VINF_IEM_RAISED_XCPT)
+        TRPMResetTrap(pVCpu);
+
+    return rcStrict;
+#endif
+}
+
+
+VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp)
+{
+    RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp)
+{
+    RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+#if 0 /* The IRET-to-v8086 mode in PATM is very optimistic, so I don't dare do this yet. */
+/**
+ * Executes a IRET instruction with default operand size.
+ *
+ * This is for PATM.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   pCtxCore            The register frame.
+ */
+VMM_INT_DECL(int) IEMExecInstr_iret(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore)
+{
+    PCPUMCTX pCtx = IEM_GET_CTX(pVCpu);
+
+    iemCtxCoreToCtx(pCtx, pCtxCore);
+    iemInitDecoder(pVCpu);
+    VBOXSTRICTRC rcStrict = iemCImpl_iret(pVCpu, 1, pVCpu->iem.s.enmDefOpSize);
+    if (rcStrict == VINF_SUCCESS)
+        iemCtxToCtxCore(pCtxCore, pCtx);
+    else
+        LogFlow(("IEMExecInstr_iret: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
+                 pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+#endif
+
+
+/**
+ * Macro used by the IEMExec* method to check the given instruction length.
+ *
+ * Will return on failure!
+ *
+ * @param   a_cbInstr   The given instruction length.
+ * @param   a_cbMin     The minimum length.
+ */
+#define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
+    AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
+                    ("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
+
+
+/**
+ * Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
+ *
+ * Only calling iemRCRawMaybeReenter in raw-mode, obviously.
+ *
+ * @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ * @param   rcStrict    The status code to fiddle.
+ */
+DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
+{
+    iemUninitExec(pVCpu);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM for executing string I/O OUT (write) instructions.
+ *
+ * This API ASSUMES that the caller has already verified that the guest code is
+ * allowed to access the I/O port.  (The I/O port is in the DX register in the
+ * guest state.)
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbValue             The size of the I/O port access (1, 2, or 4).
+ * @param   enmAddrMode         The addressing mode.
+ * @param   fRepPrefix          Indicates whether a repeat prefix is used
+ *                              (doesn't matter which for this instruction).
+ * @param   cbInstr             The instruction length in bytes.
+ * @param   iEffSeg             The effective segment address.
+ * @param   fIoChecked          Whether the access to the I/O port has been
+ *                              checked or not.  It's typically checked in the
+ *                              HM scenario.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoWrite(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
+                                                bool fRepPrefix, uint8_t cbInstr, uint8_t iEffSeg, bool fIoChecked)
+{
+    AssertMsgReturn(iEffSeg < X86_SREG_COUNT, ("%#x\n", iEffSeg), VERR_IEM_INVALID_EFF_SEG);
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
+
+    /*
+     * State init.
+     */
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    /*
+     * Switch orgy for getting to the right handler.
+     */
+    VBOXSTRICTRC rcStrict;
+    if (fRepPrefix)
+    {
+        switch (enmAddrMode)
+        {
+            case IEMMODE_16BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_rep_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_rep_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_rep_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_32BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_rep_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_rep_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_rep_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_64BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_rep_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_rep_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_rep_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            default:
+                AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
+        }
+    }
+    else
+    {
+        switch (enmAddrMode)
+        {
+            case IEMMODE_16BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_32BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_64BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            default:
+                AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
+        }
+    }
+
+    if (pVCpu->iem.s.cActiveMappings)
+        iemMemRollback(pVCpu);
+
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM for executing string I/O IN (read) instructions.
+ *
+ * This API ASSUMES that the caller has already verified that the guest code is
+ * allowed to access the I/O port.  (The I/O port is in the DX register in the
+ * guest state.)
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbValue             The size of the I/O port access (1, 2, or 4).
+ * @param   enmAddrMode         The addressing mode.
+ * @param   fRepPrefix          Indicates whether a repeat prefix is used
+ *                              (doesn't matter which for this instruction).
+ * @param   cbInstr             The instruction length in bytes.
+ * @param   fIoChecked          Whether the access to the I/O port has been
+ *                              checked or not.  It's typically checked in the
+ *                              HM scenario.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoRead(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
+                                               bool fRepPrefix, uint8_t cbInstr, bool fIoChecked)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
+
+    /*
+     * State init.
+     */
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    /*
+     * Switch orgy for getting to the right handler.
+     */
+    VBOXSTRICTRC rcStrict;
+    if (fRepPrefix)
+    {
+        switch (enmAddrMode)
+        {
+            case IEMMODE_16BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_rep_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_rep_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_rep_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_32BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_rep_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_rep_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_rep_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_64BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_rep_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_rep_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_rep_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            default:
+                AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
+        }
+    }
+    else
+    {
+        switch (enmAddrMode)
+        {
+            case IEMMODE_16BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_32BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_64BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            default:
+                AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
+        }
+    }
+
+    Assert(pVCpu->iem.s.cActiveMappings == 0 || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for rawmode to write execute an OUT instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   u16Port     The port to read.
+ * @param   fImm        Whether the port is specified using an immediate operand or
+ *                      using the implicit DX register.
+ * @param   cbReg       The register size.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedOut(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
+    Assert(cbReg <= 4 && cbReg != 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_out, u16Port, fImm, cbReg);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for rawmode to write execute an IN instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   u16Port     The port to read.
+ * @param   fImm        Whether the port is specified using an immediate operand or
+ *                      using the implicit DX.
+ * @param   cbReg       The register size.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedIn(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
+    Assert(cbReg <= 4 && cbReg != 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_in, u16Port, fImm, cbReg);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to write to a CRx register.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iCrReg      The control register number (destination).
+ * @param   iGReg       The general purpose register number (source).
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iCrReg, uint8_t iGReg)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    Assert(iCrReg < 16);
+    Assert(iGReg < 16);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Cd_Rd, iCrReg, iGReg);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to read from a CRx register.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iGReg       The general purpose register number (destination).
+ * @param   iCrReg      The control register number (source).
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4
+                        | CPUMCTX_EXTRN_APIC_TPR);
+    Assert(iCrReg < 16);
+    Assert(iGReg < 16);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Cd, iGReg, iCrReg);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to clear the CR0[TS] bit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClts(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clts);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the LMSW instruction (loads CR0).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   uValue          The value to load into CR0.
+ * @param   GCPtrEffDst     The guest-linear address if the LMSW instruction has a
+ *                          memory operand. Otherwise pass NIL_RTGCPTR.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedLmsw(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uValue, RTGCPTR GCPtrEffDst)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_lmsw, uValue, GCPtrEffDst);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the XSETBV instruction (loads XCRx).
+ *
+ * Takes input values in ecx and edx:eax of the CPU context of the calling EMT.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedXsetbv(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_xsetbv);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the WBINVD instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWbinvd(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wbinvd);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the INVD instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvd(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invd);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the INVLPG instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_PGM_SYNC_CR3
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   GCPtrPage   The effective address of the page to invalidate.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpg(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPTR GCPtrPage)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_invlpg, GCPtrPage);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the INVPCID instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_PGM_SYNC_CR3
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iEffSeg     The effective segment register.
+ * @param   GCPtrDesc   The effective address of the INVPCID descriptor.
+ * @param   uType       The invalidation type.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvpcid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDesc,
+                                                 uint64_t uType)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 4);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_invpcid, iEffSeg, GCPtrDesc, uType);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the CPUID instruction.
+ *
+ * @returns Strict VBox status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in, the usual pluss RAX and RCX.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedCpuid(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_cpuid);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the RDPMC instruction.
+ *
+ * @returns Strict VBox status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdpmc(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdpmc);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the RDTSC instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtsc(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtsc);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the RDTSCP instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.  Recommended
+ *          to include CPUMCTX_EXTRN_TSC_AUX, to avoid extra fetch call.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtscp(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_TSC_AUX);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtscp);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the RDMSR instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.  Requires RCX and
+ *          (currently) all MSRs.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_ALL_MSRS);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdmsr);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the WRMSR instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.  Requires RCX, RAX, RDX,
+ *          and (currently) all MSRs.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWrmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
+                        | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_ALL_MSRS);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wrmsr);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the MONITOR instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.
+ * @remarks ASSUMES the default segment of DS and no segment override prefixes
+ *          are used.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMonitor(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_monitor, X86_SREG_DS);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the MWAIT instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMwait(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_mwait);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the HLT instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedHlt(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_hlt);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Checks if IEM is in the process of delivering an event (interrupt or
+ * exception).
+ *
+ * @returns true if we're in the process of raising an interrupt or exception,
+ *          false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   puVector        Where to store the vector associated with the
+ *                          currently delivered event, optional.
+ * @param   pfFlags         Where to store th event delivery flags (see
+ *                          IEM_XCPT_FLAGS_XXX), optional.
+ * @param   puErr           Where to store the error code associated with the
+ *                          event, optional.
+ * @param   puCr2           Where to store the CR2 associated with the event,
+ *                          optional.
+ * @remarks The caller should check the flags to determine if the error code and
+ *          CR2 are valid for the event.
+ */
+VMM_INT_DECL(bool) IEMGetCurrentXcpt(PVMCPUCC pVCpu, uint8_t *puVector, uint32_t *pfFlags, uint32_t *puErr, uint64_t *puCr2)
+{
+    bool const fRaisingXcpt = pVCpu->iem.s.cXcptRecursions > 0;
+    if (fRaisingXcpt)
+    {
+        if (puVector)
+            *puVector = pVCpu->iem.s.uCurXcpt;
+        if (pfFlags)
+            *pfFlags = pVCpu->iem.s.fCurXcpt;
+        if (puErr)
+            *puErr = pVCpu->iem.s.uCurXcptErr;
+        if (puCr2)
+            *puCr2 = pVCpu->iem.s.uCurXcptCr2;
+    }
+    return fRaisingXcpt;
+}
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+
+/**
+ * Interface for HM and EM to emulate the CLGI instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClgi(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clgi);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the STGI instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedStgi(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_stgi);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMLOAD instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmload(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmload);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMSAVE instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmsave(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmsave);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the INVLPGA instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpga(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invlpga);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMRUN instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmrun(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMRUN_MASK);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmrun);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate \#VMEXIT.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uExitCode   The exit code.
+ * @param   uExitInfo1  The exit info. 1 field.
+ * @param   uExitInfo2  The exit info. 2 field.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2)
+{
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK);
+    VBOXSTRICTRC rcStrict = iemSvmVmexit(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+    if (pVCpu->iem.s.cActiveMappings)
+        iemMemRollback(pVCpu);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+
+/**
+ * Interface for HM and EM to read a VMCS field from the nested-guest VMCS.
+ *
+ * It is ASSUMED the caller knows what they're doing. No VMREAD instruction checks
+ * are performed. Bounds checks are strict builds only.
+ *
+ * @param   pVmcs           Pointer to the virtual VMCS.
+ * @param   u64VmcsField    The VMCS field.
+ * @param   pu64Dst         Where to store the VMCS value.
+ *
+ * @remarks May be called with interrupts disabled.
+ * @todo    This should probably be moved to CPUM someday.
+ */
+VMM_INT_DECL(void) IEMReadVmxVmcsField(PCVMXVVMCS pVmcs, uint64_t u64VmcsField, uint64_t *pu64Dst)
+{
+    AssertPtr(pVmcs);
+    AssertPtr(pu64Dst);
+    iemVmxVmreadNoCheck(pVmcs, pu64Dst, u64VmcsField);
+}
+
+
+/**
+ * Interface for HM and EM to write a VMCS field in the nested-guest VMCS.
+ *
+ * It is ASSUMED the caller knows what they're doing. No VMWRITE instruction checks
+ * are performed. Bounds checks are strict builds only.
+ *
+ * @param   pVmcs           Pointer to the virtual VMCS.
+ * @param   u64VmcsField    The VMCS field.
+ * @param   u64Val          The value to write.
+ *
+ * @remarks May be called with interrupts disabled.
+ * @todo    This should probably be moved to CPUM someday.
+ */
+VMM_INT_DECL(void) IEMWriteVmxVmcsField(PVMXVVMCS pVmcs, uint64_t u64VmcsField, uint64_t u64Val)
+{
+    AssertPtr(pVmcs);
+    iemVmxVmwriteNoCheck(pVmcs, u64Val, u64VmcsField);
+}
+
+
+/**
+ * Interface for HM and EM to virtualize x2APIC MSR accesses.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_VMX_MODIFIES_BEHAVIOR if the MSR access was virtualized.
+ * @retval  VINF_VMX_INTERCEPT_NOT_ACTIVE if the MSR access must be handled by
+ *          the x2APIC device.
+ * @retval  VERR_OUT_RANGE if the caller must raise \#GP(0).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   idMsr       The MSR being read.
+ * @param   pu64Value   Pointer to the value being written or where to store the
+ *                      value being read.
+ * @param   fWrite      Whether this is an MSR write or read access.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVirtApicAccessMsr(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Value, bool fWrite)
+{
+    Assert(pu64Value);
+
+    VBOXSTRICTRC rcStrict;
+    if (fWrite)
+        rcStrict = iemVmxVirtApicAccessMsrWrite(pVCpu, idMsr, *pu64Value);
+    else
+        rcStrict = iemVmxVirtApicAccessMsrRead(pVCpu, idMsr, pu64Value);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+
+}
+
+
+/**
+ * Interface for HM and EM to virtualize memory-mapped APIC accesses.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_VMX_MODIFIES_BEHAVIOR if the memory access was virtualized.
+ * @retval  VINF_VMX_VMEXIT if the access causes a VM-exit.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitApicAccess(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
+{
+    Assert(pExitInfo);
+    Assert(pExitEventInfo);
+    VBOXSTRICTRC rcStrict = iemVmxVmexitApicAccessWithInfo(pVCpu, pExitInfo, pExitEventInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+
+}
+
+
+/**
+ * Interface for HM and EM to perform an APIC-write emulation which may cause a
+ * VM-exit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitApicWrite(PVMCPUCC pVCpu)
+{
+    VBOXSTRICTRC rcStrict = iemVmxApicWriteEmulation(pVCpu);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate VM-exit due to expiry of the preemption timer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitPreemptTimer(PVMCPUCC pVCpu)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate VM-exit due to external interrupts.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   uVector         The external interrupt vector (pass 0 if the external
+ *                          interrupt is still pending).
+ * @param   fIntPending     Whether the external interrupt is pending or
+ *                          acknowdledged in the interrupt controller.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitExtInt(PVMCPUCC pVCpu, uint8_t uVector, bool fIntPending)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexitExtInt(pVCpu, uVector, fIntPending);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate VM-exit due to exceptions.
+ *
+ * Exception includes NMIs, software exceptions (those generated by INT3 or
+ * INTO) and privileged software exceptions (those generated by INT1/ICEBP).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitXcpt(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
+{
+    Assert(pExitInfo);
+    Assert(pExitEventInfo);
+    VBOXSTRICTRC rcStrict = iemVmxVmexitEventWithInfo(pVCpu, pExitInfo, pExitEventInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate VM-exit due to NMIs.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitXcptNmi(PVMCPUCC pVCpu)
+{
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason = VMX_EXIT_XCPT_OR_NMI;
+
+    VMXVEXITEVENTINFO ExitEventInfo;
+    RT_ZERO(ExitEventInfo);
+    ExitEventInfo.uExitIntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID,  1)
+                               | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE,   VMX_EXIT_INT_INFO_TYPE_NMI)
+                               | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, X86_XCPT_NMI);
+
+    VBOXSTRICTRC rcStrict = iemVmxVmexitEventWithInfo(pVCpu, &ExitInfo, &ExitEventInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate VM-exit due to a triple-fault.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTripleFault(PVMCPUCC pVCpu)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate VM-exit due to startup-IPI (SIPI).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   uVector         The SIPI vector.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitStartupIpi(PVMCPUCC pVCpu, uint8_t uVector)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_SIPI, uVector);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate a VM-exit.
+ *
+ * If a specialized version of a VM-exit handler exists, that must be used instead.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   uExitReason     The VM-exit reason.
+ * @param   u64ExitQual     The Exit qualification.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, uExitReason, u64ExitQual);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate a VM-exit due to an instruction.
+ *
+ * This is meant to be used for those instructions that VMX provides additional
+ * decoding information beyond just the instruction length!
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo   Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitInstrWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate a VM-exit due to an instruction.
+ *
+ * This is meant to be used for those instructions that VMX provides only the
+ * instruction length.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo   Pointer to the VM-exit information.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitInstr(PVMCPUCC pVCpu, uint32_t uExitReason, uint8_t cbInstr)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexitInstr(pVCpu, uExitReason, cbInstr);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate a trap-like VM-exit (MTF, APIC-write,
+ * Virtualized-EOI, TPR-below threshold).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTrapLike(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    Assert(pExitInfo);
+    VBOXSTRICTRC rcStrict = iemVmxVmexitTrapLikeWithInfo(pVCpu, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate a VM-exit due to a task switch.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTaskSwitch(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
+{
+    Assert(pExitInfo);
+    Assert(pExitEventInfo);
+    Assert(pExitInfo->uReason == VMX_EXIT_TASK_SWITCH);
+    VBOXSTRICTRC rcStrict = iemVmxVmexitTaskSwitchWithInfo(pVCpu, pExitInfo, pExitEventInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMREAD instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmread(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
+    Assert(pExitInfo);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    VBOXSTRICTRC   rcStrict;
+    uint8_t const  cbInstr       = pExitInfo->cbInstr;
+    bool const     fIs64BitMode  = RT_BOOL(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
+    uint64_t const u64FieldEnc   = fIs64BitMode
+                                 ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2)
+                                 : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2);
+    if (pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand)
+    {
+        if (fIs64BitMode)
+        {
+            uint64_t *pu64Dst = iemGRegRefU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1);
+            rcStrict = iemVmxVmreadReg64(pVCpu, cbInstr, pu64Dst, u64FieldEnc, pExitInfo);
+        }
+        else
+        {
+            uint32_t *pu32Dst = iemGRegRefU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1);
+            rcStrict = iemVmxVmreadReg32(pVCpu, cbInstr, pu32Dst, u64FieldEnc, pExitInfo);
+        }
+    }
+    else
+    {
+        RTGCPTR const GCPtrDst = pExitInfo->GCPtrEffAddr;
+        uint8_t const iEffSeg  = pExitInfo->InstrInfo.VmreadVmwrite.iSegReg;
+        rcStrict = iemVmxVmreadMem(pVCpu, cbInstr, iEffSeg, GCPtrDst, u64FieldEnc, pExitInfo);
+    }
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMWRITE instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmwrite(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
+    Assert(pExitInfo);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint64_t u64Val;
+    uint8_t  iEffSeg;
+    if (pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand)
+    {
+        u64Val  = iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1);
+        iEffSeg = UINT8_MAX;
+    }
+    else
+    {
+        u64Val  = pExitInfo->GCPtrEffAddr;
+        iEffSeg = pExitInfo->InstrInfo.VmreadVmwrite.iSegReg;
+    }
+    uint8_t const  cbInstr     = pExitInfo->cbInstr;
+    uint64_t const u64FieldEnc = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+                               ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2)
+                               : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2);
+    VBOXSTRICTRC rcStrict = iemVmxVmwrite(pVCpu, cbInstr, iEffSeg, u64Val, u64FieldEnc, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMPTRLD instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmptrld(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    Assert(pExitInfo);
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint8_t const iEffSeg   = pExitInfo->InstrInfo.VmxXsave.iSegReg;
+    uint8_t const cbInstr   = pExitInfo->cbInstr;
+    RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr;
+    VBOXSTRICTRC rcStrict = iemVmxVmptrld(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMPTRST instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmptrst(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    Assert(pExitInfo);
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint8_t const iEffSeg   = pExitInfo->InstrInfo.VmxXsave.iSegReg;
+    uint8_t const cbInstr   = pExitInfo->cbInstr;
+    RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr;
+    VBOXSTRICTRC rcStrict = iemVmxVmptrst(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMCLEAR instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmclear(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    Assert(pExitInfo);
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint8_t const iEffSeg   = pExitInfo->InstrInfo.VmxXsave.iSegReg;
+    uint8_t const cbInstr   = pExitInfo->cbInstr;
+    RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr;
+    VBOXSTRICTRC rcStrict = iemVmxVmclear(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMLAUNCH/VMRESUME instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   uInstrId        The instruction ID (VMXINSTRID_VMLAUNCH or
+ *                          VMXINSTRID_VMRESUME).
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmlaunchVmresume(PVMCPUCC pVCpu, uint8_t cbInstr, VMXINSTRID uInstrId)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = iemVmxVmlaunchVmresume(pVCpu, cbInstr, uInstrId);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMXON instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmxon(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    Assert(pExitInfo);
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint8_t const iEffSeg    = pExitInfo->InstrInfo.VmxXsave.iSegReg;
+    uint8_t const cbInstr    = pExitInfo->cbInstr;
+    RTGCPTR const GCPtrVmxon = pExitInfo->GCPtrEffAddr;
+    VBOXSTRICTRC rcStrict = iemVmxVmxon(pVCpu, cbInstr, iEffSeg, GCPtrVmxon, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMXOFF instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmxoff(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmxoff);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the INVVPID instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvvpid(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 4);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
+    Assert(pExitInfo);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint8_t const  iEffSeg          = pExitInfo->InstrInfo.Inv.iSegReg;
+    uint8_t const  cbInstr          = pExitInfo->cbInstr;
+    RTGCPTR const  GCPtrInvvpidDesc = pExitInfo->GCPtrEffAddr;
+    uint64_t const u64InvvpidType   = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+                                    ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.Inv.iReg2)
+                                    : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.Inv.iReg2);
+    VBOXSTRICTRC rcStrict = iemVmxInvvpid(pVCpu, cbInstr, iEffSeg, GCPtrInvvpidDesc, u64InvvpidType, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER, VMX APIC-access page accesses}
+ *
+ * @remarks The @a pvUser argument is currently unused.
+ */
+PGM_ALL_CB2_DECL(VBOXSTRICTRC) iemVmxApicAccessPageHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void *pvPhys,
+                                                           void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType,
+                                                           PGMACCESSORIGIN enmOrigin, void *pvUser)
+{
+    RT_NOREF3(pvPhys, enmOrigin, pvUser);
+
+    RTGCPHYS const GCPhysAccessBase = GCPhysFault & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+    if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
+    {
+        Assert(CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(pVCpu), VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
+        Assert(CPUMGetGuestVmxApicAccessPageAddr(IEM_GET_CTX(pVCpu)) == GCPhysAccessBase);
+
+        /** @todo NSTVMX: How are we to distinguish instruction fetch accesses here?
+         *        Currently they will go through as read accesses. */
+        uint32_t const fAccess   = enmAccessType == PGMACCESSTYPE_WRITE ? IEM_ACCESS_TYPE_WRITE : IEM_ACCESS_TYPE_READ;
+        uint16_t const offAccess = GCPhysFault & PAGE_OFFSET_MASK;
+        VBOXSTRICTRC rcStrict = iemVmxVirtApicAccessMem(pVCpu, offAccess, cbBuf, pvBuf, fAccess);
+        if (RT_FAILURE(rcStrict))
+            return rcStrict;
+
+        /* Any access on this APIC-access page has been handled, caller should not carry out the access. */
+        return VINF_SUCCESS;
+    }
+
+    Log(("iemVmxApicAccessPageHandler: Access outside VMX non-root mode, deregistering page at %#RGp\n", GCPhysAccessBase));
+    int rc = PGMHandlerPhysicalDeregister(pVM, GCPhysAccessBase);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* Instruct the caller of this handler to perform the read/write as normal memory. */
+    return VINF_PGM_HANDLER_DO_DEFAULT;
+}
+
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+#ifdef IN_RING3
+
+/**
+ * Handles the unlikely and probably fatal merge cases.
+ *
+ * @returns Merged status code.
+ * @param   rcStrict        Current EM status code.
+ * @param   rcStrictCommit  The IOM I/O or MMIO write commit status to merge
+ *                          with @a rcStrict.
+ * @param   iMemMap         The memory mapping index. For error reporting only.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          thread, for error reporting only.
+ */
+DECL_NO_INLINE(static, VBOXSTRICTRC) iemR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit,
+                                                          unsigned iMemMap, PVMCPUCC pVCpu)
+{
+    if (RT_FAILURE_NP(rcStrict))
+        return rcStrict;
+
+    if (RT_FAILURE_NP(rcStrictCommit))
+        return rcStrictCommit;
+
+    if (rcStrict == rcStrictCommit)
+        return rcStrictCommit;
+
+    AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc iMemMap=%u fAccess=%#x FirstPg=%RGp LB %u SecondPg=%RGp LB %u\n",
+                           VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict), iMemMap,
+                           pVCpu->iem.s.aMemMappings[iMemMap].fAccess,
+                           pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst,
+                           pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond));
+    return VERR_IOM_FF_STATUS_IPE;
+}
+
+
+/**
+ * Helper for IOMR3ProcessForceFlag.
+ *
+ * @returns Merged status code.
+ * @param   rcStrict        Current EM status code.
+ * @param   rcStrictCommit  The IOM I/O or MMIO write commit status to merge
+ *                          with @a rcStrict.
+ * @param   iMemMap         The memory mapping index. For error reporting only.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          thread, for error reporting only.
+ */
+DECLINLINE(VBOXSTRICTRC) iemR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, unsigned iMemMap, PVMCPUCC pVCpu)
+{
+    /* Simple. */
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RAW_TO_R3))
+        return rcStrictCommit;
+
+    if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS))
+        return rcStrict;
+
+    /* EM scheduling status codes. */
+    if (RT_LIKELY(   rcStrict >= VINF_EM_FIRST
+                  && rcStrict <= VINF_EM_LAST))
+    {
+        if (RT_LIKELY(   rcStrictCommit >= VINF_EM_FIRST
+                      && rcStrictCommit <= VINF_EM_LAST))
+            return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit;
+    }
+
+    /* Unlikely */
+    return iemR3MergeStatusSlow(rcStrict, rcStrictCommit, iMemMap, pVCpu);
+}
+
+
+/**
+ * Called by force-flag handling code when VMCPU_FF_IEM is set.
+ *
+ * @returns Merge between @a rcStrict and what the commit operation returned.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   rcStrict    The status code returned by ring-0 or raw-mode.
+ */
+VMMR3_INT_DECL(VBOXSTRICTRC) IEMR3ProcessForceFlag(PVM pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
+{
+    /*
+     * Reset the pending commit.
+     */
+    AssertMsg(  (pVCpu->iem.s.aMemMappings[0].fAccess | pVCpu->iem.s.aMemMappings[1].fAccess | pVCpu->iem.s.aMemMappings[2].fAccess)
+              & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND),
+              ("%#x %#x %#x\n",
+               pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IEM);
+
+    /*
+     * Commit the pending bounce buffers (usually just one).
+     */
+    unsigned cBufs = 0;
+    unsigned iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
+    while (iMemMap-- > 0)
+        if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND))
+        {
+            Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
+            Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
+            Assert(!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned);
+
+            uint16_t const  cbFirst  = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
+            uint16_t const  cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
+            uint8_t const  *pbBuf    = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
+
+            if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_1ST)
+            {
+                VBOXSTRICTRC rcStrictCommit1 = PGMPhysWrite(pVM,
+                                                            pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
+                                                            pbBuf,
+                                                            cbFirst,
+                                                            PGMACCESSORIGIN_IEM);
+                rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit1, iMemMap, pVCpu);
+                Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysFirst=%RGp LB %#x %Rrc => %Rrc\n",
+                     iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                     VBOXSTRICTRC_VAL(rcStrictCommit1), VBOXSTRICTRC_VAL(rcStrict)));
+            }
+
+            if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_2ND)
+            {
+                VBOXSTRICTRC rcStrictCommit2 = PGMPhysWrite(pVM,
+                                                            pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
+                                                            pbBuf + cbFirst,
+                                                            cbSecond,
+                                                            PGMACCESSORIGIN_IEM);
+                rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit2, iMemMap, pVCpu);
+                Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysSecond=%RGp LB %#x %Rrc => %Rrc\n",
+                     iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond,
+                     VBOXSTRICTRC_VAL(rcStrictCommit2), VBOXSTRICTRC_VAL(rcStrict)));
+            }
+            cBufs++;
+            pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+        }
+
+    AssertMsg(cBufs > 0 && cBufs == pVCpu->iem.s.cActiveMappings,
+              ("cBufs=%u cActiveMappings=%u - %#x %#x %#x\n", cBufs, pVCpu->iem.s.cActiveMappings,
+               pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
+    pVCpu->iem.s.cActiveMappings = 0;
+    return rcStrict;
+}
+
+#endif /* IN_RING3 */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllAImpl.asm b/src/VBox/VMM/VMMAll/IEMAllAImpl.asm
new file mode 100644
index 00000000..fc4ed29a
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllAImpl.asm
@@ -0,0 +1,3024 @@
+; $Id: IEMAllAImpl.asm $
+;; @file
+; IEM - Instruction Implementation in Assembly.
+;
+
+;
+; Copyright (C) 2011-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;   Header Files                                                               ;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%include "VBox/asmdefs.mac"
+%include "VBox/err.mac"
+%include "iprt/x86.mac"
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;   Defined Constants And Macros                                               ;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;
+; RET XX / RET wrapper for fastcall.
+;
+%macro RET_FASTCALL 1
+%ifdef RT_ARCH_X86
+ %ifdef RT_OS_WINDOWS
+    ret %1
+ %else
+    ret
+ %endif
+%else
+    ret
+%endif
+%endmacro
+
+;;
+; NAME for fastcall functions.
+;
+;; @todo 'global @fastcall@12' is still broken in yasm and requires dollar
+;         escaping (or whatever the dollar is good for here).  Thus the ugly
+;         prefix argument.
+;
+%define NAME_FASTCALL(a_Name, a_cbArgs, a_Prefix)   NAME(a_Name)
+%ifdef RT_ARCH_X86
+ %ifdef RT_OS_WINDOWS
+  %undef NAME_FASTCALL
+  %define NAME_FASTCALL(a_Name, a_cbArgs, a_Prefix) a_Prefix %+ a_Name %+ @ %+ a_cbArgs
+ %endif
+%endif
+
+;;
+; BEGINPROC for fastcall functions.
+;
+; @param        1       The function name (C).
+; @param        2       The argument size on x86.
+;
+%macro BEGINPROC_FASTCALL 2
+ %ifdef ASM_FORMAT_PE
+  export %1=NAME_FASTCALL(%1,%2,$@)
+ %endif
+ %ifdef __NASM__
+  %ifdef ASM_FORMAT_OMF
+   export NAME(%1) NAME_FASTCALL(%1,%2,$@)
+  %endif
+ %endif
+ %ifndef ASM_FORMAT_BIN
+  global NAME_FASTCALL(%1,%2,$@)
+ %endif
+NAME_FASTCALL(%1,%2,@):
+%endmacro
+
+
+;
+; We employ some macro assembly here to hid the calling convention differences.
+;
+%ifdef RT_ARCH_AMD64
+ %macro PROLOGUE_1_ARGS 0
+ %endmacro
+ %macro EPILOGUE_1_ARGS 0
+        ret
+ %endmacro
+ %macro EPILOGUE_1_ARGS_EX 0
+        ret
+ %endmacro
+
+ %macro PROLOGUE_2_ARGS 0
+ %endmacro
+ %macro EPILOGUE_2_ARGS 0
+        ret
+ %endmacro
+ %macro EPILOGUE_2_ARGS_EX 1
+        ret
+ %endmacro
+
+ %macro PROLOGUE_3_ARGS 0
+ %endmacro
+ %macro EPILOGUE_3_ARGS 0
+        ret
+ %endmacro
+ %macro EPILOGUE_3_ARGS_EX 1
+        ret
+ %endmacro
+
+ %macro PROLOGUE_4_ARGS 0
+ %endmacro
+ %macro EPILOGUE_4_ARGS 0
+        ret
+ %endmacro
+ %macro EPILOGUE_4_ARGS_EX 1
+        ret
+ %endmacro
+
+ %ifdef ASM_CALL64_GCC
+  %define A0        rdi
+  %define A0_32     edi
+  %define A0_16     di
+  %define A0_8      dil
+
+  %define A1        rsi
+  %define A1_32     esi
+  %define A1_16     si
+  %define A1_8      sil
+
+  %define A2        rdx
+  %define A2_32     edx
+  %define A2_16     dx
+  %define A2_8      dl
+
+  %define A3        rcx
+  %define A3_32     ecx
+  %define A3_16     cx
+ %endif
+
+ %ifdef ASM_CALL64_MSC
+  %define A0        rcx
+  %define A0_32     ecx
+  %define A0_16     cx
+  %define A0_8      cl
+
+  %define A1        rdx
+  %define A1_32     edx
+  %define A1_16     dx
+  %define A1_8      dl
+
+  %define A2        r8
+  %define A2_32     r8d
+  %define A2_16     r8w
+  %define A2_8      r8b
+
+  %define A3        r9
+  %define A3_32     r9d
+  %define A3_16     r9w
+ %endif
+
+ %define T0         rax
+ %define T0_32      eax
+ %define T0_16      ax
+ %define T0_8       al
+
+ %define T1         r11
+ %define T1_32      r11d
+ %define T1_16      r11w
+ %define T1_8       r11b
+
+%else
+ ; x86
+ %macro PROLOGUE_1_ARGS 0
+        push    edi
+ %endmacro
+ %macro EPILOGUE_1_ARGS 0
+        pop     edi
+        ret     0
+ %endmacro
+ %macro EPILOGUE_1_ARGS_EX 1
+        pop     edi
+        ret     %1
+ %endmacro
+
+ %macro PROLOGUE_2_ARGS 0
+        push    edi
+ %endmacro
+ %macro EPILOGUE_2_ARGS 0
+        pop     edi
+        ret     0
+ %endmacro
+ %macro EPILOGUE_2_ARGS_EX 1
+        pop     edi
+        ret     %1
+ %endmacro
+
+ %macro PROLOGUE_3_ARGS 0
+        push    ebx
+        mov     ebx, [esp + 4 + 4]
+        push    edi
+ %endmacro
+ %macro EPILOGUE_3_ARGS_EX 1
+  %if (%1) < 4
+   %error "With three args, at least 4 bytes must be remove from the stack upon return (32-bit)."
+  %endif
+        pop     edi
+        pop     ebx
+        ret     %1
+ %endmacro
+ %macro EPILOGUE_3_ARGS 0
+        EPILOGUE_3_ARGS_EX 4
+ %endmacro
+
+ %macro PROLOGUE_4_ARGS 0
+        push    ebx
+        push    edi
+        push    esi
+        mov     ebx, [esp + 12 + 4 + 0]
+        mov     esi, [esp + 12 + 4 + 4]
+ %endmacro
+ %macro EPILOGUE_4_ARGS_EX 1
+  %if (%1) < 8
+   %error "With four args, at least 8 bytes must be remove from the stack upon return (32-bit)."
+  %endif
+        pop     esi
+        pop     edi
+        pop     ebx
+        ret     %1
+ %endmacro
+ %macro EPILOGUE_4_ARGS 0
+        EPILOGUE_4_ARGS_EX 8
+ %endmacro
+
+ %define A0         ecx
+ %define A0_32      ecx
+ %define A0_16       cx
+ %define A0_8        cl
+
+ %define A1         edx
+ %define A1_32      edx
+ %define A1_16      dx
+ %define A1_8       dl
+
+ %define A2         ebx
+ %define A2_32      ebx
+ %define A2_16      bx
+ %define A2_8       bl
+
+ %define A3         esi
+ %define A3_32      esi
+ %define A3_16      si
+
+ %define T0         eax
+ %define T0_32      eax
+ %define T0_16      ax
+ %define T0_8       al
+
+ %define T1         edi
+ %define T1_32      edi
+ %define T1_16      di
+%endif
+
+
+;;
+; Load the relevant flags from [%1] if there are undefined flags (%3).
+;
+; @remarks      Clobbers T0, stack. Changes EFLAGS.
+; @param        A2      The register pointing to the flags.
+; @param        1       The parameter (A0..A3) pointing to the eflags.
+; @param        2       The set of modified flags.
+; @param        3       The set of undefined flags.
+;
+%macro IEM_MAYBE_LOAD_FLAGS 3
+ ;%if (%3) != 0
+        pushf                           ; store current flags
+        mov     T0_32, [%1]             ; load the guest flags
+        and     dword [xSP], ~(%2 | %3) ; mask out the modified and undefined flags
+        and     T0_32, (%2 | %3)        ; select the modified and undefined flags.
+        or      [xSP], T0               ; merge guest flags with host flags.
+        popf                            ; load the mixed flags.
+ ;%endif
+%endmacro
+
+;;
+; Update the flag.
+;
+; @remarks  Clobbers T0, T1, stack.
+; @param        1       The register pointing to the EFLAGS.
+; @param        2       The mask of modified flags to save.
+; @param        3       The mask of undefined flags to (maybe) save.
+;
+%macro IEM_SAVE_FLAGS 3
+ %if (%2 | %3) != 0
+        pushf
+        pop     T1
+        mov     T0_32, [%1]             ; flags
+        and     T0_32, ~(%2 | %3)       ; clear the modified & undefined flags.
+        and     T1_32, (%2 | %3)        ; select the modified and undefined flags.
+        or      T0_32, T1_32            ; combine the flags.
+        mov     [%1], T0_32             ; save the flags.
+ %endif
+%endmacro
+
+
+;;
+; Macro for implementing a binary operator.
+;
+; This will generate code for the 8, 16, 32 and 64 bit accesses with locked
+; variants, except on 32-bit system where the 64-bit accesses requires hand
+; coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the source register operand in A1 and a pointer to eflags in A2.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       Non-zero if there should be a locked version.
+; @param        3       The modified flags.
+; @param        4       The undefined flags.
+;
+%macro IEMIMPL_BIN_OP 4
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      byte [A0], A1_8
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u8
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      word [A0], A1_16
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      dword [A0], A1_32
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      qword [A0], A1
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+  %endif ; RT_ARCH_AMD64
+
+ %if %2 != 0 ; locked versions requested?
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 byte [A0], A1_8
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u8_locked
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 word [A0], A1_16
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16_locked
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 dword [A0], A1_32
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32_locked
+
+  %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 qword [A0], A1
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_ %+ %1 %+ _u64_locked
+  %endif ; RT_ARCH_AMD64
+ %endif ; locked
+%endmacro
+
+;            instr,lock,modified-flags.
+IEMIMPL_BIN_OP add,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_BIN_OP adc,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_BIN_OP sub,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_BIN_OP sbb,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_BIN_OP or,   1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), X86_EFL_AF
+IEMIMPL_BIN_OP xor,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), X86_EFL_AF
+IEMIMPL_BIN_OP and,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), X86_EFL_AF
+IEMIMPL_BIN_OP cmp,  0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_BIN_OP test, 0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), X86_EFL_AF
+
+
+;;
+; Macro for implementing a bit operator.
+;
+; This will generate code for the 16, 32 and 64 bit accesses with locked
+; variants, except on 32-bit system where the 64-bit accesses requires hand
+; coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the source register operand in A1 and a pointer to eflags in A2.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       Non-zero if there should be a locked version.
+; @param        3       The modified flags.
+; @param        4       The undefined flags.
+;
+%macro IEMIMPL_BIT_OP 4
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      word [A0], A1_16
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      dword [A0], A1_32
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      qword [A0], A1
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+  %endif ; RT_ARCH_AMD64
+
+ %if %2 != 0 ; locked versions requested?
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 word [A0], A1_16
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16_locked
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 dword [A0], A1_32
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32_locked
+
+  %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 qword [A0], A1
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_ %+ %1 %+ _u64_locked
+  %endif ; RT_ARCH_AMD64
+ %endif ; locked
+%endmacro
+IEMIMPL_BIT_OP bt,  0, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_BIT_OP btc, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_BIT_OP bts, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_BIT_OP btr, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+
+;;
+; Macro for implementing a bit search operator.
+;
+; This will generate code for the 16, 32 and 64 bit accesses, except on 32-bit
+; system where the 64-bit accesses requires hand coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the source register operand in A1 and a pointer to eflags in A2.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+;
+%macro IEMIMPL_BIT_OP 3
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        %1      T0_16, A1_16
+        jz      .unchanged_dst
+        mov     [A0], T0_16
+.unchanged_dst:
+        IEM_SAVE_FLAGS                 A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        %1      T0_32, A1_32
+        jz      .unchanged_dst
+        mov     [A0], T0_32
+.unchanged_dst:
+        IEM_SAVE_FLAGS                 A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        %1      T0, A1
+        jz      .unchanged_dst
+        mov     [A0], T0
+.unchanged_dst:
+        IEM_SAVE_FLAGS                 A2, %2, %3
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif ; RT_ARCH_AMD64
+%endmacro
+IEMIMPL_BIT_OP bsf, (X86_EFL_ZF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF)
+IEMIMPL_BIT_OP bsr, (X86_EFL_ZF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF)
+
+
+;
+; IMUL is also a similar but yet different case (no lock, no mem dst).
+; The rDX:rAX variant of imul is handled together with mul further down.
+;
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_imul_two_u16, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+        imul    A1_16, word [A0]
+        mov     [A0], A1_16
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_imul_two_u16
+
+BEGINPROC_FASTCALL iemAImpl_imul_two_u32, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+        imul    A1_32, dword [A0]
+        mov     [A0], A1_32
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_imul_two_u32
+
+%ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_imul_two_u64, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+        imul    A1, qword [A0]
+        mov     [A0], A1
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_imul_two_u64
+%endif ; RT_ARCH_AMD64
+
+
+;
+; XCHG for memory operands.  This implies locking.  No flag changes.
+;
+; Each function takes two arguments, first the pointer to the memory,
+; then the pointer to the register.  They all return void.
+;
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_xchg_u8, 8
+        PROLOGUE_2_ARGS
+        mov     T0_8, [A1]
+        xchg    [A0], T0_8
+        mov     [A1], T0_8
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_xchg_u8
+
+BEGINPROC_FASTCALL iemAImpl_xchg_u16, 8
+        PROLOGUE_2_ARGS
+        mov     T0_16, [A1]
+        xchg    [A0], T0_16
+        mov     [A1], T0_16
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_xchg_u16
+
+BEGINPROC_FASTCALL iemAImpl_xchg_u32, 8
+        PROLOGUE_2_ARGS
+        mov     T0_32, [A1]
+        xchg    [A0], T0_32
+        mov     [A1], T0_32
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_xchg_u32
+
+%ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_xchg_u64, 8
+        PROLOGUE_2_ARGS
+        mov     T0, [A1]
+        xchg    [A0], T0
+        mov     [A1], T0
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_xchg_u64
+%endif
+
+
+;
+; XADD for memory operands.
+;
+; Each function takes three arguments, first the pointer to the
+; memory/register, then the pointer to the register, and finally a pointer to
+; eflags.  They all return void.
+;
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_xadd_u8, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0_8, [A1]
+        xadd    [A0], T0_8
+        mov     [A1], T0_8
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u8
+
+BEGINPROC_FASTCALL iemAImpl_xadd_u16, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0_16, [A1]
+        xadd    [A0], T0_16
+        mov     [A1], T0_16
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u16
+
+BEGINPROC_FASTCALL iemAImpl_xadd_u32, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0_32, [A1]
+        xadd    [A0], T0_32
+        mov     [A1], T0_32
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u32
+
+%ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_xadd_u64, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0, [A1]
+        xadd    [A0], T0
+        mov     [A1], T0
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u64
+%endif ; RT_ARCH_AMD64
+
+BEGINPROC_FASTCALL iemAImpl_xadd_u8_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0_8, [A1]
+        lock xadd [A0], T0_8
+        mov     [A1], T0_8
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u8_locked
+
+BEGINPROC_FASTCALL iemAImpl_xadd_u16_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0_16, [A1]
+        lock xadd [A0], T0_16
+        mov     [A1], T0_16
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u16_locked
+
+BEGINPROC_FASTCALL iemAImpl_xadd_u32_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0_32, [A1]
+        lock xadd [A0], T0_32
+        mov     [A1], T0_32
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u32_locked
+
+%ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_xadd_u64_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0, [A1]
+        lock xadd [A0], T0
+        mov     [A1], T0
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u64_locked
+%endif ; RT_ARCH_AMD64
+
+
+;
+; CMPXCHG8B.
+;
+; These are tricky register wise, so the code is duplicated for each calling
+; convention.
+;
+; WARNING! This code make ASSUMPTIONS about which registers T1 and T0 are mapped to!
+;
+; C-proto:
+; IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
+;                                             uint32_t *pEFlags));
+;
+; Note! Identical to iemAImpl_cmpxchg16b.
+;
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_cmpxchg8b, 16
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_MSC
+        push    rbx
+
+        mov     r11, rdx                ; pu64EaxEdx (is also T1)
+        mov     r10, rcx                ; pu64Dst
+
+        mov     ebx, [r8]
+        mov     ecx, [r8 + 4]
+        IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
+        mov     eax, [r11]
+        mov     edx, [r11 + 4]
+
+        lock cmpxchg8b [r10]
+
+        mov     [r11], eax
+        mov     [r11 + 4], edx
+        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+
+        pop     rbx
+        ret
+ %else
+        push    rbx
+
+        mov     r10, rcx                ; pEFlags
+        mov     r11, rdx                ; pu64EbxEcx (is also T1)
+
+        mov     ebx, [r11]
+        mov     ecx, [r11 + 4]
+        IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
+        mov     eax, [rsi]
+        mov     edx, [rsi + 4]
+
+        lock cmpxchg8b [rdi]
+
+        mov     [rsi], eax
+        mov     [rsi + 4], edx
+        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+
+        pop     rbx
+        ret
+
+ %endif
+%else
+        push    esi
+        push    edi
+        push    ebx
+        push    ebp
+
+        mov     edi, ecx                ; pu64Dst
+        mov     esi, edx                ; pu64EaxEdx
+        mov     ecx, [esp + 16 + 4 + 0] ; pu64EbxEcx
+        mov     ebp, [esp + 16 + 4 + 4] ; pEFlags
+
+        mov     ebx, [ecx]
+        mov     ecx, [ecx + 4]
+        IEM_MAYBE_LOAD_FLAGS ebp, (X86_EFL_ZF), 0  ; clobbers T0 (eax)
+        mov     eax, [esi]
+        mov     edx, [esi + 4]
+
+        lock cmpxchg8b [edi]
+
+        mov     [esi], eax
+        mov     [esi + 4], edx
+        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, edi)
+
+        pop     ebp
+        pop     ebx
+        pop     edi
+        pop     esi
+        ret     8
+%endif
+ENDPROC iemAImpl_cmpxchg8b
+
+BEGINPROC_FASTCALL iemAImpl_cmpxchg8b_locked, 16
+        ; Lazy bird always lock prefixes cmpxchg8b.
+        jmp     NAME_FASTCALL(iemAImpl_cmpxchg8b,16,$@)
+ENDPROC iemAImpl_cmpxchg8b_locked
+
+%ifdef RT_ARCH_AMD64
+
+;
+; CMPXCHG16B.
+;
+; These are tricky register wise, so the code is duplicated for each calling
+; convention.
+;
+; WARNING! This code make ASSUMPTIONS about which registers T1 and T0 are mapped to!
+;
+; C-proto:
+; IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu1284RaxRdx, PRTUINT128U pu128RbxRcx,
+;                                              uint32_t *pEFlags));
+;
+; Note! Identical to iemAImpl_cmpxchg8b.
+;
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_cmpxchg16b, 16
+ %ifdef ASM_CALL64_MSC
+        push    rbx
+
+        mov     r11, rdx                ; pu64RaxRdx (is also T1)
+        mov     r10, rcx                ; pu64Dst
+
+        mov     rbx, [r8]
+        mov     rcx, [r8 + 8]
+        IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
+        mov     rax, [r11]
+        mov     rdx, [r11 + 8]
+
+        lock cmpxchg16b [r10]
+
+        mov     [r11], rax
+        mov     [r11 + 8], rdx
+        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+
+        pop     rbx
+        ret
+ %else
+        push    rbx
+
+        mov     r10, rcx                ; pEFlags
+        mov     r11, rdx                ; pu64RbxRcx (is also T1)
+
+        mov     rbx, [r11]
+        mov     rcx, [r11 + 8]
+        IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
+        mov     rax, [rsi]
+        mov     rdx, [rsi + 8]
+
+        lock cmpxchg16b [rdi]
+
+        mov     [rsi], eax
+        mov     [rsi + 8], edx
+        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+
+        pop     rbx
+        ret
+
+ %endif
+ENDPROC iemAImpl_cmpxchg16b
+
+BEGINPROC_FASTCALL iemAImpl_cmpxchg16b_locked, 16
+        ; Lazy bird always lock prefixes cmpxchg8b.
+        jmp     NAME_FASTCALL(iemAImpl_cmpxchg16b,16,$@)
+ENDPROC iemAImpl_cmpxchg16b_locked
+
+%endif ; RT_ARCH_AMD64
+
+
+;
+; CMPXCHG.
+;
+; WARNING! This code make ASSUMPTIONS about which registers T1 and T0 are mapped to!
+;
+; C-proto:
+; IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg,(uintX_t *puXDst, uintX_t puEax, uintX_t uReg, uint32_t *pEFlags));
+;
+BEGINCODE
+%macro IEMIMPL_CMPXCHG 2
+BEGINPROC_FASTCALL iemAImpl_cmpxchg_u8 %+ %2, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax)
+        mov     al, [A1]
+        %1 cmpxchg [A0], A2_8
+        mov     [A1], al
+        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cmpxchg_u8 %+ %2
+
+BEGINPROC_FASTCALL iemAImpl_cmpxchg_u16 %+ %2, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax)
+        mov     ax, [A1]
+        %1 cmpxchg [A0], A2_16
+        mov     [A1], ax
+        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cmpxchg_u16 %+ %2
+
+BEGINPROC_FASTCALL iemAImpl_cmpxchg_u32 %+ %2, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax)
+        mov     eax, [A1]
+        %1 cmpxchg [A0], A2_32
+        mov     [A1], eax
+        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cmpxchg_u32 %+ %2
+
+BEGINPROC_FASTCALL iemAImpl_cmpxchg_u64 %+ %2, 16
+%ifdef RT_ARCH_AMD64
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax)
+        mov     rax, [A1]
+        %1 cmpxchg [A0], A2
+        mov     [A1], rax
+        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
+        EPILOGUE_4_ARGS
+%else
+        ;
+        ; Must use cmpxchg8b here. See also iemAImpl_cmpxchg8b.
+        ;
+        push    esi
+        push    edi
+        push    ebx
+        push    ebp
+
+        mov     edi, ecx                ; pu64Dst
+        mov     esi, edx                ; pu64Rax
+        mov     ecx, [esp + 16 + 4 + 0] ; pu64Reg - Note! Pointer on 32-bit hosts!
+        mov     ebp, [esp + 16 + 4 + 4] ; pEFlags
+
+        mov     ebx, [ecx]
+        mov     ecx, [ecx + 4]
+        IEM_MAYBE_LOAD_FLAGS ebp, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0  ; clobbers T0 (eax)
+        mov     eax, [esi]
+        mov     edx, [esi + 4]
+
+        lock cmpxchg8b [edi]
+
+        ; cmpxchg8b doesn't set CF, PF, AF, SF and OF, so we have to do that.
+        jz      .cmpxchg8b_not_equal
+        cmp     eax, eax                ; just set the other flags.
+.store:
+        mov     [esi], eax
+        mov     [esi + 4], edx
+        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, edi)
+
+        pop     ebp
+        pop     ebx
+        pop     edi
+        pop     esi
+        ret     8
+
+.cmpxchg8b_not_equal:
+        cmp     [esi + 4], edx          ;; @todo FIXME - verify 64-bit compare implementation
+        jne     .store
+        cmp     [esi], eax
+        jmp     .store
+
+%endif
+ENDPROC iemAImpl_cmpxchg_u64 %+ %2
+%endmacro ; IEMIMPL_CMPXCHG
+
+IEMIMPL_CMPXCHG , ,
+IEMIMPL_CMPXCHG lock, _locked
+
+;;
+; Macro for implementing a unary operator.
+;
+; This will generate code for the 8, 16, 32 and 64 bit accesses with locked
+; variants, except on 32-bit system where the 64-bit accesses requires hand
+; coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the source register operand in A1 and a pointer to eflags in A2.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+;
+%macro IEMIMPL_UNARY_OP 3
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        %1      byte [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u8
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8_locked, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        lock %1 byte [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u8_locked
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        %1      word [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        lock %1 word [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16_locked
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        %1      dword [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        lock %1 dword [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32_locked
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        %1      qword [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        lock %1 qword [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64_locked
+ %endif ; RT_ARCH_AMD64
+
+%endmacro
+
+IEMIMPL_UNARY_OP inc, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF), 0
+IEMIMPL_UNARY_OP dec, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF), 0
+IEMIMPL_UNARY_OP neg, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_UNARY_OP not, 0, 0
+
+
+;;
+; Macro for implementing memory fence operation.
+;
+; No return value, no operands or anything.
+;
+; @param        1      The instruction.
+;
+%macro IEMIMPL_MEM_FENCE 1
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1, 0
+        %1
+        ret
+ENDPROC iemAImpl_ %+ %1
+%endmacro
+
+IEMIMPL_MEM_FENCE lfence
+IEMIMPL_MEM_FENCE sfence
+IEMIMPL_MEM_FENCE mfence
+
+;;
+; Alternative for non-SSE2 host.
+;
+BEGINPROC_FASTCALL iemAImpl_alt_mem_fence, 0
+        push    xAX
+        xchg    xAX, [xSP]
+        add     xSP, xCB
+        ret
+ENDPROC iemAImpl_alt_mem_fence
+
+
+
+;;
+; Macro for implementing a shift operation.
+;
+; This will generate code for the 8, 16, 32 and 64 bit accesses, except on
+; 32-bit system where the 64-bit accesses requires hand coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the shift count in A1 and a pointer to eflags in A2.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+;
+; Makes ASSUMPTIONS about A0, A1 and A2 assignments.
+;
+%macro IEMIMPL_SHIFT_OP 3
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
+ %ifdef ASM_CALL64_GCC
+        mov     cl, A1_8
+        %1      byte [A0], cl
+ %else
+        xchg    A1, A0
+        %1      byte [A1], cl
+ %endif
+        IEM_SAVE_FLAGS       A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u8
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
+ %ifdef ASM_CALL64_GCC
+        mov     cl, A1_8
+        %1      word [A0], cl
+ %else
+        xchg    A1, A0
+        %1      word [A1], cl
+ %endif
+        IEM_SAVE_FLAGS       A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
+ %ifdef ASM_CALL64_GCC
+        mov     cl, A1_8
+        %1      dword [A0], cl
+ %else
+        xchg    A1, A0
+        %1      dword [A1], cl
+ %endif
+        IEM_SAVE_FLAGS       A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
+ %ifdef ASM_CALL64_GCC
+        mov     cl, A1_8
+        %1      qword [A0], cl
+ %else
+        xchg    A1, A0
+        %1      qword [A1], cl
+ %endif
+        IEM_SAVE_FLAGS       A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif ; RT_ARCH_AMD64
+
+%endmacro
+
+IEMIMPL_SHIFT_OP rol, (X86_EFL_OF | X86_EFL_CF), 0
+IEMIMPL_SHIFT_OP ror, (X86_EFL_OF | X86_EFL_CF), 0
+IEMIMPL_SHIFT_OP rcl, (X86_EFL_OF | X86_EFL_CF), 0
+IEMIMPL_SHIFT_OP rcr, (X86_EFL_OF | X86_EFL_CF), 0
+IEMIMPL_SHIFT_OP shl, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
+IEMIMPL_SHIFT_OP shr, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
+IEMIMPL_SHIFT_OP sar, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
+
+
+;;
+; Macro for implementing a double precision shift operation.
+;
+; This will generate code for the 16, 32 and 64 bit accesses, except on
+; 32-bit system where the 64-bit accesses requires hand coding.
+;
+; The functions takes the destination operand (r/m) in A0, the source (reg) in
+; A1, the shift count in A2 and a pointer to the eflags variable/register in A3.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+;
+; Makes ASSUMPTIONS about A0, A1, A2 and A3 assignments.
+;
+%macro IEMIMPL_SHIFT_DBL_OP 3
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+ %ifdef ASM_CALL64_GCC
+        xchg    A3, A2
+        %1      [A0], A1_16, cl
+        xchg    A3, A2
+ %else
+        xchg    A0, A2
+        %1      [A2], A1_16, cl
+ %endif
+        IEM_SAVE_FLAGS       A3, %2, %3
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+ %ifdef ASM_CALL64_GCC
+        xchg    A3, A2
+        %1      [A0], A1_32, cl
+        xchg    A3, A2
+ %else
+        xchg    A0, A2
+        %1      [A2], A1_32, cl
+ %endif
+        IEM_SAVE_FLAGS       A3, %2, %3
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 20
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+ %ifdef ASM_CALL64_GCC
+        xchg    A3, A2
+        %1      [A0], A1, cl
+        xchg    A3, A2
+ %else
+        xchg    A0, A2
+        %1      [A2], A1, cl
+ %endif
+        IEM_SAVE_FLAGS       A3, %2, %3
+        EPILOGUE_4_ARGS_EX 12
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif ; RT_ARCH_AMD64
+
+%endmacro
+
+IEMIMPL_SHIFT_DBL_OP shld, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
+IEMIMPL_SHIFT_DBL_OP shrd, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
+
+
+;;
+; Macro for implementing a multiplication operations.
+;
+; This will generate code for the 8, 16, 32 and 64 bit accesses, except on
+; 32-bit system where the 64-bit accesses requires hand coding.
+;
+; The 8-bit function only operates on AX, so it takes no DX pointer.  The other
+; functions takes a pointer to rAX in A0, rDX in A1, the operand in A2 and a
+; pointer to eflags in A3.
+;
+; The functions all return 0 so the caller can be used for div/idiv as well as
+; for the mul/imul implementation.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+;
+; Makes ASSUMPTIONS about A0, A1, A2, A3, T0 and T1 assignments.
+;
+%macro IEMIMPL_MUL_OP 3
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
+        mov     al, [A0]
+        %1      A1_8
+        mov     [A0], ax
+        IEM_SAVE_FLAGS       A2, %2, %3
+        xor     eax, eax
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u8
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        mov     ax, [A0]
+ %ifdef ASM_CALL64_GCC
+        %1      A2_16
+        mov     [A0], ax
+        mov     [A1], dx
+ %else
+        mov     T1, A1
+        %1      A2_16
+        mov     [A0], ax
+        mov     [T1], dx
+ %endif
+        IEM_SAVE_FLAGS       A3, %2, %3
+        xor     eax, eax
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        mov     eax, [A0]
+ %ifdef ASM_CALL64_GCC
+        %1      A2_32
+        mov     [A0], eax
+        mov     [A1], edx
+ %else
+        mov     T1, A1
+        %1      A2_32
+        mov     [A0], eax
+        mov     [T1], edx
+ %endif
+        IEM_SAVE_FLAGS       A3, %2, %3
+        xor     eax, eax
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64 ; The 32-bit host version lives in IEMAllAImplC.cpp.
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 20
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        mov     rax, [A0]
+  %ifdef ASM_CALL64_GCC
+        %1      A2
+        mov     [A0], rax
+        mov     [A1], rdx
+  %else
+        mov     T1, A1
+        %1      A2
+        mov     [A0], rax
+        mov     [T1], rdx
+  %endif
+        IEM_SAVE_FLAGS       A3, %2, %3
+        xor     eax, eax
+        EPILOGUE_4_ARGS_EX 12
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif ; !RT_ARCH_AMD64
+
+%endmacro
+
+IEMIMPL_MUL_OP mul,  (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_MUL_OP imul, (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+
+
+BEGINCODE
+;;
+; Worker function for negating a 32-bit number in T1:T0
+; @uses None (T0,T1)
+iemAImpl_negate_T0_T1_u32:
+        push    0
+        push    0
+        xchg    T0_32, [xSP]
+        xchg    T1_32, [xSP + xCB]
+        sub     T0_32, [xSP]
+        sbb     T1_32, [xSP + xCB]
+        add     xSP, xCB*2
+        ret
+
+%ifdef RT_ARCH_AMD64
+;;
+; Worker function for negating a 64-bit number in T1:T0
+; @uses None (T0,T1)
+iemAImpl_negate_T0_T1_u64:
+        push    0
+        push    0
+        xchg    T0, [xSP]
+        xchg    T1, [xSP + xCB]
+        sub     T0, [xSP]
+        sbb     T1, [xSP + xCB]
+        add     xSP, xCB*2
+        ret
+%endif
+
+
+;;
+; Macro for implementing a division operations.
+;
+; This will generate code for the 8, 16, 32 and 64 bit accesses, except on
+; 32-bit system where the 64-bit accesses requires hand coding.
+;
+; The 8-bit function only operates on AX, so it takes no DX pointer.  The other
+; functions takes a pointer to rAX in A0, rDX in A1, the operand in A2 and a
+; pointer to eflags in A3.
+;
+; The functions all return 0 on success and -1 if a divide error should be
+; raised by the caller.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+; @param        4       1 if signed, 0 if unsigned.
+;
+; Makes ASSUMPTIONS about A0, A1, A2, A3, T0 and T1 assignments.
+;
+%macro IEMIMPL_DIV_OP 4
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 12
+        PROLOGUE_3_ARGS
+
+        ; div by chainsaw check.
+        test    A1_8, A1_8
+        jz      .div_zero
+
+        ; Overflow check - unsigned division is simple to verify, haven't
+        ; found a simple way to check signed division yet unfortunately.
+ %if %4 == 0
+        cmp     [A0 + 1], A1_8
+        jae     .div_overflow
+ %else
+        mov     T0_16, [A0]             ; T0 = dividend
+        mov     T1, A1                  ; T1 = saved divisor (because of missing T1_8 in 32-bit)
+        test    A1_8, A1_8
+        js      .divisor_negative
+        test    T0_16, T0_16
+        jns     .both_positive
+        neg     T0_16
+.one_of_each:                           ; OK range is 2^(result-with - 1) + (divisor - 1).
+        push    T0                      ; Start off like unsigned below.
+        shr     T0_16, 7
+        cmp     T0_8, A1_8
+        pop     T0
+        jb      .div_no_overflow
+        ja      .div_overflow
+        and     T0_8, 0x7f              ; Special case for covering (divisor - 1).
+        cmp     T0_8, A1_8
+        jae     .div_overflow
+        jmp     .div_no_overflow
+
+.divisor_negative:
+        neg     A1_8
+        test    T0_16, T0_16
+        jns     .one_of_each
+        neg     T0_16
+.both_positive:                         ; Same as unsigned shifted by sign indicator bit.
+        shr     T0_16, 7
+        cmp     T0_8, A1_8
+        jae     .div_overflow
+.div_no_overflow:
+        mov     A1, T1                  ; restore divisor
+ %endif
+
+        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
+        mov     ax, [A0]
+        %1      A1_8
+        mov     [A0], ax
+        IEM_SAVE_FLAGS       A2, %2, %3
+        xor     eax, eax
+
+.return:
+        EPILOGUE_3_ARGS
+
+.div_zero:
+.div_overflow:
+        mov     eax, -1
+        jmp     .return
+ENDPROC iemAImpl_ %+ %1 %+ _u8
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 16
+        PROLOGUE_4_ARGS
+
+        ; div by chainsaw check.
+        test    A2_16, A2_16
+        jz      .div_zero
+
+        ; Overflow check - unsigned division is simple to verify, haven't
+        ; found a simple way to check signed division yet unfortunately.
+ %if %4 == 0
+        cmp     [A1], A2_16
+        jae     .div_overflow
+ %else
+        mov     T0_16, [A1]
+        shl     T0_32, 16
+        mov     T0_16, [A0]              ; T0 = dividend
+        mov     T1, A2                   ; T1 = divisor
+        test    T1_16, T1_16
+        js      .divisor_negative
+        test    T0_32, T0_32
+        jns     .both_positive
+        neg     T0_32
+.one_of_each:                           ; OK range is 2^(result-with - 1) + (divisor - 1).
+        push    T0                      ; Start off like unsigned below.
+        shr     T0_32, 15
+        cmp     T0_16, T1_16
+        pop     T0
+        jb      .div_no_overflow
+        ja      .div_overflow
+        and     T0_16, 0x7fff           ; Special case for covering (divisor - 1).
+        cmp     T0_16, T1_16
+        jae     .div_overflow
+        jmp     .div_no_overflow
+
+.divisor_negative:
+        neg     T1_16
+        test    T0_32, T0_32
+        jns     .one_of_each
+        neg     T0_32
+.both_positive:                         ; Same as unsigned shifted by sign indicator bit.
+        shr     T0_32, 15
+        cmp     T0_16, T1_16
+        jae     .div_overflow
+.div_no_overflow:
+ %endif
+
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+ %ifdef ASM_CALL64_GCC
+        mov     T1, A2
+        mov     ax, [A0]
+        mov     dx, [A1]
+        %1      T1_16
+        mov     [A0], ax
+        mov     [A1], dx
+ %else
+        mov     T1, A1
+        mov     ax, [A0]
+        mov     dx, [T1]
+        %1      A2_16
+        mov     [A0], ax
+        mov     [T1], dx
+ %endif
+        IEM_SAVE_FLAGS       A3, %2, %3
+        xor     eax, eax
+
+.return:
+        EPILOGUE_4_ARGS
+
+.div_zero:
+.div_overflow:
+        mov     eax, -1
+        jmp     .return
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 16
+        PROLOGUE_4_ARGS
+
+        ; div by chainsaw check.
+        test    A2_32, A2_32
+        jz      .div_zero
+
+        ; Overflow check - unsigned division is simple to verify, haven't
+        ; found a simple way to check signed division yet unfortunately.
+ %if %4 == 0
+        cmp     [A1], A2_32
+        jae     .div_overflow
+ %else
+        push    A2                      ; save A2 so we modify it (we out of regs on x86).
+        mov     T0_32, [A0]             ; T0 = dividend low
+        mov     T1_32, [A1]             ; T1 = dividend high
+        test    A2_32, A2_32
+        js      .divisor_negative
+        test    T1_32, T1_32
+        jns     .both_positive
+        call    iemAImpl_negate_T0_T1_u32
+.one_of_each:                           ; OK range is 2^(result-with - 1) + (divisor - 1).
+        push    T0                      ; Start off like unsigned below.
+        shl     T1_32, 1
+        shr     T0_32, 31
+        or      T1_32, T0_32
+        cmp     T1_32, A2_32
+        pop     T0
+        jb      .div_no_overflow
+        ja      .div_overflow
+        and     T0_32, 0x7fffffff       ; Special case for covering (divisor - 1).
+        cmp     T0_32, A2_32
+        jae     .div_overflow
+        jmp     .div_no_overflow
+
+.divisor_negative:
+        neg     A2_32
+        test    T1_32, T1_32
+        jns     .one_of_each
+        call    iemAImpl_negate_T0_T1_u32
+.both_positive:                         ; Same as unsigned shifted by sign indicator bit.
+        shl     T1_32, 1
+        shr     T0_32, 31
+        or      T1_32, T0_32
+        cmp     T1_32, A2_32
+        jae     .div_overflow
+.div_no_overflow:
+        pop     A2
+ %endif
+
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        mov     eax, [A0]
+ %ifdef ASM_CALL64_GCC
+        mov     T1, A2
+        mov     eax, [A0]
+        mov     edx, [A1]
+        %1      T1_32
+        mov     [A0], eax
+        mov     [A1], edx
+ %else
+        mov     T1, A1
+        mov     eax, [A0]
+        mov     edx, [T1]
+        %1      A2_32
+        mov     [A0], eax
+        mov     [T1], edx
+ %endif
+        IEM_SAVE_FLAGS       A3, %2, %3
+        xor     eax, eax
+
+.return:
+        EPILOGUE_4_ARGS
+
+.div_overflow:
+ %if %4 != 0
+        pop     A2
+ %endif
+.div_zero:
+        mov     eax, -1
+        jmp     .return
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64 ; The 32-bit host version lives in IEMAllAImplC.cpp.
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 20
+        PROLOGUE_4_ARGS
+
+        test    A2, A2
+        jz      .div_zero
+  %if %4 == 0
+        cmp     [A1], A2
+        jae     .div_overflow
+  %else
+        push    A2                      ; save A2 so we modify it (we out of regs on x86).
+        mov     T0, [A0]                ; T0 = dividend low
+        mov     T1, [A1]                ; T1 = dividend high
+        test    A2, A2
+        js      .divisor_negative
+        test    T1, T1
+        jns     .both_positive
+        call    iemAImpl_negate_T0_T1_u64
+.one_of_each:                           ; OK range is 2^(result-with - 1) + (divisor - 1).
+        push    T0                      ; Start off like unsigned below.
+        shl     T1, 1
+        shr     T0, 63
+        or      T1, T0
+        cmp     T1, A2
+        pop     T0
+        jb      .div_no_overflow
+        ja      .div_overflow
+        mov     T1, 0x7fffffffffffffff
+        and     T0, T1                  ; Special case for covering (divisor - 1).
+        cmp     T0, A2
+        jae     .div_overflow
+        jmp     .div_no_overflow
+
+.divisor_negative:
+        neg     A2
+        test    T1, T1
+        jns     .one_of_each
+        call    iemAImpl_negate_T0_T1_u64
+.both_positive:                         ; Same as unsigned shifted by sign indicator bit.
+        shl     T1, 1
+        shr     T0, 63
+        or      T1, T0
+        cmp     T1, A2
+        jae     .div_overflow
+.div_no_overflow:
+        pop     A2
+  %endif
+
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        mov     rax, [A0]
+  %ifdef ASM_CALL64_GCC
+        mov     T1, A2
+        mov     rax, [A0]
+        mov     rdx, [A1]
+        %1      T1
+        mov     [A0], rax
+        mov     [A1], rdx
+  %else
+        mov     T1, A1
+        mov     rax, [A0]
+        mov     rdx, [T1]
+        %1      A2
+        mov     [A0], rax
+        mov     [T1], rdx
+  %endif
+        IEM_SAVE_FLAGS       A3, %2, %3
+        xor     eax, eax
+
+.return:
+        EPILOGUE_4_ARGS_EX 12
+
+.div_overflow:
+  %if %4 != 0
+        pop     A2
+  %endif
+.div_zero:
+        mov     eax, -1
+        jmp     .return
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif ; !RT_ARCH_AMD64
+
+%endmacro
+
+IEMIMPL_DIV_OP div,  0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_DIV_OP idiv, 0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 1
+
+
+;
+; BSWAP. No flag changes.
+;
+; Each function takes one argument, pointer to the value to bswap
+; (input/output). They all return void.
+;
+BEGINPROC_FASTCALL iemAImpl_bswap_u16, 4
+        PROLOGUE_1_ARGS
+        mov     T0_32, [A0]             ; just in case any of the upper bits are used.
+        db 66h
+        bswap   T0_32
+        mov     [A0], T0_32
+        EPILOGUE_1_ARGS
+ENDPROC iemAImpl_bswap_u16
+
+BEGINPROC_FASTCALL iemAImpl_bswap_u32, 4
+        PROLOGUE_1_ARGS
+        mov     T0_32, [A0]
+        bswap   T0_32
+        mov     [A0], T0_32
+        EPILOGUE_1_ARGS
+ENDPROC iemAImpl_bswap_u32
+
+BEGINPROC_FASTCALL iemAImpl_bswap_u64, 4
+%ifdef RT_ARCH_AMD64
+        PROLOGUE_1_ARGS
+        mov     T0, [A0]
+        bswap   T0
+        mov     [A0], T0
+        EPILOGUE_1_ARGS
+%else
+        PROLOGUE_1_ARGS
+        mov     T0, [A0]
+        mov     T1, [A0 + 4]
+        bswap   T0
+        bswap   T1
+        mov     [A0 + 4], T0
+        mov     [A0], T1
+        EPILOGUE_1_ARGS
+%endif
+ENDPROC iemAImpl_bswap_u64
+
+
+;;
+; Initialize the FPU for the actual instruction being emulated, this means
+; loading parts of the guest's control word and status word.
+;
+; @uses     24 bytes of stack.
+; @param    1       Expression giving the address of the FXSTATE of the guest.
+;
+%macro FPU_LD_FXSTATE_FCW_AND_SAFE_FSW 1
+        fnstenv [xSP]
+
+        ; FCW - for exception, precision and rounding control.
+        movzx   T0, word [%1 + X86FXSTATE.FCW]
+        and     T0, X86_FCW_MASK_ALL | X86_FCW_PC_MASK | X86_FCW_RC_MASK
+        mov     [xSP + X86FSTENV32P.FCW], T0_16
+
+        ; FSW - for undefined C0, C1, C2, and C3.
+        movzx   T1, word [%1 + X86FXSTATE.FSW]
+        and     T1, X86_FSW_C_MASK
+        movzx   T0, word [xSP + X86FSTENV32P.FSW]
+        and     T0, X86_FSW_TOP_MASK
+        or      T0, T1
+        mov     [xSP + X86FSTENV32P.FSW], T0_16
+
+        fldenv  [xSP]
+%endmacro
+
+
+;;
+; Need to move this as well somewhere better?
+;
+struc IEMFPURESULT
+    .r80Result  resw 5
+    .FSW        resw 1
+endstruc
+
+
+;;
+; Need to move this as well somewhere better?
+;
+struc IEMFPURESULTTWO
+    .r80Result1 resw 5
+    .FSW        resw 1
+    .r80Result2 resw 5
+endstruc
+
+
+;
+;---------------------- 16-bit signed integer operations ----------------------
+;
+
+
+;;
+; Converts a 16-bit floating point value to a 80-bit one (fpu register).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 16-bit floating point value to convert.
+;
+BEGINPROC_FASTCALL iemAImpl_fild_i16_to_r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fild    word [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fild_i16_to_r80
+
+
+;;
+; Store a 80-bit floating point value (register) as a 16-bit signed integer (memory).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 16-bit signed integer value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fist_r80_to_i16, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fistp   word [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fist_r80_to_i16
+
+
+;;
+; Store a 80-bit floating point value (register) as a 16-bit signed integer
+; (memory) with truncation.
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 16-bit signed integer value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fistt_r80_to_i16, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fisttp  dword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fistt_r80_to_i16
+
+
+;;
+; FPU instruction working on one 80-bit and one 16-bit signed integer value.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 16-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_I16 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i16, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      word [A3]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i16
+%endmacro
+
+IEMIMPL_FPU_R80_BY_I16 fiadd
+IEMIMPL_FPU_R80_BY_I16 fimul
+IEMIMPL_FPU_R80_BY_I16 fisub
+IEMIMPL_FPU_R80_BY_I16 fisubr
+IEMIMPL_FPU_R80_BY_I16 fidiv
+IEMIMPL_FPU_R80_BY_I16 fidivr
+
+
+;;
+; FPU instruction working on one 80-bit and one 16-bit signed integer value,
+; only returning FSW.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to store the output FSW.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 64-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_I16_FSW 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i16, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      word [A3]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i16
+%endmacro
+
+IEMIMPL_FPU_R80_BY_I16_FSW ficom
+
+
+
+;
+;---------------------- 32-bit signed integer operations ----------------------
+;
+
+
+;;
+; Converts a 32-bit floating point value to a 80-bit one (fpu register).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 32-bit floating point value to convert.
+;
+BEGINPROC_FASTCALL iemAImpl_fild_i32_to_r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fild    dword [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fild_i32_to_r80
+
+
+;;
+; Store a 80-bit floating point value (register) as a 32-bit signed integer (memory).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 32-bit signed integer value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fist_r80_to_i32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fistp   dword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fist_r80_to_i32
+
+
+;;
+; Store a 80-bit floating point value (register) as a 32-bit signed integer
+; (memory) with truncation.
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 32-bit signed integer value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fistt_r80_to_i32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fisttp  dword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fistt_r80_to_i32
+
+
+;;
+; FPU instruction working on one 80-bit and one 32-bit signed integer value.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 32-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_I32 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      dword [A3]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i32
+%endmacro
+
+IEMIMPL_FPU_R80_BY_I32 fiadd
+IEMIMPL_FPU_R80_BY_I32 fimul
+IEMIMPL_FPU_R80_BY_I32 fisub
+IEMIMPL_FPU_R80_BY_I32 fisubr
+IEMIMPL_FPU_R80_BY_I32 fidiv
+IEMIMPL_FPU_R80_BY_I32 fidivr
+
+
+;;
+; FPU instruction working on one 80-bit and one 32-bit signed integer value,
+; only returning FSW.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to store the output FSW.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 64-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_I32_FSW 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      dword [A3]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i32
+%endmacro
+
+IEMIMPL_FPU_R80_BY_I32_FSW ficom
+
+
+
+;
+;---------------------- 64-bit signed integer operations ----------------------
+;
+
+
+;;
+; Converts a 64-bit floating point value to a 80-bit one (fpu register).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 64-bit floating point value to convert.
+;
+BEGINPROC_FASTCALL iemAImpl_fild_i64_to_r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fild    qword [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fild_i64_to_r80
+
+
+;;
+; Store a 80-bit floating point value (register) as a 64-bit signed integer (memory).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 64-bit signed integer value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fist_r80_to_i64, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fistp   qword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fist_r80_to_i64
+
+
+;;
+; Store a 80-bit floating point value (register) as a 64-bit signed integer
+; (memory) with truncation.
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 64-bit signed integer value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fistt_r80_to_i64, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fisttp  qword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fistt_r80_to_i64
+
+
+
+;
+;---------------------- 32-bit floating point operations ----------------------
+;
+
+;;
+; Converts a 32-bit floating point value to a 80-bit one (fpu register).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 32-bit floating point value to convert.
+;
+BEGINPROC_FASTCALL iemAImpl_fld_r32_to_r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fld     dword [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fld_r32_to_r80
+
+
+;;
+; Store a 80-bit floating point value (register) as a 32-bit one (memory).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 32-bit value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fst_r80_to_r32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fst     dword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fst_r80_to_r32
+
+
+;;
+; FPU instruction working on one 80-bit and one 32-bit floating point value.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 32-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_R32 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      dword [A3]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r32
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R32 fadd
+IEMIMPL_FPU_R80_BY_R32 fmul
+IEMIMPL_FPU_R80_BY_R32 fsub
+IEMIMPL_FPU_R80_BY_R32 fsubr
+IEMIMPL_FPU_R80_BY_R32 fdiv
+IEMIMPL_FPU_R80_BY_R32 fdivr
+
+
+;;
+; FPU instruction working on one 80-bit and one 32-bit floating point value,
+; only returning FSW.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to store the output FSW.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 64-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_R32_FSW 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      dword [A3]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r32
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R32_FSW fcom
+
+
+
+;
+;---------------------- 64-bit floating point operations ----------------------
+;
+
+;;
+; Converts a 64-bit floating point value to a 80-bit one (fpu register).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 64-bit floating point value to convert.
+;
+BEGINPROC_FASTCALL iemAImpl_fld_r64_to_r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fld     qword [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fld_r64_to_r80
+
+
+;;
+; Store a 80-bit floating point value (register) as a 64-bit one (memory).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 64-bit value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fst_r80_to_r64, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fst     qword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fst_r80_to_r64
+
+
+;;
+; FPU instruction working on one 80-bit and one 64-bit floating point value.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 64-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_R64 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r64, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      qword [A3]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r64
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R64 fadd
+IEMIMPL_FPU_R80_BY_R64 fmul
+IEMIMPL_FPU_R80_BY_R64 fsub
+IEMIMPL_FPU_R80_BY_R64 fsubr
+IEMIMPL_FPU_R80_BY_R64 fdiv
+IEMIMPL_FPU_R80_BY_R64 fdivr
+
+;;
+; FPU instruction working on one 80-bit and one 64-bit floating point value,
+; only returning FSW.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to store the output FSW.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 64-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_R64_FSW 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r64, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      qword [A3]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r64
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R64_FSW fcom
+
+
+
+;
+;---------------------- 80-bit floating point operations ----------------------
+;
+
+;;
+; Loads a 80-bit floating point register value from memory.
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit floating point value to load.
+;
+BEGINPROC_FASTCALL iemAImpl_fld_r80_from_r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fld     tword [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fld_r80_from_r80
+
+
+;;
+; Store a 80-bit floating point register to memory
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 80-bit value.
+; @param    A3      Pointer to the 80-bit register value.
+;
+BEGINPROC_FASTCALL iemAImpl_fst_r80_to_r80, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fstp    tword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fst_r80_to_r80
+
+
+;;
+; FPU instruction working on two 80-bit floating point values.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the first 80-bit value (ST0)
+; @param    A3      Pointer to the second 80-bit value (STn).
+;
+%macro IEMIMPL_FPU_R80_BY_R80 2
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      %2
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R80 fadd,   {st0, st1}
+IEMIMPL_FPU_R80_BY_R80 fmul,   {st0, st1}
+IEMIMPL_FPU_R80_BY_R80 fsub,   {st0, st1}
+IEMIMPL_FPU_R80_BY_R80 fsubr,  {st0, st1}
+IEMIMPL_FPU_R80_BY_R80 fdiv,   {st0, st1}
+IEMIMPL_FPU_R80_BY_R80 fdivr,  {st0, st1}
+IEMIMPL_FPU_R80_BY_R80 fprem,  {}
+IEMIMPL_FPU_R80_BY_R80 fprem1, {}
+IEMIMPL_FPU_R80_BY_R80 fscale, {}
+
+
+;;
+; FPU instruction working on two 80-bit floating point values, ST1 and ST0,
+; storing the result in ST1 and popping the stack.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the first 80-bit value (ST1).
+; @param    A3      Pointer to the second 80-bit value (ST0).
+;
+%macro IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP fpatan
+IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP fyl2x
+IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP fyl2xp1
+
+
+;;
+; FPU instruction working on two 80-bit floating point values, only
+; returning FSW.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a uint16_t for the resulting FSW.
+; @param    A2      Pointer to the first 80-bit value.
+; @param    A3      Pointer to the second 80-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_R80_FSW 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      st0, st1
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R80_FSW fcom
+IEMIMPL_FPU_R80_BY_R80_FSW fucom
+
+
+;;
+; FPU instruction working on two 80-bit floating point values,
+; returning FSW and EFLAGS (eax).
+;
+; @param    1       The instruction
+;
+; @returns  EFLAGS in EAX.
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a uint16_t for the resulting FSW.
+; @param    A2      Pointer to the first 80-bit value.
+; @param    A3      Pointer to the second 80-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_R80_EFL 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      st1
+
+        fnstsw  word  [A1]
+        pushf
+        pop     xAX
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R80_EFL fcomi
+IEMIMPL_FPU_R80_BY_R80_EFL fucomi
+
+
+;;
+; FPU instruction working on one 80-bit floating point value.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit value.
+;
+%macro IEMIMPL_FPU_R80 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80
+%endmacro
+
+IEMIMPL_FPU_R80 fchs
+IEMIMPL_FPU_R80 fabs
+IEMIMPL_FPU_R80 f2xm1
+IEMIMPL_FPU_R80 fsqrt
+IEMIMPL_FPU_R80 frndint
+IEMIMPL_FPU_R80 fsin
+IEMIMPL_FPU_R80 fcos
+
+
+;;
+; FPU instruction working on one 80-bit floating point value, only
+; returning FSW.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a uint16_t for the resulting FSW.
+; @param    A2      Pointer to the 80-bit value.
+;
+%macro IEMIMPL_FPU_R80_FSW 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80
+%endmacro
+
+IEMIMPL_FPU_R80_FSW ftst
+IEMIMPL_FPU_R80_FSW fxam
+
+
+
+;;
+; FPU instruction loading a 80-bit floating point constant.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+;
+%macro IEMIMPL_FPU_R80_CONST 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1, 8
+        PROLOGUE_2_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+
+%endmacro
+
+IEMIMPL_FPU_R80_CONST fld1
+IEMIMPL_FPU_R80_CONST fldl2t
+IEMIMPL_FPU_R80_CONST fldl2e
+IEMIMPL_FPU_R80_CONST fldpi
+IEMIMPL_FPU_R80_CONST fldlg2
+IEMIMPL_FPU_R80_CONST fldln2
+IEMIMPL_FPU_R80_CONST fldz
+
+
+;;
+; FPU instruction working on one 80-bit floating point value, outputing two.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULTTWO for the output.
+; @param    A2      Pointer to the 80-bit value.
+;
+%macro IEMIMPL_FPU_R80_R80 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1
+
+        fnstsw  word  [A1 + IEMFPURESULTTWO.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULTTWO.r80Result2]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULTTWO.r80Result1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_r80
+%endmacro
+
+IEMIMPL_FPU_R80_R80 fptan
+IEMIMPL_FPU_R80_R80 fxtract
+IEMIMPL_FPU_R80_R80 fsincos
+
+
+
+
+;---------------------- SSE and MMX Operations ----------------------
+
+;; @todo what do we need to do for MMX?
+%macro IEMIMPL_MMX_PROLOGUE 0
+%endmacro
+%macro IEMIMPL_MMX_EPILOGUE 0
+%endmacro
+
+;; @todo what do we need to do for SSE?
+%macro IEMIMPL_SSE_PROLOGUE 0
+%endmacro
+%macro IEMIMPL_SSE_EPILOGUE 0
+%endmacro
+
+
+;;
+; Media instruction working on two full sized registers.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to the first media register size operand (input/output).
+; @param    A2      Pointer to the second media register size operand (input).
+;
+%macro IEMIMPL_MEDIA_F2 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_MMX_PROLOGUE
+
+        movq    mm0, [A1]
+        movq    mm1, [A2]
+        %1      mm0, mm1
+        movq    [A1], mm0
+
+        IEMIMPL_MMX_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu   xmm0, [A1]
+        movdqu   xmm1, [A2]
+        %1       xmm0, xmm1
+        movdqu   [A1], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_F2 pxor
+IEMIMPL_MEDIA_F2 pcmpeqb
+IEMIMPL_MEDIA_F2 pcmpeqw
+IEMIMPL_MEDIA_F2 pcmpeqd
+
+
+;;
+; Media instruction working on one full sized and one half sized register (lower half).
+;
+; @param    1       The instruction
+; @param    2       1 if MMX is included, 0 if not.
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to the first full sized media register operand (input/output).
+; @param    A2      Pointer to the second half sized media register operand (input).
+;
+%macro IEMIMPL_MEDIA_F1L1 2
+ %if %2 != 0
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_MMX_PROLOGUE
+
+        movq    mm0, [A1]
+        movd    mm1, [A2]
+        %1      mm0, mm1
+        movq    [A1], mm0
+
+        IEMIMPL_MMX_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu   xmm0, [A1]
+        movq     xmm1, [A2]
+        %1       xmm0, xmm1
+        movdqu   [A1], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_F1L1 punpcklbw,  1
+IEMIMPL_MEDIA_F1L1 punpcklwd,  1
+IEMIMPL_MEDIA_F1L1 punpckldq,  1
+IEMIMPL_MEDIA_F1L1 punpcklqdq, 0
+
+
+;;
+; Media instruction working on one full sized and one half sized register (high half).
+;
+; @param    1       The instruction
+; @param    2       1 if MMX is included, 0 if not.
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to the first full sized media register operand (input/output).
+; @param    A2      Pointer to the second full sized media register operand, where we
+;                   will only use the upper half (input).
+;
+%macro IEMIMPL_MEDIA_F1H1 2
+ %if %2 != 0
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_MMX_PROLOGUE
+
+        movq    mm0, [A1]
+        movq    mm1, [A2]
+        %1      mm0, mm1
+        movq    [A1], mm0
+
+        IEMIMPL_MMX_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu   xmm0, [A1]
+        movdqu   xmm1, [A2]
+        %1       xmm0, xmm1
+        movdqu   [A1], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_F1L1 punpckhbw,  1
+IEMIMPL_MEDIA_F1L1 punpckhwd,  1
+IEMIMPL_MEDIA_F1L1 punpckhdq,  1
+IEMIMPL_MEDIA_F1L1 punpckhqdq, 0
+
+
+;
+; Shufflers with evil 8-bit immediates.
+;
+
+BEGINPROC_FASTCALL iemAImpl_pshufw, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_MMX_PROLOGUE
+
+        movq    mm0, [A1]
+        movq    mm1, [A2]
+        lea     T0, [A3 + A3*4]         ; sizeof(pshufw+ret) == 5
+        lea     T1, [.imm0 xWrtRIP]
+        lea     T1, [T1 + T0]
+        call    T1
+        movq    [A1], mm0
+
+        IEMIMPL_MMX_EPILOGUE
+        EPILOGUE_4_ARGS
+%assign bImm 0
+%rep 256
+.imm %+ bImm:
+       pshufw    mm0, mm1, bImm
+       ret
+ %assign bImm bImm + 1
+%endrep
+.immEnd:                                ; 256*5 == 0x500
+dw 0xfaff  + (.immEnd - .imm0)          ; will cause warning if entries are too big.
+dw 0x104ff - (.immEnd - .imm0)          ; will cause warning if entries are small big.
+ENDPROC iemAImpl_pshufw
+
+
+%macro IEMIMPL_MEDIA_SSE_PSHUFXX 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm0, [A1]
+        movdqu  xmm1, [A2]
+        lea     T1, [.imm0 xWrtRIP]
+        lea     T0, [A3 + A3*2]         ; sizeof(pshufXX+ret) == 6: (A3 * 3) *2
+        lea     T1, [T1 + T0*2]
+        call    T1
+        movdqu  [A1], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+       %1       xmm0, xmm1, bImm
+       ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd:                                ; 256*6 == 0x600
+dw 0xf9ff  + (.immEnd - .imm0)          ; will cause warning if entries are too big.
+dw 0x105ff - (.immEnd - .imm0)          ; will cause warning if entries are small big.
+ENDPROC iemAImpl_ %+ %1
+%endmacro
+
+IEMIMPL_MEDIA_SSE_PSHUFXX pshufhw
+IEMIMPL_MEDIA_SSE_PSHUFXX pshuflw
+IEMIMPL_MEDIA_SSE_PSHUFXX pshufd
+
+
+;
+; Move byte mask.
+;
+
+BEGINPROC_FASTCALL iemAImpl_pmovmskb_u64, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_MMX_PROLOGUE
+
+        mov     T0,  [A1]
+        movq    mm1, [A2]
+        pmovmskb T0, mm1
+        mov     [A1], T0
+%ifdef RT_ARCH_X86
+        mov     dword [A1 + 4], 0
+%endif
+        IEMIMPL_MMX_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_pmovmskb_u64
+
+BEGINPROC_FASTCALL iemAImpl_pmovmskb_u128, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        mov     T0,  [A1]
+        movdqu  xmm1, [A2]
+        pmovmskb T0, xmm1
+        mov     [A1], T0
+%ifdef RT_ARCH_X86
+        mov     dword [A1 + 4], 0
+%endif
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_pmovmskb_u128
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllAImplC.cpp b/src/VBox/VMM/VMMAll/IEMAllAImplC.cpp
new file mode 100644
index 00000000..3ae56bee
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllAImplC.cpp
@@ -0,0 +1,1450 @@
+/* $Id: IEMAllAImplC.cpp $ */
+/** @file
+ * IEM - Instruction Implementation in Assembly, portable C variant.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include "IEMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <iprt/x86.h>
+#include <iprt/uint128.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#ifdef RT_ARCH_X86
+/**
+ * Parity calculation table.
+ *
+ * The generator code:
+ * @code
+ * #include <stdio.h>
+ *
+ * int main()
+ * {
+ *     unsigned b;
+ *     for (b = 0; b < 256; b++)
+ *     {
+ *         int cOnes = ( b       & 1)
+ *                   + ((b >> 1) & 1)
+ *                   + ((b >> 2) & 1)
+ *                   + ((b >> 3) & 1)
+ *                   + ((b >> 4) & 1)
+ *                   + ((b >> 5) & 1)
+ *                   + ((b >> 6) & 1)
+ *                   + ((b >> 7) & 1);
+ *         printf("    /" "* %#04x = %u%u%u%u%u%u%u%ub *" "/ %s,\n",
+ *                b,
+ *                (b >> 7) & 1,
+ *                (b >> 6) & 1,
+ *                (b >> 5) & 1,
+ *                (b >> 4) & 1,
+ *                (b >> 3) & 1,
+ *                (b >> 2) & 1,
+ *                (b >> 1) & 1,
+ *                 b       & 1,
+ *                cOnes & 1 ? "0" : "X86_EFL_PF");
+ *     }
+ *     return 0;
+ * }
+ * @endcode
+ */
+static uint8_t const g_afParity[256] =
+{
+    /* 0000 = 00000000b */ X86_EFL_PF,
+    /* 0x01 = 00000001b */ 0,
+    /* 0x02 = 00000010b */ 0,
+    /* 0x03 = 00000011b */ X86_EFL_PF,
+    /* 0x04 = 00000100b */ 0,
+    /* 0x05 = 00000101b */ X86_EFL_PF,
+    /* 0x06 = 00000110b */ X86_EFL_PF,
+    /* 0x07 = 00000111b */ 0,
+    /* 0x08 = 00001000b */ 0,
+    /* 0x09 = 00001001b */ X86_EFL_PF,
+    /* 0x0a = 00001010b */ X86_EFL_PF,
+    /* 0x0b = 00001011b */ 0,
+    /* 0x0c = 00001100b */ X86_EFL_PF,
+    /* 0x0d = 00001101b */ 0,
+    /* 0x0e = 00001110b */ 0,
+    /* 0x0f = 00001111b */ X86_EFL_PF,
+    /* 0x10 = 00010000b */ 0,
+    /* 0x11 = 00010001b */ X86_EFL_PF,
+    /* 0x12 = 00010010b */ X86_EFL_PF,
+    /* 0x13 = 00010011b */ 0,
+    /* 0x14 = 00010100b */ X86_EFL_PF,
+    /* 0x15 = 00010101b */ 0,
+    /* 0x16 = 00010110b */ 0,
+    /* 0x17 = 00010111b */ X86_EFL_PF,
+    /* 0x18 = 00011000b */ X86_EFL_PF,
+    /* 0x19 = 00011001b */ 0,
+    /* 0x1a = 00011010b */ 0,
+    /* 0x1b = 00011011b */ X86_EFL_PF,
+    /* 0x1c = 00011100b */ 0,
+    /* 0x1d = 00011101b */ X86_EFL_PF,
+    /* 0x1e = 00011110b */ X86_EFL_PF,
+    /* 0x1f = 00011111b */ 0,
+    /* 0x20 = 00100000b */ 0,
+    /* 0x21 = 00100001b */ X86_EFL_PF,
+    /* 0x22 = 00100010b */ X86_EFL_PF,
+    /* 0x23 = 00100011b */ 0,
+    /* 0x24 = 00100100b */ X86_EFL_PF,
+    /* 0x25 = 00100101b */ 0,
+    /* 0x26 = 00100110b */ 0,
+    /* 0x27 = 00100111b */ X86_EFL_PF,
+    /* 0x28 = 00101000b */ X86_EFL_PF,
+    /* 0x29 = 00101001b */ 0,
+    /* 0x2a = 00101010b */ 0,
+    /* 0x2b = 00101011b */ X86_EFL_PF,
+    /* 0x2c = 00101100b */ 0,
+    /* 0x2d = 00101101b */ X86_EFL_PF,
+    /* 0x2e = 00101110b */ X86_EFL_PF,
+    /* 0x2f = 00101111b */ 0,
+    /* 0x30 = 00110000b */ X86_EFL_PF,
+    /* 0x31 = 00110001b */ 0,
+    /* 0x32 = 00110010b */ 0,
+    /* 0x33 = 00110011b */ X86_EFL_PF,
+    /* 0x34 = 00110100b */ 0,
+    /* 0x35 = 00110101b */ X86_EFL_PF,
+    /* 0x36 = 00110110b */ X86_EFL_PF,
+    /* 0x37 = 00110111b */ 0,
+    /* 0x38 = 00111000b */ 0,
+    /* 0x39 = 00111001b */ X86_EFL_PF,
+    /* 0x3a = 00111010b */ X86_EFL_PF,
+    /* 0x3b = 00111011b */ 0,
+    /* 0x3c = 00111100b */ X86_EFL_PF,
+    /* 0x3d = 00111101b */ 0,
+    /* 0x3e = 00111110b */ 0,
+    /* 0x3f = 00111111b */ X86_EFL_PF,
+    /* 0x40 = 01000000b */ 0,
+    /* 0x41 = 01000001b */ X86_EFL_PF,
+    /* 0x42 = 01000010b */ X86_EFL_PF,
+    /* 0x43 = 01000011b */ 0,
+    /* 0x44 = 01000100b */ X86_EFL_PF,
+    /* 0x45 = 01000101b */ 0,
+    /* 0x46 = 01000110b */ 0,
+    /* 0x47 = 01000111b */ X86_EFL_PF,
+    /* 0x48 = 01001000b */ X86_EFL_PF,
+    /* 0x49 = 01001001b */ 0,
+    /* 0x4a = 01001010b */ 0,
+    /* 0x4b = 01001011b */ X86_EFL_PF,
+    /* 0x4c = 01001100b */ 0,
+    /* 0x4d = 01001101b */ X86_EFL_PF,
+    /* 0x4e = 01001110b */ X86_EFL_PF,
+    /* 0x4f = 01001111b */ 0,
+    /* 0x50 = 01010000b */ X86_EFL_PF,
+    /* 0x51 = 01010001b */ 0,
+    /* 0x52 = 01010010b */ 0,
+    /* 0x53 = 01010011b */ X86_EFL_PF,
+    /* 0x54 = 01010100b */ 0,
+    /* 0x55 = 01010101b */ X86_EFL_PF,
+    /* 0x56 = 01010110b */ X86_EFL_PF,
+    /* 0x57 = 01010111b */ 0,
+    /* 0x58 = 01011000b */ 0,
+    /* 0x59 = 01011001b */ X86_EFL_PF,
+    /* 0x5a = 01011010b */ X86_EFL_PF,
+    /* 0x5b = 01011011b */ 0,
+    /* 0x5c = 01011100b */ X86_EFL_PF,
+    /* 0x5d = 01011101b */ 0,
+    /* 0x5e = 01011110b */ 0,
+    /* 0x5f = 01011111b */ X86_EFL_PF,
+    /* 0x60 = 01100000b */ X86_EFL_PF,
+    /* 0x61 = 01100001b */ 0,
+    /* 0x62 = 01100010b */ 0,
+    /* 0x63 = 01100011b */ X86_EFL_PF,
+    /* 0x64 = 01100100b */ 0,
+    /* 0x65 = 01100101b */ X86_EFL_PF,
+    /* 0x66 = 01100110b */ X86_EFL_PF,
+    /* 0x67 = 01100111b */ 0,
+    /* 0x68 = 01101000b */ 0,
+    /* 0x69 = 01101001b */ X86_EFL_PF,
+    /* 0x6a = 01101010b */ X86_EFL_PF,
+    /* 0x6b = 01101011b */ 0,
+    /* 0x6c = 01101100b */ X86_EFL_PF,
+    /* 0x6d = 01101101b */ 0,
+    /* 0x6e = 01101110b */ 0,
+    /* 0x6f = 01101111b */ X86_EFL_PF,
+    /* 0x70 = 01110000b */ 0,
+    /* 0x71 = 01110001b */ X86_EFL_PF,
+    /* 0x72 = 01110010b */ X86_EFL_PF,
+    /* 0x73 = 01110011b */ 0,
+    /* 0x74 = 01110100b */ X86_EFL_PF,
+    /* 0x75 = 01110101b */ 0,
+    /* 0x76 = 01110110b */ 0,
+    /* 0x77 = 01110111b */ X86_EFL_PF,
+    /* 0x78 = 01111000b */ X86_EFL_PF,
+    /* 0x79 = 01111001b */ 0,
+    /* 0x7a = 01111010b */ 0,
+    /* 0x7b = 01111011b */ X86_EFL_PF,
+    /* 0x7c = 01111100b */ 0,
+    /* 0x7d = 01111101b */ X86_EFL_PF,
+    /* 0x7e = 01111110b */ X86_EFL_PF,
+    /* 0x7f = 01111111b */ 0,
+    /* 0x80 = 10000000b */ 0,
+    /* 0x81 = 10000001b */ X86_EFL_PF,
+    /* 0x82 = 10000010b */ X86_EFL_PF,
+    /* 0x83 = 10000011b */ 0,
+    /* 0x84 = 10000100b */ X86_EFL_PF,
+    /* 0x85 = 10000101b */ 0,
+    /* 0x86 = 10000110b */ 0,
+    /* 0x87 = 10000111b */ X86_EFL_PF,
+    /* 0x88 = 10001000b */ X86_EFL_PF,
+    /* 0x89 = 10001001b */ 0,
+    /* 0x8a = 10001010b */ 0,
+    /* 0x8b = 10001011b */ X86_EFL_PF,
+    /* 0x8c = 10001100b */ 0,
+    /* 0x8d = 10001101b */ X86_EFL_PF,
+    /* 0x8e = 10001110b */ X86_EFL_PF,
+    /* 0x8f = 10001111b */ 0,
+    /* 0x90 = 10010000b */ X86_EFL_PF,
+    /* 0x91 = 10010001b */ 0,
+    /* 0x92 = 10010010b */ 0,
+    /* 0x93 = 10010011b */ X86_EFL_PF,
+    /* 0x94 = 10010100b */ 0,
+    /* 0x95 = 10010101b */ X86_EFL_PF,
+    /* 0x96 = 10010110b */ X86_EFL_PF,
+    /* 0x97 = 10010111b */ 0,
+    /* 0x98 = 10011000b */ 0,
+    /* 0x99 = 10011001b */ X86_EFL_PF,
+    /* 0x9a = 10011010b */ X86_EFL_PF,
+    /* 0x9b = 10011011b */ 0,
+    /* 0x9c = 10011100b */ X86_EFL_PF,
+    /* 0x9d = 10011101b */ 0,
+    /* 0x9e = 10011110b */ 0,
+    /* 0x9f = 10011111b */ X86_EFL_PF,
+    /* 0xa0 = 10100000b */ X86_EFL_PF,
+    /* 0xa1 = 10100001b */ 0,
+    /* 0xa2 = 10100010b */ 0,
+    /* 0xa3 = 10100011b */ X86_EFL_PF,
+    /* 0xa4 = 10100100b */ 0,
+    /* 0xa5 = 10100101b */ X86_EFL_PF,
+    /* 0xa6 = 10100110b */ X86_EFL_PF,
+    /* 0xa7 = 10100111b */ 0,
+    /* 0xa8 = 10101000b */ 0,
+    /* 0xa9 = 10101001b */ X86_EFL_PF,
+    /* 0xaa = 10101010b */ X86_EFL_PF,
+    /* 0xab = 10101011b */ 0,
+    /* 0xac = 10101100b */ X86_EFL_PF,
+    /* 0xad = 10101101b */ 0,
+    /* 0xae = 10101110b */ 0,
+    /* 0xaf = 10101111b */ X86_EFL_PF,
+    /* 0xb0 = 10110000b */ 0,
+    /* 0xb1 = 10110001b */ X86_EFL_PF,
+    /* 0xb2 = 10110010b */ X86_EFL_PF,
+    /* 0xb3 = 10110011b */ 0,
+    /* 0xb4 = 10110100b */ X86_EFL_PF,
+    /* 0xb5 = 10110101b */ 0,
+    /* 0xb6 = 10110110b */ 0,
+    /* 0xb7 = 10110111b */ X86_EFL_PF,
+    /* 0xb8 = 10111000b */ X86_EFL_PF,
+    /* 0xb9 = 10111001b */ 0,
+    /* 0xba = 10111010b */ 0,
+    /* 0xbb = 10111011b */ X86_EFL_PF,
+    /* 0xbc = 10111100b */ 0,
+    /* 0xbd = 10111101b */ X86_EFL_PF,
+    /* 0xbe = 10111110b */ X86_EFL_PF,
+    /* 0xbf = 10111111b */ 0,
+    /* 0xc0 = 11000000b */ X86_EFL_PF,
+    /* 0xc1 = 11000001b */ 0,
+    /* 0xc2 = 11000010b */ 0,
+    /* 0xc3 = 11000011b */ X86_EFL_PF,
+    /* 0xc4 = 11000100b */ 0,
+    /* 0xc5 = 11000101b */ X86_EFL_PF,
+    /* 0xc6 = 11000110b */ X86_EFL_PF,
+    /* 0xc7 = 11000111b */ 0,
+    /* 0xc8 = 11001000b */ 0,
+    /* 0xc9 = 11001001b */ X86_EFL_PF,
+    /* 0xca = 11001010b */ X86_EFL_PF,
+    /* 0xcb = 11001011b */ 0,
+    /* 0xcc = 11001100b */ X86_EFL_PF,
+    /* 0xcd = 11001101b */ 0,
+    /* 0xce = 11001110b */ 0,
+    /* 0xcf = 11001111b */ X86_EFL_PF,
+    /* 0xd0 = 11010000b */ 0,
+    /* 0xd1 = 11010001b */ X86_EFL_PF,
+    /* 0xd2 = 11010010b */ X86_EFL_PF,
+    /* 0xd3 = 11010011b */ 0,
+    /* 0xd4 = 11010100b */ X86_EFL_PF,
+    /* 0xd5 = 11010101b */ 0,
+    /* 0xd6 = 11010110b */ 0,
+    /* 0xd7 = 11010111b */ X86_EFL_PF,
+    /* 0xd8 = 11011000b */ X86_EFL_PF,
+    /* 0xd9 = 11011001b */ 0,
+    /* 0xda = 11011010b */ 0,
+    /* 0xdb = 11011011b */ X86_EFL_PF,
+    /* 0xdc = 11011100b */ 0,
+    /* 0xdd = 11011101b */ X86_EFL_PF,
+    /* 0xde = 11011110b */ X86_EFL_PF,
+    /* 0xdf = 11011111b */ 0,
+    /* 0xe0 = 11100000b */ 0,
+    /* 0xe1 = 11100001b */ X86_EFL_PF,
+    /* 0xe2 = 11100010b */ X86_EFL_PF,
+    /* 0xe3 = 11100011b */ 0,
+    /* 0xe4 = 11100100b */ X86_EFL_PF,
+    /* 0xe5 = 11100101b */ 0,
+    /* 0xe6 = 11100110b */ 0,
+    /* 0xe7 = 11100111b */ X86_EFL_PF,
+    /* 0xe8 = 11101000b */ X86_EFL_PF,
+    /* 0xe9 = 11101001b */ 0,
+    /* 0xea = 11101010b */ 0,
+    /* 0xeb = 11101011b */ X86_EFL_PF,
+    /* 0xec = 11101100b */ 0,
+    /* 0xed = 11101101b */ X86_EFL_PF,
+    /* 0xee = 11101110b */ X86_EFL_PF,
+    /* 0xef = 11101111b */ 0,
+    /* 0xf0 = 11110000b */ X86_EFL_PF,
+    /* 0xf1 = 11110001b */ 0,
+    /* 0xf2 = 11110010b */ 0,
+    /* 0xf3 = 11110011b */ X86_EFL_PF,
+    /* 0xf4 = 11110100b */ 0,
+    /* 0xf5 = 11110101b */ X86_EFL_PF,
+    /* 0xf6 = 11110110b */ X86_EFL_PF,
+    /* 0xf7 = 11110111b */ 0,
+    /* 0xf8 = 11111000b */ 0,
+    /* 0xf9 = 11111001b */ X86_EFL_PF,
+    /* 0xfa = 11111010b */ X86_EFL_PF,
+    /* 0xfb = 11111011b */ 0,
+    /* 0xfc = 11111100b */ X86_EFL_PF,
+    /* 0xfd = 11111101b */ 0,
+    /* 0xfe = 11111110b */ 0,
+    /* 0xff = 11111111b */ X86_EFL_PF,
+};
+#endif /* RT_ARCH_X86 */
+
+
+/**
+ * Calculates the signed flag value given a result and it's bit width.
+ *
+ * The signed flag (SF) is a duplication of the most significant bit in the
+ * result.
+ *
+ * @returns X86_EFL_SF or 0.
+ * @param   a_uResult       Unsigned result value.
+ * @param   a_cBitsWidth    The width of the result (8, 16, 32, 64).
+ */
+#define X86_EFL_CALC_SF(a_uResult, a_cBitsWidth) \
+    ( (uint32_t)((a_uResult) >> ((a_cBitsWidth) - X86_EFL_SF_BIT - 1)) & X86_EFL_SF )
+
+/**
+ * Calculates the zero flag value given a result.
+ *
+ * The zero flag (ZF) indicates whether the result is zero or not.
+ *
+ * @returns X86_EFL_ZF or 0.
+ * @param   a_uResult       Unsigned result value.
+ */
+#define X86_EFL_CALC_ZF(a_uResult) \
+    ( (uint32_t)((a_uResult) == 0) << X86_EFL_ZF_BIT )
+
+/**
+ * Updates the status bits (CF, PF, AF, ZF, SF, and OF) after a logical op.
+ *
+ * CF and OF are defined to be 0 by logical operations.  AF on the other hand is
+ * undefined.  We do not set AF, as that seems to make the most sense (which
+ * probably makes it the most wrong in real life).
+ *
+ * @returns Status bits.
+ * @param   a_pfEFlags      Pointer to the 32-bit EFLAGS value to update.
+ * @param   a_uResult       Unsigned result value.
+ * @param   a_cBitsWidth    The width of the result (8, 16, 32, 64).
+ * @param   a_fExtra        Additional bits to set.
+ */
+#define IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(a_pfEFlags, a_uResult, a_cBitsWidth, a_fExtra) \
+    do { \
+        uint32_t fEflTmp = *(a_pfEFlags); \
+        fEflTmp &= ~X86_EFL_STATUS_BITS; \
+        fEflTmp |= g_afParity[(a_uResult) & 0xff]; \
+        fEflTmp |= X86_EFL_CALC_ZF(a_uResult); \
+        fEflTmp |= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
+        fEflTmp |= (a_fExtra); \
+        *(a_pfEFlags) = fEflTmp; \
+    } while (0)
+
+
+#ifdef RT_ARCH_X86
+/*
+ * There are a few 64-bit on 32-bit things we'd rather do in C.  Actually, doing
+ * it all in C is probably safer atm., optimize what's necessary later, maybe.
+ */
+
+
+/* Binary ops */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_add_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uDst    = *puDst;
+    uint64_t uResult = uDst + uSrc;
+    *puDst = uResult;
+
+    /* Calc EFLAGS. */
+    uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS;
+    fEfl |= (uResult < uDst) << X86_EFL_CF_BIT;
+    fEfl |= g_afParity[uResult & 0xff];
+    fEfl |= ((uint32_t)uResult ^ (uint32_t)uSrc ^ (uint32_t)uDst) & X86_EFL_AF;
+    fEfl |= X86_EFL_CALC_ZF(uResult);
+    fEfl |= X86_EFL_CALC_SF(uResult, 64);
+    fEfl |= (((uDst ^ uSrc ^ RT_BIT_64(63)) & (uResult ^ uDst)) >> (64 - X86_EFL_OF_BIT)) & X86_EFL_OF;
+    *pfEFlags = fEfl;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    if (!(*pfEFlags & X86_EFL_CF))
+        iemAImpl_add_u64(puDst, uSrc, pfEFlags);
+    else
+    {
+        uint64_t uDst    = *puDst;
+        uint64_t uResult = uDst + uSrc + 1;
+        *puDst = uResult;
+
+        /* Calc EFLAGS. */
+        /** @todo verify AF and OF calculations. */
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS;
+        fEfl |= (uResult <= uDst) << X86_EFL_CF_BIT;
+        fEfl |= g_afParity[uResult & 0xff];
+        fEfl |= ((uint32_t)uResult ^ (uint32_t)uSrc ^ (uint32_t)uDst) & X86_EFL_AF;
+        fEfl |= X86_EFL_CALC_ZF(uResult);
+        fEfl |= X86_EFL_CALC_SF(uResult, 64);
+        fEfl |= (((uDst ^ uSrc ^ RT_BIT_64(63)) & (uResult ^ uDst)) >> (64 - X86_EFL_OF_BIT)) & X86_EFL_OF;
+        *pfEFlags = fEfl;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uDst    = *puDst;
+    uint64_t uResult = uDst - uSrc;
+    *puDst = uResult;
+
+    /* Calc EFLAGS. */
+    uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS;
+    fEfl |= (uDst < uSrc) << X86_EFL_CF_BIT;
+    fEfl |= g_afParity[uResult & 0xff];
+    fEfl |= ((uint32_t)uResult ^ (uint32_t)uSrc ^ (uint32_t)uDst) & X86_EFL_AF;
+    fEfl |= X86_EFL_CALC_ZF(uResult);
+    fEfl |= X86_EFL_CALC_SF(uResult, 64);
+    fEfl |= (((uDst ^ uSrc) & (uResult ^ uDst)) >> (64 - X86_EFL_OF_BIT)) & X86_EFL_OF;
+    *pfEFlags = fEfl;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    if (!(*pfEFlags & X86_EFL_CF))
+        iemAImpl_sub_u64(puDst, uSrc, pfEFlags);
+    else
+    {
+        uint64_t uDst    = *puDst;
+        uint64_t uResult = uDst - uSrc - 1;
+        *puDst = uResult;
+
+        /* Calc EFLAGS. */
+        /** @todo verify AF and OF calculations. */
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS;
+        fEfl |= (uDst <= uSrc) << X86_EFL_CF_BIT;
+        fEfl |= g_afParity[uResult & 0xff];
+        fEfl |= ((uint32_t)uResult ^ (uint32_t)uSrc ^ (uint32_t)uDst) & X86_EFL_AF;
+        fEfl |= X86_EFL_CALC_ZF(uResult);
+        fEfl |= X86_EFL_CALC_SF(uResult, 64);
+        fEfl |= (((uDst ^ uSrc) & (uResult ^ uDst)) >> (64 - X86_EFL_OF_BIT)) & X86_EFL_OF;
+        *pfEFlags = fEfl;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_or_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uResult = *puDst | uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uResult = *puDst ^ uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_and_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uResult = *puDst & uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uDstTmp = *puDst;
+    iemAImpl_sub_u64(&uDstTmp, uSrc, pfEFlags);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_test_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uResult = *puDst & uSrc;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
+}
+
+
+/** 64-bit locked binary operand operation. */
+# define DO_LOCKED_BIN_OP_U64(a_Mnemonic) \
+    do { \
+        uint64_t uOld = ASMAtomicReadU64(puDst); \
+        uint64_t uTmp; \
+        uint32_t fEflTmp; \
+        do \
+        { \
+            uTmp = uOld; \
+            fEflTmp = *pfEFlags; \
+            iemAImpl_ ## a_Mnemonic ## _u64(&uTmp, uSrc, &fEflTmp); \
+        } while (!ASMAtomicCmpXchgExU64(puDst, uTmp, uOld, &uOld)); \
+        *pfEFlags = fEflTmp; \
+    } while (0)
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_add_u64_locked,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    DO_LOCKED_BIN_OP_U64(add);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u64_locked,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    DO_LOCKED_BIN_OP_U64(adc);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u64_locked,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    DO_LOCKED_BIN_OP_U64(sub);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u64_locked,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    DO_LOCKED_BIN_OP_U64(sbb);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_or_u64_locked,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    DO_LOCKED_BIN_OP_U64(or);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u64_locked,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    DO_LOCKED_BIN_OP_U64(xor);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_and_u64_locked,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    DO_LOCKED_BIN_OP_U64(and);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u64,(uint64_t *puDst, uint64_t *puReg, uint32_t *pfEFlags))
+{
+    uint64_t uDst    = *puDst;
+    uint64_t uResult = uDst;
+    iemAImpl_add_u64(&uResult, *puReg, pfEFlags);
+    *puDst = uResult;
+    *puReg = uDst;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u64_locked,(uint64_t *puDst, uint64_t *puReg, uint32_t *pfEFlags))
+{
+    uint64_t uOld = ASMAtomicReadU64(puDst);
+    uint64_t uTmpDst;
+    uint32_t fEflTmp;
+    do
+    {
+        uTmpDst = uOld;
+        fEflTmp = *pfEFlags;
+        iemAImpl_add_u64(&uTmpDst, *puReg, pfEFlags);
+    } while (!ASMAtomicCmpXchgExU64(puDst, uTmpDst, uOld, &uOld));
+    *puReg    = uOld;
+    *pfEFlags = fEflTmp;
+}
+
+
+/* Bit operations (same signature as above). */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  We set them as after an
+       logical operation (AND/OR/whatever). */
+    Assert(uSrc < 64);
+    uint64_t uDst = *puDst;
+    if (uDst & RT_BIT_64(uSrc))
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uDst, 64, X86_EFL_CF);
+    else
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uDst, 64, 0);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  We set them as after an
+       logical operation (AND/OR/whatever). */
+    Assert(uSrc < 64);
+    uint64_t fMask = RT_BIT_64(uSrc);
+    uint64_t uDst = *puDst;
+    if (uDst & fMask)
+    {
+        uDst &= ~fMask;
+        *puDst = uDst;
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uDst, 64, X86_EFL_CF);
+    }
+    else
+    {
+        uDst |= fMask;
+        *puDst = uDst;
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uDst, 64, 0);
+    }
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  We set them as after an
+       logical operation (AND/OR/whatever). */
+    Assert(uSrc < 64);
+    uint64_t fMask = RT_BIT_64(uSrc);
+    uint64_t uDst = *puDst;
+    if (uDst & fMask)
+    {
+        uDst &= ~fMask;
+        *puDst = uDst;
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uDst, 64, X86_EFL_CF);
+    }
+    else
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uDst, 64, 0);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  We set them as after an
+       logical operation (AND/OR/whatever). */
+    Assert(uSrc < 64);
+    uint64_t fMask = RT_BIT_64(uSrc);
+    uint64_t uDst = *puDst;
+    if (uDst & fMask)
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uDst, 64, X86_EFL_CF);
+    else
+    {
+        uDst |= fMask;
+        *puDst = uDst;
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uDst, 64, 0);
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u64_locked,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    DO_LOCKED_BIN_OP_U64(btc);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u64_locked,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    DO_LOCKED_BIN_OP_U64(btr);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u64_locked,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    DO_LOCKED_BIN_OP_U64(bts);
+}
+
+
+/* bit scan */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, AF, PF, CF. */
+    /** @todo check what real CPUs do. */
+    if (uSrc)
+    {
+        uint8_t  iBit;
+        uint32_t u32Src;
+        if (uSrc & UINT32_MAX)
+        {
+            iBit = 0;
+            u32Src = uSrc;
+        }
+        else
+        {
+            iBit = 32;
+            u32Src = uSrc >> 32;
+        }
+        if (!(u32Src & UINT16_MAX))
+        {
+            iBit += 16;
+            u32Src >>= 16;
+        }
+        if (!(u32Src & UINT8_MAX))
+        {
+            iBit += 8;
+            u32Src >>= 8;
+        }
+        if (!(u32Src & 0xf))
+        {
+            iBit += 4;
+            u32Src >>= 4;
+        }
+        if (!(u32Src & 0x3))
+        {
+            iBit += 2;
+            u32Src >>= 2;
+        }
+        if (!(u32Src & 1))
+        {
+            iBit += 1;
+            Assert(u32Src & 2);
+        }
+
+        *puDst     = iBit;
+        *pfEFlags &= ~X86_EFL_ZF;
+    }
+    else
+        *pfEFlags |= X86_EFL_ZF;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, AF, PF, CF. */
+    /** @todo check what real CPUs do. */
+    if (uSrc)
+    {
+        uint8_t  iBit;
+        uint32_t u32Src;
+        if (uSrc & UINT64_C(0xffffffff00000000))
+        {
+            iBit = 63;
+            u32Src = uSrc >> 32;
+        }
+        else
+        {
+            iBit = 31;
+            u32Src = uSrc;
+        }
+        if (!(u32Src & UINT32_C(0xffff0000)))
+        {
+            iBit -= 16;
+            u32Src <<= 16;
+        }
+        if (!(u32Src & UINT32_C(0xff000000)))
+        {
+            iBit -= 8;
+            u32Src <<= 8;
+        }
+        if (!(u32Src & UINT32_C(0xf0000000)))
+        {
+            iBit -= 4;
+            u32Src <<= 4;
+        }
+        if (!(u32Src & UINT32_C(0xc0000000)))
+        {
+            iBit -= 2;
+            u32Src <<= 2;
+        }
+        if (!(u32Src & UINT32_C(0x80000000)))
+        {
+            iBit -= 1;
+            Assert(u32Src & RT_BIT(30));
+        }
+
+        *puDst     = iBit;
+        *pfEFlags &= ~X86_EFL_ZF;
+    }
+    else
+        *pfEFlags |= X86_EFL_ZF;
+}
+
+
+/* Unary operands. */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u64,(uint64_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint64_t uDst    = *puDst;
+    uint64_t uResult = uDst + 1;
+    *puDst = uResult;
+
+    /*
+     * Calc EFLAGS.
+     * CF is NOT modified for hysterical raisins (allegedly for carrying and
+     * borrowing in arithmetic loops on intel 8008).
+     */
+    uint32_t fEfl = *pfEFlags & ~(X86_EFL_STATUS_BITS & ~X86_EFL_CF);
+    fEfl |= g_afParity[uResult & 0xff];
+    fEfl |= ((uint32_t)uResult ^ (uint32_t)uDst) & X86_EFL_AF;
+    fEfl |= X86_EFL_CALC_ZF(uResult);
+    fEfl |= X86_EFL_CALC_SF(uResult, 64);
+    fEfl |= (((uDst ^ RT_BIT_64(63)) & uResult) >> (64 - X86_EFL_OF_BIT)) & X86_EFL_OF;
+    *pfEFlags = fEfl;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u64,(uint64_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint64_t uDst    = *puDst;
+    uint64_t uResult = uDst - 1;
+    *puDst = uResult;
+
+    /*
+     * Calc EFLAGS.
+     * CF is NOT modified for hysterical raisins (allegedly for carrying and
+     * borrowing in arithmetic loops on intel 8008).
+     */
+    uint32_t fEfl = *pfEFlags & ~(X86_EFL_STATUS_BITS & ~X86_EFL_CF);
+    fEfl |= g_afParity[uResult & 0xff];
+    fEfl |= ((uint32_t)uResult ^ (uint32_t)uDst) & X86_EFL_AF;
+    fEfl |= X86_EFL_CALC_ZF(uResult);
+    fEfl |= X86_EFL_CALC_SF(uResult, 64);
+    fEfl |= ((uDst & (uResult ^ RT_BIT_64(63))) >> (64 - X86_EFL_OF_BIT)) & X86_EFL_OF;
+    *pfEFlags = fEfl;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_not_u64,(uint64_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint64_t uDst    = *puDst;
+    uint64_t uResult = ~uDst;
+    *puDst = uResult;
+    /* EFLAGS are not modified. */
+    RT_NOREF_PV(pfEFlags);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u64,(uint64_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint64_t uDst    = 0;
+    uint64_t uSrc    = *puDst;
+    uint64_t uResult = uDst - uSrc;
+    *puDst = uResult;
+
+    /* Calc EFLAGS. */
+    uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS;
+    fEfl |= (uSrc != 0) << X86_EFL_CF_BIT;
+    fEfl |= g_afParity[uResult & 0xff];
+    fEfl |= ((uint32_t)uResult ^ (uint32_t)uDst) & X86_EFL_AF;
+    fEfl |= X86_EFL_CALC_ZF(uResult);
+    fEfl |= X86_EFL_CALC_SF(uResult, 64);
+    fEfl |= ((uSrc & uResult) >> (64 - X86_EFL_OF_BIT)) & X86_EFL_OF;
+    *pfEFlags = fEfl;
+}
+
+
+/** 64-bit locked unary operand operation. */
+# define DO_LOCKED_UNARY_OP_U64(a_Mnemonic) \
+    do { \
+        uint64_t uOld = ASMAtomicReadU64(puDst); \
+        uint64_t uTmp; \
+        uint32_t fEflTmp; \
+        do \
+        { \
+            uTmp = uOld; \
+            fEflTmp = *pfEFlags; \
+            iemAImpl_ ## a_Mnemonic ## _u64(&uTmp, &fEflTmp); \
+        } while (!ASMAtomicCmpXchgExU64(puDst, uTmp, uOld, &uOld)); \
+        *pfEFlags = fEflTmp; \
+    } while (0)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u64_locked,(uint64_t  *puDst,  uint32_t *pfEFlags))
+{
+    DO_LOCKED_UNARY_OP_U64(inc);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u64_locked,(uint64_t  *puDst,  uint32_t *pfEFlags))
+{
+    DO_LOCKED_UNARY_OP_U64(dec);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_not_u64_locked,(uint64_t  *puDst,  uint32_t *pfEFlags))
+{
+    DO_LOCKED_UNARY_OP_U64(not);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u64_locked,(uint64_t  *puDst,  uint32_t *pfEFlags))
+{
+    DO_LOCKED_UNARY_OP_U64(neg);
+}
+
+
+/* Shift and rotate. */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_rol_u64,(uint64_t *puDst, uint8_t cShift, uint32_t *pfEFlags))
+{
+    cShift &= 63;
+    if (cShift)
+    {
+        uint64_t uDst = *puDst;
+        uint64_t uResult;
+        uResult  = uDst << cShift;
+        uResult |= uDst >> (64 - cShift);
+        *puDst = uResult;
+
+        /* Calc EFLAGS.  The OF bit is undefined if cShift > 1, we implement
+           it the same way as for 1 bit shifts. */
+        AssertCompile(X86_EFL_CF_BIT == 0);
+        uint32_t fEfl   = *pfEFlags & ~(X86_EFL_CF | X86_EFL_OF);
+        uint32_t fCarry = (uResult & 1);
+        fEfl |= fCarry;
+        fEfl |= ((uResult >> 63) ^ fCarry) << X86_EFL_OF_BIT;
+        *pfEFlags = fEfl;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_ror_u64,(uint64_t *puDst, uint8_t cShift, uint32_t *pfEFlags))
+{
+    cShift &= 63;
+    if (cShift)
+    {
+        uint64_t uDst = *puDst;
+        uint64_t uResult;
+        uResult  = uDst >> cShift;
+        uResult |= uDst << (64 - cShift);
+        *puDst = uResult;
+
+        /* Calc EFLAGS.  The OF bit is undefined if cShift > 1, we implement
+           it the same way as for 1 bit shifts (OF = OF XOR New-CF). */
+        AssertCompile(X86_EFL_CF_BIT == 0);
+        uint32_t fEfl   = *pfEFlags & ~(X86_EFL_CF | X86_EFL_OF);
+        uint32_t fCarry = (uResult >> 63) & X86_EFL_CF;
+        fEfl |= fCarry;
+        fEfl |= (((uResult >> 62) ^ fCarry) << X86_EFL_OF_BIT) & X86_EFL_OF;
+        *pfEFlags = fEfl;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_rcl_u64,(uint64_t *puDst, uint8_t cShift, uint32_t *pfEFlags))
+{
+    cShift &= 63;
+    if (cShift)
+    {
+        uint32_t fEfl = *pfEFlags;
+        uint64_t uDst = *puDst;
+        uint64_t uResult;
+        uResult = uDst << cShift;
+        AssertCompile(X86_EFL_CF_BIT == 0);
+        if (cShift > 1)
+            uResult |= uDst >> (65 - cShift);
+        uResult |= (uint64_t)(fEfl & X86_EFL_CF) << (cShift - 1);
+        *puDst = uResult;
+
+        /* Calc EFLAGS.  The OF bit is undefined if cShift > 1, we implement
+           it the same way as for 1 bit shifts. */
+        uint32_t fCarry = (uDst >> (64 - cShift)) & X86_EFL_CF;
+        fEfl &= ~(X86_EFL_CF | X86_EFL_OF);
+        fEfl |= fCarry;
+        fEfl |= ((uResult >> 63) ^ fCarry) << X86_EFL_OF_BIT;
+        *pfEFlags = fEfl;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_rcr_u64,(uint64_t *puDst, uint8_t cShift, uint32_t *pfEFlags))
+{
+    cShift &= 63;
+    if (cShift)
+    {
+        uint32_t fEfl = *pfEFlags;
+        uint64_t uDst = *puDst;
+        uint64_t uResult;
+        uResult  = uDst >> cShift;
+        AssertCompile(X86_EFL_CF_BIT == 0);
+        if (cShift > 1)
+            uResult |= uDst << (65 - cShift);
+        uResult |= (uint64_t)(fEfl & X86_EFL_CF) << (64 - cShift);
+        *puDst = uResult;
+
+        /* Calc EFLAGS.  The OF bit is undefined if cShift > 1, we implement
+           it the same way as for 1 bit shifts. */
+        uint32_t fCarry = (uDst >> (cShift - 1)) & X86_EFL_CF;
+        fEfl &= ~(X86_EFL_CF | X86_EFL_OF);
+        fEfl |= fCarry;
+        fEfl |= ((uResult >> 63) ^ fCarry) << X86_EFL_OF_BIT;
+        *pfEFlags = fEfl;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_shl_u64,(uint64_t *puDst, uint8_t cShift, uint32_t *pfEFlags))
+{
+    cShift &= 63;
+    if (cShift)
+    {
+        uint64_t uDst = *puDst;
+        uint64_t uResult = uDst << cShift;
+        *puDst = uResult;
+
+        /* Calc EFLAGS.  The OF bit is undefined if cShift > 1, we implement
+           it the same way as for 1 bit shifts.  The AF bit is undefined, we
+           always set it to zero atm. */
+        AssertCompile(X86_EFL_CF_BIT == 0);
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS;
+        uint32_t fCarry = (uDst >> (64 - cShift)) & X86_EFL_CF;
+        fEfl |= fCarry;
+        fEfl |= ((uResult >> 63) ^ fCarry) << X86_EFL_OF_BIT;
+        fEfl |= X86_EFL_CALC_SF(uResult, 64);
+        fEfl |= X86_EFL_CALC_ZF(uResult);
+        fEfl |= g_afParity[uResult & 0xff];
+        *pfEFlags = fEfl;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_shr_u64,(uint64_t *puDst, uint8_t cShift, uint32_t *pfEFlags))
+{
+    cShift &= 63;
+    if (cShift)
+    {
+        uint64_t uDst = *puDst;
+        uint64_t uResult = uDst >> cShift;
+        *puDst = uResult;
+
+        /* Calc EFLAGS.  The OF bit is undefined if cShift > 1, we implement
+           it the same way as for 1 bit shifts.  The AF bit is undefined, we
+           always set it to zero atm. */
+        AssertCompile(X86_EFL_CF_BIT == 0);
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS;
+        fEfl |= (uDst >> (cShift - 1)) & X86_EFL_CF;
+        fEfl |= (uDst >> 63) << X86_EFL_OF_BIT;
+        fEfl |= X86_EFL_CALC_SF(uResult, 64);
+        fEfl |= X86_EFL_CALC_ZF(uResult);
+        fEfl |= g_afParity[uResult & 0xff];
+        *pfEFlags = fEfl;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sar_u64,(uint64_t *puDst, uint8_t cShift, uint32_t *pfEFlags))
+{
+    cShift &= 63;
+    if (cShift)
+    {
+        uint64_t uDst = *puDst;
+        uint64_t uResult = (int64_t)uDst >> cShift;
+        *puDst = uResult;
+
+        /* Calc EFLAGS.  The OF bit is undefined if cShift > 1, we implement
+           it the same way as for 1 bit shifts (0).  The AF bit is undefined,
+           we always set it to zero atm. */
+        AssertCompile(X86_EFL_CF_BIT == 0);
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS;
+        fEfl |= (uDst >> (cShift - 1)) & X86_EFL_CF;
+        fEfl |= X86_EFL_CALC_SF(uResult, 64);
+        fEfl |= X86_EFL_CALC_ZF(uResult);
+        fEfl |= g_afParity[uResult & 0xff];
+        *pfEFlags = fEfl;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_shld_u64,(uint64_t *puDst, uint64_t uSrc, uint8_t cShift, uint32_t *pfEFlags))
+{
+    cShift &= 63;
+    if (cShift)
+    {
+        uint64_t uDst = *puDst;
+        uint64_t uResult;
+        uResult  = uDst << cShift;
+        uResult |= uSrc >> (64 - cShift);
+        *puDst = uResult;
+
+        /* Calc EFLAGS.  The OF bit is undefined if cShift > 1, we implement
+           it the same way as for 1 bit shifts.  The AF bit is undefined,
+           we always set it to zero atm. */
+        AssertCompile(X86_EFL_CF_BIT == 0);
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS;
+        fEfl |= (uDst >> (64 - cShift)) & X86_EFL_CF;
+        fEfl |= (uint32_t)((uDst >> 63) ^ (uint32_t)(uResult >> 63)) << X86_EFL_OF_BIT;
+        fEfl |= X86_EFL_CALC_SF(uResult, 64);
+        fEfl |= X86_EFL_CALC_ZF(uResult);
+        fEfl |= g_afParity[uResult & 0xff];
+        *pfEFlags = fEfl;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_shrd_u64,(uint64_t *puDst, uint64_t uSrc, uint8_t cShift, uint32_t *pfEFlags))
+{
+    cShift &= 63;
+    if (cShift)
+    {
+        uint64_t uDst = *puDst;
+        uint64_t uResult;
+        uResult  = uDst >> cShift;
+        uResult |= uSrc << (64 - cShift);
+        *puDst = uResult;
+
+        /* Calc EFLAGS.  The OF bit is undefined if cShift > 1, we implement
+           it the same way as for 1 bit shifts.  The AF bit is undefined,
+           we always set it to zero atm. */
+        AssertCompile(X86_EFL_CF_BIT == 0);
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS;
+        fEfl |= (uDst >> (cShift - 1)) & X86_EFL_CF;
+        fEfl |= (uint32_t)((uDst >> 63) ^ (uint32_t)(uResult >> 63)) << X86_EFL_OF_BIT;
+        fEfl |= X86_EFL_CALC_SF(uResult, 64);
+        fEfl |= X86_EFL_CALC_ZF(uResult);
+        fEfl |= g_afParity[uResult & 0xff];
+        *pfEFlags = fEfl;
+    }
+}
+
+
+/* misc */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64,(uint64_t *puMem, uint64_t *puReg))
+{
+    /* XCHG implies LOCK. */
+    uint64_t uOldMem = *puMem;
+    while (!ASMAtomicCmpXchgExU64(puMem, *puReg, uOldMem, &uOldMem))
+        ASMNopPause();
+    *puReg = uOldMem;
+}
+
+
+#endif /* RT_ARCH_X86 */
+#ifdef RT_ARCH_X86
+
+/* multiplication and division */
+
+
+IEM_DECL_IMPL_DEF(int, iemAImpl_mul_u64,(uint64_t *pu64RAX, uint64_t *pu64RDX, uint64_t u64Factor, uint32_t *pfEFlags))
+{
+    RTUINT128U Result;
+    RTUInt128MulU64ByU64(&Result, *pu64RAX, u64Factor);
+    *pu64RAX = Result.s.Lo;
+    *pu64RDX = Result.s.Hi;
+
+    /* MUL EFLAGS according to Skylake (similar to IMUL). */
+    *pfEFlags &= ~(X86_EFL_SF | X86_EFL_CF | X86_EFL_OF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_PF);
+    if (Result.s.Lo & RT_BIT_64(63))
+        *pfEFlags |= X86_EFL_SF;
+    *pfEFlags |= g_afParity[Result.s.Lo & 0xff]; /* (Skylake behaviour) */
+    if (Result.s.Hi != 0)
+        *pfEFlags |= X86_EFL_CF | X86_EFL_OF;
+    return 0;
+}
+
+
+IEM_DECL_IMPL_DEF(int, iemAImpl_imul_u64,(uint64_t *pu64RAX, uint64_t *pu64RDX, uint64_t u64Factor, uint32_t *pfEFlags))
+{
+    RTUINT128U Result;
+    *pfEFlags &= ~( X86_EFL_SF | X86_EFL_CF | X86_EFL_OF
+                   /* Skylake always clears: */ | X86_EFL_AF | X86_EFL_ZF
+                   /* Skylake may set: */       | X86_EFL_PF);
+
+    if ((int64_t)*pu64RAX >= 0)
+    {
+        if ((int64_t)u64Factor >= 0)
+        {
+            RTUInt128MulU64ByU64(&Result, *pu64RAX, u64Factor);
+            if (Result.s.Hi != 0 || Result.s.Lo >= UINT64_C(0x8000000000000000))
+                *pfEFlags |= X86_EFL_CF | X86_EFL_OF;
+        }
+        else
+        {
+            RTUInt128MulU64ByU64(&Result, *pu64RAX, UINT64_C(0) - u64Factor);
+            if (Result.s.Hi != 0 || Result.s.Lo > UINT64_C(0x8000000000000000))
+                *pfEFlags |= X86_EFL_CF | X86_EFL_OF;
+            RTUInt128AssignNeg(&Result);
+        }
+    }
+    else
+    {
+        if ((int64_t)u64Factor >= 0)
+        {
+            RTUInt128MulU64ByU64(&Result, UINT64_C(0) - *pu64RAX, u64Factor);
+            if (Result.s.Hi != 0 || Result.s.Lo > UINT64_C(0x8000000000000000))
+                *pfEFlags |= X86_EFL_CF | X86_EFL_OF;
+            RTUInt128AssignNeg(&Result);
+        }
+        else
+        {
+            RTUInt128MulU64ByU64(&Result, UINT64_C(0) - *pu64RAX, UINT64_C(0) - u64Factor);
+            if (Result.s.Hi != 0 || Result.s.Lo >= UINT64_C(0x8000000000000000))
+                *pfEFlags |= X86_EFL_CF | X86_EFL_OF;
+        }
+    }
+    *pu64RAX = Result.s.Lo;
+    if (Result.s.Lo & RT_BIT_64(63))
+        *pfEFlags |= X86_EFL_SF;
+    *pfEFlags |= g_afParity[Result.s.Lo & 0xff]; /* (Skylake behaviour) */
+    *pu64RDX = Result.s.Hi;
+
+    return 0;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+/** @todo Testcase: IMUL 2 and 3 operands. */
+    uint64_t u64Ign;
+    iemAImpl_imul_u64(puDst, &u64Ign, uSrc, pfEFlags);
+}
+
+
+
+IEM_DECL_IMPL_DEF(int, iemAImpl_div_u64,(uint64_t *pu64RAX, uint64_t *pu64RDX, uint64_t u64Divisor, uint32_t *pfEFlags))
+{
+    /* Note! Skylake leaves all flags alone. */
+    RT_NOREF_PV(pfEFlags);
+
+    if (   u64Divisor != 0
+        && *pu64RDX < u64Divisor)
+    {
+        RTUINT128U Dividend;
+        Dividend.s.Lo = *pu64RAX;
+        Dividend.s.Hi = *pu64RDX;
+
+        RTUINT128U Divisor;
+        Divisor.s.Lo = u64Divisor;
+        Divisor.s.Hi = 0;
+
+        RTUINT128U Remainder;
+        RTUINT128U Quotient;
+# ifdef __GNUC__ /* GCC maybe really annoying in function. */
+        Quotient.s.Lo = 0;
+        Quotient.s.Hi = 0;
+# endif
+        RTUInt128DivRem(&Quotient, &Remainder, &Dividend, &Divisor);
+        Assert(Quotient.s.Hi == 0);
+        Assert(Remainder.s.Hi == 0);
+
+        *pu64RAX = Quotient.s.Lo;
+        *pu64RDX = Remainder.s.Lo;
+        /** @todo research the undefined DIV flags. */
+        return 0;
+
+    }
+    /* #DE */
+    return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
+}
+
+
+IEM_DECL_IMPL_DEF(int, iemAImpl_idiv_u64,(uint64_t *pu64RAX, uint64_t *pu64RDX, uint64_t u64Divisor, uint32_t *pfEFlags))
+{
+    /* Note! Skylake leaves all flags alone. */
+    RT_NOREF_PV(pfEFlags);
+
+    if (u64Divisor != 0)
+    {
+        /*
+         * Convert to unsigned division.
+         */
+        RTUINT128U Dividend;
+        Dividend.s.Lo = *pu64RAX;
+        Dividend.s.Hi = *pu64RDX;
+        if ((int64_t)*pu64RDX < 0)
+            RTUInt128AssignNeg(&Dividend);
+
+        RTUINT128U Divisor;
+        Divisor.s.Hi = 0;
+        if ((int64_t)u64Divisor >= 0)
+            Divisor.s.Lo = u64Divisor;
+        else
+            Divisor.s.Lo = UINT64_C(0) - u64Divisor;
+
+        RTUINT128U Remainder;
+        RTUINT128U Quotient;
+# ifdef __GNUC__ /* GCC maybe really annoying in function. */
+        Quotient.s.Lo = 0;
+        Quotient.s.Hi = 0;
+# endif
+        RTUInt128DivRem(&Quotient, &Remainder, &Dividend, &Divisor);
+
+        /*
+         * Setup the result, checking for overflows.
+         */
+        if ((int64_t)u64Divisor >= 0)
+        {
+            if ((int64_t)*pu64RDX >= 0)
+            {
+                /* Positive divisor, positive dividend => result positive. */
+                if (Quotient.s.Hi == 0 && Quotient.s.Lo <= (uint64_t)INT64_MAX)
+                {
+                    *pu64RAX = Quotient.s.Lo;
+                    *pu64RDX = Remainder.s.Lo;
+                    return 0;
+                }
+            }
+            else
+            {
+                /* Positive divisor, positive dividend => result negative. */
+                if (Quotient.s.Hi == 0 && Quotient.s.Lo <= UINT64_C(0x8000000000000000))
+                {
+                    *pu64RAX = UINT64_C(0) - Quotient.s.Lo;
+                    *pu64RDX = UINT64_C(0) - Remainder.s.Lo;
+                    return 0;
+                }
+            }
+        }
+        else
+        {
+            if ((int64_t)*pu64RDX >= 0)
+            {
+                /* Negative divisor, positive dividend => negative quotient, positive remainder. */
+                if (Quotient.s.Hi == 0 && Quotient.s.Lo <= UINT64_C(0x8000000000000000))
+                {
+                    *pu64RAX = UINT64_C(0) - Quotient.s.Lo;
+                    *pu64RDX = Remainder.s.Lo;
+                    return 0;
+                }
+            }
+            else
+            {
+                /* Negative divisor, negative dividend => positive quotient, negative remainder. */
+                if (Quotient.s.Hi == 0 && Quotient.s.Lo <= (uint64_t)INT64_MAX)
+                {
+                    *pu64RAX = Quotient.s.Lo;
+                    *pu64RDX = UINT64_C(0) - Remainder.s.Lo;
+                    return 0;
+                }
+            }
+        }
+    }
+    /* #DE */
+    return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
+}
+
+
+#endif /* RT_ARCH_X86 */
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_arpl,(uint16_t *pu16Dst, uint16_t u16Src, uint32_t *pfEFlags))
+{
+    if ((*pu16Dst & X86_SEL_RPL) < (u16Src & X86_SEL_RPL))
+    {
+        *pu16Dst &= X86_SEL_MASK_OFF_RPL;
+        *pu16Dst |= u16Src & X86_SEL_RPL;
+
+        *pfEFlags |= X86_EFL_ZF;
+    }
+    else
+        *pfEFlags &= ~X86_EFL_ZF;
+}
+
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_fallback,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx,
+                                                      PRTUINT128U pu128RbxRcx, uint32_t *pEFlags))
+{
+    RTUINT128U u128Tmp = *pu128Dst;
+    if (   u128Tmp.s.Lo == pu128RaxRdx->s.Lo
+        && u128Tmp.s.Hi == pu128RaxRdx->s.Hi)
+    {
+        *pu128Dst = *pu128RbxRcx;
+        *pEFlags |= X86_EFL_ZF;
+    }
+    else
+    {
+        *pu128RaxRdx = u128Tmp;
+        *pEFlags &= ~X86_EFL_ZF;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_movsldup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    puDst->au32[0] = puSrc->au32[0];
+    puDst->au32[1] = puSrc->au32[0];
+    puDst->au32[2] = puSrc->au32[2];
+    puDst->au32[3] = puSrc->au32[2];
+}
+
+#ifdef IEM_WITH_VEX
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc))
+{
+    pXState->x87.aXMM[iYRegDst].au32[0] = pXState->x87.aXMM[iYRegSrc].au32[0];
+    pXState->x87.aXMM[iYRegDst].au32[1] = pXState->x87.aXMM[iYRegSrc].au32[0];
+    pXState->x87.aXMM[iYRegDst].au32[2] = pXState->x87.aXMM[iYRegSrc].au32[2];
+    pXState->x87.aXMM[iYRegDst].au32[3] = pXState->x87.aXMM[iYRegSrc].au32[2];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[0];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[0];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[2];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[2];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc))
+{
+    pXState->x87.aXMM[iYRegDst].au32[0]       = pSrc->au32[0];
+    pXState->x87.aXMM[iYRegDst].au32[1]       = pSrc->au32[0];
+    pXState->x87.aXMM[iYRegDst].au32[2]       = pSrc->au32[2];
+    pXState->x87.aXMM[iYRegDst].au32[3]       = pSrc->au32[2];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pSrc->au32[4];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pSrc->au32[4];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pSrc->au32[6];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pSrc->au32[6];
+}
+
+#endif /* IEM_WITH_VEX */
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_movshdup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    puDst->au32[0] = puSrc->au32[1];
+    puDst->au32[1] = puSrc->au32[1];
+    puDst->au32[2] = puSrc->au32[3];
+    puDst->au32[3] = puSrc->au32[3];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_movddup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, uint64_t uSrc))
+{
+    RT_NOREF(pFpuState);
+    puDst->au64[0] = uSrc;
+    puDst->au64[1] = uSrc;
+}
+
+#ifdef IEM_WITH_VEX
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc))
+{
+    pXState->x87.aXMM[iYRegDst].au64[0] = pXState->x87.aXMM[iYRegSrc].au64[0];
+    pXState->x87.aXMM[iYRegDst].au64[1] = pXState->x87.aXMM[iYRegSrc].au64[0];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au64[0] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au64[0];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au64[1] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au64[0];
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc))
+{
+    pXState->x87.aXMM[iYRegDst].au64[0]       = pSrc->au64[0];
+    pXState->x87.aXMM[iYRegDst].au64[1]       = pSrc->au64[0];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au64[0] = pSrc->au64[2];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au64[1] = pSrc->au64[2];
+}
+
+#endif /* IEM_WITH_VEX */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllCImpl.cpp.h b/src/VBox/VMM/VMMAll/IEMAllCImpl.cpp.h
new file mode 100644
index 00000000..15c75456
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllCImpl.cpp.h
@@ -0,0 +1,9046 @@
+/* $Id: IEMAllCImpl.cpp.h $ */
+/** @file
+ * IEM - Instruction Implementation in C/C++ (code include).
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#include "IEMAllCImplSvmInstr.cpp.h"
+#include "IEMAllCImplVmxInstr.cpp.h"
+
+
+/** @name Misc Helpers
+ * @{
+ */
+
+
+/**
+ * Worker function for iemHlpCheckPortIOPermission, don't call directly.
+ *
+ * @returns Strict VBox status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16Port             The port number.
+ * @param   cbOperand           The operand size.
+ */
+static VBOXSTRICTRC iemHlpCheckPortIOPermissionBitmap(PVMCPUCC pVCpu, uint16_t u16Port, uint8_t cbOperand)
+{
+    /* The TSS bits we're interested in are the same on 386 and AMD64. */
+    AssertCompile(AMD64_SEL_TYPE_SYS_TSS_BUSY  == X86_SEL_TYPE_SYS_386_TSS_BUSY);
+    AssertCompile(AMD64_SEL_TYPE_SYS_TSS_AVAIL == X86_SEL_TYPE_SYS_386_TSS_AVAIL);
+    AssertCompileMembersAtSameOffset(X86TSS32, offIoBitmap, X86TSS64, offIoBitmap);
+    AssertCompile(sizeof(X86TSS32) == sizeof(X86TSS64));
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR);
+
+    /*
+     * Check the TSS type, 16-bit TSSes doesn't have any I/O permission bitmap.
+     */
+    Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType);
+    if (RT_UNLIKELY(   pVCpu->cpum.GstCtx.tr.Attr.n.u4Type != AMD64_SEL_TYPE_SYS_TSS_BUSY
+                    && pVCpu->cpum.GstCtx.tr.Attr.n.u4Type != AMD64_SEL_TYPE_SYS_TSS_AVAIL))
+    {
+        Log(("iemHlpCheckPortIOPermissionBitmap: Port=%#x cb=%d - TSS type %#x (attr=%#x) has no I/O bitmap -> #GP(0)\n",
+             u16Port, cbOperand, pVCpu->cpum.GstCtx.tr.Attr.n.u4Type, pVCpu->cpum.GstCtx.tr.Attr.u));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Read the bitmap offset (may #PF).
+     */
+    uint16_t offBitmap;
+    VBOXSTRICTRC rcStrict = iemMemFetchSysU16(pVCpu, &offBitmap, UINT8_MAX,
+                                              pVCpu->cpum.GstCtx.tr.u64Base + RT_UOFFSETOF(X86TSS64, offIoBitmap));
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iemHlpCheckPortIOPermissionBitmap: Error reading offIoBitmap (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /*
+     * The bit range from u16Port to (u16Port + cbOperand - 1), however intel
+     * describes the CPU actually reading two bytes regardless of whether the
+     * bit range crosses a byte boundrary.  Thus the + 1 in the test below.
+     */
+    uint32_t offFirstBit = (uint32_t)u16Port / 8 + offBitmap;
+    /** @todo check if real CPUs ensures that offBitmap has a minimum value of
+     *        for instance sizeof(X86TSS32). */
+    if (offFirstBit + 1 > pVCpu->cpum.GstCtx.tr.u32Limit) /* the limit is inclusive */
+    {
+        Log(("iemHlpCheckPortIOPermissionBitmap: offFirstBit=%#x + 1 is beyond u32Limit=%#x -> #GP(0)\n",
+             offFirstBit, pVCpu->cpum.GstCtx.tr.u32Limit));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Read the necessary bits.
+     */
+    /** @todo Test the assertion in the intel manual that the CPU reads two
+     *        bytes.  The question is how this works wrt to #PF and #GP on the
+     *        2nd byte when it's not required. */
+    uint16_t bmBytes = UINT16_MAX;
+    rcStrict = iemMemFetchSysU16(pVCpu, &bmBytes, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + offFirstBit);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iemHlpCheckPortIOPermissionBitmap: Error reading I/O bitmap @%#x (%Rrc)\n", offFirstBit, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /*
+     * Perform the check.
+     */
+    uint16_t fPortMask = (1 << cbOperand) - 1;
+    bmBytes >>= (u16Port & 7);
+    if (bmBytes & fPortMask)
+    {
+        Log(("iemHlpCheckPortIOPermissionBitmap: u16Port=%#x LB %u - access denied (bm=%#x mask=%#x) -> #GP(0)\n",
+             u16Port, cbOperand, bmBytes, fPortMask));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if we are allowed to access the given I/O port, raising the
+ * appropriate exceptions if we aren't (or if the I/O bitmap is not
+ * accessible).
+ *
+ * @returns Strict VBox status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16Port             The port number.
+ * @param   cbOperand           The operand size.
+ */
+DECLINLINE(VBOXSTRICTRC) iemHlpCheckPortIOPermission(PVMCPUCC pVCpu, uint16_t u16Port, uint8_t cbOperand)
+{
+    X86EFLAGS Efl;
+    Efl.u = IEMMISC_GET_EFL(pVCpu);
+    if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
+        && (    pVCpu->iem.s.uCpl > Efl.Bits.u2IOPL
+            ||  Efl.Bits.u1VM) )
+        return iemHlpCheckPortIOPermissionBitmap(pVCpu, u16Port, cbOperand);
+    return VINF_SUCCESS;
+}
+
+
+#if 0
+/**
+ * Calculates the parity bit.
+ *
+ * @returns true if the bit is set, false if not.
+ * @param   u8Result            The least significant byte of the result.
+ */
+static bool iemHlpCalcParityFlag(uint8_t u8Result)
+{
+    /*
+     * Parity is set if the number of bits in the least significant byte of
+     * the result is even.
+     */
+    uint8_t cBits;
+    cBits  = u8Result & 1;              /* 0 */
+    u8Result >>= 1;
+    cBits += u8Result & 1;
+    u8Result >>= 1;
+    cBits += u8Result & 1;
+    u8Result >>= 1;
+    cBits += u8Result & 1;
+    u8Result >>= 1;
+    cBits += u8Result & 1;              /* 4 */
+    u8Result >>= 1;
+    cBits += u8Result & 1;
+    u8Result >>= 1;
+    cBits += u8Result & 1;
+    u8Result >>= 1;
+    cBits += u8Result & 1;
+    return !(cBits & 1);
+}
+#endif /* not used */
+
+
+/**
+ * Updates the specified flags according to a 8-bit result.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u8Result            The result to set the flags according to.
+ * @param   fToUpdate           The flags to update.
+ * @param   fUndefined          The flags that are specified as undefined.
+ */
+static void iemHlpUpdateArithEFlagsU8(PVMCPUCC pVCpu, uint8_t u8Result, uint32_t fToUpdate, uint32_t fUndefined)
+{
+    uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u;
+    iemAImpl_test_u8(&u8Result, u8Result, &fEFlags);
+    pVCpu->cpum.GstCtx.eflags.u &= ~(fToUpdate | fUndefined);
+    pVCpu->cpum.GstCtx.eflags.u |= (fToUpdate | fUndefined) & fEFlags;
+}
+
+
+/**
+ * Updates the specified flags according to a 16-bit result.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16Result           The result to set the flags according to.
+ * @param   fToUpdate           The flags to update.
+ * @param   fUndefined          The flags that are specified as undefined.
+ */
+static void iemHlpUpdateArithEFlagsU16(PVMCPUCC pVCpu, uint16_t u16Result, uint32_t fToUpdate, uint32_t fUndefined)
+{
+    uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u;
+    iemAImpl_test_u16(&u16Result, u16Result, &fEFlags);
+    pVCpu->cpum.GstCtx.eflags.u &= ~(fToUpdate | fUndefined);
+    pVCpu->cpum.GstCtx.eflags.u |= (fToUpdate | fUndefined) & fEFlags;
+}
+
+
+/**
+ * Helper used by iret.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   uCpl                The new CPL.
+ * @param   pSReg               Pointer to the segment register.
+ */
+static void iemHlpAdjustSelectorForNewCpl(PVMCPUCC pVCpu, uint8_t uCpl, PCPUMSELREG pSReg)
+{
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
+
+    if (   uCpl > pSReg->Attr.n.u2Dpl
+        && pSReg->Attr.n.u1DescType /* code or data, not system */
+        &&    (pSReg->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+           !=                         (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) /* not conforming code */
+        iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, 0);
+}
+
+
+/**
+ * Indicates that we have modified the FPU state.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(void) iemHlpUsedFpu(PVMCPUCC pVCpu)
+{
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
+}
+
+/** @} */
+
+/** @name C Implementations
+ * @{
+ */
+
+/**
+ * Implements a 16-bit popa.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_popa_16)
+{
+    RTGCPTR         GCPtrStart  = iemRegGetEffRsp(pVCpu);
+    RTGCPTR         GCPtrLast   = GCPtrStart + 15;
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * The docs are a bit hard to comprehend here, but it looks like we wrap
+     * around in real mode as long as none of the individual "popa" crosses the
+     * end of the stack segment.  In protected mode we check the whole access
+     * in one go.  For efficiency, only do the word-by-word thing if we're in
+     * danger of wrapping around.
+     */
+    /** @todo do popa boundary / wrap-around checks.  */
+    if (RT_UNLIKELY(   IEM_IS_REAL_OR_V86_MODE(pVCpu)
+                    && (pVCpu->cpum.GstCtx.cs.u32Limit < GCPtrLast)) ) /* ASSUMES 64-bit RTGCPTR */
+    {
+        /* word-by-word */
+        RTUINT64U TmpRsp;
+        TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+        rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.di, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.si, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.bp, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            iemRegAddToRspEx(pVCpu, &TmpRsp, 2); /* sp */
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.bx, &TmpRsp);
+        }
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.dx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.cx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.ax, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        }
+    }
+    else
+    {
+        uint16_t const *pa16Mem = NULL;
+        rcStrict = iemMemMap(pVCpu, (void **)&pa16Mem, 16, X86_SREG_SS, GCPtrStart, IEM_ACCESS_STACK_R);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.di = pa16Mem[7 - X86_GREG_xDI];
+            pVCpu->cpum.GstCtx.si = pa16Mem[7 - X86_GREG_xSI];
+            pVCpu->cpum.GstCtx.bp = pa16Mem[7 - X86_GREG_xBP];
+            /* skip sp */
+            pVCpu->cpum.GstCtx.bx = pa16Mem[7 - X86_GREG_xBX];
+            pVCpu->cpum.GstCtx.dx = pa16Mem[7 - X86_GREG_xDX];
+            pVCpu->cpum.GstCtx.cx = pa16Mem[7 - X86_GREG_xCX];
+            pVCpu->cpum.GstCtx.ax = pa16Mem[7 - X86_GREG_xAX];
+            rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pa16Mem, IEM_ACCESS_STACK_R);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                iemRegAddToRsp(pVCpu, 16);
+                iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            }
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements a 32-bit popa.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_popa_32)
+{
+    RTGCPTR         GCPtrStart  = iemRegGetEffRsp(pVCpu);
+    RTGCPTR         GCPtrLast   = GCPtrStart + 31;
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * The docs are a bit hard to comprehend here, but it looks like we wrap
+     * around in real mode as long as none of the individual "popa" crosses the
+     * end of the stack segment.  In protected mode we check the whole access
+     * in one go.  For efficiency, only do the word-by-word thing if we're in
+     * danger of wrapping around.
+     */
+    /** @todo do popa boundary / wrap-around checks.  */
+    if (RT_UNLIKELY(   IEM_IS_REAL_OR_V86_MODE(pVCpu)
+                    && (pVCpu->cpum.GstCtx.cs.u32Limit < GCPtrLast)) ) /* ASSUMES 64-bit RTGCPTR */
+    {
+        /* word-by-word */
+        RTUINT64U TmpRsp;
+        TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+        rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.edi, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.esi, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ebp, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            iemRegAddToRspEx(pVCpu, &TmpRsp, 2); /* sp */
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ebx, &TmpRsp);
+        }
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.edx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ecx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.eax, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+        {
+#if 1  /** @todo what actually happens with the high bits when we're in 16-bit mode? */
+            pVCpu->cpum.GstCtx.rdi &= UINT32_MAX;
+            pVCpu->cpum.GstCtx.rsi &= UINT32_MAX;
+            pVCpu->cpum.GstCtx.rbp &= UINT32_MAX;
+            pVCpu->cpum.GstCtx.rbx &= UINT32_MAX;
+            pVCpu->cpum.GstCtx.rdx &= UINT32_MAX;
+            pVCpu->cpum.GstCtx.rcx &= UINT32_MAX;
+            pVCpu->cpum.GstCtx.rax &= UINT32_MAX;
+#endif
+            pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        }
+    }
+    else
+    {
+        uint32_t const *pa32Mem;
+        rcStrict = iemMemMap(pVCpu, (void **)&pa32Mem, 32, X86_SREG_SS, GCPtrStart, IEM_ACCESS_STACK_R);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rdi = pa32Mem[7 - X86_GREG_xDI];
+            pVCpu->cpum.GstCtx.rsi = pa32Mem[7 - X86_GREG_xSI];
+            pVCpu->cpum.GstCtx.rbp = pa32Mem[7 - X86_GREG_xBP];
+            /* skip esp */
+            pVCpu->cpum.GstCtx.rbx = pa32Mem[7 - X86_GREG_xBX];
+            pVCpu->cpum.GstCtx.rdx = pa32Mem[7 - X86_GREG_xDX];
+            pVCpu->cpum.GstCtx.rcx = pa32Mem[7 - X86_GREG_xCX];
+            pVCpu->cpum.GstCtx.rax = pa32Mem[7 - X86_GREG_xAX];
+            rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pa32Mem, IEM_ACCESS_STACK_R);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                iemRegAddToRsp(pVCpu, 32);
+                iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            }
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements a 16-bit pusha.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_pusha_16)
+{
+    RTGCPTR         GCPtrTop    = iemRegGetEffRsp(pVCpu);
+    RTGCPTR         GCPtrBottom = GCPtrTop - 15;
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * The docs are a bit hard to comprehend here, but it looks like we wrap
+     * around in real mode as long as none of the individual "pushd" crosses the
+     * end of the stack segment.  In protected mode we check the whole access
+     * in one go.  For efficiency, only do the word-by-word thing if we're in
+     * danger of wrapping around.
+     */
+    /** @todo do pusha boundary / wrap-around checks.  */
+    if (RT_UNLIKELY(   GCPtrBottom > GCPtrTop
+                    && IEM_IS_REAL_OR_V86_MODE(pVCpu) ) )
+    {
+        /* word-by-word */
+        RTUINT64U TmpRsp;
+        TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+        rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.ax, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.cx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.dx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.bx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.sp, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.bp, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.si, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.di, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        }
+    }
+    else
+    {
+        GCPtrBottom--;
+        uint16_t *pa16Mem = NULL;
+        rcStrict = iemMemMap(pVCpu, (void **)&pa16Mem, 16, X86_SREG_SS, GCPtrBottom, IEM_ACCESS_STACK_W);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pa16Mem[7 - X86_GREG_xDI] = pVCpu->cpum.GstCtx.di;
+            pa16Mem[7 - X86_GREG_xSI] = pVCpu->cpum.GstCtx.si;
+            pa16Mem[7 - X86_GREG_xBP] = pVCpu->cpum.GstCtx.bp;
+            pa16Mem[7 - X86_GREG_xSP] = pVCpu->cpum.GstCtx.sp;
+            pa16Mem[7 - X86_GREG_xBX] = pVCpu->cpum.GstCtx.bx;
+            pa16Mem[7 - X86_GREG_xDX] = pVCpu->cpum.GstCtx.dx;
+            pa16Mem[7 - X86_GREG_xCX] = pVCpu->cpum.GstCtx.cx;
+            pa16Mem[7 - X86_GREG_xAX] = pVCpu->cpum.GstCtx.ax;
+            rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pa16Mem, IEM_ACCESS_STACK_W);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                iemRegSubFromRsp(pVCpu, 16);
+                iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            }
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements a 32-bit pusha.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_pusha_32)
+{
+    RTGCPTR         GCPtrTop    = iemRegGetEffRsp(pVCpu);
+    RTGCPTR         GCPtrBottom = GCPtrTop - 31;
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * The docs are a bit hard to comprehend here, but it looks like we wrap
+     * around in real mode as long as none of the individual "pusha" crosses the
+     * end of the stack segment.  In protected mode we check the whole access
+     * in one go.  For efficiency, only do the word-by-word thing if we're in
+     * danger of wrapping around.
+     */
+    /** @todo do pusha boundary / wrap-around checks.  */
+    if (RT_UNLIKELY(   GCPtrBottom > GCPtrTop
+                    && IEM_IS_REAL_OR_V86_MODE(pVCpu) ) )
+    {
+        /* word-by-word */
+        RTUINT64U TmpRsp;
+        TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+        rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.eax, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ecx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.edx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ebx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.esp, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ebp, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.esi, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.edi, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        }
+    }
+    else
+    {
+        GCPtrBottom--;
+        uint32_t *pa32Mem;
+        rcStrict = iemMemMap(pVCpu, (void **)&pa32Mem, 32, X86_SREG_SS, GCPtrBottom, IEM_ACCESS_STACK_W);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pa32Mem[7 - X86_GREG_xDI] = pVCpu->cpum.GstCtx.edi;
+            pa32Mem[7 - X86_GREG_xSI] = pVCpu->cpum.GstCtx.esi;
+            pa32Mem[7 - X86_GREG_xBP] = pVCpu->cpum.GstCtx.ebp;
+            pa32Mem[7 - X86_GREG_xSP] = pVCpu->cpum.GstCtx.esp;
+            pa32Mem[7 - X86_GREG_xBX] = pVCpu->cpum.GstCtx.ebx;
+            pa32Mem[7 - X86_GREG_xDX] = pVCpu->cpum.GstCtx.edx;
+            pa32Mem[7 - X86_GREG_xCX] = pVCpu->cpum.GstCtx.ecx;
+            pa32Mem[7 - X86_GREG_xAX] = pVCpu->cpum.GstCtx.eax;
+            rcStrict = iemMemCommitAndUnmap(pVCpu, pa32Mem, IEM_ACCESS_STACK_W);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                iemRegSubFromRsp(pVCpu, 32);
+                iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            }
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements pushf.
+ *
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_pushf, IEMMODE, enmEffOpSize)
+{
+    VBOXSTRICTRC rcStrict;
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_PUSHF))
+    {
+        Log2(("pushf: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_PUSHF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * If we're in V8086 mode some care is required (which is why we're in
+     * doing this in a C implementation).
+     */
+    uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
+    if (   (fEfl & X86_EFL_VM)
+        && X86_EFL_GET_IOPL(fEfl) != 3 )
+    {
+        Assert(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE);
+        if (   enmEffOpSize != IEMMODE_16BIT
+            || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME))
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        fEfl &= ~X86_EFL_IF;          /* (RF and VM are out of range) */
+        fEfl |= (fEfl & X86_EFL_VIF) >> (19 - 9);
+        rcStrict = iemMemStackPushU16(pVCpu, (uint16_t)fEfl);
+    }
+    else
+    {
+
+        /*
+         * Ok, clear RF and VM, adjust for ancient CPUs, and push the flags.
+         */
+        fEfl &= ~(X86_EFL_RF | X86_EFL_VM);
+
+        switch (enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
+                if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_186)
+                    fEfl |= UINT16_C(0xf000);
+                rcStrict = iemMemStackPushU16(pVCpu, (uint16_t)fEfl);
+                break;
+            case IEMMODE_32BIT:
+                rcStrict = iemMemStackPushU32(pVCpu, fEfl);
+                break;
+            case IEMMODE_64BIT:
+                rcStrict = iemMemStackPushU64(pVCpu, fEfl);
+                break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements popf.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_popf, IEMMODE, enmEffOpSize)
+{
+    uint32_t const  fEflOld = IEMMISC_GET_EFL(pVCpu);
+    VBOXSTRICTRC    rcStrict;
+    uint32_t        fEflNew;
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_POPF))
+    {
+        Log2(("popf: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_POPF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * V8086 is special as usual.
+     */
+    if (fEflOld & X86_EFL_VM)
+    {
+        /*
+         * Almost anything goes if IOPL is 3.
+         */
+        if (X86_EFL_GET_IOPL(fEflOld) == 3)
+        {
+            switch (enmEffOpSize)
+            {
+                case IEMMODE_16BIT:
+                {
+                    uint16_t u16Value;
+                    rcStrict = iemMemStackPopU16(pVCpu, &u16Value);
+                    if (rcStrict != VINF_SUCCESS)
+                        return rcStrict;
+                    fEflNew = u16Value | (fEflOld & UINT32_C(0xffff0000));
+                    break;
+                }
+                case IEMMODE_32BIT:
+                    rcStrict = iemMemStackPopU32(pVCpu, &fEflNew);
+                    if (rcStrict != VINF_SUCCESS)
+                        return rcStrict;
+                    break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+
+            const uint32_t fPopfBits = pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386
+                                     ? X86_EFL_POPF_BITS : X86_EFL_POPF_BITS_386;
+            fEflNew &=   fPopfBits & ~(X86_EFL_IOPL);
+            fEflNew |= ~(fPopfBits & ~(X86_EFL_IOPL)) & fEflOld;
+        }
+        /*
+         * Interrupt flag virtualization with CR4.VME=1.
+         */
+        else if (   enmEffOpSize == IEMMODE_16BIT
+                 && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME) )
+        {
+            uint16_t    u16Value;
+            RTUINT64U   TmpRsp;
+            TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Value, &TmpRsp);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /** @todo Is the popf VME #GP(0) delivered after updating RSP+RIP
+             *        or before? */
+            if (    (   (u16Value & X86_EFL_IF)
+                     && (fEflOld  & X86_EFL_VIP))
+                ||  (u16Value & X86_EFL_TF) )
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+
+            fEflNew = u16Value | (fEflOld & UINT32_C(0xffff0000) & ~X86_EFL_VIF);
+            fEflNew |= (fEflNew & X86_EFL_IF) << (19 - 9);
+            fEflNew &=   X86_EFL_POPF_BITS & ~(X86_EFL_IOPL | X86_EFL_IF);
+            fEflNew |= ~(X86_EFL_POPF_BITS & ~(X86_EFL_IOPL | X86_EFL_IF)) & fEflOld;
+
+            pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+        }
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    }
+    /*
+     * Not in V8086 mode.
+     */
+    else
+    {
+        /* Pop the flags. */
+        switch (enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                uint16_t u16Value;
+                rcStrict = iemMemStackPopU16(pVCpu, &u16Value);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+                fEflNew = u16Value | (fEflOld & UINT32_C(0xffff0000));
+
+                /*
+                 * Ancient CPU adjustments:
+                 *  - 8086, 80186, V20/30:
+                 *    Fixed bits 15:12 bits are not kept correctly internally, mostly for
+                 *    practical reasons (masking below).  We add them when pushing flags.
+                 *  - 80286:
+                 *    The NT and IOPL flags cannot be popped from real mode and are
+                 *    therefore always zero (since a 286 can never exit from PM and
+                 *    their initial value is zero).  This changed on a 386 and can
+                 *    therefore be used to detect 286 or 386 CPU in real mode.
+                 */
+                if (   IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_286
+                    && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
+                    fEflNew &= ~(X86_EFL_NT | X86_EFL_IOPL);
+                break;
+            }
+            case IEMMODE_32BIT:
+                rcStrict = iemMemStackPopU32(pVCpu, &fEflNew);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+                break;
+            case IEMMODE_64BIT:
+            {
+                uint64_t u64Value;
+                rcStrict = iemMemStackPopU64(pVCpu, &u64Value);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+                fEflNew = u64Value;  /** @todo testcase: Check exactly what happens if high bits are set. */
+                break;
+            }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+
+        /* Merge them with the current flags. */
+        const uint32_t fPopfBits = pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386
+                                 ? X86_EFL_POPF_BITS : X86_EFL_POPF_BITS_386;
+        if (   (fEflNew & (X86_EFL_IOPL | X86_EFL_IF)) == (fEflOld & (X86_EFL_IOPL | X86_EFL_IF))
+            || pVCpu->iem.s.uCpl == 0)
+        {
+            fEflNew &=  fPopfBits;
+            fEflNew |= ~fPopfBits & fEflOld;
+        }
+        else if (pVCpu->iem.s.uCpl <= X86_EFL_GET_IOPL(fEflOld))
+        {
+            fEflNew &=   fPopfBits & ~(X86_EFL_IOPL);
+            fEflNew |= ~(fPopfBits & ~(X86_EFL_IOPL)) & fEflOld;
+        }
+        else
+        {
+            fEflNew &=   fPopfBits & ~(X86_EFL_IOPL | X86_EFL_IF);
+            fEflNew |= ~(fPopfBits & ~(X86_EFL_IOPL | X86_EFL_IF)) & fEflOld;
+        }
+    }
+
+    /*
+     * Commit the flags.
+     */
+    Assert(fEflNew & RT_BIT_32(1));
+    IEMMISC_SET_EFL(pVCpu, fEflNew);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements an indirect call.
+ *
+ * @param   uNewPC          The new program counter (RIP) value (loaded from the
+ *                          operand).
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_call_16, uint16_t, uNewPC)
+{
+    uint16_t uOldPC = pVCpu->cpum.GstCtx.ip + cbInstr;
+    if (uNewPC > pVCpu->cpum.GstCtx.cs.u32Limit)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldPC);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    pVCpu->cpum.GstCtx.rip = uNewPC;
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements a 16-bit relative call.
+ *
+ * @param   offDisp      The displacment offset.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_call_rel_16, int16_t, offDisp)
+{
+    uint16_t uOldPC = pVCpu->cpum.GstCtx.ip + cbInstr;
+    uint16_t uNewPC = uOldPC + offDisp;
+    if (uNewPC > pVCpu->cpum.GstCtx.cs.u32Limit)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldPC);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    pVCpu->cpum.GstCtx.rip = uNewPC;
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements a 32-bit indirect call.
+ *
+ * @param   uNewPC          The new program counter (RIP) value (loaded from the
+ *                          operand).
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_call_32, uint32_t, uNewPC)
+{
+    uint32_t uOldPC = pVCpu->cpum.GstCtx.eip + cbInstr;
+    if (uNewPC > pVCpu->cpum.GstCtx.cs.u32Limit)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldPC);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    pVCpu->cpum.GstCtx.rip = uNewPC;
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements a 32-bit relative call.
+ *
+ * @param   offDisp      The displacment offset.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_call_rel_32, int32_t, offDisp)
+{
+    uint32_t uOldPC = pVCpu->cpum.GstCtx.eip + cbInstr;
+    uint32_t uNewPC = uOldPC + offDisp;
+    if (uNewPC > pVCpu->cpum.GstCtx.cs.u32Limit)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldPC);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    pVCpu->cpum.GstCtx.rip = uNewPC;
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements a 64-bit indirect call.
+ *
+ * @param   uNewPC          The new program counter (RIP) value (loaded from the
+ *                          operand).
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_call_64, uint64_t, uNewPC)
+{
+    uint64_t uOldPC = pVCpu->cpum.GstCtx.rip + cbInstr;
+    if (!IEM_IS_CANONICAL(uNewPC))
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldPC);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    pVCpu->cpum.GstCtx.rip = uNewPC;
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements a 64-bit relative call.
+ *
+ * @param   offDisp      The displacment offset.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_call_rel_64, int64_t, offDisp)
+{
+    uint64_t uOldPC = pVCpu->cpum.GstCtx.rip + cbInstr;
+    uint64_t uNewPC = uOldPC + offDisp;
+    if (!IEM_IS_CANONICAL(uNewPC))
+        return iemRaiseNotCanonical(pVCpu);
+
+    VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldPC);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    pVCpu->cpum.GstCtx.rip = uNewPC;
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements far jumps and calls thru task segments (TSS).
+ *
+ * @param   uSel            The selector.
+ * @param   enmBranch       The kind of branching we're performing.
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   pDesc           The descriptor corresponding to @a uSel. The type is
+ *                          task gate.
+ */
+IEM_CIMPL_DEF_4(iemCImpl_BranchTaskSegment, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc)
+{
+#ifndef IEM_IMPLEMENTS_TASKSWITCH
+    IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
+#else
+    Assert(enmBranch == IEMBRANCH_JUMP || enmBranch == IEMBRANCH_CALL);
+    Assert(   pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
+           || pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL);
+    RT_NOREF_PV(enmEffOpSize);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    if (   pDesc->Legacy.Gate.u2Dpl < pVCpu->iem.s.uCpl
+        || pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL))
+    {
+        Log(("BranchTaskSegment invalid priv. uSel=%04x TSS DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl,
+             pVCpu->iem.s.uCpl, (uSel & X86_SEL_RPL)));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /** @todo This is checked earlier for far jumps (see iemCImpl_FarJmp) but not
+     *        far calls (see iemCImpl_callf). Most likely in both cases it should be
+     *        checked here, need testcases. */
+    if (!pDesc->Legacy.Gen.u1Present)
+    {
+        Log(("BranchTaskSegment TSS not present uSel=%04x -> #NP\n", uSel));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr;
+    return iemTaskSwitch(pVCpu, enmBranch == IEMBRANCH_JUMP ? IEMTASKSWITCH_JUMP : IEMTASKSWITCH_CALL,
+                         uNextEip, 0 /* fFlags */, 0 /* uErr */, 0 /* uCr2 */, uSel, pDesc);
+#endif
+}
+
+
+/**
+ * Implements far jumps and calls thru task gates.
+ *
+ * @param   uSel            The selector.
+ * @param   enmBranch       The kind of branching we're performing.
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   pDesc           The descriptor corresponding to @a uSel. The type is
+ *                          task gate.
+ */
+IEM_CIMPL_DEF_4(iemCImpl_BranchTaskGate, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc)
+{
+#ifndef IEM_IMPLEMENTS_TASKSWITCH
+    IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
+#else
+    Assert(enmBranch == IEMBRANCH_JUMP || enmBranch == IEMBRANCH_CALL);
+    RT_NOREF_PV(enmEffOpSize);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    if (   pDesc->Legacy.Gate.u2Dpl < pVCpu->iem.s.uCpl
+        || pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL))
+    {
+        Log(("BranchTaskGate invalid priv. uSel=%04x TSS DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl,
+             pVCpu->iem.s.uCpl, (uSel & X86_SEL_RPL)));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /** @todo This is checked earlier for far jumps (see iemCImpl_FarJmp) but not
+     *        far calls (see iemCImpl_callf). Most likely in both cases it should be
+     *        checked here, need testcases. */
+    if (!pDesc->Legacy.Gen.u1Present)
+    {
+        Log(("BranchTaskSegment segment not present uSel=%04x -> #NP\n", uSel));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /*
+     * Fetch the new TSS descriptor from the GDT.
+     */
+    RTSEL uSelTss = pDesc->Legacy.Gate.u16Sel;
+    if (uSelTss  & X86_SEL_LDT)
+    {
+        Log(("BranchTaskGate TSS is in LDT. uSel=%04x uSelTss=%04x -> #GP\n", uSel, uSelTss));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    IEMSELDESC TssDesc;
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &TssDesc, uSelTss, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    if (TssDesc.Legacy.Gate.u4Type & X86_SEL_TYPE_SYS_TSS_BUSY_MASK)
+    {
+        Log(("BranchTaskGate TSS is busy. uSel=%04x uSelTss=%04x DescType=%#x -> #GP\n", uSel, uSelTss,
+             TssDesc.Legacy.Gate.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    if (!TssDesc.Legacy.Gate.u1Present)
+    {
+        Log(("BranchTaskGate TSS is not present. uSel=%04x uSelTss=%04x -> #NP\n", uSel, uSelTss));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSelTss & X86_SEL_MASK_OFF_RPL);
+    }
+
+    uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr;
+    return iemTaskSwitch(pVCpu, enmBranch == IEMBRANCH_JUMP ? IEMTASKSWITCH_JUMP : IEMTASKSWITCH_CALL,
+                         uNextEip, 0 /* fFlags */, 0 /* uErr */, 0 /* uCr2 */, uSelTss, &TssDesc);
+#endif
+}
+
+
+/**
+ * Implements far jumps and calls thru call gates.
+ *
+ * @param   uSel            The selector.
+ * @param   enmBranch       The kind of branching we're performing.
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   pDesc           The descriptor corresponding to @a uSel. The type is
+ *                          call gate.
+ */
+IEM_CIMPL_DEF_4(iemCImpl_BranchCallGate, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc)
+{
+#define IEM_IMPLEMENTS_CALLGATE
+#ifndef IEM_IMPLEMENTS_CALLGATE
+    IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
+#else
+    RT_NOREF_PV(enmEffOpSize);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    /* NB: Far jumps can only do intra-privilege transfers. Far calls support
+     * inter-privilege calls and are much more complex.
+     *
+     * NB: 64-bit call gate has the same type as a 32-bit call gate! If
+     * EFER.LMA=1, the gate must be 64-bit. Conversely if EFER.LMA=0, the gate
+     * must be 16-bit or 32-bit.
+     */
+    /** @todo: effective operand size is probably irrelevant here, only the
+     *         call gate bitness matters??
+     */
+    VBOXSTRICTRC    rcStrict;
+    RTPTRUNION      uPtrRet;
+    uint64_t        uNewRsp;
+    uint64_t        uNewRip;
+    uint64_t        u64Base;
+    uint32_t        cbLimit;
+    RTSEL           uNewCS;
+    IEMSELDESC      DescCS;
+
+    AssertCompile(X86_SEL_TYPE_SYS_386_CALL_GATE == AMD64_SEL_TYPE_SYS_CALL_GATE);
+    Assert(enmBranch == IEMBRANCH_JUMP || enmBranch == IEMBRANCH_CALL);
+    Assert(   pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE
+           || pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE);
+
+    /* Determine the new instruction pointer from the gate descriptor. */
+    uNewRip = pDesc->Legacy.Gate.u16OffsetLow
+            | ((uint32_t)pDesc->Legacy.Gate.u16OffsetHigh << 16)
+            | ((uint64_t)pDesc->Long.Gate.u32OffsetTop    << 32);
+
+    /* Perform DPL checks on the gate descriptor. */
+    if (   pDesc->Legacy.Gate.u2Dpl < pVCpu->iem.s.uCpl
+        || pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL))
+    {
+        Log(("BranchCallGate invalid priv. uSel=%04x Gate DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl,
+             pVCpu->iem.s.uCpl, (uSel & X86_SEL_RPL)));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+
+    /** @todo does this catch NULL selectors, too? */
+    if (!pDesc->Legacy.Gen.u1Present)
+    {
+        Log(("BranchCallGate Gate not present uSel=%04x -> #NP\n", uSel));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
+    }
+
+    /*
+     * Fetch the target CS descriptor from the GDT or LDT.
+     */
+    uNewCS = pDesc->Legacy.Gate.u16Sel;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Target CS must be a code selector. */
+    if (   !DescCS.Legacy.Gen.u1DescType
+        || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) )
+    {
+        Log(("BranchCallGate %04x:%08RX64 -> not a code selector (u1DescType=%u u4Type=%#x).\n",
+             uNewCS, uNewRip, DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
+    }
+
+    /* Privilege checks on target CS. */
+    if (enmBranch == IEMBRANCH_JUMP)
+    {
+        if (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
+        {
+            if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
+            {
+                Log(("BranchCallGate jump (conforming) bad DPL uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n",
+                     uNewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
+            }
+        }
+        else
+        {
+            if (DescCS.Legacy.Gen.u2Dpl != pVCpu->iem.s.uCpl)
+            {
+                Log(("BranchCallGate jump (non-conforming) bad DPL uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n",
+                     uNewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
+            }
+        }
+    }
+    else
+    {
+        Assert(enmBranch == IEMBRANCH_CALL);
+        if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
+        {
+            Log(("BranchCallGate call invalid priv. uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n",
+                 uNewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
+        }
+    }
+
+    /* Additional long mode checks. */
+    if (IEM_IS_LONG_MODE(pVCpu))
+    {
+        if (!DescCS.Legacy.Gen.u1Long)
+        {
+            Log(("BranchCallGate uNewCS %04x -> not a 64-bit code segment.\n", uNewCS));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
+        }
+
+        /* L vs D. */
+        if (   DescCS.Legacy.Gen.u1Long
+            && DescCS.Legacy.Gen.u1DefBig)
+        {
+            Log(("BranchCallGate uNewCS %04x -> both L and D are set.\n", uNewCS));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
+        }
+    }
+
+    if (!DescCS.Legacy.Gate.u1Present)
+    {
+        Log(("BranchCallGate target CS is not present. uSel=%04x uNewCS=%04x -> #NP(CS)\n", uSel, uNewCS));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCS);
+    }
+
+    if (enmBranch == IEMBRANCH_JUMP)
+    {
+        /** @todo: This is very similar to regular far jumps; merge! */
+        /* Jumps are fairly simple... */
+
+        /* Chop the high bits off if 16-bit gate (Intel says so). */
+        if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE)
+            uNewRip = (uint16_t)uNewRip;
+
+        /* Limit check for non-long segments. */
+        cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
+        if (DescCS.Legacy.Gen.u1Long)
+            u64Base = 0;
+        else
+        {
+            if (uNewRip > cbLimit)
+            {
+                Log(("BranchCallGate jump %04x:%08RX64 -> out of bounds (%#x) -> #GP(0)\n", uNewCS, uNewRip, cbLimit));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0);
+            }
+            u64Base = X86DESC_BASE(&DescCS.Legacy);
+        }
+
+        /* Canonical address check. */
+        if (!IEM_IS_CANONICAL(uNewRip))
+        {
+            Log(("BranchCallGate jump %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip));
+            return iemRaiseNotCanonical(pVCpu);
+        }
+
+        /*
+         * Ok, everything checked out fine.  Now set the accessed bit before
+         * committing the result into CS, CSHID and RIP.
+         */
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            /** @todo check what VT-x and AMD-V does. */
+            DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /* commit */
+        pVCpu->cpum.GstCtx.rip         = uNewRip;
+        pVCpu->cpum.GstCtx.cs.Sel      = uNewCS & X86_SEL_MASK_OFF_RPL;
+        pVCpu->cpum.GstCtx.cs.Sel     |= pVCpu->iem.s.uCpl; /** @todo is this right for conforming segs? or in general? */
+        pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
+        pVCpu->cpum.GstCtx.cs.fFlags   = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.Attr.u   = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+        pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
+        pVCpu->cpum.GstCtx.cs.u64Base  = u64Base;
+        pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+    }
+    else
+    {
+        Assert(enmBranch == IEMBRANCH_CALL);
+        /* Calls are much more complicated. */
+
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF) && (DescCS.Legacy.Gen.u2Dpl < pVCpu->iem.s.uCpl))
+        {
+            uint16_t    offNewStack;    /* Offset of new stack in TSS. */
+            uint16_t    cbNewStack;     /* Number of bytes the stack information takes up in TSS. */
+            uint8_t     uNewCSDpl;
+            uint8_t     cbWords;
+            RTSEL       uNewSS;
+            RTSEL       uOldSS;
+            uint64_t    uOldRsp;
+            IEMSELDESC  DescSS;
+            RTPTRUNION  uPtrTSS;
+            RTGCPTR     GCPtrTSS;
+            RTPTRUNION  uPtrParmWds;
+            RTGCPTR     GCPtrParmWds;
+
+            /* More privilege. This is the fun part. */
+            Assert(!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF));    /* Filtered out above. */
+
+            /*
+             * Determine new SS:rSP from the TSS.
+             */
+            Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType);
+
+            /* Figure out where the new stack pointer is stored in the TSS. */
+            uNewCSDpl = DescCS.Legacy.Gen.u2Dpl;
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY)
+                {
+                    offNewStack = RT_UOFFSETOF(X86TSS32, esp0) + uNewCSDpl * 8;
+                    cbNewStack  = RT_SIZEOFMEMB(X86TSS32, esp0) + RT_SIZEOFMEMB(X86TSS32, ss0);
+                }
+                else
+                {
+                    Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY);
+                    offNewStack = RT_UOFFSETOF(X86TSS16, sp0) + uNewCSDpl * 4;
+                    cbNewStack  = RT_SIZEOFMEMB(X86TSS16, sp0) + RT_SIZEOFMEMB(X86TSS16, ss0);
+                }
+            }
+            else
+            {
+                Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY);
+                offNewStack = RT_UOFFSETOF(X86TSS64, rsp0) + uNewCSDpl * RT_SIZEOFMEMB(X86TSS64, rsp0);
+                cbNewStack  = RT_SIZEOFMEMB(X86TSS64, rsp0);
+            }
+
+            /* Check against TSS limit. */
+            if ((uint16_t)(offNewStack + cbNewStack - 1) > pVCpu->cpum.GstCtx.tr.u32Limit)
+            {
+                Log(("BranchCallGate inner stack past TSS limit - %u > %u -> #TS(TSS)\n", offNewStack + cbNewStack - 1, pVCpu->cpum.GstCtx.tr.u32Limit));
+                return iemRaiseTaskSwitchFaultBySelector(pVCpu, pVCpu->cpum.GstCtx.tr.Sel);
+            }
+
+            GCPtrTSS = pVCpu->cpum.GstCtx.tr.u64Base + offNewStack;
+            rcStrict = iemMemMap(pVCpu, &uPtrTSS.pv, cbNewStack, UINT8_MAX, GCPtrTSS, IEM_ACCESS_SYS_R);
+            if (rcStrict != VINF_SUCCESS)
+            {
+                Log(("BranchCallGate: TSS mapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                return rcStrict;
+            }
+
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY)
+                {
+                    uNewRsp = uPtrTSS.pu32[0];
+                    uNewSS  = uPtrTSS.pu16[2];
+                }
+                else
+                {
+                    Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY);
+                    uNewRsp = uPtrTSS.pu16[0];
+                    uNewSS  = uPtrTSS.pu16[1];
+                }
+            }
+            else
+            {
+                Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY);
+                /* SS will be a NULL selector, but that's valid. */
+                uNewRsp = uPtrTSS.pu64[0];
+                uNewSS  = uNewCSDpl;
+            }
+
+            /* Done with the TSS now. */
+            rcStrict = iemMemCommitAndUnmap(pVCpu, uPtrTSS.pv, IEM_ACCESS_SYS_R);
+            if (rcStrict != VINF_SUCCESS)
+            {
+                Log(("BranchCallGate: TSS unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                return rcStrict;
+            }
+
+            /* Only used outside of long mode. */
+            cbWords = pDesc->Legacy.Gate.u5ParmCount;
+
+            /* If EFER.LMA is 0, there's extra work to do. */
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if ((uNewSS & X86_SEL_MASK_OFF_RPL) == 0)
+                {
+                    Log(("BranchCallGate new SS NULL -> #TS(NewSS)\n"));
+                    return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS);
+                }
+
+                /* Grab the new SS descriptor. */
+                rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_SS);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+
+                /* Ensure that CS.DPL == SS.RPL == SS.DPL. */
+                if (   (DescCS.Legacy.Gen.u2Dpl != (uNewSS & X86_SEL_RPL))
+                    || (DescCS.Legacy.Gen.u2Dpl != DescSS.Legacy.Gen.u2Dpl))
+                {
+                    Log(("BranchCallGate call bad RPL/DPL uNewSS=%04x SS DPL=%d CS DPL=%u -> #TS(NewSS)\n",
+                         uNewSS, DescCS.Legacy.Gen.u2Dpl, DescCS.Legacy.Gen.u2Dpl));
+                    return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS);
+                }
+
+                /* Ensure new SS is a writable data segment. */
+                if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE)
+                {
+                    Log(("BranchCallGate call new SS -> not a writable data selector (u4Type=%#x)\n", DescSS.Legacy.Gen.u4Type));
+                    return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS);
+                }
+
+                if (!DescSS.Legacy.Gen.u1Present)
+                {
+                    Log(("BranchCallGate New stack not present uSel=%04x -> #SS(NewSS)\n", uNewSS));
+                    return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSS);
+                }
+                if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE)
+                    cbNewStack = (uint16_t)sizeof(uint32_t) * (4 + cbWords);
+                else
+                    cbNewStack = (uint16_t)sizeof(uint16_t) * (4 + cbWords);
+            }
+            else
+            {
+                /* Just grab the new (NULL) SS descriptor. */
+                /** @todo testcase: Check whether the zero GDT entry is actually loaded here
+                 *        like we do... */
+                rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_SS);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+
+                cbNewStack = sizeof(uint64_t) * 4;
+            }
+
+            /** @todo: According to Intel, new stack is checked for enough space first,
+             *         then switched. According to AMD, the stack is switched first and
+             *         then pushes might fault!
+             *         NB: OS/2 Warp 3/4 actively relies on the fact that possible
+             *         incoming stack #PF happens before actual stack switch. AMD is
+             *         either lying or implicitly assumes that new state is committed
+             *         only if and when an instruction doesn't fault.
+             */
+
+            /** @todo: According to AMD, CS is loaded first, then SS.
+             *         According to Intel, it's the other way around!?
+             */
+
+            /** @todo: Intel and AMD disagree on when exactly the CPL changes! */
+
+            /* Set the accessed bit before committing new SS. */
+            if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+            {
+                rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+                DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+            }
+
+            /* Remember the old SS:rSP and their linear address. */
+            uOldSS  = pVCpu->cpum.GstCtx.ss.Sel;
+            uOldRsp = pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig ? pVCpu->cpum.GstCtx.rsp : pVCpu->cpum.GstCtx.sp;
+
+            GCPtrParmWds = pVCpu->cpum.GstCtx.ss.u64Base + uOldRsp;
+
+            /* HACK ALERT! Probe if the write to the new stack will succeed. May #SS(NewSS)
+                           or #PF, the former is not implemented in this workaround. */
+            /** @todo Proper fix callgate target stack exceptions. */
+            /** @todo testcase: Cover callgates with partially or fully inaccessible
+             *        target stacks. */
+            void    *pvNewFrame;
+            RTGCPTR  GCPtrNewStack = X86DESC_BASE(&DescSS.Legacy) + uNewRsp - cbNewStack;
+            rcStrict = iemMemMap(pVCpu, &pvNewFrame, cbNewStack, UINT8_MAX, GCPtrNewStack, IEM_ACCESS_SYS_RW);
+            if (rcStrict != VINF_SUCCESS)
+            {
+                Log(("BranchCallGate: Incoming stack (%04x:%08RX64) not accessible, rc=%Rrc\n", uNewSS, uNewRsp, VBOXSTRICTRC_VAL(rcStrict)));
+                return rcStrict;
+            }
+            rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewFrame, IEM_ACCESS_SYS_RW);
+            if (rcStrict != VINF_SUCCESS)
+            {
+                Log(("BranchCallGate: New stack probe unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                return rcStrict;
+            }
+
+            /* Commit new SS:rSP. */
+            pVCpu->cpum.GstCtx.ss.Sel      = uNewSS;
+            pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
+            pVCpu->cpum.GstCtx.ss.Attr.u   = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
+            pVCpu->cpum.GstCtx.ss.u32Limit = X86DESC_LIMIT_G(&DescSS.Legacy);
+            pVCpu->cpum.GstCtx.ss.u64Base  = X86DESC_BASE(&DescSS.Legacy);
+            pVCpu->cpum.GstCtx.ss.fFlags   = CPUMSELREG_FLAGS_VALID;
+            pVCpu->cpum.GstCtx.rsp         = uNewRsp;
+            pVCpu->iem.s.uCpl = uNewCSDpl;
+            Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+            CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
+
+            /* At this point the stack access must not fail because new state was already committed. */
+            /** @todo this can still fail due to SS.LIMIT not check.   */
+            rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbNewStack,
+                                                   &uPtrRet.pv, &uNewRsp);
+            AssertMsgReturn(rcStrict == VINF_SUCCESS, ("BranchCallGate: New stack mapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)),
+                            VERR_INTERNAL_ERROR_5);
+
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE)
+                {
+                    /* Push the old CS:rIP. */
+                    uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
+                    uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when pushing CS? */
+
+                    if (cbWords)
+                    {
+                        /* Map the relevant chunk of the old stack. */
+                        rcStrict = iemMemMap(pVCpu, &uPtrParmWds.pv, cbWords * 4, UINT8_MAX, GCPtrParmWds, IEM_ACCESS_DATA_R);
+                        if (rcStrict != VINF_SUCCESS)
+                        {
+                            Log(("BranchCallGate: Old stack mapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                            return rcStrict;
+                        }
+
+                        /* Copy the parameter (d)words. */
+                        for (int i = 0; i < cbWords; ++i)
+                            uPtrRet.pu32[2 + i] = uPtrParmWds.pu32[i];
+
+                        /* Unmap the old stack. */
+                        rcStrict = iemMemCommitAndUnmap(pVCpu, uPtrParmWds.pv, IEM_ACCESS_DATA_R);
+                        if (rcStrict != VINF_SUCCESS)
+                        {
+                            Log(("BranchCallGate: Old stack unmapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                            return rcStrict;
+                        }
+                    }
+
+                    /* Push the old SS:rSP. */
+                    uPtrRet.pu32[2 + cbWords + 0] = uOldRsp;
+                    uPtrRet.pu32[2 + cbWords + 1] = uOldSS;
+                }
+                else
+                {
+                    Assert(pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE);
+
+                    /* Push the old CS:rIP. */
+                    uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
+                    uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
+
+                    if (cbWords)
+                    {
+                        /* Map the relevant chunk of the old stack. */
+                        rcStrict = iemMemMap(pVCpu, &uPtrParmWds.pv, cbWords * 2, UINT8_MAX, GCPtrParmWds, IEM_ACCESS_DATA_R);
+                        if (rcStrict != VINF_SUCCESS)
+                        {
+                            Log(("BranchCallGate: Old stack mapping (16-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                            return rcStrict;
+                        }
+
+                        /* Copy the parameter words. */
+                        for (int i = 0; i < cbWords; ++i)
+                            uPtrRet.pu16[2 + i] = uPtrParmWds.pu16[i];
+
+                        /* Unmap the old stack. */
+                        rcStrict = iemMemCommitAndUnmap(pVCpu, uPtrParmWds.pv, IEM_ACCESS_DATA_R);
+                        if (rcStrict != VINF_SUCCESS)
+                        {
+                            Log(("BranchCallGate: Old stack unmapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                            return rcStrict;
+                        }
+                    }
+
+                    /* Push the old SS:rSP. */
+                    uPtrRet.pu16[2 + cbWords + 0] = uOldRsp;
+                    uPtrRet.pu16[2 + cbWords + 1] = uOldSS;
+                }
+            }
+            else
+            {
+                Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE);
+
+                /* For 64-bit gates, no parameters are copied. Just push old SS:rSP and CS:rIP. */
+                uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr;
+                uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when pushing CS? */
+                uPtrRet.pu64[2] = uOldRsp;
+                uPtrRet.pu64[3] = uOldSS;       /** @todo Testcase: What is written to the high words when pushing SS? */
+            }
+
+            rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp);
+            if (rcStrict != VINF_SUCCESS)
+            {
+                Log(("BranchCallGate: New stack unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                return rcStrict;
+            }
+
+            /* Chop the high bits off if 16-bit gate (Intel says so). */
+            if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE)
+                uNewRip = (uint16_t)uNewRip;
+
+            /* Limit / canonical check. */
+            cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if (uNewRip > cbLimit)
+                {
+                    Log(("BranchCallGate %04x:%08RX64 -> out of bounds (%#x)\n", uNewCS, uNewRip, cbLimit));
+                    return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0);
+                }
+                u64Base = X86DESC_BASE(&DescCS.Legacy);
+            }
+            else
+            {
+                Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE);
+                if (!IEM_IS_CANONICAL(uNewRip))
+                {
+                    Log(("BranchCallGate call %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip));
+                    return iemRaiseNotCanonical(pVCpu);
+                }
+                u64Base = 0;
+            }
+
+            /*
+             * Now set the accessed bit before
+             * writing the return address to the stack and committing the result into
+             * CS, CSHID and RIP.
+             */
+            /** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
+            if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+            {
+                rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+                /** @todo check what VT-x and AMD-V does. */
+                DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+            }
+
+            /* Commit new CS:rIP. */
+            pVCpu->cpum.GstCtx.rip         = uNewRip;
+            pVCpu->cpum.GstCtx.cs.Sel      = uNewCS & X86_SEL_MASK_OFF_RPL;
+            pVCpu->cpum.GstCtx.cs.Sel     |= pVCpu->iem.s.uCpl;
+            pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
+            pVCpu->cpum.GstCtx.cs.fFlags   = CPUMSELREG_FLAGS_VALID;
+            pVCpu->cpum.GstCtx.cs.Attr.u   = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+            pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
+            pVCpu->cpum.GstCtx.cs.u64Base  = u64Base;
+            pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+        }
+        else
+        {
+            /* Same privilege. */
+            /** @todo: This is very similar to regular far calls; merge! */
+
+            /* Check stack first - may #SS(0). */
+            /** @todo check how gate size affects pushing of CS! Does callf 16:32 in
+             *        16-bit code cause a two or four byte CS to be pushed? */
+            rcStrict = iemMemStackPushBeginSpecial(pVCpu,
+                                                   IEM_IS_LONG_MODE(pVCpu) ? 8+8
+                                                   : pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 4+4 : 2+2,
+                                                   &uPtrRet.pv, &uNewRsp);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /* Chop the high bits off if 16-bit gate (Intel says so). */
+            if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE)
+                uNewRip = (uint16_t)uNewRip;
+
+            /* Limit / canonical check. */
+            cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if (uNewRip > cbLimit)
+                {
+                    Log(("BranchCallGate %04x:%08RX64 -> out of bounds (%#x)\n", uNewCS, uNewRip, cbLimit));
+                    return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0);
+                }
+                u64Base = X86DESC_BASE(&DescCS.Legacy);
+            }
+            else
+            {
+                if (!IEM_IS_CANONICAL(uNewRip))
+                {
+                    Log(("BranchCallGate call %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip));
+                    return iemRaiseNotCanonical(pVCpu);
+                }
+                u64Base = 0;
+            }
+
+            /*
+             * Now set the accessed bit before
+             * writing the return address to the stack and committing the result into
+             * CS, CSHID and RIP.
+             */
+            /** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
+            if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+            {
+                rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+                /** @todo check what VT-x and AMD-V does. */
+                DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+            }
+
+            /* stack */
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE)
+                {
+                    uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
+                    uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when pushing CS? */
+                }
+                else
+                {
+                    Assert(pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE);
+                    uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
+                    uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
+                }
+            }
+            else
+            {
+                Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE);
+                uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr;
+                uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when pushing CS? */
+            }
+
+            rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /* commit */
+            pVCpu->cpum.GstCtx.rip         = uNewRip;
+            pVCpu->cpum.GstCtx.cs.Sel      = uNewCS & X86_SEL_MASK_OFF_RPL;
+            pVCpu->cpum.GstCtx.cs.Sel     |= pVCpu->iem.s.uCpl;
+            pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
+            pVCpu->cpum.GstCtx.cs.fFlags   = CPUMSELREG_FLAGS_VALID;
+            pVCpu->cpum.GstCtx.cs.Attr.u   = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+            pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
+            pVCpu->cpum.GstCtx.cs.u64Base  = u64Base;
+            pVCpu->iem.s.enmCpuMode  = iemCalcCpuMode(pVCpu);
+        }
+    }
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+    /* Flush the prefetch buffer. */
+# ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+# else
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+# endif
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/**
+ * Implements far jumps and calls thru system selectors.
+ *
+ * @param   uSel            The selector.
+ * @param   enmBranch       The kind of branching we're performing.
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   pDesc           The descriptor corresponding to @a uSel.
+ */
+IEM_CIMPL_DEF_4(iemCImpl_BranchSysSel, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc)
+{
+    Assert(enmBranch == IEMBRANCH_JUMP || enmBranch == IEMBRANCH_CALL);
+    Assert((uSel & X86_SEL_MASK_OFF_RPL));
+    IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    if (IEM_IS_LONG_MODE(pVCpu))
+        switch (pDesc->Legacy.Gen.u4Type)
+        {
+            case AMD64_SEL_TYPE_SYS_CALL_GATE:
+                return IEM_CIMPL_CALL_4(iemCImpl_BranchCallGate, uSel, enmBranch, enmEffOpSize, pDesc);
+
+            default:
+            case AMD64_SEL_TYPE_SYS_LDT:
+            case AMD64_SEL_TYPE_SYS_TSS_BUSY:
+            case AMD64_SEL_TYPE_SYS_TSS_AVAIL:
+            case AMD64_SEL_TYPE_SYS_TRAP_GATE:
+            case AMD64_SEL_TYPE_SYS_INT_GATE:
+                Log(("branch %04x -> wrong sys selector (64-bit): %d\n", uSel, pDesc->Legacy.Gen.u4Type));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+
+    switch (pDesc->Legacy.Gen.u4Type)
+    {
+        case X86_SEL_TYPE_SYS_286_CALL_GATE:
+        case X86_SEL_TYPE_SYS_386_CALL_GATE:
+            return IEM_CIMPL_CALL_4(iemCImpl_BranchCallGate, uSel, enmBranch, enmEffOpSize, pDesc);
+
+        case X86_SEL_TYPE_SYS_TASK_GATE:
+            return IEM_CIMPL_CALL_4(iemCImpl_BranchTaskGate, uSel, enmBranch, enmEffOpSize, pDesc);
+
+        case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
+        case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
+            return IEM_CIMPL_CALL_4(iemCImpl_BranchTaskSegment, uSel, enmBranch, enmEffOpSize, pDesc);
+
+        case X86_SEL_TYPE_SYS_286_TSS_BUSY:
+            Log(("branch %04x -> busy 286 TSS\n", uSel));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+
+        case X86_SEL_TYPE_SYS_386_TSS_BUSY:
+            Log(("branch %04x -> busy 386 TSS\n", uSel));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+
+        default:
+        case X86_SEL_TYPE_SYS_LDT:
+        case X86_SEL_TYPE_SYS_286_INT_GATE:
+        case X86_SEL_TYPE_SYS_286_TRAP_GATE:
+        case X86_SEL_TYPE_SYS_386_INT_GATE:
+        case X86_SEL_TYPE_SYS_386_TRAP_GATE:
+            Log(("branch %04x -> wrong sys selector: %d\n", uSel, pDesc->Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+}
+
+
+/**
+ * Implements far jumps.
+ *
+ * @param   uSel            The selector.
+ * @param   offSeg          The segment offset.
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_FarJmp, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize)
+{
+    NOREF(cbInstr);
+    Assert(offSeg <= UINT32_MAX);
+
+    /*
+     * Real mode and V8086 mode are easy.  The only snag seems to be that
+     * CS.limit doesn't change and the limit check is done against the current
+     * limit.
+     */
+    /** @todo Robert Collins claims (The Segment Descriptor Cache, DDJ August
+     *        1998) that up to and including the Intel 486, far control
+     *        transfers in real mode set default CS attributes (0x93) and also
+     *        set a 64K segment limit. Starting with the Pentium, the
+     *        attributes and limit are left alone but the access rights are
+     *        ignored. We only implement the Pentium+ behavior.
+     *  */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        Assert(enmEffOpSize == IEMMODE_16BIT || enmEffOpSize == IEMMODE_32BIT);
+        if (offSeg > pVCpu->cpum.GstCtx.cs.u32Limit)
+        {
+            Log(("iemCImpl_FarJmp: 16-bit limit\n"));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        if (enmEffOpSize == IEMMODE_16BIT) /** @todo WRONG, must pass this. */
+            pVCpu->cpum.GstCtx.rip       = offSeg;
+        else
+            pVCpu->cpum.GstCtx.rip       = offSeg & UINT16_MAX;
+        pVCpu->cpum.GstCtx.cs.Sel        = uSel;
+        pVCpu->cpum.GstCtx.cs.ValidSel   = uSel;
+        pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.u64Base    = (uint32_t)uSel << 4;
+        pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Protected mode. Need to parse the specified descriptor...
+     */
+    if (!(uSel & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("jmpf %04x:%08RX64 -> invalid selector, #GP(0)\n", uSel, offSeg));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Fetch the descriptor. */
+    IEMSELDESC Desc;
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Is it there? */
+    if (!Desc.Legacy.Gen.u1Present) /** @todo this is probably checked too early. Testcase! */
+    {
+        Log(("jmpf %04x:%08RX64 -> segment not present\n", uSel, offSeg));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
+    }
+
+    /*
+     * Deal with it according to its type.  We do the standard code selectors
+     * here and dispatch the system selectors to worker functions.
+     */
+    if (!Desc.Legacy.Gen.u1DescType)
+        return IEM_CIMPL_CALL_4(iemCImpl_BranchSysSel, uSel, IEMBRANCH_JUMP, enmEffOpSize, &Desc);
+
+    /* Only code segments. */
+    if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
+    {
+        Log(("jmpf %04x:%08RX64 -> not a code selector (u4Type=%#x).\n", uSel, offSeg, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+
+    /* L vs D. */
+    if (   Desc.Legacy.Gen.u1Long
+        && Desc.Legacy.Gen.u1DefBig
+        && IEM_IS_LONG_MODE(pVCpu))
+    {
+        Log(("jmpf %04x:%08RX64 -> both L and D are set.\n", uSel, offSeg));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+
+    /* DPL/RPL/CPL check, where conforming segments makes a difference. */
+    if (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
+    {
+        if (pVCpu->iem.s.uCpl < Desc.Legacy.Gen.u2Dpl)
+        {
+            Log(("jmpf %04x:%08RX64 -> DPL violation (conforming); DPL=%d CPL=%u\n",
+                 uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+    }
+    else
+    {
+        if (pVCpu->iem.s.uCpl != Desc.Legacy.Gen.u2Dpl)
+        {
+            Log(("jmpf %04x:%08RX64 -> CPL != DPL; DPL=%d CPL=%u\n", uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+        if ((uSel & X86_SEL_RPL) > pVCpu->iem.s.uCpl)
+        {
+            Log(("jmpf %04x:%08RX64 -> RPL > DPL; RPL=%d CPL=%u\n", uSel, offSeg, (uSel & X86_SEL_RPL), pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+    }
+
+    /* Chop the high bits if 16-bit (Intel says so). */
+    if (enmEffOpSize == IEMMODE_16BIT)
+        offSeg &= UINT16_MAX;
+
+    /* Limit check. (Should alternatively check for non-canonical addresses
+       here, but that is ruled out by offSeg being 32-bit, right?) */
+    uint64_t u64Base;
+    uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
+    if (Desc.Legacy.Gen.u1Long)
+        u64Base = 0;
+    else
+    {
+        if (offSeg > cbLimit)
+        {
+            Log(("jmpf %04x:%08RX64 -> out of bounds (%#x)\n", uSel, offSeg, cbLimit));
+            /** @todo: Intel says this is #GP(0)! */
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+        u64Base = X86DESC_BASE(&Desc.Legacy);
+    }
+
+    /*
+     * Ok, everything checked out fine.  Now set the accessed bit before
+     * committing the result into CS, CSHID and RIP.
+     */
+    if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        /** @todo check what VT-x and AMD-V does. */
+        Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+
+    /* commit */
+    pVCpu->cpum.GstCtx.rip = offSeg;
+    pVCpu->cpum.GstCtx.cs.Sel         = uSel & X86_SEL_MASK_OFF_RPL;
+    pVCpu->cpum.GstCtx.cs.Sel        |= pVCpu->iem.s.uCpl; /** @todo is this right for conforming segs? or in general? */
+    pVCpu->cpum.GstCtx.cs.ValidSel    = pVCpu->cpum.GstCtx.cs.Sel;
+    pVCpu->cpum.GstCtx.cs.fFlags      = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.Attr.u      = X86DESC_GET_HID_ATTR(&Desc.Legacy);
+    pVCpu->cpum.GstCtx.cs.u32Limit    = cbLimit;
+    pVCpu->cpum.GstCtx.cs.u64Base     = u64Base;
+    pVCpu->iem.s.enmCpuMode  = iemCalcCpuMode(pVCpu);
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+    /** @todo check if the hidden bits are loaded correctly for 64-bit
+     *        mode.  */
+
+    /* Flush the prefetch buffer. */
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+#else
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements far calls.
+ *
+ * This very similar to iemCImpl_FarJmp.
+ *
+ * @param   uSel            The selector.
+ * @param   offSeg          The segment offset.
+ * @param   enmEffOpSize    The operand size (in case we need it).
+ */
+IEM_CIMPL_DEF_3(iemCImpl_callf, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize)
+{
+    VBOXSTRICTRC    rcStrict;
+    uint64_t        uNewRsp;
+    RTPTRUNION      uPtrRet;
+
+    /*
+     * Real mode and V8086 mode are easy.  The only snag seems to be that
+     * CS.limit doesn't change and the limit check is done against the current
+     * limit.
+     */
+    /** @todo See comment for similar code in iemCImpl_FarJmp */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        Assert(enmEffOpSize == IEMMODE_16BIT || enmEffOpSize == IEMMODE_32BIT);
+
+        /* Check stack first - may #SS(0). */
+        rcStrict = iemMemStackPushBeginSpecial(pVCpu, enmEffOpSize == IEMMODE_32BIT ? 4+4 : 2+2,
+                                               &uPtrRet.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        /* Check the target address range. */
+        if (offSeg > UINT32_MAX)
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+
+        /* Everything is fine, push the return address. */
+        if (enmEffOpSize == IEMMODE_16BIT)
+        {
+            uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
+            uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
+        }
+        else
+        {
+            uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
+            uPtrRet.pu16[2] = pVCpu->cpum.GstCtx.cs.Sel;
+        }
+        rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        /* Branch. */
+        pVCpu->cpum.GstCtx.rip           = offSeg;
+        pVCpu->cpum.GstCtx.cs.Sel        = uSel;
+        pVCpu->cpum.GstCtx.cs.ValidSel   = uSel;
+        pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.u64Base    = (uint32_t)uSel << 4;
+        pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Protected mode. Need to parse the specified descriptor...
+     */
+    if (!(uSel & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("callf %04x:%08RX64 -> invalid selector, #GP(0)\n", uSel, offSeg));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Fetch the descriptor. */
+    IEMSELDESC Desc;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Deal with it according to its type.  We do the standard code selectors
+     * here and dispatch the system selectors to worker functions.
+     */
+    if (!Desc.Legacy.Gen.u1DescType)
+        return IEM_CIMPL_CALL_4(iemCImpl_BranchSysSel, uSel, IEMBRANCH_CALL, enmEffOpSize, &Desc);
+
+    /* Only code segments. */
+    if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
+    {
+        Log(("callf %04x:%08RX64 -> not a code selector (u4Type=%#x).\n", uSel, offSeg, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+
+    /* L vs D. */
+    if (   Desc.Legacy.Gen.u1Long
+        && Desc.Legacy.Gen.u1DefBig
+        && IEM_IS_LONG_MODE(pVCpu))
+    {
+        Log(("callf %04x:%08RX64 -> both L and D are set.\n", uSel, offSeg));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+
+    /* DPL/RPL/CPL check, where conforming segments makes a difference. */
+    if (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
+    {
+        if (pVCpu->iem.s.uCpl < Desc.Legacy.Gen.u2Dpl)
+        {
+            Log(("callf %04x:%08RX64 -> DPL violation (conforming); DPL=%d CPL=%u\n",
+                 uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+    }
+    else
+    {
+        if (pVCpu->iem.s.uCpl != Desc.Legacy.Gen.u2Dpl)
+        {
+            Log(("callf %04x:%08RX64 -> CPL != DPL; DPL=%d CPL=%u\n", uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+        if ((uSel & X86_SEL_RPL) > pVCpu->iem.s.uCpl)
+        {
+            Log(("callf %04x:%08RX64 -> RPL > DPL; RPL=%d CPL=%u\n", uSel, offSeg, (uSel & X86_SEL_RPL), pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+    }
+
+    /* Is it there? */
+    if (!Desc.Legacy.Gen.u1Present)
+    {
+        Log(("callf %04x:%08RX64 -> segment not present\n", uSel, offSeg));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
+    }
+
+    /* Check stack first - may #SS(0). */
+    /** @todo check how operand prefix affects pushing of CS! Does callf 16:32 in
+     *        16-bit code cause a two or four byte CS to be pushed? */
+    rcStrict = iemMemStackPushBeginSpecial(pVCpu,
+                                           enmEffOpSize == IEMMODE_64BIT   ? 8+8
+                                           : enmEffOpSize == IEMMODE_32BIT ? 4+4 : 2+2,
+                                           &uPtrRet.pv, &uNewRsp);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Chop the high bits if 16-bit (Intel says so). */
+    if (enmEffOpSize == IEMMODE_16BIT)
+        offSeg &= UINT16_MAX;
+
+    /* Limit / canonical check. */
+    uint64_t u64Base;
+    uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        if (!IEM_IS_CANONICAL(offSeg))
+        {
+            Log(("callf %04x:%016RX64 - not canonical -> #GP\n", uSel, offSeg));
+            return iemRaiseNotCanonical(pVCpu);
+        }
+        u64Base = 0;
+    }
+    else
+    {
+        if (offSeg > cbLimit)
+        {
+            Log(("callf %04x:%08RX64 -> out of bounds (%#x)\n", uSel, offSeg, cbLimit));
+            /** @todo: Intel says this is #GP(0)! */
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+        u64Base = X86DESC_BASE(&Desc.Legacy);
+    }
+
+    /*
+     * Now set the accessed bit before
+     * writing the return address to the stack and committing the result into
+     * CS, CSHID and RIP.
+     */
+    /** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
+    if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        /** @todo check what VT-x and AMD-V does. */
+        Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+
+    /* stack */
+    if (enmEffOpSize == IEMMODE_16BIT)
+    {
+        uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
+        uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
+    }
+    else if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
+        uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when callf is pushing CS? */
+    }
+    else
+    {
+        uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr;
+        uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when callf is pushing CS? */
+    }
+    rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* commit */
+    pVCpu->cpum.GstCtx.rip = offSeg;
+    pVCpu->cpum.GstCtx.cs.Sel         = uSel & X86_SEL_MASK_OFF_RPL;
+    pVCpu->cpum.GstCtx.cs.Sel        |= pVCpu->iem.s.uCpl;
+    pVCpu->cpum.GstCtx.cs.ValidSel    = pVCpu->cpum.GstCtx.cs.Sel;
+    pVCpu->cpum.GstCtx.cs.fFlags      = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.Attr.u      = X86DESC_GET_HID_ATTR(&Desc.Legacy);
+    pVCpu->cpum.GstCtx.cs.u32Limit    = cbLimit;
+    pVCpu->cpum.GstCtx.cs.u64Base     = u64Base;
+    pVCpu->iem.s.enmCpuMode  = iemCalcCpuMode(pVCpu);
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+    /** @todo check if the hidden bits are loaded correctly for 64-bit
+     *        mode.  */
+
+    /* Flush the prefetch buffer. */
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+#else
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements retf.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   cbPop           The amount of arguments to pop from the stack
+ *                          (bytes).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_retf, IEMMODE, enmEffOpSize, uint16_t, cbPop)
+{
+    VBOXSTRICTRC    rcStrict;
+    RTCPTRUNION     uPtrFrame;
+    uint64_t        uNewRsp;
+    uint64_t        uNewRip;
+    uint16_t        uNewCs;
+    NOREF(cbInstr);
+
+    /*
+     * Read the stack values first.
+     */
+    uint32_t        cbRetPtr = enmEffOpSize == IEMMODE_16BIT ? 2+2
+                             : enmEffOpSize == IEMMODE_32BIT ? 4+4 : 8+8;
+    rcStrict = iemMemStackPopBeginSpecial(pVCpu, cbRetPtr, &uPtrFrame.pv, &uNewRsp);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    if (enmEffOpSize == IEMMODE_16BIT)
+    {
+        uNewRip = uPtrFrame.pu16[0];
+        uNewCs  = uPtrFrame.pu16[1];
+    }
+    else if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        uNewRip = uPtrFrame.pu32[0];
+        uNewCs  = uPtrFrame.pu16[2];
+    }
+    else
+    {
+        uNewRip = uPtrFrame.pu64[0];
+        uNewCs  = uPtrFrame.pu16[4];
+    }
+    rcStrict = iemMemStackPopDoneSpecial(pVCpu, uPtrFrame.pv);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* extremely likely */ }
+    else
+        return rcStrict;
+
+    /*
+     * Real mode and V8086 mode are easy.
+     */
+    /** @todo See comment for similar code in iemCImpl_FarJmp */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);
+        /** @todo check how this is supposed to work if sp=0xfffe. */
+
+        /* Check the limit of the new EIP. */
+        /** @todo Intel pseudo code only does the limit check for 16-bit
+         *        operands, AMD does not make any distinction. What is right? */
+        if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit)
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+
+        /* commit the operation. */
+        pVCpu->cpum.GstCtx.rsp           = uNewRsp;
+        pVCpu->cpum.GstCtx.rip           = uNewRip;
+        pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+        pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+        pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.u64Base    = (uint32_t)uNewCs << 4;
+        pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+        if (cbPop)
+            iemRegAddToRsp(pVCpu, cbPop);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Protected mode is complicated, of course.
+     */
+    if (!(uNewCs & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("retf %04x:%08RX64 -> invalid selector, #GP(0)\n", uNewCs, uNewRip));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
+
+    /* Fetch the descriptor. */
+    IEMSELDESC DescCs;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &DescCs, uNewCs, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Can only return to a code selector. */
+    if (   !DescCs.Legacy.Gen.u1DescType
+        || !(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) )
+    {
+        Log(("retf %04x:%08RX64 -> not a code selector (u1DescType=%u u4Type=%#x).\n",
+             uNewCs, uNewRip, DescCs.Legacy.Gen.u1DescType, DescCs.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    /* L vs D. */
+    if (   DescCs.Legacy.Gen.u1Long /** @todo Testcase: far return to a selector with both L and D set. */
+        && DescCs.Legacy.Gen.u1DefBig
+        && IEM_IS_LONG_MODE(pVCpu))
+    {
+        Log(("retf %04x:%08RX64 -> both L & D set.\n", uNewCs, uNewRip));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    /* DPL/RPL/CPL checks. */
+    if ((uNewCs & X86_SEL_RPL) < pVCpu->iem.s.uCpl)
+    {
+        Log(("retf %04x:%08RX64 -> RPL < CPL(%d).\n", uNewCs, uNewRip, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    if (DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
+    {
+        if ((uNewCs & X86_SEL_RPL) < DescCs.Legacy.Gen.u2Dpl)
+        {
+            Log(("retf %04x:%08RX64 -> DPL violation (conforming); DPL=%u RPL=%u\n",
+                 uNewCs, uNewRip, DescCs.Legacy.Gen.u2Dpl, (uNewCs & X86_SEL_RPL)));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+        }
+    }
+    else
+    {
+        if ((uNewCs & X86_SEL_RPL) != DescCs.Legacy.Gen.u2Dpl)
+        {
+            Log(("retf %04x:%08RX64 -> RPL != DPL; DPL=%u RPL=%u\n",
+                 uNewCs, uNewRip, DescCs.Legacy.Gen.u2Dpl, (uNewCs & X86_SEL_RPL)));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+        }
+    }
+
+    /* Is it there? */
+    if (!DescCs.Legacy.Gen.u1Present)
+    {
+        Log(("retf %04x:%08RX64 -> segment not present\n", uNewCs, uNewRip));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
+    }
+
+    /*
+     * Return to outer privilege? (We'll typically have entered via a call gate.)
+     */
+    if ((uNewCs & X86_SEL_RPL) != pVCpu->iem.s.uCpl)
+    {
+        /* Read the outer stack pointer stored *after* the parameters. */
+        rcStrict = iemMemStackPopContinueSpecial(pVCpu, cbPop + cbRetPtr, &uPtrFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        uPtrFrame.pu8 += cbPop; /* Skip the parameters. */
+
+        uint16_t uNewOuterSs;
+        uint64_t uNewOuterRsp;
+        if (enmEffOpSize == IEMMODE_16BIT)
+        {
+            uNewOuterRsp = uPtrFrame.pu16[0];
+            uNewOuterSs  = uPtrFrame.pu16[1];
+        }
+        else if (enmEffOpSize == IEMMODE_32BIT)
+        {
+            uNewOuterRsp = uPtrFrame.pu32[0];
+            uNewOuterSs  = uPtrFrame.pu16[2];
+        }
+        else
+        {
+            uNewOuterRsp = uPtrFrame.pu64[0];
+            uNewOuterSs  = uPtrFrame.pu16[4];
+        }
+        uPtrFrame.pu8 -= cbPop; /* Put uPtrFrame back the way it was. */
+        rcStrict = iemMemStackPopDoneSpecial(pVCpu, uPtrFrame.pv);
+        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        { /* extremely likely */ }
+        else
+            return rcStrict;
+
+        /* Check for NULL stack selector (invalid in ring-3 and non-long mode)
+           and read the selector. */
+        IEMSELDESC DescSs;
+        if (!(uNewOuterSs & X86_SEL_MASK_OFF_RPL))
+        {
+            if (   !DescCs.Legacy.Gen.u1Long
+                || (uNewOuterSs & X86_SEL_RPL) == 3)
+            {
+                Log(("retf %04x:%08RX64 %04x:%08RX64 -> invalid stack selector, #GP\n",
+                     uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+            /** @todo Testcase: Return far to ring-1 or ring-2 with SS=0. */
+            iemMemFakeStackSelDesc(&DescSs, (uNewOuterSs & X86_SEL_RPL));
+        }
+        else
+        {
+            /* Fetch the descriptor for the new stack segment. */
+            rcStrict = iemMemFetchSelDesc(pVCpu, &DescSs, uNewOuterSs, X86_XCPT_GP);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+
+        /* Check that RPL of stack and code selectors match. */
+        if ((uNewCs & X86_SEL_RPL) != (uNewOuterSs & X86_SEL_RPL))
+        {
+            Log(("retf %04x:%08RX64 %04x:%08RX64 - SS.RPL != CS.RPL -> #GP(SS)\n", uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
+        }
+
+        /* Must be a writable data segment. */
+        if (   !DescSs.Legacy.Gen.u1DescType
+            || (DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
+            || !(DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
+        {
+            Log(("retf %04x:%08RX64 %04x:%08RX64 - SS not a writable data segment (u1DescType=%u u4Type=%#x) -> #GP(SS).\n",
+                 uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp, DescSs.Legacy.Gen.u1DescType, DescSs.Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
+        }
+
+        /* L vs D. (Not mentioned by intel.) */
+        if (   DescSs.Legacy.Gen.u1Long /** @todo Testcase: far return to a stack selector with both L and D set. */
+            && DescSs.Legacy.Gen.u1DefBig
+            && IEM_IS_LONG_MODE(pVCpu))
+        {
+            Log(("retf %04x:%08RX64 %04x:%08RX64 - SS has both L & D set -> #GP(SS).\n",
+                 uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
+        }
+
+        /* DPL/RPL/CPL checks. */
+        if (DescSs.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL))
+        {
+            Log(("retf %04x:%08RX64 %04x:%08RX64 - SS.DPL(%u) != CS.RPL (%u) -> #GP(SS).\n",
+                 uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp, DescSs.Legacy.Gen.u2Dpl, uNewCs & X86_SEL_RPL));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
+        }
+
+        /* Is it there? */
+        if (!DescSs.Legacy.Gen.u1Present)
+        {
+            Log(("retf %04x:%08RX64 %04x:%08RX64 - SS not present -> #NP(SS).\n", uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp));
+            return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
+        }
+
+        /* Calc SS limit.*/
+        uint32_t cbLimitSs = X86DESC_LIMIT_G(&DescSs.Legacy);
+
+        /* Is RIP canonical or within CS.limit? */
+        uint64_t u64Base;
+        uint32_t cbLimitCs = X86DESC_LIMIT_G(&DescCs.Legacy);
+
+        /** @todo Testcase: Is this correct? */
+        if (   DescCs.Legacy.Gen.u1Long
+            && IEM_IS_LONG_MODE(pVCpu) )
+        {
+            if (!IEM_IS_CANONICAL(uNewRip))
+            {
+                Log(("retf %04x:%08RX64 %04x:%08RX64 - not canonical -> #GP.\n", uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp));
+                return iemRaiseNotCanonical(pVCpu);
+            }
+            u64Base = 0;
+        }
+        else
+        {
+            if (uNewRip > cbLimitCs)
+            {
+                Log(("retf %04x:%08RX64 %04x:%08RX64 - out of bounds (%#x)-> #GP(CS).\n",
+                     uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp, cbLimitCs));
+                /** @todo: Intel says this is #GP(0)! */
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+            }
+            u64Base = X86DESC_BASE(&DescCs.Legacy);
+        }
+
+        /*
+         * Now set the accessed bit before
+         * writing the return address to the stack and committing the result into
+         * CS, CSHID and RIP.
+         */
+        /** @todo Testcase: Need to check WHEN exactly the CS accessed bit is set. */
+        if (!(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            /** @todo check what VT-x and AMD-V does. */
+            DescCs.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+        /** @todo Testcase: Need to check WHEN exactly the SS accessed bit is set. */
+        if (!(DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewOuterSs);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            /** @todo check what VT-x and AMD-V does. */
+            DescSs.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /* commit */
+        if (enmEffOpSize == IEMMODE_16BIT)
+            pVCpu->cpum.GstCtx.rip           = uNewRip & UINT16_MAX; /** @todo Testcase: When exactly does this occur? With call it happens prior to the limit check according to Intel... */
+        else
+            pVCpu->cpum.GstCtx.rip           = uNewRip;
+        pVCpu->cpum.GstCtx.cs.Sel            = uNewCs;
+        pVCpu->cpum.GstCtx.cs.ValidSel       = uNewCs;
+        pVCpu->cpum.GstCtx.cs.fFlags         = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.Attr.u         = X86DESC_GET_HID_ATTR(&DescCs.Legacy);
+        pVCpu->cpum.GstCtx.cs.u32Limit       = cbLimitCs;
+        pVCpu->cpum.GstCtx.cs.u64Base        = u64Base;
+        pVCpu->iem.s.enmCpuMode     = iemCalcCpuMode(pVCpu);
+        pVCpu->cpum.GstCtx.ss.Sel            = uNewOuterSs;
+        pVCpu->cpum.GstCtx.ss.ValidSel       = uNewOuterSs;
+        pVCpu->cpum.GstCtx.ss.fFlags         = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ss.Attr.u         = X86DESC_GET_HID_ATTR(&DescSs.Legacy);
+        pVCpu->cpum.GstCtx.ss.u32Limit       = cbLimitSs;
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+            pVCpu->cpum.GstCtx.ss.u64Base    = 0;
+        else
+            pVCpu->cpum.GstCtx.ss.u64Base    = X86DESC_BASE(&DescSs.Legacy);
+        if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+            pVCpu->cpum.GstCtx.sp            = (uint16_t)uNewOuterRsp;
+        else
+            pVCpu->cpum.GstCtx.rsp           = uNewOuterRsp;
+
+        pVCpu->iem.s.uCpl           = (uNewCs & X86_SEL_RPL);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.ds);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.es);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.fs);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.gs);
+
+        /** @todo check if the hidden bits are loaded correctly for 64-bit
+         *        mode. */
+
+        if (cbPop)
+            iemRegAddToRsp(pVCpu, cbPop);
+        pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+        /* Done! */
+    }
+    /*
+     * Return to the same privilege level
+     */
+    else
+    {
+        /* Limit / canonical check. */
+        uint64_t u64Base;
+        uint32_t cbLimitCs = X86DESC_LIMIT_G(&DescCs.Legacy);
+
+        /** @todo Testcase: Is this correct? */
+        if (   DescCs.Legacy.Gen.u1Long
+            && IEM_IS_LONG_MODE(pVCpu) )
+        {
+            if (!IEM_IS_CANONICAL(uNewRip))
+            {
+                Log(("retf %04x:%08RX64 - not canonical -> #GP\n", uNewCs, uNewRip));
+                return iemRaiseNotCanonical(pVCpu);
+            }
+            u64Base = 0;
+        }
+        else
+        {
+            if (uNewRip > cbLimitCs)
+            {
+                Log(("retf %04x:%08RX64 -> out of bounds (%#x)\n", uNewCs, uNewRip, cbLimitCs));
+                /** @todo: Intel says this is #GP(0)! */
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+            }
+            u64Base = X86DESC_BASE(&DescCs.Legacy);
+        }
+
+        /*
+         * Now set the accessed bit before
+         * writing the return address to the stack and committing the result into
+         * CS, CSHID and RIP.
+         */
+        /** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
+        if (!(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            /** @todo check what VT-x and AMD-V does. */
+            DescCs.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /* commit */
+        if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+            pVCpu->cpum.GstCtx.sp        = (uint16_t)uNewRsp;
+        else
+            pVCpu->cpum.GstCtx.rsp       = uNewRsp;
+        if (enmEffOpSize == IEMMODE_16BIT)
+            pVCpu->cpum.GstCtx.rip       = uNewRip & UINT16_MAX; /** @todo Testcase: When exactly does this occur? With call it happens prior to the limit check according to Intel... */
+        else
+            pVCpu->cpum.GstCtx.rip       = uNewRip;
+        pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+        pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+        pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESC_GET_HID_ATTR(&DescCs.Legacy);
+        pVCpu->cpum.GstCtx.cs.u32Limit   = cbLimitCs;
+        pVCpu->cpum.GstCtx.cs.u64Base    = u64Base;
+        /** @todo check if the hidden bits are loaded correctly for 64-bit
+         *        mode.  */
+        pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+        if (cbPop)
+            iemRegAddToRsp(pVCpu, cbPop);
+        pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+    }
+
+    /* Flush the prefetch buffer. */
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+#else
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements retn.
+ *
+ * We're doing this in C because of the \#GP that might be raised if the popped
+ * program counter is out of bounds.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   cbPop           The amount of arguments to pop from the stack
+ *                          (bytes).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_retn, IEMMODE, enmEffOpSize, uint16_t, cbPop)
+{
+    NOREF(cbInstr);
+
+    /* Fetch the RSP from the stack. */
+    VBOXSTRICTRC    rcStrict;
+    RTUINT64U       NewRip;
+    RTUINT64U       NewRsp;
+    NewRsp.u = pVCpu->cpum.GstCtx.rsp;
+
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            NewRip.u = 0;
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRip.Words.w0, &NewRsp);
+            break;
+        case IEMMODE_32BIT:
+            NewRip.u = 0;
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRip.DWords.dw0, &NewRsp);
+            break;
+        case IEMMODE_64BIT:
+            rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRip.u, &NewRsp);
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Check the new RSP before loading it. */
+    /** @todo Should test this as the intel+amd pseudo code doesn't mention half
+     *        of it.  The canonical test is performed here and for call. */
+    if (enmEffOpSize != IEMMODE_64BIT)
+    {
+        if (NewRip.DWords.dw0 > pVCpu->cpum.GstCtx.cs.u32Limit)
+        {
+            Log(("retn newrip=%llx - out of bounds (%x) -> #GP\n", NewRip.u, pVCpu->cpum.GstCtx.cs.u32Limit));
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+        }
+    }
+    else
+    {
+        if (!IEM_IS_CANONICAL(NewRip.u))
+        {
+            Log(("retn newrip=%llx - not canonical -> #GP\n", NewRip.u));
+            return iemRaiseNotCanonical(pVCpu);
+        }
+    }
+
+    /* Apply cbPop */
+    if (cbPop)
+        iemRegAddToRspEx(pVCpu, &NewRsp, cbPop);
+
+    /* Commit it. */
+    pVCpu->cpum.GstCtx.rip = NewRip.u;
+    pVCpu->cpum.GstCtx.rsp = NewRsp.u;
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+
+    /* Flush the prefetch buffer. */
+#ifndef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements enter.
+ *
+ * We're doing this in C because the instruction is insane, even for the
+ * u8NestingLevel=0 case dealing with the stack is tedious.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_enter, IEMMODE, enmEffOpSize, uint16_t, cbFrame, uint8_t, cParameters)
+{
+    /* Push RBP, saving the old value in TmpRbp. */
+    RTUINT64U       NewRsp; NewRsp.u = pVCpu->cpum.GstCtx.rsp;
+    RTUINT64U       TmpRbp; TmpRbp.u = pVCpu->cpum.GstCtx.rbp;
+    RTUINT64U       NewRbp;
+    VBOXSTRICTRC    rcStrict;
+    if (enmEffOpSize == IEMMODE_64BIT)
+    {
+        rcStrict = iemMemStackPushU64Ex(pVCpu, TmpRbp.u, &NewRsp);
+        NewRbp = NewRsp;
+    }
+    else if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        rcStrict = iemMemStackPushU32Ex(pVCpu, TmpRbp.DWords.dw0, &NewRsp);
+        NewRbp = NewRsp;
+    }
+    else
+    {
+        rcStrict = iemMemStackPushU16Ex(pVCpu, TmpRbp.Words.w0, &NewRsp);
+        NewRbp = TmpRbp;
+        NewRbp.Words.w0 = NewRsp.Words.w0;
+    }
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Copy the parameters (aka nesting levels by Intel). */
+    cParameters &= 0x1f;
+    if (cParameters > 0)
+    {
+        switch (enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+                    TmpRbp.DWords.dw0 -= 2;
+                else
+                    TmpRbp.Words.w0   -= 2;
+                do
+                {
+                    uint16_t u16Tmp;
+                    rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Tmp, &TmpRbp);
+                    if (rcStrict != VINF_SUCCESS)
+                        break;
+                    rcStrict = iemMemStackPushU16Ex(pVCpu, u16Tmp, &NewRsp);
+                } while (--cParameters > 0 && rcStrict == VINF_SUCCESS);
+                break;
+
+            case IEMMODE_32BIT:
+                if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+                    TmpRbp.DWords.dw0 -= 4;
+                else
+                    TmpRbp.Words.w0   -= 4;
+                do
+                {
+                    uint32_t u32Tmp;
+                    rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Tmp, &TmpRbp);
+                    if (rcStrict != VINF_SUCCESS)
+                        break;
+                    rcStrict = iemMemStackPushU32Ex(pVCpu, u32Tmp, &NewRsp);
+                } while (--cParameters > 0 && rcStrict == VINF_SUCCESS);
+                break;
+
+            case IEMMODE_64BIT:
+                TmpRbp.u -= 8;
+                do
+                {
+                    uint64_t u64Tmp;
+                    rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Tmp, &TmpRbp);
+                    if (rcStrict != VINF_SUCCESS)
+                        break;
+                    rcStrict = iemMemStackPushU64Ex(pVCpu, u64Tmp, &NewRsp);
+                } while (--cParameters > 0 && rcStrict == VINF_SUCCESS);
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+        if (rcStrict != VINF_SUCCESS)
+            return VINF_SUCCESS;
+
+        /* Push the new RBP */
+        if (enmEffOpSize == IEMMODE_64BIT)
+            rcStrict = iemMemStackPushU64Ex(pVCpu, NewRbp.u, &NewRsp);
+        else if (enmEffOpSize == IEMMODE_32BIT)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, NewRbp.DWords.dw0, &NewRsp);
+        else
+            rcStrict = iemMemStackPushU16Ex(pVCpu, NewRbp.Words.w0, &NewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+    }
+
+    /* Recalc RSP. */
+    iemRegSubFromRspEx(pVCpu, &NewRsp, cbFrame);
+
+    /** @todo Should probe write access at the new RSP according to AMD. */
+
+    /* Commit it. */
+    pVCpu->cpum.GstCtx.rbp = NewRbp.u;
+    pVCpu->cpum.GstCtx.rsp = NewRsp.u;
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Implements leave.
+ *
+ * We're doing this in C because messing with the stack registers is annoying
+ * since they depends on SS attributes.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_leave, IEMMODE, enmEffOpSize)
+{
+    /* Calculate the intermediate RSP from RBP and the stack attributes. */
+    RTUINT64U       NewRsp;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        NewRsp.u = pVCpu->cpum.GstCtx.rbp;
+    else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+        NewRsp.u = pVCpu->cpum.GstCtx.ebp;
+    else
+    {
+        /** @todo Check that LEAVE actually preserve the high EBP bits. */
+        NewRsp.u = pVCpu->cpum.GstCtx.rsp;
+        NewRsp.Words.w0 = pVCpu->cpum.GstCtx.bp;
+    }
+
+    /* Pop RBP according to the operand size. */
+    VBOXSTRICTRC    rcStrict;
+    RTUINT64U       NewRbp;
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            NewRbp.u = pVCpu->cpum.GstCtx.rbp;
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRbp.Words.w0, &NewRsp);
+            break;
+        case IEMMODE_32BIT:
+            NewRbp.u = 0;
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRbp.DWords.dw0, &NewRsp);
+            break;
+        case IEMMODE_64BIT:
+            rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRbp.u, &NewRsp);
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+
+    /* Commit it. */
+    pVCpu->cpum.GstCtx.rbp = NewRbp.u;
+    pVCpu->cpum.GstCtx.rsp = NewRsp.u;
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements int3 and int XX.
+ *
+ * @param   u8Int       The interrupt vector number.
+ * @param   enmInt      The int instruction type.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_int, uint8_t, u8Int, IEMINT, enmInt)
+{
+    Assert(pVCpu->iem.s.cXcptRecursions == 0);
+    return iemRaiseXcptOrInt(pVCpu,
+                             cbInstr,
+                             u8Int,
+                             IEM_XCPT_FLAGS_T_SOFT_INT | enmInt,
+                             0,
+                             0);
+}
+
+
+/**
+ * Implements iret for real mode and V8086 mode.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_iret_real_v8086, IEMMODE, enmEffOpSize)
+{
+    X86EFLAGS Efl;
+    Efl.u = IEMMISC_GET_EFL(pVCpu);
+    NOREF(cbInstr);
+
+    /*
+     * iret throws an exception if VME isn't enabled.
+     */
+    if (   Efl.Bits.u1VM
+        && Efl.Bits.u2IOPL != 3
+        && !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME))
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    /*
+     * Do the stack bits, but don't commit RSP before everything checks
+     * out right.
+     */
+    Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);
+    VBOXSTRICTRC    rcStrict;
+    RTCPTRUNION     uFrame;
+    uint16_t        uNewCs;
+    uint32_t        uNewEip;
+    uint32_t        uNewFlags;
+    uint64_t        uNewRsp;
+    if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 12, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewEip    = uFrame.pu32[0];
+        if (uNewEip > UINT16_MAX)
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+
+        uNewCs     = (uint16_t)uFrame.pu32[1];
+        uNewFlags  = uFrame.pu32[2];
+        uNewFlags &= X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF
+                   | X86_EFL_TF | X86_EFL_IF | X86_EFL_DF | X86_EFL_OF | X86_EFL_IOPL | X86_EFL_NT
+                   | X86_EFL_RF /*| X86_EFL_VM*/ | X86_EFL_AC /*|X86_EFL_VIF*/ /*|X86_EFL_VIP*/
+                   | X86_EFL_ID;
+        if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
+            uNewFlags &= ~(X86_EFL_AC | X86_EFL_ID | X86_EFL_VIF | X86_EFL_VIP);
+        uNewFlags |= Efl.u & (X86_EFL_VM | X86_EFL_VIF | X86_EFL_VIP | X86_EFL_1);
+    }
+    else
+    {
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 6, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewEip    = uFrame.pu16[0];
+        uNewCs     = uFrame.pu16[1];
+        uNewFlags  = uFrame.pu16[2];
+        uNewFlags &= X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF
+                   | X86_EFL_TF | X86_EFL_IF | X86_EFL_DF | X86_EFL_OF | X86_EFL_IOPL | X86_EFL_NT;
+        uNewFlags |= Efl.u & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF);
+        /** @todo The intel pseudo code does not indicate what happens to
+         *        reserved flags. We just ignore them. */
+        /* Ancient CPU adjustments: See iemCImpl_popf. */
+        if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_286)
+            uNewFlags &= ~(X86_EFL_NT | X86_EFL_IOPL);
+    }
+    rcStrict = iemMemStackPopDoneSpecial(pVCpu, uFrame.pv);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* extremely likely */ }
+    else
+        return rcStrict;
+
+    /** @todo Check how this is supposed to work if sp=0xfffe. */
+    Log7(("iemCImpl_iret_real_v8086: uNewCs=%#06x uNewRip=%#010x uNewFlags=%#x uNewRsp=%#18llx\n",
+          uNewCs, uNewEip, uNewFlags, uNewRsp));
+
+    /*
+     * Check the limit of the new EIP.
+     */
+    /** @todo Only the AMD pseudo code check the limit here, what's
+     *        right? */
+    if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
+        return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+
+    /*
+     * V8086 checks and flag adjustments
+     */
+    if (Efl.Bits.u1VM)
+    {
+        if (Efl.Bits.u2IOPL == 3)
+        {
+            /* Preserve IOPL and clear RF. */
+            uNewFlags &=        ~(X86_EFL_IOPL | X86_EFL_RF);
+            uNewFlags |= Efl.u & (X86_EFL_IOPL);
+        }
+        else if (   enmEffOpSize == IEMMODE_16BIT
+                 && (   !(uNewFlags & X86_EFL_IF)
+                     || !Efl.Bits.u1VIP )
+                 && !(uNewFlags & X86_EFL_TF)   )
+        {
+            /* Move IF to VIF, clear RF and preserve IF and IOPL.*/
+            uNewFlags &= ~X86_EFL_VIF;
+            uNewFlags |= (uNewFlags & X86_EFL_IF) << (19 - 9);
+            uNewFlags &=        ~(X86_EFL_IF | X86_EFL_IOPL | X86_EFL_RF);
+            uNewFlags |= Efl.u & (X86_EFL_IF | X86_EFL_IOPL);
+        }
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        Log7(("iemCImpl_iret_real_v8086: u1VM=1: adjusted uNewFlags=%#x\n", uNewFlags));
+    }
+
+    /*
+     * Commit the operation.
+     */
+#ifdef DBGFTRACE_ENABLED
+    RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/rm %04x:%04x -> %04x:%04x %x %04llx",
+                      pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewRsp);
+#endif
+    pVCpu->cpum.GstCtx.rsp           = uNewRsp;
+    pVCpu->cpum.GstCtx.rip           = uNewEip;
+    pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+    pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+    pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.u64Base    = (uint32_t)uNewCs << 4;
+    /** @todo do we load attribs and limit as well? */
+    Assert(uNewFlags & X86_EFL_1);
+    IEMMISC_SET_EFL(pVCpu, uNewFlags);
+
+    /* Flush the prefetch buffer. */
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+#else
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads a segment register when entering V8086 mode.
+ *
+ * @param   pSReg           The segment register.
+ * @param   uSeg            The segment to load.
+ */
+static void iemCImplCommonV8086LoadSeg(PCPUMSELREG pSReg, uint16_t uSeg)
+{
+    pSReg->Sel        = uSeg;
+    pSReg->ValidSel   = uSeg;
+    pSReg->fFlags     = CPUMSELREG_FLAGS_VALID;
+    pSReg->u64Base    = (uint32_t)uSeg << 4;
+    pSReg->u32Limit   = 0xffff;
+    pSReg->Attr.u     = X86_SEL_TYPE_RW_ACC | RT_BIT(4) /*!sys*/ | RT_BIT(7) /*P*/ | (3 /*DPL*/ << 5); /* VT-x wants 0xf3 */
+    /** @todo Testcase: Check if VT-x really needs this and what it does itself when
+     *        IRET'ing to V8086. */
+}
+
+
+/**
+ * Implements iret for protected mode returning to V8086 mode.
+ *
+ * @param   uNewEip         The new EIP.
+ * @param   uNewCs          The new CS.
+ * @param   uNewFlags       The new EFLAGS.
+ * @param   uNewRsp         The RSP after the initial IRET frame.
+ *
+ * @note    This can only be a 32-bit iret du to the X86_EFL_VM position.
+ */
+IEM_CIMPL_DEF_4(iemCImpl_iret_prot_v8086, uint32_t, uNewEip, uint16_t, uNewCs, uint32_t, uNewFlags, uint64_t, uNewRsp)
+{
+    RT_NOREF_PV(cbInstr);
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
+
+    /*
+     * Pop the V8086 specific frame bits off the stack.
+     */
+    VBOXSTRICTRC    rcStrict;
+    RTCPTRUNION     uFrame;
+    rcStrict = iemMemStackPopContinueSpecial(pVCpu, 24, &uFrame.pv, &uNewRsp);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    uint32_t uNewEsp = uFrame.pu32[0];
+    uint16_t uNewSs  = uFrame.pu32[1];
+    uint16_t uNewEs  = uFrame.pu32[2];
+    uint16_t uNewDs  = uFrame.pu32[3];
+    uint16_t uNewFs  = uFrame.pu32[4];
+    uint16_t uNewGs  = uFrame.pu32[5];
+    rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uFrame.pv, IEM_ACCESS_STACK_R); /* don't use iemMemStackPopCommitSpecial here. */
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Commit the operation.
+     */
+    uNewFlags &= X86_EFL_LIVE_MASK;
+    uNewFlags |= X86_EFL_RA1_MASK;
+#ifdef DBGFTRACE_ENABLED
+    RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/p/v %04x:%08x -> %04x:%04x %x %04x:%04x",
+                      pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewSs, uNewEsp);
+#endif
+    Log7(("iemCImpl_iret_prot_v8086: %04x:%08x -> %04x:%04x %x %04x:%04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewSs, uNewEsp));
+
+    IEMMISC_SET_EFL(pVCpu, uNewFlags);
+    iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.cs, uNewCs);
+    iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.ss, uNewSs);
+    iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.es, uNewEs);
+    iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.ds, uNewDs);
+    iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.fs, uNewFs);
+    iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.gs, uNewGs);
+    pVCpu->cpum.GstCtx.rip      = (uint16_t)uNewEip;
+    pVCpu->cpum.GstCtx.rsp      = uNewEsp; /** @todo check this out! */
+    pVCpu->iem.s.uCpl  = 3;
+
+    /* Flush the prefetch buffer. */
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+#else
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements iret for protected mode returning via a nested task.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_iret_prot_NestedTask, IEMMODE, enmEffOpSize)
+{
+    Log7(("iemCImpl_iret_prot_NestedTask:\n"));
+#ifndef IEM_IMPLEMENTS_TASKSWITCH
+    IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
+#else
+    RT_NOREF_PV(enmEffOpSize);
+
+    /*
+     * Read the segment selector in the link-field of the current TSS.
+     */
+    RTSEL        uSelRet;
+    VBOXSTRICTRC rcStrict = iemMemFetchSysU16(pVCpu, &uSelRet, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Fetch the returning task's TSS descriptor from the GDT.
+     */
+    if (uSelRet & X86_SEL_LDT)
+    {
+        Log(("iret_prot_NestedTask TSS not in LDT. uSelRet=%04x -> #TS\n", uSelRet));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet);
+    }
+
+    IEMSELDESC TssDesc;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &TssDesc, uSelRet, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    if (TssDesc.Legacy.Gate.u1DescType)
+    {
+        Log(("iret_prot_NestedTask Invalid TSS type. uSelRet=%04x -> #TS\n", uSelRet));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL);
+    }
+
+    if (   TssDesc.Legacy.Gate.u4Type != X86_SEL_TYPE_SYS_286_TSS_BUSY
+        && TssDesc.Legacy.Gate.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
+    {
+        Log(("iret_prot_NestedTask TSS is not busy. uSelRet=%04x DescType=%#x -> #TS\n", uSelRet, TssDesc.Legacy.Gate.u4Type));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL);
+    }
+
+    if (!TssDesc.Legacy.Gate.u1Present)
+    {
+        Log(("iret_prot_NestedTask TSS is not present. uSelRet=%04x -> #NP\n", uSelRet));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL);
+    }
+
+    uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr;
+    return iemTaskSwitch(pVCpu, IEMTASKSWITCH_IRET, uNextEip, 0 /* fFlags */, 0 /* uErr */,
+                         0 /* uCr2 */, uSelRet, &TssDesc);
+#endif
+}
+
+
+/**
+ * Implements iret for protected mode
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_iret_prot, IEMMODE, enmEffOpSize)
+{
+    NOREF(cbInstr);
+    Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);
+
+    /*
+     * Nested task return.
+     */
+    if (pVCpu->cpum.GstCtx.eflags.Bits.u1NT)
+        return IEM_CIMPL_CALL_1(iemCImpl_iret_prot_NestedTask, enmEffOpSize);
+
+    /*
+     * Normal return.
+     *
+     * Do the stack bits, but don't commit RSP before everything checks
+     * out right.
+     */
+    Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);
+    VBOXSTRICTRC    rcStrict;
+    RTCPTRUNION     uFrame;
+    uint16_t        uNewCs;
+    uint32_t        uNewEip;
+    uint32_t        uNewFlags;
+    uint64_t        uNewRsp;
+    if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 12, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewEip    = uFrame.pu32[0];
+        uNewCs     = (uint16_t)uFrame.pu32[1];
+        uNewFlags  = uFrame.pu32[2];
+    }
+    else
+    {
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 6, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewEip    = uFrame.pu16[0];
+        uNewCs     = uFrame.pu16[1];
+        uNewFlags  = uFrame.pu16[2];
+    }
+    rcStrict = iemMemStackPopDoneSpecial(pVCpu, (void *)uFrame.pv); /* don't use iemMemStackPopCommitSpecial here. */
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* extremely likely */ }
+    else
+        return rcStrict;
+    Log7(("iemCImpl_iret_prot: uNewCs=%#06x uNewEip=%#010x uNewFlags=%#x uNewRsp=%#18llx uCpl=%u\n", uNewCs, uNewEip, uNewFlags, uNewRsp, pVCpu->iem.s.uCpl));
+
+    /*
+     * We're hopefully not returning to V8086 mode...
+     */
+    if (   (uNewFlags & X86_EFL_VM)
+        && pVCpu->iem.s.uCpl == 0)
+    {
+        Assert(enmEffOpSize == IEMMODE_32BIT);
+        return IEM_CIMPL_CALL_4(iemCImpl_iret_prot_v8086, uNewEip, uNewCs, uNewFlags, uNewRsp);
+    }
+
+    /*
+     * Protected mode.
+     */
+    /* Read the CS descriptor. */
+    if (!(uNewCs & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("iret %04x:%08x -> invalid CS selector, #GP(0)\n", uNewCs, uNewEip));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    IEMSELDESC DescCS;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCs, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iret %04x:%08x - rcStrict=%Rrc when fetching CS\n", uNewCs, uNewEip, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /* Must be a code descriptor. */
+    if (!DescCS.Legacy.Gen.u1DescType)
+    {
+        Log(("iret %04x:%08x - CS is system segment (%#x) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+    if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
+    {
+        Log(("iret %04x:%08x - not code segment (%#x) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    /* Privilege checks. */
+    if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF))
+    {
+        if ((uNewCs & X86_SEL_RPL) != DescCS.Legacy.Gen.u2Dpl)
+        {
+            Log(("iret %04x:%08x - RPL != DPL (%d) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u2Dpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+        }
+    }
+    else if ((uNewCs & X86_SEL_RPL) < DescCS.Legacy.Gen.u2Dpl)
+    {
+        Log(("iret %04x:%08x - RPL < DPL (%d) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u2Dpl));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+    if ((uNewCs & X86_SEL_RPL) < pVCpu->iem.s.uCpl)
+    {
+        Log(("iret %04x:%08x - RPL < CPL (%d) -> #GP\n", uNewCs, uNewEip, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    /* Present? */
+    if (!DescCS.Legacy.Gen.u1Present)
+    {
+        Log(("iret %04x:%08x - CS not present -> #NP\n", uNewCs, uNewEip));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
+    }
+
+    uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
+
+    /*
+     * Return to outer level?
+     */
+    if ((uNewCs & X86_SEL_RPL) != pVCpu->iem.s.uCpl)
+    {
+        uint16_t    uNewSS;
+        uint32_t    uNewESP;
+        if (enmEffOpSize == IEMMODE_32BIT)
+        {
+            rcStrict = iemMemStackPopContinueSpecial(pVCpu, 8, &uFrame.pv, &uNewRsp);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+/** @todo We might be popping a 32-bit ESP from the IRET frame, but whether
+ *        16-bit or 32-bit are being loaded into SP depends on the D/B
+ *        bit of the popped SS selector it turns out. */
+            uNewESP = uFrame.pu32[0];
+            uNewSS  = (uint16_t)uFrame.pu32[1];
+        }
+        else
+        {
+            rcStrict = iemMemStackPopContinueSpecial(pVCpu, 4, &uFrame.pv, &uNewRsp);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            uNewESP = uFrame.pu16[0];
+            uNewSS  = uFrame.pu16[1];
+        }
+        rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uFrame.pv, IEM_ACCESS_STACK_R);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        Log7(("iemCImpl_iret_prot: uNewSS=%#06x uNewESP=%#010x\n", uNewSS, uNewESP));
+
+        /* Read the SS descriptor. */
+        if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
+        {
+            Log(("iret %04x:%08x/%04x:%08x -> invalid SS selector, #GP(0)\n", uNewCs, uNewEip, uNewSS, uNewESP));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        IEMSELDESC DescSS;
+        rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_GP); /** @todo Correct exception? */
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iret %04x:%08x/%04x:%08x - %Rrc when fetching SS\n",
+                 uNewCs, uNewEip, uNewSS, uNewESP, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* Privilege checks. */
+        if ((uNewSS & X86_SEL_RPL) != (uNewCs & X86_SEL_RPL))
+        {
+            Log(("iret %04x:%08x/%04x:%08x -> SS.RPL != CS.RPL -> #GP\n", uNewCs, uNewEip, uNewSS, uNewESP));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
+        }
+        if (DescSS.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL))
+        {
+            Log(("iret %04x:%08x/%04x:%08x -> SS.DPL (%d) != CS.RPL -> #GP\n",
+                 uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u2Dpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
+        }
+
+        /* Must be a writeable data segment descriptor. */
+        if (!DescSS.Legacy.Gen.u1DescType)
+        {
+            Log(("iret %04x:%08x/%04x:%08x -> SS is system segment (%#x) -> #GP\n",
+                 uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
+        }
+        if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE)
+        {
+            Log(("iret %04x:%08x/%04x:%08x - not writable data segment (%#x) -> #GP\n",
+                 uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
+        }
+
+        /* Present? */
+        if (!DescSS.Legacy.Gen.u1Present)
+        {
+            Log(("iret %04x:%08x/%04x:%08x -> SS not present -> #SS\n", uNewCs, uNewEip, uNewSS, uNewESP));
+            return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSS);
+        }
+
+        uint32_t cbLimitSs = X86DESC_LIMIT_G(&DescSS.Legacy);
+
+        /* Check EIP. */
+        if (uNewEip > cbLimitCS)
+        {
+            Log(("iret %04x:%08x/%04x:%08x -> EIP is out of bounds (%#x) -> #GP(0)\n",
+                 uNewCs, uNewEip, uNewSS, uNewESP, cbLimitCS));
+            /** @todo: Which is it, #GP(0) or #GP(sel)? */
+            return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
+        }
+
+        /*
+         * Commit the changes, marking CS and SS accessed first since
+         * that may fail.
+         */
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+        if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        uint32_t fEFlagsMask = X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF
+                             | X86_EFL_TF | X86_EFL_DF | X86_EFL_OF | X86_EFL_NT;
+        if (enmEffOpSize != IEMMODE_16BIT)
+            fEFlagsMask |= X86_EFL_RF | X86_EFL_AC | X86_EFL_ID;
+        if (pVCpu->iem.s.uCpl == 0)
+            fEFlagsMask |= X86_EFL_IF | X86_EFL_IOPL | X86_EFL_VIF | X86_EFL_VIP; /* VM is 0 */
+        else if (pVCpu->iem.s.uCpl <= pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL)
+            fEFlagsMask |= X86_EFL_IF;
+        if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
+            fEFlagsMask &= ~(X86_EFL_AC | X86_EFL_ID | X86_EFL_VIF | X86_EFL_VIP);
+        uint32_t fEFlagsNew = IEMMISC_GET_EFL(pVCpu);
+        fEFlagsNew         &= ~fEFlagsMask;
+        fEFlagsNew         |= uNewFlags & fEFlagsMask;
+#ifdef DBGFTRACE_ENABLED
+        RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%up%u %04x:%08x -> %04x:%04x %x %04x:%04x",
+                          pVCpu->iem.s.uCpl, uNewCs & X86_SEL_RPL, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
+                          uNewCs, uNewEip, uNewFlags,  uNewSS, uNewESP);
+#endif
+
+        IEMMISC_SET_EFL(pVCpu, fEFlagsNew);
+        pVCpu->cpum.GstCtx.rip           = uNewEip;
+        pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+        pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+        pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+        pVCpu->cpum.GstCtx.cs.u32Limit   = cbLimitCS;
+        pVCpu->cpum.GstCtx.cs.u64Base    = X86DESC_BASE(&DescCS.Legacy);
+        pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+
+        pVCpu->cpum.GstCtx.ss.Sel        = uNewSS;
+        pVCpu->cpum.GstCtx.ss.ValidSel   = uNewSS;
+        pVCpu->cpum.GstCtx.ss.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ss.Attr.u     = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
+        pVCpu->cpum.GstCtx.ss.u32Limit   = cbLimitSs;
+        pVCpu->cpum.GstCtx.ss.u64Base    = X86DESC_BASE(&DescSS.Legacy);
+        if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+            pVCpu->cpum.GstCtx.sp        = (uint16_t)uNewESP;
+        else
+            pVCpu->cpum.GstCtx.rsp       = uNewESP;
+
+        pVCpu->iem.s.uCpl       = uNewCs & X86_SEL_RPL;
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.ds);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.es);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.fs);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.gs);
+
+        /* Done! */
+
+    }
+    /*
+     * Return to the same level.
+     */
+    else
+    {
+        /* Check EIP. */
+        if (uNewEip > cbLimitCS)
+        {
+            Log(("iret %04x:%08x - EIP is out of bounds (%#x) -> #GP(0)\n", uNewCs, uNewEip, cbLimitCS));
+            /** @todo: Which is it, #GP(0) or #GP(sel)? */
+            return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
+        }
+
+        /*
+         * Commit the changes, marking CS first since it may fail.
+         */
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        X86EFLAGS NewEfl;
+        NewEfl.u = IEMMISC_GET_EFL(pVCpu);
+        uint32_t fEFlagsMask = X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF
+                             | X86_EFL_TF | X86_EFL_DF | X86_EFL_OF | X86_EFL_NT;
+        if (enmEffOpSize != IEMMODE_16BIT)
+            fEFlagsMask |= X86_EFL_RF | X86_EFL_AC | X86_EFL_ID;
+        if (pVCpu->iem.s.uCpl == 0)
+            fEFlagsMask |= X86_EFL_IF | X86_EFL_IOPL | X86_EFL_VIF | X86_EFL_VIP; /* VM is 0 */
+        else if (pVCpu->iem.s.uCpl <= NewEfl.Bits.u2IOPL)
+            fEFlagsMask |= X86_EFL_IF;
+        if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
+            fEFlagsMask &= ~(X86_EFL_AC | X86_EFL_ID | X86_EFL_VIF | X86_EFL_VIP);
+        NewEfl.u           &= ~fEFlagsMask;
+        NewEfl.u           |= fEFlagsMask & uNewFlags;
+#ifdef DBGFTRACE_ENABLED
+        RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%up %04x:%08x -> %04x:%04x %x %04x:%04llx",
+                          pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
+                          uNewCs, uNewEip, uNewFlags, pVCpu->cpum.GstCtx.ss.Sel, uNewRsp);
+#endif
+
+        IEMMISC_SET_EFL(pVCpu, NewEfl.u);
+        pVCpu->cpum.GstCtx.rip           = uNewEip;
+        pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+        pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+        pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+        pVCpu->cpum.GstCtx.cs.u32Limit   = cbLimitCS;
+        pVCpu->cpum.GstCtx.cs.u64Base    = X86DESC_BASE(&DescCS.Legacy);
+        pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+        if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+            pVCpu->cpum.GstCtx.sp        = (uint16_t)uNewRsp;
+        else
+            pVCpu->cpum.GstCtx.rsp       = uNewRsp;
+        /* Done! */
+    }
+
+    /* Flush the prefetch buffer. */
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+#else
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements iret for long mode
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_iret_64bit, IEMMODE, enmEffOpSize)
+{
+    NOREF(cbInstr);
+
+    /*
+     * Nested task return is not supported in long mode.
+     */
+    if (pVCpu->cpum.GstCtx.eflags.Bits.u1NT)
+    {
+        Log(("iretq with NT=1 (eflags=%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.eflags.u));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Normal return.
+     *
+     * Do the stack bits, but don't commit RSP before everything checks
+     * out right.
+     */
+    VBOXSTRICTRC    rcStrict;
+    RTCPTRUNION     uFrame;
+    uint64_t        uNewRip;
+    uint16_t        uNewCs;
+    uint16_t        uNewSs;
+    uint32_t        uNewFlags;
+    uint64_t        uNewRsp;
+    if (enmEffOpSize == IEMMODE_64BIT)
+    {
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*8, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewRip    = uFrame.pu64[0];
+        uNewCs     = (uint16_t)uFrame.pu64[1];
+        uNewFlags  = (uint32_t)uFrame.pu64[2];
+        uNewRsp    = uFrame.pu64[3];
+        uNewSs     = (uint16_t)uFrame.pu64[4];
+    }
+    else if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*4, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewRip    = uFrame.pu32[0];
+        uNewCs     = (uint16_t)uFrame.pu32[1];
+        uNewFlags  = uFrame.pu32[2];
+        uNewRsp    = uFrame.pu32[3];
+        uNewSs     = (uint16_t)uFrame.pu32[4];
+    }
+    else
+    {
+        Assert(enmEffOpSize == IEMMODE_16BIT);
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*2, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewRip    = uFrame.pu16[0];
+        uNewCs     = uFrame.pu16[1];
+        uNewFlags  = uFrame.pu16[2];
+        uNewRsp    = uFrame.pu16[3];
+        uNewSs     = uFrame.pu16[4];
+    }
+    rcStrict = iemMemStackPopDoneSpecial(pVCpu, (void *)uFrame.pv); /* don't use iemMemStackPopCommitSpecial here. */
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* extremely like */ }
+    else
+        return rcStrict;
+    Log7(("iretq stack: cs:rip=%04x:%016RX64 rflags=%016RX64 ss:rsp=%04x:%016RX64\n", uNewCs, uNewRip, uNewFlags, uNewSs, uNewRsp));
+
+    /*
+     * Check stuff.
+     */
+    /* Read the CS descriptor. */
+    if (!(uNewCs & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("iret %04x:%016RX64/%04x:%016RX64 -> invalid CS selector, #GP(0)\n", uNewCs, uNewRip, uNewSs, uNewRsp));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    IEMSELDESC DescCS;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCs, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iret %04x:%016RX64/%04x:%016RX64 - rcStrict=%Rrc when fetching CS\n",
+             uNewCs, uNewRip, uNewSs, uNewRsp, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /* Must be a code descriptor. */
+    if (   !DescCS.Legacy.Gen.u1DescType
+        || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
+    {
+        Log(("iret %04x:%016RX64/%04x:%016RX64 - CS is not a code segment T=%u T=%#xu -> #GP\n",
+             uNewCs, uNewRip, uNewSs, uNewRsp, DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    /* Privilege checks. */
+    uint8_t const uNewCpl = uNewCs & X86_SEL_RPL;
+    if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF))
+    {
+        if ((uNewCs & X86_SEL_RPL) != DescCS.Legacy.Gen.u2Dpl)
+        {
+            Log(("iret %04x:%016RX64 - RPL != DPL (%d) -> #GP\n", uNewCs, uNewRip, DescCS.Legacy.Gen.u2Dpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+        }
+    }
+    else if ((uNewCs & X86_SEL_RPL) < DescCS.Legacy.Gen.u2Dpl)
+    {
+        Log(("iret %04x:%016RX64 - RPL < DPL (%d) -> #GP\n", uNewCs, uNewRip, DescCS.Legacy.Gen.u2Dpl));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+    if ((uNewCs & X86_SEL_RPL) < pVCpu->iem.s.uCpl)
+    {
+        Log(("iret %04x:%016RX64 - RPL < CPL (%d) -> #GP\n", uNewCs, uNewRip, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    /* Present? */
+    if (!DescCS.Legacy.Gen.u1Present)
+    {
+        Log(("iret %04x:%016RX64/%04x:%016RX64 - CS not present -> #NP\n", uNewCs, uNewRip, uNewSs, uNewRsp));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
+    }
+
+    uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
+
+    /* Read the SS descriptor. */
+    IEMSELDESC DescSS;
+    if (!(uNewSs & X86_SEL_MASK_OFF_RPL))
+    {
+        if (   !DescCS.Legacy.Gen.u1Long
+            || DescCS.Legacy.Gen.u1DefBig /** @todo exactly how does iret (and others) behave with u1Long=1 and u1DefBig=1? \#GP(sel)? */
+            || uNewCpl > 2) /** @todo verify SS=0 impossible for ring-3. */
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 -> invalid SS selector, #GP(0)\n", uNewCs, uNewRip, uNewSs, uNewRsp));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+        DescSS.Legacy.u = 0;
+    }
+    else
+    {
+        rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSs, X86_XCPT_GP); /** @todo Correct exception? */
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 - %Rrc when fetching SS\n",
+                 uNewCs, uNewRip, uNewSs, uNewRsp, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+
+    /* Privilege checks. */
+    if ((uNewSs & X86_SEL_RPL) != (uNewCs & X86_SEL_RPL))
+    {
+        Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS.RPL != CS.RPL -> #GP\n", uNewCs, uNewRip, uNewSs, uNewRsp));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
+    }
+
+    uint32_t cbLimitSs;
+    if (!(uNewSs & X86_SEL_MASK_OFF_RPL))
+        cbLimitSs = UINT32_MAX;
+    else
+    {
+        if (DescSS.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL))
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS.DPL (%d) != CS.RPL -> #GP\n",
+                 uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u2Dpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
+        }
+
+        /* Must be a writeable data segment descriptor. */
+        if (!DescSS.Legacy.Gen.u1DescType)
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS is system segment (%#x) -> #GP\n",
+                 uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
+        }
+        if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE)
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 - not writable data segment (%#x) -> #GP\n",
+                 uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
+        }
+
+        /* Present? */
+        if (!DescSS.Legacy.Gen.u1Present)
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS not present -> #SS\n", uNewCs, uNewRip, uNewSs, uNewRsp));
+            return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSs);
+        }
+        cbLimitSs = X86DESC_LIMIT_G(&DescSS.Legacy);
+    }
+
+    /* Check EIP. */
+    if (DescCS.Legacy.Gen.u1Long)
+    {
+        if (!IEM_IS_CANONICAL(uNewRip))
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 -> RIP is not canonical -> #GP(0)\n",
+                 uNewCs, uNewRip, uNewSs, uNewRsp));
+            return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
+        }
+    }
+    else
+    {
+        if (uNewRip > cbLimitCS)
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 -> EIP is out of bounds (%#x) -> #GP(0)\n",
+                 uNewCs, uNewRip, uNewSs, uNewRsp, cbLimitCS));
+            /** @todo: Which is it, #GP(0) or #GP(sel)? */
+            return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
+        }
+    }
+
+    /*
+     * Commit the changes, marking CS and SS accessed first since
+     * that may fail.
+     */
+    /** @todo where exactly are these actually marked accessed by a real CPU? */
+    if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+    if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSs);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+
+    uint32_t fEFlagsMask = X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF
+                         | X86_EFL_TF | X86_EFL_DF | X86_EFL_OF | X86_EFL_NT;
+    if (enmEffOpSize != IEMMODE_16BIT)
+        fEFlagsMask |= X86_EFL_RF | X86_EFL_AC | X86_EFL_ID;
+    if (pVCpu->iem.s.uCpl == 0)
+        fEFlagsMask |= X86_EFL_IF | X86_EFL_IOPL | X86_EFL_VIF | X86_EFL_VIP; /* VM is ignored */
+    else if (pVCpu->iem.s.uCpl <= pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL)
+        fEFlagsMask |= X86_EFL_IF;
+    uint32_t fEFlagsNew = IEMMISC_GET_EFL(pVCpu);
+    fEFlagsNew         &= ~fEFlagsMask;
+    fEFlagsNew         |= uNewFlags & fEFlagsMask;
+#ifdef DBGFTRACE_ENABLED
+    RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%ul%u %08llx -> %04x:%04llx %llx %04x:%04llx",
+                      pVCpu->iem.s.uCpl, uNewCpl, pVCpu->cpum.GstCtx.rip, uNewCs, uNewRip, uNewFlags, uNewSs, uNewRsp);
+#endif
+
+    IEMMISC_SET_EFL(pVCpu, fEFlagsNew);
+    pVCpu->cpum.GstCtx.rip           = uNewRip;
+    pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+    pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+    pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+    pVCpu->cpum.GstCtx.cs.u32Limit   = cbLimitCS;
+    pVCpu->cpum.GstCtx.cs.u64Base    = X86DESC_BASE(&DescCS.Legacy);
+    pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+    if (pVCpu->cpum.GstCtx.cs.Attr.n.u1Long || pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
+        pVCpu->cpum.GstCtx.rsp       = uNewRsp;
+    else
+        pVCpu->cpum.GstCtx.sp        = (uint16_t)uNewRsp;
+    pVCpu->cpum.GstCtx.ss.Sel        = uNewSs;
+    pVCpu->cpum.GstCtx.ss.ValidSel   = uNewSs;
+    if (!(uNewSs & X86_SEL_MASK_OFF_RPL))
+    {
+        pVCpu->cpum.GstCtx.ss.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ss.Attr.u     = X86DESCATTR_UNUSABLE | (uNewCpl << X86DESCATTR_DPL_SHIFT);
+        pVCpu->cpum.GstCtx.ss.u32Limit   = UINT32_MAX;
+        pVCpu->cpum.GstCtx.ss.u64Base    = 0;
+        Log2(("iretq new SS: NULL\n"));
+    }
+    else
+    {
+        pVCpu->cpum.GstCtx.ss.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ss.Attr.u     = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
+        pVCpu->cpum.GstCtx.ss.u32Limit   = cbLimitSs;
+        pVCpu->cpum.GstCtx.ss.u64Base    = X86DESC_BASE(&DescSS.Legacy);
+        Log2(("iretq new SS: base=%#RX64 lim=%#x attr=%#x\n", pVCpu->cpum.GstCtx.ss.u64Base, pVCpu->cpum.GstCtx.ss.u32Limit, pVCpu->cpum.GstCtx.ss.Attr.u));
+    }
+
+    if (pVCpu->iem.s.uCpl != uNewCpl)
+    {
+        pVCpu->iem.s.uCpl = uNewCpl;
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.ds);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.es);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.fs);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.gs);
+    }
+
+    /* Flush the prefetch buffer. */
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+#else
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements iret.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_iret, IEMMODE, enmEffOpSize)
+{
+    bool fBlockingNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        /*
+         * Record whether NMI (or virtual-NMI) blocking is in effect during the execution
+         * of this IRET instruction. We need to provide this information as part of some
+         * VM-exits.
+         *
+         * See Intel spec. 27.2.2 "Information for VM Exits Due to Vectored Events".
+         */
+        if (IEM_VMX_IS_PINCTLS_SET(pVCpu, VMX_PIN_CTLS_VIRT_NMI))
+            pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret = pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking;
+        else
+            pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret = fBlockingNmi;
+
+        /*
+         * If "NMI exiting" is set, IRET does not affect blocking of NMIs.
+         * See Intel Spec. 25.3 "Changes To Instruction Behavior In VMX Non-root Operation".
+         */
+        if (IEM_VMX_IS_PINCTLS_SET(pVCpu, VMX_PIN_CTLS_NMI_EXIT))
+            fBlockingNmi = false;
+
+        /* Clear virtual-NMI blocking, if any, before causing any further exceptions. */
+        pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = false;
+    }
+#endif
+
+    /*
+     * The SVM nested-guest intercept for IRET takes priority over all exceptions,
+     * The NMI is still held pending (which I assume means blocking of further NMIs
+     * is in effect).
+     *
+     * See AMD spec. 15.9 "Instruction Intercepts".
+     * See AMD spec. 15.21.9 "NMI Support".
+     */
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IRET))
+    {
+        Log(("iret: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IRET, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Clear NMI blocking, if any, before causing any further exceptions.
+     * See Intel spec. 6.7.1 "Handling Multiple NMIs".
+     */
+    if (fBlockingNmi)
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
+
+    /*
+     * Call a mode specific worker.
+     */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+        return IEM_CIMPL_CALL_1(iemCImpl_iret_real_v8086, enmEffOpSize);
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        return IEM_CIMPL_CALL_1(iemCImpl_iret_64bit, enmEffOpSize);
+    return     IEM_CIMPL_CALL_1(iemCImpl_iret_prot, enmEffOpSize);
+}
+
+
+static void iemLoadallSetSelector(PVMCPUCC pVCpu, uint8_t iSegReg, uint16_t uSel)
+{
+    PCPUMSELREGHID  pHid = iemSRegGetHid(pVCpu, iSegReg);
+
+    pHid->Sel      = uSel;
+    pHid->ValidSel = uSel;
+    pHid->fFlags   = CPUMSELREG_FLAGS_VALID;
+}
+
+
+static void iemLoadall286SetDescCache(PVMCPUCC pVCpu, uint8_t iSegReg, uint8_t const *pbMem)
+{
+    PCPUMSELREGHID  pHid = iemSRegGetHid(pVCpu, iSegReg);
+
+    /* The base is in the first three bytes. */
+    pHid->u64Base  = pbMem[0] + (pbMem[1] << 8) + (pbMem[2] << 16);
+    /* The attributes are in the fourth byte. */
+    pHid->Attr.u   = pbMem[3];
+    /* The limit is in the last two bytes. */
+    pHid->u32Limit = pbMem[4] + (pbMem[5] << 8);
+}
+
+
+/**
+ * Implements 286 LOADALL (286 CPUs only).
+ */
+IEM_CIMPL_DEF_0(iemCImpl_loadall286)
+{
+    NOREF(cbInstr);
+
+    /* Data is loaded from a buffer at 800h. No checks are done on the
+     * validity of loaded state.
+     *
+     * LOADALL only loads the internal CPU state, it does not access any
+     * GDT, LDT, or similar tables.
+     */
+
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("loadall286: CPL must be 0 not %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    uint8_t const *pbMem = NULL;
+    uint16_t const *pa16Mem;
+    uint8_t const *pa8Mem;
+    RTGCPHYS GCPtrStart = 0x800;    /* Fixed table location. */
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&pbMem, 0x66, UINT8_MAX, GCPtrStart, IEM_ACCESS_SYS_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* The MSW is at offset 0x06. */
+    pa16Mem = (uint16_t const *)(pbMem + 0x06);
+    /* Even LOADALL can't clear the MSW.PE bit, though it can set it. */
+    uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0 & ~(X86_CR0_MP | X86_CR0_EM | X86_CR0_TS);
+    uNewCr0 |= *pa16Mem & (X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS);
+    uint64_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
+
+    CPUMSetGuestCR0(pVCpu, uNewCr0);
+    Assert(pVCpu->cpum.GstCtx.cr0 == uNewCr0);
+
+    /* Inform PGM if mode changed. */
+    if ((uNewCr0 & X86_CR0_PE) != (uOldCr0 & X86_CR0_PE))
+    {
+        int rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */);
+        AssertRCReturn(rc, rc);
+        /* ignore informational status codes */
+    }
+    rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER);
+
+    /* TR selector is at offset 0x16. */
+    pa16Mem = (uint16_t const *)(pbMem + 0x16);
+    pVCpu->cpum.GstCtx.tr.Sel      = pa16Mem[0];
+    pVCpu->cpum.GstCtx.tr.ValidSel = pa16Mem[0];
+    pVCpu->cpum.GstCtx.tr.fFlags   = CPUMSELREG_FLAGS_VALID;
+
+    /* Followed by FLAGS... */
+    pVCpu->cpum.GstCtx.eflags.u = pa16Mem[1] | X86_EFL_1;
+    pVCpu->cpum.GstCtx.ip       = pa16Mem[2];   /* ...and IP. */
+
+    /* LDT is at offset 0x1C. */
+    pa16Mem = (uint16_t const *)(pbMem + 0x1C);
+    pVCpu->cpum.GstCtx.ldtr.Sel      = pa16Mem[0];
+    pVCpu->cpum.GstCtx.ldtr.ValidSel = pa16Mem[0];
+    pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+
+    /* Segment registers are at offset 0x1E. */
+    pa16Mem = (uint16_t const *)(pbMem + 0x1E);
+    iemLoadallSetSelector(pVCpu, X86_SREG_DS, pa16Mem[0]);
+    iemLoadallSetSelector(pVCpu, X86_SREG_SS, pa16Mem[1]);
+    iemLoadallSetSelector(pVCpu, X86_SREG_CS, pa16Mem[2]);
+    iemLoadallSetSelector(pVCpu, X86_SREG_ES, pa16Mem[3]);
+
+    /* GPRs are at offset 0x26. */
+    pa16Mem = (uint16_t const *)(pbMem + 0x26);
+    pVCpu->cpum.GstCtx.di = pa16Mem[0];
+    pVCpu->cpum.GstCtx.si = pa16Mem[1];
+    pVCpu->cpum.GstCtx.bp = pa16Mem[2];
+    pVCpu->cpum.GstCtx.sp = pa16Mem[3];
+    pVCpu->cpum.GstCtx.bx = pa16Mem[4];
+    pVCpu->cpum.GstCtx.dx = pa16Mem[5];
+    pVCpu->cpum.GstCtx.cx = pa16Mem[6];
+    pVCpu->cpum.GstCtx.ax = pa16Mem[7];
+
+    /* Descriptor caches are at offset 0x36, 6 bytes per entry. */
+    iemLoadall286SetDescCache(pVCpu, X86_SREG_ES, pbMem + 0x36);
+    iemLoadall286SetDescCache(pVCpu, X86_SREG_CS, pbMem + 0x3C);
+    iemLoadall286SetDescCache(pVCpu, X86_SREG_SS, pbMem + 0x42);
+    iemLoadall286SetDescCache(pVCpu, X86_SREG_DS, pbMem + 0x48);
+
+    /* GDTR contents are at offset 0x4E, 6 bytes. */
+    RTGCPHYS GCPtrBase;
+    uint16_t cbLimit;
+    pa8Mem = pbMem + 0x4E;
+    /* NB: Fourth byte "should be zero"; we are ignoring it. */
+    GCPtrBase = pa8Mem[0] + (pa8Mem[1] << 8) + (pa8Mem[2] << 16);
+    cbLimit = pa8Mem[4] + (pa8Mem[5] << 8);
+    CPUMSetGuestGDTR(pVCpu, GCPtrBase, cbLimit);
+
+    /* IDTR contents are at offset 0x5A, 6 bytes. */
+    pa8Mem = pbMem + 0x5A;
+    GCPtrBase = pa8Mem[0] + (pa8Mem[1] << 8) + (pa8Mem[2] << 16);
+    cbLimit = pa8Mem[4] + (pa8Mem[5] << 8);
+    CPUMSetGuestIDTR(pVCpu, GCPtrBase, cbLimit);
+
+    Log(("LOADALL: GDTR:%08RX64/%04X, IDTR:%08RX64/%04X\n", pVCpu->cpum.GstCtx.gdtr.pGdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, pVCpu->cpum.GstCtx.idtr.pIdt, pVCpu->cpum.GstCtx.idtr.cbIdt));
+    Log(("LOADALL: CS:%04X, CS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.cs.u64Base, pVCpu->cpum.GstCtx.cs.u32Limit, pVCpu->cpum.GstCtx.cs.Attr.u));
+    Log(("LOADALL: DS:%04X, DS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.ds.u64Base, pVCpu->cpum.GstCtx.ds.u32Limit, pVCpu->cpum.GstCtx.ds.Attr.u));
+    Log(("LOADALL: ES:%04X, ES base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.es.Sel, pVCpu->cpum.GstCtx.es.u64Base, pVCpu->cpum.GstCtx.es.u32Limit, pVCpu->cpum.GstCtx.es.Attr.u));
+    Log(("LOADALL: SS:%04X, SS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ss.u64Base, pVCpu->cpum.GstCtx.ss.u32Limit, pVCpu->cpum.GstCtx.ss.Attr.u));
+    Log(("LOADALL: SI:%04X, DI:%04X, AX:%04X, BX:%04X, CX:%04X, DX:%04X\n", pVCpu->cpum.GstCtx.si, pVCpu->cpum.GstCtx.di, pVCpu->cpum.GstCtx.bx, pVCpu->cpum.GstCtx.bx, pVCpu->cpum.GstCtx.cx, pVCpu->cpum.GstCtx.dx));
+
+    rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pbMem, IEM_ACCESS_SYS_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* The CPL may change. It is taken from the "DPL fields of the SS and CS
+     * descriptor caches" but there is no word as to what happens if those are
+     * not identical (probably bad things).
+     */
+    pVCpu->iem.s.uCpl = pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl;
+
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS | CPUM_CHANGED_IDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_TR | CPUM_CHANGED_LDTR);
+
+    /* Flush the prefetch buffer. */
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+#else
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+    return rcStrict;
+}
+
+
+/**
+ * Implements SYSCALL (AMD and Intel64).
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_syscall)
+{
+    /** @todo hack, LOADALL should be decoded as such on a 286. */
+    if (RT_UNLIKELY(pVCpu->iem.s.uTargetCpu == IEMTARGETCPU_286))
+        return iemCImpl_loadall286(pVCpu, cbInstr);
+
+    /*
+     * Check preconditions.
+     *
+     * Note that CPUs described in the documentation may load a few odd values
+     * into CS and SS than we allow here.  This has yet to be checked on real
+     * hardware.
+     */
+    if (!(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SCE))
+    {
+        Log(("syscall: Not enabled in EFER -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
+    {
+        Log(("syscall: Protected mode is required -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    if (IEM_IS_GUEST_CPU_INTEL(pVCpu) && !CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+    {
+        Log(("syscall: Only available in long mode on intel -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS);
+
+    /** @todo verify RPL ignoring and CS=0xfff8 (i.e. SS == 0). */
+    /** @todo what about LDT selectors? Shouldn't matter, really. */
+    uint16_t uNewCs = (pVCpu->cpum.GstCtx.msrSTAR >> MSR_K6_STAR_SYSCALL_CS_SS_SHIFT) & X86_SEL_MASK_OFF_RPL;
+    uint16_t uNewSs = uNewCs + 8;
+    if (uNewCs == 0 || uNewSs == 0)
+    {
+        Log(("syscall: msrSTAR.CS = 0 or SS = 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Long mode and legacy mode differs. */
+    if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+    {
+        uint64_t uNewRip = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.msrLSTAR : pVCpu->cpum.GstCtx. msrCSTAR;
+
+        /* This test isn't in the docs, but I'm not trusting the guys writing
+           the MSRs to have validated the values as canonical like they should. */
+        if (!IEM_IS_CANONICAL(uNewRip))
+        {
+            Log(("syscall: Only available in long mode on intel -> #UD\n"));
+            return iemRaiseUndefinedOpcode(pVCpu);
+        }
+
+        /*
+         * Commit it.
+         */
+        Log(("syscall: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, uNewRip));
+        pVCpu->cpum.GstCtx.rcx           = pVCpu->cpum.GstCtx.rip + cbInstr;
+        pVCpu->cpum.GstCtx.rip           = uNewRip;
+
+        pVCpu->cpum.GstCtx.rflags.u     &= ~X86_EFL_RF;
+        pVCpu->cpum.GstCtx.r11           = pVCpu->cpum.GstCtx.rflags.u;
+        pVCpu->cpum.GstCtx.rflags.u     &= ~pVCpu->cpum.GstCtx.msrSFMASK;
+        pVCpu->cpum.GstCtx.rflags.u     |= X86_EFL_1;
+
+        pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_L | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC;
+        pVCpu->cpum.GstCtx.ss.Attr.u     = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_L | X86DESCATTR_DT | X86_SEL_TYPE_RW_ACC;
+    }
+    else
+    {
+        /*
+         * Commit it.
+         */
+        Log(("syscall: %04x:%08RX32 [efl=%#x] -> %04x:%08RX32\n",
+             pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, uNewCs, (uint32_t)(pVCpu->cpum.GstCtx.msrSTAR & MSR_K6_STAR_SYSCALL_EIP_MASK)));
+        pVCpu->cpum.GstCtx.rcx           = pVCpu->cpum.GstCtx.eip + cbInstr;
+        pVCpu->cpum.GstCtx.rip           = pVCpu->cpum.GstCtx.msrSTAR & MSR_K6_STAR_SYSCALL_EIP_MASK;
+        pVCpu->cpum.GstCtx.rflags.u     &= ~(X86_EFL_VM | X86_EFL_IF | X86_EFL_RF);
+
+        pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC;
+        pVCpu->cpum.GstCtx.ss.Attr.u     = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_RW_ACC;
+    }
+    pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+    pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+    pVCpu->cpum.GstCtx.cs.u64Base    = 0;
+    pVCpu->cpum.GstCtx.cs.u32Limit   = UINT32_MAX;
+    pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+
+    pVCpu->cpum.GstCtx.ss.Sel        = uNewSs;
+    pVCpu->cpum.GstCtx.ss.ValidSel   = uNewSs;
+    pVCpu->cpum.GstCtx.ss.u64Base    = 0;
+    pVCpu->cpum.GstCtx.ss.u32Limit   = UINT32_MAX;
+    pVCpu->cpum.GstCtx.ss.fFlags     = CPUMSELREG_FLAGS_VALID;
+
+    /* Flush the prefetch buffer. */
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+#else
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements SYSRET (AMD and Intel64).
+ */
+IEM_CIMPL_DEF_0(iemCImpl_sysret)
+
+{
+    RT_NOREF_PV(cbInstr);
+
+    /*
+     * Check preconditions.
+     *
+     * Note that CPUs described in the documentation may load a few odd values
+     * into CS and SS than we allow here.  This has yet to be checked on real
+     * hardware.
+     */
+    if (!(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SCE))
+    {
+        Log(("sysret: Not enabled in EFER -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (IEM_IS_GUEST_CPU_INTEL(pVCpu) && !CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+    {
+        Log(("sysret: Only available in long mode on intel -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
+    {
+        Log(("sysret: Protected mode is required -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("sysret: CPL must be 0 not %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS);
+
+    /** @todo Does SYSRET verify CS != 0 and SS != 0? Neither is valid in ring-3. */
+    uint16_t uNewCs = (pVCpu->cpum.GstCtx.msrSTAR >> MSR_K6_STAR_SYSRET_CS_SS_SHIFT) & X86_SEL_MASK_OFF_RPL;
+    uint16_t uNewSs = uNewCs + 8;
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+        uNewCs += 16;
+    if (uNewCs == 0 || uNewSs == 0)
+    {
+        Log(("sysret: msrSTAR.CS = 0 or SS = 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Commit it.
+     */
+    if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+    {
+        if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+        {
+            Log(("sysret: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64 [r11=%#llx]\n",
+                 pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, pVCpu->cpum.GstCtx.rcx, pVCpu->cpum.GstCtx.r11));
+            /* Note! We disregard intel manual regarding the RCX cananonical
+                     check, ask intel+xen why AMD doesn't do it. */
+            pVCpu->cpum.GstCtx.rip       = pVCpu->cpum.GstCtx.rcx;
+            pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_L | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC
+                            | (3 << X86DESCATTR_DPL_SHIFT);
+        }
+        else
+        {
+            Log(("sysret: %04x:%016RX64 [efl=%#llx] -> %04x:%08RX32 [r11=%#llx]\n",
+                 pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.r11));
+            pVCpu->cpum.GstCtx.rip       = pVCpu->cpum.GstCtx.ecx;
+            pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC
+                            | (3 << X86DESCATTR_DPL_SHIFT);
+        }
+        /** @todo testcase: See what kind of flags we can make SYSRET restore and
+         *        what it really ignores. RF and VM are hinted at being zero, by AMD. */
+        pVCpu->cpum.GstCtx.rflags.u      = pVCpu->cpum.GstCtx.r11 & (X86_EFL_POPF_BITS | X86_EFL_VIF | X86_EFL_VIP);
+        pVCpu->cpum.GstCtx.rflags.u     |= X86_EFL_1;
+    }
+    else
+    {
+        Log(("sysret: %04x:%08RX32 [efl=%#x] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, uNewCs, pVCpu->cpum.GstCtx.ecx));
+        pVCpu->cpum.GstCtx.rip           = pVCpu->cpum.GstCtx.rcx;
+        pVCpu->cpum.GstCtx.rflags.u     |= X86_EFL_IF;
+        pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC
+                            | (3 << X86DESCATTR_DPL_SHIFT);
+    }
+    pVCpu->cpum.GstCtx.cs.Sel        = uNewCs | 3;
+    pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs | 3;
+    pVCpu->cpum.GstCtx.cs.u64Base    = 0;
+    pVCpu->cpum.GstCtx.cs.u32Limit   = UINT32_MAX;
+    pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+
+    pVCpu->cpum.GstCtx.ss.Sel        = uNewSs | 3;
+    pVCpu->cpum.GstCtx.ss.ValidSel   = uNewSs | 3;
+    pVCpu->cpum.GstCtx.ss.fFlags     = CPUMSELREG_FLAGS_VALID;
+    /* The SS hidden bits remains unchanged says AMD. To that I say "Yeah, right!". */
+    pVCpu->cpum.GstCtx.ss.Attr.u    |= (3 << X86DESCATTR_DPL_SHIFT);
+    /** @todo Testcase: verify that SS.u1Long and SS.u1DefBig are left unchanged
+     *        on sysret. */
+
+    /* Flush the prefetch buffer. */
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+#else
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for 'pop SReg', 'mov SReg, GReg' and 'lXs GReg, reg/mem'.
+ *
+ * @param   iSegReg     The segment register number (valid).
+ * @param   uSel        The new selector value.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_LoadSReg, uint8_t, iSegReg, uint16_t, uSel)
+{
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+    uint16_t       *pSel = iemSRegRef(pVCpu, iSegReg);
+    PCPUMSELREGHID  pHid = iemSRegGetHid(pVCpu, iSegReg);
+
+    Assert(iSegReg <= X86_SREG_GS && iSegReg != X86_SREG_CS);
+
+    /*
+     * Real mode and V8086 mode are easy.
+     */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        *pSel           = uSel;
+        pHid->u64Base   = (uint32_t)uSel << 4;
+        pHid->ValidSel  = uSel;
+        pHid->fFlags    = CPUMSELREG_FLAGS_VALID;
+#if 0 /* AMD Volume 2, chapter 4.1 - "real mode segmentation" - states that limit and attributes are untouched. */
+        /** @todo Does the CPU actually load limits and attributes in the
+         *        real/V8086 mode segment load case?  It doesn't for CS in far
+         *        jumps...  Affects unreal mode.  */
+        pHid->u32Limit          = 0xffff;
+        pHid->Attr.u = 0;
+        pHid->Attr.n.u1Present  = 1;
+        pHid->Attr.n.u1DescType = 1;
+        pHid->Attr.n.u4Type     = iSegReg != X86_SREG_CS
+                                ? X86_SEL_TYPE_RW
+                                : X86_SEL_TYPE_READ | X86_SEL_TYPE_CODE;
+#endif
+        CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Protected mode.
+     *
+     * Check if it's a null segment selector value first, that's OK for DS, ES,
+     * FS and GS.  If not null, then we have to load and parse the descriptor.
+     */
+    if (!(uSel & X86_SEL_MASK_OFF_RPL))
+    {
+        Assert(iSegReg != X86_SREG_CS); /** @todo testcase for \#UD on MOV CS, ax! */
+        if (iSegReg == X86_SREG_SS)
+        {
+            /* In 64-bit kernel mode, the stack can be 0 because of the way
+               interrupts are dispatched. AMD seems to have a slighly more
+               relaxed relationship to SS.RPL than intel does. */
+            /** @todo We cannot 'mov ss, 3' in 64-bit kernel mode, can we? There is a testcase (bs-cpu-xcpt-1), but double check this! */
+            if (   pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
+                || pVCpu->iem.s.uCpl > 2
+                || (   uSel != pVCpu->iem.s.uCpl
+                    && !IEM_IS_GUEST_CPU_AMD(pVCpu)) )
+            {
+                Log(("load sreg %#x -> invalid stack selector, #GP(0)\n", uSel));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+        }
+
+        *pSel = uSel;   /* Not RPL, remember :-) */
+        iemHlpLoadNullDataSelectorProt(pVCpu, pHid, uSel);
+        if (iSegReg == X86_SREG_SS)
+            pHid->Attr.u |= pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT;
+
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pHid));
+        CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
+
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* Fetch the descriptor. */
+    IEMSELDESC Desc;
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP); /** @todo Correct exception? */
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Check GPs first. */
+    if (!Desc.Legacy.Gen.u1DescType)
+    {
+        Log(("load sreg %d (=%#x) - system selector (%#x) -> #GP\n", iSegReg, uSel, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+    if (iSegReg == X86_SREG_SS) /* SS gets different treatment */
+    {
+        if (    (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
+            || !(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
+        {
+            Log(("load sreg SS, %#x - code or read only (%#x) -> #GP\n", uSel, Desc.Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+        if ((uSel & X86_SEL_RPL) != pVCpu->iem.s.uCpl)
+        {
+            Log(("load sreg SS, %#x - RPL and CPL (%d) differs -> #GP\n", uSel, pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+        if (Desc.Legacy.Gen.u2Dpl != pVCpu->iem.s.uCpl)
+        {
+            Log(("load sreg SS, %#x - DPL (%d) and CPL (%d) differs -> #GP\n", uSel, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+    }
+    else
+    {
+        if ((Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
+        {
+            Log(("load sreg%u, %#x - execute only segment -> #GP\n", iSegReg, uSel));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+        if (   (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+            != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+        {
+#if 0 /* this is what intel says. */
+            if (   (uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
+                && pVCpu->iem.s.uCpl        > Desc.Legacy.Gen.u2Dpl)
+            {
+                Log(("load sreg%u, %#x - both RPL (%d) and CPL (%d) are greater than DPL (%d) -> #GP\n",
+                     iSegReg, uSel, (uSel & X86_SEL_RPL), pVCpu->iem.s.uCpl, Desc.Legacy.Gen.u2Dpl));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+            }
+#else /* this is what makes more sense. */
+            if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl)
+            {
+                Log(("load sreg%u, %#x - RPL (%d) is greater than DPL (%d) -> #GP\n",
+                     iSegReg, uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+            }
+            if (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl)
+            {
+                Log(("load sreg%u, %#x - CPL (%d) is greater than DPL (%d) -> #GP\n",
+                     iSegReg, uSel, pVCpu->iem.s.uCpl, Desc.Legacy.Gen.u2Dpl));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+            }
+#endif
+        }
+    }
+
+    /* Is it there? */
+    if (!Desc.Legacy.Gen.u1Present)
+    {
+        Log(("load sreg%d,%#x - segment not present -> #NP\n", iSegReg, uSel));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
+    }
+
+    /* The base and limit. */
+    uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
+    uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
+
+    /*
+     * Ok, everything checked out fine.  Now set the accessed bit before
+     * committing the result into the registers.
+     */
+    if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+
+    /* commit */
+    *pSel = uSel;
+    pHid->Attr.u   = X86DESC_GET_HID_ATTR(&Desc.Legacy);
+    pHid->u32Limit = cbLimit;
+    pHid->u64Base  = u64Base;
+    pHid->ValidSel = uSel;
+    pHid->fFlags   = CPUMSELREG_FLAGS_VALID;
+
+    /** @todo check if the hidden bits are loaded correctly for 64-bit
+     *        mode.  */
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pHid));
+
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'mov SReg, r/m'.
+ *
+ * @param   iSegReg     The segment register number (valid).
+ * @param   uSel        The new selector value.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_load_SReg, uint8_t, iSegReg, uint16_t, uSel)
+{
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_LoadSReg, iSegReg, uSel);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (iSegReg == X86_SREG_SS)
+            EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'pop SReg'.
+ *
+ * @param   iSegReg         The segment register number (valid).
+ * @param   enmEffOpSize    The efficient operand size (valid).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_pop_Sreg, uint8_t, iSegReg, IEMMODE, enmEffOpSize)
+{
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * Read the selector off the stack and join paths with mov ss, reg.
+     */
+    RTUINT64U TmpRsp;
+    TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t uSel;
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &uSel, &TmpRsp);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = IEM_CIMPL_CALL_2(iemCImpl_LoadSReg, iSegReg, uSel);
+            break;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            uint32_t u32Value;
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Value, &TmpRsp);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = IEM_CIMPL_CALL_2(iemCImpl_LoadSReg, iSegReg, (uint16_t)u32Value);
+            break;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            uint64_t u64Value;
+            rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Value, &TmpRsp);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = IEM_CIMPL_CALL_2(iemCImpl_LoadSReg, iSegReg, (uint16_t)u64Value);
+            break;
+        }
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    /*
+     * Commit the stack on success.
+     */
+    if (rcStrict == VINF_SUCCESS)
+    {
+        pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+        if (iSegReg == X86_SREG_SS)
+            EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements lgs, lfs, les, lds & lss.
+ */
+IEM_CIMPL_DEF_5(iemCImpl_load_SReg_Greg,
+                uint16_t, uSel,
+                uint64_t, offSeg,
+                uint8_t,  iSegReg,
+                uint8_t,  iGReg,
+                IEMMODE,  enmEffOpSize)
+{
+    /*
+     * Use iemCImpl_LoadSReg to do the tricky segment register loading.
+     */
+    /** @todo verify and test that mov, pop and lXs works the segment
+     *        register loading in the exact same way. */
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_LoadSReg, iSegReg, uSel);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        switch (enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                *(uint16_t *)iemGRegRef(pVCpu, iGReg) = offSeg;
+                break;
+            case IEMMODE_32BIT:
+                *(uint64_t *)iemGRegRef(pVCpu, iGReg) = offSeg;
+                break;
+            case IEMMODE_64BIT:
+                *(uint64_t *)iemGRegRef(pVCpu, iGReg) = offSeg;
+                break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Helper for VERR, VERW, LAR, and LSL and loads the descriptor into memory.
+ *
+ * @retval VINF_SUCCESS on success.
+ * @retval VINF_IEM_SELECTOR_NOT_OK if the selector isn't ok.
+ * @retval iemMemFetchSysU64 return value.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   uSel                The selector value.
+ * @param   fAllowSysDesc       Whether system descriptors are OK or not.
+ * @param   pDesc               Where to return the descriptor on success.
+ */
+static VBOXSTRICTRC iemCImpl_LoadDescHelper(PVMCPUCC pVCpu, uint16_t uSel, bool fAllowSysDesc, PIEMSELDESC pDesc)
+{
+    pDesc->Long.au64[0] = 0;
+    pDesc->Long.au64[1] = 0;
+
+    if (!(uSel & X86_SEL_MASK_OFF_RPL)) /** @todo test this on 64-bit. */
+        return VINF_IEM_SELECTOR_NOT_OK;
+
+    /* Within the table limits? */
+    RTGCPTR GCPtrBase;
+    if (uSel & X86_SEL_LDT)
+    {
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR);
+        if (   !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
+            || (uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
+            return VINF_IEM_SELECTOR_NOT_OK;
+        GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
+    }
+    else
+    {
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR);
+        if ((uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
+            return VINF_IEM_SELECTOR_NOT_OK;
+        GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
+    }
+
+    /* Fetch the descriptor. */
+    VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    if (!pDesc->Legacy.Gen.u1DescType)
+    {
+        if (!fAllowSysDesc)
+            return VINF_IEM_SELECTOR_NOT_OK;
+        if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+        {
+            rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 8);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements verr (fWrite = false) and verw (fWrite = true).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_VerX, uint16_t, uSel, bool, fWrite)
+{
+    Assert(!IEM_IS_REAL_OR_V86_MODE(pVCpu));
+
+    /** @todo figure whether the accessed bit is set or not. */
+
+    bool         fAccessible = true;
+    IEMSELDESC   Desc;
+    VBOXSTRICTRC rcStrict = iemCImpl_LoadDescHelper(pVCpu, uSel, false /*fAllowSysDesc*/, &Desc);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /* Check the descriptor, order doesn't matter much here. */
+        if (   !Desc.Legacy.Gen.u1DescType
+            || !Desc.Legacy.Gen.u1Present)
+            fAccessible = false;
+        else
+        {
+            if (  fWrite
+                ? (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE
+                : (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ))  == X86_SEL_TYPE_CODE)
+                fAccessible = false;
+
+            /** @todo testcase for the conforming behavior. */
+            if (   (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+                != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+            {
+                if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl)
+                    fAccessible = false;
+                else if (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl)
+                    fAccessible = false;
+            }
+        }
+
+    }
+    else if (rcStrict == VINF_IEM_SELECTOR_NOT_OK)
+        fAccessible = false;
+    else
+        return rcStrict;
+
+    /* commit */
+    pVCpu->cpum.GstCtx.eflags.Bits.u1ZF = fAccessible;
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements LAR and LSL with 64-bit operand size.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pu16Dst         Pointer to the destination register.
+ * @param   uSel            The selector to load details for.
+ * @param   fIsLar          true = LAR, false = LSL.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_LarLsl_u64, uint64_t *, pu64Dst, uint16_t, uSel, bool, fIsLar)
+{
+    Assert(!IEM_IS_REAL_OR_V86_MODE(pVCpu));
+
+    /** @todo figure whether the accessed bit is set or not. */
+
+    bool         fDescOk = true;
+    IEMSELDESC   Desc;
+    VBOXSTRICTRC rcStrict = iemCImpl_LoadDescHelper(pVCpu, uSel, true /*fAllowSysDesc*/, &Desc);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /*
+         * Check the descriptor type.
+         */
+        if (!Desc.Legacy.Gen.u1DescType)
+        {
+            if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+            {
+                if (Desc.Long.Gen.u5Zeros)
+                    fDescOk = false;
+                else
+                    switch (Desc.Long.Gen.u4Type)
+                    {
+                        /** @todo Intel lists 0 as valid for LSL, verify whether that's correct */
+                        case AMD64_SEL_TYPE_SYS_TSS_AVAIL:
+                        case AMD64_SEL_TYPE_SYS_TSS_BUSY:
+                        case AMD64_SEL_TYPE_SYS_LDT: /** @todo Intel lists this as invalid for LAR, AMD and 32-bit does otherwise. */
+                            break;
+                        case AMD64_SEL_TYPE_SYS_CALL_GATE:
+                            fDescOk = fIsLar;
+                            break;
+                        default:
+                            fDescOk = false;
+                            break;
+                    }
+            }
+            else
+            {
+                switch (Desc.Long.Gen.u4Type)
+                {
+                    case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
+                    case X86_SEL_TYPE_SYS_286_TSS_BUSY:
+                    case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
+                    case X86_SEL_TYPE_SYS_386_TSS_BUSY:
+                    case X86_SEL_TYPE_SYS_LDT:
+                        break;
+                    case X86_SEL_TYPE_SYS_286_CALL_GATE:
+                    case X86_SEL_TYPE_SYS_TASK_GATE:
+                    case X86_SEL_TYPE_SYS_386_CALL_GATE:
+                        fDescOk = fIsLar;
+                        break;
+                    default:
+                        fDescOk = false;
+                        break;
+                }
+            }
+        }
+        if (fDescOk)
+        {
+            /*
+             * Check the RPL/DPL/CPL interaction..
+             */
+            /** @todo testcase for the conforming behavior. */
+            if (   (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)
+                || !Desc.Legacy.Gen.u1DescType)
+            {
+                if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl)
+                    fDescOk = false;
+                else if (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl)
+                    fDescOk = false;
+            }
+        }
+
+        if (fDescOk)
+        {
+            /*
+             * All fine, start committing the result.
+             */
+            if (fIsLar)
+                *pu64Dst = Desc.Legacy.au32[1] & UINT32_C(0x00ffff00);
+            else
+                *pu64Dst = X86DESC_LIMIT_G(&Desc.Legacy);
+        }
+
+    }
+    else if (rcStrict == VINF_IEM_SELECTOR_NOT_OK)
+        fDescOk = false;
+    else
+        return rcStrict;
+
+    /* commit flags value and advance rip. */
+    pVCpu->cpum.GstCtx.eflags.Bits.u1ZF = fDescOk;
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements LAR and LSL with 16-bit operand size.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pu16Dst         Pointer to the destination register.
+ * @param   u16Sel          The selector to load details for.
+ * @param   fIsLar          true = LAR, false = LSL.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_LarLsl_u16, uint16_t *, pu16Dst, uint16_t, uSel, bool, fIsLar)
+{
+    uint64_t u64TmpDst = *pu16Dst;
+    IEM_CIMPL_CALL_3(iemCImpl_LarLsl_u64, &u64TmpDst, uSel, fIsLar);
+    *pu16Dst = u64TmpDst;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements lgdt.
+ *
+ * @param   iEffSeg         The segment of the new gdtr contents
+ * @param   GCPtrEffSrc     The address of the new gdtr contents.
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_lgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("lgdt: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_GDTR_IDTR_ACCESS, VMXINSTRID_LGDT, cbInstr);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_GDTR_WRITES))
+    {
+        Log(("lgdt: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_GDTR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Fetch the limit and base address.
+     */
+    uint16_t cbLimit;
+    RTGCPTR  GCPtrBase;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataXdtr(pVCpu, &cbLimit, &GCPtrBase, iEffSeg, GCPtrEffSrc, enmEffOpSize);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (   pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
+            || X86_IS_CANONICAL(GCPtrBase))
+        {
+            rcStrict = CPUMSetGuestGDTR(pVCpu, GCPtrBase, cbLimit);
+            if (rcStrict == VINF_SUCCESS)
+                iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        }
+        else
+        {
+            Log(("iemCImpl_lgdt: Non-canonical base %04x:%RGv\n", cbLimit, GCPtrBase));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements sgdt.
+ *
+ * @param   iEffSeg         The segment where to store the gdtr content.
+ * @param   GCPtrEffDst     The address where to store the gdtr content.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_sgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    /*
+     * Join paths with sidt.
+     * Note! No CPL or V8086 checks here, it's a really sad story, ask Intel if
+     *       you really must know.
+     */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("sgdt: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_GDTR_IDTR_ACCESS, VMXINSTRID_SGDT, cbInstr);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_GDTR_READS))
+    {
+        Log(("sgdt: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_GDTR_READ, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR);
+    VBOXSTRICTRC rcStrict = iemMemStoreDataXdtr(pVCpu, pVCpu->cpum.GstCtx.gdtr.cbGdt, pVCpu->cpum.GstCtx.gdtr.pGdt, iEffSeg, GCPtrEffDst);
+    if (rcStrict == VINF_SUCCESS)
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Implements lidt.
+ *
+ * @param   iEffSeg         The segment of the new idtr contents
+ * @param   GCPtrEffSrc     The address of the new idtr contents.
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_lidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IDTR_WRITES))
+    {
+        Log(("lidt: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IDTR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Fetch the limit and base address.
+     */
+    uint16_t cbLimit;
+    RTGCPTR  GCPtrBase;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataXdtr(pVCpu, &cbLimit, &GCPtrBase, iEffSeg, GCPtrEffSrc, enmEffOpSize);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (   pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
+            || X86_IS_CANONICAL(GCPtrBase))
+        {
+            CPUMSetGuestIDTR(pVCpu, GCPtrBase, cbLimit);
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        }
+        else
+        {
+            Log(("iemCImpl_lidt: Non-canonical base %04x:%RGv\n", cbLimit, GCPtrBase));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements sidt.
+ *
+ * @param   iEffSeg         The segment where to store the idtr content.
+ * @param   GCPtrEffDst     The address where to store the idtr content.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_sidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    /*
+     * Join paths with sgdt.
+     * Note! No CPL or V8086 checks here, it's a really sad story, ask Intel if
+     *       you really must know.
+     */
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IDTR_READS))
+    {
+        Log(("sidt: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IDTR_READ, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_IDTR);
+    VBOXSTRICTRC rcStrict = iemMemStoreDataXdtr(pVCpu, pVCpu->cpum.GstCtx.idtr.cbIdt, pVCpu->cpum.GstCtx.idtr.pIdt, iEffSeg, GCPtrEffDst);
+    if (rcStrict == VINF_SUCCESS)
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Implements lldt.
+ *
+ * @param   uNewLdt     The new LDT selector value.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_lldt, uint16_t, uNewLdt)
+{
+    /*
+     * Check preconditions.
+     */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        Log(("lldt %04x - real or v8086 mode -> #GP(0)\n", uNewLdt));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("lldt %04x - CPL is %d -> #GP(0)\n", uNewLdt, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    /* Nested-guest VMX intercept. */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("lldt: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_LLDT, cbInstr);
+    }
+    if (uNewLdt & X86_SEL_LDT)
+    {
+        Log(("lldt %04x - LDT selector -> #GP\n", uNewLdt));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewLdt);
+    }
+
+    /*
+     * Now, loading a NULL selector is easy.
+     */
+    if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
+    {
+        /* Nested-guest SVM intercept. */
+        if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_LDTR_WRITES))
+        {
+            Log(("lldt: Guest intercept -> #VMEXIT\n"));
+            IEM_SVM_UPDATE_NRIP(pVCpu);
+            IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_LDTR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        Log(("lldt %04x: Loading NULL selector.\n", uNewLdt));
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_LDTR;
+        CPUMSetGuestLDTR(pVCpu, uNewLdt);
+        pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
+        pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+        if (IEM_IS_GUEST_CPU_AMD(pVCpu))
+        {
+            /* AMD-V seems to leave the base and limit alone. */
+            pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESCATTR_UNUSABLE;
+        }
+        else
+        {
+            /* VT-x (Intel 3960x) seems to be doing the following. */
+            pVCpu->cpum.GstCtx.ldtr.Attr.u   = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D;
+            pVCpu->cpum.GstCtx.ldtr.u64Base  = 0;
+            pVCpu->cpum.GstCtx.ldtr.u32Limit = UINT32_MAX;
+        }
+
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Read the descriptor.
+     */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR | CPUMCTX_EXTRN_GDTR);
+    IEMSELDESC Desc;
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uNewLdt, X86_XCPT_GP); /** @todo Correct exception? */
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Check GPs first. */
+    if (Desc.Legacy.Gen.u1DescType)
+    {
+        Log(("lldt %#x - not system selector (type %x) -> #GP\n", uNewLdt, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
+    }
+    if (Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
+    {
+        Log(("lldt %#x - not LDT selector (type %x) -> #GP\n", uNewLdt, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
+    }
+    uint64_t u64Base;
+    if (!IEM_IS_LONG_MODE(pVCpu))
+        u64Base = X86DESC_BASE(&Desc.Legacy);
+    else
+    {
+        if (Desc.Long.Gen.u5Zeros)
+        {
+            Log(("lldt %#x - u5Zeros=%#x -> #GP\n", uNewLdt, Desc.Long.Gen.u5Zeros));
+            return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
+        }
+
+        u64Base = X86DESC64_BASE(&Desc.Long);
+        if (!IEM_IS_CANONICAL(u64Base))
+        {
+            Log(("lldt %#x - non-canonical base address %#llx -> #GP\n", uNewLdt, u64Base));
+            return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
+        }
+    }
+
+    /* NP */
+    if (!Desc.Legacy.Gen.u1Present)
+    {
+        Log(("lldt %#x - segment not present -> #NP\n", uNewLdt));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewLdt);
+    }
+
+    /* Nested-guest SVM intercept. */
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_LDTR_WRITES))
+    {
+        Log(("lldt: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_LDTR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * It checks out alright, update the registers.
+     */
+/** @todo check if the actual value is loaded or if the RPL is dropped */
+    CPUMSetGuestLDTR(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
+    pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt & X86_SEL_MASK_OFF_RPL;
+    pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.ldtr.Attr.u   = X86DESC_GET_HID_ATTR(&Desc.Legacy);
+    pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&Desc.Legacy);
+    pVCpu->cpum.GstCtx.ldtr.u64Base  = u64Base;
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements sldt GReg
+ *
+ * @param   iGReg           The general register to store the CRx value in.
+ * @param   enmEffOpSize    The operand size.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_sldt_reg, uint8_t, iGReg, uint8_t, enmEffOpSize)
+{
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("sldt: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_SLDT, cbInstr);
+    }
+
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_LDTR_READS, SVM_EXIT_LDTR_READ, 0, 0);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR);
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT: *(uint16_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.ldtr.Sel; break;
+        case IEMMODE_32BIT: *(uint64_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.ldtr.Sel; break;
+        case IEMMODE_64BIT: *(uint64_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.ldtr.Sel; break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements sldt mem.
+ *
+ * @param   iGReg           The general register to store the CRx value in.
+ * @param   iEffSeg         The effective segment register to use with @a GCPtrMem.
+ * @param   GCPtrEffDst     Where to store the 16-bit CR0 value.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_sldt_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_LDTR_READS, SVM_EXIT_LDTR_READ, 0, 0);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR);
+    VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, pVCpu->cpum.GstCtx.ldtr.Sel);
+    if (rcStrict == VINF_SUCCESS)
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Implements ltr.
+ *
+ * @param   uNewTr      The new TSS selector value.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_ltr, uint16_t, uNewTr)
+{
+    /*
+     * Check preconditions.
+     */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        Log(("ltr %04x - real or v8086 mode -> #GP(0)\n", uNewTr));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("ltr %04x - CPL is %d -> #GP(0)\n", uNewTr, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("ltr: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_LTR, cbInstr);
+    }
+    if (uNewTr & X86_SEL_LDT)
+    {
+        Log(("ltr %04x - LDT selector -> #GP\n", uNewTr));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewTr);
+    }
+    if (!(uNewTr & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("ltr %04x - NULL selector -> #GP(0)\n", uNewTr));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TR_WRITES))
+    {
+        Log(("ltr: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Read the descriptor.
+     */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_TR);
+    IEMSELDESC Desc;
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uNewTr, X86_XCPT_GP); /** @todo Correct exception? */
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Check GPs first. */
+    if (Desc.Legacy.Gen.u1DescType)
+    {
+        Log(("ltr %#x - not system selector (type %x) -> #GP\n", uNewTr, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
+    }
+    if (   Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL /* same as AMD64_SEL_TYPE_SYS_TSS_AVAIL */
+        && (   Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
+            || IEM_IS_LONG_MODE(pVCpu)) )
+    {
+        Log(("ltr %#x - not an available TSS selector (type %x) -> #GP\n", uNewTr, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
+    }
+    uint64_t u64Base;
+    if (!IEM_IS_LONG_MODE(pVCpu))
+        u64Base = X86DESC_BASE(&Desc.Legacy);
+    else
+    {
+        if (Desc.Long.Gen.u5Zeros)
+        {
+            Log(("ltr %#x - u5Zeros=%#x -> #GP\n", uNewTr, Desc.Long.Gen.u5Zeros));
+            return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
+        }
+
+        u64Base = X86DESC64_BASE(&Desc.Long);
+        if (!IEM_IS_CANONICAL(u64Base))
+        {
+            Log(("ltr %#x - non-canonical base address %#llx -> #GP\n", uNewTr, u64Base));
+            return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
+        }
+    }
+
+    /* NP */
+    if (!Desc.Legacy.Gen.u1Present)
+    {
+        Log(("ltr %#x - segment not present -> #NP\n", uNewTr));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewTr);
+    }
+
+    /*
+     * Set it busy.
+     * Note! Intel says this should lock down the whole descriptor, but we'll
+     *       restrict our selves to 32-bit for now due to lack of inline
+     *       assembly and such.
+     */
+    void *pvDesc;
+    rcStrict = iemMemMap(pVCpu, &pvDesc, 8, UINT8_MAX, pVCpu->cpum.GstCtx.gdtr.pGdt + (uNewTr & X86_SEL_MASK_OFF_RPL), IEM_ACCESS_DATA_RW);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    switch ((uintptr_t)pvDesc & 3)
+    {
+        case 0: ASMAtomicBitSet(pvDesc, 40 + 1); break;
+        case 1: ASMAtomicBitSet((uint8_t *)pvDesc + 3, 40 + 1 - 24); break;
+        case 2: ASMAtomicBitSet((uint8_t *)pvDesc + 2, 40 + 1 - 16); break;
+        case 3: ASMAtomicBitSet((uint8_t *)pvDesc + 1, 40 + 1 -  8); break;
+    }
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvDesc, IEM_ACCESS_DATA_RW);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
+
+    /*
+     * It checks out alright, update the registers.
+     */
+/** @todo check if the actual value is loaded or if the RPL is dropped */
+    CPUMSetGuestTR(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
+    pVCpu->cpum.GstCtx.tr.ValidSel = uNewTr & X86_SEL_MASK_OFF_RPL;
+    pVCpu->cpum.GstCtx.tr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.tr.Attr.u   = X86DESC_GET_HID_ATTR(&Desc.Legacy);
+    pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&Desc.Legacy);
+    pVCpu->cpum.GstCtx.tr.u64Base  = u64Base;
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements str GReg
+ *
+ * @param   iGReg           The general register to store the CRx value in.
+ * @param   enmEffOpSize    The operand size.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_str_reg, uint8_t, iGReg, uint8_t, enmEffOpSize)
+{
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("str_reg: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_STR, cbInstr);
+    }
+
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_TR_READS, SVM_EXIT_TR_READ, 0, 0);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR);
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT: *(uint16_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.tr.Sel; break;
+        case IEMMODE_32BIT: *(uint64_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.tr.Sel; break;
+        case IEMMODE_64BIT: *(uint64_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.tr.Sel; break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements str mem.
+ *
+ * @param   iGReg           The general register to store the CRx value in.
+ * @param   iEffSeg         The effective segment register to use with @a GCPtrMem.
+ * @param   GCPtrEffDst     Where to store the 16-bit CR0 value.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_str_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("str_mem: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_STR, cbInstr);
+    }
+
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_TR_READS, SVM_EXIT_TR_READ, 0, 0);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR);
+    VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, pVCpu->cpum.GstCtx.tr.Sel);
+    if (rcStrict == VINF_SUCCESS)
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Implements mov GReg,CRx.
+ *
+ * @param   iGReg           The general register to store the CRx value in.
+ * @param   iCrReg          The CRx register to read (valid).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Cd, uint8_t, iGReg, uint8_t, iCrReg)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+
+    if (IEM_SVM_IS_READ_CR_INTERCEPT_SET(pVCpu, iCrReg))
+    {
+        Log(("iemCImpl_mov_Rd_Cd: Guest intercept CR%u -> #VMEXIT\n", iCrReg));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_READ_CR0 + iCrReg, IEMACCESSCRX_MOV_CRX, iGReg);
+    }
+
+    /* Read it. */
+    uint64_t crX;
+    switch (iCrReg)
+    {
+        case 0:
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+            crX = pVCpu->cpum.GstCtx.cr0;
+            if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
+                crX |= UINT32_C(0x7fffffe0); /* All reserved CR0 flags are set on a 386, just like MSW on 286. */
+            break;
+        case 2:
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_CR2);
+            crX = pVCpu->cpum.GstCtx.cr2;
+            break;
+        case 3:
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
+            crX = pVCpu->cpum.GstCtx.cr3;
+            break;
+        case 4:
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+            crX = pVCpu->cpum.GstCtx.cr4;
+            break;
+        case 8:
+        {
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+            {
+                VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovFromCr8(pVCpu, iGReg, cbInstr);
+                if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+                    return rcStrict;
+
+                /*
+                 * If the Mov-from-CR8 doesn't cause a VM-exit, bits 7:4 of the VTPR is copied
+                 * to bits 0:3 of the destination operand. Bits 63:4 of the destination operand
+                 * are cleared.
+                 *
+                 * See Intel Spec. 29.3 "Virtualizing CR8-based TPR Accesses"
+                 */
+                if (IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_USE_TPR_SHADOW))
+                {
+                    uint32_t const uTpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR);
+                    crX = (uTpr >> 4) & 0xf;
+                    break;
+                }
+            }
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+            if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
+            {
+                PCSVMVMCBCTRL pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb)->ctrl;
+                if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, IEM_GET_CTX(pVCpu)))
+                {
+                    crX = pVmcbCtrl->IntCtrl.n.u8VTPR & 0xf;
+                    break;
+                }
+            }
+#endif
+            uint8_t uTpr;
+            int rc = APICGetTpr(pVCpu, &uTpr, NULL, NULL);
+            if (RT_SUCCESS(rc))
+                crX = uTpr >> 4;
+            else
+                crX = 0;
+            break;
+        }
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */
+    }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+        Assert(pVmcs);
+        switch (iCrReg)
+        {
+            /* CR0/CR4 reads are subject to masking when in VMX non-root mode. */
+            case 0: crX = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcs->u64Cr0Mask.u); break;
+            case 4: crX = CPUMGetGuestVmxMaskedCr4(&pVCpu->cpum.GstCtx, pVmcs->u64Cr4Mask.u); break;
+
+            case 3:
+            {
+                VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovFromCr3(pVCpu, iGReg, cbInstr);
+                if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+                    return rcStrict;
+                break;
+            }
+        }
+    }
+#endif
+
+    /* Store it. */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        *(uint64_t *)iemGRegRef(pVCpu, iGReg) = crX;
+    else
+        *(uint64_t *)iemGRegRef(pVCpu, iGReg) = (uint32_t)crX;
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements smsw GReg.
+ *
+ * @param   iGReg           The general register to store the CRx value in.
+ * @param   enmEffOpSize    The operand size.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_smsw_reg, uint8_t, iGReg, uint8_t, enmEffOpSize)
+{
+    IEM_SVM_CHECK_READ_CR0_INTERCEPT(pVCpu, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    uint64_t u64MaskedCr0;
+    if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        u64MaskedCr0 = pVCpu->cpum.GstCtx.cr0;
+    else
+    {
+        PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+        Assert(pVmcs);
+        u64MaskedCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcs->u64Cr0Mask.u);
+    }
+    uint64_t const u64GuestCr0 = u64MaskedCr0;
+#else
+    uint64_t const u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
+#endif
+
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386)
+                *(uint16_t *)iemGRegRef(pVCpu, iGReg) = (uint16_t)u64GuestCr0;
+            else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
+                *(uint16_t *)iemGRegRef(pVCpu, iGReg) = (uint16_t)u64GuestCr0 | 0xffe0;
+            else
+                *(uint16_t *)iemGRegRef(pVCpu, iGReg) = (uint16_t)u64GuestCr0 | 0xfff0;
+            break;
+
+        case IEMMODE_32BIT:
+            *(uint32_t *)iemGRegRef(pVCpu, iGReg) = (uint32_t)u64GuestCr0;
+            break;
+
+        case IEMMODE_64BIT:
+            *(uint64_t *)iemGRegRef(pVCpu, iGReg) = u64GuestCr0;
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements smsw mem.
+ *
+ * @param   iGReg           The general register to store the CR0 value in.
+ * @param   iEffSeg         The effective segment register to use with @a GCPtrMem.
+ * @param   GCPtrEffDst     Where to store the 16-bit CR0 value.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_smsw_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    IEM_SVM_CHECK_READ_CR0_INTERCEPT(pVCpu, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    uint64_t u64MaskedCr0;
+    if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        u64MaskedCr0 = pVCpu->cpum.GstCtx.cr0;
+    else
+    {
+        PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+        Assert(pVmcs);
+        u64MaskedCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcs->u64Cr0Mask.u);
+    }
+    uint64_t const u64GuestCr0 = u64MaskedCr0;
+#else
+    uint64_t const u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
+#endif
+
+    uint16_t u16Value;
+    if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386)
+        u16Value = (uint16_t)u64GuestCr0;
+    else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
+        u16Value = (uint16_t)u64GuestCr0 | 0xffe0;
+    else
+        u16Value = (uint16_t)u64GuestCr0 | 0xfff0;
+
+    VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, u16Value);
+    if (rcStrict == VINF_SUCCESS)
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Used to implemented 'mov CRx,GReg' and 'lmsw r/m16'.
+ *
+ * @param   iCrReg          The CRx register to write (valid).
+ * @param   uNewCrX         The new value.
+ * @param   enmAccessCrx    The instruction that caused the CrX load.
+ * @param   iGReg           The general register in case of a 'mov CRx,GReg'
+ *                          instruction.
+ */
+IEM_CIMPL_DEF_4(iemCImpl_load_CrX, uint8_t, iCrReg, uint64_t, uNewCrX, IEMACCESSCRX, enmAccessCrX, uint8_t, iGReg)
+{
+    VBOXSTRICTRC    rcStrict;
+    int             rc;
+#ifndef VBOX_WITH_NESTED_HWVIRT_SVM
+    RT_NOREF2(iGReg, enmAccessCrX);
+#endif
+
+    /*
+     * Try store it.
+     * Unfortunately, CPUM only does a tiny bit of the work.
+     */
+    switch (iCrReg)
+    {
+        case 0:
+        {
+            /*
+             * Perform checks.
+             */
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+
+            uint64_t const uOldCrX = pVCpu->cpum.GstCtx.cr0;
+            uint32_t const fValid  = CPUMGetGuestCR0ValidMask();
+
+            /* ET is hardcoded on 486 and later. */
+            if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_486)
+                uNewCrX |= X86_CR0_ET;
+            /* The 386 and 486 didn't #GP(0) on attempting to set reserved CR0 bits. ET was settable on 386. */
+            else if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_486)
+            {
+                uNewCrX &= fValid;
+                uNewCrX |= X86_CR0_ET;
+            }
+            else
+                uNewCrX &= X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG | X86_CR0_ET;
+
+            /* Check for reserved bits. */
+            if (uNewCrX & ~(uint64_t)fValid)
+            {
+                Log(("Trying to set reserved CR0 bits: NewCR0=%#llx InvalidBits=%#llx\n", uNewCrX, uNewCrX & ~(uint64_t)fValid));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            /* Check for invalid combinations. */
+            if (    (uNewCrX & X86_CR0_PG)
+                && !(uNewCrX & X86_CR0_PE) )
+            {
+                Log(("Trying to set CR0.PG without CR0.PE\n"));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            if (   !(uNewCrX & X86_CR0_CD)
+                && (uNewCrX & X86_CR0_NW) )
+            {
+                Log(("Trying to clear CR0.CD while leaving CR0.NW set\n"));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            if (   !(uNewCrX & X86_CR0_PG)
+                && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCIDE))
+            {
+                Log(("Trying to clear CR0.PG while leaving CR4.PCID set\n"));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            /* Long mode consistency checks. */
+            if (    (uNewCrX & X86_CR0_PG)
+                && !(uOldCrX & X86_CR0_PG)
+                &&  (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME) )
+            {
+                if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE))
+                {
+                    Log(("Trying to enabled long mode paging without CR4.PAE set\n"));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+                if (pVCpu->cpum.GstCtx.cs.Attr.n.u1Long)
+                {
+                    Log(("Trying to enabled long mode paging with a long CS descriptor loaded.\n"));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+            }
+
+            /* Check for bits that must remain set or cleared in VMX operation,
+               see Intel spec. 23.8 "Restrictions on VMX operation". */
+            if (IEM_VMX_IS_ROOT_MODE(pVCpu))
+            {
+                uint32_t const uCr0Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed0;
+                if ((uNewCrX & uCr0Fixed0) != uCr0Fixed0)
+                {
+                    Log(("Trying to clear reserved CR0 bits in VMX operation: NewCr0=%#llx MB1=%#llx\n", uNewCrX, uCr0Fixed0));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+
+                uint32_t const uCr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1;
+                if (uNewCrX & ~uCr0Fixed1)
+                {
+                    Log(("Trying to set reserved CR0 bits in VMX operation: NewCr0=%#llx MB0=%#llx\n", uNewCrX, uCr0Fixed1));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+            }
+
+            /** @todo check reserved PDPTR bits as AMD states. */
+
+            /*
+             * SVM nested-guest CR0 write intercepts.
+             */
+            if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, iCrReg))
+            {
+                Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
+                IEM_SVM_UPDATE_NRIP(pVCpu);
+                IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR0, enmAccessCrX, iGReg);
+            }
+            if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
+            {
+                /* 'lmsw' intercepts regardless of whether the TS/MP bits are actually toggled. */
+                if (   enmAccessCrX == IEMACCESSCRX_LMSW
+                    || (uNewCrX & ~(X86_CR0_TS | X86_CR0_MP)) != (uOldCrX & ~(X86_CR0_TS | X86_CR0_MP)))
+                {
+                    Assert(enmAccessCrX != IEMACCESSCRX_CLTS);
+                    Log(("iemCImpl_load_Cr%#x: lmsw or bits other than TS/MP changed: Guest intercept -> #VMEXIT\n", iCrReg));
+                    IEM_SVM_UPDATE_NRIP(pVCpu);
+                    IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_CR0_SEL_WRITE, enmAccessCrX, iGReg);
+                }
+            }
+
+            /*
+             * Change CR0.
+             */
+            CPUMSetGuestCR0(pVCpu, uNewCrX);
+            Assert(pVCpu->cpum.GstCtx.cr0 == uNewCrX);
+
+            /*
+             * Change EFER.LMA if entering or leaving long mode.
+             */
+            if (   (uNewCrX & X86_CR0_PG) != (uOldCrX & X86_CR0_PG)
+                && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME) )
+            {
+                uint64_t NewEFER = pVCpu->cpum.GstCtx.msrEFER;
+                if (uNewCrX & X86_CR0_PG)
+                    NewEFER |= MSR_K6_EFER_LMA;
+                else
+                    NewEFER &= ~MSR_K6_EFER_LMA;
+
+                CPUMSetGuestEFER(pVCpu, NewEFER);
+                Assert(pVCpu->cpum.GstCtx.msrEFER == NewEFER);
+            }
+
+            /*
+             * Inform PGM.
+             */
+            if (    (uNewCrX & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE))
+                !=  (uOldCrX & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE)) )
+            {
+                rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */);
+                AssertRCReturn(rc, rc);
+                /* ignore informational status codes */
+            }
+            rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER);
+            break;
+        }
+
+        /*
+         * CR2 can be changed without any restrictions.
+         */
+        case 2:
+        {
+            if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /*cr*/ 2))
+            {
+                Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
+                IEM_SVM_UPDATE_NRIP(pVCpu);
+                IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR2, enmAccessCrX, iGReg);
+            }
+            pVCpu->cpum.GstCtx.cr2 = uNewCrX;
+            pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_CR2;
+            rcStrict  = VINF_SUCCESS;
+            break;
+        }
+
+        /*
+         * CR3 is relatively simple, although AMD and Intel have different
+         * accounts of how setting reserved bits are handled.  We take intel's
+         * word for the lower bits and AMD's for the high bits (63:52).  The
+         * lower reserved bits are ignored and left alone; OpenBSD 5.8 relies
+         * on this.
+         */
+        /** @todo Testcase: Setting reserved bits in CR3, especially before
+         *        enabling paging. */
+        case 3:
+        {
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
+
+            /* Bit 63 being clear in the source operand with PCIDE indicates no invalidations are required. */
+            if (   (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCIDE)
+                && (uNewCrX & RT_BIT_64(63)))
+            {
+                /** @todo r=ramshankar: avoiding a TLB flush altogether here causes Windows 10
+                 *        SMP(w/o nested-paging) to hang during bootup on Skylake systems, see
+                 *        Intel spec. 4.10.4.1 "Operations that Invalidate TLBs and
+                 *        Paging-Structure Caches". */
+                uNewCrX &= ~RT_BIT_64(63);
+            }
+
+            /* Check / mask the value. */
+            if (uNewCrX & UINT64_C(0xfff0000000000000))
+            {
+                Log(("Trying to load CR3 with invalid high bits set: %#llx\n", uNewCrX));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            uint64_t fValid;
+            if (   (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
+                && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME))
+                fValid = UINT64_C(0x000fffffffffffff);
+            else
+                fValid = UINT64_C(0xffffffff);
+            if (uNewCrX & ~fValid)
+            {
+                Log(("Automatically clearing reserved MBZ bits in CR3 load: NewCR3=%#llx ClearedBits=%#llx\n",
+                     uNewCrX, uNewCrX & ~fValid));
+                uNewCrX &= fValid;
+            }
+
+            if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /*cr*/ 3))
+            {
+                Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
+                IEM_SVM_UPDATE_NRIP(pVCpu);
+                IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR3, enmAccessCrX, iGReg);
+            }
+
+            /** @todo If we're in PAE mode we should check the PDPTRs for
+             *        invalid bits. */
+
+            /* Make the change. */
+            rc = CPUMSetGuestCR3(pVCpu, uNewCrX);
+            AssertRCSuccessReturn(rc, rc);
+
+            /* Inform PGM. */
+            if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG)
+            {
+                rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
+                AssertRCReturn(rc, rc);
+                /* ignore informational status codes */
+            }
+            rcStrict = VINF_SUCCESS;
+            break;
+        }
+
+        /*
+         * CR4 is a bit more tedious as there are bits which cannot be cleared
+         * under some circumstances and such.
+         */
+        case 4:
+        {
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+            uint64_t const uOldCrX = pVCpu->cpum.GstCtx.cr4;
+
+            /* Reserved bits. */
+            uint32_t const fValid = CPUMGetGuestCR4ValidMask(pVCpu->CTX_SUFF(pVM));
+            if (uNewCrX & ~(uint64_t)fValid)
+            {
+                Log(("Trying to set reserved CR4 bits: NewCR4=%#llx InvalidBits=%#llx\n", uNewCrX, uNewCrX & ~(uint64_t)fValid));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            bool const fPcide    = !(uOldCrX & X86_CR4_PCIDE) && (uNewCrX & X86_CR4_PCIDE);
+            bool const fLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu));
+
+            /* PCIDE check. */
+            if (   fPcide
+                && (   !fLongMode
+                    || (pVCpu->cpum.GstCtx.cr3 & UINT64_C(0xfff))))
+            {
+                Log(("Trying to set PCIDE with invalid PCID or outside long mode. Pcid=%#x\n", (pVCpu->cpum.GstCtx.cr3 & UINT64_C(0xfff))));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            /* PAE check. */
+            if (   fLongMode
+                && (uOldCrX & X86_CR4_PAE)
+                && !(uNewCrX & X86_CR4_PAE))
+            {
+                Log(("Trying to set clear CR4.PAE while long mode is active\n"));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /*cr*/ 4))
+            {
+                Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
+                IEM_SVM_UPDATE_NRIP(pVCpu);
+                IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR4, enmAccessCrX, iGReg);
+            }
+
+            /* Check for bits that must remain set or cleared in VMX operation,
+               see Intel spec. 23.8 "Restrictions on VMX operation". */
+            if (IEM_VMX_IS_ROOT_MODE(pVCpu))
+            {
+                uint32_t const uCr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0;
+                if ((uNewCrX & uCr4Fixed0) != uCr4Fixed0)
+                {
+                    Log(("Trying to clear reserved CR4 bits in VMX operation: NewCr4=%#llx MB1=%#llx\n", uNewCrX, uCr4Fixed0));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+
+                uint32_t const uCr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1;
+                if (uNewCrX & ~uCr4Fixed1)
+                {
+                    Log(("Trying to set reserved CR4 bits in VMX operation: NewCr4=%#llx MB0=%#llx\n", uNewCrX, uCr4Fixed1));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+            }
+
+            /*
+             * Change it.
+             */
+            rc = CPUMSetGuestCR4(pVCpu, uNewCrX);
+            AssertRCSuccessReturn(rc, rc);
+            Assert(pVCpu->cpum.GstCtx.cr4 == uNewCrX);
+
+            /*
+             * Notify SELM and PGM.
+             */
+            /* SELM - VME may change things wrt to the TSS shadowing. */
+            if ((uNewCrX ^ uOldCrX) & X86_CR4_VME)
+                Log(("iemCImpl_load_CrX: VME %d -> %d\n", RT_BOOL(uOldCrX & X86_CR4_VME), RT_BOOL(uNewCrX & X86_CR4_VME) ));
+
+            /* PGM - flushing and mode. */
+            if ((uNewCrX ^ uOldCrX) & (X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE | X86_CR4_PCIDE /* | X86_CR4_SMEP */))
+            {
+                rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */);
+                AssertRCReturn(rc, rc);
+                /* ignore informational status codes */
+            }
+            rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER);
+            break;
+        }
+
+        /*
+         * CR8 maps to the APIC TPR.
+         */
+        case 8:
+        {
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
+            if (uNewCrX & ~(uint64_t)0xf)
+            {
+                Log(("Trying to set reserved CR8 bits (%#RX64)\n", uNewCrX));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+                && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_USE_TPR_SHADOW))
+            {
+                /*
+                 * If the Mov-to-CR8 doesn't cause a VM-exit, bits 0:3 of the source operand
+                 * is copied to bits 7:4 of the VTPR. Bits 0:3 and bits 31:8 of the VTPR are
+                 * cleared. Following this the processor performs TPR virtualization.
+                 *
+                 * However, we should not perform TPR virtualization immediately here but
+                 * after this instruction has completed.
+                 *
+                 * See Intel spec. 29.3 "Virtualizing CR8-based TPR Accesses"
+                 * See Intel spec. 27.1 "Architectural State Before A VM-exit"
+                 */
+                uint32_t const uTpr = (uNewCrX & 0xf) << 4;
+                Log(("iemCImpl_load_Cr%#x: Virtualizing TPR (%#x) write\n", iCrReg, uTpr));
+                iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_TPR, uTpr);
+                iemVmxVirtApicSetPendingWrite(pVCpu, XAPIC_OFF_TPR);
+                rcStrict = VINF_SUCCESS;
+                break;
+            }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+            if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
+            {
+                if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /*cr*/ 8))
+                {
+                    Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
+                    IEM_SVM_UPDATE_NRIP(pVCpu);
+                    IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR8, enmAccessCrX, iGReg);
+                }
+
+                PSVMVMCBCTRL pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb)->ctrl;
+                pVmcbCtrl->IntCtrl.n.u8VTPR = uNewCrX;
+                if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, IEM_GET_CTX(pVCpu)))
+                {
+                    rcStrict = VINF_SUCCESS;
+                    break;
+                }
+            }
+#endif
+            uint8_t const u8Tpr = (uint8_t)uNewCrX << 4;
+            APICSetTpr(pVCpu, u8Tpr);
+            rcStrict = VINF_SUCCESS;
+            break;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */
+    }
+
+    /*
+     * Advance the RIP on success.
+     */
+    if (RT_SUCCESS(rcStrict))
+    {
+        if (rcStrict != VINF_SUCCESS)
+            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Implements mov CRx,GReg.
+ *
+ * @param   iCrReg          The CRx register to write (valid).
+ * @param   iGReg           The general register to load the CRx value from.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_mov_Cd_Rd, uint8_t, iCrReg, uint8_t, iGReg)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+
+    /*
+     * Read the new value from the source register and call common worker.
+     */
+    uint64_t uNewCrX;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        uNewCrX = iemGRegFetchU64(pVCpu, iGReg);
+    else
+        uNewCrX = iemGRegFetchU32(pVCpu, iGReg);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict = VINF_VMX_INTERCEPT_NOT_ACTIVE;
+        switch (iCrReg)
+        {
+            case 0:
+            case 4: rcStrict = iemVmxVmexitInstrMovToCr0Cr4(pVCpu, iCrReg, &uNewCrX, iGReg, cbInstr);   break;
+            case 3: rcStrict = iemVmxVmexitInstrMovToCr3(pVCpu, uNewCrX, iGReg, cbInstr);               break;
+            case 8: rcStrict = iemVmxVmexitInstrMovToCr8(pVCpu, iGReg, cbInstr);                        break;
+        }
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+    return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, iCrReg, uNewCrX, IEMACCESSCRX_MOV_CRX, iGReg);
+}
+
+
+/**
+ * Implements 'LMSW r/m16'
+ *
+ * @param   u16NewMsw       The new value.
+ * @param   GCPtrEffDst     The guest-linear address of the source operand in case
+ *                          of a memory operand. For register operand, pass
+ *                          NIL_RTGCPTR.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_lmsw, uint16_t, u16NewMsw, RTGCPTR, GCPtrEffDst)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /* Check nested-guest VMX intercept and get updated MSW if there's no VM-exit. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict = iemVmxVmexitInstrLmsw(pVCpu, pVCpu->cpum.GstCtx.cr0, &u16NewMsw, GCPtrEffDst, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#else
+    RT_NOREF_PV(GCPtrEffDst);
+#endif
+
+    /*
+     * Compose the new CR0 value and call common worker.
+     */
+    uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0  & ~(X86_CR0_MP | X86_CR0_EM | X86_CR0_TS);
+    uNewCr0 |= u16NewMsw & (X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS);
+    return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, /*cr*/ 0, uNewCr0, IEMACCESSCRX_LMSW, UINT8_MAX /* iGReg */);
+}
+
+
+/**
+ * Implements 'CLTS'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_clts)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+    uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0;
+    uNewCr0 &= ~X86_CR0_TS;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict = iemVmxVmexitInstrClts(pVCpu, cbInstr);
+        if (rcStrict == VINF_VMX_MODIFIES_BEHAVIOR)
+            uNewCr0 |= (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS);
+        else if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+    return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, /*cr*/ 0, uNewCr0, IEMACCESSCRX_CLTS, UINT8_MAX /* iGReg */);
+}
+
+
+/**
+ * Implements mov GReg,DRx.
+ *
+ * @param   iGReg           The general register to store the DRx value in.
+ * @param   iDrReg          The DRx register to read (0-7).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Dd, uint8_t, iGReg, uint8_t, iDrReg)
+{
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * Check nested-guest VMX intercept.
+     * Unlike most other intercepts, the Mov DRx intercept takes preceedence
+     * over CPL and CR4.DE and even DR4/DR5 checks.
+     *
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovDrX(pVCpu, VMXINSTRID_MOV_FROM_DRX, iDrReg, iGReg, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+    /*
+     * Check preconditions.
+     */
+    /* Raise GPs. */
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7 | CPUMCTX_EXTRN_CR0);
+
+    if (   (iDrReg == 4 || iDrReg == 5)
+        && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE) )
+    {
+        Log(("mov r%u,dr%u: CR4.DE=1 -> #GP(0)\n", iGReg, iDrReg));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Raise #DB if general access detect is enabled. */
+    if (pVCpu->cpum.GstCtx.dr[7] & X86_DR7_GD)
+    {
+        Log(("mov r%u,dr%u: DR7.GD=1 -> #DB\n", iGReg, iDrReg));
+        return iemRaiseDebugException(pVCpu);
+    }
+
+    /*
+     * Read the debug register and store it in the specified general register.
+     */
+    uint64_t drX;
+    switch (iDrReg)
+    {
+        case 0:
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+            drX = pVCpu->cpum.GstCtx.dr[0];
+            break;
+        case 1:
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+            drX = pVCpu->cpum.GstCtx.dr[1];
+            break;
+        case 2:
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+            drX = pVCpu->cpum.GstCtx.dr[2];
+            break;
+        case 3:
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+            drX = pVCpu->cpum.GstCtx.dr[3];
+            break;
+        case 6:
+        case 4:
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
+            drX = pVCpu->cpum.GstCtx.dr[6];
+            drX |= X86_DR6_RA1_MASK;
+            drX &= ~X86_DR6_RAZ_MASK;
+            break;
+        case 7:
+        case 5:
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
+            drX = pVCpu->cpum.GstCtx.dr[7];
+            drX |=X86_DR7_RA1_MASK;
+            drX &= ~X86_DR7_RAZ_MASK;
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */
+    }
+
+    /** @todo SVM nested-guest intercept for DR8-DR15? */
+    /*
+     * Check for any SVM nested-guest intercepts for the DRx read.
+     */
+    if (IEM_SVM_IS_READ_DR_INTERCEPT_SET(pVCpu, iDrReg))
+    {
+        Log(("mov r%u,dr%u: Guest intercept -> #VMEXIT\n", iGReg, iDrReg));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_READ_DR0 + (iDrReg & 0xf),
+                              IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? (iGReg & 7) : 0, 0 /* uExitInfo2 */);
+    }
+
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        *(uint64_t *)iemGRegRef(pVCpu, iGReg) = drX;
+    else
+        *(uint64_t *)iemGRegRef(pVCpu, iGReg) = (uint32_t)drX;
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements mov DRx,GReg.
+ *
+ * @param   iDrReg          The DRx register to write (valid).
+ * @param   iGReg           The general register to load the DRx value from.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_mov_Dd_Rd, uint8_t, iDrReg, uint8_t, iGReg)
+{
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * Check nested-guest VMX intercept.
+     * Unlike most other intercepts, the Mov DRx intercept takes preceedence
+     * over CPL and CR4.DE and even DR4/DR5 checks.
+     *
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovDrX(pVCpu, VMXINSTRID_MOV_TO_DRX, iDrReg, iGReg, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+    /*
+     * Check preconditions.
+     */
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7 | CPUMCTX_EXTRN_CR4);
+
+    if (iDrReg == 4 || iDrReg == 5)
+    {
+        if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE)
+        {
+            Log(("mov dr%u,r%u: CR4.DE=1 -> #GP(0)\n", iDrReg, iGReg));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+        iDrReg += 2;
+    }
+
+    /* Raise #DB if general access detect is enabled. */
+    /** @todo is \#DB/DR7.GD raised before any reserved high bits in DR7/DR6
+     *        \#GP? */
+    if (pVCpu->cpum.GstCtx.dr[7] & X86_DR7_GD)
+    {
+        Log(("mov dr%u,r%u: DR7.GD=1 -> #DB\n", iDrReg, iGReg));
+        return iemRaiseDebugException(pVCpu);
+    }
+
+    /*
+     * Read the new value from the source register.
+     */
+    uint64_t uNewDrX;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        uNewDrX = iemGRegFetchU64(pVCpu, iGReg);
+    else
+        uNewDrX = iemGRegFetchU32(pVCpu, iGReg);
+
+    /*
+     * Adjust it.
+     */
+    switch (iDrReg)
+    {
+        case 0:
+        case 1:
+        case 2:
+        case 3:
+            /* nothing to adjust */
+            break;
+
+        case 6:
+            if (uNewDrX & X86_DR6_MBZ_MASK)
+            {
+                Log(("mov dr%u,%#llx: DR6 high bits are not zero -> #GP(0)\n", iDrReg, uNewDrX));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+            uNewDrX |= X86_DR6_RA1_MASK;
+            uNewDrX &= ~X86_DR6_RAZ_MASK;
+            break;
+
+        case 7:
+            if (uNewDrX & X86_DR7_MBZ_MASK)
+            {
+                Log(("mov dr%u,%#llx: DR7 high bits are not zero -> #GP(0)\n", iDrReg, uNewDrX));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+            uNewDrX |= X86_DR7_RA1_MASK;
+            uNewDrX &= ~X86_DR7_RAZ_MASK;
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    /** @todo SVM nested-guest intercept for DR8-DR15? */
+    /*
+     * Check for any SVM nested-guest intercepts for the DRx write.
+     */
+    if (IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(pVCpu, iDrReg))
+    {
+        Log2(("mov dr%u,r%u: Guest intercept -> #VMEXIT\n", iDrReg, iGReg));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_DR0 + (iDrReg & 0xf),
+                              IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? (iGReg & 7) : 0, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Do the actual setting.
+     */
+    if (iDrReg < 4)
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+    else if (iDrReg == 6)
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
+
+    int rc = CPUMSetGuestDRx(pVCpu, iDrReg, uNewDrX);
+    AssertRCSuccessReturn(rc, RT_SUCCESS_NP(rc) ? VERR_IEM_IPE_1 : rc);
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'INVLPG m'.
+ *
+ * @param   GCPtrPage       The effective address of the page to invalidate.
+ * @remarks Updates the RIP.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_invlpg, RTGCPTR, GCPtrPage)
+{
+    /* ring-0 only. */
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INVLPG_EXIT))
+    {
+        Log(("invlpg: Guest intercept (%RGp) -> VM-exit\n", GCPtrPage));
+        return iemVmxVmexitInstrInvlpg(pVCpu, GCPtrPage, cbInstr);
+    }
+#endif
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_INVLPG))
+    {
+        Log(("invlpg: Guest intercept (%RGp) -> #VMEXIT\n", GCPtrPage));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_INVLPG,
+                              IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? GCPtrPage : 0, 0 /* uExitInfo2 */);
+    }
+
+    int rc = PGMInvalidatePage(pVCpu, GCPtrPage);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+
+    if (rc == VINF_SUCCESS)
+        return VINF_SUCCESS;
+    if (rc == VINF_PGM_SYNC_CR3)
+        return iemSetPassUpStatus(pVCpu, rc);
+
+    AssertMsg(rc == VINF_EM_RAW_EMULATE_INSTR || RT_FAILURE_NP(rc), ("%Rrc\n", rc));
+    Log(("PGMInvalidatePage(%RGv) -> %Rrc\n", GCPtrPage, rc));
+    return rc;
+}
+
+
+/**
+ * Implements INVPCID.
+ *
+ * @param   iEffSeg              The segment of the invpcid descriptor.
+ * @param   GCPtrInvpcidDesc     The address of invpcid descriptor.
+ * @param   uInvpcidType         The invalidation type.
+ * @remarks Updates the RIP.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_invpcid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvpcidDesc, uint64_t, uInvpcidType)
+{
+    /*
+     * Check preconditions.
+     */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fInvpcid)
+        return iemRaiseUndefinedOpcode(pVCpu);
+
+    /* When in VMX non-root mode and INVPCID is not enabled, it results in #UD. */
+    if (    IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_INVPCID))
+    {
+        Log(("invpcid: Not enabled for nested-guest execution -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("invpcid: CPL != 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    if (IEM_IS_V86_MODE(pVCpu))
+    {
+        Log(("invpcid: v8086 mode -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Check nested-guest intercept.
+     *
+     * INVPCID causes a VM-exit if "enable INVPCID" and "INVLPG exiting" are
+     * both set. We have already checked the former earlier in this function.
+     *
+     * CPL and virtual-8086 mode checks take priority over this VM-exit.
+     * See Intel spec. "25.1.1 Relative Priority of Faults and VM Exits".
+     */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INVLPG_EXIT))
+    {
+        Log(("invpcid: Guest intercept -> #VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_INVPCID, VMXINSTRID_NONE, cbInstr);
+    }
+
+    if (uInvpcidType > X86_INVPCID_TYPE_MAX_VALID)
+    {
+        Log(("invpcid: invalid/unrecognized invpcid type %#RX64 -> #GP(0)\n", uInvpcidType));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);
+
+    /*
+     * Fetch the invpcid descriptor from guest memory.
+     */
+    RTUINT128U uDesc;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataU128(pVCpu, &uDesc, iEffSeg, GCPtrInvpcidDesc);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /*
+         * Validate the descriptor.
+         */
+        if (uDesc.s.Lo > 0xfff)
+        {
+            Log(("invpcid: reserved bits set in invpcid descriptor %#RX64 -> #GP(0)\n", uDesc.s.Lo));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        RTGCUINTPTR64 const GCPtrInvAddr = uDesc.s.Hi;
+        uint8_t       const uPcid        = uDesc.s.Lo & UINT64_C(0xfff);
+        uint32_t      const uCr4         = pVCpu->cpum.GstCtx.cr4;
+        uint64_t      const uCr3         = pVCpu->cpum.GstCtx.cr3;
+        switch (uInvpcidType)
+        {
+            case X86_INVPCID_TYPE_INDV_ADDR:
+            {
+                if (!IEM_IS_CANONICAL(GCPtrInvAddr))
+                {
+                    Log(("invpcid: invalidation address %#RGP is not canonical -> #GP(0)\n", GCPtrInvAddr));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+                if (  !(uCr4 & X86_CR4_PCIDE)
+                    && uPcid != 0)
+                {
+                    Log(("invpcid: invalid pcid %#x\n", uPcid));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+
+                /* Invalidate mappings for the linear address tagged with PCID except global translations. */
+                PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */);
+                break;
+            }
+
+            case X86_INVPCID_TYPE_SINGLE_CONTEXT:
+            {
+                if (  !(uCr4 & X86_CR4_PCIDE)
+                    && uPcid != 0)
+                {
+                    Log(("invpcid: invalid pcid %#x\n", uPcid));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+                /* Invalidate all mappings associated with PCID except global translations. */
+                PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */);
+                break;
+            }
+
+            case X86_INVPCID_TYPE_ALL_CONTEXT_INCL_GLOBAL:
+            {
+                PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
+                break;
+            }
+
+            case X86_INVPCID_TYPE_ALL_CONTEXT_EXCL_GLOBAL:
+            {
+                PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */);
+                break;
+            }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements INVD.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_invd)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("invd: CPL != 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_INVD, cbInstr);
+
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_INVD, SVM_EXIT_INVD, 0, 0);
+
+    /* We currently take no action here. */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements WBINVD.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_wbinvd)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("wbinvd: CPL != 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_WBINVD, cbInstr);
+
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_WBINVD, SVM_EXIT_WBINVD, 0, 0);
+
+    /* We currently take no action here. */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0xaa. */
+IEM_CIMPL_DEF_0(iemCImpl_rsm)
+{
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_RSM, SVM_EXIT_RSM, 0, 0);
+    NOREF(cbInstr);
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+
+/**
+ * Implements RDTSC.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_rdtsc)
+{
+    /*
+     * Check preconditions.
+     */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fTsc)
+        return iemRaiseUndefinedOpcode(pVCpu);
+
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+        if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_TSD)
+        {
+            Log(("rdtsc: CR4.TSD and CPL=%u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+    }
+
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDTSC_EXIT))
+    {
+        Log(("rdtsc: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDTSC, cbInstr);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDTSC))
+    {
+        Log(("rdtsc: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDTSC, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Do the job.
+     */
+    uint64_t uTicks = TMCpuTickGet(pVCpu);
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+    uTicks = CPUMApplyNestedGuestTscOffset(pVCpu, uTicks);
+#endif
+    pVCpu->cpum.GstCtx.rax = RT_LO_U32(uTicks);
+    pVCpu->cpum.GstCtx.rdx = RT_HI_U32(uTicks);
+    pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX); /* For IEMExecDecodedRdtsc. */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements RDTSC.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_rdtscp)
+{
+    /*
+     * Check preconditions.
+     */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fRdTscP)
+        return iemRaiseUndefinedOpcode(pVCpu);
+
+    if (    IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_RDTSCP))
+    {
+        Log(("rdtscp: Not enabled for VMX non-root mode -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+        if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_TSD)
+        {
+            Log(("rdtscp: CR4.TSD and CPL=%u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+    }
+
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDTSC_EXIT))
+    {
+        Log(("rdtscp: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDTSCP, cbInstr);
+    }
+    else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDTSCP))
+    {
+        Log(("rdtscp: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDTSCP, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Do the job.
+     * Query the MSR first in case of trips to ring-3.
+     */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
+    VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pVCpu->cpum.GstCtx.rcx);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /* Low dword of the TSC_AUX msr only. */
+        pVCpu->cpum.GstCtx.rcx &= UINT32_C(0xffffffff);
+
+        uint64_t uTicks = TMCpuTickGet(pVCpu);
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+        uTicks = CPUMApplyNestedGuestTscOffset(pVCpu, uTicks);
+#endif
+        pVCpu->cpum.GstCtx.rax = RT_LO_U32(uTicks);
+        pVCpu->cpum.GstCtx.rdx = RT_HI_U32(uTicks);
+        pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RCX); /* For IEMExecDecodedRdtscp. */
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements RDPMC.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_rdpmc)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+
+    if (   pVCpu->iem.s.uCpl != 0
+        && !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCE))
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDPMC_EXIT))
+    {
+        Log(("rdpmc: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDPMC, cbInstr);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDPMC))
+    {
+        Log(("rdpmc: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDPMC, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /** @todo Emulate performance counters, for now just return 0. */
+    pVCpu->cpum.GstCtx.rax = 0;
+    pVCpu->cpum.GstCtx.rdx = 0;
+    pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX);
+    /** @todo We should trigger a \#GP here if the CPU doesn't support the index in
+     *        ecx but see @bugref{3472}! */
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements RDMSR.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_rdmsr)
+{
+    /*
+     * Check preconditions.
+     */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMsr)
+        return iemRaiseUndefinedOpcode(pVCpu);
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    /*
+     * Check nested-guest intercepts.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (iemVmxIsRdmsrWrmsrInterceptSet(pVCpu, VMX_EXIT_RDMSR, pVCpu->cpum.GstCtx.ecx))
+            IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDMSR, cbInstr);
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MSR_PROT))
+    {
+        VBOXSTRICTRC rcStrict = iemSvmHandleMsrIntercept(pVCpu, pVCpu->cpum.GstCtx.ecx, false /* fWrite */);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("IEM: SVM intercepted rdmsr(%#x) failed. rc=%Rrc\n", pVCpu->cpum.GstCtx.ecx, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    /*
+     * Do the job.
+     */
+    RTUINT64U uValue;
+    /** @todo make CPUMAllMsrs.cpp import the necessary MSR state. */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
+
+    VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pVCpu->cpum.GstCtx.ecx, &uValue.u);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        pVCpu->cpum.GstCtx.rax = uValue.s.Lo;
+        pVCpu->cpum.GstCtx.rdx = uValue.s.Hi;
+        pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX);
+
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+#ifndef IN_RING3
+    /* Deferred to ring-3. */
+    if (rcStrict == VINF_CPUM_R3_MSR_READ)
+    {
+        Log(("IEM: rdmsr(%#x) -> ring-3\n", pVCpu->cpum.GstCtx.ecx));
+        return rcStrict;
+    }
+#endif
+
+    /* Often a unimplemented MSR or MSR bit, so worth logging. */
+    if (pVCpu->iem.s.cLogRelRdMsr < 32)
+    {
+        pVCpu->iem.s.cLogRelRdMsr++;
+        LogRel(("IEM: rdmsr(%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.ecx));
+    }
+    else
+        Log((   "IEM: rdmsr(%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.ecx));
+    AssertMsgReturn(rcStrict == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IPE_UNEXPECTED_STATUS);
+    return iemRaiseGeneralProtectionFault0(pVCpu);
+}
+
+
+/**
+ * Implements WRMSR.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_wrmsr)
+{
+    /*
+     * Check preconditions.
+     */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMsr)
+        return iemRaiseUndefinedOpcode(pVCpu);
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    RTUINT64U uValue;
+    uValue.s.Lo = pVCpu->cpum.GstCtx.eax;
+    uValue.s.Hi = pVCpu->cpum.GstCtx.edx;
+
+    uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
+
+    /** @todo make CPUMAllMsrs.cpp import the necessary MSR state. */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
+
+    /*
+     * Check nested-guest intercepts.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (iemVmxIsRdmsrWrmsrInterceptSet(pVCpu, VMX_EXIT_WRMSR, idMsr))
+            IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_WRMSR, cbInstr);
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MSR_PROT))
+    {
+        VBOXSTRICTRC rcStrict = iemSvmHandleMsrIntercept(pVCpu, idMsr, true /* fWrite */);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("IEM: SVM intercepted rdmsr(%#x) failed. rc=%Rrc\n", idMsr, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    /*
+     * Do the job.
+     */
+    VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, idMsr, uValue.u);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+#ifndef IN_RING3
+    /* Deferred to ring-3. */
+    if (rcStrict == VINF_CPUM_R3_MSR_WRITE)
+    {
+        Log(("IEM: wrmsr(%#x) -> ring-3\n", idMsr));
+        return rcStrict;
+    }
+#endif
+
+    /* Often a unimplemented MSR or MSR bit, so worth logging. */
+    if (pVCpu->iem.s.cLogRelWrMsr < 32)
+    {
+        pVCpu->iem.s.cLogRelWrMsr++;
+        LogRel(("IEM: wrmsr(%#x,%#x`%08x) -> #GP(0)\n", idMsr, uValue.s.Hi, uValue.s.Lo));
+    }
+    else
+        Log((   "IEM: wrmsr(%#x,%#x`%08x) -> #GP(0)\n", idMsr, uValue.s.Hi, uValue.s.Lo));
+    AssertMsgReturn(rcStrict == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IPE_UNEXPECTED_STATUS);
+    return iemRaiseGeneralProtectionFault0(pVCpu);
+}
+
+
+/**
+ * Implements 'IN eAX, port'.
+ *
+ * @param   u16Port     The source port.
+ * @param   fImm        Whether the port was specified through an immediate operand
+ *                      or the implicit DX register.
+ * @param   cbReg       The register size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_in, uint16_t, u16Port, bool, fImm, uint8_t, cbReg)
+{
+    /*
+     * CPL check
+     */
+    VBOXSTRICTRC rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, cbReg);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Check VMX nested-guest IO intercept.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        rcStrict = iemVmxVmexitInstrIo(pVCpu, VMXINSTRID_IO_IN, u16Port, fImm, cbReg, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#else
+    RT_NOREF(fImm);
+#endif
+
+    /*
+     * Check SVM nested-guest IO intercept.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
+    {
+        uint8_t cAddrSizeBits;
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: cAddrSizeBits = 16; break;
+            case IEMMODE_32BIT: cAddrSizeBits = 32; break;
+            case IEMMODE_64BIT: cAddrSizeBits = 64; break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+        rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_IN, cbReg, cAddrSizeBits, 0 /* N/A - iEffSeg */,
+                                           false /* fRep */, false /* fStrIo */, cbInstr);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("iemCImpl_in: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, cbReg,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    /*
+     * Perform the I/O.
+     */
+    uint32_t u32Value = 0;
+    rcStrict = IOMIOPortRead(pVCpu->CTX_SUFF(pVM), pVCpu, u16Port, &u32Value, cbReg);
+    if (IOM_SUCCESS(rcStrict))
+    {
+        switch (cbReg)
+        {
+            case 1: pVCpu->cpum.GstCtx.al  = (uint8_t)u32Value;  break;
+            case 2: pVCpu->cpum.GstCtx.ax  = (uint16_t)u32Value; break;
+            case 4: pVCpu->cpum.GstCtx.rax = u32Value;           break;
+            default: AssertFailedReturn(VERR_IEM_IPE_3);
+        }
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        pVCpu->iem.s.cPotentialExits++;
+        if (rcStrict != VINF_SUCCESS)
+            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+        Assert(rcStrict == VINF_SUCCESS); /* assumed below */
+
+        /*
+         * Check for I/O breakpoints.
+         */
+        uint32_t const uDr7 = pVCpu->cpum.GstCtx.dr[7];
+        if (RT_UNLIKELY(   (   (uDr7 & X86_DR7_ENABLED_MASK)
+                            && X86_DR7_ANY_RW_IO(uDr7)
+                            && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE))
+                        || DBGFBpIsHwIoArmed(pVCpu->CTX_SUFF(pVM))))
+        {
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3 | CPUMCTX_EXTRN_DR6);
+            rcStrict = DBGFBpCheckIo(pVCpu->CTX_SUFF(pVM), pVCpu, IEM_GET_CTX(pVCpu), u16Port, cbReg);
+            if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
+                rcStrict = iemRaiseDebugException(pVCpu);
+        }
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'IN eAX, DX'.
+ *
+ * @param   cbReg       The register size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_in_eAX_DX, uint8_t, cbReg)
+{
+    return IEM_CIMPL_CALL_3(iemCImpl_in, pVCpu->cpum.GstCtx.dx, false /* fImm */, cbReg);
+}
+
+
+/**
+ * Implements 'OUT port, eAX'.
+ *
+ * @param   u16Port     The destination port.
+ * @param   fImm        Whether the port was specified through an immediate operand
+ *                      or the implicit DX register.
+ * @param   cbReg       The register size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_out, uint16_t, u16Port, bool, fImm, uint8_t, cbReg)
+{
+    /*
+     * CPL check
+     */
+    VBOXSTRICTRC rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, cbReg);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Check VMX nested-guest I/O intercept.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        rcStrict = iemVmxVmexitInstrIo(pVCpu, VMXINSTRID_IO_OUT, u16Port, fImm, cbReg, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#else
+    RT_NOREF(fImm);
+#endif
+
+    /*
+     * Check SVM nested-guest I/O intercept.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
+    {
+        uint8_t cAddrSizeBits;
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: cAddrSizeBits = 16; break;
+            case IEMMODE_32BIT: cAddrSizeBits = 32; break;
+            case IEMMODE_64BIT: cAddrSizeBits = 64; break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+        rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_OUT, cbReg, cAddrSizeBits, 0 /* N/A - iEffSeg */,
+                                           false /* fRep */, false /* fStrIo */, cbInstr);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("iemCImpl_out: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, cbReg,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    /*
+     * Perform the I/O.
+     */
+    uint32_t u32Value;
+    switch (cbReg)
+    {
+        case 1: u32Value = pVCpu->cpum.GstCtx.al;  break;
+        case 2: u32Value = pVCpu->cpum.GstCtx.ax;  break;
+        case 4: u32Value = pVCpu->cpum.GstCtx.eax; break;
+        default: AssertFailedReturn(VERR_IEM_IPE_4);
+    }
+    rcStrict = IOMIOPortWrite(pVCpu->CTX_SUFF(pVM), pVCpu, u16Port, u32Value, cbReg);
+    if (IOM_SUCCESS(rcStrict))
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        pVCpu->iem.s.cPotentialExits++;
+        if (rcStrict != VINF_SUCCESS)
+            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+        Assert(rcStrict == VINF_SUCCESS); /* assumed below */
+
+        /*
+         * Check for I/O breakpoints.
+         */
+        uint32_t const uDr7 = pVCpu->cpum.GstCtx.dr[7];
+        if (RT_UNLIKELY(   (   (uDr7 & X86_DR7_ENABLED_MASK)
+                            && X86_DR7_ANY_RW_IO(uDr7)
+                            && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE))
+                        || DBGFBpIsHwIoArmed(pVCpu->CTX_SUFF(pVM))))
+        {
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3 | CPUMCTX_EXTRN_DR6);
+            rcStrict = DBGFBpCheckIo(pVCpu->CTX_SUFF(pVM), pVCpu, IEM_GET_CTX(pVCpu), u16Port, cbReg);
+            if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
+                rcStrict = iemRaiseDebugException(pVCpu);
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'OUT DX, eAX'.
+ *
+ * @param   cbReg       The register size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_out_DX_eAX, uint8_t, cbReg)
+{
+    return IEM_CIMPL_CALL_3(iemCImpl_out, pVCpu->cpum.GstCtx.dx, false /* fImm */, cbReg);
+}
+
+
+/**
+ * Implements 'CLI'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_cli)
+{
+    uint32_t        fEfl    = IEMMISC_GET_EFL(pVCpu);
+    uint32_t const  fEflOld = fEfl;
+
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4);
+    if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
+    {
+        uint8_t const uIopl = X86_EFL_GET_IOPL(fEfl);
+        if (!(fEfl & X86_EFL_VM))
+        {
+            if (pVCpu->iem.s.uCpl <= uIopl)
+                fEfl &= ~X86_EFL_IF;
+            else if (   pVCpu->iem.s.uCpl == 3
+                     && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PVI) )
+                fEfl &= ~X86_EFL_VIF;
+            else
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+        /* V8086 */
+        else if (uIopl == 3)
+            fEfl &= ~X86_EFL_IF;
+        else if (   uIopl < 3
+                 && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME) )
+            fEfl &= ~X86_EFL_VIF;
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    /* real mode */
+    else
+        fEfl &= ~X86_EFL_IF;
+
+    /* Commit. */
+    IEMMISC_SET_EFL(pVCpu, fEfl);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    Log2(("CLI: %#x -> %#x\n", fEflOld, fEfl)); NOREF(fEflOld);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'STI'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_sti)
+{
+    uint32_t        fEfl    = IEMMISC_GET_EFL(pVCpu);
+    uint32_t const  fEflOld = fEfl;
+
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4);
+    if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
+    {
+        uint8_t const uIopl = X86_EFL_GET_IOPL(fEfl);
+        if (!(fEfl & X86_EFL_VM))
+        {
+            if (pVCpu->iem.s.uCpl <= uIopl)
+                fEfl |= X86_EFL_IF;
+            else if (   pVCpu->iem.s.uCpl == 3
+                     && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PVI)
+                     && !(fEfl & X86_EFL_VIP) )
+                fEfl |= X86_EFL_VIF;
+            else
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+        /* V8086 */
+        else if (uIopl == 3)
+            fEfl |= X86_EFL_IF;
+        else if (   uIopl < 3
+                 && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME)
+                 && !(fEfl & X86_EFL_VIP) )
+            fEfl |= X86_EFL_VIF;
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    /* real mode */
+    else
+        fEfl |= X86_EFL_IF;
+
+    /* Commit. */
+    IEMMISC_SET_EFL(pVCpu, fEfl);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    if (!(fEflOld & X86_EFL_IF) && (fEfl & X86_EFL_IF))
+        EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
+    Log2(("STI: %#x -> %#x\n", fEflOld, fEfl));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'HLT'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_hlt)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_HLT_EXIT))
+    {
+        Log2(("hlt: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_HLT, cbInstr);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_HLT))
+    {
+        Log2(("hlt: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_HLT, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_EM_HALT;
+}
+
+
+/**
+ * Implements 'MONITOR'.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_monitor, uint8_t, iEffSeg)
+{
+    /*
+     * Permission checks.
+     */
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log2(("monitor: CPL != 0\n"));
+        return iemRaiseUndefinedOpcode(pVCpu); /** @todo MSR[0xC0010015].MonMwaitUserEn if we care. */
+    }
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMonitorMWait)
+    {
+        Log2(("monitor: Not in CPUID\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+    /*
+     * Check VMX guest-intercept.
+     * This should be considered a fault-like VM-exit.
+     * See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
+     */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_MONITOR_EXIT))
+    {
+        Log2(("monitor: Guest intercept -> #VMEXIT\n"));
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_MONITOR, cbInstr);
+    }
+
+    /*
+     * Gather the operands and validate them.
+     */
+    RTGCPTR  GCPtrMem   = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax;
+    uint32_t uEcx       = pVCpu->cpum.GstCtx.ecx;
+    uint32_t uEdx       = pVCpu->cpum.GstCtx.edx;
+/** @todo Test whether EAX or ECX is processed first, i.e. do we get \#PF or
+ *        \#GP first. */
+    if (uEcx != 0)
+    {
+        Log2(("monitor rax=%RX64, ecx=%RX32, edx=%RX32; ECX != 0 -> #GP(0)\n", GCPtrMem, uEcx, uEdx)); NOREF(uEdx);
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA, iEffSeg, 1, &GCPtrMem);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    RTGCPHYS GCPhysMem;
+    rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA, &GCPhysMem);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
+    {
+        /*
+         * MONITOR does not access the memory, just monitors the address. However,
+         * if the address falls in the APIC-access page, the address monitored must
+         * instead be the corresponding address in the virtual-APIC page.
+         *
+         * See Intel spec. 29.4.4 "Instruction-Specific Considerations".
+         */
+        rcStrict = iemVmxVirtApicAccessUnused(pVCpu, &GCPhysMem);
+        if (   rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE
+            && rcStrict != VINF_VMX_MODIFIES_BEHAVIOR)
+                return rcStrict;
+    }
+#endif
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MONITOR))
+    {
+        Log2(("monitor: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MONITOR, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Call EM to prepare the monitor/wait.
+     */
+    rcStrict = EMMonitorWaitPrepare(pVCpu, pVCpu->cpum.GstCtx.rax, pVCpu->cpum.GstCtx.rcx, pVCpu->cpum.GstCtx.rdx, GCPhysMem);
+    Assert(rcStrict == VINF_SUCCESS);
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'MWAIT'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_mwait)
+{
+    /*
+     * Permission checks.
+     */
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log2(("mwait: CPL != 0\n"));
+        /** @todo MSR[0xC0010015].MonMwaitUserEn if we care. (Remember to check
+         *        EFLAGS.VM then.) */
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMonitorMWait)
+    {
+        Log2(("mwait: Not in CPUID\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+    /* Check VMX nested-guest intercept. */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_MWAIT_EXIT))
+        IEM_VMX_VMEXIT_MWAIT_RET(pVCpu, EMMonitorIsArmed(pVCpu), cbInstr);
+
+    /*
+     * Gather the operands and validate them.
+     */
+    uint32_t const uEax = pVCpu->cpum.GstCtx.eax;
+    uint32_t const uEcx = pVCpu->cpum.GstCtx.ecx;
+    if (uEcx != 0)
+    {
+        /* Only supported extension is break on IRQ when IF=0. */
+        if (uEcx > 1)
+        {
+            Log2(("mwait eax=%RX32, ecx=%RX32; ECX > 1 -> #GP(0)\n", uEax, uEcx));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+        uint32_t fMWaitFeatures = 0;
+        uint32_t uIgnore = 0;
+        CPUMGetGuestCpuId(pVCpu, 5, 0, &uIgnore, &uIgnore, &fMWaitFeatures, &uIgnore);
+        if (    (fMWaitFeatures & (X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0))
+            !=                    (X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0))
+        {
+            Log2(("mwait eax=%RX32, ecx=%RX32; break-on-IRQ-IF=0 extension not enabled -> #GP(0)\n", uEax, uEcx));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        /*
+         * If the interrupt-window exiting control is set or a virtual-interrupt is pending
+         * for delivery; and interrupts are disabled the processor does not enter its
+         * mwait state but rather passes control to the next instruction.
+         *
+         * See Intel spec. 25.3 "Changes to Instruction Behavior In VMX Non-root Operation".
+         */
+        if (    IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+            && !pVCpu->cpum.GstCtx.eflags.Bits.u1IF)
+        {
+            if (   IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INT_WINDOW_EXIT)
+                || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
+            {
+                iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+                return VINF_SUCCESS;
+            }
+        }
+#endif
+    }
+
+    /*
+     * Check SVM nested-guest mwait intercepts.
+     */
+    if (   IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MWAIT_ARMED)
+        && EMMonitorIsArmed(pVCpu))
+    {
+        Log2(("mwait: Guest intercept (monitor hardware armed) -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MWAIT_ARMED, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MWAIT))
+    {
+        Log2(("mwait: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MWAIT, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Call EM to prepare the monitor/wait.
+     */
+    VBOXSTRICTRC rcStrict = EMMonitorWaitPerform(pVCpu, uEax, uEcx);
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'SWAPGS'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_swapgs)
+{
+    Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT); /* Caller checks this. */
+
+    /*
+     * Permission checks.
+     */
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log2(("swapgs: CPL != 0\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+    /*
+     * Do the job.
+     */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_GS);
+    uint64_t uOtherGsBase = pVCpu->cpum.GstCtx.msrKERNELGSBASE;
+    pVCpu->cpum.GstCtx.msrKERNELGSBASE = pVCpu->cpum.GstCtx.gs.u64Base;
+    pVCpu->cpum.GstCtx.gs.u64Base = uOtherGsBase;
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'CPUID'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_cpuid)
+{
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        Log2(("cpuid: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_CPUID, cbInstr);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CPUID))
+    {
+        Log2(("cpuid: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_CPUID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    CPUMGetGuestCpuId(pVCpu, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx,
+                      &pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
+    pVCpu->cpum.GstCtx.rax &= UINT32_C(0xffffffff);
+    pVCpu->cpum.GstCtx.rbx &= UINT32_C(0xffffffff);
+    pVCpu->cpum.GstCtx.rcx &= UINT32_C(0xffffffff);
+    pVCpu->cpum.GstCtx.rdx &= UINT32_C(0xffffffff);
+    pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX);
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    pVCpu->iem.s.cPotentialExits++;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'AAD'.
+ *
+ * @param   bImm            The immediate operand.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_aad, uint8_t, bImm)
+{
+    uint16_t const ax = pVCpu->cpum.GstCtx.ax;
+    uint8_t const  al = (uint8_t)ax + (uint8_t)(ax >> 8) * bImm;
+    pVCpu->cpum.GstCtx.ax = al;
+    iemHlpUpdateArithEFlagsU8(pVCpu, al,
+                              X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF,
+                              X86_EFL_OF | X86_EFL_AF | X86_EFL_CF);
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'AAM'.
+ *
+ * @param   bImm            The immediate operand. Cannot be 0.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_aam, uint8_t, bImm)
+{
+    Assert(bImm != 0); /* #DE on 0 is handled in the decoder. */
+
+    uint16_t const ax = pVCpu->cpum.GstCtx.ax;
+    uint8_t const  al = (uint8_t)ax % bImm;
+    uint8_t const  ah = (uint8_t)ax / bImm;
+    pVCpu->cpum.GstCtx.ax = (ah << 8) + al;
+    iemHlpUpdateArithEFlagsU8(pVCpu, al,
+                              X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF,
+                              X86_EFL_OF | X86_EFL_AF | X86_EFL_CF);
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'DAA'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_daa)
+{
+    uint8_t const  al       = pVCpu->cpum.GstCtx.al;
+    bool const     fCarry   = pVCpu->cpum.GstCtx.eflags.Bits.u1CF;
+
+    if (   pVCpu->cpum.GstCtx.eflags.Bits.u1AF
+        || (al & 0xf) >= 10)
+    {
+        pVCpu->cpum.GstCtx.al = al + 6;
+        pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
+    }
+    else
+        pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
+
+    if (al >= 0x9a || fCarry)
+    {
+        pVCpu->cpum.GstCtx.al += 0x60;
+        pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+    }
+    else
+        pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
+
+    iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'DAS'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_das)
+{
+    uint8_t const  uInputAL = pVCpu->cpum.GstCtx.al;
+    bool const     fCarry   = pVCpu->cpum.GstCtx.eflags.Bits.u1CF;
+
+    if (   pVCpu->cpum.GstCtx.eflags.Bits.u1AF
+        || (uInputAL & 0xf) >= 10)
+    {
+        pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
+        if (uInputAL < 6)
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+        pVCpu->cpum.GstCtx.al = uInputAL - 6;
+    }
+    else
+    {
+        pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
+        pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
+    }
+
+    if (uInputAL >= 0x9a || fCarry)
+    {
+        pVCpu->cpum.GstCtx.al -= 0x60;
+        pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+    }
+
+    iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'AAA'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_aaa)
+{
+    if (IEM_IS_GUEST_CPU_AMD(pVCpu))
+    {
+        if (   pVCpu->cpum.GstCtx.eflags.Bits.u1AF
+            || (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
+        {
+            iemAImpl_add_u16(&pVCpu->cpum.GstCtx.ax, 0x106, &pVCpu->cpum.GstCtx.eflags.u32);
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+        }
+        else
+        {
+            iemHlpUpdateArithEFlagsU16(pVCpu, pVCpu->cpum.GstCtx.ax, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF);
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
+        }
+        pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
+    }
+    else
+    {
+        if (   pVCpu->cpum.GstCtx.eflags.Bits.u1AF
+            || (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
+        {
+            pVCpu->cpum.GstCtx.ax += UINT16_C(0x106);
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+        }
+        else
+        {
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
+        }
+        pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
+        iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF);
+    }
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'AAS'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_aas)
+{
+    if (IEM_IS_GUEST_CPU_AMD(pVCpu))
+    {
+        if (   pVCpu->cpum.GstCtx.eflags.Bits.u1AF
+            || (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
+        {
+            iemAImpl_sub_u16(&pVCpu->cpum.GstCtx.ax, 0x106, &pVCpu->cpum.GstCtx.eflags.u32);
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+        }
+        else
+        {
+            iemHlpUpdateArithEFlagsU16(pVCpu, pVCpu->cpum.GstCtx.ax, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF);
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
+        }
+        pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
+    }
+    else
+    {
+        if (   pVCpu->cpum.GstCtx.eflags.Bits.u1AF
+            || (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
+        {
+            pVCpu->cpum.GstCtx.ax -= UINT16_C(0x106);
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+        }
+        else
+        {
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
+        }
+        pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
+        iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF);
+    }
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements the 16-bit version of 'BOUND'.
+ *
+ * @note    We have separate 16-bit and 32-bit variants of this function due to
+ *          the decoder using unsigned parameters, whereas we want signed one to
+ *          do the job.  This is significant for a recompiler.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_bound_16, int16_t, idxArray, int16_t, idxLowerBound, int16_t, idxUpperBound)
+{
+    /*
+     * Check if the index is inside the bounds, otherwise raise #BR.
+     */
+    if (   idxArray >= idxLowerBound
+        && idxArray <= idxUpperBound)
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    return iemRaiseBoundRangeExceeded(pVCpu);
+}
+
+
+/**
+ * Implements the 32-bit version of 'BOUND'.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_bound_32, int32_t, idxArray, int32_t, idxLowerBound, int32_t, idxUpperBound)
+{
+    /*
+     * Check if the index is inside the bounds, otherwise raise #BR.
+     */
+    if (   idxArray >= idxLowerBound
+        && idxArray <= idxUpperBound)
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    return iemRaiseBoundRangeExceeded(pVCpu);
+}
+
+
+
+/*
+ * Instantiate the various string operation combinations.
+ */
+#define OP_SIZE     8
+#define ADDR_SIZE   16
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     8
+#define ADDR_SIZE   32
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     8
+#define ADDR_SIZE   64
+#include "IEMAllCImplStrInstr.cpp.h"
+
+#define OP_SIZE     16
+#define ADDR_SIZE   16
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     16
+#define ADDR_SIZE   32
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     16
+#define ADDR_SIZE   64
+#include "IEMAllCImplStrInstr.cpp.h"
+
+#define OP_SIZE     32
+#define ADDR_SIZE   16
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     32
+#define ADDR_SIZE   32
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     32
+#define ADDR_SIZE   64
+#include "IEMAllCImplStrInstr.cpp.h"
+
+#define OP_SIZE     64
+#define ADDR_SIZE   32
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     64
+#define ADDR_SIZE   64
+#include "IEMAllCImplStrInstr.cpp.h"
+
+
+/**
+ * Implements 'XGETBV'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_xgetbv)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+    if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)
+    {
+        uint32_t uEcx = pVCpu->cpum.GstCtx.ecx;
+        switch (uEcx)
+        {
+            case 0:
+                break;
+
+            case 1: /** @todo Implement XCR1 support. */
+            default:
+                Log(("xgetbv ecx=%RX32 -> #GP(0)\n", uEcx));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+
+        }
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_XCRx);
+        pVCpu->cpum.GstCtx.rax = RT_LO_U32(pVCpu->cpum.GstCtx.aXcr[uEcx]);
+        pVCpu->cpum.GstCtx.rdx = RT_HI_U32(pVCpu->cpum.GstCtx.aXcr[uEcx]);
+
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+    Log(("xgetbv CR4.OSXSAVE=0 -> UD\n"));
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+
+/**
+ * Implements 'XSETBV'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_xsetbv)
+{
+    if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)
+    {
+        if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_XSETBV))
+        {
+            Log2(("xsetbv: Guest intercept -> #VMEXIT\n"));
+            IEM_SVM_UPDATE_NRIP(pVCpu);
+            IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XSETBV, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        if (pVCpu->iem.s.uCpl == 0)
+        {
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_XCRx);
+
+            if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+                IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_XSETBV, cbInstr);
+
+            uint32_t uEcx = pVCpu->cpum.GstCtx.ecx;
+            uint64_t uNewValue = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx);
+            switch (uEcx)
+            {
+                case 0:
+                {
+                    int rc = CPUMSetGuestXcr0(pVCpu, uNewValue);
+                    if (rc == VINF_SUCCESS)
+                        break;
+                    Assert(rc == VERR_CPUM_RAISE_GP_0);
+                    Log(("xsetbv ecx=%RX32 (newvalue=%RX64) -> #GP(0)\n", uEcx, uNewValue));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+
+                case 1: /** @todo Implement XCR1 support. */
+                default:
+                    Log(("xsetbv ecx=%RX32 (newvalue=%RX64) -> #GP(0)\n", uEcx, uNewValue));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+
+            }
+
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            return VINF_SUCCESS;
+        }
+
+        Log(("xsetbv cpl=%u -> GP(0)\n", pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    Log(("xsetbv CR4.OSXSAVE=0 -> UD\n"));
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+#ifdef IN_RING3
+
+/** Argument package for iemCImpl_cmpxchg16b_fallback_rendezvous_callback. */
+struct IEMCIMPLCX16ARGS
+{
+    PRTUINT128U     pu128Dst;
+    PRTUINT128U     pu128RaxRdx;
+    PRTUINT128U     pu128RbxRcx;
+    uint32_t       *pEFlags;
+# ifdef VBOX_STRICT
+    uint32_t        cCalls;
+# endif
+};
+
+/**
+ * @callback_method_impl{FNVMMEMTRENDEZVOUS,
+ *                       Worker for iemCImpl_cmpxchg16b_fallback_rendezvous}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) iemCImpl_cmpxchg16b_fallback_rendezvous_callback(PVM pVM, PVMCPUCC pVCpu, void *pvUser)
+{
+    RT_NOREF(pVM, pVCpu);
+    struct IEMCIMPLCX16ARGS *pArgs = (struct IEMCIMPLCX16ARGS *)pvUser;
+# ifdef VBOX_STRICT
+    Assert(pArgs->cCalls == 0);
+    pArgs->cCalls++;
+# endif
+
+    iemAImpl_cmpxchg16b_fallback(pArgs->pu128Dst, pArgs->pu128RaxRdx, pArgs->pu128RbxRcx, pArgs->pEFlags);
+    return VINF_SUCCESS;
+}
+
+#endif /* IN_RING3 */
+
+/**
+ * Implements 'CMPXCHG16B' fallback using rendezvous.
+ */
+IEM_CIMPL_DEF_4(iemCImpl_cmpxchg16b_fallback_rendezvous, PRTUINT128U, pu128Dst, PRTUINT128U, pu128RaxRdx,
+                PRTUINT128U, pu128RbxRcx, uint32_t *, pEFlags)
+{
+#ifdef IN_RING3
+    struct IEMCIMPLCX16ARGS Args;
+    Args.pu128Dst       = pu128Dst;
+    Args.pu128RaxRdx    = pu128RaxRdx;
+    Args.pu128RbxRcx    = pu128RbxRcx;
+    Args.pEFlags        = pEFlags;
+# ifdef VBOX_STRICT
+    Args.cCalls         = 0;
+# endif
+    VBOXSTRICTRC rcStrict = VMMR3EmtRendezvous(pVCpu->CTX_SUFF(pVM), VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE,
+                                               iemCImpl_cmpxchg16b_fallback_rendezvous_callback, &Args);
+    Assert(Args.cCalls == 1);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /* Duplicated tail code. */
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_RW);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.eflags.u = *pEFlags; /* IEM_MC_COMMIT_EFLAGS */
+            if (!(*pEFlags & X86_EFL_ZF))
+            {
+                pVCpu->cpum.GstCtx.rax = pu128RaxRdx->s.Lo;
+                pVCpu->cpum.GstCtx.rdx = pu128RaxRdx->s.Hi;
+            }
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        }
+    }
+    return rcStrict;
+#else
+    RT_NOREF(pVCpu, cbInstr, pu128Dst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+    return VERR_IEM_ASPECT_NOT_IMPLEMENTED; /* This should get us to ring-3 for now.  Should perhaps be replaced later. */
+#endif
+}
+
+
+/**
+ * Implements 'CLFLUSH' and 'CLFLUSHOPT'.
+ *
+ * This is implemented in C because it triggers a load like behaviour without
+ * actually reading anything.  Since that's not so common, it's implemented
+ * here.
+ *
+ * @param   iEffSeg         The effective segment.
+ * @param   GCPtrEff        The address of the image.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_clflush_clflushopt, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
+{
+    /*
+     * Pretend to do a load w/o reading (see also iemCImpl_monitor and iemMemMap).
+     */
+    VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA, iEffSeg, 1, &GCPtrEff);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        RTGCPHYS GCPhysMem;
+        rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrEff, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA, &GCPhysMem);
+        if (rcStrict == VINF_SUCCESS)
+        {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+                && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
+            {
+                /*
+                 * CLFLUSH/CLFLUSHOPT does not access the memory, but flushes the cache-line
+                 * that contains the address. However, if the address falls in the APIC-access
+                 * page, the address flushed must instead be the corresponding address in the
+                 * virtual-APIC page.
+                 *
+                 * See Intel spec. 29.4.4 "Instruction-Specific Considerations".
+                 */
+                rcStrict = iemVmxVirtApicAccessUnused(pVCpu, &GCPhysMem);
+                if (   rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE
+                    && rcStrict != VINF_VMX_MODIFIES_BEHAVIOR)
+                        return rcStrict;
+            }
+#endif
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            return VINF_SUCCESS;
+        }
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'FINIT' and 'FNINIT'.
+ *
+ * @param   fCheckXcpts     Whether to check for umasked pending exceptions or
+ *                          not.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_finit, bool, fCheckXcpts)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+    if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS))
+        return iemRaiseDeviceNotAvailable(pVCpu);
+
+    iemFpuActualizeStateForChange(pVCpu);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_X87);
+
+    NOREF(fCheckXcpts); /** @todo trigger pending exceptions:
+        if (fCheckXcpts && TODO )
+        return iemRaiseMathFault(pVCpu);
+     */
+
+    PX86XSAVEAREA pXState = pVCpu->cpum.GstCtx.CTX_SUFF(pXState);
+    pXState->x87.FCW   = 0x37f;
+    pXState->x87.FSW   = 0;
+    pXState->x87.FTW   = 0x00;         /* 0 - empty. */
+    pXState->x87.FPUDP = 0;
+    pXState->x87.DS    = 0; //??
+    pXState->x87.Rsrvd2= 0;
+    pXState->x87.FPUIP = 0;
+    pXState->x87.CS    = 0; //??
+    pXState->x87.Rsrvd1= 0;
+    pXState->x87.FOP   = 0;
+
+    iemHlpUsedFpu(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'FXSAVE'.
+ *
+ * @param   iEffSeg         The effective segment.
+ * @param   GCPtrEff        The address of the image.
+ * @param   enmEffOpSize    The operand size (only REX.W really matters).
+ */
+IEM_CIMPL_DEF_3(iemCImpl_fxsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX);
+
+    /*
+     * Raise exceptions.
+     */
+    if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)
+        return iemRaiseUndefinedOpcode(pVCpu);
+    if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_TS | X86_CR0_EM))
+        return iemRaiseDeviceNotAvailable(pVCpu);
+    if (GCPtrEff & 15)
+    {
+        /** @todo CPU/VM detection possible! \#AC might not be signal for
+         * all/any misalignment sizes, intel says its an implementation detail. */
+        if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
+            && pVCpu->cpum.GstCtx.eflags.Bits.u1AC
+            && pVCpu->iem.s.uCpl == 3)
+            return iemRaiseAlignmentCheckException(pVCpu);
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Access the memory.
+     */
+    void *pvMem512;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    PX86FXSTATE  pDst = (PX86FXSTATE)pvMem512;
+    PCX86FXSTATE pSrc = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+
+    /*
+     * Store the registers.
+     */
+    /** @todo CPU/VM detection possible! If CR4.OSFXSR=0 MXCSR it's
+     * implementation specific whether MXCSR and XMM0-XMM7 are saved. */
+
+    /* common for all formats */
+    pDst->FCW           = pSrc->FCW;
+    pDst->FSW           = pSrc->FSW;
+    pDst->FTW           = pSrc->FTW & UINT16_C(0xff);
+    pDst->FOP           = pSrc->FOP;
+    pDst->MXCSR         = pSrc->MXCSR;
+    pDst->MXCSR_MASK    = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
+    for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++)
+    {
+        /** @todo Testcase: What actually happens to the 6 reserved bytes? I'm clearing
+         *        them for now... */
+        pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
+        pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
+        pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
+        pDst->aRegs[i].au32[3] = 0;
+    }
+
+    /* FPU IP, CS, DP and DS. */
+    pDst->FPUIP  = pSrc->FPUIP;
+    pDst->CS     = pSrc->CS;
+    pDst->FPUDP  = pSrc->FPUDP;
+    pDst->DS     = pSrc->DS;
+    if (enmEffOpSize == IEMMODE_64BIT)
+    {
+        /* Save upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */
+        pDst->Rsrvd1 = pSrc->Rsrvd1;
+        pDst->Rsrvd2 = pSrc->Rsrvd2;
+        pDst->au32RsrvdForSoftware[0] = 0;
+    }
+    else
+    {
+        pDst->Rsrvd1 = 0;
+        pDst->Rsrvd2 = 0;
+        pDst->au32RsrvdForSoftware[0] = X86_FXSTATE_RSVD_32BIT_MAGIC;
+    }
+
+    /* XMM registers. */
+    if (   !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_FFXSR)
+        || pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
+        || pVCpu->iem.s.uCpl != 0)
+    {
+        uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
+        for (uint32_t i = 0; i < cXmmRegs; i++)
+            pDst->aXMM[i] = pSrc->aXMM[i];
+        /** @todo Testcase: What happens to the reserved XMM registers? Untouched,
+         *        right? */
+    }
+
+    /*
+     * Commit the memory.
+     */
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'FXRSTOR'.
+ *
+ * @param   GCPtrEff        The address of the image.
+ * @param   enmEffOpSize    The operand size (only REX.W really matters).
+ */
+IEM_CIMPL_DEF_3(iemCImpl_fxrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX);
+
+    /*
+     * Raise exceptions.
+     */
+    if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)
+        return iemRaiseUndefinedOpcode(pVCpu);
+    if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_TS | X86_CR0_EM))
+        return iemRaiseDeviceNotAvailable(pVCpu);
+    if (GCPtrEff & 15)
+    {
+        /** @todo CPU/VM detection possible! \#AC might not be signal for
+         * all/any misalignment sizes, intel says its an implementation detail. */
+        if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
+            && pVCpu->cpum.GstCtx.eflags.Bits.u1AC
+            && pVCpu->iem.s.uCpl == 3)
+            return iemRaiseAlignmentCheckException(pVCpu);
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Access the memory.
+     */
+    void *pvMem512;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    PCX86FXSTATE pSrc = (PCX86FXSTATE)pvMem512;
+    PX86FXSTATE  pDst = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+
+    /*
+     * Check the state for stuff which will #GP(0).
+     */
+    uint32_t const fMXCSR      = pSrc->MXCSR;
+    uint32_t const fMXCSR_MASK = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
+    if (fMXCSR & ~fMXCSR_MASK)
+    {
+        Log(("fxrstor: MXCSR=%#x (MXCSR_MASK=%#x) -> #GP(0)\n", fMXCSR, fMXCSR_MASK));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Load the registers.
+     */
+    /** @todo CPU/VM detection possible! If CR4.OSFXSR=0 MXCSR it's
+     * implementation specific whether MXCSR and XMM0-XMM7 are restored. */
+
+    /* common for all formats */
+    pDst->FCW       = pSrc->FCW;
+    pDst->FSW       = pSrc->FSW;
+    pDst->FTW       = pSrc->FTW & UINT16_C(0xff);
+    pDst->FOP       = pSrc->FOP;
+    pDst->MXCSR     = fMXCSR;
+    /* (MXCSR_MASK is read-only) */
+    for (uint32_t i = 0; i < RT_ELEMENTS(pSrc->aRegs); i++)
+    {
+        pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
+        pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
+        pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
+        pDst->aRegs[i].au32[3] = 0;
+    }
+
+    /* FPU IP, CS, DP and DS. */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        pDst->FPUIP  = pSrc->FPUIP;
+        pDst->CS     = pSrc->CS;
+        pDst->Rsrvd1 = pSrc->Rsrvd1;
+        pDst->FPUDP  = pSrc->FPUDP;
+        pDst->DS     = pSrc->DS;
+        pDst->Rsrvd2 = pSrc->Rsrvd2;
+    }
+    else
+    {
+        pDst->FPUIP  = pSrc->FPUIP;
+        pDst->CS     = pSrc->CS;
+        pDst->Rsrvd1 = 0;
+        pDst->FPUDP  = pSrc->FPUDP;
+        pDst->DS     = pSrc->DS;
+        pDst->Rsrvd2 = 0;
+    }
+
+    /* XMM registers. */
+    if (   !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_FFXSR)
+        || pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
+        || pVCpu->iem.s.uCpl != 0)
+    {
+        uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
+        for (uint32_t i = 0; i < cXmmRegs; i++)
+            pDst->aXMM[i] = pSrc->aXMM[i];
+    }
+
+    /*
+     * Commit the memory.
+     */
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemHlpUsedFpu(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'XSAVE'.
+ *
+ * @param   iEffSeg         The effective segment.
+ * @param   GCPtrEff        The address of the image.
+ * @param   enmEffOpSize    The operand size (only REX.W really matters).
+ */
+IEM_CIMPL_DEF_3(iemCImpl_xsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
+
+    /*
+     * Raise exceptions.
+     */
+    if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
+        return iemRaiseUndefinedOpcode(pVCpu);
+    /* When in VMX non-root mode and XSAVE/XRSTOR is not enabled, it results in #UD. */
+    if (    IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_XSAVES_XRSTORS))
+    {
+        Log(("xrstor: Not enabled for nested-guest execution -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)
+        return iemRaiseDeviceNotAvailable(pVCpu);
+    if (GCPtrEff & 63)
+    {
+        /** @todo CPU/VM detection possible! \#AC might not be signal for
+         * all/any misalignment sizes, intel says its an implementation detail. */
+        if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
+            && pVCpu->cpum.GstCtx.eflags.Bits.u1AC
+            && pVCpu->iem.s.uCpl == 3)
+            return iemRaiseAlignmentCheckException(pVCpu);
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Calc the requested mask.
+     */
+    uint64_t const fReqComponents = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx) & pVCpu->cpum.GstCtx.aXcr[0];
+    AssertLogRelReturn(!(fReqComponents & ~(XSAVE_C_X87 | XSAVE_C_SSE | XSAVE_C_YMM)), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
+    uint64_t const fXInUse        = pVCpu->cpum.GstCtx.aXcr[0];
+
+/** @todo figure out the exact protocol for the memory access.  Currently we
+ *        just need this crap to work halfways to make it possible to test
+ *        AVX instructions. */
+/** @todo figure out the XINUSE and XMODIFIED   */
+
+    /*
+     * Access the x87 memory state.
+     */
+    /* The x87+SSE state.  */
+    void *pvMem512;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    PX86FXSTATE  pDst = (PX86FXSTATE)pvMem512;
+    PCX86FXSTATE pSrc = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+
+    /* The header.  */
+    PX86XSAVEHDR pHdr;
+    rcStrict = iemMemMap(pVCpu, (void **)&pHdr, sizeof(&pHdr), iEffSeg, GCPtrEff + 512, IEM_ACCESS_DATA_RW);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Store the X87 state.
+     */
+    if (fReqComponents & XSAVE_C_X87)
+    {
+        /* common for all formats */
+        pDst->FCW    = pSrc->FCW;
+        pDst->FSW    = pSrc->FSW;
+        pDst->FTW    = pSrc->FTW & UINT16_C(0xff);
+        pDst->FOP    = pSrc->FOP;
+        pDst->FPUIP  = pSrc->FPUIP;
+        pDst->CS     = pSrc->CS;
+        pDst->FPUDP  = pSrc->FPUDP;
+        pDst->DS     = pSrc->DS;
+        if (enmEffOpSize == IEMMODE_64BIT)
+        {
+            /* Save upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */
+            pDst->Rsrvd1 = pSrc->Rsrvd1;
+            pDst->Rsrvd2 = pSrc->Rsrvd2;
+            pDst->au32RsrvdForSoftware[0] = 0;
+        }
+        else
+        {
+            pDst->Rsrvd1 = 0;
+            pDst->Rsrvd2 = 0;
+            pDst->au32RsrvdForSoftware[0] = X86_FXSTATE_RSVD_32BIT_MAGIC;
+        }
+        for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++)
+        {
+            /** @todo Testcase: What actually happens to the 6 reserved bytes? I'm clearing
+             *        them for now... */
+            pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
+            pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
+            pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
+            pDst->aRegs[i].au32[3] = 0;
+        }
+
+    }
+
+    if (fReqComponents & (XSAVE_C_SSE | XSAVE_C_YMM))
+    {
+        pDst->MXCSR         = pSrc->MXCSR;
+        pDst->MXCSR_MASK    = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
+    }
+
+    if (fReqComponents & XSAVE_C_SSE)
+    {
+        /* XMM registers. */
+        uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
+        for (uint32_t i = 0; i < cXmmRegs; i++)
+            pDst->aXMM[i] = pSrc->aXMM[i];
+        /** @todo Testcase: What happens to the reserved XMM registers? Untouched,
+         *        right? */
+    }
+
+    /* Commit the x87 state bits. (probably wrong) */
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Store AVX state.
+     */
+    if (fReqComponents & XSAVE_C_YMM)
+    {
+        /** @todo testcase: xsave64 vs xsave32 wrt XSAVE_C_YMM. */
+        AssertLogRelReturn(pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT] != UINT16_MAX, VERR_IEM_IPE_9);
+        PCX86XSAVEYMMHI pCompSrc = CPUMCTX_XSAVE_C_PTR(IEM_GET_CTX(pVCpu), XSAVE_C_YMM_BIT, PCX86XSAVEYMMHI);
+        PX86XSAVEYMMHI  pCompDst;
+        rcStrict = iemMemMap(pVCpu, (void **)&pCompDst, sizeof(*pCompDst), iEffSeg, GCPtrEff + pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT],
+                             IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
+        for (uint32_t i = 0; i < cXmmRegs; i++)
+            pCompDst->aYmmHi[i] = pCompSrc->aYmmHi[i];
+
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pCompDst, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * Update the header.
+     */
+    pHdr->bmXState = (pHdr->bmXState & ~fReqComponents)
+                   | (fReqComponents & fXInUse);
+
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pHdr, IEM_ACCESS_DATA_RW);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'XRSTOR'.
+ *
+ * @param   iEffSeg         The effective segment.
+ * @param   GCPtrEff        The address of the image.
+ * @param   enmEffOpSize    The operand size (only REX.W really matters).
+ */
+IEM_CIMPL_DEF_3(iemCImpl_xrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
+
+    /*
+     * Raise exceptions.
+     */
+    if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
+        return iemRaiseUndefinedOpcode(pVCpu);
+    /* When in VMX non-root mode and XSAVE/XRSTOR is not enabled, it results in #UD. */
+    if (    IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_XSAVES_XRSTORS))
+    {
+        Log(("xrstor: Not enabled for nested-guest execution -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)
+        return iemRaiseDeviceNotAvailable(pVCpu);
+    if (GCPtrEff & 63)
+    {
+        /** @todo CPU/VM detection possible! \#AC might not be signal for
+         * all/any misalignment sizes, intel says its an implementation detail. */
+        if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
+            && pVCpu->cpum.GstCtx.eflags.Bits.u1AC
+            && pVCpu->iem.s.uCpl == 3)
+            return iemRaiseAlignmentCheckException(pVCpu);
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+/** @todo figure out the exact protocol for the memory access.  Currently we
+ *        just need this crap to work halfways to make it possible to test
+ *        AVX instructions. */
+/** @todo figure out the XINUSE and XMODIFIED   */
+
+    /*
+     * Access the x87 memory state.
+     */
+    /* The x87+SSE state.  */
+    void *pvMem512;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    PCX86FXSTATE pSrc = (PCX86FXSTATE)pvMem512;
+    PX86FXSTATE  pDst = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+
+    /*
+     * Calc the requested mask
+     */
+    PX86XSAVEHDR  pHdrDst = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->Hdr;
+    PCX86XSAVEHDR pHdrSrc;
+    rcStrict = iemMemMap(pVCpu, (void **)&pHdrSrc, sizeof(&pHdrSrc), iEffSeg, GCPtrEff + 512, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    uint64_t const fReqComponents = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx) & pVCpu->cpum.GstCtx.aXcr[0];
+    AssertLogRelReturn(!(fReqComponents & ~(XSAVE_C_X87 | XSAVE_C_SSE | XSAVE_C_YMM)), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
+    //uint64_t const fXInUse        = pVCpu->cpum.GstCtx.aXcr[0];
+    uint64_t const fRstorMask     = pHdrSrc->bmXState;
+    uint64_t const fCompMask      = pHdrSrc->bmXComp;
+
+    AssertLogRelReturn(!(fCompMask & XSAVE_C_X), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
+
+    uint32_t const cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
+
+    /* We won't need this any longer. */
+    rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pHdrSrc, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Store the X87 state.
+     */
+    if (fReqComponents & XSAVE_C_X87)
+    {
+        if (fRstorMask & XSAVE_C_X87)
+        {
+            pDst->FCW    = pSrc->FCW;
+            pDst->FSW    = pSrc->FSW;
+            pDst->FTW    = pSrc->FTW & UINT16_C(0xff);
+            pDst->FOP    = pSrc->FOP;
+            pDst->FPUIP  = pSrc->FPUIP;
+            pDst->CS     = pSrc->CS;
+            pDst->FPUDP  = pSrc->FPUDP;
+            pDst->DS     = pSrc->DS;
+            if (enmEffOpSize == IEMMODE_64BIT)
+            {
+                /* Save upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */
+                pDst->Rsrvd1 = pSrc->Rsrvd1;
+                pDst->Rsrvd2 = pSrc->Rsrvd2;
+            }
+            else
+            {
+                pDst->Rsrvd1 = 0;
+                pDst->Rsrvd2 = 0;
+            }
+            for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++)
+            {
+                pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
+                pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
+                pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
+                pDst->aRegs[i].au32[3] = 0;
+            }
+        }
+        else
+        {
+            pDst->FCW   = 0x37f;
+            pDst->FSW   = 0;
+            pDst->FTW   = 0x00;         /* 0 - empty. */
+            pDst->FPUDP = 0;
+            pDst->DS    = 0; //??
+            pDst->Rsrvd2= 0;
+            pDst->FPUIP = 0;
+            pDst->CS    = 0; //??
+            pDst->Rsrvd1= 0;
+            pDst->FOP   = 0;
+            for (uint32_t i = 0; i < RT_ELEMENTS(pSrc->aRegs); i++)
+            {
+                pDst->aRegs[i].au32[0] = 0;
+                pDst->aRegs[i].au32[1] = 0;
+                pDst->aRegs[i].au32[2] = 0;
+                pDst->aRegs[i].au32[3] = 0;
+            }
+        }
+        pHdrDst->bmXState |= XSAVE_C_X87; /* playing safe for now */
+    }
+
+    /* MXCSR */
+    if (fReqComponents & (XSAVE_C_SSE | XSAVE_C_YMM))
+    {
+        if (fRstorMask & (XSAVE_C_SSE | XSAVE_C_YMM))
+            pDst->MXCSR = pSrc->MXCSR;
+        else
+            pDst->MXCSR = 0x1f80;
+    }
+
+    /* XMM registers. */
+    if (fReqComponents & XSAVE_C_SSE)
+    {
+        if (fRstorMask & XSAVE_C_SSE)
+        {
+            for (uint32_t i = 0; i < cXmmRegs; i++)
+                pDst->aXMM[i] = pSrc->aXMM[i];
+            /** @todo Testcase: What happens to the reserved XMM registers? Untouched,
+             *        right? */
+        }
+        else
+        {
+            for (uint32_t i = 0; i < cXmmRegs; i++)
+            {
+                pDst->aXMM[i].au64[0] = 0;
+                pDst->aXMM[i].au64[1] = 0;
+            }
+        }
+        pHdrDst->bmXState |= XSAVE_C_SSE; /* playing safe for now */
+    }
+
+    /* Unmap the x87 state bits (so we've don't run out of mapping). */
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Restore AVX state.
+     */
+    if (fReqComponents & XSAVE_C_YMM)
+    {
+        AssertLogRelReturn(pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT] != UINT16_MAX, VERR_IEM_IPE_9);
+        PX86XSAVEYMMHI  pCompDst = CPUMCTX_XSAVE_C_PTR(IEM_GET_CTX(pVCpu), XSAVE_C_YMM_BIT, PX86XSAVEYMMHI);
+
+        if (fRstorMask & XSAVE_C_YMM)
+        {
+            /** @todo testcase: xsave64 vs xsave32 wrt XSAVE_C_YMM. */
+            PCX86XSAVEYMMHI pCompSrc;
+            rcStrict = iemMemMap(pVCpu, (void **)&pCompSrc, sizeof(*pCompDst),
+                                 iEffSeg, GCPtrEff + pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT], IEM_ACCESS_DATA_R);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            for (uint32_t i = 0; i < cXmmRegs; i++)
+            {
+                pCompDst->aYmmHi[i].au64[0] = pCompSrc->aYmmHi[i].au64[0];
+                pCompDst->aYmmHi[i].au64[1] = pCompSrc->aYmmHi[i].au64[1];
+            }
+
+            rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pCompSrc, IEM_ACCESS_DATA_R);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+        else
+        {
+            for (uint32_t i = 0; i < cXmmRegs; i++)
+            {
+                pCompDst->aYmmHi[i].au64[0] = 0;
+                pCompDst->aYmmHi[i].au64[1] = 0;
+            }
+        }
+        pHdrDst->bmXState |= XSAVE_C_YMM; /* playing safe for now */
+    }
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+
+
+/**
+ * Implements 'STMXCSR'.
+ *
+ * @param   GCPtrEff        The address of the image.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_stmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX);
+
+    /*
+     * Raise exceptions.
+     */
+    if (   !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)
+        && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR))
+    {
+        if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
+        {
+            /*
+             * Do the job.
+             */
+            VBOXSTRICTRC rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrEff, pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+                return VINF_SUCCESS;
+            }
+            return rcStrict;
+        }
+        return iemRaiseDeviceNotAvailable(pVCpu);
+    }
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+
+/**
+ * Implements 'VSTMXCSR'.
+ *
+ * @param   GCPtrEff        The address of the image.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vstmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_XCRx);
+
+    /*
+     * Raise exceptions.
+     */
+    if (   (   !IEM_IS_GUEST_CPU_AMD(pVCpu)
+            ? (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_SSE | XSAVE_C_YMM)) == (XSAVE_C_SSE | XSAVE_C_YMM)
+            : !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)) /* AMD Jaguar CPU (f0x16,m0,s1) behaviour */
+        && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
+    {
+        if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
+        {
+            /*
+             * Do the job.
+             */
+            VBOXSTRICTRC rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrEff, pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+                return VINF_SUCCESS;
+            }
+            return rcStrict;
+        }
+        return iemRaiseDeviceNotAvailable(pVCpu);
+    }
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+
+/**
+ * Implements 'LDMXCSR'.
+ *
+ * @param   GCPtrEff        The address of the image.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_ldmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX);
+
+    /*
+     * Raise exceptions.
+     */
+    /** @todo testcase - order of LDMXCSR faults.  Does \#PF, \#GP and \#SS
+     *        happen after or before \#UD and \#EM? */
+    if (   !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)
+        && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR))
+    {
+        if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
+        {
+            /*
+             * Do the job.
+             */
+            uint32_t fNewMxCsr;
+            VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, &fNewMxCsr, iEffSeg, GCPtrEff);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                uint32_t const fMxCsrMask = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
+                if (!(fNewMxCsr & ~fMxCsrMask))
+                {
+                    pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR = fNewMxCsr;
+                    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+                    return VINF_SUCCESS;
+                }
+                Log(("lddmxcsr: New MXCSR=%#RX32 & ~MASK=%#RX32 = %#RX32 -> #GP(0)\n",
+                     fNewMxCsr, fMxCsrMask, fNewMxCsr & ~fMxCsrMask));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+            return rcStrict;
+        }
+        return iemRaiseDeviceNotAvailable(pVCpu);
+    }
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+
+/**
+ * Commmon routine for fnstenv and fnsave.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   uPtr            Where to store the state.
+ */
+static void iemCImplCommonFpuStoreEnv(PVMCPUCC pVCpu, IEMMODE enmEffOpSize, RTPTRUNION uPtr)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87);
+    PCX86FXSTATE pSrcX87 = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    if (enmEffOpSize == IEMMODE_16BIT)
+    {
+        uPtr.pu16[0] = pSrcX87->FCW;
+        uPtr.pu16[1] = pSrcX87->FSW;
+        uPtr.pu16[2] = iemFpuCalcFullFtw(pSrcX87);
+        if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+        {
+            /** @todo Testcase: How does this work when the FPUIP/CS was saved in
+             *        protected mode or long mode and we save it in real mode?  And vice
+             *        versa?  And with 32-bit operand size?  I think CPU is storing the
+             *        effective address ((CS << 4) + IP) in the offset register and not
+             *        doing any address calculations here. */
+            uPtr.pu16[3] = (uint16_t)pSrcX87->FPUIP;
+            uPtr.pu16[4] = ((pSrcX87->FPUIP >> 4) & UINT16_C(0xf000)) | pSrcX87->FOP;
+            uPtr.pu16[5] = (uint16_t)pSrcX87->FPUDP;
+            uPtr.pu16[6] = (pSrcX87->FPUDP  >> 4) & UINT16_C(0xf000);
+        }
+        else
+        {
+            uPtr.pu16[3] = pSrcX87->FPUIP;
+            uPtr.pu16[4] = pSrcX87->CS;
+            uPtr.pu16[5] = pSrcX87->FPUDP;
+            uPtr.pu16[6] = pSrcX87->DS;
+        }
+    }
+    else
+    {
+        /** @todo Testcase: what is stored in the "gray" areas? (figure 8-9 and 8-10) */
+        uPtr.pu16[0*2]   = pSrcX87->FCW;
+        uPtr.pu16[0*2+1] = 0xffff;  /* (0xffff observed on intel skylake.) */
+        uPtr.pu16[1*2]   = pSrcX87->FSW;
+        uPtr.pu16[1*2+1] = 0xffff;
+        uPtr.pu16[2*2]   = iemFpuCalcFullFtw(pSrcX87);
+        uPtr.pu16[2*2+1] = 0xffff;
+        if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+        {
+            uPtr.pu16[3*2]   = (uint16_t)pSrcX87->FPUIP;
+            uPtr.pu32[4]     = ((pSrcX87->FPUIP & UINT32_C(0xffff0000)) >> 4) | pSrcX87->FOP;
+            uPtr.pu16[5*2]   = (uint16_t)pSrcX87->FPUDP;
+            uPtr.pu32[6]     = (pSrcX87->FPUDP  & UINT32_C(0xffff0000)) >> 4;
+        }
+        else
+        {
+            uPtr.pu32[3]     = pSrcX87->FPUIP;
+            uPtr.pu16[4*2]   = pSrcX87->CS;
+            uPtr.pu16[4*2+1] = pSrcX87->FOP;
+            uPtr.pu32[5]     = pSrcX87->FPUDP;
+            uPtr.pu16[6*2]   = pSrcX87->DS;
+            uPtr.pu16[6*2+1] = 0xffff;
+        }
+    }
+}
+
+
+/**
+ * Commmon routine for fldenv and frstor
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   uPtr                Where to store the state.
+ */
+static void iemCImplCommonFpuRestoreEnv(PVMCPUCC pVCpu, IEMMODE enmEffOpSize, RTCPTRUNION uPtr)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87);
+    PX86FXSTATE pDstX87 = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    if (enmEffOpSize == IEMMODE_16BIT)
+    {
+        pDstX87->FCW = uPtr.pu16[0];
+        pDstX87->FSW = uPtr.pu16[1];
+        pDstX87->FTW = uPtr.pu16[2];
+        if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+        {
+            pDstX87->FPUIP = uPtr.pu16[3] | ((uint32_t)(uPtr.pu16[4] & UINT16_C(0xf000)) << 4);
+            pDstX87->FPUDP = uPtr.pu16[5] | ((uint32_t)(uPtr.pu16[6] & UINT16_C(0xf000)) << 4);
+            pDstX87->FOP   = uPtr.pu16[4] & UINT16_C(0x07ff);
+            pDstX87->CS    = 0;
+            pDstX87->Rsrvd1= 0;
+            pDstX87->DS    = 0;
+            pDstX87->Rsrvd2= 0;
+        }
+        else
+        {
+            pDstX87->FPUIP = uPtr.pu16[3];
+            pDstX87->CS    = uPtr.pu16[4];
+            pDstX87->Rsrvd1= 0;
+            pDstX87->FPUDP = uPtr.pu16[5];
+            pDstX87->DS    = uPtr.pu16[6];
+            pDstX87->Rsrvd2= 0;
+            /** @todo Testcase: Is FOP cleared when doing 16-bit protected mode fldenv? */
+        }
+    }
+    else
+    {
+        pDstX87->FCW = uPtr.pu16[0*2];
+        pDstX87->FSW = uPtr.pu16[1*2];
+        pDstX87->FTW = uPtr.pu16[2*2];
+        if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+        {
+            pDstX87->FPUIP = uPtr.pu16[3*2] | ((uPtr.pu32[4] & UINT32_C(0x0ffff000)) << 4);
+            pDstX87->FOP   = uPtr.pu32[4] & UINT16_C(0x07ff);
+            pDstX87->FPUDP = uPtr.pu16[5*2] | ((uPtr.pu32[6] & UINT32_C(0x0ffff000)) << 4);
+            pDstX87->CS    = 0;
+            pDstX87->Rsrvd1= 0;
+            pDstX87->DS    = 0;
+            pDstX87->Rsrvd2= 0;
+        }
+        else
+        {
+            pDstX87->FPUIP = uPtr.pu32[3];
+            pDstX87->CS    = uPtr.pu16[4*2];
+            pDstX87->Rsrvd1= 0;
+            pDstX87->FOP   = uPtr.pu16[4*2+1];
+            pDstX87->FPUDP = uPtr.pu32[5];
+            pDstX87->DS    = uPtr.pu16[6*2];
+            pDstX87->Rsrvd2= 0;
+        }
+    }
+
+    /* Make adjustments. */
+    pDstX87->FTW = iemFpuCompressFtw(pDstX87->FTW);
+    pDstX87->FCW &= ~X86_FCW_ZERO_MASK;
+    iemFpuRecalcExceptionStatus(pDstX87);
+    /** @todo Testcase: Check if ES and/or B are automatically cleared if no
+     *        exceptions are pending after loading the saved state? */
+}
+
+
+/**
+ * Implements 'FNSTENV'.
+ *
+ * @param   enmEffOpSize    The operand size (only REX.W really matters).
+ * @param   iEffSeg         The effective segment register for @a GCPtrEff.
+ * @param   GCPtrEffDst     The address of the image.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_fnstenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    RTPTRUNION   uPtr;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 14 : 28,
+                                      iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemCImplCommonFpuStoreEnv(pVCpu, enmEffOpSize, uPtr);
+
+    rcStrict = iemMemCommitAndUnmap(pVCpu, uPtr.pv, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Note: C0, C1, C2 and C3 are documented as undefined, we leave them untouched! */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'FNSAVE'.
+ *
+ * @param   GCPtrEffDst     The address of the image.
+ * @param   enmEffOpSize    The operand size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_fnsave, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87);
+
+    RTPTRUNION   uPtr;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 94 : 108,
+                                      iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemCImplCommonFpuStoreEnv(pVCpu, enmEffOpSize, uPtr);
+    PRTFLOAT80U paRegs = (PRTFLOAT80U)(uPtr.pu8 + (enmEffOpSize == IEMMODE_16BIT ? 14 : 28));
+    for (uint32_t i = 0; i < RT_ELEMENTS(pFpuCtx->aRegs); i++)
+    {
+        paRegs[i].au32[0] = pFpuCtx->aRegs[i].au32[0];
+        paRegs[i].au32[1] = pFpuCtx->aRegs[i].au32[1];
+        paRegs[i].au16[4] = pFpuCtx->aRegs[i].au16[4];
+    }
+
+    rcStrict = iemMemCommitAndUnmap(pVCpu, uPtr.pv, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Re-initialize the FPU context.
+     */
+    pFpuCtx->FCW   = 0x37f;
+    pFpuCtx->FSW   = 0;
+    pFpuCtx->FTW   = 0x00;       /* 0 - empty */
+    pFpuCtx->FPUDP = 0;
+    pFpuCtx->DS    = 0;
+    pFpuCtx->Rsrvd2= 0;
+    pFpuCtx->FPUIP = 0;
+    pFpuCtx->CS    = 0;
+    pFpuCtx->Rsrvd1= 0;
+    pFpuCtx->FOP   = 0;
+
+    iemHlpUsedFpu(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Implements 'FLDENV'.
+ *
+ * @param   enmEffOpSize    The operand size (only REX.W really matters).
+ * @param   iEffSeg         The effective segment register for @a GCPtrEff.
+ * @param   GCPtrEffSrc     The address of the image.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_fldenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
+{
+    RTCPTRUNION  uPtr;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 14 : 28,
+                                      iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemCImplCommonFpuRestoreEnv(pVCpu, enmEffOpSize, uPtr);
+
+    rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uPtr.pv, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemHlpUsedFpu(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'FRSTOR'.
+ *
+ * @param   GCPtrEffSrc     The address of the image.
+ * @param   enmEffOpSize    The operand size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_frstor, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
+{
+    RTCPTRUNION  uPtr;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 94 : 108,
+                                      iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    iemCImplCommonFpuRestoreEnv(pVCpu, enmEffOpSize, uPtr);
+    PCRTFLOAT80U paRegs = (PCRTFLOAT80U)(uPtr.pu8 + (enmEffOpSize == IEMMODE_16BIT ? 14 : 28));
+    for (uint32_t i = 0; i < RT_ELEMENTS(pFpuCtx->aRegs); i++)
+    {
+        pFpuCtx->aRegs[i].au32[0] = paRegs[i].au32[0];
+        pFpuCtx->aRegs[i].au32[1] = paRegs[i].au32[1];
+        pFpuCtx->aRegs[i].au32[2] = paRegs[i].au16[4];
+        pFpuCtx->aRegs[i].au32[3] = 0;
+    }
+
+    rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uPtr.pv, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemHlpUsedFpu(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'FLDCW'.
+ *
+ * @param   u16Fcw          The new FCW.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_fldcw, uint16_t, u16Fcw)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87);
+
+    /** @todo Testcase: Check what happens when trying to load X86_FCW_PC_RSVD. */
+    /** @todo Testcase: Try see what happens when trying to set undefined bits
+     *        (other than 6 and 7).  Currently ignoring them. */
+    /** @todo Testcase: Test that it raises and loweres the FPU exception bits
+     *        according to FSW. (This is was is currently implemented.) */
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    pFpuCtx->FCW = u16Fcw & ~X86_FCW_ZERO_MASK;
+    iemFpuRecalcExceptionStatus(pFpuCtx);
+
+    /* Note: C0, C1, C2 and C3 are documented as undefined, we leave them untouched! */
+    iemHlpUsedFpu(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Implements the underflow case of fxch.
+ *
+ * @param   iStReg              The other stack register.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_fxch_underflow, uint8_t, iStReg)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87);
+
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    unsigned const iReg1 = X86_FSW_TOP_GET(pFpuCtx->FSW);
+    unsigned const iReg2 = (iReg1 + iStReg) & X86_FSW_TOP_SMASK;
+    Assert(!(RT_BIT(iReg1) & pFpuCtx->FTW) || !(RT_BIT(iReg2) & pFpuCtx->FTW));
+
+    /** @todo Testcase: fxch underflow. Making assumptions that underflowed
+     *        registers are read as QNaN and then exchanged. This could be
+     *        wrong... */
+    if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        if (RT_BIT(iReg1) & pFpuCtx->FTW)
+        {
+            if (RT_BIT(iReg2) & pFpuCtx->FTW)
+                iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
+            else
+                pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[iStReg].r80;
+            iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
+        }
+        else
+        {
+            pFpuCtx->aRegs[iStReg].r80 = pFpuCtx->aRegs[0].r80;
+            iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
+        }
+        pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF;
+    }
+    else
+    {
+        /* raise underflow exception, don't change anything. */
+        pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_XCPT_MASK);
+        pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
+    }
+
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemHlpUsedFpu(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'FCOMI', 'FCOMIP', 'FUCOMI', and 'FUCOMIP'.
+ *
+ * @param   cToAdd              1 or 7.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_fcomi_fucomi, uint8_t, iStReg, PFNIEMAIMPLFPUR80EFL, pfnAImpl, bool, fPop)
+{
+    Assert(iStReg < 8);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87);
+
+    /*
+     * Raise exceptions.
+     */
+    if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS))
+        return iemRaiseDeviceNotAvailable(pVCpu);
+
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
+    uint16_t u16Fsw = pFpuCtx->FSW;
+    if (u16Fsw & X86_FSW_ES)
+        return iemRaiseMathFault(pVCpu);
+
+    /*
+     * Check if any of the register accesses causes #SF + #IA.
+     */
+    unsigned const iReg1 = X86_FSW_TOP_GET(u16Fsw);
+    unsigned const iReg2 = (iReg1 + iStReg) & X86_FSW_TOP_SMASK;
+    if ((pFpuCtx->FTW & (RT_BIT(iReg1) | RT_BIT(iReg2))) == (RT_BIT(iReg1) | RT_BIT(iReg2)))
+    {
+        uint32_t u32Eflags = pfnAImpl(pFpuCtx, &u16Fsw, &pFpuCtx->aRegs[0].r80, &pFpuCtx->aRegs[iStReg].r80);
+        NOREF(u32Eflags);
+
+        pFpuCtx->FSW &= ~X86_FSW_C1;
+        pFpuCtx->FSW |= u16Fsw & ~X86_FSW_TOP_MASK;
+        if (   !(u16Fsw & X86_FSW_IE)
+            || (pFpuCtx->FCW & X86_FCW_IM) )
+        {
+            pVCpu->cpum.GstCtx.eflags.u &= ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF);
+            pVCpu->cpum.GstCtx.eflags.u |= pVCpu->cpum.GstCtx.eflags.u & (X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF);
+        }
+    }
+    else if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked underflow. */
+        pFpuCtx->FSW &= ~X86_FSW_C1;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
+        pVCpu->cpum.GstCtx.eflags.u &= ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF);
+        pVCpu->cpum.GstCtx.eflags.u |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF;
+    }
+    else
+    {
+        /* Raise underflow - don't touch EFLAGS or TOP. */
+        pFpuCtx->FSW &= ~X86_FSW_C1;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
+        fPop = false;
+    }
+
+    /*
+     * Pop if necessary.
+     */
+    if (fPop)
+    {
+        pFpuCtx->FTW &= ~RT_BIT(iReg1);
+        pFpuCtx->FSW &= X86_FSW_TOP_MASK;
+        pFpuCtx->FSW |= ((iReg1 + 7) & X86_FSW_TOP_SMASK) << X86_FSW_TOP_SHIFT;
+    }
+
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemHlpUsedFpu(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h b/src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h
new file mode 100644
index 00000000..d5dc1340
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h
@@ -0,0 +1,1762 @@
+/* $Id: IEMAllCImplStrInstr.cpp.h $ */
+/** @file
+ * IEM - String Instruction Implementation Code Template.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+#if OP_SIZE == 8
+# define OP_rAX     al
+#elif OP_SIZE == 16
+# define OP_rAX     ax
+#elif OP_SIZE == 32
+# define OP_rAX     eax
+#elif OP_SIZE == 64
+# define OP_rAX     rax
+#else
+# error "Bad OP_SIZE."
+#endif
+#define OP_TYPE                     RT_CONCAT3(uint,OP_SIZE,_t)
+
+#if ADDR_SIZE == 16
+# define ADDR_rDI       di
+# define ADDR_rSI       si
+# define ADDR_rCX       cx
+# define ADDR2_TYPE     uint32_t
+# define ADDR_VMXSTRIO  0
+#elif ADDR_SIZE == 32
+# define ADDR_rDI       edi
+# define ADDR_rSI       esi
+# define ADDR_rCX       ecx
+# define ADDR2_TYPE     uint32_t
+# define ADDR_VMXSTRIO  1
+#elif ADDR_SIZE == 64
+# define ADDR_rDI       rdi
+# define ADDR_rSI       rsi
+# define ADDR_rCX       rcx
+# define ADDR2_TYPE     uint64_t
+# define ADDR_VMXSTRIO  2
+# define IS_64_BIT_CODE(a_pVCpu)    (true)
+#else
+# error "Bad ADDR_SIZE."
+#endif
+#define ADDR_TYPE                   RT_CONCAT3(uint,ADDR_SIZE,_t)
+
+#if ADDR_SIZE == 64 || OP_SIZE == 64
+# define IS_64_BIT_CODE(a_pVCpu)    (true)
+#elif ADDR_SIZE == 32
+# define IS_64_BIT_CODE(a_pVCpu)    ((a_pVCpu)->iem.s.enmCpuMode == IEMMODE_64BIT)
+#else
+# define IS_64_BIT_CODE(a_pVCpu)    (false)
+#endif
+
+/** @def IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN
+ * Used in the outer (page-by-page) loop to check for reasons for returnning
+ * before completing the instruction.   In raw-mode we temporarily enable
+ * interrupts to let the host interrupt us.  We cannot let big string operations
+ * hog the CPU, especially not in raw-mode.
+ */
+#define IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(a_pVM, a_pVCpu, a_fEflags) \
+    do { \
+        if (RT_LIKELY(   !VMCPU_FF_IS_ANY_SET(a_pVCpu, (a_fEflags) & X86_EFL_IF ? VMCPU_FF_YIELD_REPSTR_MASK \
+                                                                                : VMCPU_FF_YIELD_REPSTR_NOINT_MASK) \
+                      && !VM_FF_IS_ANY_SET(a_pVM, VM_FF_YIELD_REPSTR_MASK) \
+                      )) \
+        { /* probable */ } \
+        else  \
+        { \
+            LogFlow(("%s: Leaving early (outer)! ffcpu=%#RX64 ffvm=%#x\n", \
+                     __FUNCTION__, (uint64_t)(a_pVCpu)->fLocalForcedActions, (a_pVM)->fGlobalForcedActions)); \
+            return VINF_SUCCESS; \
+        } \
+    } while (0)
+
+/** @def IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN
+ * This is used in some of the inner loops to make sure we respond immediately
+ * to VMCPU_FF_IOM as well as outside requests.  Use this for expensive
+ * instructions. Use IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN for
+ * ones that are typically cheap. */
+#define IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(a_pVM, a_pVCpu, a_fExitExpr) \
+    do { \
+        if (RT_LIKELY(   (   !VMCPU_FF_IS_ANY_SET(a_pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_REPSTR_MASK) \
+                          && !VM_FF_IS_ANY_SET(a_pVM,         VM_FF_HIGH_PRIORITY_POST_REPSTR_MASK)) \
+                      || (a_fExitExpr) )) \
+        { /* very likely */ } \
+        else \
+        { \
+            LogFlow(("%s: Leaving early (inner)! ffcpu=%#RX64 ffvm=%#x\n", \
+                     __FUNCTION__, (uint64_t)(a_pVCpu)->fLocalForcedActions, (a_pVM)->fGlobalForcedActions)); \
+            return VINF_SUCCESS; \
+        } \
+    } while (0)
+
+
+/** @def IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN
+ * This is used in the inner loops where
+ * IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN isn't used.  It only
+ * checks the CPU FFs so that we respond immediately to the pending IOM FF
+ * (status code is hidden in IEMCPU::rcPassUp by IEM memory commit code).
+ */
+#define IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(a_pVM, a_pVCpu, a_fExitExpr) \
+    do { \
+        if (RT_LIKELY(   !VMCPU_FF_IS_ANY_SET(a_pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_REPSTR_MASK) \
+                      || (a_fExitExpr) )) \
+        { /* very likely */ } \
+        else \
+        { \
+            LogFlow(("%s: Leaving early (inner)! ffcpu=%#RX64 (ffvm=%#x)\n", \
+                     __FUNCTION__, (uint64_t)(a_pVCpu)->fLocalForcedActions, (a_pVM)->fGlobalForcedActions)); \
+            return VINF_SUCCESS; \
+        } \
+    } while (0)
+
+
+/**
+ * Implements 'REPE CMPS'.
+ */
+IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_repe_cmps_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg  = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iEffSeg) | CPUMCTX_EXTRN_ES);
+
+    PCCPUMSELREGHID pSrc1Hid     = iemSRegGetHid(pVCpu, iEffSeg);
+    uint64_t        uSrc1Base;
+    VBOXSTRICTRC    rcStrict     = iemMemSegCheckReadAccessEx(pVCpu, pSrc1Hid, iEffSeg, &uSrc1Base);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    uint64_t        uSrc2Base;
+    rcStrict = iemMemSegCheckReadAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uSrc2Base);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr       = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    ADDR_TYPE       uSrc1AddrReg = pVCpu->cpum.GstCtx.ADDR_rSI;
+    ADDR_TYPE       uSrc2AddrReg = pVCpu->cpum.GstCtx.ADDR_rDI;
+    uint32_t        uEFlags      = pVCpu->cpum.GstCtx.eflags.u;
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtSrc1Addr = uSrc1AddrReg + (ADDR2_TYPE)uSrc1Base;
+        ADDR2_TYPE  uVirtSrc2Addr = uSrc2AddrReg + (ADDR2_TYPE)uSrc2Base;
+        uint32_t    cLeftSrc1Page = (PAGE_SIZE - (uVirtSrc1Addr & PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftSrc1Page > uCounterReg)
+            cLeftSrc1Page = uCounterReg;
+        uint32_t    cLeftSrc2Page = (PAGE_SIZE - (uVirtSrc2Addr & PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        uint32_t    cLeftPage     = RT_MIN(cLeftSrc1Page, cLeftSrc2Page);
+
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Optimize reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uSrc1AddrReg < pSrc1Hid->u32Limit
+                    && uSrc1AddrReg + (cLeftPage * (OP_SIZE / 8)) <= pSrc1Hid->u32Limit
+                    && uSrc2AddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uSrc2AddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysSrc1Mem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrc1Addr, IEM_ACCESS_DATA_R, &GCPhysSrc1Mem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            RTGCPHYS GCPhysSrc2Mem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrc2Addr, IEM_ACCESS_DATA_R, &GCPhysSrc2Mem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, do a block processing
+             * until the end of the current page.
+             */
+            PGMPAGEMAPLOCK PgLockSrc2Mem;
+            OP_TYPE const *puSrc2Mem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, (void **)&puSrc2Mem, &PgLockSrc2Mem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                PGMPAGEMAPLOCK PgLockSrc1Mem;
+                OP_TYPE const *puSrc1Mem;
+                rcStrict = iemMemPageMap(pVCpu, GCPhysSrc1Mem, IEM_ACCESS_DATA_R, (void **)&puSrc1Mem, &PgLockSrc1Mem);
+                if (rcStrict == VINF_SUCCESS)
+                {
+                    if (!memcmp(puSrc2Mem, puSrc1Mem, cLeftPage * (OP_SIZE / 8)))
+                    {
+                        /* All matches, only compare the last itme to get the right eflags. */
+                        RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&puSrc1Mem[cLeftPage-1], puSrc2Mem[cLeftPage-1], &uEFlags);
+                        uSrc1AddrReg += cLeftPage * cbIncr;
+                        uSrc2AddrReg += cLeftPage * cbIncr;
+                        uCounterReg  -= cLeftPage;
+                    }
+                    else
+                    {
+                        /* Some mismatch, compare each item (and keep volatile
+                           memory in mind). */
+                        uint32_t off = 0;
+                        do
+                        {
+                            RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&puSrc1Mem[off], puSrc2Mem[off], &uEFlags);
+                            off++;
+                        } while (   off < cLeftPage
+                                 && (uEFlags & X86_EFL_ZF));
+                        uSrc1AddrReg += cbIncr * off;
+                        uSrc2AddrReg += cbIncr * off;
+                        uCounterReg  -= off;
+                    }
+
+                    /* Update the registers before looping. */
+                    pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg;
+                    pVCpu->cpum.GstCtx.ADDR_rSI = uSrc1AddrReg;
+                    pVCpu->cpum.GstCtx.ADDR_rDI = uSrc2AddrReg;
+                    pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+
+                    iemMemPageUnmap(pVCpu, GCPhysSrc1Mem, IEM_ACCESS_DATA_R, puSrc1Mem, &PgLockSrc1Mem);
+                    iemMemPageUnmap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, puSrc2Mem, &PgLockSrc2Mem);
+                    if (   uCounterReg == 0
+                        || !(uEFlags & X86_EFL_ZF))
+                        break;
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+                    continue;
+                }
+                iemMemPageUnmap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, puSrc2Mem, &PgLockSrc2Mem);
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            OP_TYPE uValue1;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue1, iEffSeg, uSrc1AddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            OP_TYPE uValue2;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue2, X86_SREG_ES, uSrc2AddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)(&uValue1, uValue2, &uEFlags);
+
+            pVCpu->cpum.GstCtx.ADDR_rSI = uSrc1AddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rDI = uSrc2AddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+            cLeftPage--;
+            IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0 || !(uEFlags & X86_EFL_ZF));
+        } while (   (int32_t)cLeftPage > 0
+                 && (uEFlags & X86_EFL_ZF));
+
+        /*
+         * Next page? Must check for interrupts and stuff here.
+         */
+        if (   uCounterReg == 0
+            || !(uEFlags & X86_EFL_ZF))
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+    }
+
+    /*
+     * Done.
+     */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'REPNE CMPS'.
+ */
+IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_repne_cmps_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iEffSeg) | CPUMCTX_EXTRN_ES);
+
+    PCCPUMSELREGHID pSrc1Hid = iemSRegGetHid(pVCpu, iEffSeg);
+    uint64_t        uSrc1Base;
+    VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pSrc1Hid, iEffSeg, &uSrc1Base);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    uint64_t        uSrc2Base;
+    rcStrict = iemMemSegCheckReadAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uSrc2Base);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr       = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    ADDR_TYPE       uSrc1AddrReg = pVCpu->cpum.GstCtx.ADDR_rSI;
+    ADDR_TYPE       uSrc2AddrReg = pVCpu->cpum.GstCtx.ADDR_rDI;
+    uint32_t        uEFlags      = pVCpu->cpum.GstCtx.eflags.u;
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtSrc1Addr = uSrc1AddrReg + (ADDR2_TYPE)uSrc1Base;
+        ADDR2_TYPE  uVirtSrc2Addr = uSrc2AddrReg + (ADDR2_TYPE)uSrc2Base;
+        uint32_t    cLeftSrc1Page = (PAGE_SIZE - (uVirtSrc1Addr & PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftSrc1Page > uCounterReg)
+            cLeftSrc1Page = uCounterReg;
+        uint32_t    cLeftSrc2Page = (PAGE_SIZE - (uVirtSrc2Addr & PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        uint32_t    cLeftPage = RT_MIN(cLeftSrc1Page, cLeftSrc2Page);
+
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Optimize reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uSrc1AddrReg < pSrc1Hid->u32Limit
+                    && uSrc1AddrReg + (cLeftPage * (OP_SIZE / 8)) <= pSrc1Hid->u32Limit
+                    && uSrc2AddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uSrc2AddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+                )
+           )
+        {
+            RTGCPHYS GCPhysSrc1Mem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrc1Addr, IEM_ACCESS_DATA_R, &GCPhysSrc1Mem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            RTGCPHYS GCPhysSrc2Mem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrc2Addr, IEM_ACCESS_DATA_R, &GCPhysSrc2Mem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, do a block processing
+             * until the end of the current page.
+             */
+            OP_TYPE const *puSrc2Mem;
+            PGMPAGEMAPLOCK PgLockSrc2Mem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, (void **)&puSrc2Mem, &PgLockSrc2Mem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                OP_TYPE const *puSrc1Mem;
+                PGMPAGEMAPLOCK PgLockSrc1Mem;
+                rcStrict = iemMemPageMap(pVCpu, GCPhysSrc1Mem, IEM_ACCESS_DATA_R, (void **)&puSrc1Mem, &PgLockSrc1Mem);
+                if (rcStrict == VINF_SUCCESS)
+                {
+                    if (memcmp(puSrc2Mem, puSrc1Mem, cLeftPage * (OP_SIZE / 8)))
+                    {
+                        /* All matches, only compare the last item to get the right eflags. */
+                        RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&puSrc1Mem[cLeftPage-1], puSrc2Mem[cLeftPage-1], &uEFlags);
+                        uSrc1AddrReg += cLeftPage * cbIncr;
+                        uSrc2AddrReg += cLeftPage * cbIncr;
+                        uCounterReg  -= cLeftPage;
+                    }
+                    else
+                    {
+                        /* Some mismatch, compare each item (and keep volatile
+                           memory in mind). */
+                        uint32_t off = 0;
+                        do
+                        {
+                            RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&puSrc1Mem[off], puSrc2Mem[off], &uEFlags);
+                            off++;
+                        } while (   off < cLeftPage
+                                 && !(uEFlags & X86_EFL_ZF));
+                        uSrc1AddrReg += cbIncr * off;
+                        uSrc2AddrReg += cbIncr * off;
+                        uCounterReg  -= off;
+                    }
+
+                    /* Update the registers before looping. */
+                    pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg;
+                    pVCpu->cpum.GstCtx.ADDR_rSI = uSrc1AddrReg;
+                    pVCpu->cpum.GstCtx.ADDR_rDI = uSrc2AddrReg;
+                    pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+
+                    iemMemPageUnmap(pVCpu, GCPhysSrc1Mem, IEM_ACCESS_DATA_R, puSrc1Mem, &PgLockSrc1Mem);
+                    iemMemPageUnmap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, puSrc2Mem, &PgLockSrc2Mem);
+                    if (   uCounterReg == 0
+                        || (uEFlags & X86_EFL_ZF))
+                        break;
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+                    continue;
+                }
+                iemMemPageUnmap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, puSrc2Mem, &PgLockSrc2Mem);
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            OP_TYPE uValue1;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue1, iEffSeg, uSrc1AddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            OP_TYPE uValue2;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue2, X86_SREG_ES, uSrc2AddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)(&uValue1, uValue2, &uEFlags);
+
+            pVCpu->cpum.GstCtx.ADDR_rSI = uSrc1AddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rDI = uSrc2AddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+            cLeftPage--;
+            IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0 || (uEFlags & X86_EFL_ZF));
+        } while (   (int32_t)cLeftPage > 0
+                 && !(uEFlags & X86_EFL_ZF));
+
+        /*
+         * Next page? Must check for interrupts and stuff here.
+         */
+        if (   uCounterReg == 0
+            || (uEFlags & X86_EFL_ZF))
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+    }
+
+    /*
+     * Done.
+     */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'REPE SCAS'.
+ */
+IEM_CIMPL_DEF_0(RT_CONCAT4(iemCImpl_repe_scas_,OP_rAX,_m,ADDR_SIZE))
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ES);
+    uint64_t        uBaseAddr;
+    VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uBaseAddr);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    OP_TYPE const   uValueReg   = pVCpu->cpum.GstCtx.OP_rAX;
+    ADDR_TYPE       uAddrReg    = pVCpu->cpum.GstCtx.ADDR_rDI;
+    uint32_t        uEFlags     = pVCpu->cpum.GstCtx.eflags.u;
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr;
+        uint32_t    cLeftPage = (PAGE_SIZE - (uVirtAddr & PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftPage > uCounterReg)
+            cLeftPage = uCounterReg;
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uAddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, IEM_ACCESS_DATA_R, &GCPhysMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, do a block processing
+             * until the end of the current page.
+             */
+            PGMPAGEMAPLOCK PgLockMem;
+            OP_TYPE const *puMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, (void **)&puMem, &PgLockMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                /* Search till we find a mismatching item. */
+                OP_TYPE  uTmpValue;
+                bool     fQuit;
+                uint32_t i = 0;
+                do
+                {
+                    uTmpValue = puMem[i++];
+                    fQuit = uTmpValue != uValueReg;
+                } while (i < cLeftPage && !fQuit);
+
+                /* Update the regs. */
+                RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&uValueReg, uTmpValue, &uEFlags);
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= i;
+                pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg    += i * cbIncr;
+                pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+                Assert(!(uEFlags & X86_EFL_ZF) == fQuit);
+                iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, puMem, &PgLockMem);
+                if (   fQuit
+                    || uCounterReg == 0)
+                    break;
+
+                /* If unaligned, we drop thru and do the page crossing access
+                   below. Otherwise, do the next page. */
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+                    continue;
+                }
+                cLeftPage = 0;
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            OP_TYPE uTmpValue;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uTmpValue, X86_SREG_ES, uAddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&uValueReg, uTmpValue, &uEFlags);
+
+            pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+            cLeftPage--;
+            IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0 || !(uEFlags & X86_EFL_ZF));
+        } while (   (int32_t)cLeftPage > 0
+                 && (uEFlags & X86_EFL_ZF));
+
+        /*
+         * Next page? Must check for interrupts and stuff here.
+         */
+        if (   uCounterReg == 0
+            || !(uEFlags & X86_EFL_ZF))
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+    }
+
+    /*
+     * Done.
+     */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'REPNE SCAS'.
+ */
+IEM_CIMPL_DEF_0(RT_CONCAT4(iemCImpl_repne_scas_,OP_rAX,_m,ADDR_SIZE))
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ES);
+    uint64_t        uBaseAddr;
+    VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uBaseAddr);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    OP_TYPE const   uValueReg   = pVCpu->cpum.GstCtx.OP_rAX;
+    ADDR_TYPE       uAddrReg    = pVCpu->cpum.GstCtx.ADDR_rDI;
+    uint32_t        uEFlags     = pVCpu->cpum.GstCtx.eflags.u;
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr;
+        uint32_t    cLeftPage = (PAGE_SIZE - (uVirtAddr & PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftPage > uCounterReg)
+            cLeftPage = uCounterReg;
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uAddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, IEM_ACCESS_DATA_R, &GCPhysMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, do a block processing
+             * until the end of the current page.
+             */
+            PGMPAGEMAPLOCK PgLockMem;
+            OP_TYPE const *puMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, (void **)&puMem, &PgLockMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                /* Search till we find a mismatching item. */
+                OP_TYPE  uTmpValue;
+                bool     fQuit;
+                uint32_t i = 0;
+                do
+                {
+                    uTmpValue = puMem[i++];
+                    fQuit = uTmpValue == uValueReg;
+                } while (i < cLeftPage && !fQuit);
+
+                /* Update the regs. */
+                RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&uValueReg, uTmpValue, &uEFlags);
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= i;
+                pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg    += i * cbIncr;
+                pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+                Assert(!!(uEFlags & X86_EFL_ZF) == fQuit);
+                iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, puMem, &PgLockMem);
+                if (   fQuit
+                    || uCounterReg == 0)
+                    break;
+
+                /* If unaligned, we drop thru and do the page crossing access
+                   below. Otherwise, do the next page. */
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+                    continue;
+                }
+                cLeftPage = 0;
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            OP_TYPE uTmpValue;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uTmpValue, X86_SREG_ES, uAddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&uValueReg, uTmpValue, &uEFlags);
+            pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+            cLeftPage--;
+            IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0 || (uEFlags & X86_EFL_ZF));
+        } while (   (int32_t)cLeftPage > 0
+                 && !(uEFlags & X86_EFL_ZF));
+
+        /*
+         * Next page? Must check for interrupts and stuff here.
+         */
+        if (   uCounterReg == 0
+            || (uEFlags & X86_EFL_ZF))
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+    }
+
+    /*
+     * Done.
+     */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+
+
+/**
+ * Implements 'REP MOVS'.
+ */
+IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_rep_movs_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iEffSeg) | CPUMCTX_EXTRN_ES);
+
+    PCCPUMSELREGHID pSrcHid = iemSRegGetHid(pVCpu, iEffSeg);
+    uint64_t        uSrcBase;
+    VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pSrcHid, iEffSeg, &uSrcBase);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    uint64_t        uDstBase;
+    rcStrict = iemMemSegCheckWriteAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uDstBase);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    ADDR_TYPE       uSrcAddrReg = pVCpu->cpum.GstCtx.ADDR_rSI;
+    ADDR_TYPE       uDstAddrReg = pVCpu->cpum.GstCtx.ADDR_rDI;
+
+    /*
+     * Be careful with handle bypassing.
+     */
+    if (pVCpu->iem.s.fBypassHandlers)
+    {
+        Log(("%s: declining because we're bypassing handlers\n", __FUNCTION__));
+        return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
+    }
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtSrcAddr = uSrcAddrReg + (ADDR2_TYPE)uSrcBase;
+        ADDR2_TYPE  uVirtDstAddr = uDstAddrReg + (ADDR2_TYPE)uDstBase;
+        uint32_t    cLeftSrcPage = (PAGE_SIZE - (uVirtSrcAddr & PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftSrcPage > uCounterReg)
+            cLeftSrcPage = uCounterReg;
+        uint32_t    cLeftDstPage = (PAGE_SIZE - (uVirtDstAddr & PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        uint32_t    cLeftPage = RT_MIN(cLeftSrcPage, cLeftDstPage);
+
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uSrcAddrReg < pSrcHid->u32Limit
+                    && uSrcAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pSrcHid->u32Limit
+                    && uDstAddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uDstAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysSrcMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrcAddr, IEM_ACCESS_DATA_R, &GCPhysSrcMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            RTGCPHYS GCPhysDstMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtDstAddr, IEM_ACCESS_DATA_W, &GCPhysDstMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, do a block processing
+             * until the end of the current page.
+             */
+            PGMPAGEMAPLOCK PgLockDstMem;
+            OP_TYPE *puDstMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysDstMem, IEM_ACCESS_DATA_W, (void **)&puDstMem, &PgLockDstMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                PGMPAGEMAPLOCK PgLockSrcMem;
+                OP_TYPE const *puSrcMem;
+                rcStrict = iemMemPageMap(pVCpu, GCPhysSrcMem, IEM_ACCESS_DATA_R, (void **)&puSrcMem, &PgLockSrcMem);
+                if (rcStrict == VINF_SUCCESS)
+                {
+                    Assert(   (GCPhysSrcMem         >> PAGE_SHIFT) != (GCPhysDstMem         >> PAGE_SHIFT)
+                           || ((uintptr_t)puSrcMem  >> PAGE_SHIFT) == ((uintptr_t)puDstMem  >> PAGE_SHIFT));
+
+                    /* Perform the operation exactly (don't use memcpy to avoid
+                       having to consider how its implementation would affect
+                       any overlapping source and destination area). */
+                    OP_TYPE const  *puSrcCur = puSrcMem;
+                    OP_TYPE        *puDstCur = puDstMem;
+                    uint32_t        cTodo    = cLeftPage;
+                    while (cTodo-- > 0)
+                        *puDstCur++ = *puSrcCur++;
+
+                    /* Update the registers. */
+                    pVCpu->cpum.GstCtx.ADDR_rSI = uSrcAddrReg += cLeftPage * cbIncr;
+                    pVCpu->cpum.GstCtx.ADDR_rDI = uDstAddrReg += cLeftPage * cbIncr;
+                    pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cLeftPage;
+
+                    iemMemPageUnmap(pVCpu, GCPhysSrcMem, IEM_ACCESS_DATA_R, puSrcMem, &PgLockSrcMem);
+                    iemMemPageUnmap(pVCpu, GCPhysDstMem, IEM_ACCESS_DATA_W, puDstMem, &PgLockDstMem);
+
+                    if (uCounterReg == 0)
+                        break;
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+                    continue;
+                }
+                iemMemPageUnmap(pVCpu, GCPhysDstMem, IEM_ACCESS_DATA_W, puDstMem, &PgLockDstMem);
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            OP_TYPE uValue;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue, iEffSeg, uSrcAddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            rcStrict = RT_CONCAT(iemMemStoreDataU,OP_SIZE)(pVCpu, X86_SREG_ES, uDstAddrReg, uValue);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            pVCpu->cpum.GstCtx.ADDR_rSI = uSrcAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rDI = uDstAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            cLeftPage--;
+            IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0);
+        } while ((int32_t)cLeftPage > 0);
+
+        /*
+         * Next page.  Must check for interrupts and stuff here.
+         */
+        if (uCounterReg == 0)
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+    }
+
+    /*
+     * Done.
+     */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'REP STOS'.
+ */
+IEM_CIMPL_DEF_0(RT_CONCAT4(iemCImpl_stos_,OP_rAX,_m,ADDR_SIZE))
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ES);
+
+    uint64_t        uBaseAddr;
+    VBOXSTRICTRC rcStrict = iemMemSegCheckWriteAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uBaseAddr);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    OP_TYPE const   uValue      = pVCpu->cpum.GstCtx.OP_rAX;
+    ADDR_TYPE       uAddrReg    = pVCpu->cpum.GstCtx.ADDR_rDI;
+
+    /*
+     * Be careful with handle bypassing.
+     */
+    /** @todo Permit doing a page if correctly aligned. */
+    if (pVCpu->iem.s.fBypassHandlers)
+    {
+        Log(("%s: declining because we're bypassing handlers\n", __FUNCTION__));
+        return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
+    }
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr;
+        uint32_t    cLeftPage = (PAGE_SIZE - (uVirtAddr & PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftPage > uCounterReg)
+            cLeftPage = uCounterReg;
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uAddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, IEM_ACCESS_DATA_W, &GCPhysMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, do a block processing
+             * until the end of the current page.
+             */
+            PGMPAGEMAPLOCK PgLockMem;
+            OP_TYPE *puMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_W, (void **)&puMem, &PgLockMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                /* Update the regs first so we can loop on cLeftPage. */
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cLeftPage;
+                pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg    += cLeftPage * cbIncr;
+
+                /* Do the memsetting. */
+#if OP_SIZE == 8
+                memset(puMem, uValue, cLeftPage);
+/*#elif OP_SIZE == 32
+                ASMMemFill32(puMem, cLeftPage * (OP_SIZE / 8), uValue);*/
+#else
+                while (cLeftPage-- > 0)
+                    *puMem++ = uValue;
+#endif
+
+                iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_W, puMem, &PgLockMem);
+
+                if (uCounterReg == 0)
+                    break;
+
+                /* If unaligned, we drop thru and do the page crossing access
+                   below. Otherwise, do the next page. */
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+                    continue;
+                }
+                cLeftPage = 0;
+            }
+            /* If we got an invalid physical address in the page table, just skip
+               ahead to the next page or the counter reaches zero.  This crazy
+               optimization is for a buggy EFI firmware that's driving me nuts. */
+            else if (rcStrict == VERR_PGM_PHYS_TLB_UNASSIGNED)
+            {
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cLeftPage;
+                pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg    += cLeftPage * cbIncr;
+                if (uCounterReg == 0)
+                    break;
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+                    continue;
+                }
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            rcStrict = RT_CONCAT(iemMemStoreDataU,OP_SIZE)(pVCpu, X86_SREG_ES, uAddrReg, uValue);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            cLeftPage--;
+            IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0);
+        } while ((int32_t)cLeftPage > 0);
+
+        /*
+         * Next page.  Must check for interrupts and stuff here.
+         */
+        if (uCounterReg == 0)
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+    }
+
+    /*
+     * Done.
+     */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'REP LODS'.
+ */
+IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_lods_,OP_rAX,_m,ADDR_SIZE), int8_t, iEffSeg)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iEffSeg));
+    PCCPUMSELREGHID pSrcHid = iemSRegGetHid(pVCpu, iEffSeg);
+    uint64_t        uBaseAddr;
+    VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pSrcHid, iEffSeg, &uBaseAddr);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    ADDR_TYPE       uAddrReg    = pVCpu->cpum.GstCtx.ADDR_rSI;
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr;
+        uint32_t    cLeftPage = (PAGE_SIZE - (uVirtAddr & PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftPage > uCounterReg)
+            cLeftPage = uCounterReg;
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uAddrReg < pSrcHid->u32Limit
+                    && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pSrcHid->u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, IEM_ACCESS_DATA_R, &GCPhysMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, we can get away with
+             * just reading the last value on the page.
+             */
+            PGMPAGEMAPLOCK PgLockMem;
+            OP_TYPE const *puMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, (void **)&puMem, &PgLockMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                /* Only get the last byte, the rest doesn't matter in direct access mode. */
+#if OP_SIZE == 32
+                pVCpu->cpum.GstCtx.rax      = puMem[cLeftPage - 1];
+#else
+                pVCpu->cpum.GstCtx.OP_rAX   = puMem[cLeftPage - 1];
+#endif
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cLeftPage;
+                pVCpu->cpum.GstCtx.ADDR_rSI = uAddrReg    += cLeftPage * cbIncr;
+                iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, puMem, &PgLockMem);
+
+                if (uCounterReg == 0)
+                    break;
+
+                /* If unaligned, we drop thru and do the page crossing access
+                   below. Otherwise, do the next page. */
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+                    continue;
+                }
+                cLeftPage = 0;
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            OP_TYPE uTmpValue;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uTmpValue, iEffSeg, uAddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+#if OP_SIZE == 32
+            pVCpu->cpum.GstCtx.rax      = uTmpValue;
+#else
+            pVCpu->cpum.GstCtx.OP_rAX   = uTmpValue;
+#endif
+            pVCpu->cpum.GstCtx.ADDR_rSI = uAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            cLeftPage--;
+            IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0);
+        } while ((int32_t)cLeftPage > 0);
+
+        if (rcStrict != VINF_SUCCESS)
+            break;
+
+        /*
+         * Next page.  Must check for interrupts and stuff here.
+         */
+        if (uCounterReg == 0)
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+    }
+
+    /*
+     * Done.
+     */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+#if OP_SIZE != 64
+
+/**
+ * Implements 'INS' (no rep)
+ */
+IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_ins_op,OP_SIZE,_addr,ADDR_SIZE), bool, fIoChecked)
+{
+    PVMCC           pVM  = pVCpu->CTX_SUFF(pVM);
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * Be careful with handle bypassing.
+     */
+    if (pVCpu->iem.s.fBypassHandlers)
+    {
+        Log(("%s: declining because we're bypassing handlers\n", __FUNCTION__));
+        return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
+    }
+
+    /*
+     * ASSUMES the #GP for I/O permission is taken first, then any #GP for
+     * segmentation and finally any #PF due to virtual address translation.
+     * ASSUMES nothing is read from the I/O port before traps are taken.
+     */
+    if (!fIoChecked)
+    {
+        rcStrict = iemHlpCheckPortIOPermission(pVCpu, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * Check nested-guest I/O intercepts.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VMXEXITINSTRINFO ExitInstrInfo;
+        ExitInstrInfo.u = 0;
+        ExitInstrInfo.StrIo.u3AddrSize = ADDR_VMXSTRIO;
+        ExitInstrInfo.StrIo.iSegReg    = X86_SREG_ES;
+        rcStrict = iemVmxVmexitInstrStrIo(pVCpu, VMXINSTRID_IO_INS, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8, false /* fRep */,
+                                          ExitInstrInfo, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
+    {
+        rcStrict = iemSvmHandleIOIntercept(pVCpu, pVCpu->cpum.GstCtx.dx, SVMIOIOTYPE_IN, OP_SIZE / 8, ADDR_SIZE, X86_SREG_ES,
+                                           false /* fRep */, true /* fStrIo */, cbInstr);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("iemCImpl_ins_op: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", pVCpu->cpum.GstCtx.dx,
+                 OP_SIZE / 8, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    OP_TYPE        *puMem;
+    rcStrict = iemMemMap(pVCpu, (void **)&puMem, OP_SIZE / 8, X86_SREG_ES, pVCpu->cpum.GstCtx.ADDR_rDI, IEM_ACCESS_DATA_W);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    uint32_t        u32Value = 0;
+    rcStrict = IOMIOPortRead(pVM, pVCpu, pVCpu->cpum.GstCtx.dx, &u32Value, OP_SIZE / 8);
+    if (IOM_SUCCESS(rcStrict))
+    {
+        *puMem = (OP_TYPE)u32Value;
+# ifdef IN_RING3
+        VBOXSTRICTRC rcStrict2 = iemMemCommitAndUnmap(pVCpu, puMem, IEM_ACCESS_DATA_W);
+# else
+        VBOXSTRICTRC rcStrict2 = iemMemCommitAndUnmapPostponeTroubleToR3(pVCpu, puMem, IEM_ACCESS_DATA_W);
+# endif
+        if (RT_LIKELY(rcStrict2 == VINF_SUCCESS))
+        {
+            if (!pVCpu->cpum.GstCtx.eflags.Bits.u1DF)
+                pVCpu->cpum.GstCtx.ADDR_rDI += OP_SIZE / 8;
+            else
+                pVCpu->cpum.GstCtx.ADDR_rDI -= OP_SIZE / 8;
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        }
+        else
+            AssertLogRelMsgFailedReturn(("rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)), RT_FAILURE_NP(rcStrict2) ? rcStrict2 : VERR_IEM_IPE_1);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'REP INS'.
+ */
+IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_rep_ins_op,OP_SIZE,_addr,ADDR_SIZE), bool, fIoChecked)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_TR);
+
+    /*
+     * Setup.
+     */
+    uint16_t const  u16Port    = pVCpu->cpum.GstCtx.dx;
+    VBOXSTRICTRC    rcStrict;
+    if (!fIoChecked)
+    {
+/** @todo check if this is too early for ecx=0. */
+        rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, OP_SIZE / 8);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * Check nested-guest I/O intercepts.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VMXEXITINSTRINFO ExitInstrInfo;
+        ExitInstrInfo.u = 0;
+        ExitInstrInfo.StrIo.u3AddrSize = ADDR_VMXSTRIO;
+        ExitInstrInfo.StrIo.iSegReg    = X86_SREG_ES;
+        rcStrict = iemVmxVmexitInstrStrIo(pVCpu, VMXINSTRID_IO_INS, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8, true /* fRep */,
+                                          ExitInstrInfo, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
+    {
+        rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_IN, OP_SIZE / 8, ADDR_SIZE, X86_SREG_ES, true /* fRep */,
+                                           true /* fStrIo */, cbInstr);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("iemCImpl_rep_ins_op: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, OP_SIZE / 8,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    uint64_t        uBaseAddr;
+    rcStrict = iemMemSegCheckWriteAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uBaseAddr);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    ADDR_TYPE       uAddrReg    = pVCpu->cpum.GstCtx.ADDR_rDI;
+
+    /*
+     * Be careful with handle bypassing.
+     */
+    if (pVCpu->iem.s.fBypassHandlers)
+    {
+        Log(("%s: declining because we're bypassing handlers\n", __FUNCTION__));
+        return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
+    }
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr;
+        uint32_t    cLeftPage = (PAGE_SIZE - (uVirtAddr & PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftPage > uCounterReg)
+            cLeftPage = uCounterReg;
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uAddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, IEM_ACCESS_DATA_W, &GCPhysMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, use the IOM
+             * string I/O interface to do the work.
+             */
+            PGMPAGEMAPLOCK PgLockMem;
+            OP_TYPE *puMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_W, (void **)&puMem, &PgLockMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                uint32_t cTransfers = cLeftPage;
+                rcStrict = IOMIOPortReadString(pVM, pVCpu, u16Port, puMem, &cTransfers, OP_SIZE / 8);
+
+                uint32_t cActualTransfers = cLeftPage - cTransfers;
+                Assert(cActualTransfers <= cLeftPage);
+                pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg    += cbIncr * cActualTransfers;
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cActualTransfers;
+                puMem += cActualTransfers;
+
+                iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_W, puMem, &PgLockMem);
+
+                if (rcStrict != VINF_SUCCESS)
+                {
+                    if (IOM_SUCCESS(rcStrict))
+                    {
+                        rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                        if (uCounterReg == 0)
+                            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+                    }
+                    return rcStrict;
+                }
+
+                /* If unaligned, we drop thru and do the page crossing access
+                   below. Otherwise, do the next page. */
+                if (uCounterReg == 0)
+                    break;
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+                    continue;
+                }
+                cLeftPage = 0;
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         *
+         * Note! We ASSUME the CPU will raise #PF or #GP before access the
+         *       I/O port, otherwise it wouldn't really be restartable.
+         */
+        /** @todo investigate what the CPU actually does with \#PF/\#GP
+         *        during INS. */
+        do
+        {
+            OP_TYPE *puMem;
+            rcStrict = iemMemMap(pVCpu, (void **)&puMem, OP_SIZE / 8, X86_SREG_ES, uAddrReg, IEM_ACCESS_DATA_W);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            uint32_t u32Value = 0;
+            rcStrict = IOMIOPortRead(pVM, pVCpu, u16Port, &u32Value, OP_SIZE / 8);
+            if (!IOM_SUCCESS(rcStrict))
+            {
+                iemMemRollback(pVCpu);
+                return rcStrict;
+            }
+
+            *puMem = (OP_TYPE)u32Value;
+# ifdef IN_RING3
+            VBOXSTRICTRC rcStrict2 = iemMemCommitAndUnmap(pVCpu, puMem, IEM_ACCESS_DATA_W);
+# else
+            VBOXSTRICTRC rcStrict2 = iemMemCommitAndUnmapPostponeTroubleToR3(pVCpu, puMem, IEM_ACCESS_DATA_W);
+# endif
+            if (rcStrict2 == VINF_SUCCESS)
+            { /* likely */ }
+            else
+                AssertLogRelMsgFailedReturn(("rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
+                                            RT_FAILURE(rcStrict2) ? rcStrict2 : VERR_IEM_IPE_1);
+
+            pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+
+            cLeftPage--;
+            if (rcStrict != VINF_SUCCESS)
+            {
+                if (uCounterReg == 0)
+                    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+                rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                return rcStrict;
+            }
+
+            IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0);
+        } while ((int32_t)cLeftPage > 0);
+
+
+        /*
+         * Next page.  Must check for interrupts and stuff here.
+         */
+        if (uCounterReg == 0)
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+    }
+
+    /*
+     * Done.
+     */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'OUTS' (no rep)
+ */
+IEM_CIMPL_DEF_2(RT_CONCAT4(iemCImpl_outs_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg, bool, fIoChecked)
+{
+    PVMCC           pVM  = pVCpu->CTX_SUFF(pVM);
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * ASSUMES the #GP for I/O permission is taken first, then any #GP for
+     * segmentation and finally any #PF due to virtual address translation.
+     * ASSUMES nothing is read from the I/O port before traps are taken.
+     */
+    if (!fIoChecked)
+    {
+        rcStrict = iemHlpCheckPortIOPermission(pVCpu, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * Check nested-guest I/O intercepts.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VMXEXITINSTRINFO ExitInstrInfo;
+        ExitInstrInfo.u = 0;
+        ExitInstrInfo.StrIo.u3AddrSize = ADDR_VMXSTRIO;
+        ExitInstrInfo.StrIo.iSegReg    = iEffSeg;
+        rcStrict = iemVmxVmexitInstrStrIo(pVCpu, VMXINSTRID_IO_OUTS, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8, false /* fRep */,
+                                          ExitInstrInfo, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
+    {
+        rcStrict = iemSvmHandleIOIntercept(pVCpu, pVCpu->cpum.GstCtx.dx, SVMIOIOTYPE_OUT, OP_SIZE / 8, ADDR_SIZE, iEffSeg,
+                                           false /* fRep */, true /* fStrIo */, cbInstr);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("iemCImpl_outs_op: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", pVCpu->cpum.GstCtx.dx,
+                 OP_SIZE / 8, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    OP_TYPE uValue;
+    rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue, iEffSeg, pVCpu->cpum.GstCtx.ADDR_rSI);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = IOMIOPortWrite(pVM, pVCpu, pVCpu->cpum.GstCtx.dx, uValue, OP_SIZE / 8);
+        if (IOM_SUCCESS(rcStrict))
+        {
+            if (!pVCpu->cpum.GstCtx.eflags.Bits.u1DF)
+                pVCpu->cpum.GstCtx.ADDR_rSI += OP_SIZE / 8;
+            else
+                pVCpu->cpum.GstCtx.ADDR_rSI -= OP_SIZE / 8;
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            if (rcStrict != VINF_SUCCESS)
+                rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'REP OUTS'.
+ */
+IEM_CIMPL_DEF_2(RT_CONCAT4(iemCImpl_rep_outs_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg, bool, fIoChecked)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    uint16_t const  u16Port    = pVCpu->cpum.GstCtx.dx;
+    VBOXSTRICTRC    rcStrict;
+    if (!fIoChecked)
+    {
+/** @todo check if this is too early for ecx=0. */
+        rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, OP_SIZE / 8);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * Check nested-guest I/O intercepts.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VMXEXITINSTRINFO ExitInstrInfo;
+        ExitInstrInfo.u = 0;
+        ExitInstrInfo.StrIo.u3AddrSize = ADDR_VMXSTRIO;
+        ExitInstrInfo.StrIo.iSegReg    = iEffSeg;
+        rcStrict = iemVmxVmexitInstrStrIo(pVCpu, VMXINSTRID_IO_OUTS, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8, true /* fRep */,
+                                          ExitInstrInfo, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
+    {
+        rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_OUT, OP_SIZE / 8, ADDR_SIZE, iEffSeg, true /* fRep */,
+                                           true /* fStrIo */, cbInstr);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("iemCImpl_rep_outs_op: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, OP_SIZE / 8,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+    {
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    PCCPUMSELREGHID pHid = iemSRegGetHid(pVCpu, iEffSeg);
+    uint64_t        uBaseAddr;
+    rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pHid, iEffSeg, &uBaseAddr);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    ADDR_TYPE       uAddrReg    = pVCpu->cpum.GstCtx.ADDR_rSI;
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr;
+        uint32_t    cLeftPage = (PAGE_SIZE - (uVirtAddr & PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftPage > uCounterReg)
+            cLeftPage = uCounterReg;
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uAddrReg < pHid->u32Limit
+                    && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pHid->u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, IEM_ACCESS_DATA_R, &GCPhysMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, we use the IOM
+             * string I/O interface to do the job.
+             */
+            PGMPAGEMAPLOCK PgLockMem;
+            OP_TYPE const *puMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, (void **)&puMem, &PgLockMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                uint32_t cTransfers = cLeftPage;
+                rcStrict = IOMIOPortWriteString(pVM, pVCpu, u16Port, puMem, &cTransfers, OP_SIZE / 8);
+
+                uint32_t cActualTransfers = cLeftPage - cTransfers;
+                Assert(cActualTransfers <= cLeftPage);
+                pVCpu->cpum.GstCtx.ADDR_rSI = uAddrReg    += cbIncr * cActualTransfers;
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cActualTransfers;
+                puMem += cActualTransfers;
+
+                iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, puMem, &PgLockMem);
+
+                if (rcStrict != VINF_SUCCESS)
+                {
+                    if (IOM_SUCCESS(rcStrict))
+                    {
+                        rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                        if (uCounterReg == 0)
+                            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+                    }
+                    return rcStrict;
+                }
+
+                if (uCounterReg == 0)
+                    break;
+
+                /* If unaligned, we drop thru and do the page crossing access
+                   below. Otherwise, do the next page. */
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+                    continue;
+                }
+                cLeftPage = 0;
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         *
+         * Note! We ASSUME the CPU will raise #PF or #GP before access the
+         *       I/O port, otherwise it wouldn't really be restartable.
+         */
+        /** @todo investigate what the CPU actually does with \#PF/\#GP
+         *        during INS. */
+        do
+        {
+            OP_TYPE uValue;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue, iEffSeg, uAddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            rcStrict = IOMIOPortWrite(pVM, pVCpu, u16Port, uValue, OP_SIZE / 8);
+            if (IOM_SUCCESS(rcStrict))
+            {
+                pVCpu->cpum.GstCtx.ADDR_rSI = uAddrReg += cbIncr;
+                pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+                cLeftPage--;
+            }
+            if (rcStrict != VINF_SUCCESS)
+            {
+                if (IOM_SUCCESS(rcStrict))
+                {
+                    if (uCounterReg == 0)
+                        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+                    rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                }
+                return rcStrict;
+            }
+            IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0);
+        } while ((int32_t)cLeftPage > 0);
+
+
+        /*
+         * Next page.  Must check for interrupts and stuff here.
+         */
+        if (uCounterReg == 0)
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+    }
+
+    /*
+     * Done.
+     */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+#endif /* OP_SIZE != 64-bit */
+
+
+#undef OP_rAX
+#undef OP_SIZE
+#undef ADDR_SIZE
+#undef ADDR_rDI
+#undef ADDR_rSI
+#undef ADDR_rCX
+#undef ADDR_rIP
+#undef ADDR2_TYPE
+#undef ADDR_TYPE
+#undef ADDR2_TYPE
+#undef ADDR_VMXSTRIO
+#undef IS_64_BIT_CODE
+#undef IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN
+#undef IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN
+#undef IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp.h b/src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp.h
new file mode 100644
index 00000000..d0bb9cb6
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp.h
@@ -0,0 +1,1430 @@
+/* $Id: IEMAllCImplSvmInstr.cpp.h $ */
+/** @file
+ * IEM - AMD-V (Secure Virtual Machine) instruction implementation.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+/**
+ * Check the common SVM instruction preconditions.
+ */
+# define IEM_SVM_INSTR_COMMON_CHECKS(a_pVCpu, a_Instr) \
+    do { \
+        if (!CPUMIsGuestSvmEnabled(IEM_GET_CTX(a_pVCpu))) \
+        { \
+            Log((RT_STR(a_Instr) ": EFER.SVME not enabled -> #UD\n")); \
+            return iemRaiseUndefinedOpcode(a_pVCpu); \
+        } \
+        if (IEM_IS_REAL_OR_V86_MODE(a_pVCpu)) \
+        { \
+            Log((RT_STR(a_Instr) ": Real or v8086 mode -> #UD\n")); \
+            return iemRaiseUndefinedOpcode(a_pVCpu); \
+        } \
+        if ((a_pVCpu)->iem.s.uCpl != 0) \
+        { \
+            Log((RT_STR(a_Instr) ": CPL != 0 -> #GP(0)\n")); \
+            return iemRaiseGeneralProtectionFault0(a_pVCpu); \
+        } \
+    } while (0)
+
+
+/**
+ * Converts an IEM exception event type to an SVM event type.
+ *
+ * @returns The SVM event type.
+ * @retval  UINT8_MAX if the specified type of event isn't among the set
+ *          of recognized IEM event types.
+ *
+ * @param   uVector         The vector of the event.
+ * @param   fIemXcptFlags   The IEM exception / interrupt flags.
+ */
+IEM_STATIC uint8_t iemGetSvmEventType(uint32_t uVector, uint32_t fIemXcptFlags)
+{
+    if (fIemXcptFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+    {
+        if (uVector != X86_XCPT_NMI)
+            return SVM_EVENT_EXCEPTION;
+        return SVM_EVENT_NMI;
+    }
+
+    /* See AMD spec. Table 15-1. "Guest Exception or Interrupt Types". */
+    if (fIemXcptFlags & (IEM_XCPT_FLAGS_BP_INSTR | IEM_XCPT_FLAGS_ICEBP_INSTR | IEM_XCPT_FLAGS_OF_INSTR))
+        return SVM_EVENT_EXCEPTION;
+
+    if (fIemXcptFlags & IEM_XCPT_FLAGS_T_EXT_INT)
+        return SVM_EVENT_EXTERNAL_IRQ;
+
+    if (fIemXcptFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
+        return SVM_EVENT_SOFTWARE_INT;
+
+    AssertMsgFailed(("iemGetSvmEventType: Invalid IEM xcpt/int. type %#x, uVector=%#x\n", fIemXcptFlags, uVector));
+    return UINT8_MAX;
+}
+
+
+/**
+ * Performs an SVM world-switch (VMRUN, \#VMEXIT) updating PGM and IEM internals.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(VBOXSTRICTRC) iemSvmWorldSwitch(PVMCPUCC pVCpu)
+{
+    /*
+     * Inform PGM about paging mode changes.
+     * We include X86_CR0_PE because PGM doesn't handle paged-real mode yet,
+     * see comment in iemMemPageTranslateAndCheckAccess().
+     */
+    int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0 | X86_CR0_PE, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER);
+# ifdef IN_RING3
+    Assert(rc != VINF_PGM_CHANGE_MODE);
+# endif
+    AssertRCReturn(rc, rc);
+
+    /* Inform CPUM (recompiler), can later be removed. */
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
+
+    /*
+     * Flush the TLB with new CR3. This is required in case the PGM mode change
+     * above doesn't actually change anything.
+     */
+    if (rc == VINF_SUCCESS)
+    {
+        rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true);
+        AssertRCReturn(rc, rc);
+    }
+
+    /* Re-initialize IEM cache/state after the drastic mode switch. */
+    iemReInitExec(pVCpu);
+    return rc;
+}
+
+
+/**
+ * SVM \#VMEXIT handler.
+ *
+ * @returns Strict VBox status code.
+ * @retval VINF_SVM_VMEXIT when the \#VMEXIT is successful.
+ * @retval VERR_SVM_VMEXIT_FAILED when the \#VMEXIT failed restoring the guest's
+ *         "host state" and a shutdown is required.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uExitCode   The exit code.
+ * @param   uExitInfo1  The exit info. 1 field.
+ * @param   uExitInfo2  The exit info. 2 field.
+ */
+IEM_STATIC VBOXSTRICTRC iemSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2)
+{
+    VBOXSTRICTRC rcStrict;
+    if (   CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))
+        || uExitCode == SVM_EXIT_INVALID)
+    {
+        LogFlow(("iemSvmVmexit: CS:RIP=%04x:%08RX64 uExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n",
+                 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode, uExitInfo1, uExitInfo2));
+
+        /*
+         * Disable the global-interrupt flag to prevent interrupts during the 'atomic' world switch.
+         */
+        CPUMSetGuestGif(&pVCpu->cpum.GstCtx, false);
+
+        /*
+         * Map the nested-guest VMCB from its location in guest memory.
+         * Write exactly what the CPU does on #VMEXIT thereby preserving most other bits in the
+         * guest's VMCB in memory, see @bugref{7243#c113} and related comment on iemSvmVmrun().
+         */
+        PSVMVMCB       pVmcbMem;
+        PGMPAGEMAPLOCK PgLockMem;
+        PSVMVMCBCTRL   pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb)->ctrl;
+        rcStrict = iemMemPageMap(pVCpu, pVCpu->cpum.GstCtx.hwvirt.svm.GCPhysVmcb, IEM_ACCESS_DATA_RW, (void **)&pVmcbMem,
+                                 &PgLockMem);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            /*
+             * Notify HM in case the nested-guest was executed using hardware-assisted SVM (which
+             * would have modified some VMCB state) that might need to be restored on #VMEXIT before
+             * writing the VMCB back to guest memory.
+             */
+            HMNotifySvmNstGstVmexit(pVCpu, IEM_GET_CTX(pVCpu));
+
+            Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
+            Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+            Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+            Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
+
+            /*
+             * Save the nested-guest state into the VMCB state-save area.
+             */
+            PSVMVMCBSTATESAVE pVmcbMemState = &pVmcbMem->guest;
+            HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), pVmcbMemState, ES, es);
+            HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), pVmcbMemState, CS, cs);
+            HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), pVmcbMemState, SS, ss);
+            HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), pVmcbMemState, DS, ds);
+            pVmcbMemState->GDTR.u32Limit   = pVCpu->cpum.GstCtx.gdtr.cbGdt;
+            pVmcbMemState->GDTR.u64Base    = pVCpu->cpum.GstCtx.gdtr.pGdt;
+            pVmcbMemState->IDTR.u32Limit   = pVCpu->cpum.GstCtx.idtr.cbIdt;
+            pVmcbMemState->IDTR.u64Base    = pVCpu->cpum.GstCtx.idtr.pIdt;
+            pVmcbMemState->u64EFER         = pVCpu->cpum.GstCtx.msrEFER;
+            pVmcbMemState->u64CR4          = pVCpu->cpum.GstCtx.cr4;
+            pVmcbMemState->u64CR3          = pVCpu->cpum.GstCtx.cr3;
+            pVmcbMemState->u64CR2          = pVCpu->cpum.GstCtx.cr2;
+            pVmcbMemState->u64CR0          = pVCpu->cpum.GstCtx.cr0;
+            /** @todo Nested paging. */
+            pVmcbMemState->u64RFlags       = pVCpu->cpum.GstCtx.rflags.u64;
+            pVmcbMemState->u64RIP          = pVCpu->cpum.GstCtx.rip;
+            pVmcbMemState->u64RSP          = pVCpu->cpum.GstCtx.rsp;
+            pVmcbMemState->u64RAX          = pVCpu->cpum.GstCtx.rax;
+            pVmcbMemState->u64DR7          = pVCpu->cpum.GstCtx.dr[7];
+            pVmcbMemState->u64DR6          = pVCpu->cpum.GstCtx.dr[6];
+            pVmcbMemState->u8CPL           = pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl;   /* See comment in CPUMGetGuestCPL(). */
+            Assert(CPUMGetGuestCPL(pVCpu) == pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl);
+            if (CPUMIsGuestSvmNestedPagingEnabled(pVCpu, IEM_GET_CTX(pVCpu)))
+                pVmcbMemState->u64PAT = pVCpu->cpum.GstCtx.msrPAT;
+
+            /*
+             * Save additional state and intercept information.
+             *
+             *   - V_IRQ: Tracked using VMCPU_FF_INTERRUPT_NESTED_GUEST force-flag and updated below.
+             *   - V_TPR: Updated by iemCImpl_load_CrX or by the physical CPU for hardware-assisted
+             *     SVM execution.
+             *   - Interrupt shadow: Tracked using VMCPU_FF_INHIBIT_INTERRUPTS and RIP.
+             */
+            PSVMVMCBCTRL pVmcbMemCtrl = &pVmcbMem->ctrl;
+            if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))           /* V_IRQ. */
+                pVmcbMemCtrl->IntCtrl.n.u1VIrqPending = 0;
+            else
+            {
+                Assert(pVmcbCtrl->IntCtrl.n.u1VIrqPending);
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
+            }
+
+            pVmcbMemCtrl->IntCtrl.n.u8VTPR = pVmcbCtrl->IntCtrl.n.u8VTPR;           /* V_TPR. */
+
+            if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)              /* Interrupt shadow. */
+                && EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip)
+            {
+                pVmcbMemCtrl->IntShadow.n.u1IntShadow = 1;
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+                LogFlow(("iemSvmVmexit: Interrupt shadow till %#RX64\n", pVCpu->cpum.GstCtx.rip));
+            }
+            else
+                pVmcbMemCtrl->IntShadow.n.u1IntShadow = 0;
+
+            /*
+             * Save nRIP, instruction length and byte fields.
+             */
+            pVmcbMemCtrl->u64NextRIP     = pVmcbCtrl->u64NextRIP;
+            pVmcbMemCtrl->cbInstrFetched = pVmcbCtrl->cbInstrFetched;
+            memcpy(&pVmcbMemCtrl->abInstr[0], &pVmcbCtrl->abInstr[0], sizeof(pVmcbMemCtrl->abInstr));
+
+            /*
+             * Save exit information.
+             */
+            pVmcbMemCtrl->u64ExitCode  = uExitCode;
+            pVmcbMemCtrl->u64ExitInfo1 = uExitInfo1;
+            pVmcbMemCtrl->u64ExitInfo2 = uExitInfo2;
+
+            /*
+             * Update the exit interrupt-information field if this #VMEXIT happened as a result
+             * of delivering an event through IEM.
+             *
+             * Don't update the exit interrupt-information field if the event wasn't being injected
+             * through IEM, as it would have been updated by real hardware if the nested-guest was
+             * executed using hardware-assisted SVM.
+             */
+            {
+                uint8_t  uExitIntVector;
+                uint32_t uExitIntErr;
+                uint32_t fExitIntFlags;
+                bool const fRaisingEvent = IEMGetCurrentXcpt(pVCpu, &uExitIntVector, &fExitIntFlags, &uExitIntErr,
+                                                             NULL /* uExitIntCr2 */);
+                if (fRaisingEvent)
+                {
+                    pVmcbCtrl->ExitIntInfo.n.u1Valid  = 1;
+                    pVmcbCtrl->ExitIntInfo.n.u8Vector = uExitIntVector;
+                    pVmcbCtrl->ExitIntInfo.n.u3Type   = iemGetSvmEventType(uExitIntVector, fExitIntFlags);
+                    if (fExitIntFlags & IEM_XCPT_FLAGS_ERR)
+                    {
+                        pVmcbCtrl->ExitIntInfo.n.u1ErrorCodeValid = true;
+                        pVmcbCtrl->ExitIntInfo.n.u32ErrorCode     = uExitIntErr;
+                    }
+                }
+            }
+
+            /*
+             * Save the exit interrupt-information field.
+             *
+             * We write the whole field including overwriting reserved bits as it was observed on an
+             * AMD Ryzen 5 Pro 1500 that the CPU does not preserve reserved bits in EXITINTINFO.
+             */
+            pVmcbMemCtrl->ExitIntInfo = pVmcbCtrl->ExitIntInfo;
+
+            /*
+             * Clear event injection.
+             */
+            pVmcbMemCtrl->EventInject.n.u1Valid = 0;
+
+            iemMemPageUnmap(pVCpu, pVCpu->cpum.GstCtx.hwvirt.svm.GCPhysVmcb, IEM_ACCESS_DATA_RW, pVmcbMem, &PgLockMem);
+        }
+
+        /*
+         * Prepare for guest's "host mode" by clearing internal processor state bits.
+         *
+         * We don't need to zero out the state-save area, just the controls should be
+         * sufficient because it has the critical bit of indicating whether we're inside
+         * the nested-guest or not.
+         */
+        memset(pVmcbCtrl, 0, sizeof(*pVmcbCtrl));
+        Assert(!CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
+
+        /*
+         * Restore the subset of force-flags that were preserved.
+         */
+        if (pVCpu->cpum.GstCtx.hwvirt.fLocalForcedActions)
+        {
+            VMCPU_FF_SET_MASK(pVCpu, pVCpu->cpum.GstCtx.hwvirt.fLocalForcedActions);
+            pVCpu->cpum.GstCtx.hwvirt.fLocalForcedActions = 0;
+        }
+
+        if (rcStrict == VINF_SUCCESS)
+        {
+            /** @todo Nested paging. */
+            /** @todo ASID. */
+
+            /*
+             * Reload the guest's "host state".
+             */
+            CPUMSvmVmExitRestoreHostState(pVCpu, IEM_GET_CTX(pVCpu));
+
+            /*
+             * Update PGM, IEM and others of a world-switch.
+             */
+            rcStrict = iemSvmWorldSwitch(pVCpu);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = VINF_SVM_VMEXIT;
+            else if (RT_SUCCESS(rcStrict))
+            {
+                LogFlow(("iemSvmVmexit: Setting passup status from iemSvmWorldSwitch %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+                iemSetPassUpStatus(pVCpu, rcStrict);
+                rcStrict = VINF_SVM_VMEXIT;
+            }
+            else
+                LogFlow(("iemSvmVmexit: iemSvmWorldSwitch unexpected failure. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+        }
+        else
+        {
+            AssertMsgFailed(("iemSvmVmexit: Mapping VMCB at %#RGp failed. rc=%Rrc\n", pVCpu->cpum.GstCtx.hwvirt.svm.GCPhysVmcb, VBOXSTRICTRC_VAL(rcStrict)));
+            rcStrict = VERR_SVM_VMEXIT_FAILED;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("iemSvmVmexit: Not in SVM guest mode! uExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitCode, uExitInfo1, uExitInfo2));
+        rcStrict = VERR_SVM_IPE_3;
+    }
+
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3)
+    /* CLGI/STGI may not have been intercepted and thus not executed in IEM. */
+    if (   HMIsEnabled(pVCpu->CTX_SUFF(pVM))
+        && HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM)))
+        return EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, false);
+# endif
+    return rcStrict;
+}
+
+
+/**
+ * Performs the operations necessary that are part of the vmrun instruction
+ * execution in the guest.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_SUCCESS successfully executed VMRUN and entered nested-guest
+ *          code execution.
+ * @retval  VINF_SVM_VMEXIT when executing VMRUN causes a \#VMEXIT
+ *          (SVM_EXIT_INVALID most likely).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The length of the VMRUN instruction.
+ * @param   GCPhysVmcb          Guest physical address of the VMCB to run.
+ */
+IEM_STATIC VBOXSTRICTRC iemSvmVmrun(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPHYS GCPhysVmcb)
+{
+    LogFlow(("iemSvmVmrun\n"));
+
+    /*
+     * Cache the physical address of the VMCB for #VMEXIT exceptions.
+     */
+    pVCpu->cpum.GstCtx.hwvirt.svm.GCPhysVmcb = GCPhysVmcb;
+
+    /*
+     * Save the host state.
+     */
+    CPUMSvmVmRunSaveHostState(IEM_GET_CTX(pVCpu), cbInstr);
+
+    /*
+     * Read the guest VMCB.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    int rc = PGMPhysSimpleReadGCPhys(pVM, pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb), GCPhysVmcb, sizeof(SVMVMCB));
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * AMD-V seems to preserve reserved fields and only writes back selected, recognized
+         * fields on #VMEXIT. However, not all reserved  bits are preserved (e.g, EXITINTINFO)
+         * but in our implementation we try to preserve as much as we possibly can.
+         *
+         * We could read the entire page here and only write back the relevant fields on
+         * #VMEXIT but since our internal VMCB is also being used by HM during hardware-assisted
+         * SVM execution, it creates a potential for a nested-hypervisor to set bits that are
+         * currently reserved but may be recognized as features bits in future CPUs causing
+         * unexpected & undesired results. Hence, we zero out unrecognized fields here as we
+         * typically enter hardware-assisted SVM soon anyway, see @bugref{7243#c113}.
+         */
+        PSVMVMCBCTRL      pVmcbCtrl   = &pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb)->ctrl;
+        PSVMVMCBSTATESAVE pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb)->guest;
+
+        RT_ZERO(pVmcbCtrl->u8Reserved0);
+        RT_ZERO(pVmcbCtrl->u8Reserved1);
+        RT_ZERO(pVmcbCtrl->u8Reserved2);
+        RT_ZERO(pVmcbNstGst->u8Reserved0);
+        RT_ZERO(pVmcbNstGst->u8Reserved1);
+        RT_ZERO(pVmcbNstGst->u8Reserved2);
+        RT_ZERO(pVmcbNstGst->u8Reserved3);
+        RT_ZERO(pVmcbNstGst->u8Reserved4);
+        RT_ZERO(pVmcbNstGst->u8Reserved5);
+        pVmcbCtrl->u32Reserved0                   = 0;
+        pVmcbCtrl->TLBCtrl.n.u24Reserved          = 0;
+        pVmcbCtrl->IntCtrl.n.u6Reserved           = 0;
+        pVmcbCtrl->IntCtrl.n.u3Reserved           = 0;
+        pVmcbCtrl->IntCtrl.n.u5Reserved           = 0;
+        pVmcbCtrl->IntCtrl.n.u24Reserved          = 0;
+        pVmcbCtrl->IntShadow.n.u30Reserved        = 0;
+        pVmcbCtrl->ExitIntInfo.n.u19Reserved      = 0;
+        pVmcbCtrl->NestedPagingCtrl.n.u29Reserved = 0;
+        pVmcbCtrl->EventInject.n.u19Reserved      = 0;
+        pVmcbCtrl->LbrVirt.n.u30Reserved          = 0;
+
+        /*
+         * Validate guest-state and controls.
+         */
+        /* VMRUN must always be intercepted. */
+        if (!CPUMIsGuestSvmCtrlInterceptSet(pVCpu, IEM_GET_CTX(pVCpu), SVM_CTRL_INTERCEPT_VMRUN))
+        {
+            Log(("iemSvmVmrun: VMRUN instruction not intercepted -> #VMEXIT\n"));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /* Nested paging. */
+        if (    pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
+            && !pVM->cpum.ro.GuestFeatures.fSvmNestedPaging)
+        {
+            Log(("iemSvmVmrun: Nested paging not supported -> Disabling\n"));
+            pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging = 0;
+        }
+
+        /* AVIC. */
+        if (    pVmcbCtrl->IntCtrl.n.u1AvicEnable
+            && !pVM->cpum.ro.GuestFeatures.fSvmAvic)
+        {
+            Log(("iemSvmVmrun: AVIC not supported -> Disabling\n"));
+            pVmcbCtrl->IntCtrl.n.u1AvicEnable = 0;
+        }
+
+        /* Last branch record (LBR) virtualization. */
+        if (    pVmcbCtrl->LbrVirt.n.u1LbrVirt
+            && !pVM->cpum.ro.GuestFeatures.fSvmLbrVirt)
+        {
+            Log(("iemSvmVmrun: LBR virtualization not supported -> Disabling\n"));
+            pVmcbCtrl->LbrVirt.n.u1LbrVirt = 0;
+        }
+
+        /* Virtualized VMSAVE/VMLOAD. */
+        if (    pVmcbCtrl->LbrVirt.n.u1VirtVmsaveVmload
+            && !pVM->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
+        {
+            Log(("iemSvmVmrun: Virtualized VMSAVE/VMLOAD not supported -> Disabling\n"));
+            pVmcbCtrl->LbrVirt.n.u1VirtVmsaveVmload = 0;
+        }
+
+        /* Virtual GIF. */
+        if (    pVmcbCtrl->IntCtrl.n.u1VGifEnable
+            && !pVM->cpum.ro.GuestFeatures.fSvmVGif)
+        {
+            Log(("iemSvmVmrun: Virtual GIF not supported -> Disabling\n"));
+            pVmcbCtrl->IntCtrl.n.u1VGifEnable = 0;
+        }
+
+        /* Guest ASID. */
+        if (!pVmcbCtrl->TLBCtrl.n.u32ASID)
+        {
+            Log(("iemSvmVmrun: Guest ASID is invalid -> #VMEXIT\n"));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /* Guest AVIC. */
+        if (    pVmcbCtrl->IntCtrl.n.u1AvicEnable
+            && !pVM->cpum.ro.GuestFeatures.fSvmAvic)
+        {
+            Log(("iemSvmVmrun: AVIC not supported -> Disabling\n"));
+            pVmcbCtrl->IntCtrl.n.u1AvicEnable = 0;
+        }
+
+        /* Guest Secure Encrypted Virtualization. */
+        if (  (   pVmcbCtrl->NestedPagingCtrl.n.u1Sev
+               || pVmcbCtrl->NestedPagingCtrl.n.u1SevEs)
+            && !pVM->cpum.ro.GuestFeatures.fSvmAvic)
+        {
+            Log(("iemSvmVmrun: SEV not supported -> Disabling\n"));
+            pVmcbCtrl->NestedPagingCtrl.n.u1Sev = 0;
+            pVmcbCtrl->NestedPagingCtrl.n.u1SevEs = 0;
+        }
+
+        /* Flush by ASID. */
+        if (   !pVM->cpum.ro.GuestFeatures.fSvmFlusbByAsid
+            &&  pVmcbCtrl->TLBCtrl.n.u8TLBFlush != SVM_TLB_FLUSH_NOTHING
+            &&  pVmcbCtrl->TLBCtrl.n.u8TLBFlush != SVM_TLB_FLUSH_ENTIRE)
+        {
+            Log(("iemSvmVmrun: Flush-by-ASID not supported -> #VMEXIT\n"));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /* IO permission bitmap. */
+        RTGCPHYS const GCPhysIOBitmap = pVmcbCtrl->u64IOPMPhysAddr;
+        if (   (GCPhysIOBitmap & X86_PAGE_4K_OFFSET_MASK)
+            || !PGMPhysIsGCPhysNormal(pVM, GCPhysIOBitmap)
+            || !PGMPhysIsGCPhysNormal(pVM, GCPhysIOBitmap + X86_PAGE_4K_SIZE)
+            || !PGMPhysIsGCPhysNormal(pVM, GCPhysIOBitmap + (X86_PAGE_4K_SIZE << 1)))
+        {
+            Log(("iemSvmVmrun: IO bitmap physaddr invalid. GCPhysIOBitmap=%#RX64 -> #VMEXIT\n", GCPhysIOBitmap));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /* MSR permission bitmap. */
+        RTGCPHYS const GCPhysMsrBitmap = pVmcbCtrl->u64MSRPMPhysAddr;
+        if (   (GCPhysMsrBitmap & X86_PAGE_4K_OFFSET_MASK)
+            || !PGMPhysIsGCPhysNormal(pVM, GCPhysMsrBitmap)
+            || !PGMPhysIsGCPhysNormal(pVM, GCPhysMsrBitmap + X86_PAGE_4K_SIZE))
+        {
+            Log(("iemSvmVmrun: MSR bitmap physaddr invalid. GCPhysMsrBitmap=%#RX64 -> #VMEXIT\n", GCPhysMsrBitmap));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /* CR0. */
+        if (   !(pVmcbNstGst->u64CR0 & X86_CR0_CD)
+            &&  (pVmcbNstGst->u64CR0 & X86_CR0_NW))
+        {
+            Log(("iemSvmVmrun: CR0 no-write through with cache disabled. CR0=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64CR0));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+        if (pVmcbNstGst->u64CR0 >> 32)
+        {
+            Log(("iemSvmVmrun: CR0 reserved bits set. CR0=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64CR0));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+        /** @todo Implement all reserved bits/illegal combinations for CR3, CR4. */
+
+        /* DR6 and DR7. */
+        if (   pVmcbNstGst->u64DR6 >> 32
+            || pVmcbNstGst->u64DR7 >> 32)
+        {
+            Log(("iemSvmVmrun: DR6 and/or DR7 reserved bits set. DR6=%#RX64 DR7=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64DR6,
+                 pVmcbNstGst->u64DR6));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /*
+         * PAT (Page Attribute Table) MSR.
+         *
+         * The CPU only validates and loads it when nested-paging is enabled.
+         * See AMD spec. "15.25.4 Nested Paging and VMRUN/#VMEXIT".
+         */
+        if (   pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
+            && !CPUMIsPatMsrValid(pVmcbNstGst->u64PAT))
+        {
+            Log(("iemSvmVmrun: PAT invalid. u64PAT=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64PAT));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /*
+         * Copy the IO permission bitmap into the cache.
+         */
+        Assert(pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvIoBitmap));
+        rc = PGMPhysSimpleReadGCPhys(pVM, pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvIoBitmap), GCPhysIOBitmap,
+                                     SVM_IOPM_PAGES * X86_PAGE_4K_SIZE);
+        if (RT_FAILURE(rc))
+        {
+            Log(("iemSvmVmrun: Failed reading the IO permission bitmap at %#RGp. rc=%Rrc\n", GCPhysIOBitmap, rc));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /*
+         * Copy the MSR permission bitmap into the cache.
+         */
+        Assert(pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap));
+        rc = PGMPhysSimpleReadGCPhys(pVM, pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap), GCPhysMsrBitmap,
+                                     SVM_MSRPM_PAGES * X86_PAGE_4K_SIZE);
+        if (RT_FAILURE(rc))
+        {
+            Log(("iemSvmVmrun: Failed reading the MSR permission bitmap at %#RGp. rc=%Rrc\n", GCPhysMsrBitmap, rc));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /*
+         * Copy segments from nested-guest VMCB state to the guest-CPU state.
+         *
+         * We do this here as we need to use the CS attributes and it's easier this way
+         * then using the VMCB format selectors. It doesn't really matter where we copy
+         * the state, we restore the guest-CPU context state on the \#VMEXIT anyway.
+         */
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), pVmcbNstGst, ES, es);
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), pVmcbNstGst, CS, cs);
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), pVmcbNstGst, SS, ss);
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), pVmcbNstGst, DS, ds);
+
+        /** @todo Segment attribute overrides by VMRUN. */
+
+        /*
+         * CPL adjustments and overrides.
+         *
+         * SS.DPL is apparently the CPU's CPL, see comment in CPUMGetGuestCPL().
+         * We shall thus adjust both CS.DPL and SS.DPL here.
+         */
+        pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl = pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl = pVmcbNstGst->u8CPL;
+        if (CPUMIsGuestInV86ModeEx(IEM_GET_CTX(pVCpu)))
+            pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl = pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl = 3;
+        if (CPUMIsGuestInRealModeEx(IEM_GET_CTX(pVCpu)))
+            pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl = pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl = 0;
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+
+        /*
+         * Continue validating guest-state and controls.
+         *
+         * We pass CR0 as 0 to CPUMIsGuestEferMsrWriteValid() below to skip the illegal
+         * EFER.LME bit transition check. We pass the nested-guest's EFER as both the
+         * old and new EFER value to not have any guest EFER bits influence the new
+         * nested-guest EFER.
+         */
+        uint64_t uValidEfer;
+        rc = CPUMIsGuestEferMsrWriteValid(pVM, 0 /* CR0 */, pVmcbNstGst->u64EFER, pVmcbNstGst->u64EFER, &uValidEfer);
+        if (RT_FAILURE(rc))
+        {
+            Log(("iemSvmVmrun: EFER invalid uOldEfer=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64EFER));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /* Validate paging and CPU mode bits. */
+        bool const fSvm                     = RT_BOOL(uValidEfer & MSR_K6_EFER_SVME);
+        bool const fLongModeSupported       = RT_BOOL(pVM->cpum.ro.GuestFeatures.fLongMode);
+        bool const fLongModeEnabled         = RT_BOOL(uValidEfer & MSR_K6_EFER_LME);
+        bool const fPaging                  = RT_BOOL(pVmcbNstGst->u64CR0 & X86_CR0_PG);
+        bool const fPae                     = RT_BOOL(pVmcbNstGst->u64CR4 & X86_CR4_PAE);
+        bool const fProtMode                = RT_BOOL(pVmcbNstGst->u64CR0 & X86_CR0_PE);
+        bool const fLongModeWithPaging      = fLongModeEnabled && fPaging;
+        bool const fLongModeConformCS       = pVCpu->cpum.GstCtx.cs.Attr.n.u1Long && pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig;
+        /* Adjust EFER.LMA (this is normally done by the CPU when system software writes CR0). */
+        if (fLongModeWithPaging)
+            uValidEfer |= MSR_K6_EFER_LMA;
+        bool const fLongModeActiveOrEnabled = RT_BOOL(uValidEfer & (MSR_K6_EFER_LME | MSR_K6_EFER_LMA));
+        if (   !fSvm
+            || (!fLongModeSupported && fLongModeActiveOrEnabled)
+            || (fLongModeWithPaging && !fPae)
+            || (fLongModeWithPaging && !fProtMode)
+            || (   fLongModeEnabled
+                && fPaging
+                && fPae
+                && fLongModeConformCS))
+        {
+            Log(("iemSvmVmrun: EFER invalid. uValidEfer=%#RX64 -> #VMEXIT\n", uValidEfer));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /*
+         * Preserve the required force-flags.
+         *
+         * We only preserve the force-flags that would affect the execution of the
+         * nested-guest (or the guest).
+         *
+         *   - VMCPU_FF_BLOCK_NMIS needs to be preserved as it blocks NMI until the
+         *     execution of a subsequent IRET instruction in the guest.
+         *
+         * The remaining FFs (e.g. timers) can stay in place so that we will be able to
+         * generate interrupts that should cause #VMEXITs for the nested-guest.
+         *
+         * VMRUN has implicit GIF (Global Interrupt Flag) handling, we don't need to
+         * preserve VMCPU_FF_INHIBIT_INTERRUPTS.
+         */
+        pVCpu->cpum.GstCtx.hwvirt.fLocalForcedActions = pVCpu->fLocalForcedActions & VMCPU_FF_BLOCK_NMIS;
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
+
+        /*
+         * Pause filter.
+         */
+        if (pVM->cpum.ro.GuestFeatures.fSvmPauseFilter)
+        {
+            pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilter = pVmcbCtrl->u16PauseFilterCount;
+            if (pVM->cpum.ro.GuestFeatures.fSvmPauseFilterThreshold)
+                pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilterThreshold = pVmcbCtrl->u16PauseFilterCount;
+        }
+
+        /*
+         * Interrupt shadow.
+         */
+        if (pVmcbCtrl->IntShadow.n.u1IntShadow)
+        {
+            LogFlow(("iemSvmVmrun: setting interrupt shadow. inhibit PC=%#RX64\n", pVmcbNstGst->u64RIP));
+            /** @todo will this cause trouble if the nested-guest is 64-bit but the guest is 32-bit? */
+            EMSetInhibitInterruptsPC(pVCpu, pVmcbNstGst->u64RIP);
+        }
+
+        /*
+         * TLB flush control.
+         * Currently disabled since it's redundant as we unconditionally flush the TLB
+         * in iemSvmWorldSwitch() below.
+         */
+# if 0
+        /** @todo @bugref{7243}: ASID based PGM TLB flushes. */
+        if (   pVmcbCtrl->TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE
+            || pVmcbCtrl->TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
+            || pVmcbCtrl->TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
+            PGMFlushTLB(pVCpu, pVmcbNstGst->u64CR3, true /* fGlobal */);
+# endif
+
+        /*
+         * Copy the remaining guest state from the VMCB to the guest-CPU context.
+         */
+        pVCpu->cpum.GstCtx.gdtr.cbGdt = pVmcbNstGst->GDTR.u32Limit;
+        pVCpu->cpum.GstCtx.gdtr.pGdt  = pVmcbNstGst->GDTR.u64Base;
+        pVCpu->cpum.GstCtx.idtr.cbIdt = pVmcbNstGst->IDTR.u32Limit;
+        pVCpu->cpum.GstCtx.idtr.pIdt  = pVmcbNstGst->IDTR.u64Base;
+        CPUMSetGuestCR0(pVCpu, pVmcbNstGst->u64CR0);
+        CPUMSetGuestCR4(pVCpu, pVmcbNstGst->u64CR4);
+        pVCpu->cpum.GstCtx.cr3        = pVmcbNstGst->u64CR3;
+        pVCpu->cpum.GstCtx.cr2        = pVmcbNstGst->u64CR2;
+        pVCpu->cpum.GstCtx.dr[6]      = pVmcbNstGst->u64DR6;
+        pVCpu->cpum.GstCtx.dr[7]      = pVmcbNstGst->u64DR7;
+        pVCpu->cpum.GstCtx.rflags.u64 = pVmcbNstGst->u64RFlags;
+        pVCpu->cpum.GstCtx.rax        = pVmcbNstGst->u64RAX;
+        pVCpu->cpum.GstCtx.rsp        = pVmcbNstGst->u64RSP;
+        pVCpu->cpum.GstCtx.rip        = pVmcbNstGst->u64RIP;
+        CPUMSetGuestEferMsrNoChecks(pVCpu, pVCpu->cpum.GstCtx.msrEFER, uValidEfer);
+        if (pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging)
+            pVCpu->cpum.GstCtx.msrPAT = pVmcbNstGst->u64PAT;
+
+        /* Mask DR6, DR7 bits mandatory set/clear bits. */
+        pVCpu->cpum.GstCtx.dr[6] &= ~(X86_DR6_RAZ_MASK | X86_DR6_MBZ_MASK);
+        pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_RA1_MASK;
+        pVCpu->cpum.GstCtx.dr[7] &= ~(X86_DR7_RAZ_MASK | X86_DR7_MBZ_MASK);
+        pVCpu->cpum.GstCtx.dr[7] |= X86_DR7_RA1_MASK;
+
+        /*
+         * Check for pending virtual interrupts.
+         */
+        if (pVmcbCtrl->IntCtrl.n.u1VIrqPending)
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
+        else
+            Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST));
+
+        /*
+         * Update PGM, IEM and others of a world-switch.
+         */
+        VBOXSTRICTRC rcStrict = iemSvmWorldSwitch(pVCpu);
+        if (rcStrict == VINF_SUCCESS)
+        { /* likely */ }
+        else if (RT_SUCCESS(rcStrict))
+        {
+            LogFlow(("iemSvmVmrun: iemSvmWorldSwitch returned %Rrc, setting passup status\n", VBOXSTRICTRC_VAL(rcStrict)));
+            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+        }
+        else
+        {
+            LogFlow(("iemSvmVmrun: iemSvmWorldSwitch unexpected failure. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /*
+         * Set the global-interrupt flag to allow interrupts in the guest.
+         */
+        CPUMSetGuestGif(&pVCpu->cpum.GstCtx, true);
+
+        /*
+         * Event injection.
+         */
+        PCSVMEVENT pEventInject = &pVmcbCtrl->EventInject;
+        pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = !pEventInject->n.u1Valid;
+        if (pEventInject->n.u1Valid)
+        {
+            uint8_t   const uVector    = pEventInject->n.u8Vector;
+            TRPMEVENT const enmType    = HMSvmEventToTrpmEventType(pEventInject, uVector);
+            uint16_t  const uErrorCode = pEventInject->n.u1ErrorCodeValid ? pEventInject->n.u32ErrorCode : 0;
+
+            /* Validate vectors for hardware exceptions, see AMD spec. 15.20 "Event Injection". */
+            if (RT_UNLIKELY(enmType == TRPM_32BIT_HACK))
+            {
+                Log(("iemSvmVmrun: Invalid event type =%#x -> #VMEXIT\n", (uint8_t)pEventInject->n.u3Type));
+                return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+            }
+            if (pEventInject->n.u3Type == SVM_EVENT_EXCEPTION)
+            {
+                if (   uVector == X86_XCPT_NMI
+                    || uVector > X86_XCPT_LAST)
+                {
+                    Log(("iemSvmVmrun: Invalid vector for hardware exception. uVector=%#x -> #VMEXIT\n", uVector));
+                    return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+                }
+                if (   uVector == X86_XCPT_BR
+                    && CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+                {
+                    Log(("iemSvmVmrun: Cannot inject #BR when not in long mode -> #VMEXIT\n"));
+                    return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+                }
+                /** @todo any others? */
+            }
+
+            /*
+             * Invalidate the exit interrupt-information field here. This field is fully updated
+             * on #VMEXIT as events other than the one below can also cause intercepts during
+             * their injection (e.g. exceptions).
+             */
+            pVmcbCtrl->ExitIntInfo.n.u1Valid = 0;
+
+            /*
+             * Clear the event injection valid bit here. While the AMD spec. mentions that the CPU
+             * clears this bit from the VMCB unconditionally on #VMEXIT, internally the CPU could be
+             * clearing it at any time, most likely before/after injecting the event. Since VirtualBox
+             * doesn't have any virtual-CPU internal representation of this bit, we clear/update the
+             * VMCB here. This also has the added benefit that we avoid the risk of injecting the event
+             * twice if we fallback to executing the nested-guest using hardware-assisted SVM after
+             * injecting the event through IEM here.
+             */
+            pVmcbCtrl->EventInject.n.u1Valid = 0;
+
+            /** @todo NRIP: Software interrupts can only be pushed properly if we support
+             *        NRIP for the nested-guest to calculate the instruction length
+             *        below. */
+            LogFlow(("iemSvmVmrun: Injecting event: %04x:%08RX64 vec=%#x type=%d uErr=%u cr2=%#RX64 cr3=%#RX64 efer=%#RX64\n",
+                     pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uVector, enmType, uErrorCode, pVCpu->cpum.GstCtx.cr2,
+                     pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.msrEFER));
+
+            /*
+             * We shall not inject the event here right away. There may be paging mode related updates
+             * as a result of the world-switch above that are yet to be honored. Instead flag the event
+             * as pending for injection.
+             */
+            TRPMAssertTrap(pVCpu, uVector, enmType);
+            if (pEventInject->n.u1ErrorCodeValid)
+                TRPMSetErrorCode(pVCpu, uErrorCode);
+            if (   enmType == TRPM_TRAP
+                && uVector == X86_XCPT_PF)
+                TRPMSetFaultAddress(pVCpu, pVCpu->cpum.GstCtx.cr2);
+        }
+        else
+            LogFlow(("iemSvmVmrun: Entering nested-guest: %04x:%08RX64 cr0=%#RX64 cr3=%#RX64 cr4=%#RX64 efer=%#RX64 efl=%#x\n",
+                     pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3,
+                     pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER, pVCpu->cpum.GstCtx.rflags.u64));
+
+        LogFlow(("iemSvmVmrun: returns %d\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3)
+        /* If CLGI/STGI isn't intercepted we force IEM-only nested-guest execution here. */
+        if (   HMIsEnabled(pVM)
+            && HMIsSvmVGifActive(pVM))
+            return EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, true);
+# endif
+
+        return rcStrict;
+    }
+
+    /* Shouldn't really happen as the caller should've validated the physical address already. */
+    Log(("iemSvmVmrun: Failed to read nested-guest VMCB at %#RGp (rc=%Rrc) -> #VMEXIT\n", GCPhysVmcb, rc));
+    return rc;
+}
+
+
+/**
+ * Checks if the event intercepts and performs the \#VMEXIT if the corresponding
+ * intercept is active.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_HM_INTERCEPT_NOT_ACTIVE if the intercept is not active or
+ *          we're not executing a nested-guest.
+ * @retval  VINF_SVM_VMEXIT if the intercept is active and the \#VMEXIT occurred
+ *          successfully.
+ * @retval  VERR_SVM_VMEXIT_FAILED if the intercept is active and the \#VMEXIT
+ *          failed and a shutdown needs to be initiated for the guest.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ * @param   u8Vector    The interrupt or exception vector.
+ * @param   fFlags      The exception flags (see IEM_XCPT_FLAGS_XXX).
+ * @param   uErr        The error-code associated with the exception.
+ * @param   uCr2        The CR2 value in case of a \#PF exception.
+ */
+IEM_STATIC VBOXSTRICTRC iemHandleSvmEventIntercept(PVMCPUCC pVCpu, uint8_t u8Vector, uint32_t fFlags, uint32_t uErr, uint64_t uCr2)
+{
+    Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
+
+    /*
+     * Handle SVM exception and software interrupt intercepts, see AMD spec. 15.12 "Exception Intercepts".
+     *
+     *   - NMI intercepts have their own exit code and do not cause SVM_EXIT_XCPT_2 #VMEXITs.
+     *   - External interrupts and software interrupts (INTn instruction) do not check the exception intercepts
+     *     even when they use a vector in the range 0 to 31.
+     *   - ICEBP should not trigger #DB intercept, but its own intercept.
+     *   - For #PF exceptions, its intercept is checked before CR2 is written by the exception.
+     */
+    /* Check NMI intercept */
+    if (   u8Vector == X86_XCPT_NMI
+        && (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+        && IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_NMI))
+    {
+        Log2(("iemHandleSvmNstGstEventIntercept: NMI intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_NMI, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /* Check ICEBP intercept. */
+    if (   (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)
+        && IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_ICEBP))
+    {
+        Log2(("iemHandleSvmNstGstEventIntercept: ICEBP intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_ICEBP, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /* Check CPU exception intercepts. */
+    if (   (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+        && IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, u8Vector))
+    {
+        Assert(u8Vector <= X86_XCPT_LAST);
+        uint64_t const uExitInfo1 = fFlags & IEM_XCPT_FLAGS_ERR ? uErr : 0;
+        uint64_t const uExitInfo2 = fFlags & IEM_XCPT_FLAGS_CR2 ? uCr2 : 0;
+        if (   IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists
+            && u8Vector == X86_XCPT_PF
+            && !(uErr & X86_TRAP_PF_ID))
+        {
+            PSVMVMCBCTRL  pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb)->ctrl;
+# ifdef IEM_WITH_CODE_TLB
+            uint8_t const *pbInstrBuf = pVCpu->iem.s.pbInstrBuf;
+            uint8_t const  cbInstrBuf = pVCpu->iem.s.cbInstrBuf;
+            pVmcbCtrl->cbInstrFetched = RT_MIN(cbInstrBuf, SVM_CTRL_GUEST_INSTR_BYTES_MAX);
+            if (   pbInstrBuf
+                && cbInstrBuf > 0)
+                memcpy(&pVmcbCtrl->abInstr[0], pbInstrBuf, pVmcbCtrl->cbInstrFetched);
+# else
+            uint8_t const cbOpcode    = pVCpu->iem.s.cbOpcode;
+            pVmcbCtrl->cbInstrFetched = RT_MIN(cbOpcode, SVM_CTRL_GUEST_INSTR_BYTES_MAX);
+            if (cbOpcode > 0)
+                memcpy(&pVmcbCtrl->abInstr[0], &pVCpu->iem.s.abOpcode[0], pVmcbCtrl->cbInstrFetched);
+# endif
+        }
+        if (u8Vector == X86_XCPT_BR)
+            IEM_SVM_UPDATE_NRIP(pVCpu);
+        Log2(("iemHandleSvmNstGstEventIntercept: Xcpt intercept u32InterceptXcpt=%#RX32 u8Vector=%#x "
+              "uExitInfo1=%#RX64 uExitInfo2=%#RX64 -> #VMEXIT\n", pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb)->ctrl.u32InterceptXcpt,
+              u8Vector, uExitInfo1, uExitInfo2));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_0 + u8Vector, uExitInfo1, uExitInfo2);
+    }
+
+    /* Check software interrupt (INTn) intercepts. */
+    if (   (fFlags & (  IEM_XCPT_FLAGS_T_SOFT_INT
+                      | IEM_XCPT_FLAGS_BP_INSTR
+                      | IEM_XCPT_FLAGS_ICEBP_INSTR
+                      | IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
+        && IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_INTN))
+    {
+        uint64_t const uExitInfo1 = IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? u8Vector : 0;
+        Log2(("iemHandleSvmNstGstEventIntercept: Software INT intercept (u8Vector=%#x) -> #VMEXIT\n", u8Vector));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SWINT, uExitInfo1, 0 /* uExitInfo2 */);
+    }
+
+    return VINF_SVM_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * Checks the SVM IO permission bitmap and performs the \#VMEXIT if the
+ * corresponding intercept is active.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_HM_INTERCEPT_NOT_ACTIVE if the intercept is not active or
+ *          we're not executing a nested-guest.
+ * @retval  VINF_SVM_VMEXIT if the intercept is active and the \#VMEXIT occurred
+ *          successfully.
+ * @retval  VERR_SVM_VMEXIT_FAILED if the intercept is active and the \#VMEXIT
+ *          failed and a shutdown needs to be initiated for the guest.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   u16Port         The IO port being accessed.
+ * @param   enmIoType       The type of IO access.
+ * @param   cbReg           The IO operand size in bytes.
+ * @param   cAddrSizeBits   The address size bits (for 16, 32 or 64).
+ * @param   iEffSeg         The effective segment number.
+ * @param   fRep            Whether this is a repeating IO instruction (REP prefix).
+ * @param   fStrIo          Whether this is a string IO instruction.
+ * @param   cbInstr         The length of the IO instruction in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemSvmHandleIOIntercept(PVMCPUCC pVCpu, uint16_t u16Port, SVMIOIOTYPE enmIoType, uint8_t cbReg,
+                                                uint8_t cAddrSizeBits, uint8_t iEffSeg, bool fRep, bool fStrIo, uint8_t cbInstr)
+{
+    Assert(IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT));
+    Assert(cAddrSizeBits == 16 || cAddrSizeBits == 32 || cAddrSizeBits == 64);
+    Assert(cbReg == 1 || cbReg == 2 || cbReg == 4 || cbReg == 8);
+
+    Log3(("iemSvmHandleIOIntercept: u16Port=%#x (%u)\n", u16Port, u16Port));
+
+    SVMIOIOEXITINFO IoExitInfo;
+    void *pvIoBitmap = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvIoBitmap);
+    bool const fIntercept = CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep,
+                                                    fStrIo, &IoExitInfo);
+    if (fIntercept)
+    {
+        Log3(("iemSvmHandleIOIntercept: u16Port=%#x (%u) -> #VMEXIT\n", u16Port, u16Port));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        return iemSvmVmexit(pVCpu, SVM_EXIT_IOIO, IoExitInfo.u, pVCpu->cpum.GstCtx.rip + cbInstr);
+    }
+
+    /** @todo remove later (for debugging as VirtualBox always traps all IO
+     *        intercepts). */
+    AssertMsgFailed(("iemSvmHandleIOIntercept: We expect an IO intercept here!\n"));
+    return VINF_SVM_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * Checks the SVM MSR permission bitmap and performs the \#VMEXIT if the
+ * corresponding intercept is active.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_HM_INTERCEPT_NOT_ACTIVE if the MSR permission bitmap does not
+ *          specify interception of the accessed MSR @a idMsr.
+ * @retval  VINF_SVM_VMEXIT if the intercept is active and the \#VMEXIT occurred
+ *          successfully.
+ * @retval  VERR_SVM_VMEXIT_FAILED if the intercept is active and the \#VMEXIT
+ *          failed and a shutdown needs to be initiated for the guest.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR being accessed in the nested-guest.
+ * @param   fWrite      Whether this is an MSR write access, @c false implies an
+ *                      MSR read.
+ * @param   cbInstr     The length of the MSR read/write instruction in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemSvmHandleMsrIntercept(PVMCPUCC pVCpu, uint32_t idMsr, bool fWrite)
+{
+    /*
+     * Check if any MSRs are being intercepted.
+     */
+    Assert(CPUMIsGuestSvmCtrlInterceptSet(pVCpu, IEM_GET_CTX(pVCpu), SVM_CTRL_INTERCEPT_MSR_PROT));
+    Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
+
+    uint64_t const uExitInfo1 = fWrite ? SVM_EXIT1_MSR_WRITE : SVM_EXIT1_MSR_READ;
+
+    /*
+     * Get the byte and bit offset of the permission bits corresponding to the MSR.
+     */
+    uint16_t offMsrpm;
+    uint8_t  uMsrpmBit;
+    int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(uMsrpmBit == 0 || uMsrpmBit == 2 || uMsrpmBit == 4 || uMsrpmBit == 6);
+        Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
+        if (fWrite)
+            ++uMsrpmBit;
+
+        /*
+         * Check if the bit is set, if so, trigger a #VMEXIT.
+         */
+        uint8_t *pbMsrpm = (uint8_t *)pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
+        pbMsrpm += offMsrpm;
+        if (*pbMsrpm & RT_BIT(uMsrpmBit))
+        {
+            IEM_SVM_UPDATE_NRIP(pVCpu);
+            return iemSvmVmexit(pVCpu, SVM_EXIT_MSR, uExitInfo1, 0 /* uExitInfo2 */);
+        }
+    }
+    else
+    {
+        /*
+         * This shouldn't happen, but if it does, cause a #VMEXIT and let the "host" (nested hypervisor) deal with it.
+         */
+        Log(("iemSvmHandleMsrIntercept: Invalid/out-of-range MSR %#RX32 fWrite=%RTbool -> #VMEXIT\n", idMsr, fWrite));
+        return iemSvmVmexit(pVCpu, SVM_EXIT_MSR, uExitInfo1, 0 /* uExitInfo2 */);
+    }
+    return VINF_SVM_INTERCEPT_NOT_ACTIVE;
+}
+
+
+
+/**
+ * Implements 'VMRUN'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmrun)
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF2(pVCpu, cbInstr);
+    return VINF_EM_RAW_EMULATE_INSTR;
+# else
+    LogFlow(("iemCImpl_vmrun\n"));
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, vmrun);
+
+    /** @todo Check effective address size using address size prefix. */
+    RTGCPHYS const GCPhysVmcb = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax;
+    if (   (GCPhysVmcb & X86_PAGE_4K_OFFSET_MASK)
+        || !PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcb))
+    {
+        Log(("vmrun: VMCB physaddr (%#RGp) not valid -> #GP(0)\n", GCPhysVmcb));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_VMRUN))
+    {
+        Log(("vmrun: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_VMRUN, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    VBOXSTRICTRC rcStrict = iemSvmVmrun(pVCpu, cbInstr, GCPhysVmcb);
+    if (rcStrict == VERR_SVM_VMEXIT_FAILED)
+    {
+        Assert(!CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
+        rcStrict = VINF_EM_TRIPLE_FAULT;
+    }
+    return rcStrict;
+# endif
+}
+
+
+/**
+ * Implements 'VMLOAD'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmload)
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF2(pVCpu, cbInstr);
+    return VINF_EM_RAW_EMULATE_INSTR;
+# else
+    LogFlow(("iemCImpl_vmload\n"));
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, vmload);
+
+    /** @todo Check effective address size using address size prefix. */
+    RTGCPHYS const GCPhysVmcb = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax;
+    if (   (GCPhysVmcb & X86_PAGE_4K_OFFSET_MASK)
+        || !PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcb))
+    {
+        Log(("vmload: VMCB physaddr (%#RGp) not valid -> #GP(0)\n", GCPhysVmcb));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_VMLOAD))
+    {
+        Log(("vmload: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_VMLOAD, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    SVMVMCBSTATESAVE VmcbNstGst;
+    VBOXSTRICTRC rcStrict = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &VmcbNstGst, GCPhysVmcb + RT_UOFFSETOF(SVMVMCB, guest),
+                                                    sizeof(SVMVMCBSTATESAVE));
+    if (rcStrict == VINF_SUCCESS)
+    {
+        LogFlow(("vmload: Loading VMCB at %#RGp enmEffAddrMode=%d\n", GCPhysVmcb, pVCpu->iem.s.enmEffAddrMode));
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, FS, fs);
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, GS, gs);
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, TR, tr);
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, LDTR, ldtr);
+
+        pVCpu->cpum.GstCtx.msrKERNELGSBASE = VmcbNstGst.u64KernelGSBase;
+        pVCpu->cpum.GstCtx.msrSTAR         = VmcbNstGst.u64STAR;
+        pVCpu->cpum.GstCtx.msrLSTAR        = VmcbNstGst.u64LSTAR;
+        pVCpu->cpum.GstCtx.msrCSTAR        = VmcbNstGst.u64CSTAR;
+        pVCpu->cpum.GstCtx.msrSFMASK       = VmcbNstGst.u64SFMASK;
+
+        pVCpu->cpum.GstCtx.SysEnter.cs     = VmcbNstGst.u64SysEnterCS;
+        pVCpu->cpum.GstCtx.SysEnter.esp    = VmcbNstGst.u64SysEnterESP;
+        pVCpu->cpum.GstCtx.SysEnter.eip    = VmcbNstGst.u64SysEnterEIP;
+
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    }
+    return rcStrict;
+# endif
+}
+
+
+/**
+ * Implements 'VMSAVE'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmsave)
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF2(pVCpu, cbInstr);
+    return VINF_EM_RAW_EMULATE_INSTR;
+# else
+    LogFlow(("iemCImpl_vmsave\n"));
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, vmsave);
+
+    /** @todo Check effective address size using address size prefix. */
+    RTGCPHYS const GCPhysVmcb = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax;
+    if (   (GCPhysVmcb & X86_PAGE_4K_OFFSET_MASK)
+        || !PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcb))
+    {
+        Log(("vmsave: VMCB physaddr (%#RGp) not valid -> #GP(0)\n", GCPhysVmcb));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_VMSAVE))
+    {
+        Log(("vmsave: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_VMSAVE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    SVMVMCBSTATESAVE VmcbNstGst;
+    VBOXSTRICTRC rcStrict = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &VmcbNstGst, GCPhysVmcb + RT_UOFFSETOF(SVMVMCB, guest),
+                                                    sizeof(SVMVMCBSTATESAVE));
+    if (rcStrict == VINF_SUCCESS)
+    {
+        LogFlow(("vmsave: Saving VMCB at %#RGp enmEffAddrMode=%d\n", GCPhysVmcb, pVCpu->iem.s.enmEffAddrMode));
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_FS | CPUMCTX_EXTRN_GS | CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_LDTR
+                                | CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_SYSENTER_MSRS);
+
+        HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, FS, fs);
+        HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, GS, gs);
+        HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, TR, tr);
+        HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, LDTR, ldtr);
+
+        VmcbNstGst.u64KernelGSBase  = pVCpu->cpum.GstCtx.msrKERNELGSBASE;
+        VmcbNstGst.u64STAR          = pVCpu->cpum.GstCtx.msrSTAR;
+        VmcbNstGst.u64LSTAR         = pVCpu->cpum.GstCtx.msrLSTAR;
+        VmcbNstGst.u64CSTAR         = pVCpu->cpum.GstCtx.msrCSTAR;
+        VmcbNstGst.u64SFMASK        = pVCpu->cpum.GstCtx.msrSFMASK;
+
+        VmcbNstGst.u64SysEnterCS    = pVCpu->cpum.GstCtx.SysEnter.cs;
+        VmcbNstGst.u64SysEnterESP   = pVCpu->cpum.GstCtx.SysEnter.esp;
+        VmcbNstGst.u64SysEnterEIP   = pVCpu->cpum.GstCtx.SysEnter.eip;
+
+        rcStrict = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVmcb + RT_UOFFSETOF(SVMVMCB, guest), &VmcbNstGst,
+                                            sizeof(SVMVMCBSTATESAVE));
+        if (rcStrict == VINF_SUCCESS)
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    }
+    return rcStrict;
+# endif
+}
+
+
+/**
+ * Implements 'CLGI'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_clgi)
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF2(pVCpu, cbInstr);
+    return VINF_EM_RAW_EMULATE_INSTR;
+# else
+    LogFlow(("iemCImpl_clgi\n"));
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, clgi);
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CLGI))
+    {
+        Log(("clgi: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_CLGI, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    CPUMSetGuestGif(&pVCpu->cpum.GstCtx, false);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+
+#  if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3)
+    return EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, true);
+#  else
+    return VINF_SUCCESS;
+#  endif
+# endif
+}
+
+
+/**
+ * Implements 'STGI'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_stgi)
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF2(pVCpu, cbInstr);
+    return VINF_EM_RAW_EMULATE_INSTR;
+# else
+    LogFlow(("iemCImpl_stgi\n"));
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, stgi);
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_STGI))
+    {
+        Log2(("stgi: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_STGI, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    CPUMSetGuestGif(&pVCpu->cpum.GstCtx, true);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+
+#  if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3)
+    return EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, false);
+#  else
+    return VINF_SUCCESS;
+#  endif
+# endif
+}
+
+
+/**
+ * Implements 'INVLPGA'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_invlpga)
+{
+    /** @todo Check effective address size using address size prefix. */
+    RTGCPTR  const GCPtrPage = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax;
+    /** @todo PGM needs virtual ASID support. */
+# if 0
+    uint32_t const uAsid     = pVCpu->cpum.GstCtx.ecx;
+# endif
+
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, invlpga);
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_INVLPGA))
+    {
+        Log2(("invlpga: Guest intercept (%RGp) -> #VMEXIT\n", GCPtrPage));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_INVLPGA, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    PGMInvalidatePage(pVCpu, GCPtrPage);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'SKINIT'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_skinit)
+{
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, invlpga);
+
+    uint32_t uIgnore;
+    uint32_t fFeaturesECX;
+    CPUMGetGuestCpuId(pVCpu, 0x80000001, 0 /* iSubLeaf */, &uIgnore, &uIgnore, &fFeaturesECX, &uIgnore);
+    if (!(fFeaturesECX & X86_CPUID_AMD_FEATURE_ECX_SKINIT))
+        return iemRaiseUndefinedOpcode(pVCpu);
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SKINIT))
+    {
+        Log2(("skinit: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SKINIT, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    RT_NOREF(cbInstr);
+    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * Implements SVM's implementation of PAUSE.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_svm_pause)
+{
+    bool fCheckIntercept = true;
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmPauseFilter)
+    {
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+
+        /* TSC based pause-filter thresholding. */
+        if (   IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmPauseFilterThreshold
+            && pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilterThreshold > 0)
+        {
+            uint64_t const uTick = TMCpuTickGet(pVCpu);
+            if (uTick - pVCpu->cpum.GstCtx.hwvirt.svm.uPrevPauseTick > pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilterThreshold)
+                pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilter = CPUMGetGuestSvmPauseFilterCount(pVCpu, IEM_GET_CTX(pVCpu));
+            pVCpu->cpum.GstCtx.hwvirt.svm.uPrevPauseTick = uTick;
+        }
+
+        /* Simple pause-filter counter. */
+        if (pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilter > 0)
+        {
+            --pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilter;
+            fCheckIntercept = false;
+        }
+    }
+
+    if (fCheckIntercept)
+        IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_PAUSE, SVM_EXIT_PAUSE, 0, 0);
+
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
+
+/**
+ * Common code for iemCImpl_vmmcall and iemCImpl_vmcall (latter in IEMAllCImplVmxInstr.cpp.h).
+ */
+IEM_CIMPL_DEF_1(iemCImpl_Hypercall, uint16_t, uDisOpcode)
+{
+    if (EMAreHypercallInstructionsEnabled(pVCpu))
+    {
+        NOREF(uDisOpcode);
+        VBOXSTRICTRC rcStrict = GIMHypercallEx(pVCpu, IEM_GET_CTX(pVCpu), uDisOpcode, cbInstr);
+        if (RT_SUCCESS(rcStrict))
+        {
+            if (rcStrict == VINF_SUCCESS)
+                iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            if (   rcStrict == VINF_SUCCESS
+                || rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
+                return VINF_SUCCESS;
+            AssertMsgReturn(rcStrict == VINF_GIM_R3_HYPERCALL, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IEM_IPE_4);
+            return rcStrict;
+        }
+        AssertMsgReturn(   rcStrict == VERR_GIM_HYPERCALL_ACCESS_DENIED
+                        || rcStrict == VERR_GIM_HYPERCALLS_NOT_AVAILABLE
+                        || rcStrict == VERR_GIM_NOT_ENABLED
+                        || rcStrict == VERR_GIM_HYPERCALL_MEMORY_READ_FAILED
+                        || rcStrict == VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED,
+                        ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IEM_IPE_4);
+
+        /* Raise #UD on all failures. */
+    }
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+
+/**
+ * Implements 'VMMCALL'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmmcall)
+{
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_VMMCALL))
+    {
+        Log(("vmmcall: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_VMMCALL, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /* This is a little bit more complicated than the VT-x version because HM/SVM may
+       patch MOV CR8 instructions to speed up APIC.TPR access for 32-bit windows guests. */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (VM_IS_HM_ENABLED(pVM))
+    {
+        int rc = HMHCMaybeMovTprSvmHypercall(pVM, pVCpu);
+        if (RT_SUCCESS(rc))
+        {
+            Log(("vmmcall: MovTpr\n"));
+            return VINF_SUCCESS;
+        }
+    }
+
+    /* Join forces with vmcall. */
+    return IEM_CIMPL_CALL_1(iemCImpl_Hypercall, OP_VMMCALL);
+}
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp.h b/src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp.h
new file mode 100644
index 00000000..a3b01dae
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp.h
@@ -0,0 +1,8953 @@
+/* $Id: IEMAllCImplVmxInstr.cpp.h $ */
+/** @file
+ * IEM - VT-x instruction implementation.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Gets the ModR/M, SIB and displacement byte(s) from decoded opcodes given their
+ * relative offsets.
+ */
+# ifdef IEM_WITH_CODE_TLB
+#  define IEM_MODRM_GET_U8(a_pVCpu, a_bModRm, a_offModRm)         do { } while (0)
+#  define IEM_SIB_GET_U8(a_pVCpu, a_bSib, a_offSib)               do { } while (0)
+#  define IEM_DISP_GET_U16(a_pVCpu, a_u16Disp, a_offDisp)         do { } while (0)
+#  define IEM_DISP_GET_S8_SX_U16(a_pVCpu, a_u16Disp, a_offDisp)   do { } while (0)
+#  define IEM_DISP_GET_U32(a_pVCpu, a_u32Disp, a_offDisp)         do { } while (0)
+#  define IEM_DISP_GET_S8_SX_U32(a_pVCpu, a_u32Disp, a_offDisp)   do { } while (0)
+#  define IEM_DISP_GET_S32_SX_U64(a_pVCpu, a_u64Disp, a_offDisp)  do { } while (0)
+#  define IEM_DISP_GET_S8_SX_U64(a_pVCpu, a_u64Disp, a_offDisp)   do { } while (0)
+#  error "Implement me: Getting ModR/M, SIB, displacement needs to work even when instruction crosses a page boundary."
+# else  /* !IEM_WITH_CODE_TLB */
+#  define IEM_MODRM_GET_U8(a_pVCpu, a_bModRm, a_offModRm) \
+    do \
+    { \
+        Assert((a_offModRm) < (a_pVCpu)->iem.s.cbOpcode); \
+        (a_bModRm) = (a_pVCpu)->iem.s.abOpcode[(a_offModRm)]; \
+    } while (0)
+
+#  define IEM_SIB_GET_U8(a_pVCpu, a_bSib, a_offSib)      IEM_MODRM_GET_U8(a_pVCpu, a_bSib, a_offSib)
+
+#  define IEM_DISP_GET_U16(a_pVCpu, a_u16Disp, a_offDisp) \
+    do \
+    { \
+        Assert((a_offDisp) + 1 < (a_pVCpu)->iem.s.cbOpcode); \
+        uint8_t const bTmpLo = (a_pVCpu)->iem.s.abOpcode[(a_offDisp)]; \
+        uint8_t const bTmpHi = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 1]; \
+        (a_u16Disp) = RT_MAKE_U16(bTmpLo, bTmpHi); \
+    } while (0)
+
+#  define IEM_DISP_GET_S8_SX_U16(a_pVCpu, a_u16Disp, a_offDisp) \
+    do \
+    { \
+        Assert((a_offDisp) < (a_pVCpu)->iem.s.cbOpcode); \
+        (a_u16Disp) = (int8_t)((a_pVCpu)->iem.s.abOpcode[(a_offDisp)]); \
+    } while (0)
+
+#  define IEM_DISP_GET_U32(a_pVCpu, a_u32Disp, a_offDisp) \
+    do \
+    { \
+        Assert((a_offDisp) + 3 < (a_pVCpu)->iem.s.cbOpcode); \
+        uint8_t const bTmp0 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp)]; \
+        uint8_t const bTmp1 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 1]; \
+        uint8_t const bTmp2 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 2]; \
+        uint8_t const bTmp3 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 3]; \
+        (a_u32Disp) = RT_MAKE_U32_FROM_U8(bTmp0, bTmp1, bTmp2, bTmp3); \
+    } while (0)
+
+#  define IEM_DISP_GET_S8_SX_U32(a_pVCpu, a_u32Disp, a_offDisp) \
+    do \
+    { \
+        Assert((a_offDisp) + 1 < (a_pVCpu)->iem.s.cbOpcode); \
+        (a_u32Disp) = (int8_t)((a_pVCpu)->iem.s.abOpcode[(a_offDisp)]); \
+    } while (0)
+
+#  define IEM_DISP_GET_S8_SX_U64(a_pVCpu, a_u64Disp, a_offDisp) \
+    do \
+    { \
+        Assert((a_offDisp) + 1 < (a_pVCpu)->iem.s.cbOpcode); \
+        (a_u64Disp) = (int8_t)((a_pVCpu)->iem.s.abOpcode[(a_offDisp)]); \
+    } while (0)
+
+#  define IEM_DISP_GET_S32_SX_U64(a_pVCpu, a_u64Disp, a_offDisp) \
+    do \
+    { \
+        Assert((a_offDisp) + 3 < (a_pVCpu)->iem.s.cbOpcode); \
+        uint8_t const bTmp0 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp)]; \
+        uint8_t const bTmp1 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 1]; \
+        uint8_t const bTmp2 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 2]; \
+        uint8_t const bTmp3 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 3]; \
+        (a_u64Disp) = (int32_t)RT_MAKE_U32_FROM_U8(bTmp0, bTmp1, bTmp2, bTmp3); \
+    } while (0)
+# endif /* !IEM_WITH_CODE_TLB */
+
+/** Gets the guest-physical address of the shadows VMCS for the given VCPU. */
+# define IEM_VMX_GET_SHADOW_VMCS(a_pVCpu)           ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysShadowVmcs)
+
+/** Whether a shadow VMCS is present for the given VCPU. */
+# define IEM_VMX_HAS_SHADOW_VMCS(a_pVCpu)           RT_BOOL(IEM_VMX_GET_SHADOW_VMCS(a_pVCpu) != NIL_RTGCPHYS)
+
+/** Gets the VMXON region pointer. */
+# define IEM_VMX_GET_VMXON_PTR(a_pVCpu)             ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon)
+
+/** Gets the guest-physical address of the current VMCS for the given VCPU. */
+# define IEM_VMX_GET_CURRENT_VMCS(a_pVCpu)          ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs)
+
+/** Whether a current VMCS is present for the given VCPU. */
+# define IEM_VMX_HAS_CURRENT_VMCS(a_pVCpu)          RT_BOOL(IEM_VMX_GET_CURRENT_VMCS(a_pVCpu) != NIL_RTGCPHYS)
+
+/** Assigns the guest-physical address of the current VMCS for the given VCPU. */
+# define IEM_VMX_SET_CURRENT_VMCS(a_pVCpu, a_GCPhysVmcs) \
+    do \
+    { \
+        Assert((a_GCPhysVmcs) != NIL_RTGCPHYS); \
+        (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs = (a_GCPhysVmcs); \
+    } while (0)
+
+/** Clears any current VMCS for the given VCPU. */
+# define IEM_VMX_CLEAR_CURRENT_VMCS(a_pVCpu) \
+    do \
+    { \
+        (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs = NIL_RTGCPHYS; \
+    } while (0)
+
+/** Check for VMX instructions requiring to be in VMX operation.
+ * @note Any changes here, check if IEMOP_HLP_IN_VMX_OPERATION needs updating. */
+# define IEM_VMX_IN_VMX_OPERATION(a_pVCpu, a_szInstr, a_InsDiagPrefix) \
+    do \
+    { \
+        if (IEM_VMX_IS_ROOT_MODE(a_pVCpu)) \
+        { /* likely */ } \
+        else \
+        { \
+            Log((a_szInstr ": Not in VMX operation (root mode) -> #UD\n")); \
+            (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.enmDiag = a_InsDiagPrefix##_VmxRoot; \
+            return iemRaiseUndefinedOpcode(a_pVCpu); \
+        } \
+    } while (0)
+
+/** Marks a VM-entry failure with a diagnostic reason, logs and returns. */
+# define IEM_VMX_VMENTRY_FAILED_RET(a_pVCpu, a_pszInstr, a_pszFailure, a_VmxDiag) \
+    do \
+    { \
+        LogRel(("%s: VM-entry failed! enmDiag=%u (%s) -> %s\n", (a_pszInstr), (a_VmxDiag), \
+               HMGetVmxDiagDesc(a_VmxDiag), (a_pszFailure))); \
+        (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.enmDiag = (a_VmxDiag); \
+        return VERR_VMX_VMENTRY_FAILED; \
+    } while (0)
+
+/** Marks a VM-exit failure with a diagnostic reason, logs and returns. */
+# define IEM_VMX_VMEXIT_FAILED_RET(a_pVCpu, a_uExitReason, a_pszFailure, a_VmxDiag) \
+    do \
+    { \
+        LogRel(("VM-exit failed! uExitReason=%u enmDiag=%u (%s) -> %s\n", (a_uExitReason), (a_VmxDiag), \
+               HMGetVmxDiagDesc(a_VmxDiag), (a_pszFailure))); \
+        (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.enmDiag  = (a_VmxDiag); \
+        return VERR_VMX_VMEXIT_FAILED; \
+    } while (0)
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** @todo NSTVMX: The following VM-exit intercepts are pending:
+ *  VMX_EXIT_IO_SMI
+ *  VMX_EXIT_SMI
+ *  VMX_EXIT_GETSEC
+ *  VMX_EXIT_RSM
+ *  VMX_EXIT_MONITOR (APIC access VM-exit caused by MONITOR pending)
+ *  VMX_EXIT_ERR_MACHINE_CHECK (we never need to raise this?)
+ *  VMX_EXIT_EPT_VIOLATION
+ *  VMX_EXIT_EPT_MISCONFIG
+ *  VMX_EXIT_INVEPT
+ *  VMX_EXIT_RDRAND
+ *  VMX_EXIT_VMFUNC
+ *  VMX_EXIT_ENCLS
+ *  VMX_EXIT_RDSEED
+ *  VMX_EXIT_PML_FULL
+ *  VMX_EXIT_XSAVES
+ *  VMX_EXIT_XRSTORS
+ */
+/**
+ * Map of VMCS field encodings to their virtual-VMCS structure offsets.
+ *
+ * The first array dimension is VMCS field encoding of Width OR'ed with Type and the
+ * second dimension is the Index, see VMXVMCSFIELD.
+ */
+uint16_t const g_aoffVmcsMap[16][VMX_V_VMCS_MAX_INDEX + 1] =
+{
+    /* VMX_VMCSFIELD_WIDTH_16BIT | VMX_VMCSFIELD_TYPE_CONTROL: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u16Vpid),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u16PostIntNotifyVector),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u16EptpIndex),
+        /*  3-10 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 11-18 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 19-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_16BIT | VMX_VMCSFIELD_TYPE_VMEXIT_INFO: */
+    {
+        /*   0-7 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /*  8-15 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 16-23 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 24-25 */ UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_16BIT | VMX_VMCSFIELD_TYPE_GUEST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, GuestEs),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, GuestCs),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, GuestSs),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, GuestDs),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, GuestFs),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, GuestGs),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, GuestLdtr),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, GuestTr),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u16GuestIntStatus),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u16PmlIndex),
+        /* 10-17 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 18-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_16BIT | VMX_VMCSFIELD_TYPE_HOST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, HostEs),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, HostCs),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, HostSs),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, HostDs),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, HostFs),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, HostGs),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, HostTr),
+        /*  7-14 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 15-22 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 23-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_64BIT | VMX_VMCSFIELD_TYPE_CONTROL: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64AddrIoBitmapA),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64AddrIoBitmapB),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64AddrMsrBitmap),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64AddrExitMsrStore),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u64AddrExitMsrLoad),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u64AddrEntryMsrLoad),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u64ExecVmcsPtr),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u64AddrPml),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u64TscOffset),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u64AddrVirtApic),
+        /*    10 */ RT_UOFFSETOF(VMXVVMCS, u64AddrApicAccess),
+        /*    11 */ RT_UOFFSETOF(VMXVVMCS, u64AddrPostedIntDesc),
+        /*    12 */ RT_UOFFSETOF(VMXVVMCS, u64VmFuncCtls),
+        /*    13 */ RT_UOFFSETOF(VMXVVMCS, u64EptpPtr),
+        /*    14 */ RT_UOFFSETOF(VMXVVMCS, u64EoiExitBitmap0),
+        /*    15 */ RT_UOFFSETOF(VMXVVMCS, u64EoiExitBitmap1),
+        /*    16 */ RT_UOFFSETOF(VMXVVMCS, u64EoiExitBitmap2),
+        /*    17 */ RT_UOFFSETOF(VMXVVMCS, u64EoiExitBitmap3),
+        /*    18 */ RT_UOFFSETOF(VMXVVMCS, u64AddrEptpList),
+        /*    19 */ RT_UOFFSETOF(VMXVVMCS, u64AddrVmreadBitmap),
+        /*    20 */ RT_UOFFSETOF(VMXVVMCS, u64AddrVmwriteBitmap),
+        /*    21 */ RT_UOFFSETOF(VMXVVMCS, u64AddrXcptVeInfo),
+        /*    22 */ RT_UOFFSETOF(VMXVVMCS, u64XssBitmap),
+        /*    23 */ RT_UOFFSETOF(VMXVVMCS, u64EnclsBitmap),
+        /*    24 */ RT_UOFFSETOF(VMXVVMCS, u64SpptPtr),
+        /*    25 */ RT_UOFFSETOF(VMXVVMCS, u64TscMultiplier)
+    },
+    /* VMX_VMCSFIELD_WIDTH_64BIT | VMX_VMCSFIELD_TYPE_VMEXIT_INFO: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64RoGuestPhysAddr),
+        /*   1-8 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /*  9-16 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 17-24 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /*    25 */ UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_64BIT | VMX_VMCSFIELD_TYPE_GUEST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64VmcsLinkPtr),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64GuestDebugCtlMsr),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPatMsr),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64GuestEferMsr),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPerfGlobalCtlMsr),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPdpte0),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPdpte1),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPdpte2),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPdpte3),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u64GuestBndcfgsMsr),
+        /*    10 */ RT_UOFFSETOF(VMXVVMCS, u64GuestRtitCtlMsr),
+        /* 11-18 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 19-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_64BIT | VMX_VMCSFIELD_TYPE_HOST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64HostPatMsr),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64HostEferMsr),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64HostPerfGlobalCtlMsr),
+        /*  3-10 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 11-18 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 19-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_32BIT | VMX_VMCSFIELD_TYPE_CONTROL: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u32PinCtls),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u32ProcCtls),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u32XcptBitmap),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u32XcptPFMask),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u32XcptPFMatch),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u32Cr3TargetCount),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u32ExitCtls),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u32ExitMsrStoreCount),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u32ExitMsrLoadCount),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u32EntryCtls),
+        /*    10 */ RT_UOFFSETOF(VMXVVMCS, u32EntryMsrLoadCount),
+        /*    11 */ RT_UOFFSETOF(VMXVVMCS, u32EntryIntInfo),
+        /*    12 */ RT_UOFFSETOF(VMXVVMCS, u32EntryXcptErrCode),
+        /*    13 */ RT_UOFFSETOF(VMXVVMCS, u32EntryInstrLen),
+        /*    14 */ RT_UOFFSETOF(VMXVVMCS, u32TprThreshold),
+        /*    15 */ RT_UOFFSETOF(VMXVVMCS, u32ProcCtls2),
+        /*    16 */ RT_UOFFSETOF(VMXVVMCS, u32PleGap),
+        /*    17 */ RT_UOFFSETOF(VMXVVMCS, u32PleWindow),
+        /* 18-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_32BIT | VMX_VMCSFIELD_TYPE_VMEXIT_INFO: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u32RoVmInstrError),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitReason),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitIntInfo),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitIntErrCode),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u32RoIdtVectoringInfo),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u32RoIdtVectoringErrCode),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitInstrLen),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitInstrInfo),
+        /*  8-15 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 16-23 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 24-25 */ UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_32BIT | VMX_VMCSFIELD_TYPE_GUEST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u32GuestEsLimit),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u32GuestCsLimit),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u32GuestSsLimit),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u32GuestDsLimit),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u32GuestFsLimit),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u32GuestGsLimit),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u32GuestLdtrLimit),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u32GuestTrLimit),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u32GuestGdtrLimit),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u32GuestIdtrLimit),
+        /*    10 */ RT_UOFFSETOF(VMXVVMCS, u32GuestEsAttr),
+        /*    11 */ RT_UOFFSETOF(VMXVVMCS, u32GuestCsAttr),
+        /*    12 */ RT_UOFFSETOF(VMXVVMCS, u32GuestSsAttr),
+        /*    13 */ RT_UOFFSETOF(VMXVVMCS, u32GuestDsAttr),
+        /*    14 */ RT_UOFFSETOF(VMXVVMCS, u32GuestFsAttr),
+        /*    15 */ RT_UOFFSETOF(VMXVVMCS, u32GuestGsAttr),
+        /*    16 */ RT_UOFFSETOF(VMXVVMCS, u32GuestLdtrAttr),
+        /*    17 */ RT_UOFFSETOF(VMXVVMCS, u32GuestTrAttr),
+        /*    18 */ RT_UOFFSETOF(VMXVVMCS, u32GuestIntrState),
+        /*    19 */ RT_UOFFSETOF(VMXVVMCS, u32GuestActivityState),
+        /*    20 */ RT_UOFFSETOF(VMXVVMCS, u32GuestSmBase),
+        /*    21 */ RT_UOFFSETOF(VMXVVMCS, u32GuestSysenterCS),
+        /*    22 */ UINT16_MAX,
+        /*    23 */ RT_UOFFSETOF(VMXVVMCS, u32PreemptTimer),
+        /* 24-25 */ UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_32BIT | VMX_VMCSFIELD_TYPE_HOST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u32HostSysenterCs),
+        /*   1-8 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /*  9-16 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 17-24 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /*    25 */ UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_NATURAL | VMX_VMCSFIELD_TYPE_CONTROL: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64Cr0Mask),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64Cr4Mask),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64Cr0ReadShadow),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64Cr4ReadShadow),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u64Cr3Target0),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u64Cr3Target1),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u64Cr3Target2),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u64Cr3Target3),
+        /*  8-15 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 16-23 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 24-25 */ UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_NATURAL | VMX_VMCSFIELD_TYPE_VMEXIT_INFO: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64RoExitQual),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64RoIoRcx),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64RoIoRsi),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64RoIoRdi),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u64RoIoRip),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u64RoGuestLinearAddr),
+        /*  6-13 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 14-21 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 22-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_NATURAL | VMX_VMCSFIELD_TYPE_GUEST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64GuestCr0),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64GuestCr3),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64GuestCr4),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64GuestEsBase),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u64GuestCsBase),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSsBase),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u64GuestDsBase),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u64GuestFsBase),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u64GuestGsBase),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u64GuestLdtrBase),
+        /*    10 */ RT_UOFFSETOF(VMXVVMCS, u64GuestTrBase),
+        /*    11 */ RT_UOFFSETOF(VMXVVMCS, u64GuestGdtrBase),
+        /*    12 */ RT_UOFFSETOF(VMXVVMCS, u64GuestIdtrBase),
+        /*    13 */ RT_UOFFSETOF(VMXVVMCS, u64GuestDr7),
+        /*    14 */ RT_UOFFSETOF(VMXVVMCS, u64GuestRsp),
+        /*    15 */ RT_UOFFSETOF(VMXVVMCS, u64GuestRip),
+        /*    16 */ RT_UOFFSETOF(VMXVVMCS, u64GuestRFlags),
+        /*    17 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPendingDbgXcpts),
+        /*    18 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSysenterEsp),
+        /*    19 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSysenterEip),
+        /* 20-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_NATURAL | VMX_VMCSFIELD_TYPE_HOST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64HostCr0),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64HostCr3),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64HostCr4),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64HostFsBase),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u64HostGsBase),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u64HostTrBase),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u64HostGdtrBase),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u64HostIdtrBase),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u64HostSysenterEsp),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u64HostSysenterEip),
+        /*    10 */ RT_UOFFSETOF(VMXVVMCS, u64HostRsp),
+        /*    11 */ RT_UOFFSETOF(VMXVVMCS, u64HostRip),
+        /* 12-19 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 20-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    }
+};
+
+
+/**
+ * Gets a host selector from the VMCS.
+ *
+ * @param   pVmcs       Pointer to the virtual VMCS.
+ * @param   iSelReg     The index of the segment register (X86_SREG_XXX).
+ */
+DECLINLINE(RTSEL) iemVmxVmcsGetHostSelReg(PCVMXVVMCS pVmcs, uint8_t iSegReg)
+{
+    Assert(iSegReg < X86_SREG_COUNT);
+    RTSEL HostSel;
+    uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_16BIT;
+    uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_HOST_STATE;
+    uint8_t  const  uWidthType = (uWidth << 2) | uType;
+    uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS16_HOST_ES_SEL, VMX_BF_VMCSFIELD_INDEX);
+    Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+    uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+    uint8_t  const *pbVmcs     = (uint8_t *)pVmcs;
+    uint8_t  const *pbField    = pbVmcs + offField;
+    HostSel = *(uint16_t *)pbField;
+    return HostSel;
+}
+
+
+/**
+ * Sets a guest segment register in the VMCS.
+ *
+ * @param   pVmcs       Pointer to the virtual VMCS.
+ * @param   iSegReg     The index of the segment register (X86_SREG_XXX).
+ * @param   pSelReg     Pointer to the segment register.
+ */
+IEM_STATIC void iemVmxVmcsSetGuestSegReg(PCVMXVVMCS pVmcs, uint8_t iSegReg, PCCPUMSELREG pSelReg)
+{
+    Assert(pSelReg);
+    Assert(iSegReg < X86_SREG_COUNT);
+
+    /* Selector. */
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_16BIT;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS16_GUEST_ES_SEL, VMX_BF_VMCSFIELD_INDEX);
+        Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t        *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t        *pbField    = pbVmcs + offField;
+        *(uint16_t *)pbField = pSelReg->Sel;
+    }
+
+    /* Limit. */
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_32BIT;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS32_GUEST_ES_LIMIT, VMX_BF_VMCSFIELD_INDEX);
+        Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t        *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t        *pbField    = pbVmcs + offField;
+        *(uint32_t *)pbField = pSelReg->u32Limit;
+    }
+
+    /* Base. */
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_NATURAL;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS_GUEST_ES_BASE, VMX_BF_VMCSFIELD_INDEX);
+        Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t  const *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t  const *pbField    = pbVmcs + offField;
+        *(uint64_t *)pbField = pSelReg->u64Base;
+    }
+
+    /* Attributes. */
+    {
+        uint32_t const fValidAttrMask = X86DESCATTR_TYPE | X86DESCATTR_DT  | X86DESCATTR_DPL | X86DESCATTR_P
+                                      | X86DESCATTR_AVL  | X86DESCATTR_L   | X86DESCATTR_D   | X86DESCATTR_G
+                                      | X86DESCATTR_UNUSABLE;
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_32BIT;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, VMX_BF_VMCSFIELD_INDEX);
+        Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t        *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t        *pbField    = pbVmcs + offField;
+        *(uint32_t *)pbField       = pSelReg->Attr.u & fValidAttrMask;
+    }
+}
+
+
+/**
+ * Gets a guest segment register from the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVmcs       Pointer to the virtual VMCS.
+ * @param   iSegReg     The index of the segment register (X86_SREG_XXX).
+ * @param   pSelReg     Where to store the segment register (only updated when
+ *                      VINF_SUCCESS is returned).
+ *
+ * @remarks Warning! This does not validate the contents of the retrieved segment
+ *          register.
+ */
+IEM_STATIC int iemVmxVmcsGetGuestSegReg(PCVMXVVMCS pVmcs, uint8_t iSegReg, PCPUMSELREG pSelReg)
+{
+    Assert(pSelReg);
+    Assert(iSegReg < X86_SREG_COUNT);
+
+    /* Selector. */
+    uint16_t u16Sel;
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_16BIT;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS16_GUEST_ES_SEL, VMX_BF_VMCSFIELD_INDEX);
+        AssertReturn(uIndex <= VMX_V_VMCS_MAX_INDEX, VERR_IEM_IPE_3);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t  const *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t  const *pbField    = pbVmcs + offField;
+        u16Sel = *(uint16_t *)pbField;
+    }
+
+    /* Limit. */
+    uint32_t u32Limit;
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_32BIT;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS32_GUEST_ES_LIMIT, VMX_BF_VMCSFIELD_INDEX);
+        AssertReturn(uIndex <= VMX_V_VMCS_MAX_INDEX, VERR_IEM_IPE_3);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t  const *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t  const *pbField    = pbVmcs + offField;
+        u32Limit = *(uint32_t *)pbField;
+    }
+
+    /* Base. */
+    uint64_t u64Base;
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_NATURAL;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS_GUEST_ES_BASE, VMX_BF_VMCSFIELD_INDEX);
+        AssertReturn(uIndex <= VMX_V_VMCS_MAX_INDEX, VERR_IEM_IPE_3);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t  const *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t  const *pbField    = pbVmcs + offField;
+        u64Base = *(uint64_t *)pbField;
+        /** @todo NSTVMX: Should we zero out high bits here for 32-bit virtual CPUs? */
+    }
+
+    /* Attributes. */
+    uint32_t u32Attr;
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_32BIT;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, VMX_BF_VMCSFIELD_INDEX);
+        AssertReturn(uIndex <= VMX_V_VMCS_MAX_INDEX, VERR_IEM_IPE_3);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t  const *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t  const *pbField    = pbVmcs + offField;
+        u32Attr = *(uint32_t *)pbField;
+    }
+
+    pSelReg->Sel      = u16Sel;
+    pSelReg->ValidSel = u16Sel;
+    pSelReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+    pSelReg->u32Limit = u32Limit;
+    pSelReg->u64Base  = u64Base;
+    pSelReg->Attr.u   = u32Attr;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Converts an IEM exception event type to a VMX event type.
+ *
+ * @returns The VMX event type.
+ * @param   uVector     The interrupt / exception vector.
+ * @param   fFlags      The IEM event flag (see IEM_XCPT_FLAGS_XXX).
+ */
+DECLINLINE(uint8_t) iemVmxGetEventType(uint32_t uVector, uint32_t fFlags)
+{
+    /* Paranoia (callers may use these interchangeably). */
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_NMI          == VMX_IDT_VECTORING_INFO_TYPE_NMI);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_HW_XCPT      == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_EXT_INT      == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_SW_XCPT      == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_SW_INT       == VMX_IDT_VECTORING_INFO_TYPE_SW_INT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_NMI          == VMX_ENTRY_INT_INFO_TYPE_NMI);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_HW_XCPT      == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_EXT_INT      == VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_SW_XCPT      == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_SW_INT       == VMX_ENTRY_INT_INFO_TYPE_SW_INT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT == VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT);
+
+    if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+    {
+        if (uVector == X86_XCPT_NMI)
+            return VMX_EXIT_INT_INFO_TYPE_NMI;
+        return VMX_EXIT_INT_INFO_TYPE_HW_XCPT;
+    }
+
+    if (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
+    {
+        if (fFlags & (IEM_XCPT_FLAGS_BP_INSTR | IEM_XCPT_FLAGS_OF_INSTR))
+            return VMX_EXIT_INT_INFO_TYPE_SW_XCPT;
+        if (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)
+            return VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT;
+        return VMX_EXIT_INT_INFO_TYPE_SW_INT;
+    }
+
+    Assert(fFlags & IEM_XCPT_FLAGS_T_EXT_INT);
+    return VMX_EXIT_INT_INFO_TYPE_EXT_INT;
+}
+
+
+/**
+ * Sets the Exit qualification VMCS field.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u64ExitQual     The Exit qualification.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitQual(PVMCPUCC pVCpu, uint64_t u64ExitQual)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    pVmcs->u64RoExitQual.u = u64ExitQual;
+}
+
+
+/**
+ * Sets the VM-exit interruption information field.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitIntInfo    The VM-exit interruption information.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitIntInfo(PVMCPUCC pVCpu, uint32_t uExitIntInfo)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    pVmcs->u32RoExitIntInfo = uExitIntInfo;
+}
+
+
+/**
+ * Sets the VM-exit interruption error code.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uErrCode    The error code.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitIntErrCode(PVMCPUCC pVCpu, uint32_t uErrCode)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    pVmcs->u32RoExitIntErrCode = uErrCode;
+}
+
+
+/**
+ * Sets the IDT-vectoring information field.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uIdtVectorInfo  The IDT-vectoring information.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetIdtVectoringInfo(PVMCPUCC pVCpu, uint32_t uIdtVectorInfo)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    pVmcs->u32RoIdtVectoringInfo = uIdtVectorInfo;
+}
+
+
+/**
+ * Sets the IDT-vectoring error code field.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uErrCode    The error code.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetIdtVectoringErrCode(PVMCPUCC pVCpu, uint32_t uErrCode)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    pVmcs->u32RoIdtVectoringErrCode = uErrCode;
+}
+
+
+/**
+ * Sets the VM-exit guest-linear address VMCS field.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uGuestLinearAddr    The VM-exit guest-linear address.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitGuestLinearAddr(PVMCPUCC pVCpu, uint64_t uGuestLinearAddr)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    pVmcs->u64RoGuestLinearAddr.u = uGuestLinearAddr;
+}
+
+
+/**
+ * Sets the VM-exit guest-physical address VMCS field.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uGuestPhysAddr  The VM-exit guest-physical address.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitGuestPhysAddr(PVMCPUCC pVCpu, uint64_t uGuestPhysAddr)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    pVmcs->u64RoGuestPhysAddr.u = uGuestPhysAddr;
+}
+
+
+/**
+ * Sets the VM-exit instruction length VMCS field.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The VM-exit instruction length in bytes.
+ *
+ * @remarks Callers may clear this field to 0. Hence, this function does not check
+ *          the validity of the instruction length.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitInstrLen(PVMCPUCC pVCpu, uint32_t cbInstr)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    pVmcs->u32RoExitInstrLen = cbInstr;
+}
+
+
+/**
+ * Sets the VM-exit instruction info. VMCS field.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitInstrInfo  The VM-exit instruction information.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitInstrInfo(PVMCPUCC pVCpu, uint32_t uExitInstrInfo)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    pVmcs->u32RoExitInstrInfo = uExitInstrInfo;
+}
+
+
+/**
+ * Sets the guest pending-debug exceptions field.
+ *
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   uGuestPendingDbgXcpts   The guest pending-debug exceptions.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetGuestPendingDbgXcpts(PVMCPUCC pVCpu, uint64_t uGuestPendingDbgXcpts)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(!(uGuestPendingDbgXcpts & VMX_VMCS_GUEST_PENDING_DEBUG_VALID_MASK));
+    pVmcs->u64GuestPendingDbgXcpts.u = uGuestPendingDbgXcpts;
+}
+
+
+/**
+ * Implements VMSucceed for VMX instruction success.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmSucceed(PVMCPUCC pVCpu)
+{
+    return CPUMSetGuestVmxVmSucceed(&pVCpu->cpum.GstCtx);
+}
+
+
+/**
+ * Implements VMFailInvalid for VMX instruction failure.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmFailInvalid(PVMCPUCC pVCpu)
+{
+    return CPUMSetGuestVmxVmFailInvalid(&pVCpu->cpum.GstCtx);
+}
+
+
+/**
+ * Implements VMFail for VMX instruction failure.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   enmInsErr   The VM instruction error.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmFail(PVMCPUCC pVCpu, VMXINSTRERR enmInsErr)
+{
+    return CPUMSetGuestVmxVmFail(&pVCpu->cpum.GstCtx, enmInsErr);
+}
+
+
+/**
+ * Checks if the given auto-load/store MSR area count is valid for the
+ * implementation.
+ *
+ * @returns @c true if it's within the valid limit, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uMsrCount   The MSR area count to check.
+ */
+DECL_FORCE_INLINE(bool) iemVmxIsAutoMsrCountValid(PCVMCPU pVCpu, uint32_t uMsrCount)
+{
+    uint64_t const u64VmxMiscMsr      = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Misc;
+    uint32_t const cMaxSupportedMsrs  = VMX_MISC_MAX_MSRS(u64VmxMiscMsr);
+    Assert(cMaxSupportedMsrs <= VMX_V_AUTOMSR_AREA_SIZE / sizeof(VMXAUTOMSR));
+    if (uMsrCount <= cMaxSupportedMsrs)
+        return true;
+    return false;
+}
+
+
+/**
+ * Flushes the current VMCS contents back to guest memory.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECL_FORCE_INLINE(int) iemVmxWriteCurrentVmcsToGstMem(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs));
+    Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
+    int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), IEM_VMX_GET_CURRENT_VMCS(pVCpu),
+                                      pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs), sizeof(VMXVVMCS));
+    return rc;
+}
+
+
+/**
+ * Populates the current VMCS contents from guest memory.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECL_FORCE_INLINE(int) iemVmxReadCurrentVmcsFromGstMem(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs));
+    Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), (void *)pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs),
+                                     IEM_VMX_GET_CURRENT_VMCS(pVCpu), sizeof(VMXVVMCS));
+    return rc;
+}
+
+
+/**
+ * Implements VMSucceed for the VMREAD instruction and increments the guest RIP.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmreadSuccess(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    iemVmxVmSucceed(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment base checks during VM-entry of a
+ * nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+IEM_STATIC VMXVDIAG iemVmxGetDiagVmentrySegBase(unsigned iSegReg)
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegBaseCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegBaseDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegBaseEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegBaseFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegBaseGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegBaseSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_1);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment base checks during VM-entry of a
+ * nested-guest that is in Virtual-8086 mode.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+IEM_STATIC VMXVDIAG iemVmxGetDiagVmentrySegBaseV86(unsigned iSegReg)
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegBaseV86Cs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegBaseV86Ds;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegBaseV86Es;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegBaseV86Fs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegBaseV86Gs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegBaseV86Ss;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_2);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment limit checks during VM-entry of a
+ * nested-guest that is in Virtual-8086 mode.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+IEM_STATIC VMXVDIAG iemVmxGetDiagVmentrySegLimitV86(unsigned iSegReg)
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegLimitV86Cs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegLimitV86Ds;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegLimitV86Es;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegLimitV86Fs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegLimitV86Gs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegLimitV86Ss;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_3);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment attribute checks during VM-entry of a
+ * nested-guest that is in Virtual-8086 mode.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+IEM_STATIC VMXVDIAG iemVmxGetDiagVmentrySegAttrV86(unsigned iSegReg)
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrV86Cs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrV86Ds;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrV86Es;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrV86Fs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrV86Gs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrV86Ss;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_4);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment attributes reserved bits failure
+ * during VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+IEM_STATIC VMXVDIAG iemVmxGetDiagVmentrySegAttrRsvd(unsigned iSegReg)
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrRsvdEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_5);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment attributes descriptor-type
+ * (code/segment or system) failure during VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+IEM_STATIC VMXVDIAG iemVmxGetDiagVmentrySegAttrDescType(unsigned iSegReg)
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_6);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment attributes descriptor-type
+ * (code/segment or system) failure during VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+IEM_STATIC VMXVDIAG iemVmxGetDiagVmentrySegAttrPresent(unsigned iSegReg)
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrPresentCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrPresentDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrPresentEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrPresentFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrPresentGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrPresentSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_7);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment attribute granularity failure during
+ * VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+IEM_STATIC VMXVDIAG iemVmxGetDiagVmentrySegAttrGran(unsigned iSegReg)
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrGranCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrGranDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrGranEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrGranFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrGranGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrGranSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_8);
+    }
+}
+
+/**
+ * Gets the instruction diagnostic for segment attribute DPL/RPL failure during
+ * VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+IEM_STATIC VMXVDIAG iemVmxGetDiagVmentrySegAttrDplRpl(unsigned iSegReg)
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrDplRplEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_9);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment attribute type accessed failure
+ * during VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+IEM_STATIC VMXVDIAG iemVmxGetDiagVmentrySegAttrTypeAcc(unsigned iSegReg)
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_10);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for guest CR3 referenced PDPTE reserved bits
+ * failure during VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The PDPTE entry index.
+ */
+IEM_STATIC VMXVDIAG iemVmxGetDiagVmentryPdpteRsvd(unsigned iPdpte)
+{
+    Assert(iPdpte < X86_PG_PAE_PDPE_ENTRIES);
+    switch (iPdpte)
+    {
+        case 0: return kVmxVDiag_Vmentry_GuestPdpte0Rsvd;
+        case 1: return kVmxVDiag_Vmentry_GuestPdpte1Rsvd;
+        case 2: return kVmxVDiag_Vmentry_GuestPdpte2Rsvd;
+        case 3: return kVmxVDiag_Vmentry_GuestPdpte3Rsvd;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_11);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for host CR3 referenced PDPTE reserved bits
+ * failure during VM-exit of a nested-guest.
+ *
+ * @param   iSegReg     The PDPTE entry index.
+ */
+IEM_STATIC VMXVDIAG iemVmxGetDiagVmexitPdpteRsvd(unsigned iPdpte)
+{
+    Assert(iPdpte < X86_PG_PAE_PDPE_ENTRIES);
+    switch (iPdpte)
+    {
+        case 0: return kVmxVDiag_Vmexit_HostPdpte0Rsvd;
+        case 1: return kVmxVDiag_Vmexit_HostPdpte1Rsvd;
+        case 2: return kVmxVDiag_Vmexit_HostPdpte2Rsvd;
+        case 3: return kVmxVDiag_Vmexit_HostPdpte3Rsvd;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_12);
+    }
+}
+
+
+/**
+ * Saves the guest control registers, debug registers and some MSRs are part of
+ * VM-exit.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC void iemVmxVmexitSaveGuestControlRegsMsrs(PVMCPUCC pVCpu)
+{
+    /*
+     * Saves the guest control registers, debug registers and some MSRs.
+     * See Intel spec. 27.3.1 "Saving Control Registers, Debug Registers and MSRs".
+     */
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+
+    /* Save control registers. */
+    pVmcs->u64GuestCr0.u = pVCpu->cpum.GstCtx.cr0;
+    pVmcs->u64GuestCr3.u = pVCpu->cpum.GstCtx.cr3;
+    pVmcs->u64GuestCr4.u = pVCpu->cpum.GstCtx.cr4;
+
+    /* Save SYSENTER CS, ESP, EIP. */
+    pVmcs->u32GuestSysenterCS    = pVCpu->cpum.GstCtx.SysEnter.cs;
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        pVmcs->u64GuestSysenterEsp.u = pVCpu->cpum.GstCtx.SysEnter.esp;
+        pVmcs->u64GuestSysenterEip.u = pVCpu->cpum.GstCtx.SysEnter.eip;
+    }
+    else
+    {
+        pVmcs->u64GuestSysenterEsp.s.Lo = pVCpu->cpum.GstCtx.SysEnter.esp;
+        pVmcs->u64GuestSysenterEip.s.Lo = pVCpu->cpum.GstCtx.SysEnter.eip;
+    }
+
+    /* Save debug registers (DR7 and IA32_DEBUGCTL MSR). */
+    if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_DEBUG)
+    {
+        pVmcs->u64GuestDr7.u = pVCpu->cpum.GstCtx.dr[7];
+        /** @todo NSTVMX: Support IA32_DEBUGCTL MSR */
+    }
+
+    /* Save PAT MSR. */
+    if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_PAT_MSR)
+        pVmcs->u64GuestPatMsr.u = pVCpu->cpum.GstCtx.msrPAT;
+
+    /* Save EFER MSR. */
+    if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_EFER_MSR)
+        pVmcs->u64GuestEferMsr.u = pVCpu->cpum.GstCtx.msrEFER;
+
+    /* We don't support clearing IA32_BNDCFGS MSR yet. */
+    Assert(!(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_CLEAR_BNDCFGS_MSR));
+
+    /* Nothing to do for SMBASE register - We don't support SMM yet. */
+}
+
+
+/**
+ * Saves the guest force-flags in preparation of entering the nested-guest.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC void iemVmxVmentrySaveNmiBlockingFF(PVMCPUCC pVCpu)
+{
+    /* We shouldn't be called multiple times during VM-entry. */
+    Assert(pVCpu->cpum.GstCtx.hwvirt.fLocalForcedActions == 0);
+
+    /* MTF should not be set outside VMX non-root mode. */
+    Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
+
+    /*
+     * Preserve the required force-flags.
+     *
+     * We cache and clear force-flags that would affect the execution of the
+     * nested-guest. Cached flags are then restored while returning to the guest
+     * if necessary.
+     *
+     *   - VMCPU_FF_INHIBIT_INTERRUPTS need not be cached as it only affects
+     *     interrupts until the completion of the current VMLAUNCH/VMRESUME
+     *     instruction. Interrupt inhibition for any nested-guest instruction
+     *     is supplied by the guest-interruptibility state VMCS field and will
+     *     be set up as part of loading the guest state.
+     *
+     *   - VMCPU_FF_BLOCK_NMIS needs to be cached as VM-exits caused before
+     *     successful VM-entry (due to invalid guest-state) need to continue
+     *     blocking NMIs if it was in effect before VM-entry.
+     *
+     *   - MTF need not be preserved as it's used only in VMX non-root mode and
+     *     is supplied through the VM-execution controls.
+     *
+     * The remaining FFs (e.g. timers, APIC updates) can stay in place so that
+     * we will be able to generate interrupts that may cause VM-exits for
+     * the nested-guest.
+     */
+    pVCpu->cpum.GstCtx.hwvirt.fLocalForcedActions = pVCpu->fLocalForcedActions & VMCPU_FF_BLOCK_NMIS;
+}
+
+
+/**
+ * Restores the guest force-flags in preparation of exiting the nested-guest.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC void iemVmxVmexitRestoreNmiBlockingFF(PVMCPUCC pVCpu)
+{
+    if (pVCpu->cpum.GstCtx.hwvirt.fLocalForcedActions)
+    {
+        VMCPU_FF_SET_MASK(pVCpu, pVCpu->cpum.GstCtx.hwvirt.fLocalForcedActions);
+        pVCpu->cpum.GstCtx.hwvirt.fLocalForcedActions = 0;
+    }
+}
+
+
+/**
+ * Perform a VMX transition updated PGM, IEM and CPUM.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC int iemVmxWorldSwitch(PVMCPUCC pVCpu)
+{
+    /*
+     * Inform PGM about paging mode changes.
+     * We include X86_CR0_PE because PGM doesn't handle paged-real mode yet,
+     * see comment in iemMemPageTranslateAndCheckAccess().
+     */
+    int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0 | X86_CR0_PE, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER);
+# ifdef IN_RING3
+    Assert(rc != VINF_PGM_CHANGE_MODE);
+# endif
+    AssertRCReturn(rc, rc);
+
+    /* Inform CPUM (recompiler), can later be removed. */
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
+
+    /*
+     * Flush the TLB with new CR3. This is required in case the PGM mode change
+     * above doesn't actually change anything.
+     */
+    if (rc == VINF_SUCCESS)
+    {
+        rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true);
+        AssertRCReturn(rc, rc);
+    }
+
+    /* Re-initialize IEM cache/state after the drastic mode switch. */
+    iemReInitExec(pVCpu);
+    return rc;
+}
+
+
+/**
+ * Calculates the current VMX-preemption timer value.
+ *
+ * @returns The current VMX-preemption timer value.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC uint32_t iemVmxCalcPreemptTimer(PVMCPUCC pVCpu)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    /*
+     * Assume the following:
+     * PreemptTimerShift = 5
+     * VmcsPreemptTimer  = 2 (i.e. need to decrement by 1 every 2 * RT_BIT(5) = 20000 TSC ticks)
+     * EntryTick         = 50000 (TSC at time of VM-entry)
+     *
+     * CurTick   Delta    PreemptTimerVal
+     * ----------------------------------
+     *  60000    10000    2
+     *  80000    30000    1
+     *  90000    40000    0  -> VM-exit.
+     *
+     * If Delta >= VmcsPreemptTimer * RT_BIT(PreemptTimerShift) cause a VMX-preemption timer VM-exit.
+     * The saved VMX-preemption timer value is calculated as follows:
+     * PreemptTimerVal = VmcsPreemptTimer - (Delta / (VmcsPreemptTimer * RT_BIT(PreemptTimerShift)))
+     * E.g.:
+     *  Delta  = 10000
+     *    Tmp    = 10000 / (2 * 10000) = 0.5
+     *    NewPt  = 2 - 0.5 = 2
+     *  Delta  = 30000
+     *    Tmp    = 30000 / (2 * 10000) = 1.5
+     *    NewPt  = 2 - 1.5 = 1
+     *  Delta  = 40000
+     *    Tmp    = 40000 / 20000 = 2
+     *    NewPt  = 2 - 2 = 0
+     */
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+    uint32_t const uVmcsPreemptVal = pVmcs->u32PreemptTimer;
+    if (uVmcsPreemptVal > 0)
+    {
+        uint64_t const uCurTick      = TMCpuTickGetNoCheck(pVCpu);
+        uint64_t const uEntryTick    = pVCpu->cpum.GstCtx.hwvirt.vmx.uEntryTick;
+        uint64_t const uDelta        = uCurTick - uEntryTick;
+        uint32_t const uPreemptTimer = uVmcsPreemptVal
+                                     - ASMDivU64ByU32RetU32(uDelta, uVmcsPreemptVal * RT_BIT(VMX_V_PREEMPT_TIMER_SHIFT));
+        return uPreemptTimer;
+    }
+    return 0;
+}
+
+
+/**
+ * Saves guest segment registers, GDTR, IDTR, LDTR, TR as part of VM-exit.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC void iemVmxVmexitSaveGuestSegRegs(PVMCPUCC pVCpu)
+{
+    /*
+     * Save guest segment registers, GDTR, IDTR, LDTR, TR.
+     * See Intel spec 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
+     */
+    /* CS, SS, ES, DS, FS, GS. */
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    for (unsigned iSegReg = 0; iSegReg < X86_SREG_COUNT; iSegReg++)
+    {
+        PCCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
+        if (!pSelReg->Attr.n.u1Unusable)
+            iemVmxVmcsSetGuestSegReg(pVmcs, iSegReg, pSelReg);
+        else
+        {
+            /*
+             * For unusable segments the attributes are undefined except for CS and SS.
+             * For the rest we don't bother preserving anything but the unusable bit.
+             */
+            switch (iSegReg)
+            {
+                case X86_SREG_CS:
+                    pVmcs->GuestCs          = pSelReg->Sel;
+                    pVmcs->u64GuestCsBase.u = pSelReg->u64Base;
+                    pVmcs->u32GuestCsLimit  = pSelReg->u32Limit;
+                    pVmcs->u32GuestCsAttr   = pSelReg->Attr.u & (  X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
+                                                                 | X86DESCATTR_UNUSABLE);
+                    break;
+
+                case X86_SREG_SS:
+                    pVmcs->GuestSs        = pSelReg->Sel;
+                    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+                        pVmcs->u64GuestSsBase.u &= UINT32_C(0xffffffff);
+                    pVmcs->u32GuestSsAttr = pSelReg->Attr.u & (X86DESCATTR_DPL | X86DESCATTR_UNUSABLE);
+                    break;
+
+                case X86_SREG_DS:
+                    pVmcs->GuestDs        = pSelReg->Sel;
+                    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+                        pVmcs->u64GuestDsBase.u &= UINT32_C(0xffffffff);
+                    pVmcs->u32GuestDsAttr = X86DESCATTR_UNUSABLE;
+                    break;
+
+                case X86_SREG_ES:
+                    pVmcs->GuestEs        = pSelReg->Sel;
+                    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+                        pVmcs->u64GuestEsBase.u &= UINT32_C(0xffffffff);
+                    pVmcs->u32GuestEsAttr = X86DESCATTR_UNUSABLE;
+                    break;
+
+                case X86_SREG_FS:
+                    pVmcs->GuestFs          = pSelReg->Sel;
+                    pVmcs->u64GuestFsBase.u = pSelReg->u64Base;
+                    pVmcs->u32GuestFsAttr   = X86DESCATTR_UNUSABLE;
+                    break;
+
+                case X86_SREG_GS:
+                    pVmcs->GuestGs          = pSelReg->Sel;
+                    pVmcs->u64GuestGsBase.u = pSelReg->u64Base;
+                    pVmcs->u32GuestGsAttr   = X86DESCATTR_UNUSABLE;
+                    break;
+            }
+        }
+    }
+
+    /* Segment attribute bits 31:17 and 11:8 MBZ. */
+    uint32_t const fValidAttrMask = X86DESCATTR_TYPE | X86DESCATTR_DT  | X86DESCATTR_DPL | X86DESCATTR_P
+                                  | X86DESCATTR_AVL  | X86DESCATTR_L   | X86DESCATTR_D   | X86DESCATTR_G
+                                  | X86DESCATTR_UNUSABLE;
+    /* LDTR. */
+    {
+        PCCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.ldtr;
+        pVmcs->GuestLdtr          = pSelReg->Sel;
+        pVmcs->u64GuestLdtrBase.u = pSelReg->u64Base;
+        Assert(X86_IS_CANONICAL(pSelReg->u64Base));
+        pVmcs->u32GuestLdtrLimit  = pSelReg->u32Limit;
+        pVmcs->u32GuestLdtrAttr   = pSelReg->Attr.u & fValidAttrMask;
+    }
+
+    /* TR. */
+    {
+        PCCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.tr;
+        pVmcs->GuestTr          = pSelReg->Sel;
+        pVmcs->u64GuestTrBase.u = pSelReg->u64Base;
+        pVmcs->u32GuestTrLimit  = pSelReg->u32Limit;
+        pVmcs->u32GuestTrAttr   = pSelReg->Attr.u & fValidAttrMask;
+    }
+
+    /* GDTR. */
+    pVmcs->u64GuestGdtrBase.u = pVCpu->cpum.GstCtx.gdtr.pGdt;
+    pVmcs->u32GuestGdtrLimit  = pVCpu->cpum.GstCtx.gdtr.cbGdt;
+
+    /* IDTR. */
+    pVmcs->u64GuestIdtrBase.u = pVCpu->cpum.GstCtx.idtr.pIdt;
+    pVmcs->u32GuestIdtrLimit  = pVCpu->cpum.GstCtx.idtr.cbIdt;
+}
+
+
+/**
+ * Saves guest non-register state as part of VM-exit.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ */
+IEM_STATIC void iemVmxVmexitSaveGuestNonRegState(PVMCPUCC pVCpu, uint32_t uExitReason)
+{
+    /*
+     * Save guest non-register state.
+     * See Intel spec. 27.3.4 "Saving Non-Register State".
+     */
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+
+    /*
+     * Activity state.
+     * Most VM-exits will occur in the active state. However, if the first instruction
+     * following the VM-entry is a HLT instruction, and the MTF VM-execution control is set,
+     * the VM-exit will be from the HLT activity state.
+     *
+     * See Intel spec. 25.5.2 "Monitor Trap Flag".
+     */
+    /** @todo NSTVMX: Does triple-fault VM-exit reflect a shutdown activity state or
+     *        not? */
+    EMSTATE const enmActivityState = EMGetState(pVCpu);
+    switch (enmActivityState)
+    {
+        case EMSTATE_HALTED:    pVmcs->u32GuestActivityState = VMX_VMCS_GUEST_ACTIVITY_HLT;     break;
+        default:                pVmcs->u32GuestActivityState = VMX_VMCS_GUEST_ACTIVITY_ACTIVE;  break;
+    }
+
+    /*
+     * Interruptibility-state.
+     */
+    /* NMI. */
+    pVmcs->u32GuestIntrState = 0;
+    if (pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+    {
+        if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking)
+            pVmcs->u32GuestIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
+    }
+    else
+    {
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+            pVmcs->u32GuestIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
+    }
+
+    /* Blocking-by-STI. */
+    if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+        && pVCpu->cpum.GstCtx.rip == EMGetInhibitInterruptsPC(pVCpu))
+    {
+        /** @todo NSTVMX: We can't distinguish between blocking-by-MovSS and blocking-by-STI
+         *        currently. */
+        pVmcs->u32GuestIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
+    }
+    /* Nothing to do for SMI/enclave. We don't support enclaves or SMM yet. */
+
+    /*
+     * Pending debug exceptions.
+     *
+     * For VM-exits where it is not applicable, we can safely zero out the field.
+     * For VM-exits where it is applicable, it's expected to be updated by the caller already.
+     */
+    if (    uExitReason != VMX_EXIT_INIT_SIGNAL
+        &&  uExitReason != VMX_EXIT_SMI
+        &&  uExitReason != VMX_EXIT_ERR_MACHINE_CHECK
+        && !VMXIsVmexitTrapLike(uExitReason))
+    {
+        /** @todo NSTVMX: also must exclude VM-exits caused by debug exceptions when
+         *        block-by-MovSS is in effect. */
+        pVmcs->u64GuestPendingDbgXcpts.u = 0;
+    }
+
+    /*
+     * Save the VMX-preemption timer value back into the VMCS if the feature is enabled.
+     *
+     * For VMX-preemption timer VM-exits, we should have already written back 0 if the
+     * feature is supported back into the VMCS, and thus there is nothing further to do here.
+     */
+    if (   uExitReason != VMX_EXIT_PREEMPT_TIMER
+        && (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
+        pVmcs->u32PreemptTimer = iemVmxCalcPreemptTimer(pVCpu);
+
+    /* PDPTEs. */
+    /* We don't support EPT yet. */
+    Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT));
+    pVmcs->u64GuestPdpte0.u = 0;
+    pVmcs->u64GuestPdpte1.u = 0;
+    pVmcs->u64GuestPdpte2.u = 0;
+    pVmcs->u64GuestPdpte3.u = 0;
+}
+
+
+/**
+ * Saves the guest-state as part of VM-exit.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ */
+IEM_STATIC void iemVmxVmexitSaveGuestState(PVMCPUCC pVCpu, uint32_t uExitReason)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    iemVmxVmexitSaveGuestControlRegsMsrs(pVCpu);
+    iemVmxVmexitSaveGuestSegRegs(pVCpu);
+
+    pVmcs->u64GuestRip.u    = pVCpu->cpum.GstCtx.rip;
+    pVmcs->u64GuestRsp.u    = pVCpu->cpum.GstCtx.rsp;
+    pVmcs->u64GuestRFlags.u = pVCpu->cpum.GstCtx.rflags.u;  /** @todo NSTVMX: Check RFLAGS.RF handling. */
+
+    iemVmxVmexitSaveGuestNonRegState(pVCpu, uExitReason);
+}
+
+
+/**
+ * Saves the guest MSRs into the VM-exit MSR-store area as part of VM-exit.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason (for diagnostic purposes).
+ */
+IEM_STATIC int iemVmxVmexitSaveGuestAutoMsrs(PVMCPUCC pVCpu, uint32_t uExitReason)
+{
+    /*
+     * Save guest MSRs.
+     * See Intel spec. 27.4 "Saving MSRs".
+     */
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    const char *const pszFailure = "VMX-abort";
+
+    /*
+     * The VM-exit MSR-store area address need not be a valid guest-physical address if the
+     * VM-exit MSR-store count is 0. If this is the case, bail early without reading it.
+     * See Intel spec. 24.7.2 "VM-Exit Controls for MSRs".
+     */
+    uint32_t const cMsrs = pVmcs->u32ExitMsrStoreCount;
+    if (!cMsrs)
+        return VINF_SUCCESS;
+
+    /*
+     * Verify the MSR auto-store count. Physical CPUs can behave unpredictably if the count
+     * is exceeded including possibly raising #MC exceptions during VMX transition. Our
+     * implementation causes a VMX-abort followed by a triple-fault.
+     */
+    bool const fIsMsrCountValid = iemVmxIsAutoMsrCountValid(pVCpu, cMsrs);
+    if (fIsMsrCountValid)
+    { /* likely */ }
+    else
+        IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrStoreCount);
+
+    /*
+     * Optimization if the nested hypervisor is using the same guest-physical page for both
+     * the VM-entry MSR-load area as well as the VM-exit MSR store area.
+     */
+    PVMXAUTOMSR    pMsrArea;
+    RTGCPHYS const GCPhysVmEntryMsrLoadArea = pVmcs->u64AddrEntryMsrLoad.u;
+    RTGCPHYS const GCPhysVmExitMsrStoreArea = pVmcs->u64AddrExitMsrStore.u;
+    if (GCPhysVmEntryMsrLoadArea == GCPhysVmExitMsrStoreArea)
+        pMsrArea = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pEntryMsrLoadArea);
+    else
+    {
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), (void *)pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pExitMsrStoreArea),
+                                         GCPhysVmExitMsrStoreArea, cMsrs * sizeof(VMXAUTOMSR));
+        if (RT_SUCCESS(rc))
+            pMsrArea = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pExitMsrStoreArea);
+        else
+        {
+            AssertMsgFailed(("VM-exit: Failed to read MSR auto-store area at %#RGp, rc=%Rrc\n", GCPhysVmExitMsrStoreArea, rc));
+            IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrStorePtrReadPhys);
+        }
+    }
+
+    /*
+     * Update VM-exit MSR store area.
+     */
+    PVMXAUTOMSR pMsr = pMsrArea;
+    Assert(pMsr);
+    for (uint32_t idxMsr = 0; idxMsr < cMsrs; idxMsr++, pMsr++)
+    {
+        if (   !pMsr->u32Reserved
+            &&  pMsr->u32Msr != MSR_IA32_SMBASE
+            &&  pMsr->u32Msr >> 8 != MSR_IA32_X2APIC_START >> 8)
+        {
+            VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pMsr->u32Msr, &pMsr->u64Value);
+            if (rcStrict == VINF_SUCCESS)
+                continue;
+
+            /*
+             * If we're in ring-0, we cannot handle returns to ring-3 at this point and continue VM-exit.
+             * If any nested hypervisor loads MSRs that require ring-3 handling, we cause a VMX-abort
+             * recording the MSR index in the auxiliary info. field and indicated further by our
+             * own, specific diagnostic code. Later, we can try implement handling of the MSR in ring-0
+             * if possible, or come up with a better, generic solution.
+             */
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uAbortAux = pMsr->u32Msr;
+            VMXVDIAG const enmDiag = rcStrict == VINF_CPUM_R3_MSR_READ
+                                   ? kVmxVDiag_Vmexit_MsrStoreRing3
+                                   : kVmxVDiag_Vmexit_MsrStore;
+            IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, enmDiag);
+        }
+        else
+        {
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uAbortAux = pMsr->u32Msr;
+            IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrStoreRsvd);
+        }
+    }
+
+    /*
+     * Commit the VM-exit MSR store are to guest memory.
+     */
+    int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVmExitMsrStoreArea, pMsrArea, cMsrs * sizeof(VMXAUTOMSR));
+    if (RT_SUCCESS(rc))
+        return VINF_SUCCESS;
+
+    NOREF(uExitReason);
+    NOREF(pszFailure);
+
+    AssertMsgFailed(("VM-exit: Failed to write MSR auto-store area at %#RGp, rc=%Rrc\n", GCPhysVmExitMsrStoreArea, rc));
+    IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrStorePtrWritePhys);
+}
+
+
+/**
+ * Performs a VMX abort (due to an fatal error during VM-exit).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   enmAbort    The VMX abort reason.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxAbort(PVMCPUCC pVCpu, VMXABORT enmAbort)
+{
+    /*
+     * Perform the VMX abort.
+     * See Intel spec. 27.7 "VMX Aborts".
+     */
+    LogFunc(("enmAbort=%u (%s) -> RESET\n", enmAbort, VMXGetAbortDesc(enmAbort)));
+
+    /* We don't support SMX yet. */
+    pVCpu->cpum.GstCtx.hwvirt.vmx.enmAbort = enmAbort;
+    if (IEM_VMX_HAS_CURRENT_VMCS(pVCpu))
+    {
+        RTGCPHYS const GCPhysVmcs  = IEM_VMX_GET_CURRENT_VMCS(pVCpu);
+        uint32_t const offVmxAbort = RT_UOFFSETOF(VMXVVMCS, enmVmxAbort);
+        PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVmcs + offVmxAbort, &enmAbort, sizeof(enmAbort));
+    }
+
+    return VINF_EM_TRIPLE_FAULT;
+}
+
+
+/**
+ * Loads host control registers, debug registers and MSRs as part of VM-exit.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC void iemVmxVmexitLoadHostControlRegsMsrs(PVMCPUCC pVCpu)
+{
+    /*
+     * Load host control registers, debug registers and MSRs.
+     * See Intel spec. 27.5.1 "Loading Host Control Registers, Debug Registers, MSRs".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    bool const fHostInLongMode = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE);
+
+    /* CR0. */
+    {
+        /* Bits 63:32, 28:19, 17, 15:6, ET, CD, NW and CR0 fixed bits are not modified. */
+        uint64_t const uCr0Mb1       = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed0;
+        uint64_t const uCr0Mb0       = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1;
+        uint64_t const fCr0IgnMask   = VMX_EXIT_HOST_CR0_IGNORE_MASK | uCr0Mb1 | ~uCr0Mb0;
+        uint64_t const uHostCr0      = pVmcs->u64HostCr0.u;
+        uint64_t const uGuestCr0     = pVCpu->cpum.GstCtx.cr0;
+        uint64_t const uValidHostCr0 = (uHostCr0 & ~fCr0IgnMask) | (uGuestCr0 & fCr0IgnMask);
+
+        /* Verify we have not modified CR0 fixed bits in VMX non-root operation. */
+        Assert((uGuestCr0 &  uCr0Mb1) == uCr0Mb1);
+        Assert((uGuestCr0 & ~uCr0Mb0) == 0);
+        CPUMSetGuestCR0(pVCpu, uValidHostCr0);
+    }
+
+    /* CR4. */
+    {
+        /* CR4 fixed bits are not modified. */
+        uint64_t const uCr4Mb1       = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0;
+        uint64_t const uCr4Mb0       = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1;
+        uint64_t const fCr4IgnMask   = uCr4Mb1 | ~uCr4Mb0;
+        uint64_t const uHostCr4      = pVmcs->u64HostCr4.u;
+        uint64_t const uGuestCr4     = pVCpu->cpum.GstCtx.cr4;
+        uint64_t       uValidHostCr4 = (uHostCr4 & ~fCr4IgnMask) | (uGuestCr4 & fCr4IgnMask);
+        if (fHostInLongMode)
+            uValidHostCr4 |= X86_CR4_PAE;
+        else
+            uValidHostCr4 &= ~(uint64_t)X86_CR4_PCIDE;
+
+        /* Verify we have not modified CR4 fixed bits in VMX non-root operation. */
+        Assert((uGuestCr4 &  uCr4Mb1) == uCr4Mb1);
+        Assert((uGuestCr4 & ~uCr4Mb0) == 0);
+        CPUMSetGuestCR4(pVCpu, uValidHostCr4);
+    }
+
+    /* CR3 (host value validated while checking host-state during VM-entry). */
+    pVCpu->cpum.GstCtx.cr3 = pVmcs->u64HostCr3.u;
+
+    /* DR7. */
+    pVCpu->cpum.GstCtx.dr[7] = X86_DR7_INIT_VAL;
+
+    /** @todo NSTVMX: Support IA32_DEBUGCTL MSR */
+
+    /* Save SYSENTER CS, ESP, EIP (host value validated while checking host-state during VM-entry). */
+    pVCpu->cpum.GstCtx.SysEnter.eip = pVmcs->u64HostSysenterEip.u;
+    pVCpu->cpum.GstCtx.SysEnter.esp = pVmcs->u64HostSysenterEsp.u;
+    pVCpu->cpum.GstCtx.SysEnter.cs  = pVmcs->u32HostSysenterCs;
+
+    /* FS, GS bases are loaded later while we load host segment registers. */
+
+    /* EFER MSR (host value validated while checking host-state during VM-entry). */
+    if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
+        pVCpu->cpum.GstCtx.msrEFER = pVmcs->u64HostEferMsr.u;
+    else if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        if (fHostInLongMode)
+            pVCpu->cpum.GstCtx.msrEFER |=  (MSR_K6_EFER_LMA | MSR_K6_EFER_LME);
+        else
+            pVCpu->cpum.GstCtx.msrEFER &= ~(MSR_K6_EFER_LMA | MSR_K6_EFER_LME);
+    }
+
+    /* We don't support IA32_PERF_GLOBAL_CTRL MSR yet. */
+
+    /* PAT MSR (host value is validated while checking host-state during VM-entry). */
+    if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_PAT_MSR)
+        pVCpu->cpum.GstCtx.msrPAT = pVmcs->u64HostPatMsr.u;
+
+    /* We don't support IA32_BNDCFGS MSR yet. */
+}
+
+
+/**
+ * Loads host segment registers, GDTR, IDTR, LDTR and TR as part of VM-exit.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC void iemVmxVmexitLoadHostSegRegs(PVMCPUCC pVCpu)
+{
+    /*
+     * Load host segment registers, GDTR, IDTR, LDTR and TR.
+     * See Intel spec. 27.5.2 "Loading Host Segment and Descriptor-Table Registers".
+     *
+     * Warning! Be careful to not touch fields that are reserved by VT-x,
+     * e.g. segment limit high bits stored in segment attributes (in bits 11:8).
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    bool const fHostInLongMode = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE);
+
+    /* CS, SS, ES, DS, FS, GS. */
+    for (unsigned iSegReg = 0; iSegReg < X86_SREG_COUNT; iSegReg++)
+    {
+        RTSEL const HostSel   = iemVmxVmcsGetHostSelReg(pVmcs, iSegReg);
+        bool const  fUnusable = RT_BOOL(HostSel == 0);
+        PCPUMSELREG pSelReg   = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
+
+        /* Selector. */
+        pSelReg->Sel      = HostSel;
+        pSelReg->ValidSel = HostSel;
+        pSelReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+
+        /* Limit. */
+        pSelReg->u32Limit = 0xffffffff;
+
+        /* Base. */
+        pSelReg->u64Base = 0;
+
+        /* Attributes. */
+        if (iSegReg == X86_SREG_CS)
+        {
+            pSelReg->Attr.n.u4Type        = X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_ACCESSED;
+            pSelReg->Attr.n.u1DescType    = 1;
+            pSelReg->Attr.n.u2Dpl         = 0;
+            pSelReg->Attr.n.u1Present     = 1;
+            pSelReg->Attr.n.u1Long        = fHostInLongMode;
+            pSelReg->Attr.n.u1DefBig      = !fHostInLongMode;
+            pSelReg->Attr.n.u1Granularity = 1;
+            Assert(!pSelReg->Attr.n.u1Unusable);
+            Assert(!fUnusable);
+        }
+        else
+        {
+            pSelReg->Attr.n.u4Type        = X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED;
+            pSelReg->Attr.n.u1DescType    = 1;
+            pSelReg->Attr.n.u2Dpl         = 0;
+            pSelReg->Attr.n.u1Present     = 1;
+            pSelReg->Attr.n.u1DefBig      = 1;
+            pSelReg->Attr.n.u1Granularity = 1;
+            pSelReg->Attr.n.u1Unusable    = fUnusable;
+        }
+    }
+
+    /* FS base. */
+    if (   !pVCpu->cpum.GstCtx.fs.Attr.n.u1Unusable
+        ||  fHostInLongMode)
+    {
+        Assert(X86_IS_CANONICAL(pVmcs->u64HostFsBase.u));
+        pVCpu->cpum.GstCtx.fs.u64Base = pVmcs->u64HostFsBase.u;
+    }
+
+    /* GS base. */
+    if (   !pVCpu->cpum.GstCtx.gs.Attr.n.u1Unusable
+        ||  fHostInLongMode)
+    {
+        Assert(X86_IS_CANONICAL(pVmcs->u64HostGsBase.u));
+        pVCpu->cpum.GstCtx.gs.u64Base = pVmcs->u64HostGsBase.u;
+    }
+
+    /* TR. */
+    Assert(X86_IS_CANONICAL(pVmcs->u64HostTrBase.u));
+    Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1Unusable);
+    pVCpu->cpum.GstCtx.tr.Sel                  = pVmcs->HostTr;
+    pVCpu->cpum.GstCtx.tr.ValidSel             = pVmcs->HostTr;
+    pVCpu->cpum.GstCtx.tr.fFlags               = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.tr.u32Limit             = X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN;
+    pVCpu->cpum.GstCtx.tr.u64Base              = pVmcs->u64HostTrBase.u;
+    pVCpu->cpum.GstCtx.tr.Attr.n.u4Type        = X86_SEL_TYPE_SYS_386_TSS_BUSY;
+    pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType    = 0;
+    pVCpu->cpum.GstCtx.tr.Attr.n.u2Dpl         = 0;
+    pVCpu->cpum.GstCtx.tr.Attr.n.u1Present     = 1;
+    pVCpu->cpum.GstCtx.tr.Attr.n.u1DefBig      = 0;
+    pVCpu->cpum.GstCtx.tr.Attr.n.u1Granularity = 0;
+
+    /* LDTR (Warning! do not touch the base and limits here). */
+    pVCpu->cpum.GstCtx.ldtr.Sel                = 0;
+    pVCpu->cpum.GstCtx.ldtr.ValidSel           = 0;
+    pVCpu->cpum.GstCtx.ldtr.fFlags             = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.ldtr.Attr.u             = X86DESCATTR_UNUSABLE;
+
+    /* GDTR. */
+    Assert(X86_IS_CANONICAL(pVmcs->u64HostGdtrBase.u));
+    pVCpu->cpum.GstCtx.gdtr.pGdt  = pVmcs->u64HostGdtrBase.u;
+    pVCpu->cpum.GstCtx.gdtr.cbGdt = 0xffff;
+
+    /* IDTR.*/
+    Assert(X86_IS_CANONICAL(pVmcs->u64HostIdtrBase.u));
+    pVCpu->cpum.GstCtx.idtr.pIdt  = pVmcs->u64HostIdtrBase.u;
+    pVCpu->cpum.GstCtx.idtr.cbIdt = 0xffff;
+}
+
+
+/**
+ * Checks host PDPTes as part of VM-exit.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason (for logging purposes).
+ */
+IEM_STATIC int iemVmxVmexitCheckHostPdptes(PVMCPUCC pVCpu, uint32_t uExitReason)
+{
+    /*
+     * Check host PDPTEs.
+     * See Intel spec. 27.5.4 "Checking and Loading Host Page-Directory-Pointer-Table Entries".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    const char *const pszFailure = "VMX-abort";
+    bool const fHostInLongMode   = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE);
+
+    if (   (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
+        && !fHostInLongMode)
+    {
+        uint64_t const uHostCr3 = pVCpu->cpum.GstCtx.cr3 & X86_CR3_PAE_PAGE_MASK;
+        X86PDPE aPdptes[X86_PG_PAE_PDPE_ENTRIES];
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), (void *)&aPdptes[0], uHostCr3, sizeof(aPdptes));
+        if (RT_SUCCESS(rc))
+        {
+            for (unsigned iPdpte = 0; iPdpte < RT_ELEMENTS(aPdptes); iPdpte++)
+            {
+                if (   !(aPdptes[iPdpte].u & X86_PDPE_P)
+                    || !(aPdptes[iPdpte].u & X86_PDPE_PAE_MBZ_MASK))
+                { /* likely */ }
+                else
+                {
+                    VMXVDIAG const enmDiag = iemVmxGetDiagVmexitPdpteRsvd(iPdpte);
+                    IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, enmDiag);
+                }
+            }
+        }
+        else
+            IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_HostPdpteCr3ReadPhys);
+    }
+
+    NOREF(pszFailure);
+    NOREF(uExitReason);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads the host MSRs from the VM-exit MSR-load area as part of VM-exit.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+IEM_STATIC int iemVmxVmexitLoadHostAutoMsrs(PVMCPUCC pVCpu, uint32_t uExitReason)
+{
+    /*
+     * Load host MSRs.
+     * See Intel spec. 27.6 "Loading MSRs".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    const char *const pszFailure = "VMX-abort";
+
+    /*
+     * The VM-exit MSR-load area address need not be a valid guest-physical address if the
+     * VM-exit MSR load count is 0. If this is the case, bail early without reading it.
+     * See Intel spec. 24.7.2 "VM-Exit Controls for MSRs".
+     */
+    uint32_t const cMsrs = pVmcs->u32ExitMsrLoadCount;
+    if (!cMsrs)
+        return VINF_SUCCESS;
+
+    /*
+     * Verify the MSR auto-load count. Physical CPUs can behave unpredictably if the count
+     * is exceeded including possibly raising #MC exceptions during VMX transition. Our
+     * implementation causes a VMX-abort followed by a triple-fault.
+     */
+    bool const fIsMsrCountValid = iemVmxIsAutoMsrCountValid(pVCpu, cMsrs);
+    if (fIsMsrCountValid)
+    { /* likely */ }
+    else
+        IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrLoadCount);
+
+    RTGCPHYS const GCPhysVmExitMsrLoadArea = pVmcs->u64AddrExitMsrLoad.u;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), (void *)pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pExitMsrLoadArea),
+                                     GCPhysVmExitMsrLoadArea, cMsrs * sizeof(VMXAUTOMSR));
+    if (RT_SUCCESS(rc))
+    {
+        PCVMXAUTOMSR pMsr = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pExitMsrLoadArea);
+        Assert(pMsr);
+        for (uint32_t idxMsr = 0; idxMsr < cMsrs; idxMsr++, pMsr++)
+        {
+            if (   !pMsr->u32Reserved
+                &&  pMsr->u32Msr != MSR_K8_FS_BASE
+                &&  pMsr->u32Msr != MSR_K8_GS_BASE
+                &&  pMsr->u32Msr != MSR_K6_EFER
+                &&  pMsr->u32Msr != MSR_IA32_SMM_MONITOR_CTL
+                &&  pMsr->u32Msr >> 8 != MSR_IA32_X2APIC_START >> 8)
+            {
+                VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, pMsr->u32Msr, pMsr->u64Value);
+                if (rcStrict == VINF_SUCCESS)
+                    continue;
+
+                /*
+                 * If we're in ring-0, we cannot handle returns to ring-3 at this point and continue VM-exit.
+                 * If any nested hypervisor loads MSRs that require ring-3 handling, we cause a VMX-abort
+                 * recording the MSR index in the auxiliary info. field and indicated further by our
+                 * own, specific diagnostic code. Later, we can try implement handling of the MSR in ring-0
+                 * if possible, or come up with a better, generic solution.
+                 */
+                pVCpu->cpum.GstCtx.hwvirt.vmx.uAbortAux = pMsr->u32Msr;
+                VMXVDIAG const enmDiag = rcStrict == VINF_CPUM_R3_MSR_WRITE
+                                       ? kVmxVDiag_Vmexit_MsrLoadRing3
+                                       : kVmxVDiag_Vmexit_MsrLoad;
+                IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, enmDiag);
+            }
+            else
+                IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrLoadRsvd);
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("VM-exit: Failed to read MSR auto-load area at %#RGp, rc=%Rrc\n", GCPhysVmExitMsrLoadArea, rc));
+        IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrLoadPtrReadPhys);
+    }
+
+    NOREF(uExitReason);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads the host state as part of VM-exit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason (for logging purposes).
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitLoadHostState(PVMCPUCC pVCpu, uint32_t uExitReason)
+{
+    /*
+     * Load host state.
+     * See Intel spec. 27.5 "Loading Host State".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    bool const fHostInLongMode = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE);
+
+    /* We cannot return from a long-mode guest to a host that is not in long mode. */
+    if (    CPUMIsGuestInLongMode(pVCpu)
+        && !fHostInLongMode)
+    {
+        Log(("VM-exit from long-mode guest to host not in long-mode -> VMX-Abort\n"));
+        return iemVmxAbort(pVCpu, VMXABORT_HOST_NOT_IN_LONG_MODE);
+    }
+
+    iemVmxVmexitLoadHostControlRegsMsrs(pVCpu);
+    iemVmxVmexitLoadHostSegRegs(pVCpu);
+
+    /*
+     * Load host RIP, RSP and RFLAGS.
+     * See Intel spec. 27.5.3 "Loading Host RIP, RSP and RFLAGS"
+     */
+    pVCpu->cpum.GstCtx.rip      = pVmcs->u64HostRip.u;
+    pVCpu->cpum.GstCtx.rsp      = pVmcs->u64HostRsp.u;
+    pVCpu->cpum.GstCtx.rflags.u = X86_EFL_1;
+
+    /* Clear address range monitoring. */
+    EMMonitorWaitClear(pVCpu);
+
+    /* Perform the VMX transition (PGM updates). */
+    VBOXSTRICTRC rcStrict = iemVmxWorldSwitch(pVCpu);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /* Check host PDPTEs (only when we've fully switched page tables_. */
+        /** @todo r=ramshankar: I don't know if PGM does this for us already or not... */
+        int rc = iemVmxVmexitCheckHostPdptes(pVCpu, uExitReason);
+        if (RT_FAILURE(rc))
+        {
+            Log(("VM-exit failed while restoring host PDPTEs -> VMX-Abort\n"));
+            return iemVmxAbort(pVCpu, VMXBOART_HOST_PDPTE);
+        }
+    }
+    else if (RT_SUCCESS(rcStrict))
+    {
+        Log3(("VM-exit: iemVmxWorldSwitch returns %Rrc (uExitReason=%u) -> Setting passup status\n", VBOXSTRICTRC_VAL(rcStrict),
+              uExitReason));
+        rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+    }
+    else
+    {
+        Log3(("VM-exit: iemVmxWorldSwitch failed! rc=%Rrc (uExitReason=%u)\n", VBOXSTRICTRC_VAL(rcStrict), uExitReason));
+        return VBOXSTRICTRC_VAL(rcStrict);
+    }
+
+    Assert(rcStrict == VINF_SUCCESS);
+
+    /* Load MSRs from the VM-exit auto-load MSR area. */
+    int rc = iemVmxVmexitLoadHostAutoMsrs(pVCpu, uExitReason);
+    if (RT_FAILURE(rc))
+    {
+        Log(("VM-exit failed while loading host MSRs -> VMX-Abort\n"));
+        return iemVmxAbort(pVCpu, VMXABORT_LOAD_HOST_MSR);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets VM-exit instruction information along with any displacement for an
+ * instruction VM-exit.
+ *
+ * @returns The VM-exit instruction information.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ * @param   uInstrId        The VM-exit instruction identity (VMXINSTRID_XXX).
+ * @param   pGCPtrDisp      Where to store the displacement field. Optional, can be
+ *                          NULL.
+ */
+IEM_STATIC uint32_t iemVmxGetExitInstrInfo(PVMCPUCC pVCpu, uint32_t uExitReason, VMXINSTRID uInstrId, PRTGCPTR pGCPtrDisp)
+{
+    RTGCPTR          GCPtrDisp;
+    VMXEXITINSTRINFO ExitInstrInfo;
+    ExitInstrInfo.u = 0;
+
+    /*
+     * Get and parse the ModR/M byte from our decoded opcodes.
+     */
+    uint8_t bRm;
+    uint8_t const offModRm = pVCpu->iem.s.offModRm;
+    IEM_MODRM_GET_U8(pVCpu, bRm, offModRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * ModR/M indicates register addressing.
+         *
+         * The primary/secondary register operands are reported in the iReg1 or iReg2
+         * fields depending on whether it is a read/write form.
+         */
+        uint8_t idxReg1;
+        uint8_t idxReg2;
+        if (!VMXINSTRID_IS_MODRM_PRIMARY_OP_W(uInstrId))
+        {
+            idxReg1 = ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg;
+            idxReg2 = (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB;
+        }
+        else
+        {
+            idxReg1 = (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB;
+            idxReg2 = ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg;
+        }
+        ExitInstrInfo.All.u2Scaling       = 0;
+        ExitInstrInfo.All.iReg1           = idxReg1;
+        ExitInstrInfo.All.u3AddrSize      = pVCpu->iem.s.enmEffAddrMode;
+        ExitInstrInfo.All.fIsRegOperand   = 1;
+        ExitInstrInfo.All.uOperandSize    = pVCpu->iem.s.enmEffOpSize;
+        ExitInstrInfo.All.iSegReg         = 0;
+        ExitInstrInfo.All.iIdxReg         = 0;
+        ExitInstrInfo.All.fIdxRegInvalid  = 1;
+        ExitInstrInfo.All.iBaseReg        = 0;
+        ExitInstrInfo.All.fBaseRegInvalid = 1;
+        ExitInstrInfo.All.iReg2           = idxReg2;
+
+        /* Displacement not applicable for register addressing. */
+        GCPtrDisp = 0;
+    }
+    else
+    {
+        /*
+         * ModR/M indicates memory addressing.
+         */
+        uint8_t  uScale        = 0;
+        bool     fBaseRegValid = false;
+        bool     fIdxRegValid  = false;
+        uint8_t  iBaseReg      = 0;
+        uint8_t  iIdxReg       = 0;
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
+        {
+            /*
+             * Parse the ModR/M, displacement for 16-bit addressing mode.
+             * See Intel instruction spec. Table 2-1. "16-Bit Addressing Forms with the ModR/M Byte".
+             */
+            uint16_t u16Disp = 0;
+            uint8_t const offDisp = offModRm + sizeof(bRm);
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
+            {
+                /* Displacement without any registers. */
+                IEM_DISP_GET_U16(pVCpu, u16Disp, offDisp);
+            }
+            else
+            {
+                /* Register (index and base). */
+                switch (bRm & X86_MODRM_RM_MASK)
+                {
+                    case 0: fBaseRegValid = true; iBaseReg = X86_GREG_xBX; fIdxRegValid = true; iIdxReg = X86_GREG_xSI; break;
+                    case 1: fBaseRegValid = true; iBaseReg = X86_GREG_xBX; fIdxRegValid = true; iIdxReg = X86_GREG_xDI; break;
+                    case 2: fBaseRegValid = true; iBaseReg = X86_GREG_xBP; fIdxRegValid = true; iIdxReg = X86_GREG_xSI; break;
+                    case 3: fBaseRegValid = true; iBaseReg = X86_GREG_xBP; fIdxRegValid = true; iIdxReg = X86_GREG_xDI; break;
+                    case 4: fIdxRegValid  = true; iIdxReg  = X86_GREG_xSI; break;
+                    case 5: fIdxRegValid  = true; iIdxReg  = X86_GREG_xDI; break;
+                    case 6: fBaseRegValid = true; iBaseReg = X86_GREG_xBP; break;
+                    case 7: fBaseRegValid = true; iBaseReg = X86_GREG_xBX; break;
+                }
+
+                /* Register + displacement. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0:                                                  break;
+                    case 1: IEM_DISP_GET_S8_SX_U16(pVCpu, u16Disp, offDisp); break;
+                    case 2: IEM_DISP_GET_U16(pVCpu, u16Disp, offDisp);       break;
+                    default:
+                    {
+                        /* Register addressing, handled at the beginning. */
+                        AssertMsgFailed(("ModR/M %#x implies register addressing, memory addressing expected!", bRm));
+                        break;
+                    }
+                }
+            }
+
+            Assert(!uScale);                    /* There's no scaling/SIB byte for 16-bit addressing. */
+            GCPtrDisp = (int16_t)u16Disp;       /* Sign-extend the displacement. */
+        }
+        else if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
+        {
+            /*
+             * Parse the ModR/M, SIB, displacement for 32-bit addressing mode.
+             * See Intel instruction spec. Table 2-2. "32-Bit Addressing Forms with the ModR/M Byte".
+             */
+            uint32_t u32Disp = 0;
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+            {
+                /* Displacement without any registers. */
+                uint8_t const offDisp = offModRm + sizeof(bRm);
+                IEM_DISP_GET_U32(pVCpu, u32Disp, offDisp);
+            }
+            else
+            {
+                /* Register (and perhaps scale, index and base). */
+                uint8_t offDisp = offModRm + sizeof(bRm);
+                iBaseReg = (bRm & X86_MODRM_RM_MASK);
+                if (iBaseReg == 4)
+                {
+                    /* An SIB byte follows the ModR/M byte, parse it. */
+                    uint8_t bSib;
+                    uint8_t const offSib = offModRm + sizeof(bRm);
+                    IEM_SIB_GET_U8(pVCpu, bSib, offSib);
+
+                    /* A displacement may follow SIB, update its offset. */
+                    offDisp += sizeof(bSib);
+
+                    /* Get the scale. */
+                    uScale = (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                    /* Get the index register. */
+                    iIdxReg = (bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK;
+                    fIdxRegValid = RT_BOOL(iIdxReg != 4);
+
+                    /* Get the base register. */
+                    iBaseReg = bSib & X86_SIB_BASE_MASK;
+                    fBaseRegValid = true;
+                    if (iBaseReg == 5)
+                    {
+                        if ((bRm & X86_MODRM_MOD_MASK) == 0)
+                        {
+                            /* Mod is 0 implies a 32-bit displacement with no base. */
+                            fBaseRegValid = false;
+                            IEM_DISP_GET_U32(pVCpu, u32Disp, offDisp);
+                        }
+                        else
+                        {
+                            /* Mod is not 0 implies an 8-bit/32-bit displacement (handled below) with an EBP base. */
+                            iBaseReg = X86_GREG_xBP;
+                        }
+                    }
+                }
+
+                /* Register + displacement. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0: /* Handled above */                              break;
+                    case 1: IEM_DISP_GET_S8_SX_U32(pVCpu, u32Disp, offDisp); break;
+                    case 2: IEM_DISP_GET_U32(pVCpu, u32Disp, offDisp);       break;
+                    default:
+                    {
+                        /* Register addressing, handled at the beginning. */
+                        AssertMsgFailed(("ModR/M %#x implies register addressing, memory addressing expected!", bRm));
+                        break;
+                    }
+                }
+            }
+
+            GCPtrDisp = (int32_t)u32Disp;       /* Sign-extend the displacement. */
+        }
+        else
+        {
+            Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT);
+
+            /*
+             * Parse the ModR/M, SIB, displacement for 64-bit addressing mode.
+             * See Intel instruction spec. 2.2 "IA-32e Mode".
+             */
+            uint64_t u64Disp = 0;
+            bool const fRipRelativeAddr = RT_BOOL((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5);
+            if (fRipRelativeAddr)
+            {
+                /*
+                 * RIP-relative addressing mode.
+                 *
+                 * The displacement is 32-bit signed implying an offset range of +/-2G.
+                 * See Intel instruction spec. 2.2.1.6 "RIP-Relative Addressing".
+                 */
+                uint8_t const offDisp = offModRm + sizeof(bRm);
+                IEM_DISP_GET_S32_SX_U64(pVCpu, u64Disp, offDisp);
+            }
+            else
+            {
+                uint8_t offDisp = offModRm + sizeof(bRm);
+
+                /*
+                 * Register (and perhaps scale, index and base).
+                 *
+                 * REX.B extends the most-significant bit of the base register. However, REX.B
+                 * is ignored while determining whether an SIB follows the opcode. Hence, we
+                 * shall OR any REX.B bit -after- inspecting for an SIB byte below.
+                 *
+                 * See Intel instruction spec. Table 2-5. "Special Cases of REX Encodings".
+                 */
+                iBaseReg = (bRm & X86_MODRM_RM_MASK);
+                if (iBaseReg == 4)
+                {
+                    /* An SIB byte follows the ModR/M byte, parse it. Displacement (if any) follows SIB. */
+                    uint8_t bSib;
+                    uint8_t const offSib = offModRm + sizeof(bRm);
+                    IEM_SIB_GET_U8(pVCpu, bSib, offSib);
+
+                    /* Displacement may follow SIB, update its offset. */
+                    offDisp += sizeof(bSib);
+
+                    /* Get the scale. */
+                    uScale = (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                    /* Get the index. */
+                    iIdxReg = ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex;
+                    fIdxRegValid = RT_BOOL(iIdxReg != 4);   /* R12 -can- be used as an index register. */
+
+                    /* Get the base. */
+                    iBaseReg = (bSib & X86_SIB_BASE_MASK);
+                    fBaseRegValid = true;
+                    if (iBaseReg == 5)
+                    {
+                        if ((bRm & X86_MODRM_MOD_MASK) == 0)
+                        {
+                            /* Mod is 0 implies a signed 32-bit displacement with no base. */
+                            IEM_DISP_GET_S32_SX_U64(pVCpu, u64Disp, offDisp);
+                        }
+                        else
+                        {
+                            /* Mod is non-zero implies an 8-bit/32-bit displacement (handled below) with RBP or R13 as base. */
+                            iBaseReg = pVCpu->iem.s.uRexB ? X86_GREG_x13 : X86_GREG_xBP;
+                        }
+                    }
+                }
+                iBaseReg |= pVCpu->iem.s.uRexB;
+
+                /* Register + displacement. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0: /* Handled above */                               break;
+                    case 1: IEM_DISP_GET_S8_SX_U64(pVCpu, u64Disp, offDisp);  break;
+                    case 2: IEM_DISP_GET_S32_SX_U64(pVCpu, u64Disp, offDisp); break;
+                    default:
+                    {
+                        /* Register addressing, handled at the beginning. */
+                        AssertMsgFailed(("ModR/M %#x implies register addressing, memory addressing expected!", bRm));
+                        break;
+                    }
+                }
+            }
+
+            GCPtrDisp = fRipRelativeAddr ? pVCpu->cpum.GstCtx.rip + u64Disp : u64Disp;
+        }
+
+        /*
+         * The primary or secondary register operand is reported in iReg2 depending
+         * on whether the primary operand is in read/write form.
+         */
+        uint8_t idxReg2;
+        if (!VMXINSTRID_IS_MODRM_PRIMARY_OP_W(uInstrId))
+        {
+            idxReg2 = bRm & X86_MODRM_RM_MASK;
+            if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
+                idxReg2 |= pVCpu->iem.s.uRexB;
+        }
+        else
+        {
+            idxReg2 = (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK;
+            if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
+                 idxReg2 |= pVCpu->iem.s.uRexReg;
+        }
+        ExitInstrInfo.All.u2Scaling      = uScale;
+        ExitInstrInfo.All.iReg1          = 0;   /* Not applicable for memory addressing. */
+        ExitInstrInfo.All.u3AddrSize     = pVCpu->iem.s.enmEffAddrMode;
+        ExitInstrInfo.All.fIsRegOperand  = 0;
+        ExitInstrInfo.All.uOperandSize   = pVCpu->iem.s.enmEffOpSize;
+        ExitInstrInfo.All.iSegReg        = pVCpu->iem.s.iEffSeg;
+        ExitInstrInfo.All.iIdxReg        = iIdxReg;
+        ExitInstrInfo.All.fIdxRegInvalid = !fIdxRegValid;
+        ExitInstrInfo.All.iBaseReg       = iBaseReg;
+        ExitInstrInfo.All.iIdxReg        = !fBaseRegValid;
+        ExitInstrInfo.All.iReg2          = idxReg2;
+    }
+
+    /*
+     * Handle exceptions to the norm for certain instructions.
+     * (e.g. some instructions convey an instruction identity in place of iReg2).
+     */
+    switch (uExitReason)
+    {
+        case VMX_EXIT_GDTR_IDTR_ACCESS:
+        {
+            Assert(VMXINSTRID_IS_VALID(uInstrId));
+            Assert(VMXINSTRID_GET_ID(uInstrId) == (uInstrId & 0x3));
+            ExitInstrInfo.GdtIdt.u2InstrId = VMXINSTRID_GET_ID(uInstrId);
+            ExitInstrInfo.GdtIdt.u2Undef0  = 0;
+            break;
+        }
+
+        case VMX_EXIT_LDTR_TR_ACCESS:
+        {
+            Assert(VMXINSTRID_IS_VALID(uInstrId));
+            Assert(VMXINSTRID_GET_ID(uInstrId) == (uInstrId & 0x3));
+            ExitInstrInfo.LdtTr.u2InstrId = VMXINSTRID_GET_ID(uInstrId);
+            ExitInstrInfo.LdtTr.u2Undef0 = 0;
+            break;
+        }
+
+        case VMX_EXIT_RDRAND:
+        case VMX_EXIT_RDSEED:
+        {
+            Assert(ExitInstrInfo.RdrandRdseed.u2OperandSize != 3);
+            break;
+        }
+    }
+
+    /* Update displacement and return the constructed VM-exit instruction information field. */
+    if (pGCPtrDisp)
+        *pGCPtrDisp = GCPtrDisp;
+
+    return ExitInstrInfo.u;
+}
+
+
+/**
+ * VMX VM-exit handler.
+ *
+ * @returns Strict VBox status code.
+ * @retval VINF_VMX_VMEXIT when the VM-exit is successful.
+ * @retval VINF_EM_TRIPLE_FAULT when VM-exit is unsuccessful and leads to a
+ *         triple-fault.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ * @param   u64ExitQual     The Exit qualification.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual)
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF3(pVCpu, uExitReason, u64ExitQual);
+    AssertMsgFailed(("VM-exit should only be invoked from ring-3 when nested-guest executes only in ring-3!\n"));
+    return VERR_IEM_IPE_7;
+# else
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    /*
+     * Import all the guest-CPU state.
+     *
+     * HM on returning to guest execution would have to reset up a whole lot of state
+     * anyway, (e.g., VM-entry/VM-exit controls) and we do not ever import a part of
+     * the state and flag reloading the entire state on re-entry. So import the entire
+     * state here, see HMNotifyVmxNstGstVmexit() for more comments.
+     */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL);
+
+    /*
+     * Ensure VM-entry interruption information valid bit is cleared.
+     *
+     * We do it here on every VM-exit so that even premature VM-exits (e.g. those caused
+     * by invalid-guest state or machine-check exceptions) also clear this bit.
+     *
+     * See Intel spec. 27.2 "Recording VM-exit Information And Updating VM-entry control fields".
+     */
+    if (VMX_ENTRY_INT_INFO_IS_VALID(pVmcs->u32EntryIntInfo))
+        pVmcs->u32EntryIntInfo &= ~VMX_ENTRY_INT_INFO_VALID;
+
+    /*
+     * Update the VM-exit reason and Exit qualification.
+     * Other VMCS read-only data fields are expected to be updated by the caller already.
+     */
+    pVmcs->u32RoExitReason = uExitReason;
+    pVmcs->u64RoExitQual.u = u64ExitQual;
+
+    Log3(("vmexit: reason=%#RX32 qual=%#RX64 cs:rip=%04x:%#RX64 cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", uExitReason,
+          pVmcs->u64RoExitQual.u, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.cr0,
+          pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4));
+
+    /*
+     * Update the IDT-vectoring information fields if the VM-exit is triggered during delivery of an event.
+     * See Intel spec. 27.2.4 "Information for VM Exits During Event Delivery".
+     */
+    {
+        uint8_t    uVector;
+        uint32_t   fFlags;
+        uint32_t   uErrCode;
+        bool const fInEventDelivery = IEMGetCurrentXcpt(pVCpu, &uVector, &fFlags, &uErrCode, NULL /* puCr2 */);
+        if (fInEventDelivery)
+        {
+            /*
+             * A VM-exit is not considered to occur during event delivery when the VM-exit is
+             * caused by a triple-fault or the original event results in a double-fault that
+             * causes the VM exit directly (exception bitmap). Therefore, we must not set the
+             * original event information into the IDT-vectoring information fields.
+             *
+             * See Intel spec. 27.2.4 "Information for VM Exits During Event Delivery".
+             */
+            if (   uExitReason != VMX_EXIT_TRIPLE_FAULT
+                && (   uExitReason != VMX_EXIT_XCPT_OR_NMI
+                    || !VMX_EXIT_INT_INFO_IS_XCPT_DF(pVmcs->u32RoExitIntInfo)))
+            {
+                uint8_t  const uIdtVectoringType = iemVmxGetEventType(uVector, fFlags);
+                uint8_t  const fErrCodeValid     = RT_BOOL(fFlags & IEM_XCPT_FLAGS_ERR);
+                uint32_t const uIdtVectoringInfo = RT_BF_MAKE(VMX_BF_IDT_VECTORING_INFO_VECTOR,         uVector)
+                                                 | RT_BF_MAKE(VMX_BF_IDT_VECTORING_INFO_TYPE,           uIdtVectoringType)
+                                                 | RT_BF_MAKE(VMX_BF_IDT_VECTORING_INFO_ERR_CODE_VALID, fErrCodeValid)
+                                                 | RT_BF_MAKE(VMX_BF_IDT_VECTORING_INFO_VALID,          1);
+                iemVmxVmcsSetIdtVectoringInfo(pVCpu, uIdtVectoringInfo);
+                iemVmxVmcsSetIdtVectoringErrCode(pVCpu, uErrCode);
+                LogFlow(("vmexit: idt_info=%#RX32 idt_err_code=%#RX32 cr2=%#RX64\n", uIdtVectoringInfo, uErrCode,
+                         pVCpu->cpum.GstCtx.cr2));
+            }
+        }
+    }
+
+    /* The following VMCS fields should always be zero since we don't support injecting SMIs into a guest. */
+    Assert(pVmcs->u64RoIoRcx.u == 0);
+    Assert(pVmcs->u64RoIoRsi.u == 0);
+    Assert(pVmcs->u64RoIoRdi.u == 0);
+    Assert(pVmcs->u64RoIoRip.u == 0);
+
+    /* We should not cause an NMI-window/interrupt-window VM-exit when injecting events as part of VM-entry. */
+    if (!CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx))
+    {
+        Assert(uExitReason != VMX_EXIT_NMI_WINDOW);
+        Assert(uExitReason != VMX_EXIT_INT_WINDOW);
+    }
+
+    /* For exception or NMI VM-exits the VM-exit interruption info. field must be valid. */
+    Assert(uExitReason != VMX_EXIT_XCPT_OR_NMI || VMX_EXIT_INT_INFO_IS_VALID(pVmcs->u32RoExitIntInfo));
+
+    /*
+     * Save the guest state back into the VMCS.
+     * We only need to save the state when the VM-entry was successful.
+     */
+    bool const fVmentryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
+    if (!fVmentryFailed)
+    {
+        /*
+         * If we support storing EFER.LMA into IA32e-mode guest field on VM-exit, we need to do that now.
+         * See Intel spec. 27.2 "Recording VM-exit Information And Updating VM-entry Control".
+         *
+         * It is not clear from the Intel spec. if this is done only when VM-entry succeeds.
+         * If a VM-exit happens before loading guest EFER, we risk restoring the host EFER.LMA
+         * as guest-CPU state would not been modified. Hence for now, we do this only when
+         * the VM-entry succeeded.
+         */
+        /** @todo r=ramshankar: Figure out if this bit gets set to host EFER.LMA on real
+         *        hardware when VM-exit fails during VM-entry (e.g. VERR_VMX_INVALID_GUEST_STATE). */
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxExitSaveEferLma)
+        {
+            if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)
+                pVmcs->u32EntryCtls |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
+            else
+                pVmcs->u32EntryCtls &= ~VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
+        }
+
+        /*
+         * The rest of the high bits of the VM-exit reason are only relevant when the VM-exit
+         * occurs in enclave mode/SMM which we don't support yet.
+         *
+         * If we ever add support for it, we can pass just the lower bits to the functions
+         * below, till then an assert should suffice.
+         */
+        Assert(!RT_HI_U16(uExitReason));
+
+        /* Save the guest state into the VMCS and restore guest MSRs from the auto-store guest MSR area. */
+        iemVmxVmexitSaveGuestState(pVCpu, uExitReason);
+        int rc = iemVmxVmexitSaveGuestAutoMsrs(pVCpu, uExitReason);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            return iemVmxAbort(pVCpu, VMXABORT_SAVE_GUEST_MSRS);
+
+        /* Clear any saved NMI-blocking state so we don't assert on next VM-entry (if it was in effect on the previous one). */
+        pVCpu->cpum.GstCtx.hwvirt.fLocalForcedActions &= ~VMCPU_FF_BLOCK_NMIS;
+    }
+    else
+    {
+        /* Restore the NMI-blocking state if VM-entry failed due to invalid guest state or while loading MSRs. */
+        uint32_t const uExitReasonBasic = VMX_EXIT_REASON_BASIC(uExitReason);
+        if (   uExitReasonBasic == VMX_EXIT_ERR_INVALID_GUEST_STATE
+            || uExitReasonBasic == VMX_EXIT_ERR_MSR_LOAD)
+            iemVmxVmexitRestoreNmiBlockingFF(pVCpu);
+    }
+
+    /*
+     * Stop any running VMX-preemption timer if necessary.
+     */
+    if (pVmcs->u32PinCtls & VMX_PIN_CTLS_PREEMPT_TIMER)
+        CPUMStopGuestVmxPremptTimer(pVCpu);
+
+    /*
+     * Clear any pending VMX nested-guest force-flags.
+     * These force-flags have no effect on (outer) guest execution and will
+     * be re-evaluated and setup on the next nested-guest VM-entry.
+     */
+    VMCPU_FF_CLEAR_MASK(pVCpu, VMCPU_FF_VMX_ALL_MASK);
+
+    /* Restore the host (outer guest) state. */
+    VBOXSTRICTRC rcStrict = iemVmxVmexitLoadHostState(pVCpu, uExitReason);
+    if (RT_SUCCESS(rcStrict))
+    {
+        Assert(rcStrict == VINF_SUCCESS);
+        rcStrict = VINF_VMX_VMEXIT;
+    }
+    else
+        Log3(("vmexit: Loading host-state failed. uExitReason=%u rc=%Rrc\n", uExitReason, VBOXSTRICTRC_VAL(rcStrict)));
+
+    /* We're no longer in nested-guest execution mode. */
+    pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode = false;
+
+    /* Notify HM that the current VMCS fields have been modified. */
+    HMNotifyVmxNstGstCurrentVmcsChanged(pVCpu);
+
+    /* Notify HM that we've completed the VM-exit. */
+    HMNotifyVmxNstGstVmexit(pVCpu);
+
+#  if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3)
+    /* Revert any IEM-only nested-guest execution policy, otherwise return rcStrict. */
+    Log(("vmexit: Disabling IEM-only EM execution policy!\n"));
+    int rcSched = EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, false);
+    if (rcSched != VINF_SUCCESS)
+        iemSetPassUpStatus(pVCpu, rcSched);
+#  endif
+    return rcStrict;
+# endif
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to instruction execution.
+ *
+ * This is intended for instructions where the caller provides all the relevant
+ * VM-exit information.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    /*
+     * For instructions where any of the following fields are not applicable:
+     *   - Exit qualification must be cleared.
+     *   - VM-exit instruction info. is undefined.
+     *   - Guest-linear address is undefined.
+     *   - Guest-physical address is undefined.
+     *
+     * The VM-exit instruction length is mandatory for all VM-exits that are caused by
+     * instruction execution. For VM-exits that are not due to instruction execution this
+     * field is undefined.
+     *
+     * In our implementation in IEM, all undefined fields are generally cleared. However,
+     * if the caller supplies information (from say the physical CPU directly) it is
+     * then possible that the undefined fields are not cleared.
+     *
+     * See Intel spec. 27.2.1 "Basic VM-Exit Information".
+     * See Intel spec. 27.2.4 "Information for VM Exits Due to Instruction Execution".
+     */
+    Assert(pExitInfo);
+    AssertMsg(pExitInfo->uReason <= VMX_EXIT_MAX, ("uReason=%u\n", pExitInfo->uReason));
+    AssertMsg(pExitInfo->cbInstr >= 1 && pExitInfo->cbInstr <= 15,
+              ("uReason=%u cbInstr=%u\n", pExitInfo->uReason, pExitInfo->cbInstr));
+
+    /* Update all the relevant fields from the VM-exit instruction information struct. */
+    iemVmxVmcsSetExitInstrInfo(pVCpu, pExitInfo->InstrInfo.u);
+    iemVmxVmcsSetExitGuestLinearAddr(pVCpu, pExitInfo->u64GuestLinearAddr);
+    iemVmxVmcsSetExitGuestPhysAddr(pVCpu, pExitInfo->u64GuestPhysAddr);
+    iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr);
+
+    /* Perform the VM-exit. */
+    return iemVmxVmexit(pVCpu, pExitInfo->uReason, pExitInfo->u64Qual);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to instruction execution.
+ *
+ * This is intended for instructions that only provide the VM-exit instruction
+ * length.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ * @param   cbInstr         The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstr(PVMCPUCC pVCpu, uint32_t uExitReason, uint8_t cbInstr)
+{
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason = uExitReason;
+    ExitInfo.cbInstr = cbInstr;
+
+#ifdef VBOX_STRICT
+    /*
+     * To prevent us from shooting ourselves in the foot.
+     * The follow instructions should convey more than just the instruction length.
+     */
+    switch (uExitReason)
+    {
+        case VMX_EXIT_INVEPT:
+        case VMX_EXIT_INVPCID:
+        case VMX_EXIT_INVVPID:
+        case VMX_EXIT_LDTR_TR_ACCESS:
+        case VMX_EXIT_GDTR_IDTR_ACCESS:
+        case VMX_EXIT_VMCLEAR:
+        case VMX_EXIT_VMPTRLD:
+        case VMX_EXIT_VMPTRST:
+        case VMX_EXIT_VMREAD:
+        case VMX_EXIT_VMWRITE:
+        case VMX_EXIT_VMXON:
+        case VMX_EXIT_XRSTORS:
+        case VMX_EXIT_XSAVES:
+        case VMX_EXIT_RDRAND:
+        case VMX_EXIT_RDSEED:
+        case VMX_EXIT_IO_INSTR:
+            AssertMsgFailedReturn(("Use iemVmxVmexitInstrNeedsInfo for uExitReason=%u\n", uExitReason), VERR_IEM_IPE_5);
+            break;
+    }
+#endif
+
+    return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to instruction execution.
+ *
+ * This is intended for instructions that have a ModR/M byte and update the VM-exit
+ * instruction information and Exit qualification fields.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ * @param   uInstrid        The instruction identity (VMXINSTRID_XXX).
+ * @param   cbInstr         The instruction length in bytes.
+ *
+ * @remarks Do not use this for INS/OUTS instruction.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrNeedsInfo(PVMCPUCC pVCpu, uint32_t uExitReason, VMXINSTRID uInstrId, uint8_t cbInstr)
+{
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason = uExitReason;
+    ExitInfo.cbInstr = cbInstr;
+
+    /*
+     * Update the Exit qualification field with displacement bytes.
+     * See Intel spec. 27.2.1 "Basic VM-Exit Information".
+     */
+    switch (uExitReason)
+    {
+        case VMX_EXIT_INVEPT:
+        case VMX_EXIT_INVPCID:
+        case VMX_EXIT_INVVPID:
+        case VMX_EXIT_LDTR_TR_ACCESS:
+        case VMX_EXIT_GDTR_IDTR_ACCESS:
+        case VMX_EXIT_VMCLEAR:
+        case VMX_EXIT_VMPTRLD:
+        case VMX_EXIT_VMPTRST:
+        case VMX_EXIT_VMREAD:
+        case VMX_EXIT_VMWRITE:
+        case VMX_EXIT_VMXON:
+        case VMX_EXIT_XRSTORS:
+        case VMX_EXIT_XSAVES:
+        case VMX_EXIT_RDRAND:
+        case VMX_EXIT_RDSEED:
+        {
+            /* Construct the VM-exit instruction information. */
+            RTGCPTR GCPtrDisp;
+            uint32_t const uInstrInfo = iemVmxGetExitInstrInfo(pVCpu, uExitReason, uInstrId, &GCPtrDisp);
+
+            /* Update the VM-exit instruction information. */
+            ExitInfo.InstrInfo.u = uInstrInfo;
+
+            /* Update the Exit qualification. */
+            ExitInfo.u64Qual = GCPtrDisp;
+            break;
+        }
+
+        default:
+            AssertMsgFailedReturn(("Use instruction-specific handler\n"), VERR_IEM_IPE_5);
+            break;
+    }
+
+    return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to INVLPG.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtrPage   The guest-linear address of the page being invalidated.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrInvlpg(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, uint8_t cbInstr)
+{
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason = VMX_EXIT_INVLPG;
+    ExitInfo.cbInstr = cbInstr;
+    ExitInfo.u64Qual = GCPtrPage;
+    Assert(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode || !RT_HI_U32(ExitInfo.u64Qual));
+
+    return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to LMSW.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uGuestCr0       The current guest CR0.
+ * @param   pu16NewMsw      The machine-status word specified in LMSW's source
+ *                          operand. This will be updated depending on the VMX
+ *                          guest/host CR0 mask if LMSW is not intercepted.
+ * @param   GCPtrEffDst     The guest-linear address of the source operand in case
+ *                          of a memory operand. For register operand, pass
+ *                          NIL_RTGCPTR.
+ * @param   cbInstr         The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrLmsw(PVMCPUCC pVCpu, uint32_t uGuestCr0, uint16_t *pu16NewMsw, RTGCPTR GCPtrEffDst,
+                                              uint8_t cbInstr)
+{
+    Assert(pu16NewMsw);
+
+    uint16_t const uNewMsw = *pu16NewMsw;
+    if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw))
+    {
+        Log2(("lmsw: Guest intercept -> VM-exit\n"));
+
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason = VMX_EXIT_MOV_CRX;
+        ExitInfo.cbInstr = cbInstr;
+
+        bool const fMemOperand = RT_BOOL(GCPtrEffDst != NIL_RTGCPTR);
+        if (fMemOperand)
+        {
+            Assert(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode || !RT_HI_U32(GCPtrEffDst));
+            ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
+        }
+
+        ExitInfo.u64Qual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER,  0) /* CR0 */
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS,    VMX_EXIT_QUAL_CRX_ACCESS_LMSW)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_LMSW_OP,   fMemOperand)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_LMSW_DATA, uNewMsw);
+
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    /*
+     * If LMSW did not cause a VM-exit, any CR0 bits in the range 0:3 that is set in the
+     * CR0 guest/host mask must be left unmodified.
+     *
+     * See Intel Spec. 25.3 "Changes To Instruction Behavior In VMX Non-root Operation".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    uint32_t const fGstHostMask     = pVmcs->u64Cr0Mask.u;
+    uint32_t const fGstHostLmswMask = fGstHostMask & (X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS);
+    *pu16NewMsw = (uGuestCr0 & fGstHostLmswMask) | (uNewMsw & ~fGstHostLmswMask);
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to CLTS.
+ *
+ * @returns Strict VBox status code.
+ * @retval VINF_VMX_MODIFIES_BEHAVIOR if the CLTS instruction did not cause a
+ *         VM-exit but must not modify the guest CR0.TS bit.
+ * @retval VINF_VMX_INTERCEPT_NOT_ACTIVE if the CLTS instruction did not cause a
+ *         VM-exit and modification to the guest CR0.TS bit is allowed (subject to
+ *         CR0 fixed bits in VMX operation).
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrClts(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    uint32_t const fGstHostMask = pVmcs->u64Cr0Mask.u;
+    uint32_t const fReadShadow  = pVmcs->u64Cr0ReadShadow.u;
+
+    /*
+     * If CR0.TS is owned by the host:
+     *   - If CR0.TS is set in the read-shadow, we must cause a VM-exit.
+     *   - If CR0.TS is cleared in the read-shadow, no VM-exit is caused and the
+     *     CLTS instruction completes without clearing CR0.TS.
+     *
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (fGstHostMask & X86_CR0_TS)
+    {
+        if (fReadShadow & X86_CR0_TS)
+        {
+            Log2(("clts: Guest intercept -> VM-exit\n"));
+
+            VMXVEXITINFO ExitInfo;
+            RT_ZERO(ExitInfo);
+            ExitInfo.uReason = VMX_EXIT_MOV_CRX;
+            ExitInfo.cbInstr = cbInstr;
+            ExitInfo.u64Qual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 0) /* CR0 */
+                             | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS,   VMX_EXIT_QUAL_CRX_ACCESS_CLTS);
+            return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+        }
+
+        return VINF_VMX_MODIFIES_BEHAVIOR;
+    }
+
+    /*
+     * If CR0.TS is not owned by the host, the CLTS instructions operates normally
+     * and may modify CR0.TS (subject to CR0 fixed bits in VMX operation).
+     */
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to 'Mov CR0,GReg' and 'Mov CR4,GReg'
+ * (CR0/CR4 write).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iCrReg      The control register (either CR0 or CR4).
+ * @param   uGuestCrX   The current guest CR0/CR4.
+ * @param   puNewCrX    Pointer to the new CR0/CR4 value. Will be updated if no
+ *                      VM-exit is caused.
+ * @param   iGReg       The general register from which the CR0/CR4 value is being
+ *                      loaded.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrMovToCr0Cr4(PVMCPUCC pVCpu, uint8_t iCrReg, uint64_t *puNewCrX, uint8_t iGReg,
+                                                     uint8_t cbInstr)
+{
+    Assert(puNewCrX);
+    Assert(iCrReg == 0 || iCrReg == 4);
+    Assert(iGReg < X86_GREG_COUNT);
+
+    uint64_t const uNewCrX = *puNewCrX;
+    if (CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX))
+    {
+        Log2(("mov_Cr_Rd: (CR%u) Guest intercept -> VM-exit\n", iCrReg));
+
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason = VMX_EXIT_MOV_CRX;
+        ExitInfo.cbInstr = cbInstr;
+        ExitInfo.u64Qual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, iCrReg)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS,   VMX_EXIT_QUAL_CRX_ACCESS_WRITE)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG,   iGReg);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    /*
+     * If the Mov-to-CR0/CR4 did not cause a VM-exit, any bits owned by the host
+     * must not be modified the instruction.
+     *
+     * See Intel Spec. 25.3 "Changes To Instruction Behavior In VMX Non-root Operation".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    uint64_t uGuestCrX;
+    uint64_t fGstHostMask;
+    if (iCrReg == 0)
+    {
+        IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+        uGuestCrX    = pVCpu->cpum.GstCtx.cr0;
+        fGstHostMask = pVmcs->u64Cr0Mask.u;
+    }
+    else
+    {
+        IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+        uGuestCrX    = pVCpu->cpum.GstCtx.cr4;
+        fGstHostMask = pVmcs->u64Cr4Mask.u;
+    }
+
+    *puNewCrX = (uGuestCrX & fGstHostMask) | (*puNewCrX & ~fGstHostMask);
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to 'Mov GReg,CR3' (CR3 read).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iGReg       The general register to which the CR3 value is being stored.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrMovFromCr3(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(iGReg < X86_GREG_COUNT);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
+
+    /*
+     * If the CR3-store exiting control is set, we must cause a VM-exit.
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_CR3_STORE_EXIT)
+    {
+        Log2(("mov_Rd_Cr: (CR3) Guest intercept -> VM-exit\n"));
+
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason = VMX_EXIT_MOV_CRX;
+        ExitInfo.cbInstr = cbInstr;
+        ExitInfo.u64Qual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 3) /* CR3 */
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS,   VMX_EXIT_QUAL_CRX_ACCESS_READ)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG,   iGReg);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to 'Mov CR3,GReg' (CR3 write).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uNewCr3     The new CR3 value.
+ * @param   iGReg       The general register from which the CR3 value is being
+ *                      loaded.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrMovToCr3(PVMCPUCC pVCpu, uint64_t uNewCr3, uint8_t iGReg, uint8_t cbInstr)
+{
+    Assert(iGReg < X86_GREG_COUNT);
+
+    /*
+     * If the CR3-load exiting control is set and the new CR3 value does not
+     * match any of the CR3-target values in the VMCS, we must cause a VM-exit.
+     *
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCr3))
+    {
+        Log2(("mov_Cr_Rd: (CR3) Guest intercept -> VM-exit\n"));
+
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason = VMX_EXIT_MOV_CRX;
+        ExitInfo.cbInstr = cbInstr;
+        ExitInfo.u64Qual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 3) /* CR3 */
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS,   VMX_EXIT_QUAL_CRX_ACCESS_WRITE)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG,   iGReg);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to 'Mov GReg,CR8' (CR8 read).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iGReg       The general register to which the CR8 value is being stored.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrMovFromCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(iGReg < X86_GREG_COUNT);
+
+    /*
+     * If the CR8-store exiting control is set, we must cause a VM-exit.
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_CR8_STORE_EXIT)
+    {
+        Log2(("mov_Rd_Cr: (CR8) Guest intercept -> VM-exit\n"));
+
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason = VMX_EXIT_MOV_CRX;
+        ExitInfo.cbInstr = cbInstr;
+        ExitInfo.u64Qual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 8) /* CR8 */
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS,   VMX_EXIT_QUAL_CRX_ACCESS_READ)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG,   iGReg);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to 'Mov CR8,GReg' (CR8 write).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iGReg       The general register from which the CR8 value is being
+ *                      loaded.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrMovToCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(iGReg < X86_GREG_COUNT);
+
+    /*
+     * If the CR8-load exiting control is set, we must cause a VM-exit.
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_CR8_LOAD_EXIT)
+    {
+        Log2(("mov_Cr_Rd: (CR8) Guest intercept -> VM-exit\n"));
+
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason = VMX_EXIT_MOV_CRX;
+        ExitInfo.cbInstr = cbInstr;
+        ExitInfo.u64Qual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 8) /* CR8 */
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS,   VMX_EXIT_QUAL_CRX_ACCESS_WRITE)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG,   iGReg);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to 'Mov DRx,GReg' (DRx write) and 'Mov
+ * GReg,DRx' (DRx read).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uInstrid    The instruction identity (VMXINSTRID_MOV_TO_DRX or
+ *                      VMXINSTRID_MOV_FROM_DRX).
+ * @param   iDrReg      The debug register being accessed.
+ * @param   iGReg       The general register to/from which the DRx value is being
+ *                      store/loaded.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrMovDrX(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint8_t iDrReg, uint8_t iGReg,
+                                                uint8_t cbInstr)
+{
+    Assert(iDrReg <= 7);
+    Assert(uInstrId == VMXINSTRID_MOV_TO_DRX || uInstrId == VMXINSTRID_MOV_FROM_DRX);
+    Assert(iGReg < X86_GREG_COUNT);
+
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT)
+    {
+        uint32_t const uDirection = uInstrId == VMXINSTRID_MOV_TO_DRX ? VMX_EXIT_QUAL_DRX_DIRECTION_WRITE
+                                                                      : VMX_EXIT_QUAL_DRX_DIRECTION_READ;
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason = VMX_EXIT_MOV_DRX;
+        ExitInfo.cbInstr = cbInstr;
+        ExitInfo.u64Qual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_DRX_REGISTER,  iDrReg)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_DRX_DIRECTION, uDirection)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_DRX_GENREG,    iGReg);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to I/O instructions (IN and OUT).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uInstrId    The VM-exit instruction identity (VMXINSTRID_IO_IN or
+ *                      VMXINSTRID_IO_OUT).
+ * @param   u16Port     The I/O port being accessed.
+ * @param   fImm        Whether the I/O port was encoded using an immediate operand
+ *                      or the implicit DX register.
+ * @param   cbAccess    The size of the I/O access in bytes (1, 2 or 4 bytes).
+ * @param   cbInstr     The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port, bool fImm, uint8_t cbAccess,
+                                            uint8_t cbInstr)
+{
+    Assert(uInstrId == VMXINSTRID_IO_IN || uInstrId == VMXINSTRID_IO_OUT);
+    Assert(cbAccess == 1 || cbAccess == 2 || cbAccess == 4);
+
+    bool const fIntercept = CPUMIsGuestVmxIoInterceptSet(pVCpu, u16Port, cbAccess);
+    if (fIntercept)
+    {
+        uint32_t const uDirection = uInstrId == VMXINSTRID_IO_IN ? VMX_EXIT_QUAL_IO_DIRECTION_IN
+                                                                 : VMX_EXIT_QUAL_IO_DIRECTION_OUT;
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason = VMX_EXIT_IO_INSTR;
+        ExitInfo.cbInstr = cbInstr;
+        ExitInfo.u64Qual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_WIDTH,     cbAccess - 1)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_DIRECTION, uDirection)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_ENCODING,  fImm)
+                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_PORT,      u16Port);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to string I/O instructions (INS and OUTS).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uInstrId        The VM-exit instruction identity (VMXINSTRID_IO_INS or
+ *                          VMXINSTRID_IO_OUTS).
+ * @param   u16Port         The I/O port being accessed.
+ * @param   cbAccess        The size of the I/O access in bytes (1, 2 or 4 bytes).
+ * @param   fRep            Whether the instruction has a REP prefix or not.
+ * @param   ExitInstrInfo   The VM-exit instruction info. field.
+ * @param   cbInstr         The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrStrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port, uint8_t cbAccess, bool fRep,
+                                               VMXEXITINSTRINFO ExitInstrInfo, uint8_t cbInstr)
+{
+    Assert(uInstrId == VMXINSTRID_IO_INS || uInstrId == VMXINSTRID_IO_OUTS);
+    Assert(cbAccess == 1 || cbAccess == 2 || cbAccess == 4);
+    Assert(ExitInstrInfo.StrIo.iSegReg < X86_SREG_COUNT);
+    Assert(ExitInstrInfo.StrIo.u3AddrSize == 0 || ExitInstrInfo.StrIo.u3AddrSize == 1 || ExitInstrInfo.StrIo.u3AddrSize == 2);
+    Assert(uInstrId != VMXINSTRID_IO_INS || ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES);
+
+    bool const fIntercept = CPUMIsGuestVmxIoInterceptSet(pVCpu, u16Port, cbAccess);
+    if (fIntercept)
+    {
+        /*
+         * Figure out the guest-linear address and the direction bit (INS/OUTS).
+         */
+        /** @todo r=ramshankar: Is there something in IEM that already does this? */
+        static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
+        uint8_t const  iSegReg       = ExitInstrInfo.StrIo.iSegReg;
+        uint8_t const  uAddrSize     = ExitInstrInfo.StrIo.u3AddrSize;
+        uint64_t const uAddrSizeMask = s_auAddrSizeMasks[uAddrSize];
+
+        uint32_t uDirection;
+        uint64_t uGuestLinearAddr;
+        if (uInstrId == VMXINSTRID_IO_INS)
+        {
+            uDirection       = VMX_EXIT_QUAL_IO_DIRECTION_IN;
+            uGuestLinearAddr = pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base + (pVCpu->cpum.GstCtx.rdi & uAddrSizeMask);
+        }
+        else
+        {
+            uDirection       = VMX_EXIT_QUAL_IO_DIRECTION_OUT;
+            uGuestLinearAddr = pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base + (pVCpu->cpum.GstCtx.rsi & uAddrSizeMask);
+        }
+
+        /*
+         * If the segment is unusable, the guest-linear address in undefined.
+         * We shall clear it for consistency.
+         *
+         * See Intel spec. 27.2.1 "Basic VM-Exit Information".
+         */
+        if (pVCpu->cpum.GstCtx.aSRegs[iSegReg].Attr.n.u1Unusable)
+            uGuestLinearAddr = 0;
+
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason            = VMX_EXIT_IO_INSTR;
+        ExitInfo.cbInstr            = cbInstr;
+        ExitInfo.u64GuestLinearAddr = uGuestLinearAddr;
+        ExitInfo.u64Qual            = RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_WIDTH,     cbAccess - 1)
+                                    | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_DIRECTION, uDirection)
+                                    | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_IS_STRING, 1)
+                                    | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_IS_REP,    fRep)
+                                    | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_ENCODING,  VMX_EXIT_QUAL_IO_ENCODING_DX)
+                                    | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_PORT,      u16Port);
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxInsOutInfo)
+            ExitInfo.InstrInfo = ExitInstrInfo;
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to MWAIT.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   fMonitorHwArmed     Whether the address-range monitor hardware is armed.
+ * @param   cbInstr             The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrMwait(PVMCPUCC pVCpu, bool fMonitorHwArmed, uint8_t cbInstr)
+{
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason = VMX_EXIT_MWAIT;
+    ExitInfo.cbInstr = cbInstr;
+    ExitInfo.u64Qual = fMonitorHwArmed;
+    return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to PAUSE.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitInstrPause(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    /*
+     * The PAUSE VM-exit is controlled by the "PAUSE exiting" control and the
+     * "PAUSE-loop exiting" control.
+     *
+     * The PLE-Gap is the maximum number of TSC ticks between two successive executions of
+     * the PAUSE instruction before we cause a VM-exit. The PLE-Window is the maximum amount
+     * of TSC ticks the guest is allowed to execute in a pause loop before we must cause
+     * a VM-exit.
+     *
+     * See Intel spec. 24.6.13 "Controls for PAUSE-Loop Exiting".
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    bool fIntercept = false;
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
+        fIntercept = true;
+    else if (   (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
+             && pVCpu->iem.s.uCpl == 0)
+    {
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+
+        /*
+         * A previous-PAUSE-tick value of 0 is used to identify the first time
+         * execution of a PAUSE instruction after VM-entry at CPL 0. We must
+         * consider this to be the first execution of PAUSE in a loop according
+         * to the Intel.
+         *
+         * All subsequent records for the previous-PAUSE-tick we ensure that it
+         * cannot be zero by OR'ing 1 to rule out the TSC wrap-around cases at 0.
+         */
+        uint64_t *puFirstPauseLoopTick = &pVCpu->cpum.GstCtx.hwvirt.vmx.uFirstPauseLoopTick;
+        uint64_t *puPrevPauseTick      = &pVCpu->cpum.GstCtx.hwvirt.vmx.uPrevPauseTick;
+        uint64_t const  uTick          = TMCpuTickGet(pVCpu);
+        uint32_t const  uPleGap        = pVmcs->u32PleGap;
+        uint32_t const  uPleWindow     = pVmcs->u32PleWindow;
+        if (   *puPrevPauseTick == 0
+            || uTick - *puPrevPauseTick > uPleGap)
+            *puFirstPauseLoopTick = uTick;
+        else if (uTick - *puFirstPauseLoopTick > uPleWindow)
+            fIntercept = true;
+
+        *puPrevPauseTick = uTick | 1;
+    }
+
+    if (fIntercept)
+        return iemVmxVmexitInstr(pVCpu, VMX_EXIT_PAUSE, cbInstr);
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to task switches.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmTaskSwitch   The cause of the task switch.
+ * @param   SelNewTss       The selector of the new TSS.
+ * @param   cbInstr         The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, RTSEL SelNewTss, uint8_t cbInstr)
+{
+    /*
+     * Task-switch VM-exits are unconditional and provide the Exit qualification.
+     *
+     * If the cause of the task switch is due to execution of CALL, IRET or the JMP
+     * instruction or delivery of the exception generated by one of these instructions
+     * lead to a task switch through a task gate in the IDT, we need to provide the
+     * VM-exit instruction length. Any other means of invoking a task switch VM-exit
+     * leaves the VM-exit instruction length field undefined.
+     *
+     * See Intel spec. 25.2 "Other Causes Of VM Exits".
+     * See Intel spec. 27.2.4 "Information for VM Exits Due to Instruction Execution".
+     */
+    Assert(cbInstr <= 15);
+
+    uint8_t uType;
+    switch (enmTaskSwitch)
+    {
+        case IEMTASKSWITCH_CALL:        uType = VMX_EXIT_QUAL_TASK_SWITCH_TYPE_CALL; break;
+        case IEMTASKSWITCH_IRET:        uType = VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IRET; break;
+        case IEMTASKSWITCH_JUMP:        uType = VMX_EXIT_QUAL_TASK_SWITCH_TYPE_JMP;  break;
+        case IEMTASKSWITCH_INT_XCPT:    uType = VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT;  break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    uint64_t const u64ExitQual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_TASK_SWITCH_NEW_TSS, SelNewTss)
+                               | RT_BF_MAKE(VMX_BF_EXIT_QUAL_TASK_SWITCH_SOURCE,  uType);
+    iemVmxVmcsSetExitInstrLen(pVCpu, cbInstr);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_TASK_SWITCH, u64ExitQual);
+}
+
+
+/**
+ * VMX VM-exit handler for trap-like VM-exits.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitTrapLikeWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    Assert(VMXIsVmexitTrapLike(pExitInfo->uReason));
+    iemVmxVmcsSetGuestPendingDbgXcpts(pVCpu, pExitInfo->u64GuestPendingDbgXcpts);
+    return iemVmxVmexit(pVCpu, pExitInfo->uReason, pExitInfo->u64Qual);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to task switches.
+ *
+ * This is intended for task switches where the caller provides all the relevant
+ * VM-exit information.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitTaskSwitchWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo,
+                                                       PCVMXVEXITEVENTINFO pExitEventInfo)
+{
+    Assert(pExitInfo->uReason == VMX_EXIT_TASK_SWITCH);
+    iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr);
+    iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo);
+    iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_TASK_SWITCH, pExitInfo->u64Qual);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to expiring of the preemption timer.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitPreemptTimer(PVMCPUCC pVCpu)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
+    Assert(pVmcs->u32PinCtls & VMX_PIN_CTLS_PREEMPT_TIMER);
+
+    /* Import the hardware virtualization state (for nested-guest VM-entry TSC-tick). */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+
+    /* Save the VMX-preemption timer value (of 0) back in to the VMCS if the CPU supports this feature. */
+    if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER)
+        pVmcs->u32PreemptTimer = 0;
+
+    /* Cause the VMX-preemption timer VM-exit. The Exit qualification MBZ. */
+    return iemVmxVmexit(pVCpu, VMX_EXIT_PREEMPT_TIMER, 0 /* u64ExitQual */);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to external interrupts.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uVector         The external interrupt vector (pass 0 if the interrupt
+ *                          is still pending since we typically won't know the
+ *                          vector).
+ * @param   fIntPending     Whether the external interrupt is pending or
+ *                          acknowledged in the interrupt controller.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitExtInt(PVMCPUCC pVCpu, uint8_t uVector, bool fIntPending)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(!fIntPending || uVector == 0);
+
+    /* The VM-exit is subject to "External interrupt exiting" being set. */
+    if (pVmcs->u32PinCtls & VMX_PIN_CTLS_EXT_INT_EXIT)
+    {
+        if (fIntPending)
+        {
+            /*
+             * If the interrupt is pending and we don't need to acknowledge the
+             * interrupt on VM-exit, cause the VM-exit immediately.
+             *
+             * See Intel spec 25.2 "Other Causes Of VM Exits".
+             */
+            if (!(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT))
+                return iemVmxVmexit(pVCpu, VMX_EXIT_EXT_INT, 0 /* u64ExitQual */);
+
+            /*
+             * If the interrupt is pending and we -do- need to acknowledge the interrupt
+             * on VM-exit, postpone VM-exit till after the interrupt controller has been
+             * acknowledged that the interrupt has been consumed. Callers would have to call
+             * us again after getting the vector (and ofc, with fIntPending with false).
+             */
+            return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+        }
+
+        /*
+         * If the interrupt is no longer pending (i.e. it has been acknowledged) and the
+         * "External interrupt exiting" and "Acknowledge interrupt on VM-exit" controls are
+         * all set, we need to record the vector of the external interrupt in the
+         * VM-exit interruption information field. Otherwise, mark this field as invalid.
+         *
+         * See Intel spec. 27.2.2 "Information for VM Exits Due to Vectored Events".
+         */
+        uint32_t uExitIntInfo;
+        if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
+        {
+            bool const fNmiUnblocking = pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret;
+            uExitIntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR,           uVector)
+                         | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE,             VMX_EXIT_INT_INFO_TYPE_EXT_INT)
+                         | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_NMI_UNBLOCK_IRET, fNmiUnblocking)
+                         | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID,            1);
+        }
+        else
+            uExitIntInfo = 0;
+        iemVmxVmcsSetExitIntInfo(pVCpu, uExitIntInfo);
+
+        /*
+         * Cause the VM-exit whether or not the vector has been stored
+         * in the VM-exit interruption-information field.
+         */
+        return iemVmxVmexit(pVCpu, VMX_EXIT_EXT_INT, 0 /* u64ExitQual */);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to a double fault caused during delivery of
+ * an event.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitEventDoubleFault(PVMCPUCC pVCpu)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    uint32_t const fXcptBitmap = pVmcs->u32XcptBitmap;
+    if (fXcptBitmap & RT_BIT(X86_XCPT_DF))
+    {
+        /*
+         * The NMI-unblocking due to IRET field need not be set for double faults.
+         * See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
+         */
+        uint32_t const uExitIntInfo   = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR,           X86_XCPT_DF)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE,             VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_ERR_CODE_VALID,   1)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_NMI_UNBLOCK_IRET, 0)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID,            1);
+        iemVmxVmcsSetExitIntInfo(pVCpu, uExitIntInfo);
+        return iemVmxVmexit(pVCpu, VMX_EXIT_XCPT_OR_NMI, 0 /* u64ExitQual */);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exit due to delivery of an events.
+ *
+ * This is intended for VM-exit due to exceptions or NMIs where the caller provides
+ * all the relevant VM-exit information.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitEventWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
+{
+    Assert(pExitInfo);
+    Assert(pExitEventInfo);
+    Assert(pExitInfo->uReason == VMX_EXIT_XCPT_OR_NMI);
+    Assert(VMX_EXIT_INT_INFO_IS_VALID(pExitEventInfo->uExitIntInfo));
+
+    iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr);
+    iemVmxVmcsSetExitIntInfo(pVCpu, pExitEventInfo->uExitIntInfo);
+    iemVmxVmcsSetExitIntErrCode(pVCpu, pExitEventInfo->uExitIntErrCode);
+    iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo);
+    iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_XCPT_OR_NMI, pExitInfo->u64Qual);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to delivery of an event.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uVector     The interrupt / exception vector.
+ * @param   fFlags      The flags (see IEM_XCPT_FLAGS_XXX).
+ * @param   uErrCode    The error code associated with the event.
+ * @param   uCr2        The CR2 value in case of a \#PF exception.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitEvent(PVMCPUCC pVCpu, uint8_t uVector, uint32_t fFlags, uint32_t uErrCode, uint64_t uCr2,
+                                          uint8_t cbInstr)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    /*
+     * If the event is being injected as part of VM-entry, it is -not- subject to event
+     * intercepts in the nested-guest. However, secondary exceptions that occur during
+     * injection of any event -are- subject to event interception.
+     *
+     * See Intel spec. 26.5.1.2 "VM Exits During Event Injection".
+     */
+    if (!CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx))
+    {
+        /*
+         * If the event is a virtual-NMI (which is an NMI being inject during VM-entry)
+         * virtual-NMI blocking must be set in effect rather than physical NMI blocking.
+         *
+         * See Intel spec. 24.6.1 "Pin-Based VM-Execution Controls".
+         */
+        if (   uVector == X86_XCPT_NMI
+            && (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+            && (pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
+            pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = true;
+        else
+            Assert(!pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking);
+
+        CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, true);
+        return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+    }
+
+    /*
+     * We are injecting an external interrupt, check if we need to cause a VM-exit now.
+     * If not, the caller will continue delivery of the external interrupt as it would
+     * normally. The interrupt is no longer pending in the interrupt controller at this
+     * point.
+     */
+    if (fFlags & IEM_XCPT_FLAGS_T_EXT_INT)
+    {
+        Assert(!VMX_IDT_VECTORING_INFO_IS_VALID(pVmcs->u32RoIdtVectoringInfo));
+        return iemVmxVmexitExtInt(pVCpu, uVector, false /* fIntPending */);
+    }
+
+    /*
+     * Evaluate intercepts for hardware exceptions, software exceptions (#BP, #OF),
+     * and privileged software exceptions (#DB generated by INT1/ICEBP) and software
+     * interrupts.
+     */
+    Assert(fFlags & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_T_SOFT_INT));
+    bool fIntercept;
+    if (   !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
+        ||  (fFlags & (IEM_XCPT_FLAGS_BP_INSTR | IEM_XCPT_FLAGS_OF_INSTR | IEM_XCPT_FLAGS_ICEBP_INSTR)))
+    {
+        fIntercept = CPUMIsGuestVmxXcptInterceptSet(&pVCpu->cpum.GstCtx, uVector, uErrCode);
+    }
+    else
+    {
+        /* Software interrupts cannot be intercepted and therefore do not cause a VM-exit. */
+        fIntercept = false;
+    }
+
+    /*
+     * Now that we've determined whether the event causes a VM-exit, we need to construct the
+     * relevant VM-exit information and cause the VM-exit.
+     */
+    if (fIntercept)
+    {
+        Assert(!(fFlags & IEM_XCPT_FLAGS_T_EXT_INT));
+
+        /* Construct the rest of the event related information fields and cause the VM-exit. */
+        uint64_t u64ExitQual;
+        if (uVector == X86_XCPT_PF)
+        {
+            Assert(fFlags & IEM_XCPT_FLAGS_CR2);
+            u64ExitQual = uCr2;
+        }
+        else if (uVector == X86_XCPT_DB)
+        {
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
+            u64ExitQual = pVCpu->cpum.GstCtx.dr[6] & VMX_VMCS_EXIT_QUAL_VALID_MASK;
+        }
+        else
+            u64ExitQual = 0;
+
+        uint8_t  const fNmiUnblocking = pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret;
+        bool     const fErrCodeValid  = RT_BOOL(fFlags & IEM_XCPT_FLAGS_ERR);
+        uint8_t  const uIntInfoType   = iemVmxGetEventType(uVector, fFlags);
+        uint32_t const uExitIntInfo   = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR,           uVector)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE,             uIntInfoType)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_ERR_CODE_VALID,   fErrCodeValid)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_NMI_UNBLOCK_IRET, fNmiUnblocking)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID,            1);
+        iemVmxVmcsSetExitIntInfo(pVCpu, uExitIntInfo);
+        iemVmxVmcsSetExitIntErrCode(pVCpu, uErrCode);
+
+        /*
+         * For VM-exits due to software exceptions (those generated by INT3 or INTO) or privileged
+         * software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
+         * length.
+         */
+        if (   (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
+            || (fFlags & (IEM_XCPT_FLAGS_BP_INSTR | IEM_XCPT_FLAGS_OF_INSTR | IEM_XCPT_FLAGS_ICEBP_INSTR)))
+            iemVmxVmcsSetExitInstrLen(pVCpu, cbInstr);
+        else
+            iemVmxVmcsSetExitInstrLen(pVCpu, 0);
+
+        return iemVmxVmexit(pVCpu, VMX_EXIT_XCPT_OR_NMI, u64ExitQual);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for APIC accesses.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offAccess   The offset of the register being accessed.
+ * @param   fAccess     The type of access (must contain IEM_ACCESS_TYPE_READ or
+ *                      IEM_ACCESS_TYPE_WRITE or IEM_ACCESS_INSTRUCTION).
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitApicAccess(PVMCPUCC pVCpu, uint16_t offAccess, uint32_t fAccess)
+{
+    Assert((fAccess & IEM_ACCESS_TYPE_READ) || (fAccess & IEM_ACCESS_TYPE_WRITE) || (fAccess & IEM_ACCESS_INSTRUCTION));
+
+    VMXAPICACCESS enmAccess;
+    bool const fInEventDelivery = IEMGetCurrentXcpt(pVCpu, NULL, NULL, NULL, NULL);
+    if (fInEventDelivery)
+        enmAccess = VMXAPICACCESS_LINEAR_EVENT_DELIVERY;
+    else if (fAccess & IEM_ACCESS_INSTRUCTION)
+        enmAccess = VMXAPICACCESS_LINEAR_INSTR_FETCH;
+    else if (fAccess & IEM_ACCESS_TYPE_WRITE)
+        enmAccess = VMXAPICACCESS_LINEAR_WRITE;
+    else
+        enmAccess = VMXAPICACCESS_LINEAR_READ;
+
+    uint64_t const u64ExitQual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_APIC_ACCESS_OFFSET, offAccess)
+                               | RT_BF_MAKE(VMX_BF_EXIT_QUAL_APIC_ACCESS_TYPE,   enmAccess);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_APIC_ACCESS, u64ExitQual);
+}
+
+
+/**
+ * VMX VM-exit handler for APIC accesses.
+ *
+ * This is intended for APIC accesses where the caller provides all the
+ * relevant VM-exit information.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitApicAccessWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo,
+                                                       PCVMXVEXITEVENTINFO pExitEventInfo)
+{
+    /* VM-exit interruption information should not be valid for APIC-access VM-exits. */
+    Assert(!VMX_EXIT_INT_INFO_IS_VALID(pExitEventInfo->uExitIntInfo));
+    Assert(pExitInfo->uReason == VMX_EXIT_APIC_ACCESS);
+    iemVmxVmcsSetExitIntInfo(pVCpu, 0);
+    iemVmxVmcsSetExitIntErrCode(pVCpu, 0);
+    iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr);
+    iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo);
+    iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_APIC_ACCESS, pExitInfo->u64Qual);
+}
+
+
+/**
+ * VMX VM-exit handler for APIC-write VM-exits.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offApic     The write to the virtual-APIC page offset that caused this
+ *                      VM-exit.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmexitApicWrite(PVMCPUCC pVCpu, uint16_t offApic)
+{
+    Assert(offApic < XAPIC_OFF_END + 4);
+    /* Write only bits 11:0 of the APIC offset into the Exit qualification field. */
+    offApic &= UINT16_C(0xfff);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_APIC_WRITE, offApic);
+}
+
+
+/**
+ * Sets virtual-APIC write emulation as pending.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offApic     The offset in the virtual-APIC page that was written.
+ */
+DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic)
+{
+    Assert(offApic < XAPIC_OFF_END + 4);
+
+    /*
+     * Record the currently updated APIC offset, as we need this later for figuring
+     * out whether to perform TPR, EOI or self-IPI virtualization as well as well
+     * as for supplying the exit qualification when causing an APIC-write VM-exit.
+     */
+    pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
+
+    /*
+     * Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
+     * virtualization or APIC-write emulation).
+     */
+    if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
+}
+
+
+/**
+ * Clears any pending virtual-APIC write emulation.
+ *
+ * @returns The virtual-APIC offset that was written before clearing it.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(uint16_t) iemVmxVirtApicClearPendingWrite(PVMCPUCC pVCpu)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+    uint8_t const offVirtApicWrite = pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite;
+    pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = 0;
+    Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE));
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
+    return offVirtApicWrite;
+}
+
+
+/**
+ * Reads a 32-bit register from the virtual-APIC page at the given offset.
+ *
+ * @returns The register from the virtual-APIC page.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   offReg  The offset of the register being read.
+ */
+IEM_STATIC uint32_t iemVmxVirtApicReadRaw32(PVMCPUCC pVCpu, uint16_t offReg)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(offReg <= VMX_V_VIRT_APIC_SIZE - sizeof(uint32_t));
+
+    uint32_t uReg;
+    RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &uReg, GCPhysVirtApic + offReg, sizeof(uReg));
+    if (RT_SUCCESS(rc))
+    { /* likely */ }
+    else
+    {
+        AssertMsgFailed(("Failed to read %u bytes at offset %#x of the virtual-APIC page at %#RGp\n", sizeof(uReg), offReg,
+                         GCPhysVirtApic));
+        uReg = 0;
+    }
+    return uReg;
+}
+
+
+/**
+ * Reads a 64-bit register from the virtual-APIC page at the given offset.
+ *
+ * @returns The register from the virtual-APIC page.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   offReg  The offset of the register being read.
+ */
+IEM_STATIC uint64_t iemVmxVirtApicReadRaw64(PVMCPUCC pVCpu, uint16_t offReg)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(offReg <= VMX_V_VIRT_APIC_SIZE - sizeof(uint64_t));
+
+    uint64_t uReg;
+    RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &uReg, GCPhysVirtApic + offReg, sizeof(uReg));
+    if (RT_SUCCESS(rc))
+    { /* likely */ }
+    else
+    {
+        AssertMsgFailed(("Failed to read %u bytes at offset %#x of the virtual-APIC page at %#RGp\n", sizeof(uReg), offReg,
+                         GCPhysVirtApic));
+        uReg = 0;
+    }
+    return uReg;
+}
+
+
+/**
+ * Writes a 32-bit register to the virtual-APIC page at the given offset.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   offReg  The offset of the register being written.
+ * @param   uReg    The register value to write.
+ */
+IEM_STATIC void iemVmxVirtApicWriteRaw32(PVMCPUCC pVCpu, uint16_t offReg, uint32_t uReg)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(offReg <= VMX_V_VIRT_APIC_SIZE - sizeof(uint32_t));
+
+    RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+    int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic + offReg, &uReg, sizeof(uReg));
+    if (RT_SUCCESS(rc))
+    { /* likely */ }
+    else
+    {
+        AssertMsgFailed(("Failed to write %u bytes at offset %#x of the virtual-APIC page at %#RGp\n", sizeof(uReg), offReg,
+                         GCPhysVirtApic));
+    }
+}
+
+
+/**
+ * Writes a 64-bit register to the virtual-APIC page at the given offset.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   offReg  The offset of the register being written.
+ * @param   uReg    The register value to write.
+ */
+IEM_STATIC void iemVmxVirtApicWriteRaw64(PVMCPUCC pVCpu, uint16_t offReg, uint64_t uReg)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(offReg <= VMX_V_VIRT_APIC_SIZE - sizeof(uint64_t));
+
+    RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+    int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic + offReg, &uReg, sizeof(uReg));
+    if (RT_SUCCESS(rc))
+    { /* likely */ }
+    else
+    {
+        AssertMsgFailed(("Failed to write %u bytes at offset %#x of the virtual-APIC page at %#RGp\n", sizeof(uReg), offReg,
+                         GCPhysVirtApic));
+    }
+}
+
+
+/**
+ * Sets the vector in a virtual-APIC 256-bit sparse register.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offReg      The offset of the 256-bit spare register.
+ * @param   uVector     The vector to set.
+ *
+ * @remarks This is based on our APIC device code.
+ */
+IEM_STATIC void iemVmxVirtApicSetVectorInReg(PVMCPUCC pVCpu, uint16_t offReg, uint8_t uVector)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    /* Determine the vector offset within the chunk. */
+    uint16_t const offVector = (uVector & UINT32_C(0xe0)) >> 1;
+
+    /* Read the chunk at the offset. */
+    uint32_t uReg;
+    RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &uReg, GCPhysVirtApic + offReg + offVector, sizeof(uReg));
+    if (RT_SUCCESS(rc))
+    {
+        /* Modify the chunk. */
+        uint16_t const idxVectorBit = uVector & UINT32_C(0x1f);
+        uReg |= RT_BIT(idxVectorBit);
+
+        /* Write the chunk. */
+        rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic + offReg + offVector, &uReg, sizeof(uReg));
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            AssertMsgFailed(("Failed to set vector %#x in 256-bit register at %#x of the virtual-APIC page at %#RGp\n",
+                             uVector, offReg, GCPhysVirtApic));
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Failed to get vector %#x in 256-bit register at %#x of the virtual-APIC page at %#RGp\n",
+                         uVector, offReg, GCPhysVirtApic));
+    }
+}
+
+
+/**
+ * Clears the vector in a virtual-APIC 256-bit sparse register.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offReg      The offset of the 256-bit spare register.
+ * @param   uVector     The vector to clear.
+ *
+ * @remarks This is based on our APIC device code.
+ */
+IEM_STATIC void iemVmxVirtApicClearVectorInReg(PVMCPUCC pVCpu, uint16_t offReg, uint8_t uVector)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    /* Determine the vector offset within the chunk. */
+    uint16_t const offVector      = (uVector & UINT32_C(0xe0)) >> 1;
+
+    /* Read the chunk at the offset. */
+    uint32_t uReg;
+    RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &uReg, GCPhysVirtApic + offReg + offVector, sizeof(uReg));
+    if (RT_SUCCESS(rc))
+    {
+        /* Modify the chunk. */
+        uint16_t const idxVectorBit = uVector & UINT32_C(0x1f);
+        uReg &= ~RT_BIT(idxVectorBit);
+
+        /* Write the chunk. */
+        rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic + offReg + offVector, &uReg, sizeof(uReg));
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            AssertMsgFailed(("Failed to clear vector %#x in 256-bit register at %#x of the virtual-APIC page at %#RGp\n",
+                             uVector, offReg, GCPhysVirtApic));
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Failed to get vector %#x in 256-bit register at %#x of the virtual-APIC page at %#RGp\n",
+                         uVector, offReg, GCPhysVirtApic));
+    }
+}
+
+
+/**
+ * Checks if a memory access to the APIC-access page must causes an APIC-access
+ * VM-exit.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offAccess   The offset of the register being accessed.
+ * @param   cbAccess    The size of the access in bytes.
+ * @param   fAccess     The type of access (must be IEM_ACCESS_TYPE_READ or
+ *                      IEM_ACCESS_TYPE_WRITE).
+ *
+ * @remarks This must not be used for MSR-based APIC-access page accesses!
+ * @sa      iemVmxVirtApicAccessMsrWrite, iemVmxVirtApicAccessMsrRead.
+ */
+IEM_STATIC bool iemVmxVirtApicIsMemAccessIntercepted(PVMCPUCC pVCpu, uint16_t offAccess, size_t cbAccess, uint32_t fAccess)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(fAccess == IEM_ACCESS_TYPE_READ || fAccess == IEM_ACCESS_TYPE_WRITE);
+
+    /*
+     * We must cause a VM-exit if any of the following are true:
+     *   - TPR shadowing isn't active.
+     *   - The access size exceeds 32-bits.
+     *   - The access is not contained within low 4 bytes of a 16-byte aligned offset.
+     *
+     * See Intel spec. 29.4.2 "Virtualizing Reads from the APIC-Access Page".
+     * See Intel spec. 29.4.3.1 "Determining Whether a Write Access is Virtualized".
+     */
+    if (   !(pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+        || cbAccess > sizeof(uint32_t)
+        || ((offAccess + cbAccess - 1) & 0xc)
+        || offAccess >= XAPIC_OFF_END + 4)
+        return true;
+
+    /*
+     * If the access is part of an operation where we have already
+     * virtualized a virtual-APIC write, we must cause a VM-exit.
+     */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
+        return true;
+
+    /*
+     * Check write accesses to the APIC-access page that cause VM-exits.
+     */
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+    {
+        if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT)
+        {
+            /*
+             * With APIC-register virtualization, a write access to any of the
+             * following registers are virtualized. Accessing any other register
+             * causes a VM-exit.
+             */
+            uint16_t const offAlignedAccess = offAccess & 0xfffc;
+            switch (offAlignedAccess)
+            {
+                case XAPIC_OFF_ID:
+                case XAPIC_OFF_TPR:
+                case XAPIC_OFF_EOI:
+                case XAPIC_OFF_LDR:
+                case XAPIC_OFF_DFR:
+                case XAPIC_OFF_SVR:
+                case XAPIC_OFF_ESR:
+                case XAPIC_OFF_ICR_LO:
+                case XAPIC_OFF_ICR_HI:
+                case XAPIC_OFF_LVT_TIMER:
+                case XAPIC_OFF_LVT_THERMAL:
+                case XAPIC_OFF_LVT_PERF:
+                case XAPIC_OFF_LVT_LINT0:
+                case XAPIC_OFF_LVT_LINT1:
+                case XAPIC_OFF_LVT_ERROR:
+                case XAPIC_OFF_TIMER_ICR:
+                case XAPIC_OFF_TIMER_DCR:
+                    break;
+                default:
+                    return true;
+            }
+        }
+        else if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)
+        {
+            /*
+             * With virtual-interrupt delivery, a write access to any of the
+             * following registers are virtualized. Accessing any other register
+             * causes a VM-exit.
+             *
+             * Note! The specification does not allow writing to offsets in-between
+             * these registers (e.g. TPR + 1 byte) unlike read accesses.
+             */
+            switch (offAccess)
+            {
+                case XAPIC_OFF_TPR:
+                case XAPIC_OFF_EOI:
+                case XAPIC_OFF_ICR_LO:
+                    break;
+                default:
+                    return true;
+            }
+        }
+        else
+        {
+            /*
+             * Without APIC-register virtualization or virtual-interrupt delivery,
+             * only TPR accesses are virtualized.
+             */
+            if (offAccess == XAPIC_OFF_TPR)
+            { /* likely */ }
+            else
+                return true;
+        }
+    }
+    else
+    {
+        /*
+         * Check read accesses to the APIC-access page that cause VM-exits.
+         */
+        if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT)
+        {
+            /*
+             * With APIC-register virtualization, a read access to any of the
+             * following registers are virtualized. Accessing any other register
+             * causes a VM-exit.
+             */
+            uint16_t const offAlignedAccess = offAccess & 0xfffc;
+            switch (offAlignedAccess)
+            {
+                /** @todo r=ramshankar: What about XAPIC_OFF_LVT_CMCI? */
+                case XAPIC_OFF_ID:
+                case XAPIC_OFF_VERSION:
+                case XAPIC_OFF_TPR:
+                case XAPIC_OFF_EOI:
+                case XAPIC_OFF_LDR:
+                case XAPIC_OFF_DFR:
+                case XAPIC_OFF_SVR:
+                case XAPIC_OFF_ISR0:    case XAPIC_OFF_ISR1:    case XAPIC_OFF_ISR2:    case XAPIC_OFF_ISR3:
+                case XAPIC_OFF_ISR4:    case XAPIC_OFF_ISR5:    case XAPIC_OFF_ISR6:    case XAPIC_OFF_ISR7:
+                case XAPIC_OFF_TMR0:    case XAPIC_OFF_TMR1:    case XAPIC_OFF_TMR2:    case XAPIC_OFF_TMR3:
+                case XAPIC_OFF_TMR4:    case XAPIC_OFF_TMR5:    case XAPIC_OFF_TMR6:    case XAPIC_OFF_TMR7:
+                case XAPIC_OFF_IRR0:    case XAPIC_OFF_IRR1:    case XAPIC_OFF_IRR2:    case XAPIC_OFF_IRR3:
+                case XAPIC_OFF_IRR4:    case XAPIC_OFF_IRR5:    case XAPIC_OFF_IRR6:    case XAPIC_OFF_IRR7:
+                case XAPIC_OFF_ESR:
+                case XAPIC_OFF_ICR_LO:
+                case XAPIC_OFF_ICR_HI:
+                case XAPIC_OFF_LVT_TIMER:
+                case XAPIC_OFF_LVT_THERMAL:
+                case XAPIC_OFF_LVT_PERF:
+                case XAPIC_OFF_LVT_LINT0:
+                case XAPIC_OFF_LVT_LINT1:
+                case XAPIC_OFF_LVT_ERROR:
+                case XAPIC_OFF_TIMER_ICR:
+                case XAPIC_OFF_TIMER_DCR:
+                    break;
+                default:
+                    return true;
+            }
+        }
+        else
+        {
+            /* Without APIC-register virtualization, only TPR accesses are virtualized. */
+            if (offAccess == XAPIC_OFF_TPR)
+            { /* likely */ }
+            else
+                return true;
+        }
+    }
+
+    /* The APIC access is virtualized, does not cause a VM-exit. */
+    return false;
+}
+
+
+/**
+ * Virtualizes a memory-based APIC access where the address is not used to access
+ * memory.
+ *
+ * This is for instructions like MONITOR, CLFLUSH, CLFLUSHOPT, ENTER which may cause
+ * page-faults but do not use the address to access memory.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pGCPhysAccess   Pointer to the guest-physical address used.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVirtApicAccessUnused(PVMCPUCC pVCpu, PRTGCPHYS pGCPhysAccess)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
+    Assert(pGCPhysAccess);
+
+    RTGCPHYS const GCPhysAccess = *pGCPhysAccess & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+    RTGCPHYS const GCPhysApic   = pVmcs->u64AddrApicAccess.u;
+    Assert(!(GCPhysApic & PAGE_OFFSET_MASK));
+
+    if (GCPhysAccess == GCPhysApic)
+    {
+        uint16_t const offAccess = *pGCPhysAccess & PAGE_OFFSET_MASK;
+        uint32_t const fAccess   = IEM_ACCESS_TYPE_READ;
+        uint16_t const cbAccess  = 1;
+        bool const fIntercept = iemVmxVirtApicIsMemAccessIntercepted(pVCpu, offAccess, cbAccess, fAccess);
+        if (fIntercept)
+            return iemVmxVmexitApicAccess(pVCpu, offAccess, fAccess);
+
+        *pGCPhysAccess = GCPhysApic | offAccess;
+        return VINF_VMX_MODIFIES_BEHAVIOR;
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * Virtualizes a memory-based APIC access.
+ *
+ * @returns VBox strict status code.
+ * @retval VINF_VMX_MODIFIES_BEHAVIOR if the access was virtualized.
+ * @retval VINF_VMX_VMEXIT if the access causes a VM-exit.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offAccess   The offset of the register being accessed (within the
+ *                      APIC-access page).
+ * @param   cbAccess    The size of the access in bytes.
+ * @param   pvData      Pointer to the data being written or where to store the data
+ *                      being read.
+ * @param   fAccess     The type of access (must contain IEM_ACCESS_TYPE_READ or
+ *                      IEM_ACCESS_TYPE_WRITE or IEM_ACCESS_INSTRUCTION).
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVirtApicAccessMem(PVMCPUCC pVCpu, uint16_t offAccess, size_t cbAccess, void *pvData,
+                                                uint32_t fAccess)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS); NOREF(pVmcs);
+    Assert(pvData);
+    Assert(   (fAccess & IEM_ACCESS_TYPE_READ)
+           || (fAccess & IEM_ACCESS_TYPE_WRITE)
+           || (fAccess & IEM_ACCESS_INSTRUCTION));
+
+    bool const fIntercept = iemVmxVirtApicIsMemAccessIntercepted(pVCpu, offAccess, cbAccess, fAccess);
+    if (fIntercept)
+        return iemVmxVmexitApicAccess(pVCpu, offAccess, fAccess);
+
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+    {
+        /*
+         * A write access to the APIC-access page that is virtualized (rather than
+         * causing a VM-exit) writes data to the virtual-APIC page.
+         */
+        uint32_t const u32Data = *(uint32_t *)pvData;
+        iemVmxVirtApicWriteRaw32(pVCpu, offAccess, u32Data);
+
+        /*
+         * Record the currently updated APIC offset, as we need this later for figuring
+         * out whether to perform TPR, EOI or self-IPI virtualization as well as well
+         * as for supplying the exit qualification when causing an APIC-write VM-exit.
+         *
+         * After completion of the current operation, we need to perform TPR virtualization,
+         * EOI virtualization or APIC-write VM-exit depending on which register was written.
+         *
+         * The current operation may be a REP-prefixed string instruction, execution of any
+         * other instruction, or delivery of an event through the IDT.
+         *
+         * Thus things like clearing bytes 3:1 of the VTPR, clearing VEOI are not to be
+         * performed now but later after completion of the current operation.
+         *
+         * See Intel spec. 29.4.3.2 "APIC-Write Emulation".
+         */
+        iemVmxVirtApicSetPendingWrite(pVCpu, offAccess);
+    }
+    else
+    {
+        /*
+         * A read access from the APIC-access page that is virtualized (rather than
+         * causing a VM-exit) returns data from the virtual-APIC page.
+         *
+         * See Intel spec. 29.4.2 "Virtualizing Reads from the APIC-Access Page".
+         */
+        Assert(cbAccess <= 4);
+        Assert(offAccess < XAPIC_OFF_END + 4);
+        static uint32_t const s_auAccessSizeMasks[] = { 0, 0xff, 0xffff, 0xffffff, 0xffffffff };
+
+        uint32_t u32Data = iemVmxVirtApicReadRaw32(pVCpu, offAccess);
+        u32Data &= s_auAccessSizeMasks[cbAccess];
+        *(uint32_t *)pvData = u32Data;
+    }
+
+    return VINF_VMX_MODIFIES_BEHAVIOR;
+}
+
+
+/**
+ * Virtualizes an MSR-based APIC read access.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_VMX_MODIFIES_BEHAVIOR if the MSR read was virtualized.
+ * @retval  VINF_VMX_INTERCEPT_NOT_ACTIVE if the MSR read access must be
+ *          handled by the x2APIC device.
+ * @retval  VERR_OUT_RANGE if the MSR read was supposed to be virtualized but was
+ *          not within the range of valid MSRs, caller must raise \#GP(0).
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The x2APIC MSR being read.
+ * @param   pu64Value   Where to store the read x2APIC MSR value (only valid when
+ *                      VINF_VMX_MODIFIES_BEHAVIOR is returned).
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVirtApicAccessMsrRead(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Value)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE);
+    Assert(pu64Value);
+
+    if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT)
+    {
+        if (   idMsr >= MSR_IA32_X2APIC_START
+            && idMsr <= MSR_IA32_X2APIC_END)
+        {
+            uint16_t const offReg   = (idMsr & 0xff) << 4;
+            uint64_t const u64Value = iemVmxVirtApicReadRaw64(pVCpu, offReg);
+            *pu64Value = u64Value;
+            return VINF_VMX_MODIFIES_BEHAVIOR;
+        }
+        return VERR_OUT_OF_RANGE;
+    }
+
+    if (idMsr == MSR_IA32_X2APIC_TPR)
+    {
+        uint16_t const offReg   = (idMsr & 0xff) << 4;
+        uint64_t const u64Value = iemVmxVirtApicReadRaw64(pVCpu, offReg);
+        *pu64Value = u64Value;
+        return VINF_VMX_MODIFIES_BEHAVIOR;
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * Virtualizes an MSR-based APIC write access.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_VMX_MODIFIES_BEHAVIOR if the MSR write was virtualized.
+ * @retval  VERR_OUT_RANGE if the MSR read was supposed to be virtualized but was
+ *          not within the range of valid MSRs, caller must raise \#GP(0).
+ * @retval  VINF_VMX_INTERCEPT_NOT_ACTIVE if the MSR must be written normally.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The x2APIC MSR being written.
+ * @param   u64Value    The value of the x2APIC MSR being written.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVirtApicAccessMsrWrite(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t u64Value)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    /*
+     * Check if the access is to be virtualized.
+     * See Intel spec. 29.5 "Virtualizing MSR-based APIC Accesses".
+     */
+    if (   idMsr == MSR_IA32_X2APIC_TPR
+        || (   (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)
+            && (   idMsr == MSR_IA32_X2APIC_EOI
+                || idMsr == MSR_IA32_X2APIC_SELF_IPI)))
+    {
+        /* Validate the MSR write depending on the register. */
+        switch (idMsr)
+        {
+            case MSR_IA32_X2APIC_TPR:
+            case MSR_IA32_X2APIC_SELF_IPI:
+            {
+                if (u64Value & UINT64_C(0xffffffffffffff00))
+                    return VERR_OUT_OF_RANGE;
+                break;
+            }
+            case MSR_IA32_X2APIC_EOI:
+            {
+                if (u64Value != 0)
+                    return VERR_OUT_OF_RANGE;
+                break;
+            }
+        }
+
+        /* Write the MSR to the virtual-APIC page. */
+        uint16_t const offReg = (idMsr & 0xff) << 4;
+        iemVmxVirtApicWriteRaw64(pVCpu, offReg, u64Value);
+
+        /*
+         * Record the currently updated APIC offset, as we need this later for figuring
+         * out whether to perform TPR, EOI or self-IPI virtualization as well as well
+         * as for supplying the exit qualification when causing an APIC-write VM-exit.
+         */
+        iemVmxVirtApicSetPendingWrite(pVCpu, offReg);
+
+        return VINF_VMX_MODIFIES_BEHAVIOR;
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * Finds the most significant set bit in a virtual-APIC 256-bit sparse register.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS when the highest set bit is found.
+ * @retval  VERR_NOT_FOUND when no bit is set.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   offReg          The offset of the APIC 256-bit sparse register.
+ * @param   pidxHighestBit  Where to store the highest bit (most significant bit)
+ *                          set in the register. Only valid when VINF_SUCCESS is
+ *                          returned.
+ *
+ * @remarks The format of the 256-bit sparse register here mirrors that found in
+ *          real APIC hardware.
+ */
+static int iemVmxVirtApicGetHighestSetBitInReg(PVMCPUCC pVCpu, uint16_t offReg, uint8_t *pidxHighestBit)
+{
+    Assert(offReg < XAPIC_OFF_END + 4);
+    Assert(pidxHighestBit);
+    Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs));
+
+    /*
+     * There are 8 contiguous fragments (of 16-bytes each) in the sparse register.
+     * However, in each fragment only the first 4 bytes are used.
+     */
+    uint8_t const cFrags = 8;
+    for (int8_t iFrag = cFrags; iFrag >= 0; iFrag--)
+    {
+        uint16_t const offFrag = iFrag * 16;
+        uint32_t const u32Frag = iemVmxVirtApicReadRaw32(pVCpu, offReg + offFrag);
+        if (!u32Frag)
+            continue;
+
+        unsigned idxHighestBit = ASMBitLastSetU32(u32Frag);
+        Assert(idxHighestBit > 0);
+        --idxHighestBit;
+        Assert(idxHighestBit <= UINT8_MAX);
+        *pidxHighestBit = idxHighestBit;
+        return VINF_SUCCESS;
+    }
+    return VERR_NOT_FOUND;
+}
+
+
+/**
+ * Evaluates pending virtual interrupts.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC void iemVmxEvalPendingVirtIntrs(PVMCPUCC pVCpu)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+
+    if (!(pVmcs->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
+    {
+        uint8_t const uRvi = RT_LO_U8(pVmcs->u16GuestIntStatus);
+        uint8_t const uPpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_PPR);
+
+        if ((uRvi >> 4) > (uPpr >> 4))
+        {
+            Log2(("eval_virt_intrs: uRvi=%#x uPpr=%#x - Signalling pending interrupt\n", uRvi, uPpr));
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
+        }
+        else
+            Log2(("eval_virt_intrs: uRvi=%#x uPpr=%#x - Nothing to do\n", uRvi, uPpr));
+    }
+}
+
+
+/**
+ * Performs PPR virtualization.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC void iemVmxPprVirtualization(PVMCPUCC pVCpu)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
+    Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+
+    /*
+     * PPR virtualization is caused in response to a VM-entry, TPR-virtualization,
+     * or EOI-virtualization.
+     *
+     * See Intel spec. 29.1.3 "PPR Virtualization".
+     */
+    uint32_t const uTpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR);
+    uint32_t const uSvi = RT_HI_U8(pVmcs->u16GuestIntStatus);
+
+    uint32_t uPpr;
+    if (((uTpr >> 4) & 0xf) >= ((uSvi >> 4) & 0xf))
+        uPpr = uTpr & 0xff;
+    else
+        uPpr = uSvi & 0xf0;
+
+    Log2(("ppr_virt: uTpr=%#x uSvi=%#x uPpr=%#x\n", uTpr, uSvi, uPpr));
+    iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_PPR, uPpr);
+}
+
+
+/**
+ * Performs VMX TPR virtualization.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxTprVirtualization(PVMCPUCC pVCpu)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
+
+    /*
+     * We should have already performed the virtual-APIC write to the TPR offset
+     * in the virtual-APIC page. We now perform TPR virtualization.
+     *
+     * See Intel spec. 29.1.2 "TPR Virtualization".
+     */
+    if (!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY))
+    {
+        uint32_t const uTprThreshold = pVmcs->u32TprThreshold;
+        uint32_t const uTpr          = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR);
+
+        /*
+         * If the VTPR falls below the TPR threshold, we must cause a VM-exit.
+         * See Intel spec. 29.1.2 "TPR Virtualization".
+         */
+        if (((uTpr >> 4) & 0xf) < uTprThreshold)
+        {
+            Log2(("tpr_virt: uTpr=%u uTprThreshold=%u -> VM-exit\n", uTpr, uTprThreshold));
+            return iemVmxVmexit(pVCpu, VMX_EXIT_TPR_BELOW_THRESHOLD, 0 /* u64ExitQual */);
+        }
+    }
+    else
+    {
+        iemVmxPprVirtualization(pVCpu);
+        iemVmxEvalPendingVirtIntrs(pVCpu);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks whether an EOI write for the given interrupt vector causes a VM-exit or
+ * not.
+ *
+ * @returns @c true if the EOI write is intercepted, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uVector     The interrupt that was acknowledged using an EOI.
+ */
+IEM_STATIC bool iemVmxIsEoiInterceptSet(PCVMCPU pVCpu, uint8_t uVector)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+
+    if (uVector < 64)
+        return RT_BOOL(pVmcs->u64EoiExitBitmap0.u & RT_BIT_64(uVector));
+    if (uVector < 128)
+        return RT_BOOL(pVmcs->u64EoiExitBitmap1.u & RT_BIT_64(uVector));
+    if (uVector < 192)
+        return RT_BOOL(pVmcs->u64EoiExitBitmap2.u & RT_BIT_64(uVector));
+    return RT_BOOL(pVmcs->u64EoiExitBitmap3.u & RT_BIT_64(uVector));
+}
+
+
+/**
+ * Performs EOI virtualization.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxEoiVirtualization(PVMCPUCC pVCpu)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+
+    /*
+     * Clear the interrupt guest-interrupt as no longer in-service (ISR)
+     * and get the next guest-interrupt that's in-service (if any).
+     *
+     * See Intel spec. 29.1.4 "EOI Virtualization".
+     */
+    uint8_t const uRvi = RT_LO_U8(pVmcs->u16GuestIntStatus);
+    uint8_t const uSvi = RT_HI_U8(pVmcs->u16GuestIntStatus);
+    Log2(("eoi_virt: uRvi=%#x uSvi=%#x\n", uRvi, uSvi));
+
+    uint8_t uVector = uSvi;
+    iemVmxVirtApicClearVectorInReg(pVCpu, XAPIC_OFF_ISR0, uVector);
+
+    uVector = 0;
+    iemVmxVirtApicGetHighestSetBitInReg(pVCpu, XAPIC_OFF_ISR0, &uVector);
+
+    if (uVector)
+        Log2(("eoi_virt: next interrupt %#x\n", uVector));
+    else
+        Log2(("eoi_virt: no interrupt pending in ISR\n"));
+
+    /* Update guest-interrupt status SVI (leave RVI portion as it is) in the VMCS. */
+    pVmcs->u16GuestIntStatus = RT_MAKE_U16(uRvi, uVector);
+
+    iemVmxPprVirtualization(pVCpu);
+    if (iemVmxIsEoiInterceptSet(pVCpu, uVector))
+        return iemVmxVmexit(pVCpu, VMX_EXIT_VIRTUALIZED_EOI, uVector);
+    iemVmxEvalPendingVirtIntrs(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Performs self-IPI virtualization.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxSelfIpiVirtualization(PVMCPUCC pVCpu)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
+
+    /*
+     * We should have already performed the virtual-APIC write to the self-IPI offset
+     * in the virtual-APIC page. We now perform self-IPI virtualization.
+     *
+     * See Intel spec. 29.1.5 "Self-IPI Virtualization".
+     */
+    uint8_t const uVector = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_ICR_LO);
+    Log2(("self_ipi_virt: uVector=%#x\n", uVector));
+    iemVmxVirtApicSetVectorInReg(pVCpu, XAPIC_OFF_IRR0, uVector);
+    uint8_t const uRvi = RT_LO_U8(pVmcs->u16GuestIntStatus);
+    uint8_t const uSvi = RT_HI_U8(pVmcs->u16GuestIntStatus);
+    if (uVector > uRvi)
+        pVmcs->u16GuestIntStatus = RT_MAKE_U16(uVector, uSvi);
+    iemVmxEvalPendingVirtIntrs(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Performs VMX APIC-write emulation.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxApicWriteEmulation(PVMCPUCC pVCpu)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    /* Import the virtual-APIC write offset (part of the hardware-virtualization state). */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+
+    /*
+     * Perform APIC-write emulation based on the virtual-APIC register written.
+     * See Intel spec. 29.4.3.2 "APIC-Write Emulation".
+     */
+    uint16_t const offApicWrite = iemVmxVirtApicClearPendingWrite(pVCpu);
+    VBOXSTRICTRC rcStrict;
+    switch (offApicWrite)
+    {
+        case XAPIC_OFF_TPR:
+        {
+            /* Clear bytes 3:1 of the VTPR and perform TPR virtualization. */
+            uint32_t uTpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR);
+            uTpr         &= UINT32_C(0x000000ff);
+            iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_TPR, uTpr);
+            Log2(("iemVmxApicWriteEmulation: TPR write %#x\n", uTpr));
+            rcStrict = iemVmxTprVirtualization(pVCpu);
+            break;
+        }
+
+        case XAPIC_OFF_EOI:
+        {
+            if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)
+            {
+                /* Clear VEOI and perform EOI virtualization. */
+                iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_EOI, 0);
+                Log2(("iemVmxApicWriteEmulation: EOI write\n"));
+                rcStrict = iemVmxEoiVirtualization(pVCpu);
+            }
+            else
+                rcStrict = iemVmxVmexitApicWrite(pVCpu, offApicWrite);
+            break;
+        }
+
+        case XAPIC_OFF_ICR_LO:
+        {
+            if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)
+            {
+                /* If the ICR_LO is valid, write it and perform self-IPI virtualization. */
+                uint32_t const uIcrLo    = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR);
+                uint32_t const fIcrLoMb0 = UINT32_C(0xfffbb700);
+                uint32_t const fIcrLoMb1 = UINT32_C(0x000000f0);
+                if (   !(uIcrLo & fIcrLoMb0)
+                    &&  (uIcrLo & fIcrLoMb1))
+                {
+                    Log2(("iemVmxApicWriteEmulation: Self-IPI virtualization with vector %#x\n", (uIcrLo & 0xff)));
+                    rcStrict = iemVmxSelfIpiVirtualization(pVCpu);
+                }
+                else
+                    rcStrict = iemVmxVmexitApicWrite(pVCpu, offApicWrite);
+            }
+            else
+                rcStrict = iemVmxVmexitApicWrite(pVCpu, offApicWrite);
+            break;
+        }
+
+        case XAPIC_OFF_ICR_HI:
+        {
+            /* Clear bytes 2:0 of VICR_HI. No other virtualization or VM-exit must occur. */
+            uint32_t uIcrHi  = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_ICR_HI);
+            uIcrHi          &= UINT32_C(0xff000000);
+            iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_ICR_HI, uIcrHi);
+            rcStrict = VINF_SUCCESS;
+            break;
+        }
+
+        default:
+        {
+            /* Writes to any other virtual-APIC register causes an APIC-write VM-exit. */
+            rcStrict = iemVmxVmexitApicWrite(pVCpu, offApicWrite);
+            break;
+        }
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Checks guest control registers, debug registers and MSRs as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+DECLINLINE(int) iemVmxVmentryCheckGuestControlRegsMsrs(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    /*
+     * Guest Control Registers, Debug Registers, and MSRs.
+     * See Intel spec. 26.3.1.1 "Checks on Guest Control Registers, Debug Registers, and MSRs".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    const char *const pszFailure  = "VM-exit";
+    bool const fUnrestrictedGuest = RT_BOOL(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
+
+    /* CR0 reserved bits. */
+    {
+        /* CR0 MB1 bits. */
+        uint64_t u64Cr0Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed0;
+        Assert(!(u64Cr0Fixed0 & (X86_CR0_NW | X86_CR0_CD)));
+        if (fUnrestrictedGuest)
+            u64Cr0Fixed0 &= ~(X86_CR0_PE | X86_CR0_PG);
+        if ((pVmcs->u64GuestCr0.u & u64Cr0Fixed0) == u64Cr0Fixed0)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr0Fixed0);
+
+        /* CR0 MBZ bits. */
+        uint64_t const u64Cr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1;
+        if (!(pVmcs->u64GuestCr0.u & ~u64Cr0Fixed1))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr0Fixed1);
+
+        /* Without unrestricted guest support, VT-x supports does not support unpaged protected mode. */
+        if (   !fUnrestrictedGuest
+            &&  (pVmcs->u64GuestCr0.u & X86_CR0_PG)
+            && !(pVmcs->u64GuestCr0.u & X86_CR0_PE))
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr0PgPe);
+    }
+
+    /* CR4 reserved bits. */
+    {
+        /* CR4 MB1 bits. */
+        uint64_t const u64Cr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0;
+        if ((pVmcs->u64GuestCr4.u & u64Cr4Fixed0) == u64Cr4Fixed0)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr4Fixed0);
+
+        /* CR4 MBZ bits. */
+        uint64_t const u64Cr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1;
+        if (!(pVmcs->u64GuestCr4.u & ~u64Cr4Fixed1))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr4Fixed1);
+    }
+
+    /* DEBUGCTL MSR. */
+    if (   !(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
+        || !(pVmcs->u64GuestDebugCtlMsr.u & ~MSR_IA32_DEBUGCTL_VALID_MASK_INTEL))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestDebugCtl);
+
+    /* 64-bit CPU checks. */
+    bool const fGstInLongMode = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        if (fGstInLongMode)
+        {
+            /* PAE must be set. */
+            if (   (pVmcs->u64GuestCr0.u & X86_CR0_PG)
+                && (pVmcs->u64GuestCr0.u & X86_CR4_PAE))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPae);
+        }
+        else
+        {
+            /* PCIDE should not be set. */
+            if (!(pVmcs->u64GuestCr4.u & X86_CR4_PCIDE))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPcide);
+        }
+
+        /* CR3. */
+        if (!(pVmcs->u64GuestCr3.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxPhysAddrWidth))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr3);
+
+        /* DR7. */
+        if (   !(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
+            || !(pVmcs->u64GuestDr7.u & X86_DR7_MBZ_MASK))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestDr7);
+
+        /* SYSENTER ESP and SYSENTER EIP. */
+        if (   X86_IS_CANONICAL(pVmcs->u64GuestSysenterEsp.u)
+            && X86_IS_CANONICAL(pVmcs->u64GuestSysenterEip.u))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSysenterEspEip);
+    }
+
+    /* We don't support IA32_PERF_GLOBAL_CTRL MSR yet. */
+    Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR));
+
+    /* PAT MSR. */
+    if (   !(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
+        ||  CPUMIsPatMsrValid(pVmcs->u64GuestPatMsr.u))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPatMsr);
+
+    /* EFER MSR. */
+    if (pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
+    {
+        uint64_t const uValidEferMask = CPUMGetGuestEferMsrValidMask(pVCpu->CTX_SUFF(pVM));
+        if (!(pVmcs->u64GuestEferMsr.u & ~uValidEferMask))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestEferMsrRsvd);
+
+        bool const fGstLma = RT_BOOL(pVmcs->u64GuestEferMsr.u & MSR_K6_EFER_LMA);
+        bool const fGstLme = RT_BOOL(pVmcs->u64GuestEferMsr.u & MSR_K6_EFER_LME);
+        if (   fGstLma == fGstInLongMode
+            && (   !(pVmcs->u64GuestCr0.u & X86_CR0_PG)
+                || fGstLma == fGstLme))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestEferMsr);
+    }
+
+    /* We don't support IA32_BNDCFGS MSR yet. */
+    Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_BNDCFGS_MSR));
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks guest segment registers, LDTR and TR as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+DECLINLINE(int) iemVmxVmentryCheckGuestSegRegs(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    /*
+     * Segment registers.
+     * See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    const char *const pszFailure  = "VM-exit";
+    bool const fGstInV86Mode      = RT_BOOL(pVmcs->u64GuestRFlags.u & X86_EFL_VM);
+    bool const fUnrestrictedGuest = RT_BOOL(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
+    bool const fGstInLongMode     = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+
+    /* Selectors. */
+    if (   !fGstInV86Mode
+        && !fUnrestrictedGuest
+        && (pVmcs->GuestSs & X86_SEL_RPL) != (pVmcs->GuestCs & X86_SEL_RPL))
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegSelCsSsRpl);
+
+    for (unsigned iSegReg = 0; iSegReg < X86_SREG_COUNT; iSegReg++)
+    {
+        CPUMSELREG SelReg;
+        int rc = iemVmxVmcsGetGuestSegReg(pVmcs, iSegReg, &SelReg);
+        if (RT_LIKELY(rc == VINF_SUCCESS))
+        { /* likely */ }
+        else
+            return rc;
+
+        /*
+         * Virtual-8086 mode checks.
+         */
+        if (fGstInV86Mode)
+        {
+            /* Base address. */
+            if (SelReg.u64Base == (uint64_t)SelReg.Sel << 4)
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegBaseV86(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+
+            /* Limit. */
+            if (SelReg.u32Limit == 0xffff)
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegLimitV86(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+
+            /* Attribute. */
+            if (SelReg.Attr.u == 0xf3)
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrV86(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+
+            /* We're done; move to checking the next segment. */
+            continue;
+        }
+
+        /* Checks done by 64-bit CPUs. */
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+        {
+            /* Base address. */
+            if (   iSegReg == X86_SREG_FS
+                || iSegReg == X86_SREG_GS)
+            {
+                if (X86_IS_CANONICAL(SelReg.u64Base))
+                { /* likely */ }
+                else
+                {
+                    VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegBase(iSegReg);
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+                }
+            }
+            else if (iSegReg == X86_SREG_CS)
+            {
+                if (!RT_HI_U32(SelReg.u64Base))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegBaseCs);
+            }
+            else
+            {
+                if (   SelReg.Attr.n.u1Unusable
+                    || !RT_HI_U32(SelReg.u64Base))
+                { /* likely */ }
+                else
+                {
+                    VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegBase(iSegReg);
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+                }
+            }
+        }
+
+        /*
+         * Checks outside Virtual-8086 mode.
+         */
+        uint8_t const uSegType     =  SelReg.Attr.n.u4Type;
+        uint8_t const fCodeDataSeg =  SelReg.Attr.n.u1DescType;
+        uint8_t const fUsable      = !SelReg.Attr.n.u1Unusable;
+        uint8_t const uDpl         =  SelReg.Attr.n.u2Dpl;
+        uint8_t const fPresent     =  SelReg.Attr.n.u1Present;
+        uint8_t const uGranularity =  SelReg.Attr.n.u1Granularity;
+        uint8_t const uDefBig      =  SelReg.Attr.n.u1DefBig;
+        uint8_t const fSegLong     =  SelReg.Attr.n.u1Long;
+
+        /* Code or usable segment. */
+        if (   iSegReg == X86_SREG_CS
+            || fUsable)
+        {
+            /* Reserved bits (bits 31:17 and bits 11:8). */
+            if (!(SelReg.Attr.u & 0xfffe0f00))
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrRsvd(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+
+            /* Descriptor type. */
+            if (fCodeDataSeg)
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrDescType(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+
+            /* Present. */
+            if (fPresent)
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrPresent(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+
+            /* Granularity. */
+            if (   ((SelReg.u32Limit & 0x00000fff) == 0x00000fff || !uGranularity)
+                && ((SelReg.u32Limit & 0xfff00000) == 0x00000000 ||  uGranularity))
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrGran(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+        }
+
+        if (iSegReg == X86_SREG_CS)
+        {
+            /* Segment Type and DPL. */
+            if (   uSegType == (X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED)
+                && fUnrestrictedGuest)
+            {
+                if (uDpl == 0)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsDplZero);
+            }
+            else if (   uSegType == (X86_SEL_TYPE_CODE | X86_SEL_TYPE_ACCESSED)
+                     || uSegType == (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_ACCESSED))
+            {
+                X86DESCATTR AttrSs; AttrSs.u = pVmcs->u32GuestSsAttr;
+                if (uDpl == AttrSs.n.u2Dpl)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsDplEqSs);
+            }
+            else if ((uSegType & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF | X86_SEL_TYPE_ACCESSED))
+                              == (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF | X86_SEL_TYPE_ACCESSED))
+            {
+                X86DESCATTR AttrSs; AttrSs.u = pVmcs->u32GuestSsAttr;
+                if (uDpl <= AttrSs.n.u2Dpl)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsDplLtSs);
+            }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsType);
+
+            /* Def/Big. */
+            if (   fGstInLongMode
+                && fSegLong)
+            {
+                if (uDefBig == 0)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsDefBig);
+            }
+        }
+        else if (iSegReg == X86_SREG_SS)
+        {
+            /* Segment Type. */
+            if (   !fUsable
+                || uSegType == (X86_SEL_TYPE_RW   | X86_SEL_TYPE_ACCESSED)
+                || uSegType == (X86_SEL_TYPE_DOWN | X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrSsType);
+
+            /* DPL. */
+            if (!fUnrestrictedGuest)
+            {
+                if (uDpl == (SelReg.Sel & X86_SEL_RPL))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrSsDplEqRpl);
+            }
+            X86DESCATTR AttrCs; AttrCs.u = pVmcs->u32GuestCsAttr;
+            if (   AttrCs.n.u4Type == (X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED)
+                || !(pVmcs->u64GuestCr0.u & X86_CR0_PE))
+            {
+                if (uDpl == 0)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrSsDplZero);
+            }
+        }
+        else
+        {
+            /* DS, ES, FS, GS. */
+            if (fUsable)
+            {
+                /* Segment type. */
+                if (uSegType & X86_SEL_TYPE_ACCESSED)
+                { /* likely */ }
+                else
+                {
+                    VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrTypeAcc(iSegReg);
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+                }
+
+                if (   !(uSegType & X86_SEL_TYPE_CODE)
+                    ||  (uSegType & X86_SEL_TYPE_READ))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsTypeRead);
+
+                /* DPL. */
+                if (   !fUnrestrictedGuest
+                    && uSegType <= (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_ACCESSED))
+                {
+                    if (uDpl >= (SelReg.Sel & X86_SEL_RPL))
+                    { /* likely */ }
+                    else
+                    {
+                        VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrDplRpl(iSegReg);
+                        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+                    }
+                }
+            }
+        }
+    }
+
+    /*
+     * LDTR.
+     */
+    {
+        CPUMSELREG Ldtr;
+        Ldtr.Sel      = pVmcs->GuestLdtr;
+        Ldtr.u32Limit = pVmcs->u32GuestLdtrLimit;
+        Ldtr.u64Base  = pVmcs->u64GuestLdtrBase.u;
+        Ldtr.Attr.u   = pVmcs->u32GuestLdtrAttr;
+
+        if (!Ldtr.Attr.n.u1Unusable)
+        {
+            /* Selector. */
+            if (!(Ldtr.Sel & X86_SEL_LDT))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegSelLdtr);
+
+            /* Base. */
+            if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+            {
+                if (X86_IS_CANONICAL(Ldtr.u64Base))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegBaseLdtr);
+            }
+
+            /* Attributes. */
+            /* Reserved bits (bits 31:17 and bits 11:8). */
+            if (!(Ldtr.Attr.u & 0xfffe0f00))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrRsvd);
+
+            if (Ldtr.Attr.n.u4Type == X86_SEL_TYPE_SYS_LDT)
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrType);
+
+            if (!Ldtr.Attr.n.u1DescType)
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrDescType);
+
+            if (Ldtr.Attr.n.u1Present)
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrPresent);
+
+            if (   ((Ldtr.u32Limit & 0x00000fff) == 0x00000fff || !Ldtr.Attr.n.u1Granularity)
+                && ((Ldtr.u32Limit & 0xfff00000) == 0x00000000 ||  Ldtr.Attr.n.u1Granularity))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrGran);
+        }
+    }
+
+    /*
+     * TR.
+     */
+    {
+        CPUMSELREG Tr;
+        Tr.Sel      = pVmcs->GuestTr;
+        Tr.u32Limit = pVmcs->u32GuestTrLimit;
+        Tr.u64Base  = pVmcs->u64GuestTrBase.u;
+        Tr.Attr.u   = pVmcs->u32GuestTrAttr;
+
+        /* Selector. */
+        if (!(Tr.Sel & X86_SEL_LDT))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegSelTr);
+
+        /* Base. */
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+        {
+            if (X86_IS_CANONICAL(Tr.u64Base))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegBaseTr);
+        }
+
+        /* Attributes. */
+        /* Reserved bits (bits 31:17 and bits 11:8). */
+        if (!(Tr.Attr.u & 0xfffe0f00))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrRsvd);
+
+        if (!Tr.Attr.n.u1Unusable)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrUnusable);
+
+        if (   Tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY
+            || (   !fGstInLongMode
+                && Tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrType);
+
+        if (!Tr.Attr.n.u1DescType)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrDescType);
+
+        if (Tr.Attr.n.u1Present)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrPresent);
+
+        if (   ((Tr.u32Limit & 0x00000fff) == 0x00000fff || !Tr.Attr.n.u1Granularity)
+            && ((Tr.u32Limit & 0xfff00000) == 0x00000000 ||  Tr.Attr.n.u1Granularity))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrGran);
+    }
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks guest GDTR and IDTR as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+DECLINLINE(int) iemVmxVmentryCheckGuestGdtrIdtr(PVMCPUCC pVCpu,  const char *pszInstr)
+{
+    /*
+     * GDTR and IDTR.
+     * See Intel spec. 26.3.1.3 "Checks on Guest Descriptor-Table Registers".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    const char *const pszFailure = "VM-exit";
+
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        /* Base. */
+        if (X86_IS_CANONICAL(pVmcs->u64GuestGdtrBase.u))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestGdtrBase);
+
+        if (X86_IS_CANONICAL(pVmcs->u64GuestIdtrBase.u))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIdtrBase);
+    }
+
+    /* Limit. */
+    if (!RT_HI_U16(pVmcs->u32GuestGdtrLimit))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestGdtrLimit);
+
+    if (!RT_HI_U16(pVmcs->u32GuestIdtrLimit))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIdtrLimit);
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks guest RIP and RFLAGS as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+DECLINLINE(int) iemVmxVmentryCheckGuestRipRFlags(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    /*
+     * RIP and RFLAGS.
+     * See Intel spec. 26.3.1.4 "Checks on Guest RIP and RFLAGS".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    const char *const pszFailure = "VM-exit";
+    bool const fGstInLongMode    = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+
+    /* RIP. */
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        X86DESCATTR AttrCs;
+        AttrCs.u = pVmcs->u32GuestCsAttr;
+        if (   !fGstInLongMode
+            || !AttrCs.n.u1Long)
+        {
+            if (!RT_HI_U32(pVmcs->u64GuestRip.u))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRipRsvd);
+        }
+
+        if (   fGstInLongMode
+            && AttrCs.n.u1Long)
+        {
+            Assert(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxLinearAddrWidth == 48);   /* Canonical. */
+            if (   IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxLinearAddrWidth < 64
+                && X86_IS_CANONICAL(pVmcs->u64GuestRip.u))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRip);
+        }
+    }
+
+    /* RFLAGS (bits 63:22 (or 31:22), bits 15, 5, 3 are reserved, bit 1 MB1). */
+    uint64_t const uGuestRFlags = IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode ? pVmcs->u64GuestRFlags.u
+                                : pVmcs->u64GuestRFlags.s.Lo;
+    if (   !(uGuestRFlags & ~(X86_EFL_LIVE_MASK | X86_EFL_RA1_MASK))
+        &&  (uGuestRFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK)
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRFlagsRsvd);
+
+    if (   fGstInLongMode
+        || !(pVmcs->u64GuestCr0.u & X86_CR0_PE))
+    {
+        if (!(uGuestRFlags & X86_EFL_VM))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRFlagsVm);
+    }
+
+    if (VMX_ENTRY_INT_INFO_IS_EXT_INT(pVmcs->u32EntryIntInfo))
+    {
+        if (uGuestRFlags & X86_EFL_IF)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRFlagsIf);
+    }
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks guest non-register state as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+DECLINLINE(int) iemVmxVmentryCheckGuestNonRegState(PVMCPUCC pVCpu,  const char *pszInstr)
+{
+    /*
+     * Guest non-register state.
+     * See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
+     */
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    const char *const pszFailure = "VM-exit";
+
+    /*
+     * Activity state.
+     */
+    uint64_t const u64GuestVmxMiscMsr = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Misc;
+    uint32_t const fActivityStateMask = RT_BF_GET(u64GuestVmxMiscMsr, VMX_BF_MISC_ACTIVITY_STATES);
+    if (!(pVmcs->u32GuestActivityState & fActivityStateMask))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateRsvd);
+
+    X86DESCATTR AttrSs; AttrSs.u = pVmcs->u32GuestSsAttr;
+    if (   !AttrSs.n.u2Dpl
+        || pVmcs->u32GuestActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT)
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateSsDpl);
+
+    if (   pVmcs->u32GuestIntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI
+        || pVmcs->u32GuestIntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
+    {
+        if (pVmcs->u32GuestActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateStiMovSs);
+    }
+
+    if (VMX_ENTRY_INT_INFO_IS_VALID(pVmcs->u32EntryIntInfo))
+    {
+        uint8_t const uType   = VMX_ENTRY_INT_INFO_TYPE(pVmcs->u32EntryIntInfo);
+        uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(pVmcs->u32EntryIntInfo);
+        AssertCompile(VMX_V_GUEST_ACTIVITY_STATE_MASK == (VMX_VMCS_GUEST_ACTIVITY_HLT | VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN));
+        switch (pVmcs->u32GuestActivityState)
+        {
+            case VMX_VMCS_GUEST_ACTIVITY_HLT:
+            {
+                if (   uType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT
+                    || uType == VMX_ENTRY_INT_INFO_TYPE_NMI
+                    || (   uType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
+                        && (   uVector == X86_XCPT_DB
+                            || uVector == X86_XCPT_MC))
+                    || (   uType   == VMX_ENTRY_INT_INFO_TYPE_OTHER_EVENT
+                        && uVector == VMX_ENTRY_INT_INFO_VECTOR_MTF))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateHlt);
+                break;
+            }
+
+            case VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN:
+            {
+                if (   uType == VMX_ENTRY_INT_INFO_TYPE_NMI
+                    || (   uType   == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
+                        && uVector == X86_XCPT_MC))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateShutdown);
+                break;
+            }
+
+            case VMX_VMCS_GUEST_ACTIVITY_ACTIVE:
+            default:
+                break;
+        }
+    }
+
+    /*
+     * Interruptibility state.
+     */
+    if (!(pVmcs->u32GuestIntrState & ~VMX_VMCS_GUEST_INT_STATE_MASK))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateRsvd);
+
+    if ((pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
+                                 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateStiMovSs);
+
+    if (    (pVmcs->u64GuestRFlags.u & X86_EFL_IF)
+        || !(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateRFlagsSti);
+
+    if (VMX_ENTRY_INT_INFO_IS_VALID(pVmcs->u32EntryIntInfo))
+    {
+        uint8_t const uType = VMX_ENTRY_INT_INFO_TYPE(pVmcs->u32EntryIntInfo);
+        if (uType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
+        {
+            if (!(pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateExtInt);
+        }
+        else if (uType == VMX_ENTRY_INT_INFO_TYPE_NMI)
+        {
+            if (!(pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)))
+            { /* likely */ }
+            else
+            {
+                /*
+                 * We don't support injecting NMIs when blocking-by-STI would be in effect.
+                 * We update the Exit qualification only when blocking-by-STI is set
+                 * without blocking-by-MovSS being set. Although in practise it  does not
+                 * make much difference since the order of checks are implementation defined.
+                 */
+                if (!(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
+                    iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_NMI_INJECT);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateNmi);
+            }
+
+            if (   !(pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+                || !(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI))
+            { /* likely */ }
+            else
+               IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateVirtNmi);
+        }
+    }
+
+    /* We don't support SMM yet. So blocking-by-SMIs must not be set. */
+    if (!(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateSmi);
+
+    /* We don't support SGX yet. So enclave-interruption must not be set. */
+    if (!(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_ENCLAVE))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateEnclave);
+
+    /*
+     * Pending debug exceptions.
+     */
+    uint64_t const uPendingDbgXcpts = IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode
+                                    ? pVmcs->u64GuestPendingDbgXcpts.u
+                                    : pVmcs->u64GuestPendingDbgXcpts.s.Lo;
+    if (!(uPendingDbgXcpts & ~VMX_VMCS_GUEST_PENDING_DEBUG_VALID_MASK))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPndDbgXcptRsvd);
+
+    if (   (pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
+        || pVmcs->u32GuestActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
+    {
+        if (   (pVmcs->u64GuestRFlags.u & X86_EFL_TF)
+            && !(pVmcs->u64GuestDebugCtlMsr.u & MSR_IA32_DEBUGCTL_BTF)
+            && !(uPendingDbgXcpts & VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS))
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPndDbgXcptBsTf);
+
+        if (   (   !(pVmcs->u64GuestRFlags.u & X86_EFL_TF)
+                ||  (pVmcs->u64GuestDebugCtlMsr.u & MSR_IA32_DEBUGCTL_BTF))
+            && (uPendingDbgXcpts & VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS))
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPndDbgXcptBsNoTf);
+    }
+
+    /* We don't support RTM (Real-time Transactional Memory) yet. */
+    if (!(uPendingDbgXcpts & VMX_VMCS_GUEST_PENDING_DEBUG_RTM))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPndDbgXcptRtm);
+
+    /*
+     * VMCS link pointer.
+     */
+    if (pVmcs->u64VmcsLinkPtr.u != UINT64_C(0xffffffffffffffff))
+    {
+        RTGCPHYS const GCPhysShadowVmcs = pVmcs->u64VmcsLinkPtr.u;
+        /* We don't support SMM yet (so VMCS link pointer cannot be the current VMCS). */
+        if (GCPhysShadowVmcs != IEM_VMX_GET_CURRENT_VMCS(pVCpu))
+        { /* likely */ }
+        else
+        {
+            iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR);
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmcsLinkPtrCurVmcs);
+        }
+
+        /* Validate the address. */
+        if (   !(GCPhysShadowVmcs & X86_PAGE_4K_OFFSET_MASK)
+            && !(GCPhysShadowVmcs >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+            &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysShadowVmcs))
+        { /* likely */ }
+        else
+        {
+            iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR);
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrVmcsLinkPtr);
+        }
+    }
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the PDPTEs referenced by the nested-guest CR3 are valid as part of
+ * VM-entry.
+ *
+ * @returns @c true if all PDPTEs are valid, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ * @param   pVmcs       Pointer to the virtual VMCS.
+ */
+IEM_STATIC int iemVmxVmentryCheckGuestPdptesForCr3(PVMCPUCC pVCpu, const char *pszInstr, PVMXVVMCS pVmcs)
+{
+    /*
+     * Check PDPTEs.
+     * See Intel spec. 4.4.1 "PDPTE Registers".
+     */
+    uint64_t const uGuestCr3 = pVmcs->u64GuestCr3.u & X86_CR3_PAE_PAGE_MASK;
+    const char *const pszFailure = "VM-exit";
+
+    X86PDPE aPdptes[X86_PG_PAE_PDPE_ENTRIES];
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), (void *)&aPdptes[0], uGuestCr3, sizeof(aPdptes));
+    if (RT_SUCCESS(rc))
+    {
+        for (unsigned iPdpte = 0; iPdpte < RT_ELEMENTS(aPdptes); iPdpte++)
+        {
+            if (   !(aPdptes[iPdpte].u & X86_PDPE_P)
+                || !(aPdptes[iPdpte].u & X86_PDPE_PAE_MBZ_MASK))
+            { /* likely */ }
+            else
+            {
+                iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_PDPTE);
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentryPdpteRsvd(iPdpte);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+        }
+    }
+    else
+    {
+        iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_PDPTE);
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPdpteCr3ReadPhys);
+    }
+
+    NOREF(pszFailure);
+    NOREF(pszInstr);
+    return rc;
+}
+
+
+/**
+ * Checks guest PDPTEs as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+DECLINLINE(int) iemVmxVmentryCheckGuestPdptes(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    /*
+     * Guest PDPTEs.
+     * See Intel spec. 26.3.1.5 "Checks on Guest Page-Directory-Pointer-Table Entries".
+     */
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    bool const fGstInLongMode = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+
+    /* Check PDPTes if the VM-entry is to a guest using PAE paging. */
+    int rc;
+    if (   !fGstInLongMode
+        && (pVmcs->u64GuestCr4.u & X86_CR4_PAE)
+        && (pVmcs->u64GuestCr0.u & X86_CR0_PG))
+    {
+        /*
+         * We don't support nested-paging for nested-guests yet.
+         *
+         * Without nested-paging for nested-guests, PDPTEs in the VMCS are not used,
+         * rather we need to check the PDPTEs referenced by the guest CR3.
+         */
+        rc = iemVmxVmentryCheckGuestPdptesForCr3(pVCpu, pszInstr, pVmcs);
+    }
+    else
+        rc = VINF_SUCCESS;
+    return rc;
+}
+
+
+/**
+ * Checks guest-state as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+IEM_STATIC int iemVmxVmentryCheckGuestState(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    int rc = iemVmxVmentryCheckGuestControlRegsMsrs(pVCpu, pszInstr);
+    if (RT_SUCCESS(rc))
+    {
+        rc = iemVmxVmentryCheckGuestSegRegs(pVCpu, pszInstr);
+        if (RT_SUCCESS(rc))
+        {
+            rc = iemVmxVmentryCheckGuestGdtrIdtr(pVCpu, pszInstr);
+            if (RT_SUCCESS(rc))
+            {
+                rc = iemVmxVmentryCheckGuestRipRFlags(pVCpu, pszInstr);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = iemVmxVmentryCheckGuestNonRegState(pVCpu, pszInstr);
+                    if (RT_SUCCESS(rc))
+                        return iemVmxVmentryCheckGuestPdptes(pVCpu, pszInstr);
+                }
+            }
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Checks host-state as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+IEM_STATIC int iemVmxVmentryCheckHostState(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    /*
+     * Host Control Registers and MSRs.
+     * See Intel spec. 26.2.2 "Checks on Host Control Registers and MSRs".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    const char * const pszFailure = "VMFail";
+
+    /* CR0 reserved bits. */
+    {
+        /* CR0 MB1 bits. */
+        uint64_t const u64Cr0Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed0;
+        if ((pVmcs->u64HostCr0.u & u64Cr0Fixed0) == u64Cr0Fixed0)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr0Fixed0);
+
+        /* CR0 MBZ bits. */
+        uint64_t const u64Cr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1;
+        if (!(pVmcs->u64HostCr0.u & ~u64Cr0Fixed1))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr0Fixed1);
+    }
+
+    /* CR4 reserved bits. */
+    {
+        /* CR4 MB1 bits. */
+        uint64_t const u64Cr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0;
+        if ((pVmcs->u64HostCr4.u & u64Cr4Fixed0) == u64Cr4Fixed0)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr4Fixed0);
+
+        /* CR4 MBZ bits. */
+        uint64_t const u64Cr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1;
+        if (!(pVmcs->u64HostCr4.u & ~u64Cr4Fixed1))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr4Fixed1);
+    }
+
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        /* CR3 reserved bits. */
+        if (!(pVmcs->u64HostCr3.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxPhysAddrWidth))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr3);
+
+        /* SYSENTER ESP and SYSENTER EIP. */
+        if (   X86_IS_CANONICAL(pVmcs->u64HostSysenterEsp.u)
+            && X86_IS_CANONICAL(pVmcs->u64HostSysenterEip.u))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostSysenterEspEip);
+    }
+
+    /* We don't support IA32_PERF_GLOBAL_CTRL MSR yet. */
+    Assert(!(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_PERF_MSR));
+
+    /* PAT MSR. */
+    if (   !(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_PAT_MSR)
+        ||  CPUMIsPatMsrValid(pVmcs->u64HostPatMsr.u))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostPatMsr);
+
+    /* EFER MSR. */
+    uint64_t const uValidEferMask = CPUMGetGuestEferMsrValidMask(pVCpu->CTX_SUFF(pVM));
+    if (   !(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
+        || !(pVmcs->u64HostEferMsr.u & ~uValidEferMask))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostEferMsrRsvd);
+
+    bool const fHostInLongMode = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE);
+    bool const fHostLma        = RT_BOOL(pVmcs->u64HostEferMsr.u & MSR_K6_EFER_LMA);
+    bool const fHostLme        = RT_BOOL(pVmcs->u64HostEferMsr.u & MSR_K6_EFER_LME);
+    if (   fHostInLongMode == fHostLma
+        && fHostInLongMode == fHostLme)
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostEferMsr);
+
+    /*
+     * Host Segment and Descriptor-Table Registers.
+     * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
+     */
+    /* Selector RPL and TI. */
+    if (   !(pVmcs->HostCs & (X86_SEL_RPL | X86_SEL_LDT))
+        && !(pVmcs->HostSs & (X86_SEL_RPL | X86_SEL_LDT))
+        && !(pVmcs->HostDs & (X86_SEL_RPL | X86_SEL_LDT))
+        && !(pVmcs->HostEs & (X86_SEL_RPL | X86_SEL_LDT))
+        && !(pVmcs->HostFs & (X86_SEL_RPL | X86_SEL_LDT))
+        && !(pVmcs->HostGs & (X86_SEL_RPL | X86_SEL_LDT))
+        && !(pVmcs->HostTr & (X86_SEL_RPL | X86_SEL_LDT)))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostSel);
+
+    /* CS and TR selectors cannot be 0. */
+    if (   pVmcs->HostCs
+        && pVmcs->HostTr)
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCsTr);
+
+    /* SS cannot be 0 if 32-bit host. */
+    if (   fHostInLongMode
+        || pVmcs->HostSs)
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostSs);
+
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        /* FS, GS, GDTR, IDTR, TR base address. */
+        if (   X86_IS_CANONICAL(pVmcs->u64HostFsBase.u)
+            && X86_IS_CANONICAL(pVmcs->u64HostFsBase.u)
+            && X86_IS_CANONICAL(pVmcs->u64HostGdtrBase.u)
+            && X86_IS_CANONICAL(pVmcs->u64HostIdtrBase.u)
+            && X86_IS_CANONICAL(pVmcs->u64HostTrBase.u))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostSegBase);
+    }
+
+    /*
+     * Host address-space size for 64-bit CPUs.
+     * See Intel spec. 26.2.4 "Checks Related to Address-Space Size".
+     */
+    bool const fGstInLongMode = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        bool const fCpuInLongMode = CPUMIsGuestInLongMode(pVCpu);
+
+        /* Logical processor in IA-32e mode. */
+        if (fCpuInLongMode)
+        {
+            if (fHostInLongMode)
+            {
+                /* PAE must be set. */
+                if (pVmcs->u64HostCr4.u & X86_CR4_PAE)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr4Pae);
+
+                /* RIP must be canonical. */
+                if (X86_IS_CANONICAL(pVmcs->u64HostRip.u))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostRip);
+            }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostLongMode);
+        }
+        else
+        {
+            /* Logical processor is outside IA-32e mode. */
+            if (   !fGstInLongMode
+                && !fHostInLongMode)
+            {
+                /* PCIDE should not be set. */
+                if (!(pVmcs->u64HostCr4.u & X86_CR4_PCIDE))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr4Pcide);
+
+                /* The high 32-bits of RIP MBZ. */
+                if (!pVmcs->u64HostRip.s.Hi)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostRipRsvd);
+            }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostGuestLongMode);
+        }
+    }
+    else
+    {
+        /* Host address-space size for 32-bit CPUs. */
+        if (   !fGstInLongMode
+            && !fHostInLongMode)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostGuestLongModeNoCpu);
+    }
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks VMCS controls fields as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ *
+ * @remarks This may update secondary-processor based VM-execution control fields
+ *          in the current VMCS if necessary.
+ */
+IEM_STATIC int iemVmxVmentryCheckCtls(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    const char * const pszFailure = "VMFail";
+
+    /*
+     * VM-execution controls.
+     * See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
+     */
+    {
+        /* Pin-based VM-execution controls. */
+        {
+            VMXCTLSMSR const PinCtls = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.PinCtls;
+            if (!(~pVmcs->u32PinCtls & PinCtls.n.allowed0))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_PinCtlsDisallowed0);
+
+            if (!(pVmcs->u32PinCtls & ~PinCtls.n.allowed1))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_PinCtlsAllowed1);
+        }
+
+        /* Processor-based VM-execution controls. */
+        {
+            VMXCTLSMSR const ProcCtls = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.ProcCtls;
+            if (!(~pVmcs->u32ProcCtls & ProcCtls.n.allowed0))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ProcCtlsDisallowed0);
+
+            if (!(pVmcs->u32ProcCtls & ~ProcCtls.n.allowed1))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ProcCtlsAllowed1);
+        }
+
+        /* Secondary processor-based VM-execution controls. */
+        if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
+        {
+            VMXCTLSMSR const ProcCtls2 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.ProcCtls2;
+            if (!(~pVmcs->u32ProcCtls2 & ProcCtls2.n.allowed0))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ProcCtls2Disallowed0);
+
+            if (!(pVmcs->u32ProcCtls2 & ~ProcCtls2.n.allowed1))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ProcCtls2Allowed1);
+        }
+        else
+            Assert(!pVmcs->u32ProcCtls2);
+
+        /* CR3-target count. */
+        if (pVmcs->u32Cr3TargetCount <= VMX_V_CR3_TARGET_COUNT)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_Cr3TargetCount);
+
+        /* I/O bitmaps physical addresses. */
+        if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS)
+        {
+            RTGCPHYS const GCPhysIoBitmapA = pVmcs->u64AddrIoBitmapA.u;
+            if (   !(GCPhysIoBitmapA & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysIoBitmapA >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysIoBitmapA))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrIoBitmapA);
+
+            RTGCPHYS const GCPhysIoBitmapB = pVmcs->u64AddrIoBitmapB.u;
+            if (   !(GCPhysIoBitmapB & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysIoBitmapB >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysIoBitmapB))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrIoBitmapB);
+        }
+
+        /* MSR bitmap physical address. */
+        if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+        {
+            RTGCPHYS const GCPhysMsrBitmap = pVmcs->u64AddrMsrBitmap.u;
+            if (   !(GCPhysMsrBitmap & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysMsrBitmap >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysMsrBitmap))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrMsrBitmap);
+        }
+
+        /* TPR shadow related controls. */
+        if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+        {
+            /* Virtual-APIC page physical address. */
+            RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+            if (   !(GCPhysVirtApic & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysVirtApic >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrVirtApicPage);
+
+            /* TPR threshold bits 31:4 MBZ without virtual-interrupt delivery. */
+            if (   !(pVmcs->u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)
+                ||  (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_TprThresholdRsvd);
+
+            /* The rest done XXX document */
+        }
+        else
+        {
+            if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE)
+                && !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT)
+                && !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY))
+            { /* likely */ }
+            else
+            {
+                if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE)
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtX2ApicTprShadow);
+                if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT)
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ApicRegVirt);
+                Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtIntDelivery);
+            }
+        }
+
+        /* NMI exiting and virtual-NMIs. */
+        if (    (pVmcs->u32PinCtls & VMX_PIN_CTLS_NMI_EXIT)
+            || !(pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtNmi);
+
+        /* Virtual-NMIs and NMI-window exiting. */
+        if (    (pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+            || !(pVmcs->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_NmiWindowExit);
+
+        /* Virtualize APIC accesses. */
+        if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
+        {
+            /* APIC-access physical address. */
+            RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u;
+            if (   !(GCPhysApicAccess & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysApicAccess >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrApicAccess);
+
+            /*
+             * Disallow APIC-access page and virtual-APIC page from being the same address.
+             * Note! This is not an Intel requirement, but one imposed by our implementation.
+             */
+            /** @todo r=ramshankar: This is done primarily to simplify recursion scenarios while
+             *        redirecting accesses between the APIC-access page and the virtual-APIC
+             *        page. If any nested hypervisor requires this, we can implement it later. */
+            if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+            {
+                RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+                if (GCPhysVirtApic != GCPhysApicAccess)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrApicAccessEqVirtApic);
+            }
+        }
+
+        /* Virtualize-x2APIC mode is mutually exclusive with virtualize-APIC accesses. */
+        if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE)
+            || !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtX2ApicVirtApic);
+
+        /* Virtual-interrupt delivery requires external interrupt exiting. */
+        if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)
+            ||  (pVmcs->u32PinCtls & VMX_PIN_CTLS_EXT_INT_EXIT))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtX2ApicVirtApic);
+
+        /* VPID. */
+        if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VPID)
+            || pVmcs->u16Vpid != 0)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_Vpid);
+
+        Assert(!(pVmcs->u32PinCtls & VMX_PIN_CTLS_POSTED_INT));             /* We don't support posted interrupts yet. */
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT));                /* We don't support EPT yet. */
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_PML));                /* We don't support PML yet. */
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST)); /* We don't support Unrestricted-guests yet. */
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VMFUNC));             /* We don't support VM functions yet. */
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT_VE));             /* We don't support EPT-violation #VE yet. */
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT));    /* We don't support Pause-loop exiting yet. */
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_TSC_SCALING));        /* We don't support TSC-scaling yet. */
+
+        /* VMCS shadowing. */
+        if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
+        {
+            /* VMREAD-bitmap physical address. */
+            RTGCPHYS const GCPhysVmreadBitmap = pVmcs->u64AddrVmreadBitmap.u;
+            if (   !(GCPhysVmreadBitmap & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysVmreadBitmap >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmreadBitmap))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrVmreadBitmap);
+
+            /* VMWRITE-bitmap physical address. */
+            RTGCPHYS const GCPhysVmwriteBitmap = pVmcs->u64AddrVmreadBitmap.u;
+            if (   !(GCPhysVmwriteBitmap & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysVmwriteBitmap >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmwriteBitmap))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrVmwriteBitmap);
+        }
+    }
+
+    /*
+     * VM-exit controls.
+     * See Intel spec. 26.2.1.2 "VM-Exit Control Fields".
+     */
+    {
+        VMXCTLSMSR const ExitCtls = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.ExitCtls;
+        if (!(~pVmcs->u32ExitCtls & ExitCtls.n.allowed0))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ExitCtlsDisallowed0);
+
+        if (!(pVmcs->u32ExitCtls & ~ExitCtls.n.allowed1))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ExitCtlsAllowed1);
+
+        /* Save preemption timer without activating it. */
+        if (    (pVmcs->u32PinCtls & VMX_PIN_CTLS_PREEMPT_TIMER)
+            || !(pVmcs->u32ProcCtls & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_SavePreemptTimer);
+
+        /* VM-exit MSR-store count and VM-exit MSR-store area address. */
+        if (pVmcs->u32ExitMsrStoreCount)
+        {
+            if (   !(pVmcs->u64AddrExitMsrStore.u & VMX_AUTOMSR_OFFSET_MASK)
+                && !(pVmcs->u64AddrExitMsrStore.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), pVmcs->u64AddrExitMsrStore.u))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrExitMsrStore);
+        }
+
+        /* VM-exit MSR-load count and VM-exit MSR-load area address. */
+        if (pVmcs->u32ExitMsrLoadCount)
+        {
+            if (   !(pVmcs->u64AddrExitMsrLoad.u & VMX_AUTOMSR_OFFSET_MASK)
+                && !(pVmcs->u64AddrExitMsrLoad.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), pVmcs->u64AddrExitMsrLoad.u))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrExitMsrLoad);
+        }
+    }
+
+    /*
+     * VM-entry controls.
+     * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
+     */
+    {
+        VMXCTLSMSR const EntryCtls = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.EntryCtls;
+        if (!(~pVmcs->u32EntryCtls & EntryCtls.n.allowed0))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryCtlsDisallowed0);
+
+        if (!(pVmcs->u32EntryCtls & ~EntryCtls.n.allowed1))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryCtlsAllowed1);
+
+        /* Event injection. */
+        uint32_t const uIntInfo = pVmcs->u32EntryIntInfo;
+        if (RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_VALID))
+        {
+            /* Type and vector. */
+            uint8_t const uType   = RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_TYPE);
+            uint8_t const uVector = RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_VECTOR);
+            uint8_t const uRsvd   = RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_RSVD_12_30);
+            if (   !uRsvd
+                && VMXIsEntryIntInfoTypeValid(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxMonitorTrapFlag, uType)
+                && VMXIsEntryIntInfoVectorValid(uVector, uType))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryIntInfoTypeVecRsvd);
+
+            /* Exception error code. */
+            if (RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID))
+            {
+                /* Delivery possible only in Unrestricted-guest mode when CR0.PE is set. */
+                if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST)
+                    ||  (pVmcs->u64GuestCr0.s.Lo & X86_CR0_PE))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryIntInfoErrCodePe);
+
+                /* Exceptions that provide an error code. */
+                if (   uType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
+                    && (   uVector == X86_XCPT_DF
+                        || uVector == X86_XCPT_TS
+                        || uVector == X86_XCPT_NP
+                        || uVector == X86_XCPT_SS
+                        || uVector == X86_XCPT_GP
+                        || uVector == X86_XCPT_PF
+                        || uVector == X86_XCPT_AC))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryIntInfoErrCodeVec);
+
+                /* Exception error-code reserved bits. */
+                if (!(pVmcs->u32EntryXcptErrCode & ~VMX_ENTRY_INT_XCPT_ERR_CODE_VALID_MASK))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryXcptErrCodeRsvd);
+
+                /* Injecting a software interrupt, software exception or privileged software exception. */
+                if (   uType == VMX_ENTRY_INT_INFO_TYPE_SW_INT
+                    || uType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT
+                    || uType == VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT)
+                {
+                    /* Instruction length must be in the range 0-15. */
+                    if (pVmcs->u32EntryInstrLen <= VMX_ENTRY_INSTR_LEN_MAX)
+                    { /* likely */ }
+                    else
+                        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryInstrLen);
+
+                    /* However, instruction length of 0 is allowed only when its CPU feature is present. */
+                    if (   pVmcs->u32EntryInstrLen != 0
+                        || IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxEntryInjectSoftInt)
+                    { /* likely */ }
+                    else
+                        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryInstrLenZero);
+                }
+            }
+        }
+
+        /* VM-entry MSR-load count and VM-entry MSR-load area address. */
+        if (pVmcs->u32EntryMsrLoadCount)
+        {
+            if (   !(pVmcs->u64AddrEntryMsrLoad.u & VMX_AUTOMSR_OFFSET_MASK)
+                && !(pVmcs->u64AddrEntryMsrLoad.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), pVmcs->u64AddrEntryMsrLoad.u))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrEntryMsrLoad);
+        }
+
+        Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM));           /* We don't support SMM yet. */
+        Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON));    /* We don't support dual-monitor treatment yet. */
+    }
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads the guest control registers, debug register and some MSRs as part of
+ * VM-entry.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC void iemVmxVmentryLoadGuestControlRegsMsrs(PVMCPUCC pVCpu)
+{
+    /*
+     * Load guest control registers, debug registers and MSRs.
+     * See Intel spec. 26.3.2.1 "Loading Guest Control Registers, Debug Registers and MSRs".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+    uint64_t const uGstCr0 = (pVmcs->u64GuestCr0.u   & ~VMX_ENTRY_GUEST_CR0_IGNORE_MASK)
+                           | (pVCpu->cpum.GstCtx.cr0 &  VMX_ENTRY_GUEST_CR0_IGNORE_MASK);
+    CPUMSetGuestCR0(pVCpu, uGstCr0);
+    CPUMSetGuestCR4(pVCpu, pVmcs->u64GuestCr4.u);
+    pVCpu->cpum.GstCtx.cr3 = pVmcs->u64GuestCr3.u;
+
+    if (pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
+        pVCpu->cpum.GstCtx.dr[7] = (pVmcs->u64GuestDr7.u & ~VMX_ENTRY_GUEST_DR7_MBZ_MASK) | VMX_ENTRY_GUEST_DR7_MB1_MASK;
+
+    pVCpu->cpum.GstCtx.SysEnter.eip = pVmcs->u64GuestSysenterEip.s.Lo;
+    pVCpu->cpum.GstCtx.SysEnter.esp = pVmcs->u64GuestSysenterEsp.s.Lo;
+    pVCpu->cpum.GstCtx.SysEnter.cs  = pVmcs->u32GuestSysenterCS;
+
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        /* FS base and GS base are loaded  while loading the rest of the guest segment registers. */
+
+        /* EFER MSR. */
+        if (!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR))
+        {
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
+            uint64_t const uHostEfer      = pVCpu->cpum.GstCtx.msrEFER;
+            bool const     fGstInLongMode = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+            bool const     fGstPaging     = RT_BOOL(uGstCr0 & X86_CR0_PG);
+            if (fGstInLongMode)
+            {
+                /* If the nested-guest is in long mode, LMA and LME are both set. */
+                Assert(fGstPaging);
+                pVCpu->cpum.GstCtx.msrEFER = uHostEfer | (MSR_K6_EFER_LMA | MSR_K6_EFER_LME);
+            }
+            else
+            {
+                /*
+                 * If the nested-guest is outside long mode:
+                 *   - With paging:    LMA is cleared, LME is cleared.
+                 *   - Without paging: LMA is cleared, LME is left unmodified.
+                 */
+                uint64_t const fLmaLmeMask = MSR_K6_EFER_LMA | (fGstPaging ? MSR_K6_EFER_LME : 0);
+                pVCpu->cpum.GstCtx.msrEFER = uHostEfer & ~fLmaLmeMask;
+            }
+        }
+        /* else: see below. */
+    }
+
+    /* PAT MSR. */
+    if (pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
+        pVCpu->cpum.GstCtx.msrPAT = pVmcs->u64GuestPatMsr.u;
+
+    /* EFER MSR. */
+    if (pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
+        pVCpu->cpum.GstCtx.msrEFER = pVmcs->u64GuestEferMsr.u;
+
+    /* We don't support IA32_PERF_GLOBAL_CTRL MSR yet. */
+    Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR));
+
+    /* We don't support IA32_BNDCFGS MSR yet. */
+    Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_BNDCFGS_MSR));
+
+    /* Nothing to do for SMBASE register - We don't support SMM yet. */
+}
+
+
+/**
+ * Loads the guest segment registers, GDTR, IDTR, LDTR and TR as part of VM-entry.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC void iemVmxVmentryLoadGuestSegRegs(PVMCPUCC pVCpu)
+{
+    /*
+     * Load guest segment registers, GDTR, IDTR, LDTR and TR.
+     * See Intel spec. 26.3.2.2 "Loading Guest Segment Registers and Descriptor-Table Registers".
+     */
+    /* CS, SS, ES, DS, FS, GS. */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    for (unsigned iSegReg = 0; iSegReg < X86_SREG_COUNT; iSegReg++)
+    {
+        PCPUMSELREG pGstSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
+        CPUMSELREG  VmcsSelReg;
+        int rc = iemVmxVmcsGetGuestSegReg(pVmcs, iSegReg, &VmcsSelReg);
+        AssertRC(rc); NOREF(rc);
+        if (!(VmcsSelReg.Attr.u & X86DESCATTR_UNUSABLE))
+        {
+            pGstSelReg->Sel      = VmcsSelReg.Sel;
+            pGstSelReg->ValidSel = VmcsSelReg.Sel;
+            pGstSelReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+            pGstSelReg->u64Base  = VmcsSelReg.u64Base;
+            pGstSelReg->u32Limit = VmcsSelReg.u32Limit;
+            pGstSelReg->Attr.u   = VmcsSelReg.Attr.u;
+        }
+        else
+        {
+            pGstSelReg->Sel      = VmcsSelReg.Sel;
+            pGstSelReg->ValidSel = VmcsSelReg.Sel;
+            pGstSelReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+            switch (iSegReg)
+            {
+                case X86_SREG_CS:
+                    pGstSelReg->u64Base  = VmcsSelReg.u64Base;
+                    pGstSelReg->u32Limit = VmcsSelReg.u32Limit;
+                    pGstSelReg->Attr.u   = VmcsSelReg.Attr.u;
+                    break;
+
+                case X86_SREG_SS:
+                    pGstSelReg->u64Base  = VmcsSelReg.u64Base & UINT32_C(0xfffffff0);
+                    pGstSelReg->u32Limit = 0;
+                    pGstSelReg->Attr.u   = (VmcsSelReg.Attr.u & X86DESCATTR_DPL) | X86DESCATTR_D | X86DESCATTR_UNUSABLE;
+                    break;
+
+                case X86_SREG_ES:
+                case X86_SREG_DS:
+                    pGstSelReg->u64Base  = 0;
+                    pGstSelReg->u32Limit = 0;
+                    pGstSelReg->Attr.u   = X86DESCATTR_UNUSABLE;
+                    break;
+
+                case X86_SREG_FS:
+                case X86_SREG_GS:
+                    pGstSelReg->u64Base  = VmcsSelReg.u64Base;
+                    pGstSelReg->u32Limit = 0;
+                    pGstSelReg->Attr.u   = X86DESCATTR_UNUSABLE;
+                    break;
+            }
+            Assert(pGstSelReg->Attr.n.u1Unusable);
+        }
+    }
+
+    /* LDTR. */
+    pVCpu->cpum.GstCtx.ldtr.Sel      = pVmcs->GuestLdtr;
+    pVCpu->cpum.GstCtx.ldtr.ValidSel = pVmcs->GuestLdtr;
+    pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    if (!(pVmcs->u32GuestLdtrAttr & X86DESCATTR_UNUSABLE))
+    {
+        pVCpu->cpum.GstCtx.ldtr.u64Base  = pVmcs->u64GuestLdtrBase.u;
+        pVCpu->cpum.GstCtx.ldtr.u32Limit = pVmcs->u32GuestLdtrLimit;
+        pVCpu->cpum.GstCtx.ldtr.Attr.u   = pVmcs->u32GuestLdtrAttr;
+    }
+    else
+    {
+        pVCpu->cpum.GstCtx.ldtr.u64Base  = 0;
+        pVCpu->cpum.GstCtx.ldtr.u32Limit = 0;
+        pVCpu->cpum.GstCtx.ldtr.Attr.u   = X86DESCATTR_UNUSABLE;
+    }
+
+    /* TR. */
+    Assert(!(pVmcs->u32GuestTrAttr & X86DESCATTR_UNUSABLE));
+    pVCpu->cpum.GstCtx.tr.Sel      = pVmcs->GuestTr;
+    pVCpu->cpum.GstCtx.tr.ValidSel = pVmcs->GuestTr;
+    pVCpu->cpum.GstCtx.tr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.tr.u64Base  = pVmcs->u64GuestTrBase.u;
+    pVCpu->cpum.GstCtx.tr.u32Limit = pVmcs->u32GuestTrLimit;
+    pVCpu->cpum.GstCtx.tr.Attr.u   = pVmcs->u32GuestTrAttr;
+
+    /* GDTR. */
+    pVCpu->cpum.GstCtx.gdtr.cbGdt = pVmcs->u32GuestGdtrLimit;
+    pVCpu->cpum.GstCtx.gdtr.pGdt  = pVmcs->u64GuestGdtrBase.u;
+
+    /* IDTR. */
+    pVCpu->cpum.GstCtx.idtr.cbIdt = pVmcs->u32GuestIdtrLimit;
+    pVCpu->cpum.GstCtx.idtr.pIdt  = pVmcs->u64GuestIdtrBase.u;
+}
+
+
+/**
+ * Loads the guest MSRs from the VM-entry MSR-load area as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+IEM_STATIC int iemVmxVmentryLoadGuestAutoMsrs(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    /*
+     * Load guest MSRs.
+     * See Intel spec. 26.4 "Loading MSRs".
+     */
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    const char *const pszFailure = "VM-exit";
+
+    /*
+     * The VM-entry MSR-load area address need not be a valid guest-physical address if the
+     * VM-entry MSR load count is 0. If this is the case, bail early without reading it.
+     * See Intel spec. 24.8.2 "VM-Entry Controls for MSRs".
+     */
+    uint32_t const cMsrs = pVmcs->u32EntryMsrLoadCount;
+    if (!cMsrs)
+        return VINF_SUCCESS;
+
+    /*
+     * Verify the MSR auto-load count. Physical CPUs can behave unpredictably if the count is
+     * exceeded including possibly raising #MC exceptions during VMX transition. Our
+     * implementation shall fail VM-entry with an VMX_EXIT_ERR_MSR_LOAD VM-exit.
+     */
+    bool const fIsMsrCountValid = iemVmxIsAutoMsrCountValid(pVCpu, cMsrs);
+    if (fIsMsrCountValid)
+    { /* likely */ }
+    else
+    {
+        iemVmxVmcsSetExitQual(pVCpu, VMX_V_AUTOMSR_AREA_SIZE / sizeof(VMXAUTOMSR));
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_MsrLoadCount);
+    }
+
+    RTGCPHYS const GCPhysVmEntryMsrLoadArea = pVmcs->u64AddrEntryMsrLoad.u;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), (void *)pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pEntryMsrLoadArea),
+                                     GCPhysVmEntryMsrLoadArea, cMsrs * sizeof(VMXAUTOMSR));
+    if (RT_SUCCESS(rc))
+    {
+        PCVMXAUTOMSR pMsr = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pEntryMsrLoadArea);
+        Assert(pMsr);
+        for (uint32_t idxMsr = 0; idxMsr < cMsrs; idxMsr++, pMsr++)
+        {
+            if (   !pMsr->u32Reserved
+                &&  pMsr->u32Msr != MSR_K8_FS_BASE
+                &&  pMsr->u32Msr != MSR_K8_GS_BASE
+                &&  pMsr->u32Msr != MSR_K6_EFER
+                &&  pMsr->u32Msr != MSR_IA32_SMM_MONITOR_CTL
+                &&  pMsr->u32Msr >> 8 != MSR_IA32_X2APIC_START >> 8)
+            {
+                VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, pMsr->u32Msr, pMsr->u64Value);
+                if (rcStrict == VINF_SUCCESS)
+                    continue;
+
+                /*
+                 * If we're in ring-0, we cannot handle returns to ring-3 at this point and continue VM-entry.
+                 * If any nested hypervisor loads MSRs that require ring-3 handling, we cause a VM-entry failure
+                 * recording the MSR index in the Exit qualification (as per the Intel spec.) and indicated
+                 * further by our own, specific diagnostic code. Later, we can try implement handling of the
+                 * MSR in ring-0 if possible, or come up with a better, generic solution.
+                 */
+                iemVmxVmcsSetExitQual(pVCpu, idxMsr);
+                VMXVDIAG const enmDiag = rcStrict == VINF_CPUM_R3_MSR_WRITE
+                                       ? kVmxVDiag_Vmentry_MsrLoadRing3
+                                       : kVmxVDiag_Vmentry_MsrLoad;
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+            else
+            {
+                iemVmxVmcsSetExitQual(pVCpu, idxMsr);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_MsrLoadRsvd);
+            }
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("%s: Failed to read MSR auto-load area at %#RGp, rc=%Rrc\n", pszInstr, GCPhysVmEntryMsrLoadArea, rc));
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_MsrLoadPtrReadPhys);
+    }
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads the guest-state non-register state as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks This must be called only after loading the nested-guest register state
+ *          (especially nested-guest RIP).
+ */
+IEM_STATIC void iemVmxVmentryLoadGuestNonRegState(PVMCPUCC pVCpu)
+{
+    /*
+     * Load guest non-register state.
+     * See Intel spec. 26.6 "Special Features of VM Entry"
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+
+    /*
+     * If VM-entry is not vectoring, block-by-STI and block-by-MovSS state must be loaded.
+     * If VM-entry is vectoring, there is no block-by-STI or block-by-MovSS.
+     *
+     * See Intel spec. 26.6.1 "Interruptibility State".
+     */
+    bool const fEntryVectoring = VMXIsVmentryVectoring(pVmcs->u32EntryIntInfo, NULL /* puEntryIntInfoType */);
+    if (   !fEntryVectoring
+        && (pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)))
+        EMSetInhibitInterruptsPC(pVCpu, pVmcs->u64GuestRip.u);
+    else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+
+    /* NMI blocking. */
+    if (pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
+    {
+        if (pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+            pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = true;
+        else
+        {
+            pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = false;
+            if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
+        }
+    }
+    else
+        pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = false;
+
+    /* SMI blocking is irrelevant. We don't support SMIs yet. */
+
+    /* Loading PDPTEs will be taken care when we switch modes. We don't support EPT yet. */
+    Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT));
+
+    /* VPID is irrelevant. We don't support VPID yet. */
+
+    /* Clear address-range monitoring. */
+    EMMonitorWaitClear(pVCpu);
+}
+
+
+/**
+ * Loads the guest VMCS referenced state (such as MSR bitmaps, I/O bitmaps etc).
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ *
+ * @remarks This assumes various VMCS related data structure pointers have already
+ *          been verified prior to calling this function.
+ */
+IEM_STATIC int iemVmxVmentryLoadGuestVmcsRefState(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    const char *const pszFailure  = "VM-exit";
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+
+    /*
+     * Virtualize APIC accesses.
+     */
+    if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
+    {
+        /* APIC-access physical address. */
+        RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u;
+
+        /*
+         * Register the handler for the APIC-access page.
+         *
+         * We don't deregister the APIC-access page handler during the VM-exit as a different
+         * nested-VCPU might be using the same guest-physical address for its APIC-access page.
+         *
+         * We leave the page registered until the first access that happens outside VMX non-root
+         * mode. Guest software is allowed to access structures such as the APIC-access page
+         * only when no logical processor with a current VMCS references it in VMX non-root mode,
+         * otherwise it can lead to unpredictable behavior including guest triple-faults.
+         *
+         * See Intel spec. 24.11.4 "Software Access to Related Structures".
+         */
+        if (!PGMHandlerPhysicalIsRegistered(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess))
+        {
+            PVMCC pVM       = pVCpu->CTX_SUFF(pVM);
+            PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
+            int rc = PGMHandlerPhysicalRegister(pVM, GCPhysApicAccess, GCPhysApicAccess + X86_PAGE_4K_SIZE - 1,
+                                                pVCpu0->iem.s.hVmxApicAccessPage, NIL_RTR3PTR /* pvUserR3 */,
+                                                NIL_RTR0PTR /* pvUserR0 */, NIL_RTRCPTR /* pvUserRC */, NULL /* pszDesc */);
+            if (RT_SUCCESS(rc))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrApicAccessHandlerReg);
+        }
+    }
+
+    /*
+     * VMCS shadowing.
+     */
+    if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
+    {
+        /* Read the VMREAD-bitmap. */
+        RTGCPHYS const GCPhysVmreadBitmap = pVmcs->u64AddrVmreadBitmap.u;
+        Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvVmreadBitmap));
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvVmreadBitmap),
+                                         GCPhysVmreadBitmap, VMX_V_VMREAD_VMWRITE_BITMAP_SIZE);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmreadBitmapPtrReadPhys);
+
+        /* Read the VMWRITE-bitmap. */
+        RTGCPHYS const GCPhysVmwriteBitmap = pVmcs->u64AddrVmwriteBitmap.u;
+        Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvVmwriteBitmap));
+        rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvVmwriteBitmap),
+                                     GCPhysVmwriteBitmap, VMX_V_VMREAD_VMWRITE_BITMAP_SIZE);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmwriteBitmapPtrReadPhys);
+    }
+
+    /*
+     * I/O bitmaps.
+     */
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS)
+    {
+        /* Read the IO bitmap A. */
+        RTGCPHYS const GCPhysIoBitmapA = pVmcs->u64AddrIoBitmapA.u;
+        Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvIoBitmap));
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvIoBitmap),
+                                         GCPhysIoBitmapA, VMX_V_IO_BITMAP_A_SIZE);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_IoBitmapAPtrReadPhys);
+
+        /* Read the IO bitmap B. */
+        RTGCPHYS const GCPhysIoBitmapB = pVmcs->u64AddrIoBitmapB.u;
+        uint8_t *pbIoBitmapB = (uint8_t *)pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvIoBitmap) + VMX_V_IO_BITMAP_A_SIZE;
+        rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbIoBitmapB, GCPhysIoBitmapB, VMX_V_IO_BITMAP_B_SIZE);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_IoBitmapBPtrReadPhys);
+    }
+
+    /*
+     * TPR shadow and Virtual-APIC page.
+     */
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+    {
+        /* Verify TPR threshold and VTPR when both virtualize-APIC accesses and virtual-interrupt delivery aren't used. */
+        if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
+            && !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY))
+        {
+            /* Read the VTPR from the virtual-APIC page. */
+            RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+            uint8_t u8VTpr;
+            int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &u8VTpr, GCPhysVirtApic + XAPIC_OFF_TPR, sizeof(u8VTpr));
+            if (RT_SUCCESS(rc))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtApicPagePtrReadPhys);
+
+            /* Bits 3:0 of the TPR-threshold must not be greater than bits 7:4 of VTPR. */
+            if ((uint8_t)RT_BF_GET(pVmcs->u32TprThreshold, VMX_BF_TPR_THRESHOLD_TPR) <= (u8VTpr & 0xf0))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_TprThresholdVTpr);
+        }
+    }
+
+    /*
+     * VMCS link pointer.
+     */
+    if (pVmcs->u64VmcsLinkPtr.u != UINT64_C(0xffffffffffffffff))
+    {
+        /* Read the VMCS-link pointer from guest memory. */
+        RTGCPHYS const GCPhysShadowVmcs = pVmcs->u64VmcsLinkPtr.u;
+        Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pShadowVmcs));
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pShadowVmcs),
+                                         GCPhysShadowVmcs, VMX_V_SHADOW_VMCS_SIZE);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR);
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmcsLinkPtrReadPhys);
+        }
+
+        /* Verify the VMCS revision specified by the guest matches what we reported to the guest. */
+        if (pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pShadowVmcs)->u32VmcsRevId.n.u31RevisionId == VMX_V_VMCS_REVISION_ID)
+        { /* likely */ }
+        else
+        {
+            iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR);
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmcsLinkPtrRevId);
+        }
+
+        /* Verify the shadow bit is set if VMCS shadowing is enabled . */
+        if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
+            || pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pShadowVmcs)->u32VmcsRevId.n.fIsShadowVmcs)
+        { /* likely */ }
+        else
+        {
+            iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR);
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmcsLinkPtrShadow);
+        }
+
+        /* Update our cache of the guest physical address of the shadow VMCS. */
+        pVCpu->cpum.GstCtx.hwvirt.vmx.GCPhysShadowVmcs = GCPhysShadowVmcs;
+    }
+
+    /*
+     * MSR bitmap.
+     */
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+    {
+        /* Read the MSR bitmap. */
+        RTGCPHYS const GCPhysMsrBitmap = pVmcs->u64AddrMsrBitmap.u;
+        Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap));
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap),
+                                         GCPhysMsrBitmap, VMX_V_MSR_BITMAP_SIZE);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_MsrBitmapPtrReadPhys);
+    }
+
+    NOREF(pszFailure);
+    NOREF(pszInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads the guest-state as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ *
+ * @remarks This must be done after all the necessary steps prior to loading of
+ *          guest-state (e.g. checking various VMCS state).
+ */
+IEM_STATIC int iemVmxVmentryLoadGuestState(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    /* Load guest control registers, MSRs (that are directly part of the VMCS). */
+    iemVmxVmentryLoadGuestControlRegsMsrs(pVCpu);
+
+    /* Load guest segment registers. */
+    iemVmxVmentryLoadGuestSegRegs(pVCpu);
+
+    /*
+     * Load guest RIP, RSP and RFLAGS.
+     * See Intel spec. 26.3.2.3 "Loading Guest RIP, RSP and RFLAGS".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    pVCpu->cpum.GstCtx.rsp      = pVmcs->u64GuestRsp.u;
+    pVCpu->cpum.GstCtx.rip      = pVmcs->u64GuestRip.u;
+    pVCpu->cpum.GstCtx.rflags.u = pVmcs->u64GuestRFlags.u;
+
+    /* Initialize the PAUSE-loop controls as part of VM-entry. */
+    pVCpu->cpum.GstCtx.hwvirt.vmx.uFirstPauseLoopTick = 0;
+    pVCpu->cpum.GstCtx.hwvirt.vmx.uPrevPauseTick      = 0;
+
+    /* Load guest non-register state (such as interrupt shadows, NMI blocking etc). */
+    iemVmxVmentryLoadGuestNonRegState(pVCpu);
+
+    /* Load VMX related structures and state referenced by the VMCS. */
+    int rc = iemVmxVmentryLoadGuestVmcsRefState(pVCpu, pszInstr);
+    if (rc == VINF_SUCCESS)
+    { /* likely */ }
+    else
+        return rc;
+
+    NOREF(pszInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Returns whether there are is a pending debug exception on VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+IEM_STATIC bool iemVmxVmentryIsPendingDebugXcpt(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    /*
+     * Pending debug exceptions.
+     * See Intel spec. 26.6.3 "Delivery of Pending Debug Exceptions after VM Entry".
+     */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    bool fPendingDbgXcpt = RT_BOOL(pVmcs->u64GuestPendingDbgXcpts.u & (  VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS
+                                                                       | VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_EN_BP));
+    if (fPendingDbgXcpt)
+    {
+        uint8_t uEntryIntInfoType;
+        bool const fEntryVectoring = VMXIsVmentryVectoring(pVmcs->u32EntryIntInfo, &uEntryIntInfoType);
+        if (fEntryVectoring)
+        {
+            switch (uEntryIntInfoType)
+            {
+                case VMX_ENTRY_INT_INFO_TYPE_EXT_INT:
+                case VMX_ENTRY_INT_INFO_TYPE_NMI:
+                case VMX_ENTRY_INT_INFO_TYPE_HW_XCPT:
+                case VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT:
+                    fPendingDbgXcpt = false;
+                    break;
+
+                case VMX_ENTRY_INT_INFO_TYPE_SW_XCPT:
+                {
+                    /*
+                     * Whether the pending debug exception for software exceptions other than
+                     * #BP and #OF is delivered after injecting the exception or is discard
+                     * is CPU implementation specific. We will discard them (easier).
+                     */
+                    uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(pVmcs->u32EntryIntInfo);
+                    if (   uVector != X86_XCPT_BP
+                        && uVector != X86_XCPT_OF)
+                        fPendingDbgXcpt = false;
+                    RT_FALL_THRU();
+                }
+                case VMX_ENTRY_INT_INFO_TYPE_SW_INT:
+                {
+                    if (!(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
+                        fPendingDbgXcpt = false;
+                    break;
+                }
+            }
+        }
+        else
+        {
+            /*
+             * When the VM-entry is not vectoring but there is blocking-by-MovSS, whether the
+             * pending debug exception is held pending or is discarded is CPU implementation
+             * specific. We will discard them (easier).
+             */
+            if (pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
+                fPendingDbgXcpt = false;
+
+            /* There's no pending debug exception in the shutdown or wait-for-SIPI state. */
+            if (pVmcs->u32GuestActivityState & (VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN | VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT))
+                fPendingDbgXcpt = false;
+        }
+    }
+
+    NOREF(pszInstr);
+    return fPendingDbgXcpt;
+}
+
+
+/**
+ * Set up the monitor-trap flag (MTF).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+IEM_STATIC void iemVmxVmentrySetupMtf(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
+    {
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_MTF);
+        Log(("%s: Monitor-trap flag set on VM-entry\n", pszInstr));
+    }
+    else
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
+    NOREF(pszInstr);
+}
+
+
+/**
+ * Sets up NMI-window exiting.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+IEM_STATIC void iemVmxVmentrySetupNmiWindow(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
+    {
+        Assert(pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI);
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW);
+        Log(("%s: NMI-window set on VM-entry\n", pszInstr));
+    }
+    else
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW));
+    NOREF(pszInstr);
+}
+
+
+/**
+ * Sets up interrupt-window exiting.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+IEM_STATIC void iemVmxVmentrySetupIntWindow(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
+    {
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW);
+        Log(("%s: Interrupt-window set on VM-entry\n", pszInstr));
+    }
+    else
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW));
+    NOREF(pszInstr);
+}
+
+
+/**
+ * Set up the VMX-preemption timer.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+IEM_STATIC void iemVmxVmentrySetupPreemptTimer(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    if (pVmcs->u32PinCtls & VMX_PIN_CTLS_PREEMPT_TIMER)
+    {
+        /*
+         * If the timer is 0, we must cause a VM-exit before executing the first
+         * nested-guest instruction. So we can flag as though the timer has already
+         * expired and we will check and cause a VM-exit at the right priority elsewhere
+         * in the code.
+         */
+        uint64_t uEntryTick;
+        uint32_t const uPreemptTimer = pVmcs->u32PreemptTimer;
+        if (uPreemptTimer)
+        {
+            int rc = CPUMStartGuestVmxPremptTimer(pVCpu, uPreemptTimer, VMX_V_PREEMPT_TIMER_SHIFT, &uEntryTick);
+            AssertRC(rc);
+            Log(("%s: VM-entry set up VMX-preemption timer at %#RX64\n", pszInstr, uEntryTick));
+        }
+        else
+        {
+            uEntryTick = TMCpuTickGetNoCheck(pVCpu);
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER);
+            Log(("%s: VM-entry set up VMX-preemption timer at %#RX64 to expire immediately!\n", pszInstr, uEntryTick));
+        }
+
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uEntryTick = uEntryTick;
+    }
+    else
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
+
+    NOREF(pszInstr);
+}
+
+
+/**
+ * Injects an event using TRPM given a VM-entry interruption info. and related
+ * fields.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   pszInstr            The VMX instruction name (for logging purposes).
+ * @param   uEntryIntInfo       The VM-entry interruption info.
+ * @param   uErrCode            The error code associated with the event if any.
+ * @param   cbInstr             The VM-entry instruction length (for software
+ *                              interrupts and software exceptions). Pass 0
+ *                              otherwise.
+ * @param   GCPtrFaultAddress   The guest CR2 if this is a \#PF event.
+ */
+IEM_STATIC void iemVmxVmentryInjectTrpmEvent(PVMCPUCC pVCpu, const char *pszInstr, uint32_t uEntryIntInfo, uint32_t uErrCode,
+                                             uint32_t cbInstr, RTGCUINTPTR GCPtrFaultAddress)
+{
+    Assert(VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
+
+    uint8_t const   uType        = VMX_ENTRY_INT_INFO_TYPE(uEntryIntInfo);
+    uint8_t const   uVector      = VMX_ENTRY_INT_INFO_VECTOR(uEntryIntInfo);
+    TRPMEVENT const enmTrpmEvent = HMVmxEventTypeToTrpmEventType(uEntryIntInfo);
+
+    Assert(uType != VMX_ENTRY_INT_INFO_TYPE_OTHER_EVENT);
+
+    int rc = TRPMAssertTrap(pVCpu, uVector, enmTrpmEvent);
+    AssertRC(rc);
+    Log(("%s: Injecting: vector=%#x type=%#x (%s)\n", pszInstr, uVector, uType, VMXGetEntryIntInfoTypeDesc(uType)));
+
+    if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(uEntryIntInfo))
+    {
+        TRPMSetErrorCode(pVCpu, uErrCode);
+        Log(("%s: Injecting: err_code=%#x\n", pszInstr, uErrCode));
+    }
+
+    if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(uEntryIntInfo))
+    {
+        TRPMSetFaultAddress(pVCpu, GCPtrFaultAddress);
+        Log(("%s: Injecting: fault_addr=%RGp\n", pszInstr, GCPtrFaultAddress));
+    }
+    else
+    {
+        if (   uType == VMX_ENTRY_INT_INFO_TYPE_SW_INT
+            || uType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT
+            || uType == VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT)
+        {
+            TRPMSetInstrLength(pVCpu, cbInstr);
+            Log(("%s: Injecting: instr_len=%u\n", pszInstr, cbInstr));
+        }
+    }
+
+    if (VMX_ENTRY_INT_INFO_TYPE(uEntryIntInfo) == VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT)
+    {
+        TRPMSetTrapDueToIcebp(pVCpu);
+        Log(("%s: Injecting: icebp\n", pszInstr));
+    }
+
+    NOREF(pszInstr);
+}
+
+
+/**
+ * Performs event injection (if any) as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+IEM_STATIC void iemVmxVmentryInjectEvent(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+
+    /*
+     * Inject events.
+     * The event that is going to be made pending for injection is not subject to VMX intercepts,
+     * thus we flag ignoring of intercepts. However, recursive exceptions if any during delivery
+     * of the current event -are- subject to intercepts, hence this flag will be flipped during
+     * the actually delivery of this event.
+     *
+     * See Intel spec. 26.5 "Event Injection".
+     */
+    uint32_t const uEntryIntInfo      = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs)->u32EntryIntInfo;
+    bool const     fEntryIntInfoValid = VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo);
+
+    CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, !fEntryIntInfoValid);
+    if (fEntryIntInfoValid)
+    {
+        if (VMX_ENTRY_INT_INFO_TYPE(uEntryIntInfo) == VMX_ENTRY_INT_INFO_TYPE_OTHER_EVENT)
+        {
+            Assert(VMX_ENTRY_INT_INFO_VECTOR(uEntryIntInfo) == VMX_ENTRY_INT_INFO_VECTOR_MTF);
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_MTF);
+        }
+        else
+            iemVmxVmentryInjectTrpmEvent(pVCpu, pszInstr, uEntryIntInfo, pVmcs->u32EntryXcptErrCode, pVmcs->u32EntryInstrLen,
+                                         pVCpu->cpum.GstCtx.cr2);
+
+        /*
+         * We need to clear the VM-entry interruption information field's valid bit on VM-exit.
+         *
+         * However, we do it here on VM-entry as well because while it isn't visible to guest
+         * software until VM-exit, when and if HM looks at the VMCS to continue nested-guest
+         * execution using hardware-assisted VMX, it will not be try to inject the event again.
+         *
+         * See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
+         */
+        pVmcs->u32EntryIntInfo &= ~VMX_ENTRY_INT_INFO_VALID;
+    }
+    else
+    {
+        /*
+         * Inject any pending guest debug exception.
+         * Unlike injecting events, this #DB injection on VM-entry is subject to #DB VMX intercept.
+         * See Intel spec. 26.6.3 "Delivery of Pending Debug Exceptions after VM Entry".
+         */
+        bool const fPendingDbgXcpt = iemVmxVmentryIsPendingDebugXcpt(pVCpu, pszInstr);
+        if (fPendingDbgXcpt)
+        {
+            uint32_t const uDbgXcptInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
+                                        | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,   VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
+                                        | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,  1);
+            iemVmxVmentryInjectTrpmEvent(pVCpu, pszInstr, uDbgXcptInfo, 0 /* uErrCode */, pVmcs->u32EntryInstrLen,
+                                         0 /* GCPtrFaultAddress */);
+        }
+    }
+
+    NOREF(pszInstr);
+}
+
+
+/**
+ * Initializes all read-only VMCS fields as part of VM-entry.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+IEM_STATIC void iemVmxVmentryInitReadOnlyFields(PVMCPUCC pVCpu)
+{
+    /*
+     * Any VMCS field which we do not establish on every VM-exit but may potentially
+     * be used on the VM-exit path of a nested hypervisor -and- is not explicitly
+     * specified to be undefined, needs to be initialized here.
+     *
+     * Thus, it is especially important to clear the Exit qualification field
+     * since it must be zero for VM-exits where it is not used. Similarly, the
+     * VM-exit interruption information field's valid bit needs to be cleared for
+     * the same reasons.
+     */
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+
+    /* 16-bit (none currently). */
+    /* 32-bit. */
+    pVmcs->u32RoVmInstrError        = 0;
+    pVmcs->u32RoExitReason          = 0;
+    pVmcs->u32RoExitIntInfo         = 0;
+    pVmcs->u32RoExitIntErrCode      = 0;
+    pVmcs->u32RoIdtVectoringInfo    = 0;
+    pVmcs->u32RoIdtVectoringErrCode = 0;
+    pVmcs->u32RoExitInstrLen        = 0;
+    pVmcs->u32RoExitInstrInfo       = 0;
+
+    /* 64-bit. */
+    pVmcs->u64RoGuestPhysAddr.u     = 0;
+
+    /* Natural-width. */
+    pVmcs->u64RoExitQual.u          = 0;
+    pVmcs->u64RoIoRcx.u             = 0;
+    pVmcs->u64RoIoRsi.u             = 0;
+    pVmcs->u64RoIoRdi.u             = 0;
+    pVmcs->u64RoIoRip.u             = 0;
+    pVmcs->u64RoGuestLinearAddr.u   = 0;
+}
+
+
+/**
+ * VMLAUNCH/VMRESUME instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   uInstrId    The instruction identity (VMXINSTRID_VMLAUNCH or
+ *                      VMXINSTRID_VMRESUME).
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmlaunchVmresume(PVMCPUCC pVCpu, uint8_t cbInstr, VMXINSTRID uInstrId)
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF3(pVCpu, cbInstr, uInstrId);
+    return VINF_EM_RAW_EMULATE_INSTR;
+# else
+    Assert(   uInstrId == VMXINSTRID_VMLAUNCH
+           || uInstrId == VMXINSTRID_VMRESUME);
+    const char *pszInstr = uInstrId == VMXINSTRID_VMRESUME ? "vmresume" : "vmlaunch";
+
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        return iemVmxVmexitInstr(pVCpu, uInstrId == VMXINSTRID_VMRESUME ? VMX_EXIT_VMRESUME : VMX_EXIT_VMLAUNCH, cbInstr);
+
+    Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
+
+    /*
+     * Basic VM-entry checks.
+     * The order of the CPL, current and shadow VMCS and block-by-MovSS are important.
+     * The checks following that do not have to follow a specific order.
+     *
+     * See Intel spec. 26.1 "Basic VM-entry Checks".
+     */
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("%s: CPL %u -> #GP(0)\n", pszInstr, pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Current VMCS valid. */
+    if (IEM_VMX_HAS_CURRENT_VMCS(pVCpu))
+    { /* likely */ }
+    else
+    {
+        Log(("%s: VMCS pointer %#RGp invalid -> VMFailInvalid\n", pszInstr, IEM_VMX_GET_CURRENT_VMCS(pVCpu)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_PtrInvalid;
+        iemVmxVmFailInvalid(pVCpu);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* Current VMCS is not a shadow VMCS. */
+    if (!pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs)->u32VmcsRevId.n.fIsShadowVmcs)
+    { /* likely */ }
+    else
+    {
+        Log(("%s: VMCS pointer %#RGp is a shadow VMCS -> VMFailInvalid\n", pszInstr, IEM_VMX_GET_CURRENT_VMCS(pVCpu)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_PtrShadowVmcs;
+        iemVmxVmFailInvalid(pVCpu);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /** @todo Distinguish block-by-MovSS from block-by-STI. Currently we
+     *        use block-by-STI here which is not quite correct. */
+    if (   !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+        ||  pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
+    { /* likely */ }
+    else
+    {
+        Log(("%s: VM entry with events blocked by MOV SS -> VMFail\n", pszInstr));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_BlocKMovSS;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMENTRY_BLOCK_MOVSS);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    if (uInstrId == VMXINSTRID_VMLAUNCH)
+    {
+        /* VMLAUNCH with non-clear VMCS. */
+        if (pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs)->fVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR)
+        { /* likely */ }
+        else
+        {
+            Log(("vmlaunch: VMLAUNCH with non-clear VMCS -> VMFail\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_VmcsClear;
+            iemVmxVmFail(pVCpu, VMXINSTRERR_VMLAUNCH_NON_CLEAR_VMCS);
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            return VINF_SUCCESS;
+        }
+    }
+    else
+    {
+        /* VMRESUME with non-launched VMCS. */
+        if (pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs)->fVmcsState == VMX_V_VMCS_LAUNCH_STATE_LAUNCHED)
+        { /* likely */ }
+        else
+        {
+            Log(("vmresume: VMRESUME with non-launched VMCS -> VMFail\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_VmcsLaunch;
+            iemVmxVmFail(pVCpu, VMXINSTRERR_VMRESUME_NON_LAUNCHED_VMCS);
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            return VINF_SUCCESS;
+        }
+    }
+
+    /*
+     * We are allowed to cache VMCS related data structures (such as I/O bitmaps, MSR bitmaps)
+     * while entering VMX non-root mode. We do some of this while checking VM-execution
+     * controls. The nested hypervisor should not make assumptions and cannot expect
+     * predictable behavior if changes to these structures are made in guest memory while
+     * executing in VMX non-root mode. As far as VirtualBox is concerned, the guest cannot
+     * modify them anyway as we cache them in host memory.
+     *
+     * See Intel spec. 24.11.4 "Software Access to Related Structures".
+     */
+    PVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
+
+    int rc = iemVmxVmentryCheckCtls(pVCpu, pszInstr);
+    if (RT_SUCCESS(rc))
+    {
+        rc = iemVmxVmentryCheckHostState(pVCpu, pszInstr);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Initialize read-only VMCS fields before VM-entry since we don't update all of them
+             * for every VM-exit. This needs to be done before invoking a VM-exit (even those
+             * ones that may occur during VM-entry below).
+             */
+            iemVmxVmentryInitReadOnlyFields(pVCpu);
+
+            /*
+             * Blocking of NMIs need to be restored if VM-entry fails due to invalid-guest state.
+             * So we save the VMCPU_FF_BLOCK_NMI force-flag here so we can restore it on
+             * VM-exit when required.
+             * See Intel spec. 26.7 "VM-entry Failures During or After Loading Guest State"
+             */
+            iemVmxVmentrySaveNmiBlockingFF(pVCpu);
+
+            rc = iemVmxVmentryCheckGuestState(pVCpu, pszInstr);
+            if (RT_SUCCESS(rc))
+            {
+                rc = iemVmxVmentryLoadGuestState(pVCpu, pszInstr);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = iemVmxVmentryLoadGuestAutoMsrs(pVCpu, pszInstr);
+                    if (RT_SUCCESS(rc))
+                    {
+                        Assert(rc != VINF_CPUM_R3_MSR_WRITE);
+
+                        /* VMLAUNCH instruction must update the VMCS launch state. */
+                        if (uInstrId == VMXINSTRID_VMLAUNCH)
+                            pVmcs->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_LAUNCHED;
+
+                        /* Perform the VMX transition (PGM updates). */
+                        VBOXSTRICTRC rcStrict = iemVmxWorldSwitch(pVCpu);
+                        if (rcStrict == VINF_SUCCESS)
+                        { /* likely */ }
+                        else if (RT_SUCCESS(rcStrict))
+                        {
+                            Log3(("%s: iemVmxWorldSwitch returns %Rrc -> Setting passup status\n", pszInstr,
+                                  VBOXSTRICTRC_VAL(rcStrict)));
+                            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                        }
+                        else
+                        {
+                            Log3(("%s: iemVmxWorldSwitch failed! rc=%Rrc\n", pszInstr, VBOXSTRICTRC_VAL(rcStrict)));
+                            return rcStrict;
+                        }
+
+                        /* Paranoia. */
+                        Assert(rcStrict == VINF_SUCCESS);
+
+                        /* We've now entered nested-guest execution. */
+                        pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode = true;
+
+                        /*
+                         * The priority of potential VM-exits during VM-entry is important.
+                         * The priorities of VM-exits and events are listed from highest
+                         * to lowest as follows:
+                         *
+                         * 1.  Event injection.
+                         * 2.  Trap on task-switch (T flag set in TSS).
+                         * 3.  TPR below threshold / APIC-write.
+                         * 4.  SMI, INIT.
+                         * 5.  MTF exit.
+                         * 6.  Debug-trap exceptions (EFLAGS.TF), pending debug exceptions.
+                         * 7.  VMX-preemption timer.
+                         * 9.  NMI-window exit.
+                         * 10. NMI injection.
+                         * 11. Interrupt-window exit.
+                         * 12. Virtual-interrupt injection.
+                         * 13. Interrupt injection.
+                         * 14. Process next instruction (fetch, decode, execute).
+                         */
+
+                        /* Setup VMX-preemption timer. */
+                        iemVmxVmentrySetupPreemptTimer(pVCpu, pszInstr);
+
+                        /* Setup monitor-trap flag. */
+                        iemVmxVmentrySetupMtf(pVCpu, pszInstr);
+
+                        /* Setup NMI-window exiting. */
+                        iemVmxVmentrySetupNmiWindow(pVCpu, pszInstr);
+
+                        /* Setup interrupt-window exiting. */
+                        iemVmxVmentrySetupIntWindow(pVCpu, pszInstr);
+
+                        /*
+                         * Inject any event that the nested hypervisor wants to inject.
+                         * Note! We cannot immediately perform the event injection here as we may have
+                         *       pending PGM operations to perform due to switching page tables and/or
+                         *       mode.
+                         */
+                        iemVmxVmentryInjectEvent(pVCpu, pszInstr);
+
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3)
+                        /* Reschedule to IEM-only execution of the nested-guest. */
+                        Log(("%s: Enabling IEM-only EM execution policy!\n", pszInstr));
+                        int rcSched = EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, true);
+                        if (rcSched != VINF_SUCCESS)
+                            iemSetPassUpStatus(pVCpu, rcSched);
+# endif
+
+                        /* Finally, done. */
+                        Log3(("%s: cs:rip=%#04x:%#RX64 cr0=%#RX64 (%#RX64) cr4=%#RX64 (%#RX64) efer=%#RX64\n",
+                              pszInstr, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.cr0,
+                              pVmcs->u64Cr0ReadShadow.u, pVCpu->cpum.GstCtx.cr4, pVmcs->u64Cr4ReadShadow.u,
+                              pVCpu->cpum.GstCtx.msrEFER));
+                        return VINF_SUCCESS;
+                    }
+                    return iemVmxVmexit(pVCpu, VMX_EXIT_ERR_MSR_LOAD | VMX_EXIT_REASON_ENTRY_FAILED,
+                                        pVmcs->u64RoExitQual.u);
+                }
+            }
+            return iemVmxVmexit(pVCpu, VMX_EXIT_ERR_INVALID_GUEST_STATE | VMX_EXIT_REASON_ENTRY_FAILED,
+                                pVmcs->u64RoExitQual.u);
+        }
+
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMENTRY_INVALID_HOST_STATE);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    iemVmxVmFail(pVCpu, VMXINSTRERR_VMENTRY_INVALID_CTLS);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+# endif
+}
+
+
+/**
+ * Checks whether an RDMSR or WRMSR instruction for the given MSR is intercepted
+ * (causes a VM-exit)  or not.
+ *
+ * @returns @c true if the instruction is intercepted, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason (VMX_EXIT_RDMSR or
+ *                          VMX_EXIT_WRMSR).
+ * @param   idMsr           The MSR.
+ */
+IEM_STATIC bool iemVmxIsRdmsrWrmsrInterceptSet(PCVMCPU pVCpu, uint32_t uExitReason, uint32_t idMsr)
+{
+    Assert(IEM_VMX_IS_NON_ROOT_MODE(pVCpu));
+    Assert(   uExitReason == VMX_EXIT_RDMSR
+           || uExitReason == VMX_EXIT_WRMSR);
+
+    /* Consult the MSR bitmap if the feature is supported. */
+    PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcs);
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+    {
+        Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap));
+        uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), idMsr);
+        if (uExitReason == VMX_EXIT_RDMSR)
+            return RT_BOOL(fMsrpm & VMXMSRPM_EXIT_RD);
+        return RT_BOOL(fMsrpm & VMXMSRPM_EXIT_WR);
+    }
+
+    /* Without MSR bitmaps, all MSR accesses are intercepted. */
+    return true;
+}
+
+
+/**
+ * VMREAD instruction execution worker that does not perform any validation checks.
+ *
+ * Callers are expected to have performed the necessary checks and to ensure the
+ * VMREAD will succeed.
+ *
+ * @param   pVmcs           Pointer to the virtual VMCS.
+ * @param   pu64Dst         Where to write the VMCS value.
+ * @param   u64VmcsField    The VMCS field.
+ *
+ * @remarks May be called with interrupts disabled.
+ */
+IEM_STATIC void iemVmxVmreadNoCheck(PCVMXVVMCS pVmcs, uint64_t *pu64Dst, uint64_t u64VmcsField)
+{
+    VMXVMCSFIELD VmcsField;
+    VmcsField.u = u64VmcsField;
+    uint8_t  const uWidth     = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_WIDTH);
+    uint8_t  const uType      = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_TYPE);
+    uint8_t  const uWidthType = (uWidth << 2) | uType;
+    uint8_t  const uIndex     = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_INDEX);
+    Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+    uint16_t const offField   = g_aoffVmcsMap[uWidthType][uIndex];
+    AssertMsg(offField < VMX_V_VMCS_SIZE, ("off=%u field=%#RX64 width=%#x type=%#x index=%#x (%u)\n", offField, u64VmcsField,
+                                           uWidth, uType, uIndex, uIndex));
+    AssertCompile(VMX_V_SHADOW_VMCS_SIZE == VMX_V_VMCS_SIZE);
+
+    /*
+     * Read the VMCS component based on the field's effective width.
+     *
+     * The effective width is 64-bit fields adjusted to 32-bits if the access-type
+     * indicates high bits (little endian).
+     *
+     * Note! The caller is responsible to trim the result and update registers
+     * or memory locations are required. Here we just zero-extend to the largest
+     * type (i.e. 64-bits).
+     */
+    uint8_t const *pbVmcs    = (uint8_t const *)pVmcs;
+    uint8_t const *pbField   = pbVmcs + offField;
+    uint8_t const  uEffWidth = VMXGetVmcsFieldWidthEff(VmcsField.u);
+    switch (uEffWidth)
+    {
+        case VMX_VMCSFIELD_WIDTH_64BIT:
+        case VMX_VMCSFIELD_WIDTH_NATURAL: *pu64Dst = *(uint64_t const *)pbField; break;
+        case VMX_VMCSFIELD_WIDTH_32BIT:   *pu64Dst = *(uint32_t const *)pbField; break;
+        case VMX_VMCSFIELD_WIDTH_16BIT:   *pu64Dst = *(uint16_t const *)pbField; break;
+    }
+}
+
+
+/**
+ * VMREAD common (memory/register) instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   pu64Dst         Where to write the VMCS value (only updated when
+ *                          VINF_SUCCESS is returned).
+ * @param   u64VmcsField    The VMCS field.
+ * @param   pExitInfo       Pointer to the VM-exit information. Optional, can be
+ *                          NULL.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmreadCommon(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t *pu64Dst, uint64_t u64VmcsField,
+                                           PCVMXVEXITINFO pExitInfo)
+{
+    /* Nested-guest intercept. */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, VMX_EXIT_VMREAD, u64VmcsField))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMREAD, VMXINSTRID_VMREAD, cbInstr);
+    }
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("vmread: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmread_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* VMCS pointer in root mode. */
+    if (   !IEM_VMX_IS_ROOT_MODE(pVCpu)
+        ||  IEM_VMX_HAS_CURRENT_VMCS(pVCpu))
+    { /* likely */ }
+    else
+    {
+        Log(("vmread: VMCS pointer %#RGp invalid -> VMFailInvalid\n", IEM_VMX_GET_CURRENT_VMCS(pVCpu)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmread_PtrInvalid;
+        iemVmxVmFailInvalid(pVCpu);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* VMCS-link pointer in non-root mode. */
+    if (   !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        ||  IEM_VMX_HAS_SHADOW_VMCS(pVCpu))
+    { /* likely */ }
+    else
+    {
+        Log(("vmread: VMCS-link pointer %#RGp invalid -> VMFailInvalid\n", IEM_VMX_GET_SHADOW_VMCS(pVCpu)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmread_LinkPtrInvalid;
+        iemVmxVmFailInvalid(pVCpu);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* Supported VMCS field. */
+    if (CPUMIsGuestVmxVmcsFieldValid(pVCpu->CTX_SUFF(pVM), u64VmcsField))
+    { /* likely */ }
+    else
+    {
+        Log(("vmread: VMCS field %#RX64 invalid -> VMFail\n", u64VmcsField));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmread_FieldInvalid;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64VmcsField;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMREAD_INVALID_COMPONENT);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Reading from the current or shadow VMCS.
+     */
+    PCVMXVVMCS pVmcs = !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+                     ? pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs)
+                     : pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pShadowVmcs);
+    Assert(pVmcs);
+    iemVmxVmreadNoCheck(pVmcs, pu64Dst, u64VmcsField);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VMREAD (64-bit register) instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   pu64Dst         Where to store the VMCS field's value.
+ * @param   u64VmcsField    The VMCS field.
+ * @param   pExitInfo       Pointer to the VM-exit information. Optional, can be
+ *                          NULL.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmreadReg64(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t *pu64Dst, uint64_t u64VmcsField,
+                                          PCVMXVEXITINFO pExitInfo)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmreadCommon(pVCpu, cbInstr, pu64Dst, u64VmcsField, pExitInfo);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        iemVmxVmreadSuccess(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    Log(("vmread/reg: iemVmxVmreadCommon failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VMREAD (32-bit register) instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   pu32Dst         Where to store the VMCS field's value.
+ * @param   u32VmcsField    The VMCS field.
+ * @param   pExitInfo       Pointer to the VM-exit information. Optional, can be
+ *                          NULL.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmreadReg32(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t *pu32Dst, uint64_t u32VmcsField,
+                                          PCVMXVEXITINFO pExitInfo)
+{
+    uint64_t u64Dst;
+    VBOXSTRICTRC rcStrict = iemVmxVmreadCommon(pVCpu, cbInstr, &u64Dst, u32VmcsField, pExitInfo);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        *pu32Dst = u64Dst;
+        iemVmxVmreadSuccess(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    Log(("vmread/reg: iemVmxVmreadCommon failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VMREAD (memory) instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   iEffSeg         The effective segment register to use with @a u64Val.
+ *                          Pass UINT8_MAX if it is a register access.
+ * @param   GCPtrDst        The guest linear address to store the VMCS field's
+ *                          value.
+ * @param   u64VmcsField    The VMCS field.
+ * @param   pExitInfo       Pointer to the VM-exit information. Optional, can be
+ *                          NULL.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmreadMem(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDst, uint64_t u64VmcsField,
+                                        PCVMXVEXITINFO pExitInfo)
+{
+    uint64_t u64Dst;
+    VBOXSTRICTRC rcStrict = iemVmxVmreadCommon(pVCpu, cbInstr, &u64Dst, u64VmcsField, pExitInfo);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /*
+         * Write the VMCS field's value to the location specified in guest-memory.
+         */
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+            rcStrict = iemMemStoreDataU64(pVCpu, iEffSeg, GCPtrDst, u64Dst);
+        else
+            rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrDst, u64Dst);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            iemVmxVmreadSuccess(pVCpu, cbInstr);
+            return VINF_SUCCESS;
+        }
+
+        Log(("vmread/mem: Failed to write to memory operand at %#RGv, rc=%Rrc\n", GCPtrDst, VBOXSTRICTRC_VAL(rcStrict)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmread_PtrMap;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrDst;
+        return rcStrict;
+    }
+
+    Log(("vmread/mem: iemVmxVmreadCommon failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VMWRITE instruction execution worker that does not perform any validation
+ * checks.
+ *
+ * Callers are expected to have performed the necessary checks and to ensure the
+ * VMWRITE will succeed.
+ *
+ * @param   pVmcs           Pointer to the virtual VMCS.
+ * @param   u64Val          The value to write.
+ * @param   u64VmcsField    The VMCS field.
+ *
+ * @remarks May be called with interrupts disabled.
+ */
+IEM_STATIC void iemVmxVmwriteNoCheck(PVMXVVMCS pVmcs, uint64_t u64Val, uint64_t u64VmcsField)
+{
+    VMXVMCSFIELD VmcsField;
+    VmcsField.u = u64VmcsField;
+    uint8_t  const uWidth     = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_WIDTH);
+    uint8_t  const uType      = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_TYPE);
+    uint8_t  const uWidthType = (uWidth << 2) | uType;
+    uint8_t  const uIndex     = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_INDEX);
+    Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+    uint16_t const offField   = g_aoffVmcsMap[uWidthType][uIndex];
+    Assert(offField < VMX_V_VMCS_SIZE);
+    AssertCompile(VMX_V_SHADOW_VMCS_SIZE == VMX_V_VMCS_SIZE);
+
+    /*
+     * Write the VMCS component based on the field's effective width.
+     *
+     * The effective width is 64-bit fields adjusted to 32-bits if the access-type
+     * indicates high bits (little endian).
+     */
+    uint8_t      *pbVmcs    = (uint8_t *)pVmcs;
+    uint8_t      *pbField   = pbVmcs + offField;
+    uint8_t const uEffWidth = VMXGetVmcsFieldWidthEff(VmcsField.u);
+    switch (uEffWidth)
+    {
+        case VMX_VMCSFIELD_WIDTH_64BIT:
+        case VMX_VMCSFIELD_WIDTH_NATURAL: *(uint64_t *)pbField = u64Val; break;
+        case VMX_VMCSFIELD_WIDTH_32BIT:   *(uint32_t *)pbField = u64Val; break;
+        case VMX_VMCSFIELD_WIDTH_16BIT:   *(uint16_t *)pbField = u64Val; break;
+    }
+}
+
+
+/**
+ * VMWRITE instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   iEffSeg         The effective segment register to use with @a u64Val.
+ *                          Pass UINT8_MAX if it is a register access.
+ * @param   u64Val          The value to write (or guest linear address to the
+ *                          value), @a iEffSeg will indicate if it's a memory
+ *                          operand.
+ * @param   u64VmcsField    The VMCS field.
+ * @param   pExitInfo       Pointer to the VM-exit information. Optional, can be
+ *                          NULL.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmwrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, uint64_t u64Val, uint64_t u64VmcsField,
+                                      PCVMXVEXITINFO pExitInfo)
+{
+    /* Nested-guest intercept. */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, VMX_EXIT_VMWRITE, u64VmcsField))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMWRITE, VMXINSTRID_VMWRITE, cbInstr);
+    }
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("vmwrite: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmwrite_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* VMCS pointer in root mode. */
+    if (   !IEM_VMX_IS_ROOT_MODE(pVCpu)
+        ||  IEM_VMX_HAS_CURRENT_VMCS(pVCpu))
+    { /* likely */ }
+    else
+    {
+        Log(("vmwrite: VMCS pointer %#RGp invalid -> VMFailInvalid\n", IEM_VMX_GET_CURRENT_VMCS(pVCpu)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmwrite_PtrInvalid;
+        iemVmxVmFailInvalid(pVCpu);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* VMCS-link pointer in non-root mode. */
+    if (   !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        ||  IEM_VMX_HAS_SHADOW_VMCS(pVCpu))
+    { /* likely */ }
+    else
+    {
+        Log(("vmwrite: VMCS-link pointer %#RGp invalid -> VMFailInvalid\n", IEM_VMX_GET_SHADOW_VMCS(pVCpu)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmwrite_LinkPtrInvalid;
+        iemVmxVmFailInvalid(pVCpu);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* If the VMWRITE instruction references memory, access the specified memory operand. */
+    bool const fIsRegOperand = iEffSeg == UINT8_MAX;
+    if (!fIsRegOperand)
+    {
+        /* Read the value from the specified guest memory location. */
+        VBOXSTRICTRC  rcStrict;
+        RTGCPTR const GCPtrVal = u64Val;
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+            rcStrict = iemMemFetchDataU64(pVCpu, &u64Val, iEffSeg, GCPtrVal);
+        else
+            rcStrict = iemMemFetchDataU32_ZX_U64(pVCpu, &u64Val, iEffSeg, GCPtrVal);
+        if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
+        {
+            Log(("vmwrite: Failed to read value from memory operand at %#RGv, rc=%Rrc\n", GCPtrVal, VBOXSTRICTRC_VAL(rcStrict)));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmwrite_PtrMap;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVal;
+            return rcStrict;
+        }
+    }
+    else
+        Assert(!pExitInfo || pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand);
+
+    /* Supported VMCS field. */
+    if (CPUMIsGuestVmxVmcsFieldValid(pVCpu->CTX_SUFF(pVM), u64VmcsField))
+    { /* likely */ }
+    else
+    {
+        Log(("vmwrite: VMCS field %#RX64 invalid -> VMFail\n", u64VmcsField));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmwrite_FieldInvalid;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64VmcsField;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMWRITE_INVALID_COMPONENT);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* Read-only VMCS field. */
+    bool const fIsFieldReadOnly = VMXIsVmcsFieldReadOnly(u64VmcsField);
+    if (   !fIsFieldReadOnly
+        ||  IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxVmwriteAll)
+    { /* likely */ }
+    else
+    {
+        Log(("vmwrite: Write to read-only VMCS component %#RX64 -> VMFail\n", u64VmcsField));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmwrite_FieldRo;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64VmcsField;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMWRITE_RO_COMPONENT);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Write to the current or shadow VMCS.
+     */
+    bool const fInVmxNonRootMode = IEM_VMX_IS_NON_ROOT_MODE(pVCpu);
+    PVMXVVMCS pVmcs = !fInVmxNonRootMode
+                    ? pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs)
+                    : pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pShadowVmcs);
+    Assert(pVmcs);
+    iemVmxVmwriteNoCheck(pVmcs, u64Val, u64VmcsField);
+
+    /* Notify HM that the VMCS content might have changed. */
+    if (!fInVmxNonRootMode)
+        HMNotifyVmxNstGstCurrentVmcsChanged(pVCpu);
+
+    iemVmxVmSucceed(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VMCLEAR instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iEffSeg     The effective segment register to use with @a GCPtrVmcs.
+ * @param   GCPtrVmcs   The linear address of the VMCS pointer.
+ * @param   pExitInfo   Pointer to the VM-exit information. Optional, can be NULL.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmclear(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPHYS GCPtrVmcs,
+                                      PCVMXVEXITINFO pExitInfo)
+{
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMCLEAR, VMXINSTRID_NONE, cbInstr);
+    }
+
+    Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("vmclear: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmclear_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Get the VMCS pointer from the location specified by the source memory operand. */
+    RTGCPHYS GCPhysVmcs;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataU64(pVCpu, &GCPhysVmcs, iEffSeg, GCPtrVmcs);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* likely */ }
+    else
+    {
+        Log(("vmclear: Failed to read VMCS physaddr from %#RGv, rc=%Rrc\n", GCPtrVmcs, VBOXSTRICTRC_VAL(rcStrict)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmclear_PtrMap;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVmcs;
+        return rcStrict;
+    }
+
+    /* VMCS pointer alignment. */
+    if (!(GCPhysVmcs & X86_PAGE_4K_OFFSET_MASK))
+    { /* likely */ }
+    else
+    {
+        Log(("vmclear: VMCS pointer not page-aligned -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmclear_PtrAlign;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMCLEAR_INVALID_PHYSADDR);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* VMCS physical-address width limits. */
+    if (!(GCPhysVmcs >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth))
+    { /* likely */ }
+    else
+    {
+        Log(("vmclear: VMCS pointer extends beyond physical-address width -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmclear_PtrWidth;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMCLEAR_INVALID_PHYSADDR);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* VMCS is not the VMXON region. */
+    if (GCPhysVmcs != pVCpu->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon)
+    { /* likely */ }
+    else
+    {
+        Log(("vmclear: VMCS pointer cannot be identical to VMXON region pointer -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmclear_PtrVmxon;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMCLEAR_VMXON_PTR);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* Ensure VMCS is not MMIO, ROM etc. This is not an Intel requirement but a
+       restriction imposed by our implementation. */
+    if (PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcs))
+    { /* likely */ }
+    else
+    {
+        Log(("vmclear: VMCS not normal memory -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmclear_PtrAbnormal;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMCLEAR_INVALID_PHYSADDR);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * VMCLEAR allows committing and clearing any valid VMCS pointer.
+     *
+     * If the current VMCS is the one being cleared, set its state to 'clear' and commit
+     * to guest memory. Otherwise, set the state of the VMCS referenced in guest memory
+     * to 'clear'.
+     */
+    uint8_t const fVmcsLaunchStateClear = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
+    if (   IEM_VMX_HAS_CURRENT_VMCS(pVCpu)
+        && IEM_VMX_GET_CURRENT_VMCS(pVCpu) == GCPhysVmcs)
+    {
+        pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs)->fVmcsState = fVmcsLaunchStateClear;
+        iemVmxWriteCurrentVmcsToGstMem(pVCpu);
+        IEM_VMX_CLEAR_CURRENT_VMCS(pVCpu);
+    }
+    else
+    {
+        AssertCompileMemberSize(VMXVVMCS, fVmcsState, sizeof(fVmcsLaunchStateClear));
+        rcStrict = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVmcs + RT_UOFFSETOF(VMXVVMCS, fVmcsState),
+                                            (const void *)&fVmcsLaunchStateClear, sizeof(fVmcsLaunchStateClear));
+        if (RT_FAILURE(rcStrict))
+            return rcStrict;
+    }
+
+    iemVmxVmSucceed(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VMPTRST instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iEffSeg     The effective segment register to use with @a GCPtrVmcs.
+ * @param   GCPtrVmcs   The linear address of where to store the current VMCS
+ *                      pointer.
+ * @param   pExitInfo   Pointer to the VM-exit information. Optional, can be NULL.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmptrst(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPHYS GCPtrVmcs,
+                                      PCVMXVEXITINFO pExitInfo)
+{
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMPTRST, VMXINSTRID_NONE, cbInstr);
+    }
+
+    Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrst: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrst_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Set the VMCS pointer to the location specified by the destination memory operand. */
+    AssertCompile(NIL_RTGCPHYS == ~(RTGCPHYS)0U);
+    VBOXSTRICTRC rcStrict = iemMemStoreDataU64(pVCpu, iEffSeg, GCPtrVmcs, IEM_VMX_GET_CURRENT_VMCS(pVCpu));
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        iemVmxVmSucceed(pVCpu);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return rcStrict;
+    }
+
+    Log(("vmptrst: Failed to store VMCS pointer to memory at destination operand %#Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrst_PtrMap;
+    pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVmcs;
+    return rcStrict;
+}
+
+
+/**
+ * VMPTRLD instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   GCPtrVmcs   The linear address of the current VMCS pointer.
+ * @param   pExitInfo   Pointer to the VM-exit information. Optional, can be NULL.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmptrld(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPHYS GCPtrVmcs,
+                                      PCVMXVEXITINFO pExitInfo)
+{
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMPTRLD, VMXINSTRID_NONE, cbInstr);
+    }
+
+    Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Get the VMCS pointer from the location specified by the source memory operand. */
+    RTGCPHYS GCPhysVmcs;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataU64(pVCpu, &GCPhysVmcs, iEffSeg, GCPtrVmcs);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: Failed to read VMCS physaddr from %#RGv, rc=%Rrc\n", GCPtrVmcs, VBOXSTRICTRC_VAL(rcStrict)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_PtrMap;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVmcs;
+        return rcStrict;
+    }
+
+    /* VMCS pointer alignment. */
+    if (!(GCPhysVmcs & X86_PAGE_4K_OFFSET_MASK))
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: VMCS pointer not page-aligned -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_PtrAlign;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INVALID_PHYSADDR);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* VMCS physical-address width limits. */
+    if (!(GCPhysVmcs >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth))
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: VMCS pointer extends beyond physical-address width -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_PtrWidth;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INVALID_PHYSADDR);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* VMCS is not the VMXON region. */
+    if (GCPhysVmcs != pVCpu->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon)
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: VMCS pointer cannot be identical to VMXON region pointer -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_PtrVmxon;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_VMXON_PTR);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* Ensure VMCS is not MMIO, ROM etc. This is not an Intel requirement but a
+       restriction imposed by our implementation. */
+    if (PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcs))
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: VMCS not normal memory -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_PtrAbnormal;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INVALID_PHYSADDR);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* Read just the VMCS revision from the VMCS. */
+    VMXVMCSREVID VmcsRevId;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &VmcsRevId, GCPhysVmcs, sizeof(VmcsRevId));
+    if (RT_SUCCESS(rc))
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: Failed to read revision identifier from VMCS at %#RGp, rc=%Rrc\n", GCPhysVmcs, rc));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_RevPtrReadPhys;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        return rc;
+    }
+
+    /*
+     * Verify the VMCS revision specified by the guest matches what we reported to the guest.
+     * Verify the VMCS is not a shadow VMCS, if the VMCS shadowing feature is supported.
+     */
+    if (   VmcsRevId.n.u31RevisionId == VMX_V_VMCS_REVISION_ID
+        && (   !VmcsRevId.n.fIsShadowVmcs
+            ||  IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxVmcsShadowing))
+    { /* likely */ }
+    else
+    {
+        if (VmcsRevId.n.u31RevisionId != VMX_V_VMCS_REVISION_ID)
+        {
+            Log(("vmptrld: VMCS revision mismatch, expected %#RX32 got %#RX32, GCPtrVmcs=%#RGv GCPhysVmcs=%#RGp -> VMFail()\n",
+                 VMX_V_VMCS_REVISION_ID, VmcsRevId.n.u31RevisionId, GCPtrVmcs, GCPhysVmcs));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_VmcsRevId;
+            iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INCORRECT_VMCS_REV);
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            return VINF_SUCCESS;
+        }
+
+        Log(("vmptrld: Shadow VMCS -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_ShadowVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INCORRECT_VMCS_REV);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * We cache only the current VMCS in CPUMCTX. Therefore, VMPTRLD should always flush
+     * the cache of an existing, current VMCS back to guest memory before loading a new,
+     * different current VMCS.
+     */
+    if (IEM_VMX_GET_CURRENT_VMCS(pVCpu) != GCPhysVmcs)
+    {
+        if (IEM_VMX_HAS_CURRENT_VMCS(pVCpu))
+        {
+            iemVmxWriteCurrentVmcsToGstMem(pVCpu);
+            IEM_VMX_CLEAR_CURRENT_VMCS(pVCpu);
+        }
+
+        /* Set the new VMCS as the current VMCS and read it from guest memory. */
+        IEM_VMX_SET_CURRENT_VMCS(pVCpu, GCPhysVmcs);
+        rc = iemVmxReadCurrentVmcsFromGstMem(pVCpu);
+        if (RT_SUCCESS(rc))
+        {
+            /* Notify HM that a new, current VMCS is loaded. */
+            HMNotifyVmxNstGstCurrentVmcsChanged(pVCpu);
+        }
+        else
+        {
+            Log(("vmptrld: Failed to read VMCS at %#RGp, rc=%Rrc\n", GCPhysVmcs, rc));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_PtrReadPhys;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+            return rc;
+        }
+    }
+
+    Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
+    iemVmxVmSucceed(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * INVVPID instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ * @param   iEffSeg             The segment of the invvpid descriptor.
+ * @param   GCPtrInvvpidDesc    The address of invvpid descriptor.
+ * @param   u64InvvpidType      The invalidation type.
+ * @param   pExitInfo           Pointer to the VM-exit information. Optional, can be
+ *                              NULL.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxInvvpid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrInvvpidDesc,
+                                      uint64_t u64InvvpidType, PCVMXVEXITINFO pExitInfo)
+{
+    /* Check if INVVPID instruction is supported, otherwise raise #UD. */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxVpid)
+        return iemRaiseUndefinedOpcode(pVCpu);
+
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_INVVPID, VMXINSTRID_NONE, cbInstr);
+    }
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("invvpid: CPL != 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Validate INVVPID invalidation type.
+     *
+     * The instruction specifies exactly ONE of the supported invalidation types.
+     *
+     * Each of the types has a bit in IA32_VMX_EPT_VPID_CAP MSR specifying if it is
+     * supported. In theory, it's possible for a CPU to not support flushing individual
+     * addresses but all the other types or any other combination. We do not take any
+     * shortcuts here by  assuming the types we currently expose to the guest.
+     */
+    uint64_t const fCaps = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64EptVpidCaps;
+    uint8_t const fTypeIndivAddr              = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
+    uint8_t const fTypeSingleCtx              = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID_SINGLE_CTX);
+    uint8_t const fTypeAllCtx                 = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID_ALL_CTX);
+    uint8_t const fTypeSingleCtxRetainGlobals = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID_SINGLE_CTX_RETAIN_GLOBALS);
+    if (   (fTypeIndivAddr              && u64InvvpidType == VMXTLBFLUSHVPID_INDIV_ADDR)
+        || (fTypeSingleCtx              && u64InvvpidType == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
+        || (fTypeAllCtx                 && u64InvvpidType == VMXTLBFLUSHVPID_ALL_CONTEXTS)
+        || (fTypeSingleCtxRetainGlobals && u64InvvpidType == VMXTLBFLUSHVPID_SINGLE_CONTEXT_RETAIN_GLOBALS))
+    { /* likely */ }
+    else
+    {
+        Log(("invvpid: invalid/unsupported invvpid type %#x -> VMFail\n", u64InvvpidType));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invvpid_TypeInvalid;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64InvvpidType;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Fetch the invvpid descriptor from guest memory.
+     */
+    RTUINT128U uDesc;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataU128(pVCpu, &uDesc, iEffSeg, GCPtrInvvpidDesc);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /*
+         * Validate the descriptor.
+         */
+        if (uDesc.s.Lo > 0xfff)
+        {
+            Log(("invvpid: reserved bits set in invvpid descriptor %#RX64 -> #GP(0)\n", uDesc.s.Lo));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invvpid_DescRsvd;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = uDesc.s.Lo;
+            iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            return VINF_SUCCESS;
+        }
+
+        IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
+        RTGCUINTPTR64 const GCPtrInvAddr = uDesc.s.Hi;
+        uint8_t       const uVpid        = uDesc.s.Lo & UINT64_C(0xfff);
+        uint64_t      const uCr3         = pVCpu->cpum.GstCtx.cr3;
+        switch (u64InvvpidType)
+        {
+            case VMXTLBFLUSHVPID_INDIV_ADDR:
+            {
+                if (uVpid != 0)
+                {
+                    if (IEM_IS_CANONICAL(GCPtrInvAddr))
+                    {
+                        /* Invalidate mappings for the linear address tagged with VPID. */
+                        /** @todo PGM support for VPID? Currently just flush everything. */
+                        PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
+                        iemVmxVmSucceed(pVCpu);
+                    }
+                    else
+                    {
+                        Log(("invvpid: invalidation address %#RGP is not canonical -> VMFail\n", GCPtrInvAddr));
+                        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invvpid_Type0InvalidAddr;
+                        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrInvAddr;
+                        iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+                    }
+                }
+                else
+                {
+                    Log(("invvpid: invalid VPID %#x for invalidation type %u -> VMFail\n", uVpid, u64InvvpidType));
+                    pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invvpid_Type0InvalidVpid;
+                    pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64InvvpidType;
+                    iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+                }
+                break;
+            }
+
+            case VMXTLBFLUSHVPID_SINGLE_CONTEXT:
+            {
+                if (uVpid != 0)
+                {
+                    /* Invalidate all mappings with VPID. */
+                    /** @todo PGM support for VPID? Currently just flush everything. */
+                    PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
+                    iemVmxVmSucceed(pVCpu);
+                }
+                else
+                {
+                    Log(("invvpid: invalid VPID %#x for invalidation type %u -> VMFail\n", uVpid, u64InvvpidType));
+                    pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invvpid_Type1InvalidVpid;
+                    pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64InvvpidType;
+                    iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+                }
+                break;
+            }
+
+            case VMXTLBFLUSHVPID_ALL_CONTEXTS:
+            {
+                /* Invalidate all mappings with non-zero VPIDs. */
+                /** @todo PGM support for VPID? Currently just flush everything. */
+                PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
+                iemVmxVmSucceed(pVCpu);
+                break;
+            }
+
+            case VMXTLBFLUSHVPID_SINGLE_CONTEXT_RETAIN_GLOBALS:
+            {
+                if (uVpid != 0)
+                {
+                    /* Invalidate all mappings with VPID except global translations. */
+                    /** @todo PGM support for VPID? Currently just flush everything. */
+                    PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
+                    iemVmxVmSucceed(pVCpu);
+                }
+                else
+                {
+                    Log(("invvpid: invalid VPID %#x for invalidation type %u -> VMFail\n", uVpid, u64InvvpidType));
+                    pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invvpid_Type3InvalidVpid;
+                    pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = uVpid;
+                    iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+                }
+                break;
+            }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VMXON instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iEffSeg     The effective segment register to use with @a
+ *                      GCPtrVmxon.
+ * @param   GCPtrVmxon  The linear address of the VMXON pointer.
+ * @param   pExitInfo   Pointer to the VM-exit information. Optional, can be NULL.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+IEM_STATIC VBOXSTRICTRC iemVmxVmxon(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPHYS GCPtrVmxon,
+                                    PCVMXVEXITINFO pExitInfo)
+{
+    if (!IEM_VMX_IS_ROOT_MODE(pVCpu))
+    {
+        /* CPL. */
+        if (pVCpu->iem.s.uCpl == 0)
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cpl;
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        /* A20M (A20 Masked) mode. */
+        if (PGMPhysIsA20Enabled(pVCpu))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: A20M mode -> #GP(0)\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_A20M;
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        /* CR0. */
+        {
+            /* CR0 MB1 bits. */
+            uint64_t const uCr0Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed0;
+            if ((pVCpu->cpum.GstCtx.cr0 & uCr0Fixed0) == uCr0Fixed0)
+            { /* likely */ }
+            else
+            {
+                Log(("vmxon: CR0 fixed0 bits cleared -> #GP(0)\n"));
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cr0Fixed0;
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            /* CR0 MBZ bits. */
+            uint64_t const uCr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1;
+            if (!(pVCpu->cpum.GstCtx.cr0 & ~uCr0Fixed1))
+            { /* likely */ }
+            else
+            {
+                Log(("vmxon: CR0 fixed1 bits set -> #GP(0)\n"));
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cr0Fixed1;
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+        }
+
+        /* CR4. */
+        {
+            /* CR4 MB1 bits. */
+            uint64_t const uCr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0;
+            if ((pVCpu->cpum.GstCtx.cr4 & uCr4Fixed0) == uCr4Fixed0)
+            { /* likely */ }
+            else
+            {
+                Log(("vmxon: CR4 fixed0 bits cleared -> #GP(0)\n"));
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cr4Fixed0;
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            /* CR4 MBZ bits. */
+            uint64_t const uCr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1;
+            if (!(pVCpu->cpum.GstCtx.cr4 & ~uCr4Fixed1))
+            { /* likely */ }
+            else
+            {
+                Log(("vmxon: CR4 fixed1 bits set -> #GP(0)\n"));
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cr4Fixed1;
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+        }
+
+        /* Feature control MSR's LOCK and VMXON bits. */
+        uint64_t const uMsrFeatCtl = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64FeatCtrl;
+        if ((uMsrFeatCtl & (MSR_IA32_FEATURE_CONTROL_LOCK | MSR_IA32_FEATURE_CONTROL_VMXON))
+                        == (MSR_IA32_FEATURE_CONTROL_LOCK | MSR_IA32_FEATURE_CONTROL_VMXON))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: Feature control lock bit or VMXON bit cleared -> #GP(0)\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_MsrFeatCtl;
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        /* Get the VMXON pointer from the location specified by the source memory operand. */
+        RTGCPHYS GCPhysVmxon;
+        VBOXSTRICTRC rcStrict = iemMemFetchDataU64(pVCpu, &GCPhysVmxon, iEffSeg, GCPtrVmxon);
+        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: Failed to read VMXON region physaddr from %#RGv, rc=%Rrc\n", GCPtrVmxon, VBOXSTRICTRC_VAL(rcStrict)));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmxon_PtrMap;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVmxon;
+            return rcStrict;
+        }
+
+        /* VMXON region pointer alignment. */
+        if (!(GCPhysVmxon & X86_PAGE_4K_OFFSET_MASK))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: VMXON region pointer not page-aligned -> VMFailInvalid\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmxon_PtrAlign;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmxon;
+            iemVmxVmFailInvalid(pVCpu);
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            return VINF_SUCCESS;
+        }
+
+        /* VMXON physical-address width limits. */
+        if (!(GCPhysVmxon >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: VMXON region pointer extends beyond physical-address width -> VMFailInvalid\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmxon_PtrWidth;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmxon;
+            iemVmxVmFailInvalid(pVCpu);
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            return VINF_SUCCESS;
+        }
+
+        /* Ensure VMXON region is not MMIO, ROM etc. This is not an Intel requirement but a
+           restriction imposed by our implementation. */
+        if (PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmxon))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: VMXON region not normal memory -> VMFailInvalid\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmxon_PtrAbnormal;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmxon;
+            iemVmxVmFailInvalid(pVCpu);
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            return VINF_SUCCESS;
+        }
+
+        /* Read the VMCS revision ID from the VMXON region. */
+        VMXVMCSREVID VmcsRevId;
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &VmcsRevId, GCPhysVmxon, sizeof(VmcsRevId));
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: Failed to read VMXON region at %#RGp, rc=%Rrc\n", GCPhysVmxon, rc));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_PtrReadPhys;
+            return rc;
+        }
+
+        /* Verify the VMCS revision specified by the guest matches what we reported to the guest. */
+        if (RT_LIKELY(VmcsRevId.u == VMX_V_VMCS_REVISION_ID))
+        { /* likely */ }
+        else
+        {
+            /* Revision ID mismatch. */
+            if (!VmcsRevId.n.fIsShadowVmcs)
+            {
+                Log(("vmxon: VMCS revision mismatch, expected %#RX32 got %#RX32 -> VMFailInvalid\n", VMX_V_VMCS_REVISION_ID,
+                     VmcsRevId.n.u31RevisionId));
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_VmcsRevId;
+                iemVmxVmFailInvalid(pVCpu);
+                iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+                return VINF_SUCCESS;
+            }
+
+            /* Shadow VMCS disallowed. */
+            Log(("vmxon: Shadow VMCS -> VMFailInvalid\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_ShadowVmcs;
+            iemVmxVmFailInvalid(pVCpu);
+            iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+            return VINF_SUCCESS;
+        }
+
+        /*
+         * Record that we're in VMX operation, block INIT, block and disable A20M.
+         */
+        pVCpu->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon    = GCPhysVmxon;
+        IEM_VMX_CLEAR_CURRENT_VMCS(pVCpu);
+        pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxRootMode = true;
+
+        /* Clear address-range monitoring. */
+        EMMonitorWaitClear(pVCpu);
+        /** @todo NSTVMX: Intel PT. */
+
+        iemVmxVmSucceed(pVCpu);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+    else if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        /* Nested-guest intercept. */
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMXON, VMXINSTRID_NONE, cbInstr);
+    }
+
+    Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl > 0)
+    {
+        Log(("vmxon: In VMX root mode: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_VmxRootCpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* VMXON when already in VMX root mode. */
+    iemVmxVmFail(pVCpu, VMXINSTRERR_VMXON_IN_VMXROOTMODE);
+    pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_VmxAlreadyRoot;
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'VMXOFF'.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmxoff)
+{
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        return iemVmxVmexitInstr(pVCpu, VMX_EXIT_VMXOFF, cbInstr);
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("vmxoff: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxoff_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Dual monitor treatment of SMIs and SMM. */
+    uint64_t const fSmmMonitorCtl = CPUMGetGuestIa32SmmMonitorCtl(pVCpu);
+    if (!(fSmmMonitorCtl & MSR_IA32_SMM_MONITOR_VALID))
+    { /* likely */ }
+    else
+    {
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMXOFF_DUAL_MON);
+        iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+        return VINF_SUCCESS;
+    }
+
+    /* Record that we're no longer in VMX root operation, block INIT, block and disable A20M. */
+    pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxRootMode = false;
+    Assert(!pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode);
+
+    if (fSmmMonitorCtl & MSR_IA32_SMM_MONITOR_VMXOFF_UNBLOCK_SMI)
+    { /** @todo NSTVMX: Unblock SMI. */ }
+
+    EMMonitorWaitClear(pVCpu);
+    /** @todo NSTVMX: Unblock and enable A20M. */
+
+    iemVmxVmSucceed(pVCpu);
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'VMXON'.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmxon, uint8_t, iEffSeg, RTGCPTR, GCPtrVmxon)
+{
+    return iemVmxVmxon(pVCpu, cbInstr, iEffSeg, GCPtrVmxon, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMLAUNCH'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmlaunch)
+{
+    return iemVmxVmlaunchVmresume(pVCpu, cbInstr, VMXINSTRID_VMLAUNCH);
+}
+
+
+/**
+ * Implements 'VMRESUME'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmresume)
+{
+    return iemVmxVmlaunchVmresume(pVCpu, cbInstr, VMXINSTRID_VMRESUME);
+}
+
+
+/**
+ * Implements 'VMPTRLD'.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmptrld, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs)
+{
+    return iemVmxVmptrld(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMPTRST'.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmptrst, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs)
+{
+    return iemVmxVmptrst(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMCLEAR'.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmclear, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs)
+{
+    return iemVmxVmclear(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMWRITE' register.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmwrite_reg, uint64_t, u64Val, uint64_t, u64VmcsField)
+{
+    return iemVmxVmwrite(pVCpu, cbInstr, UINT8_MAX /* iEffSeg */, u64Val, u64VmcsField, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMWRITE' memory.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_vmwrite_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrVal, uint32_t, u64VmcsField)
+{
+    return iemVmxVmwrite(pVCpu, cbInstr, iEffSeg, GCPtrVal, u64VmcsField,  NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMREAD' register (64-bit).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmread_reg64, uint64_t *, pu64Dst, uint64_t, u64VmcsField)
+{
+    return iemVmxVmreadReg64(pVCpu, cbInstr, pu64Dst, u64VmcsField, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMREAD' register (32-bit).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmread_reg32, uint32_t *, pu32Dst, uint32_t, u32VmcsField)
+{
+    return iemVmxVmreadReg32(pVCpu, cbInstr, pu32Dst, u32VmcsField, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMREAD' memory, 64-bit register.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_vmread_mem_reg64, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u64VmcsField)
+{
+    return iemVmxVmreadMem(pVCpu, cbInstr, iEffSeg, GCPtrDst, u64VmcsField, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMREAD' memory, 32-bit register.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_vmread_mem_reg32, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u32VmcsField)
+{
+    return iemVmxVmreadMem(pVCpu, cbInstr, iEffSeg, GCPtrDst, u32VmcsField, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'INVVPID'.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_invvpid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvvpidDesc, uint64_t, uInvvpidType)
+{
+    return iemVmxInvvpid(pVCpu, cbInstr, iEffSeg, GCPtrInvvpidDesc, uInvvpidType, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements VMX's implementation of PAUSE.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmx_pause)
+{
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict = iemVmxVmexitInstrPause(pVCpu, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+
+    /*
+     * Outside VMX non-root operation or if the PAUSE instruction does not cause
+     * a VM-exit, the instruction operates normally.
+     */
+    iemRegAddToRipAndClearRF(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+}
+
+#endif  /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/**
+ * Implements 'VMCALL'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmcall)
+{
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        return iemVmxVmexitInstr(pVCpu, VMX_EXIT_VMCALL, cbInstr);
+#endif
+
+    /* Join forces with vmmcall. */
+    return IEM_CIMPL_CALL_1(iemCImpl_Hypercall, OP_VMCALL);
+}
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructions.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructions.cpp.h
new file mode 100644
index 00000000..c7b2a897
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructions.cpp.h
@@ -0,0 +1,776 @@
+/* $Id: IEMAllInstructions.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+extern const PFNIEMOP g_apfnOneByteMap[256]; /* not static since we need to forward declare it. */
+
+#ifdef _MSC_VER
+# pragma warning(push)
+# pragma warning(disable: 4702) /* Unreachable code like return in iemOp_Grp6_lldt. */
+#endif
+
+
+/**
+ * Common worker for instructions like ADD, AND, OR, ++ with a byte
+ * memory/register as the destination.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBinaryOperator_rm_r8, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,  pu8Dst,  0);
+        IEM_MC_ARG(uint8_t,    u8Src,   1);
+        IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+        IEM_MC_FETCH_GREG_U8(u8Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_GREG_U8(pu8Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         * Note! We're putting the eflags on the stack here so we can commit them
+         *       after the memory.
+         */
+        uint32_t const fAccess = pImpl->pfnLockedU8 ? IEM_ACCESS_DATA_RW : IEM_ACCESS_DATA_R; /* CMP,TEST */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint8_t *,  pu8Dst,           0);
+        IEM_MC_ARG(uint8_t,    u8Src,            1);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        if (!pImpl->pfnLockedU8)
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MEM_MAP(pu8Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_GREG_U8(u8Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+        else
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, fAccess);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for word/dword/qword instructions like ADD, AND, OR, ++ with
+ * memory/register as the destination.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBinaryOperator_rm_rv, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *, pu16Dst, 0);
+                IEM_MC_ARG(uint16_t,   u16Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *, pu32Dst, 0);
+                IEM_MC_ARG(uint32_t,   u32Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+                if (pImpl != &g_iemAImpl_test)
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *, pu64Dst, 0);
+                IEM_MC_ARG(uint64_t,   u64Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U64(u64Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+        }
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         * Note! We're putting the eflags on the stack here so we can commit them
+         *       after the memory.
+         */
+        uint32_t const fAccess = pImpl->pfnLockedU8 ? IEM_ACCESS_DATA_RW : IEM_ACCESS_DATA_R /* CMP,TEST */;
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *, pu16Dst,          0);
+                IEM_MC_ARG(uint16_t,   u16Src,           1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                if (!pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_GREG_U16(u16Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *, pu32Dst,          0);
+                IEM_MC_ARG(uint32_t,   u32Src,           1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                if (!pImpl->pfnLockedU32)
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_GREG_U32(u32Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *, pu64Dst,          0);
+                IEM_MC_ARG(uint64_t,   u64Src,           1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                if (!pImpl->pfnLockedU64)
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_GREG_U64(u64Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for byte instructions like ADD, AND, OR, ++ with a register as
+ * the destination.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBinaryOperator_r8_rm, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,  pu8Dst,  0);
+        IEM_MC_ARG(uint8_t,    u8Src,   1);
+        IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+        IEM_MC_FETCH_GREG_U8(u8Src, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_GREG_U8(pu8Dst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_ARG(uint8_t *,  pu8Dst,  0);
+        IEM_MC_ARG(uint8_t,    u8Src,   1);
+        IEM_MC_ARG(uint32_t *, pEFlags, 2);
+        IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_MEM_U8(u8Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+        IEM_MC_REF_GREG_U8(pu8Dst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for word/dword/qword instructions like ADD, AND, OR, ++ with a
+ * register as the destination.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBinaryOperator_rv_rm, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *, pu16Dst, 0);
+                IEM_MC_ARG(uint16_t,   u16Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_GREG_U16(pu16Dst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *, pu32Dst, 0);
+                IEM_MC_ARG(uint32_t,   u32Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_GREG_U32(pu32Dst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *, pu64Dst, 0);
+                IEM_MC_ARG(uint64_t,   u64Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U64(u64Src, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_GREG_U64(pu64Dst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+        }
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t *, pu16Dst, 0);
+                IEM_MC_ARG(uint16_t,   u16Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U16(pu16Dst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint32_t *, pu32Dst, 0);
+                IEM_MC_ARG(uint32_t,   u32Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U32(pu32Dst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint64_t *, pu64Dst, 0);
+                IEM_MC_ARG(uint64_t,   u64Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U64(pu64Dst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for instructions like ADD, AND, OR, ++ with working on AL with
+ * a byte immediate.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBinaryOperator_AL_Ib, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG(uint8_t *,       pu8Dst,             0);
+    IEM_MC_ARG_CONST(uint8_t,   u8Src,/*=*/ u8Imm,  1);
+    IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+
+    IEM_MC_REF_GREG_U8(pu8Dst, X86_GREG_xAX);
+    IEM_MC_REF_EFLAGS(pEFlags);
+    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for instructions like ADD, AND, OR, ++ with working on
+ * AX/EAX/RAX with a word/dword immediate.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBinaryOperator_rAX_Iz, PCIEMOPBINSIZES, pImpl)
+{
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+            IEM_MC_ARG_CONST(uint16_t,  u16Src,/*=*/ u16Imm,    1);
+            IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+            IEM_MC_REF_GREG_U16(pu16Dst, X86_GREG_xAX);
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+            IEM_MC_ARG_CONST(uint32_t,  u32Src,/*=*/ u32Imm,    1);
+            IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+            IEM_MC_REF_GREG_U32(pu32Dst, X86_GREG_xAX);
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+            if (pImpl != &g_iemAImpl_test)
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+            IEM_MC_ARG_CONST(uint64_t,  u64Src,/*=*/ u64Imm,    1);
+            IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+            IEM_MC_REF_GREG_U64(pu64Dst, X86_GREG_xAX);
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/** Opcodes 0xf1, 0xd6. */
+FNIEMOP_DEF(iemOp_Invalid)
+{
+    IEMOP_MNEMONIC(Invalid, "Invalid");
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid with RM byte . */
+FNIEMOPRM_DEF(iemOp_InvalidWithRM)
+{
+    RT_NOREF_PV(bRm);
+    IEMOP_MNEMONIC(InvalidWithRm, "InvalidWithRM");
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid with RM byte where intel decodes any additional address encoding
+ *  bytes. */
+FNIEMOPRM_DEF(iemOp_InvalidWithRMNeedDecode)
+{
+    IEMOP_MNEMONIC(InvalidWithRMNeedDecode, "InvalidWithRMNeedDecode");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+#ifndef TST_IEM_CHECK_MC
+        if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+    }
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid with RM byte where both AMD and Intel decodes any additional
+ *  address encoding bytes. */
+FNIEMOPRM_DEF(iemOp_InvalidWithRMAllNeeded)
+{
+    IEMOP_MNEMONIC(InvalidWithRMAllNeeded, "InvalidWithRMAllNeeded");
+#ifndef TST_IEM_CHECK_MC
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        RTGCPTR      GCPtrEff;
+        VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+#endif
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid with RM byte where intel requires 8-byte immediate.
+ * Intel will also need SIB and displacement if bRm indicates memory. */
+FNIEMOPRM_DEF(iemOp_InvalidWithRMNeedImm8)
+{
+    IEMOP_MNEMONIC(InvalidWithRMNeedImm8, "InvalidWithRMNeedImm8");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+#ifndef TST_IEM_CHECK_MC
+        if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+        uint8_t bImm8;  IEM_OPCODE_GET_NEXT_U8(&bImm8);  RT_NOREF(bRm);
+    }
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid with RM byte where intel requires 8-byte immediate.
+ * Both AMD and Intel also needs SIB and displacement according to bRm. */
+FNIEMOPRM_DEF(iemOp_InvalidWithRMAllNeedImm8)
+{
+    IEMOP_MNEMONIC(InvalidWithRMAllNeedImm8, "InvalidWithRMAllNeedImm8");
+#ifndef TST_IEM_CHECK_MC
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        RTGCPTR      GCPtrEff;
+        VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+#endif
+    uint8_t bImm8;  IEM_OPCODE_GET_NEXT_U8(&bImm8);  RT_NOREF(bRm);
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid opcode where intel requires Mod R/M sequence. */
+FNIEMOP_DEF(iemOp_InvalidNeedRM)
+{
+    IEMOP_MNEMONIC(InvalidNeedRM, "InvalidNeedRM");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+        if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+    }
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid opcode where both AMD and Intel requires Mod R/M sequence. */
+FNIEMOP_DEF(iemOp_InvalidAllNeedRM)
+{
+    IEMOP_MNEMONIC(InvalidAllNeedRM, "InvalidAllNeedRM");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        RTGCPTR      GCPtrEff;
+        VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+#endif
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid opcode where intel requires Mod R/M sequence and 8-byte
+ *  immediate. */
+FNIEMOP_DEF(iemOp_InvalidNeedRMImm8)
+{
+    IEMOP_MNEMONIC(InvalidNeedRMImm8, "InvalidNeedRMImm8");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        uint8_t bRm;  IEM_OPCODE_GET_NEXT_U8(&bRm);  RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+        if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); RT_NOREF(bImm);
+    }
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid opcode where intel requires a 3rd escape byte and a Mod R/M
+ *  sequence. */
+FNIEMOP_DEF(iemOp_InvalidNeed3ByteEscRM)
+{
+    IEMOP_MNEMONIC(InvalidNeed3ByteEscRM, "InvalidNeed3ByteEscRM");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        uint8_t b3rd; IEM_OPCODE_GET_NEXT_U8(&b3rd); RT_NOREF(b3rd);
+        uint8_t bRm;  IEM_OPCODE_GET_NEXT_U8(&bRm);  RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+        if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+    }
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid opcode where intel requires a 3rd escape byte, Mod R/M sequence, and
+ *  a 8-byte immediate. */
+FNIEMOP_DEF(iemOp_InvalidNeed3ByteEscRMImm8)
+{
+    IEMOP_MNEMONIC(InvalidNeed3ByteEscRMImm8, "InvalidNeed3ByteEscRMImm8");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        uint8_t b3rd; IEM_OPCODE_GET_NEXT_U8(&b3rd); RT_NOREF(b3rd);
+        uint8_t bRm;  IEM_OPCODE_GET_NEXT_U8(&bRm);  RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+        if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 1, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); RT_NOREF(bImm);
+        IEMOP_HLP_DONE_DECODING();
+    }
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Repeats a_fn four times.  For decoding tables. */
+#define IEMOP_X4(a_fn) a_fn, a_fn, a_fn, a_fn
+
+/*
+ * Include the tables.
+ */
+#ifdef IEM_WITH_3DNOW
+# include "IEMAllInstructions3DNow.cpp.h"
+#endif
+#ifdef IEM_WITH_THREE_0F_38
+# include "IEMAllInstructionsThree0f38.cpp.h"
+#endif
+#ifdef IEM_WITH_THREE_0F_3A
+# include "IEMAllInstructionsThree0f3a.cpp.h"
+#endif
+#include "IEMAllInstructionsTwoByte0f.cpp.h"
+#ifdef IEM_WITH_VEX
+# include "IEMAllInstructionsVexMap1.cpp.h"
+# include "IEMAllInstructionsVexMap2.cpp.h"
+# include "IEMAllInstructionsVexMap3.cpp.h"
+#endif
+#include "IEMAllInstructionsOneByte.cpp.h"
+
+
+#ifdef _MSC_VER
+# pragma warning(pop)
+#endif
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h
new file mode 100644
index 00000000..810232e7
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h
@@ -0,0 +1,133 @@
+/* $Id: IEMAllInstructions3DNow.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation, 3DNow!.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @name 3DNow! instructions (0x0f 0x0f)
+ *
+ * @{
+ */
+
+/** Opcode 0x0f 0x0f 0x0c. */
+FNIEMOP_STUB(iemOp_3Dnow_pi2fw_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x0d. */
+FNIEMOP_STUB(iemOp_3Dnow_pi2fd_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x1c. */
+FNIEMOP_STUB(iemOp_3Dnow_pf2fw_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x1d. */
+FNIEMOP_STUB(iemOp_3Dnow_pf2fd_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x8a. */
+FNIEMOP_STUB(iemOp_3Dnow_pfnacc_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x8e. */
+FNIEMOP_STUB(iemOp_3Dnow_pfpnacc_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x90. */
+FNIEMOP_STUB(iemOp_3Dnow_pfcmpge_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x94. */
+FNIEMOP_STUB(iemOp_3Dnow_pfmin_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x96. */
+FNIEMOP_STUB(iemOp_3Dnow_pfrcp_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x97. */
+FNIEMOP_STUB(iemOp_3Dnow_pfrsqrt_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x9a. */
+FNIEMOP_STUB(iemOp_3Dnow_pfsub_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x9e. */
+FNIEMOP_STUB(iemOp_3Dnow_pfadd_PQ_Qq);
+
+/** Opcode 0x0f 0x0f 0xa0. */
+FNIEMOP_STUB(iemOp_3Dnow_pfcmpgt_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xa4. */
+FNIEMOP_STUB(iemOp_3Dnow_pfmax_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xa6. */
+FNIEMOP_STUB(iemOp_3Dnow_pfrcpit1_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xa7. */
+FNIEMOP_STUB(iemOp_3Dnow_pfrsqit1_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xaa. */
+FNIEMOP_STUB(iemOp_3Dnow_pfsubr_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xae. */
+FNIEMOP_STUB(iemOp_3Dnow_pfacc_PQ_Qq);
+
+/** Opcode 0x0f 0x0f 0xb0. */
+FNIEMOP_STUB(iemOp_3Dnow_pfcmpeq_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xb4. */
+FNIEMOP_STUB(iemOp_3Dnow_pfmul_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xb6. */
+FNIEMOP_STUB(iemOp_3Dnow_pfrcpit2_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xb7. */
+FNIEMOP_STUB(iemOp_3Dnow_pmulhrw_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xbb. */
+FNIEMOP_STUB(iemOp_3Dnow_pswapd_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xbf. */
+FNIEMOP_STUB(iemOp_3Dnow_pavgusb_PQ_Qq);
+
+
+/** Opcode 0x0f 0x0f. */
+FNIEMOP_DEF_1(iemOp_3DNowDispatcher, uint8_t, b)
+{
+    /* This is pretty sparse, use switch instead of table. */
+    switch (b)
+    {
+        case 0x0c: return FNIEMOP_CALL(iemOp_3Dnow_pi2fw_Pq_Qq);
+        case 0x0d: return FNIEMOP_CALL(iemOp_3Dnow_pi2fd_Pq_Qq);
+        case 0x1c: return FNIEMOP_CALL(iemOp_3Dnow_pf2fw_Pq_Qq);
+        case 0x1d: return FNIEMOP_CALL(iemOp_3Dnow_pf2fd_Pq_Qq);
+        case 0x8a: return FNIEMOP_CALL(iemOp_3Dnow_pfnacc_Pq_Qq);
+        case 0x8e: return FNIEMOP_CALL(iemOp_3Dnow_pfpnacc_Pq_Qq);
+        case 0x90: return FNIEMOP_CALL(iemOp_3Dnow_pfcmpge_Pq_Qq);
+        case 0x94: return FNIEMOP_CALL(iemOp_3Dnow_pfmin_Pq_Qq);
+        case 0x96: return FNIEMOP_CALL(iemOp_3Dnow_pfrcp_Pq_Qq);
+        case 0x97: return FNIEMOP_CALL(iemOp_3Dnow_pfrsqrt_Pq_Qq);
+        case 0x9a: return FNIEMOP_CALL(iemOp_3Dnow_pfsub_Pq_Qq);
+        case 0x9e: return FNIEMOP_CALL(iemOp_3Dnow_pfadd_PQ_Qq);
+        case 0xa0: return FNIEMOP_CALL(iemOp_3Dnow_pfcmpgt_Pq_Qq);
+        case 0xa4: return FNIEMOP_CALL(iemOp_3Dnow_pfmax_Pq_Qq);
+        case 0xa6: return FNIEMOP_CALL(iemOp_3Dnow_pfrcpit1_Pq_Qq);
+        case 0xa7: return FNIEMOP_CALL(iemOp_3Dnow_pfrsqit1_Pq_Qq);
+        case 0xaa: return FNIEMOP_CALL(iemOp_3Dnow_pfsubr_Pq_Qq);
+        case 0xae: return FNIEMOP_CALL(iemOp_3Dnow_pfacc_PQ_Qq);
+        case 0xb0: return FNIEMOP_CALL(iemOp_3Dnow_pfcmpeq_Pq_Qq);
+        case 0xb4: return FNIEMOP_CALL(iemOp_3Dnow_pfmul_Pq_Qq);
+        case 0xb6: return FNIEMOP_CALL(iemOp_3Dnow_pfrcpit2_Pq_Qq);
+        case 0xb7: return FNIEMOP_CALL(iemOp_3Dnow_pmulhrw_Pq_Qq);
+        case 0xbb: return FNIEMOP_CALL(iemOp_3Dnow_pswapd_Pq_Qq);
+        case 0xbf: return FNIEMOP_CALL(iemOp_3Dnow_pavgusb_PQ_Qq);
+        default:
+            return IEMOP_RAISE_INVALID_OPCODE();
+    }
+}
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h
new file mode 100644
index 00000000..c29b18d8
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h
@@ -0,0 +1,11818 @@
+/* $Id: IEMAllInstructionsOneByte.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+extern const PFNIEMOP g_apfnOneByteMap[256]; /* not static since we need to forward declare it. */
+
+/* Instruction group definitions: */
+
+/** @defgroup og_gen    General
+ * @{ */
+ /** @defgroup og_gen_arith     Arithmetic
+  * @{  */
+  /** @defgroup og_gen_arith_bin    Binary numbers */
+  /** @defgroup og_gen_arith_dec    Decimal numbers */
+ /** @} */
+/** @} */
+
+/** @defgroup og_stack Stack
+ * @{ */
+ /** @defgroup og_stack_sreg    Segment registers */
+/** @} */
+
+/** @defgroup og_prefix     Prefixes */
+/** @defgroup og_escapes    Escape bytes */
+
+
+
+/** @name One byte opcodes.
+ * @{
+ */
+
+/* Instruction specification format - work in progress:  */
+
+/**
+ * @opcode      0x00
+ * @opmnemonic  add
+ * @op1         rm:Eb
+ * @op2         reg:Gb
+ * @opmaps      one
+ * @openc       ModR/M
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @ophints     harmless ignores_op_sizes
+ * @opstats     add_Eb_Gb
+ * @opgroup     og_gen_arith_bin
+ * @optest              op1=1   op2=1   -> op1=2   efl&|=nc,pe,na,nz,pl,nv
+ * @optest      efl|=cf op1=1   op2=2   -> op1=3   efl&|=nc,po,na,nz,pl,nv
+ * @optest              op1=254 op2=1   -> op1=255 efl&|=nc,po,na,nz,ng,nv
+ * @optest              op1=128 op2=128 -> op1=0   efl&|=ov,pl,zf,na,po,cf
+ */
+FNIEMOP_DEF(iemOp_add_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, ADD, add, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_add);
+}
+
+
+/**
+ * @opcode      0x01
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @optest               op1=1  op2=1  -> op1=2  efl&|=nc,pe,na,nz,pl,nv
+ * @optest      efl|=cf  op1=2  op2=2  -> op1=4  efl&|=nc,pe,na,nz,pl,nv
+ * @optest      efl&~=cf op1=-1 op2=1  -> op1=0  efl&|=cf,po,af,zf,pl,nv
+ * @optest               op1=-1 op2=-1 -> op1=-2 efl&|=cf,pe,af,nz,ng,nv
+ */
+FNIEMOP_DEF(iemOp_add_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, ADD, add, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_add);
+}
+
+
+/**
+ * @opcode      0x02
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_add_Eb_Gb
+ */
+FNIEMOP_DEF(iemOp_add_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, ADD, add, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_add);
+}
+
+
+/**
+ * @opcode      0x03
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_add_Ev_Gv
+ */
+FNIEMOP_DEF(iemOp_add_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, ADD, add, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_add);
+}
+
+
+/**
+ * @opcode      0x04
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_add_Eb_Gb
+ */
+FNIEMOP_DEF(iemOp_add_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, ADD, add, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_add);
+}
+
+
+/**
+ * @opcode      0x05
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @optest      op1=1 op2=1 -> op1=2 efl&|=nv,pl,nz,na,pe
+ * @optest      efl|=cf  op1=2  op2=2  -> op1=4  efl&|=nc,pe,na,nz,pl,nv
+ * @optest      efl&~=cf op1=-1 op2=1  -> op1=0  efl&|=cf,po,af,zf,pl,nv
+ * @optest               op1=-1 op2=-1 -> op1=-2 efl&|=cf,pe,af,nz,ng,nv
+ */
+FNIEMOP_DEF(iemOp_add_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, ADD, add, rAX, Iz, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_add);
+}
+
+
+/**
+ * @opcode      0x06
+ * @opgroup     og_stack_sreg
+ */
+FNIEMOP_DEF(iemOp_push_ES)
+{
+    IEMOP_MNEMONIC1(FIXED, PUSH, push, ES, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0);
+    IEMOP_HLP_NO_64BIT();
+    return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_ES);
+}
+
+
+/**
+ * @opcode      0x07
+ * @opgroup     og_stack_sreg
+ */
+FNIEMOP_DEF(iemOp_pop_ES)
+{
+    IEMOP_MNEMONIC1(FIXED, POP, pop, ES, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0);
+    IEMOP_HLP_NO_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_ES, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0x08
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ * @optest                  op1=7 op2=12 -> op1=15   efl&|=nc,po,na,nz,pl,nv
+ * @optest      efl|=of,cf  op1=0 op2=0  -> op1=0    efl&|=nc,po,na,zf,pl,nv
+ * @optest            op1=0xee op2=0x11  -> op1=0xff efl&|=nc,po,na,nz,ng,nv
+ * @optest            op1=0xff op2=0xff  -> op1=0xff efl&|=nc,po,na,nz,ng,nv
+ */
+FNIEMOP_DEF(iemOp_or_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, OR, or, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_or);
+}
+
+
+/*
+ * @opcode      0x09
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ * @optest      efl|=of,cf  op1=12 op2=7 -> op1=15   efl&|=nc,po,na,nz,pl,nv
+ * @optest      efl|=of,cf  op1=0 op2=0  -> op1=0    efl&|=nc,po,na,zf,pl,nv
+ * @optest      op1=-2 op2=1  -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o16 / op1=0x5a5a             op2=0xa5a5             -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o32 / op1=0x5a5a5a5a         op2=0xa5a5a5a5         -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o64 / op1=0x5a5a5a5a5a5a5a5a op2=0xa5a5a5a5a5a5a5a5 -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ */
+FNIEMOP_DEF(iemOp_or_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, OR, or, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_or);
+}
+
+
+/**
+ * @opcode      0x0a
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ * @opcopytests iemOp_or_Eb_Gb
+ */
+FNIEMOP_DEF(iemOp_or_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, OR, or, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_or);
+}
+
+
+/**
+ * @opcode      0x0b
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ * @opcopytests iemOp_or_Ev_Gv
+ */
+FNIEMOP_DEF(iemOp_or_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, OR, or, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_or);
+}
+
+
+/**
+ * @opcode      0x0c
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ * @opcopytests iemOp_or_Eb_Gb
+ */
+FNIEMOP_DEF(iemOp_or_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, OR, or, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_or);
+}
+
+
+/**
+ * @opcode      0x0d
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ * @optest      efl|=of,cf  op1=12 op2=7 -> op1=15   efl&|=nc,po,na,nz,pl,nv
+ * @optest      efl|=of,cf  op1=0 op2=0  -> op1=0    efl&|=nc,po,na,zf,pl,nv
+ * @optest      op1=-2 op2=1  -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o16 / op1=0x5a5a             op2=0xa5a5     -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o32 / op1=0x5a5a5a5a         op2=0xa5a5a5a5 -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o64 / op1=0x5a5a5a5a5a5a5a5a op2=0xa5a5a5a5 -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o64 / op1=0x5a5a5a5aa5a5a5a5 op2=0x5a5a5a5a -> op1=0x5a5a5a5affffffff efl&|=nc,po,na,nz,pl,nv
+ */
+FNIEMOP_DEF(iemOp_or_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, OR, or, rAX, Iz, DISOPTYPE_HARMLESS, 0);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_or);
+}
+
+
+/**
+ * @opcode      0x0e
+ * @opgroup     og_stack_sreg
+ */
+FNIEMOP_DEF(iemOp_push_CS)
+{
+    IEMOP_MNEMONIC1(FIXED, PUSH, push, CS, DISOPTYPE_HARMLESS | DISOPTYPE_POTENTIALLY_DANGEROUS | DISOPTYPE_INVALID_64, 0);
+    IEMOP_HLP_NO_64BIT();
+    return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_CS);
+}
+
+
+/**
+ * @opcode      0x0f
+ * @opmnemonic  EscTwo0f
+ * @openc       two0f
+ * @opdisenum   OP_2B_ESC
+ * @ophints     harmless
+ * @opgroup     og_escapes
+ */
+FNIEMOP_DEF(iemOp_2byteEscape)
+{
+#ifdef VBOX_STRICT
+    /* Sanity check the table the first time around. */
+    static bool s_fTested = false;
+    if (RT_LIKELY(s_fTested)) { /* likely */  }
+    else
+    {
+        s_fTested = true;
+        Assert(g_apfnTwoByteMap[0xbc * 4 + 0] == iemOp_bsf_Gv_Ev);
+        Assert(g_apfnTwoByteMap[0xbc * 4 + 1] == iemOp_bsf_Gv_Ev);
+        Assert(g_apfnTwoByteMap[0xbc * 4 + 2] == iemOp_tzcnt_Gv_Ev);
+        Assert(g_apfnTwoByteMap[0xbc * 4 + 3] == iemOp_bsf_Gv_Ev);
+    }
+#endif
+
+    if (RT_LIKELY(IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_286))
+    {
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        IEMOP_HLP_MIN_286();
+        return FNIEMOP_CALL(g_apfnTwoByteMap[(uintptr_t)b * 4 + pVCpu->iem.s.idxPrefix]);
+    }
+    /* @opdone */
+
+    /*
+     * On the 8086 this is a POP CS instruction.
+     * For the time being we don't specify this this.
+     */
+    IEMOP_MNEMONIC1(FIXED, POP, pop, CS, DISOPTYPE_HARMLESS | DISOPTYPE_POTENTIALLY_DANGEROUS | DISOPTYPE_INVALID_64, IEMOPHINT_SKIP_PYTHON);
+    IEMOP_HLP_NO_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_ES, pVCpu->iem.s.enmEffOpSize);
+}
+
+/**
+ * @opcode      0x10
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @optest      op1=1 op2=1 efl&~=cf -> op1=2 efl&|=nc,pe,na,nz,pl,nv
+ * @optest      op1=1 op2=1 efl|=cf  -> op1=3 efl&|=nc,po,na,nz,pl,nv
+ * @optest      op1=0xff op2=0 efl|=cf -> op1=0 efl&|=cf,po,af,zf,pl,nv
+ * @optest      op1=0  op2=0 efl|=cf -> op1=1 efl&|=nc,pe,na,nz,pl,nv
+ * @optest      op1=0  op2=0 efl&~=cf -> op1=0 efl&|=nc,po,na,zf,pl,nv
+ */
+FNIEMOP_DEF(iemOp_adc_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, ADC, adc, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_adc);
+}
+
+
+/**
+ * @opcode      0x11
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @optest      op1=1 op2=1 efl&~=cf -> op1=2 efl&|=nc,pe,na,nz,pl,nv
+ * @optest      op1=1 op2=1 efl|=cf  -> op1=3 efl&|=nc,po,na,nz,pl,nv
+ * @optest      op1=-1 op2=0 efl|=cf -> op1=0 efl&|=cf,po,af,zf,pl,nv
+ * @optest      op1=0  op2=0 efl|=cf -> op1=1 efl&|=nc,pe,na,nz,pl,nv
+ * @optest      op1=0  op2=0 efl&~=cf -> op1=0 efl&|=nc,po,na,zf,pl,nv
+ */
+FNIEMOP_DEF(iemOp_adc_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, ADC, adc, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_adc);
+}
+
+
+/**
+ * @opcode      0x12
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_adc_Eb_Gb
+ */
+FNIEMOP_DEF(iemOp_adc_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, ADC, adc, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_adc);
+}
+
+
+/**
+ * @opcode      0x13
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_adc_Ev_Gv
+ */
+FNIEMOP_DEF(iemOp_adc_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, ADC, adc, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_adc);
+}
+
+
+/**
+ * @opcode      0x14
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_adc_Eb_Gb
+ */
+FNIEMOP_DEF(iemOp_adc_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, ADC, adc, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_adc);
+}
+
+
+/**
+ * @opcode      0x15
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_adc_Ev_Gv
+ */
+FNIEMOP_DEF(iemOp_adc_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, ADC, adc, rAX, Iz, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_adc);
+}
+
+
+/**
+ * @opcode      0x16
+ */
+FNIEMOP_DEF(iemOp_push_SS)
+{
+    IEMOP_MNEMONIC1(FIXED, PUSH, push, SS, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64 | DISOPTYPE_RRM_DANGEROUS, 0);
+    IEMOP_HLP_NO_64BIT();
+    return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_SS);
+}
+
+
+/**
+ * @opcode      0x17
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_pop_SS)
+{
+    IEMOP_MNEMONIC1(FIXED, POP, pop, SS, DISOPTYPE_HARMLESS | DISOPTYPE_INHIBIT_IRQS | DISOPTYPE_INVALID_64 | DISOPTYPE_RRM_DANGEROUS , 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_NO_64BIT();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_SS, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0x18
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sbb_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, SBB, sbb, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_sbb);
+}
+
+
+/**
+ * @opcode      0x19
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sbb_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, SBB, sbb, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_sbb);
+}
+
+
+/**
+ * @opcode      0x1a
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sbb_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, SBB, sbb, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_sbb);
+}
+
+
+/**
+ * @opcode      0x1b
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sbb_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, SBB, sbb, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_sbb);
+}
+
+
+/**
+ * @opcode      0x1c
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sbb_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, SBB, sbb, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_sbb);
+}
+
+
+/**
+ * @opcode      0x1d
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sbb_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, SBB, sbb, rAX, Iz, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_sbb);
+}
+
+
+/**
+ * @opcode      0x1e
+ * @opgroup     og_stack_sreg
+ */
+FNIEMOP_DEF(iemOp_push_DS)
+{
+    IEMOP_MNEMONIC1(FIXED, PUSH, push, DS, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0);
+    IEMOP_HLP_NO_64BIT();
+    return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_DS);
+}
+
+
+/**
+ * @opcode      0x1f
+ * @opgroup     og_stack_sreg
+ */
+FNIEMOP_DEF(iemOp_pop_DS)
+{
+    IEMOP_MNEMONIC1(FIXED, POP, pop, DS, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64 | DISOPTYPE_RRM_DANGEROUS, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_NO_64BIT();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_DS, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0x20
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_and_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, AND, and, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_and);
+}
+
+
+/**
+ * @opcode      0x21
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_and_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, AND, and, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_and);
+}
+
+
+/**
+ * @opcode      0x22
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_and_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, AND, and, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_and);
+}
+
+
+/**
+ * @opcode      0x23
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_and_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, AND, and, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_and);
+}
+
+
+/**
+ * @opcode      0x24
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_and_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, AND, and, AL, Ib, DISOPTYPE_HARMLESS, 0);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_and);
+}
+
+
+/**
+ * @opcode      0x25
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_and_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, AND, and, rAX, Iz, DISOPTYPE_HARMLESS, 0);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_and);
+}
+
+
+/**
+ * @opcode      0x26
+ * @opmnemonic  SEG
+ * @op1         ES
+ * @opgroup     og_prefix
+ * @openc       prefix
+ * @opdisenum   OP_SEG
+ * @ophints     harmless
+ */
+FNIEMOP_DEF(iemOp_seg_ES)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg es");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_ES;
+    pVCpu->iem.s.iEffSeg    = X86_SREG_ES;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x27
+ * @opfltest    af,cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   of
+ */
+FNIEMOP_DEF(iemOp_daa)
+{
+    IEMOP_MNEMONIC0(FIXED, DAA, daa, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); /* express implicit AL register use */
+    IEMOP_HLP_NO_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF);
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_daa);
+}
+
+
+/**
+ * @opcode      0x28
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sub_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, SUB, sub, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_sub);
+}
+
+
+/**
+ * @opcode      0x29
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sub_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, SUB, sub, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_sub);
+}
+
+
+/**
+ * @opcode      0x2a
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sub_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, SUB, sub, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_sub);
+}
+
+
+/**
+ * @opcode      0x2b
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sub_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, SUB, sub, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_sub);
+}
+
+
+/**
+ * @opcode      0x2c
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sub_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, SUB, sub, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_sub);
+}
+
+
+/**
+ * @opcode      0x2d
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sub_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, SUB, sub, rAX, Iz, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_sub);
+}
+
+
+/**
+ * @opcode      0x2e
+ * @opmnemonic  SEG
+ * @op1         CS
+ * @opgroup     og_prefix
+ * @openc       prefix
+ * @opdisenum   OP_SEG
+ * @ophints     harmless
+ */
+FNIEMOP_DEF(iemOp_seg_CS)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg cs");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_CS;
+    pVCpu->iem.s.iEffSeg    = X86_SREG_CS;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x2f
+ * @opfltest    af,cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   of
+ */
+FNIEMOP_DEF(iemOp_das)
+{
+    IEMOP_MNEMONIC0(FIXED, DAS, das, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); /* express implicit AL register use */
+    IEMOP_HLP_NO_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF);
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_das);
+}
+
+
+/**
+ * @opcode      0x30
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_xor_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, XOR, xor, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_xor);
+}
+
+
+/**
+ * @opcode      0x31
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_xor_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, XOR, xor, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_xor);
+}
+
+
+/**
+ * @opcode      0x32
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_xor_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, XOR, xor, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_xor);
+}
+
+
+/**
+ * @opcode      0x33
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_xor_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, XOR, xor, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_xor);
+}
+
+
+/**
+ * @opcode      0x34
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_xor_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, XOR, xor, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_xor);
+}
+
+
+/**
+ * @opcode      0x35
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_xor_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, XOR, xor, rAX, Iz, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_xor);
+}
+
+
+/**
+ * @opcode      0x36
+ * @opmnemonic  SEG
+ * @op1         SS
+ * @opgroup     og_prefix
+ * @openc       prefix
+ * @opdisenum   OP_SEG
+ * @ophints     harmless
+ */
+FNIEMOP_DEF(iemOp_seg_SS)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg ss");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_SS;
+    pVCpu->iem.s.iEffSeg    = X86_SREG_SS;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x37
+ * @opfltest    af,cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   pf,zf,sf,of
+ * @opgroup     og_gen_arith_dec
+ * @optest              efl&~=af ax=9      -> efl&|=nc,po,na,nz,pl,nv
+ * @optest              efl&~=af ax=0      -> efl&|=nc,po,na,zf,pl,nv
+ * @optest      intel / efl&~=af ax=0x00f0 -> ax=0x0000 efl&|=nc,po,na,zf,pl,nv
+ * @optest      amd   / efl&~=af ax=0x00f0 -> ax=0x0000 efl&|=nc,po,na,nz,pl,nv
+ * @optest              efl&~=af ax=0x00f9 -> ax=0x0009 efl&|=nc,po,na,nz,pl,nv
+ * @optest              efl|=af  ax=0      -> ax=0x0106 efl&|=cf,po,af,nz,pl,nv
+ * @optest              efl|=af  ax=0x0100 -> ax=0x0206 efl&|=cf,po,af,nz,pl,nv
+ * @optest      intel / efl|=af  ax=0x000a -> ax=0x0100 efl&|=cf,po,af,zf,pl,nv
+ * @optest      amd   / efl|=af  ax=0x000a -> ax=0x0100 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl|=af  ax=0x010a -> ax=0x0200 efl&|=cf,po,af,zf,pl,nv
+ * @optest      amd   / efl|=af  ax=0x010a -> ax=0x0200 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl|=af  ax=0x0f0a -> ax=0x1000 efl&|=cf,po,af,zf,pl,nv
+ * @optest      amd   / efl|=af  ax=0x0f0a -> ax=0x1000 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl|=af  ax=0x7f0a -> ax=0x8000 efl&|=cf,po,af,zf,pl,nv
+ * @optest      amd   / efl|=af  ax=0x7f0a -> ax=0x8000 efl&|=cf,pe,af,nz,ng,ov
+ * @optest      intel / efl|=af  ax=0xff0a -> ax=0x0000 efl&|=cf,po,af,zf,pl,nv
+ * @optest      amd   / efl|=af  ax=0xff0a -> ax=0x0000 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0xff0a -> ax=0x0000 efl&|=cf,po,af,zf,pl,nv
+ * @optest      amd   / efl&~=af ax=0xff0a -> ax=0x0000 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0x000b -> ax=0x0101 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000b -> ax=0x0101 efl&|=cf,po,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0x000c -> ax=0x0102 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000c -> ax=0x0102 efl&|=cf,po,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0x000d -> ax=0x0103 efl&|=cf,po,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000d -> ax=0x0103 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0x000e -> ax=0x0104 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000e -> ax=0x0104 efl&|=cf,po,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0x000f -> ax=0x0105 efl&|=cf,po,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000f -> ax=0x0105 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0x020f -> ax=0x0305 efl&|=cf,po,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x020f -> ax=0x0305 efl&|=cf,pe,af,nz,pl,nv
+ */
+FNIEMOP_DEF(iemOp_aaa)
+{
+    IEMOP_MNEMONIC0(FIXED, AAA, aaa, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); /* express implicit AL/AX register use */
+    IEMOP_HLP_NO_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF);
+
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_aaa);
+}
+
+
+/**
+ * @opcode      0x38
+ */
+FNIEMOP_DEF(iemOp_cmp_Eb_Gb)
+{
+    IEMOP_MNEMONIC(cmp_Eb_Gb, "cmp Eb,Gb");
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_cmp);
+}
+
+
+/**
+ * @opcode      0x39
+ */
+FNIEMOP_DEF(iemOp_cmp_Ev_Gv)
+{
+    IEMOP_MNEMONIC(cmp_Ev_Gv, "cmp Ev,Gv");
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_cmp);
+}
+
+
+/**
+ * @opcode      0x3a
+ */
+FNIEMOP_DEF(iemOp_cmp_Gb_Eb)
+{
+    IEMOP_MNEMONIC(cmp_Gb_Eb, "cmp Gb,Eb");
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_cmp);
+}
+
+
+/**
+ * @opcode      0x3b
+ */
+FNIEMOP_DEF(iemOp_cmp_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmp_Gv_Ev, "cmp Gv,Ev");
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_cmp);
+}
+
+
+/**
+ * @opcode      0x3c
+ */
+FNIEMOP_DEF(iemOp_cmp_Al_Ib)
+{
+    IEMOP_MNEMONIC(cmp_al_Ib, "cmp al,Ib");
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_cmp);
+}
+
+
+/**
+ * @opcode      0x3d
+ */
+FNIEMOP_DEF(iemOp_cmp_eAX_Iz)
+{
+    IEMOP_MNEMONIC(cmp_rAX_Iz, "cmp rAX,Iz");
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_cmp);
+}
+
+
+/**
+ * @opcode      0x3e
+ */
+FNIEMOP_DEF(iemOp_seg_DS)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg ds");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_DS;
+    pVCpu->iem.s.iEffSeg    = X86_SREG_DS;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x3f
+ * @opfltest    af,cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   pf,zf,sf,of
+ * @opgroup     og_gen_arith_dec
+ * @optest            / efl&~=af ax=0x0009 -> efl&|=nc,po,na,nz,pl,nv
+ * @optest            / efl&~=af ax=0x0000 -> efl&|=nc,po,na,zf,pl,nv
+ * @optest      intel / efl&~=af ax=0x00f0 -> ax=0x0000 efl&|=nc,po,na,zf,pl,nv
+ * @optest      amd   / efl&~=af ax=0x00f0 -> ax=0x0000 efl&|=nc,po,na,nz,pl,nv
+ * @optest            / efl&~=af ax=0x00f9 -> ax=0x0009 efl&|=nc,po,na,nz,pl,nv
+ * @optest      intel / efl|=af  ax=0x0000 -> ax=0xfe0a efl&|=cf,po,af,nz,pl,nv
+ * @optest      amd   / efl|=af  ax=0x0000 -> ax=0xfe0a efl&|=cf,po,af,nz,ng,nv
+ * @optest      intel / efl|=af  ax=0x0100 -> ax=0xff0a efl&|=cf,po,af,nz,pl,nv
+ * @optest8     amd   / efl|=af  ax=0x0100 -> ax=0xff0a efl&|=cf,po,af,nz,ng,nv
+ * @optest      intel / efl|=af  ax=0x000a -> ax=0xff04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest10    amd   / efl|=af  ax=0x000a -> ax=0xff04 efl&|=cf,pe,af,nz,ng,nv
+ * @optest            / efl|=af  ax=0x010a -> ax=0x0004 efl&|=cf,pe,af,nz,pl,nv
+ * @optest            / efl|=af  ax=0x020a -> ax=0x0104 efl&|=cf,pe,af,nz,pl,nv
+ * @optest            / efl|=af  ax=0x0f0a -> ax=0x0e04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest            / efl|=af  ax=0x7f0a -> ax=0x7e04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl|=af  ax=0xff0a -> ax=0xfe04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl|=af  ax=0xff0a -> ax=0xfe04 efl&|=cf,pe,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0xff0a -> ax=0xfe04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0xff0a -> ax=0xfe04 efl&|=cf,pe,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0xff09 -> ax=0xff09 efl&|=nc,po,na,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0xff09 -> ax=0xff09 efl&|=nc,po,na,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0x000b -> ax=0xff05 efl&|=cf,po,af,nz,pl,nv
+ * @optest22    amd   / efl&~=af ax=0x000b -> ax=0xff05 efl&|=cf,po,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0x000c -> ax=0xff06 efl&|=cf,po,af,nz,pl,nv
+ * @optest24    amd   / efl&~=af ax=0x000c -> ax=0xff06 efl&|=cf,po,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0x000d -> ax=0xff07 efl&|=cf,pe,af,nz,pl,nv
+ * @optest26    amd   / efl&~=af ax=0x000d -> ax=0xff07 efl&|=cf,pe,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0x000e -> ax=0xff08 efl&|=cf,pe,af,nz,pl,nv
+ * @optest28    amd   / efl&~=af ax=0x000e -> ax=0xff08 efl&|=cf,pe,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0x000f -> ax=0xff09 efl&|=cf,po,af,nz,pl,nv
+ * @optest30    amd   / efl&~=af ax=0x000f -> ax=0xff09 efl&|=cf,po,af,nz,ng,nv
+ * @optest31    intel / efl&~=af ax=0x00fa -> ax=0xff04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest32    amd   / efl&~=af ax=0x00fa -> ax=0xff04 efl&|=cf,pe,af,nz,ng,nv
+ * @optest33    intel / efl&~=af ax=0xfffa -> ax=0xfe04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest34    amd   / efl&~=af ax=0xfffa -> ax=0xfe04 efl&|=cf,pe,af,nz,ng,nv
+ */
+FNIEMOP_DEF(iemOp_aas)
+{
+    IEMOP_MNEMONIC0(FIXED, AAS, aas, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); /* express implicit AL/AX register use */
+    IEMOP_HLP_NO_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_OF);
+
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_aas);
+}
+
+
+/**
+ * Common 'inc/dec/not/neg register' helper.
+ */
+FNIEMOP_DEF_2(iemOpCommonUnaryGReg, PCIEMOPUNARYSIZES, pImpl, uint8_t, iReg)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint16_t *,  pu16Dst, 0);
+            IEM_MC_ARG(uint32_t *,  pEFlags, 1);
+            IEM_MC_REF_GREG_U16(pu16Dst, iReg);
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU16, pu16Dst, pEFlags);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint32_t *,  pu32Dst, 0);
+            IEM_MC_ARG(uint32_t *,  pEFlags, 1);
+            IEM_MC_REF_GREG_U32(pu32Dst, iReg);
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU32, pu32Dst, pEFlags);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint64_t *,  pu64Dst, 0);
+            IEM_MC_ARG(uint32_t *,  pEFlags, 1);
+            IEM_MC_REF_GREG_U64(pu64Dst, iReg);
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU64, pu64Dst, pEFlags);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x40
+ */
+FNIEMOP_DEF(iemOp_inc_eAX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eAX, "inc eAX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xAX);
+}
+
+
+/**
+ * @opcode      0x41
+ */
+FNIEMOP_DEF(iemOp_inc_eCX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.b");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_B;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eCX, "inc eCX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xCX);
+}
+
+
+/**
+ * @opcode      0x42
+ */
+FNIEMOP_DEF(iemOp_inc_eDX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.x");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_X;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eDX, "inc eDX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xDX);
+}
+
+
+
+/**
+ * @opcode      0x43
+ */
+FNIEMOP_DEF(iemOp_inc_eBX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.bx");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eBX, "inc eBX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xBX);
+}
+
+
+/**
+ * @opcode      0x44
+ */
+FNIEMOP_DEF(iemOp_inc_eSP)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.r");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eSP, "inc eSP");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xSP);
+}
+
+
+/**
+ * @opcode      0x45
+ */
+FNIEMOP_DEF(iemOp_inc_eBP)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rb");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eBP, "inc eBP");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xBP);
+}
+
+
+/**
+ * @opcode      0x46
+ */
+FNIEMOP_DEF(iemOp_inc_eSI)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rx");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_X;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eSI, "inc eSI");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xSI);
+}
+
+
+/**
+ * @opcode      0x47
+ */
+FNIEMOP_DEF(iemOp_inc_eDI)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rbx");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eDI, "inc eDI");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xDI);
+}
+
+
+/**
+ * @opcode      0x48
+ */
+FNIEMOP_DEF(iemOp_dec_eAX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.w");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_SIZE_REX_W;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eAX, "dec eAX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xAX);
+}
+
+
+/**
+ * @opcode      0x49
+ */
+FNIEMOP_DEF(iemOp_dec_eCX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.bw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_B | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eCX, "dec eCX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xCX);
+}
+
+
+/**
+ * @opcode      0x4a
+ */
+FNIEMOP_DEF(iemOp_dec_eDX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.xw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eDX, "dec eDX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xDX);
+}
+
+
+/**
+ * @opcode      0x4b
+ */
+FNIEMOP_DEF(iemOp_dec_eBX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.bxw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eBX, "dec eBX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xBX);
+}
+
+
+/**
+ * @opcode      0x4c
+ */
+FNIEMOP_DEF(iemOp_dec_eSP)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eSP, "dec eSP");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xSP);
+}
+
+
+/**
+ * @opcode      0x4d
+ */
+FNIEMOP_DEF(iemOp_dec_eBP)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rbw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eBP, "dec eBP");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xBP);
+}
+
+
+/**
+ * @opcode      0x4e
+ */
+FNIEMOP_DEF(iemOp_dec_eSI)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rxw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eSI, "dec eSI");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xSI);
+}
+
+
+/**
+ * @opcode      0x4f
+ */
+FNIEMOP_DEF(iemOp_dec_eDI)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rbxw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eDI, "dec eDI");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xDI);
+}
+
+
+/**
+ * Common 'push register' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonPushGReg, uint8_t, iReg)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        iReg |= pVCpu->iem.s.uRexB;
+        pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
+        pVCpu->iem.s.enmEffOpSize = !(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP) ? IEMMODE_64BIT : IEMMODE_16BIT;
+    }
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint16_t, u16Value);
+            IEM_MC_FETCH_GREG_U16(u16Value, iReg);
+            IEM_MC_PUSH_U16(u16Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Value);
+            IEM_MC_FETCH_GREG_U32(u32Value, iReg);
+            IEM_MC_PUSH_U32(u32Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_FETCH_GREG_U64(u64Value, iReg);
+            IEM_MC_PUSH_U64(u64Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x50
+ */
+FNIEMOP_DEF(iemOp_push_eAX)
+{
+    IEMOP_MNEMONIC(push_rAX, "push rAX");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xAX);
+}
+
+
+/**
+ * @opcode      0x51
+ */
+FNIEMOP_DEF(iemOp_push_eCX)
+{
+    IEMOP_MNEMONIC(push_rCX, "push rCX");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xCX);
+}
+
+
+/**
+ * @opcode      0x52
+ */
+FNIEMOP_DEF(iemOp_push_eDX)
+{
+    IEMOP_MNEMONIC(push_rDX, "push rDX");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xDX);
+}
+
+
+/**
+ * @opcode      0x53
+ */
+FNIEMOP_DEF(iemOp_push_eBX)
+{
+    IEMOP_MNEMONIC(push_rBX, "push rBX");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xBX);
+}
+
+
+/**
+ * @opcode      0x54
+ */
+FNIEMOP_DEF(iemOp_push_eSP)
+{
+    IEMOP_MNEMONIC(push_rSP, "push rSP");
+    if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_8086)
+    {
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint16_t, u16Value);
+        IEM_MC_FETCH_GREG_U16(u16Value, X86_GREG_xSP);
+        IEM_MC_SUB_LOCAL_U16(u16Value, 2);
+        IEM_MC_PUSH_U16(u16Value);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xSP);
+}
+
+
+/**
+ * @opcode      0x55
+ */
+FNIEMOP_DEF(iemOp_push_eBP)
+{
+    IEMOP_MNEMONIC(push_rBP, "push rBP");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xBP);
+}
+
+
+/**
+ * @opcode      0x56
+ */
+FNIEMOP_DEF(iemOp_push_eSI)
+{
+    IEMOP_MNEMONIC(push_rSI, "push rSI");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xSI);
+}
+
+
+/**
+ * @opcode      0x57
+ */
+FNIEMOP_DEF(iemOp_push_eDI)
+{
+    IEMOP_MNEMONIC(push_rDI, "push rDI");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xDI);
+}
+
+
+/**
+ * Common 'pop register' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonPopGReg, uint8_t, iReg)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        iReg |= pVCpu->iem.s.uRexB;
+        pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
+        pVCpu->iem.s.enmEffOpSize = !(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP) ? IEMMODE_64BIT : IEMMODE_16BIT;
+    }
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint16_t *, pu16Dst);
+            IEM_MC_REF_GREG_U16(pu16Dst, iReg);
+            IEM_MC_POP_U16(pu16Dst);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t *, pu32Dst);
+            IEM_MC_REF_GREG_U32(pu32Dst, iReg);
+            IEM_MC_POP_U32(pu32Dst);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); /** @todo testcase*/
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t *, pu64Dst);
+            IEM_MC_REF_GREG_U64(pu64Dst, iReg);
+            IEM_MC_POP_U64(pu64Dst);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x58
+ */
+FNIEMOP_DEF(iemOp_pop_eAX)
+{
+    IEMOP_MNEMONIC(pop_rAX, "pop rAX");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xAX);
+}
+
+
+/**
+ * @opcode      0x59
+ */
+FNIEMOP_DEF(iemOp_pop_eCX)
+{
+    IEMOP_MNEMONIC(pop_rCX, "pop rCX");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xCX);
+}
+
+
+/**
+ * @opcode      0x5a
+ */
+FNIEMOP_DEF(iemOp_pop_eDX)
+{
+    IEMOP_MNEMONIC(pop_rDX, "pop rDX");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xDX);
+}
+
+
+/**
+ * @opcode      0x5b
+ */
+FNIEMOP_DEF(iemOp_pop_eBX)
+{
+    IEMOP_MNEMONIC(pop_rBX, "pop rBX");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xBX);
+}
+
+
+/**
+ * @opcode      0x5c
+ */
+FNIEMOP_DEF(iemOp_pop_eSP)
+{
+    IEMOP_MNEMONIC(pop_rSP, "pop rSP");
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        if (pVCpu->iem.s.uRexB)
+            return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xSP);
+        pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
+        pVCpu->iem.s.enmEffOpSize = !(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP) ? IEMMODE_64BIT : IEMMODE_16BIT;
+    }
+
+    IEMOP_HLP_DECODED_NL_1(OP_POP, IEMOPFORM_FIXED, OP_PARM_REG_ESP,
+                           DISOPTYPE_HARMLESS | DISOPTYPE_DEFAULT_64_OP_SIZE | DISOPTYPE_REXB_EXTENDS_OPREG);
+    /** @todo add testcase for this instruction. */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint16_t, u16Dst);
+            IEM_MC_POP_U16(&u16Dst); /** @todo not correct MC, fix later. */
+            IEM_MC_STORE_GREG_U16(X86_GREG_xSP, u16Dst);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Dst);
+            IEM_MC_POP_U32(&u32Dst);
+            IEM_MC_STORE_GREG_U32(X86_GREG_xSP, u32Dst);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Dst);
+            IEM_MC_POP_U64(&u64Dst);
+            IEM_MC_STORE_GREG_U64(X86_GREG_xSP, u64Dst);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x5d
+ */
+FNIEMOP_DEF(iemOp_pop_eBP)
+{
+    IEMOP_MNEMONIC(pop_rBP, "pop rBP");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xBP);
+}
+
+
+/**
+ * @opcode      0x5e
+ */
+FNIEMOP_DEF(iemOp_pop_eSI)
+{
+    IEMOP_MNEMONIC(pop_rSI, "pop rSI");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xSI);
+}
+
+
+/**
+ * @opcode      0x5f
+ */
+FNIEMOP_DEF(iemOp_pop_eDI)
+{
+    IEMOP_MNEMONIC(pop_rDI, "pop rDI");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xDI);
+}
+
+
+/**
+ * @opcode      0x60
+ */
+FNIEMOP_DEF(iemOp_pusha)
+{
+    IEMOP_MNEMONIC(pusha, "pusha");
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_NO_64BIT();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+        return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_pusha_16);
+    Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT);
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_pusha_32);
+}
+
+
+/**
+ * @opcode      0x61
+ */
+FNIEMOP_DEF(iemOp_popa__mvex)
+{
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
+    {
+        IEMOP_MNEMONIC(popa, "popa");
+        IEMOP_HLP_MIN_186();
+        IEMOP_HLP_NO_64BIT();
+        if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+            return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_popa_16);
+        Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT);
+        return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_popa_32);
+    }
+    IEMOP_MNEMONIC(mvex, "mvex");
+    Log(("mvex prefix is not supported!\n"));
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x62
+ * @opmnemonic  bound
+ * @op1         Gv_RO
+ * @op2         Ma
+ * @opmincpu    80186
+ * @ophints     harmless invalid_64
+ * @optest      op1=0 op2=0 ->
+ * @optest      op1=1 op2=0 -> value.xcpt=5
+ * @optest      o16 / op1=0xffff op2=0x0000fffe ->
+ * @optest      o16 / op1=0xfffe op2=0x0000fffe ->
+ * @optest      o16 / op1=0x7fff op2=0x0000fffe -> value.xcpt=5
+ * @optest      o16 / op1=0x7fff op2=0x7ffffffe ->
+ * @optest      o16 / op1=0x7fff op2=0xfffe8000 -> value.xcpt=5
+ * @optest      o16 / op1=0x8000 op2=0xfffe8000 ->
+ * @optest      o16 / op1=0xffff op2=0xfffe8000 -> value.xcpt=5
+ * @optest      o16 / op1=0xfffe op2=0xfffe8000 ->
+ * @optest      o16 / op1=0xfffe op2=0x8000fffe -> value.xcpt=5
+ * @optest      o16 / op1=0x8000 op2=0x8000fffe -> value.xcpt=5
+ * @optest      o16 / op1=0x0000 op2=0x8000fffe -> value.xcpt=5
+ * @optest      o16 / op1=0x0001 op2=0x8000fffe -> value.xcpt=5
+ * @optest      o16 / op1=0xffff op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0000 op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0001 op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0002 op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0003 op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0004 op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x000e op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x000f op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0010 op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0011 op2=0x0001000f -> value.xcpt=5
+ * @optest      o32 / op1=0xffffffff op2=0x00000000fffffffe ->
+ * @optest      o32 / op1=0xfffffffe op2=0x00000000fffffffe ->
+ * @optest      o32 / op1=0x7fffffff op2=0x00000000fffffffe -> value.xcpt=5
+ * @optest      o32 / op1=0x7fffffff op2=0x7ffffffffffffffe ->
+ * @optest      o32 / op1=0x7fffffff op2=0xfffffffe80000000 -> value.xcpt=5
+ * @optest      o32 / op1=0x80000000 op2=0xfffffffe80000000 ->
+ * @optest      o32 / op1=0xffffffff op2=0xfffffffe80000000 -> value.xcpt=5
+ * @optest      o32 / op1=0xfffffffe op2=0xfffffffe80000000 ->
+ * @optest      o32 / op1=0xfffffffe op2=0x80000000fffffffe -> value.xcpt=5
+ * @optest      o32 / op1=0x80000000 op2=0x80000000fffffffe -> value.xcpt=5
+ * @optest      o32 / op1=0x00000000 op2=0x80000000fffffffe -> value.xcpt=5
+ * @optest      o32 / op1=0x00000002 op2=0x80000000fffffffe -> value.xcpt=5
+ * @optest      o32 / op1=0x00000001 op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x00000002 op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x00000003 op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x00000004 op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x00000005 op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x0000000e op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x0000000f op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x00000010 op2=0x0000000100000003 -> value.xcpt=5
+ */
+FNIEMOP_DEF(iemOp_bound_Gv_Ma__evex)
+{
+    /* The BOUND instruction is invalid 64-bit mode. In legacy and
+       compatability mode it is invalid with MOD=3.
+
+       In 32-bit mode, the EVEX prefix works by having the top two bits (MOD)
+       both be set.  In the Intel EVEX documentation (sdm vol 2) these are simply
+       given as R and X without an exact description, so we assume it builds on
+       the VEX one and means they are inverted wrt REX.R and REX.X.  Thus, just
+       like with the 3-byte VEX, 32-bit code is restrict wrt addressable registers. */
+    uint8_t bRm;
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
+    {
+        IEMOP_MNEMONIC2(RM_MEM, BOUND, bound, Gv_RO, Ma, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+        IEMOP_HLP_MIN_186();
+        IEM_OPCODE_GET_NEXT_U8(&bRm);
+        if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /** @todo testcase: check that there are two memory accesses involved.  Check
+             *        whether they're both read before the \#BR triggers. */
+            if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+            {
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t,    u16Index,       0); /* Note! All operands are actually signed. Lazy unsigned bird. */
+                IEM_MC_ARG(uint16_t,    u16LowerBounds, 1);
+                IEM_MC_ARG(uint16_t,    u16UpperBounds, 2);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_FETCH_GREG_U16(u16Index, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+                IEM_MC_FETCH_MEM_U16(u16LowerBounds, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_FETCH_MEM_U16_DISP(u16UpperBounds, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 2);
+
+                IEM_MC_CALL_CIMPL_3(iemCImpl_bound_16, u16Index, u16LowerBounds, u16UpperBounds); /* returns */
+                IEM_MC_END();
+            }
+            else /* 32-bit operands */
+            {
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint32_t,    u32Index,       0); /* Note! All operands are actually signed. Lazy unsigned bird. */
+                IEM_MC_ARG(uint32_t,    u32LowerBounds, 1);
+                IEM_MC_ARG(uint32_t,    u32UpperBounds, 2);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_FETCH_GREG_U32(u32Index, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+                IEM_MC_FETCH_MEM_U32(u32LowerBounds, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_FETCH_MEM_U32_DISP(u32UpperBounds, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 4);
+
+                IEM_MC_CALL_CIMPL_3(iemCImpl_bound_32, u32Index, u32LowerBounds, u32UpperBounds); /* returns */
+                IEM_MC_END();
+            }
+        }
+
+        /*
+         * @opdone
+         */
+        if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx512Foundation)
+        {
+            /* Note that there is no need for the CPU to fetch further bytes
+               here because MODRM.MOD == 3. */
+            Log(("evex not supported by the guest CPU!\n"));
+            return IEMOP_RAISE_INVALID_OPCODE();
+        }
+    }
+    else
+    {
+        /** @todo check how this is decoded in 64-bit mode w/o EVEX. Intel probably
+         *        does modr/m read, whereas AMD probably doesn't... */
+        if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx512Foundation)
+        {
+            Log(("evex not supported by the guest CPU!\n"));
+            return FNIEMOP_CALL(iemOp_InvalidAllNeedRM);
+        }
+        IEM_OPCODE_GET_NEXT_U8(&bRm);
+    }
+
+    IEMOP_MNEMONIC(evex, "evex");
+    uint8_t bP2; IEM_OPCODE_GET_NEXT_U8(&bP2);
+    uint8_t bP3; IEM_OPCODE_GET_NEXT_U8(&bP3);
+    Log(("evex prefix is not implemented!\n"));
+    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+}
+
+
+/** Opcode 0x63 - non-64-bit modes. */
+FNIEMOP_DEF(iemOp_arpl_Ew_Gw)
+{
+    IEMOP_MNEMONIC(arpl_Ew_Gw, "arpl Ew,Gw");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* Register */
+        IEMOP_HLP_DECODED_NL_2(OP_ARPL, IEMOPFORM_MR_REG, OP_PARM_Ew, OP_PARM_Gw, DISOPTYPE_HARMLESS);
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint16_t *,      pu16Dst,    0);
+        IEM_MC_ARG(uint16_t,        u16Src,     1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,    2);
+
+        IEM_MC_FETCH_GREG_U16(u16Src, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_arpl, pu16Dst, u16Src, pEFlags);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* Memory */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint16_t *, pu16Dst,          0);
+        IEM_MC_ARG(uint16_t,   u16Src,           1);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DECODED_NL_2(OP_ARPL, IEMOPFORM_MR_REG, OP_PARM_Ew, OP_PARM_Gw, DISOPTYPE_HARMLESS);
+        IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_GREG_U16(u16Src, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_arpl, pu16Dst, u16Src, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+
+}
+
+
+/**
+ * @opcode 0x63
+ *
+ * @note This is a weird one. It works like a regular move instruction if
+ *       REX.W isn't set, at least according to AMD docs (rev 3.15, 2009-11).
+ * @todo This definitely needs a testcase to verify the odd cases.  */
+FNIEMOP_DEF(iemOp_movsxd_Gv_Ev)
+{
+    Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT); /* Caller branched already . */
+
+    IEMOP_MNEMONIC(movsxd_Gv_Ev, "movsxd Gv,Ev");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register to register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t, u64Value);
+        IEM_MC_FETCH_GREG_U32_SX_U64(u64Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Value);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t, u64Value);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_MEM_U32_SX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+        IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Value);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x64
+ * @opmnemonic  segfs
+ * @opmincpu    80386
+ * @opgroup     og_prefixes
+ */
+FNIEMOP_DEF(iemOp_seg_FS)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg fs");
+    IEMOP_HLP_MIN_386();
+
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_FS;
+    pVCpu->iem.s.iEffSeg    = X86_SREG_FS;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x65
+ * @opmnemonic  seggs
+ * @opmincpu    80386
+ * @opgroup     og_prefixes
+ */
+FNIEMOP_DEF(iemOp_seg_GS)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg gs");
+    IEMOP_HLP_MIN_386();
+
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_GS;
+    pVCpu->iem.s.iEffSeg    = X86_SREG_GS;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x66
+ * @opmnemonic  opsize
+ * @openc       prefix
+ * @opmincpu    80386
+ * @ophints     harmless
+ * @opgroup     og_prefixes
+ */
+FNIEMOP_DEF(iemOp_op_size)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("op size");
+    IEMOP_HLP_MIN_386();
+
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_OP;
+    iemRecalEffOpSize(pVCpu);
+
+    /* For the 4 entry opcode tables, the operand prefix doesn't not count
+       when REPZ or REPNZ are present. */
+    if (pVCpu->iem.s.idxPrefix == 0)
+        pVCpu->iem.s.idxPrefix = 1;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x67
+ * @opmnemonic  addrsize
+ * @openc       prefix
+ * @opmincpu    80386
+ * @ophints     harmless
+ * @opgroup     og_prefixes
+ */
+FNIEMOP_DEF(iemOp_addr_size)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("addr size");
+    IEMOP_HLP_MIN_386();
+
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_ADDR;
+    switch (pVCpu->iem.s.enmDefAddrMode)
+    {
+        case IEMMODE_16BIT: pVCpu->iem.s.enmEffAddrMode = IEMMODE_32BIT; break;
+        case IEMMODE_32BIT: pVCpu->iem.s.enmEffAddrMode = IEMMODE_16BIT; break;
+        case IEMMODE_64BIT: pVCpu->iem.s.enmEffAddrMode = IEMMODE_32BIT; break;
+        default: AssertFailed();
+    }
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x68
+ */
+FNIEMOP_DEF(iemOp_push_Iz)
+{
+    IEMOP_MNEMONIC(push_Iz, "push Iz");
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_PUSH_U16(u16Imm);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_PUSH_U32(u32Imm);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_PUSH_U64(u64Imm);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x69
+ */
+FNIEMOP_DEF(iemOp_imul_Gv_Ev_Iz)
+{
+    IEMOP_MNEMONIC(imul_Gv_Ev_Iz, "imul Gv,Ev,Iz"); /* Gv = Ev * Iz; */
+    IEMOP_HLP_MIN_186();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+            {
+                /* register operand */
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,            0);
+                IEM_MC_ARG_CONST(uint16_t,  u16Src,/*=*/ u16Imm,1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+                IEM_MC_LOCAL(uint16_t,      u16Tmp);
+
+                IEM_MC_FETCH_GREG_U16(u16Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_LOCAL(pu16Dst, u16Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_imul_two_u16, pu16Dst, u16Src, pEFlags);
+                IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Tmp);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory operand */
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,            0);
+                IEM_MC_ARG(uint16_t,        u16Src,             1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+                IEM_MC_LOCAL(uint16_t,      u16Tmp);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 2);
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEM_MC_ASSIGN(u16Src, u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_LOCAL(pu16Dst, u16Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_imul_two_u16, pu16Dst, u16Src, pEFlags);
+                IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Tmp);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+            {
+                /* register operand */
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,            0);
+                IEM_MC_ARG_CONST(uint32_t,  u32Src,/*=*/ u32Imm,1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+                IEM_MC_LOCAL(uint32_t,      u32Tmp);
+
+                IEM_MC_FETCH_GREG_U32(u32Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_LOCAL(pu32Dst, u32Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_imul_two_u32, pu32Dst, u32Src, pEFlags);
+                IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Tmp);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory operand */
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,            0);
+                IEM_MC_ARG(uint32_t,        u32Src,             1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+                IEM_MC_LOCAL(uint32_t,      u32Tmp);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEM_MC_ASSIGN(u32Src, u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_LOCAL(pu32Dst, u32Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_imul_two_u32, pu32Dst, u32Src, pEFlags);
+                IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Tmp);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+            {
+                /* register operand */
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,            0);
+                IEM_MC_ARG_CONST(uint64_t,  u64Src,/*=*/ u64Imm,1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+                IEM_MC_LOCAL(uint64_t,      u64Tmp);
+
+                IEM_MC_FETCH_GREG_U64(u64Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_LOCAL(pu64Dst, u64Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_imul_two_u64, pu64Dst, u64Src, pEFlags);
+                IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Tmp);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory operand */
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,            0);
+                IEM_MC_ARG(uint64_t,        u64Src,             1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+                IEM_MC_LOCAL(uint64_t,      u64Tmp);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEM_MC_ASSIGN(u64Src, u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_LOCAL(pu64Dst, u64Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_imul_two_u64, pu64Dst, u64Src, pEFlags);
+                IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Tmp);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            return VINF_SUCCESS;
+        }
+    }
+    AssertFailedReturn(VERR_IEM_IPE_9);
+}
+
+
+/**
+ * @opcode      0x6a
+ */
+FNIEMOP_DEF(iemOp_push_Ib)
+{
+    IEMOP_MNEMONIC(push_Ib, "push Ib");
+    IEMOP_HLP_MIN_186();
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0,0);
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_PUSH_U16(i8Imm);
+            break;
+        case IEMMODE_32BIT:
+            IEM_MC_PUSH_U32(i8Imm);
+            break;
+        case IEMMODE_64BIT:
+            IEM_MC_PUSH_U64(i8Imm);
+            break;
+    }
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x6b
+ */
+FNIEMOP_DEF(iemOp_imul_Gv_Ev_Ib)
+{
+    IEMOP_MNEMONIC(imul_Gv_Ev_Ib, "imul Gv,Ev,Ib"); /* Gv = Ev * Iz; */
+    IEMOP_HLP_MIN_186();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+            {
+                /* register operand */
+                uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                    0);
+                IEM_MC_ARG_CONST(uint16_t,  u16Src,/*=*/ (int8_t)u8Imm, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+                IEM_MC_LOCAL(uint16_t,      u16Tmp);
+
+                IEM_MC_FETCH_GREG_U16(u16Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_LOCAL(pu16Dst, u16Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_imul_two_u16, pu16Dst, u16Src, pEFlags);
+                IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Tmp);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory operand */
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                    0);
+                IEM_MC_ARG(uint16_t,        u16Src,                     1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+                IEM_MC_LOCAL(uint16_t,      u16Tmp);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16Imm);
+                IEM_MC_ASSIGN(u16Src, u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_LOCAL(pu16Dst, u16Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_imul_two_u16, pu16Dst, u16Src, pEFlags);
+                IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Tmp);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+            {
+                /* register operand */
+                uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                    0);
+                IEM_MC_ARG_CONST(uint32_t,  u32Src,/*=*/ (int8_t)u8Imm, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+                IEM_MC_LOCAL(uint32_t,      u32Tmp);
+
+                IEM_MC_FETCH_GREG_U32(u32Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_LOCAL(pu32Dst, u32Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_imul_two_u32, pu32Dst, u32Src, pEFlags);
+                IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Tmp);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory operand */
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                    0);
+                IEM_MC_ARG(uint32_t,        u32Src,                     1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+                IEM_MC_LOCAL(uint32_t,      u32Tmp);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_S8_SX_U32(&u32Imm);
+                IEM_MC_ASSIGN(u32Src, u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_LOCAL(pu32Dst, u32Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_imul_two_u32, pu32Dst, u32Src, pEFlags);
+                IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Tmp);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+            {
+                /* register operand */
+                uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                    0);
+                IEM_MC_ARG_CONST(uint64_t,  u64Src,/*=*/ (int8_t)u8Imm, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+                IEM_MC_LOCAL(uint64_t,      u64Tmp);
+
+                IEM_MC_FETCH_GREG_U64(u64Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_LOCAL(pu64Dst, u64Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_imul_two_u64, pu64Dst, u64Src, pEFlags);
+                IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Tmp);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory operand */
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                    0);
+                IEM_MC_ARG(uint64_t,        u64Src,                     1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+                IEM_MC_LOCAL(uint64_t,      u64Tmp);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S8_SX_U64(&u64Imm);
+                IEM_MC_ASSIGN(u64Src, u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_LOCAL(pu64Dst, u64Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_imul_two_u64, pu64Dst, u64Src, pEFlags);
+                IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Tmp);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            return VINF_SUCCESS;
+    }
+    AssertFailedReturn(VERR_IEM_IPE_8);
+}
+
+
+/**
+ * @opcode      0x6c
+ */
+FNIEMOP_DEF(iemOp_insb_Yb_DX)
+{
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_insb_Yb_DX, "rep ins Yb,DX");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op8_addr16, false);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op8_addr32, false);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op8_addr64, false);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        IEMOP_MNEMONIC(ins_Yb_DX, "ins Yb,DX");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op8_addr16, false);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op8_addr32, false);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op8_addr64, false);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x6d
+ */
+FNIEMOP_DEF(iemOp_inswd_Yv_DX)
+{
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ))
+    {
+        IEMOP_MNEMONIC(rep_ins_Yv_DX, "rep ins Yv,DX");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op16_addr16, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op16_addr32, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op16_addr64, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_64BIT:
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op32_addr16, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op32_addr32, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op32_addr64, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        IEMOP_MNEMONIC(ins_Yv_DX, "ins Yv,DX");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op16_addr16, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op16_addr32, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op16_addr64, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_64BIT:
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op32_addr16, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op32_addr32, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op32_addr64, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x6e
+ */
+FNIEMOP_DEF(iemOp_outsb_Yb_DX)
+{
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_outsb_DX_Yb, "rep outs DX,Yb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op8_addr16, pVCpu->iem.s.iEffSeg, false);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op8_addr32, pVCpu->iem.s.iEffSeg, false);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op8_addr64, pVCpu->iem.s.iEffSeg, false);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        IEMOP_MNEMONIC(outs_DX_Yb, "outs DX,Yb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op8_addr16, pVCpu->iem.s.iEffSeg, false);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op8_addr32, pVCpu->iem.s.iEffSeg, false);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op8_addr64, pVCpu->iem.s.iEffSeg, false);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x6f
+ */
+FNIEMOP_DEF(iemOp_outswd_Yv_DX)
+{
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ))
+    {
+        IEMOP_MNEMONIC(rep_outs_DX_Yv, "rep outs DX,Yv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op16_addr16, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op16_addr32, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op16_addr64, pVCpu->iem.s.iEffSeg, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_64BIT:
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op32_addr16, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op32_addr32, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op32_addr64, pVCpu->iem.s.iEffSeg, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        IEMOP_MNEMONIC(outs_DX_Yv, "outs DX,Yv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op16_addr16, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op16_addr32, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op16_addr64, pVCpu->iem.s.iEffSeg, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_64BIT:
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op32_addr16, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op32_addr32, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op32_addr64, pVCpu->iem.s.iEffSeg, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x70
+ */
+FNIEMOP_DEF(iemOp_jo_Jb)
+{
+    IEMOP_MNEMONIC(jo_Jb, "jo  Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x71
+ */
+FNIEMOP_DEF(iemOp_jno_Jb)
+{
+    IEMOP_MNEMONIC(jno_Jb, "jno Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opcode      0x72
+ */
+FNIEMOP_DEF(iemOp_jc_Jb)
+{
+    IEMOP_MNEMONIC(jc_Jb, "jc/jnae Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x73
+ */
+FNIEMOP_DEF(iemOp_jnc_Jb)
+{
+    IEMOP_MNEMONIC(jnc_Jb, "jnc/jnb Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x74
+ */
+FNIEMOP_DEF(iemOp_je_Jb)
+{
+    IEMOP_MNEMONIC(je_Jb, "je/jz   Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x75
+ */
+FNIEMOP_DEF(iemOp_jne_Jb)
+{
+    IEMOP_MNEMONIC(jne_Jb, "jne/jnz Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x76
+ */
+FNIEMOP_DEF(iemOp_jbe_Jb)
+{
+    IEMOP_MNEMONIC(jbe_Jb, "jbe/jna Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x77
+ */
+FNIEMOP_DEF(iemOp_jnbe_Jb)
+{
+    IEMOP_MNEMONIC(ja_Jb, "ja/jnbe Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x78
+ */
+FNIEMOP_DEF(iemOp_js_Jb)
+{
+    IEMOP_MNEMONIC(js_Jb, "js  Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x79
+ */
+FNIEMOP_DEF(iemOp_jns_Jb)
+{
+    IEMOP_MNEMONIC(jns_Jb, "jns Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x7a
+ */
+FNIEMOP_DEF(iemOp_jp_Jb)
+{
+    IEMOP_MNEMONIC(jp_Jb, "jp  Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x7b
+ */
+FNIEMOP_DEF(iemOp_jnp_Jb)
+{
+    IEMOP_MNEMONIC(jnp_Jb, "jnp Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x7c
+ */
+FNIEMOP_DEF(iemOp_jl_Jb)
+{
+    IEMOP_MNEMONIC(jl_Jb, "jl/jnge Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x7d
+ */
+FNIEMOP_DEF(iemOp_jnl_Jb)
+{
+    IEMOP_MNEMONIC(jge_Jb, "jnl/jge Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x7e
+ */
+FNIEMOP_DEF(iemOp_jle_Jb)
+{
+    IEMOP_MNEMONIC(jle_Jb, "jle/jng Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x7f
+ */
+FNIEMOP_DEF(iemOp_jnle_Jb)
+{
+    IEMOP_MNEMONIC(jg_Jb, "jnle/jg Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+        IEM_MC_ADVANCE_RIP();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x80
+ */
+FNIEMOP_DEF(iemOp_Grp1_Eb_Ib_80)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0: IEMOP_MNEMONIC(add_Eb_Ib, "add Eb,Ib"); break;
+        case 1: IEMOP_MNEMONIC(or_Eb_Ib,  "or  Eb,Ib"); break;
+        case 2: IEMOP_MNEMONIC(adc_Eb_Ib, "adc Eb,Ib"); break;
+        case 3: IEMOP_MNEMONIC(sbb_Eb_Ib, "sbb Eb,Ib"); break;
+        case 4: IEMOP_MNEMONIC(and_Eb_Ib, "and Eb,Ib"); break;
+        case 5: IEMOP_MNEMONIC(sub_Eb_Ib, "sub Eb,Ib"); break;
+        case 6: IEMOP_MNEMONIC(xor_Eb_Ib, "xor Eb,Ib"); break;
+        case 7: IEMOP_MNEMONIC(cmp_Eb_Ib, "cmp Eb,Ib"); break;
+    }
+    PCIEMOPBINSIZES pImpl = g_apIemImplGrp1[(bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK];
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,                 0);
+        IEM_MC_ARG_CONST(uint8_t,   u8Src, /*=*/ u8Imm,     1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+        IEM_MC_REF_GREG_U8(pu8Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        uint32_t fAccess;
+        if (pImpl->pfnLockedU8)
+            fAccess = IEM_ACCESS_DATA_RW;
+        else /* CMP */
+            fAccess = IEM_ACCESS_DATA_R;
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,                 0);
+        IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        IEM_MC_ARG_CONST(uint8_t,   u8Src, /*=*/ u8Imm,     1);
+        if (pImpl->pfnLockedU8)
+            IEMOP_HLP_DONE_DECODING();
+        else
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_MEM_MAP(pu8Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+        else
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, fAccess);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x81
+ */
+FNIEMOP_DEF(iemOp_Grp1_Ev_Iz)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0: IEMOP_MNEMONIC(add_Ev_Iz, "add Ev,Iz"); break;
+        case 1: IEMOP_MNEMONIC(or_Ev_Iz,  "or  Ev,Iz"); break;
+        case 2: IEMOP_MNEMONIC(adc_Ev_Iz, "adc Ev,Iz"); break;
+        case 3: IEMOP_MNEMONIC(sbb_Ev_Iz, "sbb Ev,Iz"); break;
+        case 4: IEMOP_MNEMONIC(and_Ev_Iz, "and Ev,Iz"); break;
+        case 5: IEMOP_MNEMONIC(sub_Ev_Iz, "sub Ev,Iz"); break;
+        case 6: IEMOP_MNEMONIC(xor_Ev_Iz, "xor Ev,Iz"); break;
+        case 7: IEMOP_MNEMONIC(cmp_Ev_Iz, "cmp Ev,Iz"); break;
+    }
+    PCIEMOPBINSIZES pImpl = g_apIemImplGrp1[(bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK];
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+            {
+                /* register target */
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG_CONST(uint16_t,  u16Src, /*=*/ u16Imm,   1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory target */
+                uint32_t fAccess;
+                if (pImpl->pfnLockedU16)
+                    fAccess = IEM_ACCESS_DATA_RW;
+                else /* CMP, TEST */
+                    fAccess = IEM_ACCESS_DATA_R;
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,        u16Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 2);
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEM_MC_ASSIGN(u16Src, u16Imm);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            break;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+            {
+                /* register target */
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG_CONST(uint32_t,  u32Src, /*=*/ u32Imm,   1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory target */
+                uint32_t fAccess;
+                if (pImpl->pfnLockedU32)
+                    fAccess = IEM_ACCESS_DATA_RW;
+                else /* CMP, TEST */
+                    fAccess = IEM_ACCESS_DATA_R;
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,        u32Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEM_MC_ASSIGN(u32Src, u32Imm);
+                if (pImpl->pfnLockedU32)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            break;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+            {
+                /* register target */
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG_CONST(uint64_t,  u64Src, /*=*/ u64Imm,   1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory target */
+                uint32_t fAccess;
+                if (pImpl->pfnLockedU64)
+                    fAccess = IEM_ACCESS_DATA_RW;
+                else /* CMP */
+                    fAccess = IEM_ACCESS_DATA_R;
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,        u64Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                if (pImpl->pfnLockedU64)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_ASSIGN(u64Src, u64Imm);
+                IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+            }
+            break;
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x82
+ * @opmnemonic  grp1_82
+ * @opgroup     og_groups
+ */
+FNIEMOP_DEF(iemOp_Grp1_Eb_Ib_82)
+{
+    IEMOP_HLP_NO_64BIT(); /** @todo do we need to decode the whole instruction or is this ok? */
+    return FNIEMOP_CALL(iemOp_Grp1_Eb_Ib_80);
+}
+
+
+/**
+ * @opcode      0x83
+ */
+FNIEMOP_DEF(iemOp_Grp1_Ev_Ib)
+{
+    uint8_t bRm;   IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0: IEMOP_MNEMONIC(add_Ev_Ib, "add Ev,Ib"); break;
+        case 1: IEMOP_MNEMONIC(or_Ev_Ib,  "or  Ev,Ib"); break;
+        case 2: IEMOP_MNEMONIC(adc_Ev_Ib, "adc Ev,Ib"); break;
+        case 3: IEMOP_MNEMONIC(sbb_Ev_Ib, "sbb Ev,Ib"); break;
+        case 4: IEMOP_MNEMONIC(and_Ev_Ib, "and Ev,Ib"); break;
+        case 5: IEMOP_MNEMONIC(sub_Ev_Ib, "sub Ev,Ib"); break;
+        case 6: IEMOP_MNEMONIC(xor_Ev_Ib, "xor Ev,Ib"); break;
+        case 7: IEMOP_MNEMONIC(cmp_Ev_Ib, "cmp Ev,Ib"); break;
+    }
+    /* Note! Seems the OR, AND, and XOR instructions are present on CPUs prior
+             to the 386 even if absent in the intel reference manuals and some
+             3rd party opcode listings. */
+    PCIEMOPBINSIZES pImpl = g_apIemImplGrp1[(bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK];
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register target
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                    0);
+                IEM_MC_ARG_CONST(uint16_t,  u16Src, /*=*/ (int8_t)u8Imm,1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_32BIT:
+            {
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                    0);
+                IEM_MC_ARG_CONST(uint32_t,  u32Src, /*=*/ (int8_t)u8Imm,1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_64BIT:
+            {
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                    0);
+                IEM_MC_ARG_CONST(uint64_t,  u64Src, /*=*/ (int8_t)u8Imm,1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+            }
+        }
+    }
+    else
+    {
+        /*
+         * Memory target.
+         */
+        uint32_t fAccess;
+        if (pImpl->pfnLockedU16)
+            fAccess = IEM_ACCESS_DATA_RW;
+        else /* CMP */
+            fAccess = IEM_ACCESS_DATA_R;
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                    0);
+                IEM_MC_ARG(uint16_t,        u16Src,                     1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,            2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+                IEM_MC_ASSIGN(u16Src, (int8_t)u8Imm);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_32BIT:
+            {
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                    0);
+                IEM_MC_ARG(uint32_t,        u32Src,                     1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,            2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+                IEM_MC_ASSIGN(u32Src, (int8_t)u8Imm);
+                if (pImpl->pfnLockedU32)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_64BIT:
+            {
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                    0);
+                IEM_MC_ARG(uint64_t,        u64Src,                     1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,            2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+                IEM_MC_ASSIGN(u64Src, (int8_t)u8Imm);
+                if (pImpl->pfnLockedU64)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+            }
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x84
+ */
+FNIEMOP_DEF(iemOp_test_Eb_Gb)
+{
+    IEMOP_MNEMONIC(test_Eb_Gb, "test Eb,Gb");
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_test);
+}
+
+
+/**
+ * @opcode      0x85
+ */
+FNIEMOP_DEF(iemOp_test_Ev_Gv)
+{
+    IEMOP_MNEMONIC(test_Ev_Gv, "test Ev,Gv");
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_test);
+}
+
+
+/**
+ * @opcode      0x86
+ */
+FNIEMOP_DEF(iemOp_xchg_Eb_Gb)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_MNEMONIC(xchg_Eb_Gb, "xchg Eb,Gb");
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint8_t, uTmp1);
+        IEM_MC_LOCAL(uint8_t, uTmp2);
+
+        IEM_MC_FETCH_GREG_U8(uTmp1, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_FETCH_GREG_U8(uTmp2, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_STORE_GREG_U8((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,                              uTmp1);
+        IEM_MC_STORE_GREG_U8(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uTmp2);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+/** @todo the register must be committed separately! */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint8_t *,  pu8Mem,           0);
+        IEM_MC_ARG(uint8_t *,  pu8Reg,           1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEM_MC_MEM_MAP(pu8Mem, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_REF_GREG_U8(pu8Reg, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u8, pu8Mem, pu8Reg);
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Mem, IEM_ACCESS_DATA_RW);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x87
+ */
+FNIEMOP_DEF(iemOp_xchg_Ev_Gv)
+{
+    IEMOP_MNEMONIC(xchg_Ev_Gv, "xchg Ev,Gv");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, uTmp1);
+                IEM_MC_LOCAL(uint16_t, uTmp2);
+
+                IEM_MC_FETCH_GREG_U16(uTmp1, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_GREG_U16(uTmp2, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_STORE_GREG_U16((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,                              uTmp1);
+                IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uTmp2);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, uTmp1);
+                IEM_MC_LOCAL(uint32_t, uTmp2);
+
+                IEM_MC_FETCH_GREG_U32(uTmp1, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_GREG_U32(uTmp2, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_STORE_GREG_U32((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,                              uTmp1);
+                IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uTmp2);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, uTmp1);
+                IEM_MC_LOCAL(uint64_t, uTmp2);
+
+                IEM_MC_FETCH_GREG_U64(uTmp1, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_GREG_U64(uTmp2, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_STORE_GREG_U64((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,                              uTmp1);
+                IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uTmp2);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+/** @todo the register must be committed separately! */
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(2, 2);
+                IEM_MC_ARG(uint16_t *,  pu16Mem, 0);
+                IEM_MC_ARG(uint16_t *,  pu16Reg, 1);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEM_MC_MEM_MAP(pu16Mem, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_REF_GREG_U16(pu16Reg, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u16, pu16Mem, pu16Reg);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Mem, IEM_ACCESS_DATA_RW);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(2, 2);
+                IEM_MC_ARG(uint32_t *,  pu32Mem, 0);
+                IEM_MC_ARG(uint32_t *,  pu32Reg, 1);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEM_MC_MEM_MAP(pu32Mem, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_REF_GREG_U32(pu32Reg, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u32, pu32Mem, pu32Reg);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Mem, IEM_ACCESS_DATA_RW);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Reg);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(2, 2);
+                IEM_MC_ARG(uint64_t *,  pu64Mem, 0);
+                IEM_MC_ARG(uint64_t *,  pu64Reg, 1);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEM_MC_MEM_MAP(pu64Mem, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_REF_GREG_U64(pu64Reg, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u64, pu64Mem, pu64Reg);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Mem, IEM_ACCESS_DATA_RW);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x88
+ */
+FNIEMOP_DEF(iemOp_mov_Eb_Gb)
+{
+    IEMOP_MNEMONIC(mov_Eb_Gb, "mov Eb,Gb");
+
+    uint8_t bRm;
+    IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint8_t, u8Value);
+        IEM_MC_FETCH_GREG_U8(u8Value, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_GREG_U8((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u8Value);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're writing a register to memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint8_t, u8Value);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_GREG_U8(u8Value, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U8(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u8Value);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+
+}
+
+
+/**
+ * @opcode      0x89
+ */
+FNIEMOP_DEF(iemOp_mov_Ev_Gv)
+{
+    IEMOP_MNEMONIC(mov_Ev_Gv, "mov Ev,Gv");
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_FETCH_GREG_U16(u16Value, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_STORE_GREG_U16((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u16Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_FETCH_GREG_U32(u32Value, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_STORE_GREG_U32((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u32Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_FETCH_GREG_U64(u64Value, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_STORE_GREG_U64((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u64Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+        }
+    }
+    else
+    {
+        /*
+         * We're writing a register to memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U16(u16Value, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U32(u32Value, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u32Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U64(u64Value, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u64Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x8a
+ */
+FNIEMOP_DEF(iemOp_mov_Gb_Eb)
+{
+    IEMOP_MNEMONIC(mov_Gb_Eb, "mov Gb,Eb");
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint8_t, u8Value);
+        IEM_MC_FETCH_GREG_U8(u8Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_STORE_GREG_U8(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u8Value);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint8_t, u8Value);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_MEM_U8(u8Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+        IEM_MC_STORE_GREG_U8(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u8Value);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x8b
+ */
+FNIEMOP_DEF(iemOp_mov_Gv_Ev)
+{
+    IEMOP_MNEMONIC(mov_Gv_Ev, "mov Gv,Ev");
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_FETCH_GREG_U16(u16Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_FETCH_GREG_U32(u32Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_FETCH_GREG_U64(u64Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+        }
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * opcode      0x63
+ * @todo Table fixme
+ */
+FNIEMOP_DEF(iemOp_arpl_Ew_Gw_movsx_Gv_Ev)
+{
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
+        return FNIEMOP_CALL(iemOp_arpl_Ew_Gw);
+    if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT)
+        return FNIEMOP_CALL(iemOp_mov_Gv_Ev);
+    return FNIEMOP_CALL(iemOp_movsxd_Gv_Ev);
+}
+
+
+/**
+ * @opcode      0x8c
+ */
+FNIEMOP_DEF(iemOp_mov_Ev_Sw)
+{
+    IEMOP_MNEMONIC(mov_Ev_Sw, "mov Ev,Sw");
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * Check that the destination register exists. The REX.R prefix is ignored.
+     */
+    uint8_t const iSegReg = ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+    if (   iSegReg > X86_SREG_GS)
+        return IEMOP_RAISE_INVALID_OPCODE(); /** @todo should probably not be raised until we've fetched all the opcode bytes? */
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     * In that case, the operand size is respected and the upper bits are
+     * cleared (starting with some pentium).
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_FETCH_SREG_U16(u16Value, iSegReg);
+                IEM_MC_STORE_GREG_U16((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u16Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_FETCH_SREG_ZX_U32(u32Value, iSegReg);
+                IEM_MC_STORE_GREG_U32((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u32Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_FETCH_SREG_ZX_U64(u64Value, iSegReg);
+                IEM_MC_STORE_GREG_U64((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u64Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+        }
+    }
+    else
+    {
+        /*
+         * We're saving the register to memory.  The access is word sized
+         * regardless of operand size prefixes.
+         */
+#if 0 /* not necessary */
+        pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_16BIT;
+#endif
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint16_t,  u16Value);
+        IEM_MC_LOCAL(RTGCPTR,   GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_SREG_U16(u16Value, iSegReg);
+        IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Value);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+
+
+/**
+ * @opcode      0x8d
+ */
+FNIEMOP_DEF(iemOp_lea_Gv_M)
+{
+    IEMOP_MNEMONIC(lea_Gv_M, "lea Gv,M");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        return IEMOP_RAISE_INVALID_OPCODE(); /* no register form */
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc);
+            IEM_MC_LOCAL(uint16_t, u16Cast);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_ASSIGN_TO_SMALLER(u16Cast, GCPtrEffSrc);
+            IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Cast);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Cast);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_ASSIGN_TO_SMALLER(u32Cast, GCPtrEffSrc);
+            IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Cast);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, GCPtrEffSrc);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+    }
+    AssertFailedReturn(VERR_IEM_IPE_7);
+}
+
+
+/**
+ * @opcode      0x8e
+ */
+FNIEMOP_DEF(iemOp_mov_Sw_Ev)
+{
+    IEMOP_MNEMONIC(mov_Sw_Ev, "mov Sw,Ev");
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * The practical operand size is 16-bit.
+     */
+#if 0 /* not necessary */
+    pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_16BIT;
+#endif
+
+    /*
+     * Check that the destination register exists and can be used with this
+     * instruction.  The REX.R prefix is ignored.
+     */
+    uint8_t const iSegReg = ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+    if (   iSegReg == X86_SREG_CS
+        || iSegReg > X86_SREG_GS)
+        return IEMOP_RAISE_INVALID_OPCODE(); /** @todo should probably not be raised until we've fetched all the opcode bytes? */
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG_CONST(uint8_t, iSRegArg, iSegReg, 0);
+        IEM_MC_ARG(uint16_t,      u16Value,          1);
+        IEM_MC_FETCH_GREG_U16(u16Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_CALL_CIMPL_2(iemCImpl_load_SReg, iSRegArg, u16Value);
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're loading the register from memory.  The access is word sized
+         * regardless of operand size prefixes.
+         */
+        IEM_MC_BEGIN(2, 1);
+        IEM_MC_ARG_CONST(uint8_t, iSRegArg, iSegReg, 0);
+        IEM_MC_ARG(uint16_t,      u16Value,          1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_MEM_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+        IEM_MC_CALL_CIMPL_2(iemCImpl_load_SReg, iSRegArg, u16Value);
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x8f /0. */
+FNIEMOP_DEF_1(iemOp_pop_Ev, uint8_t, bRm)
+{
+    /* This bugger is rather annoying as it requires rSP to be updated before
+       doing the effective address calculations.  Will eventually require a
+       split between the R/M+SIB decoding and the effective address
+       calculation - which is something that is required for any attempt at
+       reusing this code for a recompiler.  It may also be good to have if we
+       need to delay #UD exception caused by invalid lock prefixes.
+
+       For now, we'll do a mostly safe interpreter-only implementation here. */
+    /** @todo What's the deal with the 'reg' field and pop Ev?  Ignorning it for
+     *        now until tests show it's checked.. */
+    IEMOP_MNEMONIC(pop_Ev, "pop Ev");
+
+    /* Register access is relatively easy and can share code. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        return FNIEMOP_CALL_1(iemOpCommonPopGReg, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+
+    /*
+     * Memory target.
+     *
+     * Intel says that RSP is incremented before it's used in any effective
+     * address calcuations.  This means some serious extra annoyance here since
+     * we decode and calculate the effective address in one step and like to
+     * delay committing registers till everything is done.
+     *
+     * So, we'll decode and calculate the effective address twice.  This will
+     * require some recoding if turned into a recompiler.
+     */
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); /* The common code does this differently. */
+
+#ifndef TST_IEM_CHECK_MC
+    /* Calc effective address with modified ESP. */
+/** @todo testcase */
+    RTGCPTR         GCPtrEff;
+    VBOXSTRICTRC    rcStrict;
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT: rcStrict = iemOpHlpCalcRmEffAddrEx(pVCpu, bRm, 0, &GCPtrEff, 2); break;
+        case IEMMODE_32BIT: rcStrict = iemOpHlpCalcRmEffAddrEx(pVCpu, bRm, 0, &GCPtrEff, 4); break;
+        case IEMMODE_64BIT: rcStrict = iemOpHlpCalcRmEffAddrEx(pVCpu, bRm, 0, &GCPtrEff, 8); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /* Perform the operation - this should be CImpl. */
+    RTUINT64U TmpRsp;
+    TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t u16Value;
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Value, &TmpRsp);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = iemMemStoreDataU16(pVCpu, pVCpu->iem.s.iEffSeg, GCPtrEff, u16Value);
+            break;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            uint32_t u32Value;
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Value, &TmpRsp);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = iemMemStoreDataU32(pVCpu, pVCpu->iem.s.iEffSeg, GCPtrEff, u32Value);
+            break;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            uint64_t u64Value;
+            rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Value, &TmpRsp);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = iemMemStoreDataU64(pVCpu, pVCpu->iem.s.iEffSeg, GCPtrEff, u64Value);
+            break;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    if (rcStrict == VINF_SUCCESS)
+    {
+        pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+        iemRegUpdateRipAndClearRF(pVCpu);
+    }
+    return rcStrict;
+
+#else
+    return VERR_IEM_IPE_2;
+#endif
+}
+
+
+/**
+ * @opcode      0x8f
+ */
+FNIEMOP_DEF(iemOp_Grp1A__xop)
+{
+    /*
+     * AMD has defined /1 thru /7 as XOP prefix.  The prefix is similar to the
+     * three byte VEX prefix, except that the mmmmm field cannot have the values
+     * 0 thru 7, because it would then be confused with pop Ev (modrm.reg == 0).
+     */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_REG_MASK) == (0 << X86_MODRM_REG_SHIFT)) /* /0 */
+        return FNIEMOP_CALL_1(iemOp_pop_Ev, bRm);
+
+    IEMOP_MNEMONIC(xop, "xop");
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXop)
+    {
+        /** @todo Test when exctly the XOP conformance checks kick in during
+         * instruction decoding and fetching (using \#PF). */
+        uint8_t bXop2;   IEM_OPCODE_GET_NEXT_U8(&bXop2);
+        uint8_t bOpcode; IEM_OPCODE_GET_NEXT_U8(&bOpcode);
+        if (   (  pVCpu->iem.s.fPrefixes
+                & (IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_LOCK | IEM_OP_PRF_REX))
+            == 0)
+        {
+            pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_XOP;
+            if ((bXop2 & 0x80 /* XOP.W */) && pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+                pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_REX_W;
+            pVCpu->iem.s.uRexReg    = (~bRm >> (7 - 3)) & 0x8;
+            pVCpu->iem.s.uRexIndex  = (~bRm >> (6 - 3)) & 0x8;
+            pVCpu->iem.s.uRexB      = (~bRm >> (5 - 3)) & 0x8;
+            pVCpu->iem.s.uVex3rdReg = (~bXop2 >> 3) & 0xf;
+            pVCpu->iem.s.uVexLength = (bXop2 >> 2) & 1;
+            pVCpu->iem.s.idxPrefix  = bXop2 & 0x3;
+
+            /** @todo XOP: Just use new tables and decoders. */
+            switch (bRm & 0x1f)
+            {
+                case 8: /* xop opcode map 8. */
+                    IEMOP_BITCH_ABOUT_STUB();
+                    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+
+                case 9: /* xop opcode map 9. */
+                    IEMOP_BITCH_ABOUT_STUB();
+                    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+
+                case 10: /* xop opcode map 10. */
+                    IEMOP_BITCH_ABOUT_STUB();
+                    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+
+                default:
+                    Log(("XOP: Invalid vvvv value: %#x!\n", bRm & 0x1f));
+                    return IEMOP_RAISE_INVALID_OPCODE();
+            }
+        }
+        else
+            Log(("XOP: Invalid prefix mix!\n"));
+    }
+    else
+        Log(("XOP: XOP support disabled!\n"));
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * Common 'xchg reg,rAX' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonXchgGRegRax, uint8_t, iReg)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    iReg |= pVCpu->iem.s.uRexB;
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint16_t, u16Tmp1);
+            IEM_MC_LOCAL(uint16_t, u16Tmp2);
+            IEM_MC_FETCH_GREG_U16(u16Tmp1, iReg);
+            IEM_MC_FETCH_GREG_U16(u16Tmp2, X86_GREG_xAX);
+            IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Tmp1);
+            IEM_MC_STORE_GREG_U16(iReg,         u16Tmp2);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint32_t, u32Tmp1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp2);
+            IEM_MC_FETCH_GREG_U32(u32Tmp1, iReg);
+            IEM_MC_FETCH_GREG_U32(u32Tmp2, X86_GREG_xAX);
+            IEM_MC_STORE_GREG_U32(X86_GREG_xAX, u32Tmp1);
+            IEM_MC_STORE_GREG_U32(iReg,         u32Tmp2);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint64_t, u64Tmp1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp2);
+            IEM_MC_FETCH_GREG_U64(u64Tmp1, iReg);
+            IEM_MC_FETCH_GREG_U64(u64Tmp2, X86_GREG_xAX);
+            IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u64Tmp1);
+            IEM_MC_STORE_GREG_U64(iReg,         u64Tmp2);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x90
+ */
+FNIEMOP_DEF(iemOp_nop)
+{
+    /* R8/R8D and RAX/EAX can be exchanged. */
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX_B)
+    {
+        IEMOP_MNEMONIC(xchg_r8_rAX, "xchg r8,rAX");
+        return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xAX);
+    }
+
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)
+    {
+        IEMOP_MNEMONIC(pause, "pause");
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmx)
+            return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmx_pause);
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvm)
+            return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_svm_pause);
+#endif
+    }
+    else
+        IEMOP_MNEMONIC(nop, "nop");
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x91
+ */
+FNIEMOP_DEF(iemOp_xchg_eCX_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rCX_rAX, "xchg rCX,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xCX);
+}
+
+
+/**
+ * @opcode      0x92
+ */
+FNIEMOP_DEF(iemOp_xchg_eDX_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rDX_rAX, "xchg rDX,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xDX);
+}
+
+
+/**
+ * @opcode      0x93
+ */
+FNIEMOP_DEF(iemOp_xchg_eBX_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rBX_rAX, "xchg rBX,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xBX);
+}
+
+
+/**
+ * @opcode      0x94
+ */
+FNIEMOP_DEF(iemOp_xchg_eSP_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rSX_rAX, "xchg rSX,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xSP);
+}
+
+
+/**
+ * @opcode      0x95
+ */
+FNIEMOP_DEF(iemOp_xchg_eBP_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rBP_rAX, "xchg rBP,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xBP);
+}
+
+
+/**
+ * @opcode      0x96
+ */
+FNIEMOP_DEF(iemOp_xchg_eSI_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rSI_rAX, "xchg rSI,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xSI);
+}
+
+
+/**
+ * @opcode      0x97
+ */
+FNIEMOP_DEF(iemOp_xchg_eDI_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rDI_rAX, "xchg rDI,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xDI);
+}
+
+
+/**
+ * @opcode      0x98
+ */
+FNIEMOP_DEF(iemOp_cbw)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEMOP_MNEMONIC(cbw, "cbw");
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 7) {
+                IEM_MC_OR_GREG_U16(X86_GREG_xAX, UINT16_C(0xff00));
+            } IEM_MC_ELSE() {
+                IEM_MC_AND_GREG_U16(X86_GREG_xAX, UINT16_C(0x00ff));
+            } IEM_MC_ENDIF();
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEMOP_MNEMONIC(cwde, "cwde");
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 15) {
+                IEM_MC_OR_GREG_U32(X86_GREG_xAX, UINT32_C(0xffff0000));
+            } IEM_MC_ELSE() {
+                IEM_MC_AND_GREG_U32(X86_GREG_xAX, UINT32_C(0x0000ffff));
+            } IEM_MC_ENDIF();
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEMOP_MNEMONIC(cdqe, "cdqe");
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 31) {
+                IEM_MC_OR_GREG_U64(X86_GREG_xAX, UINT64_C(0xffffffff00000000));
+            } IEM_MC_ELSE() {
+                IEM_MC_AND_GREG_U64(X86_GREG_xAX, UINT64_C(0x00000000ffffffff));
+            } IEM_MC_ENDIF();
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x99
+ */
+FNIEMOP_DEF(iemOp_cwd)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEMOP_MNEMONIC(cwd, "cwd");
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 15) {
+                IEM_MC_STORE_GREG_U16_CONST(X86_GREG_xDX, UINT16_C(0xffff));
+            } IEM_MC_ELSE() {
+                IEM_MC_STORE_GREG_U16_CONST(X86_GREG_xDX, 0);
+            } IEM_MC_ENDIF();
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEMOP_MNEMONIC(cdq, "cdq");
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 31) {
+                IEM_MC_STORE_GREG_U32_CONST(X86_GREG_xDX, UINT32_C(0xffffffff));
+            } IEM_MC_ELSE() {
+                IEM_MC_STORE_GREG_U32_CONST(X86_GREG_xDX, 0);
+            } IEM_MC_ENDIF();
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEMOP_MNEMONIC(cqo, "cqo");
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 63) {
+                IEM_MC_STORE_GREG_U64_CONST(X86_GREG_xDX, UINT64_C(0xffffffffffffffff));
+            } IEM_MC_ELSE() {
+                IEM_MC_STORE_GREG_U64_CONST(X86_GREG_xDX, 0);
+            } IEM_MC_ENDIF();
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x9a
+ */
+FNIEMOP_DEF(iemOp_call_Ap)
+{
+    IEMOP_MNEMONIC(call_Ap, "call Ap");
+    IEMOP_HLP_NO_64BIT();
+
+    /* Decode the far pointer address and pass it on to the far call C implementation. */
+    uint32_t offSeg;
+    if (pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT)
+        IEM_OPCODE_GET_NEXT_U32(&offSeg);
+    else
+        IEM_OPCODE_GET_NEXT_U16_ZX_U32(&offSeg);
+    uint16_t uSel;  IEM_OPCODE_GET_NEXT_U16(&uSel);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_callf, uSel, offSeg, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/** Opcode 0x9b. (aka fwait) */
+FNIEMOP_DEF(iemOp_wait)
+{
+    IEMOP_MNEMONIC(wait, "wait");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x9c
+ */
+FNIEMOP_DEF(iemOp_pushf_Fv)
+{
+    IEMOP_MNEMONIC(pushf_Fv, "pushf Fv");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_pushf, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0x9d
+ */
+FNIEMOP_DEF(iemOp_popf_Fv)
+{
+    IEMOP_MNEMONIC(popf_Fv, "popf Fv");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_popf, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0x9e
+ */
+FNIEMOP_DEF(iemOp_sahf)
+{
+    IEMOP_MNEMONIC(sahf, "sahf");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (   pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+        && !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLahfSahf)
+        return IEMOP_RAISE_INVALID_OPCODE();
+    IEM_MC_BEGIN(0, 2);
+    IEM_MC_LOCAL(uint32_t, u32Flags);
+    IEM_MC_LOCAL(uint32_t, EFlags);
+    IEM_MC_FETCH_EFLAGS(EFlags);
+    IEM_MC_FETCH_GREG_U8_ZX_U32(u32Flags, X86_GREG_xSP/*=AH*/);
+    IEM_MC_AND_LOCAL_U32(u32Flags, X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF);
+    IEM_MC_AND_LOCAL_U32(EFlags, UINT32_C(0xffffff00));
+    IEM_MC_OR_LOCAL_U32(u32Flags, X86_EFL_1);
+    IEM_MC_OR_2LOCS_U32(EFlags, u32Flags);
+    IEM_MC_COMMIT_EFLAGS(EFlags);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x9f
+ */
+FNIEMOP_DEF(iemOp_lahf)
+{
+    IEMOP_MNEMONIC(lahf, "lahf");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (   pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+        && !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLahfSahf)
+        return IEMOP_RAISE_INVALID_OPCODE();
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(uint8_t, u8Flags);
+    IEM_MC_FETCH_EFLAGS_U8(u8Flags);
+    IEM_MC_STORE_GREG_U8(X86_GREG_xSP/*=AH*/, u8Flags);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Macro used by iemOp_mov_AL_Ob, iemOp_mov_rAX_Ov, iemOp_mov_Ob_AL and
+ * iemOp_mov_Ov_rAX to fetch the moffsXX bit of the opcode and fend off lock
+ * prefixes.  Will return on failures.
+ * @param   a_GCPtrMemOff   The variable to store the offset in.
+ */
+#define IEMOP_FETCH_MOFFS_XX(a_GCPtrMemOff) \
+    do \
+    { \
+        switch (pVCpu->iem.s.enmEffAddrMode) \
+        { \
+            case IEMMODE_16BIT: \
+                IEM_OPCODE_GET_NEXT_U16_ZX_U64(&(a_GCPtrMemOff)); \
+                break; \
+            case IEMMODE_32BIT: \
+                IEM_OPCODE_GET_NEXT_U32_ZX_U64(&(a_GCPtrMemOff)); \
+                break; \
+            case IEMMODE_64BIT: \
+                IEM_OPCODE_GET_NEXT_U64(&(a_GCPtrMemOff)); \
+                break; \
+            IEM_NOT_REACHED_DEFAULT_CASE_RET(); \
+        } \
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); \
+    } while (0)
+
+/**
+ * @opcode      0xa0
+ */
+FNIEMOP_DEF(iemOp_mov_AL_Ob)
+{
+    /*
+     * Get the offset and fend off lock prefixes.
+     */
+    IEMOP_MNEMONIC(mov_AL_Ob, "mov AL,Ob");
+    RTGCPTR GCPtrMemOff;
+    IEMOP_FETCH_MOFFS_XX(GCPtrMemOff);
+
+    /*
+     * Fetch AL.
+     */
+    IEM_MC_BEGIN(0,1);
+    IEM_MC_LOCAL(uint8_t, u8Tmp);
+    IEM_MC_FETCH_MEM_U8(u8Tmp, pVCpu->iem.s.iEffSeg, GCPtrMemOff);
+    IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Tmp);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xa1
+ */
+FNIEMOP_DEF(iemOp_mov_rAX_Ov)
+{
+    /*
+     * Get the offset and fend off lock prefixes.
+     */
+    IEMOP_MNEMONIC(mov_rAX_Ov, "mov rAX,Ov");
+    RTGCPTR GCPtrMemOff;
+    IEMOP_FETCH_MOFFS_XX(GCPtrMemOff);
+
+    /*
+     * Fetch rAX.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0,1);
+            IEM_MC_LOCAL(uint16_t, u16Tmp);
+            IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrMemOff);
+            IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Tmp);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0,1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+            IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrMemOff);
+            IEM_MC_STORE_GREG_U32(X86_GREG_xAX, u32Tmp);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0,1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+            IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrMemOff);
+            IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u64Tmp);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xa2
+ */
+FNIEMOP_DEF(iemOp_mov_Ob_AL)
+{
+    /*
+     * Get the offset and fend off lock prefixes.
+     */
+    IEMOP_MNEMONIC(mov_Ob_AL, "mov Ob,AL");
+    RTGCPTR GCPtrMemOff;
+    IEMOP_FETCH_MOFFS_XX(GCPtrMemOff);
+
+    /*
+     * Store AL.
+     */
+    IEM_MC_BEGIN(0,1);
+    IEM_MC_LOCAL(uint8_t, u8Tmp);
+    IEM_MC_FETCH_GREG_U8(u8Tmp, X86_GREG_xAX);
+    IEM_MC_STORE_MEM_U8(pVCpu->iem.s.iEffSeg, GCPtrMemOff, u8Tmp);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xa3
+ */
+FNIEMOP_DEF(iemOp_mov_Ov_rAX)
+{
+    /*
+     * Get the offset and fend off lock prefixes.
+     */
+    IEMOP_MNEMONIC(mov_Ov_rAX, "mov Ov,rAX");
+    RTGCPTR GCPtrMemOff;
+    IEMOP_FETCH_MOFFS_XX(GCPtrMemOff);
+
+    /*
+     * Store rAX.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0,1);
+            IEM_MC_LOCAL(uint16_t, u16Tmp);
+            IEM_MC_FETCH_GREG_U16(u16Tmp, X86_GREG_xAX);
+            IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrMemOff, u16Tmp);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0,1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+            IEM_MC_FETCH_GREG_U32(u32Tmp, X86_GREG_xAX);
+            IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrMemOff, u32Tmp);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0,1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+            IEM_MC_FETCH_GREG_U64(u64Tmp, X86_GREG_xAX);
+            IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrMemOff, u64Tmp);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+/** Macro used by iemOp_movsb_Xb_Yb and iemOp_movswd_Xv_Yv */
+#define IEM_MOVS_CASE(ValBits, AddrBits) \
+        IEM_MC_BEGIN(0, 2); \
+        IEM_MC_LOCAL(uint##ValBits##_t, uValue); \
+        IEM_MC_LOCAL(RTGCPTR,           uAddr); \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xSI); \
+        IEM_MC_FETCH_MEM_U##ValBits(uValue, pVCpu->iem.s.iEffSeg, uAddr); \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xDI); \
+        IEM_MC_STORE_MEM_U##ValBits(X86_SREG_ES, uAddr, uValue); \
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \
+        } IEM_MC_ELSE() { \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \
+        } IEM_MC_ENDIF(); \
+        IEM_MC_ADVANCE_RIP(); \
+        IEM_MC_END();
+
+/**
+ * @opcode      0xa4
+ */
+FNIEMOP_DEF(iemOp_movsb_Xb_Yb)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_movsb_Xb_Yb, "rep movsb Xb,Yb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op8_addr16, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op8_addr32, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op8_addr64, pVCpu->iem.s.iEffSeg);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(movsb_Xb_Yb, "movsb Xb,Yb");
+
+    /*
+     * Sharing case implementation with movs[wdq] below.
+     */
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT: IEM_MOVS_CASE(8, 16); break;
+        case IEMMODE_32BIT: IEM_MOVS_CASE(8, 32); break;
+        case IEMMODE_64BIT: IEM_MOVS_CASE(8, 64); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xa5
+ */
+FNIEMOP_DEF(iemOp_movswd_Xv_Yv)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_movs_Xv_Yv, "rep movs Xv,Yv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op16_addr16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op16_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op16_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op32_addr16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op32_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op32_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_6);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op64_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op64_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(movs_Xv_Yv, "movs Xv,Yv");
+
+    /*
+     * Annoying double switch here.
+     * Using ugly macro for implementing the cases, sharing it with movsb.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_MOVS_CASE(16, 16); break;
+                case IEMMODE_32BIT: IEM_MOVS_CASE(16, 32); break;
+                case IEMMODE_64BIT: IEM_MOVS_CASE(16, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_32BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_MOVS_CASE(32, 16); break;
+                case IEMMODE_32BIT: IEM_MOVS_CASE(32, 32); break;
+                case IEMMODE_64BIT: IEM_MOVS_CASE(32, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_64BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break;
+                case IEMMODE_32BIT: IEM_MOVS_CASE(64, 32); break;
+                case IEMMODE_64BIT: IEM_MOVS_CASE(64, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    return VINF_SUCCESS;
+}
+
+#undef IEM_MOVS_CASE
+
+/** Macro used by iemOp_cmpsb_Xb_Yb and iemOp_cmpswd_Xv_Yv */
+#define IEM_CMPS_CASE(ValBits, AddrBits) \
+        IEM_MC_BEGIN(3, 3); \
+        IEM_MC_ARG(uint##ValBits##_t *, puValue1, 0); \
+        IEM_MC_ARG(uint##ValBits##_t,   uValue2,  1); \
+        IEM_MC_ARG(uint32_t *,          pEFlags,  2); \
+        IEM_MC_LOCAL(uint##ValBits##_t, uValue1); \
+        IEM_MC_LOCAL(RTGCPTR,           uAddr); \
+        \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xSI); \
+        IEM_MC_FETCH_MEM_U##ValBits(uValue1, pVCpu->iem.s.iEffSeg, uAddr); \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xDI); \
+        IEM_MC_FETCH_MEM_U##ValBits(uValue2, X86_SREG_ES, uAddr); \
+        IEM_MC_REF_LOCAL(puValue1, uValue1); \
+        IEM_MC_REF_EFLAGS(pEFlags); \
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cmp_u##ValBits, puValue1, uValue2, pEFlags); \
+        \
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \
+        } IEM_MC_ELSE() { \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \
+        } IEM_MC_ENDIF(); \
+        IEM_MC_ADVANCE_RIP(); \
+        IEM_MC_END(); \
+
+/**
+ * @opcode      0xa6
+ */
+FNIEMOP_DEF(iemOp_cmpsb_Xb_Yb)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPZ)
+    {
+        IEMOP_MNEMONIC(repz_cmps_Xb_Yb, "repz cmps Xb,Yb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op8_addr16, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op8_addr32, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op8_addr64, pVCpu->iem.s.iEffSeg);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPNZ)
+    {
+        IEMOP_MNEMONIC(repnz_cmps_Xb_Yb, "repnz cmps Xb,Yb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op8_addr16, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op8_addr32, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op8_addr64, pVCpu->iem.s.iEffSeg);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(cmps_Xb_Yb, "cmps Xb,Yb");
+
+    /*
+     * Sharing case implementation with cmps[wdq] below.
+     */
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT: IEM_CMPS_CASE(8, 16); break;
+        case IEMMODE_32BIT: IEM_CMPS_CASE(8, 32); break;
+        case IEMMODE_64BIT: IEM_CMPS_CASE(8, 64); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    return VINF_SUCCESS;
+
+}
+
+
+/**
+ * @opcode      0xa7
+ */
+FNIEMOP_DEF(iemOp_cmpswd_Xv_Yv)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPZ)
+    {
+        IEMOP_MNEMONIC(repe_cmps_Xv_Yv, "repe cmps Xv,Yv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op16_addr16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op16_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op16_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op32_addr16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op32_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op32_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_4);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op64_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op64_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPNZ)
+    {
+        IEMOP_MNEMONIC(repne_cmps_Xv_Yv, "repne cmps Xv,Yv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op16_addr16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op16_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op16_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op32_addr16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op32_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op32_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_2);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op64_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op64_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+
+    IEMOP_MNEMONIC(cmps_Xv_Yv, "cmps Xv,Yv");
+
+    /*
+     * Annoying double switch here.
+     * Using ugly macro for implementing the cases, sharing it with cmpsb.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_CMPS_CASE(16, 16); break;
+                case IEMMODE_32BIT: IEM_CMPS_CASE(16, 32); break;
+                case IEMMODE_64BIT: IEM_CMPS_CASE(16, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_32BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_CMPS_CASE(32, 16); break;
+                case IEMMODE_32BIT: IEM_CMPS_CASE(32, 32); break;
+                case IEMMODE_64BIT: IEM_CMPS_CASE(32, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_64BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break;
+                case IEMMODE_32BIT: IEM_CMPS_CASE(64, 32); break;
+                case IEMMODE_64BIT: IEM_CMPS_CASE(64, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    return VINF_SUCCESS;
+
+}
+
+#undef IEM_CMPS_CASE
+
+/**
+ * @opcode      0xa8
+ */
+FNIEMOP_DEF(iemOp_test_AL_Ib)
+{
+    IEMOP_MNEMONIC(test_al_Ib, "test al,Ib");
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_test);
+}
+
+
+/**
+ * @opcode      0xa9
+ */
+FNIEMOP_DEF(iemOp_test_eAX_Iz)
+{
+    IEMOP_MNEMONIC(test_rAX_Iz, "test rAX,Iz");
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_test);
+}
+
+
+/** Macro used by iemOp_stosb_Yb_AL and iemOp_stoswd_Yv_eAX */
+#define IEM_STOS_CASE(ValBits, AddrBits) \
+        IEM_MC_BEGIN(0, 2); \
+        IEM_MC_LOCAL(uint##ValBits##_t, uValue); \
+        IEM_MC_LOCAL(RTGCPTR, uAddr); \
+        IEM_MC_FETCH_GREG_U##ValBits(uValue, X86_GREG_xAX); \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr,  X86_GREG_xDI); \
+        IEM_MC_STORE_MEM_U##ValBits(X86_SREG_ES, uAddr, uValue); \
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+        } IEM_MC_ELSE() { \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+        } IEM_MC_ENDIF(); \
+        IEM_MC_ADVANCE_RIP(); \
+        IEM_MC_END(); \
+
+/**
+ * @opcode      0xaa
+ */
+FNIEMOP_DEF(iemOp_stosb_Yb_AL)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_stos_Yb_al, "rep stos Yb,al");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_al_m16);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_al_m32);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_al_m64);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(stos_Yb_al, "stos Yb,al");
+
+    /*
+     * Sharing case implementation with stos[wdq] below.
+     */
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT: IEM_STOS_CASE(8, 16); break;
+        case IEMMODE_32BIT: IEM_STOS_CASE(8, 32); break;
+        case IEMMODE_64BIT: IEM_STOS_CASE(8, 64); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xab
+ */
+FNIEMOP_DEF(iemOp_stoswd_Yv_eAX)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_stos_Yv_rAX, "rep stos Yv,rAX");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_ax_m16);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_ax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_ax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_eax_m16);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_eax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_eax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_9);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_rax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_rax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(stos_Yv_rAX, "stos Yv,rAX");
+
+    /*
+     * Annoying double switch here.
+     * Using ugly macro for implementing the cases, sharing it with stosb.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_STOS_CASE(16, 16); break;
+                case IEMMODE_32BIT: IEM_STOS_CASE(16, 32); break;
+                case IEMMODE_64BIT: IEM_STOS_CASE(16, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_32BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_STOS_CASE(32, 16); break;
+                case IEMMODE_32BIT: IEM_STOS_CASE(32, 32); break;
+                case IEMMODE_64BIT: IEM_STOS_CASE(32, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_64BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break;
+                case IEMMODE_32BIT: IEM_STOS_CASE(64, 32); break;
+                case IEMMODE_64BIT: IEM_STOS_CASE(64, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    return VINF_SUCCESS;
+}
+
+#undef IEM_STOS_CASE
+
+/** Macro used by iemOp_lodsb_AL_Xb and iemOp_lodswd_eAX_Xv */
+#define IEM_LODS_CASE(ValBits, AddrBits) \
+        IEM_MC_BEGIN(0, 2); \
+        IEM_MC_LOCAL(uint##ValBits##_t, uValue); \
+        IEM_MC_LOCAL(RTGCPTR, uAddr); \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xSI); \
+        IEM_MC_FETCH_MEM_U##ValBits(uValue, pVCpu->iem.s.iEffSeg, uAddr); \
+        IEM_MC_STORE_GREG_U##ValBits(X86_GREG_xAX, uValue); \
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \
+        } IEM_MC_ELSE() { \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \
+        } IEM_MC_ENDIF(); \
+        IEM_MC_ADVANCE_RIP(); \
+        IEM_MC_END();
+
+/**
+ * @opcode      0xac
+ */
+FNIEMOP_DEF(iemOp_lodsb_AL_Xb)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_lodsb_AL_Xb, "rep lodsb AL,Xb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_al_m16, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_al_m32, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_al_m64, pVCpu->iem.s.iEffSeg);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(lodsb_AL_Xb, "lodsb AL,Xb");
+
+    /*
+     * Sharing case implementation with stos[wdq] below.
+     */
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT: IEM_LODS_CASE(8, 16); break;
+        case IEMMODE_32BIT: IEM_LODS_CASE(8, 32); break;
+        case IEMMODE_64BIT: IEM_LODS_CASE(8, 64); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xad
+ */
+FNIEMOP_DEF(iemOp_lodswd_eAX_Xv)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_lods_rAX_Xv, "rep lods rAX,Xv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_ax_m16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_ax_m32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_ax_m64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_eax_m16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_eax_m32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_eax_m64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_7);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_rax_m32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_rax_m64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(lods_rAX_Xv, "lods rAX,Xv");
+
+    /*
+     * Annoying double switch here.
+     * Using ugly macro for implementing the cases, sharing it with lodsb.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_LODS_CASE(16, 16); break;
+                case IEMMODE_32BIT: IEM_LODS_CASE(16, 32); break;
+                case IEMMODE_64BIT: IEM_LODS_CASE(16, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_32BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_LODS_CASE(32, 16); break;
+                case IEMMODE_32BIT: IEM_LODS_CASE(32, 32); break;
+                case IEMMODE_64BIT: IEM_LODS_CASE(32, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_64BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break;
+                case IEMMODE_32BIT: IEM_LODS_CASE(64, 32); break;
+                case IEMMODE_64BIT: IEM_LODS_CASE(64, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    return VINF_SUCCESS;
+}
+
+#undef IEM_LODS_CASE
+
+/** Macro used by iemOp_scasb_AL_Xb and iemOp_scaswd_eAX_Xv */
+#define IEM_SCAS_CASE(ValBits, AddrBits) \
+        IEM_MC_BEGIN(3, 2); \
+        IEM_MC_ARG(uint##ValBits##_t *, puRax,   0); \
+        IEM_MC_ARG(uint##ValBits##_t,   uValue,  1); \
+        IEM_MC_ARG(uint32_t *,          pEFlags, 2); \
+        IEM_MC_LOCAL(RTGCPTR,           uAddr); \
+        \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xDI); \
+        IEM_MC_FETCH_MEM_U##ValBits(uValue, X86_SREG_ES, uAddr); \
+        IEM_MC_REF_GREG_U##ValBits(puRax, X86_GREG_xAX); \
+        IEM_MC_REF_EFLAGS(pEFlags); \
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cmp_u##ValBits, puRax, uValue, pEFlags); \
+        \
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+        } IEM_MC_ELSE() { \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+        } IEM_MC_ENDIF(); \
+        IEM_MC_ADVANCE_RIP(); \
+        IEM_MC_END();
+
+/**
+ * @opcode      0xae
+ */
+FNIEMOP_DEF(iemOp_scasb_AL_Xb)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPZ)
+    {
+        IEMOP_MNEMONIC(repe_scasb_AL_Xb, "repe scasb AL,Xb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_al_m16);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_al_m32);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_al_m64);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPNZ)
+    {
+        IEMOP_MNEMONIC(repone_scasb_AL_Xb, "repne scasb AL,Xb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_al_m16);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_al_m32);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_al_m64);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(scasb_AL_Xb, "scasb AL,Xb");
+
+    /*
+     * Sharing case implementation with stos[wdq] below.
+     */
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT: IEM_SCAS_CASE(8, 16); break;
+        case IEMMODE_32BIT: IEM_SCAS_CASE(8, 32); break;
+        case IEMMODE_64BIT: IEM_SCAS_CASE(8, 64); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xaf
+ */
+FNIEMOP_DEF(iemOp_scaswd_eAX_Xv)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPZ)
+    {
+        IEMOP_MNEMONIC(repe_scas_rAX_Xv, "repe scas rAX,Xv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_ax_m16);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_ax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_ax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_eax_m16);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_eax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_eax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_6); /** @todo It's this wrong, we can do 16-bit addressing in 64-bit mode, but not 32-bit. right? */
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_rax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_rax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPNZ)
+    {
+        IEMOP_MNEMONIC(repne_scas_rAX_Xv, "repne scas rAX,Xv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_ax_m16);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_ax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_ax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_eax_m16);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_eax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_eax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_5);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_rax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_rax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(scas_rAX_Xv, "scas rAX,Xv");
+
+    /*
+     * Annoying double switch here.
+     * Using ugly macro for implementing the cases, sharing it with scasb.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_SCAS_CASE(16, 16); break;
+                case IEMMODE_32BIT: IEM_SCAS_CASE(16, 32); break;
+                case IEMMODE_64BIT: IEM_SCAS_CASE(16, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_32BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_SCAS_CASE(32, 16); break;
+                case IEMMODE_32BIT: IEM_SCAS_CASE(32, 32); break;
+                case IEMMODE_64BIT: IEM_SCAS_CASE(32, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_64BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break;
+                case IEMMODE_32BIT: IEM_SCAS_CASE(64, 32); break;
+                case IEMMODE_64BIT: IEM_SCAS_CASE(64, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    return VINF_SUCCESS;
+}
+
+#undef IEM_SCAS_CASE
+
+/**
+ * Common 'mov r8, imm8' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonMov_r8_Ib, uint8_t, iReg)
+{
+    uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL_CONST(uint8_t, u8Value,/*=*/ u8Imm);
+    IEM_MC_STORE_GREG_U8(iReg, u8Value);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xb0
+ */
+FNIEMOP_DEF(iemOp_mov_AL_Ib)
+{
+    IEMOP_MNEMONIC(mov_AL_Ib, "mov AL,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xAX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb1
+ */
+FNIEMOP_DEF(iemOp_CL_Ib)
+{
+    IEMOP_MNEMONIC(mov_CL_Ib, "mov CL,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xCX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb2
+ */
+FNIEMOP_DEF(iemOp_DL_Ib)
+{
+    IEMOP_MNEMONIC(mov_DL_Ib, "mov DL,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xDX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb3
+ */
+FNIEMOP_DEF(iemOp_BL_Ib)
+{
+    IEMOP_MNEMONIC(mov_BL_Ib, "mov BL,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xBX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb4
+ */
+FNIEMOP_DEF(iemOp_mov_AH_Ib)
+{
+    IEMOP_MNEMONIC(mov_AH_Ib, "mov AH,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xSP | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb5
+ */
+FNIEMOP_DEF(iemOp_CH_Ib)
+{
+    IEMOP_MNEMONIC(mov_CH_Ib, "mov CH,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xBP | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb6
+ */
+FNIEMOP_DEF(iemOp_DH_Ib)
+{
+    IEMOP_MNEMONIC(mov_DH_Ib, "mov DH,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xSI | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb7
+ */
+FNIEMOP_DEF(iemOp_BH_Ib)
+{
+    IEMOP_MNEMONIC(mov_BH_Ib, "mov BH,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xDI | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * Common 'mov regX,immX' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonMov_Rv_Iv, uint8_t, iReg)
+{
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL_CONST(uint16_t, u16Value,/*=*/ u16Imm);
+            IEM_MC_STORE_GREG_U16(iReg, u16Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL_CONST(uint32_t, u32Value,/*=*/ u32Imm);
+            IEM_MC_STORE_GREG_U32(iReg, u32Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+        }
+        case IEMMODE_64BIT:
+        {
+            uint64_t u64Imm; IEM_OPCODE_GET_NEXT_U64(&u64Imm); /* 64-bit immediate! */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL_CONST(uint64_t, u64Value,/*=*/ u64Imm);
+            IEM_MC_STORE_GREG_U64(iReg, u64Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xb8
+ */
+FNIEMOP_DEF(iemOp_eAX_Iv)
+{
+    IEMOP_MNEMONIC(mov_rAX_IV, "mov rAX,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xAX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb9
+ */
+FNIEMOP_DEF(iemOp_eCX_Iv)
+{
+    IEMOP_MNEMONIC(mov_rCX_IV, "mov rCX,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xCX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xba
+ */
+FNIEMOP_DEF(iemOp_eDX_Iv)
+{
+    IEMOP_MNEMONIC(mov_rDX_IV, "mov rDX,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xDX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xbb
+ */
+FNIEMOP_DEF(iemOp_eBX_Iv)
+{
+    IEMOP_MNEMONIC(mov_rBX_IV, "mov rBX,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xBX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xbc
+ */
+FNIEMOP_DEF(iemOp_eSP_Iv)
+{
+    IEMOP_MNEMONIC(mov_rSP_IV, "mov rSP,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xSP | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xbd
+ */
+FNIEMOP_DEF(iemOp_eBP_Iv)
+{
+    IEMOP_MNEMONIC(mov_rBP_IV, "mov rBP,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xBP | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xbe
+ */
+FNIEMOP_DEF(iemOp_eSI_Iv)
+{
+    IEMOP_MNEMONIC(mov_rSI_IV, "mov rSI,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xSI | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xbf
+ */
+FNIEMOP_DEF(iemOp_eDI_Iv)
+{
+    IEMOP_MNEMONIC(mov_rDI_IV, "mov rDI,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xDI | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xc0
+ */
+FNIEMOP_DEF(iemOp_Grp2_Eb_Ib)
+{
+    IEMOP_HLP_MIN_186();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPSHIFTSIZES pImpl;
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0: pImpl = &g_iemAImpl_rol; IEMOP_MNEMONIC(rol_Eb_Ib, "rol Eb,Ib"); break;
+        case 1: pImpl = &g_iemAImpl_ror; IEMOP_MNEMONIC(ror_Eb_Ib, "ror Eb,Ib"); break;
+        case 2: pImpl = &g_iemAImpl_rcl; IEMOP_MNEMONIC(rcl_Eb_Ib, "rcl Eb,Ib"); break;
+        case 3: pImpl = &g_iemAImpl_rcr; IEMOP_MNEMONIC(rcr_Eb_Ib, "rcr Eb,Ib"); break;
+        case 4: pImpl = &g_iemAImpl_shl; IEMOP_MNEMONIC(shl_Eb_Ib, "shl Eb,Ib"); break;
+        case 5: pImpl = &g_iemAImpl_shr; IEMOP_MNEMONIC(shr_Eb_Ib, "shr Eb,Ib"); break;
+        case 7: pImpl = &g_iemAImpl_sar; IEMOP_MNEMONIC(sar_Eb_Ib, "sar Eb,Ib"); break;
+        case 6: return IEMOP_RAISE_INVALID_OPCODE();
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe stupid */
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register */
+        uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,            0);
+        IEM_MC_ARG_CONST(uint8_t,   cShiftArg, cShift, 1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+        IEM_MC_REF_GREG_U8(pu8Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint8_t *,   pu8Dst,    0);
+        IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+        uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+        IEM_MC_ASSIGN(cShiftArg, cShift);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xc1
+ */
+FNIEMOP_DEF(iemOp_Grp2_Ev_Ib)
+{
+    IEMOP_HLP_MIN_186();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPSHIFTSIZES pImpl;
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0: pImpl = &g_iemAImpl_rol; IEMOP_MNEMONIC(rol_Ev_Ib, "rol Ev,Ib"); break;
+        case 1: pImpl = &g_iemAImpl_ror; IEMOP_MNEMONIC(ror_Ev_Ib, "ror Ev,Ib"); break;
+        case 2: pImpl = &g_iemAImpl_rcl; IEMOP_MNEMONIC(rcl_Ev_Ib, "rcl Ev,Ib"); break;
+        case 3: pImpl = &g_iemAImpl_rcr; IEMOP_MNEMONIC(rcr_Ev_Ib, "rcr Ev,Ib"); break;
+        case 4: pImpl = &g_iemAImpl_shl; IEMOP_MNEMONIC(shl_Ev_Ib, "shl Ev,Ib"); break;
+        case 5: pImpl = &g_iemAImpl_shr; IEMOP_MNEMONIC(shr_Ev_Ib, "shr Ev,Ib"); break;
+        case 7: pImpl = &g_iemAImpl_sar; IEMOP_MNEMONIC(sar_Ev_Ib, "sar Ev,Ib"); break;
+        case 6: return IEMOP_RAISE_INVALID_OPCODE();
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe stupid */
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register */
+        uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,           0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg, cShift, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,           0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg, cShift, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,           0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg, cShift, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,  pu16Dst,    0);
+                IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+                IEM_MC_ASSIGN(cShiftArg, cShift);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,  pu32Dst,    0);
+                IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+                IEM_MC_ASSIGN(cShiftArg, cShift);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,  pu64Dst,    0);
+                IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+                IEM_MC_ASSIGN(cShiftArg, cShift);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0xc2
+ */
+FNIEMOP_DEF(iemOp_retn_Iw)
+{
+    IEMOP_MNEMONIC(retn_Iw, "retn Iw");
+    uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_retn, pVCpu->iem.s.enmEffOpSize, u16Imm);
+}
+
+
+/**
+ * @opcode      0xc3
+ */
+FNIEMOP_DEF(iemOp_retn)
+{
+    IEMOP_MNEMONIC(retn, "retn");
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_retn, pVCpu->iem.s.enmEffOpSize, 0);
+}
+
+
+/**
+ * @opcode      0xc4
+ */
+FNIEMOP_DEF(iemOp_les_Gv_Mp__vex3)
+{
+    /* The LDS instruction is invalid 64-bit mode. In legacy and
+       compatability mode it is invalid with MOD=3.
+       The use as a VEX prefix is made possible by assigning the inverted
+       REX.R and REX.X to the two MOD bits, since the REX bits are ignored
+       outside of 64-bit mode.  VEX is not available in real or v86 mode. */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (   pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+        || (bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT) )
+    {
+        IEMOP_MNEMONIC(vex3_prefix, "vex3");
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx)
+        {
+            /* Note! The real mode, v8086 mode and invalid prefix checks are done once
+                     the instruction is fully decoded.  Even when XCR0=3 and CR4.OSXSAVE=0. */
+            uint8_t bVex2;   IEM_OPCODE_GET_NEXT_U8(&bVex2);
+            uint8_t bOpcode; IEM_OPCODE_GET_NEXT_U8(&bOpcode);
+            pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_VEX;
+            if ((bVex2 & 0x80 /* VEX.W */) && pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+                pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_REX_W;
+            pVCpu->iem.s.uRexReg    = (~bRm >> (7 - 3)) & 0x8;
+            pVCpu->iem.s.uRexIndex  = (~bRm >> (6 - 3)) & 0x8;
+            pVCpu->iem.s.uRexB      = (~bRm >> (5 - 3)) & 0x8;
+            pVCpu->iem.s.uVex3rdReg = (~bVex2 >> 3) & 0xf;
+            pVCpu->iem.s.uVexLength = (bVex2 >> 2) & 1;
+            pVCpu->iem.s.idxPrefix  = bVex2 & 0x3;
+
+            switch (bRm & 0x1f)
+            {
+                case 1: /* 0x0f lead opcode byte. */
+#ifdef IEM_WITH_VEX
+                    return FNIEMOP_CALL(g_apfnVexMap1[(uintptr_t)bOpcode * 4 + pVCpu->iem.s.idxPrefix]);
+#else
+                    IEMOP_BITCH_ABOUT_STUB();
+                    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+
+                case 2: /* 0x0f 0x38 lead opcode bytes. */
+#ifdef IEM_WITH_VEX
+                    return FNIEMOP_CALL(g_apfnVexMap2[(uintptr_t)bOpcode * 4 + pVCpu->iem.s.idxPrefix]);
+#else
+                    IEMOP_BITCH_ABOUT_STUB();
+                    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+
+                case 3: /* 0x0f 0x3a lead opcode bytes. */
+#ifdef IEM_WITH_VEX
+                    return FNIEMOP_CALL(g_apfnVexMap3[(uintptr_t)bOpcode * 4 + pVCpu->iem.s.idxPrefix]);
+#else
+                    IEMOP_BITCH_ABOUT_STUB();
+                    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+
+                default:
+                    Log(("VEX3: Invalid vvvv value: %#x!\n", bRm & 0x1f));
+                    return IEMOP_RAISE_INVALID_OPCODE();
+            }
+        }
+        Log(("VEX3: AVX support disabled!\n"));
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+
+    IEMOP_MNEMONIC(les_Gv_Mp, "les Gv,Mp");
+    return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_ES, bRm);
+}
+
+
+/**
+ * @opcode      0xc5
+ */
+FNIEMOP_DEF(iemOp_lds_Gv_Mp__vex2)
+{
+    /* The LES instruction is invalid 64-bit mode. In legacy and
+       compatability mode it is invalid with MOD=3.
+       The use as a VEX prefix is made possible by assigning the inverted
+       REX.R to the top MOD bit, and the top bit in the inverted register
+       specifier to the bottom MOD bit, thereby effectively limiting 32-bit
+       to accessing registers 0..7 in this VEX form. */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (   pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+        || (bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_MNEMONIC(vex2_prefix, "vex2");
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx)
+        {
+            /* Note! The real mode, v8086 mode and invalid prefix checks are done once
+                     the instruction is fully decoded.  Even when XCR0=3 and CR4.OSXSAVE=0. */
+            uint8_t bOpcode; IEM_OPCODE_GET_NEXT_U8(&bOpcode);
+            pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_VEX;
+            pVCpu->iem.s.uRexReg    = (~bRm >> (7 - 3)) & 0x8;
+            pVCpu->iem.s.uVex3rdReg = (~bRm >> 3) & 0xf;
+            pVCpu->iem.s.uVexLength = (bRm >> 2) & 1;
+            pVCpu->iem.s.idxPrefix  = bRm & 0x3;
+
+#ifdef IEM_WITH_VEX
+            return FNIEMOP_CALL(g_apfnVexMap1[(uintptr_t)bOpcode * 4 + pVCpu->iem.s.idxPrefix]);
+#else
+            IEMOP_BITCH_ABOUT_STUB();
+            return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+        }
+
+        /** @todo does intel completely decode the sequence with SIB/disp before \#UD? */
+        Log(("VEX2: AVX support disabled!\n"));
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+
+    IEMOP_MNEMONIC(lds_Gv_Mp, "lds Gv,Mp");
+    return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_DS, bRm);
+}
+
+
+/**
+ * @opcode      0xc6
+ */
+FNIEMOP_DEF(iemOp_Grp11_Eb_Ib)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_REG_MASK) != (0 << X86_MODRM_REG_SHIFT)) /* only mov Eb,Ib in this group. */
+        return IEMOP_RAISE_INVALID_OPCODE();
+    IEMOP_MNEMONIC(mov_Eb_Ib, "mov Eb,Ib");
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register access */
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_STORE_GREG_U8((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u8Imm);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory access. */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_STORE_MEM_U8(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u8Imm);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xc7
+ */
+FNIEMOP_DEF(iemOp_Grp11_Ev_Iz)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_REG_MASK) != (0 << X86_MODRM_REG_SHIFT)) /* only mov Eb,Ib in this group. */
+        return IEMOP_RAISE_INVALID_OPCODE();
+    IEMOP_MNEMONIC(mov_Ev_Iz, "mov Ev,Iz");
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register access */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 0);
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_STORE_GREG_U16((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u16Imm);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 0);
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_STORE_GREG_U32((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u32Imm);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 0);
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_STORE_GREG_U64((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u64Imm);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory access. */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 2);
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Imm);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u32Imm);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u64Imm);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+
+
+/**
+ * @opcode      0xc8
+ */
+FNIEMOP_DEF(iemOp_enter_Iw_Ib)
+{
+    IEMOP_MNEMONIC(enter_Iw_Ib, "enter Iw,Ib");
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    uint16_t cbFrame;        IEM_OPCODE_GET_NEXT_U16(&cbFrame);
+    uint8_t  u8NestingLevel; IEM_OPCODE_GET_NEXT_U8(&u8NestingLevel);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_enter, pVCpu->iem.s.enmEffOpSize, cbFrame, u8NestingLevel);
+}
+
+
+/**
+ * @opcode      0xc9
+ */
+FNIEMOP_DEF(iemOp_leave)
+{
+    IEMOP_MNEMONIC(leave, "leave");
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_leave, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0xca
+ */
+FNIEMOP_DEF(iemOp_retf_Iw)
+{
+    IEMOP_MNEMONIC(retf_Iw, "retf Iw");
+    uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_retf, pVCpu->iem.s.enmEffOpSize, u16Imm);
+}
+
+
+/**
+ * @opcode      0xcb
+ */
+FNIEMOP_DEF(iemOp_retf)
+{
+    IEMOP_MNEMONIC(retf, "retf");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_retf, pVCpu->iem.s.enmEffOpSize, 0);
+}
+
+
+/**
+ * @opcode      0xcc
+ */
+FNIEMOP_DEF(iemOp_int3)
+{
+    IEMOP_MNEMONIC(int3, "int3");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_int, X86_XCPT_BP, IEMINT_INT3);
+}
+
+
+/**
+ * @opcode      0xcd
+ */
+FNIEMOP_DEF(iemOp_int_Ib)
+{
+    IEMOP_MNEMONIC(int_Ib, "int Ib");
+    uint8_t u8Int; IEM_OPCODE_GET_NEXT_U8(&u8Int);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_int, u8Int, IEMINT_INTN);
+}
+
+
+/**
+ * @opcode      0xce
+ */
+FNIEMOP_DEF(iemOp_into)
+{
+    IEMOP_MNEMONIC(into, "into");
+    IEMOP_HLP_NO_64BIT();
+
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG_CONST(uint8_t,   u8Int,  /*=*/ X86_XCPT_OF, 0);
+    IEM_MC_ARG_CONST(IEMINT,    enmInt, /*=*/ IEMINT_INTO, 1);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_int, u8Int, enmInt);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xcf
+ */
+FNIEMOP_DEF(iemOp_iret)
+{
+    IEMOP_MNEMONIC(iret, "iret");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_iret, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0xd0
+ */
+FNIEMOP_DEF(iemOp_Grp2_Eb_1)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPSHIFTSIZES pImpl;
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0: pImpl = &g_iemAImpl_rol; IEMOP_MNEMONIC(rol_Eb_1, "rol Eb,1"); break;
+        case 1: pImpl = &g_iemAImpl_ror; IEMOP_MNEMONIC(ror_Eb_1, "ror Eb,1"); break;
+        case 2: pImpl = &g_iemAImpl_rcl; IEMOP_MNEMONIC(rcl_Eb_1, "rcl Eb,1"); break;
+        case 3: pImpl = &g_iemAImpl_rcr; IEMOP_MNEMONIC(rcr_Eb_1, "rcr Eb,1"); break;
+        case 4: pImpl = &g_iemAImpl_shl; IEMOP_MNEMONIC(shl_Eb_1, "shl Eb,1"); break;
+        case 5: pImpl = &g_iemAImpl_shr; IEMOP_MNEMONIC(shr_Eb_1, "shr Eb,1"); break;
+        case 7: pImpl = &g_iemAImpl_sar; IEMOP_MNEMONIC(sar_Eb_1, "sar Eb,1"); break;
+        case 6: return IEMOP_RAISE_INVALID_OPCODE();
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe, well... */
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,             0);
+        IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=*/1,   1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+        IEM_MC_REF_GREG_U8(pu8Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,             0);
+        IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=*/1,   1);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags,        2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * @opcode      0xd1
+ */
+FNIEMOP_DEF(iemOp_Grp2_Ev_1)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPSHIFTSIZES pImpl;
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0: pImpl = &g_iemAImpl_rol; IEMOP_MNEMONIC(rol_Ev_1, "rol Ev,1"); break;
+        case 1: pImpl = &g_iemAImpl_ror; IEMOP_MNEMONIC(ror_Ev_1, "ror Ev,1"); break;
+        case 2: pImpl = &g_iemAImpl_rcl; IEMOP_MNEMONIC(rcl_Ev_1, "rcl Ev,1"); break;
+        case 3: pImpl = &g_iemAImpl_rcr; IEMOP_MNEMONIC(rcr_Ev_1, "rcr Ev,1"); break;
+        case 4: pImpl = &g_iemAImpl_shl; IEMOP_MNEMONIC(shl_Ev_1, "shl Ev,1"); break;
+        case 5: pImpl = &g_iemAImpl_shr; IEMOP_MNEMONIC(shr_Ev_1, "shr Ev,1"); break;
+        case 7: pImpl = &g_iemAImpl_sar; IEMOP_MNEMONIC(sar_Ev_1, "sar Ev,1"); break;
+        case 6: return IEMOP_RAISE_INVALID_OPCODE();
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe, well... */
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,           0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=1*/1, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,           0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=1*/1, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,           0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=1*/1, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,            0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=1*/1,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,            0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=1*/1,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,            0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=1*/1,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0xd2
+ */
+FNIEMOP_DEF(iemOp_Grp2_Eb_CL)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPSHIFTSIZES pImpl;
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0: pImpl = &g_iemAImpl_rol; IEMOP_MNEMONIC(rol_Eb_CL, "rol Eb,CL"); break;
+        case 1: pImpl = &g_iemAImpl_ror; IEMOP_MNEMONIC(ror_Eb_CL, "ror Eb,CL"); break;
+        case 2: pImpl = &g_iemAImpl_rcl; IEMOP_MNEMONIC(rcl_Eb_CL, "rcl Eb,CL"); break;
+        case 3: pImpl = &g_iemAImpl_rcr; IEMOP_MNEMONIC(rcr_Eb_CL, "rcr Eb,CL"); break;
+        case 4: pImpl = &g_iemAImpl_shl; IEMOP_MNEMONIC(shl_Eb_CL, "shl Eb,CL"); break;
+        case 5: pImpl = &g_iemAImpl_shr; IEMOP_MNEMONIC(shr_Eb_CL, "shr Eb,CL"); break;
+        case 7: pImpl = &g_iemAImpl_sar; IEMOP_MNEMONIC(sar_Eb_CL, "sar Eb,CL"); break;
+        case 6: return IEMOP_RAISE_INVALID_OPCODE();
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc, grr. */
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,   pu8Dst,     0);
+        IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+        IEM_MC_ARG(uint32_t *,  pEFlags,    2);
+        IEM_MC_REF_GREG_U8(pu8Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint8_t *,   pu8Dst,          0);
+        IEM_MC_ARG(uint8_t,     cShiftArg,       1);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xd3
+ */
+FNIEMOP_DEF(iemOp_Grp2_Ev_CL)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPSHIFTSIZES pImpl;
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0: pImpl = &g_iemAImpl_rol; IEMOP_MNEMONIC(rol_Ev_CL, "rol Ev,CL"); break;
+        case 1: pImpl = &g_iemAImpl_ror; IEMOP_MNEMONIC(ror_Ev_CL, "ror Ev,CL"); break;
+        case 2: pImpl = &g_iemAImpl_rcl; IEMOP_MNEMONIC(rcl_Ev_CL, "rcl Ev,CL"); break;
+        case 3: pImpl = &g_iemAImpl_rcr; IEMOP_MNEMONIC(rcr_Ev_CL, "rcr Ev,CL"); break;
+        case 4: pImpl = &g_iemAImpl_shl; IEMOP_MNEMONIC(shl_Ev_CL, "shl Ev,CL"); break;
+        case 5: pImpl = &g_iemAImpl_shr; IEMOP_MNEMONIC(shr_Ev_CL, "shr Ev,CL"); break;
+        case 7: pImpl = &g_iemAImpl_sar; IEMOP_MNEMONIC(sar_Ev_CL, "sar Ev,CL"); break;
+        case 6: return IEMOP_RAISE_INVALID_OPCODE();
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe stupid */
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,    0);
+                IEM_MC_ARG(uint8_t,         cShiftArg,  1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    2);
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,    0);
+                IEM_MC_ARG(uint8_t,         cShiftArg,  1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    2);
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,    0);
+                IEM_MC_ARG(uint8_t,         cShiftArg,  1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    2);
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,  pu16Dst,    0);
+                IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,  pu32Dst,    0);
+                IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,  pu64Dst,    0);
+                IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+/**
+ * @opcode      0xd4
+ */
+FNIEMOP_DEF(iemOp_aam_Ib)
+{
+    IEMOP_MNEMONIC(aam_Ib, "aam Ib");
+    uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_NO_64BIT();
+    if (!bImm)
+        return IEMOP_RAISE_DIVIDE_ERROR();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_aam, bImm);
+}
+
+
+/**
+ * @opcode      0xd5
+ */
+FNIEMOP_DEF(iemOp_aad_Ib)
+{
+    IEMOP_MNEMONIC(aad_Ib, "aad Ib");
+    uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_NO_64BIT();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_aad, bImm);
+}
+
+
+/**
+ * @opcode      0xd6
+ */
+FNIEMOP_DEF(iemOp_salc)
+{
+    IEMOP_MNEMONIC(salc, "salc");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_NO_64BIT();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+        IEM_MC_STORE_GREG_U8_CONST(X86_GREG_xAX, 0xff);
+    } IEM_MC_ELSE() {
+        IEM_MC_STORE_GREG_U8_CONST(X86_GREG_xAX, 0x00);
+    } IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xd7
+ */
+FNIEMOP_DEF(iemOp_xlat)
+{
+    IEMOP_MNEMONIC(xlat, "xlat");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_LOCAL(uint8_t,  u8Tmp);
+            IEM_MC_LOCAL(uint16_t, u16Addr);
+            IEM_MC_FETCH_GREG_U8_ZX_U16(u16Addr, X86_GREG_xAX);
+            IEM_MC_ADD_GREG_U16_TO_LOCAL(u16Addr, X86_GREG_xBX);
+            IEM_MC_FETCH_MEM16_U8(u8Tmp, pVCpu->iem.s.iEffSeg, u16Addr);
+            IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Tmp);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_LOCAL(uint8_t,  u8Tmp);
+            IEM_MC_LOCAL(uint32_t, u32Addr);
+            IEM_MC_FETCH_GREG_U8_ZX_U32(u32Addr, X86_GREG_xAX);
+            IEM_MC_ADD_GREG_U32_TO_LOCAL(u32Addr, X86_GREG_xBX);
+            IEM_MC_FETCH_MEM32_U8(u8Tmp, pVCpu->iem.s.iEffSeg, u32Addr);
+            IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Tmp);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_LOCAL(uint8_t,  u8Tmp);
+            IEM_MC_LOCAL(uint64_t, u64Addr);
+            IEM_MC_FETCH_GREG_U8_ZX_U64(u64Addr, X86_GREG_xAX);
+            IEM_MC_ADD_GREG_U64_TO_LOCAL(u64Addr, X86_GREG_xBX);
+            IEM_MC_FETCH_MEM_U8(u8Tmp, pVCpu->iem.s.iEffSeg, u64Addr);
+            IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Tmp);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+         IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and STn, and storing the
+ * result in ST0.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_st0_stN, uint8_t, bRm, PFNIEMAIMPLFPUR80, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value2,                 2);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, bRm & X86_MODRM_RM_MASK)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pr80Value2);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, 0);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and STn, and only affecting
+ * flags.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpuNoStore_st0_stN, uint8_t, bRm, PFNIEMAIMPLFPUR80FSW, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value2,                 2);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, bRm & X86_MODRM_RM_MASK)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pu16Fsw, pr80Value1, pr80Value2);
+        IEM_MC_UPDATE_FSW(u16Fsw);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(UINT8_MAX);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and STn, only affecting
+ * flags, and popping when done.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpuNoStore_st0_stN_pop, uint8_t, bRm, PFNIEMAIMPLFPUR80FSW, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value2,                 2);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, bRm & X86_MODRM_RM_MASK)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pu16Fsw, pr80Value1, pr80Value2);
+        IEM_MC_UPDATE_FSW_THEN_POP(u16Fsw);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(UINT8_MAX);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd8 11/0. */
+FNIEMOP_DEF_1(iemOp_fadd_stN,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fadd_st0_stN, "fadd st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fadd_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/1. */
+FNIEMOP_DEF_1(iemOp_fmul_stN,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fmul_st0_stN, "fmul st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fmul_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/2. */
+FNIEMOP_DEF_1(iemOp_fcom_stN,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcom_st0_stN, "fcom st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpuNoStore_st0_stN, bRm, iemAImpl_fcom_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/3. */
+FNIEMOP_DEF_1(iemOp_fcomp_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcomp_st0_stN, "fcomp st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpuNoStore_st0_stN_pop, bRm, iemAImpl_fcom_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/4. */
+FNIEMOP_DEF_1(iemOp_fsub_stN,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsub_st0_stN, "fsub st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fsub_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/5. */
+FNIEMOP_DEF_1(iemOp_fsubr_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsubr_st0_stN, "fsubr st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fsubr_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/6. */
+FNIEMOP_DEF_1(iemOp_fdiv_stN,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdiv_st0_stN, "fdiv st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fdiv_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/7. */
+FNIEMOP_DEF_1(iemOp_fdivr_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdivr_st0_stN, "fdivr st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fdivr_r80_by_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and an m32r, and storing
+ * the result in ST0.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_st0_m32r, uint8_t, bRm, PFNIEMAIMPLFPUR32, pfnAImpl)
+{
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_LOCAL(RTFLOAT32U,            r32Val2);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U,  pr32Val2,       r32Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R32(r32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pr32Val2);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, 0);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd8 !11/0. */
+FNIEMOP_DEF_1(iemOp_fadd_m32r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fadd_st0_m32r, "fadd st0,m32r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fadd_r80_by_r32);
+}
+
+
+/** Opcode 0xd8 !11/1. */
+FNIEMOP_DEF_1(iemOp_fmul_m32r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fmul_st0_m32r, "fmul st0,m32r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fmul_r80_by_r32);
+}
+
+
+/** Opcode 0xd8 !11/2. */
+FNIEMOP_DEF_1(iemOp_fcom_m32r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcom_st0_m32r, "fcom st0,m32r");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_LOCAL(RTFLOAT32U,            r32Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U,  pr32Val2,       r32Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R32(r32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fcom_r80_by_r32, pu16Fsw, pr80Value1, pr32Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd8 !11/3. */
+FNIEMOP_DEF_1(iemOp_fcomp_m32r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcomp_st0_m32r, "fcomp st0,m32r");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_LOCAL(RTFLOAT32U,            r32Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U,  pr32Val2,       r32Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R32(r32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fcom_r80_by_r32, pu16Fsw, pr80Value1, pr32Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd8 !11/4. */
+FNIEMOP_DEF_1(iemOp_fsub_m32r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsub_st0_m32r, "fsub st0,m32r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fsub_r80_by_r32);
+}
+
+
+/** Opcode 0xd8 !11/5. */
+FNIEMOP_DEF_1(iemOp_fsubr_m32r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsubr_st0_m32r, "fsubr st0,m32r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fsubr_r80_by_r32);
+}
+
+
+/** Opcode 0xd8 !11/6. */
+FNIEMOP_DEF_1(iemOp_fdiv_m32r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdiv_st0_m32r, "fdiv st0,m32r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fdiv_r80_by_r32);
+}
+
+
+/** Opcode 0xd8 !11/7. */
+FNIEMOP_DEF_1(iemOp_fdivr_m32r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdivr_st0_m32r, "fdivr st0,m32r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fdivr_r80_by_r32);
+}
+
+
+/**
+ * @opcode      0xd8
+ */
+FNIEMOP_DEF(iemOp_EscF0)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xd8 & 0x7);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fadd_stN,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fmul_stN,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcom_stN,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fcomp_stN, bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fsub_stN,  bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fsubr_stN, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fdiv_stN,  bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fdivr_stN, bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fadd_m32r,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fmul_m32r,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcom_m32r,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fcomp_m32r, bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fsub_m32r,  bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fsubr_m32r, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fdiv_m32r,  bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fdivr_m32r, bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xd9 /0 mem32real
+ * @sa  iemOp_fld_m64r */
+FNIEMOP_DEF_1(iemOp_fld_m32r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fld_m32r, "fld m32r");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_LOCAL(RTFLOAT32U,            r32Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U,  pr32Val,    r32Val, 1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R32(r32Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fld_r32_to_r80, pFpuRes, pr32Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 !11/2 mem32real */
+FNIEMOP_DEF_1(iemOp_fst_m32r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fst_m32r, "fst m32r");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PRTFLOAT32U,             pr32Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pr32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r32, pu16Fsw, pr32Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr32Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(pr32Dst);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pr32Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 !11/3 */
+FNIEMOP_DEF_1(iemOp_fstp_m32r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fstp_m32r, "fstp m32r");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PRTFLOAT32U,             pr32Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pr32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r32, pu16Fsw, pr32Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr32Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(pr32Dst);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pr32Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 !11/4 */
+FNIEMOP_DEF_1(iemOp_fldenv, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fldenv, "fldenv m14/28byte");
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG_CONST(IEMMODE,           enmEffOpSize, /*=*/ pVCpu->iem.s.enmEffOpSize,  0);
+    IEM_MC_ARG(uint8_t,                 iEffSeg,                                    1);
+    IEM_MC_ARG(RTGCPTR,                 GCPtrEffSrc,                                2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_fldenv, enmEffOpSize, iEffSeg, GCPtrEffSrc);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 !11/5 */
+FNIEMOP_DEF_1(iemOp_fldcw, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fldcw_m2byte, "fldcw m2byte");
+    IEM_MC_BEGIN(1, 1);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_ARG(uint16_t,                u16Fsw,                                     0);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FETCH_MEM_U16(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_CALL_CIMPL_1(iemCImpl_fldcw, u16Fsw);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 !11/6 */
+FNIEMOP_DEF_1(iemOp_fnstenv, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fstenv, "fstenv m14/m28byte");
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG_CONST(IEMMODE,           enmEffOpSize, /*=*/ pVCpu->iem.s.enmEffOpSize,  0);
+    IEM_MC_ARG(uint8_t,                 iEffSeg,                                    1);
+    IEM_MC_ARG(RTGCPTR,                 GCPtrEffDst,                                2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_fnstenv, enmEffOpSize, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 !11/7 */
+FNIEMOP_DEF_1(iemOp_fnstcw, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fnstcw_m2byte, "fnstcw m2byte");
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fcw);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_FETCH_FCW(u16Fcw);
+    IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Fcw);
+    IEM_MC_ADVANCE_RIP(); /* C0-C3 are documented as undefined, we leave them unmodified. */
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 0xd0, 0xd9 0xd8-0xdf, ++?.  */
+FNIEMOP_DEF(iemOp_fnop)
+{
+    IEMOP_MNEMONIC(fnop, "fnop");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    /** @todo Testcase: looks like FNOP leaves FOP alone but updates FPUIP. Could be
+     *        intel optimizations. Investigate. */
+    IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ADVANCE_RIP(); /* C0-C3 are documented as undefined, we leave them unmodified. */
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 11/0 stN */
+FNIEMOP_DEF_1(iemOp_fld_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fld_stN, "fld stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /** @todo Testcase: Check if this raises \#MF?  Intel mentioned it not. AMD
+     *        indicates that it does. */
+    IEM_MC_BEGIN(0, 2);
+    IEM_MC_LOCAL(PCRTFLOAT80U,          pr80Value);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, bRm & X86_MODRM_RM_MASK)
+        IEM_MC_SET_FPU_RESULT(FpuRes, 0 /*FSW*/, pr80Value);
+        IEM_MC_PUSH_FPU_RESULT(FpuRes);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_UNDERFLOW();
+    IEM_MC_ENDIF();
+
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 11/3 stN */
+FNIEMOP_DEF_1(iemOp_fxch_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fxch_stN, "fxch stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /** @todo Testcase: Check if this raises \#MF?  Intel mentioned it not. AMD
+     *        indicates that it does. */
+    IEM_MC_BEGIN(1, 3);
+    IEM_MC_LOCAL(PCRTFLOAT80U,          pr80Value1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,          pr80Value2);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_ARG_CONST(uint8_t,           iStReg, /*=*/ bRm & X86_MODRM_RM_MASK, 0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, bRm & X86_MODRM_RM_MASK)
+        IEM_MC_SET_FPU_RESULT(FpuRes, X86_FSW_C1, pr80Value2);
+        IEM_MC_STORE_FPUREG_R80_SRC_REF(bRm & X86_MODRM_RM_MASK, pr80Value1);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, 0);
+    IEM_MC_ELSE()
+        IEM_MC_CALL_CIMPL_1(iemCImpl_fxch_underflow, iStReg);
+    IEM_MC_ENDIF();
+
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 11/4, 0xdd 11/2. */
+FNIEMOP_DEF_1(iemOp_fstp_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fstp_st0_stN, "fstp st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /* fstp st0, st0 is frequently used as an official 'ffreep st0' sequence. */
+    uint8_t const iDstReg = bRm & X86_MODRM_RM_MASK;
+    if (!iDstReg)
+    {
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL_CONST(uint16_t,        u16Fsw, /*=*/ 0);
+        IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+        IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_IF_FPUREG_NOT_EMPTY(0)
+            IEM_MC_UPDATE_FSW_THEN_POP(u16Fsw);
+        IEM_MC_ELSE()
+            IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(0);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(PCRTFLOAT80U,          pr80Value);
+        IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+        IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+        IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+            IEM_MC_SET_FPU_RESULT(FpuRes, 0 /*FSW*/, pr80Value);
+            IEM_MC_STORE_FPU_RESULT_THEN_POP(FpuRes, iDstReg);
+        IEM_MC_ELSE()
+            IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(iDstReg);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and replaces it with the
+ * result, i.e. unary operators.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpFpu_st0, PFNIEMAIMPLFPUR80UNARY, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(2, 1);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          1);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_2(pfnAImpl, pFpuRes, pr80Value);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, 0);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 0xe0. */
+FNIEMOP_DEF(iemOp_fchs)
+{
+    IEMOP_MNEMONIC(fchs_st0, "fchs st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fchs_r80);
+}
+
+
+/** Opcode 0xd9 0xe1. */
+FNIEMOP_DEF(iemOp_fabs)
+{
+    IEMOP_MNEMONIC(fabs_st0, "fabs st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fabs_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and only returns FSW.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpFpuNoStore_st0, PFNIEMAIMPLFPUR80UNARYFSW, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(2, 1);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          1);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_2(pfnAImpl, pu16Fsw, pr80Value);
+        IEM_MC_UPDATE_FSW(u16Fsw);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(UINT8_MAX);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 0xe4. */
+FNIEMOP_DEF(iemOp_ftst)
+{
+    IEMOP_MNEMONIC(ftst_st0, "ftst st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpuNoStore_st0, iemAImpl_ftst_r80);
+}
+
+
+/** Opcode 0xd9 0xe5. */
+FNIEMOP_DEF(iemOp_fxam)
+{
+    IEMOP_MNEMONIC(fxam_st0, "fxam st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpuNoStore_st0, iemAImpl_fxam_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions pushing a constant onto the FPU stack.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpFpuPushConstant, PFNIEMAIMPLFPUR80LDCONST, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(1, 1);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,    FpuRes, 0);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_1(pfnAImpl, pFpuRes);
+        IEM_MC_PUSH_FPU_RESULT(FpuRes);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW();
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 0xe8. */
+FNIEMOP_DEF(iemOp_fld1)
+{
+    IEMOP_MNEMONIC(fld1, "fld1");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fld1);
+}
+
+
+/** Opcode 0xd9 0xe9. */
+FNIEMOP_DEF(iemOp_fldl2t)
+{
+    IEMOP_MNEMONIC(fldl2t, "fldl2t");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldl2t);
+}
+
+
+/** Opcode 0xd9 0xea. */
+FNIEMOP_DEF(iemOp_fldl2e)
+{
+    IEMOP_MNEMONIC(fldl2e, "fldl2e");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldl2e);
+}
+
+/** Opcode 0xd9 0xeb. */
+FNIEMOP_DEF(iemOp_fldpi)
+{
+    IEMOP_MNEMONIC(fldpi, "fldpi");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldpi);
+}
+
+
+/** Opcode 0xd9 0xec. */
+FNIEMOP_DEF(iemOp_fldlg2)
+{
+    IEMOP_MNEMONIC(fldlg2, "fldlg2");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldlg2);
+}
+
+/** Opcode 0xd9 0xed. */
+FNIEMOP_DEF(iemOp_fldln2)
+{
+    IEMOP_MNEMONIC(fldln2, "fldln2");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldln2);
+}
+
+
+/** Opcode 0xd9 0xee. */
+FNIEMOP_DEF(iemOp_fldz)
+{
+    IEMOP_MNEMONIC(fldz, "fldz");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldz);
+}
+
+
+/** Opcode 0xd9 0xf0. */
+FNIEMOP_DEF(iemOp_f2xm1)
+{
+    IEMOP_MNEMONIC(f2xm1_st0, "f2xm1 st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_f2xm1_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on STn and ST0, storing the result
+ * in STn, and popping the stack unless IE, DE or ZE was raised.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_stN_st0_pop, uint8_t, bRm, PFNIEMAIMPLFPUR80, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value2,                 2);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, bRm & X86_MODRM_RM_MASK, pr80Value2, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pr80Value2);
+        IEM_MC_STORE_FPU_RESULT_THEN_POP(FpuRes, bRm & X86_MODRM_RM_MASK);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(bRm & X86_MODRM_RM_MASK);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 0xf1. */
+FNIEMOP_DEF(iemOp_fyl2x)
+{
+    IEMOP_MNEMONIC(fyl2x_st0, "fyl2x st1,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, 1, iemAImpl_fyl2x_r80_by_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and having two outputs, one
+ * replacing ST0 and one pushed onto the stack.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpFpuReplace_st0_push, PFNIEMAIMPLFPUR80UNARYTWO, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(2, 1);
+    IEM_MC_LOCAL(IEMFPURESULTTWO,           FpuResTwo);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULTTWO,  pFpuResTwo, FpuResTwo,  0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value,              1);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_2(pfnAImpl, pFpuResTwo, pr80Value);
+        IEM_MC_PUSH_FPU_RESULT_TWO(FpuResTwo);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO();
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 0xf2. */
+FNIEMOP_DEF(iemOp_fptan)
+{
+    IEMOP_MNEMONIC(fptan_st0, "fptan st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpuReplace_st0_push, iemAImpl_fptan_r80_r80);
+}
+
+
+/** Opcode 0xd9 0xf3. */
+FNIEMOP_DEF(iemOp_fpatan)
+{
+    IEMOP_MNEMONIC(fpatan_st1_st0, "fpatan st1,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, 1, iemAImpl_fpatan_r80_by_r80);
+}
+
+
+/** Opcode 0xd9 0xf4. */
+FNIEMOP_DEF(iemOp_fxtract)
+{
+    IEMOP_MNEMONIC(fxtract_st0, "fxtract st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpuReplace_st0_push, iemAImpl_fxtract_r80_r80);
+}
+
+
+/** Opcode 0xd9 0xf5. */
+FNIEMOP_DEF(iemOp_fprem1)
+{
+    IEMOP_MNEMONIC(fprem1_st0_st1, "fprem1 st0,st1");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, 1, iemAImpl_fprem1_r80_by_r80);
+}
+
+
+/** Opcode 0xd9 0xf6. */
+FNIEMOP_DEF(iemOp_fdecstp)
+{
+    IEMOP_MNEMONIC(fdecstp, "fdecstp");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    /* Note! C0, C2 and C3 are documented as undefined, we clear them. */
+    /** @todo Testcase: Check whether FOP, FPUIP and FPUCS are affected by
+     *        FINCSTP and FDECSTP. */
+
+    IEM_MC_BEGIN(0,0);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FPU_STACK_DEC_TOP();
+    IEM_MC_UPDATE_FSW_CONST(0);
+
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 0xf7. */
+FNIEMOP_DEF(iemOp_fincstp)
+{
+    IEMOP_MNEMONIC(fincstp, "fincstp");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    /* Note! C0, C2 and C3 are documented as undefined, we clear them. */
+    /** @todo Testcase: Check whether FOP, FPUIP and FPUCS are affected by
+     *        FINCSTP and FDECSTP. */
+
+    IEM_MC_BEGIN(0,0);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FPU_STACK_INC_TOP();
+    IEM_MC_UPDATE_FSW_CONST(0);
+
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 0xf8. */
+FNIEMOP_DEF(iemOp_fprem)
+{
+    IEMOP_MNEMONIC(fprem_st0_st1, "fprem st0,st1");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, 1, iemAImpl_fprem_r80_by_r80);
+}
+
+
+/** Opcode 0xd9 0xf9. */
+FNIEMOP_DEF(iemOp_fyl2xp1)
+{
+    IEMOP_MNEMONIC(fyl2xp1_st1_st0, "fyl2xp1 st1,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, 1, iemAImpl_fyl2xp1_r80_by_r80);
+}
+
+
+/** Opcode 0xd9 0xfa. */
+FNIEMOP_DEF(iemOp_fsqrt)
+{
+    IEMOP_MNEMONIC(fsqrt_st0, "fsqrt st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fsqrt_r80);
+}
+
+
+/** Opcode 0xd9 0xfb. */
+FNIEMOP_DEF(iemOp_fsincos)
+{
+    IEMOP_MNEMONIC(fsincos_st0, "fsincos st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpuReplace_st0_push, iemAImpl_fsincos_r80_r80);
+}
+
+
+/** Opcode 0xd9 0xfc. */
+FNIEMOP_DEF(iemOp_frndint)
+{
+    IEMOP_MNEMONIC(frndint_st0, "frndint st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_frndint_r80);
+}
+
+
+/** Opcode 0xd9 0xfd. */
+FNIEMOP_DEF(iemOp_fscale)
+{
+    IEMOP_MNEMONIC(fscale_st0_st1, "fscale st0,st1");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, 1, iemAImpl_fscale_r80_by_r80);
+}
+
+
+/** Opcode 0xd9 0xfe. */
+FNIEMOP_DEF(iemOp_fsin)
+{
+    IEMOP_MNEMONIC(fsin_st0, "fsin st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fsin_r80);
+}
+
+
+/** Opcode 0xd9 0xff. */
+FNIEMOP_DEF(iemOp_fcos)
+{
+    IEMOP_MNEMONIC(fcos_st0, "fcos st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fcos_r80);
+}
+
+
+/** Used by iemOp_EscF1. */
+IEM_STATIC const PFNIEMOP g_apfnEscF1_E0toFF[32] =
+{
+    /* 0xe0 */  iemOp_fchs,
+    /* 0xe1 */  iemOp_fabs,
+    /* 0xe2 */  iemOp_Invalid,
+    /* 0xe3 */  iemOp_Invalid,
+    /* 0xe4 */  iemOp_ftst,
+    /* 0xe5 */  iemOp_fxam,
+    /* 0xe6 */  iemOp_Invalid,
+    /* 0xe7 */  iemOp_Invalid,
+    /* 0xe8 */  iemOp_fld1,
+    /* 0xe9 */  iemOp_fldl2t,
+    /* 0xea */  iemOp_fldl2e,
+    /* 0xeb */  iemOp_fldpi,
+    /* 0xec */  iemOp_fldlg2,
+    /* 0xed */  iemOp_fldln2,
+    /* 0xee */  iemOp_fldz,
+    /* 0xef */  iemOp_Invalid,
+    /* 0xf0 */  iemOp_f2xm1,
+    /* 0xf1 */  iemOp_fyl2x,
+    /* 0xf2 */  iemOp_fptan,
+    /* 0xf3 */  iemOp_fpatan,
+    /* 0xf4 */  iemOp_fxtract,
+    /* 0xf5 */  iemOp_fprem1,
+    /* 0xf6 */  iemOp_fdecstp,
+    /* 0xf7 */  iemOp_fincstp,
+    /* 0xf8 */  iemOp_fprem,
+    /* 0xf9 */  iemOp_fyl2xp1,
+    /* 0xfa */  iemOp_fsqrt,
+    /* 0xfb */  iemOp_fsincos,
+    /* 0xfc */  iemOp_frndint,
+    /* 0xfd */  iemOp_fscale,
+    /* 0xfe */  iemOp_fsin,
+    /* 0xff */  iemOp_fcos
+};
+
+
+/**
+ * @opcode      0xd9
+ */
+FNIEMOP_DEF(iemOp_EscF1)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xd9 & 0x7);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fld_stN, bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fxch_stN, bRm);
+            case 2:
+                if (bRm == 0xd0)
+                    return FNIEMOP_CALL(iemOp_fnop);
+                return IEMOP_RAISE_INVALID_OPCODE();
+            case 3: return FNIEMOP_CALL_1(iemOp_fstp_stN, bRm); /* Reserved. Intel behavior seems to be FSTP ST(i) though. */
+            case 4:
+            case 5:
+            case 6:
+            case 7:
+                Assert((unsigned)bRm - 0xe0U < RT_ELEMENTS(g_apfnEscF1_E0toFF));
+                return FNIEMOP_CALL(g_apfnEscF1_E0toFF[bRm - 0xe0]);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fld_m32r,  bRm);
+            case 1: return IEMOP_RAISE_INVALID_OPCODE();
+            case 2: return FNIEMOP_CALL_1(iemOp_fst_m32r,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fstp_m32r, bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fldenv,    bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fldcw,     bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fnstenv,    bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fnstcw,     bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xda 11/0. */
+FNIEMOP_DEF_1(iemOp_fcmovb_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovb_st0_stN, "fcmovb st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, bRm & X86_MODRM_RM_MASK, 0)
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xda 11/1. */
+FNIEMOP_DEF_1(iemOp_fcmove_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmove_st0_stN, "fcmove st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, bRm & X86_MODRM_RM_MASK, 0)
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xda 11/2. */
+FNIEMOP_DEF_1(iemOp_fcmovbe_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovbe_st0_stN, "fcmovbe st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, bRm & X86_MODRM_RM_MASK, 0)
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xda 11/3. */
+FNIEMOP_DEF_1(iemOp_fcmovu_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovu_st0_stN, "fcmovu st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, bRm & X86_MODRM_RM_MASK, 0)
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and STn, only affecting
+ * flags, and popping twice when done.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpFpuNoStore_st0_stN_pop_pop, PFNIEMAIMPLFPUR80FSW, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value2,                 2);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, 1)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pu16Fsw, pr80Value1, pr80Value2);
+        IEM_MC_UPDATE_FSW_THEN_POP_POP(u16Fsw);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP();
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xda 0xe9. */
+FNIEMOP_DEF(iemOp_fucompp)
+{
+    IEMOP_MNEMONIC(fucompp_st0_stN, "fucompp st0,stN");
+    return FNIEMOP_CALL_1(iemOpHlpFpuNoStore_st0_stN_pop_pop, iemAImpl_fucom_r80_by_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and an m32i, and storing
+ * the result in ST0.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_st0_m32i, uint8_t, bRm, PFNIEMAIMPLFPUI32, pfnAImpl)
+{
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,              FpuRes);
+    IEM_MC_LOCAL(int32_t,                   i32Val2);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT,     pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(int32_t const *,   pi32Val2,       i32Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I32(i32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pi32Val2);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, 0);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xda !11/0. */
+FNIEMOP_DEF_1(iemOp_fiadd_m32i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fiadd_m32i, "fiadd m32i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fiadd_r80_by_i32);
+}
+
+
+/** Opcode 0xda !11/1. */
+FNIEMOP_DEF_1(iemOp_fimul_m32i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fimul_m32i, "fimul m32i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fimul_r80_by_i32);
+}
+
+
+/** Opcode 0xda !11/2. */
+FNIEMOP_DEF_1(iemOp_ficom_m32i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(ficom_st0_m32i, "ficom st0,m32i");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,                  u16Fsw);
+    IEM_MC_LOCAL(int32_t,                   i32Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,        pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(int32_t const *,   pi32Val2,       i32Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I32(i32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_ficom_r80_by_i32, pu16Fsw, pr80Value1, pi32Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xda !11/3. */
+FNIEMOP_DEF_1(iemOp_ficomp_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(ficomp_st0_m32i, "ficomp st0,m32i");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,                  u16Fsw);
+    IEM_MC_LOCAL(int32_t,                   i32Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,        pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(int32_t const *,   pi32Val2,       i32Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I32(i32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_ficom_r80_by_i32, pu16Fsw, pr80Value1, pi32Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xda !11/4. */
+FNIEMOP_DEF_1(iemOp_fisub_m32i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisub_m32i, "fisub m32i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fisub_r80_by_i32);
+}
+
+
+/** Opcode 0xda !11/5. */
+FNIEMOP_DEF_1(iemOp_fisubr_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisubr_m32i, "fisubr m32i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fisubr_r80_by_i32);
+}
+
+
+/** Opcode 0xda !11/6. */
+FNIEMOP_DEF_1(iemOp_fidiv_m32i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fidiv_m32i, "fidiv m32i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fidiv_r80_by_i32);
+}
+
+
+/** Opcode 0xda !11/7. */
+FNIEMOP_DEF_1(iemOp_fidivr_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fidivr_m32i, "fidivr m32i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fidivr_r80_by_i32);
+}
+
+
+/**
+ * @opcode      0xda
+ */
+FNIEMOP_DEF(iemOp_EscF2)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xda & 0x7);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fcmovb_stN, bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fcmove_stN, bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcmovbe_stN, bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fcmovu_stN, bRm);
+            case 4: return IEMOP_RAISE_INVALID_OPCODE();
+            case 5:
+                if (bRm == 0xe9)
+                    return FNIEMOP_CALL(iemOp_fucompp);
+                return IEMOP_RAISE_INVALID_OPCODE();
+            case 6: return IEMOP_RAISE_INVALID_OPCODE();
+            case 7: return IEMOP_RAISE_INVALID_OPCODE();
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fiadd_m32i,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fimul_m32i,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_ficom_m32i,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_ficomp_m32i, bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fisub_m32i,  bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fisubr_m32i, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fidiv_m32i,  bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fidivr_m32i, bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xdb !11/0. */
+FNIEMOP_DEF_1(iemOp_fild_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fild_m32i, "fild m32i");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,              FpuRes);
+    IEM_MC_LOCAL(int32_t,                   i32Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT,     pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG_LOCAL_REF(int32_t const *,   pi32Val,    i32Val, 1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I32(i32Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fild_i32_to_r80, pFpuRes, pi32Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb !11/1. */
+FNIEMOP_DEF_1(iemOp_fisttp_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisttp_m32i, "fisttp m32i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int32_t *,               pi32Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fistt_r80_to_i32, pu16Fsw, pi32Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi32Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I32_CONST_BY_REF(pi32Dst, INT32_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi32Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb !11/2. */
+FNIEMOP_DEF_1(iemOp_fist_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fist_m32i, "fist m32i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int32_t *,               pi32Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i32, pu16Fsw, pi32Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi32Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I32_CONST_BY_REF(pi32Dst, INT32_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi32Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb !11/3. */
+FNIEMOP_DEF_1(iemOp_fistp_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fistp_m32i, "fistp m32i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int32_t *,               pi32Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i32, pu16Fsw, pi32Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi32Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I32_CONST_BY_REF(pi32Dst, INT32_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi32Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb !11/5. */
+FNIEMOP_DEF_1(iemOp_fld_m80r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fld_m80r, "fld m80r");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_LOCAL(RTFLOAT80U,            r80Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT80U,  pr80Val,    r80Val, 1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R80(r80Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fld_r80_from_r80, pFpuRes, pr80Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb !11/7. */
+FNIEMOP_DEF_1(iemOp_fstp_m80r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fstp_m80r, "fstp m80r");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PRTFLOAT80U,             pr80Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pr80Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r80, pu16Fsw, pr80Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr80Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(pr80Dst);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pr80Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb 11/0. */
+FNIEMOP_DEF_1(iemOp_fcmovnb_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovnb_st0_stN, "fcmovnb st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, bRm & X86_MODRM_RM_MASK, 0)
+        IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_CF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb 11/1. */
+FNIEMOP_DEF_1(iemOp_fcmovne_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovne_st0_stN, "fcmovne st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, bRm & X86_MODRM_RM_MASK, 0)
+        IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb 11/2. */
+FNIEMOP_DEF_1(iemOp_fcmovnbe_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovnbe_st0_stN, "fcmovnbe st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, bRm & X86_MODRM_RM_MASK, 0)
+        IEM_MC_IF_EFL_NO_BITS_SET(X86_EFL_CF | X86_EFL_ZF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb 11/3. */
+FNIEMOP_DEF_1(iemOp_fcmovnnu_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovnnu_st0_stN, "fcmovnnu st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, bRm & X86_MODRM_RM_MASK, 0)
+        IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_PF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb 0xe0. */
+FNIEMOP_DEF(iemOp_fneni)
+{
+    IEMOP_MNEMONIC(fneni, "fneni (8087/ign)");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb 0xe1. */
+FNIEMOP_DEF(iemOp_fndisi)
+{
+    IEMOP_MNEMONIC(fndisi, "fndisi (8087/ign)");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb 0xe2. */
+FNIEMOP_DEF(iemOp_fnclex)
+{
+    IEMOP_MNEMONIC(fnclex, "fnclex");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_CLEAR_FSW_EX();
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb 0xe3. */
+FNIEMOP_DEF(iemOp_fninit)
+{
+    IEMOP_MNEMONIC(fninit, "fninit");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_finit, false /*fCheckXcpts*/);
+}
+
+
+/** Opcode 0xdb 0xe4. */
+FNIEMOP_DEF(iemOp_fnsetpm)
+{
+    IEMOP_MNEMONIC(fnsetpm, "fnsetpm (80287/ign)");   /* set protected mode on fpu. */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdb 0xe5. */
+FNIEMOP_DEF(iemOp_frstpm)
+{
+    IEMOP_MNEMONIC(frstpm, "frstpm (80287XL/ign)"); /* reset pm, back to real mode. */
+#if 0 /* #UDs on newer CPUs */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+#else
+    return IEMOP_RAISE_INVALID_OPCODE();
+#endif
+}
+
+
+/** Opcode 0xdb 11/5. */
+FNIEMOP_DEF_1(iemOp_fucomi_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fucomi_st0_stN, "fucomi st0,stN");
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_fcomi_fucomi, bRm & X86_MODRM_RM_MASK, iemAImpl_fucomi_r80_by_r80, false /*fPop*/);
+}
+
+
+/** Opcode 0xdb 11/6. */
+FNIEMOP_DEF_1(iemOp_fcomi_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcomi_st0_stN, "fcomi st0,stN");
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_fcomi_fucomi, bRm & X86_MODRM_RM_MASK, iemAImpl_fcomi_r80_by_r80, false /*fPop*/);
+}
+
+
+/**
+ * @opcode      0xdb
+ */
+FNIEMOP_DEF(iemOp_EscF3)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xdb & 0x7);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fcmovnb_stN,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fcmovne_stN,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcmovnbe_stN, bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fcmovnnu_stN, bRm);
+            case 4:
+                switch (bRm)
+                {
+                    case 0xe0:  return FNIEMOP_CALL(iemOp_fneni);
+                    case 0xe1:  return FNIEMOP_CALL(iemOp_fndisi);
+                    case 0xe2:  return FNIEMOP_CALL(iemOp_fnclex);
+                    case 0xe3:  return FNIEMOP_CALL(iemOp_fninit);
+                    case 0xe4:  return FNIEMOP_CALL(iemOp_fnsetpm);
+                    case 0xe5:  return FNIEMOP_CALL(iemOp_frstpm);
+                    case 0xe6:  return IEMOP_RAISE_INVALID_OPCODE();
+                    case 0xe7:  return IEMOP_RAISE_INVALID_OPCODE();
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case 5: return FNIEMOP_CALL_1(iemOp_fucomi_stN, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fcomi_stN,  bRm);
+            case 7: return IEMOP_RAISE_INVALID_OPCODE();
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fild_m32i,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fisttp_m32i,bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fist_m32i,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fistp_m32i, bRm);
+            case 4: return IEMOP_RAISE_INVALID_OPCODE();
+            case 5: return FNIEMOP_CALL_1(iemOp_fld_m80r,   bRm);
+            case 6: return IEMOP_RAISE_INVALID_OPCODE();
+            case 7: return FNIEMOP_CALL_1(iemOp_fstp_m80r,  bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * Common worker for FPU instructions working on STn and ST0, and storing the
+ * result in STn unless IE, DE or ZE was raised.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_stN_st0, uint8_t, bRm, PFNIEMAIMPLFPUR80, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value2,                 2);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, bRm & X86_MODRM_RM_MASK, pr80Value2, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pr80Value2);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, bRm & X86_MODRM_RM_MASK);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(bRm & X86_MODRM_RM_MASK);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdc 11/0. */
+FNIEMOP_DEF_1(iemOp_fadd_stN_st0,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fadd_stN_st0, "fadd stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fadd_r80_by_r80);
+}
+
+
+/** Opcode 0xdc 11/1. */
+FNIEMOP_DEF_1(iemOp_fmul_stN_st0,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fmul_stN_st0, "fmul stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fmul_r80_by_r80);
+}
+
+
+/** Opcode 0xdc 11/4. */
+FNIEMOP_DEF_1(iemOp_fsubr_stN_st0,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsubr_stN_st0, "fsubr stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fsubr_r80_by_r80);
+}
+
+
+/** Opcode 0xdc 11/5. */
+FNIEMOP_DEF_1(iemOp_fsub_stN_st0,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsub_stN_st0, "fsub stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fsub_r80_by_r80);
+}
+
+
+/** Opcode 0xdc 11/6. */
+FNIEMOP_DEF_1(iemOp_fdivr_stN_st0,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdivr_stN_st0, "fdivr stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fdivr_r80_by_r80);
+}
+
+
+/** Opcode 0xdc 11/7. */
+FNIEMOP_DEF_1(iemOp_fdiv_stN_st0,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdiv_stN_st0, "fdiv stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fdiv_r80_by_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and a 64-bit floating point
+ * memory operand, and storing the result in ST0.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_ST0_m64r, uint8_t, bRm, PFNIEMAIMPLFPUR64, pfnImpl)
+{
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_LOCAL(RTFLOAT64U,            r64Factor2);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Factor1,                1);
+    IEM_MC_ARG_LOCAL_REF(PRTFLOAT64U,   pr64Factor2,    r64Factor2, 2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_FETCH_MEM_R64(r64Factor2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Factor1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnImpl, pFpuRes, pr80Factor1, pr64Factor2);
+        IEM_MC_STORE_FPU_RESULT_MEM_OP(FpuRes, 0, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(0, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdc !11/0. */
+FNIEMOP_DEF_1(iemOp_fadd_m64r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fadd_m64r, "fadd m64r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fadd_r80_by_r64);
+}
+
+
+/** Opcode 0xdc !11/1. */
+FNIEMOP_DEF_1(iemOp_fmul_m64r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fmul_m64r, "fmul m64r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fmul_r80_by_r64);
+}
+
+
+/** Opcode 0xdc !11/2. */
+FNIEMOP_DEF_1(iemOp_fcom_m64r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcom_st0_m64r, "fcom st0,m64r");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_LOCAL(RTFLOAT64U,            r64Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT64U,  pr64Val2,       r64Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R64(r64Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fcom_r80_by_r64, pu16Fsw, pr80Value1, pr64Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdc !11/3. */
+FNIEMOP_DEF_1(iemOp_fcomp_m64r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcomp_st0_m64r, "fcomp st0,m64r");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_LOCAL(RTFLOAT64U,            r64Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT64U,  pr64Val2,       r64Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R64(r64Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fcom_r80_by_r64, pu16Fsw, pr80Value1, pr64Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdc !11/4. */
+FNIEMOP_DEF_1(iemOp_fsub_m64r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsub_m64r, "fsub m64r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fsub_r80_by_r64);
+}
+
+
+/** Opcode 0xdc !11/5. */
+FNIEMOP_DEF_1(iemOp_fsubr_m64r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsubr_m64r, "fsubr m64r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fsubr_r80_by_r64);
+}
+
+
+/** Opcode 0xdc !11/6. */
+FNIEMOP_DEF_1(iemOp_fdiv_m64r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdiv_m64r, "fdiv m64r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fdiv_r80_by_r64);
+}
+
+
+/** Opcode 0xdc !11/7. */
+FNIEMOP_DEF_1(iemOp_fdivr_m64r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdivr_m64r, "fdivr m64r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fdivr_r80_by_r64);
+}
+
+
+/**
+ * @opcode      0xdc
+ */
+FNIEMOP_DEF(iemOp_EscF4)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xdc & 0x7);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fadd_stN_st0,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fmul_stN_st0,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcom_stN,      bRm); /* Marked reserved, intel behavior is that of FCOM ST(i). */
+            case 3: return FNIEMOP_CALL_1(iemOp_fcomp_stN,     bRm); /* Marked reserved, intel behavior is that of FCOMP ST(i). */
+            case 4: return FNIEMOP_CALL_1(iemOp_fsubr_stN_st0, bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fsub_stN_st0,  bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fdivr_stN_st0, bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fdiv_stN_st0,  bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fadd_m64r,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fmul_m64r,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcom_m64r,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fcomp_m64r, bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fsub_m64r,  bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fsubr_m64r, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fdiv_m64r,  bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fdivr_m64r, bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xdd !11/0.
+ * @sa iemOp_fld_m32r */
+FNIEMOP_DEF_1(iemOp_fld_m64r,    uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fld_m64r, "fld m64r");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_LOCAL(RTFLOAT64U,            r64Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT64U,  pr64Val,    r64Val, 1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_FETCH_MEM_R64(r64Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fld_r64_to_r80, pFpuRes, pr64Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdd !11/0. */
+FNIEMOP_DEF_1(iemOp_fisttp_m64i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisttp_m64i, "fisttp m64i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int64_t *,               pi64Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi64Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fistt_r80_to_i64, pu16Fsw, pi64Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi64Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I64_CONST_BY_REF(pi64Dst, INT64_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi64Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdd !11/0. */
+FNIEMOP_DEF_1(iemOp_fst_m64r,    uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fst_m64r, "fst m64r");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PRTFLOAT64U,             pr64Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pr64Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r64, pu16Fsw, pr64Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr64Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(pr64Dst);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pr64Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+
+
+/** Opcode 0xdd !11/0. */
+FNIEMOP_DEF_1(iemOp_fstp_m64r,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fstp_m64r, "fstp m64r");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PRTFLOAT64U,             pr64Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pr64Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r64, pu16Fsw, pr64Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr64Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(pr64Dst);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pr64Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdd !11/0. */
+FNIEMOP_DEF_1(iemOp_frstor,      uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(frstor, "frstor m94/108byte");
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG_CONST(IEMMODE,           enmEffOpSize, /*=*/ pVCpu->iem.s.enmEffOpSize,  0);
+    IEM_MC_ARG(uint8_t,                 iEffSeg,                                    1);
+    IEM_MC_ARG(RTGCPTR,                 GCPtrEffSrc,                                2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_frstor, enmEffOpSize, iEffSeg, GCPtrEffSrc);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdd !11/0. */
+FNIEMOP_DEF_1(iemOp_fnsave,      uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fnsave, "fnsave m94/108byte");
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG_CONST(IEMMODE,           enmEffOpSize, /*=*/ pVCpu->iem.s.enmEffOpSize,  0);
+    IEM_MC_ARG(uint8_t,                 iEffSeg,                                    1);
+    IEM_MC_ARG(RTGCPTR,                 GCPtrEffDst,                                2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_fnsave, enmEffOpSize, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+
+}
+
+/** Opcode 0xdd !11/0. */
+FNIEMOP_DEF_1(iemOp_fnstsw,      uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fnstsw_m16, "fnstsw m16");
+
+    IEM_MC_BEGIN(0, 2);
+    IEM_MC_LOCAL(uint16_t, u16Tmp);
+    IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_FETCH_FSW(u16Tmp);
+    IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Tmp);
+    IEM_MC_ADVANCE_RIP();
+
+/** @todo Debug / drop a hint to the verifier that things may differ
+ * from REM. Seen 0x4020 (iem) vs 0x4000 (rem) at 0008:801c6b88 booting
+ * NT4SP1. (X86_FSW_PE) */
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdd 11/0. */
+FNIEMOP_DEF_1(iemOp_ffree_stN,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(ffree_stN, "ffree stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    /* Note! C0, C1, C2 and C3 are documented as undefined, we leave the
+             unmodified. */
+
+    IEM_MC_BEGIN(0, 0);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FPU_STACK_FREE(bRm & X86_MODRM_RM_MASK);
+    IEM_MC_UPDATE_FPU_OPCODE_IP();
+
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdd 11/1. */
+FNIEMOP_DEF_1(iemOp_fst_stN,     uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fst_st0_stN, "fst st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 2);
+    IEM_MC_LOCAL(PCRTFLOAT80U,          pr80Value);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_SET_FPU_RESULT(FpuRes, 0 /*FSW*/, pr80Value);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, bRm & X86_MODRM_RM_MASK);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(bRm & X86_MODRM_RM_MASK);
+    IEM_MC_ENDIF();
+
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdd 11/3. */
+FNIEMOP_DEF_1(iemOp_fucom_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fucom_st0_stN, "fucom st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpuNoStore_st0_stN, bRm, iemAImpl_fucom_r80_by_r80);
+}
+
+
+/** Opcode 0xdd 11/4. */
+FNIEMOP_DEF_1(iemOp_fucomp_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fucomp_st0_stN, "fucomp st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpuNoStore_st0_stN_pop, bRm, iemAImpl_fucom_r80_by_r80);
+}
+
+
+/**
+ * @opcode      0xdd
+ */
+FNIEMOP_DEF(iemOp_EscF5)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xdd & 0x7);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_ffree_stN,   bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fxch_stN,    bRm); /* Reserved, intel behavior is that of XCHG ST(i). */
+            case 2: return FNIEMOP_CALL_1(iemOp_fst_stN,     bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fstp_stN,    bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fucom_stN_st0,bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fucomp_stN,  bRm);
+            case 6: return IEMOP_RAISE_INVALID_OPCODE();
+            case 7: return IEMOP_RAISE_INVALID_OPCODE();
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fld_m64r,    bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fisttp_m64i, bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fst_m64r,    bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fstp_m64r,   bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_frstor,      bRm);
+            case 5: return IEMOP_RAISE_INVALID_OPCODE();
+            case 6: return FNIEMOP_CALL_1(iemOp_fnsave,      bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fnstsw,      bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xde 11/0. */
+FNIEMOP_DEF_1(iemOp_faddp_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(faddp_stN_st0, "faddp stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fadd_r80_by_r80);
+}
+
+
+/** Opcode 0xde 11/0. */
+FNIEMOP_DEF_1(iemOp_fmulp_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fmulp_stN_st0, "fmulp stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fmul_r80_by_r80);
+}
+
+
+/** Opcode 0xde 0xd9. */
+FNIEMOP_DEF(iemOp_fcompp)
+{
+    IEMOP_MNEMONIC(fcompp_st0_stN, "fcompp st0,stN");
+    return FNIEMOP_CALL_1(iemOpHlpFpuNoStore_st0_stN_pop_pop, iemAImpl_fcom_r80_by_r80);
+}
+
+
+/** Opcode 0xde 11/4. */
+FNIEMOP_DEF_1(iemOp_fsubrp_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsubrp_stN_st0, "fsubrp stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fsubr_r80_by_r80);
+}
+
+
+/** Opcode 0xde 11/5. */
+FNIEMOP_DEF_1(iemOp_fsubp_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsubp_stN_st0, "fsubp stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fsub_r80_by_r80);
+}
+
+
+/** Opcode 0xde 11/6. */
+FNIEMOP_DEF_1(iemOp_fdivrp_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdivrp_stN_st0, "fdivrp stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fdivr_r80_by_r80);
+}
+
+
+/** Opcode 0xde 11/7. */
+FNIEMOP_DEF_1(iemOp_fdivp_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdivp_stN_st0, "fdivp stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fdiv_r80_by_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and an m16i, and storing
+ * the result in ST0.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_st0_m16i, uint8_t, bRm, PFNIEMAIMPLFPUI16, pfnAImpl)
+{
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,              FpuRes);
+    IEM_MC_LOCAL(int16_t,                   i16Val2);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT,     pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(int16_t const *,   pi16Val2,       i16Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I16(i16Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pi16Val2);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, 0);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xde !11/0. */
+FNIEMOP_DEF_1(iemOp_fiadd_m16i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fiadd_m16i, "fiadd m16i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fiadd_r80_by_i16);
+}
+
+
+/** Opcode 0xde !11/1. */
+FNIEMOP_DEF_1(iemOp_fimul_m16i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fimul_m16i, "fimul m16i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fimul_r80_by_i16);
+}
+
+
+/** Opcode 0xde !11/2. */
+FNIEMOP_DEF_1(iemOp_ficom_m16i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(ficom_st0_m16i, "ficom st0,m16i");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,                  u16Fsw);
+    IEM_MC_LOCAL(int16_t,                   i16Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,        pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(int16_t const *,   pi16Val2,       i16Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I16(i16Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_ficom_r80_by_i16, pu16Fsw, pr80Value1, pi16Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xde !11/3. */
+FNIEMOP_DEF_1(iemOp_ficomp_m16i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(ficomp_st0_m16i, "ficomp st0,m16i");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,                  u16Fsw);
+    IEM_MC_LOCAL(int16_t,                   i16Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,        pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(int16_t const *,   pi16Val2,       i16Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I16(i16Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_ficom_r80_by_i16, pu16Fsw, pr80Value1, pi16Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xde !11/4. */
+FNIEMOP_DEF_1(iemOp_fisub_m16i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisub_m16i, "fisub m16i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fisub_r80_by_i16);
+}
+
+
+/** Opcode 0xde !11/5. */
+FNIEMOP_DEF_1(iemOp_fisubr_m16i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisubr_m16i, "fisubr m16i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fisubr_r80_by_i16);
+}
+
+
+/** Opcode 0xde !11/6. */
+FNIEMOP_DEF_1(iemOp_fidiv_m16i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fidiv_m16i, "fidiv m16i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fidiv_r80_by_i16);
+}
+
+
+/** Opcode 0xde !11/7. */
+FNIEMOP_DEF_1(iemOp_fidivr_m16i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fidivr_m16i, "fidivr m16i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fidivr_r80_by_i16);
+}
+
+
+/**
+ * @opcode      0xde
+ */
+FNIEMOP_DEF(iemOp_EscF6)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xde & 0x7);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_faddp_stN_st0, bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fmulp_stN_st0, bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcomp_stN, bRm);
+            case 3: if (bRm == 0xd9)
+                        return FNIEMOP_CALL(iemOp_fcompp);
+                    return IEMOP_RAISE_INVALID_OPCODE();
+            case 4: return FNIEMOP_CALL_1(iemOp_fsubrp_stN_st0, bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fsubp_stN_st0, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fdivrp_stN_st0, bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fdivp_stN_st0, bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fiadd_m16i,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fimul_m16i,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_ficom_m16i,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_ficomp_m16i, bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fisub_m16i,  bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fisubr_m16i, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fidiv_m16i,  bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fidivr_m16i, bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xdf 11/0.
+ * Undocument instruction, assumed to work like ffree + fincstp.  */
+FNIEMOP_DEF_1(iemOp_ffreep_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(ffreep_stN, "ffreep stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FPU_STACK_FREE(bRm & X86_MODRM_RM_MASK);
+    IEM_MC_FPU_STACK_INC_TOP();
+    IEM_MC_UPDATE_FPU_OPCODE_IP();
+
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdf 0xe0. */
+FNIEMOP_DEF(iemOp_fnstsw_ax)
+{
+    IEMOP_MNEMONIC(fnstsw_ax, "fnstsw ax");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(uint16_t, u16Tmp);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_FETCH_FSW(u16Tmp);
+    IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Tmp);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdf 11/5. */
+FNIEMOP_DEF_1(iemOp_fucomip_st0_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fucomip_st0_stN, "fucomip st0,stN");
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_fcomi_fucomi, bRm & X86_MODRM_RM_MASK, iemAImpl_fcomi_r80_by_r80, true /*fPop*/);
+}
+
+
+/** Opcode 0xdf 11/6. */
+FNIEMOP_DEF_1(iemOp_fcomip_st0_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcomip_st0_stN, "fcomip st0,stN");
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_fcomi_fucomi, bRm & X86_MODRM_RM_MASK, iemAImpl_fcomi_r80_by_r80, true /*fPop*/);
+}
+
+
+/** Opcode 0xdf !11/0. */
+FNIEMOP_DEF_1(iemOp_fild_m16i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fild_m16i, "fild m16i");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,              FpuRes);
+    IEM_MC_LOCAL(int16_t,                   i16Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT,     pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG_LOCAL_REF(int16_t const *,   pi16Val,    i16Val, 1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I16(i16Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fild_i16_to_r80, pFpuRes, pi16Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdf !11/1. */
+FNIEMOP_DEF_1(iemOp_fisttp_m16i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisttp_m16i, "fisttp m16i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int16_t *,               pi16Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi16Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fistt_r80_to_i16, pu16Fsw, pi16Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi16Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I16_CONST_BY_REF(pi16Dst, INT16_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi16Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdf !11/2. */
+FNIEMOP_DEF_1(iemOp_fist_m16i,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fist_m16i, "fist m16i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int16_t *,               pi16Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi16Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i16, pu16Fsw, pi16Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi16Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I16_CONST_BY_REF(pi16Dst, INT16_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi16Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdf !11/3. */
+FNIEMOP_DEF_1(iemOp_fistp_m16i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fistp_m16i, "fistp m16i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int16_t *,               pi16Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi16Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i16, pu16Fsw, pi16Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi16Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I16_CONST_BY_REF(pi16Dst, INT16_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi16Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdf !11/4. */
+FNIEMOP_STUB_1(iemOp_fbld_m80d,   uint8_t, bRm);
+
+
+/** Opcode 0xdf !11/5. */
+FNIEMOP_DEF_1(iemOp_fild_m64i,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fild_m64i, "fild m64i");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,              FpuRes);
+    IEM_MC_LOCAL(int64_t,                   i64Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT,     pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG_LOCAL_REF(int64_t const *,   pi64Val,    i64Val, 1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I64(i64Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fild_i64_to_r80, pFpuRes, pi64Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdf !11/6. */
+FNIEMOP_STUB_1(iemOp_fbstp_m80d,  uint8_t, bRm);
+
+
+/** Opcode 0xdf !11/7. */
+FNIEMOP_DEF_1(iemOp_fistp_m64i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fistp_m64i, "fistp m64i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int64_t *,               pi64Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi64Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i64, pu16Fsw, pi64Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi64Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I64_CONST_BY_REF(pi64Dst, INT64_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi64Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xdf
+ */
+FNIEMOP_DEF(iemOp_EscF7)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_ffreep_stN, bRm); /* ffree + pop afterwards, since forever according to AMD. */
+            case 1: return FNIEMOP_CALL_1(iemOp_fxch_stN,   bRm); /* Reserved, behaves like FXCH ST(i) on intel. */
+            case 2: return FNIEMOP_CALL_1(iemOp_fstp_stN,   bRm); /* Reserved, behaves like FSTP ST(i) on intel. */
+            case 3: return FNIEMOP_CALL_1(iemOp_fstp_stN,   bRm); /* Reserved, behaves like FSTP ST(i) on intel. */
+            case 4: if (bRm == 0xe0)
+                        return FNIEMOP_CALL(iemOp_fnstsw_ax);
+                    return IEMOP_RAISE_INVALID_OPCODE();
+            case 5: return FNIEMOP_CALL_1(iemOp_fucomip_st0_stN, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fcomip_st0_stN,  bRm);
+            case 7: return IEMOP_RAISE_INVALID_OPCODE();
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fild_m16i,   bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fisttp_m16i, bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fist_m16i,   bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fistp_m16i,  bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fbld_m80d,   bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fild_m64i,   bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fbstp_m80d,  bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fistp_m64i,  bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0xe0
+ */
+FNIEMOP_DEF(iemOp_loopne_Jb)
+{
+    IEMOP_MNEMONIC(loopne_Jb, "loopne Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U16(X86_GREG_xCX, 1);
+            IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(X86_EFL_ZF) {
+                IEM_MC_REL_JMP_S8(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U32(X86_GREG_xCX, 1);
+            IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(X86_EFL_ZF) {
+                IEM_MC_REL_JMP_S8(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U64(X86_GREG_xCX, 1);
+            IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(X86_EFL_ZF) {
+                IEM_MC_REL_JMP_S8(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xe1
+ */
+FNIEMOP_DEF(iemOp_loope_Jb)
+{
+    IEMOP_MNEMONIC(loope_Jb, "loope Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U16(X86_GREG_xCX, 1);
+            IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(X86_EFL_ZF) {
+                IEM_MC_REL_JMP_S8(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U32(X86_GREG_xCX, 1);
+            IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(X86_EFL_ZF) {
+                IEM_MC_REL_JMP_S8(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U64(X86_GREG_xCX, 1);
+            IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(X86_EFL_ZF) {
+                IEM_MC_REL_JMP_S8(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xe2
+ */
+FNIEMOP_DEF(iemOp_loop_Jb)
+{
+    IEMOP_MNEMONIC(loop_Jb, "loop Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    /** @todo Check out the #GP case if EIP < CS.Base or EIP > CS.Limit when
+     * using the 32-bit operand size override.  How can that be restarted?  See
+     * weird pseudo code in intel manual. */
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0,0);
+            if (-(int8_t)IEM_GET_INSTR_LEN(pVCpu) != i8Imm)
+            {
+                IEM_MC_SUB_GREG_U16(X86_GREG_xCX, 1);
+                IEM_MC_IF_CX_IS_NZ() {
+                    IEM_MC_REL_JMP_S8(i8Imm);
+                } IEM_MC_ELSE() {
+                    IEM_MC_ADVANCE_RIP();
+                } IEM_MC_ENDIF();
+            }
+            else
+            {
+                IEM_MC_STORE_GREG_U16_CONST(X86_GREG_xCX, 0);
+                IEM_MC_ADVANCE_RIP();
+            }
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0,0);
+            if (-(int8_t)IEM_GET_INSTR_LEN(pVCpu) != i8Imm)
+            {
+                IEM_MC_SUB_GREG_U32(X86_GREG_xCX, 1);
+                IEM_MC_IF_ECX_IS_NZ() {
+                    IEM_MC_REL_JMP_S8(i8Imm);
+                } IEM_MC_ELSE() {
+                    IEM_MC_ADVANCE_RIP();
+                } IEM_MC_ENDIF();
+            }
+            else
+            {
+                IEM_MC_STORE_GREG_U32_CONST(X86_GREG_xCX, 0);
+                IEM_MC_ADVANCE_RIP();
+            }
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0,0);
+            if (-(int8_t)IEM_GET_INSTR_LEN(pVCpu) != i8Imm)
+            {
+                IEM_MC_SUB_GREG_U64(X86_GREG_xCX, 1);
+                IEM_MC_IF_RCX_IS_NZ() {
+                    IEM_MC_REL_JMP_S8(i8Imm);
+                } IEM_MC_ELSE() {
+                    IEM_MC_ADVANCE_RIP();
+                } IEM_MC_ENDIF();
+            }
+            else
+            {
+                IEM_MC_STORE_GREG_U64_CONST(X86_GREG_xCX, 0);
+                IEM_MC_ADVANCE_RIP();
+            }
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xe3
+ */
+FNIEMOP_DEF(iemOp_jecxz_Jb)
+{
+    IEMOP_MNEMONIC(jecxz_Jb, "jecxz Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_IF_CX_IS_NZ() {
+                IEM_MC_ADVANCE_RIP();
+            } IEM_MC_ELSE() {
+                IEM_MC_REL_JMP_S8(i8Imm);
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_IF_ECX_IS_NZ() {
+                IEM_MC_ADVANCE_RIP();
+            } IEM_MC_ELSE() {
+                IEM_MC_REL_JMP_S8(i8Imm);
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_IF_RCX_IS_NZ() {
+                IEM_MC_ADVANCE_RIP();
+            } IEM_MC_ELSE() {
+                IEM_MC_REL_JMP_S8(i8Imm);
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/** Opcode 0xe4 */
+FNIEMOP_DEF(iemOp_in_AL_Ib)
+{
+    IEMOP_MNEMONIC(in_AL_Ib, "in AL,Ib");
+    uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_in, u8Imm, true /* fImm */, 1);
+}
+
+
+/** Opcode 0xe5 */
+FNIEMOP_DEF(iemOp_in_eAX_Ib)
+{
+    IEMOP_MNEMONIC(in_eAX_Ib, "in eAX,Ib");
+    uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_in, u8Imm, true /* fImm */, pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT ? 2 : 4);
+}
+
+
+/** Opcode 0xe6 */
+FNIEMOP_DEF(iemOp_out_Ib_AL)
+{
+    IEMOP_MNEMONIC(out_Ib_AL, "out Ib,AL");
+    uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_out, u8Imm, true /* fImm */, 1);
+}
+
+
+/** Opcode 0xe7 */
+FNIEMOP_DEF(iemOp_out_Ib_eAX)
+{
+    IEMOP_MNEMONIC(out_Ib_eAX, "out Ib,eAX");
+    uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_out, u8Imm, true /* fImm */, pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT ? 2 : 4);
+}
+
+
+/**
+ * @opcode      0xe8
+ */
+FNIEMOP_DEF(iemOp_call_Jv)
+{
+    IEMOP_MNEMONIC(call_Jv, "call Jv");
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+            return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_call_rel_16, (int16_t)u16Imm);
+        }
+
+        case IEMMODE_32BIT:
+        {
+            uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+            return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_call_rel_32, (int32_t)u32Imm);
+        }
+
+        case IEMMODE_64BIT:
+        {
+            uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+            return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_call_rel_64, u64Imm);
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xe9
+ */
+FNIEMOP_DEF(iemOp_jmp_Jv)
+{
+    IEMOP_MNEMONIC(jmp_Jv, "jmp Jv");
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+            IEM_MC_BEGIN(0, 0);
+            IEM_MC_REL_JMP_S16(i16Imm);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_64BIT:
+        case IEMMODE_32BIT:
+        {
+            int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+            IEM_MC_BEGIN(0, 0);
+            IEM_MC_REL_JMP_S32(i32Imm);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xea
+ */
+FNIEMOP_DEF(iemOp_jmp_Ap)
+{
+    IEMOP_MNEMONIC(jmp_Ap, "jmp Ap");
+    IEMOP_HLP_NO_64BIT();
+
+    /* Decode the far pointer address and pass it on to the far call C implementation. */
+    uint32_t offSeg;
+    if (pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT)
+        IEM_OPCODE_GET_NEXT_U32(&offSeg);
+    else
+        IEM_OPCODE_GET_NEXT_U16_ZX_U32(&offSeg);
+    uint16_t uSel;  IEM_OPCODE_GET_NEXT_U16(&uSel);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_FarJmp, uSel, offSeg, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0xeb
+ */
+FNIEMOP_DEF(iemOp_jmp_Jb)
+{
+    IEMOP_MNEMONIC(jmp_Jb, "jmp Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_REL_JMP_S8(i8Imm);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xec */
+FNIEMOP_DEF(iemOp_in_AL_DX)
+{
+    IEMOP_MNEMONIC(in_AL_DX, "in  AL,DX");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_in_eAX_DX, 1);
+}
+
+
+/** Opcode 0xed */
+FNIEMOP_DEF(iemOp_in_eAX_DX)
+{
+    IEMOP_MNEMONIC(in_eAX_DX, "in  eAX,DX");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_in_eAX_DX, pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT ? 2 : 4);
+}
+
+
+/** Opcode 0xee */
+FNIEMOP_DEF(iemOp_out_DX_AL)
+{
+    IEMOP_MNEMONIC(out_DX_AL, "out DX,AL");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_out_DX_eAX, 1);
+}
+
+
+/** Opcode 0xef */
+FNIEMOP_DEF(iemOp_out_DX_eAX)
+{
+    IEMOP_MNEMONIC(out_DX_eAX, "out DX,eAX");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_out_DX_eAX, pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT ? 2 : 4);
+}
+
+
+/**
+ * @opcode      0xf0
+ */
+FNIEMOP_DEF(iemOp_lock)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("lock");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_LOCK;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0xf1
+ */
+FNIEMOP_DEF(iemOp_int1)
+{
+    IEMOP_MNEMONIC(int1, "int1"); /* icebp */
+    IEMOP_HLP_MIN_386(); /** @todo does not generate #UD on 286, or so they say... */
+    /** @todo testcase! */
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_int, X86_XCPT_DB, IEMINT_INT1);
+}
+
+
+/**
+ * @opcode      0xf2
+ */
+FNIEMOP_DEF(iemOp_repne)
+{
+    /* This overrides any previous REPE prefix. */
+    pVCpu->iem.s.fPrefixes &= ~IEM_OP_PRF_REPZ;
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("repne");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REPNZ;
+
+    /* For the 4 entry opcode tables, REPNZ overrides any previous
+       REPZ and operand size prefixes. */
+    pVCpu->iem.s.idxPrefix = 3;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0xf3
+ */
+FNIEMOP_DEF(iemOp_repe)
+{
+    /* This overrides any previous REPNE prefix. */
+    pVCpu->iem.s.fPrefixes &= ~IEM_OP_PRF_REPNZ;
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("repe");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REPZ;
+
+    /* For the 4 entry opcode tables, REPNZ overrides any previous
+       REPNZ and operand size prefixes. */
+    pVCpu->iem.s.idxPrefix = 2;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0xf4
+ */
+FNIEMOP_DEF(iemOp_hlt)
+{
+    IEMOP_MNEMONIC(hlt, "hlt");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_hlt);
+}
+
+
+/**
+ * @opcode      0xf5
+ */
+FNIEMOP_DEF(iemOp_cmc)
+{
+    IEMOP_MNEMONIC(cmc, "cmc");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_FLIP_EFL_BIT(X86_EFL_CF);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common implementation of 'inc/dec/not/neg Eb'.
+ *
+ * @param   bRm             The RM byte.
+ * @param   pImpl           The instruction implementation.
+ */
+FNIEMOP_DEF_2(iemOpCommonUnaryEb, uint8_t, bRm, PCIEMOPUNARYSIZES, pImpl)
+{
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register access */
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint8_t *,   pu8Dst, 0);
+        IEM_MC_ARG(uint32_t *,  pEFlags, 1);
+        IEM_MC_REF_GREG_U8(pu8Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU8, pu8Dst, pEFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory access. */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,          0);
+        IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU8, pu8Dst, pEFlags);
+        else
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnLockedU8, pu8Dst, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common implementation of 'inc/dec/not/neg Ev'.
+ *
+ * @param   bRm             The RM byte.
+ * @param   pImpl           The instruction implementation.
+ */
+FNIEMOP_DEF_2(iemOpCommonUnaryEv, uint8_t, bRm, PCIEMOPUNARYSIZES, pImpl)
+{
+    /* Registers are handled by a common worker. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, pImpl, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+
+    /* Memory we do here. */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(2, 2);
+            IEM_MC_ARG(uint16_t *,      pu16Dst,         0);
+            IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags, 1);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+            IEM_MC_FETCH_EFLAGS(EFlags);
+            if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU16, pu16Dst, pEFlags);
+            else
+                IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnLockedU16, pu16Dst, pEFlags);
+
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+            IEM_MC_COMMIT_EFLAGS(EFlags);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(2, 2);
+            IEM_MC_ARG(uint32_t *,      pu32Dst,         0);
+            IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags, 1);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+            IEM_MC_FETCH_EFLAGS(EFlags);
+            if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU32, pu32Dst, pEFlags);
+            else
+                IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnLockedU32, pu32Dst, pEFlags);
+
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+            IEM_MC_COMMIT_EFLAGS(EFlags);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(2, 2);
+            IEM_MC_ARG(uint64_t *,      pu64Dst,         0);
+            IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags, 1);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+            IEM_MC_FETCH_EFLAGS(EFlags);
+            if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU64, pu64Dst, pEFlags);
+            else
+                IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnLockedU64, pu64Dst, pEFlags);
+
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+            IEM_MC_COMMIT_EFLAGS(EFlags);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/** Opcode 0xf6 /0. */
+FNIEMOP_DEF_1(iemOp_grp3_test_Eb, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(test_Eb_Ib, "test Eb,Ib");
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register access */
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,             0);
+        IEM_MC_ARG_CONST(uint8_t,   u8Src,/*=*/u8Imm,   1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+        IEM_MC_REF_GREG_U8(pu8Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u8, pu8Dst, u8Src, pEFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory access. */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,             0);
+        IEM_MC_ARG(uint8_t,         u8Src,              1);
+        IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,    2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        IEM_MC_ASSIGN(u8Src, u8Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_R, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_R);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xf7 /0. */
+FNIEMOP_DEF_1(iemOp_grp3_test_Ev, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(test_Ev_Iv, "test Ev,Iv");
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register access */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG_CONST(uint16_t,  u16Src,/*=*/u16Imm,     1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u16, pu16Dst, u16Src, pEFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            case IEMMODE_32BIT:
+            {
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG_CONST(uint32_t,  u32Src,/*=*/u32Imm,     1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u32, pu32Dst, u32Src, pEFlags);
+                /* No clearing the high dword here - test doesn't write back the result. */
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            case IEMMODE_64BIT:
+            {
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG_CONST(uint64_t,  u64Src,/*=*/u64Imm,     1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u64, pu64Dst, u64Src, pEFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory access. */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,            0);
+                IEM_MC_ARG(uint16_t,        u16Src,             1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,    2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 2);
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEM_MC_ASSIGN(u16Src, u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_R, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_R);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            case IEMMODE_32BIT:
+            {
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,            0);
+                IEM_MC_ARG(uint32_t,        u32Src,             1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,    2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEM_MC_ASSIGN(u32Src, u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_R, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_R);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            case IEMMODE_64BIT:
+            {
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,            0);
+                IEM_MC_ARG(uint64_t,        u64Src,             1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,    2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEM_MC_ASSIGN(u64Src, u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_R, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_R);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xf6 /4, /5, /6 and /7. */
+FNIEMOP_DEF_2(iemOpCommonGrp3MulDivEb, uint8_t, bRm, PFNIEMAIMPLMULDIVU8, pfnU8)
+{
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register access */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_ARG(uint16_t *,      pu16AX,     0);
+        IEM_MC_ARG(uint8_t,         u8Value,    1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,    2);
+        IEM_MC_LOCAL(int32_t,       rc);
+
+        IEM_MC_FETCH_GREG_U8(u8Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_GREG_U16(pu16AX, X86_GREG_xAX);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_AIMPL_3(rc, pfnU8, pu16AX, u8Value, pEFlags);
+        IEM_MC_IF_LOCAL_IS_Z(rc) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_RAISE_DIVIDE_ERROR();
+        } IEM_MC_ENDIF();
+
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory access. */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint16_t *,      pu16AX,     0);
+        IEM_MC_ARG(uint8_t,         u8Value,    1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,    2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_LOCAL(int32_t,       rc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_MEM_U8(u8Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+        IEM_MC_REF_GREG_U16(pu16AX, X86_GREG_xAX);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_AIMPL_3(rc, pfnU8, pu16AX, u8Value, pEFlags);
+        IEM_MC_IF_LOCAL_IS_Z(rc) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_RAISE_DIVIDE_ERROR();
+        } IEM_MC_ENDIF();
+
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xf7 /4, /5, /6 and /7. */
+FNIEMOP_DEF_2(iemOpCommonGrp3MulDivEv, uint8_t, bRm, PCIEMOPMULDIVSIZES, pImpl)
+{
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register access */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_BEGIN(4, 1);
+                IEM_MC_ARG(uint16_t *,      pu16AX,     0);
+                IEM_MC_ARG(uint16_t *,      pu16DX,     1);
+                IEM_MC_ARG(uint16_t,        u16Value,   2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    3);
+                IEM_MC_LOCAL(int32_t,       rc);
+
+                IEM_MC_FETCH_GREG_U16(u16Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_GREG_U16(pu16AX, X86_GREG_xAX);
+                IEM_MC_REF_GREG_U16(pu16DX, X86_GREG_xDX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU16, pu16AX, pu16DX, u16Value, pEFlags);
+                IEM_MC_IF_LOCAL_IS_Z(rc) {
+                    IEM_MC_ADVANCE_RIP();
+                } IEM_MC_ELSE() {
+                    IEM_MC_RAISE_DIVIDE_ERROR();
+                } IEM_MC_ENDIF();
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            case IEMMODE_32BIT:
+            {
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_BEGIN(4, 1);
+                IEM_MC_ARG(uint32_t *,      pu32AX,     0);
+                IEM_MC_ARG(uint32_t *,      pu32DX,     1);
+                IEM_MC_ARG(uint32_t,        u32Value,   2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    3);
+                IEM_MC_LOCAL(int32_t,       rc);
+
+                IEM_MC_FETCH_GREG_U32(u32Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_GREG_U32(pu32AX, X86_GREG_xAX);
+                IEM_MC_REF_GREG_U32(pu32DX, X86_GREG_xDX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU32, pu32AX, pu32DX, u32Value, pEFlags);
+                IEM_MC_IF_LOCAL_IS_Z(rc) {
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32AX);
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32DX);
+                    IEM_MC_ADVANCE_RIP();
+                } IEM_MC_ELSE() {
+                    IEM_MC_RAISE_DIVIDE_ERROR();
+                } IEM_MC_ENDIF();
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            case IEMMODE_64BIT:
+            {
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_BEGIN(4, 1);
+                IEM_MC_ARG(uint64_t *,      pu64AX,     0);
+                IEM_MC_ARG(uint64_t *,      pu64DX,     1);
+                IEM_MC_ARG(uint64_t,        u64Value,   2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    3);
+                IEM_MC_LOCAL(int32_t,       rc);
+
+                IEM_MC_FETCH_GREG_U64(u64Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_GREG_U64(pu64AX, X86_GREG_xAX);
+                IEM_MC_REF_GREG_U64(pu64DX, X86_GREG_xDX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU64, pu64AX, pu64DX, u64Value, pEFlags);
+                IEM_MC_IF_LOCAL_IS_Z(rc) {
+                    IEM_MC_ADVANCE_RIP();
+                } IEM_MC_ELSE() {
+                    IEM_MC_RAISE_DIVIDE_ERROR();
+                } IEM_MC_ENDIF();
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory access. */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint16_t *,      pu16AX,     0);
+                IEM_MC_ARG(uint16_t *,      pu16DX,     1);
+                IEM_MC_ARG(uint16_t,        u16Value,   2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    3);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_LOCAL(int32_t,       rc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U16(pu16AX, X86_GREG_xAX);
+                IEM_MC_REF_GREG_U16(pu16DX, X86_GREG_xDX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU16, pu16AX, pu16DX, u16Value, pEFlags);
+                IEM_MC_IF_LOCAL_IS_Z(rc) {
+                    IEM_MC_ADVANCE_RIP();
+                } IEM_MC_ELSE() {
+                    IEM_MC_RAISE_DIVIDE_ERROR();
+                } IEM_MC_ENDIF();
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            case IEMMODE_32BIT:
+            {
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint32_t *,      pu32AX,     0);
+                IEM_MC_ARG(uint32_t *,      pu32DX,     1);
+                IEM_MC_ARG(uint32_t,        u32Value,   2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    3);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_LOCAL(int32_t,       rc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U32(pu32AX, X86_GREG_xAX);
+                IEM_MC_REF_GREG_U32(pu32DX, X86_GREG_xDX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU32, pu32AX, pu32DX, u32Value, pEFlags);
+                IEM_MC_IF_LOCAL_IS_Z(rc) {
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32AX);
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32DX);
+                    IEM_MC_ADVANCE_RIP();
+                } IEM_MC_ELSE() {
+                    IEM_MC_RAISE_DIVIDE_ERROR();
+                } IEM_MC_ENDIF();
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            case IEMMODE_64BIT:
+            {
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint64_t *,      pu64AX,     0);
+                IEM_MC_ARG(uint64_t *,      pu64DX,     1);
+                IEM_MC_ARG(uint64_t,        u64Value,   2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    3);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_LOCAL(int32_t,       rc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U64(pu64AX, X86_GREG_xAX);
+                IEM_MC_REF_GREG_U64(pu64DX, X86_GREG_xDX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU64, pu64AX, pu64DX, u64Value, pEFlags);
+                IEM_MC_IF_LOCAL_IS_Z(rc) {
+                    IEM_MC_ADVANCE_RIP();
+                } IEM_MC_ELSE() {
+                    IEM_MC_RAISE_DIVIDE_ERROR();
+                } IEM_MC_ENDIF();
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+/**
+ * @opcode      0xf6
+ */
+FNIEMOP_DEF(iemOp_Grp3_Eb)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0:
+            return FNIEMOP_CALL_1(iemOp_grp3_test_Eb, bRm);
+        case 1:
+/** @todo testcase: Present on <=386, most 486 (not early), Pentiums, and current CPUs too. CPUUNDOC.EXE */
+            return IEMOP_RAISE_INVALID_OPCODE();
+        case 2:
+            IEMOP_MNEMONIC(not_Eb, "not Eb");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEb, bRm, &g_iemAImpl_not);
+        case 3:
+            IEMOP_MNEMONIC(neg_Eb, "neg Eb");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEb, bRm, &g_iemAImpl_neg);
+        case 4:
+            IEMOP_MNEMONIC(mul_Eb, "mul Eb");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEb, bRm, iemAImpl_mul_u8);
+        case 5:
+            IEMOP_MNEMONIC(imul_Eb, "imul Eb");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEb, bRm, iemAImpl_imul_u8);
+        case 6:
+            IEMOP_MNEMONIC(div_Eb, "div Eb");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_OF | X86_EFL_CF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEb, bRm, iemAImpl_div_u8);
+        case 7:
+            IEMOP_MNEMONIC(idiv_Eb, "idiv Eb");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_OF | X86_EFL_CF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEb, bRm, iemAImpl_idiv_u8);
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xf7
+ */
+FNIEMOP_DEF(iemOp_Grp3_Ev)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0:
+            return FNIEMOP_CALL_1(iemOp_grp3_test_Ev, bRm);
+        case 1:
+/** @todo testcase: Present on <=386, most 486 (not early), Pentiums, and current CPUs too. CPUUNDOC.EXE */
+            return IEMOP_RAISE_INVALID_OPCODE();
+        case 2:
+            IEMOP_MNEMONIC(not_Ev, "not Ev");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEv, bRm, &g_iemAImpl_not);
+        case 3:
+            IEMOP_MNEMONIC(neg_Ev, "neg Ev");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEv, bRm, &g_iemAImpl_neg);
+        case 4:
+            IEMOP_MNEMONIC(mul_Ev, "mul Ev");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEv, bRm, &g_iemAImpl_mul);
+        case 5:
+            IEMOP_MNEMONIC(imul_Ev, "imul Ev");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEv, bRm, &g_iemAImpl_imul);
+        case 6:
+            IEMOP_MNEMONIC(div_Ev, "div Ev");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_OF | X86_EFL_CF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEv, bRm, &g_iemAImpl_div);
+        case 7:
+            IEMOP_MNEMONIC(idiv_Ev, "idiv Ev");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_OF | X86_EFL_CF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEv, bRm, &g_iemAImpl_idiv);
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xf8
+ */
+FNIEMOP_DEF(iemOp_clc)
+{
+    IEMOP_MNEMONIC(clc, "clc");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_CLEAR_EFL_BIT(X86_EFL_CF);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xf9
+ */
+FNIEMOP_DEF(iemOp_stc)
+{
+    IEMOP_MNEMONIC(stc, "stc");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_SET_EFL_BIT(X86_EFL_CF);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xfa
+ */
+FNIEMOP_DEF(iemOp_cli)
+{
+    IEMOP_MNEMONIC(cli, "cli");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_cli);
+}
+
+
+FNIEMOP_DEF(iemOp_sti)
+{
+    IEMOP_MNEMONIC(sti, "sti");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_sti);
+}
+
+
+/**
+ * @opcode      0xfc
+ */
+FNIEMOP_DEF(iemOp_cld)
+{
+    IEMOP_MNEMONIC(cld, "cld");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_CLEAR_EFL_BIT(X86_EFL_DF);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xfd
+ */
+FNIEMOP_DEF(iemOp_std)
+{
+    IEMOP_MNEMONIC(std, "std");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_SET_EFL_BIT(X86_EFL_DF);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xfe
+ */
+FNIEMOP_DEF(iemOp_Grp4)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0:
+            IEMOP_MNEMONIC(inc_Eb, "inc Eb");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEb, bRm, &g_iemAImpl_inc);
+        case 1:
+            IEMOP_MNEMONIC(dec_Eb, "dec Eb");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEb, bRm, &g_iemAImpl_dec);
+        default:
+            IEMOP_MNEMONIC(grp4_ud, "grp4-ud");
+            return IEMOP_RAISE_INVALID_OPCODE();
+    }
+}
+
+
+/**
+ * Opcode 0xff /2.
+ * @param   bRm             The RM byte.
+ */
+FNIEMOP_DEF_1(iemOp_Grp5_calln_Ev, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(calln_Ev, "calln Ev");
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* The new RIP is taken from a register. */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(1, 0);
+                IEM_MC_ARG(uint16_t, u16Target, 0);
+                IEM_MC_FETCH_GREG_U16(u16Target, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_CALL_CIMPL_1(iemCImpl_call_16, u16Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(1, 0);
+                IEM_MC_ARG(uint32_t, u32Target, 0);
+                IEM_MC_FETCH_GREG_U32(u32Target, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_CALL_CIMPL_1(iemCImpl_call_32, u32Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(1, 0);
+                IEM_MC_ARG(uint64_t, u64Target, 0);
+                IEM_MC_FETCH_GREG_U64(u64Target, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_CALL_CIMPL_1(iemCImpl_call_64, u64Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* The new RIP is taken from a register. */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(1, 1);
+                IEM_MC_ARG(uint16_t,  u16Target, 0);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_CALL_CIMPL_1(iemCImpl_call_16, u16Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(1, 1);
+                IEM_MC_ARG(uint32_t,  u32Target, 0);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_CALL_CIMPL_1(iemCImpl_call_32, u32Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(1, 1);
+                IEM_MC_ARG(uint64_t,  u64Target, 0);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_CALL_CIMPL_1(iemCImpl_call_64, u64Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+typedef IEM_CIMPL_DECL_TYPE_3(FNIEMCIMPLFARBRANCH, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmOpSize);
+
+FNIEMOP_DEF_2(iemOpHlp_Grp5_far_Ep, uint8_t, bRm, FNIEMCIMPLFARBRANCH *, pfnCImpl)
+{
+    /* Registers? How?? */
+    if (RT_LIKELY((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT)))
+    { /* likely */ }
+    else
+        return IEMOP_RAISE_INVALID_OPCODE(); /* callf eax is not legal */
+
+    /* Far pointer loaded from memory. */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(3, 1);
+            IEM_MC_ARG(uint16_t,        u16Sel,                         0);
+            IEM_MC_ARG(uint16_t,        offSeg,                         1);
+            IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize, IEMMODE_16BIT,    2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_MEM_U16_DISP(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 2);
+            IEM_MC_CALL_CIMPL_3(pfnCImpl, u16Sel, offSeg, enmEffOpSize);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            /** @todo testcase: AMD does not seem to believe in the case (see bs-cpu-xcpt-1)
+             *        and will apparently ignore REX.W, at least for the jmp far qword [rsp]
+             *        and call far qword [rsp] encodings. */
+            if (!IEM_IS_GUEST_CPU_AMD(pVCpu))
+            {
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t,        u16Sel,                         0);
+                IEM_MC_ARG(uint64_t,        offSeg,                         1);
+                IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize, IEMMODE_16BIT,    2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_FETCH_MEM_U16_DISP(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 8);
+                IEM_MC_CALL_CIMPL_3(pfnCImpl, u16Sel, offSeg, enmEffOpSize);
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+            /* AMD falls thru. */
+            RT_FALL_THRU();
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(3, 1);
+            IEM_MC_ARG(uint16_t,        u16Sel,                         0);
+            IEM_MC_ARG(uint32_t,        offSeg,                         1);
+            IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize, IEMMODE_32BIT,    2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U32(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_MEM_U16_DISP(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 4);
+            IEM_MC_CALL_CIMPL_3(pfnCImpl, u16Sel, offSeg, enmEffOpSize);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * Opcode 0xff /3.
+ * @param   bRm             The RM byte.
+ */
+FNIEMOP_DEF_1(iemOp_Grp5_callf_Ep, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(callf_Ep, "callf Ep");
+    return FNIEMOP_CALL_2(iemOpHlp_Grp5_far_Ep, bRm, iemCImpl_callf);
+}
+
+
+/**
+ * Opcode 0xff /4.
+ * @param   bRm             The RM byte.
+ */
+FNIEMOP_DEF_1(iemOp_Grp5_jmpn_Ev, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(jmpn_Ev, "jmpn Ev");
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* The new RIP is taken from a register. */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint16_t, u16Target);
+                IEM_MC_FETCH_GREG_U16(u16Target, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_SET_RIP_U16(u16Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint32_t, u32Target);
+                IEM_MC_FETCH_GREG_U32(u32Target, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_SET_RIP_U32(u32Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint64_t, u64Target);
+                IEM_MC_FETCH_GREG_U64(u64Target, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_SET_RIP_U64(u64Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* The new RIP is taken from a memory location. */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, u16Target);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_SET_RIP_U16(u16Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Target);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_SET_RIP_U32(u32Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Target);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_SET_RIP_U64(u64Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * Opcode 0xff /5.
+ * @param   bRm             The RM byte.
+ */
+FNIEMOP_DEF_1(iemOp_Grp5_jmpf_Ep, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(jmpf_Ep, "jmpf Ep");
+    return FNIEMOP_CALL_2(iemOpHlp_Grp5_far_Ep, bRm, iemCImpl_FarJmp);
+}
+
+
+/**
+ * Opcode 0xff /6.
+ * @param   bRm             The RM byte.
+ */
+FNIEMOP_DEF_1(iemOp_Grp5_push_Ev, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(push_Ev, "push Ev");
+
+    /* Registers are handled by a common worker. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        return FNIEMOP_CALL_1(iemOpCommonPushGReg, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+
+    /* Memory we do here. */
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint16_t,  u16Src);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_PUSH_U16(u16Src);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint32_t,  u32Src);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_PUSH_U32(u32Src);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint64_t,  u64Src);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_PUSH_U64(u64Src);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xff
+ */
+FNIEMOP_DEF(iemOp_Grp5)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0:
+            IEMOP_MNEMONIC(inc_Ev, "inc Ev");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEv, bRm, &g_iemAImpl_inc);
+        case 1:
+            IEMOP_MNEMONIC(dec_Ev, "dec Ev");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEv, bRm, &g_iemAImpl_dec);
+        case 2:
+            return FNIEMOP_CALL_1(iemOp_Grp5_calln_Ev, bRm);
+        case 3:
+            return FNIEMOP_CALL_1(iemOp_Grp5_callf_Ep, bRm);
+        case 4:
+            return FNIEMOP_CALL_1(iemOp_Grp5_jmpn_Ev, bRm);
+        case 5:
+            return FNIEMOP_CALL_1(iemOp_Grp5_jmpf_Ep, bRm);
+        case 6:
+            return FNIEMOP_CALL_1(iemOp_Grp5_push_Ev, bRm);
+        case 7:
+            IEMOP_MNEMONIC(grp5_ud, "grp5-ud");
+            return IEMOP_RAISE_INVALID_OPCODE();
+    }
+    AssertFailedReturn(VERR_IEM_IPE_3);
+}
+
+
+
+const PFNIEMOP g_apfnOneByteMap[256] =
+{
+    /* 0x00 */  iemOp_add_Eb_Gb,        iemOp_add_Ev_Gv,        iemOp_add_Gb_Eb,        iemOp_add_Gv_Ev,
+    /* 0x04 */  iemOp_add_Al_Ib,        iemOp_add_eAX_Iz,       iemOp_push_ES,          iemOp_pop_ES,
+    /* 0x08 */  iemOp_or_Eb_Gb,         iemOp_or_Ev_Gv,         iemOp_or_Gb_Eb,         iemOp_or_Gv_Ev,
+    /* 0x0c */  iemOp_or_Al_Ib,         iemOp_or_eAX_Iz,        iemOp_push_CS,          iemOp_2byteEscape,
+    /* 0x10 */  iemOp_adc_Eb_Gb,        iemOp_adc_Ev_Gv,        iemOp_adc_Gb_Eb,        iemOp_adc_Gv_Ev,
+    /* 0x14 */  iemOp_adc_Al_Ib,        iemOp_adc_eAX_Iz,       iemOp_push_SS,          iemOp_pop_SS,
+    /* 0x18 */  iemOp_sbb_Eb_Gb,        iemOp_sbb_Ev_Gv,        iemOp_sbb_Gb_Eb,        iemOp_sbb_Gv_Ev,
+    /* 0x1c */  iemOp_sbb_Al_Ib,        iemOp_sbb_eAX_Iz,       iemOp_push_DS,          iemOp_pop_DS,
+    /* 0x20 */  iemOp_and_Eb_Gb,        iemOp_and_Ev_Gv,        iemOp_and_Gb_Eb,        iemOp_and_Gv_Ev,
+    /* 0x24 */  iemOp_and_Al_Ib,        iemOp_and_eAX_Iz,       iemOp_seg_ES,           iemOp_daa,
+    /* 0x28 */  iemOp_sub_Eb_Gb,        iemOp_sub_Ev_Gv,        iemOp_sub_Gb_Eb,        iemOp_sub_Gv_Ev,
+    /* 0x2c */  iemOp_sub_Al_Ib,        iemOp_sub_eAX_Iz,       iemOp_seg_CS,           iemOp_das,
+    /* 0x30 */  iemOp_xor_Eb_Gb,        iemOp_xor_Ev_Gv,        iemOp_xor_Gb_Eb,        iemOp_xor_Gv_Ev,
+    /* 0x34 */  iemOp_xor_Al_Ib,        iemOp_xor_eAX_Iz,       iemOp_seg_SS,           iemOp_aaa,
+    /* 0x38 */  iemOp_cmp_Eb_Gb,        iemOp_cmp_Ev_Gv,        iemOp_cmp_Gb_Eb,        iemOp_cmp_Gv_Ev,
+    /* 0x3c */  iemOp_cmp_Al_Ib,        iemOp_cmp_eAX_Iz,       iemOp_seg_DS,           iemOp_aas,
+    /* 0x40 */  iemOp_inc_eAX,          iemOp_inc_eCX,          iemOp_inc_eDX,          iemOp_inc_eBX,
+    /* 0x44 */  iemOp_inc_eSP,          iemOp_inc_eBP,          iemOp_inc_eSI,          iemOp_inc_eDI,
+    /* 0x48 */  iemOp_dec_eAX,          iemOp_dec_eCX,          iemOp_dec_eDX,          iemOp_dec_eBX,
+    /* 0x4c */  iemOp_dec_eSP,          iemOp_dec_eBP,          iemOp_dec_eSI,          iemOp_dec_eDI,
+    /* 0x50 */  iemOp_push_eAX,         iemOp_push_eCX,         iemOp_push_eDX,         iemOp_push_eBX,
+    /* 0x54 */  iemOp_push_eSP,         iemOp_push_eBP,         iemOp_push_eSI,         iemOp_push_eDI,
+    /* 0x58 */  iemOp_pop_eAX,          iemOp_pop_eCX,          iemOp_pop_eDX,          iemOp_pop_eBX,
+    /* 0x5c */  iemOp_pop_eSP,          iemOp_pop_eBP,          iemOp_pop_eSI,          iemOp_pop_eDI,
+    /* 0x60 */  iemOp_pusha,            iemOp_popa__mvex,       iemOp_bound_Gv_Ma__evex, iemOp_arpl_Ew_Gw_movsx_Gv_Ev,
+    /* 0x64 */  iemOp_seg_FS,           iemOp_seg_GS,           iemOp_op_size,          iemOp_addr_size,
+    /* 0x68 */  iemOp_push_Iz,          iemOp_imul_Gv_Ev_Iz,    iemOp_push_Ib,          iemOp_imul_Gv_Ev_Ib,
+    /* 0x6c */  iemOp_insb_Yb_DX,       iemOp_inswd_Yv_DX,      iemOp_outsb_Yb_DX,      iemOp_outswd_Yv_DX,
+    /* 0x70 */  iemOp_jo_Jb,            iemOp_jno_Jb,           iemOp_jc_Jb,            iemOp_jnc_Jb,
+    /* 0x74 */  iemOp_je_Jb,            iemOp_jne_Jb,           iemOp_jbe_Jb,           iemOp_jnbe_Jb,
+    /* 0x78 */  iemOp_js_Jb,            iemOp_jns_Jb,           iemOp_jp_Jb,            iemOp_jnp_Jb,
+    /* 0x7c */  iemOp_jl_Jb,            iemOp_jnl_Jb,           iemOp_jle_Jb,           iemOp_jnle_Jb,
+    /* 0x80 */  iemOp_Grp1_Eb_Ib_80,    iemOp_Grp1_Ev_Iz,       iemOp_Grp1_Eb_Ib_82,    iemOp_Grp1_Ev_Ib,
+    /* 0x84 */  iemOp_test_Eb_Gb,       iemOp_test_Ev_Gv,       iemOp_xchg_Eb_Gb,       iemOp_xchg_Ev_Gv,
+    /* 0x88 */  iemOp_mov_Eb_Gb,        iemOp_mov_Ev_Gv,        iemOp_mov_Gb_Eb,        iemOp_mov_Gv_Ev,
+    /* 0x8c */  iemOp_mov_Ev_Sw,        iemOp_lea_Gv_M,         iemOp_mov_Sw_Ev,        iemOp_Grp1A__xop,
+    /* 0x90 */  iemOp_nop,              iemOp_xchg_eCX_eAX,     iemOp_xchg_eDX_eAX,     iemOp_xchg_eBX_eAX,
+    /* 0x94 */  iemOp_xchg_eSP_eAX,     iemOp_xchg_eBP_eAX,     iemOp_xchg_eSI_eAX,     iemOp_xchg_eDI_eAX,
+    /* 0x98 */  iemOp_cbw,              iemOp_cwd,              iemOp_call_Ap,          iemOp_wait,
+    /* 0x9c */  iemOp_pushf_Fv,         iemOp_popf_Fv,          iemOp_sahf,             iemOp_lahf,
+    /* 0xa0 */  iemOp_mov_AL_Ob,        iemOp_mov_rAX_Ov,       iemOp_mov_Ob_AL,        iemOp_mov_Ov_rAX,
+    /* 0xa4 */  iemOp_movsb_Xb_Yb,      iemOp_movswd_Xv_Yv,     iemOp_cmpsb_Xb_Yb,      iemOp_cmpswd_Xv_Yv,
+    /* 0xa8 */  iemOp_test_AL_Ib,       iemOp_test_eAX_Iz,      iemOp_stosb_Yb_AL,      iemOp_stoswd_Yv_eAX,
+    /* 0xac */  iemOp_lodsb_AL_Xb,      iemOp_lodswd_eAX_Xv,    iemOp_scasb_AL_Xb,      iemOp_scaswd_eAX_Xv,
+    /* 0xb0 */  iemOp_mov_AL_Ib,        iemOp_CL_Ib,            iemOp_DL_Ib,            iemOp_BL_Ib,
+    /* 0xb4 */  iemOp_mov_AH_Ib,        iemOp_CH_Ib,            iemOp_DH_Ib,            iemOp_BH_Ib,
+    /* 0xb8 */  iemOp_eAX_Iv,           iemOp_eCX_Iv,           iemOp_eDX_Iv,           iemOp_eBX_Iv,
+    /* 0xbc */  iemOp_eSP_Iv,           iemOp_eBP_Iv,           iemOp_eSI_Iv,           iemOp_eDI_Iv,
+    /* 0xc0 */  iemOp_Grp2_Eb_Ib,       iemOp_Grp2_Ev_Ib,       iemOp_retn_Iw,          iemOp_retn,
+    /* 0xc4 */  iemOp_les_Gv_Mp__vex3,  iemOp_lds_Gv_Mp__vex2,  iemOp_Grp11_Eb_Ib,      iemOp_Grp11_Ev_Iz,
+    /* 0xc8 */  iemOp_enter_Iw_Ib,      iemOp_leave,            iemOp_retf_Iw,          iemOp_retf,
+    /* 0xcc */  iemOp_int3,             iemOp_int_Ib,           iemOp_into,             iemOp_iret,
+    /* 0xd0 */  iemOp_Grp2_Eb_1,        iemOp_Grp2_Ev_1,        iemOp_Grp2_Eb_CL,       iemOp_Grp2_Ev_CL,
+    /* 0xd4 */  iemOp_aam_Ib,           iemOp_aad_Ib,           iemOp_salc,             iemOp_xlat,
+    /* 0xd8 */  iemOp_EscF0,            iemOp_EscF1,            iemOp_EscF2,            iemOp_EscF3,
+    /* 0xdc */  iemOp_EscF4,            iemOp_EscF5,            iemOp_EscF6,            iemOp_EscF7,
+    /* 0xe0 */  iemOp_loopne_Jb,        iemOp_loope_Jb,         iemOp_loop_Jb,          iemOp_jecxz_Jb,
+    /* 0xe4 */  iemOp_in_AL_Ib,         iemOp_in_eAX_Ib,        iemOp_out_Ib_AL,        iemOp_out_Ib_eAX,
+    /* 0xe8 */  iemOp_call_Jv,          iemOp_jmp_Jv,           iemOp_jmp_Ap,           iemOp_jmp_Jb,
+    /* 0xec */  iemOp_in_AL_DX,         iemOp_in_eAX_DX,        iemOp_out_DX_AL,        iemOp_out_DX_eAX,
+    /* 0xf0 */  iemOp_lock,             iemOp_int1,             iemOp_repne,            iemOp_repe,
+    /* 0xf4 */  iemOp_hlt,              iemOp_cmc,              iemOp_Grp3_Eb,          iemOp_Grp3_Ev,
+    /* 0xf8 */  iemOp_clc,              iemOp_stc,              iemOp_cli,              iemOp_sti,
+    /* 0xfc */  iemOp_cld,              iemOp_std,              iemOp_Grp4,             iemOp_Grp5,
+};
+
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py b/src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py
new file mode 100755
index 00000000..fcf5553d
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py
@@ -0,0 +1,3568 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id: IEMAllInstructionsPython.py $
+
+"""
+IEM instruction extractor.
+
+This script/module parses the IEMAllInstruction*.cpp.h files next to it and
+collects information about the instructions.  It can then be used to generate
+disassembler tables and tests.
+"""
+
+__copyright__ = \
+"""
+Copyright (C) 2017-2020 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+
+The contents of this file may alternatively be used under the terms
+of the Common Development and Distribution License Version 1.0
+(CDDL) only, as it comes in the "COPYING.CDDL" file of the
+VirtualBox OSE distribution, in which case the provisions of the
+CDDL are applicable instead of those of the GPL.
+
+You may elect to license modified versions of this file under the
+terms and conditions of either the GPL or the CDDL or both.
+"""
+__version__ = "$Revision: 135976 $"
+
+# pylint: disable=anomalous-backslash-in-string
+
+# Standard python imports.
+import os
+import re
+import sys
+
+## Only the main script needs to modify the path.
+#g_ksValidationKitDir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
+#                                    'ValidationKit');
+#sys.path.append(g_ksValidationKitDir);
+#
+#from common import utils; - Windows build boxes doesn't have pywin32.
+
+# Python 3 hacks:
+if sys.version_info[0] >= 3:
+    long = int;     # pylint: disable=redefined-builtin,invalid-name
+
+
+g_kdX86EFlagsConstants = {
+    'X86_EFL_CF':          0x00000001, # RT_BIT_32(0)
+    'X86_EFL_1':           0x00000002, # RT_BIT_32(1)
+    'X86_EFL_PF':          0x00000004, # RT_BIT_32(2)
+    'X86_EFL_AF':          0x00000010, # RT_BIT_32(4)
+    'X86_EFL_ZF':          0x00000040, # RT_BIT_32(6)
+    'X86_EFL_SF':          0x00000080, # RT_BIT_32(7)
+    'X86_EFL_TF':          0x00000100, # RT_BIT_32(8)
+    'X86_EFL_IF':          0x00000200, # RT_BIT_32(9)
+    'X86_EFL_DF':          0x00000400, # RT_BIT_32(10)
+    'X86_EFL_OF':          0x00000800, # RT_BIT_32(11)
+    'X86_EFL_IOPL':        0x00003000, # (RT_BIT_32(12) | RT_BIT_32(13))
+    'X86_EFL_NT':          0x00004000, # RT_BIT_32(14)
+    'X86_EFL_RF':          0x00010000, # RT_BIT_32(16)
+    'X86_EFL_VM':          0x00020000, # RT_BIT_32(17)
+    'X86_EFL_AC':          0x00040000, # RT_BIT_32(18)
+    'X86_EFL_VIF':         0x00080000, # RT_BIT_32(19)
+    'X86_EFL_VIP':         0x00100000, # RT_BIT_32(20)
+    'X86_EFL_ID':          0x00200000, # RT_BIT_32(21)
+    'X86_EFL_LIVE_MASK':   0x003f7fd5, # UINT32_C(0x003f7fd5)
+    'X86_EFL_RA1_MASK':    0x00000002, # RT_BIT_32(1)
+};
+
+## EFlags values allowed in \@opfltest, \@opflmodify, \@opflundef, \@opflset, and \@opflclear.
+g_kdEFlagsMnemonics = {
+    # Debugger flag notation (sorted by value):
+    'cf':   'X86_EFL_CF',   ##< Carry Flag.
+    'nc':  '!X86_EFL_CF',   ##< No Carry.
+
+    'po':   'X86_EFL_PF',   ##< Parity Pdd.
+    'pe':  '!X86_EFL_PF',   ##< Parity Even.
+
+    'af':   'X86_EFL_AF',   ##< Aux Flag.
+    'na':  '!X86_EFL_AF',   ##< No Aux.
+
+    'zr':   'X86_EFL_ZF',   ##< ZeRo.
+    'nz':  '!X86_EFL_ZF',   ##< No Zero.
+
+    'ng':   'X86_EFL_SF',   ##< NeGative (sign).
+    'pl':  '!X86_EFL_SF',   ##< PLuss (sign).
+
+    'tf':   'X86_EFL_TF',   ##< Trap flag.
+
+    'ei':   'X86_EFL_IF',   ##< Enabled Interrupts.
+    'di':  '!X86_EFL_IF',   ##< Disabled Interrupts.
+
+    'dn':   'X86_EFL_DF',   ##< DowN (string op direction).
+    'up':  '!X86_EFL_DF',   ##< UP (string op direction).
+
+    'ov':   'X86_EFL_OF',   ##< OVerflow.
+    'nv':  '!X86_EFL_OF',   ##< No Overflow.
+
+    'nt':   'X86_EFL_NT',   ##< Nested Task.
+    'rf':   'X86_EFL_RF',   ##< Resume Flag.
+    'vm':   'X86_EFL_VM',   ##< Virtual-8086 Mode.
+    'ac':   'X86_EFL_AC',   ##< Alignment Check.
+    'vif':  'X86_EFL_VIF',  ##< Virtual Interrupt Flag.
+    'vip':  'X86_EFL_VIP',  ##< Virtual Interrupt Pending.
+
+    # Reference manual notation not covered above (sorted by value):
+    'pf':   'X86_EFL_PF',
+    'zf':   'X86_EFL_ZF',
+    'sf':   'X86_EFL_SF',
+    'if':   'X86_EFL_IF',
+    'df':   'X86_EFL_DF',
+    'of':   'X86_EFL_OF',
+    'iopl': 'X86_EFL_IOPL',
+    'id':   'X86_EFL_ID',
+};
+
+## Constants and values for CR0.
+g_kdX86Cr0Constants = {
+    'X86_CR0_PE':           0x00000001, # RT_BIT_32(0)
+    'X86_CR0_MP':           0x00000002, # RT_BIT_32(1)
+    'X86_CR0_EM':           0x00000004, # RT_BIT_32(2)
+    'X86_CR0_TS':           0x00000008, # RT_BIT_32(3)
+    'X86_CR0_ET':           0x00000010, # RT_BIT_32(4)
+    'X86_CR0_NE':           0x00000020, # RT_BIT_32(5)
+    'X86_CR0_WP':           0x00010000, # RT_BIT_32(16)
+    'X86_CR0_AM':           0x00040000, # RT_BIT_32(18)
+    'X86_CR0_NW':           0x20000000, # RT_BIT_32(29)
+    'X86_CR0_CD':           0x40000000, # RT_BIT_32(30)
+    'X86_CR0_PG':           0x80000000, # RT_BIT_32(31)
+};
+
+## Constants and values for CR4.
+g_kdX86Cr4Constants = {
+    'X86_CR4_VME':          0x00000001, # RT_BIT_32(0)
+    'X86_CR4_PVI':          0x00000002, # RT_BIT_32(1)
+    'X86_CR4_TSD':          0x00000004, # RT_BIT_32(2)
+    'X86_CR4_DE':           0x00000008, # RT_BIT_32(3)
+    'X86_CR4_PSE':          0x00000010, # RT_BIT_32(4)
+    'X86_CR4_PAE':          0x00000020, # RT_BIT_32(5)
+    'X86_CR4_MCE':          0x00000040, # RT_BIT_32(6)
+    'X86_CR4_PGE':          0x00000080, # RT_BIT_32(7)
+    'X86_CR4_PCE':          0x00000100, # RT_BIT_32(8)
+    'X86_CR4_OSFXSR':       0x00000200, # RT_BIT_32(9)
+    'X86_CR4_OSXMMEEXCPT':  0x00000400, # RT_BIT_32(10)
+    'X86_CR4_VMXE':         0x00002000, # RT_BIT_32(13)
+    'X86_CR4_SMXE':         0x00004000, # RT_BIT_32(14)
+    'X86_CR4_PCIDE':        0x00020000, # RT_BIT_32(17)
+    'X86_CR4_OSXSAVE':      0x00040000, # RT_BIT_32(18)
+    'X86_CR4_SMEP':         0x00100000, # RT_BIT_32(20)
+    'X86_CR4_SMAP':         0x00200000, # RT_BIT_32(21)
+    'X86_CR4_PKE':          0x00400000, # RT_BIT_32(22)
+};
+
+## XSAVE components (XCR0).
+g_kdX86XSaveCConstants = {
+    'XSAVE_C_X87':          0x00000001,
+    'XSAVE_C_SSE':          0x00000002,
+    'XSAVE_C_YMM':          0x00000004,
+    'XSAVE_C_BNDREGS':      0x00000008,
+    'XSAVE_C_BNDCSR':       0x00000010,
+    'XSAVE_C_OPMASK':       0x00000020,
+    'XSAVE_C_ZMM_HI256':    0x00000040,
+    'XSAVE_C_ZMM_16HI':     0x00000080,
+    'XSAVE_C_PKRU':         0x00000200,
+    'XSAVE_C_LWP':          0x4000000000000000,
+    'XSAVE_C_X':            0x8000000000000000,
+    'XSAVE_C_ALL_AVX':      0x000000c4, # For clearing all AVX bits.
+    'XSAVE_C_ALL_AVX_SSE':  0x000000c6, # For clearing all AVX and SSE bits.
+};
+
+
+## \@op[1-4] locations
+g_kdOpLocations = {
+    'reg':      [], ## modrm.reg
+    'rm':       [], ## modrm.rm
+    'imm':      [], ## immediate instruction data
+    'vvvv':     [], ## VEX.vvvv
+
+    # fixed registers.
+    'AL':       [],
+    'rAX':      [],
+    'rSI':      [],
+    'rDI':      [],
+    'rFLAGS':   [],
+    'CS':       [],
+    'DS':       [],
+    'ES':       [],
+    'FS':       [],
+    'GS':       [],
+    'SS':       [],
+};
+
+## \@op[1-4] types
+##
+## Value fields:
+##    - 0: the normal IDX_ParseXXX handler (IDX_UseModRM == IDX_ParseModRM).
+##    - 1: the location (g_kdOpLocations).
+##    - 2: disassembler format string version of the type.
+##    - 3: disassembler OP_PARAM_XXX (XXX only).
+##    - 4: IEM form matching instruction.
+##
+## Note! See the A.2.1 in SDM vol 2 for the type names.
+g_kdOpTypes = {
+    # Fixed addresses
+    'Ap':           ( 'IDX_ParseImmAddrF',  'imm',    '%Ap',  'Ap',      'FIXED', ),
+
+    # ModR/M.rm
+    'Eb':           ( 'IDX_UseModRM',       'rm',     '%Eb',  'Eb',      'RM',    ),
+    'Ed':           ( 'IDX_UseModRM',       'rm',     '%Ed',  'Ed',      'RM',    ),
+    'Ed_WO':        ( 'IDX_UseModRM',       'rm',     '%Ed',  'Ed',      'RM',    ),
+    'Eq':           ( 'IDX_UseModRM',       'rm',     '%Eq',  'Eq',      'RM',    ),
+    'Eq_WO':        ( 'IDX_UseModRM',       'rm',     '%Eq',  'Eq',      'RM',    ),
+    'Ew':           ( 'IDX_UseModRM',       'rm',     '%Ew',  'Ew',      'RM',    ),
+    'Ev':           ( 'IDX_UseModRM',       'rm',     '%Ev',  'Ev',      'RM',    ),
+    'Qq':           ( 'IDX_UseModRM',       'rm',     '%Qq',  'Qq',      'RM',    ),
+    'Qq_WO':        ( 'IDX_UseModRM',       'rm',     '%Qq',  'Qq',      'RM',    ),
+    'Wss':          ( 'IDX_UseModRM',       'rm',     '%Wss', 'Wss',     'RM',    ),
+    'Wss_WO':       ( 'IDX_UseModRM',       'rm',     '%Wss', 'Wss',     'RM',    ),
+    'Wsd':          ( 'IDX_UseModRM',       'rm',     '%Wsd', 'Wsd',     'RM',    ),
+    'Wsd_WO':       ( 'IDX_UseModRM',       'rm',     '%Wsd', 'Wsd',     'RM',    ),
+    'Wps':          ( 'IDX_UseModRM',       'rm',     '%Wps', 'Wps',     'RM',    ),
+    'Wps_WO':       ( 'IDX_UseModRM',       'rm',     '%Wps', 'Wps',     'RM',    ),
+    'Wpd':          ( 'IDX_UseModRM',       'rm',     '%Wpd', 'Wpd',     'RM',    ),
+    'Wpd_WO':       ( 'IDX_UseModRM',       'rm',     '%Wpd', 'Wpd',     'RM',    ),
+    'Wdq':          ( 'IDX_UseModRM',       'rm',     '%Wdq', 'Wdq',     'RM',    ),
+    'Wdq_WO':       ( 'IDX_UseModRM',       'rm',     '%Wdq', 'Wdq',     'RM',    ),
+    'Wq':           ( 'IDX_UseModRM',       'rm',     '%Wq',  'Wq',      'RM',    ),
+    'Wq_WO':        ( 'IDX_UseModRM',       'rm',     '%Wq',  'Wq',      'RM',    ),
+    'WqZxReg_WO':   ( 'IDX_UseModRM',       'rm',     '%Wq',  'Wq',      'RM',    ),
+    'Wx':           ( 'IDX_UseModRM',       'rm',     '%Wx',  'Wx',      'RM',    ),
+    'Wx_WO':        ( 'IDX_UseModRM',       'rm',     '%Wx',  'Wx',      'RM',    ),
+
+    # ModR/M.rm - register only.
+    'Uq':           ( 'IDX_UseModRM',       'rm',     '%Uq',  'Uq',      'REG'    ),
+    'UqHi':         ( 'IDX_UseModRM',       'rm',     '%Uq',  'UqHi',    'REG'    ),
+    'Uss':          ( 'IDX_UseModRM',       'rm',     '%Uss', 'Uss',     'REG'    ),
+    'Uss_WO':       ( 'IDX_UseModRM',       'rm',     '%Uss', 'Uss',     'REG'    ),
+    'Usd':          ( 'IDX_UseModRM',       'rm',     '%Usd', 'Usd',     'REG'    ),
+    'Usd_WO':       ( 'IDX_UseModRM',       'rm',     '%Usd', 'Usd',     'REG'    ),
+    'Nq':           ( 'IDX_UseModRM',       'rm',     '%Qq',  'Nq',      'REG'    ),
+
+    # ModR/M.rm - memory only.
+    'Ma':           ( 'IDX_UseModRM',       'rm',     '%Ma',  'Ma',      'MEM',   ), ##< Only used by BOUND.
+    'Mb_RO':        ( 'IDX_UseModRM',       'rm',     '%Mb',  'Mb',      'MEM',   ),
+    'Md':           ( 'IDX_UseModRM',       'rm',     '%Md',  'Md',      'MEM',   ),
+    'Md_RO':        ( 'IDX_UseModRM',       'rm',     '%Md',  'Md',      'MEM',   ),
+    'Md_WO':        ( 'IDX_UseModRM',       'rm',     '%Md',  'Md',      'MEM',   ),
+    'Mdq':          ( 'IDX_UseModRM',       'rm',     '%Mdq', 'Mdq',     'MEM',   ),
+    'Mdq_WO':       ( 'IDX_UseModRM',       'rm',     '%Mdq', 'Mdq',     'MEM',   ),
+    'Mq':           ( 'IDX_UseModRM',       'rm',     '%Mq',  'Mq',      'MEM',   ),
+    'Mq_WO':        ( 'IDX_UseModRM',       'rm',     '%Mq',  'Mq',      'MEM',   ),
+    'Mps_WO':       ( 'IDX_UseModRM',       'rm',     '%Mps', 'Mps',     'MEM',   ),
+    'Mpd_WO':       ( 'IDX_UseModRM',       'rm',     '%Mpd', 'Mpd',     'MEM',   ),
+    'Mx':           ( 'IDX_UseModRM',       'rm',     '%Mx',  'Mx',      'MEM',   ),
+    'Mx_WO':        ( 'IDX_UseModRM',       'rm',     '%Mx',  'Mx',      'MEM',   ),
+    'M_RO':         ( 'IDX_UseModRM',       'rm',     '%M',   'M',       'MEM',   ),
+    'M_RW':         ( 'IDX_UseModRM',       'rm',     '%M',   'M',       'MEM',   ),
+
+    # ModR/M.reg
+    'Gb':           ( 'IDX_UseModRM',       'reg',    '%Gb',  'Gb',      '',      ),
+    'Gw':           ( 'IDX_UseModRM',       'reg',    '%Gw',  'Gw',      '',      ),
+    'Gv':           ( 'IDX_UseModRM',       'reg',    '%Gv',  'Gv',      '',      ),
+    'Gv_RO':        ( 'IDX_UseModRM',       'reg',    '%Gv',  'Gv',      '',      ),
+    'Pd':           ( 'IDX_UseModRM',       'reg',    '%Pd',  'Pd',      '',      ),
+    'PdZx_WO':      ( 'IDX_UseModRM',       'reg',    '%Pd',  'PdZx',    '',      ),
+    'Pq':           ( 'IDX_UseModRM',       'reg',    '%Pq',  'Pq',      '',      ),
+    'Pq_WO':        ( 'IDX_UseModRM',       'reg',    '%Pq',  'Pq',      '',      ),
+    'Vd':           ( 'IDX_UseModRM',       'reg',    '%Vd',  'Vd',      '',      ),
+    'Vd_WO':        ( 'IDX_UseModRM',       'reg',    '%Vd',  'Vd',      '',      ),
+    'VdZx_WO':      ( 'IDX_UseModRM',       'reg',    '%Vd',  'Vd',      '',      ),
+    'Vdq':          ( 'IDX_UseModRM',       'reg',    '%Vdq', 'Vdq',     '',      ),
+    'Vss':          ( 'IDX_UseModRM',       'reg',    '%Vss', 'Vss',     '',      ),
+    'Vss_WO':       ( 'IDX_UseModRM',       'reg',    '%Vss', 'Vss',     '',      ),
+    'VssZx_WO':     ( 'IDX_UseModRM',       'reg',    '%Vss', 'Vss',     '',      ),
+    'Vsd':          ( 'IDX_UseModRM',       'reg',    '%Vsd', 'Vsd',     '',      ),
+    'Vsd_WO':       ( 'IDX_UseModRM',       'reg',    '%Vsd', 'Vsd',     '',      ),
+    'VsdZx_WO':     ( 'IDX_UseModRM',       'reg',    '%Vsd', 'Vsd',     '',      ),
+    'Vps':          ( 'IDX_UseModRM',       'reg',    '%Vps', 'Vps',     '',      ),
+    'Vps_WO':       ( 'IDX_UseModRM',       'reg',    '%Vps', 'Vps',     '',      ),
+    'Vpd':          ( 'IDX_UseModRM',       'reg',    '%Vpd', 'Vpd',     '',      ),
+    'Vpd_WO':       ( 'IDX_UseModRM',       'reg',    '%Vpd', 'Vpd',     '',      ),
+    'Vq':           ( 'IDX_UseModRM',       'reg',    '%Vq',  'Vq',      '',      ),
+    'Vq_WO':        ( 'IDX_UseModRM',       'reg',    '%Vq',  'Vq',      '',      ),
+    'Vdq_WO':       ( 'IDX_UseModRM',       'reg',    '%Vdq', 'Vdq',     '',      ),
+    'VqHi':         ( 'IDX_UseModRM',       'reg',    '%Vdq', 'VdqHi',   '',      ),
+    'VqHi_WO':      ( 'IDX_UseModRM',       'reg',    '%Vdq', 'VdqHi',   '',      ),
+    'VqZx_WO':      ( 'IDX_UseModRM',       'reg',    '%Vq',  'VqZx',    '',      ),
+    'Vx':           ( 'IDX_UseModRM',       'reg',    '%Vx',  'Vx',      '',      ),
+    'Vx_WO':        ( 'IDX_UseModRM',       'reg',    '%Vx',  'Vx',      '',      ),
+
+    # VEX.vvvv
+    'HssHi':        ( 'IDX_UseModRM',       'vvvv',   '%Hx',  'HssHi',   'V',     ),
+    'HsdHi':        ( 'IDX_UseModRM',       'vvvv',   '%Hx',  'HsdHi',   'V',     ),
+    'HqHi':         ( 'IDX_UseModRM',       'vvvv',   '%Hq',  'HqHi',    'V',     ),
+
+    # Immediate values.
+    'Ib':           ( 'IDX_ParseImmByte',   'imm',    '%Ib',  'Ib',   '', ), ##< NB! Could be IDX_ParseImmByteSX for some instrs.
+    'Iw':           ( 'IDX_ParseImmUshort', 'imm',    '%Iw',  'Iw',      '',      ),
+    'Id':           ( 'IDX_ParseImmUlong',  'imm',    '%Id',  'Id',      '',      ),
+    'Iq':           ( 'IDX_ParseImmQword',  'imm',    '%Iq',  'Iq',      '',      ),
+    'Iv':           ( 'IDX_ParseImmV',      'imm',    '%Iv',  'Iv',      '',      ), ##< o16: word, o32: dword, o64: qword
+    'Iz':           ( 'IDX_ParseImmZ',      'imm',    '%Iz',  'Iz',      '',      ), ##< o16: word, o32|o64:dword
+
+    # Address operands (no ModR/M).
+    'Ob':           ( 'IDX_ParseImmAddr',   'imm',    '%Ob',  'Ob',      '',      ),
+    'Ov':           ( 'IDX_ParseImmAddr',   'imm',    '%Ov',  'Ov',      '',      ),
+
+    # Relative jump targets
+    'Jb':           ( 'IDX_ParseImmBRel',   'imm',    '%Jb',  'Jb',      '',      ),
+    'Jv':           ( 'IDX_ParseImmVRel',   'imm',    '%Jv',  'Jv',      '',      ),
+
+    # DS:rSI
+    'Xb':           ( 'IDX_ParseXb',        'rSI',    '%eSI', 'Xb',      '',      ),
+    'Xv':           ( 'IDX_ParseXv',        'rSI',    '%eSI', 'Xv',      '',      ),
+    # ES:rDI
+    'Yb':           ( 'IDX_ParseYb',        'rDI',    '%eDI', 'Yb',      '',      ),
+    'Yv':           ( 'IDX_ParseYv',        'rDI',    '%eDI', 'Yv',      '',      ),
+
+    'Fv':           ( 'IDX_ParseFixedReg',  'rFLAGS', '%Fv',  'Fv',      '',      ),
+
+    # Fixed registers.
+    'AL':           ( 'IDX_ParseFixedReg',  'AL',     'al',   'REG_AL',  '',      ),
+    'rAX':          ( 'IDX_ParseFixedReg',  'rAX',    '%eAX', 'REG_EAX', '',      ),
+    'CS':           ( 'IDX_ParseFixedReg',  'CS',     'cs',   'REG_CS',  '',      ), # 8086: push CS
+    'DS':           ( 'IDX_ParseFixedReg',  'DS',     'ds',   'REG_DS',  '',      ),
+    'ES':           ( 'IDX_ParseFixedReg',  'ES',     'es',   'REG_ES',  '',      ),
+    'FS':           ( 'IDX_ParseFixedReg',  'FS',     'fs',   'REG_FS',  '',      ),
+    'GS':           ( 'IDX_ParseFixedReg',  'GS',     'gs',   'REG_GS',  '',      ),
+    'SS':           ( 'IDX_ParseFixedReg',  'SS',     'ss',   'REG_SS',  '',      ),
+};
+
+# IDX_ParseFixedReg
+# IDX_ParseVexDest
+
+
+## IEMFORM_XXX mappings.
+g_kdIemForms = {     # sEncoding,   [ sWhere1, ... ]         opcodesub      ),
+    'RM':           ( 'ModR/M',     [ 'reg', 'rm' ],         '',            ),
+    'RM_REG':       ( 'ModR/M',     [ 'reg', 'rm' ],         '11 mr/reg',   ),
+    'RM_MEM':       ( 'ModR/M',     [ 'reg', 'rm' ],         '!11 mr/reg',  ),
+    'MR':           ( 'ModR/M',     [ 'rm', 'reg' ],         '',            ),
+    'MR_REG':       ( 'ModR/M',     [ 'rm', 'reg' ],         '11 mr/reg',   ),
+    'MR_MEM':       ( 'ModR/M',     [ 'rm', 'reg' ],         '!11 mr/reg',  ),
+    'M':            ( 'ModR/M',     [ 'rm', ],               '',            ),
+    'M_REG':        ( 'ModR/M',     [ 'rm', ],               '',            ),
+    'M_MEM':        ( 'ModR/M',     [ 'rm', ],               '',            ),
+    'R':            ( 'ModR/M',     [ 'reg', ],              '',            ),
+
+    'VEX_RM':       ( 'VEX.ModR/M', [ 'reg', 'rm' ],         '',            ),
+    'VEX_RM_REG':   ( 'VEX.ModR/M', [ 'reg', 'rm' ],         '11 mr/reg',   ),
+    'VEX_RM_MEM':   ( 'VEX.ModR/M', [ 'reg', 'rm' ],         '!11 mr/reg',  ),
+    'VEX_MR':       ( 'VEX.ModR/M', [ 'rm', 'reg' ],         '',            ),
+    'VEX_MR_REG':   ( 'VEX.ModR/M', [ 'rm', 'reg' ],         '11 mr/reg',   ),
+    'VEX_MR_MEM':   ( 'VEX.ModR/M', [ 'rm', 'reg' ],         '!11 mr/reg',  ),
+    'VEX_M':        ( 'VEX.ModR/M', [ 'rm', ],               '' ),
+    'VEX_M_REG':    ( 'VEX.ModR/M', [ 'rm', ],               '' ),
+    'VEX_M_MEM':    ( 'VEX.ModR/M', [ 'rm', ],               '' ),
+    'VEX_R':        ( 'VEX.ModR/M', [ 'reg', ],              '' ),
+    'VEX_RVM':      ( 'VEX.ModR/M', [ 'reg', 'vvvv', 'rm' ], '',            ),
+    'VEX_RVM_REG':  ( 'VEX.ModR/M', [ 'reg', 'vvvv', 'rm' ], '11 mr/reg',   ),
+    'VEX_RVM_MEM':  ( 'VEX.ModR/M', [ 'reg', 'vvvv', 'rm' ], '!11 mr/reg',  ),
+    'VEX_MVR':      ( 'VEX.ModR/M', [ 'rm', 'vvvv', 'reg' ], '',            ),
+    'VEX_MVR_REG':  ( 'VEX.ModR/M', [ 'rm', 'vvvv', 'reg' ], '11 mr/reg',   ),
+    'VEX_MVR_MEM':  ( 'VEX.ModR/M', [ 'rm', 'vvvv', 'reg' ], '!11 mr/reg',  ),
+
+    'FIXED':        ( 'fixed',      None,                    '',            ),
+};
+
+## \@oppfx values.
+g_kdPrefixes = {
+    'none': [],
+    '0x66': [],
+    '0xf3': [],
+    '0xf2': [],
+};
+
+## Special \@opcode tag values.
+g_kdSpecialOpcodes = {
+    '/reg':         [],
+    'mr/reg':       [],
+    '11 /reg':      [],
+    '!11 /reg':     [],
+    '11 mr/reg':    [],
+    '!11 mr/reg':   [],
+};
+
+## Special \@opcodesub tag values.
+## The first value is the real value for aliases.
+## The second value is for bs3cg1.
+g_kdSubOpcodes = {
+    'none':                 [ None,                 '',         ],
+    '11 mr/reg':            [ '11 mr/reg',          '',         ],
+    '11':                   [ '11 mr/reg',          '',         ], ##< alias
+    '!11 mr/reg':           [ '!11 mr/reg',         '',         ],
+    '!11':                  [ '!11 mr/reg',         '',         ], ##< alias
+    'rex.w=0':              [ 'rex.w=0',            'WZ',       ],
+    'w=0':                  [ 'rex.w=0',            '',         ], ##< alias
+    'rex.w=1':              [ 'rex.w=1',            'WNZ',      ],
+    'w=1':                  [ 'rex.w=1',            '',         ], ##< alias
+    'vex.l=0':              [ 'vex.l=0',            'L0',       ],
+    'vex.l=1':              [ 'vex.l=0',            'L1',       ],
+    '11 mr/reg vex.l=0':    [ '11 mr/reg vex.l=0',  'L0',       ],
+    '11 mr/reg vex.l=1':    [ '11 mr/reg vex.l=1',  'L1',       ],
+    '!11 mr/reg vex.l=0':   [ '!11 mr/reg vex.l=0', 'L0',       ],
+    '!11 mr/reg vex.l=1':   [ '!11 mr/reg vex.l=1', 'L1',       ],
+};
+
+## Valid values for \@openc
+g_kdEncodings = {
+    'ModR/M':       [ 'BS3CG1ENC_MODRM', ],     ##< ModR/M
+    'VEX.ModR/M':   [ 'BS3CG1ENC_VEX_MODRM', ], ##< VEX...ModR/M
+    'fixed':        [ 'BS3CG1ENC_FIXED', ],     ##< Fixed encoding (address, registers, unused, etc).
+    'VEX.fixed':    [ 'BS3CG1ENC_VEX_FIXED', ], ##< VEX + fixed encoding (address, registers, unused, etc).
+    'prefix':       [ None, ],                  ##< Prefix
+};
+
+## \@opunused, \@opinvalid, \@opinvlstyle
+g_kdInvalidStyles = {
+    'immediate':                [], ##< CPU stops decoding immediately after the opcode.
+    'vex.modrm':                [], ##< VEX+ModR/M, everyone.
+    'intel-modrm':              [], ##< Intel decodes ModR/M.
+    'intel-modrm-imm8':         [], ##< Intel decodes ModR/M and an 8-byte immediate.
+    'intel-opcode-modrm':       [], ##< Intel decodes another opcode byte followed by ModR/M. (Unused extension tables.)
+    'intel-opcode-modrm-imm8':  [], ##< Intel decodes another opcode byte followed by ModR/M and an 8-byte immediate.
+};
+
+g_kdCpuNames = {
+    '8086':     (),
+    '80186':    (),
+    '80286':    (),
+    '80386':    (),
+    '80486':    (),
+};
+
+## \@opcpuid
+g_kdCpuIdFlags = {
+    'vme':          'X86_CPUID_FEATURE_EDX_VME',
+    'tsc':          'X86_CPUID_FEATURE_EDX_TSC',
+    'msr':          'X86_CPUID_FEATURE_EDX_MSR',
+    'cx8':          'X86_CPUID_FEATURE_EDX_CX8',
+    'sep':          'X86_CPUID_FEATURE_EDX_SEP',
+    'cmov':         'X86_CPUID_FEATURE_EDX_CMOV',
+    'clfsh':        'X86_CPUID_FEATURE_EDX_CLFSH',
+    'clflushopt':   'X86_CPUID_STEXT_FEATURE_EBX_CLFLUSHOPT',
+    'mmx':          'X86_CPUID_FEATURE_EDX_MMX',
+    'fxsr':         'X86_CPUID_FEATURE_EDX_FXSR',
+    'sse':          'X86_CPUID_FEATURE_EDX_SSE',
+    'sse2':         'X86_CPUID_FEATURE_EDX_SSE2',
+    'sse3':         'X86_CPUID_FEATURE_ECX_SSE3',
+    'pclmul':       'X86_CPUID_FEATURE_ECX_DTES64',
+    'monitor':      'X86_CPUID_FEATURE_ECX_CPLDS',
+    'vmx':          'X86_CPUID_FEATURE_ECX_VMX',
+    'smx':          'X86_CPUID_FEATURE_ECX_TM2',
+    'ssse3':        'X86_CPUID_FEATURE_ECX_SSSE3',
+    'fma':          'X86_CPUID_FEATURE_ECX_FMA',
+    'cx16':         'X86_CPUID_FEATURE_ECX_CX16',
+    'pcid':         'X86_CPUID_FEATURE_ECX_PCID',
+    'sse4.1':       'X86_CPUID_FEATURE_ECX_SSE4_1',
+    'sse4.2':       'X86_CPUID_FEATURE_ECX_SSE4_2',
+    'movbe':        'X86_CPUID_FEATURE_ECX_MOVBE',
+    'popcnt':       'X86_CPUID_FEATURE_ECX_POPCNT',
+    'aes':          'X86_CPUID_FEATURE_ECX_AES',
+    'xsave':        'X86_CPUID_FEATURE_ECX_XSAVE',
+    'avx':          'X86_CPUID_FEATURE_ECX_AVX',
+    'avx2':         'X86_CPUID_STEXT_FEATURE_EBX_AVX2',
+    'f16c':         'X86_CPUID_FEATURE_ECX_F16C',
+    'rdrand':       'X86_CPUID_FEATURE_ECX_RDRAND',
+
+    'axmmx':        'X86_CPUID_AMD_FEATURE_EDX_AXMMX',
+    '3dnowext':     'X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX',
+    '3dnow':        'X86_CPUID_AMD_FEATURE_EDX_3DNOW',
+    'svm':          'X86_CPUID_AMD_FEATURE_ECX_SVM',
+    'cr8l':         'X86_CPUID_AMD_FEATURE_ECX_CR8L',
+    'abm':          'X86_CPUID_AMD_FEATURE_ECX_ABM',
+    'sse4a':        'X86_CPUID_AMD_FEATURE_ECX_SSE4A',
+    '3dnowprf':     'X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF',
+    'xop':          'X86_CPUID_AMD_FEATURE_ECX_XOP',
+    'fma4':         'X86_CPUID_AMD_FEATURE_ECX_FMA4',
+};
+
+## \@ophints values.
+g_kdHints = {
+    'invalid':               'DISOPTYPE_INVALID',               ##<
+    'harmless':              'DISOPTYPE_HARMLESS',              ##<
+    'controlflow':           'DISOPTYPE_CONTROLFLOW',           ##<
+    'potentially_dangerous': 'DISOPTYPE_POTENTIALLY_DANGEROUS', ##<
+    'dangerous':             'DISOPTYPE_DANGEROUS',             ##<
+    'portio':                'DISOPTYPE_PORTIO',                ##<
+    'privileged':            'DISOPTYPE_PRIVILEGED',            ##<
+    'privileged_notrap':     'DISOPTYPE_PRIVILEGED_NOTRAP',     ##<
+    'uncond_controlflow':    'DISOPTYPE_UNCOND_CONTROLFLOW',    ##<
+    'relative_controlflow':  'DISOPTYPE_RELATIVE_CONTROLFLOW',  ##<
+    'cond_controlflow':      'DISOPTYPE_COND_CONTROLFLOW',      ##<
+    'interrupt':             'DISOPTYPE_INTERRUPT',             ##<
+    'illegal':               'DISOPTYPE_ILLEGAL',               ##<
+    'rrm_dangerous':         'DISOPTYPE_RRM_DANGEROUS',         ##< Some additional dangerous ones when recompiling raw r0.
+    'rrm_dangerous_16':      'DISOPTYPE_RRM_DANGEROUS_16',      ##< Some additional dangerous ones when recompiling 16-bit raw r0.
+    'inhibit_irqs':          'DISOPTYPE_INHIBIT_IRQS',          ##< Will or can inhibit irqs (sti, pop ss, mov ss) */
+    'portio_read':           'DISOPTYPE_PORTIO_READ',           ##<
+    'portio_write':          'DISOPTYPE_PORTIO_WRITE',          ##<
+    'invalid_64':            'DISOPTYPE_INVALID_64',            ##< Invalid in 64 bits mode
+    'only_64':               'DISOPTYPE_ONLY_64',               ##< Only valid in 64 bits mode
+    'default_64_op_size':    'DISOPTYPE_DEFAULT_64_OP_SIZE',    ##< Default 64 bits operand size
+    'forced_64_op_size':     'DISOPTYPE_FORCED_64_OP_SIZE',     ##< Forced 64 bits operand size; regardless of prefix bytes
+    'rexb_extends_opreg':    'DISOPTYPE_REXB_EXTENDS_OPREG',    ##< REX.B extends the register field in the opcode byte
+    'mod_fixed_11':          'DISOPTYPE_MOD_FIXED_11',          ##< modrm.mod is always 11b
+    'forced_32_op_size_x86': 'DISOPTYPE_FORCED_32_OP_SIZE_X86', ##< Forced 32 bits operand size; regardless of prefix bytes
+                                                                ##  (only in 16 & 32 bits mode!)
+    'sse':                   'DISOPTYPE_SSE',                   ##< SSE,SSE2,SSE3,AVX,++ instruction. Not implemented yet!
+    'mmx':                   'DISOPTYPE_MMX',                   ##< MMX,MMXExt,3DNow,++ instruction. Not implemented yet!
+    'fpu':                   'DISOPTYPE_FPU',                   ##< FPU instruction. Not implemented yet!
+    'ignores_oz_pfx':        '',                                ##< Ignores operand size prefix 66h.
+    'ignores_rexw':          '',                                ##< Ignores REX.W.
+    'ignores_op_sizes':      '',                                ##< Shorthand for "ignores_oz_pfx | ignores_op_sizes".
+    'vex_l_zero':            '',                                ##< VEX.L must be 0.
+    'vex_l_ignored':         '',                                ##< VEX.L is ignored.
+    'lock_allowed':          '',                                ##< Lock prefix allowed.
+};
+
+## \@opxcpttype values (see SDMv2 2.4, 2.7).
+g_kdXcptTypes = {
+    'none':     [],
+    '1':        [],
+    '2':        [],
+    '3':        [],
+    '4':        [],
+    '4UA':      [],
+    '5':        [],
+    '5LZ':      [], # LZ = VEX.L must be zero.
+    '6':        [],
+    '7':        [],
+    '7LZ':      [],
+    '8':        [],
+    '11':       [],
+    '12':       [],
+    'E1':       [],
+    'E1NF':     [],
+    'E2':       [],
+    'E3':       [],
+    'E3NF':     [],
+    'E4':       [],
+    'E4NF':     [],
+    'E5':       [],
+    'E5NF':     [],
+    'E6':       [],
+    'E6NF':     [],
+    'E7NF':     [],
+    'E9':       [],
+    'E9NF':     [],
+    'E10':      [],
+    'E11':      [],
+    'E12':      [],
+    'E12NF':    [],
+};
+
+
+def _isValidOpcodeByte(sOpcode):
+    """
+    Checks if sOpcode is a valid lower case opcode byte.
+    Returns true/false.
+    """
+    if len(sOpcode) == 4:
+        if sOpcode[:2] == '0x':
+            if sOpcode[2] in '0123456789abcdef':
+                if sOpcode[3] in '0123456789abcdef':
+                    return True;
+    return False;
+
+
+class InstructionMap(object):
+    """
+    Instruction map.
+
+    The opcode map provides the lead opcode bytes (empty for the one byte
+    opcode map).  An instruction can be member of multiple opcode maps as long
+    as it uses the same opcode value within the map (because of VEX).
+    """
+
+    kdEncodings = {
+        'legacy':   [],
+        'vex1':     [], ##< VEX or EVEX prefix with vvvvv = 1
+        'vex2':     [], ##< VEX or EVEX prefix with vvvvv = 2
+        'vex3':     [], ##< VEX or EVEX prefix with vvvvv = 3
+        'xop8':     [], ##< XOP prefix with vvvvv = 8
+        'xop9':     [], ##< XOP prefix with vvvvv = 9
+        'xop10':    [], ##< XOP prefix with vvvvv = 10
+    };
+    ## Selectors.
+    ## The first value is the number of table entries required by a
+    ## decoder or disassembler for this type of selector.
+    kdSelectors = {
+        'byte':     [ 256, ], ##< next opcode byte selects the instruction (default).
+        '/r':       [   8, ], ##< modrm.reg selects the instruction.
+        'memreg /r':[  16, ], ##< modrm.reg and (modrm.mod == 3) selects the instruction.
+        'mod /r':   [  32, ], ##< modrm.reg and modrm.mod selects the instruction.
+        '!11 /r':   [   8, ], ##< modrm.reg selects the instruction with modrm.mod != 0y11.
+        '11 /r':    [   8, ], ##< modrm.reg select the instruction with modrm.mod == 0y11.
+        '11':       [  64, ], ##< modrm.reg and modrm.rm select the instruction with modrm.mod == 0y11.
+    };
+
+    def __init__(self, sName, asLeadOpcodes = None, sSelector = 'byte', sEncoding = 'legacy', sDisParse = None):
+        assert sSelector in self.kdSelectors;
+        assert sEncoding in self.kdEncodings;
+        if asLeadOpcodes is None:
+            asLeadOpcodes = [];
+        else:
+            for sOpcode in asLeadOpcodes:
+                assert _isValidOpcodeByte(sOpcode);
+        assert sDisParse is None or sDisParse.startswith('IDX_Parse');
+
+        self.sName          = sName;
+        self.asLeadOpcodes  = asLeadOpcodes;    ##< Lead opcode bytes formatted as hex strings like '0x0f'.
+        self.sSelector      = sSelector;        ##< The member selector, see kdSelectors.
+        self.sEncoding      = sEncoding;        ##< The encoding, see kdSelectors.
+        self.aoInstructions = []                # type: Instruction
+        self.sDisParse      = sDisParse;        ##< IDX_ParseXXX.
+
+    def getTableSize(self):
+        """
+        Number of table entries.   This corresponds directly to the selector.
+        """
+        return self.kdSelectors[self.sSelector][0];
+
+    def getInstructionIndex(self, oInstr):
+        """
+        Returns the table index for the instruction.
+        """
+        bOpcode = oInstr.getOpcodeByte();
+
+        # The byte selector is simple.  We need a full opcode byte and need just return it.
+        if self.sSelector == 'byte':
+            assert oInstr.sOpcode[:2] == '0x' and len(oInstr.sOpcode) == 4, str(oInstr);
+            return bOpcode;
+
+        # The other selectors needs masking and shifting.
+        if self.sSelector == '/r':
+            return (bOpcode >> 3) & 0x7;
+
+        if self.sSelector == 'mod /r':
+            return (bOpcode >> 3) & 0x1f;
+
+        if self.sSelector == 'memreg /r':
+            return ((bOpcode >> 3) & 0x7) | (int((bOpcode >> 6) == 3) << 3);
+
+        if self.sSelector == '!11 /r':
+            assert (bOpcode & 0xc0) != 0xc, str(oInstr);
+            return (bOpcode >> 3) & 0x7;
+
+        if self.sSelector == '11 /r':
+            assert (bOpcode & 0xc0) == 0xc, str(oInstr);
+            return (bOpcode >> 3) & 0x7;
+
+        if self.sSelector == '11':
+            assert (bOpcode & 0xc0) == 0xc, str(oInstr);
+            return bOpcode & 0x3f;
+
+        assert False, self.sSelector;
+        return -1;
+
+    def getInstructionsInTableOrder(self):
+        """
+        Get instructions in table order.
+
+        Returns array of instructions.  Normally there is exactly one
+        instruction per entry.  However the entry could also be None if
+        not instruction was specified for that opcode value.  Or there
+        could be a list of instructions to deal with special encodings
+        where for instance prefix (e.g. REX.W) encodes a different
+        instruction or different CPUs have different instructions or
+        prefixes in the same place.
+        """
+        # Start with empty table.
+        cTable  = self.getTableSize();
+        aoTable = [None] * cTable;
+
+        # Insert the instructions.
+        for oInstr in self.aoInstructions:
+            if oInstr.sOpcode:
+                idxOpcode = self.getInstructionIndex(oInstr);
+                assert idxOpcode < cTable, str(idxOpcode);
+
+                oExisting = aoTable[idxOpcode];
+                if oExisting is None:
+                    aoTable[idxOpcode] = oInstr;
+                elif not isinstance(oExisting, list):
+                    aoTable[idxOpcode] = list([oExisting, oInstr]);
+                else:
+                    oExisting.append(oInstr);
+
+        return aoTable;
+
+
+    def getDisasTableName(self):
+        """
+        Returns the disassembler table name for this map.
+        """
+        sName = 'g_aDisas';
+        for sWord in self.sName.split('_'):
+            if sWord == 'm':            # suffix indicating modrm.mod==mem
+                sName += '_m';
+            elif sWord == 'r':          # suffix indicating modrm.mod==reg
+                sName += '_r';
+            elif len(sWord) == 2 and re.match('^[a-f0-9][a-f0-9]$', sWord):
+                sName += '_' + sWord;
+            else:
+                sWord  = sWord.replace('grp', 'Grp');
+                sWord  = sWord.replace('map', 'Map');
+                sName += sWord[0].upper() + sWord[1:];
+        return sName;
+
+
+    def isVexMap(self):
+        """ Returns True if a VEX map. """
+        return self.sEncoding.startswith('vex');
+
+
+class TestType(object):
+    """
+    Test value type.
+
+    This base class deals with integer like values.  The fUnsigned constructor
+    parameter indicates the default stance on zero vs sign extending.  It is
+    possible to override fUnsigned=True by prefixing the value with '+' or '-'.
+    """
+    def __init__(self, sName, acbSizes = None, fUnsigned = True):
+        self.sName = sName;
+        self.acbSizes = [1, 2, 4, 8, 16, 32] if acbSizes is None else acbSizes;  # Normal sizes.
+        self.fUnsigned = fUnsigned;
+
+    class BadValue(Exception):
+        """ Bad value exception. """
+        def __init__(self, sMessage):
+            Exception.__init__(self, sMessage);
+            self.sMessage = sMessage;
+
+    ## For ascii ~ operator.
+    kdHexInv = {
+        '0': 'f',
+        '1': 'e',
+        '2': 'd',
+        '3': 'c',
+        '4': 'b',
+        '5': 'a',
+        '6': '9',
+        '7': '8',
+        '8': '7',
+        '9': '6',
+        'a': '5',
+        'b': '4',
+        'c': '3',
+        'd': '2',
+        'e': '1',
+        'f': '0',
+    };
+
+    def get(self, sValue):
+        """
+        Get the shortest normal sized byte representation of oValue.
+
+        Returns ((fSignExtend, bytearray), ) or ((fSignExtend, bytearray), (fSignExtend, bytearray), ).
+        The latter form is for AND+OR pairs where the first entry is what to
+        AND with the field and the second the one or OR with.
+
+        Raises BadValue if invalid value.
+        """
+        if not sValue:
+            raise TestType.BadValue('empty value');
+
+        # Deal with sign and detect hexadecimal or decimal.
+        fSignExtend = not self.fUnsigned;
+        if sValue[0] == '-' or sValue[0] == '+':
+            fSignExtend = True;
+            fHex = len(sValue) > 3 and sValue[1:3].lower() == '0x';
+        else:
+            fHex = len(sValue) > 2 and sValue[0:2].lower() == '0x';
+
+        # try convert it to long integer.
+        try:
+            iValue = long(sValue, 16 if fHex else 10);
+        except Exception as oXcpt:
+            raise TestType.BadValue('failed to convert "%s" to integer (%s)' % (sValue, oXcpt));
+
+        # Convert the hex string and pad it to a decent value.  Negative values
+        # needs to be manually converted to something non-negative (~-n + 1).
+        if iValue >= 0:
+            sHex = hex(iValue);
+            if sys.version_info[0] < 3:
+                assert sHex[-1] == 'L';
+                sHex = sHex[:-1];
+            assert sHex[:2] == '0x';
+            sHex = sHex[2:];
+        else:
+            sHex = hex(-iValue - 1);
+            if sys.version_info[0] < 3:
+                assert sHex[-1] == 'L';
+                sHex = sHex[:-1];
+            assert sHex[:2] == '0x';
+            sHex = ''.join([self.kdHexInv[sDigit] for sDigit in sHex[2:]]);
+            if fSignExtend and sHex[0] not in [ '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']:
+                sHex = 'f' + sHex;
+
+        cDigits = len(sHex);
+        if cDigits <= self.acbSizes[-1] * 2:
+            for cb in self.acbSizes:
+                cNaturalDigits = cb * 2;
+                if cDigits <= cNaturalDigits:
+                    break;
+        else:
+            cNaturalDigits = self.acbSizes[-1] * 2;
+            cNaturalDigits = int((cDigits + cNaturalDigits - 1) / cNaturalDigits) * cNaturalDigits;
+            assert isinstance(cNaturalDigits, int)
+
+        if cNaturalDigits != cDigits:
+            cNeeded = cNaturalDigits - cDigits;
+            if iValue >= 0:
+                sHex = ('0' * cNeeded) + sHex;
+            else:
+                sHex = ('f' * cNeeded) + sHex;
+
+        # Invert and convert to bytearray and return it.
+        abValue = bytearray([int(sHex[offHex - 2 : offHex], 16) for offHex in range(len(sHex), 0, -2)]);
+
+        return ((fSignExtend, abValue),);
+
+    def validate(self, sValue):
+        """
+        Returns True if value is okay, error message on failure.
+        """
+        try:
+            self.get(sValue);
+        except TestType.BadValue as oXcpt:
+            return oXcpt.sMessage;
+        return True;
+
+    def isAndOrPair(self, sValue):
+        """
+        Checks if sValue is a pair.
+        """
+        _ = sValue;
+        return False;
+
+
+class TestTypeEflags(TestType):
+    """
+    Special value parsing for EFLAGS/RFLAGS/FLAGS.
+    """
+
+    kdZeroValueFlags = { 'nv': 0, 'pl': 0, 'nz': 0, 'na': 0, 'pe': 0, 'nc': 0, 'di': 0, 'up': 0 };
+
+    def __init__(self, sName):
+        TestType.__init__(self, sName, acbSizes = [1, 2, 4, 8], fUnsigned = True);
+
+    def get(self, sValue):
+        fClear = 0;
+        fSet   = 0;
+        for sFlag in sValue.split(','):
+            sConstant = g_kdEFlagsMnemonics.get(sFlag, None);
+            if sConstant is None:
+                raise self.BadValue('Unknown flag "%s" in "%s"' % (sFlag, sValue))
+            if sConstant[0] == '!':
+                fClear |= g_kdX86EFlagsConstants[sConstant[1:]];
+            else:
+                fSet   |= g_kdX86EFlagsConstants[sConstant];
+
+        aoSet = TestType.get(self, '0x%x' % (fSet,));
+        if fClear != 0:
+            aoClear = TestType.get(self, '%#x' % (fClear,))
+            assert self.isAndOrPair(sValue) is True;
+            return (aoClear[0], aoSet[0]);
+        assert self.isAndOrPair(sValue) is False;
+        return aoSet;
+
+    def isAndOrPair(self, sValue):
+        for sZeroFlag in self.kdZeroValueFlags:
+            if sValue.find(sZeroFlag) >= 0:
+                return True;
+        return False;
+
+class TestTypeFromDict(TestType):
+    """
+    Special value parsing for CR0.
+    """
+
+    kdZeroValueFlags = { 'nv': 0, 'pl': 0, 'nz': 0, 'na': 0, 'pe': 0, 'nc': 0, 'di': 0, 'up': 0 };
+
+    def __init__(self, sName, kdConstantsAndValues, sConstantPrefix):
+        TestType.__init__(self, sName, acbSizes = [1, 2, 4, 8], fUnsigned = True);
+        self.kdConstantsAndValues = kdConstantsAndValues;
+        self.sConstantPrefix      = sConstantPrefix;
+
+    def get(self, sValue):
+        fValue = 0;
+        for sFlag in sValue.split(','):
+            fFlagValue = self.kdConstantsAndValues.get(self.sConstantPrefix + sFlag.upper(), None);
+            if fFlagValue is None:
+                raise self.BadValue('Unknown flag "%s" in "%s"' % (sFlag, sValue))
+            fValue |= fFlagValue;
+        return TestType.get(self, '0x%x' % (fValue,));
+
+
+class TestInOut(object):
+    """
+    One input or output state modifier.
+
+    This should be thought as values to modify BS3REGCTX and extended (needs
+    to be structured) state.
+    """
+    ## Assigned operators.
+    kasOperators = [
+        '&|=',  # Special AND(INV)+OR operator for use with EFLAGS.
+        '&~=',
+        '&=',
+        '|=',
+        '='
+    ];
+    ## Types
+    kdTypes = {
+        'uint':  TestType('uint', fUnsigned = True),
+        'int':   TestType('int'),
+        'efl':   TestTypeEflags('efl'),
+        'cr0':   TestTypeFromDict('cr0', g_kdX86Cr0Constants, 'X86_CR0_'),
+        'cr4':   TestTypeFromDict('cr4', g_kdX86Cr4Constants, 'X86_CR4_'),
+        'xcr0':  TestTypeFromDict('xcr0', g_kdX86XSaveCConstants, 'XSAVE_C_'),
+    };
+    ## CPU context fields.
+    kdFields = {
+        # name:         ( default type, [both|input|output], )
+        # Operands.
+        'op1':          ( 'uint', 'both',   ), ## \@op1
+        'op2':          ( 'uint', 'both',   ), ## \@op2
+        'op3':          ( 'uint', 'both',   ), ## \@op3
+        'op4':          ( 'uint', 'both',   ), ## \@op4
+        # Flags.
+        'efl':          ( 'efl',  'both',   ),
+        'efl_undef':    ( 'uint', 'output', ),
+        # 8-bit GPRs.
+        'al':           ( 'uint', 'both',   ),
+        'cl':           ( 'uint', 'both',   ),
+        'dl':           ( 'uint', 'both',   ),
+        'bl':           ( 'uint', 'both',   ),
+        'ah':           ( 'uint', 'both',   ),
+        'ch':           ( 'uint', 'both',   ),
+        'dh':           ( 'uint', 'both',   ),
+        'bh':           ( 'uint', 'both',   ),
+        'r8l':          ( 'uint', 'both',   ),
+        'r9l':          ( 'uint', 'both',   ),
+        'r10l':         ( 'uint', 'both',   ),
+        'r11l':         ( 'uint', 'both',   ),
+        'r12l':         ( 'uint', 'both',   ),
+        'r13l':         ( 'uint', 'both',   ),
+        'r14l':         ( 'uint', 'both',   ),
+        'r15l':         ( 'uint', 'both',   ),
+        # 16-bit GPRs.
+        'ax':           ( 'uint', 'both',   ),
+        'dx':           ( 'uint', 'both',   ),
+        'cx':           ( 'uint', 'both',   ),
+        'bx':           ( 'uint', 'both',   ),
+        'sp':           ( 'uint', 'both',   ),
+        'bp':           ( 'uint', 'both',   ),
+        'si':           ( 'uint', 'both',   ),
+        'di':           ( 'uint', 'both',   ),
+        'r8w':          ( 'uint', 'both',   ),
+        'r9w':          ( 'uint', 'both',   ),
+        'r10w':         ( 'uint', 'both',   ),
+        'r11w':         ( 'uint', 'both',   ),
+        'r12w':         ( 'uint', 'both',   ),
+        'r13w':         ( 'uint', 'both',   ),
+        'r14w':         ( 'uint', 'both',   ),
+        'r15w':         ( 'uint', 'both',   ),
+        # 32-bit GPRs.
+        'eax':          ( 'uint', 'both',   ),
+        'edx':          ( 'uint', 'both',   ),
+        'ecx':          ( 'uint', 'both',   ),
+        'ebx':          ( 'uint', 'both',   ),
+        'esp':          ( 'uint', 'both',   ),
+        'ebp':          ( 'uint', 'both',   ),
+        'esi':          ( 'uint', 'both',   ),
+        'edi':          ( 'uint', 'both',   ),
+        'r8d':          ( 'uint', 'both',   ),
+        'r9d':          ( 'uint', 'both',   ),
+        'r10d':         ( 'uint', 'both',   ),
+        'r11d':         ( 'uint', 'both',   ),
+        'r12d':         ( 'uint', 'both',   ),
+        'r13d':         ( 'uint', 'both',   ),
+        'r14d':         ( 'uint', 'both',   ),
+        'r15d':         ( 'uint', 'both',   ),
+        # 64-bit GPRs.
+        'rax':          ( 'uint', 'both',   ),
+        'rdx':          ( 'uint', 'both',   ),
+        'rcx':          ( 'uint', 'both',   ),
+        'rbx':          ( 'uint', 'both',   ),
+        'rsp':          ( 'uint', 'both',   ),
+        'rbp':          ( 'uint', 'both',   ),
+        'rsi':          ( 'uint', 'both',   ),
+        'rdi':          ( 'uint', 'both',   ),
+        'r8':           ( 'uint', 'both',   ),
+        'r9':           ( 'uint', 'both',   ),
+        'r10':          ( 'uint', 'both',   ),
+        'r11':          ( 'uint', 'both',   ),
+        'r12':          ( 'uint', 'both',   ),
+        'r13':          ( 'uint', 'both',   ),
+        'r14':          ( 'uint', 'both',   ),
+        'r15':          ( 'uint', 'both',   ),
+        # 16-bit, 32-bit or 64-bit registers according to operand size.
+        'oz.rax':       ( 'uint', 'both',   ),
+        'oz.rdx':       ( 'uint', 'both',   ),
+        'oz.rcx':       ( 'uint', 'both',   ),
+        'oz.rbx':       ( 'uint', 'both',   ),
+        'oz.rsp':       ( 'uint', 'both',   ),
+        'oz.rbp':       ( 'uint', 'both',   ),
+        'oz.rsi':       ( 'uint', 'both',   ),
+        'oz.rdi':       ( 'uint', 'both',   ),
+        'oz.r8':        ( 'uint', 'both',   ),
+        'oz.r9':        ( 'uint', 'both',   ),
+        'oz.r10':       ( 'uint', 'both',   ),
+        'oz.r11':       ( 'uint', 'both',   ),
+        'oz.r12':       ( 'uint', 'both',   ),
+        'oz.r13':       ( 'uint', 'both',   ),
+        'oz.r14':       ( 'uint', 'both',   ),
+        'oz.r15':       ( 'uint', 'both',   ),
+        # Control registers.
+        'cr0':          ( 'cr0',  'both',   ),
+        'cr4':          ( 'cr4',  'both',   ),
+        'xcr0':         ( 'xcr0', 'both',   ),
+        # FPU Registers
+        'fcw':          ( 'uint', 'both',   ),
+        'fsw':          ( 'uint', 'both',   ),
+        'ftw':          ( 'uint', 'both',   ),
+        'fop':          ( 'uint', 'both',   ),
+        'fpuip':        ( 'uint', 'both',   ),
+        'fpucs':        ( 'uint', 'both',   ),
+        'fpudp':        ( 'uint', 'both',   ),
+        'fpuds':        ( 'uint', 'both',   ),
+        'mxcsr':        ( 'uint', 'both',   ),
+        'st0':          ( 'uint', 'both',   ),
+        'st1':          ( 'uint', 'both',   ),
+        'st2':          ( 'uint', 'both',   ),
+        'st3':          ( 'uint', 'both',   ),
+        'st4':          ( 'uint', 'both',   ),
+        'st5':          ( 'uint', 'both',   ),
+        'st6':          ( 'uint', 'both',   ),
+        'st7':          ( 'uint', 'both',   ),
+        # MMX registers.
+        'mm0':          ( 'uint', 'both',   ),
+        'mm1':          ( 'uint', 'both',   ),
+        'mm2':          ( 'uint', 'both',   ),
+        'mm3':          ( 'uint', 'both',   ),
+        'mm4':          ( 'uint', 'both',   ),
+        'mm5':          ( 'uint', 'both',   ),
+        'mm6':          ( 'uint', 'both',   ),
+        'mm7':          ( 'uint', 'both',   ),
+        # SSE registers.
+        'xmm0':         ( 'uint', 'both',   ),
+        'xmm1':         ( 'uint', 'both',   ),
+        'xmm2':         ( 'uint', 'both',   ),
+        'xmm3':         ( 'uint', 'both',   ),
+        'xmm4':         ( 'uint', 'both',   ),
+        'xmm5':         ( 'uint', 'both',   ),
+        'xmm6':         ( 'uint', 'both',   ),
+        'xmm7':         ( 'uint', 'both',   ),
+        'xmm8':         ( 'uint', 'both',   ),
+        'xmm9':         ( 'uint', 'both',   ),
+        'xmm10':        ( 'uint', 'both',   ),
+        'xmm11':        ( 'uint', 'both',   ),
+        'xmm12':        ( 'uint', 'both',   ),
+        'xmm13':        ( 'uint', 'both',   ),
+        'xmm14':        ( 'uint', 'both',   ),
+        'xmm15':        ( 'uint', 'both',   ),
+        'xmm0.lo':      ( 'uint', 'both',   ),
+        'xmm1.lo':      ( 'uint', 'both',   ),
+        'xmm2.lo':      ( 'uint', 'both',   ),
+        'xmm3.lo':      ( 'uint', 'both',   ),
+        'xmm4.lo':      ( 'uint', 'both',   ),
+        'xmm5.lo':      ( 'uint', 'both',   ),
+        'xmm6.lo':      ( 'uint', 'both',   ),
+        'xmm7.lo':      ( 'uint', 'both',   ),
+        'xmm8.lo':      ( 'uint', 'both',   ),
+        'xmm9.lo':      ( 'uint', 'both',   ),
+        'xmm10.lo':     ( 'uint', 'both',   ),
+        'xmm11.lo':     ( 'uint', 'both',   ),
+        'xmm12.lo':     ( 'uint', 'both',   ),
+        'xmm13.lo':     ( 'uint', 'both',   ),
+        'xmm14.lo':     ( 'uint', 'both',   ),
+        'xmm15.lo':     ( 'uint', 'both',   ),
+        'xmm0.hi':      ( 'uint', 'both',   ),
+        'xmm1.hi':      ( 'uint', 'both',   ),
+        'xmm2.hi':      ( 'uint', 'both',   ),
+        'xmm3.hi':      ( 'uint', 'both',   ),
+        'xmm4.hi':      ( 'uint', 'both',   ),
+        'xmm5.hi':      ( 'uint', 'both',   ),
+        'xmm6.hi':      ( 'uint', 'both',   ),
+        'xmm7.hi':      ( 'uint', 'both',   ),
+        'xmm8.hi':      ( 'uint', 'both',   ),
+        'xmm9.hi':      ( 'uint', 'both',   ),
+        'xmm10.hi':     ( 'uint', 'both',   ),
+        'xmm11.hi':     ( 'uint', 'both',   ),
+        'xmm12.hi':     ( 'uint', 'both',   ),
+        'xmm13.hi':     ( 'uint', 'both',   ),
+        'xmm14.hi':     ( 'uint', 'both',   ),
+        'xmm15.hi':     ( 'uint', 'both',   ),
+        'xmm0.lo.zx':   ( 'uint', 'both',   ),
+        'xmm1.lo.zx':   ( 'uint', 'both',   ),
+        'xmm2.lo.zx':   ( 'uint', 'both',   ),
+        'xmm3.lo.zx':   ( 'uint', 'both',   ),
+        'xmm4.lo.zx':   ( 'uint', 'both',   ),
+        'xmm5.lo.zx':   ( 'uint', 'both',   ),
+        'xmm6.lo.zx':   ( 'uint', 'both',   ),
+        'xmm7.lo.zx':   ( 'uint', 'both',   ),
+        'xmm8.lo.zx':   ( 'uint', 'both',   ),
+        'xmm9.lo.zx':   ( 'uint', 'both',   ),
+        'xmm10.lo.zx':  ( 'uint', 'both',   ),
+        'xmm11.lo.zx':  ( 'uint', 'both',   ),
+        'xmm12.lo.zx':  ( 'uint', 'both',   ),
+        'xmm13.lo.zx':  ( 'uint', 'both',   ),
+        'xmm14.lo.zx':  ( 'uint', 'both',   ),
+        'xmm15.lo.zx':  ( 'uint', 'both',   ),
+        'xmm0.dw0':     ( 'uint', 'both',   ),
+        'xmm1.dw0':     ( 'uint', 'both',   ),
+        'xmm2.dw0':     ( 'uint', 'both',   ),
+        'xmm3.dw0':     ( 'uint', 'both',   ),
+        'xmm4.dw0':     ( 'uint', 'both',   ),
+        'xmm5.dw0':     ( 'uint', 'both',   ),
+        'xmm6.dw0':     ( 'uint', 'both',   ),
+        'xmm7.dw0':     ( 'uint', 'both',   ),
+        'xmm8.dw0':     ( 'uint', 'both',   ),
+        'xmm9.dw0':     ( 'uint', 'both',   ),
+        'xmm10.dw0':    ( 'uint', 'both',   ),
+        'xmm11.dw0':    ( 'uint', 'both',   ),
+        'xmm12.dw0':    ( 'uint', 'both',   ),
+        'xmm13.dw0':    ( 'uint', 'both',   ),
+        'xmm14.dw0':    ( 'uint', 'both',   ),
+        'xmm15_dw0':    ( 'uint', 'both',   ),
+        # AVX registers.
+        'ymm0':         ( 'uint', 'both',   ),
+        'ymm1':         ( 'uint', 'both',   ),
+        'ymm2':         ( 'uint', 'both',   ),
+        'ymm3':         ( 'uint', 'both',   ),
+        'ymm4':         ( 'uint', 'both',   ),
+        'ymm5':         ( 'uint', 'both',   ),
+        'ymm6':         ( 'uint', 'both',   ),
+        'ymm7':         ( 'uint', 'both',   ),
+        'ymm8':         ( 'uint', 'both',   ),
+        'ymm9':         ( 'uint', 'both',   ),
+        'ymm10':        ( 'uint', 'both',   ),
+        'ymm11':        ( 'uint', 'both',   ),
+        'ymm12':        ( 'uint', 'both',   ),
+        'ymm13':        ( 'uint', 'both',   ),
+        'ymm14':        ( 'uint', 'both',   ),
+        'ymm15':        ( 'uint', 'both',   ),
+
+        # Special ones.
+        'value.xcpt':   ( 'uint', 'output', ),
+    };
+
+    def __init__(self, sField, sOp, sValue, sType):
+        assert sField in self.kdFields;
+        assert sOp in self.kasOperators;
+        self.sField = sField;
+        self.sOp    = sOp;
+        self.sValue = sValue;
+        self.sType  = sType;
+        assert isinstance(sField, str);
+        assert isinstance(sOp, str);
+        assert isinstance(sType, str);
+        assert isinstance(sValue, str);
+
+
+class TestSelector(object):
+    """
+    One selector for an instruction test.
+    """
+    ## Selector compare operators.
+    kasCompareOps = [ '==', '!=' ];
+    ## Selector variables and their valid values.
+    kdVariables = {
+        # Operand size.
+        'size': {
+            'o16':  'size_o16',
+            'o32':  'size_o32',
+            'o64':  'size_o64',
+        },
+        # VEX.L value.
+        'vex.l': {
+            '0':    'vexl_0',
+            '1':    'vexl_1',
+        },
+        # Execution ring.
+        'ring': {
+            '0':    'ring_0',
+            '1':    'ring_1',
+            '2':    'ring_2',
+            '3':    'ring_3',
+            '0..2': 'ring_0_thru_2',
+            '1..3': 'ring_1_thru_3',
+        },
+        # Basic code mode.
+        'codebits': {
+            '64':   'code_64bit',
+            '32':   'code_32bit',
+            '16':   'code_16bit',
+        },
+        # cpu modes.
+        'mode': {
+            'real': 'mode_real',
+            'prot': 'mode_prot',
+            'long': 'mode_long',
+            'v86':  'mode_v86',
+            'smm':  'mode_smm',
+            'vmx':  'mode_vmx',
+            'svm':  'mode_svm',
+        },
+        # paging on/off
+        'paging': {
+            'on':       'paging_on',
+            'off':      'paging_off',
+        },
+        # CPU vendor
+        'vendor': {
+            'amd':      'vendor_amd',
+            'intel':    'vendor_intel',
+            'via':      'vendor_via',
+        },
+    };
+    ## Selector shorthand predicates.
+    ## These translates into variable expressions.
+    kdPredicates = {
+        'o16':          'size==o16',
+        'o32':          'size==o32',
+        'o64':          'size==o64',
+        'ring0':        'ring==0',
+        '!ring0':       'ring==1..3',
+        'ring1':        'ring==1',
+        'ring2':        'ring==2',
+        'ring3':        'ring==3',
+        'user':         'ring==3',
+        'supervisor':   'ring==0..2',
+        '16-bit':       'codebits==16',
+        '32-bit':       'codebits==32',
+        '64-bit':       'codebits==64',
+        'real':         'mode==real',
+        'prot':         'mode==prot',
+        'long':         'mode==long',
+        'v86':          'mode==v86',
+        'smm':          'mode==smm',
+        'vmx':          'mode==vmx',
+        'svm':          'mode==svm',
+        'paging':       'paging==on',
+        '!paging':      'paging==off',
+        'amd':          'vendor==amd',
+        '!amd':         'vendor!=amd',
+        'intel':        'vendor==intel',
+        '!intel':       'vendor!=intel',
+        'via':          'vendor==via',
+        '!via':         'vendor!=via',
+    };
+
+    def __init__(self, sVariable, sOp, sValue):
+        assert sVariable in self.kdVariables;
+        assert sOp in self.kasCompareOps;
+        assert sValue in self.kdVariables[sVariable];
+        self.sVariable  = sVariable;
+        self.sOp        = sOp;
+        self.sValue     = sValue;
+
+
+class InstructionTest(object):
+    """
+    Instruction test.
+    """
+
+    def __init__(self, oInstr): # type: (InstructionTest, Instruction)
+        self.oInstr         = oInstr    # type: InstructionTest
+        self.aoInputs       = []        # type: list(TestInOut)
+        self.aoOutputs      = []        # type: list(TestInOut)
+        self.aoSelectors    = []        # type: list(TestSelector)
+
+    def toString(self, fRepr = False):
+        """
+        Converts it to string representation.
+        """
+        asWords = [];
+        if self.aoSelectors:
+            for oSelector in self.aoSelectors:
+                asWords.append('%s%s%s' % (oSelector.sVariable, oSelector.sOp, oSelector.sValue,));
+            asWords.append('/');
+
+        for oModifier in self.aoInputs:
+            asWords.append('%s%s%s:%s' % (oModifier.sField, oModifier.sOp, oModifier.sValue, oModifier.sType,));
+
+        asWords.append('->');
+
+        for oModifier in self.aoOutputs:
+            asWords.append('%s%s%s:%s' % (oModifier.sField, oModifier.sOp, oModifier.sValue, oModifier.sType,));
+
+        if fRepr:
+            return '<' + ' '.join(asWords) + '>';
+        return '  '.join(asWords);
+
+    def __str__(self):
+        """ Provide string represenation. """
+        return self.toString(False);
+
+    def __repr__(self):
+        """ Provide unambigious string representation. """
+        return self.toString(True);
+
+class Operand(object):
+    """
+    Instruction operand.
+    """
+
+    def __init__(self, sWhere, sType):
+        assert sWhere in g_kdOpLocations, sWhere;
+        assert sType  in g_kdOpTypes, sType;
+        self.sWhere = sWhere;           ##< g_kdOpLocations
+        self.sType  = sType;            ##< g_kdOpTypes
+
+    def usesModRM(self):
+        """ Returns True if using some form of ModR/M encoding. """
+        return self.sType[0] in ['E', 'G', 'M'];
+
+
+
+class Instruction(object): # pylint: disable=too-many-instance-attributes
+    """
+    Instruction.
+    """
+
+    def __init__(self, sSrcFile, iLine):
+        ## @name Core attributes.
+        ## @{
+        self.sMnemonic      = None;
+        self.sBrief         = None;
+        self.asDescSections = []        # type: list(str)
+        self.aoMaps         = []        # type: list(InstructionMap)
+        self.aoOperands     = []        # type: list(Operand)
+        self.sPrefix        = None;     ##< Single prefix: None, 'none', 0x66, 0xf3, 0xf2
+        self.sOpcode        = None      # type: str
+        self.sSubOpcode     = None      # type: str
+        self.sEncoding      = None;
+        self.asFlTest       = None;
+        self.asFlModify     = None;
+        self.asFlUndefined  = None;
+        self.asFlSet        = None;
+        self.asFlClear      = None;
+        self.dHints         = {};       ##< Dictionary of instruction hints, flags, whatnot. (Dictionary for speed; dummy value).
+        self.sDisEnum       = None;     ##< OP_XXXX value.  Default is based on the uppercased mnemonic.
+        self.asCpuIds       = [];       ##< The CPUID feature bit names for this instruction. If multiple, assume AND.
+        self.asReqFeatures  = [];       ##< Which features are required to be enabled to run this instruction.
+        self.aoTests        = []        # type: list(InstructionTest)
+        self.sMinCpu        = None;     ##< Indicates the minimum CPU required for the instruction. Not set when oCpuExpr is.
+        self.oCpuExpr       = None;     ##< Some CPU restriction expression...
+        self.sGroup         = None;
+        self.fUnused        = False;    ##< Unused instruction.
+        self.fInvalid       = False;    ##< Invalid instruction (like UD2).
+        self.sInvalidStyle  = None;     ##< Invalid behviour style (g_kdInvalidStyles),
+        self.sXcptType      = None;     ##< Exception type (g_kdXcptTypes).
+        ## @}
+
+        ## @name Implementation attributes.
+        ## @{
+        self.sStats         = None;
+        self.sFunction      = None;
+        self.fStub          = False;
+        self.fUdStub        = False;
+        ## @}
+
+        ## @name Decoding info
+        ## @{
+        self.sSrcFile       = sSrcFile;
+        self.iLineCreated   = iLine;
+        self.iLineCompleted = None;
+        self.cOpTags        = 0;
+        self.iLineFnIemOpMacro  = -1;
+        self.iLineMnemonicMacro = -1;
+        ## @}
+
+        ## @name Intermediate input fields.
+        ## @{
+        self.sRawDisOpNo    = None;
+        self.asRawDisParams = [];
+        self.sRawIemOpFlags = None;
+        self.sRawOldOpcodes = None;
+        self.asCopyTests    = [];
+        ## @}
+
+    def toString(self, fRepr = False):
+        """ Turn object into a string. """
+        aasFields = [];
+
+        aasFields.append(['opcode',    self.sOpcode]);
+        aasFields.append(['mnemonic',  self.sMnemonic]);
+        for iOperand, oOperand in enumerate(self.aoOperands):
+            aasFields.append(['op%u' % (iOperand + 1,), '%s:%s' % (oOperand.sWhere, oOperand.sType,)]);
+        if self.aoMaps:         aasFields.append(['maps', ','.join([oMap.sName for oMap in self.aoMaps])]);
+        aasFields.append(['encoding',  self.sEncoding]);
+        if self.dHints:         aasFields.append(['hints', ','.join(self.dHints.keys())]);
+        aasFields.append(['disenum',   self.sDisEnum]);
+        if self.asCpuIds:       aasFields.append(['cpuid', ','.join(self.asCpuIds)]);
+        aasFields.append(['group',     self.sGroup]);
+        if self.fUnused:        aasFields.append(['unused', 'True']);
+        if self.fInvalid:       aasFields.append(['invalid', 'True']);
+        aasFields.append(['invlstyle', self.sInvalidStyle]);
+        aasFields.append(['fltest',    self.asFlTest]);
+        aasFields.append(['flmodify',  self.asFlModify]);
+        aasFields.append(['flundef',   self.asFlUndefined]);
+        aasFields.append(['flset',     self.asFlSet]);
+        aasFields.append(['flclear',   self.asFlClear]);
+        aasFields.append(['mincpu',    self.sMinCpu]);
+        aasFields.append(['stats',     self.sStats]);
+        aasFields.append(['sFunction', self.sFunction]);
+        if self.fStub:          aasFields.append(['fStub', 'True']);
+        if self.fUdStub:        aasFields.append(['fUdStub', 'True']);
+        if self.cOpTags:        aasFields.append(['optags', str(self.cOpTags)]);
+        if self.iLineFnIemOpMacro  != -1: aasFields.append(['FNIEMOP_XXX', str(self.iLineFnIemOpMacro)]);
+        if self.iLineMnemonicMacro != -1: aasFields.append(['IEMOP_MNEMMONICn', str(self.iLineMnemonicMacro)]);
+
+        sRet = '<' if fRepr else '';
+        for sField, sValue in aasFields:
+            if sValue is not None:
+                if len(sRet) > 1:
+                    sRet += '; ';
+                sRet += '%s=%s' % (sField, sValue,);
+        if fRepr:
+            sRet += '>';
+
+        return sRet;
+
+    def __str__(self):
+        """ Provide string represenation. """
+        return self.toString(False);
+
+    def __repr__(self):
+        """ Provide unambigious string representation. """
+        return self.toString(True);
+
+    def getOpcodeByte(self):
+        """
+        Decodes sOpcode into a byte range integer value.
+        Raises exception if sOpcode is None or invalid.
+        """
+        if self.sOpcode is None:
+            raise Exception('No opcode byte for %s!' % (self,));
+        sOpcode = str(self.sOpcode);    # pylint type confusion workaround.
+
+        # Full hex byte form.
+        if sOpcode[:2] == '0x':
+            return int(sOpcode, 16);
+
+        # The /r form:
+        if len(sOpcode) == 4 and sOpcode.startswith('/') and sOpcode[-1].isdigit():
+            return int(sOpcode[-1:]) << 3;
+
+        # The 11/r form:
+        if len(sOpcode) == 4 and sOpcode.startswith('11/') and sOpcode[-1].isdigit():
+            return (int(sOpcode[-1:]) << 3) | 0xc0;
+
+        # The !11/r form (returns mod=1):
+        ## @todo this doesn't really work...
+        if len(sOpcode) == 5 and sOpcode.startswith('!11/') and sOpcode[-1].isdigit():
+            return (int(sOpcode[-1:]) << 3) | 0x80;
+
+        raise Exception('unsupported opcode byte spec "%s" for %s' % (sOpcode, self,));
+
+    @staticmethod
+    def _flagsToIntegerMask(asFlags):
+        """
+        Returns the integer mask value for asFlags.
+        """
+        uRet = 0;
+        if asFlags:
+            for sFlag in asFlags:
+                sConstant = g_kdEFlagsMnemonics[sFlag];
+                assert sConstant[0] != '!', sConstant
+                uRet |= g_kdX86EFlagsConstants[sConstant];
+        return uRet;
+
+    def getTestedFlagsMask(self):
+        """ Returns asFlTest into a integer mask value """
+        return self._flagsToIntegerMask(self.asFlTest);
+
+    def getModifiedFlagsMask(self):
+        """ Returns asFlModify into a integer mask value """
+        return self._flagsToIntegerMask(self.asFlModify);
+
+    def getUndefinedFlagsMask(self):
+        """ Returns asFlUndefined into a integer mask value """
+        return self._flagsToIntegerMask(self.asFlUndefined);
+
+    def getSetFlagsMask(self):
+        """ Returns asFlSet into a integer mask value """
+        return self._flagsToIntegerMask(self.asFlSet);
+
+    def getClearedFlagsMask(self):
+        """ Returns asFlClear into a integer mask value """
+        return self._flagsToIntegerMask(self.asFlClear);
+
+    def onlyInVexMaps(self):
+        """ Returns True if only in VEX maps, otherwise False.  (No maps -> False) """
+        if not self.aoMaps:
+            return False;
+        for oMap in self.aoMaps:
+            if not oMap.isVexMap():
+                return False;
+        return True;
+
+
+
+## All the instructions.
+g_aoAllInstructions = []            # type: list(Instruction)
+
+## All the instructions indexed by statistics name (opstat).
+g_dAllInstructionsByStat = {}       # type: dict(Instruction)
+
+## All the instructions indexed by function name (opfunction).
+g_dAllInstructionsByFunction = {}   # type: dict(list(Instruction))
+
+## Instructions tagged by oponlytest
+g_aoOnlyTestInstructions = []       # type: list(Instruction)
+
+## Instruction maps.
+g_dInstructionMaps = {
+    'one':          InstructionMap('one'),
+    'grp1_80':      InstructionMap('grp1_80',   asLeadOpcodes = ['0x80',]),
+    'grp1_81':      InstructionMap('grp1_81',   asLeadOpcodes = ['0x81',], sSelector = '/r'),
+    'grp1_82':      InstructionMap('grp1_82',   asLeadOpcodes = ['0x82',], sSelector = '/r'),
+    'grp1_83':      InstructionMap('grp1_83',   asLeadOpcodes = ['0x83',], sSelector = '/r'),
+    'grp1a':        InstructionMap('grp1a',     asLeadOpcodes = ['0x8f',], sSelector = '/r'),
+    'grp2_c0':      InstructionMap('grp2_c0',   asLeadOpcodes = ['0xc0',], sSelector = '/r'),
+    'grp2_c1':      InstructionMap('grp2_c1',   asLeadOpcodes = ['0xc1',], sSelector = '/r'),
+    'grp2_d0':      InstructionMap('grp2_d0',   asLeadOpcodes = ['0xd0',], sSelector = '/r'),
+    'grp2_d1':      InstructionMap('grp2_d1',   asLeadOpcodes = ['0xd1',], sSelector = '/r'),
+    'grp2_d2':      InstructionMap('grp2_d2',   asLeadOpcodes = ['0xd2',], sSelector = '/r'),
+    'grp2_d3':      InstructionMap('grp2_d3',   asLeadOpcodes = ['0xd3',], sSelector = '/r'),
+    'grp3_f6':      InstructionMap('grp3_f6',   asLeadOpcodes = ['0xf6',], sSelector = '/r'),
+    'grp3_f7':      InstructionMap('grp3_f7',   asLeadOpcodes = ['0xf7',], sSelector = '/r'),
+    'grp4':         InstructionMap('grp4',      asLeadOpcodes = ['0xfe',], sSelector = '/r'),
+    'grp5':         InstructionMap('grp5',      asLeadOpcodes = ['0xff',], sSelector = '/r'),
+    'grp11_c6_m':   InstructionMap('grp11_c6_m',asLeadOpcodes = ['0xc6',], sSelector = '!11 /r'),
+    'grp11_c6_r':   InstructionMap('grp11_c6_r',asLeadOpcodes = ['0xc6',], sSelector = '11'),    # xabort
+    'grp11_c7_m':   InstructionMap('grp11_c7_m',asLeadOpcodes = ['0xc7',], sSelector = '!11 /r'),
+    'grp11_c7_r':   InstructionMap('grp11_c7_r',asLeadOpcodes = ['0xc7',], sSelector = '11'),    # xbegin
+
+    'two0f':        InstructionMap('two0f',     asLeadOpcodes = ['0x0f',], sDisParse = 'IDX_ParseTwoByteEsc'),
+    'grp6':         InstructionMap('grp6',      asLeadOpcodes = ['0x0f', '0x00',], sSelector = '/r'),
+    'grp7_m':       InstructionMap('grp7_m',    asLeadOpcodes = ['0x0f', '0x01',], sSelector = '!11 /r'),
+    'grp7_r':       InstructionMap('grp7_r',    asLeadOpcodes = ['0x0f', '0x01',], sSelector = '11'),
+    'grp8':         InstructionMap('grp8',      asLeadOpcodes = ['0x0f', '0xba',], sSelector = '/r'),
+    'grp9':         InstructionMap('grp9',      asLeadOpcodes = ['0x0f', '0xc7',], sSelector = 'mod /r'),
+    'grp10':        InstructionMap('grp10',     asLeadOpcodes = ['0x0f', '0xb9',], sSelector = '/r'), # UD1 /w modr/m
+    'grp12':        InstructionMap('grp12',     asLeadOpcodes = ['0x0f', '0x71',], sSelector = 'mod /r'),
+    'grp13':        InstructionMap('grp13',     asLeadOpcodes = ['0x0f', '0x72',], sSelector = 'mod /r'),
+    'grp14':        InstructionMap('grp14',     asLeadOpcodes = ['0x0f', '0x73',], sSelector = 'mod /r'),
+    'grp15':        InstructionMap('grp15',     asLeadOpcodes = ['0x0f', '0xae',], sSelector = 'memreg /r'),
+    'grp16':        InstructionMap('grp16',     asLeadOpcodes = ['0x0f', '0x18',], sSelector = 'mod /r'),
+    'grpA17':       InstructionMap('grpA17',    asLeadOpcodes = ['0x0f', '0x78',], sSelector = '/r'), # AMD: EXTRQ weirdness
+    'grpP':         InstructionMap('grpP',      asLeadOpcodes = ['0x0f', '0x0d',], sSelector = '/r'), # AMD: prefetch
+
+    'three0f38':    InstructionMap('three0f38', asLeadOpcodes = ['0x0f', '0x38',]),
+    'three0f3a':    InstructionMap('three0f3a', asLeadOpcodes = ['0x0f', '0x3a',]),
+
+    'vexmap1':      InstructionMap('vexmap1',   sEncoding = 'vex1'),
+    'vexgrp12':     InstructionMap('vexgrp12',  sEncoding = 'vex1', asLeadOpcodes = ['0x71',], sSelector = 'mod /r'),
+    'vexgrp13':     InstructionMap('vexgrp13',  sEncoding = 'vex1', asLeadOpcodes = ['0x72',], sSelector = 'mod /r'),
+    'vexgrp14':     InstructionMap('vexgrp14',  sEncoding = 'vex1', asLeadOpcodes = ['0x73',], sSelector = 'mod /r'),
+    'vexgrp15':     InstructionMap('vexgrp15',  sEncoding = 'vex1', asLeadOpcodes = ['0xae',], sSelector = 'memreg /r'),
+    'vexgrp17':     InstructionMap('vexgrp17',  sEncoding = 'vex1', asLeadOpcodes = ['0xf3',], sSelector = '/r'),
+
+    'vexmap2':      InstructionMap('vexmap2',   sEncoding = 'vex2'),
+    'vexmap3':      InstructionMap('vexmap3',   sEncoding = 'vex3'),
+
+    '3dnow':        InstructionMap('3dnow',     asLeadOpcodes = ['0x0f', '0x0f',]),
+    'xopmap8':      InstructionMap('xopmap8',   sEncoding = 'xop8'),
+    'xopmap9':      InstructionMap('xopmap9',   sEncoding = 'xop9'),
+    'xopgrp1':      InstructionMap('xopgrp1',   sEncoding = 'xop9',  asLeadOpcodes = ['0x01'], sSelector = '/r'),
+    'xopgrp2':      InstructionMap('xopgrp2',   sEncoding = 'xop9',  asLeadOpcodes = ['0x02'], sSelector = '/r'),
+    'xopgrp3':      InstructionMap('xopgrp3',   sEncoding = 'xop9',  asLeadOpcodes = ['0x12'], sSelector = '/r'),
+    'xopmap10':     InstructionMap('xopmap10',  sEncoding = 'xop10'),
+    'xopgrp4':      InstructionMap('xopgrp4',   sEncoding = 'xop10', asLeadOpcodes = ['0x12'], sSelector = '/r'),
+};
+
+
+
+class ParserException(Exception):
+    """ Parser exception """
+    def __init__(self, sMessage):
+        Exception.__init__(self, sMessage);
+
+
+class SimpleParser(object):
+    """
+    Parser of IEMAllInstruction*.cpp.h instruction specifications.
+    """
+
+    ## @name Parser state.
+    ## @{
+    kiCode              = 0;
+    kiCommentMulti      = 1;
+    ## @}
+
+    def __init__(self, sSrcFile, asLines, sDefaultMap):
+        self.sSrcFile       = sSrcFile;
+        self.asLines        = asLines;
+        self.iLine          = 0;
+        self.iState         = self.kiCode;
+        self.sComment       = '';
+        self.iCommentLine   = 0;
+        self.aoCurInstrs    = [];
+
+        assert sDefaultMap in g_dInstructionMaps;
+        self.oDefaultMap    = g_dInstructionMaps[sDefaultMap];
+
+        self.cTotalInstr    = 0;
+        self.cTotalStubs    = 0;
+        self.cTotalTagged   = 0;
+
+        self.oReMacroName   = re.compile('^[A-Za-z_][A-Za-z0-9_]*$');
+        self.oReMnemonic    = re.compile('^[A-Za-z_][A-Za-z0-9_]*$');
+        self.oReStatsName   = re.compile('^[A-Za-z_][A-Za-z0-9_]*$');
+        self.oReFunctionName= re.compile('^iemOp_[A-Za-z_][A-Za-z0-9_]*$');
+        self.oReGroupName   = re.compile('^og_[a-z0-9]+(|_[a-z0-9]+|_[a-z0-9]+_[a-z0-9]+)$');
+        self.oReDisEnum     = re.compile('^OP_[A-Z0-9_]+$');
+        self.fDebug         = True;
+
+        self.dTagHandlers   = {
+            '@opbrief':     self.parseTagOpBrief,
+            '@opdesc':      self.parseTagOpDesc,
+            '@opmnemonic':  self.parseTagOpMnemonic,
+            '@op1':         self.parseTagOpOperandN,
+            '@op2':         self.parseTagOpOperandN,
+            '@op3':         self.parseTagOpOperandN,
+            '@op4':         self.parseTagOpOperandN,
+            '@oppfx':       self.parseTagOpPfx,
+            '@opmaps':      self.parseTagOpMaps,
+            '@opcode':      self.parseTagOpcode,
+            '@opcodesub':   self.parseTagOpcodeSub,
+            '@openc':       self.parseTagOpEnc,
+            '@opfltest':    self.parseTagOpEFlags,
+            '@opflmodify':  self.parseTagOpEFlags,
+            '@opflundef':   self.parseTagOpEFlags,
+            '@opflset':     self.parseTagOpEFlags,
+            '@opflclear':   self.parseTagOpEFlags,
+            '@ophints':     self.parseTagOpHints,
+            '@opdisenum':   self.parseTagOpDisEnum,
+            '@opmincpu':    self.parseTagOpMinCpu,
+            '@opcpuid':     self.parseTagOpCpuId,
+            '@opgroup':     self.parseTagOpGroup,
+            '@opunused':    self.parseTagOpUnusedInvalid,
+            '@opinvalid':   self.parseTagOpUnusedInvalid,
+            '@opinvlstyle': self.parseTagOpUnusedInvalid,
+            '@optest':      self.parseTagOpTest,
+            '@optestign':   self.parseTagOpTestIgnore,
+            '@optestignore': self.parseTagOpTestIgnore,
+            '@opcopytests': self.parseTagOpCopyTests,
+            '@oponly':      self.parseTagOpOnlyTest,
+            '@oponlytest':  self.parseTagOpOnlyTest,
+            '@opxcpttype':  self.parseTagOpXcptType,
+            '@opstats':     self.parseTagOpStats,
+            '@opfunction':  self.parseTagOpFunction,
+            '@opdone':      self.parseTagOpDone,
+        };
+        for i in range(48):
+            self.dTagHandlers['@optest%u' % (i,)]   = self.parseTagOpTestNum;
+            self.dTagHandlers['@optest[%u]' % (i,)] = self.parseTagOpTestNum;
+
+        self.asErrors = [];
+
+    def raiseError(self, sMessage):
+        """
+        Raise error prefixed with the source and line number.
+        """
+        raise ParserException("%s:%d: error: %s" % (self.sSrcFile, self.iLine, sMessage,));
+
+    def raiseCommentError(self, iLineInComment, sMessage):
+        """
+        Similar to raiseError, but the line number is iLineInComment + self.iCommentLine.
+        """
+        raise ParserException("%s:%d: error: %s" % (self.sSrcFile, self.iCommentLine + iLineInComment, sMessage,));
+
+    def error(self, sMessage):
+        """
+        Adds an error.
+        returns False;
+        """
+        self.asErrors.append(u'%s:%d: error: %s\n' % (self.sSrcFile, self.iLine, sMessage,));
+        return False;
+
+    def errorOnLine(self, iLine, sMessage):
+        """
+        Adds an error.
+        returns False;
+        """
+        self.asErrors.append(u'%s:%d: error: %s\n' % (self.sSrcFile, iLine, sMessage,));
+        return False;
+
+    def errorComment(self, iLineInComment, sMessage):
+        """
+        Adds a comment error.
+        returns False;
+        """
+        self.asErrors.append(u'%s:%d: error: %s\n' % (self.sSrcFile, self.iCommentLine + iLineInComment, sMessage,));
+        return False;
+
+    def printErrors(self):
+        """
+        Print the errors to stderr.
+        Returns number of errors.
+        """
+        if self.asErrors:
+            sys.stderr.write(u''.join(self.asErrors));
+        return len(self.asErrors);
+
+    def debug(self, sMessage):
+        """
+        For debugging.
+        """
+        if self.fDebug:
+            print('debug: %s' % (sMessage,));
+
+
+    def addInstruction(self, iLine = None):
+        """
+        Adds an instruction.
+        """
+        oInstr = Instruction(self.sSrcFile, self.iLine if iLine is None else iLine);
+        g_aoAllInstructions.append(oInstr);
+        self.aoCurInstrs.append(oInstr);
+        return oInstr;
+
+    def deriveMnemonicAndOperandsFromStats(self, oInstr, sStats):
+        """
+        Derives the mnemonic and operands from a IEM stats base name like string.
+        """
+        if oInstr.sMnemonic is None:
+            asWords = sStats.split('_');
+            oInstr.sMnemonic = asWords[0].lower();
+            if len(asWords) > 1 and not oInstr.aoOperands:
+                for sType in asWords[1:]:
+                    if sType in g_kdOpTypes:
+                        oInstr.aoOperands.append(Operand(g_kdOpTypes[sType][1], sType));
+                    else:
+                        #return self.error('unknown operand type: %s (instruction: %s)' % (sType, oInstr))
+                        return False;
+        return True;
+
+    def doneInstructionOne(self, oInstr, iLine):
+        """
+        Complete the parsing by processing, validating and expanding raw inputs.
+        """
+        assert oInstr.iLineCompleted is None;
+        oInstr.iLineCompleted = iLine;
+
+        #
+        # Specified instructions.
+        #
+        if oInstr.cOpTags > 0:
+            if oInstr.sStats is None:
+                pass;
+
+        #
+        # Unspecified legacy stuff.  We generally only got a few things to go on here.
+        #   /** Opcode 0x0f 0x00 /0. */
+        #   FNIEMOPRM_DEF(iemOp_Grp6_sldt)
+        #
+        else:
+            #if oInstr.sRawOldOpcodes:
+            #
+            #if oInstr.sMnemonic:
+            pass;
+
+        #
+        # Common defaults.
+        #
+
+        # Guess mnemonic and operands from stats if the former is missing.
+        if oInstr.sMnemonic is None:
+            if oInstr.sStats is not None:
+                self.deriveMnemonicAndOperandsFromStats(oInstr, oInstr.sStats);
+            elif oInstr.sFunction is not None:
+                self.deriveMnemonicAndOperandsFromStats(oInstr, oInstr.sFunction.replace('iemOp_', ''));
+
+        # Derive the disassembler op enum constant from the mnemonic.
+        if oInstr.sDisEnum is None and oInstr.sMnemonic is not None:
+            oInstr.sDisEnum = 'OP_' + oInstr.sMnemonic.upper();
+
+        # Derive the IEM statistics base name from mnemonic and operand types.
+        if oInstr.sStats is None:
+            if oInstr.sFunction is not None:
+                oInstr.sStats = oInstr.sFunction.replace('iemOp_', '');
+            elif oInstr.sMnemonic is not None:
+                oInstr.sStats = oInstr.sMnemonic;
+                for oOperand in oInstr.aoOperands:
+                    if oOperand.sType:
+                        oInstr.sStats += '_' + oOperand.sType;
+
+        # Derive the IEM function name from mnemonic and operand types.
+        if oInstr.sFunction is None:
+            if oInstr.sMnemonic is not None:
+                oInstr.sFunction = 'iemOp_' + oInstr.sMnemonic;
+                for oOperand in oInstr.aoOperands:
+                    if oOperand.sType:
+                        oInstr.sFunction += '_' + oOperand.sType;
+            elif oInstr.sStats:
+                oInstr.sFunction = 'iemOp_' + oInstr.sStats;
+
+        #
+        # Apply default map and then add the instruction to all it's groups.
+        #
+        if not oInstr.aoMaps:
+            oInstr.aoMaps = [ self.oDefaultMap, ];
+        for oMap in oInstr.aoMaps:
+            oMap.aoInstructions.append(oInstr);
+
+        #
+        # Derive encoding from operands and maps.
+        #
+        if oInstr.sEncoding is None:
+            if not oInstr.aoOperands:
+                if oInstr.fUnused and oInstr.sSubOpcode:
+                    oInstr.sEncoding = 'VEX.ModR/M' if oInstr.onlyInVexMaps() else 'ModR/M';
+                else:
+                    oInstr.sEncoding = 'VEX.fixed' if oInstr.onlyInVexMaps() else 'fixed';
+            elif oInstr.aoOperands[0].usesModRM():
+                if     (len(oInstr.aoOperands) >= 2 and oInstr.aoOperands[1].sWhere == 'vvvv') \
+                    or oInstr.onlyInVexMaps():
+                    oInstr.sEncoding = 'VEX.ModR/M';
+                else:
+                    oInstr.sEncoding = 'ModR/M';
+
+        #
+        # Check the opstat value and add it to the opstat indexed dictionary.
+        #
+        if oInstr.sStats:
+            if oInstr.sStats not in g_dAllInstructionsByStat:
+                g_dAllInstructionsByStat[oInstr.sStats] = oInstr;
+            else:
+                self.error('Duplicate opstat value "%s"\nnew: %s\nold: %s'
+                           % (oInstr.sStats, oInstr, g_dAllInstructionsByStat[oInstr.sStats],));
+
+        #
+        # Add to function indexed dictionary.  We allow multiple instructions per function.
+        #
+        if oInstr.sFunction:
+            if oInstr.sFunction not in g_dAllInstructionsByFunction:
+                g_dAllInstructionsByFunction[oInstr.sFunction] = [oInstr,];
+            else:
+                g_dAllInstructionsByFunction[oInstr.sFunction].append(oInstr);
+
+        #self.debug('%d..%d: %s; %d @op tags' % (oInstr.iLineCreated, oInstr.iLineCompleted, oInstr.sFunction, oInstr.cOpTags));
+        return True;
+
+    def doneInstructions(self, iLineInComment = None):
+        """
+        Done with current instruction.
+        """
+        for oInstr in self.aoCurInstrs:
+            self.doneInstructionOne(oInstr, self.iLine if iLineInComment is None else self.iCommentLine + iLineInComment);
+            if oInstr.fStub:
+                self.cTotalStubs += 1;
+
+        self.cTotalInstr += len(self.aoCurInstrs);
+
+        self.sComment     = '';
+        self.aoCurInstrs   = [];
+        return True;
+
+    def setInstrunctionAttrib(self, sAttrib, oValue, fOverwrite = False):
+        """
+        Sets the sAttrib of all current instruction to oValue.  If fOverwrite
+        is False, only None values and empty strings are replaced.
+        """
+        for oInstr in self.aoCurInstrs:
+            if fOverwrite is not True:
+                oOldValue = getattr(oInstr, sAttrib);
+                if oOldValue is not None:
+                    continue;
+            setattr(oInstr, sAttrib, oValue);
+
+    def setInstrunctionArrayAttrib(self, sAttrib, iEntry, oValue, fOverwrite = False):
+        """
+        Sets the iEntry of the array sAttrib of all current instruction to oValue.
+        If fOverwrite is False, only None values and empty strings are replaced.
+        """
+        for oInstr in self.aoCurInstrs:
+            aoArray = getattr(oInstr, sAttrib);
+            while len(aoArray) <= iEntry:
+                aoArray.append(None);
+            if fOverwrite is True or aoArray[iEntry] is None:
+                aoArray[iEntry] = oValue;
+
+    def parseCommentOldOpcode(self, asLines):
+        """ Deals with 'Opcode 0xff /4' like comments """
+        asWords = asLines[0].split();
+        if    len(asWords) >= 2  \
+          and asWords[0] == 'Opcode'  \
+          and (   asWords[1].startswith('0x')
+               or asWords[1].startswith('0X')):
+            asWords = asWords[:1];
+            for iWord, sWord in enumerate(asWords):
+                if sWord.startswith('0X'):
+                    sWord = '0x' + sWord[:2];
+                    asWords[iWord] = asWords;
+            self.setInstrunctionAttrib('sRawOldOpcodes', ' '.join(asWords));
+
+        return False;
+
+    def ensureInstructionForOpTag(self, iTagLine):
+        """ Ensure there is an instruction for the op-tag being parsed. """
+        if not self.aoCurInstrs:
+            self.addInstruction(self.iCommentLine + iTagLine);
+        for oInstr in self.aoCurInstrs:
+            oInstr.cOpTags += 1;
+            if oInstr.cOpTags == 1:
+                self.cTotalTagged += 1;
+        return self.aoCurInstrs[-1];
+
+    @staticmethod
+    def flattenSections(aasSections):
+        """
+        Flattens multiline sections into stripped single strings.
+        Returns list of strings, on section per string.
+        """
+        asRet = [];
+        for asLines in aasSections:
+            if asLines:
+                asRet.append(' '.join([sLine.strip() for sLine in asLines]));
+        return asRet;
+
+    @staticmethod
+    def flattenAllSections(aasSections, sLineSep = ' ', sSectionSep = '\n'):
+        """
+        Flattens sections into a simple stripped string with newlines as
+        section breaks.  The final section does not sport a trailing newline.
+        """
+        # Typical: One section with a single line.
+        if len(aasSections) == 1 and len(aasSections[0]) == 1:
+            return aasSections[0][0].strip();
+
+        sRet = '';
+        for iSection, asLines in enumerate(aasSections):
+            if asLines:
+                if iSection > 0:
+                    sRet += sSectionSep;
+                sRet += sLineSep.join([sLine.strip() for sLine in asLines]);
+        return sRet;
+
+
+
+    ## @name Tag parsers
+    ## @{
+
+    def parseTagOpBrief(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:    \@opbrief
+        Value:  Text description, multiple sections, appended.
+
+        Brief description.  If not given, it's the first sentence from @opdesc.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sBrief = self.flattenAllSections(aasSections);
+        if not sBrief:
+            return self.errorComment(iTagLine, '%s: value required' % (sTag,));
+        if sBrief[-1] != '.':
+            sBrief = sBrief + '.';
+        if len(sBrief) > 180:
+            return self.errorComment(iTagLine, '%s: value too long (max 180 chars): %s' % (sTag, sBrief));
+        offDot = sBrief.find('.');
+        while 0 <= offDot < len(sBrief) - 1 and sBrief[offDot + 1] != ' ':
+            offDot = sBrief.find('.', offDot + 1);
+        if offDot >= 0 and offDot != len(sBrief) - 1:
+            return self.errorComment(iTagLine, '%s: only one sentence: %s' % (sTag, sBrief));
+
+        # Update the instruction.
+        if oInstr.sBrief is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite brief "%s" with "%s"'
+                                               % (sTag, oInstr.sBrief, sBrief,));
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpDesc(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:    \@opdesc
+        Value:  Text description, multiple sections, appended.
+
+        It is used to describe instructions.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+        if aasSections:
+            oInstr.asDescSections.extend(self.flattenSections(aasSections));
+            return True;
+
+        _ = sTag; _ = iEndLine;
+        return True;
+
+    def parseTagOpMnemonic(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:    @opmenmonic
+        Value:  mnemonic
+
+        The 'mnemonic' value must be a valid C identifier string.  Because of
+        prefixes, groups and whatnot, there times when the mnemonic isn't that
+        of an actual assembler mnemonic.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sMnemonic = self.flattenAllSections(aasSections);
+        if not self.oReMnemonic.match(sMnemonic):
+            return self.errorComment(iTagLine, '%s: invalid menmonic name: "%s"' % (sTag, sMnemonic,));
+        if oInstr.sMnemonic is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite menmonic "%s" with "%s"'
+                                     % (sTag, oInstr.sMnemonic, sMnemonic,));
+        oInstr.sMnemonic = sMnemonic
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpOperandN(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tags:  \@op1, \@op2, \@op3, \@op4
+        Value: [where:]type
+
+        The 'where' value indicates where the operand is found, like the 'reg'
+        part of the ModR/M encoding. See Instruction.kdOperandLocations for
+        a list.
+
+        The 'type' value indicates the operand type.  These follow the types
+        given in the opcode tables in the CPU reference manuals.
+        See Instruction.kdOperandTypes for a list.
+
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+        idxOp = int(sTag[-1]) - 1;
+        assert 0 <= idxOp < 4;
+
+        # flatten, split up, and validate the "where:type" value.
+        sFlattened = self.flattenAllSections(aasSections);
+        asSplit = sFlattened.split(':');
+        if len(asSplit) == 1:
+            sType  = asSplit[0];
+            sWhere = None;
+        elif len(asSplit) == 2:
+            (sWhere, sType) = asSplit;
+        else:
+            return self.errorComment(iTagLine, 'expected %s value on format "[<where>:]<type>" not "%s"' % (sTag, sFlattened,));
+
+        if sType not in g_kdOpTypes:
+            return self.errorComment(iTagLine, '%s: invalid where value "%s", valid: %s'
+                                               % (sTag, sType, ', '.join(g_kdOpTypes.keys()),));
+        if sWhere is None:
+            sWhere = g_kdOpTypes[sType][1];
+        elif sWhere not in g_kdOpLocations:
+            return self.errorComment(iTagLine, '%s: invalid where value "%s", valid: %s'
+                                               % (sTag, sWhere, ', '.join(g_kdOpLocations.keys()),));
+
+        # Insert the operand, refusing to overwrite an existing one.
+        while idxOp >= len(oInstr.aoOperands):
+            oInstr.aoOperands.append(None);
+        if oInstr.aoOperands[idxOp] is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s:%s" with "%s:%s"'
+                                               % ( sTag, oInstr.aoOperands[idxOp].sWhere, oInstr.aoOperands[idxOp].sType,
+                                                   sWhere, sType,));
+        oInstr.aoOperands[idxOp] = Operand(sWhere, sType);
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpMaps(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:    \@opmaps
+        Value:  map[,map2]
+
+        Indicates which maps the instruction is in.  There is a default map
+        associated with each input file.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten, split up and validate the value.
+        sFlattened = self.flattenAllSections(aasSections, sLineSep = ',', sSectionSep = ',');
+        asMaps = sFlattened.split(',');
+        if not asMaps:
+            return self.errorComment(iTagLine, '%s: value required' % (sTag,));
+        for sMap in asMaps:
+            if sMap not in g_dInstructionMaps:
+                return self.errorComment(iTagLine, '%s: invalid map value: %s  (valid values: %s)'
+                                                   % (sTag, sMap, ', '.join(g_dInstructionMaps.keys()),));
+
+        # Add the maps to the current list.  Throw errors on duplicates.
+        for oMap in oInstr.aoMaps:
+            if oMap.sName in asMaps:
+                return self.errorComment(iTagLine, '%s: duplicate map assignment: %s' % (sTag, oMap.sName));
+
+        for sMap in asMaps:
+            oMap = g_dInstructionMaps[sMap];
+            if oMap not in oInstr.aoMaps:
+                oInstr.aoMaps.append(oMap);
+            else:
+                self.errorComment(iTagLine, '%s: duplicate map assignment (input): %s' % (sTag, sMap));
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpPfx(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@oppfx
+        Value:      n/a|none|0x66|0xf3|0xf2
+
+        Required prefix for the instruction.  (In a (E)VEX context this is the
+        value of the 'pp' field rather than an actual prefix.)
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sFlattened = self.flattenAllSections(aasSections);
+        asPrefixes = sFlattened.split();
+        if len(asPrefixes) > 1:
+            return self.errorComment(iTagLine, '%s: max one prefix: %s' % (sTag, asPrefixes,));
+
+        sPrefix = asPrefixes[0].lower();
+        if sPrefix == 'none':
+            sPrefix = 'none';
+        elif sPrefix == 'n/a':
+            sPrefix = None;
+        else:
+            if len(sPrefix) == 2:
+                sPrefix = '0x' + sPrefix;
+            if not _isValidOpcodeByte(sPrefix):
+                return self.errorComment(iTagLine, '%s: invalid prefix: %s' % (sTag, sPrefix,));
+
+        if sPrefix is not None and sPrefix not in g_kdPrefixes:
+            return self.errorComment(iTagLine, '%s: invalid prefix: %s (valid %s)' % (sTag, sPrefix, g_kdPrefixes,));
+
+        # Set it.
+        if oInstr.sPrefix is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % ( sTag, oInstr.sPrefix, sPrefix,));
+        oInstr.sPrefix = sPrefix;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpcode(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opcode
+        Value:      0x?? | /reg (TODO: | mr/reg | 11 /reg | !11 /reg | 11 mr/reg | !11 mr/reg)
+
+        The opcode byte or sub-byte for the instruction in the context of a map.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sOpcode = self.flattenAllSections(aasSections);
+        if _isValidOpcodeByte(sOpcode):
+            pass;
+        elif len(sOpcode) == 2 and sOpcode.startswith('/') and sOpcode[-1] in '012345678':
+            pass;
+        elif len(sOpcode) == 4 and sOpcode.startswith('11/') and sOpcode[-1] in '012345678':
+            pass;
+        elif len(sOpcode) == 5 and sOpcode.startswith('!11/') and sOpcode[-1] in '012345678':
+            pass;
+        else:
+            return self.errorComment(iTagLine, '%s: invalid opcode: %s' % (sTag, sOpcode,));
+
+        # Set it.
+        if oInstr.sOpcode is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % ( sTag, oInstr.sOpcode, sOpcode,));
+        oInstr.sOpcode = sOpcode;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpcodeSub(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opcodesub
+        Value:      none | 11 mr/reg | !11 mr/reg | rex.w=0 | rex.w=1 | vex.l=0 | vex.l=1
+                    | 11 mr/reg vex.l=0 | 11 mr/reg vex.l=1 | !11 mr/reg vex.l=0 | !11 mr/reg vex.l=1
+
+        This is a simple way of dealing with encodings where the mod=3 and mod!=3
+        represents exactly two different instructions.  The more proper way would
+        be to go via maps with two members, but this is faster.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sSubOpcode = self.flattenAllSections(aasSections);
+        if sSubOpcode not in g_kdSubOpcodes:
+            return self.errorComment(iTagLine, '%s: invalid sub opcode: %s  (valid: 11, !11, none)' % (sTag, sSubOpcode,));
+        sSubOpcode = g_kdSubOpcodes[sSubOpcode][0];
+
+        # Set it.
+        if oInstr.sSubOpcode is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"'
+                                               % ( sTag, oInstr.sSubOpcode, sSubOpcode,));
+        oInstr.sSubOpcode = sSubOpcode;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpEnc(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@openc
+        Value:      ModR/M|fixed|prefix|<map name>
+
+        The instruction operand encoding style.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sEncoding = self.flattenAllSections(aasSections);
+        if sEncoding in g_kdEncodings:
+            pass;
+        elif sEncoding in g_dInstructionMaps:
+            pass;
+        elif not _isValidOpcodeByte(sEncoding):
+            return self.errorComment(iTagLine, '%s: invalid encoding: %s' % (sTag, sEncoding,));
+
+        # Set it.
+        if oInstr.sEncoding is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"'
+                                               % ( sTag, oInstr.sEncoding, sEncoding,));
+        oInstr.sEncoding = sEncoding;
+
+        _ = iEndLine;
+        return True;
+
+    ## EFlags tag to Instruction attribute name.
+    kdOpFlagToAttr = {
+        '@opfltest':    'asFlTest',
+        '@opflmodify':  'asFlModify',
+        '@opflundef':   'asFlUndefined',
+        '@opflset':     'asFlSet',
+        '@opflclear':   'asFlClear',
+    };
+
+    def parseTagOpEFlags(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tags:   \@opfltest, \@opflmodify, \@opflundef, \@opflset, \@opflclear
+        Value:  <eflags specifier>
+
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten, split up and validate the values.
+        asFlags = self.flattenAllSections(aasSections, sLineSep = ',', sSectionSep = ',').split(',');
+        if len(asFlags) == 1 and asFlags[0].lower() == 'none':
+            asFlags = [];
+        else:
+            fRc = True;
+            for iFlag, sFlag in enumerate(asFlags):
+                if sFlag not in g_kdEFlagsMnemonics:
+                    if sFlag.strip() in g_kdEFlagsMnemonics:
+                        asFlags[iFlag] = sFlag.strip();
+                    else:
+                        fRc = self.errorComment(iTagLine, '%s: invalid EFLAGS value: %s' % (sTag, sFlag,));
+            if not fRc:
+                return False;
+
+        # Set them.
+        asOld = getattr(oInstr, self.kdOpFlagToAttr[sTag]);
+        if asOld is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % ( sTag, asOld, asFlags,));
+        setattr(oInstr, self.kdOpFlagToAttr[sTag], asFlags);
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpHints(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@ophints
+        Value:      Comma or space separated list of flags and hints.
+
+        This covers the disassembler flags table and more.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten as a space separated list, split it up and validate the values.
+        asHints = self.flattenAllSections(aasSections, sLineSep = ' ', sSectionSep = ' ').replace(',', ' ').split();
+        if len(asHints) == 1 and asHints[0].lower() == 'none':
+            asHints = [];
+        else:
+            fRc = True;
+            for iHint, sHint in enumerate(asHints):
+                if sHint not in g_kdHints:
+                    if sHint.strip() in g_kdHints:
+                        sHint[iHint] = sHint.strip();
+                    else:
+                        fRc = self.errorComment(iTagLine, '%s: invalid hint value: %s' % (sTag, sHint,));
+            if not fRc:
+                return False;
+
+        # Append them.
+        for sHint in asHints:
+            if sHint not in oInstr.dHints:
+                oInstr.dHints[sHint] = True; # (dummy value, using dictionary for speed)
+            else:
+                self.errorComment(iTagLine, '%s: duplicate hint: %s' % ( sTag, sHint,));
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpDisEnum(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opdisenum
+        Value:      OP_XXXX
+
+        This is for select a specific (legacy) disassembler enum value for the
+        instruction.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and split.
+        asWords = self.flattenAllSections(aasSections).split();
+        if len(asWords) != 1:
+            self.errorComment(iTagLine, '%s: expected exactly one value: %s' % (sTag, asWords,));
+            if not asWords:
+                return False;
+        sDisEnum = asWords[0];
+        if not self.oReDisEnum.match(sDisEnum):
+            return self.errorComment(iTagLine, '%s: invalid disassembler OP_XXXX enum: %s (pattern: %s)'
+                                               % (sTag, sDisEnum, self.oReDisEnum.pattern));
+
+        # Set it.
+        if oInstr.sDisEnum is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % (sTag, oInstr.sDisEnum, sDisEnum,));
+        oInstr.sDisEnum = sDisEnum;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpMinCpu(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opmincpu
+        Value:      <simple CPU name>
+
+        Indicates when this instruction was introduced.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten the value, split into words, make sure there's just one, valid it.
+        asCpus = self.flattenAllSections(aasSections).split();
+        if len(asCpus) > 1:
+            self.errorComment(iTagLine, '%s: exactly one CPU name, please: %s' % (sTag, ' '.join(asCpus),));
+
+        sMinCpu = asCpus[0];
+        if sMinCpu in g_kdCpuNames:
+            oInstr.sMinCpu = sMinCpu;
+        else:
+            return self.errorComment(iTagLine, '%s: invalid CPU name: %s  (names: %s)'
+                                               % (sTag, sMinCpu, ','.join(sorted(g_kdCpuNames)),));
+
+        # Set it.
+        if oInstr.sMinCpu is None:
+            oInstr.sMinCpu = sMinCpu;
+        elif oInstr.sMinCpu != sMinCpu:
+            self.errorComment(iTagLine, '%s: attemting to overwrite "%s" with "%s"' % (sTag, oInstr.sMinCpu, sMinCpu,));
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpCpuId(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opcpuid
+        Value:      none | <CPUID flag specifier>
+
+        CPUID feature bit which is required for the instruction to be present.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten as a space separated list, split it up and validate the values.
+        asCpuIds = self.flattenAllSections(aasSections, sLineSep = ' ', sSectionSep = ' ').replace(',', ' ').split();
+        if len(asCpuIds) == 1 and asCpuIds[0].lower() == 'none':
+            asCpuIds = [];
+        else:
+            fRc = True;
+            for iCpuId, sCpuId in enumerate(asCpuIds):
+                if sCpuId not in g_kdCpuIdFlags:
+                    if sCpuId.strip() in g_kdCpuIdFlags:
+                        sCpuId[iCpuId] = sCpuId.strip();
+                    else:
+                        fRc = self.errorComment(iTagLine, '%s: invalid CPUID value: %s' % (sTag, sCpuId,));
+            if not fRc:
+                return False;
+
+        # Append them.
+        for sCpuId in asCpuIds:
+            if sCpuId not in oInstr.asCpuIds:
+                oInstr.asCpuIds.append(sCpuId);
+            else:
+                self.errorComment(iTagLine, '%s: duplicate CPUID: %s' % ( sTag, sCpuId,));
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpGroup(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opgroup
+        Value:      op_grp1[_subgrp2[_subsubgrp3]]
+
+        Instruction grouping.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten as a space separated list, split it up and validate the values.
+        asGroups = self.flattenAllSections(aasSections).split();
+        if len(asGroups) != 1:
+            return self.errorComment(iTagLine, '%s: exactly one group, please: %s' % (sTag, asGroups,));
+        sGroup = asGroups[0];
+        if not self.oReGroupName.match(sGroup):
+            return self.errorComment(iTagLine, '%s: invalid group name: %s (valid: %s)'
+                                               % (sTag, sGroup, self.oReGroupName.pattern));
+
+        # Set it.
+        if oInstr.sGroup is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % ( sTag, oInstr.sGroup, sGroup,));
+        oInstr.sGroup = sGroup;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpUnusedInvalid(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:    \@opunused, \@opinvalid, \@opinvlstyle
+        Value:  <invalid opcode behaviour style>
+
+        The \@opunused indicates the specification is for a currently unused
+        instruction encoding.
+
+        The \@opinvalid indicates the specification is for an invalid currently
+        instruction encoding (like UD2).
+
+        The \@opinvlstyle just indicates how CPUs decode the instruction when
+        not supported (\@opcpuid, \@opmincpu) or disabled.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten as a space separated list, split it up and validate the values.
+        asStyles = self.flattenAllSections(aasSections).split();
+        if len(asStyles) != 1:
+            return self.errorComment(iTagLine, '%s: exactly one invalid behviour style, please: %s' % (sTag, asStyles,));
+        sStyle = asStyles[0];
+        if sStyle not in g_kdInvalidStyles:
+            return self.errorComment(iTagLine, '%s: invalid invalid behaviour style: %s (valid: %s)'
+                                               % (sTag, sStyle, g_kdInvalidStyles.keys(),));
+        # Set it.
+        if oInstr.sInvalidStyle is not None:
+            return self.errorComment(iTagLine,
+                                     '%s: attempting to overwrite "%s" with "%s" (only one @opunused, @opinvalid, @opinvlstyle)'
+                                     % ( sTag, oInstr.sInvalidStyle, sStyle,));
+        oInstr.sInvalidStyle = sStyle;
+        if sTag == '@opunused':
+            oInstr.fUnused = True;
+        elif sTag == '@opinvalid':
+            oInstr.fInvalid = True;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpTest(self, sTag, aasSections, iTagLine, iEndLine): # pylint: disable=too-many-locals
+        """
+        Tag:        \@optest
+        Value:      [<selectors>[ ]?] <inputs> -> <outputs>
+        Example:    mode==64bit / in1=0xfffffffe:dw in2=1:dw -> out1=0xffffffff:dw outfl=a?,p?
+
+        The main idea here is to generate basic instruction tests.
+
+        The probably simplest way of handling the diverse input, would be to use
+        it to produce size optimized byte code for a simple interpreter that
+        modifies the register input and output states.
+
+        An alternative to the interpreter would be creating multiple tables,
+        but that becomes rather complicated wrt what goes where and then to use
+        them in an efficient manner.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        #
+        # Do it section by section.
+        #
+        for asSectionLines in aasSections:
+            #
+            # Sort the input into outputs, inputs and selector conditions.
+            #
+            sFlatSection = self.flattenAllSections([asSectionLines,]);
+            if not sFlatSection:
+                self.errorComment(iTagLine, '%s: missing value (dbg: aasSections=%s)' % ( sTag, aasSections));
+                continue;
+            oTest = InstructionTest(oInstr);
+
+            asSelectors = [];
+            asInputs    = [];
+            asOutputs   = [];
+            asCur   = asOutputs;
+            fRc     = True;
+            asWords = sFlatSection.split();
+            for iWord in range(len(asWords) - 1, -1, -1):
+                sWord = asWords[iWord];
+                # Check for array switchers.
+                if sWord == '->':
+                    if asCur != asOutputs:
+                        fRc = self.errorComment(iTagLine, '%s: "->" shall only occure once: %s' % (sTag, sFlatSection,));
+                        break;
+                    asCur = asInputs;
+                elif sWord == '/':
+                    if asCur != asInputs:
+                        fRc = self.errorComment(iTagLine, '%s: "/" shall only occure once: %s' % (sTag, sFlatSection,));
+                        break;
+                    asCur = asSelectors;
+                else:
+                    asCur.insert(0, sWord);
+
+            #
+            # Validate and add selectors.
+            #
+            for sCond in asSelectors:
+                sCondExp = TestSelector.kdPredicates.get(sCond, sCond);
+                oSelector = None;
+                for sOp in TestSelector.kasCompareOps:
+                    off = sCondExp.find(sOp);
+                    if off >= 0:
+                        sVariable = sCondExp[:off];
+                        sValue    = sCondExp[off + len(sOp):];
+                        if sVariable in TestSelector.kdVariables:
+                            if sValue in TestSelector.kdVariables[sVariable]:
+                                oSelector = TestSelector(sVariable, sOp, sValue);
+                            else:
+                                self.errorComment(iTagLine, '%s: invalid condition value "%s" in "%s" (valid: %s)'
+                                                             % ( sTag, sValue, sCond,
+                                                                 TestSelector.kdVariables[sVariable].keys(),));
+                        else:
+                            self.errorComment(iTagLine, '%s: invalid condition variable "%s" in "%s" (valid: %s)'
+                                                         % ( sTag, sVariable, sCond, TestSelector.kdVariables.keys(),));
+                        break;
+                if oSelector is not None:
+                    for oExisting in oTest.aoSelectors:
+                        if oExisting.sVariable == oSelector.sVariable:
+                            self.errorComment(iTagLine, '%s: already have a selector for variable "%s" (existing: %s, new: %s)'
+                                                         % ( sTag, oSelector.sVariable, oExisting, oSelector,));
+                    oTest.aoSelectors.append(oSelector);
+                else:
+                    fRc = self.errorComment(iTagLine, '%s: failed to parse selector: %s' % ( sTag, sCond,));
+
+            #
+            # Validate outputs and inputs, adding them to the test as we go along.
+            #
+            for asItems, sDesc, aoDst in [ (asInputs, 'input', oTest.aoInputs), (asOutputs, 'output', oTest.aoOutputs)]:
+                asValidFieldKinds = [ 'both', sDesc, ];
+                for sItem in asItems:
+                    oItem = None;
+                    for sOp in TestInOut.kasOperators:
+                        off = sItem.find(sOp);
+                        if off < 0:
+                            continue;
+                        sField     = sItem[:off];
+                        sValueType = sItem[off + len(sOp):];
+                        if     sField in TestInOut.kdFields \
+                           and TestInOut.kdFields[sField][1] in asValidFieldKinds:
+                            asSplit = sValueType.split(':', 1);
+                            sValue  = asSplit[0];
+                            sType   = asSplit[1] if len(asSplit) > 1 else TestInOut.kdFields[sField][0];
+                            if sType in TestInOut.kdTypes:
+                                oValid = TestInOut.kdTypes[sType].validate(sValue);
+                                if oValid is True:
+                                    if not TestInOut.kdTypes[sType].isAndOrPair(sValue) or sOp == '&|=':
+                                        oItem = TestInOut(sField, sOp, sValue, sType);
+                                    else:
+                                        self.errorComment(iTagLine, '%s: and-or %s value "%s" can only be used with "&|="'
+                                                                    % ( sTag, sDesc, sItem, ));
+                                else:
+                                    self.errorComment(iTagLine, '%s: invalid %s value "%s" in "%s" (type: %s): %s'
+                                                                % ( sTag, sDesc, sValue, sItem, sType, oValid, ));
+                            else:
+                                self.errorComment(iTagLine, '%s: invalid %s type "%s" in "%s" (valid types: %s)'
+                                                             % ( sTag, sDesc, sType, sItem, TestInOut.kdTypes.keys(),));
+                        else:
+                            self.errorComment(iTagLine, '%s: invalid %s field "%s" in "%s"\nvalid fields: %s'
+                                                         % ( sTag, sDesc, sField, sItem,
+                                                             ', '.join([sKey for sKey in TestInOut.kdFields
+                                                                        if TestInOut.kdFields[sKey][1] in asValidFieldKinds]),));
+                        break;
+                    if oItem is not None:
+                        for oExisting in aoDst:
+                            if oExisting.sField == oItem.sField and oExisting.sOp == oItem.sOp:
+                                self.errorComment(iTagLine,
+                                                  '%s: already have a "%s" assignment for field "%s" (existing: %s, new: %s)'
+                                                  % ( sTag, oItem.sOp, oItem.sField, oExisting, oItem,));
+                        aoDst.append(oItem);
+                    else:
+                        fRc = self.errorComment(iTagLine, '%s: failed to parse assignment: %s' % ( sTag, sItem,));
+
+            #
+            # .
+            #
+            if fRc:
+                oInstr.aoTests.append(oTest);
+            else:
+                self.errorComment(iTagLine, '%s: failed to parse test: %s' % (sTag, ' '.join(asWords),));
+                self.errorComment(iTagLine, '%s: asSelectors=%s / asInputs=%s -> asOutputs=%s'
+                                            % (sTag, asSelectors, asInputs, asOutputs,));
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpTestNum(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Numbered \@optest tag.  Either \@optest42 or \@optest[42].
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        iTest = 0;
+        if sTag[-1] == ']':
+            iTest = int(sTag[8:-1]);
+        else:
+            iTest = int(sTag[7:]);
+
+        if iTest != len(oInstr.aoTests):
+            self.errorComment(iTagLine, '%s: incorrect test number: %u, actual %u' % (sTag, iTest, len(oInstr.aoTests),));
+        return self.parseTagOpTest(sTag, aasSections, iTagLine, iEndLine);
+
+    def parseTagOpTestIgnore(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@optestign | \@optestignore
+        Value:      <value is ignored>
+
+        This is a simple trick to ignore a test while debugging another.
+
+        See also \@oponlytest.
+        """
+        _ = sTag; _ = aasSections; _ = iTagLine; _ = iEndLine;
+        return True;
+
+    def parseTagOpCopyTests(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opcopytests
+        Value:      <opstat | function> [..]
+        Example:    \@opcopytests add_Eb_Gb
+
+        Trick to avoid duplicating tests for different encodings of the same
+        operation.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten, validate and append the copy job to the instruction.  We execute
+        # them after parsing all the input so we can handle forward references.
+        asToCopy = self.flattenAllSections(aasSections).split();
+        if not asToCopy:
+            return self.errorComment(iTagLine, '%s: requires at least on reference value' % (sTag,));
+        for sToCopy in asToCopy:
+            if sToCopy not in oInstr.asCopyTests:
+                if self.oReStatsName.match(sToCopy) or self.oReFunctionName.match(sToCopy):
+                    oInstr.asCopyTests.append(sToCopy);
+                else:
+                    self.errorComment(iTagLine, '%s: invalid instruction reference (opstat or function) "%s" (valid: %s or %s)'
+                                                % (sTag, sToCopy, self.oReStatsName.pattern, self.oReFunctionName.pattern));
+            else:
+                self.errorComment(iTagLine, '%s: ignoring duplicate "%s"' % (sTag, sToCopy,));
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpOnlyTest(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@oponlytest | \@oponly
+        Value:      none
+
+        Only test instructions with this tag.  This is a trick that is handy
+        for singling out one or two new instructions or tests.
+
+        See also \@optestignore.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Validate and add instruction to only test dictionary.
+        sValue = self.flattenAllSections(aasSections).strip();
+        if sValue:
+            return self.errorComment(iTagLine, '%s: does not take any value: %s' % (sTag, sValue));
+
+        if oInstr not in g_aoOnlyTestInstructions:
+            g_aoOnlyTestInstructions.append(oInstr);
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpXcptType(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opxcpttype
+        Value:      [none|1|2|3|4|4UA|5|6|7|8|11|12|E1|E1NF|E2|E3|E3NF|E4|E4NF|E5|E5NF|E6|E6NF|E7NF|E9|E9NF|E10|E11|E12|E12NF]
+
+        Sets the SSE or AVX exception type (see SDMv2 2.4, 2.7).
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten as a space separated list, split it up and validate the values.
+        asTypes = self.flattenAllSections(aasSections).split();
+        if len(asTypes) != 1:
+            return self.errorComment(iTagLine, '%s: exactly one invalid exception type, please: %s' % (sTag, asTypes,));
+        sType = asTypes[0];
+        if sType not in g_kdXcptTypes:
+            return self.errorComment(iTagLine, '%s: invalid invalid exception type: %s (valid: %s)'
+                                               % (sTag, sType, sorted(g_kdXcptTypes.keys()),));
+        # Set it.
+        if oInstr.sXcptType is not None:
+            return self.errorComment(iTagLine,
+                                     '%s: attempting to overwrite "%s" with "%s" (only one @opxcpttype)'
+                                     % ( sTag, oInstr.sXcptType, sType,));
+        oInstr.sXcptType = sType;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpFunction(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opfunction
+        Value:      <VMM function name>
+
+        This is for explicitly setting the IEM function name.  Normally we pick
+        this up from the FNIEMOP_XXX macro invocation after the description, or
+        generate it from the mnemonic and operands.
+
+        It it thought it maybe necessary to set it when specifying instructions
+        which implementation isn't following immediately or aren't implemented yet.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sFunction = self.flattenAllSections(aasSections);
+        if not self.oReFunctionName.match(sFunction):
+            return self.errorComment(iTagLine, '%s: invalid VMM function name: "%s" (valid: %s)'
+                                               % (sTag, sFunction, self.oReFunctionName.pattern));
+
+        if oInstr.sFunction is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite VMM function name "%s" with "%s"'
+                                     % (sTag, oInstr.sFunction, sFunction,));
+        oInstr.sFunction = sFunction;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpStats(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opstats
+        Value:      <VMM statistics base name>
+
+        This is for explicitly setting the statistics name.  Normally we pick
+        this up from the IEMOP_MNEMONIC macro invocation, or generate it from
+        the mnemonic and operands.
+
+        It it thought it maybe necessary to set it when specifying instructions
+        which implementation isn't following immediately or aren't implemented yet.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sStats = self.flattenAllSections(aasSections);
+        if not self.oReStatsName.match(sStats):
+            return self.errorComment(iTagLine, '%s: invalid VMM statistics name: "%s" (valid: %s)'
+                                               % (sTag, sStats, self.oReStatsName.pattern));
+
+        if oInstr.sStats is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite VMM statistics base name "%s" with "%s"'
+                                     % (sTag, oInstr.sStats, sStats,));
+        oInstr.sStats = sStats;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpDone(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:    \@opdone
+        Value:  none
+
+        Used to explictily flush the instructions that have been specified.
+        """
+        sFlattened = self.flattenAllSections(aasSections);
+        if sFlattened != '':
+            return self.errorComment(iTagLine, '%s: takes no value, found: "%s"' % (sTag, sFlattened,));
+        _ = sTag; _ = iEndLine;
+        return self.doneInstructions();
+
+    ## @}
+
+
+    def parseComment(self):
+        """
+        Parse the current comment (self.sComment).
+
+        If it's a opcode specifiying comment, we reset the macro stuff.
+        """
+        #
+        # Reject if comment doesn't seem to contain anything interesting.
+        #
+        if    self.sComment.find('Opcode') < 0 \
+          and self.sComment.find('@') < 0:
+            return False;
+
+        #
+        # Split the comment into lines, removing leading asterisks and spaces.
+        # Also remove leading and trailing empty lines.
+        #
+        asLines = self.sComment.split('\n');
+        for iLine, sLine in enumerate(asLines):
+            asLines[iLine] = sLine.lstrip().lstrip('*').lstrip();
+
+        while asLines and not asLines[0]:
+            self.iCommentLine += 1;
+            asLines.pop(0);
+
+        while asLines and not asLines[-1]:
+            asLines.pop(len(asLines) - 1);
+
+        #
+        # Check for old style: Opcode 0x0f 0x12
+        #
+        if asLines[0].startswith('Opcode '):
+            self.parseCommentOldOpcode(asLines);
+
+        #
+        # Look for @op* tagged data.
+        #
+        cOpTags      = 0;
+        sFlatDefault = None;
+        sCurTag      = '@default';
+        iCurTagLine  = 0;
+        asCurSection = [];
+        aasSections  = [ asCurSection, ];
+        for iLine, sLine in enumerate(asLines):
+            if not sLine.startswith('@'):
+                if sLine:
+                    asCurSection.append(sLine);
+                elif asCurSection:
+                    asCurSection = [];
+                    aasSections.append(asCurSection);
+            else:
+                #
+                # Process the previous tag.
+                #
+                if not asCurSection and len(aasSections) > 1:
+                    aasSections.pop(-1);
+                if sCurTag in self.dTagHandlers:
+                    self.dTagHandlers[sCurTag](sCurTag, aasSections, iCurTagLine, iLine);
+                    cOpTags += 1;
+                elif sCurTag.startswith('@op'):
+                    self.errorComment(iCurTagLine, 'Unknown tag: %s' % (sCurTag));
+                elif sCurTag == '@default':
+                    sFlatDefault = self.flattenAllSections(aasSections);
+                elif '@op' + sCurTag[1:] in self.dTagHandlers:
+                    self.errorComment(iCurTagLine, 'Did you mean "@op%s" rather than "%s"?' % (sCurTag[1:], sCurTag));
+                elif sCurTag in ['@encoding', '@opencoding']:
+                    self.errorComment(iCurTagLine, 'Did you mean "@openc" rather than "%s"?' % (sCurTag,));
+
+                #
+                # New tag.
+                #
+                asSplit = sLine.split(None, 1);
+                sCurTag = asSplit[0].lower();
+                if len(asSplit) > 1:
+                    asCurSection = [asSplit[1],];
+                else:
+                    asCurSection = [];
+                aasSections = [asCurSection, ];
+                iCurTagLine = iLine;
+
+        #
+        # Process the final tag.
+        #
+        if not asCurSection and len(aasSections) > 1:
+            aasSections.pop(-1);
+        if sCurTag in self.dTagHandlers:
+            self.dTagHandlers[sCurTag](sCurTag, aasSections, iCurTagLine, iLine);
+            cOpTags += 1;
+        elif sCurTag.startswith('@op'):
+            self.errorComment(iCurTagLine, 'Unknown tag: %s' % (sCurTag));
+        elif sCurTag == '@default':
+            sFlatDefault = self.flattenAllSections(aasSections);
+
+        #
+        # Don't allow default text in blocks containing @op*.
+        #
+        if cOpTags > 0 and sFlatDefault:
+            self.errorComment(0, 'Untagged comment text is not allowed with @op*: %s' % (sFlatDefault,));
+
+        return True;
+
+    def parseMacroInvocation(self, sInvocation):
+        """
+        Parses a macro invocation.
+
+        Returns a tuple, first element is the offset following the macro
+        invocation. The second element is a list of macro arguments, where the
+        zero'th is the macro name.
+        """
+        # First the name.
+        offOpen = sInvocation.find('(');
+        if offOpen <= 0:
+            self.raiseError("macro invocation open parenthesis not found");
+        sName = sInvocation[:offOpen].strip();
+        if not self.oReMacroName.match(sName):
+            return self.error("invalid macro name '%s'" % (sName,));
+        asRet = [sName, ];
+
+        # Arguments.
+        iLine    = self.iLine;
+        cDepth   = 1;
+        off      = offOpen + 1;
+        offStart = off;
+        chQuote  = None;
+        while cDepth > 0:
+            if off >= len(sInvocation):
+                if iLine >= len(self.asLines):
+                    self.error('macro invocation beyond end of file');
+                    return (off, asRet);
+                sInvocation += self.asLines[iLine];
+                iLine += 1;
+            ch = sInvocation[off];
+
+            if chQuote:
+                if ch == '\\' and off + 1 < len(sInvocation):
+                    off += 1;
+                elif ch == chQuote:
+                    chQuote = None;
+            elif ch in ('"', '\'',):
+                chQuote = ch;
+            elif ch in (',', ')',):
+                if cDepth == 1:
+                    asRet.append(sInvocation[offStart:off].strip());
+                    offStart = off + 1;
+                if ch == ')':
+                    cDepth -= 1;
+            elif ch == '(':
+                cDepth += 1;
+            off += 1;
+
+        return (off, asRet);
+
+    def findAndParseMacroInvocationEx(self, sCode, sMacro):
+        """
+        Returns (len(sCode), None) if not found, parseMacroInvocation result if found.
+        """
+        offHit = sCode.find(sMacro);
+        if offHit >= 0 and sCode[offHit + len(sMacro):].strip()[0] == '(':
+            offAfter, asRet = self.parseMacroInvocation(sCode[offHit:])
+            return (offHit + offAfter, asRet);
+        return (len(sCode), None);
+
+    def findAndParseMacroInvocation(self, sCode, sMacro):
+        """
+        Returns None if not found, arguments as per parseMacroInvocation if found.
+        """
+        return self.findAndParseMacroInvocationEx(sCode, sMacro)[1];
+
+    def findAndParseFirstMacroInvocation(self, sCode, asMacro):
+        """
+        Returns same as findAndParseMacroInvocation.
+        """
+        for sMacro in asMacro:
+            asRet = self.findAndParseMacroInvocation(sCode, sMacro);
+            if asRet is not None:
+                return asRet;
+        return None;
+
+    def workerIemOpMnemonicEx(self, sMacro, sStats, sAsm, sForm, sUpper, sLower,  # pylint: disable=too-many-arguments
+                              sDisHints, sIemHints, asOperands):
+        """
+        Processes one of the a IEMOP_MNEMONIC0EX, IEMOP_MNEMONIC1EX, IEMOP_MNEMONIC2EX,
+        IEMOP_MNEMONIC3EX, and IEMOP_MNEMONIC4EX macros.
+        """
+        #
+        # Some invocation checks.
+        #
+        if sUpper != sUpper.upper():
+            self.error('%s: bad a_Upper parameter: %s' % (sMacro, sUpper,));
+        if sLower != sLower.lower():
+            self.error('%s: bad a_Lower parameter: %s' % (sMacro, sLower,));
+        if sUpper.lower() != sLower:
+            self.error('%s: a_Upper and a_Lower parameters does not match: %s vs %s' % (sMacro, sUpper, sLower,));
+        if not self.oReMnemonic.match(sLower):
+            self.error('%s: invalid a_Lower: %s  (valid: %s)' % (sMacro, sLower, self.oReMnemonic.pattern,));
+
+        #
+        # Check if sIemHints tells us to not consider this macro invocation.
+        #
+        if sIemHints.find('IEMOPHINT_SKIP_PYTHON') >= 0:
+            return True;
+
+        # Apply to the last instruction only for now.
+        if not self.aoCurInstrs:
+            self.addInstruction();
+        oInstr = self.aoCurInstrs[-1];
+        if oInstr.iLineMnemonicMacro == -1:
+            oInstr.iLineMnemonicMacro = self.iLine;
+        else:
+            self.error('%s: already saw a IEMOP_MNEMONIC* macro on line %u for this instruction'
+                       % (sMacro, oInstr.iLineMnemonicMacro,));
+
+        # Mnemonic
+        if oInstr.sMnemonic is None:
+            oInstr.sMnemonic = sLower;
+        elif oInstr.sMnemonic != sLower:
+            self.error('%s: current instruction and a_Lower does not match: %s vs %s' % (sMacro, oInstr.sMnemonic, sLower,));
+
+        # Process operands.
+        if len(oInstr.aoOperands) not in [0, len(asOperands)]:
+            self.error('%s: number of operands given by @opN does not match macro: %s vs %s'
+                       % (sMacro, len(oInstr.aoOperands), len(asOperands),));
+        for iOperand, sType in enumerate(asOperands):
+            sWhere = g_kdOpTypes.get(sType, [None, None])[1];
+            if sWhere is None:
+                self.error('%s: unknown a_Op%u value: %s' % (sMacro, iOperand + 1, sType));
+                if iOperand < len(oInstr.aoOperands): # error recovery.
+                    sWhere = oInstr.aoOperands[iOperand].sWhere;
+                    sType  = oInstr.aoOperands[iOperand].sType;
+                else:
+                    sWhere = 'reg';
+                    sType  = 'Gb';
+            if iOperand == len(oInstr.aoOperands):
+                oInstr.aoOperands.append(Operand(sWhere, sType))
+            elif oInstr.aoOperands[iOperand].sWhere != sWhere or oInstr.aoOperands[iOperand].sType != sType:
+                self.error('%s: @op%u and a_Op%u mismatch: %s:%s vs %s:%s'
+                           % (sMacro, iOperand + 1, iOperand + 1, oInstr.aoOperands[iOperand].sWhere,
+                              oInstr.aoOperands[iOperand].sType, sWhere, sType,));
+
+        # Encoding.
+        if sForm not in g_kdIemForms:
+            self.error('%s: unknown a_Form value: %s' % (sMacro, sForm,));
+        else:
+            if oInstr.sEncoding is None:
+                oInstr.sEncoding = g_kdIemForms[sForm][0];
+            elif g_kdIemForms[sForm][0] != oInstr.sEncoding:
+                self.error('%s: current instruction @openc and a_Form does not match: %s vs %s (%s)'
+                           % (sMacro, oInstr.sEncoding, g_kdIemForms[sForm], sForm));
+
+            # Check the parameter locations for the encoding.
+            if g_kdIemForms[sForm][1] is not None:
+                if len(g_kdIemForms[sForm][1]) != len(oInstr.aoOperands):
+                    self.error('%s: The a_Form=%s has a different operand count: %s (form) vs %s'
+                               % (sMacro, sForm, len(g_kdIemForms[sForm][1]), len(oInstr.aoOperands) ));
+                else:
+                    for iOperand, sWhere in enumerate(g_kdIemForms[sForm][1]):
+                        if oInstr.aoOperands[iOperand].sWhere != sWhere:
+                            self.error('%s: current instruction @op%u and a_Form location does not match: %s vs %s (%s)'
+                                       % (sMacro, iOperand + 1, oInstr.aoOperands[iOperand].sWhere, sWhere, sForm,));
+                        sOpFormMatch = g_kdOpTypes[oInstr.aoOperands[iOperand].sType][4];
+                        if    (sOpFormMatch in [ 'REG', 'MEM', ] and sForm.find('_' + sOpFormMatch) < 0) \
+                           or (sOpFormMatch in [ 'FIXED', ]      and sForm.find(sOpFormMatch) < 0) \
+                           or (sOpFormMatch == 'RM' and (sForm.find('_MEM') > 0 or sForm.find('_REG') > 0) ) \
+                           or (sOpFormMatch == 'V'  and (   not (sForm.find('VEX') > 0 or sForm.find('XOP')) \
+                                                         or sForm.replace('VEX','').find('V') < 0) ):
+                            self.error('%s: current instruction @op%u and a_Form type does not match: %s/%s vs %s'
+                                       % (sMacro, iOperand + 1, oInstr.aoOperands[iOperand].sType, sOpFormMatch, sForm, ));
+
+            # Check @opcodesub
+            if oInstr.sSubOpcode \
+              and g_kdIemForms[sForm][2] \
+              and oInstr.sSubOpcode.find(g_kdIemForms[sForm][2]) < 0:
+                self.error('%s: current instruction @opcodesub and a_Form does not match: %s vs %s (%s)'
+                            % (sMacro, oInstr.sSubOpcode, g_kdIemForms[sForm][2], sForm,));
+
+        # Stats.
+        if not self.oReStatsName.match(sStats):
+            self.error('%s: invalid a_Stats value: %s' % (sMacro, sStats,));
+        elif oInstr.sStats is None:
+            oInstr.sStats = sStats;
+        elif oInstr.sStats != sStats:
+            self.error('%s: mismatching @opstats and a_Stats value: %s vs %s'
+                       % (sMacro, oInstr.sStats, sStats,));
+
+        # Process the hints (simply merge with @ophints w/o checking anything).
+        for sHint in sDisHints.split('|'):
+            sHint = sHint.strip();
+            if sHint.startswith('DISOPTYPE_'):
+                sShortHint = sHint[len('DISOPTYPE_'):].lower();
+                if sShortHint in g_kdHints:
+                    oInstr.dHints[sShortHint] = True; # (dummy value, using dictionary for speed)
+                else:
+                    self.error('%s: unknown a_fDisHints value: %s' % (sMacro, sHint,));
+            elif sHint != '0':
+                self.error('%s: expected a_fDisHints value: %s' % (sMacro, sHint,));
+
+        for sHint in sIemHints.split('|'):
+            sHint = sHint.strip();
+            if sHint.startswith('IEMOPHINT_'):
+                sShortHint = sHint[len('IEMOPHINT_'):].lower();
+                if sShortHint in g_kdHints:
+                    oInstr.dHints[sShortHint] = True; # (dummy value, using dictionary for speed)
+                else:
+                    self.error('%s: unknown a_fIemHints value: %s' % (sMacro, sHint,));
+            elif sHint != '0':
+                self.error('%s: expected a_fIemHints value: %s' % (sMacro, sHint,));
+
+        _ = sAsm;
+        return True;
+
+    def workerIemOpMnemonic(self, sMacro, sForm, sUpper, sLower, sDisHints, sIemHints, asOperands):
+        """
+        Processes one of the a IEMOP_MNEMONIC0, IEMOP_MNEMONIC1, IEMOP_MNEMONIC2,
+        IEMOP_MNEMONIC3, and IEMOP_MNEMONIC4 macros.
+        """
+        if not asOperands:
+            return self.workerIemOpMnemonicEx(sMacro, sLower, sLower, sForm, sUpper, sLower, sDisHints, sIemHints, asOperands);
+        return self.workerIemOpMnemonicEx(sMacro, sLower + '_' + '_'.join(asOperands), sLower + ' ' + ','.join(asOperands),
+                                          sForm, sUpper, sLower, sDisHints, sIemHints, asOperands);
+
+    def checkCodeForMacro(self, sCode):
+        """
+        Checks code for relevant macro invocation.
+        """
+        #
+        # Scan macro invocations.
+        #
+        if sCode.find('(') > 0:
+            # Look for instruction decoder function definitions. ASSUME single line.
+            asArgs = self.findAndParseFirstMacroInvocation(sCode,
+                                                           [ 'FNIEMOP_DEF',
+                                                             'FNIEMOP_STUB',
+                                                             'FNIEMOP_STUB_1',
+                                                             'FNIEMOP_UD_STUB',
+                                                             'FNIEMOP_UD_STUB_1' ]);
+            if asArgs is not None:
+                sFunction = asArgs[1];
+
+                if not self.aoCurInstrs:
+                    self.addInstruction();
+                for oInstr in self.aoCurInstrs:
+                    if oInstr.iLineFnIemOpMacro == -1:
+                        oInstr.iLineFnIemOpMacro = self.iLine;
+                    else:
+                        self.error('%s: already seen a FNIEMOP_XXX macro for %s' % (asArgs[0], oInstr,) );
+                self.setInstrunctionAttrib('sFunction', sFunction);
+                self.setInstrunctionAttrib('fStub', asArgs[0].find('STUB') > 0, fOverwrite = True);
+                self.setInstrunctionAttrib('fUdStub', asArgs[0].find('UD_STUB') > 0, fOverwrite = True);
+                if asArgs[0].find('STUB') > 0:
+                    self.doneInstructions();
+                return True;
+
+            # IEMOP_HLP_DONE_VEX_DECODING_*
+            asArgs = self.findAndParseFirstMacroInvocation(sCode,
+                                                           [ 'IEMOP_HLP_DONE_VEX_DECODING',
+                                                             'IEMOP_HLP_DONE_VEX_DECODING_L0',
+                                                             'IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV',
+                                                             'IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV',
+                                                             ]);
+            if asArgs is not None:
+                sMacro = asArgs[0];
+                if sMacro in ('IEMOP_HLP_DONE_VEX_DECODING_L0', 'IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV', ):
+                    for oInstr in self.aoCurInstrs:
+                        if 'vex_l_zero' not in oInstr.dHints:
+                            if oInstr.iLineMnemonicMacro >= 0:
+                                self.errorOnLine(oInstr.iLineMnemonicMacro,
+                                                 'Missing IEMOPHINT_VEX_L_ZERO! (%s on line %d)' % (sMacro, self.iLine,));
+                            oInstr.dHints['vex_l_zero'] = True;
+                return True;
+
+            #
+            # IEMOP_MNEMONIC*
+            #
+
+            # IEMOP_MNEMONIC(a_Stats, a_szMnemonic) IEMOP_INC_STATS(a_Stats)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC');
+            if asArgs is not None:
+                if len(self.aoCurInstrs) == 1:
+                    oInstr = self.aoCurInstrs[0];
+                    if oInstr.sStats is None:
+                        oInstr.sStats = asArgs[1];
+                    self.deriveMnemonicAndOperandsFromStats(oInstr, asArgs[1]);
+
+            # IEMOP_MNEMONIC0EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC0EX');
+            if asArgs is not None:
+                self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[6], asArgs[7],
+                                           []);
+            # IEMOP_MNEMONIC1EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC1EX');
+            if asArgs is not None:
+                self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[7], asArgs[8],
+                                           [asArgs[6],]);
+            # IEMOP_MNEMONIC2EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC2EX');
+            if asArgs is not None:
+                self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[8], asArgs[9],
+                                           [asArgs[6], asArgs[7]]);
+            # IEMOP_MNEMONIC3EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC3EX');
+            if asArgs is not None:
+                self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[9],
+                                           asArgs[10], [asArgs[6], asArgs[7], asArgs[8],]);
+            # IEMOP_MNEMONIC4EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints,
+            #                   a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC4EX');
+            if asArgs is not None:
+                self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[10],
+                                           asArgs[11], [asArgs[6], asArgs[7], asArgs[8], asArgs[9],]);
+
+            # IEMOP_MNEMONIC0(a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC0');
+            if asArgs is not None:
+                self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], []);
+            # IEMOP_MNEMONIC1(a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC1');
+            if asArgs is not None:
+                self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[5], asArgs[6], [asArgs[4],]);
+            # IEMOP_MNEMONIC2(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC2');
+            if asArgs is not None:
+                self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[6], asArgs[7],
+                                         [asArgs[4], asArgs[5],]);
+            # IEMOP_MNEMONIC3(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC3');
+            if asArgs is not None:
+                self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[7], asArgs[8],
+                                         [asArgs[4], asArgs[5], asArgs[6],]);
+            # IEMOP_MNEMONIC4(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC4');
+            if asArgs is not None:
+                self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[8], asArgs[9],
+                                         [asArgs[4], asArgs[5], asArgs[6], asArgs[7],]);
+
+        return False;
+
+
+    def parse(self):
+        """
+        Parses the given file.
+        Returns number or errors.
+        Raises exception on fatal trouble.
+        """
+        #self.debug('Parsing %s' % (self.sSrcFile,));
+
+        while self.iLine < len(self.asLines):
+            sLine = self.asLines[self.iLine];
+            self.iLine  += 1;
+
+            # We only look for comments, so only lines with a slash might possibly
+            # influence the parser state.
+            offSlash = sLine.find('/');
+            if offSlash >= 0:
+                if offSlash + 1 >= len(sLine)  or  sLine[offSlash + 1] != '/'  or  self.iState != self.kiCode:
+                    offLine = 0;
+                    while offLine < len(sLine):
+                        if self.iState == self.kiCode:
+                            offHit = sLine.find('/*', offLine); # only multiline comments for now.
+                            if offHit >= 0:
+                                self.checkCodeForMacro(sLine[offLine:offHit]);
+                                self.sComment     = '';
+                                self.iCommentLine = self.iLine;
+                                self.iState       = self.kiCommentMulti;
+                                offLine = offHit + 2;
+                            else:
+                                self.checkCodeForMacro(sLine[offLine:]);
+                                offLine = len(sLine);
+
+                        elif self.iState == self.kiCommentMulti:
+                            offHit = sLine.find('*/', offLine);
+                            if offHit >= 0:
+                                self.sComment += sLine[offLine:offHit];
+                                self.iState    = self.kiCode;
+                                offLine = offHit + 2;
+                                self.parseComment();
+                            else:
+                                self.sComment += sLine[offLine:];
+                                offLine = len(sLine);
+                        else:
+                            assert False;
+                # C++ line comment.
+                elif offSlash > 0:
+                    self.checkCodeForMacro(sLine[:offSlash]);
+
+            # No slash, but append the line if in multi-line comment.
+            elif self.iState == self.kiCommentMulti:
+                #self.debug('line %d: multi' % (self.iLine,));
+                self.sComment += sLine;
+
+            # No slash, but check code line for relevant macro.
+            elif self.iState == self.kiCode and sLine.find('IEMOP_') >= 0:
+                #self.debug('line %d: macro' % (self.iLine,));
+                self.checkCodeForMacro(sLine);
+
+            # If the line is a '}' in the first position, complete the instructions.
+            elif self.iState == self.kiCode and sLine[0] == '}':
+                #self.debug('line %d: }' % (self.iLine,));
+                self.doneInstructions();
+
+        self.doneInstructions();
+        self.debug('%3s stubs out of %3s instructions in %s'
+                   % (self.cTotalStubs, self.cTotalInstr, os.path.basename(self.sSrcFile),));
+        return self.printErrors();
+
+
+def __parseFileByName(sSrcFile, sDefaultMap):
+    """
+    Parses one source file for instruction specfications.
+    """
+    #
+    # Read sSrcFile into a line array.
+    #
+    try:
+        oFile = open(sSrcFile, "r");
+    except Exception as oXcpt:
+        raise Exception("failed to open %s for reading: %s" % (sSrcFile, oXcpt,));
+    try:
+        asLines = oFile.readlines();
+    except Exception as oXcpt:
+        raise Exception("failed to read %s: %s" % (sSrcFile, oXcpt,));
+    finally:
+        oFile.close();
+
+    #
+    # Do the parsing.
+    #
+    try:
+        cErrors = SimpleParser(sSrcFile, asLines, sDefaultMap).parse();
+    except ParserException as oXcpt:
+        print(str(oXcpt));
+        raise;
+
+    return cErrors;
+
+
+def __doTestCopying():
+    """
+    Executes the asCopyTests instructions.
+    """
+    asErrors = [];
+    for oDstInstr in g_aoAllInstructions:
+        if oDstInstr.asCopyTests:
+            for sSrcInstr in oDstInstr.asCopyTests:
+                oSrcInstr = g_dAllInstructionsByStat.get(sSrcInstr, None);
+                if oSrcInstr:
+                    aoSrcInstrs = [oSrcInstr,];
+                else:
+                    aoSrcInstrs = g_dAllInstructionsByFunction.get(sSrcInstr, []);
+                if aoSrcInstrs:
+                    for oSrcInstr in aoSrcInstrs:
+                        if oSrcInstr != oDstInstr:
+                            oDstInstr.aoTests.extend(oSrcInstr.aoTests);
+                        else:
+                            asErrors.append('%s:%s: error: @opcopytests reference "%s" matches the destination\n'
+                                            % ( oDstInstr.sSrcFile, oDstInstr.iLineCreated, sSrcInstr));
+                else:
+                    asErrors.append('%s:%s: error: @opcopytests reference "%s" not found\n'
+                                    % ( oDstInstr.sSrcFile, oDstInstr.iLineCreated, sSrcInstr));
+
+    if asErrors:
+        sys.stderr.write(u''.join(asErrors));
+    return len(asErrors);
+
+
+def __applyOnlyTest():
+    """
+    If g_aoOnlyTestInstructions contains any instructions, drop aoTests from
+    all other instructions so that only these get tested.
+    """
+    if g_aoOnlyTestInstructions:
+        for oInstr in g_aoAllInstructions:
+            if oInstr.aoTests:
+                if oInstr not in g_aoOnlyTestInstructions:
+                    oInstr.aoTests = [];
+    return 0;
+
+def __parseAll():
+    """
+    Parses all the IEMAllInstruction*.cpp.h files.
+
+    Raises exception on failure.
+    """
+    sSrcDir = os.path.dirname(os.path.abspath(__file__));
+    cErrors = 0;
+    for sDefaultMap, sName in [
+        ( 'one',        'IEMAllInstructionsOneByte.cpp.h'),
+        ( 'two0f',      'IEMAllInstructionsTwoByte0f.cpp.h'),
+        ( 'three0f38',  'IEMAllInstructionsThree0f38.cpp.h'),
+        ( 'three0f3a',  'IEMAllInstructionsThree0f3a.cpp.h'),
+        ( 'vexmap1',    'IEMAllInstructionsVexMap1.cpp.h'),
+        ( 'vexmap2',    'IEMAllInstructionsVexMap2.cpp.h'),
+        ( 'vexmap3',    'IEMAllInstructionsVexMap3.cpp.h'),
+        ( '3dnow',      'IEMAllInstructions3DNow.cpp.h'),
+    ]:
+        cErrors += __parseFileByName(os.path.join(sSrcDir, sName), sDefaultMap);
+    cErrors += __doTestCopying();
+    cErrors += __applyOnlyTest();
+
+    if cErrors != 0:
+        #raise Exception('%d parse errors' % (cErrors,));
+        sys.exit(1);
+    return True;
+
+
+
+__parseAll();
+
+
+#
+# Generators (may perhaps move later).
+#
+def generateDisassemblerTables(oDstFile = sys.stdout):
+    """
+    Generates disassembler tables.
+    """
+
+    for sName, oMap in sorted(iter(g_dInstructionMaps.items()),
+                              key = lambda aKV: aKV[1].sEncoding + ''.join(aKV[1].asLeadOpcodes)):
+        assert oMap.sName == sName;
+        asLines = [];
+
+        asLines.append('/* Generated from: %-11s  Selector: %-7s  Encoding: %-7s  Lead bytes opcodes: %s */'
+                       % ( oMap.sName, oMap.sSelector, oMap.sEncoding, ' '.join(oMap.asLeadOpcodes), ));
+        asLines.append('const DISOPCODE %s[] =' % (oMap.getDisasTableName(),));
+        asLines.append('{');
+
+        aoffColumns = [4, 29, 49, 65, 77, 89, 109, 125, 141, 157, 183, 199];
+
+        aoTableOrder = oMap.getInstructionsInTableOrder();
+        for iInstr, oInstr in enumerate(aoTableOrder):
+
+            if (iInstr & 0xf) == 0:
+                if iInstr != 0:
+                    asLines.append('');
+                asLines.append('    /* %x */' % (iInstr >> 4,));
+
+            if oInstr is None:
+                pass;#asLines.append('    /* %#04x */ None,' % (iInstr));
+            elif isinstance(oInstr, list):
+                asLines.append('    /* %#04x */ ComplicatedListStuffNeedingWrapper,' % (iInstr));
+            else:
+                sMacro = 'OP';
+                cMaxOperands = 3;
+                if len(oInstr.aoOperands) > 3:
+                    sMacro = 'OPVEX'
+                    cMaxOperands = 4;
+                    assert len(oInstr.aoOperands) <= cMaxOperands;
+
+                #
+                # Format string.
+                #
+                sTmp = '%s("%s' % (sMacro, oInstr.sMnemonic,);
+                for iOperand, oOperand in enumerate(oInstr.aoOperands):
+                    sTmp += ' ' if iOperand == 0 else ',';
+                    if g_kdOpTypes[oOperand.sType][2][0] != '%':        ## @todo remove upper() later.
+                        sTmp += g_kdOpTypes[oOperand.sType][2].upper(); ## @todo remove upper() later.
+                    else:
+                        sTmp += g_kdOpTypes[oOperand.sType][2];
+                sTmp += '",';
+                asColumns = [ sTmp, ];
+
+                #
+                # Decoders.
+                #
+                iStart = len(asColumns);
+                if oInstr.sEncoding is None:
+                    pass;
+                elif oInstr.sEncoding == 'ModR/M':
+                    # ASSUME the first operand is using the ModR/M encoding
+                    assert len(oInstr.aoOperands) >= 1 and oInstr.aoOperands[0].usesModRM();
+                    asColumns.append('IDX_ParseModRM,');
+                    ## @todo IDX_ParseVexDest
+                    # Is second operand using ModR/M too?
+                    if len(oInstr.aoOperands) > 1 and oInstr.aoOperands[1].usesModRM():
+                        asColumns.append('IDX_UseModRM,')
+                elif oInstr.sEncoding in [ 'prefix', ]:
+                    for oOperand in oInstr.aoOperands:
+                        asColumns.append('0,');
+                elif oInstr.sEncoding in [ 'fixed' ]:
+                    pass;
+                elif oInstr.sEncoding == 'vex2':
+                    asColumns.append('IDX_ParseVex2b,')
+                elif oInstr.sEncoding == 'vex3':
+                    asColumns.append('IDX_ParseVex3b,')
+                elif oInstr.sEncoding in g_dInstructionMaps:
+                    asColumns.append(g_dInstructionMaps[oInstr.sEncoding].sDisParse + ',');
+                else:
+                    ## @todo
+                    #IDX_ParseTwoByteEsc,
+                    #IDX_ParseGrp1,
+                    #IDX_ParseShiftGrp2,
+                    #IDX_ParseGrp3,
+                    #IDX_ParseGrp4,
+                    #IDX_ParseGrp5,
+                    #IDX_Parse3DNow,
+                    #IDX_ParseGrp6,
+                    #IDX_ParseGrp7,
+                    #IDX_ParseGrp8,
+                    #IDX_ParseGrp9,
+                    #IDX_ParseGrp10,
+                    #IDX_ParseGrp12,
+                    #IDX_ParseGrp13,
+                    #IDX_ParseGrp14,
+                    #IDX_ParseGrp15,
+                    #IDX_ParseGrp16,
+                    #IDX_ParseThreeByteEsc4,
+                    #IDX_ParseThreeByteEsc5,
+                    #IDX_ParseModFence,
+                    #IDX_ParseEscFP,
+                    #IDX_ParseNopPause,
+                    #IDX_ParseInvOpModRM,
+                    assert False, str(oInstr);
+
+                # Check for immediates and stuff in the remaining operands.
+                for oOperand in oInstr.aoOperands[len(asColumns) - iStart:]:
+                    sIdx = g_kdOpTypes[oOperand.sType][0];
+                    if sIdx != 'IDX_UseModRM':
+                        asColumns.append(sIdx + ',');
+                asColumns.extend(['0,'] * (cMaxOperands - (len(asColumns) - iStart)));
+
+                #
+                # Opcode and operands.
+                #
+                assert oInstr.sDisEnum, str(oInstr);
+                asColumns.append(oInstr.sDisEnum + ',');
+                iStart = len(asColumns)
+                for oOperand in oInstr.aoOperands:
+                    asColumns.append('OP_PARM_' + g_kdOpTypes[oOperand.sType][3] + ',');
+                asColumns.extend(['OP_PARM_NONE,'] * (cMaxOperands - (len(asColumns) - iStart)));
+
+                #
+                # Flags.
+                #
+                sTmp = '';
+                for sHint in sorted(oInstr.dHints.keys()):
+                    sDefine = g_kdHints[sHint];
+                    if sDefine.startswith('DISOPTYPE_'):
+                        if sTmp:
+                            sTmp += ' | ' + sDefine;
+                        else:
+                            sTmp += sDefine;
+                if sTmp:
+                    sTmp += '),';
+                else:
+                    sTmp += '0),';
+                asColumns.append(sTmp);
+
+                #
+                # Format the columns into a line.
+                #
+                sLine = '';
+                for i, s in enumerate(asColumns):
+                    if len(sLine) < aoffColumns[i]:
+                        sLine += ' ' * (aoffColumns[i] - len(sLine));
+                    else:
+                        sLine += ' ';
+                    sLine += s;
+
+                # OP("psrlw %Vdq,%Wdq", IDX_ParseModRM, IDX_UseModRM, 0, OP_PSRLW, OP_PARM_Vdq, OP_PARM_Wdq, OP_PARM_NONE,
+                # DISOPTYPE_HARMLESS),
+                # define OP(pszOpcode, idxParse1, idxParse2, idxParse3, opcode, param1, param2, param3, optype) \
+                # { pszOpcode, idxParse1, idxParse2, idxParse3, 0, opcode, param1, param2, param3, 0, 0, optype }
+
+                asLines.append(sLine);
+
+        asLines.append('};');
+        asLines.append('AssertCompile(RT_ELEMENTS(%s) == %s);' % (oMap.getDisasTableName(), oMap.getTableSize(),));
+
+        #
+        # Write out the lines.
+        #
+        oDstFile.write('\n'.join(asLines));
+        oDstFile.write('\n');
+        break; #for now
+
+if __name__ == '__main__':
+    generateDisassemblerTables();
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h
new file mode 100644
index 00000000..cb8dbd7e
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h
@@ -0,0 +1,905 @@
+/* $Id: IEMAllInstructionsThree0f38.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation.
+ *
+ * @remarks IEMAllInstructionsVexMap2.cpp.h is a VEX mirror of this file.
+ *          Any update here is likely needed in that file too.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @name Three byte opcodes with first two bytes 0x0f 0x38
+ * @{
+ */
+
+/*  Opcode      0x0f 0x38 0x00. */
+FNIEMOP_STUB(iemOp_pshufb_Pq_Qq);
+/*  Opcode 0x66 0x0f 0x38 0x00. */
+FNIEMOP_STUB(iemOp_pshufb_Vx_Wx);
+/*  Opcode      0x0f 0x38 0x01. */
+FNIEMOP_STUB(iemOp_phaddw_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x01. */
+FNIEMOP_STUB(iemOp_phaddw_Vx_Wx);
+/** Opcode      0x0f 0x38 0x02. */
+FNIEMOP_STUB(iemOp_phaddd_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x02. */
+FNIEMOP_STUB(iemOp_phaddd_Vx_Wx);
+/** Opcode      0x0f 0x38 0x03. */
+FNIEMOP_STUB(iemOp_phaddsw_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x03. */
+FNIEMOP_STUB(iemOp_phaddsw_Vx_Wx);
+/** Opcode      0x0f 0x38 0x04. */
+FNIEMOP_STUB(iemOp_pmaddubsw_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x04. */
+FNIEMOP_STUB(iemOp_pmaddubsw_Vx_Wx);
+/** Opcode      0x0f 0x38 0x05. */
+FNIEMOP_STUB(iemOp_phsubw_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x05. */
+FNIEMOP_STUB(iemOp_phsubw_Vx_Wx);
+/** Opcode      0x0f 0x38 0x06. */
+FNIEMOP_STUB(iemOp_phsubd_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x06. */
+FNIEMOP_STUB(iemOp_phsubdq_Vx_Wx);
+/** Opcode      0x0f 0x38 0x07. */
+FNIEMOP_STUB(iemOp_phsubsw_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x07. */
+FNIEMOP_STUB(iemOp_phsubsw_Vx_Wx);
+/** Opcode      0x0f 0x38 0x08. */
+FNIEMOP_STUB(iemOp_psignb_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x08. */
+FNIEMOP_STUB(iemOp_psignb_Vx_Wx);
+/** Opcode      0x0f 0x38 0x09. */
+FNIEMOP_STUB(iemOp_psignw_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x09. */
+FNIEMOP_STUB(iemOp_psignw_Vx_Wx);
+/** Opcode      0x0f 0x38 0x0a. */
+FNIEMOP_STUB(iemOp_psignd_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x0a. */
+FNIEMOP_STUB(iemOp_psignd_Vx_Wx);
+/** Opcode      0x0f 0x38 0x0b. */
+FNIEMOP_STUB(iemOp_pmulhrsw_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x0b. */
+FNIEMOP_STUB(iemOp_pmulhrsw_Vx_Wx);
+/*  Opcode      0x0f 0x38 0x0c - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x0c - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x0d - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x0d - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x0e - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x0e - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x0f - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x0f - invalid (vex only). */
+
+
+/*  Opcode      0x0f 0x38 0x10 - invalid */
+/** Opcode 0x66 0x0f 0x38 0x10 (legacy only). */
+FNIEMOP_STUB(iemOp_pblendvb_Vdq_Wdq);
+/*  Opcode      0x0f 0x38 0x11 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x11 - invalid */
+/*  Opcode      0x0f 0x38 0x12 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x12 - invalid */
+/*  Opcode      0x0f 0x38 0x13 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x13 - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x14 - invalid */
+/** Opcode 0x66 0x0f 0x38 0x14 (legacy only). */
+FNIEMOP_STUB(iemOp_blendvps_Vdq_Wdq);
+/*  Opcode      0x0f 0x38 0x15 - invalid */
+/** Opcode 0x66 0x0f 0x38 0x15 (legacy only). */
+FNIEMOP_STUB(iemOp_blendvpd_Vdq_Wdq);
+/*  Opcode      0x0f 0x38 0x16 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x16 - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x17 - invalid */
+/** Opcode 0x66 0x0f 0x38 0x17 - invalid */
+FNIEMOP_STUB(iemOp_ptest_Vx_Wx);
+/*  Opcode      0x0f 0x38 0x18 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x18 - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x19 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x19 - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x1a - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x1a - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x1b - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x1b - invalid */
+/** Opcode      0x0f 0x38 0x1c. */
+FNIEMOP_STUB(iemOp_pabsb_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x1c. */
+FNIEMOP_STUB(iemOp_pabsb_Vx_Wx);
+/** Opcode      0x0f 0x38 0x1d. */
+FNIEMOP_STUB(iemOp_pabsw_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x1d. */
+FNIEMOP_STUB(iemOp_pabsw_Vx_Wx);
+/** Opcode      0x0f 0x38 0x1e. */
+FNIEMOP_STUB(iemOp_pabsd_Pq_Qq);
+/** Opcode 0x66 0x0f 0x38 0x1e. */
+FNIEMOP_STUB(iemOp_pabsd_Vx_Wx);
+/*  Opcode      0x0f 0x38 0x1f - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x1f - invalid */
+
+
+/** Opcode 0x66 0x0f 0x38 0x20. */
+FNIEMOP_STUB(iemOp_pmovsxbw_Vx_UxMq);
+/** Opcode 0x66 0x0f 0x38 0x21. */
+FNIEMOP_STUB(iemOp_pmovsxbd_Vx_UxMd);
+/** Opcode 0x66 0x0f 0x38 0x22. */
+FNIEMOP_STUB(iemOp_pmovsxbq_Vx_UxMw);
+/** Opcode 0x66 0x0f 0x38 0x23. */
+FNIEMOP_STUB(iemOp_pmovsxwd_Vx_UxMq);
+/** Opcode 0x66 0x0f 0x38 0x24. */
+FNIEMOP_STUB(iemOp_pmovsxwq_Vx_UxMd);
+/** Opcode 0x66 0x0f 0x38 0x25. */
+FNIEMOP_STUB(iemOp_pmovsxdq_Vx_UxMq);
+/*  Opcode 0x66 0x0f 0x38 0x26 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x27 - invalid */
+/** Opcode 0x66 0x0f 0x38 0x28. */
+FNIEMOP_STUB(iemOp_pmuldq_Vx_Wx);
+/** Opcode 0x66 0x0f 0x38 0x29. */
+FNIEMOP_STUB(iemOp_pcmpeqq_Vx_Wx);
+
+/**
+ * @opcode      0x2a
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse4.1
+ * @opgroup     og_sse41_cachect
+ * @opxcpttype  1
+ * @optest      op1=-1 op2=2  -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movntdqa_Vdq_Mdq)
+{
+    IEMOP_MNEMONIC2(RM_MEM, MOVNTDQA, movntdqa, Vdq_WO, Mdq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* Register, memory. */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud660f382areg
+     * @opcode      0x2a
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/** Opcode 0x66 0x0f 0x38 0x2b. */
+FNIEMOP_STUB(iemOp_packusdw_Vx_Wx);
+/*  Opcode 0x66 0x0f 0x38 0x2c - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x2d - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x2e - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x2f - invalid (vex only). */
+
+/** Opcode 0x66 0x0f 0x38 0x30. */
+FNIEMOP_STUB(iemOp_pmovzxbw_Vx_UxMq);
+/** Opcode 0x66 0x0f 0x38 0x31. */
+FNIEMOP_STUB(iemOp_pmovzxbd_Vx_UxMd);
+/** Opcode 0x66 0x0f 0x38 0x32. */
+FNIEMOP_STUB(iemOp_pmovzxbq_Vx_UxMw);
+/** Opcode 0x66 0x0f 0x38 0x33. */
+FNIEMOP_STUB(iemOp_pmovzxwd_Vx_UxMq);
+/** Opcode 0x66 0x0f 0x38 0x34. */
+FNIEMOP_STUB(iemOp_pmovzxwq_Vx_UxMd);
+/** Opcode 0x66 0x0f 0x38 0x35. */
+FNIEMOP_STUB(iemOp_pmovzxdq_Vx_UxMq);
+/*  Opcode 0x66 0x0f 0x38 0x36 - invalid (vex only). */
+/** Opcode 0x66 0x0f 0x38 0x37. */
+FNIEMOP_STUB(iemOp_pcmpgtq_Vx_Wx);
+/** Opcode 0x66 0x0f 0x38 0x38. */
+FNIEMOP_STUB(iemOp_pminsb_Vx_Wx);
+/** Opcode 0x66 0x0f 0x38 0x39. */
+FNIEMOP_STUB(iemOp_pminsd_Vx_Wx);
+/** Opcode 0x66 0x0f 0x38 0x3a. */
+FNIEMOP_STUB(iemOp_pminuw_Vx_Wx);
+/** Opcode 0x66 0x0f 0x38 0x3b. */
+FNIEMOP_STUB(iemOp_pminud_Vx_Wx);
+/** Opcode 0x66 0x0f 0x38 0x3c. */
+FNIEMOP_STUB(iemOp_pmaxsb_Vx_Wx);
+/** Opcode 0x66 0x0f 0x38 0x3d. */
+FNIEMOP_STUB(iemOp_pmaxsd_Vx_Wx);
+/** Opcode 0x66 0x0f 0x38 0x3e. */
+FNIEMOP_STUB(iemOp_pmaxuw_Vx_Wx);
+/** Opcode 0x66 0x0f 0x38 0x3f. */
+FNIEMOP_STUB(iemOp_pmaxud_Vx_Wx);
+
+
+/** Opcode 0x66 0x0f 0x38 0x40. */
+FNIEMOP_STUB(iemOp_pmulld_Vx_Wx);
+/** Opcode 0x66 0x0f 0x38 0x41. */
+FNIEMOP_STUB(iemOp_phminposuw_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0x42 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x43 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x44 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x45 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x46 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x47 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x48 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x49 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x4a - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x4b - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x4c - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x4d - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x4e - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x4f - invalid. */
+
+/*  Opcode 0x66 0x0f 0x38 0x50 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x51 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x52 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x53 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x54 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x55 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x56 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x57 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x58 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x59 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x5a - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x5b - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x5c - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x5d - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x5e - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x5f - invalid. */
+
+/*  Opcode 0x66 0x0f 0x38 0x60 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x61 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x62 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x63 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x64 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x65 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x66 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x67 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x68 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x69 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x6a - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x6b - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x6c - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x6d - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x6e - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x6f - invalid. */
+
+/*  Opcode 0x66 0x0f 0x38 0x70 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x71 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x72 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x73 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x74 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x75 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x76 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x77 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x78 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x79 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x7a - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x7b - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x7c - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x7d - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x7e - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x7f - invalid. */
+
+/** Opcode 0x66 0x0f 0x38 0x80. */
+FNIEMOP_STUB(iemOp_invept_Gy_Mdq);
+
+/** Opcode 0x66 0x0f 0x38 0x81. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF(iemOp_invvpid_Gy_Mdq)
+{
+    IEMOP_MNEMONIC(invvpid, "invvpid Gy,Mdq");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_IN_VMX_OPERATION("invvpid", kVmxVDiag_Invvpid);
+    IEMOP_HLP_VMX_INSTR("invvpid", kVmxVDiag_Invvpid);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* Register, memory. */
+        if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,  iEffSeg,          0);
+            IEM_MC_ARG(RTGCPTR,  GCPtrInvvpidDesc, 1);
+            IEM_MC_ARG(uint64_t, uInvvpidType,     2);
+            IEM_MC_FETCH_GREG_U64(uInvvpidType, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrInvvpidDesc, bRm, 0);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_invvpid, iEffSeg, GCPtrInvvpidDesc, uInvvpidType);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,  iEffSeg,          0);
+            IEM_MC_ARG(RTGCPTR,  GCPtrInvvpidDesc, 1);
+            IEM_MC_ARG(uint32_t, uInvvpidType,     2);
+            IEM_MC_FETCH_GREG_U32(uInvvpidType, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrInvvpidDesc, bRm, 0);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_invvpid, iEffSeg, GCPtrInvvpidDesc, uInvvpidType);
+            IEM_MC_END();
+        }
+    }
+    Log(("iemOp_invvpid_Gy_Mdq: invalid encoding -> #UD\n"));
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+#else
+FNIEMOP_STUB(iemOp_invvpid_Gy_Mdq);
+#endif
+
+/** Opcode 0x66 0x0f 0x38 0x82. */
+FNIEMOP_DEF(iemOp_invpcid_Gy_Mdq)
+{
+    IEMOP_MNEMONIC(invpcid, "invpcid Gy,Mdq");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* Register, memory. */
+        if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,  iEffSeg,          0);
+            IEM_MC_ARG(RTGCPTR,  GCPtrInvpcidDesc, 1);
+            IEM_MC_ARG(uint64_t, uInvpcidType,     2);
+            IEM_MC_FETCH_GREG_U64(uInvpcidType, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrInvpcidDesc, bRm, 0);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_invpcid, iEffSeg, GCPtrInvpcidDesc, uInvpcidType);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,  iEffSeg,          0);
+            IEM_MC_ARG(RTGCPTR,  GCPtrInvpcidDesc, 1);
+            IEM_MC_ARG(uint32_t, uInvpcidType,     2);
+            IEM_MC_FETCH_GREG_U32(uInvpcidType, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrInvpcidDesc, bRm, 0);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_invpcid, iEffSeg, GCPtrInvpcidDesc, uInvpcidType);
+            IEM_MC_END();
+        }
+    }
+    Log(("iemOp_invpcid_Gy_Mdq: invalid encoding -> #UD\n"));
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/*  Opcode 0x66 0x0f 0x38 0x83 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x84 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x85 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x86 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x87 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x88 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x89 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x8a - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x8b - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x8c - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x8d - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x8e - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x8f - invalid. */
+
+/*  Opcode 0x66 0x0f 0x38 0x90 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x91 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x92 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x93 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x94 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x95 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x96 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x97 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x98 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x99 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x9a - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x9b - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x9c - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x9d - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x9e - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x9f - invalid (vex only). */
+
+/*  Opcode 0x66 0x0f 0x38 0xa0 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xa1 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xa2 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xa3 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xa4 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xa5 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xa6 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xa7 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xa8 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xa9 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xaa - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xab - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xac - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xad - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xae - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xaf - invalid (vex only). */
+
+/*  Opcode 0x66 0x0f 0x38 0xb0 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xb1 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xb2 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xb3 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xb4 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xb5 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xb6 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xb7 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xb8 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xb9 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xba - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xbb - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xbc - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xbd - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xbe - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xbf - invalid (vex only). */
+
+/*  Opcode      0x0f 0x38 0xc0 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc0 - invalid. */
+/*  Opcode      0x0f 0x38 0xc1 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc1 - invalid. */
+/*  Opcode      0x0f 0x38 0xc2 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc2 - invalid. */
+/*  Opcode      0x0f 0x38 0xc3 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc3 - invalid. */
+/*  Opcode      0x0f 0x38 0xc4 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc4 - invalid. */
+/*  Opcode      0x0f 0x38 0xc5 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc5 - invalid. */
+/*  Opcode      0x0f 0x38 0xc6 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc6 - invalid. */
+/*  Opcode      0x0f 0x38 0xc7 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc7 - invalid. */
+/** Opcode      0x0f 0x38 0xc8. */
+FNIEMOP_STUB(iemOp_sha1nexte_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0xc8 - invalid. */
+/** Opcode      0x0f 0x38 0xc9. */
+FNIEMOP_STUB(iemOp_sha1msg1_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0xc9 - invalid. */
+/** Opcode      0x0f 0x38 0xca. */
+FNIEMOP_STUB(iemOp_sha1msg2_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0xca - invalid. */
+/** Opcode      0x0f 0x38 0xcb. */
+FNIEMOP_STUB(iemOp_sha256rnds2_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0xcb - invalid. */
+/** Opcode      0x0f 0x38 0xcc. */
+FNIEMOP_STUB(iemOp_sha256msg1_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0xcc - invalid. */
+/** Opcode      0x0f 0x38 0xcd. */
+FNIEMOP_STUB(iemOp_sha256msg2_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0xcd - invalid. */
+/*  Opcode      0x0f 0x38 0xce - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xce - invalid. */
+/*  Opcode      0x0f 0x38 0xcf - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xcf - invalid. */
+
+/*  Opcode 0x66 0x0f 0x38 0xd0 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd1 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd2 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd3 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd4 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd5 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd6 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd7 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd8 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd9 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xda - invalid. */
+/** Opcode 0x66 0x0f 0x38 0xdb. */
+FNIEMOP_STUB(iemOp_aesimc_Vdq_Wdq);
+/** Opcode 0x66 0x0f 0x38 0xdc. */
+FNIEMOP_STUB(iemOp_aesenc_Vdq_Wdq);
+/** Opcode 0x66 0x0f 0x38 0xdd. */
+FNIEMOP_STUB(iemOp_aesenclast_Vdq_Wdq);
+/** Opcode 0x66 0x0f 0x38 0xde. */
+FNIEMOP_STUB(iemOp_aesdec_Vdq_Wdq);
+/** Opcode 0x66 0x0f 0x38 0xdf. */
+FNIEMOP_STUB(iemOp_aesdeclast_Vdq_Wdq);
+
+/*  Opcode 0x66 0x0f 0x38 0xe0 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe1 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe2 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe3 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe4 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe5 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe6 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe7 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe8 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe9 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xea - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xeb - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xec - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xed - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xee - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xef - invalid. */
+
+
+/** Opcode      0x0f 0x38 0xf0. */
+FNIEMOP_STUB(iemOp_movbe_Gy_My);
+/** Opcode 0x66 0x0f 0x38 0xf0. */
+FNIEMOP_STUB(iemOp_movbe_Gw_Mw);
+/*  Opcode 0xf3 0x0f 0x38 0xf0 - invalid. */
+/** Opcode 0xf2 0x0f 0x38 0xf0. */
+FNIEMOP_STUB(iemOp_crc32_Gb_Eb);
+
+/** Opcode      0x0f 0x38 0xf1. */
+FNIEMOP_STUB(iemOp_movbe_My_Gy);
+/** Opcode 0x66 0x0f 0x38 0xf1. */
+FNIEMOP_STUB(iemOp_movbe_Mw_Gw);
+/*  Opcode 0xf3 0x0f 0x38 0xf1 - invalid. */
+/** Opcode 0xf2 0x0f 0x38 0xf1. */
+FNIEMOP_STUB(iemOp_crc32_Gv_Ev);
+
+/*  Opcode      0x0f 0x38 0xf2 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xf2 - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xf2 - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xf2 - invalid. */
+
+/*  Opcode      0x0f 0x38 0xf3 - invalid (vex only - group 17). */
+/*  Opcode 0x66 0x0f 0x38 0xf3 - invalid (vex only - group 17). */
+/*  Opcode 0xf3 0x0f 0x38 0xf3 - invalid (vex only - group 17). */
+/*  Opcode 0xf2 0x0f 0x38 0xf3 - invalid (vex only - group 17). */
+
+/*  Opcode      0x0f 0x38 0xf4 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xf4 - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xf4 - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xf4 - invalid. */
+
+/*  Opcode      0x0f 0x38 0xf5 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xf5 - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xf5 - invalid (vex only). */
+/*  Opcode 0xf2 0x0f 0x38 0xf5 - invalid (vex only). */
+
+/*  Opcode      0x0f 0x38 0xf6 - invalid. */
+/** Opcode 0x66 0x0f 0x38 0xf6. */
+FNIEMOP_STUB(iemOp_adcx_Gy_Ey);
+/** Opcode 0xf3 0x0f 0x38 0xf6. */
+FNIEMOP_STUB(iemOp_adox_Gy_Ey);
+/*  Opcode 0xf2 0x0f 0x38 0xf6 - invalid (vex only). */
+
+/*  Opcode      0x0f 0x38 0xf7 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xf7 - invalid (vex only). */
+/*  Opcode 0xf3 0x0f 0x38 0xf7 - invalid (vex only). */
+/*  Opcode 0xf2 0x0f 0x38 0xf7 - invalid (vex only). */
+
+/*  Opcode      0x0f 0x38 0xf8 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xf8 - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xf8 - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xf8 - invalid. */
+
+/*  Opcode      0x0f 0x38 0xf9 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xf9 - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xf9 - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xf9 - invalid. */
+
+/*  Opcode      0x0f 0x38 0xfa - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xfa - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xfa - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xfa - invalid. */
+
+/*  Opcode      0x0f 0x38 0xfb - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xfb - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xfb - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xfb - invalid. */
+
+/*  Opcode      0x0f 0x38 0xfc - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xfc - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xfc - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xfc - invalid. */
+
+/*  Opcode      0x0f 0x38 0xfd - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xfd - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xfd - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xfd - invalid. */
+
+/*  Opcode      0x0f 0x38 0xfe - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xfe - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xfe - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xfe - invalid. */
+
+/*  Opcode      0x0f 0x38 0xff - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xff - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xff - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xff - invalid. */
+
+
+/**
+ * Three byte opcode map, first two bytes are 0x0f 0x38.
+ * @sa      g_apfnVexMap2
+ */
+IEM_STATIC const PFNIEMOP g_apfnThreeByte0f38[] =
+{
+    /*          no prefix,                  066h prefix                 f3h prefix,                 f2h prefix */
+    /* 0x00 */  iemOp_pshufb_Pq_Qq,         iemOp_pshufb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x01 */  iemOp_phaddw_Pq_Qq,         iemOp_phaddw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x02 */  iemOp_phaddd_Pq_Qq,         iemOp_phaddd_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x03 */  iemOp_phaddsw_Pq_Qq,        iemOp_phaddsw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x04 */  iemOp_pmaddubsw_Pq_Qq,      iemOp_pmaddubsw_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x05 */  iemOp_phsubw_Pq_Qq,         iemOp_phsubw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x06 */  iemOp_phsubd_Pq_Qq,         iemOp_phsubdq_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x07 */  iemOp_phsubsw_Pq_Qq,        iemOp_phsubsw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x08 */  iemOp_psignb_Pq_Qq,         iemOp_psignb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x09 */  iemOp_psignw_Pq_Qq,         iemOp_psignw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0a */  iemOp_psignd_Pq_Qq,         iemOp_psignd_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0b */  iemOp_pmulhrsw_Pq_Qq,       iemOp_pmulhrsw_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x10 */  iemOp_InvalidNeedRM,        iemOp_pblendvb_Vdq_Wdq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x11 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x12 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x13 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x14 */  iemOp_InvalidNeedRM,        iemOp_blendvps_Vdq_Wdq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x15 */  iemOp_InvalidNeedRM,        iemOp_blendvpd_Vdq_Wdq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x16 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x17 */  iemOp_InvalidNeedRM,        iemOp_ptest_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x18 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x19 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1c */  iemOp_pabsb_Pq_Qq,          iemOp_pabsb_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1d */  iemOp_pabsw_Pq_Qq,          iemOp_pabsw_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1e */  iemOp_pabsd_Pq_Qq,          iemOp_pabsd_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x20 */  iemOp_InvalidNeedRM,        iemOp_pmovsxbw_Vx_UxMq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x21 */  iemOp_InvalidNeedRM,        iemOp_pmovsxbd_Vx_UxMd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x22 */  iemOp_InvalidNeedRM,        iemOp_pmovsxbq_Vx_UxMw,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x23 */  iemOp_InvalidNeedRM,        iemOp_pmovsxwd_Vx_UxMq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x24 */  iemOp_InvalidNeedRM,        iemOp_pmovsxwq_Vx_UxMd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x25 */  iemOp_InvalidNeedRM,        iemOp_pmovsxdq_Vx_UxMq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x26 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x27 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x28 */  iemOp_InvalidNeedRM,        iemOp_pmuldq_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x29 */  iemOp_InvalidNeedRM,        iemOp_pcmpeqq_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2a */  iemOp_InvalidNeedRM,        iemOp_movntdqa_Vdq_Mdq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2b */  iemOp_InvalidNeedRM,        iemOp_packusdw_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x2d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x2e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x2f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x30 */  iemOp_InvalidNeedRM,        iemOp_pmovzxbw_Vx_UxMq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x31 */  iemOp_InvalidNeedRM,        iemOp_pmovzxbd_Vx_UxMd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x32 */  iemOp_InvalidNeedRM,        iemOp_pmovzxbq_Vx_UxMw,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x33 */  iemOp_InvalidNeedRM,        iemOp_pmovzxwd_Vx_UxMq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x34 */  iemOp_InvalidNeedRM,        iemOp_pmovzxwq_Vx_UxMd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x35 */  iemOp_InvalidNeedRM,        iemOp_pmovzxdq_Vx_UxMq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x36 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x37 */  iemOp_InvalidNeedRM,        iemOp_pcmpgtq_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x38 */  iemOp_InvalidNeedRM,        iemOp_pminsb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x39 */  iemOp_InvalidNeedRM,        iemOp_pminsd_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3a */  iemOp_InvalidNeedRM,        iemOp_pminuw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3b */  iemOp_InvalidNeedRM,        iemOp_pminud_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3c */  iemOp_InvalidNeedRM,        iemOp_pmaxsb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3d */  iemOp_InvalidNeedRM,        iemOp_pmaxsd_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3e */  iemOp_InvalidNeedRM,        iemOp_pmaxuw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3f */  iemOp_InvalidNeedRM,        iemOp_pmaxud_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x40 */  iemOp_InvalidNeedRM,        iemOp_pmulld_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x41 */  iemOp_InvalidNeedRM,        iemOp_phminposuw_Vdq_Wdq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x42 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x43 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x44 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x45 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x46 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x47 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x48 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x49 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x50 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x51 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x52 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x53 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x54 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x55 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x56 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x57 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x58 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x59 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x60 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x61 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x62 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x63 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x64 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x65 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x66 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x67 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x68 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x69 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x70 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x71 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x72 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x73 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x74 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x75 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x76 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x77 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x78 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x79 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x80 */  iemOp_InvalidNeedRM,        iemOp_invept_Gy_Mdq,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x81 */  iemOp_InvalidNeedRM,        iemOp_invvpid_Gy_Mdq,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x82 */  iemOp_InvalidNeedRM,        iemOp_invpcid_Gy_Mdq,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x83 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x84 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x85 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x86 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x87 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x88 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x89 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x90 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x91 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x92 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x93 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x94 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x95 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x96 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x97 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x98 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x99 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xa0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xaa */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xab */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xac */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xad */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xae */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xaf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xb0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xba */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbc */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbd */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbe */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xc0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc8 */  iemOp_sha1nexte_Vdq_Wdq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xc9 */  iemOp_sha1msg1_Vdq_Wdq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xca */  iemOp_sha1msg2_Vdq_Wdq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xcb */  iemOp_sha256rnds2_Vdq_Wdq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xcc */  iemOp_sha256msg1_Vdq_Wdq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xcd */  iemOp_sha256msg2_Vdq_Wdq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xce */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xcf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xd0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xda */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xdb */  iemOp_InvalidNeedRM,        iemOp_aesimc_Vdq_Wdq,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdc */  iemOp_InvalidNeedRM,        iemOp_aesenc_Vdq_Wdq,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdd */  iemOp_InvalidNeedRM,        iemOp_aesenclast_Vdq_Wdq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xde */  iemOp_InvalidNeedRM,        iemOp_aesdec_Vdq_Wdq,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdf */  iemOp_InvalidNeedRM,        iemOp_aesdeclast_Vdq_Wdq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0xe0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xea */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xeb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xec */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xed */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xee */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xef */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xf0 */  iemOp_movbe_Gy_My,          iemOp_movbe_Gw_Mw,          iemOp_InvalidNeedRM,        iemOp_crc32_Gb_Eb,
+    /* 0xf1 */  iemOp_movbe_My_Gy,          iemOp_movbe_Mw_Gw,          iemOp_InvalidNeedRM,        iemOp_crc32_Gv_Ev,
+    /* 0xf2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf6 */  iemOp_InvalidNeedRM,        iemOp_adcx_Gy_Ey,           iemOp_adox_Gy_Ey,           iemOp_InvalidNeedRM,
+    /* 0xf7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfa */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfc */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfd */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfe */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xff */  IEMOP_X4(iemOp_InvalidNeedRM),
+};
+AssertCompile(RT_ELEMENTS(g_apfnThreeByte0f38) == 1024);
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h
new file mode 100644
index 00000000..e3e1ed2c
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h
@@ -0,0 +1,502 @@
+/* $Id: IEMAllInstructionsThree0f3a.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation, 0x0f 0x3a map.
+ *
+ * @remarks IEMAllInstructionsVexMap3.cpp.h is a VEX mirror of this file.
+ *          Any update here is likely needed in that file too.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @name Three byte opcodes with first two bytes 0x0f 0x3a
+ * @{
+ */
+
+/** Opcode 0x66 0x0f 0x00 - invalid (vex only). */
+/** Opcode 0x66 0x0f 0x01 - invalid (vex only). */
+/** Opcode 0x66 0x0f 0x02 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x03 - invalid */
+/** Opcode 0x66 0x0f 0x04 - invalid (vex only). */
+/** Opcode 0x66 0x0f 0x05 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x06 - invalid (vex only) */
+/*  Opcode 0x66 0x0f 0x07 - invalid */
+/** Opcode 0x66 0x0f 0x08. */
+FNIEMOP_STUB(iemOp_roundps_Vx_Wx_Ib);
+/** Opcode 0x66 0x0f 0x09. */
+FNIEMOP_STUB(iemOp_roundpd_Vx_Wx_Ib);
+/** Opcode 0x66 0x0f 0x0a. */
+FNIEMOP_STUB(iemOp_roundss_Vss_Wss_Ib);
+/** Opcode 0x66 0x0f 0x0b. */
+FNIEMOP_STUB(iemOp_roundsd_Vsd_Wsd_Ib);
+/** Opcode 0x66 0x0f 0x0c. */
+FNIEMOP_STUB(iemOp_blendps_Vx_Wx_Ib);
+/** Opcode 0x66 0x0f 0x0d. */
+FNIEMOP_STUB(iemOp_blendpd_Vx_Wx_Ib);
+/** Opcode 0x66 0x0f 0x0e. */
+FNIEMOP_STUB(iemOp_blendw_Vx_Wx_Ib);
+/** Opcode      0x0f 0x0f. */
+FNIEMOP_STUB(iemOp_palignr_Pq_Qq_Ib);
+/** Opcode 0x66 0x0f 0x0f. */
+FNIEMOP_STUB(iemOp_palignr_Vx_Wx_Ib);
+
+
+/*  Opcode 0x66 0x0f 0x10 - invalid */
+/*  Opcode 0x66 0x0f 0x11 - invalid */
+/*  Opcode 0x66 0x0f 0x12 - invalid */
+/*  Opcode 0x66 0x0f 0x13 - invalid */
+/** Opcode 0x66 0x0f 0x14. */
+FNIEMOP_STUB(iemOp_pextrb_RdMb_Vdq_Ib);
+/** Opcode 0x66 0x0f 0x15. */
+FNIEMOP_STUB(iemOp_pextrw_RdMw_Vdq_Ib);
+/** Opcode 0x66 0x0f 0x16. */
+FNIEMOP_STUB(iemOp_pextrd_q_RdMw_Vdq_Ib);
+/** Opcode 0x66 0x0f 0x17. */
+FNIEMOP_STUB(iemOp_extractps_Ed_Vdq_Ib);
+/*  Opcode 0x66 0x0f 0x18 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x19 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x1a - invalid */
+/*  Opcode 0x66 0x0f 0x1b - invalid */
+/*  Opcode 0x66 0x0f 0x1c - invalid */
+/*  Opcode 0x66 0x0f 0x1d - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x1e - invalid */
+/*  Opcode 0x66 0x0f 0x1f - invalid */
+
+
+/** Opcode 0x66 0x0f 0x20. */
+FNIEMOP_STUB(iemOp_pinsrb_Vdq_RyMb_Ib);
+/** Opcode 0x66 0x0f 0x21, */
+FNIEMOP_STUB(iemOp_insertps_Vdq_UdqMd_Ib);
+/** Opcode 0x66 0x0f 0x22. */
+FNIEMOP_STUB(iemOp_pinsrd_q_Vdq_Ey_Ib);
+/*  Opcode 0x66 0x0f 0x23 - invalid */
+/*  Opcode 0x66 0x0f 0x24 - invalid */
+/*  Opcode 0x66 0x0f 0x25 - invalid */
+/*  Opcode 0x66 0x0f 0x26 - invalid */
+/*  Opcode 0x66 0x0f 0x27 - invalid */
+/*  Opcode 0x66 0x0f 0x28 - invalid */
+/*  Opcode 0x66 0x0f 0x29 - invalid */
+/*  Opcode 0x66 0x0f 0x2a - invalid */
+/*  Opcode 0x66 0x0f 0x2b - invalid */
+/*  Opcode 0x66 0x0f 0x2c - invalid */
+/*  Opcode 0x66 0x0f 0x2d - invalid */
+/*  Opcode 0x66 0x0f 0x2e - invalid */
+/*  Opcode 0x66 0x0f 0x2f - invalid */
+
+
+/*  Opcode 0x66 0x0f 0x30 - invalid */
+/*  Opcode 0x66 0x0f 0x31 - invalid */
+/*  Opcode 0x66 0x0f 0x32 - invalid */
+/*  Opcode 0x66 0x0f 0x33 - invalid */
+/*  Opcode 0x66 0x0f 0x34 - invalid */
+/*  Opcode 0x66 0x0f 0x35 - invalid */
+/*  Opcode 0x66 0x0f 0x36 - invalid */
+/*  Opcode 0x66 0x0f 0x37 - invalid */
+/*  Opcode 0x66 0x0f 0x38 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x39 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x3a - invalid */
+/*  Opcode 0x66 0x0f 0x3b - invalid */
+/*  Opcode 0x66 0x0f 0x3c - invalid */
+/*  Opcode 0x66 0x0f 0x3d - invalid */
+/*  Opcode 0x66 0x0f 0x3e - invalid */
+/*  Opcode 0x66 0x0f 0x3f - invalid */
+
+
+/** Opcode 0x66 0x0f 0x40. */
+FNIEMOP_STUB(iemOp_dpps_Vx_Wx_Ib);
+/** Opcode 0x66 0x0f 0x41, */
+FNIEMOP_STUB(iemOp_dppd_Vdq_Wdq_Ib);
+/** Opcode 0x66 0x0f 0x42. */
+FNIEMOP_STUB(iemOp_mpsadbw_Vx_Wx_Ib);
+/*  Opcode 0x66 0x0f 0x43 - invalid */
+/** Opcode 0x66 0x0f 0x44. */
+FNIEMOP_STUB(iemOp_pclmulqdq_Vdq_Wdq_Ib);
+/*  Opcode 0x66 0x0f 0x45 - invalid */
+/*  Opcode 0x66 0x0f 0x46 - invalid (vex only)  */
+/*  Opcode 0x66 0x0f 0x47 - invalid */
+/*  Opcode 0x66 0x0f 0x48 - invalid */
+/*  Opcode 0x66 0x0f 0x49 - invalid */
+/*  Opcode 0x66 0x0f 0x4a - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x4b - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x4c - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x4d - invalid */
+/*  Opcode 0x66 0x0f 0x4e - invalid */
+/*  Opcode 0x66 0x0f 0x4f - invalid */
+
+
+/*  Opcode 0x66 0x0f 0x50 - invalid */
+/*  Opcode 0x66 0x0f 0x51 - invalid */
+/*  Opcode 0x66 0x0f 0x52 - invalid */
+/*  Opcode 0x66 0x0f 0x53 - invalid */
+/*  Opcode 0x66 0x0f 0x54 - invalid */
+/*  Opcode 0x66 0x0f 0x55 - invalid */
+/*  Opcode 0x66 0x0f 0x56 - invalid */
+/*  Opcode 0x66 0x0f 0x57 - invalid */
+/*  Opcode 0x66 0x0f 0x58 - invalid */
+/*  Opcode 0x66 0x0f 0x59 - invalid */
+/*  Opcode 0x66 0x0f 0x5a - invalid */
+/*  Opcode 0x66 0x0f 0x5b - invalid */
+/*  Opcode 0x66 0x0f 0x5c - invalid */
+/*  Opcode 0x66 0x0f 0x5d - invalid */
+/*  Opcode 0x66 0x0f 0x5e - invalid */
+/*  Opcode 0x66 0x0f 0x5f - invalid */
+
+
+/** Opcode 0x66 0x0f 0x60. */
+FNIEMOP_STUB(iemOp_pcmpestrm_Vdq_Wdq_Ib);
+/** Opcode 0x66 0x0f 0x61, */
+FNIEMOP_STUB(iemOp_pcmpestri_Vdq_Wdq_Ib);
+/** Opcode 0x66 0x0f 0x62. */
+FNIEMOP_STUB(iemOp_pcmpistrm_Vdq_Wdq_Ib);
+/** Opcode 0x66 0x0f 0x63*/
+FNIEMOP_STUB(iemOp_pcmpistri_Vdq_Wdq_Ib);
+/*  Opcode 0x66 0x0f 0x64 - invalid */
+/*  Opcode 0x66 0x0f 0x65 - invalid */
+/*  Opcode 0x66 0x0f 0x66 - invalid */
+/*  Opcode 0x66 0x0f 0x67 - invalid */
+/*  Opcode 0x66 0x0f 0x68 - invalid */
+/*  Opcode 0x66 0x0f 0x69 - invalid */
+/*  Opcode 0x66 0x0f 0x6a - invalid */
+/*  Opcode 0x66 0x0f 0x6b - invalid */
+/*  Opcode 0x66 0x0f 0x6c - invalid */
+/*  Opcode 0x66 0x0f 0x6d - invalid */
+/*  Opcode 0x66 0x0f 0x6e - invalid */
+/*  Opcode 0x66 0x0f 0x6f - invalid */
+
+/*  Opcodes 0x0f 0x70 thru 0x0f 0xb0 are unused.  */
+
+
+/*  Opcode      0x0f 0xc0 - invalid */
+/*  Opcode      0x0f 0xc1 - invalid */
+/*  Opcode      0x0f 0xc2 - invalid */
+/*  Opcode      0x0f 0xc3 - invalid */
+/*  Opcode      0x0f 0xc4 - invalid */
+/*  Opcode      0x0f 0xc5 - invalid */
+/*  Opcode      0x0f 0xc6 - invalid */
+/*  Opcode      0x0f 0xc7 - invalid */
+/*  Opcode      0x0f 0xc8 - invalid */
+/*  Opcode      0x0f 0xc9 - invalid */
+/*  Opcode      0x0f 0xca - invalid */
+/*  Opcode      0x0f 0xcb - invalid */
+/*  Opcode      0x0f 0xcc */
+FNIEMOP_STUB(iemOp_sha1rnds4_Vdq_Wdq_Ib);
+/*  Opcode      0x0f 0xcd - invalid */
+/*  Opcode      0x0f 0xce - invalid */
+/*  Opcode      0x0f 0xcf - invalid */
+
+
+/*  Opcode 0x66 0x0f 0xd0 - invalid */
+/*  Opcode 0x66 0x0f 0xd1 - invalid */
+/*  Opcode 0x66 0x0f 0xd2 - invalid */
+/*  Opcode 0x66 0x0f 0xd3 - invalid */
+/*  Opcode 0x66 0x0f 0xd4 - invalid */
+/*  Opcode 0x66 0x0f 0xd5 - invalid */
+/*  Opcode 0x66 0x0f 0xd6 - invalid */
+/*  Opcode 0x66 0x0f 0xd7 - invalid */
+/*  Opcode 0x66 0x0f 0xd8 - invalid */
+/*  Opcode 0x66 0x0f 0xd9 - invalid */
+/*  Opcode 0x66 0x0f 0xda - invalid */
+/*  Opcode 0x66 0x0f 0xdb - invalid */
+/*  Opcode 0x66 0x0f 0xdc - invalid */
+/*  Opcode 0x66 0x0f 0xdd - invalid */
+/*  Opcode 0x66 0x0f 0xde - invalid */
+/*  Opcode 0x66 0x0f 0xdf - (aeskeygenassist). */
+FNIEMOP_STUB(iemOp_aeskeygen_Vdq_Wdq_Ib);
+
+
+/*  Opcode 0xf2 0x0f 0xf0 - invalid (vex only) */
+
+
+/**
+ * Three byte opcode map, first two bytes are 0x0f 0x3a.
+ * @sa      g_apfnVexMap2
+ */
+IEM_STATIC const PFNIEMOP g_apfnThreeByte0f3a[] =
+{
+    /*          no prefix,                  066h prefix                 f3h prefix,                 f2h prefix */
+    /* 0x00 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x01 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x02 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x03 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x04 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x05 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x06 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x07 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x08 */  iemOp_InvalidNeedRMImm8,    iemOp_roundps_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x09 */  iemOp_InvalidNeedRMImm8,    iemOp_roundpd_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0a */  iemOp_InvalidNeedRMImm8,    iemOp_roundss_Vss_Wss_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0b */  iemOp_InvalidNeedRMImm8,    iemOp_roundsd_Vsd_Wsd_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0c */  iemOp_InvalidNeedRMImm8,    iemOp_blendps_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0d */  iemOp_InvalidNeedRMImm8,    iemOp_blendpd_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0e */  iemOp_InvalidNeedRMImm8,    iemOp_blendw_Vx_Wx_Ib,      iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0f */  iemOp_palignr_Pq_Qq_Ib,     iemOp_palignr_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+
+    /* 0x10 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x11 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x12 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x13 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x14 */  iemOp_InvalidNeedRMImm8,    iemOp_pextrb_RdMb_Vdq_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x15 */  iemOp_InvalidNeedRMImm8,    iemOp_pextrw_RdMw_Vdq_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x16 */  iemOp_InvalidNeedRMImm8,    iemOp_pextrd_q_RdMw_Vdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x17 */  iemOp_InvalidNeedRMImm8,    iemOp_extractps_Ed_Vdq_Ib,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x18 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x19 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x20 */  iemOp_InvalidNeedRMImm8,    iemOp_pinsrb_Vdq_RyMb_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x21 */  iemOp_InvalidNeedRMImm8,    iemOp_insertps_Vdq_UdqMd_Ib,iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x22 */  iemOp_InvalidNeedRMImm8,    iemOp_pinsrd_q_Vdq_Ey_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x23 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x24 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x25 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x26 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x27 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x28 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x29 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x30 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x31 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x32 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x33 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x34 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x35 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x36 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x37 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x38 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x39 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x40 */  iemOp_InvalidNeedRMImm8,    iemOp_dpps_Vx_Wx_Ib,        iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x41 */  iemOp_InvalidNeedRMImm8,    iemOp_dppd_Vdq_Wdq_Ib,      iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x42 */  iemOp_InvalidNeedRMImm8,    iemOp_mpsadbw_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x43 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x44 */  iemOp_InvalidNeedRMImm8,    iemOp_pclmulqdq_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x45 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x46 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x47 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x48 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x49 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x50 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x51 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x52 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x53 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x54 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x55 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x56 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x57 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x58 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x59 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x60 */  iemOp_InvalidNeedRMImm8,    iemOp_pcmpestrm_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x61 */  iemOp_InvalidNeedRMImm8,    iemOp_pcmpestri_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x62 */  iemOp_InvalidNeedRMImm8,    iemOp_pcmpistrm_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x63 */  iemOp_InvalidNeedRMImm8,    iemOp_pcmpistri_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x64 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x65 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x66 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x67 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x68 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x69 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x6a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x6b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x6c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x6d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x6e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x6f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x70 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x71 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x72 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x73 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x74 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x75 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x76 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x77 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x78 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x79 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x7a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x7b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x7c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x7d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x7e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x7f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x80 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x81 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x82 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x83 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x84 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x85 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x86 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x87 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x88 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x89 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x90 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x91 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x92 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x93 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x94 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x95 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x96 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x97 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x98 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x99 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xa0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xaa */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xab */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xac */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xad */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xae */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xaf */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xb0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xba */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbc */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbe */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbf */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xc0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xca */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xcb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xcc */  iemOp_sha1rnds4_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0xcd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xce */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xcf */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xd0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xda */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdc */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xde */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdf */  iemOp_aeskeygen_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+
+    /* 0xe0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xea */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xeb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xec */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xed */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xee */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xef */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xf0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfa */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfc */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfe */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xff */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+};
+AssertCompile(RT_ELEMENTS(g_apfnThreeByte0f3a) == 1024);
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h
new file mode 100644
index 00000000..7ab79d0c
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h
@@ -0,0 +1,9779 @@
+/* $Id: IEMAllInstructionsTwoByte0f.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation.
+ *
+ * @remarks IEMAllInstructionsVexMap1.cpp.h is a VEX mirror of this file.
+ *          Any update here is likely needed in that file too.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @name Two byte opcodes (first byte 0x0f).
+ *
+ * @{
+ */
+
+/** Opcode 0x0f 0x00 /0. */
+FNIEMOPRM_DEF(iemOp_Grp6_sldt)
+{
+    IEMOP_MNEMONIC(sldt, "sldt Rv/Mw");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DECODED_NL_1(OP_SLDT, IEMOPFORM_M_REG, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+        return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_sldt_reg, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, pVCpu->iem.s.enmEffOpSize);
+    }
+
+    /* Ignore operand size here, memory refs are always 16-bit. */
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint16_t, iEffSeg,               0);
+    IEM_MC_ARG(RTGCPTR,  GCPtrEffDst,           1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DECODED_NL_1(OP_SLDT, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_sldt_mem, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x00 /1. */
+FNIEMOPRM_DEF(iemOp_Grp6_str)
+{
+    IEMOP_MNEMONIC(str, "str Rv/Mw");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DECODED_NL_1(OP_STR, IEMOPFORM_M_REG, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+        return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_str_reg, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, pVCpu->iem.s.enmEffOpSize);
+    }
+
+    /* Ignore operand size here, memory refs are always 16-bit. */
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint16_t, iEffSeg,               0);
+    IEM_MC_ARG(RTGCPTR,  GCPtrEffDst,           1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DECODED_NL_1(OP_STR, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_str_mem, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x00 /2. */
+FNIEMOPRM_DEF(iemOp_Grp6_lldt)
+{
+    IEMOP_MNEMONIC(lldt, "lldt Ew");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DECODED_NL_1(OP_LLDT, IEMOPFORM_M_REG, OP_PARM_Ew, DISOPTYPE_DANGEROUS);
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_ARG(uint16_t, u16Sel, 0);
+        IEM_MC_FETCH_GREG_U16(u16Sel, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_CALL_CIMPL_1(iemCImpl_lldt, u16Sel);
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(1, 1);
+        IEM_MC_ARG(uint16_t, u16Sel, 0);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DECODED_NL_1(OP_LLDT, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS);
+        IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO(); /** @todo test order */
+        IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_CALL_CIMPL_1(iemCImpl_lldt, u16Sel);
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x00 /3. */
+FNIEMOPRM_DEF(iemOp_Grp6_ltr)
+{
+    IEMOP_MNEMONIC(ltr, "ltr Ew");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_ARG(uint16_t, u16Sel, 0);
+        IEM_MC_FETCH_GREG_U16(u16Sel, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_CALL_CIMPL_1(iemCImpl_ltr, u16Sel);
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(1, 1);
+        IEM_MC_ARG(uint16_t, u16Sel, 0);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO(); /** @todo test order */
+        IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_CALL_CIMPL_1(iemCImpl_ltr, u16Sel);
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x00 /3. */
+FNIEMOP_DEF_2(iemOpCommonGrp6VerX, uint8_t, bRm, bool, fWrite)
+{
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DECODED_NL_1(fWrite ? OP_VERW : OP_VERR, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint16_t,    u16Sel,            0);
+        IEM_MC_ARG_CONST(bool,  fWriteArg, fWrite, 1);
+        IEM_MC_FETCH_GREG_U16(u16Sel, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_CALL_CIMPL_2(iemCImpl_VerX, u16Sel, fWriteArg);
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(2, 1);
+        IEM_MC_ARG(uint16_t,    u16Sel,            0);
+        IEM_MC_ARG_CONST(bool,  fWriteArg, fWrite, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DECODED_NL_1(fWrite ? OP_VERW : OP_VERR, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+        IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_CALL_CIMPL_2(iemCImpl_VerX, u16Sel, fWriteArg);
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x00 /4. */
+FNIEMOPRM_DEF(iemOp_Grp6_verr)
+{
+    IEMOP_MNEMONIC(verr, "verr Ew");
+    IEMOP_HLP_MIN_286();
+    return FNIEMOP_CALL_2(iemOpCommonGrp6VerX, bRm, false);
+}
+
+
+/** Opcode 0x0f 0x00 /5. */
+FNIEMOPRM_DEF(iemOp_Grp6_verw)
+{
+    IEMOP_MNEMONIC(verw, "verw Ew");
+    IEMOP_HLP_MIN_286();
+    return FNIEMOP_CALL_2(iemOpCommonGrp6VerX, bRm, true);
+}
+
+
+/**
+ * Group 6 jump table.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup6[8] =
+{
+    iemOp_Grp6_sldt,
+    iemOp_Grp6_str,
+    iemOp_Grp6_lldt,
+    iemOp_Grp6_ltr,
+    iemOp_Grp6_verr,
+    iemOp_Grp6_verw,
+    iemOp_InvalidWithRM,
+    iemOp_InvalidWithRM
+};
+
+/** Opcode 0x0f 0x00. */
+FNIEMOP_DEF(iemOp_Grp6)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    return FNIEMOP_CALL_1(g_apfnGroup6[(bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK], bRm);
+}
+
+
+/** Opcode 0x0f 0x01 /0. */
+FNIEMOP_DEF_1(iemOp_Grp7_sgdt, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(sgdt, "sgdt Ms");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_64BIT_OP_SIZE();
+    IEM_MC_BEGIN(2, 1);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                    0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEffSrc,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_sgdt, iEffSeg, GCPtrEffSrc);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 /0. */
+FNIEMOP_DEF(iemOp_Grp7_vmcall)
+{
+    IEMOP_MNEMONIC(vmcall, "vmcall");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the VMX instructions. ASSUMING no lock for now. */
+
+    /* Note! We do not check any CPUMFEATURES::fSvm here as we (GIM) generally
+             want all hypercalls regardless of instruction used, and if a
+             hypercall isn't handled by GIM or HMSvm will raise an #UD.
+             (NEM/win makes ASSUMPTIONS about this behavior.)  */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmcall);
+}
+
+
+/** Opcode 0x0f 0x01 /0. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF(iemOp_Grp7_vmlaunch)
+{
+    IEMOP_MNEMONIC(vmlaunch, "vmlaunch");
+    IEMOP_HLP_IN_VMX_OPERATION("vmlaunch", kVmxVDiag_Vmentry);
+    IEMOP_HLP_VMX_INSTR("vmlaunch", kVmxVDiag_Vmentry);
+    IEMOP_HLP_DONE_DECODING();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmlaunch);
+}
+#else
+FNIEMOP_DEF(iemOp_Grp7_vmlaunch)
+{
+    IEMOP_BITCH_ABOUT_STUB();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+#endif
+
+
+/** Opcode 0x0f 0x01 /0. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF(iemOp_Grp7_vmresume)
+{
+    IEMOP_MNEMONIC(vmresume, "vmresume");
+    IEMOP_HLP_IN_VMX_OPERATION("vmresume", kVmxVDiag_Vmentry);
+    IEMOP_HLP_VMX_INSTR("vmresume", kVmxVDiag_Vmentry);
+    IEMOP_HLP_DONE_DECODING();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmresume);
+}
+#else
+FNIEMOP_DEF(iemOp_Grp7_vmresume)
+{
+    IEMOP_BITCH_ABOUT_STUB();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+#endif
+
+
+/** Opcode 0x0f 0x01 /0. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF(iemOp_Grp7_vmxoff)
+{
+    IEMOP_MNEMONIC(vmxoff, "vmxoff");
+    IEMOP_HLP_IN_VMX_OPERATION("vmxoff", kVmxVDiag_Vmxoff);
+    IEMOP_HLP_VMX_INSTR("vmxoff", kVmxVDiag_Vmxoff);
+    IEMOP_HLP_DONE_DECODING();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmxoff);
+}
+#else
+FNIEMOP_DEF(iemOp_Grp7_vmxoff)
+{
+    IEMOP_BITCH_ABOUT_STUB();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+#endif
+
+
+/** Opcode 0x0f 0x01 /1. */
+FNIEMOP_DEF_1(iemOp_Grp7_sidt, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(sidt, "sidt Ms");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_64BIT_OP_SIZE();
+    IEM_MC_BEGIN(2, 1);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                    0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEffSrc,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_sidt, iEffSeg, GCPtrEffSrc);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 /1. */
+FNIEMOP_DEF(iemOp_Grp7_monitor)
+{
+    IEMOP_MNEMONIC(monitor, "monitor");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo Verify that monitor is allergic to lock prefixes. */
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_monitor, pVCpu->iem.s.iEffSeg);
+}
+
+
+/** Opcode 0x0f 0x01 /1. */
+FNIEMOP_DEF(iemOp_Grp7_mwait)
+{
+    IEMOP_MNEMONIC(mwait, "mwait"); /** @todo Verify that mwait is allergic to lock prefixes. */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_mwait);
+}
+
+
+/** Opcode 0x0f 0x01 /2. */
+FNIEMOP_DEF_1(iemOp_Grp7_lgdt, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(lgdt, "lgdt");
+    IEMOP_HLP_64BIT_OP_SIZE();
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                    0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEffSrc,                                1);
+    IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSizeArg,/*=*/pVCpu->iem.s.enmEffOpSize, 2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_lgdt, iEffSeg, GCPtrEffSrc, enmEffOpSizeArg);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 0xd0. */
+FNIEMOP_DEF(iemOp_Grp7_xgetbv)
+{
+    IEMOP_MNEMONIC(xgetbv, "xgetbv");
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXSaveRstor)
+    {
+        /** @todo r=ramshankar: We should use
+         *        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX and
+         *        IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES here. */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_REPZ_OR_REPNZ_PREFIXES();
+        return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_xgetbv);
+    }
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode 0x0f 0x01 0xd1. */
+FNIEMOP_DEF(iemOp_Grp7_xsetbv)
+{
+    IEMOP_MNEMONIC(xsetbv, "xsetbv");
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXSaveRstor)
+    {
+        /** @todo r=ramshankar: We should use
+         *        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX and
+         *        IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES here. */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_REPZ_OR_REPNZ_PREFIXES();
+        return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_xsetbv);
+    }
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode 0x0f 0x01 /3. */
+FNIEMOP_DEF_1(iemOp_Grp7_lidt, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(lidt, "lidt");
+    IEMMODE enmEffOpSize = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+                         ? IEMMODE_64BIT
+                         : pVCpu->iem.s.enmEffOpSize;
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                            0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEffSrc,                        1);
+    IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSizeArg,/*=*/enmEffOpSize,  2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_lidt, iEffSeg, GCPtrEffSrc, enmEffOpSizeArg);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 0xd8. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_vmrun)
+{
+    IEMOP_MNEMONIC(vmrun, "vmrun");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmrun);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_vmrun);
+#endif
+
+/** Opcode 0x0f 0x01 0xd9. */
+FNIEMOP_DEF(iemOp_Grp7_Amd_vmmcall)
+{
+    IEMOP_MNEMONIC(vmmcall, "vmmcall");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+
+    /* Note! We do not check any CPUMFEATURES::fSvm here as we (GIM) generally
+             want all hypercalls regardless of instruction used, and if a
+             hypercall isn't handled by GIM or HMSvm will raise an #UD.
+             (NEM/win makes ASSUMPTIONS about this behavior.) */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmmcall);
+}
+
+/** Opcode 0x0f 0x01 0xda. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_vmload)
+{
+    IEMOP_MNEMONIC(vmload, "vmload");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmload);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_vmload);
+#endif
+
+
+/** Opcode 0x0f 0x01 0xdb. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_vmsave)
+{
+    IEMOP_MNEMONIC(vmsave, "vmsave");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmsave);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_vmsave);
+#endif
+
+
+/** Opcode 0x0f 0x01 0xdc. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_stgi)
+{
+    IEMOP_MNEMONIC(stgi, "stgi");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stgi);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_stgi);
+#endif
+
+
+/** Opcode 0x0f 0x01 0xdd. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_clgi)
+{
+    IEMOP_MNEMONIC(clgi, "clgi");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_clgi);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_clgi);
+#endif
+
+
+/** Opcode 0x0f 0x01 0xdf. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_invlpga)
+{
+    IEMOP_MNEMONIC(invlpga, "invlpga");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_invlpga);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_invlpga);
+#endif
+
+
+/** Opcode 0x0f 0x01 0xde. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_skinit)
+{
+    IEMOP_MNEMONIC(skinit, "skinit");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_skinit);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_skinit);
+#endif
+
+
+/** Opcode 0x0f 0x01 /4. */
+FNIEMOP_DEF_1(iemOp_Grp7_smsw, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(smsw, "smsw");
+    IEMOP_HLP_MIN_286();
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_smsw_reg, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, pVCpu->iem.s.enmEffOpSize);
+    }
+
+    /* Ignore operand size here, memory refs are always 16-bit. */
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint16_t, iEffSeg,               0);
+    IEM_MC_ARG(RTGCPTR,  GCPtrEffDst,           1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_smsw_mem, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 /6. */
+FNIEMOP_DEF_1(iemOp_Grp7_lmsw, uint8_t, bRm)
+{
+    /* The operand size is effectively ignored, all is 16-bit and only the
+       lower 3-bits are used. */
+    IEMOP_MNEMONIC(lmsw, "lmsw");
+    IEMOP_HLP_MIN_286();
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint16_t, u16Tmp,                         0);
+        IEM_MC_ARG_CONST(RTGCPTR,  GCPtrEffDst, NIL_RTGCPTR, 1);
+        IEM_MC_FETCH_GREG_U16(u16Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_CALL_CIMPL_2(iemCImpl_lmsw, u16Tmp, GCPtrEffDst);
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint16_t, u16Tmp,      0);
+        IEM_MC_ARG(RTGCPTR,  GCPtrEffDst, 1);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+        IEM_MC_CALL_CIMPL_2(iemCImpl_lmsw, u16Tmp, GCPtrEffDst);
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 /7. */
+FNIEMOP_DEF_1(iemOp_Grp7_invlpg, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(invlpg, "invlpg");
+    IEMOP_HLP_MIN_486();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(1, 1);
+    IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 0);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEM_MC_CALL_CIMPL_1(iemCImpl_invlpg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 /7. */
+FNIEMOP_DEF(iemOp_Grp7_swapgs)
+{
+    IEMOP_MNEMONIC(swapgs, "swapgs");
+    IEMOP_HLP_ONLY_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_swapgs);
+}
+
+
+/** Opcode 0x0f 0x01 /7. */
+FNIEMOP_DEF(iemOp_Grp7_rdtscp)
+{
+    IEMOP_MNEMONIC(rdtscp, "rdtscp");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rdtscp);
+}
+
+
+/**
+ * Group 7 jump table, memory variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup7Mem[8] =
+{
+    iemOp_Grp7_sgdt,
+    iemOp_Grp7_sidt,
+    iemOp_Grp7_lgdt,
+    iemOp_Grp7_lidt,
+    iemOp_Grp7_smsw,
+    iemOp_InvalidWithRM,
+    iemOp_Grp7_lmsw,
+    iemOp_Grp7_invlpg
+};
+
+
+/** Opcode 0x0f 0x01. */
+FNIEMOP_DEF(iemOp_Grp7)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+        return FNIEMOP_CALL_1(g_apfnGroup7Mem[(bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK], bRm);
+
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0:
+            switch (bRm & X86_MODRM_RM_MASK)
+            {
+                case 1: return FNIEMOP_CALL(iemOp_Grp7_vmcall);
+                case 2: return FNIEMOP_CALL(iemOp_Grp7_vmlaunch);
+                case 3: return FNIEMOP_CALL(iemOp_Grp7_vmresume);
+                case 4: return FNIEMOP_CALL(iemOp_Grp7_vmxoff);
+            }
+            return IEMOP_RAISE_INVALID_OPCODE();
+
+        case 1:
+            switch (bRm & X86_MODRM_RM_MASK)
+            {
+                case 0: return FNIEMOP_CALL(iemOp_Grp7_monitor);
+                case 1: return FNIEMOP_CALL(iemOp_Grp7_mwait);
+            }
+            return IEMOP_RAISE_INVALID_OPCODE();
+
+        case 2:
+            switch (bRm & X86_MODRM_RM_MASK)
+            {
+                case 0: return FNIEMOP_CALL(iemOp_Grp7_xgetbv);
+                case 1: return FNIEMOP_CALL(iemOp_Grp7_xsetbv);
+            }
+            return IEMOP_RAISE_INVALID_OPCODE();
+
+        case 3:
+            switch (bRm & X86_MODRM_RM_MASK)
+            {
+                case 0: return FNIEMOP_CALL(iemOp_Grp7_Amd_vmrun);
+                case 1: return FNIEMOP_CALL(iemOp_Grp7_Amd_vmmcall);
+                case 2: return FNIEMOP_CALL(iemOp_Grp7_Amd_vmload);
+                case 3: return FNIEMOP_CALL(iemOp_Grp7_Amd_vmsave);
+                case 4: return FNIEMOP_CALL(iemOp_Grp7_Amd_stgi);
+                case 5: return FNIEMOP_CALL(iemOp_Grp7_Amd_clgi);
+                case 6: return FNIEMOP_CALL(iemOp_Grp7_Amd_skinit);
+                case 7: return FNIEMOP_CALL(iemOp_Grp7_Amd_invlpga);
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+
+        case 4:
+            return FNIEMOP_CALL_1(iemOp_Grp7_smsw, bRm);
+
+        case 5:
+            return IEMOP_RAISE_INVALID_OPCODE();
+
+        case 6:
+            return FNIEMOP_CALL_1(iemOp_Grp7_lmsw, bRm);
+
+        case 7:
+            switch (bRm & X86_MODRM_RM_MASK)
+            {
+                case 0: return FNIEMOP_CALL(iemOp_Grp7_swapgs);
+                case 1: return FNIEMOP_CALL(iemOp_Grp7_rdtscp);
+            }
+            return IEMOP_RAISE_INVALID_OPCODE();
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+/** Opcode 0x0f 0x00 /3. */
+FNIEMOP_DEF_1(iemOpCommonLarLsl_Gv_Ew, bool, fIsLar)
+{
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DECODED_NL_2(fIsLar ? OP_LAR : OP_LSL, IEMOPFORM_RM_REG, OP_PARM_Gv, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,  pu16Dst,           0);
+                IEM_MC_ARG(uint16_t,    u16Sel,            1);
+                IEM_MC_ARG_CONST(bool,  fIsLarArg, fIsLar, 2);
+
+                IEM_MC_REF_GREG_U16(pu16Dst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_GREG_U16(u16Sel, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_CALL_CIMPL_3(iemCImpl_LarLsl_u16, pu16Dst, u16Sel, fIsLarArg);
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            case IEMMODE_32BIT:
+            case IEMMODE_64BIT:
+            {
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,  pu64Dst,           0);
+                IEM_MC_ARG(uint16_t,    u16Sel,            1);
+                IEM_MC_ARG_CONST(bool,  fIsLarArg, fIsLar, 2);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_GREG_U16(u16Sel, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_CALL_CIMPL_3(iemCImpl_LarLsl_u64, pu64Dst, u16Sel, fIsLarArg);
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t *,  pu16Dst,           0);
+                IEM_MC_ARG(uint16_t,    u16Sel,            1);
+                IEM_MC_ARG_CONST(bool,  fIsLarArg, fIsLar, 2);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DECODED_NL_2(fIsLar ? OP_LAR : OP_LSL, IEMOPFORM_RM_MEM, OP_PARM_Gv, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+
+                IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_REF_GREG_U16(pu16Dst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_CALL_CIMPL_3(iemCImpl_LarLsl_u16, pu16Dst, u16Sel, fIsLarArg);
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            case IEMMODE_32BIT:
+            case IEMMODE_64BIT:
+            {
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint64_t *,  pu64Dst,           0);
+                IEM_MC_ARG(uint16_t,    u16Sel,            1);
+                IEM_MC_ARG_CONST(bool,  fIsLarArg, fIsLar, 2);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DECODED_NL_2(fIsLar ? OP_LAR : OP_LSL, IEMOPFORM_RM_MEM, OP_PARM_Gv, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+/** @todo testcase: make sure it's a 16-bit read. */
+
+                IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_REF_GREG_U64(pu64Dst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_CALL_CIMPL_3(iemCImpl_LarLsl_u64, pu64Dst, u16Sel, fIsLarArg);
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+
+/** Opcode 0x0f 0x02. */
+FNIEMOP_DEF(iemOp_lar_Gv_Ew)
+{
+    IEMOP_MNEMONIC(lar, "lar Gv,Ew");
+    return FNIEMOP_CALL_1(iemOpCommonLarLsl_Gv_Ew, true);
+}
+
+
+/** Opcode 0x0f 0x03. */
+FNIEMOP_DEF(iemOp_lsl_Gv_Ew)
+{
+    IEMOP_MNEMONIC(lsl, "lsl Gv,Ew");
+    return FNIEMOP_CALL_1(iemOpCommonLarLsl_Gv_Ew, false);
+}
+
+
+/** Opcode 0x0f 0x05. */
+FNIEMOP_DEF(iemOp_syscall)
+{
+    IEMOP_MNEMONIC(syscall, "syscall"); /** @todo 286 LOADALL   */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_syscall);
+}
+
+
+/** Opcode 0x0f 0x06. */
+FNIEMOP_DEF(iemOp_clts)
+{
+    IEMOP_MNEMONIC(clts, "clts");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_clts);
+}
+
+
+/** Opcode 0x0f 0x07. */
+FNIEMOP_DEF(iemOp_sysret)
+{
+    IEMOP_MNEMONIC(sysret, "sysret");  /** @todo 386 LOADALL   */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_sysret);
+}
+
+
+/** Opcode 0x0f 0x08. */
+FNIEMOP_DEF(iemOp_invd)
+{
+    IEMOP_MNEMONIC0(FIXED, INVD, invd, DISOPTYPE_PRIVILEGED, 0);
+    IEMOP_HLP_MIN_486();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_invd);
+}
+
+
+/** Opcode 0x0f 0x09. */
+FNIEMOP_DEF(iemOp_wbinvd)
+{
+    IEMOP_MNEMONIC0(FIXED, WBINVD, wbinvd, DISOPTYPE_PRIVILEGED, 0);
+    IEMOP_HLP_MIN_486();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_wbinvd);
+}
+
+
+/** Opcode 0x0f 0x0b. */
+FNIEMOP_DEF(iemOp_ud2)
+{
+    IEMOP_MNEMONIC(ud2, "ud2");
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/** Opcode 0x0f 0x0d. */
+FNIEMOP_DEF(iemOp_nop_Ev_GrpP)
+{
+    /* AMD prefetch group, Intel implements this as NOP Ev (and so do we). */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->f3DNowPrefetch)
+    {
+        IEMOP_MNEMONIC(GrpPNotSupported, "GrpP");
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_MNEMONIC(GrpPInvalid, "GrpP");
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 2: /* Aliased to /0 for the time being. */
+        case 4: /* Aliased to /0 for the time being. */
+        case 5: /* Aliased to /0 for the time being. */
+        case 6: /* Aliased to /0 for the time being. */
+        case 7: /* Aliased to /0 for the time being. */
+        case 0: IEMOP_MNEMONIC(prefetch, "prefetch"); break;
+        case 1: IEMOP_MNEMONIC(prefetchw_1, "prefetchw"); break;
+        case 3: IEMOP_MNEMONIC(prefetchw_3, "prefetchw"); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    /* Currently a NOP. */
+    NOREF(GCPtrEffSrc);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x0e. */
+FNIEMOP_DEF(iemOp_femms)
+{
+    IEMOP_MNEMONIC(femms, "femms");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FPU_FROM_MMX_MODE();
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x0f. */
+FNIEMOP_DEF(iemOp_3Dnow)
+{
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->f3DNow)
+    {
+        IEMOP_MNEMONIC(Inv3Dnow, "3Dnow");
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+
+#ifdef IEM_WITH_3DNOW
+    /* This is pretty sparse, use switch instead of table. */
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL_1(iemOp_3DNowDispatcher, b);
+#else
+    IEMOP_BITCH_ABOUT_STUB();
+    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/**
+ * @opcode      0x10
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_movups_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, MOVUPS, movups, Vps_WO, Wps, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                              (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+
+}
+
+
+/**
+ * @opcode      0x10
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movupd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, MOVUPD, movupd, Vpd_WO, Wpd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                              (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x10
+ * @oppfx       0xf3
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_movss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, MOVSS, movss, VssZx_WO, Wss, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_XREG_U32(uSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_STORE_XREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U32_ZX_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x10
+ * @oppfx       0xf2
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movsd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, MOVSD, movsd, VsdZx_WO, Wsd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_XREG_U64(uSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_STORE_XREG_U64_ZX_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U64_ZX_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x11
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movups_Wps_Vps)
+{
+    IEMOP_MNEMONIC2(MR, MOVUPS, movups, Wps_WO, Vps, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                              ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x11
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movupd_Wpd_Vpd)
+{
+    IEMOP_MNEMONIC2(MR, MOVUPD, movupd, Wpd_WO, Vpd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                              ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x11
+ * @oppfx       0xf3
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_movss_Wss_Vss)
+{
+    IEMOP_MNEMONIC2(MR, MOVSS, movss, Wss_WO, Vss, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_XREG_U32(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_XREG_U32((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U32(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x11
+ * @oppfx       0xf2
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movsd_Wsd_Vsd)
+{
+    IEMOP_MNEMONIC2(MR, MOVSD, movsd, Wsd_WO, Vsd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_XREG_U64(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_XREG_U64((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+FNIEMOP_DEF(iemOp_movlps_Vq_Mq__movhlps)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /**
+         * @opcode      0x12
+         * @opcodesub   11 mr/reg
+         * @oppfx       none
+         * @opcpuid     sse
+         * @opgroup     og_sse_simdfp_datamove
+         * @opxcpttype  5
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(RM_REG, MOVHLPS, movhlps, Vq_WO, UqHi, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_XREG_HI_U64(uSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_STORE_XREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x12
+         * @opcodesub   !11 mr/reg
+         * @oppfx       none
+         * @opcpuid     sse
+         * @opgroup     og_sse_simdfp_datamove
+         * @opxcpttype  5
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         * @opfunction  iemOp_movlps_Vq_Mq__vmovhlps
+         */
+        IEMOP_MNEMONIC2(RM_MEM, MOVLPS, movlps, Vq_WO, Mq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x12
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movlpd_Vq_Mq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_MNEMONIC2(RM_MEM, MOVLPD, movlpd, Vq_WO, Mq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud660f12m3
+     * @opcode      0x12
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x12
+ * @oppfx       0xf3
+ * @opcpuid     sse3
+ * @opgroup     og_sse3_pcksclr_datamove
+ * @opxcpttype  4
+ * @optest      op1=-1 op2=0xdddddddd00000002eeeeeeee00000001 ->
+ *              op1=0x00000002000000020000000100000001
+ */
+FNIEMOP_DEF(iemOp_movsldup_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, MOVSLDUP, movsldup, Vdq_WO, Wdq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc, 1);
+
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_REF_XREG_U128_CONST(puSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_XREG_U128(puDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_SSE_AIMPL_2(iemAImpl_movsldup, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_REF_XREG_U128(puDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_SSE_AIMPL_2(iemAImpl_movsldup, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x12
+ * @oppfx       0xf2
+ * @opcpuid     sse3
+ * @opgroup     og_sse3_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=-1 op2=0xddddddddeeeeeeee2222222211111111 ->
+ *              op1=0x22222222111111112222222211111111
+ */
+FNIEMOP_DEF(iemOp_movddup_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, MOVDDUP, movddup, Vdq_WO, Wdq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+        IEM_MC_ARG(uint64_t,                    uSrc, 1);
+
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_XREG_U128(puDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_SSE_AIMPL_2(iemAImpl_movddup, puDst, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+        IEM_MC_ARG(uint64_t,                    uSrc, 1);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_REF_XREG_U128(puDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_SSE_AIMPL_2(iemAImpl_movddup, puDst, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x13
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movlps_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_MNEMONIC2(MR_MEM, MOVLPS, movlps, Mq_WO, Vq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud0f13m3
+     * @opcode      0x13
+     * @opcodesub   11 mr/reg
+     * @oppfx       none
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x13
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movlpd_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_MNEMONIC2(MR_MEM, MOVLPD, movlpd, Mq_WO, Vq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud660f13m3
+     * @opcode      0x13
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opmnemonic  udf30f13
+ * @opcode      0x13
+ * @oppfx       0xf3
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udf20f13
+ * @opcode      0x13
+ * @oppfx       0xf2
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+/** Opcode      0x0f 0x14 - unpcklps Vx, Wx*/
+FNIEMOP_STUB(iemOp_unpcklps_Vx_Wx);
+/** Opcode 0x66 0x0f 0x14 - unpcklpd Vx, Wx   */
+FNIEMOP_STUB(iemOp_unpcklpd_Vx_Wx);
+
+/**
+ * @opdone
+ * @opmnemonic  udf30f14
+ * @opcode      0x14
+ * @oppfx       0xf3
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udf20f14
+ * @opcode      0x14
+ * @oppfx       0xf2
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+/** Opcode      0x0f 0x15 - unpckhps Vx, Wx   */
+FNIEMOP_STUB(iemOp_unpckhps_Vx_Wx);
+/** Opcode 0x66 0x0f 0x15 - unpckhpd Vx, Wx   */
+FNIEMOP_STUB(iemOp_unpckhpd_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0x15 - invalid */
+/*  Opcode 0xf2 0x0f 0x15 - invalid */
+
+/**
+ * @opdone
+ * @opmnemonic  udf30f15
+ * @opcode      0x15
+ * @oppfx       0xf3
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udf20f15
+ * @opcode      0x15
+ * @oppfx       0xf2
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+FNIEMOP_DEF(iemOp_movhps_Vdq_Mq__movlhps_Vdq_Uq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /**
+         * @opcode      0x16
+         * @opcodesub   11 mr/reg
+         * @oppfx       none
+         * @opcpuid     sse
+         * @opgroup     og_sse_simdfp_datamove
+         * @opxcpttype  5
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(RM_REG, MOVLHPS, movlhps, VqHi_WO, Uq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_XREG_U64(uSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_STORE_XREG_HI_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x16
+         * @opcodesub   !11 mr/reg
+         * @oppfx       none
+         * @opcpuid     sse
+         * @opgroup     og_sse_simdfp_datamove
+         * @opxcpttype  5
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         * @opfunction  iemOp_movhps_Vdq_Mq__movlhps_Vdq_Uq
+         */
+        IEMOP_MNEMONIC2(RM_MEM, MOVHPS, movhps, VqHi_WO, Mq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_HI_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x16
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movhpd_Vdq_Mq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_MNEMONIC2(RM_MEM, MOVHPD, movhpd, VqHi_WO, Mq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_HI_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud660f16m3
+     * @opcode      0x16
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x16
+ * @oppfx       0xf3
+ * @opcpuid     sse3
+ * @opgroup     og_sse3_pcksclr_datamove
+ * @opxcpttype  4
+ * @optest      op1=-1 op2=0x00000002dddddddd00000001eeeeeeee ->
+ *              op1=0x00000002000000020000000100000001
+ */
+FNIEMOP_DEF(iemOp_movshdup_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, MOVSHDUP, movshdup, Vdq_WO, Wdq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc, 1);
+
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_REF_XREG_U128_CONST(puSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_XREG_U128(puDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_SSE_AIMPL_2(iemAImpl_movshdup, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_REF_XREG_U128(puDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_SSE_AIMPL_2(iemAImpl_movshdup, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opdone
+ * @opmnemonic  udf30f16
+ * @opcode      0x16
+ * @oppfx       0xf2
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+
+/**
+ * @opcode      0x17
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movhps_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_MNEMONIC2(MR_MEM, MOVHPS, movhps, Mq_WO, VqHi, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_HI_U64(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud0f17m3
+     * @opcode      0x17
+     * @opcodesub   11 mr/reg
+     * @oppfx       none
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x17
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movhpd_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_MNEMONIC2(MR_MEM, MOVHPD, movhpd, Mq_WO, VqHi, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_HI_U64(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud660f17m3
+     * @opcode      0x17
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opdone
+ * @opmnemonic  udf30f17
+ * @opcode      0x17
+ * @oppfx       0xf3
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udf20f17
+ * @opcode      0x17
+ * @oppfx       0xf2
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+
+/** Opcode 0x0f 0x18. */
+FNIEMOP_DEF(iemOp_prefetch_Grp16)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+        {
+            case 4: /* Aliased to /0 for the time being according to AMD. */
+            case 5: /* Aliased to /0 for the time being according to AMD. */
+            case 6: /* Aliased to /0 for the time being according to AMD. */
+            case 7: /* Aliased to /0 for the time being according to AMD. */
+            case 0: IEMOP_MNEMONIC(prefetchNTA, "prefetchNTA m8"); break;
+            case 1: IEMOP_MNEMONIC(prefetchT0, "prefetchT0  m8"); break;
+            case 2: IEMOP_MNEMONIC(prefetchT1, "prefetchT1  m8"); break;
+            case 3: IEMOP_MNEMONIC(prefetchT2, "prefetchT2  m8"); break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        /* Currently a NOP. */
+        NOREF(GCPtrEffSrc);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode 0x0f 0x19..0x1f. */
+FNIEMOP_DEF(iemOp_nop_Ev)
+{
+    IEMOP_MNEMONIC(nop_Ev, "nop Ev");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        /* Currently a NOP. */
+        NOREF(GCPtrEffSrc);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x20. */
+FNIEMOP_DEF(iemOp_mov_Rd_Cd)
+{
+    /* mod is ignored, as is operand size overrides. */
+    IEMOP_MNEMONIC(mov_Rd_Cd, "mov Rd,Cd");
+    IEMOP_HLP_MIN_386();
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
+    else
+        pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    uint8_t iCrReg = ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg;
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)
+    {
+        /* The lock prefix can be used to encode CR8 accesses on some CPUs. */
+        if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovCr8In32Bit)
+            return IEMOP_RAISE_INVALID_OPCODE(); /* #UD takes precedence over #GP(), see test. */
+        iCrReg |= 8;
+    }
+    switch (iCrReg)
+    {
+        case 0: case 2: case 3: case 4: case 8:
+            break;
+        default:
+            return IEMOP_RAISE_INVALID_OPCODE();
+    }
+    IEMOP_HLP_DONE_DECODING();
+
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Rd_Cd, (X86_MODRM_RM_MASK & bRm) | pVCpu->iem.s.uRexB, iCrReg);
+}
+
+
+/** Opcode 0x0f 0x21. */
+FNIEMOP_DEF(iemOp_mov_Rd_Dd)
+{
+    IEMOP_MNEMONIC(mov_Rd_Dd, "mov Rd,Dd");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX_R)
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Rd_Dd,
+                                   (X86_MODRM_RM_MASK & bRm) | pVCpu->iem.s.uRexB,
+                                   ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK));
+}
+
+
+/** Opcode 0x0f 0x22. */
+FNIEMOP_DEF(iemOp_mov_Cd_Rd)
+{
+    /* mod is ignored, as is operand size overrides. */
+    IEMOP_MNEMONIC(mov_Cd_Rd, "mov Cd,Rd");
+    IEMOP_HLP_MIN_386();
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
+    else
+        pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    uint8_t iCrReg = ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg;
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)
+    {
+        /* The lock prefix can be used to encode CR8 accesses on some CPUs. */
+        if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovCr8In32Bit)
+            return IEMOP_RAISE_INVALID_OPCODE(); /* #UD takes precedence over #GP(), see test. */
+        iCrReg |= 8;
+    }
+    switch (iCrReg)
+    {
+        case 0: case 2: case 3: case 4: case 8:
+            break;
+        default:
+            return IEMOP_RAISE_INVALID_OPCODE();
+    }
+    IEMOP_HLP_DONE_DECODING();
+
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Cd_Rd, iCrReg, (X86_MODRM_RM_MASK & bRm) | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0x23. */
+FNIEMOP_DEF(iemOp_mov_Dd_Rd)
+{
+    IEMOP_MNEMONIC(mov_Dd_Rd, "mov Dd,Rd");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX_R)
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Dd_Rd,
+                                   ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK),
+                                   (X86_MODRM_RM_MASK & bRm) | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0x24. */
+FNIEMOP_DEF(iemOp_mov_Rd_Td)
+{
+    IEMOP_MNEMONIC(mov_Rd_Td, "mov Rd,Td");
+    /** @todo works on 386 and 486. */
+    /* The RM byte is not considered, see testcase. */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode 0x0f 0x26. */
+FNIEMOP_DEF(iemOp_mov_Td_Rd)
+{
+    IEMOP_MNEMONIC(mov_Td_Rd, "mov Td,Rd");
+    /** @todo works on 386 and 486. */
+    /* The RM byte is not considered, see testcase. */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x28
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movaps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, MOVAPS, movaps, Vps_WO, Wps, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                              (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opcode      0x28
+ * @oppfx       66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movapd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, MOVAPD, movapd, Vpd_WO, Wpd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                              (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/*  Opcode 0xf3 0x0f 0x28 - invalid */
+/*  Opcode 0xf2 0x0f 0x28 - invalid */
+
+/**
+ * @opcode      0x29
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movaps_Wps_Vps)
+{
+    IEMOP_MNEMONIC2(MR, MOVAPS, movaps, Wps_WO, Vps, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                              ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opcode      0x29
+ * @oppfx       66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movapd_Wpd_Vpd)
+{
+    IEMOP_MNEMONIC2(MR, MOVAPD, movapd, Wpd_WO, Vpd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                              ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/*  Opcode 0xf3 0x0f 0x29 - invalid */
+/*  Opcode 0xf2 0x0f 0x29 - invalid */
+
+
+/** Opcode      0x0f 0x2a - cvtpi2ps Vps, Qpi */
+FNIEMOP_STUB(iemOp_cvtpi2ps_Vps_Qpi); //NEXT
+/** Opcode 0x66 0x0f 0x2a - cvtpi2pd Vpd, Qpi */
+FNIEMOP_STUB(iemOp_cvtpi2pd_Vpd_Qpi); //NEXT
+/** Opcode 0xf3 0x0f 0x2a - vcvtsi2ss Vss, Hss, Ey */
+FNIEMOP_STUB(iemOp_cvtsi2ss_Vss_Ey); //NEXT
+/** Opcode 0xf2 0x0f 0x2a - vcvtsi2sd Vsd, Hsd, Ey */
+FNIEMOP_STUB(iemOp_cvtsi2sd_Vsd_Ey); //NEXT
+
+
+/**
+ * @opcode      0x2b
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse1_cachect
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movntps_Mps_Vps)
+{
+    IEMOP_MNEMONIC2(MR_MEM, MOVNTPS, movntps, Mps_WO, Vps, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    /* The register, register encoding is invalid. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opcode      0x2b
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_cachect
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movntpd_Mpd_Vpd)
+{
+    IEMOP_MNEMONIC2(MR_MEM, MOVNTPD, movntpd, Mpd_WO, Vpd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    /* The register, register encoding is invalid. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return VINF_SUCCESS;
+}
+/*  Opcode 0xf3 0x0f 0x2b - invalid */
+/*  Opcode 0xf2 0x0f 0x2b - invalid */
+
+
+/** Opcode      0x0f 0x2c - cvttps2pi Ppi, Wps */
+FNIEMOP_STUB(iemOp_cvttps2pi_Ppi_Wps);
+/** Opcode 0x66 0x0f 0x2c - cvttpd2pi Ppi, Wpd */
+FNIEMOP_STUB(iemOp_cvttpd2pi_Ppi_Wpd);
+/** Opcode 0xf3 0x0f 0x2c - cvttss2si Gy, Wss */
+FNIEMOP_STUB(iemOp_cvttss2si_Gy_Wss);
+/** Opcode 0xf2 0x0f 0x2c - cvttsd2si Gy, Wsd */
+FNIEMOP_STUB(iemOp_cvttsd2si_Gy_Wsd);
+
+/** Opcode      0x0f 0x2d - cvtps2pi Ppi, Wps */
+FNIEMOP_STUB(iemOp_cvtps2pi_Ppi_Wps);
+/** Opcode 0x66 0x0f 0x2d - cvtpd2pi Qpi, Wpd */
+FNIEMOP_STUB(iemOp_cvtpd2pi_Qpi_Wpd);
+/** Opcode 0xf3 0x0f 0x2d - cvtss2si Gy, Wss */
+FNIEMOP_STUB(iemOp_cvtss2si_Gy_Wss);
+/** Opcode 0xf2 0x0f 0x2d - cvtsd2si Gy, Wsd */
+FNIEMOP_STUB(iemOp_cvtsd2si_Gy_Wsd);
+
+/** Opcode      0x0f 0x2e - ucomiss Vss, Wss */
+FNIEMOP_STUB(iemOp_ucomiss_Vss_Wss); // NEXT
+/** Opcode 0x66 0x0f 0x2e - ucomisd Vsd, Wsd */
+FNIEMOP_STUB(iemOp_ucomisd_Vsd_Wsd); // NEXT
+/*  Opcode 0xf3 0x0f 0x2e - invalid */
+/*  Opcode 0xf2 0x0f 0x2e - invalid */
+
+/** Opcode      0x0f 0x2f - comiss Vss, Wss */
+FNIEMOP_STUB(iemOp_comiss_Vss_Wss);
+/** Opcode 0x66 0x0f 0x2f - comisd Vsd, Wsd */
+FNIEMOP_STUB(iemOp_comisd_Vsd_Wsd);
+/*  Opcode 0xf3 0x0f 0x2f - invalid */
+/*  Opcode 0xf2 0x0f 0x2f - invalid */
+
+/** Opcode 0x0f 0x30. */
+FNIEMOP_DEF(iemOp_wrmsr)
+{
+    IEMOP_MNEMONIC(wrmsr, "wrmsr");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_wrmsr);
+}
+
+
+/** Opcode 0x0f 0x31. */
+FNIEMOP_DEF(iemOp_rdtsc)
+{
+    IEMOP_MNEMONIC(rdtsc, "rdtsc");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rdtsc);
+}
+
+
+/** Opcode 0x0f 0x33. */
+FNIEMOP_DEF(iemOp_rdmsr)
+{
+    IEMOP_MNEMONIC(rdmsr, "rdmsr");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rdmsr);
+}
+
+
+/** Opcode 0x0f 0x34. */
+FNIEMOP_DEF(iemOp_rdpmc)
+{
+    IEMOP_MNEMONIC(rdpmc, "rdpmc");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rdpmc);
+}
+
+
+/** Opcode 0x0f 0x34. */
+FNIEMOP_STUB(iemOp_sysenter);
+/** Opcode 0x0f 0x35. */
+FNIEMOP_STUB(iemOp_sysexit);
+/** Opcode 0x0f 0x37. */
+FNIEMOP_STUB(iemOp_getsec);
+
+
+/** Opcode 0x0f 0x38. */
+FNIEMOP_DEF(iemOp_3byte_Esc_0f_38)
+{
+#ifdef IEM_WITH_THREE_0F_38
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnThreeByte0f38[(uintptr_t)b * 4 + pVCpu->iem.s.idxPrefix]);
+#else
+    IEMOP_BITCH_ABOUT_STUB();
+    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/** Opcode 0x0f 0x3a. */
+FNIEMOP_DEF(iemOp_3byte_Esc_0f_3a)
+{
+#ifdef IEM_WITH_THREE_0F_3A
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnThreeByte0f3a[(uintptr_t)b * 4 + pVCpu->iem.s.idxPrefix]);
+#else
+    IEMOP_BITCH_ABOUT_STUB();
+    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/**
+ * Implements a conditional move.
+ *
+ * Wish there was an obvious way to do this where we could share and reduce
+ * code bloat.
+ *
+ * @param   a_Cnd       The conditional "microcode" operation.
+ */
+#define CMOV_X(a_Cnd) \
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT)) \
+    { \
+        switch (pVCpu->iem.s.enmEffOpSize) \
+        { \
+            case IEMMODE_16BIT: \
+                IEM_MC_BEGIN(0, 1); \
+                IEM_MC_LOCAL(uint16_t, u16Tmp); \
+                a_Cnd { \
+                    IEM_MC_FETCH_GREG_U16(u16Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB); \
+                    IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Tmp); \
+                } IEM_MC_ENDIF(); \
+                IEM_MC_ADVANCE_RIP(); \
+                IEM_MC_END(); \
+                return VINF_SUCCESS; \
+    \
+            case IEMMODE_32BIT: \
+                IEM_MC_BEGIN(0, 1); \
+                IEM_MC_LOCAL(uint32_t, u32Tmp); \
+                a_Cnd { \
+                    IEM_MC_FETCH_GREG_U32(u32Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB); \
+                    IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Tmp); \
+                } IEM_MC_ELSE() { \
+                    IEM_MC_CLEAR_HIGH_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg); \
+                } IEM_MC_ENDIF(); \
+                IEM_MC_ADVANCE_RIP(); \
+                IEM_MC_END(); \
+                return VINF_SUCCESS; \
+    \
+            case IEMMODE_64BIT: \
+                IEM_MC_BEGIN(0, 1); \
+                IEM_MC_LOCAL(uint64_t, u64Tmp); \
+                a_Cnd { \
+                    IEM_MC_FETCH_GREG_U64(u64Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB); \
+                    IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Tmp); \
+                } IEM_MC_ENDIF(); \
+                IEM_MC_ADVANCE_RIP(); \
+                IEM_MC_END(); \
+                return VINF_SUCCESS; \
+    \
+            IEM_NOT_REACHED_DEFAULT_CASE_RET(); \
+        } \
+    } \
+    else \
+    { \
+        switch (pVCpu->iem.s.enmEffOpSize) \
+        { \
+            case IEMMODE_16BIT: \
+                IEM_MC_BEGIN(0, 2); \
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc); \
+                IEM_MC_LOCAL(uint16_t, u16Tmp); \
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+                IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+                a_Cnd { \
+                    IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Tmp); \
+                } IEM_MC_ENDIF(); \
+                IEM_MC_ADVANCE_RIP(); \
+                IEM_MC_END(); \
+                return VINF_SUCCESS; \
+    \
+            case IEMMODE_32BIT: \
+                IEM_MC_BEGIN(0, 2); \
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc); \
+                IEM_MC_LOCAL(uint32_t, u32Tmp); \
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+                IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+                a_Cnd { \
+                    IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Tmp); \
+                } IEM_MC_ELSE() { \
+                    IEM_MC_CLEAR_HIGH_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg); \
+                } IEM_MC_ENDIF(); \
+                IEM_MC_ADVANCE_RIP(); \
+                IEM_MC_END(); \
+                return VINF_SUCCESS; \
+    \
+            case IEMMODE_64BIT: \
+                IEM_MC_BEGIN(0, 2); \
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc); \
+                IEM_MC_LOCAL(uint64_t, u64Tmp); \
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+                IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+                a_Cnd { \
+                    IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Tmp); \
+                } IEM_MC_ENDIF(); \
+                IEM_MC_ADVANCE_RIP(); \
+                IEM_MC_END(); \
+                return VINF_SUCCESS; \
+    \
+            IEM_NOT_REACHED_DEFAULT_CASE_RET(); \
+        } \
+    } do {} while (0)
+
+
+
+/** Opcode 0x0f 0x40. */
+FNIEMOP_DEF(iemOp_cmovo_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovo_Gv_Ev, "cmovo Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF));
+}
+
+
+/** Opcode 0x0f 0x41. */
+FNIEMOP_DEF(iemOp_cmovno_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovno_Gv_Ev, "cmovno Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_OF));
+}
+
+
+/** Opcode 0x0f 0x42. */
+FNIEMOP_DEF(iemOp_cmovc_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovc_Gv_Ev, "cmovc Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF));
+}
+
+
+/** Opcode 0x0f 0x43. */
+FNIEMOP_DEF(iemOp_cmovnc_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovnc_Gv_Ev, "cmovnc Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_CF));
+}
+
+
+/** Opcode 0x0f 0x44. */
+FNIEMOP_DEF(iemOp_cmove_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmove_Gv_Ev, "cmove Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF));
+}
+
+
+/** Opcode 0x0f 0x45. */
+FNIEMOP_DEF(iemOp_cmovne_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovne_Gv_Ev, "cmovne Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF));
+}
+
+
+/** Opcode 0x0f 0x46. */
+FNIEMOP_DEF(iemOp_cmovbe_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovbe_Gv_Ev, "cmovbe Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF));
+}
+
+
+/** Opcode 0x0f 0x47. */
+FNIEMOP_DEF(iemOp_cmovnbe_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovnbe_Gv_Ev, "cmovnbe Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_NO_BITS_SET(X86_EFL_CF | X86_EFL_ZF));
+}
+
+
+/** Opcode 0x0f 0x48. */
+FNIEMOP_DEF(iemOp_cmovs_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovs_Gv_Ev, "cmovs Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF));
+}
+
+
+/** Opcode 0x0f 0x49. */
+FNIEMOP_DEF(iemOp_cmovns_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovns_Gv_Ev, "cmovns Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_SF));
+}
+
+
+/** Opcode 0x0f 0x4a. */
+FNIEMOP_DEF(iemOp_cmovp_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovp_Gv_Ev, "cmovp Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF));
+}
+
+
+/** Opcode 0x0f 0x4b. */
+FNIEMOP_DEF(iemOp_cmovnp_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovnp_Gv_Ev, "cmovnp Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_PF));
+}
+
+
+/** Opcode 0x0f 0x4c. */
+FNIEMOP_DEF(iemOp_cmovl_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovl_Gv_Ev, "cmovl Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF));
+}
+
+
+/** Opcode 0x0f 0x4d. */
+FNIEMOP_DEF(iemOp_cmovnl_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovnl_Gv_Ev, "cmovnl Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BITS_EQ(X86_EFL_SF, X86_EFL_OF));
+}
+
+
+/** Opcode 0x0f 0x4e. */
+FNIEMOP_DEF(iemOp_cmovle_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovle_Gv_Ev, "cmovle Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF));
+}
+
+
+/** Opcode 0x0f 0x4f. */
+FNIEMOP_DEF(iemOp_cmovnle_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovnle_Gv_Ev, "cmovnle Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF));
+}
+
+#undef CMOV_X
+
+/** Opcode      0x0f 0x50 - movmskps Gy, Ups */
+FNIEMOP_STUB(iemOp_movmskps_Gy_Ups);
+/** Opcode 0x66 0x0f 0x50 - movmskpd Gy, Upd */
+FNIEMOP_STUB(iemOp_movmskpd_Gy_Upd);
+/*  Opcode 0xf3 0x0f 0x50 - invalid */
+/*  Opcode 0xf2 0x0f 0x50 - invalid */
+
+/** Opcode      0x0f 0x51 - sqrtps Vps, Wps */
+FNIEMOP_STUB(iemOp_sqrtps_Vps_Wps);
+/** Opcode 0x66 0x0f 0x51 - sqrtpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_sqrtpd_Vpd_Wpd);
+/** Opcode 0xf3 0x0f 0x51 - sqrtss Vss, Wss */
+FNIEMOP_STUB(iemOp_sqrtss_Vss_Wss);
+/** Opcode 0xf2 0x0f 0x51 - sqrtsd Vsd, Wsd */
+FNIEMOP_STUB(iemOp_sqrtsd_Vsd_Wsd);
+
+/** Opcode      0x0f 0x52 - rsqrtps Vps, Wps */
+FNIEMOP_STUB(iemOp_rsqrtps_Vps_Wps);
+/*  Opcode 0x66 0x0f 0x52 - invalid */
+/** Opcode 0xf3 0x0f 0x52 - rsqrtss Vss, Wss */
+FNIEMOP_STUB(iemOp_rsqrtss_Vss_Wss);
+/*  Opcode 0xf2 0x0f 0x52 - invalid */
+
+/** Opcode      0x0f 0x53 - rcpps Vps, Wps */
+FNIEMOP_STUB(iemOp_rcpps_Vps_Wps);
+/*  Opcode 0x66 0x0f 0x53 - invalid */
+/** Opcode 0xf3 0x0f 0x53 - rcpss Vss, Wss */
+FNIEMOP_STUB(iemOp_rcpss_Vss_Wss);
+/*  Opcode 0xf2 0x0f 0x53 - invalid */
+
+/** Opcode      0x0f 0x54 - andps Vps, Wps */
+FNIEMOP_STUB(iemOp_andps_Vps_Wps);
+/** Opcode 0x66 0x0f 0x54 - andpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_andpd_Vpd_Wpd);
+/*  Opcode 0xf3 0x0f 0x54 - invalid */
+/*  Opcode 0xf2 0x0f 0x54 - invalid */
+
+/** Opcode      0x0f 0x55 - andnps Vps, Wps */
+FNIEMOP_STUB(iemOp_andnps_Vps_Wps);
+/** Opcode 0x66 0x0f 0x55 - andnpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_andnpd_Vpd_Wpd);
+/*  Opcode 0xf3 0x0f 0x55 - invalid */
+/*  Opcode 0xf2 0x0f 0x55 - invalid */
+
+/** Opcode      0x0f 0x56 - orps Vps, Wps */
+FNIEMOP_STUB(iemOp_orps_Vps_Wps);
+/** Opcode 0x66 0x0f 0x56 - orpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_orpd_Vpd_Wpd);
+/*  Opcode 0xf3 0x0f 0x56 - invalid */
+/*  Opcode 0xf2 0x0f 0x56 - invalid */
+
+/** Opcode      0x0f 0x57 - xorps Vps, Wps */
+FNIEMOP_STUB(iemOp_xorps_Vps_Wps);
+/** Opcode 0x66 0x0f 0x57 - xorpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_xorpd_Vpd_Wpd);
+/*  Opcode 0xf3 0x0f 0x57 - invalid */
+/*  Opcode 0xf2 0x0f 0x57 - invalid */
+
+/** Opcode      0x0f 0x58 - addps Vps, Wps */
+FNIEMOP_STUB(iemOp_addps_Vps_Wps);
+/** Opcode 0x66 0x0f 0x58 - addpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_addpd_Vpd_Wpd);
+/** Opcode 0xf3 0x0f 0x58 - addss Vss, Wss */
+FNIEMOP_STUB(iemOp_addss_Vss_Wss);
+/** Opcode 0xf2 0x0f 0x58 - addsd Vsd, Wsd */
+FNIEMOP_STUB(iemOp_addsd_Vsd_Wsd);
+
+/** Opcode      0x0f 0x59 - mulps Vps, Wps */
+FNIEMOP_STUB(iemOp_mulps_Vps_Wps);
+/** Opcode 0x66 0x0f 0x59 - mulpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_mulpd_Vpd_Wpd);
+/** Opcode 0xf3 0x0f 0x59 - mulss Vss, Wss */
+FNIEMOP_STUB(iemOp_mulss_Vss_Wss);
+/** Opcode 0xf2 0x0f 0x59 - mulsd Vsd, Wsd */
+FNIEMOP_STUB(iemOp_mulsd_Vsd_Wsd);
+
+/** Opcode      0x0f 0x5a - cvtps2pd Vpd, Wps */
+FNIEMOP_STUB(iemOp_cvtps2pd_Vpd_Wps);
+/** Opcode 0x66 0x0f 0x5a - cvtpd2ps Vps, Wpd */
+FNIEMOP_STUB(iemOp_cvtpd2ps_Vps_Wpd);
+/** Opcode 0xf3 0x0f 0x5a - cvtss2sd Vsd, Wss */
+FNIEMOP_STUB(iemOp_cvtss2sd_Vsd_Wss);
+/** Opcode 0xf2 0x0f 0x5a - cvtsd2ss Vss, Wsd */
+FNIEMOP_STUB(iemOp_cvtsd2ss_Vss_Wsd);
+
+/** Opcode      0x0f 0x5b - cvtdq2ps Vps, Wdq */
+FNIEMOP_STUB(iemOp_cvtdq2ps_Vps_Wdq);
+/** Opcode 0x66 0x0f 0x5b - cvtps2dq Vdq, Wps */
+FNIEMOP_STUB(iemOp_cvtps2dq_Vdq_Wps);
+/** Opcode 0xf3 0x0f 0x5b - cvttps2dq Vdq, Wps */
+FNIEMOP_STUB(iemOp_cvttps2dq_Vdq_Wps);
+/*  Opcode 0xf2 0x0f 0x5b - invalid */
+
+/** Opcode      0x0f 0x5c - subps Vps, Wps */
+FNIEMOP_STUB(iemOp_subps_Vps_Wps);
+/** Opcode 0x66 0x0f 0x5c - subpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_subpd_Vpd_Wpd);
+/** Opcode 0xf3 0x0f 0x5c - subss Vss, Wss */
+FNIEMOP_STUB(iemOp_subss_Vss_Wss);
+/** Opcode 0xf2 0x0f 0x5c - subsd Vsd, Wsd */
+FNIEMOP_STUB(iemOp_subsd_Vsd_Wsd);
+
+/** Opcode      0x0f 0x5d - minps Vps, Wps */
+FNIEMOP_STUB(iemOp_minps_Vps_Wps);
+/** Opcode 0x66 0x0f 0x5d - minpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_minpd_Vpd_Wpd);
+/** Opcode 0xf3 0x0f 0x5d - minss Vss, Wss */
+FNIEMOP_STUB(iemOp_minss_Vss_Wss);
+/** Opcode 0xf2 0x0f 0x5d - minsd Vsd, Wsd */
+FNIEMOP_STUB(iemOp_minsd_Vsd_Wsd);
+
+/** Opcode      0x0f 0x5e - divps Vps, Wps */
+FNIEMOP_STUB(iemOp_divps_Vps_Wps);
+/** Opcode 0x66 0x0f 0x5e - divpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_divpd_Vpd_Wpd);
+/** Opcode 0xf3 0x0f 0x5e - divss Vss, Wss */
+FNIEMOP_STUB(iemOp_divss_Vss_Wss);
+/** Opcode 0xf2 0x0f 0x5e - divsd Vsd, Wsd */
+FNIEMOP_STUB(iemOp_divsd_Vsd_Wsd);
+
+/** Opcode      0x0f 0x5f - maxps Vps, Wps */
+FNIEMOP_STUB(iemOp_maxps_Vps_Wps);
+/** Opcode 0x66 0x0f 0x5f - maxpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_maxpd_Vpd_Wpd);
+/** Opcode 0xf3 0x0f 0x5f - maxss Vss, Wss */
+FNIEMOP_STUB(iemOp_maxss_Vss_Wss);
+/** Opcode 0xf2 0x0f 0x5f - maxsd Vsd, Wsd */
+FNIEMOP_STUB(iemOp_maxsd_Vsd_Wsd);
+
+/**
+ * Common worker for MMX instructions on the forms:
+ *      pxxxx mm1, mm2/mem32
+ *
+ * The 2nd operand is the first half of a register, which in the memory case
+ * means a 32-bit memory access for MMX and 128-bit aligned 64-bit or 128-bit
+ * memory accessed for MMX.
+ *
+ * Exceptions type 4.
+ */
+FNIEMOP_DEF_1(iemOpCommonMmx_LowLow_To_Full, PCIEMOPMEDIAF1L1, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,          pDst, 0);
+        IEM_MC_ARG(uint64_t const *,     pSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_XREG_U64_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint64_t const *,  pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U64_ALIGN_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for SSE2 instructions on the forms:
+ *      pxxxx xmm1, xmm2/mem128
+ *
+ * The 2nd operand is the first half of a register, which in the memory case
+ * means a 32-bit memory access for MMX and 128-bit aligned 64-bit or 128-bit
+ * memory accessed for MMX.
+ *
+ * Exceptions type 4.
+ */
+FNIEMOP_DEF_1(iemOpCommonSse_LowLow_To_Full, PCIEMOPMEDIAF1L1, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (!pImpl->pfnU64)
+        return IEMOP_RAISE_INVALID_OPCODE();
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG(uint32_t const *,    pSrc, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_REF_MREG_U64(pDst, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_REF_MREG_U32_CONST(pSrc, bRm & X86_MODRM_RM_MASK);
+        IEM_MC_CALL_MMX_AIMPL_2(pImpl->pfnU64, pDst, pSrc);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint64_t *,                  pDst,       0);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint32_t const *,  pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_REF_MREG_U64(pDst, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_CALL_MMX_AIMPL_2(pImpl->pfnU64, pDst, pSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode      0x0f 0x60 - punpcklbw Pq, Qd */
+FNIEMOP_DEF(iemOp_punpcklbw_Pq_Qd)
+{
+    IEMOP_MNEMONIC(punpcklbw, "punpcklbw Pq, Qd");
+    return FNIEMOP_CALL_1(iemOpCommonMmx_LowLow_To_Full, &g_iemAImpl_punpcklbw);
+}
+
+/** Opcode 0x66 0x0f 0x60 - punpcklbw Vx, W */
+FNIEMOP_DEF(iemOp_punpcklbw_Vx_Wx)
+{
+    IEMOP_MNEMONIC(vpunpcklbw_Vx_Wx, "vpunpcklbw Vx, Wx");
+    return FNIEMOP_CALL_1(iemOpCommonSse_LowLow_To_Full, &g_iemAImpl_punpcklbw);
+}
+
+/*  Opcode 0xf3 0x0f 0x60 - invalid */
+
+
+/** Opcode      0x0f 0x61 - punpcklwd Pq, Qd */
+FNIEMOP_DEF(iemOp_punpcklwd_Pq_Qd)
+{
+    IEMOP_MNEMONIC(punpcklwd, "punpcklwd Pq, Qd"); /** @todo AMD mark the MMX version as 3DNow!. Intel says MMX CPUID req. */
+    return FNIEMOP_CALL_1(iemOpCommonMmx_LowLow_To_Full, &g_iemAImpl_punpcklwd);
+}
+
+/** Opcode 0x66 0x0f 0x61 - punpcklwd Vx, Wx */
+FNIEMOP_DEF(iemOp_punpcklwd_Vx_Wx)
+{
+    IEMOP_MNEMONIC(vpunpcklwd_Vx_Wx, "punpcklwd Vx, Wx");
+    return FNIEMOP_CALL_1(iemOpCommonSse_LowLow_To_Full, &g_iemAImpl_punpcklwd);
+}
+
+/*  Opcode 0xf3 0x0f 0x61 - invalid */
+
+
+/** Opcode      0x0f 0x62 - punpckldq Pq, Qd */
+FNIEMOP_DEF(iemOp_punpckldq_Pq_Qd)
+{
+    IEMOP_MNEMONIC(punpckldq, "punpckldq Pq, Qd");
+    return FNIEMOP_CALL_1(iemOpCommonMmx_LowLow_To_Full, &g_iemAImpl_punpckldq);
+}
+
+/** Opcode 0x66 0x0f 0x62 - punpckldq Vx, Wx */
+FNIEMOP_DEF(iemOp_punpckldq_Vx_Wx)
+{
+    IEMOP_MNEMONIC(punpckldq_Vx_Wx, "punpckldq Vx, Wx");
+    return FNIEMOP_CALL_1(iemOpCommonSse_LowLow_To_Full, &g_iemAImpl_punpckldq);
+}
+
+/*  Opcode 0xf3 0x0f 0x62 - invalid */
+
+
+
+/** Opcode      0x0f 0x63 - packsswb Pq, Qq */
+FNIEMOP_STUB(iemOp_packsswb_Pq_Qq);
+/** Opcode 0x66 0x0f 0x63 - packsswb Vx, Wx */
+FNIEMOP_STUB(iemOp_packsswb_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0x63 - invalid */
+
+/** Opcode      0x0f 0x64 - pcmpgtb Pq, Qq */
+FNIEMOP_STUB(iemOp_pcmpgtb_Pq_Qq);
+/** Opcode 0x66 0x0f 0x64 - pcmpgtb Vx, Wx */
+FNIEMOP_STUB(iemOp_pcmpgtb_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0x64 - invalid */
+
+/** Opcode      0x0f 0x65 - pcmpgtw Pq, Qq */
+FNIEMOP_STUB(iemOp_pcmpgtw_Pq_Qq);
+/** Opcode 0x66 0x0f 0x65 - pcmpgtw Vx, Wx */
+FNIEMOP_STUB(iemOp_pcmpgtw_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0x65 - invalid */
+
+/** Opcode      0x0f 0x66 - pcmpgtd Pq, Qq */
+FNIEMOP_STUB(iemOp_pcmpgtd_Pq_Qq);
+/** Opcode 0x66 0x0f 0x66 - pcmpgtd Vx, Wx */
+FNIEMOP_STUB(iemOp_pcmpgtd_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0x66 - invalid */
+
+/** Opcode      0x0f 0x67 - packuswb Pq, Qq */
+FNIEMOP_STUB(iemOp_packuswb_Pq_Qq);
+/** Opcode 0x66 0x0f 0x67 - packuswb Vx, W */
+FNIEMOP_STUB(iemOp_packuswb_Vx_W);
+/*  Opcode 0xf3 0x0f 0x67 - invalid */
+
+
+/**
+ * Common worker for MMX instructions on the form:
+ *      pxxxx mm1, mm2/mem64
+ *
+ * The 2nd operand is the second half of a register, which in the memory case
+ * means a 64-bit memory access for MMX, and for SSE a 128-bit aligned access
+ * where it may read the full 128 bits or only the upper 64 bits.
+ *
+ * Exceptions type 4.
+ */
+FNIEMOP_DEF_1(iemOpCommonMmx_HighHigh_To_Full, PCIEMOPMEDIAF1H1, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    AssertReturn(pImpl->pfnU64, IEMOP_RAISE_INVALID_OPCODE());
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG(uint64_t const *,    pSrc, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_REF_MREG_U64(pDst, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_REF_MREG_U64_CONST(pSrc, bRm & X86_MODRM_RM_MASK);
+        IEM_MC_CALL_MMX_AIMPL_2(pImpl->pfnU64, pDst, pSrc);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint64_t *,                  pDst,       0);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint64_t const *,  pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_REF_MREG_U64(pDst, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_CALL_MMX_AIMPL_2(pImpl->pfnU64, pDst, pSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for SSE2 instructions on the form:
+ *      pxxxx xmm1, xmm2/mem128
+ *
+ * The 2nd operand is the second half of a register, which in the memory case
+ * means a 64-bit memory access for MMX, and for SSE a 128-bit aligned access
+ * where it may read the full 128 bits or only the upper 64 bits.
+ *
+ * Exceptions type 4.
+ */
+FNIEMOP_DEF_1(iemOpCommonSse_HighHigh_To_Full, PCIEMOPMEDIAF1H1, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,          pDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,         pSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_XREG_U128_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); /* Most CPUs probably only right high qword */
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode      0x0f 0x68 - punpckhbw Pq, Qd */
+FNIEMOP_DEF(iemOp_punpckhbw_Pq_Qd)
+{
+    IEMOP_MNEMONIC(punpckhbw, "punpckhbw Pq, Qd");
+    return FNIEMOP_CALL_1(iemOpCommonMmx_HighHigh_To_Full, &g_iemAImpl_punpckhbw);
+}
+
+/** Opcode 0x66 0x0f 0x68 - punpckhbw Vx, Wx */
+FNIEMOP_DEF(iemOp_punpckhbw_Vx_Wx)
+{
+    IEMOP_MNEMONIC(vpunpckhbw_Vx_Wx, "vpunpckhbw Vx, Wx");
+    return FNIEMOP_CALL_1(iemOpCommonSse_HighHigh_To_Full, &g_iemAImpl_punpckhbw);
+}
+/*  Opcode 0xf3 0x0f 0x68 - invalid */
+
+
+/** Opcode      0x0f 0x69 - punpckhwd Pq, Qd */
+FNIEMOP_DEF(iemOp_punpckhwd_Pq_Qd)
+{
+    IEMOP_MNEMONIC(punpckhwd, "punpckhwd Pq, Qd");
+    return FNIEMOP_CALL_1(iemOpCommonMmx_HighHigh_To_Full, &g_iemAImpl_punpckhwd);
+}
+
+/** Opcode 0x66 0x0f 0x69 - punpckhwd Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_punpckhwd_Vx_Wx)
+{
+    IEMOP_MNEMONIC(punpckhwd_Vx_Wx, "punpckhwd Vx, Wx");
+    return FNIEMOP_CALL_1(iemOpCommonSse_HighHigh_To_Full, &g_iemAImpl_punpckhwd);
+
+}
+/*  Opcode 0xf3 0x0f 0x69 - invalid */
+
+
+/** Opcode      0x0f 0x6a - punpckhdq Pq, Qd */
+FNIEMOP_DEF(iemOp_punpckhdq_Pq_Qd)
+{
+    IEMOP_MNEMONIC(punpckhdq, "punpckhdq Pq, Qd");
+    return FNIEMOP_CALL_1(iemOpCommonMmx_HighHigh_To_Full, &g_iemAImpl_punpckhdq);
+}
+
+/** Opcode 0x66 0x0f 0x6a - punpckhdq Vx, W */
+FNIEMOP_DEF(iemOp_punpckhdq_Vx_W)
+{
+    IEMOP_MNEMONIC(punpckhdq_Vx_W, "punpckhdq Vx, W");
+    return FNIEMOP_CALL_1(iemOpCommonSse_HighHigh_To_Full, &g_iemAImpl_punpckhdq);
+}
+/*  Opcode 0xf3 0x0f 0x6a - invalid */
+
+
+/** Opcode      0x0f 0x6b - packssdw Pq, Qd */
+FNIEMOP_STUB(iemOp_packssdw_Pq_Qd);
+/** Opcode 0x66 0x0f 0x6b - packssdw Vx, Wx */
+FNIEMOP_STUB(iemOp_packssdw_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0x6b - invalid */
+
+
+/*  Opcode      0x0f 0x6c - invalid */
+
+/** Opcode 0x66 0x0f 0x6c - punpcklqdq Vx, Wx */
+FNIEMOP_DEF(iemOp_punpcklqdq_Vx_Wx)
+{
+    IEMOP_MNEMONIC(punpcklqdq, "punpcklqdq Vx, Wx");
+    return FNIEMOP_CALL_1(iemOpCommonSse_LowLow_To_Full, &g_iemAImpl_punpcklqdq);
+}
+
+/*  Opcode 0xf3 0x0f 0x6c - invalid */
+/*  Opcode 0xf2 0x0f 0x6c - invalid */
+
+
+/*  Opcode      0x0f 0x6d - invalid */
+
+/** Opcode 0x66 0x0f 0x6d - punpckhqdq Vx, W */
+FNIEMOP_DEF(iemOp_punpckhqdq_Vx_W)
+{
+    IEMOP_MNEMONIC(punpckhqdq_Vx_W, "punpckhqdq Vx,W");
+    return FNIEMOP_CALL_1(iemOpCommonSse_HighHigh_To_Full, &g_iemAImpl_punpckhqdq);
+}
+
+/*  Opcode 0xf3 0x0f 0x6d - invalid */
+
+
+FNIEMOP_DEF(iemOp_movd_q_Pd_Ey)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x6e
+         * @opcodesub   rex.w=1
+         * @oppfx       none
+         * @opcpuid     mmx
+         * @opgroup     og_mmx_datamove
+         * @opxcpttype  5
+         * @optest      64-bit / op1=1 op2=2   -> op1=2   ftw=0xff
+         * @optest      64-bit / op1=0 op2=-42 -> op1=-42 ftw=0xff
+         */
+        IEMOP_MNEMONIC2(RM, MOVQ, movq, Pq_WO, Eq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OZ_PFX);
+        if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /* MMX, greg64 */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_GREG_U64(u64Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_STORE_MREG_U64((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK, u64Tmp);
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* MMX, [mem64] */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_MREG_U64((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK, u64Tmp);
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x6e
+         * @opcodesub   rex.w=0
+         * @oppfx       none
+         * @opcpuid     mmx
+         * @opgroup     og_mmx_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_movd_q_Pd_Ey
+         * @optest      op1=1 op2=2   -> op1=2   ftw=0xff
+         * @optest      op1=0 op2=-42 -> op1=-42 ftw=0xff
+         */
+        IEMOP_MNEMONIC2(RM, MOVD, movd, PdZx_WO, Ed, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OZ_PFX);
+        if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /* MMX, greg */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_GREG_U32_ZX_U64(u64Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_STORE_MREG_U64((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK, u64Tmp);
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* MMX, [mem] */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_MREG_U32_ZX_U64((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK, u32Tmp);
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+FNIEMOP_DEF(iemOp_movd_q_Vy_Ey)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x6e
+         * @opcodesub   rex.w=1
+         * @oppfx       0x66
+         * @opcpuid     sse2
+         * @opgroup     og_sse2_simdint_datamove
+         * @opxcpttype  5
+         * @optest      64-bit / op1=1 op2=2   -> op1=2
+         * @optest      64-bit / op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(RM, MOVQ, movq, VqZx_WO, Eq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OZ_PFX);
+        if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /* XMM, greg64 */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_GREG_U64(u64Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_STORE_XREG_U64_ZX_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* XMM, [mem64] */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_XREG_U64_ZX_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x6e
+         * @opcodesub   rex.w=0
+         * @oppfx       0x66
+         * @opcpuid     sse2
+         * @opgroup     og_sse2_simdint_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_movd_q_Vy_Ey
+         * @optest      op1=1 op2=2   -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(RM, MOVD, movd, VdZx_WO, Ed, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OZ_PFX);
+        if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /* XMM, greg32 */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_GREG_U32(u32Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_STORE_XREG_U32_ZX_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* XMM, [mem32] */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_XREG_U32_ZX_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+/*  Opcode 0xf3 0x0f 0x6e - invalid */
+
+
+/**
+ * @opcode      0x6f
+ * @oppfx       none
+ * @opcpuid     mmx
+ * @opgroup     og_mmx_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2   -> op1=2   ftw=0xff
+ * @optest      op1=0 op2=-42 -> op1=-42 ftw=0xff
+ */
+FNIEMOP_DEF(iemOp_movq_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, MOVD, movd, Pq_WO, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MREG_U64(u64Tmp, bRm & X86_MODRM_RM_MASK);
+        IEM_MC_STORE_MREG_U64((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK, u64Tmp);
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t, u64Tmp);
+        IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_MREG_U64((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK, u64Tmp);
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opcode      0x6f
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_simdint_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2   -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movdqa_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, MOVDQA, movdqa, Vdq_WO, Wdq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_COPY_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                              (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u128Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opcode      0x6f
+ * @oppfx       0xf3
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_simdint_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2   -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movdqu_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, MOVDQU, movdqu, Vdq_WO, Wdq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                              (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_MEM_U128(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u128Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode      0x0f 0x70 - pshufw Pq, Qq, Ib */
+FNIEMOP_DEF(iemOp_pshufw_Pq_Qq_Ib)
+{
+    IEMOP_MNEMONIC(pshufw_Pq_Qq, "pshufw Pq,Qq,Ib");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG(uint64_t const *,    pSrc, 1);
+        IEM_MC_ARG_CONST(uint8_t,       bEvilArg, /*=*/ bEvil, 2);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_REF_MREG_U64(pDst, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_REF_MREG_U64_CONST(pSrc, bRm & X86_MODRM_RM_MASK);
+        IEM_MC_CALL_MMX_AIMPL_3(iemAImpl_pshufw, pDst, pSrc, bEvilArg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint64_t *,                  pDst,       0);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint64_t const *,  pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+        IEM_MC_ARG_CONST(uint8_t,               bEvilArg, /*=*/ bEvil, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_REF_MREG_U64(pDst, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_CALL_MMX_AIMPL_3(iemAImpl_pshufw, pDst, pSrc, bEvilArg);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/** Opcode 0x66 0x0f 0x70 - pshufd Vx, Wx, Ib */
+FNIEMOP_DEF(iemOp_pshufd_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC(pshufd_Vx_Wx_Ib, "pshufd Vx,Wx,Ib");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(PRTUINT128U,         pDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,        pSrc, 1);
+        IEM_MC_ARG_CONST(uint8_t,       bEvilArg, /*=*/ bEvil, 2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_XREG_U128_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_pshufd, pDst, pSrc, bEvilArg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+        IEM_MC_ARG_CONST(uint8_t,               bEvilArg, /*=*/ bEvil, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_pshufd, pDst, pSrc, bEvilArg);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/** Opcode 0xf3 0x0f 0x70 - pshufhw Vx, Wx, Ib */
+FNIEMOP_DEF(iemOp_pshufhw_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC(pshufhw_Vx_Wx_Ib, "pshufhw Vx,Wx,Ib");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(PRTUINT128U,         pDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,        pSrc, 1);
+        IEM_MC_ARG_CONST(uint8_t,       bEvilArg, /*=*/ bEvil, 2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_XREG_U128_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_pshufhw, pDst, pSrc, bEvilArg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+        IEM_MC_ARG_CONST(uint8_t,               bEvilArg, /*=*/ bEvil, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_pshufhw, pDst, pSrc, bEvilArg);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/** Opcode 0xf2 0x0f 0x70 - pshuflw Vx, Wx, Ib */
+FNIEMOP_DEF(iemOp_pshuflw_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC(pshuflw_Vx_Wx_Ib, "pshuflw Vx,Wx,Ib");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(PRTUINT128U,         pDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,        pSrc, 1);
+        IEM_MC_ARG_CONST(uint8_t,       bEvilArg, /*=*/ bEvil, 2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_XREG_U128_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_pshuflw, pDst, pSrc, bEvilArg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+        IEM_MC_ARG_CONST(uint8_t,               bEvilArg, /*=*/ bEvil, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_pshuflw, pDst, pSrc, bEvilArg);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x71 11/2. */
+FNIEMOP_STUB_1(iemOp_Grp12_psrlw_Nq_Ib, uint8_t, bRm);
+
+/** Opcode 0x66 0x0f 0x71 11/2. */
+FNIEMOP_STUB_1(iemOp_Grp12_psrlw_Ux_Ib, uint8_t, bRm);
+
+/** Opcode 0x0f 0x71 11/4. */
+FNIEMOP_STUB_1(iemOp_Grp12_psraw_Nq_Ib, uint8_t, bRm);
+
+/** Opcode 0x66 0x0f 0x71 11/4. */
+FNIEMOP_STUB_1(iemOp_Grp12_psraw_Ux_Ib, uint8_t, bRm);
+
+/** Opcode 0x0f 0x71 11/6. */
+FNIEMOP_STUB_1(iemOp_Grp12_psllw_Nq_Ib, uint8_t, bRm);
+
+/** Opcode 0x66 0x0f 0x71 11/6. */
+FNIEMOP_STUB_1(iemOp_Grp12_psllw_Ux_Ib, uint8_t, bRm);
+
+
+/**
+ * Group 12 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup12RegReg[] =
+{
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /2 */ iemOp_Grp12_psrlw_Nq_Ib,   iemOp_Grp12_psrlw_Ux_Ib,    iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /3 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /4 */ iemOp_Grp12_psraw_Nq_Ib,   iemOp_Grp12_psraw_Ux_Ib,    iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /6 */ iemOp_Grp12_psllw_Nq_Ib,   iemOp_Grp12_psllw_Ux_Ib,    iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /7 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8)
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup12RegReg) == 8*4);
+
+
+/** Opcode 0x0f 0x71. */
+FNIEMOP_DEF(iemOp_Grp12)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnGroup12RegReg[  ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) * 4
+                                                  + pVCpu->iem.s.idxPrefix], bRm);
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm);
+}
+
+
+/** Opcode 0x0f 0x72 11/2. */
+FNIEMOP_STUB_1(iemOp_Grp13_psrld_Nq_Ib, uint8_t, bRm);
+
+/** Opcode 0x66 0x0f 0x72 11/2. */
+FNIEMOP_STUB_1(iemOp_Grp13_psrld_Ux_Ib, uint8_t, bRm);
+
+/** Opcode 0x0f 0x72 11/4. */
+FNIEMOP_STUB_1(iemOp_Grp13_psrad_Nq_Ib, uint8_t, bRm);
+
+/** Opcode 0x66 0x0f 0x72 11/4. */
+FNIEMOP_STUB_1(iemOp_Grp13_psrad_Ux_Ib, uint8_t, bRm);
+
+/** Opcode 0x0f 0x72 11/6. */
+FNIEMOP_STUB_1(iemOp_Grp13_pslld_Nq_Ib, uint8_t, bRm);
+
+/** Opcode 0x66 0x0f 0x72 11/6. */
+FNIEMOP_STUB_1(iemOp_Grp13_pslld_Ux_Ib, uint8_t, bRm);
+
+
+/**
+ * Group 13 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup13RegReg[] =
+{
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /2 */ iemOp_Grp13_psrld_Nq_Ib,   iemOp_Grp13_psrld_Ux_Ib,    iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /3 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /4 */ iemOp_Grp13_psrad_Nq_Ib,   iemOp_Grp13_psrad_Ux_Ib,    iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /6 */ iemOp_Grp13_pslld_Nq_Ib,   iemOp_Grp13_pslld_Ux_Ib,    iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /7 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8)
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup13RegReg) == 8*4);
+
+/** Opcode 0x0f 0x72. */
+FNIEMOP_DEF(iemOp_Grp13)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnGroup13RegReg[ ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) * 4
+                                                  + pVCpu->iem.s.idxPrefix], bRm);
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm);
+}
+
+
+/** Opcode 0x0f 0x73 11/2. */
+FNIEMOP_STUB_1(iemOp_Grp14_psrlq_Nq_Ib, uint8_t, bRm);
+
+/** Opcode 0x66 0x0f 0x73 11/2. */
+FNIEMOP_STUB_1(iemOp_Grp14_psrlq_Ux_Ib, uint8_t, bRm);
+
+/** Opcode 0x66 0x0f 0x73 11/3. */
+FNIEMOP_STUB_1(iemOp_Grp14_psrldq_Ux_Ib, uint8_t, bRm); //NEXT
+
+/** Opcode 0x0f 0x73 11/6. */
+FNIEMOP_STUB_1(iemOp_Grp14_psllq_Nq_Ib, uint8_t, bRm);
+
+/** Opcode 0x66 0x0f 0x73 11/6. */
+FNIEMOP_STUB_1(iemOp_Grp14_psllq_Ux_Ib, uint8_t, bRm);
+
+/** Opcode 0x66 0x0f 0x73 11/7. */
+FNIEMOP_STUB_1(iemOp_Grp14_pslldq_Ux_Ib, uint8_t, bRm); //NEXT
+
+/**
+ * Group 14 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup14RegReg[] =
+{
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /2 */ iemOp_Grp14_psrlq_Nq_Ib,     iemOp_Grp14_psrlq_Ux_Ib,  iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /3 */ iemOp_InvalidWithRMNeedImm8, iemOp_Grp14_psrldq_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /4 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /6 */ iemOp_Grp14_psllq_Nq_Ib,     iemOp_Grp14_psllq_Ux_Ib,  iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /7 */ iemOp_InvalidWithRMNeedImm8, iemOp_Grp14_pslldq_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup14RegReg) == 8*4);
+
+
+/** Opcode 0x0f 0x73. */
+FNIEMOP_DEF(iemOp_Grp14)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnGroup14RegReg[ ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) * 4
+                                                  + pVCpu->iem.s.idxPrefix], bRm);
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm);
+}
+
+
+/**
+ * Common worker for MMX instructions on the form:
+ *      pxxx    mm1, mm2/mem64
+ */
+FNIEMOP_DEF_1(iemOpCommonMmx_FullFull_To_Full, PCIEMOPMEDIAF2, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG(uint64_t const *,    pSrc, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_REF_MREG_U64(pDst, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_REF_MREG_U64_CONST(pSrc, bRm & X86_MODRM_RM_MASK);
+        IEM_MC_CALL_MMX_AIMPL_2(pImpl->pfnU64, pDst, pSrc);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint64_t *,                  pDst,       0);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint64_t const *,  pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_REF_MREG_U64(pDst, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_CALL_MMX_AIMPL_2(pImpl->pfnU64, pDst, pSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for SSE2 instructions on the forms:
+ *      pxxx    xmm1, xmm2/mem128
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 4. SSE2 cpuid checks.
+ */
+FNIEMOP_DEF_1(iemOpCommonSse2_FullFull_To_Full, PCIEMOPMEDIAF2, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,          pDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,         pSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_XREG_U128_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode      0x0f 0x74 - pcmpeqb Pq, Qq */
+FNIEMOP_DEF(iemOp_pcmpeqb_Pq_Qq)
+{
+    IEMOP_MNEMONIC(pcmpeqb, "pcmpeqb");
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, &g_iemAImpl_pcmpeqb);
+}
+
+/** Opcode 0x66 0x0f 0x74 - pcmpeqb Vx, Wx */
+FNIEMOP_DEF(iemOp_pcmpeqb_Vx_Wx)
+{
+    IEMOP_MNEMONIC(vpcmpeqb_Vx_Wx, "pcmpeqb");
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, &g_iemAImpl_pcmpeqb);
+}
+
+/*  Opcode 0xf3 0x0f 0x74 - invalid */
+/*  Opcode 0xf2 0x0f 0x74 - invalid */
+
+
+/** Opcode      0x0f 0x75 - pcmpeqw Pq, Qq */
+FNIEMOP_DEF(iemOp_pcmpeqw_Pq_Qq)
+{
+    IEMOP_MNEMONIC(pcmpeqw, "pcmpeqw");
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, &g_iemAImpl_pcmpeqw);
+}
+
+/** Opcode 0x66 0x0f 0x75 - pcmpeqw Vx, Wx */
+FNIEMOP_DEF(iemOp_pcmpeqw_Vx_Wx)
+{
+    IEMOP_MNEMONIC(pcmpeqw_Vx_Wx, "pcmpeqw");
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, &g_iemAImpl_pcmpeqw);
+}
+
+/*  Opcode 0xf3 0x0f 0x75 - invalid */
+/*  Opcode 0xf2 0x0f 0x75 - invalid */
+
+
+/** Opcode      0x0f 0x76 - pcmpeqd Pq, Qq */
+FNIEMOP_DEF(iemOp_pcmpeqd_Pq_Qq)
+{
+    IEMOP_MNEMONIC(pcmpeqd, "pcmpeqd");
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, &g_iemAImpl_pcmpeqd);
+}
+
+/** Opcode 0x66 0x0f 0x76 - pcmpeqd Vx, Wx */
+FNIEMOP_DEF(iemOp_pcmpeqd_Vx_Wx)
+{
+    IEMOP_MNEMONIC(pcmpeqd_Vx_Wx, "vpcmpeqd");
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, &g_iemAImpl_pcmpeqd);
+}
+
+/*  Opcode 0xf3 0x0f 0x76 - invalid */
+/*  Opcode 0xf2 0x0f 0x76 - invalid */
+
+
+/** Opcode      0x0f 0x77 - emms (vex has vzeroall and vzeroupper here) */
+FNIEMOP_DEF(iemOp_emms)
+{
+    IEMOP_MNEMONIC(emms, "emms");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FPU_FROM_MMX_MODE();
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+/*  Opcode 0x66 0x0f 0x77 - invalid */
+/*  Opcode 0xf3 0x0f 0x77 - invalid */
+/*  Opcode 0xf2 0x0f 0x77 - invalid */
+
+/** Opcode      0x0f 0x78 - VMREAD Ey, Gy */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF(iemOp_vmread_Ey_Gy)
+{
+    IEMOP_MNEMONIC(vmread, "vmread Ey,Gy");
+    IEMOP_HLP_IN_VMX_OPERATION("vmread", kVmxVDiag_Vmread);
+    IEMOP_HLP_VMX_INSTR("vmread", kVmxVDiag_Vmread);
+    IEMMODE const enmEffOpSize = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? IEMMODE_64BIT : IEMMODE_32BIT;
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+        if (enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint64_t *, pu64Dst, 0);
+            IEM_MC_ARG(uint64_t,   u64Enc,  1);
+            IEM_MC_FETCH_GREG_U64(u64Enc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_CALL_CIMPL_2(iemCImpl_vmread_reg64, pu64Dst, u64Enc);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint32_t *, pu32Dst, 0);
+            IEM_MC_ARG(uint32_t,   u32Enc,  1);
+            IEM_MC_FETCH_GREG_U32(u32Enc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_CALL_CIMPL_2(iemCImpl_vmread_reg32, pu32Dst, u32Enc);
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        if (enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,       iEffSeg,                                          0);
+            IEM_MC_ARG(RTGCPTR,       GCPtrVal,                                         1);
+            IEM_MC_ARG(uint64_t,      u64Enc,                                           2);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrVal, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+            IEM_MC_FETCH_GREG_U64(u64Enc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_vmread_mem_reg64, iEffSeg, GCPtrVal, u64Enc);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,       iEffSeg,                                          0);
+            IEM_MC_ARG(RTGCPTR,       GCPtrVal,                                         1);
+            IEM_MC_ARG(uint32_t,      u32Enc,                                           2);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrVal, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+            IEM_MC_FETCH_GREG_U32(u32Enc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_vmread_mem_reg32, iEffSeg, GCPtrVal, u32Enc);
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+#else
+FNIEMOP_STUB(iemOp_vmread_Ey_Gy);
+#endif
+
+/*  Opcode 0x66 0x0f 0x78 - AMD Group 17 */
+FNIEMOP_STUB(iemOp_AmdGrp17);
+/*  Opcode 0xf3 0x0f 0x78 - invalid */
+/*  Opcode 0xf2 0x0f 0x78 - invalid */
+
+/** Opcode      0x0f 0x79 - VMWRITE Gy, Ey */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF(iemOp_vmwrite_Gy_Ey)
+{
+    IEMOP_MNEMONIC(vmwrite, "vmwrite Gy,Ey");
+    IEMOP_HLP_IN_VMX_OPERATION("vmwrite", kVmxVDiag_Vmwrite);
+    IEMOP_HLP_VMX_INSTR("vmwrite", kVmxVDiag_Vmwrite);
+    IEMMODE const enmEffOpSize = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? IEMMODE_64BIT : IEMMODE_32BIT;
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+        if (enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint64_t, u64Val, 0);
+            IEM_MC_ARG(uint64_t, u64Enc, 1);
+            IEM_MC_FETCH_GREG_U64(u64Val, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_FETCH_GREG_U64(u64Enc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_CALL_CIMPL_2(iemCImpl_vmwrite_reg, u64Val, u64Enc);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint32_t, u32Val, 0);
+            IEM_MC_ARG(uint32_t, u32Enc, 1);
+            IEM_MC_FETCH_GREG_U32(u32Val, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_FETCH_GREG_U32(u32Enc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_CALL_CIMPL_2(iemCImpl_vmwrite_reg, u32Val, u32Enc);
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,       iEffSeg,                                          0);
+            IEM_MC_ARG(RTGCPTR,       GCPtrVal,                                         1);
+            IEM_MC_ARG(uint64_t,      u64Enc,                                           2);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrVal, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+            IEM_MC_FETCH_GREG_U64(u64Enc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_vmwrite_mem, iEffSeg, GCPtrVal, u64Enc);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,       iEffSeg,                                          0);
+            IEM_MC_ARG(RTGCPTR,       GCPtrVal,                                         1);
+            IEM_MC_ARG(uint32_t,      u32Enc,                                           2);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrVal, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+            IEM_MC_FETCH_GREG_U32(u32Enc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_vmwrite_mem, iEffSeg, GCPtrVal, u32Enc);
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+#else
+FNIEMOP_STUB(iemOp_vmwrite_Gy_Ey);
+#endif
+/*  Opcode 0x66 0x0f 0x79 - invalid */
+/*  Opcode 0xf3 0x0f 0x79 - invalid */
+/*  Opcode 0xf2 0x0f 0x79 - invalid */
+
+/*  Opcode      0x0f 0x7a - invalid */
+/*  Opcode 0x66 0x0f 0x7a - invalid */
+/*  Opcode 0xf3 0x0f 0x7a - invalid */
+/*  Opcode 0xf2 0x0f 0x7a - invalid */
+
+/*  Opcode      0x0f 0x7b - invalid */
+/*  Opcode 0x66 0x0f 0x7b - invalid */
+/*  Opcode 0xf3 0x0f 0x7b - invalid */
+/*  Opcode 0xf2 0x0f 0x7b - invalid */
+
+/*  Opcode      0x0f 0x7c - invalid */
+/** Opcode 0x66 0x0f 0x7c - haddpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_haddpd_Vpd_Wpd);
+/*  Opcode 0xf3 0x0f 0x7c - invalid */
+/** Opcode 0xf2 0x0f 0x7c - haddps Vps, Wps */
+FNIEMOP_STUB(iemOp_haddps_Vps_Wps);
+
+/*  Opcode      0x0f 0x7d - invalid */
+/** Opcode 0x66 0x0f 0x7d - hsubpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_hsubpd_Vpd_Wpd);
+/*  Opcode 0xf3 0x0f 0x7d - invalid */
+/** Opcode 0xf2 0x0f 0x7d - hsubps Vps, Wps */
+FNIEMOP_STUB(iemOp_hsubps_Vps_Wps);
+
+
+/** Opcode      0x0f 0x7e - movd_q Ey, Pd */
+FNIEMOP_DEF(iemOp_movd_q_Ey_Pd)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x7e
+         * @opcodesub   rex.w=1
+         * @oppfx       none
+         * @opcpuid     mmx
+         * @opgroup     og_mmx_datamove
+         * @opxcpttype  5
+         * @optest      64-bit / op1=1 op2=2   -> op1=2   ftw=0xff
+         * @optest      64-bit / op1=0 op2=-42 -> op1=-42 ftw=0xff
+         */
+        IEMOP_MNEMONIC2(MR, MOVQ, movq, Eq_WO, Pq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OZ_PFX);
+        if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /* greg64, MMX */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MREG_U64(u64Tmp, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+            IEM_MC_STORE_GREG_U64((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u64Tmp);
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* [mem64], MMX */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MREG_U64(u64Tmp, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+            IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u64Tmp);
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x7e
+         * @opcodesub   rex.w=0
+         * @oppfx       none
+         * @opcpuid     mmx
+         * @opgroup     og_mmx_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_movd_q_Pd_Ey
+         * @optest      op1=1 op2=2   -> op1=2   ftw=0xff
+         * @optest      op1=0 op2=-42 -> op1=-42 ftw=0xff
+         */
+        IEMOP_MNEMONIC2(MR, MOVD, movd, Ed_WO, Pd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OZ_PFX);
+        if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /* greg32, MMX */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MREG_U32(u32Tmp, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+            IEM_MC_STORE_GREG_U32((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u32Tmp);
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* [mem32], MMX */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MREG_U32(u32Tmp, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+            IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u32Tmp);
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+
+}
+
+
+FNIEMOP_DEF(iemOp_movd_q_Ey_Vy)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x7e
+         * @opcodesub   rex.w=1
+         * @oppfx       0x66
+         * @opcpuid     sse2
+         * @opgroup     og_sse2_simdint_datamove
+         * @opxcpttype  5
+         * @optest      64-bit / op1=1 op2=2   -> op1=2
+         * @optest      64-bit / op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(MR, MOVQ, movq, Eq_WO, Vq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OZ_PFX);
+        if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /* greg64, XMM */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+            IEM_MC_FETCH_XREG_U64(u64Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_GREG_U64((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u64Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* [mem64], XMM */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+            IEM_MC_FETCH_XREG_U64(u64Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u64Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x7e
+         * @opcodesub   rex.w=0
+         * @oppfx       0x66
+         * @opcpuid     sse2
+         * @opgroup     og_sse2_simdint_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_movd_q_Vy_Ey
+         * @optest      op1=1 op2=2   -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(MR, MOVD, movd, Ed_WO, Vd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OZ_PFX);
+        if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /* greg32, XMM */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+            IEM_MC_FETCH_XREG_U32(u32Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_GREG_U32((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u32Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* [mem32], XMM */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+            IEM_MC_FETCH_XREG_U32(u32Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u32Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+
+}
+
+/**
+ * @opcode      0x7e
+ * @oppfx       0xf3
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  none
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movq_Vq_Wq)
+{
+    IEMOP_MNEMONIC2(RM, MOVQ, movq, VqZx_WO, Wq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_STORE_XREG_U64_ZX_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U64_ZX_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/*  Opcode 0xf2 0x0f 0x7e - invalid */
+
+
+/** Opcode      0x0f 0x7f - movq Qq, Pq */
+FNIEMOP_DEF(iemOp_movq_Qq_Pq)
+{
+    IEMOP_MNEMONIC(movq_Qq_Pq, "movq Qq,Pq");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t, u64Tmp);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_MREG_U64(u64Tmp, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_STORE_MREG_U64(bRm & X86_MODRM_RM_MASK, u64Tmp);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t, u64Tmp);
+        IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+
+        IEM_MC_FETCH_MREG_U64(u64Tmp, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u64Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/** Opcode 0x66 0x0f 0x7f - movdqa Wx,Vx */
+FNIEMOP_DEF(iemOp_movdqa_Wx_Vx)
+{
+    IEMOP_MNEMONIC(movdqa_Wdq_Vdq, "movdqa Wx,Vx");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                              ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(u128Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u128Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/** Opcode 0xf3 0x0f 0x7f - movdqu Wx,Vx */
+FNIEMOP_DEF(iemOp_movdqu_Wx_Vx)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_MNEMONIC(movdqu_Wdq_Vdq, "movdqu Wx,Vx");
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                              ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(u128Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u128Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/*  Opcode 0xf2 0x0f 0x7f - invalid */
+
+
+
+/** Opcode 0x0f 0x80. */
+FNIEMOP_DEF(iemOp_jo_Jv)
+{
+    IEMOP_MNEMONIC(jo_Jv, "jo  Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x81. */
+FNIEMOP_DEF(iemOp_jno_Jv)
+{
+    IEMOP_MNEMONIC(jno_Jv, "jno Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x82. */
+FNIEMOP_DEF(iemOp_jc_Jv)
+{
+    IEMOP_MNEMONIC(jc_Jv, "jc/jb/jnae Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x83. */
+FNIEMOP_DEF(iemOp_jnc_Jv)
+{
+    IEMOP_MNEMONIC(jnc_Jv, "jnc/jnb/jae Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x84. */
+FNIEMOP_DEF(iemOp_je_Jv)
+{
+    IEMOP_MNEMONIC(je_Jv, "je/jz Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x85. */
+FNIEMOP_DEF(iemOp_jne_Jv)
+{
+    IEMOP_MNEMONIC(jne_Jv, "jne/jnz Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x86. */
+FNIEMOP_DEF(iemOp_jbe_Jv)
+{
+    IEMOP_MNEMONIC(jbe_Jv, "jbe/jna Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x87. */
+FNIEMOP_DEF(iemOp_jnbe_Jv)
+{
+    IEMOP_MNEMONIC(ja_Jv, "jnbe/ja Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x88. */
+FNIEMOP_DEF(iemOp_js_Jv)
+{
+    IEMOP_MNEMONIC(js_Jv, "js  Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x89. */
+FNIEMOP_DEF(iemOp_jns_Jv)
+{
+    IEMOP_MNEMONIC(jns_Jv, "jns Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x8a. */
+FNIEMOP_DEF(iemOp_jp_Jv)
+{
+    IEMOP_MNEMONIC(jp_Jv, "jp  Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x8b. */
+FNIEMOP_DEF(iemOp_jnp_Jv)
+{
+    IEMOP_MNEMONIC(jnp_Jv, "jnp Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x8c. */
+FNIEMOP_DEF(iemOp_jl_Jv)
+{
+    IEMOP_MNEMONIC(jl_Jv, "jl/jnge Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x8d. */
+FNIEMOP_DEF(iemOp_jnl_Jv)
+{
+    IEMOP_MNEMONIC(jge_Jv, "jnl/jge Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x8e. */
+FNIEMOP_DEF(iemOp_jle_Jv)
+{
+    IEMOP_MNEMONIC(jle_Jv, "jle/jng Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x8f. */
+FNIEMOP_DEF(iemOp_jnle_Jv)
+{
+    IEMOP_MNEMONIC(jg_Jv, "jnle/jg Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_ADVANCE_RIP();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x90. */
+FNIEMOP_DEF(iemOp_seto_Eb)
+{
+    IEMOP_MNEMONIC(seto_Eb, "seto Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x91. */
+FNIEMOP_DEF(iemOp_setno_Eb)
+{
+    IEMOP_MNEMONIC(setno_Eb, "setno Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x92. */
+FNIEMOP_DEF(iemOp_setc_Eb)
+{
+    IEMOP_MNEMONIC(setc_Eb, "setc Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x93. */
+FNIEMOP_DEF(iemOp_setnc_Eb)
+{
+    IEMOP_MNEMONIC(setnc_Eb, "setnc Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x94. */
+FNIEMOP_DEF(iemOp_sete_Eb)
+{
+    IEMOP_MNEMONIC(sete_Eb, "sete Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x95. */
+FNIEMOP_DEF(iemOp_setne_Eb)
+{
+    IEMOP_MNEMONIC(setne_Eb, "setne Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x96. */
+FNIEMOP_DEF(iemOp_setbe_Eb)
+{
+    IEMOP_MNEMONIC(setbe_Eb, "setbe Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x97. */
+FNIEMOP_DEF(iemOp_setnbe_Eb)
+{
+    IEMOP_MNEMONIC(setnbe_Eb, "setnbe Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x98. */
+FNIEMOP_DEF(iemOp_sets_Eb)
+{
+    IEMOP_MNEMONIC(sets_Eb, "sets Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x99. */
+FNIEMOP_DEF(iemOp_setns_Eb)
+{
+    IEMOP_MNEMONIC(setns_Eb, "setns Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x9a. */
+FNIEMOP_DEF(iemOp_setp_Eb)
+{
+    IEMOP_MNEMONIC(setp_Eb, "setp Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x9b. */
+FNIEMOP_DEF(iemOp_setnp_Eb)
+{
+    IEMOP_MNEMONIC(setnp_Eb, "setnp Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x9c. */
+FNIEMOP_DEF(iemOp_setl_Eb)
+{
+    IEMOP_MNEMONIC(setl_Eb, "setl Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x9d. */
+FNIEMOP_DEF(iemOp_setnl_Eb)
+{
+    IEMOP_MNEMONIC(setnl_Eb, "setnl Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x9e. */
+FNIEMOP_DEF(iemOp_setle_Eb)
+{
+    IEMOP_MNEMONIC(setle_Eb, "setle Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x9f. */
+FNIEMOP_DEF(iemOp_setnle_Eb)
+{
+    IEMOP_MNEMONIC(setnle_Eb, "setnle Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common 'push segment-register' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonPushSReg, uint8_t, iReg)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    Assert(iReg < X86_SREG_FS || pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint16_t, u16Value);
+            IEM_MC_FETCH_SREG_U16(u16Value, iReg);
+            IEM_MC_PUSH_U16(u16Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Value);
+            IEM_MC_FETCH_SREG_ZX_U32(u32Value, iReg);
+            IEM_MC_PUSH_U32_SREG(u32Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_FETCH_SREG_ZX_U64(u64Value, iReg);
+            IEM_MC_PUSH_U64(u64Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            break;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0xa0. */
+FNIEMOP_DEF(iemOp_push_fs)
+{
+    IEMOP_MNEMONIC(push_fs, "push fs");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_FS);
+}
+
+
+/** Opcode 0x0f 0xa1. */
+FNIEMOP_DEF(iemOp_pop_fs)
+{
+    IEMOP_MNEMONIC(pop_fs, "pop fs");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_FS, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/** Opcode 0x0f 0xa2. */
+FNIEMOP_DEF(iemOp_cpuid)
+{
+    IEMOP_MNEMONIC(cpuid, "cpuid");
+    IEMOP_HLP_MIN_486(); /* not all 486es. */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_cpuid);
+}
+
+
+/**
+ * Common worker for iemOp_bt_Ev_Gv, iemOp_btc_Ev_Gv, iemOp_btr_Ev_Gv and
+ * iemOp_bts_Ev_Gv.
+ */
+FNIEMOP_DEF_1(iemOpCommonBit_Ev_Gv, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register destination. */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,        u16Src,                 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_AND_LOCAL_U16(u16Src, 0xf);
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,        u32Src,                 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_AND_LOCAL_U32(u32Src, 0x1f);
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,        u64Src,                 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+                IEM_MC_FETCH_GREG_U64(u64Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_AND_LOCAL_U64(u64Src, 0x3f);
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory destination. */
+
+        uint32_t fAccess;
+        if (pImpl->pfnLockedU16)
+            fAccess = IEM_ACCESS_DATA_RW;
+        else /* BT */
+            fAccess = IEM_ACCESS_DATA_R;
+
+        /** @todo test negative bit offsets! */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,              pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,                u16Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+                IEM_MC_LOCAL(int16_t,               i16AddrAdj);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U16(u16Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_ASSIGN(i16AddrAdj, u16Src);
+                IEM_MC_AND_ARG_U16(u16Src, 0x0f);
+                IEM_MC_SAR_LOCAL_S16(i16AddrAdj, 4);
+                IEM_MC_SHL_LOCAL_S16(i16AddrAdj, 1);
+                IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(GCPtrEffDst, i16AddrAdj);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+
+                IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess);
+
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,              pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,                u32Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+                IEM_MC_LOCAL(int32_t,               i32AddrAdj);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U32(u32Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_ASSIGN(i32AddrAdj, u32Src);
+                IEM_MC_AND_ARG_U32(u32Src, 0x1f);
+                IEM_MC_SAR_LOCAL_S32(i32AddrAdj, 5);
+                IEM_MC_SHL_LOCAL_S32(i32AddrAdj, 2);
+                IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(GCPtrEffDst, i32AddrAdj);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+
+                IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess);
+
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,              pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,                u64Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+                IEM_MC_LOCAL(int64_t,               i64AddrAdj);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U64(u64Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_ASSIGN(i64AddrAdj, u64Src);
+                IEM_MC_AND_ARG_U64(u64Src, 0x3f);
+                IEM_MC_SAR_LOCAL_S64(i64AddrAdj, 6);
+                IEM_MC_SHL_LOCAL_S64(i64AddrAdj, 3);
+                IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(GCPtrEffDst, i64AddrAdj);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+
+                IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess);
+
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0x0f 0xa3. */
+FNIEMOP_DEF(iemOp_bt_Ev_Gv)
+{
+    IEMOP_MNEMONIC(bt_Ev_Gv, "bt  Ev,Gv");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonBit_Ev_Gv, &g_iemAImpl_bt);
+}
+
+
+/**
+ * Common worker for iemOp_shrd_Ev_Gv_Ib and iemOp_shld_Ev_Gv_Ib.
+ */
+FNIEMOP_DEF_1(iemOpCommonShldShrd_Ib, PCIEMOPSHIFTDBLSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF | X86_EFL_OF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,        u16Src,                 1);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg, /*=*/cShift, 2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU16, pu16Dst, u16Src, cShiftArg, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,        u32Src,                 1);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg, /*=*/cShift, 2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU32, pu32Dst, u32Src, cShiftArg, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,        u64Src,                 1);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg, /*=*/cShift, 2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U64(u64Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU64, pu64Dst, u64Src, cShiftArg, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint16_t *,              pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,                u16Src,                 1);
+                IEM_MC_ARG(uint8_t,                 cShiftArg,              2);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+                IEM_MC_ASSIGN(cShiftArg, cShift);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U16(u16Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU16, pu16Dst, u16Src, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint32_t *,              pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,                u32Src,                 1);
+                IEM_MC_ARG(uint8_t,                 cShiftArg,              2);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+                IEM_MC_ASSIGN(cShiftArg, cShift);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U32(u32Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU32, pu32Dst, u32Src, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint64_t *,              pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,                u64Src,                 1);
+                IEM_MC_ARG(uint8_t,                 cShiftArg,              2);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+                IEM_MC_ASSIGN(cShiftArg, cShift);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U64(u64Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU64, pu64Dst, u64Src, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * Common worker for iemOp_shrd_Ev_Gv_CL and iemOp_shld_Ev_Gv_CL.
+ */
+FNIEMOP_DEF_1(iemOpCommonShldShrd_CL, PCIEMOPSHIFTDBLSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF | X86_EFL_OF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,        u16Src,                 1);
+                IEM_MC_ARG(uint8_t,         cShiftArg,              2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU16, pu16Dst, u16Src, cShiftArg, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,        u32Src,                 1);
+                IEM_MC_ARG(uint8_t,         cShiftArg,              2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU32, pu32Dst, u32Src, cShiftArg, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,        u64Src,                 1);
+                IEM_MC_ARG(uint8_t,         cShiftArg,              2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U64(u64Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU64, pu64Dst, u64Src, cShiftArg, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint16_t *,              pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,                u16Src,                 1);
+                IEM_MC_ARG(uint8_t,                 cShiftArg,              2);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U16(u16Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU16, pu16Dst, u16Src, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint32_t *,              pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,                u32Src,                 1);
+                IEM_MC_ARG(uint8_t,                 cShiftArg,              2);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U32(u32Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU32, pu32Dst, u32Src, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint64_t *,              pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,                u64Src,                 1);
+                IEM_MC_ARG(uint8_t,                 cShiftArg,              2);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U64(u64Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU64, pu64Dst, u64Src, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+
+/** Opcode 0x0f 0xa4. */
+FNIEMOP_DEF(iemOp_shld_Ev_Gv_Ib)
+{
+    IEMOP_MNEMONIC(shld_Ev_Gv_Ib, "shld Ev,Gv,Ib");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonShldShrd_Ib, &g_iemAImpl_shld);
+}
+
+
+/** Opcode 0x0f 0xa5. */
+FNIEMOP_DEF(iemOp_shld_Ev_Gv_CL)
+{
+    IEMOP_MNEMONIC(shld_Ev_Gv_CL, "shld Ev,Gv,CL");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonShldShrd_CL, &g_iemAImpl_shld);
+}
+
+
+/** Opcode 0x0f 0xa8. */
+FNIEMOP_DEF(iemOp_push_gs)
+{
+    IEMOP_MNEMONIC(push_gs, "push gs");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_GS);
+}
+
+
+/** Opcode 0x0f 0xa9. */
+FNIEMOP_DEF(iemOp_pop_gs)
+{
+    IEMOP_MNEMONIC(pop_gs, "pop gs");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_GS, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/** Opcode 0x0f 0xaa. */
+FNIEMOP_DEF(iemOp_rsm)
+{
+    IEMOP_MNEMONIC0(FIXED, RSM, rsm, DISOPTYPE_HARMLESS, 0);
+    IEMOP_HLP_MIN_386(); /* 386SL and later. */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rsm);
+}
+
+
+
+/** Opcode 0x0f 0xab. */
+FNIEMOP_DEF(iemOp_bts_Ev_Gv)
+{
+    IEMOP_MNEMONIC(bts_Ev_Gv, "bts Ev,Gv");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonBit_Ev_Gv, &g_iemAImpl_bts);
+}
+
+
+/** Opcode 0x0f 0xac. */
+FNIEMOP_DEF(iemOp_shrd_Ev_Gv_Ib)
+{
+    IEMOP_MNEMONIC(shrd_Ev_Gv_Ib, "shrd Ev,Gv,Ib");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonShldShrd_Ib, &g_iemAImpl_shrd);
+}
+
+
+/** Opcode 0x0f 0xad. */
+FNIEMOP_DEF(iemOp_shrd_Ev_Gv_CL)
+{
+    IEMOP_MNEMONIC(shrd_Ev_Gv_CL, "shrd Ev,Gv,CL");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonShldShrd_CL, &g_iemAImpl_shrd);
+}
+
+
+/** Opcode 0x0f 0xae mem/0. */
+FNIEMOP_DEF_1(iemOp_Grp15_fxsave,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fxsave, "fxsave m512");
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fFxSaveRstor)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_fxsave, iEffSeg, GCPtrEff, enmEffOpSize);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0xae mem/1. */
+FNIEMOP_DEF_1(iemOp_Grp15_fxrstor,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fxrstor, "fxrstor m512");
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fFxSaveRstor)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_fxrstor, iEffSeg, GCPtrEff, enmEffOpSize);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opmaps      grp15
+ * @opcode      !11/2
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_mxcsrsm
+ * @opxcpttype  5
+ * @optest      op1=0      -> mxcsr=0
+ * @optest      op1=0x2083 -> mxcsr=0x2083
+ * @optest      op1=0xfffffffe -> value.xcpt=0xd
+ * @optest      op1=0x2083 cr0|=ts -> value.xcpt=0x7
+ * @optest      op1=0x2083 cr0|=em -> value.xcpt=0x6
+ * @optest      op1=0x2083 cr0|=mp -> mxcsr=0x2083
+ * @optest      op1=0x2083 cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      op1=0x2083 cr0|=ts,em -> value.xcpt=0x6
+ * @optest      op1=0x2083 cr0|=em cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      op1=0x2083 cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      op1=0x2083 cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x6
+ */
+FNIEMOP_DEF_1(iemOp_Grp15_ldmxcsr, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(M_MEM, LDMXCSR, ldmxcsr, Md_RO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_ldmxcsr, iEffSeg, GCPtrEff);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opmaps      grp15
+ * @opcode      !11/3
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_mxcsrsm
+ * @opxcpttype  5
+ * @optest      mxcsr=0      -> op1=0
+ * @optest      mxcsr=0x2083 -> op1=0x2083
+ * @optest      mxcsr=0x2084 cr0|=ts -> value.xcpt=0x7
+ * @optest      mxcsr=0x2085 cr0|=em -> value.xcpt=0x6
+ * @optest      mxcsr=0x2086 cr0|=mp -> op1=0x2086
+ * @optest      mxcsr=0x2087 cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      mxcsr=0x2088 cr0|=ts,em -> value.xcpt=0x6
+ * @optest      mxcsr=0x2089 cr0|=em cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      mxcsr=0x208a cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      mxcsr=0x208b cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x6
+ */
+FNIEMOP_DEF_1(iemOp_Grp15_stmxcsr,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(M_MEM, STMXCSR, stmxcsr, Md_WO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_stmxcsr, iEffSeg, GCPtrEff);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opmaps      grp15
+ * @opcode      !11/4
+ * @oppfx       none
+ * @opcpuid     xsave
+ * @opgroup     og_system
+ * @opxcpttype  none
+ */
+FNIEMOP_DEF_1(iemOp_Grp15_xsave,    uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(M_MEM, XSAVE, xsave, M_RW, DISOPTYPE_HARMLESS, 0);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXSaveRstor)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_xsave, iEffSeg, GCPtrEff, enmEffOpSize);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opmaps      grp15
+ * @opcode      !11/5
+ * @oppfx       none
+ * @opcpuid     xsave
+ * @opgroup     og_system
+ * @opxcpttype  none
+ */
+FNIEMOP_DEF_1(iemOp_Grp15_xrstor,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(M_MEM, XRSTOR, xrstor, M_RO, DISOPTYPE_HARMLESS, 0);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXSaveRstor)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_xrstor, iEffSeg, GCPtrEff, enmEffOpSize);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+/** Opcode 0x0f 0xae mem/6. */
+FNIEMOP_UD_STUB_1(iemOp_Grp15_xsaveopt, uint8_t, bRm);
+
+/**
+ * @opmaps      grp15
+ * @opcode      !11/7
+ * @oppfx       none
+ * @opcpuid     clfsh
+ * @opgroup     og_cachectl
+ * @optest      op1=1 ->
+ */
+FNIEMOP_DEF_1(iemOp_Grp15_clflush,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(M_MEM, CLFLUSH, clflush, Mb_RO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fClFlush)
+        return FNIEMOP_CALL_1(iemOp_InvalidWithRMAllNeeded, bRm);
+
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_clflush_clflushopt, iEffSeg, GCPtrEff);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opmaps      grp15
+ * @opcode      !11/7
+ * @oppfx       0x66
+ * @opcpuid     clflushopt
+ * @opgroup     og_cachectl
+ * @optest      op1=1 ->
+ */
+FNIEMOP_DEF_1(iemOp_Grp15_clflushopt,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(M_MEM, CLFLUSHOPT, clflushopt, Mb_RO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fClFlushOpt)
+        return FNIEMOP_CALL_1(iemOp_InvalidWithRMAllNeeded, bRm);
+
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_clflush_clflushopt, iEffSeg, GCPtrEff);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0xae 11b/5. */
+FNIEMOP_DEF_1(iemOp_Grp15_lfence,   uint8_t, bRm)
+{
+    RT_NOREF_PV(bRm);
+    IEMOP_MNEMONIC(lfence, "lfence");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(0, 0);
+    if (IEM_GET_HOST_CPU_FEATURES(pVCpu)->fSse2)
+        IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_lfence);
+    else
+        IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_alt_mem_fence);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0xae 11b/6. */
+FNIEMOP_DEF_1(iemOp_Grp15_mfence,   uint8_t, bRm)
+{
+    RT_NOREF_PV(bRm);
+    IEMOP_MNEMONIC(mfence, "mfence");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(0, 0);
+    if (IEM_GET_HOST_CPU_FEATURES(pVCpu)->fSse2)
+        IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_mfence);
+    else
+        IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_alt_mem_fence);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0xae 11b/7. */
+FNIEMOP_DEF_1(iemOp_Grp15_sfence,   uint8_t, bRm)
+{
+    RT_NOREF_PV(bRm);
+    IEMOP_MNEMONIC(sfence, "sfence");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(0, 0);
+    if (IEM_GET_HOST_CPU_FEATURES(pVCpu)->fSse2)
+        IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_sfence);
+    else
+        IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_alt_mem_fence);
+    IEM_MC_ADVANCE_RIP();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xf3 0x0f 0xae 11b/0. */
+FNIEMOP_DEF_1(iemOp_Grp15_rdfsbase, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(rdfsbase, "rdfsbase Ry");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint64_t, u64Dst, 0);
+        IEM_MC_FETCH_SREG_BASE_U64(u64Dst, X86_SREG_FS);
+        IEM_MC_STORE_GREG_U64((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u64Dst);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint32_t, u32Dst, 0);
+        IEM_MC_FETCH_SREG_BASE_U32(u32Dst, X86_SREG_FS);
+        IEM_MC_STORE_GREG_U32((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u32Dst);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xf3 0x0f 0xae 11b/1. */
+FNIEMOP_DEF_1(iemOp_Grp15_rdgsbase, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(rdgsbase, "rdgsbase Ry");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint64_t, u64Dst, 0);
+        IEM_MC_FETCH_SREG_BASE_U64(u64Dst, X86_SREG_GS);
+        IEM_MC_STORE_GREG_U64((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u64Dst);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint32_t, u32Dst, 0);
+        IEM_MC_FETCH_SREG_BASE_U32(u32Dst, X86_SREG_GS);
+        IEM_MC_STORE_GREG_U32((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u32Dst);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xf3 0x0f 0xae 11b/2. */
+FNIEMOP_DEF_1(iemOp_Grp15_wrfsbase, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(wrfsbase, "wrfsbase Ry");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint64_t, u64Dst, 0);
+        IEM_MC_FETCH_GREG_U64(u64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(u64Dst);
+        IEM_MC_STORE_SREG_BASE_U64(X86_SREG_FS, u64Dst);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint32_t, u32Dst, 0);
+        IEM_MC_FETCH_GREG_U32(u32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_STORE_SREG_BASE_U64(X86_SREG_FS, u32Dst);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xf3 0x0f 0xae 11b/3. */
+FNIEMOP_DEF_1(iemOp_Grp15_wrgsbase, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(wrgsbase, "wrgsbase Ry");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint64_t, u64Dst, 0);
+        IEM_MC_FETCH_GREG_U64(u64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(u64Dst);
+        IEM_MC_STORE_SREG_BASE_U64(X86_SREG_GS, u64Dst);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint32_t, u32Dst, 0);
+        IEM_MC_FETCH_GREG_U32(u32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_STORE_SREG_BASE_U64(X86_SREG_GS, u32Dst);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Group 15 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup15RegReg[] =
+{   /* pfx:  none,                          066h,                           0f3h,                           0f2h */
+    /* /0 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_Grp15_rdfsbase,           iemOp_InvalidWithRM,
+    /* /1 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_Grp15_rdgsbase,           iemOp_InvalidWithRM,
+    /* /2 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_Grp15_wrfsbase,           iemOp_InvalidWithRM,
+    /* /3 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_Grp15_wrgsbase,           iemOp_InvalidWithRM,
+    /* /4 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /5 */ iemOp_Grp15_lfence,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /6 */ iemOp_Grp15_mfence,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /7 */ iemOp_Grp15_sfence,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup15RegReg) == 8*4);
+
+
+/**
+ * Group 15 jump table for memory variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup15MemReg[] =
+{   /* pfx:  none,                          066h,                           0f3h,                           0f2h */
+    /* /0 */ iemOp_Grp15_fxsave,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /1 */ iemOp_Grp15_fxrstor,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /2 */ iemOp_Grp15_ldmxcsr,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /3 */ iemOp_Grp15_stmxcsr,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /4 */ iemOp_Grp15_xsave,             iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /5 */ iemOp_Grp15_xrstor,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /6 */ iemOp_Grp15_xsaveopt,          iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /7 */ iemOp_Grp15_clflush,           iemOp_Grp15_clflushopt,         iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup15MemReg) == 8*4);
+
+
+/** Opcode 0x0f 0xae. */
+FNIEMOP_DEF(iemOp_Grp15)
+{
+    IEMOP_HLP_MIN_586(); /* Not entirely accurate nor needed, but useful for debugging 286 code. */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnGroup15RegReg[ ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) * 4
+                                                  + pVCpu->iem.s.idxPrefix], bRm);
+    /* memory, register */
+    return FNIEMOP_CALL_1(g_apfnGroup15MemReg[ ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) * 4
+                                              + pVCpu->iem.s.idxPrefix], bRm);
+}
+
+
+/** Opcode 0x0f 0xaf. */
+FNIEMOP_DEF(iemOp_imul_Gv_Ev)
+{
+    IEMOP_MNEMONIC(imul_Gv_Ev, "imul Gv,Ev");
+    IEMOP_HLP_MIN_386();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_imul_two);
+}
+
+
+/** Opcode 0x0f 0xb0. */
+FNIEMOP_DEF(iemOp_cmpxchg_Eb_Gb)
+{
+    IEMOP_MNEMONIC(cmpxchg_Eb_Gb, "cmpxchg Eb,Gb");
+    IEMOP_HLP_MIN_486();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING();
+        IEM_MC_BEGIN(4, 0);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,                 0);
+        IEM_MC_ARG(uint8_t *,       pu8Al,                  1);
+        IEM_MC_ARG(uint8_t,         u8Src,                  2);
+        IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+        IEM_MC_FETCH_GREG_U8(u8Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_GREG_U8(pu8Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_GREG_U8(pu8Al, X86_GREG_xAX);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u8, pu8Dst, pu8Al, u8Src, pEFlags);
+        else
+            IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u8_locked, pu8Dst, pu8Al, u8Src, pEFlags);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,                 0);
+        IEM_MC_ARG(uint8_t *,       pu8Al,                  1);
+        IEM_MC_ARG(uint8_t,         u8Src,                  2);
+        IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        3);
+        IEM_MC_LOCAL(RTGCPTR,       GCPtrEffDst);
+        IEM_MC_LOCAL(uint8_t,       u8Al);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING();
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        IEM_MC_FETCH_GREG_U8(u8Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_FETCH_GREG_U8(u8Al, X86_GREG_xAX);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        IEM_MC_REF_LOCAL(pu8Al, u8Al);
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u8, pu8Dst, pu8Al, u8Src, pEFlags);
+        else
+            IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u8_locked, pu8Dst, pu8Al, u8Src, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Al);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/** Opcode 0x0f 0xb1. */
+FNIEMOP_DEF(iemOp_cmpxchg_Ev_Gv)
+{
+    IEMOP_MNEMONIC(cmpxchg_Ev_Gv, "cmpxchg Ev,Gv");
+    IEMOP_HLP_MIN_486();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint16_t *,      pu16Ax,                 1);
+                IEM_MC_ARG(uint16_t,        u16Src,                 2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_GREG_U16(pu16Ax, X86_GREG_xAX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u16, pu16Dst, pu16Ax, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u16_locked, pu16Dst, pu16Ax, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t *,      pu32Eax,                1);
+                IEM_MC_ARG(uint32_t,        u32Src,                 2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_GREG_U32(pu32Eax, X86_GREG_xAX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u32, pu32Dst, pu32Eax, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u32_locked, pu32Dst, pu32Eax, u32Src, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Eax);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint64_t *,      pu64Rax,                1);
+#ifdef RT_ARCH_X86
+                IEM_MC_ARG(uint64_t *,      pu64Src,                2);
+#else
+                IEM_MC_ARG(uint64_t,        u64Src,                 2);
+#endif
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_GREG_U64(pu64Rax, X86_GREG_xAX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+#ifdef RT_ARCH_X86
+                IEM_MC_REF_GREG_U64(pu64Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64, pu64Dst, pu64Rax, pu64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64_locked, pu64Dst, pu64Rax, pu64Src, pEFlags);
+#else
+                IEM_MC_FETCH_GREG_U64(u64Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64, pu64Dst, pu64Rax, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64_locked, pu64Dst, pu64Rax, u64Src, pEFlags);
+#endif
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(4, 3);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint16_t *,      pu16Ax,                 1);
+                IEM_MC_ARG(uint16_t,        u16Src,                 2);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,       GCPtrEffDst);
+                IEM_MC_LOCAL(uint16_t,      u16Ax);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING();
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_FETCH_GREG_U16(u16Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_GREG_U16(u16Ax, X86_GREG_xAX);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_REF_LOCAL(pu16Ax, u16Ax);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u16, pu16Dst, pu16Ax, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u16_locked, pu16Dst, pu16Ax, u16Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Ax);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(4, 3);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t *,      pu32Eax,                 1);
+                IEM_MC_ARG(uint32_t,        u32Src,                 2);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,       GCPtrEffDst);
+                IEM_MC_LOCAL(uint32_t,      u32Eax);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING();
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_FETCH_GREG_U32(u32Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_FETCH_GREG_U32(u32Eax, X86_GREG_xAX);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_REF_LOCAL(pu32Eax, u32Eax);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u32, pu32Dst, pu32Eax, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u32_locked, pu32Dst, pu32Eax, u32Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_STORE_GREG_U32(X86_GREG_xAX, u32Eax);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(4, 3);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint64_t *,      pu64Rax,                1);
+#ifdef RT_ARCH_X86
+                IEM_MC_ARG(uint64_t *,      pu64Src,                2);
+#else
+                IEM_MC_ARG(uint64_t,        u64Src,                 2);
+#endif
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,       GCPtrEffDst);
+                IEM_MC_LOCAL(uint64_t,      u64Rax);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING();
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_FETCH_GREG_U64(u64Rax, X86_GREG_xAX);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_REF_LOCAL(pu64Rax, u64Rax);
+#ifdef RT_ARCH_X86
+                IEM_MC_REF_GREG_U64(pu64Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64, pu64Dst, pu64Rax, pu64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64_locked, pu64Dst, pu64Rax, pu64Src, pEFlags);
+#else
+                IEM_MC_FETCH_GREG_U64(u64Src, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64, pu64Dst, pu64Rax, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64_locked, pu64Dst, pu64Rax, u64Src, pEFlags);
+#endif
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u64Rax);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+FNIEMOP_DEF_2(iemOpCommonLoadSRegAndGreg, uint8_t, iSegReg, uint8_t, bRm)
+{
+    Assert((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT)); /* Caller checks this */
+    uint8_t const iGReg = ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg;
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(5, 1);
+            IEM_MC_ARG(uint16_t,        uSel,                                    0);
+            IEM_MC_ARG(uint16_t,        offSeg,                                  1);
+            IEM_MC_ARG_CONST(uint8_t,   iSegRegArg,/*=*/iSegReg,                 2);
+            IEM_MC_ARG_CONST(uint8_t,   iGRegArg,  /*=*/iGReg,                   3);
+            IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 4);
+            IEM_MC_LOCAL(RTGCPTR,       GCPtrEff);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEff);
+            IEM_MC_FETCH_MEM_U16_DISP(uSel, pVCpu->iem.s.iEffSeg, GCPtrEff, 2);
+            IEM_MC_CALL_CIMPL_5(iemCImpl_load_SReg_Greg, uSel, offSeg, iSegRegArg, iGRegArg, enmEffOpSize);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(5, 1);
+            IEM_MC_ARG(uint16_t,        uSel,                                    0);
+            IEM_MC_ARG(uint32_t,        offSeg,                                  1);
+            IEM_MC_ARG_CONST(uint8_t,   iSegRegArg,/*=*/iSegReg,                 2);
+            IEM_MC_ARG_CONST(uint8_t,   iGRegArg,  /*=*/iGReg,                   3);
+            IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 4);
+            IEM_MC_LOCAL(RTGCPTR,       GCPtrEff);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U32(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEff);
+            IEM_MC_FETCH_MEM_U16_DISP(uSel, pVCpu->iem.s.iEffSeg, GCPtrEff, 4);
+            IEM_MC_CALL_CIMPL_5(iemCImpl_load_SReg_Greg, uSel, offSeg, iSegRegArg, iGRegArg, enmEffOpSize);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(5, 1);
+            IEM_MC_ARG(uint16_t,        uSel,                                    0);
+            IEM_MC_ARG(uint64_t,        offSeg,                                  1);
+            IEM_MC_ARG_CONST(uint8_t,   iSegRegArg,/*=*/iSegReg,                 2);
+            IEM_MC_ARG_CONST(uint8_t,   iGRegArg,  /*=*/iGReg,                   3);
+            IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 4);
+            IEM_MC_LOCAL(RTGCPTR,       GCPtrEff);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            if (IEM_IS_GUEST_CPU_AMD(pVCpu)) /** @todo testcase: rev 3.15 of the amd manuals claims it only loads a 32-bit greg. */
+                IEM_MC_FETCH_MEM_U32_SX_U64(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEff);
+            else
+                IEM_MC_FETCH_MEM_U64(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEff);
+            IEM_MC_FETCH_MEM_U16_DISP(uSel, pVCpu->iem.s.iEffSeg, GCPtrEff, 8);
+            IEM_MC_CALL_CIMPL_5(iemCImpl_load_SReg_Greg, uSel, offSeg, iSegRegArg, iGRegArg, enmEffOpSize);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/** Opcode 0x0f 0xb2. */
+FNIEMOP_DEF(iemOp_lss_Gv_Mp)
+{
+    IEMOP_MNEMONIC(lss_Gv_Mp, "lss Gv,Mp");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_SS, bRm);
+}
+
+
+/** Opcode 0x0f 0xb3. */
+FNIEMOP_DEF(iemOp_btr_Ev_Gv)
+{
+    IEMOP_MNEMONIC(btr_Ev_Gv, "btr Ev,Gv");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonBit_Ev_Gv, &g_iemAImpl_btr);
+}
+
+
+/** Opcode 0x0f 0xb4. */
+FNIEMOP_DEF(iemOp_lfs_Gv_Mp)
+{
+    IEMOP_MNEMONIC(lfs_Gv_Mp, "lfs Gv,Mp");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_FS, bRm);
+}
+
+
+/** Opcode 0x0f 0xb5. */
+FNIEMOP_DEF(iemOp_lgs_Gv_Mp)
+{
+    IEMOP_MNEMONIC(lgs_Gv_Mp, "lgs Gv,Mp");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_GS, bRm);
+}
+
+
+/** Opcode 0x0f 0xb6. */
+FNIEMOP_DEF(iemOp_movzx_Gv_Eb)
+{
+    IEMOP_MNEMONIC(movzx_Gv_Eb, "movzx Gv,Eb");
+    IEMOP_HLP_MIN_386();
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_FETCH_GREG_U8_ZX_U16(u16Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_FETCH_GREG_U8_ZX_U32(u32Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_FETCH_GREG_U8_ZX_U64(u64Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U8_ZX_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U8_ZX_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U8_ZX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0x0f 0xb7. */
+FNIEMOP_DEF(iemOp_movzx_Gv_Ew)
+{
+    IEMOP_MNEMONIC(movzx_Gv_Ew, "movzx Gv,Ew");
+    IEMOP_HLP_MIN_386();
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Not entirely sure how the operand size prefix is handled here,
+     *        assuming that it will be ignored. Would be nice to have a few
+     *        test for this. */
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Value);
+            IEM_MC_FETCH_GREG_U16_ZX_U32(u32Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_FETCH_GREG_U16_ZX_U64(u64Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint32_t, u32Value);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16_ZX_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+            IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16_ZX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+            IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode      0x0f 0xb8 - JMPE (reserved for emulator on IPF) */
+FNIEMOP_UD_STUB(iemOp_jmpe);
+/** Opcode 0xf3 0x0f 0xb8 - POPCNT Gv, Ev */
+FNIEMOP_STUB(iemOp_popcnt_Gv_Ev);
+
+
+/**
+ * @opcode      0xb9
+ * @opinvalid   intel-modrm
+ * @optest      ->
+ */
+FNIEMOP_DEF(iemOp_Grp10)
+{
+    /*
+     * AMD does not decode beyond the 0xb9 whereas intel does the modr/m bit
+     * too. See bs3-cpu-decoder-1.c32.  So, we can forward to iemOp_InvalidNeedRM.
+     */
+    Log(("iemOp_Grp10 aka UD1 -> #UD\n"));
+    IEMOP_MNEMONIC2EX(ud1, "ud1", RM, UD1, ud1, Gb, Eb, DISOPTYPE_INVALID, IEMOPHINT_IGNORES_OP_SIZES); /* just picked Gb,Eb here. */
+    return FNIEMOP_CALL(iemOp_InvalidNeedRM);
+}
+
+
+/** Opcode 0x0f 0xba. */
+FNIEMOP_DEF(iemOp_Grp8)
+{
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPBINSIZES pImpl;
+    switch ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)
+    {
+        case 0: case 1: case 2: case 3:
+            /* Both AMD and Intel want full modr/m decoding and imm8. */
+            return FNIEMOP_CALL_1(iemOp_InvalidWithRMAllNeedImm8, bRm);
+        case 4: pImpl = &g_iemAImpl_bt;  IEMOP_MNEMONIC(bt_Ev_Ib,  "bt  Ev,Ib"); break;
+        case 5: pImpl = &g_iemAImpl_bts; IEMOP_MNEMONIC(bts_Ev_Ib, "bts Ev,Ib"); break;
+        case 6: pImpl = &g_iemAImpl_btr; IEMOP_MNEMONIC(btr_Ev_Ib, "btr Ev,Ib"); break;
+        case 7: pImpl = &g_iemAImpl_btc; IEMOP_MNEMONIC(btc_Ev_Ib, "btc Ev,Ib"); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* register destination. */
+        uint8_t u8Bit; IEM_OPCODE_GET_NEXT_U8(&u8Bit);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                    0);
+                IEM_MC_ARG_CONST(uint16_t,  u16Src, /*=*/ u8Bit & 0x0f, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                    0);
+                IEM_MC_ARG_CONST(uint32_t,  u32Src, /*=*/ u8Bit & 0x1f, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                    0);
+                IEM_MC_ARG_CONST(uint64_t,  u64Src, /*=*/ u8Bit & 0x3f, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory destination. */
+
+        uint32_t fAccess;
+        if (pImpl->pfnLockedU16)
+            fAccess = IEM_ACCESS_DATA_RW;
+        else /* BT */
+            fAccess = IEM_ACCESS_DATA_R;
+
+        /** @todo test negative bit offsets! */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t *,              pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,                u16Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t u8Bit; IEM_OPCODE_GET_NEXT_U8(&u8Bit);
+                IEM_MC_ASSIGN(u16Src, u8Bit & 0x0f);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess);
+
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint32_t *,              pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,                u32Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t u8Bit; IEM_OPCODE_GET_NEXT_U8(&u8Bit);
+                IEM_MC_ASSIGN(u32Src, u8Bit & 0x1f);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess);
+
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint64_t *,              pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,                u64Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t u8Bit; IEM_OPCODE_GET_NEXT_U8(&u8Bit);
+                IEM_MC_ASSIGN(u64Src, u8Bit & 0x3f);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess);
+
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0x0f 0xbb. */
+FNIEMOP_DEF(iemOp_btc_Ev_Gv)
+{
+    IEMOP_MNEMONIC(btc_Ev_Gv, "btc Ev,Gv");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonBit_Ev_Gv, &g_iemAImpl_btc);
+}
+
+
+/** Opcode 0x0f 0xbc. */
+FNIEMOP_DEF(iemOp_bsf_Gv_Ev)
+{
+    IEMOP_MNEMONIC(bsf_Gv_Ev, "bsf Gv,Ev");
+    IEMOP_HLP_MIN_386();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_bsf);
+}
+
+
+/** Opcode 0xf3 0x0f 0xbc - TZCNT Gv, Ev */
+FNIEMOP_STUB(iemOp_tzcnt_Gv_Ev);
+
+
+/** Opcode 0x0f 0xbd. */
+FNIEMOP_DEF(iemOp_bsr_Gv_Ev)
+{
+    IEMOP_MNEMONIC(bsr_Gv_Ev, "bsr Gv,Ev");
+    IEMOP_HLP_MIN_386();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_bsr);
+}
+
+
+/** Opcode 0xf3 0x0f 0xbd - LZCNT Gv, Ev */
+FNIEMOP_STUB(iemOp_lzcnt_Gv_Ev);
+
+
+/** Opcode 0x0f 0xbe. */
+FNIEMOP_DEF(iemOp_movsx_Gv_Eb)
+{
+    IEMOP_MNEMONIC(movsx_Gv_Eb, "movsx Gv,Eb");
+    IEMOP_HLP_MIN_386();
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_FETCH_GREG_U8_SX_U16(u16Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_FETCH_GREG_U8_SX_U32(u32Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_FETCH_GREG_U8_SX_U64(u64Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U8_SX_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U8_SX_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U8_SX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0x0f 0xbf. */
+FNIEMOP_DEF(iemOp_movsx_Gv_Ew)
+{
+    IEMOP_MNEMONIC(movsx_Gv_Ew, "movsx Gv,Ew");
+    IEMOP_HLP_MIN_386();
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Not entirely sure how the operand size prefix is handled here,
+     *        assuming that it will be ignored. Would be nice to have a few
+     *        test for this. */
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Value);
+            IEM_MC_FETCH_GREG_U16_SX_U32(u32Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_FETCH_GREG_U16_SX_U64(u64Value, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint32_t, u32Value);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16_SX_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+            IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16_SX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+            IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Value);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0xc0. */
+FNIEMOP_DEF(iemOp_xadd_Eb_Gb)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_HLP_MIN_486();
+    IEMOP_MNEMONIC(xadd_Eb_Gb, "xadd Eb,Gb");
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,  pu8Dst,  0);
+        IEM_MC_ARG(uint8_t *,  pu8Reg,  1);
+        IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+        IEM_MC_REF_GREG_U8(pu8Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_REF_GREG_U8(pu8Reg, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u8, pu8Dst, pu8Reg, pEFlags);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+        IEM_MC_BEGIN(3, 3);
+        IEM_MC_ARG(uint8_t *,   pu8Dst,          0);
+        IEM_MC_ARG(uint8_t *,   pu8Reg,          1);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+        IEM_MC_LOCAL(uint8_t,  u8RegCopy);
+        IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_GREG_U8(u8RegCopy, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_LOCAL(pu8Reg, u8RegCopy);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u8, pu8Dst, pu8Reg, pEFlags);
+        else
+            IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u8_locked, pu8Dst, pu8Reg, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_STORE_GREG_U8(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u8RegCopy);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0xc1. */
+FNIEMOP_DEF(iemOp_xadd_Ev_Gv)
+{
+    IEMOP_MNEMONIC(xadd_Ev_Gv, "xadd Ev,Gv");
+    IEMOP_HLP_MIN_486();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *, pu16Dst,  0);
+                IEM_MC_ARG(uint16_t *, pu16Reg,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_REF_GREG_U16(pu16Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_GREG_U16(pu16Reg, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u16, pu16Dst, pu16Reg, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *, pu32Dst,  0);
+                IEM_MC_ARG(uint32_t *, pu32Reg,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_REF_GREG_U32(pu32Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_GREG_U32(pu32Reg, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u32, pu32Dst, pu32Reg, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Reg);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *, pu64Dst,  0);
+                IEM_MC_ARG(uint64_t *, pu64Reg,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+                IEM_MC_REF_GREG_U64(pu64Reg, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u64, pu64Dst, pu64Reg, pEFlags);
+
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 3);
+                IEM_MC_ARG(uint16_t *,  pu16Dst,         0);
+                IEM_MC_ARG(uint16_t *,  pu16Reg,         1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(uint16_t,  u16RegCopy);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_GREG_U16(u16RegCopy, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_LOCAL(pu16Reg, u16RegCopy);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u16, pu16Dst, pu16Reg, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u16_locked, pu16Dst, pu16Reg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_STORE_GREG_U16(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u16RegCopy);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 3);
+                IEM_MC_ARG(uint32_t *,  pu32Dst,         0);
+                IEM_MC_ARG(uint32_t *,  pu32Reg,         1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(uint32_t,  u32RegCopy);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_GREG_U32(u32RegCopy, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_LOCAL(pu32Reg, u32RegCopy);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u32, pu32Dst, pu32Reg, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u32_locked, pu32Dst, pu32Reg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_STORE_GREG_U32(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32RegCopy);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 3);
+                IEM_MC_ARG(uint64_t *,  pu64Dst,         0);
+                IEM_MC_ARG(uint64_t *,  pu64Reg,         1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(uint64_t,  u64RegCopy);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_GREG_U64(u64RegCopy, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_REF_LOCAL(pu64Reg, u64RegCopy);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u64, pu64Dst, pu64Reg, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u64_locked, pu64Dst, pu64Reg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_STORE_GREG_U64(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64RegCopy);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode      0x0f 0xc2 - cmpps Vps,Wps,Ib */
+FNIEMOP_STUB(iemOp_cmpps_Vps_Wps_Ib);
+/** Opcode 0x66 0x0f 0xc2 - cmppd Vpd,Wpd,Ib */
+FNIEMOP_STUB(iemOp_cmppd_Vpd_Wpd_Ib);
+/** Opcode 0xf3 0x0f 0xc2 - cmpss Vss,Wss,Ib */
+FNIEMOP_STUB(iemOp_cmpss_Vss_Wss_Ib);
+/** Opcode 0xf2 0x0f 0xc2 - cmpsd Vsd,Wsd,Ib */
+FNIEMOP_STUB(iemOp_cmpsd_Vsd_Wsd_Ib);
+
+
+/** Opcode 0x0f 0xc3. */
+FNIEMOP_DEF(iemOp_movnti_My_Gy)
+{
+    IEMOP_MNEMONIC(movnti_My_Gy, "movnti My,Gy");
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /* Only the register -> memory form makes sense, assuming #UD for the other form. */
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2)
+                    return IEMOP_RAISE_INVALID_OPCODE();
+
+                IEM_MC_FETCH_GREG_U32(u32Value, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u32Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2)
+                    return IEMOP_RAISE_INVALID_OPCODE();
+
+                IEM_MC_FETCH_GREG_U64(u64Value, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+                IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u64Value);
+                IEM_MC_ADVANCE_RIP();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_16BIT:
+                /** @todo check this form.   */
+                return IEMOP_RAISE_INVALID_OPCODE();
+        }
+    }
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return VINF_SUCCESS;
+}
+/*  Opcode 0x66 0x0f 0xc3 - invalid */
+/*  Opcode 0xf3 0x0f 0xc3 - invalid */
+/*  Opcode 0xf2 0x0f 0xc3 - invalid */
+
+/** Opcode      0x0f 0xc4 - pinsrw Pq, Ry/Mw,Ib */
+FNIEMOP_STUB(iemOp_pinsrw_Pq_RyMw_Ib);
+/** Opcode 0x66 0x0f 0xc4 - pinsrw Vdq, Ry/Mw,Ib */
+FNIEMOP_STUB(iemOp_pinsrw_Vdq_RyMw_Ib);
+/*  Opcode 0xf3 0x0f 0xc4 - invalid */
+/*  Opcode 0xf2 0x0f 0xc4 - invalid */
+
+/** Opcode      0x0f 0xc5 - pextrw Gd, Nq, Ib */
+FNIEMOP_STUB(iemOp_pextrw_Gd_Nq_Ib);
+/** Opcode 0x66 0x0f 0xc5 - pextrw Gd, Udq, Ib */
+FNIEMOP_STUB(iemOp_pextrw_Gd_Udq_Ib);
+/*  Opcode 0xf3 0x0f 0xc5 - invalid */
+/*  Opcode 0xf2 0x0f 0xc5 - invalid */
+
+/** Opcode      0x0f 0xc6 - shufps Vps, Wps, Ib */
+FNIEMOP_STUB(iemOp_shufps_Vps_Wps_Ib);
+/** Opcode 0x66 0x0f 0xc6 - shufpd Vpd, Wpd, Ib */
+FNIEMOP_STUB(iemOp_shufpd_Vpd_Wpd_Ib);
+/*  Opcode 0xf3 0x0f 0xc6 - invalid */
+/*  Opcode 0xf2 0x0f 0xc6 - invalid */
+
+
+/** Opcode 0x0f 0xc7 !11/1. */
+FNIEMOP_DEF_1(iemOp_Grp9_cmpxchg8b_Mq, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(cmpxchg8b, "cmpxchg8b Mq");
+
+    IEM_MC_BEGIN(4, 3);
+    IEM_MC_ARG(uint64_t *, pu64MemDst,     0);
+    IEM_MC_ARG(PRTUINT64U, pu64EaxEdx,     1);
+    IEM_MC_ARG(PRTUINT64U, pu64EbxEcx,     2);
+    IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 3);
+    IEM_MC_LOCAL(RTUINT64U, u64EaxEdx);
+    IEM_MC_LOCAL(RTUINT64U, u64EbxEcx);
+    IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING();
+    IEM_MC_MEM_MAP(pu64MemDst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+
+    IEM_MC_FETCH_GREG_U32(u64EaxEdx.s.Lo, X86_GREG_xAX);
+    IEM_MC_FETCH_GREG_U32(u64EaxEdx.s.Hi, X86_GREG_xDX);
+    IEM_MC_REF_LOCAL(pu64EaxEdx, u64EaxEdx);
+
+    IEM_MC_FETCH_GREG_U32(u64EbxEcx.s.Lo, X86_GREG_xBX);
+    IEM_MC_FETCH_GREG_U32(u64EbxEcx.s.Hi, X86_GREG_xCX);
+    IEM_MC_REF_LOCAL(pu64EbxEcx, u64EbxEcx);
+
+    IEM_MC_FETCH_EFLAGS(EFlags);
+    if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg8b, pu64MemDst, pu64EaxEdx, pu64EbxEcx, pEFlags);
+    else
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg8b_locked, pu64MemDst, pu64EaxEdx, pu64EbxEcx, pEFlags);
+
+    IEM_MC_MEM_COMMIT_AND_UNMAP(pu64MemDst, IEM_ACCESS_DATA_RW);
+    IEM_MC_COMMIT_EFLAGS(EFlags);
+    IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF)
+        /** @todo Testcase: Check effect of cmpxchg8b on bits 63:32 in rax and rdx. */
+        IEM_MC_STORE_GREG_U32(X86_GREG_xAX, u64EaxEdx.s.Lo);
+        IEM_MC_STORE_GREG_U32(X86_GREG_xDX, u64EaxEdx.s.Hi);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP();
+
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode REX.W 0x0f 0xc7 !11/1. */
+FNIEMOP_DEF_1(iemOp_Grp9_cmpxchg16b_Mdq, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(cmpxchg16b, "cmpxchg16b Mdq");
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovCmpXchg16b)
+    {
+#if 0
+        RT_NOREF(bRm);
+        IEMOP_BITCH_ABOUT_STUB();
+        return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#else
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_ARG(PRTUINT128U, pu128MemDst,     0);
+        IEM_MC_ARG(PRTUINT128U, pu128RaxRdx,     1);
+        IEM_MC_ARG(PRTUINT128U, pu128RbxRcx,     2);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 3);
+        IEM_MC_LOCAL(RTUINT128U, u128RaxRdx);
+        IEM_MC_LOCAL(RTUINT128U, u128RbxRcx);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING();
+        IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(GCPtrEffDst, 16);
+        IEM_MC_MEM_MAP(pu128MemDst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+
+        IEM_MC_FETCH_GREG_U64(u128RaxRdx.s.Lo, X86_GREG_xAX);
+        IEM_MC_FETCH_GREG_U64(u128RaxRdx.s.Hi, X86_GREG_xDX);
+        IEM_MC_REF_LOCAL(pu128RaxRdx, u128RaxRdx);
+
+        IEM_MC_FETCH_GREG_U64(u128RbxRcx.s.Lo, X86_GREG_xBX);
+        IEM_MC_FETCH_GREG_U64(u128RbxRcx.s.Hi, X86_GREG_xCX);
+        IEM_MC_REF_LOCAL(pu128RbxRcx, u128RbxRcx);
+
+        IEM_MC_FETCH_EFLAGS(EFlags);
+# ifdef RT_ARCH_AMD64
+        if (IEM_GET_HOST_CPU_FEATURES(pVCpu)->fMovCmpXchg16b)
+        {
+            if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+            else
+                IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_locked, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+        }
+        else
+# endif
+        {
+            /* Note! The fallback for 32-bit systems and systems without CX16 is multiple
+                     accesses and not all all atomic, which works fine on in UNI CPU guest
+                     configuration (ignoring DMA).  If guest SMP is active we have no choice
+                     but to use a rendezvous callback here.  Sigh. */
+            if (pVCpu->CTX_SUFF(pVM)->cCpus == 1)
+                IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_fallback, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+            else
+            {
+                IEM_MC_CALL_CIMPL_4(iemCImpl_cmpxchg16b_fallback_rendezvous, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+                /* Does not get here, tail code is duplicated in iemCImpl_cmpxchg16b_fallback_rendezvous. */
+            }
+        }
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu128MemDst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF)
+            IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u128RaxRdx.s.Lo);
+            IEM_MC_STORE_GREG_U64(X86_GREG_xDX, u128RaxRdx.s.Hi);
+        IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP();
+
+        IEM_MC_END();
+        return VINF_SUCCESS;
+#endif
+    }
+    Log(("cmpxchg16b -> #UD\n"));
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+FNIEMOP_DEF_1(iemOp_Grp9_cmpxchg8bOr16b, uint8_t, bRm)
+{
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+        return FNIEMOP_CALL_1(iemOp_Grp9_cmpxchg16b_Mdq, bRm);
+    return FNIEMOP_CALL_1(iemOp_Grp9_cmpxchg8b_Mq, bRm);
+}
+
+/** Opcode 0x0f 0xc7 11/6. */
+FNIEMOP_UD_STUB_1(iemOp_Grp9_rdrand_Rv, uint8_t, bRm);
+
+/** Opcode 0x0f 0xc7 !11/6. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF_1(iemOp_Grp9_vmptrld_Mq, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(vmptrld, "vmptrld");
+    IEMOP_HLP_IN_VMX_OPERATION("vmptrld", kVmxVDiag_Vmptrld);
+    IEMOP_HLP_VMX_INSTR("vmptrld", kVmxVDiag_Vmptrld);
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t, iEffSeg,     0);
+    IEM_MC_ARG(RTGCPTR, GCPtrEffSrc, 1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_vmptrld, iEffSeg, GCPtrEffSrc);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+#else
+FNIEMOP_UD_STUB_1(iemOp_Grp9_vmptrld_Mq, uint8_t, bRm);
+#endif
+
+/** Opcode 0x66 0x0f 0xc7 !11/6. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF_1(iemOp_Grp9_vmclear_Mq, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(vmclear, "vmclear");
+    IEMOP_HLP_IN_VMX_OPERATION("vmclear", kVmxVDiag_Vmclear);
+    IEMOP_HLP_VMX_INSTR("vmclear", kVmxVDiag_Vmclear);
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t, iEffSeg,     0);
+    IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_vmclear, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+#else
+FNIEMOP_UD_STUB_1(iemOp_Grp9_vmclear_Mq, uint8_t, bRm);
+#endif
+
+/** Opcode 0xf3 0x0f 0xc7 !11/6. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF_1(iemOp_Grp9_vmxon_Mq, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(vmxon, "vmxon");
+    IEMOP_HLP_VMX_INSTR("vmxon", kVmxVDiag_Vmxon);
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t, iEffSeg,     0);
+    IEM_MC_ARG(RTGCPTR, GCPtrEffSrc, 1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_vmxon, iEffSeg, GCPtrEffSrc);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+#else
+FNIEMOP_UD_STUB_1(iemOp_Grp9_vmxon_Mq, uint8_t, bRm);
+#endif
+
+/** Opcode [0xf3] 0x0f 0xc7 !11/7. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF_1(iemOp_Grp9_vmptrst_Mq, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(vmptrst, "vmptrst");
+    IEMOP_HLP_IN_VMX_OPERATION("vmptrst", kVmxVDiag_Vmptrst);
+    IEMOP_HLP_VMX_INSTR("vmptrst", kVmxVDiag_Vmptrst);
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t, iEffSeg,     0);
+    IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_vmptrst, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+#else
+FNIEMOP_UD_STUB_1(iemOp_Grp9_vmptrst_Mq, uint8_t, bRm);
+#endif
+
+/** Opcode 0x0f 0xc7 11/7. */
+FNIEMOP_UD_STUB_1(iemOp_Grp9_rdseed_Rv, uint8_t, bRm);
+
+
+/**
+ * Group 9 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup9RegReg[] =
+{   /* pfx:  none,                          066h,                           0f3h,                           0f2h */
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /2 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /3 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /4 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /6 */ iemOp_Grp9_rdrand_Rv,          iemOp_Grp9_rdrand_Rv,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /7 */ iemOp_Grp9_rdseed_Rv,          iemOp_Grp9_rdseed_Rv,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup9RegReg) == 8*4);
+
+
+/**
+ * Group 9 jump table for memory variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup9MemReg[] =
+{   /* pfx:  none,                          066h,                           0f3h,                           0f2h */
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /1 */ iemOp_Grp9_cmpxchg8bOr16b,     iemOp_Grp9_cmpxchg8bOr16b,      iemOp_Grp9_cmpxchg8bOr16b,      iemOp_Grp9_cmpxchg8bOr16b, /* see bs3-cpu-decoding-1 */
+    /* /2 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /3 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /4 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /6 */ iemOp_Grp9_vmptrld_Mq,         iemOp_Grp9_vmclear_Mq,          iemOp_Grp9_vmxon_Mq,            iemOp_InvalidWithRM,
+    /* /7 */ iemOp_Grp9_vmptrst_Mq,         iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup9MemReg) == 8*4);
+
+
+/** Opcode 0x0f 0xc7. */
+FNIEMOP_DEF(iemOp_Grp9)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_RM(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnGroup9RegReg[ ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) * 4
+                                                 + pVCpu->iem.s.idxPrefix], bRm);
+    /* memory, register */
+    return FNIEMOP_CALL_1(g_apfnGroup9MemReg[ ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) * 4
+                                             + pVCpu->iem.s.idxPrefix], bRm);
+}
+
+
+/**
+ * Common 'bswap register' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonBswapGReg, uint8_t, iReg)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(1, 0);
+            IEM_MC_ARG(uint32_t *,  pu32Dst, 0);
+            IEM_MC_REF_GREG_U32(pu32Dst, iReg);     /* Don't clear the high dword! */
+            IEM_MC_CALL_VOID_AIMPL_1(iemAImpl_bswap_u16, pu32Dst);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(1, 0);
+            IEM_MC_ARG(uint32_t *,  pu32Dst, 0);
+            IEM_MC_REF_GREG_U32(pu32Dst, iReg);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+            IEM_MC_CALL_VOID_AIMPL_1(iemAImpl_bswap_u32, pu32Dst);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(1, 0);
+            IEM_MC_ARG(uint64_t *,  pu64Dst, 0);
+            IEM_MC_REF_GREG_U64(pu64Dst, iReg);
+            IEM_MC_CALL_VOID_AIMPL_1(iemAImpl_bswap_u64, pu64Dst);
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/** Opcode 0x0f 0xc8. */
+FNIEMOP_DEF(iemOp_bswap_rAX_r8)
+{
+    IEMOP_MNEMONIC(bswap_rAX_r8, "bswap rAX/r8");
+    /* Note! Intel manuals states that R8-R15 can be accessed by using a REX.X
+             prefix.  REX.B is the correct prefix it appears.  For a parallel
+             case, see iemOp_mov_AL_Ib and iemOp_mov_eAX_Iv. */
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xAX | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xc9. */
+FNIEMOP_DEF(iemOp_bswap_rCX_r9)
+{
+    IEMOP_MNEMONIC(bswap_rCX_r9, "bswap rCX/r9");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xCX | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xca. */
+FNIEMOP_DEF(iemOp_bswap_rDX_r10)
+{
+    IEMOP_MNEMONIC(bswap_rDX_r9, "bswap rDX/r9");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xDX | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xcb. */
+FNIEMOP_DEF(iemOp_bswap_rBX_r11)
+{
+    IEMOP_MNEMONIC(bswap_rBX_r9, "bswap rBX/r9");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xBX | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xcc. */
+FNIEMOP_DEF(iemOp_bswap_rSP_r12)
+{
+    IEMOP_MNEMONIC(bswap_rSP_r12, "bswap rSP/r12");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xSP | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xcd. */
+FNIEMOP_DEF(iemOp_bswap_rBP_r13)
+{
+    IEMOP_MNEMONIC(bswap_rBP_r13, "bswap rBP/r13");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xBP | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xce. */
+FNIEMOP_DEF(iemOp_bswap_rSI_r14)
+{
+    IEMOP_MNEMONIC(bswap_rSI_r14, "bswap rSI/r14");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xSI | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xcf. */
+FNIEMOP_DEF(iemOp_bswap_rDI_r15)
+{
+    IEMOP_MNEMONIC(bswap_rDI_r15, "bswap rDI/r15");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xDI | pVCpu->iem.s.uRexB);
+}
+
+
+/*  Opcode      0x0f 0xd0 - invalid */
+/** Opcode 0x66 0x0f 0xd0 - addsubpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_addsubpd_Vpd_Wpd);
+/*  Opcode 0xf3 0x0f 0xd0 - invalid */
+/** Opcode 0xf2 0x0f 0xd0 - addsubps Vps, Wps */
+FNIEMOP_STUB(iemOp_addsubps_Vps_Wps);
+
+/** Opcode      0x0f 0xd1 - psrlw Pq, Qq */
+FNIEMOP_STUB(iemOp_psrlw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xd1 - psrlw Vx, W */
+FNIEMOP_STUB(iemOp_psrlw_Vx_W);
+/*  Opcode 0xf3 0x0f 0xd1 - invalid */
+/*  Opcode 0xf2 0x0f 0xd1 - invalid */
+
+/** Opcode      0x0f 0xd2 - psrld Pq, Qq */
+FNIEMOP_STUB(iemOp_psrld_Pq_Qq);
+/** Opcode 0x66 0x0f 0xd2 - psrld Vx, Wx */
+FNIEMOP_STUB(iemOp_psrld_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xd2 - invalid */
+/*  Opcode 0xf2 0x0f 0xd2 - invalid */
+
+/** Opcode      0x0f 0xd3 - psrlq Pq, Qq */
+FNIEMOP_STUB(iemOp_psrlq_Pq_Qq);
+/** Opcode 0x66 0x0f 0xd3 - psrlq Vx, Wx */
+FNIEMOP_STUB(iemOp_psrlq_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xd3 - invalid */
+/*  Opcode 0xf2 0x0f 0xd3 - invalid */
+
+/** Opcode      0x0f 0xd4 - paddq Pq, Qq */
+FNIEMOP_STUB(iemOp_paddq_Pq_Qq);
+/** Opcode 0x66 0x0f 0xd4 - paddq Vx, W */
+FNIEMOP_STUB(iemOp_paddq_Vx_W);
+/*  Opcode 0xf3 0x0f 0xd4 - invalid */
+/*  Opcode 0xf2 0x0f 0xd4 - invalid */
+
+/** Opcode      0x0f 0xd5 - pmullw Pq, Qq */
+FNIEMOP_STUB(iemOp_pmullw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xd5 - pmullw Vx, Wx */
+FNIEMOP_STUB(iemOp_pmullw_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xd5 - invalid */
+/*  Opcode 0xf2 0x0f 0xd5 - invalid */
+
+/*  Opcode      0x0f 0xd6 - invalid */
+
+/**
+ * @opcode      0xd6
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  none
+ * @optest      op1=-1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movq_Wq_Vq)
+{
+    IEMOP_MNEMONIC2(MR, MOVQ, movq, WqZxReg_WO, Vq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_XREG_U64_ZX_U128((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xd6
+ * @opcodesub   11 mr/reg
+ * @oppfx       f3
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_simdint_datamove
+ * @optest      op1=1 op2=2   -> op1=2   ftw=0xff
+ * @optest      op1=0 op2=-42 -> op1=-42 ftw=0xff
+ */
+FNIEMOP_DEF(iemOp_movq2dq_Vdq_Nq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_MNEMONIC2(RM_REG, MOVQ2DQ, movq2dq, VqZx_WO, Nq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MREG_U64(uSrc, bRm & X86_MODRM_RM_MASK);
+        IEM_MC_STORE_XREG_U64_ZX_U128(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udf30fd6mem
+     * @opcode      0xd6
+     * @opcodesub   !11 mr/reg
+     * @oppfx       f3
+     * @opunused    intel-modrm
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedDecode, bRm);
+}
+
+
+/**
+ * @opcode      0xd6
+ * @opcodesub   11 mr/reg
+ * @oppfx       f2
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_simdint_datamove
+ * @optest      op1=1 op2=2   -> op1=2   ftw=0xff
+ * @optest      op1=0 op2=-42 -> op1=-42 ftw=0xff
+ * @optest      op1=0 op2=0x1123456789abcdef -> op1=0x1123456789abcdef ftw=0xff
+ * @optest      op1=0 op2=0xfedcba9876543210 -> op1=0xfedcba9876543210 ftw=0xff
+ * @optest      op1=-42 op2=0xfedcba9876543210
+ *                  ->  op1=0xfedcba9876543210 ftw=0xff
+ */
+FNIEMOP_DEF(iemOp_movdq2q_Pq_Uq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_MNEMONIC2(RM_REG, MOVDQ2Q, movdq2q, Pq_WO, Uq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_STORE_MREG_U64((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK, uSrc);
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udf20fd6mem
+     * @opcode      0xd6
+     * @opcodesub   !11 mr/reg
+     * @oppfx       f2
+     * @opunused    intel-modrm
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedDecode, bRm);
+}
+
+/** Opcode      0x0f 0xd7 - pmovmskb Gd, Nq */
+FNIEMOP_DEF(iemOp_pmovmskb_Gd_Nq)
+{
+    /* Note! Taking the lazy approch here wrt the high 32-bits of the GREG. */
+    /** @todo testcase: Check that the instruction implicitly clears the high
+     *        bits in 64-bit mode.  The REX.W is first necessary when VLMAX > 256
+     *        and opcode modifications are made to work with the whole width (not
+     *        just 128). */
+    IEMOP_MNEMONIC(pmovmskb_Gd_Udq, "pmovmskb Gd,Nq");
+    /* Docs says register only. */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT)) /** @todo test that this is registers only. */
+    {
+        IEMOP_HLP_DECODED_NL_2(OP_PMOVMSKB, IEMOPFORM_RM_REG, OP_PARM_Gd, OP_PARM_Vdq, DISOPTYPE_MMX | DISOPTYPE_HARMLESS);
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG(uint64_t const *,    pSrc, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_REF_GREG_U64(pDst, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_REF_MREG_U64_CONST(pSrc, bRm & X86_MODRM_RM_MASK);
+        IEM_MC_CALL_MMX_AIMPL_2(iemAImpl_pmovmskb_u64, pDst, pSrc);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/** Opcode 0x66 0x0f 0xd7 -  */
+FNIEMOP_DEF(iemOp_pmovmskb_Gd_Ux)
+{
+    /* Note! Taking the lazy approch here wrt the high 32-bits of the GREG. */
+    /** @todo testcase: Check that the instruction implicitly clears the high
+     *        bits in 64-bit mode.  The REX.W is first necessary when VLMAX > 256
+     *        and opcode modifications are made to work with the whole width (not
+     *        just 128). */
+    IEMOP_MNEMONIC(pmovmskb_Gd_Nq, "vpmovmskb Gd, Ux");
+    /* Docs says register only. */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT)) /** @todo test that this is registers only. */
+    {
+        IEMOP_HLP_DECODED_NL_2(OP_PMOVMSKB, IEMOPFORM_RM_REG, OP_PARM_Gd, OP_PARM_Vdq, DISOPTYPE_SSE | DISOPTYPE_HARMLESS);
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,           pDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,         pSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_GREG_U64(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_REF_XREG_U128_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_CALL_SSE_AIMPL_2(iemAImpl_pmovmskb_u128, pDst, pSrc);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/*  Opcode 0xf3 0x0f 0xd7 - invalid */
+/*  Opcode 0xf2 0x0f 0xd7 - invalid */
+
+
+/** Opcode      0x0f 0xd8 - psubusb Pq, Qq */
+FNIEMOP_STUB(iemOp_psubusb_Pq_Qq);
+/** Opcode 0x66 0x0f 0xd8 - psubusb Vx, W */
+FNIEMOP_STUB(iemOp_psubusb_Vx_W);
+/*  Opcode 0xf3 0x0f 0xd8 - invalid */
+/*  Opcode 0xf2 0x0f 0xd8 - invalid */
+
+/** Opcode      0x0f 0xd9 - psubusw Pq, Qq */
+FNIEMOP_STUB(iemOp_psubusw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xd9 - psubusw Vx, Wx */
+FNIEMOP_STUB(iemOp_psubusw_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xd9 - invalid */
+/*  Opcode 0xf2 0x0f 0xd9 - invalid */
+
+/** Opcode      0x0f 0xda - pminub Pq, Qq */
+FNIEMOP_STUB(iemOp_pminub_Pq_Qq);
+/** Opcode 0x66 0x0f 0xda - pminub Vx, Wx */
+FNIEMOP_STUB(iemOp_pminub_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xda - invalid */
+/*  Opcode 0xf2 0x0f 0xda - invalid */
+
+/** Opcode      0x0f 0xdb - pand Pq, Qq */
+FNIEMOP_STUB(iemOp_pand_Pq_Qq);
+/** Opcode 0x66 0x0f 0xdb - pand Vx, W */
+FNIEMOP_STUB(iemOp_pand_Vx_W);
+/*  Opcode 0xf3 0x0f 0xdb - invalid */
+/*  Opcode 0xf2 0x0f 0xdb - invalid */
+
+/** Opcode      0x0f 0xdc - paddusb Pq, Qq */
+FNIEMOP_STUB(iemOp_paddusb_Pq_Qq);
+/** Opcode 0x66 0x0f 0xdc - paddusb Vx, Wx */
+FNIEMOP_STUB(iemOp_paddusb_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xdc - invalid */
+/*  Opcode 0xf2 0x0f 0xdc - invalid */
+
+/** Opcode      0x0f 0xdd - paddusw Pq, Qq */
+FNIEMOP_STUB(iemOp_paddusw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xdd - paddusw Vx, Wx */
+FNIEMOP_STUB(iemOp_paddusw_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xdd - invalid */
+/*  Opcode 0xf2 0x0f 0xdd - invalid */
+
+/** Opcode      0x0f 0xde - pmaxub Pq, Qq */
+FNIEMOP_STUB(iemOp_pmaxub_Pq_Qq);
+/** Opcode 0x66 0x0f 0xde - pmaxub Vx, W */
+FNIEMOP_STUB(iemOp_pmaxub_Vx_W);
+/*  Opcode 0xf3 0x0f 0xde - invalid */
+/*  Opcode 0xf2 0x0f 0xde - invalid */
+
+/** Opcode      0x0f 0xdf - pandn Pq, Qq */
+FNIEMOP_STUB(iemOp_pandn_Pq_Qq);
+/** Opcode 0x66 0x0f 0xdf - pandn Vx, Wx */
+FNIEMOP_STUB(iemOp_pandn_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xdf - invalid */
+/*  Opcode 0xf2 0x0f 0xdf - invalid */
+
+/** Opcode      0x0f 0xe0 - pavgb Pq, Qq */
+FNIEMOP_STUB(iemOp_pavgb_Pq_Qq);
+/** Opcode 0x66 0x0f 0xe0 - pavgb Vx, Wx */
+FNIEMOP_STUB(iemOp_pavgb_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xe0 - invalid */
+/*  Opcode 0xf2 0x0f 0xe0 - invalid */
+
+/** Opcode      0x0f 0xe1 - psraw Pq, Qq */
+FNIEMOP_STUB(iemOp_psraw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xe1 - psraw Vx, W */
+FNIEMOP_STUB(iemOp_psraw_Vx_W);
+/*  Opcode 0xf3 0x0f 0xe1 - invalid */
+/*  Opcode 0xf2 0x0f 0xe1 - invalid */
+
+/** Opcode      0x0f 0xe2 - psrad Pq, Qq */
+FNIEMOP_STUB(iemOp_psrad_Pq_Qq);
+/** Opcode 0x66 0x0f 0xe2 - psrad Vx, Wx */
+FNIEMOP_STUB(iemOp_psrad_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xe2 - invalid */
+/*  Opcode 0xf2 0x0f 0xe2 - invalid */
+
+/** Opcode      0x0f 0xe3 - pavgw Pq, Qq */
+FNIEMOP_STUB(iemOp_pavgw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xe3 - pavgw Vx, Wx */
+FNIEMOP_STUB(iemOp_pavgw_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xe3 - invalid */
+/*  Opcode 0xf2 0x0f 0xe3 - invalid */
+
+/** Opcode      0x0f 0xe4 - pmulhuw Pq, Qq */
+FNIEMOP_STUB(iemOp_pmulhuw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xe4 - pmulhuw Vx, W */
+FNIEMOP_STUB(iemOp_pmulhuw_Vx_W);
+/*  Opcode 0xf3 0x0f 0xe4 - invalid */
+/*  Opcode 0xf2 0x0f 0xe4 - invalid */
+
+/** Opcode      0x0f 0xe5 - pmulhw Pq, Qq */
+FNIEMOP_STUB(iemOp_pmulhw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xe5 - pmulhw Vx, Wx */
+FNIEMOP_STUB(iemOp_pmulhw_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xe5 - invalid */
+/*  Opcode 0xf2 0x0f 0xe5 - invalid */
+
+/*  Opcode      0x0f 0xe6 - invalid */
+/** Opcode 0x66 0x0f 0xe6 - cvttpd2dq Vx, Wpd */
+FNIEMOP_STUB(iemOp_cvttpd2dq_Vx_Wpd);
+/** Opcode 0xf3 0x0f 0xe6 - cvtdq2pd Vx, Wpd */
+FNIEMOP_STUB(iemOp_cvtdq2pd_Vx_Wpd);
+/** Opcode 0xf2 0x0f 0xe6 - cvtpd2dq Vx, Wpd */
+FNIEMOP_STUB(iemOp_cvtpd2dq_Vx_Wpd);
+
+
+/**
+ * @opcode      0xe7
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse1_cachect
+ * @opxcpttype  none
+ * @optest      op1=-1 op2=2  -> op1=2   ftw=0xff
+ * @optest      op1=0 op2=-42 -> op1=-42 ftw=0xff
+ */
+FNIEMOP_DEF(iemOp_movntq_Mq_Pq)
+{
+    IEMOP_MNEMONIC2(MR_MEM, MOVNTQ, movntq, Mq_WO, Pq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* Register, memory. */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MREG_U64(uSrc, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+    /**
+     * @opdone
+     * @opmnemonic  ud0fe7reg
+     * @opcode      0xe7
+     * @opcodesub   11 mr/reg
+     * @oppfx       none
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/**
+ * @opcode      0xe7
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_cachect
+ * @opxcpttype  1
+ * @optest      op1=-1 op2=2  -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movntdq_Mdq_Vdq)
+{
+    IEMOP_MNEMONIC2(MR_MEM, MOVNTDQ, movntdq, Mdq_WO, Vdq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /* Register, memory. */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud660fe7reg
+     * @opcode      0xe7
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/*  Opcode 0xf3 0x0f 0xe7 - invalid */
+/*  Opcode 0xf2 0x0f 0xe7 - invalid */
+
+
+/** Opcode      0x0f 0xe8 - psubsb Pq, Qq */
+FNIEMOP_STUB(iemOp_psubsb_Pq_Qq);
+/** Opcode 0x66 0x0f 0xe8 - psubsb Vx, W */
+FNIEMOP_STUB(iemOp_psubsb_Vx_W);
+/*  Opcode 0xf3 0x0f 0xe8 - invalid */
+/*  Opcode 0xf2 0x0f 0xe8 - invalid */
+
+/** Opcode      0x0f 0xe9 - psubsw Pq, Qq */
+FNIEMOP_STUB(iemOp_psubsw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xe9 - psubsw Vx, Wx */
+FNIEMOP_STUB(iemOp_psubsw_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xe9 - invalid */
+/*  Opcode 0xf2 0x0f 0xe9 - invalid */
+
+/** Opcode      0x0f 0xea - pminsw Pq, Qq */
+FNIEMOP_STUB(iemOp_pminsw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xea - pminsw Vx, Wx */
+FNIEMOP_STUB(iemOp_pminsw_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xea - invalid */
+/*  Opcode 0xf2 0x0f 0xea - invalid */
+
+/** Opcode      0x0f 0xeb - por Pq, Qq */
+FNIEMOP_STUB(iemOp_por_Pq_Qq);
+/** Opcode 0x66 0x0f 0xeb - por Vx, W */
+FNIEMOP_STUB(iemOp_por_Vx_W);
+/*  Opcode 0xf3 0x0f 0xeb - invalid */
+/*  Opcode 0xf2 0x0f 0xeb - invalid */
+
+/** Opcode      0x0f 0xec - paddsb Pq, Qq */
+FNIEMOP_STUB(iemOp_paddsb_Pq_Qq);
+/** Opcode 0x66 0x0f 0xec - paddsb Vx, Wx */
+FNIEMOP_STUB(iemOp_paddsb_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xec - invalid */
+/*  Opcode 0xf2 0x0f 0xec - invalid */
+
+/** Opcode      0x0f 0xed - paddsw Pq, Qq */
+FNIEMOP_STUB(iemOp_paddsw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xed - paddsw Vx, Wx */
+FNIEMOP_STUB(iemOp_paddsw_Vx_Wx);
+/*  Opcode 0xf3 0x0f 0xed - invalid */
+/*  Opcode 0xf2 0x0f 0xed - invalid */
+
+/** Opcode      0x0f 0xee - pmaxsw Pq, Qq */
+FNIEMOP_STUB(iemOp_pmaxsw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xee - pmaxsw Vx, W */
+FNIEMOP_STUB(iemOp_pmaxsw_Vx_W);
+/*  Opcode 0xf3 0x0f 0xee - invalid */
+/*  Opcode 0xf2 0x0f 0xee - invalid */
+
+
+/** Opcode      0x0f 0xef - pxor Pq, Qq */
+FNIEMOP_DEF(iemOp_pxor_Pq_Qq)
+{
+    IEMOP_MNEMONIC(pxor, "pxor");
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, &g_iemAImpl_pxor);
+}
+
+/** Opcode 0x66 0x0f 0xef - pxor Vx, Wx */
+FNIEMOP_DEF(iemOp_pxor_Vx_Wx)
+{
+    IEMOP_MNEMONIC(pxor_Vx_Wx, "pxor");
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, &g_iemAImpl_pxor);
+}
+
+/*  Opcode 0xf3 0x0f 0xef - invalid */
+/*  Opcode 0xf2 0x0f 0xef - invalid */
+
+/*  Opcode      0x0f 0xf0 - invalid */
+/*  Opcode 0x66 0x0f 0xf0 - invalid */
+/** Opcode 0xf2 0x0f 0xf0 - lddqu Vx, Mx */
+FNIEMOP_STUB(iemOp_lddqu_Vx_Mx);
+
+/** Opcode      0x0f 0xf1 - psllw Pq, Qq */
+FNIEMOP_STUB(iemOp_psllw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xf1 - psllw Vx, W */
+FNIEMOP_STUB(iemOp_psllw_Vx_W);
+/*  Opcode 0xf2 0x0f 0xf1 - invalid */
+
+/** Opcode      0x0f 0xf2 - pslld Pq, Qq */
+FNIEMOP_STUB(iemOp_pslld_Pq_Qq);
+/** Opcode 0x66 0x0f 0xf2 - pslld Vx, Wx */
+FNIEMOP_STUB(iemOp_pslld_Vx_Wx);
+/*  Opcode 0xf2 0x0f 0xf2 - invalid */
+
+/** Opcode      0x0f 0xf3 - psllq Pq, Qq */
+FNIEMOP_STUB(iemOp_psllq_Pq_Qq);
+/** Opcode 0x66 0x0f 0xf3 - psllq Vx, Wx */
+FNIEMOP_STUB(iemOp_psllq_Vx_Wx);
+/*  Opcode 0xf2 0x0f 0xf3 - invalid */
+
+/** Opcode      0x0f 0xf4 - pmuludq Pq, Qq */
+FNIEMOP_STUB(iemOp_pmuludq_Pq_Qq);
+/** Opcode 0x66 0x0f 0xf4 - pmuludq Vx, W */
+FNIEMOP_STUB(iemOp_pmuludq_Vx_W);
+/*  Opcode 0xf2 0x0f 0xf4 - invalid */
+
+/** Opcode      0x0f 0xf5 - pmaddwd Pq, Qq */
+FNIEMOP_STUB(iemOp_pmaddwd_Pq_Qq);
+/** Opcode 0x66 0x0f 0xf5 - pmaddwd Vx, Wx */
+FNIEMOP_STUB(iemOp_pmaddwd_Vx_Wx);
+/*  Opcode 0xf2 0x0f 0xf5 - invalid */
+
+/** Opcode      0x0f 0xf6 - psadbw Pq, Qq */
+FNIEMOP_STUB(iemOp_psadbw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xf6 - psadbw Vx, Wx */
+FNIEMOP_STUB(iemOp_psadbw_Vx_Wx);
+/*  Opcode 0xf2 0x0f 0xf6 - invalid */
+
+/** Opcode      0x0f 0xf7 - maskmovq Pq, Nq */
+FNIEMOP_STUB(iemOp_maskmovq_Pq_Nq);
+/** Opcode 0x66 0x0f 0xf7 - maskmovdqu Vdq, Udq */
+FNIEMOP_STUB(iemOp_maskmovdqu_Vdq_Udq);
+/*  Opcode 0xf2 0x0f 0xf7 - invalid */
+
+/** Opcode      0x0f 0xf8 - psubb Pq, Qq */
+FNIEMOP_STUB(iemOp_psubb_Pq_Qq);
+/** Opcode 0x66 0x0f 0xf8 - psubb Vx, W */
+FNIEMOP_STUB(iemOp_psubb_Vx_W);
+/*  Opcode 0xf2 0x0f 0xf8 - invalid */
+
+/** Opcode      0x0f 0xf9 - psubw Pq, Qq */
+FNIEMOP_STUB(iemOp_psubw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xf9 - psubw Vx, Wx */
+FNIEMOP_STUB(iemOp_psubw_Vx_Wx);
+/*  Opcode 0xf2 0x0f 0xf9 - invalid */
+
+/** Opcode      0x0f 0xfa - psubd Pq, Qq */
+FNIEMOP_STUB(iemOp_psubd_Pq_Qq);
+/** Opcode 0x66 0x0f 0xfa - psubd Vx, Wx */
+FNIEMOP_STUB(iemOp_psubd_Vx_Wx);
+/*  Opcode 0xf2 0x0f 0xfa - invalid */
+
+/** Opcode      0x0f 0xfb - psubq Pq, Qq */
+FNIEMOP_STUB(iemOp_psubq_Pq_Qq);
+/** Opcode 0x66 0x0f 0xfb - psubq Vx, W */
+FNIEMOP_STUB(iemOp_psubq_Vx_W);
+/*  Opcode 0xf2 0x0f 0xfb - invalid */
+
+/** Opcode      0x0f 0xfc - paddb Pq, Qq */
+FNIEMOP_STUB(iemOp_paddb_Pq_Qq);
+/** Opcode 0x66 0x0f 0xfc - paddb Vx, Wx */
+FNIEMOP_STUB(iemOp_paddb_Vx_Wx);
+/*  Opcode 0xf2 0x0f 0xfc - invalid */
+
+/** Opcode      0x0f 0xfd - paddw Pq, Qq */
+FNIEMOP_STUB(iemOp_paddw_Pq_Qq);
+/** Opcode 0x66 0x0f 0xfd - paddw Vx, Wx */
+FNIEMOP_STUB(iemOp_paddw_Vx_Wx);
+/*  Opcode 0xf2 0x0f 0xfd - invalid */
+
+/** Opcode      0x0f 0xfe - paddd Pq, Qq */
+FNIEMOP_STUB(iemOp_paddd_Pq_Qq);
+/** Opcode 0x66 0x0f 0xfe - paddd Vx, W */
+FNIEMOP_STUB(iemOp_paddd_Vx_W);
+/*  Opcode 0xf2 0x0f 0xfe - invalid */
+
+
+/** Opcode **** 0x0f 0xff - UD0 */
+FNIEMOP_DEF(iemOp_ud0)
+{
+    IEMOP_MNEMONIC(ud0, "ud0");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+        if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+        IEMOP_HLP_DONE_DECODING();
+    }
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+
+/**
+ * Two byte opcode map, first byte 0x0f.
+ *
+ * @remarks The g_apfnVexMap1 table is currently a subset of this one, so please
+ *          check if it needs updating as well when making changes.
+ */
+IEM_STATIC const PFNIEMOP g_apfnTwoByteMap[] =
+{
+    /*          no prefix,                  066h prefix                 f3h prefix,                 f2h prefix */
+    /* 0x00 */  IEMOP_X4(iemOp_Grp6),
+    /* 0x01 */  IEMOP_X4(iemOp_Grp7),
+    /* 0x02 */  IEMOP_X4(iemOp_lar_Gv_Ew),
+    /* 0x03 */  IEMOP_X4(iemOp_lsl_Gv_Ew),
+    /* 0x04 */  IEMOP_X4(iemOp_Invalid),
+    /* 0x05 */  IEMOP_X4(iemOp_syscall),
+    /* 0x06 */  IEMOP_X4(iemOp_clts),
+    /* 0x07 */  IEMOP_X4(iemOp_sysret),
+    /* 0x08 */  IEMOP_X4(iemOp_invd),
+    /* 0x09 */  IEMOP_X4(iemOp_wbinvd),
+    /* 0x0a */  IEMOP_X4(iemOp_Invalid),
+    /* 0x0b */  IEMOP_X4(iemOp_ud2),
+    /* 0x0c */  IEMOP_X4(iemOp_Invalid),
+    /* 0x0d */  IEMOP_X4(iemOp_nop_Ev_GrpP),
+    /* 0x0e */  IEMOP_X4(iemOp_femms),
+    /* 0x0f */  IEMOP_X4(iemOp_3Dnow),
+
+    /* 0x10 */  iemOp_movups_Vps_Wps,       iemOp_movupd_Vpd_Wpd,       iemOp_movss_Vss_Wss,        iemOp_movsd_Vsd_Wsd,
+    /* 0x11 */  iemOp_movups_Wps_Vps,       iemOp_movupd_Wpd_Vpd,       iemOp_movss_Wss_Vss,        iemOp_movsd_Wsd_Vsd,
+    /* 0x12 */  iemOp_movlps_Vq_Mq__movhlps, iemOp_movlpd_Vq_Mq,        iemOp_movsldup_Vdq_Wdq,     iemOp_movddup_Vdq_Wdq,
+    /* 0x13 */  iemOp_movlps_Mq_Vq,         iemOp_movlpd_Mq_Vq,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x14 */  iemOp_unpcklps_Vx_Wx,       iemOp_unpcklpd_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x15 */  iemOp_unpckhps_Vx_Wx,       iemOp_unpckhpd_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x16 */  iemOp_movhps_Vdq_Mq__movlhps_Vdq_Uq, iemOp_movhpd_Vdq_Mq, iemOp_movshdup_Vdq_Wdq,   iemOp_InvalidNeedRM,
+    /* 0x17 */  iemOp_movhps_Mq_Vq,         iemOp_movhpd_Mq_Vq,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x18 */  IEMOP_X4(iemOp_prefetch_Grp16),
+    /* 0x19 */  IEMOP_X4(iemOp_nop_Ev),
+    /* 0x1a */  IEMOP_X4(iemOp_nop_Ev),
+    /* 0x1b */  IEMOP_X4(iemOp_nop_Ev),
+    /* 0x1c */  IEMOP_X4(iemOp_nop_Ev),
+    /* 0x1d */  IEMOP_X4(iemOp_nop_Ev),
+    /* 0x1e */  IEMOP_X4(iemOp_nop_Ev),
+    /* 0x1f */  IEMOP_X4(iemOp_nop_Ev),
+
+    /* 0x20 */  iemOp_mov_Rd_Cd,            iemOp_mov_Rd_Cd,            iemOp_mov_Rd_Cd,            iemOp_mov_Rd_Cd,
+    /* 0x21 */  iemOp_mov_Rd_Dd,            iemOp_mov_Rd_Dd,            iemOp_mov_Rd_Dd,            iemOp_mov_Rd_Dd,
+    /* 0x22 */  iemOp_mov_Cd_Rd,            iemOp_mov_Cd_Rd,            iemOp_mov_Cd_Rd,            iemOp_mov_Cd_Rd,
+    /* 0x23 */  iemOp_mov_Dd_Rd,            iemOp_mov_Dd_Rd,            iemOp_mov_Dd_Rd,            iemOp_mov_Dd_Rd,
+    /* 0x24 */  iemOp_mov_Rd_Td,            iemOp_mov_Rd_Td,            iemOp_mov_Rd_Td,            iemOp_mov_Rd_Td,
+    /* 0x25 */  iemOp_Invalid,              iemOp_Invalid,              iemOp_Invalid,              iemOp_Invalid,
+    /* 0x26 */  iemOp_mov_Td_Rd,            iemOp_mov_Td_Rd,            iemOp_mov_Td_Rd,            iemOp_mov_Td_Rd,
+    /* 0x27 */  iemOp_Invalid,              iemOp_Invalid,              iemOp_Invalid,              iemOp_Invalid,
+    /* 0x28 */  iemOp_movaps_Vps_Wps,       iemOp_movapd_Vpd_Wpd,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x29 */  iemOp_movaps_Wps_Vps,       iemOp_movapd_Wpd_Vpd,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2a */  iemOp_cvtpi2ps_Vps_Qpi,     iemOp_cvtpi2pd_Vpd_Qpi,     iemOp_cvtsi2ss_Vss_Ey,      iemOp_cvtsi2sd_Vsd_Ey,
+    /* 0x2b */  iemOp_movntps_Mps_Vps,      iemOp_movntpd_Mpd_Vpd,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2c */  iemOp_cvttps2pi_Ppi_Wps,    iemOp_cvttpd2pi_Ppi_Wpd,    iemOp_cvttss2si_Gy_Wss,     iemOp_cvttsd2si_Gy_Wsd,
+    /* 0x2d */  iemOp_cvtps2pi_Ppi_Wps,     iemOp_cvtpd2pi_Qpi_Wpd,     iemOp_cvtss2si_Gy_Wss,      iemOp_cvtsd2si_Gy_Wsd,
+    /* 0x2e */  iemOp_ucomiss_Vss_Wss,      iemOp_ucomisd_Vsd_Wsd,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2f */  iemOp_comiss_Vss_Wss,       iemOp_comisd_Vsd_Wsd,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x30 */  IEMOP_X4(iemOp_wrmsr),
+    /* 0x31 */  IEMOP_X4(iemOp_rdtsc),
+    /* 0x32 */  IEMOP_X4(iemOp_rdmsr),
+    /* 0x33 */  IEMOP_X4(iemOp_rdpmc),
+    /* 0x34 */  IEMOP_X4(iemOp_sysenter),
+    /* 0x35 */  IEMOP_X4(iemOp_sysexit),
+    /* 0x36 */  IEMOP_X4(iemOp_Invalid),
+    /* 0x37 */  IEMOP_X4(iemOp_getsec),
+    /* 0x38 */  IEMOP_X4(iemOp_3byte_Esc_0f_38),
+    /* 0x39 */  IEMOP_X4(iemOp_InvalidNeed3ByteEscRM),
+    /* 0x3a */  IEMOP_X4(iemOp_3byte_Esc_0f_3a),
+    /* 0x3b */  IEMOP_X4(iemOp_InvalidNeed3ByteEscRMImm8),
+    /* 0x3c */  IEMOP_X4(iemOp_InvalidNeed3ByteEscRM),
+    /* 0x3d */  IEMOP_X4(iemOp_InvalidNeed3ByteEscRM),
+    /* 0x3e */  IEMOP_X4(iemOp_InvalidNeed3ByteEscRMImm8),
+    /* 0x3f */  IEMOP_X4(iemOp_InvalidNeed3ByteEscRMImm8),
+
+    /* 0x40 */  IEMOP_X4(iemOp_cmovo_Gv_Ev),
+    /* 0x41 */  IEMOP_X4(iemOp_cmovno_Gv_Ev),
+    /* 0x42 */  IEMOP_X4(iemOp_cmovc_Gv_Ev),
+    /* 0x43 */  IEMOP_X4(iemOp_cmovnc_Gv_Ev),
+    /* 0x44 */  IEMOP_X4(iemOp_cmove_Gv_Ev),
+    /* 0x45 */  IEMOP_X4(iemOp_cmovne_Gv_Ev),
+    /* 0x46 */  IEMOP_X4(iemOp_cmovbe_Gv_Ev),
+    /* 0x47 */  IEMOP_X4(iemOp_cmovnbe_Gv_Ev),
+    /* 0x48 */  IEMOP_X4(iemOp_cmovs_Gv_Ev),
+    /* 0x49 */  IEMOP_X4(iemOp_cmovns_Gv_Ev),
+    /* 0x4a */  IEMOP_X4(iemOp_cmovp_Gv_Ev),
+    /* 0x4b */  IEMOP_X4(iemOp_cmovnp_Gv_Ev),
+    /* 0x4c */  IEMOP_X4(iemOp_cmovl_Gv_Ev),
+    /* 0x4d */  IEMOP_X4(iemOp_cmovnl_Gv_Ev),
+    /* 0x4e */  IEMOP_X4(iemOp_cmovle_Gv_Ev),
+    /* 0x4f */  IEMOP_X4(iemOp_cmovnle_Gv_Ev),
+
+    /* 0x50 */  iemOp_movmskps_Gy_Ups,      iemOp_movmskpd_Gy_Upd,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x51 */  iemOp_sqrtps_Vps_Wps,       iemOp_sqrtpd_Vpd_Wpd,       iemOp_sqrtss_Vss_Wss,       iemOp_sqrtsd_Vsd_Wsd,
+    /* 0x52 */  iemOp_rsqrtps_Vps_Wps,      iemOp_InvalidNeedRM,        iemOp_rsqrtss_Vss_Wss,      iemOp_InvalidNeedRM,
+    /* 0x53 */  iemOp_rcpps_Vps_Wps,        iemOp_InvalidNeedRM,        iemOp_rcpss_Vss_Wss,        iemOp_InvalidNeedRM,
+    /* 0x54 */  iemOp_andps_Vps_Wps,        iemOp_andpd_Vpd_Wpd,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x55 */  iemOp_andnps_Vps_Wps,       iemOp_andnpd_Vpd_Wpd,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x56 */  iemOp_orps_Vps_Wps,         iemOp_orpd_Vpd_Wpd,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x57 */  iemOp_xorps_Vps_Wps,        iemOp_xorpd_Vpd_Wpd,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x58 */  iemOp_addps_Vps_Wps,        iemOp_addpd_Vpd_Wpd,        iemOp_addss_Vss_Wss,        iemOp_addsd_Vsd_Wsd,
+    /* 0x59 */  iemOp_mulps_Vps_Wps,        iemOp_mulpd_Vpd_Wpd,        iemOp_mulss_Vss_Wss,        iemOp_mulsd_Vsd_Wsd,
+    /* 0x5a */  iemOp_cvtps2pd_Vpd_Wps,     iemOp_cvtpd2ps_Vps_Wpd,     iemOp_cvtss2sd_Vsd_Wss,     iemOp_cvtsd2ss_Vss_Wsd,
+    /* 0x5b */  iemOp_cvtdq2ps_Vps_Wdq,     iemOp_cvtps2dq_Vdq_Wps,     iemOp_cvttps2dq_Vdq_Wps,    iemOp_InvalidNeedRM,
+    /* 0x5c */  iemOp_subps_Vps_Wps,        iemOp_subpd_Vpd_Wpd,        iemOp_subss_Vss_Wss,        iemOp_subsd_Vsd_Wsd,
+    /* 0x5d */  iemOp_minps_Vps_Wps,        iemOp_minpd_Vpd_Wpd,        iemOp_minss_Vss_Wss,        iemOp_minsd_Vsd_Wsd,
+    /* 0x5e */  iemOp_divps_Vps_Wps,        iemOp_divpd_Vpd_Wpd,        iemOp_divss_Vss_Wss,        iemOp_divsd_Vsd_Wsd,
+    /* 0x5f */  iemOp_maxps_Vps_Wps,        iemOp_maxpd_Vpd_Wpd,        iemOp_maxss_Vss_Wss,        iemOp_maxsd_Vsd_Wsd,
+
+    /* 0x60 */  iemOp_punpcklbw_Pq_Qd,      iemOp_punpcklbw_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x61 */  iemOp_punpcklwd_Pq_Qd,      iemOp_punpcklwd_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x62 */  iemOp_punpckldq_Pq_Qd,      iemOp_punpckldq_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x63 */  iemOp_packsswb_Pq_Qq,       iemOp_packsswb_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x64 */  iemOp_pcmpgtb_Pq_Qq,        iemOp_pcmpgtb_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x65 */  iemOp_pcmpgtw_Pq_Qq,        iemOp_pcmpgtw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x66 */  iemOp_pcmpgtd_Pq_Qq,        iemOp_pcmpgtd_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x67 */  iemOp_packuswb_Pq_Qq,       iemOp_packuswb_Vx_W,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x68 */  iemOp_punpckhbw_Pq_Qd,      iemOp_punpckhbw_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x69 */  iemOp_punpckhwd_Pq_Qd,      iemOp_punpckhwd_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6a */  iemOp_punpckhdq_Pq_Qd,      iemOp_punpckhdq_Vx_W,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6b */  iemOp_packssdw_Pq_Qd,       iemOp_packssdw_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6c */  iemOp_InvalidNeedRM,        iemOp_punpcklqdq_Vx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6d */  iemOp_InvalidNeedRM,        iemOp_punpckhqdq_Vx_W,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6e */  iemOp_movd_q_Pd_Ey,         iemOp_movd_q_Vy_Ey,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6f */  iemOp_movq_Pq_Qq,           iemOp_movdqa_Vdq_Wdq,       iemOp_movdqu_Vdq_Wdq,       iemOp_InvalidNeedRM,
+
+    /* 0x70 */  iemOp_pshufw_Pq_Qq_Ib,      iemOp_pshufd_Vx_Wx_Ib,      iemOp_pshufhw_Vx_Wx_Ib,     iemOp_pshuflw_Vx_Wx_Ib,
+    /* 0x71 */  IEMOP_X4(iemOp_Grp12),
+    /* 0x72 */  IEMOP_X4(iemOp_Grp13),
+    /* 0x73 */  IEMOP_X4(iemOp_Grp14),
+    /* 0x74 */  iemOp_pcmpeqb_Pq_Qq,        iemOp_pcmpeqb_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x75 */  iemOp_pcmpeqw_Pq_Qq,        iemOp_pcmpeqw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x76 */  iemOp_pcmpeqd_Pq_Qq,        iemOp_pcmpeqd_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x77 */  iemOp_emms,                 iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x78 */  iemOp_vmread_Ey_Gy,         iemOp_AmdGrp17,             iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x79 */  iemOp_vmwrite_Gy_Ey,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x7a */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x7b */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x7c */  iemOp_InvalidNeedRM,        iemOp_haddpd_Vpd_Wpd,       iemOp_InvalidNeedRM,        iemOp_haddps_Vps_Wps,
+    /* 0x7d */  iemOp_InvalidNeedRM,        iemOp_hsubpd_Vpd_Wpd,       iemOp_InvalidNeedRM,        iemOp_hsubps_Vps_Wps,
+    /* 0x7e */  iemOp_movd_q_Ey_Pd,         iemOp_movd_q_Ey_Vy,         iemOp_movq_Vq_Wq,           iemOp_InvalidNeedRM,
+    /* 0x7f */  iemOp_movq_Qq_Pq,           iemOp_movdqa_Wx_Vx,         iemOp_movdqu_Wx_Vx,         iemOp_InvalidNeedRM,
+
+    /* 0x80 */  IEMOP_X4(iemOp_jo_Jv),
+    /* 0x81 */  IEMOP_X4(iemOp_jno_Jv),
+    /* 0x82 */  IEMOP_X4(iemOp_jc_Jv),
+    /* 0x83 */  IEMOP_X4(iemOp_jnc_Jv),
+    /* 0x84 */  IEMOP_X4(iemOp_je_Jv),
+    /* 0x85 */  IEMOP_X4(iemOp_jne_Jv),
+    /* 0x86 */  IEMOP_X4(iemOp_jbe_Jv),
+    /* 0x87 */  IEMOP_X4(iemOp_jnbe_Jv),
+    /* 0x88 */  IEMOP_X4(iemOp_js_Jv),
+    /* 0x89 */  IEMOP_X4(iemOp_jns_Jv),
+    /* 0x8a */  IEMOP_X4(iemOp_jp_Jv),
+    /* 0x8b */  IEMOP_X4(iemOp_jnp_Jv),
+    /* 0x8c */  IEMOP_X4(iemOp_jl_Jv),
+    /* 0x8d */  IEMOP_X4(iemOp_jnl_Jv),
+    /* 0x8e */  IEMOP_X4(iemOp_jle_Jv),
+    /* 0x8f */  IEMOP_X4(iemOp_jnle_Jv),
+
+    /* 0x90 */  IEMOP_X4(iemOp_seto_Eb),
+    /* 0x91 */  IEMOP_X4(iemOp_setno_Eb),
+    /* 0x92 */  IEMOP_X4(iemOp_setc_Eb),
+    /* 0x93 */  IEMOP_X4(iemOp_setnc_Eb),
+    /* 0x94 */  IEMOP_X4(iemOp_sete_Eb),
+    /* 0x95 */  IEMOP_X4(iemOp_setne_Eb),
+    /* 0x96 */  IEMOP_X4(iemOp_setbe_Eb),
+    /* 0x97 */  IEMOP_X4(iemOp_setnbe_Eb),
+    /* 0x98 */  IEMOP_X4(iemOp_sets_Eb),
+    /* 0x99 */  IEMOP_X4(iemOp_setns_Eb),
+    /* 0x9a */  IEMOP_X4(iemOp_setp_Eb),
+    /* 0x9b */  IEMOP_X4(iemOp_setnp_Eb),
+    /* 0x9c */  IEMOP_X4(iemOp_setl_Eb),
+    /* 0x9d */  IEMOP_X4(iemOp_setnl_Eb),
+    /* 0x9e */  IEMOP_X4(iemOp_setle_Eb),
+    /* 0x9f */  IEMOP_X4(iemOp_setnle_Eb),
+
+    /* 0xa0 */  IEMOP_X4(iemOp_push_fs),
+    /* 0xa1 */  IEMOP_X4(iemOp_pop_fs),
+    /* 0xa2 */  IEMOP_X4(iemOp_cpuid),
+    /* 0xa3 */  IEMOP_X4(iemOp_bt_Ev_Gv),
+    /* 0xa4 */  IEMOP_X4(iemOp_shld_Ev_Gv_Ib),
+    /* 0xa5 */  IEMOP_X4(iemOp_shld_Ev_Gv_CL),
+    /* 0xa6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa8 */  IEMOP_X4(iemOp_push_gs),
+    /* 0xa9 */  IEMOP_X4(iemOp_pop_gs),
+    /* 0xaa */  IEMOP_X4(iemOp_rsm),
+    /* 0xab */  IEMOP_X4(iemOp_bts_Ev_Gv),
+    /* 0xac */  IEMOP_X4(iemOp_shrd_Ev_Gv_Ib),
+    /* 0xad */  IEMOP_X4(iemOp_shrd_Ev_Gv_CL),
+    /* 0xae */  IEMOP_X4(iemOp_Grp15),
+    /* 0xaf */  IEMOP_X4(iemOp_imul_Gv_Ev),
+
+    /* 0xb0 */  IEMOP_X4(iemOp_cmpxchg_Eb_Gb),
+    /* 0xb1 */  IEMOP_X4(iemOp_cmpxchg_Ev_Gv),
+    /* 0xb2 */  IEMOP_X4(iemOp_lss_Gv_Mp),
+    /* 0xb3 */  IEMOP_X4(iemOp_btr_Ev_Gv),
+    /* 0xb4 */  IEMOP_X4(iemOp_lfs_Gv_Mp),
+    /* 0xb5 */  IEMOP_X4(iemOp_lgs_Gv_Mp),
+    /* 0xb6 */  IEMOP_X4(iemOp_movzx_Gv_Eb),
+    /* 0xb7 */  IEMOP_X4(iemOp_movzx_Gv_Ew),
+    /* 0xb8 */  iemOp_jmpe,                 iemOp_InvalidNeedRM,        iemOp_popcnt_Gv_Ev,         iemOp_InvalidNeedRM,
+    /* 0xb9 */  IEMOP_X4(iemOp_Grp10),
+    /* 0xba */  IEMOP_X4(iemOp_Grp8),
+    /* 0xbb */  IEMOP_X4(iemOp_btc_Ev_Gv), // 0xf3?
+    /* 0xbc */  iemOp_bsf_Gv_Ev,            iemOp_bsf_Gv_Ev,            iemOp_tzcnt_Gv_Ev,          iemOp_bsf_Gv_Ev,
+    /* 0xbd */  iemOp_bsr_Gv_Ev,            iemOp_bsr_Gv_Ev,            iemOp_lzcnt_Gv_Ev,          iemOp_bsr_Gv_Ev,
+    /* 0xbe */  IEMOP_X4(iemOp_movsx_Gv_Eb),
+    /* 0xbf */  IEMOP_X4(iemOp_movsx_Gv_Ew),
+
+    /* 0xc0 */  IEMOP_X4(iemOp_xadd_Eb_Gb),
+    /* 0xc1 */  IEMOP_X4(iemOp_xadd_Ev_Gv),
+    /* 0xc2 */  iemOp_cmpps_Vps_Wps_Ib,     iemOp_cmppd_Vpd_Wpd_Ib,     iemOp_cmpss_Vss_Wss_Ib,     iemOp_cmpsd_Vsd_Wsd_Ib,
+    /* 0xc3 */  iemOp_movnti_My_Gy,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xc4 */  iemOp_pinsrw_Pq_RyMw_Ib,    iemOp_pinsrw_Vdq_RyMw_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0xc5 */  iemOp_pextrw_Gd_Nq_Ib,      iemOp_pextrw_Gd_Udq_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0xc6 */  iemOp_shufps_Vps_Wps_Ib,    iemOp_shufpd_Vpd_Wpd_Ib,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0xc7 */  IEMOP_X4(iemOp_Grp9),
+    /* 0xc8 */  IEMOP_X4(iemOp_bswap_rAX_r8),
+    /* 0xc9 */  IEMOP_X4(iemOp_bswap_rCX_r9),
+    /* 0xca */  IEMOP_X4(iemOp_bswap_rDX_r10),
+    /* 0xcb */  IEMOP_X4(iemOp_bswap_rBX_r11),
+    /* 0xcc */  IEMOP_X4(iemOp_bswap_rSP_r12),
+    /* 0xcd */  IEMOP_X4(iemOp_bswap_rBP_r13),
+    /* 0xce */  IEMOP_X4(iemOp_bswap_rSI_r14),
+    /* 0xcf */  IEMOP_X4(iemOp_bswap_rDI_r15),
+
+    /* 0xd0 */  iemOp_InvalidNeedRM,        iemOp_addsubpd_Vpd_Wpd,     iemOp_InvalidNeedRM,        iemOp_addsubps_Vps_Wps,
+    /* 0xd1 */  iemOp_psrlw_Pq_Qq,          iemOp_psrlw_Vx_W,           iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd2 */  iemOp_psrld_Pq_Qq,          iemOp_psrld_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd3 */  iemOp_psrlq_Pq_Qq,          iemOp_psrlq_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd4 */  iemOp_paddq_Pq_Qq,          iemOp_paddq_Vx_W,           iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd5 */  iemOp_pmullw_Pq_Qq,         iemOp_pmullw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd6 */  iemOp_InvalidNeedRM,        iemOp_movq_Wq_Vq,           iemOp_movq2dq_Vdq_Nq,       iemOp_movdq2q_Pq_Uq,
+    /* 0xd7 */  iemOp_pmovmskb_Gd_Nq,       iemOp_pmovmskb_Gd_Ux,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd8 */  iemOp_psubusb_Pq_Qq,        iemOp_psubusb_Vx_W,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd9 */  iemOp_psubusw_Pq_Qq,        iemOp_psubusw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xda */  iemOp_pminub_Pq_Qq,         iemOp_pminub_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdb */  iemOp_pand_Pq_Qq,           iemOp_pand_Vx_W,            iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdc */  iemOp_paddusb_Pq_Qq,        iemOp_paddusb_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdd */  iemOp_paddusw_Pq_Qq,        iemOp_paddusw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xde */  iemOp_pmaxub_Pq_Qq,         iemOp_pmaxub_Vx_W,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdf */  iemOp_pandn_Pq_Qq,          iemOp_pandn_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0xe0 */  iemOp_pavgb_Pq_Qq,          iemOp_pavgb_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe1 */  iemOp_psraw_Pq_Qq,          iemOp_psraw_Vx_W,           iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe2 */  iemOp_psrad_Pq_Qq,          iemOp_psrad_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe3 */  iemOp_pavgw_Pq_Qq,          iemOp_pavgw_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe4 */  iemOp_pmulhuw_Pq_Qq,        iemOp_pmulhuw_Vx_W,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe5 */  iemOp_pmulhw_Pq_Qq,         iemOp_pmulhw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe6 */  iemOp_InvalidNeedRM,        iemOp_cvttpd2dq_Vx_Wpd,     iemOp_cvtdq2pd_Vx_Wpd,      iemOp_cvtpd2dq_Vx_Wpd,
+    /* 0xe7 */  iemOp_movntq_Mq_Pq,         iemOp_movntdq_Mdq_Vdq,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe8 */  iemOp_psubsb_Pq_Qq,         iemOp_psubsb_Vx_W,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe9 */  iemOp_psubsw_Pq_Qq,         iemOp_psubsw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xea */  iemOp_pminsw_Pq_Qq,         iemOp_pminsw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xeb */  iemOp_por_Pq_Qq,            iemOp_por_Vx_W,             iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xec */  iemOp_paddsb_Pq_Qq,         iemOp_paddsb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xed */  iemOp_paddsw_Pq_Qq,         iemOp_paddsw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xee */  iemOp_pmaxsw_Pq_Qq,         iemOp_pmaxsw_Vx_W,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xef */  iemOp_pxor_Pq_Qq,           iemOp_pxor_Vx_Wx,           iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0xf0 */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_lddqu_Vx_Mx,
+    /* 0xf1 */  iemOp_psllw_Pq_Qq,          iemOp_psllw_Vx_W,           iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf2 */  iemOp_pslld_Pq_Qq,          iemOp_pslld_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf3 */  iemOp_psllq_Pq_Qq,          iemOp_psllq_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf4 */  iemOp_pmuludq_Pq_Qq,        iemOp_pmuludq_Vx_W,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf5 */  iemOp_pmaddwd_Pq_Qq,        iemOp_pmaddwd_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf6 */  iemOp_psadbw_Pq_Qq,         iemOp_psadbw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf7 */  iemOp_maskmovq_Pq_Nq,       iemOp_maskmovdqu_Vdq_Udq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf8 */  iemOp_psubb_Pq_Qq,          iemOp_psubb_Vx_W,           iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf9 */  iemOp_psubw_Pq_Qq,          iemOp_psubw_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfa */  iemOp_psubd_Pq_Qq,          iemOp_psubd_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfb */  iemOp_psubq_Pq_Qq,          iemOp_psubq_Vx_W,           iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfc */  iemOp_paddb_Pq_Qq,          iemOp_paddb_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfd */  iemOp_paddw_Pq_Qq,          iemOp_paddw_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfe */  iemOp_paddd_Pq_Qq,          iemOp_paddd_Vx_W,           iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xff */  IEMOP_X4(iemOp_ud0),
+};
+AssertCompile(RT_ELEMENTS(g_apfnTwoByteMap) == 1024);
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h
new file mode 100644
index 00000000..1b5f4b45
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h
@@ -0,0 +1,4054 @@
+/* $Id: IEMAllInstructionsVexMap1.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation.
+ *
+ * @remarks IEMAllInstructionsTwoByte0f.cpp.h is a legacy mirror of this file.
+ *          Any update here is likely needed in that file too.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @name VEX Opcode Map 1
+ * @{
+ */
+
+
+/*  Opcode VEX.0F 0x00 - invalid */
+/*  Opcode VEX.0F 0x01 - invalid */
+/*  Opcode VEX.0F 0x02 - invalid */
+/*  Opcode VEX.0F 0x03 - invalid */
+/*  Opcode VEX.0F 0x04 - invalid */
+/*  Opcode VEX.0F 0x05 - invalid */
+/*  Opcode VEX.0F 0x06 - invalid */
+/*  Opcode VEX.0F 0x07 - invalid */
+/*  Opcode VEX.0F 0x08 - invalid */
+/*  Opcode VEX.0F 0x09 - invalid */
+/*  Opcode VEX.0F 0x0a - invalid */
+
+/** Opcode VEX.0F 0x0b. */
+FNIEMOP_DEF(iemOp_vud2)
+{
+    IEMOP_MNEMONIC(vud2, "vud2");
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/*  Opcode VEX.0F 0x0c - invalid */
+/*  Opcode VEX.0F 0x0d - invalid */
+/*  Opcode VEX.0F 0x0e - invalid */
+/*  Opcode VEX.0F 0x0f - invalid */
+
+
+/**
+ * @opcode      0x10
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdfp_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_vmovups_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVUPS, vmovups, Vps_WO, Wps, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * 128-bit: Register, Memory
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * 256-bit: Register, Memory
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x10
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdfp_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_vmovupd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVUPD, vmovupd, Vpd_WO, Wpd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * 128-bit: Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * 256-bit: Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+FNIEMOP_DEF(iemOp_vmovss_Vss_Hss_Wss)
+{
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /**
+         * @opcode      0x10
+         * @oppfx       0xf3
+         * @opcodesub   11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamerge
+         * @opxcpttype  5
+         * @optest      op1=1 op2=0  op3=2    -> op1=2
+         * @optest      op1=0 op2=0  op3=-22  -> op1=0xffffffea
+         * @optest      op1=3 op2=-1 op3=0x77 -> op1=-4294967177
+         * @optest      op1=3 op2=-2 op3=0x77 -> op1=-8589934473
+         * @note        HssHi refers to bits 127:32.
+         */
+        IEMOP_MNEMONIC3(VEX_RVM_REG, VMOVSS, vmovss, Vss_WO, HssHi, Uss, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEMOP_HLP_DONE_VEX_DECODING();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB /*U32*/,
+                                            IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hss*/);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x10
+         * @oppfx       0xf3
+         * @opcodesub   !11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_vmovss_Vss_Hss_Wss
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-22 -> op1=-22
+         */
+        IEMOP_MNEMONIC2(VEX_RM_MEM, VMOVSS, vmovss, VssZx_WO, Md, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U32_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+FNIEMOP_DEF(iemOp_vmovsd_Vsd_Hsd_Wsd)
+{
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /**
+         * @opcode      0x10
+         * @oppfx       0xf2
+         * @opcodesub   11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamerge
+         * @opxcpttype  5
+         * @optest      op1=1 op2=0  op3=2    -> op1=2
+         * @optest      op1=0 op2=0  op3=-22  -> op1=0xffffffffffffffea
+         * @optest      op1=3 op2=-1 op3=0x77 ->
+         *              op1=0xffffffffffffffff0000000000000077
+         * @optest      op1=3 op2=0x42 op3=0x77 -> op1=0x420000000000000077
+         */
+        IEMOP_MNEMONIC3(VEX_RVM_REG, VMOVSD, vmovsd, Vsd_WO, HsdHi, Usd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEMOP_HLP_DONE_VEX_DECODING();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB /*U32*/,
+                                           IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hss*/);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x10
+         * @oppfx       0xf2
+         * @opcodesub   !11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_vmovsd_Vsd_Hsd_Wsd
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-22 -> op1=-22
+         */
+        IEMOP_MNEMONIC2(VEX_RM_MEM, VMOVSD, vmovsd, VsdZx_WO, Mq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U64_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x11
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdfp_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_vmovups_Wps_Vps)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVUPS, vmovups, Wps_WO, Vps, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                           ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                           ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * 128-bit: Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * 256-bit: Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U256(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U256(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x11
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdfp_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_vmovupd_Wpd_Vpd)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVUPD, vmovupd, Wpd_WO, Vpd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                           ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                           ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * 128-bit: Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * 256-bit: Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U256(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U256(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+FNIEMOP_DEF(iemOp_vmovss_Wss_Hss_Vss)
+{
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /**
+         * @opcode      0x11
+         * @oppfx       0xf3
+         * @opcodesub   11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamerge
+         * @opxcpttype  5
+         * @optest      op1=1 op2=0  op3=2    -> op1=2
+         * @optest      op1=0 op2=0  op3=-22  -> op1=0xffffffea
+         * @optest      op1=3 op2=-1 op3=0x77 -> op1=-4294967177
+         * @optest      op1=3 op2=0x42 op3=0x77 -> op1=0x4200000077
+         */
+        IEMOP_MNEMONIC3(VEX_MVR_REG, VMOVSS, vmovss, Uss_WO, HssHi, Vss, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEMOP_HLP_DONE_VEX_DECODING();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB /*U32*/,
+                                           ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hss*/);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x11
+         * @oppfx       0xf3
+         * @opcodesub   !11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_vmovss_Vss_Hss_Wss
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-22 -> op1=-22
+         */
+        IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVSS, vmovss, Md_WO, Vss, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U32(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+FNIEMOP_DEF(iemOp_vmovsd_Wsd_Hsd_Vsd)
+{
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /**
+         * @opcode      0x11
+         * @oppfx       0xf2
+         * @opcodesub   11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamerge
+         * @opxcpttype  5
+         * @optest      op1=1 op2=0  op3=2    -> op1=2
+         * @optest      op1=0 op2=0  op3=-22  -> op1=0xffffffffffffffea
+         * @optest      op1=3 op2=-1 op3=0x77 ->
+         *              op1=0xffffffffffffffff0000000000000077
+         * @optest      op2=0x42 op3=0x77 -> op1=0x420000000000000077
+         */
+        IEMOP_MNEMONIC3(VEX_MVR_REG, VMOVSD, vmovsd, Usd_WO, HsdHi, Vsd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEMOP_HLP_DONE_VEX_DECODING();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                           ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hss*/);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x11
+         * @oppfx       0xf2
+         * @opcodesub   !11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_vmovsd_Wsd_Hsd_Vsd
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-22 -> op1=-22
+         */
+        IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVSD, vmovsd, Mq_WO, Vsd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U64(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+FNIEMOP_DEF(iemOp_vmovlps_Vq_Hq_Mq__vmovhlps)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /**
+         * @opcode      0x12
+         * @opcodesub   11 mr/reg
+         * @oppfx       none
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamerge
+         * @opxcpttype  7LZ
+         * @optest         op2=0x2200220122022203
+         *                 op3=0x3304330533063307
+         *              -> op1=0x22002201220222033304330533063307
+         * @optest      op2=-1  op3=-42 -> op1=-42
+         * @note        op3 and op2 are only the 8-byte high XMM register halfs.
+         */
+        IEMOP_MNEMONIC3(VEX_RVM_REG, VMOVHLPS, vmovhlps, Vq_WO, HqHi, UqHi, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+
+        IEMOP_HLP_DONE_VEX_DECODING_L0();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        IEM_MC_MERGE_YREG_U64HI_U64_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                             (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                             IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x12
+         * @opcodesub   !11 mr/reg
+         * @oppfx       none
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamove
+         * @opxcpttype  5LZ
+         * @opfunction  iemOp_vmovlps_Vq_Hq_Mq__vmovhlps
+         * @optest      op1=1 op2=0 op3=0 -> op1=0
+         * @optest      op1=0 op2=-1 op3=-1 -> op1=-1
+         * @optest      op1=1 op2=2 op3=3 -> op1=0x20000000000000003
+         * @optest      op2=-1 op3=0x42 -> op1=0xffffffffffffffff0000000000000042
+         */
+        IEMOP_MNEMONIC3(VEX_RVM_MEM, VMOVLPS, vmovlps, Vq_WO, HqHi, Mq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_MERGE_YREG_U64LOCAL_U64_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                                uSrc,
+                                                IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x12
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamerge
+ * @opxcpttype  5LZ
+ * @optest      op2=0 op3=2 -> op1=2
+ * @optest      op2=0x22 op3=0x33 -> op1=0x220000000000000033
+ * @optest      op2=0xfffffff0fffffff1 op3=0xeeeeeee8eeeeeee9
+ *              -> op1=0xfffffff0fffffff1eeeeeee8eeeeeee9
+ */
+FNIEMOP_DEF(iemOp_vmovlpd_Vq_Hq_Mq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_MNEMONIC3(VEX_RVM_MEM, VMOVLPD, vmovlpd, Vq_WO, HqHi, Mq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_MERGE_YREG_U64LOCAL_U64_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                                uSrc,
+                                                IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udvex660f12m3
+     * @opcode      0x12
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x12
+ * @oppfx       0xf3
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  4
+ * @optest      vex.l==0 / op1=-1 op2=0xdddddddd00000002eeeeeeee00000001
+ *              -> op1=0x00000002000000020000000100000001
+ * @optest      vex.l==1 /
+ *                 op2=0xbbbbbbbb00000004cccccccc00000003dddddddd00000002eeeeeeee00000001
+ *              -> op1=0x0000000400000004000000030000000300000002000000020000000100000001
+ */
+FNIEMOP_DEF(iemOp_vmovsldup_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVSLDUP, vmovsldup, Vx_WO, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+            IEM_MC_ARG(PCRTUINT128U,                puSrc, 1);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_REF_XREG_U128_CONST(puSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_REF_XREG_U128(puDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_CALL_SSE_AIMPL_2(iemAImpl_movsldup, puDst, puSrc);
+            IEM_MC_CLEAR_YREG_128_UP(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG_CONST(uint8_t,   iYRegDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, 1);
+            IEM_MC_ARG_CONST(uint8_t,   iYRegSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 2);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovsldup_256_rr, iYRegDst, iYRegSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(2, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_REF_XREG_U128(puDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_CALL_SSE_AIMPL_2(iemAImpl_movsldup, puDst, puSrc);
+            IEM_MC_CLEAR_YREG_128_UP(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG_CONST(uint8_t,   iYRegDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, 1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc, uSrc, 2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovsldup_256_rm, iYRegDst, puSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x12
+ * @oppfx       0xf2
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      vex.l==0 / op2=0xddddddddeeeeeeee2222222211111111
+ *              ->         op1=0x22222222111111112222222211111111
+ * @optest      vex.l==1 / op2=0xbbbbbbbbcccccccc4444444433333333ddddddddeeeeeeee2222222211111111
+ *              ->         op1=0x4444444433333333444444443333333322222222111111112222222211111111
+ */
+FNIEMOP_DEF(iemOp_vmovddup_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVDDUP, vmovddup, Vx_WO, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+            IEM_MC_ARG(uint64_t,                    uSrc, 1);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_XREG_U64(uSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_REF_XREG_U128(puDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_CALL_SSE_AIMPL_2(iemAImpl_movddup, puDst, uSrc);
+            IEM_MC_CLEAR_YREG_128_UP(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG_CONST(uint8_t,   iYRegDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, 1);
+            IEM_MC_ARG_CONST(uint8_t,   iYRegSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, 2);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovddup_256_rr, iYRegDst, iYRegSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(2, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+            IEM_MC_ARG(uint64_t,                    uSrc, 1);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_REF_XREG_U128(puDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_CALL_SSE_AIMPL_2(iemAImpl_movddup, puDst, uSrc);
+            IEM_MC_CLEAR_YREG_128_UP(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG_CONST(uint8_t,   iYRegDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, 1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc, uSrc, 2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovddup_256_rm, iYRegDst, puSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x13
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdfp_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovlps_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVLPS, vmovlps, Mq_WO, Vq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U64(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udvex0f13m3
+     * @opcode      0x13
+     * @opcodesub   11 mr/reg
+     * @oppfx       none
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x13
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovlpd_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVLPD, vmovlpd, Mq_WO, Vq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U64(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udvex660f13m3
+     * @opcode      0x13
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/*  Opcode VEX.F3.0F 0x13 - invalid */
+/*  Opcode VEX.F2.0F 0x13 - invalid */
+
+/** Opcode VEX.0F 0x14 - vunpcklps Vx, Hx, Wx*/
+FNIEMOP_STUB(iemOp_vunpcklps_Vx_Hx_Wx);
+/** Opcode VEX.66.0F 0x14 - vunpcklpd Vx,Hx,Wx   */
+FNIEMOP_STUB(iemOp_vunpcklpd_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0x14 - invalid */
+/*  Opcode VEX.F2.0F 0x14 - invalid */
+/** Opcode VEX.0F 0x15 - vunpckhps Vx, Hx, Wx   */
+FNIEMOP_STUB(iemOp_vunpckhps_Vx_Hx_Wx);
+/** Opcode VEX.66.0F 0x15 - vunpckhpd Vx,Hx,Wx   */
+FNIEMOP_STUB(iemOp_vunpckhpd_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0x15 - invalid */
+/*  Opcode VEX.F2.0F 0x15 - invalid */
+/** Opcode VEX.0F 0x16 - vmovhpsv1 Vdq, Hq, Mq vmovlhps Vdq, Hq, Uq   */
+FNIEMOP_STUB(iemOp_vmovhpsv1_Vdq_Hq_Mq__vmovlhps_Vdq_Hq_Uq);  //NEXT
+/** Opcode VEX.66.0F 0x16 - vmovhpdv1 Vdq, Hq, Mq   */
+FNIEMOP_STUB(iemOp_vmovhpdv1_Vdq_Hq_Mq);  //NEXT
+/** Opcode VEX.F3.0F 0x16 - vmovshdup Vx, Wx   */
+FNIEMOP_STUB(iemOp_vmovshdup_Vx_Wx); //NEXT
+/*  Opcode VEX.F2.0F 0x16 - invalid */
+/** Opcode VEX.0F 0x17 - vmovhpsv1 Mq, Vq   */
+FNIEMOP_STUB(iemOp_vmovhpsv1_Mq_Vq);  //NEXT
+/** Opcode VEX.66.0F 0x17 - vmovhpdv1 Mq, Vq   */
+FNIEMOP_STUB(iemOp_vmovhpdv1_Mq_Vq);  //NEXT
+/*  Opcode VEX.F3.0F 0x17 - invalid */
+/*  Opcode VEX.F2.0F 0x17 - invalid */
+
+
+/*  Opcode VEX.0F 0x18 - invalid */
+/*  Opcode VEX.0F 0x19 - invalid */
+/*  Opcode VEX.0F 0x1a - invalid */
+/*  Opcode VEX.0F 0x1b - invalid */
+/*  Opcode VEX.0F 0x1c - invalid */
+/*  Opcode VEX.0F 0x1d - invalid */
+/*  Opcode VEX.0F 0x1e - invalid */
+/*  Opcode VEX.0F 0x1f - invalid */
+
+/*  Opcode VEX.0F 0x20 - invalid */
+/*  Opcode VEX.0F 0x21 - invalid */
+/*  Opcode VEX.0F 0x22 - invalid */
+/*  Opcode VEX.0F 0x23 - invalid */
+/*  Opcode VEX.0F 0x24 - invalid */
+/*  Opcode VEX.0F 0x25 - invalid */
+/*  Opcode VEX.0F 0x26 - invalid */
+/*  Opcode VEX.0F 0x27 - invalid */
+
+/**
+ * @opcode      0x28
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ * @note        Almost identical to vmovapd.
+ */
+FNIEMOP_DEF(iemOp_vmovaps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVAPS, vmovaps, Vps_WO, Wps, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(1, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U256_ALIGN_AVX(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0x28
+ * @oppfx       66
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ * @note        Almost identical to vmovaps
+ */
+FNIEMOP_DEF(iemOp_vmovapd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVAPD, vmovapd, Vpd_WO, Wpd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(1, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U256_ALIGN_AVX(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opmnemonic  udvexf30f28
+ * @opcode      0x28
+ * @oppfx       0xf3
+ * @opunused    vex.modrm
+ * @opcpuid     avx
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udvexf20f28
+ * @opcode      0x28
+ * @oppfx       0xf2
+ * @opunused    vex.modrm
+ * @opcpuid     avx
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opcode      0x29
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ * @note        Almost identical to vmovapd.
+ */
+FNIEMOP_DEF(iemOp_vmovaps_Wps_Vps)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVAPS, vmovaps, Wps_WO, Vps, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(1, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                           ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U256(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opcode      0x29
+ * @oppfx       66
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ * @note        Almost identical to vmovaps
+ */
+FNIEMOP_DEF(iemOp_vmovapd_Wpd_Vpd)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVAPD, vmovapd, Wpd_WO, Vpd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(1, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                           ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U256(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opmnemonic  udvexf30f29
+ * @opcode      0x29
+ * @oppfx       0xf3
+ * @opunused    vex.modrm
+ * @opcpuid     avx
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udvexf20f29
+ * @opcode      0x29
+ * @oppfx       0xf2
+ * @opunused    vex.modrm
+ * @opcpuid     avx
+ * @optest      ->
+ * @opdone
+ */
+
+
+/** Opcode VEX.0F 0x2a - invalid */
+/** Opcode VEX.66.0F 0x2a - invalid */
+/** Opcode VEX.F3.0F 0x2a - vcvtsi2ss Vss, Hss, Ey */
+FNIEMOP_STUB(iemOp_vcvtsi2ss_Vss_Hss_Ey);
+/** Opcode VEX.F2.0F 0x2a - vcvtsi2sd Vsd, Hsd, Ey */
+FNIEMOP_STUB(iemOp_vcvtsi2sd_Vsd_Hsd_Ey);
+
+
+/**
+ * @opcode      0x2b
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_cachect
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ * @note        Identical implementation to vmovntpd
+ */
+FNIEMOP_DEF(iemOp_vmovntps_Mps_Vps)
+{
+    IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVNTPS, vmovntps, Mps_WO, Vps, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * memory, register.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_XREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_YREG_U256(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    /* The register, register encoding is invalid. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opcode      0x2b
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_cachect
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ * @note        Identical implementation to vmovntps
+ */
+FNIEMOP_DEF(iemOp_vmovntpd_Mpd_Vpd)
+{
+    IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVNTPD, vmovntpd, Mpd_WO, Vpd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * memory, register.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_XREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_YREG_U256(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    /* The register, register encoding is invalid. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opmnemonic  udvexf30f2b
+ * @opcode      0x2b
+ * @oppfx       0xf3
+ * @opunused    vex.modrm
+ * @opcpuid     avx
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udvexf20f2b
+ * @opcode      0x2b
+ * @oppfx       0xf2
+ * @opunused    vex.modrm
+ * @opcpuid     avx
+ * @optest      ->
+ * @opdone
+ */
+
+
+/*  Opcode VEX.0F 0x2c - invalid */
+/*  Opcode VEX.66.0F 0x2c - invalid */
+/** Opcode VEX.F3.0F 0x2c - vcvttss2si Gy, Wss */
+FNIEMOP_STUB(iemOp_vcvttss2si_Gy_Wss);
+/** Opcode VEX.F2.0F 0x2c - vcvttsd2si Gy, Wsd */
+FNIEMOP_STUB(iemOp_vcvttsd2si_Gy_Wsd);
+
+/*  Opcode VEX.0F 0x2d - invalid */
+/*  Opcode VEX.66.0F 0x2d - invalid */
+/** Opcode VEX.F3.0F 0x2d - vcvtss2si Gy, Wss */
+FNIEMOP_STUB(iemOp_vcvtss2si_Gy_Wss);
+/** Opcode VEX.F2.0F 0x2d - vcvtsd2si Gy, Wsd */
+FNIEMOP_STUB(iemOp_vcvtsd2si_Gy_Wsd);
+
+/** Opcode VEX.0F 0x2e - vucomiss Vss, Wss */
+FNIEMOP_STUB(iemOp_vucomiss_Vss_Wss);
+/** Opcode VEX.66.0F 0x2e - vucomisd Vsd, Wsd */
+FNIEMOP_STUB(iemOp_vucomisd_Vsd_Wsd);
+/*  Opcode VEX.F3.0F 0x2e - invalid */
+/*  Opcode VEX.F2.0F 0x2e - invalid */
+
+/** Opcode VEX.0F 0x2f - vcomiss Vss, Wss */
+FNIEMOP_STUB(iemOp_vcomiss_Vss_Wss);
+/** Opcode VEX.66.0F 0x2f - vcomisd Vsd, Wsd */
+FNIEMOP_STUB(iemOp_vcomisd_Vsd_Wsd);
+/*  Opcode VEX.F3.0F 0x2f - invalid */
+/*  Opcode VEX.F2.0F 0x2f - invalid */
+
+/*  Opcode VEX.0F 0x30 - invalid */
+/*  Opcode VEX.0F 0x31 - invalid */
+/*  Opcode VEX.0F 0x32 - invalid */
+/*  Opcode VEX.0F 0x33 - invalid */
+/*  Opcode VEX.0F 0x34 - invalid */
+/*  Opcode VEX.0F 0x35 - invalid */
+/*  Opcode VEX.0F 0x36 - invalid */
+/*  Opcode VEX.0F 0x37 - invalid */
+/*  Opcode VEX.0F 0x38 - invalid */
+/*  Opcode VEX.0F 0x39 - invalid */
+/*  Opcode VEX.0F 0x3a - invalid */
+/*  Opcode VEX.0F 0x3b - invalid */
+/*  Opcode VEX.0F 0x3c - invalid */
+/*  Opcode VEX.0F 0x3d - invalid */
+/*  Opcode VEX.0F 0x3e - invalid */
+/*  Opcode VEX.0F 0x3f - invalid */
+/*  Opcode VEX.0F 0x40 - invalid */
+/*  Opcode VEX.0F 0x41 - invalid */
+/*  Opcode VEX.0F 0x42 - invalid */
+/*  Opcode VEX.0F 0x43 - invalid */
+/*  Opcode VEX.0F 0x44 - invalid */
+/*  Opcode VEX.0F 0x45 - invalid */
+/*  Opcode VEX.0F 0x46 - invalid */
+/*  Opcode VEX.0F 0x47 - invalid */
+/*  Opcode VEX.0F 0x48 - invalid */
+/*  Opcode VEX.0F 0x49 - invalid */
+/*  Opcode VEX.0F 0x4a - invalid */
+/*  Opcode VEX.0F 0x4b - invalid */
+/*  Opcode VEX.0F 0x4c - invalid */
+/*  Opcode VEX.0F 0x4d - invalid */
+/*  Opcode VEX.0F 0x4e - invalid */
+/*  Opcode VEX.0F 0x4f - invalid */
+
+/** Opcode VEX.0F 0x50 - vmovmskps Gy, Ups */
+FNIEMOP_STUB(iemOp_vmovmskps_Gy_Ups);
+/** Opcode VEX.66.0F 0x50 - vmovmskpd Gy,Upd */
+FNIEMOP_STUB(iemOp_vmovmskpd_Gy_Upd);
+/*  Opcode VEX.F3.0F 0x50 - invalid */
+/*  Opcode VEX.F2.0F 0x50 - invalid */
+
+/** Opcode VEX.0F 0x51 - vsqrtps Vps, Wps */
+FNIEMOP_STUB(iemOp_vsqrtps_Vps_Wps);
+/** Opcode VEX.66.0F 0x51 - vsqrtpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_vsqrtpd_Vpd_Wpd);
+/** Opcode VEX.F3.0F 0x51 - vsqrtss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vsqrtss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x51 - vsqrtsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vsqrtsd_Vsd_Hsd_Wsd);
+
+/** Opcode VEX.0F 0x52 - vrsqrtps Vps, Wps */
+FNIEMOP_STUB(iemOp_vrsqrtps_Vps_Wps);
+/*  Opcode VEX.66.0F 0x52 - invalid */
+/** Opcode VEX.F3.0F 0x52 - vrsqrtss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vrsqrtss_Vss_Hss_Wss);
+/*  Opcode VEX.F2.0F 0x52 - invalid */
+
+/** Opcode VEX.0F 0x53 - vrcpps Vps, Wps */
+FNIEMOP_STUB(iemOp_vrcpps_Vps_Wps);
+/*  Opcode VEX.66.0F 0x53 - invalid */
+/** Opcode VEX.F3.0F 0x53 - vrcpss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vrcpss_Vss_Hss_Wss);
+/*  Opcode VEX.F2.0F 0x53 - invalid */
+
+/** Opcode VEX.0F 0x54 - vandps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vandps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x54 - vandpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vandpd_Vpd_Hpd_Wpd);
+/*  Opcode VEX.F3.0F 0x54 - invalid */
+/*  Opcode VEX.F2.0F 0x54 - invalid */
+
+/** Opcode VEX.0F 0x55 - vandnps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vandnps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x55 - vandnpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vandnpd_Vpd_Hpd_Wpd);
+/*  Opcode VEX.F3.0F 0x55 - invalid */
+/*  Opcode VEX.F2.0F 0x55 - invalid */
+
+/** Opcode VEX.0F 0x56 - vorps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vorps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x56 - vorpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vorpd_Vpd_Hpd_Wpd);
+/*  Opcode VEX.F3.0F 0x56 - invalid */
+/*  Opcode VEX.F2.0F 0x56 - invalid */
+
+/** Opcode VEX.0F 0x57 - vxorps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vxorps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x57 - vxorpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vxorpd_Vpd_Hpd_Wpd);
+/*  Opcode VEX.F3.0F 0x57 - invalid */
+/*  Opcode VEX.F2.0F 0x57 - invalid */
+
+/** Opcode VEX.0F 0x58 - vaddps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vaddps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x58 - vaddpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vaddpd_Vpd_Hpd_Wpd);
+/** Opcode VEX.F3.0F 0x58 - vaddss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vaddss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x58 - vaddsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vaddsd_Vsd_Hsd_Wsd);
+
+/** Opcode VEX.0F 0x59 - vmulps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vmulps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x59 - vmulpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vmulpd_Vpd_Hpd_Wpd);
+/** Opcode VEX.F3.0F 0x59 - vmulss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vmulss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x59 - vmulsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vmulsd_Vsd_Hsd_Wsd);
+
+/** Opcode VEX.0F 0x5a - vcvtps2pd Vpd, Wps */
+FNIEMOP_STUB(iemOp_vcvtps2pd_Vpd_Wps);
+/** Opcode VEX.66.0F 0x5a - vcvtpd2ps Vps, Wpd */
+FNIEMOP_STUB(iemOp_vcvtpd2ps_Vps_Wpd);
+/** Opcode VEX.F3.0F 0x5a - vcvtss2sd Vsd, Hx, Wss */
+FNIEMOP_STUB(iemOp_vcvtss2sd_Vsd_Hx_Wss);
+/** Opcode VEX.F2.0F 0x5a - vcvtsd2ss Vss, Hx, Wsd */
+FNIEMOP_STUB(iemOp_vcvtsd2ss_Vss_Hx_Wsd);
+
+/** Opcode VEX.0F 0x5b - vcvtdq2ps Vps, Wdq */
+FNIEMOP_STUB(iemOp_vcvtdq2ps_Vps_Wdq);
+/** Opcode VEX.66.0F 0x5b - vcvtps2dq Vdq, Wps */
+FNIEMOP_STUB(iemOp_vcvtps2dq_Vdq_Wps);
+/** Opcode VEX.F3.0F 0x5b - vcvttps2dq Vdq, Wps */
+FNIEMOP_STUB(iemOp_vcvttps2dq_Vdq_Wps);
+/*  Opcode VEX.F2.0F 0x5b - invalid */
+
+/** Opcode VEX.0F 0x5c - vsubps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vsubps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x5c - vsubpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vsubpd_Vpd_Hpd_Wpd);
+/** Opcode VEX.F3.0F 0x5c - vsubss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vsubss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x5c - vsubsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vsubsd_Vsd_Hsd_Wsd);
+
+/** Opcode VEX.0F 0x5d - vminps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vminps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x5d - vminpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vminpd_Vpd_Hpd_Wpd);
+/** Opcode VEX.F3.0F 0x5d - vminss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vminss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x5d - vminsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vminsd_Vsd_Hsd_Wsd);
+
+/** Opcode VEX.0F 0x5e - vdivps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vdivps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x5e - vdivpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vdivpd_Vpd_Hpd_Wpd);
+/** Opcode VEX.F3.0F 0x5e - vdivss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vdivss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x5e - vdivsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vdivsd_Vsd_Hsd_Wsd);
+
+/** Opcode VEX.0F 0x5f - vmaxps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vmaxps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x5f - vmaxpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vmaxpd_Vpd_Hpd_Wpd);
+/** Opcode VEX.F3.0F 0x5f - vmaxss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vmaxss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x5f - vmaxsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vmaxsd_Vsd_Hsd_Wsd);
+
+
+///**
+// * Common worker for SSE2 instructions on the forms:
+// *      pxxxx xmm1, xmm2/mem128
+// *
+// * The 2nd operand is the first half of a register, which in the memory case
+// * means a 32-bit memory access for MMX and 128-bit aligned 64-bit or 128-bit
+// * memory accessed for MMX.
+// *
+// * Exceptions type 4.
+// */
+//FNIEMOP_DEF_1(iemOpCommonSse_LowLow_To_Full, PCIEMOPMEDIAF1L1, pImpl)
+//{
+//    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+//    if (!pImpl->pfnU64)
+//        return IEMOP_RAISE_INVALID_OPCODE();
+//    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+//    {
+//        /*
+//         * Register, register.
+//         */
+//        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+//        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//        IEM_MC_BEGIN(2, 0);
+//        IEM_MC_ARG(uint64_t *,          pDst, 0);
+//        IEM_MC_ARG(uint32_t const *,    pSrc, 1);
+//        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+//        IEM_MC_PREPARE_FPU_USAGE();
+//        IEM_MC_REF_MREG_U64(pDst, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+//        IEM_MC_REF_MREG_U32_CONST(pSrc, bRm & X86_MODRM_RM_MASK);
+//        IEM_MC_CALL_MMX_AIMPL_2(pImpl->pfnU64, pDst, pSrc);
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//    }
+//    else
+//    {
+//        /*
+//         * Register, memory.
+//         */
+//        IEM_MC_BEGIN(2, 2);
+//        IEM_MC_ARG(uint64_t *,                  pDst,       0);
+//        IEM_MC_LOCAL(uint32_t,                  uSrc);
+//        IEM_MC_ARG_LOCAL_REF(uint32_t const *,  pSrc, uSrc, 1);
+//        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+//
+//        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+//        IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+//
+//        IEM_MC_PREPARE_FPU_USAGE();
+//        IEM_MC_REF_MREG_U64(pDst, (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK);
+//        IEM_MC_CALL_MMX_AIMPL_2(pImpl->pfnU64, pDst, pSrc);
+//
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//    }
+//    return VINF_SUCCESS;
+//}
+
+
+/*  Opcode VEX.0F 0x60 - invalid */
+
+/** Opcode VEX.66.0F 0x60 - vpunpcklbw Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpunpcklbw_Vx_Hx_Wx);
+//FNIEMOP_DEF(iemOp_vpunpcklbw_Vx_Hx_Wx)
+//{
+//    IEMOP_MNEMONIC(vpunpcklbw, "vpunpcklbw Vx, Hx, Wx");
+//    return FNIEMOP_CALL_1(iemOpCommonSse_LowLow_To_Full, &g_iemAImpl_punpcklbw);
+//}
+
+/*  Opcode VEX.F3.0F 0x60 - invalid */
+
+
+/*  Opcode VEX.0F 0x61 - invalid */
+
+/** Opcode VEX.66.0F 0x61 - vpunpcklwd Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpunpcklwd_Vx_Hx_Wx);
+//FNIEMOP_DEF(iemOp_vpunpcklwd_Vx_Hx_Wx)
+//{
+//    IEMOP_MNEMONIC(vpunpcklwd, "vpunpcklwd Vx, Hx, Wx");
+//    return FNIEMOP_CALL_1(iemOpCommonSse_LowLow_To_Full, &g_iemAImpl_punpcklwd);
+//}
+
+/*  Opcode VEX.F3.0F 0x61 - invalid */
+
+
+/*  Opcode VEX.0F 0x62 - invalid */
+
+/** Opcode VEX.66.0F 0x62 - vpunpckldq Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpunpckldq_Vx_Hx_Wx);
+//FNIEMOP_DEF(iemOp_vpunpckldq_Vx_Hx_Wx)
+//{
+//    IEMOP_MNEMONIC(vpunpckldq, "vpunpckldq Vx, Hx, Wx");
+//    return FNIEMOP_CALL_1(iemOpCommonSse_LowLow_To_Full, &g_iemAImpl_punpckldq);
+//}
+
+/*  Opcode VEX.F3.0F 0x62 - invalid */
+
+
+
+/*  Opcode VEX.0F 0x63 - invalid */
+/** Opcode VEX.66.0F 0x63 - vpacksswb Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpacksswb_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0x63 - invalid */
+
+/*  Opcode VEX.0F 0x64 - invalid */
+/** Opcode VEX.66.0F 0x64 - vpcmpgtb Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpcmpgtb_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0x64 - invalid */
+
+/*  Opcode VEX.0F 0x65 - invalid */
+/** Opcode VEX.66.0F 0x65 - vpcmpgtw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpcmpgtw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0x65 - invalid */
+
+/*  Opcode VEX.0F 0x66 - invalid */
+/** Opcode VEX.66.0F 0x66 - vpcmpgtd Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpcmpgtd_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0x66 - invalid */
+
+/*  Opcode VEX.0F 0x67 - invalid */
+/** Opcode VEX.66.0F 0x67 - vpackuswb Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpackuswb_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0x67 - invalid */
+
+
+///**
+// * Common worker for SSE2 instructions on the form:
+// *      pxxxx xmm1, xmm2/mem128
+// *
+// * The 2nd operand is the second half of a register, which in the memory case
+// * means a 64-bit memory access for MMX, and for SSE a 128-bit aligned access
+// * where it may read the full 128 bits or only the upper 64 bits.
+// *
+// * Exceptions type 4.
+// */
+//FNIEMOP_DEF_1(iemOpCommonSse_HighHigh_To_Full, PCIEMOPMEDIAF1H1, pImpl)
+//{
+//    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+//    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+//    {
+//        /*
+//         * Register, register.
+//         */
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//        IEM_MC_BEGIN(2, 0);
+//        IEM_MC_ARG(PRTUINT128U,          pDst, 0);
+//        IEM_MC_ARG(PCRTUINT128U,         pSrc, 1);
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+//        IEM_MC_REF_XREG_U128_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+//        IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc);
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//    }
+//    else
+//    {
+//        /*
+//         * Register, memory.
+//         */
+//        IEM_MC_BEGIN(2, 2);
+//        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+//        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+//        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+//        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+//
+//        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); /* Most CPUs probably only right high qword */
+//
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+//        IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc);
+//
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//    }
+//    return VINF_SUCCESS;
+//}
+
+
+/*  Opcode VEX.0F 0x68 - invalid */
+
+/** Opcode VEX.66.0F 0x68 - vpunpckhbw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpunpckhbw_Vx_Hx_Wx);
+//FNIEMOP_DEF(iemOp_vpunpckhbw_Vx_Hx_Wx)
+//{
+//    IEMOP_MNEMONIC(vpunpckhbw, "vpunpckhbw Vx, Hx, Wx");
+//    return FNIEMOP_CALL_1(iemOpCommonSse_HighHigh_To_Full, &g_iemAImpl_punpckhbw);
+//}
+/*  Opcode VEX.F3.0F 0x68 - invalid */
+
+
+/*  Opcode VEX.0F 0x69 - invalid */
+
+/** Opcode VEX.66.0F 0x69 - vpunpckhwd Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpunpckhwd_Vx_Hx_Wx);
+//FNIEMOP_DEF(iemOp_vpunpckhwd_Vx_Hx_Wx)
+//{
+//    IEMOP_MNEMONIC(vpunpckhwd, "vpunpckhwd Vx, Hx, Wx");
+//    return FNIEMOP_CALL_1(iemOpCommonSse_HighHigh_To_Full, &g_iemAImpl_punpckhwd);
+//
+//}
+/*  Opcode VEX.F3.0F 0x69 - invalid */
+
+
+/*  Opcode VEX.0F 0x6a - invalid */
+
+/** Opcode VEX.66.0F 0x6a - vpunpckhdq Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpunpckhdq_Vx_Hx_W);
+//FNIEMOP_DEF(iemOp_vpunpckhdq_Vx_Hx_W)
+//{
+//    IEMOP_MNEMONIC(vpunpckhdq, "vpunpckhdq Vx, Hx, W");
+//    return FNIEMOP_CALL_1(iemOpCommonSse_HighHigh_To_Full, &g_iemAImpl_punpckhdq);
+//}
+/*  Opcode VEX.F3.0F 0x6a - invalid */
+
+
+/*  Opcode VEX.0F 0x6b - invalid */
+/** Opcode VEX.66.0F 0x6b - vpackssdw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpackssdw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0x6b - invalid */
+
+
+/*  Opcode VEX.0F 0x6c - invalid */
+
+/** Opcode VEX.66.0F 0x6c - vpunpcklqdq Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpunpcklqdq_Vx_Hx_Wx);
+//FNIEMOP_DEF(iemOp_vpunpcklqdq_Vx_Hx_Wx)
+//{
+//    IEMOP_MNEMONIC(vpunpcklqdq, "vpunpcklqdq Vx, Hx, Wx");
+//    return FNIEMOP_CALL_1(iemOpCommonSse_LowLow_To_Full, &g_iemAImpl_punpcklqdq);
+//}
+
+/*  Opcode VEX.F3.0F 0x6c - invalid */
+/*  Opcode VEX.F2.0F 0x6c - invalid */
+
+
+/*  Opcode VEX.0F 0x6d - invalid */
+
+/** Opcode VEX.66.0F 0x6d - vpunpckhqdq Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpunpckhqdq_Vx_Hx_W);
+//FNIEMOP_DEF(iemOp_vpunpckhqdq_Vx_Hx_W)
+//{
+//    IEMOP_MNEMONIC(punpckhqdq, "punpckhqdq");
+//    return FNIEMOP_CALL_1(iemOpCommonSse_HighHigh_To_Full, &g_iemAImpl_punpckhqdq);
+//}
+
+/*  Opcode VEX.F3.0F 0x6d - invalid */
+
+
+/*  Opcode VEX.0F 0x6e - invalid */
+
+FNIEMOP_DEF(iemOp_vmovd_q_Vy_Ey)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x6e
+         * @opcodesub   rex.w=1
+         * @oppfx       0x66
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdint_datamov
+         * @opxcpttype  5
+         * @optest      64-bit / op1=1 op2=2   -> op1=2
+         * @optest      64-bit / op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(VEX_RM, VMOVQ, vmovq, Vq_WO, Eq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_VEX_L_ZERO);
+        if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /* XMM, greg64 */
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_GREG_U64(u64Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_STORE_YREG_U64_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* XMM, [mem64] */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U64_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u64Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x6e
+         * @opcodesub   rex.w=0
+         * @oppfx       0x66
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdint_datamov
+         * @opxcpttype  5
+         * @opfunction  iemOp_vmovd_q_Vy_Ey
+         * @optest      op1=1 op2=2   -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(VEX_RM, VMOVD, vmovd, Vd_WO, Ed, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_VEX_L_ZERO);
+        if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /* XMM, greg32 */
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_GREG_U32(u32Tmp, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+            IEM_MC_STORE_YREG_U32_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* XMM, [mem32] */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U32_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u32Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/*  Opcode VEX.F3.0F 0x6e - invalid */
+
+
+/*  Opcode VEX.0F 0x6f - invalid */
+
+/**
+ * @opcode      0x6f
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdint_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2   -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovdqa_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVDQA, vmovdqa, Vx_WO, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * Register, memory128.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u128Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory256.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U, u256Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U256_ALIGN_AVX(u256Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u256Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opcode      0x6f
+ * @oppfx       0xf3
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdint_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2   -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovdqu_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVDQU, vmovdqu, Vx_WO, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                           (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * Register, memory128.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u128Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory256.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U, u256Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U256(u256Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, u256Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/*  Opcode VEX.0F 0x70 - invalid */
+
+/** Opcode VEX.66.0F 0x70 - vpshufd Vx, Wx, Ib */
+FNIEMOP_STUB(iemOp_vpshufd_Vx_Wx_Ib);
+//FNIEMOP_DEF(iemOp_vpshufd_Vx_Wx_Ib)
+//{
+//    IEMOP_MNEMONIC(vpshufd_Vx_Wx_Ib, "vpshufd Vx,Wx,Ib");
+//    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+//    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+//    {
+//        /*
+//         * Register, register.
+//         */
+//        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//
+//        IEM_MC_BEGIN(3, 0);
+//        IEM_MC_ARG(PRTUINT128U,         pDst, 0);
+//        IEM_MC_ARG(PCRTUINT128U,        pSrc, 1);
+//        IEM_MC_ARG_CONST(uint8_t,       bEvilArg, /*=*/ bEvil, 2);
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+//        IEM_MC_REF_XREG_U128_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+//        IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_pshufd, pDst, pSrc, bEvilArg);
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//    }
+//    else
+//    {
+//        /*
+//         * Register, memory.
+//         */
+//        IEM_MC_BEGIN(3, 2);
+//        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+//        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+//        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+//        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+//
+//        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+//        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+//        IEM_MC_ARG_CONST(uint8_t,               bEvilArg, /*=*/ bEvil, 2);
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//
+//        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+//        IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_pshufd, pDst, pSrc, bEvilArg);
+//
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//    }
+//    return VINF_SUCCESS;
+//}
+
+/** Opcode VEX.F3.0F 0x70 - vpshufhw Vx, Wx, Ib */
+FNIEMOP_STUB(iemOp_vpshufhw_Vx_Wx_Ib);
+//FNIEMOP_DEF(iemOp_vpshufhw_Vx_Wx_Ib)
+//{
+//    IEMOP_MNEMONIC(vpshufhw_Vx_Wx_Ib, "vpshufhw Vx,Wx,Ib");
+//    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+//    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+//    {
+//        /*
+//         * Register, register.
+//         */
+//        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//
+//        IEM_MC_BEGIN(3, 0);
+//        IEM_MC_ARG(PRTUINT128U,         pDst, 0);
+//        IEM_MC_ARG(PCRTUINT128U,        pSrc, 1);
+//        IEM_MC_ARG_CONST(uint8_t,       bEvilArg, /*=*/ bEvil, 2);
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+//        IEM_MC_REF_XREG_U128_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+//        IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_pshufhw, pDst, pSrc, bEvilArg);
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//    }
+//    else
+//    {
+//        /*
+//         * Register, memory.
+//         */
+//        IEM_MC_BEGIN(3, 2);
+//        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+//        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+//        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+//        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+//
+//        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+//        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+//        IEM_MC_ARG_CONST(uint8_t,               bEvilArg, /*=*/ bEvil, 2);
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//
+//        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+//        IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_pshufhw, pDst, pSrc, bEvilArg);
+//
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//    }
+//    return VINF_SUCCESS;
+//}
+
+/** Opcode VEX.F2.0F 0x70 - vpshuflw Vx, Wx, Ib */
+FNIEMOP_STUB(iemOp_vpshuflw_Vx_Wx_Ib);
+//FNIEMOP_DEF(iemOp_vpshuflw_Vx_Wx_Ib)
+//{
+//    IEMOP_MNEMONIC(vpshuflw_Vx_Wx_Ib, "vpshuflw Vx,Wx,Ib");
+//    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+//    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+//    {
+//        /*
+//         * Register, register.
+//         */
+//        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//
+//        IEM_MC_BEGIN(3, 0);
+//        IEM_MC_ARG(PRTUINT128U,         pDst, 0);
+//        IEM_MC_ARG(PCRTUINT128U,        pSrc, 1);
+//        IEM_MC_ARG_CONST(uint8_t,       bEvilArg, /*=*/ bEvil, 2);
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+//        IEM_MC_REF_XREG_U128_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+//        IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_pshuflw, pDst, pSrc, bEvilArg);
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//    }
+//    else
+//    {
+//        /*
+//         * Register, memory.
+//         */
+//        IEM_MC_BEGIN(3, 2);
+//        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+//        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+//        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+//        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+//
+//        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+//        uint8_t bEvil; IEM_OPCODE_GET_NEXT_U8(&bEvil);
+//        IEM_MC_ARG_CONST(uint8_t,               bEvilArg, /*=*/ bEvil, 2);
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//
+//        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+//        IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_pshuflw, pDst, pSrc, bEvilArg);
+//
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//    }
+//    return VINF_SUCCESS;
+//}
+
+
+/*  Opcode VEX.0F 0x71 11/2 - invalid. */
+/** Opcode VEX.66.0F 0x71 11/2. */
+FNIEMOP_STUB_1(iemOp_VGrp12_vpsrlw_Hx_Ux_Ib, uint8_t, bRm);
+
+/*  Opcode VEX.0F 0x71 11/4 - invalid */
+/** Opcode VEX.66.0F 0x71 11/4. */
+FNIEMOP_STUB_1(iemOp_VGrp12_vpsraw_Hx_Ux_Ib, uint8_t, bRm);
+
+/*  Opcode VEX.0F 0x71 11/6 - invalid */
+/** Opcode VEX.66.0F 0x71 11/6. */
+FNIEMOP_STUB_1(iemOp_VGrp12_vpsllw_Hx_Ux_Ib, uint8_t, bRm);
+
+
+/**
+ * VEX Group 12 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnVexGroup12RegReg[] =
+{
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /2 */ iemOp_InvalidWithRMNeedImm8,   iemOp_VGrp12_vpsrlw_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /3 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /4 */ iemOp_InvalidWithRMNeedImm8,   iemOp_VGrp12_vpsraw_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /6 */ iemOp_InvalidWithRMNeedImm8,   iemOp_VGrp12_vpsllw_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /7 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8)
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexGroup12RegReg) == 8*4);
+
+
+/** Opcode VEX.0F 0x71. */
+FNIEMOP_DEF(iemOp_VGrp12)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnVexGroup12RegReg[  ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) * 4
+                                                     + pVCpu->iem.s.idxPrefix], bRm);
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm);
+}
+
+
+/*  Opcode VEX.0F 0x72 11/2 - invalid. */
+/** Opcode VEX.66.0F 0x72 11/2. */
+FNIEMOP_STUB_1(iemOp_VGrp13_vpsrld_Hx_Ux_Ib, uint8_t, bRm);
+
+/*  Opcode VEX.0F 0x72 11/4 - invalid. */
+/** Opcode VEX.66.0F 0x72 11/4. */
+FNIEMOP_STUB_1(iemOp_VGrp13_vpsrad_Hx_Ux_Ib, uint8_t, bRm);
+
+/*  Opcode VEX.0F 0x72 11/6 - invalid. */
+/** Opcode VEX.66.0F 0x72 11/6. */
+FNIEMOP_STUB_1(iemOp_VGrp13_vpslld_Hx_Ux_Ib, uint8_t, bRm);
+
+
+/**
+ * Group 13 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnVexGroup13RegReg[] =
+{
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /2 */ iemOp_InvalidWithRMNeedImm8,   iemOp_VGrp13_vpsrld_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /3 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /4 */ iemOp_InvalidWithRMNeedImm8,   iemOp_VGrp13_vpsrad_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /6 */ iemOp_InvalidWithRMNeedImm8,   iemOp_VGrp13_vpslld_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /7 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8)
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexGroup13RegReg) == 8*4);
+
+/** Opcode VEX.0F 0x72. */
+FNIEMOP_DEF(iemOp_VGrp13)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnVexGroup13RegReg[ ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) * 4
+                                                     + pVCpu->iem.s.idxPrefix], bRm);
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm);
+}
+
+
+/*  Opcode VEX.0F 0x73 11/2 - invalid. */
+/** Opcode VEX.66.0F 0x73 11/2. */
+FNIEMOP_STUB_1(iemOp_VGrp14_vpsrlq_Hx_Ux_Ib, uint8_t, bRm);
+
+/** Opcode VEX.66.0F 0x73 11/3. */
+FNIEMOP_STUB_1(iemOp_VGrp14_vpsrldq_Hx_Ux_Ib, uint8_t, bRm);
+
+/*  Opcode VEX.0F 0x73 11/6 - invalid. */
+/** Opcode VEX.66.0F 0x73 11/6. */
+FNIEMOP_STUB_1(iemOp_VGrp14_vpsllq_Hx_Ux_Ib, uint8_t, bRm);
+
+/** Opcode VEX.66.0F 0x73 11/7. */
+FNIEMOP_STUB_1(iemOp_VGrp14_vpslldq_Hx_Ux_Ib, uint8_t, bRm);
+
+/**
+ * Group 14 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnVexGroup14RegReg[] =
+{
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /2 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp14_vpsrlq_Hx_Ux_Ib,  iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /3 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp14_vpsrldq_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /4 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /6 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp14_vpsllq_Hx_Ux_Ib,  iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /7 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp14_vpslldq_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexGroup14RegReg) == 8*4);
+
+
+/** Opcode VEX.0F 0x73. */
+FNIEMOP_DEF(iemOp_VGrp14)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnVexGroup14RegReg[ ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) * 4
+                                                     + pVCpu->iem.s.idxPrefix], bRm);
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm);
+}
+
+
+///**
+// * Common worker for SSE2 instructions on the forms:
+// *      pxxx    xmm1, xmm2/mem128
+// *
+// * Proper alignment of the 128-bit operand is enforced.
+// * Exceptions type 4. SSE2 cpuid checks.
+// */
+//FNIEMOP_DEF_1(iemOpCommonSse2_FullFull_To_Full, PCIEMOPMEDIAF2, pImpl)
+//{
+//    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+//    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+//    {
+//        /*
+//         * Register, register.
+//         */
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//        IEM_MC_BEGIN(2, 0);
+//        IEM_MC_ARG(PRTUINT128U,          pDst, 0);
+//        IEM_MC_ARG(PCRTUINT128U,         pSrc, 1);
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+//        IEM_MC_REF_XREG_U128_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+//        IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc);
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//    }
+//    else
+//    {
+//        /*
+//         * Register, memory.
+//         */
+//        IEM_MC_BEGIN(2, 2);
+//        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+//        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+//        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+//        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+//
+//        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+//
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_XREG_U128(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+//        IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc);
+//
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//    }
+//    return VINF_SUCCESS;
+//}
+
+
+/*  Opcode VEX.0F 0x74 - invalid */
+
+/** Opcode VEX.66.0F 0x74 - vpcmpeqb Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpcmpeqb_Vx_Hx_Wx);
+//FNIEMOP_DEF(iemOp_vpcmpeqb_Vx_Hx_Wx)
+//{
+//    IEMOP_MNEMONIC(vpcmpeqb, "vpcmpeqb");
+//    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, &g_iemAImpl_pcmpeqb);
+//}
+
+/*  Opcode VEX.F3.0F 0x74 - invalid */
+/*  Opcode VEX.F2.0F 0x74 - invalid */
+
+
+/*  Opcode VEX.0F 0x75 - invalid */
+
+/** Opcode VEX.66.0F 0x75 - vpcmpeqw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpcmpeqw_Vx_Hx_Wx);
+//FNIEMOP_DEF(iemOp_vpcmpeqw_Vx_Hx_Wx)
+//{
+//    IEMOP_MNEMONIC(vpcmpeqw, "vpcmpeqw");
+//    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, &g_iemAImpl_pcmpeqw);
+//}
+
+/*  Opcode VEX.F3.0F 0x75 - invalid */
+/*  Opcode VEX.F2.0F 0x75 - invalid */
+
+
+/*  Opcode VEX.0F 0x76 - invalid */
+
+/** Opcode VEX.66.0F 0x76 - vpcmpeqd Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpcmpeqd_Vx_Hx_Wx);
+//FNIEMOP_DEF(iemOp_vpcmpeqd_Vx_Hx_Wx)
+//{
+//    IEMOP_MNEMONIC(vpcmpeqd, "vpcmpeqd");
+//    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, &g_iemAImpl_pcmpeqd);
+//}
+
+/*  Opcode VEX.F3.0F 0x76 - invalid */
+/*  Opcode VEX.F2.0F 0x76 - invalid */
+
+
+/** Opcode VEX.0F 0x77 - vzeroupperv vzeroallv */
+FNIEMOP_STUB(iemOp_vzeroupperv__vzeroallv);
+/*  Opcode VEX.66.0F 0x77 - invalid */
+/*  Opcode VEX.F3.0F 0x77 - invalid */
+/*  Opcode VEX.F2.0F 0x77 - invalid */
+
+/*  Opcode VEX.0F 0x78 - invalid */
+/*  Opcode VEX.66.0F 0x78 - invalid */
+/*  Opcode VEX.F3.0F 0x78 - invalid */
+/*  Opcode VEX.F2.0F 0x78 - invalid */
+
+/*  Opcode VEX.0F 0x79 - invalid */
+/*  Opcode VEX.66.0F 0x79 - invalid */
+/*  Opcode VEX.F3.0F 0x79 - invalid */
+/*  Opcode VEX.F2.0F 0x79 - invalid */
+
+/*  Opcode VEX.0F 0x7a - invalid */
+/*  Opcode VEX.66.0F 0x7a - invalid */
+/*  Opcode VEX.F3.0F 0x7a - invalid */
+/*  Opcode VEX.F2.0F 0x7a - invalid */
+
+/*  Opcode VEX.0F 0x7b - invalid */
+/*  Opcode VEX.66.0F 0x7b - invalid */
+/*  Opcode VEX.F3.0F 0x7b - invalid */
+/*  Opcode VEX.F2.0F 0x7b - invalid */
+
+/*  Opcode VEX.0F 0x7c - invalid */
+/** Opcode VEX.66.0F 0x7c - vhaddpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vhaddpd_Vpd_Hpd_Wpd);
+/*  Opcode VEX.F3.0F 0x7c - invalid */
+/** Opcode VEX.F2.0F 0x7c - vhaddps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vhaddps_Vps_Hps_Wps);
+
+/*  Opcode VEX.0F 0x7d - invalid */
+/** Opcode VEX.66.0F 0x7d - vhsubpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vhsubpd_Vpd_Hpd_Wpd);
+/*  Opcode VEX.F3.0F 0x7d - invalid */
+/** Opcode VEX.F2.0F 0x7d - vhsubps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vhsubps_Vps_Hps_Wps);
+
+
+/*  Opcode VEX.0F 0x7e - invalid */
+
+FNIEMOP_DEF(iemOp_vmovd_q_Ey_Vy)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x7e
+         * @opcodesub   rex.w=1
+         * @oppfx       0x66
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdint_datamov
+         * @opxcpttype  5
+         * @optest      64-bit / op1=1 op2=2   -> op1=2
+         * @optest      64-bit / op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(VEX_MR, VMOVQ, vmovq, Eq_WO, Vq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_VEX_L_ZERO);
+        if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /* greg64, XMM */
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U64(u64Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_GREG_U64((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u64Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* [mem64], XMM */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U64(u64Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u64Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x7e
+         * @opcodesub   rex.w=0
+         * @oppfx       0x66
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdint_datamov
+         * @opxcpttype  5
+         * @opfunction  iemOp_vmovd_q_Vy_Ey
+         * @optest      op1=1 op2=2   -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(VEX_MR, VMOVD, vmovd, Ed_WO, Vd, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_VEX_L_ZERO);
+        if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        {
+            /* greg32, XMM */
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U32(u32Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_GREG_U32((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB, u32Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* [mem32], XMM */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U32(u32Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u32Tmp);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opcode      0x7e
+ * @oppfx       0xf3
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  none
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovq_Vq_Wq)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVQ, vmovq, Vq_WO, Wq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_COPY_YREG_U64_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg,
+                                      (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U64_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+
+}
+/*  Opcode VEX.F2.0F 0x7e - invalid */
+
+
+/*  Opcode VEX.0F 0x7f - invalid */
+
+/**
+ * @opcode      0x7f
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdint_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2   -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovdqa_Wx_Vx)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVDQA, vmovdqa, Wx_WO, Vx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                           ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                           ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * Register, memory128.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U128(u128Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u128Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory256.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U, u256Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U256(u256Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u256Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opcode      0x7f
+ * @oppfx       0xf3
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdint_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2   -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovdqu_Wx_Vx)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVDQU, vmovdqu, Wx_WO, Vx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                           ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                           ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * Register, memory128.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U128(u128Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u128Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory256.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U, u256Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U256(u256Tmp, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U256(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u256Tmp);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/*  Opcode VEX.F2.0F 0x7f - invalid */
+
+
+/*  Opcode VEX.0F 0x80 - invalid  */
+/*  Opcode VEX.0F 0x81 - invalid  */
+/*  Opcode VEX.0F 0x82 - invalid  */
+/*  Opcode VEX.0F 0x83 - invalid  */
+/*  Opcode VEX.0F 0x84 - invalid  */
+/*  Opcode VEX.0F 0x85 - invalid  */
+/*  Opcode VEX.0F 0x86 - invalid  */
+/*  Opcode VEX.0F 0x87 - invalid  */
+/*  Opcode VEX.0F 0x88 - invalid  */
+/*  Opcode VEX.0F 0x89 - invalid  */
+/*  Opcode VEX.0F 0x8a - invalid  */
+/*  Opcode VEX.0F 0x8b - invalid  */
+/*  Opcode VEX.0F 0x8c - invalid  */
+/*  Opcode VEX.0F 0x8d - invalid  */
+/*  Opcode VEX.0F 0x8e - invalid  */
+/*  Opcode VEX.0F 0x8f - invalid  */
+/*  Opcode VEX.0F 0x90 - invalid  */
+/*  Opcode VEX.0F 0x91 - invalid  */
+/*  Opcode VEX.0F 0x92 - invalid  */
+/*  Opcode VEX.0F 0x93 - invalid  */
+/*  Opcode VEX.0F 0x94 - invalid  */
+/*  Opcode VEX.0F 0x95 - invalid  */
+/*  Opcode VEX.0F 0x96 - invalid  */
+/*  Opcode VEX.0F 0x97 - invalid  */
+/*  Opcode VEX.0F 0x98 - invalid  */
+/*  Opcode VEX.0F 0x99 - invalid  */
+/*  Opcode VEX.0F 0x9a - invalid  */
+/*  Opcode VEX.0F 0x9b - invalid  */
+/*  Opcode VEX.0F 0x9c - invalid  */
+/*  Opcode VEX.0F 0x9d - invalid  */
+/*  Opcode VEX.0F 0x9e - invalid  */
+/*  Opcode VEX.0F 0x9f - invalid  */
+/*  Opcode VEX.0F 0xa0 - invalid  */
+/*  Opcode VEX.0F 0xa1 - invalid  */
+/*  Opcode VEX.0F 0xa2 - invalid  */
+/*  Opcode VEX.0F 0xa3 - invalid  */
+/*  Opcode VEX.0F 0xa4 - invalid  */
+/*  Opcode VEX.0F 0xa5 - invalid  */
+/*  Opcode VEX.0F 0xa6 - invalid  */
+/*  Opcode VEX.0F 0xa7 - invalid  */
+/*  Opcode VEX.0F 0xa8 - invalid  */
+/*  Opcode VEX.0F 0xa9 - invalid  */
+/*  Opcode VEX.0F 0xaa - invalid  */
+/*  Opcode VEX.0F 0xab - invalid  */
+/*  Opcode VEX.0F 0xac - invalid  */
+/*  Opcode VEX.0F 0xad - invalid  */
+
+
+/*  Opcode VEX.0F 0xae mem/0 - invalid. */
+/*  Opcode VEX.0F 0xae mem/1 - invalid. */
+
+/**
+ * @ opmaps      grp15
+ * @ opcode      !11/2
+ * @ oppfx       none
+ * @ opcpuid     sse
+ * @ opgroup     og_sse_mxcsrsm
+ * @ opxcpttype  5
+ * @ optest      op1=0      -> mxcsr=0
+ * @ optest      op1=0x2083 -> mxcsr=0x2083
+ * @ optest      op1=0xfffffffe -> value.xcpt=0xd
+ * @ optest      op1=0x2083 cr0|=ts -> value.xcpt=0x7
+ * @ optest      op1=0x2083 cr0|=em -> value.xcpt=0x6
+ * @ optest      op1=0x2083 cr0|=mp -> mxcsr=0x2083
+ * @ optest      op1=0x2083 cr4&~=osfxsr -> value.xcpt=0x6
+ * @ optest      op1=0x2083 cr0|=ts,em -> value.xcpt=0x6
+ * @ optest      op1=0x2083 cr0|=em cr4&~=osfxsr -> value.xcpt=0x6
+ * @ optest      op1=0x2083 cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x6
+ * @ optest      op1=0x2083 cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x6
+ */
+FNIEMOP_STUB_1(iemOp_VGrp15_vldmxcsr, uint8_t, bRm);
+//FNIEMOP_DEF_1(iemOp_VGrp15_vldmxcsr, uint8_t, bRm)
+//{
+//    IEMOP_MNEMONIC1(M_MEM, VLDMXCSR, vldmxcsr, MdRO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+//    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse)
+//        return IEMOP_RAISE_INVALID_OPCODE();
+//
+//    IEM_MC_BEGIN(2, 0);
+//    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+//    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+//    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+//    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//    IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+//    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+//    IEM_MC_CALL_CIMPL_2(iemCImpl_ldmxcsr, iEffSeg, GCPtrEff);
+//    IEM_MC_END();
+//    return VINF_SUCCESS;
+//}
+
+
+/**
+ * @opmaps      vexgrp15
+ * @opcode      !11/3
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_mxcsrsm
+ * @opxcpttype  5
+ * @optest      mxcsr=0      -> op1=0
+ * @optest      mxcsr=0x2083 -> op1=0x2083
+ * @optest      mxcsr=0x2084 cr0|=ts -> value.xcpt=0x7
+ * @optest      !amd / mxcsr=0x2085 cr0|=em -> op1=0x2085
+ * @optest       amd / mxcsr=0x2085 cr0|=em -> value.xcpt=0x6
+ * @optest      mxcsr=0x2086 cr0|=mp -> op1=0x2086
+ * @optest      mxcsr=0x2087 cr4&~=osfxsr -> op1=0x2087
+ * @optest      mxcsr=0x208f cr4&~=osxsave -> value.xcpt=0x6
+ * @optest      mxcsr=0x2087 cr4&~=osfxsr,osxsave -> value.xcpt=0x6
+ * @optest      !amd / mxcsr=0x2088 cr0|=ts,em -> value.xcpt=0x7
+ * @optest      amd  / mxcsr=0x2088 cr0|=ts,em -> value.xcpt=0x6
+ * @optest      !amd / mxcsr=0x2089 cr0|=em cr4&~=osfxsr -> op1=0x2089
+ * @optest      amd  / mxcsr=0x2089 cr0|=em cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      !amd / mxcsr=0x208a cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x7
+ * @optest      amd  / mxcsr=0x208a cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      !amd / mxcsr=0x208b cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x7
+ * @optest      amd  / mxcsr=0x208b cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      !amd / mxcsr=0x208c xcr0&~=all_avx -> value.xcpt=0x6
+ * @optest      amd  / mxcsr=0x208c xcr0&~=all_avx -> op1=0x208c
+ * @optest      !amd / mxcsr=0x208d xcr0&~=all_avx_sse -> value.xcpt=0x6
+ * @optest      amd  / mxcsr=0x208d xcr0&~=all_avx_sse -> op1=0x208d
+ * @optest      !amd / mxcsr=0x208e xcr0&~=all_avx cr0|=ts -> value.xcpt=0x6
+ * @optest      amd  / mxcsr=0x208e xcr0&~=all_avx cr0|=ts -> value.xcpt=0x7
+ * @optest      mxcsr=0x2082 cr0|=ts cr4&~=osxsave -> value.xcpt=0x6
+ * @optest      mxcsr=0x2081 xcr0&~=all_avx cr0|=ts cr4&~=osxsave
+ *              -> value.xcpt=0x6
+ * @remarks     AMD Jaguar CPU (f0x16,m0,s1) \#UD when CR0.EM is set.  It also
+ *              doesn't seem to check XCR0[2:1] != 11b.  This does not match the
+ *              APMv4 rev 3.17 page 509.
+ * @todo        Test this instruction on AMD Ryzen.
+ */
+FNIEMOP_DEF_1(iemOp_VGrp15_vstmxcsr,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(VEX_M_MEM, VSTMXCSR, vstmxcsr, Md_WO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+    IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_vstmxcsr, iEffSeg, GCPtrEff);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+/*  Opcode VEX.0F 0xae mem/4 - invalid. */
+/*  Opcode VEX.0F 0xae mem/5 - invalid. */
+/*  Opcode VEX.0F 0xae mem/6 - invalid. */
+/*  Opcode VEX.0F 0xae mem/7 - invalid. */
+
+/*  Opcode VEX.0F 0xae 11b/0 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/1 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/2 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/3 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/4 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/5 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/6 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/7 - invalid. */
+
+/**
+ * Vex group 15 jump table for memory variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnVexGroup15MemReg[] =
+{   /* pfx:  none,                          066h,                           0f3h,                           0f2h */
+    /* /0 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /1 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /2 */ iemOp_VGrp15_vldmxcsr,         iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /3 */ iemOp_VGrp15_vstmxcsr,         iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /4 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /5 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /6 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /7 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexGroup15MemReg) == 8*4);
+
+
+/** Opcode vex. 0xae. */
+FNIEMOP_DEF(iemOp_VGrp15)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+        /* register, register */
+        return FNIEMOP_CALL_1(iemOp_InvalidWithRM, bRm);
+
+    /* memory, register */
+    return FNIEMOP_CALL_1(g_apfnVexGroup15MemReg[ ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) * 4
+                                                 + pVCpu->iem.s.idxPrefix], bRm);
+}
+
+
+/*  Opcode VEX.0F 0xaf - invalid. */
+
+/*  Opcode VEX.0F 0xb0 - invalid. */
+/*  Opcode VEX.0F 0xb1 - invalid. */
+/*  Opcode VEX.0F 0xb2 - invalid. */
+/*  Opcode VEX.0F 0xb2 - invalid. */
+/*  Opcode VEX.0F 0xb3 - invalid. */
+/*  Opcode VEX.0F 0xb4 - invalid. */
+/*  Opcode VEX.0F 0xb5 - invalid. */
+/*  Opcode VEX.0F 0xb6 - invalid. */
+/*  Opcode VEX.0F 0xb7 - invalid. */
+/*  Opcode VEX.0F 0xb8 - invalid. */
+/*  Opcode VEX.0F 0xb9 - invalid. */
+/*  Opcode VEX.0F 0xba - invalid. */
+/*  Opcode VEX.0F 0xbb - invalid. */
+/*  Opcode VEX.0F 0xbc - invalid. */
+/*  Opcode VEX.0F 0xbd - invalid. */
+/*  Opcode VEX.0F 0xbe - invalid. */
+/*  Opcode VEX.0F 0xbf - invalid. */
+
+/*  Opcode VEX.0F 0xc0 - invalid. */
+/*  Opcode VEX.66.0F 0xc0 - invalid. */
+/*  Opcode VEX.F3.0F 0xc0 - invalid. */
+/*  Opcode VEX.F2.0F 0xc0 - invalid. */
+
+/*  Opcode VEX.0F 0xc1 - invalid. */
+/*  Opcode VEX.66.0F 0xc1 - invalid. */
+/*  Opcode VEX.F3.0F 0xc1 - invalid. */
+/*  Opcode VEX.F2.0F 0xc1 - invalid. */
+
+/** Opcode VEX.0F 0xc2 - vcmpps Vps,Hps,Wps,Ib */
+FNIEMOP_STUB(iemOp_vcmpps_Vps_Hps_Wps_Ib);
+/** Opcode VEX.66.0F 0xc2 - vcmppd Vpd,Hpd,Wpd,Ib */
+FNIEMOP_STUB(iemOp_vcmppd_Vpd_Hpd_Wpd_Ib);
+/** Opcode VEX.F3.0F 0xc2 - vcmpss Vss,Hss,Wss,Ib */
+FNIEMOP_STUB(iemOp_vcmpss_Vss_Hss_Wss_Ib);
+/** Opcode VEX.F2.0F 0xc2 - vcmpsd Vsd,Hsd,Wsd,Ib */
+FNIEMOP_STUB(iemOp_vcmpsd_Vsd_Hsd_Wsd_Ib);
+
+/*  Opcode VEX.0F 0xc3 - invalid */
+/*  Opcode VEX.66.0F 0xc3 - invalid */
+/*  Opcode VEX.F3.0F 0xc3 - invalid */
+/*  Opcode VEX.F2.0F 0xc3 - invalid */
+
+/*  Opcode VEX.0F 0xc4 - invalid */
+/** Opcode VEX.66.0F 0xc4 - vpinsrw Vdq,Hdq,Ry/Mw,Ib */
+FNIEMOP_STUB(iemOp_vpinsrw_Vdq_Hdq_RyMw_Ib);
+/*  Opcode VEX.F3.0F 0xc4 - invalid */
+/*  Opcode VEX.F2.0F 0xc4 - invalid */
+
+/*  Opcode VEX.0F 0xc5 - invlid */
+/** Opcode VEX.66.0F 0xc5 - vpextrw Gd, Udq, Ib */
+FNIEMOP_STUB(iemOp_vpextrw_Gd_Udq_Ib);
+/*  Opcode VEX.F3.0F 0xc5 - invalid */
+/*  Opcode VEX.F2.0F 0xc5 - invalid */
+
+/** Opcode VEX.0F 0xc6 - vshufps Vps,Hps,Wps,Ib */
+FNIEMOP_STUB(iemOp_vshufps_Vps_Hps_Wps_Ib);
+/** Opcode VEX.66.0F 0xc6 - vshufpd Vpd,Hpd,Wpd,Ib */
+FNIEMOP_STUB(iemOp_vshufpd_Vpd_Hpd_Wpd_Ib);
+/*  Opcode VEX.F3.0F 0xc6 - invalid */
+/*  Opcode VEX.F2.0F 0xc6 - invalid */
+
+/*  Opcode VEX.0F 0xc7 - invalid */
+/*  Opcode VEX.66.0F 0xc7 - invalid */
+/*  Opcode VEX.F3.0F 0xc7 - invalid */
+/*  Opcode VEX.F2.0F 0xc7 - invalid */
+
+/*  Opcode VEX.0F 0xc8 - invalid */
+/*  Opcode VEX.0F 0xc9 - invalid */
+/*  Opcode VEX.0F 0xca - invalid */
+/*  Opcode VEX.0F 0xcb - invalid */
+/*  Opcode VEX.0F 0xcc - invalid */
+/*  Opcode VEX.0F 0xcd - invalid */
+/*  Opcode VEX.0F 0xce - invalid */
+/*  Opcode VEX.0F 0xcf - invalid */
+
+
+/*  Opcode VEX.0F 0xd0 - invalid */
+/** Opcode VEX.66.0F 0xd0 - vaddsubpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vaddsubpd_Vpd_Hpd_Wpd);
+/*  Opcode VEX.F3.0F 0xd0 - invalid */
+/** Opcode VEX.F2.0F 0xd0 - vaddsubps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vaddsubps_Vps_Hps_Wps);
+
+/*  Opcode VEX.0F 0xd1 - invalid */
+/** Opcode VEX.66.0F 0xd1 - vpsrlw Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpsrlw_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xd1 - invalid */
+/*  Opcode VEX.F2.0F 0xd1 - invalid */
+
+/*  Opcode VEX.0F 0xd2 - invalid */
+/** Opcode VEX.66.0F 0xd2 - vpsrld Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsrld_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xd2 - invalid */
+/*  Opcode VEX.F2.0F 0xd2 - invalid */
+
+/*  Opcode VEX.0F 0xd3 - invalid */
+/** Opcode VEX.66.0F 0xd3 - vpsrlq Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsrlq_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xd3 - invalid */
+/*  Opcode VEX.F2.0F 0xd3 - invalid */
+
+/*  Opcode VEX.0F 0xd4 - invalid */
+/** Opcode VEX.66.0F 0xd4 - vpaddq Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpaddq_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xd4 - invalid */
+/*  Opcode VEX.F2.0F 0xd4 - invalid */
+
+/*  Opcode VEX.0F 0xd5 - invalid */
+/** Opcode VEX.66.0F 0xd5 - vpmullw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpmullw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xd5 - invalid */
+/*  Opcode VEX.F2.0F 0xd5 - invalid */
+
+/*  Opcode VEX.0F 0xd6 - invalid */
+
+/**
+ * @opcode      0xd6
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  none
+ * @optest      op1=-1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovq_Wq_Vq)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVQ, vmovq, Wq_WO, Vq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_COPY_YREG_U64_ZX_VLMAX((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB,
+                                      ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U64(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP();
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+/*  Opcode VEX.F3.0F 0xd6 - invalid */
+/*  Opcode VEX.F2.0F 0xd6 - invalid */
+
+
+/*  Opcode VEX.0F 0xd7 - invalid */
+
+/** Opcode VEX.66.0F 0xd7 -  */
+FNIEMOP_STUB(iemOp_vpmovmskb_Gd_Ux);
+//FNIEMOP_DEF(iemOp_vpmovmskb_Gd_Ux)
+//{
+//    /* Note! Taking the lazy approch here wrt the high 32-bits of the GREG. */
+//    /** @todo testcase: Check that the instruction implicitly clears the high
+//     *        bits in 64-bit mode.  The REX.W is first necessary when VLMAX > 256
+//     *        and opcode modifications are made to work with the whole width (not
+//     *        just 128). */
+//    IEMOP_MNEMONIC(vpmovmskb_Gd_Nq, "vpmovmskb Gd, Ux");
+//    /* Docs says register only. */
+//    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+//    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT)) /** @todo test that this is registers only. */
+//    {
+//        IEMOP_HLP_DECODED_NL_2(OP_PMOVMSKB, IEMOPFORM_RM_REG, OP_PARM_Gd, OP_PARM_Vdq, DISOPTYPE_SSE | DISOPTYPE_HARMLESS);
+//        IEM_MC_BEGIN(2, 0);
+//        IEM_MC_ARG(uint64_t *,           pDst, 0);
+//        IEM_MC_ARG(PCRTUINT128U,         pSrc, 1);
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_GREG_U64(pDst, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+//        IEM_MC_REF_XREG_U128_CONST(pSrc, (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB);
+//        IEM_MC_CALL_SSE_AIMPL_2(iemAImpl_pmovmskb_u128, pDst, pSrc);
+//        IEM_MC_ADVANCE_RIP();
+//        IEM_MC_END();
+//        return VINF_SUCCESS;
+//    }
+//    return IEMOP_RAISE_INVALID_OPCODE();
+//}
+
+/*  Opcode VEX.F3.0F 0xd7 - invalid */
+/*  Opcode VEX.F2.0F 0xd7 - invalid */
+
+
+/*  Opcode VEX.0F 0xd8 - invalid */
+/** Opcode VEX.66.0F 0xd8 - vpsubusb Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpsubusb_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xd8 - invalid */
+/*  Opcode VEX.F2.0F 0xd8 - invalid */
+
+/*  Opcode VEX.0F 0xd9 - invalid */
+/** Opcode VEX.66.0F 0xd9 - vpsubusw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsubusw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xd9 - invalid */
+/*  Opcode VEX.F2.0F 0xd9 - invalid */
+
+/*  Opcode VEX.0F 0xda - invalid */
+/** Opcode VEX.66.0F 0xda - vpminub Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpminub_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xda - invalid */
+/*  Opcode VEX.F2.0F 0xda - invalid */
+
+/*  Opcode VEX.0F 0xdb - invalid */
+/** Opcode VEX.66.0F 0xdb - vpand Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpand_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xdb - invalid */
+/*  Opcode VEX.F2.0F 0xdb - invalid */
+
+/*  Opcode VEX.0F 0xdc - invalid */
+/** Opcode VEX.66.0F 0xdc - vpaddusb Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpaddusb_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xdc - invalid */
+/*  Opcode VEX.F2.0F 0xdc - invalid */
+
+/*  Opcode VEX.0F 0xdd - invalid */
+/** Opcode VEX.66.0F 0xdd - vpaddusw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpaddusw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xdd - invalid */
+/*  Opcode VEX.F2.0F 0xdd - invalid */
+
+/*  Opcode VEX.0F 0xde - invalid */
+/** Opcode VEX.66.0F 0xde - vpmaxub Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpmaxub_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xde - invalid */
+/*  Opcode VEX.F2.0F 0xde - invalid */
+
+/*  Opcode VEX.0F 0xdf - invalid */
+/** Opcode VEX.66.0F 0xdf - vpandn Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpandn_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xdf - invalid */
+/*  Opcode VEX.F2.0F 0xdf - invalid */
+
+/*  Opcode VEX.0F 0xe0 - invalid */
+/** Opcode VEX.66.0F 0xe0 - vpavgb Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpavgb_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xe0 - invalid */
+/*  Opcode VEX.F2.0F 0xe0 - invalid */
+
+/*  Opcode VEX.0F 0xe1 - invalid */
+/** Opcode VEX.66.0F 0xe1 - vpsraw Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpsraw_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xe1 - invalid */
+/*  Opcode VEX.F2.0F 0xe1 - invalid */
+
+/*  Opcode VEX.0F 0xe2 - invalid */
+/** Opcode VEX.66.0F 0xe2 - vpsrad Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsrad_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xe2 - invalid */
+/*  Opcode VEX.F2.0F 0xe2 - invalid */
+
+/*  Opcode VEX.0F 0xe3 - invalid */
+/** Opcode VEX.66.0F 0xe3 - vpavgw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpavgw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xe3 - invalid */
+/*  Opcode VEX.F2.0F 0xe3 - invalid */
+
+/*  Opcode VEX.0F 0xe4 - invalid */
+/** Opcode VEX.66.0F 0xe4 - vpmulhuw Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpmulhuw_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xe4 - invalid */
+/*  Opcode VEX.F2.0F 0xe4 - invalid */
+
+/*  Opcode VEX.0F 0xe5 - invalid */
+/** Opcode VEX.66.0F 0xe5 - vpmulhw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpmulhw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xe5 - invalid */
+/*  Opcode VEX.F2.0F 0xe5 - invalid */
+
+/*  Opcode VEX.0F 0xe6 - invalid */
+/** Opcode VEX.66.0F 0xe6 - vcvttpd2dq Vx, Wpd */
+FNIEMOP_STUB(iemOp_vcvttpd2dq_Vx_Wpd);
+/** Opcode VEX.F3.0F 0xe6 - vcvtdq2pd Vx, Wpd */
+FNIEMOP_STUB(iemOp_vcvtdq2pd_Vx_Wpd);
+/** Opcode VEX.F2.0F 0xe6 - vcvtpd2dq Vx, Wpd */
+FNIEMOP_STUB(iemOp_vcvtpd2dq_Vx_Wpd);
+
+
+/* Opcode VEX.0F 0xe7 - invalid */
+
+/**
+ * @opcode      0xe7
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_cachect
+ * @opxcpttype  1
+ * @optest      op1=-1 op2=2  -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovntdq_Mx_Vx)
+{
+    IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVNTDQ, vmovntdq, Mx_WO, Vx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            /*
+             * 128-bit: Memory, register.
+             */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U128(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /*
+             * 256-bit: Memory, register.
+             */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U256(uSrc, ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg);
+            IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        return VINF_SUCCESS;
+    }
+    /**
+     * @opdone
+     * @opmnemonic  udvex660fe7reg
+     * @opcode      0xe7
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/*  Opcode VEX.F3.0F 0xe7 - invalid */
+/*  Opcode VEX.F2.0F 0xe7 - invalid */
+
+
+/*  Opcode VEX.0F 0xe8 - invalid */
+/** Opcode VEX.66.0F 0xe8 - vpsubsb Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpsubsb_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xe8 - invalid */
+/*  Opcode VEX.F2.0F 0xe8 - invalid */
+
+/*  Opcode VEX.0F 0xe9 - invalid */
+/** Opcode VEX.66.0F 0xe9 - vpsubsw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsubsw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xe9 - invalid */
+/*  Opcode VEX.F2.0F 0xe9 - invalid */
+
+/*  Opcode VEX.0F 0xea - invalid */
+/** Opcode VEX.66.0F 0xea - vpminsw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpminsw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xea - invalid */
+/*  Opcode VEX.F2.0F 0xea - invalid */
+
+/*  Opcode VEX.0F 0xeb - invalid */
+/** Opcode VEX.66.0F 0xeb - vpor Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpor_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xeb - invalid */
+/*  Opcode VEX.F2.0F 0xeb - invalid */
+
+/*  Opcode VEX.0F 0xec - invalid */
+/** Opcode VEX.66.0F 0xec - vpaddsb Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpaddsb_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xec - invalid */
+/*  Opcode VEX.F2.0F 0xec - invalid */
+
+/*  Opcode VEX.0F 0xed - invalid */
+/** Opcode VEX.66.0F 0xed - vpaddsw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpaddsw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xed - invalid */
+/*  Opcode VEX.F2.0F 0xed - invalid */
+
+/*  Opcode VEX.0F 0xee - invalid */
+/** Opcode VEX.66.0F 0xee - vpmaxsw Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpmaxsw_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xee - invalid */
+/*  Opcode VEX.F2.0F 0xee - invalid */
+
+
+/*  Opcode VEX.0F 0xef - invalid */
+
+/** Opcode VEX.66.0F 0xef - vpxor Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpxor_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC(vpxor, "vpxor");
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, &g_iemAImpl_pxor);
+}
+
+/*  Opcode VEX.F3.0F 0xef - invalid */
+/*  Opcode VEX.F2.0F 0xef - invalid */
+
+/*  Opcode VEX.0F 0xf0 - invalid */
+/*  Opcode VEX.66.0F 0xf0 - invalid */
+/** Opcode VEX.F2.0F 0xf0 - vlddqu Vx, Mx */
+FNIEMOP_STUB(iemOp_vlddqu_Vx_Mx);
+
+/*  Opcode VEX.0F 0xf1 - invalid */
+/** Opcode VEX.66.0F 0xf1 - vpsllw Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpsllw_Vx_Hx_W);
+/*  Opcode VEX.F2.0F 0xf1 - invalid */
+
+/*  Opcode VEX.0F 0xf2 - invalid */
+/** Opcode VEX.66.0F 0xf2 - vpslld Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpslld_Vx_Hx_Wx);
+/*  Opcode VEX.F2.0F 0xf2 - invalid */
+
+/*  Opcode VEX.0F 0xf3 - invalid */
+/** Opcode VEX.66.0F 0xf3 - vpsllq Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsllq_Vx_Hx_Wx);
+/*  Opcode VEX.F2.0F 0xf3 - invalid */
+
+/*  Opcode VEX.0F 0xf4 - invalid */
+/** Opcode VEX.66.0F 0xf4 - vpmuludq Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpmuludq_Vx_Hx_W);
+/*  Opcode VEX.F2.0F 0xf4 - invalid */
+
+/*  Opcode VEX.0F 0xf5 - invalid */
+/** Opcode VEX.66.0F 0xf5 - vpmaddwd Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpmaddwd_Vx_Hx_Wx);
+/*  Opcode VEX.F2.0F 0xf5 - invalid */
+
+/*  Opcode VEX.0F 0xf6 - invalid */
+/** Opcode VEX.66.0F 0xf6 - vpsadbw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsadbw_Vx_Hx_Wx);
+/*  Opcode VEX.F2.0F 0xf6 - invalid */
+
+/*  Opcode VEX.0F 0xf7 - invalid */
+/** Opcode VEX.66.0F 0xf7 - vmaskmovdqu Vdq, Udq */
+FNIEMOP_STUB(iemOp_vmaskmovdqu_Vdq_Udq);
+/*  Opcode VEX.F2.0F 0xf7 - invalid */
+
+/*  Opcode VEX.0F 0xf8 - invalid */
+/** Opcode VEX.66.0F 0xf8 - vpsubb Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpsubb_Vx_Hx_W);
+/*  Opcode VEX.F2.0F 0xf8 - invalid */
+
+/*  Opcode VEX.0F 0xf9 - invalid */
+/** Opcode VEX.66.0F 0xf9 - vpsubw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsubw_Vx_Hx_Wx);
+/*  Opcode VEX.F2.0F 0xf9 - invalid */
+
+/*  Opcode VEX.0F 0xfa - invalid */
+/** Opcode VEX.66.0F 0xfa - vpsubd Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsubd_Vx_Hx_Wx);
+/*  Opcode VEX.F2.0F 0xfa - invalid */
+
+/*  Opcode VEX.0F 0xfb - invalid */
+/** Opcode VEX.66.0F 0xfb - vpsubq Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpsubq_Vx_Hx_W);
+/*  Opcode VEX.F2.0F 0xfb - invalid */
+
+/*  Opcode VEX.0F 0xfc - invalid */
+/** Opcode VEX.66.0F 0xfc - vpaddb Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpaddb_Vx_Hx_Wx);
+/*  Opcode VEX.F2.0F 0xfc - invalid */
+
+/*  Opcode VEX.0F 0xfd - invalid */
+/** Opcode VEX.66.0F 0xfd - vpaddw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpaddw_Vx_Hx_Wx);
+/*  Opcode VEX.F2.0F 0xfd - invalid */
+
+/*  Opcode VEX.0F 0xfe - invalid */
+/** Opcode VEX.66.0F 0xfe - vpaddd Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpaddd_Vx_Hx_W);
+/*  Opcode VEX.F2.0F 0xfe - invalid */
+
+
+/** Opcode **** 0x0f 0xff - UD0 */
+FNIEMOP_DEF(iemOp_vud0)
+{
+    IEMOP_MNEMONIC(vud0, "vud0");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+        RTGCPTR      GCPtrEff;
+        VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+#endif
+        IEMOP_HLP_DONE_DECODING();
+    }
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+
+/**
+ * VEX opcode map \#1.
+ *
+ * @sa  g_apfnTwoByteMap
+ */
+IEM_STATIC const PFNIEMOP g_apfnVexMap1[] =
+{
+    /*          no prefix,                  066h prefix                 f3h prefix,                 f2h prefix */
+    /* 0x00 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x01 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x02 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x03 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x04 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x05 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x06 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x07 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x08 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x09 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0b */  IEMOP_X4(iemOp_vud2), /* ?? */
+    /* 0x0c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x10 */  iemOp_vmovups_Vps_Wps,      iemOp_vmovupd_Vpd_Wpd,      iemOp_vmovss_Vss_Hss_Wss,   iemOp_vmovsd_Vsd_Hsd_Wsd,
+    /* 0x11 */  iemOp_vmovups_Wps_Vps,      iemOp_vmovupd_Wpd_Vpd,      iemOp_vmovss_Wss_Hss_Vss,   iemOp_vmovsd_Wsd_Hsd_Vsd,
+    /* 0x12 */  iemOp_vmovlps_Vq_Hq_Mq__vmovhlps, iemOp_vmovlpd_Vq_Hq_Mq, iemOp_vmovsldup_Vx_Wx,    iemOp_vmovddup_Vx_Wx,
+    /* 0x13 */  iemOp_vmovlps_Mq_Vq,        iemOp_vmovlpd_Mq_Vq,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x14 */  iemOp_vunpcklps_Vx_Hx_Wx,   iemOp_vunpcklpd_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x15 */  iemOp_vunpckhps_Vx_Hx_Wx,   iemOp_vunpckhpd_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x16 */  iemOp_vmovhpsv1_Vdq_Hq_Mq__vmovlhps_Vdq_Hq_Uq, iemOp_vmovhpdv1_Vdq_Hq_Mq, iemOp_vmovshdup_Vx_Wx, iemOp_InvalidNeedRM,
+    /* 0x17 */  iemOp_vmovhpsv1_Mq_Vq,      iemOp_vmovhpdv1_Mq_Vq,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x18 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x19 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x20 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x21 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x22 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x23 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x24 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x25 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x26 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x27 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x28 */  iemOp_vmovaps_Vps_Wps,      iemOp_vmovapd_Vpd_Wpd,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x29 */  iemOp_vmovaps_Wps_Vps,      iemOp_vmovapd_Wpd_Vpd,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2a */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_vcvtsi2ss_Vss_Hss_Ey, iemOp_vcvtsi2sd_Vsd_Hsd_Ey,
+    /* 0x2b */  iemOp_vmovntps_Mps_Vps,     iemOp_vmovntpd_Mpd_Vpd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2c */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_vcvttss2si_Gy_Wss,    iemOp_vcvttsd2si_Gy_Wsd,
+    /* 0x2d */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_vcvtss2si_Gy_Wss,     iemOp_vcvtsd2si_Gy_Wsd,
+    /* 0x2e */  iemOp_vucomiss_Vss_Wss,     iemOp_vucomisd_Vsd_Wsd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2f */  iemOp_vcomiss_Vss_Wss,      iemOp_vcomisd_Vsd_Wsd,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x30 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x31 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x32 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x33 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x34 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x35 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x36 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x37 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x38 */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x39 */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x3a */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x3b */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x3c */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x3d */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x3e */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x3f */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+
+    /* 0x40 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x41 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x42 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x43 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x44 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x45 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x46 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x47 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x48 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x49 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x50 */  iemOp_vmovmskps_Gy_Ups,     iemOp_vmovmskpd_Gy_Upd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x51 */  iemOp_vsqrtps_Vps_Wps,      iemOp_vsqrtpd_Vpd_Wpd,      iemOp_vsqrtss_Vss_Hss_Wss,  iemOp_vsqrtsd_Vsd_Hsd_Wsd,
+    /* 0x52 */  iemOp_vrsqrtps_Vps_Wps,     iemOp_InvalidNeedRM,        iemOp_vrsqrtss_Vss_Hss_Wss, iemOp_InvalidNeedRM,
+    /* 0x53 */  iemOp_vrcpps_Vps_Wps,       iemOp_InvalidNeedRM,        iemOp_vrcpss_Vss_Hss_Wss,   iemOp_InvalidNeedRM,
+    /* 0x54 */  iemOp_vandps_Vps_Hps_Wps,   iemOp_vandpd_Vpd_Hpd_Wpd,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x55 */  iemOp_vandnps_Vps_Hps_Wps,  iemOp_vandnpd_Vpd_Hpd_Wpd,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x56 */  iemOp_vorps_Vps_Hps_Wps,    iemOp_vorpd_Vpd_Hpd_Wpd,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x57 */  iemOp_vxorps_Vps_Hps_Wps,   iemOp_vxorpd_Vpd_Hpd_Wpd,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x58 */  iemOp_vaddps_Vps_Hps_Wps,   iemOp_vaddpd_Vpd_Hpd_Wpd,   iemOp_vaddss_Vss_Hss_Wss,   iemOp_vaddsd_Vsd_Hsd_Wsd,
+    /* 0x59 */  iemOp_vmulps_Vps_Hps_Wps,   iemOp_vmulpd_Vpd_Hpd_Wpd,   iemOp_vmulss_Vss_Hss_Wss,   iemOp_vmulsd_Vsd_Hsd_Wsd,
+    /* 0x5a */  iemOp_vcvtps2pd_Vpd_Wps,    iemOp_vcvtpd2ps_Vps_Wpd,    iemOp_vcvtss2sd_Vsd_Hx_Wss, iemOp_vcvtsd2ss_Vss_Hx_Wsd,
+    /* 0x5b */  iemOp_vcvtdq2ps_Vps_Wdq,    iemOp_vcvtps2dq_Vdq_Wps,    iemOp_vcvttps2dq_Vdq_Wps,   iemOp_InvalidNeedRM,
+    /* 0x5c */  iemOp_vsubps_Vps_Hps_Wps,   iemOp_vsubpd_Vpd_Hpd_Wpd,   iemOp_vsubss_Vss_Hss_Wss,   iemOp_vsubsd_Vsd_Hsd_Wsd,
+    /* 0x5d */  iemOp_vminps_Vps_Hps_Wps,   iemOp_vminpd_Vpd_Hpd_Wpd,   iemOp_vminss_Vss_Hss_Wss,   iemOp_vminsd_Vsd_Hsd_Wsd,
+    /* 0x5e */  iemOp_vdivps_Vps_Hps_Wps,   iemOp_vdivpd_Vpd_Hpd_Wpd,   iemOp_vdivss_Vss_Hss_Wss,   iemOp_vdivsd_Vsd_Hsd_Wsd,
+    /* 0x5f */  iemOp_vmaxps_Vps_Hps_Wps,   iemOp_vmaxpd_Vpd_Hpd_Wpd,   iemOp_vmaxss_Vss_Hss_Wss,   iemOp_vmaxsd_Vsd_Hsd_Wsd,
+
+    /* 0x60 */  iemOp_InvalidNeedRM,        iemOp_vpunpcklbw_Vx_Hx_Wx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x61 */  iemOp_InvalidNeedRM,        iemOp_vpunpcklwd_Vx_Hx_Wx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x62 */  iemOp_InvalidNeedRM,        iemOp_vpunpckldq_Vx_Hx_Wx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x63 */  iemOp_InvalidNeedRM,        iemOp_vpacksswb_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x64 */  iemOp_InvalidNeedRM,        iemOp_vpcmpgtb_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x65 */  iemOp_InvalidNeedRM,        iemOp_vpcmpgtw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x66 */  iemOp_InvalidNeedRM,        iemOp_vpcmpgtd_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x67 */  iemOp_InvalidNeedRM,        iemOp_vpackuswb_Vx_Hx_W,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x68 */  iemOp_InvalidNeedRM,        iemOp_vpunpckhbw_Vx_Hx_Wx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x69 */  iemOp_InvalidNeedRM,        iemOp_vpunpckhwd_Vx_Hx_Wx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6a */  iemOp_InvalidNeedRM,        iemOp_vpunpckhdq_Vx_Hx_W,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6b */  iemOp_InvalidNeedRM,        iemOp_vpackssdw_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6c */  iemOp_InvalidNeedRM,        iemOp_vpunpcklqdq_Vx_Hx_Wx, iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6d */  iemOp_InvalidNeedRM,        iemOp_vpunpckhqdq_Vx_Hx_W,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6e */  iemOp_InvalidNeedRM,        iemOp_vmovd_q_Vy_Ey,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6f */  iemOp_InvalidNeedRM,        iemOp_vmovdqa_Vx_Wx,        iemOp_vmovdqu_Vx_Wx,        iemOp_InvalidNeedRM,
+
+    /* 0x70 */  iemOp_InvalidNeedRM,        iemOp_vpshufd_Vx_Wx_Ib,     iemOp_vpshufhw_Vx_Wx_Ib,    iemOp_vpshuflw_Vx_Wx_Ib,
+    /* 0x71 */  iemOp_InvalidNeedRM,        iemOp_VGrp12,               iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x72 */  iemOp_InvalidNeedRM,        iemOp_VGrp13,               iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x73 */  iemOp_InvalidNeedRM,        iemOp_VGrp14,               iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x74 */  iemOp_InvalidNeedRM,        iemOp_vpcmpeqb_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x75 */  iemOp_InvalidNeedRM,        iemOp_vpcmpeqw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x76 */  iemOp_InvalidNeedRM,        iemOp_vpcmpeqd_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x77 */  iemOp_vzeroupperv__vzeroallv, iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,       iemOp_InvalidNeedRM,
+    /* 0x78 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x79 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7c */  iemOp_InvalidNeedRM,        iemOp_vhaddpd_Vpd_Hpd_Wpd,  iemOp_InvalidNeedRM,        iemOp_vhaddps_Vps_Hps_Wps,
+    /* 0x7d */  iemOp_InvalidNeedRM,        iemOp_vhsubpd_Vpd_Hpd_Wpd,  iemOp_InvalidNeedRM,        iemOp_vhsubps_Vps_Hps_Wps,
+    /* 0x7e */  iemOp_InvalidNeedRM,        iemOp_vmovd_q_Ey_Vy,        iemOp_vmovq_Vq_Wq,          iemOp_InvalidNeedRM,
+    /* 0x7f */  iemOp_InvalidNeedRM,        iemOp_vmovdqa_Wx_Vx,        iemOp_vmovdqu_Wx_Vx,        iemOp_InvalidNeedRM,
+
+    /* 0x80 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x81 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x82 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x83 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x84 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x85 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x86 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x87 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x88 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x89 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x90 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x91 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x92 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x93 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x94 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x95 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x96 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x97 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x98 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x99 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xa0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xaa */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xab */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xac */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xad */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xae */  IEMOP_X4(iemOp_VGrp15),
+    /* 0xaf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xb0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xba */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbc */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbd */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbe */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xc0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc2 */  iemOp_vcmpps_Vps_Hps_Wps_Ib, iemOp_vcmppd_Vpd_Hpd_Wpd_Ib, iemOp_vcmpss_Vss_Hss_Wss_Ib, iemOp_vcmpsd_Vsd_Hsd_Wsd_Ib,
+    /* 0xc3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc4 */  iemOp_InvalidNeedRM,        iemOp_vpinsrw_Vdq_Hdq_RyMw_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0xc5 */  iemOp_InvalidNeedRM,        iemOp_vpextrw_Gd_Udq_Ib,       iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0xc6 */  iemOp_vshufps_Vps_Hps_Wps_Ib, iemOp_vshufpd_Vpd_Hpd_Wpd_Ib, iemOp_InvalidNeedRMImm8,iemOp_InvalidNeedRMImm8,
+    /* 0xc7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xca */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xcb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xcc */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xcd */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xce */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xcf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xd0 */  iemOp_InvalidNeedRM,        iemOp_vaddsubpd_Vpd_Hpd_Wpd, iemOp_InvalidNeedRM,       iemOp_vaddsubps_Vps_Hps_Wps,
+    /* 0xd1 */  iemOp_InvalidNeedRM,        iemOp_vpsrlw_Vx_Hx_W,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd2 */  iemOp_InvalidNeedRM,        iemOp_vpsrld_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd3 */  iemOp_InvalidNeedRM,        iemOp_vpsrlq_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd4 */  iemOp_InvalidNeedRM,        iemOp_vpaddq_Vx_Hx_W,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd5 */  iemOp_InvalidNeedRM,        iemOp_vpmullw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd6 */  iemOp_InvalidNeedRM,        iemOp_vmovq_Wq_Vq,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd7 */  iemOp_InvalidNeedRM,        iemOp_vpmovmskb_Gd_Ux,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd8 */  iemOp_InvalidNeedRM,        iemOp_vpsubusb_Vx_Hx_W,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd9 */  iemOp_InvalidNeedRM,        iemOp_vpsubusw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xda */  iemOp_InvalidNeedRM,        iemOp_vpminub_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdb */  iemOp_InvalidNeedRM,        iemOp_vpand_Vx_Hx_W,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdc */  iemOp_InvalidNeedRM,        iemOp_vpaddusb_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdd */  iemOp_InvalidNeedRM,        iemOp_vpaddusw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xde */  iemOp_InvalidNeedRM,        iemOp_vpmaxub_Vx_Hx_W,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdf */  iemOp_InvalidNeedRM,        iemOp_vpandn_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0xe0 */  iemOp_InvalidNeedRM,        iemOp_vpavgb_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe1 */  iemOp_InvalidNeedRM,        iemOp_vpsraw_Vx_Hx_W,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe2 */  iemOp_InvalidNeedRM,        iemOp_vpsrad_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe3 */  iemOp_InvalidNeedRM,        iemOp_vpavgw_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe4 */  iemOp_InvalidNeedRM,        iemOp_vpmulhuw_Vx_Hx_W,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe5 */  iemOp_InvalidNeedRM,        iemOp_vpmulhw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe6 */  iemOp_InvalidNeedRM,        iemOp_vcvttpd2dq_Vx_Wpd,    iemOp_vcvtdq2pd_Vx_Wpd,     iemOp_vcvtpd2dq_Vx_Wpd,
+    /* 0xe7 */  iemOp_InvalidNeedRM,        iemOp_vmovntdq_Mx_Vx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe8 */  iemOp_InvalidNeedRM,        iemOp_vpsubsb_Vx_Hx_W,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe9 */  iemOp_InvalidNeedRM,        iemOp_vpsubsw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xea */  iemOp_InvalidNeedRM,        iemOp_vpminsw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xeb */  iemOp_InvalidNeedRM,        iemOp_vpor_Vx_Hx_W,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xec */  iemOp_InvalidNeedRM,        iemOp_vpaddsb_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xed */  iemOp_InvalidNeedRM,        iemOp_vpaddsw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xee */  iemOp_InvalidNeedRM,        iemOp_vpmaxsw_Vx_Hx_W,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xef */  iemOp_InvalidNeedRM,        iemOp_vpxor_Vx_Hx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0xf0 */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_vlddqu_Vx_Mx,
+    /* 0xf1 */  iemOp_InvalidNeedRM,        iemOp_vpsllw_Vx_Hx_W,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf2 */  iemOp_InvalidNeedRM,        iemOp_vpslld_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf3 */  iemOp_InvalidNeedRM,        iemOp_vpsllq_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf4 */  iemOp_InvalidNeedRM,        iemOp_vpmuludq_Vx_Hx_W,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf5 */  iemOp_InvalidNeedRM,        iemOp_vpmaddwd_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf6 */  iemOp_InvalidNeedRM,        iemOp_vpsadbw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf7 */  iemOp_InvalidNeedRM,        iemOp_vmaskmovdqu_Vdq_Udq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf8 */  iemOp_InvalidNeedRM,        iemOp_vpsubb_Vx_Hx_W,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf9 */  iemOp_InvalidNeedRM,        iemOp_vpsubw_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfa */  iemOp_InvalidNeedRM,        iemOp_vpsubd_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfb */  iemOp_InvalidNeedRM,        iemOp_vpsubq_Vx_Hx_W,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfc */  iemOp_InvalidNeedRM,        iemOp_vpaddb_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfd */  iemOp_InvalidNeedRM,        iemOp_vpaddw_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfe */  iemOp_InvalidNeedRM,        iemOp_vpaddd_Vx_Hx_W,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xff */  IEMOP_X4(iemOp_vud0) /* ?? */
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexMap1) == 1024);
+/** @}  */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h
new file mode 100644
index 00000000..6bf008d7
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h
@@ -0,0 +1,940 @@
+/* $Id: IEMAllInstructionsVexMap2.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation.
+ *
+ * @remarks IEMAllInstructionsThree0f38.cpp.h is a VEX mirror of this file.
+ *          Any update here is likely needed in that file too.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @name VEX Opcode Map 2
+ * @{
+ */
+
+/*  Opcode VEX.0F38 0x00 - invalid. */
+/** Opcode VEX.66.0F38 0x00. */
+FNIEMOP_STUB(iemOp_vpshufb_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x01 - invalid. */
+/** Opcode VEX.66.0F38 0x01. */
+FNIEMOP_STUB(iemOp_vphaddw_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x02 - invalid. */
+/** Opcode VEX.66.0F38 0x02. */
+FNIEMOP_STUB(iemOp_vphaddd_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x03 - invalid. */
+/** Opcode VEX.66.0F38 0x03. */
+FNIEMOP_STUB(iemOp_vphaddsw_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x04 - invalid. */
+/** Opcode VEX.66.0F38 0x04. */
+FNIEMOP_STUB(iemOp_vpmaddubsw_Vx_Hx_Wx);
+/* Opcode VEX.0F38 0x05 - invalid. */
+/** Opcode VEX.66.0F38 0x05. */
+FNIEMOP_STUB(iemOp_vphsubw_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x06 - invalid. */
+/** Opcode VEX.66.0F38 0x06. */
+FNIEMOP_STUB(iemOp_vphsubdq_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x07 - invalid. */
+/** Opcode VEX.66.0F38 0x07. */
+FNIEMOP_STUB(iemOp_vphsubsw_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x08 - invalid. */
+/** Opcode VEX.66.0F38 0x08. */
+FNIEMOP_STUB(iemOp_vpsignb_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x09 - invalid. */
+/** Opcode VEX.66.0F38 0x09. */
+FNIEMOP_STUB(iemOp_vpsignw_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x0a - invalid. */
+/** Opcode VEX.66.0F38 0x0a. */
+FNIEMOP_STUB(iemOp_vpsignd_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x0b - invalid. */
+/** Opcode VEX.66.0F38 0x0b. */
+FNIEMOP_STUB(iemOp_vpmulhrsw_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x0c - invalid. */
+/**  Opcode VEX.66.0F38 0x0c. */
+FNIEMOP_STUB(iemOp_vpermilps_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x0d - invalid. */
+/**  Opcode VEX.66.0F38 0x0d. */
+FNIEMOP_STUB(iemOp_vpermilpd_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x0e - invalid. */
+/**  Opcode VEX.66.0F38 0x0e. */
+FNIEMOP_STUB(iemOp_vtestps_Vx_Wx);
+/*  Opcode VEX.0F38 0x0f - invalid. */
+/**  Opcode VEX.66.0F38 0x0f. */
+FNIEMOP_STUB(iemOp_vtestpd_Vx_Wx);
+
+
+/*  Opcode VEX.0F38 0x10 - invalid */
+/*  Opcode VEX.66.0F38 0x10 - invalid (legacy only). */
+/*  Opcode VEX.0F38 0x11 - invalid */
+/*  Opcode VEX.66.0F38 0x11 - invalid */
+/*  Opcode VEX.0F38 0x12 - invalid */
+/*  Opcode VEX.66.0F38 0x12 - invalid */
+/*  Opcode VEX.0F38 0x13 - invalid */
+/*  Opcode VEX.66.0F38 0x13 - invalid (vex only). */
+/*  Opcode VEX.0F38 0x14 - invalid */
+/*  Opcode VEX.66.0F38 0x14 - invalid (legacy only). */
+/*  Opcode VEX.0F38 0x15 - invalid */
+/*  Opcode VEX.66.0F38 0x15 - invalid (legacy only). */
+/*  Opcode VEX.0F38 0x16 - invalid */
+/** Opcode VEX.66.0F38 0x16. */
+FNIEMOP_STUB(iemOp_vpermps_Vqq_Hqq_Wqq);
+/*  Opcode VEX.0F38 0x17 - invalid */
+/** Opcode VEX.66.0F38 0x17 - invalid */
+FNIEMOP_STUB(iemOp_vptest_Vx_Wx);
+/*  Opcode VEX.0F38 0x18 - invalid */
+/** Opcode VEX.66.0F38 0x18. */
+FNIEMOP_STUB(iemOp_vbroadcastss_Vx_Wd);
+/*  Opcode VEX.0F38 0x19 - invalid */
+/** Opcode VEX.66.0F38 0x19. */
+FNIEMOP_STUB(iemOp_vbroadcastsd_Vqq_Wq);
+/*  Opcode VEX.0F38 0x1a - invalid */
+/** Opcode VEX.66.0F38 0x1a. */
+FNIEMOP_STUB(iemOp_vbroadcastf128_Vqq_Mdq);
+/*  Opcode VEX.0F38 0x1b - invalid */
+/*  Opcode VEX.66.0F38 0x1b - invalid */
+/*  Opcode VEX.0F38 0x1c - invalid. */
+/** Opcode VEX.66.0F38 0x1c. */
+FNIEMOP_STUB(iemOp_vpabsb_Vx_Wx);
+/*  Opcode VEX.0F38 0x1d - invalid. */
+/** Opcode VEX.66.0F38 0x1d. */
+FNIEMOP_STUB(iemOp_vpabsw_Vx_Wx);
+/*  Opcode VEX.0F38 0x1e - invalid. */
+/** Opcode VEX.66.0F38 0x1e. */
+FNIEMOP_STUB(iemOp_vpabsd_Vx_Wx);
+/*  Opcode VEX.0F38 0x1f - invalid */
+/*  Opcode VEX.66.0F38 0x1f - invalid */
+
+
+/** Opcode VEX.66.0F38 0x20. */
+FNIEMOP_STUB(iemOp_vpmovsxbw_Vx_UxMq);
+/** Opcode VEX.66.0F38 0x21. */
+FNIEMOP_STUB(iemOp_vpmovsxbd_Vx_UxMd);
+/** Opcode VEX.66.0F38 0x22. */
+FNIEMOP_STUB(iemOp_vpmovsxbq_Vx_UxMw);
+/** Opcode VEX.66.0F38 0x23. */
+FNIEMOP_STUB(iemOp_vpmovsxwd_Vx_UxMq);
+/** Opcode VEX.66.0F38 0x24. */
+FNIEMOP_STUB(iemOp_vpmovsxwq_Vx_UxMd);
+/** Opcode VEX.66.0F38 0x25. */
+FNIEMOP_STUB(iemOp_vpmovsxdq_Vx_UxMq);
+/*  Opcode VEX.66.0F38 0x26 - invalid */
+/*  Opcode VEX.66.0F38 0x27 - invalid */
+/** Opcode VEX.66.0F38 0x28. */
+FNIEMOP_STUB(iemOp_vpmuldq_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x29. */
+FNIEMOP_STUB(iemOp_vpcmpeqq_Vx_Hx_Wx);
+
+
+FNIEMOP_DEF(iemOp_vmovntdqa_Vx_Mx)
+{
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_MOD_MASK) != (3 << X86_MODRM_MOD_SHIFT))
+    {
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            /**
+             * @opcode      0x2a
+             * @opcodesub   !11 mr/reg vex.l=0
+             * @oppfx       0x66
+             * @opcpuid     avx
+             * @opgroup     og_avx_cachect
+             * @opxcpttype  1
+             * @optest      op1=-1 op2=2  -> op1=2
+             * @optest      op1=0 op2=-42 -> op1=-42
+             */
+            /* 128-bit: Memory, register. */
+            IEMOP_MNEMONIC2EX(vmovntdqa_Vdq_WO_Mdq_L0, "vmovntdqa, Vdq_WO, Mdq", VEX_RM_MEM, VMOVNTDQA, vmovntdqa, Vx_WO, Mx,
+                              DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U128_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        else
+        {
+            /**
+             * @opdone
+             * @opcode      0x2a
+             * @opcodesub   !11 mr/reg vex.l=1
+             * @oppfx       0x66
+             * @opcpuid     avx2
+             * @opgroup     og_avx2_cachect
+             * @opxcpttype  1
+             * @optest      op1=-1 op2=2  -> op1=2
+             * @optest      op1=0 op2=-42 -> op1=-42
+             */
+            /* 256-bit: Memory, register. */
+            IEMOP_MNEMONIC2EX(vmovntdqa_Vqq_WO_Mqq_L1, "vmovntdqa, Vqq_WO,Mqq", VEX_RM_MEM, VMOVNTDQA, vmovntdqa, Vx_WO, Mx,
+                              DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U256_ALIGN_AVX(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg, uSrc);
+
+            IEM_MC_ADVANCE_RIP();
+            IEM_MC_END();
+        }
+        return VINF_SUCCESS;
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udvex660f382arg
+     * @opcode      0x2a
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    return IEMOP_RAISE_INVALID_OPCODE();
+
+}
+
+
+/** Opcode VEX.66.0F38 0x2b. */
+FNIEMOP_STUB(iemOp_vpackusdw_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x2c. */
+FNIEMOP_STUB(iemOp_vmaskmovps_Vx_Hx_Mx);
+/** Opcode VEX.66.0F38 0x2d. */
+FNIEMOP_STUB(iemOp_vmaskmovpd_Vx_Hx_Mx);
+/** Opcode VEX.66.0F38 0x2e. */
+FNIEMOP_STUB(iemOp_vmaskmovps_Mx_Hx_Vx);
+/** Opcode VEX.66.0F38 0x2f. */
+FNIEMOP_STUB(iemOp_vmaskmovpd_Mx_Hx_Vx);
+
+/** Opcode VEX.66.0F38 0x30. */
+FNIEMOP_STUB(iemOp_vpmovzxbw_Vx_UxMq);
+/** Opcode VEX.66.0F38 0x31. */
+FNIEMOP_STUB(iemOp_vpmovzxbd_Vx_UxMd);
+/** Opcode VEX.66.0F38 0x32. */
+FNIEMOP_STUB(iemOp_vpmovzxbq_Vx_UxMw);
+/** Opcode VEX.66.0F38 0x33. */
+FNIEMOP_STUB(iemOp_vpmovzxwd_Vx_UxMq);
+/** Opcode VEX.66.0F38 0x34. */
+FNIEMOP_STUB(iemOp_vpmovzxwq_Vx_UxMd);
+/** Opcode VEX.66.0F38 0x35. */
+FNIEMOP_STUB(iemOp_vpmovzxdq_Vx_UxMq);
+/*  Opcode VEX.66.0F38 0x36. */
+FNIEMOP_STUB(iemOp_vpermd_Vqq_Hqq_Wqq);
+/** Opcode VEX.66.0F38 0x37. */
+FNIEMOP_STUB(iemOp_vpcmpgtq_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x38. */
+FNIEMOP_STUB(iemOp_vpminsb_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x39. */
+FNIEMOP_STUB(iemOp_vpminsd_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x3a. */
+FNIEMOP_STUB(iemOp_vpminuw_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x3b. */
+FNIEMOP_STUB(iemOp_vpminud_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x3c. */
+FNIEMOP_STUB(iemOp_vpmaxsb_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x3d. */
+FNIEMOP_STUB(iemOp_vpmaxsd_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x3e. */
+FNIEMOP_STUB(iemOp_vpmaxuw_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x3f. */
+FNIEMOP_STUB(iemOp_vpmaxud_Vx_Hx_Wx);
+
+
+/** Opcode VEX.66.0F38 0x40. */
+FNIEMOP_STUB(iemOp_vpmulld_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x41. */
+FNIEMOP_STUB(iemOp_vphminposuw_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0x42 - invalid. */
+/*  Opcode VEX.66.0F38 0x43 - invalid. */
+/*  Opcode VEX.66.0F38 0x44 - invalid. */
+/** Opcode VEX.66.0F38 0x45. */
+FNIEMOP_STUB(iemOp_vpsrlvd_q_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x46. */
+FNIEMOP_STUB(iemOp_vsravd_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x47. */
+FNIEMOP_STUB(iemOp_vpsllvd_q_Vx_Hx_Wx);
+/*  Opcode VEX.66.0F38 0x48 - invalid. */
+/*  Opcode VEX.66.0F38 0x49 - invalid. */
+/*  Opcode VEX.66.0F38 0x4a - invalid. */
+/*  Opcode VEX.66.0F38 0x4b - invalid. */
+/*  Opcode VEX.66.0F38 0x4c - invalid. */
+/*  Opcode VEX.66.0F38 0x4d - invalid. */
+/*  Opcode VEX.66.0F38 0x4e - invalid. */
+/*  Opcode VEX.66.0F38 0x4f - invalid. */
+
+/*  Opcode VEX.66.0F38 0x50 - invalid. */
+/*  Opcode VEX.66.0F38 0x51 - invalid. */
+/*  Opcode VEX.66.0F38 0x52 - invalid. */
+/*  Opcode VEX.66.0F38 0x53 - invalid. */
+/*  Opcode VEX.66.0F38 0x54 - invalid. */
+/*  Opcode VEX.66.0F38 0x55 - invalid. */
+/*  Opcode VEX.66.0F38 0x56 - invalid. */
+/*  Opcode VEX.66.0F38 0x57 - invalid. */
+/** Opcode VEX.66.0F38 0x58. */
+FNIEMOP_STUB(iemOp_vpbroadcastd_Vx_Wx);
+/** Opcode VEX.66.0F38 0x59. */
+FNIEMOP_STUB(iemOp_vpbroadcastq_Vx_Wx);
+/** Opcode VEX.66.0F38 0x5a. */
+FNIEMOP_STUB(iemOp_vbroadcasti128_Vqq_Mdq);
+/*  Opcode VEX.66.0F38 0x5b - invalid. */
+/*  Opcode VEX.66.0F38 0x5c - invalid. */
+/*  Opcode VEX.66.0F38 0x5d - invalid. */
+/*  Opcode VEX.66.0F38 0x5e - invalid. */
+/*  Opcode VEX.66.0F38 0x5f - invalid. */
+
+/*  Opcode VEX.66.0F38 0x60 - invalid. */
+/*  Opcode VEX.66.0F38 0x61 - invalid. */
+/*  Opcode VEX.66.0F38 0x62 - invalid. */
+/*  Opcode VEX.66.0F38 0x63 - invalid. */
+/*  Opcode VEX.66.0F38 0x64 - invalid. */
+/*  Opcode VEX.66.0F38 0x65 - invalid. */
+/*  Opcode VEX.66.0F38 0x66 - invalid. */
+/*  Opcode VEX.66.0F38 0x67 - invalid. */
+/*  Opcode VEX.66.0F38 0x68 - invalid. */
+/*  Opcode VEX.66.0F38 0x69 - invalid. */
+/*  Opcode VEX.66.0F38 0x6a - invalid. */
+/*  Opcode VEX.66.0F38 0x6b - invalid. */
+/*  Opcode VEX.66.0F38 0x6c - invalid. */
+/*  Opcode VEX.66.0F38 0x6d - invalid. */
+/*  Opcode VEX.66.0F38 0x6e - invalid. */
+/*  Opcode VEX.66.0F38 0x6f - invalid. */
+
+/*  Opcode VEX.66.0F38 0x70 - invalid. */
+/*  Opcode VEX.66.0F38 0x71 - invalid. */
+/*  Opcode VEX.66.0F38 0x72 - invalid. */
+/*  Opcode VEX.66.0F38 0x73 - invalid. */
+/*  Opcode VEX.66.0F38 0x74 - invalid. */
+/*  Opcode VEX.66.0F38 0x75 - invalid. */
+/*  Opcode VEX.66.0F38 0x76 - invalid. */
+/*  Opcode VEX.66.0F38 0x77 - invalid. */
+/** Opcode VEX.66.0F38 0x78. */
+FNIEMOP_STUB(iemOp_vpboardcastb_Vx_Wx);
+/** Opcode VEX.66.0F38 0x79. */
+FNIEMOP_STUB(iemOp_vpboardcastw_Vx_Wx);
+/*  Opcode VEX.66.0F38 0x7a - invalid. */
+/*  Opcode VEX.66.0F38 0x7b - invalid. */
+/*  Opcode VEX.66.0F38 0x7c - invalid. */
+/*  Opcode VEX.66.0F38 0x7d - invalid. */
+/*  Opcode VEX.66.0F38 0x7e - invalid. */
+/*  Opcode VEX.66.0F38 0x7f - invalid. */
+
+/*  Opcode VEX.66.0F38 0x80 - invalid (legacy only). */
+/*  Opcode VEX.66.0F38 0x81 - invalid (legacy only). */
+/*  Opcode VEX.66.0F38 0x82 - invalid (legacy only). */
+/*  Opcode VEX.66.0F38 0x83 - invalid. */
+/*  Opcode VEX.66.0F38 0x84 - invalid. */
+/*  Opcode VEX.66.0F38 0x85 - invalid. */
+/*  Opcode VEX.66.0F38 0x86 - invalid. */
+/*  Opcode VEX.66.0F38 0x87 - invalid. */
+/*  Opcode VEX.66.0F38 0x88 - invalid. */
+/*  Opcode VEX.66.0F38 0x89 - invalid. */
+/*  Opcode VEX.66.0F38 0x8a - invalid. */
+/*  Opcode VEX.66.0F38 0x8b - invalid. */
+/** Opcode VEX.66.0F38 0x8c. */
+FNIEMOP_STUB(iemOp_vpmaskmovd_q_Vx_Hx_Mx);
+/*  Opcode VEX.66.0F38 0x8d - invalid. */
+/** Opcode VEX.66.0F38 0x8e. */
+FNIEMOP_STUB(iemOp_vpmaskmovd_q_Mx_Vx_Hx);
+/*  Opcode VEX.66.0F38 0x8f - invalid. */
+
+/** Opcode VEX.66.0F38 0x90 (vex only). */
+FNIEMOP_STUB(iemOp_vgatherdd_q_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x91 (vex only). */
+FNIEMOP_STUB(iemOp_vgatherqd_q_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x92 (vex only). */
+FNIEMOP_STUB(iemOp_vgatherdps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x93 (vex only). */
+FNIEMOP_STUB(iemOp_vgatherqps_d_Vx_Hx_Wx);
+/*  Opcode VEX.66.0F38 0x94 - invalid. */
+/*  Opcode VEX.66.0F38 0x95 - invalid. */
+/** Opcode VEX.66.0F38 0x96 (vex only). */
+FNIEMOP_STUB(iemOp_vfmaddsub132ps_q_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x97 (vex only). */
+FNIEMOP_STUB(iemOp_vfmsubadd132ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x98 (vex only). */
+FNIEMOP_STUB(iemOp_vfmadd132ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x99 (vex only). */
+FNIEMOP_STUB(iemOp_vfmadd132ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x9a (vex only). */
+FNIEMOP_STUB(iemOp_vfmsub132ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x9b (vex only). */
+FNIEMOP_STUB(iemOp_vfmsub132ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x9c (vex only). */
+FNIEMOP_STUB(iemOp_vfnmadd132ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x9d (vex only). */
+FNIEMOP_STUB(iemOp_vfnmadd132ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x9e (vex only). */
+FNIEMOP_STUB(iemOp_vfnmsub132ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x9f (vex only). */
+FNIEMOP_STUB(iemOp_vfnmsub132ss_d_Vx_Hx_Wx);
+
+/*  Opcode VEX.66.0F38 0xa0 - invalid. */
+/*  Opcode VEX.66.0F38 0xa1 - invalid. */
+/*  Opcode VEX.66.0F38 0xa2 - invalid. */
+/*  Opcode VEX.66.0F38 0xa3 - invalid. */
+/*  Opcode VEX.66.0F38 0xa4 - invalid. */
+/*  Opcode VEX.66.0F38 0xa5 - invalid. */
+/** Opcode VEX.66.0F38 0xa6 (vex only). */
+FNIEMOP_STUB(iemOp_vfmaddsub213ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xa7 (vex only). */
+FNIEMOP_STUB(iemOp_vfmsubadd213ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xa8 (vex only). */
+FNIEMOP_STUB(iemOp_vfmadd213ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xa9 (vex only). */
+FNIEMOP_STUB(iemOp_vfmadd213ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xaa (vex only). */
+FNIEMOP_STUB(iemOp_vfmsub213ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xab (vex only). */
+FNIEMOP_STUB(iemOp_vfmsub213ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xac (vex only). */
+FNIEMOP_STUB(iemOp_vfnmadd213ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xad (vex only). */
+FNIEMOP_STUB(iemOp_vfnmadd213ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xae (vex only). */
+FNIEMOP_STUB(iemOp_vfnmsub213ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xaf (vex only). */
+FNIEMOP_STUB(iemOp_vfnmsub213ss_d_Vx_Hx_Wx);
+
+/*  Opcode VEX.66.0F38 0xb0 - invalid. */
+/*  Opcode VEX.66.0F38 0xb1 - invalid. */
+/*  Opcode VEX.66.0F38 0xb2 - invalid. */
+/*  Opcode VEX.66.0F38 0xb3 - invalid. */
+/*  Opcode VEX.66.0F38 0xb4 - invalid. */
+/*  Opcode VEX.66.0F38 0xb5 - invalid. */
+/** Opcode VEX.66.0F38 0xb6 (vex only). */
+FNIEMOP_STUB(iemOp_vfmaddsub231ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xb7 (vex only). */
+FNIEMOP_STUB(iemOp_vfmsubadd231ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xb8 (vex only). */
+FNIEMOP_STUB(iemOp_vfmadd231ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xb9 (vex only). */
+FNIEMOP_STUB(iemOp_vfmadd231ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xba (vex only). */
+FNIEMOP_STUB(iemOp_vfmsub231ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xbb (vex only). */
+FNIEMOP_STUB(iemOp_vfmsub231ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xbc (vex only). */
+FNIEMOP_STUB(iemOp_vfnmadd231ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xbd (vex only). */
+FNIEMOP_STUB(iemOp_vfnmadd231ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xbe (vex only). */
+FNIEMOP_STUB(iemOp_vfnmsub231ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xbf (vex only). */
+FNIEMOP_STUB(iemOp_vfnmsub231ss_d_Vx_Hx_Wx);
+
+/*  Opcode VEX.0F38 0xc0 - invalid. */
+/*  Opcode VEX.66.0F38 0xc0 - invalid. */
+/*  Opcode VEX.0F38 0xc1 - invalid. */
+/*  Opcode VEX.66.0F38 0xc1 - invalid. */
+/*  Opcode VEX.0F38 0xc2 - invalid. */
+/*  Opcode VEX.66.0F38 0xc2 - invalid. */
+/*  Opcode VEX.0F38 0xc3 - invalid. */
+/*  Opcode VEX.66.0F38 0xc3 - invalid. */
+/*  Opcode VEX.0F38 0xc4 - invalid. */
+/*  Opcode VEX.66.0F38 0xc4 - invalid. */
+/*  Opcode VEX.0F38 0xc5 - invalid. */
+/*  Opcode VEX.66.0F38 0xc5 - invalid. */
+/*  Opcode VEX.0F38 0xc6 - invalid. */
+/*  Opcode VEX.66.0F38 0xc6 - invalid. */
+/*  Opcode VEX.0F38 0xc7 - invalid. */
+/*  Opcode VEX.66.0F38 0xc7 - invalid. */
+/** Opcode VEX.0F38 0xc8. */
+FNIEMOP_STUB(iemOp_vsha1nexte_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0xc8 - invalid. */
+/** Opcode VEX.0F38 0xc9. */
+FNIEMOP_STUB(iemOp_vsha1msg1_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0xc9 - invalid. */
+/** Opcode VEX.0F38 0xca. */
+FNIEMOP_STUB(iemOp_vsha1msg2_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0xca - invalid. */
+/** Opcode VEX.0F38 0xcb. */
+FNIEMOP_STUB(iemOp_vsha256rnds2_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0xcb - invalid. */
+/** Opcode VEX.0F38 0xcc. */
+FNIEMOP_STUB(iemOp_vsha256msg1_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0xcc - invalid. */
+/** Opcode VEX.0F38 0xcd. */
+FNIEMOP_STUB(iemOp_vsha256msg2_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0xcd - invalid. */
+/*  Opcode VEX.0F38 0xce - invalid. */
+/*  Opcode VEX.66.0F38 0xce - invalid. */
+/*  Opcode VEX.0F38 0xcf - invalid. */
+/*  Opcode VEX.66.0F38 0xcf - invalid. */
+
+/*  Opcode VEX.66.0F38 0xd0 - invalid. */
+/*  Opcode VEX.66.0F38 0xd1 - invalid. */
+/*  Opcode VEX.66.0F38 0xd2 - invalid. */
+/*  Opcode VEX.66.0F38 0xd3 - invalid. */
+/*  Opcode VEX.66.0F38 0xd4 - invalid. */
+/*  Opcode VEX.66.0F38 0xd5 - invalid. */
+/*  Opcode VEX.66.0F38 0xd6 - invalid. */
+/*  Opcode VEX.66.0F38 0xd7 - invalid. */
+/*  Opcode VEX.66.0F38 0xd8 - invalid. */
+/*  Opcode VEX.66.0F38 0xd9 - invalid. */
+/*  Opcode VEX.66.0F38 0xda - invalid. */
+/** Opcode VEX.66.0F38 0xdb. */
+FNIEMOP_STUB(iemOp_vaesimc_Vdq_Wdq);
+/** Opcode VEX.66.0F38 0xdc. */
+FNIEMOP_STUB(iemOp_vaesenc_Vdq_Wdq);
+/** Opcode VEX.66.0F38 0xdd. */
+FNIEMOP_STUB(iemOp_vaesenclast_Vdq_Wdq);
+/** Opcode VEX.66.0F38 0xde. */
+FNIEMOP_STUB(iemOp_vaesdec_Vdq_Wdq);
+/** Opcode VEX.66.0F38 0xdf. */
+FNIEMOP_STUB(iemOp_vaesdeclast_Vdq_Wdq);
+
+/*  Opcode VEX.66.0F38 0xe0 - invalid. */
+/*  Opcode VEX.66.0F38 0xe1 - invalid. */
+/*  Opcode VEX.66.0F38 0xe2 - invalid. */
+/*  Opcode VEX.66.0F38 0xe3 - invalid. */
+/*  Opcode VEX.66.0F38 0xe4 - invalid. */
+/*  Opcode VEX.66.0F38 0xe5 - invalid. */
+/*  Opcode VEX.66.0F38 0xe6 - invalid. */
+/*  Opcode VEX.66.0F38 0xe7 - invalid. */
+/*  Opcode VEX.66.0F38 0xe8 - invalid. */
+/*  Opcode VEX.66.0F38 0xe9 - invalid. */
+/*  Opcode VEX.66.0F38 0xea - invalid. */
+/*  Opcode VEX.66.0F38 0xeb - invalid. */
+/*  Opcode VEX.66.0F38 0xec - invalid. */
+/*  Opcode VEX.66.0F38 0xed - invalid. */
+/*  Opcode VEX.66.0F38 0xee - invalid. */
+/*  Opcode VEX.66.0F38 0xef - invalid. */
+
+
+/*  Opcode VEX.0F38 0xf0 - invalid (legacy only). */
+/*  Opcode VEX.66.0F38 0xf0 - invalid (legacy only). */
+/*  Opcode VEX.F3.0F38 0xf0 - invalid. */
+/*  Opcode VEX.F2.0F38 0xf0 - invalid (legacy only). */
+
+/*  Opcode VEX.0F38 0xf1 - invalid (legacy only). */
+/*  Opcode VEX.66.0F38 0xf1 - invalid (legacy only). */
+/*  Opcode VEX.F3.0F38 0xf1 - invalid. */
+/*  Opcode VEX.F2.0F38 0xf1 - invalid (legacy only). */
+
+/*  Opcode VEX.0F38 0xf2 - invalid (vex only). */
+FNIEMOP_STUB(iemOp_andn_Gy_By_Ey);
+/*  Opcode VEX.66.0F38 0xf2 - invalid. */
+/*  Opcode VEX.F3.0F38 0xf2 - invalid. */
+/*  Opcode VEX.F2.0F38 0xf2 - invalid. */
+
+
+/*  Opcode VEX.0F38 0xf3 - invalid. */
+/*  Opcode VEX.66.0F38 0xf3 - invalid. */
+
+/*  Opcode VEX.F3.0F38 0xf3 /0 - invalid). */
+/*  Opcode VEX.F3.0F38 0xf3 /1). */
+FNIEMOP_STUB_1(iemOp_VGrp17_blsr_By_Ey, uint8_t, bRm);
+/*  Opcode VEX.F3.0F38 0xf3 /2). */
+FNIEMOP_STUB_1(iemOp_VGrp17_blsmsk_By_Ey, uint8_t, bRm);
+/*  Opcode VEX.F3.0F38 0xf3 /3). */
+FNIEMOP_STUB_1(iemOp_VGrp17_blsi_By_Ey, uint8_t, bRm);
+/*  Opcode VEX.F3.0F38 0xf3 /4 - invalid). */
+/*  Opcode VEX.F3.0F38 0xf3 /5 - invalid). */
+/*  Opcode VEX.F3.0F38 0xf3 /6 - invalid). */
+/*  Opcode VEX.F3.0F38 0xf3 /7 - invalid). */
+
+/**
+ * Group 17 jump table for the VEX.F3 variant..
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnVexGroup17_f3[] =
+{
+    /* /0 */ iemOp_InvalidWithRM,
+    /* /1 */ iemOp_VGrp17_blsr_By_Ey,
+    /* /2 */ iemOp_VGrp17_blsmsk_By_Ey,
+    /* /3 */ iemOp_VGrp17_blsi_By_Ey,
+    /* /4 */ iemOp_InvalidWithRM,
+    /* /5 */ iemOp_InvalidWithRM,
+    /* /6 */ iemOp_InvalidWithRM,
+    /* /7 */ iemOp_InvalidWithRM
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexGroup17_f3) == 8);
+
+/**  Opcode VEX.F3.0F38 0xf3 - invalid (vex only - group 17). */
+FNIEMOP_DEF(iemOp_VGrp17_f3)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    return FNIEMOP_CALL_1(g_apfnVexGroup17_f3[((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK)], bRm);
+
+}
+
+/*  Opcode VEX.F2.0F38 0xf3 - invalid (vex only - group 17). */
+
+
+/*  Opcode VEX.0F38 0xf4 - invalid. */
+/*  Opcode VEX.66.0F38 0xf4 - invalid. */
+/*  Opcode VEX.F3.0F38 0xf4 - invalid. */
+/*  Opcode VEX.F2.0F38 0xf4 - invalid. */
+
+/** Opcode VEX.0F38 0xf5 (vex only). */
+FNIEMOP_STUB(iemOp_bzhi_Gy_Ey_By);
+/*  Opcode VEX.66.0F38 0xf5 - invalid. */
+/** Opcode VEX.F3.0F38 0xf5 (vex only). */
+FNIEMOP_STUB(iemOp_pext_Gy_By_Ey);
+/** Opcode VEX.F2.0F38 0xf5 (vex only). */
+FNIEMOP_STUB(iemOp_pdep_Gy_By_Ey);
+
+/*  Opcode VEX.0F38 0xf6 - invalid. */
+/*  Opcode VEX.66.0F38 0xf6 - invalid (legacy only). */
+/*  Opcode VEX.F3.0F38 0xf6 - invalid (legacy only). */
+/*  Opcode VEX.F2.0F38 0xf6 - invalid (vex only). */
+FNIEMOP_STUB(iemOp_mulx_By_Gy_rDX_Ey);
+
+/** Opcode VEX.0F38 0xf7 (vex only). */
+FNIEMOP_STUB(iemOp_bextr_Gy_Ey_By);
+/** Opcode VEX.66.0F38 0xf7 (vex only). */
+FNIEMOP_STUB(iemOp_shlx_Gy_Ey_By);
+/** Opcode VEX.F3.0F38 0xf7 (vex only). */
+FNIEMOP_STUB(iemOp_sarx_Gy_Ey_By);
+/** Opcode VEX.F2.0F38 0xf7 (vex only). */
+FNIEMOP_STUB(iemOp_shrx_Gy_Ey_By);
+
+/*  Opcode VEX.0F38 0xf8 - invalid. */
+/*  Opcode VEX.66.0F38 0xf8 - invalid. */
+/*  Opcode VEX.F3.0F38 0xf8 - invalid. */
+/*  Opcode VEX.F2.0F38 0xf8 - invalid. */
+
+/*  Opcode VEX.0F38 0xf9 - invalid. */
+/*  Opcode VEX.66.0F38 0xf9 - invalid. */
+/*  Opcode VEX.F3.0F38 0xf9 - invalid. */
+/*  Opcode VEX.F2.0F38 0xf9 - invalid. */
+
+/*  Opcode VEX.0F38 0xfa - invalid. */
+/*  Opcode VEX.66.0F38 0xfa - invalid. */
+/*  Opcode VEX.F3.0F38 0xfa - invalid. */
+/*  Opcode VEX.F2.0F38 0xfa - invalid. */
+
+/*  Opcode VEX.0F38 0xfb - invalid. */
+/*  Opcode VEX.66.0F38 0xfb - invalid. */
+/*  Opcode VEX.F3.0F38 0xfb - invalid. */
+/*  Opcode VEX.F2.0F38 0xfb - invalid. */
+
+/*  Opcode VEX.0F38 0xfc - invalid. */
+/*  Opcode VEX.66.0F38 0xfc - invalid. */
+/*  Opcode VEX.F3.0F38 0xfc - invalid. */
+/*  Opcode VEX.F2.0F38 0xfc - invalid. */
+
+/*  Opcode VEX.0F38 0xfd - invalid. */
+/*  Opcode VEX.66.0F38 0xfd - invalid. */
+/*  Opcode VEX.F3.0F38 0xfd - invalid. */
+/*  Opcode VEX.F2.0F38 0xfd - invalid. */
+
+/*  Opcode VEX.0F38 0xfe - invalid. */
+/*  Opcode VEX.66.0F38 0xfe - invalid. */
+/*  Opcode VEX.F3.0F38 0xfe - invalid. */
+/*  Opcode VEX.F2.0F38 0xfe - invalid. */
+
+/*  Opcode VEX.0F38 0xff - invalid. */
+/*  Opcode VEX.66.0F38 0xff - invalid. */
+/*  Opcode VEX.F3.0F38 0xff - invalid. */
+/*  Opcode VEX.F2.0F38 0xff - invalid. */
+
+
+/**
+ * VEX opcode map \#2.
+ *
+ * @sa      g_apfnThreeByte0f38
+ */
+IEM_STATIC const PFNIEMOP g_apfnVexMap2[] =
+{
+    /*          no prefix,                  066h prefix                 f3h prefix,                 f2h prefix */
+    /* 0x00 */  iemOp_InvalidNeedRM,        iemOp_vpshufb_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x01 */  iemOp_InvalidNeedRM,        iemOp_vphaddw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x02 */  iemOp_InvalidNeedRM,        iemOp_vphaddd_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x03 */  iemOp_InvalidNeedRM,        iemOp_vphaddsw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x04 */  iemOp_InvalidNeedRM,        iemOp_vpmaddubsw_Vx_Hx_Wx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x05 */  iemOp_InvalidNeedRM,        iemOp_vphsubw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x06 */  iemOp_InvalidNeedRM,        iemOp_vphsubdq_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x07 */  iemOp_InvalidNeedRM,        iemOp_vphsubsw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x08 */  iemOp_InvalidNeedRM,        iemOp_vpsignb_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x09 */  iemOp_InvalidNeedRM,        iemOp_vpsignw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0a */  iemOp_InvalidNeedRM,        iemOp_vpsignd_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0b */  iemOp_InvalidNeedRM,        iemOp_vpmulhrsw_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0c */  iemOp_InvalidNeedRM,        iemOp_vpermilps_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0d */  iemOp_InvalidNeedRM,        iemOp_vpermilpd_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0e */  iemOp_InvalidNeedRM,        iemOp_vtestps_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0f */  iemOp_InvalidNeedRM,        iemOp_vtestpd_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x10 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x11 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x12 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x13 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x14 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x15 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x16 */  iemOp_InvalidNeedRM,        iemOp_vpermps_Vqq_Hqq_Wqq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x17 */  iemOp_InvalidNeedRM,        iemOp_vptest_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x18 */  iemOp_InvalidNeedRM,        iemOp_vbroadcastss_Vx_Wd,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x19 */  iemOp_InvalidNeedRM,        iemOp_vbroadcastsd_Vqq_Wq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1a */  iemOp_InvalidNeedRM,        iemOp_vbroadcastf128_Vqq_Mdq, iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,
+    /* 0x1b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1c */  iemOp_InvalidNeedRM,        iemOp_vpabsb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1d */  iemOp_InvalidNeedRM,        iemOp_vpabsw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1e */  iemOp_InvalidNeedRM,        iemOp_vpabsd_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x20 */  iemOp_InvalidNeedRM,        iemOp_vpmovsxbw_Vx_UxMq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x21 */  iemOp_InvalidNeedRM,        iemOp_vpmovsxbd_Vx_UxMd,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x22 */  iemOp_InvalidNeedRM,        iemOp_vpmovsxbq_Vx_UxMw,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x23 */  iemOp_InvalidNeedRM,        iemOp_vpmovsxwd_Vx_UxMq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x24 */  iemOp_InvalidNeedRM,        iemOp_vpmovsxwq_Vx_UxMd,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x25 */  iemOp_InvalidNeedRM,        iemOp_vpmovsxdq_Vx_UxMq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x26 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x27 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x28 */  iemOp_InvalidNeedRM,        iemOp_vpmuldq_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x29 */  iemOp_InvalidNeedRM,        iemOp_vpcmpeqq_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2a */  iemOp_InvalidNeedRM,        iemOp_vmovntdqa_Vx_Mx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2b */  iemOp_InvalidNeedRM,        iemOp_vpackusdw_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2c */  iemOp_InvalidNeedRM,        iemOp_vmaskmovps_Vx_Hx_Mx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2d */  iemOp_InvalidNeedRM,        iemOp_vmaskmovpd_Vx_Hx_Mx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2e */  iemOp_InvalidNeedRM,        iemOp_vmaskmovps_Mx_Hx_Vx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2f */  iemOp_InvalidNeedRM,        iemOp_vmaskmovpd_Mx_Hx_Vx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x30 */  iemOp_InvalidNeedRM,        iemOp_vpmovzxbw_Vx_UxMq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x31 */  iemOp_InvalidNeedRM,        iemOp_vpmovzxbd_Vx_UxMd,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x32 */  iemOp_InvalidNeedRM,        iemOp_vpmovzxbq_Vx_UxMw,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x33 */  iemOp_InvalidNeedRM,        iemOp_vpmovzxwd_Vx_UxMq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x34 */  iemOp_InvalidNeedRM,        iemOp_vpmovzxwq_Vx_UxMd,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x35 */  iemOp_InvalidNeedRM,        iemOp_vpmovzxdq_Vx_UxMq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x36 */  iemOp_InvalidNeedRM,        iemOp_vpermd_Vqq_Hqq_Wqq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x37 */  iemOp_InvalidNeedRM,        iemOp_vpcmpgtq_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x38 */  iemOp_InvalidNeedRM,        iemOp_vpminsb_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x39 */  iemOp_InvalidNeedRM,        iemOp_vpminsd_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3a */  iemOp_InvalidNeedRM,        iemOp_vpminuw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3b */  iemOp_InvalidNeedRM,        iemOp_vpminud_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3c */  iemOp_InvalidNeedRM,        iemOp_vpmaxsb_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3d */  iemOp_InvalidNeedRM,        iemOp_vpmaxsd_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3e */  iemOp_InvalidNeedRM,        iemOp_vpmaxuw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3f */  iemOp_InvalidNeedRM,        iemOp_vpmaxud_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x40 */  iemOp_InvalidNeedRM,        iemOp_vpmulld_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x41 */  iemOp_InvalidNeedRM,        iemOp_vphminposuw_Vdq_Wdq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x42 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x43 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x44 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x45 */  iemOp_InvalidNeedRM,        iemOp_vpsrlvd_q_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x46 */  iemOp_InvalidNeedRM,        iemOp_vsravd_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x47 */  iemOp_InvalidNeedRM,        iemOp_vpsllvd_q_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x48 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x49 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x50 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x51 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x52 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x53 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x54 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x55 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x56 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x57 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x58 */  iemOp_InvalidNeedRM,        iemOp_vpbroadcastd_Vx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x59 */  iemOp_InvalidNeedRM,        iemOp_vpbroadcastq_Vx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x5a */  iemOp_InvalidNeedRM,        iemOp_vbroadcasti128_Vqq_Mdq, iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,
+    /* 0x5b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x60 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x61 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x62 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x63 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x64 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x65 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x66 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x67 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x68 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x69 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x70 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x71 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x72 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x73 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x74 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x75 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x76 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x77 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x78 */  iemOp_InvalidNeedRM,        iemOp_vpboardcastb_Vx_Wx,   iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,
+    /* 0x79 */  iemOp_InvalidNeedRM,        iemOp_vpboardcastw_Vx_Wx,   iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,
+    /* 0x7a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x80 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x81 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x82 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x83 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x84 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x85 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x86 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x87 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x88 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x89 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8c */  iemOp_InvalidNeedRM,        iemOp_vpmaskmovd_q_Vx_Hx_Mx, iemOp_InvalidNeedRM,     iemOp_InvalidNeedRM,
+    /* 0x8d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8e */  iemOp_InvalidNeedRM,        iemOp_vpmaskmovd_q_Mx_Vx_Hx, iemOp_InvalidNeedRM,     iemOp_InvalidNeedRM,
+    /* 0x8f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x90 */  iemOp_InvalidNeedRM,        iemOp_vgatherdd_q_Vx_Hx_Wx, iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,
+    /* 0x91 */  iemOp_InvalidNeedRM,        iemOp_vgatherqd_q_Vx_Hx_Wx, iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,
+    /* 0x92 */  iemOp_InvalidNeedRM,        iemOp_vgatherdps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM,     iemOp_InvalidNeedRM,
+    /* 0x93 */  iemOp_InvalidNeedRM,        iemOp_vgatherqps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM,     iemOp_InvalidNeedRM,
+    /* 0x94 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x95 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x96 */  iemOp_InvalidNeedRM,        iemOp_vfmaddsub132ps_q_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x97 */  iemOp_InvalidNeedRM,        iemOp_vfmsubadd132ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x98 */  iemOp_InvalidNeedRM,        iemOp_vfmadd132ps_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x99 */  iemOp_InvalidNeedRM,        iemOp_vfmadd132ss_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x9a */  iemOp_InvalidNeedRM,        iemOp_vfmsub132ps_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x9b */  iemOp_InvalidNeedRM,        iemOp_vfmsub132ss_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x9c */  iemOp_InvalidNeedRM,        iemOp_vfnmadd132ps_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x9d */  iemOp_InvalidNeedRM,        iemOp_vfnmadd132ss_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x9e */  iemOp_InvalidNeedRM,        iemOp_vfnmsub132ps_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x9f */  iemOp_InvalidNeedRM,        iemOp_vfnmsub132ss_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+
+    /* 0xa0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa6 */  iemOp_InvalidNeedRM,        iemOp_vfmaddsub213ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xa7 */  iemOp_InvalidNeedRM,        iemOp_vfmsubadd213ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xa8 */  iemOp_InvalidNeedRM,        iemOp_vfmadd213ps_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xa9 */  iemOp_InvalidNeedRM,        iemOp_vfmadd213ss_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xaa */  iemOp_InvalidNeedRM,        iemOp_vfmsub213ps_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xab */  iemOp_InvalidNeedRM,        iemOp_vfmsub213ss_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xac */  iemOp_InvalidNeedRM,        iemOp_vfnmadd213ps_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xad */  iemOp_InvalidNeedRM,        iemOp_vfnmadd213ss_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xae */  iemOp_InvalidNeedRM,        iemOp_vfnmsub213ps_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xaf */  iemOp_InvalidNeedRM,        iemOp_vfnmsub213ss_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+
+    /* 0xb0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb6 */  iemOp_InvalidNeedRM,        iemOp_vfmaddsub231ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xb7 */  iemOp_InvalidNeedRM,        iemOp_vfmsubadd231ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xb8 */  iemOp_InvalidNeedRM,        iemOp_vfmadd231ps_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xb9 */  iemOp_InvalidNeedRM,        iemOp_vfmadd231ss_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xba */  iemOp_InvalidNeedRM,        iemOp_vfmsub231ps_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xbb */  iemOp_InvalidNeedRM,        iemOp_vfmsub231ss_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xbc */  iemOp_InvalidNeedRM,        iemOp_vfnmadd231ps_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xbd */  iemOp_InvalidNeedRM,        iemOp_vfnmadd231ss_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xbe */  iemOp_InvalidNeedRM,        iemOp_vfnmsub231ps_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xbf */  iemOp_InvalidNeedRM,        iemOp_vfnmsub231ss_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+
+    /* 0xc0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc8 */  iemOp_vsha1nexte_Vdq_Wdq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xc9 */  iemOp_vsha1msg1_Vdq_Wdq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xca */  iemOp_vsha1msg2_Vdq_Wdq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xcb */  iemOp_vsha256rnds2_Vdq_Wdq, iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xcc */  iemOp_vsha256msg1_Vdq_Wdq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xcd */  iemOp_vsha256msg2_Vdq_Wdq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xce */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xcf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xd0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xda */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xdb */  iemOp_InvalidNeedRM,        iemOp_vaesimc_Vdq_Wdq,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdc */  iemOp_InvalidNeedRM,        iemOp_vaesenc_Vdq_Wdq,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdd */  iemOp_InvalidNeedRM,        iemOp_vaesenclast_Vdq_Wdq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xde */  iemOp_InvalidNeedRM,        iemOp_vaesdec_Vdq_Wdq,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdf */  iemOp_InvalidNeedRM,        iemOp_vaesdeclast_Vdq_Wdq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0xe0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xea */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xeb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xec */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xed */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xee */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xef */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xf0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf2 */  iemOp_andn_Gy_By_Ey,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf3 */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_VGrp17_f3,            iemOp_InvalidNeedRM,
+    /* 0xf4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf5 */  iemOp_bzhi_Gy_Ey_By,        iemOp_InvalidNeedRM,        iemOp_pext_Gy_By_Ey,        iemOp_pdep_Gy_By_Ey,
+    /* 0xf6 */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_mulx_By_Gy_rDX_Ey,
+    /* 0xf7 */  iemOp_bextr_Gy_Ey_By,       iemOp_shlx_Gy_Ey_By,        iemOp_sarx_Gy_Ey_By,        iemOp_shrx_Gy_Ey_By,
+    /* 0xf8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfa */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfc */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfd */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfe */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xff */  IEMOP_X4(iemOp_InvalidNeedRM),
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexMap2) == 1024);
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h
new file mode 100644
index 00000000..f2d555ab
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h
@@ -0,0 +1,557 @@
+/* $Id: IEMAllInstructionsVexMap3.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation, 0x0f 0x3a map.
+ *
+ * @remarks IEMAllInstructionsThree0f3a.cpp.h is a VEX mirror of this file.
+ *          Any update here is likely needed in that file too.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @name VEX Opcode Map 3
+ * @{
+ */
+
+/** Opcode VEX.66.0F3A 0x00. */
+FNIEMOP_STUB(iemOp_vpermq_Vqq_Wqq_Ib);
+/** Opcode VEX.66.0F3A 0x01. */
+FNIEMOP_STUB(iemOp_vpermqd_Vqq_Wqq_Ib);
+/** Opcode VEX.66.0F3A 0x02. */
+FNIEMOP_STUB(iemOp_vpblendd_Vx_Wx_Ib);
+/*  Opcode VEX.66.0F3A 0x03 - invalid */
+/** Opcode VEX.66.0F3A 0x04. */
+FNIEMOP_STUB(iemOp_vpermilps_Vx_Wx_Ib);
+/** Opcode VEX.66.0F3A 0x05. */
+FNIEMOP_STUB(iemOp_vpermilpd_Vx_Wx_Ib);
+/** Opcode VEX.66.0F3A 0x06 (vex only) */
+FNIEMOP_STUB(iemOp_vperm2f128_Vqq_Hqq_Wqq_Ib);
+/*  Opcode VEX.66.0F3A 0x07 - invalid */
+/** Opcode VEX.66.0F3A 0x08. */
+FNIEMOP_STUB(iemOp_vroundps_Vx_Wx_Ib);
+/** Opcode VEX.66.0F3A 0x09. */
+FNIEMOP_STUB(iemOp_vroundpd_Vx_Wx_Ib);
+/** Opcode VEX.66.0F3A 0x0a. */
+FNIEMOP_STUB(iemOp_vroundss_Vss_Wss_Ib);
+/** Opcode VEX.66.0F3A 0x0b. */
+FNIEMOP_STUB(iemOp_vroundsd_Vsd_Wsd_Ib);
+/** Opcode VEX.66.0F3A 0x0c. */
+FNIEMOP_STUB(iemOp_vblendps_Vx_Hx_Wx_Ib);
+/** Opcode VEX.66.0F3A 0x0d. */
+FNIEMOP_STUB(iemOp_vblendpd_Vx_Hx_Wx_Ib);
+/** Opcode VEX.66.0F3A 0x0e. */
+FNIEMOP_STUB(iemOp_vblendw_Vx_Hx_Wx_Ib);
+/** Opcode VEX.0F3A 0x0f - invalid. */
+/** Opcode VEX.66.0F3A 0x0f. */
+FNIEMOP_STUB(iemOp_vpalignr_Vx_Hx_Wx_Ib);
+
+
+/*  Opcode VEX.66.0F3A 0x10 - invalid */
+/*  Opcode VEX.66.0F3A 0x11 - invalid */
+/*  Opcode VEX.66.0F3A 0x12 - invalid */
+/*  Opcode VEX.66.0F3A 0x13 - invalid */
+/** Opcode VEX.66.0F3A 0x14. */
+FNIEMOP_STUB(iemOp_vpextrb_RdMb_Vdq_Ib);
+/** Opcode VEX.66.0F3A 0x15. */
+FNIEMOP_STUB(iemOp_vpextrw_RdMw_Vdq_Ib);
+/** Opcode VEX.66.0F3A 0x16. */
+FNIEMOP_STUB(iemOp_vpextrd_q_RdMw_Vdq_Ib);
+/** Opcode VEX.66.0F3A 0x17. */
+FNIEMOP_STUB(iemOp_vextractps_Ed_Vdq_Ib);
+/** Opcode VEX.66.0F3A 0x18 (vex only). */
+FNIEMOP_STUB(iemOp_vinsertf128_Vqq_Hqq_Wqq_Ib);
+/** Opcode VEX.66.0F3A 0x19 (vex only). */
+FNIEMOP_STUB(iemOp_vextractf128_Wdq_Vqq_Ib);
+/*  Opcode VEX.66.0F3A 0x1a - invalid */
+/*  Opcode VEX.66.0F3A 0x1b - invalid */
+/*  Opcode VEX.66.0F3A 0x1c - invalid */
+/** Opcode VEX.66.0F3A 0x1d (vex only). */
+FNIEMOP_STUB(iemOp_vcvtps2ph_Wx_Vx_Ib);
+/*  Opcode VEX.66.0F3A 0x1e - invalid */
+/*  Opcode VEX.66.0F3A 0x1f - invalid */
+
+
+/** Opcode VEX.66.0F3A 0x20. */
+FNIEMOP_STUB(iemOp_vpinsrb_Vdq_Hdq_RyMb_Ib);
+/** Opcode VEX.66.0F3A 0x21, */
+FNIEMOP_STUB(iemOp_vinsertps_Vdq_Hdq_UdqMd_Ib);
+/** Opcode VEX.66.0F3A 0x22. */
+FNIEMOP_STUB(iemOp_vpinsrd_q_Vdq_Hdq_Ey_Ib);
+/*  Opcode VEX.66.0F3A 0x23 - invalid */
+/*  Opcode VEX.66.0F3A 0x24 - invalid */
+/*  Opcode VEX.66.0F3A 0x25 - invalid */
+/*  Opcode VEX.66.0F3A 0x26 - invalid */
+/*  Opcode VEX.66.0F3A 0x27 - invalid */
+/*  Opcode VEX.66.0F3A 0x28 - invalid */
+/*  Opcode VEX.66.0F3A 0x29 - invalid */
+/*  Opcode VEX.66.0F3A 0x2a - invalid */
+/*  Opcode VEX.66.0F3A 0x2b - invalid */
+/*  Opcode VEX.66.0F3A 0x2c - invalid */
+/*  Opcode VEX.66.0F3A 0x2d - invalid */
+/*  Opcode VEX.66.0F3A 0x2e - invalid */
+/*  Opcode VEX.66.0F3A 0x2f - invalid */
+
+
+/*  Opcode VEX.66.0F3A 0x30 - invalid */
+/*  Opcode VEX.66.0F3A 0x31 - invalid */
+/*  Opcode VEX.66.0F3A 0x32 - invalid */
+/*  Opcode VEX.66.0F3A 0x33 - invalid */
+/*  Opcode VEX.66.0F3A 0x34 - invalid */
+/*  Opcode VEX.66.0F3A 0x35 - invalid */
+/*  Opcode VEX.66.0F3A 0x36 - invalid */
+/*  Opcode VEX.66.0F3A 0x37 - invalid */
+/** Opcode VEX.66.0F3A 0x38 (vex only). */
+FNIEMOP_STUB(iemOp_vinserti128_Vqq_Hqq_Wqq_Ib);
+/** Opcode VEX.66.0F3A 0x39 (vex only). */
+FNIEMOP_STUB(iemOp_vextracti128_Wdq_Vqq_Ib);
+/*  Opcode VEX.66.0F3A 0x3a - invalid */
+/*  Opcode VEX.66.0F3A 0x3b - invalid */
+/*  Opcode VEX.66.0F3A 0x3c - invalid */
+/*  Opcode VEX.66.0F3A 0x3d - invalid */
+/*  Opcode VEX.66.0F3A 0x3e - invalid */
+/*  Opcode VEX.66.0F3A 0x3f - invalid */
+
+
+/** Opcode VEX.66.0F3A 0x40. */
+FNIEMOP_STUB(iemOp_vdpps_Vx_Hx_Wx_Ib);
+/** Opcode VEX.66.0F3A 0x41, */
+FNIEMOP_STUB(iemOp_vdppd_Vdq_Hdq_Wdq_Ib);
+/** Opcode VEX.66.0F3A 0x42. */
+FNIEMOP_STUB(iemOp_vmpsadbw_Vx_Hx_Wx_Ib);
+/*  Opcode VEX.66.0F3A 0x43 - invalid */
+/** Opcode VEX.66.0F3A 0x44. */
+FNIEMOP_STUB(iemOp_vpclmulqdq_Vdq_Hdq_Wdq_Ib);
+/*  Opcode VEX.66.0F3A 0x45 - invalid */
+/** Opcode VEX.66.0F3A 0x46 (vex only)  */
+FNIEMOP_STUB(iemOp_vperm2i128_Vqq_Hqq_Wqq_Ib);
+/*  Opcode VEX.66.0F3A 0x47 - invalid */
+/** Opcode VEX.66.0F3A 0x48 (AMD tables only). */
+FNIEMOP_STUB(iemOp_vperlmilzz2ps_Vx_Hx_Wp_Lx);
+/** Opcode VEX.66.0F3A 0x49 (AMD tables only). */
+FNIEMOP_STUB(iemOp_vperlmilzz2pd_Vx_Hx_Wp_Lx);
+/** Opcode VEX.66.0F3A 0x4a (vex only). */
+FNIEMOP_STUB(iemOp_vblendvps_Vx_Hx_Wx_Lx);
+/** Opcode VEX.66.0F3A 0x4b (vex only). */
+FNIEMOP_STUB(iemOp_vblendvpd_Vx_Hx_Wx_Lx);
+/** Opcode VEX.66.0F3A 0x4c (vex only). */
+FNIEMOP_STUB(iemOp_vpblendvb_Vx_Hx_Wx_Lx);
+/*  Opcode VEX.66.0F3A 0x4d - invalid */
+/*  Opcode VEX.66.0F3A 0x4e - invalid */
+/*  Opcode VEX.66.0F3A 0x4f - invalid */
+
+
+/*  Opcode VEX.66.0F3A 0x50 - invalid */
+/*  Opcode VEX.66.0F3A 0x51 - invalid */
+/*  Opcode VEX.66.0F3A 0x52 - invalid */
+/*  Opcode VEX.66.0F3A 0x53 - invalid */
+/*  Opcode VEX.66.0F3A 0x54 - invalid */
+/*  Opcode VEX.66.0F3A 0x55 - invalid */
+/*  Opcode VEX.66.0F3A 0x56 - invalid */
+/*  Opcode VEX.66.0F3A 0x57 - invalid */
+/*  Opcode VEX.66.0F3A 0x58 - invalid */
+/*  Opcode VEX.66.0F3A 0x59 - invalid */
+/*  Opcode VEX.66.0F3A 0x5a - invalid */
+/*  Opcode VEX.66.0F3A 0x5b - invalid */
+/** Opcode VEX.66.0F3A 0x5c (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmaddsubps_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x5d (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmaddsubpd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x5e (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmsubaddps_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x5f (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmsubaddpd_Vx_Lx_Wx_Hx);
+
+
+/** Opcode VEX.66.0F3A 0x60. */
+FNIEMOP_STUB(iemOp_vpcmpestrm_Vdq_Wdq_Ib);
+/** Opcode VEX.66.0F3A 0x61, */
+FNIEMOP_STUB(iemOp_vpcmpestri_Vdq_Wdq_Ib);
+/** Opcode VEX.66.0F3A 0x62. */
+FNIEMOP_STUB(iemOp_vpcmpistrm_Vdq_Wdq_Ib);
+/** Opcode VEX.66.0F3A 0x63*/
+FNIEMOP_STUB(iemOp_vpcmpistri_Vdq_Wdq_Ib);
+/*  Opcode VEX.66.0F3A 0x64 - invalid */
+/*  Opcode VEX.66.0F3A 0x65 - invalid */
+/*  Opcode VEX.66.0F3A 0x66 - invalid */
+/*  Opcode VEX.66.0F3A 0x67 - invalid */
+/** Opcode VEX.66.0F3A 0x68 (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmaddps_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x69 (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmaddpd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x6a (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmaddss_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x6b (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmaddsd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x6c (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmsubps_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x6d (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmsubpd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x6e (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmsubss_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x6f (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmsubsd_Vx_Lx_Wx_Hx);
+
+/*  Opcode VEX.66.0F3A 0x70 - invalid */
+/*  Opcode VEX.66.0F3A 0x71 - invalid */
+/*  Opcode VEX.66.0F3A 0x72 - invalid */
+/*  Opcode VEX.66.0F3A 0x73 - invalid */
+/*  Opcode VEX.66.0F3A 0x74 - invalid */
+/*  Opcode VEX.66.0F3A 0x75 - invalid */
+/*  Opcode VEX.66.0F3A 0x76 - invalid */
+/*  Opcode VEX.66.0F3A 0x77 - invalid */
+/** Opcode VEX.66.0F3A 0x78 (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmaddps_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x79 (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmaddpd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x7a (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmaddss_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x7b (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmaddsd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x7c (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmsubps_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x7d (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmsubpd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x7e (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmsubss_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x7f (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmsubsd_Vx_Lx_Wx_Hx);
+
+/*  Opcodes 0x0f 0x80 thru 0x0f 0xb0 are unused.  */
+
+
+/*  Opcode      0x0f 0xc0 - invalid */
+/*  Opcode      0x0f 0xc1 - invalid */
+/*  Opcode      0x0f 0xc2 - invalid */
+/*  Opcode      0x0f 0xc3 - invalid */
+/*  Opcode      0x0f 0xc4 - invalid */
+/*  Opcode      0x0f 0xc5 - invalid */
+/*  Opcode      0x0f 0xc6 - invalid */
+/*  Opcode      0x0f 0xc7 - invalid */
+/*  Opcode      0x0f 0xc8 - invalid */
+/*  Opcode      0x0f 0xc9 - invalid */
+/*  Opcode      0x0f 0xca - invalid */
+/*  Opcode      0x0f 0xcb - invalid */
+/*  Opcode      0x0f 0xcc */
+FNIEMOP_STUB(iemOp_vsha1rnds4_Vdq_Wdq_Ib);
+/*  Opcode      0x0f 0xcd - invalid */
+/*  Opcode      0x0f 0xce - invalid */
+/*  Opcode      0x0f 0xcf - invalid */
+
+
+/*  Opcode VEX.66.0F3A 0xd0 - invalid */
+/*  Opcode VEX.66.0F3A 0xd1 - invalid */
+/*  Opcode VEX.66.0F3A 0xd2 - invalid */
+/*  Opcode VEX.66.0F3A 0xd3 - invalid */
+/*  Opcode VEX.66.0F3A 0xd4 - invalid */
+/*  Opcode VEX.66.0F3A 0xd5 - invalid */
+/*  Opcode VEX.66.0F3A 0xd6 - invalid */
+/*  Opcode VEX.66.0F3A 0xd7 - invalid */
+/*  Opcode VEX.66.0F3A 0xd8 - invalid */
+/*  Opcode VEX.66.0F3A 0xd9 - invalid */
+/*  Opcode VEX.66.0F3A 0xda - invalid */
+/*  Opcode VEX.66.0F3A 0xdb - invalid */
+/*  Opcode VEX.66.0F3A 0xdc - invalid */
+/*  Opcode VEX.66.0F3A 0xdd - invalid */
+/*  Opcode VEX.66.0F3A 0xde - invalid */
+/*  Opcode VEX.66.0F3A 0xdf - (aeskeygenassist). */
+FNIEMOP_STUB(iemOp_vaeskeygen_Vdq_Wdq_Ib);
+
+
+/*  Opcode VEX.F2.0F3A (vex only) */
+FNIEMOP_STUB(iemOp_rorx_Gy_Ey_Ib);
+
+
+/**
+ * VEX opcode map \#3.
+ *
+ * @sa      g_apfnThreeByte0f3a
+ */
+IEM_STATIC const PFNIEMOP g_apfnVexMap3[] =
+{
+    /*          no prefix,                  066h prefix                 f3h prefix,                 f2h prefix */
+    /* 0x00 */  iemOp_InvalidNeedRMImm8,    iemOp_vpermq_Vqq_Wqq_Ib,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x01 */  iemOp_InvalidNeedRMImm8,    iemOp_vpermqd_Vqq_Wqq_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x02 */  iemOp_InvalidNeedRMImm8,    iemOp_vpblendd_Vx_Wx_Ib,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x03 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x04 */  iemOp_InvalidNeedRMImm8,    iemOp_vpermilps_Vx_Wx_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x05 */  iemOp_InvalidNeedRMImm8,    iemOp_vpermilpd_Vx_Wx_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x06 */  iemOp_InvalidNeedRMImm8,    iemOp_vperm2f128_Vqq_Hqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x07 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x08 */  iemOp_InvalidNeedRMImm8,    iemOp_vroundps_Vx_Wx_Ib,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x09 */  iemOp_InvalidNeedRMImm8,    iemOp_vroundpd_Vx_Wx_Ib,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0a */  iemOp_InvalidNeedRMImm8,    iemOp_vroundss_Vss_Wss_Ib,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0b */  iemOp_InvalidNeedRMImm8,    iemOp_vroundsd_Vsd_Wsd_Ib,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0c */  iemOp_InvalidNeedRMImm8,    iemOp_vblendps_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0d */  iemOp_InvalidNeedRMImm8,    iemOp_vblendpd_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0e */  iemOp_InvalidNeedRMImm8,    iemOp_vblendw_Vx_Hx_Wx_Ib,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0f */  iemOp_InvalidNeedRMImm8,    iemOp_vpalignr_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+
+    /* 0x10 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x11 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x12 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x13 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x14 */  iemOp_InvalidNeedRMImm8,    iemOp_vpextrb_RdMb_Vdq_Ib,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x15 */  iemOp_InvalidNeedRMImm8,    iemOp_vpextrw_RdMw_Vdq_Ib,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x16 */  iemOp_InvalidNeedRMImm8,    iemOp_vpextrd_q_RdMw_Vdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x17 */  iemOp_InvalidNeedRMImm8,    iemOp_vextractps_Ed_Vdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x18 */  iemOp_InvalidNeedRMImm8,    iemOp_vinsertf128_Vqq_Hqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x19 */  iemOp_InvalidNeedRMImm8,    iemOp_vextractf128_Wdq_Vqq_Ib, iemOp_InvalidNeedRMImm8,  iemOp_InvalidNeedRMImm8,
+    /* 0x1a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1d */  iemOp_InvalidNeedRMImm8,    iemOp_vcvtps2ph_Wx_Vx_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x1e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x20 */  iemOp_InvalidNeedRMImm8,    iemOp_vpinsrb_Vdq_Hdq_RyMb_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x21 */  iemOp_InvalidNeedRMImm8,    iemOp_vinsertps_Vdq_Hdq_UdqMd_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x22 */  iemOp_InvalidNeedRMImm8,    iemOp_vpinsrd_q_Vdq_Hdq_Ey_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x23 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x24 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x25 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x26 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x27 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x28 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x29 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x30 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x31 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x32 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x33 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x34 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x35 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x36 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x37 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x38 */  iemOp_InvalidNeedRMImm8,    iemOp_vinserti128_Vqq_Hqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x39 */  iemOp_InvalidNeedRMImm8,    iemOp_vextracti128_Wdq_Vqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x3a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x40 */  iemOp_InvalidNeedRMImm8,    iemOp_vdpps_Vx_Hx_Wx_Ib,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x41 */  iemOp_InvalidNeedRMImm8,    iemOp_vdppd_Vdq_Hdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x42 */  iemOp_InvalidNeedRMImm8,    iemOp_vmpsadbw_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x43 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x44 */  iemOp_InvalidNeedRMImm8,    iemOp_vpclmulqdq_Vdq_Hdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x45 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x46 */  iemOp_InvalidNeedRMImm8,    iemOp_vperm2i128_Vqq_Hqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x47 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x48 */  iemOp_InvalidNeedRMImm8,    iemOp_vperlmilzz2ps_Vx_Hx_Wp_Lx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x49 */  iemOp_InvalidNeedRMImm8,    iemOp_vperlmilzz2pd_Vx_Hx_Wp_Lx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x4a */  iemOp_InvalidNeedRMImm8,    iemOp_vblendvps_Vx_Hx_Wx_Lx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x4b */  iemOp_InvalidNeedRMImm8,    iemOp_vblendvpd_Vx_Hx_Wx_Lx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x4c */  iemOp_InvalidNeedRMImm8,    iemOp_vpblendvb_Vx_Hx_Wx_Lx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x4d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x50 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x51 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x52 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x53 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x54 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x55 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x56 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x57 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x58 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x59 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5c */  iemOp_InvalidNeedRMImm8,    iemOp_vfmaddsubps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x5d */  iemOp_InvalidNeedRMImm8,    iemOp_vfmaddsubpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x5e */  iemOp_InvalidNeedRMImm8,    iemOp_vfmsubaddps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x5f */  iemOp_InvalidNeedRMImm8,    iemOp_vfmsubaddpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+
+    /* 0x60 */  iemOp_InvalidNeedRMImm8,    iemOp_vpcmpestrm_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x61 */  iemOp_InvalidNeedRMImm8,    iemOp_vpcmpestri_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x62 */  iemOp_InvalidNeedRMImm8,    iemOp_vpcmpistrm_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x63 */  iemOp_InvalidNeedRMImm8,    iemOp_vpcmpistri_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x64 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x65 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x66 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x67 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x68 */  iemOp_InvalidNeedRMImm8,    iemOp_vfmaddps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x69 */  iemOp_InvalidNeedRMImm8,    iemOp_vfmaddpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x6a */  iemOp_InvalidNeedRMImm8,    iemOp_vfmaddss_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x6b */  iemOp_InvalidNeedRMImm8,    iemOp_vfmaddsd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x6c */  iemOp_InvalidNeedRMImm8,    iemOp_vfmsubps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x6d */  iemOp_InvalidNeedRMImm8,    iemOp_vfmsubpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x6e */  iemOp_InvalidNeedRMImm8,    iemOp_vfmsubss_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x6f */  iemOp_InvalidNeedRMImm8,    iemOp_vfmsubsd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+
+    /* 0x70 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x71 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x72 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x73 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x74 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x75 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x76 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x77 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x78 */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmaddps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x79 */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmaddpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x7a */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmaddss_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x7b */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmaddsd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x7c */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmsubps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x7d */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmsubpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x7e */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmsubss_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x7f */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmsubsd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+
+    /* 0x80 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x81 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x82 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x83 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x84 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x85 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x86 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x87 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x88 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x89 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x90 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x91 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x92 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x93 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x94 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x95 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x96 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x97 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x98 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x99 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xa0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xaa */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xab */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xac */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xad */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xae */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xaf */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xb0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xba */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbc */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbe */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbf */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xc0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xca */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xcb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xcc */  iemOp_vsha1rnds4_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0xcd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xce */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xcf */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xd0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xda */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdc */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xde */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdf */  iemOp_vaeskeygen_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+
+    /* 0xe0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xea */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xeb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xec */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xed */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xee */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xef */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xf0 */  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,    iemOp_rorx_Gy_Ey_Ib,
+    /* 0xf1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfa */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfc */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfe */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xff */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexMap3) == 1024);
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IOMAll.cpp b/src/VBox/VMM/VMMAll/IOMAll.cpp
new file mode 100644
index 00000000..ff1afcb4
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IOMAll.cpp
@@ -0,0 +1,596 @@
+/* $Id: IOMAll.cpp $ */
+/** @file
+ * IOM - Input / Output Monitor - Any Context.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_IOM_IOPORT
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/param.h>
+#include "IOMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include "IOMInline.h"
+
+
+/**
+ * Reads an I/O port register.
+ *
+ * @returns Strict VBox status code. Informational status codes other than the one documented
+ *          here are to be treated as internal failure. Use IOM_SUCCESS() to check for success.
+ * @retval  VINF_SUCCESS                Success.
+ * @retval  VINF_EM_FIRST-VINF_EM_LAST  Success with some exceptions (see IOM_SUCCESS()), the
+ *                                      status code must be passed on to EM.
+ * @retval  VINF_IOM_R3_IOPORT_READ     Defer the read to ring-3. (R0/RC only)
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   Port        The port to read.
+ * @param   pu32Value   Where to store the value read.
+ * @param   cbValue     The size of the register to read in bytes. 1, 2 or 4 bytes.
+ */
+VMMDECL(VBOXSTRICTRC) IOMIOPortRead(PVMCC pVM, PVMCPU pVCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue)
+{
+    STAM_COUNTER_INC(&pVM->iom.s.StatIoPortIn);
+    Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0);
+
+/** @todo should initialize *pu32Value here because it can happen that some
+ *        handle is buggy and doesn't handle all cases. */
+
+    /* For lookups we need to share lock IOM. */
+    int rc2 = IOM_LOCK_SHARED(pVM);
+#ifndef IN_RING3
+    if (rc2 == VERR_SEM_BUSY)
+        return VINF_IOM_R3_IOPORT_READ;
+#endif
+    AssertRC(rc2);
+
+    /*
+     * Get the entry for the current context.
+     */
+    uint16_t offPort;
+    CTX_SUFF(PIOMIOPORTENTRY) pRegEntry = iomIoPortGetEntry(pVM, Port, &offPort, &pVCpu->iom.s.idxIoPortLastRead);
+    if (pRegEntry)
+    {
+#ifdef VBOX_WITH_STATISTICS
+        PIOMIOPORTSTATSENTRY  pStats    = iomIoPortGetStats(pVM, pRegEntry, offPort);
+#endif
+
+        /*
+         * Found an entry, get the data so we can leave the IOM lock.
+         */
+        uint16_t const    fFlags        = pRegEntry->fFlags;
+        PFNIOMIOPORTNEWIN pfnInCallback = pRegEntry->pfnInCallback;
+        PPDMDEVINS        pDevIns       = pRegEntry->pDevIns;
+#ifndef IN_RING3
+        if (   pfnInCallback
+            && pDevIns
+            && pRegEntry->cPorts > 0)
+        { /* likely */ }
+        else
+        {
+            STAM_COUNTER_INC(&pStats->InRZToR3);
+            IOM_UNLOCK_SHARED(pVM);
+            return VINF_IOM_R3_IOPORT_READ;
+        }
+#endif
+        void           *pvUser       = pRegEntry->pvUser;
+        IOM_UNLOCK_SHARED(pVM);
+        AssertPtr(pDevIns);
+        AssertPtr(pfnInCallback);
+
+        /*
+         * Call the device.
+         */
+        VBOXSTRICTRC rcStrict = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_IOPORT_READ);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfIn), a);
+            rcStrict = pfnInCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? Port : offPort, pu32Value, (unsigned)cbValue);
+            STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfIn), a);
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
+
+#ifndef IN_RING3
+            if (rcStrict == VINF_IOM_R3_IOPORT_READ)
+                STAM_COUNTER_INC(&pStats->InRZToR3);
+            else
+#endif
+            {
+                STAM_COUNTER_INC(&pStats->CTX_SUFF_Z(In));
+                STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total);
+                if (rcStrict == VERR_IOM_IOPORT_UNUSED)
+                {
+                    /* make return value */
+                    rcStrict = VINF_SUCCESS;
+                    switch (cbValue)
+                    {
+                        case 1: *(uint8_t  *)pu32Value = 0xff; break;
+                        case 2: *(uint16_t *)pu32Value = 0xffff; break;
+                        case 4: *(uint32_t *)pu32Value = UINT32_C(0xffffffff); break;
+                        default:
+                            AssertMsgFailedReturn(("Invalid I/O port size %d. Port=%d\n", cbValue, Port), VERR_IOM_INVALID_IOPORT_SIZE);
+                    }
+                }
+           }
+            Log3(("IOMIOPortRead: Port=%RTiop *pu32=%08RX32 cb=%d rc=%Rrc\n", Port, *pu32Value, cbValue, VBOXSTRICTRC_VAL(rcStrict)));
+            STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total);
+        }
+        else
+            STAM_COUNTER_INC(&pStats->InRZToR3);
+        return rcStrict;
+    }
+
+    /*
+     * Ok, no handler for this port.
+     */
+    IOM_UNLOCK_SHARED(pVM);
+    switch (cbValue)
+    {
+        case 1: *(uint8_t  *)pu32Value = 0xff; break;
+        case 2: *(uint16_t *)pu32Value = 0xffff; break;
+        case 4: *(uint32_t *)pu32Value = UINT32_C(0xffffffff); break;
+        default:
+            AssertMsgFailedReturn(("Invalid I/O port size %d. Port=%d\n", cbValue, Port), VERR_IOM_INVALID_IOPORT_SIZE);
+    }
+    Log3(("IOMIOPortRead: Port=%RTiop *pu32=%08RX32 cb=%d rc=VINF_SUCCESS\n", Port, *pu32Value, cbValue));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Reads the string buffer of an I/O port register.
+ *
+ * @returns Strict VBox status code. Informational status codes other than the one documented
+ *          here are to be treated as internal failure. Use IOM_SUCCESS() to check for success.
+ * @retval  VINF_SUCCESS                Success or no string I/O callback in
+ *                                      this context.
+ * @retval  VINF_EM_FIRST-VINF_EM_LAST  Success with some exceptions (see IOM_SUCCESS()), the
+ *                                      status code must be passed on to EM.
+ * @retval  VINF_IOM_R3_IOPORT_READ     Defer the read to ring-3. (R0/RC only)
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uPort       The port to read.
+ * @param   pvDst       Pointer to the destination buffer.
+ * @param   pcTransfers Pointer to the number of transfer units to read, on return remaining transfer units.
+ * @param   cb          Size of the transfer unit (1, 2 or 4 bytes).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IOMIOPortReadString(PVMCC pVM, PVMCPU pVCpu, RTIOPORT uPort,
+                                               void *pvDst, uint32_t *pcTransfers, unsigned cb)
+{
+    STAM_COUNTER_INC(&pVM->iom.s.StatIoPortInS);
+    Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0);
+
+    /* For lookups we need to share lock IOM. */
+    int rc2 = IOM_LOCK_SHARED(pVM);
+#ifndef IN_RING3
+    if (rc2 == VERR_SEM_BUSY)
+        return VINF_IOM_R3_IOPORT_READ;
+#endif
+    AssertRC(rc2);
+
+    const uint32_t cRequestedTransfers = *pcTransfers;
+    Assert(cRequestedTransfers > 0);
+
+    /*
+     * Get the entry for the current context.
+     */
+    uint16_t offPort;
+    CTX_SUFF(PIOMIOPORTENTRY) pRegEntry = iomIoPortGetEntry(pVM, uPort, &offPort, &pVCpu->iom.s.idxIoPortLastReadStr);
+    if (pRegEntry)
+    {
+#ifdef VBOX_WITH_STATISTICS
+        PIOMIOPORTSTATSENTRY  pStats    = iomIoPortGetStats(pVM, pRegEntry, offPort);
+#endif
+
+        /*
+         * Found an entry, get the data so we can leave the IOM lock.
+         */
+        uint16_t const          fFlags           = pRegEntry->fFlags;
+        PFNIOMIOPORTNEWINSTRING pfnInStrCallback = pRegEntry->pfnInStrCallback;
+        PFNIOMIOPORTNEWIN       pfnInCallback    = pRegEntry->pfnInCallback;
+        PPDMDEVINS              pDevIns          = pRegEntry->pDevIns;
+#ifndef IN_RING3
+        if (   pfnInCallback
+            && pDevIns
+            && pRegEntry->cPorts > 0)
+        { /* likely */ }
+        else
+        {
+            STAM_COUNTER_INC(&pStats->InRZToR3);
+            IOM_UNLOCK_SHARED(pVM);
+            return VINF_IOM_R3_IOPORT_READ;
+        }
+#endif
+        void           *pvUser       = pRegEntry->pvUser;
+        IOM_UNLOCK_SHARED(pVM);
+        AssertPtr(pDevIns);
+        AssertPtr(pfnInCallback);
+
+        /*
+         * Call the device.
+         */
+        VBOXSTRICTRC rcStrict = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_IOPORT_READ);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            /*
+             * First using the string I/O callback.
+             */
+            if (pfnInStrCallback)
+            {
+                STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfIn), a);
+                rcStrict = pfnInStrCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? uPort : offPort,
+                                            (uint8_t *)pvDst, pcTransfers, cb);
+                STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfIn), a);
+            }
+
+            /*
+             * Then doing the single I/O fallback.
+             */
+            if (   *pcTransfers > 0
+                && rcStrict == VINF_SUCCESS)
+            {
+                pvDst = (uint8_t *)pvDst + (cRequestedTransfers - *pcTransfers) * cb;
+                do
+                {
+                    uint32_t u32Value = 0;
+                    STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfIn), a);
+                    rcStrict = pfnInCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? uPort : offPort, &u32Value, cb);
+                    STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfIn), a);
+                    if (rcStrict == VERR_IOM_IOPORT_UNUSED)
+                    {
+                        u32Value = UINT32_MAX;
+                        rcStrict = VINF_SUCCESS;
+                    }
+                    if (IOM_SUCCESS(rcStrict))
+                    {
+                        switch (cb)
+                        {
+                            case 4: *(uint32_t *)pvDst =           u32Value; pvDst = (uint8_t *)pvDst + 4; break;
+                            case 2: *(uint16_t *)pvDst = (uint16_t)u32Value; pvDst = (uint8_t *)pvDst + 2; break;
+                            case 1: *(uint8_t  *)pvDst = (uint8_t )u32Value; pvDst = (uint8_t *)pvDst + 1; break;
+                            default: AssertFailed();
+                        }
+                        *pcTransfers -= 1;
+                    }
+                } while (   *pcTransfers > 0
+                         && rcStrict == VINF_SUCCESS);
+            }
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
+
+#ifdef VBOX_WITH_STATISTICS
+# ifndef IN_RING3
+            if (rcStrict == VINF_IOM_R3_IOPORT_READ)
+                STAM_COUNTER_INC(&pStats->InRZToR3);
+            else
+# endif
+            {
+                STAM_COUNTER_INC(&pStats->CTX_SUFF_Z(In));
+                STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total);
+            }
+#endif
+            Log3(("IOMIOPortReadStr: uPort=%RTiop pvDst=%p pcTransfer=%p:{%#x->%#x} cb=%d rc=%Rrc\n",
+                  uPort, pvDst, pcTransfers, cRequestedTransfers, *pcTransfers, cb, VBOXSTRICTRC_VAL(rcStrict)));
+        }
+#ifndef IN_RING3
+        else
+            STAM_COUNTER_INC(&pStats->InRZToR3);
+#endif
+        return rcStrict;
+    }
+
+    /*
+     * Ok, no handler for this port.
+     */
+    IOM_UNLOCK_SHARED(pVM);
+    *pcTransfers = 0;
+    memset(pvDst, 0xff, cRequestedTransfers * cb);
+    Log3(("IOMIOPortReadStr: uPort=%RTiop (unused) pvDst=%p pcTransfer=%p:{%#x->%#x} cb=%d rc=VINF_SUCCESS\n",
+          uPort, pvDst, pcTransfers, cRequestedTransfers, *pcTransfers, cb));
+    return VINF_SUCCESS;
+}
+
+
+#ifndef IN_RING3
+/**
+ * Defers a pending I/O port write to ring-3.
+ *
+ * @returns VINF_IOM_R3_IOPORT_COMMIT_WRITE
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   Port        The port to write to.
+ * @param   u32Value    The value to write.
+ * @param   cbValue     The size of the value (1, 2, 4).
+ */
+static VBOXSTRICTRC iomIOPortRing3WritePending(PVMCPU pVCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
+{
+    Log5(("iomIOPortRing3WritePending: %#x LB %u -> %RTiop\n", u32Value, cbValue, Port));
+    AssertReturn(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0, VERR_IOM_IOPORT_IPE_1);
+    pVCpu->iom.s.PendingIOPortWrite.IOPort   = Port;
+    pVCpu->iom.s.PendingIOPortWrite.u32Value = u32Value;
+    pVCpu->iom.s.PendingIOPortWrite.cbValue  = (uint32_t)cbValue;
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
+    return VINF_IOM_R3_IOPORT_COMMIT_WRITE;
+}
+#endif
+
+
+/**
+ * Writes to an I/O port register.
+ *
+ * @returns Strict VBox status code. Informational status codes other than the one documented
+ *          here are to be treated as internal failure. Use IOM_SUCCESS() to check for success.
+ * @retval  VINF_SUCCESS                Success.
+ * @retval  VINF_EM_FIRST-VINF_EM_LAST  Success with some exceptions (see IOM_SUCCESS()), the
+ *                                      status code must be passed on to EM.
+ * @retval  VINF_IOM_R3_IOPORT_WRITE    Defer the write to ring-3. (R0/RC only)
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   Port        The port to write to.
+ * @param   u32Value    The value to write.
+ * @param   cbValue     The size of the register to read in bytes. 1, 2 or 4 bytes.
+ */
+VMMDECL(VBOXSTRICTRC) IOMIOPortWrite(PVMCC pVM, PVMCPU pVCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
+{
+    STAM_COUNTER_INC(&pVM->iom.s.StatIoPortOut);
+#ifndef IN_RING3
+    Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0);
+#endif
+
+    /* For lookups we need to share lock IOM. */
+    int rc2 = IOM_LOCK_SHARED(pVM);
+#ifndef IN_RING3
+    if (rc2 == VERR_SEM_BUSY)
+        return iomIOPortRing3WritePending(pVCpu, Port, u32Value, cbValue);
+#endif
+    AssertRC(rc2);
+
+    /*
+     * Get the entry for the current context.
+     */
+    uint16_t offPort;
+    CTX_SUFF(PIOMIOPORTENTRY) pRegEntry = iomIoPortGetEntry(pVM, Port, &offPort, &pVCpu->iom.s.idxIoPortLastWrite);
+    if (pRegEntry)
+    {
+#ifdef VBOX_WITH_STATISTICS
+        PIOMIOPORTSTATSENTRY  pStats    = iomIoPortGetStats(pVM, pRegEntry, offPort);
+#endif
+
+        /*
+         * Found an entry, get the data so we can leave the IOM lock.
+         */
+        uint16_t const     fFlags           = pRegEntry->fFlags;
+        PFNIOMIOPORTNEWOUT pfnOutCallback   = pRegEntry->pfnOutCallback;
+        PPDMDEVINS         pDevIns          = pRegEntry->pDevIns;
+#ifndef IN_RING3
+        if (   pfnOutCallback
+            && pDevIns
+            && pRegEntry->cPorts > 0)
+        { /* likely */ }
+        else
+        {
+            IOM_UNLOCK_SHARED(pVM);
+            STAM_COUNTER_INC(&pStats->OutRZToR3);
+            return iomIOPortRing3WritePending(pVCpu, Port, u32Value, cbValue);
+        }
+#endif
+        void           *pvUser       = pRegEntry->pvUser;
+        IOM_UNLOCK_SHARED(pVM);
+        AssertPtr(pDevIns);
+        AssertPtr(pfnOutCallback);
+
+        /*
+         * Call the device.
+         */
+        VBOXSTRICTRC rcStrict = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_IOPORT_WRITE);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfOut), a);
+            rcStrict = pfnOutCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? Port : offPort, u32Value, (unsigned)cbValue);
+            STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfOut), a);
+
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
+
+#ifdef VBOX_WITH_STATISTICS
+# ifndef IN_RING3
+            if (rcStrict != VINF_IOM_R3_IOPORT_WRITE)
+# endif
+            {
+                STAM_COUNTER_INC(&pStats->CTX_SUFF_Z(Out));
+                STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total);
+            }
+#endif
+            Log3(("IOMIOPortWrite: Port=%RTiop u32=%08RX32 cb=%d rc=%Rrc\n", Port, u32Value, cbValue, VBOXSTRICTRC_VAL(rcStrict)));
+        }
+#ifndef IN_RING3
+        if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
+        {
+            STAM_COUNTER_INC(&pStats->OutRZToR3);
+            return iomIOPortRing3WritePending(pVCpu, Port, u32Value, cbValue);
+        }
+#endif
+        return rcStrict;
+    }
+
+    /*
+     * Ok, no handler for that port.
+     */
+    IOM_UNLOCK_SHARED(pVM);
+    Log3(("IOMIOPortWrite: Port=%RTiop u32=%08RX32 cb=%d nop\n", Port, u32Value, cbValue));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Writes the string buffer of an I/O port register.
+ *
+ * @returns Strict VBox status code. Informational status codes other than the one documented
+ *          here are to be treated as internal failure. Use IOM_SUCCESS() to check for success.
+ * @retval  VINF_SUCCESS                Success or no string I/O callback in
+ *                                      this context.
+ * @retval  VINF_EM_FIRST-VINF_EM_LAST  Success with some exceptions (see IOM_SUCCESS()), the
+ *                                      status code must be passed on to EM.
+ * @retval  VINF_IOM_R3_IOPORT_WRITE    Defer the write to ring-3. (R0/RC only)
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uPort       The port to write to.
+ * @param   pvSrc       The guest page to read from.
+ * @param   pcTransfers Pointer to the number of transfer units to write, on
+ *                      return remaining transfer units.
+ * @param   cb          Size of the transfer unit (1, 2 or 4 bytes).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IOMIOPortWriteString(PVMCC pVM, PVMCPU pVCpu, RTIOPORT uPort, void const *pvSrc,
+                                                uint32_t *pcTransfers, unsigned cb)
+{
+    STAM_COUNTER_INC(&pVM->iom.s.StatIoPortOutS);
+    Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0);
+    Assert(cb == 1 || cb == 2 || cb == 4);
+
+    /* Take the IOM lock before performing any device I/O. */
+    int rc2 = IOM_LOCK_SHARED(pVM);
+#ifndef IN_RING3
+    if (rc2 == VERR_SEM_BUSY)
+        return VINF_IOM_R3_IOPORT_WRITE;
+#endif
+    AssertRC(rc2);
+
+    const uint32_t cRequestedTransfers = *pcTransfers;
+    Assert(cRequestedTransfers > 0);
+
+    /*
+     * Get the entry for the current context.
+     */
+    uint16_t offPort;
+    CTX_SUFF(PIOMIOPORTENTRY) pRegEntry = iomIoPortGetEntry(pVM, uPort, &offPort, &pVCpu->iom.s.idxIoPortLastWriteStr);
+    if (pRegEntry)
+    {
+#ifdef VBOX_WITH_STATISTICS
+        PIOMIOPORTSTATSENTRY  pStats    = iomIoPortGetStats(pVM, pRegEntry, offPort);
+#endif
+
+        /*
+         * Found an entry, get the data so we can leave the IOM lock.
+         */
+        uint16_t const             fFlags            = pRegEntry->fFlags;
+        PFNIOMIOPORTNEWOUTSTRING   pfnOutStrCallback = pRegEntry->pfnOutStrCallback;
+        PFNIOMIOPORTNEWOUT         pfnOutCallback    = pRegEntry->pfnOutCallback;
+        PPDMDEVINS                 pDevIns           = pRegEntry->pDevIns;
+#ifndef IN_RING3
+        if (   pfnOutCallback
+            && pDevIns
+            && pRegEntry->cPorts > 0)
+        { /* likely */ }
+        else
+        {
+            IOM_UNLOCK_SHARED(pVM);
+            STAM_COUNTER_INC(&pStats->OutRZToR3);
+            return VINF_IOM_R3_IOPORT_WRITE;
+        }
+#endif
+        void           *pvUser       = pRegEntry->pvUser;
+        IOM_UNLOCK_SHARED(pVM);
+        AssertPtr(pDevIns);
+        AssertPtr(pfnOutCallback);
+
+        /*
+         * Call the device.
+         */
+        VBOXSTRICTRC rcStrict = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_IOPORT_WRITE);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            /*
+             * First using string I/O if possible.
+             */
+            if (pfnOutStrCallback)
+            {
+                STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfOut), a);
+                rcStrict = pfnOutStrCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? uPort : offPort,
+                                             (uint8_t const *)pvSrc, pcTransfers, cb);
+                STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfOut), a);
+            }
+
+            /*
+             * Then doing the single I/O fallback.
+             */
+            if (   *pcTransfers > 0
+                && rcStrict == VINF_SUCCESS)
+            {
+                pvSrc = (uint8_t *)pvSrc + (cRequestedTransfers - *pcTransfers) * cb;
+                do
+                {
+                    uint32_t u32Value;
+                    switch (cb)
+                    {
+                        case 4: u32Value = *(uint32_t *)pvSrc; pvSrc = (uint8_t const *)pvSrc + 4; break;
+                        case 2: u32Value = *(uint16_t *)pvSrc; pvSrc = (uint8_t const *)pvSrc + 2; break;
+                        case 1: u32Value = *(uint8_t  *)pvSrc; pvSrc = (uint8_t const *)pvSrc + 1; break;
+                        default: AssertFailed(); u32Value = UINT32_MAX;
+                    }
+                    STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfOut), a);
+                    rcStrict = pfnOutCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? uPort : offPort, u32Value, cb);
+                    STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfOut), a);
+                    if (IOM_SUCCESS(rcStrict))
+                        *pcTransfers -= 1;
+                } while (   *pcTransfers > 0
+                         && rcStrict == VINF_SUCCESS);
+            }
+
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
+
+#ifdef VBOX_WITH_STATISTICS
+# ifndef IN_RING3
+            if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
+                STAM_COUNTER_INC(&pStats->OutRZToR3);
+            else
+# endif
+            {
+                STAM_COUNTER_INC(&pStats->CTX_SUFF_Z(Out));
+                STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total);
+            }
+#endif
+            Log3(("IOMIOPortWriteStr: uPort=%RTiop pvSrc=%p pcTransfer=%p:{%#x->%#x} cb=%d rcStrict=%Rrc\n",
+                  uPort, pvSrc, pcTransfers, cRequestedTransfers, *pcTransfers, cb, VBOXSTRICTRC_VAL(rcStrict)));
+        }
+#ifndef IN_RING3
+        else
+            STAM_COUNTER_INC(&pStats->OutRZToR3);
+#endif
+        return rcStrict;
+    }
+
+    /*
+     * Ok, no handler for this port.
+     */
+    IOM_UNLOCK_SHARED(pVM);
+    *pcTransfers = 0;
+    Log3(("IOMIOPortWriteStr: uPort=%RTiop (unused) pvSrc=%p pcTransfer=%p:{%#x->%#x} cb=%d rc=VINF_SUCCESS\n",
+          uPort, pvSrc, pcTransfers, cRequestedTransfers, *pcTransfers, cb));
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp b/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp
new file mode 100644
index 00000000..9e31039a
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp
@@ -0,0 +1,1274 @@
+/* $Id: IOMAllMmioNew.cpp $ */
+/** @file
+ * IOM - Input / Output Monitor - Any Context, MMIO & String I/O.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_IOM_MMIO
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/iem.h>
+#include "IOMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/hm.h>
+#include "IOMInline.h"
+
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** @def IOM_MMIO_STATS_COMMA_DECL
+ * Parameter list declaration for statistics entry pointer. */
+/** @def IOM_MMIO_STATS_COMMA_ARG
+ * Statistics entry pointer argument. */
+#if defined(VBOX_WITH_STATISTICS) || defined(DOXYGEN_RUNNING)
+# define IOM_MMIO_STATS_COMMA_DECL , PIOMMMIOSTATSENTRY pStats
+# define IOM_MMIO_STATS_COMMA_ARG  , pStats
+#else
+# define IOM_MMIO_STATS_COMMA_DECL
+# define IOM_MMIO_STATS_COMMA_ARG
+#endif
+
+
+
+#ifndef IN_RING3
+/**
+ * Defers a pending MMIO write to ring-3.
+ *
+ * @returns VINF_IOM_R3_MMIO_COMMIT_WRITE
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPhys      The write address.
+ * @param   pvBuf       The bytes being written.
+ * @param   cbBuf       How many bytes.
+ * @param   idxRegEntry The MMIO registration index (handle) if available,
+ *                      otherwise UINT32_MAX.
+ */
+static VBOXSTRICTRC iomMmioRing3WritePending(PVMCPU pVCpu, RTGCPHYS GCPhys, void const *pvBuf, size_t cbBuf,
+                                             uint32_t idxRegEntry)
+{
+    Log5(("iomMmioRing3WritePending: %RGp LB %#x (idx=%#x)\n", GCPhys, cbBuf, idxRegEntry));
+    if (pVCpu->iom.s.PendingMmioWrite.cbValue == 0)
+    {
+        pVCpu->iom.s.PendingMmioWrite.GCPhys            = GCPhys;
+        AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
+        pVCpu->iom.s.PendingMmioWrite.cbValue           = (uint32_t)cbBuf;
+        pVCpu->iom.s.PendingMmioWrite.idxMmioRegionHint = idxRegEntry;
+        memcpy(pVCpu->iom.s.PendingMmioWrite.abValue, pvBuf, cbBuf);
+    }
+    else
+    {
+        /*
+         * Join with pending if adjecent.
+         *
+         * This may happen if the stack overflows into MMIO territory and RSP/ESP/SP
+         * isn't aligned. IEM will bounce buffer the access and do one write for each
+         * page.  We get here when the 2nd page part is written.
+         */
+        uint32_t const cbOldValue = pVCpu->iom.s.PendingMmioWrite.cbValue;
+        AssertMsgReturn(GCPhys == pVCpu->iom.s.PendingMmioWrite.GCPhys + cbOldValue,
+                        ("pending %RGp LB %#x; incoming %RGp LB %#x\n",
+                         pVCpu->iom.s.PendingMmioWrite.GCPhys, cbOldValue, GCPhys, cbBuf),
+                        VERR_IOM_MMIO_IPE_1);
+        AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue) - cbOldValue, VERR_IOM_MMIO_IPE_2);
+        pVCpu->iom.s.PendingMmioWrite.cbValue = cbOldValue + (uint32_t)cbBuf;
+        memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[cbOldValue], pvBuf, cbBuf);
+    }
+
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
+    return VINF_IOM_R3_MMIO_COMMIT_WRITE;
+}
+#endif
+
+
+/**
+ * Deals with complicated MMIO writes.
+ *
+ * Complicated means unaligned or non-dword/qword sized accesses depending on
+ * the MMIO region's access mode flags.
+ *
+ * @returns Strict VBox status code. Any EM scheduling status code,
+ *          VINF_IOM_R3_MMIO_WRITE, VINF_IOM_R3_MMIO_READ_WRITE or
+ *          VINF_IOM_R3_MMIO_READ may be returned.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pRegEntry   The MMIO entry for the current context.
+ * @param   GCPhys      The physical address to start writing.
+ * @param   offRegion   MMIO region offset corresponding to @a GCPhys.
+ * @param   pvValue     Where to store the value.
+ * @param   cbValue     The size of the value to write.
+ * @param   pStats      Pointer to the statistics (never NULL).
+ */
+static VBOXSTRICTRC iomMmioDoComplicatedWrite(PVM pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry,
+                                              RTGCPHYS GCPhys, RTGCPHYS offRegion,
+                                              void const *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    AssertReturn(   (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) != IOMMMIO_FLAGS_WRITE_PASSTHRU
+                 && (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING,
+                 VERR_IOM_MMIO_IPE_1);
+    AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
+    RTGCPHYS const GCPhysStart  = GCPhys; NOREF(GCPhysStart);
+    bool const     fReadMissing = (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_READ_MISSING
+                               || (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING;
+    RT_NOREF_PV(pVCpu); /* ring-3 */
+
+    /*
+     * Do debug stop if requested.
+     */
+    VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
+#ifdef VBOX_STRICT
+    if (!(pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_WRITE))
+    { /* likely */ }
+    else
+    {
+# ifdef IN_RING3
+        LogRel(("IOM: Complicated write %#x byte at %RGp to %s, initiating debugger intervention\n", cbValue, GCPhys,
+                R3STRING(pRegEntry->pszDesc)));
+        rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
+                            "Complicated write %#x byte at %RGp to %s\n", cbValue, GCPhys, pRegEntry->pszDesc);
+        if (rc == VERR_DBGF_NOT_ATTACHED)
+            rc = VINF_SUCCESS;
+# else
+        return VINF_IOM_R3_MMIO_WRITE;
+# endif
+    }
+#endif
+
+    STAM_COUNTER_INC(&pStats->ComplicatedWrites);
+
+    /*
+     * Check if we should ignore the write.
+     */
+    if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD)
+    {
+        Assert(cbValue != 4 || (GCPhys & 3));
+        return VINF_SUCCESS;
+    }
+    if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD)
+    {
+        Assert((cbValue != 4 && cbValue != 8) || (GCPhys & (cbValue - 1)));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Split and conquer.
+     */
+    for (;;)
+    {
+        unsigned const  offAccess  = GCPhys & 3;
+        unsigned        cbThisPart = 4 - offAccess;
+        if (cbThisPart > cbValue)
+            cbThisPart = cbValue;
+
+        /*
+         * Get the missing bits (if any).
+         */
+        uint32_t u32MissingValue = 0;
+        if (fReadMissing && cbThisPart != 4)
+        {
+            VBOXSTRICTRC rc2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                          !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
+                                                          ? offRegion & ~(RTGCPHYS)3 : (GCPhys & ~(RTGCPHYS)3),
+                                                          &u32MissingValue, sizeof(u32MissingValue));
+            switch (VBOXSTRICTRC_VAL(rc2))
+            {
+                case VINF_SUCCESS:
+                    break;
+                case VINF_IOM_MMIO_UNUSED_FF:
+                    STAM_COUNTER_INC(&pStats->FFor00Reads);
+                    u32MissingValue = UINT32_C(0xffffffff);
+                    break;
+                case VINF_IOM_MMIO_UNUSED_00:
+                    STAM_COUNTER_INC(&pStats->FFor00Reads);
+                    u32MissingValue = 0;
+                    break;
+#ifndef IN_RING3
+                case VINF_IOM_R3_MMIO_READ:
+                case VINF_IOM_R3_MMIO_READ_WRITE:
+                case VINF_IOM_R3_MMIO_WRITE:
+                    LogFlow(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
+                    rc2 = iomMmioRing3WritePending(pVCpu, GCPhys, pvValue, cbValue, pRegEntry->idxSelf);
+                    if (rc == VINF_SUCCESS || rc2 < rc)
+                        rc = rc2;
+                    return rc;
+#endif
+                default:
+                    if (RT_FAILURE(rc2))
+                    {
+                        Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
+                        return rc2;
+                    }
+                    AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                    if (rc == VINF_SUCCESS || rc2 < rc)
+                        rc = rc2;
+                    break;
+            }
+        }
+
+        /*
+         * Merge missing and given bits.
+         */
+        uint32_t u32GivenMask;
+        uint32_t u32GivenValue;
+        switch (cbThisPart)
+        {
+            case 1:
+                u32GivenValue = *(uint8_t  const *)pvValue;
+                u32GivenMask  = UINT32_C(0x000000ff);
+                break;
+            case 2:
+                u32GivenValue = *(uint16_t const *)pvValue;
+                u32GivenMask  = UINT32_C(0x0000ffff);
+                break;
+            case 3:
+                u32GivenValue = RT_MAKE_U32_FROM_U8(((uint8_t const *)pvValue)[0], ((uint8_t const *)pvValue)[1],
+                                                    ((uint8_t const *)pvValue)[2], 0);
+                u32GivenMask  = UINT32_C(0x00ffffff);
+                break;
+            case 4:
+                u32GivenValue = *(uint32_t const *)pvValue;
+                u32GivenMask  = UINT32_C(0xffffffff);
+                break;
+            default:
+                AssertFailedReturn(VERR_IOM_MMIO_IPE_3);
+        }
+        if (offAccess)
+        {
+            u32GivenValue <<= offAccess * 8;
+            u32GivenMask  <<= offAccess * 8;
+        }
+
+        uint32_t u32Value = (u32MissingValue & ~u32GivenMask)
+                          | (u32GivenValue & u32GivenMask);
+
+        /*
+         * Do DWORD write to the device.
+         */
+        VBOXSTRICTRC rc2 = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                       !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
+                                                       ? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3,
+                                                       &u32Value, sizeof(u32Value));
+        switch (VBOXSTRICTRC_VAL(rc2))
+        {
+            case VINF_SUCCESS:
+                break;
+#ifndef IN_RING3
+            case VINF_IOM_R3_MMIO_READ:
+            case VINF_IOM_R3_MMIO_READ_WRITE:
+            case VINF_IOM_R3_MMIO_WRITE:
+                Log3(("iomMmioDoComplicatedWrite: deferring GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
+                AssertReturn(pVCpu->iom.s.PendingMmioWrite.cbValue == 0, VERR_IOM_MMIO_IPE_1);
+                AssertReturn(cbValue + (GCPhys & 3) <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
+                pVCpu->iom.s.PendingMmioWrite.GCPhys  = GCPhys & ~(RTGCPHYS)3;
+                pVCpu->iom.s.PendingMmioWrite.cbValue = cbValue + (GCPhys & 3);
+                *(uint32_t *)pVCpu->iom.s.PendingMmioWrite.abValue = u32Value;
+                if (cbValue > cbThisPart)
+                    memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[4],
+                           (uint8_t const *)pvValue + cbThisPart, cbValue - cbThisPart);
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
+                if (rc == VINF_SUCCESS)
+                    rc = VINF_IOM_R3_MMIO_COMMIT_WRITE;
+                return rc;
+#endif
+            default:
+                if (RT_FAILURE(rc2))
+                {
+                    Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
+                    return rc2;
+                }
+                AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                if (rc == VINF_SUCCESS || rc2 < rc)
+                    rc = rc2;
+                break;
+        }
+
+        /*
+         * Advance.
+         */
+        cbValue   -= cbThisPart;
+        if (!cbValue)
+            break;
+        GCPhys    += cbThisPart;
+        offRegion += cbThisPart;
+        pvValue    = (uint8_t const *)pvValue + cbThisPart;
+    }
+
+    return rc;
+}
+
+
+
+
+/**
+ * Wrapper which does the write.
+ */
+DECLINLINE(VBOXSTRICTRC) iomMmioDoWrite(PVMCC pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry,
+                                        RTGCPHYS GCPhys, RTGCPHYS offRegion,
+                                        const void *pvData, uint32_t cb IOM_MMIO_STATS_COMMA_DECL)
+{
+    VBOXSTRICTRC rcStrict;
+    if (RT_LIKELY(pRegEntry->pfnWriteCallback))
+    {
+        if (   (cb == 4 && !(GCPhys & 3))
+            || (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_PASSTHRU
+            || (cb == 8 && !(GCPhys & 7) && IOMMMIO_DOES_WRITE_MODE_ALLOW_QWORD(pRegEntry->fFlags)) )
+            rcStrict = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                   !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvData, cb);
+        else
+            rcStrict = iomMmioDoComplicatedWrite(pVM, pVCpu, pRegEntry, GCPhys, offRegion, pvData, cb IOM_MMIO_STATS_COMMA_ARG);
+    }
+    else
+        rcStrict = VINF_SUCCESS;
+    return rcStrict;
+}
+
+
+#ifdef IN_RING3
+/**
+ * Helper for IOMR3ProcessForceFlag() that lives here to utilize iomMmioDoWrite et al.
+ */
+VBOXSTRICTRC iomR3MmioCommitWorker(PVM pVM, PVMCPU pVCpu, PIOMMMIOENTRYR3 pRegEntry, RTGCPHYS offRegion)
+{
+# ifdef VBOX_WITH_STATISTICS
+    STAM_PROFILE_START(UnusedMacroArg, Prf);
+    PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
+# endif
+    PPDMDEVINS const         pDevIns = pRegEntry->pDevIns;
+    int rc = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), VERR_IGNORED);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, pVCpu->iom.s.PendingMmioWrite.GCPhys, offRegion,
+                                           pVCpu->iom.s.PendingMmioWrite.abValue, pVCpu->iom.s.PendingMmioWrite.cbValue
+                                           IOM_MMIO_STATS_COMMA_ARG);
+
+    PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
+    STAM_PROFILE_STOP(&pStats->ProfWriteR3, Prf);
+    return rcStrict;
+}
+#endif /* IN_RING3 */
+
+
+/**
+ * Deals with complicated MMIO reads.
+ *
+ * Complicated means unaligned or non-dword/qword sized accesses depending on
+ * the MMIO region's access mode flags.
+ *
+ * @returns Strict VBox status code. Any EM scheduling status code,
+ *          VINF_IOM_R3_MMIO_READ, VINF_IOM_R3_MMIO_READ_WRITE or
+ *          VINF_IOM_R3_MMIO_WRITE may be returned.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pRegEntry   The MMIO entry for the current context.
+ * @param   GCPhys      The physical address to start reading.
+ * @param   offRegion   MMIO region offset corresponding to @a GCPhys.
+ * @param   pvValue     Where to store the value.
+ * @param   cbValue     The size of the value to read.
+ * @param   pStats      Pointer to the statistics (never NULL).
+ */
+static VBOXSTRICTRC iomMMIODoComplicatedRead(PVM pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion,
+                                             void *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    AssertReturn(   (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD
+                 || (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD,
+                 VERR_IOM_MMIO_IPE_1);
+    AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
+#ifdef LOG_ENABLED
+    RTGCPHYS const GCPhysStart = GCPhys;
+#endif
+
+    /*
+     * Do debug stop if requested.
+     */
+    VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
+#ifdef VBOX_STRICT
+    if (pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_READ)
+    {
+# ifdef IN_RING3
+        rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
+                            "Complicated read %#x byte at %RGp to %s\n", cbValue, GCPhys, R3STRING(pRegEntry->pszDesc));
+        if (rc == VERR_DBGF_NOT_ATTACHED)
+            rc = VINF_SUCCESS;
+# else
+        return VINF_IOM_R3_MMIO_READ;
+# endif
+    }
+#endif
+
+    STAM_COUNTER_INC(&pStats->ComplicatedReads);
+
+    /*
+     * Split and conquer.
+     */
+    for (;;)
+    {
+        /*
+         * Do DWORD read from the device.
+         */
+        uint32_t u32Value;
+        VBOXSTRICTRC rcStrict2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                            !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
+                                                            ? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3,
+                                                            &u32Value, sizeof(u32Value));
+        switch (VBOXSTRICTRC_VAL(rcStrict2))
+        {
+            case VINF_SUCCESS:
+                break;
+            case VINF_IOM_MMIO_UNUSED_FF:
+                STAM_COUNTER_INC(&pStats->FFor00Reads);
+                u32Value = UINT32_C(0xffffffff);
+                break;
+            case VINF_IOM_MMIO_UNUSED_00:
+                STAM_COUNTER_INC(&pStats->FFor00Reads);
+                u32Value = 0;
+                break;
+            case VINF_IOM_R3_MMIO_READ:
+            case VINF_IOM_R3_MMIO_READ_WRITE:
+            case VINF_IOM_R3_MMIO_WRITE:
+                /** @todo What if we've split a transfer and already read
+                 * something?  Since reads can have sideeffects we could be
+                 * kind of screwed here... */
+                LogFlow(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
+                         GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
+                return rcStrict2;
+            default:
+                if (RT_FAILURE(rcStrict2))
+                {
+                    Log(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
+                         GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
+                    return rcStrict2;
+                }
+                AssertMsgReturn(rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
+                                VERR_IPE_UNEXPECTED_INFO_STATUS);
+                if (rc == VINF_SUCCESS || rcStrict2 < rc)
+                    rc = rcStrict2;
+                break;
+        }
+        u32Value >>= (GCPhys & 3) * 8;
+
+        /*
+         * Write what we've read.
+         */
+        unsigned cbThisPart = 4 - (GCPhys & 3);
+        if (cbThisPart > cbValue)
+            cbThisPart = cbValue;
+
+        switch (cbThisPart)
+        {
+            case 1:
+                *(uint8_t *)pvValue = (uint8_t)u32Value;
+                break;
+            case 2:
+                *(uint16_t *)pvValue = (uint16_t)u32Value;
+                break;
+            case 3:
+                ((uint8_t *)pvValue)[0] = RT_BYTE1(u32Value);
+                ((uint8_t *)pvValue)[1] = RT_BYTE2(u32Value);
+                ((uint8_t *)pvValue)[2] = RT_BYTE3(u32Value);
+                break;
+            case 4:
+                *(uint32_t *)pvValue = u32Value;
+                break;
+        }
+
+        /*
+         * Advance.
+         */
+        cbValue   -= cbThisPart;
+        if (!cbValue)
+            break;
+        GCPhys    += cbThisPart;
+        offRegion += cbThisPart;
+        pvValue    = (uint8_t *)pvValue + cbThisPart;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Implements VINF_IOM_MMIO_UNUSED_FF.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pvValue     Where to store the zeros.
+ * @param   cbValue     How many bytes to read.
+ * @param   pStats      Pointer to the statistics (never NULL).
+ */
+static int iomMMIODoReadFFs(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    switch (cbValue)
+    {
+        case 1: *(uint8_t  *)pvValue = UINT8_C(0xff); break;
+        case 2: *(uint16_t *)pvValue = UINT16_C(0xffff); break;
+        case 4: *(uint32_t *)pvValue = UINT32_C(0xffffffff); break;
+        case 8: *(uint64_t *)pvValue = UINT64_C(0xffffffffffffffff); break;
+        default:
+        {
+            uint8_t *pb = (uint8_t *)pvValue;
+            while (cbValue--)
+                *pb++ = UINT8_C(0xff);
+            break;
+        }
+    }
+    STAM_COUNTER_INC(&pStats->FFor00Reads);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements VINF_IOM_MMIO_UNUSED_00.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pvValue     Where to store the zeros.
+ * @param   cbValue     How many bytes to read.
+ * @param   pStats      Pointer to the statistics (never NULL).
+ */
+static int iomMMIODoRead00s(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    switch (cbValue)
+    {
+        case 1: *(uint8_t  *)pvValue = UINT8_C(0x00); break;
+        case 2: *(uint16_t *)pvValue = UINT16_C(0x0000); break;
+        case 4: *(uint32_t *)pvValue = UINT32_C(0x00000000); break;
+        case 8: *(uint64_t *)pvValue = UINT64_C(0x0000000000000000); break;
+        default:
+        {
+            uint8_t *pb = (uint8_t *)pvValue;
+            while (cbValue--)
+                *pb++ = UINT8_C(0x00);
+            break;
+        }
+    }
+    STAM_COUNTER_INC(&pStats->FFor00Reads);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Wrapper which does the read.
+ */
+DECLINLINE(VBOXSTRICTRC) iomMmioDoRead(PVMCC pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion,
+                                       void *pvValue, uint32_t cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    VBOXSTRICTRC rcStrict;
+    if (RT_LIKELY(pRegEntry->pfnReadCallback))
+    {
+        if (   (   cbValue == 4
+                && !(GCPhys & 3))
+            || (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_PASSTHRU
+            || (    cbValue == 8
+                && !(GCPhys & 7)
+                && (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD ) )
+            rcStrict = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                  !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvValue, cbValue);
+        else
+            rcStrict = iomMMIODoComplicatedRead(pVM, pRegEntry, GCPhys, offRegion, pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG);
+    }
+    else
+        rcStrict = VINF_IOM_MMIO_UNUSED_FF;
+    if (rcStrict != VINF_SUCCESS)
+    {
+        switch (VBOXSTRICTRC_VAL(rcStrict))
+        {
+            case VINF_IOM_MMIO_UNUSED_FF: rcStrict = iomMMIODoReadFFs(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
+            case VINF_IOM_MMIO_UNUSED_00: rcStrict = iomMMIODoRead00s(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
+        }
+    }
+    return rcStrict;
+}
+
+#ifndef IN_RING3
+
+/**
+ * Checks if we can handle an MMIO \#PF in R0/RC.
+ */
+DECLINLINE(bool) iomMmioCanHandlePfInRZ(PVMCC pVM, uint32_t uErrorCode, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
+{
+    if (pRegEntry->cbRegion > 0)
+    {
+        if (   pRegEntry->pfnWriteCallback
+            && pRegEntry->pfnReadCallback)
+            return true;
+
+        PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[pRegEntry->idxSelf];
+        if (  uErrorCode == UINT32_MAX
+            ? pRegEntryR3->pfnWriteCallback || pRegEntryR3->pfnReadCallback
+            : uErrorCode & X86_TRAP_PF_RW
+              ? !pRegEntry->pfnWriteCallback && pRegEntryR3->pfnWriteCallback
+              : !pRegEntry->pfnReadCallback  && pRegEntryR3->pfnReadCallback)
+            return false;
+
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Common worker for the \#PF handler and IOMMMIOPhysHandler (APIC+VT-x).
+ *
+ * @returns VBox status code (appropriate for GC return).
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uErrorCode  CPU Error code.  This is UINT32_MAX when we don't have
+ *                      any error code (the EPT misconfig hack).
+ * @param   GCPhysFault The GC physical address corresponding to pvFault.
+ * @param   pRegEntry   The MMIO entry for the current context.
+ */
+DECLINLINE(VBOXSTRICTRC) iomMmioCommonPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode,
+                                                   RTGCPHYS GCPhysFault, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
+{
+    Log(("iomMmioCommonPfHandler: GCPhysFault=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, CPUMGetGuestRIP(pVCpu) ));
+    RT_NOREF(GCPhysFault, uErrorCode);
+
+    VBOXSTRICTRC rcStrict;
+
+#ifndef IN_RING3
+    /*
+     * Should we defer the request right away?  This isn't usually the case, so
+     * do the simple test first and the try deal with uErrorCode being N/A.
+     */
+    PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
+    if (RT_LIKELY(   pDevIns
+                  && iomMmioCanHandlePfInRZ(pVM, uErrorCode, pRegEntry)))
+    {
+        /*
+         * Enter the device critsect prior to engaging IOM in case of lock contention.
+         * Note! Perhaps not a good move?
+         */
+        rcStrict = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_MMIO_READ_WRITE);
+        if (rcStrict == VINF_SUCCESS)
+        {
+#endif /* !IN_RING3 */
+
+            /*
+             * Let IEM call us back via iomMmioHandler.
+             */
+            rcStrict = IEMExecOne(pVCpu);
+
+#ifndef IN_RING3
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
+#endif
+            if (RT_SUCCESS(rcStrict))
+            { /* likely */ }
+            else if (   rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
+                     || rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
+            {
+                Log(("IOM: Hit unsupported IEM feature!\n"));
+                rcStrict = VINF_EM_RAW_EMULATE_INSTR;
+            }
+#ifndef IN_RING3
+            return rcStrict;
+        }
+        STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
+    }
+    else
+        rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
+
+# ifdef VBOX_WITH_STATISTICS
+    if (rcStrict == VINF_IOM_R3_MMIO_READ_WRITE)
+    {
+        PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
+        if (uErrorCode & X86_TRAP_PF_RW)
+        {
+            STAM_COUNTER_INC(&pStats->WriteRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
+        }
+        else
+        {
+            STAM_COUNTER_INC(&pStats->ReadRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3);
+        }
+    }
+# endif
+#else  /* IN_RING3 */
+    RT_NOREF(pVM, pRegEntry);
+#endif /* IN_RING3 */
+    return rcStrict;
+}
+
+
+/**
+ * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
+ *      \#PF access handler callback for MMIO pages.}
+ *
+ * @remarks The @a pvUser argument is the MMIO handle.
+ */
+DECLEXPORT(VBOXSTRICTRC) iomMmioPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pCtxCore,
+                                             RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
+{
+    STAM_PROFILE_START(&pVM->iom.s.StatMmioPfHandler, Prf);
+    LogFlow(("iomMmioPfHandlerNew: GCPhys=%RGp uErr=%#x pvFault=%RGv rip=%RGv\n",
+             GCPhysFault, (uint32_t)uErrorCode, pvFault, (RTGCPTR)pCtxCore->rip));
+    RT_NOREF(pvFault, pCtxCore);
+
+    /* Translate the MMIO handle to a registration entry for the current context. */
+    AssertReturn((uintptr_t)pvUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+# ifdef IN_RING0
+    AssertReturn((uintptr_t)pvUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iomr0.s.paMmioRegs[(uintptr_t)pvUser];
+# else
+    CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iom.s.paMmioRegs[(uintptr_t)pvUser];
+# endif
+
+    VBOXSTRICTRC rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
+
+    STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPfHandler, Prf);
+    return rcStrict;
+}
+
+#endif /* !IN_RING3 */
+
+#ifdef IN_RING0
+/**
+ * Physical access handler for MMIO ranges.
+ *
+ * This is actually only used by VT-x for APIC page accesses.
+ *
+ * @returns VBox status code (appropriate for GC return).
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uErrorCode  CPU Error code.
+ * @param   GCPhysFault The GC physical address.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IOMR0MmioPhysHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, RTGCPHYS GCPhysFault)
+{
+    STAM_PROFILE_START(&pVM->iom.s.StatMmioPhysHandler, Prf);
+
+    /*
+     * We don't have a range here, so look it up before calling the common function.
+     */
+    VBOXSTRICTRC rcStrict = IOM_LOCK_SHARED(pVM);
+    if (RT_SUCCESS(rcStrict))
+    {
+        RTGCPHYS offRegion;
+        CTX_SUFF(PIOMMMIOENTRY) pRegEntry = iomMmioGetEntry(pVM, GCPhysFault, &offRegion, &pVCpu->iom.s.idxMmioLastPhysHandler);
+        IOM_UNLOCK_SHARED(pVM);
+        if (RT_LIKELY(pRegEntry))
+            rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
+        else
+            rcStrict = VERR_IOM_MMIO_RANGE_NOT_FOUND;
+    }
+    else if (rcStrict == VERR_SEM_BUSY)
+        rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
+
+    STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPhysHandler, Prf);
+    return rcStrict;
+}
+#endif /* IN_RING0 */
+
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER, MMIO page accesses}
+ *
+ * @remarks The @a pvUser argument is the MMIO handle.
+ */
+PGM_ALL_CB2_DECL(VBOXSTRICTRC) iomMmioHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void *pvPhys, void *pvBuf,
+                                                 size_t cbBuf, PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
+{
+    STAM_PROFILE_START(UnusedMacroArg, Prf);
+    STAM_COUNTER_INC(&pVM->iom.s.CTX_SUFF(StatMmioHandler));
+    Log4(("iomMmioHandlerNew: GCPhysFault=%RGp cbBuf=%#x enmAccessType=%d enmOrigin=%d pvUser=%p\n", GCPhysFault, cbBuf, enmAccessType, enmOrigin, pvUser));
+
+    Assert(enmAccessType == PGMACCESSTYPE_READ || enmAccessType == PGMACCESSTYPE_WRITE);
+    AssertMsg(cbBuf >= 1, ("%zu\n", cbBuf));
+    NOREF(pvPhys); NOREF(enmOrigin);
+
+#ifdef IN_RING3
+    int const rcToRing3 = VERR_IOM_MMIO_IPE_3;
+#else
+    int const rcToRing3 = enmAccessType == PGMACCESSTYPE_READ ? VINF_IOM_R3_MMIO_READ : VINF_IOM_R3_MMIO_WRITE;
+#endif
+
+    /*
+     * Translate pvUser to an MMIO registration table entry.  We can do this
+     * without any locking as the data is static after VM creation.
+     */
+    AssertReturn((uintptr_t)pvUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn((uintptr_t)pvUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    CTX_SUFF(PIOMMMIOENTRY) const pRegEntry    = &pVM->iomr0.s.paMmioRegs[(uintptr_t)pvUser];
+    PIOMMMIOENTRYR3 const         pRegEntryR3  = &pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser];
+#else
+    CTX_SUFF(PIOMMMIOENTRY) const pRegEntry    = &pVM->iom.s.paMmioRegs[(uintptr_t)pvUser];
+#endif
+#ifdef VBOX_WITH_STATISTICS
+    PIOMMMIOSTATSENTRY const      pStats       = iomMmioGetStats(pVM, pRegEntry);  /* (Works even without ring-0 device setup.) */
+#endif
+    PPDMDEVINS const              pDevIns      = pRegEntry->pDevIns;
+
+#ifdef VBOX_STRICT
+    /*
+     * Assert the right entry in strict builds.  This may yield a false positive
+     * for SMP VMs if we're unlucky and the guest isn't well behaved.
+     */
+# ifdef IN_RING0
+    Assert(pRegEntry && (GCPhysFault - pRegEntryR3->GCPhysMapping < pRegEntryR3->cbRegion || !pRegEntryR3->fMapped));
+# else
+    Assert(pRegEntry && (GCPhysFault - pRegEntry->GCPhysMapping   < pRegEntry->cbRegion   || !pRegEntry->fMapped));
+# endif
+#endif
+
+#ifndef IN_RING3
+    /*
+     * If someone is doing FXSAVE, FXRSTOR, XSAVE, XRSTOR or other stuff dealing with
+     * large amounts of data, just go to ring-3 where we don't need to deal with partial
+     * successes.  No chance any of these will be problematic read-modify-write stuff.
+     *
+     * Also drop back if the ring-0 registration entry isn't actually used.
+     */
+    if (   RT_LIKELY(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue))
+        && pRegEntry->cbRegion != 0
+        && (  enmAccessType == PGMACCESSTYPE_READ
+            ? pRegEntry->pfnReadCallback  != NULL || pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].pfnReadCallback == NULL
+            : pRegEntry->pfnWriteCallback != NULL || pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].pfnWriteCallback == NULL)
+        && pDevIns )
+    { /* likely */ }
+    else
+    {
+        Log4(("iomMmioHandlerNew: to ring-3: to-big=%RTbool zero-size=%RTbool no-callback=%RTbool pDevIns=%p hRegion=%p\n",
+              !(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue)), !(pRegEntry->cbRegion != 0),
+              !(  enmAccessType == PGMACCESSTYPE_READ
+                ? pRegEntry->pfnReadCallback  != NULL || pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].pfnReadCallback == NULL
+                : pRegEntry->pfnWriteCallback != NULL || pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].pfnWriteCallback == NULL),
+              pDevIns, pvUser));
+        STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
+        STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3);
+        return rcToRing3;
+    }
+#endif /* !IN_RING3 */
+
+    /*
+     * If we've got an offset that's outside the region, defer to ring-3 if we
+     * can, or pretend there is nothing there.  This shouldn't happen, but can
+     * if we're unlucky with an SMP VM and the guest isn't behaving very well.
+     */
+#ifdef IN_RING0
+    RTGCPHYS const GCPhysMapping = pRegEntryR3->GCPhysMapping;
+#else
+    RTGCPHYS const GCPhysMapping = pRegEntry->GCPhysMapping;
+#endif
+    RTGCPHYS const offRegion     = GCPhysFault - GCPhysMapping;
+    if (RT_LIKELY(offRegion < pRegEntry->cbRegion && GCPhysMapping != NIL_RTGCPHYS))
+    { /* likely */ }
+    else
+    {
+        STAM_REL_COUNTER_INC(&pVM->iom.s.StatMmioStaleMappings);
+        LogRelMax(64, ("iomMmioHandlerNew: Stale access at %#RGp to range #%#x currently residing at %RGp LB %RGp\n",
+                       GCPhysFault, pRegEntry->idxSelf, GCPhysMapping, pRegEntry->cbRegion));
+#ifdef IN_RING3
+        if (enmAccessType == PGMACCESSTYPE_READ)
+            iomMMIODoReadFFs(pvBuf, cbBuf IOM_MMIO_STATS_COMMA_ARG);
+        return VINF_SUCCESS;
+#else
+        STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
+        STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3);
+        return rcToRing3;
+#endif
+    }
+
+    /*
+     * Perform locking and the access.
+     *
+     * Writes requiring a return to ring-3 are buffered by IOM so IEM can
+     * commit the instruction.
+     *
+     * Note! We may end up locking the device even when the relevant callback is
+     *       NULL.  This is supposed to be an unlikely case, so not optimized yet.
+     */
+    VBOXSTRICTRC rcStrict = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), rcToRing3);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (enmAccessType == PGMACCESSTYPE_READ)
+        {
+            /*
+             * Read.
+             */
+            rcStrict = iomMmioDoRead(pVM, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
+
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
+#ifndef IN_RING3
+            if (rcStrict == VINF_IOM_R3_MMIO_READ)
+            {
+                STAM_COUNTER_INC(&pStats->ReadRZToR3);
+                STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3);
+            }
+            else
+#endif
+                STAM_COUNTER_INC(&pStats->Reads);
+            STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfRead), Prf);
+        }
+        else
+        {
+            /*
+             * Write.
+             */
+            rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
+#ifndef IN_RING3
+            if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
+                rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
+            if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
+            {
+                STAM_COUNTER_INC(&pStats->WriteRZToR3);
+                STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
+            }
+            else if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
+            {
+                STAM_COUNTER_INC(&pStats->CommitRZToR3);
+                STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3);
+            }
+            else
+#endif
+                STAM_COUNTER_INC(&pStats->Writes);
+            STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfWrite), Prf);
+        }
+
+        /*
+         * Check the return code.
+         */
+#ifdef IN_RING3
+        AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc -  Access type %d - %RGp - %s\n",
+                                             VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pRegEntry->pszDesc));
+#else
+        AssertMsg(   rcStrict == VINF_SUCCESS
+                  || rcStrict == rcToRing3
+                  || (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE && enmAccessType == PGMACCESSTYPE_WRITE)
+                  || rcStrict == VINF_EM_DBG_STOP
+                  || rcStrict == VINF_EM_DBG_EVENT
+                  || rcStrict == VINF_EM_DBG_BREAKPOINT
+                  || rcStrict == VINF_EM_OFF
+                  || rcStrict == VINF_EM_SUSPEND
+                  || rcStrict == VINF_EM_RESET
+                  //|| rcStrict == VINF_EM_HALT       /* ?? */
+                  //|| rcStrict == VINF_EM_NO_MEMORY  /* ?? */
+                  , ("%Rrc - Access type %d - %RGp - %s #%u\n",
+                     VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pDevIns->pReg->szName, pDevIns->iInstance));
+#endif
+    }
+    /*
+     * Deal with enter-critsect failures.
+     */
+#ifndef IN_RING3
+    else if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
+    {
+        Assert(enmAccessType == PGMACCESSTYPE_WRITE);
+        rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
+        if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
+        {
+            STAM_COUNTER_INC(&pStats->CommitRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3);
+        }
+        else
+        {
+            STAM_COUNTER_INC(&pStats->WriteRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
+        }
+        STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
+    }
+    else if (rcStrict == VINF_IOM_R3_MMIO_READ)
+    {
+        Assert(enmAccessType == PGMACCESSTYPE_READ);
+        STAM_COUNTER_INC(&pStats->ReadRZToR3);
+        STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
+    }
+#endif
+    else
+        AssertMsg(RT_FAILURE_NP(rcStrict), ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * Mapping an MMIO2 page in place of an MMIO page for direct access.
+ *
+ * This is a special optimization used by the VGA device.  Call
+ * IOMMmioResetRegion() to undo the mapping.
+ *
+ * @returns VBox status code.  This API may return VINF_SUCCESS even if no
+ *          remapping is made.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance @a hRegion and @a hMmio2 are
+ *                          associated with.
+ * @param   hRegion         The handle to the MMIO region.
+ * @param   offRegion       The offset into @a hRegion of the page to be
+ *                          remapped.
+ * @param   hMmio2          The MMIO2 handle.
+ * @param   offMmio2        Offset into @a hMmio2 of the page to be use for the
+ *                          mapping.
+ * @param   fPageFlags      Page flags to set. Must be (X86_PTE_RW | X86_PTE_P)
+ *                          for the time being.
+ */
+VMMDECL(int) IOMMmioMapMmio2Page(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS offRegion,
+                                 uint64_t hMmio2, RTGCPHYS offMmio2, uint64_t fPageFlags)
+{
+    /* Currently only called from the VGA device during MMIO. */
+    Log(("IOMMmioMapMmio2Page %#RX64/%RGp -> %#RX64/%RGp flags=%RX64\n", hRegion, offRegion, hMmio2, offMmio2, fPageFlags));
+    AssertReturn(fPageFlags == (X86_PTE_RW | X86_PTE_P), VERR_INVALID_PARAMETER);
+    AssertReturn(pDevIns, VERR_INVALID_POINTER);
+
+/** @todo Why is this restricted to protected mode???  Try it in all modes! */
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+
+    /* This currently only works in real mode, protected mode without paging or with nested paging. */
+    /** @todo NEM: MMIO page aliasing. */
+    if (    !HMIsEnabled(pVM)       /* useless without VT-x/AMD-V */
+        ||  (   CPUMIsGuestInPagedProtectedMode(pVCpu)
+             && !HMIsNestedPagingActive(pVM)))
+        return VINF_SUCCESS;    /* ignore */ /** @todo return some indicator if we fail here */
+
+    /*
+     * Translate the handle into an entry and check the region offset.
+     */
+    AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(offRegion < pVM->iomr0.s.paMmioRegs[hRegion].cbRegion, VERR_OUT_OF_RANGE);
+    AssertReturn(   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
+                 || (   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
+                     && pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
+#else
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
+#endif
+    AssertReturn(offRegion < pRegEntry->cbRegion, VERR_OUT_OF_RANGE);
+    Assert((pRegEntry->cbRegion & PAGE_OFFSET_MASK) == 0);
+
+    /*
+     * When getting and using the mapping address, we must sit on the IOM lock
+     * to prevent remapping.  Shared suffices as we change nothing.
+     */
+    int rc = IOM_LOCK_SHARED(pVM);
+    if (rc == VINF_SUCCESS)
+    {
+        RTGCPHYS const GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
+        if (GCPhys != NIL_RTGCPHYS)
+        {
+            Assert(!(GCPhys & PAGE_OFFSET_MASK));
+
+            /*
+             * Do the aliasing; page align the addresses since PGM is picky.
+             */
+            rc = PGMHandlerPhysicalPageAliasMmio2(pVM, GCPhys, GCPhys + (offRegion & ~(RTGCPHYS)PAGE_OFFSET_MASK),
+                                                  pDevIns, hMmio2, offMmio2);
+        }
+        else
+            AssertFailedStmt(rc = VERR_IOM_MMIO_REGION_NOT_MAPPED);
+
+        IOM_UNLOCK_SHARED(pVM);
+    }
+
+/** @todo either ditch this or replace it with something that works in the
+ *        nested case, since we really only care about nested paging! */
+#if 0
+    /*
+     * Modify the shadow page table. Since it's an MMIO page it won't be present and we
+     * can simply prefetch it.
+     *
+     * Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
+     */
+# if 0 /* The assertion is wrong for the PGM_SYNC_CLEAR_PGM_POOL and VINF_PGM_HANDLER_ALREADY_ALIASED cases. */
+#  ifdef VBOX_STRICT
+    uint64_t fFlags;
+    RTHCPHYS HCPhys;
+    rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
+    Assert(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+#  endif
+# endif
+    rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
+    Assert(rc == VINF_SUCCESS || rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+#endif
+    return rc;
+}
+
+
+#ifdef IN_RING0 /* VT-x ring-0 only, move to IOMR0Mmio.cpp later.  */
+/**
+ * Mapping a HC page in place of an MMIO page for direct access.
+ *
+ * This is a special optimization used by the APIC in the VT-x case.  This VT-x
+ * code uses PGMHandlerPhysicalReset rather than IOMMmioResetRegion() to undo
+ * the effects here.
+ *
+ * @todo Make VT-x usage more consistent.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhys          The address of the MMIO page to be changed.
+ * @param   HCPhys          The address of the host physical page.
+ * @param   fPageFlags      Page flags to set. Must be (X86_PTE_RW | X86_PTE_P)
+ *                          for the time being.
+ */
+VMMR0_INT_DECL(int) IOMR0MmioMapMmioHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags)
+{
+    /* Currently only called from VT-x code during a page fault. */
+    Log(("IOMR0MmioMapMmioHCPage %RGp -> %RGp flags=%RX64\n", GCPhys, HCPhys, fPageFlags));
+
+    AssertReturn(fPageFlags == (X86_PTE_RW | X86_PTE_P), VERR_INVALID_PARAMETER);
+    /** @todo NEM: MMIO page aliasing?? */
+    Assert(HMIsEnabled(pVM));
+
+# ifdef VBOX_STRICT
+    /*
+     * Check input address (it's HM calling, not the device, so no region handle).
+     */
+    int rcSem = IOM_LOCK_SHARED(pVM);
+    if (rcSem == VINF_SUCCESS)
+    {
+        RTGCPHYS offIgn;
+        uint16_t idxIgn = UINT16_MAX;
+        PIOMMMIOENTRYR0 pRegEntry = iomMmioGetEntry(pVM, GCPhys, &offIgn, &idxIgn);
+        IOM_UNLOCK_SHARED(pVM);
+        Assert(pRegEntry);
+        Assert(pRegEntry && !(pRegEntry->cbRegion & PAGE_OFFSET_MASK));
+    }
+# endif
+
+    /*
+     * Do the aliasing; page align the addresses since PGM is picky.
+     */
+    GCPhys &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
+    HCPhys &= ~(RTHCPHYS)PAGE_OFFSET_MASK;
+
+    int rc = PGMHandlerPhysicalPageAliasHC(pVM, GCPhys, GCPhys, HCPhys);
+    AssertRCReturn(rc, rc);
+
+/** @todo either ditch this or replace it with something that works in the
+ *        nested case, since we really only care about nested paging! */
+
+    /*
+     * Modify the shadow page table. Since it's an MMIO page it won't be present and we
+     * can simply prefetch it.
+     *
+     * Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
+     */
+    rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
+    Assert(rc == VINF_SUCCESS || rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+    return VINF_SUCCESS;
+}
+#endif
+
+
+/**
+ * Reset a previously modified MMIO region; restore the access flags.
+ *
+ * This undoes the effects of IOMMmioMapMmio2Page() and is currently only
+ * intended for some ancient VGA hack.  However, it would be great to extend it
+ * beyond VT-x and/or nested-paging.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance @a hRegion is associated with.
+ * @param   hRegion         The handle to the MMIO region.
+ */
+VMMDECL(int) IOMMmioResetRegion(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
+{
+    Log(("IOMMMIOResetRegion %#RX64\n", hRegion));
+    AssertReturn(pDevIns, VERR_INVALID_POINTER);
+
+/** @todo Get rid of this this real/protected or nested paging restriction,
+ *        it probably shouldn't be here and would be nasty when the CPU
+ *        changes mode while we have the hack enabled... */
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+
+    /* This currently only works in real mode, protected mode without paging or with nested paging. */
+    /** @todo NEM: MMIO page aliasing. */
+    if (   !HMIsEnabled(pVM)       /* useless without VT-x/AMD-V */
+        || (   CPUMIsGuestInPagedProtectedMode(pVCpu)
+            && !HMIsNestedPagingActive(pVM)))
+        return VINF_SUCCESS;    /* ignore */
+
+    /*
+     * Translate the handle into an entry and mapping address for PGM.
+     * We have to take the lock to safely access the mapping address here.
+     */
+    AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
+                 || (   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
+                     && pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
+#else
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
+#endif
+    Assert((pRegEntry->cbRegion & PAGE_OFFSET_MASK) == 0);
+
+    int rcSem = IOM_LOCK_SHARED(pVM);
+    RTGCPHYS GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
+    if (rcSem == VINF_SUCCESS)
+        IOM_UNLOCK_SHARED(pVM);
+
+    Assert(!(GCPhys              & PAGE_OFFSET_MASK));
+    Assert(!(pRegEntry->cbRegion & PAGE_OFFSET_MASK));
+
+    /*
+     * Call PGM to do the job work.
+     *
+     * After the call, all the pages should be non-present, unless there is
+     * a page pool flush pending (unlikely).
+     */
+    int rc = PGMHandlerPhysicalReset(pVM, GCPhys);
+    AssertRC(rc);
+
+# ifdef VBOX_STRICT
+    if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
+    {
+        RTGCPHYS cb = pRegEntry->cbRegion;
+        while (cb)
+        {
+            uint64_t fFlags;
+            RTHCPHYS HCPhys;
+            rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
+            Assert(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+            cb     -= PAGE_SIZE;
+            GCPhys += PAGE_SIZE;
+        }
+    }
+# endif
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMAll/MMAll.cpp b/src/VBox/VMM/VMMAll/MMAll.cpp
new file mode 100644
index 00000000..250b1dd4
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/MMAll.cpp
@@ -0,0 +1,666 @@
+/* $Id: MMAll.cpp $ */
+/** @file
+ * MM - Memory Manager - Any Context.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM_HYPER
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vmm.h>
+#include "MMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+
+/**
+ * Lookup a host context ring-3 address.
+ *
+ * @returns Pointer to the corresponding lookup record.
+ * @returns NULL on failure.
+ * @param   pVM     The cross context VM structure.
+ * @param   R3Ptr   The host context ring-3 address to lookup.
+ * @param   poff    Where to store the offset into the HMA memory chunk.
+ */
+DECLINLINE(PMMLOOKUPHYPER) mmHyperLookupR3(PVM pVM, RTR3PTR R3Ptr, uint32_t *poff)
+{
+    /** @todo cache last lookup, this stuff ain't cheap! */
+    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.CTX_SUFF(pHyperHeap) + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        switch (pLookup->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+            {
+                const RTR3UINTPTR off = (RTR3UINTPTR)R3Ptr - (RTR3UINTPTR)pLookup->u.Locked.pvR3;
+                if (off < pLookup->cb)
+                {
+                    *poff = off;
+                    return pLookup;
+                }
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+            {
+                const RTR3UINTPTR off = (RTR3UINTPTR)R3Ptr - (RTR3UINTPTR)pLookup->u.HCPhys.pvR3;
+                if (off < pLookup->cb)
+                {
+                    *poff = off;
+                    return pLookup;
+                }
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_GCPHYS:  /* (for now we'll not allow these kind of conversions) */
+            case MMLOOKUPHYPERTYPE_MMIO2:
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+                break;
+
+            default:
+                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+                break;
+        }
+
+        /* next */
+        if (pLookup->offNext ==  (int32_t)NIL_OFFSET)
+            break;
+        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
+    }
+
+    AssertMsgFailed(("R3Ptr=%RHv is not inside the hypervisor memory area!\n", R3Ptr));
+    return NULL;
+}
+
+
+/**
+ * Lookup a host context ring-0 address.
+ *
+ * @returns Pointer to the corresponding lookup record.
+ * @returns NULL on failure.
+ * @param   pVM     The cross context VM structure.
+ * @param   R0Ptr   The host context ring-0 address to lookup.
+ * @param   poff    Where to store the offset into the HMA memory chunk.
+ */
+DECLINLINE(PMMLOOKUPHYPER) mmHyperLookupR0(PVM pVM, RTR0PTR R0Ptr, uint32_t *poff)
+{
+    AssertCompile(sizeof(RTR0PTR) == sizeof(RTR3PTR));
+
+    /** @todo cache last lookup, this stuff ain't cheap! */
+    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.CTX_SUFF(pHyperHeap) + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        switch (pLookup->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+            {
+                const RTR0UINTPTR off = (RTR0UINTPTR)R0Ptr - (RTR0UINTPTR)pLookup->u.Locked.pvR0;
+                if (off < pLookup->cb && pLookup->u.Locked.pvR0)
+                {
+                    *poff = off;
+                    return pLookup;
+                }
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+            {
+                const RTR0UINTPTR off = (RTR0UINTPTR)R0Ptr - (RTR0UINTPTR)pLookup->u.HCPhys.pvR0;
+                if (off < pLookup->cb && pLookup->u.HCPhys.pvR0)
+                {
+                    *poff = off;
+                    return pLookup;
+                }
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_GCPHYS:  /* (for now we'll not allow these kind of conversions) */
+            case MMLOOKUPHYPERTYPE_MMIO2:
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+                break;
+
+            default:
+                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+                break;
+        }
+
+        /* next */
+        if (pLookup->offNext ==  (int32_t)NIL_OFFSET)
+            break;
+        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
+    }
+
+    AssertMsgFailed(("R0Ptr=%RHv is not inside the hypervisor memory area!\n", R0Ptr));
+    return NULL;
+}
+
+
+/**
+ * Lookup a raw-mode context address.
+ *
+ * @returns Pointer to the corresponding lookup record.
+ * @returns NULL on failure.
+ * @param   pVM     The cross context VM structure.
+ * @param   RCPtr   The raw-mode context address to lookup.
+ * @param   poff    Where to store the offset into the HMA memory chunk.
+ */
+DECLINLINE(PMMLOOKUPHYPER) mmHyperLookupRC(PVM pVM, RTRCPTR RCPtr, uint32_t *poff)
+{
+    /** @todo cache last lookup this stuff ain't cheap! */
+    unsigned        offRC = (RTRCUINTPTR)RCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC;
+    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.CTX_SUFF(pHyperHeap) + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        const uint32_t off = offRC - pLookup->off;
+        if (off < pLookup->cb)
+        {
+            switch (pLookup->enmType)
+            {
+                case MMLOOKUPHYPERTYPE_LOCKED:
+                case MMLOOKUPHYPERTYPE_HCPHYS:
+                    *poff = off;
+                    return pLookup;
+                default:
+                    break;
+            }
+            AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+            *poff = 0; /* shut up gcc */
+            return NULL;
+        }
+
+        /* next */
+        if (pLookup->offNext == (int32_t)NIL_OFFSET)
+            break;
+        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
+    }
+
+    AssertMsgFailed(("RCPtr=%RRv is not inside the hypervisor memory area!\n", RCPtr));
+    *poff = 0; /* shut up gcc */
+    return NULL;
+}
+
+
+/**
+ * Lookup a current context address.
+ *
+ * @returns Pointer to the corresponding lookup record.
+ * @returns NULL on failure.
+ * @param   pVM     The cross context VM structure.
+ * @param   pv      The current context address to lookup.
+ * @param   poff    Where to store the offset into the HMA memory chunk.
+ */
+DECLINLINE(PMMLOOKUPHYPER) mmHyperLookupCC(PVM pVM, void *pv, uint32_t *poff)
+{
+#ifdef IN_RING0
+    return mmHyperLookupR0(pVM, pv, poff);
+#elif defined(IN_RING3)
+    return mmHyperLookupR3(pVM, pv, poff);
+#else
+# error "Neither IN_RING0 nor IN_RING3!"
+#endif
+}
+
+
+/**
+ * Calculate the host context ring-3 address of an offset into the HMA memory chunk.
+ *
+ * @returns the host context ring-3 address.
+ * @param   pLookup     The HMA lookup record.
+ * @param   off         The offset into the HMA memory chunk.
+ */
+DECLINLINE(RTR3PTR) mmHyperLookupCalcR3(PMMLOOKUPHYPER pLookup, uint32_t off)
+{
+    switch (pLookup->enmType)
+    {
+        case MMLOOKUPHYPERTYPE_LOCKED:
+            return (RTR3PTR)((RTR3UINTPTR)pLookup->u.Locked.pvR3 + off);
+        case MMLOOKUPHYPERTYPE_HCPHYS:
+            return (RTR3PTR)((RTR3UINTPTR)pLookup->u.HCPhys.pvR3 + off);
+        default:
+            AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+            return NIL_RTR3PTR;
+    }
+}
+
+
+/**
+ * Calculate the host context ring-0 address of an offset into the HMA memory chunk.
+ *
+ * @returns the host context ring-0 address.
+ * @param   pVM         The cross context VM structure.
+ * @param   pLookup     The HMA lookup record.
+ * @param   off         The offset into the HMA memory chunk.
+ */
+DECLINLINE(RTR0PTR) mmHyperLookupCalcR0(PVM pVM, PMMLOOKUPHYPER pLookup, uint32_t off)
+{
+    switch (pLookup->enmType)
+    {
+        case MMLOOKUPHYPERTYPE_LOCKED:
+            if (pLookup->u.Locked.pvR0)
+                return (RTR0PTR)((RTR0UINTPTR)pLookup->u.Locked.pvR0 + off);
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+            AssertMsg(VM_IS_RAW_MODE_ENABLED(pVM), ("%s\n", R3STRING(pLookup->pszDesc)));
+#else
+            AssertMsgFailed(("%s\n", R3STRING(pLookup->pszDesc))); NOREF(pVM);
+#endif
+            return NIL_RTR0PTR;
+
+        case MMLOOKUPHYPERTYPE_HCPHYS:
+            if (pLookup->u.HCPhys.pvR0)
+                return (RTR0PTR)((RTR0UINTPTR)pLookup->u.HCPhys.pvR0 + off);
+            AssertMsgFailed(("%s\n", R3STRING(pLookup->pszDesc)));
+            return NIL_RTR0PTR;
+
+        default:
+            AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+            return NIL_RTR0PTR;
+    }
+}
+
+
+/**
+ * Calculate the raw-mode context address of an offset into the HMA memory chunk.
+ *
+ * @returns the raw-mode context base address.
+ * @param   pVM         The cross context VM structure.
+ * @param   pLookup     The HMA lookup record.
+ * @param   off         The offset into the HMA memory chunk.
+ */
+DECLINLINE(RTRCPTR) mmHyperLookupCalcRC(PVM pVM, PMMLOOKUPHYPER pLookup, uint32_t off)
+{
+    return (RTRCPTR)((RTRCUINTPTR)pVM->mm.s.pvHyperAreaGC + pLookup->off + off);
+}
+
+
+/**
+ * Calculate the guest context address of an offset into the HMA memory chunk.
+ *
+ * @returns the guest context base address.
+ * @param   pVM         The cross context VM structure.
+ * @param   pLookup     The HMA lookup record.
+ * @param   off         The offset into the HMA memory chunk.
+ */
+DECLINLINE(void *) mmHyperLookupCalcCC(PVM pVM, PMMLOOKUPHYPER pLookup, uint32_t off)
+{
+#ifdef IN_RING0
+    return mmHyperLookupCalcR0(pVM, pLookup, off);
+#elif defined(IN_RING3)
+    NOREF(pVM);
+    return mmHyperLookupCalcR3(pLookup, off);
+#else
+# error "Neither IN_RING0 nor IN_RING3!"
+#endif
+}
+
+
+/**
+ * Converts a ring-0 host context address in the Hypervisor memory region to a ring-3 host context address.
+ *
+ * @returns ring-3 host context address.
+ * @param   pVM         The cross context VM structure.
+ * @param   R0Ptr       The ring-0 host context address.
+ *                      You'll be damned if this is not in the HMA! :-)
+ * @thread  The Emulation Thread.
+ */
+VMMDECL(RTR3PTR) MMHyperR0ToR3(PVM pVM, RTR0PTR R0Ptr)
+{
+    uint32_t off;
+    PMMLOOKUPHYPER pLookup = mmHyperLookupR0(pVM, R0Ptr, &off);
+    if (pLookup)
+        return mmHyperLookupCalcR3(pLookup, off);
+    return NIL_RTR3PTR;
+}
+
+
+/**
+ * Converts a ring-0 host context address in the Hypervisor memory region to a raw-mode context address.
+ *
+ * @returns raw-mode context address.
+ * @param   pVM         The cross context VM structure.
+ * @param   R0Ptr       The ring-0 host context address.
+ *                      You'll be damned if this is not in the HMA! :-)
+ * @thread  The Emulation Thread.
+ */
+VMMDECL(RTRCPTR) MMHyperR0ToRC(PVM pVM, RTR0PTR R0Ptr)
+{
+    uint32_t off;
+    PMMLOOKUPHYPER pLookup = mmHyperLookupR0(pVM, R0Ptr, &off);
+    if (pLookup)
+        return mmHyperLookupCalcRC(pVM, pLookup, off);
+    return NIL_RTRCPTR;
+}
+
+
+#ifndef IN_RING0
+/**
+ * Converts a ring-0 host context address in the Hypervisor memory region to a current context address.
+ *
+ * @returns current context address.
+ * @param   pVM         The cross context VM structure.
+ * @param   R0Ptr       The ring-0 host context address.
+ *                      You'll be damned if this is not in the HMA! :-)
+ * @thread  The Emulation Thread.
+ */
+VMMDECL(void *) MMHyperR0ToCC(PVM pVM, RTR0PTR R0Ptr)
+{
+    uint32_t off;
+    PMMLOOKUPHYPER pLookup = mmHyperLookupR0(pVM, R0Ptr, &off);
+    if (pLookup)
+        return mmHyperLookupCalcCC(pVM, pLookup, off);
+    return NULL;
+}
+#endif
+
+
+/**
+ * Converts a ring-3 host context address in the Hypervisor memory region to a ring-0 host context address.
+ *
+ * @returns ring-0 host context address.
+ * @param   pVM         The cross context VM structure.
+ * @param   R3Ptr       The ring-3 host context address.
+ *                      You'll be damned if this is not in the HMA! :-)
+ * @thread  The Emulation Thread.
+ */
+VMMDECL(RTR0PTR) MMHyperR3ToR0(PVM pVM, RTR3PTR R3Ptr)
+{
+    uint32_t off;
+    PMMLOOKUPHYPER pLookup = mmHyperLookupR3(pVM, R3Ptr, &off);
+    if (pLookup)
+        return mmHyperLookupCalcR0(pVM, pLookup, off);
+    AssertMsgFailed(("R3Ptr=%p is not inside the hypervisor memory area!\n", R3Ptr));
+    return NIL_RTR0PTR;
+}
+
+
+/**
+ * Converts a ring-3 host context address in the Hypervisor memory region to a guest context address.
+ *
+ * @returns guest context address.
+ * @param   pVM         The cross context VM structure.
+ * @param   R3Ptr       The ring-3 host context address.
+ *                      You'll be damned if this is not in the HMA! :-)
+ * @thread  The Emulation Thread.
+ */
+VMMDECL(RTRCPTR) MMHyperR3ToRC(PVM pVM, RTR3PTR R3Ptr)
+{
+    uint32_t off;
+    PMMLOOKUPHYPER pLookup = mmHyperLookupR3(pVM, R3Ptr, &off);
+    if (pLookup)
+        return mmHyperLookupCalcRC(pVM, pLookup, off);
+    AssertMsgFailed(("R3Ptr=%p is not inside the hypervisor memory area!\n", R3Ptr));
+    return NIL_RTRCPTR;
+}
+
+
+#ifndef IN_RING3
+/**
+ * Converts a ring-3 host context address in the Hypervisor memory region to a current context address.
+ *
+ * @returns current context address.
+ * @param   pVM         The cross context VM structure.
+ * @param   R3Ptr       The ring-3 host context address.
+ *                      You'll be damned if this is not in the HMA! :-)
+ * @thread  The Emulation Thread.
+ */
+VMMDECL(void *) MMHyperR3ToCC(PVM pVM, RTR3PTR R3Ptr)
+{
+    uint32_t off;
+    PMMLOOKUPHYPER pLookup = mmHyperLookupR3(pVM, R3Ptr, &off);
+    if (pLookup)
+        return mmHyperLookupCalcCC(pVM, pLookup, off);
+    return NULL;
+}
+#endif
+
+
+/**
+ * Converts a raw-mode context address in the Hypervisor memory region to a ring-3 context address.
+ *
+ * @returns ring-3 host context address.
+ * @param   pVM         The cross context VM structure.
+ * @param   RCPtr       The raw-mode context address.
+ *                      You'll be damned if this is not in the HMA! :-)
+ * @thread  The Emulation Thread.
+ */
+VMMDECL(RTR3PTR) MMHyperRCToR3(PVM pVM, RTRCPTR RCPtr)
+{
+    uint32_t off;
+    PMMLOOKUPHYPER pLookup = mmHyperLookupRC(pVM, RCPtr, &off);
+    if (pLookup)
+        return mmHyperLookupCalcR3(pLookup, off);
+    return NIL_RTR3PTR;
+}
+
+
+/**
+ * Converts a raw-mode context address in the Hypervisor memory region to a ring-0 host context address.
+ *
+ * @returns ring-0 host context address.
+ * @param   pVM         The cross context VM structure.
+ * @param   RCPtr       The raw-mode context address.
+ *                      You'll be damned if this is not in the HMA! :-)
+ * @thread  The Emulation Thread.
+ */
+VMMDECL(RTR0PTR) MMHyperRCToR0(PVM pVM, RTRCPTR RCPtr)
+{
+    uint32_t off;
+    PMMLOOKUPHYPER pLookup = mmHyperLookupRC(pVM, RCPtr, &off);
+    if (pLookup)
+        return mmHyperLookupCalcR0(pVM, pLookup, off);
+    return NIL_RTR0PTR;
+}
+
+
+/**
+ * Converts a raw-mode context address in the Hypervisor memory region to a current context address.
+ *
+ * @returns current context address.
+ * @param   pVM         The cross context VM structure.
+ * @param   RCPtr       The raw-mode host context address.
+ *                      You'll be damned if this is not in the HMA! :-)
+ * @thread  The Emulation Thread.
+ */
+VMMDECL(void *) MMHyperRCToCC(PVM pVM, RTRCPTR RCPtr)
+{
+    uint32_t off;
+    PMMLOOKUPHYPER pLookup = mmHyperLookupRC(pVM, RCPtr, &off);
+    if (pLookup)
+        return mmHyperLookupCalcCC(pVM, pLookup, off);
+    return NULL;
+}
+
+
+#ifndef IN_RING3
+/**
+ * Converts a current context address in the Hypervisor memory region to a ring-3 host context address.
+ *
+ * @returns ring-3 host context address.
+ * @param   pVM         The cross context VM structure.
+ * @param   pv          The current context address.
+ *                      You'll be damned if this is not in the HMA! :-)
+ * @thread  The Emulation Thread.
+ */
+VMMDECL(RTR3PTR) MMHyperCCToR3(PVM pVM, void *pv)
+{
+    uint32_t off;
+    PMMLOOKUPHYPER pLookup = mmHyperLookupCC(pVM, pv, &off);
+    if (pLookup)
+        return mmHyperLookupCalcR3(pLookup, off);
+    return NIL_RTR3PTR;
+}
+#endif
+
+#ifndef IN_RING0
+/**
+ * Converts a current context address in the Hypervisor memory region to a ring-0 host context address.
+ *
+ * @returns ring-0 host context address.
+ * @param   pVM         The cross context VM structure.
+ * @param   pv          The current context address.
+ *                      You'll be damned if this is not in the HMA! :-)
+ * @thread  The Emulation Thread.
+ */
+VMMDECL(RTR0PTR) MMHyperCCToR0(PVM pVM, void *pv)
+{
+    uint32_t off;
+    PMMLOOKUPHYPER pLookup = mmHyperLookupCC(pVM, pv, &off);
+    if (pLookup)
+        return mmHyperLookupCalcR0(pVM, pLookup, off);
+    return NIL_RTR0PTR;
+}
+#endif
+
+
+/**
+ * Converts a current context address in the Hypervisor memory region to a raw-mode context address.
+ *
+ * @returns guest context address.
+ * @param   pVM         The cross context VM structure.
+ * @param   pv          The current context address.
+ *                      You'll be damned if this is not in the HMA! :-)
+ * @thread  The Emulation Thread.
+ */
+VMMDECL(RTRCPTR) MMHyperCCToRC(PVM pVM, void *pv)
+{
+    uint32_t off;
+    PMMLOOKUPHYPER pLookup = mmHyperLookupCC(pVM, pv, &off);
+    if (pLookup)
+        return mmHyperLookupCalcRC(pVM, pLookup, off);
+    return NIL_RTRCPTR;
+}
+
+
+/**
+ * Gets the string name of a memory tag.
+ *
+ * @returns name of enmTag.
+ * @param   enmTag      The tag.
+ */
+const char *mmGetTagName(MMTAG enmTag)
+{
+    switch (enmTag)
+    {
+        #define TAG2STR(tag) case MM_TAG_##tag: return #tag
+
+        TAG2STR(CFGM);
+        TAG2STR(CFGM_BYTES);
+        TAG2STR(CFGM_STRING);
+        TAG2STR(CFGM_USER);
+
+        TAG2STR(CPUM_CTX);
+        TAG2STR(CPUM_CPUID);
+        TAG2STR(CPUM_MSRS);
+
+        TAG2STR(CSAM);
+        TAG2STR(CSAM_PATCH);
+
+        TAG2STR(DBGF);
+        TAG2STR(DBGF_AS);
+        TAG2STR(DBGF_INFO);
+        TAG2STR(DBGF_LINE);
+        TAG2STR(DBGF_LINE_DUP);
+        TAG2STR(DBGF_MODULE);
+        TAG2STR(DBGF_OS);
+        TAG2STR(DBGF_REG);
+        TAG2STR(DBGF_STACK);
+        TAG2STR(DBGF_SYMBOL);
+        TAG2STR(DBGF_SYMBOL_DUP);
+        TAG2STR(DBGF_TYPE);
+
+        TAG2STR(EM);
+
+        TAG2STR(IEM);
+
+        TAG2STR(IOM);
+        TAG2STR(IOM_STATS);
+
+        TAG2STR(MM);
+        TAG2STR(MM_LOOKUP_GUEST);
+        TAG2STR(MM_LOOKUP_PHYS);
+        TAG2STR(MM_LOOKUP_VIRT);
+        TAG2STR(MM_PAGE);
+
+        TAG2STR(PARAV);
+
+        TAG2STR(PATM);
+        TAG2STR(PATM_PATCH);
+
+        TAG2STR(PDM);
+        TAG2STR(PDM_DEVICE);
+        TAG2STR(PDM_DEVICE_DESC);
+        TAG2STR(PDM_DEVICE_USER);
+        TAG2STR(PDM_DRIVER);
+        TAG2STR(PDM_DRIVER_DESC);
+        TAG2STR(PDM_DRIVER_USER);
+        TAG2STR(PDM_USB);
+        TAG2STR(PDM_USB_DESC);
+        TAG2STR(PDM_USB_USER);
+        TAG2STR(PDM_LUN);
+        TAG2STR(PDM_QUEUE);
+        TAG2STR(PDM_THREAD);
+        TAG2STR(PDM_ASYNC_COMPLETION);
+#ifdef VBOX_WITH_NETSHAPER
+        TAG2STR(PDM_NET_SHAPER);
+#endif /* VBOX_WITH_NETSHAPER */
+
+        TAG2STR(PGM);
+        TAG2STR(PGM_CHUNK_MAPPING);
+        TAG2STR(PGM_HANDLERS);
+        TAG2STR(PGM_HANDLER_TYPES);
+        TAG2STR(PGM_MAPPINGS);
+        TAG2STR(PGM_PHYS);
+        TAG2STR(PGM_POOL);
+
+        TAG2STR(REM);
+
+        TAG2STR(SELM);
+
+        TAG2STR(SSM);
+
+        TAG2STR(STAM);
+
+        TAG2STR(TM);
+
+        TAG2STR(TRPM);
+
+        TAG2STR(VM);
+        TAG2STR(VM_REQ);
+
+        TAG2STR(VMM);
+
+        TAG2STR(HM);
+
+        #undef TAG2STR
+
+        default:
+        {
+            AssertMsgFailed(("Unknown tag %d! forgot to add it to the switch?\n", enmTag));
+#ifdef IN_RING3
+            static char sz[48];
+            RTStrPrintf(sz, sizeof(sz), "%d", enmTag);
+            return sz;
+#else
+            return "unknown tag!";
+#endif
+        }
+    }
+}
+
diff --git a/src/VBox/VMM/VMMAll/MMAllHyper.cpp b/src/VBox/VMM/VMMAll/MMAllHyper.cpp
new file mode 100644
index 00000000..0f440c86
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/MMAllHyper.cpp
@@ -0,0 +1,1337 @@
+/* $Id: MMAllHyper.cpp $ */
+/** @file
+ * MM - Memory Manager - Hypervisor Memory Area, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM_HYPER_HEAP
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/stam.h>
+#include "MMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#define ASSERT_L(u1, u2)    AssertMsg((u1) <  (u2), ("u1=%#x u2=%#x\n", u1, u2))
+#define ASSERT_LE(u1, u2)   AssertMsg((u1) <= (u2), ("u1=%#x u2=%#x\n", u1, u2))
+#define ASSERT_GE(u1, u2)   AssertMsg((u1) >= (u2), ("u1=%#x u2=%#x\n", u1, u2))
+#define ASSERT_ALIGN(u1)    AssertMsg(!((u1) & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("u1=%#x (%d)\n", u1, u1))
+
+#define ASSERT_OFFPREV(pHeap, pChunk)  \
+    do { Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) <= 0); \
+         Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
+         AssertMsg(     MMHYPERCHUNK_GET_OFFPREV(pChunk) != 0 \
+                   ||   (uint8_t *)(pChunk) == (pHeap)->CTX_SUFF(pbHeap), \
+                   ("pChunk=%p pvHyperHeap=%p\n", (pChunk), (pHeap)->CTX_SUFF(pbHeap))); \
+    } while (0)
+
+#define ASSERT_OFFNEXT(pHeap, pChunk) \
+    do { ASSERT_ALIGN((pChunk)->offNext); \
+         ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
+    } while (0)
+
+#define ASSERT_OFFHEAP(pHeap, pChunk) \
+    do { Assert((pChunk)->offHeap); \
+         AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
+                   ("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
+         Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
+    } while (0)
+
+#ifdef VBOX_WITH_STATISTICS
+#define ASSERT_OFFSTAT(pHeap, pChunk) \
+    do { if (MMHYPERCHUNK_ISFREE(pChunk)) \
+             Assert(!(pChunk)->offStat); \
+         else if ((pChunk)->offStat) \
+         { \
+             Assert((pChunk)->offStat); \
+             AssertMsg(!((pChunk)->offStat & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("offStat=%RX32\n", (pChunk)->offStat)); \
+             uintptr_t uPtr = (uintptr_t)(pChunk)->offStat + (uintptr_t)pChunk; NOREF(uPtr); \
+             AssertMsg(uPtr - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) < (pHeap)->offPageAligned, \
+                       ("%p - %p < %RX32\n", uPtr, (pHeap)->CTX_SUFF(pbHeap), (pHeap)->offPageAligned)); \
+         } \
+    } while (0)
+#else
+#define ASSERT_OFFSTAT(pHeap, pChunk) \
+    do { Assert(!(pChunk)->offStat); \
+    } while (0)
+#endif
+
+#define ASSERT_CHUNK(pHeap, pChunk) \
+    do { ASSERT_OFFNEXT(pHeap, pChunk); \
+         ASSERT_OFFPREV(pHeap, pChunk); \
+         ASSERT_OFFHEAP(pHeap, pChunk); \
+         ASSERT_OFFSTAT(pHeap, pChunk); \
+    } while (0)
+#define ASSERT_CHUNK_USED(pHeap, pChunk) \
+    do { ASSERT_OFFNEXT(pHeap, pChunk); \
+         ASSERT_OFFPREV(pHeap, pChunk); \
+         Assert(MMHYPERCHUNK_ISUSED(pChunk)); \
+    } while (0)
+
+#define ASSERT_FREE_OFFPREV(pHeap, pChunk)  \
+    do { ASSERT_ALIGN((pChunk)->offPrev); \
+         ASSERT_GE(((pChunk)->offPrev  & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
+         Assert((pChunk)->offPrev != MMHYPERCHUNK_GET_OFFPREV(&(pChunk)->core) || !(pChunk)->offPrev); \
+         AssertMsg(    (pChunk)->offPrev \
+                  ||   (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeHead, \
+                  ("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap),\
+                  (pHeap)->offFreeHead)); \
+    } while (0)
+
+#define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
+    do { ASSERT_ALIGN((pChunk)->offNext); \
+         ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
+         Assert((pChunk)->offNext != (pChunk)->core.offNext || !(pChunk)->offNext); \
+         AssertMsg(     (pChunk)->offNext \
+                   ||   (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeTail, \
+                   ("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk),  (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap), \
+                    (pHeap)->offFreeTail)); \
+    } while (0)
+
+#define ASSERT_FREE_CB(pHeap, pChunk) \
+    do { ASSERT_ALIGN((pChunk)->cb); \
+         Assert((pChunk)->cb > 0); \
+         if ((pChunk)->core.offNext) \
+             AssertMsg((pChunk)->cb == ((pChunk)->core.offNext - sizeof(MMHYPERCHUNK)), \
+                       ("cb=%d offNext=%d\n", (pChunk)->cb, (pChunk)->core.offNext)); \
+         else \
+             ASSERT_LE((pChunk)->cb, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
+    } while (0)
+
+#define ASSERT_CHUNK_FREE(pHeap, pChunk) \
+    do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
+         Assert(MMHYPERCHUNK_ISFREE(pChunk)); \
+         ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
+         ASSERT_FREE_OFFPREV(pHeap, pChunk); \
+         ASSERT_FREE_CB(pHeap, pChunk); \
+    } while (0)
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
+static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
+#ifdef VBOX_WITH_STATISTICS
+static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
+#ifdef IN_RING3
+static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat);
+#endif
+#endif
+static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
+#ifdef MMHYPER_HEAP_STRICT
+static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
+#endif
+
+
+
+/**
+ * Locks the hypervisor heap.
+ * This might call back to Ring-3 in order to deal with lock contention in GC and R3.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+static int mmHyperLock(PVMCC pVM)
+{
+    PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
+
+#ifdef IN_RING3
+    if (!PDMCritSectIsInitialized(&pHeap->Lock))
+        return VINF_SUCCESS;     /* early init */
+#else
+    Assert(PDMCritSectIsInitialized(&pHeap->Lock));
+#endif
+    int rc = PDMCritSectEnter(&pHeap->Lock, VERR_SEM_BUSY);
+#ifdef IN_RING0
+    if (rc == VERR_SEM_BUSY)
+        rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_MMHYPER_LOCK, 0);
+#endif
+    AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * Unlocks the hypervisor heap.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+static void mmHyperUnlock(PVM pVM)
+{
+    PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
+
+#ifdef IN_RING3
+    if (!PDMCritSectIsInitialized(&pHeap->Lock))
+        return;     /* early init */
+#endif
+    Assert(PDMCritSectIsInitialized(&pHeap->Lock));
+    PDMCritSectLeave(&pHeap->Lock);
+}
+
+/**
+ * Allocates memory in the Hypervisor (RC VMM) area.
+ * The returned memory is of course zeroed.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   cb          Number of bytes to allocate.
+ * @param   uAlignment  Required memory alignment in bytes.
+ *                      Values are 0,8,16,32,64 and PAGE_SIZE.
+ *                      0 -> default alignment, i.e. 8 bytes.
+ * @param   enmTag      The statistics tag.
+ * @param   ppv         Where to store the address to the allocated
+ *                      memory.
+ */
+static int mmHyperAllocInternal(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
+{
+    AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
+
+    /*
+     * Validate input and adjust it to reasonable values.
+     */
+    if (!uAlignment || uAlignment < MMHYPER_HEAP_ALIGN_MIN)
+        uAlignment = MMHYPER_HEAP_ALIGN_MIN;
+    uint32_t cbAligned;
+    switch (uAlignment)
+    {
+        case 8:
+        case 16:
+        case 32:
+        case 64:
+            cbAligned = RT_ALIGN_32(cb, MMHYPER_HEAP_ALIGN_MIN);
+            if (!cbAligned || cbAligned < cb)
+            {
+                Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
+                AssertMsgFailed(("Nice try.\n"));
+                return VERR_INVALID_PARAMETER;
+            }
+            break;
+
+        case PAGE_SIZE:
+            AssertMsg(RT_ALIGN_32(cb, PAGE_SIZE) == cb, ("The size isn't page aligned. (cb=%#x)\n", cb));
+            cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
+            if (!cbAligned)
+            {
+                Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
+                AssertMsgFailed(("Nice try.\n"));
+                return VERR_INVALID_PARAMETER;
+            }
+            break;
+
+        default:
+            Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
+            AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
+            return VERR_INVALID_PARAMETER;
+    }
+
+
+    /*
+     * Get heap and statisticsStatistics.
+     */
+    PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
+#ifdef VBOX_WITH_STATISTICS
+    PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
+    if (!pStat)
+    {
+        Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
+        AssertMsgFailed(("Failed to allocate statistics!\n"));
+        return VERR_MM_HYPER_NO_MEMORY;
+    }
+#else
+    NOREF(enmTag);
+#endif
+    if (uAlignment < PAGE_SIZE)
+    {
+        /*
+         * Allocate a chunk.
+         */
+        PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
+        if (pChunk)
+        {
+#ifdef VBOX_WITH_STATISTICS
+            const uint32_t cbChunk = pChunk->offNext
+                ? pChunk->offNext
+                : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
+            pStat->cbAllocated += (uint32_t)cbChunk;
+            pStat->cbCurAllocated += (uint32_t)cbChunk;
+            if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
+                pStat->cbMaxAllocated = pStat->cbCurAllocated;
+            pStat->cAllocations++;
+            pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
+#else
+            pChunk->offStat = 0;
+#endif
+            void *pv = pChunk + 1;
+            *ppv = pv;
+            ASMMemZero32(pv, cbAligned);
+            Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, pv));
+            return VINF_SUCCESS;
+        }
+    }
+    else
+    {
+        /*
+         * Allocate page aligned memory.
+         */
+        void *pv = mmHyperAllocPages(pHeap, cbAligned);
+        if (pv)
+        {
+#ifdef VBOX_WITH_STATISTICS
+            pStat->cbAllocated += cbAligned;
+            pStat->cbCurAllocated += cbAligned;
+            if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
+                pStat->cbMaxAllocated = pStat->cbCurAllocated;
+            pStat->cAllocations++;
+#endif
+            *ppv = pv;
+            /* ASMMemZero32(pv, cbAligned); - not required since memory is alloc-only and SUPR3PageAlloc zeros it. */
+            Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, ppv));
+            return VINF_SUCCESS;
+        }
+    }
+
+#ifdef VBOX_WITH_STATISTICS
+    pStat->cAllocations++;
+    pStat->cFailures++;
+#endif
+    Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
+    AssertMsgFailed(("Failed to allocate %d bytes!\n", cb));
+    return VERR_MM_HYPER_NO_MEMORY;
+}
+
+
+/**
+ * Wrapper for mmHyperAllocInternal
+ */
+VMMDECL(int) MMHyperAlloc(PVMCC pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
+{
+    int rc = mmHyperLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    LogFlow(("MMHyperAlloc %x align=%x tag=%s\n", cb, uAlignment, mmGetTagName(enmTag)));
+
+    rc = mmHyperAllocInternal(pVM, cb, uAlignment, enmTag, ppv);
+
+    mmHyperUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Duplicates a block of memory.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvSrc       The source memory block to copy from.
+ * @param   cb          Size of the source memory block.
+ * @param   uAlignment  Required memory alignment in bytes.
+ *                      Values are 0,8,16,32,64 and PAGE_SIZE.
+ *                      0 -> default alignment, i.e. 8 bytes.
+ * @param   enmTag      The statistics tag.
+ * @param   ppv         Where to store the address to the allocated
+ *                      memory.
+ */
+VMMDECL(int) MMHyperDupMem(PVMCC pVM, const void *pvSrc, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
+{
+    int rc = MMHyperAlloc(pVM, cb, uAlignment, enmTag, ppv);
+    if (RT_SUCCESS(rc))
+        memcpy(*ppv, pvSrc, cb);
+    return rc;
+}
+
+
+/**
+ * Allocates a chunk of memory from the specified heap.
+ * The caller validates the parameters of this request.
+ *
+ * @returns Pointer to the allocated chunk.
+ * @returns NULL on failure.
+ * @param   pHeap       The heap.
+ * @param   cb          Size of the memory block to allocate.
+ * @param   uAlignment  The alignment specifications for the allocated block.
+ * @internal
+ */
+static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
+{
+    Log3(("mmHyperAllocChunk: Enter cb=%#x uAlignment=%#x\n", cb, uAlignment));
+#ifdef MMHYPER_HEAP_STRICT
+    mmHyperHeapCheck(pHeap);
+#endif
+#ifdef MMHYPER_HEAP_STRICT_FENCE
+    uint32_t cbFence = RT_MAX(MMHYPER_HEAP_STRICT_FENCE_SIZE, uAlignment);
+    cb += cbFence;
+#endif
+
+    /*
+     * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
+     */
+    if (pHeap->offFreeHead == NIL_OFFSET)
+        return NULL;
+
+    /*
+     * Small alignments - from the front of the heap.
+     *
+     * Must split off free chunks at the end to prevent messing up the
+     * last free node which we take the page aligned memory from the top of.
+     */
+    PMMHYPERCHUNK     pRet = NULL;
+    PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeHead);
+    while (pFree)
+    {
+        ASSERT_CHUNK_FREE(pHeap, pFree);
+        if (pFree->cb >= cb)
+        {
+            unsigned offAlign = (uintptr_t)(&pFree->core + 1) & (uAlignment - 1);
+            if (offAlign)
+                offAlign = uAlignment - offAlign;
+            if (!offAlign || pFree->cb - offAlign >= cb)
+            {
+                Log3(("mmHyperAllocChunk: Using pFree=%p pFree->cb=%d offAlign=%d\n", pFree, pFree->cb, offAlign));
+
+                /*
+                 * Adjust the node in front.
+                 * Because of multiple alignments we need to special case allocation of the first block.
+                 */
+                if (offAlign)
+                {
+                    MMHYPERCHUNKFREE Free = *pFree;
+                    if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
+                    {
+                        /* just add a bit of memory to it. */
+                        PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&Free.core));
+                        pPrev->core.offNext += offAlign;
+                        AssertMsg(!MMHYPERCHUNK_ISFREE(&pPrev->core), ("Impossible!\n"));
+                        Log3(("mmHyperAllocChunk: Added %d bytes to %p\n", offAlign, pPrev));
+                    }
+                    else
+                    {
+                        /* make new head node, mark it USED for simplicity. */
+                        PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)pHeap->CTX_SUFF(pbHeap);
+                        Assert(pPrev == &pFree->core);
+                        pPrev->offPrev = 0;
+                        MMHYPERCHUNK_SET_TYPE(pPrev, MMHYPERCHUNK_FLAGS_USED);
+                        pPrev->offNext = offAlign;
+                        Log3(("mmHyperAllocChunk: Created new first node of %d bytes\n", offAlign));
+
+                    }
+                    Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
+                    pHeap->cbFree -= offAlign;
+
+                    /* Recreate pFree node and adjusting everything... */
+                    pFree = (PMMHYPERCHUNKFREE)((char *)pFree + offAlign);
+                    *pFree = Free;
+
+                    pFree->cb -= offAlign;
+                    if (pFree->core.offNext)
+                    {
+                        pFree->core.offNext -= offAlign;
+                        PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
+                        MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
+                        ASSERT_CHUNK(pHeap, pNext);
+                    }
+                    if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
+                        MMHYPERCHUNK_SET_OFFPREV(&pFree->core, MMHYPERCHUNK_GET_OFFPREV(&pFree->core) - offAlign);
+
+                    if (pFree->offNext)
+                    {
+                        pFree->offNext -= offAlign;
+                        PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
+                        pNext->offPrev = -(int32_t)pFree->offNext;
+                        ASSERT_CHUNK_FREE(pHeap, pNext);
+                    }
+                    else
+                        pHeap->offFreeTail += offAlign;
+                    if (pFree->offPrev)
+                    {
+                        pFree->offPrev -= offAlign;
+                        PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
+                        pPrev->offNext = -pFree->offPrev;
+                        ASSERT_CHUNK_FREE(pHeap, pPrev);
+                    }
+                    else
+                        pHeap->offFreeHead += offAlign;
+                    pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
+                    pFree->core.offStat = 0;
+                    ASSERT_CHUNK_FREE(pHeap, pFree);
+                    Log3(("mmHyperAllocChunk: Realigned pFree=%p\n", pFree));
+                }
+
+                /*
+                 * Split off a new FREE chunk?
+                 */
+                if (pFree->cb >= cb + RT_ALIGN(sizeof(MMHYPERCHUNKFREE), MMHYPER_HEAP_ALIGN_MIN))
+                {
+                    /*
+                     * Move the FREE chunk up to make room for the new USED chunk.
+                     */
+                    const int           off = cb + sizeof(MMHYPERCHUNK);
+                    PMMHYPERCHUNKFREE   pNew = (PMMHYPERCHUNKFREE)((char *)&pFree->core + off);
+                    *pNew = *pFree;
+                    pNew->cb -= off;
+                    if (pNew->core.offNext)
+                    {
+                        pNew->core.offNext -= off;
+                        PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pNew + pNew->core.offNext);
+                        MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pNew->core.offNext);
+                        ASSERT_CHUNK(pHeap, pNext);
+                    }
+                    pNew->core.offPrev  = -off;
+                    MMHYPERCHUNK_SET_TYPE(pNew, MMHYPERCHUNK_FLAGS_FREE);
+
+                    if (pNew->offNext)
+                    {
+                        pNew->offNext -= off;
+                        PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offNext);
+                        pNext->offPrev = -(int32_t)pNew->offNext;
+                        ASSERT_CHUNK_FREE(pHeap, pNext);
+                    }
+                    else
+                        pHeap->offFreeTail += off;
+                    if (pNew->offPrev)
+                    {
+                        pNew->offPrev -= off;
+                        PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offPrev);
+                        pPrev->offNext = -pNew->offPrev;
+                        ASSERT_CHUNK_FREE(pHeap, pPrev);
+                    }
+                    else
+                        pHeap->offFreeHead += off;
+                    pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
+                    pNew->core.offStat = 0;
+                    ASSERT_CHUNK_FREE(pHeap, pNew);
+
+                    /*
+                     * Update the old FREE node making it a USED node.
+                     */
+                    pFree->core.offNext = off;
+                    MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
+
+
+                    Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
+                          pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
+                    pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
+                    pRet = &pFree->core;
+                    ASSERT_CHUNK(pHeap, &pFree->core);
+                    Log3(("mmHyperAllocChunk: Created free chunk pNew=%p cb=%d\n", pNew, pNew->cb));
+                }
+                else
+                {
+                    /*
+                     * Link out of free list.
+                     */
+                    if (pFree->offNext)
+                    {
+                        PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
+                        if (pFree->offPrev)
+                        {
+                            pNext->offPrev += pFree->offPrev;
+                            PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
+                            pPrev->offNext += pFree->offNext;
+                            ASSERT_CHUNK_FREE(pHeap, pPrev);
+                        }
+                        else
+                        {
+                            pHeap->offFreeHead += pFree->offNext;
+                            pNext->offPrev = 0;
+                        }
+                        ASSERT_CHUNK_FREE(pHeap, pNext);
+                    }
+                    else
+                    {
+                        if (pFree->offPrev)
+                        {
+                            pHeap->offFreeTail += pFree->offPrev;
+                            PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
+                            pPrev->offNext = 0;
+                            ASSERT_CHUNK_FREE(pHeap, pPrev);
+                        }
+                        else
+                        {
+                            pHeap->offFreeHead = NIL_OFFSET;
+                            pHeap->offFreeTail = NIL_OFFSET;
+                        }
+                    }
+
+                    Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
+                          pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
+                    pHeap->cbFree -= pFree->cb;
+                    MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
+                    pRet = &pFree->core;
+                    ASSERT_CHUNK(pHeap, &pFree->core);
+                    Log3(("mmHyperAllocChunk: Converted free chunk %p to used chunk.\n", pFree));
+                }
+                Log3(("mmHyperAllocChunk: Returning %p\n", pRet));
+                break;
+            }
+        }
+
+        /* next */
+        pFree = pFree->offNext ? (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext) : NULL;
+    }
+
+#ifdef MMHYPER_HEAP_STRICT_FENCE
+    uint32_t *pu32End = (uint32_t *)((uint8_t *)(pRet + 1) + cb);
+    uint32_t *pu32EndReal = pRet->offNext
+                          ? (uint32_t *)((uint8_t *)pRet + pRet->offNext)
+                          : (uint32_t *)(pHeap->CTX_SUFF(pbHeap) + pHeap->cbHeap);
+    cbFence += (uintptr_t)pu32EndReal - (uintptr_t)pu32End; Assert(!(cbFence & 0x3));
+    ASMMemFill32((uint8_t *)pu32EndReal - cbFence, cbFence, MMHYPER_HEAP_STRICT_FENCE_U32);
+    pu32EndReal[-1] = cbFence;
+#endif
+#ifdef MMHYPER_HEAP_STRICT
+    mmHyperHeapCheck(pHeap);
+#endif
+    return pRet;
+}
+
+
+/**
+ * Allocates one or more pages of memory from the specified heap.
+ * The caller validates the parameters of this request.
+ *
+ * @returns Pointer to the allocated chunk.
+ * @returns NULL on failure.
+ * @param   pHeap       The heap.
+ * @param   cb          Size of the memory block to allocate.
+ * @internal
+ */
+static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
+{
+    Log3(("mmHyperAllocPages: Enter cb=%#x\n", cb));
+
+#ifdef MMHYPER_HEAP_STRICT
+    mmHyperHeapCheck(pHeap);
+#endif
+
+    /*
+     * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
+     */
+    if (pHeap->offFreeHead == NIL_OFFSET)
+        return NULL;
+
+    /*
+     * Page aligned chunks.
+     *
+     * Page aligned chunks can only be allocated from the last FREE chunk.
+     * This is for reasons of simplicity and fragmentation. Page aligned memory
+     * must also be allocated in page aligned sizes. Page aligned memory cannot
+     * be freed either.
+     *
+     * So, for this to work, the last FREE chunk needs to end on a page aligned
+     * boundary.
+     */
+    PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail);
+    ASSERT_CHUNK_FREE(pHeap, pFree);
+    if (    (((uintptr_t)(&pFree->core + 1) + pFree->cb) & (PAGE_OFFSET_MASK - 1))
+        ||  pFree->cb + sizeof(MMHYPERCHUNK) < cb)
+    {
+        Log3(("mmHyperAllocPages: Not enough/no page aligned memory!\n"));
+        return NULL;
+    }
+
+    void *pvRet;
+    if (pFree->cb > cb)
+    {
+        /*
+         * Simple, just cut the top of the free node and return it.
+         */
+        pFree->cb -= cb;
+        pvRet = (char *)(&pFree->core + 1) + pFree->cb;
+        AssertMsg(RT_ALIGN_P(pvRet, PAGE_SIZE) == pvRet, ("pvRet=%p cb=%#x pFree=%p pFree->cb=%#x\n", pvRet, cb, pFree, pFree->cb));
+        Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
+        pHeap->cbFree -= cb;
+        ASSERT_CHUNK_FREE(pHeap, pFree);
+        Log3(("mmHyperAllocPages: Allocated from pFree=%p new pFree->cb=%d\n", pFree, pFree->cb));
+    }
+    else
+    {
+        /*
+         * Unlink the FREE node.
+         */
+        pvRet = (char *)(&pFree->core + 1) + pFree->cb - cb;
+        Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
+        pHeap->cbFree -= pFree->cb;
+
+        /* a scrap of spare memory (unlikely)? add it to the sprevious chunk. */
+        if (pvRet != (void *)pFree)
+        {
+            AssertMsg(MMHYPERCHUNK_GET_OFFPREV(&pFree->core), ("How the *beep* did someone manage to allocated up all the heap with page aligned memory?!?\n"));
+            PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&pFree->core));
+            pPrev->offNext += (uintptr_t)pvRet - (uintptr_t)pFree;
+            AssertMsg(!MMHYPERCHUNK_ISFREE(pPrev), ("Free bug?\n"));
+#ifdef VBOX_WITH_STATISTICS
+            PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pPrev + pPrev->offStat);
+            pStat->cbAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
+            pStat->cbCurAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
+#endif
+            Log3(("mmHyperAllocPages: Added %d to %p (page align)\n", (uintptr_t)pvRet - (uintptr_t)pFree, pFree));
+        }
+
+        /* unlink from FREE chain. */
+        if (pFree->offPrev)
+        {
+            pHeap->offFreeTail += pFree->offPrev;
+            ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev))->offNext = 0;
+        }
+        else
+        {
+            pHeap->offFreeTail = NIL_OFFSET;
+            pHeap->offFreeHead = NIL_OFFSET;
+        }
+        Log3(("mmHyperAllocPages: Unlinked pFree=%d\n", pFree));
+    }
+    pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
+    Log3(("mmHyperAllocPages: Returning %p (page aligned)\n", pvRet));
+
+#ifdef MMHYPER_HEAP_STRICT
+    mmHyperHeapCheck(pHeap);
+#endif
+    return pvRet;
+}
+
+#ifdef VBOX_WITH_STATISTICS
+
+/**
+ * Get the statistic record for a tag.
+ *
+ * @returns Pointer to a stat record.
+ * @returns NULL on failure.
+ * @param   pHeap       The heap.
+ * @param   enmTag      The tag.
+ */
+static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
+{
+    /* try look it up first. */
+    PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
+    if (!pStat)
+    {
+        /* try allocate a new one */
+        PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
+        if (!pChunk)
+            return NULL;
+        pStat = (PMMHYPERSTAT)(pChunk + 1);
+        pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
+
+        ASMMemZero32(pStat, sizeof(*pStat));
+        pStat->Core.Key = enmTag;
+        RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
+    }
+    if (!pStat->fRegistered)
+    {
+# ifdef IN_RING3
+        mmR3HyperStatRegisterOne(pHeap->pVMR3, pStat);
+# else
+        /** @todo schedule a R3 action. */
+# endif
+    }
+    return pStat;
+}
+
+
+# ifdef IN_RING3
+/**
+ * Registers statistics with STAM.
+ *
+ */
+static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
+{
+    if (pStat->fRegistered)
+        return;
+    const char *pszTag = mmGetTagName((MMTAG)pStat->Core.Key);
+    STAMR3RegisterF(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.",           "/MM/HyperHeap/%s", pszTag);
+    STAMR3RegisterF(pVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of alloc calls.",                         "/MM/HyperHeap/%s/cAllocations", pszTag);
+    STAMR3RegisterF(pVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of free calls.",                          "/MM/HyperHeap/%s/cFrees", pszTag);
+    STAMR3RegisterF(pVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.",                            "/MM/HyperHeap/%s/cFailures", pszTag);
+    STAMR3RegisterF(pVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of allocated bytes.",               "/MM/HyperHeap/%s/cbAllocated", pszTag);
+    STAMR3RegisterF(pVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of freed bytes.",                   "/MM/HyperHeap/%s/cbFreed", pszTag);
+    STAMR3RegisterF(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.","/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
+    pStat->fRegistered = true;
+}
+# endif /* IN_RING3 */
+
+#endif /* VBOX_WITH_STATISTICS */
+
+
+/**
+ * Free memory allocated using MMHyperAlloc().
+ * The caller validates the parameters of this request.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pv          The memory to free.
+ * @remark  Try avoid free hyper memory.
+ */
+static int mmHyperFreeInternal(PVM pVM, void *pv)
+{
+    Log2(("MMHyperFree: pv=%p\n", pv));
+    if (!pv)
+        return VINF_SUCCESS;
+    AssertMsgReturn(RT_ALIGN_P(pv, MMHYPER_HEAP_ALIGN_MIN) == pv,
+                    ("Invalid pointer %p!\n", pv),
+                    VERR_INVALID_POINTER);
+
+    /*
+     * Get the heap and stats.
+     * Validate the chunk at the same time.
+     */
+    PMMHYPERCHUNK   pChunk = (PMMHYPERCHUNK)((PMMHYPERCHUNK)pv - 1);
+
+    AssertMsgReturn(    (uintptr_t)pChunk + pChunk->offNext >= (uintptr_t)pChunk
+                    ||  RT_ALIGN_32(pChunk->offNext, MMHYPER_HEAP_ALIGN_MIN) != pChunk->offNext,
+                    ("%p: offNext=%#RX32\n", pv, pChunk->offNext),
+                    VERR_INVALID_POINTER);
+
+    AssertMsgReturn(MMHYPERCHUNK_ISUSED(pChunk),
+                    ("%p: Not used!\n", pv),
+                    VERR_INVALID_POINTER);
+
+    int32_t offPrev = MMHYPERCHUNK_GET_OFFPREV(pChunk);
+    AssertMsgReturn(    (uintptr_t)pChunk + offPrev <= (uintptr_t)pChunk
+                    && !((uint32_t)-offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)),
+                    ("%p: offPrev=%#RX32!\n", pv, offPrev),
+                    VERR_INVALID_POINTER);
+
+    /* statistics */
+#ifdef VBOX_WITH_STATISTICS
+    PMMHYPERSTAT    pStat = (PMMHYPERSTAT)((uintptr_t)pChunk + pChunk->offStat);
+    AssertMsgReturn(    RT_ALIGN_P(pStat, MMHYPER_HEAP_ALIGN_MIN) == (void *)pStat
+                    &&  pChunk->offStat,
+                    ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
+                    VERR_INVALID_POINTER);
+#else
+    AssertMsgReturn(!pChunk->offStat,
+                    ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
+                    VERR_INVALID_POINTER);
+#endif
+
+    /* The heap structure. */
+    PMMHYPERHEAP    pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
+    AssertMsgReturn(    !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
+                    &&  pChunk->offHeap,
+                    ("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
+                    VERR_INVALID_POINTER);
+
+    AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
+                    ("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
+                    VERR_INVALID_POINTER);
+    Assert(pHeap == pVM->mm.s.CTX_SUFF(pHyperHeap)); NOREF(pVM);
+
+    /* Some more verifications using additional info from pHeap. */
+    AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)pHeap->CTX_SUFF(pbHeap),
+                    ("%p: offPrev=%#RX32!\n", pv, offPrev),
+                    VERR_INVALID_POINTER);
+
+    AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
+                    ("%p: offNext=%#RX32!\n", pv, pChunk->offNext),
+                    VERR_INVALID_POINTER);
+
+    AssertMsgReturn(   (uintptr_t)pv - (uintptr_t)pHeap->CTX_SUFF(pbHeap) <= pHeap->offPageAligned,
+                    ("Invalid pointer %p! (heap: %p-%p)\n", pv, pHeap->CTX_SUFF(pbHeap),
+                    (char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned),
+                    VERR_INVALID_POINTER);
+
+#ifdef MMHYPER_HEAP_STRICT
+    mmHyperHeapCheck(pHeap);
+#endif
+
+#if defined(VBOX_WITH_STATISTICS) || defined(MMHYPER_HEAP_FREE_POISON)
+    /* calc block size. */
+    const uint32_t cbChunk = pChunk->offNext
+        ? pChunk->offNext
+        : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
+#endif
+#ifdef MMHYPER_HEAP_FREE_POISON
+    /* poison the block */
+    memset(pChunk + 1, MMHYPER_HEAP_FREE_POISON, cbChunk - sizeof(*pChunk));
+#endif
+
+#ifdef MMHYPER_HEAP_FREE_DELAY
+# ifdef MMHYPER_HEAP_FREE_POISON
+    /*
+     * Check poison.
+     */
+    unsigned i = RT_ELEMENTS(pHeap->aDelayedFrees);
+    while (i-- > 0)
+        if (pHeap->aDelayedFrees[i].offChunk)
+        {
+            PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
+            const size_t cb = pCur->offNext
+                ? pCur->offNext - sizeof(*pCur)
+                : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pCur - sizeof(*pCur);
+            uint8_t *pab = (uint8_t *)(pCur + 1);
+            for (unsigned off = 0; off < cb; off++)
+                AssertReleaseMsg(pab[off] == 0xCB,
+                                 ("caller=%RTptr cb=%#zx off=%#x: %.*Rhxs\n",
+                                  pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
+        }
+# endif /* MMHYPER_HEAP_FREE_POISON */
+
+    /*
+     * Delayed freeing.
+     */
+    int rc = VINF_SUCCESS;
+    if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
+    {
+        PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
+        rc = mmHyperFree(pHeap, pChunkFree);
+    }
+    pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
+    pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
+    pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % RT_ELEMENTS(pHeap->aDelayedFrees);
+
+#else   /* !MMHYPER_HEAP_FREE_POISON */
+    /*
+     * Call the worker.
+     */
+    int rc = mmHyperFree(pHeap, pChunk);
+#endif  /* !MMHYPER_HEAP_FREE_POISON */
+
+    /*
+     * Update statistics.
+     */
+#ifdef VBOX_WITH_STATISTICS
+    pStat->cFrees++;
+    if (RT_SUCCESS(rc))
+    {
+        pStat->cbFreed        += cbChunk;
+        pStat->cbCurAllocated -= cbChunk;
+    }
+    else
+        pStat->cFailures++;
+#endif
+
+    return rc;
+}
+
+
+/**
+ * Wrapper for mmHyperFreeInternal
+ */
+VMMDECL(int) MMHyperFree(PVMCC pVM, void *pv)
+{
+    int rc;
+
+    rc = mmHyperLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    LogFlow(("MMHyperFree %p\n", pv));
+
+    rc = mmHyperFreeInternal(pVM, pv);
+
+    mmHyperUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Free memory a memory chunk.
+ *
+ * @returns VBox status code.
+ * @param   pHeap       The heap.
+ * @param   pChunk      The memory chunk to free.
+ */
+static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
+{
+    Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
+    PMMHYPERCHUNKFREE   pFree = (PMMHYPERCHUNKFREE)pChunk;
+
+    /*
+     * Insert into the free list (which is sorted on address).
+     *
+     * We'll search towards the end of the heap to locate the
+     * closest FREE chunk.
+     */
+    PMMHYPERCHUNKFREE   pLeft = NULL;
+    PMMHYPERCHUNKFREE   pRight = NULL;
+    if (pHeap->offFreeTail != NIL_OFFSET)
+    {
+        if (pFree->core.offNext)
+        {
+            pRight = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->core.offNext);
+            ASSERT_CHUNK(pHeap, &pRight->core);
+            while (!MMHYPERCHUNK_ISFREE(&pRight->core))
+            {
+                if (!pRight->core.offNext)
+                {
+                    pRight = NULL;
+                    break;
+                }
+                pRight = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->core.offNext);
+                ASSERT_CHUNK(pHeap, &pRight->core);
+            }
+        }
+        if (!pRight)
+            pRight = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail); /** @todo this can't be correct! 'pLeft = .. ; else' I think */
+        if (pRight)
+        {
+            ASSERT_CHUNK_FREE(pHeap, pRight);
+            if (pRight->offPrev)
+            {
+                pLeft = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->offPrev);
+                ASSERT_CHUNK_FREE(pHeap, pLeft);
+            }
+        }
+    }
+    if (pLeft == pFree)
+    {
+        AssertMsgFailed(("Freed twice! pv=%p (pChunk=%p)\n", pChunk + 1, pChunk));
+        return VERR_INVALID_POINTER;
+    }
+    pChunk->offStat = 0;
+
+    /*
+     * Head free chunk list?
+     */
+    if (!pLeft)
+    {
+        MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
+        pFree->offPrev = 0;
+        pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
+        if (pRight)
+        {
+            pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
+            pRight->offPrev = -(int32_t)pFree->offNext;
+        }
+        else
+        {
+            pFree->offNext = 0;
+            pHeap->offFreeTail = pHeap->offFreeHead;
+        }
+        Log3(("mmHyperFree: Inserted %p at head of free chain.\n", pFree));
+    }
+    else
+    {
+        /*
+         * Can we merge with left hand free chunk?
+         */
+        if ((char *)pLeft + pLeft->core.offNext == (char *)pFree)
+        {
+            if (pFree->core.offNext)
+            {
+                pLeft->core.offNext = pLeft->core.offNext + pFree->core.offNext;
+                MMHYPERCHUNK_SET_OFFPREV(((PMMHYPERCHUNK)((char *)pLeft + pLeft->core.offNext)), -(int32_t)pLeft->core.offNext);
+            }
+            else
+                pLeft->core.offNext = 0;
+            pFree = pLeft;
+            Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
+            pHeap->cbFree -= pLeft->cb;
+            Log3(("mmHyperFree: Merging %p into %p (cb=%d).\n", pFree, pLeft, pLeft->cb));
+        }
+        /*
+         * No, just link it into the free list then.
+         */
+        else
+        {
+            MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
+            pFree->offPrev = (uintptr_t)pLeft - (uintptr_t)pFree;
+            pLeft->offNext = -pFree->offPrev;
+            if (pRight)
+            {
+                pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
+                pRight->offPrev = -(int32_t)pFree->offNext;
+            }
+            else
+            {
+                pFree->offNext = 0;
+                pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
+            }
+            Log3(("mmHyperFree: Inserted %p after %p in free list.\n", pFree, pLeft));
+        }
+    }
+
+    /*
+     * Can we merge with right hand free chunk?
+     */
+    if (pRight && (char *)pRight == (char *)pFree + pFree->core.offNext)
+    {
+        /* core */
+        if (pRight->core.offNext)
+        {
+            pFree->core.offNext += pRight->core.offNext;
+            PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
+            MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
+            ASSERT_CHUNK(pHeap, pNext);
+        }
+        else
+            pFree->core.offNext = 0;
+
+        /* free */
+        if (pRight->offNext)
+        {
+            pFree->offNext += pRight->offNext;
+            ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext))->offPrev = -(int32_t)pFree->offNext;
+        }
+        else
+        {
+            pFree->offNext = 0;
+            pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
+        }
+        Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
+        pHeap->cbFree -= pRight->cb;
+        Log3(("mmHyperFree: Merged %p (cb=%d) into %p.\n", pRight, pRight->cb, pFree));
+    }
+
+    /* calculate the size. */
+    if (pFree->core.offNext)
+        pFree->cb = pFree->core.offNext - sizeof(MMHYPERCHUNK);
+    else
+        pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap)) - sizeof(MMHYPERCHUNK);
+    Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
+    pHeap->cbFree += pFree->cb;
+    ASSERT_CHUNK_FREE(pHeap, pFree);
+
+#ifdef MMHYPER_HEAP_STRICT
+    mmHyperHeapCheck(pHeap);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+#if defined(DEBUG) || defined(MMHYPER_HEAP_STRICT_FENCE)
+/**
+ * Dumps a heap chunk to the log.
+ *
+ * @param   pHeap       Pointer to the heap.
+ * @param   pCur        Pointer to the chunk.
+ */
+static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
+{
+    if (MMHYPERCHUNK_ISUSED(&pCur->core))
+    {
+        if (pCur->core.offStat)
+        {
+            PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
+            const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
+#ifdef IN_RING3
+            Log(("%p  %06x USED offNext=%06x offPrev=-%06x %s%s\n",
+                 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
+                 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
+                 mmGetTagName((MMTAG)pStat->Core.Key), pszSelf));
+#else
+            Log(("%p  %06x USED offNext=%06x offPrev=-%06x %d%s\n",
+                 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
+                 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
+                 (MMTAG)pStat->Core.Key, pszSelf));
+#endif
+            NOREF(pStat); NOREF(pszSelf);
+        }
+        else
+            Log(("%p  %06x USED offNext=%06x offPrev=-%06x\n",
+                 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
+                 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
+    }
+    else
+        Log(("%p  %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
+             pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
+             pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
+}
+#endif /* DEBUG || MMHYPER_HEAP_STRICT */
+
+
+#ifdef MMHYPER_HEAP_STRICT
+/**
+ * Internal consistency check.
+ */
+static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
+{
+    PMMHYPERCHUNKFREE pPrev = NULL;
+    PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
+    for (;;)
+    {
+        if (MMHYPERCHUNK_ISUSED(&pCur->core))
+            ASSERT_CHUNK_USED(pHeap, &pCur->core);
+        else
+            ASSERT_CHUNK_FREE(pHeap, pCur);
+        if (pPrev)
+            AssertMsg((int32_t)pPrev->core.offNext == -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
+                      ("pPrev->core.offNext=%d offPrev=%d\n", pPrev->core.offNext, MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
+
+# ifdef MMHYPER_HEAP_STRICT_FENCE
+        uint32_t off = (uint8_t *)pCur - pHeap->CTX_SUFF(pbHeap);
+        if (    MMHYPERCHUNK_ISUSED(&pCur->core)
+            &&  off < pHeap->offPageAligned)
+        {
+            uint32_t cbCur = pCur->core.offNext
+                           ? pCur->core.offNext
+                           : pHeap->cbHeap - off;
+            uint32_t *pu32End = ((uint32_t *)((uint8_t *)pCur + cbCur));
+            uint32_t cbFence = pu32End[-1];
+            if (RT_UNLIKELY(    cbFence >= cbCur - sizeof(*pCur)
+                            ||  cbFence < MMHYPER_HEAP_STRICT_FENCE_SIZE))
+            {
+                mmHyperHeapDumpOne(pHeap, pCur);
+                Assert(cbFence < cbCur - sizeof(*pCur));
+                Assert(cbFence >= MMHYPER_HEAP_STRICT_FENCE_SIZE);
+            }
+
+            uint32_t *pu32Bad = ASMMemFirstMismatchingU32((uint8_t *)pu32End - cbFence, cbFence - sizeof(uint32_t), MMHYPER_HEAP_STRICT_FENCE_U32);
+            if (RT_UNLIKELY(pu32Bad))
+            {
+                mmHyperHeapDumpOne(pHeap, pCur);
+                Assert(!pu32Bad);
+            }
+        }
+# endif
+
+        /* next */
+        if (!pCur->core.offNext)
+            break;
+        pPrev = pCur;
+        pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
+    }
+}
+#endif
+
+
+/**
+ * Performs consistency checks on the heap if MMHYPER_HEAP_STRICT was
+ * defined at build time.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+VMMDECL(void) MMHyperHeapCheck(PVMCC pVM)
+{
+#ifdef MMHYPER_HEAP_STRICT
+    int rc;
+
+    rc = mmHyperLock(pVM);
+    AssertRC(rc);
+    mmHyperHeapCheck(pVM->mm.s.CTX_SUFF(pHyperHeap));
+    mmHyperUnlock(pVM);
+#else
+    NOREF(pVM);
+#endif
+}
+
+
+#ifdef DEBUG
+/**
+ * Dumps the hypervisor heap to Log.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(void) MMHyperHeapDump(PVM pVM)
+{
+    Log(("MMHyperHeapDump: *** heap dump - start ***\n"));
+    PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
+    PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
+    for (;;)
+    {
+        mmHyperHeapDumpOne(pHeap, pCur);
+
+        /* next */
+        if (!pCur->core.offNext)
+            break;
+        pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
+    }
+    Log(("MMHyperHeapDump: *** heap dump - end ***\n"));
+}
+#endif
+
+
+/**
+ * Query the amount of free memory in the hypervisor heap.
+ *
+ * @returns Number of free bytes in the hypervisor heap.
+ */
+VMMDECL(size_t) MMHyperHeapGetFreeSize(PVM pVM)
+{
+    return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbFree;
+}
+
+
+/**
+ * Query the size the hypervisor heap.
+ *
+ * @returns The size of the hypervisor heap in bytes.
+ */
+VMMDECL(size_t) MMHyperHeapGetSize(PVM pVM)
+{
+    return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbHeap;
+}
+
+
+/**
+ * Converts a context neutral heap offset into a pointer.
+ *
+ * @returns Pointer to hyper heap data.
+ * @param   pVM         The cross context VM structure.
+ * @param   offHeap     The hyper heap offset.
+ */
+VMMDECL(void *) MMHyperHeapOffsetToPtr(PVM pVM, uint32_t offHeap)
+{
+    Assert(offHeap - MMYPERHEAP_HDR_SIZE <= pVM->mm.s.CTX_SUFF(pHyperHeap)->cbHeap);
+    return (uint8_t *)pVM->mm.s.CTX_SUFF(pHyperHeap) + offHeap;
+}
+
+
+/**
+ * Converts a context specific heap pointer into a neutral heap offset.
+ *
+ * @returns Heap offset.
+ * @param   pVM         The cross context VM structure.
+ * @param   pv          Pointer to the heap data.
+ */
+VMMDECL(uint32_t) MMHyperHeapPtrToOffset(PVM pVM, void *pv)
+{
+    size_t offHeap = (uint8_t *)pv - (uint8_t *)pVM->mm.s.CTX_SUFF(pHyperHeap);
+    Assert(offHeap - MMYPERHEAP_HDR_SIZE <= pVM->mm.s.CTX_SUFF(pHyperHeap)->cbHeap);
+    return (uint32_t)offHeap;
+}
+
+
+/**
+ * Query the address and size the hypervisor memory area.
+ *
+ * @returns Base address of the hypervisor area.
+ * @param   pVM         The cross context VM structure.
+ * @param   pcb         Where to store the size of the hypervisor area. (out)
+ */
+VMMDECL(RTGCPTR) MMHyperGetArea(PVM pVM, size_t *pcb)
+{
+    if (pcb)
+        *pcb = pVM->mm.s.cbHyperArea;
+    return pVM->mm.s.pvHyperAreaGC;
+}
+
+
+/**
+ * Checks if an address is within the hypervisor memory area.
+ *
+ * @returns true if inside.
+ * @returns false if outside.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPtr       The pointer to check.
+ *
+ * @note    Caller must check that we're in raw-mode before calling!
+ */
+VMMDECL(bool) MMHyperIsInsideArea(PVM pVM, RTGCPTR GCPtr)
+{
+    Assert(VM_IS_RAW_MODE_ENABLED(pVM));
+    return (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC < pVM->mm.s.cbHyperArea;
+}
+
diff --git a/src/VBox/VMM/VMMAll/Makefile.kup b/src/VBox/VMM/VMMAll/Makefile.kup
new file mode 100644
index 00000000..e69de29b
diff --git a/src/VBox/VMM/VMMAll/NEMAll.cpp b/src/VBox/VMM/VMMAll/NEMAll.cpp
new file mode 100644
index 00000000..62e27085
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/NEMAll.cpp
@@ -0,0 +1,153 @@
+/* $Id: NEMAll.cpp $ */
+/** @file
+ * NEM - Native execution manager, R0 and R3 context code.
+ */
+
+/*
+ * Copyright (C) 2018-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_NEM
+#include <VBox/vmm/nem.h>
+#include "NEMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+
+
+/**
+ * Checks if this VM is in NEM mode and is long-mode capable.
+ *
+ * Use VMR3IsLongModeAllowed() instead of this, when possible.
+ *
+ * @returns true if long mode is allowed, false otherwise.
+ * @param   pVM         The cross context VM structure.
+ * @sa      VMR3IsLongModeAllowed, HMIsLongModeAllowed
+ */
+VMM_INT_DECL(bool) NEMHCIsLongModeAllowed(PVMCC pVM)
+{
+    return pVM->nem.s.fAllow64BitGuests && VM_IS_NEM_ENABLED(pVM);
+}
+
+
+/**
+ * Physical access handler registration notification.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   enmKind     The kind of access handler.
+ * @param   GCPhys      Start of the access handling range.
+ * @param   cb          Length of the access handling range.
+ *
+ * @note    Called while holding down the PGM lock.
+ */
+VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
+{
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (VM_IS_NEM_ENABLED(pVM))
+        nemHCNativeNotifyHandlerPhysicalRegister(pVM, enmKind, GCPhys, cb);
+#else
+    RT_NOREF(pVM, enmKind, GCPhys, cb);
+#endif
+}
+
+
+VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
+                                                        int fRestoreAsRAM, bool fRestoreAsRAM2)
+{
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (VM_IS_NEM_ENABLED(pVM))
+        nemHCNativeNotifyHandlerPhysicalDeregister(pVM, enmKind, GCPhys, cb, fRestoreAsRAM, fRestoreAsRAM2);
+#else
+    RT_NOREF(pVM, enmKind, GCPhys, cb, fRestoreAsRAM, fRestoreAsRAM2);
+#endif
+}
+
+
+VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
+                                                    RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
+{
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (VM_IS_NEM_ENABLED(pVM))
+        nemHCNativeNotifyHandlerPhysicalModify(pVM, enmKind, GCPhysOld, GCPhysNew, cb, fRestoreAsRAM);
+#else
+    RT_NOREF(pVM, enmKind, GCPhysOld, GCPhysNew, cb, fRestoreAsRAM);
+#endif
+}
+
+
+VMM_INT_DECL(int)  NEMHCNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
+                                                PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    Assert(VM_IS_NEM_ENABLED(pVM));
+#ifdef VBOX_WITH_NATIVE_NEM
+    return nemHCNativeNotifyPhysPageAllocated(pVM, GCPhys, HCPhys, fPageProt, enmType, pu2State);
+#else
+    RT_NOREF(pVM, GCPhys, HCPhys, fPageProt, enmType, pu2State);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
+                                                  PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    Assert(VM_IS_NEM_ENABLED(pVM));
+#ifdef VBOX_WITH_NATIVE_NEM
+    nemHCNativeNotifyPhysPageProtChanged(pVM, GCPhys, HCPhys, fPageProt, enmType, pu2State);
+#else
+    RT_NOREF(pVM, GCPhys, HCPhys, fPageProt, enmType, pu2State);
+#endif
+}
+
+
+VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
+                                              uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    Assert(VM_IS_NEM_ENABLED(pVM));
+#ifdef VBOX_WITH_NATIVE_NEM
+    nemHCNativeNotifyPhysPageChanged(pVM, GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, pu2State);
+#else
+    RT_NOREF(pVM, GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, pu2State);
+#endif
+}
+
+
+#ifndef VBOX_WITH_NATIVE_NEM
+VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
+{
+    RT_NOREF(pVCpu, fWhat);
+    return VERR_NEM_IPE_9;
+}
+#endif
+
+
+#ifndef VBOX_WITH_NATIVE_NEM
+VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t *pcTicks, uint32_t *puAux)
+{
+    RT_NOREF(pVCpu, pcTicks, puAux);
+    AssertFailed();
+    return VERR_NEM_IPE_9;
+}
+#endif
+
+
+#ifndef VBOX_WITH_NATIVE_NEM
+VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
+{
+    RT_NOREF(pVM, pVCpu, uPausedTscValue);
+    AssertFailed();
+    return VERR_NEM_IPE_9;
+}
+#endif
+
diff --git a/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h b/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h
new file mode 100644
index 00000000..47331a85
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h
@@ -0,0 +1,4961 @@
+/* $Id: NEMAllNativeTemplate-win.cpp.h $ */
+/** @file
+ * NEM - Native execution manager, Windows code template ring-0/3.
+ */
+
+/*
+ * Copyright (C) 2018-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** Copy back a segment from hyper-V. */
+#define NEM_WIN_COPY_BACK_SEG(a_Dst, a_Src) \
+            do { \
+                (a_Dst).u64Base  = (a_Src).Base; \
+                (a_Dst).u32Limit = (a_Src).Limit; \
+                (a_Dst).ValidSel = (a_Dst).Sel = (a_Src).Selector; \
+                (a_Dst).Attr.u   = (a_Src).Attributes; \
+                (a_Dst).fFlags   = CPUMSELREG_FLAGS_VALID; \
+            } while (0)
+
+/** @def NEMWIN_ASSERT_MSG_REG_VAL
+ * Asserts the correctness of a register value in a message/context.
+ */
+#if 0
+# define NEMWIN_NEED_GET_REGISTER
+# if defined(IN_RING0) || defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
+#  define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) \
+    do { \
+        HV_REGISTER_VALUE TmpVal; \
+        nemHCWinGetRegister(a_pVCpu, a_enmReg, &TmpVal); \
+        AssertMsg(a_Expr, a_Msg); \
+    } while (0)
+# else
+#  define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) \
+    do { \
+        WHV_REGISTER_VALUE TmpVal; \
+        nemR3WinGetRegister(a_pVCpu, a_enmReg, &TmpVal); \
+        AssertMsg(a_Expr, a_Msg); \
+    } while (0)
+# endif
+#else
+# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) do { } while (0)
+#endif
+
+/** @def NEMWIN_ASSERT_MSG_REG_VAL
+ * Asserts the correctness of a 64-bit register value in a message/context.
+ */
+#define NEMWIN_ASSERT_MSG_REG_VAL64(a_pVCpu, a_enmReg, a_u64Val) \
+    NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, (a_u64Val) == TmpVal.Reg64, \
+                              (#a_u64Val "=%#RX64, expected %#RX64\n", (a_u64Val), TmpVal.Reg64))
+/** @def NEMWIN_ASSERT_MSG_REG_VAL
+ * Asserts the correctness of a segment register value in a message/context.
+ */
+#define NEMWIN_ASSERT_MSG_REG_SEG(a_pVCpu, a_enmReg, a_SReg) \
+    NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, \
+                                 (a_SReg).Base       == TmpVal.Segment.Base \
+                              && (a_SReg).Limit      == TmpVal.Segment.Limit \
+                              && (a_SReg).Selector   == TmpVal.Segment.Selector \
+                              && (a_SReg).Attributes == TmpVal.Segment.Attributes, \
+                              ( #a_SReg "=%#RX16 {%#RX64 LB %#RX32,%#RX16} expected %#RX16 {%#RX64 LB %#RX32,%#RX16}\n", \
+                               (a_SReg).Selector, (a_SReg).Base, (a_SReg).Limit, (a_SReg).Attributes, \
+                               TmpVal.Segment.Selector, TmpVal.Segment.Base, TmpVal.Segment.Limit, TmpVal.Segment.Attributes))
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** NEM_WIN_PAGE_STATE_XXX names. */
+NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
+
+/** HV_INTERCEPT_ACCESS_TYPE names. */
+static const char * const g_apszHvInterceptAccessTypes[4] = { "read", "write", "exec", "!undefined!" };
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
+                                           uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged);
+
+
+
+#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+
+/**
+ * Wrapper around VMMR0_DO_NEM_MAP_PAGES for a single page.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the caller.
+ * @param   GCPhysSrc   The source page.  Does not need to be page aligned.
+ * @param   GCPhysDst   The destination page.  Same as @a GCPhysSrc except for
+ *                      when A20 is disabled.
+ * @param   fFlags      HV_MAP_GPA_XXX.
+ */
+DECLINLINE(int) nemHCWinHypercallMapPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst, uint32_t fFlags)
+{
+#ifdef IN_RING0
+    /** @todo optimize further, caller generally has the physical address. */
+    return nemR0WinMapPages(pVM, pVCpu,
+                            GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
+                            GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
+                            1, fFlags);
+#else
+    pVCpu->nem.s.Hypercall.MapPages.GCPhysSrc   = GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
+    pVCpu->nem.s.Hypercall.MapPages.GCPhysDst   = GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
+    pVCpu->nem.s.Hypercall.MapPages.cPages      = 1;
+    pVCpu->nem.s.Hypercall.MapPages.fFlags      = fFlags;
+    return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_MAP_PAGES, 0, NULL);
+#endif
+}
+
+
+/**
+ * Wrapper around VMMR0_DO_NEM_UNMAP_PAGES for a single page.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the caller.
+ * @param   GCPhys      The page to unmap.  Does not need to be page aligned.
+ */
+DECLINLINE(int) nemHCWinHypercallUnmapPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys)
+{
+# ifdef IN_RING0
+    return nemR0WinUnmapPages(pVM, pVCpu, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, 1);
+# else
+    pVCpu->nem.s.Hypercall.UnmapPages.GCPhys    = GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
+    pVCpu->nem.s.Hypercall.UnmapPages.cPages    = 1;
+    return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_UNMAP_PAGES, 0, NULL);
+# endif
+}
+
+#endif /* NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
+#ifndef IN_RING0
+
+NEM_TMPL_STATIC int nemHCWinCopyStateToHyperV(PVMCC pVM, PVMCPUCC pVCpu)
+{
+# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) || defined(NEM_WIN_WITH_RING0_RUNLOOP)
+#  if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
+    if (pVM->nem.s.fUseRing0Runloop)
+#  endif
+    {
+        int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_EXPORT_STATE, 0, NULL);
+        AssertLogRelRCReturn(rc, rc);
+        return rc;
+    }
+# endif
+# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
+
+    /*
+     * The following is very similar to what nemR0WinExportState() does.
+     */
+    WHV_REGISTER_NAME  aenmNames[128];
+    WHV_REGISTER_VALUE aValues[128];
+
+    uint64_t const fWhat = ~pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK);
+    if (   !fWhat
+        && pVCpu->nem.s.fCurrentInterruptWindows == pVCpu->nem.s.fDesiredInterruptWindows)
+        return VINF_SUCCESS;
+    uintptr_t iReg = 0;
+
+#  define ADD_REG64(a_enmName, a_uValue) do { \
+            aenmNames[iReg]      = (a_enmName); \
+            aValues[iReg].Reg128.High64 = 0; \
+            aValues[iReg].Reg64  = (a_uValue); \
+            iReg++; \
+        } while (0)
+#  define ADD_REG128(a_enmName, a_uValueLo, a_uValueHi) do { \
+            aenmNames[iReg] = (a_enmName); \
+            aValues[iReg].Reg128.Low64  = (a_uValueLo); \
+            aValues[iReg].Reg128.High64 = (a_uValueHi); \
+            iReg++; \
+        } while (0)
+
+    /* GPRs */
+    if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_RAX)
+            ADD_REG64(WHvX64RegisterRax, pVCpu->cpum.GstCtx.rax);
+        if (fWhat & CPUMCTX_EXTRN_RCX)
+            ADD_REG64(WHvX64RegisterRcx, pVCpu->cpum.GstCtx.rcx);
+        if (fWhat & CPUMCTX_EXTRN_RDX)
+            ADD_REG64(WHvX64RegisterRdx, pVCpu->cpum.GstCtx.rdx);
+        if (fWhat & CPUMCTX_EXTRN_RBX)
+            ADD_REG64(WHvX64RegisterRbx, pVCpu->cpum.GstCtx.rbx);
+        if (fWhat & CPUMCTX_EXTRN_RSP)
+            ADD_REG64(WHvX64RegisterRsp, pVCpu->cpum.GstCtx.rsp);
+        if (fWhat & CPUMCTX_EXTRN_RBP)
+            ADD_REG64(WHvX64RegisterRbp, pVCpu->cpum.GstCtx.rbp);
+        if (fWhat & CPUMCTX_EXTRN_RSI)
+            ADD_REG64(WHvX64RegisterRsi, pVCpu->cpum.GstCtx.rsi);
+        if (fWhat & CPUMCTX_EXTRN_RDI)
+            ADD_REG64(WHvX64RegisterRdi, pVCpu->cpum.GstCtx.rdi);
+        if (fWhat & CPUMCTX_EXTRN_R8_R15)
+        {
+            ADD_REG64(WHvX64RegisterR8, pVCpu->cpum.GstCtx.r8);
+            ADD_REG64(WHvX64RegisterR9, pVCpu->cpum.GstCtx.r9);
+            ADD_REG64(WHvX64RegisterR10, pVCpu->cpum.GstCtx.r10);
+            ADD_REG64(WHvX64RegisterR11, pVCpu->cpum.GstCtx.r11);
+            ADD_REG64(WHvX64RegisterR12, pVCpu->cpum.GstCtx.r12);
+            ADD_REG64(WHvX64RegisterR13, pVCpu->cpum.GstCtx.r13);
+            ADD_REG64(WHvX64RegisterR14, pVCpu->cpum.GstCtx.r14);
+            ADD_REG64(WHvX64RegisterR15, pVCpu->cpum.GstCtx.r15);
+        }
+    }
+
+    /* RIP & Flags */
+    if (fWhat & CPUMCTX_EXTRN_RIP)
+        ADD_REG64(WHvX64RegisterRip, pVCpu->cpum.GstCtx.rip);
+    if (fWhat & CPUMCTX_EXTRN_RFLAGS)
+        ADD_REG64(WHvX64RegisterRflags, pVCpu->cpum.GstCtx.rflags.u);
+
+    /* Segments */
+#  define ADD_SEG(a_enmName, a_SReg) \
+        do { \
+            aenmNames[iReg]                  = a_enmName; \
+            aValues[iReg].Segment.Base       = (a_SReg).u64Base; \
+            aValues[iReg].Segment.Limit      = (a_SReg).u32Limit; \
+            aValues[iReg].Segment.Selector   = (a_SReg).Sel; \
+            aValues[iReg].Segment.Attributes = (a_SReg).Attr.u; \
+            iReg++; \
+        } while (0)
+    if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_ES)
+            ADD_SEG(WHvX64RegisterEs,   pVCpu->cpum.GstCtx.es);
+        if (fWhat & CPUMCTX_EXTRN_CS)
+            ADD_SEG(WHvX64RegisterCs,   pVCpu->cpum.GstCtx.cs);
+        if (fWhat & CPUMCTX_EXTRN_SS)
+            ADD_SEG(WHvX64RegisterSs,   pVCpu->cpum.GstCtx.ss);
+        if (fWhat & CPUMCTX_EXTRN_DS)
+            ADD_SEG(WHvX64RegisterDs,   pVCpu->cpum.GstCtx.ds);
+        if (fWhat & CPUMCTX_EXTRN_FS)
+            ADD_SEG(WHvX64RegisterFs,   pVCpu->cpum.GstCtx.fs);
+        if (fWhat & CPUMCTX_EXTRN_GS)
+            ADD_SEG(WHvX64RegisterGs,   pVCpu->cpum.GstCtx.gs);
+    }
+
+    /* Descriptor tables & task segment. */
+    if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_LDTR)
+            ADD_SEG(WHvX64RegisterLdtr, pVCpu->cpum.GstCtx.ldtr);
+        if (fWhat & CPUMCTX_EXTRN_TR)
+            ADD_SEG(WHvX64RegisterTr,   pVCpu->cpum.GstCtx.tr);
+        if (fWhat & CPUMCTX_EXTRN_IDTR)
+        {
+            aenmNames[iReg] = WHvX64RegisterIdtr;
+            aValues[iReg].Table.Limit = pVCpu->cpum.GstCtx.idtr.cbIdt;
+            aValues[iReg].Table.Base  = pVCpu->cpum.GstCtx.idtr.pIdt;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_GDTR)
+        {
+            aenmNames[iReg] = WHvX64RegisterGdtr;
+            aValues[iReg].Table.Limit = pVCpu->cpum.GstCtx.gdtr.cbGdt;
+            aValues[iReg].Table.Base  = pVCpu->cpum.GstCtx.gdtr.pGdt;
+            iReg++;
+        }
+    }
+
+    /* Control registers. */
+    if (fWhat & CPUMCTX_EXTRN_CR_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_CR0)
+            ADD_REG64(WHvX64RegisterCr0, pVCpu->cpum.GstCtx.cr0);
+        if (fWhat & CPUMCTX_EXTRN_CR2)
+            ADD_REG64(WHvX64RegisterCr2, pVCpu->cpum.GstCtx.cr2);
+        if (fWhat & CPUMCTX_EXTRN_CR3)
+            ADD_REG64(WHvX64RegisterCr3, pVCpu->cpum.GstCtx.cr3);
+        if (fWhat & CPUMCTX_EXTRN_CR4)
+            ADD_REG64(WHvX64RegisterCr4, pVCpu->cpum.GstCtx.cr4);
+    }
+    if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
+        ADD_REG64(WHvX64RegisterCr8, CPUMGetGuestCR8(pVCpu));
+
+    /* Debug registers. */
+/** @todo fixme. Figure out what the hyper-v version of KVM_SET_GUEST_DEBUG would be. */
+    if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
+    {
+        ADD_REG64(WHvX64RegisterDr0, pVCpu->cpum.GstCtx.dr[0]); // CPUMGetHyperDR0(pVCpu));
+        ADD_REG64(WHvX64RegisterDr1, pVCpu->cpum.GstCtx.dr[1]); // CPUMGetHyperDR1(pVCpu));
+        ADD_REG64(WHvX64RegisterDr2, pVCpu->cpum.GstCtx.dr[2]); // CPUMGetHyperDR2(pVCpu));
+        ADD_REG64(WHvX64RegisterDr3, pVCpu->cpum.GstCtx.dr[3]); // CPUMGetHyperDR3(pVCpu));
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR6)
+        ADD_REG64(WHvX64RegisterDr6, pVCpu->cpum.GstCtx.dr[6]); // CPUMGetHyperDR6(pVCpu));
+    if (fWhat & CPUMCTX_EXTRN_DR7)
+        ADD_REG64(WHvX64RegisterDr7, pVCpu->cpum.GstCtx.dr[7]); // CPUMGetHyperDR7(pVCpu));
+
+    /* Floating point state. */
+    if (fWhat & CPUMCTX_EXTRN_X87)
+    {
+        ADD_REG128(WHvX64RegisterFpMmx0, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[0].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[0].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx1, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[1].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[1].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx2, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[2].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[2].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx3, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[3].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[3].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx4, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[4].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[4].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx5, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[5].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[5].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx6, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[6].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[6].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx7, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[7].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[7].au64[1]);
+
+        aenmNames[iReg] = WHvX64RegisterFpControlStatus;
+        aValues[iReg].FpControlStatus.FpControl = pVCpu->cpum.GstCtx.pXStateR3->x87.FCW;
+        aValues[iReg].FpControlStatus.FpStatus  = pVCpu->cpum.GstCtx.pXStateR3->x87.FSW;
+        aValues[iReg].FpControlStatus.FpTag     = pVCpu->cpum.GstCtx.pXStateR3->x87.FTW;
+        aValues[iReg].FpControlStatus.Reserved  = pVCpu->cpum.GstCtx.pXStateR3->x87.FTW >> 8;
+        aValues[iReg].FpControlStatus.LastFpOp  = pVCpu->cpum.GstCtx.pXStateR3->x87.FOP;
+        aValues[iReg].FpControlStatus.LastFpRip = (pVCpu->cpum.GstCtx.pXStateR3->x87.FPUIP)
+                                                | ((uint64_t)pVCpu->cpum.GstCtx.pXStateR3->x87.CS << 32)
+                                                | ((uint64_t)pVCpu->cpum.GstCtx.pXStateR3->x87.Rsrvd1 << 48);
+        iReg++;
+
+        aenmNames[iReg] = WHvX64RegisterXmmControlStatus;
+        aValues[iReg].XmmControlStatus.LastFpRdp            = (pVCpu->cpum.GstCtx.pXStateR3->x87.FPUDP)
+                                                            | ((uint64_t)pVCpu->cpum.GstCtx.pXStateR3->x87.DS << 32)
+                                                            | ((uint64_t)pVCpu->cpum.GstCtx.pXStateR3->x87.Rsrvd2 << 48);
+        aValues[iReg].XmmControlStatus.XmmStatusControl     = pVCpu->cpum.GstCtx.pXStateR3->x87.MXCSR;
+        aValues[iReg].XmmControlStatus.XmmStatusControlMask = pVCpu->cpum.GstCtx.pXStateR3->x87.MXCSR_MASK; /** @todo ??? (Isn't this an output field?) */
+        iReg++;
+    }
+
+    /* Vector state. */
+    if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
+    {
+        ADD_REG128(WHvX64RegisterXmm0,  pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 0].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 0].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm1,  pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 1].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 1].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm2,  pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 2].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 2].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm3,  pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 3].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 3].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm4,  pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 4].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 4].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm5,  pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 5].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 5].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm6,  pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 6].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 6].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm7,  pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 7].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 7].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm8,  pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 8].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 8].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm9,  pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 9].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 9].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[10].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[10].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm11, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[11].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[11].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm12, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[12].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[12].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm13, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[13].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[13].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm14, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[14].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[14].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm15, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[15].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[15].uXmm.s.Hi);
+    }
+
+    /* MSRs */
+    // WHvX64RegisterTsc - don't touch
+    if (fWhat & CPUMCTX_EXTRN_EFER)
+        ADD_REG64(WHvX64RegisterEfer, pVCpu->cpum.GstCtx.msrEFER);
+    if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+        ADD_REG64(WHvX64RegisterKernelGsBase, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
+    if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+    {
+        ADD_REG64(WHvX64RegisterSysenterCs, pVCpu->cpum.GstCtx.SysEnter.cs);
+        ADD_REG64(WHvX64RegisterSysenterEip, pVCpu->cpum.GstCtx.SysEnter.eip);
+        ADD_REG64(WHvX64RegisterSysenterEsp, pVCpu->cpum.GstCtx.SysEnter.esp);
+    }
+    if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+    {
+        ADD_REG64(WHvX64RegisterStar, pVCpu->cpum.GstCtx.msrSTAR);
+        ADD_REG64(WHvX64RegisterLstar, pVCpu->cpum.GstCtx.msrLSTAR);
+        ADD_REG64(WHvX64RegisterCstar, pVCpu->cpum.GstCtx.msrCSTAR);
+        ADD_REG64(WHvX64RegisterSfmask, pVCpu->cpum.GstCtx.msrSFMASK);
+    }
+    if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
+    {
+        ADD_REG64(WHvX64RegisterApicBase, APICGetBaseMsrNoCheck(pVCpu));
+        ADD_REG64(WHvX64RegisterPat, pVCpu->cpum.GstCtx.msrPAT);
+#if 0 /** @todo check if WHvX64RegisterMsrMtrrCap works here... */
+        ADD_REG64(WHvX64RegisterMsrMtrrCap, CPUMGetGuestIa32MtrrCap(pVCpu));
+#endif
+        PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
+        ADD_REG64(WHvX64RegisterMsrMtrrDefType, pCtxMsrs->msr.MtrrDefType);
+        ADD_REG64(WHvX64RegisterMsrMtrrFix64k00000, pCtxMsrs->msr.MtrrFix64K_00000);
+        ADD_REG64(WHvX64RegisterMsrMtrrFix16k80000, pCtxMsrs->msr.MtrrFix16K_80000);
+        ADD_REG64(WHvX64RegisterMsrMtrrFix16kA0000, pCtxMsrs->msr.MtrrFix16K_A0000);
+        ADD_REG64(WHvX64RegisterMsrMtrrFix4kC0000,  pCtxMsrs->msr.MtrrFix4K_C0000);
+        ADD_REG64(WHvX64RegisterMsrMtrrFix4kC8000,  pCtxMsrs->msr.MtrrFix4K_C8000);
+        ADD_REG64(WHvX64RegisterMsrMtrrFix4kD0000,  pCtxMsrs->msr.MtrrFix4K_D0000);
+        ADD_REG64(WHvX64RegisterMsrMtrrFix4kD8000,  pCtxMsrs->msr.MtrrFix4K_D8000);
+        ADD_REG64(WHvX64RegisterMsrMtrrFix4kE0000,  pCtxMsrs->msr.MtrrFix4K_E0000);
+        ADD_REG64(WHvX64RegisterMsrMtrrFix4kE8000,  pCtxMsrs->msr.MtrrFix4K_E8000);
+        ADD_REG64(WHvX64RegisterMsrMtrrFix4kF0000,  pCtxMsrs->msr.MtrrFix4K_F0000);
+        ADD_REG64(WHvX64RegisterMsrMtrrFix4kF8000,  pCtxMsrs->msr.MtrrFix4K_F8000);
+        ADD_REG64(WHvX64RegisterTscAux, pCtxMsrs->msr.TscAux);
+#if 0 /** @todo these registers aren't available? Might explain something.. .*/
+        const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM);
+        if (enmCpuVendor != CPUMCPUVENDOR_AMD)
+        {
+            ADD_REG64(HvX64RegisterIa32MiscEnable, pCtxMsrs->msr.MiscEnable);
+            ADD_REG64(HvX64RegisterIa32FeatureControl, CPUMGetGuestIa32FeatureControl(pVCpu));
+        }
+#endif
+    }
+
+    /* event injection (clear it). */
+    if (fWhat & CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)
+        ADD_REG64(WHvRegisterPendingInterruption, 0);
+
+    /* Interruptibility state.  This can get a little complicated since we get
+       half of the state via HV_X64_VP_EXECUTION_STATE. */
+    if (   (fWhat & (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
+        ==          (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI) )
+    {
+        ADD_REG64(WHvRegisterInterruptState, 0);
+        if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+            && EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip)
+            aValues[iReg - 1].InterruptState.InterruptShadow = 1;
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+            aValues[iReg - 1].InterruptState.NmiMasked = 1;
+    }
+    else if (fWhat & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT)
+    {
+        if (   pVCpu->nem.s.fLastInterruptShadow
+            || (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+                && EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip))
+        {
+            ADD_REG64(WHvRegisterInterruptState, 0);
+            if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+                && EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip)
+                aValues[iReg - 1].InterruptState.InterruptShadow = 1;
+            /** @todo Retrieve NMI state, currently assuming it's zero. (yes this may happen on I/O) */
+            //if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+            //    aValues[iReg - 1].InterruptState.NmiMasked = 1;
+        }
+    }
+    else
+        Assert(!(fWhat & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI));
+
+    /* Interrupt windows. Always set if active as Hyper-V seems to be forgetful. */
+    uint8_t const fDesiredIntWin = pVCpu->nem.s.fDesiredInterruptWindows;
+    if (   fDesiredIntWin
+        || pVCpu->nem.s.fCurrentInterruptWindows != fDesiredIntWin)
+    {
+        pVCpu->nem.s.fCurrentInterruptWindows = pVCpu->nem.s.fDesiredInterruptWindows;
+        Log8(("Setting WHvX64RegisterDeliverabilityNotifications, fDesiredIntWin=%X\n", fDesiredIntWin));
+        ADD_REG64(WHvX64RegisterDeliverabilityNotifications, fDesiredIntWin);
+        Assert(aValues[iReg - 1].DeliverabilityNotifications.NmiNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_NMI));
+        Assert(aValues[iReg - 1].DeliverabilityNotifications.InterruptNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_REGULAR));
+        Assert(aValues[iReg - 1].DeliverabilityNotifications.InterruptPriority == (fDesiredIntWin & NEM_WIN_INTW_F_PRIO_MASK) >> NEM_WIN_INTW_F_PRIO_SHIFT);
+    }
+
+    /// @todo WHvRegisterPendingEvent
+
+    /*
+     * Set the registers.
+     */
+    Assert(iReg < RT_ELEMENTS(aValues));
+    Assert(iReg < RT_ELEMENTS(aenmNames));
+#  ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
+    Log12(("Calling WHvSetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
+           pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues));
+#  endif
+    HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues);
+    if (SUCCEEDED(hrc))
+    {
+        pVCpu->cpum.GstCtx.fExtrn |= CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK | CPUMCTX_EXTRN_KEEPER_NEM;
+        return VINF_SUCCESS;
+    }
+    AssertLogRelMsgFailed(("WHvSetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
+                           pVM->nem.s.hPartition, pVCpu->idCpu, iReg,
+                           hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+    return VERR_INTERNAL_ERROR;
+
+#  undef ADD_REG64
+#  undef ADD_REG128
+#  undef ADD_SEG
+
+# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
+}
+
+
+NEM_TMPL_STATIC int nemHCWinCopyStateFromHyperV(PVMCC pVM, PVMCPUCC pVCpu, uint64_t fWhat)
+{
+# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) || defined(NEM_WIN_WITH_RING0_RUNLOOP)
+#  if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
+    if (pVM->nem.s.fUseRing0Runloop)
+#  endif
+    {
+        /* See NEMR0ImportState */
+        int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_IMPORT_STATE, fWhat, NULL);
+        if (RT_SUCCESS(rc))
+            return rc;
+        if (rc == VERR_NEM_FLUSH_TLB)
+            return PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /*fGlobal*/);
+        AssertLogRelRCReturn(rc, rc);
+        return rc;
+    }
+# endif
+# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
+    WHV_REGISTER_NAME  aenmNames[128];
+
+    fWhat &= pVCpu->cpum.GstCtx.fExtrn;
+    uintptr_t iReg = 0;
+
+    /* GPRs */
+    if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_RAX)
+            aenmNames[iReg++] = WHvX64RegisterRax;
+        if (fWhat & CPUMCTX_EXTRN_RCX)
+            aenmNames[iReg++] = WHvX64RegisterRcx;
+        if (fWhat & CPUMCTX_EXTRN_RDX)
+            aenmNames[iReg++] = WHvX64RegisterRdx;
+        if (fWhat & CPUMCTX_EXTRN_RBX)
+            aenmNames[iReg++] = WHvX64RegisterRbx;
+        if (fWhat & CPUMCTX_EXTRN_RSP)
+            aenmNames[iReg++] = WHvX64RegisterRsp;
+        if (fWhat & CPUMCTX_EXTRN_RBP)
+            aenmNames[iReg++] = WHvX64RegisterRbp;
+        if (fWhat & CPUMCTX_EXTRN_RSI)
+            aenmNames[iReg++] = WHvX64RegisterRsi;
+        if (fWhat & CPUMCTX_EXTRN_RDI)
+            aenmNames[iReg++] = WHvX64RegisterRdi;
+        if (fWhat & CPUMCTX_EXTRN_R8_R15)
+        {
+            aenmNames[iReg++] = WHvX64RegisterR8;
+            aenmNames[iReg++] = WHvX64RegisterR9;
+            aenmNames[iReg++] = WHvX64RegisterR10;
+            aenmNames[iReg++] = WHvX64RegisterR11;
+            aenmNames[iReg++] = WHvX64RegisterR12;
+            aenmNames[iReg++] = WHvX64RegisterR13;
+            aenmNames[iReg++] = WHvX64RegisterR14;
+            aenmNames[iReg++] = WHvX64RegisterR15;
+        }
+    }
+
+    /* RIP & Flags */
+    if (fWhat & CPUMCTX_EXTRN_RIP)
+        aenmNames[iReg++] = WHvX64RegisterRip;
+    if (fWhat & CPUMCTX_EXTRN_RFLAGS)
+        aenmNames[iReg++] = WHvX64RegisterRflags;
+
+    /* Segments */
+    if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_ES)
+            aenmNames[iReg++] = WHvX64RegisterEs;
+        if (fWhat & CPUMCTX_EXTRN_CS)
+            aenmNames[iReg++] = WHvX64RegisterCs;
+        if (fWhat & CPUMCTX_EXTRN_SS)
+            aenmNames[iReg++] = WHvX64RegisterSs;
+        if (fWhat & CPUMCTX_EXTRN_DS)
+            aenmNames[iReg++] = WHvX64RegisterDs;
+        if (fWhat & CPUMCTX_EXTRN_FS)
+            aenmNames[iReg++] = WHvX64RegisterFs;
+        if (fWhat & CPUMCTX_EXTRN_GS)
+            aenmNames[iReg++] = WHvX64RegisterGs;
+    }
+
+    /* Descriptor tables. */
+    if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_LDTR)
+            aenmNames[iReg++] = WHvX64RegisterLdtr;
+        if (fWhat & CPUMCTX_EXTRN_TR)
+            aenmNames[iReg++] = WHvX64RegisterTr;
+        if (fWhat & CPUMCTX_EXTRN_IDTR)
+            aenmNames[iReg++] = WHvX64RegisterIdtr;
+        if (fWhat & CPUMCTX_EXTRN_GDTR)
+            aenmNames[iReg++] = WHvX64RegisterGdtr;
+    }
+
+    /* Control registers. */
+    if (fWhat & CPUMCTX_EXTRN_CR_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_CR0)
+            aenmNames[iReg++] = WHvX64RegisterCr0;
+        if (fWhat & CPUMCTX_EXTRN_CR2)
+            aenmNames[iReg++] = WHvX64RegisterCr2;
+        if (fWhat & CPUMCTX_EXTRN_CR3)
+            aenmNames[iReg++] = WHvX64RegisterCr3;
+        if (fWhat & CPUMCTX_EXTRN_CR4)
+            aenmNames[iReg++] = WHvX64RegisterCr4;
+    }
+    if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
+        aenmNames[iReg++] = WHvX64RegisterCr8;
+
+    /* Debug registers. */
+    if (fWhat & CPUMCTX_EXTRN_DR7)
+        aenmNames[iReg++] = WHvX64RegisterDr7;
+    if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
+    {
+        if (!(fWhat & CPUMCTX_EXTRN_DR7) && (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_DR7))
+        {
+            fWhat |= CPUMCTX_EXTRN_DR7;
+            aenmNames[iReg++] = WHvX64RegisterDr7;
+        }
+        aenmNames[iReg++] = WHvX64RegisterDr0;
+        aenmNames[iReg++] = WHvX64RegisterDr1;
+        aenmNames[iReg++] = WHvX64RegisterDr2;
+        aenmNames[iReg++] = WHvX64RegisterDr3;
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR6)
+        aenmNames[iReg++] = WHvX64RegisterDr6;
+
+    /* Floating point state. */
+    if (fWhat & CPUMCTX_EXTRN_X87)
+    {
+        aenmNames[iReg++] = WHvX64RegisterFpMmx0;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx1;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx2;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx3;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx4;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx5;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx6;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx7;
+        aenmNames[iReg++] = WHvX64RegisterFpControlStatus;
+    }
+    if (fWhat & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX))
+        aenmNames[iReg++] = WHvX64RegisterXmmControlStatus;
+
+    /* Vector state. */
+    if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
+    {
+        aenmNames[iReg++] = WHvX64RegisterXmm0;
+        aenmNames[iReg++] = WHvX64RegisterXmm1;
+        aenmNames[iReg++] = WHvX64RegisterXmm2;
+        aenmNames[iReg++] = WHvX64RegisterXmm3;
+        aenmNames[iReg++] = WHvX64RegisterXmm4;
+        aenmNames[iReg++] = WHvX64RegisterXmm5;
+        aenmNames[iReg++] = WHvX64RegisterXmm6;
+        aenmNames[iReg++] = WHvX64RegisterXmm7;
+        aenmNames[iReg++] = WHvX64RegisterXmm8;
+        aenmNames[iReg++] = WHvX64RegisterXmm9;
+        aenmNames[iReg++] = WHvX64RegisterXmm10;
+        aenmNames[iReg++] = WHvX64RegisterXmm11;
+        aenmNames[iReg++] = WHvX64RegisterXmm12;
+        aenmNames[iReg++] = WHvX64RegisterXmm13;
+        aenmNames[iReg++] = WHvX64RegisterXmm14;
+        aenmNames[iReg++] = WHvX64RegisterXmm15;
+    }
+
+    /* MSRs */
+    // WHvX64RegisterTsc - don't touch
+    if (fWhat & CPUMCTX_EXTRN_EFER)
+        aenmNames[iReg++] = WHvX64RegisterEfer;
+    if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+        aenmNames[iReg++] = WHvX64RegisterKernelGsBase;
+    if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+    {
+        aenmNames[iReg++] = WHvX64RegisterSysenterCs;
+        aenmNames[iReg++] = WHvX64RegisterSysenterEip;
+        aenmNames[iReg++] = WHvX64RegisterSysenterEsp;
+    }
+    if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+    {
+        aenmNames[iReg++] = WHvX64RegisterStar;
+        aenmNames[iReg++] = WHvX64RegisterLstar;
+        aenmNames[iReg++] = WHvX64RegisterCstar;
+        aenmNames[iReg++] = WHvX64RegisterSfmask;
+    }
+
+//#ifdef LOG_ENABLED
+//    const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM);
+//#endif
+    if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
+    {
+        aenmNames[iReg++] = WHvX64RegisterApicBase; /// @todo APIC BASE
+        aenmNames[iReg++] = WHvX64RegisterPat;
+#if 0 /*def LOG_ENABLED*/ /** @todo Check if WHvX64RegisterMsrMtrrCap works... */
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrCap;
+#endif
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrDefType;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix64k00000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix16k80000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix16kA0000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kC0000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kC8000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kD0000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kD8000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kE0000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kE8000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kF0000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kF8000;
+        aenmNames[iReg++] = WHvX64RegisterTscAux;
+        /** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
+//#ifdef LOG_ENABLED
+//        if (enmCpuVendor != CPUMCPUVENDOR_AMD)
+//            aenmNames[iReg++] = HvX64RegisterIa32FeatureControl;
+//#endif
+    }
+
+    /* Interruptibility. */
+    if (fWhat & (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
+    {
+        aenmNames[iReg++] = WHvRegisterInterruptState;
+        aenmNames[iReg++] = WHvX64RegisterRip;
+    }
+
+    /* event injection */
+    aenmNames[iReg++] = WHvRegisterPendingInterruption;
+    aenmNames[iReg++] = WHvRegisterPendingEvent0; /** @todo renamed to WHvRegisterPendingEvent */
+
+    size_t const cRegs = iReg;
+    Assert(cRegs < RT_ELEMENTS(aenmNames));
+
+    /*
+     * Get the registers.
+     */
+    WHV_REGISTER_VALUE aValues[128];
+    RT_ZERO(aValues);
+    Assert(RT_ELEMENTS(aValues) >= cRegs);
+    Assert(RT_ELEMENTS(aenmNames) >= cRegs);
+#  ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
+    Log12(("Calling WHvGetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
+          pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, cRegs, aValues));
+#  endif
+    HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, (uint32_t)cRegs, aValues);
+    AssertLogRelMsgReturn(SUCCEEDED(hrc),
+                          ("WHvGetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
+                           pVM->nem.s.hPartition, pVCpu->idCpu, cRegs, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
+                          , VERR_NEM_GET_REGISTERS_FAILED);
+
+    iReg = 0;
+#  define GET_REG64(a_DstVar, a_enmName) do { \
+            Assert(aenmNames[iReg] == (a_enmName)); \
+            (a_DstVar) = aValues[iReg].Reg64; \
+            iReg++; \
+        } while (0)
+#  define GET_REG64_LOG7(a_DstVar, a_enmName, a_szLogName) do { \
+            Assert(aenmNames[iReg] == (a_enmName)); \
+            if ((a_DstVar) != aValues[iReg].Reg64) \
+                Log7(("NEM/%u: " a_szLogName " changed %RX64 -> %RX64\n", pVCpu->idCpu, (a_DstVar), aValues[iReg].Reg64)); \
+            (a_DstVar) = aValues[iReg].Reg64; \
+            iReg++; \
+        } while (0)
+#  define GET_REG128(a_DstVarLo, a_DstVarHi, a_enmName) do { \
+            Assert(aenmNames[iReg] == a_enmName); \
+            (a_DstVarLo) = aValues[iReg].Reg128.Low64; \
+            (a_DstVarHi) = aValues[iReg].Reg128.High64; \
+            iReg++; \
+        } while (0)
+#  define GET_SEG(a_SReg, a_enmName) do { \
+            Assert(aenmNames[iReg] == (a_enmName)); \
+            NEM_WIN_COPY_BACK_SEG(a_SReg, aValues[iReg].Segment); \
+            iReg++; \
+        } while (0)
+
+    /* GPRs */
+    if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_RAX)
+            GET_REG64(pVCpu->cpum.GstCtx.rax, WHvX64RegisterRax);
+        if (fWhat & CPUMCTX_EXTRN_RCX)
+            GET_REG64(pVCpu->cpum.GstCtx.rcx, WHvX64RegisterRcx);
+        if (fWhat & CPUMCTX_EXTRN_RDX)
+            GET_REG64(pVCpu->cpum.GstCtx.rdx, WHvX64RegisterRdx);
+        if (fWhat & CPUMCTX_EXTRN_RBX)
+            GET_REG64(pVCpu->cpum.GstCtx.rbx, WHvX64RegisterRbx);
+        if (fWhat & CPUMCTX_EXTRN_RSP)
+            GET_REG64(pVCpu->cpum.GstCtx.rsp, WHvX64RegisterRsp);
+        if (fWhat & CPUMCTX_EXTRN_RBP)
+            GET_REG64(pVCpu->cpum.GstCtx.rbp, WHvX64RegisterRbp);
+        if (fWhat & CPUMCTX_EXTRN_RSI)
+            GET_REG64(pVCpu->cpum.GstCtx.rsi, WHvX64RegisterRsi);
+        if (fWhat & CPUMCTX_EXTRN_RDI)
+            GET_REG64(pVCpu->cpum.GstCtx.rdi, WHvX64RegisterRdi);
+        if (fWhat & CPUMCTX_EXTRN_R8_R15)
+        {
+            GET_REG64(pVCpu->cpum.GstCtx.r8, WHvX64RegisterR8);
+            GET_REG64(pVCpu->cpum.GstCtx.r9, WHvX64RegisterR9);
+            GET_REG64(pVCpu->cpum.GstCtx.r10, WHvX64RegisterR10);
+            GET_REG64(pVCpu->cpum.GstCtx.r11, WHvX64RegisterR11);
+            GET_REG64(pVCpu->cpum.GstCtx.r12, WHvX64RegisterR12);
+            GET_REG64(pVCpu->cpum.GstCtx.r13, WHvX64RegisterR13);
+            GET_REG64(pVCpu->cpum.GstCtx.r14, WHvX64RegisterR14);
+            GET_REG64(pVCpu->cpum.GstCtx.r15, WHvX64RegisterR15);
+        }
+    }
+
+    /* RIP & Flags */
+    if (fWhat & CPUMCTX_EXTRN_RIP)
+        GET_REG64(pVCpu->cpum.GstCtx.rip, WHvX64RegisterRip);
+    if (fWhat & CPUMCTX_EXTRN_RFLAGS)
+        GET_REG64(pVCpu->cpum.GstCtx.rflags.u, WHvX64RegisterRflags);
+
+    /* Segments */
+    if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_ES)
+            GET_SEG(pVCpu->cpum.GstCtx.es, WHvX64RegisterEs);
+        if (fWhat & CPUMCTX_EXTRN_CS)
+            GET_SEG(pVCpu->cpum.GstCtx.cs, WHvX64RegisterCs);
+        if (fWhat & CPUMCTX_EXTRN_SS)
+            GET_SEG(pVCpu->cpum.GstCtx.ss, WHvX64RegisterSs);
+        if (fWhat & CPUMCTX_EXTRN_DS)
+            GET_SEG(pVCpu->cpum.GstCtx.ds, WHvX64RegisterDs);
+        if (fWhat & CPUMCTX_EXTRN_FS)
+            GET_SEG(pVCpu->cpum.GstCtx.fs, WHvX64RegisterFs);
+        if (fWhat & CPUMCTX_EXTRN_GS)
+            GET_SEG(pVCpu->cpum.GstCtx.gs, WHvX64RegisterGs);
+    }
+
+    /* Descriptor tables and the task segment. */
+    if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_LDTR)
+            GET_SEG(pVCpu->cpum.GstCtx.ldtr, WHvX64RegisterLdtr);
+
+        if (fWhat & CPUMCTX_EXTRN_TR)
+        {
+            /* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY.  So,
+               avoid to trigger sanity assertions around the code, always fix this. */
+            GET_SEG(pVCpu->cpum.GstCtx.tr, WHvX64RegisterTr);
+            switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
+            {
+                case X86_SEL_TYPE_SYS_386_TSS_BUSY:
+                case X86_SEL_TYPE_SYS_286_TSS_BUSY:
+                    break;
+                case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
+                    pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
+                    break;
+                case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
+                    pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
+                    break;
+            }
+        }
+        if (fWhat & CPUMCTX_EXTRN_IDTR)
+        {
+            Assert(aenmNames[iReg] == WHvX64RegisterIdtr);
+            pVCpu->cpum.GstCtx.idtr.cbIdt = aValues[iReg].Table.Limit;
+            pVCpu->cpum.GstCtx.idtr.pIdt  = aValues[iReg].Table.Base;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_GDTR)
+        {
+            Assert(aenmNames[iReg] == WHvX64RegisterGdtr);
+            pVCpu->cpum.GstCtx.gdtr.cbGdt = aValues[iReg].Table.Limit;
+            pVCpu->cpum.GstCtx.gdtr.pGdt  = aValues[iReg].Table.Base;
+            iReg++;
+        }
+    }
+
+    /* Control registers. */
+    bool fMaybeChangedMode = false;
+    bool fUpdateCr3        = false;
+    if (fWhat & CPUMCTX_EXTRN_CR_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_CR0)
+        {
+            Assert(aenmNames[iReg] == WHvX64RegisterCr0);
+            if (pVCpu->cpum.GstCtx.cr0 != aValues[iReg].Reg64)
+            {
+                CPUMSetGuestCR0(pVCpu, aValues[iReg].Reg64);
+                fMaybeChangedMode = true;
+            }
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_CR2)
+            GET_REG64(pVCpu->cpum.GstCtx.cr2, WHvX64RegisterCr2);
+        if (fWhat & CPUMCTX_EXTRN_CR3)
+        {
+            if (pVCpu->cpum.GstCtx.cr3 != aValues[iReg].Reg64)
+            {
+                CPUMSetGuestCR3(pVCpu, aValues[iReg].Reg64);
+                fUpdateCr3 = true;
+            }
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_CR4)
+        {
+            if (pVCpu->cpum.GstCtx.cr4 != aValues[iReg].Reg64)
+            {
+                CPUMSetGuestCR4(pVCpu, aValues[iReg].Reg64);
+                fMaybeChangedMode = true;
+            }
+            iReg++;
+        }
+    }
+    if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterCr8);
+        APICSetTpr(pVCpu, (uint8_t)aValues[iReg].Reg64 << 4);
+        iReg++;
+    }
+
+    /* Debug registers. */
+    if (fWhat & CPUMCTX_EXTRN_DR7)
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterDr7);
+        if (pVCpu->cpum.GstCtx.dr[7] != aValues[iReg].Reg64)
+            CPUMSetGuestDR7(pVCpu, aValues[iReg].Reg64);
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_DR7; /* Hack alert! Avoids asserting when processing CPUMCTX_EXTRN_DR0_DR3. */
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterDr0);
+        Assert(aenmNames[iReg+3] == WHvX64RegisterDr3);
+        if (pVCpu->cpum.GstCtx.dr[0] != aValues[iReg].Reg64)
+            CPUMSetGuestDR0(pVCpu, aValues[iReg].Reg64);
+        iReg++;
+        if (pVCpu->cpum.GstCtx.dr[1] != aValues[iReg].Reg64)
+            CPUMSetGuestDR1(pVCpu, aValues[iReg].Reg64);
+        iReg++;
+        if (pVCpu->cpum.GstCtx.dr[2] != aValues[iReg].Reg64)
+            CPUMSetGuestDR2(pVCpu, aValues[iReg].Reg64);
+        iReg++;
+        if (pVCpu->cpum.GstCtx.dr[3] != aValues[iReg].Reg64)
+            CPUMSetGuestDR3(pVCpu, aValues[iReg].Reg64);
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR6)
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterDr6);
+        if (pVCpu->cpum.GstCtx.dr[6] != aValues[iReg].Reg64)
+            CPUMSetGuestDR6(pVCpu, aValues[iReg].Reg64);
+        iReg++;
+    }
+
+    /* Floating point state. */
+    if (fWhat & CPUMCTX_EXTRN_X87)
+    {
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[0].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[0].au64[1], WHvX64RegisterFpMmx0);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[1].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[1].au64[1], WHvX64RegisterFpMmx1);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[2].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[2].au64[1], WHvX64RegisterFpMmx2);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[3].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[3].au64[1], WHvX64RegisterFpMmx3);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[4].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[4].au64[1], WHvX64RegisterFpMmx4);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[5].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[5].au64[1], WHvX64RegisterFpMmx5);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[6].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[6].au64[1], WHvX64RegisterFpMmx6);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[7].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[7].au64[1], WHvX64RegisterFpMmx7);
+
+        Assert(aenmNames[iReg] == WHvX64RegisterFpControlStatus);
+        pVCpu->cpum.GstCtx.pXStateR3->x87.FCW        = aValues[iReg].FpControlStatus.FpControl;
+        pVCpu->cpum.GstCtx.pXStateR3->x87.FSW        = aValues[iReg].FpControlStatus.FpStatus;
+        pVCpu->cpum.GstCtx.pXStateR3->x87.FTW        = aValues[iReg].FpControlStatus.FpTag
+                                        /*| (aValues[iReg].FpControlStatus.Reserved << 8)*/;
+        pVCpu->cpum.GstCtx.pXStateR3->x87.FOP        = aValues[iReg].FpControlStatus.LastFpOp;
+        pVCpu->cpum.GstCtx.pXStateR3->x87.FPUIP      = (uint32_t)aValues[iReg].FpControlStatus.LastFpRip;
+        pVCpu->cpum.GstCtx.pXStateR3->x87.CS         = (uint16_t)(aValues[iReg].FpControlStatus.LastFpRip >> 32);
+        pVCpu->cpum.GstCtx.pXStateR3->x87.Rsrvd1     = (uint16_t)(aValues[iReg].FpControlStatus.LastFpRip >> 48);
+        iReg++;
+    }
+
+    if (fWhat & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX))
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterXmmControlStatus);
+        if (fWhat & CPUMCTX_EXTRN_X87)
+        {
+            pVCpu->cpum.GstCtx.pXStateR3->x87.FPUDP  = (uint32_t)aValues[iReg].XmmControlStatus.LastFpRdp;
+            pVCpu->cpum.GstCtx.pXStateR3->x87.DS     = (uint16_t)(aValues[iReg].XmmControlStatus.LastFpRdp >> 32);
+            pVCpu->cpum.GstCtx.pXStateR3->x87.Rsrvd2 = (uint16_t)(aValues[iReg].XmmControlStatus.LastFpRdp >> 48);
+        }
+        pVCpu->cpum.GstCtx.pXStateR3->x87.MXCSR      = aValues[iReg].XmmControlStatus.XmmStatusControl;
+        pVCpu->cpum.GstCtx.pXStateR3->x87.MXCSR_MASK = aValues[iReg].XmmControlStatus.XmmStatusControlMask; /** @todo ??? (Isn't this an output field?) */
+        iReg++;
+    }
+
+    /* Vector state. */
+    if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
+    {
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 0].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 0].uXmm.s.Hi, WHvX64RegisterXmm0);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 1].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 1].uXmm.s.Hi, WHvX64RegisterXmm1);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 2].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 2].uXmm.s.Hi, WHvX64RegisterXmm2);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 3].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 3].uXmm.s.Hi, WHvX64RegisterXmm3);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 4].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 4].uXmm.s.Hi, WHvX64RegisterXmm4);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 5].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 5].uXmm.s.Hi, WHvX64RegisterXmm5);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 6].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 6].uXmm.s.Hi, WHvX64RegisterXmm6);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 7].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 7].uXmm.s.Hi, WHvX64RegisterXmm7);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 8].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 8].uXmm.s.Hi, WHvX64RegisterXmm8);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 9].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 9].uXmm.s.Hi, WHvX64RegisterXmm9);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[10].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[10].uXmm.s.Hi, WHvX64RegisterXmm10);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[11].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[11].uXmm.s.Hi, WHvX64RegisterXmm11);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[12].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[12].uXmm.s.Hi, WHvX64RegisterXmm12);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[13].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[13].uXmm.s.Hi, WHvX64RegisterXmm13);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[14].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[14].uXmm.s.Hi, WHvX64RegisterXmm14);
+        GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[15].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[15].uXmm.s.Hi, WHvX64RegisterXmm15);
+    }
+
+    /* MSRs */
+    // WHvX64RegisterTsc - don't touch
+    if (fWhat & CPUMCTX_EXTRN_EFER)
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterEfer);
+        if (aValues[iReg].Reg64 != pVCpu->cpum.GstCtx.msrEFER)
+        {
+            Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, aValues[iReg].Reg64));
+            if ((aValues[iReg].Reg64 ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
+                PGMNotifyNxeChanged(pVCpu, RT_BOOL(aValues[iReg].Reg64 & MSR_K6_EFER_NXE));
+            pVCpu->cpum.GstCtx.msrEFER = aValues[iReg].Reg64;
+            fMaybeChangedMode = true;
+        }
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrKERNELGSBASE, WHvX64RegisterKernelGsBase, "MSR KERNEL_GS_BASE");
+    if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+    {
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.cs,  WHvX64RegisterSysenterCs,  "MSR SYSENTER.CS");
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.eip, WHvX64RegisterSysenterEip, "MSR SYSENTER.EIP");
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.esp, WHvX64RegisterSysenterEsp, "MSR SYSENTER.ESP");
+    }
+    if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+    {
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrSTAR,   WHvX64RegisterStar,   "MSR STAR");
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrLSTAR,  WHvX64RegisterLstar,  "MSR LSTAR");
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrCSTAR,  WHvX64RegisterCstar,  "MSR CSTAR");
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrSFMASK, WHvX64RegisterSfmask, "MSR SFMASK");
+    }
+    if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterApicBase);
+        const uint64_t uOldBase = APICGetBaseMsrNoCheck(pVCpu);
+        if (aValues[iReg].Reg64 != uOldBase)
+        {
+            Log7(("NEM/%u: MSR APICBase changed %RX64 -> %RX64 (%RX64)\n",
+                  pVCpu->idCpu, uOldBase, aValues[iReg].Reg64, aValues[iReg].Reg64 ^ uOldBase));
+            int rc2 = APICSetBaseMsr(pVCpu, aValues[iReg].Reg64);
+            AssertLogRelMsg(rc2 == VINF_SUCCESS, ("%Rrc %RX64\n", rc2, aValues[iReg].Reg64));
+        }
+        iReg++;
+
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterPat, "MSR PAT");
+#if 0 /*def LOG_ENABLED*/ /** @todo something's wrong with HvX64RegisterMtrrCap? (AMD) */
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterMsrMtrrCap);
+#endif
+        PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
+        GET_REG64_LOG7(pCtxMsrs->msr.MtrrDefType,      WHvX64RegisterMsrMtrrDefType,     "MSR MTRR_DEF_TYPE");
+        GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix64K_00000, WHvX64RegisterMsrMtrrFix64k00000, "MSR MTRR_FIX_64K_00000");
+        GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_80000, WHvX64RegisterMsrMtrrFix16k80000, "MSR MTRR_FIX_16K_80000");
+        GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_A0000, WHvX64RegisterMsrMtrrFix16kA0000, "MSR MTRR_FIX_16K_A0000");
+        GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C0000,  WHvX64RegisterMsrMtrrFix4kC0000,  "MSR MTRR_FIX_4K_C0000");
+        GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C8000,  WHvX64RegisterMsrMtrrFix4kC8000,  "MSR MTRR_FIX_4K_C8000");
+        GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D0000,  WHvX64RegisterMsrMtrrFix4kD0000,  "MSR MTRR_FIX_4K_D0000");
+        GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D8000,  WHvX64RegisterMsrMtrrFix4kD8000,  "MSR MTRR_FIX_4K_D8000");
+        GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E0000,  WHvX64RegisterMsrMtrrFix4kE0000,  "MSR MTRR_FIX_4K_E0000");
+        GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E8000,  WHvX64RegisterMsrMtrrFix4kE8000,  "MSR MTRR_FIX_4K_E8000");
+        GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F0000,  WHvX64RegisterMsrMtrrFix4kF0000,  "MSR MTRR_FIX_4K_F0000");
+        GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F8000,  WHvX64RegisterMsrMtrrFix4kF8000,  "MSR MTRR_FIX_4K_F8000");
+        GET_REG64_LOG7(pCtxMsrs->msr.TscAux,           WHvX64RegisterTscAux,             "MSR TSC_AUX");
+        /** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
+    }
+
+    /* Interruptibility. */
+    if (fWhat & (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
+    {
+        Assert(aenmNames[iReg] == WHvRegisterInterruptState);
+        Assert(aenmNames[iReg + 1] == WHvX64RegisterRip);
+
+        if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT))
+        {
+            pVCpu->nem.s.fLastInterruptShadow = aValues[iReg].InterruptState.InterruptShadow;
+            if (aValues[iReg].InterruptState.InterruptShadow)
+                EMSetInhibitInterruptsPC(pVCpu, aValues[iReg + 1].Reg64);
+            else
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+        }
+
+        if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
+        {
+            if (aValues[iReg].InterruptState.NmiMasked)
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
+            else
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
+        }
+
+        fWhat |= CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI;
+        iReg += 2;
+    }
+
+    /* Event injection. */
+    /// @todo WHvRegisterPendingInterruption
+    Assert(aenmNames[iReg] == WHvRegisterPendingInterruption);
+    if (aValues[iReg].PendingInterruption.InterruptionPending)
+    {
+        Log7(("PendingInterruption: type=%u vector=%#x errcd=%RTbool/%#x instr-len=%u nested=%u\n",
+              aValues[iReg].PendingInterruption.InterruptionType, aValues[iReg].PendingInterruption.InterruptionVector,
+              aValues[iReg].PendingInterruption.DeliverErrorCode, aValues[iReg].PendingInterruption.ErrorCode,
+              aValues[iReg].PendingInterruption.InstructionLength, aValues[iReg].PendingInterruption.NestedEvent));
+        AssertMsg((aValues[iReg].PendingInterruption.AsUINT64 & UINT64_C(0xfc00)) == 0,
+                  ("%#RX64\n", aValues[iReg].PendingInterruption.AsUINT64));
+    }
+
+    /// @todo WHvRegisterPendingEvent0 (renamed to WHvRegisterPendingEvent).
+
+    /* Almost done, just update extrn flags and maybe change PGM mode. */
+    pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
+    if (!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT))))
+        pVCpu->cpum.GstCtx.fExtrn = 0;
+
+    /* Typical. */
+    if (!fMaybeChangedMode && !fUpdateCr3)
+        return VINF_SUCCESS;
+
+    /*
+     * Slow.
+     */
+    if (fMaybeChangedMode)
+    {
+        int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER);
+        AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
+    }
+
+    if (fUpdateCr3)
+    {
+        int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3);
+        AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
+    }
+
+    return VINF_SUCCESS;
+# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
+}
+
+#endif /* !IN_RING0 */
+
+
+/**
+ * Interface for importing state on demand (used by IEM).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context CPU structure.
+ * @param   fWhat       What to import, CPUMCTX_EXTRN_XXX.
+ */
+VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
+{
+    STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
+
+#ifdef IN_RING0
+# ifdef NEM_WIN_WITH_RING0_RUNLOOP
+    return nemR0WinImportState(pVCpu->pGVM, pVCpu, &pVCpu->cpum.GstCtx, fWhat, true /*fCanUpdateCr3*/);
+# else
+    RT_NOREF(pVCpu, fWhat);
+    return VERR_NOT_IMPLEMENTED;
+# endif
+#else
+    return nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat);
+#endif
+}
+
+
+/**
+ * Query the CPU tick counter and optionally the TSC_AUX MSR value.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context CPU structure.
+ * @param   pcTicks     Where to return the CPU tick count.
+ * @param   puAux       Where to return the TSC_AUX register value.
+ */
+VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t *pcTicks, uint32_t *puAux)
+{
+    STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
+
+#ifdef IN_RING3
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
+    AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
+
+# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) || defined(NEM_WIN_WITH_RING0_RUNLOOP)
+#  if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
+    if (pVM->nem.s.fUseRing0Runloop)
+#  endif
+    {
+        /* Call ring-0 and get the values. */
+        int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_QUERY_CPU_TICK, 0, NULL);
+        AssertLogRelRCReturn(rc, rc);
+        *pcTicks = pVCpu->nem.s.Hypercall.QueryCpuTick.cTicks;
+        if (puAux)
+            *puAux = pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX
+                   ? pVCpu->nem.s.Hypercall.QueryCpuTick.uAux : CPUMGetGuestTscAux(pVCpu);
+        return VINF_SUCCESS;
+    }
+# endif
+# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
+    /* Call the offical API. */
+    WHV_REGISTER_NAME  aenmNames[2] = { WHvX64RegisterTsc, WHvX64RegisterTscAux };
+    WHV_REGISTER_VALUE aValues[2]   = { {0, 0}, {0, 0} };
+    Assert(RT_ELEMENTS(aenmNames) == RT_ELEMENTS(aValues));
+    HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, 2, aValues);
+    AssertLogRelMsgReturn(SUCCEEDED(hrc),
+                          ("WHvGetVirtualProcessorRegisters(%p, %u,{tsc,tsc_aux},2,) -> %Rhrc (Last=%#x/%u)\n",
+                           pVM->nem.s.hPartition, pVCpu->idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
+                          , VERR_NEM_GET_REGISTERS_FAILED);
+    *pcTicks = aValues[0].Reg64;
+    if (puAux)
+        *pcTicks = pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX ? aValues[0].Reg64 : CPUMGetGuestTscAux(pVCpu);
+    return VINF_SUCCESS;
+# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
+#else  /* IN_RING0 */
+# ifdef NEM_WIN_WITH_RING0_RUNLOOP
+    int rc = nemR0WinQueryCpuTick(pVCpu->pGVM, pVCpu, pcTicks, puAux);
+    if (RT_SUCCESS(rc) && puAux && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX))
+        *puAux = CPUMGetGuestTscAux(pVCpu);
+    return rc;
+# else
+    RT_NOREF(pVCpu, pcTicks, puAux);
+    return VERR_NOT_IMPLEMENTED;
+# endif
+#endif /* IN_RING0 */
+}
+
+
+/**
+ * Resumes CPU clock (TSC) on all virtual CPUs.
+ *
+ * This is called by TM when the VM is started, restored, resumed or similar.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context CPU structure of the calling EMT.
+ * @param   uPausedTscValue The TSC value at the time of pausing.
+ */
+VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
+{
+#ifdef IN_RING0
+# ifdef NEM_WIN_WITH_RING0_RUNLOOP
+    return nemR0WinResumeCpuTickOnAll(pVM, pVCpu, uPausedTscValue);
+# else
+    RT_NOREF(pVM, pVCpu, uPausedTscValue);
+    return VERR_NOT_IMPLEMENTED;
+# endif
+#else  /* IN_RING3 */
+    VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
+    AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
+
+# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) || defined(NEM_WIN_WITH_RING0_RUNLOOP)
+#  if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
+    if (pVM->nem.s.fUseRing0Runloop)
+#  endif
+    {
+        /* Call ring-0 and do it all there. */
+        return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_RESUME_CPU_TICK_ON_ALL, uPausedTscValue, NULL);
+    }
+# endif
+# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
+    /*
+     * Call the offical API to do the job.
+     */
+    if (pVM->cCpus > 1)
+        RTThreadYield(); /* Try decrease the chance that we get rescheduled in the middle. */
+
+    /* Start with the first CPU. */
+    WHV_REGISTER_NAME  enmName   = WHvX64RegisterTsc;
+    WHV_REGISTER_VALUE Value     = {0, 0};
+    Value.Reg64 = uPausedTscValue;
+    uint64_t const     uFirstTsc = ASMReadTSC();
+    HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, 0 /*iCpu*/, &enmName, 1, &Value);
+    AssertLogRelMsgReturn(SUCCEEDED(hrc),
+                          ("WHvSetVirtualProcessorRegisters(%p, 0,{tsc},2,%#RX64) -> %Rhrc (Last=%#x/%u)\n",
+                           pVM->nem.s.hPartition, uPausedTscValue, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
+                          , VERR_NEM_SET_TSC);
+
+    /* Do the other CPUs, adjusting for elapsed TSC and keeping finger crossed
+       that we don't introduce too much drift here. */
+    for (VMCPUID iCpu = 1; iCpu < pVM->cCpus; iCpu++)
+    {
+        Assert(enmName == WHvX64RegisterTsc);
+        const uint64_t offDelta = (ASMReadTSC() - uFirstTsc);
+        Value.Reg64 = uPausedTscValue + offDelta;
+        HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, iCpu, &enmName, 1, &Value);
+        AssertLogRelMsgReturn(SUCCEEDED(hrc),
+                              ("WHvSetVirtualProcessorRegisters(%p, 0,{tsc},2,%#RX64 + %#RX64) -> %Rhrc (Last=%#x/%u)\n",
+                               pVM->nem.s.hPartition, iCpu, uPausedTscValue, offDelta, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
+                              , VERR_NEM_SET_TSC);
+    }
+
+    return VINF_SUCCESS;
+# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
+#endif /* IN_RING3 */
+}
+
+#ifdef NEMWIN_NEED_GET_REGISTER
+# if defined(IN_RING0) || defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
+/** Worker for assertion macro. */
+NEM_TMPL_STATIC int nemHCWinGetRegister(PVMCPUCC pVCpu, PGVMCPU pGVCpu, uint32_t enmReg, HV_REGISTER_VALUE *pRetValue)
+{
+    RT_ZERO(*pRetValue);
+#  ifdef IN_RING3
+    RT_NOREF(pVCpu, pGVCpu, enmReg);
+    return VERR_NOT_IMPLEMENTED;
+#  else
+    NOREF(pVCpu);
+
+    /*
+     * Hypercall parameters.
+     */
+    HV_INPUT_GET_VP_REGISTERS *pInput = (HV_INPUT_GET_VP_REGISTERS *)pGVCpu->nem.s.HypercallData.pbPage;
+    AssertPtrReturn(pInput, VERR_INTERNAL_ERROR_3);
+    AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API);
+
+    pInput->PartitionId = pVCpu->pGVM->nemr0.s.idHvPartition;
+    pInput->VpIndex     = pVCpu->idCpu;
+    pInput->fFlags      = 0;
+    pInput->Names[0]    = (HV_REGISTER_NAME)enmReg;
+
+    size_t const cbInput = RT_ALIGN_Z(RT_UOFFSETOF(HV_INPUT_GET_VP_REGISTERS, Names[1]), 32);
+    HV_REGISTER_VALUE *paValues = (HV_REGISTER_VALUE *)((uint8_t *)pInput + cbInput);
+    RT_BZERO(paValues, sizeof(paValues[0]) * 1);
+
+    /*
+     * Make the hypercall and copy out the value.
+     */
+    uint64_t uResult = g_pfnHvlInvokeHypercall(HV_MAKE_CALL_INFO(HvCallGetVpRegisters, 1),
+                                               pGVCpu->nem.s.HypercallData.HCPhysPage,
+                                               pGVCpu->nem.s.HypercallData.HCPhysPage + cbInput);
+    AssertLogRelMsgReturn(uResult == HV_MAKE_CALL_REP_RET(1), ("uResult=%RX64 cRegs=%#x\n", uResult, 1),
+                          VERR_NEM_GET_REGISTERS_FAILED);
+
+    *pRetValue = paValues[0];
+    return VINF_SUCCESS;
+#  endif
+}
+# else
+/** Worker for assertion macro. */
+NEM_TMPL_STATIC int nemR3WinGetRegister(PVMCPUCC a_pVCpu, uint32_t a_enmReg, WHV_REGISTER_VALUE pValue)
+{
+    RT_ZERO(*pRetValue);
+    RT_NOREF(pVCpu, pGVCpu, enmReg);
+    return VERR_NOT_IMPLEMENTED;
+}
+# endif
+#endif
+
+
+#ifdef LOG_ENABLED
+/**
+ * Get the virtual processor running status.
+ */
+DECLINLINE(VID_PROCESSOR_STATUS) nemHCWinCpuGetRunningStatus(PVMCPUCC pVCpu)
+{
+# ifdef IN_RING0
+    NOREF(pVCpu);
+    return VidProcessorStatusUndefined;
+# else
+    RTERRVARS Saved;
+    RTErrVarsSave(&Saved);
+
+    /*
+     * This API is disabled in release builds, it seems.  On build 17101 it requires
+     * the following patch to be enabled (windbg): eb vid+12180 0f 84 98 00 00 00
+     */
+    VID_PROCESSOR_STATUS enmCpuStatus = VidProcessorStatusUndefined;
+    NTSTATUS rcNt = g_pfnVidGetVirtualProcessorRunningStatus(pVCpu->pVMR3->nem.s.hPartitionDevice, pVCpu->idCpu, &enmCpuStatus);
+    AssertRC(rcNt);
+
+    RTErrVarsRestore(&Saved);
+    return enmCpuStatus;
+# endif
+}
+#endif /* LOG_ENABLED */
+
+
+#if defined(NEM_WIN_USE_OUR_OWN_RUN_API) || defined(NEM_WIN_WITH_RING0_RUNLOOP)
+# ifdef IN_RING3 /* hopefully not needed in ring-0, as we'd need KTHREADs and KeAlertThread. */
+/**
+ * Our own WHvCancelRunVirtualProcessor that can later be moved to ring-0.
+ *
+ * This is an experiment only.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.
+ */
+NEM_TMPL_STATIC int nemHCWinCancelRunVirtualProcessor(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    /*
+     * Work the state.
+     *
+     * From the looks of things, we should let the EMT call VidStopVirtualProcessor.
+     * So, we just need to modify the state and kick the EMT if it's waiting on
+     * messages.  For the latter we use QueueUserAPC / KeAlterThread.
+     */
+    for (;;)
+    {
+        VMCPUSTATE enmState = VMCPU_GET_STATE(pVCpu);
+        switch (enmState)
+        {
+            case VMCPUSTATE_STARTED_EXEC_NEM:
+                if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED, VMCPUSTATE_STARTED_EXEC_NEM))
+                {
+                    DBGFTRACE_CUSTOM(pVM, "VMCPUSTATE_STARTED_EXEC_NEM -> CANCELED");
+                    Log8(("nemHCWinCancelRunVirtualProcessor: Switched %u to canceled state\n", pVCpu->idCpu));
+                    STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatCancelChangedState);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case VMCPUSTATE_STARTED_EXEC_NEM_WAIT:
+                if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED, VMCPUSTATE_STARTED_EXEC_NEM_WAIT))
+                {
+                    DBGFTRACE_CUSTOM(pVM, "VMCPUSTATE_STARTED_EXEC_NEM_WAIT -> CANCELED");
+#  ifdef IN_RING0
+                    NTSTATUS rcNt = KeAlertThread(??);
+                    DBGFTRACE_CUSTOM(pVM, "KeAlertThread -> %#x", rcNt);
+#  else
+                    NTSTATUS rcNt = NtAlertThread(pVCpu->nem.s.hNativeThreadHandle);
+                    DBGFTRACE_CUSTOM(pVM, "NtAlertThread -> %#x", rcNt);
+#  endif
+                    Log8(("nemHCWinCancelRunVirtualProcessor: Alerted %u: %#x\n", pVCpu->idCpu, rcNt));
+                    Assert(rcNt == STATUS_SUCCESS);
+                    if (NT_SUCCESS(rcNt))
+                    {
+                        STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatCancelAlertedThread);
+                        return VINF_SUCCESS;
+                    }
+                    AssertLogRelMsgFailedReturn(("NtAlertThread failed: %#x\n", rcNt), RTErrConvertFromNtStatus(rcNt));
+                }
+                break;
+
+            default:
+                return VINF_SUCCESS;
+        }
+
+        ASMNopPause();
+        RT_NOREF(pVM);
+    }
+}
+# endif /* IN_RING3 */
+#endif /* NEM_WIN_USE_OUR_OWN_RUN_API || NEM_WIN_WITH_RING0_RUNLOOP */
+
+
+#ifdef LOG_ENABLED
+/**
+ * Logs the current CPU state.
+ */
+NEM_TMPL_STATIC void nemHCWinLogState(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    if (LogIs3Enabled())
+    {
+# if 0 // def IN_RING3 - causes lazy state import assertions all over CPUM.
+        char szRegs[4096];
+        DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
+                        "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
+                        "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
+                        "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
+                        "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
+                        "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
+                        "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
+                        "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
+                        "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
+                        "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
+                        "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
+                        "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
+                        "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
+                        "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
+                        "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim}  idtr=%016VR{idtr_base}:%04VR{idtr_lim}  rflags=%08VR{rflags}\n"
+                        "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
+                        "tr  ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
+                        "    sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
+                        "        efer=%016VR{efer}\n"
+                        "         pat=%016VR{pat}\n"
+                        "     sf_mask=%016VR{sf_mask}\n"
+                        "krnl_gs_base=%016VR{krnl_gs_base}\n"
+                        "       lstar=%016VR{lstar}\n"
+                        "        star=%016VR{star} cstar=%016VR{cstar}\n"
+                        "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
+                        );
+
+        char szInstr[256];
+        DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
+                           DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                           szInstr, sizeof(szInstr), NULL);
+        Log3(("%s%s\n", szRegs, szInstr));
+# else
+        /** @todo stat logging in ring-0 */
+        RT_NOREF(pVM, pVCpu);
+# endif
+    }
+}
+#endif /* LOG_ENABLED */
+
+
+/** Macro used by nemHCWinExecStateToLogStr and nemR3WinExecStateToLogStr. */
+#define SWITCH_IT(a_szPrefix) \
+    do \
+        switch (u)\
+        { \
+            case 0x00: return a_szPrefix ""; \
+            case 0x01: return a_szPrefix ",Pnd"; \
+            case 0x02: return a_szPrefix ",Dbg"; \
+            case 0x03: return a_szPrefix ",Pnd,Dbg"; \
+            case 0x04: return a_szPrefix ",Shw"; \
+            case 0x05: return a_szPrefix ",Pnd,Shw"; \
+            case 0x06: return a_szPrefix ",Shw,Dbg"; \
+            case 0x07: return a_szPrefix ",Pnd,Shw,Dbg"; \
+            default: AssertFailedReturn("WTF?"); \
+        } \
+    while (0)
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Translates the execution stat bitfield into a short log string, VID version.
+ *
+ * @returns Read-only log string.
+ * @param   pMsgHdr       The header which state to summarize.
+ */
+static const char *nemHCWinExecStateToLogStr(HV_X64_INTERCEPT_MESSAGE_HEADER const *pMsgHdr)
+{
+    unsigned u = (unsigned)pMsgHdr->ExecutionState.InterruptionPending
+               | ((unsigned)pMsgHdr->ExecutionState.DebugActive << 1)
+               | ((unsigned)pMsgHdr->ExecutionState.InterruptShadow << 2);
+    if (pMsgHdr->ExecutionState.EferLma)
+        SWITCH_IT("LM");
+    else if (pMsgHdr->ExecutionState.Cr0Pe)
+        SWITCH_IT("PM");
+    else
+        SWITCH_IT("RM");
+}
+#elif defined(IN_RING3)
+/**
+ * Translates the execution stat bitfield into a short log string, WinHv version.
+ *
+ * @returns Read-only log string.
+ * @param   pExitCtx        The exit context which state to summarize.
+ */
+static const char *nemR3WinExecStateToLogStr(WHV_VP_EXIT_CONTEXT const *pExitCtx)
+{
+    unsigned u = (unsigned)pExitCtx->ExecutionState.InterruptionPending
+               | ((unsigned)pExitCtx->ExecutionState.DebugActive << 1)
+               | ((unsigned)pExitCtx->ExecutionState.InterruptShadow << 2);
+    if (pExitCtx->ExecutionState.EferLma)
+        SWITCH_IT("LM");
+    else if (pExitCtx->ExecutionState.Cr0Pe)
+        SWITCH_IT("PM");
+    else
+        SWITCH_IT("RM");
+}
+#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+#undef SWITCH_IT
+
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Advances the guest RIP and clear EFLAGS.RF, VID version.
+ *
+ * This may clear VMCPU_FF_INHIBIT_INTERRUPTS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitCtx        The exit context.
+ * @param   cbMinInstr      The minimum instruction length, or 1 if not unknown.
+ */
+DECLINLINE(void)
+nemHCWinAdvanceGuestRipAndClearRF(PVMCPUCC pVCpu, HV_X64_INTERCEPT_MESSAGE_HEADER const *pMsgHdr, uint8_t cbMinInstr)
+{
+    Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)));
+
+    /* Advance the RIP. */
+    Assert(pMsgHdr->InstructionLength >= cbMinInstr); RT_NOREF_PV(cbMinInstr);
+    pVCpu->cpum.GstCtx.rip += pMsgHdr->InstructionLength;
+    pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
+
+    /* Update interrupt inhibition. */
+    if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+    { /* likely */ }
+    else if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+}
+#elif defined(IN_RING3)
+/**
+ * Advances the guest RIP and clear EFLAGS.RF, WinHv version.
+ *
+ * This may clear VMCPU_FF_INHIBIT_INTERRUPTS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitCtx        The exit context.
+ * @param   cbMinInstr      The minimum instruction length, or 1 if not unknown.
+ */
+DECLINLINE(void) nemR3WinAdvanceGuestRipAndClearRF(PVMCPUCC pVCpu, WHV_VP_EXIT_CONTEXT const *pExitCtx, uint8_t cbMinInstr)
+{
+    Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)));
+
+    /* Advance the RIP. */
+    Assert(pExitCtx->InstructionLength >= cbMinInstr); RT_NOREF_PV(cbMinInstr);
+    pVCpu->cpum.GstCtx.rip += pExitCtx->InstructionLength;
+    pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
+
+    /* Update interrupt inhibition. */
+    if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+    { /* likely */ }
+    else if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+}
+#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+
+
+NEM_TMPL_STATIC DECLCALLBACK(int)
+nemHCWinUnmapOnePageCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint8_t *pu2NemState, void *pvUser)
+{
+    RT_NOREF_PV(pvUser);
+#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+    int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
+    AssertRC(rc);
+    if (RT_SUCCESS(rc))
+#else
+    RT_NOREF_PV(pVCpu);
+    HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
+    if (SUCCEEDED(hrc))
+#endif
+    {
+        Log5(("NEM GPA unmap all: %RGp (cMappedPages=%u)\n", GCPhys, pVM->nem.s.cMappedPages - 1));
+        *pu2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
+    }
+    else
+    {
+#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+        LogRel(("nemR3WinUnmapOnePageCallback: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
+#else
+        LogRel(("nemR3WinUnmapOnePageCallback: GCPhys=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
+                GCPhys, g_apszPageStates[*pu2NemState], hrc, hrc, RTNtLastStatusValue(),
+                RTNtLastErrorValue(), pVM->nem.s.cMappedPages));
+#endif
+        *pu2NemState = NEM_WIN_PAGE_STATE_NOT_SET;
+    }
+    if (pVM->nem.s.cMappedPages > 0)
+        ASMAtomicDecU32(&pVM->nem.s.cMappedPages);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * State to pass between nemHCWinHandleMemoryAccess / nemR3WinWHvHandleMemoryAccess
+ * and nemHCWinHandleMemoryAccessPageCheckerCallback.
+ */
+typedef struct NEMHCWINHMACPCCSTATE
+{
+    /** Input: Write access. */
+    bool    fWriteAccess;
+    /** Output: Set if we did something. */
+    bool    fDidSomething;
+    /** Output: Set it we should resume. */
+    bool    fCanResume;
+} NEMHCWINHMACPCCSTATE;
+
+/**
+ * @callback_method_impl{FNPGMPHYSNEMCHECKPAGE,
+ *      Worker for nemR3WinHandleMemoryAccess; pvUser points to a
+ *      NEMHCWINHMACPCCSTATE structure. }
+ */
+NEM_TMPL_STATIC DECLCALLBACK(int)
+nemHCWinHandleMemoryAccessPageCheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
+{
+    NEMHCWINHMACPCCSTATE *pState = (NEMHCWINHMACPCCSTATE *)pvUser;
+    pState->fDidSomething = false;
+    pState->fCanResume    = false;
+
+    /* If A20 is disabled, we may need to make another query on the masked
+       page to get the correct protection information. */
+    uint8_t  u2State = pInfo->u2NemState;
+    RTGCPHYS GCPhysSrc;
+    if (   pVM->nem.s.fA20Enabled
+        || !NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
+        GCPhysSrc = GCPhys;
+    else
+    {
+        GCPhysSrc = GCPhys & ~(RTGCPHYS)RT_BIT_32(20);
+        PGMPHYSNEMPAGEINFO Info2;
+        int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhysSrc, pState->fWriteAccess, &Info2, NULL, NULL);
+        AssertRCReturn(rc, rc);
+
+        *pInfo = Info2;
+        pInfo->u2NemState = u2State;
+    }
+
+    /*
+     * Consolidate current page state with actual page protection and access type.
+     * We don't really consider downgrades here, as they shouldn't happen.
+     */
+#ifndef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+    /** @todo Someone at microsoft please explain:
+     * I'm not sure WTF was going on, but I ended up in a loop if I remapped a
+     * readonly page as writable (unmap, then map again).  Specifically, this was an
+     * issue with the big VRAM mapping at 0xe0000000 when booing DSL 4.4.1.  So, in
+     * a hope to work around that we no longer pre-map anything, just unmap stuff
+     * and do it lazily here.  And here we will first unmap, restart, and then remap
+     * with new protection or backing.
+     */
+#endif
+    int rc;
+    switch (u2State)
+    {
+        case NEM_WIN_PAGE_STATE_UNMAPPED:
+        case NEM_WIN_PAGE_STATE_NOT_SET:
+            if (pInfo->fNemProt == NEM_PAGE_PROT_NONE)
+            {
+                Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys));
+                return VINF_SUCCESS;
+            }
+
+            /* Don't bother remapping it if it's a write request to a non-writable page. */
+            if (   pState->fWriteAccess
+                && !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE))
+            {
+                Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys));
+                return VINF_SUCCESS;
+            }
+
+            /* Map the page. */
+            rc = nemHCNativeSetPhysPage(pVM,
+                                        pVCpu,
+                                        GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
+                                        GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
+                                        pInfo->fNemProt,
+                                        &u2State,
+                                        true /*fBackingState*/);
+            pInfo->u2NemState = u2State;
+            Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n",
+                  GCPhys, g_apszPageStates[u2State], rc));
+            pState->fDidSomething = true;
+            pState->fCanResume    = true;
+            return rc;
+
+        case NEM_WIN_PAGE_STATE_READABLE:
+            if (   !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
+                && (pInfo->fNemProt & (NEM_PAGE_PROT_READ | NEM_PAGE_PROT_EXECUTE)))
+            {
+                Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys));
+                return VINF_SUCCESS;
+            }
+
+#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+            /* Upgrade page to writable. */
+/** @todo test this*/
+            if (   (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
+                && pState->fWriteAccess)
+            {
+                rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhys,
+                                              HV_MAP_GPA_READABLE   | HV_MAP_GPA_WRITABLE
+                                              | HV_MAP_GPA_EXECUTABLE | HV_MAP_GPA_EXECUTABLE_AGAIN);
+                AssertRC(rc);
+                if (RT_SUCCESS(rc))
+                {
+                    pInfo->u2NemState = NEM_WIN_PAGE_STATE_WRITABLE;
+                    pState->fDidSomething = true;
+                    pState->fCanResume    = true;
+                    Log5(("NEM GPA write-upgrade/exit: %RGp (was %s, cMappedPages=%u)\n",
+                          GCPhys, g_apszPageStates[u2State], pVM->nem.s.cMappedPages));
+                }
+            }
+            else
+            {
+                /* Need to emulate the acces. */
+                AssertBreak(pInfo->fNemProt != NEM_PAGE_PROT_NONE); /* There should be no downgrades. */
+                rc = VINF_SUCCESS;
+            }
+            return rc;
+#else
+            break;
+#endif
+
+        case NEM_WIN_PAGE_STATE_WRITABLE:
+            if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
+            {
+                if (pInfo->u2OldNemState == NEM_WIN_PAGE_STATE_WRITABLE)
+                    Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3a\n", GCPhys));
+                else
+                {
+                    pState->fCanResume = true;
+                    Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3b (%s -> %s)\n",
+                          GCPhys, g_apszPageStates[pInfo->u2OldNemState], g_apszPageStates[u2State]));
+                }
+                return VINF_SUCCESS;
+            }
+#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+            AssertFailed(); /* There should be no downgrades. */
+#endif
+            break;
+
+        default:
+            AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_NEM_IPE_4);
+    }
+
+    /*
+     * Unmap and restart the instruction.
+     * If this fails, which it does every so often, just unmap everything for now.
+     */
+#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+    rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
+    AssertRC(rc);
+    if (RT_SUCCESS(rc))
+#else
+    /** @todo figure out whether we mess up the state or if it's WHv.   */
+    HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
+    if (SUCCEEDED(hrc))
+#endif
+    {
+        pState->fDidSomething = true;
+        pState->fCanResume    = true;
+        pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
+        uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+        Log5(("NEM GPA unmapped/exit: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[u2State], cMappedPages));
+        return VINF_SUCCESS;
+    }
+#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+    LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhys, rc));
+    return rc;
+#else
+    LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
+            GCPhys, g_apszPageStates[u2State], hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue(),
+            pVM->nem.s.cMappedPages));
+
+    PGMPhysNemEnumPagesByState(pVM, pVCpu, NEM_WIN_PAGE_STATE_READABLE, nemR3WinUnmapOnePageCallback, NULL);
+    Log(("nemHCWinHandleMemoryAccessPageCheckerCallback: Unmapped all (cMappedPages=%u)\n", pVM->nem.s.cMappedPages));
+
+    pState->fDidSomething = true;
+    pState->fCanResume    = true;
+    pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
+    return VINF_SUCCESS;
+#endif
+}
+
+
+
+#if defined(IN_RING0) && defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API)
+/**
+ * Wrapper around nemR0WinImportState that converts VERR_NEM_FLUSH_TLB
+ * into informational status codes and logs+asserts statuses.
+ *
+ * @returns VBox strict status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   pGVCpu          The global (ring-0) per CPU structure.
+ * @param   fWhat           What to import.
+ * @param   pszCaller       Who is doing the importing.
+ */
+DECLINLINE(VBOXSTRICTRC) nemR0WinImportStateStrict(PGVM pGVM, PGVMCPU pGVCpu, uint64_t fWhat, const char *pszCaller)
+{
+    int rc = nemR0WinImportState(pGVM, pGVCpu, &pGVCpu->cpum.GstCtx, fWhat, true /*fCanUpdateCr3*/);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(rc == VINF_SUCCESS);
+        return VINF_SUCCESS;
+    }
+
+    if (rc == VERR_NEM_FLUSH_TLB)
+    {
+        Log4(("%s/%u: nemR0WinImportState -> %Rrc\n", pszCaller, pGVCpu->idCpu, -rc));
+        return -rc;
+    }
+    RT_NOREF(pszCaller);
+    AssertMsgFailedReturn(("%s/%u: nemR0WinImportState failed: %Rrc\n", pszCaller, pGVCpu->idCpu, rc), rc);
+}
+#endif /* IN_RING0 && NEM_WIN_TEMPLATE_MODE_OWN_RUN_API*/
+
+#if defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API) || defined(IN_RING3)
+/**
+ * Wrapper around nemR0WinImportStateStrict and nemHCWinCopyStateFromHyperV.
+ *
+ * Unlike the wrapped APIs, this checks whether it's necessary.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   fWhat           What to import.
+ * @param   pszCaller       Who is doing the importing.
+ */
+DECLINLINE(VBOXSTRICTRC) nemHCWinImportStateIfNeededStrict(PVMCPUCC pVCpu, uint64_t fWhat, const char *pszCaller)
+{
+    if (pVCpu->cpum.GstCtx.fExtrn & fWhat)
+    {
+# ifdef IN_RING0
+        return nemR0WinImportStateStrict(pVCpu->pGVM, pVCpu, fWhat, pszCaller);
+# else
+        RT_NOREF(pszCaller);
+        int rc = nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat);
+        AssertRCReturn(rc, rc);
+# endif
+    }
+    return VINF_SUCCESS;
+}
+#endif /* NEM_WIN_TEMPLATE_MODE_OWN_RUN_API || IN_RING3 */
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Copies register state from the X64 intercept message header.
+ *
+ * ASSUMES no state copied yet.
+ *
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pHdr            The X64 intercept message header.
+ * @sa      nemR3WinCopyStateFromX64Header
+ */
+DECLINLINE(void) nemHCWinCopyStateFromX64Header(PVMCPUCC pVCpu, HV_X64_INTERCEPT_MESSAGE_HEADER const *pHdr)
+{
+    Assert(   (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT))
+           ==                              (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT));
+    NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.cs, pHdr->CsSegment);
+    pVCpu->cpum.GstCtx.rip      = pHdr->Rip;
+    pVCpu->cpum.GstCtx.rflags.u = pHdr->Rflags;
+
+    pVCpu->nem.s.fLastInterruptShadow = pHdr->ExecutionState.InterruptShadow;
+    if (!pHdr->ExecutionState.InterruptShadow)
+    {
+        if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+        { /* likely */ }
+        else
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+    }
+    else
+        EMSetInhibitInterruptsPC(pVCpu, pHdr->Rip);
+
+    APICSetTpr(pVCpu, pHdr->Cr8 << 4);
+
+    pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_APIC_TPR);
+}
+#elif defined(IN_RING3)
+/**
+ * Copies register state from the (common) exit context.
+ *
+ * ASSUMES no state copied yet.
+ *
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExitCtx        The common exit context.
+ * @sa      nemHCWinCopyStateFromX64Header
+ */
+DECLINLINE(void) nemR3WinCopyStateFromX64Header(PVMCPUCC pVCpu, WHV_VP_EXIT_CONTEXT const *pExitCtx)
+{
+    Assert(   (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT))
+           ==                              (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT));
+    NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.cs, pExitCtx->Cs);
+    pVCpu->cpum.GstCtx.rip      = pExitCtx->Rip;
+    pVCpu->cpum.GstCtx.rflags.u = pExitCtx->Rflags;
+
+    pVCpu->nem.s.fLastInterruptShadow = pExitCtx->ExecutionState.InterruptShadow;
+    if (!pExitCtx->ExecutionState.InterruptShadow)
+    {
+        if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+        { /* likely */ }
+        else
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+    }
+    else
+        EMSetInhibitInterruptsPC(pVCpu, pExitCtx->Rip);
+
+    APICSetTpr(pVCpu, pExitCtx->Cr8 << 4);
+
+    pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_APIC_TPR);
+}
+#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Deals with memory intercept message.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pMsg            The message.
+ * @sa      nemR3WinHandleExitMemory
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC
+nemHCWinHandleMessageMemory(PVMCC pVM, PVMCPUCC pVCpu, HV_X64_MEMORY_INTERCEPT_MESSAGE const *pMsg)
+{
+    uint64_t const uHostTsc = ASMReadTSC();
+    Assert(   pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
+           || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
+           || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE);
+
+    /*
+     * Whatever we do, we must clear pending event injection upon resume.
+     */
+    if (pMsg->Header.ExecutionState.InterruptionPending)
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
+
+# if 0 /* Experiment: 20K -> 34K exit/s. */
+    if (   pMsg->Header.ExecutionState.EferLma
+        && pMsg->Header.CsSegment.Long
+        && pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
+    {
+        if (   pMsg->Header.Rip - (uint64_t)0xf65a < (uint64_t)(0xf662 - 0xf65a)
+            && pMsg->InstructionBytes[0] == 0x89
+            && pMsg->InstructionBytes[1] == 0x03)
+        {
+            pVCpu->cpum.GstCtx.rip    = pMsg->Header.Rip + 2;
+            pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP;
+            AssertMsg(pMsg->Header.InstructionLength == 2, ("%#x\n", pMsg->Header.InstructionLength));
+            //Log(("%RX64 msg:\n%.80Rhxd\n", pVCpu->cpum.GstCtx.rip, pMsg));
+            return VINF_SUCCESS;
+        }
+    }
+# endif
+
+    /*
+     * Ask PGM for information about the given GCPhys.  We need to check if we're
+     * out of sync first.
+     */
+    NEMHCWINHMACPCCSTATE State = { pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE, false, false };
+    PGMPHYSNEMPAGEINFO   Info;
+    int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pMsg->GuestPhysicalAddress, State.fWriteAccess, &Info,
+                                       nemHCWinHandleMemoryAccessPageCheckerCallback, &State);
+    if (RT_SUCCESS(rc))
+    {
+        if (Info.fNemProt & (  pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
+                             ? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
+        {
+            if (State.fCanResume)
+            {
+                Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
+                      pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+                      pMsg->GuestPhysicalAddress, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
+                      Info.fHasHandlers ? " handlers" : "", Info.fZeroPage    ? " zero-pg" : "",
+                      State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
+                EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
+                                 pMsg->Header.Rip + pMsg->Header.CsSegment.Base, uHostTsc);
+                return VINF_SUCCESS;
+            }
+        }
+        Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
+              pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+              pMsg->GuestPhysicalAddress, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
+              Info.fHasHandlers ? " handlers" : "", Info.fZeroPage    ? " zero-pg" : "",
+              State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
+    }
+    else
+        Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp rc=%Rrc%s; emulating (%s)\n",
+              pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+              pMsg->GuestPhysicalAddress, rc, State.fDidSomething ? " modified-backing" : "",
+              g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
+
+    /*
+     * Emulate the memory access, either access handler or special memory.
+     */
+    PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
+                                              pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
+                                            ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
+                                            : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
+                                            pMsg->Header.Rip + pMsg->Header.CsSegment.Base, uHostTsc);
+    nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
+    VBOXSTRICTRC rcStrict;
+# ifdef IN_RING0
+    rcStrict = nemR0WinImportStateStrict(pVM, pVCpu,
+                                         NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_DS | CPUMCTX_EXTRN_ES, "MemExit");
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+# else
+    rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_DS | CPUMCTX_EXTRN_ES);
+    AssertRCReturn(rc, rc);
+# endif
+
+    if (pMsg->Reserved1)
+        Log(("MemExit/Reserved1=%#x\n", pMsg->Reserved1));
+    if (pMsg->Header.ExecutionState.Reserved0 || pMsg->Header.ExecutionState.Reserved1)
+        Log(("MemExit/Hdr/State: Reserved0=%#x Reserved1=%#x\n", pMsg->Header.ExecutionState.Reserved0, pMsg->Header.ExecutionState.Reserved1));
+
+    if (!pExitRec)
+    {
+        //if (pMsg->InstructionByteCount > 0)
+        //    Log4(("InstructionByteCount=%#x %.16Rhxs\n", pMsg->InstructionByteCount, pMsg->InstructionBytes));
+        if (pMsg->InstructionByteCount > 0)
+            rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pMsg->Header.Rip,
+                                                    pMsg->InstructionBytes, pMsg->InstructionByteCount);
+        else
+            rcStrict = IEMExecOne(pVCpu);
+        /** @todo do we need to do anything wrt debugging here?   */
+    }
+    else
+    {
+        /* Frequent access or probing. */
+        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+        Log4(("MemExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+              pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    }
+    return rcStrict;
+}
+#elif defined(IN_RING3)
+/**
+ * Deals with memory access exits (WHvRunVpExitReasonMemoryAccess).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessageMemory
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC
+nemR3WinHandleExitMemory(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    uint64_t const uHostTsc = ASMReadTSC();
+    Assert(pExit->MemoryAccess.AccessInfo.AccessType != 3);
+
+    /*
+     * Whatever we do, we must clear pending event injection upon resume.
+     */
+    if (pExit->VpContext.ExecutionState.InterruptionPending)
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
+
+    /*
+     * Ask PGM for information about the given GCPhys.  We need to check if we're
+     * out of sync first.
+     */
+    NEMHCWINHMACPCCSTATE State = { pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite, false, false };
+    PGMPHYSNEMPAGEINFO   Info;
+    int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pExit->MemoryAccess.Gpa, State.fWriteAccess, &Info,
+                                       nemHCWinHandleMemoryAccessPageCheckerCallback, &State);
+    if (RT_SUCCESS(rc))
+    {
+        if (Info.fNemProt & (  pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
+                             ? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
+        {
+            if (State.fCanResume)
+            {
+                Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                      pExit->MemoryAccess.Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
+                      Info.fHasHandlers ? " handlers" : "", Info.fZeroPage    ? " zero-pg" : "",
+                      State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
+                EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
+                                 pExit->VpContext.Rip + pExit->VpContext.Cs.Base, uHostTsc);
+                return VINF_SUCCESS;
+            }
+        }
+        Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
+              pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+              pExit->MemoryAccess.Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
+              Info.fHasHandlers ? " handlers" : "", Info.fZeroPage    ? " zero-pg" : "",
+              State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
+    }
+    else
+        Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp rc=%Rrc%s; emulating (%s)\n",
+              pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+              pExit->MemoryAccess.Gpa, rc, State.fDidSomething ? " modified-backing" : "",
+              g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
+
+    /*
+     * Emulate the memory access, either access handler or special memory.
+     */
+    PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
+                                              pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
+                                            ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
+                                            : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
+                                            pExit->VpContext.Rip + pExit->VpContext.Cs.Base, uHostTsc);
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_DS | CPUMCTX_EXTRN_ES);
+    AssertRCReturn(rc, rc);
+    if (pExit->VpContext.ExecutionState.Reserved0 || pExit->VpContext.ExecutionState.Reserved1)
+        Log(("MemExit/Hdr/State: Reserved0=%#x Reserved1=%#x\n", pExit->VpContext.ExecutionState.Reserved0, pExit->VpContext.ExecutionState.Reserved1));
+
+    VBOXSTRICTRC rcStrict;
+    if (!pExitRec)
+    {
+        //if (pMsg->InstructionByteCount > 0)
+        //    Log4(("InstructionByteCount=%#x %.16Rhxs\n", pMsg->InstructionByteCount, pMsg->InstructionBytes));
+        if (pExit->MemoryAccess.InstructionByteCount > 0)
+            rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pExit->VpContext.Rip,
+                                                    pExit->MemoryAccess.InstructionBytes, pExit->MemoryAccess.InstructionByteCount);
+        else
+            rcStrict = IEMExecOne(pVCpu);
+        /** @todo do we need to do anything wrt debugging here?   */
+    }
+    else
+    {
+        /* Frequent access or probing. */
+        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+        Log4(("MemExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+              pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    }
+    return rcStrict;
+}
+#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Deals with I/O port intercept message.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pMsg            The message.
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC
+nemHCWinHandleMessageIoPort(PVMCC pVM, PVMCPUCC pVCpu, HV_X64_IO_PORT_INTERCEPT_MESSAGE const *pMsg)
+{
+    /*
+     * Assert message sanity.
+     */
+    Assert(   pMsg->AccessInfo.AccessSize == 1
+           || pMsg->AccessInfo.AccessSize == 2
+           || pMsg->AccessInfo.AccessSize == 4);
+    Assert(   pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
+           || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
+    NEMWIN_ASSERT_MSG_REG_SEG(  pVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsg->Header.Rip);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRax, pMsg->Rax);
+    if (pMsg->AccessInfo.StringOp)
+    {
+        NEMWIN_ASSERT_MSG_REG_SEG(  pVCpu, HvX64RegisterDs,  pMsg->DsSegment);
+        NEMWIN_ASSERT_MSG_REG_SEG(  pVCpu, HvX64RegisterEs,  pMsg->EsSegment);
+        NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRcx, pMsg->Rcx);
+        NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRsi, pMsg->Rsi);
+        NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdi, pMsg->Rdi);
+    }
+
+    /*
+     * Whatever we do, we must clear pending event injection upon resume.
+     */
+    if (pMsg->Header.ExecutionState.InterruptionPending)
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
+
+    /*
+     * Add history first to avoid two paths doing EMHistoryExec calls.
+     */
+    VBOXSTRICTRC rcStrict;
+    PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
+                                            !pMsg->AccessInfo.StringOp
+                                            ? (  pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
+                                               ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE)
+                                               : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ))
+                                            : (  pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
+                                               ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)
+                                               : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)),
+                                            pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
+    if (!pExitRec)
+    {
+        if (!pMsg->AccessInfo.StringOp)
+        {
+            /*
+             * Simple port I/O.
+             */
+            static uint32_t const s_fAndMask[8] =
+            {   UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX,   UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX   };
+            uint32_t const        fAndMask      = s_fAndMask[pMsg->AccessInfo.AccessSize];
+
+            nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
+            if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
+            {
+                rcStrict = IOMIOPortWrite(pVM, pVCpu, pMsg->PortNumber, (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize);
+                Log4(("IOExit/%u: %04x:%08RX64/%s: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
+                      pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+                      pMsg->PortNumber, (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) ));
+                if (IOM_SUCCESS(rcStrict))
+                    nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 1);
+# ifdef IN_RING0
+                else if (   rcStrict == VINF_IOM_R3_IOPORT_WRITE
+                         && !pVCpu->cpum.GstCtx.rflags.Bits.u1TF
+                         /** @todo check for debug breakpoints */ )
+                    return EMRZSetPendingIoPortWrite(pVCpu, pMsg->PortNumber, pMsg->Header.InstructionLength,
+                                                     pMsg->AccessInfo.AccessSize, (uint32_t)pMsg->Rax & fAndMask);
+# endif
+                else
+                {
+                    pVCpu->cpum.GstCtx.rax = pMsg->Rax;
+                    pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
+                }
+            }
+            else
+            {
+                uint32_t uValue = 0;
+                rcStrict = IOMIOPortRead(pVM, pVCpu, pMsg->PortNumber, &uValue, pMsg->AccessInfo.AccessSize);
+                Log4(("IOExit/%u: %04x:%08RX64/%s: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
+                      pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+                      pMsg->PortNumber, pMsg->AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
+                if (IOM_SUCCESS(rcStrict))
+                {
+                    if (pMsg->AccessInfo.AccessSize != 4)
+                        pVCpu->cpum.GstCtx.rax = (pMsg->Rax & ~(uint64_t)fAndMask) | (uValue & fAndMask);
+                    else
+                        pVCpu->cpum.GstCtx.rax = uValue;
+                    pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
+                    Log4(("IOExit/%u: RAX %#RX64 -> %#RX64\n", pVCpu->idCpu, pMsg->Rax, pVCpu->cpum.GstCtx.rax));
+                    nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 1);
+                }
+                else
+                {
+                    pVCpu->cpum.GstCtx.rax = pMsg->Rax;
+                    pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
+# ifdef IN_RING0
+                    if (   rcStrict == VINF_IOM_R3_IOPORT_READ
+                        && !pVCpu->cpum.GstCtx.rflags.Bits.u1TF
+                        /** @todo check for debug breakpoints */ )
+                        return EMRZSetPendingIoPortRead(pVCpu, pMsg->PortNumber, pMsg->Header.InstructionLength,
+                                                        pMsg->AccessInfo.AccessSize);
+# endif
+                }
+            }
+        }
+        else
+        {
+            /*
+             * String port I/O.
+             */
+            /** @todo Someone at Microsoft please explain how we can get the address mode
+             * from the IoPortAccess.VpContext.  CS.Attributes is only sufficient for
+             * getting the default mode, it can always be overridden by a prefix.   This
+             * forces us to interpret the instruction from opcodes, which is suboptimal.
+             * Both AMD-V and VT-x includes the address size in the exit info, at least on
+             * CPUs that are reasonably new.
+             *
+             * Of course, it's possible this is an undocumented and we just need to do some
+             * experiments to figure out how it's communicated.  Alternatively, we can scan
+             * the opcode bytes for possible evil prefixes.
+             */
+            nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
+            pVCpu->cpum.GstCtx.fExtrn &= ~(  CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDI | CPUMCTX_EXTRN_RSI
+                              | CPUMCTX_EXTRN_DS  | CPUMCTX_EXTRN_ES);
+            NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pMsg->DsSegment);
+            NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pMsg->EsSegment);
+            pVCpu->cpum.GstCtx.rax = pMsg->Rax;
+            pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
+            pVCpu->cpum.GstCtx.rdi = pMsg->Rdi;
+            pVCpu->cpum.GstCtx.rsi = pMsg->Rsi;
+# ifdef IN_RING0
+            rcStrict = nemR0WinImportStateStrict(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "IOExit");
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+# else
+            int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
+            AssertRCReturn(rc, rc);
+# endif
+
+            Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s %#x LB %u (emulating)\n",
+                  pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+                  pMsg->AccessInfo.RepPrefix ? "REP " : "",
+                  pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "OUTS" : "INS",
+                  pMsg->PortNumber, pMsg->AccessInfo.AccessSize ));
+            rcStrict = IEMExecOne(pVCpu);
+        }
+        if (IOM_SUCCESS(rcStrict))
+        {
+            /*
+             * Do debug checks.
+             */
+            if (   pMsg->Header.ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflect DR7? */
+                || (pMsg->Header.Rflags & X86_EFL_TF)
+                || DBGFBpIsHwIoArmed(pVM) )
+            {
+                /** @todo Debugging. */
+            }
+        }
+        return rcStrict;
+    }
+
+    /*
+     * Frequent exit or something needing probing.
+     * Get state and call EMHistoryExec.
+     */
+    nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
+    if (!pMsg->AccessInfo.StringOp)
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
+    else
+    {
+        pVCpu->cpum.GstCtx.fExtrn &= ~(  CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDI | CPUMCTX_EXTRN_RSI
+                          | CPUMCTX_EXTRN_DS  | CPUMCTX_EXTRN_ES);
+        NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pMsg->DsSegment);
+        NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pMsg->EsSegment);
+        pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
+        pVCpu->cpum.GstCtx.rdi = pMsg->Rdi;
+        pVCpu->cpum.GstCtx.rsi = pMsg->Rsi;
+    }
+    pVCpu->cpum.GstCtx.rax = pMsg->Rax;
+
+# ifdef IN_RING0
+    rcStrict = nemR0WinImportStateStrict(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "IOExit");
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+# else
+    int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
+    AssertRCReturn(rc, rc);
+# endif
+
+    Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s%s %#x LB %u -> EMHistoryExec\n",
+          pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+          pMsg->AccessInfo.RepPrefix ? "REP " : "",
+          pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "OUT" : "IN",
+          pMsg->AccessInfo.StringOp ? "S" : "",
+          pMsg->PortNumber, pMsg->AccessInfo.AccessSize));
+    rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+    Log4(("IOExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+          pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+          VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    return rcStrict;
+}
+#elif defined(IN_RING3)
+/**
+ * Deals with I/O port access exits (WHvRunVpExitReasonX64IoPortAccess).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessageIoPort
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitIoPort(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    Assert(   pExit->IoPortAccess.AccessInfo.AccessSize == 1
+           || pExit->IoPortAccess.AccessInfo.AccessSize == 2
+           || pExit->IoPortAccess.AccessInfo.AccessSize == 4);
+
+    /*
+     * Whatever we do, we must clear pending event injection upon resume.
+     */
+    if (pExit->VpContext.ExecutionState.InterruptionPending)
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
+
+    /*
+     * Add history first to avoid two paths doing EMHistoryExec calls.
+     */
+    PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
+                                            !pExit->IoPortAccess.AccessInfo.StringOp
+                                            ? (  pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
+                                               ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE)
+                                               : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ))
+                                            : (  pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
+                                               ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)
+                                               : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)),
+                                            pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+    if (!pExitRec)
+    {
+        VBOXSTRICTRC rcStrict;
+        if (!pExit->IoPortAccess.AccessInfo.StringOp)
+        {
+            /*
+             * Simple port I/O.
+             */
+            static uint32_t const s_fAndMask[8] =
+            {   UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX,   UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX   };
+            uint32_t const        fAndMask      = s_fAndMask[pExit->IoPortAccess.AccessInfo.AccessSize];
+            if (pExit->IoPortAccess.AccessInfo.IsWrite)
+            {
+                rcStrict = IOMIOPortWrite(pVM, pVCpu, pExit->IoPortAccess.PortNumber,
+                                          (uint32_t)pExit->IoPortAccess.Rax & fAndMask,
+                                          pExit->IoPortAccess.AccessInfo.AccessSize);
+                Log4(("IOExit/%u: %04x:%08RX64/%s: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                      pExit->IoPortAccess.PortNumber, (uint32_t)pExit->IoPortAccess.Rax & fAndMask,
+                      pExit->IoPortAccess.AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) ));
+                if (IOM_SUCCESS(rcStrict))
+                {
+                    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+                    nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 1);
+                }
+            }
+            else
+            {
+                uint32_t uValue = 0;
+                rcStrict = IOMIOPortRead(pVM, pVCpu, pExit->IoPortAccess.PortNumber, &uValue,
+                                         pExit->IoPortAccess.AccessInfo.AccessSize);
+                Log4(("IOExit/%u: %04x:%08RX64/%s: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                      pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
+                if (IOM_SUCCESS(rcStrict))
+                {
+                    if (pExit->IoPortAccess.AccessInfo.AccessSize != 4)
+                        pVCpu->cpum.GstCtx.rax = (pExit->IoPortAccess.Rax & ~(uint64_t)fAndMask) | (uValue & fAndMask);
+                    else
+                        pVCpu->cpum.GstCtx.rax = uValue;
+                    pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
+                    Log4(("IOExit/%u: RAX %#RX64 -> %#RX64\n", pVCpu->idCpu, pExit->IoPortAccess.Rax, pVCpu->cpum.GstCtx.rax));
+                    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+                    nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 1);
+                }
+            }
+        }
+        else
+        {
+            /*
+             * String port I/O.
+             */
+            /** @todo Someone at Microsoft please explain how we can get the address mode
+             * from the IoPortAccess.VpContext.  CS.Attributes is only sufficient for
+             * getting the default mode, it can always be overridden by a prefix.   This
+             * forces us to interpret the instruction from opcodes, which is suboptimal.
+             * Both AMD-V and VT-x includes the address size in the exit info, at least on
+             * CPUs that are reasonably new.
+             *
+             * Of course, it's possible this is an undocumented and we just need to do some
+             * experiments to figure out how it's communicated.  Alternatively, we can scan
+             * the opcode bytes for possible evil prefixes.
+             */
+            nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+            pVCpu->cpum.GstCtx.fExtrn &= ~(  CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDI | CPUMCTX_EXTRN_RSI
+                                           | CPUMCTX_EXTRN_DS  | CPUMCTX_EXTRN_ES);
+            NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pExit->IoPortAccess.Ds);
+            NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pExit->IoPortAccess.Es);
+            pVCpu->cpum.GstCtx.rax = pExit->IoPortAccess.Rax;
+            pVCpu->cpum.GstCtx.rcx = pExit->IoPortAccess.Rcx;
+            pVCpu->cpum.GstCtx.rdi = pExit->IoPortAccess.Rdi;
+            pVCpu->cpum.GstCtx.rsi = pExit->IoPortAccess.Rsi;
+            int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
+            AssertRCReturn(rc, rc);
+
+            Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s %#x LB %u (emulating)\n",
+                  pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                  pExit->IoPortAccess.AccessInfo.RepPrefix ? "REP " : "",
+                  pExit->IoPortAccess.AccessInfo.IsWrite ? "OUTS" : "INS",
+                  pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize ));
+            rcStrict = IEMExecOne(pVCpu);
+        }
+        if (IOM_SUCCESS(rcStrict))
+        {
+            /*
+             * Do debug checks.
+             */
+            if (   pExit->VpContext.ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflect DR7? */
+                || (pExit->VpContext.Rflags & X86_EFL_TF)
+                || DBGFBpIsHwIoArmed(pVM) )
+            {
+                /** @todo Debugging. */
+            }
+        }
+        return rcStrict;
+    }
+
+    /*
+     * Frequent exit or something needing probing.
+     * Get state and call EMHistoryExec.
+     */
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    if (!pExit->IoPortAccess.AccessInfo.StringOp)
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
+    else
+    {
+        pVCpu->cpum.GstCtx.fExtrn &= ~(  CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDI | CPUMCTX_EXTRN_RSI
+                          | CPUMCTX_EXTRN_DS  | CPUMCTX_EXTRN_ES);
+        NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pExit->IoPortAccess.Ds);
+        NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pExit->IoPortAccess.Es);
+        pVCpu->cpum.GstCtx.rcx = pExit->IoPortAccess.Rcx;
+        pVCpu->cpum.GstCtx.rdi = pExit->IoPortAccess.Rdi;
+        pVCpu->cpum.GstCtx.rsi = pExit->IoPortAccess.Rsi;
+    }
+    pVCpu->cpum.GstCtx.rax = pExit->IoPortAccess.Rax;
+    int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
+    AssertRCReturn(rc, rc);
+    Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s%s %#x LB %u -> EMHistoryExec\n",
+          pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+          pExit->IoPortAccess.AccessInfo.RepPrefix ? "REP " : "",
+          pExit->IoPortAccess.AccessInfo.IsWrite ? "OUT" : "IN",
+          pExit->IoPortAccess.AccessInfo.StringOp ? "S" : "",
+          pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize));
+    VBOXSTRICTRC rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+    Log4(("IOExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+          pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+          VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    return rcStrict;
+}
+#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Deals with interrupt window message.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pMsg            The message.
+ * @sa      nemR3WinHandleExitInterruptWindow
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC
+nemHCWinHandleMessageInterruptWindow(PVMCC pVM, PVMCPUCC pVCpu, HV_X64_INTERRUPT_WINDOW_MESSAGE const *pMsg)
+{
+    /*
+     * Assert message sanity.
+     */
+    Assert(   pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE
+           || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ   // READ & WRITE are probably not used here
+           || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
+    AssertMsg(pMsg->Type == HvX64PendingInterrupt || pMsg->Type == HvX64PendingNmi, ("%#x\n", pMsg->Type));
+
+    /*
+     * Just copy the state we've got and handle it in the loop for now.
+     */
+    EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
+                     pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
+
+    nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
+    Log4(("IntWinExit/%u: %04x:%08RX64/%s: %u IF=%d InterruptShadow=%d\n",
+          pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,  nemHCWinExecStateToLogStr(&pMsg->Header),
+          pMsg->Type, RT_BOOL(pMsg->Header.Rflags & X86_EFL_IF), pMsg->Header.ExecutionState.InterruptShadow));
+
+    /** @todo call nemHCWinHandleInterruptFF   */
+    RT_NOREF(pVM);
+    return VINF_SUCCESS;
+}
+#elif defined(IN_RING3)
+/**
+ * Deals with interrupt window exits (WHvRunVpExitReasonX64InterruptWindow).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessageInterruptWindow
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitInterruptWindow(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    /*
+     * Assert message sanity.
+     */
+    AssertMsg(   pExit->InterruptWindow.DeliverableType == WHvX64PendingInterrupt
+              || pExit->InterruptWindow.DeliverableType == WHvX64PendingNmi,
+              ("%#x\n", pExit->InterruptWindow.DeliverableType));
+
+    /*
+     * Just copy the state we've got and handle it in the loop for now.
+     */
+    EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
+                     pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    Log4(("IntWinExit/%u: %04x:%08RX64/%s: %u IF=%d InterruptShadow=%d CR8=%#x\n",
+          pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,  nemR3WinExecStateToLogStr(&pExit->VpContext),
+          pExit->InterruptWindow.DeliverableType, RT_BOOL(pExit->VpContext.Rflags & X86_EFL_IF),
+          pExit->VpContext.ExecutionState.InterruptShadow, pExit->VpContext.Cr8));
+
+    /** @todo call nemHCWinHandleInterruptFF   */
+    RT_NOREF(pVM);
+    return VINF_SUCCESS;
+}
+#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Deals with CPUID intercept message.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pMsg            The message.
+ * @sa      nemR3WinHandleExitCpuId
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessageCpuId(PVMCC pVM, PVMCPUCC pVCpu, HV_X64_CPUID_INTERCEPT_MESSAGE const *pMsg)
+{
+    /* Check message register value sanity. */
+    NEMWIN_ASSERT_MSG_REG_SEG(  pVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsg->Header.Rip);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRax, pMsg->Rax);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRcx, pMsg->Rcx);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdx, pMsg->Rdx);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRbx, pMsg->Rbx);
+
+    /* Do exit history. */
+    PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_CPUID),
+                                            pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
+    if (!pExitRec)
+    {
+        /*
+         * Soak up state and execute the instruction.
+         *
+         * Note! If this grows slightly more complicated, combine into an IEMExecDecodedCpuId
+         *       function and make everyone use it.
+         */
+        /** @todo Combine implementations into IEMExecDecodedCpuId as this will
+         *        only get weirder with nested VT-x and AMD-V support. */
+        nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
+
+        /* Copy in the low register values (top is always cleared). */
+        pVCpu->cpum.GstCtx.rax = (uint32_t)pMsg->Rax;
+        pVCpu->cpum.GstCtx.rcx = (uint32_t)pMsg->Rcx;
+        pVCpu->cpum.GstCtx.rdx = (uint32_t)pMsg->Rdx;
+        pVCpu->cpum.GstCtx.rbx = (uint32_t)pMsg->Rbx;
+        pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX);
+
+        /* Get the correct values. */
+        CPUMGetGuestCpuId(pVCpu, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx,
+                          &pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
+
+        Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 -> %08RX32 / %08RX32 / %08RX32 / %08RX32 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64)\n",
+              pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+              pMsg->Rax,                           pMsg->Rcx,              pMsg->Rdx,              pMsg->Rbx,
+              pVCpu->cpum.GstCtx.eax,              pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.ebx,
+              pMsg->DefaultResultRax, pMsg->DefaultResultRcx, pMsg->DefaultResultRdx, pMsg->DefaultResultRbx));
+
+        /* Move RIP and we're done. */
+        nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 2);
+
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Frequent exit or something needing probing.
+     * Get state and call EMHistoryExec.
+     */
+    nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
+    pVCpu->cpum.GstCtx.rax = pMsg->Rax;
+    pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
+    pVCpu->cpum.GstCtx.rdx = pMsg->Rdx;
+    pVCpu->cpum.GstCtx.rbx = pMsg->Rbx;
+    pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX);
+    Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64) ==> EMHistoryExec\n",
+          pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+          pMsg->Rax,                           pMsg->Rcx,              pMsg->Rdx,              pMsg->Rbx,
+          pMsg->DefaultResultRax, pMsg->DefaultResultRcx, pMsg->DefaultResultRdx, pMsg->DefaultResultRbx));
+# ifdef IN_RING0
+    VBOXSTRICTRC rcStrict = nemR0WinImportStateStrict(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "CpuIdExit");
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    RT_NOREF(pVM);
+# else
+    int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
+    AssertRCReturn(rc, rc);
+# endif
+    VBOXSTRICTRC rcStrictExec = EMHistoryExec(pVCpu, pExitRec, 0);
+    Log4(("CpuIdExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+          pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+          VBOXSTRICTRC_VAL(rcStrictExec), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    return rcStrictExec;
+}
+#elif defined(IN_RING3)
+/**
+ * Deals with CPUID exits (WHvRunVpExitReasonX64Cpuid).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessageCpuId
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC
+nemR3WinHandleExitCpuId(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_CPUID),
+                                            pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+    if (!pExitRec)
+    {
+        /*
+         * Soak up state and execute the instruction.
+         *
+         * Note! If this grows slightly more complicated, combine into an IEMExecDecodedCpuId
+         *       function and make everyone use it.
+         */
+        /** @todo Combine implementations into IEMExecDecodedCpuId as this will
+         *        only get weirder with nested VT-x and AMD-V support. */
+        nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+
+        /* Copy in the low register values (top is always cleared). */
+        pVCpu->cpum.GstCtx.rax = (uint32_t)pExit->CpuidAccess.Rax;
+        pVCpu->cpum.GstCtx.rcx = (uint32_t)pExit->CpuidAccess.Rcx;
+        pVCpu->cpum.GstCtx.rdx = (uint32_t)pExit->CpuidAccess.Rdx;
+        pVCpu->cpum.GstCtx.rbx = (uint32_t)pExit->CpuidAccess.Rbx;
+        pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX);
+
+        /* Get the correct values. */
+        CPUMGetGuestCpuId(pVCpu, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx,
+                          &pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
+
+        Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 -> %08RX32 / %08RX32 / %08RX32 / %08RX32 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64)\n",
+              pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+              pExit->CpuidAccess.Rax,                           pExit->CpuidAccess.Rcx,              pExit->CpuidAccess.Rdx,              pExit->CpuidAccess.Rbx,
+              pVCpu->cpum.GstCtx.eax,                           pVCpu->cpum.GstCtx.ecx,              pVCpu->cpum.GstCtx.edx,              pVCpu->cpum.GstCtx.ebx,
+              pExit->CpuidAccess.DefaultResultRax, pExit->CpuidAccess.DefaultResultRcx, pExit->CpuidAccess.DefaultResultRdx, pExit->CpuidAccess.DefaultResultRbx));
+
+        /* Move RIP and we're done. */
+        nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
+
+        RT_NOREF_PV(pVM);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Frequent exit or something needing probing.
+     * Get state and call EMHistoryExec.
+     */
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    pVCpu->cpum.GstCtx.rax = pExit->CpuidAccess.Rax;
+    pVCpu->cpum.GstCtx.rcx = pExit->CpuidAccess.Rcx;
+    pVCpu->cpum.GstCtx.rdx = pExit->CpuidAccess.Rdx;
+    pVCpu->cpum.GstCtx.rbx = pExit->CpuidAccess.Rbx;
+    pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX);
+    Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64) ==> EMHistoryExec\n",
+          pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+          pExit->CpuidAccess.Rax,                           pExit->CpuidAccess.Rcx,              pExit->CpuidAccess.Rdx,              pExit->CpuidAccess.Rbx,
+          pExit->CpuidAccess.DefaultResultRax, pExit->CpuidAccess.DefaultResultRcx, pExit->CpuidAccess.DefaultResultRdx, pExit->CpuidAccess.DefaultResultRbx));
+    int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
+    AssertRCReturn(rc, rc);
+    VBOXSTRICTRC rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+    Log4(("CpuIdExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+          pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+          VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    return rcStrict;
+}
+#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Deals with MSR intercept message.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pMsg            The message.
+ * @sa      nemR3WinHandleExitMsr
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessageMsr(PVMCPUCC pVCpu, HV_X64_MSR_INTERCEPT_MESSAGE const *pMsg)
+{
+    /*
+     * A wee bit of sanity first.
+     */
+    Assert(   pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
+           || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
+    NEMWIN_ASSERT_MSG_REG_SEG(  pVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsg->Header.Rip);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRax, pMsg->Rax);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdx, pMsg->Rdx);
+
+    /*
+     * Check CPL as that's common to both RDMSR and WRMSR.
+     */
+    VBOXSTRICTRC rcStrict;
+    if (pMsg->Header.ExecutionState.Cpl == 0)
+    {
+        /*
+         * Get all the MSR state.  Since we're getting EFER, we also need to
+         * get CR0, CR4 and CR3.
+         */
+        PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
+                                                  pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
+                                                ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE)
+                                                : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ),
+                                                pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
+
+        nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
+        rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu,
+                                                     (!pExitRec ? 0 : IEM_CPUMCTX_EXTRN_MUST_MASK)
+                                                     | CPUMCTX_EXTRN_ALL_MSRS | CPUMCTX_EXTRN_CR0
+                                                     | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4,
+                                                     "MSRs");
+        if (rcStrict == VINF_SUCCESS)
+        {
+            if (!pExitRec)
+            {
+                /*
+                 * Handle writes.
+                 */
+                if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
+                {
+                    rcStrict = CPUMSetGuestMsr(pVCpu, pMsg->MsrNumber, RT_MAKE_U64((uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx));
+                    Log4(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc\n",
+                          pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+                          pMsg->MsrNumber, (uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
+                    if (rcStrict == VINF_SUCCESS)
+                    {
+                        nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 2);
+                        return VINF_SUCCESS;
+                    }
+# ifndef IN_RING3
+                    /* move to ring-3 and handle the trap/whatever there, as we want to LogRel this. */
+                    if (rcStrict == VERR_CPUM_RAISE_GP_0)
+                        rcStrict = VINF_CPUM_R3_MSR_WRITE;
+                    return rcStrict;
+# else
+                    LogRel(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc!\n",
+                            pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+                            pMsg->MsrNumber, (uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
+# endif
+                }
+                /*
+                 * Handle reads.
+                 */
+                else
+                {
+                    uint64_t uValue = 0;
+                    rcStrict = CPUMQueryGuestMsr(pVCpu, pMsg->MsrNumber, &uValue);
+                    Log4(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n",
+                          pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+                          pMsg->MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
+                    if (rcStrict == VINF_SUCCESS)
+                    {
+                        pVCpu->cpum.GstCtx.rax = (uint32_t)uValue;
+                        pVCpu->cpum.GstCtx.rdx = uValue >> 32;
+                        pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX);
+                        nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 2);
+                        return VINF_SUCCESS;
+                    }
+# ifndef IN_RING3
+                    /* move to ring-3 and handle the trap/whatever there, as we want to LogRel this. */
+                    if (rcStrict == VERR_CPUM_RAISE_GP_0)
+                        rcStrict = VINF_CPUM_R3_MSR_READ;
+                    return rcStrict;
+# else
+                    LogRel(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n",
+                            pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+                            pMsg->MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
+# endif
+                }
+            }
+            else
+            {
+                /*
+                 * Handle frequent exit or something needing probing.
+                 */
+                Log4(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %#08x\n",
+                      pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+                      pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "WR" : "RD", pMsg->MsrNumber));
+                rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+                Log4(("MsrExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+                      pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+                      VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+                return rcStrict;
+            }
+        }
+        else
+        {
+            LogRel(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %08x -> %Rrc - msr state import\n",
+                    pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+                    pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "WR" : "RD",
+                    pMsg->MsrNumber, VBOXSTRICTRC_VAL(rcStrict) ));
+            return rcStrict;
+        }
+    }
+    else if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
+        Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); WRMSR %08x, %08x:%08x\n",
+              pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+              pMsg->Header.ExecutionState.Cpl, pMsg->MsrNumber, (uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx ));
+    else
+        Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); RDMSR %08x\n",
+              pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+              pMsg->Header.ExecutionState.Cpl, pMsg->MsrNumber));
+
+    /*
+     * If we get down here, we're supposed to #GP(0).
+     */
+    rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_ALL_MSRS, "MSR");
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_GP, TRPM_TRAP, 0, 0, 0);
+        if (rcStrict == VINF_IEM_RAISED_XCPT)
+            rcStrict = VINF_SUCCESS;
+        else if (rcStrict != VINF_SUCCESS)
+            Log4(("MsrExit/%u: Injecting #GP(0) failed: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
+    }
+    return rcStrict;
+}
+#elif defined(IN_RING3)
+/**
+ * Deals with MSR access exits (WHvRunVpExitReasonX64MsrAccess).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessageMsr
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitMsr(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    /*
+     * Check CPL as that's common to both RDMSR and WRMSR.
+     */
+    VBOXSTRICTRC rcStrict;
+    if (pExit->VpContext.ExecutionState.Cpl == 0)
+    {
+        /*
+         * Get all the MSR state.  Since we're getting EFER, we also need to
+         * get CR0, CR4 and CR3.
+         */
+        PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
+                                                  pExit->MsrAccess.AccessInfo.IsWrite
+                                                ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE)
+                                                : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ),
+                                                pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+        nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+        rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu,
+                                                     (!pExitRec ? 0 : IEM_CPUMCTX_EXTRN_MUST_MASK)
+                                                     | CPUMCTX_EXTRN_ALL_MSRS | CPUMCTX_EXTRN_CR0
+                                                     | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4,
+                                                     "MSRs");
+        if (rcStrict == VINF_SUCCESS)
+        {
+            if (!pExitRec)
+            {
+                /*
+                 * Handle writes.
+                 */
+                if (pExit->MsrAccess.AccessInfo.IsWrite)
+                {
+                    rcStrict = CPUMSetGuestMsr(pVCpu, pExit->MsrAccess.MsrNumber,
+                                               RT_MAKE_U64((uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx));
+                    Log4(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+                          pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->MsrAccess.MsrNumber,
+                          (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
+                    if (rcStrict == VINF_SUCCESS)
+                    {
+                        nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
+                        return VINF_SUCCESS;
+                    }
+                    LogRel(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc!\n", pVCpu->idCpu,
+                            pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                            pExit->MsrAccess.MsrNumber, (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx,
+                            VBOXSTRICTRC_VAL(rcStrict) ));
+                }
+                /*
+                 * Handle reads.
+                 */
+                else
+                {
+                    uint64_t uValue = 0;
+                    rcStrict = CPUMQueryGuestMsr(pVCpu, pExit->MsrAccess.MsrNumber, &uValue);
+                    Log4(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n", pVCpu->idCpu,
+                          pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                          pExit->MsrAccess.MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
+                    if (rcStrict == VINF_SUCCESS)
+                    {
+                        pVCpu->cpum.GstCtx.rax = (uint32_t)uValue;
+                        pVCpu->cpum.GstCtx.rdx = uValue >> 32;
+                        pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX);
+                        nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
+                        return VINF_SUCCESS;
+                    }
+                    LogRel(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+                            pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->MsrAccess.MsrNumber,
+                            uValue, VBOXSTRICTRC_VAL(rcStrict) ));
+                }
+            }
+            else
+            {
+                /*
+                 * Handle frequent exit or something needing probing.
+                 */
+                Log4(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %#08x\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                      pExit->MsrAccess.AccessInfo.IsWrite ? "WR" : "RD", pExit->MsrAccess.MsrNumber));
+                rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+                Log4(("MsrExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                      VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+                return rcStrict;
+            }
+        }
+        else
+        {
+            LogRel(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %08x -> %Rrc - msr state import\n",
+                    pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                    pExit->MsrAccess.AccessInfo.IsWrite ? "WR" : "RD", pExit->MsrAccess.MsrNumber, VBOXSTRICTRC_VAL(rcStrict) ));
+            return rcStrict;
+        }
+    }
+    else if (pExit->MsrAccess.AccessInfo.IsWrite)
+        Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); WRMSR %08x, %08x:%08x\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+              pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.ExecutionState.Cpl,
+              pExit->MsrAccess.MsrNumber, (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx ));
+    else
+        Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); RDMSR %08x\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+              pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.ExecutionState.Cpl,
+              pExit->MsrAccess.MsrNumber));
+
+    /*
+     * If we get down here, we're supposed to #GP(0).
+     */
+    rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_ALL_MSRS, "MSR");
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_GP, TRPM_TRAP, 0, 0, 0);
+        if (rcStrict == VINF_IEM_RAISED_XCPT)
+            rcStrict = VINF_SUCCESS;
+        else if (rcStrict != VINF_SUCCESS)
+            Log4(("MsrExit/%u: Injecting #GP(0) failed: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
+    }
+
+    RT_NOREF_PV(pVM);
+    return rcStrict;
+}
+#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+
+/**
+ * Worker for nemHCWinHandleMessageException & nemR3WinHandleExitException that
+ * checks if the given opcodes are of interest at all.
+ *
+ * @returns true if interesting, false if not.
+ * @param   cbOpcodes           Number of opcode bytes available.
+ * @param   pbOpcodes           The opcode bytes.
+ * @param   f64BitMode          Whether we're in 64-bit mode.
+ */
+DECLINLINE(bool) nemHcWinIsInterestingUndefinedOpcode(uint8_t cbOpcodes, uint8_t const *pbOpcodes, bool f64BitMode)
+{
+    /*
+     * Currently only interested in VMCALL and VMMCALL.
+     */
+    while (cbOpcodes >= 3)
+    {
+        switch (pbOpcodes[0])
+        {
+            case 0x0f:
+                switch (pbOpcodes[1])
+                {
+                    case 0x01:
+                        switch (pbOpcodes[2])
+                        {
+                            case 0xc1: /* 0f 01 c1  VMCALL */
+                                return true;
+                            case 0xd9: /* 0f 01 d9  VMMCALL */
+                                return true;
+                            default:
+                                break;
+                        }
+                        break;
+                }
+                break;
+
+            default:
+                return false;
+
+            /* prefixes */
+            case 0x40: case 0x41: case 0x42: case 0x43: case 0x44: case 0x45: case 0x46: case 0x47:
+            case 0x48: case 0x49: case 0x4a: case 0x4b: case 0x4c: case 0x4d: case 0x4e: case 0x4f:
+                if (!f64BitMode)
+                    return false;
+                RT_FALL_THRU();
+            case X86_OP_PRF_CS:
+            case X86_OP_PRF_SS:
+            case X86_OP_PRF_DS:
+            case X86_OP_PRF_ES:
+            case X86_OP_PRF_FS:
+            case X86_OP_PRF_GS:
+            case X86_OP_PRF_SIZE_OP:
+            case X86_OP_PRF_SIZE_ADDR:
+            case X86_OP_PRF_LOCK:
+            case X86_OP_PRF_REPZ:
+            case X86_OP_PRF_REPNZ:
+                cbOpcodes--;
+                pbOpcodes++;
+                continue;
+        }
+        break;
+    }
+    return false;
+}
+
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Copies state included in a exception intercept message.
+ *
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pMsg            The message.
+ * @param   fClearXcpt      Clear pending exception.
+ */
+DECLINLINE(void)
+nemHCWinCopyStateFromExceptionMessage(PVMCPUCC pVCpu, HV_X64_EXCEPTION_INTERCEPT_MESSAGE const *pMsg, bool fClearXcpt)
+{
+    nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
+    pVCpu->cpum.GstCtx.fExtrn &= ~(  CPUMCTX_EXTRN_GPRS_MASK | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_DS
+                      | (fClearXcpt ? CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT : 0) );
+    pVCpu->cpum.GstCtx.rax = pMsg->Rax;
+    pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
+    pVCpu->cpum.GstCtx.rdx = pMsg->Rdx;
+    pVCpu->cpum.GstCtx.rbx = pMsg->Rbx;
+    pVCpu->cpum.GstCtx.rsp = pMsg->Rsp;
+    pVCpu->cpum.GstCtx.rbp = pMsg->Rbp;
+    pVCpu->cpum.GstCtx.rsi = pMsg->Rsi;
+    pVCpu->cpum.GstCtx.rdi = pMsg->Rdi;
+    pVCpu->cpum.GstCtx.r8  = pMsg->R8;
+    pVCpu->cpum.GstCtx.r9  = pMsg->R9;
+    pVCpu->cpum.GstCtx.r10 = pMsg->R10;
+    pVCpu->cpum.GstCtx.r11 = pMsg->R11;
+    pVCpu->cpum.GstCtx.r12 = pMsg->R12;
+    pVCpu->cpum.GstCtx.r13 = pMsg->R13;
+    pVCpu->cpum.GstCtx.r14 = pMsg->R14;
+    pVCpu->cpum.GstCtx.r15 = pMsg->R15;
+    NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pMsg->DsSegment);
+    NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ss, pMsg->SsSegment);
+}
+#elif defined(IN_RING3)
+/**
+ * Copies state included in a exception intercept exit.
+ *
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information.
+ * @param   fClearXcpt      Clear pending exception.
+ */
+DECLINLINE(void) nemR3WinCopyStateFromExceptionMessage(PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit, bool fClearXcpt)
+{
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    if (fClearXcpt)
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
+}
+#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+
+/**
+ * Advances the guest RIP by the number of bytes specified in @a cb.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cb          RIP increment value in bytes.
+ */
+DECLINLINE(void) nemHcWinAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
+{
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    pCtx->rip += cb;
+
+    /* Update interrupt shadow. */
+    if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+        && pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+}
+
+
+/**
+ * Hacks its way around the lovely mesa driver's backdoor accesses.
+ *
+ * @sa hmR0VmxHandleMesaDrvGp
+ * @sa hmR0SvmHandleMesaDrvGp
+ */
+static int nemHcWinHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+    Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_GPRS_MASK)));
+    RT_NOREF(pCtx);
+
+    /* For now we'll just skip the instruction. */
+    nemHcWinAdvanceRip(pVCpu, 1);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
+ * backdoor logging w/o checking what it is running inside.
+ *
+ * This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
+ * backdoor port and magic numbers loaded in registers.
+ *
+ * @returns true if it is, false if it isn't.
+ * @sa      hmR0VmxIsMesaDrvGp
+ * @sa      hmR0SvmIsMesaDrvGp
+ */
+DECLINLINE(bool) nemHcWinIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, const uint8_t *pbInsn, uint32_t cbInsn)
+{
+    /* #GP(0) is already checked by caller. */
+
+    /* Check magic and port. */
+    Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RAX)));
+    if (pCtx->dx != UINT32_C(0x5658))
+        return false;
+    if (pCtx->rax != UINT32_C(0x564d5868))
+        return false;
+
+    /* Flat ring-3 CS. */
+    if (CPUMGetGuestCPL(pVCpu) != 3)
+        return false;
+    if (pCtx->cs.u64Base != 0)
+        return false;
+
+    /* 0xed:  IN eAX,dx */
+    if (cbInsn < 1) /* Play safe (shouldn't happen). */
+    {
+        uint8_t abInstr[1];
+        int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
+        if (RT_FAILURE(rc))
+            return false;
+        if (abInstr[0] != 0xed)
+            return false;
+    }
+    else
+    {
+        if (pbInsn[0] != 0xed)
+            return false;
+    }
+
+    return true;
+}
+
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Deals with exception intercept message (HvMessageTypeX64ExceptionIntercept).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pMsg            The message.
+ * @sa      nemR3WinHandleExitMsr
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC
+nemHCWinHandleMessageException(PVMCPUCC pVCpu, HV_X64_EXCEPTION_INTERCEPT_MESSAGE const *pMsg)
+{
+    /*
+     * Assert sanity.
+     */
+    Assert(   pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
+           || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
+           || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE);
+    NEMWIN_ASSERT_MSG_REG_SEG(  pVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsg->Header.Rip);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
+    NEMWIN_ASSERT_MSG_REG_SEG(  pVCpu, HvX64RegisterDs,  pMsg->DsSegment);
+    NEMWIN_ASSERT_MSG_REG_SEG(  pVCpu, HvX64RegisterSs,  pMsg->SsSegment);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRax, pMsg->Rax);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRcx, pMsg->Rcx);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdx, pMsg->Rdx);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRbx, pMsg->Rbx);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRsp, pMsg->Rsp);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRbp, pMsg->Rbp);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRsi, pMsg->Rsi);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdi, pMsg->Rdi);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR8,  pMsg->R8);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR9,  pMsg->R9);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR10, pMsg->R10);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR11, pMsg->R11);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR12, pMsg->R12);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR13, pMsg->R13);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR14, pMsg->R14);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR15, pMsg->R15);
+
+    /*
+     * Get most of the register state since we'll end up making IEM inject the
+     * event.  The exception isn't normally flaged as a pending event, so duh.
+     *
+     * Note! We can optimize this later with event injection.
+     */
+    Log4(("XcptExit/%u: %04x:%08RX64/%s: %x errcd=%#x parm=%RX64\n",
+          pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
+          pMsg->ExceptionVector, pMsg->ErrorCode, pMsg->ExceptionParameter));
+    nemHCWinCopyStateFromExceptionMessage(pVCpu, pMsg, true /*fClearXcpt*/);
+    uint64_t fWhat = NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM;
+    if (pMsg->ExceptionVector == X86_XCPT_DB)
+        fWhat |= CPUMCTX_EXTRN_DR0_DR3 | CPUMCTX_EXTRN_DR7 | CPUMCTX_EXTRN_DR6;
+    VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, fWhat, "Xcpt");
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Handle the intercept.
+     */
+    TRPMEVENT enmEvtType = TRPM_TRAP;
+    switch (pMsg->ExceptionVector)
+    {
+        /*
+         * We get undefined opcodes on VMMCALL(AMD) & VMCALL(Intel) instructions
+         * and need to turn them over to GIM.
+         *
+         * Note! We do not check fGIMTrapXcptUD here ASSUMING that GIM only wants
+         *       #UD for handling non-native hypercall instructions.  (IEM will
+         *       decode both and let the GIM provider decide whether to accept it.)
+         */
+        case X86_XCPT_UD:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUd);
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_UD),
+                             pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
+
+            if (nemHcWinIsInterestingUndefinedOpcode(pMsg->InstructionByteCount, pMsg->InstructionBytes,
+                                                     pMsg->Header.ExecutionState.EferLma && pMsg->Header.CsSegment.Long ))
+            {
+                rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pMsg->Header.Rip,
+                                                        pMsg->InstructionBytes, pMsg->InstructionByteCount);
+                Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD -> emulated -> %Rrc\n",
+                      pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
+                      nemHCWinExecStateToLogStr(&pMsg->Header), VBOXSTRICTRC_VAL(rcStrict) ));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled);
+                return rcStrict;
+            }
+            Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD [%.*Rhxs] -> re-injected\n", pVCpu->idCpu, pMsg->Header.CsSegment.Selector,
+                  pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),  pMsg->InstructionByteCount, pMsg->InstructionBytes ));
+            break;
+
+        /*
+         * Workaround the lovely mesa driver assuming that vmsvga means vmware
+         * hypervisor and tries to log stuff to the host.
+         */
+        case X86_XCPT_GP:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionGp);
+            /** @todo r=bird: Need workaround in IEM for this, right?
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_GP),
+                             pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC()); */
+            if (   !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
+                || !nemHcWinIsMesaDrvGp(pVCpu, &pVCpu->cpum.GstCtx, pMsg->InstructionBytes, pMsg->InstructionByteCount))
+            {
+# if 1 /** @todo Need to emulate instruction or we get a triple fault when trying to inject the #GP... */
+                rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pMsg->Header.Rip,
+                                                        pMsg->InstructionBytes, pMsg->InstructionByteCount);
+                Log4(("XcptExit/%u: %04x:%08RX64/%s: #GP -> emulated -> %Rrc\n",
+                      pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
+                      nemHCWinExecStateToLogStr(&pMsg->Header), VBOXSTRICTRC_VAL(rcStrict) ));
+                return rcStrict;
+# else
+                break;
+# endif
+            }
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionGpMesa);
+            return nemHcWinHandleMesaDrvGp(pVCpu, &pVCpu->cpum.GstCtx);
+
+        /*
+         * Filter debug exceptions.
+         */
+        case X86_XCPT_DB:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionDb);
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_DB),
+                             pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
+            Log4(("XcptExit/%u: %04x:%08RX64/%s: #DB - TODO\n",
+                  pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header) ));
+            break;
+
+        case X86_XCPT_BP:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionBp);
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_BP),
+                             pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
+            Log4(("XcptExit/%u: %04x:%08RX64/%s: #BP - TODO - %u\n", pVCpu->idCpu, pMsg->Header.CsSegment.Selector,
+                  pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header), pMsg->Header.InstructionLength));
+            enmEvtType = TRPM_SOFTWARE_INT; /* We're at the INT3 instruction, not after it. */
+            break;
+
+        /* This shouldn't happen. */
+        default:
+            AssertLogRelMsgFailedReturn(("ExceptionVector=%#x\n", pMsg->ExceptionVector),  VERR_IEM_IPE_6);
+    }
+
+    /*
+     * Inject it.
+     */
+    rcStrict = IEMInjectTrap(pVCpu, pMsg->ExceptionVector, enmEvtType, pMsg->ErrorCode,
+                             pMsg->ExceptionParameter /*??*/, pMsg->Header.InstructionLength);
+    Log4(("XcptExit/%u: %04x:%08RX64/%s: %#u -> injected -> %Rrc\n",
+          pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
+          nemHCWinExecStateToLogStr(&pMsg->Header), pMsg->ExceptionVector, VBOXSTRICTRC_VAL(rcStrict) ));
+    return rcStrict;
+}
+#elif defined(IN_RING3)
+/**
+ * Deals with MSR access exits (WHvRunVpExitReasonException).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemR3WinHandleExitException
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitException(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    /*
+     * Get most of the register state since we'll end up making IEM inject the
+     * event.  The exception isn't normally flaged as a pending event, so duh.
+     *
+     * Note! We can optimize this later with event injection.
+     */
+    Log4(("XcptExit/%u: %04x:%08RX64/%s: %x errcd=%#x parm=%RX64\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+          pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpException.ExceptionType,
+          pExit->VpException.ErrorCode, pExit->VpException.ExceptionParameter ));
+    nemR3WinCopyStateFromExceptionMessage(pVCpu, pExit, true /*fClearXcpt*/);
+    uint64_t fWhat = NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM;
+    if (pExit->VpException.ExceptionType == X86_XCPT_DB)
+        fWhat |= CPUMCTX_EXTRN_DR0_DR3 | CPUMCTX_EXTRN_DR7 | CPUMCTX_EXTRN_DR6;
+    VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, fWhat, "Xcpt");
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Handle the intercept.
+     */
+    TRPMEVENT enmEvtType = TRPM_TRAP;
+    switch (pExit->VpException.ExceptionType)
+    {
+        /*
+         * We get undefined opcodes on VMMCALL(AMD) & VMCALL(Intel) instructions
+         * and need to turn them over to GIM.
+         *
+         * Note! We do not check fGIMTrapXcptUD here ASSUMING that GIM only wants
+         *       #UD for handling non-native hypercall instructions.  (IEM will
+         *       decode both and let the GIM provider decide whether to accept it.)
+         */
+        case X86_XCPT_UD:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUd);
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_UD),
+                             pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+            if (nemHcWinIsInterestingUndefinedOpcode(pExit->VpException.InstructionByteCount, pExit->VpException.InstructionBytes,
+                                                     pExit->VpContext.ExecutionState.EferLma && pExit->VpContext.Cs.Long ))
+            {
+                rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pExit->VpContext.Rip,
+                                                        pExit->VpException.InstructionBytes,
+                                                        pExit->VpException.InstructionByteCount);
+                Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD -> emulated -> %Rrc\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
+                      nemR3WinExecStateToLogStr(&pExit->VpContext), VBOXSTRICTRC_VAL(rcStrict) ));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled);
+                return rcStrict;
+            }
+
+            Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD [%.*Rhxs] -> re-injected\n", pVCpu->idCpu,
+                  pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                  pExit->VpException.InstructionByteCount, pExit->VpException.InstructionBytes ));
+            break;
+
+        /*
+         * Workaround the lovely mesa driver assuming that vmsvga means vmware
+         * hypervisor and tries to log stuff to the host.
+         */
+        case X86_XCPT_GP:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionGp);
+            /** @todo r=bird: Need workaround in IEM for this, right?
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_GP),
+                             pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); */
+            if (   !pVCpu->nem.s.fTrapXcptGpForLovelyMesaDrv
+                || !nemHcWinIsMesaDrvGp(pVCpu, &pVCpu->cpum.GstCtx, pExit->VpException.InstructionBytes,
+                                        pExit->VpException.InstructionByteCount))
+            {
+# if 1 /** @todo Need to emulate instruction or we get a triple fault when trying to inject the #GP... */
+                rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pExit->VpContext.Rip,
+                                                        pExit->VpException.InstructionBytes,
+                                                        pExit->VpException.InstructionByteCount);
+                Log4(("XcptExit/%u: %04x:%08RX64/%s: #GP -> emulated -> %Rrc\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
+                      nemR3WinExecStateToLogStr(&pExit->VpContext), VBOXSTRICTRC_VAL(rcStrict) ));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled);
+                return rcStrict;
+# else
+                break;
+# endif
+            }
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionGpMesa);
+            return nemHcWinHandleMesaDrvGp(pVCpu, &pVCpu->cpum.GstCtx);
+
+        /*
+         * Filter debug exceptions.
+         */
+        case X86_XCPT_DB:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionDb);
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_DB),
+                             pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+            Log4(("XcptExit/%u: %04x:%08RX64/%s: #DB - TODO\n",
+                  pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext) ));
+            break;
+
+        case X86_XCPT_BP:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionBp);
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_BP),
+                             pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+            Log4(("XcptExit/%u: %04x:%08RX64/%s: #BP - TODO - %u\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+                  pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.InstructionLength));
+            enmEvtType = TRPM_SOFTWARE_INT; /* We're at the INT3 instruction, not after it. */
+            break;
+
+        /* This shouldn't happen. */
+        default:
+            AssertLogRelMsgFailedReturn(("ExceptionType=%#x\n", pExit->VpException.ExceptionType),  VERR_IEM_IPE_6);
+    }
+
+    /*
+     * Inject it.
+     */
+    rcStrict = IEMInjectTrap(pVCpu, pExit->VpException.ExceptionType, enmEvtType, pExit->VpException.ErrorCode,
+                             pExit->VpException.ExceptionParameter /*??*/, pExit->VpContext.InstructionLength);
+    Log4(("XcptExit/%u: %04x:%08RX64/%s: %#u -> injected -> %Rrc\n",
+          pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
+          nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpException.ExceptionType, VBOXSTRICTRC_VAL(rcStrict) ));
+
+    RT_NOREF_PV(pVM);
+    return rcStrict;
+}
+#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Deals with unrecoverable exception (triple fault).
+ *
+ * Seen WRMSR 0x201 (IA32_MTRR_PHYSMASK0) writes from grub / debian9 ending up
+ * here too.  So we'll leave it to IEM to decide.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pMsgHdr         The message header.
+ * @sa      nemR3WinHandleExitUnrecoverableException
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC
+nemHCWinHandleMessageUnrecoverableException(PVMCPUCC pVCpu, HV_X64_INTERCEPT_MESSAGE_HEADER const *pMsgHdr)
+{
+    /* Check message register value sanity. */
+    NEMWIN_ASSERT_MSG_REG_SEG(  pVCpu, HvX64RegisterCs, pMsgHdr->CsSegment);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsgHdr->Rip);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsgHdr->Rflags);
+    NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsgHdr->Cr8);
+
+# if 0
+    /*
+     * Just copy the state we've got and handle it in the loop for now.
+     */
+    nemHCWinCopyStateFromX64Header(pVCpu, pMsgHdr);
+    Log(("TripleExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT\n",
+         pVCpu->idCpu, pMsgHdr->CsSegment.Selector, pMsgHdr->Rip, nemHCWinExecStateToLogStr(&pMsg->Header), pMsgHdr->Rflags));
+    return VINF_EM_TRIPLE_FAULT;
+# else
+    /*
+     * Let IEM decide whether this is really it.
+     */
+    EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_UNRECOVERABLE_EXCEPTION),
+                     pMsgHdr->Rip + pMsgHdr->CsSegment.Base, ASMReadTSC());
+    nemHCWinCopyStateFromX64Header(pVCpu, pMsgHdr);
+    VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_ALL, "TripleExit");
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = IEMExecOne(pVCpu);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_SUCCESS\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
+                 pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags ));
+            pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; /* Make sure to reset pending #DB(0). */
+            return VINF_SUCCESS;
+        }
+        if (rcStrict == VINF_EM_TRIPLE_FAULT)
+            Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT!\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
+                 pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
+        else
+            Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (IEMExecOne)\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
+                 pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
+    }
+    else
+        Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (state import)\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
+             pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
+    return rcStrict;
+# endif
+}
+#elif defined(IN_RING3)
+/**
+ * Deals with MSR access exits (WHvRunVpExitReasonUnrecoverableException).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessageUnrecoverableException
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitUnrecoverableException(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+# if 0
+    /*
+     * Just copy the state we've got and handle it in the loop for now.
+     */
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    Log(("TripleExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+         pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags));
+    RT_NOREF_PV(pVM);
+    return VINF_EM_TRIPLE_FAULT;
+# else
+    /*
+     * Let IEM decide whether this is really it.
+     */
+    EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_UNRECOVERABLE_EXCEPTION),
+                     pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_ALL, "TripleExit");
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = IEMExecOne(pVCpu);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_SUCCESS\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+                 pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags));
+            pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; /* Make sure to reset pending #DB(0). */
+            return VINF_SUCCESS;
+        }
+        if (rcStrict == VINF_EM_TRIPLE_FAULT)
+            Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT!\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+                 pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
+        else
+            Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (IEMExecOne)\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+                 pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
+    }
+    else
+        Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (state import)\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+             pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
+    RT_NOREF_PV(pVM);
+    return rcStrict;
+# endif
+
+}
+#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Handles messages (VM exits).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pMappingHeader  The message slot mapping.
+ * @sa      nemR3WinHandleExit
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC
+nemHCWinHandleMessage(PVMCC pVM, PVMCPUCC pVCpu, VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader)
+{
+    if (pMappingHeader->enmVidMsgType == VidMessageHypervisorMessage)
+    {
+        AssertMsg(pMappingHeader->cbMessage == HV_MESSAGE_SIZE, ("%#x\n", pMappingHeader->cbMessage));
+        HV_MESSAGE const *pMsg = (HV_MESSAGE const *)(pMappingHeader + 1);
+        switch (pMsg->Header.MessageType)
+        {
+            case HvMessageTypeUnmappedGpa:
+                Assert(pMsg->Header.PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemUnmapped);
+                return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept);
+
+            case HvMessageTypeGpaIntercept:
+                Assert(pMsg->Header.PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemIntercept);
+                return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept);
+
+            case HvMessageTypeX64IoPortIntercept:
+                Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64IoPortIntercept));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitPortIo);
+                return nemHCWinHandleMessageIoPort(pVM, pVCpu, &pMsg->X64IoPortIntercept);
+
+            case HvMessageTypeX64Halt:
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
+                EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
+                                 pMsg->X64InterceptHeader.Rip + pMsg->X64InterceptHeader.CsSegment.Base, ASMReadTSC());
+                Log4(("HaltExit\n"));
+                return VINF_EM_HALT;
+
+            case HvMessageTypeX64InterruptWindow:
+                Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64InterruptWindow));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInterruptWindow);
+                return nemHCWinHandleMessageInterruptWindow(pVM, pVCpu, &pMsg->X64InterruptWindow);
+
+            case HvMessageTypeX64CpuidIntercept:
+                Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64CpuIdIntercept));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitCpuId);
+                return nemHCWinHandleMessageCpuId(pVM, pVCpu, &pMsg->X64CpuIdIntercept);
+
+            case HvMessageTypeX64MsrIntercept:
+                Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64MsrIntercept));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMsr);
+                return nemHCWinHandleMessageMsr(pVCpu, &pMsg->X64MsrIntercept);
+
+            case HvMessageTypeX64ExceptionIntercept:
+                Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64ExceptionIntercept));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitException);
+                return nemHCWinHandleMessageException(pVCpu, &pMsg->X64ExceptionIntercept);
+
+            case HvMessageTypeUnrecoverableException:
+                Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64InterceptHeader));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitUnrecoverable);
+                return nemHCWinHandleMessageUnrecoverableException(pVCpu, &pMsg->X64InterceptHeader);
+
+            case HvMessageTypeInvalidVpRegisterValue:
+            case HvMessageTypeUnsupportedFeature:
+            case HvMessageTypeTlbPageSizeMismatch:
+                LogRel(("Unimplemented msg:\n%.*Rhxd\n", (int)sizeof(*pMsg), pMsg));
+                AssertLogRelMsgFailedReturn(("Message type %#x not implemented!\n%.32Rhxd\n", pMsg->Header.MessageType, pMsg),
+                                            VERR_NEM_IPE_3);
+
+            case HvMessageTypeX64ApicEoi:
+            case HvMessageTypeX64LegacyFpError:
+            case HvMessageTypeX64RegisterIntercept:
+            case HvMessageTypeApicEoi:
+            case HvMessageTypeFerrAsserted:
+            case HvMessageTypeEventLogBufferComplete:
+            case HvMessageTimerExpired:
+                LogRel(("Unexpected msg:\n%.*Rhxd\n", (int)sizeof(*pMsg), pMsg));
+                AssertLogRelMsgFailedReturn(("Unexpected message on CPU #%u: %#x\n", pVCpu->idCpu, pMsg->Header.MessageType),
+                                            VERR_NEM_IPE_3);
+
+            default:
+                LogRel(("Unknown msg:\n%.*Rhxd\n", (int)sizeof(*pMsg), pMsg));
+                AssertLogRelMsgFailedReturn(("Unknown message on CPU #%u: %#x\n", pVCpu->idCpu, pMsg->Header.MessageType),
+                                            VERR_NEM_IPE_3);
+        }
+    }
+    else
+        AssertLogRelMsgFailedReturn(("Unexpected VID message type on CPU #%u: %#x LB %u\n",
+                                     pVCpu->idCpu, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage),
+                                    VERR_NEM_IPE_4);
+}
+#elif defined(IN_RING3)
+/**
+ * Handles VM exits.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessage
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExit(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    switch (pExit->ExitReason)
+    {
+        case WHvRunVpExitReasonMemoryAccess:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemUnmapped);
+            return nemR3WinHandleExitMemory(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonX64IoPortAccess:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitPortIo);
+            return nemR3WinHandleExitIoPort(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonX64Halt:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
+                             pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+            Log4(("HaltExit/%u\n", pVCpu->idCpu));
+            return VINF_EM_HALT;
+
+        case WHvRunVpExitReasonCanceled:
+            Log4(("CanceledExit/%u\n", pVCpu->idCpu));
+            return VINF_SUCCESS;
+
+        case WHvRunVpExitReasonX64InterruptWindow:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInterruptWindow);
+            return nemR3WinHandleExitInterruptWindow(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonX64Cpuid:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitCpuId);
+            return nemR3WinHandleExitCpuId(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonX64MsrAccess:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMsr);
+            return nemR3WinHandleExitMsr(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonException:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitException);
+            return nemR3WinHandleExitException(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonUnrecoverableException:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitUnrecoverable);
+            return nemR3WinHandleExitUnrecoverableException(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonUnsupportedFeature:
+        case WHvRunVpExitReasonInvalidVpRegisterValue:
+            LogRel(("Unimplemented exit:\n%.*Rhxd\n", (int)sizeof(*pExit), pExit));
+            AssertLogRelMsgFailedReturn(("Unexpected exit on CPU #%u: %#x\n%.32Rhxd\n",
+                                         pVCpu->idCpu, pExit->ExitReason, pExit), VERR_NEM_IPE_3);
+
+        /* Undesired exits: */
+        case WHvRunVpExitReasonNone:
+        default:
+            LogRel(("Unknown exit:\n%.*Rhxd\n", (int)sizeof(*pExit), pExit));
+            AssertLogRelMsgFailedReturn(("Unknown exit on CPU #%u: %#x!\n", pVCpu->idCpu, pExit->ExitReason), VERR_NEM_IPE_3);
+    }
+}
+#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+
+#ifdef IN_RING0
+/**
+ * Perform an I/O control operation on the partition handle (VID.SYS),
+ * restarting on alert-like behaviour.
+ *
+ * @returns NT status code.
+ * @param   pGVM            The ring-0 VM structure.
+ * @param   pGVCpu          The global (ring-0) per CPU structure.
+ * @param   fFlags          The wait flags.
+ * @param   cMillies        The timeout in milliseconds
+ */
+static NTSTATUS nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(PGVM pGVM, PGVMCPU pGVCpu, uint32_t fFlags, uint32_t cMillies)
+{
+    pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu     = pGVCpu->idCpu;
+    pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags   = fFlags;
+    pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMillies;
+    NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVCpu, pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
+                                            &pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
+                                            pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.cbInput,
+                                            NULL, 0);
+    if (rcNt == STATUS_SUCCESS)
+    { /* likely */ }
+    /*
+     * Generally, if we get down here, we have been interrupted between ACK'ing
+     * a message and waiting for the next due to a NtAlertThread call.  So, we
+     * should stop ACK'ing the previous message and get on waiting on the next.
+     * See similar stuff in nemHCWinRunGC().
+     */
+    else if (   rcNt == STATUS_TIMEOUT
+             || rcNt == STATUS_ALERTED    /* just in case */
+             || rcNt == STATUS_KERNEL_APC /* just in case */
+             || rcNt == STATUS_USER_APC   /* just in case */)
+    {
+        DBGFTRACE_CUSTOM(pGVCpu->CTX_SUFF(pVM), "IoCtlMessageSlotHandleAndGetNextRestart/1 %#x (f=%#x)", rcNt, fFlags);
+        STAM_REL_COUNTER_INC(&pGVCpu->nem.s.StatStopCpuPendingAlerts);
+        Assert(fFlags & VID_MSHAGN_F_GET_NEXT_MESSAGE);
+
+        pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu     = pGVCpu->idCpu;
+        pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags   = fFlags & ~VID_MSHAGN_F_HANDLE_MESSAGE;
+        pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMillies;
+        rcNt = nemR0NtPerformIoControl(pGVM, pGVCpu, pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
+                                       &pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
+                                       pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.cbInput,
+                                       NULL, 0);
+        DBGFTRACE_CUSTOM(pGVM, "IoCtlMessageSlotHandleAndGetNextRestart/2 %#x", rcNt);
+    }
+    return rcNt;
+}
+#endif /* IN_RING0 */
+
+
+#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+/**
+ * Worker for nemHCWinRunGC that stops the execution on the way out.
+ *
+ * The CPU was running the last time we checked, no there are no messages that
+ * needs being marked handled/whatever.  Caller checks this.
+ *
+ * @returns rcStrict on success, error status on failure.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   rcStrict        The nemHCWinRunGC return status.  This is a little
+ *                          bit unnecessary, except in internal error cases,
+ *                          since we won't need to stop the CPU if we took an
+ *                          exit.
+ * @param   pMappingHeader  The message slot mapping.
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinStopCpu(PVMCC pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict,
+                                             VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader)
+{
+# ifdef DBGFTRACE_ENABLED
+    HV_MESSAGE const volatile *pMsgForTrace = (HV_MESSAGE const volatile *)(pMappingHeader + 1);
+# endif
+
+    /*
+     * Try stopping the processor.  If we're lucky we manage to do this before it
+     * does another VM exit.
+     */
+    DBGFTRACE_CUSTOM(pVM, "nemStop#0");
+# ifdef IN_RING0
+    pVCpu->nem.s.uIoCtlBuf.idCpu = pVCpu->idCpu;
+    NTSTATUS rcNt = nemR0NtPerformIoControl(pVM, pVCpu, pVM->nemr0.s.IoCtlStopVirtualProcessor.uFunction,
+                                            &pVCpu->nem.s.uIoCtlBuf.idCpu, sizeof(pVCpu->nem.s.uIoCtlBuf.idCpu),
+                                            NULL, 0);
+    if (NT_SUCCESS(rcNt))
+    {
+        DBGFTRACE_CUSTOM(pVM, "nemStop#0: okay (%#x)", rcNt);
+        Log8(("nemHCWinStopCpu: Stopping CPU succeeded (cpu status %u)\n", nemHCWinCpuGetRunningStatus(pVCpu) ));
+        STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuSuccess);
+        return rcStrict;
+    }
+# else
+    BOOL fRet = VidStopVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu);
+    if (fRet)
+    {
+        DBGFTRACE_CUSTOM(pVM, "nemStop#0: okay");
+        Log8(("nemHCWinStopCpu: Stopping CPU succeeded (cpu status %u)\n", nemHCWinCpuGetRunningStatus(pVCpu) ));
+        STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuSuccess);
+        return rcStrict;
+    }
+# endif
+
+    /*
+     * Dang. The CPU stopped by itself and we got a couple of message to deal with.
+     */
+# ifdef IN_RING0
+    DBGFTRACE_CUSTOM(pVM, "nemStop#0: pending (%#x)", rcNt);
+    AssertLogRelMsgReturn(rcNt == ERROR_VID_STOP_PENDING, ("rcNt=%#x\n", rcNt),
+                          RT_SUCCESS(rcStrict) ?  VERR_NEM_IPE_5 : rcStrict);
+# else
+    DWORD dwErr = RTNtLastErrorValue();
+    DBGFTRACE_CUSTOM(pVM, "nemStop#0: pending (%#x)", dwErr);
+    AssertLogRelMsgReturn(dwErr == ERROR_VID_STOP_PENDING, ("dwErr=%#u (%#x)\n", dwErr, dwErr),
+                          RT_SUCCESS(rcStrict) ?  VERR_NEM_IPE_5 : rcStrict);
+# endif
+    Log8(("nemHCWinStopCpu: Stopping CPU #%u pending...\n", pVCpu->idCpu));
+    STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuPending);
+
+    /*
+     * First message: Exit or similar, sometimes VidMessageStopRequestComplete.
+     * Note! We can safely ASSUME that rcStrict isn't an important information one.
+     */
+# ifdef IN_RING0
+    rcNt = nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(pVM, pVCpu, VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /*ms*/);
+    DBGFTRACE_CUSTOM(pVM, "nemStop#1: %#x / %#x %#x %#x", rcNt, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage,
+                     pMsgForTrace->Header.MessageType);
+    AssertLogRelMsgReturn(rcNt == STATUS_SUCCESS,
+                          ("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
+                          RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
+# else
+    BOOL fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
+                                                     VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /*ms*/);
+    DBGFTRACE_CUSTOM(pVM, "nemStop#1: %d+%#x / %#x %#x %#x", fWait, RTNtLastErrorValue(), pMappingHeader->enmVidMsgType,
+                     pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
+    AssertLogRelMsgReturn(fWait, ("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
+                          RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
+# endif
+
+    VID_MESSAGE_TYPE enmVidMsgType = pMappingHeader->enmVidMsgType;
+    if (enmVidMsgType != VidMessageStopRequestComplete)
+    {
+        VBOXSTRICTRC rcStrict2 = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader);
+        if (rcStrict2 != VINF_SUCCESS && RT_SUCCESS(rcStrict))
+            rcStrict = rcStrict2;
+        DBGFTRACE_CUSTOM(pVM, "nemStop#1: handled %#x -> %d", pMsgForTrace->Header.MessageType, VBOXSTRICTRC_VAL(rcStrict));
+
+        /*
+         * Mark it as handled and get the stop request completed message, then mark
+         * that as handled too.  CPU is back into fully stopped stated then.
+         */
+# ifdef IN_RING0
+        rcNt = nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(pVM, pVCpu,
+                                                              VID_MSHAGN_F_HANDLE_MESSAGE | VID_MSHAGN_F_GET_NEXT_MESSAGE,
+                                                              30000 /*ms*/);
+        DBGFTRACE_CUSTOM(pVM, "nemStop#2: %#x / %#x %#x %#x", rcNt, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage,
+                         pMsgForTrace->Header.MessageType);
+        AssertLogRelMsgReturn(rcNt == STATUS_SUCCESS,
+                              ("2nd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
+                              RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
+# else
+        fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
+                                                    VID_MSHAGN_F_HANDLE_MESSAGE | VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /*ms*/);
+        DBGFTRACE_CUSTOM(pVM, "nemStop#2: %d+%#x / %#x %#x %#x", fWait, RTNtLastErrorValue(), pMappingHeader->enmVidMsgType,
+                         pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
+        AssertLogRelMsgReturn(fWait, ("2nd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
+                              RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
+# endif
+
+        /* It should be a stop request completed message. */
+        enmVidMsgType = pMappingHeader->enmVidMsgType;
+        AssertLogRelMsgReturn(enmVidMsgType == VidMessageStopRequestComplete,
+                              ("Unexpected 2nd message following ERROR_VID_STOP_PENDING: %#x LB %#x\n",
+                               enmVidMsgType, pMappingHeader->cbMessage),
+                              RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
+
+        /*
+         * Mark the VidMessageStopRequestComplete message as handled.
+         */
+# ifdef IN_RING0
+        rcNt = nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(pVM, pVCpu, VID_MSHAGN_F_HANDLE_MESSAGE, 30000 /*ms*/);
+        DBGFTRACE_CUSTOM(pVM, "nemStop#3: %#x / %#x %#x %#x", rcNt, pMappingHeader->enmVidMsgType,
+                         pMsgForTrace->Header.MessageType, pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
+        AssertLogRelMsgReturn(rcNt == STATUS_SUCCESS,
+                              ("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
+                              RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
+# else
+        fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu, VID_MSHAGN_F_HANDLE_MESSAGE, 30000 /*ms*/);
+        DBGFTRACE_CUSTOM(pVM, "nemStop#3: %d+%#x / %#x %#x %#x", fWait, RTNtLastErrorValue(), pMappingHeader->enmVidMsgType,
+                         pMsgForTrace->Header.MessageType, pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
+        AssertLogRelMsgReturn(fWait, ("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
+                              RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
+# endif
+        Log8(("nemHCWinStopCpu: Stopped the CPU (rcStrict=%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict) ));
+    }
+    else
+    {
+        /** @todo I'm not so sure about this now... */
+        DBGFTRACE_CUSTOM(pVM, "nemStop#9: %#x %#x %#x", pMappingHeader->enmVidMsgType,
+                         pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
+        STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuPendingOdd);
+        Log8(("nemHCWinStopCpu: Stopped the CPU (rcStrict=%Rrc) - 1st VidMessageSlotHandleAndGetNext got VidMessageStopRequestComplete.\n",
+              VBOXSTRICTRC_VAL(rcStrict) ));
+    }
+    return rcStrict;
+}
+#endif /* NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+#if defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API) || defined(IN_RING3)
+
+/**
+ * Deals with pending interrupt related force flags, may inject interrupt.
+ *
+ * @returns VBox strict status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context per CPU structure.
+ * @param   pfInterruptWindows  Where to return interrupt window flags.
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleInterruptFF(PVMCC pVM, PVMCPUCC pVCpu, uint8_t *pfInterruptWindows)
+{
+    Assert(!TRPMHasTrap(pVCpu));
+    RT_NOREF_PV(pVM);
+
+    /*
+     * First update APIC.  We ASSUME this won't need TPR/CR8.
+     */
+    if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
+    {
+        APICUpdatePendingInterrupts(pVCpu);
+        if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC
+                                      | VMCPU_FF_INTERRUPT_NMI  | VMCPU_FF_INTERRUPT_SMI))
+            return VINF_SUCCESS;
+    }
+
+    /*
+     * We don't currently implement SMIs.
+     */
+    AssertReturn(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI), VERR_NEM_IPE_0);
+
+    /*
+     * Check if we've got the minimum of state required for deciding whether we
+     * can inject interrupts and NMIs.  If we don't have it, get all we might require
+     * for injection via IEM.
+     */
+    bool const fPendingNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+    uint64_t   fNeedExtrn  = CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS
+                           | (fPendingNmi ? CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI : 0);
+    if (pVCpu->cpum.GstCtx.fExtrn & fNeedExtrn)
+    {
+        VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "IntFF");
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+    bool const fInhibitInterrupts = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+                                 && EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip;
+
+    /*
+     * NMI? Try deliver it first.
+     */
+    if (fPendingNmi)
+    {
+        if (   !fInhibitInterrupts
+            && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+        {
+            VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "NMI");
+            if (rcStrict == VINF_SUCCESS)
+            {
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+                rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_NMI, TRPM_HARDWARE_INT, 0, 0, 0);
+                Log8(("Injected NMI on %u (%d)\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
+            }
+            return rcStrict;
+        }
+        *pfInterruptWindows |= NEM_WIN_INTW_F_NMI;
+        Log8(("NMI window pending on %u\n", pVCpu->idCpu));
+    }
+
+    /*
+     * APIC or PIC interrupt?
+     */
+    if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
+    {
+        if (   !fInhibitInterrupts
+            && pVCpu->cpum.GstCtx.rflags.Bits.u1IF)
+        {
+            AssertCompile(NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT & CPUMCTX_EXTRN_APIC_TPR);
+            VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "NMI");
+            if (rcStrict == VINF_SUCCESS)
+            {
+                uint8_t bInterrupt;
+                int rc = PDMGetInterrupt(pVCpu, &bInterrupt);
+                if (RT_SUCCESS(rc))
+                {
+                    rcStrict = IEMInjectTrap(pVCpu, bInterrupt, TRPM_HARDWARE_INT, 0, 0, 0);
+                    Log8(("Injected interrupt %#x on %u (%d)\n", bInterrupt, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
+                }
+                else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
+                {
+                    *pfInterruptWindows |= ((bInterrupt >> 4) << NEM_WIN_INTW_F_PRIO_SHIFT) | NEM_WIN_INTW_F_REGULAR;
+                    Log8(("VERR_APIC_INTR_MASKED_BY_TPR: *pfInterruptWindows=%#x\n", *pfInterruptWindows));
+                }
+                else
+                    Log8(("PDMGetInterrupt failed -> %d\n", rc));
+            }
+            return rcStrict;
+        }
+        else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC) && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC))
+        {
+            /* If only an APIC interrupt is pending, we need to know its priority. Otherwise we'll
+             * likely get pointless deliverability notifications with IF=1 but TPR still too high.
+             */
+            bool fPendingIntr;
+            uint8_t u8Tpr, u8PendingIntr;
+            int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
+            AssertRC(rc);
+            *pfInterruptWindows |= (u8PendingIntr >> 4) << NEM_WIN_INTW_F_PRIO_SHIFT;
+        }
+        *pfInterruptWindows |= NEM_WIN_INTW_F_REGULAR;
+        Log8(("Interrupt window pending on %u\n", pVCpu->idCpu));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inner NEM runloop for windows.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinRunGC(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags));
+# ifdef LOG_ENABLED
+    if (LogIs3Enabled())
+        nemHCWinLogState(pVM, pVCpu);
+# endif
+
+    /*
+     * Try switch to NEM runloop state.
+     */
+    if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
+    { /* likely */ }
+    else
+    {
+        VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
+        LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * The run loop.
+     *
+     * Current approach to state updating to use the sledgehammer and sync
+     * everything every time.  This will be optimized later.
+     */
+# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+    VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader = (VID_MESSAGE_MAPPING_HEADER volatile *)pVCpu->nem.s.pvMsgSlotMapping;
+# endif
+    const bool      fSingleStepping     = DBGFIsStepping(pVCpu);
+//    const uint32_t  fCheckVmFFs         = !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK
+//                                                           : VM_FF_HP_R0_PRE_HM_STEP_MASK;
+//    const uint32_t  fCheckCpuFFs        = !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK;
+    VBOXSTRICTRC    rcStrict            = VINF_SUCCESS;
+    for (unsigned iLoop = 0;; iLoop++)
+    {
+# ifndef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+        /*
+         * Hack alert!
+         */
+        uint32_t const cMappedPages = pVM->nem.s.cMappedPages;
+        if (cMappedPages >= 4000)
+        {
+            PGMPhysNemEnumPagesByState(pVM, pVCpu, NEM_WIN_PAGE_STATE_READABLE, nemHCWinUnmapOnePageCallback, NULL);
+            Log(("nemHCWinRunGC: Unmapped all; cMappedPages=%u -> %u\n", cMappedPages, pVM->nem.s.cMappedPages));
+        }
+# endif
+
+        /*
+         * Pending interrupts or such?  Need to check and deal with this prior
+         * to the state syncing.
+         */
+        pVCpu->nem.s.fDesiredInterruptWindows = 0;
+        if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_UPDATE_APIC | VMCPU_FF_INTERRUPT_PIC
+                                     | VMCPU_FF_INTERRUPT_NMI  | VMCPU_FF_INTERRUPT_SMI))
+        {
+# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+            /* Make sure the CPU isn't executing. */
+            if (pVCpu->nem.s.fHandleAndGetFlags == VID_MSHAGN_F_GET_NEXT_MESSAGE)
+            {
+                pVCpu->nem.s.fHandleAndGetFlags = 0;
+                rcStrict = nemHCWinStopCpu(pVM, pVCpu, rcStrict, pMappingHeader);
+                if (rcStrict == VINF_SUCCESS)
+                { /* likely */ }
+                else
+                {
+                    LogFlow(("NEM/%u: breaking: nemHCWinStopCpu -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
+                    STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
+                    break;
+                }
+            }
+# endif
+
+            /* Try inject interrupt. */
+            rcStrict = nemHCWinHandleInterruptFF(pVM, pVCpu, &pVCpu->nem.s.fDesiredInterruptWindows);
+            if (rcStrict == VINF_SUCCESS)
+            { /* likely */ }
+            else
+            {
+                LogFlow(("NEM/%u: breaking: nemHCWinHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
+                break;
+            }
+        }
+
+        /*
+         * Ensure that hyper-V has the whole state.
+         * (We always update the interrupt windows settings when active as hyper-V seems
+         * to forget about it after an exit.)
+         */
+        if (      (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK))
+               !=                              (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK)
+            || (  (   pVCpu->nem.s.fDesiredInterruptWindows
+                   || pVCpu->nem.s.fCurrentInterruptWindows != pVCpu->nem.s.fDesiredInterruptWindows)
+# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+                && pVCpu->nem.s.fHandleAndGetFlags != VID_MSHAGN_F_GET_NEXT_MESSAGE /* not running */
+# endif
+               )
+           )
+        {
+# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+            AssertMsg(pVCpu->nem.s.fHandleAndGetFlags != VID_MSHAGN_F_GET_NEXT_MESSAGE /* not running */,
+                      ("%#x fExtrn=%#RX64 (%#RX64) fDesiredInterruptWindows=%d fCurrentInterruptWindows=%#x vs %#x\n",
+                       pVCpu->nem.s.fHandleAndGetFlags, pVCpu->cpum.GstCtx.fExtrn, ~pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK),
+                       pVCpu->nem.s.fDesiredInterruptWindows, pVCpu->nem.s.fCurrentInterruptWindows, pVCpu->nem.s.fDesiredInterruptWindows));
+# endif
+# ifdef IN_RING0
+            int rc2 = nemR0WinExportState(pVM, pVCpu, &pVCpu->cpum.GstCtx);
+# else
+            int rc2 = nemHCWinCopyStateToHyperV(pVM, pVCpu);
+# endif
+            AssertRCReturn(rc2, rc2);
+        }
+
+        /*
+         * Poll timers and run for a bit.
+         *
+         * With the VID approach (ring-0 or ring-3) we can specify a timeout here,
+         * so we take the time of the next timer event and uses that as a deadline.
+         * The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
+         */
+        /** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
+         *        the whole polling job when timers have changed... */
+        uint64_t       offDeltaIgnored;
+        uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
+        if (   !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
+            && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
+        {
+# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+            if (pVCpu->nem.s.fHandleAndGetFlags)
+            { /* Very likely that the CPU does NOT need starting (pending msg, running). */ }
+            else
+            {
+#  ifdef IN_RING0
+                pVCpu->nem.s.uIoCtlBuf.idCpu = pVCpu->idCpu;
+                NTSTATUS rcNt = nemR0NtPerformIoControl(pVM, pVCpu, pVM->nemr0.s.IoCtlStartVirtualProcessor.uFunction,
+                                                        &pVCpu->nem.s.uIoCtlBuf.idCpu, sizeof(pVCpu->nem.s.uIoCtlBuf.idCpu),
+                                                        NULL, 0);
+                LogFlow(("NEM/%u: IoCtlStartVirtualProcessor -> %#x\n", pVCpu->idCpu, rcNt));
+                AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("VidStartVirtualProcessor failed for CPU #%u: %#x\n", pVCpu->idCpu, rcNt),
+                                      VERR_NEM_IPE_5);
+#  else
+                AssertLogRelMsgReturn(g_pfnVidStartVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu),
+                                      ("VidStartVirtualProcessor failed for CPU #%u: %u (%#x, rcNt=%#x)\n",
+                                       pVCpu->idCpu, RTNtLastErrorValue(), RTNtLastErrorValue(), RTNtLastStatusValue()),
+                                      VERR_NEM_IPE_5);
+#  endif
+                pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
+            }
+# endif /* NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
+
+            if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
+            {
+# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+                uint64_t const  nsNow           = RTTimeNanoTS();
+                int64_t const   cNsNextTimerEvt = nsNow - nsNextTimerEvt;
+                uint32_t        cMsWait;
+                if (cNsNextTimerEvt < 100000 /* ns */)
+                    cMsWait = 0;
+                else if ((uint64_t)cNsNextTimerEvt < RT_NS_1SEC)
+                {
+                    if ((uint32_t)cNsNextTimerEvt < 2*RT_NS_1MS)
+                        cMsWait = 1;
+                    else
+                        cMsWait = ((uint32_t)cNsNextTimerEvt - 100000 /*ns*/) / RT_NS_1MS;
+                }
+                else
+                    cMsWait = RT_MS_1SEC;
+#  ifdef IN_RING0
+                pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu     = pVCpu->idCpu;
+                pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags   = pVCpu->nem.s.fHandleAndGetFlags;
+                pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMsWait;
+                NTSTATUS rcNt = nemR0NtPerformIoControl(pVM, pVCpu, pVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
+                                                        &pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
+                                                        pVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.cbInput,
+                                                        NULL, 0);
+                VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
+                if (rcNt == STATUS_SUCCESS)
+#  else
+                BOOL fRet = VidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
+                                                           pVCpu->nem.s.fHandleAndGetFlags, cMsWait);
+                VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
+                if (fRet)
+#  endif
+# else
+                WHV_RUN_VP_EXIT_CONTEXT ExitReason;
+                RT_ZERO(ExitReason);
+                LogFlow(("NEM/%u: Entry @ %04X:%08RX64 IF=%d (~~may be stale~~)\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.Bits.u1IF));
+                TMNotifyStartOfExecution(pVM, pVCpu);
+                HRESULT hrc = WHvRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, &ExitReason, sizeof(ExitReason));
+                VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
+                TMNotifyEndOfExecution(pVM, pVCpu);
+                LogFlow(("NEM/%u: Exit  @ %04X:%08RX64 IF=%d CR8=%#x \n", pVCpu->idCpu, ExitReason.VpContext.Cs.Selector, ExitReason.VpContext.Rip, RT_BOOL(ExitReason.VpContext.Rflags & X86_EFL_IF), ExitReason.VpContext.Cr8));
+                if (SUCCEEDED(hrc))
+# endif
+                {
+                    /*
+                     * Deal with the message.
+                     */
+# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+                    rcStrict = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader);
+                    pVCpu->nem.s.fHandleAndGetFlags |= VID_MSHAGN_F_HANDLE_MESSAGE;
+# else
+                    rcStrict = nemR3WinHandleExit(pVM, pVCpu, &ExitReason);
+# endif
+                    if (rcStrict == VINF_SUCCESS)
+                    { /* hopefully likely */ }
+                    else
+                    {
+                        LogFlow(("NEM/%u: breaking: nemHCWinHandleMessage -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
+                        STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
+                        break;
+                    }
+                }
+                else
+                {
+# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+
+                    /* VID.SYS merges STATUS_ALERTED and STATUS_USER_APC into STATUS_TIMEOUT,
+                       so after NtAlertThread we end up here with a STATUS_TIMEOUT.  And yeah,
+                       the error code conversion is into WAIT_XXX, i.e. NT status codes. */
+#  ifndef IN_RING0
+                    DWORD rcNt = GetLastError();
+#  endif
+                    LogFlow(("NEM/%u: VidMessageSlotHandleAndGetNext -> %#x\n", pVCpu->idCpu, rcNt));
+                    AssertLogRelMsgReturn(   rcNt == STATUS_TIMEOUT
+                                          || rcNt == STATUS_ALERTED    /* just in case */
+                                          || rcNt == STATUS_USER_APC   /* ditto */
+                                          || rcNt == STATUS_KERNEL_APC /* ditto */
+                                          , ("VidMessageSlotHandleAndGetNext failed for CPU #%u: %#x (%u)\n",
+                                             pVCpu->idCpu, rcNt, rcNt),
+                                          VERR_NEM_IPE_0);
+                    pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
+                    STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatGetMsgTimeout);
+# else
+                    AssertLogRelMsgFailedReturn(("WHvRunVirtualProcessor failed for CPU #%u: %#x (%u)\n",
+                                                 pVCpu->idCpu, hrc, GetLastError()),
+                                                VERR_NEM_IPE_0);
+# endif
+                }
+
+                /*
+                 * If no relevant FFs are pending, loop.
+                 */
+                if (   !VM_FF_IS_ANY_SET(   pVM,   !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK    : VM_FF_HP_R0_PRE_HM_STEP_MASK)
+                    && !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
+                    continue;
+
+                /** @todo Try handle pending flags, not just return to EM loops.  Take care
+                 *        not to set important RCs here unless we've handled a message. */
+                LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
+                         pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
+            }
+            else
+            {
+                LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
+            }
+        }
+        else
+        {
+            LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
+        }
+        break;
+    } /* the run loop */
+
+
+    /*
+     * If the CPU is running, make sure to stop it before we try sync back the
+     * state and return to EM.  We don't sync back the whole state if we can help it.
+     */
+# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+    if (pVCpu->nem.s.fHandleAndGetFlags == VID_MSHAGN_F_GET_NEXT_MESSAGE)
+    {
+        pVCpu->nem.s.fHandleAndGetFlags = 0;
+        rcStrict = nemHCWinStopCpu(pVM, pVCpu, rcStrict, pMappingHeader);
+    }
+# endif
+
+    if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
+        VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
+
+    if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)))
+    {
+        /* Try anticipate what we might need. */
+        uint64_t fImport = IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI;
+        if (   (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
+            || RT_FAILURE(rcStrict))
+            fImport = CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT);
+# ifdef IN_RING0 /* Ring-3 I/O port access optimizations: */
+        else if (   rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
+                 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
+            fImport = CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT;
+        else if (rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
+            fImport = CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT;
+# endif
+        else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC | VMCPU_FF_INTERRUPT_APIC
+                                          | VMCPU_FF_INTERRUPT_NMI | VMCPU_FF_INTERRUPT_SMI))
+            fImport |= IEM_CPUMCTX_EXTRN_XCPT_MASK;
+
+        if (pVCpu->cpum.GstCtx.fExtrn & fImport)
+        {
+# ifdef IN_RING0
+            int rc2 = nemR0WinImportState(pVM, pVCpu, &pVCpu->cpum.GstCtx, fImport | CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT,
+                                          true /*fCanUpdateCr3*/);
+            if (RT_SUCCESS(rc2))
+                pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
+            else if (rc2 == VERR_NEM_FLUSH_TLB)
+            {
+                pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
+                if (rcStrict == VINF_SUCCESS || rcStrict == -rc2)
+                    rcStrict = -rc2;
+                else
+                {
+                    pVCpu->nem.s.rcPending = -rc2;
+                    LogFlow(("NEM/%u: rcPending=%Rrc (rcStrict=%Rrc)\n", pVCpu->idCpu, rc2, VBOXSTRICTRC_VAL(rcStrict) ));
+                }
+            }
+# else
+            int rc2 = nemHCWinCopyStateFromHyperV(pVM, pVCpu, fImport | CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT);
+            if (RT_SUCCESS(rc2))
+                pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
+# endif
+            else if (RT_SUCCESS(rcStrict))
+                rcStrict = rc2;
+            if (!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT))))
+                pVCpu->cpum.GstCtx.fExtrn = 0;
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
+        }
+        else
+        {
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
+            pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
+        }
+    }
+    else
+    {
+        STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
+        pVCpu->cpum.GstCtx.fExtrn = 0;
+    }
+
+    LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n",
+             pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, VBOXSTRICTRC_VAL(rcStrict) ));
+    return rcStrict;
+}
+
+#endif /* defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API) || defined(IN_RING3) */
+
+/**
+ * @callback_method_impl{FNPGMPHYSNEMCHECKPAGE}
+ */
+NEM_TMPL_STATIC DECLCALLBACK(int) nemHCWinUnsetForA20CheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys,
+                                                                     PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
+{
+    /* We'll just unmap the memory. */
+    if (pInfo->u2NemState > NEM_WIN_PAGE_STATE_UNMAPPED)
+    {
+#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+        int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
+        AssertRC(rc);
+        if (RT_SUCCESS(rc))
+#else
+        HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
+        if (SUCCEEDED(hrc))
+#endif
+        {
+            uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+            Log5(("NEM GPA unmapped/A20: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[pInfo->u2NemState], cMappedPages));
+            pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
+        }
+        else
+        {
+#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+            LogRel(("nemHCWinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
+            return rc;
+#else
+            LogRel(("nemHCWinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                    GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+            return VERR_NEM_IPE_2;
+#endif
+        }
+    }
+    RT_NOREF(pVCpu, pvUser);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Unmaps a page from Hyper-V for the purpose of emulating A20 gate behavior.
+ *
+ * @returns The PGMPhysNemQueryPageInfo result.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhys          The page to unmap.
+ */
+NEM_TMPL_STATIC int nemHCWinUnmapPageForA20Gate(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys)
+{
+    PGMPHYSNEMPAGEINFO Info;
+    return PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhys, false /*fMakeWritable*/, &Info,
+                                     nemHCWinUnsetForA20CheckerCallback, NULL);
+}
+
+
+void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
+{
+    Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
+    NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
+}
+
+
+void nemHCNativeNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
+                                                int fRestoreAsRAM, bool fRestoreAsRAM2)
+{
+    Log5(("nemHCNativeNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d fRestoreAsRAM=%d fRestoreAsRAM2=%d\n",
+          GCPhys, cb, enmKind, fRestoreAsRAM, fRestoreAsRAM2));
+    NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb); NOREF(fRestoreAsRAM); NOREF(fRestoreAsRAM2);
+}
+
+
+void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
+                                            RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
+{
+    Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
+          GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
+    NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
+}
+
+
+/**
+ * Worker that maps pages into Hyper-V.
+ *
+ * This is used by the PGM physical page notifications as well as the memory
+ * access VMEXIT handlers.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.
+ * @param   GCPhysSrc       The source page address.
+ * @param   GCPhysDst       The hyper-V destination page.  This may differ from
+ *                          GCPhysSrc when A20 is disabled.
+ * @param   fPageProt       NEM_PAGE_PROT_XXX.
+ * @param   pu2State        Our page state (input/output).
+ * @param   fBackingChanged Set if the page backing is being changed.
+ * @thread  EMT(pVCpu)
+ */
+NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
+                                           uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged)
+{
+#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+    /*
+     * When using the hypercalls instead of the ring-3 APIs, we don't need to
+     * unmap memory before modifying it.  We still want to track the state though,
+     * since unmap will fail when called an unmapped page and we don't want to redo
+     * upgrades/downgrades.
+     */
+    uint8_t const u2OldState = *pu2State;
+    int rc;
+    if (fPageProt == NEM_PAGE_PROT_NONE)
+    {
+        if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
+        {
+            rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
+            if (RT_SUCCESS(rc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
+                uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+                Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
+            }
+            else
+                AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
+        }
+        else
+            rc = VINF_SUCCESS;
+    }
+    else if (fPageProt & NEM_PAGE_PROT_WRITE)
+    {
+        if (u2OldState != NEM_WIN_PAGE_STATE_WRITABLE || fBackingChanged)
+        {
+            rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
+                                            HV_MAP_GPA_READABLE   | HV_MAP_GPA_WRITABLE
+                                          | HV_MAP_GPA_EXECUTABLE | HV_MAP_GPA_EXECUTABLE_AGAIN);
+            if (RT_SUCCESS(rc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
+                uint32_t cMappedPages = u2OldState <= NEM_WIN_PAGE_STATE_UNMAPPED
+                                      ? ASMAtomicIncU32(&pVM->nem.s.cMappedPages) : pVM->nem.s.cMappedPages;
+                Log5(("NEM GPA writable/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
+                NOREF(cMappedPages);
+            }
+            else
+                AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
+        }
+        else
+            rc = VINF_SUCCESS;
+    }
+    else
+    {
+        if (u2OldState != NEM_WIN_PAGE_STATE_READABLE || fBackingChanged)
+        {
+            rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
+                                          HV_MAP_GPA_READABLE | HV_MAP_GPA_EXECUTABLE | HV_MAP_GPA_EXECUTABLE_AGAIN);
+            if (RT_SUCCESS(rc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_READABLE;
+                uint32_t cMappedPages = u2OldState <= NEM_WIN_PAGE_STATE_UNMAPPED
+                                      ? ASMAtomicIncU32(&pVM->nem.s.cMappedPages) : pVM->nem.s.cMappedPages;
+                Log5(("NEM GPA read+exec/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
+                NOREF(cMappedPages);
+            }
+            else
+                AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
+        }
+        else
+            rc = VINF_SUCCESS;
+    }
+
+    return VINF_SUCCESS;
+
+#else
+    /*
+     * Looks like we need to unmap a page before we can change the backing
+     * or even modify the protection.  This is going to be *REALLY* efficient.
+     * PGM lends us two bits to keep track of the state here.
+     */
+    uint8_t const u2OldState = *pu2State;
+    uint8_t const u2NewState = fPageProt & NEM_PAGE_PROT_WRITE ? NEM_WIN_PAGE_STATE_WRITABLE
+                             : fPageProt & NEM_PAGE_PROT_READ  ? NEM_WIN_PAGE_STATE_READABLE : NEM_WIN_PAGE_STATE_UNMAPPED;
+    if (   fBackingChanged
+        || u2NewState != u2OldState)
+    {
+        if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
+        {
+# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+            int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
+            AssertRC(rc);
+            if (RT_SUCCESS(rc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
+                uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+                if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
+                {
+                    Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
+                          GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
+                    return VINF_SUCCESS;
+                }
+            }
+            else
+            {
+                LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
+                return rc;
+            }
+# else
+            HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst, X86_PAGE_SIZE);
+            if (SUCCEEDED(hrc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
+                uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+                if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
+                {
+                    Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
+                          GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
+                    return VINF_SUCCESS;
+                }
+            }
+            else
+            {
+                LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                        GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+                return VERR_NEM_INIT_FAILED;
+            }
+# endif
+        }
+    }
+
+    /*
+     * Writeable mapping?
+     */
+    if (fPageProt & NEM_PAGE_PROT_WRITE)
+    {
+# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+        int rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
+                                            HV_MAP_GPA_READABLE   | HV_MAP_GPA_WRITABLE
+                                          | HV_MAP_GPA_EXECUTABLE | HV_MAP_GPA_EXECUTABLE_AGAIN);
+        AssertRC(rc);
+        if (RT_SUCCESS(rc))
+        {
+            *pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
+            uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+            Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
+                  GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
+            return VINF_SUCCESS;
+        }
+        LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
+        return rc;
+# else
+        void *pvPage;
+        int rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhysSrc, &pvPage);
+        if (RT_SUCCESS(rc))
+        {
+            HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvPage, GCPhysDst, X86_PAGE_SIZE,
+                                         WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute | WHvMapGpaRangeFlagWrite);
+            if (SUCCEEDED(hrc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
+                uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+                Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
+                      GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
+                return VINF_SUCCESS;
+            }
+            LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                    GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+            return VERR_NEM_INIT_FAILED;
+        }
+        LogRel(("nemHCNativeSetPhysPage/writable: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
+        return rc;
+# endif
+    }
+
+    if (fPageProt & NEM_PAGE_PROT_READ)
+    {
+# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+        int rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
+                                          HV_MAP_GPA_READABLE | HV_MAP_GPA_EXECUTABLE | HV_MAP_GPA_EXECUTABLE_AGAIN);
+        AssertRC(rc);
+        if (RT_SUCCESS(rc))
+        {
+            *pu2State = NEM_WIN_PAGE_STATE_READABLE;
+            uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+            Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
+                  GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
+            return VINF_SUCCESS;
+        }
+        LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
+        return rc;
+# else
+        const void *pvPage;
+        int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhysSrc, &pvPage);
+        if (RT_SUCCESS(rc))
+        {
+            HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhysDst, X86_PAGE_SIZE,
+                                         WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute);
+            if (SUCCEEDED(hrc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_READABLE;
+                uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+                Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
+                      GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
+                return VINF_SUCCESS;
+            }
+            LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                    GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+            return VERR_NEM_INIT_FAILED;
+        }
+        LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
+        return rc;
+# endif
+    }
+
+    /* We already unmapped it above. */
+    *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
+    return VINF_SUCCESS;
+#endif /* !NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
+}
+
+
+NEM_TMPL_STATIC int nemHCJustUnmapPageFromHyperV(PVMCC pVM, RTGCPHYS GCPhysDst, uint8_t *pu2State)
+{
+    if (*pu2State <= NEM_WIN_PAGE_STATE_UNMAPPED)
+    {
+        Log5(("nemHCJustUnmapPageFromHyperV: %RGp == unmapped\n", GCPhysDst));
+        *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
+        return VINF_SUCCESS;
+    }
+
+#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) || defined(IN_RING0)
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
+    AssertRC(rc);
+    if (RT_SUCCESS(rc))
+    {
+        uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+        Log5(("NEM GPA unmapped/just: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[*pu2State], cMappedPages));
+        *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
+        return VINF_SUCCESS;
+    }
+    LogRel(("nemHCJustUnmapPageFromHyperV/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
+    return rc;
+#else
+    HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE);
+    if (SUCCEEDED(hrc))
+    {
+        uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+        *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
+        Log5(("nemHCJustUnmapPageFromHyperV: %RGp => unmapped (total %u)\n", GCPhysDst, cMappedPages));
+        return VINF_SUCCESS;
+    }
+    LogRel(("nemHCJustUnmapPageFromHyperV(%RGp): failed! hrc=%Rhrc (%#x) Last=%#x/%u\n",
+            GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+    return VERR_NEM_IPE_6;
+#endif
+}
+
+
+int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
+                                       PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
+          GCPhys, HCPhys, fPageProt, enmType, *pu2State));
+    RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
+
+    int rc;
+#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) || defined(IN_RING0)
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    if (   pVM->nem.s.fA20Enabled
+        || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
+        rc = nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /*fBackingChanged*/);
+    else
+    {
+        /* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
+        rc = nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys | RT_BIT_32(20));
+        if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys) && RT_SUCCESS(rc))
+            rc = nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /*fBackingChanged*/);
+
+    }
+#else
+    RT_NOREF_PV(fPageProt);
+    if (   pVM->nem.s.fA20Enabled
+        || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
+        rc = nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
+    else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
+        rc = nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
+    else
+        rc = VINF_SUCCESS; /* ignore since we've got the alias page at this address. */
+#endif
+    return rc;
+}
+
+
+void nemHCNativeNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
+                                          PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    Log5(("nemHCNativeNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
+          GCPhys, HCPhys, fPageProt, enmType, *pu2State));
+    RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
+
+#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) || defined(IN_RING0)
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    if (   pVM->nem.s.fA20Enabled
+        || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
+        nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, false /*fBackingChanged*/);
+    else
+    {
+        /* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
+        nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys | RT_BIT_32(20));
+        if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
+            nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, false /*fBackingChanged*/);
+    }
+#else
+    RT_NOREF_PV(fPageProt);
+    if (   pVM->nem.s.fA20Enabled
+        || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
+        nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
+    else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
+        nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
+    /* else: ignore since we've got the alias page at this address. */
+#endif
+}
+
+
+void nemHCNativeNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
+                                     uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    Log5(("nemHCNativeNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
+          GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, *pu2State));
+    RT_NOREF_PV(HCPhysPrev); RT_NOREF_PV(HCPhysNew); RT_NOREF_PV(enmType);
+
+#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) || defined(IN_RING0)
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    if (   pVM->nem.s.fA20Enabled
+        || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
+        nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /*fBackingChanged*/);
+    else
+    {
+        /* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
+        nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys | RT_BIT_32(20));
+        if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
+            nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /*fBackingChanged*/);
+    }
+#else
+    RT_NOREF_PV(fPageProt);
+    if (   pVM->nem.s.fA20Enabled
+        || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
+        nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
+    else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
+        nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
+    /* else: ignore since we've got the alias page at this address. */
+#endif
+}
+
diff --git a/src/VBox/VMM/VMMAll/PDMAll.cpp b/src/VBox/VMM/VMMAll/PDMAll.cpp
new file mode 100644
index 00000000..6ad7ed62
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAll.cpp
@@ -0,0 +1,343 @@
+/* $Id: PDMAll.cpp $ */
+/** @file
+ * PDM Critical Sections
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/vmm/apic.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+
+#include "PDMInline.h"
+#include "dtrace/VBoxVMM.h"
+
+
+
+/**
+ * Gets the pending interrupt.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_APIC_INTR_MASKED_BY_TPR when an APIC interrupt is pending but
+ *          can't be delivered due to TPR priority.
+ * @retval  VERR_NO_DATA if there is no interrupt to be delivered (either APIC
+ *          has been software-disabled since it flagged something was pending,
+ *          or other reasons).
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pu8Interrupt    Where to store the interrupt.
+ */
+VMMDECL(int) PDMGetInterrupt(PVMCPUCC pVCpu, uint8_t *pu8Interrupt)
+{
+    /*
+     * The local APIC has a higher priority than the PIC.
+     */
+    int rc = VERR_NO_DATA;
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC))
+    {
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC);
+        uint32_t uTagSrc;
+        rc = APICGetInterrupt(pVCpu, pu8Interrupt, &uTagSrc);
+        if (RT_SUCCESS(rc))
+        {
+            if (rc == VINF_SUCCESS)
+                VBOXVMM_PDM_IRQ_GET(pVCpu, RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc), *pu8Interrupt);
+            return rc;
+        }
+        /* else if it's masked by TPR/PPR/whatever, go ahead checking the PIC. Such masked
+           interrupts shouldn't prevent ExtINT from being delivered. */
+    }
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    pdmLock(pVM);
+
+    /*
+     * Check the PIC.
+     */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC))
+    {
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);
+        Assert(pVM->pdm.s.Pic.CTX_SUFF(pDevIns));
+        Assert(pVM->pdm.s.Pic.CTX_SUFF(pfnGetInterrupt));
+        uint32_t uTagSrc;
+        int i = pVM->pdm.s.Pic.CTX_SUFF(pfnGetInterrupt)(pVM->pdm.s.Pic.CTX_SUFF(pDevIns), &uTagSrc);
+        AssertMsg(i <= 255 && i >= 0, ("i=%d\n", i));
+        if (i >= 0)
+        {
+            pdmUnlock(pVM);
+            *pu8Interrupt = (uint8_t)i;
+            VBOXVMM_PDM_IRQ_GET(pVCpu, RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc), i);
+            return VINF_SUCCESS;
+        }
+    }
+
+    /*
+     * One scenario where we may possibly get here is if the APIC signaled a pending interrupt,
+     * got an APIC MMIO/MSR VM-exit which disabled the APIC. We could, in theory, clear the APIC
+     * force-flag from all the places which disables the APIC but letting PDMGetInterrupt() fail
+     * without returning a valid interrupt still needs to be handled for the TPR masked case,
+     * so we shall just handle it here regardless if we choose to update the APIC code in the future.
+     */
+
+    pdmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Sets the pending interrupt coming from ISA source or HPET.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   u8Irq           The IRQ line.
+ * @param   u8Level         The new level.
+ * @param   uTagSrc         The IRQ tag and source tracer ID.
+ */
+VMMDECL(int) PDMIsaSetIrq(PVMCC pVM, uint8_t u8Irq, uint8_t u8Level, uint32_t uTagSrc)
+{
+    pdmLock(pVM);
+
+    /** @todo put the IRQ13 code elsewhere to avoid this unnecessary bloat. */
+    if (!uTagSrc && (u8Level & PDM_IRQ_LEVEL_HIGH)) /* FPU IRQ */
+    {
+        if (u8Level == PDM_IRQ_LEVEL_HIGH)
+            VBOXVMM_PDM_IRQ_HIGH(VMMGetCpu(pVM), 0, 0);
+        else
+            VBOXVMM_PDM_IRQ_HILO(VMMGetCpu(pVM), 0, 0);
+    }
+
+    int rc = VERR_PDM_NO_PIC_INSTANCE;
+/** @todo r=bird: This code is incorrect, as it ASSUMES the PIC and I/O APIC
+ *        are always ring-0 enabled! */
+    if (pVM->pdm.s.Pic.CTX_SUFF(pDevIns))
+    {
+        Assert(pVM->pdm.s.Pic.CTX_SUFF(pfnSetIrq));
+        pVM->pdm.s.Pic.CTX_SUFF(pfnSetIrq)(pVM->pdm.s.Pic.CTX_SUFF(pDevIns), u8Irq, u8Level, uTagSrc);
+        rc = VINF_SUCCESS;
+    }
+
+    if (pVM->pdm.s.IoApic.CTX_SUFF(pDevIns))
+    {
+        Assert(pVM->pdm.s.IoApic.CTX_SUFF(pfnSetIrq));
+
+        /*
+         * Apply Interrupt Source Override rules.
+         * See ACPI 4.0 specification 5.2.12.4 and 5.2.12.5 for details on
+         * interrupt source override.
+         * Shortly, ISA IRQ0 is electically connected to pin 2 on IO-APIC, and some OSes,
+         * notably recent OS X rely upon this configuration.
+         * If changing, also update override rules in MADT and MPS.
+         */
+        /* ISA IRQ0 routed to pin 2, all others ISA sources are identity mapped */
+        if (u8Irq == 0)
+            u8Irq = 2;
+
+        pVM->pdm.s.IoApic.CTX_SUFF(pfnSetIrq)(pVM->pdm.s.IoApic.CTX_SUFF(pDevIns), u8Irq, u8Level, uTagSrc);
+        rc = VINF_SUCCESS;
+    }
+
+    if (!uTagSrc && u8Level == PDM_IRQ_LEVEL_LOW)
+        VBOXVMM_PDM_IRQ_LOW(VMMGetCpu(pVM), 0, 0);
+    pdmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Sets the pending I/O APIC interrupt.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   u8Irq       The IRQ line.
+ * @param   u8Level     The new level.
+ * @param   uTagSrc     The IRQ tag and source tracer ID.
+ */
+VMM_INT_DECL(int) PDMIoApicSetIrq(PVM pVM, uint8_t u8Irq, uint8_t u8Level, uint32_t uTagSrc)
+{
+    if (pVM->pdm.s.IoApic.CTX_SUFF(pDevIns))
+    {
+        Assert(pVM->pdm.s.IoApic.CTX_SUFF(pfnSetIrq));
+        pVM->pdm.s.IoApic.CTX_SUFF(pfnSetIrq)(pVM->pdm.s.IoApic.CTX_SUFF(pDevIns), u8Irq, u8Level, uTagSrc);
+        return VINF_SUCCESS;
+    }
+    return VERR_PDM_NO_PIC_INSTANCE;
+}
+
+
+/**
+ * Broadcasts an EOI to the I/O APICs.
+ *
+ * @returns Strict VBox status code - only the following informational status codes:
+ * @retval  VINF_IOM_R3_MMIO_WRITE if the I/O APIC lock is contenteded and we're in R0 or RC.
+ * @retval  VINF_SUCCESS
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   uVector         The interrupt vector corresponding to the EOI.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) PDMIoApicBroadcastEoi(PVM pVM, uint8_t uVector)
+{
+    /* At present, we support only a maximum of one I/O APIC per-VM. If we ever implement having
+       multiple I/O APICs per-VM, we'll have to broadcast this EOI to all of the I/O APICs. */
+    if (pVM->pdm.s.IoApic.CTX_SUFF(pDevIns))
+    {
+        Assert(pVM->pdm.s.IoApic.CTX_SUFF(pfnSetEoi));
+        return pVM->pdm.s.IoApic.CTX_SUFF(pfnSetEoi)(pVM->pdm.s.IoApic.CTX_SUFF(pDevIns), uVector);
+    }
+
+    /* We shouldn't return failure if no I/O APIC is present. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Send a MSI to an I/O APIC.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCAddr      Request address.
+ * @param   uValue      Request value.
+ * @param   uTagSrc     The IRQ tag and source tracer ID.
+ */
+VMM_INT_DECL(int) PDMIoApicSendMsi(PVM pVM, RTGCPHYS GCAddr, uint32_t uValue, uint32_t uTagSrc)
+{
+    if (pVM->pdm.s.IoApic.CTX_SUFF(pDevIns))
+    {
+        Assert(pVM->pdm.s.IoApic.CTX_SUFF(pfnSendMsi));
+        pVM->pdm.s.IoApic.CTX_SUFF(pfnSendMsi)(pVM->pdm.s.IoApic.CTX_SUFF(pDevIns), GCAddr, uValue, uTagSrc);
+        return VINF_SUCCESS;
+    }
+    return VERR_PDM_NO_PIC_INSTANCE;
+}
+
+
+
+/**
+ * Returns the presence of an IO-APIC.
+ *
+ * @returns true if an IO-APIC is present.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(bool) PDMHasIoApic(PVM pVM)
+{
+    return pVM->pdm.s.IoApic.pDevInsR3 != NULL;
+}
+
+
+/**
+ * Returns the presence of an APIC.
+ *
+ * @returns true if an APIC is present.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(bool) PDMHasApic(PVM pVM)
+{
+    return pVM->pdm.s.Apic.pDevInsR3 != NIL_RTR3PTR;
+}
+
+
+/**
+ * Locks PDM.
+ * This might call back to Ring-3 in order to deal with lock contention in GC and R3.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+void pdmLock(PVMCC pVM)
+{
+#ifdef IN_RING3
+    int rc = PDMCritSectEnter(&pVM->pdm.s.CritSect, VERR_IGNORED);
+#else
+    int rc = PDMCritSectEnter(&pVM->pdm.s.CritSect, VERR_GENERAL_FAILURE);
+    if (rc == VERR_GENERAL_FAILURE)
+        rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PDM_LOCK, 0);
+#endif
+    AssertRC(rc);
+}
+
+
+/**
+ * Locks PDM but don't go to ring-3 if it's owned by someone.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns rc if we're in GC or R0 and can't get the lock.
+ * @param   pVM     The cross context VM structure.
+ * @param   rc      The RC to return in GC or R0 when we can't get the lock.
+ */
+int pdmLockEx(PVMCC pVM, int rc)
+{
+    return PDMCritSectEnter(&pVM->pdm.s.CritSect, rc);
+}
+
+
+/**
+ * Unlocks PDM.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+void pdmUnlock(PVMCC pVM)
+{
+    PDMCritSectLeave(&pVM->pdm.s.CritSect);
+}
+
+
+/**
+ * Converts ring 3 VMM heap pointer to a guest physical address
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pv              Ring-3 pointer.
+ * @param   pGCPhys         GC phys address (out).
+ */
+VMM_INT_DECL(int) PDMVmmDevHeapR3ToGCPhys(PVM pVM, RTR3PTR pv, RTGCPHYS *pGCPhys)
+{
+    if (RT_LIKELY(pVM->pdm.s.GCPhysVMMDevHeap != NIL_RTGCPHYS))
+    {
+        RTR3UINTPTR const offHeap = (RTR3UINTPTR)pv - (RTR3UINTPTR)pVM->pdm.s.pvVMMDevHeap;
+        if (RT_LIKELY(offHeap < pVM->pdm.s.cbVMMDevHeap))
+        {
+            *pGCPhys = pVM->pdm.s.GCPhysVMMDevHeap + offHeap;
+            return VINF_SUCCESS;
+        }
+
+        /* Don't assert here as this is called before we can catch ring-0 assertions. */
+        Log(("PDMVmmDevHeapR3ToGCPhys: pv=%p pvVMMDevHeap=%p cbVMMDevHeap=%#x\n",
+             pv, pVM->pdm.s.pvVMMDevHeap, pVM->pdm.s.cbVMMDevHeap));
+    }
+    else
+        Log(("PDMVmmDevHeapR3ToGCPhys: GCPhysVMMDevHeap=%RGp (pv=%p)\n", pVM->pdm.s.GCPhysVMMDevHeap, pv));
+    return VERR_PDM_DEV_HEAP_R3_TO_GCPHYS;
+}
+
+
+/**
+ * Checks if the vmm device heap is enabled (== vmm device's pci region mapped)
+ *
+ * @returns dev heap enabled status (true/false)
+ * @param   pVM             The cross context VM structure.
+ */
+VMM_INT_DECL(bool) PDMVmmDevHeapIsEnabled(PVM pVM)
+{
+    return pVM->pdm.s.GCPhysVMMDevHeap != NIL_RTGCPHYS;
+}
diff --git a/src/VBox/VMM/VMMAll/PDMAllCritSect.cpp b/src/VBox/VMM/VMMAll/PDMAllCritSect.cpp
new file mode 100644
index 00000000..65360907
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllCritSect.cpp
@@ -0,0 +1,830 @@
+/* $Id: PDMAllCritSect.cpp $ */
+/** @file
+ * PDM - Write-Only Critical Section, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_CRITSECT
+#include "PDMInternal.h"
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/vmm/hm.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/assert.h>
+#ifdef IN_RING3
+# include <iprt/lockvalidator.h>
+# include <iprt/semaphore.h>
+#endif
+#if defined(IN_RING3) || defined(IN_RING0)
+# include <iprt/thread.h>
+#endif
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The number loops to spin for in ring-3. */
+#define PDMCRITSECT_SPIN_COUNT_R3       20
+/** The number loops to spin for in ring-0. */
+#define PDMCRITSECT_SPIN_COUNT_R0       256
+/** The number loops to spin for in the raw-mode context. */
+#define PDMCRITSECT_SPIN_COUNT_RC       256
+
+
+/** Skips some of the overly paranoid atomic updates.
+ * Makes some assumptions about cache coherence, though not brave enough not to
+ * always end with an atomic update. */
+#define PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+
+/* Undefine the automatic VBOX_STRICT API mappings. */
+#undef PDMCritSectEnter
+#undef PDMCritSectTryEnter
+
+
+/**
+ * Gets the ring-3 native thread handle of the calling thread.
+ *
+ * @returns native thread handle (ring-3).
+ * @param   pCritSect           The critical section. This is used in R0 and RC.
+ */
+DECL_FORCE_INLINE(RTNATIVETHREAD) pdmCritSectGetNativeSelf(PCPDMCRITSECT pCritSect)
+{
+#ifdef IN_RING3
+    NOREF(pCritSect);
+    RTNATIVETHREAD  hNativeSelf = RTThreadNativeSelf();
+#else
+    AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, ("%RX32\n", pCritSect->s.Core.u32Magic),
+                    NIL_RTNATIVETHREAD);
+    PVMCC           pVM         = pCritSect->s.CTX_SUFF(pVM); AssertPtr(pVM);
+    PVMCPUCC        pVCpu       = VMMGetCpu(pVM);             AssertPtr(pVCpu);
+    RTNATIVETHREAD  hNativeSelf = pVCpu->hNativeThread;       Assert(hNativeSelf != NIL_RTNATIVETHREAD);
+#endif
+    return hNativeSelf;
+}
+
+
+/**
+ * Tail code called when we've won the battle for the lock.
+ *
+ * @returns VINF_SUCCESS.
+ *
+ * @param   pCritSect       The critical section.
+ * @param   hNativeSelf     The native handle of this thread.
+ * @param   pSrcPos         The source position of the lock operation.
+ */
+DECL_FORCE_INLINE(int) pdmCritSectEnterFirst(PPDMCRITSECT pCritSect, RTNATIVETHREAD hNativeSelf, PCRTLOCKVALSRCPOS pSrcPos)
+{
+    AssertMsg(pCritSect->s.Core.NativeThreadOwner == NIL_RTNATIVETHREAD, ("NativeThreadOwner=%p\n", pCritSect->s.Core.NativeThreadOwner));
+    Assert(!(pCritSect->s.Core.fFlags & PDMCRITSECT_FLAGS_PENDING_UNLOCK));
+
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+    pCritSect->s.Core.cNestings = 1;
+# else
+    ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 1);
+# endif
+    Assert(pCritSect->s.Core.cNestings == 1);
+    ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, hNativeSelf);
+
+# ifdef PDMCRITSECT_STRICT
+    RTLockValidatorRecExclSetOwner(pCritSect->s.Core.pValidatorRec, NIL_RTTHREAD, pSrcPos, true);
+# else
+    NOREF(pSrcPos);
+# endif
+
+    STAM_PROFILE_ADV_START(&pCritSect->s.StatLocked, l);
+    return VINF_SUCCESS;
+}
+
+
+#if defined(IN_RING3) || defined(IN_RING0)
+/**
+ * Deals with the contended case in ring-3 and ring-0.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_DESTROYED if destroyed.
+ *
+ * @param   pCritSect           The critsect.
+ * @param   hNativeSelf         The native thread handle.
+ * @param   pSrcPos             The source position of the lock operation.
+ */
+static int pdmR3R0CritSectEnterContended(PPDMCRITSECT pCritSect, RTNATIVETHREAD hNativeSelf, PCRTLOCKVALSRCPOS pSrcPos)
+{
+    /*
+     * Start waiting.
+     */
+    if (ASMAtomicIncS32(&pCritSect->s.Core.cLockers) == 0)
+        return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
+# ifdef IN_RING3
+    STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionR3);
+# else
+    STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock);
+# endif
+
+    /*
+     * The wait loop.
+     */
+    PSUPDRVSESSION  pSession    = pCritSect->s.CTX_SUFF(pVM)->pSession;
+    SUPSEMEVENT     hEvent      = (SUPSEMEVENT)pCritSect->s.Core.EventSem;
+# ifdef IN_RING3
+#  ifdef PDMCRITSECT_STRICT
+    RTTHREAD        hThreadSelf = RTThreadSelfAutoAdopt();
+    int rc2 = RTLockValidatorRecExclCheckOrder(pCritSect->s.Core.pValidatorRec, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
+    if (RT_FAILURE(rc2))
+        return rc2;
+#  else
+    RTTHREAD        hThreadSelf = RTThreadSelf();
+#  endif
+# endif
+    for (;;)
+    {
+        /*
+         * Do the wait.
+         *
+         * In ring-3 this gets cluttered by lock validation and thread state
+         * maintainence.
+         *
+         * In ring-0 we have to deal with the possibility that the thread has
+         * been signalled and the interruptible wait function returning
+         * immediately.  In that case we do normal R0/RC rcBusy handling.
+         */
+# ifdef IN_RING3
+#  ifdef PDMCRITSECT_STRICT
+        int rc9 = RTLockValidatorRecExclCheckBlocking(pCritSect->s.Core.pValidatorRec, hThreadSelf, pSrcPos,
+                                                      !(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NO_NESTING),
+                                                      RT_INDEFINITE_WAIT, RTTHREADSTATE_CRITSECT, true);
+        if (RT_FAILURE(rc9))
+            return rc9;
+#  else
+        RTThreadBlocking(hThreadSelf, RTTHREADSTATE_CRITSECT, true);
+#  endif
+        int rc = SUPSemEventWaitNoResume(pSession, hEvent, RT_INDEFINITE_WAIT);
+        RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_CRITSECT);
+# else  /* IN_RING0 */
+        int rc = SUPSemEventWaitNoResume(pSession, hEvent, RT_INDEFINITE_WAIT);
+# endif /* IN_RING0 */
+
+        /*
+         * Deal with the return code and critsect destruction.
+         */
+        if (RT_UNLIKELY(pCritSect->s.Core.u32Magic != RTCRITSECT_MAGIC))
+            return VERR_SEM_DESTROYED;
+        if (rc == VINF_SUCCESS)
+            return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
+        AssertMsg(rc == VERR_INTERRUPTED, ("rc=%Rrc\n", rc));
+
+# ifdef IN_RING0
+        /* Something is pending (signal, APC, debugger, whatever), just go back
+           to ring-3 so the kernel can deal with it when leaving kernel context.
+
+           Note! We've incremented cLockers already and cannot safely decrement
+                 it without creating a race with PDMCritSectLeave, resulting in
+                 spurious wakeups. */
+        PVMCC     pVM   = pCritSect->s.CTX_SUFF(pVM); AssertPtr(pVM);
+        PVMCPUCC  pVCpu = VMMGetCpu(pVM);             AssertPtr(pVCpu);
+        rc = VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VM_R0_PREEMPT, NULL);
+        AssertRC(rc);
+# endif
+    }
+    /* won't get here */
+}
+#endif /* IN_RING3 || IN_RING0 */
+
+
+/**
+ * Common worker for the debug and normal APIs.
+ *
+ * @returns VINF_SUCCESS if entered successfully.
+ * @returns rcBusy when encountering a busy critical section in GC/R0.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pCritSect           The PDM critical section to enter.
+ * @param   rcBusy              The status code to return when we're in GC or R0
+ * @param   pSrcPos             The source position of the lock operation.
+ */
+DECL_FORCE_INLINE(int) pdmCritSectEnter(PPDMCRITSECT pCritSect, int rcBusy, PCRTLOCKVALSRCPOS pSrcPos)
+{
+    Assert(pCritSect->s.Core.cNestings < 8);  /* useful to catch incorrect locking */
+    Assert(pCritSect->s.Core.cNestings >= 0);
+
+    /*
+     * If the critical section has already been destroyed, then inform the caller.
+     */
+    AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC,
+                    ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic),
+                    VERR_SEM_DESTROYED);
+
+    /*
+     * See if we're lucky.
+     */
+    /* NOP ... */
+    if (!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP))
+    { /* We're more likely to end up here with real critsects than a NOP one. */ }
+    else
+        return VINF_SUCCESS;
+
+    RTNATIVETHREAD hNativeSelf = pdmCritSectGetNativeSelf(pCritSect);
+    /* ... not owned ... */
+    if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1))
+        return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
+
+    /* ... or nested. */
+    if (pCritSect->s.Core.NativeThreadOwner == hNativeSelf)
+    {
+        Assert(pCritSect->s.Core.cNestings >= 1);
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+        pCritSect->s.Core.cNestings += 1;
+# else
+        ASMAtomicIncS32(&pCritSect->s.Core.cNestings);
+# endif
+        ASMAtomicIncS32(&pCritSect->s.Core.cLockers);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Spin for a bit without incrementing the counter.
+     */
+    /** @todo Move this to cfgm variables since it doesn't make sense to spin on UNI
+     *        cpu systems. */
+    int32_t cSpinsLeft = CTX_SUFF(PDMCRITSECT_SPIN_COUNT_);
+    while (cSpinsLeft-- > 0)
+    {
+        if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1))
+            return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
+        ASMNopPause();
+        /** @todo Should use monitor/mwait on e.g. &cLockers here, possibly with a
+           cli'ed pendingpreemption check up front using sti w/ instruction fusing
+           for avoiding races. Hmm ... This is assuming the other party is actually
+           executing code on another CPU ... which we could keep track of if we
+           wanted. */
+    }
+
+#ifdef IN_RING3
+    /*
+     * Take the slow path.
+     */
+    NOREF(rcBusy);
+    return pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos);
+
+#else
+# ifdef IN_RING0
+    /** @todo If preemption is disabled it means we're in VT-x/AMD-V context
+     *        and would be better off switching out of that while waiting for
+     *        the lock.  Several of the locks jumps back to ring-3 just to
+     *        get the lock, the ring-3 code will then call the kernel to do
+     *        the lock wait and when the call return it will call ring-0
+     *        again and resume via in setjmp style.  Not very efficient. */
+#  if 0
+    if (ASMIntAreEnabled()) /** @todo this can be handled as well by changing
+                             * callers not prepared for longjmp/blocking to
+                             * use PDMCritSectTryEnter. */
+    {
+        /*
+         * Leave HM context while waiting if necessary.
+         */
+        int rc;
+        if (RTThreadPreemptIsEnabled(NIL_RTTHREAD))
+        {
+            STAM_REL_COUNTER_ADD(&pCritSect->s.StatContentionRZLock,    1000000);
+            rc = pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos);
+        }
+        else
+        {
+            STAM_REL_COUNTER_ADD(&pCritSect->s.StatContentionRZLock, 1000000000);
+            PVMCC     pVM   = pCritSect->s.CTX_SUFF(pVM);
+            PVMCPUCC  pVCpu = VMMGetCpu(pVM);
+            HMR0Leave(pVM, pVCpu);
+            RTThreadPreemptRestore(NIL_RTTHREAD, XXX);
+
+            rc = pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos);
+
+            RTThreadPreemptDisable(NIL_RTTHREAD, XXX);
+            HMR0Enter(pVM, pVCpu);
+        }
+        return rc;
+    }
+#  else
+    /*
+     * We preemption hasn't been disabled, we can block here in ring-0.
+     */
+    if (   RTThreadPreemptIsEnabled(NIL_RTTHREAD)
+        && ASMIntAreEnabled())
+        return pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos);
+#  endif
+#endif /* IN_RING0 */
+
+    STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock);
+
+    /*
+     * Call ring-3 to acquire the critical section?
+     */
+    if (rcBusy == VINF_SUCCESS)
+    {
+        PVMCC     pVM   = pCritSect->s.CTX_SUFF(pVM); AssertPtr(pVM);
+        PVMCPUCC  pVCpu = VMMGetCpu(pVM);             AssertPtr(pVCpu);
+        return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_ENTER, MMHyperCCToR3(pVM, pCritSect));
+    }
+
+    /*
+     * Return busy.
+     */
+    LogFlow(("PDMCritSectEnter: locked => R3 (%Rrc)\n", rcBusy));
+    return rcBusy;
+#endif /* !IN_RING3 */
+}
+
+
+/**
+ * Enters a PDM critical section.
+ *
+ * @returns VINF_SUCCESS if entered successfully.
+ * @returns rcBusy when encountering a busy critical section in RC/R0.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pCritSect           The PDM critical section to enter.
+ * @param   rcBusy              The status code to return when we're in RC or R0
+ *                              and the section is busy.  Pass VINF_SUCCESS to
+ *                              acquired the critical section thru a ring-3
+ *                              call if necessary.
+ */
+VMMDECL(int) PDMCritSectEnter(PPDMCRITSECT pCritSect, int rcBusy)
+{
+#ifndef PDMCRITSECT_STRICT
+    return pdmCritSectEnter(pCritSect, rcBusy, NULL);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+    return pdmCritSectEnter(pCritSect, rcBusy, &SrcPos);
+#endif
+}
+
+
+/**
+ * Enters a PDM critical section, with location information for debugging.
+ *
+ * @returns VINF_SUCCESS if entered successfully.
+ * @returns rcBusy when encountering a busy critical section in RC/R0.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pCritSect           The PDM critical section to enter.
+ * @param   rcBusy              The status code to return when we're in RC or R0
+ *                              and the section is busy.   Pass VINF_SUCCESS to
+ *                              acquired the critical section thru a ring-3
+ *                              call if necessary.
+ * @param   uId                 Some kind of locking location ID.  Typically a
+ *                              return address up the stack.  Optional (0).
+ * @param   SRC_POS             The source position where to lock is being
+ *                              acquired from.  Optional.
+ */
+VMMDECL(int) PDMCritSectEnterDebug(PPDMCRITSECT pCritSect, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+#ifdef PDMCRITSECT_STRICT
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+    return pdmCritSectEnter(pCritSect, rcBusy, &SrcPos);
+#else
+    NOREF(uId); RT_SRC_POS_NOREF();
+    return pdmCritSectEnter(pCritSect, rcBusy, NULL);
+#endif
+}
+
+
+/**
+ * Common worker for the debug and normal APIs.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pCritSect   The critical section.
+ * @param   pSrcPos     The source position of the lock operation.
+ */
+static int pdmCritSectTryEnter(PPDMCRITSECT pCritSect, PCRTLOCKVALSRCPOS pSrcPos)
+{
+    /*
+     * If the critical section has already been destroyed, then inform the caller.
+     */
+    AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC,
+                    ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic),
+                    VERR_SEM_DESTROYED);
+
+    /*
+     * See if we're lucky.
+     */
+    /* NOP ... */
+    if (!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP))
+    { /* We're more likely to end up here with real critsects than a NOP one. */ }
+    else
+        return VINF_SUCCESS;
+
+    RTNATIVETHREAD hNativeSelf = pdmCritSectGetNativeSelf(pCritSect);
+    /* ... not owned ... */
+    if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1))
+        return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
+
+    /* ... or nested. */
+    if (pCritSect->s.Core.NativeThreadOwner == hNativeSelf)
+    {
+        Assert(pCritSect->s.Core.cNestings >= 1);
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+        pCritSect->s.Core.cNestings += 1;
+# else
+        ASMAtomicIncS32(&pCritSect->s.Core.cNestings);
+# endif
+        ASMAtomicIncS32(&pCritSect->s.Core.cLockers);
+        return VINF_SUCCESS;
+    }
+
+    /* no spinning */
+
+    /*
+     * Return busy.
+     */
+#ifdef IN_RING3
+    STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionR3);
+#else
+    STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock);
+#endif
+    LogFlow(("PDMCritSectTryEnter: locked\n"));
+    return VERR_SEM_BUSY;
+}
+
+
+/**
+ * Try enter a critical section.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pCritSect   The critical section.
+ */
+VMMDECL(int) PDMCritSectTryEnter(PPDMCRITSECT pCritSect)
+{
+#ifndef PDMCRITSECT_STRICT
+    return pdmCritSectTryEnter(pCritSect, NULL);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+    return pdmCritSectTryEnter(pCritSect, &SrcPos);
+#endif
+}
+
+
+/**
+ * Try enter a critical section, with location information for debugging.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pCritSect           The critical section.
+ * @param   uId                 Some kind of locking location ID.  Typically a
+ *                              return address up the stack.  Optional (0).
+ * @param   SRC_POS             The source position where to lock is being
+ *                              acquired from.  Optional.
+ */
+VMMDECL(int) PDMCritSectTryEnterDebug(PPDMCRITSECT pCritSect, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+#ifdef PDMCRITSECT_STRICT
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+    return pdmCritSectTryEnter(pCritSect, &SrcPos);
+#else
+    NOREF(uId); RT_SRC_POS_NOREF();
+    return pdmCritSectTryEnter(pCritSect, NULL);
+#endif
+}
+
+
+#ifdef IN_RING3
+/**
+ * Enters a PDM critical section.
+ *
+ * @returns VINF_SUCCESS if entered successfully.
+ * @returns rcBusy when encountering a busy critical section in GC/R0.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pCritSect           The PDM critical section to enter.
+ * @param   fCallRing3          Whether this is a VMMRZCallRing3()request.
+ */
+VMMR3DECL(int) PDMR3CritSectEnterEx(PPDMCRITSECT pCritSect, bool fCallRing3)
+{
+    int rc = PDMCritSectEnter(pCritSect, VERR_IGNORED);
+    if (    rc == VINF_SUCCESS
+        &&  fCallRing3
+        &&  pCritSect->s.Core.pValidatorRec
+        &&  pCritSect->s.Core.pValidatorRec->hThread != NIL_RTTHREAD)
+        RTLockValidatorRecExclReleaseOwnerUnchecked(pCritSect->s.Core.pValidatorRec);
+    return rc;
+}
+#endif /* IN_RING3 */
+
+
+/**
+ * Leaves a critical section entered with PDMCritSectEnter().
+ *
+ * @returns Indication whether we really exited the critical section.
+ * @retval  VINF_SUCCESS if we really exited.
+ * @retval  VINF_SEM_NESTED if we only reduced the nesting count.
+ * @retval  VERR_NOT_OWNER if you somehow ignore release assertions.
+ *
+ * @param   pCritSect           The PDM critical section to leave.
+ */
+VMMDECL(int) PDMCritSectLeave(PPDMCRITSECT pCritSect)
+{
+    AssertMsg(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic));
+    Assert(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC);
+
+    /* Check for NOP sections before asserting ownership. */
+    if (!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP))
+    { /* We're more likely to end up here with real critsects than a NOP one. */ }
+    else
+        return VINF_SUCCESS;
+
+    /*
+     * Always check that the caller is the owner (screw performance).
+     */
+    RTNATIVETHREAD const hNativeSelf = pdmCritSectGetNativeSelf(pCritSect);
+    AssertReleaseMsgReturn(pCritSect->s.Core.NativeThreadOwner == hNativeSelf,
+                           ("%p %s: %p != %p; cLockers=%d cNestings=%d\n", pCritSect, R3STRING(pCritSect->s.pszName),
+                            pCritSect->s.Core.NativeThreadOwner, hNativeSelf,
+                            pCritSect->s.Core.cLockers, pCritSect->s.Core.cNestings),
+                           VERR_NOT_OWNER);
+
+    /*
+     * Nested leave.
+     */
+    int32_t const cNestings = pCritSect->s.Core.cNestings;
+    Assert(cNestings >= 1);
+    if (cNestings > 1)
+    {
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+        pCritSect->s.Core.cNestings = cNestings - 1;
+# else
+        ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, cNestings - 1);
+# endif
+        ASMAtomicDecS32(&pCritSect->s.Core.cLockers);
+        Assert(pCritSect->s.Core.cLockers >= 0);
+        return VINF_SEM_NESTED;
+    }
+
+#ifdef IN_RING0
+# if 0 /** @todo Make SUPSemEventSignal interrupt safe (handle table++) and enable this for: defined(RT_OS_LINUX) || defined(RT_OS_OS2) */
+    if (1) /* SUPSemEventSignal is safe */
+# else
+    if (ASMIntAreEnabled())
+# endif
+#endif
+#if defined(IN_RING3) || defined(IN_RING0)
+    {
+        /*
+         * Leave for real.
+         */
+        /* update members. */
+        SUPSEMEVENT hEventToSignal  = pCritSect->s.hEventToSignal;
+        pCritSect->s.hEventToSignal = NIL_SUPSEMEVENT;
+# ifdef IN_RING3
+#  if defined(PDMCRITSECT_STRICT)
+        if (pCritSect->s.Core.pValidatorRec->hThread != NIL_RTTHREAD)
+            RTLockValidatorRecExclReleaseOwnerUnchecked(pCritSect->s.Core.pValidatorRec);
+#  endif
+        Assert(!pCritSect->s.Core.pValidatorRec || pCritSect->s.Core.pValidatorRec->hThread == NIL_RTTHREAD);
+# endif
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+        //pCritSect->s.Core.cNestings = 0; /* not really needed */
+        pCritSect->s.Core.NativeThreadOwner = NIL_RTNATIVETHREAD;
+# else
+        ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 0);
+        ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, NIL_RTNATIVETHREAD);
+# endif
+        ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK);
+
+        /* stop and decrement lockers. */
+        STAM_PROFILE_ADV_STOP(&pCritSect->s.StatLocked, l);
+        ASMCompilerBarrier();
+        if (ASMAtomicDecS32(&pCritSect->s.Core.cLockers) < 0)
+        { /* hopefully likely */ }
+        else
+        {
+            /* Someone is waiting, wake up one of them. */
+            SUPSEMEVENT     hEvent   = (SUPSEMEVENT)pCritSect->s.Core.EventSem;
+            PSUPDRVSESSION  pSession = pCritSect->s.CTX_SUFF(pVM)->pSession;
+            int rc = SUPSemEventSignal(pSession, hEvent);
+            AssertRC(rc);
+        }
+
+        /* Signal exit event. */
+        if (RT_LIKELY(hEventToSignal == NIL_SUPSEMEVENT))
+        { /* likely */ }
+        else
+        {
+            Log8(("Signalling %#p\n", hEventToSignal));
+            int rc = SUPSemEventSignal(pCritSect->s.CTX_SUFF(pVM)->pSession, hEventToSignal);
+            AssertRC(rc);
+        }
+
+# if defined(DEBUG_bird) && defined(IN_RING0)
+        VMMTrashVolatileXMMRegs();
+# endif
+    }
+#endif  /* IN_RING3 || IN_RING0 */
+#ifdef IN_RING0
+    else
+#endif
+#if defined(IN_RING0) || defined(IN_RC)
+    {
+        /*
+         * Try leave it.
+         */
+        if (pCritSect->s.Core.cLockers == 0)
+        {
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+            //pCritSect->s.Core.cNestings = 0; /* not really needed */
+# else
+            ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 0);
+# endif
+            RTNATIVETHREAD hNativeThread = pCritSect->s.Core.NativeThreadOwner;
+            ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK);
+            STAM_PROFILE_ADV_STOP(&pCritSect->s.StatLocked, l);
+
+            ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, NIL_RTNATIVETHREAD);
+            if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, -1, 0))
+                return VINF_SUCCESS;
+
+            /* darn, someone raced in on us. */
+            ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, hNativeThread);
+            STAM_PROFILE_ADV_START(&pCritSect->s.StatLocked, l);
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+            //pCritSect->s.Core.cNestings = 1;
+            Assert(pCritSect->s.Core.cNestings == 1);
+# else
+            //Assert(pCritSect->s.Core.cNestings == 0);
+            ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 1);
+# endif
+        }
+        ASMAtomicOrU32(&pCritSect->s.Core.fFlags, PDMCRITSECT_FLAGS_PENDING_UNLOCK);
+
+        /*
+         * Queue the request.
+         */
+        PVMCC       pVM   = pCritSect->s.CTX_SUFF(pVM);     AssertPtr(pVM);
+        PVMCPUCC    pVCpu = VMMGetCpu(pVM);                 AssertPtr(pVCpu);
+        uint32_t    i     = pVCpu->pdm.s.cQueuedCritSectLeaves++;
+        LogFlow(("PDMCritSectLeave: [%d]=%p => R3\n", i, pCritSect));
+        AssertFatal(i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectLeaves));
+        pVCpu->pdm.s.apQueuedCritSectLeaves[i] = MMHyperCCToR3(pVM, pCritSect);
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT);
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
+        STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves);
+        STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZUnlock);
+    }
+#endif /* IN_RING0 || IN_RC */
+
+    return VINF_SUCCESS;
+}
+
+
+#if defined(IN_RING0) || defined(IN_RING3)
+/**
+ * Schedule a event semaphore for signalling upon critsect exit.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VERR_TOO_MANY_SEMAPHORES if an event was already scheduled.
+ * @returns VERR_NOT_OWNER if we're not the critsect owner (ring-3 only).
+ * @returns VERR_SEM_DESTROYED if RTCritSectDelete was called while waiting.
+ *
+ * @param   pCritSect       The critical section.
+ * @param   hEventToSignal  The support driver event semaphore that should be
+ *                          signalled.
+ */
+VMMDECL(int) PDMHCCritSectScheduleExitEvent(PPDMCRITSECT pCritSect, SUPSEMEVENT hEventToSignal)
+{
+    AssertPtr(pCritSect);
+    Assert(!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP));
+    Assert(hEventToSignal != NIL_SUPSEMEVENT);
+# ifdef IN_RING3
+    if (RT_UNLIKELY(!RTCritSectIsOwner(&pCritSect->s.Core)))
+        return VERR_NOT_OWNER;
+# endif
+    if (RT_LIKELY(   pCritSect->s.hEventToSignal == NIL_RTSEMEVENT
+                  || pCritSect->s.hEventToSignal == hEventToSignal))
+    {
+        pCritSect->s.hEventToSignal = hEventToSignal;
+        return VINF_SUCCESS;
+    }
+    return VERR_TOO_MANY_SEMAPHORES;
+}
+#endif /* IN_RING0 || IN_RING3 */
+
+
+/**
+ * Checks the caller is the owner of the critical section.
+ *
+ * @returns true if owner.
+ * @returns false if not owner.
+ * @param   pCritSect   The critical section.
+ */
+VMMDECL(bool) PDMCritSectIsOwner(PCPDMCRITSECT pCritSect)
+{
+#ifdef IN_RING3
+    return RTCritSectIsOwner(&pCritSect->s.Core);
+#else
+    PVMCC     pVM   = pCritSect->s.CTX_SUFF(pVM); AssertPtr(pVM);
+    PVMCPUCC  pVCpu = VMMGetCpu(pVM);             AssertPtr(pVCpu);
+    if (pCritSect->s.Core.NativeThreadOwner != pVCpu->hNativeThread)
+        return false;
+    return (pCritSect->s.Core.fFlags & PDMCRITSECT_FLAGS_PENDING_UNLOCK) == 0
+        || pCritSect->s.Core.cNestings > 1;
+#endif
+}
+
+
+/**
+ * Checks the specified VCPU is the owner of the critical section.
+ *
+ * @returns true if owner.
+ * @returns false if not owner.
+ * @param   pCritSect   The critical section.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) PDMCritSectIsOwnerEx(PCPDMCRITSECT pCritSect, PVMCPUCC pVCpu)
+{
+#ifdef IN_RING3
+    NOREF(pVCpu);
+    return RTCritSectIsOwner(&pCritSect->s.Core);
+#else
+    Assert(VMCC_GET_CPU(pVCpu->CTX_SUFF(pVM), pVCpu->idCpu) == pVCpu);
+    if (pCritSect->s.Core.NativeThreadOwner != pVCpu->hNativeThread)
+        return false;
+    return (pCritSect->s.Core.fFlags & PDMCRITSECT_FLAGS_PENDING_UNLOCK) == 0
+        || pCritSect->s.Core.cNestings > 1;
+#endif
+}
+
+
+/**
+ * Checks if anyone is waiting on the critical section we own.
+ *
+ * @returns true if someone is waiting.
+ * @returns false if no one is waiting.
+ * @param   pCritSect   The critical section.
+ */
+VMMDECL(bool) PDMCritSectHasWaiters(PCPDMCRITSECT pCritSect)
+{
+    AssertReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, false);
+    Assert(pCritSect->s.Core.NativeThreadOwner == pdmCritSectGetNativeSelf(pCritSect));
+    return pCritSect->s.Core.cLockers >= pCritSect->s.Core.cNestings;
+}
+
+
+/**
+ * Checks if a critical section is initialized or not.
+ *
+ * @returns true if initialized.
+ * @returns false if not initialized.
+ * @param   pCritSect   The critical section.
+ */
+VMMDECL(bool) PDMCritSectIsInitialized(PCPDMCRITSECT pCritSect)
+{
+    return RTCritSectIsInitialized(&pCritSect->s.Core);
+}
+
+
+/**
+ * Gets the recursion depth.
+ *
+ * @returns The recursion depth.
+ * @param   pCritSect   The critical section.
+ */
+VMMDECL(uint32_t) PDMCritSectGetRecursion(PCPDMCRITSECT pCritSect)
+{
+    return RTCritSectGetRecursion(&pCritSect->s.Core);
+}
+
diff --git a/src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp b/src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp
new file mode 100644
index 00000000..36f85baa
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp
@@ -0,0 +1,97 @@
+/* $Id: PDMAllCritSectBoth.cpp $ */
+/** @file
+ * PDM - Code Common to Both Critical Section Types, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_CRITSECT
+#include "PDMInternal.h"
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/pdmcritsectrw.h>
+#include <VBox/vmm/vmcc.h>
+#include <iprt/errcore.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+
+
+#if defined(IN_RING3) || defined(IN_RING0)
+/**
+ * Process the critical sections (both types) queued for ring-3 'leave'.
+ *
+ * @param   pVCpu         The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(void) PDMCritSectBothFF(PVMCPUCC pVCpu)
+{
+    uint32_t i;
+    Assert(   pVCpu->pdm.s.cQueuedCritSectLeaves       > 0
+           || pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves > 0
+           || pVCpu->pdm.s.cQueuedCritSectRwExclLeaves > 0);
+
+    /* Shared leaves. */
+    i = pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves;
+    pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves = 0;
+    while (i-- > 0)
+    {
+# ifdef IN_RING3
+        PPDMCRITSECTRW pCritSectRw = pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i];
+# else
+        PPDMCRITSECTRW pCritSectRw = (PPDMCRITSECTRW)MMHyperR3ToCC(pVCpu->CTX_SUFF(pVM),
+                                                                   pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i]);
+# endif
+
+        pdmCritSectRwLeaveSharedQueued(pCritSectRw);
+        LogFlow(("PDMR3CritSectFF: %p (R/W)\n", pCritSectRw));
+    }
+
+    /* Last, exclusive leaves. */
+    i = pVCpu->pdm.s.cQueuedCritSectRwExclLeaves;
+    pVCpu->pdm.s.cQueuedCritSectRwExclLeaves = 0;
+    while (i-- > 0)
+    {
+# ifdef IN_RING3
+        PPDMCRITSECTRW pCritSectRw = pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i];
+# else
+        PPDMCRITSECTRW pCritSectRw = (PPDMCRITSECTRW)MMHyperR3ToCC(pVCpu->CTX_SUFF(pVM),
+                                                                   pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i]);
+# endif
+
+        pdmCritSectRwLeaveExclQueued(pCritSectRw);
+        LogFlow(("PDMR3CritSectFF: %p (R/W)\n", pCritSectRw));
+    }
+
+    /* Normal leaves. */
+    i = pVCpu->pdm.s.cQueuedCritSectLeaves;
+    pVCpu->pdm.s.cQueuedCritSectLeaves = 0;
+    while (i-- > 0)
+    {
+# ifdef IN_RING3
+        PPDMCRITSECT pCritSect = pVCpu->pdm.s.apQueuedCritSectLeaves[i];
+# else
+        PPDMCRITSECT pCritSect = (PPDMCRITSECT)MMHyperR3ToCC(pVCpu->CTX_SUFF(pVM), pVCpu->pdm.s.apQueuedCritSectLeaves[i]);
+# endif
+
+        PDMCritSectLeave(pCritSect);
+        LogFlow(("PDMR3CritSectFF: %p\n", pCritSect));
+    }
+
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PDM_CRITSECT);
+}
+#endif /* IN_RING3 || IN_RING0 */
+
diff --git a/src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp b/src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp
new file mode 100644
index 00000000..a4d9d626
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp
@@ -0,0 +1,1445 @@
+/* $Id: PDMAllCritSectRw.cpp $ */
+/** @file
+ * IPRT - Read/Write Critical Section, Generic.
+ */
+
+/*
+ * Copyright (C) 2009-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_CRITSECT
+#include "PDMInternal.h"
+#include <VBox/vmm/pdmcritsectrw.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/vmm/hm.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/assert.h>
+#ifdef IN_RING3
+# include <iprt/lockvalidator.h>
+# include <iprt/semaphore.h>
+#endif
+#if defined(IN_RING3) || defined(IN_RING0)
+# include <iprt/thread.h>
+#endif
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The number loops to spin for shared access in ring-3. */
+#define PDMCRITSECTRW_SHRD_SPIN_COUNT_R3       20
+/** The number loops to spin for shared access in ring-0. */
+#define PDMCRITSECTRW_SHRD_SPIN_COUNT_R0       128
+/** The number loops to spin for shared access in the raw-mode context. */
+#define PDMCRITSECTRW_SHRD_SPIN_COUNT_RC       128
+
+/** The number loops to spin for exclusive access in ring-3. */
+#define PDMCRITSECTRW_EXCL_SPIN_COUNT_R3       20
+/** The number loops to spin for exclusive access in ring-0. */
+#define PDMCRITSECTRW_EXCL_SPIN_COUNT_R0       256
+/** The number loops to spin for exclusive access in the raw-mode context. */
+#define PDMCRITSECTRW_EXCL_SPIN_COUNT_RC       256
+
+
+/* Undefine the automatic VBOX_STRICT API mappings. */
+#undef PDMCritSectRwEnterExcl
+#undef PDMCritSectRwTryEnterExcl
+#undef PDMCritSectRwEnterShared
+#undef PDMCritSectRwTryEnterShared
+
+
+/**
+ * Gets the ring-3 native thread handle of the calling thread.
+ *
+ * @returns native thread handle (ring-3).
+ * @param   pThis       The read/write critical section.  This is only used in
+ *                      R0 and RC.
+ */
+DECL_FORCE_INLINE(RTNATIVETHREAD) pdmCritSectRwGetNativeSelf(PCPDMCRITSECTRW pThis)
+{
+#ifdef IN_RING3
+    NOREF(pThis);
+    RTNATIVETHREAD  hNativeSelf = RTThreadNativeSelf();
+#else
+    AssertMsgReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, ("%RX32\n", pThis->s.Core.u32Magic),
+                    NIL_RTNATIVETHREAD);
+    PVMCC           pVM         = pThis->s.CTX_SUFF(pVM);   AssertPtr(pVM);
+    PVMCPUCC        pVCpu       = VMMGetCpu(pVM);           AssertPtr(pVCpu);
+    RTNATIVETHREAD  hNativeSelf = pVCpu->hNativeThread;     Assert(hNativeSelf != NIL_RTNATIVETHREAD);
+#endif
+    return hNativeSelf;
+}
+
+
+
+
+
+#ifdef IN_RING3
+/**
+ * Changes the lock validator sub-class of the read/write critical section.
+ *
+ * It is recommended to try make sure that nobody is using this critical section
+ * while changing the value.
+ *
+ * @returns The old sub-class.  RTLOCKVAL_SUB_CLASS_INVALID is returns if the
+ *          lock validator isn't compiled in or either of the parameters are
+ *          invalid.
+ * @param   pThis           Pointer to the read/write critical section.
+ * @param   uSubClass       The new sub-class value.
+ */
+VMMDECL(uint32_t) PDMR3CritSectRwSetSubClass(PPDMCRITSECTRW pThis, uint32_t uSubClass)
+{
+    AssertPtrReturn(pThis, RTLOCKVAL_SUB_CLASS_INVALID);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, RTLOCKVAL_SUB_CLASS_INVALID);
+# if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+    AssertReturn(!(pThis->s.Core.fFlags & RTCRITSECT_FLAGS_NOP), RTLOCKVAL_SUB_CLASS_INVALID);
+
+    RTLockValidatorRecSharedSetSubClass(pThis->s.Core.pValidatorRead, uSubClass);
+    return RTLockValidatorRecExclSetSubClass(pThis->s.Core.pValidatorWrite, uSubClass);
+# else
+    NOREF(uSubClass);
+    return RTLOCKVAL_SUB_CLASS_INVALID;
+# endif
+}
+#endif /* IN_RING3 */
+
+
+#ifdef IN_RING0
+/**
+ * Go back to ring-3 so the kernel can do signals, APCs and other fun things.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ */
+static void pdmR0CritSectRwYieldToRing3(PPDMCRITSECTRW pThis)
+{
+    PVMCC     pVM   = pThis->s.CTX_SUFF(pVM);     AssertPtr(pVM);
+    PVMCPUCC  pVCpu = VMMGetCpu(pVM);             AssertPtr(pVCpu);
+    int rc = VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VM_R0_PREEMPT, NULL);
+    AssertRC(rc);
+}
+#endif /* IN_RING0 */
+
+
+/**
+ * Worker that enters a read/write critical section with shard access.
+ *
+ * @returns VBox status code.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rcBusy      The busy return code for ring-0 and ring-3.
+ * @param   fTryOnly    Only try enter it, don't wait.
+ * @param   pSrcPos     The source position. (Can be NULL.)
+ * @param   fNoVal      No validation records.
+ */
+static int pdmCritSectRwEnterShared(PPDMCRITSECTRW pThis, int rcBusy, bool fTryOnly, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
+
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    NOREF(pSrcPos);
+    NOREF(fNoVal);
+#endif
+#ifdef IN_RING3
+    NOREF(rcBusy);
+#endif
+
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+    RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt();
+    if (!fTryOnly)
+    {
+        int            rc9;
+        RTNATIVETHREAD hNativeWriter;
+        ASMAtomicUoReadHandle(&pThis->s.Core.hNativeWriter, &hNativeWriter);
+        if (hNativeWriter != NIL_RTTHREAD && hNativeWriter == pdmCritSectRwGetNativeSelf(pThis))
+            rc9 = RTLockValidatorRecExclCheckOrder(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
+        else
+            rc9 = RTLockValidatorRecSharedCheckOrder(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
+        if (RT_FAILURE(rc9))
+            return rc9;
+    }
+#endif
+
+    /*
+     * Get cracking...
+     */
+    uint64_t u64State    = ASMAtomicReadU64(&pThis->s.Core.u64State);
+    uint64_t u64OldState = u64State;
+
+    for (;;)
+    {
+        if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
+        {
+            /* It flows in the right direction, try follow it before it changes. */
+            uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
+            c++;
+            Assert(c < RTCSRW_CNT_MASK / 2);
+            u64State &= ~RTCSRW_CNT_RD_MASK;
+            u64State |= c << RTCSRW_CNT_RD_SHIFT;
+            if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+            {
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+                if (!fNoVal)
+                    RTLockValidatorRecSharedAddOwner(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos);
+#endif
+                break;
+            }
+        }
+        else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0)
+        {
+            /* Wrong direction, but we're alone here and can simply try switch the direction. */
+            u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK);
+            u64State |= (UINT64_C(1) << RTCSRW_CNT_RD_SHIFT) | (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT);
+            if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+            {
+                Assert(!pThis->s.Core.fNeedReset);
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+                if (!fNoVal)
+                    RTLockValidatorRecSharedAddOwner(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos);
+#endif
+                break;
+            }
+        }
+        else
+        {
+            /* Is the writer perhaps doing a read recursion? */
+            RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pThis);
+            RTNATIVETHREAD hNativeWriter;
+            ASMAtomicUoReadHandle(&pThis->s.Core.hNativeWriter, &hNativeWriter);
+            if (hNativeSelf == hNativeWriter)
+            {
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+                if (!fNoVal)
+                {
+                    int rc9 = RTLockValidatorRecExclRecursionMixed(pThis->s.Core.pValidatorWrite, &pThis->s.Core.pValidatorRead->Core, pSrcPos);
+                    if (RT_FAILURE(rc9))
+                        return rc9;
+                }
+#endif
+                Assert(pThis->s.Core.cWriterReads < UINT32_MAX / 2);
+                ASMAtomicIncU32(&pThis->s.Core.cWriterReads);
+                STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterShared));
+                return VINF_SUCCESS; /* don't break! */
+            }
+
+            /*
+             * If we're only trying, return already.
+             */
+            if (fTryOnly)
+            {
+                STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterShared));
+                return VERR_SEM_BUSY;
+            }
+
+#if defined(IN_RING3) || defined(IN_RING0)
+# ifdef IN_RING0
+            if (   RTThreadPreemptIsEnabled(NIL_RTTHREAD)
+                && ASMIntAreEnabled())
+# endif
+            {
+                /*
+                 * Add ourselves to the queue and wait for the direction to change.
+                 */
+                uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
+                c++;
+                Assert(c < RTCSRW_CNT_MASK / 2);
+
+                uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
+                cWait++;
+                Assert(cWait <= c);
+                Assert(cWait < RTCSRW_CNT_MASK / 2);
+
+                u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK);
+                u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT);
+
+                if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+                {
+                    for (uint32_t iLoop = 0; ; iLoop++)
+                    {
+                        int rc;
+# ifdef IN_RING3
+#  if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+                        rc = RTLockValidatorRecSharedCheckBlocking(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos, true,
+                                                                   RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_READ, false);
+                        if (RT_SUCCESS(rc))
+#  else
+                        RTTHREAD hThreadSelf = RTThreadSelf();
+                        RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false);
+#  endif
+# endif
+                        {
+                            for (;;)
+                            {
+                                rc = SUPSemEventMultiWaitNoResume(pThis->s.CTX_SUFF(pVM)->pSession,
+                                                                  (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead,
+                                                                  RT_INDEFINITE_WAIT);
+                                if (   rc != VERR_INTERRUPTED
+                                    || pThis->s.Core.u32Magic != RTCRITSECTRW_MAGIC)
+                                    break;
+# ifdef IN_RING0
+                                pdmR0CritSectRwYieldToRing3(pThis);
+# endif
+                            }
+# ifdef IN_RING3
+                            RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_READ);
+# endif
+                            if (pThis->s.Core.u32Magic != RTCRITSECTRW_MAGIC)
+                                return VERR_SEM_DESTROYED;
+                        }
+                        if (RT_FAILURE(rc))
+                        {
+                            /* Decrement the counts and return the error. */
+                            for (;;)
+                            {
+                                u64OldState = u64State = ASMAtomicReadU64(&pThis->s.Core.u64State);
+                                c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; Assert(c > 0);
+                                c--;
+                                cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT; Assert(cWait > 0);
+                                cWait--;
+                                u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK);
+                                u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT);
+                                if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+                                    break;
+                            }
+                            return rc;
+                        }
+
+                        Assert(pThis->s.Core.fNeedReset);
+                        u64State = ASMAtomicReadU64(&pThis->s.Core.u64State);
+                        if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
+                            break;
+                        AssertMsg(iLoop < 1, ("%u\n", iLoop));
+                    }
+
+                    /* Decrement the wait count and maybe reset the semaphore (if we're last). */
+                    for (;;)
+                    {
+                        u64OldState = u64State;
+
+                        cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
+                        Assert(cWait > 0);
+                        cWait--;
+                        u64State &= ~RTCSRW_WAIT_CNT_RD_MASK;
+                        u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT;
+
+                        if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+                        {
+                            if (cWait == 0)
+                            {
+                                if (ASMAtomicXchgBool(&pThis->s.Core.fNeedReset, false))
+                                {
+                                    int rc = SUPSemEventMultiReset(pThis->s.CTX_SUFF(pVM)->pSession,
+                                                                   (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead);
+                                    AssertRCReturn(rc, rc);
+                                }
+                            }
+                            break;
+                        }
+                        u64State = ASMAtomicReadU64(&pThis->s.Core.u64State);
+                    }
+
+# if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+                    if (!fNoVal)
+                        RTLockValidatorRecSharedAddOwner(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos);
+# endif
+                    break;
+                }
+            }
+#endif /* IN_RING3 || IN_RING3 */
+#ifndef IN_RING3
+# ifdef IN_RING0
+            else
+# endif
+            {
+                /*
+                 * We cannot call SUPSemEventMultiWaitNoResume in this context. Go
+                 * back to ring-3 and do it there or return rcBusy.
+                 */
+                STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterShared));
+                if (rcBusy == VINF_SUCCESS)
+                {
+                    PVMCC     pVM   = pThis->s.CTX_SUFF(pVM);     AssertPtr(pVM);
+                    PVMCPUCC  pVCpu = VMMGetCpu(pVM);             AssertPtr(pVCpu);
+                    /** @todo Should actually do this in via VMMR0.cpp instead of going all the way
+                     *        back to ring-3. Goes for both kind of crit sects. */
+                    return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_SHARED, MMHyperCCToR3(pVM, pThis));
+                }
+                return rcBusy;
+            }
+#endif /* !IN_RING3 */
+        }
+
+        if (pThis->s.Core.u32Magic != RTCRITSECTRW_MAGIC)
+            return VERR_SEM_DESTROYED;
+
+        ASMNopPause();
+        u64State = ASMAtomicReadU64(&pThis->s.Core.u64State);
+        u64OldState = u64State;
+    }
+
+    /* got it! */
+    STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterShared));
+    Assert((ASMAtomicReadU64(&pThis->s.Core.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT));
+    return VINF_SUCCESS;
+
+}
+
+
+/**
+ * Enter a critical section with shared (read) access.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  rcBusy if in ring-0 or raw-mode context and it is busy.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rcBusy      The status code to return when we're in RC or R0 and the
+ *                      section is busy.   Pass VINF_SUCCESS to acquired the
+ *                      critical section thru a ring-3 call if necessary.
+ * @sa      PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterShared,
+ *          PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwLeaveShared,
+ *          RTCritSectRwEnterShared.
+ */
+VMMDECL(int) PDMCritSectRwEnterShared(PPDMCRITSECTRW pThis, int rcBusy)
+{
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterShared(pThis, rcBusy, false /*fTryOnly*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+    return pdmCritSectRwEnterShared(pThis, rcBusy, false /*fTryOnly*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+/**
+ * Enter a critical section with shared (read) access.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  rcBusy if in ring-0 or raw-mode context and it is busy.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rcBusy      The status code to return when we're in RC or R0 and the
+ *                      section is busy.   Pass VINF_SUCCESS to acquired the
+ *                      critical section thru a ring-3 call if necessary.
+ * @param   uId         Where we're entering the section.
+ * @param   SRC_POS     The source position.
+ * @sa      PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared,
+ *          PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwLeaveShared,
+ *          RTCritSectRwEnterSharedDebug.
+ */
+VMMDECL(int) PDMCritSectRwEnterSharedDebug(PPDMCRITSECTRW pThis, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+    NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterShared(pThis, rcBusy, false /*fTryOnly*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+    return pdmCritSectRwEnterShared(pThis, rcBusy, false /*fTryOnly*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+/**
+ * Try enter a critical section with shared (read) access.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwEnterShared,
+ *          PDMCritSectRwEnterSharedDebug, PDMCritSectRwLeaveShared,
+ *          RTCritSectRwTryEnterShared.
+ */
+VMMDECL(int) PDMCritSectRwTryEnterShared(PPDMCRITSECTRW pThis)
+{
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterShared(pThis, VERR_SEM_BUSY, true /*fTryOnly*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+    return pdmCritSectRwEnterShared(pThis, VERR_SEM_BUSY, true /*fTryOnly*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+/**
+ * Try enter a critical section with shared (read) access.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   uId         Where we're entering the section.
+ * @param   SRC_POS     The source position.
+ * @sa      PDMCritSectRwTryEnterShared, PDMCritSectRwEnterShared,
+ *          PDMCritSectRwEnterSharedDebug, PDMCritSectRwLeaveShared,
+ *          RTCritSectRwTryEnterSharedDebug.
+ */
+VMMDECL(int) PDMCritSectRwTryEnterSharedDebug(PPDMCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+    NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterShared(pThis, VERR_SEM_BUSY, true /*fTryOnly*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+    return pdmCritSectRwEnterShared(pThis, VERR_SEM_BUSY, true /*fTryOnly*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+#ifdef IN_RING3
+/**
+ * Enters a PDM read/write critical section with shared (read) access.
+ *
+ * @returns VINF_SUCCESS if entered successfully.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   fCallRing3  Whether this is a VMMRZCallRing3()request.
+ */
+VMMR3DECL(int) PDMR3CritSectRwEnterSharedEx(PPDMCRITSECTRW pThis, bool fCallRing3)
+{
+    return pdmCritSectRwEnterShared(pThis, VERR_SEM_BUSY, false /*fTryAgain*/, NULL, fCallRing3);
+}
+#endif
+
+
+/**
+ * Leave a critical section held with shared access.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   fNoVal      No validation records (i.e. queued release).
+ * @sa      PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared,
+ *          PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterSharedDebug,
+ *          PDMCritSectRwLeaveExcl, RTCritSectRwLeaveShared.
+ */
+static int pdmCritSectRwLeaveSharedWorker(PPDMCRITSECTRW pThis, bool fNoVal)
+{
+    /*
+     * Validate handle.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
+
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    NOREF(fNoVal);
+#endif
+
+    /*
+     * Check the direction and take action accordingly.
+     */
+    uint64_t u64State    = ASMAtomicReadU64(&pThis->s.Core.u64State);
+    uint64_t u64OldState = u64State;
+    if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
+    {
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+        if (fNoVal)
+            Assert(!RTLockValidatorRecSharedIsOwner(pThis->s.Core.pValidatorRead, NIL_RTTHREAD));
+        else
+        {
+            int rc9 = RTLockValidatorRecSharedCheckAndRelease(pThis->s.Core.pValidatorRead, NIL_RTTHREAD);
+            if (RT_FAILURE(rc9))
+                return rc9;
+        }
+#endif
+        for (;;)
+        {
+            uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
+            AssertReturn(c > 0, VERR_NOT_OWNER);
+            c--;
+
+            if (   c > 0
+                || (u64State & RTCSRW_CNT_WR_MASK) == 0)
+            {
+                /* Don't change the direction. */
+                u64State &= ~RTCSRW_CNT_RD_MASK;
+                u64State |= c << RTCSRW_CNT_RD_SHIFT;
+                if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+                    break;
+            }
+            else
+            {
+#if defined(IN_RING3) || defined(IN_RING0)
+# ifdef IN_RING0
+                if (   RTThreadPreemptIsEnabled(NIL_RTTHREAD)
+                    && ASMIntAreEnabled())
+# endif
+                {
+                    /* Reverse the direction and signal the writer threads. */
+                    u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_DIR_MASK);
+                    u64State |= RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT;
+                    if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+                    {
+                        int rc = SUPSemEventSignal(pThis->s.CTX_SUFF(pVM)->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
+                        AssertRC(rc);
+                        break;
+                    }
+                }
+#endif /* IN_RING3 || IN_RING0 */
+#ifndef IN_RING3
+# ifdef IN_RING0
+                else
+# endif
+                {
+                    /* Queue the exit request (ring-3). */
+                    PVMCC       pVM   = pThis->s.CTX_SUFF(pVM);         AssertPtr(pVM);
+                    PVMCPUCC    pVCpu = VMMGetCpu(pVM);                 AssertPtr(pVCpu);
+                    uint32_t    i     = pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves++;
+                    LogFlow(("PDMCritSectRwLeaveShared: [%d]=%p => R3 c=%d (%#llx)\n", i, pThis, c, u64State));
+                    AssertFatal(i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves));
+                    pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i] = MMHyperCCToR3(pVM, pThis);
+                    VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT);
+                    VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
+                    STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves);
+                    STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveShared);
+                    break;
+                }
+#endif
+            }
+
+            ASMNopPause();
+            u64State = ASMAtomicReadU64(&pThis->s.Core.u64State);
+            u64OldState = u64State;
+        }
+    }
+    else
+    {
+        RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pThis);
+        RTNATIVETHREAD hNativeWriter;
+        ASMAtomicUoReadHandle(&pThis->s.Core.hNativeWriter, &hNativeWriter);
+        AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
+        AssertReturn(pThis->s.Core.cWriterReads > 0, VERR_NOT_OWNER);
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+        if (!fNoVal)
+        {
+            int rc = RTLockValidatorRecExclUnwindMixed(pThis->s.Core.pValidatorWrite, &pThis->s.Core.pValidatorRead->Core);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+#endif
+        ASMAtomicDecU32(&pThis->s.Core.cWriterReads);
+    }
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Leave a critical section held with shared access.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared,
+ *          PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterSharedDebug,
+ *          PDMCritSectRwLeaveExcl, RTCritSectRwLeaveShared.
+ */
+VMMDECL(int) PDMCritSectRwLeaveShared(PPDMCRITSECTRW pThis)
+{
+    return pdmCritSectRwLeaveSharedWorker(pThis, false /*fNoVal*/);
+}
+
+
+#if defined(IN_RING3) || defined(IN_RING0)
+/**
+ * PDMCritSectBothFF interface.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ */
+void pdmCritSectRwLeaveSharedQueued(PPDMCRITSECTRW pThis)
+{
+    pdmCritSectRwLeaveSharedWorker(pThis, true /*fNoVal*/);
+}
+#endif
+
+
+/**
+ * Worker that enters a read/write critical section with exclusive access.
+ *
+ * @returns VBox status code.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rcBusy      The busy return code for ring-0 and ring-3.
+ * @param   fTryOnly    Only try enter it, don't wait.
+ * @param   pSrcPos     The source position. (Can be NULL.)
+ * @param   fNoVal      No validation records.
+ */
+static int pdmCritSectRwEnterExcl(PPDMCRITSECTRW pThis, int rcBusy, bool fTryOnly, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
+
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    NOREF(pSrcPos);
+    NOREF(fNoVal);
+#endif
+#ifdef IN_RING3
+    NOREF(rcBusy);
+#endif
+
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+    RTTHREAD hThreadSelf = NIL_RTTHREAD;
+    if (!fTryOnly)
+    {
+        hThreadSelf = RTThreadSelfAutoAdopt();
+        int rc9 = RTLockValidatorRecExclCheckOrder(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
+        if (RT_FAILURE(rc9))
+            return rc9;
+    }
+#endif
+
+    /*
+     * Check if we're already the owner and just recursing.
+     */
+    RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pThis);
+    RTNATIVETHREAD hNativeWriter;
+    ASMAtomicUoReadHandle(&pThis->s.Core.hNativeWriter, &hNativeWriter);
+    if (hNativeSelf == hNativeWriter)
+    {
+        Assert((ASMAtomicReadU64(&pThis->s.Core.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT));
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+        if (!fNoVal)
+        {
+            int rc9 = RTLockValidatorRecExclRecursion(pThis->s.Core.pValidatorWrite, pSrcPos);
+            if (RT_FAILURE(rc9))
+                return rc9;
+        }
+#endif
+        Assert(pThis->s.Core.cWriteRecursions < UINT32_MAX / 2);
+        STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterExcl));
+        ASMAtomicIncU32(&pThis->s.Core.cWriteRecursions);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Get cracking.
+     */
+    uint64_t u64State    = ASMAtomicReadU64(&pThis->s.Core.u64State);
+    uint64_t u64OldState = u64State;
+
+    for (;;)
+    {
+        if (   (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)
+            || (u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) != 0)
+        {
+            /* It flows in the right direction, try follow it before it changes. */
+            uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
+            c++;
+            Assert(c < RTCSRW_CNT_MASK / 2);
+            u64State &= ~RTCSRW_CNT_WR_MASK;
+            u64State |= c << RTCSRW_CNT_WR_SHIFT;
+            if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+                break;
+        }
+        else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0)
+        {
+            /* Wrong direction, but we're alone here and can simply try switch the direction. */
+            u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK);
+            u64State |= (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT);
+            if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+                break;
+        }
+        else if (fTryOnly)
+        {
+            /* Wrong direction and we're not supposed to wait, just return. */
+            STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl));
+            return VERR_SEM_BUSY;
+        }
+        else
+        {
+            /* Add ourselves to the write count and break out to do the wait. */
+            uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
+            c++;
+            Assert(c < RTCSRW_CNT_MASK / 2);
+            u64State &= ~RTCSRW_CNT_WR_MASK;
+            u64State |= c << RTCSRW_CNT_WR_SHIFT;
+            if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+                break;
+        }
+
+        if (pThis->s.Core.u32Magic != RTCRITSECTRW_MAGIC)
+            return VERR_SEM_DESTROYED;
+
+        ASMNopPause();
+        u64State = ASMAtomicReadU64(&pThis->s.Core.u64State);
+        u64OldState = u64State;
+    }
+
+    /*
+     * If we're in write mode now try grab the ownership. Play fair if there
+     * are threads already waiting.
+     */
+    bool fDone = (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)
+#if defined(IN_RING3)
+              && (  ((u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT) == 1
+                  || fTryOnly)
+#endif
+               ;
+    if (fDone)
+        ASMAtomicCmpXchgHandle(&pThis->s.Core.hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
+    if (!fDone)
+    {
+        STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl));
+
+#if defined(IN_RING3) || defined(IN_RING0)
+        if (   !fTryOnly
+# ifdef IN_RING0
+            && RTThreadPreemptIsEnabled(NIL_RTTHREAD)
+            && ASMIntAreEnabled()
+# endif
+           )
+        {
+
+            /*
+             * Wait for our turn.
+             */
+            for (uint32_t iLoop = 0; ; iLoop++)
+            {
+                int rc;
+# ifdef IN_RING3
+#  ifdef PDMCRITSECTRW_STRICT
+                if (hThreadSelf == NIL_RTTHREAD)
+                    hThreadSelf = RTThreadSelfAutoAdopt();
+                rc = RTLockValidatorRecExclCheckBlocking(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, true,
+                                                         RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_WRITE, false);
+                if (RT_SUCCESS(rc))
+#  else
+                RTTHREAD hThreadSelf = RTThreadSelf();
+                RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_WRITE, false);
+#  endif
+# endif
+                {
+                    for (;;)
+                    {
+                        rc = SUPSemEventWaitNoResume(pThis->s.CTX_SUFF(pVM)->pSession,
+                                                     (SUPSEMEVENT)pThis->s.Core.hEvtWrite,
+                                                     RT_INDEFINITE_WAIT);
+                        if (   rc != VERR_INTERRUPTED
+                            || pThis->s.Core.u32Magic != RTCRITSECTRW_MAGIC)
+                            break;
+# ifdef IN_RING0
+                        pdmR0CritSectRwYieldToRing3(pThis);
+# endif
+                    }
+# ifdef IN_RING3
+                    RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
+# endif
+                    if (pThis->s.Core.u32Magic != RTCRITSECTRW_MAGIC)
+                        return VERR_SEM_DESTROYED;
+                }
+                if (RT_FAILURE(rc))
+                {
+                    /* Decrement the counts and return the error. */
+                    for (;;)
+                    {
+                        u64OldState = u64State = ASMAtomicReadU64(&pThis->s.Core.u64State);
+                        uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; Assert(c > 0);
+                        c--;
+                        u64State &= ~RTCSRW_CNT_WR_MASK;
+                        u64State |= c << RTCSRW_CNT_WR_SHIFT;
+                        if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+                            break;
+                    }
+                    return rc;
+                }
+
+                u64State = ASMAtomicReadU64(&pThis->s.Core.u64State);
+                if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
+                {
+                    ASMAtomicCmpXchgHandle(&pThis->s.Core.hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
+                    if (fDone)
+                        break;
+                }
+                AssertMsg(iLoop < 1000, ("%u\n", iLoop)); /* may loop a few times here... */
+            }
+
+        }
+        else
+#endif /* IN_RING3 || IN_RING0 */
+        {
+#ifdef IN_RING3
+            /* TryEnter call - decrement the number of (waiting) writers.  */
+#else
+            /* We cannot call SUPSemEventWaitNoResume in this context. Go back to
+               ring-3 and do it there or return rcBusy. */
+#endif
+
+            for (;;)
+            {
+                u64OldState = u64State = ASMAtomicReadU64(&pThis->s.Core.u64State);
+                uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; Assert(c > 0);
+                c--;
+                u64State &= ~RTCSRW_CNT_WR_MASK;
+                u64State |= c << RTCSRW_CNT_WR_SHIFT;
+                if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+                    break;
+            }
+
+#ifdef IN_RING3
+            return VERR_SEM_BUSY;
+#else
+            if (rcBusy == VINF_SUCCESS)
+            {
+                Assert(!fTryOnly);
+                PVMCC     pVM   = pThis->s.CTX_SUFF(pVM);     AssertPtr(pVM);
+                PVMCPUCC  pVCpu = VMMGetCpu(pVM);             AssertPtr(pVCpu);
+                /** @todo Should actually do this in via VMMR0.cpp instead of going all the way
+                 *        back to ring-3. Goes for both kind of crit sects. */
+                return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_EXCL, MMHyperCCToR3(pVM, pThis));
+            }
+            return rcBusy;
+#endif
+        }
+    }
+
+    /*
+     * Got it!
+     */
+    Assert((ASMAtomicReadU64(&pThis->s.Core.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT));
+    ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 1);
+    Assert(pThis->s.Core.cWriterReads == 0);
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+    if (!fNoVal)
+        RTLockValidatorRecExclSetOwner(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, true);
+#endif
+    STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterExcl));
+    STAM_PROFILE_ADV_START(&pThis->s.StatWriteLocked, swl);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Try enter a critical section with exclusive (write) access.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  rcBusy if in ring-0 or raw-mode context and it is busy.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rcBusy      The status code to return when we're in RC or R0 and the
+ *                      section is busy.   Pass VINF_SUCCESS to acquired the
+ *                      critical section thru a ring-3 call if necessary.
+ * @sa      PDMCritSectRwEnterExclDebug, PDMCritSectRwTryEnterExcl,
+ *          PDMCritSectRwTryEnterExclDebug,
+ *          PDMCritSectEnterDebug, PDMCritSectEnter,
+ * RTCritSectRwEnterExcl.
+ */
+VMMDECL(int) PDMCritSectRwEnterExcl(PPDMCRITSECTRW pThis, int rcBusy)
+{
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterExcl(pThis, rcBusy, false /*fTryAgain*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+    return pdmCritSectRwEnterExcl(pThis, rcBusy, false /*fTryAgain*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+/**
+ * Try enter a critical section with exclusive (write) access.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  rcBusy if in ring-0 or raw-mode context and it is busy.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rcBusy      The status code to return when we're in RC or R0 and the
+ *                      section is busy.   Pass VINF_SUCCESS to acquired the
+ *                      critical section thru a ring-3 call if necessary.
+ * @param   uId         Where we're entering the section.
+ * @param   SRC_POS     The source position.
+ * @sa      PDMCritSectRwEnterExcl, PDMCritSectRwTryEnterExcl,
+ *          PDMCritSectRwTryEnterExclDebug,
+ *          PDMCritSectEnterDebug, PDMCritSectEnter,
+ *          RTCritSectRwEnterExclDebug.
+ */
+VMMDECL(int) PDMCritSectRwEnterExclDebug(PPDMCRITSECTRW pThis, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+    NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterExcl(pThis, rcBusy, false /*fTryAgain*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+    return pdmCritSectRwEnterExcl(pThis, rcBusy, false /*fTryAgain*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+/**
+ * Try enter a critical section with exclusive (write) access.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwEnterExcl, PDMCritSectRwTryEnterExclDebug,
+ *          PDMCritSectRwEnterExclDebug,
+ *          PDMCritSectTryEnter, PDMCritSectTryEnterDebug,
+ *          RTCritSectRwTryEnterExcl.
+ */
+VMMDECL(int) PDMCritSectRwTryEnterExcl(PPDMCRITSECTRW pThis)
+{
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterExcl(pThis, VERR_SEM_BUSY, true /*fTryAgain*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+    return pdmCritSectRwEnterExcl(pThis, VERR_SEM_BUSY, true /*fTryAgain*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+/**
+ * Try enter a critical section with exclusive (write) access.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   uId         Where we're entering the section.
+ * @param   SRC_POS     The source position.
+ * @sa      PDMCritSectRwTryEnterExcl, PDMCritSectRwEnterExcl,
+ *          PDMCritSectRwEnterExclDebug,
+ *          PDMCritSectTryEnterDebug, PDMCritSectTryEnter,
+ *          RTCritSectRwTryEnterExclDebug.
+ */
+VMMDECL(int) PDMCritSectRwTryEnterExclDebug(PPDMCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+    NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterExcl(pThis, VERR_SEM_BUSY, true /*fTryAgain*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+    return pdmCritSectRwEnterExcl(pThis, VERR_SEM_BUSY, true /*fTryAgain*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+#ifdef IN_RING3
+/**
+ * Enters a PDM read/write critical section with exclusive (write) access.
+ *
+ * @returns VINF_SUCCESS if entered successfully.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   fCallRing3  Whether this is a VMMRZCallRing3()request.
+ */
+VMMR3DECL(int) PDMR3CritSectRwEnterExclEx(PPDMCRITSECTRW pThis, bool fCallRing3)
+{
+    return pdmCritSectRwEnterExcl(pThis, VERR_SEM_BUSY, false /*fTryAgain*/, NULL, fCallRing3 /*fNoVal*/);
+}
+#endif /* IN_RING3 */
+
+
+/**
+ * Leave a critical section held exclusively.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   fNoVal      No validation records (i.e. queued release).
+ * @sa      PDMCritSectRwLeaveShared, RTCritSectRwLeaveExcl.
+ */
+static int pdmCritSectRwLeaveExclWorker(PPDMCRITSECTRW pThis, bool fNoVal)
+{
+    /*
+     * Validate handle.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
+
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    NOREF(fNoVal);
+#endif
+
+    RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pThis);
+    RTNATIVETHREAD hNativeWriter;
+    ASMAtomicUoReadHandle(&pThis->s.Core.hNativeWriter, &hNativeWriter);
+    AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
+
+    /*
+     * Unwind one recursion. Is it the final one?
+     */
+    if (pThis->s.Core.cWriteRecursions == 1)
+    {
+        AssertReturn(pThis->s.Core.cWriterReads == 0, VERR_WRONG_ORDER); /* (must release all read recursions before the final write.) */
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+        if (fNoVal)
+            Assert(pThis->s.Core.pValidatorWrite->hThread == NIL_RTTHREAD);
+        else
+        {
+            int rc9 = RTLockValidatorRecExclReleaseOwner(pThis->s.Core.pValidatorWrite, true);
+            if (RT_FAILURE(rc9))
+                return rc9;
+        }
+#endif
+        /*
+         * Update the state.
+         */
+#if defined(IN_RING3) || defined(IN_RING0)
+# ifdef IN_RING0
+        if (   RTThreadPreemptIsEnabled(NIL_RTTHREAD)
+            && ASMIntAreEnabled())
+# endif
+        {
+            ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 0);
+            STAM_PROFILE_ADV_STOP(&pThis->s.StatWriteLocked, swl);
+            ASMAtomicWriteHandle(&pThis->s.Core.hNativeWriter, NIL_RTNATIVETHREAD);
+
+            for (;;)
+            {
+                uint64_t u64State    = ASMAtomicReadU64(&pThis->s.Core.u64State);
+                uint64_t u64OldState = u64State;
+
+                uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
+                Assert(c > 0);
+                c--;
+
+                if (   c > 0
+                    || (u64State & RTCSRW_CNT_RD_MASK) == 0)
+                {
+                    /* Don't change the direction, wake up the next writer if any. */
+                    u64State &= ~RTCSRW_CNT_WR_MASK;
+                    u64State |= c << RTCSRW_CNT_WR_SHIFT;
+                    if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+                    {
+                        if (c > 0)
+                        {
+                            int rc = SUPSemEventSignal(pThis->s.CTX_SUFF(pVM)->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
+                            AssertRC(rc);
+                        }
+                        break;
+                    }
+                }
+                else
+                {
+                    /* Reverse the direction and signal the reader threads. */
+                    u64State &= ~(RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK);
+                    u64State |= RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT;
+                    if (ASMAtomicCmpXchgU64(&pThis->s.Core.u64State, u64State, u64OldState))
+                    {
+                        Assert(!pThis->s.Core.fNeedReset);
+                        ASMAtomicWriteBool(&pThis->s.Core.fNeedReset, true);
+                        int rc = SUPSemEventMultiSignal(pThis->s.CTX_SUFF(pVM)->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead);
+                        AssertRC(rc);
+                        break;
+                    }
+                }
+
+                ASMNopPause();
+                if (pThis->s.Core.u32Magic != RTCRITSECTRW_MAGIC)
+                    return VERR_SEM_DESTROYED;
+            }
+        }
+#endif /* IN_RING3 || IN_RING0 */
+#ifndef IN_RING3
+# ifdef IN_RING0
+        else
+# endif
+        {
+            /*
+             * We cannot call neither SUPSemEventSignal nor SUPSemEventMultiSignal,
+             * so queue the exit request (ring-3).
+             */
+            PVMCC       pVM   = pThis->s.CTX_SUFF(pVM);         AssertPtr(pVM);
+            PVMCPUCC    pVCpu = VMMGetCpu(pVM);                 AssertPtr(pVCpu);
+            uint32_t    i     = pVCpu->pdm.s.cQueuedCritSectRwExclLeaves++;
+            LogFlow(("PDMCritSectRwLeaveShared: [%d]=%p => R3\n", i, pThis));
+            AssertFatal(i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectLeaves));
+            pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i] = MMHyperCCToR3(pVM, pThis);
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT);
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
+            STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves);
+            STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveExcl);
+        }
+#endif
+    }
+    else
+    {
+        /*
+         * Not the final recursion.
+         */
+        Assert(pThis->s.Core.cWriteRecursions != 0);
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+        if (fNoVal)
+            Assert(pThis->s.Core.pValidatorWrite->hThread == NIL_RTTHREAD);
+        else
+        {
+            int rc9 = RTLockValidatorRecExclUnwind(pThis->s.Core.pValidatorWrite);
+            if (RT_FAILURE(rc9))
+                return rc9;
+        }
+#endif
+        ASMAtomicDecU32(&pThis->s.Core.cWriteRecursions);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Leave a critical section held exclusively.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwLeaveShared, RTCritSectRwLeaveExcl.
+ */
+VMMDECL(int) PDMCritSectRwLeaveExcl(PPDMCRITSECTRW pThis)
+{
+    return pdmCritSectRwLeaveExclWorker(pThis, false /*fNoVal*/);
+}
+
+
+#if defined(IN_RING3) || defined(IN_RING0)
+/**
+ * PDMCritSectBothFF interface.
+ *
+ * @param   pThis       Pointer to the read/write critical section.
+ */
+void pdmCritSectRwLeaveExclQueued(PPDMCRITSECTRW pThis)
+{
+    pdmCritSectRwLeaveExclWorker(pThis, true /*fNoVal*/);
+}
+#endif
+
+
+/**
+ * Checks the caller is the exclusive (write) owner of the critical section.
+ *
+ * @retval  true if owner.
+ * @retval  false if not owner.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwIsReadOwner, PDMCritSectIsOwner,
+ *          RTCritSectRwIsWriteOwner.
+ */
+VMMDECL(bool) PDMCritSectRwIsWriteOwner(PPDMCRITSECTRW pThis)
+{
+    /*
+     * Validate handle.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, false);
+
+    /*
+     * Check ownership.
+     */
+    RTNATIVETHREAD hNativeWriter;
+    ASMAtomicUoReadHandle(&pThis->s.Core.hNativeWriter, &hNativeWriter);
+    if (hNativeWriter == NIL_RTNATIVETHREAD)
+        return false;
+    return hNativeWriter == pdmCritSectRwGetNativeSelf(pThis);
+}
+
+
+/**
+ * Checks if the caller is one of the read owners of the critical section.
+ *
+ * @note    !CAUTION!  This API doesn't work reliably if lock validation isn't
+ *          enabled. Meaning, the answer is not trustworhty unless
+ *          RT_LOCK_STRICT or PDMCRITSECTRW_STRICT was defined at build time.
+ *          Also, make sure you do not use RTCRITSECTRW_FLAGS_NO_LOCK_VAL when
+ *          creating the semaphore.  And finally, if you used a locking class,
+ *          don't disable deadlock detection by setting cMsMinDeadlock to
+ *          RT_INDEFINITE_WAIT.
+ *
+ *          In short, only use this for assertions.
+ *
+ * @returns @c true if reader, @c false if not.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   fWannaHear  What you'd like to hear when lock validation is not
+ *                      available.  (For avoiding asserting all over the place.)
+ * @sa      PDMCritSectRwIsWriteOwner, RTCritSectRwIsReadOwner.
+ */
+VMMDECL(bool) PDMCritSectRwIsReadOwner(PPDMCRITSECTRW pThis, bool fWannaHear)
+{
+    /*
+     * Validate handle.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, false);
+
+    /*
+     * Inspect the state.
+     */
+    uint64_t u64State = ASMAtomicReadU64(&pThis->s.Core.u64State);
+    if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
+    {
+        /*
+         * It's in write mode, so we can only be a reader if we're also the
+         * current writer.
+         */
+        RTNATIVETHREAD hWriter;
+        ASMAtomicUoReadHandle(&pThis->s.Core.hNativeWriter, &hWriter);
+        if (hWriter == NIL_RTNATIVETHREAD)
+            return false;
+        return hWriter == pdmCritSectRwGetNativeSelf(pThis);
+    }
+
+    /*
+     * Read mode.  If there are no current readers, then we cannot be a reader.
+     */
+    if (!(u64State & RTCSRW_CNT_RD_MASK))
+        return false;
+
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+    /*
+     * Ask the lock validator.
+     * Note! It doesn't know everything, let's deal with that if it becomes an issue...
+     */
+    NOREF(fWannaHear);
+    return RTLockValidatorRecSharedIsOwner(pThis->s.Core.pValidatorRead, NIL_RTTHREAD);
+#else
+    /*
+     * Ok, we don't know, just tell the caller what he want to hear.
+     */
+    return fWannaHear;
+#endif
+}
+
+
+/**
+ * Gets the write recursion count.
+ *
+ * @returns The write recursion count (0 if bad critsect).
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwGetWriterReadRecursion, PDMCritSectRwGetReadCount,
+ *          RTCritSectRwGetWriteRecursion.
+ */
+VMMDECL(uint32_t) PDMCritSectRwGetWriteRecursion(PPDMCRITSECTRW pThis)
+{
+    /*
+     * Validate handle.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0);
+
+    /*
+     * Return the requested data.
+     */
+    return pThis->s.Core.cWriteRecursions;
+}
+
+
+/**
+ * Gets the read recursion count of the current writer.
+ *
+ * @returns The read recursion count (0 if bad critsect).
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwGetWriteRecursion, PDMCritSectRwGetReadCount,
+ *          RTCritSectRwGetWriterReadRecursion.
+ */
+VMMDECL(uint32_t) PDMCritSectRwGetWriterReadRecursion(PPDMCRITSECTRW pThis)
+{
+    /*
+     * Validate handle.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0);
+
+    /*
+     * Return the requested data.
+     */
+    return pThis->s.Core.cWriterReads;
+}
+
+
+/**
+ * Gets the current number of reads.
+ *
+ * This includes all read recursions, so it might be higher than the number of
+ * read owners.  It does not include reads done by the current writer.
+ *
+ * @returns The read count (0 if bad critsect).
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwGetWriteRecursion, PDMCritSectRwGetWriterReadRecursion,
+ *          RTCritSectRwGetReadCount.
+ */
+VMMDECL(uint32_t) PDMCritSectRwGetReadCount(PPDMCRITSECTRW pThis)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0);
+
+    /*
+     * Return the requested data.
+     */
+    uint64_t u64State = ASMAtomicReadU64(&pThis->s.Core.u64State);
+    if ((u64State & RTCSRW_DIR_MASK) != (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
+        return 0;
+    return (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
+}
+
+
+/**
+ * Checks if the read/write critical section is initialized or not.
+ *
+ * @retval  true if initialized.
+ * @retval  false if not initialized.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectIsInitialized, RTCritSectRwIsInitialized.
+ */
+VMMDECL(bool) PDMCritSectRwIsInitialized(PCPDMCRITSECTRW pThis)
+{
+    AssertPtr(pThis);
+    return pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC;
+}
+
diff --git a/src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp b/src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp
new file mode 100644
index 00000000..6eb36db3
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp
@@ -0,0 +1,77 @@
+/* $Id: PDMAllNetShaper.cpp $ */
+/** @file
+ * PDM Network Shaper - Limit network traffic according to bandwidth group settings.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_NET_SHAPER
+#include <VBox/vmm/pdm.h>
+#include <VBox/log.h>
+#include <iprt/time.h>
+
+#include <VBox/vmm/pdmnetshaper.h>
+#include "PDMNetShaperInternal.h"
+
+
+/**
+ * Obtain bandwidth in a bandwidth group.
+ *
+ * @returns True if bandwidth was allocated, false if not.
+ * @param   pFilter         Pointer to the filter that allocates bandwidth.
+ * @param   cbTransfer      Number of bytes to allocate.
+ */
+VMMDECL(bool) PDMNsAllocateBandwidth(PPDMNSFILTER pFilter, size_t cbTransfer)
+{
+    AssertPtrReturn(pFilter, true);
+    if (!VALID_PTR(pFilter->CTX_SUFF(pBwGroup)))
+        return true;
+
+    PPDMNSBWGROUP pBwGroup = ASMAtomicReadPtrT(&pFilter->CTX_SUFF(pBwGroup), PPDMNSBWGROUP);
+    int rc = PDMCritSectEnter(&pBwGroup->Lock, VERR_SEM_BUSY); AssertRC(rc);
+    if (RT_UNLIKELY(rc == VERR_SEM_BUSY))
+        return true;
+
+    bool fAllowed = true;
+    if (pBwGroup->cbPerSecMax)
+    {
+        /* Re-fill the bucket first */
+        uint64_t tsNow        = RTTimeSystemNanoTS();
+        uint32_t uTokensAdded = (tsNow - pBwGroup->tsUpdatedLast) * pBwGroup->cbPerSecMax / (1000 * 1000 * 1000);
+        uint32_t uTokens      = RT_MIN(pBwGroup->cbBucket, uTokensAdded + pBwGroup->cbTokensLast);
+
+        if (cbTransfer > uTokens)
+        {
+            fAllowed = false;
+            ASMAtomicWriteBool(&pFilter->fChoked, true);
+        }
+        else
+        {
+            pBwGroup->tsUpdatedLast = tsNow;
+            pBwGroup->cbTokensLast = uTokens - (uint32_t)cbTransfer;
+        }
+        Log2(("pdmNsAllocateBandwidth: BwGroup=%#p{%s} cbTransfer=%u uTokens=%u uTokensAdded=%u fAllowed=%RTbool\n",
+              pBwGroup, R3STRING(pBwGroup->pszNameR3), cbTransfer, uTokens, uTokensAdded, fAllowed));
+    }
+    else
+        Log2(("pdmNsAllocateBandwidth: BwGroup=%#p{%s} disabled fAllowed=%RTbool\n",
+              pBwGroup, R3STRING(pBwGroup->pszNameR3), fAllowed));
+
+    rc = PDMCritSectLeave(&pBwGroup->Lock); AssertRC(rc);
+    return fAllowed;
+}
+
diff --git a/src/VBox/VMM/VMMAll/PDMAllQueue.cpp b/src/VBox/VMM/VMMAll/PDMAllQueue.cpp
new file mode 100644
index 00000000..fa578d8f
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllQueue.cpp
@@ -0,0 +1,208 @@
+/* $Id: PDMAllQueue.cpp $ */
+/** @file
+ * PDM Queue - Transport data and tasks to EMT and R3.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_QUEUE
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#ifndef IN_RC
+# include <VBox/vmm/mm.h>
+#endif
+#include <VBox/vmm/vmcc.h>
+#include <iprt/errcore.h>
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+
+
+/**
+ * Allocate an item from a queue.
+ * The allocated item must be handed on to PDMR3QueueInsert() after the
+ * data have been filled in.
+ *
+ * @returns Pointer to allocated queue item.
+ * @returns NULL on failure. The queue is exhausted.
+ * @param   pQueue      The queue handle.
+ * @thread  Any thread.
+ */
+VMMDECL(PPDMQUEUEITEMCORE) PDMQueueAlloc(PPDMQUEUE pQueue)
+{
+    Assert(VALID_PTR(pQueue) && pQueue->CTX_SUFF(pVM));
+    PPDMQUEUEITEMCORE pNew;
+    uint32_t iNext;
+    uint32_t i;
+    do
+    {
+        i = pQueue->iFreeTail;
+        if (i == pQueue->iFreeHead)
+        {
+            STAM_REL_COUNTER_INC(&pQueue->StatAllocFailures);
+            return NULL;
+        }
+        pNew = pQueue->aFreeItems[i].CTX_SUFF(pItem);
+        iNext = (i + 1) % (pQueue->cItems + PDMQUEUE_FREE_SLACK);
+    } while (!ASMAtomicCmpXchgU32(&pQueue->iFreeTail, iNext, i));
+    return pNew;
+}
+
+
+/**
+ * Sets the FFs and fQueueFlushed.
+ *
+ * @param   pQueue              The PDM queue.
+ */
+static void pdmQueueSetFF(PPDMQUEUE pQueue)
+{
+    PVM pVM = pQueue->CTX_SUFF(pVM);
+    Log2(("PDMQueueInsert: VM_FF_PDM_QUEUES %d -> 1\n", VM_FF_IS_SET(pVM, VM_FF_PDM_QUEUES)));
+    VM_FF_SET(pVM, VM_FF_PDM_QUEUES);
+    ASMAtomicBitSet(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_PENDING_BIT);
+#ifdef IN_RING3
+    VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_DONE_REM);
+#endif
+}
+
+
+/**
+ * Queue an item.
+ * The item must have been obtained using PDMQueueAlloc(). Once the item
+ * have been passed to this function it must not be touched!
+ *
+ * @param   pQueue      The queue handle.
+ * @param   pItem       The item to insert.
+ * @thread  Any thread.
+ */
+VMMDECL(void) PDMQueueInsert(PPDMQUEUE pQueue, PPDMQUEUEITEMCORE pItem)
+{
+    Assert(VALID_PTR(pQueue) && pQueue->CTX_SUFF(pVM));
+    Assert(VALID_PTR(pItem));
+
+#if 0 /* the paranoid android version: */
+    void *pvNext;
+    do
+    {
+        pvNext = ASMAtomicUoReadPtr((void * volatile *)&pQueue->CTX_SUFF(pPending));
+        ASMAtomicUoWritePtr((void * volatile *)&pItem->CTX_SUFF(pNext), pvNext);
+    } while (!ASMAtomicCmpXchgPtr(&pQueue->CTX_SUFF(pPending), pItem, pvNext));
+#else
+    PPDMQUEUEITEMCORE pNext;
+    do
+    {
+        pNext = pQueue->CTX_SUFF(pPending);
+        pItem->CTX_SUFF(pNext) = pNext;
+    } while (!ASMAtomicCmpXchgPtr(&pQueue->CTX_SUFF(pPending), pItem, pNext));
+#endif
+
+    if (!pQueue->pTimer)
+        pdmQueueSetFF(pQueue);
+    STAM_REL_COUNTER_INC(&pQueue->StatInsert);
+    STAM_STATS({ ASMAtomicIncU32(&pQueue->cStatPending); });
+}
+
+
+/**
+ * Queue an item.
+ * The item must have been obtained using PDMQueueAlloc(). Once the item
+ * have been passed to this function it must not be touched!
+ *
+ * @param   pQueue          The queue handle.
+ * @param   pItem           The item to insert.
+ * @param   NanoMaxDelay    The maximum delay before processing the queue, in nanoseconds.
+ *                          This applies only to GC.
+ * @thread  Any thread.
+ */
+VMMDECL(void) PDMQueueInsertEx(PPDMQUEUE pQueue, PPDMQUEUEITEMCORE pItem, uint64_t NanoMaxDelay)
+{
+    NOREF(NanoMaxDelay);
+    PDMQueueInsert(pQueue, pItem);
+#ifdef IN_RC
+    PVM pVM = pQueue->CTX_SUFF(pVM);
+    /** @todo figure out where to put this, the next bit should go there too.
+    if (NanoMaxDelay)
+    {
+
+    }
+    else */
+    {
+        VMCPU_FF_SET(VMMGetCpu0(pVM), VMCPU_FF_TO_R3);
+        Log2(("PDMQueueInsertEx: Setting VMCPU_FF_TO_R3\n"));
+    }
+#endif
+}
+
+
+
+/**
+ * Gets the RC pointer for the specified queue.
+ *
+ * @returns The RC address of the queue.
+ * @returns NULL if pQueue is invalid.
+ * @param   pQueue          The queue handle.
+ */
+VMMDECL(RCPTRTYPE(PPDMQUEUE)) PDMQueueRCPtr(PPDMQUEUE pQueue)
+{
+    Assert(VALID_PTR(pQueue));
+    Assert(pQueue->pVMR3 && pQueue->pVMRC);
+#ifdef IN_RC
+    return pQueue;
+#else
+    return MMHyperCCToRC(pQueue->CTX_SUFF(pVM), pQueue);
+#endif
+}
+
+
+/**
+ * Gets the ring-0 pointer for the specified queue.
+ *
+ * @returns The ring-0 address of the queue.
+ * @returns NULL if pQueue is invalid.
+ * @param   pQueue          The queue handle.
+ */
+VMMDECL(R0PTRTYPE(PPDMQUEUE)) PDMQueueR0Ptr(PPDMQUEUE pQueue)
+{
+    Assert(VALID_PTR(pQueue));
+    Assert(pQueue->pVMR3 && pQueue->pVMR0);
+#ifdef IN_RING0
+    return pQueue;
+#else
+    return MMHyperCCToR0(pQueue->CTX_SUFF(pVM), pQueue);
+#endif
+}
+
+
+/**
+ * Schedule the queue for flushing (processing) if necessary.
+ *
+ * @returns @c true if necessary, @c false if not.
+ * @param   pQueue              The queue.
+ */
+VMMDECL(bool) PDMQueueFlushIfNecessary(PPDMQUEUE pQueue)
+{
+    AssertPtr(pQueue);
+    if (   pQueue->pPendingR3 != NIL_RTR3PTR
+        || pQueue->pPendingR0 != NIL_RTR0PTR
+        || pQueue->pPendingRC != NIL_RTRCPTR)
+    {
+        pdmQueueSetFF(pQueue);
+        return false;
+    }
+    return false;
+}
+
diff --git a/src/VBox/VMM/VMMAll/PDMAllTask.cpp b/src/VBox/VMM/VMMAll/PDMAllTask.cpp
new file mode 100644
index 00000000..32c2a68d
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllTask.cpp
@@ -0,0 +1,114 @@
+/* $Id: PDMAllTask.cpp $ */
+/** @file
+ * PDM Task - Asynchronous user mode tasks, all context code.
+ */
+
+/*
+ * Copyright (C) 2019-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_TASK
+#include "PDMInternal.h"
+#include <VBox/vmm/pdmtask.h>
+#include <VBox/vmm/gvm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <VBox/sup.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/semaphore.h>
+
+
+
+/**
+ * Triggers a task.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_ALREADY_POSTED is the task is already pending.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   enmType     The task owner type.
+ * @param   pvOwner     The task owner.
+ * @param   hTask       The task to trigger.
+ * @thread  Any
+ */
+VMM_INT_DECL(int)   PDMTaskTrigger(PVMCC pVM, PDMTASKTYPE enmType, RTR3PTR pvOwner, PDMTASKHANDLE hTask)
+{
+    /*
+     * Validate input and translate the handle to a task.
+     */
+    AssertReturn(pvOwner, VERR_NOT_OWNER);
+    AssertReturn(enmType >= PDMTASKTYPE_DEV && enmType <= PDMTASKTYPE_INTERNAL, VERR_NOT_OWNER);
+
+    size_t const iTask    = hTask % RT_ELEMENTS(pVM->pdm.s.aTaskSets[0].aTasks);
+    size_t const iTaskSet = hTask / RT_ELEMENTS(pVM->pdm.s.aTaskSets[0].aTasks);
+#ifdef IN_RING3
+    AssertReturn(iTaskSet < RT_ELEMENTS(pVM->pdm.s.apTaskSets), VERR_INVALID_HANDLE);
+    PPDMTASKSET pTaskSet  = pVM->pdm.s.apTaskSets[iTaskSet];
+    AssertReturn(pTaskSet, VERR_INVALID_HANDLE);
+#else
+    AssertReturn(iTaskSet < RT_ELEMENTS(pVM->pdm.s.aTaskSets),
+                 iTaskSet < RT_ELEMENTS(pVM->pdm.s.apTaskSets) ? VERR_INVALID_CONTEXT : VERR_INVALID_HANDLE);
+    PPDMTASKSET pTaskSet  = &pVM->pdm.s.aTaskSets[iTaskSet];
+#endif
+    AssertReturn(pTaskSet->u32Magic == PDMTASKSET_MAGIC, VERR_INVALID_MAGIC);
+    PPDMTASK pTask = &pTaskSet->aTasks[iTask];
+
+    /*
+     * Check task ownership.
+     */
+    AssertReturn(pvOwner == pTask->pvOwner, VERR_NOT_OWNER);
+    AssertReturn(enmType == pTask->enmType, VERR_NOT_OWNER);
+
+    /*
+     * Trigger the task, wake up the thread if the task isn't triggered yet.
+     */
+    if (!ASMAtomicBitTestAndSet(&pTaskSet->fTriggered, (uint32_t)iTask))
+    {
+        Log(("PDMTaskTrigger: Triggered %RU64 (%s)\n", hTask, R3STRING(pTask->pszName)));
+#ifdef IN_RING3
+        if (pTaskSet->hEventR3 != NIL_RTSEMEVENT)
+        {
+            int rc = RTSemEventSignal(pTaskSet->hEventR3);
+            AssertLogRelRCReturn(rc, rc);
+        }
+        else
+#endif
+        {
+            int rc = SUPSemEventSignal(pVM->pSession, pTaskSet->hEventR0);
+            AssertLogRelRCReturn(rc, rc);
+        }
+        return VINF_SUCCESS;
+    }
+    ASMAtomicIncU32(&pTask->cAlreadyTrigged);
+    Log(("PDMTaskTrigger: %RU64 (%s) was already triggered\n", hTask, R3STRING(pTask->pszName)));
+    return VINF_ALREADY_POSTED;
+}
+
+
+/**
+ * Triggers an internal task.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTask       The task to trigger.
+ * @thread  Any
+ */
+VMM_INT_DECL(int)   PDMTaskTriggerInternal(PVMCC pVM, PDMTASKHANDLE hTask)
+{
+    return PDMTaskTrigger(pVM, PDMTASKTYPE_INTERNAL, pVM->pVMR3, hTask);
+}
+
diff --git a/src/VBox/VMM/VMMAll/PGMAll.cpp b/src/VBox/VMM/VMMAll/PGMAll.cpp
new file mode 100644
index 00000000..6a06bb55
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAll.cpp
@@ -0,0 +1,3779 @@
+/* $Id: PGMAll.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor - All context code.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/hm_vmx.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include "PGMInline.h"
+#include <iprt/assert.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/string.h>
+#include <VBox/log.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPT *ppPdpt, PX86PDPAE *ppPD);
+DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
+static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
+static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT *ppPdpt, PEPTPD *ppPD);
+
+
+/*
+ * Shadow - 32-bit mode
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_32BIT
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_32BIT(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_REAL(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_32BIT_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_32BIT_PD_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllGst.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_PROT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_32BIT_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_32BIT_PD_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllGst.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_32BIT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_32BIT_PD
+#include "PGMGstDefs.h"
+#include "PGMAllGst.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - PAE mode
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_PAE
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_PAE(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_REAL(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_REAL(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_PROT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_32BIT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT_FOR_32BIT
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_PAE(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT
+#include "PGMGstDefs.h"
+#include "PGMAllGst.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - AMD64 mode
+ */
+#define PGM_SHW_TYPE               PGM_TYPE_AMD64
+#define PGM_SHW_NAME(name)         PGM_SHW_NAME_AMD64(name)
+#include "PGMAllShw.h"
+
+/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
+/** @todo retire this hack. */
+#define PGM_GST_TYPE               PGM_TYPE_PROT
+#define PGM_GST_NAME(name)         PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)         PGM_BTH_NAME_AMD64_PROT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT  PGMPOOLKIND_PAE_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT       PGMPOOLKIND_PAE_PD_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE              PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)        PGM_GST_NAME_AMD64(name)
+# define PGM_BTH_NAME(name)        PGM_BTH_NAME_AMD64_AMD64(name)
+# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
+# define BTH_PGMPOOLKIND_ROOT      PGMPOOLKIND_64BIT_PML4
+# include "PGMGstDefs.h"
+# include "PGMAllGst.h"
+# include "PGMAllBth.h"
+# undef BTH_PGMPOOLKIND_PT_FOR_BIG
+# undef BTH_PGMPOOLKIND_PT_FOR_PT
+# undef BTH_PGMPOOLKIND_ROOT
+# undef PGM_BTH_NAME
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - 32-bit nested paging mode.
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_NESTED_32BIT
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_NESTED_32BIT(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_32BIT_REAL(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_32BIT_PROT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_32BIT_PAE(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
+# include "PGMGstDefs.h"
+# include "PGMAllBth.h"
+# undef PGM_BTH_NAME
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - PAE nested paging mode.
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_NESTED_PAE
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_NESTED_PAE(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_PAE_REAL(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_PAE_PROT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_PAE_32BIT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_PAE_PAE(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_NESTED_PAE_AMD64(name)
+# include "PGMGstDefs.h"
+# include "PGMAllBth.h"
+# undef PGM_BTH_NAME
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - AMD64 nested paging mode.
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_NESTED_AMD64
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_NESTED_AMD64(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_AMD64_REAL(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_AMD64_PROT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_AMD64_PAE(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
+# include "PGMGstDefs.h"
+# include "PGMAllBth.h"
+# undef PGM_BTH_NAME
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - EPT.
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_EPT
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_EPT(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_REAL(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_EPT_PT_FOR_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_PROT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_EPT_PT_FOR_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_32BIT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_EPT_PT_FOR_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_PAE(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_EPT_PT_FOR_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_EPT_AMD64(name)
+# define BTH_PGMPOOLKIND_PT_FOR_PT  PGMPOOLKIND_EPT_PT_FOR_PHYS
+# include "PGMGstDefs.h"
+# include "PGMAllBth.h"
+# undef BTH_PGMPOOLKIND_PT_FOR_PT
+# undef PGM_BTH_NAME
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - NEM / None.
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_NONE
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_NONE(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NONE_REAL(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NONE_PROT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NONE_32BIT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE               PGM_TYPE_PAE
+#define PGM_GST_NAME(name)         PGM_GST_NAME_PAE(name)
+#define PGM_BTH_NAME(name)         PGM_BTH_NAME_NONE_PAE(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_NONE_AMD64(name)
+# include "PGMGstDefs.h"
+# include "PGMAllBth.h"
+# undef PGM_BTH_NAME
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+
+/**
+ * Guest mode data array.
+ */
+PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
+{
+    { UINT32_MAX, NULL, NULL, NULL, NULL, NULL }, /* 0 */
+    {
+        PGM_TYPE_REAL,
+        PGM_GST_NAME_REAL(GetPage),
+        PGM_GST_NAME_REAL(ModifyPage),
+        PGM_GST_NAME_REAL(GetPDE),
+        PGM_GST_NAME_REAL(Enter),
+        PGM_GST_NAME_REAL(Exit),
+#ifdef IN_RING3
+        PGM_GST_NAME_REAL(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_PROT,
+        PGM_GST_NAME_PROT(GetPage),
+        PGM_GST_NAME_PROT(ModifyPage),
+        PGM_GST_NAME_PROT(GetPDE),
+        PGM_GST_NAME_PROT(Enter),
+        PGM_GST_NAME_PROT(Exit),
+#ifdef IN_RING3
+        PGM_GST_NAME_PROT(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_32BIT,
+        PGM_GST_NAME_32BIT(GetPage),
+        PGM_GST_NAME_32BIT(ModifyPage),
+        PGM_GST_NAME_32BIT(GetPDE),
+        PGM_GST_NAME_32BIT(Enter),
+        PGM_GST_NAME_32BIT(Exit),
+#ifdef IN_RING3
+        PGM_GST_NAME_32BIT(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_PAE,
+        PGM_GST_NAME_PAE(GetPage),
+        PGM_GST_NAME_PAE(ModifyPage),
+        PGM_GST_NAME_PAE(GetPDE),
+        PGM_GST_NAME_PAE(Enter),
+        PGM_GST_NAME_PAE(Exit),
+#ifdef IN_RING3
+        PGM_GST_NAME_PAE(Relocate),
+#endif
+    },
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    {
+        PGM_TYPE_AMD64,
+        PGM_GST_NAME_AMD64(GetPage),
+        PGM_GST_NAME_AMD64(ModifyPage),
+        PGM_GST_NAME_AMD64(GetPDE),
+        PGM_GST_NAME_AMD64(Enter),
+        PGM_GST_NAME_AMD64(Exit),
+# ifdef IN_RING3
+        PGM_GST_NAME_AMD64(Relocate),
+# endif
+    },
+#endif
+};
+
+
+/**
+ * The shadow mode data array.
+ */
+PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
+{
+    { UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
+    { UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
+    { UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
+    {
+        PGM_TYPE_32BIT,
+        PGM_SHW_NAME_32BIT(GetPage),
+        PGM_SHW_NAME_32BIT(ModifyPage),
+        PGM_SHW_NAME_32BIT(Enter),
+        PGM_SHW_NAME_32BIT(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_32BIT(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_PAE,
+        PGM_SHW_NAME_PAE(GetPage),
+        PGM_SHW_NAME_PAE(ModifyPage),
+        PGM_SHW_NAME_PAE(Enter),
+        PGM_SHW_NAME_PAE(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_PAE(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_AMD64,
+        PGM_SHW_NAME_AMD64(GetPage),
+        PGM_SHW_NAME_AMD64(ModifyPage),
+        PGM_SHW_NAME_AMD64(Enter),
+        PGM_SHW_NAME_AMD64(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_AMD64(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_NESTED_32BIT,
+        PGM_SHW_NAME_NESTED_32BIT(GetPage),
+        PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
+        PGM_SHW_NAME_NESTED_32BIT(Enter),
+        PGM_SHW_NAME_NESTED_32BIT(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_NESTED_32BIT(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_NESTED_PAE,
+        PGM_SHW_NAME_NESTED_PAE(GetPage),
+        PGM_SHW_NAME_NESTED_PAE(ModifyPage),
+        PGM_SHW_NAME_NESTED_PAE(Enter),
+        PGM_SHW_NAME_NESTED_PAE(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_NESTED_PAE(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_NESTED_AMD64,
+        PGM_SHW_NAME_NESTED_AMD64(GetPage),
+        PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
+        PGM_SHW_NAME_NESTED_AMD64(Enter),
+        PGM_SHW_NAME_NESTED_AMD64(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_NESTED_AMD64(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_EPT,
+        PGM_SHW_NAME_EPT(GetPage),
+        PGM_SHW_NAME_EPT(ModifyPage),
+        PGM_SHW_NAME_EPT(Enter),
+        PGM_SHW_NAME_EPT(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_EPT(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_NONE,
+        PGM_SHW_NAME_NONE(GetPage),
+        PGM_SHW_NAME_NONE(ModifyPage),
+        PGM_SHW_NAME_NONE(Enter),
+        PGM_SHW_NAME_NONE(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_NONE(Relocate),
+#endif
+    },
+};
+
+
+/**
+ * The guest+shadow mode data array.
+ */
+PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
+{
+#if   !defined(IN_RING3) && !defined(VBOX_STRICT)
+# define PGMMODEDATABTH_NULL_ENTRY()    { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
+# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
+    { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler) }
+
+#elif !defined(IN_RING3) && defined(VBOX_STRICT)
+# define PGMMODEDATABTH_NULL_ENTRY()    { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
+# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
+    { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(AssertCR3) }
+
+#elif defined(IN_RING3) && !defined(VBOX_STRICT)
+# define PGMMODEDATABTH_NULL_ENTRY()    { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
+# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
+    { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
+
+#elif defined(IN_RING3) && defined(VBOX_STRICT)
+# define PGMMODEDATABTH_NULL_ENTRY()    { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
+# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
+    { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
+
+#else
+# error "Misconfig."
+#endif
+
+    /* 32-bit shadow paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL,  PGM_BTH_NAME_32BIT_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT,  PGM_BTH_NAME_32BIT_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64        - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE         - illegal */
+
+    /* PAE shadow paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL,  PGM_BTH_NAME_PAE_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT,  PGM_BTH_NAME_PAE_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE,   PGM_BTH_NAME_PAE_PAE),
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64        - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE         - illegal */
+
+    /* AMD64 shadow paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL,  PGM_BTH_NAME_AMD64_REAL),
+    PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT,  PGM_BTH_NAME_AMD64_PROT),
+    PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
+    PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE,   PGM_BTH_NAME_AMD64_PAE),
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
+#else
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64        - illegal */
+#endif
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE         - illegal */
+
+    /* 32-bit nested paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL,  PGM_BTH_NAME_NESTED_32BIT_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT,  PGM_BTH_NAME_NESTED_32BIT_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE,   PGM_BTH_NAME_NESTED_32BIT_PAE),
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
+#else
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64        - illegal */
+#endif
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE         - illegal */
+
+    /* PAE nested paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL,  PGM_BTH_NAME_NESTED_PAE_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT,  PGM_BTH_NAME_NESTED_PAE_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE,   PGM_BTH_NAME_NESTED_PAE_PAE),
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
+#else
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64        - illegal */
+#endif
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE         - illegal */
+
+    /* AMD64 nested paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL,  PGM_BTH_NAME_NESTED_AMD64_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT,  PGM_BTH_NAME_NESTED_AMD64_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE,   PGM_BTH_NAME_NESTED_AMD64_PAE),
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
+#else
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64        - illegal */
+#endif
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE         - illegal */
+
+    /* EPT nested paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL,  PGM_BTH_NAME_EPT_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT,  PGM_BTH_NAME_EPT_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE,   PGM_BTH_NAME_EPT_PAE),
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
+#else
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64        - illegal */
+#endif
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE         - illegal */
+
+    /* NONE / NEM: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL,  PGM_BTH_NAME_EPT_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT,  PGM_BTH_NAME_EPT_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE,   PGM_BTH_NAME_EPT_PAE),
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
+#else
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64        - illegal */
+#endif
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE         - illegal */
+
+
+#undef PGMMODEDATABTH_ENTRY
+#undef PGMMODEDATABTH_NULL_ENTRY
+};
+
+
+#ifdef IN_RING0
+/**
+ * #PF Handler.
+ *
+ * @returns VBox status code (appropriate for trap handling and GC return).
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uErr        The trap error code.
+ * @param   pRegFrame   Trap register frame.
+ * @param   pvFault     The fault address.
+ */
+VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0e, a);
+    STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = NULL; } );
+
+
+# ifdef VBOX_WITH_STATISTICS
+    /*
+     * Error code stats.
+     */
+    if (uErr & X86_TRAP_PF_US)
+    {
+        if (!(uErr & X86_TRAP_PF_P))
+        {
+            if (uErr & X86_TRAP_PF_RW)
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNotPresentWrite);
+            else
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNotPresentRead);
+        }
+        else if (uErr & X86_TRAP_PF_RW)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSWrite);
+        else if (uErr & X86_TRAP_PF_RSVD)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSReserved);
+        else if (uErr & X86_TRAP_PF_ID)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNXE);
+        else
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSRead);
+    }
+    else
+    {   /* Supervisor */
+        if (!(uErr & X86_TRAP_PF_P))
+        {
+            if (uErr & X86_TRAP_PF_RW)
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVNotPresentWrite);
+            else
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVNotPresentRead);
+        }
+        else if (uErr & X86_TRAP_PF_RW)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVWrite);
+        else if (uErr & X86_TRAP_PF_ID)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSNXE);
+        else if (uErr & X86_TRAP_PF_RSVD)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVReserved);
+    }
+# endif /* VBOX_WITH_STATISTICS */
+
+    /*
+     * Call the worker.
+     */
+    uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
+    bool fLockTaken = false;
+    int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pRegFrame, pvFault, &fLockTaken);
+    if (fLockTaken)
+    {
+        PGM_LOCK_ASSERT_OWNER(pVM);
+        pgmUnlock(pVM);
+    }
+    LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
+
+    /*
+     * Return code tweaks.
+     */
+    if (rc != VINF_SUCCESS)
+    {
+        if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
+            rc = VINF_SUCCESS;
+
+        /* Note: hack alert for difficult to reproduce problem. */
+        if (    rc == VERR_PAGE_NOT_PRESENT                 /* SMP only ; disassembly might fail. */
+            ||  rc == VERR_PAGE_TABLE_NOT_PRESENT           /* seen with UNI & SMP */
+            ||  rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT   /* seen with SMP */
+            ||  rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)     /* precaution */
+        {
+            Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pRegFrame->rip));
+            /* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
+            rc = VINF_SUCCESS;
+        }
+    }
+
+    STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPF); });
+    STAM_STATS({ if (!pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution))
+                    pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Misc; });
+    STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0e, pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution), a);
+    return rc;
+}
+#endif /* IN_RING0 */
+
+
+/**
+ * Prefetch a page
+ *
+ * Typically used to sync commonly used pages before entering raw mode
+ * after a CR3 reload.
+ *
+ * @returns VBox status code suitable for scheduling.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtrPage   Page to invalidate.
+ */
+VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
+{
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,Prefetch), a);
+
+    uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
+    int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,Prefetch), a);
+    AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+#ifndef PGM_WITHOUT_MAPPINGS
+/**
+ * Gets the mapping corresponding to the specified address (if any).
+ *
+ * @returns Pointer to the mapping.
+ * @returns NULL if not
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPtr       The guest context pointer.
+ */
+PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr)
+{
+    PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
+    while (pMapping)
+    {
+        if ((uintptr_t)GCPtr < (uintptr_t)pMapping->GCPtr)
+            break;
+        if ((uintptr_t)GCPtr - (uintptr_t)pMapping->GCPtr < pMapping->cb)
+            return pMapping;
+        pMapping = pMapping->CTX_SUFF(pNext);
+    }
+    return NULL;
+}
+#endif
+
+
+/**
+ * Verifies a range of pages for read or write access
+ *
+ * Only checks the guest's page tables
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   Addr        Guest virtual address to check
+ * @param   cbSize      Access size
+ * @param   fAccess     Access type (r/w, user/supervisor (X86_PTE_*))
+ * @remarks Current not in use.
+ */
+VMMDECL(int) PGMIsValidAccess(PVMCPUCC pVCpu, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
+{
+    /*
+     * Validate input.
+     */
+    if (fAccess & ~(X86_PTE_US | X86_PTE_RW))
+    {
+        AssertMsgFailed(("PGMIsValidAccess: invalid access type %08x\n", fAccess));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    uint64_t fPage;
+    int rc = PGMGstGetPage(pVCpu, (RTGCPTR)Addr, &fPage, NULL);
+    if (RT_FAILURE(rc))
+    {
+        Log(("PGMIsValidAccess: access violation for %RGv rc=%d\n", Addr, rc));
+        return VINF_EM_RAW_GUEST_TRAP;
+    }
+
+    /*
+     * Check if the access would cause a page fault
+     *
+     * Note that hypervisor page directories are not present in the guest's tables, so this check
+     * is sufficient.
+     */
+    bool fWrite = !!(fAccess & X86_PTE_RW);
+    bool fUser  = !!(fAccess & X86_PTE_US);
+    if (  !(fPage & X86_PTE_P)
+        || (fWrite && !(fPage & X86_PTE_RW))
+        || (fUser  && !(fPage & X86_PTE_US)) )
+    {
+        Log(("PGMIsValidAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPage, fWrite, fUser));
+        return VINF_EM_RAW_GUEST_TRAP;
+    }
+    if (    RT_SUCCESS(rc)
+        &&  PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize))
+        return PGMIsValidAccess(pVCpu, Addr + PAGE_SIZE, (cbSize > PAGE_SIZE) ? cbSize - PAGE_SIZE : 1, fAccess);
+    return rc;
+}
+
+
+/**
+ * Verifies a range of pages for read or write access
+ *
+ * Supports handling of pages marked for dirty bit tracking and CSAM
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   Addr        Guest virtual address to check
+ * @param   cbSize      Access size
+ * @param   fAccess     Access type (r/w, user/supervisor (X86_PTE_*))
+ */
+VMMDECL(int) PGMVerifyAccess(PVMCPUCC pVCpu, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    AssertMsg(!(fAccess & ~(X86_PTE_US | X86_PTE_RW)), ("PGMVerifyAccess: invalid access type %08x\n", fAccess));
+
+    /*
+     * Get going.
+     */
+    uint64_t fPageGst;
+    int rc = PGMGstGetPage(pVCpu, (RTGCPTR)Addr, &fPageGst, NULL);
+    if (RT_FAILURE(rc))
+    {
+        Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", Addr, rc));
+        return VINF_EM_RAW_GUEST_TRAP;
+    }
+
+    /*
+     * Check if the access would cause a page fault
+     *
+     * Note that hypervisor page directories are not present in the guest's tables, so this check
+     * is sufficient.
+     */
+    const bool fWrite = !!(fAccess & X86_PTE_RW);
+    const bool fUser  = !!(fAccess & X86_PTE_US);
+    if (   !(fPageGst & X86_PTE_P)
+        || (fWrite  && !(fPageGst & X86_PTE_RW))
+        || (fUser   && !(fPageGst & X86_PTE_US)) )
+    {
+        Log(("PGMVerifyAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPageGst, fWrite, fUser));
+        return VINF_EM_RAW_GUEST_TRAP;
+    }
+
+    if (!pVM->pgm.s.fNestedPaging)
+    {
+        /*
+         * Next step is to verify if we protected this page for dirty bit tracking or for CSAM scanning
+         */
+        rc = PGMShwGetPage(pVCpu, (RTGCPTR)Addr, NULL, NULL);
+        if (    rc == VERR_PAGE_NOT_PRESENT
+            ||  rc == VERR_PAGE_TABLE_NOT_PRESENT)
+        {
+            /*
+             * Page is not present in our page tables.
+             * Try to sync it!
+             */
+            Assert(X86_TRAP_PF_RW == X86_PTE_RW && X86_TRAP_PF_US == X86_PTE_US);
+            uint32_t  const uErr   = fAccess & (X86_TRAP_PF_RW | X86_TRAP_PF_US);
+            uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+            AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+            AssertReturn(g_aPgmBothModeData[idxBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
+            rc = g_aPgmBothModeData[idxBth].pfnVerifyAccessSyncPage(pVCpu, Addr, fPageGst, uErr);
+            if (rc != VINF_SUCCESS)
+                return rc;
+        }
+        else
+            AssertMsg(rc == VINF_SUCCESS, ("PGMShwGetPage %RGv failed with %Rrc\n", Addr, rc));
+    }
+
+#if 0 /* def VBOX_STRICT; triggers too often now */
+    /*
+     * This check is a bit paranoid, but useful.
+     */
+    /* Note! This will assert when writing to monitored pages (a bit annoying actually). */
+    uint64_t fPageShw;
+    rc = PGMShwGetPage(pVCpu, (RTGCPTR)Addr, &fPageShw, NULL);
+    if (    (rc == VERR_PAGE_NOT_PRESENT || RT_FAILURE(rc))
+        || (fWrite && !(fPageShw & X86_PTE_RW))
+        || (fUser  && !(fPageShw & X86_PTE_US)) )
+    {
+        AssertMsgFailed(("Unexpected access violation for %RGv! rc=%Rrc write=%d user=%d\n",
+                         Addr, rc, fWrite && !(fPageShw & X86_PTE_RW), fUser && !(fPageShw & X86_PTE_US)));
+        return VINF_EM_RAW_GUEST_TRAP;
+    }
+#endif
+
+    if (    RT_SUCCESS(rc)
+        &&  (   PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize - 1)
+             || Addr + cbSize < Addr))
+    {
+        /* Don't recursively call PGMVerifyAccess as we might run out of stack. */
+        for (;;)
+        {
+            Addr += PAGE_SIZE;
+            if (cbSize > PAGE_SIZE)
+                cbSize -= PAGE_SIZE;
+            else
+                cbSize = 1;
+            rc = PGMVerifyAccess(pVCpu, Addr, 1, fAccess);
+            if (rc != VINF_SUCCESS)
+                break;
+            if (PAGE_ADDRESS(Addr) == PAGE_ADDRESS(Addr + cbSize - 1))
+                break;
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Emulation of the invlpg instruction (HC only actually).
+ *
+ * @returns Strict VBox status code, special care required.
+ * @retval  VINF_PGM_SYNC_CR3 - handled.
+ * @retval  VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
+ * @retval  VERR_REM_FLUSHED_PAGES_OVERFLOW - not handled.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtrPage   Page to invalidate.
+ *
+ * @remark  ASSUMES the page table entry or page directory is valid. Fairly
+ *          safe, but there could be edge cases!
+ *
+ * @todo    Flush page or page directory only if necessary!
+ * @todo    VBOXSTRICTRC
+ */
+VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    int rc;
+    Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
+
+    IEMTlbInvalidatePage(pVCpu, GCPtrPage);
+
+    /*
+     * Call paging mode specific worker.
+     */
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage), a);
+    pgmLock(pVM);
+
+    uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), pgmUnlock(pVM), VERR_PGM_MODE_IPE);
+    AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, pgmUnlock(pVM), VERR_PGM_MODE_IPE);
+    rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
+
+    pgmUnlock(pVM);
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage), a);
+
+#ifdef IN_RING3
+    /*
+     * Check if we have a pending update of the CR3 monitoring.
+     */
+    if (    RT_SUCCESS(rc)
+        &&  (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3))
+    {
+        pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
+        Assert(!pVM->pgm.s.fMappingsFixed); Assert(pgmMapAreMappingsEnabled(pVM));
+    }
+#endif /* IN_RING3 */
+
+    /* Ignore all irrelevant error codes. */
+    if (    rc == VERR_PAGE_NOT_PRESENT
+        ||  rc == VERR_PAGE_TABLE_NOT_PRESENT
+        ||  rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
+        ||  rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
+        rc = VINF_SUCCESS;
+
+    return rc;
+}
+
+
+/**
+ * Executes an instruction using the interpreter.
+ *
+ * @returns VBox status code (appropriate for trap handling and GC return).
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pRegFrame   Register frame.
+ * @param   pvFault     Fault address.
+ */
+VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
+{
+    NOREF(pVM);
+    VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, pRegFrame, pvFault);
+    if (rc == VERR_EM_INTERPRETER)
+        rc = VINF_EM_RAW_EMULATE_INSTR;
+    if (rc != VINF_SUCCESS)
+        Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
+    return rc;
+}
+
+
+/**
+ * Gets effective page information (from the VMM page directory).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Guest Context virtual address of the page.
+ * @param   pfFlags     Where to store the flags. These are X86_PTE_*.
+ * @param   pHCPhys     Where to store the HC physical address of the page.
+ *                      This is page aligned.
+ * @remark  You should use PGMMapGetPage() for pages in a mapping.
+ */
+VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    pgmLock(pVM);
+
+    uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
+    AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
+    int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Modify page flags for a range of pages in the shadow context.
+ *
+ * The existing flags are ANDed with the fMask and ORed with the fFlags.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   fFlags      The OR  mask - page flags X86_PTE_*, excluding the page mask of course.
+ * @param   fMask       The AND mask - page flags X86_PTE_*.
+ *                      Be very CAREFUL when ~'ing constants which could be 32-bit!
+ * @param   fOpFlags    A combination of the PGM_MK_PK_XXX flags.
+ * @remark  You must use PGMMapModifyPage() for pages in a mapping.
+ */
+DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
+{
+    AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
+    Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 | PGM_MK_PG_IS_WRITE_FAULT)));
+
+    GCPtr &= PAGE_BASE_GC_MASK; /** @todo this ain't necessary, right... */
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    pgmLock(pVM);
+
+    uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
+    AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
+    int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, PAGE_SIZE, fFlags, fMask, fOpFlags);
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Changing the page flags for a single page in the shadow page tables so as to
+ * make it read-only.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   fOpFlags    A combination of the PGM_MK_PK_XXX flags.
+ */
+VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
+{
+    return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
+}
+
+
+/**
+ * Changing the page flags for a single page in the shadow page tables so as to
+ * make it writable.
+ *
+ * The call must know with 101% certainty that the guest page tables maps this
+ * as writable too.  This function will deal shared, zero and write monitored
+ * pages.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   fOpFlags    A combination of the PGM_MK_PK_XXX flags.
+ */
+VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
+{
+    if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
+        return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Changing the page flags for a single page in the shadow page tables so as to
+ * make it not present.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   fOpFlags    A combination of the PGM_MK_PG_XXX flags.
+ */
+VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
+{
+    return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
+}
+
+
+/**
+ * Changing the page flags for a single page in the shadow page tables so as to
+ * make it supervisor and writable.
+ *
+ * This if for dealing with CR0.WP=0 and readonly user pages.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   fBigPage    Whether or not this is a big page. If it is, we have to
+ *                      change the shadow PDE as well.  If it isn't, the caller
+ *                      has checked that the shadow PDE doesn't need changing.
+ *                      We ASSUME 4KB pages backing the big page here!
+ * @param   fOpFlags    A combination of the PGM_MK_PG_XXX flags.
+ */
+int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
+{
+    int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
+    if (rc == VINF_SUCCESS && fBigPage)
+    {
+        /* this is a bit ugly... */
+        switch (pVCpu->pgm.s.enmShadowMode)
+        {
+            case PGMMODE_32_BIT:
+            {
+                PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
+                AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
+                Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
+                pPde->n.u1Write = 1;
+                Log(("-> PDE=%#llx (32)\n", pPde->u));
+                break;
+            }
+            case PGMMODE_PAE:
+            case PGMMODE_PAE_NX:
+            {
+                PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
+                AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
+                Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
+                pPde->n.u1Write = 1;
+                Log(("-> PDE=%#llx (PAE)\n", pPde->u));
+                break;
+            }
+            default:
+                AssertFailedReturn(VERR_INTERNAL_ERROR_4);
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Gets the shadow page directory for the specified address, PAE.
+ *
+ * @returns Pointer to the shadow PD.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ * @param   uGstPdpe    Guest PDPT entry. Valid.
+ * @param   ppPD        Receives address of page directory
+ */
+int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
+{
+    const unsigned iPdPt    = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+    PX86PDPT       pPdpt    = pgmShwGetPaePDPTPtr(pVCpu);
+    PX86PDPE       pPdpe    = &pPdpt->a[iPdPt];
+    PVMCC          pVM      = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL       pPool    = pVM->pgm.s.CTX_SUFF(pPool);
+    PPGMPOOLPAGE   pShwPage;
+    int            rc;
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /* Allocate page directory if not present. */
+    if (    !pPdpe->n.u1Present
+        &&  !(pPdpe->u & X86_PDPE_PG_MASK))
+    {
+        RTGCPTR64   GCPdPt;
+        PGMPOOLKIND enmKind;
+
+        if (pVM->pgm.s.fNestedPaging || !CPUMIsGuestPagingEnabled(pVCpu))
+        {
+            /* AMD-V nested paging or real/protected mode without paging. */
+            GCPdPt  = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
+            enmKind = PGMPOOLKIND_PAE_PD_PHYS;
+        }
+        else
+        {
+            if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
+            {
+                if (!(uGstPdpe & X86_PDPE_P))
+                {
+                    /* PD not present; guest must reload CR3 to change it.
+                     * No need to monitor anything in this case.
+                     */
+                    Assert(VM_IS_RAW_MODE_ENABLED(pVM));
+
+                    GCPdPt  = uGstPdpe & X86_PDPE_PG_MASK;
+                    enmKind = PGMPOOLKIND_PAE_PD_PHYS;
+                    uGstPdpe |= X86_PDPE_P;
+                }
+                else
+                {
+                    GCPdPt  = uGstPdpe & X86_PDPE_PG_MASK;
+                    enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
+                }
+            }
+            else
+            {
+                GCPdPt  = CPUMGetGuestCR3(pVCpu);
+                enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
+            }
+        }
+
+        /* Create a reference back to the PDPT by using the index in its shadow page. */
+        rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                          pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /*fLockPage*/,
+                          &pShwPage);
+        AssertRCReturn(rc, rc);
+
+        /* The PD was cached or created; hook it up now. */
+        pPdpe->u |= pShwPage->Core.Key | (uGstPdpe & (X86_PDPE_P | X86_PDPE_A));
+        PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
+    }
+    else
+    {
+        pShwPage = pgmPoolGetPage(pPool, pPdpe->u & X86_PDPE_PG_MASK);
+        AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+        Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
+
+        pgmPoolCacheUsed(pPool, pShwPage);
+    }
+    *ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the pointer to the shadow page directory entry for an address, PAE.
+ *
+ * @returns Pointer to the PDE.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtr       The address.
+ * @param   ppShwPde    Receives the address of the pgm pool page for the shadow page directory
+ */
+DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
+{
+    const unsigned  iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+    PX86PDPT        pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
+    PVM             pVM   = pVCpu->CTX_SUFF(pVM);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT);    /* can't happen */
+    if (!pPdpt->a[iPdPt].n.u1Present)
+    {
+        LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, pPdpt->a[iPdPt].u));
+        return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
+    }
+    AssertMsg(pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK);
+    AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+    *ppShwPde = pShwPde;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Syncs the SHADOW page directory pointer for the specified address.
+ *
+ * Allocates backing pages in case the PDPT or PML4 entry is missing.
+ *
+ * The caller is responsible for making sure the guest has a valid PD before
+ * calling this function.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ * @param   uGstPml4e   Guest PML4 entry (valid).
+ * @param   uGstPdpe    Guest PDPT entry (valid).
+ * @param   ppPD        Receives address of page directory
+ */
+static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
+{
+    PVMCC          pVM           = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL       pPool         = pVM->pgm.s.CTX_SUFF(pPool);
+    const unsigned iPml4         = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    PX86PML4E      pPml4e        = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
+    bool           fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging || !CPUMIsGuestPagingEnabled(pVCpu);
+    PPGMPOOLPAGE   pShwPage;
+    int            rc;
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /* Allocate page directory pointer table if not present. */
+    if (    !pPml4e->n.u1Present
+        &&  !(pPml4e->u & X86_PML4E_PG_MASK))
+    {
+        RTGCPTR64   GCPml4;
+        PGMPOOLKIND enmKind;
+
+        Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+
+        if (fNestedPagingOrNoGstPaging)
+        {
+            /* AMD-V nested paging or real/protected mode without paging */
+            GCPml4  = (RTGCPTR64)iPml4 << X86_PML4_SHIFT;
+            enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
+        }
+        else
+        {
+            GCPml4  = uGstPml4e & X86_PML4E_PG_MASK;
+            enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
+        }
+
+        /* Create a reference back to the PDPT by using the index in its shadow page. */
+        rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                          pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /*fLockPage*/,
+                          &pShwPage);
+        AssertRCReturn(rc, rc);
+    }
+    else
+    {
+        pShwPage = pgmPoolGetPage(pPool, pPml4e->u & X86_PML4E_PG_MASK);
+        AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+        pgmPoolCacheUsed(pPool, pShwPage);
+    }
+    /* The PDPT was cached or created; hook it up now. */
+    pPml4e->u |= pShwPage->Core.Key | (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask);
+
+    const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    PX86PDPT  pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    PX86PDPE  pPdpe = &pPdpt->a[iPdPt];
+
+    /* Allocate page directory if not present. */
+    if (    !pPdpe->n.u1Present
+        &&  !(pPdpe->u & X86_PDPE_PG_MASK))
+    {
+        RTGCPTR64   GCPdPt;
+        PGMPOOLKIND enmKind;
+
+        if (fNestedPagingOrNoGstPaging)
+        {
+            /* AMD-V nested paging or real/protected mode without paging */
+            GCPdPt  = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
+            enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
+        }
+        else
+        {
+            GCPdPt  = uGstPdpe & X86_PDPE_PG_MASK;
+            enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
+        }
+
+        /* Create a reference back to the PDPT by using the index in its shadow page. */
+        rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                          pShwPage->idx, iPdPt, false /*fLockPage*/,
+                          &pShwPage);
+        AssertRCReturn(rc, rc);
+    }
+    else
+    {
+        pShwPage = pgmPoolGetPage(pPool, pPdpe->u & X86_PDPE_PG_MASK);
+        AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+        pgmPoolCacheUsed(pPool, pShwPage);
+    }
+    /* The PD was cached or created; hook it up now. */
+    pPdpe->u |= pShwPage->Core.Key | (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask);
+
+    *ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the SHADOW page directory pointer for the specified address (long mode).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ * @param   ppPml4e     Receives the address of the page map level 4 entry.
+ * @param   ppPdpt      Receives the address of the page directory pointer table.
+ * @param   ppPD        Receives the address of the page directory.
+ */
+DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPT *ppPdpt, PX86PDPAE *ppPD)
+{
+    const unsigned  iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    PCX86PML4E      pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
+
+    PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
+
+    AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
+    if (ppPml4e)
+        *ppPml4e = (PX86PML4E)pPml4e;
+
+    Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, pPml4e->u));
+
+    if (!pPml4e->n.u1Present)
+        return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
+
+    PVM             pVM      = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL        pPool    = pVM->pgm.s.CTX_SUFF(pPool);
+    PPGMPOOLPAGE    pShwPage = pgmPoolGetPage(pPool, pPml4e->u & X86_PML4E_PG_MASK);
+    AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+    const unsigned  iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    PCX86PDPT       pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    if (!pPdpt->a[iPdPt].n.u1Present)
+        return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
+
+    pShwPage = pgmPoolGetPage(pPool, pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK);
+    AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+    *ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    Log4(("pgmShwGetLongModePDPtr %RGv -> *ppPD=%p PDE=%p/%RX64\n", GCPtr, *ppPD, &(*ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (*ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
+ * backing pages in case the PDPT or PML4 entry is missing.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ * @param   ppPdpt      Receives address of pdpt
+ * @param   ppPD        Receives address of page directory
+ */
+static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT *ppPdpt, PEPTPD *ppPD)
+{
+    PVMCC          pVM   = pVCpu->CTX_SUFF(pVM);
+    const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
+    PPGMPOOL       pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PEPTPML4       pPml4;
+    PEPTPML4E      pPml4e;
+    PPGMPOOLPAGE   pShwPage;
+    int            rc;
+
+    Assert(pVM->pgm.s.fNestedPaging);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+    Assert(pPml4);
+
+    /* Allocate page directory pointer table if not present. */
+    pPml4e = &pPml4->a[iPml4];
+    if (    !pPml4e->n.u1Present
+        &&  !(pPml4e->u & EPT_PML4E_PG_MASK))
+    {
+        Assert(!(pPml4e->u & EPT_PML4E_PG_MASK));
+        RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
+
+        rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                          pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /*fLockPage*/,
+                          &pShwPage);
+        AssertRCReturn(rc, rc);
+    }
+    else
+    {
+        pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
+        AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+        pgmPoolCacheUsed(pPool, pShwPage);
+    }
+    /* The PDPT was cached or created; hook it up now and fill with the default value. */
+    pPml4e->u           = pShwPage->Core.Key;
+    pPml4e->n.u1Present = 1;
+    pPml4e->n.u1Write   = 1;
+    pPml4e->n.u1Execute = 1;
+
+    const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
+    PEPTPDPT  pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
+
+    if (ppPdpt)
+        *ppPdpt = pPdpt;
+
+    /* Allocate page directory if not present. */
+    if (    !pPdpe->n.u1Present
+        &&  !(pPdpe->u & EPT_PDPTE_PG_MASK))
+    {
+        RTGCPTR64 GCPdPt = (RTGCPTR64)iPdPt << EPT_PDPT_SHIFT;
+        rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                          pShwPage->idx, iPdPt, false /*fLockPage*/,
+                          &pShwPage);
+        AssertRCReturn(rc, rc);
+    }
+    else
+    {
+        pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
+        AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+        pgmPoolCacheUsed(pPool, pShwPage);
+    }
+    /* The PD was cached or created; hook it up now and fill with the default value. */
+    pPdpe->u            = pShwPage->Core.Key;
+    pPdpe->n.u1Present  = 1;
+    pPdpe->n.u1Write    = 1;
+    pPdpe->n.u1Execute  = 1;
+
+    *ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    return VINF_SUCCESS;
+}
+
+
+#ifdef IN_RING0
+/**
+ * Synchronizes a range of nested page table entries.
+ *
+ * The caller must own the PGM lock.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPhys              Where to start.
+ * @param   cPages              How many pages which entries should be synced.
+ * @param   enmShwPagingMode    The shadow paging mode (PGMMODE_EPT for VT-x,
+ *                              host paging mode for AMD-V).
+ */
+int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
+{
+    PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
+
+/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
+    int rc;
+    switch (enmShwPagingMode)
+    {
+        case PGMMODE_32_BIT:
+        {
+            X86PDE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
+            rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/);
+            break;
+        }
+
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX:
+        {
+            X86PDEPAE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
+            rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/);
+            break;
+        }
+
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:
+        {
+            X86PDEPAE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
+            rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/);
+            break;
+        }
+
+        case PGMMODE_EPT:
+        {
+            X86PDEPAE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
+            rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/);
+            break;
+        }
+
+        default:
+            AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+    return rc;
+}
+#endif /* IN_RING0 */
+
+
+/**
+ * Gets effective Guest OS page information.
+ *
+ * When GCPtr is in a big page, the function will return as if it was a normal
+ * 4KB page. If the need for distinguishing between big and normal page becomes
+ * necessary at a later point, a PGMGstGetPage() will be created for that
+ * purpose.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtr       Guest Context virtual address of the page.
+ * @param   pfFlags     Where to store the flags. These are X86_PTE_*, even for big pages.
+ * @param   pGCPhys     Where to store the GC physical address of the page.
+ *                      This is page aligned. The fact that the
+ */
+VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
+    AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
+    return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pfFlags, pGCPhys);
+}
+
+
+/**
+ * Performs a guest page table walk.
+ *
+ * The guest should be in paged protect mode or long mode when making a call to
+ * this function.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT on failure.  Check pWalk for details.
+ * @retval  VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
+ *          not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtr       The guest virtual address to walk by.
+ * @param   pWalk       Where to return the walk result. This is valid for some
+ *                      error codes as well.
+ */
+int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    switch (pVCpu->pgm.s.enmGuestMode)
+    {
+        case PGMMODE_32_BIT:
+            pWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
+            return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, &pWalk->u.Legacy);
+
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX:
+            pWalk->enmType = PGMPTWALKGSTTYPE_PAE;
+            return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, &pWalk->u.Pae);
+
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:
+            pWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
+            return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, &pWalk->u.Amd64);
+
+        case PGMMODE_REAL:
+        case PGMMODE_PROTECTED:
+            pWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
+            return VERR_PGM_NOT_USED_IN_MODE;
+
+        case PGMMODE_NESTED_32BIT:
+        case PGMMODE_NESTED_PAE:
+        case PGMMODE_NESTED_AMD64:
+        case PGMMODE_EPT:
+        default:
+            AssertFailed();
+            pWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
+            return VERR_PGM_NOT_USED_IN_MODE;
+    }
+}
+
+
+/**
+ * Tries to continue the previous walk.
+ *
+ * @note    Requires the caller to hold the PGM lock from the first
+ *          pgmGstPtWalk() call to the last pgmGstPtWalkNext() call.  Otherwise
+ *          we cannot use the pointers.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT on failure.  Check pWalk for details.
+ * @retval  VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
+ *          not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtr       The guest virtual address to walk by.
+ * @param   pWalk       Pointer to the previous walk result and where to return
+ *                      the result of this walk.  This is valid for some error
+ *                      codes as well.
+ */
+int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk)
+{
+    /*
+     * We can only handle successfully walks.
+     * We also limit ourselves to the next page.
+     */
+    if (   pWalk->u.Core.fSucceeded
+        && GCPtr - pWalk->u.Core.GCPtr == PAGE_SIZE)
+    {
+        Assert(pWalk->u.Core.uLevel == 0);
+        if (pWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
+        {
+            /*
+             * AMD64
+             */
+            if (!pWalk->u.Core.fGigantPage && !pWalk->u.Core.fBigPage)
+            {
+                /*
+                 * We fall back to full walk if the PDE table changes, if any
+                 * reserved bits are set, or if the effective page access changes.
+                 */
+                const uint64_t fPteSame = X86_PTE_P   | X86_PTE_RW | X86_PTE_US     | X86_PTE_PWT
+                                        | X86_PTE_PCD | X86_PTE_A  | X86_PTE_PAE_NX;
+                const uint64_t fPdeSame = X86_PDE_P   | X86_PDE_RW | X86_PDE_US     | X86_PDE_PWT
+                                        | X86_PDE_PCD | X86_PDE_A  | X86_PDE_PAE_NX | X86_PDE_PS;
+
+                if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->u.Core.GCPtr >> X86_PD_PAE_SHIFT))
+                {
+                    if (pWalk->u.Amd64.pPte)
+                    {
+                        X86PTEPAE Pte;
+                        Pte.u = pWalk->u.Amd64.pPte[1].u;
+                        if (   (Pte.u & fPteSame) == (pWalk->u.Amd64.Pte.u & fPteSame)
+                            && !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
+                        {
+
+                            pWalk->u.Core.GCPtr  = GCPtr;
+                            pWalk->u.Core.GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
+                            pWalk->u.Amd64.Pte.u = Pte.u;
+                            pWalk->u.Amd64.pPte++;
+                            return VINF_SUCCESS;
+                        }
+                    }
+                }
+                else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->u.Core.GCPtr >> X86_PDPT_SHIFT))
+                {
+                    Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
+                    if (pWalk->u.Amd64.pPde)
+                    {
+                        X86PDEPAE Pde;
+                        Pde.u = pWalk->u.Amd64.pPde[1].u;
+                        if (   (Pde.u & fPdeSame) == (pWalk->u.Amd64.Pde.u & fPdeSame)
+                            && !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
+                        {
+                            /* Get the new PTE and check out the first entry. */
+                            int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
+                                                               &pWalk->u.Amd64.pPt);
+                            if (RT_SUCCESS(rc))
+                            {
+                                pWalk->u.Amd64.pPte = &pWalk->u.Amd64.pPt->a[0];
+                                X86PTEPAE Pte;
+                                Pte.u = pWalk->u.Amd64.pPte->u;
+                                if (   (Pte.u & fPteSame) == (pWalk->u.Amd64.Pte.u & fPteSame)
+                                    && !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
+                                {
+                                    pWalk->u.Core.GCPtr  = GCPtr;
+                                    pWalk->u.Core.GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
+                                    pWalk->u.Amd64.Pte.u = Pte.u;
+                                    pWalk->u.Amd64.Pde.u = Pde.u;
+                                    pWalk->u.Amd64.pPde++;
+                                    return VINF_SUCCESS;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            else if (!pWalk->u.Core.fGigantPage)
+            {
+                if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->u.Core.GCPtr & X86_PAGE_2M_BASE_MASK))
+                {
+                    pWalk->u.Core.GCPtr   = GCPtr;
+                    pWalk->u.Core.GCPhys += PAGE_SIZE;
+                    return VINF_SUCCESS;
+                }
+            }
+            else
+            {
+                if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->u.Core.GCPtr & X86_PAGE_1G_BASE_MASK))
+                {
+                    pWalk->u.Core.GCPtr   = GCPtr;
+                    pWalk->u.Core.GCPhys += PAGE_SIZE;
+                    return VINF_SUCCESS;
+                }
+            }
+        }
+    }
+    /* Case we don't handle.  Do full walk. */
+    return pgmGstPtWalk(pVCpu, GCPtr, pWalk);
+}
+
+
+/**
+ * Checks if the page is present.
+ *
+ * @returns true if the page is present.
+ * @returns false if the page is not present.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Address within the page.
+ */
+VMMDECL(bool) PGMGstIsPagePresent(PVMCPUCC pVCpu, RTGCPTR GCPtr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    int rc = PGMGstGetPage(pVCpu, GCPtr, NULL, NULL);
+    return RT_SUCCESS(rc);
+}
+
+
+/**
+ * Sets (replaces) the page flags for a range of pages in the guest's tables.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address of the first page.
+ * @param   cb          The size of the range in bytes.
+ * @param   fFlags      Page flags X86_PTE_*, excluding the page mask of course.
+ */
+VMMDECL(int)  PGMGstSetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    return PGMGstModifyPage(pVCpu, GCPtr, cb, fFlags, 0);
+}
+
+
+/**
+ * Modify page flags for a range of pages in the guest's tables
+ *
+ * The existing flags are ANDed with the fMask and ORed with the fFlags.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   cb          Size (in bytes) of the range to apply the modification to.
+ * @param   fFlags      The OR  mask - page flags X86_PTE_*, excluding the page mask of course.
+ * @param   fMask       The AND mask - page flags X86_PTE_*, excluding the page mask of course.
+ *                      Be very CAREFUL when ~'ing constants which could be 32-bit!
+ */
+VMMDECL(int)  PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
+{
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,GstModifyPage), a);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Validate input.
+     */
+    AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
+    Assert(cb);
+
+    LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
+
+    /*
+     * Adjust input.
+     */
+    cb     += GCPtr & PAGE_OFFSET_MASK;
+    cb      = RT_ALIGN_Z(cb, PAGE_SIZE);
+    GCPtr   = (GCPtr & PAGE_BASE_GC_MASK);
+
+    /*
+     * Call worker.
+     */
+    uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
+    AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
+    int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,GstModifyPage), a);
+    return rc;
+}
+
+
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+
+/**
+ * Performs the lazy mapping of the 32-bit guest PD.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   ppPd        Where to return the pointer to the mapping.  This is
+ *                      always set.
+ */
+int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
+{
+    PVMCC       pVM = pVCpu->CTX_SUFF(pVM);
+    pgmLock(pVM);
+
+    Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
+
+    RTGCPHYS    GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAGE_MASK;
+    PPGMPAGE    pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+# ifdef VBOX_WITH_RAM_IN_KERNEL
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd);
+        if (RT_SUCCESS(rc))
+        {
+#  ifdef IN_RING3
+            pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
+            pVCpu->pgm.s.pGst32BitPdR3 = *ppPd;
+#  else
+            pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR;
+            pVCpu->pgm.s.pGst32BitPdR0 = *ppPd;
+#  endif
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+# else
+        RTHCPTR HCPtrGuestCR3;
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)&HCPtrGuestCR3);
+        if (RT_SUCCESS(rc))
+        {
+            pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
+#  ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+            pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
+#  endif
+            *ppPd = (PX86PD)HCPtrGuestCR3;
+
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+# endif
+        AssertRC(rc);
+    }
+    pgmUnlock(pVM);
+
+    *ppPd = NULL;
+    return rc;
+}
+
+
+/**
+ * Performs the lazy mapping of the PAE guest PDPT.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   ppPdpt      Where to return the pointer to the mapping.  This is
+ *                      always set.
+ */
+int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
+{
+    Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
+    PVMCC       pVM = pVCpu->CTX_SUFF(pVM);
+    pgmLock(pVM);
+
+    RTGCPHYS    GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAE_PAGE_MASK;
+    PPGMPAGE    pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+# ifdef VBOX_WITH_RAM_IN_KERNEL
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt);
+        if (RT_SUCCESS(rc))
+        {
+#  ifdef IN_RING3
+            pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
+            pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt;
+#  else
+            pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
+            pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt;
+#  endif
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+# else
+        RTHCPTR HCPtrGuestCR3;
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)&HCPtrGuestCR3);
+        if (RT_SUCCESS(rc))
+        {
+            pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
+#  ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+            pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
+#  endif
+            *ppPdpt = (PX86PDPT)HCPtrGuestCR3;
+
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+# endif
+        AssertRC(rc);
+    }
+
+    pgmUnlock(pVM);
+    *ppPdpt = NULL;
+    return rc;
+}
+
+
+/**
+ * Performs the lazy mapping / updating of a PAE guest PD.
+ *
+ * @returns Pointer to the mapping.
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   iPdpt       Which PD entry to map (0..3).
+ * @param   ppPd        Where to return the pointer to the mapping.  This is
+ *                      always set.
+ */
+int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
+{
+    PVMCC           pVM         = pVCpu->CTX_SUFF(pVM);
+    pgmLock(pVM);
+
+    PX86PDPT        pGuestPDPT  = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
+    Assert(pGuestPDPT);
+    Assert(pGuestPDPT->a[iPdpt].n.u1Present);
+    RTGCPHYS        GCPhys      = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
+    bool const      fChanged    = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
+
+    PPGMPAGE        pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+# ifdef VBOX_WITH_RAM_IN_KERNEL
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd);
+        AssertRC(rc);
+        if (RT_SUCCESS(rc))
+        {
+#  ifdef IN_RING3
+            pVCpu->pgm.s.apGstPaePDsR0[iPdpt]          = NIL_RTR0PTR;
+            pVCpu->pgm.s.apGstPaePDsR3[iPdpt]          = *ppPd;
+#  else
+            pVCpu->pgm.s.apGstPaePDsR3[iPdpt]          = NIL_RTR3PTR;
+            pVCpu->pgm.s.apGstPaePDsR0[iPdpt]          = *ppPd;
+#  endif
+            if (fChanged)
+                pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt]   = GCPhys;
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+# else
+        RTHCPTR     HCPtr       = NIL_RTHCPTR;
+#  ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, &HCPtr);
+        AssertRC(rc);
+#  endif
+        if (RT_SUCCESS(rc))
+        {
+            pVCpu->pgm.s.apGstPaePDsR3[iPdpt]          = (R3PTRTYPE(PX86PDPAE))HCPtr;
+#  ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+            pVCpu->pgm.s.apGstPaePDsR0[iPdpt]          = (R0PTRTYPE(PX86PDPAE))HCPtr;
+#  endif
+            if (fChanged)
+                pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt]   = GCPhys;
+
+            *ppPd = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt];
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+# endif
+    }
+
+    /* Invalid page or some failure, invalidate the entry. */
+    pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt]   = NIL_RTGCPHYS;
+    pVCpu->pgm.s.apGstPaePDsR3[iPdpt]      = NIL_RTR3PTR;
+# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    pVCpu->pgm.s.apGstPaePDsR0[iPdpt]      = NIL_RTR0PTR;
+# endif
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Performs the lazy mapping of the 32-bit guest PD.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   ppPml4      Where to return the pointer to the mapping.  This will
+ *                      always be set.
+ */
+int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
+{
+    Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
+    PVMCC       pVM = pVCpu->CTX_SUFF(pVM);
+    pgmLock(pVM);
+
+    RTGCPHYS    GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_AMD64_PAGE_MASK;
+    PPGMPAGE    pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+# ifdef VBOX_WITH_RAM_IN_KERNEL
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4);
+        if (RT_SUCCESS(rc))
+        {
+#  ifdef IN_RING3
+            pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
+            pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4;
+#  else
+            pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
+            pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4;
+#  endif
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+# else
+        RTHCPTR HCPtrGuestCR3;
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)&HCPtrGuestCR3);
+        if (RT_SUCCESS(rc))
+        {
+            pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
+#  ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+            pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
+#  endif
+            *ppPml4 = (PX86PML4)HCPtrGuestCR3;
+
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+# endif
+    }
+
+    pgmUnlock(pVM);
+    *ppPml4 = NULL;
+    return rc;
+}
+
+#endif /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+
+
+/**
+ * Gets the PAE PDPEs values cached by the CPU.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   paPdpes             Where to return the four PDPEs. The array
+ *                              pointed to must have 4 entries.
+ */
+VMM_INT_DECL(int) PGMGstGetPaePdpes(PVMCPUCC pVCpu, PX86PDPE paPdpes)
+{
+    Assert(pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
+
+    paPdpes[0] = pVCpu->pgm.s.aGstPaePdpeRegs[0];
+    paPdpes[1] = pVCpu->pgm.s.aGstPaePdpeRegs[1];
+    paPdpes[2] = pVCpu->pgm.s.aGstPaePdpeRegs[2];
+    paPdpes[3] = pVCpu->pgm.s.aGstPaePdpeRegs[3];
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the PAE PDPEs values cached by the CPU.
+ *
+ * @remarks This must be called *AFTER* PGMUpdateCR3.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   paPdpes             The four PDPE values. The array pointed to must
+ *                              have exactly 4 entries.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+VMM_INT_DECL(void) PGMGstUpdatePaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPdpes)
+{
+    Assert(pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGstPaePdpeRegs); i++)
+    {
+        if (pVCpu->pgm.s.aGstPaePdpeRegs[i].u != paPdpes[i].u)
+        {
+            pVCpu->pgm.s.aGstPaePdpeRegs[i] = paPdpes[i];
+
+            /* Force lazy remapping if it changed in any way. */
+            pVCpu->pgm.s.apGstPaePDsR3[i]     = 0;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+            pVCpu->pgm.s.apGstPaePDsR0[i]     = 0;
+#endif
+            pVCpu->pgm.s.aGCPhysGstPaePDs[i]  = NIL_RTGCPHYS;
+        }
+    }
+
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
+}
+
+
+/**
+ * Gets the current CR3 register value for the shadow memory context.
+ * @returns CR3 value.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
+{
+    PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+    AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
+    return pPoolPage->Core.Key;
+}
+
+
+/**
+ * Performs and schedules necessary updates following a CR3 load or reload.
+ *
+ * This will normally involve mapping the guest PD or nPDPT
+ *
+ * @returns VBox status code.
+ * @retval  VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
+ *          safely be ignored and overridden since the FF will be set too then.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cr3         The new cr3.
+ * @param   fGlobal     Indicates whether this is a global flush or not.
+ */
+VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal)
+{
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLB), a);
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Always flag the necessary updates; necessary for hardware acceleration
+     */
+    /** @todo optimize this, it shouldn't always be necessary. */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+    if (fGlobal)
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    LogFlow(("PGMFlushTLB: cr3=%RX64 OldCr3=%RX64 fGlobal=%d\n", cr3, pVCpu->pgm.s.GCPhysCR3, fGlobal));
+
+    /*
+     * Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
+     */
+    int rc = VINF_SUCCESS;
+    RTGCPHYS GCPhysCR3;
+    switch (pVCpu->pgm.s.enmGuestMode)
+    {
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX:
+            GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
+            break;
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:
+            GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_AMD64_PAGE_MASK);
+            break;
+        default:
+            GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
+            break;
+    }
+    PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
+
+    RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
+    if (GCPhysOldCR3 != GCPhysCR3)
+    {
+        uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+        AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+        AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
+
+        pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
+        rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
+        if (RT_LIKELY(rc == VINF_SUCCESS))
+        {
+            if (pgmMapAreMappingsFloating(pVM))
+                pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
+        }
+        else
+        {
+            AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
+            Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_PGM_SYNC_CR3));
+            pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
+            pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_MAP_CR3;
+            if (pgmMapAreMappingsFloating(pVM))
+                pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_MONITOR_CR3;
+        }
+
+        if (fGlobal)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBNewCR3Global));
+        else
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBNewCR3));
+    }
+    else
+    {
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+        PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+        if (pPool->cDirtyPages)
+        {
+            pgmLock(pVM);
+            pgmPoolResetDirtyPages(pVM);
+            pgmUnlock(pVM);
+        }
+#endif
+        /*
+         * Check if we have a pending update of the CR3 monitoring.
+         */
+        if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
+        {
+            pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
+            Assert(!pVM->pgm.s.fMappingsFixed); Assert(pgmMapAreMappingsEnabled(pVM));
+        }
+        if (fGlobal)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBSameCR3Global));
+        else
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBSameCR3));
+    }
+
+    IEMTlbInvalidateAll(pVCpu, false /*fVmm*/);
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLB), a);
+    return rc;
+}
+
+
+/**
+ * Performs and schedules necessary updates following a CR3 load or reload when
+ * using nested or extended paging.
+ *
+ * This API is an alternative to PGMFlushTLB that avoids actually flushing the
+ * TLB and triggering a SyncCR3.
+ *
+ * This will normally involve mapping the guest PD or nPDPT
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS.
+ * @retval  VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
+ *          paging modes).  This can safely be ignored and overridden since the
+ *          FF will be set too then.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cr3         The new cr3.
+ */
+VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    LogFlow(("PGMUpdateCR3: cr3=%RX64 OldCr3=%RX64\n", cr3, pVCpu->pgm.s.GCPhysCR3));
+
+    /* We assume we're only called in nested paging mode. */
+    Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging || pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
+    Assert(!pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM)));
+    Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3));
+
+    /*
+     * Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
+     */
+    int rc = VINF_SUCCESS;
+    RTGCPHYS GCPhysCR3;
+    switch (pVCpu->pgm.s.enmGuestMode)
+    {
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX:
+            GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
+            break;
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:
+            GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_AMD64_PAGE_MASK);
+            break;
+        default:
+            GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
+            break;
+    }
+    PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
+
+    if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
+    {
+        uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+        AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+        AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
+
+        pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
+        rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
+
+        AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. */ /** @todo this isn't true for the mac, but we need hw to test/fix this. */
+    }
+
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
+    return rc;
+}
+
+
+/**
+ * Synchronize the paging structures.
+ *
+ * This function is called in response to the VM_FF_PGM_SYNC_CR3 and
+ * VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
+ * in several places, most importantly whenever the CR3 is loaded.
+ *
+ * @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
+ * @retval  VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
+ *          the VMM into guest context.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cr0         Guest context CR0 register
+ * @param   cr3         Guest context CR3 register
+ * @param   cr4         Guest context CR4 register
+ * @param   fGlobal     Including global page directories or not
+ */
+VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
+{
+    int rc;
+
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * The pool may have pending stuff and even require a return to ring-3 to
+     * clear the whole thing.
+     */
+    rc = pgmPoolSyncCR3(pVCpu);
+    if (rc != VINF_SUCCESS)
+        return rc;
+
+    /*
+     * We might be called when we shouldn't.
+     *
+     * The mode switching will ensure that the PD is resynced after every mode
+     * switch.  So, if we find ourselves here when in protected or real mode
+     * we can safely clear the FF and return immediately.
+     */
+    if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
+    {
+        Assert((cr0 & (X86_CR0_PG | X86_CR0_PE)) != (X86_CR0_PG | X86_CR0_PE));
+        Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+        return VINF_SUCCESS;
+    }
+
+    /* If global pages are not supported, then all flushes are global. */
+    if (!(cr4 & X86_CR4_PGE))
+        fGlobal = true;
+    LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
+             VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
+
+    /*
+     * Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
+     * This should be done before SyncCR3.
+     */
+    if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
+    {
+        pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
+
+        RTGCPHYS GCPhysCR3Old = pVCpu->pgm.s.GCPhysCR3; NOREF(GCPhysCR3Old);
+        RTGCPHYS GCPhysCR3;
+        switch (pVCpu->pgm.s.enmGuestMode)
+        {
+            case PGMMODE_PAE:
+            case PGMMODE_PAE_NX:
+                GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
+                break;
+            case PGMMODE_AMD64:
+            case PGMMODE_AMD64_NX:
+                GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_AMD64_PAGE_MASK);
+                break;
+            default:
+                GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
+                break;
+        }
+        PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
+
+        if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
+        {
+            uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+            AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+            AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
+            pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
+            rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
+        }
+
+        /* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
+        if (    rc == VINF_PGM_SYNC_CR3
+            ||  (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
+        {
+            Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
+#ifdef IN_RING3
+            rc = pgmPoolSyncCR3(pVCpu);
+#else
+            if (rc == VINF_PGM_SYNC_CR3)
+                pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3Old;
+            return VINF_PGM_SYNC_CR3;
+#endif
+        }
+        AssertRCReturn(rc, rc);
+        AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
+    }
+
+    /*
+     * Let the 'Bth' function do the work and we'll just keep track of the flags.
+     */
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
+
+    uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
+    rc = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
+    AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("rc=%Rrc\n", rc));
+    if (rc == VINF_SUCCESS)
+    {
+        if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
+        {
+            /* Go back to ring 3 if a pgm pool sync is again pending. */
+            return VINF_PGM_SYNC_CR3;
+        }
+
+        if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
+        {
+            Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+        }
+
+        /*
+         * Check if we have a pending update of the CR3 monitoring.
+         */
+        if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
+        {
+            pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
+            Assert(!pVCpu->CTX_SUFF(pVM)->pgm.s.fMappingsFixed);
+            Assert(pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM)));
+        }
+    }
+
+    /*
+     * Now flush the CR3 (guest context).
+     */
+    if (rc == VINF_SUCCESS)
+        PGM_INVL_VCPU_TLBS(pVCpu);
+    return rc;
+}
+
+
+/**
+ * Called whenever CR0 or CR4 in a way which may affect the paging mode.
+ *
+ * @returns VBox status code, with the following informational code for
+ *          VM scheduling.
+ * @retval  VINF_SUCCESS if the was no change, or it was successfully dealt with.
+ * @retval  VINF_PGM_CHANGE_MODE if we're in RC the mode changes.  This will
+ *          NOT be returned in ring-3 or ring-0.
+ * @retval  VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cr0         The new cr0.
+ * @param   cr4         The new cr4.
+ * @param   efer        The new extended feature enable register.
+ */
+VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Calc the new guest mode.
+     *
+     * Note! We check PG before PE and without requiring PE because of the
+     *       special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
+     */
+    PGMMODE enmGuestMode;
+    if (cr0 & X86_CR0_PG)
+    {
+        if (!(cr4 & X86_CR4_PAE))
+        {
+            bool const fPse = !!(cr4 & X86_CR4_PSE);
+            if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
+                Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
+            pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
+            enmGuestMode = PGMMODE_32_BIT;
+        }
+        else if (!(efer & MSR_K6_EFER_LME))
+        {
+            if (!(efer & MSR_K6_EFER_NXE))
+                enmGuestMode = PGMMODE_PAE;
+            else
+                enmGuestMode = PGMMODE_PAE_NX;
+        }
+        else
+        {
+            if (!(efer & MSR_K6_EFER_NXE))
+                enmGuestMode = PGMMODE_AMD64;
+            else
+                enmGuestMode = PGMMODE_AMD64_NX;
+        }
+    }
+    else if (!(cr0 & X86_CR0_PE))
+        enmGuestMode = PGMMODE_REAL;
+    else
+        enmGuestMode = PGMMODE_PROTECTED;
+
+    /*
+     * Did it change?
+     */
+    if (pVCpu->pgm.s.enmGuestMode == enmGuestMode)
+        return VINF_SUCCESS;
+
+    /* Flush the TLB */
+    PGM_INVL_VCPU_TLBS(pVCpu);
+    return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode);
+}
+
+
+/**
+ * Converts a PGMMODE value to a PGM_TYPE_* \#define.
+ *
+ * @returns PGM_TYPE_*.
+ * @param   pgmMode     The mode value to convert.
+ */
+DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
+{
+    switch (pgmMode)
+    {
+        case PGMMODE_REAL:          return PGM_TYPE_REAL;
+        case PGMMODE_PROTECTED:     return PGM_TYPE_PROT;
+        case PGMMODE_32_BIT:        return PGM_TYPE_32BIT;
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX:        return PGM_TYPE_PAE;
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:      return PGM_TYPE_AMD64;
+        case PGMMODE_NESTED_32BIT:  return PGM_TYPE_NESTED_32BIT;
+        case PGMMODE_NESTED_PAE:    return PGM_TYPE_NESTED_PAE;
+        case PGMMODE_NESTED_AMD64:  return PGM_TYPE_NESTED_AMD64;
+        case PGMMODE_EPT:           return PGM_TYPE_EPT;
+        case PGMMODE_NONE:          return PGM_TYPE_NONE;
+        default:
+            AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
+    }
+}
+
+
+/**
+ * Calculates the shadow paging mode.
+ *
+ * @returns The shadow paging mode.
+ * @param   pVM             The cross context VM structure.
+ * @param   enmGuestMode    The guest mode.
+ * @param   enmHostMode     The host mode.
+ * @param   enmShadowMode   The current shadow mode.
+ */
+static PGMMODE pgmCalcShadowMode(PVM pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
+{
+    switch (enmGuestMode)
+    {
+        /*
+         * When switching to real or protected mode we don't change
+         * anything since it's likely that we'll switch back pretty soon.
+         *
+         * During pgmR3InitPaging we'll end up here with PGMMODE_INVALID
+         * and is supposed to determine which shadow paging and switcher to
+         * use during init.
+         */
+        case PGMMODE_REAL:
+        case PGMMODE_PROTECTED:
+            if (    enmShadowMode != PGMMODE_INVALID
+                && VM_IS_RAW_MODE_ENABLED(pVM) /* always switch in hm and nem modes! */)
+                break; /* (no change) */
+
+            switch (enmHostMode)
+            {
+                case SUPPAGINGMODE_32_BIT:
+                case SUPPAGINGMODE_32_BIT_GLOBAL:
+                    enmShadowMode = PGMMODE_32_BIT;
+                    break;
+
+                case SUPPAGINGMODE_PAE:
+                case SUPPAGINGMODE_PAE_NX:
+                case SUPPAGINGMODE_PAE_GLOBAL:
+                case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                case SUPPAGINGMODE_AMD64:
+                case SUPPAGINGMODE_AMD64_GLOBAL:
+                case SUPPAGINGMODE_AMD64_NX:
+                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                default:
+                    AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
+            }
+            break;
+
+        case PGMMODE_32_BIT:
+            switch (enmHostMode)
+            {
+                case SUPPAGINGMODE_32_BIT:
+                case SUPPAGINGMODE_32_BIT_GLOBAL:
+                    enmShadowMode = PGMMODE_32_BIT;
+                    break;
+
+                case SUPPAGINGMODE_PAE:
+                case SUPPAGINGMODE_PAE_NX:
+                case SUPPAGINGMODE_PAE_GLOBAL:
+                case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                case SUPPAGINGMODE_AMD64:
+                case SUPPAGINGMODE_AMD64_GLOBAL:
+                case SUPPAGINGMODE_AMD64_NX:
+                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                default:
+                    AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
+            }
+            break;
+
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
+            switch (enmHostMode)
+            {
+                case SUPPAGINGMODE_32_BIT:
+                case SUPPAGINGMODE_32_BIT_GLOBAL:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                case SUPPAGINGMODE_PAE:
+                case SUPPAGINGMODE_PAE_NX:
+                case SUPPAGINGMODE_PAE_GLOBAL:
+                case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                case SUPPAGINGMODE_AMD64:
+                case SUPPAGINGMODE_AMD64_GLOBAL:
+                case SUPPAGINGMODE_AMD64_NX:
+                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                default:
+                    AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
+            }
+            break;
+
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:
+            switch (enmHostMode)
+            {
+                case SUPPAGINGMODE_32_BIT:
+                case SUPPAGINGMODE_32_BIT_GLOBAL:
+                    enmShadowMode = PGMMODE_AMD64;
+                    break;
+
+                case SUPPAGINGMODE_PAE:
+                case SUPPAGINGMODE_PAE_NX:
+                case SUPPAGINGMODE_PAE_GLOBAL:
+                case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_AMD64;
+                    break;
+
+                case SUPPAGINGMODE_AMD64:
+                case SUPPAGINGMODE_AMD64_GLOBAL:
+                case SUPPAGINGMODE_AMD64_NX:
+                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_AMD64;
+                    break;
+
+                default:
+                    AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
+            }
+            break;
+
+        default:
+            AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
+    }
+
+    /*
+     * Override the shadow mode when NEM or nested paging is active.
+     */
+    if (VM_IS_NEM_ENABLED(pVM))
+    {
+        pVM->pgm.s.fNestedPaging = true;
+        enmShadowMode = PGMMODE_NONE;
+    }
+    else
+    {
+        bool fNestedPaging = HMIsNestedPagingActive(pVM);
+        pVM->pgm.s.fNestedPaging = fNestedPaging;
+        if (fNestedPaging)
+        {
+            if (HMIsVmxActive(pVM))
+                enmShadowMode = PGMMODE_EPT;
+            else
+            {
+                /* The nested SVM paging depends on the host one. */
+                Assert(HMIsSvmActive(pVM));
+                if (   enmGuestMode == PGMMODE_AMD64
+                    || enmGuestMode == PGMMODE_AMD64_NX)
+                    enmShadowMode = PGMMODE_NESTED_AMD64;
+                else
+                    switch (pVM->pgm.s.enmHostMode)
+                    {
+                        case SUPPAGINGMODE_32_BIT:
+                        case SUPPAGINGMODE_32_BIT_GLOBAL:
+                            enmShadowMode = PGMMODE_NESTED_32BIT;
+                            break;
+
+                        case SUPPAGINGMODE_PAE:
+                        case SUPPAGINGMODE_PAE_GLOBAL:
+                        case SUPPAGINGMODE_PAE_NX:
+                        case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                            enmShadowMode = PGMMODE_NESTED_PAE;
+                            break;
+
+                        case SUPPAGINGMODE_AMD64:
+                        case SUPPAGINGMODE_AMD64_GLOBAL:
+                        case SUPPAGINGMODE_AMD64_NX:
+                        case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                            enmShadowMode = PGMMODE_NESTED_AMD64;
+                            break;
+
+                        default:
+                            AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
+                    }
+            }
+        }
+    }
+
+    return enmShadowMode;
+}
+
+
+/**
+ * Performs the actual mode change.
+ * This is called by PGMChangeMode and pgmR3InitPaging().
+ *
+ * @returns VBox status code. May suspend or power off the VM on error, but this
+ *          will trigger using FFs and not informational status codes.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmGuestMode    The new guest mode. This is assumed to be different from
+ *                          the current mode.
+ */
+VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode)
+{
+    Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
+    STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
+
+    /*
+     * Calc the shadow mode and switcher.
+     */
+    PGMMODE enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
+
+    /*
+     * Exit old mode(s).
+     */
+    /* shadow */
+    if (enmShadowMode != pVCpu->pgm.s.enmShadowMode)
+    {
+        LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n",  PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
+        uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
+        if (   idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
+            && g_aPgmShadowModeData[idxOldShw].pfnExit)
+        {
+            int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
+            AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
+        }
+    }
+    else
+        LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n",  PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
+
+    /* guest */
+    uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
+    if (   idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
+        && g_aPgmGuestModeData[idxOldGst].pfnExit)
+    {
+        int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
+        AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
+    }
+    pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
+
+    /*
+     * Change the paging mode data indexes.
+     */
+    uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
+    AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPDE, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
+#ifdef IN_RING3
+    AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
+#endif
+
+    uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
+    AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
+#ifdef IN_RING3
+    AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
+#endif
+
+    uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
+    AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
+#ifdef VBOX_STRICT
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
+#endif
+
+    /*
+     * Enter new shadow mode (if changed).
+     */
+    if (enmShadowMode != pVCpu->pgm.s.enmShadowMode)
+    {
+        pVCpu->pgm.s.enmShadowMode = enmShadowMode;
+        int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu, enmGuestMode >= PGMMODE_AMD64);
+        AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
+                                    pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
+    }
+
+    /*
+     * Always flag the necessary updates
+     */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+
+    /*
+     * Enter the new guest and shadow+guest modes.
+     */
+    /* Calc the new CR3 value. */
+    RTGCPHYS GCPhysCR3;
+    switch (enmGuestMode)
+    {
+        case PGMMODE_REAL:
+        case PGMMODE_PROTECTED:
+            GCPhysCR3 = NIL_RTGCPHYS;
+            break;
+
+        case PGMMODE_32_BIT:
+            GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
+            break;
+
+        case PGMMODE_PAE_NX:
+        case PGMMODE_PAE:
+            if (!pVM->cpum.ro.GuestFeatures.fPae)
+                return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
+                                         N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
+            GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
+            break;
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+        case PGMMODE_AMD64_NX:
+        case PGMMODE_AMD64:
+            GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
+            break;
+#endif
+        default:
+            AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
+    }
+
+    /* Enter the new guest mode.  */
+    pVCpu->pgm.s.enmGuestMode = enmGuestMode;
+    int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
+    int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
+
+    /* Set the new guest CR3. */
+    pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
+
+    /* status codes. */
+    AssertRC(rc);
+    AssertRC(rc2);
+    if (RT_SUCCESS(rc))
+    {
+        rc = rc2;
+        if (RT_SUCCESS(rc)) /* no informational status codes. */
+            rc = VINF_SUCCESS;
+    }
+
+    /*
+     * Notify HM.
+     */
+    HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
+    return rc;
+}
+
+
+/**
+ * Called by CPUM or REM when CR0.WP changes to 1.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT
+ */
+VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
+{
+    /*
+     * Netware WP0+RO+US hack cleanup when WP0 -> WP1.
+     *
+     * Use the counter to judge whether there might be pool pages with active
+     * hacks in them.  If there are, we will be running the risk of messing up
+     * the guest by allowing it to write to read-only pages.  Thus, we have to
+     * clear the page pool ASAP if there is the slightest chance.
+     */
+    if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
+    {
+        Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
+
+        Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
+        pVCpu->pgm.s.cNetwareWp0Hacks = 0;
+        pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    }
+}
+
+
+/**
+ * Gets the current guest paging mode.
+ *
+ * If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
+ *
+ * @returns The current paging mode.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
+{
+    return pVCpu->pgm.s.enmGuestMode;
+}
+
+
+/**
+ * Gets the current shadow paging mode.
+ *
+ * @returns The current paging mode.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
+{
+    return pVCpu->pgm.s.enmShadowMode;
+}
+
+
+/**
+ * Gets the current host paging mode.
+ *
+ * @returns The current paging mode.
+ * @param   pVM             The cross context VM structure.
+ */
+VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
+{
+    switch (pVM->pgm.s.enmHostMode)
+    {
+        case SUPPAGINGMODE_32_BIT:
+        case SUPPAGINGMODE_32_BIT_GLOBAL:
+            return PGMMODE_32_BIT;
+
+        case SUPPAGINGMODE_PAE:
+        case SUPPAGINGMODE_PAE_GLOBAL:
+            return PGMMODE_PAE;
+
+        case SUPPAGINGMODE_PAE_NX:
+        case SUPPAGINGMODE_PAE_GLOBAL_NX:
+            return PGMMODE_PAE_NX;
+
+        case SUPPAGINGMODE_AMD64:
+        case SUPPAGINGMODE_AMD64_GLOBAL:
+            return PGMMODE_AMD64;
+
+        case SUPPAGINGMODE_AMD64_NX:
+        case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+            return PGMMODE_AMD64_NX;
+
+        default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
+    }
+
+    return PGMMODE_INVALID;
+}
+
+
+/**
+ * Get mode name.
+ *
+ * @returns read-only name string.
+ * @param   enmMode     The mode which name is desired.
+ */
+VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
+{
+    switch (enmMode)
+    {
+        case PGMMODE_REAL:          return "Real";
+        case PGMMODE_PROTECTED:     return "Protected";
+        case PGMMODE_32_BIT:        return "32-bit";
+        case PGMMODE_PAE:           return "PAE";
+        case PGMMODE_PAE_NX:        return "PAE+NX";
+        case PGMMODE_AMD64:         return "AMD64";
+        case PGMMODE_AMD64_NX:      return "AMD64+NX";
+        case PGMMODE_NESTED_32BIT:  return "Nested-32";
+        case PGMMODE_NESTED_PAE:    return "Nested-PAE";
+        case PGMMODE_NESTED_AMD64:  return "Nested-AMD64";
+        case PGMMODE_EPT:           return "EPT";
+        case PGMMODE_NONE:          return "None";
+        default:                    return "unknown mode value";
+    }
+}
+
+
+/**
+ * Gets the physical address represented in the guest CR3 as PGM sees it.
+ *
+ * This is mainly for logging and debugging.
+ *
+ * @returns PGM's guest CR3 value.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(RTGCPHYS) PGMGetGuestCR3Phys(PVMCPU pVCpu)
+{
+    return pVCpu->pgm.s.GCPhysCR3;
+}
+
+
+
+/**
+ * Notification from CPUM that the EFER.NXE bit has changed.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the CPU for
+ *                      which EFER changed.
+ * @param   fNxe        The new NXE state.
+ */
+VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
+{
+/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
+    Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
+
+    pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
+    if (fNxe)
+    {
+        /*pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A */
+        pVCpu->pgm.s.fGstPaeMbzPteMask       &= ~X86_PTE_PAE_NX;
+        pVCpu->pgm.s.fGstPaeMbzPdeMask       &= ~X86_PDE_PAE_NX;
+        pVCpu->pgm.s.fGstPaeMbzBigPdeMask    &= ~X86_PDE2M_PAE_NX;
+        /*pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A */
+        pVCpu->pgm.s.fGstAmd64MbzPteMask     &= ~X86_PTE_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPdeMask     &= ~X86_PDE_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64MbzBigPdeMask  &= ~X86_PDE2M_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPdpeMask    &= ~X86_PDPE_LM_NX;
+        pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPml4eMask   &= ~X86_PML4E_NX;
+
+        pVCpu->pgm.s.fGst64ShadowedPteMask        |= X86_PTE_PAE_NX;
+        pVCpu->pgm.s.fGst64ShadowedPdeMask        |= X86_PDE_PAE_NX;
+        pVCpu->pgm.s.fGst64ShadowedBigPdeMask     |= X86_PDE2M_PAE_NX;
+        pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask |= X86_PDE2M_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask    |= X86_PDPE_LM_NX;
+        pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask   |= X86_PML4E_NX;
+    }
+    else
+    {
+        /*pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A */
+        pVCpu->pgm.s.fGstPaeMbzPteMask       |= X86_PTE_PAE_NX;
+        pVCpu->pgm.s.fGstPaeMbzPdeMask       |= X86_PDE_PAE_NX;
+        pVCpu->pgm.s.fGstPaeMbzBigPdeMask    |= X86_PDE2M_PAE_NX;
+        /*pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A */
+        pVCpu->pgm.s.fGstAmd64MbzPteMask     |= X86_PTE_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPdeMask     |= X86_PDE_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64MbzBigPdeMask  |= X86_PDE2M_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPdpeMask    |= X86_PDPE_LM_NX;
+        pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask |= X86_PDPE_LM_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPml4eMask   |= X86_PML4E_NX;
+
+        pVCpu->pgm.s.fGst64ShadowedPteMask        &= ~X86_PTE_PAE_NX;
+        pVCpu->pgm.s.fGst64ShadowedPdeMask        &= ~X86_PDE_PAE_NX;
+        pVCpu->pgm.s.fGst64ShadowedBigPdeMask     &= ~X86_PDE2M_PAE_NX;
+        pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask    &= ~X86_PDPE_LM_NX;
+        pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask   &= ~X86_PML4E_NX;
+    }
+}
+
+
+/**
+ * Check if any pgm pool pages are marked dirty (not monitored)
+ *
+ * @returns bool locked/not locked
+ * @param   pVM         The cross context VM structure.
+ */
+VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
+{
+    return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
+}
+
+
+/**
+ * Check if this VCPU currently owns the PGM lock.
+ *
+ * @returns bool owner/not owner
+ * @param   pVM         The cross context VM structure.
+ */
+VMMDECL(bool) PGMIsLockOwner(PVM pVM)
+{
+    return PDMCritSectIsOwner(&pVM->pgm.s.CritSectX);
+}
+
+
+/**
+ * Enable or disable large page usage
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   fUseLargePages  Use/not use large pages
+ */
+VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    pVM->fUseLargePages = fUseLargePages;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Acquire the PGM lock.
+ *
+ * @returns VBox status code
+ * @param   pVM         The cross context VM structure.
+ * @param   SRC_POS     The source position of the caller (RT_SRC_POS).
+ */
+#if (defined(VBOX_STRICT) && defined(IN_RING3)) || defined(DOXYGEN_RUNNING)
+int pgmLockDebug(PVMCC pVM, RT_SRC_POS_DECL)
+#else
+int pgmLock(PVMCC pVM)
+#endif
+{
+#if defined(VBOX_STRICT) && defined(IN_RING3)
+    int rc = PDMCritSectEnterDebug(&pVM->pgm.s.CritSectX, VERR_SEM_BUSY, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
+#else
+    int rc = PDMCritSectEnter(&pVM->pgm.s.CritSectX, VERR_SEM_BUSY);
+#endif
+#ifdef IN_RING0
+    if (rc == VERR_SEM_BUSY)
+        rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_LOCK, 0);
+#endif
+    AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Release the PGM lock.
+ *
+ * @returns VBox status code
+ * @param   pVM         The cross context VM structure.
+ */
+void pgmUnlock(PVM pVM)
+{
+    uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
+    pVM->pgm.s.cDeprecatedPageLocks = 0;
+    int rc = PDMCritSectLeave(&pVM->pgm.s.CritSectX);
+    if (rc == VINF_SEM_NESTED)
+        pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
+}
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+
+/**
+ * Common worker for pgmRZDynMapGCPageOffInlined and pgmRZDynMapGCPageV2Inlined.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPhys      The guest physical address of the page to map.  The
+ *                      offset bits are not ignored.
+ * @param   ppv         Where to return the address corresponding to @a GCPhys.
+ * @param   SRC_POS     The source position of the caller (RT_SRC_POS).
+ */
+int pgmRZDynMapGCPageCommon(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
+{
+    pgmLock(pVM);
+
+    /*
+     * Convert it to a writable page and it on to the dynamic mapper.
+     */
+    int rc;
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+    if (RT_LIKELY(pPage))
+    {
+        rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+        if (RT_SUCCESS(rc))
+        {
+            void *pv;
+            rc = pgmRZDynMapHCPageInlined(pVCpu, PGM_PAGE_GET_HCPHYS(pPage), &pv RTLOG_COMMA_SRC_POS_ARGS);
+            if (RT_SUCCESS(rc))
+                *ppv = (void *)((uintptr_t)pv | ((uintptr_t)GCPhys & PAGE_OFFSET_MASK));
+        }
+        else
+            AssertRC(rc);
+    }
+    else
+    {
+        AssertMsgFailed(("Invalid physical address %RGp!\n", GCPhys));
+        rc = VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+#if !defined(IN_R0) || defined(LOG_ENABLED)
+
+/** Format handler for PGMPAGE.
+ * @copydoc FNRTSTRFORMATTYPE */
+static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
+                                                     const char *pszType, void const *pvValue,
+                                                     int cchWidth, int cchPrecision, unsigned fFlags,
+                                                     void *pvUser)
+{
+    size_t    cch;
+    PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
+    if (RT_VALID_PTR(pPage))
+    {
+        char szTmp[64+80];
+
+        cch = 0;
+
+        /* The single char state stuff. */
+        static const char s_achPageStates[4]    = { 'Z', 'A', 'W', 'S' };
+        szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
+
+# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) || cchPrecision == (lvl) || cchPrecision >= (lvl)+10 )
+        if (IS_PART_INCLUDED(5))
+        {
+            static const char s_achHandlerStates[4] = { '-', 't', 'w', 'a' };
+            szTmp[cch++] = s_achHandlerStates[PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)];
+        }
+
+        /* The type. */
+        if (IS_PART_INCLUDED(4))
+        {
+            szTmp[cch++] = ':';
+            static const char s_achPageTypes[8][4]  = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
+            szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
+            szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
+            szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
+        }
+
+        /* The numbers. */
+        if (IS_PART_INCLUDED(3))
+        {
+            szTmp[cch++] = ':';
+            cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD | RTSTR_F_64BIT);
+        }
+
+        if (IS_PART_INCLUDED(2))
+        {
+            szTmp[cch++] = ':';
+            cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD | RTSTR_F_32BIT);
+        }
+
+        if (IS_PART_INCLUDED(6))
+        {
+            szTmp[cch++] = ':';
+            static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
+            szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
+            cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD | RTSTR_F_16BIT);
+        }
+# undef IS_PART_INCLUDED
+
+        cch = pfnOutput(pvArgOutput, szTmp, cch);
+    }
+    else
+        cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
+    NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
+    return cch;
+}
+
+
+/** Format handler for PGMRAMRANGE.
+ * @copydoc FNRTSTRFORMATTYPE */
+static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
+                                                         const char *pszType, void const *pvValue,
+                                                         int cchWidth, int cchPrecision, unsigned fFlags,
+                                                         void *pvUser)
+{
+    size_t              cch;
+    PGMRAMRANGE const  *pRam = (PGMRAMRANGE const *)pvValue;
+    if (VALID_PTR(pRam))
+    {
+        char szTmp[80];
+        cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
+        cch = pfnOutput(pvArgOutput, szTmp, cch);
+    }
+    else
+        cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
+    NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
+    return cch;
+}
+
+/** Format type andlers to be registered/deregistered. */
+static const struct
+{
+    char                szType[24];
+    PFNRTSTRFORMATTYPE  pfnHandler;
+} g_aPgmFormatTypes[] =
+{
+    { "pgmpage",        pgmFormatTypeHandlerPage },
+    { "pgmramrange",    pgmFormatTypeHandlerRamRange }
+};
+
+#endif /* !IN_R0 || LOG_ENABLED */
+
+/**
+ * Registers the global string format types.
+ *
+ * This should be called at module load time or in some other manner that ensure
+ * that it's called exactly one time.
+ *
+ * @returns IPRT status code on RTStrFormatTypeRegister failure.
+ */
+VMMDECL(int) PGMRegisterStringFormatTypes(void)
+{
+#if !defined(IN_R0) || defined(LOG_ENABLED)
+    int         rc = VINF_SUCCESS;
+    unsigned    i;
+    for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
+    {
+        rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
+# ifdef IN_RING0
+        if (rc == VERR_ALREADY_EXISTS)
+        {
+            /* in case of cleanup failure in ring-0 */
+            RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
+            rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
+        }
+# endif
+    }
+    if (RT_FAILURE(rc))
+        while (i-- > 0)
+            RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
+
+    return rc;
+#else
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/**
+ * Deregisters the global string format types.
+ *
+ * This should be called at module unload time or in some other manner that
+ * ensure that it's called exactly one time.
+ */
+VMMDECL(void) PGMDeregisterStringFormatTypes(void)
+{
+#if !defined(IN_R0) || defined(LOG_ENABLED)
+    for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
+        RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
+#endif
+}
+
+#ifdef VBOX_STRICT
+
+# ifndef PGM_WITHOUT_MAPPINGS
+/**
+ * Asserts that there are no mapping conflicts.
+ *
+ * @returns Number of conflicts.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(unsigned) PGMAssertNoMappingConflicts(PVM pVM)
+{
+    unsigned cErrors = 0;
+
+    /* Only applies to raw mode -> 1 VPCU */
+    Assert(pVM->cCpus == 1);
+    PVMCPU pVCpu = &VMCC_GET_CPU_0(pVM);
+
+    /*
+     * Check for mapping conflicts.
+     */
+    for (PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
+         pMapping;
+         pMapping = pMapping->CTX_SUFF(pNext))
+    {
+        /** @todo This is slow and should be optimized, but since it's just assertions I don't care now. */
+        for (RTGCPTR GCPtr = pMapping->GCPtr; GCPtr <= pMapping->GCPtrLast; GCPtr += PAGE_SIZE)
+        {
+            int rc = PGMGstGetPage(pVCpu, (RTGCPTR)GCPtr, NULL, NULL);
+            if (rc != VERR_PAGE_TABLE_NOT_PRESENT)
+            {
+                AssertMsgFailed(("Conflict at %RGv with %s\n", GCPtr, R3STRING(pMapping->pszDesc)));
+                cErrors++;
+                break;
+            }
+        }
+    }
+
+    return cErrors;
+}
+# endif /* !PGM_WITHOUT_MAPPINGS */
+
+
+/**
+ * Asserts that everything related to the guest CR3 is correctly shadowed.
+ *
+ * This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
+ * and assert the correctness of the guest CR3 mapping before asserting that the
+ * shadow page tables is in sync with the guest page tables.
+ *
+ * @returns Number of conflicts.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   cr3     The current guest CR3 register value.
+ * @param   cr4     The current guest CR4 register value.
+ */
+VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
+{
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
+
+    uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
+
+    pgmLock(pVM);
+    unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
+    pgmUnlock(pVM);
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
+    return cErrors;
+}
+
+#endif /* VBOX_STRICT */
+
diff --git a/src/VBox/VMM/VMMAll/PGMAllBth.h b/src/VBox/VMM/VMMAll/PGMAllBth.h
new file mode 100644
index 00000000..ae464bc6
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllBth.h
@@ -0,0 +1,4611 @@
+/* $Id: PGMAllBth.h $ */
+/** @file
+ * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
+ *
+ * @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
+ *          PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
+ *          bird: WTF does this mean these days?  Looking at PGMAll.cpp it's
+ *
+ * @remarks This file is one big \#ifdef-orgy!
+ *
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifdef _MSC_VER
+/** @todo we're generating unnecessary code in nested/ept shadow mode and for
+ *        real/prot-guest+RC mode. */
+# pragma warning(disable: 4505)
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
+#ifndef IN_RING3
+PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
+#endif
+PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
+static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
+static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
+static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
+#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
+#else
+static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
+#endif
+PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
+PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
+PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
+#ifdef VBOX_STRICT
+PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
+#endif
+PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
+PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu);
+
+#ifdef IN_RING3
+PGM_BTH_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
+#endif
+RT_C_DECLS_END
+
+
+
+
+/*
+ * Filter out some illegal combinations of guest and shadow paging, so we can
+ * remove redundant checks inside functions.
+ */
+#if    PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE \
+    && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
+# error "Invalid combination; PAE guest implies PAE shadow"
+#endif
+
+#if    (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
+    && !(   PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 \
+         || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE)
+# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
+#endif
+
+#if     (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \
+    && !(   PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE \
+         || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE)
+# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
+#endif
+
+#if    (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE) \
+    || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
+# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
+#endif
+
+
+/**
+ * Enters the shadow+guest mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPhysCR3   The physical address from the CR3 register.
+ */
+PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
+{
+    /* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches;
+     * Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables.
+     */
+#if  (   (   PGM_SHW_TYPE == PGM_TYPE_32BIT \
+          || PGM_SHW_TYPE == PGM_TYPE_PAE    \
+          || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
+      && (   PGM_GST_TYPE == PGM_TYPE_REAL   \
+          || PGM_GST_TYPE == PGM_TYPE_PROT))
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    Assert((HMIsNestedPagingActive(pVM) || VM_IS_NEM_ENABLED(pVM)) == pVM->pgm.s.fNestedPaging);
+    Assert(!pVM->pgm.s.fNestedPaging);
+
+    pgmLock(pVM);
+    /* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes),
+     *       but any calls to GC need a proper shadow page setup as well.
+     */
+    /* Free the previous root mapping if still active. */
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+    if (pOldShwPageCR3)
+    {
+        Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
+
+        /* Mark the page as unlocked; allow flushing again. */
+        pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+
+# ifndef PGM_WITHOUT_MAPPINGS
+        /* Remove the hypervisor mappings from the shadow page table. */
+        pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+# endif
+
+        pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
+        pVCpu->pgm.s.pShwPageCR3R3 = NIL_RTR3PTR;
+        pVCpu->pgm.s.pShwPageCR3R0 = NIL_RTR0PTR;
+    }
+
+    /* construct a fake address. */
+    GCPhysCR3 = RT_BIT_64(63);
+    PPGMPOOLPAGE pNewShwPageCR3;
+    int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                          NIL_PGMPOOL_IDX, UINT32_MAX, false /*fLockPage*/,
+                          &pNewShwPageCR3);
+    AssertRCReturn(rc, rc);
+
+    pVCpu->pgm.s.pShwPageCR3R3 = (R3PTRTYPE(PPGMPOOLPAGE))MMHyperCCToR3(pVM, pNewShwPageCR3);
+    pVCpu->pgm.s.pShwPageCR3R0 = (R0PTRTYPE(PPGMPOOLPAGE))MMHyperCCToR0(pVM, pNewShwPageCR3);
+
+    /* Mark the page as locked; disallow flushing. */
+    pgmPoolLockPage(pPool, pNewShwPageCR3);
+
+    /* Set the current hypervisor CR3. */
+    CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
+
+# ifndef PGM_WITHOUT_MAPPINGS
+    /* Apply all hypervisor mappings to the new CR3. */
+    rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
+# endif
+
+    pgmUnlock(pVM);
+    return rc;
+#else
+    NOREF(pVCpu); NOREF(GCPhysCR3);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+#ifndef IN_RING3
+
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+/**
+ * Deal with a guest page fault.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_EM_RAW_GUEST_TRAP
+ * @retval  VINF_EM_RAW_EMULATE_INSTR
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pGstWalk        The guest page table walk result.
+ * @param   uErr            The error code.
+ */
+PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPUCC pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
+{
+#  if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE)
+    /*
+     * Check for write conflicts with our hypervisor mapping.
+     *
+     * If the  guest happens to access a non-present page, where our hypervisor
+     * is currently mapped, then we'll create a #PF storm in the guest.
+     */
+    if (   (uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RW)) == (X86_TRAP_PF_P | X86_TRAP_PF_RW)
+        && pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM))
+        && MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
+    {
+        /* Force a CR3 sync to check for conflicts and emulate the instruction. */
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
+        return VINF_EM_RAW_EMULATE_INSTR;
+    }
+#  endif
+
+    /*
+     * Calc the error code for the guest trap.
+     */
+    uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
+                     ? uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID)
+                     : uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US);
+    if (   pGstWalk->Core.fRsvdError
+        || pGstWalk->Core.fBadPhysAddr)
+    {
+        uNewErr |= X86_TRAP_PF_RSVD | X86_TRAP_PF_P;
+        Assert(!pGstWalk->Core.fNotPresent);
+    }
+    else if (!pGstWalk->Core.fNotPresent)
+        uNewErr |= X86_TRAP_PF_P;
+    TRPMSetErrorCode(pVCpu, uNewErr);
+
+    LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pGstWalk->Core.GCPtr, uErr, pGstWalk->Core.uLevel));
+    STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
+    return VINF_EM_RAW_GUEST_TRAP;
+}
+# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
+
+
+#if !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
+/**
+ * Deal with a guest page fault.
+ *
+ * The caller has taken the PGM lock.
+ *
+ * @returns Strict VBox status code.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   uErr            The error code.
+ * @param   pRegFrame       The register frame.
+ * @param   pvFault         The fault address.
+ * @param   pPage           The guest page at @a pvFault.
+ * @param   pGstWalk        The guest page table walk result.
+ * @param   pfLockTaken     PGM lock taken here or not (out).  This is true
+ *                          when we're called.
+ */
+static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
+                                                                RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING)
+                                                                , PGSTPTWALK pGstWalk
+# endif
+                                                                )
+{
+# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    GSTPDE const    PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
+# endif
+    PVMCC           pVM         = pVCpu->CTX_SUFF(pVM);
+    VBOXSTRICTRC    rcStrict;
+
+    if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
+    {
+        /*
+         * Physical page access handler.
+         */
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        const RTGCPHYS  GCPhysFault = pGstWalk->Core.GCPhys;
+# else
+        const RTGCPHYS  GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
+# endif
+        PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
+        if (pCur)
+        {
+            PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+
+#  ifdef PGM_SYNC_N_PAGES
+            /*
+             * If the region is write protected and we got a page not present fault, then sync
+             * the pages. If the fault was caused by a read, then restart the instruction.
+             * In case of write access continue to the GC write handler.
+             *
+             * ASSUMES that there is only one handler per page or that they have similar write properties.
+             */
+            if (   !(uErr & X86_TRAP_PF_P)
+                &&  pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
+            {
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+                rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#   else
+                rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#   endif
+                if (    RT_FAILURE(rcStrict)
+                    || !(uErr & X86_TRAP_PF_RW)
+                    || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
+                {
+                    AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
+                    STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
+                    return rcStrict;
+                }
+            }
+#  endif
+#  ifdef PGM_WITH_MMIO_OPTIMIZATIONS
+            /*
+             * If the access was not thru a #PF(RSVD|...) resync the page.
+             */
+            if (   !(uErr & X86_TRAP_PF_RSVD)
+                && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+                && pGstWalk->Core.fEffectiveRW
+                && !pGstWalk->Core.fEffectiveUS /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
+#   endif
+               )
+            {
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+                rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#   else
+                rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#   endif
+                if (    RT_FAILURE(rcStrict)
+                    || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
+                {
+                    AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
+                    STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
+                    return rcStrict;
+                }
+            }
+#  endif
+
+            AssertMsg(   pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
+                      || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
+                      ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
+                       pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
+            if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysWrite);
+            else
+            {
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAll);
+                if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAllOpt);
+            }
+
+            if (pCurType->CTX_SUFF(pfnPfHandler))
+            {
+                PPGMPOOL    pPool  = pVM->pgm.s.CTX_SUFF(pPool);
+                void       *pvUser = pCur->CTX_SUFF(pvUser);
+
+                STAM_PROFILE_START(&pCur->Stat, h);
+                if (pCur->hType != pPool->hAccessHandlerType)
+                {
+                    pgmUnlock(pVM);
+                    *pfLockTaken = false;
+                }
+
+                rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
+
+#  ifdef VBOX_WITH_STATISTICS
+                pgmLock(pVM);
+                pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
+                if (pCur)
+                    STAM_PROFILE_STOP(&pCur->Stat, h);
+                pgmUnlock(pVM);
+#  endif
+            }
+            else
+                rcStrict = VINF_EM_RAW_EMULATE_INSTR;
+
+            STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
+            return rcStrict;
+        }
+    }
+
+    /*
+     * There is a handled area of the page, but this fault doesn't belong to it.
+     * We must emulate the instruction.
+     *
+     * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
+     * we first check if this was a page-not-present fault for a page with only
+     * write access handlers. Restart the instruction if it wasn't a write access.
+     */
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
+
+    if (    !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
+        &&  !(uErr & X86_TRAP_PF_P))
+    {
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#  else
+        rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#  endif
+        if (    RT_FAILURE(rcStrict)
+            ||  rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
+            ||  !(uErr & X86_TRAP_PF_RW))
+        {
+            AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
+            STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
+            return rcStrict;
+        }
+    }
+
+    /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
+     *        It's writing to an unhandled part of the LDT page several million times.
+     */
+    rcStrict = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
+    LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
+    STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
+    return rcStrict;
+} /* if any kind of handler */
+# endif /* !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE*/
+
+
+/**
+ * \#PF Handler for raw-mode guest execution.
+ *
+ * @returns VBox status code (appropriate for trap handling and GC return).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uErr        The trap error code.
+ * @param   pRegFrame   Trap register frame.
+ * @param   pvFault     The fault address.
+ * @param   pfLockTaken PGM lock taken here or not (out)
+ */
+PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
+
+    *pfLockTaken = false;
+
+# if  (   PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT \
+       || PGM_GST_TYPE == PGM_TYPE_PAE   || PGM_GST_TYPE == PGM_TYPE_AMD64) \
+    && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
+    && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
+    && PGM_SHW_TYPE != PGM_TYPE_NONE
+    int rc;
+
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    /*
+     * Walk the guest page translation tables and check if it's a guest fault.
+     */
+    GSTPTWALK GstWalk;
+    rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
+    if (RT_FAILURE_NP(rc))
+        return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
+
+    /* assert some GstWalk sanity. */
+#   if PGM_GST_TYPE == PGM_TYPE_AMD64
+    /*AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u));  - not always true with SMP guests. */
+#   endif
+#   if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
+    /*AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto */
+#   endif
+    /*AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto */
+    /*AssertMsg(GstWalk.Core.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto */
+    Assert(GstWalk.Core.fSucceeded);
+
+    if (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID))
+    {
+        if (    (   (uErr & X86_TRAP_PF_RW)
+                 && !GstWalk.Core.fEffectiveRW
+                 && (   (uErr & X86_TRAP_PF_US)
+                     || CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
+            ||  ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
+            ||  ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
+           )
+            return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
+    }
+
+    /*
+     * Set the accessed and dirty flags.
+     */
+#   if PGM_GST_TYPE == PGM_TYPE_AMD64
+    GstWalk.Pml4e.u     |= X86_PML4E_A;
+    GstWalk.pPml4e->u   |= X86_PML4E_A;
+    GstWalk.Pdpe.u      |= X86_PDPE_A;
+    GstWalk.pPdpe->u    |= X86_PDPE_A;
+#   endif
+    if (GstWalk.Core.fBigPage)
+    {
+        Assert(GstWalk.Pde.b.u1Size);
+        if (uErr & X86_TRAP_PF_RW)
+        {
+            GstWalk.Pde.u   |= X86_PDE4M_A | X86_PDE4M_D;
+            GstWalk.pPde->u |= X86_PDE4M_A | X86_PDE4M_D;
+        }
+        else
+        {
+            GstWalk.Pde.u   |= X86_PDE4M_A;
+            GstWalk.pPde->u |= X86_PDE4M_A;
+        }
+    }
+    else
+    {
+        Assert(!GstWalk.Pde.b.u1Size);
+        GstWalk.Pde.u   |= X86_PDE_A;
+        GstWalk.pPde->u |= X86_PDE_A;
+        if (uErr & X86_TRAP_PF_RW)
+        {
+#   ifdef VBOX_WITH_STATISTICS
+            if (!GstWalk.Pte.n.u1Dirty)
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
+            else
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
+#   endif
+            GstWalk.Pte.u   |= X86_PTE_A | X86_PTE_D;
+            GstWalk.pPte->u |= X86_PTE_A | X86_PTE_D;
+        }
+        else
+        {
+            GstWalk.Pte.u   |= X86_PTE_A;
+            GstWalk.pPte->u |= X86_PTE_A;
+        }
+        Assert(GstWalk.Pte.u == GstWalk.pPte->u);
+    }
+    AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u,
+              ("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
+#  else  /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
+    GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A}; /** @todo eliminate this */
+#  endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
+
+    /* Take the big lock now. */
+    *pfLockTaken = true;
+    pgmLock(pVM);
+
+#  ifdef PGM_WITH_MMIO_OPTIMIZATIONS
+    /*
+     * If it is a reserved bit fault we know that it is an MMIO (access
+     * handler) related fault and can skip some 200 lines of code.
+     */
+    if (uErr & X86_TRAP_PF_RSVD)
+    {
+        Assert(uErr & X86_TRAP_PF_P);
+        PPGMPAGE pPage;
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        rc = pgmPhysGetPageEx(pVM, GstWalk.Core.GCPhys, &pPage);
+        if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+            return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
+                                                                                 pfLockTaken, &GstWalk));
+        rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
+#   else
+        rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
+        if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+            return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
+                                                                                 pfLockTaken));
+        rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
+#   endif
+        AssertRC(rc);
+        PGM_INVL_PG(pVCpu, pvFault);
+        return rc; /* Restart with the corrected entry. */
+    }
+#  endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
+
+    /*
+     * Fetch the guest PDE, PDPE and PML4E.
+     */
+#  if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDDst = pvFault >> SHW_PD_SHIFT;
+    PX86PD          pPDDst = pgmShwGet32BitPDPtr(pVCpu);
+
+#  elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    const unsigned  iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK;   /* pPDDst index, not used with the pool. */
+    PX86PDPAE       pPDDst;
+#   if PGM_GST_TYPE == PGM_TYPE_PAE
+    rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
+#   else
+    rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst);       /* RW, US and A are reserved in PAE mode. */
+#   endif
+    AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+
+#  elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    const unsigned  iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PX86PDPAE       pPDDst;
+#   if PGM_GST_TYPE == PGM_TYPE_PROT  /* (AMD-V nested paging) */
+    rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A,
+                                 X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A, &pPDDst);
+#   else
+    rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
+#   endif
+    AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+
+#  elif PGM_SHW_TYPE == PGM_TYPE_EPT
+    const unsigned  iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PEPTPD          pPDDst;
+    rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
+    AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+#  endif
+    Assert(pPDDst);
+
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    /*
+     * Dirty page handling.
+     *
+     * If we successfully correct the write protection fault due to dirty bit
+     * tracking, then return immediately.
+     */
+    if (uErr & X86_TRAP_PF_RW)  /* write fault? */
+    {
+        STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
+        rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
+        STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
+        if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
+        {
+            STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
+                        = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
+                          ? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
+                          : &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
+            Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
+            return VINF_SUCCESS;
+        }
+#ifdef DEBUG_bird
+        AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); // - triggers with smp w7 guests.
+        AssertMsg(GstWalk.Core.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); // - ditto.
+#endif
+    }
+
+#   if 0 /* rarely useful; leave for debugging. */
+    STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
+#   endif
+#  endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
+
+    /*
+     * A common case is the not-present error caused by lazy page table syncing.
+     *
+     * It is IMPORTANT that we weed out any access to non-present shadow PDEs
+     * here so we can safely assume that the shadow PT is present when calling
+     * SyncPage later.
+     *
+     * On failure, we ASSUME that SyncPT is out of memory or detected some kind
+     * of mapping conflict and defer to SyncCR3 in R3.
+     * (Again, we do NOT support access handlers for non-present guest pages.)
+     *
+     */
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    Assert(GstWalk.Pde.n.u1Present);
+#  endif
+    if (    !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
+        &&  !pPDDst->a[iPDDst].n.u1Present)
+    {
+        STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
+        rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
+#  else
+        LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
+        rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
+#  endif
+        if (RT_SUCCESS(rc))
+            return rc;
+        Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
+        return VINF_PGM_SYNC_CR3;
+    }
+
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
+    /*
+     * Check if this address is within any of our mappings.
+     *
+     * This is *very* fast and it's gonna save us a bit of effort below and prevent
+     * us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
+     * (BTW, it's impossible to have physical access handlers in a mapping.)
+     */
+    if (pgmMapAreMappingsEnabled(pVM))
+    {
+        PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
+        for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
+        {
+            if (pvFault < pMapping->GCPtr)
+                break;
+            if (pvFault - pMapping->GCPtr < pMapping->cb)
+            {
+                /*
+                 * The first thing we check is if we've got an undetected conflict.
+                 */
+                if (pgmMapAreMappingsFloating(pVM))
+                {
+                    unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
+                    while (iPT-- > 0)
+                        if (GstWalk.pPde[iPT].n.u1Present)
+                        {
+                            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
+                            Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
+                            VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
+                            STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
+                            return VINF_PGM_SYNC_CR3;
+                        }
+                }
+
+                /*
+                 * Pretend we're not here and let the guest handle the trap.
+                 */
+                TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
+                LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
+                STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
+                return VINF_EM_RAW_GUEST_TRAP;
+            }
+        }
+    } /* pgmAreMappingsEnabled(&pVM->pgm.s) */
+#  endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
+
+    /*
+     * Check if this fault address is flagged for special treatment,
+     * which means we'll have to figure out the physical address and
+     * check flags associated with it.
+     *
+     * ASSUME that we can limit any special access handling to pages
+     * in page tables which the guest believes to be present.
+     */
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+#  else
+    RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK);
+#  endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
+    PPGMPAGE pPage;
+    rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+    if (RT_FAILURE(rc))
+    {
+        /*
+         * When the guest accesses invalid physical memory (e.g. probing
+         * of RAM or accessing a remapped MMIO range), then we'll fall
+         * back to the recompiler to emulate the instruction.
+         */
+        LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
+        STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
+        return VINF_EM_RAW_EMULATE_INSTR;
+    }
+
+    /*
+     * Any handlers for this page?
+     */
+    if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
+                                                                             &GstWalk));
+# else
+        return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
+# endif
+
+    /*
+     * We are here only if page is present in Guest page tables and
+     * trap is not handled by our handlers.
+     *
+     * Check it for page out-of-sync situation.
+     */
+    if (!(uErr & X86_TRAP_PF_P))
+    {
+        /*
+         * Page is not present in our page tables. Try to sync it!
+         */
+        if (uErr & X86_TRAP_PF_US)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
+        else /* supervisor */
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
+
+        if (PGM_PAGE_IS_BALLOONED(pPage))
+        {
+            /* Emulate reads from ballooned pages as they are not present in
+               our shadow page tables. (Required for e.g. Solaris guests; soft
+               ecc, random nr generator.) */
+            rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
+            LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
+            STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
+            return rc;
+        }
+
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#   else
+        rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#   endif
+        if (RT_SUCCESS(rc))
+        {
+            /* The page was successfully synced, return to the guest. */
+            STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
+            return VINF_SUCCESS;
+        }
+    }
+    else /* uErr & X86_TRAP_PF_P: */
+    {
+        /*
+         * Write protected pages are made writable when the guest makes the
+         * first write to it.  This happens for pages that are shared, write
+         * monitored or not yet allocated.
+         *
+         * We may also end up here when CR0.WP=0 in the guest.
+         *
+         * Also, a side effect of not flushing global PDEs are out of sync
+         * pages due to physical monitored regions, that are no longer valid.
+         * Assume for now it only applies to the read/write flag.
+         */
+        if (uErr & X86_TRAP_PF_RW)
+        {
+            /*
+             * Check if it is a read-only page.
+             */
+            if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
+            {
+                Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
+                Assert(!PGM_PAGE_IS_ZERO(pPage));
+                AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
+                STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
+
+                rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+                if (rc != VINF_SUCCESS)
+                {
+                    AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
+                    return rc;
+                }
+                if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                    return VINF_EM_NO_MEMORY;
+            }
+
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+            /*
+             * Check to see if we need to emulate the instruction if CR0.WP=0.
+             */
+            if (    !GstWalk.Core.fEffectiveRW
+                &&  (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
+                &&  CPUMGetGuestCPL(pVCpu) < 3)
+            {
+                Assert((uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_P)) == (X86_TRAP_PF_RW | X86_TRAP_PF_P));
+
+                /*
+                 * The Netware WP0+RO+US hack.
+                 *
+                 * Netware sometimes(/always?) runs with WP0.  It has been observed doing
+                 * excessive write accesses to pages which are mapped with US=1 and RW=0
+                 * while WP=0.  This causes a lot of exits and extremely slow execution.
+                 * To avoid trapping and emulating every write here, we change the shadow
+                 * page table entry to map it as US=0 and RW=1 until user mode tries to
+                 * access it again (see further below).  We count these shadow page table
+                 * changes so we can avoid having to clear the page pool every time the WP
+                 * bit changes to 1 (see PGMCr0WpEnabled()).
+                 */
+#    if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) && 1
+                if (   GstWalk.Core.fEffectiveUS
+                    && !GstWalk.Core.fEffectiveRW
+                    && (GstWalk.Core.fBigPage || GstWalk.Pde.n.u1Write)
+                    && pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
+                {
+                    Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, GstWalk.Core.fBigPage));
+                    rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, GstWalk.Core.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
+                    if (rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3)
+                    {
+                        PGM_INVL_PG(pVCpu, pvFault);
+                        pVCpu->pgm.s.cNetwareWp0Hacks++;
+                        STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Wp0RoUsHack; });
+                        return rc;
+                    }
+                    AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
+                    Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
+                }
+#    endif
+
+                /* Interpret the access. */
+                rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
+                Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), GstWalk.Core.fBigPage, GstWalk.Core.fEffectiveUS));
+                if (RT_SUCCESS(rc))
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
+                else
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
+                STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
+                return rc;
+            }
+#   endif
+            /// @todo count the above case; else
+            if (uErr & X86_TRAP_PF_US)
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
+            else /* supervisor */
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
+
+            /*
+             * Sync the page.
+             *
+             * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
+             *       page is not present, which is not true in this case.
+             */
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+            rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
+#   else
+            rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
+#   endif
+            if (RT_SUCCESS(rc))
+            {
+               /*
+                * Page was successfully synced, return to guest but invalidate
+                * the TLB first as the page is very likely to be in it.
+                */
+#   if PGM_SHW_TYPE == PGM_TYPE_EPT
+                HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
+#   else
+                PGM_INVL_PG(pVCpu, pvFault);
+#   endif
+#   ifdef VBOX_STRICT
+                RTGCPHYS GCPhys2 = RTGCPHYS_MAX;
+                uint64_t fPageGst = UINT64_MAX;
+                if (!pVM->pgm.s.fNestedPaging)
+                {
+                    rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
+                    AssertMsg(RT_SUCCESS(rc) && ((fPageGst & X86_PTE_RW) || ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
+                    LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
+                }
+#    if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
+                uint64_t fPageShw = 0;
+                rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
+                AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) || pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
+                          ("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GCPhys2, fPageGst, pvFault));
+#    endif
+#   endif /* VBOX_STRICT */
+                STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
+                return VINF_SUCCESS;
+            }
+        }
+#    if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        /*
+         * Check for Netware WP0+RO+US hack from above and undo it when user
+         * mode accesses the page again.
+         */
+        else if (    GstWalk.Core.fEffectiveUS
+                 && !GstWalk.Core.fEffectiveRW
+                 && (GstWalk.Core.fBigPage || GstWalk.Pde.n.u1Write)
+                 &&  pVCpu->pgm.s.cNetwareWp0Hacks > 0
+                 &&  (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
+                 &&  CPUMGetGuestCPL(pVCpu) == 3
+                 &&  pVM->cCpus == 1
+                )
+        {
+            Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
+            rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
+            if (RT_SUCCESS(rc))
+            {
+                PGM_INVL_PG(pVCpu, pvFault);
+                pVCpu->pgm.s.cNetwareWp0Hacks--;
+                STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Wp0RoUsUnhack; });
+                return VINF_SUCCESS;
+            }
+        }
+#    endif /* PGM_WITH_PAGING */
+
+        /** @todo else: why are we here? */
+
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
+        /*
+         * Check for VMM page flags vs. Guest page flags consistency.
+         * Currently only for debug purposes.
+         */
+        if (RT_SUCCESS(rc))
+        {
+            /* Get guest page flags. */
+            uint64_t fPageGst;
+            int rc2 = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
+            if (RT_SUCCESS(rc2))
+            {
+                uint64_t fPageShw = 0;
+                rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
+
+#if 0
+                /*
+                 * Compare page flags.
+                 * Note: we have AVL, A, D bits desynced.
+                 */
+                AssertMsg(      (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
+                             == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
+                          || (   pVCpu->pgm.s.cNetwareWp0Hacks > 0
+                              &&    (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US))
+                                 == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US))
+                              && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW
+                              && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US),
+                          ("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
+                           pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
+01:01:15.623511 00:08:43.266063 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US)
+01:01:15.623511 00:08:43.266064 Location  : e:\vbox\svn\trunk\srcPage flags mismatch! pvFault=fffff801b0d7b000 uErr=11 GCPhys=0000000019b52000 fPageShw=0 fPageGst=77b0000000000121 rc=0
+
+01:01:15.625516 00:08:43.268051 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US)
+01:01:15.625516 00:08:43.268051 Location  :
+e:\vbox\svn\trunk\srcPage flags mismatch!
+pvFault=fffff801b0d7b000
+                uErr=11     X86_TRAP_PF_ID | X86_TRAP_PF_P
+GCPhys=0000000019b52000
+fPageShw=0
+fPageGst=77b0000000000121
+rc=0
+#endif
+
+            }
+            else
+                AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
+        }
+        else
+            AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
+#   endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
+    }
+
+
+    /*
+     * If we get here it is because something failed above, i.e. most like guru
+     * meditiation time.
+     */
+    LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
+            __PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs.Sel, pRegFrame->rip));
+    return rc;
+
+# else  /* Nested paging, EPT except PGM_GST_TYPE = PROT, NONE.   */
+    NOREF(uErr); NOREF(pRegFrame); NOREF(pvFault);
+    AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
+    return VERR_PGM_NOT_USED_IN_MODE;
+# endif
+}
+
+#endif /* !IN_RING3 */
+
+
+/**
+ * Emulation of the invlpg instruction.
+ *
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtrPage   Page to invalidate.
+ *
+ * @remark  ASSUMES that the guest is updating before invalidating. This order
+ *          isn't required by the CPU, so this is speculative and could cause
+ *          trouble.
+ * @remark  No TLB shootdown is done on any other VCPU as we assume that
+ *          invlpg emulation is the *only* reason for calling this function.
+ *          (The guest has to shoot down TLB entries on other CPUs itself)
+ *          Currently true, but keep in mind!
+ *
+ * @todo    Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
+ *          Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
+ */
+PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
+{
+#if    PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
+    && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
+    && PGM_SHW_TYPE != PGM_TYPE_NONE
+    int rc;
+    PVMCC    pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
+
+    /*
+     * Get the shadow PD entry and skip out if this PD isn't present.
+     * (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
+     */
+# if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDDst    = (uint32_t)GCPtrPage >> SHW_PD_SHIFT;
+    PX86PDE         pPdeDst   = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+#  ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
+    if (!pShwPde)
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
+        return VINF_SUCCESS;
+    }
+#  else
+    Assert(pShwPde);
+#  endif
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    const unsigned  iPdpt     = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT;
+    PX86PDPT        pPdptDst  = pgmShwGetPaePDPTPtr(pVCpu);
+
+    /* If the shadow PDPE isn't present, then skip the invalidate. */
+#  ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
+    if (!pPdptDst || !pPdptDst->a[iPdpt].n.u1Present)
+#  else
+    if (!pPdptDst->a[iPdpt].n.u1Present)
+#  endif
+    {
+        Assert(!pPdptDst || !(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
+        PGM_INVL_PG(pVCpu, GCPtrPage);
+        return VINF_SUCCESS;
+    }
+
+    const unsigned  iPDDst  = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PPGMPOOLPAGE    pShwPde = NULL;
+    PX86PDPAE       pPDDst;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
+    AssertRCSuccessReturn(rc, rc);
+    Assert(pShwPde);
+
+    pPDDst             = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
+    PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
+
+# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
+    /* PML4 */
+    /*const unsigned  iPml4     = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;*/
+    const unsigned  iPdpt     = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    const unsigned  iPDDst    = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PX86PDPAE       pPDDst;
+    PX86PDPT        pPdptDst;
+    PX86PML4E       pPml4eDst;
+    rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
+    if (rc != VINF_SUCCESS)
+    {
+        AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
+        PGM_INVL_PG(pVCpu, GCPtrPage);
+        return VINF_SUCCESS;
+    }
+    Assert(pPDDst);
+
+    PX86PDEPAE  pPdeDst  = &pPDDst->a[iPDDst];
+    PX86PDPE    pPdpeDst = &pPdptDst->a[iPdpt];
+
+    if (!pPdpeDst->n.u1Present)
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
+        PGM_INVL_PG(pVCpu, GCPtrPage);
+        return VINF_SUCCESS;
+    }
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
+    Assert(pShwPde);
+
+# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
+
+    const SHWPDE PdeDst = *pPdeDst;
+    if (!PdeDst.n.u1Present)
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
+        PGM_INVL_PG(pVCpu, GCPtrPage);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Get the guest PD entry and calc big page.
+     */
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+    PGSTPD          pPDSrc      = pgmGstGet32bitPDPtr(pVCpu);
+    const unsigned  iPDSrc      = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
+    GSTPDE          PdeSrc      = pPDSrc->a[iPDSrc];
+# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
+    unsigned        iPDSrc = 0;
+#  if PGM_GST_TYPE == PGM_TYPE_PAE
+    X86PDPE         PdpeSrcIgn;
+    PX86PDPAE       pPDSrc      = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
+#  else /* AMD64 */
+    PX86PML4E       pPml4eSrcIgn;
+    X86PDPE         PdpeSrcIgn;
+    PX86PDPAE       pPDSrc      = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
+#  endif
+    GSTPDE          PdeSrc;
+
+    if (pPDSrc)
+        PdeSrc = pPDSrc->a[iPDSrc];
+    else
+        PdeSrc.u = 0;
+# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
+    const bool      fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
+    const bool      fIsBigPage  = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
+    if (fWasBigPage != fIsBigPage)
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
+
+# ifdef IN_RING3
+    /*
+     * If a CR3 Sync is pending we may ignore the invalidate page operation
+     * depending on the kind of sync and if it's a global page or not.
+     * This doesn't make sense in GC/R0 so we'll skip it entirely there.
+     */
+#  ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
+    if (    VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
+        || (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
+            && fIsBigPage
+            && PdeSrc.b.u1Global
+           )
+       )
+#  else
+    if (VM_FF_IS_ANY_SET(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
+#  endif
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
+        return VINF_SUCCESS;
+    }
+# endif /* IN_RING3 */
+
+    /*
+     * Deal with the Guest PDE.
+     */
+    rc = VINF_SUCCESS;
+    if (PdeSrc.n.u1Present)
+    {
+        Assert(     PdeSrc.n.u1User == PdeDst.n.u1User
+               &&   (PdeSrc.n.u1Write || !PdeDst.n.u1Write || pVCpu->pgm.s.cNetwareWp0Hacks > 0));
+# ifndef PGM_WITHOUT_MAPPING
+        if (PdeDst.u & PGM_PDFLAGS_MAPPING)
+        {
+            /*
+             * Conflict - Let SyncPT deal with it to avoid duplicate code.
+             */
+            Assert(pgmMapAreMappingsEnabled(pVM));
+            Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
+            rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
+        }
+        else
+# endif /* !PGM_WITHOUT_MAPPING */
+        if (!fIsBigPage)
+        {
+            /*
+             * 4KB - page.
+             */
+            PPGMPOOLPAGE    pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
+            RTGCPHYS        GCPhys   = GST_GET_PDE_GCPHYS(PdeSrc);
+
+# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+            /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
+            GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (PAGE_SIZE / 2)));
+# endif
+            if (pShwPage->GCPhys == GCPhys)
+            {
+                /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
+                PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+
+                PGSTPT pPTSrc;
+                rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
+                if (RT_SUCCESS(rc))
+                {
+                    const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
+                    GSTPTE PteSrc = pPTSrc->a[iPTSrc];
+                    const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+                    PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
+                    Log2(("SyncPage: 4K  %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
+                            GCPtrPage, PteSrc.n.u1Present,
+                            PteSrc.n.u1Write & PdeSrc.n.u1Write,
+                            PteSrc.n.u1User  & PdeSrc.n.u1User,
+                            (uint64_t)PteSrc.u,
+                            SHW_PTE_LOG64(pPTDst->a[iPTDst]),
+                            SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
+                }
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
+                PGM_INVL_PG(pVCpu, GCPtrPage);
+            }
+            else
+            {
+                /*
+                 * The page table address changed.
+                 */
+                LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
+                         GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
+                pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
+                ASMAtomicWriteSize(pPdeDst, 0);
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
+                PGM_INVL_VCPU_TLBS(pVCpu);
+            }
+        }
+        else
+        {
+            /*
+             * 2/4MB - page.
+             */
+            /* Before freeing the page, check if anything really changed. */
+            PPGMPOOLPAGE    pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
+            RTGCPHYS        GCPhys   = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
+# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+            /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
+            GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
+# endif
+            if (    pShwPage->GCPhys == GCPhys
+                &&  pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
+            {
+                /* ASSUMES a the given bits are identical for 4M and normal PDEs */
+                /** @todo This test is wrong as it cannot check the G bit!
+                 *        FIXME */
+                if (        (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
+                        ==  (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
+                    &&  (   PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
+                         || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
+                {
+                    LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
+                    return VINF_SUCCESS;
+                }
+            }
+
+            /*
+             * Ok, the page table is present and it's been changed in the guest.
+             * If we're in host context, we'll just mark it as not present taking the lazy approach.
+             * We could do this for some flushes in GC too, but we need an algorithm for
+             * deciding which 4MB pages containing code likely to be executed very soon.
+             */
+            LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
+                     GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+            pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
+            ASMAtomicWriteSize(pPdeDst, 0);
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
+            PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
+        }
+    }
+    else
+    {
+        /*
+         * Page directory is not present, mark shadow PDE not present.
+         */
+        if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
+        {
+            pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
+            ASMAtomicWriteSize(pPdeDst, 0);
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
+            PGM_INVL_PG(pVCpu, GCPtrPage);
+        }
+        else
+        {
+            Assert(pgmMapAreMappingsEnabled(pVM));
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
+        }
+    }
+    return rc;
+
+#else /* guest real and protected mode, nested + ept, none. */
+    /* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
+    NOREF(pVCpu); NOREF(GCPtrPage);
+    return VINF_SUCCESS;
+#endif
+}
+
+#if PGM_SHW_TYPE != PGM_TYPE_NONE
+
+/**
+ * Update the tracking of shadowed pages.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pShwPage    The shadow page.
+ * @param   HCPhys      The physical page we is being dereferenced.
+ * @param   iPte        Shadow PTE index
+ * @param   GCPhysPage  Guest physical address (only valid if pShwPage->fDirty is set)
+ */
+DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
+                                                        RTGCPHYS GCPhysPage)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+# if    defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
+     && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
+     && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
+
+    /* Use the hint we retrieved from the cached guest PT. */
+    if (pShwPage->fDirty)
+    {
+        PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+        Assert(pShwPage->cPresent);
+        Assert(pPool->cPresent);
+        pShwPage->cPresent--;
+        pPool->cPresent--;
+
+        PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
+        AssertRelease(pPhysPage);
+        pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
+        return;
+    }
+# else
+  NOREF(GCPhysPage);
+# endif
+
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
+    LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
+
+    /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done:
+     *      1. have a medium sized HCPhys -> GCPhys TLB (hash?)
+     *      2. write protect all shadowed pages. I.e. implement caching.
+     */
+    /** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
+
+    /*
+     * Find the guest address.
+     */
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        unsigned iPage = pRam->cb >> PAGE_SHIFT;
+        while (iPage-- > 0)
+        {
+            if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
+            {
+                PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+                Assert(pShwPage->cPresent);
+                Assert(pPool->cPresent);
+                pShwPage->cPresent--;
+                pPool->cPresent--;
+
+                pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
+                STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
+                return;
+            }
+        }
+    }
+
+    for (;;)
+        AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
+}
+
+
+/**
+ * Update the tracking of shadowed pages.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pShwPage    The shadow page.
+ * @param   u16         The top 16-bit of the pPage->HCPhys.
+ * @param   pPage       Pointer to the guest page. this will be modified.
+ * @param   iPTDst      The index into the shadow table.
+ */
+DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Just deal with the simple first time here.
+     */
+    if (!u16)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
+        u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
+        /* Save the page table index. */
+        PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
+    }
+    else
+        u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
+
+    /* write back */
+    Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x  iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
+    PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
+
+    /* update statistics. */
+    pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
+    pShwPage->cPresent++;
+    if (pShwPage->iFirstPresent > iPTDst)
+        pShwPage->iFirstPresent = iPTDst;
+}
+
+
+/**
+ * Modifies a shadow PTE to account for access handlers.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The page in question.
+ * @param   fPteSrc     The shadowed flags of the source PTE.  Must include the
+ *                      A (accessed) bit so it can be emulated correctly.
+ * @param   pPteDst     The shadow PTE (output).  This is temporary storage and
+ *                      does not need to be set atomically.
+ */
+DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVMCC pVM, PCPGMPAGE pPage, uint64_t fPteSrc, PSHWPTE pPteDst)
+{
+    NOREF(pVM); RT_NOREF_PV(fPteSrc);
+
+    /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
+     *  Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
+    if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+    {
+        LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
+# if PGM_SHW_TYPE == PGM_TYPE_EPT
+        pPteDst->u             = PGM_PAGE_GET_HCPHYS(pPage);
+        pPteDst->n.u1Present   = 1;
+        pPteDst->n.u1Execute   = 1;
+        pPteDst->n.u1IgnorePAT = 1;
+        pPteDst->n.u3EMT       = VMX_EPT_MEMTYPE_WB;
+        /* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
+# else
+        if (fPteSrc & X86_PTE_A)
+        {
+            SHW_PTE_SET(*pPteDst, fPteSrc | PGM_PAGE_GET_HCPHYS(pPage));
+            SHW_PTE_SET_RO(*pPteDst);
+        }
+        else
+            SHW_PTE_SET(*pPteDst, 0);
+# endif
+    }
+# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
+#  if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
+    else if (   PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
+             && (   BTH_IS_NP_ACTIVE(pVM)
+                 || (fPteSrc & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
+#   if PGM_SHW_TYPE == PGM_TYPE_AMD64
+             && pVM->pgm.s.fLessThan52PhysicalAddressBits
+#   endif
+            )
+    {
+        LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
+#   if PGM_SHW_TYPE == PGM_TYPE_EPT
+        /* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
+        pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
+        /* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
+        pPteDst->n.u1Present = 0;
+        pPteDst->n.u1Write   = 1;
+        pPteDst->n.u1Execute = 0;
+        /* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
+        pPteDst->n.u3EMT     = 7;
+#   else
+        /* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64).  */
+        SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX | X86_PTE_P);
+#   endif
+    }
+#  endif
+# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
+    else
+    {
+        LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
+        SHW_PTE_SET(*pPteDst, 0);
+    }
+    /** @todo count these kinds of entries. */
+}
+
+
+/**
+ * Creates a 4K shadow page for a guest page.
+ *
+ * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
+ * physical address.  The PdeSrc argument only the flags are used.  No page
+ * structured will be mapped in this function.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pPteDst     Destination page table entry.
+ * @param   PdeSrc      Source page directory entry (i.e. Guest OS page directory entry).
+ *                      Can safely assume that only the flags are being used.
+ * @param   PteSrc      Source page table entry (i.e. Guest OS page table entry).
+ * @param   pShwPage    Pointer to the shadow page.
+ * @param   iPTDst      The index into the shadow table.
+ *
+ * @remark  Not used for 2/4MB pages!
+ */
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING)
+static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
+                                         PPGMPOOLPAGE pShwPage, unsigned iPTDst)
+# else
+static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
+                                         PPGMPOOLPAGE pShwPage, unsigned iPTDst)
+# endif
+{
+    PVMCC    pVM = pVCpu->CTX_SUFF(pVM);
+    RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
+
+# if    defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
+     && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
+     && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
+
+    if (pShwPage->fDirty)
+    {
+        PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+        PGSTPT pGstPT;
+
+        /* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
+        pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
+        GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
+        pGstPT->a[iPTDst].u = PteSrc.u;
+    }
+# else
+    Assert(!pShwPage->fDirty);
+# endif
+
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    if (   PteSrc.n.u1Present
+        && GST_IS_PTE_VALID(pVCpu, PteSrc))
+# endif
+    {
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
+#  endif
+        PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
+
+        /*
+         * Find the ram range.
+         */
+        PPGMPAGE pPage;
+        int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
+        if (RT_SUCCESS(rc))
+        {
+            /* Ignore ballooned pages.
+               Don't return errors or use a fatal assert here as part of a
+               shadow sync range might included ballooned pages. */
+            if (PGM_PAGE_IS_BALLOONED(pPage))
+            {
+                Assert(!SHW_PTE_IS_P(*pPteDst)); /** @todo user tracking needs updating if this triggers. */
+                return;
+            }
+
+# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+            /* Make the page writable if necessary. */
+            if (    PGM_PAGE_GET_TYPE(pPage)  == PGMPAGETYPE_RAM
+                &&  (   PGM_PAGE_IS_ZERO(pPage)
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+                     || (   PteSrc.n.u1Write
+#  else
+                     || (   1
+#  endif
+                         && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+#  ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
+                         && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
+#  endif
+#  ifdef VBOX_WITH_PAGE_SHARING
+                         && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
+#  endif
+                        )
+                     )
+               )
+            {
+                rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
+                AssertRC(rc);
+            }
+# endif
+
+            /*
+             * Make page table entry.
+             */
+            SHWPTE PteDst;
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+            uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
+# else
+            uint64_t fGstShwPteFlags = X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D;
+# endif
+            if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+                PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, fGstShwPteFlags, &PteDst);
+            else
+            {
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+                /*
+                 * If the page or page directory entry is not marked accessed,
+                 * we mark the page not present.
+                 */
+                if (!PteSrc.n.u1Accessed || !PdeSrc.n.u1Accessed)
+                {
+                    LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
+                    SHW_PTE_SET(PteDst, 0);
+                }
+                /*
+                 * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
+                 * when the page is modified.
+                 */
+                else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
+                {
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
+                    SHW_PTE_SET(PteDst,
+                                  fGstShwPteFlags
+                                | PGM_PAGE_GET_HCPHYS(pPage)
+                                | PGM_PTFLAGS_TRACK_DIRTY);
+                    SHW_PTE_SET_RO(PteDst);
+                }
+                else
+# endif
+                {
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
+# if PGM_SHW_TYPE == PGM_TYPE_EPT
+                    PteDst.u             = PGM_PAGE_GET_HCPHYS(pPage);
+                    PteDst.n.u1Present   = 1;
+                    PteDst.n.u1Write     = 1;
+                    PteDst.n.u1Execute   = 1;
+                    PteDst.n.u1IgnorePAT = 1;
+                    PteDst.n.u3EMT       = VMX_EPT_MEMTYPE_WB;
+                    /* PteDst.n.u1Size = 0 */
+# else
+                    SHW_PTE_SET(PteDst, fGstShwPteFlags | PGM_PAGE_GET_HCPHYS(pPage));
+# endif
+                }
+
+                /*
+                 * Make sure only allocated pages are mapped writable.
+                 */
+                if (    SHW_PTE_IS_P_RW(PteDst)
+                    &&  PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
+                {
+                    /* Still applies to shared pages. */
+                    Assert(!PGM_PAGE_IS_ZERO(pPage));
+                    SHW_PTE_SET_RO(PteDst);   /** @todo this isn't quite working yet. Why, isn't it? */
+                    Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
+                }
+            }
+
+            /*
+             * Keep user track up to date.
+             */
+            if (SHW_PTE_IS_P(PteDst))
+            {
+                if (!SHW_PTE_IS_P(*pPteDst))
+                    PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
+                else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
+                {
+                    Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst), SHW_PTE_LOG64(PteDst)));
+                    PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
+                    PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
+                }
+            }
+            else if (SHW_PTE_IS_P(*pPteDst))
+            {
+                Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
+                PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
+            }
+
+            /*
+             * Update statistics and commit the entry.
+             */
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+            if (!PteSrc.n.u1Global)
+                pShwPage->fSeenNonGlobal = true;
+# endif
+            SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
+            return;
+        }
+
+/** @todo count these three different kinds. */
+        Log2(("SyncPageWorker: invalid address in Pte\n"));
+    }
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    else if (!PteSrc.n.u1Present)
+        Log2(("SyncPageWorker: page not present in Pte\n"));
+    else
+        Log2(("SyncPageWorker: invalid Pte\n"));
+# endif
+
+    /*
+     * The page is not present or the PTE is bad. Replace the shadow PTE by
+     * an empty entry, making sure to keep the user tracking up to date.
+     */
+    if (SHW_PTE_IS_P(*pPteDst))
+    {
+        Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
+        PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
+    }
+    SHW_PTE_ATOMIC_SET(*pPteDst, 0);
+}
+
+
+/**
+ * Syncs a guest OS page.
+ *
+ * There are no conflicts at this point, neither is there any need for
+ * page table allocations.
+ *
+ * When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
+ * When called in AMD64 guest mode, the guest PML4E shall be valid.
+ *
+ * @returns VBox status code.
+ * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   PdeSrc      Page directory entry of the guest.
+ * @param   GCPtrPage   Guest context page address.
+ * @param   cPages      Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
+ * @param   uErr        Fault error (X86_TRAP_PF_*).
+ */
+static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
+{
+    PVMCC    pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
+    LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
+    RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+# if   (   PGM_GST_TYPE == PGM_TYPE_32BIT \
+        || PGM_GST_TYPE == PGM_TYPE_PAE \
+        || PGM_GST_TYPE == PGM_TYPE_AMD64) \
+    && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
+
+    /*
+     * Assert preconditions.
+     */
+    Assert(PdeSrc.n.u1Present);
+    Assert(cPages);
+#  if 0 /* rarely useful; leave for debugging. */
+    STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
+#  endif
+
+    /*
+     * Get the shadow PDE, find the shadow page table in the pool.
+     */
+#  if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDDst   = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PX86PDE         pPdeDst  = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+    Assert(pShwPde);
+
+#  elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    const unsigned  iPDDst  = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PPGMPOOLPAGE    pShwPde = NULL;
+    PX86PDPAE       pPDDst;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
+    AssertRCSuccessReturn(rc2, rc2);
+    Assert(pShwPde);
+
+    pPDDst             = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
+    PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
+
+#  elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    const unsigned  iPDDst   = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    const unsigned  iPdpt    = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    PX86PDPAE       pPDDst   = NULL;            /* initialized to shut up gcc */
+    PX86PDPT        pPdptDst = NULL;            /* initialized to shut up gcc */
+
+    int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
+    AssertRCSuccessReturn(rc2, rc2);
+    Assert(pPDDst && pPdptDst);
+    PX86PDEPAE      pPdeDst = &pPDDst->a[iPDDst];
+#  endif
+    SHWPDE          PdeDst   = *pPdeDst;
+
+    /*
+     * - In the guest SMP case we could have blocked while another VCPU reused
+     *   this page table.
+     * - With W7-64 we may also take this path when the A bit is cleared on
+     *   higher level tables (PDPE/PML4E).  The guest does not invalidate the
+     *   relevant TLB entries.  If we're write monitoring any page mapped by
+     *   the modified entry, we may end up here with a "stale" TLB entry.
+     */
+    if (!PdeDst.n.u1Present)
+    {
+        Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
+        AssertMsg(pVM->cCpus > 1 || (uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RW)) == (X86_TRAP_PF_P | X86_TRAP_PF_RW),
+                  ("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
+        if (uErr & X86_TRAP_PF_P)
+            PGM_INVL_PG(pVCpu, GCPtrPage);
+        return VINF_SUCCESS;    /* force the instruction to be executed again. */
+    }
+
+    PPGMPOOLPAGE    pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
+    Assert(pShwPage);
+
+#  if PGM_GST_TYPE == PGM_TYPE_AMD64
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde  = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
+    Assert(pShwPde);
+#  endif
+
+    /*
+     * Check that the page is present and that the shadow PDE isn't out of sync.
+     */
+    const bool      fBigPage  = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
+    const bool      fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
+    RTGCPHYS        GCPhys;
+    if (!fBigPage)
+    {
+        GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
+#  if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+        /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
+        GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (PAGE_SIZE / 2)));
+#  endif
+    }
+    else
+    {
+        GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
+#  if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+        /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
+        GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
+#  endif
+    }
+    /** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
+    if (    fPdeValid
+        &&  pShwPage->GCPhys == GCPhys
+        &&  PdeSrc.n.u1Present
+        &&  PdeSrc.n.u1User == PdeDst.n.u1User
+        &&  (PdeSrc.n.u1Write == PdeDst.n.u1Write || !PdeDst.n.u1Write)
+#  if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
+        &&  (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute || !GST_IS_NX_ACTIVE(pVCpu))
+#  endif
+       )
+    {
+        /*
+         * Check that the PDE is marked accessed already.
+         * Since we set the accessed bit *before* getting here on a #PF, this
+         * check is only meant for dealing with non-#PF'ing paths.
+         */
+        if (PdeSrc.n.u1Accessed)
+        {
+            PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+            if (!fBigPage)
+            {
+                /*
+                 * 4KB Page - Map the guest page table.
+                 */
+                PGSTPT pPTSrc;
+                int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
+                if (RT_SUCCESS(rc))
+                {
+#  ifdef PGM_SYNC_N_PAGES
+                    Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
+                    if (    cPages > 1
+                        &&  !(uErr & X86_TRAP_PF_P)
+                        &&  !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+                    {
+                        /*
+                         * This code path is currently only taken when the caller is PGMTrap0eHandler
+                         * for non-present pages!
+                         *
+                         * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
+                         * deal with locality.
+                         */
+                        unsigned        iPTDst      = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+#   if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+                        /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
+                        const unsigned  offPTSrc    = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
+#   else
+                        const unsigned  offPTSrc    = 0;
+#   endif
+                        const unsigned  iPTDstEnd   = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
+                        if (iPTDst < PGM_SYNC_NR_PAGES / 2)
+                            iPTDst = 0;
+                        else
+                            iPTDst -= PGM_SYNC_NR_PAGES / 2;
+
+                        for (; iPTDst < iPTDstEnd; iPTDst++)
+                        {
+                            const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
+
+                            if (   pPteSrc->n.u1Present
+                                && !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
+                            {
+                                RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
+                                NOREF(GCPtrCurPage);
+                                PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
+                                Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
+                                      GCPtrCurPage, pPteSrc->n.u1Present,
+                                      pPteSrc->n.u1Write & PdeSrc.n.u1Write,
+                                      pPteSrc->n.u1User  & PdeSrc.n.u1User,
+                                      (uint64_t)pPteSrc->u,
+                                      SHW_PTE_LOG64(pPTDst->a[iPTDst]),
+                                      SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
+                            }
+                        }
+                    }
+                    else
+#  endif /* PGM_SYNC_N_PAGES */
+                    {
+                        const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
+                        GSTPTE PteSrc = pPTSrc->a[iPTSrc];
+                        const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+                        PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
+                        Log2(("SyncPage: 4K  %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
+                              GCPtrPage, PteSrc.n.u1Present,
+                              PteSrc.n.u1Write & PdeSrc.n.u1Write,
+                              PteSrc.n.u1User  & PdeSrc.n.u1User,
+                              (uint64_t)PteSrc.u,
+                              SHW_PTE_LOG64(pPTDst->a[iPTDst]),
+                              SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
+                    }
+                }
+                else /* MMIO or invalid page: emulated in #PF handler. */
+                {
+                    LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
+                    Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
+                }
+            }
+            else
+            {
+                /*
+                 * 4/2MB page - lazy syncing shadow 4K pages.
+                 * (There are many causes of getting here, it's no longer only CSAM.)
+                 */
+                /* Calculate the GC physical address of this 4KB shadow page. */
+                GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) | (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
+                /* Find ram range. */
+                PPGMPAGE pPage;
+                int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+                if (RT_SUCCESS(rc))
+                {
+                    AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
+
+#  ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+                    /* Try to make the page writable if necessary. */
+                    if (    PGM_PAGE_GET_TYPE(pPage)  == PGMPAGETYPE_RAM
+                        &&  (   PGM_PAGE_IS_ZERO(pPage)
+                             || (   PdeSrc.n.u1Write
+                                 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+#   ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
+                                 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
+#   endif
+#   ifdef VBOX_WITH_PAGE_SHARING
+                                 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
+#   endif
+                                 )
+                             )
+                       )
+                    {
+                        rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+                        AssertRC(rc);
+                    }
+#  endif
+
+                    /*
+                     * Make shadow PTE entry.
+                     */
+                    SHWPTE PteDst;
+                    if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+                        PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
+                    else
+                        SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) | PGM_PAGE_GET_HCPHYS(pPage));
+
+                    const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+                    if (    SHW_PTE_IS_P(PteDst)
+                        &&  !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
+                        PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
+
+                    /* Make sure only allocated pages are mapped writable. */
+                    if (    SHW_PTE_IS_P_RW(PteDst)
+                        &&  PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
+                    {
+                        /* Still applies to shared pages. */
+                        Assert(!PGM_PAGE_IS_ZERO(pPage));
+                        SHW_PTE_SET_RO(PteDst);   /** @todo this isn't quite working yet... */
+                        Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
+                    }
+
+                    SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
+
+                    /*
+                     * If the page is not flagged as dirty and is writable, then make it read-only
+                     * at PD level, so we can set the dirty bit when the page is modified.
+                     *
+                     * ASSUMES that page access handlers are implemented on page table entry level.
+                     *      Thus we will first catch the dirty access and set PDE.D and restart. If
+                     *      there is an access handler, we'll trap again and let it work on the problem.
+                     */
+                    /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
+                     * As for invlpg, it simply frees the whole shadow PT.
+                     * ...It's possibly because the guest clears it and the guest doesn't really tell us... */
+                    if (    !PdeSrc.b.u1Dirty
+                        &&  PdeSrc.b.u1Write)
+                    {
+                        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
+                        PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
+                        PdeDst.n.u1Write = 0;
+                    }
+                    else
+                    {
+                        PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
+                        PdeDst.n.u1Write = PdeSrc.n.u1Write;
+                    }
+                    ASMAtomicWriteSize(pPdeDst, PdeDst.u);
+                    Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
+                          GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
+                          PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
+                }
+                else
+                {
+                    LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
+                    /** @todo must wipe the shadow page table entry in this
+                     *        case. */
+                }
+            }
+            PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+            return VINF_SUCCESS;
+        }
+
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
+    }
+    else if (fPdeValid)
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
+        Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
+              GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
+    }
+    else
+    {
+/// @todo        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
+        Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
+              GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
+    }
+
+    /*
+     * Mark the PDE not present.  Restart the instruction and let #PF call SyncPT.
+     * Yea, I'm lazy.
+     */
+    pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
+    ASMAtomicWriteSize(pPdeDst, 0);
+
+    PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+    PGM_INVL_VCPU_TLBS(pVCpu);
+    return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
+
+
+# elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
+    && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
+    && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT)
+    NOREF(PdeSrc);
+
+#  ifdef PGM_SYNC_N_PAGES
+    /*
+     * Get the shadow PDE, find the shadow page table in the pool.
+     */
+#   if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    X86PDE          PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
+
+#   elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    X86PDEPAE       PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
+
+#   elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    const unsigned  iPDDst   = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    const unsigned  iPdpt    = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
+    PX86PDPAE       pPDDst   = NULL;            /* initialized to shut up gcc */
+    X86PDEPAE       PdeDst;
+    PX86PDPT        pPdptDst = NULL;            /* initialized to shut up gcc */
+
+    int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
+    AssertRCSuccessReturn(rc, rc);
+    Assert(pPDDst && pPdptDst);
+    PdeDst = pPDDst->a[iPDDst];
+#   elif PGM_SHW_TYPE == PGM_TYPE_EPT
+    const unsigned  iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PEPTPD          pPDDst;
+    EPTPDE          PdeDst;
+
+    int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
+    if (rc != VINF_SUCCESS)
+    {
+        AssertRC(rc);
+        return rc;
+    }
+    Assert(pPDDst);
+    PdeDst = pPDDst->a[iPDDst];
+#   endif
+    /* In the guest SMP case we could have blocked while another VCPU reused this page table. */
+    if (!PdeDst.n.u1Present)
+    {
+        AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
+        Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
+        return VINF_SUCCESS;    /* force the instruction to be executed again. */
+    }
+
+    /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
+    if (PdeDst.n.u1Size)
+    {
+        Assert(pVM->pgm.s.fNestedPaging);
+        Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
+        return VINF_SUCCESS;
+    }
+
+    /* Mask away the page offset. */
+    GCPtrPage &= ~((RTGCPTR)0xfff);
+
+    PPGMPOOLPAGE  pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
+    PSHWPT        pPTDst   = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+
+    Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
+    if (    cPages > 1
+        &&  !(uErr & X86_TRAP_PF_P)
+        &&  !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+    {
+        /*
+         * This code path is currently only taken when the caller is PGMTrap0eHandler
+         * for non-present pages!
+         *
+         * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
+         * deal with locality.
+         */
+        unsigned        iPTDst    = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        const unsigned  iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
+        if (iPTDst < PGM_SYNC_NR_PAGES / 2)
+            iPTDst = 0;
+        else
+            iPTDst -= PGM_SYNC_NR_PAGES / 2;
+        for (; iPTDst < iPTDstEnd; iPTDst++)
+        {
+            if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
+            {
+                RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
+                                                          | (iPTDst << PAGE_SHIFT));
+
+                PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
+                Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
+                      GCPtrCurPage,
+                      SHW_PTE_LOG64(pPTDst->a[iPTDst]),
+                      SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
+
+                if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                    break;
+            }
+            else
+                Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
+        }
+    }
+    else
+#  endif /* PGM_SYNC_N_PAGES */
+    {
+        const unsigned  iPTDst       = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        RTGCPTR         GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
+                                                          | (iPTDst << PAGE_SHIFT));
+
+        PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
+
+        Log2(("SyncPage: 4K  %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
+              GCPtrPage,
+              SHW_PTE_LOG64(pPTDst->a[iPTDst]),
+              SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
+    }
+    return VINF_SUCCESS;
+
+# else
+    NOREF(PdeSrc);
+    AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
+    return VERR_PGM_NOT_USED_IN_MODE;
+# endif
+}
+
+#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
+#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
+
+/**
+ * CheckPageFault helper for returning a page fault indicating a non-present
+ * (NP) entry in the page translation structures.
+ *
+ * @returns VINF_EM_RAW_GUEST_TRAP.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uErr            The error code of the shadow fault.  Corrections to
+ *                          TRPM's copy will be made if necessary.
+ * @param   GCPtrPage       For logging.
+ * @param   uPageFaultLevel For logging.
+ */
+DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPUCC pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
+{
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
+    AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
+    AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
+    if (uErr & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P))
+        TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD | X86_TRAP_PF_P));
+
+    Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
+    RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
+    return VINF_EM_RAW_GUEST_TRAP;
+}
+
+
+/**
+ * CheckPageFault helper for returning a page fault indicating a reserved bit
+ * (RSVD) error in the page translation structures.
+ *
+ * @returns VINF_EM_RAW_GUEST_TRAP.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uErr            The error code of the shadow fault.  Corrections to
+ *                          TRPM's copy will be made if necessary.
+ * @param   GCPtrPage       For logging.
+ * @param   uPageFaultLevel For logging.
+ */
+DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPUCC pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
+{
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
+    if ((uErr & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD | X86_TRAP_PF_P))
+        TRPMSetErrorCode(pVCpu, uErr | X86_TRAP_PF_RSVD | X86_TRAP_PF_P);
+
+    Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
+    RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
+    return VINF_EM_RAW_GUEST_TRAP;
+}
+
+
+/**
+ * CheckPageFault helper for returning a page protection fault (P).
+ *
+ * @returns VINF_EM_RAW_GUEST_TRAP.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uErr            The error code of the shadow fault.  Corrections to
+ *                          TRPM's copy will be made if necessary.
+ * @param   GCPtrPage       For logging.
+ * @param   uPageFaultLevel For logging.
+ */
+DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPUCC pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
+{
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
+    AssertMsg(uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID), ("%#x\n", uErr));
+    if ((uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
+        TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) | X86_TRAP_PF_P);
+
+    Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
+    RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
+    return VINF_EM_RAW_GUEST_TRAP;
+}
+
+
+/**
+ * Handle dirty bit tracking faults.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uErr        Page fault error code.
+ * @param   pPdeSrc     Guest page directory entry.
+ * @param   pPdeDst     Shadow page directory entry.
+ * @param   GCPtrPage   Guest context page address.
+ */
+static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
+                                             RTGCPTR GCPtrPage)
+{
+    PVMCC       pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL    pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    NOREF(uErr);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Handle big page.
+     */
+    if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
+    {
+        if (    pPdeDst->n.u1Present
+            &&  (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
+        {
+            SHWPDE PdeDst = *pPdeDst;
+
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
+            Assert(pPdeSrc->b.u1Write);
+
+            /* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
+             *       fault again and take this path to only invalidate the entry (see below).
+             */
+            PdeDst.n.u1Write      = 1;
+            PdeDst.n.u1Accessed   = 1;
+            PdeDst.au32[0]       &= ~PGM_PDFLAGS_TRACK_DIRTY;
+            ASMAtomicWriteSize(pPdeDst, PdeDst.u);
+            PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
+            return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;    /* restarts the instruction. */
+        }
+
+# ifdef IN_RING0
+        /* Check for stale TLB entry; only applies to the SMP guest case. */
+        if (    pVM->cCpus > 1
+            &&  pPdeDst->n.u1Write
+            &&  pPdeDst->n.u1Accessed)
+        {
+            PPGMPOOLPAGE    pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
+            if (pShwPage)
+            {
+                PSHWPT      pPTDst   = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+                PSHWPTE     pPteDst  = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
+                if (SHW_PTE_IS_P_RW(*pPteDst))
+                {
+                    /* Stale TLB entry. */
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
+                    PGM_INVL_PG(pVCpu, GCPtrPage);
+                    return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;    /* restarts the instruction. */
+                }
+            }
+        }
+# endif /* IN_RING0 */
+        return VINF_PGM_NO_DIRTY_BIT_TRACKING;
+    }
+
+    /*
+     * Map the guest page table.
+     */
+    PGSTPT pPTSrc;
+    int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
+    if (RT_FAILURE(rc))
+    {
+        AssertRC(rc);
+        return rc;
+    }
+
+    if (pPdeDst->n.u1Present)
+    {
+        GSTPTE const  *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
+        const GSTPTE   PteSrc  = *pPteSrc;
+
+        /*
+         * Map shadow page table.
+         */
+        PPGMPOOLPAGE    pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
+        if (pShwPage)
+        {
+            PSHWPT      pPTDst   = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+            PSHWPTE     pPteDst  = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
+            if (SHW_PTE_IS_P(*pPteDst))    /** @todo Optimize accessed bit emulation? */
+            {
+                if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
+                {
+                    PPGMPAGE pPage  = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
+                    SHWPTE   PteDst = *pPteDst;
+
+                    LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
+
+                    Assert(PteSrc.n.u1Write);
+
+                    /* Note: No need to invalidate this entry on other VCPUs as a stale TLB
+                     *       entry will not harm; write access will simply fault again and
+                     *       take this path to only invalidate the entry.
+                     */
+                    if (RT_LIKELY(pPage))
+                    {
+                        if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+                        {
+                            //AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
+                            Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
+                            /* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
+                            SHW_PTE_SET_RO(PteDst);
+                        }
+                        else
+                        {
+                            if (   PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
+                                && PGM_PAGE_GET_TYPE(pPage)  == PGMPAGETYPE_RAM)
+                            {
+                                rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
+                                AssertRC(rc);
+                            }
+                            if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
+                                SHW_PTE_SET_RW(PteDst);
+                            else
+                            {
+                                /* Still applies to shared pages. */
+                                Assert(!PGM_PAGE_IS_ZERO(pPage));
+                                SHW_PTE_SET_RO(PteDst);
+                            }
+                        }
+                    }
+                    else
+                        SHW_PTE_SET_RW(PteDst);  /** @todo r=bird: This doesn't make sense to me. */
+
+                    SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) | X86_PTE_D | X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
+                    SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
+                    PGM_INVL_PG(pVCpu, GCPtrPage);
+                    return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;    /* restarts the instruction. */
+                }
+
+# ifdef IN_RING0
+                /* Check for stale TLB entry; only applies to the SMP guest case. */
+                if (    pVM->cCpus > 1
+                    &&  SHW_PTE_IS_RW(*pPteDst)
+                    &&  SHW_PTE_IS_A(*pPteDst))
+                {
+                    /* Stale TLB entry. */
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
+                    PGM_INVL_PG(pVCpu, GCPtrPage);
+                    return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;    /* restarts the instruction. */
+                }
+# endif
+            }
+        }
+        else
+            AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
+    }
+
+    return VINF_PGM_NO_DIRTY_BIT_TRACKING;
+}
+
+#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
+
+/**
+ * Sync a shadow page table.
+ *
+ * The shadow page table is not present in the shadow PDE.
+ *
+ * Handles mapping conflicts.
+ *
+ * This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
+ * conflict), and Trap0eHandler.
+ *
+ * A precondition for this method is that the shadow PDE is not present.  The
+ * caller must take the PGM lock before checking this and continue to hold it
+ * when calling this method.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iPDSrc      Page directory index.
+ * @param   pPDSrc      Source page directory (i.e. Guest OS page directory).
+ *                      Assume this is a temporary mapping.
+ * @param   GCPtrPage   GC Pointer of the page that caused the fault
+ */
+static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
+{
+    PVMCC       pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL    pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
+
+#if 0 /* rarely useful; leave for debugging. */
+    STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
+#endif
+    LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+#if (   PGM_GST_TYPE == PGM_TYPE_32BIT \
+     || PGM_GST_TYPE == PGM_TYPE_PAE \
+     || PGM_GST_TYPE == PGM_TYPE_AMD64) \
+ && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
+ && PGM_SHW_TYPE != PGM_TYPE_NONE
+    int             rc       = VINF_SUCCESS;
+
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
+
+    /*
+     * Some input validation first.
+     */
+    AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
+
+    /*
+     * Get the relevant shadow PDE entry.
+     */
+# if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDDst   = GCPtrPage >> SHW_PD_SHIFT;
+    PSHWPDE         pPdeDst  = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde  = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+    Assert(pShwPde);
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    const unsigned  iPDDst   = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PPGMPOOLPAGE    pShwPde  = NULL;
+    PX86PDPAE       pPDDst;
+    PSHWPDE         pPdeDst;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
+    AssertRCSuccessReturn(rc, rc);
+    Assert(pShwPde);
+
+    pPDDst  = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
+    pPdeDst = &pPDDst->a[iPDDst];
+
+# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    const unsigned  iPdpt    = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    const unsigned  iPDDst   = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PX86PDPAE       pPDDst   = NULL;            /* initialized to shut up gcc */
+    PX86PDPT        pPdptDst = NULL;            /* initialized to shut up gcc */
+    rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
+    AssertRCSuccessReturn(rc, rc);
+    Assert(pPDDst);
+    PSHWPDE         pPdeDst  = &pPDDst->a[iPDDst];
+# endif
+    SHWPDE          PdeDst   = *pPdeDst;
+
+# if PGM_GST_TYPE == PGM_TYPE_AMD64
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde  = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
+    Assert(pShwPde);
+# endif
+
+# ifndef PGM_WITHOUT_MAPPINGS
+    /*
+     * Check for conflicts.
+     * RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
+     * R3: Simply resolve the conflict.
+     */
+    if (PdeDst.u & PGM_PDFLAGS_MAPPING)
+    {
+        Assert(pgmMapAreMappingsEnabled(pVM));
+#  ifndef IN_RING3
+        Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
+        STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
+        return VERR_ADDRESS_CONFLICT;
+
+#  else  /* IN_RING3 */
+        PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
+        Assert(pMapping);
+#   if PGM_GST_TYPE == PGM_TYPE_32BIT
+        rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
+#   elif PGM_GST_TYPE == PGM_TYPE_PAE
+        rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
+#   else
+        AssertFailed(); NOREF(pMapping); /* can't happen for amd64 */
+#   endif
+        if (RT_FAILURE(rc))
+        {
+            STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
+            return rc;
+        }
+        PdeDst = *pPdeDst;
+#  endif /* IN_RING3 */
+    }
+# endif /* !PGM_WITHOUT_MAPPINGS */
+    Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
+
+    /*
+     * Sync the page directory entry.
+     */
+    GSTPDE      PdeSrc = pPDSrc->a[iPDSrc];
+    const bool  fPageTable = !PdeSrc.b.u1Size || !GST_IS_PSE_ACTIVE(pVCpu);
+    if (   PdeSrc.n.u1Present
+        && (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
+    {
+        /*
+         * Allocate & map the page table.
+         */
+        PSHWPT          pPTDst;
+        PPGMPOOLPAGE    pShwPage;
+        RTGCPHYS        GCPhys;
+        if (fPageTable)
+        {
+            GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
+# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+            /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
+            GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (PAGE_SIZE / 2)));
+# endif
+            rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                              pShwPde->idx, iPDDst, false /*fLockPage*/,
+                              &pShwPage);
+        }
+        else
+        {
+            PGMPOOLACCESS enmAccess;
+# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
+            const bool  fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
+# else
+            const bool  fNoExecute = false;
+# endif
+
+            GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
+# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+            /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
+            GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
+# endif
+            /* Determine the right kind of large page to avoid incorrect cached entry reuse. */
+            if (PdeSrc.n.u1User)
+            {
+                if (PdeSrc.n.u1Write)
+                    enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
+                else
+                    enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX  : PGMPOOLACCESS_USER_R;
+            }
+            else
+            {
+                if (PdeSrc.n.u1Write)
+                    enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
+                else
+                    enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX  : PGMPOOLACCESS_SUPERVISOR_R;
+            }
+            rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
+                              pShwPde->idx, iPDDst, false /*fLockPage*/,
+                              &pShwPage);
+        }
+        if (rc == VINF_SUCCESS)
+            pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+        else if (rc == VINF_PGM_CACHED_PAGE)
+        {
+            /*
+             * The PT was cached, just hook it up.
+             */
+            if (fPageTable)
+                PdeDst.u = pShwPage->Core.Key | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
+            else
+            {
+                PdeDst.u = pShwPage->Core.Key | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
+                /* (see explanation and assumptions further down.) */
+                if (    !PdeSrc.b.u1Dirty
+                    &&  PdeSrc.b.u1Write)
+                {
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
+                    PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
+                    PdeDst.b.u1Write = 0;
+                }
+            }
+            ASMAtomicWriteSize(pPdeDst, PdeDst.u);
+            PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+            return VINF_SUCCESS;
+        }
+        else
+            AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+        /** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
+         * Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
+         * irrelevant at this point. */
+        PdeDst.u &= X86_PDE_AVL_MASK;
+        PdeDst.u |= pShwPage->Core.Key;
+
+        /*
+         * Page directory has been accessed (this is a fault situation, remember).
+         */
+        /** @todo
+         * Well, when the caller is PrefetchPage or InvalidatePage is isn't a
+         * fault situation.  What's more, the Trap0eHandler has already set the
+         * accessed bit.  So, it's actually just VerifyAccessSyncPage which
+         * might need setting the accessed flag.
+         *
+         * The best idea is to leave this change to the caller and add an
+         * assertion that it's set already. */
+        pPDSrc->a[iPDSrc].n.u1Accessed = 1;
+        if (fPageTable)
+        {
+            /*
+             * Page table - 4KB.
+             *
+             * Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
+             */
+            Log2(("SyncPT:   4K  %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
+                  GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
+            PGSTPT pPTSrc;
+            rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Start by syncing the page directory entry so CSAM's TLB trick works.
+                 */
+                PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | X86_PDE_AVL_MASK))
+                         | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
+                ASMAtomicWriteSize(pPdeDst, PdeDst.u);
+                PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+
+                /*
+                 * Directory/page user or supervisor privilege: (same goes for read/write)
+                 *
+                 * Directory    Page    Combined
+                 * U/S          U/S     U/S
+                 *  0            0       0
+                 *  0            1       0
+                 *  1            0       0
+                 *  1            1       1
+                 *
+                 * Simple AND operation. Table listed for completeness.
+                 *
+                 */
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
+# ifdef PGM_SYNC_N_PAGES
+                unsigned        iPTBase   = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+                unsigned        iPTDst    = iPTBase;
+                const unsigned  iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
+                if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
+                    iPTDst = 0;
+                else
+                    iPTDst -= PGM_SYNC_NR_PAGES / 2;
+# else /* !PGM_SYNC_N_PAGES */
+                unsigned        iPTDst    = 0;
+                const unsigned  iPTDstEnd = RT_ELEMENTS(pPTDst->a);
+# endif /* !PGM_SYNC_N_PAGES */
+                RTGCPTR         GCPtrCur  = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
+                                          | ((RTGCPTR)iPTDst << PAGE_SHIFT);
+# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+                /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
+                const unsigned  offPTSrc  = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
+# else
+                const unsigned  offPTSrc  = 0;
+# endif
+                for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += PAGE_SIZE)
+                {
+                    const unsigned iPTSrc = iPTDst + offPTSrc;
+                    const GSTPTE   PteSrc = pPTSrc->a[iPTSrc];
+
+                    if (PteSrc.n.u1Present)
+                    {
+                        PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
+                        Log2(("SyncPT:   4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
+                              GCPtrCur,
+                              PteSrc.n.u1Present,
+                              PteSrc.n.u1Write & PdeSrc.n.u1Write,
+                              PteSrc.n.u1User  & PdeSrc.n.u1User,
+                              (uint64_t)PteSrc.u,
+                              SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
+                              (RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
+                    }
+                    /* else: the page table was cleared by the pool */
+                } /* for PTEs */
+            }
+        }
+        else
+        {
+            /*
+             * Big page - 2/4MB.
+             *
+             * We'll walk the ram range list in parallel and optimize lookups.
+             * We will only sync one shadow page table at a time.
+             */
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
+
+            /**
+             * @todo It might be more efficient to sync only a part of the 4MB
+             *       page (similar to what we do for 4KB PDs).
+             */
+
+            /*
+             * Start by syncing the page directory entry.
+             */
+            PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
+                     | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
+
+            /*
+             * If the page is not flagged as dirty and is writable, then make it read-only
+             * at PD level, so we can set the dirty bit when the page is modified.
+             *
+             * ASSUMES that page access handlers are implemented on page table entry level.
+             *      Thus we will first catch the dirty access and set PDE.D and restart. If
+             *      there is an access handler, we'll trap again and let it work on the problem.
+             */
+            /** @todo move the above stuff to a section in the PGM documentation. */
+            Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
+            if (    !PdeSrc.b.u1Dirty
+                &&  PdeSrc.b.u1Write)
+            {
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
+                PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
+                PdeDst.b.u1Write = 0;
+            }
+            ASMAtomicWriteSize(pPdeDst, PdeDst.u);
+            PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+
+            /*
+             * Fill the shadow page table.
+             */
+            /* Get address and flags from the source PDE. */
+            SHWPTE PteDstBase;
+            SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
+
+            /* Loop thru the entries in the shadow PT. */
+            const RTGCPTR   GCPtr  = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
+            Log2(("SyncPT:   BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
+                  GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
+                  GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
+            PPGMRAMRANGE    pRam   = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
+            unsigned        iPTDst = 0;
+            while (     iPTDst < RT_ELEMENTS(pPTDst->a)
+                   &&   !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+            {
+                if (pRam && GCPhys >= pRam->GCPhys)
+                {
+# ifndef PGM_WITH_A20
+                    unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
+# endif
+                    do
+                    {
+                        /* Make shadow PTE. */
+# ifdef PGM_WITH_A20
+                        PPGMPAGE    pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+# else
+                        PPGMPAGE    pPage = &pRam->aPages[iHCPage];
+# endif
+                        SHWPTE      PteDst;
+
+# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+                        /* Try to make the page writable if necessary. */
+                        if (    PGM_PAGE_GET_TYPE(pPage)  == PGMPAGETYPE_RAM
+                            &&  (   PGM_PAGE_IS_ZERO(pPage)
+                                 || (   SHW_PTE_IS_RW(PteDstBase)
+                                     && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+#  ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
+                                     && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
+#  endif
+#  ifdef VBOX_WITH_PAGE_SHARING
+                                     && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
+#  endif
+                                     && !PGM_PAGE_IS_BALLOONED(pPage))
+                                 )
+                           )
+                        {
+                            rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+                            AssertRCReturn(rc, rc);
+                            if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+                                break;
+                        }
+# endif
+
+                        if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+                            PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, SHW_PTE_GET_U(PteDstBase), &PteDst);
+                        else if (PGM_PAGE_IS_BALLOONED(pPage))
+                            SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
+                        else
+                            SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) | SHW_PTE_GET_U(PteDstBase));
+
+                        /* Only map writable pages writable. */
+                        if (    SHW_PTE_IS_P_RW(PteDst)
+                            &&  PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
+                        {
+                            /* Still applies to shared pages. */
+                            Assert(!PGM_PAGE_IS_ZERO(pPage));
+                            SHW_PTE_SET_RO(PteDst);   /** @todo this isn't quite working yet... */
+                            Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))));
+                        }
+
+                        if (SHW_PTE_IS_P(PteDst))
+                            PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
+
+                        /* commit it (not atomic, new table) */
+                        pPTDst->a[iPTDst] = PteDst;
+                        Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
+                              (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
+                              SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
+
+                        /* advance */
+                        GCPhys += PAGE_SIZE;
+                        PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
+# ifndef PGM_WITH_A20
+                        iHCPage++;
+# endif
+                        iPTDst++;
+                    } while (   iPTDst < RT_ELEMENTS(pPTDst->a)
+                             && GCPhys <= pRam->GCPhysLast);
+
+                    /* Advance ram range list. */
+                    while (pRam && GCPhys > pRam->GCPhysLast)
+                        pRam = pRam->CTX_SUFF(pNext);
+                }
+                else if (pRam)
+                {
+                    Log(("Invalid pages at %RGp\n", GCPhys));
+                    do
+                    {
+                        SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
+                        GCPhys += PAGE_SIZE;
+                        iPTDst++;
+                    } while (   iPTDst < RT_ELEMENTS(pPTDst->a)
+                             && GCPhys < pRam->GCPhys);
+                    PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
+                }
+                else
+                {
+                    Log(("Invalid pages at %RGp (2)\n", GCPhys));
+                    for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
+                        SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
+                }
+            } /* while more PTEs */
+        } /* 4KB / 4MB */
+    }
+    else
+        AssertRelease(!PdeDst.n.u1Present);
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
+    if (RT_FAILURE(rc))
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
+    return rc;
+
+#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
+    && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
+    && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
+    && PGM_SHW_TYPE != PGM_TYPE_NONE
+    NOREF(iPDSrc); NOREF(pPDSrc);
+
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
+
+    /*
+     * Validate input a little bit.
+     */
+    int             rc = VINF_SUCCESS;
+# if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDDst  = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PSHWPDE         pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+    Assert(pShwPde);
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    const unsigned  iPDDst  = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PPGMPOOLPAGE    pShwPde = NULL;             /* initialized to shut up gcc */
+    PX86PDPAE       pPDDst;
+    PSHWPDE         pPdeDst;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
+    AssertRCSuccessReturn(rc, rc);
+    Assert(pShwPde);
+
+    pPDDst  = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
+    pPdeDst = &pPDDst->a[iPDDst];
+
+# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    const unsigned  iPdpt   = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    const unsigned  iPDDst  = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PX86PDPAE       pPDDst  = NULL;             /* initialized to shut up gcc */
+    PX86PDPT        pPdptDst= NULL;             /* initialized to shut up gcc */
+    rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
+    AssertRCSuccessReturn(rc, rc);
+    Assert(pPDDst);
+    PSHWPDE         pPdeDst = &pPDDst->a[iPDDst];
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
+    Assert(pShwPde);
+
+# elif PGM_SHW_TYPE == PGM_TYPE_EPT
+    const unsigned  iPdpt   = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
+    const unsigned  iPDDst  = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PEPTPD          pPDDst;
+    PEPTPDPT        pPdptDst;
+
+    rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
+    if (rc != VINF_SUCCESS)
+    {
+        STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
+        AssertRC(rc);
+        return rc;
+    }
+    Assert(pPDDst);
+    PSHWPDE         pPdeDst = &pPDDst->a[iPDDst];
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
+    Assert(pShwPde);
+# endif
+    SHWPDE          PdeDst = *pPdeDst;
+
+    Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
+    Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
+
+# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
+    if (    BTH_IS_NP_ACTIVE(pVM)
+        && !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Large page support. */
+    {
+        /* Check if we allocated a big page before for this 2 MB range. */
+        PPGMPAGE pPage;
+        rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
+        if (RT_SUCCESS(rc))
+        {
+            RTHCPHYS HCPhys = NIL_RTHCPHYS;
+            if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
+            {
+                if (PGM_A20_IS_ENABLED(pVCpu))
+                {
+                    STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
+                    AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+                    HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+                }
+                else
+                {
+                    PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
+                    pVM->pgm.s.cLargePagesDisabled++;
+                }
+            }
+            else if (   PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
+                     && PGM_A20_IS_ENABLED(pVCpu))
+            {
+                /* Recheck the entire 2 MB range to see if we can use it again as a large page. */
+                rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
+                if (RT_SUCCESS(rc))
+                {
+                    Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+                    Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
+                    HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+                }
+            }
+            else if (   PGMIsUsingLargePages(pVM)
+                     && PGM_A20_IS_ENABLED(pVCpu))
+            {
+                rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
+                if (RT_SUCCESS(rc))
+                {
+                    Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+                    Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
+                    HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+                }
+                else
+                    LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
+            }
+
+            if (HCPhys != NIL_RTHCPHYS)
+            {
+                PdeDst.u &= X86_PDE_AVL_MASK;
+                PdeDst.u |= HCPhys;
+                PdeDst.n.u1Present   = 1;
+                PdeDst.n.u1Write     = 1;
+                PdeDst.b.u1Size      = 1;
+#  if  PGM_SHW_TYPE == PGM_TYPE_EPT
+                PdeDst.n.u1Execute   = 1;
+                PdeDst.b.u1IgnorePAT = 1;
+                PdeDst.b.u3EMT       = VMX_EPT_MEMTYPE_WB;
+#  else
+                PdeDst.n.u1User      = 1;
+#  endif
+                ASMAtomicWriteSize(pPdeDst, PdeDst.u);
+
+                Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
+                /* Add a reference to the first page only. */
+                PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
+
+                STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
+                return VINF_SUCCESS;
+            }
+        }
+    }
+# endif /* defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE */
+
+    /*
+     * Allocate & map the page table.
+     */
+    PSHWPT          pPTDst;
+    PPGMPOOLPAGE    pShwPage;
+    RTGCPHYS        GCPhys;
+
+    /* Virtual address = physical address */
+    GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
+    rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
+                      PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /*fLockPage*/,
+                      &pShwPage);
+    if (    rc == VINF_SUCCESS
+        ||  rc == VINF_PGM_CACHED_PAGE)
+        pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    else
+    {
+       STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
+       AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+    }
+
+    if (rc == VINF_SUCCESS)
+    {
+        /* New page table; fully set it up. */
+        Assert(pPTDst);
+
+        /* Mask away the page offset. */
+        GCPtrPage &= ~(RTGCPTR)PAGE_OFFSET_MASK;
+
+        for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
+        {
+            RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
+                                                      | (iPTDst << PAGE_SHIFT));
+
+            PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
+            Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
+                  GCPtrCurPage,
+                  SHW_PTE_LOG64(pPTDst->a[iPTDst]),
+                  SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
+
+            if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                break;
+        }
+    }
+    else
+        rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
+
+    /* Save the new PDE. */
+    PdeDst.u &= X86_PDE_AVL_MASK;
+    PdeDst.u |= pShwPage->Core.Key;
+    PdeDst.n.u1Present  = 1;
+    PdeDst.n.u1Write    = 1;
+# if PGM_SHW_TYPE == PGM_TYPE_EPT
+    PdeDst.n.u1Execute  = 1;
+# else
+    PdeDst.n.u1User     = 1;
+    PdeDst.n.u1Accessed = 1;
+# endif
+    ASMAtomicWriteSize(pPdeDst, PdeDst.u);
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
+    if (RT_FAILURE(rc))
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
+    return rc;
+
+#else
+    NOREF(iPDSrc); NOREF(pPDSrc);
+    AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
+    return VERR_PGM_NOT_USED_IN_MODE;
+#endif
+}
+
+
+
+/**
+ * Prefetch a page/set of pages.
+ *
+ * Typically used to sync commonly used pages before entering raw mode
+ * after a CR3 reload.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtrPage   Page to invalidate.
+ */
+PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
+{
+#if (   PGM_GST_TYPE == PGM_TYPE_32BIT \
+     || PGM_GST_TYPE == PGM_TYPE_REAL \
+     || PGM_GST_TYPE == PGM_TYPE_PROT \
+     || PGM_GST_TYPE == PGM_TYPE_PAE \
+     || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
+ && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
+ && PGM_SHW_TYPE != PGM_TYPE_NONE
+    /*
+     * Check that all Guest levels thru the PDE are present, getting the
+     * PD and PDE in the processes.
+     */
+    int             rc     = VINF_SUCCESS;
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+#  if PGM_GST_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
+    PGSTPD          pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
+#  elif PGM_GST_TYPE == PGM_TYPE_PAE
+    unsigned        iPDSrc;
+    X86PDPE         PdpeSrc;
+    PGSTPD          pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
+    if (!pPDSrc)
+        return VINF_SUCCESS; /* not present */
+#  elif PGM_GST_TYPE == PGM_TYPE_AMD64
+    unsigned        iPDSrc;
+    PX86PML4E       pPml4eSrc;
+    X86PDPE         PdpeSrc;
+    PGSTPD          pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
+    if (!pPDSrc)
+        return VINF_SUCCESS; /* not present */
+#  endif
+    const GSTPDE    PdeSrc = pPDSrc->a[iPDSrc];
+# else
+    PGSTPD          pPDSrc = NULL;
+    const unsigned  iPDSrc = 0;
+    GSTPDE          PdeSrc;
+
+    PdeSrc.u            = 0; /* faked so we don't have to #ifdef everything */
+    PdeSrc.n.u1Present  = 1;
+    PdeSrc.n.u1Write    = 1;
+    PdeSrc.n.u1Accessed = 1;
+    PdeSrc.n.u1User     = 1;
+# endif
+
+    if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
+    {
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        pgmLock(pVM);
+
+# if PGM_SHW_TYPE == PGM_TYPE_32BIT
+        const X86PDE    PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE
+        const unsigned  iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+        PX86PDPAE       pPDDst;
+        X86PDEPAE       PdeDst;
+#   if PGM_GST_TYPE != PGM_TYPE_PAE
+        X86PDPE         PdpeSrc;
+
+        /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
+        PdpeSrc.u  = X86_PDPE_P;   /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
+#   endif
+        rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
+        if (rc != VINF_SUCCESS)
+        {
+            pgmUnlock(pVM);
+            AssertRC(rc);
+            return rc;
+        }
+        Assert(pPDDst);
+        PdeDst = pPDDst->a[iPDDst];
+
+# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+        const unsigned  iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+        PX86PDPAE       pPDDst;
+        X86PDEPAE       PdeDst;
+
+#  if PGM_GST_TYPE == PGM_TYPE_PROT
+        /* AMD-V nested paging */
+        X86PML4E        Pml4eSrc;
+        X86PDPE         PdpeSrc;
+        PX86PML4E       pPml4eSrc = &Pml4eSrc;
+
+        /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
+        Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
+        PdpeSrc.u  = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
+#  endif
+
+        rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
+        if (rc != VINF_SUCCESS)
+        {
+            pgmUnlock(pVM);
+            AssertRC(rc);
+            return rc;
+        }
+        Assert(pPDDst);
+        PdeDst = pPDDst->a[iPDDst];
+# endif
+        if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
+        {
+            if (!PdeDst.n.u1Present)
+            {
+                /** @todo r=bird: This guy will set the A bit on the PDE,
+                 *    probably harmless. */
+                rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
+            }
+            else
+            {
+                /* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
+                 *       R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
+                 *       makes no sense to prefetch more than one page.
+                 */
+                rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
+                if (RT_SUCCESS(rc))
+                    rc = VINF_SUCCESS;
+            }
+        }
+        pgmUnlock(pVM);
+    }
+    return rc;
+
+#elif PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
+    NOREF(pVCpu); NOREF(GCPtrPage);
+    return VINF_SUCCESS; /* ignore */
+#else
+    AssertCompile(0);
+#endif
+}
+
+
+
+
+/**
+ * Syncs a page during a PGMVerifyAccess() call.
+ *
+ * @returns VBox status code (informational included).
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtrPage   The address of the page to sync.
+ * @param   fPage       The effective guest page flags.
+ * @param   uErr        The trap error code.
+ * @remarks This will normally never be called on invalid guest page
+ *          translation entries.
+ */
+PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
+
+    LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
+    RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(fPage); RT_NOREF_PV(uErr);
+
+    Assert(!pVM->pgm.s.fNestedPaging);
+#if   (   PGM_GST_TYPE == PGM_TYPE_32BIT \
+       || PGM_GST_TYPE == PGM_TYPE_REAL \
+       || PGM_GST_TYPE == PGM_TYPE_PROT \
+       || PGM_GST_TYPE == PGM_TYPE_PAE \
+       || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
+    && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
+    && PGM_SHW_TYPE != PGM_TYPE_NONE
+
+    /*
+     * Get guest PD and index.
+     */
+    /** @todo Performance: We've done all this a jiffy ago in the
+     *        PGMGstGetPage call. */
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+#  if PGM_GST_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
+    PGSTPD          pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
+
+#  elif PGM_GST_TYPE == PGM_TYPE_PAE
+    unsigned        iPDSrc = 0;
+    X86PDPE         PdpeSrc;
+    PGSTPD          pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
+    if (RT_UNLIKELY(!pPDSrc))
+    {
+        Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
+        return VINF_EM_RAW_GUEST_TRAP;
+    }
+
+#  elif PGM_GST_TYPE == PGM_TYPE_AMD64
+    unsigned        iPDSrc = 0;         /* shut up gcc */
+    PX86PML4E       pPml4eSrc = NULL;   /* ditto */
+    X86PDPE         PdpeSrc;
+    PGSTPD          pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
+    if (RT_UNLIKELY(!pPDSrc))
+    {
+        Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
+        return VINF_EM_RAW_GUEST_TRAP;
+    }
+#  endif
+
+# else  /* !PGM_WITH_PAGING */
+    PGSTPD          pPDSrc = NULL;
+    const unsigned  iPDSrc = 0;
+# endif /* !PGM_WITH_PAGING */
+    int             rc = VINF_SUCCESS;
+
+    pgmLock(pVM);
+
+    /*
+     * First check if the shadow pd is present.
+     */
+# if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    PX86PDE         pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    PX86PDEPAE      pPdeDst;
+    const unsigned  iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PX86PDPAE       pPDDst;
+#   if PGM_GST_TYPE != PGM_TYPE_PAE
+    /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
+    X86PDPE         PdpeSrc;
+    PdpeSrc.u  = X86_PDPE_P;   /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
+#   endif
+    rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
+    if (rc != VINF_SUCCESS)
+    {
+        pgmUnlock(pVM);
+        AssertRC(rc);
+        return rc;
+    }
+    Assert(pPDDst);
+    pPdeDst = &pPDDst->a[iPDDst];
+
+# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    const unsigned  iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PX86PDPAE       pPDDst;
+    PX86PDEPAE      pPdeDst;
+
+#  if PGM_GST_TYPE == PGM_TYPE_PROT
+    /* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
+    X86PML4E        Pml4eSrc;
+    X86PDPE         PdpeSrc;
+    PX86PML4E       pPml4eSrc = &Pml4eSrc;
+    Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
+    PdpeSrc.u  = X86_PDPE_P  | X86_PDPE_RW  | X86_PDPE_US  | X86_PDPE_A;
+#  endif
+
+    rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
+    if (rc != VINF_SUCCESS)
+    {
+        pgmUnlock(pVM);
+        AssertRC(rc);
+        return rc;
+    }
+    Assert(pPDDst);
+    pPdeDst = &pPDDst->a[iPDDst];
+# endif
+
+    if (!pPdeDst->n.u1Present)
+    {
+        rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
+        if (rc != VINF_SUCCESS)
+        {
+            PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+            pgmUnlock(pVM);
+            AssertRC(rc);
+            return rc;
+        }
+    }
+
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    /* Check for dirty bit fault */
+    rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
+    if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
+        Log(("PGMVerifyAccess: success (dirty)\n"));
+    else
+# endif
+    {
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        GSTPDE PdeSrc       = pPDSrc->a[iPDSrc];
+# else
+        GSTPDE PdeSrc;
+        PdeSrc.u            = 0; /* faked so we don't have to #ifdef everything */
+        PdeSrc.n.u1Present  = 1;
+        PdeSrc.n.u1Write    = 1;
+        PdeSrc.n.u1Accessed = 1;
+        PdeSrc.n.u1User     = 1;
+# endif
+
+        Assert(rc != VINF_EM_RAW_GUEST_TRAP);
+        if (uErr & X86_TRAP_PF_US)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
+        else /* supervisor */
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
+
+        rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
+        if (RT_SUCCESS(rc))
+        {
+            /* Page was successfully synced */
+            Log2(("PGMVerifyAccess: success (sync)\n"));
+            rc = VINF_SUCCESS;
+        }
+        else
+        {
+            Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
+            rc = VINF_EM_RAW_GUEST_TRAP;
+        }
+    }
+    PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+    pgmUnlock(pVM);
+    return rc;
+
+#else  /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
+
+    AssertLogRelMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
+    return VERR_PGM_NOT_USED_IN_MODE;
+#endif /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
+}
+
+
+/**
+ * Syncs the paging hierarchy starting at CR3.
+ *
+ * @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
+ *          informational status codes.
+ * @retval  VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
+ *          the VMM into guest context.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cr0         Guest context CR0 register.
+ * @param   cr3         Guest context CR3 register. Not subjected to the A20
+ *                      mask.
+ * @param   cr4         Guest context CR4 register.
+ * @param   fGlobal     Including global page directories or not
+ */
+PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
+    NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
+
+    LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
+
+#if !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    pgmLock(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    if (pPool->cDirtyPages)
+        pgmPoolResetDirtyPages(pVM);
+    pgmUnlock(pVM);
+# endif
+#endif /* !NESTED && !EPT */
+
+#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
+    /*
+     * Nested / EPT / None - No work.
+     */
+    Assert(!pgmMapAreMappingsEnabled(pVM));
+    return VINF_SUCCESS;
+
+#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    /*
+     * AMD64 (Shw & Gst) - No need to check all paging levels; we zero
+     * out the shadow parts when the guest modifies its tables.
+     */
+    Assert(!pgmMapAreMappingsEnabled(pVM));
+    return VINF_SUCCESS;
+
+#else /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
+
+#  ifndef PGM_WITHOUT_MAPPINGS
+    /*
+     * Check for and resolve conflicts with our guest mappings if they
+     * are enabled and not fixed.
+     */
+    if (pgmMapAreMappingsFloating(pVM))
+    {
+        int rc = pgmMapResolveConflicts(pVM);
+        Assert(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3);
+        if (rc == VINF_SUCCESS)
+        { /* likely */ }
+        else if (rc == VINF_PGM_SYNC_CR3)
+        {
+            LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
+            return VINF_PGM_SYNC_CR3;
+        }
+        else if (RT_FAILURE(rc))
+            return rc;
+        else
+            AssertMsgFailed(("%Rrc\n", rc));
+    }
+#  else
+    Assert(!pgmMapAreMappingsEnabled(pVM));
+#  endif
+    return VINF_SUCCESS;
+#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
+}
+
+
+
+
+#ifdef VBOX_STRICT
+
+/**
+ * Checks that the shadow page table is in sync with the guest one.
+ *
+ * @returns The number of errors.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cr3         Guest context CR3 register.
+ * @param   cr4         Guest context CR4 register.
+ * @param   GCPtr       Where to start. Defaults to 0.
+ * @param   cb          How much to check. Defaults to everything.
+ */
+PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
+{
+    NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
+#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
+    return 0;
+#else
+    unsigned cErrors = 0;
+    PVMCC    pVM     = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL pPool   = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
+
+# if PGM_GST_TYPE == PGM_TYPE_PAE
+    /** @todo currently broken; crashes below somewhere */
+    AssertFailed();
+# endif
+
+# if   PGM_GST_TYPE == PGM_TYPE_32BIT \
+    || PGM_GST_TYPE == PGM_TYPE_PAE \
+    || PGM_GST_TYPE == PGM_TYPE_AMD64
+
+    bool            fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
+    PPGMCPU         pPGM = &pVCpu->pgm.s;
+    RTGCPHYS        GCPhysGst;              /* page address derived from the guest page tables. */
+    RTHCPHYS        HCPhysShw;              /* page address derived from the shadow page tables. */
+#  ifndef IN_RING0
+    RTHCPHYS        HCPhys;                 /* general usage. */
+#  endif
+    int             rc;
+
+    /*
+     * Check that the Guest CR3 and all its mappings are correct.
+     */
+    AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
+                    ("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
+                    false);
+#  if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
+#   if 0
+#    if PGM_GST_TYPE == PGM_TYPE_32BIT
+    rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
+#    else
+    rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
+#    endif
+    AssertRCReturn(rc, 1);
+    HCPhys = NIL_RTHCPHYS;
+    rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
+    AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
+#   endif
+#   if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
+    pgmGstGet32bitPDPtr(pVCpu);
+    RTGCPHYS GCPhys;
+    rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
+    AssertRCReturn(rc, 1);
+    AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
+#   endif
+#  endif /* !IN_RING0 */
+
+    /*
+     * Get and check the Shadow CR3.
+     */
+#  if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    unsigned        cPDEs       = X86_PG_ENTRIES;
+    unsigned        cIncrement  = X86_PG_ENTRIES * PAGE_SIZE;
+#  elif PGM_SHW_TYPE == PGM_TYPE_PAE
+#   if PGM_GST_TYPE == PGM_TYPE_32BIT
+    unsigned        cPDEs       = X86_PG_PAE_ENTRIES * 4;   /* treat it as a 2048 entry table. */
+#   else
+    unsigned        cPDEs       = X86_PG_PAE_ENTRIES;
+#   endif
+    unsigned        cIncrement  = X86_PG_PAE_ENTRIES * PAGE_SIZE;
+#  elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    unsigned        cPDEs       = X86_PG_PAE_ENTRIES;
+    unsigned        cIncrement  = X86_PG_PAE_ENTRIES * PAGE_SIZE;
+#  endif
+    if (cb != ~(RTGCPTR)0)
+        cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
+
+/** @todo call the other two PGMAssert*() functions. */
+
+#  if PGM_GST_TYPE == PGM_TYPE_AMD64
+    unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+
+    for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
+    {
+        PPGMPOOLPAGE    pShwPdpt = NULL;
+        PX86PML4E       pPml4eSrc;
+        PX86PML4E       pPml4eDst;
+        RTGCPHYS        GCPhysPdptSrc;
+
+        pPml4eSrc     = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
+        pPml4eDst     = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
+
+        /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
+        if (!pPml4eDst->n.u1Present)
+        {
+            GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
+            continue;
+        }
+
+        pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
+        GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
+
+        if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
+        {
+            AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
+            GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
+            cErrors++;
+            continue;
+        }
+
+        if (GCPhysPdptSrc != pShwPdpt->GCPhys)
+        {
+            AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
+            GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
+            cErrors++;
+            continue;
+        }
+
+        if (    pPml4eDst->n.u1User      != pPml4eSrc->n.u1User
+            ||  pPml4eDst->n.u1Write     != pPml4eSrc->n.u1Write
+            ||  pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
+        {
+            AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
+            GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
+            cErrors++;
+            continue;
+        }
+#  else  /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
+    {
+#  endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
+
+#  if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
+        /*
+         * Check the PDPTEs too.
+         */
+        unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
+
+        for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
+        {
+            unsigned        iPDSrc  = 0;        /* initialized to shut up gcc */
+            PPGMPOOLPAGE    pShwPde = NULL;
+            PX86PDPE        pPdpeDst;
+            RTGCPHYS        GCPhysPdeSrc;
+            X86PDPE         PdpeSrc;
+            PdpeSrc.u = 0;                      /* initialized to shut up gcc 4.5 */
+#   if PGM_GST_TYPE == PGM_TYPE_PAE
+            PGSTPD          pPDSrc    = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
+            PX86PDPT        pPdptDst  = pgmShwGetPaePDPTPtr(pVCpu);
+#   else
+            PX86PML4E       pPml4eSrcIgn;
+            PX86PDPT        pPdptDst;
+            PX86PDPAE       pPDDst;
+            PGSTPD          pPDSrc    = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
+
+            rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
+            if (rc != VINF_SUCCESS)
+            {
+                AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
+                GCPtr += 512 * _2M;
+                continue;   /* next PDPTE */
+            }
+            Assert(pPDDst);
+#   endif
+            Assert(iPDSrc == 0);
+
+            pPdpeDst = &pPdptDst->a[iPdpt];
+
+            if (!pPdpeDst->n.u1Present)
+            {
+                GCPtr += 512 * _2M;
+                continue;   /* next PDPTE */
+            }
+
+            pShwPde      = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
+            GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
+
+            if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
+            {
+                AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
+                GCPtr += 512 * _2M;
+                cErrors++;
+                continue;
+            }
+
+            if (GCPhysPdeSrc != pShwPde->GCPhys)
+            {
+#   if PGM_GST_TYPE == PGM_TYPE_AMD64
+                AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
+#   else
+                AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
+#   endif
+                GCPtr += 512 * _2M;
+                cErrors++;
+                continue;
+            }
+
+#   if PGM_GST_TYPE == PGM_TYPE_AMD64
+            if (    pPdpeDst->lm.u1User      != PdpeSrc.lm.u1User
+                ||  pPdpeDst->lm.u1Write     != PdpeSrc.lm.u1Write
+                ||  pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
+            {
+                AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
+                GCPtr += 512 * _2M;
+                cErrors++;
+                continue;
+            }
+#   endif
+
+#  else  /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
+        {
+#  endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
+#  if PGM_GST_TYPE == PGM_TYPE_32BIT
+            GSTPD const    *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
+#   if PGM_SHW_TYPE == PGM_TYPE_32BIT
+            PCX86PD         pPDDst = pgmShwGet32BitPDPtr(pVCpu);
+#   endif
+#  endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
+            /*
+            * Iterate the shadow page directory.
+            */
+            GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
+            unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
+
+            for (;
+                iPDDst < cPDEs;
+                iPDDst++, GCPtr += cIncrement)
+            {
+#  if PGM_SHW_TYPE == PGM_TYPE_PAE
+                const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
+#  else
+                const SHWPDE PdeDst = pPDDst->a[iPDDst];
+#  endif
+                if (PdeDst.u & PGM_PDFLAGS_MAPPING)
+                {
+                    Assert(pgmMapAreMappingsEnabled(pVM));
+                    if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
+                    {
+                        AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
+                        cErrors++;
+                        continue;
+                    }
+                }
+                else if (   (PdeDst.u & X86_PDE_P)
+                        || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY))
+                        )
+                {
+                    HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
+                    PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
+                    if (!pPoolPage)
+                    {
+                        AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
+                                        HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
+                        cErrors++;
+                        continue;
+                    }
+                    const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
+
+                    if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD))
+                    {
+                        AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
+                                        GCPtr, (uint64_t)PdeDst.u));
+                        cErrors++;
+                    }
+
+                    if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D))
+                    {
+                        AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
+                                        GCPtr, (uint64_t)PdeDst.u));
+                        cErrors++;
+                    }
+
+                    const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
+                    if (!PdeSrc.n.u1Present)
+                    {
+                        AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
+                                        GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
+                        cErrors++;
+                        continue;
+                    }
+
+                    if (    !PdeSrc.b.u1Size
+                        ||  !fBigPagesSupported)
+                    {
+                        GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
+#  if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+                        GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | ((iPDDst & 1) * (PAGE_SIZE / 2)));
+#  endif
+                    }
+                    else
+                    {
+#  if PGM_GST_TYPE == PGM_TYPE_32BIT
+                        if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
+                        {
+                            AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
+                                            GCPtr, (uint64_t)PdeSrc.u));
+                            cErrors++;
+                            continue;
+                        }
+#  endif
+                        GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
+#  if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+                        GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
+#  endif
+                    }
+
+                    if (    pPoolPage->enmKind
+                        !=  (!PdeSrc.b.u1Size || !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
+                    {
+                        AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
+                                        pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
+                        cErrors++;
+                    }
+
+                    PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
+                    if (!pPhysPage)
+                    {
+                        AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
+                                        GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
+                        cErrors++;
+                        continue;
+                    }
+
+                    if (GCPhysGst != pPoolPage->GCPhys)
+                    {
+                        AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
+                                        GCPhysGst, pPoolPage->GCPhys, GCPtr));
+                        cErrors++;
+                        continue;
+                    }
+
+                    if (    !PdeSrc.b.u1Size
+                        ||  !fBigPagesSupported)
+                    {
+                        /*
+                        * Page Table.
+                        */
+                        const GSTPT *pPTSrc;
+                        rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1)),
+                                                 &pPTSrc);
+                        if (RT_FAILURE(rc))
+                        {
+                            AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
+                                            GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
+                            cErrors++;
+                            continue;
+                        }
+                        if (    (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))
+                            !=  (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)))
+                        {
+                            /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
+                            // (This problem will go away when/if we shadow multiple CR3s.)
+                            AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
+                                            GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                            cErrors++;
+                            continue;
+                        }
+                        if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
+                        {
+                            AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
+                                            GCPtr, (uint64_t)PdeDst.u));
+                            cErrors++;
+                            continue;
+                        }
+
+                        /* iterate the page table. */
+#  if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+                        /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
+                        const unsigned offPTSrc  = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
+#  else
+                        const unsigned offPTSrc  = 0;
+#  endif
+                        for (unsigned iPT = 0, off = 0;
+                            iPT < RT_ELEMENTS(pPTDst->a);
+                            iPT++, off += PAGE_SIZE)
+                        {
+                            const SHWPTE PteDst = pPTDst->a[iPT];
+
+                            /* skip not-present and dirty tracked entries. */
+                            if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
+                                continue;
+                            Assert(SHW_PTE_IS_P(PteDst));
+
+                            const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
+                            if (!PteSrc.n.u1Present)
+                            {
+#  ifdef IN_RING3
+                                PGMAssertHandlerAndFlagsInSync(pVM);
+                                DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 | DBGFPGDMP_FLAGS_CURRENT_MODE
+                                                   | DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_HEADER | DBGFPGDMP_FLAGS_PRINT_CR3,
+                                                   0, 0, UINT64_MAX, 99, NULL);
+#  endif
+                                AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
+                                                GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
+                                                (uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
+                                cErrors++;
+                                continue;
+                            }
+
+                            uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
+#  if 1 /** @todo sync accessed bit properly... */
+                            fIgnoreFlags |= X86_PTE_A;
+#  endif
+
+                            /* match the physical addresses */
+                            HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
+                            GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
+
+#  ifdef IN_RING3
+                            rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
+                            if (RT_FAILURE(rc))
+                            {
+                                if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
+                                {
+                                    AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                     GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                    cErrors++;
+                                    continue;
+                                }
+                            }
+                            else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
+                            {
+                                AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                                continue;
+                            }
+#  endif
+
+                            pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
+                            if (!pPhysPage)
+                            {
+#  ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
+                                if (HCPhysShw != MMR3PageDummyHCPhys(pVM))  /** @todo this is wrong. */
+                                {
+                                    AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                     GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                    cErrors++;
+                                    continue;
+                                }
+#  endif
+                                if (SHW_PTE_IS_RW(PteDst))
+                                {
+                                    AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                     GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                    cErrors++;
+                                }
+                                fIgnoreFlags |= X86_PTE_RW;
+                            }
+                            else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
+                            {
+                                AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                                continue;
+                            }
+
+                            /* flags */
+                            if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
+                            {
+                                if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
+                                {
+                                    if (SHW_PTE_IS_RW(PteDst))
+                                    {
+                                        AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                        continue;
+                                    }
+                                    fIgnoreFlags |= X86_PTE_RW;
+                                }
+                                else
+                                {
+                                    if (   SHW_PTE_IS_P(PteDst)
+#  if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
+                                        && !PGM_PAGE_IS_MMIO(pPhysPage)
+#  endif
+                                       )
+                                    {
+                                        AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                        continue;
+                                    }
+                                    fIgnoreFlags |= X86_PTE_P;
+                                }
+                            }
+                            else
+                            {
+                                if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
+                                {
+                                    if (SHW_PTE_IS_RW(PteDst))
+                                    {
+                                        AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                        continue;
+                                    }
+                                    if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
+                                    {
+                                        AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                        continue;
+                                    }
+                                    if (SHW_PTE_IS_D(PteDst))
+                                    {
+                                        AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                    }
+#  if 0 /** @todo sync access bit properly... */
+                                    if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
+                                    {
+                                        AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                        GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                    }
+                                    fIgnoreFlags |= X86_PTE_RW;
+#  else
+                                    fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
+#  endif
+                                }
+                                else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
+                                {
+                                    /* access bit emulation (not implemented). */
+                                    if (PteSrc.n.u1Accessed || SHW_PTE_IS_P(PteDst))
+                                    {
+                                        AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                        continue;
+                                    }
+                                    if (!SHW_PTE_IS_A(PteDst))
+                                    {
+                                        AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                    }
+                                    fIgnoreFlags |= X86_PTE_P;
+                                }
+#  ifdef DEBUG_sandervl
+                                fIgnoreFlags |= X86_PTE_D | X86_PTE_A;
+#  endif
+                            }
+
+                            if (    (PteSrc.u & ~fIgnoreFlags)                != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
+                                &&  (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
+                            )
+                            {
+                                AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                 GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
+                                                 fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                                continue;
+                            }
+                        } /* foreach PTE */
+                    }
+                    else
+                    {
+                        /*
+                        * Big Page.
+                        */
+                        uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
+                        if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
+                        {
+                            if (PdeDst.n.u1Write)
+                            {
+                                AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
+                                                GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                                cErrors++;
+                                continue;
+                            }
+                            if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
+                            {
+                                AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                                cErrors++;
+                                continue;
+                            }
+#  if 0 /** @todo sync access bit properly... */
+                            if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
+                            {
+                                AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                                cErrors++;
+                            }
+                            fIgnoreFlags |= X86_PTE_RW;
+#  else
+                            fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
+#  endif
+                        }
+                        else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
+                        {
+                            /* access bit emulation (not implemented). */
+                            if (PdeSrc.b.u1Accessed || PdeDst.n.u1Present)
+                            {
+                                AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
+                                                GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                                cErrors++;
+                                continue;
+                            }
+                            if (!PdeDst.n.u1Accessed)
+                            {
+                                AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
+                                                GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                                cErrors++;
+                            }
+                            fIgnoreFlags |= X86_PTE_P;
+                        }
+
+                        if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
+                        {
+                            AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
+                                            GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
+                                            fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                            cErrors++;
+                        }
+
+                        /* iterate the page table. */
+                        for (unsigned iPT = 0, off = 0;
+                            iPT < RT_ELEMENTS(pPTDst->a);
+                            iPT++, off += PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + PAGE_SIZE))
+                        {
+                            const SHWPTE PteDst = pPTDst->a[iPT];
+
+                            if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
+                            {
+                                AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                 GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                            }
+
+                            /* skip not-present entries. */
+                            if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
+                                continue;
+
+                            fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT | X86_PTE_D | X86_PTE_A | X86_PTE_G | X86_PTE_PAE_NX;
+
+                            /* match the physical addresses */
+                            HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
+
+#  ifdef IN_RING3
+                            rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
+                            if (RT_FAILURE(rc))
+                            {
+                                if (HCPhysShw != MMR3PageDummyHCPhys(pVM))  /** @todo this is wrong. */
+                                {
+                                    AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                     GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                    cErrors++;
+                                }
+                            }
+                            else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
+                            {
+                                AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                                continue;
+                            }
+#  endif
+                            pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
+                            if (!pPhysPage)
+                            {
+#  ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
+                                if (HCPhysShw != MMR3PageDummyHCPhys(pVM))  /** @todo this is wrong. */
+                                {
+                                    AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                     GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                    cErrors++;
+                                    continue;
+                                }
+#  endif
+                                if (SHW_PTE_IS_RW(PteDst))
+                                {
+                                    AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                     GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                    cErrors++;
+                                }
+                                fIgnoreFlags |= X86_PTE_RW;
+                            }
+                            else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
+                            {
+                                AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                                continue;
+                            }
+
+                            /* flags */
+                            if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
+                            {
+                                if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
+                                {
+                                    if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
+                                    {
+                                        if (SHW_PTE_IS_RW(PteDst))
+                                        {
+                                            AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                             GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                            cErrors++;
+                                            continue;
+                                        }
+                                        fIgnoreFlags |= X86_PTE_RW;
+                                    }
+                                }
+                                else
+                                {
+                                    if (   SHW_PTE_IS_P(PteDst)
+#  if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
+                                        && !PGM_PAGE_IS_MMIO(pPhysPage)
+#  endif
+                                        )
+                                    {
+                                        AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                        continue;
+                                    }
+                                    fIgnoreFlags |= X86_PTE_P;
+                                }
+                            }
+
+                            if (    (PdeSrc.u & ~fIgnoreFlags)                != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
+                                &&  (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
+                            )
+                            {
+                                AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                 GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
+                                                 fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                                continue;
+                            }
+                        } /* for each PTE */
+                    }
+                }
+                /* not present */
+
+            } /* for each PDE */
+
+        } /* for each PDPTE */
+
+    } /* for each PML4E */
+
+#  ifdef DEBUG
+    if (cErrors)
+        LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
+#  endif
+# endif /* GST is in {32BIT, PAE, AMD64} */
+    return cErrors;
+#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
+}
+#endif /* VBOX_STRICT */
+
+
+/**
+ * Sets up the CR3 for shadow paging
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhysCR3       The physical address in the CR3 register.  (A20
+ *                          mask already applied.)
+ */
+PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
+
+    /* Update guest paging info. */
+#if PGM_GST_TYPE == PGM_TYPE_32BIT \
+ || PGM_GST_TYPE == PGM_TYPE_PAE \
+ || PGM_GST_TYPE == PGM_TYPE_AMD64
+
+    LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
+    PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
+
+    /*
+     * Map the page CR3 points at.
+     */
+    RTHCPTR     HCPtrGuestCR3;
+    pgmLock(pVM);
+    PPGMPAGE    pPageCR3 = pgmPhysGetPage(pVM, GCPhysCR3);
+    AssertReturn(pPageCR3, VERR_PGM_INVALID_CR3_ADDR);
+    /** @todo this needs some reworking wrt. locking?  */
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    HCPtrGuestCR3 = NIL_RTHCPTR;
+    int rc = VINF_SUCCESS;
+# else
+    int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void **)&HCPtrGuestCR3); /** @todo r=bird: This GCPhysCR3 masking isn't necessary. */
+# endif
+    pgmUnlock(pVM);
+    if (RT_SUCCESS(rc))
+    {
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+#  ifdef VBOX_WITH_RAM_IN_KERNEL
+#   ifdef IN_RING3
+        pVCpu->pgm.s.pGst32BitPdR3 = (PX86PD)HCPtrGuestCR3;
+        pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
+#   else
+        pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR3PTR;
+        pVCpu->pgm.s.pGst32BitPdR0 = (PX86PD)HCPtrGuestCR3;
+#   endif
+#  else
+        pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
+#   ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+        pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
+#   endif
+#  endif
+
+# elif PGM_GST_TYPE == PGM_TYPE_PAE
+#  ifdef VBOX_WITH_RAM_IN_KERNEL
+#   ifdef IN_RING3
+        pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCR3;
+        pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
+#   else
+        pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
+        pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCR3;
+#   endif
+#  else
+        pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
+#   ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+        pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
+#   endif
+#  endif
+
+        /*
+         * Map the 4 PDs too.
+         */
+        PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
+        for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
+        {
+            pVCpu->pgm.s.aGstPaePdpeRegs[i].u = pGuestPDPT->a[i].u;
+            if (pGuestPDPT->a[i].n.u1Present)
+            {
+                RTHCPTR     HCPtr;
+                RTGCPHYS    GCPhys = PGM_A20_APPLY(pVCpu, pGuestPDPT->a[i].u & X86_PDPE_PG_MASK);
+                pgmLock(pVM);
+                PPGMPAGE    pPage  = pgmPhysGetPage(pVM, GCPhys);
+                AssertReturn(pPage, VERR_PGM_INVALID_PDPE_ADDR);
+#  ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+                HCPtr = NIL_RTHCPTR;
+                int rc2 = VINF_SUCCESS;
+#  else
+                int rc2 = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
+#  endif
+                pgmUnlock(pVM);
+                if (RT_SUCCESS(rc2))
+                {
+#  ifdef VBOX_WITH_RAM_IN_KERNEL
+#   ifdef IN_RING3
+                    pVCpu->pgm.s.apGstPaePDsR3[i]     = (PX86PDPAE)HCPtr;
+                    pVCpu->pgm.s.apGstPaePDsR0[i]     = NIL_RTR0PTR;
+#   else
+                    pVCpu->pgm.s.apGstPaePDsR3[i]     = NIL_RTR3PTR;
+                    pVCpu->pgm.s.apGstPaePDsR0[i]     = (PX86PDPAE)HCPtr;
+#   endif
+#  else
+                    pVCpu->pgm.s.apGstPaePDsR3[i]     = (R3PTRTYPE(PX86PDPAE))HCPtr;
+#   ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+                    pVCpu->pgm.s.apGstPaePDsR0[i]     = (R0PTRTYPE(PX86PDPAE))HCPtr;
+#   endif
+#  endif
+                    pVCpu->pgm.s.aGCPhysGstPaePDs[i]  = GCPhys;
+                    continue;
+                }
+                AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
+            }
+
+            pVCpu->pgm.s.apGstPaePDsR3[i]     = 0;
+#  ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+            pVCpu->pgm.s.apGstPaePDsR0[i]     = 0;
+#  endif
+            pVCpu->pgm.s.aGCPhysGstPaePDs[i]  = NIL_RTGCPHYS;
+        }
+
+# elif PGM_GST_TYPE == PGM_TYPE_AMD64
+#  ifdef VBOX_WITH_RAM_IN_KERNEL
+#   ifdef IN_RING3
+        pVCpu->pgm.s.pGstAmd64Pml4R3 = (PX86PML4)HCPtrGuestCR3;
+        pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
+#   else
+        pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
+        pVCpu->pgm.s.pGstAmd64Pml4R0 = (PX86PML4)HCPtrGuestCR3;
+#   endif
+#  else
+        pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
+#   ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+        pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
+#   endif
+#  endif
+# endif
+    }
+    else
+        AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
+
+#else /* prot/real stub */
+    int rc = VINF_SUCCESS;
+#endif
+
+    /*
+     * Update shadow paging info for guest modes with paging (32-bit, PAE, AMD64).
+     */
+# if  (   (   PGM_SHW_TYPE == PGM_TYPE_32BIT \
+           || PGM_SHW_TYPE == PGM_TYPE_PAE    \
+           || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
+       && (   PGM_GST_TYPE != PGM_TYPE_REAL   \
+           && PGM_GST_TYPE != PGM_TYPE_PROT))
+
+    Assert(!pVM->pgm.s.fNestedPaging);
+    PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
+
+    /*
+     * Update the shadow root page as well since that's not fixed.
+     */
+    PPGMPOOL     pPool             = pVM->pgm.s.CTX_SUFF(pPool);
+    PPGMPOOLPAGE pOldShwPageCR3    = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+    PPGMPOOLPAGE pNewShwPageCR3;
+
+    pgmLock(pVM);
+
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    if (pPool->cDirtyPages)
+        pgmPoolResetDirtyPages(pVM);
+# endif
+
+    Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
+    rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                      NIL_PGMPOOL_IDX, UINT32_MAX, true /*fLockPage*/,
+                      &pNewShwPageCR3);
+    AssertFatalRC(rc);
+    rc = VINF_SUCCESS;
+
+    pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
+#  ifdef IN_RING0
+    pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+#  else
+    pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+#  endif
+
+#  ifndef PGM_WITHOUT_MAPPINGS
+    /*
+     * Apply all hypervisor mappings to the new CR3.
+     * Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
+     * make sure we check for conflicts in the new CR3 root.
+     */
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+#   endif
+    rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
+    AssertRCReturn(rc, rc);
+#  endif
+
+    /* Set the current hypervisor CR3. */
+    CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
+
+    /* Clean up the old CR3 root. */
+    if (    pOldShwPageCR3
+        &&  pOldShwPageCR3 != pNewShwPageCR3    /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
+    {
+        Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
+#  ifndef PGM_WITHOUT_MAPPINGS
+        /* Remove the hypervisor mappings from the shadow page table. */
+        pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
+#  endif
+        /* Mark the page as unlocked; allow flushing again. */
+        pgmPoolUnlockPage(pPool, pOldShwPageCR3);
+
+        pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
+    }
+    pgmUnlock(pVM);
+# else
+    NOREF(GCPhysCR3);
+# endif
+
+    return rc;
+}
+
+/**
+ * Unmaps the shadow CR3.
+ *
+ * @returns VBox status, no specials.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu)
+{
+    LogFlow(("UnmapCR3\n"));
+
+    int   rc  = VINF_SUCCESS;
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
+
+    /*
+     * Update guest paging info.
+     */
+#if PGM_GST_TYPE == PGM_TYPE_32BIT
+    pVCpu->pgm.s.pGst32BitPdR3 = 0;
+# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    pVCpu->pgm.s.pGst32BitPdR0 = 0;
+# endif
+
+#elif PGM_GST_TYPE == PGM_TYPE_PAE
+    pVCpu->pgm.s.pGstPaePdptR3 = 0;
+# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    pVCpu->pgm.s.pGstPaePdptR0 = 0;
+# endif
+    for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
+    {
+        pVCpu->pgm.s.apGstPaePDsR3[i]    = 0;
+# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+        pVCpu->pgm.s.apGstPaePDsR0[i]    = 0;
+# endif
+        pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
+    }
+
+#elif PGM_GST_TYPE == PGM_TYPE_AMD64
+    pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
+# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
+# endif
+
+#else /* prot/real mode stub */
+    /* nothing to do */
+#endif
+
+    /*
+     * Update shadow paging info.
+     */
+#if  (   (   PGM_SHW_TYPE == PGM_TYPE_32BIT \
+          || PGM_SHW_TYPE == PGM_TYPE_PAE \
+          || PGM_SHW_TYPE == PGM_TYPE_AMD64))
+# if PGM_GST_TYPE != PGM_TYPE_REAL
+    Assert(!pVM->pgm.s.fNestedPaging);
+# endif
+    pgmLock(pVM);
+
+# ifndef PGM_WITHOUT_MAPPINGS
+    if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
+        /* Remove the hypervisor mappings from the shadow page table. */
+        pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+# endif
+
+    if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
+    {
+        PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+        if (pPool->cDirtyPages)
+            pgmPoolResetDirtyPages(pVM);
+# endif
+
+        /* Mark the page as unlocked; allow flushing again. */
+        pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+
+        pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
+        pVCpu->pgm.s.pShwPageCR3R3 = 0;
+        pVCpu->pgm.s.pShwPageCR3R0 = 0;
+    }
+
+    pgmUnlock(pVM);
+#endif
+
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMAll/PGMAllGst.h b/src/VBox/VMM/VMMAll/PGMAllGst.h
new file mode 100644
index 00000000..4c70e80c
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllGst.h
@@ -0,0 +1,521 @@
+/* $Id: PGMAllGst.h $ */
+/** @file
+ * VBox - Page Manager, Guest Paging Template - All context code.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+#if PGM_GST_TYPE == PGM_TYPE_32BIT \
+ || PGM_GST_TYPE == PGM_TYPE_PAE \
+ || PGM_GST_TYPE == PGM_TYPE_AMD64
+static int PGM_GST_NAME(Walk)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PGSTPTWALK pWalk);
+#endif
+PGM_GST_DECL(int,  GetPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys);
+PGM_GST_DECL(int,  ModifyPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask);
+PGM_GST_DECL(int,  GetPDE)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPDE);
+
+#ifdef IN_RING3 /* r3 only for now.  */
+PGM_GST_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
+PGM_GST_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
+PGM_GST_DECL(int, Exit)(PVMCPUCC pVCpu);
+#endif
+RT_C_DECLS_END
+
+
+/**
+ * Enters the guest mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPhysCR3   The physical address from the CR3 register.
+ */
+PGM_GST_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
+{
+    /*
+     * Map and monitor CR3
+     */
+    uintptr_t idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
+    return g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
+}
+
+
+/**
+ * Exits the guest mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+PGM_GST_DECL(int, Exit)(PVMCPUCC pVCpu)
+{
+    uintptr_t idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
+    return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
+}
+
+
+#if PGM_GST_TYPE == PGM_TYPE_32BIT \
+ || PGM_GST_TYPE == PGM_TYPE_PAE \
+ || PGM_GST_TYPE == PGM_TYPE_AMD64
+
+
+DECLINLINE(int) PGM_GST_NAME(WalkReturnNotPresent)(PVMCPUCC pVCpu, PGSTPTWALK pWalk, int iLevel)
+{
+    NOREF(iLevel); NOREF(pVCpu);
+    pWalk->Core.fNotPresent     = true;
+    pWalk->Core.uLevel          = (uint8_t)iLevel;
+    return VERR_PAGE_TABLE_NOT_PRESENT;
+}
+
+DECLINLINE(int) PGM_GST_NAME(WalkReturnBadPhysAddr)(PVMCPUCC pVCpu, PGSTPTWALK pWalk, int iLevel, int rc)
+{
+    AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc); NOREF(pVCpu);
+    pWalk->Core.fBadPhysAddr    = true;
+    pWalk->Core.uLevel          = (uint8_t)iLevel;
+    return VERR_PAGE_TABLE_NOT_PRESENT;
+}
+
+DECLINLINE(int) PGM_GST_NAME(WalkReturnRsvdError)(PVMCPUCC pVCpu, PGSTPTWALK pWalk, int iLevel)
+{
+    NOREF(pVCpu);
+    pWalk->Core.fRsvdError      = true;
+    pWalk->Core.uLevel          = (uint8_t)iLevel;
+    return VERR_PAGE_TABLE_NOT_PRESENT;
+}
+
+
+/**
+ * Performs a guest page table walk.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT on failure.  Check pWalk for details.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtr       The guest virtual address to walk by.
+ * @param   pWalk       Where to return the walk result. This is always set.
+ */
+DECLINLINE(int) PGM_GST_NAME(Walk)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PGSTPTWALK pWalk)
+{
+    int rc;
+
+    /*
+     * Init the walking structure.
+     */
+    RT_ZERO(*pWalk);
+    pWalk->Core.GCPtr = GCPtr;
+
+# if PGM_GST_TYPE == PGM_TYPE_32BIT \
+  || PGM_GST_TYPE == PGM_TYPE_PAE
+    /*
+     * Boundary check for PAE and 32-bit (prevents trouble further down).
+     */
+    if (RT_UNLIKELY(GCPtr >= _4G))
+        return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 8);
+# endif
+
+    uint32_t register fEffective = X86_PTE_RW | X86_PTE_US | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_A | 1;
+    {
+# if PGM_GST_TYPE == PGM_TYPE_AMD64
+        /*
+         * The PMLE4.
+         */
+        rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pWalk->pPml4);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 4, rc);
+
+        PX86PML4E register  pPml4e;
+        pWalk->pPml4e  = pPml4e  = &pWalk->pPml4->a[(GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK];
+        X86PML4E  register  Pml4e;
+        pWalk->Pml4e.u = Pml4e.u = pPml4e->u;
+
+        if (Pml4e.n.u1Present) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 4);
+
+        if (RT_LIKELY(GST_IS_PML4E_VALID(pVCpu, Pml4e))) { /* likely */ }
+        else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 4);
+
+        pWalk->Core.fEffective = fEffective = ((uint32_t)Pml4e.u & (X86_PML4E_RW  | X86_PML4E_US | X86_PML4E_PWT | X86_PML4E_PCD | X86_PML4E_A))
+                                            | ((uint32_t)(Pml4e.u >> 63) ^ 1) /*NX */;
+
+        /*
+         * The PDPE.
+         */
+        rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, Pml4e.u & X86_PML4E_PG_MASK, &pWalk->pPdpt);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 3, rc);
+
+# elif PGM_GST_TYPE == PGM_TYPE_PAE
+        rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pWalk->pPdpt);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
+# endif
+    }
+    {
+# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
+        PX86PDPE register   pPdpe;
+        pWalk->pPdpe  = pPdpe  = &pWalk->pPdpt->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
+        X86PDPE  register   Pdpe;
+        pWalk->Pdpe.u = Pdpe.u = pPdpe->u;
+
+        if (Pdpe.n.u1Present) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 3);
+
+        if (RT_LIKELY(GST_IS_PDPE_VALID(pVCpu, Pdpe))) { /* likely */ }
+        else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 3);
+
+# if PGM_GST_TYPE == PGM_TYPE_AMD64
+        pWalk->Core.fEffective = fEffective &= ((uint32_t)Pdpe.u & (X86_PDPE_RW  | X86_PDPE_US | X86_PDPE_PWT | X86_PDPE_PCD | X86_PDPE_A))
+                                             | ((uint32_t)(Pdpe.u >> 63) ^ 1) /*NX */;
+# else
+        pWalk->Core.fEffective = fEffective  = X86_PDPE_RW  | X86_PDPE_US | X86_PDPE_A
+                                             | ((uint32_t)Pdpe.u & (X86_PDPE_PWT | X86_PDPE_PCD))
+                                             | ((uint32_t)(Pdpe.u >> 63) ^ 1) /*NX */;
+# endif
+
+        /*
+         * The PDE.
+         */
+        rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, Pdpe.u & X86_PDPE_PG_MASK, &pWalk->pPd);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 2, rc);
+# elif PGM_GST_TYPE == PGM_TYPE_32BIT
+        rc = pgmGstGet32bitPDPtrEx(pVCpu, &pWalk->pPd);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
+# endif
+    }
+    {
+        PGSTPDE register    pPde;
+        pWalk->pPde  = pPde  = &pWalk->pPd->a[(GCPtr >> GST_PD_SHIFT) & GST_PD_MASK];
+        GSTPDE              Pde;
+        pWalk->Pde.u = Pde.u = pPde->u;
+        if (Pde.n.u1Present) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 2);
+        if (Pde.n.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
+        {
+            if (RT_LIKELY(GST_IS_BIG_PDE_VALID(pVCpu, Pde))) { /* likely */ }
+            else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2);
+
+            /*
+             * We're done.
+             */
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+            fEffective &= Pde.u & (X86_PDE4M_RW  | X86_PDE4M_US | X86_PDE4M_PWT | X86_PDE4M_PCD | X86_PDE4M_A);
+# else
+            fEffective &= ((uint32_t)Pde.u & (X86_PDE4M_RW  | X86_PDE4M_US | X86_PDE4M_PWT | X86_PDE4M_PCD | X86_PDE4M_A))
+                        | ((uint32_t)(Pde.u >> 63) ^ 1) /*NX */;
+# endif
+            fEffective |= (uint32_t)Pde.u & (X86_PDE4M_D | X86_PDE4M_G);
+            fEffective |= (uint32_t)(Pde.u & X86_PDE4M_PAT) >> X86_PDE4M_PAT_SHIFT;
+            pWalk->Core.fEffective = fEffective;
+
+            pWalk->Core.fEffectiveRW = !!(fEffective & X86_PTE_RW);
+            pWalk->Core.fEffectiveUS = !!(fEffective & X86_PTE_US);
+# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
+            pWalk->Core.fEffectiveNX = !(fEffective & 1) && GST_IS_NX_ACTIVE(pVCpu);
+# else
+            pWalk->Core.fEffectiveNX = false;
+# endif
+            pWalk->Core.fBigPage     = true;
+            pWalk->Core.fSucceeded   = true;
+
+            pWalk->Core.GCPhys       = GST_GET_BIG_PDE_GCPHYS(pVCpu->CTX_SUFF(pVM), Pde)
+                                     | (GCPtr & GST_BIG_PAGE_OFFSET_MASK);
+            PGM_A20_APPLY_TO_VAR(pVCpu, pWalk->Core.GCPhys);
+            return VINF_SUCCESS;
+        }
+
+        if (RT_UNLIKELY(!GST_IS_PDE_VALID(pVCpu, Pde)))
+            return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2);
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+        pWalk->Core.fEffective = fEffective &= Pde.u & (X86_PDE_RW  | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD | X86_PDE_A);
+# else
+        pWalk->Core.fEffective = fEffective &= ((uint32_t)Pde.u & (X86_PDE_RW  | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD | X86_PDE_A))
+                                             | ((uint32_t)(Pde.u >> 63) ^ 1) /*NX */;
+# endif
+
+        /*
+         * The PTE.
+         */
+        rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GST_GET_PDE_GCPHYS(Pde), &pWalk->pPt);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 1, rc);
+    }
+    {
+        PGSTPTE register    pPte;
+        pWalk->pPte  = pPte  = &pWalk->pPt->a[(GCPtr >> GST_PT_SHIFT) & GST_PT_MASK];
+        GSTPTE  register    Pte;
+        pWalk->Pte.u = Pte.u = pPte->u;
+
+        if (Pte.n.u1Present) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 1);
+
+        if (RT_LIKELY(GST_IS_PTE_VALID(pVCpu, Pte))) { /* likely */ }
+        else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 1);
+
+        /*
+         * We're done.
+         */
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+        fEffective &= Pte.u & (X86_PTE_RW  | X86_PTE_US | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_A);
+# else
+        fEffective &= ((uint32_t)Pte.u & (X86_PTE_RW  | X86_PTE_US | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_A))
+                    | ((uint32_t)(Pte.u >> 63) ^ 1) /*NX */;
+# endif
+        fEffective |= (uint32_t)Pte.u & (X86_PTE_D | X86_PTE_PAT | X86_PTE_G);
+        pWalk->Core.fEffective = fEffective;
+
+        pWalk->Core.fEffectiveRW = !!(fEffective & X86_PTE_RW);
+        pWalk->Core.fEffectiveUS = !!(fEffective & X86_PTE_US);
+# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
+        pWalk->Core.fEffectiveNX = !(fEffective & 1) && GST_IS_NX_ACTIVE(pVCpu);
+# else
+        pWalk->Core.fEffectiveNX = false;
+# endif
+        pWalk->Core.fSucceeded   = true;
+
+        pWalk->Core.GCPhys       = GST_GET_PDE_GCPHYS(Pte)
+                                 | (GCPtr & PAGE_OFFSET_MASK);
+        return VINF_SUCCESS;
+    }
+}
+
+#endif /* 32BIT, PAE, AMD64 */
+
+/**
+ * Gets effective Guest OS page information.
+ *
+ * When GCPtr is in a big page, the function will return as if it was a normal
+ * 4KB page. If the need for distinguishing between big and normal page becomes
+ * necessary at a later point, a PGMGstGetPage Ex() will be created for that
+ * purpose.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Guest Context virtual address of the page.
+ * @param   pfFlags     Where to store the flags. These are X86_PTE_*, even for big pages.
+ * @param   pGCPhys     Where to store the GC physical address of the page.
+ *                      This is page aligned!
+ */
+PGM_GST_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys)
+{
+#if PGM_GST_TYPE == PGM_TYPE_REAL \
+ || PGM_GST_TYPE == PGM_TYPE_PROT
+    /*
+     * Fake it.
+     */
+    if (pfFlags)
+        *pfFlags = X86_PTE_P | X86_PTE_RW | X86_PTE_US;
+    if (pGCPhys)
+        *pGCPhys = GCPtr & PAGE_BASE_GC_MASK;
+    NOREF(pVCpu);
+    return VINF_SUCCESS;
+
+#elif PGM_GST_TYPE == PGM_TYPE_32BIT \
+   || PGM_GST_TYPE == PGM_TYPE_PAE \
+   || PGM_GST_TYPE == PGM_TYPE_AMD64
+
+    GSTPTWALK Walk;
+    int rc = PGM_GST_NAME(Walk)(pVCpu, GCPtr, &Walk);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    if (pGCPhys)
+        *pGCPhys = Walk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+
+    if (pfFlags)
+    {
+        if (!Walk.Core.fBigPage)
+            *pfFlags = (Walk.Pte.u & ~(GST_PTE_PG_MASK | X86_PTE_RW | X86_PTE_US))                      /* NX not needed */
+                     | (Walk.Core.fEffectiveRW ? X86_PTE_RW : 0)
+                     | (Walk.Core.fEffectiveUS ? X86_PTE_US : 0)
+# if PGM_WITH_NX(PGM_GST_TYPE, PGM_GST_TYPE)
+                     | (Walk.Core.fEffectiveNX ? X86_PTE_PAE_NX : 0)
+# endif
+                     ;
+        else
+        {
+            *pfFlags = (Walk.Pde.u & ~(GST_PTE_PG_MASK | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_PS))   /* NX not needed */
+                     | ((Walk.Pde.u & X86_PDE4M_PAT) >> X86_PDE4M_PAT_SHIFT)
+                     | (Walk.Core.fEffectiveRW ? X86_PTE_RW : 0)
+                     | (Walk.Core.fEffectiveUS ? X86_PTE_US : 0)
+# if PGM_WITH_NX(PGM_GST_TYPE, PGM_GST_TYPE)
+                     | (Walk.Core.fEffectiveNX ? X86_PTE_PAE_NX : 0)
+# endif
+                     ;
+        }
+    }
+
+    return VINF_SUCCESS;
+
+#else
+# error "shouldn't be here!"
+    /* something else... */
+    return VERR_NOT_SUPPORTED;
+#endif
+}
+
+
+/**
+ * Modify page flags for a range of pages in the guest's tables
+ *
+ * The existing flags are ANDed with the fMask and ORed with the fFlags.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range. Page aligned!
+ * @param   cb          Size (in bytes) of the page range to apply the modification to. Page aligned!
+ * @param   fFlags      The OR  mask - page flags X86_PTE_*, excluding the page mask of course.
+ * @param   fMask       The AND mask - page flags X86_PTE_*.
+ */
+PGM_GST_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
+{
+    Assert((cb & PAGE_OFFSET_MASK) == 0); RT_NOREF_PV(cb);
+
+#if PGM_GST_TYPE == PGM_TYPE_32BIT \
+ || PGM_GST_TYPE == PGM_TYPE_PAE \
+ || PGM_GST_TYPE == PGM_TYPE_AMD64
+    for (;;)
+    {
+        GSTPTWALK Walk;
+        int rc = PGM_GST_NAME(Walk)(pVCpu, GCPtr, &Walk);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        if (!Walk.Core.fBigPage)
+        {
+            /*
+             * 4KB Page table, process
+             *
+             * Walk pages till we're done.
+             */
+            unsigned iPTE = (GCPtr >> GST_PT_SHIFT) & GST_PT_MASK;
+            while (iPTE < RT_ELEMENTS(Walk.pPt->a))
+            {
+                GSTPTE Pte = Walk.pPt->a[iPTE];
+                Pte.u = (Pte.u & (fMask | X86_PTE_PAE_PG_MASK))
+                      | (fFlags & ~GST_PTE_PG_MASK);
+                Walk.pPt->a[iPTE] = Pte;
+
+                /* next page */
+                cb -= PAGE_SIZE;
+                if (!cb)
+                    return VINF_SUCCESS;
+                GCPtr += PAGE_SIZE;
+                iPTE++;
+            }
+        }
+        else
+        {
+            /*
+             * 2/4MB Page table
+             */
+            GSTPDE PdeNew;
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+            PdeNew.u = (Walk.Pde.u & (fMask | ((fMask & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT) | GST_PDE_BIG_PG_MASK | X86_PDE4M_PG_HIGH_MASK | X86_PDE4M_PS))
+# else
+            PdeNew.u = (Walk.Pde.u & (fMask | ((fMask & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT) | GST_PDE_BIG_PG_MASK | X86_PDE4M_PS))
+# endif
+                     | (fFlags & ~GST_PTE_PG_MASK)
+                     | ((fFlags & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT);
+            *Walk.pPde = PdeNew;
+
+            /* advance */
+            const unsigned cbDone = GST_BIG_PAGE_SIZE - (GCPtr & GST_BIG_PAGE_OFFSET_MASK);
+            if (cbDone >= cb)
+                return VINF_SUCCESS;
+            cb    -= cbDone;
+            GCPtr += cbDone;
+        }
+    }
+
+#else
+    /* real / protected mode: ignore. */
+    NOREF(pVCpu); NOREF(GCPtr); NOREF(fFlags); NOREF(fMask);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/**
+ * Retrieve guest PDE information.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Guest context pointer.
+ * @param   pPDE        Pointer to guest PDE structure.
+ */
+PGM_GST_DECL(int, GetPDE)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPDE)
+{
+#if PGM_GST_TYPE == PGM_TYPE_32BIT \
+ || PGM_GST_TYPE == PGM_TYPE_PAE   \
+ || PGM_GST_TYPE == PGM_TYPE_AMD64
+
+# if PGM_GST_TYPE != PGM_TYPE_AMD64
+    /* Boundary check. */
+    if (RT_UNLIKELY(GCPtr >= _4G))
+        return VERR_PAGE_TABLE_NOT_PRESENT;
+# endif
+
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+    unsigned    iPd = (GCPtr >> GST_PD_SHIFT) & GST_PD_MASK;
+    PX86PD      pPd = pgmGstGet32bitPDPtr(pVCpu);
+
+# elif PGM_GST_TYPE == PGM_TYPE_PAE
+    unsigned    iPd = 0;                /* shut up gcc */
+    PCX86PDPAE  pPd = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPd, NULL);
+
+# elif PGM_GST_TYPE == PGM_TYPE_AMD64
+    PX86PML4E   pPml4eIgn;
+    X86PDPE     PdpeIgn;
+    unsigned    iPd = 0;                /* shut up gcc */
+    PCX86PDPAE  pPd = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eIgn, &PdpeIgn, &iPd);
+    /* Note! We do not return an effective PDE here like we do for the PTE in GetPage method. */
+# endif
+
+    if (RT_LIKELY(pPd))
+        pPDE->u = (X86PGPAEUINT)pPd->a[iPd].u;
+    else
+        pPDE->u = 0;
+    return VINF_SUCCESS;
+
+#else
+    NOREF(pVCpu); NOREF(GCPtr); NOREF(pPDE);
+    AssertFailed();
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+#ifdef IN_RING3
+/**
+ * Relocate any GC pointers related to guest mode paging.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offDelta    The relocation offset.
+ */
+PGM_GST_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta)
+{
+    RT_NOREF(pVCpu, offDelta);
+    return VINF_SUCCESS;
+}
+#endif
diff --git a/src/VBox/VMM/VMMAll/PGMAllHandler.cpp b/src/VBox/VMM/VMMAll/PGMAllHandler.cpp
new file mode 100644
index 00000000..06d02bc5
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllHandler.cpp
@@ -0,0 +1,1768 @@
+/* $Id: PGMAllHandler.cpp $ */
+/** @file
+ * PGM - Page Manager / Monitor, Access Handlers.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/dbgf.h>
+#ifdef IN_RING0
+# include <VBox/vmm/pdmdev.h>
+#endif
+#include "PGMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include "PGMInline.h"
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/string.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/vmm/selm.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int  pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(PVMCC pVM, PPGMPHYSHANDLER pCur, PPGMRAMRANGE pRam);
+static void pgmHandlerPhysicalDeregisterNotifyREMAndNEM(PVMCC pVM, PPGMPHYSHANDLER pCur, int fRestoreRAM);
+static void pgmHandlerPhysicalResetRamFlags(PVMCC pVM, PPGMPHYSHANDLER pCur);
+
+
+/**
+ * Internal worker for releasing a physical handler type registration reference.
+ *
+ * @returns New reference count. UINT32_MAX if invalid input (asserted).
+ * @param   pVM         The cross context VM structure.
+ * @param   pType       Pointer to the type registration.
+ */
+DECLINLINE(uint32_t) pgmHandlerPhysicalTypeRelease(PVMCC pVM, PPGMPHYSHANDLERTYPEINT pType)
+{
+    AssertMsgReturn(pType->u32Magic == PGMPHYSHANDLERTYPEINT_MAGIC, ("%#x\n", pType->u32Magic), UINT32_MAX);
+    uint32_t cRefs = ASMAtomicDecU32(&pType->cRefs);
+    if (cRefs == 0)
+    {
+        pgmLock(pVM);
+        pType->u32Magic = PGMPHYSHANDLERTYPEINT_MAGIC_DEAD;
+        RTListOff32NodeRemove(&pType->ListNode);
+        pgmUnlock(pVM);
+        MMHyperFree(pVM, pType);
+    }
+    return cRefs;
+}
+
+
+/**
+ * Internal worker for retaining a physical handler type registration reference.
+ *
+ * @returns New reference count. UINT32_MAX if invalid input (asserted).
+ * @param   pVM         The cross context VM structure.
+ * @param   pType       Pointer to the type registration.
+ */
+DECLINLINE(uint32_t) pgmHandlerPhysicalTypeRetain(PVM pVM, PPGMPHYSHANDLERTYPEINT pType)
+{
+    NOREF(pVM);
+    AssertMsgReturn(pType->u32Magic == PGMPHYSHANDLERTYPEINT_MAGIC, ("%#x\n", pType->u32Magic), UINT32_MAX);
+    uint32_t cRefs = ASMAtomicIncU32(&pType->cRefs);
+    Assert(cRefs < _1M && cRefs > 0);
+    return cRefs;
+}
+
+
+/**
+ * Releases a reference to a physical handler type registration.
+ *
+ * @returns New reference count. UINT32_MAX if invalid input (asserted).
+ * @param   pVM         The cross context VM structure.
+ * @param   hType       The type regiration handle.
+ */
+VMMDECL(uint32_t) PGMHandlerPhysicalTypeRelease(PVMCC pVM, PGMPHYSHANDLERTYPE hType)
+{
+    if (hType != NIL_PGMPHYSHANDLERTYPE)
+        return pgmHandlerPhysicalTypeRelease(pVM, PGMPHYSHANDLERTYPEINT_FROM_HANDLE(pVM, hType));
+    return 0;
+}
+
+
+/**
+ * Retains a reference to a physical handler type registration.
+ *
+ * @returns New reference count. UINT32_MAX if invalid input (asserted).
+ * @param   pVM         The cross context VM structure.
+ * @param   hType       The type regiration handle.
+ */
+VMMDECL(uint32_t) PGMHandlerPhysicalTypeRetain(PVM pVM, PGMPHYSHANDLERTYPE hType)
+{
+    return pgmHandlerPhysicalTypeRetain(pVM, PGMPHYSHANDLERTYPEINT_FROM_HANDLE(pVM, hType));
+}
+
+
+/**
+ * Creates a physical access handler.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS when successfully installed.
+ * @retval  VINF_PGM_GCPHYS_ALIASED when the shadow PTs could be updated because
+ *          the guest page aliased or/and mapped by multiple PTs. A CR3 sync has been
+ *          flagged together with a pool clearing.
+ * @retval  VERR_PGM_HANDLER_PHYSICAL_CONFLICT if the range conflicts with an existing
+ *          one. A debug assertion is raised.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   hType           The handler type registration handle.
+ * @param   pvUserR3        User argument to the R3 handler.
+ * @param   pvUserR0        User argument to the R0 handler.
+ * @param   pvUserRC        User argument to the RC handler. This can be a value
+ *                          less that 0x10000 or a (non-null) pointer that is
+ *                          automatically relocated.
+ * @param   pszDesc         Description of this handler.  If NULL, the type
+ *                          description will be used instead.
+ * @param   ppPhysHandler   Where to return the access handler structure on
+ *                          success.
+ */
+int pgmHandlerPhysicalExCreate(PVMCC pVM, PGMPHYSHANDLERTYPE hType, RTR3PTR pvUserR3, RTR0PTR pvUserR0, RTRCPTR pvUserRC,
+                               R3PTRTYPE(const char *) pszDesc, PPGMPHYSHANDLER *ppPhysHandler)
+{
+    PPGMPHYSHANDLERTYPEINT pType = PGMPHYSHANDLERTYPEINT_FROM_HANDLE(pVM, hType);
+    Log(("pgmHandlerPhysicalExCreate: pvUserR3=%RHv pvUserR0=%RHv pvUserGC=%RRv hType=%#x (%d, %s) pszDesc=%RHv:%s\n",
+         pvUserR3, pvUserR0, pvUserRC, hType, pType->enmKind, R3STRING(pType->pszDesc), pszDesc, R3STRING(pszDesc)));
+
+    /*
+     * Validate input.
+     */
+    AssertPtr(ppPhysHandler);
+    AssertReturn(pType->u32Magic == PGMPHYSHANDLERTYPEINT_MAGIC, VERR_INVALID_HANDLE);
+    AssertMsgReturn(    (RTRCUINTPTR)pvUserRC < 0x10000
+                    ||  MMHyperR3ToRC(pVM, MMHyperRCToR3(pVM, pvUserRC)) == pvUserRC,
+                    ("Not RC pointer! pvUserRC=%RRv\n", pvUserRC),
+                    VERR_INVALID_PARAMETER);
+#if 0 /* No longer valid. */
+    AssertMsgReturn(    (RTR0UINTPTR)pvUserR0 < 0x10000
+                    ||  MMHyperR3ToR0(pVM, MMHyperR0ToR3(pVM, pvUserR0)) == pvUserR0,
+                    ("Not R0 pointer! pvUserR0=%RHv\n", pvUserR0),
+                    VERR_INVALID_PARAMETER);
+#endif
+
+    /*
+     * Allocate and initialize the new entry.
+     */
+    PPGMPHYSHANDLER pNew;
+    int rc = MMHyperAlloc(pVM, sizeof(*pNew), 0, MM_TAG_PGM_HANDLERS, (void **)&pNew);
+    if (RT_SUCCESS(rc))
+    {
+        pNew->Core.Key      = NIL_RTGCPHYS;
+        pNew->Core.KeyLast  = NIL_RTGCPHYS;
+        pNew->cPages        = 0;
+        pNew->cAliasedPages = 0;
+        pNew->cTmpOffPages  = 0;
+        pNew->pvUserR3      = pvUserR3;
+        pNew->pvUserR0      = pvUserR0;
+        pNew->hType         = hType;
+        pNew->pszDesc       = pszDesc != NIL_RTR3PTR ? pszDesc : pType->pszDesc;
+        pgmHandlerPhysicalTypeRetain(pVM, pType);
+        *ppPhysHandler = pNew;
+        return VINF_SUCCESS;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Duplicates a physical access handler.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS when successfully installed.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pPhysHandlerSrc The source handler to duplicate
+ * @param   ppPhysHandler   Where to return the access handler structure on
+ *                          success.
+ */
+int pgmHandlerPhysicalExDup(PVMCC pVM, PPGMPHYSHANDLER pPhysHandlerSrc, PPGMPHYSHANDLER *ppPhysHandler)
+{
+    return pgmHandlerPhysicalExCreate(pVM,
+                                      pPhysHandlerSrc->hType,
+                                      pPhysHandlerSrc->pvUserR3,
+                                      pPhysHandlerSrc->pvUserR0,
+                                      NIL_RTR0PTR,
+                                      pPhysHandlerSrc->pszDesc,
+                                      ppPhysHandler);
+}
+
+
+/**
+ * Register a access handler for a physical range.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS when successfully installed.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pPhysHandler    The physical handler.
+ * @param   GCPhys          Start physical address.
+ * @param   GCPhysLast      Last physical address. (inclusive)
+ */
+int pgmHandlerPhysicalExRegister(PVMCC pVM, PPGMPHYSHANDLER pPhysHandler, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtr(pPhysHandler);
+    PPGMPHYSHANDLERTYPEINT pType = PGMPHYSHANDLERTYPEINT_FROM_HANDLE(pVM, pPhysHandler->hType);
+    Assert(pType->u32Magic == PGMPHYSHANDLERTYPEINT_MAGIC);
+    Log(("pgmHandlerPhysicalExRegister: GCPhys=%RGp GCPhysLast=%RGp hType=%#x (%d, %s) pszDesc=%RHv:%s\n",
+         GCPhys, GCPhysLast, pPhysHandler->hType, pType->enmKind, R3STRING(pType->pszDesc), pPhysHandler->pszDesc, R3STRING(pPhysHandler->pszDesc)));
+    AssertReturn(pPhysHandler->Core.Key == NIL_RTGCPHYS, VERR_WRONG_ORDER);
+
+    AssertMsgReturn(GCPhys < GCPhysLast, ("GCPhys >= GCPhysLast (%#x >= %#x)\n", GCPhys, GCPhysLast), VERR_INVALID_PARAMETER);
+    switch (pType->enmKind)
+    {
+        case PGMPHYSHANDLERKIND_WRITE:
+            break;
+        case PGMPHYSHANDLERKIND_MMIO:
+        case PGMPHYSHANDLERKIND_ALL:
+            /* Simplification for PGMPhysRead, PGMR0Trap0eHandlerNPMisconfig and others: Full pages. */
+            AssertMsgReturn(!(GCPhys & PAGE_OFFSET_MASK), ("%RGp\n", GCPhys), VERR_INVALID_PARAMETER);
+            AssertMsgReturn((GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK, ("%RGp\n", GCPhysLast), VERR_INVALID_PARAMETER);
+            break;
+        default:
+            AssertMsgFailed(("Invalid input enmKind=%d!\n", pType->enmKind));
+            return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * We require the range to be within registered ram.
+     * There is no apparent need to support ranges which cover more than one ram range.
+     */
+    PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
+    if (   !pRam
+        || GCPhysLast > pRam->GCPhysLast)
+    {
+#ifdef IN_RING3
+        DBGFR3Info(pVM->pUVM, "phys", NULL, NULL);
+#endif
+        AssertMsgFailed(("No RAM range for %RGp-%RGp\n", GCPhys, GCPhysLast));
+        return VERR_PGM_HANDLER_PHYSICAL_NO_RAM_RANGE;
+    }
+    Assert(GCPhys >= pRam->GCPhys && GCPhys < pRam->GCPhysLast);
+    Assert(GCPhysLast <= pRam->GCPhysLast && GCPhysLast >= pRam->GCPhys);
+
+    /*
+     * Try insert into list.
+     */
+    pPhysHandler->Core.Key     = GCPhys;
+    pPhysHandler->Core.KeyLast = GCPhysLast;
+    pPhysHandler->cPages       = (GCPhysLast - (GCPhys & X86_PTE_PAE_PG_MASK) + PAGE_SIZE) >> PAGE_SHIFT;
+
+    pgmLock(pVM);
+    if (RTAvlroGCPhysInsert(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, &pPhysHandler->Core))
+    {
+        int rc = pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(pVM, pPhysHandler, pRam);
+        if (rc == VINF_PGM_SYNC_CR3)
+            rc = VINF_PGM_GCPHYS_ALIASED;
+
+#if defined(IN_RING3) || defined(IN_RING0)
+        NEMHCNotifyHandlerPhysicalRegister(pVM, pType->enmKind, GCPhys, GCPhysLast - GCPhys + 1);
+#endif
+        pgmUnlock(pVM);
+
+        if (rc != VINF_SUCCESS)
+            Log(("PGMHandlerPhysicalRegisterEx: returns %Rrc (%RGp-%RGp)\n", rc, GCPhys, GCPhysLast));
+        return rc;
+    }
+    pgmUnlock(pVM);
+
+    pPhysHandler->Core.Key     = NIL_RTGCPHYS;
+    pPhysHandler->Core.KeyLast = NIL_RTGCPHYS;
+
+#if defined(IN_RING3) && defined(VBOX_STRICT)
+    DBGFR3Info(pVM->pUVM, "handlers", "phys nostats", NULL);
+#endif
+    AssertMsgFailed(("Conflict! GCPhys=%RGp GCPhysLast=%RGp pszDesc=%s/%s\n",
+                     GCPhys, GCPhysLast, R3STRING(pPhysHandler->pszDesc), R3STRING(pType->pszDesc)));
+    return VERR_PGM_HANDLER_PHYSICAL_CONFLICT;
+}
+
+
+/**
+ * Register a access handler for a physical range.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS when successfully installed.
+ * @retval  VINF_PGM_GCPHYS_ALIASED when the shadow PTs could be updated because
+ *          the guest page aliased or/and mapped by multiple PTs. A CR3 sync has been
+ *          flagged together with a pool clearing.
+ * @retval  VERR_PGM_HANDLER_PHYSICAL_CONFLICT if the range conflicts with an existing
+ *          one. A debug assertion is raised.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Start physical address.
+ * @param   GCPhysLast      Last physical address. (inclusive)
+ * @param   hType           The handler type registration handle.
+ * @param   pvUserR3        User argument to the R3 handler.
+ * @param   pvUserR0        User argument to the R0 handler.
+ * @param   pvUserRC        User argument to the RC handler. This can be a value
+ *                          less that 0x10000 or a (non-null) pointer that is
+ *                          automatically relocated.
+ * @param   pszDesc         Description of this handler.  If NULL, the type
+ *                          description will be used instead.
+ */
+VMMDECL(int) PGMHandlerPhysicalRegister(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, PGMPHYSHANDLERTYPE hType,
+                                        RTR3PTR pvUserR3, RTR0PTR pvUserR0, RTRCPTR pvUserRC, R3PTRTYPE(const char *) pszDesc)
+{
+#ifdef LOG_ENABLED
+    PPGMPHYSHANDLERTYPEINT pType = PGMPHYSHANDLERTYPEINT_FROM_HANDLE(pVM, hType);
+    Log(("PGMHandlerPhysicalRegister: GCPhys=%RGp GCPhysLast=%RGp pvUserR3=%RHv pvUserR0=%RHv pvUserGC=%RRv hType=%#x (%d, %s) pszDesc=%RHv:%s\n",
+         GCPhys, GCPhysLast, pvUserR3, pvUserR0, pvUserRC, hType, pType->enmKind, R3STRING(pType->pszDesc), pszDesc, R3STRING(pszDesc)));
+#endif
+
+    PPGMPHYSHANDLER pNew;
+    int rc = pgmHandlerPhysicalExCreate(pVM, hType, pvUserR3, pvUserR0, pvUserRC, pszDesc, &pNew);
+    if (RT_SUCCESS(rc))
+    {
+        rc = pgmHandlerPhysicalExRegister(pVM, pNew, GCPhys, GCPhysLast);
+        if (RT_SUCCESS(rc))
+            return rc;
+        pgmHandlerPhysicalExDestroy(pVM, pNew);
+    }
+    return rc;
+}
+
+
+/**
+ * Sets ram range flags and attempts updating shadow PTs.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS when shadow PTs was successfully updated.
+ * @retval  VINF_PGM_SYNC_CR3 when the shadow PTs could be updated because
+ *          the guest page aliased or/and mapped by multiple PTs. FFs set.
+ * @param   pVM     The cross context VM structure.
+ * @param   pCur    The physical handler.
+ * @param   pRam    The RAM range.
+ */
+static int pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(PVMCC pVM, PPGMPHYSHANDLER pCur, PPGMRAMRANGE pRam)
+{
+    /*
+     * Iterate the guest ram pages updating the flags and flushing PT entries
+     * mapping the page.
+     */
+    bool                    fFlushTLBs = false;
+    int                     rc         = VINF_SUCCESS;
+    PPGMPHYSHANDLERTYPEINT  pCurType   = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+    const unsigned          uState     = pCurType->uState;
+    uint32_t                cPages     = pCur->cPages;
+    uint32_t                i          = (pCur->Core.Key - pRam->GCPhys) >> PAGE_SHIFT;
+    for (;;)
+    {
+        PPGMPAGE pPage = &pRam->aPages[i];
+        AssertMsg(pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO || PGM_PAGE_IS_MMIO(pPage),
+                  ("%RGp %R[pgmpage]\n", pRam->GCPhys + (i << PAGE_SHIFT), pPage));
+
+        /* Only do upgrades. */
+        if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) < uState)
+        {
+            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, uState);
+
+            const RTGCPHYS GCPhysPage = pRam->GCPhys + (i << PAGE_SHIFT);
+            int rc2 = pgmPoolTrackUpdateGCPhys(pVM, GCPhysPage, pPage,
+                                               false /* allow updates of PTEs (instead of flushing) */, &fFlushTLBs);
+            if (rc2 != VINF_SUCCESS && rc == VINF_SUCCESS)
+                rc = rc2;
+
+            /* Tell NEM about the protection update. */
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+                NEMHCNotifyPhysPageProtChanged(pVM, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage),
+                                               pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+        }
+
+        /* next */
+        if (--cPages == 0)
+            break;
+        i++;
+    }
+
+    if (fFlushTLBs)
+    {
+        PGM_INVL_ALL_VCPU_TLBS(pVM);
+        Log(("pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs: flushing guest TLBs; rc=%d\n", rc));
+    }
+    else
+        Log(("pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs: doesn't flush guest TLBs. rc=%Rrc; sync flags=%x VMCPU_FF_PGM_SYNC_CR3=%d\n", rc, VMMGetCpu(pVM)->pgm.s.fSyncFlags, VMCPU_FF_IS_SET(VMMGetCpu(pVM), VMCPU_FF_PGM_SYNC_CR3)));
+
+    return rc;
+}
+
+
+/**
+ * Deregister a physical page access handler.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pPhysHandler    The handler to deregister (but not free).
+ * @param   fRestoreAsRAM   How this will likely be restored, if we know (true,
+ *                          false, or if we don't know -1).
+ */
+int pgmHandlerPhysicalExDeregister(PVMCC pVM, PPGMPHYSHANDLER pPhysHandler, int fRestoreAsRAM)
+{
+    LogFlow(("pgmHandlerPhysicalExDeregister: Removing Range %RGp-%RGp %s fRestoreAsRAM=%d\n",
+             pPhysHandler->Core.Key, pPhysHandler->Core.KeyLast, R3STRING(pPhysHandler->pszDesc), fRestoreAsRAM));
+    AssertReturn(pPhysHandler->Core.Key != NIL_RTGCPHYS, VERR_PGM_HANDLER_NOT_FOUND);
+
+    /*
+     * Remove the handler from the tree.
+     */
+    pgmLock(pVM);
+    PPGMPHYSHANDLER pRemoved = (PPGMPHYSHANDLER)RTAvlroGCPhysRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,
+                                                                    pPhysHandler->Core.Key);
+    if (pRemoved == pPhysHandler)
+    {
+        /*
+         * Clear the page bits, notify the REM about this change and clear
+         * the cache.
+         */
+        pgmHandlerPhysicalResetRamFlags(pVM, pPhysHandler);
+        pgmHandlerPhysicalDeregisterNotifyREMAndNEM(pVM, pPhysHandler, fRestoreAsRAM);
+        pVM->pgm.s.pLastPhysHandlerR0 = 0;
+        pVM->pgm.s.pLastPhysHandlerR3 = 0;
+
+        pPhysHandler->Core.Key     = NIL_RTGCPHYS;
+        pPhysHandler->Core.KeyLast = NIL_RTGCPHYS;
+
+        pgmUnlock(pVM);
+
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Both of the failure conditions here are considered internal processing
+     * errors because they can only be caused by race conditions or corruption.
+     * If we ever need to handle concurrent deregistration, we have to move
+     * the NIL_RTGCPHYS check inside the PGM lock.
+     */
+    if (pRemoved)
+        RTAvlroGCPhysInsert(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, &pRemoved->Core);
+
+    pgmUnlock(pVM);
+
+    if (!pRemoved)
+        AssertMsgFailed(("Didn't find range starting at %RGp in the tree!\n", pPhysHandler->Core.Key));
+    else
+        AssertMsgFailed(("Found different handle at %RGp in the tree: got %p insteaded of %p\n",
+                         pPhysHandler->Core.Key, pRemoved, pPhysHandler));
+    return VERR_PGM_HANDLER_IPE_1;
+}
+
+
+/**
+ * Destroys (frees) a physical handler.
+ *
+ * The caller must deregister it before destroying it!
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pHandler    The handler to free.  NULL if ignored.
+ */
+int pgmHandlerPhysicalExDestroy(PVMCC pVM, PPGMPHYSHANDLER pHandler)
+{
+    if (pHandler)
+    {
+        AssertPtr(pHandler);
+        AssertReturn(pHandler->Core.Key == NIL_RTGCPHYS, VERR_WRONG_ORDER);
+        PGMHandlerPhysicalTypeRelease(pVM, pHandler->hType);
+        MMHyperFree(pVM, pHandler);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregister a physical page access handler.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      Start physical address.
+ */
+VMMDECL(int)  PGMHandlerPhysicalDeregister(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    /*
+     * Find the handler.
+     */
+    pgmLock(pVM);
+    PPGMPHYSHANDLER pRemoved = (PPGMPHYSHANDLER)RTAvlroGCPhysRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
+    if (pRemoved)
+    {
+        LogFlow(("PGMHandlerPhysicalDeregister: Removing Range %RGp-%RGp %s\n",
+                 pRemoved->Core.Key, pRemoved->Core.KeyLast, R3STRING(pRemoved->pszDesc)));
+
+        /*
+         * Clear the page bits, notify the REM about this change and clear
+         * the cache.
+         */
+        pgmHandlerPhysicalResetRamFlags(pVM, pRemoved);
+        pgmHandlerPhysicalDeregisterNotifyREMAndNEM(pVM, pRemoved, -1);
+        pVM->pgm.s.pLastPhysHandlerR0 = 0;
+        pVM->pgm.s.pLastPhysHandlerR3 = 0;
+
+        pgmUnlock(pVM);
+
+        pRemoved->Core.Key = NIL_RTGCPHYS;
+        pgmHandlerPhysicalExDestroy(pVM, pRemoved);
+        return VINF_SUCCESS;
+    }
+
+    pgmUnlock(pVM);
+
+    AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys));
+    return VERR_PGM_HANDLER_NOT_FOUND;
+}
+
+
+/**
+ * Shared code with modify.
+ */
+static void pgmHandlerPhysicalDeregisterNotifyREMAndNEM(PVMCC pVM, PPGMPHYSHANDLER pCur, int fRestoreAsRAM)
+{
+    PPGMPHYSHANDLERTYPEINT  pCurType    = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+    RTGCPHYS                GCPhysStart = pCur->Core.Key;
+    RTGCPHYS                GCPhysLast  = pCur->Core.KeyLast;
+
+    /*
+     * Page align the range.
+     *
+     * Since we've reset (recalculated) the physical handler state of all pages
+     * we can make use of the page states to figure out whether a page should be
+     * included in the REM notification or not.
+     */
+    if (   (pCur->Core.Key           & PAGE_OFFSET_MASK)
+        || ((pCur->Core.KeyLast + 1) & PAGE_OFFSET_MASK))
+    {
+        Assert(pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO);
+
+        if (GCPhysStart & PAGE_OFFSET_MASK)
+        {
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhysStart);
+            if (    pPage
+                &&  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
+            {
+                RTGCPHYS GCPhys = (GCPhysStart + (PAGE_SIZE - 1)) & X86_PTE_PAE_PG_MASK;
+                if (    GCPhys > GCPhysLast
+                    ||  GCPhys < GCPhysStart)
+                    return;
+                GCPhysStart = GCPhys;
+            }
+            else
+                GCPhysStart &= X86_PTE_PAE_PG_MASK;
+            Assert(!pPage || PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO); /* these are page aligned atm! */
+        }
+
+        if (GCPhysLast & PAGE_OFFSET_MASK)
+        {
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhysLast);
+            if (    pPage
+                &&  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
+            {
+                RTGCPHYS GCPhys = (GCPhysLast & X86_PTE_PAE_PG_MASK) - 1;
+                if (    GCPhys < GCPhysStart
+                    ||  GCPhys > GCPhysLast)
+                    return;
+                GCPhysLast = GCPhys;
+            }
+            else
+                GCPhysLast |= PAGE_OFFSET_MASK;
+            Assert(!pPage || PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO); /* these are page aligned atm! */
+        }
+    }
+
+    /*
+     * Tell REM and NEM.
+     */
+    const bool fRestoreAsRAM2 = pCurType->pfnHandlerR3
+                             && pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO; /** @todo this isn't entirely correct. */
+    /** @todo do we need this notification? */
+    NEMHCNotifyHandlerPhysicalDeregister(pVM, pCurType->enmKind, GCPhysStart, GCPhysLast - GCPhysStart + 1,
+                                         fRestoreAsRAM, fRestoreAsRAM2);
+}
+
+
+/**
+ * pgmHandlerPhysicalResetRamFlags helper that checks for other handlers on
+ * edge pages.
+ */
+DECLINLINE(void) pgmHandlerPhysicalRecalcPageState(PVMCC pVM, RTGCPHYS GCPhys, bool fAbove, PPGMRAMRANGE *ppRamHint)
+{
+    /*
+     * Look for other handlers.
+     */
+    unsigned uState = PGM_PAGE_HNDL_PHYS_STATE_NONE;
+    for (;;)
+    {
+        PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys, fAbove);
+        if (   !pCur
+            || ((fAbove ? pCur->Core.Key : pCur->Core.KeyLast) >> PAGE_SHIFT) != (GCPhys >> PAGE_SHIFT))
+            break;
+        PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+        uState = RT_MAX(uState, pCurType->uState);
+
+        /* next? */
+        RTGCPHYS GCPhysNext = fAbove
+                            ? pCur->Core.KeyLast + 1
+                            : pCur->Core.Key - 1;
+        if ((GCPhysNext >> PAGE_SHIFT) != (GCPhys >> PAGE_SHIFT))
+            break;
+        GCPhys = GCPhysNext;
+    }
+
+    /*
+     * Update if we found something that is a higher priority
+     * state than the current.
+     */
+    if (uState != PGM_PAGE_HNDL_PHYS_STATE_NONE)
+    {
+        PPGMPAGE pPage;
+        int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, ppRamHint);
+        if (    RT_SUCCESS(rc)
+            &&  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) < uState)
+        {
+            /* This should normally not be necessary. */
+            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, uState);
+            bool fFlushTLBs ;
+            rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, false /*fFlushPTEs*/, &fFlushTLBs);
+            if (RT_SUCCESS(rc) && fFlushTLBs)
+                PGM_INVL_ALL_VCPU_TLBS(pVM);
+            else
+                AssertRC(rc);
+
+            /* Tell NEM about the protection update. */
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+                NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
+                                               pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+        }
+        else
+            AssertRC(rc);
+    }
+}
+
+
+/**
+ * Resets an aliased page.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pPage           The page.
+ * @param   GCPhysPage      The page address in case it comes in handy.
+ * @param   fDoAccounting   Whether to perform accounting.  (Only set during
+ *                          reset where pgmR3PhysRamReset doesn't have the
+ *                          handler structure handy.)
+ */
+void pgmHandlerPhysicalResetAliasedPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage, bool fDoAccounting)
+{
+    Assert(   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
+           || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO);
+    Assert(PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_DISABLED);
+    RTHCPHYS const HCPhysPrev = PGM_PAGE_GET_HCPHYS(pPage);
+
+    /*
+     * Flush any shadow page table references *first*.
+     */
+    bool fFlushTLBs = false;
+    int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhysPage, pPage, true /*fFlushPTEs*/, &fFlushTLBs);
+    AssertLogRelRCReturnVoid(rc);
+    HMFlushTlbOnAllVCpus(pVM);
+
+    /*
+     * Make it an MMIO/Zero page.
+     */
+    PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
+    PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_MMIO);
+    PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
+    PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
+    PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, PGM_PAGE_HNDL_PHYS_STATE_ALL);
+
+    /* Flush its TLB entry. */
+    pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhysPage);
+
+    /*
+     * Do accounting for pgmR3PhysRamReset.
+     */
+    if (fDoAccounting)
+    {
+        PPGMPHYSHANDLER pHandler = pgmHandlerPhysicalLookup(pVM, GCPhysPage);
+        if (RT_LIKELY(pHandler))
+        {
+            Assert(pHandler->cAliasedPages > 0);
+            pHandler->cAliasedPages--;
+        }
+        else
+            AssertFailed();
+    }
+
+    /*
+     * Tell NEM about the protection change.
+     */
+    if (VM_IS_NEM_ENABLED(pVM))
+    {
+        uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+        NEMHCNotifyPhysPageChanged(pVM, GCPhysPage, HCPhysPrev, pVM->pgm.s.HCPhysZeroPg,
+                                   NEM_PAGE_PROT_NONE, PGMPAGETYPE_MMIO, &u2State);
+        PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+    }
+}
+
+
+/**
+ * Resets ram range flags.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS when shadow PTs was successfully updated.
+ * @param   pVM     The cross context VM structure.
+ * @param   pCur    The physical handler.
+ *
+ * @remark  We don't start messing with the shadow page tables, as we've
+ *          already got code in Trap0e which deals with out of sync handler
+ *          flags (originally conceived for global pages).
+ */
+static void pgmHandlerPhysicalResetRamFlags(PVMCC pVM, PPGMPHYSHANDLER pCur)
+{
+    /*
+     * Iterate the guest ram pages updating the state.
+     */
+    RTUINT          cPages   = pCur->cPages;
+    RTGCPHYS        GCPhys   = pCur->Core.Key;
+    PPGMRAMRANGE    pRamHint = NULL;
+    for (;;)
+    {
+        PPGMPAGE pPage;
+        int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
+        if (RT_SUCCESS(rc))
+        {
+            /* Reset aliased MMIO pages to MMIO, since this aliasing is our business.
+               (We don't flip MMIO to RAM though, that's PGMPhys.cpp's job.)  */
+            bool fNemNotifiedAlready = false;
+            if (   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
+                || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO)
+            {
+                Assert(pCur->cAliasedPages > 0);
+                pgmHandlerPhysicalResetAliasedPage(pVM, pPage, GCPhys, false /*fDoAccounting*/);
+                pCur->cAliasedPages--;
+                fNemNotifiedAlready = true;
+            }
+#ifdef VBOX_STRICT
+            PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+            AssertMsg(pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO || PGM_PAGE_IS_MMIO(pPage), ("%RGp %R[pgmpage]\n", GCPhys, pPage));
+#endif
+            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, PGM_PAGE_HNDL_PHYS_STATE_NONE);
+
+            /* Tell NEM about the protection change. */
+            if (VM_IS_NEM_ENABLED(pVM) && !fNemNotifiedAlready)
+            {
+                uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+                NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
+                                               pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+        }
+        else
+            AssertRC(rc);
+
+        /* next */
+        if (--cPages == 0)
+            break;
+        GCPhys += PAGE_SIZE;
+    }
+
+    pCur->cAliasedPages = 0;
+    pCur->cTmpOffPages  = 0;
+
+    /*
+     * Check for partial start and end pages.
+     */
+    if (pCur->Core.Key & PAGE_OFFSET_MASK)
+        pgmHandlerPhysicalRecalcPageState(pVM, pCur->Core.Key - 1, false /* fAbove */, &pRamHint);
+    if ((pCur->Core.KeyLast & PAGE_OFFSET_MASK) != PAGE_OFFSET_MASK)
+        pgmHandlerPhysicalRecalcPageState(pVM, pCur->Core.KeyLast + 1, true /* fAbove */, &pRamHint);
+}
+
+
+/**
+ * Modify a physical page access handler.
+ *
+ * Modification can only be done to the range it self, not the type or anything else.
+ *
+ * @returns VBox status code.
+ *          For all return codes other than VERR_PGM_HANDLER_NOT_FOUND and VINF_SUCCESS the range is deregistered
+ *          and a new registration must be performed!
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhysCurrent   Current location.
+ * @param   GCPhys          New location.
+ * @param   GCPhysLast      New last location.
+ */
+VMMDECL(int) PGMHandlerPhysicalModify(PVMCC pVM, RTGCPHYS GCPhysCurrent, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast)
+{
+    /*
+     * Remove it.
+     */
+    int rc;
+    pgmLock(pVM);
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysCurrent);
+    if (pCur)
+    {
+        /*
+         * Clear the ram flags. (We're gonna move or free it!)
+         */
+        pgmHandlerPhysicalResetRamFlags(pVM, pCur);
+        PPGMPHYSHANDLERTYPEINT const pCurType      = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+        bool const                   fRestoreAsRAM = pCurType->pfnHandlerR3 /** @todo this isn't entirely correct. */
+                                                  && pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO;
+
+        /*
+         * Validate the new range, modify and reinsert.
+         */
+        if (GCPhysLast >= GCPhys)
+        {
+            /*
+             * We require the range to be within registered ram.
+             * There is no apparent need to support ranges which cover more than one ram range.
+             */
+            PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
+            if (   pRam
+                && GCPhys <= pRam->GCPhysLast
+                && GCPhysLast >= pRam->GCPhys)
+            {
+                pCur->Core.Key      = GCPhys;
+                pCur->Core.KeyLast  = GCPhysLast;
+                pCur->cPages        = (GCPhysLast - (GCPhys & X86_PTE_PAE_PG_MASK) + 1) >> PAGE_SHIFT;
+
+                if (RTAvlroGCPhysInsert(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, &pCur->Core))
+                {
+                    RTGCPHYS            const cb            = GCPhysLast - GCPhys + 1;
+                    PGMPHYSHANDLERKIND  const enmKind       = pCurType->enmKind;
+
+                    /*
+                     * Set ram flags, flush shadow PT entries and finally tell REM about this.
+                     */
+                    rc = pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(pVM, pCur, pRam);
+
+                    /** @todo NEM: not sure we need this notification... */
+                    NEMHCNotifyHandlerPhysicalModify(pVM, enmKind, GCPhysCurrent, GCPhys, cb, fRestoreAsRAM);
+
+                    pgmUnlock(pVM);
+
+                    PGM_INVL_ALL_VCPU_TLBS(pVM);
+                    Log(("PGMHandlerPhysicalModify: GCPhysCurrent=%RGp -> GCPhys=%RGp GCPhysLast=%RGp\n",
+                         GCPhysCurrent, GCPhys, GCPhysLast));
+                    return VINF_SUCCESS;
+                }
+
+                AssertMsgFailed(("Conflict! GCPhys=%RGp GCPhysLast=%RGp\n", GCPhys, GCPhysLast));
+                rc = VERR_PGM_HANDLER_PHYSICAL_CONFLICT;
+            }
+            else
+            {
+                AssertMsgFailed(("No RAM range for %RGp-%RGp\n", GCPhys, GCPhysLast));
+                rc = VERR_PGM_HANDLER_PHYSICAL_NO_RAM_RANGE;
+            }
+        }
+        else
+        {
+            AssertMsgFailed(("Invalid range %RGp-%RGp\n", GCPhys, GCPhysLast));
+            rc = VERR_INVALID_PARAMETER;
+        }
+
+        /*
+         * Invalid new location, flush the cache and free it.
+         * We've only gotta notify REM and free the memory.
+         */
+        pgmHandlerPhysicalDeregisterNotifyREMAndNEM(pVM, pCur, -1);
+        pVM->pgm.s.pLastPhysHandlerR0 = 0;
+        pVM->pgm.s.pLastPhysHandlerR3 = 0;
+        PGMHandlerPhysicalTypeRelease(pVM, pCur->hType);
+        MMHyperFree(pVM, pCur);
+    }
+    else
+    {
+        AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhysCurrent));
+        rc = VERR_PGM_HANDLER_NOT_FOUND;
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Changes the user callback arguments associated with a physical access handler.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Start physical address of the handler.
+ * @param   pvUserR3        User argument to the R3 handler.
+ * @param   pvUserR0        User argument to the R0 handler.
+ */
+VMMDECL(int) PGMHandlerPhysicalChangeUserArgs(PVMCC pVM, RTGCPHYS GCPhys, RTR3PTR pvUserR3, RTR0PTR pvUserR0)
+{
+    /*
+     * Find the handler.
+     */
+    int rc = VINF_SUCCESS;
+    pgmLock(pVM);
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
+    if (pCur)
+    {
+        /*
+         * Change arguments.
+         */
+        pCur->pvUserR3 = pvUserR3;
+        pCur->pvUserR0 = pvUserR0;
+    }
+    else
+    {
+        AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys));
+        rc = VERR_PGM_HANDLER_NOT_FOUND;
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Splits a physical access handler in two.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Start physical address of the handler.
+ * @param   GCPhysSplit     The split address.
+ */
+VMMDECL(int) PGMHandlerPhysicalSplit(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysSplit)
+{
+    AssertReturn(GCPhys < GCPhysSplit, VERR_INVALID_PARAMETER);
+
+    /*
+     * Do the allocation without owning the lock.
+     */
+    PPGMPHYSHANDLER pNew;
+    int rc = MMHyperAlloc(pVM, sizeof(*pNew), 0, MM_TAG_PGM_HANDLERS, (void **)&pNew);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Get the handler.
+     */
+    pgmLock(pVM);
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
+    if (RT_LIKELY(pCur))
+    {
+        if (RT_LIKELY(GCPhysSplit <= pCur->Core.KeyLast))
+        {
+            /*
+             * Create new handler node for the 2nd half.
+             */
+            *pNew = *pCur;
+            pNew->Core.Key      = GCPhysSplit;
+            pNew->cPages        = (pNew->Core.KeyLast - (pNew->Core.Key & X86_PTE_PAE_PG_MASK) + PAGE_SIZE) >> PAGE_SHIFT;
+
+            pCur->Core.KeyLast  = GCPhysSplit - 1;
+            pCur->cPages        = (pCur->Core.KeyLast - (pCur->Core.Key & X86_PTE_PAE_PG_MASK) + PAGE_SIZE) >> PAGE_SHIFT;
+
+            if (RT_LIKELY(RTAvlroGCPhysInsert(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, &pNew->Core)))
+            {
+                LogFlow(("PGMHandlerPhysicalSplit: %RGp-%RGp and %RGp-%RGp\n",
+                         pCur->Core.Key, pCur->Core.KeyLast, pNew->Core.Key, pNew->Core.KeyLast));
+                pgmUnlock(pVM);
+                return VINF_SUCCESS;
+            }
+            AssertMsgFailed(("whu?\n"));
+            rc = VERR_PGM_PHYS_HANDLER_IPE;
+        }
+        else
+        {
+            AssertMsgFailed(("outside range: %RGp-%RGp split %RGp\n", pCur->Core.Key, pCur->Core.KeyLast, GCPhysSplit));
+            rc = VERR_INVALID_PARAMETER;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys));
+        rc = VERR_PGM_HANDLER_NOT_FOUND;
+    }
+    pgmUnlock(pVM);
+    MMHyperFree(pVM, pNew);
+    return rc;
+}
+
+
+/**
+ * Joins up two adjacent physical access handlers which has the same callbacks.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys1         Start physical address of the first handler.
+ * @param   GCPhys2         Start physical address of the second handler.
+ */
+VMMDECL(int) PGMHandlerPhysicalJoin(PVMCC pVM, RTGCPHYS GCPhys1, RTGCPHYS GCPhys2)
+{
+    /*
+     * Get the handlers.
+     */
+    int rc;
+    pgmLock(pVM);
+    PPGMPHYSHANDLER pCur1 = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys1);
+    if (RT_LIKELY(pCur1))
+    {
+        PPGMPHYSHANDLER pCur2 = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys2);
+        if (RT_LIKELY(pCur2))
+        {
+            /*
+             * Make sure that they are adjacent, and that they've got the same callbacks.
+             */
+            if (RT_LIKELY(pCur1->Core.KeyLast + 1 == pCur2->Core.Key))
+            {
+                if (RT_LIKELY(pCur1->hType == pCur2->hType))
+                {
+                    PPGMPHYSHANDLER pCur3 = (PPGMPHYSHANDLER)RTAvlroGCPhysRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys2);
+                    if (RT_LIKELY(pCur3 == pCur2))
+                    {
+                        pCur1->Core.KeyLast  = pCur2->Core.KeyLast;
+                        pCur1->cPages        = (pCur1->Core.KeyLast - (pCur1->Core.Key & X86_PTE_PAE_PG_MASK) + PAGE_SIZE) >> PAGE_SHIFT;
+                        LogFlow(("PGMHandlerPhysicalJoin: %RGp-%RGp %RGp-%RGp\n",
+                                 pCur1->Core.Key, pCur1->Core.KeyLast, pCur2->Core.Key, pCur2->Core.KeyLast));
+                        pVM->pgm.s.pLastPhysHandlerR0 = 0;
+                        pVM->pgm.s.pLastPhysHandlerR3 = 0;
+                        PGMHandlerPhysicalTypeRelease(pVM, pCur2->hType);
+                        MMHyperFree(pVM, pCur2);
+                        pgmUnlock(pVM);
+                        return VINF_SUCCESS;
+                    }
+
+                    Assert(pCur3 == pCur2);
+                    rc = VERR_PGM_PHYS_HANDLER_IPE;
+                }
+                else
+                {
+                    AssertMsgFailed(("mismatching handlers\n"));
+                    rc = VERR_ACCESS_DENIED;
+                }
+            }
+            else
+            {
+                AssertMsgFailed(("not adjacent: %RGp-%RGp %RGp-%RGp\n",
+                                 pCur1->Core.Key, pCur1->Core.KeyLast, pCur2->Core.Key, pCur2->Core.KeyLast));
+                rc = VERR_INVALID_PARAMETER;
+            }
+        }
+        else
+        {
+            AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys2));
+            rc = VERR_PGM_HANDLER_NOT_FOUND;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys1));
+        rc = VERR_PGM_HANDLER_NOT_FOUND;
+    }
+    pgmUnlock(pVM);
+    return rc;
+
+}
+
+
+/**
+ * Resets any modifications to individual pages in a physical page access
+ * handler region.
+ *
+ * This is used in pair with PGMHandlerPhysicalPageTempOff(),
+ * PGMHandlerPhysicalPageAliasMmio2() or PGMHandlerPhysicalPageAliasHC().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The start address of the handler regions, i.e. what you
+ *                      passed to PGMR3HandlerPhysicalRegister(),
+ *                      PGMHandlerPhysicalRegisterEx() or
+ *                      PGMHandlerPhysicalModify().
+ */
+VMMDECL(int) PGMHandlerPhysicalReset(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    LogFlow(("PGMHandlerPhysicalReset GCPhys=%RGp\n", GCPhys));
+    pgmLock(pVM);
+
+    /*
+     * Find the handler.
+     */
+    int rc;
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
+    if (RT_LIKELY(pCur))
+    {
+        /*
+         * Validate kind.
+         */
+        PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+        switch (pCurType->enmKind)
+        {
+            case PGMPHYSHANDLERKIND_WRITE:
+            case PGMPHYSHANDLERKIND_ALL:
+            case PGMPHYSHANDLERKIND_MMIO: /* NOTE: Only use when clearing MMIO ranges with aliased MMIO2 pages! */
+            {
+                STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysHandlerReset)); /** @todo move out of switch */
+                PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
+                Assert(pRam);
+                Assert(pRam->GCPhys     <= pCur->Core.Key);
+                Assert(pRam->GCPhysLast >= pCur->Core.KeyLast);
+
+                if (pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO)
+                {
+                    /*
+                     * Reset all the PGMPAGETYPE_MMIO2_ALIAS_MMIO pages first and that's it.
+                     * This could probably be optimized a bit wrt to flushing, but I'm too lazy
+                     * to do that now...
+                     */
+                    if (pCur->cAliasedPages)
+                    {
+                        PPGMPAGE    pPage = &pRam->aPages[(pCur->Core.Key - pRam->GCPhys) >> PAGE_SHIFT];
+                        uint32_t    cLeft = pCur->cPages;
+                        while (cLeft-- > 0)
+                        {
+                            if (   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
+                                || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO)
+                            {
+                                Assert(pCur->cAliasedPages > 0);
+                                pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)cLeft << PAGE_SHIFT),
+                                                                   false /*fDoAccounting*/);
+                                --pCur->cAliasedPages;
+#ifndef VBOX_STRICT
+                                if (pCur->cAliasedPages == 0)
+                                    break;
+#endif
+                            }
+                            Assert(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO);
+                            pPage++;
+                        }
+                        Assert(pCur->cAliasedPages == 0);
+                    }
+                }
+                else if (pCur->cTmpOffPages > 0)
+                {
+                    /*
+                     * Set the flags and flush shadow PT entries.
+                     */
+                    rc = pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(pVM, pCur, pRam);
+                }
+
+                pCur->cAliasedPages = 0;
+                pCur->cTmpOffPages  = 0;
+
+                rc = VINF_SUCCESS;
+                break;
+            }
+
+            /*
+             * Invalid.
+             */
+            default:
+                AssertMsgFailed(("Invalid type %d! Corruption!\n",  pCurType->enmKind));
+                rc = VERR_PGM_PHYS_HANDLER_IPE;
+                break;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Didn't find MMIO Range starting at %#x\n", GCPhys));
+        rc = VERR_PGM_HANDLER_NOT_FOUND;
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Temporarily turns off the access monitoring of a page within a monitored
+ * physical write/all page access handler region.
+ *
+ * Use this when no further \#PFs are required for that page. Be aware that
+ * a page directory sync might reset the flags, and turn on access monitoring
+ * for the page.
+ *
+ * The caller must do required page table modifications.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   GCPhys              The start address of the access handler. This
+ *                              must be a fully page aligned range or we risk
+ *                              messing up other handlers installed for the
+ *                              start and end pages.
+ * @param   GCPhysPage          The physical address of the page to turn off
+ *                              access monitoring for.
+ */
+VMMDECL(int)  PGMHandlerPhysicalPageTempOff(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysPage)
+{
+    LogFlow(("PGMHandlerPhysicalPageTempOff GCPhysPage=%RGp\n", GCPhysPage));
+
+    pgmLock(pVM);
+    /*
+     * Validate the range.
+     */
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
+    if (RT_LIKELY(pCur))
+    {
+        if (RT_LIKELY(    GCPhysPage >= pCur->Core.Key
+                      &&  GCPhysPage <= pCur->Core.KeyLast))
+        {
+            Assert(!(pCur->Core.Key & PAGE_OFFSET_MASK));
+            Assert((pCur->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
+
+            PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+            AssertReturnStmt(   pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE
+                             || pCurType->enmKind == PGMPHYSHANDLERKIND_ALL,
+                             pgmUnlock(pVM), VERR_ACCESS_DENIED);
+
+            /*
+             * Change the page status.
+             */
+            PPGMPAGE pPage;
+            int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
+            AssertReturnStmt(RT_SUCCESS_NP(rc), pgmUnlock(pVM), rc);
+            if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
+            {
+                PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, PGM_PAGE_HNDL_PHYS_STATE_DISABLED);
+                pCur->cTmpOffPages++;
+
+                /* Tell NEM about the protection change (VGA is using this to track dirty pages). */
+                if (VM_IS_NEM_ENABLED(pVM))
+                {
+                    uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                    PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+                    NEMHCNotifyPhysPageProtChanged(pVM, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage),
+                                                   pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+                    PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+                }
+            }
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+        pgmUnlock(pVM);
+        AssertMsgFailed(("The page %#x is outside the range %#x-%#x\n",
+                         GCPhysPage, pCur->Core.Key, pCur->Core.KeyLast));
+        return VERR_INVALID_PARAMETER;
+    }
+    pgmUnlock(pVM);
+    AssertMsgFailed(("Specified physical handler start address %#x is invalid.\n", GCPhys));
+    return VERR_PGM_HANDLER_NOT_FOUND;
+}
+
+
+/**
+ * Resolves an MMIO2 page.
+ *
+ * Caller as taken the PGM lock.
+ *
+ * @returns Pointer to the page if valid, NULL otherwise
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device owning it.
+ * @param   hMmio2          The MMIO2 region.
+ * @param   offMmio2Page    The offset into the region.
+ */
+static PPGMPAGE pgmPhysResolveMmio2PageLocked(PVMCC pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS offMmio2Page)
+{
+    /* Only works if the handle is in the handle table! */
+    AssertReturn(hMmio2 != 0, NULL);
+    hMmio2--;
+
+    /* Must check the first one for PGMREGMMIO2RANGE_F_FIRST_CHUNK. */
+    AssertReturn(hMmio2 < RT_ELEMENTS(pVM->pgm.s.apMmio2RangesR3), NULL);
+    PPGMREGMMIO2RANGE pCur = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[hMmio2];
+    AssertReturn(pCur, NULL);
+    AssertReturn(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK, NULL);
+
+    /* Loop thru the sub-ranges till we find the one covering offMmio2. */
+    for (;;)
+    {
+        AssertReturn(pCur->fFlags & PGMREGMMIO2RANGE_F_MMIO2, NULL);
+#ifdef IN_RING3
+        AssertReturn(pCur->pDevInsR3 == pDevIns, NULL);
+#else
+        AssertReturn(pCur->pDevInsR3 == pDevIns->pDevInsForR3, NULL);
+#endif
+
+        /* Does it match the offset? */
+        if (offMmio2Page < pCur->cbReal)
+            return &pCur->RamRange.aPages[offMmio2Page >> PAGE_SHIFT];
+
+        /* Advance if we can. */
+        AssertReturn(!(pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK), NULL);
+        offMmio2Page -= pCur->cbReal;
+        hMmio2++;
+        AssertReturn(hMmio2 < RT_ELEMENTS(pVM->pgm.s.apMmio2RangesR3), NULL);
+        pCur = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[hMmio2];
+        AssertReturn(pCur, NULL);
+    }
+}
+
+
+/**
+ * Replaces an MMIO page with an MMIO2 page.
+ *
+ * This is a worker for IOMMMIOMapMMIO2Page that works in a similar way to
+ * PGMHandlerPhysicalPageTempOff but for an MMIO page. Since an MMIO page has no
+ * backing, the caller must provide a replacement page. For various reasons the
+ * replacement page must be an MMIO2 page.
+ *
+ * The caller must do required page table modifications. You can get away
+ * without making any modifications since it's an MMIO page, the cost is an extra
+ * \#PF which will the resync the page.
+ *
+ * Call PGMHandlerPhysicalReset() to restore the MMIO page.
+ *
+ * The caller may still get handler callback even after this call and must be
+ * able to deal correctly with such calls. The reason for these callbacks are
+ * either that we're executing in the recompiler (which doesn't know about this
+ * arrangement) or that we've been restored from saved state (where we won't
+ * save the change).
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   GCPhys              The start address of the access handler. This
+ *                              must be a fully page aligned range or we risk
+ *                              messing up other handlers installed for the
+ *                              start and end pages.
+ * @param   GCPhysPage          The physical address of the page to turn off
+ *                              access monitoring for and replace with the MMIO2
+ *                              page.
+ * @param   pDevIns             The device instance owning @a hMmio2.
+ * @param   hMmio2              Handle to the MMIO2 region containing the page
+ *                              to remap in the the MMIO page at @a GCPhys.
+ * @param   offMmio2PageRemap   The offset into @a hMmio2 of the MMIO2 page that
+ *                              should serve as backing memory.
+ *
+ * @remark  May cause a page pool flush if used on a page that is already
+ *          aliased.
+ *
+ * @note    This trick does only work reliably if the two pages are never ever
+ *          mapped in the same page table. If they are the page pool code will
+ *          be confused should either of them be flushed. See the special case
+ *          of zero page aliasing mentioned in #3170.
+ *
+ */
+VMMDECL(int)  PGMHandlerPhysicalPageAliasMmio2(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysPage,
+                                               PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS offMmio2PageRemap)
+{
+    pgmLock(pVM);
+
+    /*
+     * Resolve the MMIO2 reference.
+     */
+    PPGMPAGE pPageRemap = pgmPhysResolveMmio2PageLocked(pVM, pDevIns, hMmio2, offMmio2PageRemap);
+    if (RT_LIKELY(pPageRemap))
+        AssertMsgReturnStmt(PGM_PAGE_GET_TYPE(pPageRemap) == PGMPAGETYPE_MMIO2,
+                            ("hMmio2=%RU64 offMmio2PageRemap=%RGp %R[pgmpage]\n", hMmio2, offMmio2PageRemap, pPageRemap),
+                            pgmUnlock(pVM), VERR_PGM_PHYS_NOT_MMIO2);
+    else
+    {
+        pgmUnlock(pVM);
+        return VERR_OUT_OF_RANGE;
+    }
+
+    /*
+     * Lookup and validate the range.
+     */
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
+    if (RT_LIKELY(pCur))
+    {
+        if (RT_LIKELY(   GCPhysPage >= pCur->Core.Key
+                      && GCPhysPage <= pCur->Core.KeyLast))
+        {
+            PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+            AssertReturnStmt(pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO, pgmUnlock(pVM), VERR_ACCESS_DENIED);
+            AssertReturnStmt(!(pCur->Core.Key & PAGE_OFFSET_MASK), pgmUnlock(pVM), VERR_INVALID_PARAMETER);
+            AssertReturnStmt((pCur->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK, pgmUnlock(pVM), VERR_INVALID_PARAMETER);
+
+            /*
+             * Validate the page.
+             */
+            PPGMPAGE pPage;
+            int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
+            AssertReturnStmt(RT_SUCCESS_NP(rc), pgmUnlock(pVM), rc);
+            if (PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO)
+            {
+                AssertMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO,
+                                ("GCPhysPage=%RGp %R[pgmpage]\n", GCPhysPage, pPage),
+                                VERR_PGM_PHYS_NOT_MMIO2);
+                if (PGM_PAGE_GET_HCPHYS(pPage) == PGM_PAGE_GET_HCPHYS(pPageRemap))
+                {
+                    pgmUnlock(pVM);
+                    return VINF_PGM_HANDLER_ALREADY_ALIASED;
+                }
+
+                /*
+                 * The page is already mapped as some other page, reset it
+                 * to an MMIO/ZERO page before doing the new mapping.
+                 */
+                Log(("PGMHandlerPhysicalPageAliasMmio2: GCPhysPage=%RGp (%R[pgmpage]; %RHp -> %RHp\n",
+                     GCPhysPage, pPage, PGM_PAGE_GET_HCPHYS(pPage), PGM_PAGE_GET_HCPHYS(pPageRemap)));
+                pgmHandlerPhysicalResetAliasedPage(pVM, pPage, GCPhysPage, false /*fDoAccounting*/);
+                pCur->cAliasedPages--;
+            }
+            Assert(PGM_PAGE_IS_ZERO(pPage));
+
+            /*
+             * Do the actual remapping here.
+             * This page now serves as an alias for the backing memory specified.
+             */
+            LogFlow(("PGMHandlerPhysicalPageAliasMmio2: %RGp (%R[pgmpage]) alias for %RU64/%RGp (%R[pgmpage])\n",
+                     GCPhysPage, pPage, hMmio2, offMmio2PageRemap, pPageRemap ));
+            PGM_PAGE_SET_HCPHYS(pVM, pPage, PGM_PAGE_GET_HCPHYS(pPageRemap));
+            PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_MMIO2_ALIAS_MMIO);
+            PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
+            PGM_PAGE_SET_PAGEID(pVM, pPage, PGM_PAGE_GET_PAGEID(pPageRemap));
+            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, PGM_PAGE_HNDL_PHYS_STATE_DISABLED);
+            pCur->cAliasedPages++;
+            Assert(pCur->cAliasedPages <= pCur->cPages);
+
+            /* Flush its TLB entry. */
+            pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhysPage);
+
+            /* Tell NEM about the backing and protection change. */
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                NEMHCNotifyPhysPageChanged(pVM, GCPhysPage, pVM->pgm.s.HCPhysZeroPg, PGM_PAGE_GET_HCPHYS(pPage),
+                                           pgmPhysPageCalcNemProtection(pPage, PGMPAGETYPE_MMIO2_ALIAS_MMIO),
+                                           PGMPAGETYPE_MMIO2_ALIAS_MMIO, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+            LogFlow(("PGMHandlerPhysicalPageAliasMmio2: => %R[pgmpage]\n", pPage));
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+
+        pgmUnlock(pVM);
+        AssertMsgFailed(("The page %#x is outside the range %#x-%#x\n",
+                         GCPhysPage, pCur->Core.Key, pCur->Core.KeyLast));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    pgmUnlock(pVM);
+    AssertMsgFailed(("Specified physical handler start address %#x is invalid.\n", GCPhys));
+    return VERR_PGM_HANDLER_NOT_FOUND;
+}
+
+
+/**
+ * Replaces an MMIO page with an arbitrary HC page in the shadow page tables.
+ *
+ * This differs from PGMHandlerPhysicalPageAliasMmio2 in that the page doesn't
+ * need to be a known MMIO2 page and that only shadow paging may access the
+ * page. The latter distinction is important because the only use for this
+ * feature is for mapping the special APIC access page that VT-x uses to detect
+ * APIC MMIO operations, the page is shared between all guest CPUs and actually
+ * not written to. At least at the moment.
+ *
+ * The caller must do required page table modifications. You can get away
+ * without making any modifications since it's an MMIO page, the cost is an extra
+ * \#PF which will the resync the page.
+ *
+ * Call PGMHandlerPhysicalReset() to restore the MMIO page.
+ *
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   GCPhys              The start address of the access handler. This
+ *                              must be a fully page aligned range or we risk
+ *                              messing up other handlers installed for the
+ *                              start and end pages.
+ * @param   GCPhysPage          The physical address of the page to turn off
+ *                              access monitoring for.
+ * @param   HCPhysPageRemap     The physical address of the HC page that
+ *                              serves as backing memory.
+ *
+ * @remark  May cause a page pool flush if used on a page that is already
+ *          aliased.
+ */
+VMMDECL(int)  PGMHandlerPhysicalPageAliasHC(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysPage, RTHCPHYS HCPhysPageRemap)
+{
+///    Assert(!IOMIsLockOwner(pVM)); /* We mustn't own any other locks when calling this */
+    pgmLock(pVM);
+
+    /*
+     * Lookup and validate the range.
+     */
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
+    if (RT_LIKELY(pCur))
+    {
+        if (RT_LIKELY(    GCPhysPage >= pCur->Core.Key
+                      &&  GCPhysPage <= pCur->Core.KeyLast))
+        {
+            PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+            AssertReturnStmt(pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO, pgmUnlock(pVM), VERR_ACCESS_DENIED);
+            AssertReturnStmt(!(pCur->Core.Key & PAGE_OFFSET_MASK), pgmUnlock(pVM), VERR_INVALID_PARAMETER);
+            AssertReturnStmt((pCur->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK, pgmUnlock(pVM), VERR_INVALID_PARAMETER);
+
+            /*
+             * Get and validate the pages.
+             */
+            PPGMPAGE pPage;
+            int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
+            AssertReturnStmt(RT_SUCCESS_NP(rc), pgmUnlock(pVM), rc);
+            if (PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO)
+            {
+                pgmUnlock(pVM);
+                AssertMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO,
+                                ("GCPhysPage=%RGp %R[pgmpage]\n", GCPhysPage, pPage),
+                                VERR_PGM_PHYS_NOT_MMIO2);
+                return VINF_PGM_HANDLER_ALREADY_ALIASED;
+            }
+            Assert(PGM_PAGE_IS_ZERO(pPage));
+
+            /*
+             * Do the actual remapping here.
+             * This page now serves as an alias for the backing memory
+             * specified as far as shadow paging is concerned.
+             */
+            LogFlow(("PGMHandlerPhysicalPageAliasHC: %RGp (%R[pgmpage]) alias for %RHp\n",
+                     GCPhysPage, pPage, HCPhysPageRemap));
+            PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhysPageRemap);
+            PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_SPECIAL_ALIAS_MMIO);
+            PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
+            PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
+            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, PGM_PAGE_HNDL_PHYS_STATE_DISABLED);
+            pCur->cAliasedPages++;
+            Assert(pCur->cAliasedPages <= pCur->cPages);
+
+            /* Flush its TLB entry. */
+            pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhysPage);
+
+            /* Tell NEM about the backing and protection change. */
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                NEMHCNotifyPhysPageChanged(pVM, GCPhysPage, pVM->pgm.s.HCPhysZeroPg, PGM_PAGE_GET_HCPHYS(pPage),
+                                           pgmPhysPageCalcNemProtection(pPage, PGMPAGETYPE_SPECIAL_ALIAS_MMIO),
+                                           PGMPAGETYPE_SPECIAL_ALIAS_MMIO, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+            LogFlow(("PGMHandlerPhysicalPageAliasHC: => %R[pgmpage]\n", pPage));
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+        pgmUnlock(pVM);
+        AssertMsgFailed(("The page %#x is outside the range %#x-%#x\n",
+                         GCPhysPage, pCur->Core.Key, pCur->Core.KeyLast));
+        return VERR_INVALID_PARAMETER;
+    }
+    pgmUnlock(pVM);
+
+    AssertMsgFailed(("Specified physical handler start address %#x is invalid.\n", GCPhys));
+    return VERR_PGM_HANDLER_NOT_FOUND;
+}
+
+
+/**
+ * Checks if a physical range is handled
+ *
+ * @returns boolean
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      Start physical address earlier passed to PGMR3HandlerPhysicalRegister().
+ * @remarks Caller must take the PGM lock...
+ * @thread  EMT.
+ */
+VMMDECL(bool) PGMHandlerPhysicalIsRegistered(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    /*
+     * Find the handler.
+     */
+    pgmLock(pVM);
+    PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
+    if (pCur)
+    {
+#ifdef VBOX_STRICT
+        Assert(GCPhys >= pCur->Core.Key && GCPhys <= pCur->Core.KeyLast);
+        PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+        Assert(   pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE
+               || pCurType->enmKind == PGMPHYSHANDLERKIND_ALL
+               || pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO);
+#endif
+        pgmUnlock(pVM);
+        return true;
+    }
+    pgmUnlock(pVM);
+    return false;
+}
+
+
+/**
+ * Checks if it's an disabled all access handler or write access handler at the
+ * given address.
+ *
+ * @returns true if it's an all access handler, false if it's a write access
+ *          handler.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The address of the page with a disabled handler.
+ *
+ * @remarks The caller, PGMR3PhysTlbGCPhys2Ptr, must hold the PGM lock.
+ */
+bool pgmHandlerPhysicalIsAll(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    pgmLock(pVM);
+    PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
+    if (!pCur)
+    {
+        pgmUnlock(pVM);
+        AssertFailed();
+        return true;
+    }
+    PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+    Assert(   pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE
+           || pCurType->enmKind == PGMPHYSHANDLERKIND_ALL
+           || pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO); /* sanity */
+    /* Only whole pages can be disabled. */
+    Assert(   pCur->Core.Key     <= (GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK)
+           && pCur->Core.KeyLast >= (GCPhys | PAGE_OFFSET_MASK));
+
+    bool bRet = pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE;
+    pgmUnlock(pVM);
+    return bRet;
+}
+
+#ifdef VBOX_STRICT
+
+/**
+ * State structure used by the PGMAssertHandlerAndFlagsInSync() function
+ * and its AVL enumerators.
+ */
+typedef struct PGMAHAFIS
+{
+    /** The current physical address. */
+    RTGCPHYS    GCPhys;
+    /** Number of errors. */
+    unsigned    cErrors;
+    /** Pointer to the VM. */
+    PVM         pVM;
+} PGMAHAFIS, *PPGMAHAFIS;
+
+
+/**
+ * Asserts that the handlers+guest-page-tables == ramrange-flags and
+ * that the physical addresses associated with virtual handlers are correct.
+ *
+ * @returns Number of mismatches.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(unsigned) PGMAssertHandlerAndFlagsInSync(PVM pVM)
+{
+    PPGM        pPGM = &pVM->pgm.s;
+    PGMAHAFIS   State;
+    State.GCPhys  = 0;
+    State.cErrors = 0;
+    State.pVM     = pVM;
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Check the RAM flags against the handlers.
+     */
+    for (PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRangesX); pRam; pRam = pRam->CTX_SUFF(pNext))
+    {
+        const uint32_t cPages = pRam->cb >> PAGE_SHIFT;
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            PGMPAGE const *pPage = &pRam->aPages[iPage];
+            if (PGM_PAGE_HAS_ANY_HANDLERS(pPage))
+            {
+                State.GCPhys = pRam->GCPhys + (iPage << PAGE_SHIFT);
+
+                /*
+                 * Physical first - calculate the state based on the handlers
+                 *                  active on the page, then compare.
+                 */
+                if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
+                {
+                    /* the first */
+                    PPGMPHYSHANDLER pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pPGM->CTX_SUFF(pTrees)->PhysHandlers, State.GCPhys);
+                    if (!pPhys)
+                    {
+                        pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysGetBestFit(&pPGM->CTX_SUFF(pTrees)->PhysHandlers, State.GCPhys, true);
+                        if (    pPhys
+                            &&  pPhys->Core.Key > (State.GCPhys + PAGE_SIZE - 1))
+                            pPhys = NULL;
+                        Assert(!pPhys || pPhys->Core.Key >= State.GCPhys);
+                    }
+                    if (pPhys)
+                    {
+                        PPGMPHYSHANDLERTYPEINT pPhysType = (PPGMPHYSHANDLERTYPEINT)MMHyperHeapOffsetToPtr(pVM, pPhys->hType);
+                        unsigned uState = pPhysType->uState;
+
+                        /* more? */
+                        while (pPhys->Core.KeyLast < (State.GCPhys | PAGE_OFFSET_MASK))
+                        {
+                            PPGMPHYSHANDLER pPhys2 = (PPGMPHYSHANDLER)RTAvlroGCPhysGetBestFit(&pPGM->CTX_SUFF(pTrees)->PhysHandlers,
+                                                                                              pPhys->Core.KeyLast + 1, true);
+                            if (    !pPhys2
+                                ||  pPhys2->Core.Key > (State.GCPhys | PAGE_OFFSET_MASK))
+                                break;
+                            PPGMPHYSHANDLERTYPEINT pPhysType2 = (PPGMPHYSHANDLERTYPEINT)MMHyperHeapOffsetToPtr(pVM, pPhys2->hType);
+                            uState = RT_MAX(uState, pPhysType2->uState);
+                            pPhys = pPhys2;
+                        }
+
+                        /* compare.*/
+                        if (    PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != uState
+                            &&  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
+                        {
+                            AssertMsgFailed(("ram range vs phys handler flags mismatch. GCPhys=%RGp state=%d expected=%d %s\n",
+                                             State.GCPhys, PGM_PAGE_GET_HNDL_PHYS_STATE(pPage), uState, pPhysType->pszDesc));
+                            State.cErrors++;
+                        }
+                    }
+                    else
+                    {
+                        AssertMsgFailed(("ram range vs phys handler mismatch. no handler for GCPhys=%RGp\n", State.GCPhys));
+                        State.cErrors++;
+                    }
+                }
+            }
+        } /* foreach page in ram range. */
+    } /* foreach ram range. */
+
+    /*
+     * Do the reverse check for physical handlers.
+     */
+    /** @todo */
+
+    return State.cErrors;
+}
+
+#endif /* VBOX_STRICT */
+
diff --git a/src/VBox/VMM/VMMAll/PGMAllMap.cpp b/src/VBox/VMM/VMMAll/PGMAllMap.cpp
new file mode 100644
index 00000000..f70575d2
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllMap.cpp
@@ -0,0 +1,930 @@
+/* $Id: PGMAllMap.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor - All context code.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/em.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include "PGMInline.h"
+#include <VBox/err.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/assert.h>
+
+
+#ifndef PGM_WITHOUT_MAPPINGS
+
+/**
+ * Maps a range of physical pages at a given virtual address
+ * in the guest context.
+ *
+ * The GC virtual address range must be within an existing mapping.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPtr       Where to map the page(s). Must be page aligned.
+ * @param   HCPhys      Start of the range of physical pages. Must be page aligned.
+ * @param   cbPages     Number of bytes to map. Must be page aligned.
+ * @param   fFlags      Page flags (X86_PTE_*).
+ */
+VMMDECL(int) PGMMap(PVM pVM, RTGCUINTPTR GCPtr, RTHCPHYS HCPhys, uint32_t cbPages, unsigned fFlags)
+{
+    AssertMsg(pVM->pgm.s.offVM, ("Bad init order\n"));
+
+    /*
+     * Validate input.
+     */
+    AssertMsg(RT_ALIGN_T(GCPtr, PAGE_SIZE, RTGCUINTPTR) == GCPtr, ("Invalid alignment GCPtr=%#x\n", GCPtr));
+    AssertMsg(cbPages > 0 && RT_ALIGN_32(cbPages, PAGE_SIZE) == cbPages, ("Invalid cbPages=%#x\n",  cbPages));
+    AssertMsg(!(fFlags & X86_PDE_PG_MASK), ("Invalid flags %#x\n", fFlags));
+
+    /* hypervisor defaults */
+    if (!fFlags)
+        fFlags = X86_PTE_P | X86_PTE_A | X86_PTE_D;
+
+    /*
+     * Find the mapping.
+     */
+    PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings);
+    while (pCur)
+    {
+        if (GCPtr - pCur->GCPtr < pCur->cb)
+        {
+            if (GCPtr + cbPages - 1 > pCur->GCPtrLast)
+            {
+                AssertMsgFailed(("Invalid range!!\n"));
+                return VERR_INVALID_PARAMETER;
+            }
+
+            /*
+             * Setup PTE.
+             */
+            X86PTEPAE Pte;
+            Pte.u = fFlags | (HCPhys & X86_PTE_PAE_PG_MASK);
+
+            /*
+             * Update the page tables.
+             */
+            for (;;)
+            {
+                RTGCUINTPTR     off = GCPtr - pCur->GCPtr;
+                const unsigned  iPT = off >> X86_PD_SHIFT;
+                const unsigned  iPageNo = (off >> PAGE_SHIFT) & X86_PT_MASK;
+
+                /* 32-bit */
+                pCur->aPTs[iPT].CTX_SUFF(pPT)->a[iPageNo].u = (uint32_t)Pte.u;      /* ASSUMES HCPhys < 4GB and/or that we're never gonna do 32-bit on a PAE host! */
+
+                /* pae */
+                PGMSHWPTEPAE_SET(pCur->aPTs[iPT].CTX_SUFF(paPaePTs)[iPageNo / 512].a[iPageNo % 512], Pte.u);
+
+                /* next */
+                cbPages -= PAGE_SIZE;
+                if (!cbPages)
+                    break;
+                GCPtr += PAGE_SIZE;
+                Pte.u += PAGE_SIZE;
+            }
+
+            return VINF_SUCCESS;
+        }
+
+        /* next */
+        pCur = pCur->CTX_SUFF(pNext);
+    }
+
+    AssertMsgFailed(("GCPtr=%#x was not found in any mapping ranges!\n",  GCPtr));
+    return VERR_INVALID_PARAMETER;
+}
+
+
+/**
+ * Sets (replaces) the page flags for a range of pages in a mapping.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   cb          Size (in bytes) of the range to apply the modification to.
+ * @param   fFlags      Page flags X86_PTE_*, excluding the page mask of course.
+ */
+VMMDECL(int) PGMMapSetPage(PVM pVM, RTGCPTR GCPtr, uint64_t cb, uint64_t fFlags)
+{
+    return PGMMapModifyPage(pVM, GCPtr, cb, fFlags, 0);
+}
+
+
+/**
+ * Modify page flags for a range of pages in a mapping.
+ *
+ * The existing flags are ANDed with the fMask and ORed with the fFlags.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   cb          Size (in bytes) of the range to apply the modification to.
+ * @param   fFlags      The OR  mask - page flags X86_PTE_*, excluding the page mask of course.
+ * @param   fMask       The AND mask - page flags X86_PTE_*, excluding the page mask of course.
+ */
+VMMDECL(int)  PGMMapModifyPage(PVM pVM, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
+{
+    /*
+     * Validate input.
+     */
+    AssertMsg(!(fFlags & (X86_PTE_PAE_PG_MASK | X86_PTE_PAE_MBZ_MASK_NX)), ("fFlags=%#x\n", fFlags));
+    Assert(cb);
+
+    /*
+     * Align the input.
+     */
+    cb     += (RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK;
+    cb      = RT_ALIGN_Z(cb, PAGE_SIZE);
+    GCPtr   = (RTGCPTR)((RTGCUINTPTR)GCPtr & PAGE_BASE_GC_MASK);
+
+    /*
+     * Find the mapping.
+     */
+    PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings);
+    while (pCur)
+    {
+        RTGCUINTPTR off = (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pCur->GCPtr;
+        if (off < pCur->cb)
+        {
+            AssertMsgReturn(off + cb <= pCur->cb,
+                            ("Invalid page range %#x LB%#x. mapping '%s' %#x to %#x\n",
+                             GCPtr, cb, pCur->pszDesc, pCur->GCPtr, pCur->GCPtrLast),
+                            VERR_INVALID_PARAMETER);
+
+            /*
+             * Perform the requested operation.
+             */
+            while (cb > 0)
+            {
+                unsigned iPT  = off >> X86_PD_SHIFT;
+                unsigned iPTE = (off >> PAGE_SHIFT) & X86_PT_MASK;
+                while (cb > 0 && iPTE < RT_ELEMENTS(pCur->aPTs[iPT].CTX_SUFF(pPT)->a))
+                {
+                    /* 32-Bit */
+                    pCur->aPTs[iPT].CTX_SUFF(pPT)->a[iPTE].u &= fMask | X86_PTE_PG_MASK;
+                    pCur->aPTs[iPT].CTX_SUFF(pPT)->a[iPTE].u |= fFlags & ~X86_PTE_PG_MASK;
+
+                    /* PAE */
+                    PPGMSHWPTEPAE pPtePae = &pCur->aPTs[iPT].CTX_SUFF(paPaePTs)[iPTE / 512].a[iPTE % 512];
+                    PGMSHWPTEPAE_SET(*pPtePae,
+                                       (  PGMSHWPTEPAE_GET_U(*pPtePae)
+                                        & (fMask | X86_PTE_PAE_PG_MASK))
+                                     | (fFlags & ~(X86_PTE_PAE_PG_MASK | X86_PTE_PAE_MBZ_MASK_NX)));
+
+                    /* invalidate tls */
+                    PGM_INVL_PG(VMMGetCpu(pVM), (RTGCUINTPTR)pCur->GCPtr + off);
+
+                    /* next */
+                    iPTE++;
+                    cb -= PAGE_SIZE;
+                    off += PAGE_SIZE;
+                }
+            }
+
+            return VINF_SUCCESS;
+        }
+        /* next */
+        pCur = pCur->CTX_SUFF(pNext);
+    }
+
+    AssertMsgFailed(("Page range %#x LB%#x not found\n", GCPtr, cb));
+    return VERR_INVALID_PARAMETER;
+}
+
+
+/**
+ * Get information about a page in a mapping.
+ *
+ * This differs from PGMShwGetPage and PGMGstGetPage in that it only consults
+ * the page table to calculate the flags.
+ *
+ * @returns VINF_SUCCESS, VERR_PAGE_NOT_PRESENT or VERR_NOT_FOUND.
+ * @param   pVM                 The cross context VM structure.
+ * @param   GCPtr               The page address.
+ * @param   pfFlags             Where to return the flags.  Optional.
+ * @param   pHCPhys             Where to return the address.  Optional.
+ */
+VMMDECL(int) PGMMapGetPage(PVM pVM, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
+{
+    /*
+     * Find the mapping.
+     */
+    GCPtr &= PAGE_BASE_GC_MASK;
+    PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings);
+    while (pCur)
+    {
+        RTGCUINTPTR off = (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pCur->GCPtr;
+        if (off < pCur->cb)
+        {
+            /*
+             * Dig out the information.
+             */
+            int             rc      = VINF_SUCCESS;
+            unsigned        iPT     = off >> X86_PD_SHIFT;
+            unsigned        iPTE    = (off >> PAGE_SHIFT) & X86_PT_MASK;
+            PCPGMSHWPTEPAE  pPtePae = &pCur->aPTs[iPT].CTX_SUFF(paPaePTs)[iPTE / 512].a[iPTE % 512];
+            if (PGMSHWPTEPAE_IS_P(*pPtePae))
+            {
+                if (pfFlags)
+                    *pfFlags = PGMSHWPTEPAE_GET_U(*pPtePae) & ~X86_PTE_PAE_PG_MASK;
+                if (pHCPhys)
+                    *pHCPhys = PGMSHWPTEPAE_GET_HCPHYS(*pPtePae);
+            }
+            else
+                rc = VERR_PAGE_NOT_PRESENT;
+            return rc;
+        }
+        /* next */
+        pCur = pCur->CTX_SUFF(pNext);
+    }
+
+    return VERR_NOT_FOUND;
+}
+
+
+/**
+ * Sets all PDEs involved with the mapping in the shadow page table.
+ *
+ * Ignored if mappings are disabled (i.e. if HM is enabled).
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pMap        Pointer to the mapping in question.
+ * @param   iNewPDE     The index of the 32-bit PDE corresponding to the base of the mapping.
+ */
+void pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE)
+{
+    Log4(("pgmMapSetShadowPDEs new pde %x (mappings enabled %d)\n", iNewPDE, pgmMapAreMappingsEnabled(pVM)));
+
+    if (!pgmMapAreMappingsEnabled(pVM))
+        return;
+
+    /* This only applies to raw mode where we only support 1 VCPU. */
+    PVMCPU pVCpu = VMMGetCpu0(pVM);
+    if (!pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
+        return;    /* too early */
+
+    PGMMODE enmShadowMode = PGMGetShadowMode(pVCpu);
+    Assert(enmShadowMode <= PGMMODE_PAE_NX);
+
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    /*
+     * Insert the page tables into the shadow page directories.
+     */
+    unsigned i = pMap->cPTs;
+    iNewPDE += i;
+    while (i-- > 0)
+    {
+        iNewPDE--;
+
+        switch (enmShadowMode)
+        {
+            case PGMMODE_32_BIT:
+            {
+                PX86PD pShw32BitPd = pgmShwGet32BitPDPtr(pVCpu);
+                AssertFatal(pShw32BitPd);
+
+                /* Free any previous user, unless it's us. */
+                Assert(   (pShw32BitPd->a[iNewPDE].u & (X86_PDE_P | PGM_PDFLAGS_MAPPING)) != (X86_PDE_P | PGM_PDFLAGS_MAPPING)
+                       || (pShw32BitPd->a[iNewPDE].u & X86_PDE_PG_MASK) == pMap->aPTs[i].HCPhysPT);
+                if (    pShw32BitPd->a[iNewPDE].n.u1Present
+                    &&  !(pShw32BitPd->a[iNewPDE].u & PGM_PDFLAGS_MAPPING))
+                    pgmPoolFree(pVM, pShw32BitPd->a[iNewPDE].u & X86_PDE_PG_MASK, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iNewPDE);
+
+                /* Default mapping page directory flags are read/write and supervisor; individual page attributes determine the final flags. */
+                pShw32BitPd->a[iNewPDE].u = PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US
+                                          | (uint32_t)pMap->aPTs[i].HCPhysPT;
+                PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShw32BitPd);
+                break;
+            }
+
+            case PGMMODE_PAE:
+            case PGMMODE_PAE_NX:
+            {
+                const uint32_t  iPdPt     = iNewPDE / 256;
+                unsigned        iPaePde   = iNewPDE * 2 % 512;
+                PX86PDPT        pShwPdpt  = pgmShwGetPaePDPTPtr(pVCpu);
+                Assert(pShwPdpt);
+
+                /*
+                 * Get the shadow PD.
+                 * If no PD, sync it (PAE guest) or fake (not present or 32-bit guest).
+                 * Note! The RW, US and A bits are reserved for PAE PDPTEs. Setting the
+                 *       accessed bit causes invalid VT-x guest state errors.
+                 */
+                PX86PDPAE pShwPaePd = pgmShwGetPaePDPtr(pVCpu, iPdPt << X86_PDPT_SHIFT);
+                if (!pShwPaePd)
+                {
+                    X86PDPE     GstPdpe;
+                    if (PGMGetGuestMode(pVCpu) < PGMMODE_PAE)
+                        GstPdpe.u = X86_PDPE_P;
+                    else
+                    {
+                        PX86PDPE pGstPdpe = pgmGstGetPaePDPEPtr(pVCpu, iPdPt << X86_PDPT_SHIFT);
+                        if (pGstPdpe)
+                            GstPdpe = *pGstPdpe;
+                        else
+                            GstPdpe.u = X86_PDPE_P;
+                    }
+                    int rc = pgmShwSyncPaePDPtr(pVCpu, iPdPt << X86_PDPT_SHIFT, GstPdpe.u, &pShwPaePd);
+                    AssertFatalRC(rc);
+                }
+                Assert(pShwPaePd);
+
+                /*
+                 * Mark the page as locked; disallow flushing.
+                 */
+                PPGMPOOLPAGE    pPoolPagePd = pgmPoolGetPage(pPool, pShwPdpt->a[iPdPt].u & X86_PDPE_PG_MASK);
+                AssertFatal(pPoolPagePd);
+                if (!pgmPoolIsPageLocked(pPoolPagePd))
+                    pgmPoolLockPage(pPool, pPoolPagePd);
+# ifdef VBOX_STRICT
+                else if (pShwPaePd->a[iPaePde].u & PGM_PDFLAGS_MAPPING)
+                {
+                    Assert(PGMGetGuestMode(pVCpu) >= PGMMODE_PAE); /** @todo We may hit this during reset, will fix later. */
+                    AssertFatalMsg(   (pShwPaePd->a[iPaePde].u & X86_PDE_PAE_PG_MASK) == pMap->aPTs[i].HCPhysPaePT0
+                                   || !PGMMODE_WITH_PAGING(PGMGetGuestMode(pVCpu)),
+                                   ("%RX64 vs %RX64\n", pShwPaePd->a[iPaePde+1].u & X86_PDE_PAE_PG_MASK, pMap->aPTs[i].HCPhysPaePT0));
+                    Assert(pShwPaePd->a[iPaePde+1].u & PGM_PDFLAGS_MAPPING);
+                    AssertFatalMsg(   (pShwPaePd->a[iPaePde+1].u & X86_PDE_PAE_PG_MASK) == pMap->aPTs[i].HCPhysPaePT1
+                                   || !PGMMODE_WITH_PAGING(PGMGetGuestMode(pVCpu)),
+                                   ("%RX64 vs %RX64\n", pShwPaePd->a[iPaePde+1].u & X86_PDE_PAE_PG_MASK, pMap->aPTs[i].HCPhysPaePT1));
+                }
+# endif
+
+                /*
+                 * Insert our first PT, freeing anything we might be replacing unless it's a mapping (i.e. us).
+                 */
+                Assert(   (pShwPaePd->a[iPaePde].u & (X86_PDE_P | PGM_PDFLAGS_MAPPING)) != (X86_PDE_P | PGM_PDFLAGS_MAPPING)
+                       || (pShwPaePd->a[iPaePde].u & X86_PDE_PAE_PG_MASK) == pMap->aPTs[i].HCPhysPaePT0);
+                if (    pShwPaePd->a[iPaePde].n.u1Present
+                    &&  !(pShwPaePd->a[iPaePde].u & PGM_PDFLAGS_MAPPING))
+                {
+                    Assert(!(pShwPaePd->a[iPaePde].u & PGM_PDFLAGS_MAPPING));
+                    pgmPoolFree(pVM, pShwPaePd->a[iPaePde].u & X86_PDE_PAE_PG_MASK, pPoolPagePd->idx, iPaePde);
+                }
+                pShwPaePd->a[iPaePde].u = PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US
+                                        | pMap->aPTs[i].HCPhysPaePT0;
+
+                /* 2nd 2 MB PDE of the 4 MB region, same as above. */
+                iPaePde++;
+                AssertFatal(iPaePde < 512);
+                Assert(   (pShwPaePd->a[iPaePde].u & (X86_PDE_P | PGM_PDFLAGS_MAPPING)) != (X86_PDE_P | PGM_PDFLAGS_MAPPING)
+                       || (pShwPaePd->a[iPaePde].u & X86_PDE_PAE_PG_MASK) == pMap->aPTs[i].HCPhysPaePT1);
+                if (    pShwPaePd->a[iPaePde].n.u1Present
+                    &&  !(pShwPaePd->a[iPaePde].u & PGM_PDFLAGS_MAPPING))
+                    pgmPoolFree(pVM, pShwPaePd->a[iPaePde].u & X86_PDE_PG_MASK, pPoolPagePd->idx, iPaePde);
+                pShwPaePd->a[iPaePde].u = PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US
+                                        | pMap->aPTs[i].HCPhysPaePT1;
+
+                /*
+                 * Set the PGM_PDFLAGS_MAPPING flag in the page directory pointer entry. (legacy PAE guest mode)
+                 */
+                pShwPdpt->a[iPdPt].u |= PGM_PLXFLAGS_MAPPING;
+
+                PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPaePd);
+                PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPdpt);
+                break;
+            }
+
+            default:
+                AssertFailed();
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clears all PDEs involved with the mapping in the shadow page table.
+ *
+ * Ignored if mappings are disabled (i.e. if HM is enabled).
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pShwPageCR3     CR3 root page
+ * @param   pMap            Pointer to the mapping in question.
+ * @param   iOldPDE         The index of the 32-bit PDE corresponding to the base of the mapping.
+ * @param   fDeactivateCR3  Set if it's pgmMapDeactivateCR3 calling.
+ */
+void pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3)
+{
+    Log(("pgmMapClearShadowPDEs: old pde %x (cPTs=%x) (mappings enabled %d) fDeactivateCR3=%RTbool\n", iOldPDE, pMap->cPTs, pgmMapAreMappingsEnabled(pVM), fDeactivateCR3));
+
+    /*
+     * Skip this if it doesn't apply.
+     */
+    if (!pgmMapAreMappingsEnabled(pVM))
+        return;
+
+    Assert(pShwPageCR3);
+
+    /* This only applies to raw mode where we only support 1 VCPU. */
+    PVMCPU pVCpu = VMMGetCpu0(pVM);
+# error fixme
+
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    PX86PDPT pCurrentShwPdpt = NULL;
+    if (    PGMGetGuestMode(pVCpu) >= PGMMODE_PAE
+        &&  pShwPageCR3 != pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
+        pCurrentShwPdpt = pgmShwGetPaePDPTPtr(pVCpu);
+
+    unsigned i = pMap->cPTs;
+    PGMMODE  enmShadowMode = PGMGetShadowMode(pVCpu);
+
+    iOldPDE += i;
+    while (i-- > 0)
+    {
+        iOldPDE--;
+
+        switch(enmShadowMode)
+        {
+            case PGMMODE_32_BIT:
+            {
+                PX86PD          pShw32BitPd = (PX86PD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPageCR3);
+                AssertFatal(pShw32BitPd);
+
+                Assert(!pShw32BitPd->a[iOldPDE].n.u1Present || (pShw32BitPd->a[iOldPDE].u & PGM_PDFLAGS_MAPPING));
+                pShw32BitPd->a[iOldPDE].u = 0;
+                break;
+            }
+
+            case PGMMODE_PAE:
+            case PGMMODE_PAE_NX:
+            {
+                const unsigned  iPdpt     = iOldPDE / 256;      /* iOldPDE * 2 / 512; iOldPDE is in 4 MB pages */
+                unsigned        iPaePde   = iOldPDE * 2 % 512;
+                PX86PDPT        pShwPdpt  = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPageCR3);
+                PX86PDPAE       pShwPaePd = pgmShwGetPaePDPtr(pVCpu, pShwPdpt, (iPdpt << X86_PDPT_SHIFT));
+
+                /*
+                 * Clear the PGM_PDFLAGS_MAPPING flag for the page directory pointer entry. (legacy PAE guest mode)
+                 */
+                if (fDeactivateCR3)
+                    pShwPdpt->a[iPdpt].u &= ~PGM_PLXFLAGS_MAPPING;
+                else if (pShwPdpt->a[iPdpt].u & PGM_PLXFLAGS_MAPPING)
+                {
+                    /* See if there are any other mappings here. This is suboptimal code. */
+                    pShwPdpt->a[iPdpt].u &= ~PGM_PLXFLAGS_MAPPING;
+                    for (PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings); pCur; pCur = pCur->CTX_SUFF(pNext))
+                        if (    pCur != pMap
+                            &&  (   (pCur->GCPtr >> X86_PDPT_SHIFT) == iPdpt
+                                 || (pCur->GCPtrLast >> X86_PDPT_SHIFT) == iPdpt))
+                        {
+                            pShwPdpt->a[iPdpt].u |= PGM_PLXFLAGS_MAPPING;
+                            break;
+                        }
+                }
+
+                /*
+                 * If the page directory of the old CR3 is reused in the new one, then don't
+                 * clear the hypervisor mappings.
+                 */
+                if (    pCurrentShwPdpt
+                    &&  (pCurrentShwPdpt->a[iPdpt].u & X86_PDPE_PG_MASK) == (pShwPdpt->a[iPdpt].u & X86_PDPE_PG_MASK) )
+                {
+                    LogFlow(("pgmMapClearShadowPDEs: Pdpe %d reused -> don't clear hypervisor mappings!\n", iPdpt));
+                    break;
+                }
+
+                /*
+                 * Clear the mappings in the PD.
+                 */
+                AssertFatal(pShwPaePd);
+                Assert(!pShwPaePd->a[iPaePde].n.u1Present || (pShwPaePd->a[iPaePde].u & PGM_PDFLAGS_MAPPING));
+                pShwPaePd->a[iPaePde].u = 0;
+
+                iPaePde++;
+                AssertFatal(iPaePde < 512);
+                Assert(!pShwPaePd->a[iPaePde].n.u1Present || (pShwPaePd->a[iPaePde].u & PGM_PDFLAGS_MAPPING));
+                pShwPaePd->a[iPaePde].u = 0;
+
+                /*
+                 * Unlock the shadow pool PD page if the PDPTE no longer holds any mappings.
+                 */
+                if (    fDeactivateCR3
+                    ||  !(pShwPdpt->a[iPdpt].u & PGM_PLXFLAGS_MAPPING))
+                {
+                    PPGMPOOLPAGE pPoolPagePd = pgmPoolGetPage(pPool, pShwPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
+                    AssertFatal(pPoolPagePd);
+                    if (pgmPoolIsPageLocked(pPoolPagePd))
+                        pgmPoolUnlockPage(pPool, pPoolPagePd);
+                }
+                break;
+            }
+
+            default:
+                AssertFailed();
+                break;
+        }
+    }
+
+    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pCurrentShwPdpt);
+}
+
+# if defined(VBOX_STRICT) && !defined(IN_RING0)
+
+/**
+ * Clears all PDEs involved with the mapping in the shadow page table.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pShwPageCR3 CR3 root page
+ * @param   pMap        Pointer to the mapping in question.
+ * @param   iPDE        The index of the 32-bit PDE corresponding to the base of the mapping.
+ */
+static void pgmMapCheckShadowPDEs(PVM pVM, PVMCPU pVCpu, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iPDE)
+{
+    Assert(pShwPageCR3);
+
+    uint32_t i = pMap->cPTs;
+    PGMMODE  enmShadowMode = PGMGetShadowMode(pVCpu);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    iPDE += i;
+    while (i-- > 0)
+    {
+        iPDE--;
+
+        switch (enmShadowMode)
+        {
+            case PGMMODE_32_BIT:
+            {
+                PCX86PD         pShw32BitPd = (PCX86PD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPageCR3);
+                AssertFatal(pShw32BitPd);
+
+                AssertMsg(pShw32BitPd->a[iPDE].u == (PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | (uint32_t)pMap->aPTs[i].HCPhysPT),
+                          ("Expected %x vs %x; iPDE=%#x %RGv %s\n",
+                           pShw32BitPd->a[iPDE].u,  (PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | (uint32_t)pMap->aPTs[i].HCPhysPT),
+                           iPDE, pMap->GCPtr, R3STRING(pMap->pszDesc) ));
+                break;
+            }
+
+            case PGMMODE_PAE:
+            case PGMMODE_PAE_NX:
+            {
+                const unsigned  iPdpt     = iPDE / 256;         /* iPDE * 2 / 512; iPDE is in 4 MB pages */
+                unsigned        iPaePDE   = iPDE * 2 % 512;
+                PX86PDPT        pShwPdpt  = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPageCR3);
+                PCX86PDPAE      pShwPaePd = pgmShwGetPaePDPtr(pVCpu, pShwPdpt, iPdpt << X86_PDPT_SHIFT);
+                AssertFatal(pShwPaePd);
+
+                AssertMsg(pShwPaePd->a[iPaePDE].u == (PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | pMap->aPTs[i].HCPhysPaePT0),
+                          ("Expected %RX64 vs %RX64; iPDE=%#x iPdpt=%#x iPaePDE=%#x %RGv %s\n",
+                           pShwPaePd->a[iPaePDE].u,  (PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | pMap->aPTs[i].HCPhysPaePT0),
+                           iPDE, iPdpt, iPaePDE, pMap->GCPtr, R3STRING(pMap->pszDesc) ));
+
+                iPaePDE++;
+                AssertFatal(iPaePDE < 512);
+
+                AssertMsg(pShwPaePd->a[iPaePDE].u == (PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | pMap->aPTs[i].HCPhysPaePT1),
+                          ("Expected %RX64 vs %RX64; iPDE=%#x iPdpt=%#x iPaePDE=%#x %RGv %s\n",
+                           pShwPaePd->a[iPaePDE].u,  (PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | pMap->aPTs[i].HCPhysPaePT1),
+                           iPDE, iPdpt, iPaePDE, pMap->GCPtr, R3STRING(pMap->pszDesc) ));
+
+                AssertMsg(pShwPdpt->a[iPdpt].u & PGM_PLXFLAGS_MAPPING,
+                          ("%RX64; iPdpt=%#x iPDE=%#x iPaePDE=%#x %RGv %s\n",
+                           pShwPdpt->a[iPdpt].u,
+                           iPDE, iPdpt, iPaePDE, pMap->GCPtr, R3STRING(pMap->pszDesc) ));
+
+                PCPGMPOOLPAGE   pPoolPagePd = pgmPoolGetPage(pPool, pShwPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
+                AssertFatal(pPoolPagePd);
+                AssertMsg(pPoolPagePd->cLocked, (".idx=%d .type=%d\n", pPoolPagePd->idx, pPoolPagePd->enmKind));
+                break;
+            }
+
+            default:
+                AssertFailed();
+                break;
+        }
+    }
+}
+
+
+/**
+ * Check the hypervisor mappings in the active CR3.
+ *
+ * Ignored if mappings are disabled (i.e. if HM is enabled).
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+VMMDECL(void) PGMMapCheck(PVM pVM)
+{
+    /*
+     * Can skip this if mappings are disabled.
+     */
+    if (!pgmMapAreMappingsEnabled(pVM))
+        return;
+
+    /* This only applies to raw mode where we only support 1 VCPU. */
+    Assert(pVM->cCpus == 1);
+    PVMCPU pVCpu = VMMGetCpu0(pVM);
+    Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+
+    /*
+     * Iterate mappings.
+     */
+    pgmLock(pVM);                           /* to avoid assertions */
+    for (PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings); pCur; pCur = pCur->CTX_SUFF(pNext))
+    {
+        unsigned iPDE = pCur->GCPtr >> X86_PD_SHIFT;
+        pgmMapCheckShadowPDEs(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pCur, iPDE);
+    }
+    pgmUnlock(pVM);
+}
+
+
+# endif /* defined(VBOX_STRICT) && !defined(IN_RING0) */
+
+/**
+ * Apply the hypervisor mappings to the active CR3.
+ *
+ * Ignored if mappings are disabled (i.e. if HM is enabled).
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pShwPageCR3 CR3 root page
+ */
+int pgmMapActivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3)
+{
+    RT_NOREF_PV(pShwPageCR3);
+
+    /*
+     * Skip this if it doesn't apply.
+     */
+    if (!pgmMapAreMappingsEnabled(pVM))
+        return VINF_SUCCESS;
+
+    /* Note! This might not be logged successfully in RC because we usually
+             cannot flush the log at this point. */
+    Log4(("pgmMapActivateCR3: fixed mappings=%RTbool idxShwPageCR3=%#x\n", pVM->pgm.s.fMappingsFixed, pShwPageCR3 ? pShwPageCR3->idx : NIL_PGMPOOL_IDX));
+
+# ifdef VBOX_STRICT
+    PVMCPU pVCpu = VMMGetCpu0(pVM);
+    Assert(pShwPageCR3 && pShwPageCR3 == pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+# endif
+
+    /*
+     * Iterate mappings.
+     */
+    for (PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings); pCur; pCur = pCur->CTX_SUFF(pNext))
+    {
+        unsigned iPDE = pCur->GCPtr >> X86_PD_SHIFT;
+        pgmMapSetShadowPDEs(pVM, pCur, iPDE);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Remove the hypervisor mappings from the specified CR3
+ *
+ * Ignored if mappings are disabled (i.e. if HM is enabled).
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pShwPageCR3 CR3 root page
+ */
+int pgmMapDeactivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3)
+{
+    /*
+     * Skip this if it doesn't apply.
+     */
+    if (!pgmMapAreMappingsEnabled(pVM))
+        return VINF_SUCCESS;
+
+    Assert(pShwPageCR3);
+    Log4(("pgmMapDeactivateCR3: fixed mappings=%d idxShwPageCR3=%#x\n", pVM->pgm.s.fMappingsFixed, pShwPageCR3 ? pShwPageCR3->idx : NIL_PGMPOOL_IDX));
+
+    /*
+     * Iterate mappings.
+     */
+    for (PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings); pCur; pCur = pCur->CTX_SUFF(pNext))
+    {
+        unsigned iPDE = pCur->GCPtr >> X86_PD_SHIFT;
+        pgmMapClearShadowPDEs(pVM, pShwPageCR3, pCur, iPDE, true /*fDeactivateCR3*/);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks guest PD for conflicts with VMM GC mappings.
+ *
+ * @returns true if conflict detected.
+ * @returns false if not.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMDECL(bool) PGMMapHasConflicts(PVM pVM)
+{
+    /*
+     * Can skip this if mappings are safely fixed.
+     */
+    if (!pgmMapAreMappingsFloating(pVM))
+        return false;
+    AssertReturn(pgmMapAreMappingsEnabled(pVM), false);
+
+    /* This only applies to raw mode where we only support 1 VCPU. */
+    PVMCPU pVCpu = &VMCC_GET_CPU_0(pVM);
+
+    PGMMODE const enmGuestMode = PGMGetGuestMode(pVCpu);
+    Assert(enmGuestMode <= PGMMODE_PAE_NX);
+
+    /*
+     * Iterate mappings.
+     */
+    if (enmGuestMode == PGMMODE_32_BIT)
+    {
+        /*
+         * Resolve the page directory.
+         */
+        PX86PD pPD = pgmGstGet32bitPDPtr(pVCpu);
+        Assert(pPD);
+
+        for (PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings); pCur; pCur = pCur->CTX_SUFF(pNext))
+        {
+            unsigned iPDE = pCur->GCPtr >> X86_PD_SHIFT;
+            unsigned iPT = pCur->cPTs;
+            while (iPT-- > 0)
+                if (pPD->a[iPDE + iPT].n.u1Present /** @todo PGMGstGetPDE. */)
+                {
+                    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3DetectedConflicts);
+
+# ifdef IN_RING3
+                    Log(("PGMHasMappingConflicts: Conflict was detected at %08RX32 for mapping %s (32 bits)\n"
+                         "                        iPDE=%#x iPT=%#x PDE=%RGp.\n",
+                        (iPT + iPDE) << X86_PD_SHIFT, pCur->pszDesc,
+                        iPDE, iPT, pPD->a[iPDE + iPT].au32[0]));
+# else
+                    Log(("PGMHasMappingConflicts: Conflict was detected at %08RX32 for mapping (32 bits)\n"
+                         "                        iPDE=%#x iPT=%#x PDE=%RGp.\n",
+                        (iPT + iPDE) << X86_PD_SHIFT,
+                        iPDE, iPT, pPD->a[iPDE + iPT].au32[0]));
+# endif
+                    return true;
+                }
+        }
+    }
+    else if (   enmGuestMode == PGMMODE_PAE
+             || enmGuestMode == PGMMODE_PAE_NX)
+    {
+        for (PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings); pCur; pCur = pCur->CTX_SUFF(pNext))
+        {
+            RTGCPTR   GCPtr = pCur->GCPtr;
+
+            unsigned  iPT = pCur->cb >> X86_PD_PAE_SHIFT;
+            while (iPT-- > 0)
+            {
+                X86PDEPAE Pde = pgmGstGetPaePDE(pVCpu, GCPtr);
+
+                if (Pde.n.u1Present)
+                {
+                    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3DetectedConflicts);
+# ifdef IN_RING3
+                    Log(("PGMHasMappingConflicts: Conflict was detected at %RGv for mapping %s (PAE)\n"
+                         "                        PDE=%016RX64.\n",
+                        GCPtr, pCur->pszDesc, Pde.u));
+# else
+                    Log(("PGMHasMappingConflicts: Conflict was detected at %RGv for mapping (PAE)\n"
+                         "                        PDE=%016RX64.\n",
+                        GCPtr, Pde.u));
+# endif
+                    return true;
+                }
+                GCPtr += (1 << X86_PD_PAE_SHIFT);
+            }
+        }
+    }
+    else
+        AssertFailed();
+
+    return false;
+}
+
+
+/**
+ * Checks and resolves (ring 3 only) guest conflicts with the guest mappings.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ */
+int pgmMapResolveConflicts(PVM pVM)
+{
+    /* The caller is expected to check these two conditions. */
+    Assert(!pVM->pgm.s.fMappingsFixed);
+    Assert(pgmMapAreMappingsEnabled(pVM));
+
+    /* This only applies to raw mode where we only support 1 VCPU. */
+    Assert(pVM->cCpus == 1);
+    PVMCPU          pVCpu        = &VMCC_GET_CPU_0(pVM);
+    PGMMODE const   enmGuestMode = PGMGetGuestMode(pVCpu);
+    Assert(enmGuestMode <= PGMMODE_PAE_NX);
+
+    if (enmGuestMode == PGMMODE_32_BIT)
+    {
+        /*
+         * Resolve the page directory.
+         */
+        PX86PD pPD = pgmGstGet32bitPDPtr(pVCpu);
+        Assert(pPD);
+
+        /*
+         * Iterate mappings.
+         */
+        for (PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings); pCur; )
+        {
+            PPGMMAPPING pNext = pCur->CTX_SUFF(pNext);
+            unsigned    iPDE  = pCur->GCPtr >> X86_PD_SHIFT;
+            unsigned    iPT   = pCur->cPTs;
+            while (iPT-- > 0)
+            {
+                if (pPD->a[iPDE + iPT].n.u1Present /** @todo PGMGstGetPDE. */)
+                {
+                    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3DetectedConflicts);
+
+# ifdef IN_RING3
+                    Log(("PGMHasMappingConflicts: Conflict was detected at %08RX32 for mapping %s (32 bits)\n"
+                         "                        iPDE=%#x iPT=%#x PDE=%RGp.\n",
+                         (iPT + iPDE) << X86_PD_SHIFT, pCur->pszDesc,
+                         iPDE, iPT, pPD->a[iPDE + iPT].au32[0]));
+                    int rc = pgmR3SyncPTResolveConflict(pVM, pCur, pPD, iPDE << X86_PD_SHIFT);
+                    AssertRCReturn(rc, rc);
+                    break;
+# else
+                    Log(("PGMHasMappingConflicts: Conflict was detected at %08RX32 for mapping (32 bits)\n"
+                         "                        iPDE=%#x iPT=%#x PDE=%RGp.\n",
+                         (iPT + iPDE) << X86_PD_SHIFT,
+                         iPDE, iPT, pPD->a[iPDE + iPT].au32[0]));
+                    return VINF_PGM_SYNC_CR3;
+# endif
+                }
+            }
+            pCur = pNext;
+        }
+    }
+    else if (   enmGuestMode == PGMMODE_PAE
+             || enmGuestMode == PGMMODE_PAE_NX)
+    {
+        /*
+         * Iterate mappings.
+         */
+        for (PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings); pCur;)
+        {
+            PPGMMAPPING pNext = pCur->CTX_SUFF(pNext);
+            RTGCPTR     GCPtr = pCur->GCPtr;
+            unsigned    iPT   = pCur->cb >> X86_PD_PAE_SHIFT;
+            while (iPT-- > 0)
+            {
+                X86PDEPAE Pde = pgmGstGetPaePDE(pVCpu, GCPtr);
+
+                if (Pde.n.u1Present)
+                {
+                    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3DetectedConflicts);
+# ifdef IN_RING3
+                    Log(("PGMHasMappingConflicts: Conflict was detected at %RGv for mapping %s (PAE)\n"
+                         "                        PDE=%016RX64.\n",
+                         GCPtr, pCur->pszDesc, Pde.u));
+                    int rc = pgmR3SyncPTResolveConflictPAE(pVM, pCur, pCur->GCPtr);
+                    AssertRCReturn(rc, rc);
+                    break;
+# else
+                    Log(("PGMHasMappingConflicts: Conflict was detected at %RGv for mapping (PAE)\n"
+                         "                        PDE=%016RX64.\n",
+                         GCPtr, Pde.u));
+                    return VINF_PGM_SYNC_CR3;
+# endif
+                }
+                GCPtr += (1 << X86_PD_PAE_SHIFT);
+            }
+            pCur = pNext;
+        }
+    }
+    else
+        AssertFailed();
+
+    Assert(!PGMMapHasConflicts(pVM));
+    return VINF_SUCCESS;
+}
+
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
diff --git a/src/VBox/VMM/VMMAll/PGMAllPhys.cpp b/src/VBox/VMM/VMMAll/PGMAllPhys.cpp
new file mode 100644
index 00000000..cef90ce6
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllPhys.cpp
@@ -0,0 +1,4729 @@
+/* $Id: PGMAllPhys.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor, Physical Memory Addressing.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_PHYS
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/nem.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include "PGMInline.h"
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/asm-amd64-x86.h>
+#include <VBox/log.h>
+#ifdef IN_RING3
+# include <iprt/thread.h>
+#endif
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** Enable the physical TLB. */
+#define PGM_WITH_PHYS_TLB
+
+/** @def PGM_HANDLER_PHYS_IS_VALID_STATUS
+ * Checks if valid physical access handler return code (normal handler, not PF).
+ *
+ * Checks if the given strict status code is one of the expected ones for a
+ * physical access handler in the current context.
+ *
+ * @returns true or false.
+ * @param   a_rcStrict      The status code.
+ * @param   a_fWrite        Whether it is a write or read being serviced.
+ *
+ * @remarks We wish to keep the list of statuses here as short as possible.
+ *          When changing, please make sure to update the PGMPhysRead,
+ *          PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
+ */
+#ifdef IN_RING3
+# define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
+    (   (a_rcStrict) == VINF_SUCCESS \
+     || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
+#elif defined(IN_RING0)
+#define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
+    (   (a_rcStrict) == VINF_SUCCESS \
+     || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT \
+     \
+     || (a_rcStrict) == ((a_fWrite) ? VINF_IOM_R3_MMIO_WRITE : VINF_IOM_R3_MMIO_READ) \
+     || (a_rcStrict) == VINF_IOM_R3_MMIO_READ_WRITE \
+     || ((a_rcStrict) == VINF_IOM_R3_MMIO_COMMIT_WRITE && (a_fWrite)) \
+     \
+     || (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR  \
+     || (a_rcStrict) == VINF_EM_DBG_STOP \
+     || (a_rcStrict) == VINF_EM_DBG_EVENT \
+     || (a_rcStrict) == VINF_EM_DBG_BREAKPOINT \
+     || (a_rcStrict) == VINF_EM_OFF \
+     || (a_rcStrict) == VINF_EM_SUSPEND \
+     || (a_rcStrict) == VINF_EM_RESET \
+    )
+#else
+# error "Context?"
+#endif
+
+/** @def PGM_HANDLER_VIRT_IS_VALID_STATUS
+ * Checks if valid virtual access handler return code (normal handler, not PF).
+ *
+ * Checks if the given strict status code is one of the expected ones for a
+ * virtual access handler in the current context.
+ *
+ * @returns true or false.
+ * @param   a_rcStrict      The status code.
+ * @param   a_fWrite        Whether it is a write or read being serviced.
+ *
+ * @remarks We wish to keep the list of statuses here as short as possible.
+ *          When changing, please make sure to update the PGMPhysRead,
+ *          PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
+ */
+#ifdef IN_RING3
+# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
+    (   (a_rcStrict) == VINF_SUCCESS \
+     || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
+#elif defined(IN_RING0)
+# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
+    (false /* no virtual handlers in ring-0! */ )
+#else
+# error "Context?"
+#endif
+
+
+
+#ifndef IN_RING3
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER,
+ *      Dummy for forcing ring-3 handling of the access.}
+ */
+DECLEXPORT(VBOXSTRICTRC)
+pgmPhysHandlerRedirectToHC(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
+                           PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
+{
+    NOREF(pVM); NOREF(pVCpu); NOREF(GCPhys); NOREF(pvPhys); NOREF(pvBuf); NOREF(cbBuf);
+    NOREF(enmAccessType); NOREF(enmOrigin); NOREF(pvUser);
+    return VINF_EM_RAW_EMULATE_INSTR;
+}
+
+
+/**
+ * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
+ *      Dummy for forcing ring-3 handling of the access.}
+ */
+VMMDECL(VBOXSTRICTRC) pgmPhysPfHandlerRedirectToHC(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
+                                                   RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
+{
+    NOREF(pVM); NOREF(pVCpu); NOREF(uErrorCode); NOREF(pRegFrame); NOREF(pvFault); NOREF(GCPhysFault); NOREF(pvUser);
+    return VINF_EM_RAW_EMULATE_INSTR;
+}
+
+
+/**
+ * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
+ *      \#PF access handler callback for guest ROM range write access.}
+ *
+ * @remarks The @a pvUser argument points to the PGMROMRANGE.
+ */
+DECLEXPORT(VBOXSTRICTRC) pgmPhysRomWritePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
+                                                  RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
+{
+    int             rc;
+    PPGMROMRANGE    pRom = (PPGMROMRANGE)pvUser;
+    uint32_t        iPage = (GCPhysFault - pRom->GCPhys) >> PAGE_SHIFT;
+    NOREF(uErrorCode); NOREF(pvFault);
+
+    Assert(uErrorCode & X86_TRAP_PF_RW); /* This shall not be used for read access! */
+
+    Assert(iPage < (pRom->cb >> PAGE_SHIFT));
+    switch (pRom->aPages[iPage].enmProt)
+    {
+        case PGMROMPROT_READ_ROM_WRITE_IGNORE:
+        case PGMROMPROT_READ_RAM_WRITE_IGNORE:
+        {
+            /*
+             * If it's a simple instruction which doesn't change the cpu state
+             * we will simply skip it. Otherwise we'll have to defer it to REM.
+             */
+            uint32_t     cbOp;
+            PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
+            rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
+            if (     RT_SUCCESS(rc)
+                &&   pDis->uCpuMode == DISCPUMODE_32BIT  /** @todo why does this matter? */
+                &&  !(pDis->fPrefix & (DISPREFIX_REPNE | DISPREFIX_REP | DISPREFIX_SEG)))
+            {
+                switch (pDis->bOpCode)
+                {
+                    /** @todo Find other instructions we can safely skip, possibly
+                     * adding this kind of detection to DIS or EM. */
+                    case OP_MOV:
+                        pRegFrame->rip += cbOp;
+                        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestROMWriteHandled);
+                        return VINF_SUCCESS;
+                }
+            }
+            break;
+        }
+
+        case PGMROMPROT_READ_RAM_WRITE_RAM:
+            pRom->aPages[iPage].LiveSave.fWrittenTo = true;
+            rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & X86_PTE_PG_MASK);
+            AssertRC(rc);
+            break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */
+
+        case PGMROMPROT_READ_ROM_WRITE_RAM:
+            /* Handle it in ring-3 because it's *way* easier there. */
+            pRom->aPages[iPage].LiveSave.fWrittenTo = true;
+            break;
+
+        default:
+            AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
+                                   pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
+                                  VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestROMWriteUnhandled);
+    return VINF_EM_RAW_EMULATE_INSTR;
+}
+
+#endif /* !IN_RING3 */
+
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER,
+ *      Access handler callback for ROM write accesses.}
+ *
+ * @remarks The @a pvUser argument points to the PGMROMRANGE.
+ */
+PGM_ALL_CB2_DECL(VBOXSTRICTRC)
+pgmPhysRomWriteHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
+                       PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
+{
+    PPGMROMRANGE    pRom     = (PPGMROMRANGE)pvUser;
+    const uint32_t  iPage    = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
+    Assert(iPage < (pRom->cb >> PAGE_SHIFT));
+    PPGMROMPAGE     pRomPage = &pRom->aPages[iPage];
+    Log5(("pgmPhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
+    NOREF(pVCpu); NOREF(pvPhys); NOREF(enmOrigin);
+
+    if (enmAccessType == PGMACCESSTYPE_READ)
+    {
+        switch (pRomPage->enmProt)
+        {
+            /*
+             * Take the default action.
+             */
+            case PGMROMPROT_READ_ROM_WRITE_IGNORE:
+            case PGMROMPROT_READ_RAM_WRITE_IGNORE:
+            case PGMROMPROT_READ_ROM_WRITE_RAM:
+            case PGMROMPROT_READ_RAM_WRITE_RAM:
+                return VINF_PGM_HANDLER_DO_DEFAULT;
+
+            default:
+                AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
+                                       pRom->aPages[iPage].enmProt, iPage, GCPhys),
+                                      VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+        }
+    }
+    else
+    {
+        Assert(enmAccessType == PGMACCESSTYPE_WRITE);
+        switch (pRomPage->enmProt)
+        {
+            /*
+             * Ignore writes.
+             */
+            case PGMROMPROT_READ_ROM_WRITE_IGNORE:
+            case PGMROMPROT_READ_RAM_WRITE_IGNORE:
+                return VINF_SUCCESS;
+
+            /*
+             * Write to the RAM page.
+             */
+            case PGMROMPROT_READ_ROM_WRITE_RAM:
+            case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's *way* simpler that way. */
+            {
+                /* This should be impossible now, pvPhys doesn't work cross page anylonger. */
+                Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
+
+                /*
+                 * Take the lock, do lazy allocation, map the page and copy the data.
+                 *
+                 * Note that we have to bypass the mapping TLB since it works on
+                 * guest physical addresses and entering the shadow page would
+                 * kind of screw things up...
+                 */
+                int rc = pgmLock(pVM);
+                AssertRC(rc);
+
+                PPGMPAGE pShadowPage = &pRomPage->Shadow;
+                if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
+                {
+                    pShadowPage = pgmPhysGetPage(pVM, GCPhys);
+                    AssertLogRelReturn(pShadowPage, VERR_PGM_PHYS_PAGE_GET_IPE);
+                }
+
+                void *pvDstPage;
+                rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
+                if (RT_SUCCESS(rc))
+                {
+                    memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
+                    pRomPage->LiveSave.fWrittenTo = true;
+
+                    AssertMsg(    rc == VINF_SUCCESS
+                              || (  rc == VINF_PGM_SYNC_CR3
+                                  && VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
+                              , ("%Rrc\n", rc));
+                    rc = VINF_SUCCESS;
+                }
+
+                pgmUnlock(pVM);
+                return rc;
+            }
+
+            default:
+                AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
+                                       pRom->aPages[iPage].enmProt, iPage, GCPhys),
+                                      VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+        }
+    }
+}
+
+
+/**
+ * Invalidates the RAM range TLBs.
+ *
+ * @param   pVM                 The cross context VM structure.
+ */
+void pgmPhysInvalidRamRangeTlbs(PVMCC pVM)
+{
+    pgmLock(pVM);
+    RT_ZERO(pVM->pgm.s.apRamRangesTlbR3);
+    RT_ZERO(pVM->pgm.s.apRamRangesTlbR0);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Tests if a value of type RTGCPHYS is negative if the type had been signed
+ * instead of unsigned.
+ *
+ * @returns @c true if negative, @c false if positive or zero.
+ * @param   a_GCPhys        The value to test.
+ * @todo    Move me to iprt/types.h.
+ */
+#define RTGCPHYS_IS_NEGATIVE(a_GCPhys)  ((a_GCPhys) & ((RTGCPHYS)1 << (sizeof(RTGCPHYS)*8 - 1)))
+
+
+/**
+ * Slow worker for pgmPhysGetRange.
+ *
+ * @copydoc pgmPhysGetRange
+ */
+PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
+
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
+    while (pRam)
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            return pRam;
+        }
+        if (RTGCPHYS_IS_NEGATIVE(off))
+            pRam = pRam->CTX_SUFF(pLeft);
+        else
+            pRam = pRam->CTX_SUFF(pRight);
+    }
+    return NULL;
+}
+
+
+/**
+ * Slow worker for pgmPhysGetRangeAtOrAbove.
+ *
+ * @copydoc pgmPhysGetRangeAtOrAbove
+ */
+PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
+
+    PPGMRAMRANGE pLastLeft = NULL;
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
+    while (pRam)
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            return pRam;
+        }
+        if (RTGCPHYS_IS_NEGATIVE(off))
+        {
+            pLastLeft = pRam;
+            pRam = pRam->CTX_SUFF(pLeft);
+        }
+        else
+            pRam = pRam->CTX_SUFF(pRight);
+    }
+    return pLastLeft;
+}
+
+
+/**
+ * Slow worker for pgmPhysGetPage.
+ *
+ * @copydoc pgmPhysGetPage
+ */
+PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
+
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
+    while (pRam)
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            return &pRam->aPages[off >> PAGE_SHIFT];
+        }
+
+        if (RTGCPHYS_IS_NEGATIVE(off))
+            pRam = pRam->CTX_SUFF(pLeft);
+        else
+            pRam = pRam->CTX_SUFF(pRight);
+    }
+    return NULL;
+}
+
+
+/**
+ * Slow worker for pgmPhysGetPageEx.
+ *
+ * @copydoc pgmPhysGetPageEx
+ */
+int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
+
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
+    while (pRam)
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+            return VINF_SUCCESS;
+        }
+
+        if (RTGCPHYS_IS_NEGATIVE(off))
+            pRam = pRam->CTX_SUFF(pLeft);
+        else
+            pRam = pRam->CTX_SUFF(pRight);
+    }
+
+    *ppPage = NULL;
+    return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+}
+
+
+/**
+ * Slow worker for pgmPhysGetPageAndRangeEx.
+ *
+ * @copydoc pgmPhysGetPageAndRangeEx
+ */
+int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
+
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
+    while (pRam)
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            *ppRam  = pRam;
+            *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+            return VINF_SUCCESS;
+        }
+
+        if (RTGCPHYS_IS_NEGATIVE(off))
+            pRam = pRam->CTX_SUFF(pLeft);
+        else
+            pRam = pRam->CTX_SUFF(pRight);
+    }
+
+    *ppRam  = NULL;
+    *ppPage = NULL;
+    return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+}
+
+
+/**
+ * Checks if Address Gate 20 is enabled or not.
+ *
+ * @returns true if enabled.
+ * @returns false if disabled.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu)
+{
+    LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu->pgm.s.fA20Enabled));
+    return pVCpu->pgm.s.fA20Enabled;
+}
+
+
+/**
+ * Validates a GC physical address.
+ *
+ * @returns true if valid.
+ * @returns false if invalid.
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPhys  The physical address to validate.
+ */
+VMMDECL(bool) PGMPhysIsGCPhysValid(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+    return pPage != NULL;
+}
+
+
+/**
+ * Checks if a GC physical address is a normal page,
+ * i.e. not ROM, MMIO or reserved.
+ *
+ * @returns true if normal.
+ * @returns false if invalid, ROM, MMIO or reserved page.
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPhys  The physical address to check.
+ */
+VMMDECL(bool) PGMPhysIsGCPhysNormal(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+    return pPage
+        && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
+}
+
+
+/**
+ * Converts a GC physical address to a HC physical address.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
+ *          page but has no physical backing.
+ * @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
+ *          GC physical address.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPhys  The GC physical address to convert.
+ * @param   pHCPhys Where to store the HC physical address on success.
+ */
+VMM_INT_DECL(int) PGMPhysGCPhys2HCPhys(PVMCC pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
+{
+    pgmLock(pVM);
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+    if (RT_SUCCESS(rc))
+        *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) | (GCPhys & PAGE_OFFSET_MASK);
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Invalidates all page mapping TLBs.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+void pgmPhysInvalidatePageMapTLB(PVMCC pVM)
+{
+    pgmLock(pVM);
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushes);
+
+    /* Clear the R3 & R0 TLBs completely. */
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR0.aEntries); i++)
+    {
+        pVM->pgm.s.PhysTlbR0.aEntries[i].GCPhys = NIL_RTGCPHYS;
+        pVM->pgm.s.PhysTlbR0.aEntries[i].pPage = 0;
+#ifndef VBOX_WITH_RAM_IN_KERNEL
+        pVM->pgm.s.PhysTlbR0.aEntries[i].pMap = 0;
+#endif
+        pVM->pgm.s.PhysTlbR0.aEntries[i].pv = 0;
+    }
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR3.aEntries); i++)
+    {
+        pVM->pgm.s.PhysTlbR3.aEntries[i].GCPhys = NIL_RTGCPHYS;
+        pVM->pgm.s.PhysTlbR3.aEntries[i].pPage = 0;
+        pVM->pgm.s.PhysTlbR3.aEntries[i].pMap = 0;
+        pVM->pgm.s.PhysTlbR3.aEntries[i].pv = 0;
+    }
+
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Invalidates a page mapping TLB entry
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPhys  GCPhys entry to flush
+ */
+void pgmPhysInvalidatePageMapTLBEntry(PVM pVM, RTGCPHYS GCPhys)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushEntry);
+
+    unsigned const idx = PGM_PAGER3MAPTLB_IDX(GCPhys);
+
+    pVM->pgm.s.PhysTlbR0.aEntries[idx].GCPhys = NIL_RTGCPHYS;
+    pVM->pgm.s.PhysTlbR0.aEntries[idx].pPage = 0;
+#ifndef VBOX_WITH_RAM_IN_KERNEL
+    pVM->pgm.s.PhysTlbR0.aEntries[idx].pMap = 0;
+#endif
+    pVM->pgm.s.PhysTlbR0.aEntries[idx].pv = 0;
+
+    pVM->pgm.s.PhysTlbR3.aEntries[idx].GCPhys = NIL_RTGCPHYS;
+    pVM->pgm.s.PhysTlbR3.aEntries[idx].pPage = 0;
+    pVM->pgm.s.PhysTlbR3.aEntries[idx].pMap = 0;
+    pVM->pgm.s.PhysTlbR3.aEntries[idx].pv = 0;
+}
+
+
+/**
+ * Makes sure that there is at least one handy page ready for use.
+ *
+ * This will also take the appropriate actions when reaching water-marks.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_EM_NO_MEMORY if we're really out of memory.
+ *
+ * @param   pVM     The cross context VM structure.
+ *
+ * @remarks Must be called from within the PGM critical section. It may
+ *          nip back to ring-3/0 in some cases.
+ */
+static int pgmPhysEnsureHandyPage(PVMCC pVM)
+{
+    AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
+
+    /*
+     * Do we need to do anything special?
+     */
+#ifdef IN_RING3
+    if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
+#else
+    if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
+#endif
+    {
+        /*
+         * Allocate pages only if we're out of them, or in ring-3, almost out.
+         */
+#ifdef IN_RING3
+        if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
+#else
+        if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
+#endif
+        {
+            Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n",
+                 pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY) ));
+#ifdef IN_RING3
+            int rc = PGMR3PhysAllocateHandyPages(pVM);
+#else
+            int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES, 0);
+#endif
+            if (RT_UNLIKELY(rc != VINF_SUCCESS))
+            {
+                if (RT_FAILURE(rc))
+                    return rc;
+                AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                if (!pVM->pgm.s.cHandyPages)
+                {
+                    LogRel(("PGM: no more handy pages!\n"));
+                    return VERR_EM_NO_MEMORY;
+                }
+                Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
+                Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY));
+#ifndef IN_RING3
+                VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
+#endif
+            }
+            AssertMsgReturn(    pVM->pgm.s.cHandyPages > 0
+                            &&  pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
+                            ("%u\n", pVM->pgm.s.cHandyPages),
+                            VERR_PGM_HANDY_PAGE_IPE);
+        }
+        else
+        {
+            if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
+                VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
+#ifndef IN_RING3
+            if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
+            {
+                Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
+                VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
+            }
+#endif
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Replace a zero or shared page with new page that we can write to.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success, pPage is modified.
+ * @retval  VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
+ * @retval  VERR_EM_NO_MEMORY if we're totally out of memory.
+ *
+ * @todo    Propagate VERR_EM_NO_MEMORY up the call tree.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure. This will
+ *                      be modified on success.
+ * @param   GCPhys      The address of the page.
+ *
+ * @remarks Must be called from within the PGM critical section. It may
+ *          nip back to ring-3/0 in some cases.
+ *
+ * @remarks This function shouldn't really fail, however if it does
+ *          it probably means we've screwed up the size of handy pages and/or
+ *          the low-water mark. Or, that some device I/O is causing a lot of
+ *          pages to be allocated while while the host is in a low-memory
+ *          condition. This latter should be handled elsewhere and in a more
+ *          controlled manner, it's on the @bugref{3170} todo list...
+ */
+int pgmPhysAllocPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
+{
+    LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys));
+
+    /*
+     * Prereqs.
+     */
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    AssertMsg(PGM_PAGE_IS_ZERO(pPage) || PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
+    Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
+
+# ifdef PGM_WITH_LARGE_PAGES
+    /*
+     * Try allocate a large page if applicable.
+     */
+    if (   PGMIsUsingLargePages(pVM)
+        && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
+        && !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Implement large pages support. */
+    {
+        RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
+        PPGMPAGE pBasePage;
+
+        int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pBasePage);
+        AssertRCReturn(rc, rc);     /* paranoia; can't happen. */
+        if (PGM_PAGE_GET_PDE_TYPE(pBasePage) == PGM_PAGE_PDE_TYPE_DONTCARE)
+        {
+            rc = pgmPhysAllocLargePage(pVM, GCPhys);
+            if (rc == VINF_SUCCESS)
+                return rc;
+        }
+        /* Mark the base as type page table, so we don't check over and over again. */
+        PGM_PAGE_SET_PDE_TYPE(pVM, pBasePage, PGM_PAGE_PDE_TYPE_PT);
+
+        /* fall back to 4KB pages. */
+    }
+# endif
+
+    /*
+     * Flush any shadow page table mappings of the page.
+     * When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any.
+     */
+    bool fFlushTLBs = false;
+    int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, true /*fFlushTLBs*/, &fFlushTLBs);
+    AssertMsgReturn(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS);
+
+    /*
+     * Ensure that we've got a page handy, take it and use it.
+     */
+    int rc2 = pgmPhysEnsureHandyPage(pVM);
+    if (RT_FAILURE(rc2))
+    {
+        if (fFlushTLBs)
+            PGM_INVL_ALL_VCPU_TLBS(pVM);
+        Assert(rc2 == VERR_EM_NO_MEMORY);
+        return rc2;
+    }
+    /* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    AssertMsg(PGM_PAGE_IS_ZERO(pPage) || PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
+    Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
+
+    uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
+    AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
+    Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_RTHCPHYS);
+    Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
+    Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
+    Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
+
+    /*
+     * There are one or two action to be taken the next time we allocate handy pages:
+     *      - Tell the GMM (global memory manager) what the page is being used for.
+     *        (Speeds up replacement operations - sharing and defragmenting.)
+     *      - If the current backing is shared, it must be freed.
+     */
+    const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
+    pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+
+    void const *pvSharedPage = NULL;
+    if (PGM_PAGE_IS_SHARED(pPage))
+    {
+        /* Mark this shared page for freeing/dereferencing. */
+        pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
+        Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
+
+        Log(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
+             GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageReplaceShared));
+        pVM->pgm.s.cSharedPages--;
+
+        /* Grab the address of the page so we can make a copy later on. (safe) */
+        rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvSharedPage);
+        AssertRC(rc);
+    }
+    else
+    {
+        Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
+        pVM->pgm.s.cZeroPages--;
+    }
+
+    /*
+     * Do the PGMPAGE modifications.
+     */
+    pVM->pgm.s.cPrivatePages++;
+    PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
+    PGM_PAGE_SET_PAGEID(pVM, pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
+    PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
+    PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PT);
+    pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
+
+    /* Copy the shared page contents to the replacement page. */
+    if (pvSharedPage)
+    {
+        /* Get the virtual address of the new page. */
+        PGMPAGEMAPLOCK  PgMpLck;
+        void           *pvNewPage;
+        rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvNewPage, &PgMpLck); AssertRC(rc);
+        if (RT_SUCCESS(rc))
+        {
+            memcpy(pvNewPage, pvSharedPage, PAGE_SIZE); /** @todo todo write ASMMemCopyPage */
+            pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+        }
+    }
+
+    if (    fFlushTLBs
+        &&  rc != VINF_PGM_GCPHYS_ALIASED)
+        PGM_INVL_ALL_VCPU_TLBS(pVM);
+
+    /*
+     * Notify NEM about the mapping change for this page.
+     *
+     * Note! Shadow ROM pages are complicated as they can definitely be
+     *       allocated while not visible, so play safe.
+     */
+    if (VM_IS_NEM_ENABLED(pVM))
+    {
+        PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+        if (   enmType != PGMPAGETYPE_ROM_SHADOW
+            || pgmPhysGetPage(pVM, GCPhys) == pPage)
+        {
+            uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+            rc2 = NEMHCNotifyPhysPageAllocated(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, HCPhys,
+                                               pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+            if (RT_SUCCESS(rc))
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            else
+                rc = rc2;
+        }
+    }
+
+    return rc;
+}
+
+#ifdef PGM_WITH_LARGE_PAGES
+
+/**
+ * Replace a 2 MB range of zero pages with new pages that we can write to.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success, pPage is modified.
+ * @retval  VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
+ * @retval  VERR_EM_NO_MEMORY if we're totally out of memory.
+ *
+ * @todo    Propagate VERR_EM_NO_MEMORY up the call tree.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The address of the page.
+ *
+ * @remarks Must be called from within the PGM critical section. It may
+ *          nip back to ring-3/0 in some cases.
+ */
+int pgmPhysAllocLargePage(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
+    LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase));
+    Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */
+
+    /*
+     * Prereqs.
+     */
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    Assert(PGMIsUsingLargePages(pVM));
+
+    PPGMPAGE pFirstPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pFirstPage);
+    if (    RT_SUCCESS(rc)
+        &&  PGM_PAGE_GET_TYPE(pFirstPage) == PGMPAGETYPE_RAM)
+    {
+        unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pFirstPage);
+
+        /* Don't call this function for already allocated pages. */
+        Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE);
+
+        if (   uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE
+            && PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ZERO)
+        {
+            /* Lazy approach: check all pages in the 2 MB range.
+             * The whole range must be ram and unallocated. */
+            GCPhys = GCPhysBase;
+            unsigned iPage;
+            for (iPage = 0; iPage < _2M/PAGE_SIZE; iPage++)
+            {
+                PPGMPAGE pSubPage;
+                rc = pgmPhysGetPageEx(pVM, GCPhys, &pSubPage);
+                if  (   RT_FAILURE(rc)
+                     || PGM_PAGE_GET_TYPE(pSubPage)  != PGMPAGETYPE_RAM      /* Anything other than ram implies monitoring. */
+                     || PGM_PAGE_GET_STATE(pSubPage) != PGM_PAGE_STATE_ZERO) /* Allocated, monitored or shared means we can't use a large page here */
+                {
+                    LogFlow(("Found page %RGp with wrong attributes (type=%d; state=%d); cancel check. rc=%d\n", GCPhys, PGM_PAGE_GET_TYPE(pSubPage), PGM_PAGE_GET_STATE(pSubPage), rc));
+                    break;
+                }
+                Assert(PGM_PAGE_GET_PDE_TYPE(pSubPage) == PGM_PAGE_PDE_TYPE_DONTCARE);
+                GCPhys += PAGE_SIZE;
+            }
+            if (iPage != _2M/PAGE_SIZE)
+            {
+                /* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */
+                STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
+                PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PT);
+                return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
+            }
+
+            /*
+             * Do the allocation.
+             */
+# ifdef IN_RING3
+            rc = PGMR3PhysAllocateLargeHandyPage(pVM, GCPhysBase);
+# else
+            rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE, GCPhysBase);
+# endif
+            if (RT_SUCCESS(rc))
+            {
+                Assert(PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ALLOCATED);
+                pVM->pgm.s.cLargePages++;
+                return VINF_SUCCESS;
+            }
+
+            /* If we fail once, it most likely means the host's memory is too
+               fragmented; don't bother trying again. */
+            LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
+            PGMSetLargePageUsage(pVM, false);
+            return rc;
+        }
+    }
+    return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
+}
+
+
+/**
+ * Recheck the entire 2 MB range to see if we can use it again as a large page.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success, the large page can be used again
+ * @retval  VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The address of the page.
+ * @param   pLargePage  Page structure of the base page
+ */
+int pgmPhysRecheckLargePage(PVMCC pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
+{
+    STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
+
+    Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */
+
+    GCPhys &= X86_PDE2M_PAE_PG_MASK;
+
+    /* Check the base page. */
+    Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
+    if (    PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED
+        ||  PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM
+        ||  PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
+    {
+        LogFlow(("pgmPhysRecheckLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage)));
+        return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
+    }
+
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
+    /* Check all remaining pages in the 2 MB range. */
+    unsigned i;
+    GCPhys += PAGE_SIZE;
+    for (i = 1; i < _2M/PAGE_SIZE; i++)
+    {
+        PPGMPAGE pPage;
+        int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+        AssertRCBreak(rc);
+
+        if (    PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+            ||  PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
+            ||  PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
+            ||  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
+        {
+            LogFlow(("pgmPhysRecheckLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)));
+            break;
+        }
+
+        GCPhys += PAGE_SIZE;
+    }
+    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
+
+    if (i == _2M/PAGE_SIZE)
+    {
+        PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE);
+        pVM->pgm.s.cLargePagesDisabled--;
+        Log(("pgmPhysRecheckLargePage: page %RGp can be reused!\n", GCPhys - _2M));
+        return VINF_SUCCESS;
+    }
+
+    return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
+}
+
+#endif /* PGM_WITH_LARGE_PAGES */
+
+
+/**
+ * Deal with a write monitored page.
+ *
+ * @returns VBox strict status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure.
+ * @param   GCPhys      The guest physical address of the page.
+ *                      PGMPhysReleasePageMappingLock() passes NIL_RTGCPHYS in a
+ *                      very unlikely situation where it is okay that we let NEM
+ *                      fix the page access in a lazy fasion.
+ *
+ * @remarks Called from within the PGM critical section.
+ */
+void pgmPhysPageMakeWriteMonitoredWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
+{
+    Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED);
+    PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
+    PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
+    Assert(pVM->pgm.s.cMonitoredPages > 0);
+    pVM->pgm.s.cMonitoredPages--;
+    pVM->pgm.s.cWrittenToPages++;
+
+    /*
+     * Notify NEM about the protection change so we won't spin forever.
+     *
+     * Note! NEM need to be handle to lazily correct page protection as we cannot
+     *       really get it 100% right here it seems.  The page pool does this too.
+     */
+    if (VM_IS_NEM_ENABLED(pVM) && GCPhys != NIL_RTGCPHYS)
+    {
+        uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+        PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+        NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
+                                       pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+        PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+    }
+}
+
+
+/**
+ * Deal with pages that are not writable, i.e. not in the ALLOCATED state.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure.
+ * @param   GCPhys      The address of the page.
+ *
+ * @remarks Called from within the PGM critical section.
+ */
+int pgmPhysPageMakeWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    switch (PGM_PAGE_GET_STATE(pPage))
+    {
+        case PGM_PAGE_STATE_WRITE_MONITORED:
+            pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
+            RT_FALL_THRU();
+        default: /* to shut up GCC */
+        case PGM_PAGE_STATE_ALLOCATED:
+            return VINF_SUCCESS;
+
+        /*
+         * Zero pages can be dummy pages for MMIO or reserved memory,
+         * so we need to check the flags before joining cause with
+         * shared page replacement.
+         */
+        case PGM_PAGE_STATE_ZERO:
+            if (PGM_PAGE_IS_MMIO(pPage))
+                return VERR_PGM_PHYS_PAGE_RESERVED;
+            RT_FALL_THRU();
+        case PGM_PAGE_STATE_SHARED:
+            return pgmPhysAllocPage(pVM, pPage, GCPhys);
+
+        /* Not allowed to write to ballooned pages. */
+        case PGM_PAGE_STATE_BALLOONED:
+            return VERR_PGM_PHYS_PAGE_BALLOONED;
+    }
+}
+
+
+/**
+ * Internal usage: Map the page specified by its GMM ID.
+ *
+ * This is similar to pgmPhysPageMap
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   idPage      The Page ID.
+ * @param   HCPhys      The physical address (for SUPR0HCPhysToVirt).
+ * @param   ppv         Where to store the mapping address.
+ *
+ * @remarks Called from within the PGM critical section.  The mapping is only
+ *          valid while you are inside this section.
+ */
+int pgmPhysPageMapByPageID(PVMCC pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
+{
+    /*
+     * Validation.
+     */
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    AssertReturn(HCPhys && !(HCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT;
+    AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER);
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    /*
+     * Map it by HCPhys.
+     */
+    return pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv  RTLOG_COMMA_SRC_POS);
+
+#elif defined(IN_RING0) && defined(VBOX_WITH_RAM_IN_KERNEL)
+# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
+    return SUPR0HCPhysToVirt(HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, ppv);
+# else
+    return GMMR0PageIdToVirt(pVM, idPage, ppv);
+# endif
+
+#else
+    /*
+     * Find/make Chunk TLB entry for the mapping chunk.
+     */
+    PPGMCHUNKR3MAP pMap;
+    PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
+    if (pTlbe->idChunk == idChunk)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
+        pMap = pTlbe->pChunk;
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
+
+        /*
+         * Find the chunk, map it if necessary.
+         */
+        pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
+        if (pMap)
+            pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
+        else
+        {
+# ifdef IN_RING0
+            int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
+            AssertRCReturn(rc, rc);
+            pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
+            Assert(pMap);
+# else
+            int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
+            if (RT_FAILURE(rc))
+                return rc;
+# endif
+        }
+
+        /*
+         * Enter it into the Chunk TLB.
+         */
+        pTlbe->idChunk = idChunk;
+        pTlbe->pChunk = pMap;
+    }
+
+    *ppv = (uint8_t *)pMap->pv + ((idPage &GMM_PAGEID_IDX_MASK) << PAGE_SHIFT);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/**
+ * Maps a page into the current virtual address space so it can be accessed.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure.
+ * @param   GCPhys      The address of the page.
+ * @param   ppMap       Where to store the address of the mapping tracking structure.
+ * @param   ppv         Where to store the mapping address of the page. The page
+ *                      offset is masked off!
+ *
+ * @remarks Called from within the PGM critical section.
+ */
+static int pgmPhysPageMapCommon(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    NOREF(GCPhys);
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    /*
+     * Just some sketchy GC/R0-darwin code.
+     */
+    *ppMap = NULL;
+    RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+    Assert(HCPhys != pVM->pgm.s.HCPhysZeroPg);
+    pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
+    return VINF_SUCCESS;
+
+#else /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+
+
+    /*
+     * Special cases: MMIO2, ZERO and specially aliased MMIO pages.
+     */
+    if (   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2
+        || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
+    {
+        /* Decode the page id to a page in a MMIO2 ram range. */
+        uint8_t  idMmio2 = PGM_MMIO2_PAGEID_GET_MMIO2_ID(PGM_PAGE_GET_PAGEID(pPage));
+        uint32_t iPage   = PGM_MMIO2_PAGEID_GET_IDX(PGM_PAGE_GET_PAGEID(pPage));
+        AssertLogRelMsgReturn((uint8_t)(idMmio2 - 1U) < RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)),
+                              ("idMmio2=%u size=%u type=%u GCPHys=%#RGp Id=%u State=%u", idMmio2,
+                               RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)), PGM_PAGE_GET_TYPE(pPage), GCPhys,
+                               pPage->s.idPage, pPage->s.uStateY),
+                              VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
+        PPGMREGMMIO2RANGE pMmio2Range = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[idMmio2 - 1];
+        AssertLogRelReturn(pMmio2Range, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
+        AssertLogRelReturn(pMmio2Range->idMmio2 == idMmio2, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
+        AssertLogRelReturn(iPage < (pMmio2Range->RamRange.cb >> PAGE_SHIFT), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
+        *ppMap = NULL;
+# if   defined(IN_RING0) && defined(VBOX_WITH_RAM_IN_KERNEL) && defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+        return SUPR0HCPhysToVirt(PGM_PAGE_GET_HCPHYS(pPage), ppv);
+# elif defined(IN_RING0) && defined(VBOX_WITH_RAM_IN_KERNEL)
+        *ppv = (uint8_t *)pMmio2Range->pvR0 + ((uintptr_t)iPage << PAGE_SHIFT);
+        return VINF_SUCCESS;
+# else
+        *ppv = (uint8_t *)pMmio2Range->RamRange.pvR3 + ((uintptr_t)iPage << PAGE_SHIFT);
+        return VINF_SUCCESS;
+# endif
+    }
+
+    const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
+    if (idChunk == NIL_GMM_CHUNKID)
+    {
+        AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage),
+                        VERR_PGM_PHYS_PAGE_MAP_IPE_1);
+        if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+        {
+            AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage),
+                            VERR_PGM_PHYS_PAGE_MAP_IPE_3);
+            AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage)== pVM->pgm.s.HCPhysZeroPg, ("pPage=%R[pgmpage]\n", pPage),
+                            VERR_PGM_PHYS_PAGE_MAP_IPE_4);
+            *ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
+        }
+        else
+            *ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
+        *ppMap = NULL;
+        return VINF_SUCCESS;
+    }
+
+# if defined(IN_RING0) && defined(VBOX_WITH_RAM_IN_KERNEL) && defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+    /*
+     * Just use the physical address.
+     */
+    *ppMap = NULL;
+    return SUPR0HCPhysToVirt(PGM_PAGE_GET_HCPHYS(pPage), ppv);
+
+# elif defined(IN_RING0) && defined(VBOX_WITH_RAM_IN_KERNEL)
+    /*
+     * Go by page ID thru GMMR0.
+     */
+    *ppMap = NULL;
+    return GMMR0PageIdToVirt(pVM, PGM_PAGE_GET_PAGEID(pPage), ppv);
+
+# else
+    /*
+     * Find/make Chunk TLB entry for the mapping chunk.
+     */
+    PPGMCHUNKR3MAP pMap;
+    PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
+    if (pTlbe->idChunk == idChunk)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
+        pMap = pTlbe->pChunk;
+        AssertPtr(pMap->pv);
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
+
+        /*
+         * Find the chunk, map it if necessary.
+         */
+        pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
+        if (pMap)
+        {
+            AssertPtr(pMap->pv);
+            pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
+        }
+        else
+        {
+# ifdef IN_RING0
+            int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
+            AssertRCReturn(rc, rc);
+            pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
+            Assert(pMap);
+# else
+            int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
+            if (RT_FAILURE(rc))
+                return rc;
+# endif
+            AssertPtr(pMap->pv);
+        }
+
+        /*
+         * Enter it into the Chunk TLB.
+         */
+        pTlbe->idChunk = idChunk;
+        pTlbe->pChunk = pMap;
+    }
+
+    *ppv = (uint8_t *)pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << PAGE_SHIFT);
+    *ppMap = pMap;
+    return VINF_SUCCESS;
+# endif /* !IN_RING0 || !VBOX_WITH_RAM_IN_KERNEL */
+#endif /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+}
+
+
+/**
+ * Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable.
+ *
+ * This is typically used is paths where we cannot use the TLB methods (like ROM
+ * pages) or where there is no point in using them since we won't get many hits.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure.
+ * @param   GCPhys      The address of the page.
+ * @param   ppv         Where to store the mapping address of the page. The page
+ *                      offset is masked off!
+ *
+ * @remarks Called from within the PGM critical section.  The mapping is only
+ *          valid while you are inside section.
+ */
+int pgmPhysPageMakeWritableAndMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
+{
+    int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+    if (RT_SUCCESS(rc))
+    {
+        AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc));
+        PPGMPAGEMAP pMapIgnore;
+        int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
+        if (RT_FAILURE(rc2)) /* preserve rc */
+            rc = rc2;
+    }
+    return rc;
+}
+
+
+/**
+ * Maps a page into the current virtual address space so it can be accessed for
+ * both writing and reading.
+ *
+ * This is typically used is paths where we cannot use the TLB methods (like ROM
+ * pages) or where there is no point in using them since we won't get many hits.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure. Must be in the
+ *                      allocated state.
+ * @param   GCPhys      The address of the page.
+ * @param   ppv         Where to store the mapping address of the page. The page
+ *                      offset is masked off!
+ *
+ * @remarks Called from within the PGM critical section.  The mapping is only
+ *          valid while you are inside section.
+ */
+int pgmPhysPageMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
+{
+    Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+    PPGMPAGEMAP pMapIgnore;
+    return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
+}
+
+
+/**
+ * Maps a page into the current virtual address space so it can be accessed for
+ * reading.
+ *
+ * This is typically used is paths where we cannot use the TLB methods (like ROM
+ * pages) or where there is no point in using them since we won't get many hits.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure.
+ * @param   GCPhys      The address of the page.
+ * @param   ppv         Where to store the mapping address of the page. The page
+ *                      offset is masked off!
+ *
+ * @remarks Called from within the PGM critical section.  The mapping is only
+ *          valid while you are inside this section.
+ */
+int pgmPhysPageMapReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
+{
+    PPGMPAGEMAP pMapIgnore;
+    return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
+}
+
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+
+/**
+ * Load a guest page into the ring-3 physical TLB.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address in question.
+ */
+int pgmPhysPageLoadIntoTlb(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Find the ram range and page and hand it over to the with-page function.
+     * 99.8% of requests are expected to be in the first range.
+     */
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+    if (!pPage)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
+        return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+    }
+
+    return pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
+}
+
+
+/**
+ * Load a guest page into the ring-3 physical TLB.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       Pointer to the PGMPAGE structure corresponding to
+ *                      GCPhys.
+ * @param   GCPhys      The guest physical address in question.
+ */
+int pgmPhysPageLoadIntoTlbWithPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
+
+    /*
+     * Map the page.
+     * Make a special case for the zero page as it is kind of special.
+     */
+    PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTX_SUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
+    if (    !PGM_PAGE_IS_ZERO(pPage)
+        &&  !PGM_PAGE_IS_BALLOONED(pPage))
+    {
+        void *pv;
+        PPGMPAGEMAP pMap;
+        int rc = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMap, &pv);
+        if (RT_FAILURE(rc))
+            return rc;
+# if !defined(IN_RING0) || !defined(VBOX_WITH_RAM_IN_KERNEL)
+        pTlbe->pMap = pMap;
+# endif
+        pTlbe->pv = pv;
+        Assert(!((uintptr_t)pTlbe->pv & PAGE_OFFSET_MASK));
+    }
+    else
+    {
+        AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg, ("%RGp/%R[pgmpage]\n", GCPhys, pPage));
+# if !defined(IN_RING0) || !defined(VBOX_WITH_RAM_IN_KERNEL)
+        pTlbe->pMap = NULL;
+# endif
+        pTlbe->pv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
+    }
+# ifdef PGM_WITH_PHYS_TLB
+    if (    PGM_PAGE_GET_TYPE(pPage) < PGMPAGETYPE_ROM_SHADOW
+        ||  PGM_PAGE_GET_TYPE(pPage) > PGMPAGETYPE_ROM)
+        pTlbe->GCPhys = GCPhys & X86_PTE_PAE_PG_MASK;
+    else
+        pTlbe->GCPhys = NIL_RTGCPHYS; /* ROM: Problematic because of the two pages. :-/ */
+# else
+    pTlbe->GCPhys = NIL_RTGCPHYS;
+# endif
+    pTlbe->pPage = pPage;
+    return VINF_SUCCESS;
+}
+
+#endif /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+
+/**
+ * Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
+ * own the PGM lock and therefore not need to lock the mapped page.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address of the page that should be mapped.
+ * @param   pPage       Pointer to the PGMPAGE structure for the page.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ *
+ * @internal
+ * @deprecated Use pgmPhysGCPhys2CCPtrInternalEx.
+ */
+int pgmPhysGCPhys2CCPtrInternalDepr(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
+{
+    int rc;
+    AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    pVM->pgm.s.cDeprecatedPageLocks++;
+
+    /*
+     * Make sure the page is writable.
+     */
+    if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
+    {
+        rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+        if (RT_FAILURE(rc))
+            return rc;
+        AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
+    }
+    Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
+
+    /*
+     * Get the mapping address.
+     */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    void *pv;
+    rc = pgmRZDynMapHCPageInlined(VMMGetCpu(pVM),
+                                  PGM_PAGE_GET_HCPHYS(pPage),
+                                  &pv
+                                  RTLOG_COMMA_SRC_POS);
+    if (RT_FAILURE(rc))
+        return rc;
+    *ppv = (void *)((uintptr_t)pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+#else
+    PPGMPAGEMAPTLBE pTlbe;
+    rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+    if (RT_FAILURE(rc))
+        return rc;
+    *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+#endif
+    return VINF_SUCCESS;
+}
+
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+
+/**
+ * Locks a page mapping for writing.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pPage               The page.
+ * @param   pTlbe               The mapping TLB entry for the page.
+ * @param   pLock               The lock structure (output).
+ */
+DECLINLINE(void) pgmPhysPageMapLockForWriting(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
+{
+# if !defined(IN_RING0) || !defined(VBOX_WITH_RAM_IN_KERNEL)
+    PPGMPAGEMAP pMap = pTlbe->pMap;
+    if (pMap)
+        pMap->cRefs++;
+# else
+    RT_NOREF(pTlbe);
+# endif
+
+    unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
+    if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
+    {
+        if (cLocks == 0)
+            pVM->pgm.s.cWriteLockedPages++;
+        PGM_PAGE_INC_WRITE_LOCKS(pPage);
+    }
+    else if (cLocks != PGM_PAGE_MAX_LOCKS)
+    {
+        PGM_PAGE_INC_WRITE_LOCKS(pPage);
+        AssertMsgFailed(("%R[pgmpage] is entering permanent write locked state!\n", pPage));
+# if !defined(IN_RING0) || !defined(VBOX_WITH_RAM_IN_KERNEL)
+        if (pMap)
+            pMap->cRefs++; /* Extra ref to prevent it from going away. */
+# endif
+    }
+
+    pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE;
+# if !defined(IN_RING0) || !defined(VBOX_WITH_RAM_IN_KERNEL)
+    pLock->pvMap = pMap;
+# else
+    pLock->pvMap = NULL;
+# endif
+}
+
+/**
+ * Locks a page mapping for reading.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pPage               The page.
+ * @param   pTlbe               The mapping TLB entry for the page.
+ * @param   pLock               The lock structure (output).
+ */
+DECLINLINE(void) pgmPhysPageMapLockForReading(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
+{
+# if !defined(IN_RING0) || !defined(VBOX_WITH_RAM_IN_KERNEL)
+    PPGMPAGEMAP pMap = pTlbe->pMap;
+    if (pMap)
+        pMap->cRefs++;
+# else
+    RT_NOREF(pTlbe);
+# endif
+
+    unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
+    if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
+    {
+        if (cLocks == 0)
+            pVM->pgm.s.cReadLockedPages++;
+        PGM_PAGE_INC_READ_LOCKS(pPage);
+    }
+    else if (cLocks != PGM_PAGE_MAX_LOCKS)
+    {
+        PGM_PAGE_INC_READ_LOCKS(pPage);
+        AssertMsgFailed(("%R[pgmpage] is entering permanent read locked state!\n", pPage));
+# if !defined(IN_RING0) || !defined(VBOX_WITH_RAM_IN_KERNEL)
+        if (pMap)
+            pMap->cRefs++; /* Extra ref to prevent it from going away. */
+# endif
+    }
+
+    pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ;
+# if !defined(IN_RING0) || !defined(VBOX_WITH_RAM_IN_KERNEL)
+    pLock->pvMap = pMap;
+# else
+    pLock->pvMap = NULL;
+# endif
+}
+
+#endif /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+
+
+/**
+ * Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
+ * own the PGM lock and have access to the page structure.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address of the page that should be mapped.
+ * @param   pPage       Pointer to the PGMPAGE structure for the page.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that
+ *                      pgmPhysReleaseInternalPageMappingLock needs.
+ *
+ * @internal
+ */
+int pgmPhysGCPhys2CCPtrInternal(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    int rc;
+    AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Make sure the page is writable.
+     */
+    if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
+    {
+        rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+        if (RT_FAILURE(rc))
+            return rc;
+        AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
+    }
+    Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
+
+    /*
+     * Do the job.
+     */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    void *pv;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    rc = pgmRZDynMapHCPageInlined(pVCpu,
+                                  PGM_PAGE_GET_HCPHYS(pPage),
+                                  &pv
+                                  RTLOG_COMMA_SRC_POS);
+    if (RT_FAILURE(rc))
+        return rc;
+    *ppv = (void *)((uintptr_t)pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+    pLock->pvPage = pv;
+    pLock->pVCpu  = pVCpu;
+
+#else
+    PPGMPAGEMAPTLBE pTlbe;
+    rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+    if (RT_FAILURE(rc))
+        return rc;
+    pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
+    *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to
+ * own the PGM lock and have access to the page structure.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address of the page that should be mapped.
+ * @param   pPage       Pointer to the PGMPAGE structure for the page.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that
+ *                      pgmPhysReleaseInternalPageMappingLock needs.
+ *
+ * @internal
+ */
+int pgmPhysGCPhys2CCPtrInternalReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
+
+    /*
+     * Do the job.
+     */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    void *pv;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    int rc = pgmRZDynMapHCPageInlined(pVCpu,
+                                      PGM_PAGE_GET_HCPHYS(pPage),
+                                      &pv
+                                      RTLOG_COMMA_SRC_POS); /** @todo add a read only flag? */
+    if (RT_FAILURE(rc))
+        return rc;
+    *ppv = (void *)((uintptr_t)pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+    pLock->pvPage = pv;
+    pLock->pVCpu  = pVCpu;
+
+#else
+    PPGMPAGEMAPTLBE pTlbe;
+    int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+    if (RT_FAILURE(rc))
+        return rc;
+    pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
+    *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Requests the mapping of a guest page into the current context.
+ *
+ * This API should only be used for very short term, as it will consume scarse
+ * resources (R0 and GC) in the mapping cache. When you're done with the page,
+ * call PGMPhysReleasePageMappingLock() ASAP to release it.
+ *
+ * This API will assume your intention is to write to the page, and will
+ * therefore replace shared and zero pages. If you do not intend to modify
+ * the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address of the page that should be
+ *                      mapped.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that
+ *                      PGMPhysReleasePageMappingLock needs.
+ *
+ * @remarks The caller is responsible for dealing with access handlers.
+ * @todo    Add an informational return code for pages with access handlers?
+ *
+ * @remark  Avoid calling this API from within critical sections (other than
+ *          the PGM one) because of the deadlock risk. External threads may
+ *          need to delegate jobs to the EMTs.
+ * @remarks Only one page is mapped!  Make no assumption about what's after or
+ *          before the returned page!
+ * @thread  Any thread.
+ */
+VMM_INT_DECL(int) PGMPhysGCPhys2CCPtr(PVMCC pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    /*
+     * Find the page and make sure it's writable.
+     */
+    PPGMPAGE pPage;
+    rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+        if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
+            rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+        if (RT_SUCCESS(rc))
+        {
+            AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
+
+            PVMCPU pVCpu = VMMGetCpu(pVM);
+            void  *pv;
+            rc = pgmRZDynMapHCPageInlined(pVCpu,
+                                          PGM_PAGE_GET_HCPHYS(pPage),
+                                          &pv
+                                          RTLOG_COMMA_SRC_POS);
+            if (RT_SUCCESS(rc))
+            {
+                AssertRCSuccess(rc);
+
+                pv = (void *)((uintptr_t)pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+                *ppv = pv;
+                pLock->pvPage = pv;
+                pLock->pVCpu  = pVCpu;
+            }
+        }
+    }
+
+#else  /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+    /*
+     * Query the Physical TLB entry for the page (may fail).
+     */
+    PPGMPAGEMAPTLBE pTlbe;
+    rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * If the page is shared, the zero page, or being write monitored
+         * it must be converted to a page that's writable if possible.
+         */
+        PPGMPAGE pPage = pTlbe->pPage;
+        if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
+        {
+            rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+            if (RT_SUCCESS(rc))
+            {
+                AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
+                rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+            }
+        }
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Now, just perform the locking and calculate the return address.
+             */
+            pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
+            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+        }
+    }
+
+#endif /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Requests the mapping of a guest page into the current context.
+ *
+ * This API should only be used for very short term, as it will consume scarse
+ * resources (R0 and GC) in the mapping cache.  When you're done with the page,
+ * call PGMPhysReleasePageMappingLock() ASAP to release it.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address of the page that should be
+ *                      mapped.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that
+ *                      PGMPhysReleasePageMappingLock needs.
+ *
+ * @remarks The caller is responsible for dealing with access handlers.
+ * @todo    Add an informational return code for pages with access handlers?
+ *
+ * @remarks Avoid calling this API from within critical sections (other than
+ *          the PGM one) because of the deadlock risk.
+ * @remarks Only one page is mapped!  Make no assumption about what's after or
+ *          before the returned page!
+ * @thread  Any thread.
+ */
+VMM_INT_DECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVMCC pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    /*
+     * Find the page and make sure it's readable.
+     */
+    PPGMPAGE pPage;
+    rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+        if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+        else
+        {
+            PVMCPU pVCpu = VMMGetCpu(pVM);
+            void  *pv;
+            rc = pgmRZDynMapHCPageInlined(pVCpu,
+                                          PGM_PAGE_GET_HCPHYS(pPage),
+                                          &pv
+                                          RTLOG_COMMA_SRC_POS); /** @todo add a read only flag? */
+            if (RT_SUCCESS(rc))
+            {
+                AssertRCSuccess(rc);
+
+                pv = (void *)((uintptr_t)pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+                *ppv = pv;
+                pLock->pvPage = pv;
+                pLock->pVCpu  = pVCpu;
+            }
+        }
+    }
+
+#else  /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+    /*
+     * Query the Physical TLB entry for the page (may fail).
+     */
+    PPGMPAGEMAPTLBE pTlbe;
+    rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
+    if (RT_SUCCESS(rc))
+    {
+        /* MMIO pages doesn't have any readable backing. */
+        PPGMPAGE pPage = pTlbe->pPage;
+        if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+        else
+        {
+            /*
+             * Now, just perform the locking and calculate the return address.
+             */
+            pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
+            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+        }
+    }
+
+#endif /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Requests the mapping of a guest page given by virtual address into the current context.
+ *
+ * This API should only be used for very short term, as it will consume
+ * scarse resources (R0 and GC) in the mapping cache. When you're done
+ * with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
+ *
+ * This API will assume your intention is to write to the page, and will
+ * therefore replace shared and zero pages. If you do not intend to modify
+ * the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
+ * @retval  VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The guest physical address of the page that should be
+ *                      mapped.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that PGMPhysReleasePageMappingLock needs.
+ *
+ * @remark  Avoid calling this API from within critical sections (other than
+ *          the PGM one) because of the deadlock risk.
+ * @thread  EMT
+ */
+VMM_INT_DECL(int) PGMPhysGCPtr2CCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
+    RTGCPHYS GCPhys;
+    int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
+    if (RT_SUCCESS(rc))
+        rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
+    return rc;
+}
+
+
+/**
+ * Requests the mapping of a guest page given by virtual address into the current context.
+ *
+ * This API should only be used for very short term, as it will consume
+ * scarse resources (R0 and GC) in the mapping cache. When you're done
+ * with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
+ * @retval  VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The guest physical address of the page that should be
+ *                      mapped.
+ * @param   ppv         Where to store the address corresponding to GCPtr.
+ * @param   pLock       Where to store the lock information that PGMPhysReleasePageMappingLock needs.
+ *
+ * @remark  Avoid calling this API from within critical sections (other than
+ *          the PGM one) because of the deadlock risk.
+ * @thread  EMT
+ */
+VMM_INT_DECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPUCC pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
+    RTGCPHYS GCPhys;
+    int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
+    if (RT_SUCCESS(rc))
+        rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
+    return rc;
+}
+
+
+/**
+ * Release the mapping of a guest page.
+ *
+ * This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
+ * PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pLock       The lock structure initialized by the mapping function.
+ */
+VMMDECL(void) PGMPhysReleasePageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    Assert(pLock->pvPage != NULL);
+    Assert(pLock->pVCpu == VMMGetCpu(pVM)); RT_NOREF_PV(pVM);
+    PGM_DYNMAP_UNUSED_HINT(pLock->pVCpu, pLock->pvPage);
+    pLock->pVCpu  = NULL;
+    pLock->pvPage = NULL;
+
+#else
+# if !defined(IN_RING0) || !defined(VBOX_WITH_RAM_IN_KERNEL)
+    PPGMPAGEMAP pMap       = (PPGMPAGEMAP)pLock->pvMap;
+# endif
+    PPGMPAGE    pPage      = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
+    bool        fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
+
+    pLock->uPageAndType = 0;
+    pLock->pvMap = NULL;
+
+    pgmLock(pVM);
+    if (fWriteLock)
+    {
+        unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
+        Assert(cLocks > 0);
+        if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
+        {
+            if (cLocks == 1)
+            {
+                Assert(pVM->pgm.s.cWriteLockedPages > 0);
+                pVM->pgm.s.cWriteLockedPages--;
+            }
+            PGM_PAGE_DEC_WRITE_LOCKS(pPage);
+        }
+
+        if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED)
+        { /* probably extremely likely */ }
+        else
+            pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS);
+    }
+    else
+    {
+        unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
+        Assert(cLocks > 0);
+        if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
+        {
+            if (cLocks == 1)
+            {
+                Assert(pVM->pgm.s.cReadLockedPages > 0);
+                pVM->pgm.s.cReadLockedPages--;
+            }
+            PGM_PAGE_DEC_READ_LOCKS(pPage);
+        }
+    }
+
+# if !defined(IN_RING0) || !defined(VBOX_WITH_RAM_IN_KERNEL)
+    if (pMap)
+    {
+        Assert(pMap->cRefs >= 1);
+        pMap->cRefs--;
+    }
+# endif
+    pgmUnlock(pVM);
+#endif /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+}
+
+
+#ifdef IN_RING3
+/**
+ * Release the mapping of multiple guest pages.
+ *
+ * This is the counter part to PGMR3PhysBulkGCPhys2CCPtrExternal() and
+ * PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal().
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   cPages      Number of pages to unlock.
+ * @param   paLocks     Array of locks lock structure initialized by the mapping
+ *                      function.
+ */
+VMMDECL(void) PGMPhysBulkReleasePageMappingLocks(PVMCC pVM, uint32_t cPages, PPGMPAGEMAPLOCK paLocks)
+{
+    Assert(cPages > 0);
+    bool const fWriteLock = (paLocks[0].uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
+#ifdef VBOX_STRICT
+    for (uint32_t i = 1; i < cPages; i++)
+    {
+        Assert(fWriteLock == ((paLocks[i].uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE));
+        AssertPtr(paLocks[i].uPageAndType);
+    }
+#endif
+
+    pgmLock(pVM);
+    if (fWriteLock)
+    {
+        /*
+         * Write locks:
+         */
+        for (uint32_t i = 0; i < cPages; i++)
+        {
+            PPGMPAGE pPage  = (PPGMPAGE)(paLocks[i].uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
+            unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
+            Assert(cLocks > 0);
+            if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
+            {
+                if (cLocks == 1)
+                {
+                    Assert(pVM->pgm.s.cWriteLockedPages > 0);
+                    pVM->pgm.s.cWriteLockedPages--;
+                }
+                PGM_PAGE_DEC_WRITE_LOCKS(pPage);
+            }
+
+            if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED)
+            { /* probably extremely likely */ }
+            else
+                pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS);
+
+            PPGMPAGEMAP pMap = (PPGMPAGEMAP)paLocks[i].pvMap;
+            if (pMap)
+            {
+                Assert(pMap->cRefs >= 1);
+                pMap->cRefs--;
+            }
+
+            /* Yield the lock: */
+            if ((i & 1023) == 1023)
+            {
+                pgmLock(pVM);
+                pgmUnlock(pVM);
+            }
+        }
+    }
+    else
+    {
+        /*
+         * Read locks:
+         */
+        for (uint32_t i = 0; i < cPages; i++)
+        {
+            PPGMPAGE pPage  = (PPGMPAGE)(paLocks[i].uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
+            unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
+            Assert(cLocks > 0);
+            if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
+            {
+                if (cLocks == 1)
+                {
+                    Assert(pVM->pgm.s.cReadLockedPages > 0);
+                    pVM->pgm.s.cReadLockedPages--;
+                }
+                PGM_PAGE_DEC_READ_LOCKS(pPage);
+            }
+
+            PPGMPAGEMAP pMap = (PPGMPAGEMAP)paLocks[i].pvMap;
+            if (pMap)
+            {
+                Assert(pMap->cRefs >= 1);
+                pMap->cRefs--;
+            }
+
+            /* Yield the lock: */
+            if ((i & 1023) == 1023)
+            {
+                pgmLock(pVM);
+                pgmUnlock(pVM);
+            }
+        }
+    }
+    pgmUnlock(pVM);
+
+    RT_BZERO(paLocks, sizeof(paLocks[0]) * cPages);
+}
+#endif /* IN_RING3 */
+
+
+/**
+ * Release the internal mapping of a guest page.
+ *
+ * This is the counter part of pgmPhysGCPhys2CCPtrInternalEx and
+ * pgmPhysGCPhys2CCPtrInternalReadOnly.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pLock       The lock structure initialized by the mapping function.
+ *
+ * @remarks Caller must hold the PGM lock.
+ */
+void pgmPhysReleaseInternalPageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PGMPhysReleasePageMappingLock(pVM, pLock); /* lazy for now */
+}
+
+
+/**
+ * Converts a GC physical address to a HC ring-3 pointer.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
+ *          page but has no physical backing.
+ * @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
+ *          GC physical address.
+ * @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
+ *          a dynamic ram chunk boundary
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address to convert.
+ * @param   pR3Ptr      Where to store the R3 pointer on success.
+ *
+ * @deprecated  Avoid when possible!
+ */
+int pgmPhysGCPhys2R3Ptr(PVMCC pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
+{
+/** @todo this is kind of hacky and needs some more work. */
+#ifndef DEBUG_sandervl
+    VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
+#endif
+
+    Log(("pgmPhysGCPhys2R3Ptr(,%RGp,): dont use this API!\n", GCPhys)); /** @todo eliminate this API! */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    NOREF(pVM); NOREF(pR3Ptr); RT_NOREF_PV(GCPhys);
+    AssertFailedReturn(VERR_NOT_IMPLEMENTED);
+#else
+    pgmLock(pVM);
+
+    PPGMRAMRANGE pRam;
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
+    if (RT_SUCCESS(rc))
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
+
+    pgmUnlock(pVM);
+    Assert(rc <= VINF_SUCCESS);
+    return rc;
+#endif
+}
+
+#if 0 /*def VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+
+/**
+ * Maps and locks a guest CR3 or PD (PAE) page.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
+ *          page but has no physical backing.
+ * @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
+ *          GC physical address.
+ * @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
+ *          a dynamic ram chunk boundary
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address to convert.
+ * @param   pR3Ptr      Where to store the R3 pointer on success.  This may or
+ *                      may not be valid in ring-0 depending on the
+ *                      VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 build option.
+ *
+ * @remarks The caller must own the PGM lock.
+ */
+int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
+{
+
+    PPGMRAMRANGE pRam;
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
+    if (RT_SUCCESS(rc))
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
+    Assert(rc <= VINF_SUCCESS);
+    return rc;
+}
+
+
+int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
+{
+
+}
+
+#endif
+
+/**
+ * Converts a guest pointer to a GC physical address.
+ *
+ * This uses the current CR3/CR0/CR4 of the guest.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The guest pointer to convert.
+ * @param   pGCPhys     Where to store the GC physical address.
+ */
+VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPUCC pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
+{
+    int rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtr, NULL, pGCPhys);
+    if (pGCPhys && RT_SUCCESS(rc))
+        *pGCPhys |= (RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK;
+    return rc;
+}
+
+
+/**
+ * Converts a guest pointer to a HC physical address.
+ *
+ * This uses the current CR3/CR0/CR4 of the guest.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The guest pointer to convert.
+ * @param   pHCPhys     Where to store the HC physical address.
+ */
+VMM_INT_DECL(int) PGMPhysGCPtr2HCPhys(PVMCPUCC pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    RTGCPHYS GCPhys;
+    int rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtr, NULL, &GCPhys);
+    if (RT_SUCCESS(rc))
+        rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys | ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), pHCPhys);
+    return rc;
+}
+
+
+
+#undef LOG_GROUP
+#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
+
+
+#if defined(IN_RING3) && defined(SOME_UNUSED_FUNCTION)
+/**
+ * Cache PGMPhys memory access
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pCache          Cache structure pointer
+ * @param   GCPhys          GC physical address
+ * @param   pbHC            HC pointer corresponding to physical page
+ *
+ * @thread  EMT.
+ */
+static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE *pCache, RTGCPHYS GCPhys, uint8_t *pbR3)
+{
+    uint32_t iCacheIndex;
+
+    Assert(VM_IS_EMT(pVM));
+
+    GCPhys = PHYS_PAGE_ADDRESS(GCPhys);
+    pbR3   = (uint8_t *)PAGE_ADDRESS(pbR3);
+
+    iCacheIndex = ((GCPhys >> PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
+
+    ASMBitSet(&pCache->aEntries, iCacheIndex);
+
+    pCache->Entry[iCacheIndex].GCPhys = GCPhys;
+    pCache->Entry[iCacheIndex].pbR3   = pbR3;
+}
+#endif /* IN_RING3 */
+
+
+/**
+ * Deals with reading from a page with one or more ALL access handlers.
+ *
+ * @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
+ *          See PGM_HANDLER_PHYS_IS_VALID_STATUS and
+ *          PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The page descriptor.
+ * @param   GCPhys      The physical address to start reading at.
+ * @param   pvBuf       Where to put the bits we read.
+ * @param   cb          How much to read - less or equal to a page.
+ * @param   enmOrigin   The origin of this call.
+ */
+static VBOXSTRICTRC pgmPhysReadHandler(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb,
+                                       PGMACCESSORIGIN enmOrigin)
+{
+    /*
+     * The most frequent access here is MMIO and shadowed ROM.
+     * The current code ASSUMES all these access handlers covers full pages!
+     */
+
+    /*
+     * Whatever we do we need the source page, map it first.
+     */
+    PGMPAGEMAPLOCK PgMpLck;
+    const void    *pvSrc = NULL;
+    int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc, &PgMpLck);
+/** @todo Check how this can work for MMIO pages? */
+    if (RT_FAILURE(rc))
+    {
+        AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
+                               GCPhys, pPage, rc));
+        memset(pvBuf, 0xff, cb);
+        return VINF_SUCCESS;
+    }
+
+    VBOXSTRICTRC rcStrict = VINF_PGM_HANDLER_DO_DEFAULT;
+
+    /*
+     * Deal with any physical handlers.
+     */
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    PPGMPHYSHANDLER pPhys = NULL;
+    if (   PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL
+        || PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
+    {
+        pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
+        AssertReleaseMsg(pPhys, ("GCPhys=%RGp cb=%#x\n", GCPhys, cb));
+        Assert(GCPhys >= pPhys->Core.Key && GCPhys <= pPhys->Core.KeyLast);
+        Assert((pPhys->Core.Key     & PAGE_OFFSET_MASK) == 0);
+        Assert((pPhys->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
+#ifndef IN_RING3
+        if (enmOrigin != PGMACCESSORIGIN_IEM)
+        {
+            /* Cannot reliably handle informational status codes in this context */
+            pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+            return VERR_PGM_PHYS_WR_HIT_HANDLER;
+        }
+#endif
+        PFNPGMPHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pPhys)->CTX_SUFF(pfnHandler); Assert(pfnHandler);
+        void *pvUser = pPhys->CTX_SUFF(pvUser);
+
+        Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pPhys->pszDesc) ));
+        STAM_PROFILE_START(&pPhys->Stat, h);
+        PGM_LOCK_ASSERT_OWNER(pVM);
+
+        /* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
+        pgmUnlock(pVM);
+        rcStrict = pfnHandler(pVM, pVCpu, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, enmOrigin, pvUser);
+        pgmLock(pVM);
+
+#ifdef VBOX_WITH_STATISTICS
+        pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
+        if (pPhys)
+            STAM_PROFILE_STOP(&pPhys->Stat, h);
+#else
+        pPhys = NULL; /* might not be valid anymore. */
+#endif
+        AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, false),
+                        ("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys));
+        if (   rcStrict != VINF_PGM_HANDLER_DO_DEFAULT
+            && !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+        {
+            pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+            return rcStrict;
+        }
+    }
+
+    /*
+     * Take the default action.
+     */
+    if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
+    {
+        memcpy(pvBuf, pvSrc, cb);
+        rcStrict = VINF_SUCCESS;
+    }
+    pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+    return rcStrict;
+}
+
+
+/**
+ * Read physical memory.
+ *
+ * This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
+ * want to ignore those.
+ *
+ * @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
+ *          code in ring-3.  Use PGM_PHYS_RW_IS_SUCCESS to check.
+ * @retval  VINF_SUCCESS in all context - read completed.
+ *
+ * @retval  VINF_EM_OFF in RC and R0 - read completed.
+ * @retval  VINF_EM_SUSPEND in RC and R0 - read completed.
+ * @retval  VINF_EM_RESET in RC and R0 - read completed.
+ * @retval  VINF_EM_HALT in RC and R0 - read completed.
+ * @retval  VINF_SELM_SYNC_GDT in RC only - read completed.
+ *
+ * @retval  VINF_EM_DBG_STOP in RC and R0 - read completed.
+ * @retval  VINF_EM_DBG_BREAKPOINT in RC and R0 - read completed.
+ * @retval  VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
+ *
+ * @retval  VINF_IOM_R3_MMIO_READ in RC and R0.
+ * @retval  VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
+ *
+ * @retval  VINF_PATM_CHECK_PATCH_PAGE in RC only.
+ *
+ * @retval  VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
+ *          haven't been cleared for strict status codes yet.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Physical address start reading from.
+ * @param   pvBuf           Where to put the read bits.
+ * @param   cbRead          How many bytes to read.
+ * @param   enmOrigin       The origin of this call.
+ */
+VMMDECL(VBOXSTRICTRC) PGMPhysRead(PVMCC pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin)
+{
+    AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
+    LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
+
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysRead));
+    STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysReadBytes), cbRead);
+
+    pgmLock(pVM);
+
+    /*
+     * Copy loop on ram ranges.
+     */
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
+    for (;;)
+    {
+        /* Inside range or not? */
+        if (pRam && GCPhys >= pRam->GCPhys)
+        {
+            /*
+             * Must work our way thru this page by page.
+             */
+            RTGCPHYS off = GCPhys - pRam->GCPhys;
+            while (off < pRam->cb)
+            {
+                unsigned iPage = off >> PAGE_SHIFT;
+                PPGMPAGE pPage = &pRam->aPages[iPage];
+                size_t   cb    = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
+                if (cb > cbRead)
+                    cb = cbRead;
+
+                /*
+                 * Normal page? Get the pointer to it.
+                 */
+                if (   !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
+                    && !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+                {
+                    /*
+                     * Get the pointer to the page.
+                     */
+                    PGMPAGEMAPLOCK PgMpLck;
+                    const void    *pvSrc;
+                    int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
+                    if (RT_SUCCESS(rc))
+                    {
+                        memcpy(pvBuf, pvSrc, cb);
+                        pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+                    }
+                    else
+                    {
+                        AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
+                                               pRam->GCPhys + off, pPage, rc));
+                        memset(pvBuf, 0xff, cb);
+                    }
+                }
+                /*
+                 * Have ALL/MMIO access handlers.
+                 */
+                else
+                {
+                    VBOXSTRICTRC rcStrict2 = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
+                    if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
+                        PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
+                    else
+                    {
+                        memset(pvBuf, 0xff, cb);
+                        pgmUnlock(pVM);
+                        return rcStrict2;
+                    }
+                }
+
+                /* next page */
+                if (cb >= cbRead)
+                {
+                    pgmUnlock(pVM);
+                    return rcStrict;
+                }
+                cbRead -= cb;
+                off    += cb;
+                pvBuf   = (char *)pvBuf + cb;
+            } /* walk pages in ram range. */
+
+            GCPhys = pRam->GCPhysLast + 1;
+        }
+        else
+        {
+            LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
+
+            /*
+             * Unassigned address space.
+             */
+            size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
+            if (cb >= cbRead)
+            {
+                memset(pvBuf, 0xff, cbRead);
+                break;
+            }
+            memset(pvBuf, 0xff, cb);
+
+            cbRead -= cb;
+            pvBuf   = (char *)pvBuf + cb;
+            GCPhys += cb;
+        }
+
+        /* Advance range if necessary. */
+        while (pRam && GCPhys > pRam->GCPhysLast)
+            pRam = pRam->CTX_SUFF(pNext);
+    } /* Ram range walk */
+
+    pgmUnlock(pVM);
+    return rcStrict;
+}
+
+
+/**
+ * Deals with writing to a page with one or more WRITE or ALL access handlers.
+ *
+ * @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
+ *          See PGM_HANDLER_PHYS_IS_VALID_STATUS and
+ *          PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The page descriptor.
+ * @param   GCPhys      The physical address to start writing at.
+ * @param   pvBuf       What to write.
+ * @param   cbWrite     How much to write - less or equal to a page.
+ * @param   enmOrigin   The origin of this call.
+ */
+static VBOXSTRICTRC pgmPhysWriteHandler(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite,
+                                        PGMACCESSORIGIN enmOrigin)
+{
+    PGMPAGEMAPLOCK  PgMpLck;
+    void           *pvDst = NULL;
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * Give priority to physical handlers (like #PF does).
+     *
+     * Hope for a lonely physical handler first that covers the whole
+     * write area. This should be a pretty frequent case with MMIO and
+     * the heavy usage of full page handlers in the page pool.
+     */
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
+    if (pCur)
+    {
+        Assert(GCPhys >= pCur->Core.Key && GCPhys <= pCur->Core.KeyLast);
+#ifndef IN_RING3
+        if (enmOrigin != PGMACCESSORIGIN_IEM)
+            /* Cannot reliably handle informational status codes in this context */
+            return VERR_PGM_PHYS_WR_HIT_HANDLER;
+#endif
+        size_t cbRange = pCur->Core.KeyLast - GCPhys + 1;
+        if (cbRange > cbWrite)
+            cbRange = cbWrite;
+
+        Assert(PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->CTX_SUFF(pfnHandler));
+        Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n",
+              GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
+        if (!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
+            rcStrict = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
+        else
+            rcStrict = VINF_SUCCESS;
+        if (RT_SUCCESS(rcStrict))
+        {
+            PFNPGMPHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->CTX_SUFF(pfnHandler);
+            void *pvUser = pCur->CTX_SUFF(pvUser);
+            STAM_PROFILE_START(&pCur->Stat, h);
+
+            /* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
+            PGM_LOCK_ASSERT_OWNER(pVM);
+            pgmUnlock(pVM);
+            rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, pvUser);
+            pgmLock(pVM);
+
+#ifdef VBOX_WITH_STATISTICS
+            pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
+            if (pCur)
+                STAM_PROFILE_STOP(&pCur->Stat, h);
+#else
+            pCur = NULL; /* might not be valid anymore. */
+#endif
+            if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
+            {
+                if (pvDst)
+                    memcpy(pvDst, pvBuf, cbRange);
+                rcStrict = VINF_SUCCESS;
+            }
+            else
+                AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, true),
+                                ("rcStrict=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n",
+                                 VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pCur ? R3STRING(pCur->pszDesc) : ""));
+        }
+        else
+            AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
+                                         GCPhys, pPage, VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
+        if (RT_LIKELY(cbRange == cbWrite) || !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+        {
+            if (pvDst)
+                pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+            return rcStrict;
+        }
+
+        /* more fun to be had below */
+        cbWrite -= cbRange;
+        GCPhys  += cbRange;
+        pvBuf    = (uint8_t *)pvBuf + cbRange;
+        pvDst    = (uint8_t *)pvDst + cbRange;
+    }
+    else /* The handler is somewhere else in the page, deal with it below. */
+        rcStrict = VINF_SUCCESS;
+    Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)); /* MMIO handlers are all PAGE_SIZEed! */
+
+    /*
+     * Deal with all the odd ends (used to be deal with virt+phys).
+     */
+    Assert(rcStrict != VINF_PGM_HANDLER_DO_DEFAULT);
+
+    /* We need a writable destination page. */
+    if (!pvDst)
+    {
+        int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
+        AssertLogRelMsgReturn(RT_SUCCESS(rc2),
+                              ("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n", GCPhys, pPage, rc2),
+                              rc2);
+    }
+
+    /* The loop state (big + ugly). */
+    PPGMPHYSHANDLER pPhys       = NULL;
+    uint32_t        offPhys     = PAGE_SIZE;
+    uint32_t        offPhysLast = PAGE_SIZE;
+    bool            fMorePhys   = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage);
+
+    /* The loop. */
+    for (;;)
+    {
+        if (fMorePhys && !pPhys)
+        {
+            pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
+            if (pPhys)
+            {
+                offPhys = 0;
+                offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
+            }
+            else
+            {
+                pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,
+                                                                 GCPhys, true /* fAbove */);
+                if (    pPhys
+                    &&  pPhys->Core.Key <= GCPhys + (cbWrite - 1))
+                {
+                    offPhys     = pPhys->Core.Key     - GCPhys;
+                    offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
+                }
+                else
+                {
+                    pPhys     = NULL;
+                    fMorePhys = false;
+                    offPhys   = offPhysLast = PAGE_SIZE;
+                }
+            }
+        }
+
+        /*
+         * Handle access to space without handlers (that's easy).
+         */
+        VBOXSTRICTRC rcStrict2 = VINF_PGM_HANDLER_DO_DEFAULT;
+        uint32_t cbRange = (uint32_t)cbWrite;
+
+        /*
+         * Physical handler.
+         */
+        if (!offPhys)
+        {
+#ifndef IN_RING3
+            if (enmOrigin != PGMACCESSORIGIN_IEM)
+                /* Cannot reliably handle informational status codes in this context */
+                return VERR_PGM_PHYS_WR_HIT_HANDLER;
+#endif
+            if (cbRange > offPhysLast + 1)
+                cbRange = offPhysLast + 1;
+
+            PFNPGMPHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pPhys)->CTX_SUFF(pfnHandler);
+            void *pvUser = pPhys->CTX_SUFF(pvUser);
+
+            Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) ));
+            STAM_PROFILE_START(&pPhys->Stat, h);
+
+            /* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
+            PGM_LOCK_ASSERT_OWNER(pVM);
+            pgmUnlock(pVM);
+            rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, pvUser);
+            pgmLock(pVM);
+
+#ifdef VBOX_WITH_STATISTICS
+            pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
+            if (pPhys)
+                STAM_PROFILE_STOP(&pPhys->Stat, h);
+#else
+            pPhys = NULL; /* might not be valid anymore. */
+#endif
+            AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict2, true),
+                            ("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n", VBOXSTRICTRC_VAL(rcStrict2),
+                             VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage,  pPhys ? R3STRING(pPhys->pszDesc) : ""));
+        }
+
+        /*
+         * Execute the default action and merge the status codes.
+         */
+        if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT)
+        {
+            memcpy(pvDst, pvBuf, cbRange);
+            rcStrict2 = VINF_SUCCESS;
+        }
+        else if (!PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
+        {
+            pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+            return rcStrict2;
+        }
+        else
+            PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
+
+        /*
+         * Advance if we've got more stuff to do.
+         */
+        if (cbRange >= cbWrite)
+        {
+            pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+            return rcStrict;
+        }
+
+
+        cbWrite         -= cbRange;
+        GCPhys          += cbRange;
+        pvBuf            = (uint8_t *)pvBuf + cbRange;
+        pvDst            = (uint8_t *)pvDst + cbRange;
+
+        offPhys         -= cbRange;
+        offPhysLast     -= cbRange;
+    }
+}
+
+
+/**
+ * Write to physical memory.
+ *
+ * This API respects access handlers and MMIO. Use PGMPhysSimpleWriteGCPhys() if you
+ * want to ignore those.
+ *
+ * @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
+ *          code in ring-3.  Use PGM_PHYS_RW_IS_SUCCESS to check.
+ * @retval  VINF_SUCCESS in all context - write completed.
+ *
+ * @retval  VINF_EM_OFF in RC and R0 - write completed.
+ * @retval  VINF_EM_SUSPEND in RC and R0 - write completed.
+ * @retval  VINF_EM_RESET in RC and R0 - write completed.
+ * @retval  VINF_EM_HALT in RC and R0 - write completed.
+ * @retval  VINF_SELM_SYNC_GDT in RC only - write completed.
+ *
+ * @retval  VINF_EM_DBG_STOP in RC and R0 - write completed.
+ * @retval  VINF_EM_DBG_BREAKPOINT in RC and R0 - write completed.
+ * @retval  VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
+ *
+ * @retval  VINF_IOM_R3_MMIO_WRITE in RC and R0.
+ * @retval  VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
+ * @retval  VINF_IOM_R3_MMIO_COMMIT_WRITE in RC and R0.
+ *
+ * @retval  VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT in RC only - write completed.
+ * @retval  VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT in RC only.
+ * @retval  VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT in RC only.
+ * @retval  VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT in RC only.
+ * @retval  VINF_CSAM_PENDING_ACTION in RC only.
+ * @retval  VINF_PATM_CHECK_PATCH_PAGE in RC only.
+ *
+ * @retval  VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
+ *          haven't been cleared for strict status codes yet.
+ *
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Physical address to write to.
+ * @param   pvBuf           What to write.
+ * @param   cbWrite         How many bytes to write.
+ * @param   enmOrigin       Who is calling.
+ */
+VMMDECL(VBOXSTRICTRC) PGMPhysWrite(PVMCC pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin)
+{
+    AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()! enmOrigin=%d\n", enmOrigin));
+    AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
+    LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
+
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWrite));
+    STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
+
+    pgmLock(pVM);
+
+    /*
+     * Copy loop on ram ranges.
+     */
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
+    for (;;)
+    {
+        /* Inside range or not? */
+        if (pRam && GCPhys >= pRam->GCPhys)
+        {
+            /*
+             * Must work our way thru this page by page.
+             */
+            RTGCPTR off = GCPhys - pRam->GCPhys;
+            while (off < pRam->cb)
+            {
+                RTGCPTR     iPage = off >> PAGE_SHIFT;
+                PPGMPAGE    pPage = &pRam->aPages[iPage];
+                size_t      cb    = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
+                if (cb > cbWrite)
+                    cb = cbWrite;
+
+                /*
+                 * Normal page? Get the pointer to it.
+                 */
+                if (   !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
+                    && !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+                {
+                    PGMPAGEMAPLOCK PgMpLck;
+                    void          *pvDst;
+                    int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
+                    if (RT_SUCCESS(rc))
+                    {
+                        Assert(!PGM_PAGE_IS_BALLOONED(pPage));
+                        memcpy(pvDst, pvBuf, cb);
+                        pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+                    }
+                    /* Ignore writes to ballooned pages. */
+                    else if (!PGM_PAGE_IS_BALLOONED(pPage))
+                        AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
+                                                pRam->GCPhys + off, pPage, rc));
+                }
+                /*
+                 * Active WRITE or ALL access handlers.
+                 */
+                else
+                {
+                    VBOXSTRICTRC rcStrict2 = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
+                    if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
+                        PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
+                    else
+                    {
+                        pgmUnlock(pVM);
+                        return rcStrict2;
+                    }
+                }
+
+                /* next page */
+                if (cb >= cbWrite)
+                {
+                    pgmUnlock(pVM);
+                    return rcStrict;
+                }
+
+                cbWrite -= cb;
+                off     += cb;
+                pvBuf    = (const char *)pvBuf + cb;
+            } /* walk pages in ram range */
+
+            GCPhys = pRam->GCPhysLast + 1;
+        }
+        else
+        {
+            /*
+             * Unassigned address space, skip it.
+             */
+            if (!pRam)
+                break;
+            size_t cb = pRam->GCPhys - GCPhys;
+            if (cb >= cbWrite)
+                break;
+            cbWrite -= cb;
+            pvBuf   = (const char *)pvBuf + cb;
+            GCPhys += cb;
+        }
+
+        /* Advance range if necessary. */
+        while (pRam && GCPhys > pRam->GCPhysLast)
+            pRam = pRam->CTX_SUFF(pNext);
+    } /* Ram range walk */
+
+    pgmUnlock(pVM);
+    return rcStrict;
+}
+
+
+/**
+ * Read from guest physical memory by GC physical address, bypassing
+ * MMIO and access handlers.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvDst       The destination address.
+ * @param   GCPhysSrc   The source address (GC physical address).
+ * @param   cb          The number of bytes to read.
+ */
+VMMDECL(int) PGMPhysSimpleReadGCPhys(PVMCC pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
+{
+    /*
+     * Treat the first page as a special case.
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    /* map the 1st page */
+    void const *pvSrc;
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* optimize for the case where access is completely within the first page. */
+    size_t cbPage = PAGE_SIZE - (GCPhysSrc & PAGE_OFFSET_MASK);
+    if (RT_LIKELY(cb <= cbPage))
+    {
+        memcpy(pvDst, pvSrc, cb);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        return VINF_SUCCESS;
+    }
+
+    /* copy to the end of the page. */
+    memcpy(pvDst, pvSrc, cbPage);
+    PGMPhysReleasePageMappingLock(pVM, &Lock);
+    GCPhysSrc += cbPage;
+    pvDst = (uint8_t *)pvDst + cbPage;
+    cb -= cbPage;
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* map the page */
+        rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* last page? */
+        if (cb <= PAGE_SIZE)
+        {
+            memcpy(pvDst, pvSrc, cb);
+            PGMPhysReleasePageMappingLock(pVM, &Lock);
+            return VINF_SUCCESS;
+        }
+
+        /* copy the entire page and advance */
+        memcpy(pvDst, pvSrc, PAGE_SIZE);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        GCPhysSrc += PAGE_SIZE;
+        pvDst = (uint8_t *)pvDst + PAGE_SIZE;
+        cb -= PAGE_SIZE;
+    }
+    /* won't ever get here. */
+}
+
+
+/**
+ * Write to guest physical memory referenced by GC pointer.
+ * Write memory to GC physical address in guest physical memory.
+ *
+ * This will bypass MMIO and access handlers.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhysDst   The GC physical address of the destination.
+ * @param   pvSrc       The source buffer.
+ * @param   cb          The number of bytes to write.
+ */
+VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVMCC pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
+{
+    LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
+
+    /*
+     * Treat the first page as a special case.
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    /* map the 1st page */
+    void *pvDst;
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* optimize for the case where access is completely within the first page. */
+    size_t cbPage = PAGE_SIZE - (GCPhysDst & PAGE_OFFSET_MASK);
+    if (RT_LIKELY(cb <= cbPage))
+    {
+        memcpy(pvDst, pvSrc, cb);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        return VINF_SUCCESS;
+    }
+
+    /* copy to the end of the page. */
+    memcpy(pvDst, pvSrc, cbPage);
+    PGMPhysReleasePageMappingLock(pVM, &Lock);
+    GCPhysDst += cbPage;
+    pvSrc = (const uint8_t *)pvSrc + cbPage;
+    cb -= cbPage;
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* map the page */
+        rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* last page? */
+        if (cb <= PAGE_SIZE)
+        {
+            memcpy(pvDst, pvSrc, cb);
+            PGMPhysReleasePageMappingLock(pVM, &Lock);
+            return VINF_SUCCESS;
+        }
+
+        /* copy the entire page and advance */
+        memcpy(pvDst, pvSrc, PAGE_SIZE);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        GCPhysDst += PAGE_SIZE;
+        pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
+        cb -= PAGE_SIZE;
+    }
+    /* won't ever get here. */
+}
+
+
+/**
+ * Read from guest physical memory referenced by GC pointer.
+ *
+ * This function uses the current CR3/CR0/CR4 of the guest and will
+ * bypass access handlers and not set any accessed bits.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pvDst       The destination address.
+ * @param   GCPtrSrc    The source address (GC pointer).
+ * @param   cb          The number of bytes to read.
+ */
+VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPUCC pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+/** @todo fix the macro / state handling: VMCPU_ASSERT_EMT_OR_GURU(pVCpu); */
+
+    /*
+     * Treat the first page as a special case.
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleRead));
+    STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
+
+    /* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive
+     * when many VCPUs are fighting for the lock.
+     */
+    pgmLock(pVM);
+
+    /* map the 1st page */
+    void const *pvSrc;
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
+    if (RT_FAILURE(rc))
+    {
+        pgmUnlock(pVM);
+        return rc;
+    }
+
+    /* optimize for the case where access is completely within the first page. */
+    size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
+    if (RT_LIKELY(cb <= cbPage))
+    {
+        memcpy(pvDst, pvSrc, cb);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        pgmUnlock(pVM);
+        return VINF_SUCCESS;
+    }
+
+    /* copy to the end of the page. */
+    memcpy(pvDst, pvSrc, cbPage);
+    PGMPhysReleasePageMappingLock(pVM, &Lock);
+    GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
+    pvDst = (uint8_t *)pvDst + cbPage;
+    cb -= cbPage;
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* map the page */
+        rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
+        if (RT_FAILURE(rc))
+        {
+            pgmUnlock(pVM);
+            return rc;
+        }
+
+        /* last page? */
+        if (cb <= PAGE_SIZE)
+        {
+            memcpy(pvDst, pvSrc, cb);
+            PGMPhysReleasePageMappingLock(pVM, &Lock);
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+
+        /* copy the entire page and advance */
+        memcpy(pvDst, pvSrc, PAGE_SIZE);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + PAGE_SIZE);
+        pvDst = (uint8_t *)pvDst + PAGE_SIZE;
+        cb -= PAGE_SIZE;
+    }
+    /* won't ever get here. */
+}
+
+
+/**
+ * Write to guest physical memory referenced by GC pointer.
+ *
+ * This function uses the current CR3/CR0/CR4 of the guest and will
+ * bypass access handlers and not set dirty or accessed bits.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtrDst    The destination address (GC pointer).
+ * @param   pvSrc       The source address.
+ * @param   cb          The number of bytes to write.
+ */
+VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Treat the first page as a special case.
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWrite));
+    STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
+
+    /* map the 1st page */
+    void *pvDst;
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* optimize for the case where access is completely within the first page. */
+    size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
+    if (RT_LIKELY(cb <= cbPage))
+    {
+        memcpy(pvDst, pvSrc, cb);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        return VINF_SUCCESS;
+    }
+
+    /* copy to the end of the page. */
+    memcpy(pvDst, pvSrc, cbPage);
+    PGMPhysReleasePageMappingLock(pVM, &Lock);
+    GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
+    pvSrc = (const uint8_t *)pvSrc + cbPage;
+    cb -= cbPage;
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* map the page */
+        rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* last page? */
+        if (cb <= PAGE_SIZE)
+        {
+            memcpy(pvDst, pvSrc, cb);
+            PGMPhysReleasePageMappingLock(pVM, &Lock);
+            return VINF_SUCCESS;
+        }
+
+        /* copy the entire page and advance */
+        memcpy(pvDst, pvSrc, PAGE_SIZE);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
+        pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
+        cb -= PAGE_SIZE;
+    }
+    /* won't ever get here. */
+}
+
+
+/**
+ * Write to guest physical memory referenced by GC pointer and update the PTE.
+ *
+ * This function uses the current CR3/CR0/CR4 of the guest and will
+ * bypass access handlers but will set any dirty and accessed bits in the PTE.
+ *
+ * If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtrDst    The destination address (GC pointer).
+ * @param   pvSrc       The source address.
+ * @param   cb          The number of bytes to write.
+ */
+VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Treat the first page as a special case.
+     * Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    /* map the 1st page */
+    void *pvDst;
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* optimize for the case where access is completely within the first page. */
+    size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
+    if (RT_LIKELY(cb <= cbPage))
+    {
+        memcpy(pvDst, pvSrc, cb);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc);
+        return VINF_SUCCESS;
+    }
+
+    /* copy to the end of the page. */
+    memcpy(pvDst, pvSrc, cbPage);
+    PGMPhysReleasePageMappingLock(pVM, &Lock);
+    rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc);
+    GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
+    pvSrc = (const uint8_t *)pvSrc + cbPage;
+    cb -= cbPage;
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* map the page */
+        rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* last page? */
+        if (cb <= PAGE_SIZE)
+        {
+            memcpy(pvDst, pvSrc, cb);
+            PGMPhysReleasePageMappingLock(pVM, &Lock);
+            rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc);
+            return VINF_SUCCESS;
+        }
+
+        /* copy the entire page and advance */
+        memcpy(pvDst, pvSrc, PAGE_SIZE);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc);
+        GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
+        pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
+        cb -= PAGE_SIZE;
+    }
+    /* won't ever get here. */
+}
+
+
+/**
+ * Read from guest physical memory referenced by GC pointer.
+ *
+ * This function uses the current CR3/CR0/CR4 of the guest and will
+ * respect access handlers and set accessed bits.
+ *
+ * @returns Strict VBox status, see PGMPhysRead for details.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
+ *          specified virtual address.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pvDst       The destination address.
+ * @param   GCPtrSrc    The source address (GC pointer).
+ * @param   cb          The number of bytes to read.
+ * @param   enmOrigin   Who is calling.
+ * @thread  EMT(pVCpu)
+ */
+VMMDECL(VBOXSTRICTRC) PGMPhysReadGCPtr(PVMCPUCC pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
+{
+    RTGCPHYS    GCPhys;
+    uint64_t    fFlags;
+    int         rc;
+    PVMCC       pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Anything to do?
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
+
+    /*
+     * Optimize reads within a single page.
+     */
+    if (((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
+    {
+        /* Convert virtual to physical address + flags */
+        rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
+        AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
+        GCPhys |= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
+
+        /* mark the guest page as accessed. */
+        if (!(fFlags & X86_PTE_A))
+        {
+            rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
+            AssertRC(rc);
+        }
+
+        return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
+    }
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* Convert virtual to physical address + flags */
+        rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
+        AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
+        GCPhys |= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
+
+        /* mark the guest page as accessed. */
+        if (!(fFlags & X86_PTE_A))
+        {
+            rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
+            AssertRC(rc);
+        }
+
+        /* copy */
+        size_t cbRead = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
+        if (cbRead < cb)
+        {
+            VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, GCPhys, pvDst, cbRead, enmOrigin);
+            if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+            { /* likely */ }
+            else
+                return rcStrict;
+        }
+        else    /* Last page (cbRead is PAGE_SIZE, we only need cb!) */
+            return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
+
+        /* next */
+        Assert(cb > cbRead);
+        cb         -= cbRead;
+        pvDst       = (uint8_t *)pvDst + cbRead;
+        GCPtrSrc   += cbRead;
+    }
+}
+
+
+/**
+ * Write to guest physical memory referenced by GC pointer.
+ *
+ * This function uses the current CR3/CR0/CR4 of the guest and will
+ * respect access handlers and set dirty and accessed bits.
+ *
+ * @returns Strict VBox status, see PGMPhysWrite for details.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
+ *          specified virtual address.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtrDst    The destination address (GC pointer).
+ * @param   pvSrc       The source address.
+ * @param   cb          The number of bytes to write.
+ * @param   enmOrigin       Who is calling.
+ */
+VMMDECL(VBOXSTRICTRC) PGMPhysWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
+{
+    RTGCPHYS    GCPhys;
+    uint64_t    fFlags;
+    int         rc;
+    PVMCC       pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Anything to do?
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
+
+    /*
+     * Optimize writes within a single page.
+     */
+    if (((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
+    {
+        /* Convert virtual to physical address + flags */
+        rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
+        AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
+        GCPhys |= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
+
+        /* Mention when we ignore X86_PTE_RW... */
+        if (!(fFlags & X86_PTE_RW))
+            Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
+
+        /* Mark the guest page as accessed and dirty if necessary. */
+        if ((fFlags & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D))
+        {
+            rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D));
+            AssertRC(rc);
+        }
+
+        return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
+    }
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* Convert virtual to physical address + flags */
+        rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
+        AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
+        GCPhys |= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
+
+        /* Mention when we ignore X86_PTE_RW... */
+        if (!(fFlags & X86_PTE_RW))
+            Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
+
+        /* Mark the guest page as accessed and dirty if necessary. */
+        if ((fFlags & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D))
+        {
+            rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D));
+            AssertRC(rc);
+        }
+
+        /* copy */
+        size_t cbWrite = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
+        if (cbWrite < cb)
+        {
+            VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite, enmOrigin);
+            if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+            { /* likely */ }
+            else
+                return rcStrict;
+        }
+        else    /* Last page (cbWrite is PAGE_SIZE, we only need cb!) */
+            return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
+
+        /* next */
+        Assert(cb > cbWrite);
+        cb         -= cbWrite;
+        pvSrc       = (uint8_t *)pvSrc + cbWrite;
+        GCPtrDst   += cbWrite;
+    }
+}
+
+
+/**
+ * Performs a read of guest virtual memory for instruction emulation.
+ *
+ * This will check permissions, raise exceptions and update the access bits.
+ *
+ * The current implementation will bypass all access handlers. It may later be
+ * changed to at least respect MMIO.
+ *
+ *
+ * @returns VBox status code suitable to scheduling.
+ * @retval  VINF_SUCCESS if the read was performed successfully.
+ * @retval  VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
+ * @retval  VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pCtxCore    The context core.
+ * @param   pvDst       Where to put the bytes we've read.
+ * @param   GCPtrSrc    The source address.
+ * @param   cb          The number of bytes to read. Not more than a page.
+ *
+ * @remark  This function will dynamically map physical pages in GC. This may unmap
+ *          mappings done by the caller. Be careful!
+ */
+VMMDECL(int) PGMPhysInterpretedRead(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb)
+{
+    NOREF(pCtxCore);
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    Assert(cb <= PAGE_SIZE);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+/** @todo r=bird: This isn't perfect!
+ *  -# It's not checking for reserved bits being 1.
+ *  -# It's not correctly dealing with the access bit.
+ *  -# It's not respecting MMIO memory or any other access handlers.
+ */
+    /*
+     * 1. Translate virtual to physical. This may fault.
+     * 2. Map the physical address.
+     * 3. Do the read operation.
+     * 4. Set access bits if required.
+     */
+    int rc;
+    unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
+    if (cb <= cb1)
+    {
+        /*
+         * Not crossing pages.
+         */
+        RTGCPHYS GCPhys;
+        uint64_t fFlags;
+        rc = PGMGstGetPage(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
+        if (RT_SUCCESS(rc))
+        {
+            /** @todo we should check reserved bits ... */
+            PGMPAGEMAPLOCK PgMpLck;
+            void const    *pvSrc;
+            rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &PgMpLck);
+            switch (rc)
+            {
+                case VINF_SUCCESS:
+                    Log(("PGMPhysInterpretedRead: pvDst=%p pvSrc=%p cb=%d\n", pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb));
+                    memcpy(pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
+                    PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
+                    break;
+                case VERR_PGM_PHYS_PAGE_RESERVED:
+                case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
+                    memset(pvDst, 0xff, cb);
+                    break;
+                default:
+                    Assert(RT_FAILURE_NP(rc));
+                    return rc;
+            }
+
+            /** @todo access bit emulation isn't 100% correct. */
+            if (!(fFlags & X86_PTE_A))
+            {
+                rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
+                AssertRC(rc);
+            }
+            return VINF_SUCCESS;
+        }
+    }
+    else
+    {
+        /*
+         * Crosses pages.
+         */
+        size_t cb2 = cb - cb1;
+        uint64_t fFlags1;
+        RTGCPHYS GCPhys1;
+        uint64_t fFlags2;
+        RTGCPHYS GCPhys2;
+        rc = PGMGstGetPage(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
+        if (RT_SUCCESS(rc))
+        {
+            rc = PGMGstGetPage(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
+            if (RT_SUCCESS(rc))
+            {
+                /** @todo we should check reserved bits ... */
+                AssertMsgFailed(("cb=%d cb1=%d cb2=%d GCPtrSrc=%RGv\n", cb, cb1, cb2, GCPtrSrc));
+                PGMPAGEMAPLOCK PgMpLck;
+                void const *pvSrc1;
+                rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc1, &PgMpLck);
+                switch (rc)
+                {
+                    case VINF_SUCCESS:
+                        memcpy(pvDst, (uint8_t *)pvSrc1 + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
+                        PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
+                        break;
+                    case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
+                        memset(pvDst, 0xff, cb1);
+                        break;
+                    default:
+                        Assert(RT_FAILURE_NP(rc));
+                        return rc;
+                }
+
+                void const *pvSrc2;
+                rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc2, &PgMpLck);
+                switch (rc)
+                {
+                    case VINF_SUCCESS:
+                        memcpy((uint8_t *)pvDst + cb1, pvSrc2, cb2);
+                        PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
+                        break;
+                    case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
+                        memset((uint8_t *)pvDst + cb1, 0xff, cb2);
+                        break;
+                    default:
+                        Assert(RT_FAILURE_NP(rc));
+                        return rc;
+                }
+
+                if (!(fFlags1 & X86_PTE_A))
+                {
+                    rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
+                    AssertRC(rc);
+                }
+                if (!(fFlags2 & X86_PTE_A))
+                {
+                    rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
+                    AssertRC(rc);
+                }
+                return VINF_SUCCESS;
+            }
+        }
+    }
+
+    /*
+     * Raise a #PF.
+     */
+    uint32_t uErr;
+
+    /* Get the current privilege level. */
+    uint32_t cpl = CPUMGetGuestCPL(pVCpu);
+    switch (rc)
+    {
+        case VINF_SUCCESS:
+            uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD | X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
+            break;
+
+        case VERR_PAGE_NOT_PRESENT:
+        case VERR_PAGE_TABLE_NOT_PRESENT:
+            uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
+            break;
+
+        default:
+            AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
+            return rc;
+    }
+    Log(("PGMPhysInterpretedRead: GCPtrSrc=%RGv cb=%#x -> #PF(%#x)\n", GCPtrSrc, cb, uErr));
+    rc = TRPMAssertXcptPF(pVCpu, GCPtrSrc, uErr);
+    if (RT_SUCCESS(rc))
+        return VINF_EM_RAW_GUEST_TRAP;
+    return rc;
+}
+
+
+/**
+ * Performs a read of guest virtual memory for instruction emulation.
+ *
+ * This will check permissions, raise exceptions and update the access bits.
+ *
+ * The current implementation will bypass all access handlers. It may later be
+ * changed to at least respect MMIO.
+ *
+ *
+ * @returns VBox status code suitable to scheduling.
+ * @retval  VINF_SUCCESS if the read was performed successfully.
+ * @retval  VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
+ * @retval  VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pCtxCore    The context core.
+ * @param   pvDst       Where to put the bytes we've read.
+ * @param   GCPtrSrc    The source address.
+ * @param   cb          The number of bytes to read. Not more than a page.
+ * @param   fRaiseTrap  If set the trap will be raised on as per spec, if clear
+ *                      an appropriate error status will be returned (no
+ *                      informational at all).
+ *
+ *
+ * @remarks Takes the PGM lock.
+ * @remarks A page fault on the 2nd page of the access will be raised without
+ *          writing the bits on the first page since we're ASSUMING that the
+ *          caller is emulating an instruction access.
+ * @remarks This function will dynamically map physical pages in GC. This may
+ *          unmap mappings done by the caller. Be careful!
+ */
+VMMDECL(int) PGMPhysInterpretedReadNoHandlers(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb,
+                                              bool fRaiseTrap)
+{
+    NOREF(pCtxCore);
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    Assert(cb <= PAGE_SIZE);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * 1. Translate virtual to physical. This may fault.
+     * 2. Map the physical address.
+     * 3. Do the read operation.
+     * 4. Set access bits if required.
+     */
+    int rc;
+    unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
+    if (cb <= cb1)
+    {
+        /*
+         * Not crossing pages.
+         */
+        RTGCPHYS    GCPhys;
+        uint64_t    fFlags;
+        rc = PGMGstGetPage(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
+        if (RT_SUCCESS(rc))
+        {
+            if (1) /** @todo we should check reserved bits ... */
+            {
+                const void *pvSrc;
+                PGMPAGEMAPLOCK Lock;
+                rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &Lock);
+                switch (rc)
+                {
+                    case VINF_SUCCESS:
+                        Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d\n",
+                               pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb));
+                        memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
+                        PGMPhysReleasePageMappingLock(pVM, &Lock);
+                        break;
+                    case VERR_PGM_PHYS_PAGE_RESERVED:
+                    case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
+                        memset(pvDst, 0xff, cb);
+                        break;
+                    default:
+                        AssertMsgFailed(("%Rrc\n", rc));
+                        AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                        return rc;
+                }
+
+                if (!(fFlags & X86_PTE_A))
+                {
+                    /** @todo access bit emulation isn't 100% correct. */
+                    rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
+                    AssertRC(rc);
+                }
+                return VINF_SUCCESS;
+            }
+        }
+    }
+    else
+    {
+        /*
+         * Crosses pages.
+         */
+        size_t      cb2 = cb - cb1;
+        uint64_t    fFlags1;
+        RTGCPHYS    GCPhys1;
+        uint64_t    fFlags2;
+        RTGCPHYS    GCPhys2;
+        rc = PGMGstGetPage(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
+        if (RT_SUCCESS(rc))
+        {
+            rc = PGMGstGetPage(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
+            if (RT_SUCCESS(rc))
+            {
+                if (1) /** @todo we should check reserved bits ... */
+                {
+                    const void *pvSrc;
+                    PGMPAGEMAPLOCK Lock;
+                    rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc, &Lock);
+                    switch (rc)
+                    {
+                        case VINF_SUCCESS:
+                            Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d [2]\n",
+                                   pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb1));
+                            memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
+                            PGMPhysReleasePageMappingLock(pVM, &Lock);
+                            break;
+                        case VERR_PGM_PHYS_PAGE_RESERVED:
+                        case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
+                            memset(pvDst, 0xff, cb1);
+                            break;
+                        default:
+                            AssertMsgFailed(("%Rrc\n", rc));
+                            AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                            return rc;
+                    }
+
+                    rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc, &Lock);
+                    switch (rc)
+                    {
+                        case VINF_SUCCESS:
+                            memcpy((uint8_t *)pvDst + cb1, pvSrc, cb2);
+                            PGMPhysReleasePageMappingLock(pVM, &Lock);
+                            break;
+                        case VERR_PGM_PHYS_PAGE_RESERVED:
+                        case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
+                            memset((uint8_t *)pvDst + cb1, 0xff, cb2);
+                            break;
+                        default:
+                            AssertMsgFailed(("%Rrc\n", rc));
+                            AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                            return rc;
+                    }
+
+                    if (!(fFlags1 & X86_PTE_A))
+                    {
+                        rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
+                        AssertRC(rc);
+                    }
+                    if (!(fFlags2 & X86_PTE_A))
+                    {
+                        rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
+                        AssertRC(rc);
+                    }
+                    return VINF_SUCCESS;
+                }
+                /* sort out which page */
+            }
+            else
+                GCPtrSrc += cb1; /* fault on 2nd page */
+        }
+    }
+
+    /*
+     * Raise a #PF if we're allowed to do that.
+     */
+    /* Calc the error bits. */
+    uint32_t cpl = CPUMGetGuestCPL(pVCpu);
+    uint32_t uErr;
+    switch (rc)
+    {
+        case VINF_SUCCESS:
+            uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD | X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
+            rc = VERR_ACCESS_DENIED;
+            break;
+
+        case VERR_PAGE_NOT_PRESENT:
+        case VERR_PAGE_TABLE_NOT_PRESENT:
+            uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
+            break;
+
+        default:
+            AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
+            AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+            return rc;
+    }
+    if (fRaiseTrap)
+    {
+        Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrSrc, cb, uErr));
+        rc = TRPMAssertXcptPF(pVCpu, GCPtrSrc, uErr);
+        if (RT_SUCCESS(rc))
+            return VINF_EM_RAW_GUEST_TRAP;
+        return rc;
+    }
+    Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrSrc, cb, uErr));
+    return rc;
+}
+
+
+/**
+ * Performs a write to guest virtual memory for instruction emulation.
+ *
+ * This will check permissions, raise exceptions and update the dirty and access
+ * bits.
+ *
+ * @returns VBox status code suitable to scheduling.
+ * @retval  VINF_SUCCESS if the read was performed successfully.
+ * @retval  VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
+ * @retval  VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pCtxCore    The context core.
+ * @param   GCPtrDst    The destination address.
+ * @param   pvSrc       What to write.
+ * @param   cb          The number of bytes to write. Not more than a page.
+ * @param   fRaiseTrap  If set the trap will be raised on as per spec, if clear
+ *                      an appropriate error status will be returned (no
+ *                      informational at all).
+ *
+ * @remarks Takes the PGM lock.
+ * @remarks A page fault on the 2nd page of the access will be raised without
+ *          writing the bits on the first page since we're ASSUMING that the
+ *          caller is emulating an instruction access.
+ * @remarks This function will dynamically map physical pages in GC. This may
+ *          unmap mappings done by the caller. Be careful!
+ */
+VMMDECL(int) PGMPhysInterpretedWriteNoHandlers(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, RTGCPTR GCPtrDst, const void *pvSrc,
+                                               size_t cb, bool fRaiseTrap)
+{
+    NOREF(pCtxCore);
+    Assert(cb <= PAGE_SIZE);
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * 1. Translate virtual to physical. This may fault.
+     * 2. Map the physical address.
+     * 3. Do the write operation.
+     * 4. Set access bits if required.
+     */
+    /** @todo Since this method is frequently used by EMInterpret or IOM
+     *        upon a write fault to an write access monitored page, we can
+     *        reuse the guest page table walking from the \#PF code. */
+    int rc;
+    unsigned cb1 = PAGE_SIZE - (GCPtrDst & PAGE_OFFSET_MASK);
+    if (cb <= cb1)
+    {
+        /*
+         * Not crossing pages.
+         */
+        RTGCPHYS    GCPhys;
+        uint64_t    fFlags;
+        rc = PGMGstGetPage(pVCpu, GCPtrDst, &fFlags, &GCPhys);
+        if (RT_SUCCESS(rc))
+        {
+            if (    (fFlags & X86_PTE_RW)                   /** @todo Also check reserved bits. */
+                ||  (   !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
+                     &&   CPUMGetGuestCPL(pVCpu) <= 2) ) /** @todo it's 2, right? Check cpl check below as well. */
+            {
+                void *pvDst;
+                PGMPAGEMAPLOCK Lock;
+                rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, &pvDst, &Lock);
+                switch (rc)
+                {
+                    case VINF_SUCCESS:
+                        Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
+                               (uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc,  cb));
+                        memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb);
+                        PGMPhysReleasePageMappingLock(pVM, &Lock);
+                        break;
+                    case VERR_PGM_PHYS_PAGE_RESERVED:
+                    case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
+                        /* bit bucket */
+                        break;
+                    default:
+                        AssertMsgFailed(("%Rrc\n", rc));
+                        AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                        return rc;
+                }
+
+                if (!(fFlags & (X86_PTE_A | X86_PTE_D)))
+                {
+                    /** @todo dirty & access bit emulation isn't 100% correct. */
+                    rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D));
+                    AssertRC(rc);
+                }
+                return VINF_SUCCESS;
+            }
+            rc = VERR_ACCESS_DENIED;
+        }
+    }
+    else
+    {
+        /*
+         * Crosses pages.
+         */
+        size_t      cb2 = cb - cb1;
+        uint64_t    fFlags1;
+        RTGCPHYS    GCPhys1;
+        uint64_t    fFlags2;
+        RTGCPHYS    GCPhys2;
+        rc = PGMGstGetPage(pVCpu, GCPtrDst, &fFlags1, &GCPhys1);
+        if (RT_SUCCESS(rc))
+        {
+            rc = PGMGstGetPage(pVCpu, GCPtrDst + cb1, &fFlags2, &GCPhys2);
+            if (RT_SUCCESS(rc))
+            {
+                if (    (   (fFlags1 & X86_PTE_RW)  /** @todo Also check reserved bits. */
+                         && (fFlags2 & X86_PTE_RW))
+                    ||  (   !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
+                         &&   CPUMGetGuestCPL(pVCpu) <= 2) )
+                {
+                    void *pvDst;
+                    PGMPAGEMAPLOCK Lock;
+                    rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys1, &pvDst, &Lock);
+                    switch (rc)
+                    {
+                        case VINF_SUCCESS:
+                            Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
+                                   (uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb1));
+                            memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb1);
+                            PGMPhysReleasePageMappingLock(pVM, &Lock);
+                            break;
+                        case VERR_PGM_PHYS_PAGE_RESERVED:
+                        case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
+                            /* bit bucket */
+                            break;
+                        default:
+                            AssertMsgFailed(("%Rrc\n", rc));
+                            AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                            return rc;
+                    }
+
+                    rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys2, &pvDst, &Lock);
+                    switch (rc)
+                    {
+                        case VINF_SUCCESS:
+                            memcpy(pvDst, (const uint8_t *)pvSrc + cb1, cb2);
+                            PGMPhysReleasePageMappingLock(pVM, &Lock);
+                            break;
+                        case VERR_PGM_PHYS_PAGE_RESERVED:
+                        case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
+                            /* bit bucket */
+                            break;
+                        default:
+                            AssertMsgFailed(("%Rrc\n", rc));
+                            AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                            return rc;
+                    }
+
+                    if (!(fFlags1 & (X86_PTE_A | X86_PTE_RW)))
+                    {
+                        rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, (X86_PTE_A | X86_PTE_RW), ~(uint64_t)(X86_PTE_A | X86_PTE_RW));
+                        AssertRC(rc);
+                    }
+                    if (!(fFlags2 & (X86_PTE_A | X86_PTE_RW)))
+                    {
+                        rc = PGMGstModifyPage(pVCpu, GCPtrDst + cb1, 1, (X86_PTE_A | X86_PTE_RW), ~(uint64_t)(X86_PTE_A | X86_PTE_RW));
+                        AssertRC(rc);
+                    }
+                    return VINF_SUCCESS;
+                }
+                if ((fFlags1 & (X86_PTE_RW)) == X86_PTE_RW)
+                    GCPtrDst += cb1; /* fault on the 2nd page. */
+                rc = VERR_ACCESS_DENIED;
+            }
+            else
+                GCPtrDst += cb1; /* fault on the 2nd page. */
+        }
+    }
+
+    /*
+     * Raise a #PF if we're allowed to do that.
+     */
+    /* Calc the error bits. */
+    uint32_t uErr;
+    uint32_t cpl = CPUMGetGuestCPL(pVCpu);
+    switch (rc)
+    {
+        case VINF_SUCCESS:
+            uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD | X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
+            rc = VERR_ACCESS_DENIED;
+            break;
+
+        case VERR_ACCESS_DENIED:
+            uErr = (cpl >= 2) ? X86_TRAP_PF_RW | X86_TRAP_PF_US : X86_TRAP_PF_RW;
+            break;
+
+        case VERR_PAGE_NOT_PRESENT:
+        case VERR_PAGE_TABLE_NOT_PRESENT:
+            uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
+            break;
+
+        default:
+            AssertMsgFailed(("rc=%Rrc GCPtrDst=%RGv cb=%#x\n", rc, GCPtrDst, cb));
+            AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+            return rc;
+    }
+    if (fRaiseTrap)
+    {
+        Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrDst, cb, uErr));
+        rc = TRPMAssertXcptPF(pVCpu, GCPtrDst, uErr);
+        if (RT_SUCCESS(rc))
+            return VINF_EM_RAW_GUEST_TRAP;
+        return rc;
+    }
+    Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrDst, cb, uErr));
+    return rc;
+}
+
+
+/**
+ * Return the page type of the specified physical address.
+ *
+ * @returns The page type.
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Guest physical address
+ */
+VMM_INT_DECL(PGMPAGETYPE) PGMPhysGetPageType(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    pgmLock(pVM);
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+    PGMPAGETYPE enmPgType = pPage ? (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage) : PGMPAGETYPE_INVALID;
+    pgmUnlock(pVM);
+
+    return enmPgType;
+}
+
+
+/**
+ * Converts a GC physical address to a HC ring-3 pointer, with some
+ * additional checks.
+ *
+ * @returns VBox status code (no informational statuses).
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.
+ * @param   GCPhys          The GC physical address to convert.  This API mask
+ *                          the A20 line when necessary.
+ * @param   puTlbPhysRev    Where to read the physical TLB revision.  Needs to
+ *                          be done while holding the PGM lock.
+ * @param   ppb             Where to store the pointer corresponding to GCPhys
+ *                          on success.
+ * @param   pfTlb           The TLB flags and revision.  We only add stuff.
+ *
+ * @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr and
+ *          PGMPhysIemGCPhys2Ptr.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(int) PGMPhysIemGCPhys2PtrNoLock(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint64_t const volatile *puTlbPhysRev,
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+                                             R3PTRTYPE(uint8_t *) *ppb,
+#else
+                                             R3R0PTRTYPE(uint8_t *) *ppb,
+#endif
+                                             uint64_t *pfTlb)
+{
+    PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
+    Assert(!(GCPhys & X86_PAGE_OFFSET_MASK));
+
+    pgmLock(pVM);
+
+    PPGMRAMRANGE pRam;
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
+    if (RT_SUCCESS(rc))
+    {
+        if (!PGM_PAGE_IS_BALLOONED(pPage))
+        {
+            if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+            {
+                if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
+                {
+                    /*
+                     * No access handler.
+                     */
+                    switch (PGM_PAGE_GET_STATE(pPage))
+                    {
+                        case PGM_PAGE_STATE_ALLOCATED:
+                            *pfTlb |= *puTlbPhysRev;
+                            break;
+                        case PGM_PAGE_STATE_BALLOONED:
+                            AssertFailed();
+                            RT_FALL_THRU();
+                        case PGM_PAGE_STATE_ZERO:
+                        case PGM_PAGE_STATE_SHARED:
+                        case PGM_PAGE_STATE_WRITE_MONITORED:
+                            *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE;
+                            break;
+                    }
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+                    *pfTlb |= PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
+                    *ppb = NULL;
+#else
+                    PPGMPAGEMAPTLBE pTlbe;
+                    rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+                    AssertLogRelRCReturn(rc, rc);
+                    *ppb = (uint8_t *)pTlbe->pv;
+#endif
+                }
+                else if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+                {
+                    /*
+                     * MMIO or similar all access handler: Catch all access.
+                     */
+                    *pfTlb |= *puTlbPhysRev
+                           | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
+                    *ppb   = NULL;
+                }
+                else
+                {
+                    /*
+                     * Write access handler: Catch write accesses if active.
+                     */
+                    if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+                        *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE;
+                    else
+                        switch (PGM_PAGE_GET_STATE(pPage))
+                        {
+                            case PGM_PAGE_STATE_ALLOCATED:
+                                *pfTlb |= *puTlbPhysRev;
+                                break;
+                            case PGM_PAGE_STATE_BALLOONED:
+                                AssertFailed();
+                                RT_FALL_THRU();
+                            case PGM_PAGE_STATE_ZERO:
+                            case PGM_PAGE_STATE_SHARED:
+                            case PGM_PAGE_STATE_WRITE_MONITORED:
+                                *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE;
+                                break;
+                        }
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+                    *pfTlb |= PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
+                    *ppb = NULL;
+#else
+                    PPGMPAGEMAPTLBE pTlbe;
+                    rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+                    AssertLogRelRCReturn(rc, rc);
+                    *ppb = (uint8_t *)pTlbe->pv;
+#endif
+                }
+            }
+            else
+            {
+                /* Alias MMIO: For now, we catch all access.  */
+                *pfTlb |= *puTlbPhysRev
+                       |  PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
+                *ppb    = NULL;
+            }
+        }
+        else
+        {
+            /* Ballooned: Shouldn't get here, but we read zero page via PGMPhysRead and writes goes to /dev/null. */
+            *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
+            *ppb    = NULL;
+        }
+        Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp *ppb=%p *pfTlb=%#RX64 pPage=%R[pgmpage]\n", GCPhys, *ppb, *pfTlb, pPage));
+    }
+    else
+    {
+        *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
+        *ppb    = NULL;
+        Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp *ppb=%p *pfTlb=%#RX64 (rc=%Rrc)\n", GCPhys, *ppb, *pfTlb, rc));
+    }
+
+    pgmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Converts a GC physical address to a HC ring-3 pointer, with some
+ * additional checks.
+ *
+ * @returns VBox status code (no informational statuses).
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
+ *          access handler of some kind.
+ * @retval  VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
+ *          accesses or is odd in any way.
+ * @retval  VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.
+ * @param   GCPhys          The GC physical address to convert.  This API mask
+ *                          the A20 line when necessary.
+ * @param   fWritable       Whether write access is required.
+ * @param   fByPassHandlers Whether to bypass access handlers.
+ * @param   ppv             Where to store the pointer corresponding to GCPhys
+ *                          on success.
+ * @param   pLock
+ *
+ * @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr.
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(int) PGMPhysIemGCPhys2Ptr(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers,
+                                       void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
+
+    pgmLock(pVM);
+
+    PPGMRAMRANGE pRam;
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
+    if (RT_SUCCESS(rc))
+    {
+        if (PGM_PAGE_IS_BALLOONED(pPage))
+            rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
+        else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+            rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
+        else if (   !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
+                 || (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
+            rc = VINF_SUCCESS;
+        else
+        {
+            if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
+            {
+                Assert(!fByPassHandlers || PGM_PAGE_IS_MMIO(pPage));
+                rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
+            }
+            else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
+            {
+                Assert(!fByPassHandlers);
+                rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
+            }
+        }
+        if (RT_SUCCESS(rc))
+        {
+            int rc2;
+
+            /* Make sure what we return is writable. */
+            if (fWritable)
+                switch (PGM_PAGE_GET_STATE(pPage))
+                {
+                    case PGM_PAGE_STATE_ALLOCATED:
+                        break;
+                    case PGM_PAGE_STATE_BALLOONED:
+                        AssertFailed();
+                        break;
+                    case PGM_PAGE_STATE_ZERO:
+                    case PGM_PAGE_STATE_SHARED:
+                    case PGM_PAGE_STATE_WRITE_MONITORED:
+                        rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
+                        AssertLogRelRCReturn(rc2, rc2);
+                        break;
+                }
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+            void *pv;
+            rc = pgmRZDynMapHCPageInlined(pVCpu,
+                                          PGM_PAGE_GET_HCPHYS(pPage),
+                                          &pv
+                                          RTLOG_COMMA_SRC_POS);
+            if (RT_FAILURE(rc))
+                return rc;
+            *ppv = (void *)((uintptr_t)pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+            pLock->pvPage = pv;
+            pLock->pVCpu  = pVCpu;
+
+#else
+            /* Get a ring-3 mapping of the address. */
+            PPGMPAGEMAPTLBE pTlbe;
+            rc2 = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+            AssertLogRelRCReturn(rc2, rc2);
+
+            /* Lock it and calculate the address. */
+            if (fWritable)
+                pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
+            else
+                pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
+            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+#endif
+
+            Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] *ppv=%p\n", GCPhys, rc, pPage, *ppv));
+        }
+        else
+            Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
+
+        /* else: handler catching all access, no pointer returned. */
+    }
+    else
+        rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Checks if the give GCPhys page requires special handling for the given access
+ * because it's MMIO or otherwise monitored.
+ *
+ * @returns VBox status code (no informational statuses).
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
+ *          access handler of some kind.
+ * @retval  VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
+ *          accesses or is odd in any way.
+ * @retval  VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The GC physical address to convert.  Since this is
+ *                          only used for filling the REM TLB, the A20 mask must
+ *                          be applied before calling this API.
+ * @param   fWritable       Whether write access is required.
+ * @param   fByPassHandlers Whether to bypass access handlers.
+ *
+ * @remarks This is a watered down version PGMPhysIemGCPhys2Ptr and really just
+ *          a stop gap thing that should be removed once there is a better TLB
+ *          for virtual address accesses.
+ */
+VMM_INT_DECL(int) PGMPhysIemQueryAccess(PVMCC pVM, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers)
+{
+    pgmLock(pVM);
+    PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
+
+    PPGMRAMRANGE pRam;
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
+    if (RT_SUCCESS(rc))
+    {
+        if (PGM_PAGE_IS_BALLOONED(pPage))
+            rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
+        else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+            rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
+        else if (   !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
+                 || (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
+            rc = VINF_SUCCESS;
+        else
+        {
+            if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
+            {
+                Assert(!fByPassHandlers || PGM_PAGE_IS_MMIO(pPage));
+                rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
+            }
+            else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
+            {
+                Assert(!fByPassHandlers);
+                rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
+            }
+        }
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Interface used by NEM to check what to do on a memory access exit.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per virtual CPU structure.
+ *                          Optional.
+ * @param   GCPhys          The guest physical address.
+ * @param   fMakeWritable   Whether to try make the page writable or not.  If it
+ *                          cannot be made writable, NEM_PAGE_PROT_WRITE won't
+ *                          be returned and the return code will be unaffected
+ * @param   pInfo           Where to return the page information.  This is
+ *                          initialized even on failure.
+ * @param   pfnChecker      Page in-sync checker callback.  Optional.
+ * @param   pvUser          User argument to pass to pfnChecker.
+ */
+VMM_INT_DECL(int) PGMPhysNemPageInfoChecker(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, bool fMakeWritable, PPGMPHYSNEMPAGEINFO pInfo,
+                                            PFNPGMPHYSNEMCHECKPAGE pfnChecker, void *pvUser)
+{
+    pgmLock(pVM);
+
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+        /* Try make it writable if requested. */
+        pInfo->u2OldNemState = PGM_PAGE_GET_NEM_STATE(pPage);
+        if (fMakeWritable)
+            switch (PGM_PAGE_GET_STATE(pPage))
+            {
+                case PGM_PAGE_STATE_SHARED:
+                case PGM_PAGE_STATE_WRITE_MONITORED:
+                case PGM_PAGE_STATE_ZERO:
+                    rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+                    if (rc == VERR_PGM_PHYS_PAGE_RESERVED)
+                        rc = VINF_SUCCESS;
+                    break;
+            }
+
+        /* Fill in the info. */
+        pInfo->HCPhys       = PGM_PAGE_GET_HCPHYS(pPage);
+        pInfo->u2NemState   = PGM_PAGE_GET_NEM_STATE(pPage);
+        pInfo->fHasHandlers = PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) ? 1 : 0;
+        PGMPAGETYPE const enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+        pInfo->enmType      = enmType;
+        pInfo->fNemProt     = pgmPhysPageCalcNemProtection(pPage, enmType);
+        switch (PGM_PAGE_GET_STATE(pPage))
+        {
+            case PGM_PAGE_STATE_ALLOCATED:
+                pInfo->fZeroPage = 0;
+                break;
+
+            case PGM_PAGE_STATE_ZERO:
+                pInfo->fZeroPage = 1;
+                break;
+
+            case PGM_PAGE_STATE_WRITE_MONITORED:
+                pInfo->fZeroPage = 0;
+                break;
+
+            case PGM_PAGE_STATE_SHARED:
+                pInfo->fZeroPage = 0;
+                break;
+
+            case PGM_PAGE_STATE_BALLOONED:
+                pInfo->fZeroPage = 1;
+                break;
+
+            default:
+                pInfo->fZeroPage = 1;
+                AssertFailedStmt(rc = VERR_PGM_PHYS_PAGE_GET_IPE);
+        }
+
+        /* Call the checker and update NEM state. */
+        if (pfnChecker)
+        {
+            rc = pfnChecker(pVM, pVCpu, GCPhys, pInfo, pvUser);
+            PGM_PAGE_SET_NEM_STATE(pPage, pInfo->u2NemState);
+        }
+
+        /* Done. */
+        pgmUnlock(pVM);
+    }
+    else
+    {
+        pgmUnlock(pVM);
+
+        pInfo->HCPhys       = NIL_RTHCPHYS;
+        pInfo->fNemProt     = NEM_PAGE_PROT_NONE;
+        pInfo->u2NemState   = 0;
+        pInfo->fHasHandlers = 0;
+        pInfo->fZeroPage    = 0;
+        pInfo->enmType      = PGMPAGETYPE_INVALID;
+    }
+
+    return rc;
+}
+
+
+/**
+ * NEM helper that performs @a pfnCallback on pages with NEM state @a uMinState
+ * or higher.
+ *
+ * @returns VBox status code from callback.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.  This is
+ *                          optional as its only for passing to callback.
+ * @param   uMinState       The minimum NEM state value to call on.
+ * @param   pfnCallback     The callback function.
+ * @param   pvUser          User argument for the callback.
+ */
+VMM_INT_DECL(int) PGMPhysNemEnumPagesByState(PVMCC pVM, PVMCPUCC pVCpu, uint8_t uMinState,
+                                             PFNPGMPHYSNEMENUMCALLBACK pfnCallback, void *pvUser)
+{
+    /*
+     * Just brute force this problem.
+     */
+    pgmLock(pVM);
+    int rc = VINF_SUCCESS;
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX); pRam; pRam = pRam->CTX_SUFF(pNext))
+    {
+        uint32_t const cPages = pRam->cb >> X86_PAGE_SHIFT;
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            uint8_t u2State = PGM_PAGE_GET_NEM_STATE(&pRam->aPages[iPage]);
+            if (u2State < uMinState)
+            { /* likely */ }
+            else
+            {
+                rc = pfnCallback(pVM, pVCpu, pRam->GCPhys + ((RTGCPHYS)iPage << X86_PAGE_SHIFT), &u2State, pvUser);
+                if (RT_SUCCESS(rc))
+                    PGM_PAGE_SET_NEM_STATE(&pRam->aPages[iPage], u2State);
+                else
+                    break;
+            }
+        }
+    }
+    pgmUnlock(pVM);
+
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMAll/PGMAllPool.cpp b/src/VBox/VMM/VMMAll/PGMAllPool.cpp
new file mode 100644
index 00000000..88d94fca
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllPool.cpp
@@ -0,0 +1,5552 @@
+/* $Id: PGMAllPool.cpp $ */
+/** @file
+ * PGM Shadow Page Pool.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_POOL
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/cpum.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include "PGMInline.h"
+#include <VBox/disopcode.h>
+#include <VBox/vmm/hm_vmx.h>
+
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+#if 0 /* unused */
+DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
+DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
+#endif /* unused */
+static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
+static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
+static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
+static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
+#if defined(LOG_ENABLED) || defined(VBOX_STRICT)
+static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
+#endif
+#if 0 /*defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT)*/
+static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT);
+#endif
+
+int             pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage);
+PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt);
+void            pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt);
+void            pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt);
+
+RT_C_DECLS_END
+
+
+#if 0 /* unused */
+/**
+ * Checks if the specified page pool kind is for a 4MB or 2MB guest page.
+ *
+ * @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
+ * @param   enmKind     The page kind.
+ */
+DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
+{
+    switch (enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+            return true;
+        default:
+            return false;
+    }
+}
+#endif /* unused */
+
+
+/**
+ * Flushes a chain of pages sharing the same access monitor.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       A page in the chain.
+ */
+void pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
+
+    /*
+     * Find the list head.
+     */
+    uint16_t idx = pPage->idx;
+    if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
+    {
+        while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
+        {
+            idx = pPage->iMonitoredPrev;
+            Assert(idx != pPage->idx);
+            pPage = &pPool->aPages[idx];
+        }
+    }
+
+    /*
+     * Iterate the list flushing each shadow page.
+     */
+    for (;;)
+    {
+        idx = pPage->iMonitoredNext;
+        Assert(idx != pPage->idx);
+        if (pPage->idx >= PGMPOOL_IDX_FIRST)
+        {
+            int rc2 = pgmPoolFlushPage(pPool, pPage);
+            AssertRC(rc2);
+        }
+        /* next */
+        if (idx == NIL_PGMPOOL_IDX)
+            break;
+        pPage = &pPool->aPages[idx];
+    }
+}
+
+
+/**
+ * Wrapper for getting the current context pointer to the entry being modified.
+ *
+ * @returns VBox status code suitable for scheduling.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvDst       Destination address
+ * @param   pvSrc       Pointer to the mapping of @a GCPhysSrc or NULL depending
+ *                      on the context (e.g. \#PF in R0 & RC).
+ * @param   GCPhysSrc   The source guest physical address.
+ * @param   cb          Size of data to read
+ */
+DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVMCC pVM, void *pvDst, void const *pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
+{
+#if defined(IN_RING3)
+    NOREF(pVM); NOREF(GCPhysSrc);
+    memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
+    return VINF_SUCCESS;
+#else
+    /** @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
+    NOREF(pvSrc);
+    return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
+#endif
+}
+
+
+/**
+ * Process shadow entries before they are changed by the guest.
+ *
+ * For PT entries we will clear them. For PD entries, we'll simply check
+ * for mapping conflicts and set the SyncCR3 FF if found.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pPool       The pool.
+ * @param   pPage       The head page.
+ * @param   GCPhysFault The guest physical fault address.
+ * @param   pvAddress   Pointer to the mapping of @a GCPhysFault or NULL
+ *                      depending on the context (e.g. \#PF in R0 & RC).
+ * @param   cbWrite     Write size; might be zero if the caller knows we're not crossing entry boundaries
+ */
+static void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault,
+                                        void const *pvAddress, unsigned cbWrite)
+{
+    AssertMsg(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX, ("%u (idx=%u)\n", pPage->iMonitoredPrev, pPage->idx));
+    const unsigned  off = GCPhysFault & PAGE_OFFSET_MASK;
+    PVMCC           pVM = pPool->CTX_SUFF(pVM);
+    NOREF(pVCpu);
+
+    LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp cbWrite=%d\n",
+             (RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
+
+    for (;;)
+    {
+       union
+        {
+            void           *pv;
+            PX86PT          pPT;
+            PPGMSHWPTPAE    pPTPae;
+            PX86PD          pPD;
+            PX86PDPAE       pPDPae;
+            PX86PDPT        pPDPT;
+            PX86PML4        pPML4;
+        } uShw;
+
+        LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s write=%#x\n",
+                 pPage->idx, pPage->GCPhys, pPage->iMonitoredNext, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
+
+        uShw.pv = NULL;
+        switch (pPage->enmKind)
+        {
+            case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+            {
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PTE);
+                LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
+                if (uShw.pPT->a[iShw].n.u1Present)
+                {
+                    X86PTE GstPte;
+
+                    int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
+                    AssertRC(rc);
+                    Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
+                    pgmPoolTracDerefGCPhysHint(pPool, pPage,
+                                               uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK,
+                                               GstPte.u & X86_PTE_PG_MASK,
+                                               iShw);
+                    ASMAtomicWriteU32(&uShw.pPT->a[iShw].u, 0);
+                }
+                break;
+            }
+
+            /* page/2 sized */
+            case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+            {
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
+                {
+                    const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
+                    LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
+                    if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
+                    {
+                        X86PTE GstPte;
+                        int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
+                        AssertRC(rc);
+
+                        Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
+                        pgmPoolTracDerefGCPhysHint(pPool, pPage,
+                                                   PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
+                                                   GstPte.u & X86_PTE_PG_MASK,
+                                                   iShw);
+                        PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+            case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+            case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+            case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+            {
+                unsigned iGst     = off / sizeof(X86PDE);
+                unsigned iShwPdpt = iGst / 256;
+                unsigned iShw     = (iGst % 256) * 2;
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+
+                LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
+                if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
+                {
+                    for (unsigned i = 0; i < 2; i++)
+                    {
+                        if (uShw.pPDPae->a[iShw+i].n.u1Present)
+                        {
+                            LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw+i, uShw.pPDPae->a[iShw+i].u));
+                            pgmPoolFree(pVM,
+                                        uShw.pPDPae->a[iShw+i].u & X86_PDE_PAE_PG_MASK,
+                                        pPage->idx,
+                                        iShw + i);
+                            ASMAtomicWriteU64(&uShw.pPDPae->a[iShw+i].u, 0);
+                        }
+
+                        /* paranoia / a bit assumptive. */
+                        if (    (off & 3)
+                            &&  (off & 3) + cbWrite > 4)
+                        {
+                            const unsigned iShw2 = iShw + 2 + i;
+                            if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
+                            {
+                                if (uShw.pPDPae->a[iShw2].n.u1Present)
+                                {
+                                    LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
+                                    pgmPoolFree(pVM,
+                                                uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
+                                                pPage->idx,
+                                                iShw2);
+                                    ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
+                                }
+                            }
+                        }
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PTEPAE);
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
+                if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
+                {
+                    X86PTEPAE GstPte;
+                    int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
+                    AssertRC(rc);
+
+                    Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]), GstPte.u & X86_PTE_PAE_PG_MASK));
+                    pgmPoolTracDerefGCPhysHint(pPool, pPage,
+                                               PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
+                                               GstPte.u & X86_PTE_PAE_PG_MASK,
+                                               iShw);
+                    PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(X86PTEPAE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
+                    AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
+
+                    if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw2]))
+                    {
+                        X86PTEPAE GstPte;
+                        int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte,
+                                                             pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL,
+                                                             GCPhysFault + sizeof(GstPte), sizeof(GstPte));
+                        AssertRC(rc);
+                        Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]), GstPte.u & X86_PTE_PAE_PG_MASK));
+                        pgmPoolTracDerefGCPhysHint(pPool, pPage,
+                                                   PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]),
+                                                   GstPte.u & X86_PTE_PAE_PG_MASK,
+                                                   iShw2);
+                        PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw2], 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_32BIT_PD:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PTE);         // ASSUMING 32-bit guest paging!
+
+                LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
+                if (uShw.pPD->a[iShw].n.u1Present)
+                {
+                    LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
+                    pgmPoolFree(pVM,
+                                uShw.pPD->a[iShw].u & X86_PDE_PAE_PG_MASK,
+                                pPage->idx,
+                                iShw);
+                    ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
+                }
+                /* paranoia / a bit assumptive. */
+                if (    (off & 3)
+                    &&  (off & 3) + cbWrite > sizeof(X86PTE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
+                    if (    iShw2 != iShw
+                        &&  iShw2 < RT_ELEMENTS(uShw.pPD->a))
+                    {
+                        if (uShw.pPD->a[iShw2].n.u1Present)
+                        {
+                            LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPD->a[iShw2].u));
+                            pgmPoolFree(pVM,
+                                        uShw.pPD->a[iShw2].u & X86_PDE_PAE_PG_MASK,
+                                        pPage->idx,
+                                        iShw2);
+                            ASMAtomicWriteU32(&uShw.pPD->a[iShw2].u, 0);
+                        }
+                    }
+                }
+#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). - not working any longer... */
+                if (    uShw.pPD->a[iShw].n.u1Present
+                    &&  !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
+                {
+                    LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
+                    pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
+                }
+#endif
+                break;
+            }
+
+            case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PDEPAE);
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
+
+                /*
+                 * Causes trouble when the guest uses a PDE to refer to the whole page table level
+                 * structure. (Invalidate here; faults later on when it tries to change the page
+                 * table entries -> recheck; probably only applies to the RC case.)
+                 */
+                if (uShw.pPDPae->a[iShw].n.u1Present)
+                {
+                    LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
+                    pgmPoolFree(pVM,
+                                uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
+                                pPage->idx,
+                                iShw);
+                    ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(X86PDEPAE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
+                    AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
+
+                    if (uShw.pPDPae->a[iShw2].n.u1Present)
+                    {
+                        LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
+                        pgmPoolFree(pVM,
+                                    uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
+                                    pPage->idx,
+                                    iShw2);
+                        ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_PAE_PDPT:
+            {
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
+                /*
+                 * Hopefully this doesn't happen very often:
+                 * - touching unused parts of the page
+                 * - messing with the bits of pd pointers without changing the physical address
+                 */
+                /* PDPT roots are not page aligned; 32 byte only! */
+                const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
+
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = offPdpt / sizeof(X86PDPE);
+                if (iShw < X86_PG_PAE_PDPE_ENTRIES)          /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
+                {
+                    if (uShw.pPDPT->a[iShw].n.u1Present)
+                    {
+                        LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
+                        pgmPoolFree(pVM,
+                                    uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK,
+                                    pPage->idx,
+                                    iShw);
+                        ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
+                    }
+
+                    /* paranoia / a bit assumptive. */
+                    if (    (offPdpt & 7)
+                        &&  (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
+                    {
+                        const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
+                        if (    iShw2 != iShw
+                            &&  iShw2 < X86_PG_PAE_PDPE_ENTRIES)
+                        {
+                            if (uShw.pPDPT->a[iShw2].n.u1Present)
+                            {
+                                LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
+                                pgmPoolFree(pVM,
+                                            uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK,
+                                            pPage->idx,
+                                            iShw2);
+                                ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
+                            }
+                        }
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+            {
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PDEPAE);
+                Assert(!(uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING));
+                if (uShw.pPDPae->a[iShw].n.u1Present)
+                {
+                    LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
+                    pgmPoolFree(pVM,
+                                uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
+                                pPage->idx,
+                                iShw);
+                    ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
+                }
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(X86PDEPAE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
+                    AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
+
+                    Assert(!(uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING));
+                    if (uShw.pPDPae->a[iShw2].n.u1Present)
+                    {
+                        LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
+                        pgmPoolFree(pVM,
+                                    uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
+                                    pPage->idx,
+                                    iShw2);
+                        ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+            {
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
+                /*
+                 * Hopefully this doesn't happen very often:
+                 * - messing with the bits of pd pointers without changing the physical address
+                 */
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PDPE);
+                if (uShw.pPDPT->a[iShw].n.u1Present)
+                {
+                    LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
+                    pgmPoolFree(pVM, uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
+                }
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(X86PDPE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
+                    if (uShw.pPDPT->a[iShw2].n.u1Present)
+                    {
+                        LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
+                        pgmPoolFree(pVM, uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK, pPage->idx, iShw2);
+                        ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_64BIT_PML4:
+            {
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPML4));
+                /*
+                 * Hopefully this doesn't happen very often:
+                 * - messing with the bits of pd pointers without changing the physical address
+                 */
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PDPE);
+                if (uShw.pPML4->a[iShw].n.u1Present)
+                {
+                    LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPML4->a[iShw].u));
+                    pgmPoolFree(pVM, uShw.pPML4->a[iShw].u & X86_PML4E_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0);
+                }
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(X86PDPE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
+                    if (uShw.pPML4->a[iShw2].n.u1Present)
+                    {
+                        LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPML4->a[iShw2].u));
+                        pgmPoolFree(pVM, uShw.pPML4->a[iShw2].u & X86_PML4E_PG_MASK, pPage->idx, iShw2);
+                        ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            default:
+                AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
+        }
+        PGM_DYNMAP_UNUSED_HINT_VM(pVM, uShw.pv);
+
+        /* next */
+        if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
+            return;
+        pPage = &pPool->aPages[pPage->iMonitoredNext];
+    }
+}
+
+#ifndef IN_RING3
+
+/**
+ * Checks if a access could be a fork operation in progress.
+ *
+ * Meaning, that the guest is setting up the parent process for Copy-On-Write.
+ *
+ * @returns true if it's likely that we're forking, otherwise false.
+ * @param   pPool       The pool.
+ * @param   pDis        The disassembled instruction.
+ * @param   offFault    The access offset.
+ */
+DECLINLINE(bool) pgmRZPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault)
+{
+    /*
+     * i386 linux is using btr to clear X86_PTE_RW.
+     * The functions involved are (2.6.16 source inspection):
+     *      clear_bit
+     *      ptep_set_wrprotect
+     *      copy_one_pte
+     *      copy_pte_range
+     *      copy_pmd_range
+     *      copy_pud_range
+     *      copy_page_range
+     *      dup_mmap
+     *      dup_mm
+     *      copy_mm
+     *      copy_process
+     *      do_fork
+     */
+    if (    pDis->pCurInstr->uOpcode == OP_BTR
+        &&  !(offFault & 4)
+        /** @todo Validate that the bit index is X86_PTE_RW. */
+            )
+    {
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,Fork)); RT_NOREF_PV(pPool);
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Determine whether the page is likely to have been reused.
+ *
+ * @returns true if we consider the page as being reused for a different purpose.
+ * @returns false if we consider it to still be a paging page.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pRegFrame   Trap register frame.
+ * @param   pDis        The disassembly info for the faulting instruction.
+ * @param   pvFault     The fault address.
+ * @param   pPage       The pool page being accessed.
+ *
+ * @remark  The REP prefix check is left to the caller because of STOSD/W.
+ */
+DECLINLINE(bool) pgmRZPoolMonitorIsReused(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pDis, RTGCPTR pvFault,
+                                          PPGMPOOLPAGE pPage)
+{
+    /* Locked (CR3, PDPTR*4) should not be reusable.  Considering them as
+       such may cause loops booting tst-ubuntu-15_10-64-efi, ++. */
+    if (pPage->cLocked)
+    {
+        Log2(("pgmRZPoolMonitorIsReused: %RGv (%p) can't have been resued, because it's locked!\n", pvFault, pPage));
+        return false;
+    }
+
+    /** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */
+    if (   HMHasPendingIrq(pVM)
+        && pRegFrame->rsp - pvFault < 32)
+    {
+        /* Fault caused by stack writes while trying to inject an interrupt event. */
+        Log(("pgmRZPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
+        return true;
+    }
+
+    LogFlow(("Reused instr %RGv %d at %RGv param1.fUse=%llx param1.reg=%d\n", pRegFrame->rip, pDis->pCurInstr->uOpcode, pvFault, pDis->Param1.fUse,  pDis->Param1.Base.idxGenReg));
+
+    /* Non-supervisor mode write means it's used for something else. */
+    if (CPUMGetGuestCPL(pVCpu) == 3)
+        return true;
+
+    switch (pDis->pCurInstr->uOpcode)
+    {
+        /* call implies the actual push of the return address faulted */
+        case OP_CALL:
+            Log4(("pgmRZPoolMonitorIsReused: CALL\n"));
+            return true;
+        case OP_PUSH:
+            Log4(("pgmRZPoolMonitorIsReused: PUSH\n"));
+            return true;
+        case OP_PUSHF:
+            Log4(("pgmRZPoolMonitorIsReused: PUSHF\n"));
+            return true;
+        case OP_PUSHA:
+            Log4(("pgmRZPoolMonitorIsReused: PUSHA\n"));
+            return true;
+        case OP_FXSAVE:
+            Log4(("pgmRZPoolMonitorIsReused: FXSAVE\n"));
+            return true;
+        case OP_MOVNTI:     /* solaris - block_zero_no_xmm */
+            Log4(("pgmRZPoolMonitorIsReused: MOVNTI\n"));
+            return true;
+        case OP_MOVNTDQ:    /* solaris - hwblkclr & hwblkpagecopy */
+            Log4(("pgmRZPoolMonitorIsReused: MOVNTDQ\n"));
+            return true;
+        case OP_MOVSWD:
+        case OP_STOSWD:
+            if (    pDis->fPrefix == (DISPREFIX_REP|DISPREFIX_REX)
+                &&  pRegFrame->rcx >= 0x40
+               )
+            {
+                Assert(pDis->uCpuMode == DISCPUMODE_64BIT);
+
+                Log(("pgmRZPoolMonitorIsReused: OP_STOSQ\n"));
+                return true;
+            }
+            break;
+
+        default:
+            /*
+             * Anything having ESP on the left side means stack writes.
+             */
+            if (    (    (pDis->Param1.fUse & DISUSE_REG_GEN32)
+                     ||  (pDis->Param1.fUse & DISUSE_REG_GEN64))
+                &&  (pDis->Param1.Base.idxGenReg == DISGREG_ESP))
+            {
+                Log4(("pgmRZPoolMonitorIsReused: ESP\n"));
+                return true;
+            }
+            break;
+    }
+
+    /*
+     * Page table updates are very very unlikely to be crossing page boundraries,
+     * and we don't want to deal with that in pgmPoolMonitorChainChanging and such.
+     */
+    uint32_t const cbWrite = DISGetParamSize(pDis, &pDis->Param1);
+    if ( (((uintptr_t)pvFault + cbWrite) >> X86_PAGE_SHIFT) != ((uintptr_t)pvFault >> X86_PAGE_SHIFT) )
+    {
+        Log4(("pgmRZPoolMonitorIsReused: cross page write\n"));
+        return true;
+    }
+
+    /*
+     * Nobody does an unaligned 8 byte write to a page table, right.
+     */
+    if (cbWrite >= 8 && ((uintptr_t)pvFault & 7) != 0)
+    {
+        Log4(("pgmRZPoolMonitorIsReused: Unaligned 8+ byte write\n"));
+        return true;
+    }
+
+    return false;
+}
+
+
+/**
+ * Flushes the page being accessed.
+ *
+ * @returns VBox status code suitable for scheduling.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pPool       The pool.
+ * @param   pPage       The pool page (head).
+ * @param   pDis        The disassembly of the write instruction.
+ * @param   pRegFrame   The trap register frame.
+ * @param   GCPhysFault The fault address as guest physical address.
+ * @param   pvFault     The fault address.
+ * @todo VBOXSTRICTRC
+ */
+static int pgmRZPoolAccessPfHandlerFlush(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
+                                         PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
+{
+    NOREF(pVM); NOREF(GCPhysFault);
+
+    /*
+     * First, do the flushing.
+     */
+    pgmPoolMonitorChainFlush(pPool, pPage);
+
+    /*
+     * Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
+     * Must do this in raw mode (!); XP boot will fail otherwise.
+     */
+    int rc = VINF_SUCCESS;
+    VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, pRegFrame, pvFault, EMCODETYPE_ALL);
+    if (rc2 == VINF_SUCCESS)
+    { /* do nothing */ }
+    else if (rc2 == VINF_EM_RESCHEDULE)
+    {
+        rc = VBOXSTRICTRC_VAL(rc2);
+# ifndef IN_RING3
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
+# endif
+    }
+    else if (rc2 == VERR_EM_INTERPRETER)
+    {
+        rc = VINF_EM_RAW_EMULATE_INSTR;
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
+    }
+    else if (RT_FAILURE_NP(rc2))
+        rc = VBOXSTRICTRC_VAL(rc2);
+    else
+        AssertMsgFailed(("%Rrc\n", VBOXSTRICTRC_VAL(rc2))); /* ASSUMES no complicated stuff here. */
+
+    LogFlow(("pgmRZPoolAccessPfHandlerFlush: returns %Rrc (flushed)\n", rc));
+    return rc;
+}
+
+
+/**
+ * Handles the STOSD write accesses.
+ *
+ * @returns VBox status code suitable for scheduling.
+ * @param   pVM         The cross context VM structure.
+ * @param   pPool       The pool.
+ * @param   pPage       The pool page (head).
+ * @param   pDis        The disassembly of the write instruction.
+ * @param   pRegFrame   The trap register frame.
+ * @param   GCPhysFault The fault address as guest physical address.
+ * @param   pvFault     The fault address.
+ */
+DECLINLINE(int) pgmRZPoolAccessPfHandlerSTOSD(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
+                                              PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
+{
+    unsigned uIncrement = pDis->Param1.cb;
+    NOREF(pVM);
+
+    Assert(pDis->uCpuMode == DISCPUMODE_32BIT || pDis->uCpuMode == DISCPUMODE_64BIT);
+    Assert(pRegFrame->rcx <= 0x20);
+
+# ifdef VBOX_STRICT
+    if (pDis->uOpMode == DISCPUMODE_32BIT)
+        Assert(uIncrement == 4);
+    else
+        Assert(uIncrement == 8);
+# endif
+
+    Log3(("pgmRZPoolAccessPfHandlerSTOSD\n"));
+
+    /*
+     * Increment the modification counter and insert it into the list
+     * of modified pages the first time.
+     */
+    if (!pPage->cModifications++)
+        pgmPoolMonitorModifiedInsert(pPool, pPage);
+
+    /*
+     * Execute REP STOSD.
+     *
+     * This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
+     * write situation, meaning that it's safe to write here.
+     */
+    PVMCPUCC    pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
+    RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
+    while (pRegFrame->rcx)
+    {
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+        uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
+        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
+        PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
+# else
+        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
+# endif
+        PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->rax, uIncrement);
+        pu32           += uIncrement;
+        GCPhysFault    += uIncrement;
+        pRegFrame->rdi += uIncrement;
+        pRegFrame->rcx--;
+    }
+    pRegFrame->rip += pDis->cbInstr;
+
+    LogFlow(("pgmRZPoolAccessPfHandlerSTOSD: returns\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Handles the simple write accesses.
+ *
+ * @returns VBox status code suitable for scheduling.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pPool       The pool.
+ * @param   pPage       The pool page (head).
+ * @param   pDis        The disassembly of the write instruction.
+ * @param   pRegFrame   The trap register frame.
+ * @param   GCPhysFault The fault address as guest physical address.
+ * @param   pvFault     The fault address.
+ * @param   pfReused    Reused state (in/out)
+ */
+DECLINLINE(int) pgmRZPoolAccessPfHandlerSimple(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
+                                               PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault, bool *pfReused)
+{
+    Log3(("pgmRZPoolAccessPfHandlerSimple\n"));
+    NOREF(pVM);
+    NOREF(pfReused); /* initialized by caller */
+
+    /*
+     * Increment the modification counter and insert it into the list
+     * of modified pages the first time.
+     */
+    if (!pPage->cModifications++)
+        pgmPoolMonitorModifiedInsert(pPool, pPage);
+
+    /*
+     * Clear all the pages. ASSUMES that pvFault is readable.
+     */
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
+# endif
+
+    uint32_t cbWrite = DISGetParamSize(pDis, &pDis->Param1);
+    if (cbWrite <= 8)
+        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, cbWrite);
+    else if (cbWrite <= 16)
+    {
+        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, 8);
+        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + 8, NULL, cbWrite - 8);
+    }
+    else
+    {
+        Assert(cbWrite <= 32);
+        for (uint32_t off = 0; off < cbWrite; off += 8)
+            pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + off, NULL, RT_MIN(8, cbWrite - off));
+    }
+
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
+# endif
+
+    /*
+     * Interpret the instruction.
+     */
+    VBOXSTRICTRC rc = EMInterpretInstructionDisasState(pVCpu, pDis, pRegFrame, pvFault, EMCODETYPE_ALL);
+    if (RT_SUCCESS(rc))
+        AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rc))); /* ASSUMES no complicated stuff here. */
+    else if (rc == VERR_EM_INTERPRETER)
+    {
+        LogFlow(("pgmRZPoolAccessPfHandlerSimple: Interpretation failed for %04x:%RGv - opcode=%d\n",
+                  pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode));
+        rc = VINF_EM_RAW_EMULATE_INSTR;
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
+    }
+
+# if 0 /* experimental code */
+    if (rc == VINF_SUCCESS)
+    {
+        switch (pPage->enmKind)
+        {
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        {
+            X86PTEPAE GstPte;
+            int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvFault, GCPhysFault, sizeof(GstPte));
+            AssertRC(rc);
+
+            /* Check the new value written by the guest. If present and with a bogus physical address, then
+             * it's fairly safe to assume the guest is reusing the PT.
+             */
+            if (GstPte.n.u1Present)
+            {
+                RTHCPHYS HCPhys = -1;
+                int rc = PGMPhysGCPhys2HCPhys(pVM, GstPte.u & X86_PTE_PAE_PG_MASK, &HCPhys);
+                if (rc != VINF_SUCCESS)
+                {
+                    *pfReused = true;
+                    STAM_COUNTER_INC(&pPool->StatForceFlushReused);
+                }
+            }
+            break;
+        }
+        }
+    }
+# endif
+
+    LogFlow(("pgmRZPoolAccessPfHandlerSimple: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
+    return VBOXSTRICTRC_VAL(rc);
+}
+
+
+/**
+ * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
+ *      \#PF access handler callback for page table pages.}
+ *
+ * @remarks The @a pvUser argument points to the PGMPOOLPAGE.
+ */
+DECLEXPORT(VBOXSTRICTRC) pgmRZPoolAccessPfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
+                                                  RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
+{
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorRZ, a);
+    PPGMPOOL        pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PPGMPOOLPAGE    pPage = (PPGMPOOLPAGE)pvUser;
+    unsigned        cMaxModifications;
+    bool            fForcedFlush = false;
+    NOREF(uErrorCode);
+
+    LogFlow(("pgmRZPoolAccessPfHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
+
+    pgmLock(pVM);
+    if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
+    {
+        /* Pool page changed while we were waiting for the lock; ignore. */
+        Log(("CPU%d: pgmRZPoolAccessPfHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
+        STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
+        pgmUnlock(pVM);
+        return VINF_SUCCESS;
+    }
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    if (pPage->fDirty)
+    {
+        Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH));
+        pgmUnlock(pVM);
+        return VINF_SUCCESS;    /* SMP guest case where we were blocking on the pgm lock while the same page was being marked dirty. */
+    }
+# endif
+
+# if 0 /* test code defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) */
+    if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
+    {
+        void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
+        void *pvGst;
+        int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+        pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
+        PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+        PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
+    }
+# endif
+
+    /*
+     * Disassemble the faulting instruction.
+     */
+    PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
+    int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
+    if (RT_UNLIKELY(rc != VINF_SUCCESS))
+    {
+        AssertMsg(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT, ("Unexpected rc %d\n", rc));
+        pgmUnlock(pVM);
+        return rc;
+    }
+
+    Assert(pPage->enmKind != PGMPOOLKIND_FREE);
+
+    /*
+     * We should ALWAYS have the list head as user parameter. This
+     * is because we use that page to record the changes.
+     */
+    Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
+
+# ifdef IN_RING0
+    /* Maximum nr of modifications depends on the page type. */
+    if (    pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+        ||  pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
+        cMaxModifications = 4;
+    else
+        cMaxModifications = 24;
+# else
+    cMaxModifications = 48;
+# endif
+
+    /*
+     * Incremental page table updates should weigh more than random ones.
+     * (Only applies when started from offset 0)
+     */
+    pVCpu->pgm.s.cPoolAccessHandler++;
+    if (    pPage->GCPtrLastAccessHandlerRip >= pRegFrame->rip - 0x40      /* observed loops in Windows 7 x64 */
+        &&  pPage->GCPtrLastAccessHandlerRip <  pRegFrame->rip + 0x40
+        &&  pvFault == (pPage->GCPtrLastAccessHandlerFault + pDis->Param1.cb)
+        &&  pVCpu->pgm.s.cPoolAccessHandler == pPage->cLastAccessHandler + 1)
+    {
+        Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
+        Assert(pPage->cModifications < 32000);
+        pPage->cModifications               = pPage->cModifications * 2;
+        pPage->GCPtrLastAccessHandlerFault  = pvFault;
+        pPage->cLastAccessHandler           = pVCpu->pgm.s.cPoolAccessHandler;
+        if (pPage->cModifications >= cMaxModifications)
+        {
+            STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushReinit);
+            fForcedFlush = true;
+        }
+    }
+
+    if (pPage->cModifications >= cMaxModifications)
+        Log(("Mod overflow %RGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
+
+    /*
+     * Check if it's worth dealing with.
+     */
+    bool fReused = false;
+    bool fNotReusedNotForking = false;
+    if (    (   pPage->cModifications < cMaxModifications   /** @todo \#define */ /** @todo need to check that it's not mapping EIP. */ /** @todo adjust this! */
+             || pgmPoolIsPageLocked(pPage)
+            )
+        &&  !(fReused = pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault, pPage))
+        &&  !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
+    {
+        /*
+         * Simple instructions, no REP prefix.
+         */
+        if (!(pDis->fPrefix & (DISPREFIX_REP | DISPREFIX_REPNE)))
+        {
+            rc = pgmRZPoolAccessPfHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault, &fReused);
+            if (fReused)
+                goto flushPage;
+
+            /* A mov instruction to change the first page table entry will be remembered so we can detect
+             * full page table changes early on. This will reduce the amount of unnecessary traps we'll take.
+             */
+            if (   rc == VINF_SUCCESS
+                && !pPage->cLocked                      /* only applies to unlocked pages as we can't free locked ones (e.g. cr3 root). */
+                && pDis->pCurInstr->uOpcode == OP_MOV
+                && (pvFault & PAGE_OFFSET_MASK) == 0)
+            {
+                pPage->GCPtrLastAccessHandlerFault = pvFault;
+                pPage->cLastAccessHandler          = pVCpu->pgm.s.cPoolAccessHandler;
+                pPage->GCPtrLastAccessHandlerRip   = pRegFrame->rip;
+                /* Make sure we don't kick out a page too quickly. */
+                if (pPage->cModifications > 8)
+                    pPage->cModifications = 2;
+            }
+            else if (pPage->GCPtrLastAccessHandlerFault == pvFault)
+            {
+                /* ignore the 2nd write to this page table entry. */
+                pPage->cLastAccessHandler       = pVCpu->pgm.s.cPoolAccessHandler;
+            }
+            else
+            {
+                pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
+                pPage->GCPtrLastAccessHandlerRip   = 0;
+            }
+
+            STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
+            pgmUnlock(pVM);
+            return rc;
+        }
+
+        /*
+         * Windows is frequently doing small memset() operations (netio test 4k+).
+         * We have to deal with these or we'll kill the cache and performance.
+         */
+        if (    pDis->pCurInstr->uOpcode == OP_STOSWD
+            &&  !pRegFrame->eflags.Bits.u1DF
+            &&  pDis->uOpMode == pDis->uCpuMode
+            &&  pDis->uAddrMode == pDis->uCpuMode)
+        {
+            bool fValidStosd = false;
+
+            if (    pDis->uCpuMode == DISCPUMODE_32BIT
+                &&  pDis->fPrefix == DISPREFIX_REP
+                &&  pRegFrame->ecx <= 0x20
+                &&  pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
+                &&  !((uintptr_t)pvFault & 3)
+                &&  (pRegFrame->eax == 0 || pRegFrame->eax == 0x80) /* the two values observed. */
+                )
+            {
+                fValidStosd = true;
+                pRegFrame->rcx &= 0xffffffff;   /* paranoia */
+            }
+            else
+            if (    pDis->uCpuMode == DISCPUMODE_64BIT
+                &&  pDis->fPrefix == (DISPREFIX_REP | DISPREFIX_REX)
+                &&  pRegFrame->rcx <= 0x20
+                &&  pRegFrame->rcx * 8 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
+                &&  !((uintptr_t)pvFault & 7)
+                &&  (pRegFrame->rax == 0 || pRegFrame->rax == 0x80) /* the two values observed. */
+                )
+            {
+                fValidStosd = true;
+            }
+
+            if (fValidStosd)
+            {
+                rc = pgmRZPoolAccessPfHandlerSTOSD(pVM, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
+                STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZRepStosd, a);
+                pgmUnlock(pVM);
+                return rc;
+            }
+        }
+
+        /* REP prefix, don't bother. */
+        STAM_COUNTER_INC(&pPool->StatMonitorPfRZRepPrefix);
+        Log4(("pgmRZPoolAccessPfHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
+              pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode, pDis->fPrefix));
+        fNotReusedNotForking = true;
+    }
+
+# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) && defined(IN_RING0)
+    /* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This
+     * leads to pgm pool trashing and an excessive amount of write faults due to page monitoring.
+     */
+    if (    pPage->cModifications >= cMaxModifications
+        &&  !fForcedFlush
+        &&  (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT || pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
+        &&  (   fNotReusedNotForking
+             || (   !pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault, pPage)
+                 && !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
+            )
+       )
+    {
+        Assert(!pgmPoolIsPageLocked(pPage));
+        Assert(pPage->fDirty == false);
+
+        /* Flush any monitored duplicates as we will disable write protection. */
+        if (    pPage->iMonitoredNext != NIL_PGMPOOL_IDX
+            ||  pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
+        {
+            PPGMPOOLPAGE pPageHead = pPage;
+
+            /* Find the monitor head. */
+            while (pPageHead->iMonitoredPrev != NIL_PGMPOOL_IDX)
+                pPageHead = &pPool->aPages[pPageHead->iMonitoredPrev];
+
+            while (pPageHead)
+            {
+                unsigned idxNext = pPageHead->iMonitoredNext;
+
+                if (pPageHead != pPage)
+                {
+                    STAM_COUNTER_INC(&pPool->StatDirtyPageDupFlush);
+                    Log(("Flush duplicate page idx=%d GCPhys=%RGp type=%s\n", pPageHead->idx, pPageHead->GCPhys, pgmPoolPoolKindToStr(pPageHead->enmKind)));
+                    int rc2 = pgmPoolFlushPage(pPool, pPageHead);
+                    AssertRC(rc2);
+                }
+
+                if (idxNext == NIL_PGMPOOL_IDX)
+                    break;
+
+                pPageHead = &pPool->aPages[idxNext];
+            }
+        }
+
+        /* The flushing above might fail for locked pages, so double check. */
+        if (    pPage->iMonitoredNext == NIL_PGMPOOL_IDX
+            &&  pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
+        {
+            pgmPoolAddDirtyPage(pVM, pPool, pPage);
+
+            /* Temporarily allow write access to the page table again. */
+            rc = PGMHandlerPhysicalPageTempOff(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK, pPage->GCPhys & PAGE_BASE_GC_MASK);
+            if (rc == VINF_SUCCESS)
+            {
+                rc = PGMShwMakePageWritable(pVCpu, pvFault, PGM_MK_PG_IS_WRITE_FAULT);
+                AssertMsg(rc == VINF_SUCCESS
+                        /* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
+                        ||  rc == VERR_PAGE_TABLE_NOT_PRESENT
+                        ||  rc == VERR_PAGE_NOT_PRESENT,
+                        ("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc));
+#  ifdef VBOX_STRICT
+                pPage->GCPtrDirtyFault = pvFault;
+#  endif
+
+                STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, a);
+                pgmUnlock(pVM);
+                return rc;
+            }
+        }
+    }
+# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT && IN_RING0 */
+
+    STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushModOverflow);
+flushPage:
+    /*
+     * Not worth it, so flush it.
+     *
+     * If we considered it to be reused, don't go back to ring-3
+     * to emulate failed instructions since we usually cannot
+     * interpret then. This may be a bit risky, in which case
+     * the reuse detection must be fixed.
+     */
+    rc = pgmRZPoolAccessPfHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
+    if (    rc == VINF_EM_RAW_EMULATE_INSTR
+        &&  fReused)
+    {
+        /* Make sure that the current instruction still has shadow page backing, otherwise we'll end up in a loop. */
+        if (PGMShwGetPage(pVCpu, pRegFrame->rip, NULL, NULL) == VINF_SUCCESS)
+            rc = VINF_SUCCESS;  /* safe to restart the instruction. */
+    }
+    STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZFlushPage, a);
+    pgmUnlock(pVM);
+    return rc;
+}
+
+#endif /* !IN_RING3 */
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER,
+ *      Access handler for shadowed page table pages.}
+ *
+ * @remarks Only uses the VINF_PGM_HANDLER_DO_DEFAULT status.
+ */
+PGM_ALL_CB2_DECL(VBOXSTRICTRC)
+pgmPoolAccessHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
+                     PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
+{
+    PPGMPOOL        pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    STAM_PROFILE_START(&pPool->CTX_SUFF_Z(StatMonitor), a);
+    PPGMPOOLPAGE    pPage = (PPGMPOOLPAGE)pvUser;
+    LogFlow(("PGM_ALL_CB_DECL: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
+             GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
+
+    NOREF(pvPhys); NOREF(pvBuf); NOREF(enmAccessType);
+
+    pgmLock(pVM);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Collect stats on the access.
+     */
+    AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Sizes)) == 19);
+    if (cbBuf <= 16 && cbBuf > 0)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[cbBuf - 1]);
+    else if (cbBuf >= 17 && cbBuf < 32)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[16]);
+    else if (cbBuf >= 32 && cbBuf < 64)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[17]);
+    else if (cbBuf >= 64)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[18]);
+
+    uint8_t cbAlign;
+    switch (pPage->enmKind)
+    {
+        default:
+            cbAlign = 7;
+            break;
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            cbAlign = 3;
+            break;
+    }
+    AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Misaligned)) == 7);
+    if ((uint8_t)GCPhys & cbAlign)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Misaligned)[((uint8_t)GCPhys & cbAlign) - 1]);
+#endif
+
+    /*
+     * Make sure the pool page wasn't modified by a different CPU.
+     */
+    if (PHYS_PAGE_ADDRESS(GCPhys) == PHYS_PAGE_ADDRESS(pPage->GCPhys))
+    {
+        Assert(pPage->enmKind != PGMPOOLKIND_FREE);
+
+        /* The max modification count before flushing depends on the context and page type. */
+#ifdef IN_RING3
+        uint16_t const cMaxModifications = 96; /* it's cheaper here, right? */
+#else
+        uint16_t cMaxModifications;
+        if (    pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+            ||  pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
+            cMaxModifications = 4;
+        else
+            cMaxModifications = 24;
+#endif
+
+        /*
+         * We don't have to be very sophisticated about this since there are relativly few calls here.
+         * However, we must try our best to detect any non-cpu accesses (disk / networking).
+         */
+        if (   (   pPage->cModifications < cMaxModifications
+                || pgmPoolIsPageLocked(pPage) )
+            && enmOrigin != PGMACCESSORIGIN_DEVICE
+            && cbBuf <= 16)
+        {
+            /* Clear the shadow entry. */
+            if (!pPage->cModifications++)
+                pgmPoolMonitorModifiedInsert(pPool, pPage);
+
+            if (cbBuf <= 8)
+                pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, (uint32_t)cbBuf);
+            else
+            {
+                pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, 8);
+                pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys + 8, (uint8_t *)pvBuf + 8, (uint32_t)cbBuf - 8);
+            }
+        }
+        else
+            pgmPoolMonitorChainFlush(pPool, pPage);
+
+        STAM_PROFILE_STOP_EX(&pPool->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
+    }
+    else
+        Log(("CPU%d: PGM_ALL_CB_DECL pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
+    pgmUnlock(pVM);
+    return VINF_PGM_HANDLER_DO_DEFAULT;
+}
+
+
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+
+# if defined(VBOX_STRICT) && !defined(IN_RING3)
+
+/**
+ * Check references to guest physical memory in a PAE / PAE page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ * @param   pGstPT      The guest page table.
+ */
+static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
+{
+    unsigned cErrors    = 0;
+    int      LastRc     = -1;           /* initialized to shut up gcc */
+    unsigned LastPTE    = ~0U;          /* initialized to shut up gcc */
+    RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
+    PVMCC    pVM        = pPool->CTX_SUFF(pVM);
+
+#  ifdef VBOX_STRICT
+    for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
+        AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]),  pPage->iFirstPresent));
+#  endif
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            RTHCPHYS HCPhys = NIL_RTHCPHYS;
+            int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
+            if (    rc != VINF_SUCCESS
+                ||  PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
+            {
+                Log(("rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
+                LastPTE     = i;
+                LastRc      = rc;
+                LastHCPhys  = HCPhys;
+                cErrors++;
+
+                RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
+                rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
+                AssertRC(rc);
+
+                for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
+                {
+                    PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
+
+                    if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
+                    {
+                        PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
+
+                        for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
+                        {
+                            if (   PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
+                                && PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
+                            {
+                                Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
+                            }
+                        }
+
+                        PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
+                    }
+                }
+            }
+        }
+    }
+    AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
+}
+
+
+/**
+ * Check references to guest physical memory in a PAE / 32-bit page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ * @param   pGstPT      The guest page table.
+ */
+static void pgmPoolTrackCheckPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
+{
+    unsigned cErrors    = 0;
+    int      LastRc     = -1;           /* initialized to shut up gcc */
+    unsigned LastPTE    = ~0U;          /* initialized to shut up gcc */
+    RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
+    PVMCC    pVM        = pPool->CTX_SUFF(pVM);
+
+#  ifdef VBOX_STRICT
+    for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
+        AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]),  pPage->iFirstPresent));
+#  endif
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            RTHCPHYS HCPhys = NIL_RTHCPHYS;
+            int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
+            if (    rc != VINF_SUCCESS
+                ||  PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
+            {
+                Log(("rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
+                LastPTE     = i;
+                LastRc      = rc;
+                LastHCPhys  = HCPhys;
+                cErrors++;
+
+                RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
+                rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
+                AssertRC(rc);
+
+                for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
+                {
+                    PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
+
+                    if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
+                    {
+                        PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
+
+                        for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
+                        {
+                            if (   PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
+                                && PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
+                            {
+                                Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
+                            }
+                        }
+
+                        PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
+                    }
+                }
+            }
+        }
+    }
+    AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
+}
+
+# endif /* VBOX_STRICT && !IN_RING3 */
+
+/**
+ * Clear references to guest physical memory in a PAE / PAE page table.
+ *
+ * @returns nr of changed PTEs
+ * @param   pPool           The pool.
+ * @param   pPage           The page.
+ * @param   pShwPT          The shadow page table (mapping of the page).
+ * @param   pGstPT          The guest page table.
+ * @param   pOldGstPT       The old cached guest page table.
+ * @param   fAllowRemoval   Bail out as soon as we encounter an invalid PTE
+ * @param   pfFlush         Flush reused page table (out)
+ */
+DECLINLINE(unsigned) pgmPoolTrackFlushPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT,
+                                               PCX86PTPAE pOldGstPT, bool fAllowRemoval, bool *pfFlush)
+{
+    unsigned cChanged = 0;
+
+# ifdef VBOX_STRICT
+    for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
+        AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]),  pPage->iFirstPresent));
+# endif
+    *pfFlush = false;
+
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        /* Check the new value written by the guest. If present and with a bogus physical address, then
+         * it's fairly safe to assume the guest is reusing the PT.
+         */
+        if (    fAllowRemoval
+            &&  pGstPT->a[i].n.u1Present)
+        {
+            if (!PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
+            {
+                *pfFlush = true;
+                return ++cChanged;
+            }
+        }
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            /* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
+            if ((pGstPT->a[i].u & X86_PTE_PAE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
+            {
+# ifdef VBOX_STRICT
+                RTHCPHYS HCPhys = NIL_RTGCPHYS;
+                int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
+                AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %RX64 old %RX64 shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
+# endif
+                uint64_t uHostAttr  = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G | X86_PTE_PAE_NX);
+                bool     fHostRW    = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
+                uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G | X86_PTE_PAE_NX);
+                bool     fGuestRW   = !!(pGstPT->a[i].u & X86_PTE_RW);
+
+                if (    uHostAttr == uGuestAttr
+                    &&  fHostRW <= fGuestRW)
+                    continue;
+            }
+            cChanged++;
+            /* Something was changed, so flush it. */
+            Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%RX64\n",
+                  i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
+            pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK, i);
+            PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
+        }
+    }
+    return cChanged;
+}
+
+
+/**
+ * Clear references to guest physical memory in a PAE / PAE page table.
+ *
+ * @returns nr of changed PTEs
+ * @param   pPool           The pool.
+ * @param   pPage           The page.
+ * @param   pShwPT          The shadow page table (mapping of the page).
+ * @param   pGstPT          The guest page table.
+ * @param   pOldGstPT       The old cached guest page table.
+ * @param   fAllowRemoval   Bail out as soon as we encounter an invalid PTE
+ * @param   pfFlush         Flush reused page table (out)
+ */
+DECLINLINE(unsigned) pgmPoolTrackFlushPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT,
+                                                 PCX86PT pOldGstPT, bool fAllowRemoval, bool *pfFlush)
+{
+    unsigned cChanged = 0;
+
+# ifdef VBOX_STRICT
+    for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
+        AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]),  pPage->iFirstPresent));
+# endif
+    *pfFlush = false;
+
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        /* Check the new value written by the guest. If present and with a bogus physical address, then
+         * it's fairly safe to assume the guest is reusing the PT.
+         */
+        if (    fAllowRemoval
+            &&  pGstPT->a[i].n.u1Present)
+        {
+            if (!PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PG_MASK))
+            {
+                *pfFlush = true;
+                return ++cChanged;
+            }
+        }
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            /* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
+            if ((pGstPT->a[i].u & X86_PTE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PG_MASK))
+            {
+# ifdef VBOX_STRICT
+                RTHCPHYS HCPhys = NIL_RTGCPHYS;
+                int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
+                AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %x old %x shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
+# endif
+                uint64_t uHostAttr  = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G);
+                bool     fHostRW    = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
+                uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G);
+                bool     fGuestRW   = !!(pGstPT->a[i].u & X86_PTE_RW);
+
+                if (    uHostAttr == uGuestAttr
+                    &&  fHostRW <= fGuestRW)
+                    continue;
+            }
+            cChanged++;
+            /* Something was changed, so flush it. */
+            Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%x\n",
+                  i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK));
+            pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK, i);
+            PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
+        }
+    }
+    return cChanged;
+}
+
+
+/**
+ * Flush a dirty page
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pPool           The pool.
+ * @param   idxSlot         Dirty array slot index
+ * @param   fAllowRemoval   Allow a reused page table to be removed
+ */
+static void pgmPoolFlushDirtyPage(PVMCC pVM, PPGMPOOL pPool, unsigned idxSlot, bool fAllowRemoval = false)
+{
+    AssertCompile(RT_ELEMENTS(pPool->aidxDirtyPages) == RT_ELEMENTS(pPool->aDirtyPages));
+
+    Assert(idxSlot < RT_ELEMENTS(pPool->aDirtyPages));
+    unsigned idxPage = pPool->aidxDirtyPages[idxSlot];
+    if (idxPage == NIL_PGMPOOL_IDX)
+        return;
+
+    PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
+    Assert(pPage->idx == idxPage);
+    Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
+
+    AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, idxSlot));
+    Log(("Flush dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
+
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PVMCPU   pVCpu = VMMGetCpu(pVM);
+    uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
+# endif
+
+    /* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */
+    int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK);
+    Assert(rc == VINF_SUCCESS);
+    pPage->fDirty = false;
+
+# ifdef VBOX_STRICT
+    uint64_t fFlags = 0;
+    RTHCPHYS HCPhys;
+    rc = PGMShwGetPage(VMMGetCpu(pVM), pPage->GCPtrDirtyFault, &fFlags, &HCPhys);
+    AssertMsg(      (   rc == VINF_SUCCESS
+                     && (!(fFlags & X86_PTE_RW) || HCPhys != pPage->Core.Key))
+              /* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
+              ||    rc == VERR_PAGE_TABLE_NOT_PRESENT
+              ||    rc == VERR_PAGE_NOT_PRESENT,
+              ("PGMShwGetPage -> GCPtr=%RGv rc=%d flags=%RX64\n", pPage->GCPtrDirtyFault, rc, fFlags));
+# endif
+
+    /* Flush those PTEs that have changed. */
+    STAM_PROFILE_START(&pPool->StatTrackDeref,a);
+    void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+    void *pvGst;
+    rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+    bool  fFlush;
+    unsigned cChanges;
+
+    if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
+        cChanges = pgmPoolTrackFlushPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst,
+                                             (PCX86PTPAE)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
+    else
+        cChanges = pgmPoolTrackFlushPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst,
+                                               (PCX86PT)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
+
+    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
+    STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
+    /* Note: we might want to consider keeping the dirty page active in case there were many changes. */
+
+    /* This page is likely to be modified again, so reduce the nr of modifications just a bit here. */
+    Assert(pPage->cModifications);
+    if (cChanges < 4)
+        pPage->cModifications = 1;      /* must use > 0 here */
+    else
+        pPage->cModifications = RT_MAX(1, pPage->cModifications / 2);
+
+    STAM_COUNTER_INC(&pPool->StatResetDirtyPages);
+    if (pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages))
+        pPool->idxFreeDirtyPage = idxSlot;
+
+    pPool->cDirtyPages--;
+    pPool->aidxDirtyPages[idxSlot] = NIL_PGMPOOL_IDX;
+    Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
+    if (fFlush)
+    {
+        Assert(fAllowRemoval);
+        Log(("Flush reused page table!\n"));
+        pgmPoolFlushPage(pPool, pPage);
+        STAM_COUNTER_INC(&pPool->StatForceFlushReused);
+    }
+    else
+        Log(("Removed dirty page %RGp cMods=%d cChanges=%d\n", pPage->GCPhys, pPage->cModifications, cChanges));
+
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
+# endif
+}
+
+
+# ifndef IN_RING3
+/**
+ * Add a new dirty page
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ */
+void pgmPoolAddDirtyPage(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    AssertCompile(RT_ELEMENTS(pPool->aDirtyPages) == 8 || RT_ELEMENTS(pPool->aDirtyPages) == 16);
+    Assert(!pPage->fDirty);
+
+    unsigned idxFree = pPool->idxFreeDirtyPage;
+    Assert(idxFree < RT_ELEMENTS(pPool->aDirtyPages));
+    Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
+
+    if (pPool->cDirtyPages >= RT_ELEMENTS(pPool->aDirtyPages))
+    {
+        STAM_COUNTER_INC(&pPool->StatDirtyPageOverFlowFlush);
+        pgmPoolFlushDirtyPage(pVM, pPool, idxFree, true /* allow removal of reused page tables*/);
+    }
+    Assert(pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages));
+    AssertMsg(pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX, ("idxFree=%d cDirtyPages=%d\n", idxFree, pPool->cDirtyPages));
+
+    Log(("Add dirty page %RGp (slot=%d)\n", pPage->GCPhys, idxFree));
+
+    /*
+     * Make a copy of the guest page table as we require valid GCPhys addresses
+     * when removing references to physical pages.
+     * (The HCPhys linear lookup is *extremely* expensive!)
+     */
+    void *pvGst;
+    int   rc  = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+    memcpy(&pPool->aDirtyPages[idxFree].aPage[0], pvGst, (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT) ? PAGE_SIZE : PAGE_SIZE/2);
+#  ifdef VBOX_STRICT
+    void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+    if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
+        pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
+    else
+        pgmPoolTrackCheckPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
+    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
+#  endif
+    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+
+    STAM_COUNTER_INC(&pPool->StatDirtyPage);
+    pPage->fDirty                    = true;
+    pPage->idxDirtyEntry             = (uint8_t)idxFree; Assert(pPage->idxDirtyEntry == idxFree);
+    pPool->aidxDirtyPages[idxFree]   = pPage->idx;
+    pPool->cDirtyPages++;
+
+    pPool->idxFreeDirtyPage        = (pPool->idxFreeDirtyPage + 1) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
+    if (    pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages)
+        &&  pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
+    {
+        unsigned i;
+        for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+        {
+            idxFree = (pPool->idxFreeDirtyPage + i) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
+            if (pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX)
+            {
+                pPool->idxFreeDirtyPage = idxFree;
+                break;
+            }
+        }
+        Assert(i != RT_ELEMENTS(pPool->aDirtyPages));
+    }
+
+    Assert(pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages) || pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX);
+
+    /*
+     * Clear all references to this shadow table. See @bugref{7298}.
+     */
+    pgmPoolTrackClearPageUsers(pPool, pPage);
+}
+# endif /* !IN_RING3 */
+
+
+/**
+ * Check if the specified page is dirty (not write monitored)
+ *
+ * @return dirty or not
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Guest physical address
+ */
+bool pgmPoolIsDirtyPageSlow(PVM pVM, RTGCPHYS GCPhys)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    if (!pPool->cDirtyPages)
+        return false;
+
+    GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+    {
+        unsigned idxPage = pPool->aidxDirtyPages[i];
+        if (idxPage != NIL_PGMPOOL_IDX)
+        {
+            PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
+            if (pPage->GCPhys == GCPhys)
+                return true;
+        }
+    }
+    return false;
+}
+
+
+/**
+ * Reset all dirty pages by reinstating page monitoring.
+ *
+ * @param   pVM             The cross context VM structure.
+ */
+void pgmPoolResetDirtyPages(PVMCC pVM)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
+
+    if (!pPool->cDirtyPages)
+        return;
+
+    Log(("pgmPoolResetDirtyPages\n"));
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+        pgmPoolFlushDirtyPage(pVM, pPool, i, true /* allow removal of reused page tables*/);
+
+    pPool->idxFreeDirtyPage = 0;
+    if (    pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
+        &&  pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
+    {
+        unsigned i;
+        for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+        {
+            if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
+            {
+                pPool->idxFreeDirtyPage = i;
+                break;
+            }
+        }
+        AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
+    }
+
+    Assert(pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX || pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages));
+    return;
+}
+
+
+/**
+ * Invalidate the PT entry for the specified page
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPtrPage       Guest page to invalidate
+ */
+void pgmPoolResetDirtyPage(PVM pVM, RTGCPTR GCPtrPage)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
+
+    if (!pPool->cDirtyPages)
+        return;
+
+    Log(("pgmPoolResetDirtyPage %RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+    {
+    /** @todo What was intended here??? This looks incomplete... */
+    }
+}
+
+
+/**
+ * Reset all dirty pages by reinstating page monitoring.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhysPT        Physical address of the page table
+ */
+void pgmPoolInvalidateDirtyPage(PVMCC pVM, RTGCPHYS GCPhysPT)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
+    unsigned idxDirtyPage = RT_ELEMENTS(pPool->aDirtyPages);
+
+    if (!pPool->cDirtyPages)
+        return;
+
+    GCPhysPT = GCPhysPT & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+    {
+        unsigned idxPage = pPool->aidxDirtyPages[i];
+        if (idxPage != NIL_PGMPOOL_IDX)
+        {
+            PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
+            if (pPage->GCPhys == GCPhysPT)
+            {
+                idxDirtyPage = i;
+                break;
+            }
+        }
+    }
+
+    if (idxDirtyPage != RT_ELEMENTS(pPool->aDirtyPages))
+    {
+        pgmPoolFlushDirtyPage(pVM, pPool, idxDirtyPage, true /* allow removal of reused page tables*/);
+        if (    pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
+            &&  pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
+        {
+            unsigned i;
+            for (i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+            {
+                if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
+                {
+                    pPool->idxFreeDirtyPage = i;
+                    break;
+                }
+            }
+            AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
+        }
+    }
+}
+
+#endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
+
+/**
+ * Inserts a page into the GCPhys hash table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ */
+DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
+    Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
+    uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
+    pPage->iNext = pPool->aiHash[iHash];
+    pPool->aiHash[iHash] = pPage->idx;
+}
+
+
+/**
+ * Removes a page from the GCPhys hash table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ */
+DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
+    uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
+    if (pPool->aiHash[iHash] == pPage->idx)
+        pPool->aiHash[iHash] = pPage->iNext;
+    else
+    {
+        uint16_t iPrev = pPool->aiHash[iHash];
+        for (;;)
+        {
+            const int16_t i = pPool->aPages[iPrev].iNext;
+            if (i == pPage->idx)
+            {
+                pPool->aPages[iPrev].iNext = pPage->iNext;
+                break;
+            }
+            if (i == NIL_PGMPOOL_IDX)
+            {
+                AssertReleaseMsgFailed(("GCPhys=%RGp idx=%d\n", pPage->GCPhys, pPage->idx));
+                break;
+            }
+            iPrev = i;
+        }
+    }
+    pPage->iNext = NIL_PGMPOOL_IDX;
+}
+
+
+/**
+ * Frees up one cache page.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @param   pPool       The pool.
+ * @param   iUser       The user index.
+ */
+static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
+{
+    const PVMCC pVM = pPool->CTX_SUFF(pVM);
+    Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
+    STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
+
+    /*
+     * Select one page from the tail of the age list.
+     */
+    PPGMPOOLPAGE    pPage;
+    for (unsigned iLoop = 0; ; iLoop++)
+    {
+        uint16_t iToFree = pPool->iAgeTail;
+        if (iToFree == iUser && iUser != NIL_PGMPOOL_IDX)
+            iToFree = pPool->aPages[iToFree].iAgePrev;
+/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
+        if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
+        {
+            uint16_t i = pPool->aPages[iToFree].iAgePrev;
+            for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
+            {
+                if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
+                    continue;
+                iToFree = i;
+                break;
+            }
+        }
+*/
+        Assert(iToFree != iUser);
+        AssertReleaseMsg(iToFree != NIL_PGMPOOL_IDX,
+                         ("iToFree=%#x (iAgeTail=%#x) iUser=%#x iLoop=%u - pPool=%p LB %#zx\n",
+                          iToFree, pPool->iAgeTail, iUser, iLoop, pPool,
+                            RT_UOFFSETOF_DYN(PGMPOOL, aPages[pPool->cMaxPages])
+                          + pPool->cMaxUsers * sizeof(PGMPOOLUSER)
+                          + pPool->cMaxPhysExts * sizeof(PGMPOOLPHYSEXT) ));
+
+        pPage = &pPool->aPages[iToFree];
+
+        /*
+         * Reject any attempts at flushing the currently active shadow CR3 mapping.
+         * Call pgmPoolCacheUsed to move the page to the head of the age list.
+         */
+        if (   !pgmPoolIsPageLocked(pPage)
+            && pPage->idx >= PGMPOOL_IDX_FIRST /* paranoia (#6349) */)
+            break;
+        LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
+        pgmPoolCacheUsed(pPool, pPage);
+        AssertLogRelReturn(iLoop < 8192, VERR_PGM_POOL_TOO_MANY_LOOPS);
+    }
+
+    /*
+     * Found a usable page, flush it and return.
+     */
+    int rc = pgmPoolFlushPage(pPool, pPage);
+    /* This flush was initiated by us and not the guest, so explicitly flush the TLB. */
+    /** @todo find out why this is necessary; pgmPoolFlushPage should trigger a flush if one is really needed. */
+    if (rc == VINF_SUCCESS)
+        PGM_INVL_ALL_VCPU_TLBS(pVM);
+    return rc;
+}
+
+
+/**
+ * Checks if a kind mismatch is really a page being reused
+ * or if it's just normal remappings.
+ *
+ * @returns true if reused and the cached page (enmKind1) should be flushed
+ * @returns false if not reused.
+ * @param   enmKind1    The kind of the cached page.
+ * @param   enmKind2    The kind of the requested page.
+ */
+static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
+{
+    switch (enmKind1)
+    {
+        /*
+         * Never reuse them. There is no remapping in non-paging mode.
+         */
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
+            return false;
+
+        /*
+         * It's perfectly fine to reuse these, except for PAE and non-paging stuff.
+         */
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_PAE_PDPT:
+            switch (enmKind2)
+            {
+                case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+                case PGMPOOLKIND_64BIT_PML4:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                    return true;
+                default:
+                    return false;
+            }
+
+        /*
+         * It's perfectly fine to reuse these, except for PAE and non-paging stuff.
+         */
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+        case PGMPOOLKIND_64BIT_PML4:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+            switch (enmKind2)
+            {
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                    return true;
+                default:
+                    return false;
+            }
+
+        /*
+         * These cannot be flushed, and it's common to reuse the PDs as PTs.
+         */
+        case PGMPOOLKIND_ROOT_NESTED:
+            return false;
+
+        default:
+            AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
+    }
+}
+
+
+/**
+ * Attempts to satisfy a pgmPoolAlloc request from the cache.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_PGM_CACHED_PAGE on success.
+ * @retval  VERR_FILE_NOT_FOUND if not found.
+ * @param   pPool       The pool.
+ * @param   GCPhys      The GC physical address of the page we're gonna shadow.
+ * @param   enmKind     The kind of mapping.
+ * @param   enmAccess   Access type for the mapping (only relevant for big pages)
+ * @param   fA20Enabled Whether the CPU has the A20 gate enabled.
+ * @param   iUser       The shadow page pool index of the user table.  This is
+ *                      NIL_PGMPOOL_IDX for root pages.
+ * @param   iUserTable  The index into the user table (shadowed).  Ignored if
+ *                      root page
+ * @param   ppPage      Where to store the pointer to the page.
+ */
+static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
+                             uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
+{
+    /*
+     * Look up the GCPhys in the hash.
+     */
+    unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
+    Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
+    if (i != NIL_PGMPOOL_IDX)
+    {
+        do
+        {
+            PPGMPOOLPAGE pPage = &pPool->aPages[i];
+            Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
+            if (pPage->GCPhys == GCPhys)
+            {
+                if (   (PGMPOOLKIND)pPage->enmKind == enmKind
+                    && (PGMPOOLACCESS)pPage->enmAccess == enmAccess
+                    && pPage->fA20Enabled == fA20Enabled)
+                {
+                    /* Put it at the start of the use list to make sure pgmPoolTrackAddUser
+                     * doesn't flush it in case there are no more free use records.
+                     */
+                    pgmPoolCacheUsed(pPool, pPage);
+
+                    int rc = VINF_SUCCESS;
+                    if (iUser != NIL_PGMPOOL_IDX)
+                        rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
+                    if (RT_SUCCESS(rc))
+                    {
+                        Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
+                        *ppPage = pPage;
+                        if (pPage->cModifications)
+                            pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
+                        STAM_COUNTER_INC(&pPool->StatCacheHits);
+                        return VINF_PGM_CACHED_PAGE;
+                    }
+                    return rc;
+                }
+
+                if ((PGMPOOLKIND)pPage->enmKind != enmKind)
+                {
+                    /*
+                     * The kind is different. In some cases we should now flush the page
+                     * as it has been reused, but in most cases this is normal remapping
+                     * of PDs as PT or big pages using the GCPhys field in a slightly
+                     * different way than the other kinds.
+                     */
+                    if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
+                    {
+                        STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
+                        pgmPoolFlushPage(pPool, pPage);
+                        break;
+                    }
+                }
+            }
+
+            /* next */
+            i = pPage->iNext;
+        } while (i != NIL_PGMPOOL_IDX);
+    }
+
+    Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
+    STAM_COUNTER_INC(&pPool->StatCacheMisses);
+    return VERR_FILE_NOT_FOUND;
+}
+
+
+/**
+ * Inserts a page into the cache.
+ *
+ * @param   pPool           The pool.
+ * @param   pPage           The cached page.
+ * @param   fCanBeCached    Set if the page is fit for caching from the caller's point of view.
+ */
+static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
+{
+    /*
+     * Insert into the GCPhys hash if the page is fit for that.
+     */
+    Assert(!pPage->fCached);
+    if (fCanBeCached)
+    {
+        pPage->fCached = true;
+        pgmPoolHashInsert(pPool, pPage);
+        Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
+              pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
+        STAM_COUNTER_INC(&pPool->StatCacheCacheable);
+    }
+    else
+    {
+        Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
+              pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
+        STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
+    }
+
+    /*
+     * Insert at the head of the age list.
+     */
+    pPage->iAgePrev = NIL_PGMPOOL_IDX;
+    pPage->iAgeNext = pPool->iAgeHead;
+    if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
+        pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
+    else
+        pPool->iAgeTail = pPage->idx;
+    pPool->iAgeHead = pPage->idx;
+}
+
+
+/**
+ * Flushes a cached page.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ */
+static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
+
+    /*
+     * Remove the page from the hash.
+     */
+    if (pPage->fCached)
+    {
+        pPage->fCached = false;
+        pgmPoolHashRemove(pPool, pPage);
+    }
+    else
+        Assert(pPage->iNext == NIL_PGMPOOL_IDX);
+
+    /*
+     * Remove it from the age list.
+     */
+    if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
+        pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
+    else
+        pPool->iAgeTail = pPage->iAgePrev;
+    if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
+        pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
+    else
+        pPool->iAgeHead = pPage->iAgeNext;
+    pPage->iAgeNext = NIL_PGMPOOL_IDX;
+    pPage->iAgePrev = NIL_PGMPOOL_IDX;
+}
+
+
+/**
+ * Looks for pages sharing the monitor.
+ *
+ * @returns Pointer to the head page.
+ * @returns NULL if not found.
+ * @param   pPool       The Pool
+ * @param   pNewPage    The page which is going to be monitored.
+ */
+static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
+{
+    /*
+     * Look up the GCPhys in the hash.
+     */
+    RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+    unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
+    if (i == NIL_PGMPOOL_IDX)
+        return NULL;
+    do
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[i];
+        if (    pPage->GCPhys - GCPhys < PAGE_SIZE
+            &&  pPage != pNewPage)
+        {
+            switch (pPage->enmKind)
+            {
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+                case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+                case PGMPOOLKIND_64BIT_PML4:
+                case PGMPOOLKIND_32BIT_PD:
+                case PGMPOOLKIND_PAE_PDPT:
+                {
+                    /* find the head */
+                    while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
+                    {
+                        Assert(pPage->iMonitoredPrev != pPage->idx);
+                        pPage = &pPool->aPages[pPage->iMonitoredPrev];
+                    }
+                    return pPage;
+                }
+
+                /* ignore, no monitoring. */
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                case PGMPOOLKIND_ROOT_NESTED:
+                case PGMPOOLKIND_PAE_PD_PHYS:
+                case PGMPOOLKIND_PAE_PDPT_PHYS:
+                case PGMPOOLKIND_32BIT_PD_PHYS:
+                case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+                    break;
+                default:
+                    AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
+            }
+        }
+
+        /* next */
+        i = pPage->iNext;
+    } while (i != NIL_PGMPOOL_IDX);
+    return NULL;
+}
+
+
+/**
+ * Enabled write monitoring of a guest page.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ */
+static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK));
+
+    /*
+     * Filter out the relevant kinds.
+     */
+    switch (pPage->enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+        case PGMPOOLKIND_64BIT_PML4:
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_PAE_PDPT:
+            break;
+
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+        case PGMPOOLKIND_ROOT_NESTED:
+            /* Nothing to monitor here. */
+            return VINF_SUCCESS;
+
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+            /* Nothing to monitor here. */
+            return VINF_SUCCESS;
+        default:
+            AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
+    }
+
+    /*
+     * Install handler.
+     */
+    int rc;
+    PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
+    if (pPageHead)
+    {
+        Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
+        Assert(pPageHead->iMonitoredPrev != pPage->idx);
+
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+        if (pPageHead->fDirty)
+            pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPageHead->idxDirtyEntry, false /* do not remove */);
+#endif
+
+        pPage->iMonitoredPrev = pPageHead->idx;
+        pPage->iMonitoredNext = pPageHead->iMonitoredNext;
+        if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
+            pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
+        pPageHead->iMonitoredNext = pPage->idx;
+        rc = VINF_SUCCESS;
+    }
+    else
+    {
+        Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
+        PVMCC pVM = pPool->CTX_SUFF(pVM);
+        const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+        rc = PGMHandlerPhysicalRegister(pVM, GCPhysPage, GCPhysPage + PAGE_OFFSET_MASK, pPool->hAccessHandlerType,
+                                        MMHyperCCToR3(pVM, pPage), MMHyperCCToR0(pVM, pPage), MMHyperCCToRC(pVM, pPage),
+                                        NIL_RTR3PTR /*pszDesc*/);
+        /** @todo we should probably deal with out-of-memory conditions here, but for now increasing
+         * the heap size should suffice. */
+        AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc));
+        PVMCPU pVCpu = VMMGetCpu(pVM);
+        AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), ("fSyncFlags=%x syncff=%d\n", pVCpu->pgm.s.fSyncFlags, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
+    }
+    pPage->fMonitored = true;
+    return rc;
+}
+
+
+/**
+ * Disables write monitoring of a guest page.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ */
+static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    /*
+     * Filter out the relevant kinds.
+     */
+    switch (pPage->enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+        case PGMPOOLKIND_64BIT_PML4:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_PAE_PDPT:
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+            break;
+
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+        case PGMPOOLKIND_ROOT_NESTED:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            /* Nothing to monitor here. */
+            Assert(!pPage->fMonitored);
+            return VINF_SUCCESS;
+
+        default:
+            AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
+    }
+    Assert(pPage->fMonitored);
+
+    /*
+     * Remove the page from the monitored list or uninstall it if last.
+     */
+    const PVMCC pVM = pPool->CTX_SUFF(pVM);
+    int rc;
+    if (    pPage->iMonitoredNext != NIL_PGMPOOL_IDX
+        ||  pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
+    {
+        if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
+        {
+            PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
+            pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
+            rc = PGMHandlerPhysicalChangeUserArgs(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK,
+                                                  MMHyperCCToR3(pVM, pNewHead), MMHyperCCToR0(pVM, pNewHead));
+
+            AssertFatalRCSuccess(rc);
+            pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
+        }
+        else
+        {
+            pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
+            if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
+            {
+                pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
+                pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
+            }
+            pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
+            rc = VINF_SUCCESS;
+        }
+    }
+    else
+    {
+        rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
+        AssertFatalRC(rc);
+        PVMCPU pVCpu = VMMGetCpu(pVM);
+        AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3),
+                  ("%#x %#x\n", pVCpu->pgm.s.fSyncFlags, pVM->fGlobalForcedActions));
+    }
+    pPage->fMonitored = false;
+
+    /*
+     * Remove it from the list of modified pages (if in it).
+     */
+    pgmPoolMonitorModifiedRemove(pPool, pPage);
+
+    return rc;
+}
+
+
+/**
+ * Inserts the page into the list of modified pages.
+ *
+ * @param   pPool   The pool.
+ * @param   pPage   The page.
+ */
+void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
+    AssertMsg(   pPage->iModifiedNext == NIL_PGMPOOL_IDX
+              && pPage->iModifiedPrev == NIL_PGMPOOL_IDX
+              && pPool->iModifiedHead != pPage->idx,
+              ("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
+               pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
+               pPool->iModifiedHead, pPool->cModifiedPages));
+
+    pPage->iModifiedNext = pPool->iModifiedHead;
+    if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
+        pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
+    pPool->iModifiedHead = pPage->idx;
+    pPool->cModifiedPages++;
+#ifdef VBOX_WITH_STATISTICS
+    if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
+        pPool->cModifiedPagesHigh = pPool->cModifiedPages;
+#endif
+}
+
+
+/**
+ * Removes the page from the list of modified pages and resets the
+ * modification counter.
+ *
+ * @param   pPool   The pool.
+ * @param   pPage   The page which is believed to be in the list of modified pages.
+ */
+static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
+    if (pPool->iModifiedHead == pPage->idx)
+    {
+        Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
+        pPool->iModifiedHead = pPage->iModifiedNext;
+        if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
+        {
+            pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
+            pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+        }
+        pPool->cModifiedPages--;
+    }
+    else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
+    {
+        pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
+        if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
+        {
+            pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
+            pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+        }
+        pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
+        pPool->cModifiedPages--;
+    }
+    else
+        Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
+    pPage->cModifications = 0;
+}
+
+
+/**
+ * Zaps the list of modified pages, resetting their modification counters in the process.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+static void pgmPoolMonitorModifiedClearAll(PVMCC pVM)
+{
+    pgmLock(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
+
+    unsigned cPages = 0; NOREF(cPages);
+
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    pgmPoolResetDirtyPages(pVM);
+#endif
+
+    uint16_t idx = pPool->iModifiedHead;
+    pPool->iModifiedHead = NIL_PGMPOOL_IDX;
+    while (idx != NIL_PGMPOOL_IDX)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[idx];
+        idx = pPage->iModifiedNext;
+        pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+        pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
+        pPage->cModifications = 0;
+        Assert(++cPages);
+    }
+    AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
+    pPool->cModifiedPages = 0;
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Handle SyncCR3 pool tasks
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully added.
+ * @retval  VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
+ * @param   pVCpu     The cross context virtual CPU structure.
+ * @remark  Should only be used when monitoring is available, thus placed in
+ *          the PGMPOOL_WITH_MONITORING \#ifdef.
+ */
+int pgmPoolSyncCR3(PVMCPUCC pVCpu)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    LogFlow(("pgmPoolSyncCR3 fSyncFlags=%x\n", pVCpu->pgm.s.fSyncFlags));
+
+    /*
+     * When monitoring shadowed pages, we reset the modification counters on CR3 sync.
+     * Occasionally we will have to clear all the shadow page tables because we wanted
+     * to monitor a page which was mapped by too many shadowed page tables. This operation
+     * sometimes referred to as a 'lightweight flush'.
+     */
+# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
+    if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
+        pgmR3PoolClearAll(pVM, false /*fFlushRemTlb*/);
+# else  /* !IN_RING3 */
+    if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
+    {
+        Log(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
+
+        /* Make sure all other VCPUs return to ring 3. */
+        if (pVM->cCpus > 1)
+        {
+            VM_FF_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
+            PGM_INVL_ALL_VCPU_TLBS(pVM);
+        }
+        return VINF_PGM_SYNC_CR3;
+    }
+# endif /* !IN_RING3 */
+    else
+    {
+        pgmPoolMonitorModifiedClearAll(pVM);
+
+        /* pgmPoolMonitorModifiedClearAll can cause a pgm pool flush (dirty page clearing), so make sure we handle this! */
+        if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
+        {
+            Log(("pgmPoolMonitorModifiedClearAll caused a pgm flush -> call pgmPoolSyncCR3 again!\n"));
+            return pgmPoolSyncCR3(pVCpu);
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Frees up at least one user entry.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully added.
+ *
+ * @param   pPool       The pool.
+ * @param   iUser       The user index.
+ */
+static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
+{
+    STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
+    /*
+     * Just free cached pages in a braindead fashion.
+     */
+    /** @todo walk the age list backwards and free the first with usage. */
+    int rc = VINF_SUCCESS;
+    do
+    {
+        int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
+        if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
+            rc = rc2;
+    } while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
+    return rc;
+}
+
+
+/**
+ * Inserts a page into the cache.
+ *
+ * This will create user node for the page, insert it into the GCPhys
+ * hash, and insert it into the age list.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully added.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ * @param   GCPhys      The GC physical address of the page we're gonna shadow.
+ * @param   iUser       The user index.
+ * @param   iUserTable  The user table index.
+ */
+DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
+{
+    int rc = VINF_SUCCESS;
+    PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
+
+    LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser=%d iUserTable=%x\n", GCPhys, iUser, iUserTable)); RT_NOREF_PV(GCPhys);
+
+    if (iUser != NIL_PGMPOOL_IDX)
+    {
+#ifdef VBOX_STRICT
+        /*
+         * Check that the entry doesn't already exists.
+         */
+        if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
+        {
+            uint16_t i = pPage->iUserHead;
+            do
+            {
+                Assert(i < pPool->cMaxUsers);
+                AssertMsg(paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
+                i = paUsers[i].iNext;
+            } while (i != NIL_PGMPOOL_USER_INDEX);
+        }
+#endif
+
+        /*
+         * Find free a user node.
+         */
+        uint16_t i = pPool->iUserFreeHead;
+        if (i == NIL_PGMPOOL_USER_INDEX)
+        {
+            rc = pgmPoolTrackFreeOneUser(pPool, iUser);
+            if (RT_FAILURE(rc))
+                return rc;
+            i = pPool->iUserFreeHead;
+        }
+
+        /*
+         * Unlink the user node from the free list,
+         * initialize and insert it into the user list.
+         */
+        pPool->iUserFreeHead = paUsers[i].iNext;
+        paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
+        paUsers[i].iUser = iUser;
+        paUsers[i].iUserTable = iUserTable;
+        pPage->iUserHead = i;
+    }
+    else
+        pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
+
+
+    /*
+     * Insert into cache and enable monitoring of the guest page if enabled.
+     *
+     * Until we implement caching of all levels, including the CR3 one, we'll
+     * have to make sure we don't try monitor & cache any recursive reuse of
+     * a monitored CR3 page. Because all windows versions are doing this we'll
+     * have to be able to do combined access monitoring, CR3 + PT and
+     * PD + PT (guest PAE).
+     *
+     * Update:
+     * We're now cooperating with the CR3 monitor if an uncachable page is found.
+     */
+    const bool fCanBeMonitored = true;
+    pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
+    if (fCanBeMonitored)
+    {
+        rc = pgmPoolMonitorInsert(pPool, pPage);
+        AssertRC(rc);
+    }
+    return rc;
+}
+
+
+/**
+ * Adds a user reference to a page.
+ *
+ * This will move the page to the head of the
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully added.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ * @param   iUser       The user index.
+ * @param   iUserTable  The user table.
+ */
+static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
+{
+    Log3(("pgmPoolTrackAddUser: GCPhys=%RGp iUser=%x iUserTable=%x\n", pPage->GCPhys, iUser, iUserTable));
+    PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
+    Assert(iUser != NIL_PGMPOOL_IDX);
+
+#  ifdef VBOX_STRICT
+    /*
+     * Check that the entry doesn't already exists. We only allow multiple
+     * users of top-level paging structures (SHW_POOL_ROOT_IDX).
+     */
+    if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
+    {
+        uint16_t i = pPage->iUserHead;
+        do
+        {
+            Assert(i < pPool->cMaxUsers);
+            /** @todo this assertion looks odd... Shouldn't it be && here? */
+            AssertMsg(paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
+            i = paUsers[i].iNext;
+        } while (i != NIL_PGMPOOL_USER_INDEX);
+    }
+#  endif
+
+    /*
+     * Allocate a user node.
+     */
+    uint16_t i = pPool->iUserFreeHead;
+    if (i == NIL_PGMPOOL_USER_INDEX)
+    {
+        int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
+        if (RT_FAILURE(rc))
+            return rc;
+        i = pPool->iUserFreeHead;
+    }
+    pPool->iUserFreeHead = paUsers[i].iNext;
+
+    /*
+     * Initialize the user node and insert it.
+     */
+    paUsers[i].iNext = pPage->iUserHead;
+    paUsers[i].iUser = iUser;
+    paUsers[i].iUserTable = iUserTable;
+    pPage->iUserHead = i;
+
+#  ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    if (pPage->fDirty)
+        pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPage->idxDirtyEntry, false /* do not remove */);
+#  endif
+
+    /*
+     * Tell the cache to update its replacement stats for this page.
+     */
+    pgmPoolCacheUsed(pPool, pPage);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Frees a user record associated with a page.
+ *
+ * This does not clear the entry in the user table, it simply replaces the
+ * user record to the chain of free records.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The shadow page.
+ * @param   iUser       The shadow page pool index of the user table.
+ * @param   iUserTable  The index into the user table (shadowed).
+ *
+ * @remarks Don't call this for root pages.
+ */
+static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
+{
+    Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
+    PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
+    Assert(iUser != NIL_PGMPOOL_IDX);
+
+    /*
+     * Unlink and free the specified user entry.
+     */
+
+    /* Special: For PAE and 32-bit paging, there is usually no more than one user. */
+    uint16_t i = pPage->iUserHead;
+    if (    i != NIL_PGMPOOL_USER_INDEX
+        &&  paUsers[i].iUser == iUser
+        &&  paUsers[i].iUserTable == iUserTable)
+    {
+        pPage->iUserHead = paUsers[i].iNext;
+
+        paUsers[i].iUser = NIL_PGMPOOL_IDX;
+        paUsers[i].iNext = pPool->iUserFreeHead;
+        pPool->iUserFreeHead = i;
+        return;
+    }
+
+    /* General: Linear search. */
+    uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
+    while (i != NIL_PGMPOOL_USER_INDEX)
+    {
+        if (    paUsers[i].iUser == iUser
+            &&  paUsers[i].iUserTable == iUserTable)
+        {
+            if (iPrev != NIL_PGMPOOL_USER_INDEX)
+                paUsers[iPrev].iNext = paUsers[i].iNext;
+            else
+                pPage->iUserHead = paUsers[i].iNext;
+
+            paUsers[i].iUser = NIL_PGMPOOL_IDX;
+            paUsers[i].iNext = pPool->iUserFreeHead;
+            pPool->iUserFreeHead = i;
+            return;
+        }
+        iPrev = i;
+        i = paUsers[i].iNext;
+    }
+
+    /* Fatal: didn't find it */
+    AssertFatalMsgFailed(("Didn't find the user entry! iUser=%d iUserTable=%#x GCPhys=%RGp\n",
+                          iUser, iUserTable, pPage->GCPhys));
+}
+
+
+#if 0 /* unused */
+/**
+ * Gets the entry size of a shadow table.
+ *
+ * @param   enmKind     The kind of page.
+ *
+ * @returns The size of the entry in bytes. That is, 4 or 8.
+ * @returns If the kind is not for a table, an assertion is raised and 0 is
+ *          returned.
+ */
+DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
+{
+    switch (enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            return 4;
+
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+        case PGMPOOLKIND_64BIT_PML4:
+        case PGMPOOLKIND_PAE_PDPT:
+        case PGMPOOLKIND_ROOT_NESTED:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+            return 8;
+
+        default:
+            AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
+    }
+}
+#endif /* unused */
+
+#if 0 /* unused */
+/**
+ * Gets the entry size of a guest table.
+ *
+ * @param   enmKind     The kind of page.
+ *
+ * @returns The size of the entry in bytes. That is, 0, 4 or 8.
+ * @returns If the kind is not for a table, an assertion is raised and 0 is
+ *          returned.
+ */
+DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
+{
+    switch (enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+            return 4;
+
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+        case PGMPOOLKIND_64BIT_PML4:
+        case PGMPOOLKIND_PAE_PDPT:
+            return 8;
+
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+        case PGMPOOLKIND_ROOT_NESTED:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            /** @todo can we return 0? (nobody is calling this...) */
+            AssertFailed();
+            return 0;
+
+        default:
+            AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
+    }
+}
+#endif /* unused */
+
+
+/**
+ * Checks one shadow page table entry for a mapping of a physical page.
+ *
+ * @returns true / false indicating removal of all relevant PTEs
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPhysPage   The guest page in question.
+ * @param   fFlushPTEs  Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
+ * @param   iShw        The shadow page table.
+ * @param   iPte        Page table entry or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown
+ */
+static bool pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw, uint16_t iPte)
+{
+    LogFlow(("pgmPoolTrackFlushGCPhysPTInt: pPhysPage=%RHp iShw=%d iPte=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw, iPte));
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    bool     fRet  = false;
+
+    /*
+     * Assert sanity.
+     */
+    Assert(iPte != NIL_PGMPOOL_PHYSEXT_IDX_PTE);
+    AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
+    PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
+
+    /*
+     * Then, clear the actual mappings to the page in the shadow PT.
+     */
+    switch (pPage->enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        {
+            const uint32_t  u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
+            PX86PT          pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+            uint32_t        u32AndMask = 0;
+            uint32_t        u32OrMask  = 0;
+
+            if (!fFlushPTEs)
+            {
+                switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
+                {
+                    case PGM_PAGE_HNDL_PHYS_STATE_NONE:         /* No handler installed. */
+                    case PGM_PAGE_HNDL_PHYS_STATE_DISABLED:     /* Monitoring is temporarily disabled. */
+                        u32OrMask = X86_PTE_RW;
+                        u32AndMask = UINT32_MAX;
+                        fRet = true;
+                        STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
+                        break;
+
+                    case PGM_PAGE_HNDL_PHYS_STATE_WRITE:        /* Write access is monitored. */
+                        u32OrMask = 0;
+                        u32AndMask = ~X86_PTE_RW;
+                        fRet = true;
+                        STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
+                        break;
+                    default:
+                        /* (shouldn't be here, will assert below) */
+                        STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
+                        break;
+                }
+            }
+            else
+                STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
+
+            /* Update the counter if we're removing references. */
+            if (!u32AndMask)
+            {
+                Assert(pPage->cPresent);
+                Assert(pPool->cPresent);
+                pPage->cPresent--;
+                pPool->cPresent--;
+            }
+
+            if ((pPT->a[iPte].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
+            {
+                X86PTE Pte;
+
+                Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32\n", iPte, pPT->a[iPte]));
+                Pte.u = (pPT->a[iPte].u & u32AndMask) | u32OrMask;
+                if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
+                    Pte.n.u1Write = 0;    /* need to disallow writes when dirty bit tracking is still active. */
+
+                ASMAtomicWriteU32(&pPT->a[iPte].u, Pte.u);
+                PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
+                return fRet;
+            }
+#ifdef LOG_ENABLED
+            Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
+            for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
+                if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
+                {
+                    Log(("i=%d cFound=%d\n", i, ++cFound));
+                }
+#endif
+            AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u32=%RX32 poolkind=%x\n", pPage->iFirstPresent, pPage->cPresent, u32, pPage->enmKind));
+            /*PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);*/
+            break;
+        }
+
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:   /* physical mask the same as PAE; RW bit as well; be careful! */
+        {
+            const uint64_t  u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
+            PPGMSHWPTPAE    pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+            uint64_t        u64OrMask  = 0;
+            uint64_t        u64AndMask = 0;
+
+            if (!fFlushPTEs)
+            {
+                switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
+                {
+                    case PGM_PAGE_HNDL_PHYS_STATE_NONE:         /* No handler installed. */
+                    case PGM_PAGE_HNDL_PHYS_STATE_DISABLED:     /* Monitoring is temporarily disabled. */
+                        u64OrMask = X86_PTE_RW;
+                        u64AndMask = UINT64_MAX;
+                        fRet = true;
+                        STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
+                        break;
+
+                    case PGM_PAGE_HNDL_PHYS_STATE_WRITE:        /* Write access is monitored. */
+                        u64OrMask = 0;
+                        u64AndMask = ~(uint64_t)X86_PTE_RW;
+                        fRet = true;
+                        STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
+                        break;
+
+                    default:
+                        /* (shouldn't be here, will assert below) */
+                        STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
+                        break;
+                }
+            }
+            else
+                STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
+
+            /* Update the counter if we're removing references. */
+            if (!u64AndMask)
+            {
+                Assert(pPage->cPresent);
+                Assert(pPool->cPresent);
+                pPage->cPresent--;
+                pPool->cPresent--;
+            }
+
+            if ((PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK | X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX)) == u64)
+            {
+                X86PTEPAE Pte;
+
+                Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64\n", iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
+                Pte.u = (PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & u64AndMask) | u64OrMask;
+                if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
+                    Pte.n.u1Write = 0;    /* need to disallow writes when dirty bit tracking is still active. */
+
+                PGMSHWPTEPAE_ATOMIC_SET(pPT->a[iPte], Pte.u);
+                PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
+                return fRet;
+            }
+#ifdef LOG_ENABLED
+            Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
+            Log(("Found %RX64 expected %RX64\n", PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK | X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX), u64));
+            for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
+                if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & (X86_PTE_PAE_PG_MASK | X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX)) == u64)
+                    Log(("i=%d cFound=%d\n", i, ++cFound));
+#endif
+            AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u64=%RX64 poolkind=%x iPte=%d PT=%RX64\n", pPage->iFirstPresent, pPage->cPresent, u64, pPage->enmKind, iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
+            /*PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);*/
+            break;
+        }
+
+#ifdef PGM_WITH_LARGE_PAGES
+        /* Large page case only. */
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+        {
+            Assert(pVM->pgm.s.fNestedPaging);
+
+            const uint64_t  u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PDE4M_P | X86_PDE4M_PS;
+            PEPTPD          pPD = (PEPTPD)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+
+            if ((pPD->a[iPte].u & (EPT_PDE2M_PG_MASK | X86_PDE4M_P | X86_PDE4M_PS)) == u64)
+            {
+                Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
+                STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
+                pPD->a[iPte].u = 0;
+                PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
+
+                /* Update the counter as we're removing references. */
+                Assert(pPage->cPresent);
+                Assert(pPool->cPresent);
+                pPage->cPresent--;
+                pPool->cPresent--;
+
+                return fRet;
+            }
+# ifdef LOG_ENABLED
+            Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
+            for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
+                if ((pPD->a[i].u & (EPT_PDE2M_PG_MASK | X86_PDE4M_P | X86_PDE4M_PS)) == u64)
+                    Log(("i=%d cFound=%d\n", i, ++cFound));
+# endif
+            AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
+            /*PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);*/
+            break;
+        }
+
+        /* AMD-V nested paging */ /** @todo merge with EPT as we only check the parts that are identical. */
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        {
+            Assert(pVM->pgm.s.fNestedPaging);
+
+            const uint64_t  u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PDE4M_P | X86_PDE4M_PS;
+            PX86PD          pPD = (PX86PD)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+
+            if ((pPD->a[iPte].u & (X86_PDE2M_PAE_PG_MASK | X86_PDE4M_P | X86_PDE4M_PS)) == u64)
+            {
+                Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
+                STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
+                pPD->a[iPte].u = 0;
+                PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
+
+                /* Update the counter as we're removing references. */
+                Assert(pPage->cPresent);
+                Assert(pPool->cPresent);
+                pPage->cPresent--;
+                pPool->cPresent--;
+                return fRet;
+            }
+# ifdef LOG_ENABLED
+            Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
+            for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
+                if ((pPD->a[i].u & (X86_PDE2M_PAE_PG_MASK | X86_PDE4M_P | X86_PDE4M_PS)) == u64)
+                    Log(("i=%d cFound=%d\n", i, ++cFound));
+# endif
+            AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
+            /*PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);*/
+            break;
+        }
+#endif /* PGM_WITH_LARGE_PAGES */
+
+        default:
+            AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
+    }
+
+    /* not reached. */
+#ifndef _MSC_VER
+    return fRet;
+#endif
+}
+
+
+/**
+ * Scans one shadow page table for mappings of a physical page.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPhysPage   The guest page in question.
+ * @param   fFlushPTEs  Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
+ * @param   iShw        The shadow page table.
+ */
+static void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
+
+    /* We should only come here with when there's only one reference to this physical page. */
+    Assert(PGMPOOL_TD_GET_CREFS(PGM_PAGE_GET_TRACKING(pPhysPage)) == 1);
+
+    Log2(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%RHp iShw=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw));
+    STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
+    bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, iShw, PGM_PAGE_GET_PTE_INDEX(pPhysPage));
+    if (!fKeptPTEs)
+        PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
+    STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
+}
+
+
+/**
+ * Flushes a list of shadow page tables mapping the same physical page.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPhysPage   The guest page in question.
+ * @param   fFlushPTEs  Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
+ * @param   iPhysExt    The physical cross reference extent list to flush.
+ */
+static void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iPhysExt)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    bool     fKeepList = false;
+
+    STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
+    Log2(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%RHp iPhysExt=%u\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iPhysExt));
+
+    const uint16_t iPhysExtStart = iPhysExt;
+    PPGMPOOLPHYSEXT pPhysExt;
+    do
+    {
+        Assert(iPhysExt < pPool->cMaxPhysExts);
+        pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
+        for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
+        {
+            if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
+            {
+                bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, pPhysExt->aidx[i], pPhysExt->apte[i]);
+                if (!fKeptPTEs)
+                {
+                    pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
+                    pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+                }
+                else
+                    fKeepList = true;
+            }
+        }
+        /* next */
+        iPhysExt = pPhysExt->iNext;
+    } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
+
+    if (!fKeepList)
+    {
+        /* insert the list into the free list and clear the ram range entry. */
+        pPhysExt->iNext = pPool->iPhysExtFreeHead;
+        pPool->iPhysExtFreeHead = iPhysExtStart;
+        /* Invalidate the tracking data. */
+        PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
+    }
+
+    STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
+}
+
+
+/**
+ * Flushes all shadow page table mappings of the given guest page.
+ *
+ * This is typically called when the host page backing the guest one has been
+ * replaced or when the page protection was changed due to a guest access
+ * caught by the monitoring.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if all references has been successfully cleared.
+ * @retval  VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page
+ *          pool cleaning. FF and sync flags are set.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhysPage  GC physical address of the page in question
+ * @param   pPhysPage   The guest page in question.
+ * @param   fFlushPTEs  Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
+ * @param   pfFlushTLBs This is set to @a true if the shadow TLBs should be
+ *                      flushed, it is NOT touched if this isn't necessary.
+ *                      The caller MUST initialized this to @a false.
+ */
+int pgmPoolTrackUpdateGCPhys(PVMCC pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs)
+{
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    pgmLock(pVM);
+    int rc = VINF_SUCCESS;
+
+#ifdef PGM_WITH_LARGE_PAGES
+    /* Is this page part of a large page? */
+    if (PGM_PAGE_GET_PDE_TYPE(pPhysPage) == PGM_PAGE_PDE_TYPE_PDE)
+    {
+        RTGCPHYS GCPhysBase = GCPhysPage & X86_PDE2M_PAE_PG_MASK;
+        GCPhysPage &= X86_PDE_PAE_PG_MASK;
+
+        /* Fetch the large page base. */
+        PPGMPAGE pLargePage;
+        if (GCPhysBase != GCPhysPage)
+        {
+            pLargePage = pgmPhysGetPage(pVM, GCPhysBase);
+            AssertFatal(pLargePage);
+        }
+        else
+            pLargePage = pPhysPage;
+
+        Log(("pgmPoolTrackUpdateGCPhys: update large page PDE for %RGp (%RGp)\n", GCPhysBase, GCPhysPage));
+
+        if (PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE)
+        {
+            /* Mark the large page as disabled as we need to break it up to change a single page in the 2 MB range. */
+            PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
+            pVM->pgm.s.cLargePagesDisabled++;
+
+            /* Update the base as that *only* that one has a reference and there's only one PDE to clear. */
+            rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhysBase, pLargePage, fFlushPTEs, pfFlushTLBs);
+
+            *pfFlushTLBs = true;
+            pgmUnlock(pVM);
+            return rc;
+        }
+    }
+#else
+    NOREF(GCPhysPage);
+#endif /* PGM_WITH_LARGE_PAGES */
+
+    const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
+    if (u16)
+    {
+        /*
+         * The zero page is currently screwing up the tracking and we'll
+         * have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+         * is defined, zero pages won't normally be mapped. Some kind of solution
+         * will be needed for this problem of course, but it will have to wait...
+         */
+        if (    PGM_PAGE_IS_ZERO(pPhysPage)
+            ||  PGM_PAGE_IS_BALLOONED(pPhysPage))
+            rc = VINF_PGM_GCPHYS_ALIASED;
+        else
+        {
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 /** @todo we can drop this now. */
+            /* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow and
+               pgmPoolTrackFlushGCPhysPTs will/may kill the pool otherwise. */
+            uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
+# endif
+
+            if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
+            {
+                Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
+                pgmPoolTrackFlushGCPhysPT(pVM,
+                                          pPhysPage,
+                                          fFlushPTEs,
+                                          PGMPOOL_TD_GET_IDX(u16));
+            }
+            else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
+                pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, fFlushPTEs, PGMPOOL_TD_GET_IDX(u16));
+            else
+                rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
+            *pfFlushTLBs = true;
+
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+            PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
+# endif
+        }
+    }
+
+    if (rc == VINF_PGM_GCPHYS_ALIASED)
+    {
+        pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        rc = VINF_PGM_SYNC_CR3;
+    }
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Scans all shadow page tables for mappings of a physical page.
+ *
+ * This may be slow, but it's most likely more efficient than cleaning
+ * out the entire page pool / cache.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if all references has been successfully cleared.
+ * @retval  VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
+ *          a page pool cleaning.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPhysPage   The guest page in question.
+ */
+int pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
+    LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
+             pPool->cUsedPages, pPool->cPresent, pPhysPage));
+
+    /*
+     * There is a limit to what makes sense.
+     */
+    if (    pPool->cPresent > 1024
+        &&  pVM->cCpus == 1)
+    {
+        LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
+        STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
+        return VINF_PGM_GCPHYS_ALIASED;
+    }
+
+    /*
+     * Iterate all the pages until we've encountered all that in use.
+     * This is simple but not quite optimal solution.
+     */
+    const uint64_t  u64   = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;   /** @todo drop X86_PTE_P here as we always test if present separately, anyway. */
+    const uint32_t  u32   = u64;                                          /** @todo move into the 32BIT_PT_xx case */
+    unsigned        cLeft = pPool->cUsedPages;
+    unsigned        iPage = pPool->cCurPages;
+    while (--iPage >= PGMPOOL_IDX_FIRST)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
+        if (    pPage->GCPhys != NIL_RTGCPHYS
+            &&  pPage->cPresent)
+        {
+            switch (pPage->enmKind)
+            {
+                /*
+                 * We only care about shadow page tables.
+                 */
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                {
+                    unsigned    cPresent = pPage->cPresent;
+                    PX86PT      pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
+                        if (pPT->a[i].n.u1Present)
+                        {
+                            if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
+                            {
+                                //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
+                                pPT->a[i].u = 0;
+
+                                /* Update the counter as we're removing references. */
+                                Assert(pPage->cPresent);
+                                Assert(pPool->cPresent);
+                                pPage->cPresent--;
+                                pPool->cPresent--;
+                            }
+                            if (!--cPresent)
+                                break;
+                        }
+                    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
+                    break;
+                }
+
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                {
+                    unsigned        cPresent = pPage->cPresent;
+                    PPGMSHWPTPAE    pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
+                        if (PGMSHWPTEPAE_IS_P(pPT->a[i]))
+                        {
+                            if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
+                            {
+                                //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
+                                PGMSHWPTEPAE_SET(pPT->a[i], 0); /// @todo why not atomic?
+
+                                /* Update the counter as we're removing references. */
+                                Assert(pPage->cPresent);
+                                Assert(pPool->cPresent);
+                                pPage->cPresent--;
+                                pPool->cPresent--;
+                            }
+                            if (!--cPresent)
+                                break;
+                        }
+                    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
+                    break;
+                }
+
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                {
+                    unsigned  cPresent = pPage->cPresent;
+                    PEPTPT    pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
+                        if (pPT->a[i].n.u1Present)
+                        {
+                            if ((pPT->a[i].u & (EPT_PTE_PG_MASK | X86_PTE_P)) == u64)
+                            {
+                                //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
+                                pPT->a[i].u = 0;
+
+                                /* Update the counter as we're removing references. */
+                                Assert(pPage->cPresent);
+                                Assert(pPool->cPresent);
+                                pPage->cPresent--;
+                                pPool->cPresent--;
+                            }
+                            if (!--cPresent)
+                                break;
+                        }
+                    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
+                    break;
+                }
+            }
+
+            if (!--cLeft)
+                break;
+        }
+    }
+
+    PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
+    STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
+
+    /*
+     * There is a limit to what makes sense. The above search is very expensive, so force a pgm pool flush.
+     */
+    if (pPool->cPresent > 1024)
+    {
+        LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
+        return VINF_PGM_GCPHYS_ALIASED;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Clears the user entry in a user table.
+ *
+ * This is used to remove all references to a page when flushing it.
+ */
+static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
+{
+    Assert(pUser->iUser != NIL_PGMPOOL_IDX);
+    Assert(pUser->iUser < pPool->cCurPages);
+    uint32_t iUserTable = pUser->iUserTable;
+
+    /*
+     * Map the user page.  Ignore references made by fictitious pages.
+     */
+    PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
+    LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
+    union
+    {
+        uint64_t       *pau64;
+        uint32_t       *pau32;
+    } u;
+    if (pUserPage->idx < PGMPOOL_IDX_FIRST)
+    {
+        Assert(!pUserPage->pvPageR3);
+        return;
+    }
+    u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
+
+
+    /* Safety precaution in case we change the paging for other modes too in the future. */
+    Assert(!pgmPoolIsPageLocked(pPage)); RT_NOREF_PV(pPage);
+
+#ifdef VBOX_STRICT
+    /*
+     * Some sanity checks.
+     */
+    switch (pUserPage->enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            Assert(iUserTable < X86_PG_ENTRIES);
+            break;
+        case PGMPOOLKIND_PAE_PDPT:
+        case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+            Assert(iUserTable < 4);
+            Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
+            break;
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+            Assert(iUserTable < X86_PG_PAE_ENTRIES);
+            break;
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+            Assert(iUserTable < X86_PG_PAE_ENTRIES);
+            Assert(!(u.pau64[iUserTable] & PGM_PDFLAGS_MAPPING));
+            break;
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+            Assert(iUserTable < X86_PG_PAE_ENTRIES);
+            Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
+            break;
+        case PGMPOOLKIND_64BIT_PML4:
+            Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
+            /* GCPhys >> PAGE_SHIFT is the index here */
+            break;
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+            Assert(iUserTable < X86_PG_PAE_ENTRIES);
+            break;
+
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+            Assert(iUserTable < X86_PG_PAE_ENTRIES);
+            break;
+
+        case PGMPOOLKIND_ROOT_NESTED:
+            Assert(iUserTable < X86_PG_PAE_ENTRIES);
+            break;
+
+        default:
+            AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
+            break;
+    }
+#endif /* VBOX_STRICT */
+
+    /*
+     * Clear the entry in the user page.
+     */
+    switch (pUserPage->enmKind)
+    {
+        /* 32-bit entries */
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            ASMAtomicWriteU32(&u.pau32[iUserTable], 0);
+            break;
+
+        /* 64-bit entries */
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+        case PGMPOOLKIND_64BIT_PML4:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PDPT:
+        case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+        case PGMPOOLKIND_ROOT_NESTED:
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+            ASMAtomicWriteU64(&u.pau64[iUserTable], 0);
+            break;
+
+        default:
+            AssertFatalMsgFailed(("enmKind=%d iUser=%d iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
+    }
+    PGM_DYNMAP_UNUSED_HINT_VM(pPool->CTX_SUFF(pVM), u.pau64);
+}
+
+
+/**
+ * Clears all users of a page.
+ */
+static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    /*
+     * Free all the user records.
+     */
+    LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
+
+    PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
+    uint16_t i = pPage->iUserHead;
+    while (i != NIL_PGMPOOL_USER_INDEX)
+    {
+        /* Clear enter in user table. */
+        pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
+
+        /* Free it. */
+        const uint16_t iNext = paUsers[i].iNext;
+        paUsers[i].iUser = NIL_PGMPOOL_IDX;
+        paUsers[i].iNext = pPool->iUserFreeHead;
+        pPool->iUserFreeHead = i;
+
+        /* Next. */
+        i = iNext;
+    }
+    pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
+}
+
+
+/**
+ * Allocates a new physical cross reference extent.
+ *
+ * @returns Pointer to the allocated extent on success. NULL if we're out of them.
+ * @param   pVM         The cross context VM structure.
+ * @param   piPhysExt   Where to store the phys ext index.
+ */
+PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    uint16_t iPhysExt = pPool->iPhysExtFreeHead;
+    if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
+    {
+        STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
+        return NULL;
+    }
+    PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
+    pPool->iPhysExtFreeHead = pPhysExt->iNext;
+    pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
+    *piPhysExt = iPhysExt;
+    return pPhysExt;
+}
+
+
+/**
+ * Frees a physical cross reference extent.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   iPhysExt    The extent to free.
+ */
+void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    Assert(iPhysExt < pPool->cMaxPhysExts);
+    PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
+    for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
+    {
+        pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
+        pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+    }
+    pPhysExt->iNext = pPool->iPhysExtFreeHead;
+    pPool->iPhysExtFreeHead = iPhysExt;
+}
+
+
+/**
+ * Frees a physical cross reference extent.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   iPhysExt    The extent to free.
+ */
+void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    const uint16_t  iPhysExtStart = iPhysExt;
+    PPGMPOOLPHYSEXT pPhysExt;
+    do
+    {
+        Assert(iPhysExt < pPool->cMaxPhysExts);
+        pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
+        for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
+        {
+            pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
+            pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        }
+
+        /* next */
+        iPhysExt = pPhysExt->iNext;
+    } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
+
+    pPhysExt->iNext = pPool->iPhysExtFreeHead;
+    pPool->iPhysExtFreeHead = iPhysExtStart;
+}
+
+
+/**
+ * Insert a reference into a list of physical cross reference extents.
+ *
+ * @returns The new tracking data for PGMPAGE.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   iPhysExt    The physical extent index of the list head.
+ * @param   iShwPT      The shadow page table index.
+ * @param   iPte        Page table entry
+ *
+ */
+static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT, uint16_t iPte)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PPGMPOOL        pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
+
+    /*
+     * Special common cases.
+     */
+    if (paPhysExts[iPhysExt].aidx[1] == NIL_PGMPOOL_IDX)
+    {
+        paPhysExts[iPhysExt].aidx[1] = iShwPT;
+        paPhysExts[iPhysExt].apte[1] = iPte;
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedMany);
+        LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,%d pte %d,}\n", iPhysExt, iShwPT, iPte));
+        return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
+    }
+    if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
+    {
+        paPhysExts[iPhysExt].aidx[2] = iShwPT;
+        paPhysExts[iPhysExt].apte[2] = iPte;
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedMany);
+        LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,,%d pte %d}\n", iPhysExt, iShwPT, iPte));
+        return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
+    }
+    AssertCompile(RT_ELEMENTS(paPhysExts[iPhysExt].aidx) == 3);
+
+    /*
+     * General treatment.
+     */
+    const uint16_t iPhysExtStart = iPhysExt;
+    unsigned cMax = 15;
+    for (;;)
+    {
+        Assert(iPhysExt < pPool->cMaxPhysExts);
+        for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
+            if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
+            {
+                paPhysExts[iPhysExt].aidx[i] = iShwPT;
+                paPhysExts[iPhysExt].apte[i] = iPte;
+                STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedMany);
+                LogFlow(("pgmPoolTrackPhysExtInsert: %d:{%d pte %d} i=%d cMax=%d\n", iPhysExt, iShwPT, iPte, i, cMax));
+                return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
+            }
+        if (!--cMax)
+        {
+            STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackOverflows);
+            pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
+            LogFlow(("pgmPoolTrackPhysExtInsert: overflow (1) iShwPT=%d\n", iShwPT));
+            return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
+        }
+
+        /* advance */
+        iPhysExt = paPhysExts[iPhysExt].iNext;
+        if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
+            break;
+    }
+
+    /*
+     * Add another extent to the list.
+     */
+    PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
+    if (!pNew)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackNoExtentsLeft);
+        pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
+        LogFlow(("pgmPoolTrackPhysExtInsert: pgmPoolTrackPhysExtAlloc failed iShwPT=%d\n", iShwPT));
+        return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
+    }
+    pNew->iNext = iPhysExtStart;
+    pNew->aidx[0] = iShwPT;
+    pNew->apte[0] = iPte;
+    LogFlow(("pgmPoolTrackPhysExtInsert: added new extent %d:{%d pte %d}->%d\n", iPhysExt, iShwPT, iPte, iPhysExtStart));
+    return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
+}
+
+
+/**
+ * Add a reference to guest physical page where extents are in use.
+ *
+ * @returns The new tracking data for PGMPAGE.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPhysPage   Pointer to the aPages entry in the ram range.
+ * @param   u16         The ram range flags (top 16-bits).
+ * @param   iShwPT      The shadow page table index.
+ * @param   iPte        Page table entry
+ */
+uint16_t pgmPoolTrackPhysExtAddref(PVMCC pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte)
+{
+    pgmLock(pVM);
+    if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
+    {
+        /*
+         * Convert to extent list.
+         */
+        Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
+        uint16_t iPhysExt;
+        PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
+        if (pPhysExt)
+        {
+            LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
+            STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliased);
+            pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
+            pPhysExt->apte[0] = PGM_PAGE_GET_PTE_INDEX(pPhysPage);
+            pPhysExt->aidx[1] = iShwPT;
+            pPhysExt->apte[1] = iPte;
+            u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
+        }
+        else
+            u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
+    }
+    else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
+    {
+        /*
+         * Insert into the extent list.
+         */
+        u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT, iPte);
+    }
+    else
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedLots);
+    pgmUnlock(pVM);
+    return u16;
+}
+
+
+/**
+ * Clear references to guest physical memory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pPhysPage   Pointer to the aPages entry in the ram range.
+ * @param   iPte        Shadow PTE index
+ */
+void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage, uint16_t iPte)
+{
+    PVMCC          pVM = pPool->CTX_SUFF(pVM);
+    const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
+    AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
+
+    uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
+    if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
+    {
+        pgmLock(pVM);
+
+        uint16_t        iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
+        PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
+        do
+        {
+            Assert(iPhysExt < pPool->cMaxPhysExts);
+
+            /*
+             * Look for the shadow page and check if it's all freed.
+             */
+            for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
+            {
+                if (    paPhysExts[iPhysExt].aidx[i] == pPage->idx
+                    &&  paPhysExts[iPhysExt].apte[i] == iPte)
+                {
+                    paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
+                    paPhysExts[iPhysExt].apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+
+                    for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
+                        if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
+                        {
+                            Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
+                            pgmUnlock(pVM);
+                            return;
+                        }
+
+                    /* we can free the node. */
+                    const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
+                    if (    iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
+                        &&  iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
+                    {
+                        /* lonely node */
+                        pgmPoolTrackPhysExtFree(pVM, iPhysExt);
+                        Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
+                        PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
+                    }
+                    else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
+                    {
+                        /* head */
+                        Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
+                        PGM_PAGE_SET_TRACKING(pVM, pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
+                        pgmPoolTrackPhysExtFree(pVM, iPhysExt);
+                    }
+                    else
+                    {
+                        /* in list */
+                        Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d in list\n", pPhysPage, pPage->idx));
+                        paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
+                        pgmPoolTrackPhysExtFree(pVM, iPhysExt);
+                    }
+                    iPhysExt = iPhysExtNext;
+                    pgmUnlock(pVM);
+                    return;
+                }
+            }
+
+            /* next */
+            iPhysExtPrev = iPhysExt;
+            iPhysExt = paPhysExts[iPhysExt].iNext;
+        } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
+
+        pgmUnlock(pVM);
+        AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
+    }
+    else /* nothing to do */
+        Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
+}
+
+/**
+ * Clear references to guest physical memory.
+ *
+ * This is the same as pgmPoolTracDerefGCPhysHint except that the guest
+ * physical address is assumed to be correct, so the linear search can be
+ * skipped and we can assert at an earlier point.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   HCPhys      The host physical address corresponding to the guest page.
+ * @param   GCPhys      The guest physical address corresponding to HCPhys.
+ * @param   iPte        Shadow PTE index
+ */
+static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys, uint16_t iPte)
+{
+    /*
+     * Lookup the page and check if it checks out before derefing it.
+     */
+    PVMCC    pVM       = pPool->CTX_SUFF(pVM);
+    PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhys);
+    if (pPhysPage)
+    {
+        Assert(PGM_PAGE_GET_HCPHYS(pPhysPage));
+#ifdef LOG_ENABLED
+        RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(pPhysPage);
+        Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
+#endif
+        if (PGM_PAGE_GET_HCPHYS(pPhysPage) == HCPhys)
+        {
+            Assert(pPage->cPresent);
+            Assert(pPool->cPresent);
+            pPage->cPresent--;
+            pPool->cPresent--;
+            pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
+            return;
+        }
+
+        AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page has HCPhys=%RHp\n",
+                              HCPhys, GCPhys, PGM_PAGE_GET_HCPHYS(pPhysPage)));
+    }
+    AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
+}
+
+
+/**
+ * Clear references to guest physical memory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   HCPhys      The host physical address corresponding to the guest page.
+ * @param   GCPhysHint  The guest physical address which may corresponding to HCPhys.
+ * @param   iPte        Shadow pte index
+ */
+void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte)
+{
+    Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
+
+    /*
+     * Try the hint first.
+     */
+    RTHCPHYS HCPhysHinted;
+    PVMCC    pVM       = pPool->CTX_SUFF(pVM);
+    PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysHint);
+    if (pPhysPage)
+    {
+        HCPhysHinted = PGM_PAGE_GET_HCPHYS(pPhysPage);
+        Assert(HCPhysHinted);
+        if (HCPhysHinted == HCPhys)
+        {
+            Assert(pPage->cPresent);
+            Assert(pPool->cPresent);
+            pPage->cPresent--;
+            pPool->cPresent--;
+            pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
+            return;
+        }
+    }
+    else
+        HCPhysHinted = UINT64_C(0xdeadbeefdeadbeef);
+
+    /*
+     * Damn, the hint didn't work.  We'll have to do an expensive linear search.
+     */
+    STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
+    PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRangesX);
+    while (pRam)
+    {
+        unsigned iPage = pRam->cb >> PAGE_SHIFT;
+        while (iPage-- > 0)
+        {
+            if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
+            {
+                Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
+                      HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
+                Assert(pPage->cPresent);
+                Assert(pPool->cPresent);
+                pPage->cPresent--;
+                pPool->cPresent--;
+                pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage], iPte);
+                return;
+            }
+        }
+        pRam = pRam->CTX_SUFF(pNext);
+    }
+
+    AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp (Hinted page has HCPhys = %RHp)\n", HCPhys, GCPhysHint, HCPhysHinted));
+}
+
+
+/**
+ * Clear references to guest physical memory in a 32-bit / 32-bit page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ * @param   pGstPT      The guest page table.
+ */
+DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
+{
+    RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        Assert(!(pShwPT->a[i].u & RT_BIT_32(10)));
+        if (pShwPT->a[i].n.u1Present)
+        {
+            Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
+                  i, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
+            pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & fPgMask, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to guest physical memory in a PAE / 32-bit page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ * @param   pGstPT      The guest page table (just a half one).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
+{
+    RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        Assert(   (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
+               || (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
+                  i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PG_MASK));
+            pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to guest physical memory in a PAE / PAE page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ * @param   pGstPT      The guest page table.
+ */
+DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
+{
+    RTGCPHYS const fPgMask = pPage->fA20Enabled ? X86_PTE_PAE_PG_MASK : X86_PTE_PAE_PG_MASK & ~RT_BIT_64(20);
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        Assert(   (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
+               || (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
+                  i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
+            pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to guest physical memory in a 32-bit / 4MB page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
+{
+    RTGCPHYS const  GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
+    RTGCPHYS        GCPhys        = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
+    {
+        Assert(!(pShwPT->a[i].u & RT_BIT_32(10)));
+        if (pShwPT->a[i].n.u1Present)
+        {
+            Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
+                  i, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys));
+            pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to guest physical memory in a PAE / 2/4MB page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT)
+{
+    RTGCPHYS const  GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
+    RTGCPHYS        GCPhys        = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
+    {
+        Assert(   (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
+               || (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
+                  i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys));
+            pgmPoolTracDerefGCPhys(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys & GCPhysA20Mask, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in an EPT page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page directory pointer table (mapping of the
+ *                      page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
+{
+    RTGCPHYS const  GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
+    RTGCPHYS        GCPhys        = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
+    {
+        Assert((pShwPT->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
+        if (pShwPT->a[i].n.u1Present)
+        {
+            Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
+                  i, pShwPT->a[i].u & EPT_PTE_PG_MASK, pPage->GCPhys));
+            pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & EPT_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in a 32 bits page directory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPD      The shadow page directory (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
+    {
+        if (    pShwPD->a[i].n.u1Present
+            &&  !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
+           )
+        {
+            PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
+            if (pSubPage)
+                pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+            else
+                AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPD      The shadow page directory (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
+    {
+        if (   pShwPD->a[i].n.u1Present
+            && !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING))
+        {
+#ifdef PGM_WITH_LARGE_PAGES
+            if (pShwPD->a[i].b.u1Size)
+            {
+                Log4(("pgmPoolTrackDerefPDPae: i=%d pde=%RX64 GCPhys=%RX64\n",
+                      i, pShwPD->a[i].u & X86_PDE2M_PAE_PG_MASK, pPage->GCPhys));
+                pgmPoolTracDerefGCPhys(pPool, pPage, pShwPD->a[i].u & X86_PDE2M_PAE_PG_MASK,
+                                       pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
+                                       i);
+            }
+            else
+#endif
+            {
+                Assert((pShwPD->a[i].u & (X86_PDE_PAE_MBZ_MASK_NX | UINT64_C(0x7ff0000000000000))) == 0);
+                PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
+                if (pSubPage)
+                    pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+                else
+                    AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & X86_PDE_PAE_PG_MASK));
+                /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
+            }
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in a PAE page directory pointer table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPDPT   The shadow page directory pointer table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
+{
+    for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
+    {
+        Assert((pShwPDPT->a[i].u & (X86_PDPE_PAE_MBZ_MASK | UINT64_C(0x7ff0000000000200))) == 0);
+        if (    pShwPDPT->a[i].n.u1Present
+            &&  !(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING)
+           )
+        {
+            PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
+            if (pSubPage)
+                pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+            else
+                AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in a 64-bit page directory pointer table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPDPT   The shadow page directory pointer table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
+    {
+        Assert((pShwPDPT->a[i].u & (X86_PDPE_LM_MBZ_MASK_NX | UINT64_C(0x7ff0000000000200))) == 0);
+        if (pShwPDPT->a[i].n.u1Present)
+        {
+            PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
+            if (pSubPage)
+                pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+            else
+                AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
+            /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in a 64-bit level 4 page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPML4    The shadow page directory pointer table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
+    {
+        Assert((pShwPML4->a[i].u & (X86_PML4E_MBZ_MASK_NX | UINT64_C(0x7ff0000000000200))) == 0);
+        if (pShwPML4->a[i].n.u1Present)
+        {
+            PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPML4->a[i].u & X86_PDPE_PG_MASK);
+            if (pSubPage)
+                pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+            else
+                AssertFatalMsgFailed(("%RX64\n", pShwPML4->a[i].u & X86_PML4E_PG_MASK));
+            /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in an EPT page directory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPD      The shadow page directory (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
+    {
+        Assert((pShwPD->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
+        if (pShwPD->a[i].n.u1Present)
+        {
+#ifdef PGM_WITH_LARGE_PAGES
+            if (pShwPD->a[i].b.u1Size)
+            {
+                Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n",
+                      i, pShwPD->a[i].u & X86_PDE2M_PAE_PG_MASK, pPage->GCPhys));
+                pgmPoolTracDerefGCPhys(pPool, pPage, pShwPD->a[i].u & X86_PDE2M_PAE_PG_MASK,
+                                       pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
+                                       i);
+            }
+            else
+#endif
+            {
+                PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & EPT_PDE_PG_MASK);
+                if (pSubPage)
+                    pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+                else
+                    AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
+            }
+            /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in an EPT page directory pointer table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPDPT   The shadow page directory pointer table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
+    {
+        Assert((pShwPDPT->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
+        if (pShwPDPT->a[i].n.u1Present)
+        {
+            PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK);
+            if (pSubPage)
+                pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+            else
+                AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK));
+            /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
+        }
+    }
+}
+
+
+/**
+ * Clears all references made by this page.
+ *
+ * This includes other shadow pages and GC physical addresses.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ */
+static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    /*
+     * Map the shadow page and take action according to the page kind.
+     */
+    PVMCC pVM   = pPool->CTX_SUFF(pVM);
+    void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+    switch (pPage->enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        {
+            STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
+            void *pvGst;
+            int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+            pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
+            PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+            STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
+            break;
+        }
+
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        {
+            STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
+            void *pvGst;
+            int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+            pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
+            PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+            STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
+            break;
+        }
+
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        {
+            STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
+            void *pvGst;
+            int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+            pgmPoolTrackDerefPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
+            PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+            STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
+            break;
+        }
+
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        {
+            STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
+            pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
+            STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
+            break;
+        }
+
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:   /* treat it like a 2 MB page */
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        {
+            STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
+            pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PPGMSHWPTPAE)pvShw);
+            STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
+            break;
+        }
+
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+            pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
+            break;
+
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+        case PGMPOOLKIND_32BIT_PD:
+            pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
+            break;
+
+        case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+        case PGMPOOLKIND_PAE_PDPT:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+            pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
+            break;
+
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+            pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
+            break;
+
+        case PGMPOOLKIND_64BIT_PML4:
+            pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
+            break;
+
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+            pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
+            break;
+
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+            pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
+            break;
+
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+            pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
+            break;
+
+        default:
+            AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
+    }
+
+    /* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
+    STAM_PROFILE_START(&pPool->StatZeroPage, z);
+    ASMMemZeroPage(pvShw);
+    STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
+    pPage->fZeroed = true;
+    Assert(!pPage->cPresent);
+    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
+}
+
+
+/**
+ * Flushes a pool page.
+ *
+ * This moves the page to the free list after removing all user references to it.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @param   pPool       The pool.
+ * @param   pPage       The shadow page.
+ * @param   fFlush      Flush the TLBS when required (should only be false in very specific use cases!!)
+ */
+int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush)
+{
+    PVMCC   pVM = pPool->CTX_SUFF(pVM);
+    bool    fFlushRequired = false;
+
+    int rc = VINF_SUCCESS;
+    STAM_PROFILE_START(&pPool->StatFlushPage, f);
+    LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
+             pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
+
+    /*
+     * Reject any attempts at flushing any of the special root pages (shall
+     * not happen).
+     */
+    AssertMsgReturn(pPage->idx >= PGMPOOL_IDX_FIRST,
+                    ("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n",
+                     pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx),
+                    VINF_SUCCESS);
+
+    pgmLock(pVM);
+
+    /*
+     * Quietly reject any attempts at flushing the currently active shadow CR3 mapping
+     */
+    if (pgmPoolIsPageLocked(pPage))
+    {
+        AssertMsg(   pPage->enmKind == PGMPOOLKIND_64BIT_PML4
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PDPT
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
+                  || pPage->enmKind == PGMPOOLKIND_32BIT_PD
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD
+                  || pPage->enmKind == PGMPOOLKIND_ROOT_NESTED,
+                  ("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind));
+        Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
+        pgmUnlock(pVM);
+        return VINF_SUCCESS;
+    }
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    /* Start a subset so we won't run out of mapping space. */
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
+#endif
+
+    /*
+     * Mark the page as being in need of an ASMMemZeroPage().
+     */
+    pPage->fZeroed = false;
+
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    if (pPage->fDirty)
+        pgmPoolFlushDirtyPage(pVM, pPool, pPage->idxDirtyEntry, false /* do not remove */);
+#endif
+
+    /* If there are any users of this table, then we *must* issue a tlb flush on all VCPUs. */
+    if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
+        fFlushRequired = true;
+
+    /*
+     * Clear the page.
+     */
+    pgmPoolTrackClearPageUsers(pPool, pPage);
+    STAM_PROFILE_START(&pPool->StatTrackDeref,a);
+    pgmPoolTrackDeref(pPool, pPage);
+    STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
+
+    /*
+     * Flush it from the cache.
+     */
+    pgmPoolCacheFlushPage(pPool, pPage);
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    /* Heavy stuff done. */
+    PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
+#endif
+
+    /*
+     * Deregistering the monitoring.
+     */
+    if (pPage->fMonitored)
+        rc = pgmPoolMonitorFlush(pPool, pPage);
+
+    /*
+     * Free the page.
+     */
+    Assert(pPage->iNext == NIL_PGMPOOL_IDX);
+    pPage->iNext = pPool->iFreeHead;
+    pPool->iFreeHead = pPage->idx;
+    pPage->enmKind = PGMPOOLKIND_FREE;
+    pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
+    pPage->GCPhys = NIL_RTGCPHYS;
+    pPage->fReusedFlushPending = false;
+
+    pPool->cUsedPages--;
+
+    /* Flush the TLBs of all VCPUs if required. */
+    if (    fFlushRequired
+        &&  fFlush)
+    {
+        PGM_INVL_ALL_VCPU_TLBS(pVM);
+    }
+
+    pgmUnlock(pVM);
+    STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
+    return rc;
+}
+
+
+/**
+ * Frees a usage of a pool page.
+ *
+ * The caller is responsible to updating the user table so that it no longer
+ * references the shadow page.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The shadow page.
+ * @param   iUser       The shadow page pool index of the user table.
+ *                      NIL_PGMPOOL_IDX for root pages.
+ * @param   iUserTable  The index into the user table (shadowed). Ignored if
+ *                      root page.
+ */
+void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
+{
+    PVMCC pVM = pPool->CTX_SUFF(pVM);
+
+    STAM_PROFILE_START(&pPool->StatFree, a);
+    LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%d iUserTable=%#x\n",
+             pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
+    AssertReturnVoid(pPage->idx >= PGMPOOL_IDX_FIRST); /* paranoia (#6349) */
+
+    pgmLock(pVM);
+    if (iUser != NIL_PGMPOOL_IDX)
+        pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
+    if (!pPage->fCached)
+        pgmPoolFlushPage(pPool, pPage);
+    pgmUnlock(pVM);
+    STAM_PROFILE_STOP(&pPool->StatFree, a);
+}
+
+
+/**
+ * Makes one or more free page free.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ *
+ * @param   pPool       The pool.
+ * @param   enmKind     Page table kind
+ * @param   iUser       The user of the page.
+ */
+static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
+{
+    PVMCC pVM = pPool->CTX_SUFF(pVM);
+    LogFlow(("pgmPoolMakeMoreFreePages: enmKind=%d iUser=%d\n", enmKind, iUser));
+    NOREF(enmKind);
+
+    /*
+     * If the pool isn't full grown yet, expand it.
+     */
+    if (pPool->cCurPages < pPool->cMaxPages)
+    {
+        STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
+#ifdef IN_RING3
+        int rc = PGMR3PoolGrow(pVM, VMMGetCpu(pVM));
+#else
+        int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_POOL_GROW, 0);
+#endif
+        if (RT_FAILURE(rc))
+            return rc;
+        STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
+        if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
+            return VINF_SUCCESS;
+    }
+
+    /*
+     * Free one cached page.
+     */
+    return pgmPoolCacheFreeOne(pPool, iUser);
+}
+
+
+/**
+ * Allocates a page from the pool.
+ *
+ * This page may actually be a cached page and not in need of any processing
+ * on the callers part.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if a NEW page was allocated.
+ * @retval  VINF_PGM_CACHED_PAGE if a CACHED page was returned.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address of the page we're gonna shadow.
+ *                      For 4MB and 2MB PD entries, it's the first address the
+ *                      shadow PT is covering.
+ * @param   enmKind     The kind of mapping.
+ * @param   enmAccess   Access type for the mapping (only relevant for big pages)
+ * @param   fA20Enabled Whether the A20 gate is enabled or not.
+ * @param   iUser       The shadow page pool index of the user table.  Root
+ *                      pages should pass NIL_PGMPOOL_IDX.
+ * @param   iUserTable  The index into the user table (shadowed).  Ignored for
+ *                      root pages (iUser == NIL_PGMPOOL_IDX).
+ * @param   fLockPage   Lock the page
+ * @param   ppPage      Where to store the pointer to the page. NULL is stored here on failure.
+ */
+int pgmPoolAlloc(PVMCC pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
+                 uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
+    LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%d iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
+    *ppPage = NULL;
+    /** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
+     *  (TRPMR3SyncIDT) because of FF priority. Try fix that?
+     *  Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
+
+    pgmLock(pVM);
+
+    if (pPool->fCacheEnabled)
+    {
+        int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, enmAccess, fA20Enabled, iUser, iUserTable, ppPage);
+        if (RT_SUCCESS(rc2))
+        {
+            if (fLockPage)
+                pgmPoolLockPage(pPool, *ppPage);
+            pgmUnlock(pVM);
+            STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
+            LogFlow(("pgmPoolAlloc: cached returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, *ppPage, (*ppPage)->Core.Key, (*ppPage)->idx));
+            return rc2;
+        }
+    }
+
+    /*
+     * Allocate a new one.
+     */
+    int         rc = VINF_SUCCESS;
+    uint16_t    iNew = pPool->iFreeHead;
+    if (iNew == NIL_PGMPOOL_IDX)
+    {
+        rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
+        if (RT_FAILURE(rc))
+        {
+            pgmUnlock(pVM);
+            Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
+            STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
+            return rc;
+        }
+        iNew = pPool->iFreeHead;
+        AssertReleaseMsgReturn(iNew != NIL_PGMPOOL_IDX, ("iNew=%#x\n", iNew), VERR_PGM_POOL_IPE);
+    }
+
+    /* unlink the free head */
+    PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
+    pPool->iFreeHead = pPage->iNext;
+    pPage->iNext = NIL_PGMPOOL_IDX;
+
+    /*
+     * Initialize it.
+     */
+    pPool->cUsedPages++;                /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
+    pPage->enmKind = enmKind;
+    pPage->enmAccess = enmAccess;
+    pPage->GCPhys = GCPhys;
+    pPage->fA20Enabled = fA20Enabled;
+    pPage->fSeenNonGlobal = false;      /* Set this to 'true' to disable this feature. */
+    pPage->fMonitored = false;
+    pPage->fCached = false;
+    pPage->fDirty = false;
+    pPage->fReusedFlushPending = false;
+    pPage->cModifications = 0;
+    pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+    pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
+    pPage->cPresent = 0;
+    pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
+    pPage->idxDirtyEntry = 0;
+    pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
+    pPage->GCPtrLastAccessHandlerRip   = NIL_RTGCPTR;
+    pPage->cLastAccessHandler = 0;
+    pPage->cLocked = 0;
+# ifdef VBOX_STRICT
+    pPage->GCPtrDirtyFault = NIL_RTGCPTR;
+# endif
+
+    /*
+     * Insert into the tracking and cache. If this fails, free the page.
+     */
+    int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
+    if (RT_FAILURE(rc3))
+    {
+        pPool->cUsedPages--;
+        pPage->enmKind      = PGMPOOLKIND_FREE;
+        pPage->enmAccess    = PGMPOOLACCESS_DONTCARE;
+        pPage->GCPhys       = NIL_RTGCPHYS;
+        pPage->iNext        = pPool->iFreeHead;
+        pPool->iFreeHead    = pPage->idx;
+        pgmUnlock(pVM);
+        STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
+        Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
+        return rc3;
+    }
+
+    /*
+     * Commit the allocation, clear the page and return.
+     */
+#ifdef VBOX_WITH_STATISTICS
+    if (pPool->cUsedPages > pPool->cUsedPagesHigh)
+        pPool->cUsedPagesHigh = pPool->cUsedPages;
+#endif
+
+    if (!pPage->fZeroed)
+    {
+        STAM_PROFILE_START(&pPool->StatZeroPage, z);
+        void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+        ASMMemZeroPage(pv);
+        STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
+    }
+
+    *ppPage = pPage;
+    if (fLockPage)
+        pgmPoolLockPage(pPool, pPage);
+    pgmUnlock(pVM);
+    LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
+             rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
+    STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
+    return rc;
+}
+
+
+/**
+ * Frees a usage of a pool page.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   HCPhys      The HC physical address of the shadow page.
+ * @param   iUser       The shadow page pool index of the user table.
+ *                      NIL_PGMPOOL_IDX if root page.
+ * @param   iUserTable  The index into the user table (shadowed).  Ignored if
+ *                      root page.
+ */
+void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
+{
+    LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%d iUserTable=%#x\n", HCPhys, iUser, iUserTable));
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
+}
+
+
+/**
+ * Internal worker for finding a 'in-use' shadow page give by it's physical address.
+ *
+ * @returns Pointer to the shadow page structure.
+ * @param   pPool       The pool.
+ * @param   HCPhys      The HC physical address of the shadow page.
+ */
+PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
+{
+    PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
+
+    /*
+     * Look up the page.
+     */
+    PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
+
+    AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
+    return pPage;
+}
+
+
+/**
+ * Internal worker for finding a page for debugging purposes, no assertions.
+ *
+ * @returns Pointer to the shadow page structure.  NULL on if not found.
+ * @param   pPool       The pool.
+ * @param   HCPhys      The HC physical address of the shadow page.
+ */
+PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys)
+{
+    PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
+    return (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
+}
+
+
+/**
+ * Internal worker for PGM_HCPHYS_2_PTR.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   HCPhys      The HC physical address of the shadow page.
+ * @param   ppv         Where to return the address.
+ */
+int pgmPoolHCPhys2Ptr(PVM pVM, RTHCPHYS HCPhys, void **ppv)
+{
+    PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pVM->pgm.s.CTX_SUFF(pPool)->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
+    AssertMsgReturn(pPage && pPage->enmKind != PGMPOOLKIND_FREE,
+                    ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0),
+                    VERR_PGM_POOL_GET_PAGE_FAILED);
+    *ppv = (uint8_t *)pPage->CTX_SUFF(pvPage) + (HCPhys & PAGE_OFFSET_MASK);
+    return VINF_SUCCESS;
+}
+
+#ifdef IN_RING3 /* currently only used in ring 3; save some space in the R0 & GC modules (left it here as we might need it elsewhere later on) */
+
+/**
+ * Flush the specified page if present
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPhys  Guest physical address of the page to flush
+ */
+void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Look up the GCPhys in the hash.
+     */
+    GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+    unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
+    if (i == NIL_PGMPOOL_IDX)
+        return;
+
+    do
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[i];
+        if (pPage->GCPhys - GCPhys < PAGE_SIZE)
+        {
+            switch (pPage->enmKind)
+            {
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+                case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+                case PGMPOOLKIND_64BIT_PML4:
+                case PGMPOOLKIND_32BIT_PD:
+                case PGMPOOLKIND_PAE_PDPT:
+                {
+                    Log(("PGMPoolFlushPage: found pgm pool pages for %RGp\n", GCPhys));
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+                    if (pPage->fDirty)
+                        STAM_COUNTER_INC(&pPool->StatForceFlushDirtyPage);
+                    else
+# endif
+                        STAM_COUNTER_INC(&pPool->StatForceFlushPage);
+                    Assert(!pgmPoolIsPageLocked(pPage));
+                    pgmPoolMonitorChainFlush(pPool, pPage);
+                    return;
+                }
+
+                /* ignore, no monitoring. */
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                case PGMPOOLKIND_ROOT_NESTED:
+                case PGMPOOLKIND_PAE_PD_PHYS:
+                case PGMPOOLKIND_PAE_PDPT_PHYS:
+                case PGMPOOLKIND_32BIT_PD_PHYS:
+                case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+                    break;
+
+                default:
+                    AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
+            }
+        }
+
+        /* next */
+        i = pPage->iNext;
+    } while (i != NIL_PGMPOOL_IDX);
+    return;
+}
+
+
+/**
+ * Reset CPU on hot plugging.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu              The cross context virtual CPU structure.
+ */
+void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu)
+{
+    pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
+
+    pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+}
+
+
+/**
+ * Flushes the entire cache.
+ *
+ * It will assert a global CR3 flush (FF) and assumes the caller is aware of
+ * this and execute this CR3 flush.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+void pgmR3PoolReset(PVM pVM)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    STAM_PROFILE_START(&pPool->StatR3Reset, a);
+    LogFlow(("pgmR3PoolReset:\n"));
+
+    /*
+     * If there are no pages in the pool, there is nothing to do.
+     */
+    if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
+    {
+        STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
+        return;
+    }
+
+    /*
+     * Exit the shadow mode since we're going to clear everything,
+     * including the root page.
+     */
+    VMCC_FOR_EACH_VMCPU(pVM)
+        pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
+    VMCC_FOR_EACH_VMCPU_END(pVM);
+
+
+    /*
+     * Nuke the free list and reinsert all pages into it.
+     */
+    for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[i];
+
+        if (pPage->fMonitored)
+            pgmPoolMonitorFlush(pPool, pPage);
+        pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+        pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
+        pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
+        pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
+        pPage->GCPhys     = NIL_RTGCPHYS;
+        pPage->enmKind    = PGMPOOLKIND_FREE;
+        pPage->enmAccess  = PGMPOOLACCESS_DONTCARE;
+        Assert(pPage->idx == i);
+        pPage->iNext      = i + 1;
+        pPage->fA20Enabled = true;
+        pPage->fZeroed    = false;       /* This could probably be optimized, but better safe than sorry. */
+        pPage->fSeenNonGlobal = false;
+        pPage->fMonitored = false;
+        pPage->fDirty     = false;
+        pPage->fCached    = false;
+        pPage->fReusedFlushPending = false;
+        pPage->iUserHead  = NIL_PGMPOOL_USER_INDEX;
+        pPage->cPresent = 0;
+        pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
+        pPage->cModifications = 0;
+        pPage->iAgeNext   = NIL_PGMPOOL_IDX;
+        pPage->iAgePrev   = NIL_PGMPOOL_IDX;
+        pPage->idxDirtyEntry = 0;
+        pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
+        pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
+        pPage->cLastAccessHandler = 0;
+        pPage->cLocked    = 0;
+# ifdef VBOX_STRICT
+        pPage->GCPtrDirtyFault = NIL_RTGCPTR;
+# endif
+    }
+    pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
+    pPool->iFreeHead = PGMPOOL_IDX_FIRST;
+    pPool->cUsedPages = 0;
+
+    /*
+     * Zap and reinitialize the user records.
+     */
+    pPool->cPresent = 0;
+    pPool->iUserFreeHead = 0;
+    PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
+    const unsigned cMaxUsers = pPool->cMaxUsers;
+    for (unsigned i = 0; i < cMaxUsers; i++)
+    {
+        paUsers[i].iNext = i + 1;
+        paUsers[i].iUser = NIL_PGMPOOL_IDX;
+        paUsers[i].iUserTable = 0xfffffffe;
+    }
+    paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
+
+    /*
+     * Clear all the GCPhys links and rebuild the phys ext free list.
+     */
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        unsigned iPage = pRam->cb >> PAGE_SHIFT;
+        while (iPage-- > 0)
+            PGM_PAGE_SET_TRACKING(pVM, &pRam->aPages[iPage], 0);
+    }
+
+    pPool->iPhysExtFreeHead = 0;
+    PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
+    const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
+    for (unsigned i = 0; i < cMaxPhysExts; i++)
+    {
+        paPhysExts[i].iNext = i + 1;
+        paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+    }
+    paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
+
+    /*
+     * Just zap the modified list.
+     */
+    pPool->cModifiedPages = 0;
+    pPool->iModifiedHead = NIL_PGMPOOL_IDX;
+
+    /*
+     * Clear the GCPhys hash and the age list.
+     */
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
+        pPool->aiHash[i] = NIL_PGMPOOL_IDX;
+    pPool->iAgeHead = NIL_PGMPOOL_IDX;
+    pPool->iAgeTail = NIL_PGMPOOL_IDX;
+
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    /* Clear all dirty pages. */
+    pPool->idxFreeDirtyPage = 0;
+    pPool->cDirtyPages      = 0;
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aidxDirtyPages); i++)
+        pPool->aidxDirtyPages[i] = NIL_PGMPOOL_IDX;
+# endif
+
+    /*
+     * Reinsert active pages into the hash and ensure monitoring chains are correct.
+     */
+    VMCC_FOR_EACH_VMCPU(pVM)
+    {
+        /*
+         * Re-enter the shadowing mode and assert Sync CR3 FF.
+         */
+        pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+    }
+    VMCC_FOR_EACH_VMCPU_END(pVM);
+
+    STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
+}
+
+#endif /* IN_RING3 */
+
+#if defined(LOG_ENABLED) || defined(VBOX_STRICT)
+/**
+ * Stringifies a PGMPOOLKIND value.
+ */
+static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
+{
+    switch ((PGMPOOLKIND)enmKind)
+    {
+        case PGMPOOLKIND_INVALID:
+            return "PGMPOOLKIND_INVALID";
+        case PGMPOOLKIND_FREE:
+            return "PGMPOOLKIND_FREE";
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+            return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+            return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+            return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+            return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+            return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+            return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+            return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+            return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
+        case PGMPOOLKIND_32BIT_PD:
+            return "PGMPOOLKIND_32BIT_PD";
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            return "PGMPOOLKIND_32BIT_PD_PHYS";
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+            return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+            return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+            return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+            return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+            return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
+        case PGMPOOLKIND_PAE_PD_PHYS:
+            return "PGMPOOLKIND_PAE_PD_PHYS";
+        case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+            return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
+        case PGMPOOLKIND_PAE_PDPT:
+            return "PGMPOOLKIND_PAE_PDPT";
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+            return "PGMPOOLKIND_PAE_PDPT_PHYS";
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+            return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+            return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+            return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+            return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
+        case PGMPOOLKIND_64BIT_PML4:
+            return "PGMPOOLKIND_64BIT_PML4";
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+            return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+            return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+            return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
+        case PGMPOOLKIND_ROOT_NESTED:
+            return "PGMPOOLKIND_ROOT_NESTED";
+    }
+    return "Unknown kind!";
+}
+#endif /* LOG_ENABLED || VBOX_STRICT */
+
diff --git a/src/VBox/VMM/VMMAll/PGMAllShw.h b/src/VBox/VMM/VMMAll/PGMAllShw.h
new file mode 100644
index 00000000..e6012c16
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllShw.h
@@ -0,0 +1,617 @@
+/* $Id: PGMAllShw.h $ */
+/** @file
+ * VBox - Page Manager, Shadow Paging Template - All context code.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#undef SHWPT
+#undef PSHWPT
+#undef SHWPTE
+#undef PSHWPTE
+#undef SHWPD
+#undef PSHWPD
+#undef SHWPDE
+#undef PSHWPDE
+#undef SHW_PDE_PG_MASK
+#undef SHW_PD_SHIFT
+#undef SHW_PD_MASK
+#undef SHW_PTE_PG_MASK
+#undef SHW_PTE_IS_P
+#undef SHW_PTE_IS_RW
+#undef SHW_PTE_IS_US
+#undef SHW_PTE_IS_A
+#undef SHW_PTE_IS_D
+#undef SHW_PTE_IS_P_RW
+#undef SHW_PTE_IS_TRACK_DIRTY
+#undef SHW_PTE_GET_HCPHYS
+#undef SHW_PTE_GET_U
+#undef SHW_PTE_LOG64
+#undef SHW_PTE_SET
+#undef SHW_PTE_ATOMIC_SET
+#undef SHW_PTE_ATOMIC_SET2
+#undef SHW_PTE_SET_RO
+#undef SHW_PTE_SET_RW
+#undef SHW_PT_SHIFT
+#undef SHW_PT_MASK
+#undef SHW_TOTAL_PD_ENTRIES
+#undef SHW_PDPT_SHIFT
+#undef SHW_PDPT_MASK
+#undef SHW_PDPE_PG_MASK
+
+#if PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_NESTED_32BIT
+# define SHWPT                          X86PT
+# define PSHWPT                         PX86PT
+# define SHWPTE                         X86PTE
+# define PSHWPTE                        PX86PTE
+# define SHWPD                          X86PD
+# define PSHWPD                         PX86PD
+# define SHWPDE                         X86PDE
+# define PSHWPDE                        PX86PDE
+# define SHW_PDE_PG_MASK                X86_PDE_PG_MASK
+# define SHW_PD_SHIFT                   X86_PD_SHIFT
+# define SHW_PD_MASK                    X86_PD_MASK
+# define SHW_TOTAL_PD_ENTRIES           X86_PG_ENTRIES
+# define SHW_PTE_PG_MASK                X86_PTE_PG_MASK
+# define SHW_PTE_IS_P(Pte)              ( (Pte).n.u1Present )
+# define SHW_PTE_IS_RW(Pte)             ( (Pte).n.u1Write )
+# define SHW_PTE_IS_US(Pte)             ( (Pte).n.u1User )
+# define SHW_PTE_IS_A(Pte)              ( (Pte).n.u1Accessed )
+# define SHW_PTE_IS_D(Pte)              ( (Pte).n.u1Dirty )
+# define SHW_PTE_IS_P_RW(Pte)           ( (Pte).n.u1Present && (Pte).n.u1Write )
+# define SHW_PTE_IS_TRACK_DIRTY(Pte)    ( !!((Pte).u & PGM_PTFLAGS_TRACK_DIRTY) )
+# define SHW_PTE_GET_HCPHYS(Pte)        ( (Pte).u & X86_PTE_PG_MASK )
+# define SHW_PTE_LOG64(Pte)             ( (uint64_t)(Pte).u )
+# define SHW_PTE_GET_U(Pte)             ( (Pte).u )             /**< Use with care. */
+# define SHW_PTE_SET(Pte, uNew)         do { (Pte).u = (uNew); } while (0)
+# define SHW_PTE_ATOMIC_SET(Pte, uNew)  do { ASMAtomicWriteU32(&(Pte).u, (uNew)); } while (0)
+# define SHW_PTE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU32(&(Pte).u, (Pte2).u); } while (0)
+# define SHW_PTE_SET_RO(Pte)            do { (Pte).n.u1Write = 0; } while (0)
+# define SHW_PTE_SET_RW(Pte)            do { (Pte).n.u1Write = 1; } while (0)
+# define SHW_PT_SHIFT                   X86_PT_SHIFT
+# define SHW_PT_MASK                    X86_PT_MASK
+
+#elif PGM_SHW_TYPE == PGM_TYPE_EPT
+# define SHWPT                          EPTPT
+# define PSHWPT                         PEPTPT
+# define SHWPTE                         EPTPTE
+# define PSHWPTE                        PEPTPTE
+# define SHWPD                          EPTPD
+# define PSHWPD                         PEPTPD
+# define SHWPDE                         EPTPDE
+# define PSHWPDE                        PEPTPDE
+# define SHW_PDE_PG_MASK                EPT_PDE_PG_MASK
+# define SHW_PD_SHIFT                   EPT_PD_SHIFT
+# define SHW_PD_MASK                    EPT_PD_MASK
+# define SHW_PTE_PG_MASK                EPT_PTE_PG_MASK
+# define SHW_PTE_IS_P(Pte)              ( (Pte).n.u1Present )  /* Approximation, works for us. */
+# define SHW_PTE_IS_RW(Pte)             ( (Pte).n.u1Write )
+# define SHW_PTE_IS_US(Pte)             ( true )
+# define SHW_PTE_IS_A(Pte)              ( true )
+# define SHW_PTE_IS_D(Pte)              ( true )
+# define SHW_PTE_IS_P_RW(Pte)           ( (Pte).n.u1Present && (Pte).n.u1Write )
+# define SHW_PTE_IS_TRACK_DIRTY(Pte)    ( false )
+# define SHW_PTE_GET_HCPHYS(Pte)        ( (Pte).u & X86_PTE_PG_MASK )
+# define SHW_PTE_LOG64(Pte)             ( (Pte).u )
+# define SHW_PTE_GET_U(Pte)             ( (Pte).u )             /**< Use with care. */
+# define SHW_PTE_SET(Pte, uNew)         do { (Pte).u = (uNew); } while (0)
+# define SHW_PTE_ATOMIC_SET(Pte, uNew)  do { ASMAtomicWriteU64(&(Pte).u, (uNew)); } while (0)
+# define SHW_PTE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU64(&(Pte).u, (Pte2).u); } while (0)
+# define SHW_PTE_SET_RO(Pte)            do { (Pte).n.u1Write = 0; } while (0)
+# define SHW_PTE_SET_RW(Pte)            do { (Pte).n.u1Write = 1; } while (0)
+# define SHW_PT_SHIFT                   EPT_PT_SHIFT
+# define SHW_PT_MASK                    EPT_PT_MASK
+# define SHW_PDPT_SHIFT                 EPT_PDPT_SHIFT
+# define SHW_PDPT_MASK                  EPT_PDPT_MASK
+# define SHW_PDPE_PG_MASK               EPT_PDPE_PG_MASK
+# define SHW_TOTAL_PD_ENTRIES           (EPT_PG_AMD64_ENTRIES*EPT_PG_AMD64_PDPE_ENTRIES)
+
+#else
+# define SHWPT                          PGMSHWPTPAE
+# define PSHWPT                         PPGMSHWPTPAE
+# define SHWPTE                         PGMSHWPTEPAE
+# define PSHWPTE                        PPGMSHWPTEPAE
+# define SHWPD                          X86PDPAE
+# define PSHWPD                         PX86PDPAE
+# define SHWPDE                         X86PDEPAE
+# define PSHWPDE                        PX86PDEPAE
+# define SHW_PDE_PG_MASK                X86_PDE_PAE_PG_MASK
+# define SHW_PD_SHIFT                   X86_PD_PAE_SHIFT
+# define SHW_PD_MASK                    X86_PD_PAE_MASK
+# define SHW_PTE_PG_MASK                X86_PTE_PAE_PG_MASK
+# define SHW_PTE_IS_P(Pte)              PGMSHWPTEPAE_IS_P(Pte)
+# define SHW_PTE_IS_RW(Pte)             PGMSHWPTEPAE_IS_RW(Pte)
+# define SHW_PTE_IS_US(Pte)             PGMSHWPTEPAE_IS_US(Pte)
+# define SHW_PTE_IS_A(Pte)              PGMSHWPTEPAE_IS_A(Pte)
+# define SHW_PTE_IS_D(Pte)              PGMSHWPTEPAE_IS_D(Pte)
+# define SHW_PTE_IS_P_RW(Pte)           PGMSHWPTEPAE_IS_P_RW(Pte)
+# define SHW_PTE_IS_TRACK_DIRTY(Pte)    PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte)
+# define SHW_PTE_GET_HCPHYS(Pte)        PGMSHWPTEPAE_GET_HCPHYS(Pte)
+# define SHW_PTE_LOG64(Pte)             PGMSHWPTEPAE_GET_LOG(Pte)
+# define SHW_PTE_GET_U(Pte)             PGMSHWPTEPAE_GET_U(Pte)     /**< Use with care. */
+# define SHW_PTE_SET(Pte, uNew)         PGMSHWPTEPAE_SET(Pte, uNew)
+# define SHW_PTE_ATOMIC_SET(Pte, uNew)  PGMSHWPTEPAE_ATOMIC_SET(Pte, uNew)
+# define SHW_PTE_ATOMIC_SET2(Pte, Pte2) PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2)
+# define SHW_PTE_SET_RO(Pte)            PGMSHWPTEPAE_SET_RO(Pte)
+# define SHW_PTE_SET_RW(Pte)            PGMSHWPTEPAE_SET_RW(Pte)
+# define SHW_PT_SHIFT                   X86_PT_PAE_SHIFT
+# define SHW_PT_MASK                    X86_PT_PAE_MASK
+
+# if PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64 || /* whatever: */ PGM_SHW_TYPE == PGM_TYPE_NONE
+#  define SHW_PDPT_SHIFT                X86_PDPT_SHIFT
+#  define SHW_PDPT_MASK                 X86_PDPT_MASK_AMD64
+#  define SHW_PDPE_PG_MASK              X86_PDPE_PG_MASK
+#  define SHW_TOTAL_PD_ENTRIES          (X86_PG_AMD64_ENTRIES * X86_PG_AMD64_PDPE_ENTRIES)
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE
+#  define SHW_PDPT_SHIFT                X86_PDPT_SHIFT
+#  define SHW_PDPT_MASK                 X86_PDPT_MASK_PAE
+#  define SHW_PDPE_PG_MASK              X86_PDPE_PG_MASK
+#  define SHW_TOTAL_PD_ENTRIES          (X86_PG_PAE_ENTRIES * X86_PG_PAE_PDPE_ENTRIES)
+
+# else
+#  error "Misconfigured PGM_SHW_TYPE or something..."
+# endif
+#endif
+
+#if PGM_SHW_TYPE == PGM_TYPE_NONE && PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
+# error "PGM_TYPE_IS_NESTED_OR_EPT is true for PGM_TYPE_NONE!"
+#endif
+
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+PGM_SHW_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCUINTPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys);
+PGM_SHW_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCUINTPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags);
+PGM_SHW_DECL(int, Enter)(PVMCPUCC pVCpu, bool fIs64BitsPagingMode);
+PGM_SHW_DECL(int, Exit)(PVMCPUCC pVCpu);
+#ifdef IN_RING3
+PGM_SHW_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
+#endif
+RT_C_DECLS_END
+
+
+/**
+ * Enters the shadow mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   fIs64BitsPagingMode     New shadow paging mode is for 64 bits? (only relevant for 64 bits guests on a 32 bits AMD-V nested paging host)
+ */
+PGM_SHW_DECL(int, Enter)(PVMCPUCC pVCpu, bool fIs64BitsPagingMode)
+{
+#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
+
+# if PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && HC_ARCH_BITS == 32
+    /* Must distinguish between 32 and 64 bits guest paging modes as we'll use
+       a different shadow paging root/mode in both cases. */
+    RTGCPHYS     GCPhysCR3 = (fIs64BitsPagingMode) ? RT_BIT_64(63) : RT_BIT_64(62);
+# else
+    RTGCPHYS     GCPhysCR3 = RT_BIT_64(63); NOREF(fIs64BitsPagingMode);
+# endif
+    PPGMPOOLPAGE pNewShwPageCR3;
+    PVMCC        pVM       = pVCpu->CTX_SUFF(pVM);
+
+    Assert((HMIsNestedPagingActive(pVM) || VM_IS_NEM_ENABLED(pVM)) == pVM->pgm.s.fNestedPaging);
+    Assert(pVM->pgm.s.fNestedPaging);
+    Assert(!pVCpu->pgm.s.pShwPageCR3R3);
+
+    pgmLock(pVM);
+
+    int rc = pgmPoolAlloc(pVM, GCPhysCR3, PGMPOOLKIND_ROOT_NESTED, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                          NIL_PGMPOOL_IDX, UINT32_MAX, true /*fLockPage*/,
+                          &pNewShwPageCR3);
+    AssertLogRelRCReturnStmt(rc, pgmUnlock(pVM), rc);
+
+    pVCpu->pgm.s.pShwPageCR3R3 = (R3PTRTYPE(PPGMPOOLPAGE))MMHyperCCToR3(pVM, pNewShwPageCR3);
+    pVCpu->pgm.s.pShwPageCR3R0 = (R0PTRTYPE(PPGMPOOLPAGE))MMHyperCCToR0(pVM, pNewShwPageCR3);
+
+    pgmUnlock(pVM);
+
+    Log(("Enter nested shadow paging mode: root %RHv phys %RHp\n", pVCpu->pgm.s.pShwPageCR3R3, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->Core.Key));
+#else
+    NOREF(pVCpu); NOREF(fIs64BitsPagingMode);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exits the shadow mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+PGM_SHW_DECL(int, Exit)(PVMCPUCC pVCpu)
+{
+#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
+    {
+        PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+        pgmLock(pVM);
+
+        /* Do *not* unlock this page as we have two of them floating around in the 32-bit host & 64-bit guest case.
+         * We currently assert when you try to free one of them; don't bother to really allow this.
+         *
+         * Note that this is two nested paging root pages max. This isn't a leak. They are reused.
+         */
+        /* pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); */
+
+        pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
+        pVCpu->pgm.s.pShwPageCR3R3 = 0;
+        pVCpu->pgm.s.pShwPageCR3R0 = 0;
+
+        pgmUnlock(pVM);
+
+        Log(("Leave nested shadow paging mode\n"));
+    }
+#else
+    RT_NOREF_PV(pVCpu);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets effective page information (from the VMM page directory).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Guest Context virtual address of the page.
+ * @param   pfFlags     Where to store the flags. These are X86_PTE_*.
+ * @param   pHCPhys     Where to store the HC physical address of the page.
+ *                      This is page aligned.
+ * @remark  You should use PGMMapGetPage() for pages in a mapping.
+ */
+PGM_SHW_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCUINTPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
+{
+#if PGM_SHW_TYPE == PGM_TYPE_NONE
+    RT_NOREF(pVCpu, GCPtr);
+    AssertFailed();
+    *pfFlags = 0;
+    *pHCPhys = NIL_RTHCPHYS;
+    return VERR_PGM_SHW_NONE_IPE;
+
+#else  /* PGM_SHW_TYPE != PGM_TYPE_NONE */
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Get the PDE.
+     */
+# if PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64
+    X86PDEPAE Pde;
+
+    /* PML4 */
+    X86PML4E        Pml4e = pgmShwGetLongModePML4E(pVCpu, GCPtr);
+    if (!Pml4e.n.u1Present)
+        return VERR_PAGE_TABLE_NOT_PRESENT;
+
+    /* PDPT */
+    PX86PDPT        pPDPT;
+    int rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pml4e.u & X86_PML4E_PG_MASK, &pPDPT);
+    if (RT_FAILURE(rc))
+        return rc;
+    const unsigned  iPDPT = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
+    X86PDPE         Pdpe = pPDPT->a[iPDPT];
+    if (!Pdpe.n.u1Present)
+        return VERR_PAGE_TABLE_NOT_PRESENT;
+
+    /* PD */
+    PX86PDPAE       pPd;
+    rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pdpe.u & X86_PDPE_PG_MASK, &pPd);
+    if (RT_FAILURE(rc))
+        return rc;
+    const unsigned  iPd = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    Pde = pPd->a[iPd];
+
+    /* Merge accessed, write, user and no-execute bits into the PDE. */
+    Pde.n.u1Accessed  &= Pml4e.n.u1Accessed & Pdpe.lm.u1Accessed;
+    Pde.n.u1Write     &= Pml4e.n.u1Write & Pdpe.lm.u1Write;
+    Pde.n.u1User      &= Pml4e.n.u1User & Pdpe.lm.u1User;
+    Pde.n.u1NoExecute |= Pml4e.n.u1NoExecute | Pdpe.lm.u1NoExecute;
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE
+    X86PDEPAE       Pde = pgmShwGetPaePDE(pVCpu, GCPtr);
+
+# elif PGM_SHW_TYPE == PGM_TYPE_EPT
+    const unsigned  iPd = ((GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PEPTPD          pPDDst;
+    EPTPDE          Pde;
+
+    int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtr, NULL, &pPDDst);
+    if (rc != VINF_SUCCESS) /** @todo this function isn't expected to return informational status codes. Check callers / fix. */
+    {
+        AssertRC(rc);
+        return rc;
+    }
+    Assert(pPDDst);
+    Pde = pPDDst->a[iPd];
+
+# elif PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_NESTED_32BIT
+    X86PDE          Pde = pgmShwGet32BitPDE(pVCpu, GCPtr);
+# else
+#  error "Misconfigured PGM_SHW_TYPE or something..."
+# endif
+    if (!Pde.n.u1Present)
+        return VERR_PAGE_TABLE_NOT_PRESENT;
+
+    /* Deal with large pages. */
+    if (Pde.b.u1Size)
+    {
+        /*
+         * Store the results.
+         * RW and US flags depend on the entire page translation hierarchy - except for
+         * legacy PAE which has a simplified PDPE.
+         */
+        if (pfFlags)
+        {
+            *pfFlags = (Pde.u & ~SHW_PDE_PG_MASK);
+# if PGM_WITH_NX(PGM_SHW_TYPE, PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64
+            if (   (Pde.u & X86_PTE_PAE_NX)
+#  if PGM_WITH_NX(PGM_SHW_TYPE, PGM_SHW_TYPE)
+                && CPUMIsGuestNXEnabled(pVCpu) /** @todo why do we have to check the guest state here? */
+#  endif
+               )
+                *pfFlags |= X86_PTE_PAE_NX;
+# endif
+        }
+
+        if (pHCPhys)
+            *pHCPhys = (Pde.u & SHW_PDE_PG_MASK) + (GCPtr & (RT_BIT(SHW_PD_SHIFT) - 1) & X86_PAGE_4K_BASE_MASK);
+
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Get PT entry.
+     */
+    PSHWPT          pPT;
+    if (!(Pde.u & PGM_PDFLAGS_MAPPING))
+    {
+        int rc2 = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pde.u & SHW_PDE_PG_MASK, &pPT);
+        if (RT_FAILURE(rc2))
+            return rc2;
+    }
+    else /* mapping: */
+    {
+# if  PGM_SHW_TYPE == PGM_TYPE_AMD64 \
+  || PGM_SHW_TYPE == PGM_TYPE_EPT \
+  || defined(PGM_WITHOUT_MAPPINGS)
+        AssertFailed(); /* can't happen */
+        pPT = NULL;     /* shut up MSC */
+# else
+        Assert(pgmMapAreMappingsEnabled(pVM));
+
+        PPGMMAPPING pMap = pgmGetMapping(pVM, (RTGCPTR)GCPtr);
+        AssertMsgReturn(pMap, ("GCPtr=%RGv\n", GCPtr), VERR_PGM_MAPPING_IPE);
+#  if PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_NESTED_32BIT
+        pPT = pMap->aPTs[(GCPtr - pMap->GCPtr) >> X86_PD_SHIFT].CTX_SUFF(pPT);
+#  else /* PAE */
+        pPT = pMap->aPTs[(GCPtr - pMap->GCPtr) >> X86_PD_SHIFT].CTX_SUFF(paPaePTs);
+#  endif
+# endif
+    }
+    const unsigned  iPt = (GCPtr >> SHW_PT_SHIFT) & SHW_PT_MASK;
+    SHWPTE          Pte = pPT->a[iPt];
+    if (!SHW_PTE_IS_P(Pte))
+        return VERR_PAGE_NOT_PRESENT;
+
+    /*
+     * Store the results.
+     * RW and US flags depend on the entire page translation hierarchy - except for
+     * legacy PAE which has a simplified PDPE.
+     */
+    if (pfFlags)
+    {
+        *pfFlags = (SHW_PTE_GET_U(Pte) & ~SHW_PTE_PG_MASK)
+                 & ((Pde.u & (X86_PTE_RW | X86_PTE_US)) | ~(uint64_t)(X86_PTE_RW | X86_PTE_US));
+
+# if PGM_WITH_NX(PGM_SHW_TYPE, PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64
+        /* The NX bit is determined by a bitwise OR between the PT and PD */
+        if (   ((SHW_PTE_GET_U(Pte) | Pde.u) & X86_PTE_PAE_NX)
+#  if PGM_WITH_NX(PGM_SHW_TYPE, PGM_SHW_TYPE)
+            && CPUMIsGuestNXEnabled(pVCpu) /** @todo why do we have to check the guest state here? */
+#  endif
+           )
+            *pfFlags |= X86_PTE_PAE_NX;
+# endif
+    }
+
+    if (pHCPhys)
+        *pHCPhys = SHW_PTE_GET_HCPHYS(Pte);
+
+    return VINF_SUCCESS;
+#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
+}
+
+
+/**
+ * Modify page flags for a range of pages in the shadow context.
+ *
+ * The existing flags are ANDed with the fMask and ORed with the fFlags.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range. Page aligned!
+ * @param   cb          Size (in bytes) of the range to apply the modification to. Page aligned!
+ * @param   fFlags      The OR  mask - page flags X86_PTE_*, excluding the page mask of course.
+ * @param   fMask       The AND mask - page flags X86_PTE_*.
+ *                      Be extremely CAREFUL with ~'ing values because they can be 32-bit!
+ * @param   fOpFlags    A combination of the PGM_MK_PK_XXX flags.
+ * @remark  You must use PGMMapModifyPage() for pages in a mapping.
+ */
+PGM_SHW_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCUINTPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
+{
+#if PGM_SHW_TYPE == PGM_TYPE_NONE
+    RT_NOREF(pVCpu, GCPtr, cb, fFlags, fMask, fOpFlags);
+    AssertFailed();
+    return VERR_PGM_SHW_NONE_IPE;
+
+#else  /* PGM_SHW_TYPE != PGM_TYPE_NONE */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Walk page tables and pages till we're done.
+     */
+    int rc;
+    for (;;)
+    {
+        /*
+         * Get the PDE.
+         */
+# if PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64
+        X86PDEPAE       Pde;
+        /* PML4 */
+        X86PML4E        Pml4e = pgmShwGetLongModePML4E(pVCpu, GCPtr);
+        if (!Pml4e.n.u1Present)
+            return VERR_PAGE_TABLE_NOT_PRESENT;
+
+        /* PDPT */
+        PX86PDPT        pPDPT;
+        rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pml4e.u & X86_PML4E_PG_MASK, &pPDPT);
+        if (RT_FAILURE(rc))
+            return rc;
+        const unsigned  iPDPT = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
+        X86PDPE         Pdpe = pPDPT->a[iPDPT];
+        if (!Pdpe.n.u1Present)
+            return VERR_PAGE_TABLE_NOT_PRESENT;
+
+        /* PD */
+        PX86PDPAE       pPd;
+        rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pdpe.u & X86_PDPE_PG_MASK, &pPd);
+        if (RT_FAILURE(rc))
+            return rc;
+        const unsigned iPd = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
+        Pde = pPd->a[iPd];
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE
+        X86PDEPAE       Pde = pgmShwGetPaePDE(pVCpu, GCPtr);
+
+# elif PGM_SHW_TYPE == PGM_TYPE_EPT
+        const unsigned  iPd = ((GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK);
+        PEPTPD          pPDDst;
+        EPTPDE          Pde;
+
+        rc = pgmShwGetEPTPDPtr(pVCpu, GCPtr, NULL, &pPDDst);
+        if (rc != VINF_SUCCESS)
+        {
+            AssertRC(rc);
+            return rc;
+        }
+        Assert(pPDDst);
+        Pde = pPDDst->a[iPd];
+
+# else /* PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_NESTED_32BIT */
+        X86PDE          Pde = pgmShwGet32BitPDE(pVCpu, GCPtr);
+# endif
+        if (!Pde.n.u1Present)
+            return VERR_PAGE_TABLE_NOT_PRESENT;
+
+        AssertFatal(!Pde.b.u1Size);
+
+        /*
+         * Map the page table.
+         */
+        PSHWPT          pPT;
+        rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pde.u & SHW_PDE_PG_MASK, &pPT);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        unsigned        iPTE = (GCPtr >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        while (iPTE < RT_ELEMENTS(pPT->a))
+        {
+            if (SHW_PTE_IS_P(pPT->a[iPTE]))
+            {
+                SHWPTE const    OrgPte = pPT->a[iPTE];
+                SHWPTE          NewPte;
+
+                SHW_PTE_SET(NewPte, (SHW_PTE_GET_U(OrgPte) & (fMask | SHW_PTE_PG_MASK)) | (fFlags & ~SHW_PTE_PG_MASK));
+                if (!SHW_PTE_IS_P(NewPte))
+                {
+                    /** @todo Some CSAM code path might end up here and upset
+                     *  the page pool. */
+                    AssertFailed();
+                }
+                else if (   SHW_PTE_IS_RW(NewPte)
+                         && !SHW_PTE_IS_RW(OrgPte)
+                         && !(fOpFlags & PGM_MK_PG_IS_MMIO2) )
+                {
+                    /** @todo Optimize \#PF handling by caching data.  We can
+                     *        then use this when PGM_MK_PG_IS_WRITE_FAULT is
+                     *        set instead of resolving the guest physical
+                     *        address yet again. */
+                    RTGCPHYS GCPhys;
+                    uint64_t fGstPte;
+                    rc = PGMGstGetPage(pVCpu, GCPtr, &fGstPte, &GCPhys);
+                    AssertRC(rc);
+                    if (RT_SUCCESS(rc))
+                    {
+                        Assert((fGstPte & X86_PTE_RW) || !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP /* allow netware hack */));
+                        PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+                        Assert(pPage);
+                        if (pPage)
+                        {
+                            rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+                            AssertRCReturn(rc, rc);
+                            Log(("%s: pgmPhysPageMakeWritable on %RGv / %RGp %R[pgmpage]\n", __PRETTY_FUNCTION__, GCPtr, GCPhys, pPage));
+                        }
+                    }
+                }
+
+                SHW_PTE_ATOMIC_SET2(pPT->a[iPTE], NewPte);
+# if PGM_SHW_TYPE == PGM_TYPE_EPT
+                HMInvalidatePhysPage(pVM, (RTGCPHYS)GCPtr);
+# else
+                PGM_INVL_PG_ALL_VCPU(pVM, GCPtr);
+# endif
+            }
+
+            /* next page */
+            cb -= PAGE_SIZE;
+            if (!cb)
+                return VINF_SUCCESS;
+            GCPtr += PAGE_SIZE;
+            iPTE++;
+        }
+    }
+#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
+}
+
+
+#ifdef IN_RING3
+/**
+ * Relocate any GC pointers related to shadow mode paging.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offDelta    The relocation offset.
+ */
+PGM_SHW_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta)
+{
+    RT_NOREF(pVCpu, offDelta);
+    return VINF_SUCCESS;
+}
+#endif
+
diff --git a/src/VBox/VMM/VMMAll/SELMAll.cpp b/src/VBox/VMM/VMMAll/SELMAll.cpp
new file mode 100644
index 00000000..6124fb40
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/SELMAll.cpp
@@ -0,0 +1,388 @@
+/* $Id: SELMAll.cpp $ */
+/** @file
+ * SELM All contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_SELM
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/hm.h>
+#include "SELMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <VBox/vmm/vmm.h>
+#include <iprt/x86.h>
+#include <iprt/string.h>
+
+
+
+/**
+ * Converts a GC selector based address to a flat address.
+ *
+ * No limit checks are done. Use the SELMToFlat*() or SELMValidate*() functions
+ * for that.
+ *
+ * @returns Flat address.
+ * @param   pVM         The cross context VM structure.
+ * @param   SelReg      Selector register
+ * @param   pCtxCore    CPU context
+ * @param   Addr        Address part.
+ */
+VMMDECL(RTGCPTR) SELMToFlat(PVMCC pVM, DISSELREG SelReg, PCPUMCTXCORE pCtxCore, RTGCPTR Addr)
+{
+    PCPUMSELREG    pSReg;
+    PVMCPUCC       pVCpu = VMMGetCpu(pVM);
+
+    int rc = DISFetchRegSegEx(pCtxCore, SelReg, &pSReg); AssertRC(rc);
+
+    /*
+     * Deal with real & v86 mode first.
+     */
+    if (    pCtxCore->eflags.Bits.u1VM
+        ||  CPUMIsGuestInRealMode(pVCpu))
+    {
+        uint32_t uFlat = (uint32_t)Addr & 0xffff;
+        if (CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg))
+            uFlat += (uint32_t)pSReg->u64Base;
+        else
+            uFlat += (uint32_t)pSReg->Sel << 4;
+        return (RTGCPTR)uFlat;
+    }
+
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtxCore->cs));
+
+    /* 64 bits mode: CS, DS, ES and SS are treated as if each segment base is 0
+       (Intel® 64 and IA-32 Architectures Software Developer's Manual: 3.4.2.1). */
+    if (    pCtxCore->cs.Attr.n.u1Long
+        &&  CPUMIsGuestInLongMode(pVCpu))
+    {
+        switch (SelReg)
+        {
+            case DISSELREG_FS:
+            case DISSELREG_GS:
+                return (RTGCPTR)(pSReg->u64Base + Addr);
+
+            default:
+                return Addr;    /* base 0 */
+        }
+    }
+
+    /* AMD64 manual: compatibility mode ignores the high 32 bits when calculating an effective address. */
+    Assert(pSReg->u64Base <= 0xffffffff);
+    return (uint32_t)pSReg->u64Base + (uint32_t)Addr;
+}
+
+
+/**
+ * Converts a GC selector based address to a flat address.
+ *
+ * Some basic checking is done, but not all kinds yet.
+ *
+ * @returns VBox status
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   SelReg      Selector register.
+ * @param   pCtxCore    CPU context.
+ * @param   Addr        Address part.
+ * @param   fFlags      SELMTOFLAT_FLAGS_*
+ *                      GDT entires are valid.
+ * @param   ppvGC       Where to store the GC flat address.
+ */
+VMMDECL(int) SELMToFlatEx(PVMCPU pVCpu, DISSELREG SelReg, PCPUMCTXCORE pCtxCore, RTGCPTR Addr, uint32_t fFlags, PRTGCPTR ppvGC)
+{
+    /*
+     * Fetch the selector first.
+     */
+    PCPUMSELREG pSReg;
+    int rc = DISFetchRegSegEx(pCtxCore, SelReg, &pSReg);
+    AssertRCReturn(rc, rc); AssertPtr(pSReg);
+
+    /*
+     * Deal with real & v86 mode first.
+     */
+    if (    pCtxCore->eflags.Bits.u1VM
+        ||  CPUMIsGuestInRealMode(pVCpu))
+    {
+        if (ppvGC)
+        {
+            uint32_t uFlat = (uint32_t)Addr & 0xffff;
+            if (CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg))
+                *ppvGC = (uint32_t)pSReg->u64Base + uFlat;
+            else
+                *ppvGC = ((uint32_t)pSReg->Sel << 4) + uFlat;
+        }
+        return VINF_SUCCESS;
+    }
+
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtxCore->cs));
+
+    /* 64 bits mode: CS, DS, ES and SS are treated as if each segment base is 0
+       (Intel® 64 and IA-32 Architectures Software Developer's Manual: 3.4.2.1). */
+    RTGCPTR  pvFlat;
+    bool     fCheckLimit   = true;
+    if (    pCtxCore->cs.Attr.n.u1Long
+        &&  CPUMIsGuestInLongMode(pVCpu))
+    {
+        fCheckLimit = false;
+        switch (SelReg)
+        {
+            case DISSELREG_FS:
+            case DISSELREG_GS:
+                pvFlat = pSReg->u64Base + Addr;
+                break;
+
+            default:
+                pvFlat = Addr;
+                break;
+        }
+    }
+    else
+    {
+        /* AMD64 manual: compatibility mode ignores the high 32 bits when calculating an effective address. */
+        Assert(pSReg->u64Base <= UINT32_C(0xffffffff));
+        pvFlat  = (uint32_t)pSReg->u64Base + (uint32_t)Addr;
+        Assert(pvFlat <= UINT32_MAX);
+    }
+
+    /*
+     * Check type if present.
+     */
+    if (pSReg->Attr.n.u1Present)
+    {
+        switch (pSReg->Attr.n.u4Type)
+        {
+            /* Read only selector type. */
+            case X86_SEL_TYPE_RO:
+            case X86_SEL_TYPE_RO_ACC:
+            case X86_SEL_TYPE_RW:
+            case X86_SEL_TYPE_RW_ACC:
+            case X86_SEL_TYPE_EO:
+            case X86_SEL_TYPE_EO_ACC:
+            case X86_SEL_TYPE_ER:
+            case X86_SEL_TYPE_ER_ACC:
+                if (!(fFlags & SELMTOFLAT_FLAGS_NO_PL))
+                {
+                    /** @todo fix this mess */
+                }
+                /* check limit. */
+                if (fCheckLimit && Addr > pSReg->u32Limit)
+                    return VERR_OUT_OF_SELECTOR_BOUNDS;
+                /* ok */
+                if (ppvGC)
+                    *ppvGC = pvFlat;
+                return VINF_SUCCESS;
+
+            case X86_SEL_TYPE_EO_CONF:
+            case X86_SEL_TYPE_EO_CONF_ACC:
+            case X86_SEL_TYPE_ER_CONF:
+            case X86_SEL_TYPE_ER_CONF_ACC:
+                if (!(fFlags & SELMTOFLAT_FLAGS_NO_PL))
+                {
+                    /** @todo fix this mess */
+                }
+                /* check limit. */
+                if (fCheckLimit && Addr > pSReg->u32Limit)
+                    return VERR_OUT_OF_SELECTOR_BOUNDS;
+                /* ok */
+                if (ppvGC)
+                    *ppvGC = pvFlat;
+                return VINF_SUCCESS;
+
+            case X86_SEL_TYPE_RO_DOWN:
+            case X86_SEL_TYPE_RO_DOWN_ACC:
+            case X86_SEL_TYPE_RW_DOWN:
+            case X86_SEL_TYPE_RW_DOWN_ACC:
+                if (!(fFlags & SELMTOFLAT_FLAGS_NO_PL))
+                {
+                    /** @todo fix this mess */
+                }
+                /* check limit. */
+                if (fCheckLimit)
+                {
+                    if (!pSReg->Attr.n.u1Granularity && Addr > UINT32_C(0xffff))
+                        return VERR_OUT_OF_SELECTOR_BOUNDS;
+                    if (Addr <= pSReg->u32Limit)
+                        return VERR_OUT_OF_SELECTOR_BOUNDS;
+                }
+                /* ok */
+                if (ppvGC)
+                    *ppvGC = pvFlat;
+                return VINF_SUCCESS;
+
+            default:
+                return VERR_INVALID_SELECTOR;
+
+        }
+    }
+    return VERR_SELECTOR_NOT_PRESENT;
+}
+
+
+
+/**
+ * Validates and converts a GC selector based code address to a flat
+ * address when in real or v8086 mode.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   SelCS   Selector part.
+ * @param   pSReg   The hidden CS register part. Optional.
+ * @param   Addr    Address part.
+ * @param   ppvFlat Where to store the flat address.
+ */
+DECLINLINE(int) selmValidateAndConvertCSAddrRealMode(PVMCPU pVCpu, RTSEL SelCS, PCCPUMSELREGHID pSReg, RTGCPTR Addr,
+                                                     PRTGCPTR ppvFlat)
+{
+    NOREF(pVCpu);
+    uint32_t uFlat = Addr & 0xffff;
+    if (!pSReg || !CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg))
+        uFlat += (uint32_t)SelCS << 4;
+    else
+        uFlat += (uint32_t)pSReg->u64Base;
+    *ppvFlat = uFlat;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Validates and converts a GC selector based code address to a flat address
+ * when in protected/long mode using the standard hidden selector registers
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   SelCPL      Current privilege level.  Get this from SS - CS might be
+ *                      conforming!  A full selector can be passed, we'll only
+ *                      use the RPL part.
+ * @param   SelCS       Selector part.
+ * @param   pSRegCS     The full CS selector register.
+ * @param   Addr        The address (think IP/EIP/RIP).
+ * @param   ppvFlat     Where to store the flat address upon successful return.
+ */
+DECLINLINE(int) selmValidateAndConvertCSAddrHidden(PVMCPU pVCpu, RTSEL SelCPL, RTSEL SelCS, PCCPUMSELREGHID pSRegCS,
+                                                   RTGCPTR Addr, PRTGCPTR ppvFlat)
+{
+    NOREF(SelCPL); NOREF(SelCS);
+
+    /*
+     * Check if present.
+     */
+    if (pSRegCS->Attr.n.u1Present)
+    {
+        /*
+         * Type check.
+         */
+        if (     pSRegCS->Attr.n.u1DescType == 1
+            &&  (pSRegCS->Attr.n.u4Type & X86_SEL_TYPE_CODE))
+        {
+            /* 64 bits mode: CS, DS, ES and SS are treated as if each segment base is 0
+               (Intel® 64 and IA-32 Architectures Software Developer's Manual: 3.4.2.1). */
+            if (    pSRegCS->Attr.n.u1Long
+                &&  CPUMIsGuestInLongMode(pVCpu))
+            {
+                *ppvFlat = Addr;
+                return VINF_SUCCESS;
+            }
+
+            /*
+             * Limit check. Note that the limit in the hidden register is the
+             * final value. The granularity bit was included in its calculation.
+             */
+            uint32_t u32Limit = pSRegCS->u32Limit;
+            if ((uint32_t)Addr <= u32Limit)
+            {
+                *ppvFlat = (uint32_t)Addr + (uint32_t)pSRegCS->u64Base;
+                return VINF_SUCCESS;
+            }
+
+            return VERR_OUT_OF_SELECTOR_BOUNDS;
+        }
+        return VERR_NOT_CODE_SELECTOR;
+    }
+    return VERR_SELECTOR_NOT_PRESENT;
+}
+
+
+/**
+ * Validates and converts a GC selector based code address to a flat address.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   Efl         Current EFLAGS.
+ * @param   SelCPL      Current privilege level.  Get this from SS - CS might be
+ *                      conforming!  A full selector can be passed, we'll only
+ *                      use the RPL part.
+ * @param   SelCS       Selector part.
+ * @param   pSRegCS     The full CS selector register.
+ * @param   Addr        The address (think IP/EIP/RIP).
+ * @param   ppvFlat     Where to store the flat address upon successful return.
+ */
+VMMDECL(int) SELMValidateAndConvertCSAddr(PVMCPU pVCpu, X86EFLAGS Efl, RTSEL SelCPL, RTSEL SelCS, PCPUMSELREG pSRegCS,
+                                          RTGCPTR Addr, PRTGCPTR ppvFlat)
+{
+    if (    Efl.Bits.u1VM
+        ||  CPUMIsGuestInRealMode(pVCpu))
+        return selmValidateAndConvertCSAddrRealMode(pVCpu, SelCS, pSRegCS, Addr, ppvFlat);
+
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSRegCS));
+    Assert(pSRegCS->Sel == SelCS);
+
+    return selmValidateAndConvertCSAddrHidden(pVCpu, SelCPL, SelCS, pSRegCS, Addr, ppvFlat);
+}
+
+
+/**
+ * Gets info about the current TSS.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if we've got a TSS loaded.
+ * @retval  VERR_SELM_NO_TSS if we haven't got a TSS (rather unlikely).
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   pGCPtrTss           Where to store the TSS address.
+ * @param   pcbTss              Where to store the TSS size limit.
+ * @param   pfCanHaveIOBitmap   Where to store the can-have-I/O-bitmap indicator. (optional)
+ */
+VMMDECL(int) SELMGetTSSInfo(PVM pVM, PVMCPU pVCpu, PRTGCUINTPTR pGCPtrTss, PRTGCUINTPTR pcbTss, bool *pfCanHaveIOBitmap)
+{
+    NOREF(pVM);
+
+    /*
+     * The TR hidden register is always valid.
+     */
+    CPUMSELREGHID trHid;
+    RTSEL tr = CPUMGetGuestTR(pVCpu, &trHid);
+    if (!(tr & X86_SEL_MASK_OFF_RPL))
+        return VERR_SELM_NO_TSS;
+
+    *pGCPtrTss = trHid.u64Base;
+    *pcbTss    = trHid.u32Limit + (trHid.u32Limit != UINT32_MAX); /* be careful. */
+    if (pfCanHaveIOBitmap)
+        *pfCanHaveIOBitmap = trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
+                          || trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY;
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMAll/TMAll.cpp b/src/VBox/VMM/VMMAll/TMAll.cpp
new file mode 100644
index 00000000..f45ad6dd
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/TMAll.cpp
@@ -0,0 +1,2617 @@
+/* $Id: TMAll.cpp $ */
+/** @file
+ * TM - Timeout Manager, all contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_TM
+#ifdef DEBUG_bird
+# define DBGFTRACE_DISABLED /* annoying */
+#endif
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/dbgftrace.h>
+#ifdef IN_RING3
+#endif
+#include <VBox/vmm/pdmdev.h> /* (for TMTIMER_GET_CRITSECT implementation) */
+#include "TMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/sup.h>
+#include <iprt/time.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/asm-math.h>
+#ifdef IN_RING3
+# include <iprt/thread.h>
+#endif
+
+#include "TMInline.h"
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#ifdef VBOX_STRICT
+/** @def TMTIMER_GET_CRITSECT
+ * Helper for safely resolving the critical section for a timer belonging to a
+ * device instance.
+ * @todo needs reworking later as it uses PDMDEVINSR0::pDevInsR0RemoveMe.  */
+# ifdef IN_RING3
+#  define TMTIMER_GET_CRITSECT(pTimer) ((pTimer)->pCritSect)
+# else
+#  define TMTIMER_GET_CRITSECT(pTimer) tmRZTimerGetCritSect(pTimer)
+# endif
+#endif
+
+/** @def TMTIMER_ASSERT_CRITSECT
+ * Checks that the caller owns the critical section if one is associated with
+ * the timer. */
+#ifdef VBOX_STRICT
+# define TMTIMER_ASSERT_CRITSECT(pTimer) \
+    do { \
+        if ((pTimer)->pCritSect) \
+        { \
+            VMSTATE      enmState; \
+            PPDMCRITSECT pCritSect = TMTIMER_GET_CRITSECT(pTimer); \
+            AssertMsg(   pCritSect \
+                      && (   PDMCritSectIsOwner(pCritSect) \
+                          || (enmState = (pTimer)->CTX_SUFF(pVM)->enmVMState) == VMSTATE_CREATING \
+                          || enmState == VMSTATE_RESETTING \
+                          || enmState == VMSTATE_RESETTING_LS ),\
+                      ("pTimer=%p (%s) pCritSect=%p (%s)\n", pTimer, R3STRING(pTimer->pszDesc), \
+                       (pTimer)->pCritSect, R3STRING(PDMR3CritSectName((pTimer)->pCritSect)) )); \
+        } \
+    } while (0)
+#else
+# define TMTIMER_ASSERT_CRITSECT(pTimer) do { } while (0)
+#endif
+
+/** @def TMTIMER_ASSERT_SYNC_CRITSECT_ORDER
+ * Checks for lock order trouble between the timer critsect and the critical
+ * section critsect.  The virtual sync critsect must always be entered before
+ * the one associated with the timer (see TMR3TimerQueuesDo).  It is OK if there
+ * isn't any critical section associated with the timer or if the calling thread
+ * doesn't own it, ASSUMING of course that the thread using this macro is going
+ * to enter the virtual sync critical section anyway.
+ *
+ * @remarks This is a sligtly relaxed timer locking attitude compared to
+ *          TMTIMER_ASSERT_CRITSECT, however, the calling device/whatever code
+ *          should know what it's doing if it's stopping or starting a timer
+ *          without taking the device lock.
+ */
+#ifdef VBOX_STRICT
+# define TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer) \
+    do { \
+        if ((pTimer)->pCritSect) \
+        { \
+            VMSTATE      enmState; \
+            PPDMCRITSECT pCritSect = TMTIMER_GET_CRITSECT(pTimer); \
+            AssertMsg(   pCritSect \
+                      && (   !PDMCritSectIsOwner(pCritSect) \
+                          || PDMCritSectIsOwner(&pVM->tm.s.VirtualSyncLock) \
+                          || (enmState = (pVM)->enmVMState) == VMSTATE_CREATING \
+                          || enmState == VMSTATE_RESETTING \
+                          || enmState == VMSTATE_RESETTING_LS ),\
+                      ("pTimer=%p (%s) pCritSect=%p (%s)\n", pTimer, R3STRING(pTimer->pszDesc), \
+                       (pTimer)->pCritSect, R3STRING(PDMR3CritSectName((pTimer)->pCritSect)) )); \
+        } \
+    } while (0)
+#else
+# define TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer) do { } while (0)
+#endif
+
+
+#if defined(VBOX_STRICT) && defined(IN_RING0)
+/**
+ * Helper for  TMTIMER_GET_CRITSECT
+ * @todo This needs a redo!
+ */
+DECLINLINE(PPDMCRITSECT) tmRZTimerGetCritSect(PTMTIMER pTimer)
+{
+    if (pTimer->enmType == TMTIMERTYPE_DEV)
+    {
+        RTCCUINTREG fSavedFlags = ASMAddFlags(X86_EFL_AC); /** @todo fix ring-3 pointer use */
+        PPDMDEVINSR0        pDevInsR0 = ((struct PDMDEVINSR3 *)pTimer->u.Dev.pDevIns)->pDevInsR0RemoveMe; /* !ring-3 read! */
+        ASMSetFlags(fSavedFlags);
+        struct PDMDEVINSR3 *pDevInsR3 = pDevInsR0->pDevInsForR3R0;
+        if (pTimer->pCritSect == pDevInsR3->pCritSectRoR3)
+            return pDevInsR0->pCritSectRoR0;
+        uintptr_t offCritSect = (uintptr_t)pTimer->pCritSect - (uintptr_t)pDevInsR3->pvInstanceDataR3;
+        if (offCritSect < pDevInsR0->pReg->cbInstanceShared)
+            return (PPDMCRITSECT)((uintptr_t)pDevInsR0->pvInstanceDataR0 + offCritSect);
+    }
+    return (PPDMCRITSECT)MMHyperR3ToCC((pTimer)->CTX_SUFF(pVM), pTimer->pCritSect);
+}
+#endif /* VBOX_STRICT && IN_RING0 */
+
+
+/**
+ * Notification that execution is about to start.
+ *
+ * This call must always be paired with a TMNotifyEndOfExecution call.
+ *
+ * The function may, depending on the configuration, resume the TSC and future
+ * clocks that only ticks when we're executing guest code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(void) TMNotifyStartOfExecution(PVMCC pVM, PVMCPUCC pVCpu)
+{
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    pVCpu->tm.s.u64NsTsStartExecuting = RTTimeNanoTS();
+#endif
+    if (pVM->tm.s.fTSCTiedToExecution)
+        tmCpuTickResume(pVM, pVCpu);
+}
+
+
+/**
+ * Notification that execution has ended.
+ *
+ * This call must always be paired with a TMNotifyStartOfExecution call.
+ *
+ * The function may, depending on the configuration, suspend the TSC and future
+ * clocks that only ticks when we're executing guest code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(void) TMNotifyEndOfExecution(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    if (pVM->tm.s.fTSCTiedToExecution)
+        tmCpuTickPause(pVCpu);
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    uint64_t const u64NsTs           = RTTimeNanoTS();
+    uint64_t const cNsTotalNew       = u64NsTs - pVCpu->tm.s.u64NsTsStartTotal;
+    uint64_t const cNsExecutingDelta = u64NsTs - pVCpu->tm.s.u64NsTsStartExecuting;
+    uint64_t const cNsExecutingNew   = pVCpu->tm.s.cNsExecuting + cNsExecutingDelta;
+    uint64_t const cNsOtherNew       = cNsTotalNew - cNsExecutingNew - pVCpu->tm.s.cNsHalted;
+
+# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS)
+    STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsExecuting, cNsExecutingDelta);
+    if (cNsExecutingDelta < 5000)
+        STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsExecTiny, cNsExecutingDelta);
+    else if (cNsExecutingDelta < 50000)
+        STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsExecShort, cNsExecutingDelta);
+    else
+        STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsExecLong, cNsExecutingDelta);
+    STAM_REL_COUNTER_ADD(&pVCpu->tm.s.StatNsTotal, cNsTotalNew - pVCpu->tm.s.cNsTotal);
+    int64_t  const cNsOtherNewDelta  = cNsOtherNew - pVCpu->tm.s.cNsOther;
+    if (cNsOtherNewDelta > 0)
+        STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsOther, cNsOtherNewDelta); /* (the period before execution) */
+# endif
+
+    uint32_t uGen = ASMAtomicIncU32(&pVCpu->tm.s.uTimesGen); Assert(uGen & 1);
+    pVCpu->tm.s.cNsExecuting = cNsExecutingNew;
+    pVCpu->tm.s.cNsTotal     = cNsTotalNew;
+    pVCpu->tm.s.cNsOther     = cNsOtherNew;
+    pVCpu->tm.s.cPeriodsExecuting++;
+    ASMAtomicWriteU32(&pVCpu->tm.s.uTimesGen, (uGen | 1) + 1);
+#endif
+}
+
+
+/**
+ * Notification that the cpu is entering the halt state
+ *
+ * This call must always be paired with a TMNotifyEndOfExecution call.
+ *
+ * The function may, depending on the configuration, resume the TSC and future
+ * clocks that only ticks when we're halted.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(void) TMNotifyStartOfHalt(PVMCPUCC pVCpu)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    pVCpu->tm.s.u64NsTsStartHalting = RTTimeNanoTS();
+#endif
+
+    if (    pVM->tm.s.fTSCTiedToExecution
+        &&  !pVM->tm.s.fTSCNotTiedToHalt)
+        tmCpuTickResume(pVM, pVCpu);
+}
+
+
+/**
+ * Notification that the cpu is leaving the halt state
+ *
+ * This call must always be paired with a TMNotifyStartOfHalt call.
+ *
+ * The function may, depending on the configuration, suspend the TSC and future
+ * clocks that only ticks when we're halted.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(void) TMNotifyEndOfHalt(PVMCPUCC pVCpu)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    if (    pVM->tm.s.fTSCTiedToExecution
+        &&  !pVM->tm.s.fTSCNotTiedToHalt)
+        tmCpuTickPause(pVCpu);
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    uint64_t const u64NsTs        = RTTimeNanoTS();
+    uint64_t const cNsTotalNew    = u64NsTs - pVCpu->tm.s.u64NsTsStartTotal;
+    uint64_t const cNsHaltedDelta = u64NsTs - pVCpu->tm.s.u64NsTsStartHalting;
+    uint64_t const cNsHaltedNew   = pVCpu->tm.s.cNsHalted + cNsHaltedDelta;
+    uint64_t const cNsOtherNew    = cNsTotalNew - pVCpu->tm.s.cNsExecuting - cNsHaltedNew;
+
+# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS)
+    STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsHalted, cNsHaltedDelta);
+    STAM_REL_COUNTER_ADD(&pVCpu->tm.s.StatNsTotal, cNsTotalNew - pVCpu->tm.s.cNsTotal);
+    int64_t  const cNsOtherNewDelta  = cNsOtherNew - pVCpu->tm.s.cNsOther;
+    if (cNsOtherNewDelta > 0)
+        STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsOther, cNsOtherNewDelta); /* (the period before halting) */
+# endif
+
+    uint32_t uGen = ASMAtomicIncU32(&pVCpu->tm.s.uTimesGen); Assert(uGen & 1);
+    pVCpu->tm.s.cNsHalted = cNsHaltedNew;
+    pVCpu->tm.s.cNsTotal  = cNsTotalNew;
+    pVCpu->tm.s.cNsOther  = cNsOtherNew;
+    pVCpu->tm.s.cPeriodsHalted++;
+    ASMAtomicWriteU32(&pVCpu->tm.s.uTimesGen, (uGen | 1) + 1);
+#endif
+}
+
+
+/**
+ * Raise the timer force action flag and notify the dedicated timer EMT.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+DECLINLINE(void) tmScheduleNotify(PVMCC pVM)
+{
+    PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu);
+    if (!VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+    {
+        Log5(("TMAll(%u): FF: 0 -> 1\n", __LINE__));
+        VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+#ifdef IN_RING3
+        VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM);
+#endif
+        STAM_COUNTER_INC(&pVM->tm.s.StatScheduleSetFF);
+    }
+}
+
+
+/**
+ * Schedule the queue which was changed.
+ */
+DECLINLINE(void) tmSchedule(PTMTIMER pTimer)
+{
+    PVMCC pVM = pTimer->CTX_SUFF(pVM);
+    if (    VM_IS_EMT(pVM)
+        &&  RT_SUCCESS(TM_TRY_LOCK_TIMERS(pVM)))
+    {
+        STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a);
+        Log3(("tmSchedule: tmTimerQueueSchedule\n"));
+        tmTimerQueueSchedule(pVM, &pVM->tm.s.CTX_SUFF(paTimerQueues)[pTimer->enmClock]);
+#ifdef VBOX_STRICT
+        tmTimerQueuesSanityChecks(pVM, "tmSchedule");
+#endif
+        STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a);
+        TM_UNLOCK_TIMERS(pVM);
+    }
+    else
+    {
+        TMTIMERSTATE enmState = pTimer->enmState;
+        if (TMTIMERSTATE_IS_PENDING_SCHEDULING(enmState))
+            tmScheduleNotify(pVM);
+    }
+}
+
+
+/**
+ * Try change the state to enmStateNew from enmStateOld
+ * and link the timer into the scheduling queue.
+ *
+ * @returns Success indicator.
+ * @param   pTimer          Timer in question.
+ * @param   enmStateNew     The new timer state.
+ * @param   enmStateOld     The old timer state.
+ */
+DECLINLINE(bool) tmTimerTry(PTMTIMER pTimer, TMTIMERSTATE enmStateNew, TMTIMERSTATE enmStateOld)
+{
+    /*
+     * Attempt state change.
+     */
+    bool fRc;
+    TM_TRY_SET_STATE(pTimer, enmStateNew, enmStateOld, fRc);
+    return fRc;
+}
+
+
+/**
+ * Links the timer onto the scheduling queue.
+ *
+ * @param   pQueue  The timer queue the timer belongs to.
+ * @param   pTimer  The timer.
+ *
+ * @todo    FIXME: Look into potential race with the thread running the queues
+ *          and stuff.
+ */
+DECLINLINE(void) tmTimerLinkSchedule(PTMTIMERQUEUE pQueue, PTMTIMER pTimer)
+{
+    Assert(!pTimer->offScheduleNext);
+    const int32_t offHeadNew = (intptr_t)pTimer - (intptr_t)pQueue;
+    int32_t offHead;
+    do
+    {
+        offHead = pQueue->offSchedule;
+        if (offHead)
+            pTimer->offScheduleNext = ((intptr_t)pQueue + offHead) - (intptr_t)pTimer;
+        else
+            pTimer->offScheduleNext = 0;
+    } while (!ASMAtomicCmpXchgS32(&pQueue->offSchedule, offHeadNew, offHead));
+}
+
+
+/**
+ * Try change the state to enmStateNew from enmStateOld
+ * and link the timer into the scheduling queue.
+ *
+ * @returns Success indicator.
+ * @param   pTimer          Timer in question.
+ * @param   enmStateNew     The new timer state.
+ * @param   enmStateOld     The old timer state.
+ */
+DECLINLINE(bool) tmTimerTryWithLink(PTMTIMER pTimer, TMTIMERSTATE enmStateNew, TMTIMERSTATE enmStateOld)
+{
+    if (tmTimerTry(pTimer, enmStateNew, enmStateOld))
+    {
+        tmTimerLinkSchedule(&pTimer->CTX_SUFF(pVM)->tm.s.CTX_SUFF(paTimerQueues)[pTimer->enmClock], pTimer);
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Links a timer into the active list of a timer queue.
+ *
+ * @param   pQueue          The queue.
+ * @param   pTimer          The timer.
+ * @param   u64Expire       The timer expiration time.
+ *
+ * @remarks Called while owning the relevant queue lock.
+ */
+DECL_FORCE_INLINE(void) tmTimerQueueLinkActive(PTMTIMERQUEUE pQueue, PTMTIMER pTimer, uint64_t u64Expire)
+{
+    Assert(!pTimer->offNext);
+    Assert(!pTimer->offPrev);
+    Assert(pTimer->enmState == TMTIMERSTATE_ACTIVE || pTimer->enmClock != TMCLOCK_VIRTUAL_SYNC); /* (active is not a stable state) */
+
+    PTMTIMER pCur = TMTIMER_GET_HEAD(pQueue);
+    if (pCur)
+    {
+        for (;; pCur = TMTIMER_GET_NEXT(pCur))
+        {
+            if (pCur->u64Expire > u64Expire)
+            {
+                const PTMTIMER pPrev = TMTIMER_GET_PREV(pCur);
+                TMTIMER_SET_NEXT(pTimer, pCur);
+                TMTIMER_SET_PREV(pTimer, pPrev);
+                if (pPrev)
+                    TMTIMER_SET_NEXT(pPrev, pTimer);
+                else
+                {
+                    TMTIMER_SET_HEAD(pQueue, pTimer);
+                    ASMAtomicWriteU64(&pQueue->u64Expire, u64Expire);
+                    DBGFTRACE_U64_TAG2(pTimer->CTX_SUFF(pVM), u64Expire, "tmTimerQueueLinkActive head", R3STRING(pTimer->pszDesc));
+                }
+                TMTIMER_SET_PREV(pCur, pTimer);
+                return;
+            }
+            if (!pCur->offNext)
+            {
+                TMTIMER_SET_NEXT(pCur, pTimer);
+                TMTIMER_SET_PREV(pTimer, pCur);
+                DBGFTRACE_U64_TAG2(pTimer->CTX_SUFF(pVM), u64Expire, "tmTimerQueueLinkActive tail", R3STRING(pTimer->pszDesc));
+                return;
+            }
+        }
+    }
+    else
+    {
+        TMTIMER_SET_HEAD(pQueue, pTimer);
+        ASMAtomicWriteU64(&pQueue->u64Expire, u64Expire);
+        DBGFTRACE_U64_TAG2(pTimer->CTX_SUFF(pVM), u64Expire, "tmTimerQueueLinkActive empty", R3STRING(pTimer->pszDesc));
+    }
+}
+
+
+
+/**
+ * Schedules the given timer on the given queue.
+ *
+ * @param   pQueue      The timer queue.
+ * @param   pTimer      The timer that needs scheduling.
+ *
+ * @remarks Called while owning the lock.
+ */
+DECLINLINE(void) tmTimerQueueScheduleOne(PTMTIMERQUEUE pQueue, PTMTIMER pTimer)
+{
+    Assert(pQueue->enmClock != TMCLOCK_VIRTUAL_SYNC);
+
+    /*
+     * Processing.
+     */
+    unsigned cRetries = 2;
+    do
+    {
+        TMTIMERSTATE enmState = pTimer->enmState;
+        switch (enmState)
+        {
+            /*
+             * Reschedule timer (in the active list).
+             */
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+                if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_PENDING_SCHEDULE, TMTIMERSTATE_PENDING_RESCHEDULE)))
+                    break; /* retry */
+                tmTimerQueueUnlinkActive(pQueue, pTimer);
+                RT_FALL_THRU();
+
+            /*
+             * Schedule timer (insert into the active list).
+             */
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+                Assert(!pTimer->offNext); Assert(!pTimer->offPrev);
+                if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_ACTIVE, TMTIMERSTATE_PENDING_SCHEDULE)))
+                    break; /* retry */
+                tmTimerQueueLinkActive(pQueue, pTimer, pTimer->u64Expire);
+                return;
+
+            /*
+             * Stop the timer in active list.
+             */
+            case TMTIMERSTATE_PENDING_STOP:
+                if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_PENDING_STOP_SCHEDULE, TMTIMERSTATE_PENDING_STOP)))
+                    break; /* retry */
+                tmTimerQueueUnlinkActive(pQueue, pTimer);
+                RT_FALL_THRU();
+
+            /*
+             * Stop the timer (not on the active list).
+             */
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                Assert(!pTimer->offNext); Assert(!pTimer->offPrev);
+                if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_STOPPED, TMTIMERSTATE_PENDING_STOP_SCHEDULE)))
+                    break;
+                return;
+
+            /*
+             * The timer is pending destruction by TMR3TimerDestroy, our caller.
+             * Nothing to do here.
+             */
+            case TMTIMERSTATE_DESTROY:
+                break;
+
+            /*
+             * Postpone these until they get into the right state.
+             */
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+                tmTimerLinkSchedule(pQueue, pTimer);
+                STAM_COUNTER_INC(&pTimer->CTX_SUFF(pVM)->tm.s.CTX_SUFF_Z(StatPostponed));
+                return;
+
+            /*
+             * None of these can be in the schedule.
+             */
+            case TMTIMERSTATE_FREE:
+            case TMTIMERSTATE_STOPPED:
+            case TMTIMERSTATE_ACTIVE:
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+            default:
+                AssertMsgFailed(("Timer (%p) in the scheduling list has an invalid state %s (%d)!",
+                                 pTimer, tmTimerState(pTimer->enmState), pTimer->enmState));
+                return;
+        }
+    } while (cRetries-- > 0);
+}
+
+
+/**
+ * Schedules the specified timer queue.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pQueue          The queue to schedule.
+ *
+ * @remarks Called while owning the lock.
+ */
+void tmTimerQueueSchedule(PVM pVM, PTMTIMERQUEUE pQueue)
+{
+    TM_ASSERT_TIMER_LOCK_OWNERSHIP(pVM);
+    NOREF(pVM);
+
+    /*
+     * Dequeue the scheduling list and iterate it.
+     */
+    int32_t offNext = ASMAtomicXchgS32(&pQueue->offSchedule, 0);
+    Log2(("tmTimerQueueSchedule: pQueue=%p:{.enmClock=%d, offNext=%RI32, .u64Expired=%'RU64}\n", pQueue, pQueue->enmClock, offNext, pQueue->u64Expire));
+    if (!offNext)
+        return;
+    PTMTIMER pNext = (PTMTIMER)((intptr_t)pQueue + offNext);
+    while (pNext)
+    {
+        /*
+         * Unlink the head timer and find the next one.
+         */
+        PTMTIMER pTimer = pNext;
+        pNext = pNext->offScheduleNext ? (PTMTIMER)((intptr_t)pNext + pNext->offScheduleNext) : NULL;
+        pTimer->offScheduleNext = 0;
+
+        /*
+         * Do the scheduling.
+         */
+        Log2(("tmTimerQueueSchedule: %p:{.enmState=%s, .enmClock=%d, .enmType=%d, .pszDesc=%s}\n",
+              pTimer, tmTimerState(pTimer->enmState), pTimer->enmClock, pTimer->enmType, R3STRING(pTimer->pszDesc)));
+        tmTimerQueueScheduleOne(pQueue, pTimer);
+        Log2(("tmTimerQueueSchedule: %p: new %s\n", pTimer, tmTimerState(pTimer->enmState)));
+    } /* foreach timer in current schedule batch. */
+    Log2(("tmTimerQueueSchedule: u64Expired=%'RU64\n", pQueue->u64Expire));
+}
+
+
+#ifdef VBOX_STRICT
+/**
+ * Checks that the timer queues are sane.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pszWhere    Caller location clue.
+ *
+ * @remarks Called while owning the lock.
+ */
+void tmTimerQueuesSanityChecks(PVM pVM, const char *pszWhere)
+{
+    TM_ASSERT_TIMER_LOCK_OWNERSHIP(pVM);
+
+    /*
+     * Check the linking of the active lists.
+     */
+    bool fHaveVirtualSyncLock = false;
+    for (int i = 0; i < TMCLOCK_MAX; i++)
+    {
+        PTMTIMERQUEUE pQueue = &pVM->tm.s.CTX_SUFF(paTimerQueues)[i];
+        Assert((int)pQueue->enmClock == i);
+        if (pQueue->enmClock == TMCLOCK_VIRTUAL_SYNC)
+        {
+            if (PDMCritSectTryEnter(&pVM->tm.s.VirtualSyncLock) != VINF_SUCCESS)
+                continue;
+            fHaveVirtualSyncLock = true;
+        }
+        PTMTIMER pPrev = NULL;
+        for (PTMTIMER pCur = TMTIMER_GET_HEAD(pQueue); pCur; pPrev = pCur, pCur = TMTIMER_GET_NEXT(pCur))
+        {
+            AssertMsg((int)pCur->enmClock == i, ("%s: %d != %d\n", pszWhere, pCur->enmClock, i));
+            AssertMsg(TMTIMER_GET_PREV(pCur) == pPrev, ("%s: %p != %p\n", pszWhere, TMTIMER_GET_PREV(pCur), pPrev));
+            TMTIMERSTATE enmState = pCur->enmState;
+            switch (enmState)
+            {
+                case TMTIMERSTATE_ACTIVE:
+                    AssertMsg(  !pCur->offScheduleNext
+                              || pCur->enmState != TMTIMERSTATE_ACTIVE,
+                              ("%s: %RI32\n", pszWhere, pCur->offScheduleNext));
+                    break;
+                case TMTIMERSTATE_PENDING_STOP:
+                case TMTIMERSTATE_PENDING_RESCHEDULE:
+                case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+                    break;
+                default:
+                    AssertMsgFailed(("%s: Invalid state enmState=%d %s\n", pszWhere, enmState, tmTimerState(enmState)));
+                    break;
+            }
+        }
+    }
+
+
+# ifdef IN_RING3
+    /*
+     * Do the big list and check that active timers all are in the active lists.
+     */
+    PTMTIMERR3 pPrev = NULL;
+    for (PTMTIMERR3 pCur = pVM->tm.s.pCreated; pCur; pPrev = pCur, pCur = pCur->pBigNext)
+    {
+        Assert(pCur->pBigPrev == pPrev);
+        Assert((unsigned)pCur->enmClock < (unsigned)TMCLOCK_MAX);
+
+        TMTIMERSTATE enmState = pCur->enmState;
+        switch (enmState)
+        {
+            case TMTIMERSTATE_ACTIVE:
+            case TMTIMERSTATE_PENDING_STOP:
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+                if (fHaveVirtualSyncLock || pCur->enmClock != TMCLOCK_VIRTUAL_SYNC)
+                {
+                    PTMTIMERR3 pCurAct = TMTIMER_GET_HEAD(&pVM->tm.s.CTX_SUFF(paTimerQueues)[pCur->enmClock]);
+                    Assert(pCur->offPrev || pCur == pCurAct);
+                    while (pCurAct && pCurAct != pCur)
+                        pCurAct = TMTIMER_GET_NEXT(pCurAct);
+                    Assert(pCurAct == pCur);
+                }
+                break;
+
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+            case TMTIMERSTATE_STOPPED:
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+                if (fHaveVirtualSyncLock || pCur->enmClock != TMCLOCK_VIRTUAL_SYNC)
+                {
+                    Assert(!pCur->offNext);
+                    Assert(!pCur->offPrev);
+                    for (PTMTIMERR3 pCurAct = TMTIMER_GET_HEAD(&pVM->tm.s.CTX_SUFF(paTimerQueues)[pCur->enmClock]);
+                          pCurAct;
+                          pCurAct = TMTIMER_GET_NEXT(pCurAct))
+                    {
+                        Assert(pCurAct != pCur);
+                        Assert(TMTIMER_GET_NEXT(pCurAct) != pCur);
+                        Assert(TMTIMER_GET_PREV(pCurAct) != pCur);
+                    }
+                }
+                break;
+
+            /* ignore */
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+                break;
+
+            /* shouldn't get here! */
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_DESTROY:
+            default:
+                AssertMsgFailed(("Invalid state enmState=%d %s\n", enmState, tmTimerState(enmState)));
+                break;
+        }
+    }
+# endif /* IN_RING3 */
+
+    if (fHaveVirtualSyncLock)
+        PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+}
+#endif /* !VBOX_STRICT */
+
+#ifdef VBOX_HIGH_RES_TIMERS_HACK
+
+/**
+ * Worker for tmTimerPollInternal that handles misses when the dedicated timer
+ * EMT is polling.
+ *
+ * @returns See tmTimerPollInternal.
+ * @param   pVM                 The cross context VM structure.
+ * @param   u64Now              Current virtual clock timestamp.
+ * @param   u64Delta            The delta to the next even in ticks of the
+ *                              virtual clock.
+ * @param   pu64Delta           Where to return the delta.
+ */
+DECLINLINE(uint64_t) tmTimerPollReturnMiss(PVM pVM, uint64_t u64Now, uint64_t u64Delta, uint64_t *pu64Delta)
+{
+    Assert(!(u64Delta & RT_BIT_64(63)));
+
+    if (!pVM->tm.s.fVirtualWarpDrive)
+    {
+        *pu64Delta = u64Delta;
+        return u64Delta + u64Now + pVM->tm.s.u64VirtualOffset;
+    }
+
+    /*
+     * Warp drive adjustments - this is the reverse of what tmVirtualGetRaw is doing.
+     */
+    uint64_t const u64Start = pVM->tm.s.u64VirtualWarpDriveStart;
+    uint32_t const u32Pct   = pVM->tm.s.u32VirtualWarpDrivePercentage;
+
+    uint64_t u64GipTime = u64Delta + u64Now + pVM->tm.s.u64VirtualOffset;
+    u64GipTime -= u64Start; /* the start is GIP time. */
+    if (u64GipTime >= u64Delta)
+    {
+        ASMMultU64ByU32DivByU32(u64GipTime, 100, u32Pct);
+        ASMMultU64ByU32DivByU32(u64Delta, 100, u32Pct);
+    }
+    else
+    {
+        u64Delta -= u64GipTime;
+        ASMMultU64ByU32DivByU32(u64GipTime, 100, u32Pct);
+        u64Delta += u64GipTime;
+    }
+    *pu64Delta = u64Delta;
+    u64GipTime += u64Start;
+    return u64GipTime;
+}
+
+
+/**
+ * Worker for tmTimerPollInternal dealing with returns on virtual CPUs other
+ * than the one dedicated to timer work.
+ *
+ * @returns See tmTimerPollInternal.
+ * @param   pVM                 The cross context VM structure.
+ * @param   u64Now              Current virtual clock timestamp.
+ * @param   pu64Delta           Where to return the delta.
+ */
+DECL_FORCE_INLINE(uint64_t) tmTimerPollReturnOtherCpu(PVM pVM, uint64_t u64Now, uint64_t *pu64Delta)
+{
+    static const uint64_t s_u64OtherRet = 500000000; /* 500 ms for non-timer EMTs. */
+    *pu64Delta = s_u64OtherRet;
+    return u64Now + pVM->tm.s.u64VirtualOffset + s_u64OtherRet;
+}
+
+
+/**
+ * Worker for tmTimerPollInternal.
+ *
+ * @returns See tmTimerPollInternal.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pVCpuDst    The cross context virtual CPU structure of the dedicated
+ *                      timer EMT.
+ * @param   u64Now      Current virtual clock timestamp.
+ * @param   pu64Delta   Where to return the delta.
+ * @param   pCounter    The statistics counter to update.
+ */
+DECL_FORCE_INLINE(uint64_t) tmTimerPollReturnHit(PVM pVM, PVMCPU pVCpu, PVMCPU pVCpuDst, uint64_t u64Now,
+                                                 uint64_t *pu64Delta, PSTAMCOUNTER pCounter)
+{
+    STAM_COUNTER_INC(pCounter); NOREF(pCounter);
+    if (pVCpuDst != pVCpu)
+        return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta);
+    *pu64Delta = 0;
+    return 0;
+}
+
+/**
+ * Common worker for TMTimerPollGIP and TMTimerPoll.
+ *
+ * This function is called before FFs are checked in the inner execution EM loops.
+ *
+ * @returns The GIP timestamp of the next event.
+ *          0 if the next event has already expired.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pu64Delta   Where to store the delta.
+ *
+ * @thread  The emulation thread.
+ *
+ * @remarks GIP uses ns ticks.
+ */
+DECL_FORCE_INLINE(uint64_t) tmTimerPollInternal(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pu64Delta)
+{
+    PVMCPU                  pVCpuDst      = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu);
+    const uint64_t          u64Now        = TMVirtualGetNoCheck(pVM);
+    STAM_COUNTER_INC(&pVM->tm.s.StatPoll);
+
+    /*
+     * Return straight away if the timer FF is already set ...
+     */
+    if (VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+        return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollAlreadySet);
+
+    /*
+     * ... or if timers are being run.
+     */
+    if (ASMAtomicReadBool(&pVM->tm.s.fRunningQueues))
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatPollRunning);
+        return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta);
+    }
+
+    /*
+     * Check for TMCLOCK_VIRTUAL expiration.
+     */
+    const uint64_t  u64Expire1 = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL].u64Expire);
+    const int64_t   i64Delta1  = u64Expire1 - u64Now;
+    if (i64Delta1 <= 0)
+    {
+        if (!VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+        {
+            Log5(("TMAll(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+            VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+        }
+        LogFlow(("TMTimerPoll: expire1=%'RU64 <= now=%'RU64\n", u64Expire1, u64Now));
+        return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtual);
+    }
+
+    /*
+     * Check for TMCLOCK_VIRTUAL_SYNC expiration.
+     * This isn't quite as straight forward if in a catch-up, not only do
+     * we have to adjust the 'now' but when have to adjust the delta as well.
+     */
+
+    /*
+     * Optimistic lockless approach.
+     */
+    uint64_t u64VirtualSyncNow;
+    uint64_t u64Expire2 = ASMAtomicUoReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+    if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking))
+    {
+        if (!ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+        {
+            u64VirtualSyncNow = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync);
+            if (RT_LIKELY(   ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking)
+                          && !ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp)
+                          && u64VirtualSyncNow == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync)
+                          && u64Expire2 == ASMAtomicUoReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire)))
+            {
+                u64VirtualSyncNow = u64Now - u64VirtualSyncNow;
+                int64_t i64Delta2 = u64Expire2 - u64VirtualSyncNow;
+                if (i64Delta2 > 0)
+                {
+                    STAM_COUNTER_INC(&pVM->tm.s.StatPollSimple);
+                    STAM_COUNTER_INC(&pVM->tm.s.StatPollMiss);
+
+                    if (pVCpu == pVCpuDst)
+                        return tmTimerPollReturnMiss(pVM, u64Now, RT_MIN(i64Delta1, i64Delta2), pu64Delta);
+                    return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta);
+                }
+
+                if (    !pVM->tm.s.fRunningQueues
+                    &&  !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+                {
+                    Log5(("TMAll(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+                    VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+                }
+
+                STAM_COUNTER_INC(&pVM->tm.s.StatPollSimple);
+                LogFlow(("TMTimerPoll: expire2=%'RU64 <= now=%'RU64\n", u64Expire2, u64Now));
+                return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtualSync);
+            }
+        }
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatPollSimple);
+        LogFlow(("TMTimerPoll: stopped\n"));
+        return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtualSync);
+    }
+
+    /*
+     * Complicated lockless approach.
+     */
+    uint64_t    off;
+    uint32_t    u32Pct = 0;
+    bool        fCatchUp;
+    int         cOuterTries = 42;
+    for (;; cOuterTries--)
+    {
+        fCatchUp   = ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp);
+        off        = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync);
+        u64Expire2 = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+        if (fCatchUp)
+        {
+            /* No changes allowed, try get a consistent set of parameters. */
+            uint64_t const u64Prev    = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev);
+            uint64_t const offGivenUp = ASMAtomicReadU64(&pVM->tm.s.offVirtualSyncGivenUp);
+            u32Pct                    = ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage);
+            if (    (   u64Prev    == ASMAtomicReadU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev)
+                     && offGivenUp == ASMAtomicReadU64(&pVM->tm.s.offVirtualSyncGivenUp)
+                     && u32Pct     == ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage)
+                     && off        == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync)
+                     && u64Expire2 == ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire)
+                     && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp)
+                     && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking))
+                ||  cOuterTries <= 0)
+            {
+                uint64_t u64Delta = u64Now - u64Prev;
+                if (RT_LIKELY(!(u64Delta >> 32)))
+                {
+                    uint64_t u64Sub = ASMMultU64ByU32DivByU32(u64Delta, u32Pct, 100);
+                    if (off > u64Sub + offGivenUp)
+                        off -= u64Sub;
+                    else /* we've completely caught up. */
+                        off = offGivenUp;
+                }
+                else
+                    /* More than 4 seconds since last time (or negative), ignore it. */
+                    Log(("TMVirtualGetSync: u64Delta=%RX64 (NoLock)\n", u64Delta));
+
+                /* Check that we're still running and in catch up. */
+                if (    ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking)
+                    &&  ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+                    break;
+            }
+        }
+        else if (   off        == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync)
+                 && u64Expire2 == ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire)
+                 && !ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp)
+                 && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking))
+            break; /* Got an consistent offset */
+
+        /* Repeat the initial checks before iterating. */
+        if (VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+            return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollAlreadySet);
+        if (ASMAtomicUoReadBool(&pVM->tm.s.fRunningQueues))
+        {
+            STAM_COUNTER_INC(&pVM->tm.s.StatPollRunning);
+            return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta);
+        }
+        if (!ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking))
+        {
+            LogFlow(("TMTimerPoll: stopped\n"));
+            return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtualSync);
+        }
+        if (cOuterTries <= 0)
+            break; /* that's enough */
+    }
+    if (cOuterTries <= 0)
+        STAM_COUNTER_INC(&pVM->tm.s.StatPollELoop);
+    u64VirtualSyncNow = u64Now - off;
+
+    /* Calc delta and see if we've got a virtual sync hit. */
+    int64_t i64Delta2 = u64Expire2 - u64VirtualSyncNow;
+    if (i64Delta2 <= 0)
+    {
+        if (    !pVM->tm.s.fRunningQueues
+            &&  !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+        {
+            Log5(("TMAll(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+            VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+        }
+        STAM_COUNTER_INC(&pVM->tm.s.StatPollVirtualSync);
+        LogFlow(("TMTimerPoll: expire2=%'RU64 <= now=%'RU64\n", u64Expire2, u64Now));
+        return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtualSync);
+    }
+
+    /*
+     * Return the time left to the next event.
+     */
+    STAM_COUNTER_INC(&pVM->tm.s.StatPollMiss);
+    if (pVCpu == pVCpuDst)
+    {
+        if (fCatchUp)
+            i64Delta2 = ASMMultU64ByU32DivByU32(i64Delta2, 100, u32Pct + 100);
+        return tmTimerPollReturnMiss(pVM, u64Now, RT_MIN(i64Delta1, i64Delta2), pu64Delta);
+    }
+    return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta);
+}
+
+
+/**
+ * Set FF if we've passed the next virtual event.
+ *
+ * This function is called before FFs are checked in the inner execution EM loops.
+ *
+ * @returns true if timers are pending, false if not.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @thread  The emulation thread.
+ */
+VMMDECL(bool) TMTimerPollBool(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    uint64_t off = 0;
+    tmTimerPollInternal(pVM, pVCpu, &off);
+    return off == 0;
+}
+
+
+/**
+ * Set FF if we've passed the next virtual event.
+ *
+ * This function is called before FFs are checked in the inner execution EM loops.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @thread  The emulation thread.
+ */
+VMM_INT_DECL(void) TMTimerPollVoid(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    uint64_t off;
+    tmTimerPollInternal(pVM, pVCpu, &off);
+}
+
+
+/**
+ * Set FF if we've passed the next virtual event.
+ *
+ * This function is called before FFs are checked in the inner execution EM loops.
+ *
+ * @returns The GIP timestamp of the next event.
+ *          0 if the next event has already expired.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pu64Delta   Where to store the delta.
+ * @thread  The emulation thread.
+ */
+VMM_INT_DECL(uint64_t) TMTimerPollGIP(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pu64Delta)
+{
+    return tmTimerPollInternal(pVM, pVCpu, pu64Delta);
+}
+
+#endif /* VBOX_HIGH_RES_TIMERS_HACK */
+
+/**
+ * Gets the host context ring-3 pointer of the timer.
+ *
+ * @returns HC R3 pointer.
+ * @param   pTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(PTMTIMERR3) TMTimerR3Ptr(PTMTIMER pTimer)
+{
+    return (PTMTIMERR3)MMHyperCCToR3(pTimer->CTX_SUFF(pVM), pTimer);
+}
+
+
+/**
+ * Gets the host context ring-0 pointer of the timer.
+ *
+ * @returns HC R0 pointer.
+ * @param   pTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(PTMTIMERR0) TMTimerR0Ptr(PTMTIMER pTimer)
+{
+    return (PTMTIMERR0)MMHyperCCToR0(pTimer->CTX_SUFF(pVM), pTimer);
+}
+
+
+/**
+ * Gets the RC pointer of the timer.
+ *
+ * @returns RC pointer.
+ * @param   pTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(PTMTIMERRC) TMTimerRCPtr(PTMTIMER pTimer)
+{
+    return (PTMTIMERRC)MMHyperCCToRC(pTimer->CTX_SUFF(pVM), pTimer);
+}
+
+
+/**
+ * Locks the timer clock.
+ *
+ * @returns VINF_SUCCESS on success, @a rcBusy if busy, and VERR_NOT_SUPPORTED
+ *          if the clock does not have a lock.
+ * @param   pTimer              The timer which clock lock we wish to take.
+ * @param   rcBusy              What to return in ring-0 and raw-mode context
+ *                              if the lock is busy.  Pass VINF_SUCCESS to
+ *                              acquired the critical section thru a ring-3
+                                call if necessary.
+ *
+ * @remarks Currently only supported on timers using the virtual sync clock.
+ */
+VMMDECL(int) TMTimerLock(PTMTIMER pTimer, int rcBusy)
+{
+    AssertPtr(pTimer);
+    AssertReturn(pTimer->enmClock == TMCLOCK_VIRTUAL_SYNC, VERR_NOT_SUPPORTED);
+    return PDMCritSectEnter(&pTimer->CTX_SUFF(pVM)->tm.s.VirtualSyncLock, rcBusy);
+}
+
+
+/**
+ * Unlocks a timer clock locked by TMTimerLock.
+ *
+ * @param   pTimer              The timer which clock to unlock.
+ */
+VMMDECL(void) TMTimerUnlock(PTMTIMER pTimer)
+{
+    AssertPtr(pTimer);
+    AssertReturnVoid(pTimer->enmClock == TMCLOCK_VIRTUAL_SYNC);
+    PDMCritSectLeave(&pTimer->CTX_SUFF(pVM)->tm.s.VirtualSyncLock);
+}
+
+
+/**
+ * Checks if the current thread owns the timer clock lock.
+ *
+ * @returns @c true if its the owner, @c false if not.
+ * @param   pTimer              The timer handle.
+ */
+VMMDECL(bool) TMTimerIsLockOwner(PTMTIMER pTimer)
+{
+    AssertPtr(pTimer);
+    AssertReturn(pTimer->enmClock == TMCLOCK_VIRTUAL_SYNC, false);
+    return PDMCritSectIsOwner(&pTimer->CTX_SUFF(pVM)->tm.s.VirtualSyncLock);
+}
+
+
+/**
+ * Optimized TMTimerSet code path for starting an inactive timer.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pTimer          The timer handle.
+ * @param   u64Expire       The new expire time.
+ */
+static int tmTimerSetOptimizedStart(PVM pVM, PTMTIMER pTimer, uint64_t u64Expire)
+{
+    Assert(!pTimer->offPrev);
+    Assert(!pTimer->offNext);
+    Assert(pTimer->enmState == TMTIMERSTATE_ACTIVE);
+
+    TMCLOCK const enmClock = pTimer->enmClock;
+
+    /*
+     * Calculate and set the expiration time.
+     */
+    if (enmClock == TMCLOCK_VIRTUAL_SYNC)
+    {
+        uint64_t u64Last = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSync);
+        AssertMsgStmt(u64Expire >= u64Last,
+                      ("exp=%#llx last=%#llx\n", u64Expire, u64Last),
+                      u64Expire = u64Last);
+    }
+    ASMAtomicWriteU64(&pTimer->u64Expire, u64Expire);
+    Log2(("tmTimerSetOptimizedStart: %p:{.pszDesc='%s', .u64Expire=%'RU64}\n", pTimer, R3STRING(pTimer->pszDesc), u64Expire));
+
+    /*
+     * Link the timer into the active list.
+     */
+    tmTimerQueueLinkActive(&pVM->tm.s.CTX_SUFF(paTimerQueues)[enmClock], pTimer, u64Expire);
+
+    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetOpt);
+    TM_UNLOCK_TIMERS(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * TMTimerSet for the virtual sync timer queue.
+ *
+ * This employs a greatly simplified state machine by always acquiring the
+ * queue lock and bypassing the scheduling list.
+ *
+ * @returns VBox status code
+ * @param   pVM                 The cross context VM structure.
+ * @param   pTimer              The timer handle.
+ * @param   u64Expire           The expiration time.
+ */
+static int tmTimerVirtualSyncSet(PVMCC pVM, PTMTIMER pTimer, uint64_t u64Expire)
+{
+    STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetVs), a);
+    VM_ASSERT_EMT(pVM);
+    TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer);
+    int rc = PDMCritSectEnter(&pVM->tm.s.VirtualSyncLock, VINF_SUCCESS);
+    AssertRCReturn(rc, rc);
+
+    PTMTIMERQUEUE   pQueue   = &pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC];
+    TMTIMERSTATE    enmState = pTimer->enmState;
+    switch (enmState)
+    {
+        case TMTIMERSTATE_EXPIRED_DELIVER:
+        case TMTIMERSTATE_STOPPED:
+            if (enmState == TMTIMERSTATE_EXPIRED_DELIVER)
+                STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetVsStExpDeliver);
+            else
+                STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetVsStStopped);
+
+            AssertMsg(u64Expire >= pVM->tm.s.u64VirtualSync,
+                      ("%'RU64 < %'RU64 %s\n", u64Expire, pVM->tm.s.u64VirtualSync, R3STRING(pTimer->pszDesc)));
+            pTimer->u64Expire = u64Expire;
+            TM_SET_STATE(pTimer, TMTIMERSTATE_ACTIVE);
+            tmTimerQueueLinkActive(pQueue, pTimer, u64Expire);
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_ACTIVE:
+            STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetVsStActive);
+            tmTimerQueueUnlinkActive(pQueue, pTimer);
+            pTimer->u64Expire = u64Expire;
+            tmTimerQueueLinkActive(pQueue, pTimer, u64Expire);
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_PENDING_RESCHEDULE:
+        case TMTIMERSTATE_PENDING_STOP:
+        case TMTIMERSTATE_PENDING_SCHEDULE:
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+        case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+        case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_DESTROY:
+        case TMTIMERSTATE_FREE:
+            AssertLogRelMsgFailed(("Invalid timer state %s: %s\n", tmTimerState(enmState), R3STRING(pTimer->pszDesc)));
+            rc = VERR_TM_INVALID_STATE;
+            break;
+
+        default:
+            AssertMsgFailed(("Unknown timer state %d: %s\n", enmState, R3STRING(pTimer->pszDesc)));
+            rc = VERR_TM_UNKNOWN_STATE;
+            break;
+    }
+
+    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetVs), a);
+    PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+    return rc;
+}
+
+
+/**
+ * Arm a timer with a (new) expire time.
+ *
+ * @returns VBox status code.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ * @param   u64Expire       New expire time.
+ */
+VMMDECL(int) TMTimerSet(PTMTIMER pTimer, uint64_t u64Expire)
+{
+    PVMCC pVM = pTimer->CTX_SUFF(pVM);
+    STAM_COUNTER_INC(&pTimer->StatSetAbsolute);
+
+    /* Treat virtual sync timers specially. */
+    if (pTimer->enmClock == TMCLOCK_VIRTUAL_SYNC)
+        return tmTimerVirtualSyncSet(pVM, pTimer, u64Expire);
+
+    STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+    TMTIMER_ASSERT_CRITSECT(pTimer);
+
+    DBGFTRACE_U64_TAG2(pVM, u64Expire, "TMTimerSet", R3STRING(pTimer->pszDesc));
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Gather optimization info.
+     */
+    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSet);
+    TMTIMERSTATE enmOrgState = pTimer->enmState;
+    switch (enmOrgState)
+    {
+        case TMTIMERSTATE_STOPPED:                  STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStStopped); break;
+        case TMTIMERSTATE_EXPIRED_DELIVER:          STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStExpDeliver); break;
+        case TMTIMERSTATE_ACTIVE:                   STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStActive); break;
+        case TMTIMERSTATE_PENDING_STOP:             STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStPendStop); break;
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStPendStopSched); break;
+        case TMTIMERSTATE_PENDING_SCHEDULE:         STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStPendSched); break;
+        case TMTIMERSTATE_PENDING_RESCHEDULE:       STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStPendResched); break;
+        default:                                    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStOther); break;
+    }
+#endif
+
+    /*
+     * The most common case is setting the timer again during the callback.
+     * The second most common case is starting a timer at some other time.
+     */
+#if 1
+    TMTIMERSTATE enmState1 = pTimer->enmState;
+    if (    enmState1 == TMTIMERSTATE_EXPIRED_DELIVER
+        ||  (   enmState1 == TMTIMERSTATE_STOPPED
+             && pTimer->pCritSect))
+    {
+        /* Try take the TM lock and check the state again. */
+        if (RT_SUCCESS_NP(TM_TRY_LOCK_TIMERS(pVM)))
+        {
+            if (RT_LIKELY(tmTimerTry(pTimer, TMTIMERSTATE_ACTIVE, enmState1)))
+            {
+                tmTimerSetOptimizedStart(pVM, pTimer, u64Expire);
+                STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+                return VINF_SUCCESS;
+            }
+            TM_UNLOCK_TIMERS(pVM);
+        }
+    }
+#endif
+
+    /*
+     * Unoptimized code path.
+     */
+    int cRetries = 1000;
+    do
+    {
+        /*
+         * Change to any of the SET_EXPIRE states if valid and then to SCHEDULE or RESCHEDULE.
+         */
+        TMTIMERSTATE enmState = pTimer->enmState;
+        Log2(("TMTimerSet: %p:{.enmState=%s, .pszDesc='%s'} cRetries=%d u64Expire=%'RU64\n",
+              pTimer, tmTimerState(enmState), R3STRING(pTimer->pszDesc), cRetries, u64Expire));
+        switch (enmState)
+        {
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+            case TMTIMERSTATE_STOPPED:
+                if (tmTimerTryWithLink(pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState))
+                {
+                    Assert(!pTimer->offPrev);
+                    Assert(!pTimer->offNext);
+                    pTimer->u64Expire = u64Expire;
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE);
+                    tmSchedule(pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState))
+                {
+                    pTimer->u64Expire = u64Expire;
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE);
+                    tmSchedule(pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+
+            case TMTIMERSTATE_ACTIVE:
+                if (tmTimerTryWithLink(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE, enmState))
+                {
+                    pTimer->u64Expire = u64Expire;
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE);
+                    tmSchedule(pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+            case TMTIMERSTATE_PENDING_STOP:
+                if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE, enmState))
+                {
+                    pTimer->u64Expire = u64Expire;
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE);
+                    tmSchedule(pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+#ifdef IN_RING3
+                if (!RTThreadYield())
+                    RTThreadSleep(1);
+#else
+/** @todo call host context and yield after a couple of iterations */
+#endif
+                break;
+
+            /*
+             * Invalid states.
+             */
+            case TMTIMERSTATE_DESTROY:
+            case TMTIMERSTATE_FREE:
+                AssertMsgFailed(("Invalid timer state %d (%s)\n", enmState, R3STRING(pTimer->pszDesc)));
+                return VERR_TM_INVALID_STATE;
+            default:
+                AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, R3STRING(pTimer->pszDesc)));
+                return VERR_TM_UNKNOWN_STATE;
+        }
+    } while (cRetries-- > 0);
+
+    AssertMsgFailed(("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, R3STRING(pTimer->pszDesc)));
+    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+    return VERR_TM_TIMER_UNSTABLE_STATE;
+}
+
+
+/**
+ * Return the current time for the specified clock, setting pu64Now if not NULL.
+ *
+ * @returns Current time.
+ * @param   pVM             The cross context VM structure.
+ * @param   enmClock        The clock to query.
+ * @param   pu64Now         Optional pointer where to store the return time
+ */
+DECL_FORCE_INLINE(uint64_t) tmTimerSetRelativeNowWorker(PVMCC pVM, TMCLOCK enmClock, uint64_t *pu64Now)
+{
+    uint64_t u64Now;
+    switch (enmClock)
+    {
+        case TMCLOCK_VIRTUAL_SYNC:
+            u64Now = TMVirtualSyncGet(pVM);
+            break;
+        case TMCLOCK_VIRTUAL:
+            u64Now = TMVirtualGet(pVM);
+            break;
+        case TMCLOCK_REAL:
+            u64Now = TMRealGet(pVM);
+            break;
+        default:
+            AssertFatalMsgFailed(("%d\n", enmClock));
+    }
+
+    if (pu64Now)
+        *pu64Now = u64Now;
+    return u64Now;
+}
+
+
+/**
+ * Optimized TMTimerSetRelative code path.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pTimer          The timer handle.
+ * @param   cTicksToNext    Clock ticks until the next time expiration.
+ * @param   pu64Now         Where to return the current time stamp used.
+ *                          Optional.
+ */
+static int tmTimerSetRelativeOptimizedStart(PVMCC pVM, PTMTIMER pTimer, uint64_t cTicksToNext, uint64_t *pu64Now)
+{
+    Assert(!pTimer->offPrev);
+    Assert(!pTimer->offNext);
+    Assert(pTimer->enmState == TMTIMERSTATE_ACTIVE);
+
+    /*
+     * Calculate and set the expiration time.
+     */
+    TMCLOCK const   enmClock  = pTimer->enmClock;
+    uint64_t const  u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, enmClock, pu64Now);
+    pTimer->u64Expire         = u64Expire;
+    Log2(("tmTimerSetRelativeOptimizedStart: %p:{.pszDesc='%s', .u64Expire=%'RU64} cTicksToNext=%'RU64\n", pTimer, R3STRING(pTimer->pszDesc), u64Expire, cTicksToNext));
+
+    /*
+     * Link the timer into the active list.
+     */
+    DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerSetRelativeOptimizedStart", R3STRING(pTimer->pszDesc));
+    tmTimerQueueLinkActive(&pVM->tm.s.CTX_SUFF(paTimerQueues)[enmClock], pTimer, u64Expire);
+
+    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeOpt);
+    TM_UNLOCK_TIMERS(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * TMTimerSetRelative for the virtual sync timer queue.
+ *
+ * This employs a greatly simplified state machine by always acquiring the
+ * queue lock and bypassing the scheduling list.
+ *
+ * @returns VBox status code
+ * @param   pVM                 The cross context VM structure.
+ * @param   pTimer              The timer to (re-)arm.
+ * @param   cTicksToNext        Clock ticks until the next time expiration.
+ * @param   pu64Now             Where to return the current time stamp used.
+ *                              Optional.
+ */
+static int tmTimerVirtualSyncSetRelative(PVMCC pVM, PTMTIMER pTimer, uint64_t cTicksToNext, uint64_t *pu64Now)
+{
+    STAM_PROFILE_START(pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelativeVs), a);
+    VM_ASSERT_EMT(pVM);
+    TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer);
+    int rc = PDMCritSectEnter(&pVM->tm.s.VirtualSyncLock, VINF_SUCCESS);
+    AssertRCReturn(rc, rc);
+
+    /* Calculate the expiration tick. */
+    uint64_t u64Expire = TMVirtualSyncGetNoCheck(pVM);
+    if (pu64Now)
+        *pu64Now = u64Expire;
+    u64Expire += cTicksToNext;
+
+    /* Update the timer. */
+    PTMTIMERQUEUE   pQueue    = &pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC];
+    TMTIMERSTATE    enmState  = pTimer->enmState;
+    switch (enmState)
+    {
+        case TMTIMERSTATE_EXPIRED_DELIVER:
+        case TMTIMERSTATE_STOPPED:
+            if (enmState == TMTIMERSTATE_EXPIRED_DELIVER)
+                STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeVsStExpDeliver);
+            else
+                STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeVsStStopped);
+            pTimer->u64Expire = u64Expire;
+            TM_SET_STATE(pTimer, TMTIMERSTATE_ACTIVE);
+            tmTimerQueueLinkActive(pQueue, pTimer, u64Expire);
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_ACTIVE:
+            STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeVsStActive);
+            tmTimerQueueUnlinkActive(pQueue, pTimer);
+            pTimer->u64Expire = u64Expire;
+            tmTimerQueueLinkActive(pQueue, pTimer, u64Expire);
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_PENDING_RESCHEDULE:
+        case TMTIMERSTATE_PENDING_STOP:
+        case TMTIMERSTATE_PENDING_SCHEDULE:
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+        case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+        case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_DESTROY:
+        case TMTIMERSTATE_FREE:
+            AssertLogRelMsgFailed(("Invalid timer state %s: %s\n", tmTimerState(enmState), R3STRING(pTimer->pszDesc)));
+            rc = VERR_TM_INVALID_STATE;
+            break;
+
+        default:
+            AssertMsgFailed(("Unknown timer state %d: %s\n", enmState, R3STRING(pTimer->pszDesc)));
+            rc = VERR_TM_UNKNOWN_STATE;
+            break;
+    }
+
+    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelativeVs), a);
+    PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+    return rc;
+}
+
+
+/**
+ * Arm a timer with a expire time relative to the current time.
+ *
+ * @returns VBox status code.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ * @param   cTicksToNext    Clock ticks until the next time expiration.
+ * @param   pu64Now         Where to return the current time stamp used.
+ *                          Optional.
+ */
+VMMDECL(int) TMTimerSetRelative(PTMTIMER pTimer, uint64_t cTicksToNext, uint64_t *pu64Now)
+{
+    PVMCC pVM = pTimer->CTX_SUFF(pVM);
+    STAM_COUNTER_INC(&pTimer->StatSetRelative);
+
+    /* Treat virtual sync timers specially. */
+    if (pTimer->enmClock == TMCLOCK_VIRTUAL_SYNC)
+        return tmTimerVirtualSyncSetRelative(pVM, pTimer, cTicksToNext, pu64Now);
+
+    STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelative), a);
+    TMTIMER_ASSERT_CRITSECT(pTimer);
+
+    DBGFTRACE_U64_TAG2(pVM, cTicksToNext, "TMTimerSetRelative", R3STRING(pTimer->pszDesc));
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Gather optimization info.
+     */
+    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelative);
+    TMTIMERSTATE enmOrgState = pTimer->enmState;
+    switch (enmOrgState)
+    {
+        case TMTIMERSTATE_STOPPED:                  STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStStopped); break;
+        case TMTIMERSTATE_EXPIRED_DELIVER:          STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStExpDeliver); break;
+        case TMTIMERSTATE_ACTIVE:                   STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStActive); break;
+        case TMTIMERSTATE_PENDING_STOP:             STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStPendStop); break;
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStPendStopSched); break;
+        case TMTIMERSTATE_PENDING_SCHEDULE:         STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStPendSched); break;
+        case TMTIMERSTATE_PENDING_RESCHEDULE:       STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStPendResched); break;
+        default:                                    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStOther); break;
+    }
+#endif
+
+    /*
+     * Try to take the TM lock and optimize the common cases.
+     *
+     * With the TM lock we can safely make optimizations like immediate
+     * scheduling and we can also be 100% sure that we're not racing the
+     * running of the timer queues. As an additional restraint we require the
+     * timer to have a critical section associated with to be 100% there aren't
+     * concurrent operations on the timer. (This latter isn't necessary any
+     * longer as this isn't supported for any timers, critsect or not.)
+     *
+     * Note! Lock ordering doesn't apply when we only tries to
+     *       get the innermost locks.
+     */
+    bool fOwnTMLock = RT_SUCCESS_NP(TM_TRY_LOCK_TIMERS(pVM));
+#if 1
+    if (    fOwnTMLock
+        &&  pTimer->pCritSect)
+    {
+        TMTIMERSTATE enmState = pTimer->enmState;
+        if (RT_LIKELY(  (   enmState == TMTIMERSTATE_EXPIRED_DELIVER
+                         || enmState == TMTIMERSTATE_STOPPED)
+                      && tmTimerTry(pTimer, TMTIMERSTATE_ACTIVE, enmState)))
+        {
+            tmTimerSetRelativeOptimizedStart(pVM, pTimer, cTicksToNext, pu64Now);
+            STAM_PROFILE_STOP(&pTimer->CTX_SUFF(pVM)->tm.s.CTX_SUFF_Z(StatTimerSetRelative), a);
+            return VINF_SUCCESS;
+        }
+
+        /* Optimize other states when it becomes necessary. */
+    }
+#endif
+
+    /*
+     * Unoptimized path.
+     */
+    int             rc;
+    TMCLOCK const   enmClock = pTimer->enmClock;
+    for (int cRetries = 1000; ; cRetries--)
+    {
+        /*
+         * Change to any of the SET_EXPIRE states if valid and then to SCHEDULE or RESCHEDULE.
+         */
+        TMTIMERSTATE enmState = pTimer->enmState;
+        switch (enmState)
+        {
+            case TMTIMERSTATE_STOPPED:
+                if (enmClock == TMCLOCK_VIRTUAL_SYNC)
+                {
+                    /** @todo To fix assertion in tmR3TimerQueueRunVirtualSync:
+                     *              Figure a safe way of activating this timer while the queue is
+                     *              being run.
+                     *        (99.9% sure this that the assertion is caused by DevAPIC.cpp
+                     *        re-starting the timer in response to a initial_count write.) */
+                }
+                RT_FALL_THRU();
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+                if (tmTimerTryWithLink(pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState))
+                {
+                    Assert(!pTimer->offPrev);
+                    Assert(!pTimer->offNext);
+                    pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, enmClock, pu64Now);
+                    Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [EXP/STOP]\n",
+                          pTimer, tmTimerState(enmState), R3STRING(pTimer->pszDesc), pTimer->u64Expire, cRetries));
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE);
+                    tmSchedule(pTimer);
+                    rc = VINF_SUCCESS;
+                    break;
+                }
+                rc = VERR_TRY_AGAIN;
+                break;
+
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState))
+                {
+                    pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, enmClock, pu64Now);
+                    Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [PEND_SCHED]\n",
+                          pTimer, tmTimerState(enmState), R3STRING(pTimer->pszDesc), pTimer->u64Expire, cRetries));
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE);
+                    tmSchedule(pTimer);
+                    rc = VINF_SUCCESS;
+                    break;
+                }
+                rc = VERR_TRY_AGAIN;
+                break;
+
+
+            case TMTIMERSTATE_ACTIVE:
+                if (tmTimerTryWithLink(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE, enmState))
+                {
+                    pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, enmClock, pu64Now);
+                    Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [ACTIVE]\n",
+                          pTimer, tmTimerState(enmState), R3STRING(pTimer->pszDesc), pTimer->u64Expire, cRetries));
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE);
+                    tmSchedule(pTimer);
+                    rc = VINF_SUCCESS;
+                    break;
+                }
+                rc = VERR_TRY_AGAIN;
+                break;
+
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+            case TMTIMERSTATE_PENDING_STOP:
+                if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE, enmState))
+                {
+                    pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, enmClock, pu64Now);
+                    Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [PEND_RESCH/STOP]\n",
+                          pTimer, tmTimerState(enmState), R3STRING(pTimer->pszDesc), pTimer->u64Expire, cRetries));
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE);
+                    tmSchedule(pTimer);
+                    rc = VINF_SUCCESS;
+                    break;
+                }
+                rc = VERR_TRY_AGAIN;
+                break;
+
+
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+#ifdef IN_RING3
+                if (!RTThreadYield())
+                    RTThreadSleep(1);
+#else
+/** @todo call host context and yield after a couple of iterations */
+#endif
+                rc = VERR_TRY_AGAIN;
+                break;
+
+            /*
+             * Invalid states.
+             */
+            case TMTIMERSTATE_DESTROY:
+            case TMTIMERSTATE_FREE:
+                AssertMsgFailed(("Invalid timer state %d (%s)\n", enmState, R3STRING(pTimer->pszDesc)));
+                rc = VERR_TM_INVALID_STATE;
+                break;
+
+            default:
+                AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, R3STRING(pTimer->pszDesc)));
+                rc = VERR_TM_UNKNOWN_STATE;
+                break;
+        }
+
+        /* switch + loop is tedious to break out of. */
+        if (rc == VINF_SUCCESS)
+            break;
+
+        if (rc != VERR_TRY_AGAIN)
+        {
+            tmTimerSetRelativeNowWorker(pVM, enmClock, pu64Now);
+            break;
+        }
+        if (cRetries <= 0)
+        {
+            AssertMsgFailed(("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, R3STRING(pTimer->pszDesc)));
+            rc = VERR_TM_TIMER_UNSTABLE_STATE;
+            tmTimerSetRelativeNowWorker(pVM, enmClock, pu64Now);
+            break;
+        }
+
+        /*
+         * Retry to gain locks.
+         */
+        if (!fOwnTMLock)
+            fOwnTMLock = RT_SUCCESS_NP(TM_TRY_LOCK_TIMERS(pVM));
+
+    } /* for (;;) */
+
+    /*
+     * Clean up and return.
+     */
+    if (fOwnTMLock)
+        TM_UNLOCK_TIMERS(pVM);
+
+    STAM_PROFILE_STOP(&pTimer->CTX_SUFF(pVM)->tm.s.CTX_SUFF_Z(StatTimerSetRelative), a);
+    return rc;
+}
+
+
+/**
+ * Drops a hint about the frequency of the timer.
+ *
+ * This is used by TM and the VMM to calculate how often guest execution needs
+ * to be interrupted.  The hint is automatically cleared by TMTimerStop.
+ *
+ * @returns VBox status code.
+ * @param   pTimer          Timer handle as returned by one of the create
+ *                          functions.
+ * @param   uHzHint         The frequency hint.  Pass 0 to clear the hint.
+ *
+ * @remarks We're using an integer hertz value here since anything above 1 HZ
+ *          is not going to be any trouble satisfying scheduling wise.  The
+ *          range where it makes sense is >= 100 HZ.
+ */
+VMMDECL(int) TMTimerSetFrequencyHint(PTMTIMER pTimer, uint32_t uHzHint)
+{
+    TMTIMER_ASSERT_CRITSECT(pTimer);
+
+    uint32_t const uHzOldHint = pTimer->uHzHint;
+    pTimer->uHzHint = uHzHint;
+
+    PVM pVM = pTimer->CTX_SUFF(pVM);
+    uint32_t const uMaxHzHint = pVM->tm.s.uMaxHzHint;
+    if (   uHzHint    >  uMaxHzHint
+        || uHzOldHint >= uMaxHzHint)
+        ASMAtomicWriteBool(&pVM->tm.s.fHzHintNeedsUpdating, true);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * TMTimerStop for the virtual sync timer queue.
+ *
+ * This employs a greatly simplified state machine by always acquiring the
+ * queue lock and bypassing the scheduling list.
+ *
+ * @returns VBox status code
+ * @param   pVM                 The cross context VM structure.
+ * @param   pTimer              The timer handle.
+ */
+static int tmTimerVirtualSyncStop(PVMCC pVM, PTMTIMER pTimer)
+{
+    STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerStopVs), a);
+    VM_ASSERT_EMT(pVM);
+    TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer);
+    int rc = PDMCritSectEnter(&pVM->tm.s.VirtualSyncLock, VINF_SUCCESS);
+    AssertRCReturn(rc, rc);
+
+    /* Reset the HZ hint. */
+    if (pTimer->uHzHint)
+    {
+        if (pTimer->uHzHint >= pVM->tm.s.uMaxHzHint)
+            ASMAtomicWriteBool(&pVM->tm.s.fHzHintNeedsUpdating, true);
+        pTimer->uHzHint = 0;
+    }
+
+    /* Update the timer state. */
+    PTMTIMERQUEUE   pQueue   = &pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC];
+    TMTIMERSTATE    enmState = pTimer->enmState;
+    switch (enmState)
+    {
+        case TMTIMERSTATE_ACTIVE:
+            tmTimerQueueUnlinkActive(pQueue, pTimer);
+            TM_SET_STATE(pTimer, TMTIMERSTATE_STOPPED);
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_EXPIRED_DELIVER:
+            TM_SET_STATE(pTimer, TMTIMERSTATE_STOPPED);
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_STOPPED:
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_PENDING_RESCHEDULE:
+        case TMTIMERSTATE_PENDING_STOP:
+        case TMTIMERSTATE_PENDING_SCHEDULE:
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+        case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+        case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_DESTROY:
+        case TMTIMERSTATE_FREE:
+            AssertLogRelMsgFailed(("Invalid timer state %s: %s\n", tmTimerState(enmState), R3STRING(pTimer->pszDesc)));
+            rc = VERR_TM_INVALID_STATE;
+            break;
+
+        default:
+            AssertMsgFailed(("Unknown timer state %d: %s\n", enmState, R3STRING(pTimer->pszDesc)));
+            rc = VERR_TM_UNKNOWN_STATE;
+            break;
+    }
+
+    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStopVs), a);
+    PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+    return rc;
+}
+
+
+/**
+ * Stop the timer.
+ * Use TMR3TimerArm() to "un-stop" the timer.
+ *
+ * @returns VBox status code.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ */
+VMMDECL(int) TMTimerStop(PTMTIMER pTimer)
+{
+    PVMCC pVM = pTimer->CTX_SUFF(pVM);
+    STAM_COUNTER_INC(&pTimer->StatStop);
+
+    /* Treat virtual sync timers specially. */
+    if (pTimer->enmClock == TMCLOCK_VIRTUAL_SYNC)
+        return tmTimerVirtualSyncStop(pVM, pTimer);
+
+    STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a);
+    TMTIMER_ASSERT_CRITSECT(pTimer);
+
+    /*
+     * Reset the HZ hint.
+     */
+    if (pTimer->uHzHint)
+    {
+        if (pTimer->uHzHint >= pVM->tm.s.uMaxHzHint)
+            ASMAtomicWriteBool(&pVM->tm.s.fHzHintNeedsUpdating, true);
+        pTimer->uHzHint = 0;
+    }
+
+    /** @todo see if this function needs optimizing. */
+    int cRetries = 1000;
+    do
+    {
+        /*
+         * Change to any of the SET_EXPIRE states if valid and then to SCHEDULE or RESCHEDULE.
+         */
+        TMTIMERSTATE    enmState = pTimer->enmState;
+        Log2(("TMTimerStop: %p:{.enmState=%s, .pszDesc='%s'} cRetries=%d\n",
+              pTimer, tmTimerState(enmState), R3STRING(pTimer->pszDesc), cRetries));
+        switch (enmState)
+        {
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+                //AssertMsgFailed(("You don't stop an expired timer dude!\n"));
+                return VERR_INVALID_PARAMETER;
+
+            case TMTIMERSTATE_STOPPED:
+            case TMTIMERSTATE_PENDING_STOP:
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a);
+                return VINF_SUCCESS;
+
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+                if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_STOP_SCHEDULE, enmState))
+                {
+                    tmSchedule(pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+                if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_STOP, enmState))
+                {
+                    tmSchedule(pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case TMTIMERSTATE_ACTIVE:
+                if (tmTimerTryWithLink(pTimer, TMTIMERSTATE_PENDING_STOP, enmState))
+                {
+                    tmSchedule(pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+#ifdef IN_RING3
+                if (!RTThreadYield())
+                    RTThreadSleep(1);
+#else
+/** @todo call host and yield cpu after a while. */
+#endif
+                break;
+
+            /*
+             * Invalid states.
+             */
+            case TMTIMERSTATE_DESTROY:
+            case TMTIMERSTATE_FREE:
+                AssertMsgFailed(("Invalid timer state %d (%s)\n", enmState, R3STRING(pTimer->pszDesc)));
+                return VERR_TM_INVALID_STATE;
+            default:
+                AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, R3STRING(pTimer->pszDesc)));
+                return VERR_TM_UNKNOWN_STATE;
+        }
+    } while (cRetries-- > 0);
+
+    AssertMsgFailed(("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, R3STRING(pTimer->pszDesc)));
+    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a);
+    return VERR_TM_TIMER_UNSTABLE_STATE;
+}
+
+
+/**
+ * Get the current clock time.
+ * Handy for calculating the new expire time.
+ *
+ * @returns Current clock time.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ */
+VMMDECL(uint64_t) TMTimerGet(PTMTIMER pTimer)
+{
+    PVMCC pVM = pTimer->CTX_SUFF(pVM);
+    STAM_COUNTER_INC(&pTimer->StatGet);
+
+    uint64_t u64;
+    switch (pTimer->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+            u64 = TMVirtualGet(pVM);
+            break;
+        case TMCLOCK_VIRTUAL_SYNC:
+            u64 = TMVirtualSyncGet(pVM);
+            break;
+        case TMCLOCK_REAL:
+            u64 = TMRealGet(pVM);
+            break;
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pTimer->enmClock));
+            return UINT64_MAX;
+    }
+    //Log2(("TMTimerGet: returns %'RU64 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+    //      u64, pTimer, tmTimerState(pTimer->enmState), R3STRING(pTimer->pszDesc)));
+    return u64;
+}
+
+
+/**
+ * Get the frequency of the timer clock.
+ *
+ * @returns Clock frequency (as Hz of course).
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ */
+VMMDECL(uint64_t) TMTimerGetFreq(PTMTIMER pTimer)
+{
+    switch (pTimer->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            return TMCLOCK_FREQ_VIRTUAL;
+
+        case TMCLOCK_REAL:
+            return TMCLOCK_FREQ_REAL;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pTimer->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Get the expire time of the timer.
+ * Only valid for active timers.
+ *
+ * @returns Expire time of the timer.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ */
+VMMDECL(uint64_t) TMTimerGetExpire(PTMTIMER pTimer)
+{
+    TMTIMER_ASSERT_CRITSECT(pTimer);
+    int cRetries = 1000;
+    do
+    {
+        TMTIMERSTATE    enmState = pTimer->enmState;
+        switch (enmState)
+        {
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+            case TMTIMERSTATE_STOPPED:
+            case TMTIMERSTATE_PENDING_STOP:
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                Log2(("TMTimerGetExpire: returns ~0 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+                      pTimer, tmTimerState(pTimer->enmState), R3STRING(pTimer->pszDesc)));
+                return ~(uint64_t)0;
+
+            case TMTIMERSTATE_ACTIVE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+                Log2(("TMTimerGetExpire: returns %'RU64 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+                      pTimer->u64Expire, pTimer, tmTimerState(pTimer->enmState), R3STRING(pTimer->pszDesc)));
+                return pTimer->u64Expire;
+
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+#ifdef IN_RING3
+                if (!RTThreadYield())
+                    RTThreadSleep(1);
+#endif
+                break;
+
+            /*
+             * Invalid states.
+             */
+            case TMTIMERSTATE_DESTROY:
+            case TMTIMERSTATE_FREE:
+                AssertMsgFailed(("Invalid timer state %d (%s)\n", enmState, R3STRING(pTimer->pszDesc)));
+                Log2(("TMTimerGetExpire: returns ~0 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+                      pTimer, tmTimerState(pTimer->enmState), R3STRING(pTimer->pszDesc)));
+                return ~(uint64_t)0;
+            default:
+                AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, R3STRING(pTimer->pszDesc)));
+                return ~(uint64_t)0;
+        }
+    } while (cRetries-- > 0);
+
+    AssertMsgFailed(("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, R3STRING(pTimer->pszDesc)));
+    Log2(("TMTimerGetExpire: returns ~0 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+          pTimer, tmTimerState(pTimer->enmState), R3STRING(pTimer->pszDesc)));
+    return ~(uint64_t)0;
+}
+
+
+/**
+ * Checks if a timer is active or not.
+ *
+ * @returns True if active.
+ * @returns False if not active.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ */
+VMMDECL(bool) TMTimerIsActive(PTMTIMER pTimer)
+{
+    TMTIMERSTATE enmState = pTimer->enmState;
+    switch (enmState)
+    {
+        case TMTIMERSTATE_STOPPED:
+        case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+        case TMTIMERSTATE_EXPIRED_DELIVER:
+        case TMTIMERSTATE_PENDING_STOP:
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+            Log2(("TMTimerIsActive: returns false (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+                  pTimer, tmTimerState(pTimer->enmState), R3STRING(pTimer->pszDesc)));
+            return false;
+
+        case TMTIMERSTATE_ACTIVE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE:
+        case TMTIMERSTATE_PENDING_SCHEDULE:
+        case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+            Log2(("TMTimerIsActive: returns true (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+                  pTimer, tmTimerState(pTimer->enmState), R3STRING(pTimer->pszDesc)));
+            return true;
+
+        /*
+         * Invalid states.
+         */
+        case TMTIMERSTATE_DESTROY:
+        case TMTIMERSTATE_FREE:
+            AssertMsgFailed(("Invalid timer state %s (%s)\n", tmTimerState(enmState), R3STRING(pTimer->pszDesc)));
+            Log2(("TMTimerIsActive: returns false (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+                  pTimer, tmTimerState(pTimer->enmState), R3STRING(pTimer->pszDesc)));
+            return false;
+        default:
+            AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, R3STRING(pTimer->pszDesc)));
+            return false;
+    }
+}
+
+
+/* -=-=-=-=-=-=- Convenience APIs -=-=-=-=-=-=- */
+
+
+/**
+ * Arm a timer with a (new) expire time relative to current time.
+ *
+ * @returns VBox status code.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ * @param   cMilliesToNext  Number of milliseconds to the next tick.
+ */
+VMMDECL(int) TMTimerSetMillies(PTMTIMER pTimer, uint32_t cMilliesToNext)
+{
+    switch (pTimer->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return TMTimerSetRelative(pTimer, cMilliesToNext * UINT64_C(1000000), NULL);
+
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return TMTimerSetRelative(pTimer, cMilliesToNext * UINT64_C(1000000), NULL);
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return TMTimerSetRelative(pTimer, cMilliesToNext, NULL);
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pTimer->enmClock));
+            return VERR_TM_TIMER_BAD_CLOCK;
+    }
+}
+
+
+/**
+ * Arm a timer with a (new) expire time relative to current time.
+ *
+ * @returns VBox status code.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ * @param   cMicrosToNext   Number of microseconds to the next tick.
+ */
+VMMDECL(int) TMTimerSetMicro(PTMTIMER pTimer, uint64_t cMicrosToNext)
+{
+    switch (pTimer->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return TMTimerSetRelative(pTimer, cMicrosToNext * 1000, NULL);
+
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return TMTimerSetRelative(pTimer, cMicrosToNext * 1000, NULL);
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return TMTimerSetRelative(pTimer, cMicrosToNext / 1000, NULL);
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pTimer->enmClock));
+            return VERR_TM_TIMER_BAD_CLOCK;
+    }
+}
+
+
+/**
+ * Arm a timer with a (new) expire time relative to current time.
+ *
+ * @returns VBox status code.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ * @param   cNanosToNext    Number of nanoseconds to the next tick.
+ */
+VMMDECL(int) TMTimerSetNano(PTMTIMER pTimer, uint64_t cNanosToNext)
+{
+    switch (pTimer->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return TMTimerSetRelative(pTimer, cNanosToNext, NULL);
+
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return TMTimerSetRelative(pTimer, cNanosToNext, NULL);
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return TMTimerSetRelative(pTimer, cNanosToNext / 1000000, NULL);
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pTimer->enmClock));
+            return VERR_TM_TIMER_BAD_CLOCK;
+    }
+}
+
+
+/**
+ * Get the current clock time as nanoseconds.
+ *
+ * @returns The timer clock as nanoseconds.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ */
+VMMDECL(uint64_t) TMTimerGetNano(PTMTIMER pTimer)
+{
+    return TMTimerToNano(pTimer, TMTimerGet(pTimer));
+}
+
+
+/**
+ * Get the current clock time as microseconds.
+ *
+ * @returns The timer clock as microseconds.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ */
+VMMDECL(uint64_t) TMTimerGetMicro(PTMTIMER pTimer)
+{
+    return TMTimerToMicro(pTimer, TMTimerGet(pTimer));
+}
+
+
+/**
+ * Get the current clock time as milliseconds.
+ *
+ * @returns The timer clock as milliseconds.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ */
+VMMDECL(uint64_t) TMTimerGetMilli(PTMTIMER pTimer)
+{
+    return TMTimerToMilli(pTimer, TMTimerGet(pTimer));
+}
+
+
+/**
+ * Converts the specified timer clock time to nanoseconds.
+ *
+ * @returns nanoseconds.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ * @param   u64Ticks        The clock ticks.
+ * @remark  There could be rounding errors here. We just do a simple integer divide
+ *          without any adjustments.
+ */
+VMMDECL(uint64_t) TMTimerToNano(PTMTIMER pTimer, uint64_t u64Ticks)
+{
+    switch (pTimer->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return u64Ticks;
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return u64Ticks * 1000000;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pTimer->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Converts the specified timer clock time to microseconds.
+ *
+ * @returns microseconds.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ * @param   u64Ticks        The clock ticks.
+ * @remark  There could be rounding errors here. We just do a simple integer divide
+ *          without any adjustments.
+ */
+VMMDECL(uint64_t) TMTimerToMicro(PTMTIMER pTimer, uint64_t u64Ticks)
+{
+    switch (pTimer->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return u64Ticks / 1000;
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return u64Ticks * 1000;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pTimer->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Converts the specified timer clock time to milliseconds.
+ *
+ * @returns milliseconds.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ * @param   u64Ticks        The clock ticks.
+ * @remark  There could be rounding errors here. We just do a simple integer divide
+ *          without any adjustments.
+ */
+VMMDECL(uint64_t) TMTimerToMilli(PTMTIMER pTimer, uint64_t u64Ticks)
+{
+    switch (pTimer->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return u64Ticks / 1000000;
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return u64Ticks;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pTimer->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Converts the specified nanosecond timestamp to timer clock ticks.
+ *
+ * @returns timer clock ticks.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ * @param   cNanoSecs       The nanosecond value ticks to convert.
+ * @remark  There could be rounding and overflow errors here.
+ */
+VMMDECL(uint64_t) TMTimerFromNano(PTMTIMER pTimer, uint64_t cNanoSecs)
+{
+    switch (pTimer->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return cNanoSecs;
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return cNanoSecs / 1000000;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pTimer->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Converts the specified microsecond timestamp to timer clock ticks.
+ *
+ * @returns timer clock ticks.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ * @param   cMicroSecs      The microsecond value ticks to convert.
+ * @remark  There could be rounding and overflow errors here.
+ */
+VMMDECL(uint64_t) TMTimerFromMicro(PTMTIMER pTimer, uint64_t cMicroSecs)
+{
+    switch (pTimer->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return cMicroSecs * 1000;
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return cMicroSecs / 1000;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pTimer->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Converts the specified millisecond timestamp to timer clock ticks.
+ *
+ * @returns timer clock ticks.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ * @param   cMilliSecs      The millisecond value ticks to convert.
+ * @remark  There could be rounding and overflow errors here.
+ */
+VMMDECL(uint64_t) TMTimerFromMilli(PTMTIMER pTimer, uint64_t cMilliSecs)
+{
+    switch (pTimer->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return cMilliSecs * 1000000;
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return cMilliSecs;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pTimer->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Convert state to string.
+ *
+ * @returns Readonly status name.
+ * @param   enmState    State.
+ */
+const char *tmTimerState(TMTIMERSTATE enmState)
+{
+    switch (enmState)
+    {
+#define CASE(num, state) \
+            case TMTIMERSTATE_##state: \
+                AssertCompile(TMTIMERSTATE_##state == (num)); \
+                return #num "-" #state
+        CASE( 1,STOPPED);
+        CASE( 2,ACTIVE);
+        CASE( 3,EXPIRED_GET_UNLINK);
+        CASE( 4,EXPIRED_DELIVER);
+        CASE( 5,PENDING_STOP);
+        CASE( 6,PENDING_STOP_SCHEDULE);
+        CASE( 7,PENDING_SCHEDULE_SET_EXPIRE);
+        CASE( 8,PENDING_SCHEDULE);
+        CASE( 9,PENDING_RESCHEDULE_SET_EXPIRE);
+        CASE(10,PENDING_RESCHEDULE);
+        CASE(11,DESTROY);
+        CASE(12,FREE);
+        default:
+            AssertMsgFailed(("Invalid state enmState=%d\n", enmState));
+            return "Invalid state!";
+#undef CASE
+    }
+}
+
+
+/**
+ * Gets the highest frequency hint for all the important timers.
+ *
+ * @returns The highest frequency.  0 if no timers care.
+ * @param   pVM         The cross context VM structure.
+ */
+static uint32_t tmGetFrequencyHint(PVM pVM)
+{
+    /*
+     * Query the value, recalculate it if necessary.
+     *
+     * The "right" highest frequency value isn't so important that we'll block
+     * waiting on the timer semaphore.
+     */
+    uint32_t uMaxHzHint = ASMAtomicUoReadU32(&pVM->tm.s.uMaxHzHint);
+    if (RT_UNLIKELY(ASMAtomicReadBool(&pVM->tm.s.fHzHintNeedsUpdating)))
+    {
+        if (RT_SUCCESS(TM_TRY_LOCK_TIMERS(pVM)))
+        {
+            ASMAtomicWriteBool(&pVM->tm.s.fHzHintNeedsUpdating, false);
+
+            /*
+             * Loop over the timers associated with each clock.
+             */
+            uMaxHzHint = 0;
+            for (int i = 0; i < TMCLOCK_MAX; i++)
+            {
+                PTMTIMERQUEUE pQueue = &pVM->tm.s.CTX_SUFF(paTimerQueues)[i];
+                for (PTMTIMER pCur = TMTIMER_GET_HEAD(pQueue); pCur; pCur = TMTIMER_GET_NEXT(pCur))
+                {
+                    uint32_t uHzHint = ASMAtomicUoReadU32(&pCur->uHzHint);
+                    if (uHzHint > uMaxHzHint)
+                    {
+                        switch (pCur->enmState)
+                        {
+                            case TMTIMERSTATE_ACTIVE:
+                            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+                            case TMTIMERSTATE_EXPIRED_DELIVER:
+                            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+                            case TMTIMERSTATE_PENDING_SCHEDULE:
+                            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+                            case TMTIMERSTATE_PENDING_RESCHEDULE:
+                                uMaxHzHint = uHzHint;
+                                break;
+
+                            case TMTIMERSTATE_STOPPED:
+                            case TMTIMERSTATE_PENDING_STOP:
+                            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                            case TMTIMERSTATE_DESTROY:
+                            case TMTIMERSTATE_FREE:
+                                break;
+                            /* no default, want gcc warnings when adding more states. */
+                        }
+                    }
+                }
+            }
+            ASMAtomicWriteU32(&pVM->tm.s.uMaxHzHint, uMaxHzHint);
+            Log(("tmGetFrequencyHint: New value %u Hz\n", uMaxHzHint));
+            TM_UNLOCK_TIMERS(pVM);
+        }
+    }
+    return uMaxHzHint;
+}
+
+
+/**
+ * Calculates a host timer frequency that would be suitable for the current
+ * timer load.
+ *
+ * This will take the highest timer frequency, adjust for catch-up and warp
+ * driver, and finally add a little fudge factor.  The caller (VMM) will use
+ * the result to adjust the per-cpu preemption timer.
+ *
+ * @returns The highest frequency.  0 if no important timers around.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMM_INT_DECL(uint32_t) TMCalcHostTimerFrequency(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    uint32_t uHz = tmGetFrequencyHint(pVM);
+
+    /* Catch up, we have to be more aggressive than the % indicates at the
+       beginning of the effort. */
+    if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+    {
+        uint32_t u32Pct = ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage);
+        if (ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+        {
+            if (u32Pct <= 100)
+                u32Pct = u32Pct * pVM->tm.s.cPctHostHzFudgeFactorCatchUp100 / 100;
+            else if (u32Pct <= 200)
+                u32Pct = u32Pct * pVM->tm.s.cPctHostHzFudgeFactorCatchUp200 / 100;
+            else if (u32Pct <= 400)
+                u32Pct = u32Pct * pVM->tm.s.cPctHostHzFudgeFactorCatchUp400 / 100;
+            uHz *= u32Pct + 100;
+            uHz /= 100;
+        }
+    }
+
+    /* Warp drive. */
+    if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualWarpDrive))
+    {
+        uint32_t u32Pct = ASMAtomicReadU32(&pVM->tm.s.u32VirtualWarpDrivePercentage);
+        if (ASMAtomicReadBool(&pVM->tm.s.fVirtualWarpDrive))
+        {
+            uHz *= u32Pct;
+            uHz /= 100;
+        }
+    }
+
+    /* Fudge factor. */
+    if (pVCpu->idCpu == pVM->tm.s.idTimerCpu)
+        uHz *= pVM->tm.s.cPctHostHzFudgeFactorTimerCpu;
+    else
+        uHz *= pVM->tm.s.cPctHostHzFudgeFactorOtherCpu;
+    uHz /= 100;
+
+    /* Make sure it isn't too high. */
+    if (uHz > pVM->tm.s.cHostHzMax)
+        uHz = pVM->tm.s.cHostHzMax;
+
+    return uHz;
+}
+
+
+/**
+ * Whether the guest virtual clock is ticking.
+ *
+ * @returns true if ticking, false otherwise.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(bool) TMVirtualIsTicking(PVM pVM)
+{
+    return RT_BOOL(pVM->tm.s.cVirtualTicking);
+}
+
diff --git a/src/VBox/VMM/VMMAll/TMAllCpu.cpp b/src/VBox/VMM/VMMAll/TMAllCpu.cpp
new file mode 100644
index 00000000..a6304740
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/TMAllCpu.cpp
@@ -0,0 +1,605 @@
+/* $Id: TMAllCpu.cpp $ */
+/** @file
+ * TM - Timeout Manager, CPU Time, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_TM
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/nem.h>
+#include <iprt/asm-amd64-x86.h> /* for SUPGetCpuHzFromGIP */
+#include "TMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/sup.h>
+
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <iprt/asm-math.h>
+#include <iprt/assert.h>
+#include <VBox/log.h>
+
+
+/**
+ * Gets the raw cpu tick from current virtual time.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fCheckTimers    Whether to check timers.
+ */
+DECLINLINE(uint64_t) tmCpuTickGetRawVirtual(PVMCC pVM, bool fCheckTimers)
+{
+    uint64_t u64;
+    if (fCheckTimers)
+        u64 = TMVirtualSyncGet(pVM);
+    else
+        u64 = TMVirtualSyncGetNoCheck(pVM);
+    return ASMMultU64ByU32DivByU32(u64, pVM->tm.s.cTSCTicksPerSecond, TMCLOCK_FREQ_VIRTUAL);
+}
+
+
+#ifdef IN_RING3
+/**
+ * Used by tmR3CpuTickParavirtEnable and tmR3CpuTickParavirtDisable.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+uint64_t tmR3CpuTickGetRawVirtualNoCheck(PVM pVM)
+{
+    return tmCpuTickGetRawVirtual(pVM, false /*fCheckTimers*/);
+}
+#endif
+
+
+/**
+ * Resumes the CPU timestamp counter ticking.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @internal
+ */
+int tmCpuTickResume(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    if (!pVCpu->tm.s.fTSCTicking)
+    {
+        pVCpu->tm.s.fTSCTicking = true;
+
+        /** @todo Test that pausing and resuming doesn't cause lag! (I.e. that we're
+         *        unpaused before the virtual time and stopped after it. */
+        switch (pVM->tm.s.enmTSCMode)
+        {
+            case TMTSCMODE_REAL_TSC_OFFSET:
+                pVCpu->tm.s.offTSCRawSrc = SUPReadTsc() - pVCpu->tm.s.u64TSC;
+                break;
+            case TMTSCMODE_VIRT_TSC_EMULATED:
+            case TMTSCMODE_DYNAMIC:
+                pVCpu->tm.s.offTSCRawSrc = tmCpuTickGetRawVirtual(pVM, false /* don't check for pending timers */)
+                                         - pVCpu->tm.s.u64TSC;
+                break;
+            case TMTSCMODE_NATIVE_API:
+                pVCpu->tm.s.offTSCRawSrc = 0; /** @todo ?? */
+                /* Looks like this is only used by weird modes and MSR TSC writes.  We cannot support either on NEM/win. */
+                break;
+            default:
+                AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+        }
+        return VINF_SUCCESS;
+    }
+    AssertFailed();
+    return VERR_TM_TSC_ALREADY_TICKING;
+}
+
+
+/**
+ * Resumes the CPU timestamp counter ticking.
+ *
+ * @returns VINF_SUCCESS or VERR_TM_VIRTUAL_TICKING_IPE (asserted).
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+int tmCpuTickResumeLocked(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    if (!pVCpu->tm.s.fTSCTicking)
+    {
+        /* TSC must be ticking before calling tmCpuTickGetRawVirtual()! */
+        pVCpu->tm.s.fTSCTicking = true;
+        uint32_t c = ASMAtomicIncU32(&pVM->tm.s.cTSCsTicking);
+        AssertMsgReturn(c <= pVM->cCpus, ("%u vs %u\n", c, pVM->cCpus), VERR_TM_VIRTUAL_TICKING_IPE);
+        if (c == 1)
+        {
+            /* The first VCPU to resume. */
+            uint64_t    offTSCRawSrcOld = pVCpu->tm.s.offTSCRawSrc;
+
+            STAM_COUNTER_INC(&pVM->tm.s.StatTSCResume);
+
+            /* When resuming, use the TSC value of the last stopped VCPU to avoid the TSC going back. */
+            switch (pVM->tm.s.enmTSCMode)
+            {
+                case TMTSCMODE_REAL_TSC_OFFSET:
+                    pVCpu->tm.s.offTSCRawSrc = SUPReadTsc() - pVM->tm.s.u64LastPausedTSC;
+                    break;
+                case TMTSCMODE_VIRT_TSC_EMULATED:
+                case TMTSCMODE_DYNAMIC:
+                    pVCpu->tm.s.offTSCRawSrc = tmCpuTickGetRawVirtual(pVM, false /* don't check for pending timers */)
+                                             - pVM->tm.s.u64LastPausedTSC;
+                    break;
+                case TMTSCMODE_NATIVE_API:
+                {
+                    int rc = NEMHCResumeCpuTickOnAll(pVM, pVCpu, pVM->tm.s.u64LastPausedTSC);
+                    AssertRCReturn(rc, rc);
+                    pVCpu->tm.s.offTSCRawSrc = offTSCRawSrcOld = 0;
+                    break;
+                }
+                default:
+                    AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+            }
+
+            /* Calculate the offset addendum for other VCPUs to use. */
+            pVM->tm.s.offTSCPause = pVCpu->tm.s.offTSCRawSrc - offTSCRawSrcOld;
+        }
+        else
+        {
+            /* All other VCPUs (if any). */
+            pVCpu->tm.s.offTSCRawSrc += pVM->tm.s.offTSCPause;
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Pauses the CPU timestamp counter ticking.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @internal
+ */
+int tmCpuTickPause(PVMCPUCC pVCpu)
+{
+    if (pVCpu->tm.s.fTSCTicking)
+    {
+        pVCpu->tm.s.u64TSC = TMCpuTickGetNoCheck(pVCpu);
+        pVCpu->tm.s.fTSCTicking = false;
+        return VINF_SUCCESS;
+    }
+    AssertFailed();
+    return VERR_TM_TSC_ALREADY_PAUSED;
+}
+
+
+/**
+ * Pauses the CPU timestamp counter ticking.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @internal
+ */
+int tmCpuTickPauseLocked(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    if (pVCpu->tm.s.fTSCTicking)
+    {
+        pVCpu->tm.s.u64TSC = TMCpuTickGetNoCheck(pVCpu);
+        pVCpu->tm.s.fTSCTicking = false;
+
+        uint32_t c = ASMAtomicDecU32(&pVM->tm.s.cTSCsTicking);
+        AssertMsgReturn(c < pVM->cCpus, ("%u vs %u\n", c, pVM->cCpus), VERR_TM_VIRTUAL_TICKING_IPE);
+        if (c == 0)
+        {
+            /* When the last TSC stops, remember the value. */
+            STAM_COUNTER_INC(&pVM->tm.s.StatTSCPause);
+            pVM->tm.s.u64LastPausedTSC = pVCpu->tm.s.u64TSC;
+        }
+        return VINF_SUCCESS;
+    }
+    AssertFailed();
+    return VERR_TM_TSC_ALREADY_PAUSED;
+}
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * Record why we refused to use offsetted TSC.
+ *
+ * Used by TMCpuTickCanUseRealTSC() and TMCpuTickGetDeadlineAndTscOffset().
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+DECLINLINE(void) tmCpuTickRecordOffsettedTscRefusal(PVM pVM, PVMCPU pVCpu)
+{
+    /* Sample the reason for refusing. */
+    if (pVM->tm.s.enmTSCMode != TMTSCMODE_DYNAMIC)
+       STAM_COUNTER_INC(&pVM->tm.s.StatTSCNotFixed);
+    else if (!pVCpu->tm.s.fTSCTicking)
+       STAM_COUNTER_INC(&pVM->tm.s.StatTSCNotTicking);
+    else if (pVM->tm.s.enmTSCMode != TMTSCMODE_REAL_TSC_OFFSET)
+    {
+        if (pVM->tm.s.fVirtualSyncCatchUp)
+        {
+           if (pVM->tm.s.u32VirtualSyncCatchUpPercentage <= 10)
+               STAM_COUNTER_INC(&pVM->tm.s.StatTSCCatchupLE010);
+           else if (pVM->tm.s.u32VirtualSyncCatchUpPercentage <= 25)
+               STAM_COUNTER_INC(&pVM->tm.s.StatTSCCatchupLE025);
+           else if (pVM->tm.s.u32VirtualSyncCatchUpPercentage <= 100)
+               STAM_COUNTER_INC(&pVM->tm.s.StatTSCCatchupLE100);
+           else
+               STAM_COUNTER_INC(&pVM->tm.s.StatTSCCatchupOther);
+        }
+        else if (!pVM->tm.s.fVirtualSyncTicking)
+           STAM_COUNTER_INC(&pVM->tm.s.StatTSCSyncNotTicking);
+        else if (pVM->tm.s.fVirtualWarpDrive)
+           STAM_COUNTER_INC(&pVM->tm.s.StatTSCWarp);
+    }
+}
+#endif /* VBOX_WITH_STATISTICS */
+
+
+/**
+ * Checks if AMD-V / VT-x can use an offsetted hardware TSC or not.
+ *
+ * @returns true/false accordingly.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   poffRealTsc     The offset against the TSC of the current host CPU,
+ *                          if pfOffsettedTsc is set to true.
+ * @param   pfParavirtTsc   Where to return whether paravirt TSC is enabled.
+ *
+ * @thread  EMT(pVCpu).
+ * @see     TMCpuTickGetDeadlineAndTscOffset().
+ */
+VMM_INT_DECL(bool) TMCpuTickCanUseRealTSC(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *poffRealTsc, bool *pfParavirtTsc)
+{
+    Assert(pVCpu->tm.s.fTSCTicking || DBGFIsStepping(pVCpu));
+
+    *pfParavirtTsc = pVM->tm.s.fParavirtTscEnabled;
+
+    /*
+     * In real TSC mode it's easy, we just need the delta & offTscRawSrc and
+     * the CPU will add them to RDTSC and RDTSCP at runtime.
+     *
+     * In tmCpuTickGetInternal we do:
+     *          SUPReadTsc() - pVCpu->tm.s.offTSCRawSrc;
+     * Where SUPReadTsc() does:
+     *          ASMReadTSC() - pGipCpu->i64TscDelta;
+     * Which means tmCpuTickGetInternal actually does:
+     *          ASMReadTSC() - pGipCpu->i64TscDelta - pVCpu->tm.s.offTSCRawSrc;
+     * So, the offset to be ADDED to RDTSC[P] is:
+     *          offRealTsc = -(pGipCpu->i64TscDelta + pVCpu->tm.s.offTSCRawSrc)
+     */
+    if (pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET)
+    {
+        /** @todo We should negate both deltas!  It's soo weird that we do the
+         *        exact opposite of what the hardware implements. */
+#ifdef IN_RING3
+        *poffRealTsc = 0 - pVCpu->tm.s.offTSCRawSrc - SUPGetTscDelta();
+#else
+        *poffRealTsc = 0 - pVCpu->tm.s.offTSCRawSrc - SUPGetTscDeltaByCpuSetIndex(pVCpu->iHostCpuSet);
+#endif
+        return true;
+    }
+
+    /*
+     * We require:
+     *     1. A fixed TSC, this is checked at init time.
+     *     2. That the TSC is ticking (we shouldn't be here if it isn't)
+     *     3. Either that we're using the real TSC as time source or
+     *          a) we don't have any lag to catch up, and
+     *          b) the virtual sync clock hasn't been halted by an expired timer, and
+     *          c) we're not using warp drive (accelerated virtual guest time).
+     */
+    if (   pVM->tm.s.enmTSCMode == TMTSCMODE_DYNAMIC
+        && !pVM->tm.s.fVirtualSyncCatchUp
+        && RT_LIKELY(pVM->tm.s.fVirtualSyncTicking)
+        && !pVM->tm.s.fVirtualWarpDrive)
+    {
+        /* The source is the timer synchronous virtual clock. */
+        uint64_t u64Now = tmCpuTickGetRawVirtual(pVM, false /* don't check for pending timers */)
+                        - pVCpu->tm.s.offTSCRawSrc;
+        /** @todo When we start collecting statistics on how much time we spend executing
+         * guest code before exiting, we should check this against the next virtual sync
+         * timer timeout. If it's lower than the avg. length, we should trap rdtsc to increase
+         * the chance that we'll get interrupted right after the timer expired. */
+        if (u64Now >= pVCpu->tm.s.u64TSCLastSeen)
+        {
+            *poffRealTsc = u64Now - ASMReadTSC();
+            return true;    /** @todo count this? */
+        }
+    }
+
+#ifdef VBOX_WITH_STATISTICS
+    tmCpuTickRecordOffsettedTscRefusal(pVM, pVCpu);
+#endif
+    return false;
+}
+
+
+/**
+ * Calculates the number of host CPU ticks till the next virtual sync deadline.
+ *
+ * @note    To save work, this function will not bother calculating the accurate
+ *          tick count for deadlines that are more than a second ahead.
+ *
+ * @returns The number of host cpu ticks to the next deadline.  Max one second.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   cNsToDeadline   The number of nano seconds to the next virtual
+ *                          sync deadline.
+ */
+DECLINLINE(uint64_t) tmCpuCalcTicksToDeadline(PVMCPU pVCpu, uint64_t cNsToDeadline)
+{
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL <= _4G);
+#ifdef IN_RING3
+    RT_NOREF_PV(pVCpu);
+    uint64_t uCpuHz = SUPGetCpuHzFromGip(g_pSUPGlobalInfoPage);
+#else
+    uint64_t uCpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
+#endif
+    if (RT_UNLIKELY(cNsToDeadline >= TMCLOCK_FREQ_VIRTUAL))
+        return uCpuHz;
+    uint64_t cTicks = ASMMultU64ByU32DivByU32(uCpuHz, cNsToDeadline, TMCLOCK_FREQ_VIRTUAL);
+    if (cTicks > 4000)
+        cTicks -= 4000; /* fudge to account for overhead */
+    else
+        cTicks >>= 1;
+    return cTicks;
+}
+
+
+/**
+ * Gets the next deadline in host CPU clock ticks and the TSC offset if we can
+ * use the raw TSC.
+ *
+ * @returns The number of host CPU clock ticks to the next timer deadline.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   poffRealTsc     The offset against the TSC of the current host CPU,
+ *                          if pfOffsettedTsc is set to true.
+ * @param   pfOffsettedTsc  Where to return whether TSC offsetting can be used.
+ * @param   pfParavirtTsc   Where to return whether paravirt TSC is enabled.
+ *
+ * @thread  EMT(pVCpu).
+ * @see     TMCpuTickCanUseRealTSC().
+ */
+VMM_INT_DECL(uint64_t) TMCpuTickGetDeadlineAndTscOffset(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *poffRealTsc,
+                                                        bool *pfOffsettedTsc, bool *pfParavirtTsc)
+{
+    Assert(pVCpu->tm.s.fTSCTicking || DBGFIsStepping(pVCpu));
+
+    *pfParavirtTsc = pVM->tm.s.fParavirtTscEnabled;
+
+    /*
+     * Same logic as in TMCpuTickCanUseRealTSC.
+     */
+    if (pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET)
+    {
+        /** @todo We should negate both deltas!  It's soo weird that we do the
+         *        exact opposite of what the hardware implements. */
+#ifdef IN_RING3
+        *poffRealTsc     = 0 - pVCpu->tm.s.offTSCRawSrc - SUPGetTscDelta();
+#else
+        *poffRealTsc     = 0 - pVCpu->tm.s.offTSCRawSrc - SUPGetTscDeltaByCpuSetIndex(pVCpu->iHostCpuSet);
+#endif
+        *pfOffsettedTsc  = true;
+        return tmCpuCalcTicksToDeadline(pVCpu, TMVirtualSyncGetNsToDeadline(pVM));
+    }
+
+    /*
+     * Same logic as in TMCpuTickCanUseRealTSC.
+     */
+    if (   pVM->tm.s.enmTSCMode == TMTSCMODE_DYNAMIC
+        && !pVM->tm.s.fVirtualSyncCatchUp
+        && RT_LIKELY(pVM->tm.s.fVirtualSyncTicking)
+        && !pVM->tm.s.fVirtualWarpDrive)
+    {
+        /* The source is the timer synchronous virtual clock. */
+        uint64_t cNsToDeadline;
+        uint64_t u64NowVirtSync = TMVirtualSyncGetWithDeadlineNoCheck(pVM, &cNsToDeadline);
+        uint64_t u64Now = ASMMultU64ByU32DivByU32(u64NowVirtSync, pVM->tm.s.cTSCTicksPerSecond, TMCLOCK_FREQ_VIRTUAL);
+        u64Now -= pVCpu->tm.s.offTSCRawSrc;
+        *poffRealTsc     = u64Now - ASMReadTSC();
+        *pfOffsettedTsc  = u64Now >= pVCpu->tm.s.u64TSCLastSeen;
+        return tmCpuCalcTicksToDeadline(pVCpu, cNsToDeadline);
+    }
+
+#ifdef VBOX_WITH_STATISTICS
+    tmCpuTickRecordOffsettedTscRefusal(pVM, pVCpu);
+#endif
+    *pfOffsettedTsc  = false;
+    *poffRealTsc     = 0;
+    return tmCpuCalcTicksToDeadline(pVCpu, TMVirtualSyncGetNsToDeadline(pVM));
+}
+
+
+/**
+ * Read the current CPU timestamp counter.
+ *
+ * @returns Gets the CPU tsc.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   fCheckTimers    Whether to check timers.
+ */
+DECLINLINE(uint64_t) tmCpuTickGetInternal(PVMCPUCC pVCpu, bool fCheckTimers)
+{
+    uint64_t u64;
+
+    if (RT_LIKELY(pVCpu->tm.s.fTSCTicking))
+    {
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        switch (pVM->tm.s.enmTSCMode)
+        {
+            case TMTSCMODE_REAL_TSC_OFFSET:
+                u64 = SUPReadTsc();
+                break;
+            case TMTSCMODE_VIRT_TSC_EMULATED:
+            case TMTSCMODE_DYNAMIC:
+                u64 = tmCpuTickGetRawVirtual(pVM, fCheckTimers);
+                break;
+            case TMTSCMODE_NATIVE_API:
+            {
+                u64 = 0;
+                int rcNem = NEMHCQueryCpuTick(pVCpu, &u64, NULL);
+                AssertLogRelRCReturn(rcNem, SUPReadTsc());
+                break;
+            }
+            default:
+                AssertFailedBreakStmt(u64 = SUPReadTsc());
+        }
+        u64 -= pVCpu->tm.s.offTSCRawSrc;
+
+        /* Always return a value higher than what the guest has already seen. */
+        if (RT_LIKELY(u64 > pVCpu->tm.s.u64TSCLastSeen))
+            pVCpu->tm.s.u64TSCLastSeen = u64;
+        else
+        {
+            STAM_COUNTER_INC(&pVM->tm.s.StatTSCUnderflow);
+            pVCpu->tm.s.u64TSCLastSeen += 64;   /** @todo choose a good increment here */
+            u64 = pVCpu->tm.s.u64TSCLastSeen;
+        }
+    }
+    else
+        u64 = pVCpu->tm.s.u64TSC;
+    return u64;
+}
+
+
+/**
+ * Read the current CPU timestamp counter.
+ *
+ * @returns Gets the CPU tsc.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(uint64_t) TMCpuTickGet(PVMCPUCC pVCpu)
+{
+    return tmCpuTickGetInternal(pVCpu, true /* fCheckTimers */);
+}
+
+
+/**
+ * Read the current CPU timestamp counter, don't check for expired timers.
+ *
+ * @returns Gets the CPU tsc.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(uint64_t) TMCpuTickGetNoCheck(PVMCPUCC pVCpu)
+{
+    return tmCpuTickGetInternal(pVCpu, false /* fCheckTimers */);
+}
+
+
+/**
+ * Sets the current CPU timestamp counter.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   u64Tick     The new timestamp value.
+ *
+ * @thread  EMT which TSC is to be set.
+ */
+VMM_INT_DECL(int) TMCpuTickSet(PVMCC pVM, PVMCPUCC pVCpu, uint64_t u64Tick)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    STAM_COUNTER_INC(&pVM->tm.s.StatTSCSet);
+
+    /*
+     * This is easier to do when the TSC is paused since resume will
+     * do all the calculations for us. Actually, we don't need to
+     * call tmCpuTickPause here since we overwrite u64TSC anyway.
+     */
+    bool        fTSCTicking    = pVCpu->tm.s.fTSCTicking;
+    pVCpu->tm.s.fTSCTicking    = false;
+    pVCpu->tm.s.u64TSC         = u64Tick;
+    pVCpu->tm.s.u64TSCLastSeen = u64Tick;
+    if (fTSCTicking)
+        tmCpuTickResume(pVM, pVCpu);
+    /** @todo Try help synchronizing it better among the virtual CPUs? */
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Sets the last seen CPU timestamp counter.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   u64LastSeenTick     The last seen timestamp value.
+ *
+ * @thread  EMT which TSC is to be set.
+ */
+VMM_INT_DECL(int) TMCpuTickSetLastSeen(PVMCPUCC pVCpu, uint64_t u64LastSeenTick)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    LogFlow(("TMCpuTickSetLastSeen %RX64\n", u64LastSeenTick));
+    if (pVCpu->tm.s.u64TSCLastSeen < u64LastSeenTick)
+        pVCpu->tm.s.u64TSCLastSeen = u64LastSeenTick;
+    return VINF_SUCCESS;
+}
+
+/**
+ * Gets the last seen CPU timestamp counter of the guest.
+ *
+ * @returns the last seen TSC.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(uint64_t) TMCpuTickGetLastSeen(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    return pVCpu->tm.s.u64TSCLastSeen;
+}
+
+
+/**
+ * Get the timestamp frequency.
+ *
+ * @returns Number of ticks per second.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(uint64_t) TMCpuTicksPerSecond(PVMCC pVM)
+{
+    if (   pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET
+        && g_pSUPGlobalInfoPage->u32Mode != SUPGIPMODE_INVARIANT_TSC)
+    {
+#ifdef IN_RING3
+        uint64_t cTSCTicksPerSecond = SUPGetCpuHzFromGip(g_pSUPGlobalInfoPage);
+#elif defined(IN_RING0)
+        uint64_t cTSCTicksPerSecond = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, RTMpCpuIdToSetIndex(RTMpCpuId()));
+#else
+        uint64_t cTSCTicksPerSecond = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, VMMGetCpu(pVM)->iHostCpuSet);
+#endif
+        if (RT_LIKELY(cTSCTicksPerSecond != ~(uint64_t)0))
+            return cTSCTicksPerSecond;
+    }
+    return pVM->tm.s.cTSCTicksPerSecond;
+}
+
+
+/**
+ * Whether the TSC is ticking for the VCPU.
+ *
+ * @returns true if ticking, false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) TMCpuTickIsTicking(PVMCPUCC pVCpu)
+{
+    return pVCpu->tm.s.fTSCTicking;
+}
+
diff --git a/src/VBox/VMM/VMMAll/TMAllReal.cpp b/src/VBox/VMM/VMMAll/TMAllReal.cpp
new file mode 100644
index 00000000..d284610a
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/TMAllReal.cpp
@@ -0,0 +1,53 @@
+/* $Id: TMAllReal.cpp $ */
+/** @file
+ * TM - Timeout Manager, Real Time, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_TM
+#include <VBox/vmm/tm.h>
+#include "TMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <iprt/time.h>
+
+
+/**
+ * Gets the current TMCLOCK_REAL time.
+ *
+ * @returns Real time.
+ * @param   pVM             The cross context VM structure.
+ */
+VMM_INT_DECL(uint64_t) TMRealGet(PVM pVM)
+{
+    NOREF(pVM);
+    return RTTimeMilliTS();
+}
+
+
+/**
+ * Gets the frequency of the TMCLOCK_REAL clock.
+ *
+ * @returns frequency.
+ * @param   pVM             The cross context VM structure.
+ */
+VMM_INT_DECL(uint64_t) TMRealGetFreq(PVM pVM)
+{
+    NOREF(pVM);
+    return TMCLOCK_FREQ_REAL;
+}
+
diff --git a/src/VBox/VMM/VMMAll/TMAllVirtual.cpp b/src/VBox/VMM/VMMAll/TMAllVirtual.cpp
new file mode 100644
index 00000000..53894bc9
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/TMAllVirtual.cpp
@@ -0,0 +1,998 @@
+/* $Id: TMAllVirtual.cpp $ */
+/** @file
+ * TM - Timeout Manager, Virtual Time, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_TM
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/dbgftrace.h>
+#ifdef IN_RING3
+# include <iprt/thread.h>
+#endif
+#include "TMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/sup.h>
+
+#include <iprt/time.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/asm-math.h>
+
+
+
+/**
+ * @interface_method_impl{RTTIMENANOTSDATA,pfnBad}
+ */
+DECLCALLBACK(DECLEXPORT(void)) tmVirtualNanoTSBad(PRTTIMENANOTSDATA pData, uint64_t u64NanoTS, uint64_t u64DeltaPrev,
+                                                  uint64_t u64PrevNanoTS)
+{
+    PVM pVM = RT_FROM_MEMBER(pData, VM, CTX_SUFF(tm.s.VirtualGetRawData));
+    pData->cBadPrev++;
+    if ((int64_t)u64DeltaPrev < 0)
+        LogRel(("TM: u64DeltaPrev=%RI64 u64PrevNanoTS=0x%016RX64 u64NanoTS=0x%016RX64 pVM=%p\n",
+                u64DeltaPrev, u64PrevNanoTS, u64NanoTS, pVM));
+    else
+        Log(("TM: u64DeltaPrev=%RI64 u64PrevNanoTS=0x%016RX64 u64NanoTS=0x%016RX64 pVM=%p (debugging?)\n",
+             u64DeltaPrev, u64PrevNanoTS, u64NanoTS, pVM));
+}
+
+
+/**
+ * @interface_method_impl{RTTIMENANOTSDATA,pfnRediscover}
+ *
+ * This is the initial worker, so the first call in each context ends up here.
+ * It is also used should the delta rating of the host CPUs change or if the
+ * fGetGipCpu feature the current worker relies upon becomes unavailable.  The
+ * last two events may occur as CPUs are taken online.
+ */
+DECLCALLBACK(DECLEXPORT(uint64_t)) tmVirtualNanoTSRediscover(PRTTIMENANOTSDATA pData)
+{
+    PVM pVM = RT_FROM_MEMBER(pData, VM, CTX_SUFF(tm.s.VirtualGetRawData));
+
+    /*
+     * We require a valid GIP for the selection below.  Invalid GIP is fatal.
+     */
+    PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
+    AssertFatalMsg(RT_VALID_PTR(pGip), ("pVM=%p pGip=%p\n", pVM, pGip));
+    AssertFatalMsg(pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC, ("pVM=%p pGip=%p u32Magic=%#x\n", pVM, pGip, pGip->u32Magic));
+    AssertFatalMsg(pGip->u32Mode > SUPGIPMODE_INVALID && pGip->u32Mode < SUPGIPMODE_END,
+                   ("pVM=%p pGip=%p u32Mode=%#x\n", pVM, pGip, pGip->u32Mode));
+
+    /*
+     * Determine the new worker.
+     */
+    PFNTIMENANOTSINTERNAL   pfnWorker;
+    bool const              fLFence = RT_BOOL(ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_SSE2);
+    switch (pGip->u32Mode)
+    {
+        case SUPGIPMODE_SYNC_TSC:
+        case SUPGIPMODE_INVARIANT_TSC:
+#ifdef IN_RING0
+            if (pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO)
+                pfnWorker = fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta    : RTTimeNanoTSLegacySyncInvarNoDelta;
+            else
+                pfnWorker = fLFence ? RTTimeNanoTSLFenceSyncInvarWithDelta  : RTTimeNanoTSLegacySyncInvarWithDelta;
+#else
+            if (pGip->fGetGipCpu & SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS)
+                pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_PRACTICALLY_ZERO
+                          ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta             : RTTimeNanoTSLegacySyncInvarNoDelta
+                          : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseIdtrLim : RTTimeNanoTSLegacySyncInvarWithDeltaUseIdtrLim;
+            else if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS)
+                pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_PRACTICALLY_ZERO
+                          ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta            : RTTimeNanoTSLegacySyncInvarNoDelta
+                          : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseRdtscp : RTTimeNanoTSLegacySyncInvarWithDeltaUseRdtscp;
+            else if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_0B)
+                pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO
+                          ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta                 : RTTimeNanoTSLegacySyncInvarNoDelta
+                          : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseApicIdExt0B : RTTimeNanoTSLegacySyncInvarWithDeltaUseApicIdExt0B;
+            else if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_8000001E)
+                pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO
+                          ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta                       : RTTimeNanoTSLegacySyncInvarNoDelta
+                          : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseApicIdExt8000001E : RTTimeNanoTSLegacySyncInvarWithDeltaUseApicIdExt8000001E;
+            else
+                pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO
+                          ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta            : RTTimeNanoTSLegacySyncInvarNoDelta
+                          : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseApicId : RTTimeNanoTSLegacySyncInvarWithDeltaUseApicId;
+#endif
+            break;
+
+        case SUPGIPMODE_ASYNC_TSC:
+#ifdef IN_RING0
+            pfnWorker = fLFence ? RTTimeNanoTSLFenceAsync : RTTimeNanoTSLegacyAsync;
+#else
+            if (pGip->fGetGipCpu & SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS)
+                pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseIdtrLim     : RTTimeNanoTSLegacyAsyncUseIdtrLim;
+            else if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS)
+                pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseRdtscp      : RTTimeNanoTSLegacyAsyncUseRdtscp;
+            else if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL)
+                pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseRdtscpGroupChNumCl : RTTimeNanoTSLegacyAsyncUseRdtscpGroupChNumCl;
+            else if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_0B)
+                pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseApicIdExt0B : RTTimeNanoTSLegacyAsyncUseApicIdExt0B;
+            else if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_8000001E)
+                pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseApicIdExt8000001E  : RTTimeNanoTSLegacyAsyncUseApicIdExt8000001E;
+            else
+                pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseApicId      : RTTimeNanoTSLegacyAsyncUseApicId;
+#endif
+            break;
+
+        default:
+            AssertFatalMsgFailed(("pVM=%p pGip=%p u32Mode=%#x\n", pVM, pGip, pGip->u32Mode));
+    }
+
+    /*
+     * Update the pfnVirtualGetRaw pointer and call the worker we selected.
+     */
+    ASMAtomicWritePtr((void * volatile *)&CTX_SUFF(pVM->tm.s.pfnVirtualGetRaw), (void *)(uintptr_t)pfnWorker);
+    return pfnWorker(pData);
+}
+
+
+/**
+ * @interface_method_impl{RTTIMENANOTSDATA,pfnBadCpuIndex}
+ */
+DECLEXPORT(uint64_t) tmVirtualNanoTSBadCpuIndex(PRTTIMENANOTSDATA pData, uint16_t idApic, uint16_t iCpuSet, uint16_t iGipCpu)
+{
+    PVM pVM = RT_FROM_MEMBER(pData, VM, CTX_SUFF(tm.s.VirtualGetRawData));
+    AssertFatalMsgFailed(("pVM=%p idApic=%#x iCpuSet=%#x iGipCpu=%#x\n", pVM, idApic, iCpuSet, iGipCpu));
+#ifndef _MSC_VER
+    return UINT64_MAX;
+#endif
+}
+
+
+/**
+ * Wrapper around the IPRT GIP time methods.
+ */
+DECLINLINE(uint64_t) tmVirtualGetRawNanoTS(PVMCC pVM)
+{
+# ifdef IN_RING3
+    uint64_t u64 = CTXALLSUFF(pVM->tm.s.pfnVirtualGetRaw)(&CTXALLSUFF(pVM->tm.s.VirtualGetRawData));
+# else  /* !IN_RING3 */
+    uint32_t cPrevSteps = pVM->tm.s.CTX_SUFF(VirtualGetRawData).c1nsSteps;
+    uint64_t u64 = pVM->tm.s.CTX_SUFF(pfnVirtualGetRaw)(&pVM->tm.s.CTX_SUFF(VirtualGetRawData));
+    if (cPrevSteps != pVM->tm.s.CTX_SUFF(VirtualGetRawData).c1nsSteps)
+        VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
+# endif /* !IN_RING3 */
+    /*DBGFTRACE_POS_U64(pVM, u64);*/
+    return u64;
+}
+
+
+/**
+ * Get the time when we're not running at 100%
+ *
+ * @returns The timestamp.
+ * @param   pVM     The cross context VM structure.
+ */
+static uint64_t tmVirtualGetRawNonNormal(PVMCC pVM)
+{
+    /*
+     * Recalculate the RTTimeNanoTS() value for the period where
+     * warp drive has been enabled.
+     */
+    uint64_t u64 = tmVirtualGetRawNanoTS(pVM);
+    u64 -= pVM->tm.s.u64VirtualWarpDriveStart;
+    u64 *= pVM->tm.s.u32VirtualWarpDrivePercentage;
+    u64 /= 100;
+    u64 += pVM->tm.s.u64VirtualWarpDriveStart;
+
+    /*
+     * Now we apply the virtual time offset.
+     * (Which is the negated tmVirtualGetRawNanoTS() value for when the virtual
+     * machine started if it had been running continuously without any suspends.)
+     */
+    u64 -= pVM->tm.s.u64VirtualOffset;
+    return u64;
+}
+
+
+/**
+ * Get the raw virtual time.
+ *
+ * @returns The current time stamp.
+ * @param   pVM     The cross context VM structure.
+ */
+DECLINLINE(uint64_t) tmVirtualGetRaw(PVMCC pVM)
+{
+    if (RT_LIKELY(!pVM->tm.s.fVirtualWarpDrive))
+        return tmVirtualGetRawNanoTS(pVM) - pVM->tm.s.u64VirtualOffset;
+    return tmVirtualGetRawNonNormal(pVM);
+}
+
+
+/**
+ * Inlined version of tmVirtualGetEx.
+ */
+DECLINLINE(uint64_t) tmVirtualGet(PVMCC pVM, bool fCheckTimers)
+{
+    uint64_t u64;
+    if (RT_LIKELY(pVM->tm.s.cVirtualTicking))
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualGet);
+        u64 = tmVirtualGetRaw(pVM);
+
+        /*
+         * Use the chance to check for expired timers.
+         */
+        if (fCheckTimers)
+        {
+            PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu);
+            if (    !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)
+                &&  !pVM->tm.s.fRunningQueues
+                &&  (   pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL].u64Expire <= u64
+                     || (   pVM->tm.s.fVirtualSyncTicking
+                         && pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire <= u64 - pVM->tm.s.offVirtualSync
+                        )
+                    )
+                &&  !pVM->tm.s.fRunningQueues
+               )
+            {
+                STAM_COUNTER_INC(&pVM->tm.s.StatVirtualGetSetFF);
+                Log5(("TMAllVirtual(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+                VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+#ifdef IN_RING3
+                VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM);
+#endif
+            }
+        }
+    }
+    else
+        u64 = pVM->tm.s.u64Virtual;
+    return u64;
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL time
+ *
+ * @returns The timestamp.
+ * @param   pVM     The cross context VM structure.
+ *
+ * @remark  While the flow of time will never go backwards, the speed of the
+ *          progress varies due to inaccurate RTTimeNanoTS and TSC. The latter can be
+ *          influenced by power saving (SpeedStep, PowerNow!), while the former
+ *          makes use of TSC and kernel timers.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualGet(PVMCC pVM)
+{
+    return tmVirtualGet(pVM, true /*fCheckTimers*/);
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL time without checking
+ * timers or anything.
+ *
+ * Meaning, this has no side effect on FFs like TMVirtualGet may have.
+ *
+ * @returns The timestamp.
+ * @param   pVM     The cross context VM structure.
+ *
+ * @remarks See TMVirtualGet.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualGetNoCheck(PVMCC pVM)
+{
+    return tmVirtualGet(pVM, false /*fCheckTimers*/);
+}
+
+
+/**
+ * Converts the dead line interval from TMCLOCK_VIRTUAL to host nano seconds.
+ *
+ * @returns Host nano second count.
+ * @param   pVM                     The cross context VM structure.
+ * @param   cVirtTicksToDeadline    The TMCLOCK_VIRTUAL interval.
+ */
+DECLINLINE(uint64_t) tmVirtualVirtToNsDeadline(PVM pVM, uint64_t cVirtTicksToDeadline)
+{
+    if (RT_UNLIKELY(pVM->tm.s.fVirtualWarpDrive))
+        return ASMMultU64ByU32DivByU32(cVirtTicksToDeadline, 100, pVM->tm.s.u32VirtualWarpDrivePercentage);
+    return cVirtTicksToDeadline;
+}
+
+
+/**
+ * tmVirtualSyncGetLocked worker for handling catch-up when owning the lock.
+ *
+ * @returns The timestamp.
+ * @param   pVM                 The cross context VM structure.
+ * @param   u64                 raw virtual time.
+ * @param   off                 offVirtualSync.
+ * @param   pcNsToDeadline      Where to return the number of nano seconds to
+ *                              the next virtual sync timer deadline. Can be
+ *                              NULL.
+ */
+DECLINLINE(uint64_t) tmVirtualSyncGetHandleCatchUpLocked(PVMCC pVM, uint64_t u64, uint64_t off, uint64_t *pcNsToDeadline)
+{
+    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLocked);
+
+    /*
+     * Don't make updates until we've check the timer queue.
+     */
+    bool            fUpdatePrev = true;
+    bool            fUpdateOff  = true;
+    bool            fStop       = false;
+    const uint64_t  u64Prev     = pVM->tm.s.u64VirtualSyncCatchUpPrev;
+    uint64_t        u64Delta    = u64 - u64Prev;
+    if (RT_LIKELY(!(u64Delta >> 32)))
+    {
+        uint64_t u64Sub = ASMMultU64ByU32DivByU32(u64Delta, pVM->tm.s.u32VirtualSyncCatchUpPercentage, 100);
+        if (off > u64Sub + pVM->tm.s.offVirtualSyncGivenUp)
+        {
+            off -= u64Sub;
+            Log4(("TM: %'RU64/-%'8RU64: sub %RU32 [vsghcul]\n", u64 - off, off - pVM->tm.s.offVirtualSyncGivenUp, u64Sub));
+        }
+        else
+        {
+            /* we've completely caught up. */
+            STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
+            off = pVM->tm.s.offVirtualSyncGivenUp;
+            fStop = true;
+            Log4(("TM: %'RU64/0: caught up [vsghcul]\n", u64));
+        }
+    }
+    else
+    {
+        /* More than 4 seconds since last time (or negative), ignore it. */
+        fUpdateOff = false;
+        fUpdatePrev = !(u64Delta & RT_BIT_64(63));
+        Log(("TMVirtualGetSync: u64Delta=%RX64\n", u64Delta));
+    }
+
+    /*
+     * Complete the calculation of the current TMCLOCK_VIRTUAL_SYNC time. The current
+     * approach is to never pass the head timer. So, when we do stop the clock and
+     * set the timer pending flag.
+     */
+    u64 -= off;
+
+    uint64_t u64Last = ASMAtomicUoReadU64(&pVM->tm.s.u64VirtualSync);
+    if (u64Last > u64)
+    {
+        u64 = u64Last + 1;
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetAdjLast);
+    }
+
+    uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+    if (u64 < u64Expire)
+    {
+        ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64);
+        if (fUpdateOff)
+            ASMAtomicWriteU64(&pVM->tm.s.offVirtualSync, off);
+        if (fStop)
+            ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
+        if (fUpdatePrev)
+            ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev, u64);
+        if (pcNsToDeadline)
+        {
+            uint64_t cNsToDeadline = u64Expire - u64;
+            if (pVM->tm.s.fVirtualSyncCatchUp)
+                cNsToDeadline = ASMMultU64ByU32DivByU32(cNsToDeadline, 100,
+                                                        pVM->tm.s.u32VirtualSyncCatchUpPercentage + 100);
+            *pcNsToDeadline = tmVirtualVirtToNsDeadline(pVM, cNsToDeadline);
+        }
+        PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+    }
+    else
+    {
+        u64 = u64Expire;
+        ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64);
+        ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false);
+
+        VM_FF_SET(pVM, VM_FF_TM_VIRTUAL_SYNC);
+        PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu);
+        VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+        Log5(("TMAllVirtual(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+        Log4(("TM: %'RU64/-%'8RU64: exp tmr=>ff [vsghcul]\n", u64, pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp));
+        PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+
+        if (pcNsToDeadline)
+            *pcNsToDeadline = 0;
+#ifdef IN_RING3
+        VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM);
+#endif
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetSetFF);
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetExpired);
+    }
+    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLocked);
+
+    Log6(("tmVirtualSyncGetHandleCatchUpLocked -> %'RU64\n", u64));
+    DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetHandleCatchUpLocked");
+    return u64;
+}
+
+
+/**
+ * tmVirtualSyncGetEx worker for when we get the lock.
+ *
+ * @returns timesamp.
+ * @param   pVM                 The cross context VM structure.
+ * @param   u64                 The virtual clock timestamp.
+ * @param   pcNsToDeadline      Where to return the number of nano seconds to
+ *                              the next virtual sync timer deadline.  Can be
+ *                              NULL.
+ */
+DECLINLINE(uint64_t) tmVirtualSyncGetLocked(PVMCC pVM, uint64_t u64, uint64_t *pcNsToDeadline)
+{
+    /*
+     * Not ticking?
+     */
+    if (!pVM->tm.s.fVirtualSyncTicking)
+    {
+        u64 = ASMAtomicUoReadU64(&pVM->tm.s.u64VirtualSync);
+        PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+        if (pcNsToDeadline)
+            *pcNsToDeadline = 0;
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLocked);
+        Log6(("tmVirtualSyncGetLocked -> %'RU64 [stopped]\n", u64));
+        DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetLocked-stopped");
+        return u64;
+    }
+
+    /*
+     * Handle catch up in a separate function.
+     */
+    uint64_t off = ASMAtomicUoReadU64(&pVM->tm.s.offVirtualSync);
+    if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+        return tmVirtualSyncGetHandleCatchUpLocked(pVM, u64, off, pcNsToDeadline);
+
+    /*
+     * Complete the calculation of the current TMCLOCK_VIRTUAL_SYNC time. The current
+     * approach is to never pass the head timer. So, when we do stop the clock and
+     * set the timer pending flag.
+     */
+    u64 -= off;
+
+    uint64_t u64Last = ASMAtomicUoReadU64(&pVM->tm.s.u64VirtualSync);
+    if (u64Last > u64)
+    {
+        u64 = u64Last + 1;
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetAdjLast);
+    }
+
+    uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+    if (u64 < u64Expire)
+    {
+        ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64);
+        PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+        if (pcNsToDeadline)
+            *pcNsToDeadline = tmVirtualVirtToNsDeadline(pVM, u64Expire - u64);
+    }
+    else
+    {
+        u64 = u64Expire;
+        ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64);
+        ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false);
+
+        VM_FF_SET(pVM, VM_FF_TM_VIRTUAL_SYNC);
+        PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu);
+        VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+        Log5(("TMAllVirtual(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+        Log4(("TM: %'RU64/-%'8RU64: exp tmr=>ff [vsgl]\n", u64, pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp));
+        PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+
+#ifdef IN_RING3
+        VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM);
+#endif
+        if (pcNsToDeadline)
+            *pcNsToDeadline = 0;
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetSetFF);
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetExpired);
+    }
+    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLocked);
+    Log6(("tmVirtualSyncGetLocked -> %'RU64\n", u64));
+    DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetLocked");
+    return u64;
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL_SYNC time.
+ *
+ * @returns The timestamp.
+ * @param   pVM                 The cross context VM structure.
+ * @param   fCheckTimers        Check timers or not
+ * @param   pcNsToDeadline      Where to return the number of nano seconds to
+ *                              the next virtual sync timer deadline.  Can be
+ *                              NULL.
+ * @thread  EMT.
+ */
+DECLINLINE(uint64_t) tmVirtualSyncGetEx(PVMCC pVM, bool fCheckTimers, uint64_t *pcNsToDeadline)
+{
+    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGet);
+
+    uint64_t u64;
+    if (!pVM->tm.s.fVirtualSyncTicking)
+    {
+        if (pcNsToDeadline)
+            *pcNsToDeadline = 0;
+        u64 = pVM->tm.s.u64VirtualSync;
+        DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetEx-stopped1");
+        return u64;
+    }
+
+    /*
+     * Query the virtual clock and do the usual expired timer check.
+     */
+    Assert(pVM->tm.s.cVirtualTicking);
+    u64 = tmVirtualGetRaw(pVM);
+    if (fCheckTimers)
+    {
+        PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu);
+        if (    !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)
+            &&  pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL].u64Expire <= u64)
+        {
+            Log5(("TMAllVirtual(%u): FF: 0 -> 1\n", __LINE__));
+            VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+#ifdef IN_RING3
+            VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM /** @todo |VMNOTIFYFF_FLAGS_POKE*/);
+#endif
+            STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetSetFF);
+        }
+    }
+
+    /*
+     * If we can get the lock, get it.  The result is much more reliable.
+     *
+     * Note! This is where all clock source devices branch off because they
+     *       will be owning the lock already.  The 'else' is taken by code
+     *       which is less picky or hasn't been adjusted yet
+     */
+    if (PDMCritSectTryEnter(&pVM->tm.s.VirtualSyncLock) == VINF_SUCCESS)
+        return tmVirtualSyncGetLocked(pVM, u64, pcNsToDeadline);
+
+    /*
+     * When the clock is ticking, not doing catch ups and not running into an
+     * expired time, we can get away without locking. Try this first.
+     */
+    uint64_t off;
+    if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking))
+    {
+        if (!ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+        {
+            off = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync);
+            if (RT_LIKELY(   ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking)
+                          && !ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp)
+                          && off == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync)))
+            {
+                off = u64 - off;
+                uint64_t const u64Expire = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+                if (off < u64Expire)
+                {
+                    if (pcNsToDeadline)
+                        *pcNsToDeadline = tmVirtualVirtToNsDeadline(pVM, u64Expire - off);
+                    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLockless);
+                    Log6(("tmVirtualSyncGetEx -> %'RU64 [lockless]\n", off));
+                    DBGFTRACE_U64_TAG(pVM, off, "tmVirtualSyncGetEx-lockless");
+                    return off;
+                }
+            }
+        }
+    }
+    else
+    {
+        off = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSync);
+        if (RT_LIKELY(!ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking)))
+        {
+            if (pcNsToDeadline)
+                *pcNsToDeadline = 0;
+            STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLockless);
+            Log6(("tmVirtualSyncGetEx -> %'RU64 [lockless/stopped]\n", off));
+            DBGFTRACE_U64_TAG(pVM, off, "tmVirtualSyncGetEx-stopped2");
+            return off;
+        }
+    }
+
+    /*
+     * Read the offset and adjust if we're playing catch-up.
+     *
+     * The catch-up adjusting work by us decrementing the offset by a percentage of
+     * the time elapsed since the previous TMVirtualGetSync call.
+     *
+     * It's possible to get a very long or even negative interval between two read
+     * for the following reasons:
+     *  - Someone might have suspended the process execution, frequently the case when
+     *    debugging the process.
+     *  - We might be on a different CPU which TSC isn't quite in sync with the
+     *    other CPUs in the system.
+     *  - Another thread is racing us and we might have been preempted while inside
+     *    this function.
+     *
+     * Assuming nano second virtual time, we can simply ignore any intervals which has
+     * any of the upper 32 bits set.
+     */
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    int cOuterTries = 42;
+    for (;; cOuterTries--)
+    {
+        /* Try grab the lock, things get simpler when owning the lock. */
+        int rcLock = PDMCritSectTryEnter(&pVM->tm.s.VirtualSyncLock);
+        if (RT_SUCCESS_NP(rcLock))
+            return tmVirtualSyncGetLocked(pVM, u64, pcNsToDeadline);
+
+        /* Re-check the ticking flag. */
+        if (!ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking))
+        {
+            off = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSync);
+            if (   ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking)
+                && cOuterTries > 0)
+                continue;
+            if (pcNsToDeadline)
+                *pcNsToDeadline = 0;
+            Log6(("tmVirtualSyncGetEx -> %'RU64 [stopped]\n", off));
+            DBGFTRACE_U64_TAG(pVM, off, "tmVirtualSyncGetEx-stopped3");
+            return off;
+        }
+
+        off = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync);
+        if (ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+        {
+            /* No changes allowed, try get a consistent set of parameters. */
+            uint64_t const u64Prev    = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev);
+            uint64_t const offGivenUp = ASMAtomicReadU64(&pVM->tm.s.offVirtualSyncGivenUp);
+            uint32_t const u32Pct     = ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage);
+            if (    (   u64Prev    == ASMAtomicReadU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev)
+                     && offGivenUp == ASMAtomicReadU64(&pVM->tm.s.offVirtualSyncGivenUp)
+                     && u32Pct     == ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage)
+                     && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+                ||  cOuterTries <= 0)
+            {
+                uint64_t u64Delta = u64 - u64Prev;
+                if (RT_LIKELY(!(u64Delta >> 32)))
+                {
+                    uint64_t u64Sub = ASMMultU64ByU32DivByU32(u64Delta, u32Pct, 100);
+                    if (off > u64Sub + offGivenUp)
+                    {
+                        off -= u64Sub;
+                        Log4(("TM: %'RU64/-%'8RU64: sub %RU32 [NoLock]\n", u64 - off, pVM->tm.s.offVirtualSync - offGivenUp, u64Sub));
+                    }
+                    else
+                    {
+                        /* we've completely caught up. */
+                        STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
+                        off = offGivenUp;
+                        Log4(("TM: %'RU64/0: caught up [NoLock]\n", u64));
+                    }
+                }
+                else
+                    /* More than 4 seconds since last time (or negative), ignore it. */
+                    Log(("TMVirtualGetSync: u64Delta=%RX64 (NoLock)\n", u64Delta));
+
+                /* Check that we're still running and in catch up. */
+                if (    ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking)
+                    &&  ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+                    break;
+                if (cOuterTries <= 0)
+                    break; /* enough */
+            }
+        }
+        else if (   off == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync)
+                 && !ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+            break; /* Got an consistent offset */
+        else if (cOuterTries <= 0)
+            break; /* enough */
+    }
+    if (cOuterTries <= 0)
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetELoop);
+
+    /*
+     * Complete the calculation of the current TMCLOCK_VIRTUAL_SYNC time. The current
+     * approach is to never pass the head timer. So, when we do stop the clock and
+     * set the timer pending flag.
+     */
+    u64 -= off;
+/** @todo u64VirtualSyncLast */
+    uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+    if (u64 >= u64Expire)
+    {
+        PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu);
+        if (!VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+        {
+            Log5(("TMAllVirtual(%u): FF: %d -> 1 (NoLock)\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+            VM_FF_SET(pVM, VM_FF_TM_VIRTUAL_SYNC); /* Hmm? */
+            VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+#ifdef IN_RING3
+            VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM);
+#endif
+            STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetSetFF);
+            Log4(("TM: %'RU64/-%'8RU64: exp tmr=>ff [NoLock]\n", u64, pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp));
+        }
+        else
+            Log4(("TM: %'RU64/-%'8RU64: exp tmr [NoLock]\n", u64, pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp));
+        if (pcNsToDeadline)
+            *pcNsToDeadline = 0;
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetExpired);
+    }
+    else if (pcNsToDeadline)
+    {
+        uint64_t cNsToDeadline = u64Expire - u64;
+        if (ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+            cNsToDeadline = ASMMultU64ByU32DivByU32(cNsToDeadline, 100,
+                                                    ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage) + 100);
+        *pcNsToDeadline = tmVirtualVirtToNsDeadline(pVM, cNsToDeadline);
+    }
+
+    Log6(("tmVirtualSyncGetEx -> %'RU64\n", u64));
+    DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetEx-nolock");
+    return u64;
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL_SYNC time.
+ *
+ * @returns The timestamp.
+ * @param   pVM             The cross context VM structure.
+ * @thread  EMT.
+ * @remarks May set the timer and virtual sync FFs.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualSyncGet(PVMCC pVM)
+{
+    return tmVirtualSyncGetEx(pVM, true /*fCheckTimers*/, NULL /*pcNsToDeadline*/);
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL_SYNC time without checking timers running on
+ * TMCLOCK_VIRTUAL.
+ *
+ * @returns The timestamp.
+ * @param   pVM             The cross context VM structure.
+ * @thread  EMT.
+ * @remarks May set the timer and virtual sync FFs.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualSyncGetNoCheck(PVMCC pVM)
+{
+    return tmVirtualSyncGetEx(pVM, false /*fCheckTimers*/, NULL /*pcNsToDeadline*/);
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL_SYNC time.
+ *
+ * @returns The timestamp.
+ * @param   pVM     The cross context VM structure.
+ * @param   fCheckTimers    Check timers on the virtual clock or not.
+ * @thread  EMT.
+ * @remarks May set the timer and virtual sync FFs.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualSyncGetEx(PVMCC pVM, bool fCheckTimers)
+{
+    return tmVirtualSyncGetEx(pVM, fCheckTimers, NULL /*pcNsToDeadline*/);
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL_SYNC time and ticks to the next deadline
+ * without checking timers running on TMCLOCK_VIRTUAL.
+ *
+ * @returns The timestamp.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pcNsToDeadline      Where to return the number of nano seconds to
+ *                              the next virtual sync timer deadline.
+ * @thread  EMT.
+ * @remarks May set the timer and virtual sync FFs.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualSyncGetWithDeadlineNoCheck(PVMCC pVM, uint64_t *pcNsToDeadline)
+{
+    uint64_t cNsToDeadlineTmp;       /* try convince the compiler to skip the if tests. */
+    uint64_t u64Now = tmVirtualSyncGetEx(pVM, false /*fCheckTimers*/, &cNsToDeadlineTmp);
+    *pcNsToDeadline = cNsToDeadlineTmp;
+    return u64Now;
+}
+
+
+/**
+ * Gets the number of nano seconds to the next virtual sync deadline.
+ *
+ * @returns The number of TMCLOCK_VIRTUAL ticks.
+ * @param   pVM                 The cross context VM structure.
+ * @thread  EMT.
+ * @remarks May set the timer and virtual sync FFs.
+ */
+VMMDECL(uint64_t) TMVirtualSyncGetNsToDeadline(PVMCC pVM)
+{
+    uint64_t cNsToDeadline;
+    tmVirtualSyncGetEx(pVM, false /*fCheckTimers*/, &cNsToDeadline);
+    return cNsToDeadline;
+}
+
+
+/**
+ * Gets the current lag of the synchronous virtual clock (relative to the virtual clock).
+ *
+ * @return  The current lag.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualSyncGetLag(PVMCC pVM)
+{
+    return pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp;
+}
+
+
+/**
+ * Get the current catch-up percent.
+ *
+ * @return  The current catch0up percent. 0 means running at the same speed as the virtual clock.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(uint32_t) TMVirtualSyncGetCatchUpPct(PVMCC pVM)
+{
+    if (pVM->tm.s.fVirtualSyncCatchUp)
+        return pVM->tm.s.u32VirtualSyncCatchUpPercentage;
+    return 0;
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL frequency.
+ *
+ * @returns The frequency.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualGetFreq(PVM pVM)
+{
+    NOREF(pVM);
+    return TMCLOCK_FREQ_VIRTUAL;
+}
+
+
+/**
+ * Worker for TMR3PauseClocks.
+ *
+ * @returns VINF_SUCCESS or VERR_TM_VIRTUAL_TICKING_IPE (asserted).
+ * @param   pVM     The cross context VM structure.
+ */
+int tmVirtualPauseLocked(PVMCC pVM)
+{
+    uint32_t c = ASMAtomicDecU32(&pVM->tm.s.cVirtualTicking);
+    AssertMsgReturn(c < pVM->cCpus, ("%u vs %u\n", c, pVM->cCpus), VERR_TM_VIRTUAL_TICKING_IPE);
+    if (c == 0)
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualPause);
+        pVM->tm.s.u64Virtual = tmVirtualGetRaw(pVM);
+        ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for TMR3ResumeClocks.
+ *
+ * @returns VINF_SUCCESS or VERR_TM_VIRTUAL_TICKING_IPE (asserted).
+ * @param   pVM     The cross context VM structure.
+ */
+int tmVirtualResumeLocked(PVMCC pVM)
+{
+    uint32_t c = ASMAtomicIncU32(&pVM->tm.s.cVirtualTicking);
+    AssertMsgReturn(c <= pVM->cCpus, ("%u vs %u\n", c, pVM->cCpus), VERR_TM_VIRTUAL_TICKING_IPE);
+    if (c == 1)
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualResume);
+        pVM->tm.s.u64VirtualRawPrev         = 0;
+        pVM->tm.s.u64VirtualWarpDriveStart  = tmVirtualGetRawNanoTS(pVM);
+        pVM->tm.s.u64VirtualOffset          = pVM->tm.s.u64VirtualWarpDriveStart - pVM->tm.s.u64Virtual;
+        ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, true);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Converts from virtual ticks to nanoseconds.
+ *
+ * @returns nanoseconds.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64VirtualTicks The virtual ticks to convert.
+ * @remark  There could be rounding errors here. We just do a simple integer divide
+ *          without any adjustments.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualToNano(PVM pVM, uint64_t u64VirtualTicks)
+{
+    NOREF(pVM);
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    return u64VirtualTicks;
+}
+
+
+/**
+ * Converts from virtual ticks to microseconds.
+ *
+ * @returns microseconds.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64VirtualTicks The virtual ticks to convert.
+ * @remark  There could be rounding errors here. We just do a simple integer divide
+ *          without any adjustments.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualToMicro(PVM pVM, uint64_t u64VirtualTicks)
+{
+    NOREF(pVM);
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    return u64VirtualTicks / 1000;
+}
+
+
+/**
+ * Converts from virtual ticks to milliseconds.
+ *
+ * @returns milliseconds.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64VirtualTicks The virtual ticks to convert.
+ * @remark  There could be rounding errors here. We just do a simple integer divide
+ *          without any adjustments.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualToMilli(PVM pVM, uint64_t u64VirtualTicks)
+{
+    NOREF(pVM);
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    return u64VirtualTicks / 1000000;
+}
+
+
+/**
+ * Converts from nanoseconds to virtual ticks.
+ *
+ * @returns virtual ticks.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64NanoTS       The nanosecond value ticks to convert.
+ * @remark  There could be rounding and overflow errors here.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualFromNano(PVM pVM, uint64_t u64NanoTS)
+{
+    NOREF(pVM);
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    return u64NanoTS;
+}
+
+
+/**
+ * Converts from microseconds to virtual ticks.
+ *
+ * @returns virtual ticks.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64MicroTS      The microsecond value ticks to convert.
+ * @remark  There could be rounding and overflow errors here.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualFromMicro(PVM pVM, uint64_t u64MicroTS)
+{
+    NOREF(pVM);
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    return u64MicroTS * 1000;
+}
+
+
+/**
+ * Converts from milliseconds to virtual ticks.
+ *
+ * @returns virtual ticks.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64MilliTS      The millisecond value ticks to convert.
+ * @remark  There could be rounding and overflow errors here.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualFromMilli(PVM pVM, uint64_t u64MilliTS)
+{
+    NOREF(pVM);
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    return u64MilliTS * 1000000;
+}
+
diff --git a/src/VBox/VMM/VMMAll/TRPMAll.cpp b/src/VBox/VMM/VMMAll/TRPMAll.cpp
new file mode 100644
index 00000000..85427118
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/TRPMAll.cpp
@@ -0,0 +1,429 @@
+/* $Id: TRPMAll.cpp $ */
+/** @file
+ * TRPM - Trap Monitor - Any Context.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_TRPM
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/dbgf.h>
+#include "TRPMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/vmm/em.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/param.h>
+#include <iprt/x86.h>
+
+
+
+/**
+ * Query info about the current active trap/interrupt.
+ * If no trap is active active an error code is returned.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   pu8TrapNo               Where to store the trap number.
+ * @param   penmType                Where to store the trap type
+ */
+VMMDECL(int) TRPMQueryTrap(PVMCPU pVCpu, uint8_t *pu8TrapNo, TRPMEVENT *penmType)
+{
+    /*
+     * Check if we have a trap at present.
+     */
+    if (pVCpu->trpm.s.uActiveVector != ~0U)
+    {
+        if (pu8TrapNo)
+            *pu8TrapNo = (uint8_t)pVCpu->trpm.s.uActiveVector;
+        if (penmType)
+            *penmType = pVCpu->trpm.s.enmActiveType;
+        return VINF_SUCCESS;
+    }
+
+    return VERR_TRPM_NO_ACTIVE_TRAP;
+}
+
+
+/**
+ * Gets the trap number for the current trap.
+ *
+ * The caller is responsible for making sure there is an active trap which
+ * takes an error code when making this request.
+ *
+ * @returns The current trap number.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ */
+VMMDECL(uint8_t) TRPMGetTrapNo(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    return (uint8_t)pVCpu->trpm.s.uActiveVector;
+}
+
+
+/**
+ * Gets the error code for the current trap.
+ *
+ * The caller is responsible for making sure there is an active trap which
+ * takes an error code when making this request.
+ *
+ * @returns Error code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ */
+VMMDECL(uint32_t) TRPMGetErrorCode(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+#ifdef VBOX_STRICT
+    switch (pVCpu->trpm.s.uActiveVector)
+    {
+        case X86_XCPT_TS:
+        case X86_XCPT_NP:
+        case X86_XCPT_SS:
+        case X86_XCPT_GP:
+        case X86_XCPT_PF:
+        case X86_XCPT_AC:
+        case X86_XCPT_DF:
+            break;
+        default:
+            AssertMsgFailed(("This trap (%#x) doesn't have any error code\n", pVCpu->trpm.s.uActiveVector));
+            break;
+    }
+#endif
+    return pVCpu->trpm.s.uActiveErrorCode;
+}
+
+
+/**
+ * Gets the fault address for the current trap.
+ *
+ * The caller is responsible for making sure there is an active trap 0x0e when
+ * making this request.
+ *
+ * @returns Fault address associated with the trap.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ */
+VMMDECL(RTGCUINTPTR) TRPMGetFaultAddress(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    AssertMsg(pVCpu->trpm.s.uActiveVector == X86_XCPT_PF, ("Not page-fault trap!\n"));
+    return pVCpu->trpm.s.uActiveCR2;
+}
+
+
+/**
+ * Gets the instruction-length for the current trap (only relevant for software
+ * interrupts and software exceptions \#BP and \#OF).
+ *
+ * The caller is responsible for making sure there is an active trap 0x0e when
+ * making this request.
+ *
+ * @returns Fault address associated with the trap.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ */
+VMMDECL(uint8_t) TRPMGetInstrLength(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    return pVCpu->trpm.s.cbInstr;
+}
+
+
+/**
+ * Checks if the current \#DB exception is due to an INT1/ICEBP instruction.
+ *
+ * The caller is responsible for making sure there is an active trap.
+ *
+ * @returns @c true if it's due to INT1/ICEBP, @c false if not.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMDECL(bool) TRPMIsTrapDueToIcebp(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    return pVCpu->trpm.s.fIcebp;
+}
+
+
+/**
+ * Clears the current active trap/exception/interrupt.
+ *
+ * The caller is responsible for making sure there is an active trap
+ * when making this request.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ */
+VMMDECL(int) TRPMResetTrap(PVMCPU pVCpu)
+{
+    /*
+     * Cannot reset non-existing trap!
+     */
+    if (pVCpu->trpm.s.uActiveVector == ~0U)
+    {
+        AssertMsgFailed(("No active trap!\n"));
+        return VERR_TRPM_NO_ACTIVE_TRAP;
+    }
+
+    /*
+     * Reset it.
+     */
+    pVCpu->trpm.s.uActiveVector = ~0U;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Assert trap/exception/interrupt.
+ *
+ * The caller is responsible for making sure there is no active trap
+ * when making this request.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   u8TrapNo            The trap vector to assert.
+ * @param   enmType             Trap type.
+ */
+VMMDECL(int) TRPMAssertTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType)
+{
+    Log2(("TRPMAssertTrap: u8TrapNo=%02x type=%d\n", u8TrapNo, enmType));
+
+    /*
+     * Cannot assert a trap when one is already active.
+     */
+    if (pVCpu->trpm.s.uActiveVector != ~0U)
+    {
+        AssertMsgFailed(("CPU%d: Active trap %#x\n", pVCpu->idCpu, pVCpu->trpm.s.uActiveVector));
+        return VERR_TRPM_ACTIVE_TRAP;
+    }
+
+    pVCpu->trpm.s.uActiveVector               = u8TrapNo;
+    pVCpu->trpm.s.enmActiveType               = enmType;
+    pVCpu->trpm.s.uActiveErrorCode            = ~0U;
+    pVCpu->trpm.s.uActiveCR2                  = 0xdeadface;
+    pVCpu->trpm.s.cbInstr                     = UINT8_MAX;
+    pVCpu->trpm.s.fIcebp                      = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Assert a page-fault exception.
+ *
+ * The caller is responsible for making sure there is no active trap
+ * when making this request.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uCR2                The new fault address.
+ * @param   uErrorCode          The error code for the page-fault.
+ */
+VMMDECL(int) TRPMAssertXcptPF(PVMCPUCC pVCpu, RTGCUINTPTR uCR2, uint32_t uErrorCode)
+{
+    Log2(("TRPMAssertXcptPF: uCR2=%RGv uErrorCode=%#RX32\n", uCR2, uErrorCode));
+
+    /*
+     * Cannot assert a trap when one is already active.
+     */
+    if (pVCpu->trpm.s.uActiveVector != ~0U)
+    {
+        AssertMsgFailed(("CPU%d: Active trap %#x\n", pVCpu->idCpu, pVCpu->trpm.s.uActiveVector));
+        return VERR_TRPM_ACTIVE_TRAP;
+    }
+
+    pVCpu->trpm.s.uActiveVector               = X86_XCPT_PF;
+    pVCpu->trpm.s.enmActiveType               = TRPM_TRAP;
+    pVCpu->trpm.s.uActiveErrorCode            = uErrorCode;
+    pVCpu->trpm.s.uActiveCR2                  = uCR2;
+    pVCpu->trpm.s.cbInstr                     = UINT8_MAX;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the error code of the current trap.
+ * (This function is for use in trap handlers and such.)
+ *
+ * The caller is responsible for making sure there is an active trap
+ * which takes an errorcode when making this request.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uErrorCode          The new error code.
+ */
+VMMDECL(void) TRPMSetErrorCode(PVMCPU pVCpu, uint32_t uErrorCode)
+{
+    Log2(("TRPMSetErrorCode: uErrorCode=%#RX32\n", uErrorCode));
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    AssertMsg(   pVCpu->trpm.s.enmActiveType == TRPM_TRAP
+              || (   pVCpu->trpm.s.enmActiveType == TRPM_SOFTWARE_INT && pVCpu->trpm.s.uActiveVector == X86_XCPT_DB),
+              ("Not hardware exception or privileged software exception (INT1/ICEBP)!\n"));
+    pVCpu->trpm.s.uActiveErrorCode = uErrorCode;
+#ifdef VBOX_STRICT
+    if (pVCpu->trpm.s.enmActiveType == TRPM_TRAP)
+    {
+        switch (pVCpu->trpm.s.uActiveVector)
+        {
+            case X86_XCPT_TS: case X86_XCPT_NP: case X86_XCPT_SS: case X86_XCPT_GP: case X86_XCPT_PF:
+                AssertMsg(uErrorCode != ~0U, ("Invalid uErrorCode=%#x u8TrapNo=%u\n", uErrorCode, pVCpu->trpm.s.uActiveVector));
+                break;
+            case X86_XCPT_AC: case X86_XCPT_DF:
+                AssertMsg(uErrorCode == 0,   ("Invalid uErrorCode=%#x u8TrapNo=%u\n", uErrorCode, pVCpu->trpm.s.uActiveVector));
+                break;
+            default:
+                AssertMsg(uErrorCode == ~0U, ("Invalid uErrorCode=%#x u8TrapNo=%u\n", uErrorCode, pVCpu->trpm.s.uActiveVector));
+                break;
+        }
+    }
+#endif
+}
+
+
+/**
+ * Sets the fault address of the current \#PF trap. (This function is for use in
+ * trap handlers and such.)
+ *
+ * The caller is responsible for making sure there is an active trap 0e
+ * when making this request.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uCR2                The new fault address (cr2 register).
+ */
+VMMDECL(void) TRPMSetFaultAddress(PVMCPU pVCpu, RTGCUINTPTR uCR2)
+{
+    Log2(("TRPMSetFaultAddress: uCR2=%RGv\n", uCR2));
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    AssertMsg(pVCpu->trpm.s.enmActiveType == TRPM_TRAP, ("Not hardware exception!\n"));
+    AssertMsg(pVCpu->trpm.s.uActiveVector == X86_XCPT_PF, ("Not trap 0e!\n"));
+    pVCpu->trpm.s.uActiveCR2 = uCR2;
+}
+
+
+/**
+ * Sets the instruction-length of the current trap (relevant for software
+ * interrupts and software exceptions like \#BP, \#OF).
+ *
+ * The caller is responsible for making sure there is an active trap when making
+ * this request.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length.
+ */
+VMMDECL(void) TRPMSetInstrLength(PVMCPU pVCpu, uint8_t cbInstr)
+{
+    Log2(("TRPMSetInstrLength: cbInstr=%u\n", cbInstr));
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    AssertMsg(   pVCpu->trpm.s.enmActiveType == TRPM_SOFTWARE_INT
+              || (   pVCpu->trpm.s.enmActiveType == TRPM_TRAP
+                  && (   pVCpu->trpm.s.uActiveVector == X86_XCPT_BP
+                      || pVCpu->trpm.s.uActiveVector == X86_XCPT_OF)),
+              ("Invalid trap type %#x\n", pVCpu->trpm.s.enmActiveType));
+    pVCpu->trpm.s.cbInstr = cbInstr;
+}
+
+
+/**
+ * Sets if the current \#DB exception is due to an INT1/ICEBP instruction.
+ *
+ * The caller is responsible for making sure there is an active trap and it's a
+ * \#DB.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMDECL(void) TRPMSetTrapDueToIcebp(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.enmActiveType == TRPM_SOFTWARE_INT, ("Trap type for INT1/ICEBP invalid!"));
+    AssertMsg(pVCpu->trpm.s.uActiveVector == X86_XCPT_DB, ("INT1/ICEBP must be indicated by a #DB!\n"));
+    pVCpu->trpm.s.fIcebp = true;
+}
+
+
+/**
+ * Checks if the current active trap/interrupt/exception/fault/whatever is a software
+ * interrupt or not.
+ *
+ * The caller is responsible for making sure there is an active trap
+ * when making this request.
+ *
+ * @returns true if software interrupt, false if not.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMDECL(bool) TRPMIsSoftwareInterrupt(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    return (pVCpu->trpm.s.enmActiveType == TRPM_SOFTWARE_INT);
+}
+
+
+/**
+ * Check if there is an active trap.
+ *
+ * @returns true if trap active, false if not.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMDECL(bool) TRPMHasTrap(PVMCPU pVCpu)
+{
+    return pVCpu->trpm.s.uActiveVector != ~0U;
+}
+
+
+/**
+ * Query all info about the current active trap/interrupt.
+ * If no trap is active active an error code is returned.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pu8TrapNo       Where to store the trap number.
+ * @param   pEnmType        Where to store the trap type.
+ * @param   puErrorCode     Where to store the error code associated with some
+ *                          traps. ~0U is stored if the trap has no error code.
+ * @param   puCR2           Where to store the CR2 associated with a trap 0E.
+ * @param   pcbInstr        Where to store the instruction-length associated with
+ *                          some traps.
+ * @param   pfIcebp         Where to store whether the trap is a \#DB caused by an
+ *                          INT1/ICEBP instruction.
+ */
+VMMDECL(int) TRPMQueryTrapAll(PVMCPU pVCpu, uint8_t *pu8TrapNo, TRPMEVENT *pEnmType, uint32_t *puErrorCode, PRTGCUINTPTR puCR2,
+                              uint8_t *pcbInstr, bool *pfIcebp)
+{
+    /*
+     * Check if we have an active trap.
+     */
+    if (pVCpu->trpm.s.uActiveVector == ~0U)
+        return VERR_TRPM_NO_ACTIVE_TRAP;
+
+    if (pu8TrapNo)
+        *pu8TrapNo      = (uint8_t)pVCpu->trpm.s.uActiveVector;
+    if (pEnmType)
+        *pEnmType       = pVCpu->trpm.s.enmActiveType;
+    if (puErrorCode)
+        *puErrorCode    = pVCpu->trpm.s.uActiveErrorCode;
+    if (puCR2)
+        *puCR2          = pVCpu->trpm.s.uActiveCR2;
+    if (pcbInstr)
+        *pcbInstr       = pVCpu->trpm.s.cbInstr;
+    if (pfIcebp)
+        *pfIcebp        = pVCpu->trpm.s.fIcebp;
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMAll/VMAll.cpp b/src/VBox/VMM/VMMAll/VMAll.cpp
new file mode 100644
index 00000000..835d83f0
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/VMAll.cpp
@@ -0,0 +1,429 @@
+/* $Id: VMAll.cpp $ */
+/** @file
+ * VM - Virtual Machine All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VM
+#include "VMInternal.h"
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+
+/**
+ * Sets the error message.
+ *
+ * @returns rc. Meaning you can do:
+ *    @code
+ *    return VM_SET_ERROR(pVM, VERR_OF_YOUR_CHOICE, "descriptive message");
+ *    @endcode
+ * @param   pVM             The cross context VM structure.
+ * @param   rc              VBox status code.
+ * @param   SRC_POS         Use RT_SRC_POS.
+ * @param   pszFormat       Error message format string.
+ * @param   ...             Error message arguments.
+ * @thread  Any
+ */
+VMMDECL(int) VMSetError(PVMCC pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    va_list args;
+    va_start(args, pszFormat);
+    int rc2 = VMSetErrorV(pVM, rc, RT_SRC_POS_ARGS, pszFormat, args); Assert(rc == rc2); NOREF(rc2);
+    va_end(args);
+    return rc;
+}
+
+
+/**
+ * Sets the error message.
+ *
+ * @returns rc. Meaning you can do:
+ *    @code
+ *    return VM_SET_ERROR(pVM, VERR_OF_YOUR_CHOICE, "descriptive message");
+ *    @endcode
+ * @param   pVM             The cross context VM structure.
+ * @param   rc              VBox status code.
+ * @param   SRC_POS         Use RT_SRC_POS.
+ * @param   pszFormat       Error message format string.
+ * @param   args            Error message arguments.
+ * @thread  Any
+ */
+VMMDECL(int) VMSetErrorV(PVMCC pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list args)
+{
+#ifdef IN_RING3
+    /*
+     * Switch to EMT.
+     */
+    va_list va2;
+    va_copy(va2, args); /* Have to make a copy here or GCC will break. */
+    VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)vmR3SetErrorUV, 7,   /* ASSUMES 3 source pos args! */
+                            pVM->pUVM, rc, RT_SRC_POS_ARGS, pszFormat, &va2);
+    va_end(va2);
+
+#else
+    /*
+     * We're already on the EMT thread and can safely create a VMERROR chunk.
+     */
+    vmSetErrorCopy(pVM, rc, RT_SRC_POS_ARGS, pszFormat, args);
+    VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_VM_SET_ERROR, 0);
+#endif
+    return rc;
+}
+
+
+/**
+ * Copies the error to a VMERROR structure.
+ *
+ * This is mainly intended for Ring-0 and GC where the error must be copied to
+ * memory accessible from ring-3. But it's just possible that we might add
+ * APIs for retrieving the VMERROR copy later.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   rc              VBox status code.
+ * @param   SRC_POS         Use RT_SRC_POS.
+ * @param   pszFormat       Error message format string.
+ * @param   args            Error message arguments.
+ * @thread  EMT
+ */
+void vmSetErrorCopy(PVM pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list args)
+{
+    NOREF(pVM); NOREF(rc); RT_SRC_POS_NOREF(); NOREF(pszFormat); NOREF(args);
+#if 0 /// @todo implement Ring-0 and GC VMSetError
+    /*
+     * Create the untranslated message copy.
+     */
+    /* free any old message. */
+    MMHyperFree(pVM, MMHyperR32Ctx(pVM, pVM->vm.s.pError));
+    pVM->vm.s.pError = NULL;
+
+    /* calc reasonable start size. */
+    size_t cchFile = pszFile ? strlen(pszFile) : 0;
+    size_t cchFunction = pszFunction ? strlen(pszFunction) : 0;
+    size_t cchFormat = strlen(pszFormat);
+    size_t cb = sizeof(VMERROR)
+              + cchFile + 1
+              + cchFunction + 1
+              + cchFormat + 32;
+
+    /* allocate it */
+    void *pv;
+    int rc2 = MMHyperAlloc(pVM, cb, 0, MM_TAG_VM, &pv);
+    if (RT_SUCCESS(rc2))
+    {
+        /* initialize it. */
+        PVMERROR pErr = (PVMERROR)pv;
+        pErr->cbAllocated = cb;
+        pErr->iLine = iLine;
+        pErr->off = sizeof(VMERROR);
+        pErr->offFile = pErr->offFunction = 0;
+
+        if (cchFile)
+        {
+            pErr->offFile = pErr->off;
+            memcpy((uint8_t *)pErr + pErr->off, pszFile, cchFile + 1);
+            pErr->off += cchFile + 1;
+        }
+
+        if (cchFunction)
+        {
+            pErr->offFunction = pErr->off;
+            memcpy((uint8_t *)pErr + pErr->off, pszFunction, cchFunction + 1);
+            pErr->off += cchFunction + 1;
+        }
+
+        pErr->offMessage = pErr->off;
+
+        /* format the message (pErr might be reallocated) */
+        VMSETERRORFMTARGS Args;
+        Args.pVM = pVM;
+        Args.pErr = pErr;
+
+        va_list va2;
+        va_copy(va2, args);
+        RTStrFormatV(vmSetErrorFmtOut, &pErr, NULL, NULL, &pszFormatTmp, args);
+        va_end(va2);
+
+        /* done. */
+        pVM->vm.s.pErrorR3 = MMHyper2HC(pVM, (uintptr_t)pArgs.pErr);
+    }
+#endif
+}
+
+
+/**
+ * Sets the runtime error message.
+ *
+ * As opposed VMSetError(), this method is intended to inform the VM user about
+ * errors and error-like conditions that happen at an arbitrary point during VM
+ * execution (like "host memory low" or "out of host disk space").
+ *
+ * @returns VBox status code. For some flags the status code <b>must</b> be
+ *          propagated up the stack.
+ *
+ * @param   pVM             The cross context VM structure.
+ *
+ * @param   fFlags          Flags indicating which actions to take.
+ *                          See VMSETRTERR_FLAGS_* for details on each flag.
+ *
+ * @param   pszErrorId      Unique error identification string. This is used by
+ *                          the frontends and maybe other devices or drivers, so
+ *                          once an ID has been selected it's essentially
+ *                          unchangable. Employ camelcase when constructing the
+ *                          string, leave out spaces.
+ *
+ *                          The registered runtime error callbacks should string
+ *                          switch on this and handle the ones it knows
+ *                          specifically and the unknown ones generically.
+ *
+ * @param   pszFormat       Error message format string.
+ * @param   ...             Error message arguments.
+ *
+ * @thread  Any
+ */
+VMMDECL(int) VMSetRuntimeError(PVMCC pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    int rc = VMSetRuntimeErrorV(pVM, fFlags, pszErrorId, pszFormat, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * va_list version of VMSetRuntimeError.
+ *
+ * @returns VBox status code. For some flags the status code <b>must</b> be
+ *          propagated up the stack.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fFlags          Flags indicating which actions to take. See
+ *                          VMSETRTERR_FLAGS_*.
+ * @param   pszErrorId      Error ID string.
+ * @param   pszFormat       Error message format string.
+ * @param   va              Error message arguments.
+ *
+ * @thread  Any
+ */
+VMMDECL(int) VMSetRuntimeErrorV(PVMCC pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
+{
+    Log(("VMSetRuntimeErrorV: fFlags=%#x pszErrorId=%s\n", fFlags, pszErrorId));
+
+    /*
+     * Relaxed parameter validation.
+     */
+    AssertPtr(pVM);
+    AssertMsg(!(fFlags & ~(VMSETRTERR_FLAGS_NO_WAIT | VMSETRTERR_FLAGS_SUSPEND | VMSETRTERR_FLAGS_FATAL)), ("%#x\n", fFlags));
+    Assert(!(fFlags & VMSETRTERR_FLAGS_NO_WAIT) || !VM_IS_EMT(pVM));
+    Assert(!(fFlags & VMSETRTERR_FLAGS_SUSPEND) || !(fFlags & VMSETRTERR_FLAGS_FATAL));
+    AssertPtr(pszErrorId);
+    Assert(*pszErrorId);
+    Assert(RTStrEnd(pszErrorId, 128) != NULL);
+    AssertPtr(pszFormat);
+    Assert(RTStrEnd(pszFormat, 512) != NULL);
+
+#ifdef IN_RING3
+    /*
+     * Switch to EMT.
+     *
+     * If it's a no-wait request, we have to format the message into a buffer
+     * here since the variable arguments list will become invalid once we call
+     * va_end and return.
+     */
+    int rc;
+    if (    !(fFlags & VMSETRTERR_FLAGS_NO_WAIT)
+        ||  VM_IS_EMT(pVM))
+    {
+        fFlags &= ~VMSETRTERR_FLAGS_NO_WAIT;
+
+        va_list va2;
+        va_copy(va2, va); /* Have to make a copy here or GCC will break. */
+        rc = VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY,
+                                     (PFNRT)vmR3SetRuntimeErrorV, 5, pVM, fFlags, pszErrorId, pszFormat, &va2);
+        va_end(va2);
+    }
+    else
+    {
+        char *pszMessage = MMR3HeapAPrintfV(pVM, MM_TAG_VM, pszFormat, va);
+        rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY,
+                               (PFNRT)vmR3SetRuntimeError, 4, pVM, fFlags, pszErrorId, pszMessage);
+        if (RT_FAILURE(rc))
+            MMR3HeapFree(pszMessage);
+    }
+
+#else
+    /*
+     * We're already on the EMT and can safely create a VMRUNTIMEERROR chunk.
+     */
+    AssertReleaseMsgFailed(("Congratulations! You will have the pleasure of debugging the RC/R0 path.\n"));
+    vmSetRuntimeErrorCopy(pVM, fFlags, pszErrorId, pszFormat, va);
+
+    int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_VM_SET_RUNTIME_ERROR, 0);
+#endif
+
+    Log(("VMSetRuntimeErrorV: returns %Rrc (pszErrorId=%s)\n", rc, pszErrorId));
+    return rc;
+}
+
+
+/**
+ * Copies the error to a VMRUNTIMEERROR structure.
+ *
+ * This is mainly intended for Ring-0 and RC where the error must be copied to
+ * memory accessible from ring-3. But it's just possible that we might add
+ * APIs for retrieving the VMRUNTIMEERROR copy later.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fFlags          The error flags.
+ * @param   pszErrorId      Error ID string.
+ * @param   pszFormat       Error message format string.
+ * @param   va              Error message arguments. This is of course spoiled
+ *                          by this call.
+ * @thread  EMT
+ */
+void vmSetRuntimeErrorCopy(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
+{
+    NOREF(pVM); NOREF(fFlags); NOREF(pszErrorId); NOREF(pszFormat); NOREF(va);
+#if 0 /// @todo implement Ring-0 and GC VMSetError
+    /*
+     * Create the untranslated message copy.
+     */
+    /* free any old message. */
+    MMHyperFree(pVM, MMHyperR32Ctx(pVM, pVM->vm.s.pRuntimeErrorR3));
+    pVM->vm.s.pRuntimeErrorR3 = NULL;
+
+    /* calc reasonable start size. */
+    size_t cchErrorID = pszErrorId ? strlen(pszErrorId) : 0;
+    size_t cchFormat = strlen(pszFormat);
+    size_t cb = sizeof(VMRUNTIMEERROR)
+              + cchErrorID + 1
+              + cchFormat + 32;
+
+    /* allocate it */
+    void *pv;
+    int rc2 = MMHyperAlloc(pVM, cb, 0, MM_TAG_VM, &pv);
+    if (RT_SUCCESS(rc2))
+    {
+        /* initialize it. */
+        PVMRUNTIMEERROR pErr = (PVMRUNTIMEERROR)pv;
+        pErr->cbAllocated = cb;
+        pErr->fFlags = fFlags;
+        pErr->off = sizeof(PVMRUNTIMEERROR);
+        pErr->offErrorID = 0;
+
+        if (cchErrorID)
+        {
+            pErr->offErrorID = pErr->off;
+            memcpy((uint8_t *)pErr + pErr->off, pszErrorId, cchErrorID + 1);
+            pErr->off += cchErrorID + 1;
+        }
+
+        pErr->offMessage = pErr->off;
+
+        /* format the message (pErr might be reallocated) */
+        VMSETRUNTIMEERRORFMTARGS Args;
+        Args.pVM = pVM;
+        Args.pErr = pErr;
+
+        va_list va2;
+        va_copy(va2, args);
+        RTStrFormatV(vmSetRuntimeErrorFmtOut, &pErr, NULL, NULL, &pszFormatTmp, args);
+        va_end(va2);
+
+        /* done. */
+        pVM->vm.s.pErrorRuntimeR3 = MMHyper2HC(pVM, (uintptr_t)pArgs.pErr);
+    }
+#endif
+}
+
+
+/**
+ * Gets the name of VM state.
+ *
+ * @returns Pointer to a read-only string with the state name.
+ * @param   enmState    The state.
+ */
+VMMDECL(const char *) VMGetStateName(VMSTATE enmState)
+{
+    switch (enmState)
+    {
+#define MY_CASE(enm) case VMSTATE_##enm: return #enm;
+        MY_CASE(CREATING);
+        MY_CASE(CREATED);
+        MY_CASE(RUNNING);
+        MY_CASE(LOADING);
+        MY_CASE(LOAD_FAILURE);
+        MY_CASE(SAVING);
+        MY_CASE(SUSPENDED);
+        MY_CASE(RESETTING);
+        MY_CASE(GURU_MEDITATION);
+        MY_CASE(OFF);
+        MY_CASE(DESTROYING);
+        MY_CASE(TERMINATED);
+#undef MY_CASE
+        default:
+            return "Unknown";
+    }
+}
+
+
+/**
+ * Gets the total reset count.
+ *
+ * @returns Reset count. UINT32_MAX if @a pVM is invalid.
+ * @param   pVM         The VM handle.
+ */
+VMMDECL(uint32_t) VMGetResetCount(PVMCC pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
+    return pVM->vm.s.cResets;
+}
+
+
+/**
+ * Gets the soft reset count.
+ *
+ * @returns Soft reset count. UINT32_MAX if @a pVM is invalid.
+ * @param   pVM         The VM handle.
+ */
+VMMDECL(uint32_t) VMGetSoftResetCount(PVMCC pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
+    return pVM->vm.s.cSoftResets;
+}
+
+
+/**
+ * Gets the hard reset count.
+ *
+ * @returns Hard reset count. UINT32_MAX if @a pVM is invalid.
+ * @param   pVM         The VM handle.
+ */
+VMMDECL(uint32_t) VMGetHardResetCount(PVMCC pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
+    return pVM->vm.s.cHardResets;
+}
+
diff --git a/src/VBox/VMM/VMMAll/VMMAll.cpp b/src/VBox/VMM/VMMAll/VMMAll.cpp
new file mode 100644
index 00000000..e827e84f
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/VMMAll.cpp
@@ -0,0 +1,370 @@
+/* $Id: VMMAll.cpp $ */
+/** @file
+ * VMM All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <VBox/vmm/vmm.h>
+#include "VMMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#ifdef IN_RING0
+# include <VBox/vmm/gvm.h>
+#endif
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/vmcpuset.h>
+#include <VBox/param.h>
+#include <iprt/thread.h>
+#include <iprt/mp.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** User counter for the vmmInitFormatTypes function (pro forma). */
+static volatile uint32_t g_cFormatTypeUsers = 0;
+
+
+/**
+ * Helper that formats a decimal number in the range 0..9999.
+ *
+ * @returns The length of the formatted number.
+ * @param   pszBuf              Output buffer with sufficient space.
+ * @param   uNumber             The number to format.
+ */
+static unsigned vmmFormatTypeShortNumber(char *pszBuf, uint32_t uNumber)
+{
+    unsigned  off = 0;
+    if (uNumber >= 10)
+    {
+        if (uNumber >= 100)
+        {
+            if (uNumber >= 1000)
+                pszBuf[off++] = ((uNumber / 1000) % 10) + '0';
+            pszBuf[off++] = ((uNumber / 100) % 10) + '0';
+        }
+        pszBuf[off++] = ((uNumber / 10) % 10) + '0';
+    }
+    pszBuf[off++] = (uNumber % 10) + '0';
+    pszBuf[off] = '\0';
+    return off;
+}
+
+
+/**
+ * @callback_method_impl{FNRTSTRFORMATTYPE, vmsetcpu}
+ */
+static DECLCALLBACK(size_t) vmmFormatTypeVmCpuSet(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
+                                                  const char *pszType, void const *pvValue,
+                                                  int cchWidth, int cchPrecision, unsigned fFlags,
+                                                  void *pvUser)
+{
+    NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(fFlags);
+
+    PCVMCPUSET  pSet   = (PCVMCPUSET)pvValue;
+    uint32_t    cCpus  = 0;
+    uint32_t    iCpu   = RT_ELEMENTS(pSet->au32Bitmap) * 32;
+    while (iCpu--)
+        if (VMCPUSET_IS_PRESENT(pSet, iCpu))
+            cCpus++;
+
+    char szTmp[32];
+    AssertCompile(RT_ELEMENTS(pSet->au32Bitmap) * 32 < 999);
+    if (cCpus == 1)
+    {
+        iCpu = RT_ELEMENTS(pSet->au32Bitmap) * 32;
+        while (iCpu--)
+            if (VMCPUSET_IS_PRESENT(pSet, iCpu))
+            {
+                szTmp[0] = 'c';
+                szTmp[1] = 'p';
+                szTmp[2] = 'u';
+                return pfnOutput(pvArgOutput, szTmp, 3 + vmmFormatTypeShortNumber(&szTmp[3], iCpu));
+            }
+        cCpus = 0;
+    }
+    if (cCpus == 0)
+        return pfnOutput(pvArgOutput, RT_STR_TUPLE("<empty>"));
+    if (cCpus == RT_ELEMENTS(pSet->au32Bitmap) * 32)
+        return pfnOutput(pvArgOutput, RT_STR_TUPLE("<full>"));
+
+    /*
+     * Print cpus that are present: {1,2,7,9 ... }
+     */
+    size_t cchRet = pfnOutput(pvArgOutput, "{", 1);
+
+    cCpus = 0;
+    iCpu  = 0;
+    while (iCpu < RT_ELEMENTS(pSet->au32Bitmap) * 32)
+    {
+        if (VMCPUSET_IS_PRESENT(pSet, iCpu))
+        {
+            /* Output the first cpu number. */
+            int off = 0;
+            if (cCpus != 0)
+                szTmp[off++] = ',';
+            cCpus++;
+            off += vmmFormatTypeShortNumber(&szTmp[off], iCpu);
+
+            /* Check for sequence. */
+            uint32_t const iStart = ++iCpu;
+            while (   iCpu < RT_ELEMENTS(pSet->au32Bitmap) * 32
+                   && VMCPUSET_IS_PRESENT(pSet, iCpu))
+            {
+                iCpu++;
+                cCpus++;
+            }
+            if (iCpu != iStart)
+            {
+                szTmp[off++] = '-';
+                off += vmmFormatTypeShortNumber(&szTmp[off], iCpu);
+            }
+
+            /* Terminate and output. */
+            szTmp[off] = '\0';
+            cchRet += pfnOutput(pvArgOutput, szTmp, off);
+        }
+        iCpu++;
+    }
+
+    cchRet += pfnOutput(pvArgOutput, "}", 1);
+    NOREF(pvUser);
+    return cchRet;
+}
+
+
+/**
+ * Registers the VMM wide format types.
+ *
+ * Called by VMMR3Init, VMMR0Init and VMMRCInit.
+ */
+int vmmInitFormatTypes(void)
+{
+    int rc = VINF_SUCCESS;
+    if (ASMAtomicIncU32(&g_cFormatTypeUsers) == 1)
+        rc = RTStrFormatTypeRegister("vmcpuset", vmmFormatTypeVmCpuSet, NULL);
+    return rc;
+}
+
+
+/**
+ * Counterpart to vmmInitFormatTypes, called by VMMR3Term and VMMR0Term.
+ */
+void vmmTermFormatTypes(void)
+{
+    if (ASMAtomicDecU32(&g_cFormatTypeUsers) == 0)
+        RTStrFormatTypeDeregister("vmcpuset");
+}
+
+
+/**
+ * Gets the ID of the virtual CPU associated with the calling thread.
+ *
+ * @returns The CPU ID. NIL_VMCPUID if the thread isn't an EMT.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @internal
+ */
+VMMDECL(VMCPUID) VMMGetCpuId(PVMCC pVM)
+{
+#if defined(IN_RING3)
+    return VMR3GetVMCPUId(pVM);
+
+#elif defined(IN_RING0)
+    if (pVM->cCpus == 1)
+        return 0;
+    VMCPUID const cCpus = pVM->cCpus;
+
+    /* Search first by host cpu id (most common case)
+     * and then by native thread id (page fusion case).
+     */
+    if (!RTThreadPreemptIsEnabled(NIL_RTTHREAD))
+    {
+        /** @todo r=ramshankar: This doesn't buy us anything in terms of performance
+         *        leaving it here for hysterical raisins and as a reference if we
+         *        implemented a hashing approach in the future. */
+        RTCPUID idHostCpu = RTMpCpuId();
+
+        /** @todo optimize for large number of VCPUs when that becomes more common. */
+        for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+        {
+            PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+            if (pVCpu->idHostCpu == idHostCpu)
+                return pVCpu->idCpu;
+        }
+    }
+
+    /* RTThreadGetNativeSelf had better be cheap. */
+    RTNATIVETHREAD hThread = RTThreadNativeSelf();
+
+    /** @todo optimize for large number of VCPUs when that becomes more common. */
+    for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+        if (pVCpu->hNativeThreadR0 == hThread)
+            return pVCpu->idCpu;
+    }
+    return NIL_VMCPUID;
+
+#else /* RC: Always EMT(0) */
+    NOREF(pVM);
+    return 0;
+#endif
+}
+
+
+/**
+ * Returns the VMCPU of the calling EMT.
+ *
+ * @returns The VMCPU pointer. NULL if not an EMT.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @internal
+ */
+VMMDECL(PVMCPUCC) VMMGetCpu(PVMCC pVM)
+{
+#ifdef IN_RING3
+    VMCPUID idCpu = VMR3GetVMCPUId(pVM);
+    if (idCpu == NIL_VMCPUID)
+        return NULL;
+    Assert(idCpu < pVM->cCpus);
+    return VMCC_GET_CPU(pVM, idCpu);
+
+#elif defined(IN_RING0)
+    VMCPUID const cCpus = pVM->cCpus;
+    if (pVM->cCpus == 1)
+        return VMCC_GET_CPU_0(pVM);
+
+    /*
+     * Search first by host cpu id (most common case)
+     * and then by native thread id (page fusion case).
+     */
+    if (!RTThreadPreemptIsEnabled(NIL_RTTHREAD))
+    {
+        /** @todo r=ramshankar: This doesn't buy us anything in terms of performance
+         *        leaving it here for hysterical raisins and as a reference if we
+         *        implemented a hashing approach in the future. */
+        RTCPUID idHostCpu = RTMpCpuId();
+
+        /** @todo optimize for large number of VCPUs when that becomes more common. */
+        for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+        {
+            PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+            if (pVCpu->idHostCpu == idHostCpu)
+                return pVCpu;
+        }
+    }
+
+    /* RTThreadGetNativeSelf had better be cheap. */
+    RTNATIVETHREAD hThread = RTThreadNativeSelf();
+
+    /** @todo optimize for large number of VCPUs when that becomes more common.
+     * Use a map like GIP does that's indexed by the host CPU index.  */
+    for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+        if (pVCpu->hNativeThreadR0 == hThread)
+            return pVCpu;
+    }
+    return NULL;
+
+#else /* RC: Always EMT(0) */
+    RT_NOREF(pVM);
+    return &g_VCpu0;
+#endif /* IN_RING0 */
+}
+
+
+/**
+ * Returns the VMCPU of the first EMT thread.
+ *
+ * @returns The VMCPU pointer.
+ * @param   pVM         The cross context VM structure.
+ * @internal
+ */
+VMMDECL(PVMCPUCC) VMMGetCpu0(PVMCC pVM)
+{
+    Assert(pVM->cCpus == 1);
+    return VMCC_GET_CPU_0(pVM);
+}
+
+
+/**
+ * Returns the VMCPU of the specified virtual CPU.
+ *
+ * @returns The VMCPU pointer. NULL if idCpu is invalid.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   idCpu       The ID of the virtual CPU.
+ * @internal
+ */
+VMMDECL(PVMCPUCC) VMMGetCpuById(PVMCC pVM, RTCPUID idCpu)
+{
+    AssertReturn(idCpu < pVM->cCpus, NULL);
+    return VMCC_GET_CPU(pVM, idCpu);
+}
+
+
+/**
+ * Gets the VBOX_SVN_REV.
+ *
+ * This is just to avoid having to compile a bunch of big files
+ * and requires less Makefile mess.
+ *
+ * @returns VBOX_SVN_REV.
+ */
+VMM_INT_DECL(uint32_t) VMMGetSvnRev(void)
+{
+    return VBOX_SVN_REV;
+}
+
+
+/**
+ * Checks whether we're in a ring-3 call or not.
+ *
+ * @returns true / false.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT
+ */
+VMM_INT_DECL(bool) VMMIsInRing3Call(PVMCPU pVCpu)
+{
+#ifdef RT_ARCH_X86
+    return pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
+#else
+    return pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
+#endif
+}
+
+
+/**
+ * Returns the build type for matching components.
+ *
+ * @returns Build type value.
+ */
+uint32_t vmmGetBuildType(void)
+{
+    uint32_t uRet = 0xbeef0000;
+#ifdef DEBUG
+    uRet |= RT_BIT_32(0);
+#endif
+#ifdef VBOX_WITH_STATISTICS
+    uRet |= RT_BIT_32(1);
+#endif
+    return uRet;
+}
+
diff --git a/src/VBox/VMM/VMMAll/VMMAllA.asm b/src/VBox/VMM/VMMAll/VMMAllA.asm
new file mode 100644
index 00000000..2fa6127c
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/VMMAllA.asm
@@ -0,0 +1,83 @@
+; $Id: VMMAllA.asm $
+;; @file
+; VMM - All Contexts Assembly Routines.
+;
+
+;
+; Copyright (C) 2009-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+;*******************************************************************************
+;* Header Files                                                                *
+;*******************************************************************************
+%include "VBox/asmdefs.mac"
+
+
+;*******************************************************************************
+;* Defined Constants And Macros                                                *
+;*******************************************************************************
+%ifdef IN_RING3
+ %ifdef RT_ARCH_AMD64
+  %define VMM_TRASH_XMM_REGS
+ %endif
+%endif
+%ifdef IN_RING0
+ %ifdef RT_ARCH_AMD64
+  %ifdef RT_OS_WINDOWS
+   %define VMM_TRASH_XMM_REGS
+  %endif
+ %endif
+%endif
+%ifndef VMM_TRASH_XMM_REGS
+ %ifdef VBOX_WITH_KERNEL_USING_XMM
+  %define VMM_TRASH_XMM_REGS
+ %endif
+%endif
+
+BEGINCODE
+
+
+;;
+; Trashes the volatile XMM registers in the current ABI.
+;
+BEGINPROC VMMTrashVolatileXMMRegs
+%ifdef VMM_TRASH_XMM_REGS
+        push    xBP
+        mov     xBP, xSP
+
+        ; take whatever is on the stack.
+        and     xSP, ~15
+        sub     xSP, 80h
+
+        movdqa  xmm0, [xSP + 0]
+        movdqa  xmm1, [xSP + 010h]
+        movdqa  xmm2, [xSP + 020h]
+        movdqa  xmm3, [xSP + 030h]
+        movdqa  xmm4, [xSP + 040h]
+        movdqa  xmm5, [xSP + 050h]
+ %ifdef ASM_CALL64_GCC
+        movdqa  xmm6, [xSP + 060h]
+        movdqa  xmm7, [xSP + 070h]
+        movdqa  xmm8, [xSP + 000h]
+        movdqa  xmm9, [xSP + 010h]
+        movdqa  xmm10,[xSP + 020h]
+        movdqa  xmm11,[xSP + 030h]
+        movdqa  xmm12,[xSP + 040h]
+        movdqa  xmm13,[xSP + 050h]
+        movdqa  xmm14,[xSP + 060h]
+        movdqa  xmm15,[xSP + 070h]
+ %endif
+        leave
+%endif ; VMM_TRASH_XMM_REGS
+        xor     eax, eax                ; for good measure.
+        ret
+ENDPROC   VMMTrashVolatileXMMRegs
+
diff --git a/src/VBox/VMM/VMMR0/CPUMR0.cpp b/src/VBox/VMM/VMMR0/CPUMR0.cpp
new file mode 100644
index 00000000..04cbaa97
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/CPUMR0.cpp
@@ -0,0 +1,954 @@
+/* $Id: CPUMR0.cpp $ */
+/** @file
+ * CPUM - Host Context Ring 0.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_CPUM
+#include <VBox/vmm/cpum.h>
+#include "CPUMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/gvm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/vmm/hm.h>
+#include <iprt/assert.h>
+#include <iprt/asm-amd64-x86.h>
+#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+# include <iprt/mem.h>
+# include <iprt/memobj.h>
+# include <VBox/apic.h>
+#endif
+#include <iprt/x86.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+/**
+ * Local APIC mappings.
+ */
+typedef struct CPUMHOSTLAPIC
+{
+    /** Indicates that the entry is in use and have valid data. */
+    bool        fEnabled;
+    /** Whether it's operating in X2APIC mode (EXTD). */
+    bool        fX2Apic;
+    /** The APIC version number. */
+    uint32_t    uVersion;
+    /** The physical address of the APIC registers. */
+    RTHCPHYS    PhysBase;
+    /** The memory object entering the physical address. */
+    RTR0MEMOBJ  hMemObj;
+    /** The mapping object for hMemObj. */
+    RTR0MEMOBJ  hMapObj;
+    /** The mapping address APIC registers.
+     * @remarks Different CPUs may use the same physical address to map their
+     *          APICs, so this pointer is only valid when on the CPU owning the
+     *          APIC. */
+    void       *pv;
+} CPUMHOSTLAPIC;
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+static CPUMHOSTLAPIC g_aLApics[RTCPUSET_MAX_CPUS];
+#endif
+
+/**
+ * CPUID bits to unify among all cores.
+ */
+static struct
+{
+    uint32_t uLeaf;  /**< Leaf to check. */
+    uint32_t uEcx;   /**< which bits in ecx to unify between CPUs. */
+    uint32_t uEdx;   /**< which bits in edx to unify between CPUs. */
+}
+const g_aCpuidUnifyBits[] =
+{
+    {
+        0x00000001,
+        X86_CPUID_FEATURE_ECX_CX16 | X86_CPUID_FEATURE_ECX_MONITOR,
+        X86_CPUID_FEATURE_EDX_CX8
+    }
+};
+
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+static int  cpumR0MapLocalApics(void);
+static void cpumR0UnmapLocalApics(void);
+#endif
+static int  cpumR0SaveHostDebugState(PVMCPUCC pVCpu);
+
+
+/**
+ * Does the Ring-0 CPU initialization once during module load.
+ * XXX Host-CPU hot-plugging?
+ */
+VMMR0_INT_DECL(int) CPUMR0ModuleInit(void)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+    rc = cpumR0MapLocalApics();
+#endif
+    return rc;
+}
+
+
+/**
+ * Terminate the module.
+ */
+VMMR0_INT_DECL(int) CPUMR0ModuleTerm(void)
+{
+#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+    cpumR0UnmapLocalApics();
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Check the CPUID features of this particular CPU and disable relevant features
+ * for the guest which do not exist on this CPU. We have seen systems where the
+ * X86_CPUID_FEATURE_ECX_MONITOR feature flag is only set on some host CPUs, see
+ * @bugref{5436}.
+ *
+ * @note This function might be called simultaneously on more than one CPU!
+ *
+ * @param   idCpu       The identifier for the CPU the function is called on.
+ * @param   pvUser1     Pointer to the VM structure.
+ * @param   pvUser2     Ignored.
+ */
+static DECLCALLBACK(void) cpumR0CheckCpuid(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    PVMCC     pVM   = (PVMCC)pvUser1;
+
+    NOREF(idCpu); NOREF(pvUser2);
+    for (uint32_t i = 0; i < RT_ELEMENTS(g_aCpuidUnifyBits); i++)
+    {
+        /* Note! Cannot use cpumCpuIdGetLeaf from here because we're not
+                 necessarily in the VM process context.  So, we using the
+                 legacy arrays as temporary storage. */
+
+        uint32_t   uLeaf = g_aCpuidUnifyBits[i].uLeaf;
+        PCPUMCPUID pLegacyLeaf;
+        if (uLeaf < RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdPatmStd))
+            pLegacyLeaf = &pVM->cpum.s.aGuestCpuIdPatmStd[uLeaf];
+        else if (uLeaf - UINT32_C(0x80000000) < RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdPatmExt))
+            pLegacyLeaf = &pVM->cpum.s.aGuestCpuIdPatmExt[uLeaf - UINT32_C(0x80000000)];
+        else if (uLeaf - UINT32_C(0xc0000000) < RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdPatmCentaur))
+            pLegacyLeaf = &pVM->cpum.s.aGuestCpuIdPatmCentaur[uLeaf - UINT32_C(0xc0000000)];
+        else
+            continue;
+
+        uint32_t eax, ebx, ecx, edx;
+        ASMCpuIdExSlow(uLeaf, 0, 0, 0, &eax, &ebx, &ecx, &edx);
+
+        ASMAtomicAndU32(&pLegacyLeaf->uEcx, ecx | ~g_aCpuidUnifyBits[i].uEcx);
+        ASMAtomicAndU32(&pLegacyLeaf->uEdx, edx | ~g_aCpuidUnifyBits[i].uEdx);
+    }
+}
+
+
+/**
+ * Does Ring-0 CPUM initialization.
+ *
+ * This is mainly to check that the Host CPU mode is compatible
+ * with VBox.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR0_INT_DECL(int) CPUMR0InitVM(PVMCC pVM)
+{
+    LogFlow(("CPUMR0Init: %p\n", pVM));
+
+    /*
+     * Check CR0 & CR4 flags.
+     */
+    uint32_t u32CR0 = ASMGetCR0();
+    if ((u32CR0 & (X86_CR0_PE | X86_CR0_PG)) != (X86_CR0_PE | X86_CR0_PG)) /* a bit paranoid perhaps.. */
+    {
+        Log(("CPUMR0Init: PE or PG not set. cr0=%#x\n", u32CR0));
+        return VERR_UNSUPPORTED_CPU_MODE;
+    }
+
+    /*
+     * Check for sysenter and syscall usage.
+     */
+    if (ASMHasCpuId())
+    {
+        /*
+         * SYSENTER/SYSEXIT
+         *
+         * Intel docs claim you should test both the flag and family, model &
+         * stepping because some Pentium Pro CPUs have the SEP cpuid flag set,
+         * but don't support it.  AMD CPUs may support this feature in legacy
+         * mode, they've banned it from long mode.  Since we switch to 32-bit
+         * mode when entering raw-mode context the feature would become
+         * accessible again on AMD CPUs, so we have to check regardless of
+         * host bitness.
+         */
+        uint32_t u32CpuVersion;
+        uint32_t u32Dummy;
+        uint32_t fFeatures; /* (Used further down to check for MSRs, so don't clobber.) */
+        ASMCpuId(1, &u32CpuVersion, &u32Dummy, &u32Dummy, &fFeatures);
+        uint32_t const u32Family   = u32CpuVersion >> 8;
+        uint32_t const u32Model    = (u32CpuVersion >> 4) & 0xF;
+        uint32_t const u32Stepping = u32CpuVersion & 0xF;
+        if (    (fFeatures & X86_CPUID_FEATURE_EDX_SEP)
+            &&  (   u32Family   != 6    /* (> pentium pro) */
+                 || u32Model    >= 3
+                 || u32Stepping >= 3
+                 || !ASMIsIntelCpu())
+           )
+        {
+            /*
+             * Read the MSR and see if it's in use or not.
+             */
+            uint32_t u32 = ASMRdMsr_Low(MSR_IA32_SYSENTER_CS);
+            if (u32)
+            {
+                pVM->cpum.s.fHostUseFlags |= CPUM_USE_SYSENTER;
+                Log(("CPUMR0Init: host uses sysenter cs=%08x%08x\n", ASMRdMsr_High(MSR_IA32_SYSENTER_CS), u32));
+            }
+        }
+
+        /*
+         * SYSCALL/SYSRET
+         *
+         * This feature is indicated by the SEP bit returned in EDX by CPUID
+         * function 0x80000001.  Intel CPUs only supports this feature in
+         * long mode.  Since we're not running 64-bit guests in raw-mode there
+         * are no issues with 32-bit intel hosts.
+         */
+        uint32_t cExt = 0;
+        ASMCpuId(0x80000000, &cExt, &u32Dummy, &u32Dummy, &u32Dummy);
+        if (ASMIsValidExtRange(cExt))
+        {
+            uint32_t fExtFeaturesEDX = ASMCpuId_EDX(0x80000001);
+            if (fExtFeaturesEDX & X86_CPUID_EXT_FEATURE_EDX_SYSCALL)
+            {
+#ifdef RT_ARCH_X86
+                if (!ASMIsIntelCpu())
+#endif
+                {
+                    uint64_t fEfer = ASMRdMsr(MSR_K6_EFER);
+                    if (fEfer & MSR_K6_EFER_SCE)
+                    {
+                        pVM->cpum.s.fHostUseFlags |= CPUM_USE_SYSCALL;
+                        Log(("CPUMR0Init: host uses syscall\n"));
+                    }
+                }
+            }
+        }
+
+        /*
+         * Copy MSR_IA32_ARCH_CAPABILITIES bits over into the host and guest feature
+         * structure and as well as the guest MSR.
+         * Note! we assume this happens after the CPUMR3Init is done, so CPUID bits are settled.
+         */
+        pVM->cpum.s.HostFeatures.fArchRdclNo             = 0;
+        pVM->cpum.s.HostFeatures.fArchIbrsAll            = 0;
+        pVM->cpum.s.HostFeatures.fArchRsbOverride        = 0;
+        pVM->cpum.s.HostFeatures.fArchVmmNeedNotFlushL1d = 0;
+        pVM->cpum.s.HostFeatures.fArchMdsNo              = 0;
+        uint32_t const cStdRange = ASMCpuId_EAX(0);
+        if (   ASMIsValidStdRange(cStdRange)
+            && cStdRange >= 7)
+        {
+            uint32_t fEdxFeatures = ASMCpuId_EDX(7);
+            if (   (fEdxFeatures & X86_CPUID_STEXT_FEATURE_EDX_ARCHCAP)
+                && (fFeatures & X86_CPUID_FEATURE_EDX_MSR))
+            {
+                /* Host: */
+                uint64_t fArchVal = ASMRdMsr(MSR_IA32_ARCH_CAPABILITIES);
+                pVM->cpum.s.HostFeatures.fArchRdclNo             = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_RDCL_NO);
+                pVM->cpum.s.HostFeatures.fArchIbrsAll            = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_IBRS_ALL);
+                pVM->cpum.s.HostFeatures.fArchRsbOverride        = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_RSBO);
+                pVM->cpum.s.HostFeatures.fArchVmmNeedNotFlushL1d = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_VMM_NEED_NOT_FLUSH_L1D);
+                pVM->cpum.s.HostFeatures.fArchMdsNo              = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_MDS_NO);
+
+                /* guest: */
+                if (!pVM->cpum.s.GuestFeatures.fArchCap)
+                    fArchVal = 0;
+                else if (!pVM->cpum.s.GuestFeatures.fIbrs)
+                    fArchVal &= ~MSR_IA32_ARCH_CAP_F_IBRS_ALL;
+                VMCC_FOR_EACH_VMCPU_STMT(pVM, pVCpu->cpum.s.GuestMsrs.msr.ArchCaps = fArchVal);
+                pVM->cpum.s.GuestFeatures.fArchRdclNo             = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_RDCL_NO);
+                pVM->cpum.s.GuestFeatures.fArchIbrsAll            = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_IBRS_ALL);
+                pVM->cpum.s.GuestFeatures.fArchRsbOverride        = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_RSBO);
+                pVM->cpum.s.GuestFeatures.fArchVmmNeedNotFlushL1d = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_VMM_NEED_NOT_FLUSH_L1D);
+                pVM->cpum.s.GuestFeatures.fArchMdsNo              = RT_BOOL(fArchVal & MSR_IA32_ARCH_CAP_F_MDS_NO);
+            }
+            else
+                pVM->cpum.s.HostFeatures.fArchCap = 0;
+        }
+
+        /*
+         * Unify/cross check some CPUID feature bits on all available CPU cores
+         * and threads.  We've seen CPUs where the monitor support differed.
+         *
+         * Because the hyper heap isn't always mapped into ring-0, we cannot
+         * access it from a RTMpOnAll callback.  We use the legacy CPUID arrays
+         * as temp ring-0 accessible memory instead, ASSUMING that they're all
+         * up to date when we get here.
+         */
+        RTMpOnAll(cpumR0CheckCpuid, pVM, NULL);
+
+        for (uint32_t i = 0; i < RT_ELEMENTS(g_aCpuidUnifyBits); i++)
+        {
+            bool            fIgnored;
+            uint32_t        uLeaf = g_aCpuidUnifyBits[i].uLeaf;
+            PCPUMCPUIDLEAF  pLeaf = cpumCpuIdGetLeafEx(pVM, uLeaf, 0, &fIgnored);
+            if (pLeaf)
+            {
+                PCPUMCPUID pLegacyLeaf;
+                if (uLeaf < RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdPatmStd))
+                    pLegacyLeaf = &pVM->cpum.s.aGuestCpuIdPatmStd[uLeaf];
+                else if (uLeaf - UINT32_C(0x80000000) < RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdPatmExt))
+                    pLegacyLeaf = &pVM->cpum.s.aGuestCpuIdPatmExt[uLeaf - UINT32_C(0x80000000)];
+                else if (uLeaf - UINT32_C(0xc0000000) < RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdPatmCentaur))
+                    pLegacyLeaf = &pVM->cpum.s.aGuestCpuIdPatmCentaur[uLeaf - UINT32_C(0xc0000000)];
+                else
+                    continue;
+
+                pLeaf->uEcx = pLegacyLeaf->uEcx;
+                pLeaf->uEdx = pLegacyLeaf->uEdx;
+            }
+        }
+
+    }
+
+
+    /*
+     * Check if debug registers are armed.
+     * This ASSUMES that DR7.GD is not set, or that it's handled transparently!
+     */
+    uint32_t u32DR7 = ASMGetDR7();
+    if (u32DR7 & X86_DR7_ENABLED_MASK)
+    {
+        VMCC_FOR_EACH_VMCPU_STMT(pVM, pVCpu->cpum.s.fUseFlags |= CPUM_USE_DEBUG_REGS_HOST);
+        Log(("CPUMR0Init: host uses debug registers (dr7=%x)\n", u32DR7));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Trap handler for device-not-available fault (\#NM).
+ * Device not available, FP or (F)WAIT instruction.
+ *
+ * @returns VBox status code.
+ * @retval VINF_SUCCESS           if the guest FPU state is loaded.
+ * @retval VINF_EM_RAW_GUEST_TRAP if it is a guest trap.
+ * @retval VINF_CPUM_HOST_CR0_MODIFIED if we modified the host CR0.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMR0_INT_DECL(int) CPUMR0Trap07Handler(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    Assert(pVM->cpum.s.HostFeatures.fFxSaveRstor);
+    Assert(ASMGetCR4() & X86_CR4_OSFXSR);
+
+    /* If the FPU state has already been loaded, then it's a guest trap. */
+    if (CPUMIsGuestFPUStateActive(pVCpu))
+    {
+        Assert(    ((pVCpu->cpum.s.Guest.cr0 & (X86_CR0_MP | X86_CR0_EM | X86_CR0_TS)) == (X86_CR0_MP | X86_CR0_TS))
+               ||  ((pVCpu->cpum.s.Guest.cr0 & (X86_CR0_MP | X86_CR0_EM | X86_CR0_TS)) == (X86_CR0_MP | X86_CR0_TS | X86_CR0_EM)));
+        return VINF_EM_RAW_GUEST_TRAP;
+    }
+
+    /*
+     * There are two basic actions:
+     *   1. Save host fpu and restore guest fpu.
+     *   2. Generate guest trap.
+     *
+     * When entering the hypervisor we'll always enable MP (for proper wait
+     * trapping) and TS (for intercepting all fpu/mmx/sse stuff). The EM flag
+     * is taken from the guest OS in order to get proper SSE handling.
+     *
+     *
+     * Actions taken depending on the guest CR0 flags:
+     *
+     *   3    2    1
+     *  TS | EM | MP | FPUInstr | WAIT :: VMM Action
+     * ------------------------------------------------------------------------
+     *   0 |  0 |  0 | Exec     | Exec :: Clear TS & MP, Save HC, Load GC.
+     *   0 |  0 |  1 | Exec     | Exec :: Clear TS, Save HC, Load GC.
+     *   0 |  1 |  0 | #NM      | Exec :: Clear TS & MP, Save HC, Load GC.
+     *   0 |  1 |  1 | #NM      | Exec :: Clear TS, Save HC, Load GC.
+     *   1 |  0 |  0 | #NM      | Exec :: Clear MP, Save HC, Load GC. (EM is already cleared.)
+     *   1 |  0 |  1 | #NM      | #NM  :: Go to guest taking trap there.
+     *   1 |  1 |  0 | #NM      | Exec :: Clear MP, Save HC, Load GC. (EM is already set.)
+     *   1 |  1 |  1 | #NM      | #NM  :: Go to guest taking trap there.
+     */
+
+    switch (pVCpu->cpum.s.Guest.cr0 & (X86_CR0_MP | X86_CR0_EM | X86_CR0_TS))
+    {
+        case X86_CR0_MP | X86_CR0_TS:
+        case X86_CR0_MP | X86_CR0_TS | X86_CR0_EM:
+            return VINF_EM_RAW_GUEST_TRAP;
+        default:
+            break;
+    }
+
+    return CPUMR0LoadGuestFPU(pVM, pVCpu);
+}
+
+
+/**
+ * Saves the host-FPU/XMM state (if necessary) and (always) loads the guest-FPU
+ * state into the CPU.
+ *
+ * @returns VINF_SUCCESS on success, host CR0 unmodified.
+ * @returns VINF_CPUM_HOST_CR0_MODIFIED on success when the host CR0 was
+ *          modified and VT-x needs to update the value in the VMCS.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMR0_INT_DECL(int) CPUMR0LoadGuestFPU(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    int rc;
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(!(pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST));
+    Assert(!(pVCpu->cpum.s.fUseFlags & CPUM_SYNC_FPU_STATE));
+
+    if (!pVM->cpum.s.HostFeatures.fLeakyFxSR)
+    {
+        Assert(!(pVCpu->cpum.s.fUseFlags & CPUM_USED_MANUAL_XMM_RESTORE));
+        rc = cpumR0SaveHostRestoreGuestFPUState(&pVCpu->cpum.s);
+    }
+    else
+    {
+        Assert(!(pVCpu->cpum.s.fUseFlags & CPUM_USED_MANUAL_XMM_RESTORE) || (pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_HOST));
+        /** @todo r=ramshankar: Can't we used a cached value here
+         *        instead of reading the MSR? host EFER doesn't usually
+         *        change. */
+        uint64_t uHostEfer = ASMRdMsr(MSR_K6_EFER);
+        if (!(uHostEfer & MSR_K6_EFER_FFXSR))
+            rc = cpumR0SaveHostRestoreGuestFPUState(&pVCpu->cpum.s);
+        else
+        {
+            RTCCUINTREG const uSavedFlags = ASMIntDisableFlags();
+            pVCpu->cpum.s.fUseFlags |= CPUM_USED_MANUAL_XMM_RESTORE;
+            ASMWrMsr(MSR_K6_EFER, uHostEfer & ~MSR_K6_EFER_FFXSR);
+            rc = cpumR0SaveHostRestoreGuestFPUState(&pVCpu->cpum.s);
+            ASMWrMsr(MSR_K6_EFER, uHostEfer | MSR_K6_EFER_FFXSR);
+            ASMSetFlags(uSavedFlags);
+        }
+    }
+    Assert(   (pVCpu->cpum.s.fUseFlags & (CPUM_USED_FPU_GUEST | CPUM_USED_FPU_HOST | CPUM_USED_FPU_SINCE_REM))
+           ==                            (CPUM_USED_FPU_GUEST | CPUM_USED_FPU_HOST | CPUM_USED_FPU_SINCE_REM));
+    return rc;
+}
+
+
+/**
+ * Saves the guest FPU/XMM state if needed, restores the host FPU/XMM state as
+ * needed.
+ *
+ * @returns true if we saved the guest state.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMR0_INT_DECL(bool) CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(PVMCPUCC pVCpu)
+{
+    bool fSavedGuest;
+    Assert(pVCpu->CTX_SUFF(pVM)->cpum.s.HostFeatures.fFxSaveRstor);
+    Assert(ASMGetCR4() & X86_CR4_OSFXSR);
+    if (pVCpu->cpum.s.fUseFlags & (CPUM_USED_FPU_GUEST | CPUM_USED_FPU_HOST))
+    {
+        fSavedGuest = RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST);
+        if (!(pVCpu->cpum.s.fUseFlags & CPUM_USED_MANUAL_XMM_RESTORE))
+            cpumR0SaveGuestRestoreHostFPUState(&pVCpu->cpum.s);
+        else
+        {
+            /* Temporarily clear MSR_K6_EFER_FFXSR or else we'll be unable to
+               save/restore the XMM state with fxsave/fxrstor. */
+            uint64_t uHostEfer = ASMRdMsr(MSR_K6_EFER);
+            if (uHostEfer & MSR_K6_EFER_FFXSR)
+            {
+                RTCCUINTREG const uSavedFlags = ASMIntDisableFlags();
+                ASMWrMsr(MSR_K6_EFER, uHostEfer & ~MSR_K6_EFER_FFXSR);
+                cpumR0SaveGuestRestoreHostFPUState(&pVCpu->cpum.s);
+                ASMWrMsr(MSR_K6_EFER, uHostEfer | MSR_K6_EFER_FFXSR);
+                ASMSetFlags(uSavedFlags);
+            }
+            else
+                cpumR0SaveGuestRestoreHostFPUState(&pVCpu->cpum.s);
+            pVCpu->cpum.s.fUseFlags &= ~CPUM_USED_MANUAL_XMM_RESTORE;
+        }
+    }
+    else
+        fSavedGuest = false;
+    Assert(!(  pVCpu->cpum.s.fUseFlags
+             & (CPUM_USED_FPU_GUEST | CPUM_USED_FPU_HOST | CPUM_SYNC_FPU_STATE | CPUM_USED_MANUAL_XMM_RESTORE)));
+    return fSavedGuest;
+}
+
+
+/**
+ * Saves the host debug state, setting CPUM_USED_HOST_DEBUG_STATE and loading
+ * DR7 with safe values.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+static int cpumR0SaveHostDebugState(PVMCPUCC pVCpu)
+{
+    /*
+     * Save the host state.
+     */
+    pVCpu->cpum.s.Host.dr0 = ASMGetDR0();
+    pVCpu->cpum.s.Host.dr1 = ASMGetDR1();
+    pVCpu->cpum.s.Host.dr2 = ASMGetDR2();
+    pVCpu->cpum.s.Host.dr3 = ASMGetDR3();
+    pVCpu->cpum.s.Host.dr6 = ASMGetDR6();
+    /** @todo dr7 might already have been changed to 0x400; don't care right now as it's harmless. */
+    pVCpu->cpum.s.Host.dr7 = ASMGetDR7();
+
+    /* Preemption paranoia. */
+    ASMAtomicOrU32(&pVCpu->cpum.s.fUseFlags, CPUM_USED_DEBUG_REGS_HOST);
+
+    /*
+     * Make sure DR7 is harmless or else we could trigger breakpoints when
+     * load guest or hypervisor DRx values later.
+     */
+    if (pVCpu->cpum.s.Host.dr7 != X86_DR7_INIT_VAL)
+        ASMSetDR7(X86_DR7_INIT_VAL);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Saves the guest DRx state residing in host registers and restore the host
+ * register values.
+ *
+ * The guest DRx state is only saved if CPUMR0LoadGuestDebugState was called,
+ * since it's assumed that we're shadowing the guest DRx register values
+ * accurately when using the combined hypervisor debug register values
+ * (CPUMR0LoadHyperDebugState).
+ *
+ * @returns true if either guest or hypervisor debug registers were loaded.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   fDr6        Whether to include DR6 or not.
+ * @thread  EMT(pVCpu)
+ */
+VMMR0_INT_DECL(bool) CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(PVMCPUCC pVCpu, bool fDr6)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    bool const fDrXLoaded = RT_BOOL(pVCpu->cpum.s.fUseFlags & (CPUM_USED_DEBUG_REGS_GUEST | CPUM_USED_DEBUG_REGS_HYPER));
+
+    /*
+     * Do we need to save the guest DRx registered loaded into host registers?
+     * (DR7 and DR6 (if fDr6 is true) are left to the caller.)
+     */
+    if (pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_GUEST)
+    {
+        pVCpu->cpum.s.Guest.dr[0] = ASMGetDR0();
+        pVCpu->cpum.s.Guest.dr[1] = ASMGetDR1();
+        pVCpu->cpum.s.Guest.dr[2] = ASMGetDR2();
+        pVCpu->cpum.s.Guest.dr[3] = ASMGetDR3();
+        if (fDr6)
+            pVCpu->cpum.s.Guest.dr[6] = ASMGetDR6();
+    }
+    ASMAtomicAndU32(&pVCpu->cpum.s.fUseFlags, ~(  CPUM_USED_DEBUG_REGS_GUEST | CPUM_USED_DEBUG_REGS_HYPER
+                                                | CPUM_SYNC_DEBUG_REGS_GUEST | CPUM_SYNC_DEBUG_REGS_HYPER));
+
+    /*
+     * Restore the host's debug state. DR0-3, DR6 and only then DR7!
+     */
+    if (pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_HOST)
+    {
+        /* A bit of paranoia first... */
+        uint64_t uCurDR7 = ASMGetDR7();
+        if (uCurDR7 != X86_DR7_INIT_VAL)
+            ASMSetDR7(X86_DR7_INIT_VAL);
+
+        ASMSetDR0(pVCpu->cpum.s.Host.dr0);
+        ASMSetDR1(pVCpu->cpum.s.Host.dr1);
+        ASMSetDR2(pVCpu->cpum.s.Host.dr2);
+        ASMSetDR3(pVCpu->cpum.s.Host.dr3);
+        /** @todo consider only updating if they differ, esp. DR6. Need to figure how
+         *        expensive DRx reads are over DRx writes.  */
+        ASMSetDR6(pVCpu->cpum.s.Host.dr6);
+        ASMSetDR7(pVCpu->cpum.s.Host.dr7);
+
+        ASMAtomicAndU32(&pVCpu->cpum.s.fUseFlags, ~CPUM_USED_DEBUG_REGS_HOST);
+    }
+
+    return fDrXLoaded;
+}
+
+
+/**
+ * Saves the guest DRx state if it resides host registers.
+ *
+ * This does NOT clear any use flags, so the host registers remains loaded with
+ * the guest DRx state upon return.  The purpose is only to make sure the values
+ * in the CPU context structure is up to date.
+ *
+ * @returns true if the host registers contains guest values, false if not.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   fDr6        Whether to include DR6 or not.
+ * @thread  EMT(pVCpu)
+ */
+VMMR0_INT_DECL(bool) CPUMR0DebugStateMaybeSaveGuest(PVMCPUCC pVCpu, bool fDr6)
+{
+    /*
+     * Do we need to save the guest DRx registered loaded into host registers?
+     * (DR7 and DR6 (if fDr6 is true) are left to the caller.)
+     */
+    if (pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_GUEST)
+    {
+        pVCpu->cpum.s.Guest.dr[0] = ASMGetDR0();
+        pVCpu->cpum.s.Guest.dr[1] = ASMGetDR1();
+        pVCpu->cpum.s.Guest.dr[2] = ASMGetDR2();
+        pVCpu->cpum.s.Guest.dr[3] = ASMGetDR3();
+        if (fDr6)
+            pVCpu->cpum.s.Guest.dr[6] = ASMGetDR6();
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Lazily sync in the debug state.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   fDr6        Whether to include DR6 or not.
+ * @thread  EMT(pVCpu)
+ */
+VMMR0_INT_DECL(void) CPUMR0LoadGuestDebugState(PVMCPUCC pVCpu, bool fDr6)
+{
+    /*
+     * Save the host state and disarm all host BPs.
+     */
+    cpumR0SaveHostDebugState(pVCpu);
+    Assert(ASMGetDR7() == X86_DR7_INIT_VAL);
+
+    /*
+     * Activate the guest state DR0-3.
+     * DR7 and DR6 (if fDr6 is true) are left to the caller.
+     */
+    ASMSetDR0(pVCpu->cpum.s.Guest.dr[0]);
+    ASMSetDR1(pVCpu->cpum.s.Guest.dr[1]);
+    ASMSetDR2(pVCpu->cpum.s.Guest.dr[2]);
+    ASMSetDR3(pVCpu->cpum.s.Guest.dr[3]);
+    if (fDr6)
+        ASMSetDR6(pVCpu->cpum.s.Guest.dr[6]);
+
+    ASMAtomicOrU32(&pVCpu->cpum.s.fUseFlags, CPUM_USED_DEBUG_REGS_GUEST);
+}
+
+
+/**
+ * Lazily sync in the hypervisor debug state
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   fDr6        Whether to include DR6 or not.
+ * @thread  EMT(pVCpu)
+ */
+VMMR0_INT_DECL(void) CPUMR0LoadHyperDebugState(PVMCPUCC pVCpu, bool fDr6)
+{
+    /*
+     * Save the host state and disarm all host BPs.
+     */
+    cpumR0SaveHostDebugState(pVCpu);
+    Assert(ASMGetDR7() == X86_DR7_INIT_VAL);
+
+    /*
+     * Make sure the hypervisor values are up to date.
+     */
+    CPUMRecalcHyperDRx(pVCpu, UINT8_MAX /* no loading, please */, true);
+
+    /*
+     * Activate the guest state DR0-3.
+     * DR7 and DR6 (if fDr6 is true) are left to the caller.
+     */
+    ASMSetDR0(pVCpu->cpum.s.Hyper.dr[0]);
+    ASMSetDR1(pVCpu->cpum.s.Hyper.dr[1]);
+    ASMSetDR2(pVCpu->cpum.s.Hyper.dr[2]);
+    ASMSetDR3(pVCpu->cpum.s.Hyper.dr[3]);
+    if (fDr6)
+        ASMSetDR6(X86_DR6_INIT_VAL);
+
+    ASMAtomicOrU32(&pVCpu->cpum.s.fUseFlags, CPUM_USED_DEBUG_REGS_HYPER);
+}
+
+#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+
+/**
+ * Per-CPU callback that probes the CPU for APIC support.
+ *
+ * @param   idCpu       The identifier for the CPU the function is called on.
+ * @param   pvUser1     Ignored.
+ * @param   pvUser2     Ignored.
+ */
+static DECLCALLBACK(void) cpumR0MapLocalApicCpuProber(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    NOREF(pvUser1); NOREF(pvUser2);
+    int iCpu = RTMpCpuIdToSetIndex(idCpu);
+    AssertReturnVoid(iCpu >= 0 && (unsigned)iCpu < RT_ELEMENTS(g_aLApics));
+
+    /*
+     * Check for APIC support.
+     */
+    uint32_t uMaxLeaf, u32EBX, u32ECX, u32EDX;
+    ASMCpuId(0, &uMaxLeaf, &u32EBX, &u32ECX, &u32EDX);
+    if (   (   ASMIsIntelCpuEx(u32EBX, u32ECX, u32EDX)
+            || ASMIsAmdCpuEx(u32EBX, u32ECX, u32EDX)
+            || ASMIsViaCentaurCpuEx(u32EBX, u32ECX, u32EDX)
+            || ASMIsShanghaiCpuEx(u32EBX, u32ECX, u32EDX)
+            || ASMIsHygonCpuEx(u32EBX, u32ECX, u32EDX))
+        && ASMIsValidStdRange(uMaxLeaf))
+    {
+        uint32_t uDummy;
+        ASMCpuId(1, &uDummy, &u32EBX, &u32ECX, &u32EDX);
+        if (    (u32EDX & X86_CPUID_FEATURE_EDX_APIC)
+            &&  (u32EDX & X86_CPUID_FEATURE_EDX_MSR))
+        {
+            /*
+             * Safe to access the MSR. Read it and calc the BASE (a little complicated).
+             */
+            uint64_t u64ApicBase = ASMRdMsr(MSR_IA32_APICBASE);
+            uint64_t u64Mask     = MSR_IA32_APICBASE_BASE_MIN;
+
+            /* see Intel Manual: Local APIC Status and Location: MAXPHYADDR default is bit 36 */
+            uint32_t uMaxExtLeaf;
+            ASMCpuId(0x80000000, &uMaxExtLeaf, &u32EBX, &u32ECX, &u32EDX);
+            if (   uMaxExtLeaf >= UINT32_C(0x80000008)
+                && ASMIsValidExtRange(uMaxExtLeaf))
+            {
+                uint32_t u32PhysBits;
+                ASMCpuId(0x80000008, &u32PhysBits, &u32EBX, &u32ECX, &u32EDX);
+                u32PhysBits &= 0xff;
+                u64Mask = ((UINT64_C(1) << u32PhysBits) - 1) & UINT64_C(0xfffffffffffff000);
+            }
+
+            AssertCompile(sizeof(g_aLApics[iCpu].PhysBase) == sizeof(u64ApicBase));
+            g_aLApics[iCpu].PhysBase    = u64ApicBase & u64Mask;
+            g_aLApics[iCpu].fEnabled    = RT_BOOL(u64ApicBase & MSR_IA32_APICBASE_EN);
+            g_aLApics[iCpu].fX2Apic     =    (u64ApicBase & (MSR_IA32_APICBASE_EXTD | MSR_IA32_APICBASE_EN))
+                                          == (MSR_IA32_APICBASE_EXTD | MSR_IA32_APICBASE_EN);
+        }
+    }
+}
+
+
+
+/**
+ * Per-CPU callback that verifies our APIC expectations.
+ *
+ * @param   idCpu       The identifier for the CPU the function is called on.
+ * @param   pvUser1     Ignored.
+ * @param   pvUser2     Ignored.
+ */
+static DECLCALLBACK(void) cpumR0MapLocalApicCpuChecker(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    NOREF(pvUser1); NOREF(pvUser2);
+
+    int iCpu = RTMpCpuIdToSetIndex(idCpu);
+    AssertReturnVoid(iCpu >= 0 && (unsigned)iCpu < RT_ELEMENTS(g_aLApics));
+    if (!g_aLApics[iCpu].fEnabled)
+        return;
+
+    /*
+     * 0x0X       82489 external APIC
+     * 0x1X       Local APIC
+     * 0x2X..0xFF reserved
+     */
+    uint32_t uApicVersion;
+    if (g_aLApics[iCpu].fX2Apic)
+        uApicVersion = ApicX2RegRead32(APIC_REG_VERSION);
+    else
+        uApicVersion = ApicRegRead(g_aLApics[iCpu].pv, APIC_REG_VERSION);
+    if ((APIC_REG_VERSION_GET_VER(uApicVersion) & 0xF0) == 0x10)
+    {
+        g_aLApics[iCpu].uVersion    = uApicVersion;
+
+# if 0 /* enable if you need it. */
+        if (g_aLApics[iCpu].fX2Apic)
+            SUPR0Printf("CPUM: X2APIC %02u - ver %#010x, lint0=%#07x lint1=%#07x pc=%#07x thmr=%#07x cmci=%#07x\n",
+                        iCpu, uApicVersion,
+                        ApicX2RegRead32(APIC_REG_LVT_LINT0), ApicX2RegRead32(APIC_REG_LVT_LINT1),
+                        ApicX2RegRead32(APIC_REG_LVT_PC), ApicX2RegRead32(APIC_REG_LVT_THMR),
+                        ApicX2RegRead32(APIC_REG_LVT_CMCI));
+        else
+        {
+            SUPR0Printf("CPUM: APIC %02u at %RGp (mapped at %p) - ver %#010x, lint0=%#07x lint1=%#07x pc=%#07x thmr=%#07x cmci=%#07x\n",
+                        iCpu, g_aLApics[iCpu].PhysBase, g_aLApics[iCpu].pv, uApicVersion,
+                        ApicRegRead(g_aLApics[iCpu].pv, APIC_REG_LVT_LINT0), ApicRegRead(g_aLApics[iCpu].pv, APIC_REG_LVT_LINT1),
+                        ApicRegRead(g_aLApics[iCpu].pv, APIC_REG_LVT_PC), ApicRegRead(g_aLApics[iCpu].pv, APIC_REG_LVT_THMR),
+                        ApicRegRead(g_aLApics[iCpu].pv, APIC_REG_LVT_CMCI));
+            if (uApicVersion & 0x80000000)
+            {
+                uint32_t uExtFeatures = ApicRegRead(g_aLApics[iCpu].pv, 0x400);
+                uint32_t cEiLvt = (uExtFeatures >> 16) & 0xff;
+                SUPR0Printf("CPUM: APIC %02u: ExtSpace available. extfeat=%08x eilvt[0..3]=%08x %08x %08x %08x\n",
+                            iCpu,
+                            ApicRegRead(g_aLApics[iCpu].pv, 0x400),
+                            cEiLvt >= 1 ? ApicRegRead(g_aLApics[iCpu].pv, 0x500) : 0,
+                            cEiLvt >= 2 ? ApicRegRead(g_aLApics[iCpu].pv, 0x510) : 0,
+                            cEiLvt >= 3 ? ApicRegRead(g_aLApics[iCpu].pv, 0x520) : 0,
+                            cEiLvt >= 4 ? ApicRegRead(g_aLApics[iCpu].pv, 0x530) : 0);
+            }
+        }
+# endif
+    }
+    else
+    {
+        g_aLApics[iCpu].fEnabled = false;
+        g_aLApics[iCpu].fX2Apic  = false;
+        SUPR0Printf("VBox/CPUM: Unsupported APIC version %#x (iCpu=%d)\n", uApicVersion, iCpu);
+    }
+}
+
+
+/**
+ * Map the MMIO page of each local APIC in the system.
+ */
+static int cpumR0MapLocalApics(void)
+{
+    /*
+     * Check that we'll always stay within the array bounds.
+     */
+    if (RTMpGetArraySize() > RT_ELEMENTS(g_aLApics))
+    {
+        LogRel(("CPUM: Too many real CPUs/cores/threads - %u, max %u\n", RTMpGetArraySize(), RT_ELEMENTS(g_aLApics)));
+        return VERR_TOO_MANY_CPUS;
+    }
+
+    /*
+     * Create mappings for all online CPUs we think have legacy APICs.
+     */
+    int rc = RTMpOnAll(cpumR0MapLocalApicCpuProber, NULL, NULL);
+
+    for (unsigned iCpu = 0; RT_SUCCESS(rc) && iCpu < RT_ELEMENTS(g_aLApics); iCpu++)
+    {
+        if (g_aLApics[iCpu].fEnabled && !g_aLApics[iCpu].fX2Apic)
+        {
+            rc = RTR0MemObjEnterPhys(&g_aLApics[iCpu].hMemObj, g_aLApics[iCpu].PhysBase,
+                                     PAGE_SIZE, RTMEM_CACHE_POLICY_MMIO);
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTR0MemObjMapKernel(&g_aLApics[iCpu].hMapObj, g_aLApics[iCpu].hMemObj, (void *)-1,
+                                         PAGE_SIZE, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
+                if (RT_SUCCESS(rc))
+                {
+                    g_aLApics[iCpu].pv = RTR0MemObjAddress(g_aLApics[iCpu].hMapObj);
+                    continue;
+                }
+                RTR0MemObjFree(g_aLApics[iCpu].hMemObj, true /* fFreeMappings */);
+            }
+            g_aLApics[iCpu].fEnabled = false;
+        }
+        g_aLApics[iCpu].pv = NULL;
+    }
+
+    /*
+     * Check the APICs.
+     */
+    if (RT_SUCCESS(rc))
+        rc = RTMpOnAll(cpumR0MapLocalApicCpuChecker, NULL, NULL);
+
+    if (RT_FAILURE(rc))
+    {
+        cpumR0UnmapLocalApics();
+        return rc;
+    }
+
+# ifdef LOG_ENABLED
+    /*
+     * Log the result (pretty useless, requires enabling CPUM in VBoxDrv
+     * and !VBOX_WITH_R0_LOGGING).
+     */
+    if (LogIsEnabled())
+    {
+        uint32_t cEnabled = 0;
+        uint32_t cX2Apics = 0;
+        for (unsigned iCpu = 0; iCpu < RT_ELEMENTS(g_aLApics); iCpu++)
+            if (g_aLApics[iCpu].fEnabled)
+            {
+                cEnabled++;
+                cX2Apics += g_aLApics[iCpu].fX2Apic;
+            }
+        Log(("CPUM: %u APICs, %u X2APICs\n", cEnabled, cX2Apics));
+    }
+# endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Unmap the Local APIC of all host CPUs.
+ */
+static void cpumR0UnmapLocalApics(void)
+{
+    for (unsigned iCpu = RT_ELEMENTS(g_aLApics); iCpu-- > 0;)
+    {
+        if (g_aLApics[iCpu].pv)
+        {
+            RTR0MemObjFree(g_aLApics[iCpu].hMapObj, true /* fFreeMappings */);
+            RTR0MemObjFree(g_aLApics[iCpu].hMemObj, true /* fFreeMappings */);
+            g_aLApics[iCpu].hMapObj  = NIL_RTR0MEMOBJ;
+            g_aLApics[iCpu].hMemObj  = NIL_RTR0MEMOBJ;
+            g_aLApics[iCpu].fEnabled = false;
+            g_aLApics[iCpu].fX2Apic  = false;
+            g_aLApics[iCpu].pv       = NULL;
+        }
+    }
+}
+
+
+/**
+ * Updates CPUMCPU::pvApicBase and CPUMCPU::fX2Apic prior to world switch.
+ *
+ * Writes the Local APIC mapping address of the current host CPU to CPUMCPU so
+ * the world switchers can access the APIC registers for the purpose of
+ * disabling and re-enabling the NMIs.  Must be called with disabled preemption
+ * or disabled interrupts!
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   iHostCpuSet The CPU set index of the current host CPU.
+ */
+VMMR0_INT_DECL(void) CPUMR0SetLApic(PVMCPUCC pVCpu, uint32_t iHostCpuSet)
+{
+    Assert(iHostCpuSet <= RT_ELEMENTS(g_aLApics));
+    pVCpu->cpum.s.pvApicBase = g_aLApics[iHostCpuSet].pv;
+    pVCpu->cpum.s.fX2Apic    = g_aLApics[iHostCpuSet].fX2Apic;
+//    Log6(("CPUMR0SetLApic: pvApicBase=%p fX2Apic=%d\n", g_aLApics[idxCpu].pv, g_aLApics[idxCpu].fX2Apic));
+}
+
+#endif /* VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI */
+
diff --git a/src/VBox/VMM/VMMR0/CPUMR0A.asm b/src/VBox/VMM/VMMR0/CPUMR0A.asm
new file mode 100644
index 00000000..3452a815
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/CPUMR0A.asm
@@ -0,0 +1,358 @@
+ ; $Id: CPUMR0A.asm $
+;; @file
+; CPUM - Ring-0 Assembly Routines (supporting HM and IEM).
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+
+;*******************************************************************************
+;* Header Files                                                                *
+;*******************************************************************************
+%define RT_ASM_WITH_SEH64
+%include "iprt/asmdefs.mac"
+%include "VBox/asmdefs.mac"
+%include "VBox/vmm/vm.mac"
+%include "VBox/err.mac"
+%include "VBox/vmm/stam.mac"
+%include "CPUMInternal.mac"
+%include "iprt/x86.mac"
+%include "VBox/vmm/cpum.mac"
+
+
+BEGINCODE
+
+;;
+; Makes sure the EMTs have a FPU state associated with them on hosts where we're
+; allowed to use it in ring-0 too.
+;
+; This ensure that we don't have to allocate the state lazily while trying to execute
+; guest code with preemption disabled or worse.
+;
+; @cproto VMMR0_INT_DECL(void) CPUMR0RegisterVCpuThread(PVMCPU pVCpu);
+;
+BEGINPROC CPUMR0RegisterVCpuThread
+        push    xBP
+        SEH64_PUSH_xBP
+        mov     xBP, xSP
+        SEH64_SET_FRAME_xBP 0
+SEH64_END_PROLOGUE
+
+%ifdef VMM_R0_TOUCH_FPU
+        movdqa  xmm0, xmm0              ; hope this is harmless.
+%endif
+
+.return:
+        xor     eax, eax                ; paranoia
+        leave
+        ret
+ENDPROC   CPUMR0RegisterVCpuThread
+
+
+%ifdef VMM_R0_TOUCH_FPU
+;;
+; Touches the host FPU state.
+;
+; @uses nothing (well, maybe cr0)
+;
+ %ifndef RT_ASM_WITH_SEH64 ; workaround for yasm 1.3.0 bug (error: prologue -1 bytes, must be <256)
+ALIGNCODE(16)
+ %endif
+BEGINPROC CPUMR0TouchHostFpu
+        push    xBP
+        SEH64_PUSH_xBP
+        mov     xBP, xSP
+        SEH64_SET_FRAME_xBP 0
+SEH64_END_PROLOGUE
+
+        movdqa  xmm0, xmm0              ; Hope this is harmless.
+
+        leave
+        ret
+ENDPROC   CPUMR0TouchHostFpu
+%endif ; VMM_R0_TOUCH_FPU
+
+
+;;
+; Saves the host FPU/SSE/AVX state and restores the guest FPU/SSE/AVX state.
+;
+; @returns  VINF_SUCCESS (0) or VINF_CPUM_HOST_CR0_MODIFIED. (EAX)
+; @param    pCpumCpu  x86:[ebp+8] gcc:rdi msc:rcx     CPUMCPU pointer
+;
+; @remarks  64-bit Windows drivers shouldn't use AVX registers without saving+loading:
+;               https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
+;           However the compiler docs have different idea:
+;               https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
+;           We'll go with the former for now.
+;
+%ifndef RT_ASM_WITH_SEH64 ; workaround for yasm 1.3.0 bug (error: prologue -1 bytes, must be <256)
+ALIGNCODE(16)
+%endif
+BEGINPROC cpumR0SaveHostRestoreGuestFPUState
+        push    xBP
+        SEH64_PUSH_xBP
+        mov     xBP, xSP
+        SEH64_SET_FRAME_xBP 0
+SEH64_END_PROLOGUE
+
+        ;
+        ; Prologue - xAX+xDX must be free for XSAVE/XRSTOR input.
+        ;
+%ifdef RT_ARCH_AMD64
+ %ifdef RT_OS_WINDOWS
+        mov     r11, rcx
+ %else
+        mov     r11, rdi
+ %endif
+ %define pCpumCpu   r11
+ %define pXState    r10
+%else
+        push    ebx
+        push    esi
+        mov     ebx, dword [ebp + 8]
+ %define pCpumCpu ebx
+ %define pXState  esi
+%endif
+
+        pushf                           ; The darwin kernel can get upset or upset things if an
+        cli                             ; interrupt occurs while we're doing fxsave/fxrstor/cr0.
+
+        ;
+        ; Save the host state.
+        ;
+        test    dword [pCpumCpu + CPUMCPU.fUseFlags], CPUM_USED_FPU_HOST
+        jnz     .already_saved_host
+
+        CPUMRZ_TOUCH_FPU_CLEAR_CR0_FPU_TRAPS_SET_RC xCX, xAX, pCpumCpu ; xCX is the return value for VT-x; xAX is scratch.
+
+        CPUMR0_SAVE_HOST
+
+%ifdef VBOX_WITH_KERNEL_USING_XMM
+        jmp     .load_guest
+%endif
+.already_saved_host:
+%ifdef VBOX_WITH_KERNEL_USING_XMM
+        ; If we didn't save the host state, we must save the non-volatile XMM registers.
+        mov     pXState, [pCpumCpu + CPUMCPU.Host.pXStateR0]
+        stmxcsr [pXState + X86FXSTATE.MXCSR]
+        movdqa  [pXState + X86FXSTATE.xmm6 ], xmm6
+        movdqa  [pXState + X86FXSTATE.xmm7 ], xmm7
+        movdqa  [pXState + X86FXSTATE.xmm8 ], xmm8
+        movdqa  [pXState + X86FXSTATE.xmm9 ], xmm9
+        movdqa  [pXState + X86FXSTATE.xmm10], xmm10
+        movdqa  [pXState + X86FXSTATE.xmm11], xmm11
+        movdqa  [pXState + X86FXSTATE.xmm12], xmm12
+        movdqa  [pXState + X86FXSTATE.xmm13], xmm13
+        movdqa  [pXState + X86FXSTATE.xmm14], xmm14
+        movdqa  [pXState + X86FXSTATE.xmm15], xmm15
+
+        ;
+        ; Load the guest state.
+        ;
+.load_guest:
+%endif
+        CPUMR0_LOAD_GUEST
+
+%ifdef VBOX_WITH_KERNEL_USING_XMM
+        ; Restore the non-volatile xmm registers. ASSUMING 64-bit host.
+        mov     pXState, [pCpumCpu + CPUMCPU.Host.pXStateR0]
+        movdqa  xmm6,  [pXState + X86FXSTATE.xmm6]
+        movdqa  xmm7,  [pXState + X86FXSTATE.xmm7]
+        movdqa  xmm8,  [pXState + X86FXSTATE.xmm8]
+        movdqa  xmm9,  [pXState + X86FXSTATE.xmm9]
+        movdqa  xmm10, [pXState + X86FXSTATE.xmm10]
+        movdqa  xmm11, [pXState + X86FXSTATE.xmm11]
+        movdqa  xmm12, [pXState + X86FXSTATE.xmm12]
+        movdqa  xmm13, [pXState + X86FXSTATE.xmm13]
+        movdqa  xmm14, [pXState + X86FXSTATE.xmm14]
+        movdqa  xmm15, [pXState + X86FXSTATE.xmm15]
+        ldmxcsr        [pXState + X86FXSTATE.MXCSR]
+%endif
+
+        or      dword [pCpumCpu + CPUMCPU.fUseFlags], (CPUM_USED_FPU_GUEST | CPUM_USED_FPU_SINCE_REM | CPUM_USED_FPU_HOST)
+        popf
+
+        mov     eax, ecx
+.return:
+%ifdef RT_ARCH_X86
+        pop     esi
+        pop     ebx
+%endif
+        leave
+        ret
+ENDPROC   cpumR0SaveHostRestoreGuestFPUState
+
+
+;;
+; Saves the guest FPU/SSE/AVX state and restores the host FPU/SSE/AVX state.
+;
+; @param    pCpumCpu  x86:[ebp+8] gcc:rdi msc:rcx     CPUMCPU pointer
+;
+; @remarks  64-bit Windows drivers shouldn't use AVX registers without saving+loading:
+;               https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
+;           However the compiler docs have different idea:
+;               https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
+;           We'll go with the former for now.
+;
+%ifndef RT_ASM_WITH_SEH64 ; workaround for yasm 1.3.0 bug (error: prologue -1 bytes, must be <256)
+ALIGNCODE(16)
+%endif
+BEGINPROC cpumR0SaveGuestRestoreHostFPUState
+        push    xBP
+        SEH64_PUSH_xBP
+        mov     xBP, xSP
+        SEH64_SET_FRAME_xBP 0
+SEH64_END_PROLOGUE
+
+        ;
+        ; Prologue - xAX+xDX must be free for XSAVE/XRSTOR input.
+        ;
+%ifdef RT_ARCH_AMD64
+ %ifdef RT_OS_WINDOWS
+        mov     r11, rcx
+ %else
+        mov     r11, rdi
+ %endif
+ %define pCpumCpu   r11
+ %define pXState    r10
+%else
+        push    ebx
+        push    esi
+        mov     ebx, dword [ebp + 8]
+ %define pCpumCpu   ebx
+ %define pXState    esi
+%endif
+        pushf                           ; The darwin kernel can get upset or upset things if an
+        cli                             ; interrupt occurs while we're doing fxsave/fxrstor/cr0.
+
+ %ifdef VBOX_WITH_KERNEL_USING_XMM
+        ;
+        ; Copy non-volatile XMM registers to the host state so we can use
+        ; them while saving the guest state (we've gotta do this anyway).
+        ;
+        mov     pXState, [pCpumCpu + CPUMCPU.Host.pXStateR0]
+        stmxcsr [pXState + X86FXSTATE.MXCSR]
+        movdqa  [pXState + X86FXSTATE.xmm6], xmm6
+        movdqa  [pXState + X86FXSTATE.xmm7], xmm7
+        movdqa  [pXState + X86FXSTATE.xmm8], xmm8
+        movdqa  [pXState + X86FXSTATE.xmm9], xmm9
+        movdqa  [pXState + X86FXSTATE.xmm10], xmm10
+        movdqa  [pXState + X86FXSTATE.xmm11], xmm11
+        movdqa  [pXState + X86FXSTATE.xmm12], xmm12
+        movdqa  [pXState + X86FXSTATE.xmm13], xmm13
+        movdqa  [pXState + X86FXSTATE.xmm14], xmm14
+        movdqa  [pXState + X86FXSTATE.xmm15], xmm15
+ %endif
+
+        ;
+        ; Save the guest state if necessary.
+        ;
+        test    dword [pCpumCpu + CPUMCPU.fUseFlags], CPUM_USED_FPU_GUEST
+        jz      .load_only_host
+
+ %ifdef VBOX_WITH_KERNEL_USING_XMM
+        ; Load the guest XMM register values we already saved in HMR0VMXStartVMWrapXMM.
+        mov     pXState, [pCpumCpu + CPUMCPU.Guest.pXStateR0]
+        movdqa  xmm0,  [pXState + X86FXSTATE.xmm0]
+        movdqa  xmm1,  [pXState + X86FXSTATE.xmm1]
+        movdqa  xmm2,  [pXState + X86FXSTATE.xmm2]
+        movdqa  xmm3,  [pXState + X86FXSTATE.xmm3]
+        movdqa  xmm4,  [pXState + X86FXSTATE.xmm4]
+        movdqa  xmm5,  [pXState + X86FXSTATE.xmm5]
+        movdqa  xmm6,  [pXState + X86FXSTATE.xmm6]
+        movdqa  xmm7,  [pXState + X86FXSTATE.xmm7]
+        movdqa  xmm8,  [pXState + X86FXSTATE.xmm8]
+        movdqa  xmm9,  [pXState + X86FXSTATE.xmm9]
+        movdqa  xmm10, [pXState + X86FXSTATE.xmm10]
+        movdqa  xmm11, [pXState + X86FXSTATE.xmm11]
+        movdqa  xmm12, [pXState + X86FXSTATE.xmm12]
+        movdqa  xmm13, [pXState + X86FXSTATE.xmm13]
+        movdqa  xmm14, [pXState + X86FXSTATE.xmm14]
+        movdqa  xmm15, [pXState + X86FXSTATE.xmm15]
+        ldmxcsr        [pXState + X86FXSTATE.MXCSR]
+ %endif
+        CPUMR0_SAVE_GUEST
+
+        ;
+        ; Load the host state.
+        ;
+.load_only_host:
+        CPUMR0_LOAD_HOST
+
+        ; Restore the CR0 value we saved in cpumR0SaveHostRestoreGuestFPUState or
+        ; in cpumRZSaveHostFPUState.
+        mov     xCX, [pCpumCpu + CPUMCPU.Host.cr0Fpu]
+        CPUMRZ_RESTORE_CR0_IF_TS_OR_EM_SET xCX
+        and     dword [pCpumCpu + CPUMCPU.fUseFlags], ~(CPUM_USED_FPU_GUEST | CPUM_USED_FPU_HOST)
+
+        popf
+%ifdef RT_ARCH_X86
+        pop     esi
+        pop     ebx
+%endif
+        leave
+        ret
+%undef pCpumCpu
+%undef pXState
+ENDPROC   cpumR0SaveGuestRestoreHostFPUState
+
+
+%if ARCH_BITS == 32
+ %ifdef VBOX_WITH_64_BITS_GUESTS
+;;
+; Restores the host's FPU/SSE/AVX state from pCpumCpu->Host.
+;
+; @param    pCpumCpu  x86:[ebp+8] gcc:rdi msc:rcx     CPUMCPU pointer
+;
+  %ifndef RT_ASM_WITH_SEH64 ; workaround for yasm 1.3.0 bug (error: prologue -1 bytes, must be <256)
+ALIGNCODE(16)
+  %endif
+BEGINPROC cpumR0RestoreHostFPUState
+        ;
+        ; Prologue - xAX+xDX must be free for XSAVE/XRSTOR input.
+        ;
+        push    ebp
+        mov     ebp, esp
+        push    ebx
+        push    esi
+        mov     ebx, dword [ebp + 8]
+  %define pCpumCpu ebx
+  %define pXState  esi
+
+        ;
+        ; Restore host CPU state.
+        ;
+        pushf                           ; The darwin kernel can get upset or upset things if an
+        cli                             ; interrupt occurs while we're doing fxsave/fxrstor/cr0.
+
+        CPUMR0_LOAD_HOST
+
+        ; Restore the CR0 value we saved in cpumR0SaveHostRestoreGuestFPUState or
+        ; in cpumRZSaveHostFPUState.
+        ;; @todo What about XCR0?
+        mov     xCX, [pCpumCpu + CPUMCPU.Host.cr0Fpu]
+        CPUMRZ_RESTORE_CR0_IF_TS_OR_EM_SET xCX
+
+        and     dword [pCpumCpu + CPUMCPU.fUseFlags], ~CPUM_USED_FPU_HOST
+        popf
+
+        pop     esi
+        pop     ebx
+        leave
+        ret
+  %undef pCpumCPu
+  %undef pXState
+ENDPROC   cpumR0RestoreHostFPUState
+ %endif ; VBOX_WITH_64_BITS_GUESTS
+%endif  ; ARCH_BITS == 32
+
diff --git a/src/VBox/VMM/VMMR0/EMR0.cpp b/src/VBox/VMM/VMMR0/EMR0.cpp
new file mode 100644
index 00000000..8cc3b0fb
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/EMR0.cpp
@@ -0,0 +1,61 @@
+/* $Id: EMR0.cpp $ */
+/** @file
+ * EM - Host Context Ring 0.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_EM
+#include <VBox/vmm/em.h>
+#include "EMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/gvm.h>
+#include <iprt/errcore.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+
+
+
+/**
+ * Adjusts EM configuration options.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The ring-0 VM structure.
+ */
+VMMR0_INT_DECL(int) EMR0InitVM(PGVM pGVM)
+{
+    /*
+     * Override ring-0 exit optimizations settings.
+     */
+    PVMCPUCC pVCpu0 = &pGVM->aCpus[0];
+    bool fEnabledR0                = pVCpu0->em.s.fExitOptimizationEnabled
+                                  && pVCpu0->em.s.fExitOptimizationEnabledR0
+                                  && (RTThreadPreemptIsPossible() || RTThreadPreemptIsPendingTrusty());
+    bool fEnabledR0PreemptDisabled = fEnabledR0
+                                  && pVCpu0->em.s.fExitOptimizationEnabledR0PreemptDisabled
+                                  && RTThreadPreemptIsPendingTrusty();
+    for (VMCPUID idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = &pGVM->aCpus[idCpu];
+        pVCpu->em.s.fExitOptimizationEnabledR0                = fEnabledR0;
+        pVCpu->em.s.fExitOptimizationEnabledR0PreemptDisabled = fEnabledR0PreemptDisabled;
+    }
+
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMR0/GIMR0.cpp b/src/VBox/VMM/VMMR0/GIMR0.cpp
new file mode 100644
index 00000000..69667273
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/GIMR0.cpp
@@ -0,0 +1,111 @@
+/* $Id: GIMR0.cpp $ */
+/** @file
+ * Guest Interface Manager (GIM) - Host Context Ring-0.
+ */
+
+/*
+ * Copyright (C) 2014-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/gim.h>
+#include "GIMInternal.h"
+#include "GIMHvInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/err.h>
+
+
+/**
+ * Does ring-0 per-VM GIM initialization.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR0_INT_DECL(int) GIMR0InitVM(PVMCC pVM)
+{
+    if (!GIMIsEnabled(pVM))
+        return VINF_SUCCESS;
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimR0HvInitVM(pVM);
+
+        default:
+            break;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Does ring-0 per-VM GIM termination.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR0_INT_DECL(int) GIMR0TermVM(PVMCC pVM)
+{
+    if (!GIMIsEnabled(pVM))
+        return VINF_SUCCESS;
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimR0HvTermVM(pVM);
+
+        default:
+            break;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Updates the paravirtualized TSC supported by the GIM provider.
+ *
+ * @returns VBox status code.
+ * @retval VINF_SUCCESS if the paravirt. TSC is setup and in use.
+ * @retval VERR_GIM_NOT_ENABLED if no GIM provider is configured for this VM.
+ * @retval VERR_GIM_PVTSC_NOT_AVAILABLE if the GIM provider does not support any
+ *         paravirt. TSC.
+ * @retval VERR_GIM_PVTSC_NOT_IN_USE if the GIM provider supports paravirt. TSC
+ *         but the guest isn't currently using it.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   u64Offset   The computed TSC offset.
+ *
+ * @thread EMT(pVCpu)
+ */
+VMMR0_INT_DECL(int) GIMR0UpdateParavirtTsc(PVMCC pVM, uint64_t u64Offset)
+{
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimR0HvUpdateParavirtTsc(pVM, u64Offset);
+
+        case GIMPROVIDERID_KVM:
+            return VINF_SUCCESS;
+
+        case GIMPROVIDERID_NONE:
+            return VERR_GIM_NOT_ENABLED;
+
+        default:
+            break;
+    }
+    return VERR_GIM_PVTSC_NOT_AVAILABLE;
+}
+
diff --git a/src/VBox/VMM/VMMR0/GIMR0Hv.cpp b/src/VBox/VMM/VMMR0/GIMR0Hv.cpp
new file mode 100644
index 00000000..a4ec1d3b
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/GIMR0Hv.cpp
@@ -0,0 +1,182 @@
+/* $Id: GIMR0Hv.cpp $ */
+/** @file
+ * Guest Interface Manager (GIM), Hyper-V - Host Context Ring-0.
+ */
+
+/*
+ * Copyright (C) 2014-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/tm.h>
+#include "GIMInternal.h"
+#include "GIMHvInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/err.h>
+
+#include <iprt/spinlock.h>
+
+
+#if 0
+/**
+ * Allocates and maps one physically contiguous page. The allocated page is
+ * zero'd out.
+ *
+ * @returns IPRT status code.
+ * @param   pMemObj         Pointer to the ring-0 memory object.
+ * @param   ppVirt          Where to store the virtual address of the
+ *                          allocation.
+ * @param   pPhys           Where to store the physical address of the
+ *                          allocation.
+ */
+static int gimR0HvPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
+{
+    AssertPtr(pMemObj);
+    AssertPtr(ppVirt);
+    AssertPtr(pHCPhys);
+
+    int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
+    if (RT_FAILURE(rc))
+        return rc;
+    *ppVirt  = RTR0MemObjAddress(*pMemObj);
+    *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
+    ASMMemZero32(*ppVirt, PAGE_SIZE);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Frees and unmaps an allocated physical page.
+ *
+ * @param   pMemObj         Pointer to the ring-0 memory object.
+ * @param   ppVirt          Where to re-initialize the virtual address of
+ *                          allocation as 0.
+ * @param   pHCPhys         Where to re-initialize the physical address of the
+ *                          allocation as 0.
+ */
+static void gimR0HvPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
+{
+    AssertPtr(pMemObj);
+    AssertPtr(ppVirt);
+    AssertPtr(pHCPhys);
+    if (*pMemObj != NIL_RTR0MEMOBJ)
+    {
+        int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
+        AssertRC(rc);
+        *pMemObj = NIL_RTR0MEMOBJ;
+        *ppVirt  = 0;
+        *pHCPhys = 0;
+    }
+}
+#endif
+
+/**
+ * Updates Hyper-V's reference TSC page.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   u64Offset   The computed TSC offset.
+ * @thread  EMT.
+ */
+VMM_INT_DECL(int) gimR0HvUpdateParavirtTsc(PVMCC pVM, uint64_t u64Offset)
+{
+    Assert(GIMIsEnabled(pVM));
+    bool fHvTscEnabled = MSR_GIM_HV_REF_TSC_IS_ENABLED(pVM->gim.s.u.Hv.u64TscPageMsr);
+    if (RT_UNLIKELY(!fHvTscEnabled))
+        return VERR_GIM_PVTSC_NOT_ENABLED;
+
+    /** @todo this is buggy when large pages are used due to a PGM limitation, see
+     *        @bugref{7532}.
+     *
+     *        In any case, we do not ever update this page while the guest is
+     *        running after setting it up (in ring-3, see gimR3HvEnableTscPage()) as
+     *        the TSC offset is handled in the VMCS/VMCB (HM) or by trapping RDTSC
+     *        (raw-mode). */
+#if 0
+    PCGIMHV          pcHv     = &pVM->gim.s.u.Hv;
+    PCGIMMMIO2REGION pcRegion = &pcHv->aMmio2Regions[GIM_HV_REF_TSC_PAGE_REGION_IDX];
+    PGIMHVREFTSC     pRefTsc  = (PGIMHVREFTSC)pcRegion->CTX_SUFF(pvPage);
+    Assert(pRefTsc);
+
+    /*
+     * Hyper-V reports the reference time in 100 nanosecond units.
+     */
+    uint64_t u64Tsc100Ns = pcHv->cTscTicksPerSecond / RT_NS_10MS;
+    int64_t i64TscOffset = (int64_t)u64Offset / u64Tsc100Ns;
+
+    /*
+     * The TSC page can be simulatenously read by other VCPUs in the guest. The
+     * spinlock is only for protecting simultaneous hypervisor writes from other
+     * EMTs.
+     */
+    RTSpinlockAcquire(pcHv->hSpinlockR0);
+    if (pRefTsc->i64TscOffset != i64TscOffset)
+    {
+        if (pRefTsc->u32TscSequence < UINT32_C(0xfffffffe))
+            ASMAtomicIncU32(&pRefTsc->u32TscSequence);
+        else
+            ASMAtomicWriteU32(&pRefTsc->u32TscSequence, 1);
+        ASMAtomicWriteS64(&pRefTsc->i64TscOffset, i64TscOffset);
+    }
+    RTSpinlockRelease(pcHv->hSpinlockR0);
+
+    Assert(pRefTsc->u32TscSequence != 0);
+    Assert(pRefTsc->u32TscSequence != UINT32_C(0xffffffff));
+#else
+    NOREF(u64Offset);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Does ring-0 per-VM GIM Hyper-V initialization.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR0_INT_DECL(int) gimR0HvInitVM(PVMCC pVM)
+{
+    AssertPtr(pVM);
+    Assert(GIMIsEnabled(pVM));
+
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    Assert(pHv->hSpinlockR0 == NIL_RTSPINLOCK);
+
+    int rc = RTSpinlockCreate(&pHv->hSpinlockR0, RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE, "Hyper-V");
+    return rc;
+}
+
+
+/**
+ * Does ring-0 per-VM GIM Hyper-V termination.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR0_INT_DECL(int) gimR0HvTermVM(PVMCC pVM)
+{
+    AssertPtr(pVM);
+    Assert(GIMIsEnabled(pVM));
+
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    RTSpinlockDestroy(pHv->hSpinlockR0);
+    pHv->hSpinlockR0 = NIL_RTSPINLOCK;
+
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMR0/GMMR0.cpp b/src/VBox/VMM/VMMR0/GMMR0.cpp
new file mode 100644
index 00000000..3c4f77ff
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/GMMR0.cpp
@@ -0,0 +1,5746 @@
+/* $Id: GMMR0.cpp $ */
+/** @file
+ * GMM - Global Memory Manager.
+ */
+
+/*
+ * Copyright (C) 2007-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_gmm    GMM - The Global Memory Manager
+ *
+ * As the name indicates, this component is responsible for global memory
+ * management. Currently only guest RAM is allocated from the GMM, but this
+ * may change to include shadow page tables and other bits later.
+ *
+ * Guest RAM is managed as individual pages, but allocated from the host OS
+ * in chunks for reasons of portability / efficiency. To minimize the memory
+ * footprint all tracking structure must be as small as possible without
+ * unnecessary performance penalties.
+ *
+ * The allocation chunks has fixed sized, the size defined at compile time
+ * by the #GMM_CHUNK_SIZE \#define.
+ *
+ * Each chunk is given an unique ID. Each page also has a unique ID. The
+ * relationship between the two IDs is:
+ * @code
+ *  GMM_CHUNK_SHIFT = log2(GMM_CHUNK_SIZE / PAGE_SIZE);
+ *  idPage = (idChunk << GMM_CHUNK_SHIFT) | iPage;
+ * @endcode
+ * Where iPage is the index of the page within the chunk. This ID scheme
+ * permits for efficient chunk and page lookup, but it relies on the chunk size
+ * to be set at compile time. The chunks are organized in an AVL tree with their
+ * IDs being the keys.
+ *
+ * The physical address of each page in an allocation chunk is maintained by
+ * the #RTR0MEMOBJ and obtained using #RTR0MemObjGetPagePhysAddr. There is no
+ * need to duplicate this information (it'll cost 8-bytes per page if we did).
+ *
+ * So what do we need to track per page? Most importantly we need to know
+ * which state the page is in:
+ *   - Private - Allocated for (eventually) backing one particular VM page.
+ *   - Shared  - Readonly page that is used by one or more VMs and treated
+ *               as COW by PGM.
+ *   - Free    - Not used by anyone.
+ *
+ * For the page replacement operations (sharing, defragmenting and freeing)
+ * to be somewhat efficient, private pages needs to be associated with a
+ * particular page in a particular VM.
+ *
+ * Tracking the usage of shared pages is impractical and expensive, so we'll
+ * settle for a reference counting system instead.
+ *
+ * Free pages will be chained on LIFOs
+ *
+ * On 64-bit systems we will use a 64-bit bitfield per page, while on 32-bit
+ * systems a 32-bit bitfield will have to suffice because of address space
+ * limitations. The #GMMPAGE structure shows the details.
+ *
+ *
+ * @section sec_gmm_alloc_strat Page Allocation Strategy
+ *
+ * The strategy for allocating pages has to take fragmentation and shared
+ * pages into account, or we may end up with with 2000 chunks with only
+ * a few pages in each. Shared pages cannot easily be reallocated because
+ * of the inaccurate usage accounting (see above). Private pages can be
+ * reallocated by a defragmentation thread in the same manner that sharing
+ * is done.
+ *
+ * The first approach is to manage the free pages in two sets depending on
+ * whether they are mainly for the allocation of shared or private pages.
+ * In the initial implementation there will be almost no possibility for
+ * mixing shared and private pages in the same chunk (only if we're really
+ * stressed on memory), but when we implement forking of VMs and have to
+ * deal with lots of COW pages it'll start getting kind of interesting.
+ *
+ * The sets are lists of chunks with approximately the same number of
+ * free pages. Say the chunk size is 1MB, meaning 256 pages, and a set
+ * consists of 16 lists. So, the first list will contain the chunks with
+ * 1-7 free pages, the second covers 8-15, and so on. The chunks will be
+ * moved between the lists as pages are freed up or allocated.
+ *
+ *
+ * @section sec_gmm_costs       Costs
+ *
+ * The per page cost in kernel space is 32-bit plus whatever RTR0MEMOBJ
+ * entails. In addition there is the chunk cost of approximately
+ * (sizeof(RT0MEMOBJ) + sizeof(CHUNK)) / 2^CHUNK_SHIFT bytes per page.
+ *
+ * On Windows the per page #RTR0MEMOBJ cost is 32-bit on 32-bit windows
+ * and 64-bit on 64-bit windows (a PFN_NUMBER in the MDL). So, 64-bit per page.
+ * The cost on Linux is identical, but here it's because of sizeof(struct page *).
+ *
+ *
+ * @section sec_gmm_legacy      Legacy Mode for Non-Tier-1 Platforms
+ *
+ * In legacy mode the page source is locked user pages and not
+ * #RTR0MemObjAllocPhysNC, this means that a page can only be allocated
+ * by the VM that locked it. We will make no attempt at implementing
+ * page sharing on these systems, just do enough to make it all work.
+ *
+ * @note With 6.1 really dropping 32-bit support, the legacy mode is obsoleted
+ *       under the assumption that there is sufficient kernel virtual address
+ *       space to map all of the guest memory allocations.  So, we'll be using
+ *       #RTR0MemObjAllocPage on some platforms as an alternative to
+ *       #RTR0MemObjAllocPhysNC.
+ *
+ *
+ * @subsection sub_gmm_locking  Serializing
+ *
+ * One simple fast mutex will be employed in the initial implementation, not
+ * two as mentioned in @ref sec_pgmPhys_Serializing.
+ *
+ * @see @ref sec_pgmPhys_Serializing
+ *
+ *
+ * @section sec_gmm_overcommit  Memory Over-Commitment Management
+ *
+ * The GVM will have to do the system wide memory over-commitment
+ * management. My current ideas are:
+ *      - Per VM oc policy that indicates how much to initially commit
+ *        to it and what to do in a out-of-memory situation.
+ *      - Prevent overtaxing the host.
+ *
+ * There are some challenges here, the main ones are configurability and
+ * security. Should we for instance permit anyone to request 100% memory
+ * commitment? Who should be allowed to do runtime adjustments of the
+ * config. And how to prevent these settings from being lost when the last
+ * VM process exits? The solution is probably to have an optional root
+ * daemon the will keep VMMR0.r0 in memory and enable the security measures.
+ *
+ *
+ *
+ * @section sec_gmm_numa        NUMA
+ *
+ * NUMA considerations will be designed and implemented a bit later.
+ *
+ * The preliminary guesses is that we will have to try allocate memory as
+ * close as possible to the CPUs the VM is executed on (EMT and additional CPU
+ * threads). Which means it's mostly about allocation and sharing policies.
+ * Both the scheduler and allocator interface will to supply some NUMA info
+ * and we'll need to have a way to calc access costs.
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GMM
+#include <VBox/rawpci.h>
+#include <VBox/vmm/gmm.h>
+#include "GMMR0Internal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/log.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/VMMDev.h>
+#include <iprt/asm.h>
+#include <iprt/avl.h>
+#ifdef VBOX_STRICT
+# include <iprt/crc.h>
+#endif
+#include <iprt/critsect.h>
+#include <iprt/list.h>
+#include <iprt/mem.h>
+#include <iprt/memobj.h>
+#include <iprt/mp.h>
+#include <iprt/semaphore.h>
+#include <iprt/spinlock.h>
+#include <iprt/string.h>
+#include <iprt/time.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** @def VBOX_USE_CRIT_SECT_FOR_GIANT
+ * Use a critical section instead of a fast mutex for the giant GMM lock.
+ *
+ * @remarks This is primarily a way of avoiding the deadlock checks in the
+ *          windows driver verifier. */
+#if defined(RT_OS_WINDOWS) || defined(RT_OS_DARWIN) || defined(DOXYGEN_RUNNING)
+# define VBOX_USE_CRIT_SECT_FOR_GIANT
+#endif
+
+#if (!defined(VBOX_WITH_RAM_IN_KERNEL) || defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)) \
+ && !defined(RT_OS_DARWIN)
+/** Enable the legacy mode code (will be dropped soon). */
+# define GMM_WITH_LEGACY_MODE
+#endif
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/** Pointer to set of free chunks.  */
+typedef struct GMMCHUNKFREESET *PGMMCHUNKFREESET;
+
+/**
+ * The per-page tracking structure employed by the GMM.
+ *
+ * On 32-bit hosts we'll some trickery is necessary to compress all
+ * the information into 32-bits. When the fSharedFree member is set,
+ * the 30th bit decides whether it's a free page or not.
+ *
+ * Because of the different layout on 32-bit and 64-bit hosts, macros
+ * are used to get and set some of the data.
+ */
+typedef union GMMPAGE
+{
+#if HC_ARCH_BITS == 64
+    /** Unsigned integer view. */
+    uint64_t u;
+
+    /** The common view. */
+    struct GMMPAGECOMMON
+    {
+        uint32_t    uStuff1 : 32;
+        uint32_t    uStuff2 : 30;
+        /** The page state. */
+        uint32_t    u2State : 2;
+    } Common;
+
+    /** The view of a private page. */
+    struct GMMPAGEPRIVATE
+    {
+        /** The guest page frame number. (Max addressable: 2 ^ 44 - 16) */
+        uint32_t    pfn;
+        /** The GVM handle. (64K VMs) */
+        uint32_t    hGVM : 16;
+        /** Reserved. */
+        uint32_t    u16Reserved : 14;
+        /** The page state. */
+        uint32_t    u2State : 2;
+    } Private;
+
+    /** The view of a shared page. */
+    struct GMMPAGESHARED
+    {
+        /** The host page frame number. (Max addressable: 2 ^ 44 - 16) */
+        uint32_t    pfn;
+        /** The reference count (64K VMs). */
+        uint32_t    cRefs : 16;
+        /** Used for debug checksumming. */
+        uint32_t    u14Checksum : 14;
+        /** The page state. */
+        uint32_t    u2State : 2;
+    } Shared;
+
+    /** The view of a free page. */
+    struct GMMPAGEFREE
+    {
+        /** The index of the next page in the free list. UINT16_MAX is NIL. */
+        uint16_t    iNext;
+        /** Reserved. Checksum or something? */
+        uint16_t    u16Reserved0;
+        /** Reserved. Checksum or something? */
+        uint32_t    u30Reserved1 : 30;
+        /** The page state. */
+        uint32_t    u2State : 2;
+    } Free;
+
+#else /* 32-bit */
+    /** Unsigned integer view. */
+    uint32_t u;
+
+    /** The common view. */
+    struct GMMPAGECOMMON
+    {
+        uint32_t    uStuff : 30;
+        /** The page state. */
+        uint32_t    u2State : 2;
+    } Common;
+
+    /** The view of a private page. */
+    struct GMMPAGEPRIVATE
+    {
+        /** The guest page frame number. (Max addressable: 2 ^ 36) */
+        uint32_t    pfn : 24;
+        /** The GVM handle. (127 VMs) */
+        uint32_t    hGVM : 7;
+        /** The top page state bit, MBZ. */
+        uint32_t    fZero : 1;
+    } Private;
+
+    /** The view of a shared page. */
+    struct GMMPAGESHARED
+    {
+        /** The reference count. */
+        uint32_t    cRefs : 30;
+        /** The page state. */
+        uint32_t    u2State : 2;
+    } Shared;
+
+    /** The view of a free page. */
+    struct GMMPAGEFREE
+    {
+        /** The index of the next page in the free list. UINT16_MAX is NIL. */
+        uint32_t    iNext : 16;
+        /** Reserved. Checksum or something? */
+        uint32_t    u14Reserved : 14;
+        /** The page state. */
+        uint32_t    u2State : 2;
+    } Free;
+#endif
+} GMMPAGE;
+AssertCompileSize(GMMPAGE, sizeof(RTHCUINTPTR));
+/** Pointer to a GMMPAGE. */
+typedef GMMPAGE *PGMMPAGE;
+
+
+/** @name The Page States.
+ * @{ */
+/** A private page. */
+#define GMM_PAGE_STATE_PRIVATE          0
+/** A private page - alternative value used on the 32-bit implementation.
+ * This will never be used on 64-bit hosts. */
+#define GMM_PAGE_STATE_PRIVATE_32       1
+/** A shared page. */
+#define GMM_PAGE_STATE_SHARED           2
+/** A free page. */
+#define GMM_PAGE_STATE_FREE             3
+/** @} */
+
+
+/** @def GMM_PAGE_IS_PRIVATE
+ *
+ * @returns true if private, false if not.
+ * @param   pPage       The GMM page.
+ */
+#if HC_ARCH_BITS == 64
+# define GMM_PAGE_IS_PRIVATE(pPage) ( (pPage)->Common.u2State == GMM_PAGE_STATE_PRIVATE )
+#else
+# define GMM_PAGE_IS_PRIVATE(pPage) ( (pPage)->Private.fZero == 0 )
+#endif
+
+/** @def GMM_PAGE_IS_SHARED
+ *
+ * @returns true if shared, false if not.
+ * @param   pPage       The GMM page.
+ */
+#define GMM_PAGE_IS_SHARED(pPage)   ( (pPage)->Common.u2State == GMM_PAGE_STATE_SHARED )
+
+/** @def GMM_PAGE_IS_FREE
+ *
+ * @returns true if free, false if not.
+ * @param   pPage       The GMM page.
+ */
+#define GMM_PAGE_IS_FREE(pPage)     ( (pPage)->Common.u2State == GMM_PAGE_STATE_FREE )
+
+/** @def GMM_PAGE_PFN_LAST
+ * The last valid guest pfn range.
+ * @remark Some of the values outside the range has special meaning,
+ *         see GMM_PAGE_PFN_UNSHAREABLE.
+ */
+#if HC_ARCH_BITS == 64
+# define GMM_PAGE_PFN_LAST           UINT32_C(0xfffffff0)
+#else
+# define GMM_PAGE_PFN_LAST           UINT32_C(0x00fffff0)
+#endif
+AssertCompile(GMM_PAGE_PFN_LAST        == (GMM_GCPHYS_LAST >> PAGE_SHIFT));
+
+/** @def GMM_PAGE_PFN_UNSHAREABLE
+ * Indicates that this page isn't used for normal guest memory and thus isn't shareable.
+ */
+#if HC_ARCH_BITS == 64
+# define GMM_PAGE_PFN_UNSHAREABLE   UINT32_C(0xfffffff1)
+#else
+# define GMM_PAGE_PFN_UNSHAREABLE   UINT32_C(0x00fffff1)
+#endif
+AssertCompile(GMM_PAGE_PFN_UNSHAREABLE == (GMM_GCPHYS_UNSHAREABLE >> PAGE_SHIFT));
+
+
+/**
+ * A GMM allocation chunk ring-3 mapping record.
+ *
+ * This should really be associated with a session and not a VM, but
+ * it's simpler to associated with a VM and cleanup with the VM object
+ * is destroyed.
+ */
+typedef struct GMMCHUNKMAP
+{
+    /** The mapping object. */
+    RTR0MEMOBJ          hMapObj;
+    /** The VM owning the mapping. */
+    PGVM                pGVM;
+} GMMCHUNKMAP;
+/** Pointer to a GMM allocation chunk mapping. */
+typedef struct GMMCHUNKMAP *PGMMCHUNKMAP;
+
+
+/**
+ * A GMM allocation chunk.
+ */
+typedef struct GMMCHUNK
+{
+    /** The AVL node core.
+     * The Key is the chunk ID.  (Giant mtx.) */
+    AVLU32NODECORE      Core;
+    /** The memory object.
+     * Either from RTR0MemObjAllocPhysNC or RTR0MemObjLockUser depending on
+     * what the host can dish up with.  (Chunk mtx protects mapping accesses
+     * and related frees.) */
+    RTR0MEMOBJ          hMemObj;
+#if defined(VBOX_WITH_RAM_IN_KERNEL) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+    /** Pointer to the kernel mapping. */
+    uint8_t            *pbMapping;
+#endif
+    /** Pointer to the next chunk in the free list.  (Giant mtx.) */
+    PGMMCHUNK           pFreeNext;
+    /** Pointer to the previous chunk in the free list. (Giant mtx.) */
+    PGMMCHUNK           pFreePrev;
+    /** Pointer to the free set this chunk belongs to.  NULL for
+     * chunks with no free pages. (Giant mtx.) */
+    PGMMCHUNKFREESET    pSet;
+    /** List node in the chunk list (GMM::ChunkList).  (Giant mtx.) */
+    RTLISTNODE          ListNode;
+    /** Pointer to an array of mappings.  (Chunk mtx.) */
+    PGMMCHUNKMAP        paMappingsX;
+    /** The number of mappings.  (Chunk mtx.) */
+    uint16_t            cMappingsX;
+    /** The mapping lock this chunk is using using.  UINT16_MAX if nobody is
+     *  mapping or freeing anything.  (Giant mtx.) */
+    uint8_t volatile    iChunkMtx;
+    /** GMM_CHUNK_FLAGS_XXX. (Giant mtx.) */
+    uint8_t             fFlags;
+    /** The head of the list of free pages. UINT16_MAX is the NIL value.
+     *  (Giant mtx.) */
+    uint16_t            iFreeHead;
+    /** The number of free pages.  (Giant mtx.) */
+    uint16_t            cFree;
+    /** The GVM handle of the VM that first allocated pages from this chunk, this
+     * is used as a preference when there are several chunks to choose from.
+     * When in bound memory mode this isn't a preference any longer.  (Giant
+     * mtx.) */
+    uint16_t            hGVM;
+    /** The ID of the NUMA node the memory mostly resides on.  (Reserved for
+     *  future use.)  (Giant mtx.) */
+    uint16_t            idNumaNode;
+    /** The number of private pages.  (Giant mtx.) */
+    uint16_t            cPrivate;
+    /** The number of shared pages.  (Giant mtx.) */
+    uint16_t            cShared;
+    /** The pages.  (Giant mtx.) */
+    GMMPAGE             aPages[GMM_CHUNK_SIZE >> PAGE_SHIFT];
+} GMMCHUNK;
+
+/** Indicates that the NUMA properies of the memory is unknown. */
+#define GMM_CHUNK_NUMA_ID_UNKNOWN   UINT16_C(0xfffe)
+
+/** @name GMM_CHUNK_FLAGS_XXX - chunk flags.
+ * @{ */
+/** Indicates that the chunk is a large page (2MB). */
+#define GMM_CHUNK_FLAGS_LARGE_PAGE  UINT16_C(0x0001)
+#ifdef GMM_WITH_LEGACY_MODE
+/** Indicates that the chunk was locked rather than allocated directly. */
+# define GMM_CHUNK_FLAGS_SEEDED     UINT16_C(0x0002)
+#endif
+/** @}  */
+
+
+/**
+ * An allocation chunk TLB entry.
+ */
+typedef struct GMMCHUNKTLBE
+{
+    /** The chunk id. */
+    uint32_t            idChunk;
+    /** Pointer to the chunk. */
+    PGMMCHUNK           pChunk;
+} GMMCHUNKTLBE;
+/** Pointer to an allocation chunk TLB entry. */
+typedef GMMCHUNKTLBE *PGMMCHUNKTLBE;
+
+
+/** The number of entries in the allocation chunk TLB. */
+#define GMM_CHUNKTLB_ENTRIES        32
+/** Gets the TLB entry index for the given Chunk ID. */
+#define GMM_CHUNKTLB_IDX(idChunk)   ( (idChunk) & (GMM_CHUNKTLB_ENTRIES - 1) )
+
+/**
+ * An allocation chunk TLB.
+ */
+typedef struct GMMCHUNKTLB
+{
+    /** The TLB entries. */
+    GMMCHUNKTLBE    aEntries[GMM_CHUNKTLB_ENTRIES];
+} GMMCHUNKTLB;
+/** Pointer to an allocation chunk TLB. */
+typedef GMMCHUNKTLB *PGMMCHUNKTLB;
+
+
+/**
+ * The GMM instance data.
+ */
+typedef struct GMM
+{
+    /** Magic / eye catcher. GMM_MAGIC */
+    uint32_t            u32Magic;
+    /** The number of threads waiting on the mutex. */
+    uint32_t            cMtxContenders;
+#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
+    /** The critical section protecting the GMM.
+     * More fine grained locking can be implemented later if necessary. */
+    RTCRITSECT          GiantCritSect;
+#else
+    /** The fast mutex protecting the GMM.
+     * More fine grained locking can be implemented later if necessary. */
+    RTSEMFASTMUTEX      hMtx;
+#endif
+#ifdef VBOX_STRICT
+    /** The current mutex owner. */
+    RTNATIVETHREAD      hMtxOwner;
+#endif
+    /** Spinlock protecting the AVL tree.
+     * @todo Make this a read-write spinlock as we should allow concurrent
+     *       lookups. */
+    RTSPINLOCK          hSpinLockTree;
+    /** The chunk tree.
+     * Protected by hSpinLockTree. */
+    PAVLU32NODECORE     pChunks;
+    /** Chunk freeing generation - incremented whenever a chunk is freed.  Used
+     * for validating the per-VM chunk TLB entries.  Valid range is 1 to 2^62
+     * (exclusive), though higher numbers may temporarily occure while
+     * invalidating the individual TLBs during wrap-around processing. */
+    uint64_t volatile   idFreeGeneration;
+    /** The chunk TLB.
+     * Protected by hSpinLockTree. */
+    GMMCHUNKTLB         ChunkTLB;
+    /** The private free set. */
+    GMMCHUNKFREESET     PrivateX;
+    /** The shared free set. */
+    GMMCHUNKFREESET     Shared;
+
+    /** Shared module tree (global).
+     * @todo separate trees for distinctly different guest OSes. */
+    PAVLLU32NODECORE    pGlobalSharedModuleTree;
+    /** Sharable modules (count of nodes in pGlobalSharedModuleTree). */
+    uint32_t            cShareableModules;
+
+    /** The chunk list.  For simplifying the cleanup process and avoid tree
+     * traversal. */
+    RTLISTANCHOR        ChunkList;
+
+    /** The maximum number of pages we're allowed to allocate.
+     * @gcfgm{GMM/MaxPages,64-bit, Direct.}
+     * @gcfgm{GMM/PctPages,32-bit, Relative to the number of host pages.} */
+    uint64_t            cMaxPages;
+    /** The number of pages that has been reserved.
+     * The deal is that cReservedPages - cOverCommittedPages <= cMaxPages. */
+    uint64_t            cReservedPages;
+    /** The number of pages that we have over-committed in reservations. */
+    uint64_t            cOverCommittedPages;
+    /** The number of actually allocated (committed if you like) pages. */
+    uint64_t            cAllocatedPages;
+    /** The number of pages that are shared. A subset of cAllocatedPages. */
+    uint64_t            cSharedPages;
+    /** The number of pages that are actually shared between VMs. */
+    uint64_t            cDuplicatePages;
+    /** The number of pages that are shared that has been left behind by
+     * VMs not doing proper cleanups. */
+    uint64_t            cLeftBehindSharedPages;
+    /** The number of allocation chunks.
+     * (The number of pages we've allocated from the host can be derived from this.) */
+    uint32_t            cChunks;
+    /** The number of current ballooned pages. */
+    uint64_t            cBalloonedPages;
+
+#ifndef GMM_WITH_LEGACY_MODE
+# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
+    /** Whether #RTR0MemObjAllocPhysNC works.   */
+    bool                fHasWorkingAllocPhysNC;
+# else
+    bool                fPadding;
+# endif
+#else
+    /** The legacy allocation mode indicator.
+     * This is determined at initialization time. */
+    bool                fLegacyAllocationMode;
+#endif
+    /** The bound memory mode indicator.
+     * When set, the memory will be bound to a specific VM and never
+     * shared. This is always set if fLegacyAllocationMode is set.
+     * (Also determined at initialization time.) */
+    bool                fBoundMemoryMode;
+    /** The number of registered VMs. */
+    uint16_t            cRegisteredVMs;
+
+    /** The number of freed chunks ever.  This is used a list generation to
+     *  avoid restarting the cleanup scanning when the list wasn't modified. */
+    uint32_t            cFreedChunks;
+    /** The previous allocated Chunk ID.
+     * Used as a hint to avoid scanning the whole bitmap. */
+    uint32_t            idChunkPrev;
+    /** Chunk ID allocation bitmap.
+     * Bits of allocated IDs are set, free ones are clear.
+     * The NIL id (0) is marked allocated. */
+    uint32_t            bmChunkId[(GMM_CHUNKID_LAST + 1 + 31) / 32];
+
+    /** The index of the next mutex to use. */
+    uint32_t            iNextChunkMtx;
+    /** Chunk locks for reducing lock contention without having to allocate
+     * one lock per chunk. */
+    struct
+    {
+        /** The mutex */
+        RTSEMFASTMUTEX      hMtx;
+        /** The number of threads currently using this mutex. */
+        uint32_t volatile   cUsers;
+    } aChunkMtx[64];
+} GMM;
+/** Pointer to the GMM instance. */
+typedef GMM *PGMM;
+
+/** The value of GMM::u32Magic (Katsuhiro Otomo). */
+#define GMM_MAGIC       UINT32_C(0x19540414)
+
+
+/**
+ * GMM chunk mutex state.
+ *
+ * This is returned by gmmR0ChunkMutexAcquire and is used by the other
+ * gmmR0ChunkMutex* methods.
+ */
+typedef struct GMMR0CHUNKMTXSTATE
+{
+    PGMM                pGMM;
+    /** The index of the chunk mutex. */
+    uint8_t             iChunkMtx;
+    /** The relevant flags (GMMR0CHUNK_MTX_XXX). */
+    uint8_t             fFlags;
+} GMMR0CHUNKMTXSTATE;
+/** Pointer to a chunk mutex state. */
+typedef GMMR0CHUNKMTXSTATE *PGMMR0CHUNKMTXSTATE;
+
+/** @name GMMR0CHUNK_MTX_XXX
+ * @{ */
+#define GMMR0CHUNK_MTX_INVALID          UINT32_C(0)
+#define GMMR0CHUNK_MTX_KEEP_GIANT       UINT32_C(1)
+#define GMMR0CHUNK_MTX_RETAKE_GIANT     UINT32_C(2)
+#define GMMR0CHUNK_MTX_DROP_GIANT       UINT32_C(3)
+#define GMMR0CHUNK_MTX_END              UINT32_C(4)
+/** @} */
+
+
+/** The maximum number of shared modules per-vm. */
+#define GMM_MAX_SHARED_PER_VM_MODULES   2048
+/** The maximum number of shared modules GMM is allowed to track. */
+#define GMM_MAX_SHARED_GLOBAL_MODULES   16834
+
+
+/**
+ * Argument packet for gmmR0SharedModuleCleanup.
+ */
+typedef struct GMMR0SHMODPERVMDTORARGS
+{
+    PGVM    pGVM;
+    PGMM    pGMM;
+} GMMR0SHMODPERVMDTORARGS;
+
+/**
+ * Argument packet for gmmR0CheckSharedModule.
+ */
+typedef struct GMMCHECKSHAREDMODULEINFO
+{
+    PGVM                    pGVM;
+    VMCPUID                 idCpu;
+} GMMCHECKSHAREDMODULEINFO;
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** Pointer to the GMM instance data. */
+static PGMM g_pGMM = NULL;
+
+/** Macro for obtaining and validating the g_pGMM pointer.
+ *
+ * On failure it will return from the invoking function with the specified
+ * return value.
+ *
+ * @param   pGMM    The name of the pGMM variable.
+ * @param   rc      The return value on failure. Use VERR_GMM_INSTANCE for VBox
+ *                  status codes.
+ */
+#define GMM_GET_VALID_INSTANCE(pGMM, rc) \
+    do { \
+        (pGMM) = g_pGMM; \
+        AssertPtrReturn((pGMM), (rc)); \
+        AssertMsgReturn((pGMM)->u32Magic == GMM_MAGIC, ("%p - %#x\n", (pGMM), (pGMM)->u32Magic), (rc)); \
+    } while (0)
+
+/** Macro for obtaining and validating the g_pGMM pointer, void function
+ * variant.
+ *
+ * On failure it will return from the invoking function.
+ *
+ * @param   pGMM    The name of the pGMM variable.
+ */
+#define GMM_GET_VALID_INSTANCE_VOID(pGMM) \
+    do { \
+        (pGMM) = g_pGMM; \
+        AssertPtrReturnVoid((pGMM)); \
+        AssertMsgReturnVoid((pGMM)->u32Magic == GMM_MAGIC, ("%p - %#x\n", (pGMM), (pGMM)->u32Magic)); \
+    } while (0)
+
+
+/** @def GMM_CHECK_SANITY_UPON_ENTERING
+ * Checks the sanity of the GMM instance data before making changes.
+ *
+ * This is macro is a stub by default and must be enabled manually in the code.
+ *
+ * @returns true if sane, false if not.
+ * @param   pGMM    The name of the pGMM variable.
+ */
+#if defined(VBOX_STRICT) && defined(GMMR0_WITH_SANITY_CHECK) && 0
+# define GMM_CHECK_SANITY_UPON_ENTERING(pGMM)   (gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0)
+#else
+# define GMM_CHECK_SANITY_UPON_ENTERING(pGMM)   (true)
+#endif
+
+/** @def GMM_CHECK_SANITY_UPON_LEAVING
+ * Checks the sanity of the GMM instance data after making changes.
+ *
+ * This is macro is a stub by default and must be enabled manually in the code.
+ *
+ * @returns true if sane, false if not.
+ * @param   pGMM    The name of the pGMM variable.
+ */
+#if defined(VBOX_STRICT) && defined(GMMR0_WITH_SANITY_CHECK) && 0
+# define GMM_CHECK_SANITY_UPON_LEAVING(pGMM)    (gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0)
+#else
+# define GMM_CHECK_SANITY_UPON_LEAVING(pGMM)    (true)
+#endif
+
+/** @def GMM_CHECK_SANITY_IN_LOOPS
+ * Checks the sanity of the GMM instance in the allocation loops.
+ *
+ * This is macro is a stub by default and must be enabled manually in the code.
+ *
+ * @returns true if sane, false if not.
+ * @param   pGMM    The name of the pGMM variable.
+ */
+#if defined(VBOX_STRICT) && defined(GMMR0_WITH_SANITY_CHECK) && 0
+# define GMM_CHECK_SANITY_IN_LOOPS(pGMM)        (gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0)
+#else
+# define GMM_CHECK_SANITY_IN_LOOPS(pGMM)        (true)
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(int)    gmmR0TermDestroyChunk(PAVLU32NODECORE pNode, void *pvGMM);
+static bool                 gmmR0CleanupVMScanChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk);
+DECLINLINE(void)            gmmR0UnlinkChunk(PGMMCHUNK pChunk);
+DECLINLINE(void)            gmmR0LinkChunk(PGMMCHUNK pChunk, PGMMCHUNKFREESET pSet);
+DECLINLINE(void)            gmmR0SelectSetAndLinkChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk);
+#ifdef GMMR0_WITH_SANITY_CHECK
+static uint32_t             gmmR0SanityCheck(PGMM pGMM, const char *pszFunction, unsigned uLineNo);
+#endif
+static bool                 gmmR0FreeChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem);
+DECLINLINE(void)            gmmR0FreePrivatePage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage);
+DECLINLINE(void)            gmmR0FreeSharedPage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage);
+static int                  gmmR0UnmapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk);
+#ifdef VBOX_WITH_PAGE_SHARING
+static void                 gmmR0SharedModuleCleanup(PGMM pGMM, PGVM pGVM);
+# ifdef VBOX_STRICT
+static uint32_t             gmmR0StrictPageChecksum(PGMM pGMM, PGVM pGVM, uint32_t idPage);
+# endif
+#endif
+
+
+
+/**
+ * Initializes the GMM component.
+ *
+ * This is called when the VMMR0.r0 module is loaded and protected by the
+ * loader semaphore.
+ *
+ * @returns VBox status code.
+ */
+GMMR0DECL(int) GMMR0Init(void)
+{
+    LogFlow(("GMMInit:\n"));
+
+    /*
+     * Allocate the instance data and the locks.
+     */
+    PGMM pGMM = (PGMM)RTMemAllocZ(sizeof(*pGMM));
+    if (!pGMM)
+        return VERR_NO_MEMORY;
+
+    pGMM->u32Magic = GMM_MAGIC;
+    for (unsigned i = 0; i < RT_ELEMENTS(pGMM->ChunkTLB.aEntries); i++)
+        pGMM->ChunkTLB.aEntries[i].idChunk = NIL_GMM_CHUNKID;
+    RTListInit(&pGMM->ChunkList);
+    ASMBitSet(&pGMM->bmChunkId[0], NIL_GMM_CHUNKID);
+
+#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
+    int rc = RTCritSectInit(&pGMM->GiantCritSect);
+#else
+    int rc = RTSemFastMutexCreate(&pGMM->hMtx);
+#endif
+    if (RT_SUCCESS(rc))
+    {
+        unsigned iMtx;
+        for (iMtx = 0; iMtx < RT_ELEMENTS(pGMM->aChunkMtx); iMtx++)
+        {
+            rc = RTSemFastMutexCreate(&pGMM->aChunkMtx[iMtx].hMtx);
+            if (RT_FAILURE(rc))
+                break;
+        }
+        pGMM->hSpinLockTree = NIL_RTSPINLOCK;
+        if (RT_SUCCESS(rc))
+            rc = RTSpinlockCreate(&pGMM->hSpinLockTree, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "gmm-chunk-tree");
+        if (RT_SUCCESS(rc))
+        {
+#ifndef GMM_WITH_LEGACY_MODE
+            /*
+             * Figure out how we're going to allocate stuff (only applicable to
+             * host with linear physical memory mappings).
+             */
+            pGMM->fBoundMemoryMode = false;
+# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
+            pGMM->fHasWorkingAllocPhysNC = false;
+
+            RTR0MEMOBJ hMemObj;
+            rc = RTR0MemObjAllocPhysNC(&hMemObj, GMM_CHUNK_SIZE, NIL_RTHCPHYS);
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTR0MemObjFree(hMemObj, true);
+                AssertRC(rc);
+                pGMM->fHasWorkingAllocPhysNC = true;
+            }
+            else if (rc != VERR_NOT_SUPPORTED)
+                SUPR0Printf("GMMR0Init: Warning! RTR0MemObjAllocPhysNC(, %u, NIL_RTHCPHYS) -> %d!\n", GMM_CHUNK_SIZE, rc);
+# endif
+#else /* GMM_WITH_LEGACY_MODE */
+            /*
+             * Check and see if RTR0MemObjAllocPhysNC works.
+             */
+# if 0 /* later, see @bufref{3170}. */
+            RTR0MEMOBJ MemObj;
+            rc = RTR0MemObjAllocPhysNC(&MemObj, _64K, NIL_RTHCPHYS);
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTR0MemObjFree(MemObj, true);
+                AssertRC(rc);
+            }
+            else if (rc == VERR_NOT_SUPPORTED)
+                pGMM->fLegacyAllocationMode = pGMM->fBoundMemoryMode = true;
+            else
+                SUPR0Printf("GMMR0Init: RTR0MemObjAllocPhysNC(,64K,Any) -> %d!\n", rc);
+# else
+#  if defined(RT_OS_WINDOWS) || (defined(RT_OS_SOLARIS) && ARCH_BITS == 64) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD)
+            pGMM->fLegacyAllocationMode = false;
+#   if ARCH_BITS == 32
+            /* Don't reuse possibly partial chunks because of the virtual
+               address space limitation. */
+            pGMM->fBoundMemoryMode      = true;
+#   else
+            pGMM->fBoundMemoryMode      = false;
+#   endif
+#  else
+            pGMM->fLegacyAllocationMode = true;
+            pGMM->fBoundMemoryMode      = true;
+#  endif
+# endif
+#endif /* GMM_WITH_LEGACY_MODE */
+
+            /*
+             * Query system page count and guess a reasonable cMaxPages value.
+             */
+            pGMM->cMaxPages = UINT32_MAX; /** @todo IPRT function for query ram size and such. */
+
+            /*
+             * The idFreeGeneration value should be set so we actually trigger the
+             * wrap-around invalidation handling during a typical test run.
+             */
+            pGMM->idFreeGeneration = UINT64_MAX / 4 - 128;
+
+            g_pGMM = pGMM;
+#ifdef GMM_WITH_LEGACY_MODE
+            LogFlow(("GMMInit: pGMM=%p fLegacyAllocationMode=%RTbool fBoundMemoryMode=%RTbool\n", pGMM, pGMM->fLegacyAllocationMode, pGMM->fBoundMemoryMode));
+#elif defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+            LogFlow(("GMMInit: pGMM=%p fBoundMemoryMode=%RTbool fHasWorkingAllocPhysNC=%RTbool\n", pGMM, pGMM->fBoundMemoryMode, pGMM->fHasWorkingAllocPhysNC));
+#else
+            LogFlow(("GMMInit: pGMM=%p fBoundMemoryMode=%RTbool\n", pGMM, pGMM->fBoundMemoryMode));
+#endif
+            return VINF_SUCCESS;
+        }
+
+        /*
+         * Bail out.
+         */
+        RTSpinlockDestroy(pGMM->hSpinLockTree);
+        while (iMtx-- > 0)
+            RTSemFastMutexDestroy(pGMM->aChunkMtx[iMtx].hMtx);
+#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
+        RTCritSectDelete(&pGMM->GiantCritSect);
+#else
+        RTSemFastMutexDestroy(pGMM->hMtx);
+#endif
+    }
+
+    pGMM->u32Magic = 0;
+    RTMemFree(pGMM);
+    SUPR0Printf("GMMR0Init: failed! rc=%d\n", rc);
+    return rc;
+}
+
+
+/**
+ * Terminates the GMM component.
+ */
+GMMR0DECL(void) GMMR0Term(void)
+{
+    LogFlow(("GMMTerm:\n"));
+
+    /*
+     * Take care / be paranoid...
+     */
+    PGMM pGMM = g_pGMM;
+    if (!VALID_PTR(pGMM))
+        return;
+    if (pGMM->u32Magic != GMM_MAGIC)
+    {
+        SUPR0Printf("GMMR0Term: u32Magic=%#x\n", pGMM->u32Magic);
+        return;
+    }
+
+    /*
+     * Undo what init did and free all the resources we've acquired.
+     */
+    /* Destroy the fundamentals. */
+    g_pGMM = NULL;
+    pGMM->u32Magic    = ~GMM_MAGIC;
+#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
+    RTCritSectDelete(&pGMM->GiantCritSect);
+#else
+    RTSemFastMutexDestroy(pGMM->hMtx);
+    pGMM->hMtx        = NIL_RTSEMFASTMUTEX;
+#endif
+    RTSpinlockDestroy(pGMM->hSpinLockTree);
+    pGMM->hSpinLockTree = NIL_RTSPINLOCK;
+
+    /* Free any chunks still hanging around. */
+    RTAvlU32Destroy(&pGMM->pChunks, gmmR0TermDestroyChunk, pGMM);
+
+    /* Destroy the chunk locks. */
+    for (unsigned iMtx = 0; iMtx < RT_ELEMENTS(pGMM->aChunkMtx); iMtx++)
+    {
+        Assert(pGMM->aChunkMtx[iMtx].cUsers == 0);
+        RTSemFastMutexDestroy(pGMM->aChunkMtx[iMtx].hMtx);
+        pGMM->aChunkMtx[iMtx].hMtx = NIL_RTSEMFASTMUTEX;
+    }
+
+    /* Finally the instance data itself. */
+    RTMemFree(pGMM);
+    LogFlow(("GMMTerm: done\n"));
+}
+
+
+/**
+ * RTAvlU32Destroy callback.
+ *
+ * @returns 0
+ * @param   pNode   The node to destroy.
+ * @param   pvGMM   The GMM handle.
+ */
+static DECLCALLBACK(int) gmmR0TermDestroyChunk(PAVLU32NODECORE pNode, void *pvGMM)
+{
+    PGMMCHUNK pChunk = (PGMMCHUNK)pNode;
+
+    if (pChunk->cFree != (GMM_CHUNK_SIZE >> PAGE_SHIFT))
+        SUPR0Printf("GMMR0Term: %RKv/%#x: cFree=%d cPrivate=%d cShared=%d cMappings=%d\n", pChunk,
+                    pChunk->Core.Key, pChunk->cFree, pChunk->cPrivate, pChunk->cShared, pChunk->cMappingsX);
+
+    int rc = RTR0MemObjFree(pChunk->hMemObj, true /* fFreeMappings */);
+    if (RT_FAILURE(rc))
+    {
+        SUPR0Printf("GMMR0Term: %RKv/%#x: RTRMemObjFree(%RKv,true) -> %d (cMappings=%d)\n", pChunk,
+                    pChunk->Core.Key, pChunk->hMemObj, rc, pChunk->cMappingsX);
+        AssertRC(rc);
+    }
+    pChunk->hMemObj = NIL_RTR0MEMOBJ;
+
+    RTMemFree(pChunk->paMappingsX);
+    pChunk->paMappingsX = NULL;
+
+    RTMemFree(pChunk);
+    NOREF(pvGMM);
+    return 0;
+}
+
+
+/**
+ * Initializes the per-VM data for the GMM.
+ *
+ * This is called from within the GVMM lock (from GVMMR0CreateVM)
+ * and should only initialize the data members so GMMR0CleanupVM
+ * can deal with them. We reserve no memory or anything here,
+ * that's done later in GMMR0InitVM.
+ *
+ * @param   pGVM    Pointer to the Global VM structure.
+ */
+GMMR0DECL(int) GMMR0InitPerVMData(PGVM pGVM)
+{
+    AssertCompile(RT_SIZEOFMEMB(GVM,gmm.s) <= RT_SIZEOFMEMB(GVM,gmm.padding));
+
+    pGVM->gmm.s.Stats.enmPolicy = GMMOCPOLICY_INVALID;
+    pGVM->gmm.s.Stats.enmPriority = GMMPRIORITY_INVALID;
+    pGVM->gmm.s.Stats.fMayAllocate = false;
+
+    pGVM->gmm.s.hChunkTlbSpinLock = NIL_RTSPINLOCK;
+    int rc = RTSpinlockCreate(&pGVM->gmm.s.hChunkTlbSpinLock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "per-vm-chunk-tlb");
+    AssertRCReturn(rc, rc);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Acquires the GMM giant lock.
+ *
+ * @returns Assert status code from RTSemFastMutexRequest.
+ * @param   pGMM        Pointer to the GMM instance.
+ */
+static int gmmR0MutexAcquire(PGMM pGMM)
+{
+    ASMAtomicIncU32(&pGMM->cMtxContenders);
+#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
+    int rc = RTCritSectEnter(&pGMM->GiantCritSect);
+#else
+    int rc = RTSemFastMutexRequest(pGMM->hMtx);
+#endif
+    ASMAtomicDecU32(&pGMM->cMtxContenders);
+    AssertRC(rc);
+#ifdef VBOX_STRICT
+    pGMM->hMtxOwner = RTThreadNativeSelf();
+#endif
+    return rc;
+}
+
+
+/**
+ * Releases the GMM giant lock.
+ *
+ * @returns Assert status code from RTSemFastMutexRequest.
+ * @param   pGMM        Pointer to the GMM instance.
+ */
+static int gmmR0MutexRelease(PGMM pGMM)
+{
+#ifdef VBOX_STRICT
+    pGMM->hMtxOwner = NIL_RTNATIVETHREAD;
+#endif
+#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
+    int rc = RTCritSectLeave(&pGMM->GiantCritSect);
+#else
+    int rc = RTSemFastMutexRelease(pGMM->hMtx);
+    AssertRC(rc);
+#endif
+    return rc;
+}
+
+
+/**
+ * Yields the GMM giant lock if there is contention and a certain minimum time
+ * has elapsed since we took it.
+ *
+ * @returns @c true if the mutex was yielded, @c false if not.
+ * @param   pGMM            Pointer to the GMM instance.
+ * @param   puLockNanoTS    Where the lock acquisition time stamp is kept
+ *                          (in/out).
+ */
+static bool gmmR0MutexYield(PGMM pGMM, uint64_t *puLockNanoTS)
+{
+    /*
+     * If nobody is contending the mutex, don't bother checking the time.
+     */
+    if (ASMAtomicReadU32(&pGMM->cMtxContenders) == 0)
+        return false;
+
+    /*
+     * Don't yield if we haven't executed for at least 2 milliseconds.
+     */
+    uint64_t uNanoNow = RTTimeSystemNanoTS();
+    if (uNanoNow - *puLockNanoTS < UINT32_C(2000000))
+        return false;
+
+    /*
+     * Yield the mutex.
+     */
+#ifdef VBOX_STRICT
+    pGMM->hMtxOwner = NIL_RTNATIVETHREAD;
+#endif
+    ASMAtomicIncU32(&pGMM->cMtxContenders);
+#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
+    int rc1 = RTCritSectLeave(&pGMM->GiantCritSect); AssertRC(rc1);
+#else
+    int rc1 = RTSemFastMutexRelease(pGMM->hMtx); AssertRC(rc1);
+#endif
+
+    RTThreadYield();
+
+#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
+    int rc2 = RTCritSectEnter(&pGMM->GiantCritSect); AssertRC(rc2);
+#else
+    int rc2 = RTSemFastMutexRequest(pGMM->hMtx); AssertRC(rc2);
+#endif
+    *puLockNanoTS = RTTimeSystemNanoTS();
+    ASMAtomicDecU32(&pGMM->cMtxContenders);
+#ifdef VBOX_STRICT
+    pGMM->hMtxOwner = RTThreadNativeSelf();
+#endif
+
+    return true;
+}
+
+
+/**
+ * Acquires a chunk lock.
+ *
+ * The caller must own the giant lock.
+ *
+ * @returns Assert status code from RTSemFastMutexRequest.
+ * @param   pMtxState   The chunk mutex state info.  (Avoids
+ *                      passing the same flags and stuff around
+ *                      for subsequent release and drop-giant
+ *                      calls.)
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   pChunk      Pointer to the chunk.
+ * @param   fFlags      Flags regarding the giant lock, GMMR0CHUNK_MTX_XXX.
+ */
+static int gmmR0ChunkMutexAcquire(PGMMR0CHUNKMTXSTATE pMtxState, PGMM pGMM, PGMMCHUNK pChunk, uint32_t fFlags)
+{
+    Assert(fFlags > GMMR0CHUNK_MTX_INVALID && fFlags < GMMR0CHUNK_MTX_END);
+    Assert(pGMM->hMtxOwner == RTThreadNativeSelf());
+
+    pMtxState->pGMM   = pGMM;
+    pMtxState->fFlags = (uint8_t)fFlags;
+
+    /*
+     * Get the lock index and reference the lock.
+     */
+    Assert(pGMM->hMtxOwner == RTThreadNativeSelf());
+    uint32_t iChunkMtx = pChunk->iChunkMtx;
+    if (iChunkMtx == UINT8_MAX)
+    {
+        iChunkMtx = pGMM->iNextChunkMtx++;
+        iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
+
+        /* Try get an unused one... */
+        if (pGMM->aChunkMtx[iChunkMtx].cUsers)
+        {
+            iChunkMtx = pGMM->iNextChunkMtx++;
+            iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
+            if (pGMM->aChunkMtx[iChunkMtx].cUsers)
+            {
+                iChunkMtx = pGMM->iNextChunkMtx++;
+                iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
+                if (pGMM->aChunkMtx[iChunkMtx].cUsers)
+                {
+                    iChunkMtx = pGMM->iNextChunkMtx++;
+                    iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
+                }
+            }
+        }
+
+        pChunk->iChunkMtx = iChunkMtx;
+    }
+    AssertCompile(RT_ELEMENTS(pGMM->aChunkMtx) < UINT8_MAX);
+    pMtxState->iChunkMtx = (uint8_t)iChunkMtx;
+    ASMAtomicIncU32(&pGMM->aChunkMtx[iChunkMtx].cUsers);
+
+    /*
+     * Drop the giant?
+     */
+    if (fFlags != GMMR0CHUNK_MTX_KEEP_GIANT)
+    {
+        /** @todo GMM life cycle cleanup (we may race someone
+         *        destroying and cleaning up GMM)? */
+        gmmR0MutexRelease(pGMM);
+    }
+
+    /*
+     * Take the chunk mutex.
+     */
+    int rc = RTSemFastMutexRequest(pGMM->aChunkMtx[iChunkMtx].hMtx);
+    AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * Releases the GMM giant lock.
+ *
+ * @returns Assert status code from RTSemFastMutexRequest.
+ * @param   pMtxState   Pointer to the chunk mutex state.
+ * @param   pChunk      Pointer to the chunk if it's still
+ *                      alive, NULL if it isn't.  This is used to deassociate
+ *                      the chunk from the mutex on the way out so a new one
+ *                      can be selected next time, thus avoiding contented
+ *                      mutexes.
+ */
+static int gmmR0ChunkMutexRelease(PGMMR0CHUNKMTXSTATE pMtxState, PGMMCHUNK pChunk)
+{
+    PGMM pGMM = pMtxState->pGMM;
+
+    /*
+     * Release the chunk mutex and reacquire the giant if requested.
+     */
+    int rc = RTSemFastMutexRelease(pGMM->aChunkMtx[pMtxState->iChunkMtx].hMtx);
+    AssertRC(rc);
+    if (pMtxState->fFlags == GMMR0CHUNK_MTX_RETAKE_GIANT)
+        rc = gmmR0MutexAcquire(pGMM);
+    else
+        Assert((pMtxState->fFlags != GMMR0CHUNK_MTX_DROP_GIANT) == (pGMM->hMtxOwner == RTThreadNativeSelf()));
+
+    /*
+     * Drop the chunk mutex user reference and deassociate it from the chunk
+     * when possible.
+     */
+    if (   ASMAtomicDecU32(&pGMM->aChunkMtx[pMtxState->iChunkMtx].cUsers) == 0
+        && pChunk
+        && RT_SUCCESS(rc) )
+    {
+        if (pMtxState->fFlags != GMMR0CHUNK_MTX_DROP_GIANT)
+            pChunk->iChunkMtx = UINT8_MAX;
+        else
+        {
+            rc = gmmR0MutexAcquire(pGMM);
+            if (RT_SUCCESS(rc))
+            {
+                if (pGMM->aChunkMtx[pMtxState->iChunkMtx].cUsers == 0)
+                    pChunk->iChunkMtx = UINT8_MAX;
+                rc = gmmR0MutexRelease(pGMM);
+            }
+        }
+    }
+
+    pMtxState->pGMM = NULL;
+    return rc;
+}
+
+
+/**
+ * Drops the giant GMM lock we kept in gmmR0ChunkMutexAcquire while keeping the
+ * chunk locked.
+ *
+ * This only works if gmmR0ChunkMutexAcquire was called with
+ * GMMR0CHUNK_MTX_KEEP_GIANT.  gmmR0ChunkMutexRelease will retake the giant
+ * mutex, i.e. behave as if GMMR0CHUNK_MTX_RETAKE_GIANT was used.
+ *
+ * @returns VBox status code (assuming success is ok).
+ * @param   pMtxState   Pointer to the chunk mutex state.
+ */
+static int gmmR0ChunkMutexDropGiant(PGMMR0CHUNKMTXSTATE pMtxState)
+{
+    AssertReturn(pMtxState->fFlags == GMMR0CHUNK_MTX_KEEP_GIANT, VERR_GMM_MTX_FLAGS);
+    Assert(pMtxState->pGMM->hMtxOwner == RTThreadNativeSelf());
+    pMtxState->fFlags = GMMR0CHUNK_MTX_RETAKE_GIANT;
+    /** @todo GMM life cycle cleanup (we may race someone
+     *        destroying and cleaning up GMM)? */
+    return gmmR0MutexRelease(pMtxState->pGMM);
+}
+
+
+/**
+ * For experimenting with NUMA affinity and such.
+ *
+ * @returns The current NUMA Node ID.
+ */
+static uint16_t gmmR0GetCurrentNumaNodeId(void)
+{
+#if 1
+    return GMM_CHUNK_NUMA_ID_UNKNOWN;
+#else
+    return RTMpCpuId() / 16;
+#endif
+}
+
+
+
+/**
+ * Cleans up when a VM is terminating.
+ *
+ * @param   pGVM    Pointer to the Global VM structure.
+ */
+GMMR0DECL(void) GMMR0CleanupVM(PGVM pGVM)
+{
+    LogFlow(("GMMR0CleanupVM: pGVM=%p:{.hSelf=%#x}\n", pGVM, pGVM->hSelf));
+
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE_VOID(pGMM);
+
+#ifdef VBOX_WITH_PAGE_SHARING
+    /*
+     * Clean up all registered shared modules first.
+     */
+    gmmR0SharedModuleCleanup(pGMM, pGVM);
+#endif
+
+    gmmR0MutexAcquire(pGMM);
+    uint64_t uLockNanoTS = RTTimeSystemNanoTS();
+    GMM_CHECK_SANITY_UPON_ENTERING(pGMM);
+
+    /*
+     * The policy is 'INVALID' until the initial reservation
+     * request has been serviced.
+     */
+    if (    pGVM->gmm.s.Stats.enmPolicy > GMMOCPOLICY_INVALID
+        &&  pGVM->gmm.s.Stats.enmPolicy < GMMOCPOLICY_END)
+    {
+        /*
+         * If it's the last VM around, we can skip walking all the chunk looking
+         * for the pages owned by this VM and instead flush the whole shebang.
+         *
+         * This takes care of the eventuality that a VM has left shared page
+         * references behind (shouldn't happen of course, but you never know).
+         */
+        Assert(pGMM->cRegisteredVMs);
+        pGMM->cRegisteredVMs--;
+
+        /*
+         * Walk the entire pool looking for pages that belong to this VM
+         * and leftover mappings.  (This'll only catch private pages,
+         * shared pages will be 'left behind'.)
+         */
+        /** @todo r=bird: This scanning+freeing could be optimized in bound mode! */
+        uint64_t    cPrivatePages = pGVM->gmm.s.Stats.cPrivatePages; /* save */
+
+        unsigned    iCountDown = 64;
+        bool        fRedoFromStart;
+        PGMMCHUNK   pChunk;
+        do
+        {
+            fRedoFromStart = false;
+            RTListForEachReverse(&pGMM->ChunkList, pChunk, GMMCHUNK, ListNode)
+            {
+                uint32_t const cFreeChunksOld = pGMM->cFreedChunks;
+                if (   (   !pGMM->fBoundMemoryMode
+                        || pChunk->hGVM == pGVM->hSelf)
+                    && gmmR0CleanupVMScanChunk(pGMM, pGVM, pChunk))
+                {
+                    /* We left the giant mutex, so reset the yield counters. */
+                    uLockNanoTS = RTTimeSystemNanoTS();
+                    iCountDown  = 64;
+                }
+                else
+                {
+                    /* Didn't leave it, so do normal yielding. */
+                    if (!iCountDown)
+                        gmmR0MutexYield(pGMM, &uLockNanoTS);
+                    else
+                        iCountDown--;
+                }
+                if (pGMM->cFreedChunks != cFreeChunksOld)
+                {
+                    fRedoFromStart = true;
+                    break;
+                }
+            }
+        } while (fRedoFromStart);
+
+        if (pGVM->gmm.s.Stats.cPrivatePages)
+            SUPR0Printf("GMMR0CleanupVM: hGVM=%#x has %#x private pages that cannot be found!\n", pGVM->hSelf, pGVM->gmm.s.Stats.cPrivatePages);
+
+        pGMM->cAllocatedPages -= cPrivatePages;
+
+        /*
+         * Free empty chunks.
+         */
+        PGMMCHUNKFREESET pPrivateSet = pGMM->fBoundMemoryMode ? &pGVM->gmm.s.Private : &pGMM->PrivateX;
+        do
+        {
+            fRedoFromStart = false;
+            iCountDown = 10240;
+            pChunk = pPrivateSet->apLists[GMM_CHUNK_FREE_SET_UNUSED_LIST];
+            while (pChunk)
+            {
+                PGMMCHUNK pNext = pChunk->pFreeNext;
+                Assert(pChunk->cFree == GMM_CHUNK_NUM_PAGES);
+                if (   !pGMM->fBoundMemoryMode
+                    || pChunk->hGVM == pGVM->hSelf)
+                {
+                    uint64_t const idGenerationOld = pPrivateSet->idGeneration;
+                    if (gmmR0FreeChunk(pGMM, pGVM, pChunk, true /*fRelaxedSem*/))
+                    {
+                        /* We've left the giant mutex, restart? (+1 for our unlink) */
+                        fRedoFromStart = pPrivateSet->idGeneration != idGenerationOld + 1;
+                        if (fRedoFromStart)
+                            break;
+                        uLockNanoTS = RTTimeSystemNanoTS();
+                        iCountDown = 10240;
+                    }
+                }
+
+                /* Advance and maybe yield the lock. */
+                pChunk = pNext;
+                if (--iCountDown == 0)
+                {
+                    uint64_t const idGenerationOld = pPrivateSet->idGeneration;
+                    fRedoFromStart = gmmR0MutexYield(pGMM, &uLockNanoTS)
+                                  && pPrivateSet->idGeneration != idGenerationOld;
+                    if (fRedoFromStart)
+                        break;
+                    iCountDown = 10240;
+                }
+            }
+        } while (fRedoFromStart);
+
+        /*
+         * Account for shared pages that weren't freed.
+         */
+        if (pGVM->gmm.s.Stats.cSharedPages)
+        {
+            Assert(pGMM->cSharedPages >= pGVM->gmm.s.Stats.cSharedPages);
+            SUPR0Printf("GMMR0CleanupVM: hGVM=%#x left %#x shared pages behind!\n", pGVM->hSelf, pGVM->gmm.s.Stats.cSharedPages);
+            pGMM->cLeftBehindSharedPages += pGVM->gmm.s.Stats.cSharedPages;
+        }
+
+        /*
+         * Clean up balloon statistics in case the VM process crashed.
+         */
+        Assert(pGMM->cBalloonedPages >= pGVM->gmm.s.Stats.cBalloonedPages);
+        pGMM->cBalloonedPages -= pGVM->gmm.s.Stats.cBalloonedPages;
+
+        /*
+         * Update the over-commitment management statistics.
+         */
+        pGMM->cReservedPages -= pGVM->gmm.s.Stats.Reserved.cBasePages
+                              + pGVM->gmm.s.Stats.Reserved.cFixedPages
+                              + pGVM->gmm.s.Stats.Reserved.cShadowPages;
+        switch (pGVM->gmm.s.Stats.enmPolicy)
+        {
+            case GMMOCPOLICY_NO_OC:
+                break;
+            default:
+                /** @todo Update GMM->cOverCommittedPages */
+                break;
+        }
+    }
+
+    /* zap the GVM data. */
+    pGVM->gmm.s.Stats.enmPolicy    = GMMOCPOLICY_INVALID;
+    pGVM->gmm.s.Stats.enmPriority  = GMMPRIORITY_INVALID;
+    pGVM->gmm.s.Stats.fMayAllocate = false;
+
+    GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+    gmmR0MutexRelease(pGMM);
+
+    /*
+     * Destroy the spinlock.
+     */
+    RTSPINLOCK hSpinlock = NIL_RTSPINLOCK;
+    ASMAtomicXchgHandle(&pGVM->gmm.s.hChunkTlbSpinLock, NIL_RTSPINLOCK, &hSpinlock);
+    RTSpinlockDestroy(hSpinlock);
+
+    LogFlow(("GMMR0CleanupVM: returns\n"));
+}
+
+
+/**
+ * Scan one chunk for private pages belonging to the specified VM.
+ *
+ * @note    This function may drop the giant mutex!
+ *
+ * @returns @c true if we've temporarily dropped the giant mutex, @c false if
+ *          we didn't.
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   pGVM        The global VM handle.
+ * @param   pChunk      The chunk to scan.
+ */
+static bool gmmR0CleanupVMScanChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk)
+{
+    Assert(!pGMM->fBoundMemoryMode || pChunk->hGVM == pGVM->hSelf);
+
+    /*
+     * Look for pages belonging to the VM.
+     * (Perform some internal checks while we're scanning.)
+     */
+#ifndef VBOX_STRICT
+    if (pChunk->cFree != (GMM_CHUNK_SIZE >> PAGE_SHIFT))
+#endif
+    {
+        unsigned cPrivate = 0;
+        unsigned cShared = 0;
+        unsigned cFree = 0;
+
+        gmmR0UnlinkChunk(pChunk);       /* avoiding cFreePages updates. */
+
+        uint16_t hGVM = pGVM->hSelf;
+        unsigned iPage = (GMM_CHUNK_SIZE >> PAGE_SHIFT);
+        while (iPage-- > 0)
+            if (GMM_PAGE_IS_PRIVATE(&pChunk->aPages[iPage]))
+            {
+                if (pChunk->aPages[iPage].Private.hGVM == hGVM)
+                {
+                    /*
+                     * Free the page.
+                     *
+                     * The reason for not using gmmR0FreePrivatePage here is that we
+                     * must *not* cause the chunk to be freed from under us - we're in
+                     * an AVL tree walk here.
+                     */
+                    pChunk->aPages[iPage].u = 0;
+                    pChunk->aPages[iPage].Free.iNext = pChunk->iFreeHead;
+                    pChunk->aPages[iPage].Free.u2State = GMM_PAGE_STATE_FREE;
+                    pChunk->iFreeHead = iPage;
+                    pChunk->cPrivate--;
+                    pChunk->cFree++;
+                    pGVM->gmm.s.Stats.cPrivatePages--;
+                    cFree++;
+                }
+                else
+                    cPrivate++;
+            }
+            else if (GMM_PAGE_IS_FREE(&pChunk->aPages[iPage]))
+                cFree++;
+            else
+                cShared++;
+
+        gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk);
+
+        /*
+         * Did it add up?
+         */
+        if (RT_UNLIKELY(    pChunk->cFree != cFree
+                        ||  pChunk->cPrivate != cPrivate
+                        ||  pChunk->cShared != cShared))
+        {
+            SUPR0Printf("gmmR0CleanupVMScanChunk: Chunk %RKv/%#x has bogus stats - free=%d/%d private=%d/%d shared=%d/%d\n",
+                        pChunk, pChunk->Core.Key, pChunk->cFree, cFree, pChunk->cPrivate, cPrivate, pChunk->cShared, cShared);
+            pChunk->cFree = cFree;
+            pChunk->cPrivate = cPrivate;
+            pChunk->cShared = cShared;
+        }
+    }
+
+    /*
+     * If not in bound memory mode, we should reset the hGVM field
+     * if it has our handle in it.
+     */
+    if (pChunk->hGVM == pGVM->hSelf)
+    {
+        if (!g_pGMM->fBoundMemoryMode)
+            pChunk->hGVM = NIL_GVM_HANDLE;
+        else if (pChunk->cFree != GMM_CHUNK_NUM_PAGES)
+        {
+            SUPR0Printf("gmmR0CleanupVMScanChunk: %RKv/%#x: cFree=%#x - it should be 0 in bound mode!\n",
+                        pChunk, pChunk->Core.Key, pChunk->cFree);
+            AssertMsgFailed(("%p/%#x: cFree=%#x - it should be 0 in bound mode!\n", pChunk, pChunk->Core.Key, pChunk->cFree));
+
+            gmmR0UnlinkChunk(pChunk);
+            pChunk->cFree = GMM_CHUNK_NUM_PAGES;
+            gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk);
+        }
+    }
+
+    /*
+     * Look for a mapping belonging to the terminating VM.
+     */
+    GMMR0CHUNKMTXSTATE MtxState;
+    gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT);
+    unsigned cMappings = pChunk->cMappingsX;
+    for (unsigned i = 0; i < cMappings; i++)
+        if (pChunk->paMappingsX[i].pGVM == pGVM)
+        {
+            gmmR0ChunkMutexDropGiant(&MtxState);
+
+            RTR0MEMOBJ hMemObj = pChunk->paMappingsX[i].hMapObj;
+
+            cMappings--;
+            if (i < cMappings)
+                 pChunk->paMappingsX[i] = pChunk->paMappingsX[cMappings];
+            pChunk->paMappingsX[cMappings].pGVM    = NULL;
+            pChunk->paMappingsX[cMappings].hMapObj = NIL_RTR0MEMOBJ;
+            Assert(pChunk->cMappingsX - 1U == cMappings);
+            pChunk->cMappingsX = cMappings;
+
+            int rc = RTR0MemObjFree(hMemObj, false /* fFreeMappings (NA) */);
+            if (RT_FAILURE(rc))
+            {
+                SUPR0Printf("gmmR0CleanupVMScanChunk: %RKv/%#x: mapping #%x: RTRMemObjFree(%RKv,false) -> %d \n",
+                            pChunk, pChunk->Core.Key, i, hMemObj, rc);
+                AssertRC(rc);
+            }
+
+            gmmR0ChunkMutexRelease(&MtxState, pChunk);
+            return true;
+        }
+
+    gmmR0ChunkMutexRelease(&MtxState, pChunk);
+    return false;
+}
+
+
+/**
+ * The initial resource reservations.
+ *
+ * This will make memory reservations according to policy and priority. If there aren't
+ * sufficient resources available to sustain the VM this function will fail and all
+ * future allocations requests will fail as well.
+ *
+ * These are just the initial reservations made very very early during the VM creation
+ * process and will be adjusted later in the GMMR0UpdateReservation call after the
+ * ring-3 init has completed.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_GMM_MEMORY_RESERVATION_DECLINED
+ * @retval  VERR_GMM_
+ *
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   idCpu           The VCPU id - must be zero.
+ * @param   cBasePages      The number of pages that may be allocated for the base RAM and ROMs.
+ *                          This does not include MMIO2 and similar.
+ * @param   cShadowPages    The number of pages that may be allocated for shadow paging structures.
+ * @param   cFixedPages     The number of pages that may be allocated for fixed objects like the
+ *                          hyper heap, MMIO2 and similar.
+ * @param   enmPolicy       The OC policy to use on this VM.
+ * @param   enmPriority     The priority in an out-of-memory situation.
+ *
+ * @thread  The creator thread / EMT(0).
+ */
+GMMR0DECL(int) GMMR0InitialReservation(PGVM pGVM, VMCPUID idCpu, uint64_t cBasePages, uint32_t cShadowPages,
+                                       uint32_t cFixedPages, GMMOCPOLICY enmPolicy, GMMPRIORITY enmPriority)
+{
+    LogFlow(("GMMR0InitialReservation: pGVM=%p cBasePages=%#llx cShadowPages=%#x cFixedPages=%#x enmPolicy=%d enmPriority=%d\n",
+             pGVM, cBasePages, cShadowPages, cFixedPages, enmPolicy, enmPriority));
+
+    /*
+     * Validate, get basics and take the semaphore.
+     */
+    AssertReturn(idCpu == 0, VERR_INVALID_CPU_ID);
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    AssertReturn(cBasePages, VERR_INVALID_PARAMETER);
+    AssertReturn(cShadowPages, VERR_INVALID_PARAMETER);
+    AssertReturn(cFixedPages, VERR_INVALID_PARAMETER);
+    AssertReturn(enmPolicy > GMMOCPOLICY_INVALID && enmPolicy < GMMOCPOLICY_END, VERR_INVALID_PARAMETER);
+    AssertReturn(enmPriority > GMMPRIORITY_INVALID && enmPriority < GMMPRIORITY_END, VERR_INVALID_PARAMETER);
+
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        if (    !pGVM->gmm.s.Stats.Reserved.cBasePages
+            &&  !pGVM->gmm.s.Stats.Reserved.cFixedPages
+            &&  !pGVM->gmm.s.Stats.Reserved.cShadowPages)
+        {
+            /*
+             * Check if we can accommodate this.
+             */
+            /* ... later ... */
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Update the records.
+                 */
+                pGVM->gmm.s.Stats.Reserved.cBasePages   = cBasePages;
+                pGVM->gmm.s.Stats.Reserved.cFixedPages  = cFixedPages;
+                pGVM->gmm.s.Stats.Reserved.cShadowPages = cShadowPages;
+                pGVM->gmm.s.Stats.enmPolicy             = enmPolicy;
+                pGVM->gmm.s.Stats.enmPriority           = enmPriority;
+                pGVM->gmm.s.Stats.fMayAllocate          = true;
+
+                pGMM->cReservedPages += cBasePages + cFixedPages + cShadowPages;
+                pGMM->cRegisteredVMs++;
+            }
+        }
+        else
+            rc = VERR_WRONG_ORDER;
+        GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+    gmmR0MutexRelease(pGMM);
+    LogFlow(("GMMR0InitialReservation: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VMMR0 request wrapper for GMMR0InitialReservation.
+ *
+ * @returns see GMMR0InitialReservation.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   idCpu           The VCPU id.
+ * @param   pReq            Pointer to the request packet.
+ */
+GMMR0DECL(int) GMMR0InitialReservationReq(PGVM pGVM, VMCPUID idCpu, PGMMINITIALRESERVATIONREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
+
+    return GMMR0InitialReservation(pGVM, idCpu, pReq->cBasePages, pReq->cShadowPages,
+                                   pReq->cFixedPages, pReq->enmPolicy, pReq->enmPriority);
+}
+
+
+/**
+ * This updates the memory reservation with the additional MMIO2 and ROM pages.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_GMM_MEMORY_RESERVATION_DECLINED
+ *
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   idCpu           The VCPU id.
+ * @param   cBasePages      The number of pages that may be allocated for the base RAM and ROMs.
+ *                          This does not include MMIO2 and similar.
+ * @param   cShadowPages    The number of pages that may be allocated for shadow paging structures.
+ * @param   cFixedPages     The number of pages that may be allocated for fixed objects like the
+ *                          hyper heap, MMIO2 and similar.
+ *
+ * @thread  EMT(idCpu)
+ */
+GMMR0DECL(int) GMMR0UpdateReservation(PGVM pGVM, VMCPUID idCpu, uint64_t cBasePages,
+                                      uint32_t cShadowPages, uint32_t cFixedPages)
+{
+    LogFlow(("GMMR0UpdateReservation: pGVM=%p cBasePages=%#llx cShadowPages=%#x cFixedPages=%#x\n",
+             pGVM, cBasePages, cShadowPages, cFixedPages));
+
+    /*
+     * Validate, get basics and take the semaphore.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    AssertReturn(cBasePages, VERR_INVALID_PARAMETER);
+    AssertReturn(cShadowPages, VERR_INVALID_PARAMETER);
+    AssertReturn(cFixedPages, VERR_INVALID_PARAMETER);
+
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        if (    pGVM->gmm.s.Stats.Reserved.cBasePages
+            &&  pGVM->gmm.s.Stats.Reserved.cFixedPages
+            &&  pGVM->gmm.s.Stats.Reserved.cShadowPages)
+        {
+            /*
+             * Check if we can accommodate this.
+             */
+            /* ... later ... */
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Update the records.
+                 */
+                pGMM->cReservedPages -= pGVM->gmm.s.Stats.Reserved.cBasePages
+                                      + pGVM->gmm.s.Stats.Reserved.cFixedPages
+                                      + pGVM->gmm.s.Stats.Reserved.cShadowPages;
+                pGMM->cReservedPages += cBasePages + cFixedPages + cShadowPages;
+
+                pGVM->gmm.s.Stats.Reserved.cBasePages   = cBasePages;
+                pGVM->gmm.s.Stats.Reserved.cFixedPages  = cFixedPages;
+                pGVM->gmm.s.Stats.Reserved.cShadowPages = cShadowPages;
+            }
+        }
+        else
+            rc = VERR_WRONG_ORDER;
+        GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+    gmmR0MutexRelease(pGMM);
+    LogFlow(("GMMR0UpdateReservation: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VMMR0 request wrapper for GMMR0UpdateReservation.
+ *
+ * @returns see GMMR0UpdateReservation.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   idCpu           The VCPU id.
+ * @param   pReq            Pointer to the request packet.
+ */
+GMMR0DECL(int) GMMR0UpdateReservationReq(PGVM pGVM, VMCPUID idCpu, PGMMUPDATERESERVATIONREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
+
+    return GMMR0UpdateReservation(pGVM, idCpu, pReq->cBasePages, pReq->cShadowPages, pReq->cFixedPages);
+}
+
+#ifdef GMMR0_WITH_SANITY_CHECK
+
+/**
+ * Performs sanity checks on a free set.
+ *
+ * @returns Error count.
+ *
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   pSet        Pointer to the set.
+ * @param   pszSetName  The set name.
+ * @param   pszFunction The function from which it was called.
+ * @param   uLine       The line number.
+ */
+static uint32_t gmmR0SanityCheckSet(PGMM pGMM, PGMMCHUNKFREESET pSet, const char *pszSetName,
+                                    const char *pszFunction, unsigned uLineNo)
+{
+    uint32_t cErrors = 0;
+
+    /*
+     * Count the free pages in all the chunks and match it against pSet->cFreePages.
+     */
+    uint32_t cPages = 0;
+    for (unsigned i = 0; i < RT_ELEMENTS(pSet->apLists); i++)
+    {
+        for (PGMMCHUNK pCur = pSet->apLists[i]; pCur; pCur = pCur->pFreeNext)
+        {
+            /** @todo check that the chunk is hash into the right set. */
+            cPages += pCur->cFree;
+        }
+    }
+    if (RT_UNLIKELY(cPages != pSet->cFreePages))
+    {
+        SUPR0Printf("GMM insanity: found %#x pages in the %s set, expected %#x. (%s, line %u)\n",
+                    cPages, pszSetName, pSet->cFreePages, pszFunction, uLineNo);
+        cErrors++;
+    }
+
+    return cErrors;
+}
+
+
+/**
+ * Performs some sanity checks on the GMM while owning lock.
+ *
+ * @returns Error count.
+ *
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   pszFunction The function from which it is called.
+ * @param   uLineNo     The line number.
+ */
+static uint32_t gmmR0SanityCheck(PGMM pGMM, const char *pszFunction, unsigned uLineNo)
+{
+    uint32_t cErrors = 0;
+
+    cErrors += gmmR0SanityCheckSet(pGMM, &pGMM->PrivateX, "private", pszFunction, uLineNo);
+    cErrors += gmmR0SanityCheckSet(pGMM, &pGMM->Shared,   "shared",  pszFunction, uLineNo);
+    /** @todo add more sanity checks. */
+
+    return cErrors;
+}
+
+#endif /* GMMR0_WITH_SANITY_CHECK */
+
+/**
+ * Looks up a chunk in the tree and fill in the TLB entry for it.
+ *
+ * This is not expected to fail and will bitch if it does.
+ *
+ * @returns Pointer to the allocation chunk, NULL if not found.
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   idChunk     The ID of the chunk to find.
+ * @param   pTlbe       Pointer to the TLB entry.
+ *
+ * @note    Caller owns spinlock.
+ */
+static PGMMCHUNK gmmR0GetChunkSlow(PGMM pGMM, uint32_t idChunk, PGMMCHUNKTLBE pTlbe)
+{
+    PGMMCHUNK pChunk = (PGMMCHUNK)RTAvlU32Get(&pGMM->pChunks, idChunk);
+    AssertMsgReturn(pChunk, ("Chunk %#x not found!\n", idChunk), NULL);
+    pTlbe->idChunk = idChunk;
+    pTlbe->pChunk = pChunk;
+    return pChunk;
+}
+
+
+/**
+ * Finds a allocation chunk, spin-locked.
+ *
+ * This is not expected to fail and will bitch if it does.
+ *
+ * @returns Pointer to the allocation chunk, NULL if not found.
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   idChunk     The ID of the chunk to find.
+ */
+DECLINLINE(PGMMCHUNK) gmmR0GetChunkLocked(PGMM pGMM, uint32_t idChunk)
+{
+    /*
+     * Do a TLB lookup, branch if not in the TLB.
+     */
+    PGMMCHUNKTLBE pTlbe  = &pGMM->ChunkTLB.aEntries[GMM_CHUNKTLB_IDX(idChunk)];
+    PGMMCHUNK     pChunk = pTlbe->pChunk;
+    if (   pChunk == NULL
+        || pTlbe->idChunk != idChunk)
+        pChunk = gmmR0GetChunkSlow(pGMM, idChunk, pTlbe);
+    return pChunk;
+}
+
+
+/**
+ * Finds a allocation chunk.
+ *
+ * This is not expected to fail and will bitch if it does.
+ *
+ * @returns Pointer to the allocation chunk, NULL if not found.
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   idChunk     The ID of the chunk to find.
+ */
+DECLINLINE(PGMMCHUNK) gmmR0GetChunk(PGMM pGMM, uint32_t idChunk)
+{
+    RTSpinlockAcquire(pGMM->hSpinLockTree);
+    PGMMCHUNK pChunk = gmmR0GetChunkLocked(pGMM, idChunk);
+    RTSpinlockRelease(pGMM->hSpinLockTree);
+    return pChunk;
+}
+
+
+/**
+ * Finds a page.
+ *
+ * This is not expected to fail and will bitch if it does.
+ *
+ * @returns Pointer to the page, NULL if not found.
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   idPage      The ID of the page to find.
+ */
+DECLINLINE(PGMMPAGE) gmmR0GetPage(PGMM pGMM, uint32_t idPage)
+{
+    PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
+    if (RT_LIKELY(pChunk))
+        return &pChunk->aPages[idPage & GMM_PAGEID_IDX_MASK];
+    return NULL;
+}
+
+
+#if 0 /* unused */
+/**
+ * Gets the host physical address for a page given by it's ID.
+ *
+ * @returns The host physical address or NIL_RTHCPHYS.
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   idPage      The ID of the page to find.
+ */
+DECLINLINE(RTHCPHYS) gmmR0GetPageHCPhys(PGMM pGMM,  uint32_t idPage)
+{
+    PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
+    if (RT_LIKELY(pChunk))
+        return RTR0MemObjGetPagePhysAddr(pChunk->hMemObj, idPage & GMM_PAGEID_IDX_MASK);
+    return NIL_RTHCPHYS;
+}
+#endif /* unused */
+
+
+/**
+ * Selects the appropriate free list given the number of free pages.
+ *
+ * @returns Free list index.
+ * @param   cFree       The number of free pages in the chunk.
+ */
+DECLINLINE(unsigned) gmmR0SelectFreeSetList(unsigned cFree)
+{
+    unsigned iList = cFree >> GMM_CHUNK_FREE_SET_SHIFT;
+    AssertMsg(iList < RT_SIZEOFMEMB(GMMCHUNKFREESET, apLists) / RT_SIZEOFMEMB(GMMCHUNKFREESET, apLists[0]),
+              ("%d (%u)\n", iList, cFree));
+    return iList;
+}
+
+
+/**
+ * Unlinks the chunk from the free list it's currently on (if any).
+ *
+ * @param   pChunk      The allocation chunk.
+ */
+DECLINLINE(void) gmmR0UnlinkChunk(PGMMCHUNK pChunk)
+{
+    PGMMCHUNKFREESET pSet = pChunk->pSet;
+    if (RT_LIKELY(pSet))
+    {
+        pSet->cFreePages -= pChunk->cFree;
+        pSet->idGeneration++;
+
+        PGMMCHUNK pPrev = pChunk->pFreePrev;
+        PGMMCHUNK pNext = pChunk->pFreeNext;
+        if (pPrev)
+            pPrev->pFreeNext = pNext;
+        else
+            pSet->apLists[gmmR0SelectFreeSetList(pChunk->cFree)] = pNext;
+        if (pNext)
+            pNext->pFreePrev = pPrev;
+
+        pChunk->pSet = NULL;
+        pChunk->pFreeNext = NULL;
+        pChunk->pFreePrev = NULL;
+    }
+    else
+    {
+        Assert(!pChunk->pFreeNext);
+        Assert(!pChunk->pFreePrev);
+        Assert(!pChunk->cFree);
+    }
+}
+
+
+/**
+ * Links the chunk onto the appropriate free list in the specified free set.
+ *
+ * If no free entries, it's not linked into any list.
+ *
+ * @param   pChunk      The allocation chunk.
+ * @param   pSet        The free set.
+ */
+DECLINLINE(void) gmmR0LinkChunk(PGMMCHUNK pChunk, PGMMCHUNKFREESET pSet)
+{
+    Assert(!pChunk->pSet);
+    Assert(!pChunk->pFreeNext);
+    Assert(!pChunk->pFreePrev);
+
+    if (pChunk->cFree > 0)
+    {
+        pChunk->pSet = pSet;
+        pChunk->pFreePrev = NULL;
+        unsigned const iList = gmmR0SelectFreeSetList(pChunk->cFree);
+        pChunk->pFreeNext = pSet->apLists[iList];
+        if (pChunk->pFreeNext)
+            pChunk->pFreeNext->pFreePrev = pChunk;
+        pSet->apLists[iList] = pChunk;
+
+        pSet->cFreePages += pChunk->cFree;
+        pSet->idGeneration++;
+    }
+}
+
+
+/**
+ * Links the chunk onto the appropriate free list in the specified free set.
+ *
+ * If no free entries, it's not linked into any list.
+ *
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   pGVM        Pointer to the kernel-only VM instace data.
+ * @param   pChunk      The allocation chunk.
+ */
+DECLINLINE(void) gmmR0SelectSetAndLinkChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk)
+{
+    PGMMCHUNKFREESET pSet;
+    if (pGMM->fBoundMemoryMode)
+        pSet = &pGVM->gmm.s.Private;
+    else if (pChunk->cShared)
+        pSet = &pGMM->Shared;
+    else
+        pSet = &pGMM->PrivateX;
+    gmmR0LinkChunk(pChunk, pSet);
+}
+
+
+/**
+ * Frees a Chunk ID.
+ *
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   idChunk     The Chunk ID to free.
+ */
+static void gmmR0FreeChunkId(PGMM pGMM, uint32_t idChunk)
+{
+    AssertReturnVoid(idChunk != NIL_GMM_CHUNKID);
+    AssertMsg(ASMBitTest(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk));
+    ASMAtomicBitClear(&pGMM->bmChunkId[0], idChunk);
+}
+
+
+/**
+ * Allocates a new Chunk ID.
+ *
+ * @returns The Chunk ID.
+ * @param   pGMM        Pointer to the GMM instance.
+ */
+static uint32_t gmmR0AllocateChunkId(PGMM pGMM)
+{
+    AssertCompile(!((GMM_CHUNKID_LAST + 1) & 31)); /* must be a multiple of 32 */
+    AssertCompile(NIL_GMM_CHUNKID == 0);
+
+    /*
+     * Try the next sequential one.
+     */
+    int32_t idChunk = ++pGMM->idChunkPrev;
+#if 0 /** @todo enable this code */
+    if (    idChunk <= GMM_CHUNKID_LAST
+        &&  idChunk > NIL_GMM_CHUNKID
+        &&  !ASMAtomicBitTestAndSet(&pVMM->bmChunkId[0], idChunk))
+        return idChunk;
+#endif
+
+    /*
+     * Scan sequentially from the last one.
+     */
+    if (    (uint32_t)idChunk < GMM_CHUNKID_LAST
+        &&  idChunk > NIL_GMM_CHUNKID)
+    {
+        idChunk = ASMBitNextClear(&pGMM->bmChunkId[0], GMM_CHUNKID_LAST + 1, idChunk - 1);
+        if (idChunk > NIL_GMM_CHUNKID)
+        {
+            AssertMsgReturn(!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk), NIL_GMM_CHUNKID);
+            return pGMM->idChunkPrev = idChunk;
+        }
+    }
+
+    /*
+     * Ok, scan from the start.
+     * We're not racing anyone, so there is no need to expect failures or have restart loops.
+     */
+    idChunk = ASMBitFirstClear(&pGMM->bmChunkId[0], GMM_CHUNKID_LAST + 1);
+    AssertMsgReturn(idChunk > NIL_GMM_CHUNKID, ("%#x\n", idChunk), NIL_GVM_HANDLE);
+    AssertMsgReturn(!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk), NIL_GMM_CHUNKID);
+
+    return pGMM->idChunkPrev = idChunk;
+}
+
+
+/**
+ * Allocates one private page.
+ *
+ * Worker for gmmR0AllocatePages.
+ *
+ * @param   pChunk      The chunk to allocate it from.
+ * @param   hGVM        The GVM handle of the VM requesting memory.
+ * @param   pPageDesc   The page descriptor.
+ */
+static void gmmR0AllocatePage(PGMMCHUNK pChunk, uint32_t hGVM, PGMMPAGEDESC pPageDesc)
+{
+    /* update the chunk stats. */
+    if (pChunk->hGVM == NIL_GVM_HANDLE)
+        pChunk->hGVM = hGVM;
+    Assert(pChunk->cFree);
+    pChunk->cFree--;
+    pChunk->cPrivate++;
+
+    /* unlink the first free page. */
+    const uint32_t iPage = pChunk->iFreeHead;
+    AssertReleaseMsg(iPage < RT_ELEMENTS(pChunk->aPages), ("%d\n", iPage));
+    PGMMPAGE pPage = &pChunk->aPages[iPage];
+    Assert(GMM_PAGE_IS_FREE(pPage));
+    pChunk->iFreeHead = pPage->Free.iNext;
+    Log3(("A pPage=%p iPage=%#x/%#x u2State=%d iFreeHead=%#x iNext=%#x\n",
+          pPage, iPage, (pChunk->Core.Key << GMM_CHUNKID_SHIFT) | iPage,
+          pPage->Common.u2State, pChunk->iFreeHead, pPage->Free.iNext));
+
+    /* make the page private. */
+    pPage->u = 0;
+    AssertCompile(GMM_PAGE_STATE_PRIVATE == 0);
+    pPage->Private.hGVM = hGVM;
+    AssertCompile(NIL_RTHCPHYS >= GMM_GCPHYS_LAST);
+    AssertCompile(GMM_GCPHYS_UNSHAREABLE >= GMM_GCPHYS_LAST);
+    if (pPageDesc->HCPhysGCPhys <= GMM_GCPHYS_LAST)
+        pPage->Private.pfn = pPageDesc->HCPhysGCPhys >> PAGE_SHIFT;
+    else
+        pPage->Private.pfn = GMM_PAGE_PFN_UNSHAREABLE; /* unshareable / unassigned - same thing. */
+
+    /* update the page descriptor. */
+    pPageDesc->HCPhysGCPhys = RTR0MemObjGetPagePhysAddr(pChunk->hMemObj, iPage);
+    Assert(pPageDesc->HCPhysGCPhys != NIL_RTHCPHYS);
+    pPageDesc->idPage = (pChunk->Core.Key << GMM_CHUNKID_SHIFT) | iPage;
+    pPageDesc->idSharedPage = NIL_GMM_PAGEID;
+}
+
+
+/**
+ * Picks the free pages from a chunk.
+ *
+ * @returns The new page descriptor table index.
+ * @param   pChunk      The chunk.
+ * @param   hGVM        The affinity of the chunk. NIL_GVM_HANDLE for no
+ *                      affinity.
+ * @param   iPage       The current page descriptor table index.
+ * @param   cPages      The total number of pages to allocate.
+ * @param   paPages     The page descriptor table (input + ouput).
+ */
+static uint32_t gmmR0AllocatePagesFromChunk(PGMMCHUNK pChunk, uint16_t const hGVM, uint32_t iPage, uint32_t cPages,
+                                            PGMMPAGEDESC paPages)
+{
+    PGMMCHUNKFREESET pSet = pChunk->pSet; Assert(pSet);
+    gmmR0UnlinkChunk(pChunk);
+
+    for (; pChunk->cFree && iPage < cPages; iPage++)
+        gmmR0AllocatePage(pChunk, hGVM, &paPages[iPage]);
+
+    gmmR0LinkChunk(pChunk, pSet);
+    return iPage;
+}
+
+
+/**
+ * Registers a new chunk of memory.
+ *
+ * This is called by both gmmR0AllocateOneChunk and GMMR0SeedChunk.
+ *
+ * @returns VBox status code.  On success, the giant GMM lock will be held, the
+ *          caller must release it (ugly).
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   pSet        Pointer to the set.
+ * @param   hMemObj     The memory object for the chunk.
+ * @param   hGVM        The affinity of the chunk. NIL_GVM_HANDLE for no
+ *                      affinity.
+ * @param   fChunkFlags The chunk flags, GMM_CHUNK_FLAGS_XXX.
+ * @param   ppChunk     Chunk address (out).  Optional.
+ *
+ * @remarks The caller must not own the giant GMM mutex.
+ *          The giant GMM mutex will be acquired and returned acquired in
+ *          the success path.   On failure, no locks will be held.
+ */
+static int gmmR0RegisterChunk(PGMM pGMM, PGMMCHUNKFREESET pSet, RTR0MEMOBJ hMemObj, uint16_t hGVM, uint16_t fChunkFlags,
+                              PGMMCHUNK *ppChunk)
+{
+    Assert(pGMM->hMtxOwner != RTThreadNativeSelf());
+    Assert(hGVM != NIL_GVM_HANDLE || pGMM->fBoundMemoryMode);
+#ifdef GMM_WITH_LEGACY_MODE
+    Assert(fChunkFlags == 0 || fChunkFlags == GMM_CHUNK_FLAGS_LARGE_PAGE || fChunkFlags == GMM_CHUNK_FLAGS_SEEDED);
+#else
+    Assert(fChunkFlags == 0 || fChunkFlags == GMM_CHUNK_FLAGS_LARGE_PAGE);
+#endif
+
+#if defined(VBOX_WITH_RAM_IN_KERNEL) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+    /*
+     * Get a ring-0 mapping of the object.
+     */
+# ifdef GMM_WITH_LEGACY_MODE
+    uint8_t *pbMapping = !(fChunkFlags & GMM_CHUNK_FLAGS_SEEDED) ? (uint8_t *)RTR0MemObjAddress(hMemObj) : NULL;
+# else
+    uint8_t *pbMapping = (uint8_t *)RTR0MemObjAddress(hMemObj);
+# endif
+    if (!pbMapping)
+    {
+        RTR0MEMOBJ hMapObj;
+        int rc = RTR0MemObjMapKernel(&hMapObj, hMemObj, (void *)-1, 0,  RTMEM_PROT_READ | RTMEM_PROT_WRITE);
+        if (RT_SUCCESS(rc))
+            pbMapping = (uint8_t *)RTR0MemObjAddress(hMapObj);
+        else
+            return rc;
+        AssertPtr(pbMapping);
+    }
+#endif
+
+    /*
+     * Allocate a chunk.
+     */
+    int rc;
+    PGMMCHUNK pChunk = (PGMMCHUNK)RTMemAllocZ(sizeof(*pChunk));
+    if (pChunk)
+    {
+        /*
+         * Initialize it.
+         */
+        pChunk->hMemObj     = hMemObj;
+#if defined(VBOX_WITH_RAM_IN_KERNEL) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+        pChunk->pbMapping   = pbMapping;
+#endif
+        pChunk->cFree       = GMM_CHUNK_NUM_PAGES;
+        pChunk->hGVM        = hGVM;
+        /*pChunk->iFreeHead = 0;*/
+        pChunk->idNumaNode  = gmmR0GetCurrentNumaNodeId();
+        pChunk->iChunkMtx   = UINT8_MAX;
+        pChunk->fFlags      = fChunkFlags;
+        for (unsigned iPage = 0; iPage < RT_ELEMENTS(pChunk->aPages) - 1; iPage++)
+        {
+            pChunk->aPages[iPage].Free.u2State = GMM_PAGE_STATE_FREE;
+            pChunk->aPages[iPage].Free.iNext = iPage + 1;
+        }
+        pChunk->aPages[RT_ELEMENTS(pChunk->aPages) - 1].Free.u2State = GMM_PAGE_STATE_FREE;
+        pChunk->aPages[RT_ELEMENTS(pChunk->aPages) - 1].Free.iNext   = UINT16_MAX;
+
+        /*
+         * Allocate a Chunk ID and insert it into the tree.
+         * This has to be done behind the mutex of course.
+         */
+        rc = gmmR0MutexAcquire(pGMM);
+        if (RT_SUCCESS(rc))
+        {
+            if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+            {
+                pChunk->Core.Key = gmmR0AllocateChunkId(pGMM);
+                if (   pChunk->Core.Key != NIL_GMM_CHUNKID
+                    && pChunk->Core.Key <= GMM_CHUNKID_LAST)
+                {
+                    RTSpinlockAcquire(pGMM->hSpinLockTree);
+                    if (RTAvlU32Insert(&pGMM->pChunks, &pChunk->Core))
+                    {
+                        pGMM->cChunks++;
+                        RTListAppend(&pGMM->ChunkList, &pChunk->ListNode);
+                        RTSpinlockRelease(pGMM->hSpinLockTree);
+
+                        gmmR0LinkChunk(pChunk, pSet);
+
+                        LogFlow(("gmmR0RegisterChunk: pChunk=%p id=%#x cChunks=%d\n", pChunk, pChunk->Core.Key, pGMM->cChunks));
+
+                        if (ppChunk)
+                            *ppChunk = pChunk;
+                        GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+                        return VINF_SUCCESS;
+                    }
+                    RTSpinlockRelease(pGMM->hSpinLockTree);
+                }
+
+                /* bail out */
+                rc = VERR_GMM_CHUNK_INSERT;
+            }
+            else
+                rc = VERR_GMM_IS_NOT_SANE;
+            gmmR0MutexRelease(pGMM);
+        }
+
+        RTMemFree(pChunk);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+    return rc;
+}
+
+
+/**
+ * Allocate a new chunk, immediately pick the requested pages from it, and adds
+ * what's remaining to the specified free set.
+ *
+ * @note    This will leave the giant mutex while allocating the new chunk!
+ *
+ * @returns VBox status code.
+ * @param   pGMM        Pointer to the GMM instance data.
+ * @param   pGVM        Pointer to the kernel-only VM instace data.
+ * @param   pSet        Pointer to the free set.
+ * @param   cPages      The number of pages requested.
+ * @param   paPages     The page descriptor table (input + output).
+ * @param   piPage      The pointer to the page descriptor table index variable.
+ *                      This will be updated.
+ */
+static int gmmR0AllocateChunkNew(PGMM pGMM, PGVM pGVM, PGMMCHUNKFREESET pSet, uint32_t cPages,
+                                 PGMMPAGEDESC paPages, uint32_t *piPage)
+{
+    gmmR0MutexRelease(pGMM);
+
+    RTR0MEMOBJ hMemObj;
+#ifndef GMM_WITH_LEGACY_MODE
+    int rc;
+# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
+    if (pGMM->fHasWorkingAllocPhysNC)
+        rc = RTR0MemObjAllocPhysNC(&hMemObj, GMM_CHUNK_SIZE, NIL_RTHCPHYS);
+    else
+# endif
+        rc = RTR0MemObjAllocPage(&hMemObj, GMM_CHUNK_SIZE, false /*fExecutable*/);
+#else
+    int rc = RTR0MemObjAllocPhysNC(&hMemObj, GMM_CHUNK_SIZE, NIL_RTHCPHYS);
+#endif
+    if (RT_SUCCESS(rc))
+    {
+        /** @todo Duplicate gmmR0RegisterChunk here so we can avoid chaining up the
+         *        free pages first and then unchaining them right afterwards. Instead
+         *        do as much work as possible without holding the giant lock. */
+        PGMMCHUNK pChunk;
+        rc = gmmR0RegisterChunk(pGMM, pSet, hMemObj, pGVM->hSelf, 0 /*fChunkFlags*/, &pChunk);
+        if (RT_SUCCESS(rc))
+        {
+            *piPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, *piPage, cPages, paPages);
+            return VINF_SUCCESS;
+        }
+
+        /* bail out */
+        RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
+    }
+
+    int rc2 = gmmR0MutexAcquire(pGMM);
+    AssertRCReturn(rc2, RT_FAILURE(rc) ? rc : rc2);
+    return rc;
+
+}
+
+
+/**
+ * As a last restort we'll pick any page we can get.
+ *
+ * @returns The new page descriptor table index.
+ * @param   pSet        The set to pick from.
+ * @param   pGVM        Pointer to the global VM structure.
+ * @param   iPage       The current page descriptor table index.
+ * @param   cPages      The total number of pages to allocate.
+ * @param   paPages     The page descriptor table (input + ouput).
+ */
+static uint32_t gmmR0AllocatePagesIndiscriminately(PGMMCHUNKFREESET pSet, PGVM pGVM,
+                                                   uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
+{
+    unsigned iList = RT_ELEMENTS(pSet->apLists);
+    while (iList-- > 0)
+    {
+        PGMMCHUNK pChunk = pSet->apLists[iList];
+        while (pChunk)
+        {
+            PGMMCHUNK pNext = pChunk->pFreeNext;
+
+            iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages);
+            if (iPage >= cPages)
+                return iPage;
+
+            pChunk = pNext;
+        }
+    }
+    return iPage;
+}
+
+
+/**
+ * Pick pages from empty chunks on the same NUMA node.
+ *
+ * @returns The new page descriptor table index.
+ * @param   pSet        The set to pick from.
+ * @param   pGVM        Pointer to the global VM structure.
+ * @param   iPage       The current page descriptor table index.
+ * @param   cPages      The total number of pages to allocate.
+ * @param   paPages     The page descriptor table (input + ouput).
+ */
+static uint32_t gmmR0AllocatePagesFromEmptyChunksOnSameNode(PGMMCHUNKFREESET pSet, PGVM pGVM,
+                                                            uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
+{
+    PGMMCHUNK pChunk = pSet->apLists[GMM_CHUNK_FREE_SET_UNUSED_LIST];
+    if (pChunk)
+    {
+        uint16_t const idNumaNode = gmmR0GetCurrentNumaNodeId();
+        while (pChunk)
+        {
+            PGMMCHUNK pNext = pChunk->pFreeNext;
+
+            if (pChunk->idNumaNode == idNumaNode)
+            {
+                pChunk->hGVM = pGVM->hSelf;
+                iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages);
+                if (iPage >= cPages)
+                {
+                    pGVM->gmm.s.idLastChunkHint = pChunk->cFree ? pChunk->Core.Key : NIL_GMM_CHUNKID;
+                    return iPage;
+                }
+            }
+
+            pChunk = pNext;
+        }
+    }
+    return iPage;
+}
+
+
+/**
+ * Pick pages from non-empty chunks on the same NUMA node.
+ *
+ * @returns The new page descriptor table index.
+ * @param   pSet        The set to pick from.
+ * @param   pGVM        Pointer to the global VM structure.
+ * @param   iPage       The current page descriptor table index.
+ * @param   cPages      The total number of pages to allocate.
+ * @param   paPages     The page descriptor table (input + ouput).
+ */
+static uint32_t gmmR0AllocatePagesFromSameNode(PGMMCHUNKFREESET pSet, PGVM pGVM,
+                                               uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
+{
+    /** @todo start by picking from chunks with about the right size first?  */
+    uint16_t const  idNumaNode = gmmR0GetCurrentNumaNodeId();
+    unsigned        iList      = GMM_CHUNK_FREE_SET_UNUSED_LIST;
+    while (iList-- > 0)
+    {
+        PGMMCHUNK pChunk = pSet->apLists[iList];
+        while (pChunk)
+        {
+            PGMMCHUNK pNext = pChunk->pFreeNext;
+
+            if (pChunk->idNumaNode == idNumaNode)
+            {
+                iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages);
+                if (iPage >= cPages)
+                {
+                    pGVM->gmm.s.idLastChunkHint = pChunk->cFree ? pChunk->Core.Key : NIL_GMM_CHUNKID;
+                    return iPage;
+                }
+            }
+
+            pChunk = pNext;
+        }
+    }
+    return iPage;
+}
+
+
+/**
+ * Pick pages that are in chunks already associated with the VM.
+ *
+ * @returns The new page descriptor table index.
+ * @param   pGMM        Pointer to the GMM instance data.
+ * @param   pGVM        Pointer to the global VM structure.
+ * @param   pSet        The set to pick from.
+ * @param   iPage       The current page descriptor table index.
+ * @param   cPages      The total number of pages to allocate.
+ * @param   paPages     The page descriptor table (input + ouput).
+ */
+static uint32_t gmmR0AllocatePagesAssociatedWithVM(PGMM pGMM, PGVM pGVM, PGMMCHUNKFREESET pSet,
+                                                   uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
+{
+    uint16_t const hGVM = pGVM->hSelf;
+
+    /* Hint. */
+    if (pGVM->gmm.s.idLastChunkHint != NIL_GMM_CHUNKID)
+    {
+        PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pGVM->gmm.s.idLastChunkHint);
+        if (pChunk && pChunk->cFree)
+        {
+            iPage = gmmR0AllocatePagesFromChunk(pChunk, hGVM, iPage, cPages, paPages);
+            if (iPage >= cPages)
+                return iPage;
+        }
+    }
+
+    /* Scan. */
+    for (unsigned iList = 0; iList < RT_ELEMENTS(pSet->apLists); iList++)
+    {
+        PGMMCHUNK pChunk = pSet->apLists[iList];
+        while (pChunk)
+        {
+            PGMMCHUNK pNext = pChunk->pFreeNext;
+
+            if (pChunk->hGVM == hGVM)
+            {
+                iPage = gmmR0AllocatePagesFromChunk(pChunk, hGVM, iPage, cPages, paPages);
+                if (iPage >= cPages)
+                {
+                    pGVM->gmm.s.idLastChunkHint = pChunk->cFree ? pChunk->Core.Key : NIL_GMM_CHUNKID;
+                    return iPage;
+                }
+            }
+
+            pChunk = pNext;
+        }
+    }
+    return iPage;
+}
+
+
+
+/**
+ * Pick pages in bound memory mode.
+ *
+ * @returns The new page descriptor table index.
+ * @param   pGVM        Pointer to the global VM structure.
+ * @param   iPage       The current page descriptor table index.
+ * @param   cPages      The total number of pages to allocate.
+ * @param   paPages     The page descriptor table (input + ouput).
+ */
+static uint32_t gmmR0AllocatePagesInBoundMode(PGVM pGVM, uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
+{
+    for (unsigned iList = 0; iList < RT_ELEMENTS(pGVM->gmm.s.Private.apLists); iList++)
+    {
+        PGMMCHUNK pChunk = pGVM->gmm.s.Private.apLists[iList];
+        while (pChunk)
+        {
+            Assert(pChunk->hGVM == pGVM->hSelf);
+            PGMMCHUNK pNext = pChunk->pFreeNext;
+            iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages);
+            if (iPage >= cPages)
+                return iPage;
+            pChunk = pNext;
+        }
+    }
+    return iPage;
+}
+
+
+/**
+ * Checks if we should start picking pages from chunks of other VMs because
+ * we're getting close to the system memory or reserved limit.
+ *
+ * @returns @c true if we should, @c false if we should first try allocate more
+ *          chunks.
+ */
+static bool gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLimits(PGVM pGVM)
+{
+    /*
+     * Don't allocate a new chunk if we're
+     */
+    uint64_t cPgReserved  = pGVM->gmm.s.Stats.Reserved.cBasePages
+                          + pGVM->gmm.s.Stats.Reserved.cFixedPages
+                          - pGVM->gmm.s.Stats.cBalloonedPages
+                          /** @todo what about shared pages? */;
+    uint64_t cPgAllocated = pGVM->gmm.s.Stats.Allocated.cBasePages
+                          + pGVM->gmm.s.Stats.Allocated.cFixedPages;
+    uint64_t cPgDelta = cPgReserved - cPgAllocated;
+    if (cPgDelta < GMM_CHUNK_NUM_PAGES * 4)
+        return true;
+    /** @todo make the threshold configurable, also test the code to see if
+     *        this ever kicks in (we might be reserving too much or smth). */
+
+    /*
+     * Check how close we're to the max memory limit and how many fragments
+     * there are?...
+     */
+    /** @todo  */
+
+    return false;
+}
+
+
+/**
+ * Checks if we should start picking pages from chunks of other VMs because
+ * there is a lot of free pages around.
+ *
+ * @returns @c true if we should, @c false if we should first try allocate more
+ *          chunks.
+ */
+static bool gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLotsFree(PGMM pGMM)
+{
+    /*
+     * Setting the limit at 16 chunks (32 MB) at the moment.
+     */
+    if (pGMM->PrivateX.cFreePages >= GMM_CHUNK_NUM_PAGES * 16)
+        return true;
+    return false;
+}
+
+
+/**
+ * Common worker for GMMR0AllocateHandyPages and GMMR0AllocatePages.
+ *
+ * @returns VBox status code:
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_GMM_SEED_ME if seeding via GMMR0SeedChunk or
+ *          gmmR0AllocateMoreChunks is necessary.
+ * @retval  VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages.
+ * @retval  VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit,
+ *          that is we're trying to allocate more than we've reserved.
+ *
+ * @param   pGMM        Pointer to the GMM instance data.
+ * @param   pGVM        Pointer to the VM.
+ * @param   cPages      The number of pages to allocate.
+ * @param   paPages     Pointer to the page descriptors. See GMMPAGEDESC for
+ *                      details on what is expected on input.
+ * @param   enmAccount  The account to charge.
+ *
+ * @remarks Call takes the giant GMM lock.
+ */
+static int gmmR0AllocatePagesNew(PGMM pGMM, PGVM pGVM, uint32_t cPages, PGMMPAGEDESC paPages, GMMACCOUNT enmAccount)
+{
+    Assert(pGMM->hMtxOwner == RTThreadNativeSelf());
+
+    /*
+     * Check allocation limits.
+     */
+    if (RT_UNLIKELY(pGMM->cAllocatedPages + cPages > pGMM->cMaxPages))
+        return VERR_GMM_HIT_GLOBAL_LIMIT;
+
+    switch (enmAccount)
+    {
+        case GMMACCOUNT_BASE:
+            if (RT_UNLIKELY(  pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.cBalloonedPages + cPages
+                            > pGVM->gmm.s.Stats.Reserved.cBasePages))
+            {
+                Log(("gmmR0AllocatePages:Base: Reserved=%#llx Allocated+Ballooned+Requested=%#llx+%#llx+%#x!\n",
+                     pGVM->gmm.s.Stats.Reserved.cBasePages, pGVM->gmm.s.Stats.Allocated.cBasePages,
+                     pGVM->gmm.s.Stats.cBalloonedPages, cPages));
+                return VERR_GMM_HIT_VM_ACCOUNT_LIMIT;
+            }
+            break;
+        case GMMACCOUNT_SHADOW:
+            if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cShadowPages + cPages > pGVM->gmm.s.Stats.Reserved.cShadowPages))
+            {
+                Log(("gmmR0AllocatePages:Shadow: Reserved=%#x Allocated+Requested=%#x+%#x!\n",
+                     pGVM->gmm.s.Stats.Reserved.cShadowPages, pGVM->gmm.s.Stats.Allocated.cShadowPages, cPages));
+                return VERR_GMM_HIT_VM_ACCOUNT_LIMIT;
+            }
+            break;
+        case GMMACCOUNT_FIXED:
+            if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cFixedPages + cPages > pGVM->gmm.s.Stats.Reserved.cFixedPages))
+            {
+                Log(("gmmR0AllocatePages:Fixed: Reserved=%#x Allocated+Requested=%#x+%#x!\n",
+                     pGVM->gmm.s.Stats.Reserved.cFixedPages, pGVM->gmm.s.Stats.Allocated.cFixedPages, cPages));
+                return VERR_GMM_HIT_VM_ACCOUNT_LIMIT;
+            }
+            break;
+        default:
+            AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+
+#ifdef GMM_WITH_LEGACY_MODE
+    /*
+     * If we're in legacy memory mode, it's easy to figure if we have
+     * sufficient number of pages up-front.
+     */
+    if (   pGMM->fLegacyAllocationMode
+        && pGVM->gmm.s.Private.cFreePages < cPages)
+    {
+        Assert(pGMM->fBoundMemoryMode);
+        return VERR_GMM_SEED_ME;
+    }
+#endif
+
+    /*
+     * Update the accounts before we proceed because we might be leaving the
+     * protection of the global mutex and thus run the risk of permitting
+     * too much memory to be allocated.
+     */
+    switch (enmAccount)
+    {
+        case GMMACCOUNT_BASE:   pGVM->gmm.s.Stats.Allocated.cBasePages   += cPages; break;
+        case GMMACCOUNT_SHADOW: pGVM->gmm.s.Stats.Allocated.cShadowPages += cPages; break;
+        case GMMACCOUNT_FIXED:  pGVM->gmm.s.Stats.Allocated.cFixedPages  += cPages; break;
+        default:                AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+    pGVM->gmm.s.Stats.cPrivatePages += cPages;
+    pGMM->cAllocatedPages           += cPages;
+
+#ifdef GMM_WITH_LEGACY_MODE
+    /*
+     * Part two of it's-easy-in-legacy-memory-mode.
+     */
+    if (pGMM->fLegacyAllocationMode)
+    {
+        uint32_t iPage = gmmR0AllocatePagesInBoundMode(pGVM, 0, cPages, paPages);
+        AssertReleaseReturn(iPage == cPages, VERR_GMM_ALLOC_PAGES_IPE);
+        return VINF_SUCCESS;
+    }
+#endif
+
+    /*
+     * Bound mode is also relatively straightforward.
+     */
+    uint32_t iPage = 0;
+    int rc = VINF_SUCCESS;
+    if (pGMM->fBoundMemoryMode)
+    {
+        iPage = gmmR0AllocatePagesInBoundMode(pGVM, iPage, cPages, paPages);
+        if (iPage < cPages)
+            do
+                rc = gmmR0AllocateChunkNew(pGMM, pGVM, &pGVM->gmm.s.Private, cPages, paPages, &iPage);
+            while (iPage < cPages && RT_SUCCESS(rc));
+    }
+    /*
+     * Shared mode is trickier as we should try archive the same locality as
+     * in bound mode, but smartly make use of non-full chunks allocated by
+     * other VMs if we're low on memory.
+     */
+    else
+    {
+        /* Pick the most optimal pages first. */
+        iPage = gmmR0AllocatePagesAssociatedWithVM(pGMM, pGVM, &pGMM->PrivateX, iPage, cPages, paPages);
+        if (iPage < cPages)
+        {
+            /* Maybe we should try getting pages from chunks "belonging" to
+               other VMs before allocating more chunks? */
+            bool fTriedOnSameAlready = false;
+            if (gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLimits(pGVM))
+            {
+                iPage = gmmR0AllocatePagesFromSameNode(&pGMM->PrivateX, pGVM, iPage, cPages, paPages);
+                fTriedOnSameAlready = true;
+            }
+
+            /* Allocate memory from empty chunks. */
+            if (iPage < cPages)
+                iPage = gmmR0AllocatePagesFromEmptyChunksOnSameNode(&pGMM->PrivateX, pGVM, iPage, cPages, paPages);
+
+            /* Grab empty shared chunks. */
+            if (iPage < cPages)
+                iPage = gmmR0AllocatePagesFromEmptyChunksOnSameNode(&pGMM->Shared, pGVM, iPage, cPages, paPages);
+
+            /* If there is a lof of free pages spread around, try not waste
+               system memory on more chunks. (Should trigger defragmentation.) */
+            if (   !fTriedOnSameAlready
+                && gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLotsFree(pGMM))
+            {
+                iPage = gmmR0AllocatePagesFromSameNode(&pGMM->PrivateX, pGVM, iPage, cPages, paPages);
+                if (iPage < cPages)
+                    iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->PrivateX, pGVM, iPage, cPages, paPages);
+            }
+
+            /*
+             * Ok, try allocate new chunks.
+             */
+            if (iPage < cPages)
+            {
+                do
+                    rc = gmmR0AllocateChunkNew(pGMM, pGVM, &pGMM->PrivateX, cPages, paPages, &iPage);
+                while (iPage < cPages && RT_SUCCESS(rc));
+
+                /* If the host is out of memory, take whatever we can get. */
+                if (   (rc == VERR_NO_MEMORY || rc == VERR_NO_PHYS_MEMORY)
+                    && pGMM->PrivateX.cFreePages + pGMM->Shared.cFreePages >= cPages - iPage)
+                {
+                    iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->PrivateX, pGVM, iPage, cPages, paPages);
+                    if (iPage < cPages)
+                        iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->Shared, pGVM, iPage, cPages, paPages);
+                    AssertRelease(iPage == cPages);
+                    rc = VINF_SUCCESS;
+                }
+            }
+        }
+    }
+
+    /*
+     * Clean up on failure.  Since this is bound to be a low-memory condition
+     * we will give back any empty chunks that might be hanging around.
+     */
+    if (RT_FAILURE(rc))
+    {
+        /* Update the statistics. */
+        pGVM->gmm.s.Stats.cPrivatePages -= cPages;
+        pGMM->cAllocatedPages           -= cPages - iPage;
+        switch (enmAccount)
+        {
+            case GMMACCOUNT_BASE:   pGVM->gmm.s.Stats.Allocated.cBasePages   -= cPages; break;
+            case GMMACCOUNT_SHADOW: pGVM->gmm.s.Stats.Allocated.cShadowPages -= cPages; break;
+            case GMMACCOUNT_FIXED:  pGVM->gmm.s.Stats.Allocated.cFixedPages  -= cPages; break;
+            default:                AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+        }
+
+        /* Release the pages. */
+        while (iPage-- > 0)
+        {
+            uint32_t idPage = paPages[iPage].idPage;
+            PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage);
+            if (RT_LIKELY(pPage))
+            {
+                Assert(GMM_PAGE_IS_PRIVATE(pPage));
+                Assert(pPage->Private.hGVM == pGVM->hSelf);
+                gmmR0FreePrivatePage(pGMM, pGVM, idPage, pPage);
+            }
+            else
+                AssertMsgFailed(("idPage=%#x\n", idPage));
+
+            paPages[iPage].idPage       = NIL_GMM_PAGEID;
+            paPages[iPage].idSharedPage = NIL_GMM_PAGEID;
+            paPages[iPage].HCPhysGCPhys = NIL_RTHCPHYS;
+        }
+
+        /* Free empty chunks. */
+        /** @todo  */
+
+        /* return the fail status on failure */
+        return rc;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Updates the previous allocations and allocates more pages.
+ *
+ * The handy pages are always taken from the 'base' memory account.
+ * The allocated pages are not cleared and will contains random garbage.
+ *
+ * @returns VBox status code:
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_NOT_OWNER if the caller is not an EMT.
+ * @retval  VERR_GMM_PAGE_NOT_FOUND if one of the pages to update wasn't found.
+ * @retval  VERR_GMM_PAGE_NOT_PRIVATE if one of the pages to update wasn't a
+ *          private page.
+ * @retval  VERR_GMM_PAGE_NOT_SHARED if one of the pages to update wasn't a
+ *          shared page.
+ * @retval  VERR_GMM_NOT_PAGE_OWNER if one of the pages to be updated wasn't
+ *          owned by the VM.
+ * @retval  VERR_GMM_SEED_ME if seeding via GMMR0SeedChunk is necessary.
+ * @retval  VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages.
+ * @retval  VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit,
+ *          that is we're trying to allocate more than we've reserved.
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   idCpu               The VCPU id.
+ * @param   cPagesToUpdate      The number of pages to update (starting from the head).
+ * @param   cPagesToAlloc       The number of pages to allocate (starting from the head).
+ * @param   paPages             The array of page descriptors.
+ *                              See GMMPAGEDESC for details on what is expected on input.
+ * @thread  EMT(idCpu)
+ */
+GMMR0DECL(int) GMMR0AllocateHandyPages(PGVM pGVM, VMCPUID idCpu, uint32_t cPagesToUpdate,
+                                       uint32_t cPagesToAlloc, PGMMPAGEDESC paPages)
+{
+    LogFlow(("GMMR0AllocateHandyPages: pGVM=%p cPagesToUpdate=%#x cPagesToAlloc=%#x paPages=%p\n",
+             pGVM, cPagesToUpdate, cPagesToAlloc, paPages));
+
+    /*
+     * Validate, get basics and take the semaphore.
+     * (This is a relatively busy path, so make predictions where possible.)
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    AssertPtrReturn(paPages, VERR_INVALID_PARAMETER);
+    AssertMsgReturn(    (cPagesToUpdate && cPagesToUpdate < 1024)
+                    ||  (cPagesToAlloc  && cPagesToAlloc < 1024),
+                    ("cPagesToUpdate=%#x cPagesToAlloc=%#x\n", cPagesToUpdate, cPagesToAlloc),
+                    VERR_INVALID_PARAMETER);
+
+    unsigned iPage = 0;
+    for (; iPage < cPagesToUpdate; iPage++)
+    {
+        AssertMsgReturn(    (    paPages[iPage].HCPhysGCPhys <= GMM_GCPHYS_LAST
+                             && !(paPages[iPage].HCPhysGCPhys & PAGE_OFFSET_MASK))
+                        ||  paPages[iPage].HCPhysGCPhys == NIL_RTHCPHYS
+                        ||  paPages[iPage].HCPhysGCPhys == GMM_GCPHYS_UNSHAREABLE,
+                        ("#%#x: %RHp\n", iPage, paPages[iPage].HCPhysGCPhys),
+                        VERR_INVALID_PARAMETER);
+        AssertMsgReturn(    paPages[iPage].idPage <= GMM_PAGEID_LAST
+                        /*||  paPages[iPage].idPage == NIL_GMM_PAGEID*/,
+                        ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER);
+        AssertMsgReturn(    paPages[iPage].idPage <= GMM_PAGEID_LAST
+                        /*||  paPages[iPage].idSharedPage == NIL_GMM_PAGEID*/,
+                        ("#%#x: %#x\n", iPage, paPages[iPage].idSharedPage), VERR_INVALID_PARAMETER);
+    }
+
+    for (; iPage < cPagesToAlloc; iPage++)
+    {
+        AssertMsgReturn(paPages[iPage].HCPhysGCPhys == NIL_RTHCPHYS,   ("#%#x: %RHp\n", iPage, paPages[iPage].HCPhysGCPhys), VERR_INVALID_PARAMETER);
+        AssertMsgReturn(paPages[iPage].idPage       == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idPage),        VERR_INVALID_PARAMETER);
+        AssertMsgReturn(paPages[iPage].idSharedPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idSharedPage),  VERR_INVALID_PARAMETER);
+    }
+
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        /* No allocations before the initial reservation has been made! */
+        if (RT_LIKELY(    pGVM->gmm.s.Stats.Reserved.cBasePages
+                      &&  pGVM->gmm.s.Stats.Reserved.cFixedPages
+                      &&  pGVM->gmm.s.Stats.Reserved.cShadowPages))
+        {
+            /*
+             * Perform the updates.
+             * Stop on the first error.
+             */
+            for (iPage = 0; iPage < cPagesToUpdate; iPage++)
+            {
+                if (paPages[iPage].idPage != NIL_GMM_PAGEID)
+                {
+                    PGMMPAGE pPage = gmmR0GetPage(pGMM, paPages[iPage].idPage);
+                    if (RT_LIKELY(pPage))
+                    {
+                        if (RT_LIKELY(GMM_PAGE_IS_PRIVATE(pPage)))
+                        {
+                            if (RT_LIKELY(pPage->Private.hGVM == pGVM->hSelf))
+                            {
+                                AssertCompile(NIL_RTHCPHYS > GMM_GCPHYS_LAST && GMM_GCPHYS_UNSHAREABLE > GMM_GCPHYS_LAST);
+                                if (RT_LIKELY(paPages[iPage].HCPhysGCPhys <= GMM_GCPHYS_LAST))
+                                    pPage->Private.pfn = paPages[iPage].HCPhysGCPhys >> PAGE_SHIFT;
+                                else if (paPages[iPage].HCPhysGCPhys == GMM_GCPHYS_UNSHAREABLE)
+                                    pPage->Private.pfn = GMM_PAGE_PFN_UNSHAREABLE;
+                                /* else: NIL_RTHCPHYS nothing */
+
+                                paPages[iPage].idPage       = NIL_GMM_PAGEID;
+                                paPages[iPage].HCPhysGCPhys = NIL_RTHCPHYS;
+                            }
+                            else
+                            {
+                                Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not owner! hGVM=%#x hSelf=%#x\n",
+                                     iPage, paPages[iPage].idPage, pPage->Private.hGVM, pGVM->hSelf));
+                                rc = VERR_GMM_NOT_PAGE_OWNER;
+                                break;
+                            }
+                        }
+                        else
+                        {
+                            Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not private! %.*Rhxs (type %d)\n", iPage, paPages[iPage].idPage, sizeof(*pPage), pPage, pPage->Common.u2State));
+                            rc = VERR_GMM_PAGE_NOT_PRIVATE;
+                            break;
+                        }
+                    }
+                    else
+                    {
+                        Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not found! (private)\n", iPage, paPages[iPage].idPage));
+                        rc = VERR_GMM_PAGE_NOT_FOUND;
+                        break;
+                    }
+                }
+
+                if (paPages[iPage].idSharedPage != NIL_GMM_PAGEID)
+                {
+                    PGMMPAGE pPage = gmmR0GetPage(pGMM, paPages[iPage].idSharedPage);
+                    if (RT_LIKELY(pPage))
+                    {
+                        if (RT_LIKELY(GMM_PAGE_IS_SHARED(pPage)))
+                        {
+                            AssertCompile(NIL_RTHCPHYS > GMM_GCPHYS_LAST && GMM_GCPHYS_UNSHAREABLE > GMM_GCPHYS_LAST);
+                            Assert(pPage->Shared.cRefs);
+                            Assert(pGVM->gmm.s.Stats.cSharedPages);
+                            Assert(pGVM->gmm.s.Stats.Allocated.cBasePages);
+
+                            Log(("GMMR0AllocateHandyPages: free shared page %x cRefs=%d\n", paPages[iPage].idSharedPage, pPage->Shared.cRefs));
+                            pGVM->gmm.s.Stats.cSharedPages--;
+                            pGVM->gmm.s.Stats.Allocated.cBasePages--;
+                            if (!--pPage->Shared.cRefs)
+                                gmmR0FreeSharedPage(pGMM, pGVM, paPages[iPage].idSharedPage, pPage);
+                            else
+                            {
+                                Assert(pGMM->cDuplicatePages);
+                                pGMM->cDuplicatePages--;
+                            }
+
+                            paPages[iPage].idSharedPage = NIL_GMM_PAGEID;
+                        }
+                        else
+                        {
+                            Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not shared!\n", iPage, paPages[iPage].idSharedPage));
+                            rc = VERR_GMM_PAGE_NOT_SHARED;
+                            break;
+                        }
+                    }
+                    else
+                    {
+                        Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not found! (shared)\n", iPage, paPages[iPage].idSharedPage));
+                        rc = VERR_GMM_PAGE_NOT_FOUND;
+                        break;
+                    }
+                }
+            } /* for each page to update */
+
+            if (RT_SUCCESS(rc) && cPagesToAlloc > 0)
+            {
+#if defined(VBOX_STRICT) && 0 /** @todo re-test this later. Appeared to be a PGM init bug. */
+                for (iPage = 0; iPage < cPagesToAlloc; iPage++)
+                {
+                    Assert(paPages[iPage].HCPhysGCPhys  == NIL_RTHCPHYS);
+                    Assert(paPages[iPage].idPage        == NIL_GMM_PAGEID);
+                    Assert(paPages[iPage].idSharedPage  == NIL_GMM_PAGEID);
+                }
+#endif
+
+                /*
+                 * Join paths with GMMR0AllocatePages for the allocation.
+                 * Note! gmmR0AllocateMoreChunks may leave the protection of the mutex!
+                 */
+                rc = gmmR0AllocatePagesNew(pGMM, pGVM, cPagesToAlloc, paPages, GMMACCOUNT_BASE);
+            }
+        }
+        else
+            rc = VERR_WRONG_ORDER;
+        GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+    gmmR0MutexRelease(pGMM);
+    LogFlow(("GMMR0AllocateHandyPages: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Allocate one or more pages.
+ *
+ * This is typically used for ROMs and MMIO2 (VRAM) during VM creation.
+ * The allocated pages are not cleared and will contain random garbage.
+ *
+ * @returns VBox status code:
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_NOT_OWNER if the caller is not an EMT.
+ * @retval  VERR_GMM_SEED_ME if seeding via GMMR0SeedChunk is necessary.
+ * @retval  VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages.
+ * @retval  VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit,
+ *          that is we're trying to allocate more than we've reserved.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The VCPU id.
+ * @param   cPages      The number of pages to allocate.
+ * @param   paPages     Pointer to the page descriptors.
+ *                      See GMMPAGEDESC for details on what is expected on
+ *                      input.
+ * @param   enmAccount  The account to charge.
+ *
+ * @thread  EMT.
+ */
+GMMR0DECL(int) GMMR0AllocatePages(PGVM pGVM, VMCPUID idCpu, uint32_t cPages, PGMMPAGEDESC paPages, GMMACCOUNT enmAccount)
+{
+    LogFlow(("GMMR0AllocatePages: pGVM=%p cPages=%#x paPages=%p enmAccount=%d\n", pGVM, cPages, paPages, enmAccount));
+
+    /*
+     * Validate, get basics and take the semaphore.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    AssertPtrReturn(paPages, VERR_INVALID_PARAMETER);
+    AssertMsgReturn(enmAccount > GMMACCOUNT_INVALID && enmAccount < GMMACCOUNT_END, ("%d\n", enmAccount), VERR_INVALID_PARAMETER);
+    AssertMsgReturn(cPages > 0 && cPages < RT_BIT(32 - PAGE_SHIFT), ("%#x\n", cPages), VERR_INVALID_PARAMETER);
+
+    for (unsigned iPage = 0; iPage < cPages; iPage++)
+    {
+        AssertMsgReturn(    paPages[iPage].HCPhysGCPhys == NIL_RTHCPHYS
+                        ||  paPages[iPage].HCPhysGCPhys == GMM_GCPHYS_UNSHAREABLE
+                        ||  (    enmAccount == GMMACCOUNT_BASE
+                             &&  paPages[iPage].HCPhysGCPhys <= GMM_GCPHYS_LAST
+                             && !(paPages[iPage].HCPhysGCPhys & PAGE_OFFSET_MASK)),
+                        ("#%#x: %RHp enmAccount=%d\n", iPage, paPages[iPage].HCPhysGCPhys, enmAccount),
+                        VERR_INVALID_PARAMETER);
+        AssertMsgReturn(paPages[iPage].idPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER);
+        AssertMsgReturn(paPages[iPage].idSharedPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idSharedPage), VERR_INVALID_PARAMETER);
+    }
+
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+
+        /* No allocations before the initial reservation has been made! */
+        if (RT_LIKELY(    pGVM->gmm.s.Stats.Reserved.cBasePages
+                      &&  pGVM->gmm.s.Stats.Reserved.cFixedPages
+                      &&  pGVM->gmm.s.Stats.Reserved.cShadowPages))
+            rc = gmmR0AllocatePagesNew(pGMM, pGVM, cPages, paPages, enmAccount);
+        else
+            rc = VERR_WRONG_ORDER;
+        GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+    gmmR0MutexRelease(pGMM);
+    LogFlow(("GMMR0AllocatePages: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VMMR0 request wrapper for GMMR0AllocatePages.
+ *
+ * @returns see GMMR0AllocatePages.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The VCPU id.
+ * @param   pReq        Pointer to the request packet.
+ */
+GMMR0DECL(int) GMMR0AllocatePagesReq(PGVM pGVM, VMCPUID idCpu, PGMMALLOCATEPAGESREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq >= RT_UOFFSETOF(GMMALLOCATEPAGESREQ, aPages[0]),
+                    ("%#x < %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF(GMMALLOCATEPAGESREQ, aPages[0])),
+                    VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(GMMALLOCATEPAGESREQ, aPages[pReq->cPages]),
+                    ("%#x != %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF_DYN(GMMALLOCATEPAGESREQ, aPages[pReq->cPages])),
+                    VERR_INVALID_PARAMETER);
+
+    return GMMR0AllocatePages(pGVM, idCpu, pReq->cPages, &pReq->aPages[0], pReq->enmAccount);
+}
+
+
+/**
+ * Allocate a large page to represent guest RAM
+ *
+ * The allocated pages are not cleared and will contains random garbage.
+ *
+ * @returns VBox status code:
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_NOT_OWNER if the caller is not an EMT.
+ * @retval  VERR_GMM_SEED_ME if seeding via GMMR0SeedChunk is necessary.
+ * @retval  VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages.
+ * @retval  VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit,
+ *          that is we're trying to allocate more than we've reserved.
+ * @returns see GMMR0AllocatePages.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The VCPU id.
+ * @param   cbPage      Large page size.
+ * @param   pIdPage     Where to return the GMM page ID of the page.
+ * @param   pHCPhys     Where to return the host physical address of the page.
+ */
+GMMR0DECL(int)  GMMR0AllocateLargePage(PGVM pGVM, VMCPUID idCpu, uint32_t cbPage, uint32_t *pIdPage, RTHCPHYS *pHCPhys)
+{
+    LogFlow(("GMMR0AllocateLargePage: pGVM=%p cbPage=%x\n", pGVM, cbPage));
+
+    AssertReturn(cbPage == GMM_CHUNK_SIZE, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pIdPage, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
+
+    /*
+     * Validate, get basics and take the semaphore.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+#ifdef GMM_WITH_LEGACY_MODE
+    // /* Not supported in legacy mode where we allocate the memory in ring 3 and lock it in ring 0. */
+    // if (pGMM->fLegacyAllocationMode)
+    //     return VERR_NOT_SUPPORTED;
+#endif
+
+    *pHCPhys = NIL_RTHCPHYS;
+    *pIdPage = NIL_GMM_PAGEID;
+
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        const unsigned cPages = (GMM_CHUNK_SIZE >> PAGE_SHIFT);
+        if (RT_UNLIKELY(  pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.cBalloonedPages + cPages
+                        > pGVM->gmm.s.Stats.Reserved.cBasePages))
+        {
+            Log(("GMMR0AllocateLargePage: Reserved=%#llx Allocated+Requested=%#llx+%#x!\n",
+                 pGVM->gmm.s.Stats.Reserved.cBasePages, pGVM->gmm.s.Stats.Allocated.cBasePages, cPages));
+            gmmR0MutexRelease(pGMM);
+            return VERR_GMM_HIT_VM_ACCOUNT_LIMIT;
+        }
+
+        /*
+         * Allocate a new large page chunk.
+         *
+         * Note! We leave the giant GMM lock temporarily as the allocation might
+         *       take a long time.  gmmR0RegisterChunk will retake it (ugly).
+         */
+        AssertCompile(GMM_CHUNK_SIZE == _2M);
+        gmmR0MutexRelease(pGMM);
+
+        RTR0MEMOBJ hMemObj;
+        rc = RTR0MemObjAllocPhysEx(&hMemObj, GMM_CHUNK_SIZE, NIL_RTHCPHYS, GMM_CHUNK_SIZE);
+        if (RT_SUCCESS(rc))
+        {
+            PGMMCHUNKFREESET pSet = pGMM->fBoundMemoryMode ? &pGVM->gmm.s.Private : &pGMM->PrivateX;
+            PGMMCHUNK pChunk;
+            rc = gmmR0RegisterChunk(pGMM, pSet, hMemObj, pGVM->hSelf, GMM_CHUNK_FLAGS_LARGE_PAGE, &pChunk);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Allocate all the pages in the chunk.
+                 */
+                /* Unlink the new chunk from the free list. */
+                gmmR0UnlinkChunk(pChunk);
+
+                /** @todo rewrite this to skip the looping. */
+                /* Allocate all pages. */
+                GMMPAGEDESC PageDesc;
+                gmmR0AllocatePage(pChunk, pGVM->hSelf, &PageDesc);
+
+                /* Return the first page as we'll use the whole chunk as one big page. */
+                *pIdPage = PageDesc.idPage;
+                *pHCPhys = PageDesc.HCPhysGCPhys;
+
+                for (unsigned i = 1; i < cPages; i++)
+                    gmmR0AllocatePage(pChunk, pGVM->hSelf, &PageDesc);
+
+                /* Update accounting. */
+                pGVM->gmm.s.Stats.Allocated.cBasePages += cPages;
+                pGVM->gmm.s.Stats.cPrivatePages        += cPages;
+                pGMM->cAllocatedPages                  += cPages;
+
+                gmmR0LinkChunk(pChunk, pSet);
+                gmmR0MutexRelease(pGMM);
+                LogFlow(("GMMR0AllocateLargePage: returns VINF_SUCCESS\n"));
+                return VINF_SUCCESS;
+            }
+            RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
+        }
+    }
+    else
+    {
+        gmmR0MutexRelease(pGMM);
+        rc = VERR_GMM_IS_NOT_SANE;
+    }
+
+    LogFlow(("GMMR0AllocateLargePage: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Free a large page.
+ *
+ * @returns VBox status code:
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The VCPU id.
+ * @param   idPage      The large page id.
+ */
+GMMR0DECL(int)  GMMR0FreeLargePage(PGVM pGVM, VMCPUID idCpu, uint32_t idPage)
+{
+    LogFlow(("GMMR0FreeLargePage: pGVM=%p idPage=%x\n", pGVM, idPage));
+
+    /*
+     * Validate, get basics and take the semaphore.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+#ifdef GMM_WITH_LEGACY_MODE
+    // /* Not supported in legacy mode where we allocate the memory in ring 3 and lock it in ring 0. */
+    // if (pGMM->fLegacyAllocationMode)
+    //     return VERR_NOT_SUPPORTED;
+#endif
+
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        const unsigned cPages = (GMM_CHUNK_SIZE >> PAGE_SHIFT);
+
+        if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cBasePages < cPages))
+        {
+            Log(("GMMR0FreeLargePage: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cBasePages, cPages));
+            gmmR0MutexRelease(pGMM);
+            return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
+        }
+
+        PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage);
+        if (RT_LIKELY(   pPage
+                      && GMM_PAGE_IS_PRIVATE(pPage)))
+        {
+            PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
+            Assert(pChunk);
+            Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES);
+            Assert(pChunk->cPrivate > 0);
+
+            /* Release the memory immediately. */
+            gmmR0FreeChunk(pGMM, NULL, pChunk, false /*fRelaxedSem*/); /** @todo this can be relaxed too! */
+
+            /* Update accounting. */
+            pGVM->gmm.s.Stats.Allocated.cBasePages -= cPages;
+            pGVM->gmm.s.Stats.cPrivatePages        -= cPages;
+            pGMM->cAllocatedPages                  -= cPages;
+        }
+        else
+            rc = VERR_GMM_PAGE_NOT_FOUND;
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+
+    gmmR0MutexRelease(pGMM);
+    LogFlow(("GMMR0FreeLargePage: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VMMR0 request wrapper for GMMR0FreeLargePage.
+ *
+ * @returns see GMMR0FreeLargePage.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The VCPU id.
+ * @param   pReq        Pointer to the request packet.
+ */
+GMMR0DECL(int) GMMR0FreeLargePageReq(PGVM pGVM, VMCPUID idCpu, PGMMFREELARGEPAGEREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMFREEPAGESREQ),
+                    ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(GMMFREEPAGESREQ)),
+                    VERR_INVALID_PARAMETER);
+
+    return GMMR0FreeLargePage(pGVM, idCpu, pReq->idPage);
+}
+
+
+/**
+ * @callback_method_impl{FNGVMMR0ENUMCALLBACK,
+ * Used by gmmR0FreeChunkFlushPerVmTlbs().}
+ */
+static DECLCALLBACK(int) gmmR0InvalidatePerVmChunkTlbCallback(PGVM pGVM, void *pvUser)
+{
+    RT_NOREF(pvUser);
+    if (pGVM->gmm.s.hChunkTlbSpinLock != NIL_RTSPINLOCK)
+    {
+        RTSpinlockAcquire(pGVM->gmm.s.hChunkTlbSpinLock);
+        uintptr_t i = RT_ELEMENTS(pGVM->gmm.s.aChunkTlbEntries);
+        while (i-- > 0)
+        {
+            pGVM->gmm.s.aChunkTlbEntries[i].idGeneration = UINT64_MAX;
+            pGVM->gmm.s.aChunkTlbEntries[i].pChunk       = NULL;
+        }
+        RTSpinlockRelease(pGVM->gmm.s.hChunkTlbSpinLock);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Called by gmmR0FreeChunk when we reach the threshold for wrapping around the
+ * free generation ID value.
+ *
+ * This is done at 2^62 - 1, which allows us to drop all locks and as it will
+ * take a while before 12 exa (2 305 843 009 213 693 952) calls to
+ * gmmR0FreeChunk can be made and causes a real wrap-around.  We do two
+ * invalidation passes and resets the generation ID between then.  This will
+ * make sure there are no false positives.
+ *
+ * @param   pGMM        Pointer to the GMM instance.
+ */
+static void gmmR0FreeChunkFlushPerVmTlbs(PGMM pGMM)
+{
+    /*
+     * First invalidation pass.
+     */
+    int rc = GVMMR0EnumVMs(gmmR0InvalidatePerVmChunkTlbCallback, NULL);
+    AssertRCSuccess(rc);
+
+    /*
+     * Reset the generation number.
+     */
+    RTSpinlockAcquire(pGMM->hSpinLockTree);
+    ASMAtomicWriteU64(&pGMM->idFreeGeneration, 1);
+    RTSpinlockRelease(pGMM->hSpinLockTree);
+
+    /*
+     * Second invalidation pass.
+     */
+    rc = GVMMR0EnumVMs(gmmR0InvalidatePerVmChunkTlbCallback, NULL);
+    AssertRCSuccess(rc);
+}
+
+
+/**
+ * Frees a chunk, giving it back to the host OS.
+ *
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   pGVM        This is set when called from GMMR0CleanupVM so we can
+ *                      unmap and free the chunk in one go.
+ * @param   pChunk      The chunk to free.
+ * @param   fRelaxedSem Whether we can release the semaphore while doing the
+ *                      freeing (@c true) or not.
+ */
+static bool gmmR0FreeChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem)
+{
+    Assert(pChunk->Core.Key != NIL_GMM_CHUNKID);
+
+    GMMR0CHUNKMTXSTATE MtxState;
+    gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT);
+
+    /*
+     * Cleanup hack! Unmap the chunk from the callers address space.
+     * This shouldn't happen, so screw lock contention...
+     */
+    if (    pChunk->cMappingsX
+#ifdef GMM_WITH_LEGACY_MODE
+        &&  (!pGMM->fLegacyAllocationMode || (pChunk->fFlags & GMM_CHUNK_FLAGS_LARGE_PAGE))
+#endif
+        &&  pGVM)
+        gmmR0UnmapChunkLocked(pGMM, pGVM, pChunk);
+
+    /*
+     * If there are current mappings of the chunk, then request the
+     * VMs to unmap them. Reposition the chunk in the free list so
+     * it won't be a likely candidate for allocations.
+     */
+    if (pChunk->cMappingsX)
+    {
+        /** @todo R0 -> VM request */
+        /* The chunk can be mapped by more than one VM if fBoundMemoryMode is false! */
+        Log(("gmmR0FreeChunk: chunk still has %d mappings; don't free!\n", pChunk->cMappingsX));
+        gmmR0ChunkMutexRelease(&MtxState, pChunk);
+        return false;
+    }
+
+
+    /*
+     * Save and trash the handle.
+     */
+    RTR0MEMOBJ const hMemObj = pChunk->hMemObj;
+    pChunk->hMemObj = NIL_RTR0MEMOBJ;
+
+    /*
+     * Unlink it from everywhere.
+     */
+    gmmR0UnlinkChunk(pChunk);
+
+    RTSpinlockAcquire(pGMM->hSpinLockTree);
+
+    RTListNodeRemove(&pChunk->ListNode);
+
+    PAVLU32NODECORE pCore = RTAvlU32Remove(&pGMM->pChunks, pChunk->Core.Key);
+    Assert(pCore == &pChunk->Core); NOREF(pCore);
+
+    PGMMCHUNKTLBE pTlbe = &pGMM->ChunkTLB.aEntries[GMM_CHUNKTLB_IDX(pChunk->Core.Key)];
+    if (pTlbe->pChunk == pChunk)
+    {
+        pTlbe->idChunk = NIL_GMM_CHUNKID;
+        pTlbe->pChunk = NULL;
+    }
+
+    Assert(pGMM->cChunks > 0);
+    pGMM->cChunks--;
+
+    uint64_t const idFreeGeneration = ASMAtomicIncU64(&pGMM->idFreeGeneration);
+
+    RTSpinlockRelease(pGMM->hSpinLockTree);
+
+    /*
+     * Free the Chunk ID before dropping the locks and freeing the rest.
+     */
+    gmmR0FreeChunkId(pGMM, pChunk->Core.Key);
+    pChunk->Core.Key = NIL_GMM_CHUNKID;
+
+    pGMM->cFreedChunks++;
+
+    gmmR0ChunkMutexRelease(&MtxState, NULL);
+    if (fRelaxedSem)
+        gmmR0MutexRelease(pGMM);
+
+    if (idFreeGeneration == UINT64_MAX / 4)
+        gmmR0FreeChunkFlushPerVmTlbs(pGMM);
+
+    RTMemFree(pChunk->paMappingsX);
+    pChunk->paMappingsX = NULL;
+
+    RTMemFree(pChunk);
+
+#if defined(VBOX_WITH_RAM_IN_KERNEL) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+    int rc = RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
+#else
+    int rc = RTR0MemObjFree(hMemObj, false /* fFreeMappings */);
+#endif
+    AssertLogRelRC(rc);
+
+    if (fRelaxedSem)
+        gmmR0MutexAcquire(pGMM);
+    return fRelaxedSem;
+}
+
+
+/**
+ * Free page worker.
+ *
+ * The caller does all the statistic decrementing, we do all the incrementing.
+ *
+ * @param   pGMM        Pointer to the GMM instance data.
+ * @param   pGVM        Pointer to the GVM instance.
+ * @param   pChunk      Pointer to the chunk this page belongs to.
+ * @param   idPage      The Page ID.
+ * @param   pPage       Pointer to the page.
+ */
+static void gmmR0FreePageWorker(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, uint32_t idPage, PGMMPAGE pPage)
+{
+    Log3(("F pPage=%p iPage=%#x/%#x u2State=%d iFreeHead=%#x\n",
+          pPage, pPage - &pChunk->aPages[0], idPage, pPage->Common.u2State, pChunk->iFreeHead)); NOREF(idPage);
+
+    /*
+     * Put the page on the free list.
+     */
+    pPage->u = 0;
+    pPage->Free.u2State = GMM_PAGE_STATE_FREE;
+    Assert(pChunk->iFreeHead < RT_ELEMENTS(pChunk->aPages) || pChunk->iFreeHead == UINT16_MAX);
+    pPage->Free.iNext = pChunk->iFreeHead;
+    pChunk->iFreeHead = pPage - &pChunk->aPages[0];
+
+    /*
+     * Update statistics (the cShared/cPrivate stats are up to date already),
+     * and relink the chunk if necessary.
+     */
+    unsigned const cFree = pChunk->cFree;
+    if (   !cFree
+        || gmmR0SelectFreeSetList(cFree) != gmmR0SelectFreeSetList(cFree + 1))
+    {
+        gmmR0UnlinkChunk(pChunk);
+        pChunk->cFree++;
+        gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk);
+    }
+    else
+    {
+        pChunk->cFree = cFree + 1;
+        pChunk->pSet->cFreePages++;
+    }
+
+    /*
+     * If the chunk becomes empty, consider giving memory back to the host OS.
+     *
+     * The current strategy is to try give it back if there are other chunks
+     * in this free list, meaning if there are at least 240 free pages in this
+     * category. Note that since there are probably mappings of the chunk,
+     * it won't be freed up instantly, which probably screws up this logic
+     * a bit...
+     */
+    /** @todo Do this on the way out. */
+    if (RT_LIKELY(   pChunk->cFree != GMM_CHUNK_NUM_PAGES
+                  || pChunk->pFreeNext == NULL
+                  || pChunk->pFreePrev == NULL /** @todo this is probably misfiring, see reset... */))
+    { /* likely */ }
+#ifdef GMM_WITH_LEGACY_MODE
+    else if (RT_LIKELY(pGMM->fLegacyAllocationMode && !(pChunk->fFlags & GMM_CHUNK_FLAGS_LARGE_PAGE)))
+    { /* likely */ }
+#endif
+    else
+        gmmR0FreeChunk(pGMM, NULL, pChunk, false);
+
+}
+
+
+/**
+ * Frees a shared page, the page is known to exist and be valid and such.
+ *
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   pGVM        Pointer to the GVM instance.
+ * @param   idPage      The page id.
+ * @param   pPage       The page structure.
+ */
+DECLINLINE(void) gmmR0FreeSharedPage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage)
+{
+    PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
+    Assert(pChunk);
+    Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES);
+    Assert(pChunk->cShared > 0);
+    Assert(pGMM->cSharedPages > 0);
+    Assert(pGMM->cAllocatedPages > 0);
+    Assert(!pPage->Shared.cRefs);
+
+    pChunk->cShared--;
+    pGMM->cAllocatedPages--;
+    pGMM->cSharedPages--;
+    gmmR0FreePageWorker(pGMM, pGVM, pChunk, idPage, pPage);
+}
+
+
+/**
+ * Frees a private page, the page is known to exist and be valid and such.
+ *
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   pGVM        Pointer to the GVM instance.
+ * @param   idPage      The page id.
+ * @param   pPage       The page structure.
+ */
+DECLINLINE(void) gmmR0FreePrivatePage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage)
+{
+    PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
+    Assert(pChunk);
+    Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES);
+    Assert(pChunk->cPrivate > 0);
+    Assert(pGMM->cAllocatedPages > 0);
+
+    pChunk->cPrivate--;
+    pGMM->cAllocatedPages--;
+    gmmR0FreePageWorker(pGMM, pGVM, pChunk, idPage, pPage);
+}
+
+
+/**
+ * Common worker for GMMR0FreePages and GMMR0BalloonedPages.
+ *
+ * @returns VBox status code:
+ * @retval  xxx
+ *
+ * @param   pGMM        Pointer to the GMM instance data.
+ * @param   pGVM        Pointer to the VM.
+ * @param   cPages      The number of pages to free.
+ * @param   paPages     Pointer to the page descriptors.
+ * @param   enmAccount  The account this relates to.
+ */
+static int gmmR0FreePages(PGMM pGMM, PGVM pGVM, uint32_t cPages, PGMMFREEPAGEDESC paPages, GMMACCOUNT enmAccount)
+{
+    /*
+     * Check that the request isn't impossible wrt to the account status.
+     */
+    switch (enmAccount)
+    {
+        case GMMACCOUNT_BASE:
+            if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cBasePages < cPages))
+            {
+                Log(("gmmR0FreePages: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cBasePages, cPages));
+                return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
+            }
+            break;
+        case GMMACCOUNT_SHADOW:
+            if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cShadowPages < cPages))
+            {
+                Log(("gmmR0FreePages: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cShadowPages, cPages));
+                return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
+            }
+            break;
+        case GMMACCOUNT_FIXED:
+            if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cFixedPages < cPages))
+            {
+                Log(("gmmR0FreePages: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cFixedPages, cPages));
+                return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
+            }
+            break;
+        default:
+            AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+
+    /*
+     * Walk the descriptors and free the pages.
+     *
+     * Statistics (except the account) are being updated as we go along,
+     * unlike the alloc code. Also, stop on the first error.
+     */
+    int rc = VINF_SUCCESS;
+    uint32_t iPage;
+    for (iPage = 0; iPage < cPages; iPage++)
+    {
+        uint32_t idPage = paPages[iPage].idPage;
+        PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage);
+        if (RT_LIKELY(pPage))
+        {
+            if (RT_LIKELY(GMM_PAGE_IS_PRIVATE(pPage)))
+            {
+                if (RT_LIKELY(pPage->Private.hGVM == pGVM->hSelf))
+                {
+                    Assert(pGVM->gmm.s.Stats.cPrivatePages);
+                    pGVM->gmm.s.Stats.cPrivatePages--;
+                    gmmR0FreePrivatePage(pGMM, pGVM, idPage, pPage);
+                }
+                else
+                {
+                    Log(("gmmR0AllocatePages: #%#x/%#x: not owner! hGVM=%#x hSelf=%#x\n", iPage, idPage,
+                         pPage->Private.hGVM, pGVM->hSelf));
+                    rc = VERR_GMM_NOT_PAGE_OWNER;
+                    break;
+                }
+            }
+            else if (RT_LIKELY(GMM_PAGE_IS_SHARED(pPage)))
+            {
+                Assert(pGVM->gmm.s.Stats.cSharedPages);
+                Assert(pPage->Shared.cRefs);
+#if defined(VBOX_WITH_PAGE_SHARING) && defined(VBOX_STRICT) && HC_ARCH_BITS == 64
+                if (pPage->Shared.u14Checksum)
+                {
+                    uint32_t uChecksum = gmmR0StrictPageChecksum(pGMM, pGVM, idPage);
+                    uChecksum &= UINT32_C(0x00003fff);
+                    AssertMsg(!uChecksum || uChecksum == pPage->Shared.u14Checksum,
+                              ("%#x vs %#x - idPage=%#x\n", uChecksum, pPage->Shared.u14Checksum, idPage));
+                }
+#endif
+                pGVM->gmm.s.Stats.cSharedPages--;
+                if (!--pPage->Shared.cRefs)
+                    gmmR0FreeSharedPage(pGMM, pGVM, idPage, pPage);
+                else
+                {
+                    Assert(pGMM->cDuplicatePages);
+                    pGMM->cDuplicatePages--;
+                }
+            }
+            else
+            {
+                Log(("gmmR0AllocatePages: #%#x/%#x: already free!\n", iPage, idPage));
+                rc = VERR_GMM_PAGE_ALREADY_FREE;
+                break;
+            }
+        }
+        else
+        {
+            Log(("gmmR0AllocatePages: #%#x/%#x: not found!\n", iPage, idPage));
+            rc = VERR_GMM_PAGE_NOT_FOUND;
+            break;
+        }
+        paPages[iPage].idPage = NIL_GMM_PAGEID;
+    }
+
+    /*
+     * Update the account.
+     */
+    switch (enmAccount)
+    {
+        case GMMACCOUNT_BASE:   pGVM->gmm.s.Stats.Allocated.cBasePages   -= iPage; break;
+        case GMMACCOUNT_SHADOW: pGVM->gmm.s.Stats.Allocated.cShadowPages -= iPage; break;
+        case GMMACCOUNT_FIXED:  pGVM->gmm.s.Stats.Allocated.cFixedPages  -= iPage; break;
+        default:
+            AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+
+    /*
+     * Any threshold stuff to be done here?
+     */
+
+    return rc;
+}
+
+
+/**
+ * Free one or more pages.
+ *
+ * This is typically used at reset time or power off.
+ *
+ * @returns VBox status code:
+ * @retval  xxx
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The VCPU id.
+ * @param   cPages      The number of pages to allocate.
+ * @param   paPages     Pointer to the page descriptors containing the page IDs
+ *                      for each page.
+ * @param   enmAccount  The account this relates to.
+ * @thread  EMT.
+ */
+GMMR0DECL(int) GMMR0FreePages(PGVM pGVM, VMCPUID idCpu, uint32_t cPages, PGMMFREEPAGEDESC paPages, GMMACCOUNT enmAccount)
+{
+    LogFlow(("GMMR0FreePages: pGVM=%p cPages=%#x paPages=%p enmAccount=%d\n", pGVM, cPages, paPages, enmAccount));
+
+    /*
+     * Validate input and get the basics.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    AssertPtrReturn(paPages, VERR_INVALID_PARAMETER);
+    AssertMsgReturn(enmAccount > GMMACCOUNT_INVALID && enmAccount < GMMACCOUNT_END, ("%d\n", enmAccount), VERR_INVALID_PARAMETER);
+    AssertMsgReturn(cPages > 0 && cPages < RT_BIT(32 - PAGE_SHIFT), ("%#x\n", cPages), VERR_INVALID_PARAMETER);
+
+    for (unsigned iPage = 0; iPage < cPages; iPage++)
+        AssertMsgReturn(    paPages[iPage].idPage <= GMM_PAGEID_LAST
+                        /*||  paPages[iPage].idPage == NIL_GMM_PAGEID*/,
+                        ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER);
+
+    /*
+     * Take the semaphore and call the worker function.
+     */
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        rc = gmmR0FreePages(pGMM, pGVM, cPages, paPages, enmAccount);
+        GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+    gmmR0MutexRelease(pGMM);
+    LogFlow(("GMMR0FreePages: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VMMR0 request wrapper for GMMR0FreePages.
+ *
+ * @returns see GMMR0FreePages.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The VCPU id.
+ * @param   pReq        Pointer to the request packet.
+ */
+GMMR0DECL(int) GMMR0FreePagesReq(PGVM pGVM, VMCPUID idCpu, PGMMFREEPAGESREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq >= RT_UOFFSETOF(GMMFREEPAGESREQ, aPages[0]),
+                    ("%#x < %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF(GMMFREEPAGESREQ, aPages[0])),
+                    VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(GMMFREEPAGESREQ, aPages[pReq->cPages]),
+                    ("%#x != %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF_DYN(GMMFREEPAGESREQ, aPages[pReq->cPages])),
+                    VERR_INVALID_PARAMETER);
+
+    return GMMR0FreePages(pGVM, idCpu, pReq->cPages, &pReq->aPages[0], pReq->enmAccount);
+}
+
+
+/**
+ * Report back on a memory ballooning request.
+ *
+ * The request may or may not have been initiated by the GMM. If it was initiated
+ * by the GMM it is important that this function is called even if no pages were
+ * ballooned.
+ *
+ * @returns VBox status code:
+ * @retval  VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH
+ * @retval  VERR_GMM_ATTEMPT_TO_DEFLATE_TOO_MUCH
+ * @retval  VERR_GMM_OVERCOMMITTED_TRY_AGAIN_IN_A_BIT - reset condition
+ *          indicating that we won't necessarily have sufficient RAM to boot
+ *          the VM again and that it should pause until this changes (we'll try
+ *          balloon some other VM).  (For standard deflate we have little choice
+ *          but to hope the VM won't use the memory that was returned to it.)
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   idCpu               The VCPU id.
+ * @param   enmAction           Inflate/deflate/reset.
+ * @param   cBalloonedPages     The number of pages that was ballooned.
+ *
+ * @thread  EMT(idCpu)
+ */
+GMMR0DECL(int) GMMR0BalloonedPages(PGVM pGVM, VMCPUID idCpu, GMMBALLOONACTION enmAction, uint32_t cBalloonedPages)
+{
+    LogFlow(("GMMR0BalloonedPages: pGVM=%p enmAction=%d cBalloonedPages=%#x\n",
+             pGVM, enmAction, cBalloonedPages));
+
+    AssertMsgReturn(cBalloonedPages < RT_BIT(32 - PAGE_SHIFT), ("%#x\n", cBalloonedPages), VERR_INVALID_PARAMETER);
+
+    /*
+     * Validate input and get the basics.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Take the semaphore and do some more validations.
+     */
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        switch (enmAction)
+        {
+            case GMMBALLOONACTION_INFLATE:
+            {
+                if (RT_LIKELY(pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.cBalloonedPages + cBalloonedPages
+                              <= pGVM->gmm.s.Stats.Reserved.cBasePages))
+                {
+                    /*
+                     * Record the ballooned memory.
+                     */
+                    pGMM->cBalloonedPages += cBalloonedPages;
+                    if (pGVM->gmm.s.Stats.cReqBalloonedPages)
+                    {
+                        /* Codepath never taken. Might be interesting in the future to request ballooned memory from guests in low memory conditions.. */
+                        AssertFailed();
+
+                        pGVM->gmm.s.Stats.cBalloonedPages            += cBalloonedPages;
+                        pGVM->gmm.s.Stats.cReqActuallyBalloonedPages += cBalloonedPages;
+                        Log(("GMMR0BalloonedPages: +%#x - Global=%#llx / VM: Total=%#llx Req=%#llx Actual=%#llx (pending)\n",
+                             cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages,
+                             pGVM->gmm.s.Stats.cReqBalloonedPages, pGVM->gmm.s.Stats.cReqActuallyBalloonedPages));
+                    }
+                    else
+                    {
+                        pGVM->gmm.s.Stats.cBalloonedPages += cBalloonedPages;
+                        Log(("GMMR0BalloonedPages: +%#x - Global=%#llx / VM: Total=%#llx (user)\n",
+                             cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages));
+                    }
+                }
+                else
+                {
+                    Log(("GMMR0BalloonedPages: cBasePages=%#llx Total=%#llx cBalloonedPages=%#llx Reserved=%#llx\n",
+                         pGVM->gmm.s.Stats.Allocated.cBasePages, pGVM->gmm.s.Stats.cBalloonedPages, cBalloonedPages,
+                         pGVM->gmm.s.Stats.Reserved.cBasePages));
+                    rc = VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
+                }
+                break;
+            }
+
+            case GMMBALLOONACTION_DEFLATE:
+            {
+                /* Deflate. */
+                if (pGVM->gmm.s.Stats.cBalloonedPages >= cBalloonedPages)
+                {
+                    /*
+                     * Record the ballooned memory.
+                     */
+                    Assert(pGMM->cBalloonedPages >= cBalloonedPages);
+                    pGMM->cBalloonedPages             -= cBalloonedPages;
+                    pGVM->gmm.s.Stats.cBalloonedPages -= cBalloonedPages;
+                    if (pGVM->gmm.s.Stats.cReqDeflatePages)
+                    {
+                        AssertFailed(); /* This is path is for later. */
+                        Log(("GMMR0BalloonedPages: -%#x - Global=%#llx / VM: Total=%#llx Req=%#llx\n",
+                             cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages, pGVM->gmm.s.Stats.cReqDeflatePages));
+
+                        /*
+                         * Anything we need to do here now when the request has been completed?
+                         */
+                        pGVM->gmm.s.Stats.cReqDeflatePages = 0;
+                    }
+                    else
+                        Log(("GMMR0BalloonedPages: -%#x - Global=%#llx / VM: Total=%#llx (user)\n",
+                             cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages));
+                }
+                else
+                {
+                    Log(("GMMR0BalloonedPages: Total=%#llx cBalloonedPages=%#llx\n", pGVM->gmm.s.Stats.cBalloonedPages, cBalloonedPages));
+                    rc = VERR_GMM_ATTEMPT_TO_DEFLATE_TOO_MUCH;
+                }
+                break;
+            }
+
+            case GMMBALLOONACTION_RESET:
+            {
+                /* Reset to an empty balloon. */
+                Assert(pGMM->cBalloonedPages >= pGVM->gmm.s.Stats.cBalloonedPages);
+
+                pGMM->cBalloonedPages             -= pGVM->gmm.s.Stats.cBalloonedPages;
+                pGVM->gmm.s.Stats.cBalloonedPages  = 0;
+                break;
+            }
+
+            default:
+                rc = VERR_INVALID_PARAMETER;
+                break;
+        }
+        GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+
+    gmmR0MutexRelease(pGMM);
+    LogFlow(("GMMR0BalloonedPages: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VMMR0 request wrapper for GMMR0BalloonedPages.
+ *
+ * @returns see GMMR0BalloonedPages.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The VCPU id.
+ * @param   pReq        Pointer to the request packet.
+ */
+GMMR0DECL(int) GMMR0BalloonedPagesReq(PGVM pGVM, VMCPUID idCpu, PGMMBALLOONEDPAGESREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMBALLOONEDPAGESREQ),
+                    ("%#x < %#x\n", pReq->Hdr.cbReq, sizeof(GMMBALLOONEDPAGESREQ)),
+                    VERR_INVALID_PARAMETER);
+
+    return GMMR0BalloonedPages(pGVM, idCpu, pReq->enmAction, pReq->cBalloonedPages);
+}
+
+
+/**
+ * Return memory statistics for the hypervisor
+ *
+ * @returns VBox status code.
+ * @param   pReq        Pointer to the request packet.
+ */
+GMMR0DECL(int) GMMR0QueryHypervisorMemoryStatsReq(PGMMMEMSTATSREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMMEMSTATSREQ),
+                    ("%#x < %#x\n", pReq->Hdr.cbReq, sizeof(GMMMEMSTATSREQ)),
+                    VERR_INVALID_PARAMETER);
+
+    /*
+     * Validate input and get the basics.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    pReq->cAllocPages     = pGMM->cAllocatedPages;
+    pReq->cFreePages      = (pGMM->cChunks << (GMM_CHUNK_SHIFT- PAGE_SHIFT)) - pGMM->cAllocatedPages;
+    pReq->cBalloonedPages = pGMM->cBalloonedPages;
+    pReq->cMaxPages       = pGMM->cMaxPages;
+    pReq->cSharedPages    = pGMM->cDuplicatePages;
+    GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Return memory statistics for the VM
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       Cpu id.
+ * @param   pReq        Pointer to the request packet.
+ *
+ * @thread  EMT(idCpu)
+ */
+GMMR0DECL(int) GMMR0QueryMemoryStatsReq(PGVM pGVM, VMCPUID idCpu, PGMMMEMSTATSREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMMEMSTATSREQ),
+                    ("%#x < %#x\n", pReq->Hdr.cbReq, sizeof(GMMMEMSTATSREQ)),
+                    VERR_INVALID_PARAMETER);
+
+    /*
+     * Validate input and get the basics.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Take the semaphore and do some more validations.
+     */
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        pReq->cAllocPages     = pGVM->gmm.s.Stats.Allocated.cBasePages;
+        pReq->cBalloonedPages = pGVM->gmm.s.Stats.cBalloonedPages;
+        pReq->cMaxPages       = pGVM->gmm.s.Stats.Reserved.cBasePages;
+        pReq->cFreePages      = pReq->cMaxPages - pReq->cAllocPages;
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+
+    gmmR0MutexRelease(pGMM);
+    LogFlow(("GMMR3QueryVMMemoryStats: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Worker for gmmR0UnmapChunk and gmmr0FreeChunk.
+ *
+ * Don't call this in legacy allocation mode!
+ *
+ * @returns VBox status code.
+ * @param   pGMM        Pointer to the GMM instance data.
+ * @param   pGVM        Pointer to the Global VM structure.
+ * @param   pChunk      Pointer to the chunk to be unmapped.
+ */
+static int gmmR0UnmapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk)
+{
+    RT_NOREF_PV(pGMM);
+#ifdef GMM_WITH_LEGACY_MODE
+    Assert(!pGMM->fLegacyAllocationMode || (pChunk->fFlags & GMM_CHUNK_FLAGS_LARGE_PAGE));
+#endif
+
+    /*
+     * Find the mapping and try unmapping it.
+     */
+    uint32_t cMappings = pChunk->cMappingsX;
+    for (uint32_t i = 0; i < cMappings; i++)
+    {
+        Assert(pChunk->paMappingsX[i].pGVM && pChunk->paMappingsX[i].hMapObj != NIL_RTR0MEMOBJ);
+        if (pChunk->paMappingsX[i].pGVM == pGVM)
+        {
+            /* unmap */
+            int rc = RTR0MemObjFree(pChunk->paMappingsX[i].hMapObj, false /* fFreeMappings (NA) */);
+            if (RT_SUCCESS(rc))
+            {
+                /* update the record. */
+                cMappings--;
+                if (i < cMappings)
+                    pChunk->paMappingsX[i] = pChunk->paMappingsX[cMappings];
+                pChunk->paMappingsX[cMappings].hMapObj = NIL_RTR0MEMOBJ;
+                pChunk->paMappingsX[cMappings].pGVM    = NULL;
+                Assert(pChunk->cMappingsX - 1U == cMappings);
+                pChunk->cMappingsX = cMappings;
+            }
+
+            return rc;
+        }
+    }
+
+    Log(("gmmR0UnmapChunk: Chunk %#x is not mapped into pGVM=%p/%#x\n", pChunk->Core.Key, pGVM, pGVM->hSelf));
+    return VERR_GMM_CHUNK_NOT_MAPPED;
+}
+
+
+/**
+ * Unmaps a chunk previously mapped into the address space of the current process.
+ *
+ * @returns VBox status code.
+ * @param   pGMM        Pointer to the GMM instance data.
+ * @param   pGVM        Pointer to the Global VM structure.
+ * @param   pChunk      Pointer to the chunk to be unmapped.
+ * @param   fRelaxedSem Whether we can release the semaphore while doing the
+ *                      mapping (@c true) or not.
+ */
+static int gmmR0UnmapChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem)
+{
+#ifdef GMM_WITH_LEGACY_MODE
+    if (!pGMM->fLegacyAllocationMode || (pChunk->fFlags & GMM_CHUNK_FLAGS_LARGE_PAGE))
+    {
+#endif
+        /*
+         * Lock the chunk and if possible leave the giant GMM lock.
+         */
+        GMMR0CHUNKMTXSTATE MtxState;
+        int rc = gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk,
+                                        fRelaxedSem ? GMMR0CHUNK_MTX_RETAKE_GIANT : GMMR0CHUNK_MTX_KEEP_GIANT);
+        if (RT_SUCCESS(rc))
+        {
+            rc = gmmR0UnmapChunkLocked(pGMM, pGVM, pChunk);
+            gmmR0ChunkMutexRelease(&MtxState, pChunk);
+        }
+        return rc;
+#ifdef GMM_WITH_LEGACY_MODE
+    }
+
+    if (pChunk->hGVM == pGVM->hSelf)
+        return VINF_SUCCESS;
+
+    Log(("gmmR0UnmapChunk: Chunk %#x is not mapped into pGVM=%p/%#x (legacy)\n", pChunk->Core.Key, pGVM, pGVM->hSelf));
+    return VERR_GMM_CHUNK_NOT_MAPPED;
+#endif
+}
+
+
+/**
+ * Worker for gmmR0MapChunk.
+ *
+ * @returns VBox status code.
+ * @param   pGMM        Pointer to the GMM instance data.
+ * @param   pGVM        Pointer to the Global VM structure.
+ * @param   pChunk      Pointer to the chunk to be mapped.
+ * @param   ppvR3       Where to store the ring-3 address of the mapping.
+ *                      In the VERR_GMM_CHUNK_ALREADY_MAPPED case, this will be
+ *                      contain the address of the existing mapping.
+ */
+static int gmmR0MapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, PRTR3PTR ppvR3)
+{
+#ifdef GMM_WITH_LEGACY_MODE
+    /*
+     * If we're in legacy mode this is simple.
+     */
+    if (pGMM->fLegacyAllocationMode && !(pChunk->fFlags & GMM_CHUNK_FLAGS_LARGE_PAGE))
+    {
+        if (pChunk->hGVM != pGVM->hSelf)
+        {
+            Log(("gmmR0MapChunk: chunk %#x is already mapped at %p!\n", pChunk->Core.Key, *ppvR3));
+            return VERR_GMM_CHUNK_NOT_FOUND;
+        }
+
+        *ppvR3 = RTR0MemObjAddressR3(pChunk->hMemObj);
+        return VINF_SUCCESS;
+    }
+#else
+    RT_NOREF(pGMM);
+#endif
+
+    /*
+     * Check to see if the chunk is already mapped.
+     */
+    for (uint32_t i = 0; i < pChunk->cMappingsX; i++)
+    {
+        Assert(pChunk->paMappingsX[i].pGVM && pChunk->paMappingsX[i].hMapObj != NIL_RTR0MEMOBJ);
+        if (pChunk->paMappingsX[i].pGVM == pGVM)
+        {
+            *ppvR3 = RTR0MemObjAddressR3(pChunk->paMappingsX[i].hMapObj);
+            Log(("gmmR0MapChunk: chunk %#x is already mapped at %p!\n", pChunk->Core.Key, *ppvR3));
+#ifdef VBOX_WITH_PAGE_SHARING
+            /* The ring-3 chunk cache can be out of sync; don't fail. */
+            return VINF_SUCCESS;
+#else
+            return VERR_GMM_CHUNK_ALREADY_MAPPED;
+#endif
+        }
+    }
+
+    /*
+     * Do the mapping.
+     */
+    RTR0MEMOBJ hMapObj;
+    int rc = RTR0MemObjMapUser(&hMapObj, pChunk->hMemObj, (RTR3PTR)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
+    if (RT_SUCCESS(rc))
+    {
+        /* reallocate the array? assumes few users per chunk (usually one). */
+        unsigned iMapping = pChunk->cMappingsX;
+        if (   iMapping <= 3
+            || (iMapping & 3) == 0)
+        {
+            unsigned cNewSize = iMapping <= 3
+                              ? iMapping + 1
+                              : iMapping + 4;
+            Assert(cNewSize < 4 || RT_ALIGN_32(cNewSize, 4) == cNewSize);
+            if (RT_UNLIKELY(cNewSize > UINT16_MAX))
+            {
+                rc = RTR0MemObjFree(hMapObj, false /* fFreeMappings (NA) */); AssertRC(rc);
+                return VERR_GMM_TOO_MANY_CHUNK_MAPPINGS;
+            }
+
+            void *pvMappings = RTMemRealloc(pChunk->paMappingsX, cNewSize * sizeof(pChunk->paMappingsX[0]));
+            if (RT_UNLIKELY(!pvMappings))
+            {
+                rc = RTR0MemObjFree(hMapObj, false /* fFreeMappings (NA) */); AssertRC(rc);
+                return VERR_NO_MEMORY;
+            }
+            pChunk->paMappingsX = (PGMMCHUNKMAP)pvMappings;
+        }
+
+        /* insert new entry */
+        pChunk->paMappingsX[iMapping].hMapObj = hMapObj;
+        pChunk->paMappingsX[iMapping].pGVM    = pGVM;
+        Assert(pChunk->cMappingsX == iMapping);
+        pChunk->cMappingsX = iMapping + 1;
+
+        *ppvR3 = RTR0MemObjAddressR3(hMapObj);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Maps a chunk into the user address space of the current process.
+ *
+ * @returns VBox status code.
+ * @param   pGMM        Pointer to the GMM instance data.
+ * @param   pGVM        Pointer to the Global VM structure.
+ * @param   pChunk      Pointer to the chunk to be mapped.
+ * @param   fRelaxedSem Whether we can release the semaphore while doing the
+ *                      mapping (@c true) or not.
+ * @param   ppvR3       Where to store the ring-3 address of the mapping.
+ *                      In the VERR_GMM_CHUNK_ALREADY_MAPPED case, this will be
+ *                      contain the address of the existing mapping.
+ */
+static int gmmR0MapChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem, PRTR3PTR ppvR3)
+{
+    /*
+     * Take the chunk lock and leave the giant GMM lock when possible, then
+     * call the worker function.
+     */
+    GMMR0CHUNKMTXSTATE MtxState;
+    int rc = gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk,
+                                    fRelaxedSem ? GMMR0CHUNK_MTX_RETAKE_GIANT : GMMR0CHUNK_MTX_KEEP_GIANT);
+    if (RT_SUCCESS(rc))
+    {
+        rc = gmmR0MapChunkLocked(pGMM, pGVM, pChunk, ppvR3);
+        gmmR0ChunkMutexRelease(&MtxState, pChunk);
+    }
+
+    return rc;
+}
+
+
+
+#if defined(VBOX_WITH_PAGE_SHARING) || (defined(VBOX_STRICT) && HC_ARCH_BITS == 64)
+/**
+ * Check if a chunk is mapped into the specified VM
+ *
+ * @returns mapped yes/no
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   pGVM        Pointer to the Global VM structure.
+ * @param   pChunk      Pointer to the chunk to be mapped.
+ * @param   ppvR3       Where to store the ring-3 address of the mapping.
+ */
+static bool gmmR0IsChunkMapped(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, PRTR3PTR ppvR3)
+{
+    GMMR0CHUNKMTXSTATE MtxState;
+    gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT);
+    for (uint32_t i = 0; i < pChunk->cMappingsX; i++)
+    {
+        Assert(pChunk->paMappingsX[i].pGVM && pChunk->paMappingsX[i].hMapObj != NIL_RTR0MEMOBJ);
+        if (pChunk->paMappingsX[i].pGVM == pGVM)
+        {
+            *ppvR3 = RTR0MemObjAddressR3(pChunk->paMappingsX[i].hMapObj);
+            gmmR0ChunkMutexRelease(&MtxState, pChunk);
+            return true;
+        }
+    }
+    *ppvR3 = NULL;
+    gmmR0ChunkMutexRelease(&MtxState, pChunk);
+    return false;
+}
+#endif /* VBOX_WITH_PAGE_SHARING || (VBOX_STRICT && 64-BIT) */
+
+
+/**
+ * Map a chunk and/or unmap another chunk.
+ *
+ * The mapping and unmapping applies to the current process.
+ *
+ * This API does two things because it saves a kernel call per mapping when
+ * when the ring-3 mapping cache is full.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   idChunkMap      The chunk to map. NIL_GMM_CHUNKID if nothing to map.
+ * @param   idChunkUnmap    The chunk to unmap. NIL_GMM_CHUNKID if nothing to unmap.
+ * @param   ppvR3           Where to store the address of the mapped chunk. NULL is ok if nothing to map.
+ * @thread  EMT ???
+ */
+GMMR0DECL(int) GMMR0MapUnmapChunk(PGVM pGVM, uint32_t idChunkMap, uint32_t idChunkUnmap, PRTR3PTR ppvR3)
+{
+    LogFlow(("GMMR0MapUnmapChunk: pGVM=%p idChunkMap=%#x idChunkUnmap=%#x ppvR3=%p\n",
+             pGVM, idChunkMap, idChunkUnmap, ppvR3));
+
+    /*
+     * Validate input and get the basics.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVM(pGVM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    AssertCompile(NIL_GMM_CHUNKID == 0);
+    AssertMsgReturn(idChunkMap <= GMM_CHUNKID_LAST, ("%#x\n", idChunkMap), VERR_INVALID_PARAMETER);
+    AssertMsgReturn(idChunkUnmap <= GMM_CHUNKID_LAST, ("%#x\n", idChunkUnmap), VERR_INVALID_PARAMETER);
+
+    if (    idChunkMap == NIL_GMM_CHUNKID
+        &&  idChunkUnmap == NIL_GMM_CHUNKID)
+        return VERR_INVALID_PARAMETER;
+
+    if (idChunkMap != NIL_GMM_CHUNKID)
+    {
+        AssertPtrReturn(ppvR3, VERR_INVALID_POINTER);
+        *ppvR3 = NIL_RTR3PTR;
+    }
+
+    /*
+     * Take the semaphore and do the work.
+     *
+     * The unmapping is done last since it's easier to undo a mapping than
+     * undoing an unmapping. The ring-3 mapping cache cannot not be so big
+     * that it pushes the user virtual address space to within a chunk of
+     * it it's limits, so, no problem here.
+     */
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        PGMMCHUNK pMap = NULL;
+        if (idChunkMap != NIL_GVM_HANDLE)
+        {
+            pMap = gmmR0GetChunk(pGMM, idChunkMap);
+            if (RT_LIKELY(pMap))
+                rc = gmmR0MapChunk(pGMM, pGVM, pMap, true /*fRelaxedSem*/, ppvR3);
+            else
+            {
+                Log(("GMMR0MapUnmapChunk: idChunkMap=%#x\n", idChunkMap));
+                rc = VERR_GMM_CHUNK_NOT_FOUND;
+            }
+        }
+/** @todo split this operation, the bail out might (theoretcially) not be
+ *        entirely safe. */
+
+        if (    idChunkUnmap != NIL_GMM_CHUNKID
+            &&  RT_SUCCESS(rc))
+        {
+            PGMMCHUNK pUnmap = gmmR0GetChunk(pGMM, idChunkUnmap);
+            if (RT_LIKELY(pUnmap))
+                rc = gmmR0UnmapChunk(pGMM, pGVM, pUnmap, true /*fRelaxedSem*/);
+            else
+            {
+                Log(("GMMR0MapUnmapChunk: idChunkUnmap=%#x\n", idChunkUnmap));
+                rc = VERR_GMM_CHUNK_NOT_FOUND;
+            }
+
+            if (RT_FAILURE(rc) && pMap)
+                gmmR0UnmapChunk(pGMM, pGVM, pMap, false /*fRelaxedSem*/);
+        }
+
+        GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+    gmmR0MutexRelease(pGMM);
+
+    LogFlow(("GMMR0MapUnmapChunk: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VMMR0 request wrapper for GMMR0MapUnmapChunk.
+ *
+ * @returns see GMMR0MapUnmapChunk.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   pReq        Pointer to the request packet.
+ */
+GMMR0DECL(int)  GMMR0MapUnmapChunkReq(PGVM pGVM, PGMMMAPUNMAPCHUNKREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
+
+    return GMMR0MapUnmapChunk(pGVM, pReq->idChunkMap, pReq->idChunkUnmap, &pReq->pvR3);
+}
+
+
+/**
+ * Legacy mode API for supplying pages.
+ *
+ * The specified user address points to a allocation chunk sized block that
+ * will be locked down and used by the GMM when the GM asks for pages.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The VCPU id.
+ * @param   pvR3        Pointer to the chunk size memory block to lock down.
+ */
+GMMR0DECL(int) GMMR0SeedChunk(PGVM pGVM, VMCPUID idCpu, RTR3PTR pvR3)
+{
+#ifdef GMM_WITH_LEGACY_MODE
+    /*
+     * Validate input and get the basics.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    AssertPtrReturn(pvR3, VERR_INVALID_POINTER);
+    AssertReturn(!(PAGE_OFFSET_MASK & pvR3), VERR_INVALID_POINTER);
+
+    if (!pGMM->fLegacyAllocationMode)
+    {
+        Log(("GMMR0SeedChunk: not in legacy allocation mode!\n"));
+        return VERR_NOT_SUPPORTED;
+    }
+
+    /*
+     * Lock the memory and add it as new chunk with our hGVM.
+     * (The GMM locking is done inside gmmR0RegisterChunk.)
+     */
+    RTR0MEMOBJ hMemObj;
+    rc = RTR0MemObjLockUser(&hMemObj, pvR3, GMM_CHUNK_SIZE, RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
+    if (RT_SUCCESS(rc))
+    {
+        rc = gmmR0RegisterChunk(pGMM, &pGVM->gmm.s.Private, hMemObj, pGVM->hSelf, GMM_CHUNK_FLAGS_SEEDED, NULL);
+        if (RT_SUCCESS(rc))
+            gmmR0MutexRelease(pGMM);
+        else
+            RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
+    }
+
+    LogFlow(("GMMR0SeedChunk: rc=%d (pvR3=%p)\n", rc, pvR3));
+    return rc;
+#else
+    RT_NOREF(pGVM, idCpu, pvR3);
+    return VERR_NOT_SUPPORTED;
+#endif
+}
+
+#if defined(VBOX_WITH_RAM_IN_KERNEL) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+
+/**
+ * Gets the ring-0 virtual address for the given page.
+ *
+ * This is used by PGM when IEM and such wants to access guest RAM from ring-0.
+ * One of the ASSUMPTIONS here is that the @a idPage is used by the VM and the
+ * corresponding chunk will remain valid beyond the call (at least till the EMT
+ * returns to ring-3).
+ *
+ * @returns VBox status code.
+ * @param   pGVM        Pointer to the kernel-only VM instace data.
+ * @param   idPage      The page ID.
+ * @param   ppv         Where to store the address.
+ * @thread  EMT
+ */
+GMMR0DECL(int)  GMMR0PageIdToVirt(PGVM pGVM, uint32_t idPage, void **ppv)
+{
+    *ppv = NULL;
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+
+    uint32_t const idChunk = idPage >> GMM_CHUNKID_SHIFT;
+
+    /*
+     * Start with the per-VM TLB.
+     */
+    RTSpinlockAcquire(pGVM->gmm.s.hChunkTlbSpinLock);
+
+    PGMMPERVMCHUNKTLBE pTlbe = &pGVM->gmm.s.aChunkTlbEntries[GMMPERVM_CHUNKTLB_IDX(idChunk)];
+    PGMMCHUNK pChunk = pTlbe->pChunk;
+    if (   pChunk              != NULL
+        && pTlbe->idGeneration == ASMAtomicUoReadU64(&pGMM->idFreeGeneration)
+        && pChunk->Core.Key    == idChunk)
+        pGVM->R0Stats.gmm.cChunkTlbHits++; /* hopefully this is a likely outcome */
+    else
+    {
+        pGVM->R0Stats.gmm.cChunkTlbMisses++;
+
+        /*
+         * Look it up in the chunk tree.
+         */
+        RTSpinlockAcquire(pGMM->hSpinLockTree);
+        pChunk = gmmR0GetChunkLocked(pGMM, idChunk);
+        if (RT_LIKELY(pChunk))
+        {
+            pTlbe->idGeneration = pGMM->idFreeGeneration;
+            RTSpinlockRelease(pGMM->hSpinLockTree);
+            pTlbe->pChunk       = pChunk;
+        }
+        else
+        {
+            RTSpinlockRelease(pGMM->hSpinLockTree);
+            RTSpinlockRelease(pGVM->gmm.s.hChunkTlbSpinLock);
+            AssertMsgFailed(("idPage=%#x\n", idPage));
+            return VERR_GMM_PAGE_NOT_FOUND;
+        }
+    }
+
+    RTSpinlockRelease(pGVM->gmm.s.hChunkTlbSpinLock);
+
+    /*
+     * Got a chunk, now validate the page ownership and calcuate it's address.
+     */
+    const GMMPAGE * const pPage = &pChunk->aPages[idPage & GMM_PAGEID_IDX_MASK];
+    if (RT_LIKELY(   (   GMM_PAGE_IS_PRIVATE(pPage)
+                      && pPage->Private.hGVM == pGVM->hSelf)
+                  || GMM_PAGE_IS_SHARED(pPage)))
+    {
+        AssertPtr(pChunk->pbMapping);
+        *ppv = &pChunk->pbMapping[(idPage & GMM_PAGEID_IDX_MASK) << PAGE_SHIFT];
+        return VINF_SUCCESS;
+    }
+    AssertMsgFailed(("idPage=%#x is-private=%RTbool Private.hGVM=%u pGVM->hGVM=%u\n",
+                     idPage, GMM_PAGE_IS_PRIVATE(pPage), pPage->Private.hGVM, pGVM->hSelf));
+    return VERR_GMM_NOT_PAGE_OWNER;
+}
+
+#endif
+
+#ifdef VBOX_WITH_PAGE_SHARING
+
+# ifdef VBOX_STRICT
+/**
+ * For checksumming shared pages in strict builds.
+ *
+ * The purpose is making sure that a page doesn't change.
+ *
+ * @returns Checksum, 0 on failure.
+ * @param   pGMM        The GMM instance data.
+ * @param   pGVM        Pointer to the kernel-only VM instace data.
+ * @param   idPage      The page ID.
+ */
+static uint32_t gmmR0StrictPageChecksum(PGMM pGMM, PGVM pGVM, uint32_t idPage)
+{
+    PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
+    AssertMsgReturn(pChunk, ("idPage=%#x\n", idPage), 0);
+
+    uint8_t *pbChunk;
+    if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
+        return 0;
+    uint8_t const *pbPage = pbChunk + ((idPage & GMM_PAGEID_IDX_MASK) << PAGE_SHIFT);
+
+    return RTCrc32(pbPage, PAGE_SIZE);
+}
+# endif /* VBOX_STRICT */
+
+
+/**
+ * Calculates the module hash value.
+ *
+ * @returns Hash value.
+ * @param   pszModuleName   The module name.
+ * @param   pszVersion      The module version string.
+ */
+static uint32_t gmmR0ShModCalcHash(const char *pszModuleName, const char *pszVersion)
+{
+    return RTStrHash1ExN(3, pszModuleName, RTSTR_MAX, "::", (size_t)2, pszVersion, RTSTR_MAX);
+}
+
+
+/**
+ * Finds a global module.
+ *
+ * @returns Pointer to the global module on success, NULL if not found.
+ * @param   pGMM            The GMM instance data.
+ * @param   uHash           The hash as calculated by gmmR0ShModCalcHash.
+ * @param   cbModule        The module size.
+ * @param   enmGuestOS      The guest OS type.
+ * @param   cRegions        The number of regions.
+ * @param   pszModuleName   The module name.
+ * @param   pszVersion      The module version.
+ * @param   paRegions       The region descriptions.
+ */
+static PGMMSHAREDMODULE gmmR0ShModFindGlobal(PGMM pGMM, uint32_t uHash, uint32_t cbModule, VBOXOSFAMILY enmGuestOS,
+                                             uint32_t cRegions, const char *pszModuleName, const char *pszVersion,
+                                             struct VMMDEVSHAREDREGIONDESC const *paRegions)
+{
+    for (PGMMSHAREDMODULE pGblMod = (PGMMSHAREDMODULE)RTAvllU32Get(&pGMM->pGlobalSharedModuleTree, uHash);
+         pGblMod;
+         pGblMod = (PGMMSHAREDMODULE)pGblMod->Core.pList)
+    {
+        if (pGblMod->cbModule   != cbModule)
+            continue;
+        if (pGblMod->enmGuestOS != enmGuestOS)
+            continue;
+        if (pGblMod->cRegions   != cRegions)
+            continue;
+        if (strcmp(pGblMod->szName, pszModuleName))
+            continue;
+        if (strcmp(pGblMod->szVersion, pszVersion))
+            continue;
+
+        uint32_t i;
+        for (i = 0; i < cRegions; i++)
+        {
+            uint32_t off = paRegions[i].GCRegionAddr & PAGE_OFFSET_MASK;
+            if (pGblMod->aRegions[i].off != off)
+                break;
+
+            uint32_t cb  = RT_ALIGN_32(paRegions[i].cbRegion + off, PAGE_SIZE);
+            if (pGblMod->aRegions[i].cb != cb)
+                break;
+        }
+
+        if (i == cRegions)
+            return pGblMod;
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Creates a new global module.
+ *
+ * @returns VBox status code.
+ * @param   pGMM            The GMM instance data.
+ * @param   uHash           The hash as calculated by gmmR0ShModCalcHash.
+ * @param   cbModule        The module size.
+ * @param   enmGuestOS      The guest OS type.
+ * @param   cRegions        The number of regions.
+ * @param   pszModuleName   The module name.
+ * @param   pszVersion      The module version.
+ * @param   paRegions       The region descriptions.
+ * @param   ppGblMod        Where to return the new module on success.
+ */
+static int gmmR0ShModNewGlobal(PGMM pGMM, uint32_t uHash, uint32_t cbModule, VBOXOSFAMILY enmGuestOS,
+                               uint32_t cRegions, const char *pszModuleName, const char *pszVersion,
+                               struct VMMDEVSHAREDREGIONDESC const *paRegions, PGMMSHAREDMODULE *ppGblMod)
+{
+    Log(("gmmR0ShModNewGlobal: %s %s size %#x os %u rgn %u\n", pszModuleName, pszVersion, cbModule, enmGuestOS, cRegions));
+    if (pGMM->cShareableModules >= GMM_MAX_SHARED_GLOBAL_MODULES)
+    {
+        Log(("gmmR0ShModNewGlobal: Too many modules\n"));
+        return VERR_GMM_TOO_MANY_GLOBAL_MODULES;
+    }
+
+    PGMMSHAREDMODULE pGblMod = (PGMMSHAREDMODULE)RTMemAllocZ(RT_UOFFSETOF_DYN(GMMSHAREDMODULE, aRegions[cRegions]));
+    if (!pGblMod)
+    {
+        Log(("gmmR0ShModNewGlobal: No memory\n"));
+        return VERR_NO_MEMORY;
+    }
+
+    pGblMod->Core.Key   = uHash;
+    pGblMod->cbModule   = cbModule;
+    pGblMod->cRegions   = cRegions;
+    pGblMod->cUsers     = 1;
+    pGblMod->enmGuestOS = enmGuestOS;
+    strcpy(pGblMod->szName, pszModuleName);
+    strcpy(pGblMod->szVersion, pszVersion);
+
+    for (uint32_t i = 0; i < cRegions; i++)
+    {
+        Log(("gmmR0ShModNewGlobal: rgn[%u]=%RGvLB%#x\n", i, paRegions[i].GCRegionAddr, paRegions[i].cbRegion));
+        pGblMod->aRegions[i].off        = paRegions[i].GCRegionAddr & PAGE_OFFSET_MASK;
+        pGblMod->aRegions[i].cb         = paRegions[i].cbRegion + pGblMod->aRegions[i].off;
+        pGblMod->aRegions[i].cb         = RT_ALIGN_32(pGblMod->aRegions[i].cb, PAGE_SIZE);
+        pGblMod->aRegions[i].paidPages  = NULL; /* allocated when needed. */
+    }
+
+    bool fInsert = RTAvllU32Insert(&pGMM->pGlobalSharedModuleTree, &pGblMod->Core);
+    Assert(fInsert); NOREF(fInsert);
+    pGMM->cShareableModules++;
+
+    *ppGblMod = pGblMod;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deletes a global module which is no longer referenced by anyone.
+ *
+ * @param   pGMM                The GMM instance data.
+ * @param   pGblMod             The module to delete.
+ */
+static void gmmR0ShModDeleteGlobal(PGMM pGMM, PGMMSHAREDMODULE pGblMod)
+{
+    Assert(pGblMod->cUsers == 0);
+    Assert(pGMM->cShareableModules > 0 && pGMM->cShareableModules <= GMM_MAX_SHARED_GLOBAL_MODULES);
+
+    void *pvTest = RTAvllU32RemoveNode(&pGMM->pGlobalSharedModuleTree, &pGblMod->Core);
+    Assert(pvTest == pGblMod); NOREF(pvTest);
+    pGMM->cShareableModules--;
+
+    uint32_t i = pGblMod->cRegions;
+    while (i-- > 0)
+    {
+        if (pGblMod->aRegions[i].paidPages)
+        {
+            /* We don't doing anything to the pages as they are handled by the
+               copy-on-write mechanism in PGM. */
+            RTMemFree(pGblMod->aRegions[i].paidPages);
+            pGblMod->aRegions[i].paidPages = NULL;
+        }
+    }
+    RTMemFree(pGblMod);
+}
+
+
+static int gmmR0ShModNewPerVM(PGVM pGVM, RTGCPTR GCBaseAddr, uint32_t cRegions, const VMMDEVSHAREDREGIONDESC *paRegions,
+                              PGMMSHAREDMODULEPERVM *ppRecVM)
+{
+    if (pGVM->gmm.s.Stats.cShareableModules >= GMM_MAX_SHARED_PER_VM_MODULES)
+        return VERR_GMM_TOO_MANY_PER_VM_MODULES;
+
+    PGMMSHAREDMODULEPERVM pRecVM;
+    pRecVM = (PGMMSHAREDMODULEPERVM)RTMemAllocZ(RT_UOFFSETOF_DYN(GMMSHAREDMODULEPERVM, aRegionsGCPtrs[cRegions]));
+    if (!pRecVM)
+        return VERR_NO_MEMORY;
+
+    pRecVM->Core.Key = GCBaseAddr;
+    for (uint32_t i = 0; i < cRegions; i++)
+        pRecVM->aRegionsGCPtrs[i] = paRegions[i].GCRegionAddr;
+
+    bool fInsert = RTAvlGCPtrInsert(&pGVM->gmm.s.pSharedModuleTree, &pRecVM->Core);
+    Assert(fInsert); NOREF(fInsert);
+    pGVM->gmm.s.Stats.cShareableModules++;
+
+    *ppRecVM = pRecVM;
+    return VINF_SUCCESS;
+}
+
+
+static void gmmR0ShModDeletePerVM(PGMM pGMM, PGVM pGVM, PGMMSHAREDMODULEPERVM pRecVM, bool fRemove)
+{
+    /*
+     * Free the per-VM module.
+     */
+    PGMMSHAREDMODULE pGblMod = pRecVM->pGlobalModule;
+    pRecVM->pGlobalModule    = NULL;
+
+    if (fRemove)
+    {
+        void *pvTest = RTAvlGCPtrRemove(&pGVM->gmm.s.pSharedModuleTree, pRecVM->Core.Key);
+        Assert(pvTest == &pRecVM->Core); NOREF(pvTest);
+    }
+
+    RTMemFree(pRecVM);
+
+    /*
+     * Release the global module.
+     * (In the registration bailout case, it might not be.)
+     */
+    if (pGblMod)
+    {
+        Assert(pGblMod->cUsers > 0);
+        pGblMod->cUsers--;
+        if (pGblMod->cUsers == 0)
+            gmmR0ShModDeleteGlobal(pGMM, pGblMod);
+    }
+}
+
+#endif /* VBOX_WITH_PAGE_SHARING */
+
+/**
+ * Registers a new shared module for the VM.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   idCpu           The VCPU id.
+ * @param   enmGuestOS      The guest OS type.
+ * @param   pszModuleName   The module name.
+ * @param   pszVersion      The module version.
+ * @param   GCPtrModBase    The module base address.
+ * @param   cbModule        The module size.
+ * @param   cRegions        The mumber of shared region descriptors.
+ * @param   paRegions       Pointer to an array of shared region(s).
+ * @thread  EMT(idCpu)
+ */
+GMMR0DECL(int) GMMR0RegisterSharedModule(PGVM pGVM, VMCPUID idCpu, VBOXOSFAMILY enmGuestOS, char *pszModuleName,
+                                         char *pszVersion, RTGCPTR GCPtrModBase, uint32_t cbModule,
+                                         uint32_t cRegions, struct VMMDEVSHAREDREGIONDESC const *paRegions)
+{
+#ifdef VBOX_WITH_PAGE_SHARING
+    /*
+     * Validate input and get the basics.
+     *
+     * Note! Turns out the module size does necessarily match the size of the
+     *       regions. (iTunes on XP)
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    if (RT_UNLIKELY(cRegions > VMMDEVSHAREDREGIONDESC_MAX))
+        return VERR_GMM_TOO_MANY_REGIONS;
+
+    if (RT_UNLIKELY(cbModule == 0 || cbModule > _1G))
+        return VERR_GMM_BAD_SHARED_MODULE_SIZE;
+
+    uint32_t cbTotal = 0;
+    for (uint32_t i = 0; i < cRegions; i++)
+    {
+        if (RT_UNLIKELY(paRegions[i].cbRegion == 0 || paRegions[i].cbRegion > _1G))
+            return VERR_GMM_SHARED_MODULE_BAD_REGIONS_SIZE;
+
+        cbTotal += paRegions[i].cbRegion;
+        if (RT_UNLIKELY(cbTotal > _1G))
+            return VERR_GMM_SHARED_MODULE_BAD_REGIONS_SIZE;
+    }
+
+    AssertPtrReturn(pszModuleName, VERR_INVALID_POINTER);
+    if (RT_UNLIKELY(!memchr(pszModuleName, '\0', GMM_SHARED_MODULE_MAX_NAME_STRING)))
+        return VERR_GMM_MODULE_NAME_TOO_LONG;
+
+    AssertPtrReturn(pszVersion, VERR_INVALID_POINTER);
+    if (RT_UNLIKELY(!memchr(pszVersion, '\0', GMM_SHARED_MODULE_MAX_VERSION_STRING)))
+        return VERR_GMM_MODULE_NAME_TOO_LONG;
+
+    uint32_t const uHash = gmmR0ShModCalcHash(pszModuleName, pszVersion);
+    Log(("GMMR0RegisterSharedModule %s %s base %RGv size %x hash %x\n", pszModuleName, pszVersion, GCPtrModBase, cbModule, uHash));
+
+    /*
+     * Take the semaphore and do some more validations.
+     */
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        /*
+         * Check if this module is already locally registered and register
+         * it if it isn't.  The base address is a unique module identifier
+         * locally.
+         */
+        PGMMSHAREDMODULEPERVM pRecVM = (PGMMSHAREDMODULEPERVM)RTAvlGCPtrGet(&pGVM->gmm.s.pSharedModuleTree, GCPtrModBase);
+        bool fNewModule = pRecVM == NULL;
+        if (fNewModule)
+        {
+            rc = gmmR0ShModNewPerVM(pGVM, GCPtrModBase, cRegions, paRegions, &pRecVM);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Find a matching global module, register a new one if needed.
+                 */
+                PGMMSHAREDMODULE pGblMod = gmmR0ShModFindGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions,
+                                                                pszModuleName, pszVersion, paRegions);
+                if (!pGblMod)
+                {
+                    Assert(fNewModule);
+                    rc = gmmR0ShModNewGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions,
+                                             pszModuleName, pszVersion, paRegions, &pGblMod);
+                    if (RT_SUCCESS(rc))
+                    {
+                        pRecVM->pGlobalModule = pGblMod; /* (One referenced returned by gmmR0ShModNewGlobal.) */
+                        Log(("GMMR0RegisterSharedModule: new module %s %s\n", pszModuleName, pszVersion));
+                    }
+                    else
+                        gmmR0ShModDeletePerVM(pGMM, pGVM, pRecVM, true /*fRemove*/);
+                }
+                else
+                {
+                    Assert(pGblMod->cUsers > 0 && pGblMod->cUsers < UINT32_MAX / 2);
+                    pGblMod->cUsers++;
+                    pRecVM->pGlobalModule = pGblMod;
+
+                    Log(("GMMR0RegisterSharedModule: new per vm module %s %s, gbl users %d\n", pszModuleName, pszVersion, pGblMod->cUsers));
+                }
+            }
+        }
+        else
+        {
+            /*
+             * Attempt to re-register an existing module.
+             */
+            PGMMSHAREDMODULE pGblMod = gmmR0ShModFindGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions,
+                                                            pszModuleName, pszVersion, paRegions);
+            if (pRecVM->pGlobalModule == pGblMod)
+            {
+                Log(("GMMR0RegisterSharedModule: already registered %s %s, gbl users %d\n", pszModuleName, pszVersion, pGblMod->cUsers));
+                rc = VINF_GMM_SHARED_MODULE_ALREADY_REGISTERED;
+            }
+            else
+            {
+                /** @todo may have to unregister+register when this happens in case it's caused
+                 * by VBoxService crashing and being restarted... */
+                Log(("GMMR0RegisterSharedModule: Address clash!\n"
+                     "  incoming at %RGvLB%#x %s %s rgns %u\n"
+                     "  existing at %RGvLB%#x %s %s rgns %u\n",
+                     GCPtrModBase, cbModule, pszModuleName, pszVersion, cRegions,
+                     pRecVM->Core.Key, pRecVM->pGlobalModule->cbModule, pRecVM->pGlobalModule->szName,
+                     pRecVM->pGlobalModule->szVersion, pRecVM->pGlobalModule->cRegions));
+                rc = VERR_GMM_SHARED_MODULE_ADDRESS_CLASH;
+            }
+        }
+        GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+
+    gmmR0MutexRelease(pGMM);
+    return rc;
+#else
+
+    NOREF(pGVM); NOREF(idCpu); NOREF(enmGuestOS); NOREF(pszModuleName); NOREF(pszVersion);
+    NOREF(GCPtrModBase); NOREF(cbModule); NOREF(cRegions); NOREF(paRegions);
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/**
+ * VMMR0 request wrapper for GMMR0RegisterSharedModule.
+ *
+ * @returns see GMMR0RegisterSharedModule.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The VCPU id.
+ * @param   pReq        Pointer to the request packet.
+ */
+GMMR0DECL(int) GMMR0RegisterSharedModuleReq(PGVM pGVM, VMCPUID idCpu, PGMMREGISTERSHAREDMODULEREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(   pReq->Hdr.cbReq >= sizeof(*pReq)
+                    && pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(GMMREGISTERSHAREDMODULEREQ, aRegions[pReq->cRegions]),
+                    ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
+
+    /* Pass back return code in the request packet to preserve informational codes. (VMMR3CallR0 chokes on them) */
+    pReq->rc = GMMR0RegisterSharedModule(pGVM, idCpu, pReq->enmGuestOS, pReq->szName, pReq->szVersion,
+                                         pReq->GCBaseAddr, pReq->cbModule, pReq->cRegions, pReq->aRegions);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Unregisters a shared module for the VM
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   idCpu           The VCPU id.
+ * @param   pszModuleName   The module name.
+ * @param   pszVersion      The module version.
+ * @param   GCPtrModBase    The module base address.
+ * @param   cbModule        The module size.
+ */
+GMMR0DECL(int) GMMR0UnregisterSharedModule(PGVM pGVM, VMCPUID idCpu, char *pszModuleName, char *pszVersion,
+                                           RTGCPTR GCPtrModBase, uint32_t cbModule)
+{
+#ifdef VBOX_WITH_PAGE_SHARING
+    /*
+     * Validate input and get the basics.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    AssertPtrReturn(pszModuleName, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszVersion, VERR_INVALID_POINTER);
+    if (RT_UNLIKELY(!memchr(pszModuleName, '\0', GMM_SHARED_MODULE_MAX_NAME_STRING)))
+        return VERR_GMM_MODULE_NAME_TOO_LONG;
+    if (RT_UNLIKELY(!memchr(pszVersion, '\0', GMM_SHARED_MODULE_MAX_VERSION_STRING)))
+        return VERR_GMM_MODULE_NAME_TOO_LONG;
+
+    Log(("GMMR0UnregisterSharedModule %s %s base=%RGv size %x\n", pszModuleName, pszVersion, GCPtrModBase, cbModule));
+
+    /*
+     * Take the semaphore and do some more validations.
+     */
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        /*
+         * Locate and remove the specified module.
+         */
+        PGMMSHAREDMODULEPERVM pRecVM = (PGMMSHAREDMODULEPERVM)RTAvlGCPtrGet(&pGVM->gmm.s.pSharedModuleTree, GCPtrModBase);
+        if (pRecVM)
+        {
+            /** @todo Do we need to do more validations here, like that the
+             *        name + version + cbModule matches? */
+            NOREF(cbModule);
+            Assert(pRecVM->pGlobalModule);
+            gmmR0ShModDeletePerVM(pGMM, pGVM, pRecVM, true /*fRemove*/);
+        }
+        else
+            rc = VERR_GMM_SHARED_MODULE_NOT_FOUND;
+
+        GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+
+    gmmR0MutexRelease(pGMM);
+    return rc;
+#else
+
+    NOREF(pGVM); NOREF(idCpu); NOREF(pszModuleName); NOREF(pszVersion); NOREF(GCPtrModBase); NOREF(cbModule);
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/**
+ * VMMR0 request wrapper for GMMR0UnregisterSharedModule.
+ *
+ * @returns see GMMR0UnregisterSharedModule.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The VCPU id.
+ * @param   pReq        Pointer to the request packet.
+ */
+GMMR0DECL(int)  GMMR0UnregisterSharedModuleReq(PGVM pGVM, VMCPUID idCpu, PGMMUNREGISTERSHAREDMODULEREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
+
+    return GMMR0UnregisterSharedModule(pGVM, idCpu, pReq->szName, pReq->szVersion, pReq->GCBaseAddr, pReq->cbModule);
+}
+
+#ifdef VBOX_WITH_PAGE_SHARING
+
+/**
+ * Increase the use count of a shared page, the page is known to exist and be valid and such.
+ *
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   pGVM        Pointer to the GVM instance.
+ * @param   pPage       The page structure.
+ */
+DECLINLINE(void) gmmR0UseSharedPage(PGMM pGMM, PGVM pGVM, PGMMPAGE pPage)
+{
+    Assert(pGMM->cSharedPages > 0);
+    Assert(pGMM->cAllocatedPages > 0);
+
+    pGMM->cDuplicatePages++;
+
+    pPage->Shared.cRefs++;
+    pGVM->gmm.s.Stats.cSharedPages++;
+    pGVM->gmm.s.Stats.Allocated.cBasePages++;
+}
+
+
+/**
+ * Converts a private page to a shared page, the page is known to exist and be valid and such.
+ *
+ * @param   pGMM        Pointer to the GMM instance.
+ * @param   pGVM        Pointer to the GVM instance.
+ * @param   HCPhys      Host physical address
+ * @param   idPage      The Page ID
+ * @param   pPage       The page structure.
+ * @param   pPageDesc   Shared page descriptor
+ */
+DECLINLINE(void) gmmR0ConvertToSharedPage(PGMM pGMM, PGVM pGVM, RTHCPHYS HCPhys, uint32_t idPage, PGMMPAGE pPage,
+                                          PGMMSHAREDPAGEDESC pPageDesc)
+{
+    PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
+    Assert(pChunk);
+    Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES);
+    Assert(GMM_PAGE_IS_PRIVATE(pPage));
+
+    pChunk->cPrivate--;
+    pChunk->cShared++;
+
+    pGMM->cSharedPages++;
+
+    pGVM->gmm.s.Stats.cSharedPages++;
+    pGVM->gmm.s.Stats.cPrivatePages--;
+
+    /* Modify the page structure. */
+    pPage->Shared.pfn         = (uint32_t)(uint64_t)(HCPhys >> PAGE_SHIFT);
+    pPage->Shared.cRefs       = 1;
+#ifdef VBOX_STRICT
+    pPageDesc->u32StrictChecksum = gmmR0StrictPageChecksum(pGMM, pGVM, idPage);
+    pPage->Shared.u14Checksum = pPageDesc->u32StrictChecksum;
+#else
+    NOREF(pPageDesc);
+    pPage->Shared.u14Checksum = 0;
+#endif
+    pPage->Shared.u2State     = GMM_PAGE_STATE_SHARED;
+}
+
+
+static int gmmR0SharedModuleCheckPageFirstTime(PGMM pGMM, PGVM pGVM, PGMMSHAREDMODULE pModule,
+                                               unsigned idxRegion, unsigned idxPage,
+                                               PGMMSHAREDPAGEDESC pPageDesc, PGMMSHAREDREGIONDESC pGlobalRegion)
+{
+    NOREF(pModule);
+
+    /* Easy case: just change the internal page type. */
+    PGMMPAGE pPage = gmmR0GetPage(pGMM, pPageDesc->idPage);
+    AssertMsgReturn(pPage, ("idPage=%#x (GCPhys=%RGp HCPhys=%RHp idxRegion=%#x idxPage=%#x) #1\n",
+                            pPageDesc->idPage, pPageDesc->GCPhys, pPageDesc->HCPhys, idxRegion, idxPage),
+                    VERR_PGM_PHYS_INVALID_PAGE_ID);
+    NOREF(idxRegion);
+
+    AssertMsg(pPageDesc->GCPhys == (pPage->Private.pfn << 12), ("desc %RGp gmm %RGp\n", pPageDesc->HCPhys, (pPage->Private.pfn << 12)));
+
+    gmmR0ConvertToSharedPage(pGMM, pGVM, pPageDesc->HCPhys, pPageDesc->idPage, pPage, pPageDesc);
+
+    /* Keep track of these references. */
+    pGlobalRegion->paidPages[idxPage] = pPageDesc->idPage;
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Checks specified shared module range for changes
+ *
+ * Performs the following tasks:
+ *  - If a shared page is new, then it changes the GMM page type to shared and
+ *    returns it in the pPageDesc descriptor.
+ *  - If a shared page already exists, then it checks if the VM page is
+ *    identical and if so frees the VM page and returns the shared page in
+ *    pPageDesc descriptor.
+ *
+ * @remarks ASSUMES the caller has acquired the GMM semaphore!!
+ *
+ * @returns VBox status code.
+ * @param   pGVM        Pointer to the GVM instance data.
+ * @param   pModule     Module description
+ * @param   idxRegion   Region index
+ * @param   idxPage     Page index
+ * @param   pPageDesc   Page descriptor
+ */
+GMMR0DECL(int) GMMR0SharedModuleCheckPage(PGVM pGVM, PGMMSHAREDMODULE pModule, uint32_t idxRegion, uint32_t idxPage,
+                                          PGMMSHAREDPAGEDESC pPageDesc)
+{
+    int     rc;
+    PGMM    pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    pPageDesc->u32StrictChecksum = 0;
+
+    AssertMsgReturn(idxRegion < pModule->cRegions,
+                    ("idxRegion=%#x cRegions=%#x %s %s\n", idxRegion, pModule->cRegions, pModule->szName, pModule->szVersion),
+                    VERR_INVALID_PARAMETER);
+
+    uint32_t const cPages = pModule->aRegions[idxRegion].cb >> PAGE_SHIFT;
+    AssertMsgReturn(idxPage < cPages,
+                    ("idxRegion=%#x cRegions=%#x %s %s\n", idxRegion, pModule->cRegions, pModule->szName, pModule->szVersion),
+                    VERR_INVALID_PARAMETER);
+
+    LogFlow(("GMMR0SharedModuleCheckRange %s base %RGv region %d idxPage %d\n", pModule->szName, pModule->Core.Key, idxRegion, idxPage));
+
+    /*
+     * First time; create a page descriptor array.
+     */
+    PGMMSHAREDREGIONDESC pGlobalRegion = &pModule->aRegions[idxRegion];
+    if (!pGlobalRegion->paidPages)
+    {
+        Log(("Allocate page descriptor array for %d pages\n", cPages));
+        pGlobalRegion->paidPages = (uint32_t *)RTMemAlloc(cPages * sizeof(pGlobalRegion->paidPages[0]));
+        AssertReturn(pGlobalRegion->paidPages, VERR_NO_MEMORY);
+
+        /* Invalidate all descriptors. */
+        uint32_t i = cPages;
+        while (i-- > 0)
+            pGlobalRegion->paidPages[i] = NIL_GMM_PAGEID;
+    }
+
+    /*
+     * We've seen this shared page for the first time?
+     */
+    if (pGlobalRegion->paidPages[idxPage] == NIL_GMM_PAGEID)
+    {
+        Log(("New shared page guest %RGp host %RHp\n", pPageDesc->GCPhys, pPageDesc->HCPhys));
+        return gmmR0SharedModuleCheckPageFirstTime(pGMM, pGVM, pModule, idxRegion, idxPage, pPageDesc, pGlobalRegion);
+    }
+
+    /*
+     * We've seen it before...
+     */
+    Log(("Replace existing page guest %RGp host %RHp id %#x -> id %#x\n",
+         pPageDesc->GCPhys, pPageDesc->HCPhys, pPageDesc->idPage, pGlobalRegion->paidPages[idxPage]));
+    Assert(pPageDesc->idPage != pGlobalRegion->paidPages[idxPage]);
+
+    /*
+     * Get the shared page source.
+     */
+    PGMMPAGE pPage = gmmR0GetPage(pGMM, pGlobalRegion->paidPages[idxPage]);
+    AssertMsgReturn(pPage, ("idPage=%#x (idxRegion=%#x idxPage=%#x) #2\n", pPageDesc->idPage, idxRegion, idxPage),
+                    VERR_PGM_PHYS_INVALID_PAGE_ID);
+
+    if (pPage->Common.u2State != GMM_PAGE_STATE_SHARED)
+    {
+        /*
+         * Page was freed at some point; invalidate this entry.
+         */
+        /** @todo this isn't really bullet proof. */
+        Log(("Old shared page was freed -> create a new one\n"));
+        pGlobalRegion->paidPages[idxPage] = NIL_GMM_PAGEID;
+        return gmmR0SharedModuleCheckPageFirstTime(pGMM, pGVM, pModule, idxRegion, idxPage, pPageDesc, pGlobalRegion);
+    }
+
+    Log(("Replace existing page guest host %RHp -> %RHp\n", pPageDesc->HCPhys, ((uint64_t)pPage->Shared.pfn) << PAGE_SHIFT));
+
+    /*
+     * Calculate the virtual address of the local page.
+     */
+    PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pPageDesc->idPage >> GMM_CHUNKID_SHIFT);
+    AssertMsgReturn(pChunk, ("idPage=%#x (idxRegion=%#x idxPage=%#x) #4\n", pPageDesc->idPage, idxRegion, idxPage),
+                    VERR_PGM_PHYS_INVALID_PAGE_ID);
+
+    uint8_t *pbChunk;
+    AssertMsgReturn(gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk),
+                    ("idPage=%#x (idxRegion=%#x idxPage=%#x) #3\n", pPageDesc->idPage, idxRegion, idxPage),
+                    VERR_PGM_PHYS_INVALID_PAGE_ID);
+    uint8_t  *pbLocalPage = pbChunk + ((pPageDesc->idPage & GMM_PAGEID_IDX_MASK) << PAGE_SHIFT);
+
+    /*
+     * Calculate the virtual address of the shared page.
+     */
+    pChunk = gmmR0GetChunk(pGMM, pGlobalRegion->paidPages[idxPage] >> GMM_CHUNKID_SHIFT);
+    Assert(pChunk); /* can't fail as gmmR0GetPage succeeded. */
+
+    /*
+     * Get the virtual address of the physical page; map the chunk into the VM
+     * process if not already done.
+     */
+    if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
+    {
+        Log(("Map chunk into process!\n"));
+        rc = gmmR0MapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/, (PRTR3PTR)&pbChunk);
+        AssertRCReturn(rc, rc);
+    }
+    uint8_t *pbSharedPage = pbChunk + ((pGlobalRegion->paidPages[idxPage] & GMM_PAGEID_IDX_MASK) << PAGE_SHIFT);
+
+#ifdef VBOX_STRICT
+    pPageDesc->u32StrictChecksum = RTCrc32(pbSharedPage, PAGE_SIZE);
+    uint32_t uChecksum = pPageDesc->u32StrictChecksum & UINT32_C(0x00003fff);
+    AssertMsg(!uChecksum || uChecksum == pPage->Shared.u14Checksum || !pPage->Shared.u14Checksum,
+              ("%#x vs %#x - idPage=%#x - %s %s\n", uChecksum, pPage->Shared.u14Checksum,
+               pGlobalRegion->paidPages[idxPage], pModule->szName, pModule->szVersion));
+#endif
+
+    /** @todo write ASMMemComparePage. */
+    if (memcmp(pbSharedPage, pbLocalPage, PAGE_SIZE))
+    {
+        Log(("Unexpected differences found between local and shared page; skip\n"));
+        /* Signal to the caller that this one hasn't changed. */
+        pPageDesc->idPage = NIL_GMM_PAGEID;
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Free the old local page.
+     */
+    GMMFREEPAGEDESC PageDesc;
+    PageDesc.idPage = pPageDesc->idPage;
+    rc = gmmR0FreePages(pGMM, pGVM, 1, &PageDesc, GMMACCOUNT_BASE);
+    AssertRCReturn(rc, rc);
+
+    gmmR0UseSharedPage(pGMM, pGVM, pPage);
+
+    /*
+     * Pass along the new physical address & page id.
+     */
+    pPageDesc->HCPhys = ((uint64_t)pPage->Shared.pfn) << PAGE_SHIFT;
+    pPageDesc->idPage = pGlobalRegion->paidPages[idxPage];
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * RTAvlGCPtrDestroy callback.
+ *
+ * @returns 0 or VERR_GMM_INSTANCE.
+ * @param   pNode       The node to destroy.
+ * @param   pvArgs      Pointer to an argument packet.
+ */
+static DECLCALLBACK(int) gmmR0CleanupSharedModule(PAVLGCPTRNODECORE pNode, void *pvArgs)
+{
+    gmmR0ShModDeletePerVM(((GMMR0SHMODPERVMDTORARGS *)pvArgs)->pGMM,
+                          ((GMMR0SHMODPERVMDTORARGS *)pvArgs)->pGVM,
+                          (PGMMSHAREDMODULEPERVM)pNode,
+                          false /*fRemove*/);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Used by GMMR0CleanupVM to clean up shared modules.
+ *
+ * This is called without taking the GMM lock so that it can be yielded as
+ * needed here.
+ *
+ * @param   pGMM        The GMM handle.
+ * @param   pGVM        The global VM handle.
+ */
+static void gmmR0SharedModuleCleanup(PGMM pGMM, PGVM pGVM)
+{
+    gmmR0MutexAcquire(pGMM);
+    GMM_CHECK_SANITY_UPON_ENTERING(pGMM);
+
+    GMMR0SHMODPERVMDTORARGS Args;
+    Args.pGVM = pGVM;
+    Args.pGMM = pGMM;
+    RTAvlGCPtrDestroy(&pGVM->gmm.s.pSharedModuleTree, gmmR0CleanupSharedModule, &Args);
+
+    AssertMsg(pGVM->gmm.s.Stats.cShareableModules == 0, ("%d\n", pGVM->gmm.s.Stats.cShareableModules));
+    pGVM->gmm.s.Stats.cShareableModules = 0;
+
+    gmmR0MutexRelease(pGMM);
+}
+
+#endif /* VBOX_WITH_PAGE_SHARING */
+
+/**
+ * Removes all shared modules for the specified VM
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The VCPU id.
+ */
+GMMR0DECL(int) GMMR0ResetSharedModules(PGVM pGVM, VMCPUID idCpu)
+{
+#ifdef VBOX_WITH_PAGE_SHARING
+    /*
+     * Validate input and get the basics.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Take the semaphore and do some more validations.
+     */
+    gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        Log(("GMMR0ResetSharedModules\n"));
+        GMMR0SHMODPERVMDTORARGS Args;
+        Args.pGVM = pGVM;
+        Args.pGMM = pGMM;
+        RTAvlGCPtrDestroy(&pGVM->gmm.s.pSharedModuleTree, gmmR0CleanupSharedModule, &Args);
+        pGVM->gmm.s.Stats.cShareableModules = 0;
+
+        rc = VINF_SUCCESS;
+        GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+
+    gmmR0MutexRelease(pGMM);
+    return rc;
+#else
+    RT_NOREF(pGVM, idCpu);
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+#ifdef VBOX_WITH_PAGE_SHARING
+
+/**
+ * Tree enumeration callback for checking a shared module.
+ */
+static DECLCALLBACK(int) gmmR0CheckSharedModule(PAVLGCPTRNODECORE pNode, void *pvUser)
+{
+    GMMCHECKSHAREDMODULEINFO   *pArgs   = (GMMCHECKSHAREDMODULEINFO*)pvUser;
+    PGMMSHAREDMODULEPERVM       pRecVM  = (PGMMSHAREDMODULEPERVM)pNode;
+    PGMMSHAREDMODULE            pGblMod = pRecVM->pGlobalModule;
+
+    Log(("gmmR0CheckSharedModule: check %s %s base=%RGv size=%x\n",
+         pGblMod->szName, pGblMod->szVersion, pGblMod->Core.Key, pGblMod->cbModule));
+
+    int rc = PGMR0SharedModuleCheck(pArgs->pGVM, pArgs->pGVM, pArgs->idCpu, pGblMod, pRecVM->aRegionsGCPtrs);
+    if (RT_FAILURE(rc))
+        return rc;
+    return VINF_SUCCESS;
+}
+
+#endif /* VBOX_WITH_PAGE_SHARING */
+
+/**
+ * Check all shared modules for the specified VM.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The calling EMT number.
+ * @thread  EMT(idCpu)
+ */
+GMMR0DECL(int) GMMR0CheckSharedModules(PGVM pGVM, VMCPUID idCpu)
+{
+#ifdef VBOX_WITH_PAGE_SHARING
+    /*
+     * Validate input and get the basics.
+     */
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+# ifndef DEBUG_sandervl
+    /*
+     * Take the semaphore and do some more validations.
+     */
+    gmmR0MutexAcquire(pGMM);
+# endif
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        /*
+         * Walk the tree, checking each module.
+         */
+        Log(("GMMR0CheckSharedModules\n"));
+
+        GMMCHECKSHAREDMODULEINFO Args;
+        Args.pGVM     = pGVM;
+        Args.idCpu    = idCpu;
+        rc = RTAvlGCPtrDoWithAll(&pGVM->gmm.s.pSharedModuleTree, true /* fFromLeft */, gmmR0CheckSharedModule, &Args);
+
+        Log(("GMMR0CheckSharedModules done (rc=%Rrc)!\n", rc));
+        GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+
+# ifndef DEBUG_sandervl
+    gmmR0MutexRelease(pGMM);
+# endif
+    return rc;
+#else
+    RT_NOREF(pGVM, idCpu);
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
+
+/**
+ * Worker for GMMR0FindDuplicatePageReq.
+ *
+ * @returns true if duplicate, false if not.
+ */
+static bool gmmR0FindDupPageInChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, uint8_t const *pbSourcePage)
+{
+    bool fFoundDuplicate = false;
+    /* Only take chunks not mapped into this VM process; not entirely correct. */
+    uint8_t *pbChunk;
+    if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
+    {
+        int rc = gmmR0MapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/, (PRTR3PTR)&pbChunk);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Look for duplicate pages
+             */
+            uintptr_t iPage = (GMM_CHUNK_SIZE >> PAGE_SHIFT);
+            while (iPage-- > 0)
+            {
+                if (GMM_PAGE_IS_PRIVATE(&pChunk->aPages[iPage]))
+                {
+                    uint8_t *pbDestPage = pbChunk + (iPage  << PAGE_SHIFT);
+                    if (!memcmp(pbSourcePage, pbDestPage, PAGE_SIZE))
+                    {
+                        fFoundDuplicate = true;
+                        break;
+                    }
+                }
+            }
+            gmmR0UnmapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/);
+        }
+    }
+    return fFoundDuplicate;
+}
+
+
+/**
+ * Find a duplicate of the specified page in other active VMs
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   pReq        Pointer to the request packet.
+ */
+GMMR0DECL(int) GMMR0FindDuplicatePageReq(PGVM pGVM, PGMMFINDDUPLICATEPAGEREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
+
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+
+    int rc = GVMMR0ValidateGVM(pGVM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Take the semaphore and do some more validations.
+     */
+    rc = gmmR0MutexAcquire(pGMM);
+    if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
+    {
+        uint8_t  *pbChunk;
+        PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pReq->idPage >> GMM_CHUNKID_SHIFT);
+        if (pChunk)
+        {
+            if (gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
+            {
+                uint8_t *pbSourcePage = pbChunk + ((pReq->idPage & GMM_PAGEID_IDX_MASK) << PAGE_SHIFT);
+                PGMMPAGE pPage = gmmR0GetPage(pGMM, pReq->idPage);
+                if (pPage)
+                {
+                    /*
+                     * Walk the chunks
+                     */
+                    pReq->fDuplicate = false;
+                    RTListForEach(&pGMM->ChunkList, pChunk, GMMCHUNK, ListNode)
+                    {
+                        if (gmmR0FindDupPageInChunk(pGMM, pGVM, pChunk, pbSourcePage))
+                        {
+                            pReq->fDuplicate = true;
+                            break;
+                        }
+                    }
+                }
+                else
+                {
+                    AssertFailed();
+                    rc = VERR_PGM_PHYS_INVALID_PAGE_ID;
+                }
+            }
+            else
+                AssertFailed();
+        }
+        else
+            AssertFailed();
+    }
+    else
+        rc = VERR_GMM_IS_NOT_SANE;
+
+    gmmR0MutexRelease(pGMM);
+    return rc;
+}
+
+#endif /* VBOX_STRICT && HC_ARCH_BITS == 64 */
+
+
+/**
+ * Retrieves the GMM statistics visible to the caller.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pStats      Where to put the statistics.
+ * @param   pSession    The current session.
+ * @param   pGVM        The GVM to obtain statistics for. Optional.
+ */
+GMMR0DECL(int) GMMR0QueryStatistics(PGMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM)
+{
+    LogFlow(("GVMMR0QueryStatistics: pStats=%p pSession=%p pGVM=%p\n", pStats, pSession, pGVM));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pSession, VERR_INVALID_POINTER);
+    AssertPtrReturn(pStats, VERR_INVALID_POINTER);
+    pStats->cMaxPages = 0; /* (crash before taking the mutex...) */
+
+    PGMM pGMM;
+    GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
+
+    /*
+     * Validate the VM handle, if not NULL, and lock the GMM.
+     */
+    int rc;
+    if (pGVM)
+    {
+        rc = GVMMR0ValidateGVM(pGVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    rc = gmmR0MutexAcquire(pGMM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Copy out the GMM statistics.
+     */
+    pStats->cMaxPages                   = pGMM->cMaxPages;
+    pStats->cReservedPages              = pGMM->cReservedPages;
+    pStats->cOverCommittedPages         = pGMM->cOverCommittedPages;
+    pStats->cAllocatedPages             = pGMM->cAllocatedPages;
+    pStats->cSharedPages                = pGMM->cSharedPages;
+    pStats->cDuplicatePages             = pGMM->cDuplicatePages;
+    pStats->cLeftBehindSharedPages      = pGMM->cLeftBehindSharedPages;
+    pStats->cBalloonedPages             = pGMM->cBalloonedPages;
+    pStats->cChunks                     = pGMM->cChunks;
+    pStats->cFreedChunks                = pGMM->cFreedChunks;
+    pStats->cShareableModules           = pGMM->cShareableModules;
+    pStats->idFreeGeneration            = pGMM->idFreeGeneration;
+    RT_ZERO(pStats->au64Reserved);
+
+    /*
+     * Copy out the VM statistics.
+     */
+    if (pGVM)
+        pStats->VMStats = pGVM->gmm.s.Stats;
+    else
+        RT_ZERO(pStats->VMStats);
+
+    gmmR0MutexRelease(pGMM);
+    return rc;
+}
+
+
+/**
+ * VMMR0 request wrapper for GMMR0QueryStatistics.
+ *
+ * @returns see GMMR0QueryStatistics.
+ * @param   pGVM        The global (ring-0) VM structure. Optional.
+ * @param   pReq        Pointer to the request packet.
+ */
+GMMR0DECL(int) GMMR0QueryStatisticsReq(PGVM pGVM, PGMMQUERYSTATISTICSSREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
+
+    return GMMR0QueryStatistics(&pReq->Stats, pReq->pSession, pGVM);
+}
+
+
+/**
+ * Resets the specified GMM statistics.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pStats      Which statistics to reset, that is, non-zero fields
+ *                      indicates which to reset.
+ * @param   pSession    The current session.
+ * @param   pGVM        The GVM to reset statistics for. Optional.
+ */
+GMMR0DECL(int) GMMR0ResetStatistics(PCGMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM)
+{
+    NOREF(pStats); NOREF(pSession); NOREF(pGVM);
+    /* Currently nothing we can reset at the moment. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VMMR0 request wrapper for GMMR0ResetStatistics.
+ *
+ * @returns see GMMR0ResetStatistics.
+ * @param   pGVM        The global (ring-0) VM structure. Optional.
+ * @param   pReq        Pointer to the request packet.
+ */
+GMMR0DECL(int) GMMR0ResetStatisticsReq(PGVM pGVM, PGMMRESETSTATISTICSSREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
+
+    return GMMR0ResetStatistics(&pReq->Stats, pReq->pSession, pGVM);
+}
+
diff --git a/src/VBox/VMM/VMMR0/GMMR0Internal.h b/src/VBox/VMM/VMMR0/GMMR0Internal.h
new file mode 100644
index 00000000..7bc3833d
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/GMMR0Internal.h
@@ -0,0 +1,116 @@
+/* $Id: GMMR0Internal.h $ */
+/** @file
+ * GMM - The Global Memory Manager, Internal Header.
+ */
+
+/*
+ * Copyright (C) 2007-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_VMMR0_GMMR0Internal_h
+#define VMM_INCLUDED_SRC_VMMR0_GMMR0Internal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/vmm/gmm.h>
+#include <iprt/avl.h>
+
+
+/**
+ * Shared module registration info (per VM)
+ */
+typedef struct GMMSHAREDMODULEPERVM
+{
+    /** Tree node. */
+    AVLGCPTRNODECORE            Core;
+    /** Pointer to global shared module info. */
+    PGMMSHAREDMODULE            pGlobalModule;
+    /** Pointer to the region addresses.
+     *
+     * They can differe between VMs because of address space scrambling or
+     * simply different loading order. */
+    RTGCPTR64                   aRegionsGCPtrs[1];
+} GMMSHAREDMODULEPERVM;
+/** Pointer to a GMMSHAREDMODULEPERVM. */
+typedef GMMSHAREDMODULEPERVM *PGMMSHAREDMODULEPERVM;
+
+
+/** Pointer to a GMM allocation chunk. */
+typedef struct GMMCHUNK *PGMMCHUNK;
+
+
+/** The GMMCHUNK::cFree shift count employed by gmmR0SelectFreeSetList. */
+#define GMM_CHUNK_FREE_SET_SHIFT    4
+/** Index of the list containing completely unused chunks.
+ * The code ASSUMES this is the last list. */
+#define GMM_CHUNK_FREE_SET_UNUSED_LIST  (GMM_CHUNK_NUM_PAGES >> GMM_CHUNK_FREE_SET_SHIFT)
+
+/**
+ * A set of free chunks.
+ */
+typedef struct GMMCHUNKFREESET
+{
+    /** The number of free pages in the set. */
+    uint64_t            cFreePages;
+    /** The generation ID for the set.  This is incremented whenever
+     *  something is linked or unlinked from this set. */
+    uint64_t            idGeneration;
+    /** Chunks ordered by increasing number of free pages.
+     *  In the final list the chunks are completely unused. */
+    PGMMCHUNK           apLists[GMM_CHUNK_FREE_SET_UNUSED_LIST + 1];
+} GMMCHUNKFREESET;
+
+
+/**
+ * A per-VM allocation chunk lookup TLB entry (for GMMR0PageIdToVirt).
+ */
+typedef struct GMMPERVMCHUNKTLBE
+{
+    /** The GMM::idFreeGeneration value this is valid for. */
+    uint64_t            idGeneration;
+    /** The chunk. */
+    PGMMCHUNK           pChunk;
+} GMMPERVMCHUNKTLBE;
+/** Poitner to a per-VM allocation chunk TLB entry. */
+typedef GMMPERVMCHUNKTLBE *PGMMPERVMCHUNKTLBE;
+
+/** The number of entries in the allocation chunk lookup TLB. */
+#define GMMPERVM_CHUNKTLB_ENTRIES           32
+/** Gets the TLB entry index for the given Chunk ID. */
+#define GMMPERVM_CHUNKTLB_IDX(a_idChunk)    ( (a_idChunk) & (GMMPERVM_CHUNKTLB_ENTRIES - 1) )
+
+
+/**
+ * The per-VM GMM data.
+ */
+typedef struct GMMPERVM
+{
+    /** Free set for use in bound mode. */
+    GMMCHUNKFREESET     Private;
+    /** The VM statistics. */
+    GMMVMSTATS          Stats;
+    /** Shared module tree (per-vm). */
+    PAVLGCPTRNODECORE   pSharedModuleTree;
+    /** Hints at the last chunk we allocated some memory from. */
+    uint32_t            idLastChunkHint;
+    uint32_t            u32Padding;
+
+    /** Spinlock protecting the chunk lookup TLB. */
+    RTSPINLOCK          hChunkTlbSpinLock;
+    /** The chunk lookup TLB used by GMMR0PageIdToVirt. */
+    GMMPERVMCHUNKTLBE   aChunkTlbEntries[GMMPERVM_CHUNKTLB_ENTRIES];
+} GMMPERVM;
+/** Pointer to the per-VM GMM data. */
+typedef GMMPERVM *PGMMPERVM;
+
+#endif /* !VMM_INCLUDED_SRC_VMMR0_GMMR0Internal_h */
+
diff --git a/src/VBox/VMM/VMMR0/GVMMR0.cpp b/src/VBox/VMM/VMMR0/GVMMR0.cpp
new file mode 100644
index 00000000..2de316d5
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/GVMMR0.cpp
@@ -0,0 +1,3029 @@
+/* $Id: GVMMR0.cpp $ */
+/** @file
+ * GVMM - Global VM Manager.
+ */
+
+/*
+ * Copyright (C) 2007-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_gvmm   GVMM - The Global VM Manager
+ *
+ * The Global VM Manager lives in ring-0.  Its main function at the moment is
+ * to manage a list of all running VMs, keep a ring-0 only structure (GVM) for
+ * each of them, and assign them unique identifiers (so GMM can track page
+ * owners).  The GVMM also manage some of the host CPU resources, like the
+ * periodic preemption timer.
+ *
+ * The GVMM will create a ring-0 object for each VM when it is registered, this
+ * is both for session cleanup purposes and for having a point where it is
+ * possible to implement usage polices later (in SUPR0ObjRegister).
+ *
+ *
+ * @section  sec_gvmm_ppt       Periodic Preemption Timer (PPT)
+ *
+ * On system that sports a high resolution kernel timer API, we use per-cpu
+ * timers to generate interrupts that preempts VT-x, AMD-V and raw-mode guest
+ * execution.  The timer frequency is calculating by taking the max
+ * TMCalcHostTimerFrequency for all VMs running on a CPU for the last ~160 ms
+ * (RT_ELEMENTS((PGVMMHOSTCPU)0, Ppt.aHzHistory) *
+ * GVMMHOSTCPU_PPT_HIST_INTERVAL_NS).
+ *
+ * The TMCalcHostTimerFrequency() part of the things gets its takes the max
+ * TMTimerSetFrequencyHint() value and adjusts by the current catch-up percent,
+ * warp drive percent and some fudge factors.  VMMR0.cpp reports the result via
+ * GVMMR0SchedUpdatePeriodicPreemptionTimer() before switching to the VT-x,
+ * AMD-V and raw-mode execution environments.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GVMM
+#include <VBox/vmm/gvmm.h>
+#include <VBox/vmm/gmm.h>
+#include "GVMMR0Internal.h"
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/vmm.h>
+#ifdef VBOX_WITH_NEM_R0
+# include <VBox/vmm/nem.h>
+#endif
+#include <VBox/vmm/vmcpuset.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/critsect.h>
+#include <iprt/mem.h>
+#include <iprt/semaphore.h>
+#include <iprt/time.h>
+#include <VBox/log.h>
+#include <iprt/thread.h>
+#include <iprt/process.h>
+#include <iprt/param.h>
+#include <iprt/string.h>
+#include <iprt/assert.h>
+#include <iprt/mem.h>
+#include <iprt/memobj.h>
+#include <iprt/mp.h>
+#include <iprt/cpuset.h>
+#include <iprt/spinlock.h>
+#include <iprt/timer.h>
+
+#include "dtrace/VBoxVMM.h"
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS) || defined(DOXYGEN_RUNNING)
+/** Define this to enable the periodic preemption timer. */
+# define GVMM_SCHED_WITH_PPT
+#endif
+
+
+/** @def GVMM_CHECK_SMAP_SETUP
+ * SMAP check setup. */
+/** @def GVMM_CHECK_SMAP_CHECK
+ * Checks that the AC flag is set if SMAP is enabled. If AC is not set,
+ * it will be logged and @a a_BadExpr is executed. */
+/** @def GVMM_CHECK_SMAP_CHECK2
+ * Checks that the AC flag is set if SMAP is enabled.  If AC is not set, it will
+ * be logged, written to the VMs assertion text buffer, and @a a_BadExpr is
+ * executed. */
+#if (defined(VBOX_STRICT) || 1) && !defined(VBOX_WITH_RAM_IN_KERNEL)
+# define GVMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = SUPR0GetKernelFeatures()
+# define GVMM_CHECK_SMAP_CHECK(a_BadExpr) \
+    do { \
+        if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
+        { \
+            RTCCUINTREG fEflCheck = ASMGetFlags(); \
+            if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
+            { /* likely */ } \
+            else \
+            { \
+                SUPR0Printf("%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
+                a_BadExpr; \
+            } \
+        } \
+    } while (0)
+# define GVMM_CHECK_SMAP_CHECK2(a_pGVM, a_BadExpr) \
+    do { \
+        if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
+        { \
+            RTCCUINTREG fEflCheck = ASMGetFlags(); \
+            if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
+            { /* likely */ } \
+            else \
+            { \
+                SUPR0BadContext((a_pGVM) ? (a_pGVM)->pSession : NULL, __FILE__, __LINE__, "EFLAGS.AC is zero!"); \
+                a_BadExpr; \
+            } \
+        } \
+    } while (0)
+#else
+# define GVMM_CHECK_SMAP_SETUP()           uint32_t const fKernelFeatures = 0
+# define GVMM_CHECK_SMAP_CHECK(a_BadExpr)           NOREF(fKernelFeatures)
+# define GVMM_CHECK_SMAP_CHECK2(a_pGVM, a_BadExpr)   NOREF(fKernelFeatures)
+#endif
+
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+
+/**
+ * Global VM handle.
+ */
+typedef struct GVMHANDLE
+{
+    /** The index of the next handle in the list (free or used). (0 is nil.) */
+    uint16_t volatile   iNext;
+    /** Our own index / handle value. */
+    uint16_t            iSelf;
+    /** The process ID of the handle owner.
+     * This is used for access checks. */
+    RTPROCESS           ProcId;
+    /** The pointer to the ring-0 only (aka global) VM structure. */
+    PGVM                pGVM;
+    /** The virtual machine object. */
+    void               *pvObj;
+    /** The session this VM is associated with. */
+    PSUPDRVSESSION      pSession;
+    /** The ring-0 handle of the EMT0 thread.
+     * This is used for ownership checks as well as looking up a VM handle by thread
+     * at times like assertions. */
+    RTNATIVETHREAD      hEMT0;
+} GVMHANDLE;
+/** Pointer to a global VM handle. */
+typedef GVMHANDLE *PGVMHANDLE;
+
+/** Number of GVM handles (including the NIL handle). */
+#if HC_ARCH_BITS == 64
+# define GVMM_MAX_HANDLES   8192
+#else
+# define GVMM_MAX_HANDLES   128
+#endif
+
+/**
+ * Per host CPU GVMM data.
+ */
+typedef struct GVMMHOSTCPU
+{
+    /** Magic number (GVMMHOSTCPU_MAGIC). */
+    uint32_t volatile   u32Magic;
+    /** The CPU ID. */
+    RTCPUID             idCpu;
+    /** The CPU set index. */
+    uint32_t            idxCpuSet;
+
+#ifdef GVMM_SCHED_WITH_PPT
+    /** Periodic preemption timer data. */
+    struct
+    {
+        /** The handle to the periodic preemption timer. */
+        PRTTIMER            pTimer;
+        /** Spinlock protecting the data below. */
+        RTSPINLOCK          hSpinlock;
+        /** The smalles Hz that we need to care about. (static) */
+        uint32_t            uMinHz;
+        /** The number of ticks between each historization. */
+        uint32_t            cTicksHistoriziationInterval;
+        /** The current historization tick (counting up to
+         * cTicksHistoriziationInterval and then resetting). */
+        uint32_t            iTickHistorization;
+        /** The current timer interval.  This is set to 0 when inactive. */
+        uint32_t            cNsInterval;
+        /** The current timer frequency.  This is set to 0 when inactive. */
+        uint32_t            uTimerHz;
+        /** The current max frequency reported by the EMTs.
+         * This gets historicize and reset by the timer callback.  This is
+         * read without holding the spinlock, so needs atomic updating. */
+        uint32_t volatile   uDesiredHz;
+        /** Whether the timer was started or not. */
+        bool volatile       fStarted;
+        /** Set if we're starting timer. */
+        bool volatile       fStarting;
+        /** The index of the next history entry (mod it). */
+        uint32_t            iHzHistory;
+        /** Historicized uDesiredHz values.  The array wraps around, new entries
+         * are added at iHzHistory. This is updated approximately every
+         * GVMMHOSTCPU_PPT_HIST_INTERVAL_NS by the timer callback. */
+        uint32_t            aHzHistory[8];
+        /** Statistics counter for recording the number of interval changes. */
+        uint32_t            cChanges;
+        /** Statistics counter for recording the number of timer starts. */
+        uint32_t            cStarts;
+    } Ppt;
+#endif /* GVMM_SCHED_WITH_PPT */
+
+} GVMMHOSTCPU;
+/** Pointer to the per host CPU GVMM data. */
+typedef GVMMHOSTCPU *PGVMMHOSTCPU;
+/** The GVMMHOSTCPU::u32Magic value (Petra, Tanya & Rachel Haden). */
+#define GVMMHOSTCPU_MAGIC   UINT32_C(0x19711011)
+/** The interval on history entry should cover (approximately) give in
+ *  nanoseconds. */
+#define GVMMHOSTCPU_PPT_HIST_INTERVAL_NS    UINT32_C(20000000)
+
+
+/**
+ * The GVMM instance data.
+ */
+typedef struct GVMM
+{
+    /** Eyecatcher / magic. */
+    uint32_t            u32Magic;
+    /** The index of the head of the free handle chain. (0 is nil.) */
+    uint16_t volatile   iFreeHead;
+    /** The index of the head of the active handle chain. (0 is nil.) */
+    uint16_t volatile   iUsedHead;
+    /** The number of VMs. */
+    uint16_t volatile   cVMs;
+    /** Alignment padding. */
+    uint16_t            u16Reserved;
+    /** The number of EMTs. */
+    uint32_t volatile   cEMTs;
+    /** The number of EMTs that have halted in GVMMR0SchedHalt. */
+    uint32_t volatile   cHaltedEMTs;
+    /** Mini lock for restricting early wake-ups to one thread. */
+    bool volatile       fDoingEarlyWakeUps;
+    bool                afPadding[3]; /**< explicit alignment padding. */
+    /** When the next halted or sleeping EMT will wake up.
+     * This is set to 0 when it needs recalculating and to UINT64_MAX when
+     * there are no halted or sleeping EMTs in the GVMM. */
+    uint64_t            uNsNextEmtWakeup;
+    /** The lock used to serialize VM creation, destruction and associated events that
+     * isn't performance critical. Owners may acquire the list lock. */
+    RTCRITSECT          CreateDestroyLock;
+    /** The lock used to serialize used list updates and accesses.
+     * This indirectly includes scheduling since the scheduler will have to walk the
+     * used list to examin running VMs. Owners may not acquire any other locks. */
+    RTCRITSECTRW        UsedLock;
+    /** The handle array.
+     * The size of this array defines the maximum number of currently running VMs.
+     * The first entry is unused as it represents the NIL handle. */
+    GVMHANDLE           aHandles[GVMM_MAX_HANDLES];
+
+    /** @gcfgm{/GVMM/cEMTsMeansCompany, 32-bit, 0, UINT32_MAX, 1}
+     * The number of EMTs that means we no longer consider ourselves alone on a
+     * CPU/Core.
+     */
+    uint32_t            cEMTsMeansCompany;
+    /** @gcfgm{/GVMM/MinSleepAlone,32-bit, 0, 100000000, 750000, ns}
+     * The minimum sleep time for when we're alone, in nano seconds.
+     */
+    uint32_t            nsMinSleepAlone;
+    /** @gcfgm{/GVMM/MinSleepCompany,32-bit,0, 100000000, 15000, ns}
+     * The minimum sleep time for when we've got company, in nano seconds.
+     */
+    uint32_t            nsMinSleepCompany;
+    /** @gcfgm{/GVMM/EarlyWakeUp1, 32-bit, 0, 100000000, 25000, ns}
+     * The limit for the first round of early wake-ups, given in nano seconds.
+     */
+    uint32_t            nsEarlyWakeUp1;
+    /** @gcfgm{/GVMM/EarlyWakeUp2, 32-bit, 0, 100000000, 50000, ns}
+     * The limit for the second round of early wake-ups, given in nano seconds.
+     */
+    uint32_t            nsEarlyWakeUp2;
+
+    /** Set if we're doing early wake-ups.
+     * This reflects  nsEarlyWakeUp1 and nsEarlyWakeUp2.  */
+    bool volatile       fDoEarlyWakeUps;
+
+    /** The number of entries in the host CPU array (aHostCpus). */
+    uint32_t            cHostCpus;
+    /** Per host CPU data (variable length). */
+    GVMMHOSTCPU         aHostCpus[1];
+} GVMM;
+AssertCompileMemberAlignment(GVMM, CreateDestroyLock, 8);
+AssertCompileMemberAlignment(GVMM, UsedLock, 8);
+AssertCompileMemberAlignment(GVMM, uNsNextEmtWakeup, 8);
+/** Pointer to the GVMM instance data. */
+typedef GVMM *PGVMM;
+
+/** The GVMM::u32Magic value (Charlie Haden). */
+#define GVMM_MAGIC      UINT32_C(0x19370806)
+
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** Pointer to the GVMM instance data.
+ * (Just my general dislike for global variables.) */
+static PGVMM g_pGVMM = NULL;
+
+/** Macro for obtaining and validating the g_pGVMM pointer.
+ * On failure it will return from the invoking function with the specified return value.
+ *
+ * @param   pGVMM   The name of the pGVMM variable.
+ * @param   rc      The return value on failure. Use VERR_GVMM_INSTANCE for VBox
+ *                  status codes.
+ */
+#define GVMM_GET_VALID_INSTANCE(pGVMM, rc) \
+    do { \
+        (pGVMM) = g_pGVMM;\
+        AssertPtrReturn((pGVMM), (rc)); \
+        AssertMsgReturn((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic), (rc)); \
+    } while (0)
+
+/** Macro for obtaining and validating the g_pGVMM pointer, void function variant.
+ * On failure it will return from the invoking function.
+ *
+ * @param   pGVMM   The name of the pGVMM variable.
+ */
+#define GVMM_GET_VALID_INSTANCE_VOID(pGVMM) \
+    do { \
+        (pGVMM) = g_pGVMM;\
+        AssertPtrReturnVoid((pGVMM)); \
+        AssertMsgReturnVoid((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic)); \
+    } while (0)
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static void gvmmR0InitPerVMData(PGVM pGVM, int16_t hSelf, VMCPUID cCpus, PSUPDRVSESSION pSession);
+static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void *pvObj, void *pvGVMM, void *pvHandle);
+static int gvmmR0ByGVM(PGVM pGVM, PGVMM *ppGVMM, bool fTakeUsedLock);
+static int gvmmR0ByGVMandEMT(PGVM pGVM, VMCPUID idCpu, PGVMM *ppGVMM);
+
+#ifdef GVMM_SCHED_WITH_PPT
+static DECLCALLBACK(void) gvmmR0SchedPeriodicPreemptionTimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
+#endif
+
+
+/**
+ * Initializes the GVMM.
+ *
+ * This is called while owning the loader semaphore (see supdrvIOCtl_LdrLoad()).
+ *
+ * @returns VBox status code.
+ */
+GVMMR0DECL(int) GVMMR0Init(void)
+{
+    LogFlow(("GVMMR0Init:\n"));
+
+    /*
+     * Allocate and initialize the instance data.
+     */
+    uint32_t cHostCpus = RTMpGetArraySize();
+    AssertMsgReturn(cHostCpus > 0 && cHostCpus < _64K, ("%d", (int)cHostCpus), VERR_GVMM_HOST_CPU_RANGE);
+
+    PGVMM pGVMM = (PGVMM)RTMemAllocZ(RT_UOFFSETOF_DYN(GVMM, aHostCpus[cHostCpus]));
+    if (!pGVMM)
+        return VERR_NO_MEMORY;
+    int rc = RTCritSectInitEx(&pGVMM->CreateDestroyLock, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE,
+                              "GVMM-CreateDestroyLock");
+    if (RT_SUCCESS(rc))
+    {
+        rc = RTCritSectRwInitEx(&pGVMM->UsedLock, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE, "GVMM-UsedLock");
+        if (RT_SUCCESS(rc))
+        {
+            pGVMM->u32Magic = GVMM_MAGIC;
+            pGVMM->iUsedHead = 0;
+            pGVMM->iFreeHead = 1;
+
+            /* the nil handle */
+            pGVMM->aHandles[0].iSelf = 0;
+            pGVMM->aHandles[0].iNext = 0;
+
+            /* the tail */
+            unsigned i = RT_ELEMENTS(pGVMM->aHandles) - 1;
+            pGVMM->aHandles[i].iSelf = i;
+            pGVMM->aHandles[i].iNext = 0; /* nil */
+
+            /* the rest */
+            while (i-- > 1)
+            {
+                pGVMM->aHandles[i].iSelf = i;
+                pGVMM->aHandles[i].iNext = i + 1;
+            }
+
+            /* The default configuration values. */
+            uint32_t cNsResolution = RTSemEventMultiGetResolution();
+            pGVMM->cEMTsMeansCompany     = 1;                           /** @todo should be adjusted to relative to the cpu count or something... */
+            if (cNsResolution >= 5*RT_NS_100US)
+            {
+                pGVMM->nsMinSleepAlone   = 750000 /* ns (0.750 ms) */;  /** @todo this should be adjusted to be 75% (or something) of the scheduler granularity... */
+                pGVMM->nsMinSleepCompany =  15000 /* ns (0.015 ms) */;
+                pGVMM->nsEarlyWakeUp1    =  25000 /* ns (0.025 ms) */;
+                pGVMM->nsEarlyWakeUp2    =  50000 /* ns (0.050 ms) */;
+            }
+            else if (cNsResolution > RT_NS_100US)
+            {
+                pGVMM->nsMinSleepAlone   = cNsResolution / 2;
+                pGVMM->nsMinSleepCompany = cNsResolution / 4;
+                pGVMM->nsEarlyWakeUp1    = 0;
+                pGVMM->nsEarlyWakeUp2    = 0;
+            }
+            else
+            {
+                pGVMM->nsMinSleepAlone   = 2000;
+                pGVMM->nsMinSleepCompany = 2000;
+                pGVMM->nsEarlyWakeUp1    = 0;
+                pGVMM->nsEarlyWakeUp2    = 0;
+            }
+            pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
+
+            /* The host CPU data. */
+            pGVMM->cHostCpus = cHostCpus;
+            uint32_t    iCpu = cHostCpus;
+            RTCPUSET    PossibleSet;
+            RTMpGetSet(&PossibleSet);
+            while (iCpu-- > 0)
+            {
+                pGVMM->aHostCpus[iCpu].idxCpuSet        = iCpu;
+#ifdef GVMM_SCHED_WITH_PPT
+                pGVMM->aHostCpus[iCpu].Ppt.pTimer       = NULL;
+                pGVMM->aHostCpus[iCpu].Ppt.hSpinlock    = NIL_RTSPINLOCK;
+                pGVMM->aHostCpus[iCpu].Ppt.uMinHz       = 5; /** @todo Add some API which figures this one out. (not *that* important) */
+                pGVMM->aHostCpus[iCpu].Ppt.cTicksHistoriziationInterval = 1;
+                //pGVMM->aHostCpus[iCpu].Ppt.iTickHistorization           = 0;
+                //pGVMM->aHostCpus[iCpu].Ppt.cNsInterval  = 0;
+                //pGVMM->aHostCpus[iCpu].Ppt.uTimerHz     = 0;
+                //pGVMM->aHostCpus[iCpu].Ppt.uDesiredHz   = 0;
+                //pGVMM->aHostCpus[iCpu].Ppt.fStarted     = false;
+                //pGVMM->aHostCpus[iCpu].Ppt.fStarting    = false;
+                //pGVMM->aHostCpus[iCpu].Ppt.iHzHistory   = 0;
+                //pGVMM->aHostCpus[iCpu].Ppt.aHzHistory   = {0};
+#endif
+
+                if (RTCpuSetIsMember(&PossibleSet, iCpu))
+                {
+                    pGVMM->aHostCpus[iCpu].idCpu        = RTMpCpuIdFromSetIndex(iCpu);
+                    pGVMM->aHostCpus[iCpu].u32Magic     = GVMMHOSTCPU_MAGIC;
+
+#ifdef GVMM_SCHED_WITH_PPT
+                    rc = RTTimerCreateEx(&pGVMM->aHostCpus[iCpu].Ppt.pTimer,
+                                         50*1000*1000 /* whatever */,
+                                         RTTIMER_FLAGS_CPU(iCpu) | RTTIMER_FLAGS_HIGH_RES,
+                                         gvmmR0SchedPeriodicPreemptionTimerCallback,
+                                         &pGVMM->aHostCpus[iCpu]);
+                    if (RT_SUCCESS(rc))
+                        rc = RTSpinlockCreate(&pGVMM->aHostCpus[iCpu].Ppt.hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "GVMM/CPU");
+                    if (RT_FAILURE(rc))
+                    {
+                        while (iCpu < cHostCpus)
+                        {
+                            RTTimerDestroy(pGVMM->aHostCpus[iCpu].Ppt.pTimer);
+                            RTSpinlockDestroy(pGVMM->aHostCpus[iCpu].Ppt.hSpinlock);
+                            pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
+                            iCpu++;
+                        }
+                        break;
+                    }
+#endif
+                }
+                else
+                {
+                    pGVMM->aHostCpus[iCpu].idCpu        = NIL_RTCPUID;
+                    pGVMM->aHostCpus[iCpu].u32Magic     = 0;
+                }
+            }
+            if (RT_SUCCESS(rc))
+            {
+                g_pGVMM = pGVMM;
+                LogFlow(("GVMMR0Init: pGVMM=%p cHostCpus=%u\n", pGVMM, cHostCpus));
+                return VINF_SUCCESS;
+            }
+
+            /* bail out. */
+            RTCritSectRwDelete(&pGVMM->UsedLock);
+        }
+        RTCritSectDelete(&pGVMM->CreateDestroyLock);
+    }
+
+    RTMemFree(pGVMM);
+    return rc;
+}
+
+
+/**
+ * Terminates the GVM.
+ *
+ * This is called while owning the loader semaphore (see supdrvLdrFree()).
+ * And unless something is wrong, there should be absolutely no VMs
+ * registered at this point.
+ */
+GVMMR0DECL(void) GVMMR0Term(void)
+{
+    LogFlow(("GVMMR0Term:\n"));
+
+    PGVMM pGVMM = g_pGVMM;
+    g_pGVMM = NULL;
+    if (RT_UNLIKELY(!RT_VALID_PTR(pGVMM)))
+    {
+        SUPR0Printf("GVMMR0Term: pGVMM=%RKv\n", pGVMM);
+        return;
+    }
+
+    /*
+     * First of all, stop all active timers.
+     */
+    uint32_t cActiveTimers = 0;
+    uint32_t iCpu = pGVMM->cHostCpus;
+    while (iCpu-- > 0)
+    {
+        ASMAtomicWriteU32(&pGVMM->aHostCpus[iCpu].u32Magic, ~GVMMHOSTCPU_MAGIC);
+#ifdef GVMM_SCHED_WITH_PPT
+        if (    pGVMM->aHostCpus[iCpu].Ppt.pTimer != NULL
+            &&  RT_SUCCESS(RTTimerStop(pGVMM->aHostCpus[iCpu].Ppt.pTimer)))
+            cActiveTimers++;
+#endif
+    }
+    if (cActiveTimers)
+        RTThreadSleep(1); /* fudge */
+
+    /*
+     * Invalidate the and free resources.
+     */
+    pGVMM->u32Magic = ~GVMM_MAGIC;
+    RTCritSectRwDelete(&pGVMM->UsedLock);
+    RTCritSectDelete(&pGVMM->CreateDestroyLock);
+
+    pGVMM->iFreeHead = 0;
+    if (pGVMM->iUsedHead)
+    {
+        SUPR0Printf("GVMMR0Term: iUsedHead=%#x! (cVMs=%#x cEMTs=%#x)\n", pGVMM->iUsedHead, pGVMM->cVMs, pGVMM->cEMTs);
+        pGVMM->iUsedHead = 0;
+    }
+
+#ifdef GVMM_SCHED_WITH_PPT
+    iCpu = pGVMM->cHostCpus;
+    while (iCpu-- > 0)
+    {
+        RTTimerDestroy(pGVMM->aHostCpus[iCpu].Ppt.pTimer);
+        pGVMM->aHostCpus[iCpu].Ppt.pTimer = NULL;
+        RTSpinlockDestroy(pGVMM->aHostCpus[iCpu].Ppt.hSpinlock);
+        pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
+    }
+#endif
+
+    RTMemFree(pGVMM);
+}
+
+
+/**
+ * A quick hack for setting global config values.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSession    The session handle. Used for authentication.
+ * @param   pszName     The variable name.
+ * @param   u64Value    The new value.
+ */
+GVMMR0DECL(int) GVMMR0SetConfig(PSUPDRVSESSION pSession, const char *pszName, uint64_t u64Value)
+{
+    /*
+     * Validate input.
+     */
+    PGVMM pGVMM;
+    GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
+    AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
+    AssertPtrReturn(pszName, VERR_INVALID_POINTER);
+
+    /*
+     * String switch time!
+     */
+    if (strncmp(pszName, RT_STR_TUPLE("/GVMM/")))
+        return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
+    int rc = VINF_SUCCESS;
+    pszName += sizeof("/GVMM/") - 1;
+    if (!strcmp(pszName, "cEMTsMeansCompany"))
+    {
+        if (u64Value <= UINT32_MAX)
+            pGVMM->cEMTsMeansCompany = u64Value;
+        else
+            rc = VERR_OUT_OF_RANGE;
+    }
+    else if (!strcmp(pszName, "MinSleepAlone"))
+    {
+        if (u64Value <= RT_NS_100MS)
+            pGVMM->nsMinSleepAlone = u64Value;
+        else
+            rc = VERR_OUT_OF_RANGE;
+    }
+    else if (!strcmp(pszName, "MinSleepCompany"))
+    {
+        if (u64Value <= RT_NS_100MS)
+            pGVMM->nsMinSleepCompany = u64Value;
+        else
+            rc = VERR_OUT_OF_RANGE;
+    }
+    else if (!strcmp(pszName, "EarlyWakeUp1"))
+    {
+        if (u64Value <= RT_NS_100MS)
+        {
+            pGVMM->nsEarlyWakeUp1 = u64Value;
+            pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
+        }
+        else
+            rc = VERR_OUT_OF_RANGE;
+    }
+    else if (!strcmp(pszName, "EarlyWakeUp2"))
+    {
+        if (u64Value <= RT_NS_100MS)
+        {
+            pGVMM->nsEarlyWakeUp2 = u64Value;
+            pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
+        }
+        else
+            rc = VERR_OUT_OF_RANGE;
+    }
+    else
+        rc = VERR_CFGM_VALUE_NOT_FOUND;
+    return rc;
+}
+
+
+/**
+ * A quick hack for getting global config values.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSession    The session handle. Used for authentication.
+ * @param   pszName     The variable name.
+ * @param   pu64Value   Where to return the value.
+ */
+GVMMR0DECL(int) GVMMR0QueryConfig(PSUPDRVSESSION pSession, const char *pszName, uint64_t *pu64Value)
+{
+    /*
+     * Validate input.
+     */
+    PGVMM pGVMM;
+    GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
+    AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
+    AssertPtrReturn(pszName, VERR_INVALID_POINTER);
+    AssertPtrReturn(pu64Value, VERR_INVALID_POINTER);
+
+    /*
+     * String switch time!
+     */
+    if (strncmp(pszName, RT_STR_TUPLE("/GVMM/")))
+        return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
+    int rc = VINF_SUCCESS;
+    pszName += sizeof("/GVMM/") - 1;
+    if (!strcmp(pszName, "cEMTsMeansCompany"))
+        *pu64Value = pGVMM->cEMTsMeansCompany;
+    else if (!strcmp(pszName, "MinSleepAlone"))
+        *pu64Value = pGVMM->nsMinSleepAlone;
+    else if (!strcmp(pszName, "MinSleepCompany"))
+        *pu64Value = pGVMM->nsMinSleepCompany;
+    else if (!strcmp(pszName, "EarlyWakeUp1"))
+        *pu64Value = pGVMM->nsEarlyWakeUp1;
+    else if (!strcmp(pszName, "EarlyWakeUp2"))
+        *pu64Value = pGVMM->nsEarlyWakeUp2;
+    else
+        rc = VERR_CFGM_VALUE_NOT_FOUND;
+    return rc;
+}
+
+
+/**
+ * Acquire the 'used' lock in shared mode.
+ *
+ * This prevents destruction of the VM while we're in ring-0.
+ *
+ * @returns IPRT status code, see RTSemFastMutexRequest.
+ * @param   a_pGVMM     The GVMM instance data.
+ * @sa      GVMMR0_USED_SHARED_UNLOCK, GVMMR0_USED_EXCLUSIVE_LOCK
+ */
+#define GVMMR0_USED_SHARED_LOCK(a_pGVMM)        RTCritSectRwEnterShared(&(a_pGVMM)->UsedLock)
+
+/**
+ * Release the 'used' lock in when owning it in shared mode.
+ *
+ * @returns IPRT status code, see RTSemFastMutexRequest.
+ * @param   a_pGVMM     The GVMM instance data.
+ * @sa      GVMMR0_USED_SHARED_LOCK
+ */
+#define GVMMR0_USED_SHARED_UNLOCK(a_pGVMM)      RTCritSectRwLeaveShared(&(a_pGVMM)->UsedLock)
+
+/**
+ * Acquire the 'used' lock in exclusive mode.
+ *
+ * Only use this function when making changes to the used list.
+ *
+ * @returns IPRT status code, see RTSemFastMutexRequest.
+ * @param   a_pGVMM     The GVMM instance data.
+ * @sa      GVMMR0_USED_EXCLUSIVE_UNLOCK
+ */
+#define GVMMR0_USED_EXCLUSIVE_LOCK(a_pGVMM)     RTCritSectRwEnterExcl(&(a_pGVMM)->UsedLock)
+
+/**
+ * Release the 'used' lock when owning it in exclusive mode.
+ *
+ * @returns IPRT status code, see RTSemFastMutexRelease.
+ * @param   a_pGVMM     The GVMM instance data.
+ * @sa      GVMMR0_USED_EXCLUSIVE_LOCK, GVMMR0_USED_SHARED_UNLOCK
+ */
+#define GVMMR0_USED_EXCLUSIVE_UNLOCK(a_pGVMM)   RTCritSectRwLeaveExcl(&(a_pGVMM)->UsedLock)
+
+
+/**
+ * Try acquire the 'create & destroy' lock.
+ *
+ * @returns IPRT status code, see RTSemFastMutexRequest.
+ * @param   pGVMM   The GVMM instance data.
+ */
+DECLINLINE(int) gvmmR0CreateDestroyLock(PGVMM pGVMM)
+{
+    LogFlow(("++gvmmR0CreateDestroyLock(%p)\n", pGVMM));
+    int rc = RTCritSectEnter(&pGVMM->CreateDestroyLock);
+    LogFlow(("gvmmR0CreateDestroyLock(%p)->%Rrc\n", pGVMM, rc));
+    return rc;
+}
+
+
+/**
+ * Release the 'create & destroy' lock.
+ *
+ * @returns IPRT status code, see RTSemFastMutexRequest.
+ * @param   pGVMM   The GVMM instance data.
+ */
+DECLINLINE(int) gvmmR0CreateDestroyUnlock(PGVMM pGVMM)
+{
+    LogFlow(("--gvmmR0CreateDestroyUnlock(%p)\n", pGVMM));
+    int rc = RTCritSectLeave(&pGVMM->CreateDestroyLock);
+    AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * Request wrapper for the GVMMR0CreateVM API.
+ *
+ * @returns VBox status code.
+ * @param   pReq        The request buffer.
+ * @param   pSession    The session handle. The VM will be associated with this.
+ */
+GVMMR0DECL(int) GVMMR0CreateVMReq(PGVMMCREATEVMREQ pReq, PSUPDRVSESSION pSession)
+{
+    /*
+     * Validate the request.
+     */
+    if (!RT_VALID_PTR(pReq))
+        return VERR_INVALID_POINTER;
+    if (pReq->Hdr.cbReq != sizeof(*pReq))
+        return VERR_INVALID_PARAMETER;
+    if (pReq->pSession != pSession)
+        return VERR_INVALID_POINTER;
+
+    /*
+     * Execute it.
+     */
+    PGVM pGVM;
+    pReq->pVMR0 = NULL;
+    pReq->pVMR3 = NIL_RTR3PTR;
+    int rc = GVMMR0CreateVM(pSession, pReq->cCpus, &pGVM);
+    if (RT_SUCCESS(rc))
+    {
+        pReq->pVMR0 = pGVM; /** @todo don't expose this to ring-3, use a unique random number instead. */
+        pReq->pVMR3 = pGVM->pVMR3;
+    }
+    return rc;
+}
+
+
+/**
+ * Allocates the VM structure and registers it with GVM.
+ *
+ * The caller will become the VM owner and there by the EMT.
+ *
+ * @returns VBox status code.
+ * @param   pSession    The support driver session.
+ * @param   cCpus       Number of virtual CPUs for the new VM.
+ * @param   ppGVM       Where to store the pointer to the VM structure.
+ *
+ * @thread  EMT.
+ */
+GVMMR0DECL(int) GVMMR0CreateVM(PSUPDRVSESSION pSession, uint32_t cCpus, PGVM *ppGVM)
+{
+    LogFlow(("GVMMR0CreateVM: pSession=%p\n", pSession));
+    PGVMM pGVMM;
+    GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
+
+    AssertPtrReturn(ppGVM, VERR_INVALID_POINTER);
+    *ppGVM = NULL;
+
+    if (    cCpus == 0
+        ||  cCpus > VMM_MAX_CPU_COUNT)
+        return VERR_INVALID_PARAMETER;
+
+    RTNATIVETHREAD hEMT0 = RTThreadNativeSelf();
+    AssertReturn(hEMT0 != NIL_RTNATIVETHREAD, VERR_GVMM_BROKEN_IPRT);
+    RTPROCESS      ProcId = RTProcSelf();
+    AssertReturn(ProcId != NIL_RTPROCESS, VERR_GVMM_BROKEN_IPRT);
+
+    /*
+     * The whole allocation process is protected by the lock.
+     */
+    int rc = gvmmR0CreateDestroyLock(pGVMM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Only one VM per session.
+     */
+    if (SUPR0GetSessionVM(pSession) != NULL)
+    {
+        gvmmR0CreateDestroyUnlock(pGVMM);
+        SUPR0Printf("GVMMR0CreateVM: The session %p already got a VM: %p\n", pSession, SUPR0GetSessionVM(pSession));
+        return VERR_ALREADY_EXISTS;
+    }
+
+    /*
+     * Allocate a handle first so we don't waste resources unnecessarily.
+     */
+    uint16_t iHandle = pGVMM->iFreeHead;
+    if (iHandle)
+    {
+        PGVMHANDLE pHandle = &pGVMM->aHandles[iHandle];
+
+        /* consistency checks, a bit paranoid as always. */
+        if (    !pHandle->pGVM
+            &&  !pHandle->pvObj
+            &&  pHandle->iSelf == iHandle)
+        {
+            pHandle->pvObj = SUPR0ObjRegister(pSession, SUPDRVOBJTYPE_VM, gvmmR0HandleObjDestructor, pGVMM, pHandle);
+            if (pHandle->pvObj)
+            {
+                /*
+                 * Move the handle from the free to used list and perform permission checks.
+                 */
+                rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
+                AssertRC(rc);
+
+                pGVMM->iFreeHead = pHandle->iNext;
+                pHandle->iNext = pGVMM->iUsedHead;
+                pGVMM->iUsedHead = iHandle;
+                pGVMM->cVMs++;
+
+                pHandle->pGVM     = NULL;
+                pHandle->pSession = pSession;
+                pHandle->hEMT0    = NIL_RTNATIVETHREAD;
+                pHandle->ProcId   = NIL_RTPROCESS;
+
+                GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
+
+                rc = SUPR0ObjVerifyAccess(pHandle->pvObj, pSession, NULL);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Allocate memory for the VM structure (combined VM + GVM).
+                     */
+                    const uint32_t  cbVM      = RT_UOFFSETOF_DYN(GVM, aCpus[cCpus]);
+                    const uint32_t  cPages    = RT_ALIGN_32(cbVM, PAGE_SIZE) >> PAGE_SHIFT;
+                    RTR0MEMOBJ      hVMMemObj = NIL_RTR0MEMOBJ;
+                    rc = RTR0MemObjAllocPage(&hVMMemObj, cPages << PAGE_SHIFT, false /* fExecutable */);
+                    if (RT_SUCCESS(rc))
+                    {
+                        PGVM pGVM = (PGVM)RTR0MemObjAddress(hVMMemObj);
+                        AssertPtr(pGVM);
+
+                        /*
+                         * Initialise the structure.
+                         */
+                        RT_BZERO(pGVM, cPages << PAGE_SHIFT);
+                        gvmmR0InitPerVMData(pGVM, iHandle, cCpus, pSession);
+                        pGVM->gvmm.s.VMMemObj  = hVMMemObj;
+                        rc = GMMR0InitPerVMData(pGVM);
+                        int rc2 = PGMR0InitPerVMData(pGVM);
+                        PDMR0InitPerVMData(pGVM);
+                        IOMR0InitPerVMData(pGVM);
+                        if (RT_SUCCESS(rc) && RT_SUCCESS(rc2))
+                        {
+                            /*
+                             * Allocate page array.
+                             * This currently have to be made available to ring-3, but this is should change eventually.
+                             */
+                            rc = RTR0MemObjAllocPage(&pGVM->gvmm.s.VMPagesMemObj, cPages * sizeof(SUPPAGE), false /* fExecutable */);
+                            if (RT_SUCCESS(rc))
+                            {
+                                PSUPPAGE paPages = (PSUPPAGE)RTR0MemObjAddress(pGVM->gvmm.s.VMPagesMemObj); AssertPtr(paPages);
+                                for (uint32_t iPage = 0; iPage < cPages; iPage++)
+                                {
+                                    paPages[iPage].uReserved = 0;
+                                    paPages[iPage].Phys = RTR0MemObjGetPagePhysAddr(pGVM->gvmm.s.VMMemObj, iPage);
+                                    Assert(paPages[iPage].Phys != NIL_RTHCPHYS);
+                                }
+
+                                /*
+                                 * Map the page array, VM and VMCPU structures into ring-3.
+                                 */
+                                AssertCompileSizeAlignment(VM, PAGE_SIZE);
+                                rc = RTR0MemObjMapUserEx(&pGVM->gvmm.s.VMMapObj, pGVM->gvmm.s.VMMemObj, (RTR3PTR)-1, 0,
+                                                         RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS,
+                                                         0 /*offSub*/, sizeof(VM));
+                                for (VMCPUID i = 0; i < cCpus && RT_SUCCESS(rc); i++)
+                                {
+                                    AssertCompileSizeAlignment(VMCPU, PAGE_SIZE);
+                                    rc = RTR0MemObjMapUserEx(&pGVM->aCpus[i].gvmm.s.VMCpuMapObj, pGVM->gvmm.s.VMMemObj,
+                                                             (RTR3PTR)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS,
+                                                             RT_UOFFSETOF_DYN(GVM, aCpus[i]), sizeof(VMCPU));
+                                }
+                                if (RT_SUCCESS(rc))
+                                    rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMPagesMapObj, pGVM->gvmm.s.VMPagesMemObj, (RTR3PTR)-1,
+                                                           0 /* uAlignment */, RTMEM_PROT_READ | RTMEM_PROT_WRITE,
+                                                           NIL_RTR0PROCESS);
+                                if (RT_SUCCESS(rc))
+                                {
+                                    /*
+                                     * Initialize all the VM pointers.
+                                     */
+                                    PVMR3 pVMR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMMapObj);
+                                    AssertPtr((void *)pVMR3);
+
+                                    for (VMCPUID i = 0; i < cCpus; i++)
+                                    {
+                                        pGVM->aCpus[i].pVMR0 = pGVM;
+                                        pGVM->aCpus[i].pVMR3 = pVMR3;
+                                        pGVM->apCpusR3[i] = RTR0MemObjAddressR3(pGVM->aCpus[i].gvmm.s.VMCpuMapObj);
+                                        pGVM->aCpus[i].pVCpuR3 = pGVM->apCpusR3[i];
+                                        pGVM->apCpusR0[i] = &pGVM->aCpus[i];
+                                        AssertPtr((void *)pGVM->apCpusR3[i]);
+                                    }
+
+                                    pGVM->paVMPagesR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMPagesMapObj);
+                                    AssertPtr((void *)pGVM->paVMPagesR3);
+
+                                    /*
+                                     * Complete the handle - take the UsedLock sem just to be careful.
+                                     */
+                                    rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
+                                    AssertRC(rc);
+
+                                    pHandle->pGVM                   = pGVM;
+                                    pHandle->hEMT0                  = hEMT0;
+                                    pHandle->ProcId                 = ProcId;
+                                    pGVM->pVMR3                     = pVMR3;
+                                    pGVM->pVMR3Unsafe               = pVMR3;
+                                    pGVM->aCpus[0].hEMT             = hEMT0;
+                                    pGVM->aCpus[0].hNativeThreadR0  = hEMT0;
+                                    pGVMM->cEMTs += cCpus;
+
+                                    /* Associate it with the session and create the context hook for EMT0. */
+                                    rc = SUPR0SetSessionVM(pSession, pGVM, pGVM);
+                                    if (RT_SUCCESS(rc))
+                                    {
+                                        rc = VMMR0ThreadCtxHookCreateForEmt(&pGVM->aCpus[0]);
+                                        if (RT_SUCCESS(rc))
+                                        {
+                                            /*
+                                             * Done!
+                                             */
+                                            VBOXVMM_R0_GVMM_VM_CREATED(pGVM, pGVM, ProcId, (void *)hEMT0, cCpus);
+
+                                            GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
+                                            gvmmR0CreateDestroyUnlock(pGVMM);
+
+                                            CPUMR0RegisterVCpuThread(&pGVM->aCpus[0]);
+
+                                            *ppGVM = pGVM;
+                                            Log(("GVMMR0CreateVM: pVMR3=%p pGVM=%p hGVM=%d\n", pVMR3, pGVM, iHandle));
+                                            return VINF_SUCCESS;
+                                        }
+
+                                        SUPR0SetSessionVM(pSession, NULL, NULL);
+                                    }
+                                    GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
+                                }
+
+                                /* Cleanup mappings. */
+                                if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
+                                {
+                                    RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */);
+                                    pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
+                                }
+                                for (VMCPUID i = 0; i < cCpus; i++)
+                                    if (pGVM->aCpus[i].gvmm.s.VMCpuMapObj != NIL_RTR0MEMOBJ)
+                                    {
+                                        RTR0MemObjFree(pGVM->aCpus[i].gvmm.s.VMCpuMapObj, false /* fFreeMappings */);
+                                        pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
+                                    }
+                                if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
+                                {
+                                    RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */);
+                                    pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
+                                }
+                            }
+                        }
+                        else if (RT_SUCCESS(rc))
+                            rc = rc2;
+                    }
+                }
+                /* else: The user wasn't permitted to create this VM. */
+
+                /*
+                 * The handle will be freed by gvmmR0HandleObjDestructor as we release the
+                 * object reference here. A little extra mess because of non-recursive lock.
+                 */
+                void *pvObj = pHandle->pvObj;
+                pHandle->pvObj = NULL;
+                gvmmR0CreateDestroyUnlock(pGVMM);
+
+                SUPR0ObjRelease(pvObj, pSession);
+
+                SUPR0Printf("GVMMR0CreateVM: failed, rc=%Rrc\n", rc);
+                return rc;
+            }
+
+            rc = VERR_NO_MEMORY;
+        }
+        else
+            rc = VERR_GVMM_IPE_1;
+    }
+    else
+        rc = VERR_GVM_TOO_MANY_VMS;
+
+    gvmmR0CreateDestroyUnlock(pGVMM);
+    return rc;
+}
+
+
+/**
+ * Initializes the per VM data belonging to GVMM.
+ *
+ * @param   pGVM        Pointer to the global VM structure.
+ * @param   hSelf       The handle.
+ * @param   cCpus       The CPU count.
+ * @param   pSession    The session this VM is associated with.
+ */
+static void gvmmR0InitPerVMData(PGVM pGVM, int16_t hSelf, VMCPUID cCpus, PSUPDRVSESSION pSession)
+{
+    AssertCompile(RT_SIZEOFMEMB(GVM,gvmm.s) <= RT_SIZEOFMEMB(GVM,gvmm.padding));
+    AssertCompile(RT_SIZEOFMEMB(GVMCPU,gvmm.s) <= RT_SIZEOFMEMB(GVMCPU,gvmm.padding));
+    AssertCompileMemberAlignment(VM, cpum, 64);
+    AssertCompileMemberAlignment(VM, tm, 64);
+
+    /* GVM: */
+    pGVM->u32Magic         = GVM_MAGIC;
+    pGVM->hSelf            = hSelf;
+    pGVM->cCpus            = cCpus;
+    pGVM->pSession         = pSession;
+    pGVM->pSelf            = pGVM;
+
+    /* VM: */
+    pGVM->enmVMState       = VMSTATE_CREATING;
+    pGVM->hSelfUnsafe      = hSelf;
+    pGVM->pSessionUnsafe   = pSession;
+    pGVM->pVMR0ForCall     = pGVM;
+    pGVM->cCpusUnsafe      = cCpus;
+    pGVM->uCpuExecutionCap = 100; /* default is no cap. */
+    pGVM->uStructVersion   = 1;
+    pGVM->cbSelf           = sizeof(VM);
+    pGVM->cbVCpu           = sizeof(VMCPU);
+
+    /* GVMM: */
+    pGVM->gvmm.s.VMMemObj       = NIL_RTR0MEMOBJ;
+    pGVM->gvmm.s.VMMapObj       = NIL_RTR0MEMOBJ;
+    pGVM->gvmm.s.VMPagesMemObj  = NIL_RTR0MEMOBJ;
+    pGVM->gvmm.s.VMPagesMapObj  = NIL_RTR0MEMOBJ;
+    pGVM->gvmm.s.fDoneVMMR0Init = false;
+    pGVM->gvmm.s.fDoneVMMR0Term = false;
+
+    /*
+     * Per virtual CPU.
+     */
+    for (VMCPUID i = 0; i < pGVM->cCpus; i++)
+    {
+        pGVM->aCpus[i].idCpu                 = i;
+        pGVM->aCpus[i].idCpuUnsafe           = i;
+        pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
+        pGVM->aCpus[i].gvmm.s.VMCpuMapObj    = NIL_RTR0MEMOBJ;
+        pGVM->aCpus[i].hEMT                  = NIL_RTNATIVETHREAD;
+        pGVM->aCpus[i].pGVM                  = pGVM;
+        pGVM->aCpus[i].idHostCpu             = NIL_RTCPUID;
+        pGVM->aCpus[i].iHostCpuSet           = UINT32_MAX;
+        pGVM->aCpus[i].hNativeThread         = NIL_RTNATIVETHREAD;
+        pGVM->aCpus[i].hNativeThreadR0       = NIL_RTNATIVETHREAD;
+        pGVM->aCpus[i].enmState              = VMCPUSTATE_STOPPED;
+        pGVM->aCpus[i].pVCpuR0ForVtg         = &pGVM->aCpus[i];
+    }
+}
+
+
+/**
+ * Does the VM initialization.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The global (ring-0) VM structure.
+ */
+GVMMR0DECL(int) GVMMR0InitVM(PGVM pGVM)
+{
+    LogFlow(("GVMMR0InitVM: pGVM=%p\n", pGVM));
+
+    int rc = VERR_INTERNAL_ERROR_3;
+    if (   !pGVM->gvmm.s.fDoneVMMR0Init
+        && pGVM->aCpus[0].gvmm.s.HaltEventMulti == NIL_RTSEMEVENTMULTI)
+    {
+        for (VMCPUID i = 0; i < pGVM->cCpus; i++)
+        {
+            rc = RTSemEventMultiCreate(&pGVM->aCpus[i].gvmm.s.HaltEventMulti);
+            if (RT_FAILURE(rc))
+            {
+                pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
+                break;
+            }
+        }
+    }
+    else
+        rc = VERR_WRONG_ORDER;
+
+    LogFlow(("GVMMR0InitVM: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Indicates that we're done with the ring-0 initialization
+ * of the VM.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @thread  EMT(0)
+ */
+GVMMR0DECL(void) GVMMR0DoneInitVM(PGVM pGVM)
+{
+    /* Set the indicator. */
+    pGVM->gvmm.s.fDoneVMMR0Init = true;
+}
+
+
+/**
+ * Indicates that we're doing the ring-0 termination of the VM.
+ *
+ * @returns true if termination hasn't been done already, false if it has.
+ * @param   pGVM        Pointer to the global VM structure. Optional.
+ * @thread  EMT(0) or session cleanup thread.
+ */
+GVMMR0DECL(bool) GVMMR0DoingTermVM(PGVM pGVM)
+{
+    /* Validate the VM structure, state and handle. */
+    AssertPtrReturn(pGVM, false);
+
+    /* Set the indicator. */
+    if (pGVM->gvmm.s.fDoneVMMR0Term)
+        return false;
+    pGVM->gvmm.s.fDoneVMMR0Term = true;
+    return true;
+}
+
+
+/**
+ * Destroys the VM, freeing all associated resources (the ring-0 ones anyway).
+ *
+ * This is call from the vmR3DestroyFinalBit and from a error path in VMR3Create,
+ * and the caller is not the EMT thread, unfortunately. For security reasons, it
+ * would've been nice if the caller was actually the EMT thread or that we somehow
+ * could've associated the calling thread with the VM up front.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The global (ring-0) VM structure.
+ *
+ * @thread  EMT(0) if it's associated with the VM, otherwise any thread.
+ */
+GVMMR0DECL(int) GVMMR0DestroyVM(PGVM pGVM)
+{
+    LogFlow(("GVMMR0DestroyVM: pGVM=%p\n", pGVM));
+    PGVMM pGVMM;
+    GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
+
+    /*
+     * Validate the VM structure, state and caller.
+     */
+    AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
+    AssertReturn(!((uintptr_t)pGVM & PAGE_OFFSET_MASK), VERR_INVALID_POINTER);
+    AssertMsgReturn(pGVM->enmVMState >= VMSTATE_CREATING && pGVM->enmVMState <= VMSTATE_TERMINATED, ("%d\n", pGVM->enmVMState),
+                    VERR_WRONG_ORDER);
+
+    uint32_t        hGVM = pGVM->hSelf;
+    ASMCompilerBarrier();
+    AssertReturn(hGVM != NIL_GVM_HANDLE, VERR_INVALID_VM_HANDLE);
+    AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_VM_HANDLE);
+
+    PGVMHANDLE      pHandle = &pGVMM->aHandles[hGVM];
+    AssertReturn(pHandle->pGVM == pGVM, VERR_NOT_OWNER);
+
+    RTPROCESS       ProcId = RTProcSelf();
+    RTNATIVETHREAD  hSelf  = RTThreadNativeSelf();
+    AssertReturn(   (   pHandle->hEMT0  == hSelf
+                     && pHandle->ProcId == ProcId)
+                 || pHandle->hEMT0 == NIL_RTNATIVETHREAD, VERR_NOT_OWNER);
+
+    /*
+     * Lookup the handle and destroy the object.
+     * Since the lock isn't recursive and we'll have to leave it before dereferencing the
+     * object, we take some precautions against racing callers just in case...
+     */
+    int rc = gvmmR0CreateDestroyLock(pGVMM);
+    AssertRC(rc);
+
+    /* Be careful here because we might theoretically be racing someone else cleaning up. */
+    if (   pHandle->pGVM == pGVM
+        && (   (   pHandle->hEMT0  == hSelf
+                && pHandle->ProcId == ProcId)
+            || pHandle->hEMT0 == NIL_RTNATIVETHREAD)
+        && RT_VALID_PTR(pHandle->pvObj)
+        && RT_VALID_PTR(pHandle->pSession)
+        && RT_VALID_PTR(pHandle->pGVM)
+        && pHandle->pGVM->u32Magic == GVM_MAGIC)
+    {
+        /* Check that other EMTs have deregistered. */
+        uint32_t cNotDeregistered = 0;
+        for (VMCPUID idCpu = 1; idCpu < pGVM->cCpus; idCpu++)
+            cNotDeregistered += pGVM->aCpus[idCpu].hEMT != ~(RTNATIVETHREAD)1; /* see GVMMR0DeregisterVCpu for the value */
+        if (cNotDeregistered == 0)
+        {
+            /* Grab the object pointer. */
+            void *pvObj = pHandle->pvObj;
+            pHandle->pvObj = NULL;
+            gvmmR0CreateDestroyUnlock(pGVMM);
+
+            SUPR0ObjRelease(pvObj, pHandle->pSession);
+        }
+        else
+        {
+            gvmmR0CreateDestroyUnlock(pGVMM);
+            rc = VERR_GVMM_NOT_ALL_EMTS_DEREGISTERED;
+        }
+    }
+    else
+    {
+        SUPR0Printf("GVMMR0DestroyVM: pHandle=%RKv:{.pGVM=%p, .hEMT0=%p, .ProcId=%u, .pvObj=%p} pGVM=%p hSelf=%p\n",
+                    pHandle, pHandle->pGVM, pHandle->hEMT0, pHandle->ProcId, pHandle->pvObj, pGVM, hSelf);
+        gvmmR0CreateDestroyUnlock(pGVMM);
+        rc = VERR_GVMM_IPE_2;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Performs VM cleanup task as part of object destruction.
+ *
+ * @param   pGVM        The GVM pointer.
+ */
+static void gvmmR0CleanupVM(PGVM pGVM)
+{
+    if (    pGVM->gvmm.s.fDoneVMMR0Init
+        &&  !pGVM->gvmm.s.fDoneVMMR0Term)
+    {
+        if (    pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ
+            &&  RTR0MemObjAddress(pGVM->gvmm.s.VMMemObj) == pGVM)
+        {
+            LogFlow(("gvmmR0CleanupVM: Calling VMMR0TermVM\n"));
+            VMMR0TermVM(pGVM, NIL_VMCPUID);
+        }
+        else
+            AssertMsgFailed(("gvmmR0CleanupVM: VMMemObj=%p pGVM=%p\n", pGVM->gvmm.s.VMMemObj, pGVM));
+    }
+
+    GMMR0CleanupVM(pGVM);
+#ifdef VBOX_WITH_NEM_R0
+    NEMR0CleanupVM(pGVM);
+#endif
+    PDMR0CleanupVM(pGVM);
+    IOMR0CleanupVM(pGVM);
+    PGMR0CleanupVM(pGVM);
+
+    AssertCompile(NIL_RTTHREADCTXHOOK == (RTTHREADCTXHOOK)0); /* Depends on zero initialized memory working for NIL at the moment. */
+    for (VMCPUID idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
+    {
+        /** @todo Can we busy wait here for all thread-context hooks to be
+         *        deregistered before releasing (destroying) it? Only until we find a
+         *        solution for not deregistering hooks everytime we're leaving HMR0
+         *        context. */
+        VMMR0ThreadCtxHookDestroyForEmt(&pGVM->aCpus[idCpu]);
+    }
+}
+
+
+/**
+ * @callback_method_impl{FNSUPDRVDESTRUCTOR,VM handle destructor}
+ *
+ * pvUser1 is the GVM instance pointer.
+ * pvUser2 is the handle pointer.
+ */
+static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void *pvObj, void *pvUser1, void *pvUser2)
+{
+    LogFlow(("gvmmR0HandleObjDestructor: %p %p %p\n", pvObj, pvUser1, pvUser2));
+
+    NOREF(pvObj);
+
+    /*
+     * Some quick, paranoid, input validation.
+     */
+    PGVMHANDLE pHandle = (PGVMHANDLE)pvUser2;
+    AssertPtr(pHandle);
+    PGVMM pGVMM = (PGVMM)pvUser1;
+    Assert(pGVMM == g_pGVMM);
+    const uint16_t iHandle = pHandle - &pGVMM->aHandles[0];
+    if (    !iHandle
+        ||  iHandle >= RT_ELEMENTS(pGVMM->aHandles)
+        ||  iHandle != pHandle->iSelf)
+    {
+        SUPR0Printf("GVM: handle %d is out of range or corrupt (iSelf=%d)!\n", iHandle, pHandle->iSelf);
+        return;
+    }
+
+    int rc = gvmmR0CreateDestroyLock(pGVMM);
+    AssertRC(rc);
+    rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
+    AssertRC(rc);
+
+    /*
+     * This is a tad slow but a doubly linked list is too much hassle.
+     */
+    if (RT_UNLIKELY(pHandle->iNext >= RT_ELEMENTS(pGVMM->aHandles)))
+    {
+        SUPR0Printf("GVM: used list index %d is out of range!\n", pHandle->iNext);
+        GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
+        gvmmR0CreateDestroyUnlock(pGVMM);
+        return;
+    }
+
+    if (pGVMM->iUsedHead == iHandle)
+        pGVMM->iUsedHead = pHandle->iNext;
+    else
+    {
+        uint16_t iPrev = pGVMM->iUsedHead;
+        int c = RT_ELEMENTS(pGVMM->aHandles) + 2;
+        while (iPrev)
+        {
+            if (RT_UNLIKELY(iPrev >= RT_ELEMENTS(pGVMM->aHandles)))
+            {
+                SUPR0Printf("GVM: used list index %d is out of range!\n", iPrev);
+                GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
+                gvmmR0CreateDestroyUnlock(pGVMM);
+                return;
+            }
+            if (RT_UNLIKELY(c-- <= 0))
+            {
+                iPrev = 0;
+                break;
+            }
+
+            if (pGVMM->aHandles[iPrev].iNext == iHandle)
+                break;
+            iPrev = pGVMM->aHandles[iPrev].iNext;
+        }
+        if (!iPrev)
+        {
+            SUPR0Printf("GVM: can't find the handle previous previous of %d!\n", pHandle->iSelf);
+            GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
+            gvmmR0CreateDestroyUnlock(pGVMM);
+            return;
+        }
+
+        Assert(pGVMM->aHandles[iPrev].iNext == iHandle);
+        pGVMM->aHandles[iPrev].iNext = pHandle->iNext;
+    }
+    pHandle->iNext = 0;
+    pGVMM->cVMs--;
+
+    /*
+     * Do the global cleanup round.
+     */
+    PGVM pGVM = pHandle->pGVM;
+    if (   RT_VALID_PTR(pGVM)
+        && pGVM->u32Magic == GVM_MAGIC)
+    {
+        pGVMM->cEMTs -= pGVM->cCpus;
+
+        if (pGVM->pSession)
+            SUPR0SetSessionVM(pGVM->pSession, NULL, NULL);
+
+        GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
+
+        gvmmR0CleanupVM(pGVM);
+
+        /*
+         * Do the GVMM cleanup - must be done last.
+         */
+        /* The VM and VM pages mappings/allocations. */
+        if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
+        {
+            rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */); AssertRC(rc);
+            pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
+        }
+
+        if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
+        {
+            rc = RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */); AssertRC(rc);
+            pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
+        }
+
+        if (pGVM->gvmm.s.VMPagesMemObj != NIL_RTR0MEMOBJ)
+        {
+            rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMemObj, false /* fFreeMappings */); AssertRC(rc);
+            pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
+        }
+
+        for (VMCPUID i = 0; i < pGVM->cCpus; i++)
+        {
+            if (pGVM->aCpus[i].gvmm.s.HaltEventMulti != NIL_RTSEMEVENTMULTI)
+            {
+                rc = RTSemEventMultiDestroy(pGVM->aCpus[i].gvmm.s.HaltEventMulti); AssertRC(rc);
+                pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
+            }
+            if (pGVM->aCpus[i].gvmm.s.VMCpuMapObj != NIL_RTR0MEMOBJ)
+            {
+                rc = RTR0MemObjFree(pGVM->aCpus[i].gvmm.s.VMCpuMapObj, false /* fFreeMappings */); AssertRC(rc);
+                pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
+            }
+        }
+
+        /* the GVM structure itself. */
+        pGVM->u32Magic |= UINT32_C(0x80000000);
+        Assert(pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ);
+        rc = RTR0MemObjFree(pGVM->gvmm.s.VMMemObj, true /*fFreeMappings*/); AssertRC(rc);
+        pGVM = NULL;
+
+        /* Re-acquire the UsedLock before freeing the handle since we're updating handle fields. */
+        rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
+        AssertRC(rc);
+    }
+    /* else: GVMMR0CreateVM cleanup. */
+
+    /*
+     * Free the handle.
+     */
+    pHandle->iNext = pGVMM->iFreeHead;
+    pGVMM->iFreeHead = iHandle;
+    ASMAtomicWriteNullPtr(&pHandle->pGVM);
+    ASMAtomicWriteNullPtr(&pHandle->pvObj);
+    ASMAtomicWriteNullPtr(&pHandle->pSession);
+    ASMAtomicWriteHandle(&pHandle->hEMT0,        NIL_RTNATIVETHREAD);
+    ASMAtomicWriteU32(&pHandle->ProcId,          NIL_RTPROCESS);
+
+    GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
+    gvmmR0CreateDestroyUnlock(pGVMM);
+    LogFlow(("gvmmR0HandleObjDestructor: returns\n"));
+}
+
+
+/**
+ * Registers the calling thread as the EMT of a Virtual CPU.
+ *
+ * Note that VCPU 0 is automatically registered during VM creation.
+ *
+ * @returns VBox status code
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       VCPU id to register the current thread as.
+ */
+GVMMR0DECL(int) GVMMR0RegisterVCpu(PGVM pGVM, VMCPUID idCpu)
+{
+    AssertReturn(idCpu != 0, VERR_INVALID_FUNCTION);
+
+    /*
+     * Validate the VM structure, state and handle.
+     */
+    PGVMM pGVMM;
+    int rc = gvmmR0ByGVM(pGVM, &pGVMM, false /* fTakeUsedLock */); /** @todo take lock here. */
+    if (RT_SUCCESS(rc))
+    {
+        if (idCpu < pGVM->cCpus)
+        {
+            /* Check that the EMT isn't already assigned to a thread. */
+            if (pGVM->aCpus[idCpu].hEMT == NIL_RTNATIVETHREAD)
+            {
+                Assert(pGVM->aCpus[idCpu].hNativeThreadR0 == NIL_RTNATIVETHREAD);
+
+                /* A thread may only be one EMT. */
+                RTNATIVETHREAD const hNativeSelf = RTThreadNativeSelf();
+                for (VMCPUID iCpu = 0; iCpu < pGVM->cCpus; iCpu++)
+                    AssertBreakStmt(pGVM->aCpus[iCpu].hEMT != hNativeSelf, rc = VERR_INVALID_PARAMETER);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Do the assignment, then try setup the hook. Undo if that fails.
+                     */
+                    pGVM->aCpus[idCpu].hNativeThreadR0 = pGVM->aCpus[idCpu].hEMT = RTThreadNativeSelf();
+
+                    rc = VMMR0ThreadCtxHookCreateForEmt(&pGVM->aCpus[idCpu]);
+                    if (RT_SUCCESS(rc))
+                        CPUMR0RegisterVCpuThread(&pGVM->aCpus[idCpu]);
+                    else
+                        pGVM->aCpus[idCpu].hNativeThreadR0 = pGVM->aCpus[idCpu].hEMT = NIL_RTNATIVETHREAD;
+                }
+            }
+            else
+                rc = VERR_ACCESS_DENIED;
+        }
+        else
+            rc = VERR_INVALID_CPU_ID;
+    }
+    return rc;
+}
+
+
+/**
+ * Deregisters the calling thread as the EMT of a Virtual CPU.
+ *
+ * Note that VCPU 0 shall call GVMMR0DestroyVM intead of this API.
+ *
+ * @returns VBox status code
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       VCPU id to register the current thread as.
+ */
+GVMMR0DECL(int) GVMMR0DeregisterVCpu(PGVM pGVM, VMCPUID idCpu)
+{
+    AssertReturn(idCpu != 0, VERR_INVALID_FUNCTION);
+
+    /*
+     * Validate the VM structure, state and handle.
+     */
+    PGVMM pGVMM;
+    int rc = gvmmR0ByGVMandEMT(pGVM, idCpu, &pGVMM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Take the destruction lock and recheck the handle state to
+         * prevent racing GVMMR0DestroyVM.
+         */
+        gvmmR0CreateDestroyLock(pGVMM);
+        uint32_t hSelf = pGVM->hSelf;
+        ASMCompilerBarrier();
+        if (   hSelf < RT_ELEMENTS(pGVMM->aHandles)
+            && pGVMM->aHandles[hSelf].pvObj != NULL
+            && pGVMM->aHandles[hSelf].pGVM  == pGVM)
+        {
+            /*
+             * Do per-EMT cleanups.
+             */
+            VMMR0ThreadCtxHookDestroyForEmt(&pGVM->aCpus[idCpu]);
+
+            /*
+             * Invalidate hEMT.  We don't use NIL here as that would allow
+             * GVMMR0RegisterVCpu to be called again, and we don't want that.
+             */
+            AssertCompile(~(RTNATIVETHREAD)1 != NIL_RTNATIVETHREAD);
+            pGVM->aCpus[idCpu].hEMT           = ~(RTNATIVETHREAD)1;
+            pGVM->aCpus[idCpu].hNativeThreadR0 = NIL_RTNATIVETHREAD;
+        }
+
+        gvmmR0CreateDestroyUnlock(pGVMM);
+    }
+    return rc;
+}
+
+
+/**
+ * Lookup a GVM structure by its handle.
+ *
+ * @returns The GVM pointer on success, NULL on failure.
+ * @param   hGVM    The global VM handle. Asserts on bad handle.
+ */
+GVMMR0DECL(PGVM) GVMMR0ByHandle(uint32_t hGVM)
+{
+    PGVMM pGVMM;
+    GVMM_GET_VALID_INSTANCE(pGVMM, NULL);
+
+    /*
+     * Validate.
+     */
+    AssertReturn(hGVM != NIL_GVM_HANDLE, NULL);
+    AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), NULL);
+
+    /*
+     * Look it up.
+     */
+    PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
+    AssertPtrReturn(pHandle->pvObj, NULL);
+    PGVM pGVM = pHandle->pGVM;
+    AssertPtrReturn(pGVM, NULL);
+
+    return pGVM;
+}
+
+
+/**
+ * Check that the given GVM and VM structures match up.
+ *
+ * The calling thread must be in the same process as the VM. All current lookups
+ * are by threads inside the same process, so this will not be an issue.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   ppGVMM          Where to store the pointer to the GVMM instance data.
+ * @param   fTakeUsedLock   Whether to take the used lock or not.  We take it in
+ *                          shared mode when requested.
+ *
+ *                          Be very careful if not taking the lock as it's
+ *                          possible that the VM will disappear then!
+ *
+ * @remark  This will not assert on an invalid pGVM but try return silently.
+ */
+static int gvmmR0ByGVM(PGVM pGVM, PGVMM *ppGVMM, bool fTakeUsedLock)
+{
+    /*
+     * Check the pointers.
+     */
+    int rc;
+    if (RT_LIKELY(   RT_VALID_PTR(pGVM)
+                  && ((uintptr_t)pGVM & PAGE_OFFSET_MASK) == 0 ))
+    {
+        /*
+         * Get the pGVMM instance and check the VM handle.
+         */
+        PGVMM pGVMM;
+        GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
+
+        uint16_t hGVM = pGVM->hSelf;
+        if (RT_LIKELY(   hGVM != NIL_GVM_HANDLE
+                      && hGVM < RT_ELEMENTS(pGVMM->aHandles)))
+        {
+            RTPROCESS const pidSelf = RTProcSelf();
+            PGVMHANDLE      pHandle = &pGVMM->aHandles[hGVM];
+            if (fTakeUsedLock)
+            {
+                rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
+                AssertRCReturn(rc, rc);
+            }
+
+            if (RT_LIKELY(   pHandle->pGVM   == pGVM
+                          && pHandle->ProcId == pidSelf
+                          && RT_VALID_PTR(pHandle->pvObj)))
+            {
+                /*
+                 * Some more VM data consistency checks.
+                 */
+                if (RT_LIKELY(   pGVM->cCpusUnsafe == pGVM->cCpus
+                              && pGVM->hSelfUnsafe == hGVM
+                              && pGVM->pSelf       == pGVM))
+                {
+                    if (RT_LIKELY(   pGVM->enmVMState >= VMSTATE_CREATING
+                                  && pGVM->enmVMState <= VMSTATE_TERMINATED))
+                    {
+                        *ppGVMM = pGVMM;
+                        return VINF_SUCCESS;
+                    }
+                    rc = VERR_INCONSISTENT_VM_HANDLE;
+                }
+                else
+                    rc = VERR_INCONSISTENT_VM_HANDLE;
+            }
+            else
+                rc = VERR_INVALID_VM_HANDLE;
+
+            if (fTakeUsedLock)
+                GVMMR0_USED_SHARED_UNLOCK(pGVMM);
+        }
+        else
+            rc = VERR_INVALID_VM_HANDLE;
+    }
+    else
+        rc = VERR_INVALID_POINTER;
+    return rc;
+}
+
+
+/**
+ * Validates a GVM/VM pair.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The global (ring-0) VM structure.
+ */
+GVMMR0DECL(int) GVMMR0ValidateGVM(PGVM pGVM)
+{
+    PGVMM pGVMM;
+    return gvmmR0ByGVM(pGVM, &pGVMM, false /*fTakeUsedLock*/);
+}
+
+
+/**
+ * Check that the given GVM and VM structures match up.
+ *
+ * The calling thread must be in the same process as the VM. All current lookups
+ * are by threads inside the same process, so this will not be an issue.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   idCpu           The (alleged) Virtual CPU ID of the calling EMT.
+ * @param   ppGVMM          Where to store the pointer to the GVMM instance data.
+ * @thread  EMT
+ *
+ * @remarks This will assert in all failure paths.
+ */
+static int gvmmR0ByGVMandEMT(PGVM pGVM, VMCPUID idCpu, PGVMM *ppGVMM)
+{
+    /*
+     * Check the pointers.
+     */
+    AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
+    AssertReturn(((uintptr_t)pGVM & PAGE_OFFSET_MASK) == 0, VERR_INVALID_POINTER);
+
+    /*
+     * Get the pGVMM instance and check the VM handle.
+     */
+    PGVMM pGVMM;
+    GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
+
+    uint16_t hGVM = pGVM->hSelf;
+    ASMCompilerBarrier();
+    AssertReturn(   hGVM != NIL_GVM_HANDLE
+                 && hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_VM_HANDLE);
+
+    RTPROCESS const pidSelf = RTProcSelf();
+    PGVMHANDLE      pHandle = &pGVMM->aHandles[hGVM];
+    AssertReturn(   pHandle->pGVM   == pGVM
+                 && pHandle->ProcId == pidSelf
+                 && RT_VALID_PTR(pHandle->pvObj),
+                 VERR_INVALID_HANDLE);
+
+    /*
+     * Check the EMT claim.
+     */
+    RTNATIVETHREAD const hAllegedEMT = RTThreadNativeSelf();
+    AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
+    AssertReturn(pGVM->aCpus[idCpu].hEMT == hAllegedEMT, VERR_NOT_OWNER);
+
+    /*
+     * Some more VM data consistency checks.
+     */
+    AssertReturn(pGVM->cCpusUnsafe == pGVM->cCpus, VERR_INCONSISTENT_VM_HANDLE);
+    AssertReturn(pGVM->hSelfUnsafe == hGVM, VERR_INCONSISTENT_VM_HANDLE);
+    AssertReturn(   pGVM->enmVMState >= VMSTATE_CREATING
+                 && pGVM->enmVMState <= VMSTATE_TERMINATED, VERR_INCONSISTENT_VM_HANDLE);
+
+    *ppGVMM = pGVMM;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Validates a GVM/EMT pair.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The Virtual CPU ID of the calling EMT.
+ * @thread  EMT(idCpu)
+ */
+GVMMR0DECL(int) GVMMR0ValidateGVMandEMT(PGVM pGVM, VMCPUID idCpu)
+{
+    PGVMM pGVMM;
+    return gvmmR0ByGVMandEMT(pGVM, idCpu, &pGVMM);
+}
+
+
+/**
+ * Looks up the VM belonging to the specified EMT thread.
+ *
+ * This is used by the assertion machinery in VMMR0.cpp to avoid causing
+ * unnecessary kernel panics when the EMT thread hits an assertion. The
+ * call may or not be an EMT thread.
+ *
+ * @returns Pointer to the VM on success, NULL on failure.
+ * @param   hEMT    The native thread handle of the EMT.
+ *                  NIL_RTNATIVETHREAD means the current thread
+ */
+GVMMR0DECL(PVMCC) GVMMR0GetVMByEMT(RTNATIVETHREAD hEMT)
+{
+    /*
+     * No Assertions here as we're usually called in a AssertMsgN or
+     * RTAssert* context.
+     */
+    PGVMM pGVMM = g_pGVMM;
+    if (    !RT_VALID_PTR(pGVMM)
+        ||  pGVMM->u32Magic != GVMM_MAGIC)
+        return NULL;
+
+    if (hEMT == NIL_RTNATIVETHREAD)
+        hEMT = RTThreadNativeSelf();
+    RTPROCESS ProcId = RTProcSelf();
+
+    /*
+     * Search the handles in a linear fashion as we don't dare to take the lock (assert).
+     */
+/** @todo introduce some pid hash table here, please. */
+    for (unsigned i = 1; i < RT_ELEMENTS(pGVMM->aHandles); i++)
+    {
+        if (    pGVMM->aHandles[i].iSelf == i
+            &&  pGVMM->aHandles[i].ProcId == ProcId
+            &&  RT_VALID_PTR(pGVMM->aHandles[i].pvObj)
+            &&  RT_VALID_PTR(pGVMM->aHandles[i].pGVM))
+        {
+            if (pGVMM->aHandles[i].hEMT0 == hEMT)
+                return pGVMM->aHandles[i].pGVM;
+
+            /* This is fearly safe with the current process per VM approach. */
+            PGVM pGVM = pGVMM->aHandles[i].pGVM;
+            VMCPUID const cCpus = pGVM->cCpus;
+            ASMCompilerBarrier();
+            if (    cCpus < 1
+                ||  cCpus > VMM_MAX_CPU_COUNT)
+                continue;
+            for (VMCPUID idCpu = 1; idCpu < cCpus; idCpu++)
+                if (pGVM->aCpus[idCpu].hEMT == hEMT)
+                    return pGVMM->aHandles[i].pGVM;
+        }
+    }
+    return NULL;
+}
+
+
+/**
+ * Looks up the GVMCPU belonging to the specified EMT thread.
+ *
+ * This is used by the assertion machinery in VMMR0.cpp to avoid causing
+ * unnecessary kernel panics when the EMT thread hits an assertion. The
+ * call may or not be an EMT thread.
+ *
+ * @returns Pointer to the VM on success, NULL on failure.
+ * @param   hEMT    The native thread handle of the EMT.
+ *                  NIL_RTNATIVETHREAD means the current thread
+ */
+GVMMR0DECL(PGVMCPU) GVMMR0GetGVCpuByEMT(RTNATIVETHREAD hEMT)
+{
+    /*
+     * No Assertions here as we're usually called in a AssertMsgN,
+     * RTAssert*, Log and LogRel contexts.
+     */
+    PGVMM pGVMM = g_pGVMM;
+    if (   !RT_VALID_PTR(pGVMM)
+        || pGVMM->u32Magic != GVMM_MAGIC)
+        return NULL;
+
+    if (hEMT == NIL_RTNATIVETHREAD)
+        hEMT = RTThreadNativeSelf();
+    RTPROCESS ProcId = RTProcSelf();
+
+    /*
+     * Search the handles in a linear fashion as we don't dare to take the lock (assert).
+     */
+/** @todo introduce some pid hash table here, please. */
+    for (unsigned i = 1; i < RT_ELEMENTS(pGVMM->aHandles); i++)
+    {
+        if (   pGVMM->aHandles[i].iSelf == i
+            && pGVMM->aHandles[i].ProcId == ProcId
+            && RT_VALID_PTR(pGVMM->aHandles[i].pvObj)
+            && RT_VALID_PTR(pGVMM->aHandles[i].pGVM))
+        {
+            PGVM pGVM = pGVMM->aHandles[i].pGVM;
+            if (pGVMM->aHandles[i].hEMT0 == hEMT)
+                return &pGVM->aCpus[0];
+
+            /* This is fearly safe with the current process per VM approach. */
+            VMCPUID const cCpus = pGVM->cCpus;
+            ASMCompilerBarrier();
+            ASMCompilerBarrier();
+            if (   cCpus < 1
+                || cCpus > VMM_MAX_CPU_COUNT)
+                continue;
+            for (VMCPUID idCpu = 1; idCpu < cCpus; idCpu++)
+                if (pGVM->aCpus[idCpu].hEMT == hEMT)
+                    return &pGVM->aCpus[idCpu];
+        }
+    }
+    return NULL;
+}
+
+
+/**
+ * This is will wake up expired and soon-to-be expired VMs.
+ *
+ * @returns Number of VMs that has been woken up.
+ * @param   pGVMM       Pointer to the GVMM instance data.
+ * @param   u64Now      The current time.
+ */
+static unsigned gvmmR0SchedDoWakeUps(PGVMM pGVMM, uint64_t u64Now)
+{
+    /*
+     * Skip this if we've got disabled because of high resolution wakeups or by
+     * the user.
+     */
+    if (!pGVMM->fDoEarlyWakeUps)
+        return 0;
+
+/** @todo Rewrite this algorithm. See performance defect XYZ. */
+
+    /*
+     * A cheap optimization to stop wasting so much time here on big setups.
+     */
+    const uint64_t  uNsEarlyWakeUp2 = u64Now + pGVMM->nsEarlyWakeUp2;
+    if (   pGVMM->cHaltedEMTs == 0
+        || uNsEarlyWakeUp2 > pGVMM->uNsNextEmtWakeup)
+        return 0;
+
+    /*
+     * Only one thread doing this at a time.
+     */
+    if (!ASMAtomicCmpXchgBool(&pGVMM->fDoingEarlyWakeUps, true, false))
+        return 0;
+
+    /*
+     * The first pass will wake up VMs which have actually expired
+     * and look for VMs that should be woken up in the 2nd and 3rd passes.
+     */
+    const uint64_t  uNsEarlyWakeUp1 = u64Now + pGVMM->nsEarlyWakeUp1;
+    uint64_t        u64Min          = UINT64_MAX;
+    unsigned        cWoken          = 0;
+    unsigned        cHalted         = 0;
+    unsigned        cTodo2nd        = 0;
+    unsigned        cTodo3rd        = 0;
+    for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
+         i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
+         i = pGVMM->aHandles[i].iNext)
+    {
+        PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
+        if (    RT_VALID_PTR(pCurGVM)
+            &&  pCurGVM->u32Magic == GVM_MAGIC)
+        {
+            for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
+            {
+                PGVMCPU     pCurGVCpu = &pCurGVM->aCpus[idCpu];
+                uint64_t    u64       = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
+                if (u64)
+                {
+                    if (u64 <= u64Now)
+                    {
+                        if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
+                        {
+                            int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
+                            AssertRC(rc);
+                            cWoken++;
+                        }
+                    }
+                    else
+                    {
+                        cHalted++;
+                        if (u64 <= uNsEarlyWakeUp1)
+                            cTodo2nd++;
+                        else if (u64 <= uNsEarlyWakeUp2)
+                            cTodo3rd++;
+                        else if (u64 < u64Min)
+                            u64 = u64Min;
+                    }
+                }
+            }
+        }
+        AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
+    }
+
+    if (cTodo2nd)
+    {
+        for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
+             i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
+             i = pGVMM->aHandles[i].iNext)
+        {
+            PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
+            if (    RT_VALID_PTR(pCurGVM)
+                &&  pCurGVM->u32Magic == GVM_MAGIC)
+            {
+                for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
+                {
+                    PGVMCPU     pCurGVCpu = &pCurGVM->aCpus[idCpu];
+                    uint64_t    u64       = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
+                    if (   u64
+                        && u64 <= uNsEarlyWakeUp1)
+                    {
+                        if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
+                        {
+                            int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
+                            AssertRC(rc);
+                            cWoken++;
+                        }
+                    }
+                }
+            }
+            AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
+        }
+    }
+
+    if (cTodo3rd)
+    {
+        for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
+             i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
+             i = pGVMM->aHandles[i].iNext)
+        {
+            PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
+            if (    RT_VALID_PTR(pCurGVM)
+                &&  pCurGVM->u32Magic == GVM_MAGIC)
+            {
+                for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
+                {
+                    PGVMCPU     pCurGVCpu = &pCurGVM->aCpus[idCpu];
+                    uint64_t    u64       = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
+                    if (   u64
+                        && u64 <= uNsEarlyWakeUp2)
+                    {
+                        if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
+                        {
+                            int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
+                            AssertRC(rc);
+                            cWoken++;
+                        }
+                    }
+                }
+            }
+            AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
+        }
+    }
+
+    /*
+     * Set the minimum value.
+     */
+    pGVMM->uNsNextEmtWakeup = u64Min;
+
+    ASMAtomicWriteBool(&pGVMM->fDoingEarlyWakeUps, false);
+    return cWoken;
+}
+
+
+/**
+ * Halt the EMT thread.
+ *
+ * @returns VINF_SUCCESS normal wakeup (timeout or kicked by other thread).
+ *          VERR_INTERRUPTED if a signal was scheduled for the thread.
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   pGVCpu              The global (ring-0) CPU structure of the calling
+ *                              EMT.
+ * @param   u64ExpireGipTime    The time for the sleep to expire expressed as GIP time.
+ * @thread  EMT(pGVCpu).
+ */
+GVMMR0DECL(int) GVMMR0SchedHalt(PGVM pGVM, PGVMCPU pGVCpu, uint64_t u64ExpireGipTime)
+{
+    LogFlow(("GVMMR0SchedHalt: pGVM=%p pGVCpu=%p(%d) u64ExpireGipTime=%#RX64\n",
+             pGVM, pGVCpu, pGVCpu->idCpu, u64ExpireGipTime));
+    GVMM_CHECK_SMAP_SETUP();
+    GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+
+    PGVMM pGVMM;
+    GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
+
+    pGVM->gvmm.s.StatsSched.cHaltCalls++;
+    Assert(!pGVCpu->gvmm.s.u64HaltExpire);
+
+    /*
+     * If we're doing early wake-ups, we must take the UsedList lock before we
+     * start querying the current time.
+     * Note! Interrupts must NOT be disabled at this point because we ask for GIP time!
+     */
+    bool const fDoEarlyWakeUps = pGVMM->fDoEarlyWakeUps;
+    if (fDoEarlyWakeUps)
+    {
+        int rc2 = GVMMR0_USED_SHARED_LOCK(pGVMM); AssertRC(rc2);
+        GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+    }
+
+    pGVCpu->gvmm.s.iCpuEmt = ASMGetApicId();
+
+    /* GIP hack: We might are frequently sleeping for short intervals where the
+       difference between GIP and system time matters on systems with high resolution
+       system time. So, convert the input from GIP to System time in that case. */
+    Assert(ASMGetFlags() & X86_EFL_IF);
+    const uint64_t u64NowSys = RTTimeSystemNanoTS();
+    const uint64_t u64NowGip = RTTimeNanoTS();
+    GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+
+    if (fDoEarlyWakeUps)
+    {
+        pGVM->gvmm.s.StatsSched.cHaltWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64NowGip);
+        GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+    }
+
+    /*
+     * Go to sleep if we must...
+     * Cap the sleep time to 1 second to be on the safe side.
+     */
+    int rc;
+    uint64_t cNsInterval = u64ExpireGipTime - u64NowGip;
+    if (    u64NowGip < u64ExpireGipTime
+        &&  cNsInterval >= (pGVMM->cEMTs > pGVMM->cEMTsMeansCompany
+                            ? pGVMM->nsMinSleepCompany
+                            : pGVMM->nsMinSleepAlone))
+    {
+        pGVM->gvmm.s.StatsSched.cHaltBlocking++;
+        if (cNsInterval > RT_NS_1SEC)
+            u64ExpireGipTime = u64NowGip + RT_NS_1SEC;
+        ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, u64ExpireGipTime);
+        ASMAtomicIncU32(&pGVMM->cHaltedEMTs);
+        if (fDoEarlyWakeUps)
+        {
+            if (u64ExpireGipTime < pGVMM->uNsNextEmtWakeup)
+                pGVMM->uNsNextEmtWakeup = u64ExpireGipTime;
+            GVMMR0_USED_SHARED_UNLOCK(pGVMM);
+        }
+        GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+
+        rc = RTSemEventMultiWaitEx(pGVCpu->gvmm.s.HaltEventMulti,
+                                   RTSEMWAIT_FLAGS_ABSOLUTE | RTSEMWAIT_FLAGS_NANOSECS | RTSEMWAIT_FLAGS_INTERRUPTIBLE,
+                                   u64NowGip > u64NowSys ? u64ExpireGipTime : u64NowSys + cNsInterval);
+        GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+
+        ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, 0);
+        ASMAtomicDecU32(&pGVMM->cHaltedEMTs);
+
+        /* Reset the semaphore to try prevent a few false wake-ups. */
+        if (rc == VINF_SUCCESS)
+        {
+            RTSemEventMultiReset(pGVCpu->gvmm.s.HaltEventMulti);
+            GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+        }
+        else if (rc == VERR_TIMEOUT)
+        {
+            pGVM->gvmm.s.StatsSched.cHaltTimeouts++;
+            rc = VINF_SUCCESS;
+        }
+    }
+    else
+    {
+        pGVM->gvmm.s.StatsSched.cHaltNotBlocking++;
+        if (fDoEarlyWakeUps)
+            GVMMR0_USED_SHARED_UNLOCK(pGVMM);
+        GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+        RTSemEventMultiReset(pGVCpu->gvmm.s.HaltEventMulti);
+        GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+        rc = VINF_SUCCESS;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Halt the EMT thread.
+ *
+ * @returns VINF_SUCCESS normal wakeup (timeout or kicked by other thread).
+ *          VERR_INTERRUPTED if a signal was scheduled for the thread.
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   idCpu               The Virtual CPU ID of the calling EMT.
+ * @param   u64ExpireGipTime    The time for the sleep to expire expressed as GIP time.
+ * @thread  EMT(idCpu).
+ */
+GVMMR0DECL(int) GVMMR0SchedHaltReq(PGVM pGVM, VMCPUID idCpu, uint64_t u64ExpireGipTime)
+{
+    GVMM_CHECK_SMAP_SETUP();
+    GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+    PGVMM pGVMM;
+    int rc = gvmmR0ByGVMandEMT(pGVM, idCpu, &pGVMM);
+    if (RT_SUCCESS(rc))
+    {
+        GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+        rc = GVMMR0SchedHalt(pGVM, &pGVM->aCpus[idCpu], u64ExpireGipTime);
+    }
+    GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+    return rc;
+}
+
+
+
+/**
+ * Worker for GVMMR0SchedWakeUp and GVMMR0SchedWakeUpAndPokeCpus that wakes up
+ * the a sleeping EMT.
+ *
+ * @retval  VINF_SUCCESS if successfully woken up.
+ * @retval  VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   pGVCpu              The global (ring-0) VCPU structure.
+ */
+DECLINLINE(int) gvmmR0SchedWakeUpOne(PGVM pGVM, PGVMCPU pGVCpu)
+{
+    pGVM->gvmm.s.StatsSched.cWakeUpCalls++;
+
+    /*
+     * Signal the semaphore regardless of whether it's current blocked on it.
+     *
+     * The reason for this is that there is absolutely no way we can be 100%
+     * certain that it isn't *about* go to go to sleep on it and just got
+     * delayed a bit en route. So, we will always signal the semaphore when
+     * the it is flagged as halted in the VMM.
+     */
+/** @todo we can optimize some of that by means of the pVCpu->enmState now. */
+    int rc;
+    if (pGVCpu->gvmm.s.u64HaltExpire)
+    {
+        rc = VINF_SUCCESS;
+        ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, 0);
+    }
+    else
+    {
+        rc = VINF_GVM_NOT_BLOCKED;
+        pGVM->gvmm.s.StatsSched.cWakeUpNotHalted++;
+    }
+
+    int rc2 = RTSemEventMultiSignal(pGVCpu->gvmm.s.HaltEventMulti);
+    AssertRC(rc2);
+
+    return rc;
+}
+
+
+/**
+ * Wakes up the halted EMT thread so it can service a pending request.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully woken up.
+ * @retval  VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   idCpu               The Virtual CPU ID of the EMT to wake up.
+ * @param   fTakeUsedLock       Take the used lock or not
+ * @thread  Any but EMT(idCpu).
+ */
+GVMMR0DECL(int) GVMMR0SchedWakeUpEx(PGVM pGVM, VMCPUID idCpu, bool fTakeUsedLock)
+{
+    GVMM_CHECK_SMAP_SETUP();
+    GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+
+    /*
+     * Validate input and take the UsedLock.
+     */
+    PGVMM pGVMM;
+    int rc = gvmmR0ByGVM(pGVM, &pGVMM, fTakeUsedLock);
+    GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+    if (RT_SUCCESS(rc))
+    {
+        if (idCpu < pGVM->cCpus)
+        {
+            /*
+             * Do the actual job.
+             */
+            rc = gvmmR0SchedWakeUpOne(pGVM, &pGVM->aCpus[idCpu]);
+            GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+
+            if (fTakeUsedLock && pGVMM->fDoEarlyWakeUps)
+            {
+                /*
+                 * While we're here, do a round of scheduling.
+                 */
+                Assert(ASMGetFlags() & X86_EFL_IF);
+                const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
+                pGVM->gvmm.s.StatsSched.cWakeUpWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
+                GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            }
+        }
+        else
+            rc = VERR_INVALID_CPU_ID;
+
+        if (fTakeUsedLock)
+        {
+            int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
+            AssertRC(rc2);
+            GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+        }
+    }
+
+    LogFlow(("GVMMR0SchedWakeUpEx: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Wakes up the halted EMT thread so it can service a pending request.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully woken up.
+ * @retval  VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   idCpu               The Virtual CPU ID of the EMT to wake up.
+ * @thread  Any but EMT(idCpu).
+ */
+GVMMR0DECL(int) GVMMR0SchedWakeUp(PGVM pGVM, VMCPUID idCpu)
+{
+    return GVMMR0SchedWakeUpEx(pGVM, idCpu, true /* fTakeUsedLock */);
+}
+
+
+/**
+ * Wakes up the halted EMT thread so it can service a pending request, no GVM
+ * parameter and no used locking.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully woken up.
+ * @retval  VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   idCpu               The Virtual CPU ID of the EMT to wake up.
+ * @thread  Any but EMT(idCpu).
+ * @deprecated  Don't use in new code if possible!  Use the GVM variant.
+ */
+GVMMR0DECL(int) GVMMR0SchedWakeUpNoGVMNoLock(PGVM pGVM, VMCPUID idCpu)
+{
+    GVMM_CHECK_SMAP_SETUP();
+    GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+    PGVMM pGVMM;
+    int rc = gvmmR0ByGVM(pGVM, &pGVMM, false /*fTakeUsedLock*/);
+    GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+    if (RT_SUCCESS(rc))
+        rc = GVMMR0SchedWakeUpEx(pGVM, idCpu, false /*fTakeUsedLock*/);
+    return rc;
+}
+
+
+/**
+ * Worker common to GVMMR0SchedPoke and GVMMR0SchedWakeUpAndPokeCpus that pokes
+ * the Virtual CPU if it's still busy executing guest code.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if poked successfully.
+ * @retval  VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+DECLINLINE(int) gvmmR0SchedPokeOne(PGVM pGVM, PVMCPUCC pVCpu)
+{
+    pGVM->gvmm.s.StatsSched.cPokeCalls++;
+
+    RTCPUID idHostCpu = pVCpu->idHostCpu;
+    if (    idHostCpu == NIL_RTCPUID
+        ||  VMCPU_GET_STATE(pVCpu) != VMCPUSTATE_STARTED_EXEC)
+    {
+        pGVM->gvmm.s.StatsSched.cPokeNotBusy++;
+        return VINF_GVM_NOT_BUSY_IN_GC;
+    }
+
+    /* Note: this function is not implemented on Darwin and Linux (kernel < 2.6.19) */
+    RTMpPokeCpu(idHostCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Pokes an EMT if it's still busy running guest code.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if poked successfully.
+ * @retval  VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   idCpu               The ID of the virtual CPU to poke.
+ * @param   fTakeUsedLock       Take the used lock or not
+ */
+GVMMR0DECL(int) GVMMR0SchedPokeEx(PGVM pGVM, VMCPUID idCpu, bool fTakeUsedLock)
+{
+    /*
+     * Validate input and take the UsedLock.
+     */
+    PGVMM pGVMM;
+    int rc = gvmmR0ByGVM(pGVM, &pGVMM, fTakeUsedLock);
+    if (RT_SUCCESS(rc))
+    {
+        if (idCpu < pGVM->cCpus)
+            rc = gvmmR0SchedPokeOne(pGVM, &pGVM->aCpus[idCpu]);
+        else
+            rc = VERR_INVALID_CPU_ID;
+
+        if (fTakeUsedLock)
+        {
+            int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
+            AssertRC(rc2);
+        }
+    }
+
+    LogFlow(("GVMMR0SchedWakeUpAndPokeCpus: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Pokes an EMT if it's still busy running guest code.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if poked successfully.
+ * @retval  VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   idCpu               The ID of the virtual CPU to poke.
+ */
+GVMMR0DECL(int) GVMMR0SchedPoke(PGVM pGVM, VMCPUID idCpu)
+{
+    return GVMMR0SchedPokeEx(pGVM, idCpu, true /* fTakeUsedLock */);
+}
+
+
+/**
+ * Pokes an EMT if it's still busy running guest code, no GVM parameter and no
+ * used locking.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if poked successfully.
+ * @retval  VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   idCpu               The ID of the virtual CPU to poke.
+ *
+ * @deprecated  Don't use in new code if possible!  Use the GVM variant.
+ */
+GVMMR0DECL(int) GVMMR0SchedPokeNoGVMNoLock(PGVM pGVM, VMCPUID idCpu)
+{
+    PGVMM pGVMM;
+    int rc = gvmmR0ByGVM(pGVM, &pGVMM, false /*fTakeUsedLock*/);
+    if (RT_SUCCESS(rc))
+    {
+        if (idCpu < pGVM->cCpus)
+            rc = gvmmR0SchedPokeOne(pGVM, &pGVM->aCpus[idCpu]);
+        else
+            rc = VERR_INVALID_CPU_ID;
+    }
+    return rc;
+}
+
+
+/**
+ * Wakes up a set of halted EMT threads so they can service pending request.
+ *
+ * @returns VBox status code, no informational stuff.
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   pSleepSet           The set of sleepers to wake up.
+ * @param   pPokeSet            The set of CPUs to poke.
+ */
+GVMMR0DECL(int) GVMMR0SchedWakeUpAndPokeCpus(PGVM pGVM, PCVMCPUSET pSleepSet, PCVMCPUSET pPokeSet)
+{
+    AssertPtrReturn(pSleepSet, VERR_INVALID_POINTER);
+    AssertPtrReturn(pPokeSet, VERR_INVALID_POINTER);
+    GVMM_CHECK_SMAP_SETUP();
+    GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+    RTNATIVETHREAD hSelf = RTThreadNativeSelf();
+
+    /*
+     * Validate input and take the UsedLock.
+     */
+    PGVMM pGVMM;
+    int rc = gvmmR0ByGVM(pGVM, &pGVMM, true /* fTakeUsedLock */);
+    GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+    if (RT_SUCCESS(rc))
+    {
+        rc = VINF_SUCCESS;
+        VMCPUID idCpu = pGVM->cCpus;
+        while (idCpu-- > 0)
+        {
+            /* Don't try poke or wake up ourselves. */
+            if (pGVM->aCpus[idCpu].hEMT == hSelf)
+                continue;
+
+            /* just ignore errors for now. */
+            if (VMCPUSET_IS_PRESENT(pSleepSet, idCpu))
+            {
+                gvmmR0SchedWakeUpOne(pGVM, &pGVM->aCpus[idCpu]);
+                GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            }
+            else if (VMCPUSET_IS_PRESENT(pPokeSet, idCpu))
+            {
+                gvmmR0SchedPokeOne(pGVM, &pGVM->aCpus[idCpu]);
+                GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            }
+        }
+
+        int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
+        AssertRC(rc2);
+        GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+    }
+
+    LogFlow(("GVMMR0SchedWakeUpAndPokeCpus: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VMMR0 request wrapper for GVMMR0SchedWakeUpAndPokeCpus.
+ *
+ * @returns see GVMMR0SchedWakeUpAndPokeCpus.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   pReq            Pointer to the request packet.
+ */
+GVMMR0DECL(int) GVMMR0SchedWakeUpAndPokeCpusReq(PGVM pGVM, PGVMMSCHEDWAKEUPANDPOKECPUSREQ pReq)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
+
+    return GVMMR0SchedWakeUpAndPokeCpus(pGVM, &pReq->SleepSet, &pReq->PokeSet);
+}
+
+
+
+/**
+ * Poll the schedule to see if someone else should get a chance to run.
+ *
+ * This is a bit hackish and will not work too well if the machine is
+ * under heavy load from non-VM processes.
+ *
+ * @returns VINF_SUCCESS if not yielded.
+ *          VINF_GVM_YIELDED if an attempt to switch to a different VM task was made.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   idCpu           The Virtual CPU ID of the calling EMT.
+ * @param   fYield          Whether to yield or not.
+ *                          This is for when we're spinning in the halt loop.
+ * @thread  EMT(idCpu).
+ */
+GVMMR0DECL(int) GVMMR0SchedPoll(PGVM pGVM, VMCPUID idCpu, bool fYield)
+{
+    /*
+     * Validate input.
+     */
+    PGVMM pGVMM;
+    int rc = gvmmR0ByGVMandEMT(pGVM, idCpu, &pGVMM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * We currently only implement helping doing wakeups (fYield = false), so don't
+         * bother taking the lock if gvmmR0SchedDoWakeUps is not going to do anything.
+         */
+        if (!fYield && pGVMM->fDoEarlyWakeUps)
+        {
+            rc = GVMMR0_USED_SHARED_LOCK(pGVMM); AssertRC(rc);
+            pGVM->gvmm.s.StatsSched.cPollCalls++;
+
+            Assert(ASMGetFlags() & X86_EFL_IF);
+            const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
+
+            pGVM->gvmm.s.StatsSched.cPollWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
+
+            GVMMR0_USED_SHARED_UNLOCK(pGVMM);
+        }
+        /*
+         * Not quite sure what we could do here...
+         */
+        else if (fYield)
+            rc = VERR_NOT_IMPLEMENTED; /** @todo implement this... */
+        else
+            rc = VINF_SUCCESS;
+    }
+
+    LogFlow(("GVMMR0SchedWakeUp: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+#ifdef GVMM_SCHED_WITH_PPT
+/**
+ * Timer callback for the periodic preemption timer.
+ *
+ * @param   pTimer      The timer handle.
+ * @param   pvUser      Pointer to the per cpu structure.
+ * @param   iTick       The current tick.
+ */
+static DECLCALLBACK(void) gvmmR0SchedPeriodicPreemptionTimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
+{
+    PGVMMHOSTCPU pCpu = (PGVMMHOSTCPU)pvUser;
+    NOREF(pTimer); NOREF(iTick);
+
+    /*
+     * Termination check
+     */
+    if (pCpu->u32Magic != GVMMHOSTCPU_MAGIC)
+        return;
+
+    /*
+     * Do the house keeping.
+     */
+    RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
+
+    if (++pCpu->Ppt.iTickHistorization >= pCpu->Ppt.cTicksHistoriziationInterval)
+    {
+        /*
+         * Historicize the max frequency.
+         */
+        uint32_t iHzHistory = ++pCpu->Ppt.iHzHistory % RT_ELEMENTS(pCpu->Ppt.aHzHistory);
+        pCpu->Ppt.aHzHistory[iHzHistory] = pCpu->Ppt.uDesiredHz;
+        pCpu->Ppt.iTickHistorization = 0;
+        pCpu->Ppt.uDesiredHz         = 0;
+
+        /*
+         * Check if the current timer frequency.
+         */
+        uint32_t uHistMaxHz = 0;
+        for (uint32_t i = 0; i < RT_ELEMENTS(pCpu->Ppt.aHzHistory); i++)
+            if (pCpu->Ppt.aHzHistory[i] > uHistMaxHz)
+                uHistMaxHz = pCpu->Ppt.aHzHistory[i];
+        if (uHistMaxHz == pCpu->Ppt.uTimerHz)
+            RTSpinlockRelease(pCpu->Ppt.hSpinlock);
+        else if (uHistMaxHz)
+        {
+            /*
+             * Reprogram it.
+             */
+            pCpu->Ppt.cChanges++;
+            pCpu->Ppt.iTickHistorization    = 0;
+            pCpu->Ppt.uTimerHz              = uHistMaxHz;
+            uint32_t const cNsInterval      = RT_NS_1SEC / uHistMaxHz;
+            pCpu->Ppt.cNsInterval           = cNsInterval;
+            if (cNsInterval < GVMMHOSTCPU_PPT_HIST_INTERVAL_NS)
+                pCpu->Ppt.cTicksHistoriziationInterval = (  GVMMHOSTCPU_PPT_HIST_INTERVAL_NS
+                                                          + GVMMHOSTCPU_PPT_HIST_INTERVAL_NS / 2 - 1)
+                                                       / cNsInterval;
+            else
+                pCpu->Ppt.cTicksHistoriziationInterval = 1;
+            RTSpinlockRelease(pCpu->Ppt.hSpinlock);
+
+            /*SUPR0Printf("Cpu%u: change to %u Hz / %u ns\n", pCpu->idxCpuSet, uHistMaxHz, cNsInterval);*/
+            RTTimerChangeInterval(pTimer, cNsInterval);
+        }
+        else
+        {
+            /*
+             * Stop it.
+             */
+            pCpu->Ppt.fStarted    = false;
+            pCpu->Ppt.uTimerHz    = 0;
+            pCpu->Ppt.cNsInterval = 0;
+            RTSpinlockRelease(pCpu->Ppt.hSpinlock);
+
+            /*SUPR0Printf("Cpu%u: stopping (%u Hz)\n", pCpu->idxCpuSet, uHistMaxHz);*/
+            RTTimerStop(pTimer);
+        }
+    }
+    else
+        RTSpinlockRelease(pCpu->Ppt.hSpinlock);
+}
+#endif /* GVMM_SCHED_WITH_PPT */
+
+
+/**
+ * Updates the periodic preemption timer for the calling CPU.
+ *
+ * The caller must have disabled preemption!
+ * The caller must check that the host can do high resolution timers.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idHostCpu   The current host CPU id.
+ * @param   uHz         The desired frequency.
+ */
+GVMMR0DECL(void) GVMMR0SchedUpdatePeriodicPreemptionTimer(PGVM pGVM, RTCPUID idHostCpu, uint32_t uHz)
+{
+    NOREF(pGVM);
+#ifdef GVMM_SCHED_WITH_PPT
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(RTTimerCanDoHighResolution());
+
+    /*
+     * Resolve the per CPU data.
+     */
+    uint32_t    iCpu  = RTMpCpuIdToSetIndex(idHostCpu);
+    PGVMM       pGVMM = g_pGVMM;
+    if (   !RT_VALID_PTR(pGVMM)
+        || pGVMM->u32Magic != GVMM_MAGIC)
+        return;
+    AssertMsgReturnVoid(iCpu < pGVMM->cHostCpus, ("iCpu=%d cHostCpus=%d\n", iCpu, pGVMM->cHostCpus));
+    PGVMMHOSTCPU pCpu = &pGVMM->aHostCpus[iCpu];
+    AssertMsgReturnVoid(   pCpu->u32Magic == GVMMHOSTCPU_MAGIC
+                        && pCpu->idCpu    == idHostCpu,
+                        ("u32Magic=%#x idCpu=% idHostCpu=%d\n", pCpu->u32Magic, pCpu->idCpu, idHostCpu));
+
+    /*
+     * Check whether we need to do anything about the timer.
+     * We have to be a little bit careful since we might be race the timer
+     * callback here.
+     */
+    if (uHz > 16384)
+        uHz = 16384;  /** @todo add a query method for this! */
+    if (RT_UNLIKELY(   uHz > ASMAtomicReadU32(&pCpu->Ppt.uDesiredHz)
+                    && uHz >= pCpu->Ppt.uMinHz
+                    && !pCpu->Ppt.fStarting /* solaris paranoia */))
+    {
+        RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
+
+        pCpu->Ppt.uDesiredHz = uHz;
+        uint32_t cNsInterval = 0;
+        if (!pCpu->Ppt.fStarted)
+        {
+            pCpu->Ppt.cStarts++;
+            pCpu->Ppt.fStarted              = true;
+            pCpu->Ppt.fStarting             = true;
+            pCpu->Ppt.iTickHistorization    = 0;
+            pCpu->Ppt.uTimerHz              = uHz;
+            pCpu->Ppt.cNsInterval           = cNsInterval = RT_NS_1SEC / uHz;
+            if (cNsInterval < GVMMHOSTCPU_PPT_HIST_INTERVAL_NS)
+                pCpu->Ppt.cTicksHistoriziationInterval = (  GVMMHOSTCPU_PPT_HIST_INTERVAL_NS
+                                                          + GVMMHOSTCPU_PPT_HIST_INTERVAL_NS / 2 - 1)
+                                                       / cNsInterval;
+            else
+                pCpu->Ppt.cTicksHistoriziationInterval = 1;
+        }
+
+        RTSpinlockRelease(pCpu->Ppt.hSpinlock);
+
+        if (cNsInterval)
+        {
+            RTTimerChangeInterval(pCpu->Ppt.pTimer, cNsInterval);
+            int rc = RTTimerStart(pCpu->Ppt.pTimer, cNsInterval);
+            AssertRC(rc);
+
+            RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
+            if (RT_FAILURE(rc))
+                pCpu->Ppt.fStarted = false;
+            pCpu->Ppt.fStarting = false;
+            RTSpinlockRelease(pCpu->Ppt.hSpinlock);
+        }
+    }
+#else  /* !GVMM_SCHED_WITH_PPT */
+    NOREF(idHostCpu); NOREF(uHz);
+#endif /* !GVMM_SCHED_WITH_PPT */
+}
+
+
+/**
+ * Calls @a pfnCallback for each VM in the system.
+ *
+ * This will enumerate the VMs while holding the global VM used list lock in
+ * shared mode.  So, only suitable for simple work.  If more expensive work
+ * needs doing, a different approach must be taken as using this API would
+ * otherwise block VM creation and destruction.
+ *
+ * @returns VBox status code.
+ * @param   pfnCallback     The callback function.
+ * @param   pvUser          User argument to the callback.
+ */
+GVMMR0DECL(int) GVMMR0EnumVMs(PFNGVMMR0ENUMCALLBACK pfnCallback, void *pvUser)
+{
+    PGVMM pGVMM;
+    GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
+
+    int rc = VINF_SUCCESS;
+    GVMMR0_USED_SHARED_LOCK(pGVMM);
+    for (unsigned i = pGVMM->iUsedHead, cLoops = 0;
+         i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
+         i = pGVMM->aHandles[i].iNext, cLoops++)
+    {
+        PGVM pGVM = pGVMM->aHandles[i].pGVM;
+        if (   RT_VALID_PTR(pGVM)
+            && RT_VALID_PTR(pGVMM->aHandles[i].pvObj)
+            && pGVM->u32Magic == GVM_MAGIC)
+        {
+            rc = pfnCallback(pGVM, pvUser);
+            if (rc != VINF_SUCCESS)
+                break;
+        }
+
+        AssertBreak(cLoops < RT_ELEMENTS(pGVMM->aHandles) * 4); /* paranoia */
+    }
+    GVMMR0_USED_SHARED_UNLOCK(pGVMM);
+    return rc;
+}
+
+
+/**
+ * Retrieves the GVMM statistics visible to the caller.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pStats      Where to put the statistics.
+ * @param   pSession    The current session.
+ * @param   pGVM        The GVM to obtain statistics for. Optional.
+ */
+GVMMR0DECL(int) GVMMR0QueryStatistics(PGVMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM)
+{
+    LogFlow(("GVMMR0QueryStatistics: pStats=%p pSession=%p pGVM=%p\n", pStats, pSession, pGVM));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pSession, VERR_INVALID_POINTER);
+    AssertPtrReturn(pStats, VERR_INVALID_POINTER);
+    pStats->cVMs = 0; /* (crash before taking the sem...) */
+
+    /*
+     * Take the lock and get the VM statistics.
+     */
+    PGVMM pGVMM;
+    if (pGVM)
+    {
+        int rc = gvmmR0ByGVM(pGVM, &pGVMM, true /*fTakeUsedLock*/);
+        if (RT_FAILURE(rc))
+            return rc;
+        pStats->SchedVM = pGVM->gvmm.s.StatsSched;
+    }
+    else
+    {
+        GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
+        memset(&pStats->SchedVM, 0, sizeof(pStats->SchedVM));
+
+        int rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Enumerate the VMs and add the ones visible to the statistics.
+     */
+    pStats->cVMs = 0;
+    pStats->cEMTs = 0;
+    memset(&pStats->SchedSum, 0, sizeof(pStats->SchedSum));
+
+    for (unsigned i = pGVMM->iUsedHead;
+         i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
+         i = pGVMM->aHandles[i].iNext)
+    {
+        PGVM pOtherGVM = pGVMM->aHandles[i].pGVM;
+        void *pvObj = pGVMM->aHandles[i].pvObj;
+        if (    RT_VALID_PTR(pvObj)
+            &&  RT_VALID_PTR(pOtherGVM)
+            &&  pOtherGVM->u32Magic == GVM_MAGIC
+            &&  RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
+        {
+            pStats->cVMs++;
+            pStats->cEMTs += pOtherGVM->cCpus;
+
+            pStats->SchedSum.cHaltCalls        += pOtherGVM->gvmm.s.StatsSched.cHaltCalls;
+            pStats->SchedSum.cHaltBlocking     += pOtherGVM->gvmm.s.StatsSched.cHaltBlocking;
+            pStats->SchedSum.cHaltTimeouts     += pOtherGVM->gvmm.s.StatsSched.cHaltTimeouts;
+            pStats->SchedSum.cHaltNotBlocking  += pOtherGVM->gvmm.s.StatsSched.cHaltNotBlocking;
+            pStats->SchedSum.cHaltWakeUps      += pOtherGVM->gvmm.s.StatsSched.cHaltWakeUps;
+
+            pStats->SchedSum.cWakeUpCalls      += pOtherGVM->gvmm.s.StatsSched.cWakeUpCalls;
+            pStats->SchedSum.cWakeUpNotHalted  += pOtherGVM->gvmm.s.StatsSched.cWakeUpNotHalted;
+            pStats->SchedSum.cWakeUpWakeUps    += pOtherGVM->gvmm.s.StatsSched.cWakeUpWakeUps;
+
+            pStats->SchedSum.cPokeCalls        += pOtherGVM->gvmm.s.StatsSched.cPokeCalls;
+            pStats->SchedSum.cPokeNotBusy      += pOtherGVM->gvmm.s.StatsSched.cPokeNotBusy;
+
+            pStats->SchedSum.cPollCalls        += pOtherGVM->gvmm.s.StatsSched.cPollCalls;
+            pStats->SchedSum.cPollHalts        += pOtherGVM->gvmm.s.StatsSched.cPollHalts;
+            pStats->SchedSum.cPollWakeUps      += pOtherGVM->gvmm.s.StatsSched.cPollWakeUps;
+        }
+    }
+
+    /*
+     * Copy out the per host CPU statistics.
+     */
+    uint32_t iDstCpu = 0;
+    uint32_t cSrcCpus = pGVMM->cHostCpus;
+    for (uint32_t iSrcCpu = 0; iSrcCpu < cSrcCpus; iSrcCpu++)
+    {
+        if (pGVMM->aHostCpus[iSrcCpu].idCpu != NIL_RTCPUID)
+        {
+            pStats->aHostCpus[iDstCpu].idCpu      = pGVMM->aHostCpus[iSrcCpu].idCpu;
+            pStats->aHostCpus[iDstCpu].idxCpuSet  = pGVMM->aHostCpus[iSrcCpu].idxCpuSet;
+#ifdef GVMM_SCHED_WITH_PPT
+            pStats->aHostCpus[iDstCpu].uDesiredHz = pGVMM->aHostCpus[iSrcCpu].Ppt.uDesiredHz;
+            pStats->aHostCpus[iDstCpu].uTimerHz   = pGVMM->aHostCpus[iSrcCpu].Ppt.uTimerHz;
+            pStats->aHostCpus[iDstCpu].cChanges   = pGVMM->aHostCpus[iSrcCpu].Ppt.cChanges;
+            pStats->aHostCpus[iDstCpu].cStarts    = pGVMM->aHostCpus[iSrcCpu].Ppt.cStarts;
+#else
+            pStats->aHostCpus[iDstCpu].uDesiredHz = 0;
+            pStats->aHostCpus[iDstCpu].uTimerHz   = 0;
+            pStats->aHostCpus[iDstCpu].cChanges   = 0;
+            pStats->aHostCpus[iDstCpu].cStarts    = 0;
+#endif
+            iDstCpu++;
+            if (iDstCpu >= RT_ELEMENTS(pStats->aHostCpus))
+                break;
+        }
+    }
+    pStats->cHostCpus = iDstCpu;
+
+    GVMMR0_USED_SHARED_UNLOCK(pGVMM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VMMR0 request wrapper for GVMMR0QueryStatistics.
+ *
+ * @returns see GVMMR0QueryStatistics.
+ * @param   pGVM            The global (ring-0) VM structure. Optional.
+ * @param   pReq            Pointer to the request packet.
+ * @param   pSession        The current session.
+ */
+GVMMR0DECL(int) GVMMR0QueryStatisticsReq(PGVM pGVM, PGVMMQUERYSTATISTICSSREQ pReq, PSUPDRVSESSION pSession)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
+    AssertReturn(pReq->pSession == pSession, VERR_INVALID_PARAMETER);
+
+    return GVMMR0QueryStatistics(&pReq->Stats, pSession, pGVM);
+}
+
+
+/**
+ * Resets the specified GVMM statistics.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pStats      Which statistics to reset, that is, non-zero fields indicates which to reset.
+ * @param   pSession    The current session.
+ * @param   pGVM        The GVM to reset statistics for. Optional.
+ */
+GVMMR0DECL(int) GVMMR0ResetStatistics(PCGVMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM)
+{
+    LogFlow(("GVMMR0ResetStatistics: pStats=%p pSession=%p pGVM=%p\n", pStats, pSession, pGVM));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pSession, VERR_INVALID_POINTER);
+    AssertPtrReturn(pStats, VERR_INVALID_POINTER);
+
+    /*
+     * Take the lock and get the VM statistics.
+     */
+    PGVMM pGVMM;
+    if (pGVM)
+    {
+        int rc = gvmmR0ByGVM(pGVM, &pGVMM, true /*fTakeUsedLock*/);
+        if (RT_FAILURE(rc))
+            return rc;
+#       define MAYBE_RESET_FIELD(field) \
+            do { if (pStats->SchedVM. field ) { pGVM->gvmm.s.StatsSched. field = 0; } } while (0)
+        MAYBE_RESET_FIELD(cHaltCalls);
+        MAYBE_RESET_FIELD(cHaltBlocking);
+        MAYBE_RESET_FIELD(cHaltTimeouts);
+        MAYBE_RESET_FIELD(cHaltNotBlocking);
+        MAYBE_RESET_FIELD(cHaltWakeUps);
+        MAYBE_RESET_FIELD(cWakeUpCalls);
+        MAYBE_RESET_FIELD(cWakeUpNotHalted);
+        MAYBE_RESET_FIELD(cWakeUpWakeUps);
+        MAYBE_RESET_FIELD(cPokeCalls);
+        MAYBE_RESET_FIELD(cPokeNotBusy);
+        MAYBE_RESET_FIELD(cPollCalls);
+        MAYBE_RESET_FIELD(cPollHalts);
+        MAYBE_RESET_FIELD(cPollWakeUps);
+#       undef MAYBE_RESET_FIELD
+    }
+    else
+    {
+        GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
+
+        int rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Enumerate the VMs and add the ones visible to the statistics.
+     */
+    if (!ASMMemIsZero(&pStats->SchedSum, sizeof(pStats->SchedSum)))
+    {
+        for (unsigned i = pGVMM->iUsedHead;
+             i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
+             i = pGVMM->aHandles[i].iNext)
+        {
+            PGVM pOtherGVM = pGVMM->aHandles[i].pGVM;
+            void *pvObj = pGVMM->aHandles[i].pvObj;
+            if (    RT_VALID_PTR(pvObj)
+                &&  RT_VALID_PTR(pOtherGVM)
+                &&  pOtherGVM->u32Magic == GVM_MAGIC
+                &&  RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
+            {
+#               define MAYBE_RESET_FIELD(field) \
+                    do { if (pStats->SchedSum. field ) { pOtherGVM->gvmm.s.StatsSched. field = 0; } } while (0)
+                MAYBE_RESET_FIELD(cHaltCalls);
+                MAYBE_RESET_FIELD(cHaltBlocking);
+                MAYBE_RESET_FIELD(cHaltTimeouts);
+                MAYBE_RESET_FIELD(cHaltNotBlocking);
+                MAYBE_RESET_FIELD(cHaltWakeUps);
+                MAYBE_RESET_FIELD(cWakeUpCalls);
+                MAYBE_RESET_FIELD(cWakeUpNotHalted);
+                MAYBE_RESET_FIELD(cWakeUpWakeUps);
+                MAYBE_RESET_FIELD(cPokeCalls);
+                MAYBE_RESET_FIELD(cPokeNotBusy);
+                MAYBE_RESET_FIELD(cPollCalls);
+                MAYBE_RESET_FIELD(cPollHalts);
+                MAYBE_RESET_FIELD(cPollWakeUps);
+#               undef MAYBE_RESET_FIELD
+            }
+        }
+    }
+
+    GVMMR0_USED_SHARED_UNLOCK(pGVMM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VMMR0 request wrapper for GVMMR0ResetStatistics.
+ *
+ * @returns see GVMMR0ResetStatistics.
+ * @param   pGVM            The global (ring-0) VM structure. Optional.
+ * @param   pReq            Pointer to the request packet.
+ * @param   pSession        The current session.
+ */
+GVMMR0DECL(int) GVMMR0ResetStatisticsReq(PGVM pGVM, PGVMMRESETSTATISTICSSREQ pReq, PSUPDRVSESSION pSession)
+{
+    /*
+     * Validate input and pass it on.
+     */
+    AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
+    AssertReturn(pReq->pSession == pSession, VERR_INVALID_PARAMETER);
+
+    return GVMMR0ResetStatistics(&pReq->Stats, pSession, pGVM);
+}
+
diff --git a/src/VBox/VMM/VMMR0/GVMMR0Internal.h b/src/VBox/VMM/VMMR0/GVMMR0Internal.h
new file mode 100644
index 00000000..85f7ccef
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/GVMMR0Internal.h
@@ -0,0 +1,73 @@
+/* $Id: GVMMR0Internal.h $ */
+/** @file
+ * GVMM - The Global VM Manager, Internal header.
+ */
+
+/*
+ * Copyright (C) 2007-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_VMMR0_GVMMR0Internal_h
+#define VMM_INCLUDED_SRC_VMMR0_GVMMR0Internal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <iprt/mem.h>
+
+/**
+ * The GVMM per VM data.
+ */
+typedef struct GVMMPERVCPU
+{
+    /** The time the halted EMT thread expires.
+     * 0 if the EMT thread is blocked here. */
+    uint64_t volatile   u64HaltExpire;
+    /** The event semaphore the EMT thread is blocking on. */
+    RTSEMEVENTMULTI     HaltEventMulti;
+    /** The ring-3 mapping of the VMCPU structure. */
+    RTR0MEMOBJ          VMCpuMapObj;
+    /** The APIC ID of the CPU that EMT was scheduled on the last time we checked.
+     * @todo Extend to 32-bit and use most suitable APIC ID function when we
+     *       start using this for something sensible... */
+    uint8_t             iCpuEmt;
+} GVMMPERVCPU;
+/** Pointer to the GVMM per VCPU data. */
+typedef GVMMPERVCPU *PGVMMPERVCPU;
+
+/**
+ * The GVMM per VM data.
+ */
+typedef struct GVMMPERVM
+{
+    /** The shared VM data structure allocation object (PVMR0). */
+    RTR0MEMOBJ          VMMemObj;
+    /** The Ring-3 mapping of the shared VM data structure (PVMR3). */
+    RTR0MEMOBJ          VMMapObj;
+    /** The allocation object for the VM pages. */
+    RTR0MEMOBJ          VMPagesMemObj;
+    /** The ring-3 mapping of the VM pages. */
+    RTR0MEMOBJ          VMPagesMapObj;
+
+    /** The scheduler statistics. */
+    GVMMSTATSSCHED      StatsSched;
+
+    /** Whether the per-VM ring-0 initialization has been performed. */
+    bool                fDoneVMMR0Init;
+    /** Whether the per-VM ring-0 termination is being or has been performed. */
+    bool                fDoneVMMR0Term;
+} GVMMPERVM;
+/** Pointer to the GVMM per VM data. */
+typedef GVMMPERVM *PGVMMPERVM;
+
+
+#endif /* !VMM_INCLUDED_SRC_VMMR0_GVMMR0Internal_h */
+
diff --git a/src/VBox/VMM/VMMR0/HMR0.cpp b/src/VBox/VMM/VMMR0/HMR0.cpp
new file mode 100644
index 00000000..5d3e3533
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/HMR0.cpp
@@ -0,0 +1,1862 @@
+/* $Id: HMR0.cpp $ */
+/** @file
+ * Hardware Assisted Virtualization Manager (HM) - Host Context Ring-0.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_HM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pgm.h>
+#include "HMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/hm_svm.h>
+#include <VBox/vmm/hmvmxinline.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/cpuset.h>
+#include <iprt/mem.h>
+#include <iprt/memobj.h>
+#include <iprt/once.h>
+#include <iprt/param.h>
+#include <iprt/power.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/x86.h>
+#include "HMVMXR0.h"
+#include "HMSVMR0.h"
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(void) hmR0EnableCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2);
+static DECLCALLBACK(void) hmR0DisableCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2);
+static DECLCALLBACK(void) hmR0InitIntelCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2);
+static DECLCALLBACK(void) hmR0InitAmdCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2);
+static DECLCALLBACK(void) hmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser);
+static DECLCALLBACK(void) hmR0MpEventCallback(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvData);
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * This is used to manage the status code of a RTMpOnAll in HM.
+ */
+typedef struct HMR0FIRSTRC
+{
+    /** The status code. */
+    int32_t volatile    rc;
+    /** The ID of the CPU reporting the first failure. */
+    RTCPUID volatile    idCpu;
+} HMR0FIRSTRC;
+/** Pointer to a first return code structure. */
+typedef HMR0FIRSTRC *PHMR0FIRSTRC;
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/**
+ * Global data.
+ */
+static struct
+{
+    /** Per CPU globals. */
+    HMPHYSCPU                       aCpuInfo[RTCPUSET_MAX_CPUS];
+
+    /** @name Ring-0 method table for AMD-V and VT-x specific operations.
+     * @{ */
+    DECLR0CALLBACKMEMBER(int,          pfnEnterSession, (PVMCPUCC pVCpu));
+    DECLR0CALLBACKMEMBER(void,         pfnThreadCtxCallback, (RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit));
+    DECLR0CALLBACKMEMBER(int,          pfnCallRing3Callback, (PVMCPUCC pVCpu, VMMCALLRING3 enmOperation));
+    DECLR0CALLBACKMEMBER(int,          pfnExportHostState, (PVMCPUCC pVCpu));
+    DECLR0CALLBACKMEMBER(VBOXSTRICTRC, pfnRunGuestCode, (PVMCPUCC pVCpu));
+    DECLR0CALLBACKMEMBER(int,          pfnEnableCpu, (PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage,
+                                                      bool fEnabledByHost, PCSUPHWVIRTMSRS pHwvirtMsrs));
+    DECLR0CALLBACKMEMBER(int,          pfnDisableCpu, (PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage));
+    DECLR0CALLBACKMEMBER(int,          pfnInitVM, (PVMCC pVM));
+    DECLR0CALLBACKMEMBER(int,          pfnTermVM, (PVMCC pVM));
+    DECLR0CALLBACKMEMBER(int,          pfnSetupVM, (PVMCC pVM));
+    /** @} */
+
+    /** Hardware-virtualization data. */
+    struct
+    {
+        union
+        {
+            /** VT-x data. */
+            struct
+            {
+                /** Host CR4 value (set by ring-0 VMX init) */
+                uint64_t                    u64HostCr4;
+                /** Host EFER value (set by ring-0 VMX init) */
+                uint64_t                    u64HostMsrEfer;
+                /** Host SMM monitor control (used for logging/diagnostics) */
+                uint64_t                    u64HostSmmMonitorCtl;
+                /** Last instruction error. */
+                uint32_t                    ulLastInstrError;
+                /** The shift mask employed by the VMX-Preemption timer. */
+                uint8_t                     cPreemptTimerShift;
+                /** Padding. */
+                uint8_t                     abPadding[3];
+                /** Whether we're using the preemption timer or not. */
+                bool                        fUsePreemptTimer;
+                /** Whether we're using SUPR0EnableVTx or not. */
+                bool                        fUsingSUPR0EnableVTx;
+                /** Set if we've called SUPR0EnableVTx(true) and should disable it during
+                 * module termination. */
+                bool                        fCalledSUPR0EnableVTx;
+                /** Set to by us to indicate VMX is supported by the CPU. */
+                bool                        fSupported;
+            } vmx;
+
+            /** AMD-V data. */
+            struct
+            {
+                /** SVM revision. */
+                uint32_t                    u32Rev;
+                /** SVM feature bits from cpuid 0x8000000a */
+                uint32_t                    u32Features;
+                /** Padding. */
+                bool                        afPadding[3];
+                /** Set by us to indicate SVM is supported by the CPU. */
+                bool                        fSupported;
+            } svm;
+        } u;
+        /** Maximum allowed ASID/VPID (inclusive). */
+        uint32_t                    uMaxAsid;
+        /** MSRs. */
+        SUPHWVIRTMSRS               Msrs;
+    } hwvirt;
+
+    /** Last recorded error code during HM ring-0 init. */
+    int32_t                         rcInit;
+
+    /** If set, VT-x/AMD-V is enabled globally at init time, otherwise it's
+     * enabled and disabled each time it's used to execute guest code. */
+    bool                            fGlobalInit;
+    /** Indicates whether the host is suspending or not.  We'll refuse a few
+     *  actions when the host is being suspended to speed up the suspending and
+     *  avoid trouble. */
+    bool volatile                   fSuspended;
+
+    /** Whether we've already initialized all CPUs.
+     * @remarks We could check the EnableAllCpusOnce state, but this is
+     *          simpler and hopefully easier to understand. */
+    bool                            fEnabled;
+    /** Serialize initialization in HMR0EnableAllCpus. */
+    RTONCE                          EnableAllCpusOnce;
+} g_HmR0;
+
+
+/**
+ * Initializes a first return code structure.
+ *
+ * @param   pFirstRc            The structure to init.
+ */
+static void hmR0FirstRcInit(PHMR0FIRSTRC pFirstRc)
+{
+    pFirstRc->rc    = VINF_SUCCESS;
+    pFirstRc->idCpu = NIL_RTCPUID;
+}
+
+
+/**
+ * Try set the status code (success ignored).
+ *
+ * @param   pFirstRc            The first return code structure.
+ * @param   rc                  The status code.
+ */
+static void hmR0FirstRcSetStatus(PHMR0FIRSTRC pFirstRc, int rc)
+{
+    if (   RT_FAILURE(rc)
+        && ASMAtomicCmpXchgS32(&pFirstRc->rc, rc, VINF_SUCCESS))
+        pFirstRc->idCpu = RTMpCpuId();
+}
+
+
+/**
+ * Get the status code of a first return code structure.
+ *
+ * @returns The status code; VINF_SUCCESS or error status, no informational or
+ *          warning errors.
+ * @param   pFirstRc            The first return code structure.
+ */
+static int hmR0FirstRcGetStatus(PHMR0FIRSTRC pFirstRc)
+{
+    return pFirstRc->rc;
+}
+
+
+#ifdef VBOX_STRICT
+# ifndef DEBUG_bird
+/**
+ * Get the CPU ID on which the failure status code was reported.
+ *
+ * @returns The CPU ID, NIL_RTCPUID if no failure was reported.
+ * @param   pFirstRc            The first return code structure.
+ */
+static RTCPUID hmR0FirstRcGetCpuId(PHMR0FIRSTRC pFirstRc)
+{
+    return pFirstRc->idCpu;
+}
+# endif
+#endif /* VBOX_STRICT */
+
+
+/** @name Dummy callback handlers.
+ * @{ */
+
+static DECLCALLBACK(int) hmR0DummyEnter(PVMCPUCC pVCpu)
+{
+    RT_NOREF1(pVCpu);
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(void) hmR0DummyThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
+{
+    RT_NOREF3(enmEvent, pVCpu, fGlobalInit);
+}
+
+static DECLCALLBACK(int) hmR0DummyEnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage,
+                                            bool fEnabledBySystem, PCSUPHWVIRTMSRS pHwvirtMsrs)
+{
+    RT_NOREF6(pHostCpu, pVM, pvCpuPage, HCPhysCpuPage, fEnabledBySystem, pHwvirtMsrs);
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(int) hmR0DummyDisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
+{
+    RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(int) hmR0DummyInitVM(PVMCC pVM)
+{
+    RT_NOREF1(pVM);
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(int) hmR0DummyTermVM(PVMCC pVM)
+{
+    RT_NOREF1(pVM);
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(int) hmR0DummySetupVM(PVMCC pVM)
+{
+    RT_NOREF1(pVM);
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(int) hmR0DummyCallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation)
+{
+    RT_NOREF2(pVCpu, enmOperation);
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(VBOXSTRICTRC) hmR0DummyRunGuestCode(PVMCPUCC pVCpu)
+{
+    RT_NOREF(pVCpu);
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(int) hmR0DummyExportHostState(PVMCPUCC pVCpu)
+{
+    RT_NOREF1(pVCpu);
+    return VINF_SUCCESS;
+}
+
+/** @} */
+
+
+/**
+ * Intel specific initialization code.
+ *
+ * @returns VBox status code (will only fail if out of memory).
+ */
+static int hmR0InitIntel(void)
+{
+    /* Read this MSR now as it may be useful for error reporting when initializing VT-x fails. */
+    g_HmR0.hwvirt.Msrs.u.vmx.u64FeatCtrl = ASMRdMsr(MSR_IA32_FEATURE_CONTROL);
+
+    /*
+     * First try use native kernel API for controlling VT-x.
+     * (This is only supported by some Mac OS X kernels atm.)
+     */
+    int rc = g_HmR0.rcInit = SUPR0EnableVTx(true /* fEnable */);
+    g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx = rc != VERR_NOT_SUPPORTED;
+    if (g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx)
+    {
+        AssertLogRelMsg(rc == VINF_SUCCESS || rc == VERR_VMX_IN_VMX_ROOT_MODE || rc == VERR_VMX_NO_VMX, ("%Rrc\n", rc));
+        if (RT_SUCCESS(rc))
+        {
+            g_HmR0.hwvirt.u.vmx.fSupported = true;
+            rc = SUPR0EnableVTx(false /* fEnable */);
+            AssertLogRelRC(rc);
+        }
+    }
+    else
+    {
+        HMR0FIRSTRC FirstRc;
+        hmR0FirstRcInit(&FirstRc);
+        g_HmR0.rcInit = RTMpOnAll(hmR0InitIntelCpu, &FirstRc, NULL);
+        if (RT_SUCCESS(g_HmR0.rcInit))
+            g_HmR0.rcInit = hmR0FirstRcGetStatus(&FirstRc);
+    }
+
+    if (RT_SUCCESS(g_HmR0.rcInit))
+    {
+        /* Read CR4 and EFER for logging/diagnostic purposes. */
+        g_HmR0.hwvirt.u.vmx.u64HostCr4     = ASMGetCR4();
+        g_HmR0.hwvirt.u.vmx.u64HostMsrEfer = ASMRdMsr(MSR_K6_EFER);
+
+        /* Get VMX MSRs for determining VMX features we can ultimately use. */
+        SUPR0GetHwvirtMsrs(&g_HmR0.hwvirt.Msrs, SUPVTCAPS_VT_X, false /* fForce */);
+
+        /*
+         * Nested KVM workaround: Intel SDM section 34.15.5 describes that
+         * MSR_IA32_SMM_MONITOR_CTL depends on bit 49 of MSR_IA32_VMX_BASIC while
+         * table 35-2 says that this MSR is available if either VMX or SMX is supported.
+         */
+        uint64_t const uVmxBasicMsr = g_HmR0.hwvirt.Msrs.u.vmx.u64Basic;
+        if (RT_BF_GET(uVmxBasicMsr, VMX_BF_BASIC_DUAL_MON))
+            g_HmR0.hwvirt.u.vmx.u64HostSmmMonitorCtl = ASMRdMsr(MSR_IA32_SMM_MONITOR_CTL);
+
+        /* Initialize VPID - 16 bits ASID. */
+        g_HmR0.hwvirt.uMaxAsid = 0x10000; /* exclusive */
+
+        /*
+         * If the host OS has not enabled VT-x for us, try enter VMX root mode
+         * to really verify if VT-x is usable.
+         */
+        if (!g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx)
+        {
+            /* Allocate a temporary VMXON region. */
+            RTR0MEMOBJ hScatchMemObj;
+            rc = RTR0MemObjAllocCont(&hScatchMemObj, PAGE_SIZE, false /* fExecutable */);
+            if (RT_FAILURE(rc))
+            {
+                LogRel(("hmR0InitIntel: RTR0MemObjAllocCont(,PAGE_SIZE,false) -> %Rrc\n", rc));
+                return rc;
+            }
+            void          *pvScatchPage      = RTR0MemObjAddress(hScatchMemObj);
+            RTHCPHYS const HCPhysScratchPage = RTR0MemObjGetPagePhysAddr(hScatchMemObj, 0);
+            ASMMemZeroPage(pvScatchPage);
+
+            /* Set revision dword at the beginning of the VMXON structure. */
+            *(uint32_t *)pvScatchPage = RT_BF_GET(uVmxBasicMsr, VMX_BF_BASIC_VMCS_ID);
+
+            /* Make sure we don't get rescheduled to another CPU during this probe. */
+            RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+
+            /* Enable CR4.VMXE if it isn't already set. */
+            RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
+
+            /*
+             * The only way of checking if we're in VMX root mode or not is to try and enter it.
+             * There is no instruction or control bit that tells us if we're in VMX root mode.
+             * Therefore, try and enter VMX root mode here.
+             */
+            rc = VMXEnable(HCPhysScratchPage);
+            if (RT_SUCCESS(rc))
+            {
+                g_HmR0.hwvirt.u.vmx.fSupported = true;
+                VMXDisable();
+            }
+            else
+            {
+                /*
+                 * KVM leaves the CPU in VMX root mode. Not only is  this not allowed,
+                 * it will crash the host when we enter raw mode, because:
+                 *
+                 *   (a) clearing X86_CR4_VMXE in CR4 causes a #GP (we no longer modify
+                 *       this bit), and
+                 *   (b) turning off paging causes a #GP  (unavoidable when switching
+                 *       from long to 32 bits mode or 32 bits to PAE).
+                 *
+                 * They should fix their code, but until they do we simply refuse to run.
+                 */
+                g_HmR0.rcInit = VERR_VMX_IN_VMX_ROOT_MODE;
+                Assert(g_HmR0.hwvirt.u.vmx.fSupported == false);
+            }
+
+            /* Restore CR4.VMXE if it wasn't set prior to us setting it above. */
+            if (!(uOldCr4 & X86_CR4_VMXE))
+                SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
+
+            /* Restore interrupts. */
+            ASMSetFlags(fEFlags);
+
+            RTR0MemObjFree(hScatchMemObj, false);
+        }
+
+        if (g_HmR0.hwvirt.u.vmx.fSupported)
+        {
+            rc = VMXR0GlobalInit();
+            if (RT_FAILURE(rc))
+                g_HmR0.rcInit = rc;
+
+            /*
+             * Install the VT-x methods.
+             */
+            g_HmR0.pfnEnterSession      = VMXR0Enter;
+            g_HmR0.pfnThreadCtxCallback = VMXR0ThreadCtxCallback;
+            g_HmR0.pfnCallRing3Callback = VMXR0CallRing3Callback;
+            g_HmR0.pfnExportHostState   = VMXR0ExportHostState;
+            g_HmR0.pfnRunGuestCode      = VMXR0RunGuestCode;
+            g_HmR0.pfnEnableCpu         = VMXR0EnableCpu;
+            g_HmR0.pfnDisableCpu        = VMXR0DisableCpu;
+            g_HmR0.pfnInitVM            = VMXR0InitVM;
+            g_HmR0.pfnTermVM            = VMXR0TermVM;
+            g_HmR0.pfnSetupVM           = VMXR0SetupVM;
+
+            /*
+             * Check for the VMX-Preemption Timer and adjust for the "VMX-Preemption
+             * Timer Does Not Count Down at the Rate Specified" CPU erratum.
+             */
+            VMXCTLSMSR PinCtls;
+            PinCtls.u = g_HmR0.hwvirt.Msrs.u.vmx.u64PinCtls;
+            if (PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER)
+            {
+                uint64_t const uVmxMiscMsr = g_HmR0.hwvirt.Msrs.u.vmx.u64Misc;
+                g_HmR0.hwvirt.u.vmx.fUsePreemptTimer   = true;
+                g_HmR0.hwvirt.u.vmx.cPreemptTimerShift = RT_BF_GET(uVmxMiscMsr, VMX_BF_MISC_PREEMPT_TIMER_TSC);
+                if (HMIsSubjectToVmxPreemptTimerErratum())
+                    g_HmR0.hwvirt.u.vmx.cPreemptTimerShift = 0; /* This is about right most of the time here. */
+            }
+        }
+    }
+#ifdef LOG_ENABLED
+    else
+        SUPR0Printf("hmR0InitIntelCpu failed with rc=%Rrc\n", g_HmR0.rcInit);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * AMD-specific initialization code.
+ *
+ * @returns VBox status code (will only fail if out of memory).
+ */
+static int hmR0InitAmd(void)
+{
+    /* Call the global AMD-V initialization routine (should only fail in out-of-memory situations). */
+    int rc = SVMR0GlobalInit();
+    if (RT_FAILURE(rc))
+    {
+        g_HmR0.rcInit = rc;
+        return rc;
+    }
+
+    /*
+     * Install the AMD-V methods.
+     */
+    g_HmR0.pfnEnterSession      = SVMR0Enter;
+    g_HmR0.pfnThreadCtxCallback = SVMR0ThreadCtxCallback;
+    g_HmR0.pfnCallRing3Callback = SVMR0CallRing3Callback;
+    g_HmR0.pfnExportHostState   = SVMR0ExportHostState;
+    g_HmR0.pfnRunGuestCode      = SVMR0RunGuestCode;
+    g_HmR0.pfnEnableCpu         = SVMR0EnableCpu;
+    g_HmR0.pfnDisableCpu        = SVMR0DisableCpu;
+    g_HmR0.pfnInitVM            = SVMR0InitVM;
+    g_HmR0.pfnTermVM            = SVMR0TermVM;
+    g_HmR0.pfnSetupVM           = SVMR0SetupVM;
+
+    /* Query AMD features. */
+    uint32_t u32Dummy;
+    ASMCpuId(0x8000000a, &g_HmR0.hwvirt.u.svm.u32Rev, &g_HmR0.hwvirt.uMaxAsid, &u32Dummy, &g_HmR0.hwvirt.u.svm.u32Features);
+
+    /*
+     * We need to check if AMD-V has been properly initialized on all CPUs.
+     * Some BIOSes might do a poor job.
+     */
+    HMR0FIRSTRC FirstRc;
+    hmR0FirstRcInit(&FirstRc);
+    rc = RTMpOnAll(hmR0InitAmdCpu, &FirstRc, NULL);
+    AssertRC(rc);
+    if (RT_SUCCESS(rc))
+        rc = hmR0FirstRcGetStatus(&FirstRc);
+#ifndef DEBUG_bird
+    AssertMsg(rc == VINF_SUCCESS || rc == VERR_SVM_IN_USE,
+              ("hmR0InitAmdCpu failed for cpu %d with rc=%Rrc\n", hmR0FirstRcGetCpuId(&FirstRc), rc));
+#endif
+    if (RT_SUCCESS(rc))
+    {
+        SUPR0GetHwvirtMsrs(&g_HmR0.hwvirt.Msrs, SUPVTCAPS_AMD_V, false /* fForce */);
+        g_HmR0.hwvirt.u.svm.fSupported = true;
+    }
+    else
+    {
+        g_HmR0.rcInit = rc;
+        if (rc == VERR_SVM_DISABLED || rc == VERR_SVM_IN_USE)
+            rc = VINF_SUCCESS; /* Don't fail if AMD-V is disabled or in use. */
+    }
+    return rc;
+}
+
+
+/**
+ * Does global Ring-0 HM initialization (at module init).
+ *
+ * @returns VBox status code.
+ */
+VMMR0_INT_DECL(int) HMR0Init(void)
+{
+    /*
+     * Initialize the globals.
+     */
+    g_HmR0.fEnabled = false;
+    static RTONCE s_OnceInit = RTONCE_INITIALIZER;
+    g_HmR0.EnableAllCpusOnce = s_OnceInit;
+    for (unsigned i = 0; i < RT_ELEMENTS(g_HmR0.aCpuInfo); i++)
+    {
+        g_HmR0.aCpuInfo[i].idCpu        = NIL_RTCPUID;
+        g_HmR0.aCpuInfo[i].hMemObj      = NIL_RTR0MEMOBJ;
+        g_HmR0.aCpuInfo[i].HCPhysMemObj = NIL_RTHCPHYS;
+        g_HmR0.aCpuInfo[i].pvMemObj     = NULL;
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        g_HmR0.aCpuInfo[i].n.svm.hNstGstMsrpm      = NIL_RTR0MEMOBJ;
+        g_HmR0.aCpuInfo[i].n.svm.HCPhysNstGstMsrpm = NIL_RTHCPHYS;
+        g_HmR0.aCpuInfo[i].n.svm.pvNstGstMsrpm     = NULL;
+#endif
+    }
+
+    /* Fill in all callbacks with placeholders. */
+    g_HmR0.pfnEnterSession      = hmR0DummyEnter;
+    g_HmR0.pfnThreadCtxCallback = hmR0DummyThreadCtxCallback;
+    g_HmR0.pfnCallRing3Callback = hmR0DummyCallRing3Callback;
+    g_HmR0.pfnExportHostState   = hmR0DummyExportHostState;
+    g_HmR0.pfnRunGuestCode      = hmR0DummyRunGuestCode;
+    g_HmR0.pfnEnableCpu         = hmR0DummyEnableCpu;
+    g_HmR0.pfnDisableCpu        = hmR0DummyDisableCpu;
+    g_HmR0.pfnInitVM            = hmR0DummyInitVM;
+    g_HmR0.pfnTermVM            = hmR0DummyTermVM;
+    g_HmR0.pfnSetupVM           = hmR0DummySetupVM;
+
+    /* Default is global VT-x/AMD-V init. */
+    g_HmR0.fGlobalInit         = true;
+
+    /*
+     * Make sure aCpuInfo is big enough for all the CPUs on this system.
+     */
+    if (RTMpGetArraySize() > RT_ELEMENTS(g_HmR0.aCpuInfo))
+    {
+        LogRel(("HM: Too many real CPUs/cores/threads - %u, max %u\n", RTMpGetArraySize(), RT_ELEMENTS(g_HmR0.aCpuInfo)));
+        return VERR_TOO_MANY_CPUS;
+    }
+
+    /*
+     * Check for VT-x or AMD-V support.
+     * Return failure only in out-of-memory situations.
+     */
+    uint32_t fCaps = 0;
+    int rc = SUPR0GetVTSupport(&fCaps);
+    if (RT_SUCCESS(rc))
+    {
+        if (fCaps & SUPVTCAPS_VT_X)
+        {
+            rc = hmR0InitIntel();
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+        else
+        {
+            Assert(fCaps & SUPVTCAPS_AMD_V);
+            rc = hmR0InitAmd();
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+    }
+    else
+        g_HmR0.rcInit = VERR_UNSUPPORTED_CPU;
+
+    /*
+     * Register notification callbacks that we can use to disable/enable CPUs
+     * when brought offline/online or suspending/resuming.
+     */
+    if (!g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx)
+    {
+        rc = RTMpNotificationRegister(hmR0MpEventCallback, NULL);
+        AssertRC(rc);
+
+        rc = RTPowerNotificationRegister(hmR0PowerCallback, NULL);
+        AssertRC(rc);
+    }
+
+    /* We return success here because module init shall not fail if HM fails to initialize. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Does global Ring-0 HM termination (at module termination).
+ *
+ * @returns VBox status code.
+ */
+VMMR0_INT_DECL(int) HMR0Term(void)
+{
+    int rc;
+    if (   g_HmR0.hwvirt.u.vmx.fSupported
+        && g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx)
+    {
+        /*
+         * Simple if the host OS manages VT-x.
+         */
+        Assert(g_HmR0.fGlobalInit);
+
+        if (g_HmR0.hwvirt.u.vmx.fCalledSUPR0EnableVTx)
+        {
+            rc = SUPR0EnableVTx(false /* fEnable */);
+            g_HmR0.hwvirt.u.vmx.fCalledSUPR0EnableVTx = false;
+        }
+        else
+            rc = VINF_SUCCESS;
+
+        for (unsigned iCpu = 0; iCpu < RT_ELEMENTS(g_HmR0.aCpuInfo); iCpu++)
+        {
+            g_HmR0.aCpuInfo[iCpu].fConfigured = false;
+            Assert(g_HmR0.aCpuInfo[iCpu].hMemObj == NIL_RTR0MEMOBJ);
+        }
+    }
+    else
+    {
+        Assert(!g_HmR0.hwvirt.u.vmx.fSupported || !g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx);
+
+        /* Doesn't really matter if this fails. */
+        rc = RTMpNotificationDeregister(hmR0MpEventCallback, NULL);  AssertRC(rc);
+        rc = RTPowerNotificationDeregister(hmR0PowerCallback, NULL); AssertRC(rc);
+
+        /*
+         * Disable VT-x/AMD-V on all CPUs if we enabled it before.
+         */
+        if (g_HmR0.fGlobalInit)
+        {
+            HMR0FIRSTRC FirstRc;
+            hmR0FirstRcInit(&FirstRc);
+            rc = RTMpOnAll(hmR0DisableCpuCallback, NULL /* pvUser 1 */, &FirstRc);
+            Assert(RT_SUCCESS(rc) || rc == VERR_NOT_SUPPORTED);
+            if (RT_SUCCESS(rc))
+                rc = hmR0FirstRcGetStatus(&FirstRc);
+        }
+
+        /*
+         * Free the per-cpu pages used for VT-x and AMD-V.
+         */
+        for (unsigned i = 0; i < RT_ELEMENTS(g_HmR0.aCpuInfo); i++)
+        {
+            if (g_HmR0.aCpuInfo[i].hMemObj != NIL_RTR0MEMOBJ)
+            {
+                RTR0MemObjFree(g_HmR0.aCpuInfo[i].hMemObj, false);
+                g_HmR0.aCpuInfo[i].hMemObj      = NIL_RTR0MEMOBJ;
+                g_HmR0.aCpuInfo[i].HCPhysMemObj = NIL_RTHCPHYS;
+                g_HmR0.aCpuInfo[i].pvMemObj     = NULL;
+            }
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+            if (g_HmR0.aCpuInfo[i].n.svm.hNstGstMsrpm != NIL_RTR0MEMOBJ)
+            {
+                RTR0MemObjFree(g_HmR0.aCpuInfo[i].n.svm.hNstGstMsrpm, false);
+                g_HmR0.aCpuInfo[i].n.svm.hNstGstMsrpm      = NIL_RTR0MEMOBJ;
+                g_HmR0.aCpuInfo[i].n.svm.HCPhysNstGstMsrpm = NIL_RTHCPHYS;
+                g_HmR0.aCpuInfo[i].n.svm.pvNstGstMsrpm     = NULL;
+            }
+#endif
+        }
+    }
+
+    /** @todo This needs cleaning up. There's no matching
+     *        hmR0TermIntel()/hmR0TermAmd() and all the VT-x/AMD-V specific bits
+     *        should move into their respective modules. */
+    /* Finally, call global VT-x/AMD-V termination. */
+    if (g_HmR0.hwvirt.u.vmx.fSupported)
+        VMXR0GlobalTerm();
+    else if (g_HmR0.hwvirt.u.svm.fSupported)
+        SVMR0GlobalTerm();
+
+    return rc;
+}
+
+
+/**
+ * Worker function used by hmR0PowerCallback() and HMR0Init() to initalize VT-x
+ * on a CPU.
+ *
+ * @param   idCpu       The identifier for the CPU the function is called on.
+ * @param   pvUser1     Pointer to the first RC structure.
+ * @param   pvUser2     Ignored.
+ */
+static DECLCALLBACK(void) hmR0InitIntelCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser1;
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
+    NOREF(idCpu); NOREF(pvUser2);
+
+    int rc = SUPR0GetVmxUsability(NULL /* pfIsSmxModeAmbiguous */);
+    hmR0FirstRcSetStatus(pFirstRc, rc);
+}
+
+
+/**
+ * Worker function used by hmR0PowerCallback() and HMR0Init() to initalize AMD-V
+ * on a CPU.
+ *
+ * @param   idCpu       The identifier for the CPU the function is called on.
+ * @param   pvUser1     Pointer to the first RC structure.
+ * @param   pvUser2     Ignored.
+ */
+static DECLCALLBACK(void) hmR0InitAmdCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser1;
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
+    NOREF(idCpu); NOREF(pvUser2);
+
+    int rc = SUPR0GetSvmUsability(true /* fInitSvm */);
+    hmR0FirstRcSetStatus(pFirstRc, rc);
+}
+
+
+/**
+ * Enable VT-x or AMD-V on the current CPU
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure. Can be NULL.
+ * @param   idCpu   The identifier for the CPU the function is called on.
+ *
+ * @remarks Maybe called with interrupts disabled!
+ */
+static int hmR0EnableCpu(PVMCC pVM, RTCPUID idCpu)
+{
+    PHMPHYSCPU pHostCpu = &g_HmR0.aCpuInfo[idCpu];
+
+    Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
+    Assert(idCpu < RT_ELEMENTS(g_HmR0.aCpuInfo));
+    Assert(!pHostCpu->fConfigured);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    pHostCpu->idCpu = idCpu;
+    /* Do NOT reset cTlbFlushes here, see @bugref{6255}. */
+
+    int rc;
+    if (   g_HmR0.hwvirt.u.vmx.fSupported
+        && g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx)
+        rc = g_HmR0.pfnEnableCpu(pHostCpu, pVM, NULL /* pvCpuPage */, NIL_RTHCPHYS, true, &g_HmR0.hwvirt.Msrs);
+    else
+    {
+        AssertLogRelMsgReturn(pHostCpu->hMemObj != NIL_RTR0MEMOBJ, ("hmR0EnableCpu failed idCpu=%u.\n", idCpu), VERR_HM_IPE_1);
+        rc = g_HmR0.pfnEnableCpu(pHostCpu, pVM, pHostCpu->pvMemObj, pHostCpu->HCPhysMemObj, false, &g_HmR0.hwvirt.Msrs);
+    }
+    if (RT_SUCCESS(rc))
+        pHostCpu->fConfigured = true;
+    return rc;
+}
+
+
+/**
+ * Worker function passed to RTMpOnAll() that is to be called on all CPUs.
+ *
+ * @param   idCpu       The identifier for the CPU the function is called on.
+ * @param   pvUser1     Opaque pointer to the VM (can be NULL!).
+ * @param   pvUser2     The 2nd user argument.
+ */
+static DECLCALLBACK(void) hmR0EnableCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    PVMCC           pVM      = (PVMCC)pvUser1;     /* can be NULL! */
+    PHMR0FIRSTRC    pFirstRc = (PHMR0FIRSTRC)pvUser2;
+    AssertReturnVoid(g_HmR0.fGlobalInit);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    hmR0FirstRcSetStatus(pFirstRc, hmR0EnableCpu(pVM, idCpu));
+}
+
+
+/**
+ * RTOnce callback employed by HMR0EnableAllCpus.
+ *
+ * @returns VBox status code.
+ * @param   pvUser          Pointer to the VM.
+ */
+static DECLCALLBACK(int32_t) hmR0EnableAllCpuOnce(void *pvUser)
+{
+    PVMCC pVM = (PVMCC)pvUser;
+
+    /*
+     * Indicate that we've initialized.
+     *
+     * Note! There is a potential race between this function and the suspend
+     *       notification.  Kind of unlikely though, so ignored for now.
+     */
+    AssertReturn(!g_HmR0.fEnabled, VERR_HM_ALREADY_ENABLED_IPE);
+    ASMAtomicWriteBool(&g_HmR0.fEnabled, true);
+
+    /*
+     * The global init variable is set by the first VM.
+     */
+    g_HmR0.fGlobalInit = pVM->hm.s.fGlobalInit;
+
+#ifdef VBOX_STRICT
+    for (unsigned i = 0; i < RT_ELEMENTS(g_HmR0.aCpuInfo); i++)
+    {
+        Assert(g_HmR0.aCpuInfo[i].hMemObj      == NIL_RTR0MEMOBJ);
+        Assert(g_HmR0.aCpuInfo[i].HCPhysMemObj == NIL_RTHCPHYS);
+        Assert(g_HmR0.aCpuInfo[i].pvMemObj     == NULL);
+        Assert(!g_HmR0.aCpuInfo[i].fConfigured);
+        Assert(!g_HmR0.aCpuInfo[i].cTlbFlushes);
+        Assert(!g_HmR0.aCpuInfo[i].uCurrentAsid);
+# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        Assert(g_HmR0.aCpuInfo[i].n.svm.hNstGstMsrpm      == NIL_RTR0MEMOBJ);
+        Assert(g_HmR0.aCpuInfo[i].n.svm.HCPhysNstGstMsrpm == NIL_RTHCPHYS);
+        Assert(g_HmR0.aCpuInfo[i].n.svm.pvNstGstMsrpm     == NULL);
+# endif
+    }
+#endif
+
+    int rc;
+    if (   g_HmR0.hwvirt.u.vmx.fSupported
+        && g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx)
+    {
+        /*
+         * Global VT-x initialization API (only darwin for now).
+         */
+        rc = SUPR0EnableVTx(true /* fEnable */);
+        if (RT_SUCCESS(rc))
+        {
+            g_HmR0.hwvirt.u.vmx.fCalledSUPR0EnableVTx = true;
+            /* If the host provides a VT-x init API, then we'll rely on that for global init. */
+            g_HmR0.fGlobalInit = pVM->hm.s.fGlobalInit = true;
+        }
+        else
+            AssertMsgFailed(("hmR0EnableAllCpuOnce/SUPR0EnableVTx: rc=%Rrc\n", rc));
+    }
+    else
+    {
+        /*
+         * We're doing the job ourselves.
+         */
+        /* Allocate one page per cpu for the global VT-x and AMD-V pages */
+        for (unsigned i = 0; i < RT_ELEMENTS(g_HmR0.aCpuInfo); i++)
+        {
+            Assert(g_HmR0.aCpuInfo[i].hMemObj == NIL_RTR0MEMOBJ);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+            Assert(g_HmR0.aCpuInfo[i].n.svm.hNstGstMsrpm == NIL_RTR0MEMOBJ);
+#endif
+            if (RTMpIsCpuPossible(RTMpCpuIdFromSetIndex(i)))
+            {
+                /** @todo NUMA */
+                rc = RTR0MemObjAllocCont(&g_HmR0.aCpuInfo[i].hMemObj, PAGE_SIZE, false /* executable R0 mapping */);
+                AssertLogRelRCReturn(rc, rc);
+
+                g_HmR0.aCpuInfo[i].HCPhysMemObj = RTR0MemObjGetPagePhysAddr(g_HmR0.aCpuInfo[i].hMemObj, 0);
+                Assert(g_HmR0.aCpuInfo[i].HCPhysMemObj != NIL_RTHCPHYS);
+                Assert(!(g_HmR0.aCpuInfo[i].HCPhysMemObj & PAGE_OFFSET_MASK));
+
+                g_HmR0.aCpuInfo[i].pvMemObj     = RTR0MemObjAddress(g_HmR0.aCpuInfo[i].hMemObj);
+                AssertPtr(g_HmR0.aCpuInfo[i].pvMemObj);
+                ASMMemZeroPage(g_HmR0.aCpuInfo[i].pvMemObj);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+                rc = RTR0MemObjAllocCont(&g_HmR0.aCpuInfo[i].n.svm.hNstGstMsrpm, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
+                                         false /* executable R0 mapping */);
+                AssertLogRelRCReturn(rc, rc);
+
+                g_HmR0.aCpuInfo[i].n.svm.HCPhysNstGstMsrpm = RTR0MemObjGetPagePhysAddr(g_HmR0.aCpuInfo[i].n.svm.hNstGstMsrpm, 0);
+                Assert(g_HmR0.aCpuInfo[i].n.svm.HCPhysNstGstMsrpm != NIL_RTHCPHYS);
+                Assert(!(g_HmR0.aCpuInfo[i].n.svm.HCPhysNstGstMsrpm & PAGE_OFFSET_MASK));
+
+                g_HmR0.aCpuInfo[i].n.svm.pvNstGstMsrpm    = RTR0MemObjAddress(g_HmR0.aCpuInfo[i].n.svm.hNstGstMsrpm);
+                AssertPtr(g_HmR0.aCpuInfo[i].n.svm.pvNstGstMsrpm);
+                ASMMemFill32(g_HmR0.aCpuInfo[i].n.svm.pvNstGstMsrpm, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
+#endif
+            }
+        }
+
+        rc = VINF_SUCCESS;
+    }
+
+    if (   RT_SUCCESS(rc)
+        && g_HmR0.fGlobalInit)
+    {
+        /* First time, so initialize each cpu/core. */
+        HMR0FIRSTRC FirstRc;
+        hmR0FirstRcInit(&FirstRc);
+        rc = RTMpOnAll(hmR0EnableCpuCallback, (void *)pVM, &FirstRc);
+        if (RT_SUCCESS(rc))
+            rc = hmR0FirstRcGetStatus(&FirstRc);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Sets up HM on all cpus.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMR0_INT_DECL(int) HMR0EnableAllCpus(PVMCC pVM)
+{
+    /* Make sure we don't touch HM after we've disabled HM in preparation of a suspend. */
+    if (ASMAtomicReadBool(&g_HmR0.fSuspended))
+        return VERR_HM_SUSPEND_PENDING;
+
+    return RTOnce(&g_HmR0.EnableAllCpusOnce, hmR0EnableAllCpuOnce, pVM);
+}
+
+
+/**
+ * Disable VT-x or AMD-V on the current CPU.
+ *
+ * @returns VBox status code.
+ * @param   idCpu       The identifier for the CPU this function is called on.
+ *
+ * @remarks Must be called with preemption disabled.
+ */
+static int hmR0DisableCpu(RTCPUID idCpu)
+{
+    PHMPHYSCPU pHostCpu = &g_HmR0.aCpuInfo[idCpu];
+
+    Assert(!g_HmR0.hwvirt.u.vmx.fSupported || !g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
+    Assert(idCpu < RT_ELEMENTS(g_HmR0.aCpuInfo));
+    Assert(!pHostCpu->fConfigured || pHostCpu->hMemObj != NIL_RTR0MEMOBJ);
+    AssertRelease(idCpu == RTMpCpuId());
+
+    if (pHostCpu->hMemObj == NIL_RTR0MEMOBJ)
+        return pHostCpu->fConfigured ? VERR_NO_MEMORY : VINF_SUCCESS /* not initialized. */;
+    AssertPtr(pHostCpu->pvMemObj);
+    Assert(pHostCpu->HCPhysMemObj != NIL_RTHCPHYS);
+
+    int rc;
+    if (pHostCpu->fConfigured)
+    {
+        rc = g_HmR0.pfnDisableCpu(pHostCpu, pHostCpu->pvMemObj, pHostCpu->HCPhysMemObj);
+        AssertRCReturn(rc, rc);
+
+        pHostCpu->fConfigured = false;
+        pHostCpu->idCpu = NIL_RTCPUID;
+    }
+    else
+        rc = VINF_SUCCESS; /* nothing to do */
+    return rc;
+}
+
+
+/**
+ * Worker function passed to RTMpOnAll() that is to be called on the target
+ * CPUs.
+ *
+ * @param   idCpu       The identifier for the CPU the function is called on.
+ * @param   pvUser1     The 1st user argument.
+ * @param   pvUser2     Opaque pointer to the FirstRc.
+ */
+static DECLCALLBACK(void) hmR0DisableCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser2; NOREF(pvUser1);
+    AssertReturnVoid(g_HmR0.fGlobalInit);
+    hmR0FirstRcSetStatus(pFirstRc, hmR0DisableCpu(idCpu));
+}
+
+
+/**
+ * Worker function passed to RTMpOnSpecific() that is to be called on the target
+ * CPU.
+ *
+ * @param   idCpu       The identifier for the CPU the function is called on.
+ * @param   pvUser1     Null, not used.
+ * @param   pvUser2     Null, not used.
+ */
+static DECLCALLBACK(void) hmR0DisableCpuOnSpecificCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    NOREF(pvUser1);
+    NOREF(pvUser2);
+    hmR0DisableCpu(idCpu);
+}
+
+
+/**
+ * Callback function invoked when a cpu goes online or offline.
+ *
+ * @param   enmEvent            The Mp event.
+ * @param   idCpu               The identifier for the CPU the function is called on.
+ * @param   pvData              Opaque data (PVMCC pointer).
+ */
+static DECLCALLBACK(void) hmR0MpEventCallback(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvData)
+{
+    NOREF(pvData);
+    Assert(!g_HmR0.hwvirt.u.vmx.fSupported || !g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx);
+
+    /*
+     * We only care about uninitializing a CPU that is going offline. When a
+     * CPU comes online, the initialization is done lazily in HMR0Enter().
+     */
+    switch (enmEvent)
+    {
+        case RTMPEVENT_OFFLINE:
+        {
+            RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
+            RTThreadPreemptDisable(&PreemptState);
+            if (idCpu == RTMpCpuId())
+            {
+                int rc = hmR0DisableCpu(idCpu);
+                AssertRC(rc);
+                RTThreadPreemptRestore(&PreemptState);
+            }
+            else
+            {
+                RTThreadPreemptRestore(&PreemptState);
+                RTMpOnSpecific(idCpu, hmR0DisableCpuOnSpecificCallback, NULL /* pvUser1 */, NULL /* pvUser2 */);
+            }
+            break;
+        }
+
+        default:
+            break;
+    }
+}
+
+
+/**
+ * Called whenever a system power state change occurs.
+ *
+ * @param   enmEvent        The Power event.
+ * @param   pvUser          User argument.
+ */
+static DECLCALLBACK(void) hmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser)
+{
+    NOREF(pvUser);
+    Assert(!g_HmR0.hwvirt.u.vmx.fSupported || !g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx);
+
+#ifdef LOG_ENABLED
+    if (enmEvent == RTPOWEREVENT_SUSPEND)
+        SUPR0Printf("hmR0PowerCallback RTPOWEREVENT_SUSPEND\n");
+    else
+        SUPR0Printf("hmR0PowerCallback RTPOWEREVENT_RESUME\n");
+#endif
+
+    if (enmEvent == RTPOWEREVENT_SUSPEND)
+        ASMAtomicWriteBool(&g_HmR0.fSuspended, true);
+
+    if (g_HmR0.fEnabled)
+    {
+        int         rc;
+        HMR0FIRSTRC FirstRc;
+        hmR0FirstRcInit(&FirstRc);
+
+        if (enmEvent == RTPOWEREVENT_SUSPEND)
+        {
+            if (g_HmR0.fGlobalInit)
+            {
+                /* Turn off VT-x or AMD-V on all CPUs. */
+                rc = RTMpOnAll(hmR0DisableCpuCallback, NULL /* pvUser 1 */, &FirstRc);
+                Assert(RT_SUCCESS(rc) || rc == VERR_NOT_SUPPORTED);
+            }
+            /* else nothing to do here for the local init case */
+        }
+        else
+        {
+            /* Reinit the CPUs from scratch as the suspend state might have
+               messed with the MSRs. (lousy BIOSes as usual) */
+            if (g_HmR0.hwvirt.u.vmx.fSupported)
+                rc = RTMpOnAll(hmR0InitIntelCpu, &FirstRc, NULL);
+            else
+                rc = RTMpOnAll(hmR0InitAmdCpu, &FirstRc, NULL);
+            Assert(RT_SUCCESS(rc) || rc == VERR_NOT_SUPPORTED);
+            if (RT_SUCCESS(rc))
+                rc = hmR0FirstRcGetStatus(&FirstRc);
+#ifdef LOG_ENABLED
+            if (RT_FAILURE(rc))
+                SUPR0Printf("hmR0PowerCallback hmR0InitXxxCpu failed with %Rc\n", rc);
+#endif
+            if (g_HmR0.fGlobalInit)
+            {
+                /* Turn VT-x or AMD-V back on on all CPUs. */
+                rc = RTMpOnAll(hmR0EnableCpuCallback, NULL /* pVM */, &FirstRc /* output ignored */);
+                Assert(RT_SUCCESS(rc) || rc == VERR_NOT_SUPPORTED);
+            }
+            /* else nothing to do here for the local init case */
+        }
+    }
+
+    if (enmEvent == RTPOWEREVENT_RESUME)
+        ASMAtomicWriteBool(&g_HmR0.fSuspended, false);
+}
+
+
+/**
+ * Does ring-0 per-VM HM initialization.
+ *
+ * This will call the CPU specific init. routine which may initialize and allocate
+ * resources for virtual CPUs.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks This is called after HMR3Init(), see vmR3CreateU() and
+ *          vmR3InitRing3().
+ */
+VMMR0_INT_DECL(int) HMR0InitVM(PVMCC pVM)
+{
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+
+    /* Make sure we don't touch HM after we've disabled HM in preparation of a suspend. */
+    if (ASMAtomicReadBool(&g_HmR0.fSuspended))
+        return VERR_HM_SUSPEND_PENDING;
+
+    /*
+     * Copy globals to the VM structure.
+     */
+    Assert(!(pVM->hm.s.vmx.fSupported && pVM->hm.s.svm.fSupported));
+    if (pVM->hm.s.vmx.fSupported)
+    {
+        pVM->hm.s.vmx.fUsePreemptTimer     &= g_HmR0.hwvirt.u.vmx.fUsePreemptTimer; /* Can be overridden by CFGM in HMR3Init(). */
+        pVM->hm.s.vmx.cPreemptTimerShift    = g_HmR0.hwvirt.u.vmx.cPreemptTimerShift;
+        pVM->hm.s.vmx.u64HostCr4            = g_HmR0.hwvirt.u.vmx.u64HostCr4;
+        pVM->hm.s.vmx.u64HostMsrEfer        = g_HmR0.hwvirt.u.vmx.u64HostMsrEfer;
+        pVM->hm.s.vmx.u64HostSmmMonitorCtl  = g_HmR0.hwvirt.u.vmx.u64HostSmmMonitorCtl;
+        HMGetVmxMsrsFromHwvirtMsrs(&g_HmR0.hwvirt.Msrs, &pVM->hm.s.vmx.Msrs);
+        /* If you need to tweak host MSRs for testing VMX R0 code, do it here. */
+
+        /* Enable VPID if supported and configured. */
+        if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VPID)
+            pVM->hm.s.vmx.fVpid = pVM->hm.s.vmx.fAllowVpid; /* Can be overridden by CFGM in HMR3Init(). */
+
+        /* Use VMCS shadowing if supported. */
+        Assert(!pVM->hm.s.vmx.fUseVmcsShadowing);
+        if (   pVM->cpum.ro.GuestFeatures.fVmx
+            && (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VMCS_SHADOWING))
+            pVM->hm.s.vmx.fUseVmcsShadowing = true;
+
+        /* Use the VMCS controls for swapping the EFER MSR if supported. */
+        Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
+        if (   (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
+            && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1  & VMX_EXIT_CTLS_LOAD_EFER_MSR)
+            && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1  & VMX_EXIT_CTLS_SAVE_EFER_MSR))
+            pVM->hm.s.vmx.fSupportsVmcsEfer = true;
+
+#if 0
+        /* Enable APIC register virtualization and virtual-interrupt delivery if supported. */
+        if (   (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT)
+            && (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY))
+            pVM->hm.s.fVirtApicRegs = true;
+
+        /* Enable posted-interrupt processing if supported. */
+        /** @todo Add and query IPRT API for host OS support for posted-interrupt IPI
+         *        here. */
+        if (   (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1  & VMX_PIN_CTLS_POSTED_INT)
+            && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT))
+            pVM->hm.s.fPostedIntrs = true;
+#endif
+    }
+    else if (pVM->hm.s.svm.fSupported)
+    {
+        pVM->hm.s.svm.u32Rev      = g_HmR0.hwvirt.u.svm.u32Rev;
+        pVM->hm.s.svm.u32Features = g_HmR0.hwvirt.u.svm.u32Features;
+        pVM->hm.s.svm.u64MsrHwcr  = g_HmR0.hwvirt.Msrs.u.svm.u64MsrHwcr;
+        /* If you need to tweak host MSRs for testing SVM R0 code, do it here. */
+    }
+    pVM->hm.s.rcInit              = g_HmR0.rcInit;
+    pVM->hm.s.uMaxAsid            = g_HmR0.hwvirt.uMaxAsid;
+
+    /*
+     * Set default maximum inner loops in ring-0 before returning to ring-3.
+     * Can be overriden using CFGM.
+     */
+    if (!pVM->hm.s.cMaxResumeLoops)
+    {
+        pVM->hm.s.cMaxResumeLoops       = 1024;
+        if (RTThreadPreemptIsPendingTrusty())
+            pVM->hm.s.cMaxResumeLoops   = 8192;
+    }
+
+    /*
+     * Initialize some per-VCPU fields.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+        pVCpu->hm.s.idEnteredCpu   = NIL_RTCPUID;
+        pVCpu->hm.s.idLastCpu      = NIL_RTCPUID;
+
+        /* We'll aways increment this the first time (host uses ASID 0). */
+        AssertReturn(!pVCpu->hm.s.uCurrentAsid, VERR_HM_IPE_3);
+    }
+
+    /*
+     * Get host kernel features that HM might need to know in order
+     * to co-operate and function properly with the host OS (e.g. SMAP).
+     *
+     * Technically, we could do this as part of the pre-init VM procedure
+     * but it shouldn't be done later than this point so we do it here.
+     */
+    pVM->hm.s.fHostKernelFeatures = SUPR0GetKernelFeatures();
+
+    /*
+     * Call the hardware specific initialization method.
+     */
+    return g_HmR0.pfnInitVM(pVM);
+}
+
+
+/**
+ * Does ring-0 per VM HM termination.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR0_INT_DECL(int) HMR0TermVM(PVMCC pVM)
+{
+    Log(("HMR0TermVM: %p\n", pVM));
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+
+    /*
+     * Call the hardware specific method.
+     *
+     * Note! We might be preparing for a suspend, so the pfnTermVM() functions should probably not
+     * mess with VT-x/AMD-V features on the CPU, currently all they do is free memory so this is safe.
+     */
+    return g_HmR0.pfnTermVM(pVM);
+}
+
+
+/**
+ * Sets up a VT-x or AMD-V session.
+ *
+ * This is mostly about setting up the hardware VM state.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR0_INT_DECL(int) HMR0SetupVM(PVMCC pVM)
+{
+    Log(("HMR0SetupVM: %p\n", pVM));
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+
+    /* Make sure we don't touch HM after we've disabled HM in preparation of a suspend. */
+    AssertReturn(!ASMAtomicReadBool(&g_HmR0.fSuspended), VERR_HM_SUSPEND_PENDING);
+
+    /* On first entry we'll sync everything. */
+    VMCC_FOR_EACH_VMCPU_STMT(pVM, pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_ALL_GUEST);
+
+    /*
+     * Call the hardware specific setup VM method. This requires the CPU to be
+     * enabled for AMD-V/VT-x and preemption to be prevented.
+     */
+    RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
+    RTThreadPreemptDisable(&PreemptState);
+    RTCPUID const idCpu = RTMpCpuId();
+
+    /* Enable VT-x or AMD-V if local init is required. */
+    int rc;
+    if (!g_HmR0.fGlobalInit)
+    {
+        Assert(!g_HmR0.hwvirt.u.vmx.fSupported || !g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx);
+        rc = hmR0EnableCpu(pVM, idCpu);
+        if (RT_FAILURE(rc))
+        {
+            RTThreadPreemptRestore(&PreemptState);
+            return rc;
+        }
+    }
+
+    /* Setup VT-x or AMD-V. */
+    rc = g_HmR0.pfnSetupVM(pVM);
+
+    /* Disable VT-x or AMD-V if local init was done before. */
+    if (!g_HmR0.fGlobalInit)
+    {
+        Assert(!g_HmR0.hwvirt.u.vmx.fSupported || !g_HmR0.hwvirt.u.vmx.fUsingSUPR0EnableVTx);
+        int rc2 = hmR0DisableCpu(idCpu);
+        AssertRC(rc2);
+    }
+
+    RTThreadPreemptRestore(&PreemptState);
+    return rc;
+}
+
+
+/**
+ * Notification callback before performing a longjump to ring-3.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmOperation    The operation causing the ring-3 longjump.
+ * @param   pvUser          User argument, currently unused, NULL.
+ */
+static DECLCALLBACK(int) hmR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
+{
+    RT_NOREF(pvUser);
+    Assert(pVCpu);
+    Assert(g_HmR0.pfnCallRing3Callback);
+    return g_HmR0.pfnCallRing3Callback(pVCpu, enmOperation);
+}
+
+
+/**
+ * Turns on HM on the CPU if necessary and initializes the bare minimum state
+ * required for entering HM context.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+VMMR0_INT_DECL(int) hmR0EnterCpu(PVMCPUCC pVCpu)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    int              rc       = VINF_SUCCESS;
+    RTCPUID const    idCpu    = RTMpCpuId();
+    PHMPHYSCPU       pHostCpu = &g_HmR0.aCpuInfo[idCpu];
+    AssertPtr(pHostCpu);
+
+    /* Enable VT-x or AMD-V if local init is required, or enable if it's a freshly onlined CPU. */
+    if (!pHostCpu->fConfigured)
+        rc = hmR0EnableCpu(pVCpu->CTX_SUFF(pVM), idCpu);
+
+    /* Register a callback to fire prior to performing a longjmp to ring-3 so HM can disable VT-x/AMD-V if needed. */
+    VMMRZCallRing3SetNotification(pVCpu, hmR0CallRing3Callback, NULL /* pvUser */);
+
+    /* Reload host-state (back from ring-3/migrated CPUs) and shared guest/host bits. */
+    if (g_HmR0.hwvirt.u.vmx.fSupported)
+        pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE;
+    else
+        pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE;
+
+    Assert(pHostCpu->idCpu == idCpu && pHostCpu->idCpu != NIL_RTCPUID);
+    pVCpu->hm.s.idEnteredCpu = idCpu;
+    return rc;
+}
+
+
+/**
+ * Enters the VT-x or AMD-V session.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu      The cross context virtual CPU structure.
+ *
+ * @remarks This is called with preemption disabled.
+ */
+VMMR0_INT_DECL(int) HMR0Enter(PVMCPUCC pVCpu)
+{
+    /* Make sure we can't enter a session after we've disabled HM in preparation of a suspend. */
+    AssertReturn(!ASMAtomicReadBool(&g_HmR0.fSuspended), VERR_HM_SUSPEND_PENDING);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    /* Load the bare minimum state required for entering HM. */
+    int rc = hmR0EnterCpu(pVCpu);
+    if (RT_SUCCESS(rc))
+    {
+        if (g_HmR0.hwvirt.u.vmx.fSupported)
+        {
+            Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
+                                           == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
+        }
+        else
+        {
+            Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
+                                           == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
+        }
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+        AssertReturn(!VMMR0ThreadCtxHookIsEnabled(pVCpu), VERR_HM_IPE_5);
+        bool const fStartedSet = PGMR0DynMapStartOrMigrateAutoSet(pVCpu);
+#endif
+
+        /* Keep track of the CPU owning the VMCS for debugging scheduling weirdness and ring-3 calls. */
+        rc = g_HmR0.pfnEnterSession(pVCpu);
+        AssertMsgRCReturnStmt(rc, ("rc=%Rrc pVCpu=%p\n", rc, pVCpu),  pVCpu->hm.s.idEnteredCpu = NIL_RTCPUID, rc);
+
+        /* Exports the host-state as we may be resuming code after a longjmp and quite
+           possibly now be scheduled on a different CPU. */
+        rc = g_HmR0.pfnExportHostState(pVCpu);
+        AssertMsgRCReturnStmt(rc, ("rc=%Rrc pVCpu=%p\n", rc, pVCpu),  pVCpu->hm.s.idEnteredCpu = NIL_RTCPUID, rc);
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+        if (fStartedSet)
+            PGMRZDynMapReleaseAutoSet(pVCpu);
+#endif
+    }
+    return rc;
+}
+
+
+/**
+ * Deinitializes the bare minimum state used for HM context and if necessary
+ * disable HM on the CPU.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+VMMR0_INT_DECL(int) HMR0LeaveCpu(PVMCPUCC pVCpu)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_HM_WRONG_CPU);
+
+    RTCPUID const idCpu    = RTMpCpuId();
+    PCHMPHYSCPU   pHostCpu = &g_HmR0.aCpuInfo[idCpu];
+
+    if (   !g_HmR0.fGlobalInit
+        && pHostCpu->fConfigured)
+    {
+        int rc = hmR0DisableCpu(idCpu);
+        AssertRCReturn(rc, rc);
+        Assert(!pHostCpu->fConfigured);
+        Assert(pHostCpu->idCpu == NIL_RTCPUID);
+
+        /* For obtaining a non-zero ASID/VPID on next re-entry. */
+        pVCpu->hm.s.idLastCpu = NIL_RTCPUID;
+    }
+
+    /* Clear it while leaving HM context, hmPokeCpuForTlbFlush() relies on this. */
+    pVCpu->hm.s.idEnteredCpu = NIL_RTCPUID;
+
+    /* De-register the longjmp-to-ring 3 callback now that we have reliquished hardware resources. */
+    VMMRZCallRing3RemoveNotification(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Thread-context hook for HM.
+ *
+ * @param   enmEvent        The thread-context event.
+ * @param   pvUser          Opaque pointer to the VMCPU.
+ */
+VMMR0_INT_DECL(void) HMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, void *pvUser)
+{
+    PVMCPUCC pVCpu = (PVMCPUCC)pvUser;
+    Assert(pVCpu);
+    Assert(g_HmR0.pfnThreadCtxCallback);
+
+    g_HmR0.pfnThreadCtxCallback(enmEvent, pVCpu, g_HmR0.fGlobalInit);
+}
+
+
+/**
+ * Runs guest code in a hardware accelerated VM.
+ *
+ * @returns Strict VBox status code. (VBOXSTRICTRC isn't used because it's
+ *          called from setjmp assembly.)
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remarks Can be called with preemption enabled if thread-context hooks are
+ *          used!!!
+ */
+VMMR0_INT_DECL(int) HMR0RunGuestCode(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    RT_NOREF(pVM);
+
+#ifdef VBOX_STRICT
+    /* With thread-context hooks we would be running this code with preemption enabled. */
+    if (!RTThreadPreemptIsEnabled(NIL_RTTHREAD))
+    {
+        PCHMPHYSCPU pHostCpu = &g_HmR0.aCpuInfo[RTMpCpuId()];
+        Assert(!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
+        Assert(pHostCpu->fConfigured);
+        AssertReturn(!ASMAtomicReadBool(&g_HmR0.fSuspended), VERR_HM_SUSPEND_PENDING);
+    }
+#endif
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    AssertReturn(!VMMR0ThreadCtxHookIsEnabled(pVCpu), VERR_HM_IPE_4);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    PGMRZDynMapStartAutoSet(pVCpu);
+#endif
+
+    VBOXSTRICTRC rcStrict = g_HmR0.pfnRunGuestCode(pVCpu);
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    PGMRZDynMapReleaseAutoSet(pVCpu);
+#endif
+    return VBOXSTRICTRC_VAL(rcStrict);
+}
+
+
+/**
+ * Notification from CPUM that it has unloaded the guest FPU/SSE/AVX state from
+ * the host CPU and that guest access to it must be intercepted.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ */
+VMMR0_INT_DECL(void) HMR0NotifyCpumUnloadedGuestFpuState(PVMCPUCC pVCpu)
+{
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
+}
+
+
+/**
+ * Notification from CPUM that it has modified the host CR0 (because of FPU).
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ */
+VMMR0_INT_DECL(void) HMR0NotifyCpumModifiedHostCr0(PVMCPUCC pVCpu)
+{
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT);
+}
+
+
+/**
+ * Returns suspend status of the host.
+ *
+ * @returns Suspend pending or not.
+ */
+VMMR0_INT_DECL(bool) HMR0SuspendPending(void)
+{
+    return ASMAtomicReadBool(&g_HmR0.fSuspended);
+}
+
+
+/**
+ * Invalidates a guest page from the host TLB.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCVirt      Page to invalidate.
+ */
+VMMR0_INT_DECL(int) HMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (pVM->hm.s.vmx.fSupported)
+        return VMXR0InvalidatePage(pVCpu, GCVirt);
+    return SVMR0InvalidatePage(pVCpu, GCVirt);
+}
+
+
+/**
+ * Returns the cpu structure for the current cpu.
+ * Keep in mind that there is no guarantee it will stay the same (long jumps to ring 3!!!).
+ *
+ * @returns The cpu structure pointer.
+ */
+VMMR0_INT_DECL(PHMPHYSCPU) hmR0GetCurrentCpu(void)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    RTCPUID const idCpu = RTMpCpuId();
+    Assert(idCpu < RT_ELEMENTS(g_HmR0.aCpuInfo));
+    return &g_HmR0.aCpuInfo[idCpu];
+}
+
+
+/**
+ * Interface for importing state on demand (used by IEM).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context CPU structure.
+ * @param   fWhat       What to import, CPUMCTX_EXTRN_XXX.
+ */
+VMMR0_INT_DECL(int) HMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
+{
+    if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
+        return VMXR0ImportStateOnDemand(pVCpu, fWhat);
+    return SVMR0ImportStateOnDemand(pVCpu, fWhat);
+}
+
+#ifdef VBOX_STRICT
+
+/**
+ * Dumps a descriptor.
+ *
+ * @param   pDesc    Descriptor to dump.
+ * @param   Sel      The selector.
+ * @param   pszSel   The name of the selector.
+ */
+VMMR0_INT_DECL(void) hmR0DumpDescriptor(PCX86DESCHC pDesc, RTSEL Sel, const char *pszSel)
+{
+    /*
+     * Make variable description string.
+     */
+    static struct
+    {
+        unsigned    cch;
+        const char *psz;
+    } const s_aTypes[32] =
+    {
+# define STRENTRY(str) { sizeof(str) - 1, str }
+
+        /* system */
+# if HC_ARCH_BITS == 64
+        STRENTRY("Reserved0 "),                  /* 0x00 */
+        STRENTRY("Reserved1 "),                  /* 0x01 */
+        STRENTRY("LDT "),                        /* 0x02 */
+        STRENTRY("Reserved3 "),                  /* 0x03 */
+        STRENTRY("Reserved4 "),                  /* 0x04 */
+        STRENTRY("Reserved5 "),                  /* 0x05 */
+        STRENTRY("Reserved6 "),                  /* 0x06 */
+        STRENTRY("Reserved7 "),                  /* 0x07 */
+        STRENTRY("Reserved8 "),                  /* 0x08 */
+        STRENTRY("TSS64Avail "),                 /* 0x09 */
+        STRENTRY("ReservedA "),                  /* 0x0a */
+        STRENTRY("TSS64Busy "),                  /* 0x0b */
+        STRENTRY("Call64 "),                     /* 0x0c */
+        STRENTRY("ReservedD "),                  /* 0x0d */
+        STRENTRY("Int64 "),                      /* 0x0e */
+        STRENTRY("Trap64 "),                     /* 0x0f */
+# else
+        STRENTRY("Reserved0 "),                  /* 0x00 */
+        STRENTRY("TSS16Avail "),                 /* 0x01 */
+        STRENTRY("LDT "),                        /* 0x02 */
+        STRENTRY("TSS16Busy "),                  /* 0x03 */
+        STRENTRY("Call16 "),                     /* 0x04 */
+        STRENTRY("Task "),                       /* 0x05 */
+        STRENTRY("Int16 "),                      /* 0x06 */
+        STRENTRY("Trap16 "),                     /* 0x07 */
+        STRENTRY("Reserved8 "),                  /* 0x08 */
+        STRENTRY("TSS32Avail "),                 /* 0x09 */
+        STRENTRY("ReservedA "),                  /* 0x0a */
+        STRENTRY("TSS32Busy "),                  /* 0x0b */
+        STRENTRY("Call32 "),                     /* 0x0c */
+        STRENTRY("ReservedD "),                  /* 0x0d */
+        STRENTRY("Int32 "),                      /* 0x0e */
+        STRENTRY("Trap32 "),                     /* 0x0f */
+# endif
+        /* non system */
+        STRENTRY("DataRO "),                     /* 0x10 */
+        STRENTRY("DataRO Accessed "),            /* 0x11 */
+        STRENTRY("DataRW "),                     /* 0x12 */
+        STRENTRY("DataRW Accessed "),            /* 0x13 */
+        STRENTRY("DataDownRO "),                 /* 0x14 */
+        STRENTRY("DataDownRO Accessed "),        /* 0x15 */
+        STRENTRY("DataDownRW "),                 /* 0x16 */
+        STRENTRY("DataDownRW Accessed "),        /* 0x17 */
+        STRENTRY("CodeEO "),                     /* 0x18 */
+        STRENTRY("CodeEO Accessed "),            /* 0x19 */
+        STRENTRY("CodeER "),                     /* 0x1a */
+        STRENTRY("CodeER Accessed "),            /* 0x1b */
+        STRENTRY("CodeConfEO "),                 /* 0x1c */
+        STRENTRY("CodeConfEO Accessed "),        /* 0x1d */
+        STRENTRY("CodeConfER "),                 /* 0x1e */
+        STRENTRY("CodeConfER Accessed ")         /* 0x1f */
+# undef SYSENTRY
+    };
+# define ADD_STR(psz, pszAdd) do { strcpy(psz, pszAdd); psz += strlen(pszAdd); } while (0)
+    char        szMsg[128];
+    char       *psz = &szMsg[0];
+    unsigned    i = pDesc->Gen.u1DescType << 4 | pDesc->Gen.u4Type;
+    memcpy(psz, s_aTypes[i].psz, s_aTypes[i].cch);
+    psz += s_aTypes[i].cch;
+
+    if (pDesc->Gen.u1Present)
+        ADD_STR(psz, "Present ");
+    else
+        ADD_STR(psz, "Not-Present ");
+# if HC_ARCH_BITS == 64
+    if (pDesc->Gen.u1Long)
+        ADD_STR(psz, "64-bit ");
+    else
+        ADD_STR(psz, "Comp ");
+# else
+    if (pDesc->Gen.u1Granularity)
+        ADD_STR(psz, "Page ");
+    if (pDesc->Gen.u1DefBig)
+        ADD_STR(psz, "32-bit ");
+    else
+        ADD_STR(psz, "16-bit ");
+# endif
+# undef ADD_STR
+    *psz = '\0';
+
+    /*
+     * Limit and Base and format the output.
+     */
+#ifdef LOG_ENABLED
+    uint32_t u32Limit = X86DESC_LIMIT_G(pDesc);
+
+# if HC_ARCH_BITS == 64
+    uint64_t const u64Base  = X86DESC64_BASE(pDesc);
+    Log(("  %s { %#04x - %#RX64 %#RX64 - base=%#RX64 limit=%#08x dpl=%d } %s\n", pszSel,
+         Sel, pDesc->au64[0], pDesc->au64[1], u64Base, u32Limit, pDesc->Gen.u2Dpl, szMsg));
+# else
+    uint32_t const u32Base  = X86DESC_BASE(pDesc);
+    Log(("  %s { %#04x - %#08x %#08x - base=%#08x limit=%#08x dpl=%d } %s\n", pszSel,
+         Sel, pDesc->au32[0], pDesc->au32[1], u32Base, u32Limit, pDesc->Gen.u2Dpl, szMsg));
+# endif
+#else
+    NOREF(Sel); NOREF(pszSel);
+#endif
+}
+
+
+/**
+ * Formats a full register dump.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   fFlags  The dumping flags (HM_DUMP_REG_FLAGS_XXX).
+ */
+VMMR0_INT_DECL(void) hmR0DumpRegs(PVMCPUCC pVCpu, uint32_t fFlags)
+{
+    /*
+     * Format the flags.
+     */
+    static struct
+    {
+        const char *pszSet;
+        const char *pszClear;
+        uint32_t    fFlag;
+    } const s_aFlags[] =
+    {
+        { "vip", NULL, X86_EFL_VIP },
+        { "vif", NULL, X86_EFL_VIF },
+        { "ac",  NULL, X86_EFL_AC  },
+        { "vm",  NULL, X86_EFL_VM  },
+        { "rf",  NULL, X86_EFL_RF  },
+        { "nt",  NULL, X86_EFL_NT  },
+        { "ov",  "nv", X86_EFL_OF  },
+        { "dn",  "up", X86_EFL_DF  },
+        { "ei",  "di", X86_EFL_IF  },
+        { "tf",  NULL, X86_EFL_TF  },
+        { "nt",  "pl", X86_EFL_SF  },
+        { "nz",  "zr", X86_EFL_ZF  },
+        { "ac",  "na", X86_EFL_AF  },
+        { "po",  "pe", X86_EFL_PF  },
+        { "cy",  "nc", X86_EFL_CF  },
+    };
+    char szEFlags[80];
+    char *psz = szEFlags;
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    uint32_t uEFlags = pCtx->eflags.u32;
+    for (unsigned i = 0; i < RT_ELEMENTS(s_aFlags); i++)
+    {
+        const char *pszAdd = s_aFlags[i].fFlag & uEFlags ? s_aFlags[i].pszSet : s_aFlags[i].pszClear;
+        if (pszAdd)
+        {
+            strcpy(psz, pszAdd);
+            psz += strlen(pszAdd);
+            *psz++ = ' ';
+        }
+    }
+    psz[-1] = '\0';
+
+    if (fFlags & HM_DUMP_REG_FLAGS_GPRS)
+    {
+        /*
+         * Format the registers.
+         */
+        if (CPUMIsGuestIn64BitCode(pVCpu))
+        {
+            Log(("rax=%016RX64 rbx=%016RX64 rcx=%016RX64 rdx=%016RX64\n"
+                 "rsi=%016RX64 rdi=%016RX64 r8 =%016RX64 r9 =%016RX64\n"
+                 "r10=%016RX64 r11=%016RX64 r12=%016RX64 r13=%016RX64\n"
+                 "r14=%016RX64 r15=%016RX64\n"
+                 "rip=%016RX64 rsp=%016RX64 rbp=%016RX64 iopl=%d %*s\n"
+                 "cs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                 "ds={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                 "es={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                 "fs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                 "gs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                 "ss={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                 "cr0=%016RX64 cr2=%016RX64 cr3=%016RX64 cr4=%016RX64\n"
+                 "dr0=%016RX64 dr1=%016RX64 dr2=%016RX64 dr3=%016RX64\n"
+                 "dr4=%016RX64 dr5=%016RX64 dr6=%016RX64 dr7=%016RX64\n"
+                 "gdtr=%016RX64:%04x  idtr=%016RX64:%04x  eflags=%08x\n"
+                 "ldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
+                 "tr  ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
+                 "SysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
+                 ,
+                 pCtx->rax, pCtx->rbx, pCtx->rcx, pCtx->rdx, pCtx->rsi, pCtx->rdi,
+                 pCtx->r8, pCtx->r9, pCtx->r10, pCtx->r11, pCtx->r12, pCtx->r13,
+                 pCtx->r14, pCtx->r15,
+                 pCtx->rip, pCtx->rsp, pCtx->rbp, X86_EFL_GET_IOPL(uEFlags), 31, szEFlags,
+                 pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit, pCtx->cs.Attr.u,
+                 pCtx->ds.Sel, pCtx->ds.u64Base, pCtx->ds.u32Limit, pCtx->ds.Attr.u,
+                 pCtx->es.Sel, pCtx->es.u64Base, pCtx->es.u32Limit, pCtx->es.Attr.u,
+                 pCtx->fs.Sel, pCtx->fs.u64Base, pCtx->fs.u32Limit, pCtx->fs.Attr.u,
+                 pCtx->gs.Sel, pCtx->gs.u64Base, pCtx->gs.u32Limit, pCtx->gs.Attr.u,
+                 pCtx->ss.Sel, pCtx->ss.u64Base, pCtx->ss.u32Limit, pCtx->ss.Attr.u,
+                 pCtx->cr0,  pCtx->cr2, pCtx->cr3,  pCtx->cr4,
+                 pCtx->dr[0],  pCtx->dr[1], pCtx->dr[2],  pCtx->dr[3],
+                 pCtx->dr[4],  pCtx->dr[5], pCtx->dr[6],  pCtx->dr[7],
+                 pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, uEFlags,
+                 pCtx->ldtr.Sel, pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit, pCtx->ldtr.Attr.u,
+                 pCtx->tr.Sel, pCtx->tr.u64Base, pCtx->tr.u32Limit, pCtx->tr.Attr.u,
+                 pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp));
+        }
+        else
+            Log(("eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
+                 "eip=%08x esp=%08x ebp=%08x iopl=%d %*s\n"
+                 "cs={%04x base=%016RX64 limit=%08x flags=%08x} dr0=%08RX64 dr1=%08RX64\n"
+                 "ds={%04x base=%016RX64 limit=%08x flags=%08x} dr2=%08RX64 dr3=%08RX64\n"
+                 "es={%04x base=%016RX64 limit=%08x flags=%08x} dr4=%08RX64 dr5=%08RX64\n"
+                 "fs={%04x base=%016RX64 limit=%08x flags=%08x} dr6=%08RX64 dr7=%08RX64\n"
+                 "gs={%04x base=%016RX64 limit=%08x flags=%08x} cr0=%08RX64 cr2=%08RX64\n"
+                 "ss={%04x base=%016RX64 limit=%08x flags=%08x} cr3=%08RX64 cr4=%08RX64\n"
+                 "gdtr=%016RX64:%04x  idtr=%016RX64:%04x  eflags=%08x\n"
+                 "ldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
+                 "tr  ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
+                 "SysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
+                 ,
+                 pCtx->eax, pCtx->ebx, pCtx->ecx, pCtx->edx, pCtx->esi, pCtx->edi,
+                 pCtx->eip, pCtx->esp, pCtx->ebp, X86_EFL_GET_IOPL(uEFlags), 31, szEFlags,
+                 pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit, pCtx->cs.Attr.u, pCtx->dr[0],  pCtx->dr[1],
+                 pCtx->ds.Sel, pCtx->ds.u64Base, pCtx->ds.u32Limit, pCtx->ds.Attr.u, pCtx->dr[2],  pCtx->dr[3],
+                 pCtx->es.Sel, pCtx->es.u64Base, pCtx->es.u32Limit, pCtx->es.Attr.u, pCtx->dr[4],  pCtx->dr[5],
+                 pCtx->fs.Sel, pCtx->fs.u64Base, pCtx->fs.u32Limit, pCtx->fs.Attr.u, pCtx->dr[6],  pCtx->dr[7],
+                 pCtx->gs.Sel, pCtx->gs.u64Base, pCtx->gs.u32Limit, pCtx->gs.Attr.u, pCtx->cr0,  pCtx->cr2,
+                 pCtx->ss.Sel, pCtx->ss.u64Base, pCtx->ss.u32Limit, pCtx->ss.Attr.u, pCtx->cr3,  pCtx->cr4,
+                 pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, uEFlags,
+                 pCtx->ldtr.Sel, pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit, pCtx->ldtr.Attr.u,
+                 pCtx->tr.Sel, pCtx->tr.u64Base, pCtx->tr.u32Limit, pCtx->tr.Attr.u,
+                 pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp));
+    }
+
+    if (fFlags & HM_DUMP_REG_FLAGS_FPU)
+    {
+        PCX86FXSTATE pFpuCtx = &pCtx->CTX_SUFF(pXState)->x87;
+        Log(("FPU:\n"
+            "FCW=%04x FSW=%04x FTW=%02x\n"
+            "FOP=%04x FPUIP=%08x CS=%04x Rsrvd1=%04x\n"
+            "FPUDP=%04x DS=%04x Rsvrd2=%04x MXCSR=%08x MXCSR_MASK=%08x\n"
+            ,
+            pFpuCtx->FCW,   pFpuCtx->FSW,   pFpuCtx->FTW,
+            pFpuCtx->FOP,   pFpuCtx->FPUIP, pFpuCtx->CS, pFpuCtx->Rsrvd1,
+            pFpuCtx->FPUDP, pFpuCtx->DS,    pFpuCtx->Rsrvd2,
+            pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK));
+        NOREF(pFpuCtx);
+    }
+
+    if (fFlags & HM_DUMP_REG_FLAGS_MSRS)
+    {
+        Log(("MSR:\n"
+            "EFER         =%016RX64\n"
+            "PAT          =%016RX64\n"
+            "STAR         =%016RX64\n"
+            "CSTAR        =%016RX64\n"
+            "LSTAR        =%016RX64\n"
+            "SFMASK       =%016RX64\n"
+            "KERNELGSBASE =%016RX64\n",
+            pCtx->msrEFER,
+            pCtx->msrPAT,
+            pCtx->msrSTAR,
+            pCtx->msrCSTAR,
+            pCtx->msrLSTAR,
+            pCtx->msrSFMASK,
+            pCtx->msrKERNELGSBASE));
+    }
+}
+
+#endif /* VBOX_STRICT */
+
diff --git a/src/VBox/VMM/VMMR0/HMR0A.asm b/src/VBox/VMM/VMMR0/HMR0A.asm
new file mode 100644
index 00000000..fc1fc67f
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/HMR0A.asm
@@ -0,0 +1,1705 @@
+; $Id: HMR0A.asm $
+;; @file
+; HM - Ring-0 VMX, SVM world-switch and helper routines.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+;*********************************************************************************************************************************
+;*  Header Files                                                                                                                 *
+;*********************************************************************************************************************************
+%include "VBox/asmdefs.mac"
+%include "VBox/err.mac"
+%include "VBox/vmm/hm_vmx.mac"
+%include "VBox/vmm/cpum.mac"
+%include "VBox/vmm/vm.mac"
+%include "iprt/x86.mac"
+%include "HMInternal.mac"
+
+%ifdef RT_OS_OS2 ;; @todo fix OMF support in yasm and kick nasm out completely.
+ %macro vmwrite 2,
+    int3
+ %endmacro
+ %define vmlaunch int3
+ %define vmresume int3
+ %define vmsave int3
+ %define vmload int3
+ %define vmrun int3
+ %define clgi int3
+ %define stgi int3
+ %macro invlpga 2,
+    int3
+ %endmacro
+%endif
+
+;*********************************************************************************************************************************
+;*  Defined Constants And Macros                                                                                                 *
+;*********************************************************************************************************************************
+;; The offset of the XMM registers in X86FXSTATE.
+; Use define because I'm too lazy to convert the struct.
+%define XMM_OFF_IN_X86FXSTATE   160
+
+;; Spectre filler for 32-bit mode.
+; Some user space address that points to a 4MB page boundrary in hope that it
+; will somehow make it less useful.
+%define SPECTRE_FILLER32        0x227fffff
+;; Spectre filler for 64-bit mode.
+; Choosen to be an invalid address (also with 5 level paging).
+%define SPECTRE_FILLER64        0x02204204207fffff
+;; Spectre filler for the current CPU mode.
+%ifdef RT_ARCH_AMD64
+ %define SPECTRE_FILLER         SPECTRE_FILLER64
+%else
+ %define SPECTRE_FILLER         SPECTRE_FILLER32
+%endif
+
+;;
+; Determine skipping restoring of GDTR, IDTR, TR across VMX non-root operation.
+;
+%ifdef RT_ARCH_AMD64
+ %define VMX_SKIP_GDTR
+ %define VMX_SKIP_TR
+ %define VBOX_SKIP_RESTORE_SEG
+ %ifdef RT_OS_DARWIN
+  ; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't
+  ; risk loading a stale LDT value or something invalid.
+  %define HM_64_BIT_USE_NULL_SEL
+  ; Darwin (Mavericks) uses IDTR limit to store the CPU Id so we need to restore it always.
+  ; See @bugref{6875}.
+ %else
+  %define VMX_SKIP_IDTR
+ %endif
+%endif
+
+;; @def MYPUSHAD
+; Macro generating an equivalent to PUSHAD instruction.
+
+;; @def MYPOPAD
+; Macro generating an equivalent to POPAD instruction.
+
+;; @def MYPUSHSEGS
+; Macro saving all segment registers on the stack.
+; @param 1  Full width register name.
+; @param 2  16-bit register name for \a 1.
+
+;; @def MYPOPSEGS
+; Macro restoring all segment registers on the stack.
+; @param 1  Full width register name.
+; @param 2  16-bit register name for \a 1.
+
+%ifdef ASM_CALL64_GCC
+ %macro MYPUSHAD64 0
+   push    r15
+   push    r14
+   push    r13
+   push    r12
+   push    rbx
+ %endmacro
+ %macro MYPOPAD64 0
+   pop     rbx
+   pop     r12
+   pop     r13
+   pop     r14
+   pop     r15
+ %endmacro
+
+%else ; ASM_CALL64_MSC
+ %macro MYPUSHAD64 0
+   push    r15
+   push    r14
+   push    r13
+   push    r12
+   push    rbx
+   push    rsi
+   push    rdi
+ %endmacro
+ %macro MYPOPAD64 0
+   pop     rdi
+   pop     rsi
+   pop     rbx
+   pop     r12
+   pop     r13
+   pop     r14
+   pop     r15
+ %endmacro
+%endif
+
+%ifdef VBOX_SKIP_RESTORE_SEG
+ %macro MYPUSHSEGS64 2
+ %endmacro
+
+ %macro MYPOPSEGS64 2
+ %endmacro
+%else       ; !VBOX_SKIP_RESTORE_SEG
+ ; Trashes, rax, rdx & rcx.
+ %macro MYPUSHSEGS64 2
+  %ifndef HM_64_BIT_USE_NULL_SEL
+   mov     %2, es
+   push    %1
+   mov     %2, ds
+   push    %1
+  %endif
+
+   ; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode,
+   ; Solaris OTOH doesn't and we must save it.
+   mov     ecx, MSR_K8_FS_BASE
+   rdmsr
+   push    rdx
+   push    rax
+  %ifndef HM_64_BIT_USE_NULL_SEL
+   push    fs
+  %endif
+
+   ; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel.
+   ; The same happens on exit.
+   mov     ecx, MSR_K8_GS_BASE
+   rdmsr
+   push    rdx
+   push    rax
+  %ifndef HM_64_BIT_USE_NULL_SEL
+   push    gs
+  %endif
+ %endmacro
+
+ ; trashes, rax, rdx & rcx
+ %macro MYPOPSEGS64 2
+   ; Note: do not step through this code with a debugger!
+  %ifndef HM_64_BIT_USE_NULL_SEL
+   xor     eax, eax
+   mov     ds, ax
+   mov     es, ax
+   mov     fs, ax
+   mov     gs, ax
+  %endif
+
+  %ifndef HM_64_BIT_USE_NULL_SEL
+   pop     gs
+  %endif
+   pop     rax
+   pop     rdx
+   mov     ecx, MSR_K8_GS_BASE
+   wrmsr
+
+  %ifndef HM_64_BIT_USE_NULL_SEL
+   pop     fs
+  %endif
+   pop     rax
+   pop     rdx
+   mov     ecx, MSR_K8_FS_BASE
+   wrmsr
+   ; Now it's safe to step again
+
+  %ifndef HM_64_BIT_USE_NULL_SEL
+   pop     %1
+   mov     ds, %2
+   pop     %1
+   mov     es, %2
+  %endif
+ %endmacro
+%endif ; VBOX_SKIP_RESTORE_SEG
+
+%macro MYPUSHAD32 0
+  pushad
+%endmacro
+%macro MYPOPAD32 0
+  popad
+%endmacro
+
+%macro MYPUSHSEGS32 2
+  push    ds
+  push    es
+  push    fs
+  push    gs
+%endmacro
+%macro MYPOPSEGS32 2
+  pop     gs
+  pop     fs
+  pop     es
+  pop     ds
+%endmacro
+
+%ifdef RT_ARCH_AMD64
+ %define MYPUSHAD       MYPUSHAD64
+ %define MYPOPAD        MYPOPAD64
+ %define MYPUSHSEGS     MYPUSHSEGS64
+ %define MYPOPSEGS      MYPOPSEGS64
+%else
+ %define MYPUSHAD       MYPUSHAD32
+ %define MYPOPAD        MYPOPAD32
+ %define MYPUSHSEGS     MYPUSHSEGS32
+ %define MYPOPSEGS      MYPOPSEGS32
+%endif
+
+;;
+; Creates an indirect branch prediction barrier on CPUs that need and supports that.
+; @clobbers eax, edx, ecx
+; @param    1   How to address CPUMCTX.
+; @param    2   Which flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
+%macro INDIRECT_BRANCH_PREDICTION_BARRIER 2
+    test    byte [%1 + CPUMCTX.fWorldSwitcher], %2
+    jz      %%no_indirect_branch_barrier
+    mov     ecx, MSR_IA32_PRED_CMD
+    mov     eax, MSR_IA32_PRED_CMD_F_IBPB
+    xor     edx, edx
+    wrmsr
+%%no_indirect_branch_barrier:
+%endmacro
+
+;;
+; Creates an indirect branch prediction and L1D barrier on CPUs that need and supports that.
+; @clobbers eax, edx, ecx
+; @param    1   How to address CPUMCTX.
+; @param    2   Which IBPB flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
+; @param    3   Which FLUSH flag to test for (CPUMCTX_WSF_L1D_ENTRY)
+; @param    4   Which MDS flag to test for (CPUMCTX_WSF_MDS_ENTRY)
+%macro INDIRECT_BRANCH_PREDICTION_AND_L1_CACHE_BARRIER 4
+    ; Only one test+jmp when disabled CPUs.
+    test    byte [%1 + CPUMCTX.fWorldSwitcher], (%2 | %3 | %4)
+    jz      %%no_barrier_needed
+
+    ; The eax:edx value is the same for both.
+    AssertCompile(MSR_IA32_PRED_CMD_F_IBPB == MSR_IA32_FLUSH_CMD_F_L1D)
+    mov     eax, MSR_IA32_PRED_CMD_F_IBPB
+    xor     edx, edx
+
+    ; Indirect branch barrier.
+    test    byte [%1 + CPUMCTX.fWorldSwitcher], %2
+    jz      %%no_indirect_branch_barrier
+    mov     ecx, MSR_IA32_PRED_CMD
+    wrmsr
+%%no_indirect_branch_barrier:
+
+    ; Level 1 data cache flush.
+    test    byte [%1 + CPUMCTX.fWorldSwitcher], %3
+    jz      %%no_cache_flush_barrier
+    mov     ecx, MSR_IA32_FLUSH_CMD
+    wrmsr
+    jmp     %%no_mds_buffer_flushing    ; MDS flushing is included in L1D_FLUSH
+%%no_cache_flush_barrier:
+
+    ; MDS buffer flushing.
+    test    byte [%1 + CPUMCTX.fWorldSwitcher], %4
+    jz      %%no_mds_buffer_flushing
+    sub     xSP, xSP
+    mov     [xSP], ds
+    verw    [xSP]
+    add     xSP, xSP
+%%no_mds_buffer_flushing:
+
+%%no_barrier_needed:
+%endmacro
+
+
+;*********************************************************************************************************************************
+;*  External Symbols                                                                                                             *
+;*********************************************************************************************************************************
+%ifdef VBOX_WITH_KERNEL_USING_XMM
+extern NAME(CPUMIsGuestFPUStateActive)
+%endif
+
+
+BEGINCODE
+
+
+;;
+; Restores host-state fields.
+;
+; @returns VBox status code
+; @param   f32RestoreHost x86: [ebp + 08h]  msc: ecx  gcc: edi   RestoreHost flags.
+; @param   pRestoreHost   x86: [ebp + 0ch]  msc: rdx  gcc: rsi   Pointer to the RestoreHost struct.
+;
+ALIGNCODE(16)
+BEGINPROC VMXRestoreHostState
+%ifdef RT_ARCH_AMD64
+ %ifndef ASM_CALL64_GCC
+    ; Use GCC's input registers since we'll be needing both rcx and rdx further
+    ; down with the wrmsr instruction.  Use the R10 and R11 register for saving
+    ; RDI and RSI since MSC preserve the two latter registers.
+    mov         r10, rdi
+    mov         r11, rsi
+    mov         rdi, rcx
+    mov         rsi, rdx
+ %endif
+
+    test        edi, VMX_RESTORE_HOST_GDTR
+    jz          .test_idtr
+    lgdt        [rsi + VMXRESTOREHOST.HostGdtr]
+
+.test_idtr:
+    test        edi, VMX_RESTORE_HOST_IDTR
+    jz          .test_ds
+    lidt        [rsi + VMXRESTOREHOST.HostIdtr]
+
+.test_ds:
+    test        edi, VMX_RESTORE_HOST_SEL_DS
+    jz          .test_es
+    mov         ax, [rsi + VMXRESTOREHOST.uHostSelDS]
+    mov         ds, eax
+
+.test_es:
+    test        edi, VMX_RESTORE_HOST_SEL_ES
+    jz          .test_tr
+    mov         ax, [rsi + VMXRESTOREHOST.uHostSelES]
+    mov         es, eax
+
+.test_tr:
+    test        edi, VMX_RESTORE_HOST_SEL_TR
+    jz          .test_fs
+    ; When restoring the TR, we must first clear the busy flag or we'll end up faulting.
+    mov         dx, [rsi + VMXRESTOREHOST.uHostSelTR]
+    mov         ax, dx
+    and         eax, X86_SEL_MASK_OFF_RPL                       ; mask away TI and RPL bits leaving only the descriptor offset
+    test        edi, VMX_RESTORE_HOST_GDT_READ_ONLY | VMX_RESTORE_HOST_GDT_NEED_WRITABLE
+    jnz         .gdt_readonly
+    add         rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2]  ; xAX <- descriptor offset + GDTR.pGdt.
+    and         dword [rax + 4], ~RT_BIT(9)                     ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
+    ltr         dx
+    jmp short   .test_fs
+.gdt_readonly:
+    test        edi, VMX_RESTORE_HOST_GDT_NEED_WRITABLE
+    jnz         .gdt_readonly_need_writable
+    mov         rcx, cr0
+    mov         r9, rcx
+    add         rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2]  ; xAX <- descriptor offset + GDTR.pGdt.
+    and         rcx, ~X86_CR0_WP
+    mov         cr0, rcx
+    and         dword [rax + 4], ~RT_BIT(9)                     ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
+    ltr         dx
+    mov         cr0, r9
+    jmp short   .test_fs
+.gdt_readonly_need_writable:
+    add         rax, qword [rsi + VMXRESTOREHOST.HostGdtrRw + 2]  ; xAX <- descriptor offset + GDTR.pGdtRw
+    and         dword [rax + 4], ~RT_BIT(9)                     ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
+    lgdt        [rsi + VMXRESTOREHOST.HostGdtrRw]
+    ltr         dx
+    lgdt        [rsi + VMXRESTOREHOST.HostGdtr]                 ; load the original GDT
+
+.test_fs:
+    ;
+    ; When restoring the selector values for FS and GS, we'll temporarily trash
+    ; the base address (at least the high 32-bit bits, but quite possibly the
+    ; whole base address), the wrmsr will restore it correctly. (VT-x actually
+    ; restores the base correctly when leaving guest mode, but not the selector
+    ; value, so there is little problem with interrupts being enabled prior to
+    ; this restore job.)
+    ; We'll disable ints once for both FS and GS as that's probably faster.
+    ;
+    test        edi, VMX_RESTORE_HOST_SEL_FS | VMX_RESTORE_HOST_SEL_GS
+    jz          .restore_success
+    pushfq
+    cli                                   ; (see above)
+
+    test        edi, VMX_RESTORE_HOST_SEL_FS
+    jz          .test_gs
+    mov         ax, word [rsi + VMXRESTOREHOST.uHostSelFS]
+    mov         fs, eax
+    mov         eax, dword [rsi + VMXRESTOREHOST.uHostFSBase]         ; uHostFSBase - Lo
+    mov         edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h]    ; uHostFSBase - Hi
+    mov         ecx, MSR_K8_FS_BASE
+    wrmsr
+
+.test_gs:
+    test        edi, VMX_RESTORE_HOST_SEL_GS
+    jz          .restore_flags
+    mov         ax, word [rsi + VMXRESTOREHOST.uHostSelGS]
+    mov         gs, eax
+    mov         eax, dword [rsi + VMXRESTOREHOST.uHostGSBase]         ; uHostGSBase - Lo
+    mov         edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h]    ; uHostGSBase - Hi
+    mov         ecx, MSR_K8_GS_BASE
+    wrmsr
+
+.restore_flags:
+    popfq
+
+.restore_success:
+    mov         eax, VINF_SUCCESS
+ %ifndef ASM_CALL64_GCC
+    ; Restore RDI and RSI on MSC.
+    mov         rdi, r10
+    mov         rsi, r11
+ %endif
+%else  ; RT_ARCH_X86
+    mov         eax, VERR_NOT_IMPLEMENTED
+%endif
+    ret
+ENDPROC VMXRestoreHostState
+
+
+;;
+; Dispatches an NMI to the host.
+;
+ALIGNCODE(16)
+BEGINPROC VMXDispatchHostNmi
+    ; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts".
+    int 2
+    ret
+ENDPROC VMXDispatchHostNmi
+
+
+;;
+; Executes VMWRITE, 64-bit value.
+;
+; @returns VBox status code.
+; @param   idxField   x86: [ebp + 08h]  msc: rcx  gcc: rdi   VMCS index.
+; @param   u64Data    x86: [ebp + 0ch]  msc: rdx  gcc: rsi   VM field value.
+;
+ALIGNCODE(16)
+BEGINPROC VMXWriteVmcs64
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_GCC
+    and         edi, 0ffffffffh
+    xor         rax, rax
+    vmwrite     rdi, rsi
+ %else
+    and         ecx, 0ffffffffh
+    xor         rax, rax
+    vmwrite     rcx, rdx
+ %endif
+%else  ; RT_ARCH_X86
+    mov         ecx, [esp + 4]          ; idxField
+    lea         edx, [esp + 8]          ; &u64Data
+    vmwrite     ecx, [edx]              ; low dword
+    jz          .done
+    jc          .done
+    inc         ecx
+    xor         eax, eax
+    vmwrite     ecx, [edx + 4]          ; high dword
+.done:
+%endif ; RT_ARCH_X86
+    jnc         .valid_vmcs
+    mov         eax, VERR_VMX_INVALID_VMCS_PTR
+    ret
+.valid_vmcs:
+    jnz         .the_end
+    mov         eax, VERR_VMX_INVALID_VMCS_FIELD
+.the_end:
+    ret
+ENDPROC VMXWriteVmcs64
+
+
+;;
+; Executes VMREAD, 64-bit value.
+;
+; @returns VBox status code.
+; @param   idxField        VMCS index.
+; @param   pData           Where to store VM field value.
+;
+;DECLASM(int) VMXReadVmcs64(uint32_t idxField, uint64_t *pData);
+ALIGNCODE(16)
+BEGINPROC VMXReadVmcs64
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_GCC
+    and         edi, 0ffffffffh
+    xor         rax, rax
+    vmread      [rsi], rdi
+ %else
+    and         ecx, 0ffffffffh
+    xor         rax, rax
+    vmread      [rdx], rcx
+ %endif
+%else  ; RT_ARCH_X86
+    mov         ecx, [esp + 4]          ; idxField
+    mov         edx, [esp + 8]          ; pData
+    vmread      [edx], ecx              ; low dword
+    jz          .done
+    jc          .done
+    inc         ecx
+    xor         eax, eax
+    vmread      [edx + 4], ecx          ; high dword
+.done:
+%endif ; RT_ARCH_X86
+    jnc         .valid_vmcs
+    mov         eax, VERR_VMX_INVALID_VMCS_PTR
+    ret
+.valid_vmcs:
+    jnz         .the_end
+    mov         eax, VERR_VMX_INVALID_VMCS_FIELD
+.the_end:
+    ret
+ENDPROC VMXReadVmcs64
+
+
+;;
+; Executes VMREAD, 32-bit value.
+;
+; @returns VBox status code.
+; @param   idxField        VMCS index.
+; @param   pu32Data        Where to store VM field value.
+;
+;DECLASM(int) VMXReadVmcs32(uint32_t idxField, uint32_t *pu32Data);
+ALIGNCODE(16)
+BEGINPROC VMXReadVmcs32
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_GCC
+    and     edi, 0ffffffffh
+    xor     rax, rax
+    vmread  r10, rdi
+    mov     [rsi], r10d
+ %else
+    and     ecx, 0ffffffffh
+    xor     rax, rax
+    vmread  r10, rcx
+    mov     [rdx], r10d
+ %endif
+%else  ; RT_ARCH_X86
+    mov     ecx, [esp + 4]              ; idxField
+    mov     edx, [esp + 8]              ; pu32Data
+    xor     eax, eax
+    vmread  [edx], ecx
+%endif ; RT_ARCH_X86
+    jnc     .valid_vmcs
+    mov     eax, VERR_VMX_INVALID_VMCS_PTR
+    ret
+.valid_vmcs:
+    jnz     .the_end
+    mov     eax, VERR_VMX_INVALID_VMCS_FIELD
+.the_end:
+    ret
+ENDPROC VMXReadVmcs32
+
+
+;;
+; Executes VMWRITE, 32-bit value.
+;
+; @returns VBox status code.
+; @param   idxField        VMCS index.
+; @param   u32Data         Where to store VM field value.
+;
+;DECLASM(int) VMXWriteVmcs32(uint32_t idxField, uint32_t u32Data);
+ALIGNCODE(16)
+BEGINPROC VMXWriteVmcs32
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_GCC
+    and     edi, 0ffffffffh
+    and     esi, 0ffffffffh
+    xor     rax, rax
+    vmwrite rdi, rsi
+ %else
+    and     ecx, 0ffffffffh
+    and     edx, 0ffffffffh
+    xor     rax, rax
+    vmwrite rcx, rdx
+ %endif
+%else  ; RT_ARCH_X86
+    mov     ecx, [esp + 4]              ; idxField
+    mov     edx, [esp + 8]              ; u32Data
+    xor     eax, eax
+    vmwrite ecx, edx
+%endif ; RT_ARCH_X86
+    jnc     .valid_vmcs
+    mov     eax, VERR_VMX_INVALID_VMCS_PTR
+    ret
+.valid_vmcs:
+    jnz     .the_end
+    mov     eax, VERR_VMX_INVALID_VMCS_FIELD
+.the_end:
+    ret
+ENDPROC VMXWriteVmcs32
+
+
+;;
+; Executes VMXON.
+;
+; @returns VBox status code.
+; @param   HCPhysVMXOn      Physical address of VMXON structure.
+;
+;DECLASM(int) VMXEnable(RTHCPHYS HCPhysVMXOn);
+BEGINPROC VMXEnable
+%ifdef RT_ARCH_AMD64
+    xor     rax, rax
+ %ifdef ASM_CALL64_GCC
+    push    rdi
+ %else
+    push    rcx
+ %endif
+    vmxon   [rsp]
+%else  ; RT_ARCH_X86
+    xor     eax, eax
+    vmxon   [esp + 4]
+%endif ; RT_ARCH_X86
+    jnc     .good
+    mov     eax, VERR_VMX_INVALID_VMXON_PTR
+    jmp     .the_end
+
+.good:
+    jnz     .the_end
+    mov     eax, VERR_VMX_VMXON_FAILED
+
+.the_end:
+%ifdef RT_ARCH_AMD64
+    add     rsp, 8
+%endif
+    ret
+ENDPROC VMXEnable
+
+
+;;
+; Executes VMXOFF.
+;
+;DECLASM(void) VMXDisable(void);
+BEGINPROC VMXDisable
+    vmxoff
+.the_end:
+    ret
+ENDPROC VMXDisable
+
+
+;;
+; Executes VMCLEAR.
+;
+; @returns VBox status code.
+; @param   HCPhysVmcs     Physical address of VM control structure.
+;
+;DECLASM(int) VMXClearVmcs(RTHCPHYS HCPhysVmcs);
+ALIGNCODE(16)
+BEGINPROC VMXClearVmcs
+%ifdef RT_ARCH_AMD64
+    xor     rax, rax
+ %ifdef ASM_CALL64_GCC
+    push    rdi
+ %else
+    push    rcx
+ %endif
+    vmclear [rsp]
+%else  ; RT_ARCH_X86
+    xor     eax, eax
+    vmclear [esp + 4]
+%endif ; RT_ARCH_X86
+    jnc     .the_end
+    mov     eax, VERR_VMX_INVALID_VMCS_PTR
+.the_end:
+%ifdef RT_ARCH_AMD64
+    add     rsp, 8
+%endif
+    ret
+ENDPROC VMXClearVmcs
+
+
+;;
+; Executes VMPTRLD.
+;
+; @returns VBox status code.
+; @param   HCPhysVmcs     Physical address of VMCS structure.
+;
+;DECLASM(int) VMXLoadVmcs(RTHCPHYS HCPhysVmcs);
+ALIGNCODE(16)
+BEGINPROC VMXLoadVmcs
+%ifdef RT_ARCH_AMD64
+    xor     rax, rax
+ %ifdef ASM_CALL64_GCC
+    push    rdi
+ %else
+    push    rcx
+ %endif
+    vmptrld [rsp]
+%else
+    xor     eax, eax
+    vmptrld [esp + 4]
+%endif
+    jnc     .the_end
+    mov     eax, VERR_VMX_INVALID_VMCS_PTR
+.the_end:
+%ifdef RT_ARCH_AMD64
+    add     rsp, 8
+%endif
+    ret
+ENDPROC VMXLoadVmcs
+
+
+;;
+; Executes VMPTRST.
+;
+; @returns VBox status code.
+; @param    [esp + 04h]  gcc:rdi  msc:rcx   Param 1 - First parameter - Address that will receive the current pointer.
+;
+;DECLASM(int) VMXGetCurrentVmcs(RTHCPHYS *pVMCS);
+BEGINPROC VMXGetCurrentVmcs
+%ifdef RT_OS_OS2
+    mov     eax, VERR_NOT_SUPPORTED
+    ret
+%else
+ %ifdef RT_ARCH_AMD64
+  %ifdef ASM_CALL64_GCC
+    vmptrst qword [rdi]
+  %else
+    vmptrst qword [rcx]
+  %endif
+ %else
+    vmptrst qword [esp+04h]
+ %endif
+    xor     eax, eax
+.the_end:
+    ret
+%endif
+ENDPROC VMXGetCurrentVmcs
+
+;;
+; Invalidate a page using INVEPT.
+;
+; @param   enmTlbFlush  msc:ecx  gcc:edi  x86:[esp+04]  Type of flush.
+; @param   pDescriptor  msc:edx  gcc:esi  x86:[esp+08]  Descriptor pointer.
+;
+;DECLASM(int) VMXR0InvEPT(VMXTLBFLUSHEPT enmTlbFlush, uint64_t *pDescriptor);
+BEGINPROC VMXR0InvEPT
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_GCC
+    and         edi, 0ffffffffh
+    xor         rax, rax
+;    invept      rdi, qword [rsi]
+    DB          0x66, 0x0F, 0x38, 0x80, 0x3E
+ %else
+    and         ecx, 0ffffffffh
+    xor         rax, rax
+;    invept      rcx, qword [rdx]
+    DB          0x66, 0x0F, 0x38, 0x80, 0xA
+ %endif
+%else
+    mov         ecx, [esp + 4]
+    mov         edx, [esp + 8]
+    xor         eax, eax
+;    invept      ecx, qword [edx]
+    DB          0x66, 0x0F, 0x38, 0x80, 0xA
+%endif
+    jnc         .valid_vmcs
+    mov         eax, VERR_VMX_INVALID_VMCS_PTR
+    ret
+.valid_vmcs:
+    jnz         .the_end
+    mov         eax, VERR_INVALID_PARAMETER
+.the_end:
+    ret
+ENDPROC VMXR0InvEPT
+
+
+;;
+; Invalidate a page using INVVPID.
+;
+; @param   enmTlbFlush  msc:ecx  gcc:edi  x86:[esp+04]  Type of flush
+; @param   pDescriptor  msc:edx  gcc:esi  x86:[esp+08]  Descriptor pointer
+;
+;DECLASM(int) VMXR0InvVPID(VMXTLBFLUSHVPID enmTlbFlush, uint64_t *pDescriptor);
+BEGINPROC VMXR0InvVPID
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_GCC
+    and         edi, 0ffffffffh
+    xor         rax, rax
+;    invvpid     rdi, qword [rsi]
+    DB          0x66, 0x0F, 0x38, 0x81, 0x3E
+ %else
+    and         ecx, 0ffffffffh
+    xor         rax, rax
+;    invvpid     rcx, qword [rdx]
+    DB          0x66, 0x0F, 0x38, 0x81, 0xA
+ %endif
+%else
+    mov         ecx, [esp + 4]
+    mov         edx, [esp + 8]
+    xor         eax, eax
+;    invvpid     ecx, qword [edx]
+    DB          0x66, 0x0F, 0x38, 0x81, 0xA
+%endif
+    jnc         .valid_vmcs
+    mov         eax, VERR_VMX_INVALID_VMCS_PTR
+    ret
+.valid_vmcs:
+    jnz         .the_end
+    mov         eax, VERR_INVALID_PARAMETER
+.the_end:
+    ret
+ENDPROC VMXR0InvVPID
+
+
+%if GC_ARCH_BITS == 64
+;;
+; Executes INVLPGA.
+;
+; @param   pPageGC  msc:rcx  gcc:rdi  x86:[esp+04]  Virtual page to invalidate
+; @param   uASID    msc:rdx  gcc:rsi  x86:[esp+0C]  Tagged TLB id
+;
+;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID);
+BEGINPROC SVMR0InvlpgA
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_GCC
+    mov     rax, rdi
+    mov     rcx, rsi
+ %else
+    mov     rax, rcx
+    mov     rcx, rdx
+ %endif
+%else
+    mov     eax, [esp + 4]
+    mov     ecx, [esp + 0Ch]
+%endif
+    invlpga [xAX], ecx
+    ret
+ENDPROC SVMR0InvlpgA
+
+%else ; GC_ARCH_BITS != 64
+;;
+; Executes INVLPGA
+;
+; @param   pPageGC  msc:ecx  gcc:edi  x86:[esp+04]  Virtual page to invalidate
+; @param   uASID    msc:edx  gcc:esi  x86:[esp+08]  Tagged TLB id
+;
+;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID);
+BEGINPROC SVMR0InvlpgA
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_GCC
+    movzx   rax, edi
+    mov     ecx, esi
+ %else
+    ; from http://www.cs.cmu.edu/~fp/courses/15213-s06/misc/asm64-handout.pdf:
+    ; "Perhaps unexpectedly, instructions that move or generate 32-bit register
+    ;  values also set the upper 32 bits of the register to zero. Consequently
+    ;  there is no need for an instruction movzlq."
+    mov     eax, ecx
+    mov     ecx, edx
+ %endif
+%else
+    mov     eax, [esp + 4]
+    mov     ecx, [esp + 8]
+%endif
+    invlpga [xAX], ecx
+    ret
+ENDPROC SVMR0InvlpgA
+
+%endif ; GC_ARCH_BITS != 64
+
+
+%ifdef VBOX_WITH_KERNEL_USING_XMM
+
+;;
+; Wrapper around vmx.pfnStartVM that preserves host XMM registers and
+; load the guest ones when necessary.
+;
+; @cproto       DECLASM(int) hmR0VMXStartVMWrapXMM(RTHCUINT fResume, PCPUMCTX pCtx, void *pvUnused, PVM pVM,
+;                                                  PVMCPU pVCpu, PFNHMVMXSTARTVM pfnStartVM);
+;
+; @returns      eax
+;
+; @param        fResumeVM       msc:rcx
+; @param        pCtx            msc:rdx
+; @param        pvUnused        msc:r8
+; @param        pVM             msc:r9
+; @param        pVCpu           msc:[rbp+30h]   The cross context virtual CPU structure of the calling EMT.
+; @param        pfnStartVM      msc:[rbp+38h]
+;
+; @remarks      This is essentially the same code as hmR0SVMRunWrapXMM, only the parameters differ a little bit.
+;
+; @remarks      Drivers shouldn't use AVX registers without saving+loading:
+;                   https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
+;               However the compiler docs have different idea:
+;                   https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
+;               We'll go with the former for now.
+;
+; ASSUMING 64-bit and windows for now.
+;
+ALIGNCODE(16)
+BEGINPROC hmR0VMXStartVMWrapXMM
+        push    xBP
+        mov     xBP, xSP
+        sub     xSP, 0b0h + 040h ; Don't bother optimizing the frame size.
+
+        ; Spill input parameters.
+        mov     [xBP + 010h], rcx       ; fResumeVM
+        mov     [xBP + 018h], rdx       ; pCtx
+        mov     [xBP + 020h], r8        ; pvUnused
+        mov     [xBP + 028h], r9        ; pVM
+
+        ; Ask CPUM whether we've started using the FPU yet.
+        mov     rcx, [xBP + 30h]        ; pVCpu
+        call    NAME(CPUMIsGuestFPUStateActive)
+        test    al, al
+        jnz     .guest_fpu_state_active
+
+        ; No need to mess with XMM registers just call the start routine and return.
+        mov     r11, [xBP + 38h]        ; pfnStartVM
+        mov     r10, [xBP + 30h]        ; pVCpu
+        mov     [xSP + 020h], r10
+        mov     rcx, [xBP + 010h]       ; fResumeVM
+        mov     rdx, [xBP + 018h]       ; pCtx
+        mov     r8,  [xBP + 020h]       ; pvUnused
+        mov     r9,  [xBP + 028h]       ; pVM
+        call    r11
+
+        leave
+        ret
+
+ALIGNCODE(8)
+.guest_fpu_state_active:
+        ; Save the non-volatile host XMM registers.
+        movdqa  [rsp + 040h + 000h], xmm6
+        movdqa  [rsp + 040h + 010h], xmm7
+        movdqa  [rsp + 040h + 020h], xmm8
+        movdqa  [rsp + 040h + 030h], xmm9
+        movdqa  [rsp + 040h + 040h], xmm10
+        movdqa  [rsp + 040h + 050h], xmm11
+        movdqa  [rsp + 040h + 060h], xmm12
+        movdqa  [rsp + 040h + 070h], xmm13
+        movdqa  [rsp + 040h + 080h], xmm14
+        movdqa  [rsp + 040h + 090h], xmm15
+        stmxcsr [rsp + 040h + 0a0h]
+
+        mov     r10, [xBP + 018h]       ; pCtx
+        mov     eax, [r10 + CPUMCTX.fXStateMask]
+        test    eax, eax
+        jz      .guest_fpu_state_manually
+
+        ;
+        ; Using XSAVE to load the guest XMM, YMM and ZMM registers.
+        ;
+        and     eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
+        xor     edx, edx
+        mov     r10, [r10 + CPUMCTX.pXStateR0]
+        xrstor  [r10]
+
+        ; Make the call (same as in the other case).
+        mov     r11, [xBP + 38h]        ; pfnStartVM
+        mov     r10, [xBP + 30h]        ; pVCpu
+        mov     [xSP + 020h], r10
+        mov     rcx, [xBP + 010h]       ; fResumeVM
+        mov     rdx, [xBP + 018h]       ; pCtx
+        mov     r8,  [xBP + 020h]       ; pvUnused
+        mov     r9,  [xBP + 028h]       ; pVM
+        call    r11
+
+        mov     r11d, eax               ; save return value (xsave below uses eax)
+
+        ; Save the guest XMM registers.
+        mov     r10, [xBP + 018h]       ; pCtx
+        mov     eax, [r10 + CPUMCTX.fXStateMask]
+        and     eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
+        xor     edx, edx
+        mov     r10, [r10 + CPUMCTX.pXStateR0]
+        xsave  [r10]
+
+        mov     eax, r11d               ; restore return value
+
+.restore_non_volatile_host_xmm_regs:
+        ; Load the non-volatile host XMM registers.
+        movdqa  xmm6,  [rsp + 040h + 000h]
+        movdqa  xmm7,  [rsp + 040h + 010h]
+        movdqa  xmm8,  [rsp + 040h + 020h]
+        movdqa  xmm9,  [rsp + 040h + 030h]
+        movdqa  xmm10, [rsp + 040h + 040h]
+        movdqa  xmm11, [rsp + 040h + 050h]
+        movdqa  xmm12, [rsp + 040h + 060h]
+        movdqa  xmm13, [rsp + 040h + 070h]
+        movdqa  xmm14, [rsp + 040h + 080h]
+        movdqa  xmm15, [rsp + 040h + 090h]
+        ldmxcsr        [rsp + 040h + 0a0h]
+        leave
+        ret
+
+        ;
+        ; No XSAVE, load and save the guest XMM registers manually.
+        ;
+.guest_fpu_state_manually:
+        ; Load the full guest XMM register state.
+        mov     r10, [r10 + CPUMCTX.pXStateR0]
+        movdqa  xmm0,  [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
+        movdqa  xmm1,  [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
+        movdqa  xmm2,  [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
+        movdqa  xmm3,  [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
+        movdqa  xmm4,  [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
+        movdqa  xmm5,  [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
+        movdqa  xmm6,  [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
+        movdqa  xmm7,  [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
+        movdqa  xmm8,  [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
+        movdqa  xmm9,  [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
+        movdqa  xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
+        movdqa  xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
+        movdqa  xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
+        movdqa  xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
+        movdqa  xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
+        movdqa  xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
+        ldmxcsr        [r10 + X86FXSTATE.MXCSR]
+
+        ; Make the call (same as in the other case).
+        mov     r11, [xBP + 38h]        ; pfnStartVM
+        mov     r10, [xBP + 30h]        ; pVCpu
+        mov     [xSP + 020h], r10
+        mov     rcx, [xBP + 010h]       ; fResumeVM
+        mov     rdx, [xBP + 018h]       ; pCtx
+        mov     r8,  [xBP + 020h]       ; pvUnused
+        mov     r9,  [xBP + 028h]       ; pVM
+        call    r11
+
+        ; Save the guest XMM registers.
+        mov     r10, [xBP + 018h]       ; pCtx
+        mov     r10, [r10 + CPUMCTX.pXStateR0]
+        stmxcsr [r10 + X86FXSTATE.MXCSR]
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
+        jmp     .restore_non_volatile_host_xmm_regs
+ENDPROC   hmR0VMXStartVMWrapXMM
+
+;;
+; Wrapper around svm.pfnVMRun that preserves host XMM registers and
+; load the guest ones when necessary.
+;
+; @cproto       DECLASM(int) hmR0SVMRunWrapXMM(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu,
+;                                              PFNHMSVMVMRUN pfnVMRun);
+;
+; @returns      eax
+;
+; @param        HCPhysVmcbHost  msc:rcx
+; @param        HCPhysVmcb      msc:rdx
+; @param        pCtx            msc:r8
+; @param        pVM             msc:r9
+; @param        pVCpu           msc:[rbp+30h]   The cross context virtual CPU structure of the calling EMT.
+; @param        pfnVMRun        msc:[rbp+38h]
+;
+; @remarks      This is essentially the same code as hmR0VMXStartVMWrapXMM, only the parameters differ a little bit.
+;
+; @remarks      Drivers shouldn't use AVX registers without saving+loading:
+;                   https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
+;               However the compiler docs have different idea:
+;                   https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
+;               We'll go with the former for now.
+;
+; ASSUMING 64-bit and windows for now.
+ALIGNCODE(16)
+BEGINPROC hmR0SVMRunWrapXMM
+        push    xBP
+        mov     xBP, xSP
+        sub     xSP, 0b0h + 040h        ; don't bother optimizing the frame size
+
+        ; Spill input parameters.
+        mov     [xBP + 010h], rcx       ; HCPhysVmcbHost
+        mov     [xBP + 018h], rdx       ; HCPhysVmcb
+        mov     [xBP + 020h], r8        ; pCtx
+        mov     [xBP + 028h], r9        ; pVM
+
+        ; Ask CPUM whether we've started using the FPU yet.
+        mov     rcx, [xBP + 30h]        ; pVCpu
+        call    NAME(CPUMIsGuestFPUStateActive)
+        test    al, al
+        jnz     .guest_fpu_state_active
+
+        ; No need to mess with XMM registers just call the start routine and return.
+        mov     r11, [xBP + 38h]        ; pfnVMRun
+        mov     r10, [xBP + 30h]        ; pVCpu
+        mov     [xSP + 020h], r10
+        mov     rcx, [xBP + 010h]       ; HCPhysVmcbHost
+        mov     rdx, [xBP + 018h]       ; HCPhysVmcb
+        mov     r8,  [xBP + 020h]       ; pCtx
+        mov     r9,  [xBP + 028h]       ; pVM
+        call    r11
+
+        leave
+        ret
+
+ALIGNCODE(8)
+.guest_fpu_state_active:
+        ; Save the non-volatile host XMM registers.
+        movdqa  [rsp + 040h + 000h], xmm6
+        movdqa  [rsp + 040h + 010h], xmm7
+        movdqa  [rsp + 040h + 020h], xmm8
+        movdqa  [rsp + 040h + 030h], xmm9
+        movdqa  [rsp + 040h + 040h], xmm10
+        movdqa  [rsp + 040h + 050h], xmm11
+        movdqa  [rsp + 040h + 060h], xmm12
+        movdqa  [rsp + 040h + 070h], xmm13
+        movdqa  [rsp + 040h + 080h], xmm14
+        movdqa  [rsp + 040h + 090h], xmm15
+        stmxcsr [rsp + 040h + 0a0h]
+
+        mov     r10, [xBP + 020h]       ; pCtx
+        mov     eax, [r10 + CPUMCTX.fXStateMask]
+        test    eax, eax
+        jz      .guest_fpu_state_manually
+
+        ;
+        ; Using XSAVE.
+        ;
+        and     eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
+        xor     edx, edx
+        mov     r10, [r10 + CPUMCTX.pXStateR0]
+        xrstor  [r10]
+
+        ; Make the call (same as in the other case).
+        mov     r11, [xBP + 38h]        ; pfnVMRun
+        mov     r10, [xBP + 30h]        ; pVCpu
+        mov     [xSP + 020h], r10
+        mov     rcx, [xBP + 010h]       ; HCPhysVmcbHost
+        mov     rdx, [xBP + 018h]       ; HCPhysVmcb
+        mov     r8,  [xBP + 020h]       ; pCtx
+        mov     r9,  [xBP + 028h]       ; pVM
+        call    r11
+
+        mov     r11d, eax               ; save return value (xsave below uses eax)
+
+        ; Save the guest XMM registers.
+        mov     r10, [xBP + 020h]       ; pCtx
+        mov     eax, [r10 + CPUMCTX.fXStateMask]
+        and     eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
+        xor     edx, edx
+        mov     r10, [r10 + CPUMCTX.pXStateR0]
+        xsave  [r10]
+
+        mov     eax, r11d               ; restore return value
+
+.restore_non_volatile_host_xmm_regs:
+        ; Load the non-volatile host XMM registers.
+        movdqa  xmm6,  [rsp + 040h + 000h]
+        movdqa  xmm7,  [rsp + 040h + 010h]
+        movdqa  xmm8,  [rsp + 040h + 020h]
+        movdqa  xmm9,  [rsp + 040h + 030h]
+        movdqa  xmm10, [rsp + 040h + 040h]
+        movdqa  xmm11, [rsp + 040h + 050h]
+        movdqa  xmm12, [rsp + 040h + 060h]
+        movdqa  xmm13, [rsp + 040h + 070h]
+        movdqa  xmm14, [rsp + 040h + 080h]
+        movdqa  xmm15, [rsp + 040h + 090h]
+        ldmxcsr [rsp + 040h + 0a0h]
+        leave
+        ret
+
+        ;
+        ; No XSAVE, load and save the guest XMM registers manually.
+        ;
+.guest_fpu_state_manually:
+        ; Load the full guest XMM register state.
+        mov     r10, [r10 + CPUMCTX.pXStateR0]
+        movdqa  xmm0,  [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
+        movdqa  xmm1,  [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
+        movdqa  xmm2,  [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
+        movdqa  xmm3,  [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
+        movdqa  xmm4,  [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
+        movdqa  xmm5,  [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
+        movdqa  xmm6,  [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
+        movdqa  xmm7,  [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
+        movdqa  xmm8,  [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
+        movdqa  xmm9,  [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
+        movdqa  xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
+        movdqa  xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
+        movdqa  xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
+        movdqa  xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
+        movdqa  xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
+        movdqa  xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
+        ldmxcsr        [r10 + X86FXSTATE.MXCSR]
+
+        ; Make the call (same as in the other case).
+        mov     r11, [xBP + 38h]        ; pfnVMRun
+        mov     r10, [xBP + 30h]        ; pVCpu
+        mov     [xSP + 020h], r10
+        mov     rcx, [xBP + 010h]       ; HCPhysVmcbHost
+        mov     rdx, [xBP + 018h]       ; HCPhysVmcb
+        mov     r8,  [xBP + 020h]       ; pCtx
+        mov     r9,  [xBP + 028h]       ; pVM
+        call    r11
+
+        ; Save the guest XMM registers.
+        mov     r10, [xBP + 020h]       ; pCtx
+        mov     r10, [r10 + CPUMCTX.pXStateR0]
+        stmxcsr [r10 + X86FXSTATE.MXCSR]
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
+        movdqa  [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
+        jmp     .restore_non_volatile_host_xmm_regs
+ENDPROC   hmR0SVMRunWrapXMM
+
+%endif ; VBOX_WITH_KERNEL_USING_XMM
+
+
+%ifdef RT_ARCH_AMD64
+;; @def RESTORE_STATE_VM64
+; Macro restoring essential host state and updating guest state
+; for 64-bit host, 64-bit guest for VT-x.
+;
+%macro RESTORE_STATE_VM64 0
+    ; Restore base and limit of the IDTR & GDTR.
+ %ifndef VMX_SKIP_IDTR
+    lidt    [xSP]
+    add     xSP, xCB * 2
+ %endif
+ %ifndef VMX_SKIP_GDTR
+    lgdt    [xSP]
+    add     xSP, xCB * 2
+ %endif
+
+    push    xDI
+ %ifndef VMX_SKIP_TR
+    mov     xDI, [xSP + xCB * 3]        ; pCtx (*3 to skip the saved xDI, TR, LDTR)
+ %else
+    mov     xDI, [xSP + xCB * 2]        ; pCtx (*2 to skip the saved xDI, LDTR)
+ %endif
+
+    mov     qword [xDI + CPUMCTX.eax], rax
+    mov     rax, SPECTRE_FILLER64
+    mov     qword [xDI + CPUMCTX.ebx], rbx
+    mov     rbx, rax
+    mov     qword [xDI + CPUMCTX.ecx], rcx
+    mov     rcx, rax
+    mov     qword [xDI + CPUMCTX.edx], rdx
+    mov     rdx, rax
+    mov     qword [xDI + CPUMCTX.esi], rsi
+    mov     rsi, rax
+    mov     qword [xDI + CPUMCTX.ebp], rbp
+    mov     rbp, rax
+    mov     qword [xDI + CPUMCTX.r8],  r8
+    mov     r8, rax
+    mov     qword [xDI + CPUMCTX.r9],  r9
+    mov     r9, rax
+    mov     qword [xDI + CPUMCTX.r10], r10
+    mov     r10, rax
+    mov     qword [xDI + CPUMCTX.r11], r11
+    mov     r11, rax
+    mov     qword [xDI + CPUMCTX.r12], r12
+    mov     r12, rax
+    mov     qword [xDI + CPUMCTX.r13], r13
+    mov     r13, rax
+    mov     qword [xDI + CPUMCTX.r14], r14
+    mov     r14, rax
+    mov     qword [xDI + CPUMCTX.r15], r15
+    mov     r15, rax
+    mov     rax, cr2
+    mov     qword [xDI + CPUMCTX.cr2], rax
+
+    pop     xAX                                 ; The guest rdi we pushed above
+    mov     qword [xDI + CPUMCTX.edi], rax
+
+    ; Fight spectre.
+    INDIRECT_BRANCH_PREDICTION_BARRIER xDI, CPUMCTX_WSF_IBPB_EXIT
+
+ %ifndef VMX_SKIP_TR
+    ; Restore TSS selector; must mark it as not busy before using ltr!
+    ; ASSUME that this is supposed to be 'BUSY' (saves 20-30 ticks on the T42p).
+    ; @todo get rid of sgdt
+    pop     xBX         ; Saved TR
+    sub     xSP, xCB * 2
+    sgdt    [xSP]
+    mov     xAX, xBX
+    and     eax, X86_SEL_MASK_OFF_RPL           ; mask away TI and RPL bits leaving only the descriptor offset
+    add     xAX, [xSP + 2]                      ; eax <- GDTR.address + descriptor offset
+    and     dword [xAX + 4], ~RT_BIT(9)         ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
+    ltr     bx
+    add     xSP, xCB * 2
+ %endif
+
+    pop     xAX         ; Saved LDTR
+    cmp     eax, 0
+    je      %%skip_ldt_write64
+    lldt    ax
+
+%%skip_ldt_write64:
+    pop     xSI         ; pCtx (needed in rsi by the macros below)
+
+    ; Restore segment registers.
+    MYPOPSEGS xAX, ax
+
+    ; Restore the host XCR0 if necessary.
+    pop     xCX
+    test    ecx, ecx
+    jnz     %%xcr0_after_skip
+    pop     xAX
+    pop     xDX
+    xsetbv                              ; ecx is already zero.
+%%xcr0_after_skip:
+
+    ; Restore general purpose registers.
+    MYPOPAD
+%endmacro
+
+
+;;
+; Prepares for and executes VMLAUNCH/VMRESUME (64 bits guest mode)
+;
+; @returns VBox status code
+; @param    fResume    msc:rcx, gcc:rdi       Whether to use vmlauch/vmresume.
+; @param    pCtx       msc:rdx, gcc:rsi       Pointer to the guest-CPU context.
+; @param    pvUnused   msc:r8,  gcc:rdx       Unused argument.
+; @param    pVM        msc:r9,  gcc:rcx       The cross context VM structure.
+; @param    pVCpu      msc:[ebp+30], gcc:r8   The cross context virtual CPU structure of the calling EMT.
+;
+ALIGNCODE(16)
+BEGINPROC VMXR0StartVM64
+    push    xBP
+    mov     xBP, xSP
+
+    pushf
+    cli
+
+    ; Save all general purpose host registers.
+    MYPUSHAD
+
+    ; First we have to save some final CPU context registers.
+    lea     r10, [.vmlaunch64_done wrt rip]
+    mov     rax, VMX_VMCS_HOST_RIP      ; return address (too difficult to continue after VMLAUNCH?)
+    vmwrite rax, r10
+    ; Note: ASSUMES success!
+
+    ;
+    ; Unify the input parameter registers.
+    ;
+%ifdef ASM_CALL64_GCC
+    ; fResume already in rdi
+    ; pCtx    already in rsi
+    mov     rbx, rdx        ; pvUnused
+%else
+    mov     rdi, rcx        ; fResume
+    mov     rsi, rdx        ; pCtx
+    mov     rbx, r8         ; pvUnused
+%endif
+
+    ;
+    ; Save the host XCR0 and load the guest one if necessary.
+    ; Note! Trashes rdx and rcx.
+    ;
+%ifdef ASM_CALL64_MSC
+    mov     rax, [xBP + 30h]            ; pVCpu
+%else
+    mov     rax, r8                     ; pVCpu
+%endif
+    test    byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
+    jz      .xcr0_before_skip
+
+    xor     ecx, ecx
+    xgetbv                              ; save the host one on the stack
+    push    xDX
+    push    xAX
+
+    mov     eax, [xSI + CPUMCTX.aXcr]   ; load the guest one
+    mov     edx, [xSI + CPUMCTX.aXcr + 4]
+    xor     ecx, ecx                    ; paranoia
+    xsetbv
+
+    push    0                           ; indicate that we must restore XCR0 (popped into ecx, thus 0)
+    jmp     .xcr0_before_done
+
+.xcr0_before_skip:
+    push    3fh                         ; indicate that we need not
+.xcr0_before_done:
+
+    ;
+    ; Save segment registers.
+    ; Note! Trashes rdx & rcx, so we moved it here (amd64 case).
+    ;
+    MYPUSHSEGS xAX, ax
+
+    ; Save the pCtx pointer.
+    push    xSI
+
+    ; Save host LDTR.
+    xor     eax, eax
+    sldt    ax
+    push    xAX
+
+%ifndef VMX_SKIP_TR
+    ; The host TR limit is reset to 0x67; save & restore it manually.
+    str     eax
+    push    xAX
+%endif
+
+%ifndef VMX_SKIP_GDTR
+    ; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
+    sub     xSP, xCB * 2
+    sgdt    [xSP]
+%endif
+%ifndef VMX_SKIP_IDTR
+    sub     xSP, xCB * 2
+    sidt    [xSP]
+%endif
+
+    ; Load CR2 if necessary (may be expensive as writing CR2 is a synchronizing instruction).
+    mov     rbx, qword [xSI + CPUMCTX.cr2]
+    mov     rdx, cr2
+    cmp     rbx, rdx
+    je      .skip_cr2_write
+    mov     cr2, rbx
+
+.skip_cr2_write:
+    mov     eax, VMX_VMCS_HOST_RSP
+    vmwrite xAX, xSP
+    ; Note: ASSUMES success!
+    ; Don't mess with ESP anymore!!!
+
+    ; Fight spectre and similar.
+    INDIRECT_BRANCH_PREDICTION_AND_L1_CACHE_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY, CPUMCTX_WSF_L1D_ENTRY, CPUMCTX_WSF_MDS_ENTRY
+
+    ; Load guest general purpose registers.
+    mov     rax, qword [xSI + CPUMCTX.eax]
+    mov     rbx, qword [xSI + CPUMCTX.ebx]
+    mov     rcx, qword [xSI + CPUMCTX.ecx]
+    mov     rdx, qword [xSI + CPUMCTX.edx]
+    mov     rbp, qword [xSI + CPUMCTX.ebp]
+    mov     r8,  qword [xSI + CPUMCTX.r8]
+    mov     r9,  qword [xSI + CPUMCTX.r9]
+    mov     r10, qword [xSI + CPUMCTX.r10]
+    mov     r11, qword [xSI + CPUMCTX.r11]
+    mov     r12, qword [xSI + CPUMCTX.r12]
+    mov     r13, qword [xSI + CPUMCTX.r13]
+    mov     r14, qword [xSI + CPUMCTX.r14]
+    mov     r15, qword [xSI + CPUMCTX.r15]
+
+    ; Resume or start VM?
+    cmp     xDI, 0                  ; fResume
+
+    ; Load guest rdi & rsi.
+    mov     rdi, qword [xSI + CPUMCTX.edi]
+    mov     rsi, qword [xSI + CPUMCTX.esi]
+
+    je      .vmlaunch64_launch
+
+    vmresume
+    jc      near .vmxstart64_invalid_vmcs_ptr
+    jz      near .vmxstart64_start_failed
+    jmp     .vmlaunch64_done;      ; here if vmresume detected a failure
+
+.vmlaunch64_launch:
+    vmlaunch
+    jc      near .vmxstart64_invalid_vmcs_ptr
+    jz      near .vmxstart64_start_failed
+    jmp     .vmlaunch64_done;      ; here if vmlaunch detected a failure
+
+ALIGNCODE(16)
+.vmlaunch64_done:
+    RESTORE_STATE_VM64
+    mov     eax, VINF_SUCCESS
+
+.vmstart64_end:
+    popf
+    pop     xBP
+    ret
+
+.vmxstart64_invalid_vmcs_ptr:
+    RESTORE_STATE_VM64
+    mov     eax, VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
+    jmp     .vmstart64_end
+
+.vmxstart64_start_failed:
+    RESTORE_STATE_VM64
+    mov     eax, VERR_VMX_UNABLE_TO_START_VM
+    jmp     .vmstart64_end
+ENDPROC VMXR0StartVM64
+%endif ; RT_ARCH_AMD64
+
+
+;;
+; Clears the MDS buffers using VERW.
+ALIGNCODE(16)
+BEGINPROC hmR0MdsClear
+        sub     xSP, xCB
+        mov     [xSP], ds
+        verw    [xSP]
+        add     xSP, xCB
+        ret
+ENDPROC   hmR0MdsClear
+
+
+%ifdef RT_ARCH_AMD64
+;;
+; Prepares for and executes VMRUN (32-bit and 64-bit guests).
+;
+; @returns  VBox status code
+; @param    HCPhysVmcbHost  msc:rcx,gcc:rdi     Physical address of host VMCB.
+; @param    HCPhysVmcb      msc:rdx,gcc:rsi     Physical address of guest VMCB.
+; @param    pCtx            msc:r8,gcc:rdx      Pointer to the guest-CPU context.
+; @param    pVM             msc:r9,gcc:rcx      The cross context VM structure.
+; @param    pVCpu           msc:[rsp+28],gcc:r8 The cross context virtual CPU structure of the calling EMT.
+;
+ALIGNCODE(16)
+BEGINPROC SVMR0VMRun
+    ; Fake a cdecl stack frame
+ %ifdef ASM_CALL64_GCC
+    push    r8                ; pVCpu
+    push    rcx               ; pVM
+    push    rdx               ; pCtx
+    push    rsi               ; HCPhysVmcb
+    push    rdi               ; HCPhysVmcbHost
+ %else
+    mov     rax, [rsp + 28h]
+    push    rax               ; rbp + 30h pVCpu
+    push    r9                ; rbp + 28h pVM
+    push    r8                ; rbp + 20h pCtx
+    push    rdx               ; rbp + 18h HCPhysVmcb
+    push    rcx               ; rbp + 10h HCPhysVmcbHost
+ %endif
+    push    0                 ; rbp + 08h "fake ret addr"
+    push    rbp               ; rbp + 00h
+    mov     rbp, rsp
+    pushf
+
+    ; Manual save and restore:
+    ;  - General purpose registers except RIP, RSP, RAX
+    ;
+    ; Trashed:
+    ;  - CR2 (we don't care)
+    ;  - LDTR (reset to 0)
+    ;  - DRx (presumably not changed at all)
+    ;  - DR7 (reset to 0x400)
+
+    ; Save all general purpose host registers.
+    MYPUSHAD
+
+    ; Load pCtx into xSI.
+    mov     xSI, [rbp + xCB * 2 + RTHCPHYS_CB * 2]
+
+    ; Save the host XCR0 and load the guest one if necessary.
+    mov     rax, [xBP + 30h]                ; pVCpu
+    test    byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
+    jz      .xcr0_before_skip
+
+    xor     ecx, ecx
+    xgetbv                                  ; save the host XCR0 on the stack
+    push    xDX
+    push    xAX
+
+    mov     xSI, [xBP + xCB * 2 + RTHCPHYS_CB * 2]  ; pCtx
+    mov     eax, [xSI + CPUMCTX.aXcr]       ; load the guest XCR0
+    mov     edx, [xSI + CPUMCTX.aXcr + 4]
+    xor     ecx, ecx                        ; paranoia
+    xsetbv
+
+    push    0                               ; indicate that we must restore XCR0 (popped into ecx, thus 0)
+    jmp     .xcr0_before_done
+
+.xcr0_before_skip:
+    push    3fh                             ; indicate that we need not restore XCR0
+.xcr0_before_done:
+
+    ; Save guest CPU-context pointer for simplifying saving of the GPRs afterwards.
+    push    rsi
+
+    ; Save host fs, gs, sysenter msr etc.
+    mov     rax, [rbp + xCB * 2]            ; HCPhysVmcbHost (64 bits physical address; x86: take low dword only)
+    push    rax                             ; save for the vmload after vmrun
+    vmsave
+
+    ; Fight spectre.
+    INDIRECT_BRANCH_PREDICTION_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY
+
+    ; Setup rax for VMLOAD.
+    mov     rax, [rbp + xCB * 2 + RTHCPHYS_CB] ; HCPhysVmcb (64 bits physical address; take low dword only)
+
+    ; Load guest general purpose registers (rax is loaded from the VMCB by VMRUN).
+    mov     rbx, qword [xSI + CPUMCTX.ebx]
+    mov     rcx, qword [xSI + CPUMCTX.ecx]
+    mov     rdx, qword [xSI + CPUMCTX.edx]
+    mov     rdi, qword [xSI + CPUMCTX.edi]
+    mov     rbp, qword [xSI + CPUMCTX.ebp]
+    mov     r8,  qword [xSI + CPUMCTX.r8]
+    mov     r9,  qword [xSI + CPUMCTX.r9]
+    mov     r10, qword [xSI + CPUMCTX.r10]
+    mov     r11, qword [xSI + CPUMCTX.r11]
+    mov     r12, qword [xSI + CPUMCTX.r12]
+    mov     r13, qword [xSI + CPUMCTX.r13]
+    mov     r14, qword [xSI + CPUMCTX.r14]
+    mov     r15, qword [xSI + CPUMCTX.r15]
+    mov     rsi, qword [xSI + CPUMCTX.esi]
+
+    ; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
+    clgi
+    sti
+
+    ; Load guest FS, GS, Sysenter MSRs etc.
+    vmload
+
+    ; Run the VM.
+    vmrun
+
+    ; Save guest fs, gs, sysenter msr etc.
+    vmsave
+
+    ; Load host fs, gs, sysenter msr etc.
+    pop     rax                         ; load HCPhysVmcbHost (pushed above)
+    vmload
+
+    ; Set the global interrupt flag again, but execute cli to make sure IF=0.
+    cli
+    stgi
+
+    ; Pop the context pointer (pushed above) and save the guest GPRs (sans RSP and RAX).
+    pop     rax
+
+    mov     qword [rax + CPUMCTX.ebx], rbx
+    mov     rbx, SPECTRE_FILLER64
+    mov     qword [rax + CPUMCTX.ecx], rcx
+    mov     rcx, rbx
+    mov     qword [rax + CPUMCTX.edx], rdx
+    mov     rdx, rbx
+    mov     qword [rax + CPUMCTX.esi], rsi
+    mov     rsi, rbx
+    mov     qword [rax + CPUMCTX.edi], rdi
+    mov     rdi, rbx
+    mov     qword [rax + CPUMCTX.ebp], rbp
+    mov     rbp, rbx
+    mov     qword [rax + CPUMCTX.r8],  r8
+    mov     r8, rbx
+    mov     qword [rax + CPUMCTX.r9],  r9
+    mov     r9, rbx
+    mov     qword [rax + CPUMCTX.r10], r10
+    mov     r10, rbx
+    mov     qword [rax + CPUMCTX.r11], r11
+    mov     r11, rbx
+    mov     qword [rax + CPUMCTX.r12], r12
+    mov     r12, rbx
+    mov     qword [rax + CPUMCTX.r13], r13
+    mov     r13, rbx
+    mov     qword [rax + CPUMCTX.r14], r14
+    mov     r14, rbx
+    mov     qword [rax + CPUMCTX.r15], r15
+    mov     r15, rbx
+
+    ; Fight spectre.  Note! Trashes rax!
+    INDIRECT_BRANCH_PREDICTION_BARRIER rax, CPUMCTX_WSF_IBPB_EXIT
+
+    ; Restore the host xcr0 if necessary.
+    pop     xCX
+    test    ecx, ecx
+    jnz     .xcr0_after_skip
+    pop     xAX
+    pop     xDX
+    xsetbv                              ; ecx is already zero
+.xcr0_after_skip:
+
+    ; Restore host general purpose registers.
+    MYPOPAD
+
+    mov     eax, VINF_SUCCESS
+
+    popf
+    pop     rbp
+    add     rsp, 6 * xCB
+    ret
+ENDPROC SVMR0VMRun
+%endif ; RT_ARCH_AMD64
+
diff --git a/src/VBox/VMM/VMMR0/HMSVMR0.cpp b/src/VBox/VMM/VMMR0/HMSVMR0.cpp
new file mode 100644
index 00000000..9b71f272
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/HMSVMR0.cpp
@@ -0,0 +1,7847 @@
+/* $Id: HMSVMR0.cpp $ */
+/** @file
+ * HM SVM (AMD-V) - Host Context Ring-0.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_HM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/thread.h>
+
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/apic.h>
+#include "HMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include "HMSVMR0.h"
+#include "dtrace/VBoxVMM.h"
+
+#ifdef DEBUG_ramshankar
+# define HMSVM_SYNC_FULL_GUEST_STATE
+# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
+# define HMSVM_ALWAYS_TRAP_PF
+# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
+#endif
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_STATISTICS
+# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
+        if ((u64ExitCode) == SVM_EXIT_NPF) \
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
+        else \
+            STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
+        } while (0)
+
+# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+#  define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
+        if ((u64ExitCode) == SVM_EXIT_NPF) \
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
+        else \
+            STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
+        } while (0)
+# endif
+#else
+# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode)           do { } while (0)
+# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+#  define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode)   do { } while (0)
+# endif
+#endif /* !VBOX_WITH_STATISTICS */
+
+/** If we decide to use a function table approach this can be useful to
+ *  switch to a "static DECLCALLBACK(int)". */
+#define HMSVM_EXIT_DECL                 static int
+
+/**
+ * Subset of the guest-CPU state that is kept by SVM R0 code while executing the
+ * guest using hardware-assisted SVM.
+ *
+ * This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
+ * are swapped and restored across the world-switch and also registers like
+ * EFER, PAT MSR etc. which cannot be modified by the guest without causing a
+ * \#VMEXIT.
+ */
+#define HMSVM_CPUMCTX_EXTRN_ALL         (  CPUMCTX_EXTRN_RIP            \
+                                         | CPUMCTX_EXTRN_RFLAGS         \
+                                         | CPUMCTX_EXTRN_RAX            \
+                                         | CPUMCTX_EXTRN_RSP            \
+                                         | CPUMCTX_EXTRN_SREG_MASK      \
+                                         | CPUMCTX_EXTRN_CR0            \
+                                         | CPUMCTX_EXTRN_CR2            \
+                                         | CPUMCTX_EXTRN_CR3            \
+                                         | CPUMCTX_EXTRN_TABLE_MASK     \
+                                         | CPUMCTX_EXTRN_DR6            \
+                                         | CPUMCTX_EXTRN_DR7            \
+                                         | CPUMCTX_EXTRN_KERNEL_GS_BASE \
+                                         | CPUMCTX_EXTRN_SYSCALL_MSRS   \
+                                         | CPUMCTX_EXTRN_SYSENTER_MSRS  \
+                                         | CPUMCTX_EXTRN_HWVIRT         \
+                                         | CPUMCTX_EXTRN_HM_SVM_MASK)
+
+/**
+ * Subset of the guest-CPU state that is shared between the guest and host.
+ */
+#define HMSVM_CPUMCTX_SHARED_STATE      CPUMCTX_EXTRN_DR_MASK
+
+/** Macro for importing guest state from the VMCB back into CPUMCTX.  */
+#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
+    do { \
+        if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
+            hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
+    } while (0)
+
+/** Assert that the required state bits are fetched. */
+#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz)      AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
+                                                                  ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
+                                                                  (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
+
+/** Assert that preemption is disabled or covered by thread-context hooks. */
+#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu)              Assert(   VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
+                                                               || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+/** Assert that we haven't migrated CPUs when thread-context hooks are not
+ *  used. */
+#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu)                  AssertMsg(   VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
+                                                                  || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
+                                                                  ("Illegal migration! Entered on CPU %u Current %u\n", \
+                                                                   (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()));
+
+/** Assert that we're not executing a nested-guest. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx)       Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
+#else
+# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx)       do { NOREF((a_pCtx)); } while (0)
+#endif
+
+/** Assert that we're executing a nested-guest. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx)           Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
+#else
+# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx)           do { NOREF((a_pCtx)); } while (0)
+#endif
+
+/** Macro for checking and returning from the using function for
+ * \#VMEXIT intercepts that maybe caused during delivering of another
+ * event in the guest. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
+    do \
+    { \
+        int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
+        if (RT_LIKELY(rc == VINF_SUCCESS))        { /* continue #VMEXIT handling */ } \
+        else if (     rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS;            } \
+        else if (     rc == VINF_EM_RESET \
+                 &&   CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
+        { \
+            HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
+            return VBOXSTRICTRC_TODO(IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0)); \
+        } \
+        else \
+            return rc; \
+    } while (0)
+#else
+# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
+    do \
+    { \
+        int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
+        if (RT_LIKELY(rc == VINF_SUCCESS))        { /* continue #VMEXIT handling */ } \
+        else if (     rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS;            } \
+        else \
+            return rc; \
+    } while (0)
+#endif
+
+/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
+ * instruction that exited. */
+#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
+    do { \
+        if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
+            (a_rc) = VINF_EM_DBG_STEPPED; \
+    } while (0)
+
+/** Validate segment descriptor granularity bit. */
+#ifdef VBOX_STRICT
+# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
+    AssertMsg(   !(a_pCtx)->reg.Attr.n.u1Present \
+              || (   (a_pCtx)->reg.Attr.n.u1Granularity \
+                  ? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
+                  :  (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
+              ("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
+              (a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
+#else
+# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg)      do { } while (0)
+#endif
+
+/**
+ * Exception bitmap mask for all contributory exceptions.
+ *
+ * Page fault is deliberately excluded here as it's conditional as to whether
+ * it's contributory or benign. Page faults are handled separately.
+ */
+#define HMSVM_CONTRIBUTORY_XCPT_MASK  (  RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
+                                       | RT_BIT(X86_XCPT_DE))
+
+/**
+ * Mandatory/unconditional guest control intercepts.
+ *
+ * SMIs can and do happen in normal operation. We need not intercept them
+ * while executing the guest (or nested-guest).
+ */
+#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS           (  SVM_CTRL_INTERCEPT_INTR          \
+                                                         | SVM_CTRL_INTERCEPT_NMI           \
+                                                         | SVM_CTRL_INTERCEPT_INIT          \
+                                                         | SVM_CTRL_INTERCEPT_RDPMC         \
+                                                         | SVM_CTRL_INTERCEPT_CPUID         \
+                                                         | SVM_CTRL_INTERCEPT_RSM           \
+                                                         | SVM_CTRL_INTERCEPT_HLT           \
+                                                         | SVM_CTRL_INTERCEPT_IOIO_PROT     \
+                                                         | SVM_CTRL_INTERCEPT_MSR_PROT      \
+                                                         | SVM_CTRL_INTERCEPT_INVLPGA       \
+                                                         | SVM_CTRL_INTERCEPT_SHUTDOWN      \
+                                                         | SVM_CTRL_INTERCEPT_FERR_FREEZE   \
+                                                         | SVM_CTRL_INTERCEPT_VMRUN         \
+                                                         | SVM_CTRL_INTERCEPT_SKINIT        \
+                                                         | SVM_CTRL_INTERCEPT_WBINVD        \
+                                                         | SVM_CTRL_INTERCEPT_MONITOR       \
+                                                         | SVM_CTRL_INTERCEPT_MWAIT         \
+                                                         | SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
+                                                         | SVM_CTRL_INTERCEPT_XSETBV)
+
+/** @name VMCB Clean Bits.
+ *
+ * These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
+ * AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
+ * memory.
+ *
+ * @{ */
+/** All intercepts vectors, TSC offset, PAUSE filter counter. */
+#define HMSVM_VMCB_CLEAN_INTERCEPTS             RT_BIT(0)
+/** I/O permission bitmap, MSR permission bitmap. */
+#define HMSVM_VMCB_CLEAN_IOPM_MSRPM             RT_BIT(1)
+/** ASID.  */
+#define HMSVM_VMCB_CLEAN_ASID                   RT_BIT(2)
+/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
+V_INTR_VECTOR. */
+#define HMSVM_VMCB_CLEAN_INT_CTRL               RT_BIT(3)
+/** Nested Paging: Nested CR3 (nCR3), PAT. */
+#define HMSVM_VMCB_CLEAN_NP                     RT_BIT(4)
+/** Control registers (CR0, CR3, CR4, EFER). */
+#define HMSVM_VMCB_CLEAN_CRX_EFER               RT_BIT(5)
+/** Debug registers (DR6, DR7). */
+#define HMSVM_VMCB_CLEAN_DRX                    RT_BIT(6)
+/** GDT, IDT limit and base. */
+#define HMSVM_VMCB_CLEAN_DT                     RT_BIT(7)
+/** Segment register: CS, SS, DS, ES limit and base. */
+#define HMSVM_VMCB_CLEAN_SEG                    RT_BIT(8)
+/** CR2.*/
+#define HMSVM_VMCB_CLEAN_CR2                    RT_BIT(9)
+/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
+#define HMSVM_VMCB_CLEAN_LBR                    RT_BIT(10)
+/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
+PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
+#define HMSVM_VMCB_CLEAN_AVIC                   RT_BIT(11)
+/** Mask of all valid VMCB Clean bits. */
+#define HMSVM_VMCB_CLEAN_ALL                    (  HMSVM_VMCB_CLEAN_INTERCEPTS  \
+                                                 | HMSVM_VMCB_CLEAN_IOPM_MSRPM  \
+                                                 | HMSVM_VMCB_CLEAN_ASID        \
+                                                 | HMSVM_VMCB_CLEAN_INT_CTRL    \
+                                                 | HMSVM_VMCB_CLEAN_NP          \
+                                                 | HMSVM_VMCB_CLEAN_CRX_EFER    \
+                                                 | HMSVM_VMCB_CLEAN_DRX         \
+                                                 | HMSVM_VMCB_CLEAN_DT          \
+                                                 | HMSVM_VMCB_CLEAN_SEG         \
+                                                 | HMSVM_VMCB_CLEAN_CR2         \
+                                                 | HMSVM_VMCB_CLEAN_LBR         \
+                                                 | HMSVM_VMCB_CLEAN_AVIC)
+/** @} */
+
+/** @name SVM transient.
+ *
+ * A state structure for holding miscellaneous information across AMD-V
+ * VMRUN/\#VMEXIT operation, restored after the transition.
+ *
+ * @{ */
+typedef struct SVMTRANSIENT
+{
+    /** The host's rflags/eflags. */
+    RTCCUINTREG     fEFlags;
+    /** The \#VMEXIT exit code (the EXITCODE field in the VMCB). */
+    uint64_t        u64ExitCode;
+
+    /** The guest's TPR value used for TPR shadowing. */
+    uint8_t         u8GuestTpr;
+    /** Alignment. */
+    uint8_t         abAlignment0[7];
+
+    /** Pointer to the currently executing VMCB. */
+    PSVMVMCB        pVmcb;
+
+    /** Whether we are currently executing a nested-guest. */
+    bool            fIsNestedGuest;
+    /** Whether the guest debug state was active at the time of \#VMEXIT. */
+    bool            fWasGuestDebugStateActive;
+    /** Whether the hyper debug state was active at the time of \#VMEXIT. */
+    bool            fWasHyperDebugStateActive;
+    /** Whether the TSC offset mode needs to be updated. */
+    bool            fUpdateTscOffsetting;
+    /** Whether the TSC_AUX MSR needs restoring on \#VMEXIT. */
+    bool            fRestoreTscAuxMsr;
+    /** Whether the \#VMEXIT was caused by a page-fault during delivery of a
+     *  contributary exception or a page-fault. */
+    bool            fVectoringDoublePF;
+    /** Whether the \#VMEXIT was caused by a page-fault during delivery of an
+     *  external interrupt or NMI. */
+    bool            fVectoringPF;
+    /** Padding. */
+    bool            afPadding0;
+} SVMTRANSIENT;
+/** Pointer to SVM transient state. */
+typedef SVMTRANSIENT *PSVMTRANSIENT;
+/** Pointer to a const SVM transient state. */
+typedef const SVMTRANSIENT *PCSVMTRANSIENT;
+
+AssertCompileSizeAlignment(SVMTRANSIENT, sizeof(uint64_t));
+AssertCompileMemberAlignment(SVMTRANSIENT, u64ExitCode, sizeof(uint64_t));
+AssertCompileMemberAlignment(SVMTRANSIENT, pVmcb,       sizeof(uint64_t));
+/** @}  */
+
+/**
+ * MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
+ */
+typedef enum SVMMSREXITREAD
+{
+    /** Reading this MSR causes a \#VMEXIT. */
+    SVMMSREXIT_INTERCEPT_READ = 0xb,
+    /** Reading this MSR does not cause a \#VMEXIT. */
+    SVMMSREXIT_PASSTHRU_READ
+} SVMMSREXITREAD;
+
+/**
+ * MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
+ */
+typedef enum SVMMSREXITWRITE
+{
+    /** Writing to this MSR causes a \#VMEXIT. */
+    SVMMSREXIT_INTERCEPT_WRITE = 0xd,
+    /** Writing to this MSR does not cause a \#VMEXIT. */
+    SVMMSREXIT_PASSTHRU_WRITE
+} SVMMSREXITWRITE;
+
+/**
+ * SVM \#VMEXIT handler.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pSvmTransient   Pointer to the SVM-transient structure.
+ */
+typedef int FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
+static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
+
+
+/** @name \#VMEXIT handlers.
+ * @{
+ */
+static FNSVMEXITHANDLER hmR0SvmExitIntr;
+static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
+static FNSVMEXITHANDLER hmR0SvmExitInvd;
+static FNSVMEXITHANDLER hmR0SvmExitCpuid;
+static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
+static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
+static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
+static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
+static FNSVMEXITHANDLER hmR0SvmExitHlt;
+static FNSVMEXITHANDLER hmR0SvmExitMonitor;
+static FNSVMEXITHANDLER hmR0SvmExitMwait;
+static FNSVMEXITHANDLER hmR0SvmExitShutdown;
+static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
+static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
+static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
+static FNSVMEXITHANDLER hmR0SvmExitMsr;
+static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
+static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
+static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
+static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
+static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
+static FNSVMEXITHANDLER hmR0SvmExitVIntr;
+static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
+static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
+static FNSVMEXITHANDLER hmR0SvmExitPause;
+static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
+static FNSVMEXITHANDLER hmR0SvmExitIret;
+static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
+static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
+static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
+static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
+static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
+static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
+static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
+#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) || defined(VBOX_WITH_NESTED_HWVIRT_SVM)
+static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+static FNSVMEXITHANDLER hmR0SvmExitClgi;
+static FNSVMEXITHANDLER hmR0SvmExitStgi;
+static FNSVMEXITHANDLER hmR0SvmExitVmload;
+static FNSVMEXITHANDLER hmR0SvmExitVmsave;
+static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
+static FNSVMEXITHANDLER hmR0SvmExitVmrun;
+static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
+static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
+#endif
+/** @} */
+
+static int hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+static int hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** Ring-0 memory object for the IO bitmap. */
+static RTR0MEMOBJ           g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
+/** Physical address of the IO bitmap. */
+static RTHCPHYS             g_HCPhysIOBitmap;
+/** Pointer to the IO bitmap. */
+static R0PTRTYPE(void *)    g_pvIOBitmap;
+
+#ifdef VBOX_STRICT
+# define HMSVM_LOG_RBP_RSP      RT_BIT_32(0)
+# define HMSVM_LOG_CR_REGS      RT_BIT_32(1)
+# define HMSVM_LOG_CS           RT_BIT_32(2)
+# define HMSVM_LOG_SS           RT_BIT_32(3)
+# define HMSVM_LOG_FS           RT_BIT_32(4)
+# define HMSVM_LOG_GS           RT_BIT_32(5)
+# define HMSVM_LOG_LBR          RT_BIT_32(6)
+# define HMSVM_LOG_ALL          (  HMSVM_LOG_RBP_RSP \
+                                 | HMSVM_LOG_CR_REGS \
+                                 | HMSVM_LOG_CS \
+                                 | HMSVM_LOG_SS \
+                                 | HMSVM_LOG_FS \
+                                 | HMSVM_LOG_GS \
+                                 | HMSVM_LOG_LBR)
+
+/**
+ * Dumps virtual CPU state and additional info. to the logger for diagnostics.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ * @param   pszPrefix   Log prefix.
+ * @param   fFlags      Log flags, see HMSVM_LOG_XXX.
+ * @param   uVerbose    The verbosity level, currently unused.
+ */
+static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
+{
+    RT_NOREF2(pVCpu, uVerbose);
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+
+    HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
+    Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
+
+    if (fFlags & HMSVM_LOG_RBP_RSP)
+    {
+        HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RBP);
+        Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
+    }
+
+    if (fFlags & HMSVM_LOG_CR_REGS)
+    {
+        HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4);
+        Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
+    }
+
+    if (fFlags & HMSVM_LOG_CS)
+    {
+        HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
+        Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
+              pCtx->cs.u32Limit, pCtx->cs.Attr.u));
+    }
+    if (fFlags & HMSVM_LOG_SS)
+    {
+        HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
+        Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
+              pCtx->ss.u32Limit, pCtx->ss.Attr.u));
+    }
+    if (fFlags & HMSVM_LOG_FS)
+    {
+        HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
+        Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
+              pCtx->fs.u32Limit, pCtx->fs.Attr.u));
+    }
+    if (fFlags & HMSVM_LOG_GS)
+    {
+        HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
+        Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
+              pCtx->gs.u32Limit, pCtx->gs.Attr.u));
+    }
+
+    PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
+    if (fFlags & HMSVM_LOG_LBR)
+    {
+        Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
+              pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
+    }
+    NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
+}
+#endif  /* VBOX_STRICT */
+
+
+/**
+ * Sets up and activates AMD-V on the current CPU.
+ *
+ * @returns VBox status code.
+ * @param   pHostCpu        The HM physical-CPU structure.
+ * @param   pVM             The cross context VM structure. Can be
+ *                          NULL after a resume!
+ * @param   pvCpuPage       Pointer to the global CPU page.
+ * @param   HCPhysCpuPage   Physical address of the global CPU page.
+ * @param   fEnabledByHost  Whether the host OS has already initialized AMD-V.
+ * @param   pHwvirtMsrs     Pointer to the hardware-virtualization MSRs (currently
+ *                          unused).
+ */
+VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
+                              PCSUPHWVIRTMSRS pHwvirtMsrs)
+{
+    Assert(!fEnabledByHost);
+    Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
+    Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
+    Assert(pvCpuPage); NOREF(pvCpuPage);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
+
+    /* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+
+    /*
+     * We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
+     */
+    uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
+    if (u64HostEfer & MSR_K6_EFER_SVME)
+    {
+        /* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
+        if (   pVM
+            && pVM->hm.s.svm.fIgnoreInUseError)
+            pHostCpu->fIgnoreAMDVInUseError = true;
+
+        if (!pHostCpu->fIgnoreAMDVInUseError)
+        {
+            ASMSetFlags(fEFlags);
+            return VERR_SVM_IN_USE;
+        }
+    }
+
+    /* Turn on AMD-V in the EFER MSR. */
+    ASMWrMsr(MSR_K6_EFER, u64HostEfer | MSR_K6_EFER_SVME);
+
+    /* Write the physical page address where the CPU will store the host state while executing the VM. */
+    ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
+
+    /* Restore interrupts. */
+    ASMSetFlags(fEFlags);
+
+    /*
+     * Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
+     * non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
+     * ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
+     * entirely with before executing any guest code.
+     */
+    pHostCpu->fFlushAsidBeforeUse = true;
+
+    /*
+     * Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
+     */
+    ++pHostCpu->cTlbFlushes;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deactivates AMD-V on the current CPU.
+ *
+ * @returns VBox status code.
+ * @param   pHostCpu        The HM physical-CPU structure.
+ * @param   pvCpuPage       Pointer to the global CPU page.
+ * @param   HCPhysCpuPage   Physical address of the global CPU page.
+ */
+VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
+{
+    RT_NOREF1(pHostCpu);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    AssertReturn(   HCPhysCpuPage
+                 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
+    AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
+
+    /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+
+    /* Turn off AMD-V in the EFER MSR. */
+    uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
+    ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
+
+    /* Invalidate host state physical address. */
+    ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
+
+    /* Restore interrupts. */
+    ASMSetFlags(fEFlags);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Does global AMD-V initialization (called during module initialization).
+ *
+ * @returns VBox status code.
+ */
+VMMR0DECL(int) SVMR0GlobalInit(void)
+{
+    /*
+     * Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
+     * intercept all IO accesses, it's done once globally here instead of per-VM.
+     */
+    Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
+    int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    g_pvIOBitmap     = RTR0MemObjAddress(g_hMemObjIOBitmap);
+    g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
+
+    /* Set all bits to intercept all IO accesses. */
+    ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Does global AMD-V termination (called during module termination).
+ */
+VMMR0DECL(void) SVMR0GlobalTerm(void)
+{
+    if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
+    {
+        RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
+        g_pvIOBitmap      = NULL;
+        g_HCPhysIOBitmap  = 0;
+        g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
+    }
+}
+
+
+/**
+ * Frees any allocated per-VCPU structures for a VM.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
+{
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+        AssertPtr(pVCpu);
+
+        if (pVCpu->hm.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
+        {
+            RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcbHost, false);
+            pVCpu->hm.s.svm.HCPhysVmcbHost   = 0;
+            pVCpu->hm.s.svm.hMemObjVmcbHost  = NIL_RTR0MEMOBJ;
+        }
+
+        if (pVCpu->hm.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
+        {
+            RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcb, false);
+            pVCpu->hm.s.svm.pVmcb            = NULL;
+            pVCpu->hm.s.svm.HCPhysVmcb       = 0;
+            pVCpu->hm.s.svm.hMemObjVmcb      = NIL_RTR0MEMOBJ;
+        }
+
+        if (pVCpu->hm.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
+        {
+            RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjMsrBitmap, false);
+            pVCpu->hm.s.svm.pvMsrBitmap      = NULL;
+            pVCpu->hm.s.svm.HCPhysMsrBitmap  = 0;
+            pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
+        }
+    }
+}
+
+
+/**
+ * Does per-VM AMD-V initialization.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
+{
+    int rc = VERR_INTERNAL_ERROR_5;
+
+    /*
+     * Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
+     */
+    uint32_t u32Family;
+    uint32_t u32Model;
+    uint32_t u32Stepping;
+    if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
+    {
+        Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
+        pVM->hm.s.svm.fAlwaysFlushTLB = true;
+    }
+
+    /*
+     * Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+        pVCpu->hm.s.svm.hMemObjVmcbHost  = NIL_RTR0MEMOBJ;
+        pVCpu->hm.s.svm.hMemObjVmcb      = NIL_RTR0MEMOBJ;
+        pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
+    }
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+
+        /*
+         * Initialize the hardware-assisted SVM guest-execution handler.
+         * We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
+         */
+        pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun;
+
+        /*
+         * Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
+         * FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
+         */
+        rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
+        if (RT_FAILURE(rc))
+            goto failure_cleanup;
+
+        void *pvVmcbHost               = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcbHost);
+        pVCpu->hm.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcbHost, 0 /* iPage */);
+        Assert(pVCpu->hm.s.svm.HCPhysVmcbHost < _4G);
+        ASMMemZeroPage(pvVmcbHost);
+
+        /*
+         * Allocate one page for the guest-state VMCB.
+         */
+        rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
+        if (RT_FAILURE(rc))
+            goto failure_cleanup;
+
+        pVCpu->hm.s.svm.pVmcb           = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcb);
+        pVCpu->hm.s.svm.HCPhysVmcb      = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcb, 0 /* iPage */);
+        Assert(pVCpu->hm.s.svm.HCPhysVmcb < _4G);
+        ASMMemZeroPage(pVCpu->hm.s.svm.pVmcb);
+
+        /*
+         * Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
+         * SVM to not require one.
+         */
+        rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
+                                 false /* fExecutable */);
+        if (RT_FAILURE(rc))
+            goto failure_cleanup;
+
+        pVCpu->hm.s.svm.pvMsrBitmap     = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjMsrBitmap);
+        pVCpu->hm.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
+        /* Set all bits to intercept all MSR accesses (changed later on). */
+        ASMMemFill32(pVCpu->hm.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
+   }
+
+    return VINF_SUCCESS;
+
+failure_cleanup:
+    hmR0SvmFreeStructs(pVM);
+    return rc;
+}
+
+
+/**
+ * Does per-VM AMD-V termination.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
+{
+    hmR0SvmFreeStructs(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Returns whether the VMCB Clean Bits feature is supported.
+ *
+ * @returns @c true if supported, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   fIsNestedGuest  Whether we are currently executing the nested-guest.
+ */
+DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
+{
+    PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    bool const fHostVmcbCleanBits = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
+    if (!fIsNestedGuest)
+        return fHostVmcbCleanBits;
+    return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
+}
+
+
+/**
+ * Returns whether the decode assists feature is supported.
+ *
+ * @returns @c true if supported, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
+    {
+        return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
+            &&  pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
+    }
+#endif
+    return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
+}
+
+
+/**
+ * Returns whether the NRIP_SAVE feature is supported.
+ *
+ * @returns @c true if supported, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
+    {
+        return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
+            &&  pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
+    }
+#endif
+    return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
+}
+
+
+/**
+ * Sets the permission bits for the specified MSR in the MSRPM bitmap.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pbMsrBitmap     Pointer to the MSR bitmap.
+ * @param   idMsr           The MSR for which the permissions are being set.
+ * @param   enmRead         MSR read permissions.
+ * @param   enmWrite        MSR write permissions.
+ *
+ * @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
+ *          caller needs to take care of this.
+ */
+static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
+                                    SVMMSREXITWRITE enmWrite)
+{
+    bool const  fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
+    uint16_t    offMsrpm;
+    uint8_t     uMsrpmBit;
+    int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
+    AssertRC(rc);
+
+    Assert(uMsrpmBit == 0 || uMsrpmBit == 2 || uMsrpmBit == 4 || uMsrpmBit == 6);
+    Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
+
+    pbMsrBitmap += offMsrpm;
+    if (enmRead == SVMMSREXIT_INTERCEPT_READ)
+        *pbMsrBitmap |= RT_BIT(uMsrpmBit);
+    else
+    {
+        if (!fInNestedGuestMode)
+            *pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        else
+        {
+            /* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
+            uint8_t const *pbNstGstMsrBitmap = (uint8_t *)pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
+            pbNstGstMsrBitmap += offMsrpm;
+            if (!(*pbNstGstMsrBitmap & RT_BIT(uMsrpmBit)))
+                *pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
+            else
+                Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
+        }
+#endif
+    }
+
+    if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
+        *pbMsrBitmap |= RT_BIT(uMsrpmBit + 1);
+    else
+    {
+        if (!fInNestedGuestMode)
+            *pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        else
+        {
+            /* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
+            uint8_t const *pbNstGstMsrBitmap = (uint8_t *)pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
+            pbNstGstMsrBitmap += offMsrpm;
+            if (!(*pbNstGstMsrBitmap & RT_BIT(uMsrpmBit + 1)))
+                *pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
+            else
+                Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
+        }
+#endif
+    }
+}
+
+
+/**
+ * Sets up AMD-V for the specified VM.
+ * This function is only called once per-VM during initalization.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+    Assert(pVM->hm.s.svm.fSupported);
+
+    bool const fPauseFilter          = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
+    bool const fPauseFilterThreshold = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
+    bool const fUsePauseFilter       = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
+
+    bool const fLbrVirt              = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
+    bool const fUseLbrVirt           = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    bool const fVirtVmsaveVmload     = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
+    bool const fUseVirtVmsaveVmload  = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && pVM->hm.s.fNestedPaging;
+
+    bool const fVGif                 = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VGIF);
+    bool const fUseVGif              = fVGif && pVM->hm.s.svm.fVGif;
+#endif
+
+    PVMCPUCC     pVCpu0 = VMCC_GET_CPU_0(pVM);
+    PSVMVMCB     pVmcb0 = pVCpu0->hm.s.svm.pVmcb;
+    AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
+    PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
+
+    /* Always trap #AC for reasons of security. */
+    pVmcbCtrl0->u32InterceptXcpt |= RT_BIT_32(X86_XCPT_AC);
+
+    /* Always trap #DB for reasons of security. */
+    pVmcbCtrl0->u32InterceptXcpt |= RT_BIT_32(X86_XCPT_DB);
+
+    /* Trap exceptions unconditionally (debug purposes). */
+#ifdef HMSVM_ALWAYS_TRAP_PF
+    pVmcbCtrl0->u32InterceptXcpt |= RT_BIT_32(X86_XCPT_PF);
+#endif
+#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
+    /* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
+    pVmcbCtrl0->u32InterceptXcpt |= RT_BIT_32(X86_XCPT_BP)
+                                 | RT_BIT_32(X86_XCPT_DE)
+                                 | RT_BIT_32(X86_XCPT_NM)
+                                 | RT_BIT_32(X86_XCPT_UD)
+                                 | RT_BIT_32(X86_XCPT_NP)
+                                 | RT_BIT_32(X86_XCPT_SS)
+                                 | RT_BIT_32(X86_XCPT_GP)
+                                 | RT_BIT_32(X86_XCPT_PF)
+                                 | RT_BIT_32(X86_XCPT_MF)
+                                 ;
+#endif
+
+    /* Apply the exceptions intercepts needed by the GIM provider. */
+    if (pVCpu0->hm.s.fGIMTrapXcptUD)
+        pVmcbCtrl0->u32InterceptXcpt |= RT_BIT(X86_XCPT_UD);
+
+    /* The mesa 3d driver hack needs #GP. */
+    if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
+        pVmcbCtrl0->u32InterceptXcpt |= RT_BIT(X86_XCPT_GP);
+
+    /* Set up unconditional intercepts and conditions. */
+    pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
+                                 | SVM_CTRL_INTERCEPT_VMMCALL;
+
+#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
+    pVmcbCtrl0->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_TASK_SWITCH;
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    /* Virtualized VMSAVE/VMLOAD. */
+    pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = fUseVirtVmsaveVmload;
+    if (!fUseVirtVmsaveVmload)
+        pVmcbCtrl0->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_VMSAVE
+                                     |  SVM_CTRL_INTERCEPT_VMLOAD;
+
+    /* Virtual GIF. */
+    pVmcbCtrl0->IntCtrl.n.u1VGifEnable = fUseVGif;
+    if (!fUseVGif)
+        pVmcbCtrl0->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_CLGI
+                                     |  SVM_CTRL_INTERCEPT_STGI;
+#endif
+
+    /* CR4 writes must always be intercepted for tracking PGM mode changes. */
+    pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
+
+    /* Intercept all DRx reads and writes by default. Changed later on. */
+    pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
+    pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
+
+    /* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
+    pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
+
+    /* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
+       and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
+    pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
+
+    /* Set the IO permission bitmap physical addresses. */
+    pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
+
+    /* LBR virtualization. */
+    pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
+
+    /* The host ASID MBZ, for the guest start with 1. */
+    pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
+
+    /* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
+    pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = pVM->hm.s.fNestedPaging;
+
+    /* Without Nested Paging, we need additionally intercepts. */
+    if (!pVM->hm.s.fNestedPaging)
+    {
+        /* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
+        pVmcbCtrl0->u16InterceptRdCRx |= RT_BIT(3);
+        pVmcbCtrl0->u16InterceptWrCRx |= RT_BIT(3);
+
+        /* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
+        pVmcbCtrl0->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_INVLPG
+                                     |  SVM_CTRL_INTERCEPT_TASK_SWITCH;
+
+        /* Page faults must be intercepted to implement shadow paging. */
+        pVmcbCtrl0->u32InterceptXcpt |= RT_BIT(X86_XCPT_PF);
+    }
+
+    /* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
+    if (fUsePauseFilter)
+    {
+        Assert(pVM->hm.s.svm.cPauseFilter > 0);
+        pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
+        if (fPauseFilterThreshold)
+            pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
+        pVmcbCtrl0->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_PAUSE;
+    }
+
+    /*
+     * Setup the MSR permission bitmap.
+     * The following MSRs are saved/restored automatically during the world-switch.
+     * Don't intercept guest read/write accesses to these MSRs.
+     */
+    uint8_t *pbMsrBitmap0 = (uint8_t *)pVCpu0->hm.s.svm.pvMsrBitmap;
+    hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR,          SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
+    hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR,          SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
+    hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR,           SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
+    hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK,        SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
+    hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE,        SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
+    hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE,        SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
+    hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
+    hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS,  SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
+    hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
+    hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
+    pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hm.s.svm.HCPhysMsrBitmap;
+
+    /* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
+    Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
+
+    for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPUCC     pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
+        PSVMVMCB     pVmcbCur = pVCpuCur->hm.s.svm.pVmcb;
+        AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
+        PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
+
+        /* Copy the VMCB control area. */
+        memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
+
+        /* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
+        uint8_t *pbMsrBitmapCur = (uint8_t *)pVCpuCur->hm.s.svm.pvMsrBitmap;
+        memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
+        pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hm.s.svm.HCPhysMsrBitmap;
+
+        /* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
+        Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
+
+        /* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
+        Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
+    }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
+            fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
+#else
+    LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets a pointer to the currently active guest (or nested-guest) VMCB.
+ *
+ * @returns Pointer to the current context VMCB.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
+{
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
+        return pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
+#endif
+    return pVCpu->hm.s.svm.pVmcb;
+}
+
+
+/**
+ * Gets a pointer to the nested-guest VMCB cache.
+ *
+ * @returns Pointer to the nested-guest VMCB cache.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
+{
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
+    return &pVCpu->hm.s.svm.NstGstVmcbCache;
+#else
+    RT_NOREF(pVCpu);
+    return NULL;
+#endif
+}
+
+
+/**
+ * Invalidates a guest page by guest virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCVirt      Guest virtual address of the page to invalidate.
+ */
+VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
+{
+    Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
+
+    bool const fFlushPending = pVCpu->CTX_SUFF(pVM)->hm.s.svm.fAlwaysFlushTLB || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+
+    /* Skip it if a TLB flush is already pending. */
+    if (!fFlushPending)
+    {
+        Log4Func(("%#RGv\n", GCVirt));
+
+        PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+        AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
+
+        SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Flushes the appropriate tagged-TLB entries.
+ *
+ * @param   pHostCpu    The HM physical-CPU structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ */
+static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    /*
+     * Force a TLB flush for the first world switch if the current CPU differs from the one
+     * we ran on last. This can happen both for start & resume due to long jumps back to
+     * ring-3.
+     *
+     * We also force a TLB flush every time when executing a nested-guest VCPU as there is no
+     * correlation between it and the physical CPU.
+     *
+     * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
+     * flushing the TLB, so we cannot reuse the ASIDs without flushing.
+     */
+    bool fNewAsid = false;
+    Assert(pHostCpu->idCpu != NIL_RTCPUID);
+    if (   pVCpu->hm.s.idLastCpu   != pHostCpu->idCpu
+        || pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        || CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
+#endif
+        )
+    {
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
+        pVCpu->hm.s.fForceTLBFlush = true;
+        fNewAsid = true;
+    }
+
+    /* Set TLB flush state as checked until we return from the world switch. */
+    ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
+
+    /* Check for explicit TLB flushes. */
+    if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
+    {
+        pVCpu->hm.s.fForceTLBFlush = true;
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
+    }
+
+    /*
+     * If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
+     * This Host CPU requirement takes precedence.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (pVM->hm.s.svm.fAlwaysFlushTLB)
+    {
+        pHostCpu->uCurrentAsid           = 1;
+        pVCpu->hm.s.uCurrentAsid         = 1;
+        pVCpu->hm.s.cTlbFlushes          = pHostCpu->cTlbFlushes;
+        pVCpu->hm.s.idLastCpu            = pHostCpu->idCpu;
+        pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
+
+        /* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
+        pVmcb->ctrl.u32VmcbCleanBits    &= ~HMSVM_VMCB_CLEAN_NP;
+    }
+    else
+    {
+        pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
+        if (pVCpu->hm.s.fForceTLBFlush)
+        {
+            /* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
+            pVmcb->ctrl.u32VmcbCleanBits    &= ~HMSVM_VMCB_CLEAN_NP;
+
+            if (fNewAsid)
+            {
+                ++pHostCpu->uCurrentAsid;
+
+                bool fHitASIDLimit = false;
+                if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
+                {
+                    pHostCpu->uCurrentAsid = 1;      /* Wraparound at 1; host uses 0 */
+                    pHostCpu->cTlbFlushes++;         /* All VCPUs that run on this host CPU must use a new ASID. */
+                    fHitASIDLimit      = true;
+                }
+
+                if (   fHitASIDLimit
+                    || pHostCpu->fFlushAsidBeforeUse)
+                {
+                    pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
+                    pHostCpu->fFlushAsidBeforeUse = false;
+                }
+
+                pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
+                pVCpu->hm.s.idLastCpu    = pHostCpu->idCpu;
+                pVCpu->hm.s.cTlbFlushes  = pHostCpu->cTlbFlushes;
+            }
+            else
+            {
+                if (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
+                    pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
+                else
+                    pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
+            }
+
+            pVCpu->hm.s.fForceTLBFlush = false;
+        }
+    }
+
+    /* Update VMCB with the ASID. */
+    if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hm.s.uCurrentAsid)
+    {
+        pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hm.s.uCurrentAsid;
+        pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
+    }
+
+    AssertMsg(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu,
+              ("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hm.s.idLastCpu, pHostCpu->idCpu));
+    AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
+              ("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
+    AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
+              ("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
+    AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
+              ("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
+
+#ifdef VBOX_WITH_STATISTICS
+    if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
+    else if (   pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
+             || pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
+    {
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
+    }
+    else
+    {
+        Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
+    }
+#endif
+}
+
+
+/**
+ * Sets an exception intercept in the specified VMCB.
+ *
+ * @param   pVmcb       Pointer to the VM control block.
+ * @param   uXcpt       The exception (X86_XCPT_*).
+ */
+DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
+{
+    if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
+    {
+        pVmcb->ctrl.u32InterceptXcpt |= RT_BIT(uXcpt);
+        pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
+    }
+}
+
+
+/**
+ * Clears an exception intercept in the specified VMCB.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ * @param   uXcpt       The exception (X86_XCPT_*).
+ *
+ * @remarks This takes into account if we're executing a nested-guest and only
+ *          removes the exception intercept if both the guest -and- nested-guest
+ *          are not intercepting it.
+ */
+DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
+{
+    Assert(uXcpt != X86_XCPT_DB);
+    Assert(uXcpt != X86_XCPT_AC);
+    Assert(uXcpt != X86_XCPT_GP);
+#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
+    if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
+    {
+        bool fRemove = true;
+# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        /* Only remove the intercept if the nested-guest is also not intercepting it! */
+        PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+        if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+        {
+            PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
+            fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
+        }
+# else
+        RT_NOREF(pVCpu);
+# endif
+        if (fRemove)
+        {
+            pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
+            pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
+        }
+    }
+#else
+    RT_NOREF3(pVCpu, pVmcb, uXcpt);
+#endif
+}
+
+
+/**
+ * Sets a control intercept in the specified VMCB.
+ *
+ * @param   pVmcb           Pointer to the VM control block.
+ * @param   fCtrlIntercept  The control intercept (SVM_CTRL_INTERCEPT_*).
+ */
+DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
+{
+    if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
+    {
+        pVmcb->ctrl.u64InterceptCtrl |= fCtrlIntercept;
+        pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
+    }
+}
+
+
+/**
+ * Clears a control intercept in the specified VMCB.
+ *
+ * @returns @c true if the intercept is still set, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcb           Pointer to the VM control block.
+ * @param   fCtrlIntercept  The control intercept (SVM_CTRL_INTERCEPT_*).
+ *
+ * @remarks This takes into account if we're executing a nested-guest and only
+ *          removes the control intercept if both the guest -and- nested-guest
+ *          are not intercepting it.
+ */
+static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
+{
+    if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
+    {
+        bool fRemove = true;
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        /* Only remove the control intercept if the nested-guest is also not intercepting it! */
+        if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
+        {
+            PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
+            fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
+        }
+#else
+        RT_NOREF(pVCpu);
+#endif
+        if (fRemove)
+        {
+            pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
+            pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
+        }
+    }
+
+    return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
+}
+
+
+/**
+ * Exports the guest (or nested-guest) CR0 into the VMCB.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ *
+ * @remarks This assumes we always pre-load the guest FPU.
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    PCPUMCTX       pCtx = &pVCpu->cpum.GstCtx;
+    uint64_t const uGuestCr0  = pCtx->cr0;
+    uint64_t       uShadowCr0 = uGuestCr0;
+
+    /* Always enable caching. */
+    uShadowCr0 &= ~(X86_CR0_CD | X86_CR0_NW);
+
+    /* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
+    if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
+    {
+        uShadowCr0 |= X86_CR0_PG      /* Use shadow page tables. */
+                   |  X86_CR0_WP;     /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
+    }
+
+    /*
+     * Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
+     * lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
+     * see @bugref{7243#c103}.
+     */
+    if (!(uGuestCr0 & X86_CR0_NE))
+    {
+        uShadowCr0 |= X86_CR0_NE;
+        hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
+    }
+    else
+        hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
+
+    /*
+     * If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
+     *
+     * CR0 writes still needs interception as PGM requires tracking paging mode changes,
+     * see @bugref{6944}.
+     *
+     * We also don't ever want to honor weird things like cache disable from the guest.
+     * However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
+     * write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
+     */
+    if (uShadowCr0 == uGuestCr0)
+    {
+        if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+        {
+            pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
+            pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
+            Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
+        }
+        else
+        {
+            /* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
+            PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
+            pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx       & ~RT_BIT(0))
+                                          | (pVmcbNstGstCache->u16InterceptRdCRx &  RT_BIT(0));
+            pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx       & ~RT_BIT(0))
+                                          | (pVmcbNstGstCache->u16InterceptWrCRx &  RT_BIT(0));
+        }
+    }
+    else
+    {
+        pVmcb->ctrl.u16InterceptRdCRx |= RT_BIT(0);
+        pVmcb->ctrl.u16InterceptWrCRx |= RT_BIT(0);
+    }
+    pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
+
+    Assert(!RT_HI_U32(uShadowCr0));
+    if (pVmcb->guest.u64CR0 != uShadowCr0)
+    {
+        pVmcb->guest.u64CR0 = uShadowCr0;
+        pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
+    }
+}
+
+
+/**
+ * Exports the guest (or nested-guest) CR3 into the VMCB.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    PVMCC      pVM  = pVCpu->CTX_SUFF(pVM);
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    if (pVM->hm.s.fNestedPaging)
+    {
+        pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
+        pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
+        pVmcb->guest.u64CR3 = pCtx->cr3;
+        Assert(pVmcb->ctrl.u64NestedPagingCR3);
+    }
+    else
+        pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
+
+    pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
+}
+
+
+/**
+ * Exports the guest (or nested-guest) CR4 into the VMCB.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    uint64_t uShadowCr4 = pCtx->cr4;
+    if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
+    {
+        switch (pVCpu->hm.s.enmShadowMode)
+        {
+            case PGMMODE_REAL:
+            case PGMMODE_PROTECTED:     /* Protected mode, no paging. */
+                return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
+
+            case PGMMODE_32_BIT:        /* 32-bit paging. */
+                uShadowCr4 &= ~X86_CR4_PAE;
+                break;
+
+            case PGMMODE_PAE:           /* PAE paging. */
+            case PGMMODE_PAE_NX:        /* PAE paging with NX enabled. */
+                /** Must use PAE paging as we could use physical memory > 4 GB */
+                uShadowCr4 |= X86_CR4_PAE;
+                break;
+
+            case PGMMODE_AMD64:         /* 64-bit AMD paging (long mode). */
+            case PGMMODE_AMD64_NX:      /* 64-bit AMD paging (long mode) with NX enabled. */
+#ifdef VBOX_WITH_64_BITS_GUESTS
+                break;
+#else
+                return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
+#endif
+
+            default:                    /* shut up gcc */
+                return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
+        }
+    }
+
+    /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
+    pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
+
+    /* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
+    if (uShadowCr4 == pCtx->cr4)
+    {
+        if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+            pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
+        else
+        {
+            /* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
+            PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
+            pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx       & ~RT_BIT(4))
+                                          | (pVmcbNstGstCache->u16InterceptRdCRx &  RT_BIT(4));
+        }
+    }
+    else
+        pVmcb->ctrl.u16InterceptRdCRx |= RT_BIT(4);
+
+    /* CR4 writes are always intercepted (both guest, nested-guest) for tracking PGM mode changes. */
+    Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
+
+    /* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
+    Assert(!RT_HI_U32(uShadowCr4));
+    pVmcb->guest.u64CR4 = uShadowCr4;
+    pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER | HMSVM_VMCB_CLEAN_INTERCEPTS);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exports the guest (or nested-guest) control registers into the VMCB.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
+    {
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
+            hmR0SvmExportGuestCR0(pVCpu, pVmcb);
+
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
+        {
+            pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
+            pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
+        }
+
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
+            hmR0SvmExportGuestCR3(pVCpu, pVmcb);
+
+        /* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
+        {
+            int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+
+        pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exports the guest (or nested-guest) segment registers into the VMCB.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+
+    /* Guest segment registers. */
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
+    {
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
+            HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
+
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
+        {
+            HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
+            pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
+        }
+
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
+            HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
+
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
+            HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
+
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
+            HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
+
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
+            HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
+
+        pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
+    }
+
+    /* Guest TR. */
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
+        HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
+
+    /* Guest LDTR. */
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
+        HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
+
+    /* Guest GDTR. */
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
+    {
+        pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
+        pVmcb->guest.GDTR.u64Base  = pCtx->gdtr.pGdt;
+        pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
+    }
+
+    /* Guest IDTR. */
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
+    {
+        pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
+        pVmcb->guest.IDTR.u64Base  = pCtx->idtr.pIdt;
+        pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
+    }
+
+    pVCpu->hm.s.fCtxChanged &= ~(  HM_CHANGED_GUEST_SREG_MASK
+                                 | HM_CHANGED_GUEST_TABLE_MASK);
+}
+
+
+/**
+ * Exports the guest (or nested-guest) MSRs into the VMCB.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+
+    /* Guest Sysenter MSRs. */
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
+    {
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
+            pVmcb->guest.u64SysEnterCS  = pCtx->SysEnter.cs;
+
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
+            pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
+
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
+            pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
+    }
+
+    /*
+     * Guest EFER MSR.
+     * AMD-V requires guest EFER.SVME to be set. Weird.
+     * See AMD spec. 15.5.1 "Basic Operation" | "Canonicalization and Consistency Checks".
+     */
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
+    {
+        pVmcb->guest.u64EFER = pCtx->msrEFER | MSR_K6_EFER_SVME;
+        pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
+    }
+
+    /* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
+    if (   !CPUMIsGuestInLongModeEx(pCtx)
+        && (pCtx->msrEFER & MSR_K6_EFER_LME))
+    {
+        pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
+        pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
+    }
+
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
+    {
+        pVmcb->guest.u64STAR   = pCtx->msrSTAR;
+        pVmcb->guest.u64LSTAR  = pCtx->msrLSTAR;
+        pVmcb->guest.u64CSTAR  = pCtx->msrCSTAR;
+        pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
+    }
+
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
+        pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
+
+    pVCpu->hm.s.fCtxChanged &= ~(  HM_CHANGED_GUEST_SYSENTER_MSR_MASK
+                                 | HM_CHANGED_GUEST_EFER_MSR
+                                 | HM_CHANGED_GUEST_SYSCALL_MSRS
+                                 | HM_CHANGED_GUEST_KERNEL_GS_BASE);
+
+    /*
+     * Setup the PAT MSR (applicable for Nested Paging only).
+     *
+     * The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
+     * as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
+     *
+     * While guests can modify and see the modified values through the shadow values,
+     * we shall not honor any guest modifications of this MSR to ensure caching is always
+     * enabled similar to how we clear CR0.CD and NW bits.
+     *
+     * For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
+     */
+    pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
+
+    /* Enable the last branch record bit if LBR virtualization is enabled. */
+    if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
+        pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
+}
+
+
+/**
+ * Exports the guest (or nested-guest) debug state into the VMCB and programs
+ * the necessary intercepts accordingly.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ *
+ * @remarks No-long-jump zone!!!
+ * @remarks Requires EFLAGS to be up-to-date in the VMCB!
+ */
+static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+
+    /*
+     * Anyone single stepping on the host side? If so, we'll have to use the
+     * trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
+     * the VMM level like the VT-x implementations does.
+     */
+    bool       fInterceptMovDRx = false;
+    bool const fStepping = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
+    if (fStepping)
+    {
+        pVCpu->hm.s.fClearTrapFlag = true;
+        pVmcb->guest.u64RFlags |= X86_EFL_TF;
+        fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
+    }
+
+    if (   fStepping
+        || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
+    {
+        /*
+         * Use the combined guest and host DRx values found in the hypervisor
+         * register set because the debugger has breakpoints active or someone
+         * is single stepping on the host side.
+         *
+         * Note! DBGF expects a clean DR6 state before executing guest code.
+         */
+        if (!CPUMIsHyperDebugStateActive(pVCpu))
+        {
+            CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
+            Assert(!CPUMIsGuestDebugStateActive(pVCpu));
+            Assert(CPUMIsHyperDebugStateActive(pVCpu));
+        }
+
+        /* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
+        if (   pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
+            || pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
+        {
+            pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
+            pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
+            pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
+        }
+
+        /** @todo If we cared, we could optimize to allow the guest to read registers
+         *        with the same values. */
+        fInterceptMovDRx = true;
+        pVCpu->hm.s.fUsingHyperDR7 = true;
+        Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
+    }
+    else
+    {
+        /*
+         * Update DR6, DR7 with the guest values if necessary.
+         */
+        if (   pVmcb->guest.u64DR7 != pCtx->dr[7]
+            || pVmcb->guest.u64DR6 != pCtx->dr[6])
+        {
+            pVmcb->guest.u64DR7 = pCtx->dr[7];
+            pVmcb->guest.u64DR6 = pCtx->dr[6];
+            pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
+        }
+        pVCpu->hm.s.fUsingHyperDR7 = false;
+
+        /*
+         * If the guest has enabled debug registers, we need to load them prior to
+         * executing guest code so they'll trigger at the right time.
+         */
+        if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
+        {
+            if (!CPUMIsGuestDebugStateActive(pVCpu))
+            {
+                CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
+                Assert(!CPUMIsHyperDebugStateActive(pVCpu));
+                Assert(CPUMIsGuestDebugStateActive(pVCpu));
+            }
+            Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
+        }
+        /*
+         * If no debugging enabled, we'll lazy load DR0-3. We don't need to
+         * intercept #DB as DR6 is updated in the VMCB.
+         *
+         * Note! If we cared and dared, we could skip intercepting \#DB here.
+         *       However, \#DB shouldn't be performance critical, so we'll play safe
+         *       and keep the code similar to the VT-x code and always intercept it.
+         */
+        else if (!CPUMIsGuestDebugStateActive(pVCpu))
+            fInterceptMovDRx = true;
+    }
+
+    Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
+    if (fInterceptMovDRx)
+    {
+        if (   pVmcb->ctrl.u16InterceptRdDRx != 0xffff
+            || pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
+        {
+            pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
+            pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
+            pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
+        }
+    }
+    else
+    {
+        if (   pVmcb->ctrl.u16InterceptRdDRx
+            || pVmcb->ctrl.u16InterceptWrDRx)
+        {
+            pVmcb->ctrl.u16InterceptRdDRx = 0;
+            pVmcb->ctrl.u16InterceptWrDRx = 0;
+            pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
+        }
+    }
+    Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
+}
+
+/**
+ * Exports the hardware virtualization state into the nested-guest
+ * VMCB.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   pVmcb   Pointer to the VM control block.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
+    {
+        if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
+        {
+            PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+            PCVM      pVM  = pVCpu->CTX_SUFF(pVM);
+
+            HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);                                /* Nested VGIF is not supported yet. */
+            Assert(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VGIF);    /* Physical hardware supports VGIF. */
+            Assert(HMIsSvmVGifActive(pVM));                                        /* Outer VM has enabled VGIF. */
+            NOREF(pVM);
+
+            pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
+        }
+
+        /*
+         * Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
+         * since SVM doesn't have a preemption timer.
+         *
+         * We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
+         * nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
+         * and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
+         */
+        PVMCC          pVM = pVCpu->CTX_SUFF(pVM);
+        PSVMVMCBCTRL   pVmcbCtrl = &pVmcb->ctrl;
+        uint16_t const uGuestPauseFilterCount     = pVM->hm.s.svm.cPauseFilter;
+        uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
+        if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
+        {
+            PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+            pVmcbCtrl->u16PauseFilterCount     = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
+            pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
+        }
+        else
+        {
+            /** @todo r=ramshankar: We can turn these assignments into assertions. */
+            pVmcbCtrl->u16PauseFilterCount     = uGuestPauseFilterCount;
+            pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
+        }
+        pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
+
+        pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
+    }
+}
+
+
+/**
+ * Exports the guest APIC TPR state into the VMCB.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ */
+static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
+
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
+    {
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        if (   PDMHasApic(pVM)
+            && APICIsEnabled(pVCpu))
+        {
+            bool    fPendingIntr;
+            uint8_t u8Tpr;
+            int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
+            AssertRCReturn(rc, rc);
+
+            /* Assume that we need to trap all TPR accesses and thus need not check on
+               every #VMEXIT if we should update the TPR. */
+            Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
+            pVCpu->hm.s.svm.fSyncVTpr = false;
+
+            if (!pVM->hm.s.fTPRPatchingActive)
+            {
+                /* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
+                pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
+
+                /* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
+                   can deliver the interrupt to the guest. */
+                if (fPendingIntr)
+                    pVmcb->ctrl.u16InterceptWrCRx |= RT_BIT(8);
+                else
+                {
+                    pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
+                    pVCpu->hm.s.svm.fSyncVTpr = true;
+                }
+
+                pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS | HMSVM_VMCB_CLEAN_INT_CTRL);
+            }
+            else
+            {
+                /* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
+                pVmcb->guest.u64LSTAR = u8Tpr;
+                uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
+
+                /* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
+                if (fPendingIntr)
+                    hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
+                else
+                {
+                    hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
+                    pVCpu->hm.s.svm.fSyncVTpr = true;
+                }
+                pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
+            }
+        }
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets up the exception interrupts required for guest execution in the VMCB.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
+
+    /* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
+    {
+        /* Trap #UD for GIM provider (e.g. for hypercalls). */
+        if (pVCpu->hm.s.fGIMTrapXcptUD)
+            hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
+        else
+            hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
+
+        /* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
+        if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
+            hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
+        else
+            hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
+
+        /* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
+    }
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+/**
+ * Merges guest and nested-guest intercepts for executing the nested-guest using
+ * hardware-assisted SVM.
+ *
+ * This merges the guest and nested-guest intercepts in a way that if the outer
+ * guest intercept is set we need to intercept it in the nested-guest as
+ * well.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcbNstGst     Pointer to the nested-guest VM control block.
+ */
+static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
+{
+    PVMCC          pVM             = pVCpu->CTX_SUFF(pVM);
+    PCSVMVMCB    pVmcb           = pVCpu->hm.s.svm.pVmcb;
+    PSVMVMCB     pVmcbNstGst     = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
+    PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
+
+    /* Merge the guest's CR intercepts into the nested-guest VMCB. */
+    pVmcbNstGstCtrl->u16InterceptRdCRx |= pVmcb->ctrl.u16InterceptRdCRx;
+    pVmcbNstGstCtrl->u16InterceptWrCRx |= pVmcb->ctrl.u16InterceptWrCRx;
+
+    /* Always intercept CR4 writes for tracking PGM mode changes. */
+    pVmcbNstGstCtrl->u16InterceptWrCRx |= RT_BIT(4);
+
+    /* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
+    if (!pVM->hm.s.fNestedPaging)
+    {
+        pVmcbNstGstCtrl->u16InterceptRdCRx |= RT_BIT(3);
+        pVmcbNstGstCtrl->u16InterceptWrCRx |= RT_BIT(3);
+    }
+
+    /** @todo Figure out debugging with nested-guests, till then just intercept
+     *        all DR[0-15] accesses. */
+    pVmcbNstGstCtrl->u16InterceptRdDRx |= 0xffff;
+    pVmcbNstGstCtrl->u16InterceptWrDRx |= 0xffff;
+
+    /*
+     * Merge the guest's exception intercepts into the nested-guest VMCB.
+     *
+     * - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
+     * while executing the nested-guest.
+     *
+     * - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
+     * be tweaked later depending on how we wish to implement breakpoints.
+     *
+     * - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
+     *        loaded into their guests, not ours.
+     *
+     * Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
+     * for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
+     */
+#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
+    pVmcbNstGstCtrl->u32InterceptXcpt  |= pVmcb->ctrl.u32InterceptXcpt
+                                       & ~(  RT_BIT(X86_XCPT_UD)
+                                           | RT_BIT(X86_XCPT_BP)
+                                           | (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
+#else
+    pVmcbNstGstCtrl->u32InterceptXcpt  |= pVmcb->ctrl.u32InterceptXcpt;
+#endif
+
+    /*
+     * Adjust intercepts while executing the nested-guest that differ from the
+     * outer guest intercepts.
+     *
+     * - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
+     *   that belong to the nested-guest to the outer guest.
+     *
+     * - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
+     *   the nested-guest, the physical CPU raises a \#UD exception as expected.
+     */
+    pVmcbNstGstCtrl->u64InterceptCtrl  |= (pVmcb->ctrl.u64InterceptCtrl & ~(  SVM_CTRL_INTERCEPT_VINTR
+                                                                            | SVM_CTRL_INTERCEPT_VMMCALL))
+                                       |  HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
+
+    Assert(   (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
+           == HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
+
+    /* Finally, update the VMCB clean bits. */
+    pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
+}
+#endif
+
+
+/**
+ * Enters the AMD-V session.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
+{
+    AssertPtr(pVCpu);
+    Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    LogFlowFunc(("pVCpu=%p\n", pVCpu));
+    Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
+                                   == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
+
+    pVCpu->hm.s.fLeaveDone = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Thread-context callback for AMD-V.
+ *
+ * @param   enmEvent        The thread-context event.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   fGlobalInit     Whether global VT-x/AMD-V init. is used.
+ * @thread  EMT(pVCpu)
+ */
+VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
+{
+    NOREF(fGlobalInit);
+
+    switch (enmEvent)
+    {
+        case RTTHREADCTXEVENT_OUT:
+        {
+            Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+            Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
+            VMCPU_ASSERT_EMT(pVCpu);
+
+            /* No longjmps (log-flush, locks) in this fragile context. */
+            VMMRZCallRing3Disable(pVCpu);
+
+            if (!pVCpu->hm.s.fLeaveDone)
+            {
+                hmR0SvmLeave(pVCpu, false /* fImportState */);
+                pVCpu->hm.s.fLeaveDone = true;
+            }
+
+            /* Leave HM context, takes care of local init (term). */
+            int rc = HMR0LeaveCpu(pVCpu);
+            AssertRC(rc); NOREF(rc);
+
+            /* Restore longjmp state. */
+            VMMRZCallRing3Enable(pVCpu);
+            STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
+            break;
+        }
+
+        case RTTHREADCTXEVENT_IN:
+        {
+            Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+            Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
+            VMCPU_ASSERT_EMT(pVCpu);
+
+            /* No longjmps (log-flush, locks) in this fragile context. */
+            VMMRZCallRing3Disable(pVCpu);
+
+            /*
+             * Initialize the bare minimum state required for HM. This takes care of
+             * initializing AMD-V if necessary (onlined CPUs, local init etc.)
+             */
+            int rc = hmR0EnterCpu(pVCpu);
+            AssertRC(rc); NOREF(rc);
+            Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
+                                           == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
+
+            pVCpu->hm.s.fLeaveDone = false;
+
+            /* Restore longjmp state. */
+            VMMRZCallRing3Enable(pVCpu);
+            break;
+        }
+
+        default:
+            break;
+    }
+}
+
+
+/**
+ * Saves the host state.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
+{
+    NOREF(pVCpu);
+
+    /* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
+    ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exports the guest or nested-guest state from the virtual-CPU context into the
+ * VMCB.
+ *
+ * Also sets up the appropriate VMRUN function to execute guest or nested-guest
+ * code based on the virtual-CPU mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pSvmTransient   Pointer to the SVM-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
+{
+    STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
+
+    PSVMVMCB  pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    PCCPUMCTX pCtx  = &pVCpu->cpum.GstCtx;
+    Assert(pVmcb);
+
+    pVmcb->guest.u64RIP    = pCtx->rip;
+    pVmcb->guest.u64RSP    = pCtx->rsp;
+    pVmcb->guest.u64RFlags = pCtx->eflags.u32;
+    pVmcb->guest.u64RAX    = pCtx->rax;
+
+    bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+
+    int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
+    AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
+    hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
+    hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
+    hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
+
+    ASMSetFlags(fEFlags);
+
+    if (!fIsNestedGuest)
+    {
+        /* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
+           otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
+        hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
+        hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
+    }
+
+    /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
+    uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
+                         | HM_CHANGED_GUEST_RFLAGS
+                         | HM_CHANGED_GUEST_GPRS_MASK
+                         | HM_CHANGED_GUEST_X87
+                         | HM_CHANGED_GUEST_SSE_AVX
+                         | HM_CHANGED_GUEST_OTHER_XSAVE
+                         | HM_CHANGED_GUEST_XCRx
+                         | HM_CHANGED_GUEST_TSC_AUX
+                         | HM_CHANGED_GUEST_OTHER_MSRS;
+    if (fIsNestedGuest)
+        fUnusedMask |= HM_CHANGED_SVM_XCPT_INTERCEPTS
+                    |  HM_CHANGED_GUEST_APIC_TPR;
+
+    ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~(  fUnusedMask
+                                                  | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
+
+#ifdef VBOX_STRICT
+    /*
+     * All of the guest-CPU state and SVM keeper bits should be exported here by now,
+     * except for the host-context and/or shared host-guest context bits.
+     */
+    uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
+    AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
+              ("fCtxChanged=%#RX64\n", fCtxChanged));
+
+    /*
+     * If we need to log state that isn't always imported, we'll need to import them here.
+     * See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
+     */
+    hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags */, 0 /* uVerbose */);
+#endif
+
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
+    return VINF_SUCCESS;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+/**
+ * Merges the guest and nested-guest MSR permission bitmap.
+ *
+ * If the guest is intercepting an MSR we need to intercept it regardless of
+ * whether the nested-guest is intercepting it or not.
+ *
+ * @param   pHostCpu    The HM physical-CPU structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jmp zone!!!
+ */
+DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
+{
+    uint64_t const *pu64GstMsrpm    = (uint64_t const *)pVCpu->hm.s.svm.pvMsrBitmap;
+    uint64_t const *pu64NstGstMsrpm = (uint64_t const *)pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
+    uint64_t       *pu64DstMsrpm    = (uint64_t *)pHostCpu->n.svm.pvNstGstMsrpm;
+
+    /* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
+    uint32_t const offRsvdQwords = 0x1800 >> 3;
+    for (uint32_t i = 0; i < offRsvdQwords; i++)
+        pu64DstMsrpm[i] = pu64NstGstMsrpm[i] | pu64GstMsrpm[i];
+}
+
+
+/**
+ * Caches the nested-guest VMCB fields before we modify them for execution using
+ * hardware-assisted SVM.
+ *
+ * @returns true if the VMCB was previously already cached, false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ *
+ * @sa      HMNotifySvmNstGstVmexit.
+ */
+static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
+{
+    /*
+     * Cache the nested-guest programmed VMCB fields if we have not cached it yet.
+     * Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
+     *
+     * Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
+     * cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
+     */
+    PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    bool const fWasCached = pVmcbNstGstCache->fCacheValid;
+    if (!fWasCached)
+    {
+        PCSVMVMCB      pVmcbNstGst    = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
+        PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
+        pVmcbNstGstCache->u16InterceptRdCRx       = pVmcbNstGstCtrl->u16InterceptRdCRx;
+        pVmcbNstGstCache->u16InterceptWrCRx       = pVmcbNstGstCtrl->u16InterceptWrCRx;
+        pVmcbNstGstCache->u16InterceptRdDRx       = pVmcbNstGstCtrl->u16InterceptRdDRx;
+        pVmcbNstGstCache->u16InterceptWrDRx       = pVmcbNstGstCtrl->u16InterceptWrDRx;
+        pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
+        pVmcbNstGstCache->u16PauseFilterCount     = pVmcbNstGstCtrl->u16PauseFilterCount;
+        pVmcbNstGstCache->u32InterceptXcpt        = pVmcbNstGstCtrl->u32InterceptXcpt;
+        pVmcbNstGstCache->u64InterceptCtrl        = pVmcbNstGstCtrl->u64InterceptCtrl;
+        pVmcbNstGstCache->u64TSCOffset            = pVmcbNstGstCtrl->u64TSCOffset;
+        pVmcbNstGstCache->fVIntrMasking           = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
+        pVmcbNstGstCache->fNestedPaging           = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
+        pVmcbNstGstCache->fLbrVirt                = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
+        pVmcbNstGstCache->fCacheValid             = true;
+        Log4Func(("Cached VMCB fields\n"));
+    }
+
+    return fWasCached;
+}
+
+
+/**
+ * Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
+ *
+ * This is done the first time we enter nested-guest execution using SVM R0
+ * until the nested-guest \#VMEXIT (not to be confused with physical CPU
+ * \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
+{
+    PSVMVMCB     pVmcbNstGst     = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
+    PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
+
+    HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
+
+    /*
+     * First cache the nested-guest VMCB fields we may potentially modify.
+     */
+    bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
+    if (!fVmcbCached)
+    {
+        /*
+         * The IOPM of the nested-guest can be ignored because the the guest always
+         * intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
+         * than the nested-guest IOPM and swap the field back on the #VMEXIT.
+         */
+        pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
+
+        /*
+         * Use the same nested-paging as the outer guest. We can't dynamically switch off
+         * nested-paging suddenly while executing a VM (see assertion at the end of
+         * Trap0eHandler() in PGMAllBth.h).
+         */
+        pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
+
+        /* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
+        pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
+
+        /*
+         * Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
+         * to the nested-guest intercepts and we always run with V_INTR_MASKING.
+         */
+        pVCpu->hm.s.svm.fSyncVTpr = false;
+
+#ifdef DEBUG_ramshankar
+        /* For debugging purposes - copy the LBR info. from outer guest VMCB. */
+        pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
+#endif
+
+        /*
+         * If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
+         * need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
+         */
+        if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
+            pVmcbNstGstCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_VMSAVE
+                                              |  SVM_CTRL_INTERCEPT_VMLOAD;
+
+        /*
+         * If we don't expose Virtual GIF feature to the outer guest, we need to intercept
+         * CLGI/STGI instructions executed by the nested-guest.
+         */
+        if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
+            pVmcbNstGstCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_CLGI
+                                              |  SVM_CTRL_INTERCEPT_STGI;
+
+        /* Merge the guest and nested-guest intercepts. */
+        hmR0SvmMergeVmcbCtrlsNested(pVCpu);
+
+        /* Update the VMCB clean bits. */
+        pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
+    }
+    else
+    {
+        Assert(!pVCpu->hm.s.svm.fSyncVTpr);
+        Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
+        Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
+    }
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
+
+
+/**
+ * Exports the state shared between the host and guest (or nested-guest) into
+ * the VMCB.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
+    {
+        /** @todo Figure out stepping with nested-guest. */
+        PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+        if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+            hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
+        else
+        {
+            pVmcb->guest.u64DR6 = pCtx->dr[6];
+            pVmcb->guest.u64DR7 = pCtx->dr[7];
+        }
+    }
+
+    pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
+    AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
+              ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
+}
+
+
+/**
+ * Worker for SVMR0ImportStateOnDemand.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   fWhat   What to import, CPUMCTX_EXTRN_XXX.
+ */
+static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
+{
+    STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
+
+    PCPUMCTX           pCtx       = &pVCpu->cpum.GstCtx;
+    PCSVMVMCB          pVmcb      = hmR0SvmGetCurrentVmcb(pVCpu);
+    PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
+    PCSVMVMCBCTRL      pVmcbCtrl  = &pVmcb->ctrl;
+
+    /*
+     * We disable interrupts to make the updating of the state and in particular
+     * the fExtrn modification atomic wrt to preemption hooks.
+     */
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+
+    fWhat &= pCtx->fExtrn;
+    if (fWhat)
+    {
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        if (fWhat & CPUMCTX_EXTRN_HWVIRT)
+        {
+            if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
+            {
+                Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));    /* We don't yet support passing VGIF feature to the guest. */
+                Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM)));    /* VM has configured it. */
+                CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
+            }
+        }
+
+        if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
+        {
+            if (  !pVmcbCtrl->IntCtrl.n.u1VIrqPending
+                && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
+        }
+#endif
+
+        if (fWhat & CPUMCTX_EXTRN_HM_SVM_INT_SHADOW)
+        {
+            if (pVmcbCtrl->IntShadow.n.u1IntShadow)
+                EMSetInhibitInterruptsPC(pVCpu, pVmcbGuest->u64RIP);
+            else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+        }
+
+        if (fWhat & CPUMCTX_EXTRN_RIP)
+            pCtx->rip = pVmcbGuest->u64RIP;
+
+        if (fWhat & CPUMCTX_EXTRN_RFLAGS)
+            pCtx->eflags.u32 = pVmcbGuest->u64RFlags;
+
+        if (fWhat & CPUMCTX_EXTRN_RSP)
+            pCtx->rsp = pVmcbGuest->u64RSP;
+
+        if (fWhat & CPUMCTX_EXTRN_RAX)
+            pCtx->rax = pVmcbGuest->u64RAX;
+
+        if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
+        {
+            if (fWhat & CPUMCTX_EXTRN_CS)
+            {
+                HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
+                /* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
+                /** @todo SELM might need to be fixed as it too should not care about the
+                 *        granularity bit. See @bugref{6785}. */
+                if (   !pCtx->cs.Attr.n.u1Granularity
+                    &&  pCtx->cs.Attr.n.u1Present
+                    &&  pCtx->cs.u32Limit > UINT32_C(0xfffff))
+                {
+                    Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
+                    pCtx->cs.Attr.n.u1Granularity = 1;
+                }
+                HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
+            }
+            if (fWhat & CPUMCTX_EXTRN_SS)
+            {
+                HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
+                HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
+                /*
+                 * Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
+                 * VMCB and uses that and thus it's possible that when the CPL changes during
+                 * guest execution that the SS DPL isn't updated by AMD-V. Observed on some
+                 * AMD Fusion CPUs with 64-bit guests.
+                 *
+                 * See AMD spec. 15.5.1 "Basic operation".
+                 */
+                Assert(!(pVmcbGuest->u8CPL & ~0x3));
+                uint8_t const uCpl = pVmcbGuest->u8CPL;
+                if (pCtx->ss.Attr.n.u2Dpl != uCpl)
+                    pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
+            }
+            if (fWhat & CPUMCTX_EXTRN_DS)
+            {
+                HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
+                HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
+            }
+            if (fWhat & CPUMCTX_EXTRN_ES)
+            {
+                HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
+                HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
+            }
+            if (fWhat & CPUMCTX_EXTRN_FS)
+            {
+                HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
+                HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
+            }
+            if (fWhat & CPUMCTX_EXTRN_GS)
+            {
+                HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
+                HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
+            }
+        }
+
+        if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
+        {
+            if (fWhat & CPUMCTX_EXTRN_TR)
+            {
+                /*
+                 * Fixup TR attributes so it's compatible with Intel. Important when saved-states
+                 * are used between Intel and AMD, see @bugref{6208#c39}.
+                 * ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
+                 */
+                HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
+                if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
+                {
+                    if (   pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
+                        || CPUMIsGuestInLongModeEx(pCtx))
+                        pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
+                    else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
+                        pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
+                }
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_LDTR)
+                HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
+
+            if (fWhat & CPUMCTX_EXTRN_GDTR)
+            {
+                pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
+                pCtx->gdtr.pGdt  = pVmcbGuest->GDTR.u64Base;
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_IDTR)
+            {
+                pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
+                pCtx->idtr.pIdt  = pVmcbGuest->IDTR.u64Base;
+            }
+        }
+
+        if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+        {
+            pCtx->msrSTAR   = pVmcbGuest->u64STAR;
+            pCtx->msrLSTAR  = pVmcbGuest->u64LSTAR;
+            pCtx->msrCSTAR  = pVmcbGuest->u64CSTAR;
+            pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
+        }
+
+        if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+        {
+            pCtx->SysEnter.cs  = pVmcbGuest->u64SysEnterCS;
+            pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
+            pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
+        }
+
+        if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+            pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
+
+        if (fWhat & CPUMCTX_EXTRN_DR_MASK)
+        {
+            if (fWhat & CPUMCTX_EXTRN_DR6)
+            {
+                if (!pVCpu->hm.s.fUsingHyperDR7)
+                    pCtx->dr[6] = pVmcbGuest->u64DR6;
+                else
+                    CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_DR7)
+            {
+                if (!pVCpu->hm.s.fUsingHyperDR7)
+                    pCtx->dr[7] = pVmcbGuest->u64DR7;
+                else
+                    Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
+            }
+        }
+
+        if (fWhat & CPUMCTX_EXTRN_CR_MASK)
+        {
+            if (fWhat & CPUMCTX_EXTRN_CR0)
+            {
+                /* We intercept changes to all CR0 bits except maybe TS & MP bits. */
+                uint64_t const uCr0 = (pCtx->cr0          & ~(X86_CR0_TS | X86_CR0_MP))
+                                    | (pVmcbGuest->u64CR0 &  (X86_CR0_TS | X86_CR0_MP));
+                VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
+                CPUMSetGuestCR0(pVCpu, uCr0);
+                VMMRZCallRing3Enable(pVCpu);
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_CR2)
+                pCtx->cr2 = pVmcbGuest->u64CR2;
+
+            if (fWhat & CPUMCTX_EXTRN_CR3)
+            {
+                if (   pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
+                    && pCtx->cr3 != pVmcbGuest->u64CR3)
+                {
+                    CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
+                    VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
+                }
+            }
+
+            /* Changes to CR4 are always intercepted. */
+        }
+
+        /* Update fExtrn. */
+        pCtx->fExtrn &= ~fWhat;
+
+        /* If everything has been imported, clear the HM keeper bit. */
+        if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
+        {
+            pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
+            Assert(!pCtx->fExtrn);
+        }
+    }
+    else
+        Assert(!pCtx->fExtrn || (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
+
+    ASMSetFlags(fEFlags);
+
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
+
+    /*
+     * Honor any pending CR3 updates.
+     *
+     * Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
+     * -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
+     * -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
+     * handling -> hmR0SvmImportGuestState() and here we are.
+     *
+     * The reason for such complicated handling is because VM-exits that call into PGM expect
+     * CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
+     * postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
+     * VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
+     *
+     * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
+     * and does not process force-flag like regular exits to ring-3 either, we cover for it here.
+     */
+    if (   VMMRZCallRing3IsEnabled(pVCpu)
+        && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
+    {
+        Assert(pCtx->cr3 == pVmcbGuest->u64CR3);
+        PGMUpdateCR3(pVCpu, pCtx->cr3);
+    }
+}
+
+
+/**
+ * Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
+ * context.
+ *
+ * Currently there is no residual state left in the CPU that is not updated in the
+ * VMCB.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   fWhat   What to import, CPUMCTX_EXTRN_XXX.
+ */
+VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
+{
+    hmR0SvmImportGuestState(pVCpu, fWhat);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Does the necessary state syncing before returning to ring-3 for any reason
+ * (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   fImportState    Whether to import the guest state from the VMCB back
+ *                          to the guest-CPU context.
+ *
+ * @remarks No-long-jmp zone!!!
+ */
+static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+    Assert(VMMR0IsLogFlushDisabled(pVCpu));
+
+    /*
+     * !!! IMPORTANT !!!
+     * If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
+     */
+
+    /* Save the guest state if necessary. */
+    if (fImportState)
+        hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+
+    /* Restore host FPU state if necessary and resync on next R0 reentry. */
+    CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
+    Assert(!CPUMIsGuestFPUStateActive(pVCpu));
+
+    /*
+     * Restore host debug registers if necessary and resync on next R0 reentry.
+     */
+#ifdef VBOX_STRICT
+    if (CPUMIsHyperDebugStateActive(pVCpu))
+    {
+        PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb; /** @todo nested-guest. */
+        Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
+        Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
+    }
+#endif
+    CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
+    Assert(!CPUMIsHyperDebugStateActive(pVCpu));
+    Assert(!CPUMIsGuestDebugStateActive(pVCpu));
+
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
+
+    VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
+}
+
+
+/**
+ * Leaves the AMD-V session.
+ *
+ * Only used while returning to ring-3 either due to longjump or exits to
+ * ring-3.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
+{
+    HM_DISABLE_PREEMPT(pVCpu);
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
+       and done this from the SVMR0ThreadCtxCallback(). */
+    if (!pVCpu->hm.s.fLeaveDone)
+    {
+        hmR0SvmLeave(pVCpu, true /* fImportState */);
+        pVCpu->hm.s.fLeaveDone = true;
+    }
+
+    /*
+     * !!! IMPORTANT !!!
+     * If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
+     */
+
+    /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here!  */
+    /* Deregister hook now that we've left HM context before re-enabling preemption. */
+    VMMR0ThreadCtxHookDisable(pVCpu);
+
+    /* Leave HM context. This takes care of local init (term). */
+    int rc = HMR0LeaveCpu(pVCpu);
+
+    HM_RESTORE_PREEMPT();
+    return rc;
+}
+
+
+/**
+ * Does the necessary state syncing before doing a longjmp to ring-3.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jmp zone!!!
+ */
+static int hmR0SvmLongJmpToRing3(PVMCPUCC pVCpu)
+{
+    return hmR0SvmLeaveSession(pVCpu);
+}
+
+
+/**
+ * VMMRZCallRing3() callback wrapper which saves the guest state (or restores
+ * any remaining host state) before we longjump to ring-3 and possibly get
+ * preempted.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmOperation    The operation causing the ring-3 longjump.
+ */
+VMMR0DECL(int) SVMR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation)
+{
+    if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
+    {
+        /*
+         * !!! IMPORTANT !!!
+         * If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
+         * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
+         */
+        VMMRZCallRing3RemoveNotification(pVCpu);
+        VMMRZCallRing3Disable(pVCpu);
+        HM_DISABLE_PREEMPT(pVCpu);
+
+        /* Import the entire guest state. */
+        hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+
+        /* Restore host FPU state if necessary and resync on next R0 reentry. */
+        CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
+
+        /* Restore host debug registers if necessary and resync on next R0 reentry. */
+        CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
+
+        /* Deregister the hook now that we've left HM context before re-enabling preemption. */
+        /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here!  */
+        VMMR0ThreadCtxHookDisable(pVCpu);
+
+        /* Leave HM context. This takes care of local init (term). */
+        HMR0LeaveCpu(pVCpu);
+
+        HM_RESTORE_PREEMPT();
+        return VINF_SUCCESS;
+    }
+
+    Assert(pVCpu);
+    Assert(VMMRZCallRing3IsEnabled(pVCpu));
+    HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
+
+    VMMRZCallRing3Disable(pVCpu);
+    Assert(VMMR0IsLogFlushDisabled(pVCpu));
+
+    Log4Func(("Calling hmR0SvmLongJmpToRing3\n"));
+    int rc = hmR0SvmLongJmpToRing3(pVCpu);
+    AssertRCReturn(rc, rc);
+
+    VMMRZCallRing3Enable(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Take necessary actions before going back to ring-3.
+ *
+ * An action requires us to go back to ring-3. This function does the necessary
+ * steps before we can safely return to ring-3. This is not the same as longjmps
+ * to ring-3, this is voluntary.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   rcExit      The reason for exiting to ring-3. Can be
+ *                      VINF_VMM_UNKNOWN_RING3_CALL.
+ */
+static int hmR0SvmExitToRing3(PVMCPUCC pVCpu, int rcExit)
+{
+    Assert(pVCpu);
+    HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
+
+    /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
+    VMMRZCallRing3Disable(pVCpu);
+    Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", rcExit, (uint64_t)pVCpu->fLocalForcedActions,
+              pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
+
+    /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
+    if (pVCpu->hm.s.Event.fPending)
+    {
+        hmR0SvmPendingEventToTrpmTrap(pVCpu);
+        Assert(!pVCpu->hm.s.Event.fPending);
+    }
+
+    /* Sync. the necessary state for going back to ring-3. */
+    hmR0SvmLeaveSession(pVCpu);
+    STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
+
+    /* Thread-context hooks are unregistered at this point!!! */
+    /* Ring-3 callback notifications are unregistered at this point!!! */
+
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
+                             | CPUM_CHANGED_LDTR
+                             | CPUM_CHANGED_GDTR
+                             | CPUM_CHANGED_IDTR
+                             | CPUM_CHANGED_TR
+                             | CPUM_CHANGED_HIDDEN_SEL_REGS);
+    if (   pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
+        && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
+    {
+        CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
+    }
+
+    /* Update the exit-to-ring 3 reason. */
+    pVCpu->hm.s.rcLastExitToR3 = rcExit;
+
+    /* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
+    if (   rcExit != VINF_EM_RAW_INTERRUPT
+        || CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
+    {
+        Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+    }
+
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
+    VMMRZCallRing3Enable(pVCpu);
+
+    /*
+     * If we're emulating an instruction, we shouldn't have any TRPM traps pending
+     * and if we're injecting an event we should have a TRPM trap pending.
+     */
+    AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu),
+                     pVCpu->hm.s.u32HMError = rcExit,
+                     VERR_SVM_IPE_5);
+    AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu),
+                     pVCpu->hm.s.u32HMError = rcExit,
+                     VERR_SVM_IPE_4);
+
+    return rcExit;
+}
+
+
+/**
+ * Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
+ * intercepts.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    /*
+     * Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
+     * and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
+     * RDTSC/RDTSCP as well.
+     */
+    bool       fParavirtTsc;
+    uint64_t   uTscOffset;
+    bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
+
+    bool fIntercept;
+    if (fCanUseRealTsc)
+         fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC | SVM_CTRL_INTERCEPT_RDTSCP);
+    else
+    {
+        hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC | SVM_CTRL_INTERCEPT_RDTSCP);
+        fIntercept = true;
+    }
+
+    if (!fIntercept)
+    {
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        /* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
+        if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
+            uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
+#endif
+
+        /* Update the TSC offset in the VMCB and the relevant clean bits. */
+        pVmcb->ctrl.u64TSCOffset = uTscOffset;
+        pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
+    }
+
+    /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
+       information before every VM-entry, hence we have nothing to do here at the moment. */
+    if (fParavirtTsc)
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
+}
+
+
+/**
+ * Sets an event as a pending event to be injected into the guest.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   pEvent              Pointer to the SVM event.
+ * @param   GCPtrFaultAddress   The fault-address (CR2) in case it's a
+ *                              page-fault.
+ *
+ * @remarks Statistics counter assumes this is a guest event being reflected to
+ *          the guest i.e. 'StatInjectPendingReflect' is incremented always.
+ */
+DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
+{
+    Assert(!pVCpu->hm.s.Event.fPending);
+    Assert(pEvent->n.u1Valid);
+
+    pVCpu->hm.s.Event.u64IntInfo        = pEvent->u;
+    pVCpu->hm.s.Event.fPending          = true;
+    pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
+
+    Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
+              (uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
+}
+
+
+/**
+ * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
+{
+    SVMEVENT Event;
+    Event.u          = 0;
+    Event.n.u1Valid  = 1;
+    Event.n.u3Type   = SVM_EVENT_EXCEPTION;
+    Event.n.u8Vector = X86_XCPT_UD;
+    hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets a debug (\#DB) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
+{
+    SVMEVENT Event;
+    Event.u          = 0;
+    Event.n.u1Valid  = 1;
+    Event.n.u3Type   = SVM_EVENT_EXCEPTION;
+    Event.n.u8Vector = X86_XCPT_DB;
+    hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets a page fault (\#PF) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u32ErrCode      The error-code for the page-fault.
+ * @param   uFaultAddress   The page fault address (CR2).
+ *
+ * @remarks This updates the guest CR2 with @a uFaultAddress!
+ */
+DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
+{
+    SVMEVENT Event;
+    Event.u                  = 0;
+    Event.n.u1Valid          = 1;
+    Event.n.u3Type           = SVM_EVENT_EXCEPTION;
+    Event.n.u8Vector         = X86_XCPT_PF;
+    Event.n.u1ErrorCodeValid = 1;
+    Event.n.u32ErrorCode     = u32ErrCode;
+
+    /* Update CR2 of the guest. */
+    HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
+    if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
+    {
+        pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
+        /* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
+    }
+
+    hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
+}
+
+
+/**
+ * Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
+{
+    SVMEVENT Event;
+    Event.u          = 0;
+    Event.n.u1Valid  = 1;
+    Event.n.u3Type   = SVM_EVENT_EXCEPTION;
+    Event.n.u8Vector = X86_XCPT_MF;
+    hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets a double fault (\#DF) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
+{
+    SVMEVENT Event;
+    Event.u                  = 0;
+    Event.n.u1Valid          = 1;
+    Event.n.u3Type           = SVM_EVENT_EXCEPTION;
+    Event.n.u8Vector         = X86_XCPT_DF;
+    Event.n.u1ErrorCodeValid = 1;
+    Event.n.u32ErrorCode     = 0;
+    hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Injects an event into the guest upon VMRUN by updating the relevant field
+ * in the VMCB.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the guest VM control block.
+ * @param   pEvent      Pointer to the event.
+ *
+ * @remarks No-long-jump zone!!!
+ * @remarks Requires CR0!
+ */
+DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
+{
+    Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
+    pVmcb->ctrl.EventInject.u = pEvent->u;
+    if (   pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
+        || pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
+    {
+        Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
+        STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedXcptsR0[pEvent->n.u8Vector]);
+    }
+    else
+        STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
+    RT_NOREF(pVCpu);
+
+    Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
+              (uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
+}
+
+
+
+/**
+ * Converts any TRPM trap into a pending HM event. This is typically used when
+ * entering from ring-3 (not longjmp returns).
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
+{
+    Assert(TRPMHasTrap(pVCpu));
+    Assert(!pVCpu->hm.s.Event.fPending);
+
+    uint8_t     uVector;
+    TRPMEVENT   enmTrpmEvent;
+    uint32_t    uErrCode;
+    RTGCUINTPTR GCPtrFaultAddress;
+    uint8_t     cbInstr;
+
+    int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
+    AssertRC(rc);
+
+    SVMEVENT Event;
+    Event.u          = 0;
+    Event.n.u1Valid  = 1;
+    Event.n.u8Vector = uVector;
+
+    /* Refer AMD spec. 15.20 "Event Injection" for the format. */
+    if (enmTrpmEvent == TRPM_TRAP)
+    {
+        Event.n.u3Type = SVM_EVENT_EXCEPTION;
+        switch (uVector)
+        {
+            case X86_XCPT_NMI:
+            {
+                Event.n.u3Type = SVM_EVENT_NMI;
+                break;
+            }
+
+            case X86_XCPT_BP:
+            case X86_XCPT_OF:
+                AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
+                RT_FALL_THRU();
+
+            case X86_XCPT_PF:
+            case X86_XCPT_DF:
+            case X86_XCPT_TS:
+            case X86_XCPT_NP:
+            case X86_XCPT_SS:
+            case X86_XCPT_GP:
+            case X86_XCPT_AC:
+            {
+                Event.n.u1ErrorCodeValid = 1;
+                Event.n.u32ErrorCode     = uErrCode;
+                break;
+            }
+        }
+    }
+    else if (enmTrpmEvent == TRPM_HARDWARE_INT)
+        Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
+    else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
+        Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
+    else
+        AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
+
+    rc = TRPMResetTrap(pVCpu);
+    AssertRC(rc);
+
+    Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
+          !!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
+
+    hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
+}
+
+
+/**
+ * Converts any pending SVM event into a TRPM trap. Typically used when leaving
+ * AMD-V to execute any instruction.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu->hm.s.Event.fPending);
+    Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
+
+    SVMEVENT Event;
+    Event.u = pVCpu->hm.s.Event.u64IntInfo;
+
+    uint8_t   uVector     = Event.n.u8Vector;
+    TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
+
+    Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
+
+    int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
+    AssertRC(rc);
+
+    if (Event.n.u1ErrorCodeValid)
+        TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
+
+    if (   enmTrapType == TRPM_TRAP
+        && uVector     == X86_XCPT_PF)
+    {
+        TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
+        Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
+    }
+    else if (enmTrapType == TRPM_SOFTWARE_INT)
+        TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
+    pVCpu->hm.s.Event.fPending = false;
+}
+
+
+/**
+ * Checks if the guest (or nested-guest) has an interrupt shadow active right
+ * now.
+ *
+ * @returns @c true if the interrupt shadow is active, @c false otherwise.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
+ */
+static bool hmR0SvmIsIntrShadowActive(PVMCPUCC pVCpu)
+{
+    /*
+     * Instructions like STI and MOV SS inhibit interrupts till the next instruction
+     * completes. Check if we should inhibit interrupts or clear any existing
+     * interrupt inhibition.
+     */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+    {
+        if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
+        {
+            /*
+             * We can clear the inhibit force flag as even if we go back to the recompiler
+             * without executing guest code in AMD-V, the flag's condition to be cleared is
+             * met and thus the cleared state is correct.
+             */
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+            return false;
+        }
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Sets the virtual interrupt intercept control in the VMCB.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   pVmcb   Pointer to the VM control block.
+ */
+static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
+
+    /*
+     * When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
+     * is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
+     * (updating ISR at the right time) and inject the interrupt.
+     *
+     * With AVIC is supported, we could make use of the asynchronously delivery without
+     * #VMEXIT and we would be passing the AVIC page to SVM.
+     *
+     * In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
+     * is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
+     */
+    Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
+    pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
+    pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
+    hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
+    Log4(("Set VINTR intercept\n"));
+}
+
+
+/**
+ * Clears the virtual interrupt intercept control in the VMCB as
+ * we are figured the guest is unable process any interrupts
+ * at this point of time.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   pVmcb   Pointer to the VM control block.
+ */
+static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
+
+    PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
+    if (    pVmcbCtrl->IntCtrl.n.u1VIrqPending
+        || (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
+    {
+        pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
+        pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
+        hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
+        Log4(("Cleared VINTR intercept\n"));
+    }
+}
+
+
+/**
+ * Evaluates the event to be delivered to the guest and sets it as the pending
+ * event.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pSvmTransient   Pointer to the SVM transient structure.
+ */
+static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
+{
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
+                              | CPUMCTX_EXTRN_RFLAGS
+                              | CPUMCTX_EXTRN_HM_SVM_INT_SHADOW
+                              | CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
+
+    Assert(!pVCpu->hm.s.Event.fPending);
+    PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    Assert(pVmcb);
+
+    bool const fGif        = CPUMGetGuestGif(pCtx);
+    bool const fIntShadow  = hmR0SvmIsIntrShadowActive(pVCpu);
+    bool const fBlockNmi   = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
+
+    Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
+              fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC),
+              VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
+
+    /** @todo SMI. SMIs take priority over NMIs. */
+
+    /*
+     * Check if the guest or nested-guest can receive NMIs.
+     * Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
+     * NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
+     */
+    if (    VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
+        && !fBlockNmi)
+    {
+        if (    fGif
+            && !fIntShadow)
+        {
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
+            {
+                Log4(("Intercepting NMI -> #VMEXIT\n"));
+                HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+                return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
+            }
+#endif
+            Log4(("Setting NMI pending for injection\n"));
+            SVMEVENT Event;
+            Event.u = 0;
+            Event.n.u1Valid  = 1;
+            Event.n.u8Vector = X86_XCPT_NMI;
+            Event.n.u3Type   = SVM_EVENT_NMI;
+            hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+        }
+        else if (!fGif)
+            hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
+        else if (!pSvmTransient->fIsNestedGuest)
+            hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
+        /* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
+    }
+    /*
+     * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
+     * returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
+     * it from the APIC device.
+     *
+     * For nested-guests, physical interrupts always take priority over virtual interrupts.
+     * We don't need to inject nested-guest virtual interrupts here, we can let the hardware
+     * do that work when we execute nested-guest code esp. since all the required information
+     * is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
+     * the virtual interrupt controller.
+     *
+     * See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
+     */
+    else if (   VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
+             && !pVCpu->hm.s.fSingleInstruction)
+    {
+        bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u32 & X86_EFL_IF)
+                                                              : CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
+        if (    fGif
+            && !fBlockInt
+            && !fIntShadow)
+        {
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
+            {
+                Log4(("Intercepting INTR -> #VMEXIT\n"));
+                HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+                return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
+            }
+#endif
+            uint8_t u8Interrupt;
+            int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
+            if (RT_SUCCESS(rc))
+            {
+                Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
+                SVMEVENT Event;
+                Event.u = 0;
+                Event.n.u1Valid  = 1;
+                Event.n.u8Vector = u8Interrupt;
+                Event.n.u3Type   = SVM_EVENT_EXTERNAL_IRQ;
+                hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
+            }
+            else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
+            {
+                /*
+                 * AMD-V has no TPR thresholding feature. TPR and the force-flag will be
+                 * updated eventually when the TPR is written by the guest.
+                 */
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
+            }
+            else
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
+        }
+        else if (!fGif)
+            hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
+        else if (!pSvmTransient->fIsNestedGuest)
+            hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
+        /* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Injects any pending events into the guest (or nested-guest).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ *
+ * @remarks Must only be called when we are guaranteed to enter
+ *          hardware-assisted SVM execution and not return to ring-3
+ *          prematurely.
+ */
+static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    Assert(!TRPMHasTrap(pVCpu));
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+
+    bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
+#ifdef VBOX_STRICT
+    PCCPUMCTX  pCtx       = &pVCpu->cpum.GstCtx;
+    bool const fGif       = CPUMGetGuestGif(pCtx);
+    bool       fAllowInt  = fGif;
+    if (fGif)
+    {
+        /*
+         * For nested-guests we have no way to determine if we're injecting a physical or
+         * virtual interrupt at this point. Hence the partial verification below.
+         */
+        if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+            fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) || CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
+        else
+            fAllowInt = RT_BOOL(pCtx->eflags.u32 & X86_EFL_IF);
+    }
+#endif
+
+    if (pVCpu->hm.s.Event.fPending)
+    {
+        SVMEVENT Event;
+        Event.u = pVCpu->hm.s.Event.u64IntInfo;
+        Assert(Event.n.u1Valid);
+
+        /*
+         * Validate event injection pre-conditions.
+         */
+        if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
+        {
+            Assert(fAllowInt);
+            Assert(!fIntShadow);
+        }
+        else if (Event.n.u3Type == SVM_EVENT_NMI)
+        {
+            Assert(fGif);
+            Assert(!fIntShadow);
+        }
+
+        /*
+         * Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
+         * do this only when we are surely going to inject the NMI as otherwise if we return
+         * to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
+         * SVM R0.
+         *
+         * With VT-x, this is handled by the Guest interruptibility information VMCS field
+         * which will set the VMCS field after actually delivering the NMI which we read on
+         * VM-exit to determine the state.
+         */
+        if (    Event.n.u3Type   == SVM_EVENT_NMI
+            &&  Event.n.u8Vector == X86_XCPT_NMI
+            && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+        {
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
+        }
+
+        /*
+         * Inject it (update VMCB for injection by the hardware).
+         */
+        Log4(("Injecting pending HM event\n"));
+        hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
+        pVCpu->hm.s.Event.fPending = false;
+
+        if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
+        else
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
+    }
+    else
+        Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
+
+    /*
+     * We could have injected an NMI through IEM and continue guest execution using
+     * hardware-assisted SVM. In which case, we would not have any events pending (above)
+     * but we still need to intercept IRET in order to eventually clear NMI inhibition.
+     */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+        hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
+
+    /*
+     * Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
+     *
+     * For nested-guests: We need to update it too for the scenario where IEM executes
+     * the nested-guest but execution later continues here with an interrupt shadow active.
+     */
+    pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
+}
+
+
+/**
+ * Reports world-switch error and dumps some useful debug info.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   rcVMRun         The return code from VMRUN (or
+ *                          VERR_SVM_INVALID_GUEST_STATE for invalid
+ *                          guest-state).
+ */
+static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
+{
+    HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
+    HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+
+    if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
+    {
+#ifdef VBOX_STRICT
+        hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
+        PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+        Log4(("ctrl.u32VmcbCleanBits                 %#RX32\n",   pVmcb->ctrl.u32VmcbCleanBits));
+        Log4(("ctrl.u16InterceptRdCRx                %#x\n",      pVmcb->ctrl.u16InterceptRdCRx));
+        Log4(("ctrl.u16InterceptWrCRx                %#x\n",      pVmcb->ctrl.u16InterceptWrCRx));
+        Log4(("ctrl.u16InterceptRdDRx                %#x\n",      pVmcb->ctrl.u16InterceptRdDRx));
+        Log4(("ctrl.u16InterceptWrDRx                %#x\n",      pVmcb->ctrl.u16InterceptWrDRx));
+        Log4(("ctrl.u32InterceptXcpt                 %#x\n",      pVmcb->ctrl.u32InterceptXcpt));
+        Log4(("ctrl.u64InterceptCtrl                 %#RX64\n",   pVmcb->ctrl.u64InterceptCtrl));
+        Log4(("ctrl.u64IOPMPhysAddr                  %#RX64\n",   pVmcb->ctrl.u64IOPMPhysAddr));
+        Log4(("ctrl.u64MSRPMPhysAddr                 %#RX64\n",   pVmcb->ctrl.u64MSRPMPhysAddr));
+        Log4(("ctrl.u64TSCOffset                     %#RX64\n",   pVmcb->ctrl.u64TSCOffset));
+
+        Log4(("ctrl.TLBCtrl.u32ASID                  %#x\n",      pVmcb->ctrl.TLBCtrl.n.u32ASID));
+        Log4(("ctrl.TLBCtrl.u8TLBFlush               %#x\n",      pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
+        Log4(("ctrl.TLBCtrl.u24Reserved              %#x\n",      pVmcb->ctrl.TLBCtrl.n.u24Reserved));
+
+        Log4(("ctrl.IntCtrl.u8VTPR                   %#x\n",      pVmcb->ctrl.IntCtrl.n.u8VTPR));
+        Log4(("ctrl.IntCtrl.u1VIrqPending            %#x\n",      pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
+        Log4(("ctrl.IntCtrl.u1VGif                   %#x\n",      pVmcb->ctrl.IntCtrl.n.u1VGif));
+        Log4(("ctrl.IntCtrl.u6Reserved0              %#x\n",      pVmcb->ctrl.IntCtrl.n.u6Reserved));
+        Log4(("ctrl.IntCtrl.u4VIntrPrio              %#x\n",      pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
+        Log4(("ctrl.IntCtrl.u1IgnoreTPR              %#x\n",      pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
+        Log4(("ctrl.IntCtrl.u3Reserved               %#x\n",      pVmcb->ctrl.IntCtrl.n.u3Reserved));
+        Log4(("ctrl.IntCtrl.u1VIntrMasking           %#x\n",      pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
+        Log4(("ctrl.IntCtrl.u1VGifEnable             %#x\n",      pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
+        Log4(("ctrl.IntCtrl.u5Reserved1              %#x\n",      pVmcb->ctrl.IntCtrl.n.u5Reserved));
+        Log4(("ctrl.IntCtrl.u8VIntrVector            %#x\n",      pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
+        Log4(("ctrl.IntCtrl.u24Reserved              %#x\n",      pVmcb->ctrl.IntCtrl.n.u24Reserved));
+
+        Log4(("ctrl.IntShadow.u1IntShadow            %#x\n",      pVmcb->ctrl.IntShadow.n.u1IntShadow));
+        Log4(("ctrl.IntShadow.u1GuestIntMask         %#x\n",      pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
+        Log4(("ctrl.u64ExitCode                      %#RX64\n",   pVmcb->ctrl.u64ExitCode));
+        Log4(("ctrl.u64ExitInfo1                     %#RX64\n",   pVmcb->ctrl.u64ExitInfo1));
+        Log4(("ctrl.u64ExitInfo2                     %#RX64\n",   pVmcb->ctrl.u64ExitInfo2));
+        Log4(("ctrl.ExitIntInfo.u8Vector             %#x\n",      pVmcb->ctrl.ExitIntInfo.n.u8Vector));
+        Log4(("ctrl.ExitIntInfo.u3Type               %#x\n",      pVmcb->ctrl.ExitIntInfo.n.u3Type));
+        Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid     %#x\n",      pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
+        Log4(("ctrl.ExitIntInfo.u19Reserved          %#x\n",      pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
+        Log4(("ctrl.ExitIntInfo.u1Valid              %#x\n",      pVmcb->ctrl.ExitIntInfo.n.u1Valid));
+        Log4(("ctrl.ExitIntInfo.u32ErrorCode         %#x\n",      pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
+        Log4(("ctrl.NestedPagingCtrl.u1NestedPaging  %#x\n",      pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
+        Log4(("ctrl.NestedPagingCtrl.u1Sev           %#x\n",      pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
+        Log4(("ctrl.NestedPagingCtrl.u1SevEs         %#x\n",      pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
+        Log4(("ctrl.EventInject.u8Vector             %#x\n",      pVmcb->ctrl.EventInject.n.u8Vector));
+        Log4(("ctrl.EventInject.u3Type               %#x\n",      pVmcb->ctrl.EventInject.n.u3Type));
+        Log4(("ctrl.EventInject.u1ErrorCodeValid     %#x\n",      pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
+        Log4(("ctrl.EventInject.u19Reserved          %#x\n",      pVmcb->ctrl.EventInject.n.u19Reserved));
+        Log4(("ctrl.EventInject.u1Valid              %#x\n",      pVmcb->ctrl.EventInject.n.u1Valid));
+        Log4(("ctrl.EventInject.u32ErrorCode         %#x\n",      pVmcb->ctrl.EventInject.n.u32ErrorCode));
+
+        Log4(("ctrl.u64NestedPagingCR3               %#RX64\n",   pVmcb->ctrl.u64NestedPagingCR3));
+
+        Log4(("ctrl.LbrVirt.u1LbrVirt                %#x\n",      pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
+        Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload       %#x\n",      pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
+
+        Log4(("guest.CS.u16Sel                       %RTsel\n",   pVmcb->guest.CS.u16Sel));
+        Log4(("guest.CS.u16Attr                      %#x\n",      pVmcb->guest.CS.u16Attr));
+        Log4(("guest.CS.u32Limit                     %#RX32\n",   pVmcb->guest.CS.u32Limit));
+        Log4(("guest.CS.u64Base                      %#RX64\n",   pVmcb->guest.CS.u64Base));
+        Log4(("guest.DS.u16Sel                       %#RTsel\n",  pVmcb->guest.DS.u16Sel));
+        Log4(("guest.DS.u16Attr                      %#x\n",      pVmcb->guest.DS.u16Attr));
+        Log4(("guest.DS.u32Limit                     %#RX32\n",   pVmcb->guest.DS.u32Limit));
+        Log4(("guest.DS.u64Base                      %#RX64\n",   pVmcb->guest.DS.u64Base));
+        Log4(("guest.ES.u16Sel                       %RTsel\n",   pVmcb->guest.ES.u16Sel));
+        Log4(("guest.ES.u16Attr                      %#x\n",      pVmcb->guest.ES.u16Attr));
+        Log4(("guest.ES.u32Limit                     %#RX32\n",   pVmcb->guest.ES.u32Limit));
+        Log4(("guest.ES.u64Base                      %#RX64\n",   pVmcb->guest.ES.u64Base));
+        Log4(("guest.FS.u16Sel                       %RTsel\n",   pVmcb->guest.FS.u16Sel));
+        Log4(("guest.FS.u16Attr                      %#x\n",      pVmcb->guest.FS.u16Attr));
+        Log4(("guest.FS.u32Limit                     %#RX32\n",   pVmcb->guest.FS.u32Limit));
+        Log4(("guest.FS.u64Base                      %#RX64\n",   pVmcb->guest.FS.u64Base));
+        Log4(("guest.GS.u16Sel                       %RTsel\n",   pVmcb->guest.GS.u16Sel));
+        Log4(("guest.GS.u16Attr                      %#x\n",      pVmcb->guest.GS.u16Attr));
+        Log4(("guest.GS.u32Limit                     %#RX32\n",   pVmcb->guest.GS.u32Limit));
+        Log4(("guest.GS.u64Base                      %#RX64\n",   pVmcb->guest.GS.u64Base));
+
+        Log4(("guest.GDTR.u32Limit                   %#RX32\n",   pVmcb->guest.GDTR.u32Limit));
+        Log4(("guest.GDTR.u64Base                    %#RX64\n",   pVmcb->guest.GDTR.u64Base));
+
+        Log4(("guest.LDTR.u16Sel                     %RTsel\n",   pVmcb->guest.LDTR.u16Sel));
+        Log4(("guest.LDTR.u16Attr                    %#x\n",      pVmcb->guest.LDTR.u16Attr));
+        Log4(("guest.LDTR.u32Limit                   %#RX32\n",   pVmcb->guest.LDTR.u32Limit));
+        Log4(("guest.LDTR.u64Base                    %#RX64\n",   pVmcb->guest.LDTR.u64Base));
+
+        Log4(("guest.IDTR.u32Limit                   %#RX32\n",   pVmcb->guest.IDTR.u32Limit));
+        Log4(("guest.IDTR.u64Base                    %#RX64\n",   pVmcb->guest.IDTR.u64Base));
+
+        Log4(("guest.TR.u16Sel                       %RTsel\n",   pVmcb->guest.TR.u16Sel));
+        Log4(("guest.TR.u16Attr                      %#x\n",      pVmcb->guest.TR.u16Attr));
+        Log4(("guest.TR.u32Limit                     %#RX32\n",   pVmcb->guest.TR.u32Limit));
+        Log4(("guest.TR.u64Base                      %#RX64\n",   pVmcb->guest.TR.u64Base));
+
+        Log4(("guest.u8CPL                           %#x\n",      pVmcb->guest.u8CPL));
+        Log4(("guest.u64CR0                          %#RX64\n",   pVmcb->guest.u64CR0));
+        Log4(("guest.u64CR2                          %#RX64\n",   pVmcb->guest.u64CR2));
+        Log4(("guest.u64CR3                          %#RX64\n",   pVmcb->guest.u64CR3));
+        Log4(("guest.u64CR4                          %#RX64\n",   pVmcb->guest.u64CR4));
+        Log4(("guest.u64DR6                          %#RX64\n",   pVmcb->guest.u64DR6));
+        Log4(("guest.u64DR7                          %#RX64\n",   pVmcb->guest.u64DR7));
+
+        Log4(("guest.u64RIP                          %#RX64\n",   pVmcb->guest.u64RIP));
+        Log4(("guest.u64RSP                          %#RX64\n",   pVmcb->guest.u64RSP));
+        Log4(("guest.u64RAX                          %#RX64\n",   pVmcb->guest.u64RAX));
+        Log4(("guest.u64RFlags                       %#RX64\n",   pVmcb->guest.u64RFlags));
+
+        Log4(("guest.u64SysEnterCS                   %#RX64\n",   pVmcb->guest.u64SysEnterCS));
+        Log4(("guest.u64SysEnterEIP                  %#RX64\n",   pVmcb->guest.u64SysEnterEIP));
+        Log4(("guest.u64SysEnterESP                  %#RX64\n",   pVmcb->guest.u64SysEnterESP));
+
+        Log4(("guest.u64EFER                         %#RX64\n",   pVmcb->guest.u64EFER));
+        Log4(("guest.u64STAR                         %#RX64\n",   pVmcb->guest.u64STAR));
+        Log4(("guest.u64LSTAR                        %#RX64\n",   pVmcb->guest.u64LSTAR));
+        Log4(("guest.u64CSTAR                        %#RX64\n",   pVmcb->guest.u64CSTAR));
+        Log4(("guest.u64SFMASK                       %#RX64\n",   pVmcb->guest.u64SFMASK));
+        Log4(("guest.u64KernelGSBase                 %#RX64\n",   pVmcb->guest.u64KernelGSBase));
+        Log4(("guest.u64PAT                          %#RX64\n",   pVmcb->guest.u64PAT));
+        Log4(("guest.u64DBGCTL                       %#RX64\n",   pVmcb->guest.u64DBGCTL));
+        Log4(("guest.u64BR_FROM                      %#RX64\n",   pVmcb->guest.u64BR_FROM));
+        Log4(("guest.u64BR_TO                        %#RX64\n",   pVmcb->guest.u64BR_TO));
+        Log4(("guest.u64LASTEXCPFROM                 %#RX64\n",   pVmcb->guest.u64LASTEXCPFROM));
+        Log4(("guest.u64LASTEXCPTO                   %#RX64\n",   pVmcb->guest.u64LASTEXCPTO));
+
+        NOREF(pVmcb);
+#endif  /* VBOX_STRICT */
+    }
+    else
+        Log4Func(("rcVMRun=%d\n", rcVMRun));
+}
+
+
+/**
+ * Check per-VM and per-VCPU force flag actions that require us to go back to
+ * ring-3 for one reason or another.
+ *
+ * @returns VBox status code (information status code included).
+ * @retval VINF_SUCCESS if we don't have any actions that require going back to
+ *         ring-3.
+ * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
+ * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
+ *         interrupts)
+ * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
+ *         all EMTs to be in ring-3.
+ * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
+ * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
+ *         to the EM loop.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+static int hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
+{
+    Assert(VMMRZCallRing3IsEnabled(pVCpu));
+    Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
+
+    /* Could happen as a result of longjump. */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
+        PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
+
+    /* Update pending interrupts into the APIC's IRR. */
+    if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
+        APICUpdatePendingInterrupts(pVCpu);
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (   VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
+                            ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
+        || VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
+                               ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
+    {
+        /* Pending PGM C3 sync. */
+        if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
+        {
+            int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
+                                VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+            if (rc != VINF_SUCCESS)
+            {
+                Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
+                return rc;
+            }
+        }
+
+        /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
+        /* -XXX- what was that about single stepping?  */
+        if (   VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
+            || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
+        {
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
+            int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
+            Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
+            return rc;
+        }
+
+        /* Pending VM request packets, such as hardware interrupts. */
+        if (   VM_FF_IS_SET(pVM, VM_FF_REQUEST)
+            || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
+        {
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
+            Log4Func(("Pending VM request forcing us back to ring-3\n"));
+            return VINF_EM_PENDING_REQUEST;
+        }
+
+        /* Pending PGM pool flushes. */
+        if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
+        {
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
+            Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
+            return VINF_PGM_POOL_FLUSH_PENDING;
+        }
+
+        /* Pending DMA requests. */
+        if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
+        {
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
+            Log4Func(("Pending DMA request forcing us back to ring-3\n"));
+            return VINF_EM_RAW_TO_R3;
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Does the preparations before executing guest code in AMD-V.
+ *
+ * This may cause longjmps to ring-3 and may even result in rescheduling to the
+ * recompiler. We must be cautious what we do here regarding committing
+ * guest-state information into the VMCB assuming we assuredly execute the guest
+ * in AMD-V. If we fall back to the recompiler after updating the VMCB and
+ * clearing the common-state (TRPM/forceflags), we must undo those changes so
+ * that the recompiler can (and should) use them when it resumes guest
+ * execution. Otherwise such operations must be done when we can no longer
+ * exit to ring-3.
+ *
+ * @returns VBox status code (informational status codes included).
+ * @retval VINF_SUCCESS if we can proceed with running the guest.
+ * @retval VINF_* scheduling changes, we have to go back to ring-3.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pSvmTransient   Pointer to the SVM transient structure.
+ */
+static int hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
+    if (pSvmTransient->fIsNestedGuest)
+    {
+        Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
+        return VINF_EM_RESCHEDULE_REM;
+    }
+#endif
+
+    /* Check force flag actions that might require us to go back to ring-3. */
+    int rc = hmR0SvmCheckForceFlags(pVCpu);
+    if (rc != VINF_SUCCESS)
+        return rc;
+
+    if (TRPMHasTrap(pVCpu))
+        hmR0SvmTrpmTrapToPendingEvent(pVCpu);
+    else if (!pVCpu->hm.s.Event.fPending)
+    {
+        VBOXSTRICTRC rcStrict = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
+        if (   rcStrict != VINF_SUCCESS
+            || pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
+        {
+            /* If a nested-guest VM-exit occurred, bail. */
+            if (pSvmTransient->fIsNestedGuest)
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
+            return VBOXSTRICTRC_VAL(rcStrict);
+        }
+    }
+
+    /*
+     * On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
+     * Just do it in software, see @bugref{8411}.
+     * NB: If we could continue a task switch exit we wouldn't need to do this.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (RT_UNLIKELY(   !pVM->hm.s.svm.u32Features
+                    &&  pVCpu->hm.s.Event.fPending
+                    &&  SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
+        return VINF_EM_RAW_INJECT_TRPM_EVENT;
+
+#ifdef HMSVM_SYNC_FULL_GUEST_STATE
+    Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    /*
+     * Set up the nested-guest VMCB for execution using hardware-assisted SVM.
+     */
+    if (pSvmTransient->fIsNestedGuest)
+        hmR0SvmSetupVmcbNested(pVCpu);
+#endif
+
+    /*
+     * Export the guest state bits that are not shared with the host in any way as we can
+     * longjmp or get preempted in the midst of exporting some of the state.
+     */
+    rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
+    AssertRCReturn(rc, rc);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
+
+    /* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
+    Assert(!pSvmTransient->fIsNestedGuest || pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
+
+    /*
+     * If we're not intercepting TPR changes in the guest, save the guest TPR before the
+     * world-switch so we can update it on the way back if the guest changed the TPR.
+     */
+    if (pVCpu->hm.s.svm.fSyncVTpr)
+    {
+        Assert(!pSvmTransient->fIsNestedGuest);
+        PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
+        if (pVM->hm.s.fTPRPatchingActive)
+            pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
+        else
+            pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
+    }
+
+    /*
+     * No longjmps to ring-3 from this point on!!!
+     *
+     * Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
+     * better than a kernel panic. This also disables flushing of the R0-logger instance.
+     */
+    VMMRZCallRing3Disable(pVCpu);
+
+    /*
+     * We disable interrupts so that we don't miss any interrupts that would flag preemption
+     * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
+     * preemption disabled for a while.  Since this is purly to aid the
+     * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
+     * disable interrupt on NT.
+     *
+     * We need to check for force-flags that could've possible been altered since we last
+     * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
+     * see @bugref{6398}).
+     *
+     * We also check a couple of other force-flags as a last opportunity to get the EMT back
+     * to ring-3 before executing guest code.
+     */
+    pSvmTransient->fEFlags = ASMIntDisableFlags();
+    if (   VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
+        || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
+    {
+        ASMSetFlags(pSvmTransient->fEFlags);
+        VMMRZCallRing3Enable(pVCpu);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
+        return VINF_EM_RAW_TO_R3;
+    }
+    if (RTThreadPreemptIsPending(NIL_RTTHREAD))
+    {
+        ASMSetFlags(pSvmTransient->fEFlags);
+        VMMRZCallRing3Enable(pVCpu);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
+        return VINF_EM_RAW_INTERRUPT;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
+ * doing so.
+ *
+ * This means there is no backing out to ring-3 or anywhere else at this point.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pSvmTransient   Pointer to the SVM transient structure.
+ *
+ * @remarks Called with preemption disabled.
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+    Assert(VMMR0IsLogFlushDisabled(pVCpu));
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);            /* Indicate the start of guest execution. */
+
+    PVMCC      pVM   = pVCpu->CTX_SUFF(pVM);
+    PSVMVMCB pVmcb = pSvmTransient->pVmcb;
+
+    hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
+
+    if (!CPUMIsGuestFPUStateActive(pVCpu))
+    {
+        STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
+        CPUMR0LoadGuestFPU(pVM, pVCpu);     /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
+        STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
+    }
+
+    /* Load the state shared between host and guest (FPU, debug). */
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
+        hmR0SvmExportSharedState(pVCpu, pVmcb);
+
+    pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;        /* Preemption might set this, nothing to do on AMD-V. */
+    AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
+
+    PHMPHYSCPU    pHostCpu         = hmR0GetCurrentCpu();
+    RTCPUID const idHostCpu        = pHostCpu->idCpu;
+    bool const    fMigratedHostCpu = idHostCpu != pVCpu->hm.s.idLastCpu;
+
+    /* Setup TSC offsetting. */
+    if (   pSvmTransient->fUpdateTscOffsetting
+        || fMigratedHostCpu)
+    {
+        hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
+        pSvmTransient->fUpdateTscOffsetting = false;
+    }
+
+    /* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
+    if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC | SVM_CTRL_INTERCEPT_RDTSCP)))
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
+    else
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
+
+    /* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
+    if (fMigratedHostCpu)
+        pVmcb->ctrl.u32VmcbCleanBits = 0;
+
+    /* Store status of the shared guest-host state at the time of VMRUN. */
+    pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
+    pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    uint8_t *pbMsrBitmap;
+    if (!pSvmTransient->fIsNestedGuest)
+        pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
+    else
+    {
+        hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
+
+        /* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
+        pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
+        pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
+    }
+#else
+    uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
+#endif
+
+    ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);    /* Used for TLB flushing, set this across the world switch. */
+    /* Flush the appropriate tagged-TLB entries. */
+    hmR0SvmFlushTaggedTlb(pHostCpu,  pVCpu, pVmcb);
+    Assert(pVCpu->hm.s.idLastCpu == idHostCpu);
+
+    STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
+
+    TMNotifyStartOfExecution(pVM, pVCpu);                       /* Finally, notify TM to resume its clocks as we're about
+                                                                   to start executing. */
+
+    /*
+     * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
+     * (that don't cause exits) reads the guest MSR, see @bugref{3324}.
+     *
+     * This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
+     */
+    if (   pVM->cpum.ro.HostFeatures.fRdTscP
+        && !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
+    {
+        uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
+        pVCpu->hm.s.svm.u64HostTscAux   = ASMRdMsr(MSR_K8_TSC_AUX);
+        if (uGuestTscAux != pVCpu->hm.s.svm.u64HostTscAux)
+            ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
+        hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
+        pSvmTransient->fRestoreTscAuxMsr = true;
+    }
+    else
+    {
+        hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
+        pSvmTransient->fRestoreTscAuxMsr = false;
+    }
+    pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
+
+    /*
+     * If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
+     * virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
+     * from the VMCB.
+     */
+    bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
+    if (!fSupportsVmcbCleanBits)
+        pVmcb->ctrl.u32VmcbCleanBits = 0;
+}
+
+
+/**
+ * Wrapper for running the guest (or nested-guest) code in AMD-V.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   HCPhysVmcb  The host physical address of the VMCB.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
+{
+    /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    pCtx->fExtrn |= HMSVM_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
+
+    /*
+     * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
+     * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
+     * callee-saved and thus the need for this XMM wrapper.
+     *
+     * Refer MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+#ifdef VBOX_WITH_KERNEL_USING_XMM
+    return hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, HCPhysVmcb, pCtx, pVM, pVCpu, pVCpu->hm.s.svm.pfnVMRun);
+#else
+    return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, HCPhysVmcb, pCtx, pVM, pVCpu);
+#endif
+}
+
+
+/**
+ * Performs some essential restoration of state after running guest (or
+ * nested-guest) code in AMD-V.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pSvmTransient   Pointer to the SVM transient structure.
+ * @param   rcVMRun         Return code of VMRUN.
+ *
+ * @remarks Called with interrupts disabled.
+ * @remarks No-long-jump zone!!! This function will however re-enable longjmps
+ *          unconditionally when it is safe to do so.
+ */
+static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, int rcVMRun)
+{
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+
+    uint64_t const uHostTsc = ASMReadTSC();                     /* Read the TSC as soon as possible. */
+    ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false);   /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
+    ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits);            /* Initialized in vmR3CreateUVM(): used for EMT poking. */
+
+    PSVMVMCB     pVmcb     = pSvmTransient->pVmcb;
+    PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
+
+    /* TSC read must be done early for maximum accuracy. */
+    if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
+    {
+        if (!pSvmTransient->fIsNestedGuest)
+            TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVmcbCtrl->u64TSCOffset);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        else
+        {
+            /* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
+            uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uHostTsc + pVmcbCtrl->u64TSCOffset);
+            TMCpuTickSetLastSeen(pVCpu, uGstTsc);
+        }
+#endif
+    }
+
+    if (pSvmTransient->fRestoreTscAuxMsr)
+    {
+        uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
+        CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
+        if (u64GuestTscAuxMsr != pVCpu->hm.s.svm.u64HostTscAux)
+            ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.svm.u64HostTscAux);
+    }
+
+    STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    TMNotifyEndOfExecution(pVM, pVCpu);                         /* Notify TM that the guest is no longer running. */
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
+
+    Assert(!(ASMGetFlags() & X86_EFL_IF));
+    ASMSetFlags(pSvmTransient->fEFlags);                        /* Enable interrupts. */
+    VMMRZCallRing3Enable(pVCpu);                                /* It is now safe to do longjmps to ring-3!!! */
+
+    /* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
+    if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
+    {
+        Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", rcVMRun));
+        return;
+    }
+
+    pSvmTransient->u64ExitCode        = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
+    pSvmTransient->fVectoringDoublePF = false;                  /* Vectoring double page-fault needs to be determined later. */
+    pSvmTransient->fVectoringPF       = false;                  /* Vectoring page-fault needs to be determined later. */
+    pVmcbCtrl->u32VmcbCleanBits       = HMSVM_VMCB_CLEAN_ALL;   /* Mark the VMCB-state cache as unmodified by VMM. */
+
+#ifdef HMSVM_SYNC_FULL_GUEST_STATE
+    hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+    Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
+#else
+    /*
+     * Always import the following:
+     *
+     *   - RIP for exit optimizations and evaluating event injection on re-entry.
+     *   - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
+     *     state on preemption.
+     *   - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
+     *   - CS for exit optimizations.
+     *   - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
+     *     assembly switcher code.
+     *   - Shared state (only DR7 currently) for exporting shared debug state on preemption.
+     */
+    hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
+                                 | CPUMCTX_EXTRN_RFLAGS
+                                 | CPUMCTX_EXTRN_RAX
+                                 | CPUMCTX_EXTRN_RSP
+                                 | CPUMCTX_EXTRN_CS
+                                 | CPUMCTX_EXTRN_HWVIRT
+                                 | CPUMCTX_EXTRN_HM_SVM_INT_SHADOW
+                                 | CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
+                                 | HMSVM_CPUMCTX_SHARED_STATE);
+#endif
+
+    if (   pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
+        && pVCpu->hm.s.svm.fSyncVTpr)
+    {
+        Assert(!pSvmTransient->fIsNestedGuest);
+        /* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
+        if (   pVM->hm.s.fTPRPatchingActive
+            && (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
+        {
+            int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
+            AssertRC(rc);
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
+        }
+        /* Sync TPR when we aren't intercepting CR8 writes. */
+        else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
+        {
+            int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
+            AssertRC(rc);
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
+        }
+    }
+
+#ifdef DEBUG_ramshankar
+    if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
+    {
+        hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+        hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
+                        0 /* uVerbose */);
+    }
+#endif
+
+    HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
+    EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
+                     pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
+}
+
+
+/**
+ * Runs the guest code using AMD-V.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pcLoops     Pointer to the number of executed loops.
+ */
+static int hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
+{
+    uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
+    Assert(pcLoops);
+    Assert(*pcLoops <= cMaxResumeLoops);
+
+    SVMTRANSIENT SvmTransient;
+    RT_ZERO(SvmTransient);
+    SvmTransient.fUpdateTscOffsetting = true;
+    SvmTransient.pVmcb = pVCpu->hm.s.svm.pVmcb;
+
+    int rc = VERR_INTERNAL_ERROR_5;
+    for (;;)
+    {
+        Assert(!HMR0SuspendPending());
+        HMSVM_ASSERT_CPU_SAFE(pVCpu);
+
+        /* Preparatory work for running nested-guest code, this may force us to return to
+           ring-3.  This bugger disables interrupts on VINF_SUCCESS! */
+        STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
+        rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
+        if (rc != VINF_SUCCESS)
+            break;
+
+        /*
+         * No longjmps to ring-3 from this point on!!!
+         *
+         * Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
+         * better than a kernel panic. This also disables flushing of the R0-logger instance.
+         */
+        hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
+        rc = hmR0SvmRunGuest(pVCpu, pVCpu->hm.s.svm.HCPhysVmcb);
+
+        /* Restore any residual host-state and save any bits shared between host and guest
+           into the guest-CPU state.  Re-enables interrupts! */
+        hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
+
+        if (RT_UNLIKELY(   rc != VINF_SUCCESS                               /* Check for VMRUN errors. */
+                        || SvmTransient.u64ExitCode == SVM_EXIT_INVALID))   /* Check for invalid guest-state errors. */
+        {
+            if (rc == VINF_SUCCESS)
+                rc = VERR_SVM_INVALID_GUEST_STATE;
+            STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
+            hmR0SvmReportWorldSwitchError(pVCpu, rc);
+            break;
+        }
+
+        /* Handle the #VMEXIT. */
+        HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
+        STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
+        VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
+        rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
+        STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
+        if (rc != VINF_SUCCESS)
+            break;
+        if (++(*pcLoops) >= cMaxResumeLoops)
+        {
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
+            rc = VINF_EM_RAW_INTERRUPT;
+            break;
+        }
+    }
+
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
+    return rc;
+}
+
+
+/**
+ * Runs the guest code using AMD-V in single step mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pcLoops     Pointer to the number of executed loops.
+ */
+static int hmR0SvmRunGuestCodeStep(PVMCPUCC pVCpu, uint32_t *pcLoops)
+{
+    uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
+    Assert(pcLoops);
+    Assert(*pcLoops <= cMaxResumeLoops);
+
+    SVMTRANSIENT SvmTransient;
+    RT_ZERO(SvmTransient);
+    SvmTransient.fUpdateTscOffsetting = true;
+    SvmTransient.pVmcb = pVCpu->hm.s.svm.pVmcb;
+
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    uint16_t uCsStart  = pCtx->cs.Sel;
+    uint64_t uRipStart = pCtx->rip;
+
+    int rc = VERR_INTERNAL_ERROR_5;
+    for (;;)
+    {
+        Assert(!HMR0SuspendPending());
+        AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
+                  ("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
+                  (unsigned)RTMpCpuId(), *pcLoops));
+
+        /* Preparatory work for running nested-guest code, this may force us to return to
+           ring-3.  This bugger disables interrupts on VINF_SUCCESS! */
+        STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
+        rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
+        if (rc != VINF_SUCCESS)
+            break;
+
+        /*
+         * No longjmps to ring-3 from this point on!!!
+         *
+         * Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
+         * better than a kernel panic. This also disables flushing of the R0-logger instance.
+         */
+        hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
+
+        rc = hmR0SvmRunGuest(pVCpu, pVCpu->hm.s.svm.HCPhysVmcb);
+
+        /* Restore any residual host-state and save any bits shared between host and guest
+           into the guest-CPU state.  Re-enables interrupts! */
+        hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
+
+        if (RT_UNLIKELY(   rc != VINF_SUCCESS                               /* Check for VMRUN errors. */
+                        || SvmTransient.u64ExitCode == SVM_EXIT_INVALID))   /* Check for invalid guest-state errors. */
+        {
+            if (rc == VINF_SUCCESS)
+                rc = VERR_SVM_INVALID_GUEST_STATE;
+            STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
+            hmR0SvmReportWorldSwitchError(pVCpu, rc);
+            return rc;
+        }
+
+        /* Handle the #VMEXIT. */
+        HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
+        STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
+        VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
+        rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
+        STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
+        if (rc != VINF_SUCCESS)
+            break;
+        if (++(*pcLoops) >= cMaxResumeLoops)
+        {
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
+            rc = VINF_EM_RAW_INTERRUPT;
+            break;
+        }
+
+        /*
+         * Did the RIP change, if so, consider it a single step.
+         * Otherwise, make sure one of the TFs gets set.
+         */
+        if (   pCtx->rip    != uRipStart
+            || pCtx->cs.Sel != uCsStart)
+        {
+            rc = VINF_EM_DBG_STEPPED;
+            break;
+        }
+        pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR_MASK;
+    }
+
+    /*
+     * Clear the X86_EFL_TF if necessary.
+     */
+    if (pVCpu->hm.s.fClearTrapFlag)
+    {
+        pVCpu->hm.s.fClearTrapFlag = false;
+        pCtx->eflags.Bits.u1TF = 0;
+    }
+
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
+    return rc;
+}
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+/**
+ * Runs the nested-guest code using AMD-V.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pcLoops     Pointer to the number of executed loops. If we're switching
+ *                      from the guest-code execution loop to this nested-guest
+ *                      execution loop pass the remainder value, else pass 0.
+ */
+static int hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
+{
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
+    Assert(pcLoops);
+    Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops);
+
+    SVMTRANSIENT SvmTransient;
+    RT_ZERO(SvmTransient);
+    SvmTransient.fUpdateTscOffsetting = true;
+    SvmTransient.pVmcb = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
+    SvmTransient.fIsNestedGuest = true;
+
+    int rc = VERR_INTERNAL_ERROR_4;
+    for (;;)
+    {
+        Assert(!HMR0SuspendPending());
+        HMSVM_ASSERT_CPU_SAFE(pVCpu);
+
+        /* Preparatory work for running nested-guest code, this may force us to return to
+           ring-3.  This bugger disables interrupts on VINF_SUCCESS! */
+        STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
+        rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
+        if (    rc != VINF_SUCCESS
+            || !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+            break;
+
+        /*
+         * No longjmps to ring-3 from this point on!!!
+         *
+         * Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
+         * better than a kernel panic. This also disables flushing of the R0-logger instance.
+         */
+        hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
+
+        rc = hmR0SvmRunGuest(pVCpu, pCtx->hwvirt.svm.HCPhysVmcb);
+
+        /* Restore any residual host-state and save any bits shared between host and guest
+           into the guest-CPU state.  Re-enables interrupts! */
+        hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
+
+        if (RT_LIKELY(   rc == VINF_SUCCESS
+                      && SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
+        { /* extremely likely */ }
+        else
+        {
+            /* VMRUN failed, shouldn't really happen, Guru. */
+            if (rc != VINF_SUCCESS)
+                break;
+
+            /* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
+            HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+            AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
+            rc = VBOXSTRICTRC_TODO(IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0));
+            break;
+        }
+
+        /* Handle the #VMEXIT. */
+        HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
+        STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
+        VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pCtx->hwvirt.svm.CTX_SUFF(pVmcb));
+        rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
+        STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
+        if (rc == VINF_SUCCESS)
+        {
+            if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+            {
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
+                rc = VINF_SVM_VMEXIT;
+            }
+            else
+            {
+                if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
+                    continue;
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
+                rc = VINF_EM_RAW_INTERRUPT;
+            }
+        }
+        else
+            Assert(rc != VINF_SVM_VMEXIT);
+        break;
+        /** @todo NSTSVM: handle single-stepping. */
+    }
+
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
+    return rc;
+}
+#endif
+
+
+/**
+ * Runs the guest code using AMD-V.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
+{
+    AssertPtr(pVCpu);
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    Assert(VMMRZCallRing3IsEnabled(pVCpu));
+    Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
+    HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
+
+    uint32_t cLoops = 0;
+    int      rc;
+    for (;;)
+    {
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
+#else
+        NOREF(pCtx);
+        bool const fInNestedGuestMode = false;
+#endif
+        if (!fInNestedGuestMode)
+        {
+            if (!pVCpu->hm.s.fSingleInstruction)
+                rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
+            else
+                rc = hmR0SvmRunGuestCodeStep(pVCpu, &cLoops);
+        }
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        else
+            rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
+
+        if (rc == VINF_SVM_VMRUN)
+        {
+            Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
+            continue;
+        }
+        if (rc == VINF_SVM_VMEXIT)
+        {
+            Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
+            continue;
+        }
+#endif
+        break;
+    }
+
+    /* Fixup error codes. */
+    if (rc == VERR_EM_INTERPRETER)
+        rc = VINF_EM_RAW_EMULATE_INSTR;
+    else if (rc == VINF_EM_RESET)
+        rc = VINF_EM_TRIPLE_FAULT;
+
+    /* Prepare to return to ring-3. This will remove longjmp notifications. */
+    rc = hmR0SvmExitToRing3(pVCpu, rc);
+    Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
+    Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
+    return rc;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+/**
+ * Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
+ *
+ * @param   pvIoBitmap      Pointer to the nested-guest IO bitmap.
+ * @param   pIoExitInfo     Pointer to the SVMIOIOEXITINFO.
+ */
+static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
+{
+    const uint16_t    u16Port       = pIoExitInfo->n.u16Port;
+    const SVMIOIOTYPE enmIoType     = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
+    const uint8_t     cbReg         = (pIoExitInfo->u  >> SVM_IOIO_OP_SIZE_SHIFT)   & 7;
+    const uint8_t     cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
+    const uint8_t     iEffSeg       = pIoExitInfo->n.u3Seg;
+    const bool        fRep          = pIoExitInfo->n.u1Rep;
+    const bool        fStrIo        = pIoExitInfo->n.u1Str;
+
+    return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
+                                   NULL /* pIoExitInfo */);
+}
+
+
+/**
+ * Handles a nested-guest \#VMEXIT (for all EXITCODE values except
+ * SVM_EXIT_INVALID).
+ *
+ * @returns VBox status code (informational status codes included).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pSvmTransient   Pointer to the SVM transient structure.
+ */
+static int hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
+    Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
+    Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
+
+    /*
+     * We import the complete state here because we use separate VMCBs for the guest and the
+     * nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
+     * the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
+     */
+#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
+    do { \
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
+        return VBOXSTRICTRC_TODO(IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2))); \
+    } while (0)
+
+    /*
+     * For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
+     * nested-guest. If it isn't, it should be handled by the (outer) guest.
+     */
+    PSVMVMCB       pVmcbNstGst     = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
+    PCCPUMCTX      pCtx            = &pVCpu->cpum.GstCtx;
+    PSVMVMCBCTRL   pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
+    uint64_t const uExitCode       = pVmcbNstGstCtrl->u64ExitCode;
+    uint64_t const uExitInfo1      = pVmcbNstGstCtrl->u64ExitInfo1;
+    uint64_t const uExitInfo2      = pVmcbNstGstCtrl->u64ExitInfo2;
+
+    Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
+    switch (uExitCode)
+    {
+        case SVM_EXIT_CPUID:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_RDTSC:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_RDTSCP:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_MONITOR:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_MWAIT:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitMwait(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_HLT:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitHlt(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_MSR:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
+            {
+                uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
+                uint16_t offMsrpm;
+                uint8_t  uMsrpmBit;
+                int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
+                if (RT_SUCCESS(rc))
+                {
+                    Assert(uMsrpmBit == 0 || uMsrpmBit == 2 || uMsrpmBit == 4 || uMsrpmBit == 6);
+                    Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
+
+                    uint8_t const *pbMsrBitmap = (uint8_t const *)pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
+                    pbMsrBitmap               += offMsrpm;
+                    bool const fInterceptRead  = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
+                    bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
+
+                    if (   (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
+                        || (fInterceptRead  && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
+                    {
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    }
+                }
+                else
+                {
+                    /*
+                     * MSRs not covered by the MSRPM automatically cause an #VMEXIT.
+                     * See AMD-V spec. "15.11 MSR Intercepts".
+                     */
+                    Assert(rc == VERR_OUT_OF_RANGE);
+                    NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                }
+            }
+            return hmR0SvmExitMsr(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_IOIO:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
+            {
+                void *pvIoBitmap = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvIoBitmap);
+                SVMIOIOEXITINFO IoExitInfo;
+                IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
+                bool const fIntercept = hmR0SvmIsIoInterceptSet(pvIoBitmap, &IoExitInfo);
+                if (fIntercept)
+                    NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            }
+            return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_XCPT_PF:
+        {
+            PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+            if (pVM->hm.s.fNestedPaging)
+            {
+                uint32_t const u32ErrCode    = pVmcbNstGstCtrl->u64ExitInfo1;
+                uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
+
+                /* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
+                if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
+                    NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
+
+                /* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
+                HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
+                hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
+                return VINF_SUCCESS;
+            }
+            return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_XCPT_UD:
+        {
+            if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            hmR0SvmSetPendingXcptUD(pVCpu);
+            return VINF_SUCCESS;
+        }
+
+        case SVM_EXIT_XCPT_MF:
+        {
+            if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_XCPT_DB:
+        {
+            if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_XCPT_AC:
+        {
+            if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_XCPT_BP:
+        {
+            if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_READ_CR0:
+        case SVM_EXIT_READ_CR3:
+        case SVM_EXIT_READ_CR4:
+        {
+            uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
+            if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_CR0_SEL_WRITE:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_WRITE_CR0:
+        case SVM_EXIT_WRITE_CR3:
+        case SVM_EXIT_WRITE_CR4:
+        case SVM_EXIT_WRITE_CR8:    /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
+        {
+            uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
+            Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
+
+            if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_PAUSE:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitPause(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_VINTR:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_INTR:
+        case SVM_EXIT_NMI:
+        case SVM_EXIT_SMI:
+        case SVM_EXIT_XCPT_NMI:     /* Should not occur, SVM_EXIT_NMI is used instead. */
+        {
+            /*
+             * We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
+             *
+             * Although we don't intercept SMIs, the nested-guest might. Therefore, we might
+             * get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
+             * nested-guest #VMEXIT.
+             *
+             * We shall import the complete state here as we may cause #VMEXITs from ring-3
+             * while trying to inject interrupts, see comment at the top of this function.
+             */
+            HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
+            return hmR0SvmExitIntr(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_FERR_FREEZE:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_INVLPG:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_WBINVD:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_INVD:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitInvd(pVCpu, pSvmTransient);
+        }
+
+        case SVM_EXIT_RDPMC:
+        {
+            if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
+                NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+            return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
+        }
+
+        default:
+        {
+            switch (uExitCode)
+            {
+                case SVM_EXIT_READ_DR0:     case SVM_EXIT_READ_DR1:     case SVM_EXIT_READ_DR2:     case SVM_EXIT_READ_DR3:
+                case SVM_EXIT_READ_DR6:     case SVM_EXIT_READ_DR7:     case SVM_EXIT_READ_DR8:     case SVM_EXIT_READ_DR9:
+                case SVM_EXIT_READ_DR10:    case SVM_EXIT_READ_DR11:    case SVM_EXIT_READ_DR12:    case SVM_EXIT_READ_DR13:
+                case SVM_EXIT_READ_DR14:    case SVM_EXIT_READ_DR15:
+                {
+                    uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
+                    if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_WRITE_DR0:    case SVM_EXIT_WRITE_DR1:    case SVM_EXIT_WRITE_DR2:    case SVM_EXIT_WRITE_DR3:
+                case SVM_EXIT_WRITE_DR6:    case SVM_EXIT_WRITE_DR7:    case SVM_EXIT_WRITE_DR8:    case SVM_EXIT_WRITE_DR9:
+                case SVM_EXIT_WRITE_DR10:   case SVM_EXIT_WRITE_DR11:   case SVM_EXIT_WRITE_DR12:   case SVM_EXIT_WRITE_DR13:
+                case SVM_EXIT_WRITE_DR14:   case SVM_EXIT_WRITE_DR15:
+                {
+                    uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
+                    if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_XCPT_DE:
+                /*   SVM_EXIT_XCPT_DB: */       /* Handled above. */
+                /*   SVM_EXIT_XCPT_NMI: */      /* Handled above. */
+                /*   SVM_EXIT_XCPT_BP: */       /* Handled above. */
+                case SVM_EXIT_XCPT_OF:
+                case SVM_EXIT_XCPT_BR:
+                /*   SVM_EXIT_XCPT_UD: */       /* Handled above. */
+                case SVM_EXIT_XCPT_NM:
+                case SVM_EXIT_XCPT_DF:
+                case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
+                case SVM_EXIT_XCPT_TS:
+                case SVM_EXIT_XCPT_NP:
+                case SVM_EXIT_XCPT_SS:
+                case SVM_EXIT_XCPT_GP:
+                /*   SVM_EXIT_XCPT_PF: */       /* Handled above. */
+                case SVM_EXIT_XCPT_15:          /* Reserved.      */
+                /*   SVM_EXIT_XCPT_MF: */       /* Handled above. */
+                /*   SVM_EXIT_XCPT_AC: */       /* Handled above. */
+                case SVM_EXIT_XCPT_MC:
+                case SVM_EXIT_XCPT_XF:
+                case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
+                case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
+                case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
+                {
+                    uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
+                    if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_XSETBV:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_TASK_SWITCH:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_IRET:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    return hmR0SvmExitIret(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_SHUTDOWN:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_VMMCALL:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_CLGI:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                     return hmR0SvmExitClgi(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_STGI:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                     return hmR0SvmExitStgi(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_VMLOAD:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    return hmR0SvmExitVmload(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_VMSAVE:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_INVLPGA:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_VMRUN:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    return hmR0SvmExitVmrun(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_RSM:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    hmR0SvmSetPendingXcptUD(pVCpu);
+                    return VINF_SUCCESS;
+                }
+
+                case SVM_EXIT_SKINIT:
+                {
+                    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
+                        NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+                    hmR0SvmSetPendingXcptUD(pVCpu);
+                    return VINF_SUCCESS;
+                }
+
+                case SVM_EXIT_NPF:
+                {
+                    Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
+                    return hmR0SvmExitNestedPF(pVCpu, pSvmTransient);
+                }
+
+                case SVM_EXIT_INIT:  /* We shouldn't get INIT signals while executing a nested-guest. */
+                    return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
+
+                default:
+                {
+                    AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
+                    pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
+                    return VERR_SVM_UNKNOWN_EXIT;
+                }
+            }
+        }
+    }
+    /* not reached */
+
+#undef NST_GST_VMEXIT_CALL_RET
+}
+#endif
+
+
+/**
+ * Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
+ *
+ * @returns VBox status code (informational status codes included).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pSvmTransient   Pointer to the SVM transient structure.
+ */
+static int hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
+    Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
+
+#ifdef DEBUG_ramshankar
+# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
+        do { \
+            if ((a_fDbg) == 1) \
+                HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
+            int rc = a_CallExpr; \
+            if ((a_fDbg) == 1) \
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
+            return rc; \
+        } while (0)
+#else
+# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
+#endif
+
+    /*
+     * The ordering of the case labels is based on most-frequently-occurring #VMEXITs
+     * for most guests under normal workloads (for some definition of "normal").
+     */
+    uint64_t const uExitCode = pSvmTransient->u64ExitCode;
+    switch (uExitCode)
+    {
+        case SVM_EXIT_NPF:          VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pSvmTransient));
+        case SVM_EXIT_IOIO:         VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pSvmTransient));
+        case SVM_EXIT_RDTSC:        VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pSvmTransient));
+        case SVM_EXIT_RDTSCP:       VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pSvmTransient));
+        case SVM_EXIT_CPUID:        VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pSvmTransient));
+        case SVM_EXIT_XCPT_PF:      VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pSvmTransient));
+        case SVM_EXIT_MSR:          VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pSvmTransient));
+        case SVM_EXIT_MONITOR:      VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pSvmTransient));
+        case SVM_EXIT_MWAIT:        VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pSvmTransient));
+        case SVM_EXIT_HLT:          VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pSvmTransient));
+
+        case SVM_EXIT_XCPT_NMI:     /* Should not occur, SVM_EXIT_NMI is used instead. */
+        case SVM_EXIT_INTR:
+        case SVM_EXIT_NMI:          VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pSvmTransient));
+
+        case SVM_EXIT_READ_CR0:
+        case SVM_EXIT_READ_CR3:
+        case SVM_EXIT_READ_CR4:     VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pSvmTransient));
+
+        case SVM_EXIT_CR0_SEL_WRITE:
+        case SVM_EXIT_WRITE_CR0:
+        case SVM_EXIT_WRITE_CR3:
+        case SVM_EXIT_WRITE_CR4:
+        case SVM_EXIT_WRITE_CR8:    VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pSvmTransient));
+
+        case SVM_EXIT_VINTR:        VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pSvmTransient));
+        case SVM_EXIT_PAUSE:        VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pSvmTransient));
+        case SVM_EXIT_VMMCALL:      VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pSvmTransient));
+        case SVM_EXIT_INVLPG:       VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pSvmTransient));
+        case SVM_EXIT_WBINVD:       VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pSvmTransient));
+        case SVM_EXIT_INVD:         VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pSvmTransient));
+        case SVM_EXIT_RDPMC:        VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pSvmTransient));
+        case SVM_EXIT_IRET:         VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pSvmTransient));
+        case SVM_EXIT_XCPT_UD:      VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pSvmTransient));
+        case SVM_EXIT_XCPT_MF:      VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pSvmTransient));
+        case SVM_EXIT_XCPT_DB:      VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pSvmTransient));
+        case SVM_EXIT_XCPT_AC:      VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pSvmTransient));
+        case SVM_EXIT_XCPT_BP:      VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pSvmTransient));
+        case SVM_EXIT_XCPT_GP:      VMEXIT_CALL_RET(0, hmR0SvmExitXcptGP(pVCpu, pSvmTransient));
+        case SVM_EXIT_XSETBV:       VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pSvmTransient));
+        case SVM_EXIT_FERR_FREEZE:  VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient));
+
+        default:
+        {
+            switch (pSvmTransient->u64ExitCode)
+            {
+                case SVM_EXIT_READ_DR0:     case SVM_EXIT_READ_DR1:     case SVM_EXIT_READ_DR2:     case SVM_EXIT_READ_DR3:
+                case SVM_EXIT_READ_DR6:     case SVM_EXIT_READ_DR7:     case SVM_EXIT_READ_DR8:     case SVM_EXIT_READ_DR9:
+                case SVM_EXIT_READ_DR10:    case SVM_EXIT_READ_DR11:    case SVM_EXIT_READ_DR12:    case SVM_EXIT_READ_DR13:
+                case SVM_EXIT_READ_DR14:    case SVM_EXIT_READ_DR15:
+                    VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pSvmTransient));
+
+                case SVM_EXIT_WRITE_DR0:    case SVM_EXIT_WRITE_DR1:    case SVM_EXIT_WRITE_DR2:    case SVM_EXIT_WRITE_DR3:
+                case SVM_EXIT_WRITE_DR6:    case SVM_EXIT_WRITE_DR7:    case SVM_EXIT_WRITE_DR8:    case SVM_EXIT_WRITE_DR9:
+                case SVM_EXIT_WRITE_DR10:   case SVM_EXIT_WRITE_DR11:   case SVM_EXIT_WRITE_DR12:   case SVM_EXIT_WRITE_DR13:
+                case SVM_EXIT_WRITE_DR14:   case SVM_EXIT_WRITE_DR15:
+                    VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pSvmTransient));
+
+                case SVM_EXIT_TASK_SWITCH:  VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient));
+                case SVM_EXIT_SHUTDOWN:     VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pSvmTransient));
+
+                case SVM_EXIT_SMI:
+                case SVM_EXIT_INIT:
+                {
+                    /*
+                     * We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
+                     * If it ever does, we want to know about it so log the exit code and bail.
+                     */
+                    VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pSvmTransient));
+                }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+                case SVM_EXIT_CLGI:     VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pSvmTransient));
+                case SVM_EXIT_STGI:     VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pSvmTransient));
+                case SVM_EXIT_VMLOAD:   VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pSvmTransient));
+                case SVM_EXIT_VMSAVE:   VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pSvmTransient));
+                case SVM_EXIT_INVLPGA:  VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pSvmTransient));
+                case SVM_EXIT_VMRUN:    VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pSvmTransient));
+#else
+                case SVM_EXIT_CLGI:
+                case SVM_EXIT_STGI:
+                case SVM_EXIT_VMLOAD:
+                case SVM_EXIT_VMSAVE:
+                case SVM_EXIT_INVLPGA:
+                case SVM_EXIT_VMRUN:
+#endif
+                case SVM_EXIT_RSM:
+                case SVM_EXIT_SKINIT:
+                {
+                    hmR0SvmSetPendingXcptUD(pVCpu);
+                    return VINF_SUCCESS;
+                }
+
+#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
+                case SVM_EXIT_XCPT_DE:
+                /*   SVM_EXIT_XCPT_DB: */       /* Handled above. */
+                /*   SVM_EXIT_XCPT_NMI: */      /* Handled above. */
+                /*   SVM_EXIT_XCPT_BP: */       /* Handled above. */
+                case SVM_EXIT_XCPT_OF:
+                case SVM_EXIT_XCPT_BR:
+                /*   SVM_EXIT_XCPT_UD: */       /* Handled above. */
+                case SVM_EXIT_XCPT_NM:
+                case SVM_EXIT_XCPT_DF:
+                case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
+                case SVM_EXIT_XCPT_TS:
+                case SVM_EXIT_XCPT_NP:
+                case SVM_EXIT_XCPT_SS:
+                /*   SVM_EXIT_XCPT_GP: */       /* Handled above. */
+                /*   SVM_EXIT_XCPT_PF: */
+                case SVM_EXIT_XCPT_15:          /* Reserved. */
+                /*   SVM_EXIT_XCPT_MF: */       /* Handled above. */
+                /*   SVM_EXIT_XCPT_AC: */       /* Handled above. */
+                case SVM_EXIT_XCPT_MC:
+                case SVM_EXIT_XCPT_XF:
+                case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
+                case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
+                case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
+                    VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient));
+#endif  /* HMSVM_ALWAYS_TRAP_ALL_XCPTS */
+
+                default:
+                {
+                    AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
+                    pVCpu->hm.s.u32HMError = uExitCode;
+                    return VERR_SVM_UNKNOWN_EXIT;
+                }
+            }
+        }
+    }
+    /* not reached */
+#undef VMEXIT_CALL_RET
+}
+
+
+#ifdef VBOX_STRICT
+/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
+# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
+    RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
+
+# define HMSVM_ASSERT_PREEMPT_CPUID() \
+    do \
+    { \
+         RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
+         AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
+    } while (0)
+
+# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
+    do { \
+        AssertPtr((a_pVCpu)); \
+        AssertPtr((a_pSvmTransient)); \
+        Assert(ASMIntAreEnabled()); \
+        HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
+        HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
+        Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu)); \
+        HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
+        if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
+            HMSVM_ASSERT_PREEMPT_CPUID(); \
+    } while (0)
+#else
+# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
+    do { \
+        RT_NOREF2(a_pVCpu, a_pSvmTransient); \
+    } while (0)
+#endif
+
+
+/**
+ * Gets the IEM exception flags for the specified SVM event.
+ *
+ * @returns The IEM exception flags.
+ * @param   pEvent      Pointer to the SVM event.
+ *
+ * @remarks This function currently only constructs flags required for
+ *          IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
+ *          and CR2 aspects of an exception are not included).
+ */
+static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
+{
+    uint8_t const uEventType = pEvent->n.u3Type;
+    uint32_t      fIemXcptFlags;
+    switch (uEventType)
+    {
+        case SVM_EVENT_EXCEPTION:
+            /*
+             * Only INT3 and INTO instructions can raise #BP and #OF exceptions.
+             * See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
+             */
+            if (pEvent->n.u8Vector == X86_XCPT_BP)
+            {
+                fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_BP_INSTR;
+                break;
+            }
+            if (pEvent->n.u8Vector == X86_XCPT_OF)
+            {
+                fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_OF_INSTR;
+                break;
+            }
+            /** @todo How do we distinguish ICEBP \#DB from the regular one? */
+            RT_FALL_THRU();
+        case SVM_EVENT_NMI:
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
+            break;
+
+        case SVM_EVENT_EXTERNAL_IRQ:
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
+            break;
+
+        case SVM_EVENT_SOFTWARE_INT:
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
+            break;
+
+        default:
+            fIemXcptFlags = 0;
+            AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
+            break;
+    }
+    return fIemXcptFlags;
+}
+
+
+/**
+ * Handle a condition that occurred while delivering an event through the guest
+ * IDT.
+ *
+ * @returns VBox status code (informational error codes included).
+ * @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
+ * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
+ *         continue execution of the guest which will delivery the \#DF.
+ * @retval VINF_EM_RESET if we detected a triple-fault condition.
+ * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pSvmTransient   Pointer to the SVM transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0SvmCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    int rc = VINF_SUCCESS;
+    PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
+
+    Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
+          pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
+          pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
+
+    /*
+     * The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
+     * be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
+     *
+     * See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
+     */
+    if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
+    {
+        IEMXCPTRAISE     enmRaise;
+        IEMXCPTRAISEINFO fRaiseInfo;
+        bool const       fExitIsHwXcpt  = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
+        uint8_t const    uIdtVector     = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
+        if (fExitIsHwXcpt)
+        {
+            uint8_t  const uExitVector      = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
+            uint32_t const fIdtVectorFlags  = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
+            uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
+            enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
+        }
+        else
+        {
+            /*
+             * If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
+             * then we end up here.
+             *
+             * If the event was:
+             *   - a software interrupt, we can re-execute the instruction which will
+             *     regenerate the event.
+             *   - an NMI, we need to clear NMI blocking and re-inject the NMI.
+             *   - a hardware exception or external interrupt, we re-inject it.
+             */
+            fRaiseInfo = IEMXCPTRAISEINFO_NONE;
+            if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
+                enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
+            else
+                enmRaise = IEMXCPTRAISE_PREV_EVENT;
+        }
+
+        switch (enmRaise)
+        {
+            case IEMXCPTRAISE_CURRENT_XCPT:
+            case IEMXCPTRAISE_PREV_EVENT:
+            {
+                /* For software interrupts, we shall re-execute the instruction. */
+                if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
+                {
+                    RTGCUINTPTR GCPtrFaultAddress = 0;
+
+                    /* If we are re-injecting an NMI, clear NMI blocking. */
+                    if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
+                        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
+
+                    /* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
+                    if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
+                    {
+                        pSvmTransient->fVectoringPF = true;
+                        Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n",
+                                  pVCpu->cpum.GstCtx.cr2));
+                    }
+                    else if (   pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
+                             && uIdtVector == X86_XCPT_PF)
+                    {
+                        /*
+                         * If the previous exception was a #PF, we need to recover the CR2 value.
+                         * This can't happen with shadow paging.
+                         */
+                        GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
+                    }
+
+                    /*
+                     * Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
+                     * exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
+                     */
+                    Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
+                    STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
+                    hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
+
+                    Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
+                              pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
+                              pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
+                }
+                break;
+            }
+
+            case IEMXCPTRAISE_REEXEC_INSTR:
+            {
+                Assert(rc == VINF_SUCCESS);
+                break;
+            }
+
+            case IEMXCPTRAISE_DOUBLE_FAULT:
+            {
+                /*
+                 * Determing a vectoring double #PF condition. Used later, when PGM evaluates
+                 * the second #PF as a guest #PF (and not a shadow #PF) and needs to be
+                 * converted into a #DF.
+                 */
+                if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
+                {
+                    Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pVCpu->cpum.GstCtx.cr2));
+                    pSvmTransient->fVectoringDoublePF = true;
+                    Assert(rc == VINF_SUCCESS);
+                }
+                else
+                {
+                    STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
+                    hmR0SvmSetPendingXcptDF(pVCpu);
+                    rc = VINF_HM_DOUBLE_FAULT;
+                }
+                break;
+            }
+
+            case IEMXCPTRAISE_TRIPLE_FAULT:
+            {
+                rc = VINF_EM_RESET;
+                break;
+            }
+
+            case IEMXCPTRAISE_CPU_HANG:
+            {
+                rc = VERR_EM_GUEST_CPU_HANG;
+                break;
+            }
+
+            default:
+                AssertMsgFailedBreakStmt(("Bogus enmRaise value: %d (%#x)\n", enmRaise, enmRaise), rc = VERR_SVM_IPE_2);
+        }
+    }
+    Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET || rc == VERR_EM_GUEST_CPU_HANG);
+    return rc;
+}
+
+
+/**
+ * Advances the guest RIP by the number of bytes specified in @a cb.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cb          RIP increment value in bytes.
+ */
+DECLINLINE(void) hmR0SvmAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
+{
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    pCtx->rip += cb;
+
+    /* Update interrupt shadow. */
+    if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+        && pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+}
+
+
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+
+/** @name \#VMEXIT handlers.
+ * @{
+ */
+
+/**
+ * \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
+ * signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
+ */
+HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
+        STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
+    else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
+
+    /*
+     * AMD-V has no preemption timer and the generic periodic preemption timer has no way to
+     * signal -before- the timer fires if the current interrupt is our own timer or a some
+     * other host interrupt. We also cannot examine what interrupt it is until the host
+     * actually take the interrupt.
+     *
+     * Going back to executing guest code here unconditionally causes random scheduling
+     * problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
+     */
+    return VINF_EM_RAW_INTERRUPT;
+}
+
+
+/**
+ * \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    VBOXSTRICTRC rcStrict;
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
+        PCSVMVMCB     pVmcb   = hmR0SvmGetCurrentVmcb(pVCpu);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedWbinvd(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    VBOXSTRICTRC rcStrict;
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
+        PCSVMVMCB     pVmcb   = hmR0SvmGetCurrentVmcb(pVCpu);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedInvd(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
+    VBOXSTRICTRC rcStrict;
+    PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
+                                                            EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
+                                                            pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
+    if (!pExitRec)
+    {
+        bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+        if (fSupportsNextRipSave)
+        {
+            PCSVMVMCB     pVmcb   = hmR0SvmGetCurrentVmcb(pVCpu);
+            uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+            rcStrict = IEMExecDecodedCpuid(pVCpu, cbInstr);
+        }
+        else
+        {
+            HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+            rcStrict = IEMExecOne(pVCpu);
+        }
+
+        if (rcStrict == VINF_IEM_RAISED_XCPT)
+        {
+            rcStrict = VINF_SUCCESS;
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        }
+        HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    }
+    else
+    {
+        /*
+         * Frequent exit or something needing probing.  Get state and call EMHistoryExec.
+         */
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+
+        Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
+
+        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+
+        Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    }
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    VBOXSTRICTRC rcStrict;
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
+        PCSVMVMCB     pVmcb   = hmR0SvmGetCurrentVmcb(pVCpu);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedRdtsc(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (rcStrict == VINF_SUCCESS)
+        pSvmTransient->fUpdateTscOffsetting = true;
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    VBOXSTRICTRC rcStrict;
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_TSC_AUX);
+        PCSVMVMCB     pVmcb   = hmR0SvmGetCurrentVmcb(pVCpu);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedRdtscp(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (rcStrict == VINF_SUCCESS)
+        pSvmTransient->fUpdateTscOffsetting = true;
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    VBOXSTRICTRC rcStrict;
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
+        PCSVMVMCB     pVmcb   = hmR0SvmGetCurrentVmcb(pVCpu);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedRdpmc(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
+
+    VBOXSTRICTRC rcStrict;
+    bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
+    bool const fSupportsNextRipSave   = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (   fSupportsDecodeAssists
+        && fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
+        PCSVMVMCB     pVmcb     = hmR0SvmGetCurrentVmcb(pVCpu);
+        uint8_t const cbInstr   = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
+        rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_VAL(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    VBOXSTRICTRC rcStrict;
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
+        PCSVMVMCB     pVmcb   = hmR0SvmGetCurrentVmcb(pVCpu);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedHlt(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (   rcStrict == VINF_EM_HALT
+        || rcStrict == VINF_SUCCESS)
+        rcStrict = EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx) ? VINF_SUCCESS : VINF_EM_HALT;
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    if (rcStrict != VINF_SUCCESS)
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
+    return VBOXSTRICTRC_VAL(rcStrict);;
+}
+
+
+/**
+ * \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    /*
+     * If the instruction length is supplied by the CPU is 3 bytes, we can be certain that no
+     * segment override prefix is present (and thus use the default segment DS). Otherwise, a
+     * segment override prefix or other prefixes might be used, in which case we fallback to
+     * IEMExecOne() to figure out.
+     */
+    VBOXSTRICTRC  rcStrict;
+    PCSVMVMCB     pVmcb   = hmR0SvmGetCurrentVmcb(pVCpu);
+    uint8_t const cbInstr = hmR0SvmSupportsNextRipSave(pVCpu) ? pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip : 0;
+    if (cbInstr)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
+        rcStrict = IEMExecDecodedMonitor(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    VBOXSTRICTRC rcStrict;
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
+        PCSVMVMCB     pVmcb   = hmR0SvmGetCurrentVmcb(pVCpu);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedMwait(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (   rcStrict == VINF_EM_HALT
+        && EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
+        rcStrict = VINF_SUCCESS;
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
+ * \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+    return VINF_EM_RESET;
+}
+
+
+/**
+ * \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+    AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
+                     pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
+    RT_NOREF(pVmcb);
+    pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
+    return VERR_SVM_UNEXPECTED_EXIT;
+}
+
+
+/**
+ * \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    Log4Func(("CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
+#ifdef VBOX_WITH_STATISTICS
+    switch (pSvmTransient->u64ExitCode)
+    {
+        case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
+        case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
+        case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
+        case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
+        case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
+    }
+#endif
+
+    bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
+    bool const fSupportsNextRipSave   = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (   fSupportsDecodeAssists
+        && fSupportsNextRipSave)
+    {
+        PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+        bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
+        if (fMovCRx)
+        {
+            HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR_MASK
+                                            | CPUMCTX_EXTRN_APIC_TPR);
+            uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
+            uint8_t const iCrReg  = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
+            uint8_t const iGReg   = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
+            VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
+            HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+            return VBOXSTRICTRC_VAL(rcStrict);
+        }
+        /* else: SMSW instruction, fall back below to IEM for this. */
+    }
+
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+    VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
+    AssertMsg(   rcStrict == VINF_SUCCESS
+              || rcStrict == VINF_PGM_SYNC_CR3
+              || rcStrict == VINF_IEM_RAISED_XCPT,
+              ("hmR0SvmExitReadCRx: IEMExecOne failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    uint64_t const uExitCode = pSvmTransient->u64ExitCode;
+    uint8_t  const iCrReg    = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
+    Assert(iCrReg <= 15);
+
+    VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
+    PCPUMCTX     pCtx = &pVCpu->cpum.GstCtx;
+    bool         fDecodedInstr = false;
+    bool const   fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
+    bool const   fSupportsNextRipSave   = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (   fSupportsDecodeAssists
+        && fSupportsNextRipSave)
+    {
+        PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+        bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
+        if (fMovCRx)
+        {
+            HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4
+                                            | CPUMCTX_EXTRN_APIC_TPR);
+            uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
+            uint8_t const iGReg   = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
+            Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
+            rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
+            fDecodedInstr = true;
+        }
+        /* else: LMSW or CLTS instruction, fall back below to IEM for this. */
+    }
+
+    if (!fDecodedInstr)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        Log4Func(("iCrReg=%#x\n", iCrReg));
+        rcStrict = IEMExecOne(pVCpu);
+        if (RT_UNLIKELY(   rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
+                        || rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
+            rcStrict = VERR_EM_INTERPRETER;
+    }
+
+    if (rcStrict == VINF_SUCCESS)
+    {
+        switch (iCrReg)
+        {
+            case 0:
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
+                break;
+
+            case 2:
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
+                break;
+
+            case 3:
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
+                break;
+
+            case 4:
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
+                break;
+
+            case 8:
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
+                break;
+
+            default:
+            {
+                AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
+                                 pSvmTransient->u64ExitCode, iCrReg));
+                break;
+            }
+        }
+        HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    }
+    else
+        Assert(rcStrict == VERR_EM_INTERPRETER || rcStrict == VINF_PGM_SYNC_CR3);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ */
+static VBOXSTRICTRC hmR0SvmExitReadMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
+{
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
+    Log4Func(("idMsr=%#RX32\n", pVCpu->cpum.GstCtx.ecx));
+
+    VBOXSTRICTRC rcStrict;
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (fSupportsNextRipSave)
+    {
+        /** @todo Optimize this: Only retrieve the MSR bits we need here. CPUMAllMsrs.cpp
+         *  can ask for what it needs instead of using CPUMCTX_EXTRN_ALL_MSRS. */
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedRdmsr(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_ALL_MSRS);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    AssertMsg(   rcStrict == VINF_SUCCESS
+              || rcStrict == VINF_IEM_RAISED_XCPT
+              || rcStrict == VINF_CPUM_R3_MSR_READ,
+              ("hmR0SvmExitReadMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return rcStrict;
+}
+
+
+/**
+ * \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcb       Pointer to the VM control block.
+ * @param   pSvmTransient   Pointer to the SVM-transient structure.
+ */
+static VBOXSTRICTRC hmR0SvmExitWriteMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
+{
+    PCPUMCTX pCtx  = &pVCpu->cpum.GstCtx;
+    uint32_t const idMsr = pCtx->ecx;
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
+    Log4Func(("idMsr=%#RX32\n", idMsr));
+
+    /*
+     * Handle TPR patching MSR writes.
+     * We utilitize the LSTAR MSR for patching.
+     */
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (   pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive
+        && idMsr == MSR_K8_LSTAR)
+    {
+        unsigned cbInstr;
+        if (fSupportsNextRipSave)
+            cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        else
+        {
+            PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
+            int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
+            if (   rc == VINF_SUCCESS
+                && pDis->pCurInstr->uOpcode == OP_WRMSR)
+                Assert(cbInstr > 0);
+            else
+                cbInstr = 0;
+        }
+
+        /* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
+        if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
+        {
+            int rc = APICSetTpr(pVCpu, pCtx->eax & 0xff);
+            AssertRCReturn(rc, rc);
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
+        }
+
+        int rc = VINF_SUCCESS;
+        hmR0SvmAdvanceRip(pVCpu, cbInstr);
+        HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
+        return rc;
+    }
+
+    /*
+     * Handle regular MSR writes.
+     */
+    VBOXSTRICTRC rcStrict;
+    if (fSupportsNextRipSave)
+    {
+        /** @todo Optimize this: We don't need to get much of the MSR state here
+         * since we're only updating.  CPUMAllMsrs.cpp can ask for what it needs and
+         * clear the applicable extern flags. */
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedWrmsr(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_ALL_MSRS);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    AssertMsg(   rcStrict == VINF_SUCCESS
+              || rcStrict == VINF_IEM_RAISED_XCPT
+              || rcStrict == VINF_CPUM_R3_MSR_WRITE,
+              ("hmR0SvmExitWriteMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /* If this is an X2APIC WRMSR access, update the APIC TPR state. */
+        if (   idMsr >= MSR_IA32_X2APIC_START
+            && idMsr <= MSR_IA32_X2APIC_END)
+        {
+            /*
+             * We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
+             * When full APIC register virtualization is implemented we'll have to make sure
+             * APIC state is saved from the VMCB before IEM changes it.
+             */
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
+        }
+        else
+        {
+            switch (idMsr)
+            {
+                case MSR_IA32_TSC:          pSvmTransient->fUpdateTscOffsetting = true;                                     break;
+                case MSR_K6_EFER:           ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR);          break;
+                case MSR_K8_FS_BASE:        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS);                break;
+                case MSR_K8_GS_BASE:        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS);                break;
+                case MSR_IA32_SYSENTER_CS:  ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR);   break;
+                case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);  break;
+                case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);  break;
+            }
+        }
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return rcStrict;
+}
+
+
+/**
+ * \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
+ * \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
+        return VBOXSTRICTRC_TODO(hmR0SvmExitReadMsr(pVCpu, pVmcb));
+
+    Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
+    return VBOXSTRICTRC_TODO(hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient));
+}
+
+
+/**
+ * \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
+
+    /** @todo Stepping with nested-guest. */
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+    {
+        /* We should -not- get this #VMEXIT if the guest's debug registers were active. */
+        if (pSvmTransient->fWasGuestDebugStateActive)
+        {
+            AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
+            pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
+            return VERR_SVM_UNEXPECTED_EXIT;
+        }
+
+        /*
+         * Lazy DR0-3 loading.
+         */
+        if (!pSvmTransient->fWasHyperDebugStateActive)
+        {
+            Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
+            Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
+
+            /* Don't intercept DRx read and writes. */
+            PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
+            pVmcb->ctrl.u16InterceptRdDRx = 0;
+            pVmcb->ctrl.u16InterceptWrDRx = 0;
+            pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
+
+            /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
+            VMMRZCallRing3Disable(pVCpu);
+            HM_DISABLE_PREEMPT(pVCpu);
+
+            /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
+            CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
+            Assert(CPUMIsGuestDebugStateActive(pVCpu));
+
+            HM_RESTORE_PREEMPT();
+            VMMRZCallRing3Enable(pVCpu);
+
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
+            return VINF_SUCCESS;
+        }
+    }
+
+    /*
+     * Interpret the read/writing of DRx.
+     */
+    /** @todo Decode assist.  */
+    VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
+    Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
+    if (RT_LIKELY(rc == VINF_SUCCESS))
+    {
+        /* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
+        /** @todo CPUM should set this flag! */
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
+        HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
+    }
+    else
+        Assert(rc == VERR_EM_INTERPRETER);
+    return VBOXSTRICTRC_TODO(rc);
+}
+
+
+/**
+ * \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    /* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
+    int rc = hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
+    STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
+    return rc;
+}
+
+
+/**
+ * \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+
+    /** @todo decode assists... */
+    VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+        pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
+        Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%RTbool (cr4=%#RX64)\n", pCtx->aXcr[0], pVCpu->hm.s.fLoadSaveGuestXcr0,
+                  pCtx->cr4));
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK);
+
+    /* I/O operation lookup arrays. */
+    static uint32_t const s_aIOSize[8]  = { 0, 1, 2, 0, 4, 0, 0, 0 };                   /* Size of the I/O accesses in bytes. */
+    static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 };  /* AND masks for saving
+                                                                                           the result (in AL/AX/EAX). */
+    PVMCC      pVM   = pVCpu->CTX_SUFF(pVM);
+    PCPUMCTX pCtx  = &pVCpu->cpum.GstCtx;
+    PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+
+    Log4Func(("CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
+
+    /* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
+    SVMIOIOEXITINFO IoExitInfo;
+    IoExitInfo.u       = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
+    uint32_t uIOWidth  = (IoExitInfo.u >> 4) & 0x7;
+    uint32_t cbValue   = s_aIOSize[uIOWidth];
+    uint32_t uAndVal   = s_aIOOpAnd[uIOWidth];
+
+    if (RT_UNLIKELY(!cbValue))
+    {
+        AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
+        return VERR_EM_INTERPRETER;
+    }
+
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
+    VBOXSTRICTRC rcStrict;
+    PCEMEXITREC pExitRec = NULL;
+    if (   !pVCpu->hm.s.fSingleInstruction
+        && !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
+        pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
+                                                    !IoExitInfo.n.u1Str
+                                                    ? IoExitInfo.n.u1Type == SVM_IOIO_READ
+                                                    ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
+                                                    : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
+                                                    : IoExitInfo.n.u1Type == SVM_IOIO_READ
+                                                    ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
+                                                    : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
+                                                    pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
+    if (!pExitRec)
+    {
+        bool fUpdateRipAlready = false;
+        if (IoExitInfo.n.u1Str)
+        {
+            /* INS/OUTS - I/O String instruction. */
+            /** @todo Huh? why can't we use the segment prefix information given by AMD-V
+             *        in EXITINFO1? Investigate once this thing is up and running. */
+            Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
+                      IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
+            AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
+            static IEMMODE const s_aenmAddrMode[8] =
+            {
+                (IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
+            };
+            IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
+            if (enmAddrMode != (IEMMODE)-1)
+            {
+                uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
+                if (cbInstr <= 15 && cbInstr >= 1)
+                {
+                    Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
+                    if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
+                    {
+                        /* Don't know exactly how to detect whether u3Seg is valid, currently
+                           only enabling it for Bulldozer and later with NRIP.  OS/2 broke on
+                           2384 Opterons when only checking NRIP. */
+                        bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+                        if (   fSupportsNextRipSave
+                            && pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
+                        {
+                            AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS || cbInstr > 1U + IoExitInfo.n.u1Rep,
+                                      ("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
+                            rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
+                                                            IoExitInfo.n.u3Seg, true /*fIoChecked*/);
+                        }
+                        else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
+                            rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
+                                                            X86_SREG_DS, true /*fIoChecked*/);
+                        else
+                            rcStrict = IEMExecOne(pVCpu);
+                        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
+                    }
+                    else
+                    {
+                        AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /*=0*/, ("%#x\n", IoExitInfo.n.u3Seg));
+                        rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
+                                                       true /*fIoChecked*/);
+                        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
+                    }
+                }
+                else
+                {
+                    AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
+                    rcStrict = IEMExecOne(pVCpu);
+                }
+            }
+            else
+            {
+                AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
+                rcStrict = IEMExecOne(pVCpu);
+            }
+            fUpdateRipAlready = true;
+        }
+        else
+        {
+            /* IN/OUT - I/O instruction. */
+            Assert(!IoExitInfo.n.u1Rep);
+
+            uint8_t const cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
+            if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
+            {
+                rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
+                if (    rcStrict == VINF_IOM_R3_IOPORT_WRITE
+                    && !pCtx->eflags.Bits.u1TF)
+                    rcStrict = EMRZSetPendingIoPortWrite(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue, pCtx->eax & uAndVal);
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
+            }
+            else
+            {
+                uint32_t u32Val = 0;
+                rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
+                if (IOM_SUCCESS(rcStrict))
+                {
+                    /* Save result of I/O IN instr. in AL/AX/EAX. */
+                    /** @todo r=bird: 32-bit op size should clear high bits of rax! */
+                    pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
+                }
+                else if (    rcStrict == VINF_IOM_R3_IOPORT_READ
+                         && !pCtx->eflags.Bits.u1TF)
+                    rcStrict = EMRZSetPendingIoPortRead(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue);
+
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
+            }
+        }
+
+        if (IOM_SUCCESS(rcStrict))
+        {
+            /* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
+            if (!fUpdateRipAlready)
+                pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
+
+            /*
+             * If any I/O breakpoints are armed, we need to check if one triggered
+             * and take appropriate action.
+             * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
+             */
+            /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
+             *  execution engines about whether hyper BPs and such are pending. */
+            HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
+            uint32_t const uDr7 = pCtx->dr[7];
+            if (RT_UNLIKELY(   (   (uDr7 & X86_DR7_ENABLED_MASK)
+                                && X86_DR7_ANY_RW_IO(uDr7)
+                                && (pCtx->cr4 & X86_CR4_DE))
+                            || DBGFBpIsHwIoArmed(pVM)))
+            {
+                /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
+                VMMRZCallRing3Disable(pVCpu);
+                HM_DISABLE_PREEMPT(pVCpu);
+
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
+                CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /*fDr6*/);
+
+                VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, &pVCpu->cpum.GstCtx, IoExitInfo.n.u16Port, cbValue);
+                if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
+                {
+                    /* Raise #DB. */
+                    pVmcb->guest.u64DR6 = pCtx->dr[6];
+                    pVmcb->guest.u64DR7 = pCtx->dr[7];
+                    pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
+                    hmR0SvmSetPendingXcptDB(pVCpu);
+                }
+                /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
+                   however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
+                else if (   rcStrict2 != VINF_SUCCESS
+                         && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
+                    rcStrict = rcStrict2;
+                AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
+
+                HM_RESTORE_PREEMPT();
+                VMMRZCallRing3Enable(pVCpu);
+            }
+
+            HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+        }
+
+#ifdef VBOX_STRICT
+        if (   rcStrict == VINF_IOM_R3_IOPORT_READ
+            || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
+            Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
+        else if (   rcStrict == VINF_IOM_R3_IOPORT_WRITE
+                 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
+                 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
+            Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
+        else
+        {
+            /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
+             *        statuses, that the VMM device and some others may return. See
+             *        IOM_SUCCESS() for guidance. */
+            AssertMsg(   RT_FAILURE(rcStrict)
+                      || rcStrict == VINF_SUCCESS
+                      || rcStrict == VINF_EM_RAW_EMULATE_INSTR
+                      || rcStrict == VINF_EM_DBG_BREAKPOINT
+                      || rcStrict == VINF_EM_RAW_GUEST_TRAP
+                      || rcStrict == VINF_EM_RAW_TO_R3
+                      || rcStrict == VINF_TRPM_XCPT_DISPATCHED
+                      || rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+        }
+#endif
+    }
+    else
+    {
+        /*
+         * Frequent exit or something needing probing.  Get state and call EMHistoryExec.
+         */
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+        STAM_COUNTER_INC(!IoExitInfo.n.u1Str
+                         ? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
+                         : IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
+        Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
+              IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
+
+        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+
+        Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    }
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+    HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
+
+    PVMCC      pVM  = pVCpu->CTX_SUFF(pVM);
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    Assert(pVM->hm.s.fNestedPaging);
+
+    /* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
+    PSVMVMCB pVmcb           = hmR0SvmGetCurrentVmcb(pVCpu);
+    RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
+    uint32_t u32ErrCode      = pVmcb->ctrl.u64ExitInfo1;    /* Note! High bits in EXITINFO1 may contain additional info and are
+                                                               thus intentionally not copied into u32ErrCode. */
+
+    Log4Func(("#NPF at CS:RIP=%04x:%#RX64 GCPhysFaultAddr=%RGp ErrCode=%#x \n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
+              u32ErrCode));
+
+    /*
+     * TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
+     */
+    if (   pVM->hm.s.fTprPatchingAllowed
+        && (GCPhysFaultAddr & PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
+        && (   !(u32ErrCode & X86_TRAP_PF_P)                                                             /* Not present */
+            || (u32ErrCode & (X86_TRAP_PF_P | X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P | X86_TRAP_PF_RSVD))  /* MMIO page. */
+        && !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
+        && !CPUMIsGuestInLongModeEx(pCtx)
+        && !CPUMGetGuestCPL(pVCpu)
+        && pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
+    {
+        RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
+        GCPhysApicBase &= PAGE_BASE_GC_MASK;
+
+        if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
+        {
+            /* Only attempt to patch the instruction once. */
+            PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
+            if (!pPatch)
+                return VINF_EM_HM_PATCH_TPR_INSTR;
+        }
+    }
+
+    /*
+     * Determine the nested paging mode.
+     */
+/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
+    PGMMODE const enmNestedPagingMode = PGMGetHostMode(pVM);
+
+    /*
+     * MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
+     */
+    Assert((u32ErrCode & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
+    if ((u32ErrCode & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD | X86_TRAP_PF_P))
+    {
+        /*
+         * If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
+         * injecting the original pending event would most likely cause the same MMIO #NPF.
+         */
+        if (pVCpu->hm.s.Event.fPending)
+        {
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
+            return VINF_EM_RAW_INJECT_TRPM_EVENT;
+        }
+
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
+        VBOXSTRICTRC rcStrict;
+        PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
+                                                                EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
+                                                                pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
+        if (!pExitRec)
+        {
+
+            rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
+                                                     u32ErrCode);
+
+            /*
+             * If we succeed, resume guest execution.
+             *
+             * If we fail in interpreting the instruction because we couldn't get the guest
+             * physical address of the page containing the instruction via the guest's page
+             * tables (we would invalidate the guest page in the host TLB), resume execution
+             * which would cause a guest page fault to let the guest handle this weird case.
+             *
+             * See @bugref{6043}.
+             */
+            if (   rcStrict == VINF_SUCCESS
+                || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
+                || rcStrict == VERR_PAGE_NOT_PRESENT)
+            {
+                /* Successfully handled MMIO operation. */
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
+                rcStrict = VINF_SUCCESS;
+            }
+        }
+        else
+        {
+            /*
+             * Frequent exit or something needing probing.  Get state and call EMHistoryExec.
+             */
+            Assert(pCtx == &pVCpu->cpum.GstCtx);
+            HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+            Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
+                  pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
+
+            rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+
+            Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+                  pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+                  VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+        }
+        return VBOXSTRICTRC_TODO(rcStrict);
+    }
+
+    /*
+     * Nested page-fault.
+     */
+    TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
+    int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
+    TRPMResetTrap(pVCpu);
+
+    Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%#RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
+
+    /*
+     * Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
+     */
+    if (   rc == VINF_SUCCESS
+        || rc == VERR_PAGE_TABLE_NOT_PRESENT
+        || rc == VERR_PAGE_NOT_PRESENT)
+    {
+        /* We've successfully synced our shadow page tables. */
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
+        rc = VINF_SUCCESS;
+    }
+
+    /*
+     * If delivering an event causes an #NPF (and not MMIO), we shall resolve the fault and
+     * re-inject the original event.
+     */
+    if (pVCpu->hm.s.Event.fPending)
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
+
+    return rc;
+}
+
+
+/**
+ * \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
+ * \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
+
+    /* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
+    PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    hmR0SvmClearIntWindowExiting(pVCpu, pVmcb);
+
+    /* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
+ * \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
+
+#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
+    Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
+#endif
+
+    /* Check if this task-switch occurred while delivering an event through the guest IDT. */
+    if (pVCpu->hm.s.Event.fPending)  /* Can happen with exceptions/NMI. See @bugref{8411}. */
+    {
+        /*
+         * AMD-V provides us with the exception which caused the TS; we collect
+         * the information in the call to hmR0SvmCheckExitDueToEventDelivery().
+         */
+        Log4Func(("TS occurred during event delivery\n"));
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
+        return VINF_EM_RAW_INJECT_TRPM_EVENT;
+    }
+
+    /** @todo Emulate task switch someday, currently just going back to ring-3 for
+     *        emulation. */
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
+    return VERR_EM_INTERPRETER;
+}
+
+
+/**
+ * \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (pVM->hm.s.fTprPatchingAllowed)
+    {
+        int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
+        if (rc != VERR_NOT_FOUND)
+        {
+            Log4Func(("hmEmulateSvmMovTpr returns %Rrc\n", rc));
+            return rc;
+        }
+    }
+
+    if (EMAreHypercallInstructionsEnabled(pVCpu))
+    {
+        unsigned cbInstr;
+        if (hmR0SvmSupportsNextRipSave(pVCpu))
+        {
+            PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+            cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        }
+        else
+        {
+            PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
+            int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
+            if (   rc == VINF_SUCCESS
+                && pDis->pCurInstr->uOpcode == OP_VMMCALL)
+                Assert(cbInstr > 0);
+            else
+                cbInstr = 0;
+        }
+
+        VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
+        if (RT_SUCCESS(rcStrict))
+        {
+            /* Only update the RIP if we're continuing guest execution and not in the case
+               of say VINF_GIM_R3_HYPERCALL. */
+            if (rcStrict == VINF_SUCCESS)
+                hmR0SvmAdvanceRip(pVCpu, cbInstr);
+
+            return VBOXSTRICTRC_VAL(rcStrict);
+        }
+        else
+            Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
+    }
+
+    hmR0SvmSetPendingXcptUD(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    unsigned cbInstr;
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (fSupportsNextRipSave)
+    {
+        PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+        cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+    }
+    else
+    {
+        PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
+        int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
+        if (   rc == VINF_SUCCESS
+            && pDis->pCurInstr->uOpcode == OP_PAUSE)
+            Assert(cbInstr > 0);
+        else
+            cbInstr = 0;
+    }
+
+    /** @todo The guest has likely hit a contended spinlock. We might want to
+     *        poke a schedule different guest VCPU. */
+    hmR0SvmAdvanceRip(pVCpu, cbInstr);
+    return VINF_EM_RAW_INTERRUPT;
+}
+
+
+/**
+ * \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
+ * \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
+    Assert(!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE));
+
+    Log4Func(("Raising IRQ 13 in response to #FERR\n"));
+    return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq */, 1 /* u8Level */, 0 /* uTagSrc */);
+}
+
+
+/**
+ * \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    /* Clear NMI blocking. */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
+
+    /* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
+    PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_IRET);
+
+    /* Deliver the pending NMI via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
+ * Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+    HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
+
+    /* See AMD spec. 15.12.15 "#PF (Page Fault)". */
+    PVMCC             pVM           = pVCpu->CTX_SUFF(pVM);
+    PCPUMCTX        pCtx          = &pVCpu->cpum.GstCtx;
+    PSVMVMCB        pVmcb         = hmR0SvmGetCurrentVmcb(pVCpu);
+    uint32_t        uErrCode      = pVmcb->ctrl.u64ExitInfo1;
+    uint64_t const  uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
+
+#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) || defined(HMSVM_ALWAYS_TRAP_PF)
+    if (pVM->hm.s.fNestedPaging)
+    {
+        pVCpu->hm.s.Event.fPending = false;     /* In case it's a contributory or vectoring #PF. */
+        if (   !pSvmTransient->fVectoringDoublePF
+            || CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+        {
+            /* A genuine guest #PF, reflect it to the guest. */
+            hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
+            Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
+                      uFaultAddress, uErrCode));
+        }
+        else
+        {
+            /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
+            hmR0SvmSetPendingXcptDF(pVCpu);
+            Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
+        }
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
+        return VINF_SUCCESS;
+    }
+#endif
+
+    Assert(!pVM->hm.s.fNestedPaging);
+
+    /*
+     * TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
+     */
+    if (   pVM->hm.s.fTprPatchingAllowed
+        && (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
+        && !(uErrCode & X86_TRAP_PF_P)                /* Not present. */
+        && !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
+        && !CPUMIsGuestInLongModeEx(pCtx)
+        && !CPUMGetGuestCPL(pVCpu)
+        && pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
+    {
+        RTGCPHYS GCPhysApicBase;
+        GCPhysApicBase  = APICGetBaseMsrNoCheck(pVCpu);
+        GCPhysApicBase &= PAGE_BASE_GC_MASK;
+
+        /* Check if the page at the fault-address is the APIC base. */
+        RTGCPHYS GCPhysPage;
+        int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, NULL /* pfFlags */, &GCPhysPage);
+        if (   rc2 == VINF_SUCCESS
+            && GCPhysPage == GCPhysApicBase)
+        {
+            /* Only attempt to patch the instruction once. */
+            PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
+            if (!pPatch)
+                return VINF_EM_HM_PATCH_TPR_INSTR;
+        }
+    }
+
+    Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
+              pCtx->rip, uErrCode, pCtx->cr3));
+
+    /*
+     * If it's a vectoring #PF, emulate injecting the original event injection as
+     * PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
+     * event injection that caused a #PF. See @bugref{6607}.
+     */
+    if (pSvmTransient->fVectoringPF)
+    {
+        Assert(pVCpu->hm.s.Event.fPending);
+        return VINF_EM_RAW_INJECT_TRPM_EVENT;
+    }
+
+    TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
+    int rc = PGMTrap0eHandler(pVCpu, uErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
+
+    Log4Func(("#PF: rc=%Rrc\n", rc));
+
+    if (rc == VINF_SUCCESS)
+    {
+        /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
+        TRPMResetTrap(pVCpu);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+        return rc;
+    }
+
+    if (rc == VINF_EM_RAW_GUEST_TRAP)
+    {
+        pVCpu->hm.s.Event.fPending = false;     /* In case it's a contributory or vectoring #PF. */
+
+        /*
+         * If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
+         * we should simply forward the #PF to the guest and is up to the nested-hypervisor to
+         * determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
+         */
+        if (  !pSvmTransient->fVectoringDoublePF
+            || CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+        {
+            /* It's a guest (or nested-guest) page fault and needs to be reflected. */
+            uErrCode = TRPMGetErrorCode(pVCpu);        /* The error code might have been changed. */
+            TRPMResetTrap(pVCpu);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+            /* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
+            if (   CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
+                && CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
+                return VBOXSTRICTRC_TODO(IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress));
+#endif
+
+            hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
+        }
+        else
+        {
+            /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
+            TRPMResetTrap(pVCpu);
+            hmR0SvmSetPendingXcptDF(pVCpu);
+            Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
+        }
+
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
+        return VINF_SUCCESS;
+    }
+
+    TRPMResetTrap(pVCpu);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
+    return rc;
+}
+
+
+/**
+ * \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
+ * Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
+
+    /* Paranoia; Ensure we cannot be called as a result of event delivery. */
+    PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
+    Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid);  NOREF(pVmcb);
+
+    int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
+    if (pVCpu->hm.s.fGIMTrapXcptUD)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+        uint8_t cbInstr = 0;
+        VBOXSTRICTRC rcStrict = GIMXcptUD(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            /* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
+            hmR0SvmAdvanceRip(pVCpu, cbInstr);
+            rc = VINF_SUCCESS;
+            HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
+        }
+        else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
+            rc = VINF_SUCCESS;
+        else if (rcStrict == VINF_GIM_R3_HYPERCALL)
+            rc = VINF_GIM_R3_HYPERCALL;
+        else
+            Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
+    }
+
+    /* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
+    if (RT_FAILURE(rc))
+    {
+        hmR0SvmSetPendingXcptUD(pVCpu);
+        rc = VINF_SUCCESS;
+    }
+
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
+    return rc;
+}
+
+
+/**
+ * \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
+ * Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
+
+    PCPUMCTX pCtx  = &pVCpu->cpum.GstCtx;
+    PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+
+    /* Paranoia; Ensure we cannot be called as a result of event delivery. */
+    Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
+
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
+
+    if (!(pCtx->cr0 & X86_CR0_NE))
+    {
+        PVMCC       pVM  = pVCpu->CTX_SUFF(pVM);
+        PDISSTATE pDis = &pVCpu->hm.s.DisState;
+        unsigned  cbInstr;
+        int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbInstr);
+        if (RT_SUCCESS(rc))
+        {
+            /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
+            rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq */, 1 /* u8Level */, 0 /* uTagSrc */);
+            if (RT_SUCCESS(rc))
+                hmR0SvmAdvanceRip(pVCpu, cbInstr);
+        }
+        else
+            Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
+        return rc;
+    }
+
+    hmR0SvmSetPendingXcptMF(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
+ * \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+    HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
+
+    if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
+    {
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
+        return VINF_EM_RAW_INJECT_TRPM_EVENT;
+    }
+
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
+
+    /*
+     * This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
+     * breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
+     * handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
+     */
+    PVMCC      pVM   = pVCpu->CTX_SUFF(pVM);
+    PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
+    PCPUMCTX pCtx  = &pVCpu->cpum.GstCtx;
+    int rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
+    if (rc == VINF_EM_RAW_GUEST_TRAP)
+    {
+        Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
+        if (CPUMIsHyperDebugStateActive(pVCpu))
+            CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) | pVmcb->guest.u64DR6);
+
+        /* Reflect the exception back to the guest. */
+        hmR0SvmSetPendingXcptDB(pVCpu);
+        rc = VINF_SUCCESS;
+    }
+
+    /*
+     * Update DR6.
+     */
+    if (CPUMIsHyperDebugStateActive(pVCpu))
+    {
+        Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
+        pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
+        pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
+    }
+    else
+    {
+        AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
+        Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
+    }
+
+    return rc;
+}
+
+
+/**
+ * \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
+ * Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
+
+    SVMEVENT Event;
+    Event.u          = 0;
+    Event.n.u1Valid  = 1;
+    Event.n.u3Type   = SVM_EVENT_EXCEPTION;
+    Event.n.u8Vector = X86_XCPT_AC;
+    Event.n.u1ErrorCodeValid = 1;
+    hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
+ * Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+    HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
+
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    int rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
+    if (rc == VINF_EM_RAW_GUEST_TRAP)
+    {
+        SVMEVENT Event;
+        Event.u          = 0;
+        Event.n.u1Valid  = 1;
+        Event.n.u3Type   = SVM_EVENT_EXCEPTION;
+        Event.n.u8Vector = X86_XCPT_BP;
+        hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
+        rc = VINF_SUCCESS;
+    }
+
+    Assert(rc == VINF_SUCCESS || rc == VINF_EM_DBG_BREAKPOINT);
+    return rc;
+}
+
+
+/**
+ * Hacks its way around the lovely mesa driver's backdoor accesses.
+ *
+ * @sa hmR0VmxHandleMesaDrvGp
+ */
+static int hmR0SvmHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
+{
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS  | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_GPRS_MASK);
+    Log(("hmR0SvmHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n",
+         pVmcb->guest.CS.u16Sel, pVmcb->guest.u64RIP, pCtx->rcx, pCtx->rbx));
+    RT_NOREF(pCtx, pVmcb);
+
+    /* For now we'll just skip the instruction. */
+    hmR0SvmAdvanceRip(pVCpu, 1);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
+ * backdoor logging w/o checking what it is running inside.
+ *
+ * This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
+ * backdoor port and magic numbers loaded in registers.
+ *
+ * @returns true if it is, false if it isn't.
+ * @sa      hmR0VmxIsMesaDrvGp
+ */
+DECLINLINE(bool) hmR0SvmIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
+{
+    /* Check magic and port. */
+    Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RCX)));
+    /*Log8(("hmR0SvmIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax, pCtx->rdx));*/
+    if (pCtx->dx != UINT32_C(0x5658))
+        return false;
+    if ((pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax) != UINT32_C(0x564d5868))
+        return false;
+
+    /* Check that it is #GP(0). */
+    if (pVmcb->ctrl.u64ExitInfo1 != 0)
+        return false;
+
+    /* Flat ring-3 CS. */
+    /*Log8(("hmR0SvmIsMesaDrvGp: u8CPL=%d base=%RX64\n", pVmcb->guest.u8CPL, pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base));*/
+    if (pVmcb->guest.u8CPL != 3)
+        return false;
+    if ((pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base) != 0)
+        return false;
+
+    /* 0xed:  IN eAX,dx */
+    if (pVmcb->ctrl.cbInstrFetched < 1) /* unlikely, it turns out. */
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS  | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_GPRS_MASK
+                                        | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);
+        uint8_t abInstr[1];
+        int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
+        /*Log8(("hmR0SvmIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0])); */
+        if (RT_FAILURE(rc))
+            return false;
+        if (abInstr[0] != 0xed)
+            return false;
+    }
+    else
+    {
+        /*Log8(("hmR0SvmIsMesaDrvGp: %#x\n", pVmcb->ctrl.abInstr));*/
+        if (pVmcb->ctrl.abInstr[0] != 0xed)
+            return false;
+    }
+    return true;
+}
+
+
+/**
+ * \#VMEXIT handler for general protection faults (SVM_EXIT_XCPT_BP).
+ * Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitXcptGP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
+
+    PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
+
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    if (   !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
+        || !hmR0SvmIsMesaDrvGp(pVCpu, pCtx, pVmcb))
+    {
+        SVMEVENT Event;
+        Event.u                  = 0;
+        Event.n.u1Valid          = 1;
+        Event.n.u3Type           = SVM_EVENT_EXCEPTION;
+        Event.n.u8Vector         = X86_XCPT_GP;
+        Event.n.u1ErrorCodeValid = 1;
+        Event.n.u32ErrorCode     = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
+        hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
+        return VINF_SUCCESS;
+    }
+    return hmR0SvmHandleMesaDrvGp(pVCpu, pCtx, pVmcb);
+}
+
+
+#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) || defined(VBOX_WITH_NESTED_HWVIRT_SVM)
+/**
+ * \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
+
+    PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    uint8_t const  uVector  = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
+    uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
+    Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
+    Assert(uVector <= X86_XCPT_LAST);
+    Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
+
+    SVMEVENT Event;
+    Event.u          = 0;
+    Event.n.u1Valid  = 1;
+    Event.n.u3Type   = SVM_EVENT_EXCEPTION;
+    Event.n.u8Vector = uVector;
+    switch (uVector)
+    {
+        /* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
+        case X86_XCPT_PF:   AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
+        case X86_XCPT_DF:
+        case X86_XCPT_TS:
+        case X86_XCPT_NP:
+        case X86_XCPT_SS:
+        case X86_XCPT_GP:
+        case X86_XCPT_AC:
+        {
+            Event.n.u1ErrorCodeValid = 1;
+            Event.n.u32ErrorCode     = uErrCode;
+            break;
+        }
+    }
+
+#ifdef VBOX_WITH_STATISTICS
+    switch (uVector)
+    {
+        case X86_XCPT_DE:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);     break;
+        case X86_XCPT_DB:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);     break;
+        case X86_XCPT_BP:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);     break;
+        case X86_XCPT_OF:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF);     break;
+        case X86_XCPT_BR:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR);     break;
+        case X86_XCPT_UD:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);     break;
+        case X86_XCPT_NM:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF);     break;
+        case X86_XCPT_DF:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF);     break;
+        case X86_XCPT_TS:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);     break;
+        case X86_XCPT_NP:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);     break;
+        case X86_XCPT_SS:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);     break;
+        case X86_XCPT_GP:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);     break;
+        case X86_XCPT_PF:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);     break;
+        case X86_XCPT_MF:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);     break;
+        case X86_XCPT_AC:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);     break;
+        case X86_XCPT_XF:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);     break;
+        default:
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
+            break;
+    }
+#endif
+
+    hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
+    return VINF_SUCCESS;
+}
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+/**
+ * \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    Assert(pVmcb);
+    Assert(!pVmcb->ctrl.IntCtrl.n.u1VGifEnable);
+
+    VBOXSTRICTRC   rcStrict;
+    bool const     fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
+    if (fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | fImport);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | fImport);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (rcStrict == VINF_SUCCESS)
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    /*
+     * When VGIF is not used we always intercept STGI instructions. When VGIF is used,
+     * we only intercept STGI when events are pending for GIF to become 1.
+     */
+    PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
+        hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_STGI);
+
+    VBOXSTRICTRC   rcStrict;
+    bool const     fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
+    if (fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | fImport);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | fImport);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (rcStrict == VINF_SUCCESS)
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    Assert(pVmcb);
+    Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
+
+    VBOXSTRICTRC   rcStrict;
+    bool const     fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    uint64_t const fImport = CPUMCTX_EXTRN_FS   | CPUMCTX_EXTRN_GS   | CPUMCTX_EXTRN_KERNEL_GS_BASE
+                           | CPUMCTX_EXTRN_TR   | CPUMCTX_EXTRN_LDTR | CPUMCTX_EXTRN_SYSCALL_MSRS
+                           | CPUMCTX_EXTRN_SYSENTER_MSRS;
+    if (fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | fImport);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | fImport);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (rcStrict == VINF_SUCCESS)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS              | HM_CHANGED_GUEST_GS
+                                                 | HM_CHANGED_GUEST_TR              | HM_CHANGED_GUEST_LDTR
+                                                 | HM_CHANGED_GUEST_KERNEL_GS_BASE  | HM_CHANGED_GUEST_SYSCALL_MSRS
+                                                 | HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
+    Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
+
+    VBOXSTRICTRC rcStrict;
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+
+    VBOXSTRICTRC rcStrict;
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    if (fSupportsNextRipSave)
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
+        PCSVMVMCB     pVmcb   = hmR0SvmGetCurrentVmcb(pVCpu);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
+    }
+    else
+    {
+        HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    /* We shall import the entire state here, just in case we enter and continue execution of
+       the nested-guest with hardware-assisted SVM in ring-0, we would be switching VMCBs and
+       could lose lose part of CPU state. */
+    HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
+
+    VBOXSTRICTRC rcStrict;
+    bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
+    STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
+    if (fSupportsNextRipSave)
+    {
+        PCSVMVMCB     pVmcb   = hmR0SvmGetCurrentVmcb(pVCpu);
+        uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
+        rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
+    }
+    else
+    {
+        /* We use IEMExecOneBypassEx() here as it supresses attempt to continue emulating any
+           instruction(s) when interrupt inhibition is set as part of emulating the VMRUN
+           instruction itself, see @bugref{7243#c126} */
+        rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), NULL /* pcbWritten */);
+    }
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
+
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = VINF_SVM_VMRUN;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
+ * Unconditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
+
+    if (pVCpu->hm.s.Event.fPending)
+    {
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
+        return VINF_EM_RAW_INJECT_TRPM_EVENT;
+    }
+
+    hmR0SvmSetPendingXcptDB(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
+ * Conditional \#VMEXIT.
+ */
+HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
+{
+    HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
+    HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
+
+    SVMEVENT Event;
+    Event.u          = 0;
+    Event.n.u1Valid  = 1;
+    Event.n.u3Type   = SVM_EVENT_EXCEPTION;
+    Event.n.u8Vector = X86_XCPT_BP;
+    hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
+    return VINF_SUCCESS;
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMR0/HMSVMR0.h b/src/VBox/VMM/VMMR0/HMSVMR0.h
new file mode 100644
index 00000000..3ba777ea
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/HMSVMR0.h
@@ -0,0 +1,82 @@
+/* $Id: HMSVMR0.h $ */
+/** @file
+ * HM SVM (AMD-V) - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_VMMR0_HMSVMR0_h
+#define VMM_INCLUDED_SRC_VMMR0_HMSVMR0_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/hm_svm.h>
+
+RT_C_DECLS_BEGIN
+
+/** @defgroup grp_svm_int  Internal
+ * @ingroup grp_svm
+ * @internal
+ * @{
+ */
+
+#ifdef IN_RING0
+
+VMMR0DECL(int)          SVMR0GlobalInit(void);
+VMMR0DECL(void)         SVMR0GlobalTerm(void);
+VMMR0DECL(int)          SVMR0Enter(PVMCPUCC pVCpu);
+VMMR0DECL(void)         SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit);
+VMMR0DECL(int)          SVMR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation);
+VMMR0DECL(int)          SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvPageCpu, RTHCPHYS HCPhysCpuPage,
+                                       bool fEnabledBySystem, PCSUPHWVIRTMSRS pHwvirtMsrs);
+VMMR0DECL(int)          SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys);
+VMMR0DECL(int)          SVMR0InitVM(PVMCC pVM);
+VMMR0DECL(int)          SVMR0TermVM(PVMCC pVM);
+VMMR0DECL(int)          SVMR0SetupVM(PVMCC pVM);
+VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu);
+VMMR0DECL(int)          SVMR0ExportHostState(PVMCPUCC pVCpu);
+VMMR0DECL(int)          SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat);
+VMMR0DECL(int)          SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt);
+
+/**
+ * Prepares for and executes VMRUN (64-bit register context).
+ *
+ * @returns VBox status code.
+ * @param   pVMCBHostPhys   Physical address of host VMCB.
+ * @param   pVMCBPhys       Physical address of the VMCB.
+ * @param   pCtx            Pointer to the guest CPU context.
+ * @param   pVM             The cross context VM structure. (Not used.)
+ * @param   pVCpu           The cross context virtual CPU structure. (Not used.)
+ */
+DECLASM(int) SVMR0VMRun(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVMCC pVM, PVMCPUCC pVCpu);
+
+/**
+ * Executes INVLPGA.
+ *
+ * @param   pPageGC         Virtual page to invalidate.
+ * @param   u32ASID         Tagged TLB id.
+ */
+DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t u32ASID);
+
+#endif /* IN_RING0 */
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_VMMR0_HMSVMR0_h */
+
diff --git a/src/VBox/VMM/VMMR0/HMVMXR0.cpp b/src/VBox/VMM/VMMR0/HMVMXR0.cpp
new file mode 100644
index 00000000..4ceb3e36
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/HMVMXR0.cpp
@@ -0,0 +1,17380 @@
+/* $Id: HMVMXR0.cpp $ */
+/** @file
+ * HM VMX (Intel VT-x) - Host Context Ring-0.
+ */
+
+/*
+ * Copyright (C) 2012-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_HM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <iprt/x86.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/thread.h>
+#include <iprt/mem.h>
+#include <iprt/mp.h>
+
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/apic.h>
+#include "HMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/hmvmxinline.h>
+#include "HMVMXR0.h"
+#include "dtrace/VBoxVMM.h"
+
+#ifdef DEBUG_ramshankar
+# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
+# define HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
+# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
+# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
+# define HMVMX_ALWAYS_CLEAN_TRANSIENT
+# define HMVMX_ALWAYS_CHECK_GUEST_STATE
+# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
+# define HMVMX_ALWAYS_TRAP_PF
+# define HMVMX_ALWAYS_FLUSH_TLB
+# define HMVMX_ALWAYS_SWAP_EFER
+#endif
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** Use the function table. */
+#define HMVMX_USE_FUNCTION_TABLE
+
+/** Determine which tagged-TLB flush handler to use. */
+#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID             0
+#define HMVMX_FLUSH_TAGGED_TLB_EPT                  1
+#define HMVMX_FLUSH_TAGGED_TLB_VPID                 2
+#define HMVMX_FLUSH_TAGGED_TLB_NONE                 3
+
+/**
+ * Flags to skip redundant reads of some common VMCS fields that are not part of
+ * the guest-CPU or VCPU state but are needed while handling VM-exits.
+ */
+#define HMVMX_READ_IDT_VECTORING_INFO               RT_BIT_32(0)
+#define HMVMX_READ_IDT_VECTORING_ERROR_CODE         RT_BIT_32(1)
+#define HMVMX_READ_EXIT_QUALIFICATION               RT_BIT_32(2)
+#define HMVMX_READ_EXIT_INSTR_LEN                   RT_BIT_32(3)
+#define HMVMX_READ_EXIT_INTERRUPTION_INFO           RT_BIT_32(4)
+#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE     RT_BIT_32(5)
+#define HMVMX_READ_EXIT_INSTR_INFO                  RT_BIT_32(6)
+#define HMVMX_READ_GUEST_LINEAR_ADDR                RT_BIT_32(7)
+#define HMVMX_READ_GUEST_PHYSICAL_ADDR              RT_BIT_32(8)
+#define HMVMX_READ_GUEST_PENDING_DBG_XCPTS          RT_BIT_32(9)
+
+/** All the VMCS fields required for processing of exception/NMI VM-exits. */
+#define HMVMX_READ_XCPT_INFO         (  HMVMX_READ_EXIT_INTERRUPTION_INFO        \
+                                      | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE  \
+                                      | HMVMX_READ_EXIT_INSTR_LEN                \
+                                      | HMVMX_READ_IDT_VECTORING_INFO            \
+                                      | HMVMX_READ_IDT_VECTORING_ERROR_CODE)
+
+/** Assert that all the given fields have been read from the VMCS. */
+#ifdef VBOX_STRICT
+# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
+        do { \
+            uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
+            Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
+        } while (0)
+#else
+# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
+#endif
+
+/**
+ * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
+ * guest using hardware-assisted VMX.
+ *
+ * This excludes state like GPRs (other than RSP) which are always are
+ * swapped and restored across the world-switch and also registers like EFER,
+ * MSR which cannot be modified by the guest without causing a VM-exit.
+ */
+#define HMVMX_CPUMCTX_EXTRN_ALL      (  CPUMCTX_EXTRN_RIP             \
+                                      | CPUMCTX_EXTRN_RFLAGS          \
+                                      | CPUMCTX_EXTRN_RSP             \
+                                      | CPUMCTX_EXTRN_SREG_MASK       \
+                                      | CPUMCTX_EXTRN_TABLE_MASK      \
+                                      | CPUMCTX_EXTRN_KERNEL_GS_BASE  \
+                                      | CPUMCTX_EXTRN_SYSCALL_MSRS    \
+                                      | CPUMCTX_EXTRN_SYSENTER_MSRS   \
+                                      | CPUMCTX_EXTRN_TSC_AUX         \
+                                      | CPUMCTX_EXTRN_OTHER_MSRS      \
+                                      | CPUMCTX_EXTRN_CR0             \
+                                      | CPUMCTX_EXTRN_CR3             \
+                                      | CPUMCTX_EXTRN_CR4             \
+                                      | CPUMCTX_EXTRN_DR7             \
+                                      | CPUMCTX_EXTRN_HWVIRT          \
+                                      | CPUMCTX_EXTRN_HM_VMX_MASK)
+
+/**
+ * Exception bitmap mask for real-mode guests (real-on-v86).
+ *
+ * We need to intercept all exceptions manually except:
+ * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
+ *   due to bugs in Intel CPUs.
+ * - \#PF need not be intercepted even in real-mode if we have nested paging
+ * support.
+ */
+#define HMVMX_REAL_MODE_XCPT_MASK    (  RT_BIT(X86_XCPT_DE)  /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI)   \
+                                      | RT_BIT(X86_XCPT_BP)             | RT_BIT(X86_XCPT_OF)    | RT_BIT(X86_XCPT_BR)    \
+                                      | RT_BIT(X86_XCPT_UD)             | RT_BIT(X86_XCPT_NM)    | RT_BIT(X86_XCPT_DF)    \
+                                      | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS)    | RT_BIT(X86_XCPT_NP)    \
+                                      | RT_BIT(X86_XCPT_SS)             | RT_BIT(X86_XCPT_GP)   /* RT_BIT(X86_XCPT_PF) */ \
+                                      | RT_BIT(X86_XCPT_MF)  /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC)    \
+                                      | RT_BIT(X86_XCPT_XF))
+
+/** Maximum VM-instruction error number. */
+#define HMVMX_INSTR_ERROR_MAX        28
+
+/** Profiling macro. */
+#ifdef HM_PROFILE_EXIT_DISPATCH
+# define HMVMX_START_EXIT_DISPATCH_PROF()           STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
+# define HMVMX_STOP_EXIT_DISPATCH_PROF()            STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
+#else
+# define HMVMX_START_EXIT_DISPATCH_PROF()           do { } while (0)
+# define HMVMX_STOP_EXIT_DISPATCH_PROF()            do { } while (0)
+#endif
+
+/** Assert that preemption is disabled or covered by thread-context hooks. */
+#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu)          Assert(   VMMR0ThreadCtxHookIsEnabled((a_pVCpu))   \
+                                                           || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
+
+/** Assert that we haven't migrated CPUs when thread-context hooks are not
+ *  used. */
+#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu)              AssertMsg(   VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
+                                                              || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
+                                                              ("Illegal migration! Entered on CPU %u Current %u\n", \
+                                                              (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
+
+/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
+ *  context. */
+#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz)  AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
+                                                              ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
+                                                              (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
+
+/** Log the VM-exit reason with an easily visible marker to identify it in a
+ *  potential sea of logging data. */
+#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
+    do { \
+        Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
+             HMGetVmxExitName(a_uExitReason))); \
+    } while (0) \
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * VMX per-VCPU transient state.
+ *
+ * A state structure for holding miscellaneous information across
+ * VMX non-root operation and restored after the transition.
+ *
+ * Note: The members are ordered and aligned such that the most
+ * frequently used ones (in the guest execution loop) fall within
+ * the first cache line.
+ */
+typedef struct VMXTRANSIENT
+{
+   /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
+   uint32_t            fVmcsFieldsRead;
+   /** The guest's TPR value used for TPR shadowing. */
+   uint8_t             u8GuestTpr;
+   uint8_t             abAlignment0[3];
+
+   /** Whether the VM-exit was caused by a page-fault during delivery of an
+    *  external interrupt or NMI. */
+   bool                fVectoringPF;
+   /** Whether the VM-exit was caused by a page-fault during delivery of a
+    *  contributory exception or a page-fault. */
+   bool                fVectoringDoublePF;
+   /** Whether the VM-entry failed or not. */
+   bool                fVMEntryFailed;
+   /** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
+    *  area after VM-exit. */
+   bool                fRemoveTscAuxMsr;
+   /** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
+   bool                fUpdatedTscOffsettingAndPreemptTimer;
+   /** Whether we are currently executing a nested-guest. */
+   bool                fIsNestedGuest;
+   /** Whether the guest debug state was active at the time of VM-exit. */
+   bool                fWasGuestDebugStateActive;
+   /** Whether the hyper debug state was active at the time of VM-exit. */
+   bool                fWasHyperDebugStateActive;
+
+   /** The basic VM-exit reason. */
+   uint32_t            uExitReason;
+   /** The VM-exit interruption error code. */
+   uint32_t            uExitIntErrorCode;
+
+   /** The host's rflags/eflags. */
+   RTCCUINTREG         fEFlags;
+
+   /** The VM-exit exit code qualification. */
+   uint64_t            uExitQual;
+
+   /** The VMCS info. object. */
+   PVMXVMCSINFO        pVmcsInfo;
+
+   /** The VM-exit interruption-information field. */
+   uint32_t            uExitIntInfo;
+   /** The VM-exit instruction-length field. */
+   uint32_t            cbExitInstr;
+
+   /** The VM-exit instruction-information field. */
+   VMXEXITINSTRINFO    ExitInstrInfo;
+   /** IDT-vectoring information field. */
+   uint32_t            uIdtVectoringInfo;
+
+   /** IDT-vectoring error code. */
+   uint32_t            uIdtVectoringErrorCode;
+   uint32_t            u32Alignment0;
+
+   /** The Guest-linear address. */
+   uint64_t            uGuestLinearAddr;
+
+   /** The Guest-physical address. */
+   uint64_t            uGuestPhysicalAddr;
+
+   /** The Guest pending-debug exceptions. */
+   uint64_t            uGuestPendingDbgXcpts;
+
+   /** The VM-entry interruption-information field. */
+   uint32_t            uEntryIntInfo;
+   /** The VM-entry exception error code field. */
+   uint32_t            uEntryXcptErrorCode;
+
+   /** The VM-entry instruction length field. */
+   uint32_t            cbEntryInstr;
+} VMXTRANSIENT;
+AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
+AssertCompileMemberAlignment(VMXTRANSIENT, fVmcsFieldsRead,        8);
+AssertCompileMemberAlignment(VMXTRANSIENT, fVectoringPF,           8);
+AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason,            8);
+AssertCompileMemberAlignment(VMXTRANSIENT, fEFlags,                8);
+AssertCompileMemberAlignment(VMXTRANSIENT, uExitQual,              8);
+AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo,              8);
+AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo,           8);
+AssertCompileMemberAlignment(VMXTRANSIENT, ExitInstrInfo,          8);
+AssertCompileMemberAlignment(VMXTRANSIENT, uIdtVectoringErrorCode, 8);
+AssertCompileMemberAlignment(VMXTRANSIENT, uGuestLinearAddr,       8);
+AssertCompileMemberAlignment(VMXTRANSIENT, uGuestPhysicalAddr,     8);
+AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo,          8);
+AssertCompileMemberAlignment(VMXTRANSIENT, cbEntryInstr,           8);
+/** Pointer to VMX transient state. */
+typedef VMXTRANSIENT *PVMXTRANSIENT;
+/** Pointer to a const VMX transient state. */
+typedef const VMXTRANSIENT *PCVMXTRANSIENT;
+
+/**
+ * VMX page allocation information.
+ */
+typedef struct
+{
+    uint32_t    fValid;       /**< Whether to allocate this page (e.g, based on a CPU feature). */
+    uint32_t    uPadding0;    /**< Padding to ensure array of these structs are aligned to a multiple of 8. */
+    PRTHCPHYS   pHCPhys;      /**< Where to store the host-physical address of the allocation. */
+    PRTR0PTR    ppVirt;       /**< Where to store the host-virtual address of the allocation. */
+} VMXPAGEALLOCINFO;
+/** Pointer to VMX page-allocation info. */
+typedef VMXPAGEALLOCINFO *PVMXPAGEALLOCINFO;
+/** Pointer to a const VMX page-allocation info. */
+typedef const VMXPAGEALLOCINFO *PCVMXPAGEALLOCINFO;
+AssertCompileSizeAlignment(VMXPAGEALLOCINFO, 8);
+
+/**
+ * Memory operand read or write access.
+ */
+typedef enum VMXMEMACCESS
+{
+    VMXMEMACCESS_READ  = 0,
+    VMXMEMACCESS_WRITE = 1
+} VMXMEMACCESS;
+
+/**
+ * VMX VM-exit handler.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+#ifndef HMVMX_USE_FUNCTION_TABLE
+typedef VBOXSTRICTRC               FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
+#else
+typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
+/** Pointer to VM-exit handler. */
+typedef FNVMXEXITHANDLER          *PFNVMXEXITHANDLER;
+#endif
+
+/**
+ * VMX VM-exit handler, non-strict status code.
+ *
+ * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
+ *
+ * @returns VBox status code, no informational status code returned.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
+ *          use of that status code will be replaced with VINF_EM_SOMETHING
+ *          later when switching over to IEM.
+ */
+#ifndef HMVMX_USE_FUNCTION_TABLE
+typedef int                        FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
+#else
+typedef FNVMXEXITHANDLER           FNVMXEXITHANDLERNSRC;
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+#ifndef HMVMX_USE_FUNCTION_TABLE
+DECLINLINE(VBOXSTRICTRC)           hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
+# define HMVMX_EXIT_DECL           DECLINLINE(VBOXSTRICTRC)
+# define HMVMX_EXIT_NSRC_DECL      DECLINLINE(int)
+#else
+# define HMVMX_EXIT_DECL           static DECLCALLBACK(VBOXSTRICTRC)
+# define HMVMX_EXIT_NSRC_DECL      HMVMX_EXIT_DECL
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+DECLINLINE(VBOXSTRICTRC)           hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
+#endif
+
+static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
+
+/** @name VM-exit handler prototypes.
+ * @{
+ */
+static FNVMXEXITHANDLER            hmR0VmxExitXcptOrNmi;
+static FNVMXEXITHANDLER            hmR0VmxExitExtInt;
+static FNVMXEXITHANDLER            hmR0VmxExitTripleFault;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitIntWindow;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitNmiWindow;
+static FNVMXEXITHANDLER            hmR0VmxExitTaskSwitch;
+static FNVMXEXITHANDLER            hmR0VmxExitCpuid;
+static FNVMXEXITHANDLER            hmR0VmxExitGetsec;
+static FNVMXEXITHANDLER            hmR0VmxExitHlt;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitInvd;
+static FNVMXEXITHANDLER            hmR0VmxExitInvlpg;
+static FNVMXEXITHANDLER            hmR0VmxExitRdpmc;
+static FNVMXEXITHANDLER            hmR0VmxExitVmcall;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+static FNVMXEXITHANDLER            hmR0VmxExitVmclear;
+static FNVMXEXITHANDLER            hmR0VmxExitVmlaunch;
+static FNVMXEXITHANDLER            hmR0VmxExitVmptrld;
+static FNVMXEXITHANDLER            hmR0VmxExitVmptrst;
+static FNVMXEXITHANDLER            hmR0VmxExitVmread;
+static FNVMXEXITHANDLER            hmR0VmxExitVmresume;
+static FNVMXEXITHANDLER            hmR0VmxExitVmwrite;
+static FNVMXEXITHANDLER            hmR0VmxExitVmxoff;
+static FNVMXEXITHANDLER            hmR0VmxExitVmxon;
+static FNVMXEXITHANDLER            hmR0VmxExitInvvpid;
+#endif
+static FNVMXEXITHANDLER            hmR0VmxExitRdtsc;
+static FNVMXEXITHANDLER            hmR0VmxExitMovCRx;
+static FNVMXEXITHANDLER            hmR0VmxExitMovDRx;
+static FNVMXEXITHANDLER            hmR0VmxExitIoInstr;
+static FNVMXEXITHANDLER            hmR0VmxExitRdmsr;
+static FNVMXEXITHANDLER            hmR0VmxExitWrmsr;
+static FNVMXEXITHANDLER            hmR0VmxExitMwait;
+static FNVMXEXITHANDLER            hmR0VmxExitMtf;
+static FNVMXEXITHANDLER            hmR0VmxExitMonitor;
+static FNVMXEXITHANDLER            hmR0VmxExitPause;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitTprBelowThreshold;
+static FNVMXEXITHANDLER            hmR0VmxExitApicAccess;
+static FNVMXEXITHANDLER            hmR0VmxExitEptViolation;
+static FNVMXEXITHANDLER            hmR0VmxExitEptMisconfig;
+static FNVMXEXITHANDLER            hmR0VmxExitRdtscp;
+static FNVMXEXITHANDLER            hmR0VmxExitPreemptTimer;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitWbinvd;
+static FNVMXEXITHANDLER            hmR0VmxExitXsetbv;
+static FNVMXEXITHANDLER            hmR0VmxExitInvpcid;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitSetPendingXcptUD;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitErrInvalidGuestState;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitErrUnexpected;
+/** @} */
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/** @name Nested-guest VM-exit handler prototypes.
+ * @{
+ */
+static FNVMXEXITHANDLER            hmR0VmxExitXcptOrNmiNested;
+static FNVMXEXITHANDLER            hmR0VmxExitTripleFaultNested;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitIntWindowNested;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitNmiWindowNested;
+static FNVMXEXITHANDLER            hmR0VmxExitTaskSwitchNested;
+static FNVMXEXITHANDLER            hmR0VmxExitHltNested;
+static FNVMXEXITHANDLER            hmR0VmxExitInvlpgNested;
+static FNVMXEXITHANDLER            hmR0VmxExitRdpmcNested;
+static FNVMXEXITHANDLER            hmR0VmxExitVmreadVmwriteNested;
+static FNVMXEXITHANDLER            hmR0VmxExitRdtscNested;
+static FNVMXEXITHANDLER            hmR0VmxExitMovCRxNested;
+static FNVMXEXITHANDLER            hmR0VmxExitMovDRxNested;
+static FNVMXEXITHANDLER            hmR0VmxExitIoInstrNested;
+static FNVMXEXITHANDLER            hmR0VmxExitRdmsrNested;
+static FNVMXEXITHANDLER            hmR0VmxExitWrmsrNested;
+static FNVMXEXITHANDLER            hmR0VmxExitMwaitNested;
+static FNVMXEXITHANDLER            hmR0VmxExitMtfNested;
+static FNVMXEXITHANDLER            hmR0VmxExitMonitorNested;
+static FNVMXEXITHANDLER            hmR0VmxExitPauseNested;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitTprBelowThresholdNested;
+static FNVMXEXITHANDLER            hmR0VmxExitApicAccessNested;
+static FNVMXEXITHANDLER            hmR0VmxExitApicWriteNested;
+static FNVMXEXITHANDLER            hmR0VmxExitVirtEoiNested;
+static FNVMXEXITHANDLER            hmR0VmxExitRdtscpNested;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitWbinvdNested;
+static FNVMXEXITHANDLER            hmR0VmxExitInvpcidNested;
+static FNVMXEXITHANDLERNSRC        hmR0VmxExitErrInvalidGuestStateNested;
+static FNVMXEXITHANDLER            hmR0VmxExitInstrNested;
+static FNVMXEXITHANDLER            hmR0VmxExitInstrWithInfoNested;
+/** @} */
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Array of all VMCS fields.
+ * Any fields added to the VT-x spec. should be added here.
+ *
+ * Currently only used to derive shadow VMCS fields for hardware-assisted execution
+ * of nested-guests.
+ */
+static const uint32_t g_aVmcsFields[] =
+{
+    /* 16-bit control fields. */
+    VMX_VMCS16_VPID,
+    VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
+    VMX_VMCS16_EPTP_INDEX,
+
+    /* 16-bit guest-state fields. */
+    VMX_VMCS16_GUEST_ES_SEL,
+    VMX_VMCS16_GUEST_CS_SEL,
+    VMX_VMCS16_GUEST_SS_SEL,
+    VMX_VMCS16_GUEST_DS_SEL,
+    VMX_VMCS16_GUEST_FS_SEL,
+    VMX_VMCS16_GUEST_GS_SEL,
+    VMX_VMCS16_GUEST_LDTR_SEL,
+    VMX_VMCS16_GUEST_TR_SEL,
+    VMX_VMCS16_GUEST_INTR_STATUS,
+    VMX_VMCS16_GUEST_PML_INDEX,
+
+    /* 16-bits host-state fields. */
+    VMX_VMCS16_HOST_ES_SEL,
+    VMX_VMCS16_HOST_CS_SEL,
+    VMX_VMCS16_HOST_SS_SEL,
+    VMX_VMCS16_HOST_DS_SEL,
+    VMX_VMCS16_HOST_FS_SEL,
+    VMX_VMCS16_HOST_GS_SEL,
+    VMX_VMCS16_HOST_TR_SEL,
+
+    /* 64-bit control fields. */
+    VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
+    VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
+    VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
+    VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
+    VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
+    VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
+    VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
+    VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
+    VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
+    VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
+    VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
+    VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
+    VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
+    VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
+    VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
+    VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
+    VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
+    VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
+    VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
+    VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
+    VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
+    VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
+    VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
+    VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
+    VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
+    VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
+    VMX_VMCS64_CTRL_EPTP_FULL,
+    VMX_VMCS64_CTRL_EPTP_HIGH,
+    VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
+    VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
+    VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
+    VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
+    VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
+    VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
+    VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
+    VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
+    VMX_VMCS64_CTRL_EPTP_LIST_FULL,
+    VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
+    VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
+    VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
+    VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
+    VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
+    VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
+    VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
+    VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
+    VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
+    VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
+    VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
+    VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
+    VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
+
+    /* 64-bit read-only data fields. */
+    VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
+    VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
+
+    /* 64-bit guest-state fields. */
+    VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
+    VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
+    VMX_VMCS64_GUEST_DEBUGCTL_FULL,
+    VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
+    VMX_VMCS64_GUEST_PAT_FULL,
+    VMX_VMCS64_GUEST_PAT_HIGH,
+    VMX_VMCS64_GUEST_EFER_FULL,
+    VMX_VMCS64_GUEST_EFER_HIGH,
+    VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
+    VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
+    VMX_VMCS64_GUEST_PDPTE0_FULL,
+    VMX_VMCS64_GUEST_PDPTE0_HIGH,
+    VMX_VMCS64_GUEST_PDPTE1_FULL,
+    VMX_VMCS64_GUEST_PDPTE1_HIGH,
+    VMX_VMCS64_GUEST_PDPTE2_FULL,
+    VMX_VMCS64_GUEST_PDPTE2_HIGH,
+    VMX_VMCS64_GUEST_PDPTE3_FULL,
+    VMX_VMCS64_GUEST_PDPTE3_HIGH,
+    VMX_VMCS64_GUEST_BNDCFGS_FULL,
+    VMX_VMCS64_GUEST_BNDCFGS_HIGH,
+
+    /* 64-bit host-state fields. */
+    VMX_VMCS64_HOST_PAT_FULL,
+    VMX_VMCS64_HOST_PAT_HIGH,
+    VMX_VMCS64_HOST_EFER_FULL,
+    VMX_VMCS64_HOST_EFER_HIGH,
+    VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
+    VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
+
+    /* 32-bit control fields. */
+    VMX_VMCS32_CTRL_PIN_EXEC,
+    VMX_VMCS32_CTRL_PROC_EXEC,
+    VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
+    VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
+    VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
+    VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
+    VMX_VMCS32_CTRL_EXIT,
+    VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
+    VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
+    VMX_VMCS32_CTRL_ENTRY,
+    VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
+    VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
+    VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
+    VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
+    VMX_VMCS32_CTRL_TPR_THRESHOLD,
+    VMX_VMCS32_CTRL_PROC_EXEC2,
+    VMX_VMCS32_CTRL_PLE_GAP,
+    VMX_VMCS32_CTRL_PLE_WINDOW,
+
+    /* 32-bits read-only fields. */
+    VMX_VMCS32_RO_VM_INSTR_ERROR,
+    VMX_VMCS32_RO_EXIT_REASON,
+    VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
+    VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
+    VMX_VMCS32_RO_IDT_VECTORING_INFO,
+    VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
+    VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
+    VMX_VMCS32_RO_EXIT_INSTR_INFO,
+
+    /* 32-bit guest-state fields. */
+    VMX_VMCS32_GUEST_ES_LIMIT,
+    VMX_VMCS32_GUEST_CS_LIMIT,
+    VMX_VMCS32_GUEST_SS_LIMIT,
+    VMX_VMCS32_GUEST_DS_LIMIT,
+    VMX_VMCS32_GUEST_FS_LIMIT,
+    VMX_VMCS32_GUEST_GS_LIMIT,
+    VMX_VMCS32_GUEST_LDTR_LIMIT,
+    VMX_VMCS32_GUEST_TR_LIMIT,
+    VMX_VMCS32_GUEST_GDTR_LIMIT,
+    VMX_VMCS32_GUEST_IDTR_LIMIT,
+    VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_INT_STATE,
+    VMX_VMCS32_GUEST_ACTIVITY_STATE,
+    VMX_VMCS32_GUEST_SMBASE,
+    VMX_VMCS32_GUEST_SYSENTER_CS,
+    VMX_VMCS32_PREEMPT_TIMER_VALUE,
+
+    /* 32-bit host-state fields. */
+    VMX_VMCS32_HOST_SYSENTER_CS,
+
+    /* Natural-width control fields. */
+    VMX_VMCS_CTRL_CR0_MASK,
+    VMX_VMCS_CTRL_CR4_MASK,
+    VMX_VMCS_CTRL_CR0_READ_SHADOW,
+    VMX_VMCS_CTRL_CR4_READ_SHADOW,
+    VMX_VMCS_CTRL_CR3_TARGET_VAL0,
+    VMX_VMCS_CTRL_CR3_TARGET_VAL1,
+    VMX_VMCS_CTRL_CR3_TARGET_VAL2,
+    VMX_VMCS_CTRL_CR3_TARGET_VAL3,
+
+    /* Natural-width read-only data fields. */
+    VMX_VMCS_RO_EXIT_QUALIFICATION,
+    VMX_VMCS_RO_IO_RCX,
+    VMX_VMCS_RO_IO_RSI,
+    VMX_VMCS_RO_IO_RDI,
+    VMX_VMCS_RO_IO_RIP,
+    VMX_VMCS_RO_GUEST_LINEAR_ADDR,
+
+    /* Natural-width guest-state field */
+    VMX_VMCS_GUEST_CR0,
+    VMX_VMCS_GUEST_CR3,
+    VMX_VMCS_GUEST_CR4,
+    VMX_VMCS_GUEST_ES_BASE,
+    VMX_VMCS_GUEST_CS_BASE,
+    VMX_VMCS_GUEST_SS_BASE,
+    VMX_VMCS_GUEST_DS_BASE,
+    VMX_VMCS_GUEST_FS_BASE,
+    VMX_VMCS_GUEST_GS_BASE,
+    VMX_VMCS_GUEST_LDTR_BASE,
+    VMX_VMCS_GUEST_TR_BASE,
+    VMX_VMCS_GUEST_GDTR_BASE,
+    VMX_VMCS_GUEST_IDTR_BASE,
+    VMX_VMCS_GUEST_DR7,
+    VMX_VMCS_GUEST_RSP,
+    VMX_VMCS_GUEST_RIP,
+    VMX_VMCS_GUEST_RFLAGS,
+    VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
+    VMX_VMCS_GUEST_SYSENTER_ESP,
+    VMX_VMCS_GUEST_SYSENTER_EIP,
+
+    /* Natural-width host-state fields */
+    VMX_VMCS_HOST_CR0,
+    VMX_VMCS_HOST_CR3,
+    VMX_VMCS_HOST_CR4,
+    VMX_VMCS_HOST_FS_BASE,
+    VMX_VMCS_HOST_GS_BASE,
+    VMX_VMCS_HOST_TR_BASE,
+    VMX_VMCS_HOST_GDTR_BASE,
+    VMX_VMCS_HOST_IDTR_BASE,
+    VMX_VMCS_HOST_SYSENTER_ESP,
+    VMX_VMCS_HOST_SYSENTER_EIP,
+    VMX_VMCS_HOST_RSP,
+    VMX_VMCS_HOST_RIP
+};
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+static const uint32_t g_aVmcsSegBase[] =
+{
+    VMX_VMCS_GUEST_ES_BASE,
+    VMX_VMCS_GUEST_CS_BASE,
+    VMX_VMCS_GUEST_SS_BASE,
+    VMX_VMCS_GUEST_DS_BASE,
+    VMX_VMCS_GUEST_FS_BASE,
+    VMX_VMCS_GUEST_GS_BASE
+};
+static const uint32_t g_aVmcsSegSel[] =
+{
+    VMX_VMCS16_GUEST_ES_SEL,
+    VMX_VMCS16_GUEST_CS_SEL,
+    VMX_VMCS16_GUEST_SS_SEL,
+    VMX_VMCS16_GUEST_DS_SEL,
+    VMX_VMCS16_GUEST_FS_SEL,
+    VMX_VMCS16_GUEST_GS_SEL
+};
+static const uint32_t g_aVmcsSegLimit[] =
+{
+    VMX_VMCS32_GUEST_ES_LIMIT,
+    VMX_VMCS32_GUEST_CS_LIMIT,
+    VMX_VMCS32_GUEST_SS_LIMIT,
+    VMX_VMCS32_GUEST_DS_LIMIT,
+    VMX_VMCS32_GUEST_FS_LIMIT,
+    VMX_VMCS32_GUEST_GS_LIMIT
+};
+static const uint32_t g_aVmcsSegAttr[] =
+{
+    VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
+};
+AssertCompile(RT_ELEMENTS(g_aVmcsSegSel)   == X86_SREG_COUNT);
+AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
+AssertCompile(RT_ELEMENTS(g_aVmcsSegBase)  == X86_SREG_COUNT);
+AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr)  == X86_SREG_COUNT);
+
+#ifdef HMVMX_USE_FUNCTION_TABLE
+/**
+ * VMX_EXIT dispatch table.
+ */
+static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
+{
+    /*  0  VMX_EXIT_XCPT_OR_NMI             */  hmR0VmxExitXcptOrNmi,
+    /*  1  VMX_EXIT_EXT_INT                 */  hmR0VmxExitExtInt,
+    /*  2  VMX_EXIT_TRIPLE_FAULT            */  hmR0VmxExitTripleFault,
+    /*  3  VMX_EXIT_INIT_SIGNAL             */  hmR0VmxExitErrUnexpected,
+    /*  4  VMX_EXIT_SIPI                    */  hmR0VmxExitErrUnexpected,
+    /*  5  VMX_EXIT_IO_SMI                  */  hmR0VmxExitErrUnexpected,
+    /*  6  VMX_EXIT_SMI                     */  hmR0VmxExitErrUnexpected,
+    /*  7  VMX_EXIT_INT_WINDOW              */  hmR0VmxExitIntWindow,
+    /*  8  VMX_EXIT_NMI_WINDOW              */  hmR0VmxExitNmiWindow,
+    /*  9  VMX_EXIT_TASK_SWITCH             */  hmR0VmxExitTaskSwitch,
+    /* 10  VMX_EXIT_CPUID                   */  hmR0VmxExitCpuid,
+    /* 11  VMX_EXIT_GETSEC                  */  hmR0VmxExitGetsec,
+    /* 12  VMX_EXIT_HLT                     */  hmR0VmxExitHlt,
+    /* 13  VMX_EXIT_INVD                    */  hmR0VmxExitInvd,
+    /* 14  VMX_EXIT_INVLPG                  */  hmR0VmxExitInvlpg,
+    /* 15  VMX_EXIT_RDPMC                   */  hmR0VmxExitRdpmc,
+    /* 16  VMX_EXIT_RDTSC                   */  hmR0VmxExitRdtsc,
+    /* 17  VMX_EXIT_RSM                     */  hmR0VmxExitErrUnexpected,
+    /* 18  VMX_EXIT_VMCALL                  */  hmR0VmxExitVmcall,
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /* 19  VMX_EXIT_VMCLEAR                 */  hmR0VmxExitVmclear,
+    /* 20  VMX_EXIT_VMLAUNCH                */  hmR0VmxExitVmlaunch,
+    /* 21  VMX_EXIT_VMPTRLD                 */  hmR0VmxExitVmptrld,
+    /* 22  VMX_EXIT_VMPTRST                 */  hmR0VmxExitVmptrst,
+    /* 23  VMX_EXIT_VMREAD                  */  hmR0VmxExitVmread,
+    /* 24  VMX_EXIT_VMRESUME                */  hmR0VmxExitVmresume,
+    /* 25  VMX_EXIT_VMWRITE                 */  hmR0VmxExitVmwrite,
+    /* 26  VMX_EXIT_VMXOFF                  */  hmR0VmxExitVmxoff,
+    /* 27  VMX_EXIT_VMXON                   */  hmR0VmxExitVmxon,
+#else
+    /* 19  VMX_EXIT_VMCLEAR                 */  hmR0VmxExitSetPendingXcptUD,
+    /* 20  VMX_EXIT_VMLAUNCH                */  hmR0VmxExitSetPendingXcptUD,
+    /* 21  VMX_EXIT_VMPTRLD                 */  hmR0VmxExitSetPendingXcptUD,
+    /* 22  VMX_EXIT_VMPTRST                 */  hmR0VmxExitSetPendingXcptUD,
+    /* 23  VMX_EXIT_VMREAD                  */  hmR0VmxExitSetPendingXcptUD,
+    /* 24  VMX_EXIT_VMRESUME                */  hmR0VmxExitSetPendingXcptUD,
+    /* 25  VMX_EXIT_VMWRITE                 */  hmR0VmxExitSetPendingXcptUD,
+    /* 26  VMX_EXIT_VMXOFF                  */  hmR0VmxExitSetPendingXcptUD,
+    /* 27  VMX_EXIT_VMXON                   */  hmR0VmxExitSetPendingXcptUD,
+#endif
+    /* 28  VMX_EXIT_MOV_CRX                 */  hmR0VmxExitMovCRx,
+    /* 29  VMX_EXIT_MOV_DRX                 */  hmR0VmxExitMovDRx,
+    /* 30  VMX_EXIT_IO_INSTR                */  hmR0VmxExitIoInstr,
+    /* 31  VMX_EXIT_RDMSR                   */  hmR0VmxExitRdmsr,
+    /* 32  VMX_EXIT_WRMSR                   */  hmR0VmxExitWrmsr,
+    /* 33  VMX_EXIT_ERR_INVALID_GUEST_STATE */  hmR0VmxExitErrInvalidGuestState,
+    /* 34  VMX_EXIT_ERR_MSR_LOAD            */  hmR0VmxExitErrUnexpected,
+    /* 35  UNDEFINED                        */  hmR0VmxExitErrUnexpected,
+    /* 36  VMX_EXIT_MWAIT                   */  hmR0VmxExitMwait,
+    /* 37  VMX_EXIT_MTF                     */  hmR0VmxExitMtf,
+    /* 38  UNDEFINED                        */  hmR0VmxExitErrUnexpected,
+    /* 39  VMX_EXIT_MONITOR                 */  hmR0VmxExitMonitor,
+    /* 40  VMX_EXIT_PAUSE                   */  hmR0VmxExitPause,
+    /* 41  VMX_EXIT_ERR_MACHINE_CHECK       */  hmR0VmxExitErrUnexpected,
+    /* 42  UNDEFINED                        */  hmR0VmxExitErrUnexpected,
+    /* 43  VMX_EXIT_TPR_BELOW_THRESHOLD     */  hmR0VmxExitTprBelowThreshold,
+    /* 44  VMX_EXIT_APIC_ACCESS             */  hmR0VmxExitApicAccess,
+    /* 45  VMX_EXIT_VIRTUALIZED_EOI         */  hmR0VmxExitErrUnexpected,
+    /* 46  VMX_EXIT_GDTR_IDTR_ACCESS        */  hmR0VmxExitErrUnexpected,
+    /* 47  VMX_EXIT_LDTR_TR_ACCESS          */  hmR0VmxExitErrUnexpected,
+    /* 48  VMX_EXIT_EPT_VIOLATION           */  hmR0VmxExitEptViolation,
+    /* 49  VMX_EXIT_EPT_MISCONFIG           */  hmR0VmxExitEptMisconfig,
+    /* 50  VMX_EXIT_INVEPT                  */  hmR0VmxExitSetPendingXcptUD,
+    /* 51  VMX_EXIT_RDTSCP                  */  hmR0VmxExitRdtscp,
+    /* 52  VMX_EXIT_PREEMPT_TIMER           */  hmR0VmxExitPreemptTimer,
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /* 53  VMX_EXIT_INVVPID                 */  hmR0VmxExitInvvpid,
+#else
+    /* 53  VMX_EXIT_INVVPID                 */  hmR0VmxExitSetPendingXcptUD,
+#endif
+    /* 54  VMX_EXIT_WBINVD                  */  hmR0VmxExitWbinvd,
+    /* 55  VMX_EXIT_XSETBV                  */  hmR0VmxExitXsetbv,
+    /* 56  VMX_EXIT_APIC_WRITE              */  hmR0VmxExitErrUnexpected,
+    /* 57  VMX_EXIT_RDRAND                  */  hmR0VmxExitErrUnexpected,
+    /* 58  VMX_EXIT_INVPCID                 */  hmR0VmxExitInvpcid,
+    /* 59  VMX_EXIT_VMFUNC                  */  hmR0VmxExitErrUnexpected,
+    /* 60  VMX_EXIT_ENCLS                   */  hmR0VmxExitErrUnexpected,
+    /* 61  VMX_EXIT_RDSEED                  */  hmR0VmxExitErrUnexpected,
+    /* 62  VMX_EXIT_PML_FULL                */  hmR0VmxExitErrUnexpected,
+    /* 63  VMX_EXIT_XSAVES                  */  hmR0VmxExitErrUnexpected,
+    /* 64  VMX_EXIT_XRSTORS                 */  hmR0VmxExitErrUnexpected,
+    /* 65  UNDEFINED                        */  hmR0VmxExitErrUnexpected,
+    /* 66  VMX_EXIT_SPP_EVENT               */  hmR0VmxExitErrUnexpected,
+    /* 67  VMX_EXIT_UMWAIT                  */  hmR0VmxExitErrUnexpected,
+    /* 68  VMX_EXIT_TPAUSE                  */  hmR0VmxExitErrUnexpected,
+};
+#endif /* HMVMX_USE_FUNCTION_TABLE */
+
+#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
+static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
+{
+    /*  0 */ "(Not Used)",
+    /*  1 */ "VMCALL executed in VMX root operation.",
+    /*  2 */ "VMCLEAR with invalid physical address.",
+    /*  3 */ "VMCLEAR with VMXON pointer.",
+    /*  4 */ "VMLAUNCH with non-clear VMCS.",
+    /*  5 */ "VMRESUME with non-launched VMCS.",
+    /*  6 */ "VMRESUME after VMXOFF",
+    /*  7 */ "VM-entry with invalid control fields.",
+    /*  8 */ "VM-entry with invalid host state fields.",
+    /*  9 */ "VMPTRLD with invalid physical address.",
+    /* 10 */ "VMPTRLD with VMXON pointer.",
+    /* 11 */ "VMPTRLD with incorrect revision identifier.",
+    /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
+    /* 13 */ "VMWRITE to read-only VMCS component.",
+    /* 14 */ "(Not Used)",
+    /* 15 */ "VMXON executed in VMX root operation.",
+    /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
+    /* 17 */ "VM-entry with non-launched executing VMCS.",
+    /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
+    /* 19 */ "VMCALL with non-clear VMCS.",
+    /* 20 */ "VMCALL with invalid VM-exit control fields.",
+    /* 21 */ "(Not Used)",
+    /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
+    /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
+    /* 24 */ "VMCALL with invalid SMM-monitor features.",
+    /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
+    /* 26 */ "VM-entry with events blocked by MOV SS.",
+    /* 27 */ "(Not Used)",
+    /* 28 */ "Invalid operand to INVEPT/INVVPID."
+};
+#endif /* VBOX_STRICT && LOG_ENABLED */
+
+
+/**
+ * Checks if the given MSR is part of the lastbranch-from-IP MSR stack.
+ * @returns @c true if it's part of LBR stack, @c false otherwise.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   idMsr       The MSR.
+ * @param   pidxMsr     Where to store the index of the MSR in the LBR MSR array.
+ *                      Optional, can be NULL.
+ *
+ * @remarks Must only be called when LBR is enabled.
+ */
+DECL_FORCE_INLINE(bool) hmR0VmxIsLbrBranchFromMsr(PCVM pVM, uint32_t idMsr, uint32_t *pidxMsr)
+{
+    Assert(pVM->hm.s.vmx.fLbr);
+    Assert(pVM->hm.s.vmx.idLbrFromIpMsrFirst);
+    uint32_t const cLbrStack = pVM->hm.s.vmx.idLbrFromIpMsrLast - pVM->hm.s.vmx.idLbrFromIpMsrFirst + 1;
+    uint32_t const idxMsr    = idMsr - pVM->hm.s.vmx.idLbrFromIpMsrFirst;
+    if (idxMsr < cLbrStack)
+    {
+        if (pidxMsr)
+            *pidxMsr = idxMsr;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Checks if the given MSR is part of the lastbranch-to-IP MSR stack.
+ * @returns @c true if it's part of LBR stack, @c false otherwise.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   idMsr       The MSR.
+ * @param   pidxMsr     Where to store the index of the MSR in the LBR MSR array.
+ *                      Optional, can be NULL.
+ *
+ * @remarks Must only be called when LBR is enabled and when lastbranch-to-IP MSRs
+ *          are supported by the CPU (see hmR0VmxSetupLbrMsrRange).
+ */
+DECL_FORCE_INLINE(bool) hmR0VmxIsLbrBranchToMsr(PCVM pVM, uint32_t idMsr, uint32_t *pidxMsr)
+{
+    Assert(pVM->hm.s.vmx.fLbr);
+    if (pVM->hm.s.vmx.idLbrToIpMsrFirst)
+    {
+        uint32_t const cLbrStack = pVM->hm.s.vmx.idLbrToIpMsrLast - pVM->hm.s.vmx.idLbrToIpMsrFirst + 1;
+        uint32_t const idxMsr    = idMsr - pVM->hm.s.vmx.idLbrToIpMsrFirst;
+        if (idxMsr < cLbrStack)
+        {
+            if (pidxMsr)
+                *pidxMsr = idxMsr;
+            return true;
+        }
+    }
+    return false;
+}
+
+
+/**
+ * Gets the CR0 guest/host mask.
+ *
+ * These bits typically does not change through the lifetime of a VM. Any bit set in
+ * this mask is owned by the host/hypervisor and would cause a VM-exit when modified
+ * by the guest.
+ *
+ * @returns The CR0 guest/host mask.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static uint64_t hmR0VmxGetFixedCr0Mask(PCVMCPUCC pVCpu)
+{
+    /*
+     * Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
+     * to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
+     *
+     * Furthermore, modifications to any bits that are reserved/unspecified currently
+     * by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
+     * when future CPUs specify and use currently reserved/unspecified bits.
+     */
+    /** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
+     *        enmGuestMode to be in-sync with the current mode. See @bugref{6398}
+     *        and @bugref{6944}. */
+    PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    return (  X86_CR0_PE
+            | X86_CR0_NE
+            | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
+            | X86_CR0_PG
+            | VMX_EXIT_HOST_CR0_IGNORE_MASK);
+}
+
+
+/**
+ * Gets the CR4 guest/host mask.
+ *
+ * These bits typically does not change through the lifetime of a VM. Any bit set in
+ * this mask is owned by the host/hypervisor and would cause a VM-exit when modified
+ * by the guest.
+ *
+ * @returns The CR4 guest/host mask.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static uint64_t hmR0VmxGetFixedCr4Mask(PCVMCPUCC pVCpu)
+{
+    /*
+     * We construct a mask of all CR4 bits that the guest can modify without causing
+     * a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
+     * a VM-exit when the guest attempts to modify them when executing using
+     * hardware-assisted VMX.
+     *
+     * When a feature is not exposed to the guest (and may be present on the host),
+     * we want to intercept guest modifications to the bit so we can emulate proper
+     * behavior (e.g., #GP).
+     *
+     * Furthermore, only modifications to those bits that don't require immediate
+     * emulation is allowed. For e.g., PCIDE is excluded because the behavior
+     * depends on CR3 which might not always be the guest value while executing
+     * using hardware-assisted VMX.
+     */
+    PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    bool const fFsGsBase    = pVM->cpum.ro.GuestFeatures.fFsGsBase;
+    bool const fXSaveRstor  = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
+    bool const fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
+
+    /*
+     * Paranoia.
+     * Ensure features exposed to the guest are present on the host.
+     */
+    Assert(!fFsGsBase    || pVM->cpum.ro.HostFeatures.fFsGsBase);
+    Assert(!fXSaveRstor  || pVM->cpum.ro.HostFeatures.fXSaveRstor);
+    Assert(!fFxSaveRstor || pVM->cpum.ro.HostFeatures.fFxSaveRstor);
+
+    uint64_t const fGstMask = (  X86_CR4_PVI
+                               | X86_CR4_TSD
+                               | X86_CR4_DE
+                               | X86_CR4_MCE
+                               | X86_CR4_PCE
+                               | X86_CR4_OSXMMEEXCPT
+                               | (fFsGsBase    ? X86_CR4_FSGSBASE : 0)
+                               | (fXSaveRstor  ? X86_CR4_OSXSAVE  : 0)
+                               | (fFxSaveRstor ? X86_CR4_OSFXSR   : 0));
+    return ~fGstMask;
+}
+
+
+/**
+ * Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
+ * area.
+ *
+ * @returns @c true if it's different, @c false otherwise.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
+{
+    return RT_BOOL(   pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
+                   && pVmcsInfo->pvGuestMsrStore);
+}
+
+
+/**
+ * Sets the given Processor-based VM-execution controls.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   uProcCtls       The Processor-based VM-execution controls to set.
+ */
+static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
+{
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
+    {
+        pVmcsInfo->u32ProcCtls |= uProcCtls;
+        int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+        AssertRC(rc);
+    }
+}
+
+
+/**
+ * Removes the given Processor-based VM-execution controls.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   uProcCtls       The Processor-based VM-execution controls to remove.
+ *
+ * @remarks When executing a nested-guest, this will not remove any of the specified
+ *          controls if the nested hypervisor has set any one of them.
+ */
+static void hmR0VmxRemoveProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
+{
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    if (pVmcsInfo->u32ProcCtls & uProcCtls)
+    {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
+                               ? true
+                               : !CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uProcCtls);
+#else
+        NOREF(pVCpu);
+        bool const fRemoveCtls = true;
+#endif
+        if (fRemoveCtls)
+        {
+            pVmcsInfo->u32ProcCtls &= ~uProcCtls;
+            int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+            AssertRC(rc);
+        }
+    }
+}
+
+
+/**
+ * Sets the TSC offset for the current VMCS.
+ *
+ * @param   uTscOffset  The TSC offset to set.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static void hmR0VmxSetTscOffsetVmcs(PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
+{
+    if (pVmcsInfo->u64TscOffset != uTscOffset)
+    {
+        int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
+        AssertRC(rc);
+        pVmcsInfo->u64TscOffset = uTscOffset;
+    }
+}
+
+
+/**
+ * Adds one or more exceptions to the exception bitmap and commits it to the current
+ * VMCS.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   uXcptMask       The exception(s) to add.
+ */
+static void hmR0VmxAddXcptInterceptMask(PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
+{
+    PVMXVMCSINFO pVmcsInfo   = pVmxTransient->pVmcsInfo;
+    uint32_t     uXcptBitmap = pVmcsInfo->u32XcptBitmap;
+    if ((uXcptBitmap & uXcptMask) != uXcptMask)
+    {
+        uXcptBitmap |= uXcptMask;
+        int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
+        AssertRC(rc);
+        pVmcsInfo->u32XcptBitmap = uXcptBitmap;
+    }
+}
+
+
+/**
+ * Adds an exception to the exception bitmap and commits it to the current VMCS.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   uXcpt           The exception to add.
+ */
+static void hmR0VmxAddXcptIntercept(PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
+{
+    Assert(uXcpt <= X86_XCPT_LAST);
+    hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
+}
+
+
+/**
+ * Remove one or more exceptions from the exception bitmap and commits it to the
+ * current VMCS.
+ *
+ * This takes care of not removing the exception intercept if a nested-guest
+ * requires the exception to be intercepted.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   uXcptMask       The exception(s) to remove.
+ */
+static int hmR0VmxRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
+{
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
+    if (u32XcptBitmap & uXcptMask)
+    {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        if (!pVmxTransient->fIsNestedGuest)
+        { /* likely */ }
+        else
+        {
+            PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+            uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
+        }
+#endif
+#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
+        uXcptMask &= ~(  RT_BIT(X86_XCPT_BP)
+                       | RT_BIT(X86_XCPT_DE)
+                       | RT_BIT(X86_XCPT_NM)
+                       | RT_BIT(X86_XCPT_TS)
+                       | RT_BIT(X86_XCPT_UD)
+                       | RT_BIT(X86_XCPT_NP)
+                       | RT_BIT(X86_XCPT_SS)
+                       | RT_BIT(X86_XCPT_GP)
+                       | RT_BIT(X86_XCPT_PF)
+                       | RT_BIT(X86_XCPT_MF));
+#elif defined(HMVMX_ALWAYS_TRAP_PF)
+        uXcptMask &= ~RT_BIT(X86_XCPT_PF);
+#endif
+        if (uXcptMask)
+        {
+            /* Validate we are not removing any essential exception intercepts. */
+            Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || !(uXcptMask & RT_BIT(X86_XCPT_PF)));
+            NOREF(pVCpu);
+            Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
+            Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
+
+            /* Remove it from the exception bitmap. */
+            u32XcptBitmap &= ~uXcptMask;
+
+            /* Commit and update the cache if necessary. */
+            if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
+            {
+                int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
+                AssertRC(rc);
+                pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
+            }
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Remove an exceptions from the exception bitmap and commits it to the current
+ * VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   uXcpt           The exception to remove.
+ */
+static int hmR0VmxRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
+{
+    return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
+}
+
+
+/**
+ * Loads the VMCS specified by the VMCS info. object.
+ *
+ * @returns VBox status code.
+ * @param   pVmcsInfo       The VMCS info. object.
+ *
+ * @remarks Can be called with interrupts disabled.
+ */
+static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
+{
+    Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
+    if (RT_SUCCESS(rc))
+        pVmcsInfo->fVmcsState |= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
+    return rc;
+}
+
+
+/**
+ * Clears the VMCS specified by the VMCS info. object.
+ *
+ * @returns VBox status code.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Can be called with interrupts disabled.
+ */
+static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
+{
+    Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
+    if (RT_SUCCESS(rc))
+        pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
+    return rc;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Loads the shadow VMCS specified by the VMCS info. object.
+ *
+ * @returns VBox status code.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Can be called with interrupts disabled.
+ */
+static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
+
+    int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
+    if (RT_SUCCESS(rc))
+        pVmcsInfo->fShadowVmcsState |= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
+    return rc;
+}
+
+
+/**
+ * Clears the shadow VMCS specified by the VMCS info. object.
+ *
+ * @returns VBox status code.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Can be called with interrupts disabled.
+ */
+static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
+
+    int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
+    if (RT_SUCCESS(rc))
+        pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
+    return rc;
+}
+
+
+/**
+ * Switches from and to the specified VMCSes.
+ *
+ * @returns VBox status code.
+ * @param   pVmcsInfoFrom   The VMCS info. object we are switching from.
+ * @param   pVmcsInfoTo     The VMCS info. object we are switching to.
+ *
+ * @remarks Called with interrupts disabled.
+ */
+static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
+{
+    /*
+     * Clear the VMCS we are switching out if it has not already been cleared.
+     * This will sync any CPU internal data back to the VMCS.
+     */
+    if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
+    {
+        int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * The shadow VMCS, if any, would not be active at this point since we
+             * would have cleared it while importing the virtual hardware-virtualization
+             * state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
+             * clear the shadow VMCS here, just assert for safety.
+             */
+            Assert(!pVmcsInfoFrom->pvShadowVmcs || pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
+        }
+        else
+            return rc;
+    }
+
+    /*
+     * Clear the VMCS we are switching to if it has not already been cleared.
+     * This will initialize the VMCS launch state to "clear" required for loading it.
+     *
+     * See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
+     */
+    if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
+    {
+        int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            return rc;
+    }
+
+    /*
+     * Finally, load the VMCS we are switching to.
+     */
+    return hmR0VmxLoadVmcs(pVmcsInfoTo);
+}
+
+
+/**
+ * Switches between the guest VMCS and the nested-guest VMCS as specified by the
+ * caller.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   fSwitchToNstGstVmcs     Whether to switch to the nested-guest VMCS (pass
+ *                                  true) or guest VMCS (pass false).
+ */
+static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
+{
+    /* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
+    HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
+
+    PVMXVMCSINFO pVmcsInfoFrom;
+    PVMXVMCSINFO pVmcsInfoTo;
+    if (fSwitchToNstGstVmcs)
+    {
+        pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
+        pVmcsInfoTo   = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
+    }
+    else
+    {
+        pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
+        pVmcsInfoTo   = &pVCpu->hm.s.vmx.VmcsInfo;
+    }
+
+    /*
+     * Disable interrupts to prevent being preempted while we switch the current VMCS as the
+     * preemption hook code path acquires the current VMCS.
+     */
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+
+    int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
+    if (RT_SUCCESS(rc))
+    {
+        pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
+
+        /*
+         * If we are switching to a VMCS that was executed on a different host CPU or was
+         * never executed before, flag that we need to export the host state before executing
+         * guest/nested-guest code using hardware-assisted VMX.
+         *
+         * This could probably be done in a preemptible context since the preemption hook
+         * will flag the necessary change in host context. However, since preemption is
+         * already disabled and to avoid making assumptions about host specific code in
+         * RTMpCpuId when called with preemption enabled, we'll do this while preemption is
+         * disabled.
+         */
+        if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
+        { /* likely */ }
+        else
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
+
+        ASMSetFlags(fEFlags);
+
+        /*
+         * We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
+         * flag that we need to update the host MSR values there. Even if we decide in the
+         * future to share the VM-exit MSR-store area page between the guest and nested-guest,
+         * if its content differs, we would have to update the host MSRs anyway.
+         */
+        pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
+    }
+    else
+        ASMSetFlags(fEFlags);
+    return rc;
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/**
+ * Updates the VM's last error record.
+ *
+ * If there was a VMX instruction error, reads the error data from the VMCS and
+ * updates VCPU's last error record as well.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ *                  Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
+ *                  VERR_VMX_INVALID_VMCS_FIELD.
+ * @param   rc      The error code.
+ */
+static void hmR0VmxUpdateErrorRecord(PVMCPUCC pVCpu, int rc)
+{
+    if (   rc == VERR_VMX_INVALID_VMCS_FIELD
+        || rc == VERR_VMX_UNABLE_TO_START_VM)
+    {
+        AssertPtrReturnVoid(pVCpu);
+        VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
+    }
+    pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
+}
+
+
+#ifdef VBOX_STRICT
+/**
+ * Reads the VM-entry interruption-information field from the VMCS into the VMX
+ * transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
+    AssertRC(rc);
+}
+
+
+/**
+ * Reads the VM-entry exception error code field from the VMCS into
+ * the VMX transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
+    AssertRC(rc);
+}
+
+
+/**
+ * Reads the VM-entry exception error code field from the VMCS into
+ * the VMX transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
+    AssertRC(rc);
+}
+#endif /* VBOX_STRICT */
+
+
+/**
+ * Reads the VM-exit interruption-information field from the VMCS into the VMX
+ * transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
+    {
+        int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
+        AssertRC(rc);
+        pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
+    }
+}
+
+
+/**
+ * Reads the VM-exit interruption error code from the VMCS into the VMX
+ * transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
+    {
+        int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
+        AssertRC(rc);
+        pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
+    }
+}
+
+
+/**
+ * Reads the VM-exit instruction length field from the VMCS into the VMX
+ * transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
+    {
+        int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
+        AssertRC(rc);
+        pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
+    }
+}
+
+
+/**
+ * Reads the VM-exit instruction-information field from the VMCS into
+ * the VMX transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
+    {
+        int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
+        AssertRC(rc);
+        pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
+    }
+}
+
+
+/**
+ * Reads the Exit Qualification from the VMCS into the VMX transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) hmR0VmxReadExitQualVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
+    {
+        int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
+        AssertRC(rc);
+        pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
+    }
+}
+
+
+/**
+ * Reads the Guest-linear address from the VMCS into the VMX transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) hmR0VmxReadGuestLinearAddrVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
+    {
+        int rc = VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
+        AssertRC(rc);
+        pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_GUEST_LINEAR_ADDR;
+    }
+}
+
+
+/**
+ * Reads the Guest-physical address from the VMCS into the VMX transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) hmR0VmxReadGuestPhysicalAddrVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
+    {
+        int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
+        AssertRC(rc);
+        pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_GUEST_PHYSICAL_ADDR;
+    }
+}
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Reads the Guest pending-debug exceptions from the VMCS into the VMX transient
+ * structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) hmR0VmxReadGuestPendingDbgXctps(PVMXTRANSIENT pVmxTransient)
+{
+    if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
+    {
+        int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
+        AssertRC(rc);
+        pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_GUEST_PENDING_DBG_XCPTS;
+    }
+}
+#endif
+
+/**
+ * Reads the IDT-vectoring information field from the VMCS into the VMX
+ * transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+DECLINLINE(void) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
+    {
+        int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
+        AssertRC(rc);
+        pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
+    }
+}
+
+
+/**
+ * Reads the IDT-vectoring error code from the VMCS into the VMX
+ * transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
+    {
+        int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
+        AssertRC(rc);
+        pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
+    }
+}
+
+#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
+/**
+ * Reads all relevant read-only VMCS fields into the VMX transient structure.
+ *
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+static void hmR0VmxReadAllRoFieldsVmcs(PVMXTRANSIENT pVmxTransient)
+{
+    int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION,             &pVmxTransient->uExitQual);
+    rc    |= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH,            &pVmxTransient->cbExitInstr);
+    rc    |= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO,              &pVmxTransient->ExitInstrInfo.u);
+    rc    |= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO,           &pVmxTransient->uIdtVectoringInfo);
+    rc    |= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,     &pVmxTransient->uIdtVectoringErrorCode);
+    rc    |= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,       &pVmxTransient->uExitIntInfo);
+    rc    |= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
+    rc    |= VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR,              &pVmxTransient->uGuestLinearAddr);
+    rc    |= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,         &pVmxTransient->uGuestPhysicalAddr);
+    AssertRC(rc);
+    pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION
+                                   |  HMVMX_READ_EXIT_INSTR_LEN
+                                   |  HMVMX_READ_EXIT_INSTR_INFO
+                                   |  HMVMX_READ_IDT_VECTORING_INFO
+                                   |  HMVMX_READ_IDT_VECTORING_ERROR_CODE
+                                   |  HMVMX_READ_EXIT_INTERRUPTION_INFO
+                                   |  HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
+                                   |  HMVMX_READ_GUEST_LINEAR_ADDR
+                                   |  HMVMX_READ_GUEST_PHYSICAL_ADDR;
+}
+#endif
+
+/**
+ * Enters VMX root mode operation on the current CPU.
+ *
+ * @returns VBox status code.
+ * @param   pHostCpu        The HM physical-CPU structure.
+ * @param   pVM             The cross context VM structure. Can be
+ *                          NULL, after a resume.
+ * @param   HCPhysCpuPage   Physical address of the VMXON region.
+ * @param   pvCpuPage       Pointer to the VMXON region.
+ */
+static int hmR0VmxEnterRootMode(PHMPHYSCPU pHostCpu, PVMCC pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
+{
+    Assert(pHostCpu);
+    Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
+    Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
+    Assert(pvCpuPage);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    if (pVM)
+    {
+        /* Write the VMCS revision identifier to the VMXON region. */
+        *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
+    }
+
+    /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+
+    /* Enable the VMX bit in CR4 if necessary. */
+    RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
+
+    /* Record whether VMXE was already prior to us enabling it above. */
+    pHostCpu->fVmxeAlreadyEnabled = RT_BOOL(uOldCr4 & X86_CR4_VMXE);
+
+    /* Enter VMX root mode. */
+    int rc = VMXEnable(HCPhysCpuPage);
+    if (RT_FAILURE(rc))
+    {
+        /* Restore CR4.VMXE if it was not set prior to our attempt to set it above. */
+        if (!pHostCpu->fVmxeAlreadyEnabled)
+            SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
+
+        if (pVM)
+            pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
+    }
+
+    /* Restore interrupts. */
+    ASMSetFlags(fEFlags);
+    return rc;
+}
+
+
+/**
+ * Exits VMX root mode operation on the current CPU.
+ *
+ * @returns VBox status code.
+ * @param   pHostCpu        The HM physical-CPU structure.
+ */
+static int hmR0VmxLeaveRootMode(PHMPHYSCPU pHostCpu)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+
+    /* If we're for some reason not in VMX root mode, then don't leave it. */
+    RTCCUINTREG const uHostCr4 = ASMGetCR4();
+
+    int rc;
+    if (uHostCr4 & X86_CR4_VMXE)
+    {
+        /* Exit VMX root mode and clear the VMX bit in CR4. */
+        VMXDisable();
+
+        /* Clear CR4.VMXE only if it was clear prior to use setting it. */
+        if (!pHostCpu->fVmxeAlreadyEnabled)
+            SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
+
+        rc = VINF_SUCCESS;
+    }
+    else
+        rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
+
+    /* Restore interrupts. */
+    ASMSetFlags(fEFlags);
+    return rc;
+}
+
+
+/**
+ * Allocates pages specified as specified by an array of VMX page allocation info
+ * objects.
+ *
+ * The pages contents are zero'd after allocation.
+ *
+ * @returns VBox status code.
+ * @param   phMemObj        Where to return the handle to the allocation.
+ * @param   paAllocInfo     The pointer to the first element of the VMX
+ *                          page-allocation info object array.
+ * @param   cEntries        The number of elements in the @a paAllocInfo array.
+ */
+static int hmR0VmxPagesAllocZ(PRTR0MEMOBJ phMemObj, PVMXPAGEALLOCINFO paAllocInfo, uint32_t cEntries)
+{
+    *phMemObj = NIL_RTR0MEMOBJ;
+
+    /* Figure out how many pages to allocate. */
+    uint32_t cPages = 0;
+    for (uint32_t iPage = 0; iPage < cEntries; iPage++)
+        cPages += !!paAllocInfo[iPage].fValid;
+
+    /* Allocate the pages. */
+    if (cPages)
+    {
+        size_t const cbPages = cPages << PAGE_SHIFT;
+        int rc = RTR0MemObjAllocPage(phMemObj, cbPages, false /* fExecutable */);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* Zero the contents and assign each page to the corresponding VMX page-allocation entry. */
+        void *pvFirstPage = RTR0MemObjAddress(*phMemObj);
+        RT_BZERO(pvFirstPage, cbPages);
+
+        uint32_t iPage = 0;
+        for (uint32_t i = 0; i < cEntries; i++)
+            if (paAllocInfo[i].fValid)
+            {
+                RTHCPHYS const HCPhysPage = RTR0MemObjGetPagePhysAddr(*phMemObj, iPage);
+                void          *pvPage     = (void *)((uintptr_t)pvFirstPage + (iPage << X86_PAGE_4K_SHIFT));
+                Assert(HCPhysPage && HCPhysPage != NIL_RTHCPHYS);
+                AssertPtr(pvPage);
+
+                Assert(paAllocInfo[iPage].pHCPhys);
+                Assert(paAllocInfo[iPage].ppVirt);
+                *paAllocInfo[iPage].pHCPhys = HCPhysPage;
+                *paAllocInfo[iPage].ppVirt  = pvPage;
+
+                /* Move to next page. */
+                ++iPage;
+            }
+
+        /* Make sure all valid (requested) pages have been assigned. */
+        Assert(iPage == cPages);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Frees pages allocated using hmR0VmxPagesAllocZ.
+ *
+ * @param   hMemObj     The ring-0 memory object associated with the allocation.
+ */
+DECL_FORCE_INLINE(void) hmR0VmxPagesFree(RTR0MEMOBJ hMemObj)
+{
+    /* We can cleanup wholesale since it's all one allocation. */
+    RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
+}
+
+
+/**
+ * Initializes a VMCS info. object.
+ *
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static void hmR0VmxVmcsInfoInit(PVMXVMCSINFO pVmcsInfo)
+{
+    memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
+
+    Assert(pVmcsInfo->hMemObj == NIL_RTR0MEMOBJ);
+    pVmcsInfo->HCPhysVmcs          = NIL_RTHCPHYS;
+    pVmcsInfo->HCPhysShadowVmcs    = NIL_RTHCPHYS;
+    pVmcsInfo->HCPhysMsrBitmap     = NIL_RTHCPHYS;
+    pVmcsInfo->HCPhysGuestMsrLoad  = NIL_RTHCPHYS;
+    pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
+    pVmcsInfo->HCPhysHostMsrLoad   = NIL_RTHCPHYS;
+    pVmcsInfo->HCPhysVirtApic      = NIL_RTHCPHYS;
+    pVmcsInfo->HCPhysEPTP          = NIL_RTHCPHYS;
+    pVmcsInfo->u64VmcsLinkPtr      = NIL_RTHCPHYS;
+    pVmcsInfo->idHostCpuState      = NIL_RTCPUID;
+    pVmcsInfo->idHostCpuExec       = NIL_RTCPUID;
+}
+
+
+/**
+ * Frees the VT-x structures for a VMCS info. object.
+ *
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static void hmR0VmxVmcsInfoFree(PVMXVMCSINFO pVmcsInfo)
+{
+    if (pVmcsInfo->hMemObj != NIL_RTR0MEMOBJ)
+    {
+        hmR0VmxPagesFree(pVmcsInfo->hMemObj);
+        hmR0VmxVmcsInfoInit(pVmcsInfo);
+    }
+}
+
+
+/**
+ * Allocates the VT-x structures for a VMCS info. object.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcsInfo       The VMCS info. object.
+ * @param   fIsNstGstVmcs   Whether this is a nested-guest VMCS.
+ *
+ * @remarks The caller is expected to take care of any and all allocation failures.
+ *          This function will not perform any cleanup for failures half-way
+ *          through.
+ */
+static int hmR0VmxAllocVmcsInfo(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    bool const fMsrBitmaps = RT_BOOL(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS);
+    bool const fShadowVmcs = !fIsNstGstVmcs ? pVM->hm.s.vmx.fUseVmcsShadowing : pVM->cpum.ro.GuestFeatures.fVmxVmcsShadowing;
+    Assert(!pVM->cpum.ro.GuestFeatures.fVmxVmcsShadowing);  /* VMCS shadowing is not yet exposed to the guest. */
+    VMXPAGEALLOCINFO aAllocInfo[] =
+    {
+        { true,        0 /* Unused */, &pVmcsInfo->HCPhysVmcs,         &pVmcsInfo->pvVmcs         },
+        { true,        0 /* Unused */, &pVmcsInfo->HCPhysGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad },
+        { true,        0 /* Unused */, &pVmcsInfo->HCPhysHostMsrLoad,  &pVmcsInfo->pvHostMsrLoad  },
+        { fMsrBitmaps, 0 /* Unused */, &pVmcsInfo->HCPhysMsrBitmap,    &pVmcsInfo->pvMsrBitmap    },
+        { fShadowVmcs, 0 /* Unused */, &pVmcsInfo->HCPhysShadowVmcs,   &pVmcsInfo->pvShadowVmcs   },
+    };
+
+    int rc = hmR0VmxPagesAllocZ(&pVmcsInfo->hMemObj, &aAllocInfo[0], RT_ELEMENTS(aAllocInfo));
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
+     * Because they contain a symmetric list of guest MSRs to load on VM-entry and store on VM-exit.
+     */
+    AssertCompile(RT_ELEMENTS(aAllocInfo) > 0);
+    Assert(pVmcsInfo->HCPhysGuestMsrLoad != NIL_RTHCPHYS);
+    pVmcsInfo->pvGuestMsrStore     = pVmcsInfo->pvGuestMsrLoad;
+    pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
+
+    /*
+     * Get the virtual-APIC page rather than allocating them again.
+     */
+    if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
+    {
+        if (!fIsNstGstVmcs)
+        {
+            if (PDMHasApic(pVM))
+            {
+                rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic, NULL /*pR3Ptr*/);
+                if (RT_FAILURE(rc))
+                    return rc;
+                Assert(pVmcsInfo->pbVirtApic);
+                Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
+            }
+        }
+        else
+        {
+            pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(&pVCpu->cpum.GstCtx, &pVmcsInfo->HCPhysVirtApic);
+            Assert(pVmcsInfo->pbVirtApic);
+            Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Free all VT-x structures for the VM.
+ *
+ * @returns IPRT status code.
+ * @param   pVM     The cross context VM structure.
+ */
+static void hmR0VmxStructsFree(PVMCC pVM)
+{
+    hmR0VmxPagesFree(pVM->hm.s.vmx.hMemObj);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (pVM->hm.s.vmx.fUseVmcsShadowing)
+    {
+        RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
+        RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
+    }
+#endif
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+        hmR0VmxVmcsInfoFree(&pVCpu->hm.s.vmx.VmcsInfo);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        if (pVM->cpum.ro.GuestFeatures.fVmx)
+            hmR0VmxVmcsInfoFree(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
+#endif
+    }
+}
+
+
+/**
+ * Allocate all VT-x structures for the VM.
+ *
+ * @returns IPRT status code.
+ * @param   pVM     The cross context VM structure.
+ *
+ * @remarks This functions will cleanup on memory allocation failures.
+ */
+static int hmR0VmxStructsAlloc(PVMCC pVM)
+{
+    /*
+     * Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
+     * The VMCS size cannot be more than 4096 bytes.
+     *
+     * See Intel spec. Appendix A.1 "Basic VMX Information".
+     */
+    uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
+    if (cbVmcs <= X86_PAGE_4K_SIZE)
+    { /* likely */ }
+    else
+    {
+        VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
+        return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+    }
+
+    /*
+     * Allocate per-VM VT-x structures.
+     */
+    bool const fVirtApicAccess   = RT_BOOL(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
+    bool const fUseVmcsShadowing = pVM->hm.s.vmx.fUseVmcsShadowing;
+    VMXPAGEALLOCINFO aAllocInfo[] =
+    {
+        { fVirtApicAccess,   0 /* Unused */, &pVM->hm.s.vmx.HCPhysApicAccess,    (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess },
+        { fUseVmcsShadowing, 0 /* Unused */, &pVM->hm.s.vmx.HCPhysVmreadBitmap,  &pVM->hm.s.vmx.pvVmreadBitmap         },
+        { fUseVmcsShadowing, 0 /* Unused */, &pVM->hm.s.vmx.HCPhysVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap        },
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+        { true,              0 /* Unused */, &pVM->hm.s.vmx.HCPhysScratch,       &(PRTR0PTR)pVM->hm.s.vmx.pbScratch    },
+#endif
+    };
+
+    int rc = hmR0VmxPagesAllocZ(&pVM->hm.s.vmx.hMemObj, &aAllocInfo[0], RT_ELEMENTS(aAllocInfo));
+    if (RT_SUCCESS(rc))
+    {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        /* Allocate the shadow VMCS-fields array. */
+        if (fUseVmcsShadowing)
+        {
+            Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
+            Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
+            pVM->hm.s.vmx.paShadowVmcsFields   = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
+            pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
+            if (!pVM->hm.s.vmx.paShadowVmcsFields || !pVM->hm.s.vmx.paShadowVmcsRoFields)
+                rc = VERR_NO_MEMORY;
+        }
+#endif
+
+        /*
+         * Allocate per-VCPU VT-x structures.
+         */
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus && RT_SUCCESS(rc); idCpu++)
+        {
+            /* Allocate the guest VMCS structures. */
+            PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+            rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            /* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
+            if (pVM->cpum.ro.GuestFeatures.fVmx && RT_SUCCESS(rc))
+                rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
+#endif
+        }
+        if (RT_SUCCESS(rc))
+            return VINF_SUCCESS;
+    }
+    hmR0VmxStructsFree(pVM);
+    return rc;
+}
+
+
+/**
+ * Pre-initializes non-zero fields in VMX structures that will be allocated.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+static void hmR0VmxStructsInit(PVMCC pVM)
+{
+    /* Paranoia. */
+    Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    Assert(pVM->hm.s.vmx.pbScratch == NULL);
+#endif
+
+    /*
+     * Initialize members up-front so we can cleanup en masse on allocation failures.
+     */
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
+#endif
+    pVM->hm.s.vmx.HCPhysApicAccess    = NIL_RTHCPHYS;
+    pVM->hm.s.vmx.HCPhysVmreadBitmap  = NIL_RTHCPHYS;
+    pVM->hm.s.vmx.HCPhysVmwriteBitmap = NIL_RTHCPHYS;
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+        hmR0VmxVmcsInfoInit(&pVCpu->hm.s.vmx.VmcsInfo);
+        hmR0VmxVmcsInfoInit(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
+    }
+}
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
+ *
+ * @returns @c true if the MSR is intercepted, @c false otherwise.
+ * @param   pvMsrBitmap     The MSR bitmap.
+ * @param   offMsr          The MSR byte offset.
+ * @param   iBit            The bit offset from the byte offset.
+ */
+DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
+{
+    uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
+    Assert(pbMsrBitmap);
+    Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
+    return ASMBitTest(pbMsrBitmap + offMsr, iBit);
+}
+#endif
+
+/**
+ * Sets the permission bits for the specified MSR in the given MSR bitmap.
+ *
+ * If the passed VMCS is a nested-guest VMCS, this function ensures that the
+ * read/write intercept is cleared from the MSR bitmap used for hardware-assisted
+ * VMX execution of the nested-guest, only if nested-guest is also not intercepting
+ * the read/write access of this MSR.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcsInfo       The VMCS info. object.
+ * @param   fIsNstGstVmcs   Whether this is a nested-guest VMCS.
+ * @param   idMsr           The MSR value.
+ * @param   fMsrpm          The MSR permissions (see VMXMSRPM_XXX). This must
+ *                          include both a read -and- a write permission!
+ *
+ * @sa      CPUMGetVmxMsrPermission.
+ * @remarks Can be called with interrupts disabled.
+ */
+static void hmR0VmxSetMsrPermission(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
+{
+    uint8_t *pbMsrBitmap = (uint8_t *)pVmcsInfo->pvMsrBitmap;
+    Assert(pbMsrBitmap);
+    Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
+
+    /*
+     * MSR-bitmap Layout:
+     *   Byte index            MSR range            Interpreted as
+     * 0x000 - 0x3ff    0x00000000 - 0x00001fff    Low MSR read bits.
+     * 0x400 - 0x7ff    0xc0000000 - 0xc0001fff    High MSR read bits.
+     * 0x800 - 0xbff    0x00000000 - 0x00001fff    Low MSR write bits.
+     * 0xc00 - 0xfff    0xc0000000 - 0xc0001fff    High MSR write bits.
+     *
+     * A bit corresponding to an MSR within the above range causes a VM-exit
+     * if the bit is 1 on executions of RDMSR/WRMSR.  If an MSR falls out of
+     * the MSR range, it always cause a VM-exit.
+     *
+     * See Intel spec. 24.6.9 "MSR-Bitmap Address".
+     */
+    uint16_t const offBitmapRead  = 0;
+    uint16_t const offBitmapWrite = 0x800;
+    uint16_t       offMsr;
+    int32_t        iBit;
+    if (idMsr <= UINT32_C(0x00001fff))
+    {
+        offMsr = 0;
+        iBit   = idMsr;
+    }
+    else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
+    {
+        offMsr = 0x400;
+        iBit   = idMsr - UINT32_C(0xc0000000);
+    }
+    else
+        AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
+
+    /*
+     * Set the MSR read permission.
+     */
+    uint16_t const offMsrRead = offBitmapRead + offMsr;
+    Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
+    if (fMsrpm & VMXMSRPM_ALLOW_RD)
+    {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        bool const fClear = !fIsNstGstVmcs ? true
+                          : !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
+#else
+        RT_NOREF2(pVCpu, fIsNstGstVmcs);
+        bool const fClear = true;
+#endif
+        if (fClear)
+            ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
+    }
+    else
+        ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
+
+    /*
+     * Set the MSR write permission.
+     */
+    uint16_t const offMsrWrite = offBitmapWrite + offMsr;
+    Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
+    if (fMsrpm & VMXMSRPM_ALLOW_WR)
+    {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        bool const fClear = !fIsNstGstVmcs ? true
+                          : !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
+#else
+        RT_NOREF2(pVCpu, fIsNstGstVmcs);
+        bool const fClear = true;
+#endif
+        if (fClear)
+            ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
+    }
+    else
+        ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
+}
+
+
+/**
+ * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
+ * area.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ * @param   cMsrs       The number of MSRs.
+ */
+static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
+{
+    /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
+    uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
+    if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
+    {
+        /* Commit the MSR counts to the VMCS and update the cache. */
+        if (pVmcsInfo->cEntryMsrLoad != cMsrs)
+        {
+            int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);   AssertRC(rc);
+            rc     = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);   AssertRC(rc);
+            rc     = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,  cMsrs);   AssertRC(rc);
+            pVmcsInfo->cEntryMsrLoad = cMsrs;
+            pVmcsInfo->cExitMsrStore = cMsrs;
+            pVmcsInfo->cExitMsrLoad  = cMsrs;
+        }
+        return VINF_SUCCESS;
+    }
+
+    LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
+    pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
+    return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+}
+
+
+/**
+ * Adds a new (or updates the value of an existing) guest/host MSR
+ * pair to be swapped during the world-switch as part of the
+ * auto-load/store MSR area in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   idMsr           The MSR.
+ * @param   uGuestMsrValue  Value of the guest MSR.
+ * @param   fSetReadWrite   Whether to set the guest read/write access of this
+ *                          MSR (thus not causing a VM-exit).
+ * @param   fUpdateHostMsr  Whether to update the value of the host MSR if
+ *                          necessary.
+ */
+static int hmR0VmxAddAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
+                                      bool fSetReadWrite, bool fUpdateHostMsr)
+{
+    PVMXVMCSINFO pVmcsInfo     = pVmxTransient->pVmcsInfo;
+    bool const   fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
+    PVMXAUTOMSR  pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
+    uint32_t     cMsrs         = pVmcsInfo->cEntryMsrLoad;
+    uint32_t     i;
+
+    /* Paranoia. */
+    Assert(pGuestMsrLoad);
+
+    LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
+
+    /* Check if the MSR already exists in the VM-entry MSR-load area. */
+    for (i = 0; i < cMsrs; i++)
+    {
+        if (pGuestMsrLoad[i].u32Msr == idMsr)
+            break;
+    }
+
+    bool fAdded = false;
+    if (i == cMsrs)
+    {
+        /* The MSR does not exist, bump the MSR count to make room for the new MSR. */
+        ++cMsrs;
+        int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
+        AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
+
+        /* Set the guest to read/write this MSR without causing VM-exits. */
+        if (   fSetReadWrite
+            && (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
+            hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
+
+        Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
+        fAdded = true;
+    }
+
+    /* Update the MSR value for the newly added or already existing MSR. */
+    pGuestMsrLoad[i].u32Msr   = idMsr;
+    pGuestMsrLoad[i].u64Value = uGuestMsrValue;
+
+    /* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
+    if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
+    {
+        PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
+        pGuestMsrStore[i].u32Msr   = idMsr;
+        pGuestMsrStore[i].u64Value = uGuestMsrValue;
+    }
+
+    /* Update the corresponding slot in the host MSR area. */
+    PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
+    Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
+    Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
+    pHostMsr[i].u32Msr = idMsr;
+
+    /*
+     * Only if the caller requests to update the host MSR value AND we've newly added the
+     * MSR to the host MSR area do we actually update the value. Otherwise, it will be
+     * updated by hmR0VmxUpdateAutoLoadHostMsrs().
+     *
+     * We do this for performance reasons since reading MSRs may be quite expensive.
+     */
+    if (fAdded)
+    {
+        if (fUpdateHostMsr)
+        {
+            Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+            Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+            pHostMsr[i].u64Value = ASMRdMsr(idMsr);
+        }
+        else
+        {
+            /* Someone else can do the work. */
+            pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Removes a guest/host MSR pair to be swapped during the world-switch from the
+ * auto-load/store MSR area in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   idMsr           The MSR.
+ */
+static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t idMsr)
+{
+    PVMXVMCSINFO pVmcsInfo     = pVmxTransient->pVmcsInfo;
+    bool const   fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
+    PVMXAUTOMSR  pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
+    uint32_t     cMsrs         = pVmcsInfo->cEntryMsrLoad;
+
+    LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
+
+    for (uint32_t i = 0; i < cMsrs; i++)
+    {
+        /* Find the MSR. */
+        if (pGuestMsrLoad[i].u32Msr == idMsr)
+        {
+            /*
+             * If it's the last MSR, we only need to reduce the MSR count.
+             * If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
+             */
+            if (i < cMsrs - 1)
+            {
+                /* Remove it from the VM-entry MSR-load area. */
+                pGuestMsrLoad[i].u32Msr   = pGuestMsrLoad[cMsrs - 1].u32Msr;
+                pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
+
+                /* Remove it from the VM-exit MSR-store area if it's in a different page. */
+                if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
+                {
+                    PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
+                    Assert(pGuestMsrStore[i].u32Msr == idMsr);
+                    pGuestMsrStore[i].u32Msr   = pGuestMsrStore[cMsrs - 1].u32Msr;
+                    pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
+                }
+
+                /* Remove it from the VM-exit MSR-load area. */
+                PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
+                Assert(pHostMsr[i].u32Msr == idMsr);
+                pHostMsr[i].u32Msr   = pHostMsr[cMsrs - 1].u32Msr;
+                pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
+            }
+
+            /* Reduce the count to reflect the removed MSR and bail. */
+            --cMsrs;
+            break;
+        }
+    }
+
+    /* Update the VMCS if the count changed (meaning the MSR was found and removed). */
+    if (cMsrs != pVmcsInfo->cEntryMsrLoad)
+    {
+        int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
+        AssertRCReturn(rc, rc);
+
+        /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
+        if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+            hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD | VMXMSRPM_EXIT_WR);
+
+        Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
+        return VINF_SUCCESS;
+    }
+
+    return VERR_NOT_FOUND;
+}
+
+
+/**
+ * Checks if the specified guest MSR is part of the VM-entry MSR-load area.
+ *
+ * @returns @c true if found, @c false otherwise.
+ * @param   pVmcsInfo   The VMCS info. object.
+ * @param   idMsr       The MSR to find.
+ */
+static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
+{
+    PCVMXAUTOMSR   pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
+    uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
+    Assert(pMsrs);
+    Assert(sizeof(*pMsrs) * cMsrs <= X86_PAGE_4K_SIZE);
+    for (uint32_t i = 0; i < cMsrs; i++)
+    {
+        if (pMsrs[i].u32Msr == idMsr)
+            return true;
+    }
+    return false;
+}
+
+
+/**
+ * Updates the value of all host MSRs in the VM-exit MSR-load area.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
+    uint32_t const cMsrs     = pVmcsInfo->cExitMsrLoad;
+    Assert(pHostMsrLoad);
+    Assert(sizeof(*pHostMsrLoad) * cMsrs <= X86_PAGE_4K_SIZE);
+    LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
+    for (uint32_t i = 0; i < cMsrs; i++)
+    {
+        /*
+         * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
+         * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
+         */
+        if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
+            pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
+        else
+            pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
+    }
+}
+
+
+/**
+ * Saves a set of host MSRs to allow read/write passthru access to the guest and
+ * perform lazy restoration of the host MSRs while leaving VT-x.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxLazySaveHostMsrs(PVMCPUCC pVCpu)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    /*
+     * Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
+     */
+    if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
+    {
+        Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST));  /* Guest MSRs better not be loaded now. */
+        if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
+        {
+            pVCpu->hm.s.vmx.u64HostMsrLStar        = ASMRdMsr(MSR_K8_LSTAR);
+            pVCpu->hm.s.vmx.u64HostMsrStar         = ASMRdMsr(MSR_K6_STAR);
+            pVCpu->hm.s.vmx.u64HostMsrSfMask       = ASMRdMsr(MSR_K8_SF_MASK);
+            pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
+        }
+        pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
+    }
+}
+
+
+/**
+ * Checks whether the MSR belongs to the set of guest MSRs that we restore
+ * lazily while leaving VT-x.
+ *
+ * @returns true if it does, false otherwise.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   idMsr   The MSR to check.
+ */
+static bool hmR0VmxIsLazyGuestMsr(PCVMCPUCC pVCpu, uint32_t idMsr)
+{
+    if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
+    {
+        switch (idMsr)
+        {
+            case MSR_K8_LSTAR:
+            case MSR_K6_STAR:
+            case MSR_K8_SF_MASK:
+            case MSR_K8_KERNEL_GS_BASE:
+                return true;
+        }
+    }
+    return false;
+}
+
+
+/**
+ * Loads a set of guests MSRs to allow read/passthru to the guest.
+ *
+ * The name of this function is slightly confusing. This function does NOT
+ * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
+ * common prefix for functions dealing with "lazy restoration" of the shared
+ * MSRs.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxLazyLoadGuestMsrs(PVMCPUCC pVCpu)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+
+    Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
+    if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
+    {
+        /*
+         * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
+         * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
+         * we can skip a few MSR writes.
+         *
+         * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
+         * guest MSR values in the guest-CPU context might be different to what's currently
+         * loaded in the CPU. In either case, we need to write the new guest MSR values to the
+         * CPU, see @bugref{8728}.
+         */
+        PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+        if (   !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
+            && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
+            && pCtx->msrLSTAR        == pVCpu->hm.s.vmx.u64HostMsrLStar
+            && pCtx->msrSTAR         == pVCpu->hm.s.vmx.u64HostMsrStar
+            && pCtx->msrSFMASK       == pVCpu->hm.s.vmx.u64HostMsrSfMask)
+        {
+#ifdef VBOX_STRICT
+            Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
+            Assert(ASMRdMsr(MSR_K8_LSTAR)          == pCtx->msrLSTAR);
+            Assert(ASMRdMsr(MSR_K6_STAR)           == pCtx->msrSTAR);
+            Assert(ASMRdMsr(MSR_K8_SF_MASK)        == pCtx->msrSFMASK);
+#endif
+        }
+        else
+        {
+            ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
+            ASMWrMsr(MSR_K8_LSTAR,          pCtx->msrLSTAR);
+            ASMWrMsr(MSR_K6_STAR,           pCtx->msrSTAR);
+            ASMWrMsr(MSR_K8_SF_MASK,        pCtx->msrSFMASK);
+        }
+    }
+    pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
+}
+
+
+/**
+ * Performs lazy restoration of the set of host MSRs if they were previously
+ * loaded with guest MSR values.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ * @remarks The guest MSRs should have been saved back into the guest-CPU
+ *          context by hmR0VmxImportGuestState()!!!
+ */
+static void hmR0VmxLazyRestoreHostMsrs(PVMCPUCC pVCpu)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+
+    if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
+    {
+        Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
+        if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
+        {
+            ASMWrMsr(MSR_K8_LSTAR,          pVCpu->hm.s.vmx.u64HostMsrLStar);
+            ASMWrMsr(MSR_K6_STAR,           pVCpu->hm.s.vmx.u64HostMsrStar);
+            ASMWrMsr(MSR_K8_SF_MASK,        pVCpu->hm.s.vmx.u64HostMsrSfMask);
+            ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
+        }
+    }
+    pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
+}
+
+
+/**
+ * Verifies that our cached values of the VMCS fields are all consistent with
+ * what's actually present in the VMCS.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if all our caches match their respective VMCS fields.
+ * @retval  VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
+ *                                            VMCS content. HMCPU error-field is
+ *                                            updated, see VMX_VCI_XXX.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcsInfo       The VMCS info. object.
+ * @param   fIsNstGstVmcs   Whether this is a nested-guest VMCS.
+ */
+static int hmR0VmxCheckCachedVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
+{
+    const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
+
+    uint32_t u32Val;
+    int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
+    AssertRC(rc);
+    AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
+                        ("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
+                        pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
+                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+
+    rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
+    AssertRC(rc);
+    AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
+                        ("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
+                        pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
+                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+
+    rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
+    AssertRC(rc);
+    AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
+                        ("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
+                        pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
+                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+
+    rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
+    AssertRC(rc);
+    AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
+                        ("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
+                        pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
+                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+
+    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
+    {
+        rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
+        AssertRC(rc);
+        AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
+                            ("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
+                            pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
+                            VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+    }
+
+    rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
+    AssertRC(rc);
+    AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
+                        ("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
+                        pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
+                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+
+    uint64_t u64Val;
+    rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
+    AssertRC(rc);
+    AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
+                        ("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
+                        pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
+                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+
+    NOREF(pcszVmcs);
+    return VINF_SUCCESS;
+}
+
+
+#ifdef VBOX_STRICT
+/**
+ * Verifies that our cached host EFER MSR value has not changed since we cached it.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static void hmR0VmxCheckHostEferMsr(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
+    {
+        uint64_t const uHostEferMsr      = ASMRdMsr(MSR_K6_EFER);
+        uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
+        uint64_t       uVmcsEferMsrVmcs;
+        int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
+        AssertRC(rc);
+
+        AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
+                            ("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
+        AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
+                            ("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
+    }
+}
+
+
+/**
+ * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
+ * VMCS are correct.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcsInfo       The VMCS info. object.
+ * @param   fIsNstGstVmcs   Whether this is a nested-guest VMCS.
+ */
+static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    /* Read the various MSR-area counts from the VMCS. */
+    uint32_t cEntryLoadMsrs;
+    uint32_t cExitStoreMsrs;
+    uint32_t cExitLoadMsrs;
+    int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs);  AssertRC(rc);
+    rc     = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs);  AssertRC(rc);
+    rc     = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,  &cExitLoadMsrs);   AssertRC(rc);
+
+    /* Verify all the MSR counts are the same. */
+    Assert(cEntryLoadMsrs == cExitStoreMsrs);
+    Assert(cExitStoreMsrs == cExitLoadMsrs);
+    uint32_t const cMsrs = cExitLoadMsrs;
+
+    /* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
+    Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
+
+    /* Verify the MSR counts are within the allocated page size. */
+    Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
+
+    /* Verify the relevant contents of the MSR areas match. */
+    PCVMXAUTOMSR pGuestMsrLoad  = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
+    PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
+    PCVMXAUTOMSR pHostMsrLoad   = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
+    bool const   fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
+    for (uint32_t i = 0; i < cMsrs; i++)
+    {
+        /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
+        if (fSeparateExitMsrStorePage)
+        {
+            AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
+                                ("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
+                                 pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
+        }
+
+        AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
+                            ("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
+                             pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
+
+        uint64_t const u64HostMsr = ASMRdMsr(pHostMsrLoad->u32Msr);
+        AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64HostMsr,
+                            ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
+                             pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64HostMsr, cMsrs));
+
+        /* Verify that cached host EFER MSR matches what's loaded the CPU. */
+        bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
+        if (fIsEferMsr)
+        {
+            AssertMsgReturnVoid(u64HostMsr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
+                                ("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
+                                 pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64HostMsr, cMsrs));
+        }
+
+        /* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
+        if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+        {
+            uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
+            if (fIsEferMsr)
+            {
+                AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
+                AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
+            }
+            else
+            {
+                /* Verify LBR MSRs (used only for debugging) are intercepted. We don't passthru these MSRs to the guest yet. */
+                PCVM pVM = pVCpu->CTX_SUFF(pVM);
+                if (   pVM->hm.s.vmx.fLbr
+                    && (   hmR0VmxIsLbrBranchFromMsr(pVM, pGuestMsrLoad->u32Msr, NULL /* pidxMsr */)
+                        || hmR0VmxIsLbrBranchToMsr(pVM, pGuestMsrLoad->u32Msr, NULL /* pidxMsr */)
+                        || pGuestMsrLoad->u32Msr == pVM->hm.s.vmx.idLbrTosMsr))
+                {
+                    AssertMsgReturnVoid((fMsrpm & VMXMSRPM_MASK) == VMXMSRPM_EXIT_RD_WR,
+                                        ("u32Msr=%#RX32 cMsrs=%u Passthru read/write for LBR MSRs!\n",
+                                         pGuestMsrLoad->u32Msr, cMsrs));
+                }
+                else if (!fIsNstGstVmcs)
+                {
+                    AssertMsgReturnVoid((fMsrpm & VMXMSRPM_MASK) == VMXMSRPM_ALLOW_RD_WR,
+                                        ("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
+                }
+                else
+                {
+                    /*
+                     * A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
+                     * execute a nested-guest with MSR passthrough.
+                     *
+                     * Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
+                     * allow passthrough too.
+                     */
+                    void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
+                    Assert(pvMsrBitmapNstGst);
+                    uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
+                    AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
+                                        ("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
+                                         pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
+                }
+            }
+        }
+
+        /* Move to the next MSR. */
+        pHostMsrLoad++;
+        pGuestMsrLoad++;
+        pGuestMsrStore++;
+    }
+}
+#endif /* VBOX_STRICT */
+
+
+/**
+ * Flushes the TLB using EPT.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.  Can be NULL depending on @a enmTlbFlush.
+ * @param   pVmcsInfo       The VMCS info. object. Can be NULL depending on @a
+ *                          enmTlbFlush.
+ * @param   enmTlbFlush     Type of flush.
+ *
+ * @remarks Caller is responsible for making sure this function is called only
+ *          when NestedPaging is supported and providing @a enmTlbFlush that is
+ *          supported by the CPU.
+ * @remarks Can be called with interrupts disabled.
+ */
+static void hmR0VmxFlushEpt(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
+{
+    uint64_t au64Descriptor[2];
+    if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
+        au64Descriptor[0] = 0;
+    else
+    {
+        Assert(pVCpu);
+        Assert(pVmcsInfo);
+        au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
+    }
+    au64Descriptor[1] = 0;                       /* MBZ. Intel spec. 33.3 "VMX Instructions" */
+
+    int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
+    AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
+
+    if (   RT_SUCCESS(rc)
+        && pVCpu)
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
+}
+
+
+/**
+ * Flushes the TLB using VPID.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.  Can be NULL depending on @a enmTlbFlush.
+ * @param   enmTlbFlush     Type of flush.
+ * @param   GCPtr           Virtual address of the page to flush (can be 0 depending
+ *                          on @a enmTlbFlush).
+ *
+ * @remarks Can be called with interrupts disabled.
+ */
+static void hmR0VmxFlushVpid(PVMCPUCC pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
+{
+    Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
+
+    uint64_t au64Descriptor[2];
+    if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
+    {
+        au64Descriptor[0] = 0;
+        au64Descriptor[1] = 0;
+    }
+    else
+    {
+        AssertPtr(pVCpu);
+        AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
+        AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
+        au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
+        au64Descriptor[1] = GCPtr;
+    }
+
+    int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
+    AssertMsg(rc == VINF_SUCCESS,
+              ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
+
+    if (   RT_SUCCESS(rc)
+        && pVCpu)
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
+    NOREF(rc);
+}
+
+
+/**
+ * Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
+ * otherwise there is nothing really to invalidate.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   GCVirt  Guest virtual address of the page to invalidate.
+ */
+VMMR0DECL(int) VMXR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
+{
+    AssertPtr(pVCpu);
+    LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
+
+    if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
+    {
+        /*
+         * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
+         * the EPT case. See @bugref{6043} and @bugref{6177}.
+         *
+         * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
+         * as this function maybe called in a loop with individual addresses.
+         */
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        if (pVM->hm.s.vmx.fVpid)
+        {
+            bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
+            if (fVpidFlush)
+            {
+                hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
+            }
+            else
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+        }
+        else if (pVM->hm.s.fNestedPaging)
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
+ * case where neither EPT nor VPID is supported by the CPU.
+ *
+ * @param   pHostCpu    The HM physical-CPU structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remarks Called with interrupts disabled.
+ */
+static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
+{
+    AssertPtr(pVCpu);
+    AssertPtr(pHostCpu);
+
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
+
+    Assert(pHostCpu->idCpu != NIL_RTCPUID);
+    pVCpu->hm.s.idLastCpu      = pHostCpu->idCpu;
+    pVCpu->hm.s.cTlbFlushes    = pHostCpu->cTlbFlushes;
+    pVCpu->hm.s.fForceTLBFlush = false;
+    return;
+}
+
+
+/**
+ * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
+ *
+ * @param   pHostCpu    The HM physical-CPU structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks  All references to "ASID" in this function pertains to "VPID" in Intel's
+ *           nomenclature. The reason is, to avoid confusion in compare statements
+ *           since the host-CPU copies are named "ASID".
+ *
+ * @remarks  Called with interrupts disabled.
+ */
+static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+#ifdef VBOX_WITH_STATISTICS
+    bool fTlbFlushed = false;
+# define HMVMX_SET_TAGGED_TLB_FLUSHED()       do { fTlbFlushed = true; } while (0)
+# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT()    do { \
+                                                if (!fTlbFlushed) \
+                                                    STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
+                                              } while (0)
+#else
+# define HMVMX_SET_TAGGED_TLB_FLUSHED()       do { } while (0)
+# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT()    do { } while (0)
+#endif
+
+    AssertPtr(pVCpu);
+    AssertPtr(pHostCpu);
+    Assert(pHostCpu->idCpu != NIL_RTCPUID);
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
+              ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
+               "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
+
+    /*
+     * Force a TLB flush for the first world-switch if the current CPU differs from the one we
+     * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
+     * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
+     * cannot reuse the current ASID anymore.
+     */
+    if (   pVCpu->hm.s.idLastCpu   != pHostCpu->idCpu
+        || pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
+    {
+        ++pHostCpu->uCurrentAsid;
+        if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
+        {
+            pHostCpu->uCurrentAsid = 1;            /* Wraparound to 1; host uses 0. */
+            pHostCpu->cTlbFlushes++;               /* All VCPUs that run on this host CPU must use a new VPID. */
+            pHostCpu->fFlushAsidBeforeUse = true;  /* All VCPUs that run on this host CPU must flush their new VPID before use. */
+        }
+
+        pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
+        pVCpu->hm.s.idLastCpu    = pHostCpu->idCpu;
+        pVCpu->hm.s.cTlbFlushes  = pHostCpu->cTlbFlushes;
+
+        /*
+         * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
+         * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
+         */
+        hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
+        HMVMX_SET_TAGGED_TLB_FLUSHED();
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
+    }
+    else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))    /* Check for explicit TLB flushes. */
+    {
+        /*
+         * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
+         * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
+         * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
+         * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
+         * mappings, see @bugref{6568}.
+         *
+         * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
+         */
+        hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
+        HMVMX_SET_TAGGED_TLB_FLUSHED();
+    }
+    else if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
+    {
+        /*
+         * The nested-guest specifies its own guest-physical address to use as the APIC-access
+         * address which requires flushing the TLB of EPT cached structures.
+         *
+         * See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
+         */
+        hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
+        pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
+        HMVMX_SET_TAGGED_TLB_FLUSHED();
+    }
+
+
+    pVCpu->hm.s.fForceTLBFlush = false;
+    HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
+
+    Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
+    Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
+    AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
+              ("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
+    AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
+              ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
+               pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
+    AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
+              ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
+
+    /* Update VMCS with the VPID. */
+    int rc  = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
+    AssertRC(rc);
+
+#undef HMVMX_SET_TAGGED_TLB_FLUSHED
+}
+
+
+/**
+ * Flushes the tagged-TLB entries for EPT CPUs as necessary.
+ *
+ * @param   pHostCpu    The HM physical-CPU structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Called with interrupts disabled.
+ */
+static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+    AssertPtr(pVCpu);
+    AssertPtr(pHostCpu);
+    Assert(pHostCpu->idCpu != NIL_RTCPUID);
+    AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
+    AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
+
+    /*
+     * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
+     * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
+     */
+    if (   pVCpu->hm.s.idLastCpu   != pHostCpu->idCpu
+        || pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
+    {
+        pVCpu->hm.s.fForceTLBFlush = true;
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
+    }
+
+    /* Check for explicit TLB flushes. */
+    if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
+    {
+        pVCpu->hm.s.fForceTLBFlush = true;
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
+    }
+
+    /* Check for TLB flushes while switching to/from a nested-guest. */
+    if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
+    {
+        pVCpu->hm.s.fForceTLBFlush = true;
+        pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
+    }
+
+    pVCpu->hm.s.idLastCpu   = pHostCpu->idCpu;
+    pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
+
+    if (pVCpu->hm.s.fForceTLBFlush)
+    {
+        hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
+        pVCpu->hm.s.fForceTLBFlush = false;
+    }
+}
+
+
+/**
+ * Flushes the tagged-TLB entries for VPID CPUs as necessary.
+ *
+ * @param   pHostCpu    The HM physical-CPU structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remarks Called with interrupts disabled.
+ */
+static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
+{
+    AssertPtr(pVCpu);
+    AssertPtr(pHostCpu);
+    Assert(pHostCpu->idCpu != NIL_RTCPUID);
+    AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
+    AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
+
+    /*
+     * Force a TLB flush for the first world switch if the current CPU differs from the one we
+     * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
+     * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
+     * cannot reuse the current ASID anymore.
+     */
+    if (   pVCpu->hm.s.idLastCpu   != pHostCpu->idCpu
+        || pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
+    {
+        pVCpu->hm.s.fForceTLBFlush = true;
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
+    }
+
+    /* Check for explicit TLB flushes. */
+    if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
+    {
+        /*
+         * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
+         * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
+         * fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
+         * include fExplicitFlush's too) - an obscure corner case.
+         */
+        pVCpu->hm.s.fForceTLBFlush = true;
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
+    }
+
+    /* Check for TLB flushes while switching to/from a nested-guest. */
+    if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
+    {
+        pVCpu->hm.s.fForceTLBFlush = true;
+        pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
+    }
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
+    if (pVCpu->hm.s.fForceTLBFlush)
+    {
+        ++pHostCpu->uCurrentAsid;
+        if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
+        {
+            pHostCpu->uCurrentAsid        = 1;     /* Wraparound to 1; host uses 0 */
+            pHostCpu->cTlbFlushes++;               /* All VCPUs that run on this host CPU must use a new VPID. */
+            pHostCpu->fFlushAsidBeforeUse = true;  /* All VCPUs that run on this host CPU must flush their new VPID before use. */
+        }
+
+        pVCpu->hm.s.fForceTLBFlush = false;
+        pVCpu->hm.s.cTlbFlushes    = pHostCpu->cTlbFlushes;
+        pVCpu->hm.s.uCurrentAsid   = pHostCpu->uCurrentAsid;
+        if (pHostCpu->fFlushAsidBeforeUse)
+        {
+            if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
+                hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
+            else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
+            {
+                hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
+                pHostCpu->fFlushAsidBeforeUse = false;
+            }
+            else
+            {
+                /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
+                AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
+            }
+        }
+    }
+
+    AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
+              ("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
+    AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
+              ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
+               pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
+    AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
+              ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
+
+    int rc  = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
+    AssertRC(rc);
+}
+
+
+/**
+ * Flushes the guest TLB entry based on CPU capabilities.
+ *
+ * @param   pHostCpu    The HM physical-CPU structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Called with interrupts disabled.
+ */
+static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+#ifdef HMVMX_ALWAYS_FLUSH_TLB
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+#endif
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    switch (pVM->hm.s.vmx.enmTlbFlushType)
+    {
+        case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
+        case VMXTLBFLUSHTYPE_EPT:      hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo);  break;
+        case VMXTLBFLUSHTYPE_VPID:     hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu);            break;
+        case VMXTLBFLUSHTYPE_NONE:     hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu);            break;
+        default:
+            AssertMsgFailed(("Invalid flush-tag function identifier\n"));
+            break;
+    }
+    /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
+}
+
+
+/**
+ * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
+ * TLB entries from the host TLB before VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+static int hmR0VmxSetupTaggedTlb(PVMCC pVM)
+{
+    /*
+     * Determine optimal flush type for nested paging.
+     * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
+     * unrestricted guest execution (see hmR3InitFinalizeR0()).
+     */
+    if (pVM->hm.s.fNestedPaging)
+    {
+        if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
+        {
+            if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
+                pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
+            else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
+                pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
+            else
+            {
+                /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
+                pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
+                VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
+                return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+            }
+
+            /* Make sure the write-back cacheable memory type for EPT is supported. */
+            if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
+            {
+                pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
+                VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
+                return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+            }
+
+            /* EPT requires a page-walk length of 4. */
+            if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
+            {
+                pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
+                VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
+                return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+            }
+        }
+        else
+        {
+            /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
+            pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
+            VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
+            return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+        }
+    }
+
+    /*
+     * Determine optimal flush type for VPID.
+     */
+    if (pVM->hm.s.vmx.fVpid)
+    {
+        if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
+        {
+            if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
+                pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
+            else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
+                pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
+            else
+            {
+                /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
+                if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
+                    LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
+                if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
+                    LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
+                pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
+                pVM->hm.s.vmx.fVpid = false;
+            }
+        }
+        else
+        {
+            /*  Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
+            Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
+            pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
+            pVM->hm.s.vmx.fVpid = false;
+        }
+    }
+
+    /*
+     * Setup the handler for flushing tagged-TLBs.
+     */
+    if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
+        pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
+    else if (pVM->hm.s.fNestedPaging)
+        pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
+    else if (pVM->hm.s.vmx.fVpid)
+        pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
+    else
+        pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets up the LBR MSR ranges based on the host CPU.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+static int hmR0VmxSetupLbrMsrRange(PVMCC pVM)
+{
+    Assert(pVM->hm.s.vmx.fLbr);
+    uint32_t idLbrFromIpMsrFirst;
+    uint32_t idLbrFromIpMsrLast;
+    uint32_t idLbrToIpMsrFirst;
+    uint32_t idLbrToIpMsrLast;
+    uint32_t idLbrTosMsr;
+
+    /*
+     * Determine the LBR MSRs supported for this host CPU family and model.
+     *
+     * See Intel spec. 17.4.8 "LBR Stack".
+     * See Intel "Model-Specific Registers" spec.
+     */
+    uint32_t const uFamilyModel = (pVM->cpum.ro.HostFeatures.uFamily << 8)
+                                | pVM->cpum.ro.HostFeatures.uModel;
+    switch (uFamilyModel)
+    {
+        case 0x0f01: case 0x0f02:
+            idLbrFromIpMsrFirst = MSR_P4_LASTBRANCH_0;
+            idLbrFromIpMsrLast  = MSR_P4_LASTBRANCH_3;
+            idLbrToIpMsrFirst   = 0x0;
+            idLbrToIpMsrLast    = 0x0;
+            idLbrTosMsr         = MSR_P4_LASTBRANCH_TOS;
+            break;
+
+        case 0x065c: case 0x065f: case 0x064e: case 0x065e: case 0x068e:
+        case 0x069e: case 0x0655: case 0x0666: case 0x067a: case 0x0667:
+        case 0x066a: case 0x066c: case 0x067d: case 0x067e:
+            idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
+            idLbrFromIpMsrLast  = MSR_LASTBRANCH_31_FROM_IP;
+            idLbrToIpMsrFirst   = MSR_LASTBRANCH_0_TO_IP;
+            idLbrToIpMsrLast    = MSR_LASTBRANCH_31_TO_IP;
+            idLbrTosMsr         = MSR_LASTBRANCH_TOS;
+            break;
+
+        case 0x063d: case 0x0647: case 0x064f: case 0x0656: case 0x063c:
+        case 0x0645: case 0x0646: case 0x063f: case 0x062a: case 0x062d:
+        case 0x063a: case 0x063e: case 0x061a: case 0x061e: case 0x061f:
+        case 0x062e: case 0x0625: case 0x062c: case 0x062f:
+            idLbrFromIpMsrFirst = MSR_LASTBRANCH_0_FROM_IP;
+            idLbrFromIpMsrLast  = MSR_LASTBRANCH_15_FROM_IP;
+            idLbrToIpMsrFirst   = MSR_LASTBRANCH_0_TO_IP;
+            idLbrToIpMsrLast    = MSR_LASTBRANCH_15_TO_IP;
+            idLbrTosMsr         = MSR_LASTBRANCH_TOS;
+            break;
+
+        case 0x0617: case 0x061d: case 0x060f:
+            idLbrFromIpMsrFirst = MSR_CORE2_LASTBRANCH_0_FROM_IP;
+            idLbrFromIpMsrLast  = MSR_CORE2_LASTBRANCH_3_FROM_IP;
+            idLbrToIpMsrFirst   = MSR_CORE2_LASTBRANCH_0_TO_IP;
+            idLbrToIpMsrLast    = MSR_CORE2_LASTBRANCH_3_TO_IP;
+            idLbrTosMsr         = MSR_CORE2_LASTBRANCH_TOS;
+            break;
+
+        /* Atom and related microarchitectures we don't care about:
+        case 0x0637: case 0x064a: case 0x064c: case 0x064d: case 0x065a:
+        case 0x065d: case 0x061c: case 0x0626: case 0x0627: case 0x0635:
+        case 0x0636: */
+        /* All other CPUs: */
+        default:
+        {
+            LogRelFunc(("Could not determine LBR stack size for the CPU model %#x\n", uFamilyModel));
+            VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_UNKNOWN;
+            return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+        }
+    }
+
+    /*
+     * Validate.
+     */
+    uint32_t const cLbrStack = idLbrFromIpMsrLast - idLbrFromIpMsrFirst + 1;
+    PCVMCPU pVCpu0 = VMCC_GET_CPU_0(pVM);
+    AssertCompile(   RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrFromIpMsr)
+                  == RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrToIpMsr));
+    if (cLbrStack > RT_ELEMENTS(pVCpu0->hm.s.vmx.VmcsInfo.au64LbrFromIpMsr))
+    {
+        LogRelFunc(("LBR stack size of the CPU (%u) exceeds our buffer size\n", cLbrStack));
+        VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_LBR_STACK_SIZE_OVERFLOW;
+        return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+    }
+    NOREF(pVCpu0);
+
+    /*
+     * Update the LBR info. to the VM struct. for use later.
+     */
+    pVM->hm.s.vmx.idLbrTosMsr         = idLbrTosMsr;
+    pVM->hm.s.vmx.idLbrFromIpMsrFirst = idLbrFromIpMsrFirst;
+    pVM->hm.s.vmx.idLbrFromIpMsrLast  = idLbrFromIpMsrLast;
+
+    pVM->hm.s.vmx.idLbrToIpMsrFirst   = idLbrToIpMsrFirst;
+    pVM->hm.s.vmx.idLbrToIpMsrLast    = idLbrToIpMsrLast;
+    return VINF_SUCCESS;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Sets up the shadow VMCS fields arrays.
+ *
+ * This function builds arrays of VMCS fields to sync the shadow VMCS later while
+ * executing the guest.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+static int hmR0VmxSetupShadowVmcsFieldsArrays(PVMCC pVM)
+{
+    /*
+     * Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
+     * when the host does not support it.
+     */
+    bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
+    if (   !fGstVmwriteAll
+        || (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
+    { /* likely. */ }
+    else
+    {
+        LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
+        VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
+        return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+    }
+
+    uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
+    uint32_t       cRwFields   = 0;
+    uint32_t       cRoFields   = 0;
+    for (uint32_t i = 0; i < cVmcsFields; i++)
+    {
+        VMXVMCSFIELD VmcsField;
+        VmcsField.u = g_aVmcsFields[i];
+
+        /*
+         * We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
+         * Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
+         * in the shadow VMCS fields array as they would be redundant.
+         *
+         * If the VMCS field depends on a CPU feature that is not exposed to the guest,
+         * we must not include it in the shadow VMCS fields array. Guests attempting to
+         * VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
+         * the required behavior.
+         */
+        if (   VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
+            && CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
+        {
+            /*
+             * Read-only fields are placed in a separate array so that while syncing shadow
+             * VMCS fields later (which is more performance critical) we can avoid branches.
+             *
+             * However, if the guest can write to all fields (including read-only fields),
+             * we treat it a as read/write field. Otherwise, writing to these fields would
+             * cause a VMWRITE instruction error while syncing the shadow VMCS.
+             */
+            if (   fGstVmwriteAll
+                || !VMXIsVmcsFieldReadOnly(VmcsField.u))
+                pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
+            else
+                pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
+        }
+    }
+
+    /* Update the counts. */
+    pVM->hm.s.vmx.cShadowVmcsFields   = cRwFields;
+    pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets up the VMREAD and VMWRITE bitmaps.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+static void hmR0VmxSetupVmreadVmwriteBitmaps(PVMCC pVM)
+{
+    /*
+     * By default, ensure guest attempts to access any VMCS fields cause VM-exits.
+     */
+    uint32_t const cbBitmap        = X86_PAGE_4K_SIZE;
+    uint8_t       *pbVmreadBitmap  = (uint8_t *)pVM->hm.s.vmx.pvVmreadBitmap;
+    uint8_t       *pbVmwriteBitmap = (uint8_t *)pVM->hm.s.vmx.pvVmwriteBitmap;
+    ASMMemFill32(pbVmreadBitmap,  cbBitmap, UINT32_C(0xffffffff));
+    ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
+
+    /*
+     * Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
+     * VMREAD and VMWRITE bitmaps.
+     */
+    {
+        uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
+        uint32_t const  cShadowVmcsFields  = pVM->hm.s.vmx.cShadowVmcsFields;
+        for (uint32_t i = 0; i < cShadowVmcsFields; i++)
+        {
+            uint32_t const uVmcsField = paShadowVmcsFields[i];
+            Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
+            Assert(uVmcsField >> 3 < cbBitmap);
+            ASMBitClear(pbVmreadBitmap  + (uVmcsField >> 3), uVmcsField & 7);
+            ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
+        }
+    }
+
+    /*
+     * Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
+     * if the host supports VMWRITE to all supported VMCS fields.
+     */
+    if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
+    {
+        uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
+        uint32_t const  cShadowVmcsRoFields  = pVM->hm.s.vmx.cShadowVmcsRoFields;
+        for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
+        {
+            uint32_t const uVmcsField = paShadowVmcsRoFields[i];
+            Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
+            Assert(uVmcsField >> 3 < cbBitmap);
+            ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
+        }
+    }
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/**
+ * Sets up the virtual-APIC page address for the VMCS.
+ *
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+DECLINLINE(void) hmR0VmxSetupVmcsVirtApicAddr(PCVMXVMCSINFO pVmcsInfo)
+{
+    RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
+    Assert(HCPhysVirtApic != NIL_RTHCPHYS);
+    Assert(!(HCPhysVirtApic & 0xfff));                       /* Bits 11:0 MBZ. */
+    int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
+    AssertRC(rc);
+}
+
+
+/**
+ * Sets up the MSR-bitmap address for the VMCS.
+ *
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+DECLINLINE(void) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
+{
+    RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
+    Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
+    Assert(!(HCPhysMsrBitmap & 0xfff));                      /* Bits 11:0 MBZ. */
+    int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
+    AssertRC(rc);
+}
+
+
+/**
+ * Sets up the APIC-access page address for the VMCS.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) hmR0VmxSetupVmcsApicAccessAddr(PVMCPUCC pVCpu)
+{
+    RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
+    Assert(HCPhysApicAccess != NIL_RTHCPHYS);
+    Assert(!(HCPhysApicAccess & 0xfff));                     /* Bits 11:0 MBZ. */
+    int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
+    AssertRC(rc);
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Sets up the VMREAD bitmap address for the VMCS.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPUCC pVCpu)
+{
+    RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
+    Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
+    Assert(!(HCPhysVmreadBitmap & 0xfff));                     /* Bits 11:0 MBZ. */
+    int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
+    AssertRC(rc);
+}
+
+
+/**
+ * Sets up the VMWRITE bitmap address for the VMCS.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPUCC pVCpu)
+{
+    RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
+    Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
+    Assert(!(HCPhysVmwriteBitmap & 0xfff));                     /* Bits 11:0 MBZ. */
+    int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
+    AssertRC(rc);
+}
+#endif
+
+
+/**
+ * Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
+ * in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
+{
+    RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
+    Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
+    Assert(!(HCPhysGuestMsrLoad & 0xf));                     /* Bits 3:0 MBZ. */
+
+    RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
+    Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
+    Assert(!(HCPhysGuestMsrStore & 0xf));                    /* Bits 3:0 MBZ. */
+
+    RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
+    Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
+    Assert(!(HCPhysHostMsrLoad & 0xf));                      /* Bits 3:0 MBZ. */
+
+    int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);   AssertRC(rc);
+    rc     = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);  AssertRC(rc);
+    rc     = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,  HCPhysHostMsrLoad);    AssertRC(rc);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcsInfo       The VMCS info. object.
+ */
+static void hmR0VmxSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
+
+    /*
+     * By default, ensure guest attempts to access any MSR cause VM-exits.
+     * This shall later be relaxed for specific MSRs as necessary.
+     *
+     * Note: For nested-guests, the entire bitmap will be merged prior to
+     * executing the nested-guest using hardware-assisted VMX and hence there
+     * is no need to perform this operation. See hmR0VmxMergeMsrBitmapNested.
+     */
+    Assert(pVmcsInfo->pvMsrBitmap);
+    ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
+
+    /*
+     * The guest can access the following MSRs (read, write) without causing
+     * VM-exits; they are loaded/stored automatically using fields in the VMCS.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS,  VMXMSRPM_ALLOW_RD_WR);
+    hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
+    hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
+    hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE,        VMXMSRPM_ALLOW_RD_WR);
+    hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE,        VMXMSRPM_ALLOW_RD_WR);
+
+    /*
+     * The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
+     * associated with then. We never need to intercept access (writes need to be
+     * executed without causing a VM-exit, reads will #GP fault anyway).
+     *
+     * The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
+     * read/write them. We swap the the guest/host MSR value using the
+     * auto-load/store MSR area.
+     */
+    if (pVM->cpum.ro.GuestFeatures.fIbpb)
+        hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD,  VMXMSRPM_ALLOW_RD_WR);
+    if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
+        hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
+    if (pVM->cpum.ro.GuestFeatures.fIbrs)
+        hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
+
+    /*
+     * Allow full read/write access for the following MSRs (mandatory for VT-x)
+     * required for 64-bit guests.
+     */
+    if (pVM->hm.s.fAllow64BitGuests)
+    {
+        hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR,          VMXMSRPM_ALLOW_RD_WR);
+        hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR,           VMXMSRPM_ALLOW_RD_WR);
+        hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK,        VMXMSRPM_ALLOW_RD_WR);
+        hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
+    }
+
+    /*
+     * IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
+     */
+#ifdef VBOX_STRICT
+    Assert(pVmcsInfo->pvMsrBitmap);
+    uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
+    Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
+#endif
+}
+
+
+/**
+ * Sets up pin-based VM-execution controls in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static int hmR0VmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    uint32_t       fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0;      /* Bits set here must always be set. */
+    uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1;      /* Bits cleared here must always be cleared. */
+
+    fVal |= VMX_PIN_CTLS_EXT_INT_EXIT                        /* External interrupts cause a VM-exit. */
+         |  VMX_PIN_CTLS_NMI_EXIT;                           /* Non-maskable interrupts (NMIs) cause a VM-exit. */
+
+    if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
+        fVal |= VMX_PIN_CTLS_VIRT_NMI;                       /* Use virtual NMIs and virtual-NMI blocking features. */
+
+    /* Enable the VMX-preemption timer. */
+    if (pVM->hm.s.vmx.fUsePreemptTimer)
+    {
+        Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
+        fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
+    }
+
+#if 0
+    /* Enable posted-interrupt processing. */
+    if (pVM->hm.s.fPostedIntrs)
+    {
+        Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1  & VMX_PIN_CTLS_POSTED_INT);
+        Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
+        fVal |= VMX_PIN_CTLS_POSTED_INT;
+    }
+#endif
+
+    if ((fVal & fZap) != fVal)
+    {
+        LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
+                    pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
+        pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
+        return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+    }
+
+    /* Commit it to the VMCS and update our cache. */
+    int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
+    AssertRC(rc);
+    pVmcsInfo->u32PinCtls = fVal;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets up secondary processor-based VM-execution controls in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static int hmR0VmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    uint32_t       fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0;    /* Bits set here must be set in the VMCS. */
+    uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;    /* Bits cleared here must be cleared in the VMCS. */
+
+    /* WBINVD causes a VM-exit. */
+    if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
+        fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
+
+    /* Enable EPT (aka nested-paging). */
+    if (pVM->hm.s.fNestedPaging)
+        fVal |= VMX_PROC_CTLS2_EPT;
+
+    /* Enable the INVPCID instruction if we expose it to the guest and is supported
+       by the hardware. Without this, guest executing INVPCID would cause a #UD. */
+    if (   pVM->cpum.ro.GuestFeatures.fInvpcid
+        && (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
+        fVal |= VMX_PROC_CTLS2_INVPCID;
+
+    /* Enable VPID. */
+    if (pVM->hm.s.vmx.fVpid)
+        fVal |= VMX_PROC_CTLS2_VPID;
+
+    /* Enable unrestricted guest execution. */
+    if (pVM->hm.s.vmx.fUnrestrictedGuest)
+        fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
+
+#if 0
+    if (pVM->hm.s.fVirtApicRegs)
+    {
+        /* Enable APIC-register virtualization. */
+        Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
+        fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
+
+        /* Enable virtual-interrupt delivery. */
+        Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
+        fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
+    }
+#endif
+
+    /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
+       where the TPR shadow resides. */
+    /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
+     *        done dynamically. */
+    if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
+    {
+        fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
+        hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
+   }
+
+    /* Enable the RDTSCP instruction if we expose it to the guest and is supported
+       by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
+    if (   pVM->cpum.ro.GuestFeatures.fRdTscP
+        && (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
+        fVal |= VMX_PROC_CTLS2_RDTSCP;
+
+    /* Enable Pause-Loop exiting. */
+    if (   (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
+        && pVM->hm.s.vmx.cPleGapTicks
+        && pVM->hm.s.vmx.cPleWindowTicks)
+    {
+        fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
+
+        int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);          AssertRC(rc);
+        rc     = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);    AssertRC(rc);
+    }
+
+    if ((fVal & fZap) != fVal)
+    {
+        LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
+                    pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
+        pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
+        return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+    }
+
+    /* Commit it to the VMCS and update our cache. */
+    int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
+    AssertRC(rc);
+    pVmcsInfo->u32ProcCtls2 = fVal;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets up processor-based VM-execution controls in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static int hmR0VmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    uint32_t       fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;     /* Bits set here must be set in the VMCS. */
+    uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;     /* Bits cleared here must be cleared in the VMCS. */
+
+    fVal |= VMX_PROC_CTLS_HLT_EXIT                                    /* HLT causes a VM-exit. */
+         |  VMX_PROC_CTLS_USE_TSC_OFFSETTING                          /* Use TSC-offsetting. */
+         |  VMX_PROC_CTLS_MOV_DR_EXIT                                 /* MOV DRx causes a VM-exit. */
+         |  VMX_PROC_CTLS_UNCOND_IO_EXIT                              /* All IO instructions cause a VM-exit. */
+         |  VMX_PROC_CTLS_RDPMC_EXIT                                  /* RDPMC causes a VM-exit. */
+         |  VMX_PROC_CTLS_MONITOR_EXIT                                /* MONITOR causes a VM-exit. */
+         |  VMX_PROC_CTLS_MWAIT_EXIT;                                 /* MWAIT causes a VM-exit. */
+
+    /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
+    if (   !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
+        ||  (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
+    {
+        pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
+        return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+    }
+
+    /* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
+    if (!pVM->hm.s.fNestedPaging)
+    {
+        Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
+        fVal |= VMX_PROC_CTLS_INVLPG_EXIT
+             |  VMX_PROC_CTLS_CR3_LOAD_EXIT
+             |  VMX_PROC_CTLS_CR3_STORE_EXIT;
+    }
+
+    /* Use TPR shadowing if supported by the CPU. */
+    if (   PDMHasApic(pVM)
+        && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
+    {
+        fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW;                /* CR8 reads from the Virtual-APIC page. */
+                                                             /* CR8 writes cause a VM-exit based on TPR threshold. */
+        Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
+        Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
+        hmR0VmxSetupVmcsVirtApicAddr(pVmcsInfo);
+    }
+    else
+    {
+        /* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
+           invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
+        if (pVM->hm.s.fAllow64BitGuests)
+        {
+            fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT             /* CR8 reads cause a VM-exit. */
+                 |  VMX_PROC_CTLS_CR8_LOAD_EXIT;             /* CR8 writes cause a VM-exit. */
+        }
+    }
+
+    /* Use MSR-bitmaps if supported by the CPU. */
+    if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+    {
+        fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
+        hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
+    }
+
+    /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
+    if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
+        fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
+
+    if ((fVal & fZap) != fVal)
+    {
+        LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
+                    pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
+        pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
+        return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+    }
+
+    /* Commit it to the VMCS and update our cache. */
+    int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
+    AssertRC(rc);
+    pVmcsInfo->u32ProcCtls = fVal;
+
+    /* Set up MSR permissions that don't change through the lifetime of the VM. */
+    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+        hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
+
+    /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
+    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
+        return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
+
+    /* Sanity check, should not really happen. */
+    if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
+    { /* likely */ }
+    else
+    {
+        pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
+        return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+    }
+
+    /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets up miscellaneous (everything other than Pin, Processor and secondary
+ * Processor-based VM-execution) control fields in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static int hmR0VmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
+    {
+        hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
+        hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
+    }
+#endif
+
+    Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
+    int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
+    AssertRC(rc);
+
+    rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
+    if (RT_SUCCESS(rc))
+    {
+        uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
+        uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
+
+        rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);    AssertRC(rc);
+        rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);    AssertRC(rc);
+
+        pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
+        pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
+
+        if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fLbr)
+        {
+            rc = VMXWriteVmcsNw(VMX_VMCS64_GUEST_DEBUGCTL_FULL, MSR_IA32_DEBUGCTL_LBR);
+            AssertRC(rc);
+        }
+        return VINF_SUCCESS;
+    }
+    else
+        LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Sets up the initial exception bitmap in the VMCS based on static conditions.
+ *
+ * We shall setup those exception intercepts that don't change during the
+ * lifetime of the VM here. The rest are done dynamically while loading the
+ * guest state.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static void hmR0VmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    /*
+     * The following exceptions are always intercepted:
+     *
+     * #AC - To prevent the guest from hanging the CPU.
+     * #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
+     *       recursive #DBs can cause a CPU hang.
+     * #PF - To sync our shadow page tables when nested-paging is not used.
+     */
+    bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
+    uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
+                               | RT_BIT(X86_XCPT_DB)
+                               | (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
+
+    /* Commit it to the VMCS. */
+    int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
+    AssertRC(rc);
+
+    /* Update our cache of the exception bitmap. */
+    pVmcsInfo->u32XcptBitmap = uXcptBitmap;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static int hmR0VmxSetupVmcsCtlsNested(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
+    int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
+    AssertRC(rc);
+
+    rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
+    if (RT_SUCCESS(rc))
+    {
+        if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+            hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
+
+        /* Paranoia - We've not yet initialized these, they shall be done while merging the VMCS. */
+        Assert(!pVmcsInfo->u64Cr0Mask);
+        Assert(!pVmcsInfo->u64Cr4Mask);
+        return VINF_SUCCESS;
+    }
+    else
+        LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
+    return rc;
+}
+#endif
+
+
+/**
+ * Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
+ * VMX.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcsInfo       The VMCS info. object.
+ * @param   fIsNstGstVmcs   Whether this is a nested-guest VMCS.
+ */
+static int hmR0VmxSetupVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
+{
+    Assert(pVmcsInfo->pvVmcs);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    /* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    *(uint32_t *)pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
+    const char * const pszVmcs     = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
+
+    LogFlowFunc(("\n"));
+
+    /*
+     * Initialize the VMCS using VMCLEAR before loading the VMCS.
+     * See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
+     */
+    int rc = hmR0VmxClearVmcs(pVmcsInfo);
+    if (RT_SUCCESS(rc))
+    {
+        rc = hmR0VmxLoadVmcs(pVmcsInfo);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Initialize the hardware-assisted VMX execution handler for guest and nested-guest VMCS.
+             * The host is always 64-bit since we no longer support 32-bit hosts.
+             * Currently we have just a single handler for all guest modes as well, see @bugref{6208#c73}.
+             */
+            pVmcsInfo->pfnStartVM = VMXR0StartVM64;
+            if (!fIsNstGstVmcs)
+            {
+                rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
+                    if (RT_SUCCESS(rc))
+                    {
+                        rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
+                        if (RT_SUCCESS(rc))
+                        {
+                            hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                            /*
+                             * If a shadow VMCS is allocated for the VMCS info. object, initialize the
+                             * VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
+                             * making it fit for use when VMCS shadowing is later enabled.
+                             */
+                            if (pVmcsInfo->pvShadowVmcs)
+                            {
+                                VMXVMCSREVID VmcsRevId;
+                                VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
+                                VmcsRevId.n.fIsShadowVmcs = 1;
+                                *(uint32_t *)pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
+                                rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
+                                if (RT_SUCCESS(rc))
+                                { /* likely */ }
+                                else
+                                    LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
+                            }
+#endif
+                        }
+                        else
+                            LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
+                    }
+                    else
+                        LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
+                }
+                else
+                    LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
+            }
+            else
+            {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
+                if (RT_SUCCESS(rc))
+                { /* likely */ }
+                else
+                    LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
+#else
+                AssertFailed();
+#endif
+            }
+        }
+        else
+            LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
+    }
+    else
+        LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
+
+    /* Sync any CPU internal VMCS data back into our VMCS in memory. */
+    if (RT_SUCCESS(rc))
+    {
+        rc = hmR0VmxClearVmcs(pVmcsInfo);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
+    }
+
+    /*
+     * Update the last-error record both for failures and success, so we
+     * can propagate the status code back to ring-3 for diagnostics.
+     */
+    hmR0VmxUpdateErrorRecord(pVCpu, rc);
+    NOREF(pszVmcs);
+    return rc;
+}
+
+
+/**
+ * Does global VT-x initialization (called during module initialization).
+ *
+ * @returns VBox status code.
+ */
+VMMR0DECL(int) VMXR0GlobalInit(void)
+{
+#ifdef HMVMX_USE_FUNCTION_TABLE
+    AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
+# ifdef VBOX_STRICT
+    for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
+        Assert(g_apfnVMExitHandlers[i]);
+# endif
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Does global VT-x termination (called during module termination).
+ */
+VMMR0DECL(void) VMXR0GlobalTerm()
+{
+    /* Nothing to do currently. */
+}
+
+
+/**
+ * Sets up and activates VT-x on the current CPU.
+ *
+ * @returns VBox status code.
+ * @param   pHostCpu        The HM physical-CPU structure.
+ * @param   pVM             The cross context VM structure.  Can be
+ *                          NULL after a host resume operation.
+ * @param   pvCpuPage       Pointer to the VMXON region (can be NULL if @a
+ *                          fEnabledByHost is @c true).
+ * @param   HCPhysCpuPage   Physical address of the VMXON region (can be 0 if
+ *                          @a fEnabledByHost is @c true).
+ * @param   fEnabledByHost  Set if SUPR0EnableVTx() or similar was used to
+ *                          enable VT-x on the host.
+ * @param   pHwvirtMsrs     Pointer to the hardware-virtualization MSRs.
+ */
+VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
+                              PCSUPHWVIRTMSRS pHwvirtMsrs)
+{
+    AssertPtr(pHostCpu);
+    AssertPtr(pHwvirtMsrs);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    /* Enable VT-x if it's not already enabled by the host. */
+    if (!fEnabledByHost)
+    {
+        int rc = hmR0VmxEnterRootMode(pHostCpu, pVM, HCPhysCpuPage, pvCpuPage);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
+     * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
+     * invalidated when flushing by VPID.
+     */
+    if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
+    {
+        hmR0VmxFlushEpt(NULL /* pVCpu */, NULL /* pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
+        pHostCpu->fFlushAsidBeforeUse = false;
+    }
+    else
+        pHostCpu->fFlushAsidBeforeUse = true;
+
+    /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
+    ++pHostCpu->cTlbFlushes;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deactivates VT-x on the current CPU.
+ *
+ * @returns VBox status code.
+ * @param   pHostCpu        The HM physical-CPU structure.
+ * @param   pvCpuPage       Pointer to the VMXON region.
+ * @param   HCPhysCpuPage   Physical address of the VMXON region.
+ *
+ * @remarks This function should never be called when SUPR0EnableVTx() or
+ *          similar was used to enable VT-x on the host.
+ */
+VMMR0DECL(int) VMXR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
+{
+    RT_NOREF2(pvCpuPage, HCPhysCpuPage);
+
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    return hmR0VmxLeaveRootMode(pHostCpu);
+}
+
+
+/**
+ * Does per-VM VT-x initialization.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ */
+VMMR0DECL(int) VMXR0InitVM(PVMCC pVM)
+{
+    AssertPtr(pVM);
+    LogFlowFunc(("pVM=%p\n", pVM));
+
+    hmR0VmxStructsInit(pVM);
+    int rc = hmR0VmxStructsAlloc(pVM);
+    if (RT_FAILURE(rc))
+    {
+        LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
+        return rc;
+    }
+
+    /* Setup the crash dump page. */
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
+    *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Does per-VM VT-x termination.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR0DECL(int) VMXR0TermVM(PVMCC pVM)
+{
+    AssertPtr(pVM);
+    LogFlowFunc(("pVM=%p\n", pVM));
+
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
+    {
+        Assert(pVM->hm.s.vmx.pvScratch);
+        ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
+    }
+#endif
+    hmR0VmxStructsFree(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets up the VM for execution using hardware-assisted VMX.
+ * This function is only called once per-VM during initialization.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR0DECL(int) VMXR0SetupVM(PVMCC pVM)
+{
+    AssertPtr(pVM);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    LogFlowFunc(("pVM=%p\n", pVM));
+
+    /*
+     * At least verify if VMX is enabled, since we can't check if we're in VMX root mode or not
+     * without causing a #GP.
+     */
+    RTCCUINTREG const uHostCr4 = ASMGetCR4();
+    if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
+    { /* likely */ }
+    else
+        return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
+
+    /*
+     * Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable *must*
+     * always be allocated. We no longer support the highly unlikely case of unrestricted guest
+     * without pRealModeTSS, see hmR3InitFinalizeR0Intel().
+     */
+    if (   !pVM->hm.s.vmx.fUnrestrictedGuest
+        &&  (   !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
+             || !pVM->hm.s.vmx.pRealModeTSS))
+    {
+        LogRelFunc(("Invalid real-on-v86 state.\n"));
+        return VERR_INTERNAL_ERROR;
+    }
+
+    /* Initialize these always, see hmR3InitFinalizeR0().*/
+    pVM->hm.s.vmx.enmTlbFlushEpt  = VMXTLBFLUSHEPT_NONE;
+    pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
+
+    /* Setup the tagged-TLB flush handlers. */
+    int rc = hmR0VmxSetupTaggedTlb(pVM);
+    if (RT_FAILURE(rc))
+    {
+        LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
+        return rc;
+    }
+
+    /* Determine LBR capabilities. */
+    if (pVM->hm.s.vmx.fLbr)
+    {
+        rc = hmR0VmxSetupLbrMsrRange(pVM);
+        if (RT_FAILURE(rc))
+        {
+            LogRelFunc(("Failed to setup LBR MSR range. rc=%Rrc\n", rc));
+            return rc;
+        }
+    }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
+    if (pVM->hm.s.vmx.fUseVmcsShadowing)
+    {
+        rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
+        if (RT_SUCCESS(rc))
+            hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
+        else
+        {
+            LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
+            return rc;
+        }
+    }
+#endif
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+        Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
+
+        rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo,  false /* fIsNstGstVmcs */);
+        if (RT_SUCCESS(rc))
+        {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            if (pVM->cpum.ro.GuestFeatures.fVmx)
+            {
+                rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
+                if (RT_SUCCESS(rc))
+                { /* likely */ }
+                else
+                {
+                    LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
+                    return rc;
+                }
+            }
+#endif
+        }
+        else
+        {
+            LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
+            return rc;
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
+ * the VMCS.
+ */
+static void hmR0VmxExportHostControlRegs(void)
+{
+    int rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR0, ASMGetCR0());    AssertRC(rc);
+    rc     = VMXWriteVmcsNw(VMX_VMCS_HOST_CR3, ASMGetCR3());    AssertRC(rc);
+    rc     = VMXWriteVmcsNw(VMX_VMCS_HOST_CR4, ASMGetCR4());    AssertRC(rc);
+}
+
+
+/**
+ * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
+ * the host-state area in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static int hmR0VmxExportHostSegmentRegs(PVMCPUCC pVCpu)
+{
+/**
+ * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
+ * requirements. See hmR0VmxExportHostSegmentRegs().
+ */
+#define VMXLOCAL_ADJUST_HOST_SEG(a_Seg, a_selValue) \
+    if ((a_selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
+    { \
+        bool fValidSelector = true; \
+        if ((a_selValue) & X86_SEL_LDT) \
+        { \
+            uint32_t const uAttr = ASMGetSegAttr(a_selValue); \
+            fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
+        } \
+        if (fValidSelector) \
+        { \
+            pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##a_Seg; \
+            pVCpu->hm.s.vmx.RestoreHost.uHostSel##a_Seg = (a_selValue); \
+        } \
+        (a_selValue) = 0; \
+    }
+
+    /*
+     * If we've executed guest code using hardware-assisted VMX, the host-state bits
+     * will be messed up. We should -not- save the messed up state without restoring
+     * the original host-state, see @bugref{7240}.
+     *
+     * This apparently can happen (most likely the FPU changes), deal with it rather than
+     * asserting. Was observed booting Solaris 10u10 32-bit guest.
+     */
+    if (   (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
+        && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
+    {
+        Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
+                  pVCpu->idCpu));
+        VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
+    }
+    pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
+
+    /*
+     * Host segment registers.
+     */
+    RTSEL uSelES = ASMGetES();
+    RTSEL uSelCS = ASMGetCS();
+    RTSEL uSelSS = ASMGetSS();
+    RTSEL uSelDS = ASMGetDS();
+    RTSEL uSelFS = ASMGetFS();
+    RTSEL uSelGS = ASMGetGS();
+    RTSEL uSelTR = ASMGetTR();
+
+    /*
+     * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
+     * gain VM-entry and restore them before we get preempted.
+     *
+     * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
+     */
+    VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
+    VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
+    VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
+    VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
+
+    /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers"  */
+    Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
+    Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
+    Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
+    Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
+    Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
+    Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
+    Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
+    Assert(uSelCS);
+    Assert(uSelTR);
+
+    /* Write these host selector fields into the host-state area in the VMCS. */
+    int rc = VMXWriteVmcs16(VMX_VMCS16_HOST_CS_SEL, uSelCS);    AssertRC(rc);
+    rc     = VMXWriteVmcs16(VMX_VMCS16_HOST_SS_SEL, uSelSS);    AssertRC(rc);
+    rc     = VMXWriteVmcs16(VMX_VMCS16_HOST_DS_SEL, uSelDS);    AssertRC(rc);
+    rc     = VMXWriteVmcs16(VMX_VMCS16_HOST_ES_SEL, uSelES);    AssertRC(rc);
+    rc     = VMXWriteVmcs16(VMX_VMCS16_HOST_FS_SEL, uSelFS);    AssertRC(rc);
+    rc     = VMXWriteVmcs16(VMX_VMCS16_HOST_GS_SEL, uSelGS);    AssertRC(rc);
+    rc     = VMXWriteVmcs16(VMX_VMCS16_HOST_TR_SEL, uSelTR);    AssertRC(rc);
+
+    /*
+     * Host GDTR and IDTR.
+     */
+    RTGDTR Gdtr;
+    RTIDTR Idtr;
+    RT_ZERO(Gdtr);
+    RT_ZERO(Idtr);
+    ASMGetGDTR(&Gdtr);
+    ASMGetIDTR(&Idtr);
+    rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);    AssertRC(rc);
+    rc = VMXWriteVmcsNw(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);    AssertRC(rc);
+
+    /*
+     * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
+     * them to the maximum limit (0xffff) on every VM-exit.
+     */
+    if (Gdtr.cbGdt != 0xffff)
+        pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
+
+    /*
+     * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
+     * Intel spec. 6.2 "Exception and Interrupt Vectors".)  Therefore if the host has the limit
+     * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
+     * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
+     * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
+     * alignment in at least one consumer).  So, we're only allowing the IDTR.LIMIT to be left
+     * at 0xffff on hosts where we are sure it won't cause trouble.
+     */
+#if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
+    if (Idtr.cbIdt <  0x0fff)
+#else
+    if (Idtr.cbIdt != 0xffff)
+#endif
+    {
+        pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
+        AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
+        memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
+    }
+
+    /*
+     * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
+     * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
+     * RPL should be too in most cases.
+     */
+    AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
+                    ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
+
+    PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
+    uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
+
+    /*
+     * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
+     * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
+     * restoration if the host has something else. Task switching is not supported in 64-bit
+     * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
+     * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
+     *
+     * [1] See Intel spec. 3.5 "System Descriptor Types".
+     * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    Assert(pDesc->System.u4Type == 11);
+    if (   pDesc->System.u16LimitLow != 0x67
+        || pDesc->System.u4LimitHigh)
+    {
+        pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
+        /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
+        if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
+            pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
+        pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
+    }
+
+    /*
+     * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
+     */
+    if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
+    {
+        AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
+        memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
+        if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
+        {
+            /* The GDT is read-only but the writable GDT is available. */
+            pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
+            pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
+            rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
+            AssertRCReturn(rc, rc);
+        }
+    }
+
+    rc = VMXWriteVmcsNw(VMX_VMCS_HOST_TR_BASE, uTRBase);
+    AssertRC(rc);
+
+    /*
+     * Host FS base and GS base.
+     */
+    uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
+    uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
+    rc = VMXWriteVmcsNw(VMX_VMCS_HOST_FS_BASE, u64FSBase);  AssertRC(rc);
+    rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GS_BASE, u64GSBase);  AssertRC(rc);
+
+    /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
+    if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
+        pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
+    if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
+        pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
+
+    return VINF_SUCCESS;
+#undef VMXLOCAL_ADJUST_HOST_SEG
+}
+
+
+/**
+ * Exports certain host MSRs in the VM-exit MSR-load area and some in the
+ * host-state area of the VMCS.
+ *
+ * These MSRs will be automatically restored on the host after every successful
+ * VM-exit.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxExportHostMsrs(PVMCPUCC pVCpu)
+{
+    AssertPtr(pVCpu);
+
+    /*
+     * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
+     * rather than swapping them on every VM-entry.
+     */
+    hmR0VmxLazySaveHostMsrs(pVCpu);
+
+    /*
+     * Host Sysenter MSRs.
+     */
+    int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));   AssertRC(rc);
+    rc     = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP,  ASMRdMsr(MSR_IA32_SYSENTER_ESP));      AssertRC(rc);
+    rc     = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP,  ASMRdMsr(MSR_IA32_SYSENTER_EIP));      AssertRC(rc);
+
+    /*
+     * Host EFER MSR.
+     *
+     * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
+     * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (pVM->hm.s.vmx.fSupportsVmcsEfer)
+    {
+        rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
+        AssertRC(rc);
+    }
+
+    /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
+     *        hmR0VmxExportGuestEntryExitCtls(). */
+}
+
+
+/**
+ * Figures out if we need to swap the EFER MSR which is particularly expensive.
+ *
+ * We check all relevant bits. For now, that's everything besides LMA/LME, as
+ * these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
+ *
+ * @returns true if we need to load guest EFER, false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks Requires EFER, CR4.
+ * @remarks No-long-jump zone!!!
+ */
+static bool hmR0VmxShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+#ifdef HMVMX_ALWAYS_SWAP_EFER
+    RT_NOREF2(pVCpu, pVmxTransient);
+    return true;
+#else
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    uint64_t const u64HostEfer  = pVM->hm.s.vmx.u64HostMsrEfer;
+    uint64_t const u64GuestEfer = pCtx->msrEFER;
+
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * For nested-guests, we shall honor swapping the EFER MSR when requested by
+     * the nested-guest.
+     */
+    if (   pVmxTransient->fIsNestedGuest
+        && (   CPUMIsGuestVmxEntryCtlsSet(pCtx, VMX_ENTRY_CTLS_LOAD_EFER_MSR)
+            || CPUMIsGuestVmxExitCtlsSet(pCtx, VMX_EXIT_CTLS_SAVE_EFER_MSR)
+            || CPUMIsGuestVmxExitCtlsSet(pCtx, VMX_EXIT_CTLS_LOAD_EFER_MSR)))
+        return true;
+# else
+    RT_NOREF(pVmxTransient);
+#endif
+
+    /*
+     * For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
+     * to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
+     */
+    if (   CPUMIsGuestInLongModeEx(pCtx)
+        && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
+        return true;
+
+    /*
+     * If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
+     * as it affects guest paging. 64-bit paging implies CR4.PAE as well.
+     *
+     * See Intel spec. 4.5 "IA-32e Paging".
+     * See Intel spec. 4.1.1 "Three Paging Modes".
+     *
+     * Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
+     * import CR4 and CR0 from the VMCS here as those bits are always up to date.
+     */
+    Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
+    Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
+    if (   (pCtx->cr4 & X86_CR4_PAE)
+        && (pCtx->cr0 & X86_CR0_PG))
+    {
+        /*
+         * If nested paging is not used, verify that the guest paging mode matches the
+         * shadow paging mode which is/will be placed in the VMCS (which is what will
+         * actually be used while executing the guest and not the CR4 shadow value).
+         */
+        AssertMsg(pVM->hm.s.fNestedPaging || (   pVCpu->hm.s.enmShadowMode == PGMMODE_PAE
+                                              || pVCpu->hm.s.enmShadowMode == PGMMODE_PAE_NX
+                                              || pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64
+                                              || pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64_NX),
+                  ("enmShadowMode=%u\n", pVCpu->hm.s.enmShadowMode));
+        if ((u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
+        {
+            /* Verify that the host is NX capable. */
+            Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
+            return true;
+        }
+    }
+
+    return false;
+#endif
+}
+
+
+/**
+ * Exports the guest state with appropriate VM-entry and VM-exit controls in the
+ * VMCS.
+ *
+ * This is typically required when the guest changes paging mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks Requires EFER.
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0VmxExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
+    {
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        PVMXVMCSINFO pVmcsInfo      = pVmxTransient->pVmcsInfo;
+
+        /*
+         * VM-entry controls.
+         */
+        {
+            uint32_t       fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0;    /* Bits set here must be set in the VMCS. */
+            uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1;    /* Bits cleared here must be cleared in the VMCS. */
+
+            /*
+             * Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
+             * The first VT-x capable CPUs only supported the 1-setting of this bit.
+             *
+             * For nested-guests, this is a mandatory VM-entry control. It's also
+             * required because we do not want to leak host bits to the nested-guest.
+             */
+            fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
+
+            /*
+             * Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
+             *
+             * For nested-guests, the "IA-32e mode guest" control we initialize with what is
+             * required to get the nested-guest working with hardware-assisted VMX execution.
+             * It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
+             * can skip intercepting changes to the EFER MSR. This is why it it needs to be done
+             * here rather than while merging the guest VMCS controls.
+             */
+            if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
+            {
+                Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
+                fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
+            }
+            else
+                Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
+
+            /*
+             * If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
+             *
+             * For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
+             * regardless of whether the nested-guest VMCS specifies it because we are free to
+             * load whatever MSRs we require and we do not need to modify the guest visible copy
+             * of the VM-entry MSR load area.
+             */
+            if (   pVM->hm.s.vmx.fSupportsVmcsEfer
+                && hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
+                fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
+            else
+                Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
+
+            /*
+             * The following should -not- be set (since we're not in SMM mode):
+             * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
+             * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
+             */
+
+            /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
+             *        VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
+
+            if ((fVal & fZap) == fVal)
+            { /* likely */ }
+            else
+            {
+                Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
+                          pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
+                pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
+                return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+            }
+
+            /* Commit it to the VMCS. */
+            if (pVmcsInfo->u32EntryCtls != fVal)
+            {
+                int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
+                AssertRC(rc);
+                pVmcsInfo->u32EntryCtls = fVal;
+            }
+        }
+
+        /*
+         * VM-exit controls.
+         */
+        {
+            uint32_t       fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0;     /* Bits set here must be set in the VMCS. */
+            uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1;     /* Bits cleared here must be cleared in the VMCS. */
+
+            /*
+             * Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
+             * supported the 1-setting of this bit.
+             *
+             * For nested-guests, we set the "save debug controls" as the converse
+             * "load debug controls" is mandatory for nested-guests anyway.
+             */
+            fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
+
+            /*
+             * Set the host long mode active (EFER.LMA) bit (which Intel calls
+             * "Host address-space size") if necessary. On VM-exit, VT-x sets both the
+             * host EFER.LMA and EFER.LME bit to this value. See assertion in
+             * hmR0VmxExportHostMsrs().
+             *
+             * For nested-guests, we always set this bit as we do not support 32-bit
+             * hosts.
+             */
+            fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
+
+            /*
+             * If the VMCS EFER MSR fields are supported by the hardware, we use it.
+             *
+             * For nested-guests, we should use the "save IA32_EFER" control if we also
+             * used the "load IA32_EFER" control while exporting VM-entry controls.
+             */
+            if (   pVM->hm.s.vmx.fSupportsVmcsEfer
+                && hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
+            {
+                fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
+                     |  VMX_EXIT_CTLS_LOAD_EFER_MSR;
+            }
+
+            /*
+             * Enable saving of the VMX-preemption timer value on VM-exit.
+             * For nested-guests, currently not exposed/used.
+             */
+            if (    pVM->hm.s.vmx.fUsePreemptTimer
+                && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
+                fVal |= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
+
+            /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
+            Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
+
+            /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
+             *        VMX_EXIT_CTLS_SAVE_PAT_MSR,
+             *        VMX_EXIT_CTLS_LOAD_PAT_MSR. */
+
+            if ((fVal & fZap) == fVal)
+            { /* likely */ }
+            else
+            {
+                Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
+                          pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
+                pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
+                return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+            }
+
+            /* Commit it to the VMCS. */
+            if (pVmcsInfo->u32ExitCtls != fVal)
+            {
+                int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
+                AssertRC(rc);
+                pVmcsInfo->u32ExitCtls = fVal;
+            }
+        }
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the TPR threshold in the VMCS.
+ *
+ * @param   pVmcsInfo           The VMCS info. object.
+ * @param   u32TprThreshold     The TPR threshold (task-priority class only).
+ */
+DECLINLINE(void) hmR0VmxApicSetTprThreshold(PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
+{
+    Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK));         /* Bits 31:4 MBZ. */
+    Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
+    RT_NOREF(pVmcsInfo);
+    int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
+    AssertRC(rc);
+}
+
+
+/**
+ * Exports the guest APIC TPR state into the VMCS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
+
+        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+        if (!pVmxTransient->fIsNestedGuest)
+        {
+            if (   PDMHasApic(pVCpu->CTX_SUFF(pVM))
+                && APICIsEnabled(pVCpu))
+            {
+                /*
+                 * Setup TPR shadowing.
+                 */
+                if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+                {
+                    bool    fPendingIntr  = false;
+                    uint8_t u8Tpr         = 0;
+                    uint8_t u8PendingIntr = 0;
+                    int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
+                    AssertRC(rc);
+
+                    /*
+                     * If there are interrupts pending but masked by the TPR, instruct VT-x to
+                     * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
+                     * priority of the pending interrupt so we can deliver the interrupt. If there
+                     * are no interrupts pending, set threshold to 0 to not cause any
+                     * TPR-below-threshold VM-exits.
+                     */
+                    uint32_t u32TprThreshold = 0;
+                    if (fPendingIntr)
+                    {
+                        /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
+                           (which is the Task-Priority Class). */
+                        const uint8_t u8PendingPriority = u8PendingIntr >> 4;
+                        const uint8_t u8TprPriority     = u8Tpr >> 4;
+                        if (u8PendingPriority <= u8TprPriority)
+                            u32TprThreshold = u8PendingPriority;
+                    }
+
+                    hmR0VmxApicSetTprThreshold(pVmcsInfo, u32TprThreshold);
+                }
+            }
+        }
+        /* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
+    }
+}
+
+
+/**
+ * Gets the guest interruptibility-state and updates related force-flags.
+ *
+ * @returns Guest's interruptibility-state.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static uint32_t hmR0VmxGetGuestIntrStateAndUpdateFFs(PVMCPUCC pVCpu)
+{
+    /*
+     * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
+     */
+    uint32_t fIntrState = 0;
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+    {
+        /* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
+
+        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+        if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
+        {
+            if (pCtx->eflags.Bits.u1IF)
+                fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
+            else
+                fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
+        }
+        else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+        {
+            /*
+             * We can clear the inhibit force flag as even if we go back to the recompiler
+             * without executing guest code in VT-x, the flag's condition to be cleared is
+             * met and thus the cleared state is correct.
+             */
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+        }
+    }
+
+    /*
+     * Check if we should inhibit NMI delivery.
+     */
+    if (CPUMIsGuestNmiBlocking(pVCpu))
+        fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
+
+    /*
+     * Validate.
+     */
+#ifdef VBOX_STRICT
+    /* We don't support block-by-SMI yet.*/
+    Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
+
+    /* Block-by-STI must not be set when interrupts are disabled. */
+    if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
+        Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF);
+    }
+#endif
+
+    return fIntrState;
+}
+
+
+/**
+ * Exports the exception intercepts required for guest execution in the VMCS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
+    {
+        /* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
+        if (   !pVmxTransient->fIsNestedGuest
+            &&  pVCpu->hm.s.fGIMTrapXcptUD)
+            hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
+        else
+            hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
+
+        /* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
+    }
+}
+
+
+/**
+ * Exports the guest's RIP into the guest-state area in the VMCS.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxExportGuestRip(PVMCPUCC pVCpu)
+{
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
+
+        int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
+        AssertRC(rc);
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
+        Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
+    }
+}
+
+
+/**
+ * Exports the guest's RSP into the guest-state area in the VMCS.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxExportGuestRsp(PVMCPUCC pVCpu)
+{
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
+
+        int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
+        AssertRC(rc);
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
+        Log4Func(("rsp=%#RX64\n", pVCpu->cpum.GstCtx.rsp));
+    }
+}
+
+
+/**
+ * Exports the guest's RFLAGS into the guest-state area in the VMCS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
+
+        /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
+           Let us assert it as such and use 32-bit VMWRITE. */
+        Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
+        X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
+        Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
+        Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
+
+        /*
+         * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
+         * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
+         * can run the real-mode guest code under Virtual 8086 mode.
+         */
+        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+        if (pVmcsInfo->RealMode.fRealOnV86Active)
+        {
+            Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
+            Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
+            Assert(!pVmxTransient->fIsNestedGuest);
+            pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32;    /* Save the original eflags of the real-mode guest. */
+            fEFlags.Bits.u1VM   = 1;                         /* Set the Virtual 8086 mode bit. */
+            fEFlags.Bits.u2IOPL = 0;                         /* Change IOPL to 0, otherwise certain instructions won't fault. */
+        }
+
+        int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
+        AssertRC(rc);
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
+        Log4Func(("eflags=%#RX32\n", fEFlags.u32));
+    }
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Copies the nested-guest VMCS to the shadow VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+
+    /*
+     * Disable interrupts so we don't get preempted while the shadow VMCS is the
+     * current VMCS, as we may try saving guest lazy MSRs.
+     *
+     * Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
+     * calling the import VMCS code which is currently performing the guest MSR reads
+     * (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
+     * and the rest of the VMX leave session machinery.
+     */
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+
+    int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Copy all guest read/write VMCS fields.
+         *
+         * We don't check for VMWRITE failures here for performance reasons and
+         * because they are not expected to fail, barring irrecoverable conditions
+         * like hardware errors.
+         */
+        uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
+        for (uint32_t i = 0; i < cShadowVmcsFields; i++)
+        {
+            uint64_t       u64Val;
+            uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
+            IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
+            VMXWriteVmcs64(uVmcsField, u64Val);
+        }
+
+        /*
+         * If the host CPU supports writing all VMCS fields, copy the guest read-only
+         * VMCS fields, so the guest can VMREAD them without causing a VM-exit.
+         */
+        if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
+        {
+            uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
+            for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
+            {
+                uint64_t       u64Val;
+                uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
+                IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
+                VMXWriteVmcs64(uVmcsField, u64Val);
+            }
+        }
+
+        rc  = hmR0VmxClearShadowVmcs(pVmcsInfo);
+        rc |= hmR0VmxLoadVmcs(pVmcsInfo);
+    }
+
+    ASMSetFlags(fEFlags);
+    return rc;
+}
+
+
+/**
+ * Copies the shadow VMCS to the nested-guest VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Called with interrupts disabled.
+ */
+static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+
+    int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Copy guest read/write fields from the shadow VMCS.
+         * Guest read-only fields cannot be modified, so no need to copy them.
+         *
+         * We don't check for VMREAD failures here for performance reasons and
+         * because they are not expected to fail, barring irrecoverable conditions
+         * like hardware errors.
+         */
+        uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
+        for (uint32_t i = 0; i < cShadowVmcsFields; i++)
+        {
+            uint64_t       u64Val;
+            uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
+            VMXReadVmcs64(uVmcsField, &u64Val);
+            IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
+        }
+
+        rc  = hmR0VmxClearShadowVmcs(pVmcsInfo);
+        rc |= hmR0VmxLoadVmcs(pVmcsInfo);
+    }
+    return rc;
+}
+
+
+/**
+ * Enables VMCS shadowing for the given VMCS info. object.
+ *
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxEnableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
+{
+    uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
+    if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
+    {
+        Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
+        uProcCtls2 |= VMX_PROC_CTLS2_VMCS_SHADOWING;
+        int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);                            AssertRC(rc);
+        rc     = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);  AssertRC(rc);
+        pVmcsInfo->u32ProcCtls2   = uProcCtls2;
+        pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
+        Log4Func(("Enabled\n"));
+    }
+}
+
+
+/**
+ * Disables VMCS shadowing for the given VMCS info. object.
+ *
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxDisableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
+{
+    /*
+     * We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
+     * VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
+     * to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
+     * Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
+     *
+     * See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
+     * See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
+     */
+    uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
+    if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
+    {
+        uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
+        int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);                AssertRC(rc);
+        rc     = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);     AssertRC(rc);
+        pVmcsInfo->u32ProcCtls2   = uProcCtls2;
+        pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
+        Log4Func(("Disabled\n"));
+    }
+}
+#endif
+
+
+/**
+ * Exports the guest hardware-virtualization state.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0VmxExportGuestHwvirtState(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
+    {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        /*
+         * Check if the VMX feature is exposed to the guest and if the host CPU supports
+         * VMCS shadowing.
+         */
+        if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
+        {
+            /*
+             * If the nested hypervisor has loaded a current VMCS and is in VMX root mode,
+             * copy the nested hypervisor's current VMCS into the shadow VMCS and enable
+             * VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
+             *
+             * We check for VMX root mode here in case the guest executes VMXOFF without
+             * clearing the current VMCS pointer and our VMXOFF instruction emulation does
+             * not clear the current VMCS pointer.
+             */
+            PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+            if (   CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
+                && !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
+                && CPUMIsGuestVmxCurrentVmcsValid(&pVCpu->cpum.GstCtx))
+            {
+                /* Paranoia. */
+                Assert(!pVmxTransient->fIsNestedGuest);
+
+                /*
+                 * For performance reasons, also check if the nested hypervisor's current VMCS
+                 * was newly loaded or modified before copying it to the shadow VMCS.
+                 */
+                if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
+                {
+                    int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
+                    AssertRCReturn(rc, rc);
+                    pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
+                }
+                hmR0VmxEnableVmcsShadowing(pVmcsInfo);
+            }
+            else
+                hmR0VmxDisableVmcsShadowing(pVmcsInfo);
+        }
+#else
+        NOREF(pVmxTransient);
+#endif
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exports the guest CR0 control register into the guest-state area in the VMCS.
+ *
+ * The guest FPU state is always pre-loaded hence we don't need to bother about
+ * sharing FPU related CR0 bits between the guest and host.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0VmxExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
+    {
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+
+        uint64_t       fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0;
+        uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
+        if (pVM->hm.s.vmx.fUnrestrictedGuest)
+            fSetCr0 &= ~(uint64_t)(X86_CR0_PE | X86_CR0_PG);
+        else
+            Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
+
+        if (!pVmxTransient->fIsNestedGuest)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+            uint64_t       u64GuestCr0  = pVCpu->cpum.GstCtx.cr0;
+            uint64_t const u64ShadowCr0 = u64GuestCr0;
+            Assert(!RT_HI_U32(u64GuestCr0));
+
+            /*
+             * Setup VT-x's view of the guest CR0.
+             */
+            uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
+            if (pVM->hm.s.fNestedPaging)
+            {
+                if (CPUMIsGuestPagingEnabled(pVCpu))
+                {
+                    /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
+                    uProcCtls &= ~(  VMX_PROC_CTLS_CR3_LOAD_EXIT
+                                   | VMX_PROC_CTLS_CR3_STORE_EXIT);
+                }
+                else
+                {
+                    /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
+                    uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
+                              |  VMX_PROC_CTLS_CR3_STORE_EXIT;
+                }
+
+                /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
+                if (pVM->hm.s.vmx.fUnrestrictedGuest)
+                    uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
+            }
+            else
+            {
+                /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
+                u64GuestCr0 |= X86_CR0_WP;
+            }
+
+            /*
+             * Guest FPU bits.
+             *
+             * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
+             * using CR0.TS.
+             *
+             * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
+             * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
+             */
+            u64GuestCr0 |= X86_CR0_NE;
+
+            /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
+            bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
+
+            /*
+             * Update exception intercepts.
+             */
+            uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
+            if (pVmcsInfo->RealMode.fRealOnV86Active)
+            {
+                Assert(PDMVmmDevHeapIsEnabled(pVM));
+                Assert(pVM->hm.s.vmx.pRealModeTSS);
+                uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
+            }
+            else
+            {
+                /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
+                uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
+                if (fInterceptMF)
+                    uXcptBitmap |= RT_BIT(X86_XCPT_MF);
+            }
+
+            /* Additional intercepts for debugging, define these yourself explicitly. */
+#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
+            uXcptBitmap |= 0
+                        |  RT_BIT(X86_XCPT_BP)
+                        |  RT_BIT(X86_XCPT_DE)
+                        |  RT_BIT(X86_XCPT_NM)
+                        |  RT_BIT(X86_XCPT_TS)
+                        |  RT_BIT(X86_XCPT_UD)
+                        |  RT_BIT(X86_XCPT_NP)
+                        |  RT_BIT(X86_XCPT_SS)
+                        |  RT_BIT(X86_XCPT_GP)
+                        |  RT_BIT(X86_XCPT_PF)
+                        |  RT_BIT(X86_XCPT_MF)
+                        ;
+#elif defined(HMVMX_ALWAYS_TRAP_PF)
+            uXcptBitmap |= RT_BIT(X86_XCPT_PF);
+#endif
+            if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
+                uXcptBitmap |= RT_BIT(X86_XCPT_GP);
+            Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
+
+            /* Apply the hardware specified CR0 fixed bits and enable caching. */
+            u64GuestCr0 |= fSetCr0;
+            u64GuestCr0 &= fZapCr0;
+            u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD | X86_CR0_NW);
+
+            /* Commit the CR0 and related fields to the guest VMCS. */
+            int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0);               AssertRC(rc);
+            rc     = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);   AssertRC(rc);
+            if (uProcCtls != pVmcsInfo->u32ProcCtls)
+            {
+                rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
+                AssertRC(rc);
+            }
+            if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
+            {
+                rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
+                AssertRC(rc);
+            }
+
+            /* Update our caches. */
+            pVmcsInfo->u32ProcCtls   = uProcCtls;
+            pVmcsInfo->u32XcptBitmap = uXcptBitmap;
+
+            Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
+        }
+        else
+        {
+            /*
+             * With nested-guests, we may have extended the guest/host mask here since we
+             * merged in the outer guest's mask. Thus, the merged mask can include more bits
+             * (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
+             * originally supplied. We must copy those bits from the nested-guest CR0 into
+             * the nested-guest CR0 read-shadow.
+             */
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+            uint64_t       u64GuestCr0  = pVCpu->cpum.GstCtx.cr0;
+            uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
+            Assert(!RT_HI_U32(u64GuestCr0));
+            Assert(u64GuestCr0 & X86_CR0_NE);
+
+            /* Apply the hardware specified CR0 fixed bits and enable caching. */
+            u64GuestCr0 |= fSetCr0;
+            u64GuestCr0 &= fZapCr0;
+            u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD | X86_CR0_NW);
+
+            /* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
+            int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0);               AssertRC(rc);
+            rc     = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);   AssertRC(rc);
+
+            Log4Func(("cr0=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
+        }
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exports the guest control registers (CR3, CR4) into the guest-state area
+ * in the VMCS.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
+ *          without unrestricted guest access and the VMMDev is not presently
+ *          mapped (e.g. EFI32).
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    int rc  = VINF_SUCCESS;
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Guest CR2.
+     * It's always loaded in the assembler code. Nothing to do here.
+     */
+
+    /*
+     * Guest CR3.
+     */
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
+
+        if (pVM->hm.s.fNestedPaging)
+        {
+            PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+            pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
+
+            /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
+            Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
+            Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
+            Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
+
+            /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
+            pVmcsInfo->HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
+                                  |  (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
+
+            /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
+            AssertMsg(   ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3      /* Bits 3:5 (EPT page walk length - 1) must be 3. */
+                      && ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0,     /* Bits 7:11 MBZ. */
+                         ("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
+            AssertMsg(  !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01)           /* Bit 6 (EPT accessed & dirty bit). */
+                      || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
+                         ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
+
+            rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
+            AssertRC(rc);
+
+            uint64_t  u64GuestCr3;
+            PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+            if (   pVM->hm.s.vmx.fUnrestrictedGuest
+                || CPUMIsGuestPagingEnabledEx(pCtx))
+            {
+                /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
+                if (CPUMIsGuestInPAEModeEx(pCtx))
+                {
+                    rc  = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
+                    AssertRC(rc);
+                    rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);     AssertRC(rc);
+                    rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);     AssertRC(rc);
+                    rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);     AssertRC(rc);
+                    rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);     AssertRC(rc);
+                }
+
+                /*
+                 * The guest's view of its CR3 is unblemished with nested paging when the
+                 * guest is using paging or we have unrestricted guest execution to handle
+                 * the guest when it's not using paging.
+                 */
+                u64GuestCr3 = pCtx->cr3;
+            }
+            else
+            {
+                /*
+                 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
+                 * thinks it accesses physical memory directly, we use our identity-mapped
+                 * page table to map guest-linear to guest-physical addresses. EPT takes care
+                 * of translating it to host-physical addresses.
+                 */
+                RTGCPHYS GCPhys;
+                Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
+
+                /* We obtain it here every time as the guest could have relocated this PCI region. */
+                rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
+                if (RT_SUCCESS(rc))
+                { /* likely */ }
+                else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
+                {
+                    Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
+                    return VINF_EM_RESCHEDULE_REM;  /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
+                }
+                else
+                    AssertMsgFailedReturn(("%Rrc\n",  rc), rc);
+
+                u64GuestCr3 = GCPhys;
+            }
+
+            Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
+            rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, u64GuestCr3);
+            AssertRC(rc);
+        }
+        else
+        {
+            Assert(!pVmxTransient->fIsNestedGuest);
+            /* Non-nested paging case, just use the hypervisor's CR3. */
+            RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
+
+            Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
+            rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
+            AssertRC(rc);
+        }
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
+    }
+
+    /*
+     * Guest CR4.
+     * ASSUMES this is done everytime we get in from ring-3! (XCR0)
+     */
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
+    {
+        PCPUMCTX     pCtx      = &pVCpu->cpum.GstCtx;
+        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+
+        uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0;
+        uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
+
+        /*
+         * With nested-guests, we may have extended the guest/host mask here (since we
+         * merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
+         * mask can include more bits (to read from the nested-guest CR4 read-shadow) than
+         * the nested hypervisor originally supplied. Thus, we should, in essence, copy
+         * those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
+         */
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+        uint64_t       u64GuestCr4  = pCtx->cr4;
+        uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
+                                    ? pCtx->cr4
+                                    : CPUMGetGuestVmxMaskedCr4(pCtx, pVmcsInfo->u64Cr4Mask);
+        Assert(!RT_HI_U32(u64GuestCr4));
+
+        /*
+         * Setup VT-x's view of the guest CR4.
+         *
+         * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
+         * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
+         * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
+         *
+         * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
+         */
+        if (pVmcsInfo->RealMode.fRealOnV86Active)
+        {
+            Assert(pVM->hm.s.vmx.pRealModeTSS);
+            Assert(PDMVmmDevHeapIsEnabled(pVM));
+            u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
+        }
+
+        if (pVM->hm.s.fNestedPaging)
+        {
+            if (   !CPUMIsGuestPagingEnabledEx(pCtx)
+                && !pVM->hm.s.vmx.fUnrestrictedGuest)
+            {
+                /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
+                u64GuestCr4 |= X86_CR4_PSE;
+                /* Our identity mapping is a 32-bit page directory. */
+                u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
+            }
+            /* else use guest CR4.*/
+        }
+        else
+        {
+            Assert(!pVmxTransient->fIsNestedGuest);
+
+            /*
+             * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
+             * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
+             */
+            switch (pVCpu->hm.s.enmShadowMode)
+            {
+                case PGMMODE_REAL:              /* Real-mode. */
+                case PGMMODE_PROTECTED:         /* Protected mode without paging. */
+                case PGMMODE_32_BIT:            /* 32-bit paging. */
+                {
+                    u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
+                    break;
+                }
+
+                case PGMMODE_PAE:               /* PAE paging. */
+                case PGMMODE_PAE_NX:            /* PAE paging with NX. */
+                {
+                    u64GuestCr4 |= X86_CR4_PAE;
+                    break;
+                }
+
+                case PGMMODE_AMD64:             /* 64-bit AMD paging (long mode). */
+                case PGMMODE_AMD64_NX:          /* 64-bit AMD paging (long mode) with NX enabled. */
+                {
+#ifdef VBOX_WITH_64_BITS_GUESTS
+                    /* For our assumption in hmR0VmxShouldSwapEferMsr. */
+                    Assert(u64GuestCr4 & X86_CR4_PAE);
+                    break;
+#endif
+                }
+                default:
+                    AssertFailed();
+                    return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
+            }
+        }
+
+        /* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
+        u64GuestCr4 |= fSetCr4;
+        u64GuestCr4 &= fZapCr4;
+
+        /* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
+        rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR4, u64GuestCr4);               AssertRC(rc);
+        rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);   AssertRC(rc);
+
+        /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
+        pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
+
+        Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
+    }
+    return rc;
+}
+
+
+/**
+ * Exports the guest debug registers into the guest-state area in the VMCS.
+ * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
+ *
+ * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0VmxExportSharedDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    /** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
+     *        stepping. */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    if (pVmxTransient->fIsNestedGuest)
+    {
+        int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
+        AssertRC(rc);
+
+        /* Always intercept Mov DRx accesses for the nested-guest for now. */
+        pVmcsInfo->u32ProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
+        rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+        AssertRC(rc);
+        return VINF_SUCCESS;
+    }
+
+#ifdef VBOX_STRICT
+    /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
+    if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
+    {
+        /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
+        Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
+        Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
+    }
+#endif
+
+    bool     fSteppingDB      = false;
+    bool     fInterceptMovDRx = false;
+    uint32_t uProcCtls        = pVmcsInfo->u32ProcCtls;
+    if (pVCpu->hm.s.fSingleInstruction)
+    {
+        /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
+        {
+            uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
+            Assert(fSteppingDB == false);
+        }
+        else
+        {
+            pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
+            pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
+            pVCpu->hm.s.fClearTrapFlag = true;
+            fSteppingDB = true;
+        }
+    }
+
+    uint64_t u64GuestDr7;
+    if (   fSteppingDB
+        || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
+    {
+        /*
+         * Use the combined guest and host DRx values found in the hypervisor register set
+         * because the hypervisor debugger has breakpoints active or someone is single stepping
+         * on the host side without a monitor trap flag.
+         *
+         * Note! DBGF expects a clean DR6 state before executing guest code.
+         */
+        if (!CPUMIsHyperDebugStateActive(pVCpu))
+        {
+            CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
+            Assert(CPUMIsHyperDebugStateActive(pVCpu));
+            Assert(!CPUMIsGuestDebugStateActive(pVCpu));
+        }
+
+        /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
+        u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
+        pVCpu->hm.s.fUsingHyperDR7 = true;
+        fInterceptMovDRx = true;
+    }
+    else
+    {
+        /*
+         * If the guest has enabled debug registers, we need to load them prior to
+         * executing guest code so they'll trigger at the right time.
+         */
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
+        if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
+        {
+            if (!CPUMIsGuestDebugStateActive(pVCpu))
+            {
+                CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
+                Assert(CPUMIsGuestDebugStateActive(pVCpu));
+                Assert(!CPUMIsHyperDebugStateActive(pVCpu));
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
+            }
+            Assert(!fInterceptMovDRx);
+        }
+        else if (!CPUMIsGuestDebugStateActive(pVCpu))
+        {
+            /*
+             * If no debugging enabled, we'll lazy load DR0-3.  Unlike on AMD-V, we
+             * must intercept #DB in order to maintain a correct DR6 guest value, and
+             * because we need to intercept it to prevent nested #DBs from hanging the
+             * CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
+             */
+            fInterceptMovDRx = true;
+        }
+
+        /* Update DR7 with the actual guest value. */
+        u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
+        pVCpu->hm.s.fUsingHyperDR7 = false;
+    }
+
+    if (fInterceptMovDRx)
+        uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
+    else
+        uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
+
+    /*
+     * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
+     * monitor-trap flag and update our cache.
+     */
+    if (uProcCtls != pVmcsInfo->u32ProcCtls)
+    {
+        int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
+        AssertRC(rc);
+        pVmcsInfo->u32ProcCtls = uProcCtls;
+    }
+
+    /*
+     * Update guest DR7.
+     */
+    int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, u64GuestDr7);
+    AssertRC(rc);
+
+    /*
+     * If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
+     * we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
+     *
+     * See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
+     */
+    if (fSteppingDB)
+    {
+        Assert(pVCpu->hm.s.fSingleInstruction);
+        Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
+
+        uint32_t fIntrState = 0;
+        rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
+        AssertRC(rc);
+
+        if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
+        {
+            fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
+            rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
+            AssertRC(rc);
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+#ifdef VBOX_STRICT
+/**
+ * Strict function to validate segment registers.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Will import guest CR0 on strict builds during validation of
+ *          segments.
+ */
+static void hmR0VmxValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    /*
+     * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
+     *
+     * The reason we check for attribute value 0 in this function and not just the unusable bit is
+     * because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
+     * unusable bit and doesn't change the guest-context value.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
+    if (   !pVM->hm.s.vmx.fUnrestrictedGuest
+        && (   !CPUMIsGuestInRealModeEx(pCtx)
+            && !CPUMIsGuestInV86ModeEx(pCtx)))
+    {
+        /* Protected mode checks */
+        /* CS */
+        Assert(pCtx->cs.Attr.n.u1Present);
+        Assert(!(pCtx->cs.Attr.u & 0xf00));
+        Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
+        Assert(   (pCtx->cs.u32Limit & 0xfff) == 0xfff
+               || !(pCtx->cs.Attr.n.u1Granularity));
+        Assert(   !(pCtx->cs.u32Limit & 0xfff00000)
+               || (pCtx->cs.Attr.n.u1Granularity));
+        /* CS cannot be loaded with NULL in protected mode. */
+        Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
+        if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
+            Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
+        else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
+            Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
+        else
+            AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
+        /* SS */
+        Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
+        Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
+        if (   !(pCtx->cr0 & X86_CR0_PE)
+            || pCtx->cs.Attr.n.u4Type == 3)
+        {
+            Assert(!pCtx->ss.Attr.n.u2Dpl);
+        }
+        if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
+        {
+            Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
+            Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
+            Assert(pCtx->ss.Attr.n.u1Present);
+            Assert(!(pCtx->ss.Attr.u & 0xf00));
+            Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
+            Assert(   (pCtx->ss.u32Limit & 0xfff) == 0xfff
+                   || !(pCtx->ss.Attr.n.u1Granularity));
+            Assert(   !(pCtx->ss.u32Limit & 0xfff00000)
+                   || (pCtx->ss.Attr.n.u1Granularity));
+        }
+        /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
+        if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
+        {
+            Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
+            Assert(pCtx->ds.Attr.n.u1Present);
+            Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
+            Assert(!(pCtx->ds.Attr.u & 0xf00));
+            Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
+            Assert(   (pCtx->ds.u32Limit & 0xfff) == 0xfff
+                   || !(pCtx->ds.Attr.n.u1Granularity));
+            Assert(   !(pCtx->ds.u32Limit & 0xfff00000)
+                   || (pCtx->ds.Attr.n.u1Granularity));
+            Assert(   !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                   || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
+        }
+        if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
+        {
+            Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
+            Assert(pCtx->es.Attr.n.u1Present);
+            Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
+            Assert(!(pCtx->es.Attr.u & 0xf00));
+            Assert(!(pCtx->es.Attr.u & 0xfffe0000));
+            Assert(   (pCtx->es.u32Limit & 0xfff) == 0xfff
+                   || !(pCtx->es.Attr.n.u1Granularity));
+            Assert(   !(pCtx->es.u32Limit & 0xfff00000)
+                   || (pCtx->es.Attr.n.u1Granularity));
+            Assert(   !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                   || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
+        }
+        if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
+        {
+            Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
+            Assert(pCtx->fs.Attr.n.u1Present);
+            Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
+            Assert(!(pCtx->fs.Attr.u & 0xf00));
+            Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
+            Assert(   (pCtx->fs.u32Limit & 0xfff) == 0xfff
+                   || !(pCtx->fs.Attr.n.u1Granularity));
+            Assert(   !(pCtx->fs.u32Limit & 0xfff00000)
+                   || (pCtx->fs.Attr.n.u1Granularity));
+            Assert(   !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                   || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
+        }
+        if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
+        {
+            Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
+            Assert(pCtx->gs.Attr.n.u1Present);
+            Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
+            Assert(!(pCtx->gs.Attr.u & 0xf00));
+            Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
+            Assert(   (pCtx->gs.u32Limit & 0xfff) == 0xfff
+                   || !(pCtx->gs.Attr.n.u1Granularity));
+            Assert(   !(pCtx->gs.u32Limit & 0xfff00000)
+                   || (pCtx->gs.Attr.n.u1Granularity));
+            Assert(   !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                   || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
+        }
+        /* 64-bit capable CPUs. */
+        Assert(!RT_HI_U32(pCtx->cs.u64Base));
+        Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
+        Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
+        Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
+    }
+    else if (   CPUMIsGuestInV86ModeEx(pCtx)
+             || (   CPUMIsGuestInRealModeEx(pCtx)
+                 && !pVM->hm.s.vmx.fUnrestrictedGuest))
+    {
+        /* Real and v86 mode checks. */
+        /* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
+        uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
+        if (pVmcsInfo->RealMode.fRealOnV86Active)
+        {
+            u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
+            u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
+        }
+        else
+        {
+            u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
+            u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
+        }
+
+        /* CS */
+        AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
+        Assert(pCtx->cs.u32Limit == 0xffff);
+        Assert(u32CSAttr == 0xf3);
+        /* SS */
+        Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
+        Assert(pCtx->ss.u32Limit == 0xffff);
+        Assert(u32SSAttr == 0xf3);
+        /* DS */
+        Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
+        Assert(pCtx->ds.u32Limit == 0xffff);
+        Assert(u32DSAttr == 0xf3);
+        /* ES */
+        Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
+        Assert(pCtx->es.u32Limit == 0xffff);
+        Assert(u32ESAttr == 0xf3);
+        /* FS */
+        Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
+        Assert(pCtx->fs.u32Limit == 0xffff);
+        Assert(u32FSAttr == 0xf3);
+        /* GS */
+        Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
+        Assert(pCtx->gs.u32Limit == 0xffff);
+        Assert(u32GSAttr == 0xf3);
+        /* 64-bit capable CPUs. */
+        Assert(!RT_HI_U32(pCtx->cs.u64Base));
+        Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
+        Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
+        Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
+    }
+}
+#endif /* VBOX_STRICT */
+
+
+/**
+ * Exports a guest segment register into the guest-state area in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ * @param   iSegReg     The segment register number (X86_SREG_XXX).
+ * @param   pSelReg     Pointer to the segment selector.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0VmxExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
+{
+    Assert(iSegReg < X86_SREG_COUNT);
+    uint32_t const idxSel   = g_aVmcsSegSel[iSegReg];
+    uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
+    uint32_t const idxBase  = g_aVmcsSegBase[iSegReg];
+    uint32_t const idxAttr  = g_aVmcsSegAttr[iSegReg];
+
+    uint32_t u32Access = pSelReg->Attr.u;
+    if (pVmcsInfo->RealMode.fRealOnV86Active)
+    {
+        /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
+        u32Access = 0xf3;
+        Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
+        Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
+        RT_NOREF_PV(pVCpu);
+    }
+    else
+    {
+        /*
+         * The way to differentiate between whether this is really a null selector or was just
+         * a selector loaded with 0 in real-mode is using the segment attributes. A selector
+         * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
+         * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
+         * NULL selectors loaded in protected-mode have their attribute as 0.
+         */
+        if (!u32Access)
+            u32Access = X86DESCATTR_UNUSABLE;
+    }
+
+    /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
+    AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
+              ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
+
+    /*
+     * Commit it to the VMCS.
+     */
+    int rc = VMXWriteVmcs32(idxSel,   pSelReg->Sel);        AssertRC(rc);
+    rc     = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);   AssertRC(rc);
+    rc     = VMXWriteVmcsNw(idxBase,  pSelReg->u64Base);    AssertRC(rc);
+    rc     = VMXWriteVmcs32(idxAttr,  u32Access);           AssertRC(rc);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
+ * area in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks Will import guest CR0 on strict builds during validation of
+ *          segments.
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0VmxExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    int   rc  = VERR_INTERNAL_ERROR_5;
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PCCPUMCTX    pCtx      = &pVCpu->cpum.GstCtx;
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+
+    /*
+     * Guest Segment registers: CS, SS, DS, ES, FS, GS.
+     */
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
+    {
+        if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
+            if (pVmcsInfo->RealMode.fRealOnV86Active)
+                pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
+            rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
+        }
+
+        if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
+            if (pVmcsInfo->RealMode.fRealOnV86Active)
+                pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
+            rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
+        }
+
+        if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
+            if (pVmcsInfo->RealMode.fRealOnV86Active)
+                pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
+            rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
+        }
+
+        if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
+            if (pVmcsInfo->RealMode.fRealOnV86Active)
+                pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
+            rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
+        }
+
+        if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
+            if (pVmcsInfo->RealMode.fRealOnV86Active)
+                pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
+            rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
+        }
+
+        if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
+            if (pVmcsInfo->RealMode.fRealOnV86Active)
+                pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
+            rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
+        }
+
+#ifdef VBOX_STRICT
+        hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
+#endif
+        Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
+                  pCtx->cs.Attr.u));
+    }
+
+    /*
+     * Guest TR.
+     */
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
+
+        /*
+         * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
+         * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
+         * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
+         */
+        uint16_t u16Sel;
+        uint32_t u32Limit;
+        uint64_t u64Base;
+        uint32_t u32AccessRights;
+        if (!pVmcsInfo->RealMode.fRealOnV86Active)
+        {
+            u16Sel          = pCtx->tr.Sel;
+            u32Limit        = pCtx->tr.u32Limit;
+            u64Base         = pCtx->tr.u64Base;
+            u32AccessRights = pCtx->tr.Attr.u;
+        }
+        else
+        {
+            Assert(!pVmxTransient->fIsNestedGuest);
+            Assert(pVM->hm.s.vmx.pRealModeTSS);
+            Assert(PDMVmmDevHeapIsEnabled(pVM));    /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
+
+            /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
+            RTGCPHYS GCPhys;
+            rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
+            AssertRCReturn(rc, rc);
+
+            X86DESCATTR DescAttr;
+            DescAttr.u           = 0;
+            DescAttr.n.u1Present = 1;
+            DescAttr.n.u4Type    = X86_SEL_TYPE_SYS_386_TSS_BUSY;
+
+            u16Sel          = 0;
+            u32Limit        = HM_VTX_TSS_SIZE;
+            u64Base         = GCPhys;
+            u32AccessRights = DescAttr.u;
+        }
+
+        /* Validate. */
+        Assert(!(u16Sel & RT_BIT(2)));
+        AssertMsg(   (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
+                  || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
+        AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
+        Assert(!(u32AccessRights & RT_BIT(4)));                 /* System MBZ.*/
+        Assert(u32AccessRights & RT_BIT(7));                    /* Present MB1.*/
+        Assert(!(u32AccessRights & 0xf00));                     /* 11:8 MBZ. */
+        Assert(!(u32AccessRights & 0xfffe0000));                /* 31:17 MBZ. */
+        Assert(   (u32Limit & 0xfff) == 0xfff
+               || !(u32AccessRights & RT_BIT(15)));             /* Granularity MBZ. */
+        Assert(   !(pCtx->tr.u32Limit & 0xfff00000)
+               || (u32AccessRights & RT_BIT(15)));              /* Granularity MB1. */
+
+        rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_TR_SEL,           u16Sel);             AssertRC(rc);
+        rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT,         u32Limit);           AssertRC(rc);
+        rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);    AssertRC(rc);
+        rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_TR_BASE,            u64Base);            AssertRC(rc);
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
+        Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
+    }
+
+    /*
+     * Guest GDTR.
+     */
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
+
+        rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);     AssertRC(rc);
+        rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_GDTR_BASE,  pCtx->gdtr.pGdt);        AssertRC(rc);
+
+        /* Validate. */
+        Assert(!(pCtx->gdtr.cbGdt & 0xffff0000));          /* Bits 31:16 MBZ. */
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
+        Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
+    }
+
+    /*
+     * Guest LDTR.
+     */
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
+
+        /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
+        uint32_t u32Access;
+        if (   !pVmxTransient->fIsNestedGuest
+            && !pCtx->ldtr.Attr.u)
+            u32Access = X86DESCATTR_UNUSABLE;
+        else
+            u32Access = pCtx->ldtr.Attr.u;
+
+        rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_LDTR_SEL,           pCtx->ldtr.Sel);       AssertRC(rc);
+        rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT,         pCtx->ldtr.u32Limit);  AssertRC(rc);
+        rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);            AssertRC(rc);
+        rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_LDTR_BASE,            pCtx->ldtr.u64Base);   AssertRC(rc);
+
+        /* Validate. */
+        if (!(u32Access & X86DESCATTR_UNUSABLE))
+        {
+            Assert(!(pCtx->ldtr.Sel & RT_BIT(2)));              /* TI MBZ. */
+            Assert(pCtx->ldtr.Attr.n.u4Type == 2);              /* Type MB2 (LDT). */
+            Assert(!pCtx->ldtr.Attr.n.u1DescType);              /* System MBZ. */
+            Assert(pCtx->ldtr.Attr.n.u1Present == 1);           /* Present MB1. */
+            Assert(!pCtx->ldtr.Attr.n.u4LimitHigh);             /* 11:8 MBZ. */
+            Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000));          /* 31:17 MBZ. */
+            Assert(   (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
+                   || !pCtx->ldtr.Attr.n.u1Granularity);        /* Granularity MBZ. */
+            Assert(   !(pCtx->ldtr.u32Limit & 0xfff00000)
+                   || pCtx->ldtr.Attr.n.u1Granularity);         /* Granularity MB1. */
+        }
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
+        Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
+    }
+
+    /*
+     * Guest IDTR.
+     */
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
+
+        rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);     AssertRC(rc);
+        rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_IDTR_BASE,  pCtx->idtr.pIdt);        AssertRC(rc);
+
+        /* Validate. */
+        Assert(!(pCtx->idtr.cbIdt & 0xffff0000));          /* Bits 31:16 MBZ. */
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
+        Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
+ * areas.
+ *
+ * These MSRs will automatically be loaded to the host CPU on every successful
+ * VM-entry and stored from the host CPU on every successful VM-exit.
+ *
+ * We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
+ * actual host MSR values are not- updated here for performance reasons. See
+ * hmR0VmxExportHostMsrs().
+ *
+ * We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0VmxExportGuestMsrs(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    AssertPtr(pVCpu);
+    AssertPtr(pVmxTransient);
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+
+    /*
+     * MSRs that we use the auto-load/store MSR area in the VMCS.
+     * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(),
+     * nothing to do here. The host MSR values are updated when it's safe in
+     * hmR0VmxLazySaveHostMsrs().
+     *
+     * For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
+     * loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
+     * emulation. The merged MSR permission bitmap will ensure that we get VM-exits
+     * for any MSR that are not part of the lazy MSRs so we do not need to place
+     * those MSRs into the auto-load/store MSR area. Nothing to do here.
+     */
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
+    {
+        /* No auto-load/store MSRs currently. */
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
+    }
+
+    /*
+     * Guest Sysenter MSRs.
+     */
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
+
+        if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
+        {
+            int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
+        }
+
+        if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
+        {
+            int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
+        }
+
+        if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
+        {
+            int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
+        }
+    }
+
+    /*
+     * Guest/host EFER MSR.
+     */
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
+    {
+        /* Whether we are using the VMCS to swap the EFER MSR must have been
+           determined earlier while exporting VM-entry/VM-exit controls. */
+        Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
+
+        if (hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
+        {
+            /*
+             * EFER.LME is written by software, while EFER.LMA is set by the CPU to (CR0.PG & EFER.LME).
+             * This means a guest can set EFER.LME=1 while CR0.PG=0 and EFER.LMA can remain 0.
+             * VT-x requires that "IA-32e mode guest" VM-entry control must be identical to EFER.LMA
+             * and to CR0.PG. Without unrestricted execution, CR0.PG (used for VT-x, not the shadow)
+             * must always be 1. This forces us to effectively clear both EFER.LMA and EFER.LME until
+             * the guest has also set CR0.PG=1. Otherwise, we would run into an invalid-guest state
+             * during VM-entry.
+             */
+            uint64_t uGuestEferMsr = pCtx->msrEFER;
+            if (!pVM->hm.s.vmx.fUnrestrictedGuest)
+            {
+                if (!(pCtx->msrEFER & MSR_K6_EFER_LMA))
+                    uGuestEferMsr &= ~MSR_K6_EFER_LME;
+                else
+                    Assert((pCtx->msrEFER & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
+            }
+
+            /*
+             * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
+             * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
+             */
+            if (pVM->hm.s.vmx.fSupportsVmcsEfer)
+            {
+                int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, uGuestEferMsr);
+                AssertRC(rc);
+            }
+            else
+            {
+                /*
+                 * We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
+                 * continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
+                 */
+                int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, uGuestEferMsr,
+                                                    false /* fSetReadWrite */, false /* fUpdateHostMsr */);
+                AssertRCReturn(rc, rc);
+            }
+
+            Log4Func(("efer=%#RX64 shadow=%#RX64\n", uGuestEferMsr, pCtx->msrEFER));
+        }
+        else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
+            hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
+    }
+
+    /*
+     * Other MSRs.
+     */
+    if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
+    {
+        /* Speculation Control (R/W). */
+        HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
+        if (pVM->cpum.ro.GuestFeatures.fIbrs)
+        {
+            int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
+                                                false /* fSetReadWrite */, false /* fUpdateHostMsr */);
+            AssertRCReturn(rc, rc);
+        }
+
+        /* Last Branch Record. */
+        if (pVM->hm.s.vmx.fLbr)
+        {
+            uint32_t const idFromIpMsrStart = pVM->hm.s.vmx.idLbrFromIpMsrFirst;
+            uint32_t const idToIpMsrStart   = pVM->hm.s.vmx.idLbrToIpMsrFirst;
+            uint32_t const cLbrStack        = pVM->hm.s.vmx.idLbrFromIpMsrLast - pVM->hm.s.vmx.idLbrFromIpMsrFirst + 1;
+            Assert(cLbrStack <= 32);
+            for (uint32_t i = 0; i < cLbrStack; i++)
+            {
+                int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, idFromIpMsrStart + i,
+                                                    pVmxTransient->pVmcsInfo->au64LbrFromIpMsr[i],
+                                                    false /* fSetReadWrite */, false /* fUpdateHostMsr */);
+                AssertRCReturn(rc, rc);
+
+                /* Some CPUs don't have a Branch-To-IP MSR (P4 and related Xeons). */
+                if (idToIpMsrStart != 0)
+                {
+                    rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, idToIpMsrStart + i,
+                                                    pVmxTransient->pVmcsInfo->au64LbrToIpMsr[i],
+                                                    false /* fSetReadWrite */, false /* fUpdateHostMsr */);
+                    AssertRCReturn(rc, rc);
+                }
+            }
+
+            /* Add LBR top-of-stack MSR (which contains the index to the most recent record). */
+            int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, pVM->hm.s.vmx.idLbrTosMsr,
+                                                pVmxTransient->pVmcsInfo->u64LbrTosMsr, false /* fSetReadWrite */,
+                                                false /* fUpdateHostMsr */);
+            AssertRCReturn(rc, rc);
+        }
+
+        ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Wrapper for running the guest code in VT-x.
+ *
+ * @returns VBox status code, no informational status codes.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+DECLINLINE(int) hmR0VmxRunGuest(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
+
+    /** @todo Add stats for VMRESUME vs VMLAUNCH. */
+
+    /*
+     * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
+     * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
+     * callee-saved and thus the need for this XMM wrapper.
+     *
+     * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
+     */
+    PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+#ifdef VBOX_WITH_KERNEL_USING_XMM
+    int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, NULL /*pvUnused*/, pVM, pVCpu, pVmcsInfo->pfnStartVM);
+#else
+    int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, NULL /*pvUnused*/, pVM, pVCpu);
+#endif
+    AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Reports world-switch error and dumps some useful debug info.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   rcVMRun         The return code from VMLAUNCH/VMRESUME.
+ * @param   pVmxTransient   The VMX-transient structure (only
+ *                          exitReason updated).
+ */
+static void hmR0VmxReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
+{
+    Assert(pVCpu);
+    Assert(pVmxTransient);
+    HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
+
+    Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
+    switch (rcVMRun)
+    {
+        case VERR_VMX_INVALID_VMXON_PTR:
+            AssertFailed();
+            break;
+        case VINF_SUCCESS:                  /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
+        case VERR_VMX_UNABLE_TO_START_VM:   /* VMLAUNCH/VMRESUME itself failed. */
+        {
+            int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
+            rc    |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
+            AssertRC(rc);
+            hmR0VmxReadExitQualVmcs(pVmxTransient);
+
+            pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
+            /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
+               Cannot do it here as we may have been long preempted. */
+
+#ifdef VBOX_STRICT
+                PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+                Log4(("uExitReason        %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
+                     pVmxTransient->uExitReason));
+                Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
+                Log4(("InstrError         %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
+                if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
+                    Log4(("InstrError Desc.  \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
+                else
+                    Log4(("InstrError Desc.    Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
+                Log4(("Entered host CPU   %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
+                Log4(("Current host CPU   %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
+
+                static struct
+                {
+                    /** Name of the field to log. */
+                    const char     *pszName;
+                    /** The VMCS field. */
+                    uint32_t        uVmcsField;
+                    /** Whether host support of this field needs to be checked. */
+                    bool            fCheckSupport;
+                } const s_aVmcsFields[] =
+                {
+                    { "VMX_VMCS32_CTRL_PIN_EXEC",                 VMX_VMCS32_CTRL_PIN_EXEC,                   false  },
+                    { "VMX_VMCS32_CTRL_PROC_EXEC",                VMX_VMCS32_CTRL_PROC_EXEC,                  false  },
+                    { "VMX_VMCS32_CTRL_PROC_EXEC2",               VMX_VMCS32_CTRL_PROC_EXEC2,                 true   },
+                    { "VMX_VMCS32_CTRL_ENTRY",                    VMX_VMCS32_CTRL_ENTRY,                      false  },
+                    { "VMX_VMCS32_CTRL_EXIT",                     VMX_VMCS32_CTRL_EXIT,                       false  },
+                    { "VMX_VMCS32_CTRL_CR3_TARGET_COUNT",         VMX_VMCS32_CTRL_CR3_TARGET_COUNT,           false  },
+                    { "VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO",  VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,    false  },
+                    { "VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE",  VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,    false  },
+                    { "VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH",       VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,         false  },
+                    { "VMX_VMCS32_CTRL_TPR_THRESHOLD",            VMX_VMCS32_CTRL_TPR_THRESHOLD,              false  },
+                    { "VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT",     VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,       false  },
+                    { "VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT",      VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,        false  },
+                    { "VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT",     VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,       false  },
+                    { "VMX_VMCS32_CTRL_EXCEPTION_BITMAP",         VMX_VMCS32_CTRL_EXCEPTION_BITMAP,           false  },
+                    { "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK",     VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,       false  },
+                    { "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH",    VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,      false  },
+                    { "VMX_VMCS_CTRL_CR0_MASK",                   VMX_VMCS_CTRL_CR0_MASK,                     false  },
+                    { "VMX_VMCS_CTRL_CR0_READ_SHADOW",            VMX_VMCS_CTRL_CR0_READ_SHADOW,              false  },
+                    { "VMX_VMCS_CTRL_CR4_MASK",                   VMX_VMCS_CTRL_CR4_MASK,                     false  },
+                    { "VMX_VMCS_CTRL_CR4_READ_SHADOW",            VMX_VMCS_CTRL_CR4_READ_SHADOW,              false  },
+                    { "VMX_VMCS64_CTRL_EPTP_FULL",                VMX_VMCS64_CTRL_EPTP_FULL,                  true   },
+                    { "VMX_VMCS_GUEST_RIP",                       VMX_VMCS_GUEST_RIP,                         false  },
+                    { "VMX_VMCS_GUEST_RSP",                       VMX_VMCS_GUEST_RSP,                         false  },
+                    { "VMX_VMCS_GUEST_RFLAGS",                    VMX_VMCS_GUEST_RFLAGS,                      false  },
+                    { "VMX_VMCS16_VPID",                          VMX_VMCS16_VPID,                            true,  },
+                    { "VMX_VMCS_HOST_CR0",                        VMX_VMCS_HOST_CR0,                          false  },
+                    { "VMX_VMCS_HOST_CR3",                        VMX_VMCS_HOST_CR3,                          false  },
+                    { "VMX_VMCS_HOST_CR4",                        VMX_VMCS_HOST_CR4,                          false  },
+                    /* The order of selector fields below are fixed! */
+                    { "VMX_VMCS16_HOST_ES_SEL",                   VMX_VMCS16_HOST_ES_SEL,                     false  },
+                    { "VMX_VMCS16_HOST_CS_SEL",                   VMX_VMCS16_HOST_CS_SEL,                     false  },
+                    { "VMX_VMCS16_HOST_SS_SEL",                   VMX_VMCS16_HOST_SS_SEL,                     false  },
+                    { "VMX_VMCS16_HOST_DS_SEL",                   VMX_VMCS16_HOST_DS_SEL,                     false  },
+                    { "VMX_VMCS16_HOST_FS_SEL",                   VMX_VMCS16_HOST_FS_SEL,                     false  },
+                    { "VMX_VMCS16_HOST_GS_SEL",                   VMX_VMCS16_HOST_GS_SEL,                     false  },
+                    { "VMX_VMCS16_HOST_TR_SEL",                   VMX_VMCS16_HOST_TR_SEL,                     false  },
+                    /* End of ordered selector fields. */
+                    { "VMX_VMCS_HOST_TR_BASE",                    VMX_VMCS_HOST_TR_BASE,                      false  },
+                    { "VMX_VMCS_HOST_GDTR_BASE",                  VMX_VMCS_HOST_GDTR_BASE,                    false  },
+                    { "VMX_VMCS_HOST_IDTR_BASE",                  VMX_VMCS_HOST_IDTR_BASE,                    false  },
+                    { "VMX_VMCS32_HOST_SYSENTER_CS",              VMX_VMCS32_HOST_SYSENTER_CS,                false  },
+                    { "VMX_VMCS_HOST_SYSENTER_EIP",               VMX_VMCS_HOST_SYSENTER_EIP,                 false  },
+                    { "VMX_VMCS_HOST_SYSENTER_ESP",               VMX_VMCS_HOST_SYSENTER_ESP,                 false  },
+                    { "VMX_VMCS_HOST_RSP",                        VMX_VMCS_HOST_RSP,                          false  },
+                    { "VMX_VMCS_HOST_RIP",                        VMX_VMCS_HOST_RIP,                          false  }
+                };
+
+                RTGDTR      HostGdtr;
+                ASMGetGDTR(&HostGdtr);
+
+                uint32_t const cVmcsFields = RT_ELEMENTS(s_aVmcsFields);
+                for (uint32_t i = 0; i < cVmcsFields; i++)
+                {
+                    uint32_t const uVmcsField = s_aVmcsFields[i].uVmcsField;
+
+                    bool fSupported;
+                    if (!s_aVmcsFields[i].fCheckSupport)
+                        fSupported = true;
+                    else
+                    {
+                        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+                        switch (uVmcsField)
+                        {
+                            case VMX_VMCS64_CTRL_EPTP_FULL:  fSupported = pVM->hm.s.fNestedPaging;      break;
+                            case VMX_VMCS16_VPID:            fSupported = pVM->hm.s.vmx.fVpid;          break;
+                            case VMX_VMCS32_CTRL_PROC_EXEC2:
+                                fSupported = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
+                                break;
+                            default:
+                                AssertMsgFailedReturnVoid(("Failed to provide VMCS field support for %#RX32\n", uVmcsField));
+                        }
+                    }
+
+                    if (fSupported)
+                    {
+                        uint8_t const uWidth = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_WIDTH);
+                        switch (uWidth)
+                        {
+                            case VMX_VMCSFIELD_WIDTH_16BIT:
+                            {
+                                uint16_t u16Val;
+                                rc = VMXReadVmcs16(uVmcsField, &u16Val);
+                                AssertRC(rc);
+                                Log4(("%-40s = %#RX16\n", s_aVmcsFields[i].pszName, u16Val));
+
+                                if (   uVmcsField >= VMX_VMCS16_HOST_ES_SEL
+                                    && uVmcsField <= VMX_VMCS16_HOST_TR_SEL)
+                                {
+                                    if (u16Val < HostGdtr.cbGdt)
+                                    {
+                                        /* Order of selectors in s_apszSel is fixed and matches the order in s_aVmcsFields. */
+                                        static const char * const s_apszSel[] = { "Host ES", "Host CS", "Host SS", "Host DS",
+                                                                                  "Host FS", "Host GS", "Host TR" };
+                                        uint8_t const idxSel = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_INDEX);
+                                        Assert(idxSel < RT_ELEMENTS(s_apszSel));
+                                        PCX86DESCHC pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u16Val & X86_SEL_MASK));
+                                        hmR0DumpDescriptor(pDesc, u16Val, s_apszSel[idxSel]);
+                                    }
+                                    else
+                                        Log4(("  Selector value exceeds GDT limit!\n"));
+                                }
+                                break;
+                            }
+
+                            case VMX_VMCSFIELD_WIDTH_32BIT:
+                            {
+                                uint32_t u32Val;
+                                rc = VMXReadVmcs32(uVmcsField, &u32Val);
+                                AssertRC(rc);
+                                Log4(("%-40s = %#RX32\n", s_aVmcsFields[i].pszName, u32Val));
+                                break;
+                            }
+
+                            case VMX_VMCSFIELD_WIDTH_64BIT:
+                            case VMX_VMCSFIELD_WIDTH_NATURAL:
+                            {
+                                uint64_t u64Val;
+                                rc = VMXReadVmcs64(uVmcsField, &u64Val);
+                                AssertRC(rc);
+                                Log4(("%-40s = %#RX64\n", s_aVmcsFields[i].pszName, u64Val));
+                                break;
+                            }
+                        }
+                    }
+                }
+
+                Log4(("MSR_K6_EFER            = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
+                Log4(("MSR_K8_CSTAR           = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
+                Log4(("MSR_K8_LSTAR           = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
+                Log4(("MSR_K6_STAR            = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
+                Log4(("MSR_K8_SF_MASK         = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
+                Log4(("MSR_K8_KERNEL_GS_BASE  = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
+#endif /* VBOX_STRICT */
+            break;
+        }
+
+        default:
+            /* Impossible */
+            AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
+            break;
+    }
+}
+
+
+/**
+ * Sets up the usage of TSC-offsetting and updates the VMCS.
+ *
+ * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
+ * VMX-preemption timer.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    bool         fOffsettedTsc;
+    bool         fParavirtTsc;
+    uint64_t     uTscOffset;
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+
+    if (pVM->hm.s.vmx.fUsePreemptTimer)
+    {
+        uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
+
+        /* Make sure the returned values have sane upper and lower boundaries. */
+        uint64_t u64CpuHz  = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
+        cTicksToDeadline   = RT_MIN(cTicksToDeadline, u64CpuHz / 64);      /* 1/64th of a second,  15.625ms. */
+        cTicksToDeadline   = RT_MAX(cTicksToDeadline, u64CpuHz / 32768);   /* 1/32768th of a second,  ~30us. */
+        cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
+
+        /** @todo r=ramshankar: We need to find a way to integrate nested-guest
+         *        preemption timers here. We probably need to clamp the preemption timer,
+         *        after converting the timer value to the host. */
+        uint32_t const cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
+        int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
+        AssertRC(rc);
+    }
+    else
+        fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
+
+    if (fParavirtTsc)
+    {
+        /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
+           information before every VM-entry, hence disable it for performance sake. */
+#if 0
+        int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
+        AssertRC(rc);
+#endif
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
+    }
+
+    if (   fOffsettedTsc
+        && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
+    {
+        if (pVmxTransient->fIsNestedGuest)
+            uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
+        hmR0VmxSetTscOffsetVmcs(pVmcsInfo, uTscOffset);
+        hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
+    }
+    else
+    {
+        /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
+        hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
+    }
+}
+
+
+/**
+ * Gets the IEM exception flags for the specified vector and IDT vectoring /
+ * VM-exit interruption info type.
+ *
+ * @returns The IEM exception flags.
+ * @param   uVector         The event vector.
+ * @param   uVmxEventType   The VMX event type.
+ *
+ * @remarks This function currently only constructs flags required for
+ *          IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
+ *          and CR2 aspects of an exception are not included).
+ */
+static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
+{
+    uint32_t fIemXcptFlags;
+    switch (uVmxEventType)
+    {
+        case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
+        case VMX_IDT_VECTORING_INFO_TYPE_NMI:
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
+            break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
+            break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
+            break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
+        {
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
+            if (uVector == X86_XCPT_BP)
+                fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
+            else if (uVector == X86_XCPT_OF)
+                fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
+            else
+            {
+                fIemXcptFlags = 0;
+                AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
+            }
+            break;
+        }
+
+        case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
+            break;
+
+        default:
+            fIemXcptFlags = 0;
+            AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
+            break;
+    }
+    return fIemXcptFlags;
+}
+
+
+/**
+ * Sets an event as a pending event to be injected into the guest.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   u32IntInfo          The VM-entry interruption-information field.
+ * @param   cbInstr             The VM-entry instruction length in bytes (for
+ *                              software interrupts, exceptions and privileged
+ *                              software exceptions).
+ * @param   u32ErrCode          The VM-entry exception error code.
+ * @param   GCPtrFaultAddress   The fault-address (CR2) in case it's a
+ *                              page-fault.
+ */
+DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
+                                        RTGCUINTPTR GCPtrFaultAddress)
+{
+    Assert(!pVCpu->hm.s.Event.fPending);
+    pVCpu->hm.s.Event.fPending          = true;
+    pVCpu->hm.s.Event.u64IntInfo        = u32IntInfo;
+    pVCpu->hm.s.Event.u32ErrCode        = u32ErrCode;
+    pVCpu->hm.s.Event.cbInstr           = cbInstr;
+    pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
+}
+
+
+/**
+ * Sets an external interrupt as pending-for-injection into the VM.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u8Interrupt     The external interrupt vector.
+ */
+DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR,          u8Interrupt)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPUCC pVCpu)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_NMI)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_ENTRY_INT_INFO_TYPE_NMI)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPUCC pVCpu)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_DF)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPUCC pVCpu)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_UD)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets a debug (\#DB) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPUCC pVCpu)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_DB)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   u32ErrCode  The error code for the general-protection exception.
+ */
+DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_GP)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets a stack (\#SS) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   u32ErrCode  The error code for the stack exception.
+ */
+DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_SS)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/**
+ * Fixes up attributes for the specified segment register.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pSelReg     The segment register that needs fixing.
+ * @param   idxSel      The VMCS field for the corresponding segment register.
+ */
+static void hmR0VmxFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
+{
+    Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
+
+    /*
+     * If VT-x marks the segment as unusable, most other bits remain undefined:
+     *   - For CS the L, D and G bits have meaning.
+     *   - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
+     *   - For the remaining data segments no bits are defined.
+     *
+     * The present bit and the unusable bit has been observed to be set at the
+     * same time (the selector was supposed to be invalid as we started executing
+     * a V8086 interrupt in ring-0).
+     *
+     * What should be important for the rest of the VBox code, is that the P bit is
+     * cleared.  Some of the other VBox code recognizes the unusable bit, but
+     * AMD-V certainly don't, and REM doesn't really either.  So, to be on the
+     * safe side here, we'll strip off P and other bits we don't care about.  If
+     * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
+     *
+     * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
+     */
+#ifdef VBOX_STRICT
+    uint32_t const uAttr = pSelReg->Attr.u;
+#endif
+
+    /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
+    pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L    | X86DESCATTR_D  | X86DESCATTR_G
+                     | X86DESCATTR_DPL      | X86DESCATTR_TYPE | X86DESCATTR_DT;
+
+#ifdef VBOX_STRICT
+    VMMRZCallRing3Disable(pVCpu);
+    Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
+# ifdef DEBUG_bird
+    AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
+              ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
+               idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
+# endif
+    VMMRZCallRing3Enable(pVCpu);
+    NOREF(uAttr);
+#endif
+    RT_NOREF2(pVCpu, idxSel);
+}
+
+
+/**
+ * Imports a guest segment register from the current VMCS into the guest-CPU
+ * context.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iSegReg     The segment register number (X86_SREG_XXX).
+ *
+ * @remarks Called with interrupts and/or preemption disabled.
+ */
+static void hmR0VmxImportGuestSegReg(PVMCPUCC pVCpu, uint8_t iSegReg)
+{
+    Assert(iSegReg < X86_SREG_COUNT);
+
+    uint32_t const idxSel   = g_aVmcsSegSel[iSegReg];
+    uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
+    uint32_t const idxAttr  = g_aVmcsSegAttr[iSegReg];
+    uint32_t const idxBase  = g_aVmcsSegBase[iSegReg];
+
+    uint16_t u16Sel;
+    uint64_t u64Base;
+    uint32_t u32Limit, u32Attr;
+    int rc = VMXReadVmcs16(idxSel,   &u16Sel);      AssertRC(rc);
+    rc     = VMXReadVmcs32(idxLimit, &u32Limit);    AssertRC(rc);
+    rc     = VMXReadVmcs32(idxAttr,  &u32Attr);     AssertRC(rc);
+    rc     = VMXReadVmcsNw(idxBase,  &u64Base);     AssertRC(rc);
+
+    PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
+    pSelReg->Sel      = u16Sel;
+    pSelReg->ValidSel = u16Sel;
+    pSelReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+    pSelReg->u32Limit = u32Limit;
+    pSelReg->u64Base  = u64Base;
+    pSelReg->Attr.u   = u32Attr;
+    if (u32Attr & X86DESCATTR_UNUSABLE)
+        hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
+}
+
+
+/**
+ * Imports the guest LDTR from the current VMCS into the guest-CPU context.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks Called with interrupts and/or preemption disabled.
+ */
+static void hmR0VmxImportGuestLdtr(PVMCPUCC pVCpu)
+{
+    uint16_t u16Sel;
+    uint64_t u64Base;
+    uint32_t u32Limit, u32Attr;
+    int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_LDTR_SEL,           &u16Sel);       AssertRC(rc);
+    rc     = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT,         &u32Limit);     AssertRC(rc);
+    rc     = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);      AssertRC(rc);
+    rc     = VMXReadVmcsNw(VMX_VMCS_GUEST_LDTR_BASE,            &u64Base);      AssertRC(rc);
+
+    pVCpu->cpum.GstCtx.ldtr.Sel      = u16Sel;
+    pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
+    pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
+    pVCpu->cpum.GstCtx.ldtr.u64Base  = u64Base;
+    pVCpu->cpum.GstCtx.ldtr.Attr.u   = u32Attr;
+    if (u32Attr & X86DESCATTR_UNUSABLE)
+        hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
+}
+
+
+/**
+ * Imports the guest TR from the current VMCS into the guest-CPU context.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks Called with interrupts and/or preemption disabled.
+ */
+static void hmR0VmxImportGuestTr(PVMCPUCC pVCpu)
+{
+    uint16_t u16Sel;
+    uint64_t u64Base;
+    uint32_t u32Limit, u32Attr;
+    int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_TR_SEL,           &u16Sel);     AssertRC(rc);
+    rc     = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT,         &u32Limit);   AssertRC(rc);
+    rc     = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);    AssertRC(rc);
+    rc     = VMXReadVmcsNw(VMX_VMCS_GUEST_TR_BASE,            &u64Base);    AssertRC(rc);
+
+    pVCpu->cpum.GstCtx.tr.Sel      = u16Sel;
+    pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
+    pVCpu->cpum.GstCtx.tr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
+    pVCpu->cpum.GstCtx.tr.u64Base  = u64Base;
+    pVCpu->cpum.GstCtx.tr.Attr.u   = u32Attr;
+    /* TR is the only selector that can never be unusable. */
+    Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
+}
+
+
+/**
+ * Imports the guest RIP from the VMCS back into the guest-CPU context.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks Called with interrupts and/or preemption disabled, should not assert!
+ * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
+ *          instead!!!
+ */
+static void hmR0VmxImportGuestRip(PVMCPUCC pVCpu)
+{
+    uint64_t u64Val;
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
+    {
+        int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
+        AssertRC(rc);
+
+        pCtx->rip = u64Val;
+        EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
+        pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
+    }
+}
+
+
+/**
+ * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Called with interrupts and/or preemption disabled, should not assert!
+ * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
+ *          instead!!!
+ */
+static void hmR0VmxImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
+    {
+        uint64_t u64Val;
+        int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
+        AssertRC(rc);
+
+        pCtx->rflags.u64 = u64Val;
+        if (pVmcsInfo->RealMode.fRealOnV86Active)
+        {
+            pCtx->eflags.Bits.u1VM   = 0;
+            pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
+        }
+        pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
+    }
+}
+
+
+/**
+ * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
+ * context.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Called with interrupts and/or preemption disabled, try not to assert and
+ *          do not log!
+ * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
+ *          instead!!!
+ */
+static void hmR0VmxImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+    uint32_t u32Val;
+    int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);    AssertRC(rc);
+    if (!u32Val)
+    {
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+        CPUMSetGuestNmiBlocking(pVCpu, false);
+    }
+    else
+    {
+        /*
+         * We must import RIP here to set our EM interrupt-inhibited state.
+         * We also import RFLAGS as our code that evaluates pending interrupts
+         * before VM-entry requires it.
+         */
+        hmR0VmxImportGuestRip(pVCpu);
+        hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
+
+        if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
+            EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
+        else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+
+        bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
+        CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
+    }
+}
+
+
+/**
+ * Worker for VMXR0ImportStateOnDemand.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ * @param   fWhat       What to import, CPUMCTX_EXTRN_XXX.
+ */
+static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
+{
+    int      rc   = VINF_SUCCESS;
+    PVMCC    pVM  = pVCpu->CTX_SUFF(pVM);
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    uint32_t u32Val;
+
+    /*
+     * Note! This is hack to workaround a mysterious BSOD observed with release builds
+     *       on Windows 10 64-bit hosts. Profile and debug builds are not affected and
+     *       neither are other host platforms.
+     *
+     *       Committing this temporarily as it prevents BSOD.
+     *
+     * Update: This is very likely a compiler optimization bug, see @bugref{9180}.
+     */
+#ifdef RT_OS_WINDOWS
+    if (pVM == 0 || pVM == (void *)(uintptr_t)-1)
+        return VERR_HM_IPE_1;
+#endif
+
+    STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
+
+    /*
+     * We disable interrupts to make the updating of the state and in particular
+     * the fExtrn modification atomic wrt to preemption hooks.
+     */
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+
+    fWhat &= pCtx->fExtrn;
+    if (fWhat)
+    {
+        do
+        {
+            if (fWhat & CPUMCTX_EXTRN_RIP)
+                hmR0VmxImportGuestRip(pVCpu);
+
+            if (fWhat & CPUMCTX_EXTRN_RFLAGS)
+                hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
+
+            if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
+                hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
+
+            if (fWhat & CPUMCTX_EXTRN_RSP)
+            {
+                rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RSP, &pCtx->rsp);
+                AssertRC(rc);
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
+            {
+                bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
+                if (fWhat & CPUMCTX_EXTRN_CS)
+                {
+                    hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
+                    hmR0VmxImportGuestRip(pVCpu);
+                    if (fRealOnV86Active)
+                        pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
+                    EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
+                }
+                if (fWhat & CPUMCTX_EXTRN_SS)
+                {
+                    hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
+                    if (fRealOnV86Active)
+                        pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
+                }
+                if (fWhat & CPUMCTX_EXTRN_DS)
+                {
+                    hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
+                    if (fRealOnV86Active)
+                        pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
+                }
+                if (fWhat & CPUMCTX_EXTRN_ES)
+                {
+                    hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
+                    if (fRealOnV86Active)
+                        pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
+                }
+                if (fWhat & CPUMCTX_EXTRN_FS)
+                {
+                    hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
+                    if (fRealOnV86Active)
+                        pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
+                }
+                if (fWhat & CPUMCTX_EXTRN_GS)
+                {
+                    hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
+                    if (fRealOnV86Active)
+                        pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
+                }
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
+            {
+                if (fWhat & CPUMCTX_EXTRN_LDTR)
+                    hmR0VmxImportGuestLdtr(pVCpu);
+
+                if (fWhat & CPUMCTX_EXTRN_GDTR)
+                {
+                    rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE,    &pCtx->gdtr.pGdt);  AssertRC(rc);
+                    rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);           AssertRC(rc);
+                    pCtx->gdtr.cbGdt = u32Val;
+                }
+
+                /* Guest IDTR. */
+                if (fWhat & CPUMCTX_EXTRN_IDTR)
+                {
+                    rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE,    &pCtx->idtr.pIdt);  AssertRC(rc);
+                    rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);           AssertRC(rc);
+                    pCtx->idtr.cbIdt = u32Val;
+                }
+
+                /* Guest TR. */
+                if (fWhat & CPUMCTX_EXTRN_TR)
+                {
+                    /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
+                       don't need to import that one. */
+                    if (!pVmcsInfo->RealMode.fRealOnV86Active)
+                        hmR0VmxImportGuestTr(pVCpu);
+                }
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_DR7)
+            {
+                if (!pVCpu->hm.s.fUsingHyperDR7)
+                    rc = VMXReadVmcsNw(VMX_VMCS_GUEST_DR7, &pCtx->dr[7]);   AssertRC(rc);
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+            {
+                rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP,  &pCtx->SysEnter.eip);  AssertRC(rc);
+                rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP,  &pCtx->SysEnter.esp);  AssertRC(rc);
+                rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);              AssertRC(rc);
+                pCtx->SysEnter.cs = u32Val;
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+            {
+                if (   pVM->hm.s.fAllow64BitGuests
+                    && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
+                    pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+            {
+                if (   pVM->hm.s.fAllow64BitGuests
+                    && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
+                {
+                    pCtx->msrLSTAR  = ASMRdMsr(MSR_K8_LSTAR);
+                    pCtx->msrSTAR   = ASMRdMsr(MSR_K6_STAR);
+                    pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
+                }
+            }
+
+            if (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
+            {
+                PCVMXAUTOMSR   pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
+                uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
+                Assert(pMsrs);
+                Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
+                Assert(sizeof(*pMsrs) * cMsrs <= X86_PAGE_4K_SIZE);
+                for (uint32_t i = 0; i < cMsrs; i++)
+                {
+                    uint32_t const idMsr = pMsrs[i].u32Msr;
+                    switch (idMsr)
+                    {
+                        case MSR_K8_TSC_AUX:        CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value);     break;
+                        case MSR_IA32_SPEC_CTRL:    CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value);   break;
+                        case MSR_K6_EFER:           /* Can't be changed without causing a VM-exit */  break;
+                        default:
+                        {
+                            uint32_t idxLbrMsr;
+                            if (pVM->hm.s.vmx.fLbr)
+                            {
+                                if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
+                                {
+                                    Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfo->au64LbrFromIpMsr));
+                                    pVmcsInfo->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
+                                    break;
+                                }
+                                else if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
+                                {
+                                    Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfo->au64LbrFromIpMsr));
+                                    pVmcsInfo->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
+                                    break;
+                                }
+                                else if (idMsr == pVM->hm.s.vmx.idLbrTosMsr)
+                                {
+                                    pVmcsInfo->u64LbrTosMsr = pMsrs[i].u64Value;
+                                    break;
+                                }
+                                /* Fallthru (no break) */
+                            }
+                            pCtx->fExtrn = 0;
+                            pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
+                            ASMSetFlags(fEFlags);
+                            AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
+                            return VERR_HM_UNEXPECTED_LD_ST_MSR;
+                        }
+                    }
+                }
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_CR_MASK)
+            {
+                if (fWhat & CPUMCTX_EXTRN_CR0)
+                {
+                    uint64_t u64Cr0;
+                    uint64_t u64Shadow;
+                    rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0,            &u64Cr0);       AssertRC(rc);
+                    rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);    AssertRC(rc);
+#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
+                    u64Cr0 = (u64Cr0    & ~pVmcsInfo->u64Cr0Mask)
+                           | (u64Shadow &  pVmcsInfo->u64Cr0Mask);
+#else
+                    if (!CPUMIsGuestInVmxNonRootMode(pCtx))
+                    {
+                        u64Cr0 = (u64Cr0    & ~pVmcsInfo->u64Cr0Mask)
+                               | (u64Shadow &  pVmcsInfo->u64Cr0Mask);
+                    }
+                    else
+                    {
+                        /*
+                         * We've merged the guest and nested-guest's CR0 guest/host mask while executing
+                         * the nested-guest using hardware-assisted VMX. Accordingly we need to
+                         * re-construct CR0. See @bugref{9180#c95} for details.
+                         */
+                        PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
+                        PCVMXVVMCS    pVmcsNstGst  = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+                        u64Cr0 = (u64Cr0                     & ~pVmcsInfo->u64Cr0Mask)
+                               | (pVmcsNstGst->u64GuestCr0.u &  pVmcsNstGst->u64Cr0Mask.u)
+                               | (u64Shadow                  & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
+                    }
+#endif
+                    VMMRZCallRing3Disable(pVCpu);   /* May call into PGM which has Log statements. */
+                    CPUMSetGuestCR0(pVCpu, u64Cr0);
+                    VMMRZCallRing3Enable(pVCpu);
+                }
+
+                if (fWhat & CPUMCTX_EXTRN_CR4)
+                {
+                    uint64_t u64Cr4;
+                    uint64_t u64Shadow;
+                    rc  = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4,            &u64Cr4);      AssertRC(rc);
+                    rc |= VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);   AssertRC(rc);
+#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
+                    u64Cr4 = (u64Cr4    & ~pVmcsInfo->u64Cr4Mask)
+                           | (u64Shadow &  pVmcsInfo->u64Cr4Mask);
+#else
+                    if (!CPUMIsGuestInVmxNonRootMode(pCtx))
+                    {
+                        u64Cr4 = (u64Cr4    & ~pVmcsInfo->u64Cr4Mask)
+                               | (u64Shadow &  pVmcsInfo->u64Cr4Mask);
+                    }
+                    else
+                    {
+                        /*
+                         * We've merged the guest and nested-guest's CR4 guest/host mask while executing
+                         * the nested-guest using hardware-assisted VMX. Accordingly we need to
+                         * re-construct CR4. See @bugref{9180#c95} for details.
+                         */
+                        PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
+                        PCVMXVVMCS    pVmcsNstGst  = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+                        u64Cr4 = (u64Cr4                     & ~pVmcsInfo->u64Cr4Mask)
+                               | (pVmcsNstGst->u64GuestCr4.u &  pVmcsNstGst->u64Cr4Mask.u)
+                               | (u64Shadow                  & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
+                    }
+#endif
+                    pCtx->cr4 = u64Cr4;
+                }
+
+                if (fWhat & CPUMCTX_EXTRN_CR3)
+                {
+                    /* CR0.PG bit changes are always intercepted, so it's up to date. */
+                    if (   pVM->hm.s.vmx.fUnrestrictedGuest
+                        || (   pVM->hm.s.fNestedPaging
+                            && CPUMIsGuestPagingEnabledEx(pCtx)))
+                    {
+                        uint64_t u64Cr3;
+                        rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR3, &u64Cr3);  AssertRC(rc);
+                        if (pCtx->cr3 != u64Cr3)
+                        {
+                            pCtx->cr3 = u64Cr3;
+                            VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
+                        }
+
+                        /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
+                           Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
+                        if (CPUMIsGuestInPAEModeEx(pCtx))
+                        {
+                            rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);     AssertRC(rc);
+                            rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);     AssertRC(rc);
+                            rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);     AssertRC(rc);
+                            rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);     AssertRC(rc);
+                            VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
+                        }
+                    }
+                }
+            }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            if (fWhat & CPUMCTX_EXTRN_HWVIRT)
+            {
+                if (   (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
+                    && !CPUMIsGuestInVmxNonRootMode(pCtx))
+                {
+                    Assert(CPUMIsGuestInVmxRootMode(pCtx));
+                    rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
+                    if (RT_SUCCESS(rc))
+                    { /* likely */ }
+                    else
+                        break;
+                }
+            }
+#endif
+        } while (0);
+
+        if (RT_SUCCESS(rc))
+        {
+            /* Update fExtrn. */
+            pCtx->fExtrn &= ~fWhat;
+
+            /* If everything has been imported, clear the HM keeper bit. */
+            if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
+            {
+                pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
+                Assert(!pCtx->fExtrn);
+            }
+        }
+    }
+    else
+        AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
+
+    /*
+     * Restore interrupts.
+     */
+    ASMSetFlags(fEFlags);
+
+    STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
+
+    if (RT_SUCCESS(rc))
+    { /* likely */ }
+    else
+        return rc;
+
+    /*
+     * Honor any pending CR3 updates.
+     *
+     * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
+     * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
+     * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
+     *
+     * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
+     * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
+     * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
+     * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
+     *
+     * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
+     */
+    if (VMMRZCallRing3IsEnabled(pVCpu))
+    {
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
+        {
+            Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
+            PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
+        }
+
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
+            PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
+
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Saves the guest state from the VMCS into the guest-CPU context.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   fWhat   What to import, CPUMCTX_EXTRN_XXX.
+ */
+VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
+{
+    AssertPtr(pVCpu);
+    PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+    return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
+}
+
+
+/**
+ * Check per-VM and per-VCPU force flag actions that require us to go back to
+ * ring-3 for one reason or another.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too)
+ * @retval VINF_SUCCESS if we don't have any actions that require going back to
+ *         ring-3.
+ * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
+ * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
+ *         interrupts)
+ * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
+ *         all EMTs to be in ring-3.
+ * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
+ * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
+ *         to the EM loop.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   fStepping       Whether we are single-stepping the guest using the
+ *                          hypervisor debugger.
+ *
+ * @remarks This might cause nested-guest VM-exits, caller must check if the guest
+ *          is no longer in VMX non-root mode.
+ */
+static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, bool fStepping)
+{
+    Assert(VMMRZCallRing3IsEnabled(pVCpu));
+
+    /*
+     * Update pending interrupts into the APIC's IRR.
+     */
+    if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
+        APICUpdatePendingInterrupts(pVCpu);
+
+    /*
+     * Anything pending?  Should be more likely than not if we're doing a good job.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (  !fStepping
+        ?    !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
+          && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
+        :    !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
+          && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
+        return VINF_SUCCESS;
+
+    /* Pending PGM C3 sync. */
+    if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
+    {
+        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+        Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
+        VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
+                                           VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+        if (rcStrict != VINF_SUCCESS)
+        {
+            AssertRC(VBOXSTRICTRC_VAL(rcStrict));
+            Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+
+    /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
+    if (   VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
+        || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
+    {
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
+        int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
+        Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
+        return rc;
+    }
+
+    /* Pending VM request packets, such as hardware interrupts. */
+    if (   VM_FF_IS_SET(pVM, VM_FF_REQUEST)
+        || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
+    {
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
+        Log4Func(("Pending VM request forcing us back to ring-3\n"));
+        return VINF_EM_PENDING_REQUEST;
+    }
+
+    /* Pending PGM pool flushes. */
+    if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
+    {
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
+        Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
+        return VINF_PGM_POOL_FLUSH_PENDING;
+    }
+
+    /* Pending DMA requests. */
+    if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
+    {
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
+        Log4Func(("Pending DMA request forcing us back to ring-3\n"));
+        return VINF_EM_RAW_TO_R3;
+    }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * Pending nested-guest events.
+     *
+     * Please note the priority of these events are specified and important.
+     * See Intel spec. 29.4.3.2 "APIC-Write Emulation".
+     * See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts".
+     */
+    if (pVmxTransient->fIsNestedGuest)
+    {
+        /* Pending nested-guest APIC-write. */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
+        {
+            Log4Func(("Pending nested-guest APIC-write\n"));
+            VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
+            Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
+            return rcStrict;
+        }
+
+        /* Pending nested-guest monitor-trap flag (MTF). */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF))
+        {
+            Log4Func(("Pending nested-guest MTF\n"));
+            VBOXSTRICTRC rcStrict = IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */);
+            Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
+            return rcStrict;
+        }
+
+        /* Pending nested-guest VMX-preemption timer expired. */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
+        {
+            Log4Func(("Pending nested-guest MTF\n"));
+            VBOXSTRICTRC rcStrict = IEMExecVmxVmexitPreemptTimer(pVCpu);
+            Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
+            return rcStrict;
+        }
+    }
+#else
+    NOREF(pVmxTransient);
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Converts any TRPM trap into a pending HM event. This is typically used when
+ * entering from ring-3 (not longjmp returns).
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void hmR0VmxTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
+{
+    Assert(TRPMHasTrap(pVCpu));
+    Assert(!pVCpu->hm.s.Event.fPending);
+
+    uint8_t     uVector;
+    TRPMEVENT   enmTrpmEvent;
+    uint32_t    uErrCode;
+    RTGCUINTPTR GCPtrFaultAddress;
+    uint8_t     cbInstr;
+    bool        fIcebp;
+
+    int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
+    AssertRC(rc);
+
+    uint32_t u32IntInfo;
+    u32IntInfo  = uVector | VMX_IDT_VECTORING_INFO_VALID;
+    u32IntInfo |= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
+
+    rc = TRPMResetTrap(pVCpu);
+    AssertRC(rc);
+    Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
+          u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
+
+    hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
+}
+
+
+/**
+ * Converts the pending HM event into a TRPM trap.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void hmR0VmxPendingEventToTrpmTrap(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu->hm.s.Event.fPending);
+
+    /* If a trap was already pending, we did something wrong! */
+    Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
+
+    uint32_t const  u32IntInfo  = pVCpu->hm.s.Event.u64IntInfo;
+    uint32_t const  uVector     = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
+    TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
+
+    Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
+
+    int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
+    AssertRC(rc);
+
+    if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
+        TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
+
+    if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
+        TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
+    else
+    {
+        uint8_t const uVectorType = VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo);
+        switch (uVectorType)
+        {
+            case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
+                TRPMSetTrapDueToIcebp(pVCpu);
+                RT_FALL_THRU();
+            case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
+            case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
+            {
+                AssertMsg(   uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
+                          || (   uVector == X86_XCPT_BP /* INT3 */
+                              || uVector == X86_XCPT_OF /* INTO */
+                              || uVector == X86_XCPT_DB /* INT1 (ICEBP) */),
+                          ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
+                TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
+                break;
+            }
+        }
+    }
+
+    /* We're now done converting the pending event. */
+    pVCpu->hm.s.Event.fPending = false;
+}
+
+
+/**
+ * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
+ * cause a VM-exit as soon as the guest is in a state to receive interrupts.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static void hmR0VmxSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
+    {
+        if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
+        {
+            pVmcsInfo->u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
+            int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+            AssertRC(rc);
+        }
+    } /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
+}
+
+
+/**
+ * Clears the interrupt-window exiting control in the VMCS.
+ *
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
+{
+    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
+    {
+        pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
+        int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+        AssertRC(rc);
+    }
+}
+
+
+/**
+ * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
+ * cause a VM-exit as soon as the guest is in a state to receive NMIs.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static void hmR0VmxSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
+    {
+        if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
+        {
+            pVmcsInfo->u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
+            int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+            AssertRC(rc);
+            Log4Func(("Setup NMI-window exiting\n"));
+        }
+    } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
+}
+
+
+/**
+ * Clears the NMI-window exiting control in the VMCS.
+ *
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
+{
+    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
+    {
+        pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
+        int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+        AssertRC(rc);
+    }
+}
+
+
+/**
+ * Does the necessary state syncing before returning to ring-3 for any reason
+ * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   fImportState    Whether to import the guest state from the VMCS back
+ *                          to the guest-CPU context.
+ *
+ * @remarks No-long-jmp zone!!!
+ */
+static int hmR0VmxLeave(PVMCPUCC pVCpu, bool fImportState)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+
+    RTCPUID const idCpu = RTMpCpuId();
+    Log4Func(("HostCpuId=%u\n", idCpu));
+
+    /*
+     * !!! IMPORTANT !!!
+     * If you modify code here, check whether VMXR0CallRing3Callback() needs to be updated too.
+     */
+
+    /* Save the guest state if necessary. */
+    PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+    if (fImportState)
+    {
+        int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+        AssertRCReturn(rc, rc);
+    }
+
+    /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
+    CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
+    Assert(!CPUMIsGuestFPUStateActive(pVCpu));
+
+    /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
+#ifdef VBOX_STRICT
+    if (CPUMIsHyperDebugStateActive(pVCpu))
+        Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
+#endif
+    CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
+    Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
+    Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
+
+    /* Restore host-state bits that VT-x only restores partially. */
+    if (   (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
+        && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
+    {
+        Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
+        VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
+    }
+    pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
+
+    /* Restore the lazy host MSRs as we're leaving VT-x context. */
+    if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
+    {
+        /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
+        if (!fImportState)
+        {
+            int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
+            AssertRCReturn(rc, rc);
+        }
+        hmR0VmxLazyRestoreHostMsrs(pVCpu);
+        Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
+    }
+    else
+        pVCpu->hm.s.vmx.fLazyMsrs = 0;
+
+    /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
+    pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
+
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
+    STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
+
+    VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
+
+    /** @todo This partially defeats the purpose of having preemption hooks.
+     *  The problem is, deregistering the hooks should be moved to a place that
+     *  lasts until the EMT is about to be destroyed not everytime while leaving HM
+     *  context.
+     */
+    int rc = hmR0VmxClearVmcs(pVmcsInfo);
+    AssertRCReturn(rc, rc);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * A valid shadow VMCS is made active as part of VM-entry. It is necessary to
+     * clear a shadow VMCS before allowing that VMCS to become active on another
+     * logical processor. We may or may not be importing guest state which clears
+     * it, so cover for it here.
+     *
+     * See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
+     */
+    if (   pVmcsInfo->pvShadowVmcs
+        && pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
+    {
+        rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Flag that we need to re-export the host state if we switch to this VMCS before
+     * executing guest or nested-guest code.
+     */
+    pVmcsInfo->idHostCpuState = NIL_RTCPUID;
+#endif
+
+    Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
+    NOREF(idCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Leaves the VT-x session.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jmp zone!!!
+ */
+static int hmR0VmxLeaveSession(PVMCPUCC pVCpu)
+{
+    HM_DISABLE_PREEMPT(pVCpu);
+    HMVMX_ASSERT_CPU_SAFE(pVCpu);
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
+       and done this from the VMXR0ThreadCtxCallback(). */
+    if (!pVCpu->hm.s.fLeaveDone)
+    {
+        int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
+        AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
+        pVCpu->hm.s.fLeaveDone = true;
+    }
+    Assert(!pVCpu->cpum.GstCtx.fExtrn);
+
+    /*
+     * !!! IMPORTANT !!!
+     * If you modify code here, make sure to check whether VMXR0CallRing3Callback() needs to be updated too.
+     */
+
+    /* Deregister hook now that we've left HM context before re-enabling preemption. */
+    /** @todo Deregistering here means we need to VMCLEAR always
+     *        (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
+     *        for calling VMMR0ThreadCtxHookDisable here! */
+    VMMR0ThreadCtxHookDisable(pVCpu);
+
+    /* Leave HM context. This takes care of local init (term) and deregistering the longjmp-to-ring-3 callback. */
+    int rc = HMR0LeaveCpu(pVCpu);
+    HM_RESTORE_PREEMPT();
+    return rc;
+}
+
+
+/**
+ * Does the necessary state syncing before doing a longjmp to ring-3.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jmp zone!!!
+ */
+DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPUCC pVCpu)
+{
+    return hmR0VmxLeaveSession(pVCpu);
+}
+
+
+/**
+ * Take necessary actions before going back to ring-3.
+ *
+ * An action requires us to go back to ring-3. This function does the necessary
+ * steps before we can safely return to ring-3. This is not the same as longjmps
+ * to ring-3, this is voluntary and prepares the guest so it may continue
+ * executing outside HM (recompiler/IEM).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   rcExit  The reason for exiting to ring-3. Can be
+ *                  VINF_VMM_UNKNOWN_RING3_CALL.
+ */
+static int hmR0VmxExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
+{
+    HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
+
+    PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+    if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
+    {
+        VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
+        pVCpu->hm.s.vmx.LastError.u32VmcsRev   = *(uint32_t *)pVmcsInfo->pvVmcs;
+        pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
+        /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
+    }
+
+    /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
+    VMMRZCallRing3Disable(pVCpu);
+    Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
+
+    /*
+     * Convert any pending HM events back to TRPM due to premature exits to ring-3.
+     * We need to do this only on returns to ring-3 and not for longjmps to ring3.
+     *
+     * This is because execution may continue from ring-3 and we would need to inject
+     * the event from there (hence place it back in TRPM).
+     */
+    if (pVCpu->hm.s.Event.fPending)
+    {
+        hmR0VmxPendingEventToTrpmTrap(pVCpu);
+        Assert(!pVCpu->hm.s.Event.fPending);
+
+        /* Clear the events from the VMCS. */
+        int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);    AssertRC(rc);
+        rc     = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);         AssertRC(rc);
+    }
+#ifdef VBOX_STRICT
+    /*
+     * We check for rcExit here since for errors like VERR_VMX_UNABLE_TO_START_VM (which are
+     * fatal), we don't care about verifying duplicate injection of events. Errors like
+     * VERR_EM_INTERPRET are converted to their VINF_* counterparts -prior- to  calling this
+     * function so those should and will be checked below.
+     */
+    else if (RT_SUCCESS(rcExit))
+    {
+        /*
+         * Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
+         * This can be pretty hard to debug otherwise, interrupts might get injected twice
+         * occasionally, see @bugref{9180#c42}.
+         *
+         * However, if the VM-entry failed, any VM entry-interruption info. field would
+         * be left unmodified as the event would not have been injected to the guest. In
+         * such cases, don't assert, we're not going to continue guest execution anyway.
+         */
+        uint32_t uExitReason;
+        uint32_t uEntryIntInfo;
+        int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
+        rc    |= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
+        AssertRC(rc);
+        AssertMsg(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) || !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo),
+                  ("uExitReason=%#RX32 uEntryIntInfo=%#RX32 rcExit=%d\n", uExitReason, uEntryIntInfo, VBOXSTRICTRC_VAL(rcExit)));
+    }
+#endif
+
+    /*
+     * Clear the interrupt-window and NMI-window VMCS controls as we could have got
+     * a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
+     * (e.g. TPR below threshold).
+     */
+    if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
+    {
+        hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
+        hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
+    }
+
+    /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
+       and if we're injecting an event we should have a TRPM trap pending. */
+    AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
+#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
+    AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
+#endif
+
+    /* Save guest state and restore host state bits. */
+    int rc = hmR0VmxLeaveSession(pVCpu);
+    AssertRCReturn(rc, rc);
+    STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
+
+    /* Thread-context hooks are unregistered at this point!!! */
+    /* Ring-3 callback notifications are unregistered at this point!!! */
+
+    /* Sync recompiler state. */
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
+                             | CPUM_CHANGED_LDTR
+                             | CPUM_CHANGED_GDTR
+                             | CPUM_CHANGED_IDTR
+                             | CPUM_CHANGED_TR
+                             | CPUM_CHANGED_HIDDEN_SEL_REGS);
+    if (   pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
+        && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
+        CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
+
+    Assert(!pVCpu->hm.s.fClearTrapFlag);
+
+    /* Update the exit-to-ring 3 reason. */
+    pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
+
+    /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
+    if (   rcExit != VINF_EM_RAW_INTERRUPT
+        || CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
+    {
+        Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+    }
+
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
+    VMMRZCallRing3Enable(pVCpu);
+    return rc;
+}
+
+
+/**
+ * VMMRZCallRing3() callback wrapper which saves the guest state before we
+ * longjump to ring-3 and possibly get preempted.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmOperation    The operation causing the ring-3 longjump.
+ */
+VMMR0DECL(int) VMXR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation)
+{
+    if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
+    {
+        /*
+         * !!! IMPORTANT !!!
+         * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
+         * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
+         */
+        VMMRZCallRing3RemoveNotification(pVCpu);
+        VMMRZCallRing3Disable(pVCpu);
+        HM_DISABLE_PREEMPT(pVCpu);
+
+        PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+        hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+        CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
+        CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
+
+        /* Restore host-state bits that VT-x only restores partially. */
+        if (   (pVCpu->hm.s.vmx.fRestoreHostFlags &  VMX_RESTORE_HOST_REQUIRED)
+            && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
+            VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
+        pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
+
+        /* Restore the lazy host MSRs as we're leaving VT-x context. */
+        if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
+            hmR0VmxLazyRestoreHostMsrs(pVCpu);
+
+        /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
+        pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
+        VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
+
+        /* Clear the current VMCS data back to memory (shadow VMCS if any would have been
+           cleared as part of importing the guest state above. */
+        hmR0VmxClearVmcs(pVmcsInfo);
+
+        /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here!  */
+        VMMR0ThreadCtxHookDisable(pVCpu);
+
+        /* Leave HM context. This takes care of local init (term). */
+        HMR0LeaveCpu(pVCpu);
+        HM_RESTORE_PREEMPT();
+        return VINF_SUCCESS;
+    }
+
+    Assert(pVCpu);
+    Assert(VMMRZCallRing3IsEnabled(pVCpu));
+    HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
+
+    VMMRZCallRing3Disable(pVCpu);
+    Assert(VMMR0IsLogFlushDisabled(pVCpu));
+
+    Log4Func(("-> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
+
+    int rc = hmR0VmxLongJmpToRing3(pVCpu);
+    AssertRCReturn(rc, rc);
+
+    VMMRZCallRing3Enable(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
+ * stack.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   uValue  The value to push to the guest stack.
+ */
+static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPUCC pVCpu, uint16_t uValue)
+{
+    /*
+     * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
+     * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
+     * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
+     */
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    if (pCtx->sp == 1)
+        return VINF_EM_RESET;
+    pCtx->sp -= sizeof(uint16_t);       /* May wrap around which is expected behaviour. */
+    int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
+    AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * Injects an event into the guest upon VM-entry by updating the relevant fields
+ * in the VM-entry area in the VMCS.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_SUCCESS if the event is successfully injected into the VMCS.
+ * @retval  VINF_EM_RESET if event injection resulted in a triple-fault.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   pEvent          The event being injected.
+ * @param   pfIntrState     Pointer to the VT-x guest-interruptibility-state. This
+ *                          will be updated if necessary. This cannot not be NULL.
+ * @param   fStepping       Whether we're single-stepping guest execution and should
+ *                          return VINF_EM_DBG_STEPPED if the event is injected
+ *                          directly (registers modified by us, not by hardware on
+ *                          VM-entry).
+ */
+static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
+                                           uint32_t *pfIntrState)
+{
+    /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
+    AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
+    Assert(pfIntrState);
+
+    PCPUMCTX          pCtx       = &pVCpu->cpum.GstCtx;
+    uint32_t          u32IntInfo = pEvent->u64IntInfo;
+    uint32_t const    u32ErrCode = pEvent->u32ErrCode;
+    uint32_t const    cbInstr    = pEvent->cbInstr;
+    RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
+    uint8_t const     uVector    = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
+    uint32_t const    uIntType   = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
+
+#ifdef VBOX_STRICT
+    /*
+     * Validate the error-code-valid bit for hardware exceptions.
+     * No error codes for exceptions in real-mode.
+     *
+     * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
+     */
+    if (   uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
+        && !CPUMIsGuestInRealModeEx(pCtx))
+    {
+        switch (uVector)
+        {
+            case X86_XCPT_PF:
+            case X86_XCPT_DF:
+            case X86_XCPT_TS:
+            case X86_XCPT_NP:
+            case X86_XCPT_SS:
+            case X86_XCPT_GP:
+            case X86_XCPT_AC:
+                AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
+                          ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
+                RT_FALL_THRU();
+            default:
+                break;
+        }
+    }
+
+    /* Cannot inject an NMI when block-by-MOV SS is in effect. */
+    Assert(   uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
+           || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
+#endif
+
+    if (   uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
+        || uIntType == VMX_EXIT_INT_INFO_TYPE_NMI
+        || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT
+        || uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
+    {
+        Assert(uVector <= X86_XCPT_LAST);
+        Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_NMI          || uVector == X86_XCPT_NMI);
+        Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT || uVector == X86_XCPT_DB);
+        STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedXcptsR0[uVector]);
+    }
+    else
+        STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
+
+    /*
+     * Hardware interrupts & exceptions cannot be delivered through the software interrupt
+     * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
+     * interrupt handler in the (real-mode) guest.
+     *
+     * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
+     * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
+     */
+    if (CPUMIsGuestInRealModeEx(pCtx))     /* CR0.PE bit changes are always intercepted, so it's up to date. */
+    {
+        if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
+        {
+            /*
+             * For CPUs with unrestricted guest execution enabled and with the guest
+             * in real-mode, we must not set the deliver-error-code bit.
+             *
+             * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
+             */
+            u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
+        }
+        else
+        {
+            PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+            Assert(PDMVmmDevHeapIsEnabled(pVM));
+            Assert(pVM->hm.s.vmx.pRealModeTSS);
+            Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
+
+            /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
+            PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+            int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK
+                                                              | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
+            AssertRCReturn(rc2, rc2);
+
+            /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
+            size_t const cbIdtEntry = sizeof(X86IDTR16);
+            if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
+            {
+                /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
+                if (uVector == X86_XCPT_DF)
+                    return VINF_EM_RESET;
+
+                /* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
+                   No error codes for exceptions in real-mode. */
+                if (uVector == X86_XCPT_GP)
+                {
+                    uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_DF)
+                                               | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
+                                               | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
+                                               | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+                    HMEVENT EventXcptDf;
+                    RT_ZERO(EventXcptDf);
+                    EventXcptDf.u64IntInfo = uXcptDfInfo;
+                    return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
+                }
+
+                /*
+                 * If we're injecting an event with no valid IDT entry, inject a #GP.
+                 * No error codes for exceptions in real-mode.
+                 *
+                 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
+                 */
+                uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_GP)
+                                           | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
+                                           | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
+                                           | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+                HMEVENT EventXcptGp;
+                RT_ZERO(EventXcptGp);
+                EventXcptGp.u64IntInfo = uXcptGpInfo;
+                return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
+            }
+
+            /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
+            uint16_t uGuestIp = pCtx->ip;
+            if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
+            {
+                Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
+                /* #BP and #OF are both benign traps, we need to resume the next instruction. */
+                uGuestIp = pCtx->ip + (uint16_t)cbInstr;
+            }
+            else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
+                uGuestIp = pCtx->ip + (uint16_t)cbInstr;
+
+            /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
+            X86IDTR16 IdtEntry;
+            RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
+            rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
+            AssertRCReturn(rc2, rc2);
+
+            /* Construct the stack frame for the interrupt/exception handler. */
+            VBOXSTRICTRC rcStrict;
+            rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
+                if (rcStrict == VINF_SUCCESS)
+                    rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
+            }
+
+            /* Clear the required eflag bits and jump to the interrupt/exception handler. */
+            if (rcStrict == VINF_SUCCESS)
+            {
+                pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
+                pCtx->rip         = IdtEntry.offSel;
+                pCtx->cs.Sel      = IdtEntry.uSel;
+                pCtx->cs.ValidSel = IdtEntry.uSel;
+                pCtx->cs.u64Base  = IdtEntry.uSel << cbIdtEntry;
+                if (   uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
+                    && uVector  == X86_XCPT_PF)
+                    pCtx->cr2 = GCPtrFault;
+
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS  | HM_CHANGED_GUEST_CR2
+                                                         | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
+                                                         | HM_CHANGED_GUEST_RSP);
+
+                /*
+                 * If we delivered a hardware exception (other than an NMI) and if there was
+                 * block-by-STI in effect, we should clear it.
+                 */
+                if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
+                {
+                    Assert(   uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
+                           && uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
+                    Log4Func(("Clearing inhibition due to STI\n"));
+                    *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
+                }
+
+                Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
+                      u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
+
+                /*
+                 * The event has been truly dispatched to the guest. Mark it as no longer pending so
+                 * we don't attempt to undo it if we are returning to ring-3 before executing guest code.
+                 */
+                pVCpu->hm.s.Event.fPending = false;
+
+                /*
+                 * If we eventually support nested-guest execution without unrestricted guest execution,
+                 * we should set fInterceptEvents here.
+                 */
+                Assert(!pVmxTransient->fIsNestedGuest);
+
+                /* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
+                if (fStepping)
+                    rcStrict = VINF_EM_DBG_STEPPED;
+            }
+            AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
+                      ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+
+    /*
+     * Validate.
+     */
+    Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo));                     /* Bit 31 (Valid bit) must be set by caller. */
+    Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK));       /* Bits 30:12 MBZ. */
+
+    /*
+     * Inject the event into the VMCS.
+     */
+    int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
+    if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
+        rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
+    rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
+    AssertRC(rc);
+
+    /*
+     * Update guest CR2 if this is a page-fault.
+     */
+    if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
+        pCtx->cr2 = GCPtrFault;
+
+    Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Evaluates the event to be delivered to the guest and sets it as the pending
+ * event.
+ *
+ * Toggling of interrupt force-flags here is safe since we update TRPM on premature
+ * exits to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must
+ * NOT restore these force-flags.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   pfIntrState     Where to store the VT-x guest-interruptibility state.
+ */
+static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
+{
+    Assert(pfIntrState);
+    Assert(!TRPMHasTrap(pVCpu));
+
+    /*
+     * Compute/update guest-interruptibility state related FFs.
+     * The FFs will be used below while evaluating events to be injected.
+     */
+    *pfIntrState = hmR0VmxGetGuestIntrStateAndUpdateFFs(pVCpu);
+
+    /*
+     * Evaluate if a new event needs to be injected.
+     * An event that's already pending has already performed all necessary checks.
+     */
+    PVMXVMCSINFO pVmcsInfo      = pVmxTransient->pVmcsInfo;
+    bool const   fIsNestedGuest = pVmxTransient->fIsNestedGuest;
+    if (   !pVCpu->hm.s.Event.fPending
+        && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+    {
+        /** @todo SMI. SMIs take priority over NMIs. */
+
+        /*
+         * NMIs.
+         * NMIs take priority over external interrupts.
+         */
+        PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
+        {
+            /*
+             * For a guest, the FF always indicates the guest's ability to receive an NMI.
+             *
+             * For a nested-guest, the FF always indicates the outer guest's ability to
+             * receive an NMI while the guest-interruptibility state bit depends on whether
+             * the nested-hypervisor is using virtual-NMIs.
+             */
+            if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+            {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                if (   fIsNestedGuest
+                    && CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_NMI_EXIT))
+                    return IEMExecVmxVmexitXcptNmi(pVCpu);
+#endif
+                hmR0VmxSetPendingXcptNmi(pVCpu);
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+                Log4Func(("NMI pending injection\n"));
+
+                /* We've injected the NMI, bail. */
+                return VINF_SUCCESS;
+            }
+            else if (!fIsNestedGuest)
+                hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
+        }
+
+        /*
+         * External interrupts (PIC/APIC).
+         * Once PDMGetInterrupt() returns a valid interrupt we -must- deliver it.
+         * We cannot re-request the interrupt from the controller again.
+         */
+        if (    VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
+            && !pVCpu->hm.s.fSingleInstruction)
+        {
+            Assert(!DBGFIsStepping(pVCpu));
+            int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
+            AssertRC(rc);
+
+            /*
+             * We must not check EFLAGS directly when executing a nested-guest, use
+             * CPUMIsGuestPhysIntrEnabled() instead as EFLAGS.IF does not control the blocking of
+             * external interrupts when "External interrupt exiting" is set. This fixes a nasty
+             * SMP hang while executing nested-guest VCPUs on spinlocks which aren't rescued by
+             * other VM-exits (like a preemption timer), see @bugref{9562#c18}.
+             *
+             * See Intel spec. 25.4.1 "Event Blocking".
+             */
+            if (CPUMIsGuestPhysIntrEnabled(pVCpu))
+            {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                if (    fIsNestedGuest
+                    &&  CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
+                {
+                    VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector */, true /* fIntPending */);
+                    if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+                        return rcStrict;
+                }
+#endif
+                uint8_t u8Interrupt;
+                rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
+                if (RT_SUCCESS(rc))
+                {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                    if (   fIsNestedGuest
+                        && CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
+                    {
+                        VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
+                        Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
+                        return rcStrict;
+                    }
+#endif
+                    hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
+                    Log4Func(("External interrupt (%#x) pending injection\n", u8Interrupt));
+                }
+                else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
+                {
+                    STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
+
+                    if (   !fIsNestedGuest
+                        && (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
+                        hmR0VmxApicSetTprThreshold(pVmcsInfo, u8Interrupt >> 4);
+                    /* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
+
+                    /*
+                     * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
+                     * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
+                     * need to re-set this force-flag here.
+                     */
+                }
+                else
+                    STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
+
+                /* We've injected the interrupt or taken necessary action, bail. */
+                return VINF_SUCCESS;
+            }
+            else if (!fIsNestedGuest)
+                hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
+        }
+    }
+    else if (!fIsNestedGuest)
+    {
+        /*
+         * An event is being injected or we are in an interrupt shadow. Check if another event is
+         * pending. If so, instruct VT-x to cause a VM-exit as soon as the guest is ready to accept
+         * the pending event.
+         */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
+            hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
+        else if (   VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
+                 && !pVCpu->hm.s.fSingleInstruction)
+            hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
+    }
+    /* else: for nested-guests, NMI/interrupt-window exiting will be picked up when merging VMCS controls. */
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Injects any pending events into the guest if the guest is in a state to
+ * receive them.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   fIntrState      The VT-x guest-interruptibility state.
+ * @param   fStepping       Whether we are single-stepping the guest using the
+ *                          hypervisor debugger and should return
+ *                          VINF_EM_DBG_STEPPED if the event was dispatched
+ *                          directly.
+ */
+static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
+{
+    HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
+    Assert(VMMRZCallRing3IsEnabled(pVCpu));
+
+#ifdef VBOX_STRICT
+    /*
+     * Verify guest-interruptibility state.
+     *
+     * We put this in a scoped block so we do not accidentally use fBlockSti or fBlockMovSS,
+     * since injecting an event may modify the interruptibility state and we must thus always
+     * use fIntrState.
+     */
+    {
+        bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
+        bool const fBlockSti   = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
+        Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
+        Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF);     /* Cannot set block-by-STI when interrupts are disabled. */
+        Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));    /* We don't support block-by-SMI yet.*/
+        Assert(!TRPMHasTrap(pVCpu));
+        NOREF(fBlockMovSS); NOREF(fBlockSti);
+    }
+#endif
+
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    if (pVCpu->hm.s.Event.fPending)
+    {
+        /*
+         * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
+         * pending even while injecting an event and in this case, we want a VM-exit as soon as
+         * the guest is ready for the next interrupt, see @bugref{6208#c45}.
+         *
+         * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
+         */
+        uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
+#ifdef VBOX_STRICT
+        if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
+        {
+            Assert(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
+            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
+            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
+        }
+        else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
+        {
+            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
+            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
+            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
+        }
+#endif
+        Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
+              uIntType));
+
+        /*
+         * Inject the event and get any changes to the guest-interruptibility state.
+         *
+         * The guest-interruptibility state may need to be updated if we inject the event
+         * into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
+         */
+        rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
+        AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
+
+        if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
+        else
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
+    }
+
+    /*
+     * Deliver any pending debug exceptions if the guest is single-stepping using EFLAGS.TF and
+     * is an interrupt shadow (block-by-STI or block-by-MOV SS).
+     */
+    if (   (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
+        && !pVmxTransient->fIsNestedGuest)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
+
+        if (!pVCpu->hm.s.fSingleInstruction)
+        {
+            /*
+             * Set or clear the BS bit depending on whether the trap flag is active or not. We need
+             * to do both since we clear the BS bit from the VMCS while exiting to ring-3.
+             */
+            Assert(!DBGFIsStepping(pVCpu));
+            uint8_t const fTrapFlag = !!(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_TF);
+            int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, fTrapFlag << VMX_BF_VMCS_PENDING_DBG_XCPT_BS_SHIFT);
+            AssertRC(rc);
+        }
+        else
+        {
+            /*
+             * We must not deliver a debug exception when single-stepping over STI/Mov-SS in the
+             * hypervisor debugger using EFLAGS.TF but rather clear interrupt inhibition. However,
+             * we take care of this case in hmR0VmxExportSharedDebugState and also the case if
+             * we use MTF, so just make sure it's called before executing guest-code.
+             */
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
+        }
+    }
+    /* else: for nested-guest currently handling while merging controls. */
+
+    /*
+     * Finally, update the guest-interruptibility state.
+     *
+     * This is required for the real-on-v86 software interrupt injection, for
+     * pending debug exceptions as well as updates to the guest state from ring-3 (IEM).
+     */
+    int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
+    AssertRC(rc);
+
+    /*
+     * There's no need to clear the VM-entry interruption-information field here if we're not
+     * injecting anything. VT-x clears the valid bit on every VM-exit.
+     *
+     * See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
+     */
+
+    Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
+    return rcStrict;
+}
+
+
+/**
+ * Enters the VT-x session.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMR0DECL(int) VMXR0Enter(PVMCPUCC pVCpu)
+{
+    AssertPtr(pVCpu);
+    Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    LogFlowFunc(("pVCpu=%p\n", pVCpu));
+    Assert((pVCpu->hm.s.fCtxChanged &  (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
+                                    == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
+
+#ifdef VBOX_STRICT
+    /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
+    RTCCUINTREG uHostCr4 = ASMGetCR4();
+    if (!(uHostCr4 & X86_CR4_VMXE))
+    {
+        LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
+        return VERR_VMX_X86_CR4_VMXE_CLEARED;
+    }
+#endif
+
+    /*
+     * Load the appropriate VMCS as the current and active one.
+     */
+    PVMXVMCSINFO pVmcsInfo;
+    bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
+    if (!fInNestedGuestMode)
+        pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
+    else
+        pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
+    int rc = hmR0VmxLoadVmcs(pVmcsInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
+        pVCpu->hm.s.fLeaveDone = false;
+        Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
+
+        /*
+         * Do the EMT scheduled L1D flush here if needed.
+         */
+        if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
+            ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
+        else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
+            hmR0MdsClear();
+    }
+    return rc;
+}
+
+
+/**
+ * The thread-context callback (only on platforms which support it).
+ *
+ * @param   enmEvent        The thread-context event.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   fGlobalInit     Whether global VT-x/AMD-V init. was used.
+ * @thread  EMT(pVCpu)
+ */
+VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
+{
+    AssertPtr(pVCpu);
+    RT_NOREF1(fGlobalInit);
+
+    switch (enmEvent)
+    {
+        case RTTHREADCTXEVENT_OUT:
+        {
+            Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+            Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
+            VMCPU_ASSERT_EMT(pVCpu);
+
+            /* No longjmps (logger flushes, locks) in this fragile context. */
+            VMMRZCallRing3Disable(pVCpu);
+            Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
+
+            /* Restore host-state (FPU, debug etc.) */
+            if (!pVCpu->hm.s.fLeaveDone)
+            {
+                /*
+                 * Do -not- import the guest-state here as we might already be in the middle of importing
+                 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
+                 */
+                hmR0VmxLeave(pVCpu, false /* fImportState */);
+                pVCpu->hm.s.fLeaveDone = true;
+            }
+
+            /* Leave HM context, takes care of local init (term). */
+            int rc = HMR0LeaveCpu(pVCpu);
+            AssertRC(rc);
+
+            /* Restore longjmp state. */
+            VMMRZCallRing3Enable(pVCpu);
+            STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
+            break;
+        }
+
+        case RTTHREADCTXEVENT_IN:
+        {
+            Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+            Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
+            VMCPU_ASSERT_EMT(pVCpu);
+
+            /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
+            VMMRZCallRing3Disable(pVCpu);
+            Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
+
+            /* Initialize the bare minimum state required for HM. This takes care of
+               initializing VT-x if necessary (onlined CPUs, local init etc.) */
+            int rc = hmR0EnterCpu(pVCpu);
+            AssertRC(rc);
+            Assert((pVCpu->hm.s.fCtxChanged &  (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
+                                            == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
+
+            /* Load the active VMCS as the current one. */
+            PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+            rc = hmR0VmxLoadVmcs(pVmcsInfo);
+            AssertRC(rc);
+            Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
+            pVCpu->hm.s.fLeaveDone = false;
+
+            /* Do the EMT scheduled L1D flush if needed. */
+            if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
+                ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
+
+            /* Restore longjmp state. */
+            VMMRZCallRing3Enable(pVCpu);
+            break;
+        }
+
+        default:
+            break;
+    }
+}
+
+
+/**
+ * Exports the host state into the VMCS host-state area.
+ * Sets up the VM-exit MSR-load area.
+ *
+ * The CPU state will be loaded from these fields on every successful VM-exit.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0VmxExportHostState(PVMCPUCC pVCpu)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    int rc = VINF_SUCCESS;
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
+    {
+        hmR0VmxExportHostControlRegs();
+
+        rc = hmR0VmxExportHostSegmentRegs(pVCpu);
+        AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
+
+        hmR0VmxExportHostMsrs(pVCpu);
+
+        pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
+    }
+    return rc;
+}
+
+
+/**
+ * Saves the host state in the VMCS host-state.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+VMMR0DECL(int) VMXR0ExportHostState(PVMCPUCC pVCpu)
+{
+    AssertPtr(pVCpu);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    /*
+     * Export the host state here while entering HM context.
+     * When thread-context hooks are used, we might get preempted and have to re-save the host
+     * state but most of the time we won't be, so do it here before we disable interrupts.
+     */
+    return hmR0VmxExportHostState(pVCpu);
+}
+
+
+/**
+ * Exports the guest state into the VMCS guest-state area.
+ *
+ * The will typically be done before VM-entry when the guest-CPU state and the
+ * VMCS state may potentially be out of sync.
+ *
+ * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
+ * VM-entry controls.
+ * Sets up the appropriate VMX non-root function to execute guest code based on
+ * the guest CPU mode.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
+ *          without unrestricted guest execution and the VMMDev is not presently
+ *          mapped (e.g. EFI32).
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    AssertPtr(pVCpu);
+    HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
+    LogFlowFunc(("pVCpu=%p\n", pVCpu));
+
+    STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
+
+    /*
+     * Determine real-on-v86 mode.
+     * Used when the guest is in real-mode and unrestricted guest execution is not used.
+     */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    if (    pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
+        || !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
+        pVmcsInfo->RealMode.fRealOnV86Active = false;
+    else
+    {
+        Assert(!pVmxTransient->fIsNestedGuest);
+        pVmcsInfo->RealMode.fRealOnV86Active = true;
+    }
+
+    /*
+     * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
+     * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
+     */
+    int rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
+    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
+
+    rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
+    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
+
+    VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
+    if (rcStrict == VINF_SUCCESS)
+    { /* likely */ }
+    else
+    {
+        Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
+        return rcStrict;
+    }
+
+    rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
+    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
+
+    rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
+    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
+
+    hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
+    hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
+    hmR0VmxExportGuestRip(pVCpu);
+    hmR0VmxExportGuestRsp(pVCpu);
+    hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
+
+    rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
+    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
+
+    /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
+    ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~(  (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
+                                                  |  HM_CHANGED_GUEST_CR2
+                                                  | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
+                                                  |  HM_CHANGED_GUEST_X87
+                                                  |  HM_CHANGED_GUEST_SSE_AVX
+                                                  |  HM_CHANGED_GUEST_OTHER_XSAVE
+                                                  |  HM_CHANGED_GUEST_XCRx
+                                                  |  HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
+                                                  |  HM_CHANGED_GUEST_SYSCALL_MSRS   /* Part of lazy or auto load-store MSRs. */
+                                                  |  HM_CHANGED_GUEST_TSC_AUX
+                                                  |  HM_CHANGED_GUEST_OTHER_MSRS
+                                                  | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
+
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
+    return rc;
+}
+
+
+/**
+ * Exports the state shared between the host and guest into the VMCS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxExportSharedState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
+    {
+        int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
+        AssertRC(rc);
+        pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
+
+        /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
+        if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
+            hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
+    }
+
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
+    {
+        hmR0VmxLazyLoadGuestMsrs(pVCpu);
+        pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
+    }
+
+    AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
+              ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
+}
+
+
+/**
+ * Worker for loading the guest-state bits in the inner VT-x execution loop.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
+ *          without unrestricted guest execution and the VMMDev is not presently
+ *          mapped (e.g. EFI32).
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+    Assert(VMMR0IsLogFlushDisabled(pVCpu));
+
+#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+#endif
+
+    /*
+     * For many VM-exits only RIP/RSP/RFLAGS (and HWVIRT state when executing a nested-guest)
+     * changes. First try to export only these without going through all other changed-flag checks.
+     */
+    VBOXSTRICTRC   rcStrict;
+    uint64_t const fCtxMask     = HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE;
+    uint64_t const fMinimalMask = HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT;
+    uint64_t const fCtxChanged  = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
+
+    /* If only RIP/RSP/RFLAGS/HWVIRT changed, export only those (quicker, happens more often).*/
+    if (    (fCtxChanged & fMinimalMask)
+        && !(fCtxChanged & (fCtxMask & ~fMinimalMask)))
+    {
+        hmR0VmxExportGuestRip(pVCpu);
+        hmR0VmxExportGuestRsp(pVCpu);
+        hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
+        rcStrict = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
+    }
+    /* If anything else also changed, go through the full export routine and export as required. */
+    else if (fCtxChanged & fCtxMask)
+    {
+        rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
+        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        { /* likely */}
+        else
+        {
+            AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
+                                                           VBOXSTRICTRC_VAL(rcStrict)));
+            Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+            return rcStrict;
+        }
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
+    }
+    /* Nothing changed, nothing to load here. */
+    else
+        rcStrict = VINF_SUCCESS;
+
+#ifdef VBOX_STRICT
+    /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
+    uint64_t const fCtxChangedCur = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
+    AssertMsg(!(fCtxChangedCur & fCtxMask), ("fCtxChangedCur=%#RX64\n", fCtxChangedCur));
+#endif
+    return rcStrict;
+}
+
+
+/**
+ * Tries to determine what part of the guest-state VT-x has deemed as invalid
+ * and update error record fields accordingly.
+ *
+ * @returns VMX_IGS_* error codes.
+ * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
+ *         wrong with the guest state.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks This function assumes our cache of the VMCS controls
+ *          are valid, i.e. hmR0VmxCheckCachedVmcsCtls() succeeded.
+ */
+static uint32_t hmR0VmxCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+#define HMVMX_ERROR_BREAK(err)              { uError = (err); break; }
+#define HMVMX_CHECK_BREAK(expr, err)        do { \
+                                                if (!(expr)) { uError = (err); break; } \
+                                            } while (0)
+
+    PVMCC    pVM    = pVCpu->CTX_SUFF(pVM);
+    PCPUMCTX pCtx   = &pVCpu->cpum.GstCtx;
+    uint32_t uError = VMX_IGS_ERROR;
+    uint32_t u32IntrState = 0;
+    bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
+    do
+    {
+        int rc;
+
+        /*
+         * Guest-interruptibility state.
+         *
+         * Read this first so that any check that fails prior to those that actually
+         * require the guest-interruptibility state would still reflect the correct
+         * VMCS value and avoids causing further confusion.
+         */
+        rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
+        AssertRC(rc);
+
+        uint32_t u32Val;
+        uint64_t u64Val;
+
+        /*
+         * CR0.
+         */
+        /** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
+        uint64_t       fSetCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
+        uint64_t const fZapCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
+        /* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
+           See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
+        if (fUnrestrictedGuest)
+            fSetCr0 &= ~(uint64_t)(X86_CR0_PE | X86_CR0_PG);
+
+        uint64_t u64GuestCr0;
+        rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64GuestCr0);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
+        HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
+        if (   !fUnrestrictedGuest
+            &&  (u64GuestCr0 & X86_CR0_PG)
+            && !(u64GuestCr0 & X86_CR0_PE))
+            HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
+
+        /*
+         * CR4.
+         */
+        /** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
+        uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
+        uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
+
+        uint64_t u64GuestCr4;
+        rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64GuestCr4);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
+        HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
+
+        /*
+         * IA32_DEBUGCTL MSR.
+         */
+        rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
+        AssertRC(rc);
+        if (   (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
+            && (u64Val & 0xfffffe3c))                           /* Bits 31:9, bits 5:2 MBZ. */
+        {
+            HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
+        }
+        uint64_t u64DebugCtlMsr = u64Val;
+
+#ifdef VBOX_STRICT
+        rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
+        AssertRC(rc);
+        Assert(u32Val == pVmcsInfo->u32EntryCtls);
+#endif
+        bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+
+        /*
+         * RIP and RFLAGS.
+         */
+        rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
+        AssertRC(rc);
+        /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
+        if (   !fLongModeGuest
+            || !pCtx->cs.Attr.n.u1Long)
+            HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
+        /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
+         *        must be identical if the "IA-32e mode guest" VM-entry
+         *        control is 1 and CS.L is 1. No check applies if the
+         *        CPU supports 64 linear-address bits. */
+
+        /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
+        rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)),                     /* Bit 63:22, Bit 15, 5, 3 MBZ. */
+                          VMX_IGS_RFLAGS_RESERVED);
+        HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1);       /* Bit 1 MB1. */
+        uint32_t const u32Eflags = u64Val;
+
+        if (   fLongModeGuest
+            || (   fUnrestrictedGuest
+                && !(u64GuestCr0 & X86_CR0_PE)))
+        {
+            HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
+        }
+
+        uint32_t u32EntryInfo;
+        rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
+        AssertRC(rc);
+        if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
+            HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
+
+        /*
+         * 64-bit checks.
+         */
+        if (fLongModeGuest)
+        {
+            HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG,  VMX_IGS_CR0_PG_LONGMODE);
+            HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
+        }
+
+        if (   !fLongModeGuest
+            && (u64GuestCr4 & X86_CR4_PCIDE))
+            HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
+
+        /** @todo CR3 field must be such that bits 63:52 and bits in the range
+         *        51:32 beyond the processor's physical-address width are 0. */
+
+        if (   (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
+            && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
+            HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
+
+        rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
+
+        rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
+
+        /*
+         * PERF_GLOBAL MSR.
+         */
+        if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
+        {
+            rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
+                              VMX_IGS_PERF_GLOBAL_MSR_RESERVED);        /* Bits 63:35, bits 31:2 MBZ. */
+        }
+
+        /*
+         * PAT MSR.
+         */
+        if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
+        {
+            rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
+            for (unsigned i = 0; i < 8; i++)
+            {
+                uint8_t u8Val = (u64Val & 0xff);
+                if (   u8Val != 0 /* UC */
+                    && u8Val != 1 /* WC */
+                    && u8Val != 4 /* WT */
+                    && u8Val != 5 /* WP */
+                    && u8Val != 6 /* WB */
+                    && u8Val != 7 /* UC- */)
+                    HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
+                u64Val >>= 8;
+            }
+        }
+
+        /*
+         * EFER MSR.
+         */
+        if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
+        {
+            Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
+            rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
+                              VMX_IGS_EFER_MSR_RESERVED);               /* Bits 63:12, bit 9, bits 7:1 MBZ. */
+            HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(  pVmcsInfo->u32EntryCtls
+                                                                           & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
+                              VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
+            /** @todo r=ramshankar: Unrestricted check here is probably wrong, see
+             *        iemVmxVmentryCheckGuestState(). */
+            HMVMX_CHECK_BREAK(   fUnrestrictedGuest
+                              || !(u64GuestCr0 & X86_CR0_PG)
+                              || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
+                              VMX_IGS_EFER_LMA_LME_MISMATCH);
+        }
+
+        /*
+         * Segment registers.
+         */
+        HMVMX_CHECK_BREAK(   (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
+                          || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
+        if (!(u32Eflags & X86_EFL_VM))
+        {
+            /* CS */
+            HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
+            HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
+            HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
+            HMVMX_CHECK_BREAK(   (pCtx->cs.u32Limit & 0xfff) == 0xfff
+                              || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
+            HMVMX_CHECK_BREAK(   !(pCtx->cs.u32Limit & 0xfff00000)
+                              || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
+            /* CS cannot be loaded with NULL in protected mode. */
+            HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
+            HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
+            if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
+                HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
+            else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
+                HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
+            else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
+                HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
+            else
+                HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
+
+            /* SS */
+            HMVMX_CHECK_BREAK(   pVM->hm.s.vmx.fUnrestrictedGuest
+                              || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
+            HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
+            if (   !(pCtx->cr0 & X86_CR0_PE)
+                || pCtx->cs.Attr.n.u4Type == 3)
+                HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
+
+            if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
+            {
+                HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
+                HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
+                HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(   (pCtx->ss.u32Limit & 0xfff) == 0xfff
+                                  || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->ss.u32Limit & 0xfff00000)
+                                  || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
+            }
+
+            /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
+            if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
+            {
+                HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
+                HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
+                HMVMX_CHECK_BREAK(   pVM->hm.s.vmx.fUnrestrictedGuest
+                                  || pCtx->ds.Attr.n.u4Type > 11
+                                  || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
+                HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(   (pCtx->ds.u32Limit & 0xfff) == 0xfff
+                                  || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->ds.u32Limit & 0xfff00000)
+                                  || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                                  || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
+            }
+            if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
+            {
+                HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
+                HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
+                HMVMX_CHECK_BREAK(   pVM->hm.s.vmx.fUnrestrictedGuest
+                                  || pCtx->es.Attr.n.u4Type > 11
+                                  || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
+                HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(   (pCtx->es.u32Limit & 0xfff) == 0xfff
+                                  || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->es.u32Limit & 0xfff00000)
+                                  || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                                  || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
+            }
+            if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
+            {
+                HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
+                HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
+                HMVMX_CHECK_BREAK(   pVM->hm.s.vmx.fUnrestrictedGuest
+                                  || pCtx->fs.Attr.n.u4Type > 11
+                                  || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
+                HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(   (pCtx->fs.u32Limit & 0xfff) == 0xfff
+                                  || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->fs.u32Limit & 0xfff00000)
+                                  || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                                  || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
+            }
+            if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
+            {
+                HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
+                HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
+                HMVMX_CHECK_BREAK(   pVM->hm.s.vmx.fUnrestrictedGuest
+                                  || pCtx->gs.Attr.n.u4Type > 11
+                                  || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
+                HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(   (pCtx->gs.u32Limit & 0xfff) == 0xfff
+                                  || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->gs.u32Limit & 0xfff00000)
+                                  || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                                  || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
+            }
+            /* 64-bit capable CPUs. */
+            HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
+            HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
+            HMVMX_CHECK_BREAK(   (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
+                              || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
+            HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
+            HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
+                              VMX_IGS_LONGMODE_SS_BASE_INVALID);
+            HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
+                              VMX_IGS_LONGMODE_DS_BASE_INVALID);
+            HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
+                              VMX_IGS_LONGMODE_ES_BASE_INVALID);
+        }
+        else
+        {
+            /* V86 mode checks. */
+            uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
+            if (pVmcsInfo->RealMode.fRealOnV86Active)
+            {
+                u32CSAttr = 0xf3;   u32SSAttr = 0xf3;
+                u32DSAttr = 0xf3;   u32ESAttr = 0xf3;
+                u32FSAttr = 0xf3;   u32GSAttr = 0xf3;
+            }
+            else
+            {
+                u32CSAttr = pCtx->cs.Attr.u;   u32SSAttr = pCtx->ss.Attr.u;
+                u32DSAttr = pCtx->ds.Attr.u;   u32ESAttr = pCtx->es.Attr.u;
+                u32FSAttr = pCtx->fs.Attr.u;   u32GSAttr = pCtx->gs.Attr.u;
+            }
+
+            /* CS */
+            HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
+            HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
+            HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
+            /* SS */
+            HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
+            HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
+            HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
+            /* DS */
+            HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
+            HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
+            HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
+            /* ES */
+            HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
+            HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
+            HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
+            /* FS */
+            HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
+            HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
+            HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
+            /* GS */
+            HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
+            HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
+            HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
+            /* 64-bit capable CPUs. */
+            HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
+            HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
+            HMVMX_CHECK_BREAK(   (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
+                              || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
+            HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
+            HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
+                              VMX_IGS_LONGMODE_SS_BASE_INVALID);
+            HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
+                              VMX_IGS_LONGMODE_DS_BASE_INVALID);
+            HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
+                              VMX_IGS_LONGMODE_ES_BASE_INVALID);
+        }
+
+        /*
+         * TR.
+         */
+        HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
+        /* 64-bit capable CPUs. */
+        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
+        if (fLongModeGuest)
+            HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11,           /* 64-bit busy TSS. */
+                              VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
+        else
+            HMVMX_CHECK_BREAK(   pCtx->tr.Attr.n.u4Type == 3          /* 16-bit busy TSS. */
+                              || pCtx->tr.Attr.n.u4Type == 11,        /* 32-bit busy TSS.*/
+                              VMX_IGS_TR_ATTR_TYPE_INVALID);
+        HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
+        HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
+        HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED);   /* Bits 11:8 MBZ. */
+        HMVMX_CHECK_BREAK(   (pCtx->tr.u32Limit & 0xfff) == 0xfff
+                          || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
+        HMVMX_CHECK_BREAK(   !(pCtx->tr.u32Limit & 0xfff00000)
+                          || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
+        HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
+
+        /*
+         * GDTR and IDTR (64-bit capable checks).
+         */
+        rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
+
+        rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
+
+        rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID);      /* Bits 31:16 MBZ. */
+
+        rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID);      /* Bits 31:16 MBZ. */
+
+        /*
+         * Guest Non-Register State.
+         */
+        /* Activity State. */
+        uint32_t u32ActivityState;
+        rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(   !u32ActivityState
+                          || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
+                             VMX_IGS_ACTIVITY_STATE_INVALID);
+        HMVMX_CHECK_BREAK(   !(pCtx->ss.Attr.n.u2Dpl)
+                          || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
+
+        if (   u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
+            || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
+            HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
+
+        /** @todo Activity state and injecting interrupts. Left as a todo since we
+         *        currently don't use activity states but ACTIVE. */
+
+        HMVMX_CHECK_BREAK(   !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
+                          || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
+
+        /* Guest interruptibility-state. */
+        HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
+        HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
+                                       != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
+                          VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
+        HMVMX_CHECK_BREAK(   (u32Eflags & X86_EFL_IF)
+                          || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
+                          VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
+        if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
+        {
+            HMVMX_CHECK_BREAK(   !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
+                              && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
+                              VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
+        }
+        else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
+        {
+            HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
+                              VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
+            HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
+                              VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
+        }
+        /** @todo Assumes the processor is not in SMM. */
+        HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
+                          VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
+        HMVMX_CHECK_BREAK(   !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
+                          || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
+                             VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
+        if (   (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+            && VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
+            HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI), VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
+
+        /* Pending debug exceptions. */
+        rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
+        AssertRC(rc);
+        /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
+        HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
+        u32Val = u64Val;    /* For pending debug exceptions checks below. */
+
+        if (   (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
+            || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
+            || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
+        {
+            if (   (u32Eflags & X86_EFL_TF)
+                && !(u64DebugCtlMsr & RT_BIT_64(1)))    /* Bit 1 is IA32_DEBUGCTL.BTF. */
+            {
+                /* Bit 14 is PendingDebug.BS. */
+                HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
+            }
+            if (   !(u32Eflags & X86_EFL_TF)
+                || (u64DebugCtlMsr & RT_BIT_64(1)))     /* Bit 1 is IA32_DEBUGCTL.BTF. */
+            {
+                /* Bit 14 is PendingDebug.BS. */
+                HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
+            }
+        }
+
+        /* VMCS link pointer. */
+        rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
+        AssertRC(rc);
+        if (u64Val != UINT64_C(0xffffffffffffffff))
+        {
+            HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
+            /** @todo Bits beyond the processor's physical-address width MBZ. */
+            /** @todo SMM checks. */
+            Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
+            Assert(pVmcsInfo->pvShadowVmcs);
+            VMXVMCSREVID VmcsRevId;
+            VmcsRevId.u = *(uint32_t *)pVmcsInfo->pvShadowVmcs;
+            HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
+                              VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
+            HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
+                              VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
+        }
+
+        /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
+         *        not using nested paging? */
+        if (   pVM->hm.s.fNestedPaging
+            && !fLongModeGuest
+            && CPUMIsGuestInPAEModeEx(pCtx))
+        {
+            rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
+
+            rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
+
+            rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
+
+            rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
+        }
+
+        /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
+        if (uError == VMX_IGS_ERROR)
+            uError = VMX_IGS_REASON_NOT_FOUND;
+    } while (0);
+
+    pVCpu->hm.s.u32HMError = uError;
+    pVCpu->hm.s.vmx.LastError.u32GuestIntrState = u32IntrState;
+    return uError;
+
+#undef HMVMX_ERROR_BREAK
+#undef HMVMX_CHECK_BREAK
+}
+
+
+/**
+ * Map the APIC-access page for virtualizing APIC accesses.
+ *
+ * This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
+ * this not done as part of exporting guest state, see @bugref{8721}.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static int hmR0VmxMapHCApicAccessPage(PVMCPUCC pVCpu)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
+
+    Assert(PDMHasApic(pVM));
+    Assert(u64MsrApicBase);
+
+    RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
+    Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
+
+    /* Unalias the existing mapping. */
+    int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
+    AssertRCReturn(rc, rc);
+
+    /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
+    Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
+    rc = IOMR0MmioMapMmioHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
+    AssertRCReturn(rc, rc);
+
+    /* Update the per-VCPU cache of the APIC base MSR. */
+    pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker function passed to RTMpOnSpecific() that is to be called on the target
+ * CPU.
+ *
+ * @param   idCpu       The ID for the CPU the function is called on.
+ * @param   pvUser1     Null, not used.
+ * @param   pvUser2     Null, not used.
+ */
+static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    RT_NOREF3(idCpu, pvUser1, pvUser2);
+    VMXDispatchHostNmi();
+}
+
+
+/**
+ * Dispatching an NMI on the host CPU that received it.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object corresponding to the VMCS that was
+ *                      executing when receiving the host NMI in VMX non-root
+ *                      operation.
+ */
+static int hmR0VmxExitHostNmi(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+    RTCPUID const idCpu = pVmcsInfo->idHostCpuExec;
+    Assert(idCpu != NIL_RTCPUID);
+
+    /*
+     * We don't want to delay dispatching the NMI any more than we have to. However,
+     * we have already chosen -not- to dispatch NMIs when interrupts were still disabled
+     * after executing guest or nested-guest code for the following reasons:
+     *
+     *   - We would need to perform VMREADs with interrupts disabled and is orders of
+     *     magnitude worse when we run as a nested hypervisor without VMCS shadowing
+     *     supported by the host hypervisor.
+     *
+     *   - It affects the common VM-exit scenario and keeps interrupts disabled for a
+     *     longer period of time just for handling an edge case like host NMIs which do
+     *     not occur nearly as frequently as other VM-exits.
+     *
+     * Let's cover the most likely scenario first. Check if we are on the target CPU
+     * and dispatch the NMI right away. This should be much faster than calling into
+     * RTMpOnSpecific() machinery.
+     */
+    bool fDispatched = false;
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+    if (idCpu == RTMpCpuId())
+    {
+        VMXDispatchHostNmi();
+        fDispatched = true;
+    }
+    ASMSetFlags(fEFlags);
+    if (fDispatched)
+    {
+        STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * RTMpOnSpecific() waits until the worker function has run on the target CPU. So
+     * there should be no race or recursion even if we are unlucky enough to be preempted
+     * (to the target CPU) without dispatching the host NMI above.
+     */
+    STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
+    return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 */,  NULL /* pvUser2 */);
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Merges the guest with the nested-guest MSR bitmap in preparation of executing the
+ * nested-guest using hardware-assisted VMX.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   pVmcsInfoNstGst     The nested-guest VMCS info. object.
+ * @param   pVmcsInfoGst        The guest VMCS info. object.
+ */
+static void hmR0VmxMergeMsrBitmapNested(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
+{
+    uint32_t const cbMsrBitmap    = X86_PAGE_4K_SIZE;
+    uint64_t       *pu64MsrBitmap = (uint64_t *)pVmcsInfoNstGst->pvMsrBitmap;
+    Assert(pu64MsrBitmap);
+
+    /*
+     * We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
+     * MSR that is intercepted by the guest is also intercepted while executing the
+     * nested-guest using hardware-assisted VMX.
+     *
+     * Note! If the nested-guest is not using an MSR bitmap, every MSR must cause a
+     *       nested-guest VM-exit even if the outer guest is not intercepting some
+     *       MSRs. We cannot assume the caller has initialized the nested-guest
+     *       MSR bitmap in this case.
+     *
+     *       The nested hypervisor may also switch whether it uses MSR bitmaps for
+     *       each of its VM-entry, hence initializing it once per-VM while setting
+     *       up the nested-guest VMCS is not sufficient.
+     */
+    PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+    {
+        uint64_t const *pu64MsrBitmapNstGst = (uint64_t const *)pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
+        uint64_t const *pu64MsrBitmapGst    = (uint64_t const *)pVmcsInfoGst->pvMsrBitmap;
+        Assert(pu64MsrBitmapNstGst);
+        Assert(pu64MsrBitmapGst);
+
+        uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
+        for (uint32_t i = 0; i < cFrags; i++)
+            pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] | pu64MsrBitmapGst[i];
+    }
+    else
+        ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
+}
+
+
+/**
+ * Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
+ * hardware-assisted VMX execution of the nested-guest.
+ *
+ * For a guest, we don't modify these controls once we set up the VMCS and hence
+ * this function is never called.
+ *
+ * For nested-guests since the nested hypervisor provides these controls on every
+ * nested-guest VM-entry and could potentially change them everytime we need to
+ * merge them before every nested-guest VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static int hmR0VmxMergeVmcsNested(PVMCPUCC pVCpu)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
+    PCVMXVVMCS    pVmcsNstGst  = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+    Assert(pVmcsNstGst);
+
+    /*
+     * Merge the controls with the requirements of the guest VMCS.
+     *
+     * We do not need to validate the nested-guest VMX features specified in the nested-guest
+     * VMCS with the features supported by the physical CPU as it's already done by the
+     * VMLAUNCH/VMRESUME instruction emulation.
+     *
+     * This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
+     * derived from the VMX features supported by the physical CPU.
+     */
+
+    /* Pin-based VM-execution controls. */
+    uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls | pVmcsInfoGst->u32PinCtls;
+
+    /* Processor-based VM-execution controls. */
+    uint32_t       u32ProcCtls = (pVmcsNstGst->u32ProcCtls  & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
+                               | (pVmcsInfoGst->u32ProcCtls & ~(  VMX_PROC_CTLS_INT_WINDOW_EXIT
+                                                                | VMX_PROC_CTLS_NMI_WINDOW_EXIT
+                                                                | VMX_PROC_CTLS_USE_TPR_SHADOW
+                                                                | VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
+
+    /* Secondary processor-based VM-execution controls. */
+    uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2  & ~VMX_PROC_CTLS2_VPID)
+                                | (pVmcsInfoGst->u32ProcCtls2 & ~(  VMX_PROC_CTLS2_VIRT_APIC_ACCESS
+                                                                  | VMX_PROC_CTLS2_INVPCID
+                                                                  | VMX_PROC_CTLS2_VMCS_SHADOWING
+                                                                  | VMX_PROC_CTLS2_RDTSCP
+                                                                  | VMX_PROC_CTLS2_XSAVES_XRSTORS
+                                                                  | VMX_PROC_CTLS2_APIC_REG_VIRT
+                                                                  | VMX_PROC_CTLS2_VIRT_INT_DELIVERY
+                                                                  | VMX_PROC_CTLS2_VMFUNC));
+
+    /*
+     * VM-entry controls:
+     * These controls contains state that depends on the nested-guest state (primarily
+     * EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
+     * VM-exit. Although the nested hypervisor cannot change it, we need to in order to
+     * properly continue executing the nested-guest if the EFER MSR changes but does not
+     * cause a nested-guest VM-exits.
+     *
+     * VM-exit controls:
+     * These controls specify the host state on return. We cannot use the controls from
+     * the nested hypervisor state as is as it would contain the guest state rather than
+     * the host state. Since the host state is subject to change (e.g. preemption, trips
+     * to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
+     * through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
+     *
+     * VM-entry MSR-load:
+     * The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
+     * context by the VMLAUNCH/VMRESUME instruction emulation.
+     *
+     * VM-exit MSR-store:
+     * The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
+     * back into the VM-exit MSR-store area.
+     *
+     * VM-exit MSR-load areas:
+     * This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
+     * can entirely ignore what the nested hypervisor wants to load here.
+     */
+
+    /*
+     * Exception bitmap.
+     *
+     * We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
+     * here (and avoid doing anything while exporting nested-guest state), but to keep the
+     * code more flexible if intercepting exceptions become more dynamic in the future we do
+     * it as part of exporting the nested-guest state.
+     */
+    uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap | pVmcsInfoGst->u32XcptBitmap;
+
+    /*
+     * CR0/CR4 guest/host mask.
+     *
+     * Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
+     * cause VM-exits, so we need to merge them here.
+     */
+    uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u | pVmcsInfoGst->u64Cr0Mask;
+    uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u | pVmcsInfoGst->u64Cr4Mask;
+
+    /*
+     * Page-fault error-code mask and match.
+     *
+     * Although we require unrestricted guest execution (and thereby nested-paging) for
+     * hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
+     * normally intercept #PFs, it might intercept them for debugging purposes.
+     *
+     * If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
+     * If the outer guest is intercepting #PFs, we must intercept all #PFs.
+     */
+    uint32_t u32XcptPFMask;
+    uint32_t u32XcptPFMatch;
+    if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
+    {
+        u32XcptPFMask  = pVmcsNstGst->u32XcptPFMask;
+        u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
+    }
+    else
+    {
+        u32XcptPFMask  = 0;
+        u32XcptPFMatch = 0;
+    }
+
+    /*
+     * Pause-Loop exiting.
+     */
+    uint32_t const cPleGapTicks    = RT_MIN(pVM->hm.s.vmx.cPleGapTicks,    pVmcsNstGst->u32PleGap);
+    uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
+
+    /*
+     * Pending debug exceptions.
+     * Currently just copy whatever the nested-guest provides us.
+     */
+    uint64_t const uPendingDbgXcpts = pVmcsNstGst->u64GuestPendingDbgXcpts.u;
+
+    /*
+     * I/O Bitmap.
+     *
+     * We do not use the I/O bitmap that may be provided by the nested hypervisor as we always
+     * intercept all I/O port accesses.
+     */
+    Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
+    Assert(!(u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS));
+
+    /*
+     * VMCS shadowing.
+     *
+     * We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
+     * enabled while executing the nested-guest.
+     */
+    Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
+
+    /*
+     * APIC-access page.
+     */
+    RTHCPHYS HCPhysApicAccess;
+    if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
+    {
+        Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
+        RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
+
+        /** @todo NSTVMX: This is not really correct but currently is required to make
+         *        things work. We need to re-enable the page handler when we fallback to
+         *        IEM execution of the nested-guest! */
+        PGMHandlerPhysicalPageTempOff(pVM, GCPhysApicAccess, GCPhysApicAccess);
+
+        void          *pvPage;
+        PGMPAGEMAPLOCK PgLockApicAccess;
+        int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgLockApicAccess);
+        if (RT_SUCCESS(rc))
+        {
+            rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
+            AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
+
+            /** @todo Handle proper releasing of page-mapping lock later. */
+            PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockApicAccess);
+        }
+        else
+            return rc;
+    }
+    else
+        HCPhysApicAccess = 0;
+
+    /*
+     * Virtual-APIC page and TPR threshold.
+     */
+    RTHCPHYS HCPhysVirtApic;
+    uint32_t u32TprThreshold;
+    if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+    {
+        Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
+        RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
+
+        void          *pvPage;
+        PGMPAGEMAPLOCK PgLockVirtApic;
+        int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &PgLockVirtApic);
+        if (RT_SUCCESS(rc))
+        {
+            rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
+            AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
+
+            /** @todo Handle proper releasing of page-mapping lock later. */
+            PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockVirtApic);
+        }
+        else
+            return rc;
+
+        u32TprThreshold = pVmcsNstGst->u32TprThreshold;
+    }
+    else
+    {
+        HCPhysVirtApic  = 0;
+        u32TprThreshold = 0;
+
+        /*
+         * We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
+         * used by the nested hypervisor. Preventing MMIO accesses to the physical APIC will
+         * be taken care of by EPT/shadow paging.
+         */
+        if (pVM->hm.s.fAllow64BitGuests)
+        {
+            u32ProcCtls |= VMX_PROC_CTLS_CR8_STORE_EXIT
+                        |  VMX_PROC_CTLS_CR8_LOAD_EXIT;
+        }
+    }
+
+    /*
+     * Validate basic assumptions.
+     */
+    PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
+    Assert(pVM->hm.s.vmx.fAllowUnrestricted);
+    Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
+    Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
+
+    /*
+     * Commit it to the nested-guest VMCS.
+     */
+    int rc = VINF_SUCCESS;
+    if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
+        rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
+    if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
+        rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
+    if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
+        rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
+    if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
+        rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
+    if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
+        rc |= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
+    if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
+        rc |= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
+    if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
+        rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
+    if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
+        rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
+    if (   !(u32ProcCtls  & VMX_PROC_CTLS_PAUSE_EXIT)
+        &&  (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
+    {
+        Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
+        rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
+        rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
+    }
+    if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+    {
+        rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
+        rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
+    }
+    if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
+        rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
+    rc |= VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, uPendingDbgXcpts);
+    AssertRC(rc);
+
+    /*
+     * Update the nested-guest VMCS cache.
+     */
+    pVmcsInfoNstGst->u32PinCtls     = u32PinCtls;
+    pVmcsInfoNstGst->u32ProcCtls    = u32ProcCtls;
+    pVmcsInfoNstGst->u32ProcCtls2   = u32ProcCtls2;
+    pVmcsInfoNstGst->u32XcptBitmap  = u32XcptBitmap;
+    pVmcsInfoNstGst->u64Cr0Mask     = u64Cr0Mask;
+    pVmcsInfoNstGst->u64Cr4Mask     = u64Cr4Mask;
+    pVmcsInfoNstGst->u32XcptPFMask  = u32XcptPFMask;
+    pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
+    pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
+
+    /*
+     * We need to flush the TLB if we are switching the APIC-access page address.
+     * See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
+     */
+    if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
+        pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
+
+    /*
+     * MSR bitmap.
+     *
+     * The MSR bitmap address has already been initialized while setting up the nested-guest
+     * VMCS, here we need to merge the MSR bitmaps.
+     */
+    if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+        hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
+
+    return VINF_SUCCESS;
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/**
+ * Does the preparations before executing guest code in VT-x.
+ *
+ * This may cause longjmps to ring-3 and may even result in rescheduling to the
+ * recompiler/IEM. We must be cautious what we do here regarding committing
+ * guest-state information into the VMCS assuming we assuredly execute the
+ * guest in VT-x mode.
+ *
+ * If we fall back to the recompiler/IEM after updating the VMCS and clearing
+ * the common-state (TRPM/forceflags), we must undo those changes so that the
+ * recompiler/IEM can (and should) use them when it resumes guest execution.
+ * Otherwise such operations must be done when we can no longer exit to ring-3.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_SUCCESS if we can proceed with running the guest, interrupts
+ *          have been disabled.
+ * @retval  VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
+ *          pending events).
+ * @retval  VINF_EM_RESET if a triple-fault occurs while injecting a
+ *          double-fault into the guest.
+ * @retval  VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
+ *          dispatched directly.
+ * @retval  VINF_* scheduling changes, we have to go back to ring-3.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   fStepping       Whether we are single-stepping the guest in the
+ *                          hypervisor debugger. Makes us ignore some of the reasons
+ *                          for returning to ring-3, and return VINF_EM_DBG_STEPPED
+ *                          if event dispatching took place.
+ */
+static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
+{
+    Assert(VMMRZCallRing3IsEnabled(pVCpu));
+
+    Log4Func(("fIsNested=%RTbool fStepping=%RTbool\n", pVmxTransient->fIsNestedGuest, fStepping));
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
+    if (pVmxTransient->fIsNestedGuest)
+    {
+        RT_NOREF2(pVCpu, fStepping);
+        Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
+        return VINF_EM_RESCHEDULE_REM;
+    }
+#endif
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PGMRZDynMapFlushAutoSet(pVCpu);
+#endif
+
+    /*
+     * Check and process force flag actions, some of which might require us to go back to ring-3.
+     */
+    VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pVmxTransient, fStepping);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /* FFs don't get set all the time. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        if (   pVmxTransient->fIsNestedGuest
+            && !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
+        {
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
+            return VINF_VMX_VMEXIT;
+        }
+#endif
+    }
+    else
+        return rcStrict;
+
+    /*
+     * Virtualize memory-mapped accesses to the physical APIC (may take locks).
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (   !pVCpu->hm.s.vmx.u64GstMsrApicBase
+        && (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
+        && PDMHasApic(pVM))
+    {
+        int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
+        AssertRCReturn(rc, rc);
+    }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * Merge guest VMCS controls with the nested-guest VMCS controls.
+     *
+     * Even if we have not executed the guest prior to this (e.g. when resuming from a
+     * saved state), we should be okay with merging controls as we initialize the
+     * guest VMCS controls as part of VM setup phase.
+     */
+    if (   pVmxTransient->fIsNestedGuest
+        && !pVCpu->hm.s.vmx.fMergedNstGstCtls)
+    {
+        int rc = hmR0VmxMergeVmcsNested(pVCpu);
+        AssertRCReturn(rc, rc);
+        pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
+    }
+#endif
+
+    /*
+     * Evaluate events to be injected into the guest.
+     *
+     * Events in TRPM can be injected without inspecting the guest state.
+     * If any new events (interrupts/NMI) are pending currently, we try to set up the
+     * guest to cause a VM-exit the next time they are ready to receive the event.
+     *
+     * With nested-guests, evaluating pending events may cause VM-exits. Also, verify
+     * that the event in TRPM that we will inject using hardware-assisted VMX is -not-
+     * subject to interecption. Otherwise, we should have checked and injected them
+     * manually elsewhere (IEM).
+     */
+    if (TRPMHasTrap(pVCpu))
+    {
+        Assert(!pVmxTransient->fIsNestedGuest || !CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
+        hmR0VmxTrpmTrapToPendingEvent(pVCpu);
+    }
+
+    uint32_t fIntrState;
+    rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * While evaluating pending events if something failed (unlikely) or if we were
+     * preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
+     */
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    if (   pVmxTransient->fIsNestedGuest
+        && !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
+    {
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
+        return VINF_VMX_VMEXIT;
+    }
+#else
+    Assert(rcStrict == VINF_SUCCESS);
+#endif
+
+    /*
+     * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
+     * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
+     * also result in triple-faulting the VM.
+     *
+     * With nested-guests, the above does not apply since unrestricted guest execution is a
+     * requirement. Regardless, we do this here to avoid duplicating code elsewhere.
+     */
+    rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* likely */ }
+    else
+    {
+        AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
+                  ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /*
+     * A longjump might result in importing CR3 even for VM-exits that don't necessarily
+     * import CR3 themselves. We will need to update them here, as even as late as the above
+     * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
+     * the below force flags to be set.
+     */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
+    {
+        Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
+        int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
+        AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
+                        ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
+    }
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
+    {
+        PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
+    }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /* Paranoia. */
+    Assert(!pVmxTransient->fIsNestedGuest || CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
+#endif
+
+    /*
+     * No longjmps to ring-3 from this point on!!!
+     * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
+     * This also disables flushing of the R0-logger instance (if any).
+     */
+    VMMRZCallRing3Disable(pVCpu);
+
+    /*
+     * Export the guest state bits.
+     *
+     * We cannot perform longjmps while loading the guest state because we do not preserve the
+     * host/guest state (although the VMCS will be preserved) across longjmps which can cause
+     * CPU migration.
+     *
+     * If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
+     * registers. Hence, exporting of the guest state needs to be done -after- injection of events.
+     */
+    rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* likely */ }
+    else
+    {
+        VMMRZCallRing3Enable(pVCpu);
+        return rcStrict;
+    }
+
+    /*
+     * We disable interrupts so that we don't miss any interrupts that would flag preemption
+     * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
+     * preemption disabled for a while.  Since this is purely to aid the
+     * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
+     * disable interrupt on NT.
+     *
+     * We need to check for force-flags that could've possible been altered since we last
+     * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
+     * see @bugref{6398}).
+     *
+     * We also check a couple of other force-flags as a last opportunity to get the EMT back
+     * to ring-3 before executing guest code.
+     */
+    pVmxTransient->fEFlags = ASMIntDisableFlags();
+
+    if (   (   !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
+            && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
+        || (   fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
+            && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
+    {
+        if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
+        {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            /*
+             * If we are executing a nested-guest make sure that we should intercept subsequent
+             * events. The one we are injecting might be part of VM-entry. This is mainly to keep
+             * the VM-exit instruction emulation happy.
+             */
+            if (pVmxTransient->fIsNestedGuest)
+                CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, true);
+#endif
+
+            /*
+             * We've injected any pending events. This is really the point of no return (to ring-3).
+             *
+             * Note! The caller expects to continue with interrupts & longjmps disabled on successful
+             *       returns from this function, so do -not- enable them here.
+             */
+            pVCpu->hm.s.Event.fPending = false;
+            return VINF_SUCCESS;
+        }
+
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
+        rcStrict = VINF_EM_RAW_INTERRUPT;
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
+        rcStrict = VINF_EM_RAW_TO_R3;
+    }
+
+    ASMSetFlags(pVmxTransient->fEFlags);
+    VMMRZCallRing3Enable(pVCpu);
+
+    return rcStrict;
+}
+
+
+/**
+ * Final preparations before executing guest code using hardware-assisted VMX.
+ *
+ * We can no longer get preempted to a different host CPU and there are no returns
+ * to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
+ * failures), this function is not intended to fail sans unrecoverable hardware
+ * errors.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks Called with preemption disabled.
+ * @remarks No-long-jump zone!!!
+ */
+static void hmR0VmxPreRunGuestCommitted(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+    Assert(VMMR0IsLogFlushDisabled(pVCpu));
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(!pVCpu->hm.s.Event.fPending);
+
+    /*
+     * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
+     */
+    VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
+
+    PVMCC         pVM          = pVCpu->CTX_SUFF(pVM);
+    PVMXVMCSINFO  pVmcsInfo    = pVmxTransient->pVmcsInfo;
+    PHMPHYSCPU    pHostCpu     = hmR0GetCurrentCpu();
+    RTCPUID const idCurrentCpu = pHostCpu->idCpu;
+
+    if (!CPUMIsGuestFPUStateActive(pVCpu))
+    {
+        STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
+        if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
+            pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
+        STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
+    }
+
+    /*
+     * Re-export the host state bits as we may've been preempted (only happens when
+     * thread-context hooks are used or when the VM start function changes) or if
+     * the host CR0 is modified while loading the guest FPU state above.
+     *
+     * The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
+     * changed the switcher back to 32-bit, we *must* save the 32-bit host state here,
+     * see @bugref{8432}.
+     *
+     * This may also happen when switching to/from a nested-guest VMCS without leaving
+     * ring-0.
+     */
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
+    {
+        hmR0VmxExportHostState(pVCpu);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
+    }
+    Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
+
+    /*
+     * Export the state shared between host and guest (FPU, debug, lazy MSRs).
+     */
+    if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
+        hmR0VmxExportSharedState(pVCpu, pVmxTransient);
+    AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
+
+    /*
+     * Store status of the shared guest/host debug state at the time of VM-entry.
+     */
+    pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
+    pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
+
+    /*
+     * Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
+     * more than one conditional check. The post-run side of our code shall determine
+     * if it needs to sync. the virtual APIC TPR with the TPR-shadow.
+     */
+    if (pVmcsInfo->pbVirtApic)
+        pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
+
+    /*
+     * Update the host MSRs values in the VM-exit MSR-load area.
+     */
+    if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
+    {
+        if (pVmcsInfo->cExitMsrLoad > 0)
+            hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
+        pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
+    }
+
+    /*
+     * Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
+     * VMX-preemption timer based on the next virtual sync clock deadline.
+     */
+    if (   !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
+        || idCurrentCpu != pVCpu->hm.s.idLastCpu)
+    {
+        hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
+        pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
+    }
+
+    /* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
+    bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
+    if (!fIsRdtscIntercepted)
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
+    else
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
+
+    ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);    /* Used for TLB flushing, set this across the world switch. */
+    hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo);          /* Invalidate the appropriate guest entries from the TLB. */
+    Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
+    pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu;      /* Record the error reporting info. with the current host CPU. */
+    pVmcsInfo->idHostCpuState = idCurrentCpu;                   /* Record the CPU for which the host-state has been exported. */
+    pVmcsInfo->idHostCpuExec  = idCurrentCpu;                   /* Record the CPU on which we shall execute. */
+
+    STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
+
+    TMNotifyStartOfExecution(pVM, pVCpu);                       /* Notify TM to resume its clocks when TSC is tied to execution,
+                                                                   as we're about to start executing the guest. */
+
+    /*
+     * Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
+     *
+     * This is done this late as updating the TSC offsetting/preemption timer above
+     * figures out if we can skip intercepting RDTSCP by calculating the number of
+     * host CPU ticks till the next virtual sync deadline (for the dynamic case).
+     */
+    if (   (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
+        && !fIsRdtscIntercepted)
+    {
+        hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
+
+        /* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
+           it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
+        int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
+                                            true /* fSetReadWrite */, true /* fUpdateHostMsr */);
+        AssertRC(rc);
+        Assert(!pVmxTransient->fRemoveTscAuxMsr);
+        pVmxTransient->fRemoveTscAuxMsr = true;
+    }
+
+#ifdef VBOX_STRICT
+    Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
+    hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
+    hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
+    AssertRC(hmR0VmxCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
+#endif
+
+#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
+    /** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
+     *        Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
+     *        see @bugref{9180#c54}. */
+    uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
+    if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
+        Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
+#endif
+}
+
+
+/**
+ * First C routine invoked after running guest code using hardware-assisted VMX.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   rcVMRun         Return code of VMLAUNCH/VMRESUME.
+ *
+ * @remarks Called with interrupts disabled, and returns with interrupts enabled!
+ *
+ * @remarks No-long-jump zone!!! This function will however re-enable longjmps
+ *          unconditionally when it is safe to do so.
+ */
+static void hmR0VmxPostRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
+{
+    uint64_t const uHostTsc = ASMReadTSC();                     /** @todo We can do a lot better here, see @bugref{9180#c38}. */
+
+    ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false);   /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
+    ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits);            /* Initialized in vmR3CreateUVM(): used for EMT poking. */
+    pVCpu->hm.s.fCtxChanged            = 0;                     /* Exits/longjmps to ring-3 requires saving the guest state. */
+    pVmxTransient->fVmcsFieldsRead     = 0;                     /* Transient fields need to be read from the VMCS. */
+    pVmxTransient->fVectoringPF        = false;                 /* Vectoring page-fault needs to be determined later. */
+    pVmxTransient->fVectoringDoublePF  = false;                 /* Vectoring double page-fault needs to be determined later. */
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
+    {
+        uint64_t uGstTsc;
+        if (!pVmxTransient->fIsNestedGuest)
+            uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
+        else
+        {
+            uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
+            uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
+        }
+        TMCpuTickSetLastSeen(pVCpu, uGstTsc);                           /* Update TM with the guest TSC. */
+    }
+
+    STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
+    TMNotifyEndOfExecution(pVCpu->CTX_SUFF(pVM), pVCpu);                /* Notify TM that the guest is no longer running. */
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
+
+    pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED;     /* Some host state messed up by VMX needs restoring. */
+    pVmcsInfo->fVmcsState |= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED;          /* Use VMRESUME instead of VMLAUNCH in the next run. */
+#ifdef VBOX_STRICT
+    hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);                          /* Verify that the host EFER MSR wasn't modified. */
+#endif
+    Assert(!ASMIntAreEnabled());
+    ASMSetFlags(pVmxTransient->fEFlags);                                /* Enable interrupts. */
+    Assert(!VMMRZCallRing3IsEnabled(pVCpu));
+
+#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
+    /*
+     * Clean all the VMCS fields in the transient structure before reading
+     * anything from the VMCS.
+     */
+    pVmxTransient->uExitReason            = 0;
+    pVmxTransient->uExitIntErrorCode      = 0;
+    pVmxTransient->uExitQual              = 0;
+    pVmxTransient->uGuestLinearAddr       = 0;
+    pVmxTransient->uExitIntInfo           = 0;
+    pVmxTransient->cbExitInstr            = 0;
+    pVmxTransient->ExitInstrInfo.u        = 0;
+    pVmxTransient->uEntryIntInfo          = 0;
+    pVmxTransient->uEntryXcptErrorCode    = 0;
+    pVmxTransient->cbEntryInstr           = 0;
+    pVmxTransient->uIdtVectoringInfo      = 0;
+    pVmxTransient->uIdtVectoringErrorCode = 0;
+#endif
+
+    /*
+     * Save the basic VM-exit reason and check if the VM-entry failed.
+     * See Intel spec. 24.9.1 "Basic VM-exit Information".
+     */
+    uint32_t uExitReason;
+    int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
+    AssertRC(rc);
+    pVmxTransient->uExitReason    = VMX_EXIT_REASON_BASIC(uExitReason);
+    pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
+
+    /*
+     * Log the VM-exit before logging anything else as otherwise it might be a
+     * tad confusing what happens before and after the world-switch.
+     */
+    HMVMX_LOG_EXIT(pVCpu, uExitReason);
+
+    /*
+     * Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
+     * bitmap permissions, if it was added before VM-entry.
+     */
+    if (pVmxTransient->fRemoveTscAuxMsr)
+    {
+        hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
+        pVmxTransient->fRemoveTscAuxMsr = false;
+    }
+
+    /*
+     * Check if VMLAUNCH/VMRESUME succeeded.
+     * If this failed, we cause a guru meditation and cease further execution.
+     *
+     * However, if we are executing a nested-guest we might fail if we use the
+     * fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
+     */
+    if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
+    {
+        /*
+         * Update the VM-exit history array here even if the VM-entry failed due to:
+         *   - Invalid guest state.
+         *   - MSR loading.
+         *   - Machine-check event.
+         *
+         * In any of the above cases we will still have a "valid" VM-exit reason
+         * despite @a fVMEntryFailed being false.
+         *
+         * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
+         *
+         * Note! We don't have CS or RIP at this point.  Will probably address that later
+         *       by amending the history entry added here.
+         */
+        EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
+                         UINT64_MAX, uHostTsc);
+
+        if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
+        {
+            VMMRZCallRing3Enable(pVCpu);
+
+            Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
+            Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
+
+#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
+            hmR0VmxReadAllRoFieldsVmcs(pVmxTransient);
+#endif
+
+            /*
+             * Import the guest-interruptibility state always as we need it while evaluating
+             * injecting events on re-entry.
+             *
+             * We don't import CR0 (when unrestricted guest execution is unavailable) despite
+             * checking for real-mode while exporting the state because all bits that cause
+             * mode changes wrt CR0 are intercepted.
+             */
+            uint64_t const fImportMask = CPUMCTX_EXTRN_HM_VMX_INT_STATE
+#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
+                                       | HMVMX_CPUMCTX_EXTRN_ALL
+#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
+                                       | CPUMCTX_EXTRN_RFLAGS
+#endif
+                                       ;
+            rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImportMask);
+            AssertRC(rc);
+
+            /*
+             * Sync the TPR shadow with our APIC state.
+             */
+            if (   !pVmxTransient->fIsNestedGuest
+                && (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
+            {
+                Assert(pVmcsInfo->pbVirtApic);
+                if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
+                {
+                    rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
+                    AssertRC(rc);
+                    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
+                }
+            }
+
+            Assert(VMMRZCallRing3IsEnabled(pVCpu));
+            return;
+        }
+    }
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    else if (pVmxTransient->fIsNestedGuest)
+        AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
+#endif
+    else
+        Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
+
+    VMMRZCallRing3Enable(pVCpu);
+}
+
+
+/**
+ * Runs the guest code using hardware-assisted VMX the normal way.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pcLoops     Pointer to the number of executed loops.
+ */
+static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
+{
+    uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
+    Assert(pcLoops);
+    Assert(*pcLoops <= cMaxResumeLoops);
+    Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * Switch to the guest VMCS as we may have transitioned from executing the nested-guest
+     * without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
+     * guest VMCS while entering the VMX ring-0 session.
+     */
+    if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
+    {
+        int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
+            return rc;
+        }
+    }
+#endif
+
+    VMXTRANSIENT VmxTransient;
+    RT_ZERO(VmxTransient);
+    VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+
+    /* Paranoia. */
+    Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
+
+    VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
+    for (;;)
+    {
+        Assert(!HMR0SuspendPending());
+        HMVMX_ASSERT_CPU_SAFE(pVCpu);
+        STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
+
+        /*
+         * Preparatory work for running nested-guest code, this may force us to
+         * return to ring-3.
+         *
+         * Warning! This bugger disables interrupts on VINF_SUCCESS!
+         */
+        rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
+        if (rcStrict != VINF_SUCCESS)
+            break;
+
+        /* Interrupts are disabled at this point! */
+        hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
+        int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
+        hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
+        /* Interrupts are re-enabled at this point! */
+
+        /*
+         * Check for errors with running the VM (VMLAUNCH/VMRESUME).
+         */
+        if (RT_SUCCESS(rcRun))
+        { /* very likely */ }
+        else
+        {
+            STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
+            hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
+            return rcRun;
+        }
+
+        /*
+         * Profile the VM-exit.
+         */
+        AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
+        STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
+        STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
+        HMVMX_START_EXIT_DISPATCH_PROF();
+
+        VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
+
+        /*
+         * Handle the VM-exit.
+         */
+#ifdef HMVMX_USE_FUNCTION_TABLE
+        rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
+#else
+        rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
+#endif
+        STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            if (++(*pcLoops) <= cMaxResumeLoops)
+                continue;
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
+            rcStrict = VINF_EM_RAW_INTERRUPT;
+        }
+        break;
+    }
+
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
+    return rcStrict;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Runs the nested-guest code using hardware-assisted VMX.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pcLoops     Pointer to the number of executed loops.
+ *
+ * @sa      hmR0VmxRunGuestCodeNormal.
+ */
+static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
+{
+    uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
+    Assert(pcLoops);
+    Assert(*pcLoops <= cMaxResumeLoops);
+    Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
+
+    /*
+     * Switch to the nested-guest VMCS as we may have transitioned from executing the
+     * guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
+     * loaded the nested-guest VMCS while entering the VMX ring-0 session.
+     */
+    if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
+    {
+        int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
+            return rc;
+        }
+    }
+
+    VMXTRANSIENT VmxTransient;
+    RT_ZERO(VmxTransient);
+    VmxTransient.pVmcsInfo      = hmGetVmxActiveVmcsInfo(pVCpu);
+    VmxTransient.fIsNestedGuest = true;
+
+    /* Paranoia. */
+    Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
+
+    VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
+    for (;;)
+    {
+        Assert(!HMR0SuspendPending());
+        HMVMX_ASSERT_CPU_SAFE(pVCpu);
+        STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
+
+        /*
+         * Preparatory work for running guest code, this may force us to
+         * return to ring-3.
+         *
+         * Warning! This bugger disables interrupts on VINF_SUCCESS!
+         */
+        rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
+        if (rcStrict != VINF_SUCCESS)
+            break;
+
+        /* Interrupts are disabled at this point! */
+        hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
+        int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
+        hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
+        /* Interrupts are re-enabled at this point! */
+
+        /*
+         * Check for errors with running the VM (VMLAUNCH/VMRESUME).
+         */
+        if (RT_SUCCESS(rcRun))
+        { /* very likely */ }
+        else
+        {
+            STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
+            hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
+            return rcRun;
+        }
+
+        /*
+         * Profile the VM-exit.
+         */
+        AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
+        STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
+        STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
+        HMVMX_START_EXIT_DISPATCH_PROF();
+
+        VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
+
+        /*
+         * Handle the VM-exit.
+         */
+        rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
+        STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
+            {
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
+                rcStrict = VINF_VMX_VMEXIT;
+            }
+            else
+            {
+                if (++(*pcLoops) <= cMaxResumeLoops)
+                    continue;
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
+                rcStrict = VINF_EM_RAW_INTERRUPT;
+            }
+        }
+        else
+            Assert(rcStrict != VINF_VMX_VMEXIT);
+        break;
+    }
+
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
+    return rcStrict;
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
+ *  probes.
+ *
+ * The following few functions and associated structure contains the bloat
+ * necessary for providing detailed debug events and dtrace probes as well as
+ * reliable host side single stepping.  This works on the principle of
+ * "subclassing" the normal execution loop and workers.  We replace the loop
+ * method completely and override selected helpers to add necessary adjustments
+ * to their core operation.
+ *
+ * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
+ * any performance for debug and analysis features.
+ *
+ * @{
+ */
+
+/**
+ * Transient per-VCPU debug state of VMCS and related info. we save/restore in
+ * the debug run loop.
+ */
+typedef struct VMXRUNDBGSTATE
+{
+    /** The RIP we started executing at.  This is for detecting that we stepped.  */
+    uint64_t    uRipStart;
+    /** The CS we started executing with.  */
+    uint16_t    uCsStart;
+
+    /** Whether we've actually modified the 1st execution control field. */
+    bool        fModifiedProcCtls : 1;
+    /** Whether we've actually modified the 2nd execution control field. */
+    bool        fModifiedProcCtls2 : 1;
+    /** Whether we've actually modified the exception bitmap. */
+    bool        fModifiedXcptBitmap : 1;
+
+    /** We desire the modified the CR0 mask to be cleared. */
+    bool        fClearCr0Mask : 1;
+    /** We desire the modified the CR4 mask to be cleared. */
+    bool        fClearCr4Mask : 1;
+    /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
+    uint32_t    fCpe1Extra;
+    /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
+    uint32_t    fCpe1Unwanted;
+    /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
+    uint32_t    fCpe2Extra;
+    /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
+    uint32_t    bmXcptExtra;
+    /** The sequence number of the Dtrace provider settings the state was
+     *  configured against. */
+    uint32_t    uDtraceSettingsSeqNo;
+    /** VM-exits to check (one bit per VM-exit). */
+    uint32_t    bmExitsToCheck[3];
+
+    /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
+    uint32_t    fProcCtlsInitial;
+    /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
+    uint32_t    fProcCtls2Initial;
+    /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
+    uint32_t    bmXcptInitial;
+} VMXRUNDBGSTATE;
+AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
+typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
+
+
+/**
+ * Initializes the VMXRUNDBGSTATE structure.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   pDbgState       The debug state to initialize.
+ */
+static void hmR0VmxRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
+{
+    pDbgState->uRipStart            = pVCpu->cpum.GstCtx.rip;
+    pDbgState->uCsStart             = pVCpu->cpum.GstCtx.cs.Sel;
+
+    pDbgState->fModifiedProcCtls    = false;
+    pDbgState->fModifiedProcCtls2   = false;
+    pDbgState->fModifiedXcptBitmap  = false;
+    pDbgState->fClearCr0Mask        = false;
+    pDbgState->fClearCr4Mask        = false;
+    pDbgState->fCpe1Extra           = 0;
+    pDbgState->fCpe1Unwanted        = 0;
+    pDbgState->fCpe2Extra           = 0;
+    pDbgState->bmXcptExtra          = 0;
+    pDbgState->fProcCtlsInitial     = pVmxTransient->pVmcsInfo->u32ProcCtls;
+    pDbgState->fProcCtls2Initial    = pVmxTransient->pVmcsInfo->u32ProcCtls2;
+    pDbgState->bmXcptInitial        = pVmxTransient->pVmcsInfo->u32XcptBitmap;
+}
+
+
+/**
+ * Updates the VMSC fields with changes requested by @a pDbgState.
+ *
+ * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
+ * immediately before executing guest code, i.e. when interrupts are disabled.
+ * We don't check status codes here as we cannot easily assert or return in the
+ * latter case.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   pDbgState       The debug state.
+ */
+static void hmR0VmxPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
+{
+    /*
+     * Ensure desired flags in VMCS control fields are set.
+     * (Ignoring write failure here, as we're committed and it's just debug extras.)
+     *
+     * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
+     *       there should be no stale data in pCtx at this point.
+     */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    if (   (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
+        || (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
+    {
+        pVmcsInfo->u32ProcCtls |= pDbgState->fCpe1Extra;
+        pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
+        VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+        Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
+        pDbgState->fModifiedProcCtls   = true;
+    }
+
+    if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
+    {
+        pVmcsInfo->u32ProcCtls2  |= pDbgState->fCpe2Extra;
+        VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
+        Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
+        pDbgState->fModifiedProcCtls2  = true;
+    }
+
+    if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
+    {
+        pVmcsInfo->u32XcptBitmap |= pDbgState->bmXcptExtra;
+        VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
+        Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
+        pDbgState->fModifiedXcptBitmap = true;
+    }
+
+    if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
+    {
+        pVmcsInfo->u64Cr0Mask = 0;
+        VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, 0);
+        Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
+    }
+
+    if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
+    {
+        pVmcsInfo->u64Cr4Mask = 0;
+        VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, 0);
+        Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
+    }
+
+    NOREF(pVCpu);
+}
+
+
+/**
+ * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
+ * re-entry next time around.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   pDbgState       The debug state.
+ * @param   rcStrict        The return code from executing the guest using single
+ *                          stepping.
+ */
+static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
+                                               VBOXSTRICTRC rcStrict)
+{
+    /*
+     * Restore VM-exit control settings as we may not reenter this function the
+     * next time around.
+     */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+
+    /* We reload the initial value, trigger what we can of recalculations the
+       next time around.  From the looks of things, that's all that's required atm. */
+    if (pDbgState->fModifiedProcCtls)
+    {
+        if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
+            pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
+        int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
+        AssertRC(rc2);
+        pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
+    }
+
+    /* We're currently the only ones messing with this one, so just restore the
+       cached value and reload the field. */
+    if (   pDbgState->fModifiedProcCtls2
+        && pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
+    {
+        int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
+        AssertRC(rc2);
+        pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
+    }
+
+    /* If we've modified the exception bitmap, we restore it and trigger
+       reloading and partial recalculation the next time around. */
+    if (pDbgState->fModifiedXcptBitmap)
+        pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
+
+    return rcStrict;
+}
+
+
+/**
+ * Configures VM-exit controls for current DBGF and DTrace settings.
+ *
+ * This updates @a pDbgState and the VMCS execution control fields to reflect
+ * the necessary VM-exits demanded by DBGF and DTrace.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure. May update
+ *                          fUpdatedTscOffsettingAndPreemptTimer.
+ * @param   pDbgState       The debug state.
+ */
+static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
+{
+    /*
+     * Take down the dtrace serial number so we can spot changes.
+     */
+    pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
+    ASMCompilerBarrier();
+
+    /*
+     * We'll rebuild most of the middle block of data members (holding the
+     * current settings) as we go along here, so start by clearing it all.
+     */
+    pDbgState->bmXcptExtra      = 0;
+    pDbgState->fCpe1Extra       = 0;
+    pDbgState->fCpe1Unwanted    = 0;
+    pDbgState->fCpe2Extra       = 0;
+    for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
+        pDbgState->bmExitsToCheck[i] = 0;
+
+    /*
+     * Software interrupts (INT XXh) - no idea how to trigger these...
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (   DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
+        || VBOXVMM_INT_SOFTWARE_ENABLED())
+    {
+        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
+    }
+
+    /*
+     * INT3 breakpoints - triggered by #BP exceptions.
+     */
+    if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
+        pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
+
+    /*
+     * Exception bitmap and XCPT events+probes.
+     */
+    for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
+        if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
+            pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
+
+    if (VBOXVMM_XCPT_DE_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
+    if (VBOXVMM_XCPT_DB_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
+    if (VBOXVMM_XCPT_BP_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
+    if (VBOXVMM_XCPT_OF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
+    if (VBOXVMM_XCPT_BR_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
+    if (VBOXVMM_XCPT_UD_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
+    if (VBOXVMM_XCPT_NM_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
+    if (VBOXVMM_XCPT_DF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
+    if (VBOXVMM_XCPT_TS_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
+    if (VBOXVMM_XCPT_NP_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
+    if (VBOXVMM_XCPT_SS_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
+    if (VBOXVMM_XCPT_GP_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
+    if (VBOXVMM_XCPT_PF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
+    if (VBOXVMM_XCPT_MF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
+    if (VBOXVMM_XCPT_AC_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
+    if (VBOXVMM_XCPT_XF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
+    if (VBOXVMM_XCPT_VE_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
+    if (VBOXVMM_XCPT_SX_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
+
+    if (pDbgState->bmXcptExtra)
+        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
+
+    /*
+     * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
+     *
+     * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
+     *       So, when adding/changing/removing please don't forget to update it.
+     *
+     * Some of the macros are picking up local variables to save horizontal space,
+     * (being able to see it in a table is the lesser evil here).
+     */
+#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
+        (    DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
+         ||  RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
+#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
+        if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
+        {   AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
+            ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
+        } else do { } while (0)
+#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
+        if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
+        { \
+            (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
+            AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
+            ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
+        } else do { } while (0)
+#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
+        if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
+        { \
+            (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
+            AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
+            ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
+        } else do { } while (0)
+#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
+        if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
+        { \
+            (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
+            AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
+            ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
+        } else do { } while (0)
+
+    SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH,         VMX_EXIT_TASK_SWITCH);   /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION,   VMX_EXIT_EPT_VIOLATION); /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG,   VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
+    SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS,    VMX_EXIT_APIC_ACCESS);   /* feature dependent, nothing to enable here */
+    SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE,     VMX_EXIT_APIC_WRITE);    /* feature dependent, nothing to enable here */
+
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID,              VMX_EXIT_CPUID);         /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID,              VMX_EXIT_CPUID);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC,             VMX_EXIT_GETSEC);        /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC,             VMX_EXIT_GETSEC);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT,               VMX_EXIT_HLT,      VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT,               VMX_EXIT_HLT);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD,               VMX_EXIT_INVD);          /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD,               VMX_EXIT_INVD);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG,             VMX_EXIT_INVLPG,   VMX_PROC_CTLS_INVLPG_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG,             VMX_EXIT_INVLPG);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC,              VMX_EXIT_RDPMC,    VMX_PROC_CTLS_RDPMC_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC,              VMX_EXIT_RDPMC);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC,              VMX_EXIT_RDTSC,    VMX_PROC_CTLS_RDTSC_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC,              VMX_EXIT_RDTSC);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM,                VMX_EXIT_RSM);           /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM,                VMX_EXIT_RSM);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL,           VMX_EXIT_VMCALL);        /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL,           VMX_EXIT_VMCALL);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR,        VMX_EXIT_VMCLEAR);       /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR,        VMX_EXIT_VMCLEAR);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH,       VMX_EXIT_VMLAUNCH);      /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH,       VMX_EXIT_VMLAUNCH);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD,        VMX_EXIT_VMPTRLD);       /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD,        VMX_EXIT_VMPTRLD);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST,        VMX_EXIT_VMPTRST);       /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST,        VMX_EXIT_VMPTRST);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD,         VMX_EXIT_VMREAD);        /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD,         VMX_EXIT_VMREAD);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME,       VMX_EXIT_VMRESUME);      /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME,       VMX_EXIT_VMRESUME);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE,        VMX_EXIT_VMWRITE);       /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE,        VMX_EXIT_VMWRITE);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF,         VMX_EXIT_VMXOFF);        /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF,         VMX_EXIT_VMXOFF);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON,          VMX_EXIT_VMXON);         /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON,          VMX_EXIT_VMXON);
+
+    if (   IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
+        || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
+    {
+        int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4
+                                                                        | CPUMCTX_EXTRN_APIC_TPR);
+        AssertRC(rc);
+
+#if 0 /** @todo fix me */
+        pDbgState->fClearCr0Mask = true;
+        pDbgState->fClearCr4Mask = true;
+#endif
+        if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
+            pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
+        if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
+            pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
+        pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
+        /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT.  It would
+                 require clearing here and in the loop if we start using it. */
+        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
+    }
+    else
+    {
+        if (pDbgState->fClearCr0Mask)
+        {
+            pDbgState->fClearCr0Mask = false;
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
+        }
+        if (pDbgState->fClearCr4Mask)
+        {
+            pDbgState->fClearCr4Mask = false;
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
+        }
+    }
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ,           VMX_EXIT_MOV_CRX);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE,          VMX_EXIT_MOV_CRX);
+
+    if (   IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
+        || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
+    {
+        /** @todo later, need to fix handler as it assumes this won't usually happen. */
+        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
+    }
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ,           VMX_EXIT_MOV_DRX);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE,          VMX_EXIT_MOV_DRX);
+
+    SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR,              VMX_EXIT_RDMSR,    VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR,              VMX_EXIT_RDMSR);
+    SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR,              VMX_EXIT_WRMSR,    VMX_PROC_CTLS_USE_MSR_BITMAPS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR,              VMX_EXIT_WRMSR);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT,              VMX_EXIT_MWAIT,    VMX_PROC_CTLS_MWAIT_EXIT);   /* paranoia */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT,              VMX_EXIT_MWAIT);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR,            VMX_EXIT_MONITOR,  VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR,            VMX_EXIT_MONITOR);
+#if 0 /** @todo too slow, fix handler. */
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE,              VMX_EXIT_PAUSE,    VMX_PROC_CTLS_PAUSE_EXIT);
+#endif
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE,              VMX_EXIT_PAUSE);
+
+    if (   IS_EITHER_ENABLED(pVM, INSTR_SGDT)
+        || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
+        || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
+        || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
+    {
+        pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
+        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
+    }
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT,               VMX_EXIT_GDTR_IDTR_ACCESS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT,               VMX_EXIT_GDTR_IDTR_ACCESS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT,               VMX_EXIT_GDTR_IDTR_ACCESS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT,               VMX_EXIT_GDTR_IDTR_ACCESS);
+
+    if (   IS_EITHER_ENABLED(pVM, INSTR_SLDT)
+        || IS_EITHER_ENABLED(pVM, INSTR_STR)
+        || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
+        || IS_EITHER_ENABLED(pVM, INSTR_LTR))
+    {
+        pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
+        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
+    }
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT,               VMX_EXIT_LDTR_TR_ACCESS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR,                VMX_EXIT_LDTR_TR_ACCESS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT,               VMX_EXIT_LDTR_TR_ACCESS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR,                VMX_EXIT_LDTR_TR_ACCESS);
+
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT,         VMX_EXIT_INVEPT);        /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT,         VMX_EXIT_INVEPT);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP,             VMX_EXIT_RDTSCP,   VMX_PROC_CTLS_RDTSC_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP,             VMX_EXIT_RDTSCP);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID,        VMX_EXIT_INVVPID);       /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID,        VMX_EXIT_INVVPID);
+    SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD,             VMX_EXIT_WBINVD,   VMX_PROC_CTLS2_WBINVD_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD,             VMX_EXIT_WBINVD);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV,             VMX_EXIT_XSETBV);        /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV,             VMX_EXIT_XSETBV);
+    SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND,             VMX_EXIT_RDRAND,   VMX_PROC_CTLS2_RDRAND_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND,             VMX_EXIT_RDRAND);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID,        VMX_EXIT_INVPCID,  VMX_PROC_CTLS_INVLPG_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID,        VMX_EXIT_INVPCID);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC,         VMX_EXIT_VMFUNC);        /* unconditional for the current setup */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC,         VMX_EXIT_VMFUNC);
+    SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED,             VMX_EXIT_RDSEED,   VMX_PROC_CTLS2_RDSEED_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED,             VMX_EXIT_RDSEED);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES,             VMX_EXIT_XSAVES);        /* unconditional (enabled by host, guest cfg) */
+    SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES,              VMX_EXIT_XSAVES);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS,            VMX_EXIT_XRSTORS);       /* unconditional (enabled by host, guest cfg) */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS,            VMX_EXIT_XRSTORS);
+
+#undef IS_EITHER_ENABLED
+#undef SET_ONLY_XBM_IF_EITHER_EN
+#undef SET_CPE1_XBM_IF_EITHER_EN
+#undef SET_CPEU_XBM_IF_EITHER_EN
+#undef SET_CPE2_XBM_IF_EITHER_EN
+
+    /*
+     * Sanitize the control stuff.
+     */
+    pDbgState->fCpe2Extra       &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
+    if (pDbgState->fCpe2Extra)
+        pDbgState->fCpe1Extra   |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
+    pDbgState->fCpe1Extra       &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
+    pDbgState->fCpe1Unwanted    &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
+    if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
+    {
+        pVCpu->hm.s.fDebugWantRdTscExit ^= true;
+        pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
+    }
+
+    Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
+          pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
+          pDbgState->fClearCr0Mask ? " clr-cr0" : "",
+          pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
+}
+
+
+/**
+ * Fires off DBGF events and dtrace probes for a VM-exit, when it's
+ * appropriate.
+ *
+ * The caller has checked the VM-exit against the
+ * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
+ * already, so we don't have to do that either.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   uExitReason     The VM-exit reason.
+ *
+ * @remarks The name of this function is displayed by dtrace, so keep it short
+ *          and to the point. No longer than 33 chars long, please.
+ */
+static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
+{
+    /*
+     * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
+     * same time check whether any corresponding Dtrace event is enabled (fDtrace).
+     *
+     * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
+     *       does.  Must add/change/remove both places.  Same ordering, please.
+     *
+     *       Added/removed events must also be reflected in the next section
+     *       where we dispatch dtrace events.
+     */
+    bool            fDtrace1   = false;
+    bool            fDtrace2   = false;
+    DBGFEVENTTYPE   enmEvent1  = DBGFEVENT_END;
+    DBGFEVENTTYPE   enmEvent2  = DBGFEVENT_END;
+    uint32_t        uEventArg  = 0;
+#define SET_EXIT(a_EventSubName) \
+        do { \
+            enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_,  a_EventSubName); \
+            fDtrace2  = RT_CONCAT3(VBOXVMM_EXIT_,   a_EventSubName, _ENABLED)(); \
+        } while (0)
+#define SET_BOTH(a_EventSubName) \
+        do { \
+            enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
+            enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_,  a_EventSubName); \
+            fDtrace1  = RT_CONCAT3(VBOXVMM_INSTR_,  a_EventSubName, _ENABLED)(); \
+            fDtrace2  = RT_CONCAT3(VBOXVMM_EXIT_,   a_EventSubName, _ENABLED)(); \
+        } while (0)
+    switch (uExitReason)
+    {
+        case VMX_EXIT_MTF:
+            return hmR0VmxExitMtf(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_XCPT_OR_NMI:
+        {
+            uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
+            switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
+            {
+                case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
+                case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
+                case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
+                    if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
+                    {
+                        if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
+                        {
+                            hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
+                            uEventArg = pVmxTransient->uExitIntErrorCode;
+                        }
+                        enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
+                        switch (enmEvent1)
+                        {
+                            case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
+                            case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
+                            case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
+                            case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
+                            case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
+                            case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
+                            case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
+                            case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
+                            case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
+                            case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
+                            case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
+                            case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
+                            case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
+                            case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
+                            case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
+                            case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
+                            case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
+                            case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
+                            default:                                                      break;
+                        }
+                    }
+                    else
+                        AssertFailed();
+                    break;
+
+                case VMX_EXIT_INT_INFO_TYPE_SW_INT:
+                    uEventArg = idxVector;
+                    enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
+                    fDtrace1  = VBOXVMM_INT_SOFTWARE_ENABLED();
+                    break;
+            }
+            break;
+        }
+
+        case VMX_EXIT_TRIPLE_FAULT:
+            enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
+            //fDtrace1  = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
+            break;
+        case VMX_EXIT_TASK_SWITCH:      SET_EXIT(TASK_SWITCH); break;
+        case VMX_EXIT_EPT_VIOLATION:    SET_EXIT(VMX_EPT_VIOLATION); break;
+        case VMX_EXIT_EPT_MISCONFIG:    SET_EXIT(VMX_EPT_MISCONFIG); break;
+        case VMX_EXIT_APIC_ACCESS:      SET_EXIT(VMX_VAPIC_ACCESS); break;
+        case VMX_EXIT_APIC_WRITE:       SET_EXIT(VMX_VAPIC_WRITE); break;
+
+        /* Instruction specific VM-exits: */
+        case VMX_EXIT_CPUID:            SET_BOTH(CPUID); break;
+        case VMX_EXIT_GETSEC:           SET_BOTH(GETSEC); break;
+        case VMX_EXIT_HLT:              SET_BOTH(HALT); break;
+        case VMX_EXIT_INVD:             SET_BOTH(INVD); break;
+        case VMX_EXIT_INVLPG:           SET_BOTH(INVLPG); break;
+        case VMX_EXIT_RDPMC:            SET_BOTH(RDPMC); break;
+        case VMX_EXIT_RDTSC:            SET_BOTH(RDTSC); break;
+        case VMX_EXIT_RSM:              SET_BOTH(RSM); break;
+        case VMX_EXIT_VMCALL:           SET_BOTH(VMM_CALL); break;
+        case VMX_EXIT_VMCLEAR:          SET_BOTH(VMX_VMCLEAR); break;
+        case VMX_EXIT_VMLAUNCH:         SET_BOTH(VMX_VMLAUNCH); break;
+        case VMX_EXIT_VMPTRLD:          SET_BOTH(VMX_VMPTRLD); break;
+        case VMX_EXIT_VMPTRST:          SET_BOTH(VMX_VMPTRST); break;
+        case VMX_EXIT_VMREAD:           SET_BOTH(VMX_VMREAD); break;
+        case VMX_EXIT_VMRESUME:         SET_BOTH(VMX_VMRESUME); break;
+        case VMX_EXIT_VMWRITE:          SET_BOTH(VMX_VMWRITE); break;
+        case VMX_EXIT_VMXOFF:           SET_BOTH(VMX_VMXOFF); break;
+        case VMX_EXIT_VMXON:            SET_BOTH(VMX_VMXON); break;
+        case VMX_EXIT_MOV_CRX:
+            hmR0VmxReadExitQualVmcs(pVmxTransient);
+            if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
+                SET_BOTH(CRX_READ);
+            else
+                SET_BOTH(CRX_WRITE);
+            uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
+            break;
+        case VMX_EXIT_MOV_DRX:
+            hmR0VmxReadExitQualVmcs(pVmxTransient);
+            if (   VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
+                == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
+                SET_BOTH(DRX_READ);
+            else
+                SET_BOTH(DRX_WRITE);
+            uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
+            break;
+        case VMX_EXIT_RDMSR:            SET_BOTH(RDMSR); break;
+        case VMX_EXIT_WRMSR:            SET_BOTH(WRMSR); break;
+        case VMX_EXIT_MWAIT:            SET_BOTH(MWAIT); break;
+        case VMX_EXIT_MONITOR:          SET_BOTH(MONITOR); break;
+        case VMX_EXIT_PAUSE:            SET_BOTH(PAUSE); break;
+        case VMX_EXIT_GDTR_IDTR_ACCESS:
+            hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+            switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
+            {
+                case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
+                case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
+                case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
+                case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
+            }
+            break;
+
+        case VMX_EXIT_LDTR_TR_ACCESS:
+            hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+            switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
+            {
+                case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
+                case VMX_YYTR_INSINFO_II_STR:  SET_BOTH(STR); break;
+                case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
+                case VMX_YYTR_INSINFO_II_LTR:  SET_BOTH(LTR); break;
+            }
+            break;
+
+        case VMX_EXIT_INVEPT:           SET_BOTH(VMX_INVEPT); break;
+        case VMX_EXIT_RDTSCP:           SET_BOTH(RDTSCP); break;
+        case VMX_EXIT_INVVPID:          SET_BOTH(VMX_INVVPID); break;
+        case VMX_EXIT_WBINVD:           SET_BOTH(WBINVD); break;
+        case VMX_EXIT_XSETBV:           SET_BOTH(XSETBV); break;
+        case VMX_EXIT_RDRAND:           SET_BOTH(RDRAND); break;
+        case VMX_EXIT_INVPCID:          SET_BOTH(VMX_INVPCID); break;
+        case VMX_EXIT_VMFUNC:           SET_BOTH(VMX_VMFUNC); break;
+        case VMX_EXIT_RDSEED:           SET_BOTH(RDSEED); break;
+        case VMX_EXIT_XSAVES:           SET_BOTH(XSAVES); break;
+        case VMX_EXIT_XRSTORS:          SET_BOTH(XRSTORS); break;
+
+        /* Events that aren't relevant at this point. */
+        case VMX_EXIT_EXT_INT:
+        case VMX_EXIT_INT_WINDOW:
+        case VMX_EXIT_NMI_WINDOW:
+        case VMX_EXIT_TPR_BELOW_THRESHOLD:
+        case VMX_EXIT_PREEMPT_TIMER:
+        case VMX_EXIT_IO_INSTR:
+            break;
+
+        /* Errors and unexpected events. */
+        case VMX_EXIT_INIT_SIGNAL:
+        case VMX_EXIT_SIPI:
+        case VMX_EXIT_IO_SMI:
+        case VMX_EXIT_SMI:
+        case VMX_EXIT_ERR_INVALID_GUEST_STATE:
+        case VMX_EXIT_ERR_MSR_LOAD:
+        case VMX_EXIT_ERR_MACHINE_CHECK:
+        case VMX_EXIT_PML_FULL:
+        case VMX_EXIT_VIRTUALIZED_EOI:
+            break;
+
+        default:
+            AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
+            break;
+    }
+#undef SET_BOTH
+#undef SET_EXIT
+
+    /*
+     * Dtrace tracepoints go first.   We do them here at once so we don't
+     * have to copy the guest state saving and stuff a few dozen times.
+     * Down side is that we've got to repeat the switch, though this time
+     * we use enmEvent since the probes are a subset of what DBGF does.
+     */
+    if (fDtrace1 || fDtrace2)
+    {
+        hmR0VmxReadExitQualVmcs(pVmxTransient);
+        hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+        switch (enmEvent1)
+        {
+            /** @todo consider which extra parameters would be helpful for each probe.   */
+            case DBGFEVENT_END: break;
+            case DBGFEVENT_XCPT_DE:                 VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_DB:                 VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
+            case DBGFEVENT_XCPT_BP:                 VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_OF:                 VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_BR:                 VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_UD:                 VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_NM:                 VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_DF:                 VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_TS:                 VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
+            case DBGFEVENT_XCPT_NP:                 VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
+            case DBGFEVENT_XCPT_SS:                 VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
+            case DBGFEVENT_XCPT_GP:                 VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
+            case DBGFEVENT_XCPT_PF:                 VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
+            case DBGFEVENT_XCPT_MF:                 VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_AC:                 VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_XF:                 VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_VE:                 VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_SX:                 VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
+            case DBGFEVENT_INTERRUPT_SOFTWARE:      VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_INSTR_CPUID:             VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
+            case DBGFEVENT_INSTR_GETSEC:            VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_HALT:              VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_INVD:              VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_INVLPG:            VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_RDPMC:             VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_RDTSC:             VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_RSM:               VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_CRX_READ:          VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_INSTR_CRX_WRITE:         VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_INSTR_DRX_READ:          VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_INSTR_DRX_WRITE:         VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_INSTR_RDMSR:             VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
+            case DBGFEVENT_INSTR_WRMSR:             VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
+                                                                        RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
+            case DBGFEVENT_INSTR_MWAIT:             VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_MONITOR:           VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_PAUSE:             VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_SGDT:              VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_SIDT:              VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_LGDT:              VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_LIDT:              VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_SLDT:              VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_STR:               VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_LLDT:              VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_LTR:               VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_RDTSCP:            VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_WBINVD:            VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_XSETBV:            VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_RDRAND:            VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_RDSEED:            VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_XSAVES:            VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_XRSTORS:           VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMM_CALL:          VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMCLEAR:       VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMLAUNCH:      VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMPTRLD:       VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMPTRST:       VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMREAD:        VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMRESUME:      VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMWRITE:       VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMXOFF:        VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMXON:         VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_INVEPT:        VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_INVVPID:       VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_INVPCID:       VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMFUNC:        VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
+            default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
+        }
+        switch (enmEvent2)
+        {
+            /** @todo consider which extra parameters would be helpful for each probe. */
+            case DBGFEVENT_END: break;
+            case DBGFEVENT_EXIT_TASK_SWITCH:        VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_CPUID:              VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
+            case DBGFEVENT_EXIT_GETSEC:             VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_HALT:               VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_INVD:               VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_INVLPG:             VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_RDPMC:              VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_RDTSC:              VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_RSM:                VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_CRX_READ:           VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_EXIT_CRX_WRITE:          VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_EXIT_DRX_READ:           VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_EXIT_DRX_WRITE:          VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_EXIT_RDMSR:              VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
+            case DBGFEVENT_EXIT_WRMSR:              VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
+                                                                       RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
+            case DBGFEVENT_EXIT_MWAIT:              VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_MONITOR:            VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_PAUSE:              VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_SGDT:               VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_SIDT:               VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_LGDT:               VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_LIDT:               VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_SLDT:               VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_STR:                VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_LLDT:               VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_LTR:                VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_RDTSCP:             VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_WBINVD:             VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_XSETBV:             VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_RDRAND:             VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_RDSEED:             VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_XSAVES:             VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_XRSTORS:            VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMM_CALL:           VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMCLEAR:        VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMLAUNCH:       VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMPTRLD:        VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMPTRST:        VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMREAD:         VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMRESUME:       VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMWRITE:        VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMXOFF:         VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMXON:          VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_INVEPT:         VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_INVVPID:        VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_INVPCID:        VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMFUNC:         VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG:  VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_EPT_VIOLATION:  VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS:   VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VAPIC_WRITE:    VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
+            default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
+        }
+    }
+
+    /*
+     * Fire of the DBGF event, if enabled (our check here is just a quick one,
+     * the DBGF call will do a full check).
+     *
+     * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
+     * Note! If we have to events, we prioritize the first, i.e. the instruction
+     *       one, in order to avoid event nesting.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (   enmEvent1 != DBGFEVENT_END
+        && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
+    {
+        hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
+        VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+    else if (   enmEvent2 != DBGFEVENT_END
+             && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
+    {
+        hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
+        VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Single-stepping VM-exit filtering.
+ *
+ * This is preprocessing the VM-exits and deciding whether we've gotten far
+ * enough to return VINF_EM_DBG_STEPPED already.  If not, normal VM-exit
+ * handling is performed.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   pDbgState       The debug state.
+ */
+DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
+{
+    /*
+     * Expensive (saves context) generic dtrace VM-exit probe.
+     */
+    uint32_t const uExitReason = pVmxTransient->uExitReason;
+    if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
+    { /* more likely */ }
+    else
+    {
+        hmR0VmxReadExitQualVmcs(pVmxTransient);
+        int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+        AssertRC(rc);
+        VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
+    }
+
+    /*
+     * Check for host NMI, just to get that out of the way.
+     */
+    if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
+    { /* normally likely */ }
+    else
+    {
+        hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
+        uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
+        if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
+            return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
+    }
+
+    /*
+     * Check for single stepping event if we're stepping.
+     */
+    if (pVCpu->hm.s.fSingleInstruction)
+    {
+        switch (uExitReason)
+        {
+            case VMX_EXIT_MTF:
+                return hmR0VmxExitMtf(pVCpu, pVmxTransient);
+
+            /* Various events: */
+            case VMX_EXIT_XCPT_OR_NMI:
+            case VMX_EXIT_EXT_INT:
+            case VMX_EXIT_TRIPLE_FAULT:
+            case VMX_EXIT_INT_WINDOW:
+            case VMX_EXIT_NMI_WINDOW:
+            case VMX_EXIT_TASK_SWITCH:
+            case VMX_EXIT_TPR_BELOW_THRESHOLD:
+            case VMX_EXIT_APIC_ACCESS:
+            case VMX_EXIT_EPT_VIOLATION:
+            case VMX_EXIT_EPT_MISCONFIG:
+            case VMX_EXIT_PREEMPT_TIMER:
+
+            /* Instruction specific VM-exits: */
+            case VMX_EXIT_CPUID:
+            case VMX_EXIT_GETSEC:
+            case VMX_EXIT_HLT:
+            case VMX_EXIT_INVD:
+            case VMX_EXIT_INVLPG:
+            case VMX_EXIT_RDPMC:
+            case VMX_EXIT_RDTSC:
+            case VMX_EXIT_RSM:
+            case VMX_EXIT_VMCALL:
+            case VMX_EXIT_VMCLEAR:
+            case VMX_EXIT_VMLAUNCH:
+            case VMX_EXIT_VMPTRLD:
+            case VMX_EXIT_VMPTRST:
+            case VMX_EXIT_VMREAD:
+            case VMX_EXIT_VMRESUME:
+            case VMX_EXIT_VMWRITE:
+            case VMX_EXIT_VMXOFF:
+            case VMX_EXIT_VMXON:
+            case VMX_EXIT_MOV_CRX:
+            case VMX_EXIT_MOV_DRX:
+            case VMX_EXIT_IO_INSTR:
+            case VMX_EXIT_RDMSR:
+            case VMX_EXIT_WRMSR:
+            case VMX_EXIT_MWAIT:
+            case VMX_EXIT_MONITOR:
+            case VMX_EXIT_PAUSE:
+            case VMX_EXIT_GDTR_IDTR_ACCESS:
+            case VMX_EXIT_LDTR_TR_ACCESS:
+            case VMX_EXIT_INVEPT:
+            case VMX_EXIT_RDTSCP:
+            case VMX_EXIT_INVVPID:
+            case VMX_EXIT_WBINVD:
+            case VMX_EXIT_XSETBV:
+            case VMX_EXIT_RDRAND:
+            case VMX_EXIT_INVPCID:
+            case VMX_EXIT_VMFUNC:
+            case VMX_EXIT_RDSEED:
+            case VMX_EXIT_XSAVES:
+            case VMX_EXIT_XRSTORS:
+            {
+                int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
+                AssertRCReturn(rc, rc);
+                if (   pVCpu->cpum.GstCtx.rip    != pDbgState->uRipStart
+                    || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
+                    return VINF_EM_DBG_STEPPED;
+                break;
+            }
+
+            /* Errors and unexpected events: */
+            case VMX_EXIT_INIT_SIGNAL:
+            case VMX_EXIT_SIPI:
+            case VMX_EXIT_IO_SMI:
+            case VMX_EXIT_SMI:
+            case VMX_EXIT_ERR_INVALID_GUEST_STATE:
+            case VMX_EXIT_ERR_MSR_LOAD:
+            case VMX_EXIT_ERR_MACHINE_CHECK:
+            case VMX_EXIT_PML_FULL:
+            case VMX_EXIT_VIRTUALIZED_EOI:
+            case VMX_EXIT_APIC_WRITE:  /* Some talk about this being fault like, so I guess we must process it? */
+                break;
+
+            default:
+                AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
+                break;
+        }
+    }
+
+    /*
+     * Check for debugger event breakpoints and dtrace probes.
+     */
+    if (   uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
+        && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
+    {
+        VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * Normal processing.
+     */
+#ifdef HMVMX_USE_FUNCTION_TABLE
+    return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
+#else
+    return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
+#endif
+}
+
+
+/**
+ * Single steps guest code using hardware-assisted VMX.
+ *
+ * This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
+ * but single-stepping through the hypervisor debugger.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pcLoops     Pointer to the number of executed loops.
+ *
+ * @note    Mostly the same as hmR0VmxRunGuestCodeNormal().
+ */
+static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
+{
+    uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
+    Assert(pcLoops);
+    Assert(*pcLoops <= cMaxResumeLoops);
+
+    VMXTRANSIENT VmxTransient;
+    RT_ZERO(VmxTransient);
+    VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+
+    /* Set HMCPU indicators.  */
+    bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
+    pVCpu->hm.s.fSingleInstruction     = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
+    pVCpu->hm.s.fDebugWantRdTscExit    = false;
+    pVCpu->hm.s.fUsingDebugLoop        = true;
+
+    /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps.  */
+    VMXRUNDBGSTATE DbgState;
+    hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
+    hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
+
+    /*
+     * The loop.
+     */
+    VBOXSTRICTRC rcStrict  = VERR_INTERNAL_ERROR_5;
+    for (;;)
+    {
+        Assert(!HMR0SuspendPending());
+        HMVMX_ASSERT_CPU_SAFE(pVCpu);
+        STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
+        bool fStepping = pVCpu->hm.s.fSingleInstruction;
+
+        /* Set up VM-execution controls the next two can respond to. */
+        hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
+
+        /*
+         * Preparatory work for running guest code, this may force us to
+         * return to ring-3.
+         *
+         * Warning! This bugger disables interrupts on VINF_SUCCESS!
+         */
+        rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
+        if (rcStrict != VINF_SUCCESS)
+            break;
+
+        /* Interrupts are disabled at this point! */
+        hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
+
+        /* Override any obnoxious code in the above two calls. */
+        hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
+
+        /*
+         * Finally execute the guest.
+         */
+        int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
+
+        hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
+        /* Interrupts are re-enabled at this point! */
+
+        /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
+        if (RT_SUCCESS(rcRun))
+        { /* very likely */ }
+        else
+        {
+            STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
+            hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
+            return rcRun;
+        }
+
+        /* Profile the VM-exit. */
+        AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
+        STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
+        STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
+        HMVMX_START_EXIT_DISPATCH_PROF();
+
+        VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
+
+        /*
+         * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
+         */
+        rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
+        STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
+        if (rcStrict != VINF_SUCCESS)
+            break;
+        if (++(*pcLoops) > cMaxResumeLoops)
+        {
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
+            rcStrict = VINF_EM_RAW_INTERRUPT;
+            break;
+        }
+
+        /*
+         * Stepping: Did the RIP change, if so, consider it a single step.
+         * Otherwise, make sure one of the TFs gets set.
+         */
+        if (fStepping)
+        {
+            int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
+            AssertRC(rc);
+            if (   pVCpu->cpum.GstCtx.rip    != DbgState.uRipStart
+                || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
+            {
+                rcStrict = VINF_EM_DBG_STEPPED;
+                break;
+            }
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
+        }
+
+        /*
+         * Update when dtrace settings changes (DBGF kicks us, so no need to check).
+         */
+        if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
+            hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
+    }
+
+    /*
+     * Clear the X86_EFL_TF if necessary.
+     */
+    if (pVCpu->hm.s.fClearTrapFlag)
+    {
+        int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
+        AssertRC(rc);
+        pVCpu->hm.s.fClearTrapFlag = false;
+        pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
+    }
+    /** @todo there seems to be issues with the resume flag when the monitor trap
+     *        flag is pending without being used. Seen early in bios init when
+     *        accessing APIC page in protected mode. */
+
+    /*
+     * Restore VM-exit control settings as we may not re-enter this function the
+     * next time around.
+     */
+    rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
+
+    /* Restore HMCPU indicators. */
+    pVCpu->hm.s.fUsingDebugLoop     = false;
+    pVCpu->hm.s.fDebugWantRdTscExit = false;
+    pVCpu->hm.s.fSingleInstruction  = fSavedSingleInstruction;
+
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
+    return rcStrict;
+}
+
+
+/** @} */
+
+
+/**
+ * Checks if any expensive dtrace probes are enabled and we should go to the
+ * debug loop.
+ *
+ * @returns true if we should use debug loop, false if not.
+ */
+static bool hmR0VmxAnyExpensiveProbesEnabled(void)
+{
+    /* It's probably faster to OR the raw 32-bit counter variables together.
+       Since the variables are in an array and the probes are next to one
+       another (more or less), we have good locality.  So, better read
+       eight-nine cache lines ever time and only have one conditional, than
+       128+ conditionals, right? */
+    return (  VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
+            | VBOXVMM_XCPT_DE_ENABLED_RAW()
+            | VBOXVMM_XCPT_DB_ENABLED_RAW()
+            | VBOXVMM_XCPT_BP_ENABLED_RAW()
+            | VBOXVMM_XCPT_OF_ENABLED_RAW()
+            | VBOXVMM_XCPT_BR_ENABLED_RAW()
+            | VBOXVMM_XCPT_UD_ENABLED_RAW()
+            | VBOXVMM_XCPT_NM_ENABLED_RAW()
+            | VBOXVMM_XCPT_DF_ENABLED_RAW()
+            | VBOXVMM_XCPT_TS_ENABLED_RAW()
+            | VBOXVMM_XCPT_NP_ENABLED_RAW()
+            | VBOXVMM_XCPT_SS_ENABLED_RAW()
+            | VBOXVMM_XCPT_GP_ENABLED_RAW()
+            | VBOXVMM_XCPT_PF_ENABLED_RAW()
+            | VBOXVMM_XCPT_MF_ENABLED_RAW()
+            | VBOXVMM_XCPT_AC_ENABLED_RAW()
+            | VBOXVMM_XCPT_XF_ENABLED_RAW()
+            | VBOXVMM_XCPT_VE_ENABLED_RAW()
+            | VBOXVMM_XCPT_SX_ENABLED_RAW()
+            | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
+            | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
+           ) != 0
+        || (  VBOXVMM_INSTR_HALT_ENABLED_RAW()
+            | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
+            | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
+            | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
+            | VBOXVMM_INSTR_INVD_ENABLED_RAW()
+            | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
+            | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
+            | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
+            | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
+            | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
+            | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
+            | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
+            | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
+            | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
+            | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
+            | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
+            | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
+            | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
+            | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
+            | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
+            | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
+            | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
+            | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
+            | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
+            | VBOXVMM_INSTR_STR_ENABLED_RAW()
+            | VBOXVMM_INSTR_LTR_ENABLED_RAW()
+            | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
+            | VBOXVMM_INSTR_RSM_ENABLED_RAW()
+            | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
+            | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
+            | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
+            | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
+            | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
+           ) != 0
+        || (  VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
+            | VBOXVMM_EXIT_HALT_ENABLED_RAW()
+            | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
+            | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
+            | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
+            | VBOXVMM_EXIT_INVD_ENABLED_RAW()
+            | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
+            | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
+            | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
+            | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
+            | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
+            | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
+            | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
+            | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
+            | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
+            | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
+            | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
+            | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
+            | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
+            | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
+            | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
+            | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
+            | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
+            | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
+            | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
+            | VBOXVMM_EXIT_STR_ENABLED_RAW()
+            | VBOXVMM_EXIT_LTR_ENABLED_RAW()
+            | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
+            | VBOXVMM_EXIT_RSM_ENABLED_RAW()
+            | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
+            | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
+            | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
+            | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
+            | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
+           ) != 0;
+}
+
+
+/**
+ * Runs the guest using hardware-assisted VMX.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPUCC pVCpu)
+{
+    AssertPtr(pVCpu);
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    Assert(VMMRZCallRing3IsEnabled(pVCpu));
+    Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
+    HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
+
+    VBOXSTRICTRC rcStrict;
+    uint32_t     cLoops = 0;
+    for (;;)
+    {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
+#else
+        NOREF(pCtx);
+        bool const fInNestedGuestMode = false;
+#endif
+        if (!fInNestedGuestMode)
+        {
+            if (   !pVCpu->hm.s.fUseDebugLoop
+                && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
+                && !DBGFIsStepping(pVCpu)
+                && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
+                rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
+            else
+                rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
+        }
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        else
+            rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
+
+        if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
+        {
+            Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
+            continue;
+        }
+        if (rcStrict == VINF_VMX_VMEXIT)
+        {
+            Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
+            continue;
+        }
+#endif
+        break;
+    }
+
+    int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
+    switch (rcLoop)
+    {
+        case VERR_EM_INTERPRETER:   rcStrict = VINF_EM_RAW_EMULATE_INSTR;   break;
+        case VINF_EM_RESET:         rcStrict = VINF_EM_TRIPLE_FAULT;        break;
+    }
+
+    int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
+    if (RT_FAILURE(rc2))
+    {
+        pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
+        rcStrict = rc2;
+    }
+    Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
+    Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
+    return rcStrict;
+}
+
+
+#ifndef HMVMX_USE_FUNCTION_TABLE
+/**
+ * Handles a guest VM-exit from hardware-assisted VMX execution.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+#ifdef DEBUG_ramshankar
+# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
+       do { \
+            if (a_fSave != 0) \
+                hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
+            VBOXSTRICTRC rcStrict = a_CallExpr; \
+            if (a_fSave != 0) \
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
+            return rcStrict; \
+        } while (0)
+#else
+# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
+#endif
+    uint32_t const uExitReason = pVmxTransient->uExitReason;
+    switch (uExitReason)
+    {
+        case VMX_EXIT_EPT_MISCONFIG:           VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
+        case VMX_EXIT_EPT_VIOLATION:           VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
+        case VMX_EXIT_IO_INSTR:                VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
+        case VMX_EXIT_CPUID:                   VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
+        case VMX_EXIT_RDTSC:                   VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
+        case VMX_EXIT_RDTSCP:                  VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
+        case VMX_EXIT_APIC_ACCESS:             VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
+        case VMX_EXIT_XCPT_OR_NMI:             VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
+        case VMX_EXIT_MOV_CRX:                 VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
+        case VMX_EXIT_EXT_INT:                 VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
+        case VMX_EXIT_INT_WINDOW:              VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
+        case VMX_EXIT_TPR_BELOW_THRESHOLD:     VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
+        case VMX_EXIT_MWAIT:                   VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
+        case VMX_EXIT_MONITOR:                 VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
+        case VMX_EXIT_TASK_SWITCH:             VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
+        case VMX_EXIT_PREEMPT_TIMER:           VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
+        case VMX_EXIT_RDMSR:                   VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
+        case VMX_EXIT_WRMSR:                   VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMCALL:                  VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
+        case VMX_EXIT_MOV_DRX:                 VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
+        case VMX_EXIT_HLT:                     VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
+        case VMX_EXIT_INVD:                    VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
+        case VMX_EXIT_INVLPG:                  VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
+        case VMX_EXIT_MTF:                     VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
+        case VMX_EXIT_PAUSE:                   VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
+        case VMX_EXIT_WBINVD:                  VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
+        case VMX_EXIT_XSETBV:                  VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
+        case VMX_EXIT_INVPCID:                 VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
+        case VMX_EXIT_GETSEC:                  VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
+        case VMX_EXIT_RDPMC:                   VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        case VMX_EXIT_VMCLEAR:                 VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMLAUNCH:                VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMPTRLD:                 VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMPTRST:                 VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMREAD:                  VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMRESUME:                VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMWRITE:                 VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMXOFF:                  VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMXON:                   VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
+        case VMX_EXIT_INVVPID:                 VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
+        case VMX_EXIT_INVEPT:                  VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
+#else
+        case VMX_EXIT_VMCLEAR:
+        case VMX_EXIT_VMLAUNCH:
+        case VMX_EXIT_VMPTRLD:
+        case VMX_EXIT_VMPTRST:
+        case VMX_EXIT_VMREAD:
+        case VMX_EXIT_VMRESUME:
+        case VMX_EXIT_VMWRITE:
+        case VMX_EXIT_VMXOFF:
+        case VMX_EXIT_VMXON:
+        case VMX_EXIT_INVVPID:
+        case VMX_EXIT_INVEPT:
+            return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
+#endif
+
+        case VMX_EXIT_TRIPLE_FAULT:            return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
+        case VMX_EXIT_NMI_WINDOW:              return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
+        case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_INIT_SIGNAL:
+        case VMX_EXIT_SIPI:
+        case VMX_EXIT_IO_SMI:
+        case VMX_EXIT_SMI:
+        case VMX_EXIT_ERR_MSR_LOAD:
+        case VMX_EXIT_ERR_MACHINE_CHECK:
+        case VMX_EXIT_PML_FULL:
+        case VMX_EXIT_VIRTUALIZED_EOI:
+        case VMX_EXIT_GDTR_IDTR_ACCESS:
+        case VMX_EXIT_LDTR_TR_ACCESS:
+        case VMX_EXIT_APIC_WRITE:
+        case VMX_EXIT_RDRAND:
+        case VMX_EXIT_RSM:
+        case VMX_EXIT_VMFUNC:
+        case VMX_EXIT_ENCLS:
+        case VMX_EXIT_RDSEED:
+        case VMX_EXIT_XSAVES:
+        case VMX_EXIT_XRSTORS:
+        case VMX_EXIT_UMWAIT:
+        case VMX_EXIT_TPAUSE:
+        default:
+            return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
+    }
+#undef VMEXIT_CALL_RET
+}
+#endif /* !HMVMX_USE_FUNCTION_TABLE */
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Handles a nested-guest VM-exit from hardware-assisted VMX execution.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    uint32_t const uExitReason = pVmxTransient->uExitReason;
+    switch (uExitReason)
+    {
+        case VMX_EXIT_EPT_MISCONFIG:            return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
+        case VMX_EXIT_EPT_VIOLATION:            return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
+        case VMX_EXIT_XCPT_OR_NMI:              return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_IO_INSTR:                 return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_HLT:                      return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
+
+        /*
+         * We shouldn't direct host physical interrupts to the nested-guest.
+         */
+        case VMX_EXIT_EXT_INT:
+            return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
+
+        /*
+         * Instructions that cause VM-exits unconditionally or the condition is
+         * always is taken solely from the nested hypervisor (meaning if the VM-exit
+         * happens, it's guaranteed to be a nested-guest VM-exit).
+         *
+         *   - Provides VM-exit instruction length ONLY.
+         */
+        case VMX_EXIT_CPUID:              /* Unconditional. */
+        case VMX_EXIT_VMCALL:
+        case VMX_EXIT_GETSEC:
+        case VMX_EXIT_INVD:
+        case VMX_EXIT_XSETBV:
+        case VMX_EXIT_VMLAUNCH:
+        case VMX_EXIT_VMRESUME:
+        case VMX_EXIT_VMXOFF:
+        case VMX_EXIT_ENCLS:              /* Condition specified solely by nested hypervisor. */
+        case VMX_EXIT_VMFUNC:
+            return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
+
+        /*
+         * Instructions that cause VM-exits unconditionally or the condition is
+         * always is taken solely from the nested hypervisor (meaning if the VM-exit
+         * happens, it's guaranteed to be a nested-guest VM-exit).
+         *
+         *   - Provides VM-exit instruction length.
+         *   - Provides VM-exit information.
+         *   - Optionally provides Exit qualification.
+         *
+         * Since Exit qualification is 0 for all VM-exits where it is not
+         * applicable, reading and passing it to the guest should produce
+         * defined behavior.
+         *
+         * See Intel spec. 27.2.1 "Basic VM-Exit Information".
+         */
+        case VMX_EXIT_INVEPT:             /* Unconditional. */
+        case VMX_EXIT_INVVPID:
+        case VMX_EXIT_VMCLEAR:
+        case VMX_EXIT_VMPTRLD:
+        case VMX_EXIT_VMPTRST:
+        case VMX_EXIT_VMXON:
+        case VMX_EXIT_GDTR_IDTR_ACCESS:   /* Condition specified solely by nested hypervisor. */
+        case VMX_EXIT_LDTR_TR_ACCESS:
+        case VMX_EXIT_RDRAND:
+        case VMX_EXIT_RDSEED:
+        case VMX_EXIT_XSAVES:
+        case VMX_EXIT_XRSTORS:
+        case VMX_EXIT_UMWAIT:
+        case VMX_EXIT_TPAUSE:
+            return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_RDTSC:                    return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_RDTSCP:                   return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_RDMSR:                    return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_WRMSR:                    return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_INVLPG:                   return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_INVPCID:                  return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_TASK_SWITCH:              return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_WBINVD:                   return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_MTF:                      return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_APIC_ACCESS:              return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_APIC_WRITE:               return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_VIRTUALIZED_EOI:          return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_MOV_CRX:                  return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_INT_WINDOW:               return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_NMI_WINDOW:               return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_TPR_BELOW_THRESHOLD:      return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_MWAIT:                    return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_MONITOR:                  return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_PAUSE:                    return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_PREEMPT_TIMER:
+        {
+            /** @todo NSTVMX: Preempt timer. */
+            return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
+        }
+
+        case VMX_EXIT_MOV_DRX:                  return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_RDPMC:                    return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_VMREAD:
+        case VMX_EXIT_VMWRITE:                  return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_TRIPLE_FAULT:             return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_ERR_INVALID_GUEST_STATE:  return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_INIT_SIGNAL:
+        case VMX_EXIT_SIPI:
+        case VMX_EXIT_IO_SMI:
+        case VMX_EXIT_SMI:
+        case VMX_EXIT_ERR_MSR_LOAD:
+        case VMX_EXIT_ERR_MACHINE_CHECK:
+        case VMX_EXIT_PML_FULL:
+        case VMX_EXIT_RSM:
+        default:
+            return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
+    }
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/** @name VM-exit helpers.
+ * @{
+ */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+
+/** Macro for VM-exits called unexpectedly. */
+#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
+    do { \
+        (a_pVCpu)->hm.s.u32HMError = (a_HmError); \
+        return VERR_VMX_UNEXPECTED_EXIT; \
+    } while (0)
+
+#ifdef VBOX_STRICT
+/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
+# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
+    RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
+
+# define HMVMX_ASSERT_PREEMPT_CPUID() \
+    do { \
+         RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
+         AssertMsg(idAssertCpu == idAssertCpuNow,  ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
+    } while (0)
+
+# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
+    do { \
+        AssertPtr((a_pVCpu)); \
+        AssertPtr((a_pVmxTransient)); \
+        Assert((a_pVmxTransient)->fVMEntryFailed == false); \
+        Assert((a_pVmxTransient)->pVmcsInfo); \
+        Assert(ASMIntAreEnabled()); \
+        HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
+        HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
+        Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
+        HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
+        if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
+            HMVMX_ASSERT_PREEMPT_CPUID(); \
+        HMVMX_STOP_EXIT_DISPATCH_PROF(); \
+    } while (0)
+
+# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
+    do { \
+        HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
+        Assert((a_pVmxTransient)->fIsNestedGuest); \
+    } while (0)
+
+# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
+    do { \
+        Log4Func(("\n")); \
+    } while (0)
+#else
+# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
+    do { \
+        HMVMX_STOP_EXIT_DISPATCH_PROF(); \
+        NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
+    } while (0)
+
+# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
+    do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
+
+# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient)      do { } while (0)
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/** Macro that does the necessary privilege checks and intercepted VM-exits for
+ *  guests that attempted to execute a VMX instruction. */
+# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
+        if (rcStrictTmp == VINF_SUCCESS) \
+        { /* likely */ } \
+        else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
+        { \
+            Assert((a_pVCpu)->hm.s.Event.fPending); \
+            Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
+            return VINF_SUCCESS; \
+        } \
+        else \
+        { \
+            int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
+            AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
+        } \
+    } while (0)
+
+/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
+# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
+                                                           (a_pGCPtrEffAddr)); \
+        if (rcStrictTmp == VINF_SUCCESS) \
+        { /* likely */ } \
+        else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
+        { \
+            uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
+            Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
+            NOREF(uXcptTmp); \
+            return VINF_SUCCESS; \
+        } \
+        else \
+        { \
+            Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
+            return rcStrictTmp; \
+        } \
+    } while (0)
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/**
+ * Advances the guest RIP by the specified number of bytes.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     Number of bytes to advance the RIP by.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
+{
+    /* Advance the RIP. */
+    pVCpu->cpum.GstCtx.rip += cbInstr;
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
+
+    /* Update interrupt inhibition. */
+    if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+        && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+}
+
+
+/**
+ * Advances the guest RIP after reading it from the VMCS.
+ *
+ * @returns VBox status code, no informational status codes.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int hmR0VmxAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
+    AssertRCReturn(rc, rc);
+
+    hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Handle a condition that occurred while delivering an event through the guest or
+ * nested-guest IDT.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_SUCCESS if we should continue handling the VM-exit.
+ * @retval  VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
+ *          to continue execution of the guest which will delivery the \#DF.
+ * @retval  VINF_EM_RESET if we detected a triple-fault condition.
+ * @retval  VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ *          Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
+ *          is due to an EPT violation, PML full or SPP-related event.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    Assert(!pVCpu->hm.s.Event.fPending);
+    HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
+    if (   pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
+        || pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
+        || pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
+        HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
+
+    VBOXSTRICTRC   rcStrict       = VINF_SUCCESS;
+    PCVMXVMCSINFO  pVmcsInfo      = pVmxTransient->pVmcsInfo;
+    uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
+    uint32_t const uExitIntInfo   = pVmxTransient->uExitIntInfo;
+    if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
+    {
+        uint32_t const uIdtVector     = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
+        uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
+
+        /*
+         * If the event was a software interrupt (generated with INT n) or a software exception
+         * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
+         * can handle the VM-exit and continue guest execution which will re-execute the
+         * instruction rather than re-injecting the exception, as that can cause premature
+         * trips to ring-3 before injection and involve TRPM which currently has no way of
+         * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
+         * the problem).
+         */
+        IEMXCPTRAISE     enmRaise;
+        IEMXCPTRAISEINFO fRaiseInfo;
+        if (   uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
+            || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
+            || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
+        {
+            enmRaise   = IEMXCPTRAISE_REEXEC_INSTR;
+            fRaiseInfo = IEMXCPTRAISEINFO_NONE;
+        }
+        else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
+        {
+            uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
+            uint8_t const  uExitVector     = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
+            Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
+
+            uint32_t const fIdtVectorFlags  = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
+            uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
+
+            enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
+
+            /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
+            if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
+            {
+                pVmxTransient->fVectoringPF = true;
+                enmRaise = IEMXCPTRAISE_PREV_EVENT;
+            }
+        }
+        else
+        {
+            /*
+             * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
+             * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
+             * It is sufficient to reflect the original event to the guest after handling the VM-exit.
+             */
+            Assert(   uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
+                   || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
+                   || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
+            enmRaise   = IEMXCPTRAISE_PREV_EVENT;
+            fRaiseInfo = IEMXCPTRAISEINFO_NONE;
+        }
+
+        /*
+         * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
+         * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
+         * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
+         * subsequent VM-entry would fail, see @bugref{7445}.
+         *
+         * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
+         */
+        if (   uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
+            && enmRaise == IEMXCPTRAISE_PREV_EVENT
+            && (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+            && CPUMIsGuestNmiBlocking(pVCpu))
+        {
+            CPUMSetGuestNmiBlocking(pVCpu, false);
+        }
+
+        switch (enmRaise)
+        {
+            case IEMXCPTRAISE_CURRENT_XCPT:
+            {
+                Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
+                Assert(rcStrict == VINF_SUCCESS);
+                break;
+            }
+
+            case IEMXCPTRAISE_PREV_EVENT:
+            {
+                uint32_t u32ErrCode;
+                if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
+                    u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
+                else
+                    u32ErrCode = 0;
+
+                /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
+                hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
+                                       u32ErrCode, pVCpu->cpum.GstCtx.cr2);
+
+                Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
+                          pVCpu->hm.s.Event.u32ErrCode));
+                Assert(rcStrict == VINF_SUCCESS);
+                break;
+            }
+
+            case IEMXCPTRAISE_REEXEC_INSTR:
+                Assert(rcStrict == VINF_SUCCESS);
+                break;
+
+            case IEMXCPTRAISE_DOUBLE_FAULT:
+            {
+                /*
+                 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
+                 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
+                 */
+                if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
+                {
+                    pVmxTransient->fVectoringDoublePF = true;
+                    Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
+                          pVCpu->cpum.GstCtx.cr2));
+                    rcStrict = VINF_SUCCESS;
+                }
+                else
+                {
+                    STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
+                    hmR0VmxSetPendingXcptDF(pVCpu);
+                    Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
+                              uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
+                    rcStrict = VINF_HM_DOUBLE_FAULT;
+                }
+                break;
+            }
+
+            case IEMXCPTRAISE_TRIPLE_FAULT:
+            {
+                Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
+                          VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
+                rcStrict = VINF_EM_RESET;
+                break;
+            }
+
+            case IEMXCPTRAISE_CPU_HANG:
+            {
+                Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
+                rcStrict = VERR_EM_GUEST_CPU_HANG;
+                break;
+            }
+
+            default:
+            {
+                AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
+                rcStrict = VERR_VMX_IPE_2;
+                break;
+            }
+        }
+    }
+    else if (   (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+             && !CPUMIsGuestNmiBlocking(pVCpu))
+    {
+        if (    VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
+             && VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
+             && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
+        {
+            /*
+             * Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
+             * the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
+             * that virtual NMIs remain blocked until the IRET execution is completed.
+             *
+             * See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
+             */
+            CPUMSetGuestNmiBlocking(pVCpu, true);
+            Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
+        }
+        else if (   pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
+                 || pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
+                 || pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
+        {
+            /*
+             * Execution of IRET caused an EPT violation, page-modification log-full event or
+             * SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
+             * guest or nested-guest NMI handler). We need to set the block-by-NMI field so
+             * that virtual NMIs remain blocked until the IRET execution is completed.
+             *
+             * See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
+             */
+            if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
+            {
+                CPUMSetGuestNmiBlocking(pVCpu, true);
+                Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
+            }
+        }
+    }
+
+    Assert(   rcStrict == VINF_SUCCESS  || rcStrict == VINF_HM_DOUBLE_FAULT
+           || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
+    return rcStrict;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
+ * guest attempting to execute a VMX instruction.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_SUCCESS if we should continue handling the VM-exit.
+ * @retval  VINF_HM_PENDING_XCPT if an exception was raised.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ *
+ * @todo    NSTVMX: Document other error codes when VM-exit is implemented.
+ * @remarks No-long-jump zone!!!
+ */
+static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
+{
+    HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
+                              | CPUMCTX_EXTRN_CS  | CPUMCTX_EXTRN_EFER);
+
+    /*
+     * The physical CPU would have already checked the CPU mode/code segment.
+     * We shall just assert here for paranoia.
+     * See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
+     */
+    Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
+    Assert(   !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
+           ||  CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
+
+    if (uExitReason == VMX_EXIT_VMXON)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+
+        /*
+         * We check CR4.VMXE because it is required to be always set while in VMX operation
+         * by physical CPUs and our CR4 read-shadow is only consulted when executing specific
+         * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
+         * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
+         */
+        if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
+        {
+            Log4Func(("CR4.VMXE is not set -> #UD\n"));
+            hmR0VmxSetPendingXcptUD(pVCpu);
+            return VINF_HM_PENDING_XCPT;
+        }
+    }
+    else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
+    {
+        /*
+         * The guest has not entered VMX operation but attempted to execute a VMX instruction
+         * (other than VMXON), we need to raise a #UD.
+         */
+        Log4Func(("Not in VMX root mode -> #UD\n"));
+        hmR0VmxSetPendingXcptUD(pVCpu);
+        return VINF_HM_PENDING_XCPT;
+    }
+
+    /* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Decodes the memory operand of an instruction that caused a VM-exit.
+ *
+ * The Exit qualification field provides the displacement field for memory
+ * operand instructions, if any.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_SUCCESS if the operand was successfully decoded.
+ * @retval  VINF_HM_PENDING_XCPT if an exception was raised while decoding the
+ *          operand.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitInstrInfo  The VM-exit instruction information field.
+ * @param   enmMemAccess    The memory operand's access type (read or write).
+ * @param   GCPtrDisp       The instruction displacement field, if any. For
+ *                          RIP-relative addressing pass RIP + displacement here.
+ * @param   pGCPtrMem       Where to store the effective destination memory address.
+ *
+ * @remarks Warning! This function ASSUMES the instruction cannot be used in real or
+ *          virtual-8086 mode hence skips those checks while verifying if the
+ *          segment is valid.
+ */
+static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
+                                            PRTGCPTR pGCPtrMem)
+{
+    Assert(pGCPtrMem);
+    Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
+    HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER
+                              | CPUMCTX_EXTRN_CR0);
+
+    static uint64_t const s_auAddrSizeMasks[]   = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
+    static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
+    AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
+
+    VMXEXITINSTRINFO ExitInstrInfo;
+    ExitInstrInfo.u = uExitInstrInfo;
+    uint8_t const   uAddrSize     =  ExitInstrInfo.All.u3AddrSize;
+    uint8_t const   iSegReg       =  ExitInstrInfo.All.iSegReg;
+    bool const      fIdxRegValid  = !ExitInstrInfo.All.fIdxRegInvalid;
+    uint8_t const   iIdxReg       =  ExitInstrInfo.All.iIdxReg;
+    uint8_t const   uScale        =  ExitInstrInfo.All.u2Scaling;
+    bool const      fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
+    uint8_t const   iBaseReg      =  ExitInstrInfo.All.iBaseReg;
+    bool const      fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
+    bool const      fIsLongMode   =  CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
+
+    /*
+     * Validate instruction information.
+     * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
+     */
+    AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
+                          ("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
+    AssertLogRelMsgReturn(iSegReg  < X86_SREG_COUNT,
+                          ("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
+    AssertLogRelMsgReturn(fIsMemOperand,
+                          ("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
+
+    /*
+     * Compute the complete effective address.
+     *
+     * See AMD instruction spec. 1.4.2 "SIB Byte Format"
+     * See AMD spec. 4.5.2 "Segment Registers".
+     */
+    RTGCPTR GCPtrMem = GCPtrDisp;
+    if (fBaseRegValid)
+        GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
+    if (fIdxRegValid)
+        GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
+
+    RTGCPTR const GCPtrOff = GCPtrMem;
+    if (   !fIsLongMode
+        || iSegReg >= X86_SREG_FS)
+        GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
+    GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
+
+    /*
+     * Validate effective address.
+     * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
+     */
+    uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
+    Assert(cbAccess > 0);
+    if (fIsLongMode)
+    {
+        if (X86_IS_CANONICAL(GCPtrMem))
+        {
+            *pGCPtrMem = GCPtrMem;
+            return VINF_SUCCESS;
+        }
+
+        /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
+         *        "Data Limit Checks in 64-bit Mode". */
+        Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
+        hmR0VmxSetPendingXcptGP(pVCpu, 0);
+        return VINF_HM_PENDING_XCPT;
+    }
+
+    /*
+     * This is a watered down version of iemMemApplySegment().
+     * Parts that are not applicable for VMX instructions like real-or-v8086 mode
+     * and segment CPL/DPL checks are skipped.
+     */
+    RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
+    RTGCPTR32 const GCPtrLast32  = GCPtrFirst32 + cbAccess - 1;
+    PCCPUMSELREG    pSel         = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
+
+    /* Check if the segment is present and usable. */
+    if (    pSel->Attr.n.u1Present
+        && !pSel->Attr.n.u1Unusable)
+    {
+        Assert(pSel->Attr.n.u1DescType);
+        if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
+        {
+            /* Check permissions for the data segment. */
+            if (   enmMemAccess == VMXMEMACCESS_WRITE
+                && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
+            {
+                Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
+                hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
+                return VINF_HM_PENDING_XCPT;
+            }
+
+            /* Check limits if it's a normal data segment. */
+            if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
+            {
+                if (   GCPtrFirst32 > pSel->u32Limit
+                    || GCPtrLast32  > pSel->u32Limit)
+                {
+                    Log4Func(("Data segment limit exceeded. "
+                              "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
+                              GCPtrLast32, pSel->u32Limit));
+                    if (iSegReg == X86_SREG_SS)
+                        hmR0VmxSetPendingXcptSS(pVCpu, 0);
+                    else
+                        hmR0VmxSetPendingXcptGP(pVCpu, 0);
+                    return VINF_HM_PENDING_XCPT;
+                }
+            }
+            else
+            {
+               /* Check limits if it's an expand-down data segment.
+                  Note! The upper boundary is defined by the B bit, not the G bit! */
+               if (   GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
+                   || GCPtrLast32  > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
+               {
+                   Log4Func(("Expand-down data segment limit exceeded. "
+                             "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
+                             GCPtrLast32, pSel->u32Limit));
+                   if (iSegReg == X86_SREG_SS)
+                       hmR0VmxSetPendingXcptSS(pVCpu, 0);
+                   else
+                       hmR0VmxSetPendingXcptGP(pVCpu, 0);
+                   return VINF_HM_PENDING_XCPT;
+               }
+            }
+        }
+        else
+        {
+            /* Check permissions for the code segment. */
+            if (   enmMemAccess == VMXMEMACCESS_WRITE
+                || (   enmMemAccess == VMXMEMACCESS_READ
+                    && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
+            {
+                Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
+                Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
+                hmR0VmxSetPendingXcptGP(pVCpu, 0);
+                return VINF_HM_PENDING_XCPT;
+            }
+
+            /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
+            if (   GCPtrFirst32 > pSel->u32Limit
+                || GCPtrLast32  > pSel->u32Limit)
+            {
+                Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
+                          GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
+                if (iSegReg == X86_SREG_SS)
+                    hmR0VmxSetPendingXcptSS(pVCpu, 0);
+                else
+                    hmR0VmxSetPendingXcptGP(pVCpu, 0);
+                return VINF_HM_PENDING_XCPT;
+            }
+        }
+    }
+    else
+    {
+        Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
+        hmR0VmxSetPendingXcptGP(pVCpu, 0);
+        return VINF_HM_PENDING_XCPT;
+    }
+
+    *pGCPtrMem = GCPtrMem;
+    return VINF_SUCCESS;
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/**
+ * VM-exit helper for LMSW.
+ */
+static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
+{
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
+    AssertMsg(   rcStrict == VINF_SUCCESS
+              || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
+    Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit helper for CLTS.
+ */
+static VBOXSTRICTRC hmR0VmxExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
+{
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
+    AssertMsg(   rcStrict == VINF_SUCCESS
+              || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
+    Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit helper for MOV from CRx (CRx read).
+ */
+static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
+{
+    Assert(iCrReg < 16);
+    Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
+
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
+    AssertMsg(   rcStrict == VINF_SUCCESS
+              || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    if (iGReg == X86_GREG_xSP)
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
+    else
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+#ifdef VBOX_WITH_STATISTICS
+    switch (iCrReg)
+    {
+        case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
+        case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
+        case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
+        case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
+        case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
+    }
+#endif
+    Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit helper for MOV to CRx (CRx write).
+ */
+static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
+{
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
+    AssertMsg(   rcStrict == VINF_SUCCESS
+              || rcStrict == VINF_IEM_RAISED_XCPT
+              || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    switch (iCrReg)
+    {
+        case 0:
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0
+                                                     | HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
+            Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
+            break;
+
+        case 2:
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
+            /* Nothing to do here, CR2 it's not part of the VMCS. */
+            break;
+
+        case 3:
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
+            Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
+            break;
+
+        case 4:
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
+            Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
+                      pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
+            break;
+
+        case 8:
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
+                             HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
+            break;
+
+        default:
+            AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
+            break;
+    }
+
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit exception handler for \#PF (Page-fault exception).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+
+    if (!pVM->hm.s.fNestedPaging)
+    { /* likely */ }
+    else
+    {
+#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
+        Assert(pVmxTransient->fIsNestedGuest || pVCpu->hm.s.fUsingDebugLoop);
+#endif
+        pVCpu->hm.s.Event.fPending = false;                  /* In case it's a contributory or vectoring #PF. */
+        if (!pVmxTransient->fVectoringDoublePF)
+        {
+            hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
+                                   pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
+        }
+        else
+        {
+            /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
+            Assert(!pVmxTransient->fIsNestedGuest);
+            hmR0VmxSetPendingXcptDF(pVCpu);
+            Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
+        }
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
+        return VINF_SUCCESS;
+    }
+
+    Assert(!pVmxTransient->fIsNestedGuest);
+
+    /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
+       of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
+    if (pVmxTransient->fVectoringPF)
+    {
+        Assert(pVCpu->hm.s.Event.fPending);
+        return VINF_EM_RAW_INJECT_TRPM_EVENT;
+    }
+
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+    AssertRCReturn(rc, rc);
+
+    Log4Func(("#PF: cs:rip=%#04x:%#RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
+              pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pCtx->cr3));
+
+    TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
+    rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
+
+    Log4Func(("#PF: rc=%Rrc\n", rc));
+    if (rc == VINF_SUCCESS)
+    {
+        /*
+         * This is typically a shadow page table sync or a MMIO instruction. But we may have
+         * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
+         */
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+        TRPMResetTrap(pVCpu);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
+        return rc;
+    }
+
+    if (rc == VINF_EM_RAW_GUEST_TRAP)
+    {
+        if (!pVmxTransient->fVectoringDoublePF)
+        {
+            /* It's a guest page fault and needs to be reflected to the guest. */
+            uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
+            TRPMResetTrap(pVCpu);
+            pVCpu->hm.s.Event.fPending = false;                 /* In case it's a contributory #PF. */
+            hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
+                                   uGstErrorCode, pVmxTransient->uExitQual);
+        }
+        else
+        {
+            /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
+            TRPMResetTrap(pVCpu);
+            pVCpu->hm.s.Event.fPending = false;     /* Clear pending #PF to replace it with #DF. */
+            hmR0VmxSetPendingXcptDF(pVCpu);
+            Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
+        }
+
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
+        return VINF_SUCCESS;
+    }
+
+    TRPMResetTrap(pVCpu);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
+    return rc;
+}
+
+
+/**
+ * VM-exit exception handler for \#MF (Math Fault: floating point exception).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
+
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
+    AssertRCReturn(rc, rc);
+
+    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
+    {
+        /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
+        rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
+
+        /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
+         *        provides VM-exit instruction length. If this causes problem later,
+         *        disassemble the instruction like it's done on AMD-V. */
+        int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
+        AssertRCReturn(rc2, rc2);
+        return rc;
+    }
+
+    hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
+                           pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit exception handler for \#BP (Breakpoint exception).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
+
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+    AssertRCReturn(rc, rc);
+
+    if (!pVmxTransient->fIsNestedGuest)
+        rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
+    else
+        rc = VINF_EM_RAW_GUEST_TRAP;
+
+    if (rc == VINF_EM_RAW_GUEST_TRAP)
+    {
+        hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
+                               pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+        rc = VINF_SUCCESS;
+    }
+
+    Assert(rc == VINF_SUCCESS || rc == VINF_EM_DBG_BREAKPOINT);
+    return rc;
+}
+
+
+/**
+ * VM-exit exception handler for \#AC (Alignment-check exception).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
+
+    /* Re-inject it. We'll detect any nesting before getting here. */
+    hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
+                           pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit exception handler for \#DB (Debug exception).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
+
+    /*
+     * Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
+     */
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+
+    /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
+    uint64_t const uDR6 = X86_DR6_INIT_VAL
+                        | (pVmxTransient->uExitQual & (  X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3
+                                                       | X86_DR6_BD | X86_DR6_BS));
+
+    int rc;
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    if (!pVmxTransient->fIsNestedGuest)
+    {
+        rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
+
+        /*
+         * Prevents stepping twice over the same instruction when the guest is stepping using
+         * EFLAGS.TF and the hypervisor debugger is stepping using MTF.
+         * Testcase: DOSQEMM, break (using "ba x 1") at cs:rip 0x70:0x774 and step (using "t").
+         */
+        if (   rc == VINF_EM_DBG_STEPPED
+            && (pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG))
+        {
+            Assert(pVCpu->hm.s.fSingleInstruction);
+            rc = VINF_EM_RAW_GUEST_TRAP;
+        }
+    }
+    else
+        rc = VINF_EM_RAW_GUEST_TRAP;
+    Log6Func(("rc=%Rrc\n", rc));
+    if (rc == VINF_EM_RAW_GUEST_TRAP)
+    {
+        /*
+         * The exception was for the guest.  Update DR6, DR7.GD and
+         * IA32_DEBUGCTL.LBR before forwarding it.
+         * See Intel spec. 27.1 "Architectural State before a VM-Exit".
+         */
+        VMMRZCallRing3Disable(pVCpu);
+        HM_DISABLE_PREEMPT(pVCpu);
+
+        pCtx->dr[6] &= ~X86_DR6_B_MASK;
+        pCtx->dr[6] |= uDR6;
+        if (CPUMIsGuestDebugStateActive(pVCpu))
+            ASMSetDR6(pCtx->dr[6]);
+
+        HM_RESTORE_PREEMPT();
+        VMMRZCallRing3Enable(pVCpu);
+
+        rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
+        AssertRCReturn(rc, rc);
+
+        /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
+        pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
+
+        /* Paranoia. */
+        pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
+        pCtx->dr[7] |= X86_DR7_RA1_MASK;
+
+        rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
+        AssertRC(rc);
+
+        /*
+         * Raise #DB in the guest.
+         *
+         * It is important to reflect exactly what the VM-exit gave us (preserving the
+         * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
+         * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
+         * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
+         *
+         * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
+         * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
+         */
+        hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
+                               pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Not a guest trap, must be a hypervisor related debug event then.
+     * Update DR6 in case someone is interested in it.
+     */
+    AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
+    AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
+    CPUMSetHyperDR6(pVCpu, uDR6);
+
+    return rc;
+}
+
+
+/**
+ * Hacks its way around the lovely mesa driver's backdoor accesses.
+ *
+ * @sa hmR0SvmHandleMesaDrvGp.
+ */
+static int hmR0VmxHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
+{
+    LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
+    RT_NOREF(pCtx);
+
+    /* For now we'll just skip the instruction. */
+    return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
+ * backdoor logging w/o checking what it is running inside.
+ *
+ * This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
+ * backdoor port and magic numbers loaded in registers.
+ *
+ * @returns true if it is, false if it isn't.
+ * @sa      hmR0SvmIsMesaDrvGp.
+ */
+DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
+{
+    /* 0xed:  IN eAX,dx */
+    uint8_t abInstr[1];
+    if (pVmxTransient->cbExitInstr != sizeof(abInstr))
+        return false;
+
+    /* Check that it is #GP(0). */
+    if (pVmxTransient->uExitIntErrorCode != 0)
+        return false;
+
+    /* Check magic and port. */
+    Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RCX)));
+    /*Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));*/
+    if (pCtx->rax != UINT32_C(0x564d5868))
+        return false;
+    if (pCtx->dx != UINT32_C(0x5658))
+        return false;
+
+    /* Flat ring-3 CS. */
+    AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
+    Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
+    /*Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));*/
+    if (pCtx->cs.Attr.n.u2Dpl != 3)
+        return false;
+    if (pCtx->cs.u64Base != 0)
+        return false;
+
+    /* Check opcode. */
+    AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
+    Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
+    /*Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));*/
+    if (RT_FAILURE(rc))
+        return false;
+    if (abInstr[0] != 0xed)
+        return false;
+
+    return true;
+}
+
+
+/**
+ * VM-exit exception handler for \#GP (General-protection exception).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
+
+    PCPUMCTX     pCtx      = &pVCpu->cpum.GstCtx;
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    if (pVmcsInfo->RealMode.fRealOnV86Active)
+    { /* likely */ }
+    else
+    {
+#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
+        Assert(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv || pVmxTransient->fIsNestedGuest);
+#endif
+        /*
+         * If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
+         * executing a nested-guest, reflect #GP to the guest or nested-guest.
+         */
+        int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+        AssertRCReturn(rc, rc);
+        Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
+                  pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
+
+        if (    pVmxTransient->fIsNestedGuest
+            || !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
+            || !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
+            hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
+                                   pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+        else
+            rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
+        return rc;
+    }
+
+    Assert(CPUMIsGuestInRealModeEx(pCtx));
+    Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
+    Assert(!pVmxTransient->fIsNestedGuest);
+
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (!CPUMIsGuestInRealModeEx(pCtx))
+        {
+            /*
+             * The guest is no longer in real-mode, check if we can continue executing the
+             * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
+             */
+            pVmcsInfo->RealMode.fRealOnV86Active = false;
+            if (HMCanExecuteVmxGuest(pVCpu->pVMR0, pVCpu, pCtx))
+            {
+                Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+            }
+            else
+            {
+                Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
+                rcStrict = VINF_EM_RESCHEDULE;
+            }
+        }
+        else
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    return VBOXSTRICTRC_VAL(rcStrict);
+}
+
+
+/**
+ * VM-exit exception handler wrapper for all other exceptions that are not handled
+ * by a specific handler.
+ *
+ * This simply re-injects the exception back into the VM without any special
+ * processing.
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC hmR0VmxExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
+    PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVmcsInfo->RealMode.fRealOnV86Active || pVmxTransient->fIsNestedGuest,
+              ("uVector=%#x u32XcptBitmap=%#X32\n",
+               VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
+    NOREF(pVmcsInfo);
+#endif
+
+    /*
+     * Re-inject the exception into the guest. This cannot be a double-fault condition which
+     * would have been handled while checking exits due to event delivery.
+     */
+    uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
+
+#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
+    AssertRCReturn(rc, rc);
+    Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
+#endif
+
+#ifdef VBOX_WITH_STATISTICS
+    switch (uVector)
+    {
+        case X86_XCPT_DE:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);     break;
+        case X86_XCPT_DB:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);     break;
+        case X86_XCPT_BP:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);     break;
+        case X86_XCPT_OF:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF);     break;
+        case X86_XCPT_BR:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR);     break;
+        case X86_XCPT_UD:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);     break;
+        case X86_XCPT_NM:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF);     break;
+        case X86_XCPT_DF:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF);     break;
+        case X86_XCPT_TS:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);     break;
+        case X86_XCPT_NP:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);     break;
+        case X86_XCPT_SS:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);     break;
+        case X86_XCPT_GP:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);     break;
+        case X86_XCPT_PF:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);     break;
+        case X86_XCPT_MF:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);     break;
+        case X86_XCPT_AC:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);     break;
+        case X86_XCPT_XF:   STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);     break;
+        default:
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
+            break;
+    }
+#endif
+
+    /* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
+    Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
+    NOREF(uVector);
+
+    /* Re-inject the original exception into the guest. */
+    hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
+                           pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit exception handler for all exceptions (except NMIs!).
+ *
+ * @remarks This may be called for both guests and nested-guests. Take care to not
+ *          make assumptions and avoid doing anything that is not relevant when
+ *          executing a nested-guest (e.g., Mesa driver hacks).
+ */
+static VBOXSTRICTRC hmR0VmxExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
+
+    /*
+     * If this VM-exit occurred while delivering an event through the guest IDT, take
+     * action based on the return code and additional hints (e.g. for page-faults)
+     * that will be updated in the VMX transient structure.
+     */
+    VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /*
+         * If an exception caused a VM-exit due to delivery of an event, the original
+         * event may have to be re-injected into the guest. We shall reinject it and
+         * continue guest execution. However, page-fault is a complicated case and
+         * needs additional processing done in hmR0VmxExitXcptPF().
+         */
+        Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
+        uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
+        if (   !pVCpu->hm.s.Event.fPending
+            || uVector == X86_XCPT_PF)
+        {
+            switch (uVector)
+            {
+                case X86_XCPT_PF: return hmR0VmxExitXcptPF(pVCpu, pVmxTransient);
+                case X86_XCPT_GP: return hmR0VmxExitXcptGP(pVCpu, pVmxTransient);
+                case X86_XCPT_MF: return hmR0VmxExitXcptMF(pVCpu, pVmxTransient);
+                case X86_XCPT_DB: return hmR0VmxExitXcptDB(pVCpu, pVmxTransient);
+                case X86_XCPT_BP: return hmR0VmxExitXcptBP(pVCpu, pVmxTransient);
+                case X86_XCPT_AC: return hmR0VmxExitXcptAC(pVCpu, pVmxTransient);
+                default:
+                    return hmR0VmxExitXcptOthers(pVCpu, pVmxTransient);
+            }
+        }
+        /* else: inject pending event before resuming guest execution. */
+    }
+    else if (rcStrict == VINF_HM_DOUBLE_FAULT)
+    {
+        Assert(pVCpu->hm.s.Event.fPending);
+        rcStrict = VINF_SUCCESS;
+    }
+
+    return rcStrict;
+}
+/** @} */
+
+
+/** @name VM-exit handlers.
+ * @{
+ */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+
+/**
+ * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
+ */
+HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
+    /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
+    if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
+        return VINF_SUCCESS;
+    return VINF_EM_RAW_INTERRUPT;
+}
+
+
+/**
+ * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
+
+    hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
+
+    uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
+    uint8_t const  uVector      = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
+    Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
+
+    PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    Assert(   !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
+           && uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
+    NOREF(pVmcsInfo);
+
+    VBOXSTRICTRC rcStrict;
+    switch (uExitIntType)
+    {
+        /*
+         * Host physical NMIs:
+         *     This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
+         *     injected it ourselves and anything we inject is not going to cause a VM-exit directly
+         *     for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
+         *
+         *     See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
+         *     See Intel spec. 27.5.5 "Updating Non-Register State".
+         */
+        case VMX_EXIT_INT_INFO_TYPE_NMI:
+        {
+            rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
+            break;
+        }
+
+        /*
+         * Privileged software exceptions (#DB from ICEBP),
+         * Software exceptions (#BP and #OF),
+         * Hardware exceptions:
+         *     Process the required exceptions and resume guest execution if possible.
+         */
+        case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
+            Assert(uVector == X86_XCPT_DB);
+            RT_FALL_THRU();
+        case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
+            Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
+            RT_FALL_THRU();
+        case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
+        {
+            NOREF(uVector);
+            hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
+            hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+            hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
+            hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
+
+            rcStrict = hmR0VmxExitXcpt(pVCpu, pVmxTransient);
+            break;
+        }
+
+        default:
+        {
+            pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
+            rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
+            AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
+            break;
+        }
+    }
+
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
+ */
+HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
+
+    /* Evaluate and deliver pending events and resume guest execution. */
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
+ */
+HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
+    {
+        AssertMsgFailed(("Unexpected NMI-window exit.\n"));
+        HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
+    }
+
+    Assert(!CPUMIsGuestNmiBlocking(pVCpu));
+
+    /*
+     * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
+     * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
+     */
+    uint32_t fIntrState;
+    int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
+    AssertRC(rc);
+    Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
+    if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
+    {
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+
+        fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
+        rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
+        AssertRC(rc);
+    }
+
+    /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
+    hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
+
+    /* Evaluate and deliver pending events and resume guest execution. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
+ */
+HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
+ */
+HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /*
+     * Get the state we need and update the exit history entry.
+     */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict;
+    PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
+                                                            EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
+                                                            pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
+    if (!pExitRec)
+    {
+        /*
+         * Regular CPUID instruction execution.
+         */
+        rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
+        if (rcStrict == VINF_SUCCESS)
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+        else if (rcStrict == VINF_IEM_RAISED_XCPT)
+        {
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+            rcStrict = VINF_SUCCESS;
+        }
+    }
+    else
+    {
+        /*
+         * Frequent exit or something needing probing.  Get state and call EMHistoryExec.
+         */
+        int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+        AssertRCReturn(rc2, rc2);
+
+        Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
+
+        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+
+        Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
+    AssertRCReturn(rc, rc);
+
+    if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
+        return VINF_EM_RAW_EMULATE_INSTR;
+
+    AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
+    HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
+}
+
+
+/**
+ * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        /* If we get a spurious VM-exit when TSC offsetting is enabled,
+           we must reset offsetting on VM-entry. See @bugref{6634}. */
+        if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
+            pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        /* If we get a spurious VM-exit when TSC offsetting is enabled,
+           we must reset offsetting on VM-reentry. See @bugref{6634}. */
+        if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
+            pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4    | CPUMCTX_EXTRN_CR0
+                                                     | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
+    AssertRCReturn(rc, rc);
+
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
+    if (RT_LIKELY(rc == VINF_SUCCESS))
+    {
+        rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
+        Assert(pVmxTransient->cbExitInstr == 2);
+    }
+    else
+    {
+        AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
+        rc = VERR_EM_INTERPRETER;
+    }
+    return rc;
+}
+
+
+/**
+ * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
+    if (EMAreHypercallInstructionsEnabled(pVCpu))
+    {
+        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+        int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0
+                                                         | CPUMCTX_EXTRN_SS  | CPUMCTX_EXTRN_CS     | CPUMCTX_EXTRN_EFER);
+        AssertRCReturn(rc, rc);
+
+        /* Perform the hypercall. */
+        rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
+            AssertRCReturn(rc, rc);
+        }
+        else
+            Assert(   rcStrict == VINF_GIM_R3_HYPERCALL
+                   || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
+                   || RT_FAILURE(rcStrict));
+
+        /* If the hypercall changes anything other than guest's general-purpose registers,
+           we would need to reload the guest changed bits here before VM-entry. */
+    }
+    else
+        Log4Func(("Hypercalls not enabled\n"));
+
+    /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
+    if (RT_FAILURE(rcStrict))
+    {
+        hmR0VmxSetPendingXcptUD(pVCpu);
+        rcStrict = VINF_SUCCESS;
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
+
+    if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    else
+        AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
+                         VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
+    if (rcStrict == VINF_SUCCESS)
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
+    if (RT_SUCCESS(rcStrict))
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+        if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
+            rcStrict = VINF_SUCCESS;
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    return VINF_EM_RESET;
+}
+
+
+/**
+ * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);            /* Advancing the RIP above should've imported eflags. */
+    if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx))    /* Requires eflags. */
+        rc = VINF_SUCCESS;
+    else
+        rc = VINF_EM_HALT;
+
+    if (rc != VINF_SUCCESS)
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
+    return rc;
+}
+
+
+/**
+ * VM-exit handler for instructions that result in a \#UD exception delivered to
+ * the guest.
+ */
+HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    hmR0VmxSetPendingXcptUD(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit handler for expiry of the VMX-preemption timer.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
+    pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
+
+    /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
+    return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
+                                                                                : HM_CHANGED_RAISED_XCPT_MASK);
+
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /** @todo Enable the new code after finding a reliably guest test-case. */
+#if 1
+    return VERR_EM_INTERPRETER;
+#else
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
+                                                                    | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    /* Paranoia. Ensure this has a memory operand. */
+    Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
+
+    uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
+    Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
+    uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
+                                                         : pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
+
+    RTGCPTR GCPtrDesc;
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
+                                                  GCPtrDesc, uType);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+#endif
+}
+
+
+/**
+ * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
+ * VM-exit.
+ */
+HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+    AssertRCReturn(rc, rc);
+
+    rc = hmR0VmxCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
+    NOREF(uInvalidReason);
+
+#ifdef VBOX_STRICT
+    uint32_t fIntrState;
+    uint64_t u64Val;
+    hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
+    hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
+    hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
+
+    Log4(("uInvalidReason                             %u\n",     uInvalidReason));
+    Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO    %#RX32\n", pVmxTransient->uEntryIntInfo));
+    Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE    %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
+    Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH         %#RX32\n", pVmxTransient->cbEntryInstr));
+
+    rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);            AssertRC(rc);
+    Log4(("VMX_VMCS32_GUEST_INT_STATE                 %#RX32\n", fIntrState));
+    rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Val);                        AssertRC(rc);
+    Log4(("VMX_VMCS_GUEST_CR0                         %#RX64\n", u64Val));
+    rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_MASK, &u64Val);                    AssertRC(rc);
+    Log4(("VMX_VMCS_CTRL_CR0_MASK                     %#RX64\n", u64Val));
+    rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val);             AssertRC(rc);
+    Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW              %#RX64\n", u64Val));
+    rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_MASK, &u64Val);                    AssertRC(rc);
+    Log4(("VMX_VMCS_CTRL_CR4_MASK                     %#RX64\n", u64Val));
+    rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val);             AssertRC(rc);
+    Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW              %#RX64\n", u64Val));
+    if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
+    {
+        rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val);             AssertRC(rc);
+        Log4(("VMX_VMCS64_CTRL_EPTP_FULL                  %#RX64\n", u64Val));
+    }
+    hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
+#endif
+
+    return VERR_VMX_INVALID_GUEST_STATE;
+}
+
+/**
+ * VM-exit handler for all undefined/unexpected reasons. Should never happen.
+ */
+HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    /*
+     * Cumulative notes of all recognized but unexpected VM-exits.
+     *
+     * 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
+     *    nested-paging is used.
+     *
+     * 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
+     *    emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
+     *    this function (and thereby stop VM execution) for handling such instructions.
+     *
+     *
+     * VMX_EXIT_INIT_SIGNAL:
+     *    INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
+     *    It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
+     *    VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
+     *
+     *    See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
+     *    See Intel spec. 29.3 "VMX Instructions" for "VMXON".
+     *    See Intel spec. "23.8 Restrictions on VMX operation".
+     *
+     * VMX_EXIT_SIPI:
+     *    SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
+     *    activity state is used. We don't make use of it as our guests don't have direct
+     *    access to the host local APIC.
+     *
+     *    See Intel spec. 25.3 "Other Causes of VM-exits".
+     *
+     * VMX_EXIT_IO_SMI:
+     * VMX_EXIT_SMI:
+     *    This can only happen if we support dual-monitor treatment of SMI, which can be
+     *    activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
+     *    monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
+     *    VMX root mode or receive an SMI. If we get here, something funny is going on.
+     *
+     *    See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
+     *    See Intel spec. 25.3 "Other Causes of VM-Exits"
+     *
+     * VMX_EXIT_ERR_MSR_LOAD:
+     *    Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
+     *    and typically indicates a bug in the hypervisor code. We thus cannot not resume
+     *    execution.
+     *
+     *    See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
+     *
+     * VMX_EXIT_ERR_MACHINE_CHECK:
+     *    Machine check exceptions indicates a fatal/unrecoverable hardware condition
+     *    including but not limited to system bus, ECC, parity, cache and TLB errors. A
+     *    #MC exception abort class exception is raised. We thus cannot assume a
+     *    reasonable chance of continuing any sort of execution and we bail.
+     *
+     *    See Intel spec. 15.1 "Machine-check Architecture".
+     *    See Intel spec. 27.1 "Architectural State Before A VM Exit".
+     *
+     * VMX_EXIT_PML_FULL:
+     * VMX_EXIT_VIRTUALIZED_EOI:
+     * VMX_EXIT_APIC_WRITE:
+     *    We do not currently support any of these features and thus they are all unexpected
+     *    VM-exits.
+     *
+     * VMX_EXIT_GDTR_IDTR_ACCESS:
+     * VMX_EXIT_LDTR_TR_ACCESS:
+     * VMX_EXIT_RDRAND:
+     * VMX_EXIT_RSM:
+     * VMX_EXIT_VMFUNC:
+     * VMX_EXIT_ENCLS:
+     * VMX_EXIT_RDSEED:
+     * VMX_EXIT_XSAVES:
+     * VMX_EXIT_XRSTORS:
+     * VMX_EXIT_UMWAIT:
+     * VMX_EXIT_TPAUSE:
+     *    These VM-exits are -not- caused unconditionally by execution of the corresponding
+     *    instruction. Any VM-exit for these instructions indicate a hardware problem,
+     *    unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
+     *
+     *    See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
+    HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
+}
+
+
+/**
+ * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
+ */
+HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /** @todo Optimize this: We currently drag in the whole MSR state
+     * (CPUMCTX_EXTRN_ALL_MSRS) here.  We should optimize this to only get
+     * MSRs required.  That would require changes to IEM and possibly CPUM too.
+     * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
+    PVMXVMCSINFO   pVmcsInfo = pVmxTransient->pVmcsInfo;
+    uint32_t const idMsr     = pVCpu->cpum.GstCtx.ecx;
+    uint64_t       fImport   = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS;
+    switch (idMsr)
+    {
+        case MSR_K8_FS_BASE: fImport |= CPUMCTX_EXTRN_FS; break;
+        case MSR_K8_GS_BASE: fImport |= CPUMCTX_EXTRN_GS; break;
+    }
+
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
+    AssertRCReturn(rc, rc);
+
+    Log4Func(("ecx=%#RX32\n", idMsr));
+
+#ifdef VBOX_STRICT
+    Assert(!pVmxTransient->fIsNestedGuest);
+    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+    {
+        if (   hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
+            && idMsr != MSR_K6_EFER)
+        {
+            AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
+            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
+        }
+        if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
+        {
+            Assert(pVmcsInfo->pvMsrBitmap);
+            uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
+            if (fMsrpm & VMXMSRPM_ALLOW_RD)
+            {
+                AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
+                HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
+            }
+        }
+    }
+#endif
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
+    if (rcStrict == VINF_SUCCESS)
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    else
+        AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
+ */
+HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /** @todo Optimize this: We currently drag in the whole MSR state
+     * (CPUMCTX_EXTRN_ALL_MSRS) here.  We should optimize this to only get
+     * MSRs required.  That would require changes to IEM and possibly CPUM too.
+     * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
+    uint32_t const idMsr    = pVCpu->cpum.GstCtx.ecx;
+    uint64_t       fImport  = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS;
+
+    /*
+     * The FS and GS base MSRs are not part of the above all-MSRs mask.
+     * Although we don't need to fetch the base as it will be overwritten shortly, while
+     * loading guest-state we would also load the entire segment register including limit
+     * and attributes and thus we need to load them here.
+     */
+    switch (idMsr)
+    {
+        case MSR_K8_FS_BASE: fImport |= CPUMCTX_EXTRN_FS; break;
+        case MSR_K8_GS_BASE: fImport |= CPUMCTX_EXTRN_GS; break;
+    }
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
+    AssertRCReturn(rc, rc);
+
+    Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
+
+    if (rcStrict == VINF_SUCCESS)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+
+        /* If this is an X2APIC WRMSR access, update the APIC state as well. */
+        if (    idMsr == MSR_IA32_APICBASE
+            || (   idMsr >= MSR_IA32_X2APIC_START
+                && idMsr <= MSR_IA32_X2APIC_END))
+        {
+            /*
+             * We've already saved the APIC related guest-state (TPR) in post-run phase.
+             * When full APIC register virtualization is implemented we'll have to make
+             * sure APIC state is saved from the VMCS before IEM changes it.
+             */
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
+        }
+        else if (idMsr == MSR_IA32_TSC)        /* Windows 7 does this during bootup. See @bugref{6398}. */
+            pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
+        else if (idMsr == MSR_K6_EFER)
+        {
+            /*
+             * If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
+             * as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
+             * We care about the other bits as well, SCE and NXE. See @bugref{7368}.
+             */
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
+        }
+
+        /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
+        if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
+        {
+            switch (idMsr)
+            {
+                case MSR_IA32_SYSENTER_CS:  ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR);  break;
+                case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
+                case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
+                case MSR_K8_FS_BASE:        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS);               break;
+                case MSR_K8_GS_BASE:        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS);               break;
+                case MSR_K6_EFER:           /* Nothing to do, already handled above. */                                    break;
+                default:
+                {
+                    if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
+                        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
+                    else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
+                        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
+                    break;
+                }
+            }
+        }
+#ifdef VBOX_STRICT
+        else
+        {
+            /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
+            switch (idMsr)
+            {
+                case MSR_IA32_SYSENTER_CS:
+                case MSR_IA32_SYSENTER_EIP:
+                case MSR_IA32_SYSENTER_ESP:
+                case MSR_K8_FS_BASE:
+                case MSR_K8_GS_BASE:
+                {
+                    AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
+                    HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
+                }
+
+                /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
+                default:
+                {
+                    if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
+                    {
+                        /* EFER MSR writes are always intercepted. */
+                        if (idMsr != MSR_K6_EFER)
+                        {
+                            AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
+                                             idMsr));
+                            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
+                        }
+                    }
+
+                    if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
+                    {
+                        Assert(pVmcsInfo->pvMsrBitmap);
+                        uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
+                        if (fMsrpm & VMXMSRPM_ALLOW_WR)
+                        {
+                            AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
+                            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+#endif  /* VBOX_STRICT */
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    else
+        AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /** @todo The guest has likely hit a contended spinlock. We might want to
+     *        poke a schedule different guest VCPU. */
+    int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
+    if (RT_SUCCESS(rc))
+        return VINF_EM_RAW_INTERRUPT;
+
+    AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VM-exit handler for when the TPR value is lowered below the specified
+ * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
+ */
+HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
+
+    /*
+     * The TPR shadow would've been synced with the APIC TPR in the post-run phase.
+     * We'll re-evaluate pending interrupts and inject them before the next VM
+     * entry so we can just continue execution here.
+     */
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
+ * VM-exit.
+ *
+ * @retval VINF_SUCCESS when guest execution can continue.
+ * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
+ * @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
+ *         incompatible guest state for VMX execution (real-on-v86 case).
+ */
+HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+
+    VBOXSTRICTRC rcStrict;
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    uint64_t const uExitQual   = pVmxTransient->uExitQual;
+    uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
+    switch (uAccessType)
+    {
+        /*
+         * MOV to CRx.
+         */
+        case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
+        {
+            int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
+            AssertRCReturn(rc, rc);
+
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+            uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
+            uint8_t const  iGReg   = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
+            uint8_t const  iCrReg  = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
+
+            /*
+             * MOV to CR3 only cause a VM-exit when one or more of the following are true:
+             *   - When nested paging isn't used.
+             *   - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
+             *   - We are executing in the VM debug loop.
+             */
+            Assert(   iCrReg != 3
+                   || !pVM->hm.s.fNestedPaging
+                   || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
+                   || pVCpu->hm.s.fUsingDebugLoop);
+
+            /* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
+            Assert(   iCrReg != 8
+                   || !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
+
+            rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
+            AssertMsg(   rcStrict == VINF_SUCCESS
+                      || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+            /*
+             * This is a kludge for handling switches back to real mode when we try to use
+             * V86 mode to run real mode code directly.  Problem is that V86 mode cannot
+             * deal with special selector values, so we have to return to ring-3 and run
+             * there till the selector values are V86 mode compatible.
+             *
+             * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
+             *       latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
+             *       this function.
+             */
+            if (   iCrReg == 0
+                && rcStrict == VINF_SUCCESS
+                && !pVM->hm.s.vmx.fUnrestrictedGuest
+                && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
+                && (uOldCr0 & X86_CR0_PE)
+                && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
+            {
+                /** @todo Check selectors rather than returning all the time.  */
+                Assert(!pVmxTransient->fIsNestedGuest);
+                Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
+                rcStrict = VINF_EM_RESCHEDULE_REM;
+            }
+            break;
+        }
+
+        /*
+         * MOV from CRx.
+         */
+        case VMX_EXIT_QUAL_CRX_ACCESS_READ:
+        {
+            uint8_t const iGReg  = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
+            uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
+
+            /*
+             * MOV from CR3 only cause a VM-exit when one or more of the following are true:
+             *   - When nested paging isn't used.
+             *   - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
+             *   - We are executing in the VM debug loop.
+             */
+            Assert(   iCrReg != 3
+                   || !pVM->hm.s.fNestedPaging
+                   || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
+                   || pVCpu->hm.s.fUsingDebugLoop);
+
+            /* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
+            Assert(   iCrReg != 8
+                   || !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
+
+            rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
+            break;
+        }
+
+        /*
+         * CLTS (Clear Task-Switch Flag in CR0).
+         */
+        case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
+        {
+            rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
+            break;
+        }
+
+        /*
+         * LMSW (Load Machine-Status Word into CR0).
+         * LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
+         */
+        case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
+        {
+            RTGCPTR        GCPtrEffDst;
+            uint8_t const  cbInstr     = pVmxTransient->cbExitInstr;
+            uint16_t const uMsw        = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
+            bool const     fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
+            if (fMemOperand)
+            {
+                hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
+                GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
+            }
+            else
+                GCPtrEffDst = NIL_RTGCPTR;
+            rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
+            break;
+        }
+
+        default:
+        {
+            AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
+            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
+        }
+    }
+
+    Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
+                                   == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
+    Assert(rcStrict != VINF_IEM_RAISED_XCPT);
+
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
+    NOREF(pVM);
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
+
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK
+                                                     | CPUMCTX_EXTRN_EFER);
+    /* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
+    AssertRCReturn(rc, rc);
+
+    /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
+    uint32_t const uIOPort      = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
+    uint8_t  const uIOSize      = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
+    bool     const fIOWrite     = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
+    bool     const fIOString    = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
+    bool     const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
+    bool     const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
+    AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
+
+    /*
+     * Update exit history to see if this exit can be optimized.
+     */
+    VBOXSTRICTRC rcStrict;
+    PCEMEXITREC  pExitRec = NULL;
+    if (   !fGstStepping
+        && !fDbgStepping)
+        pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
+                                                    !fIOString
+                                                    ? !fIOWrite
+                                                    ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
+                                                    : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
+                                                    : !fIOWrite
+                                                    ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
+                                                    : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
+                                                    pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
+    if (!pExitRec)
+    {
+        static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 };                    /* Size of the I/O accesses in bytes. */
+        static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff };   /* AND masks for saving result in AL/AX/EAX. */
+
+        uint32_t const cbValue  = s_aIOSizes[uIOSize];
+        uint32_t const cbInstr  = pVmxTransient->cbExitInstr;
+        bool  fUpdateRipAlready = false; /* ugly hack, should be temporary. */
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        if (fIOString)
+        {
+            /*
+             * INS/OUTS - I/O String instruction.
+             *
+             * Use instruction-information if available, otherwise fall back on
+             * interpreting the instruction.
+             */
+            Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
+            AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
+            bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
+            if (fInsOutsInfo)
+            {
+                hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+                AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
+                AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
+                IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
+                bool const fRep           = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
+                if (fIOWrite)
+                    rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
+                                                    pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
+                else
+                {
+                    /*
+                     * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
+                     * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
+                     * See Intel Instruction spec. for "INS".
+                     * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
+                     */
+                    rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
+                }
+            }
+            else
+                rcStrict = IEMExecOne(pVCpu);
+
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
+            fUpdateRipAlready = true;
+        }
+        else
+        {
+            /*
+             * IN/OUT - I/O instruction.
+             */
+            Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
+            uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
+            Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
+            if (fIOWrite)
+            {
+                rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
+                if (    rcStrict == VINF_IOM_R3_IOPORT_WRITE
+                    && !pCtx->eflags.Bits.u1TF)
+                    rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
+            }
+            else
+            {
+                uint32_t u32Result = 0;
+                rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
+                if (IOM_SUCCESS(rcStrict))
+                {
+                    /* Save result of I/O IN instr. in AL/AX/EAX. */
+                    pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
+                }
+                if (    rcStrict == VINF_IOM_R3_IOPORT_READ
+                    && !pCtx->eflags.Bits.u1TF)
+                    rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
+            }
+        }
+
+        if (IOM_SUCCESS(rcStrict))
+        {
+            if (!fUpdateRipAlready)
+            {
+                hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
+            }
+
+            /*
+             * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
+             * while booting Fedora 17 64-bit guest.
+             *
+             * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
+             */
+            if (fIOString)
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
+
+            /*
+             * If any I/O breakpoints are armed, we need to check if one triggered
+             * and take appropriate action.
+             * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
+             */
+            rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
+            AssertRCReturn(rc, rc);
+
+            /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
+             *  execution engines about whether hyper BPs and such are pending. */
+            uint32_t const uDr7 = pCtx->dr[7];
+            if (RT_UNLIKELY(   (   (uDr7 & X86_DR7_ENABLED_MASK)
+                                && X86_DR7_ANY_RW_IO(uDr7)
+                                && (pCtx->cr4 & X86_CR4_DE))
+                            || DBGFBpIsHwIoArmed(pVM)))
+            {
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
+
+                /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
+                VMMRZCallRing3Disable(pVCpu);
+                HM_DISABLE_PREEMPT(pVCpu);
+
+                bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
+
+                VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
+                if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
+                {
+                    /* Raise #DB. */
+                    if (fIsGuestDbgActive)
+                        ASMSetDR6(pCtx->dr[6]);
+                    if (pCtx->dr[7] != uDr7)
+                        pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
+
+                    hmR0VmxSetPendingXcptDB(pVCpu);
+                }
+                /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
+                   however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
+                else if (   rcStrict2 != VINF_SUCCESS
+                         && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
+                    rcStrict = rcStrict2;
+                AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
+
+                HM_RESTORE_PREEMPT();
+                VMMRZCallRing3Enable(pVCpu);
+            }
+        }
+
+#ifdef VBOX_STRICT
+        if (   rcStrict == VINF_IOM_R3_IOPORT_READ
+            || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
+            Assert(!fIOWrite);
+        else if (   rcStrict == VINF_IOM_R3_IOPORT_WRITE
+                 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
+                 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
+            Assert(fIOWrite);
+        else
+        {
+# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
+           *        statuses, that the VMM device and some others may return. See
+           *        IOM_SUCCESS() for guidance. */
+            AssertMsg(   RT_FAILURE(rcStrict)
+                      || rcStrict == VINF_SUCCESS
+                      || rcStrict == VINF_EM_RAW_EMULATE_INSTR
+                      || rcStrict == VINF_EM_DBG_BREAKPOINT
+                      || rcStrict == VINF_EM_RAW_GUEST_TRAP
+                      || rcStrict == VINF_EM_RAW_TO_R3
+                      || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+# endif
+        }
+#endif
+        STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
+    }
+    else
+    {
+        /*
+         * Frequent exit or something needing probing.  Get state and call EMHistoryExec.
+         */
+        int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+        AssertRCReturn(rc2, rc2);
+        STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
+                         : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
+        Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
+              fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
+
+        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+
+        Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /* Check if this task-switch occurred while delivery an event through the guest IDT. */
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
+    {
+        hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
+        if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
+        {
+            uint32_t uErrCode;
+            if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
+            {
+                hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
+                uErrCode = pVmxTransient->uIdtVectoringErrorCode;
+            }
+            else
+                uErrCode = 0;
+
+            RTGCUINTPTR GCPtrFaultAddress;
+            if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
+                GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
+            else
+                GCPtrFaultAddress = 0;
+
+            hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+
+            hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
+                                   pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
+
+            Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
+                      VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
+            return VINF_EM_RAW_INJECT_TRPM_EVENT;
+        }
+    }
+
+    /* Fall back to the interpreter to emulate the task-switch. */
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
+    return VERR_EM_INTERPRETER;
+}
+
+
+/**
+ * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
+    int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+    AssertRC(rc);
+    return VINF_EM_DBG_STEPPED;
+}
+
+
+/**
+ * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
+
+    hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
+    hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
+    hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
+
+    /*
+     * If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
+     */
+    VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
+        if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
+        {
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
+            return VINF_EM_RAW_INJECT_TRPM_EVENT;
+        }
+    }
+    else
+    {
+        Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
+        return rcStrict;
+    }
+
+    /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
+    AssertRCReturn(rc, rc);
+
+    /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
+    uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
+    switch (uAccessType)
+    {
+        case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
+        case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
+        {
+            AssertMsg(   !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+                      || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
+                      ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
+
+            RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase;    /* Always up-to-date, as it is not part of the VMCS. */
+            GCPhys &= PAGE_BASE_GC_MASK;
+            GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
+            Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
+                 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
+
+            rcStrict = IOMR0MmioPhysHandler(pVCpu->CTX_SUFF(pVM), pVCpu,
+                                            uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW, GCPhys);
+            Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            if (   rcStrict == VINF_SUCCESS
+                || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
+                || rcStrict == VERR_PAGE_NOT_PRESENT)
+            {
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
+                                                         | HM_CHANGED_GUEST_APIC_TPR);
+                rcStrict = VINF_SUCCESS;
+            }
+            break;
+        }
+
+        default:
+        {
+            Log4Func(("uAccessType=%#x\n", uAccessType));
+            rcStrict = VINF_EM_RAW_EMULATE_INSTR;
+            break;
+        }
+    }
+
+    if (rcStrict != VINF_SUCCESS)
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+
+    /* We might get this VM-exit if the nested-guest is not intercepting MOV DRx accesses. */
+    if (!pVmxTransient->fIsNestedGuest)
+    {
+        /* We should -not- get this VM-exit if the guest's debug registers were active. */
+        if (pVmxTransient->fWasGuestDebugStateActive)
+        {
+            AssertMsgFailed(("Unexpected MOV DRx exit\n"));
+            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
+        }
+
+        if (   !pVCpu->hm.s.fSingleInstruction
+            && !pVmxTransient->fWasHyperDebugStateActive)
+        {
+            Assert(!DBGFIsStepping(pVCpu));
+            Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
+
+            /* Don't intercept MOV DRx any more. */
+            pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
+            int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+            AssertRC(rc);
+
+            /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
+            VMMRZCallRing3Disable(pVCpu);
+            HM_DISABLE_PREEMPT(pVCpu);
+
+            /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
+            CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
+            Assert(CPUMIsGuestDebugStateActive(pVCpu));
+
+            HM_RESTORE_PREEMPT();
+            VMMRZCallRing3Enable(pVCpu);
+
+#ifdef VBOX_WITH_STATISTICS
+            hmR0VmxReadExitQualVmcs(pVmxTransient);
+            if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
+            else
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
+#endif
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
+            return VINF_SUCCESS;
+        }
+    }
+
+    /*
+     * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
+     * The EFER MSR is always up-to-date.
+     * Update the segment registers and DR7 from the CPU.
+     */
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
+    AssertRCReturn(rc, rc);
+    Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
+    {
+        rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
+                                 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
+                                 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
+        if (RT_SUCCESS(rc))
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
+    }
+    else
+    {
+        rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
+                                VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
+                                VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
+    }
+
+    Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
+    if (RT_SUCCESS(rc))
+    {
+        int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
+        AssertRCReturn(rc2, rc2);
+        return VINF_SUCCESS;
+    }
+    return rc;
+}
+
+
+/**
+ * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
+
+    hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
+    hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
+    hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
+
+    /*
+     * If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
+     */
+    VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        /*
+         * In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
+         * instruction emulation to inject the original event. Otherwise, injecting the original event
+         * using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
+         */
+        if (!pVCpu->hm.s.Event.fPending)
+        { /* likely */ }
+        else
+        {
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            /** @todo NSTVMX: Think about how this should be handled. */
+            if (pVmxTransient->fIsNestedGuest)
+                return VERR_VMX_IPE_3;
+#endif
+            return VINF_EM_RAW_INJECT_TRPM_EVENT;
+        }
+    }
+    else
+    {
+        Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
+        return rcStrict;
+    }
+
+    /*
+     * Get sufficient state and update the exit history entry.
+     */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
+    AssertRCReturn(rc, rc);
+
+    RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
+    PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
+                                                            EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
+                                                            pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
+    if (!pExitRec)
+    {
+        /*
+         * If we succeed, resume guest execution.
+         * If we fail in interpreting the instruction because we couldn't get the guest physical address
+         * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
+         * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
+         * weird case. See @bugref{6043}.
+         */
+        PVMCC    pVM  = pVCpu->CTX_SUFF(pVM);
+        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+/** @todo bird: We can probably just go straight to IOM here and assume that
+ *        it's MMIO, then fall back on PGM if that hunch didn't work out so
+ *        well.  However, we need to address that aliasing workarounds that
+ *        PGMR0Trap0eHandlerNPMisconfig implements.  So, some care is needed.
+ *
+ *        Might also be interesting to see if we can get this done more or
+ *        less locklessly inside IOM.  Need to consider the lookup table
+ *        updating and use a bit more carefully first (or do all updates via
+ *        rendezvous) */
+        rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
+        Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
+        if (   rcStrict == VINF_SUCCESS
+            || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
+            || rcStrict == VERR_PAGE_NOT_PRESENT)
+        {
+            /* Successfully handled MMIO operation. */
+            ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
+                                                     | HM_CHANGED_GUEST_APIC_TPR);
+            rcStrict = VINF_SUCCESS;
+        }
+    }
+    else
+    {
+        /*
+         * Frequent exit or something needing probing. Call EMHistoryExec.
+         */
+        Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
+
+        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+
+        Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
+
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
+    hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
+    hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
+
+    /*
+     * If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
+     */
+    VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        /*
+         * If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
+         * we shall resolve the nested #PF and re-inject the original event.
+         */
+        if (pVCpu->hm.s.Event.fPending)
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
+    }
+    else
+    {
+        Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
+        return rcStrict;
+    }
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
+    AssertRCReturn(rc, rc);
+
+    RTGCPHYS const GCPhys    = pVmxTransient->uGuestPhysicalAddr;
+    uint64_t const uExitQual = pVmxTransient->uExitQual;
+    AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
+
+    RTGCUINT uErrorCode = 0;
+    if (uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
+        uErrorCode |= X86_TRAP_PF_ID;
+    if (uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
+        uErrorCode |= X86_TRAP_PF_RW;
+    if (uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
+        uErrorCode |= X86_TRAP_PF_P;
+
+    PVMCC    pVM  = pVCpu->CTX_SUFF(pVM);
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
+
+    /*
+     * Handle the pagefault trap for the nested shadow table.
+     */
+    TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
+    rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
+    TRPMResetTrap(pVCpu);
+
+    /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
+    if (   rcStrict == VINF_SUCCESS
+        || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
+        || rcStrict == VERR_PAGE_NOT_PRESENT)
+    {
+        /* Successfully synced our nested page tables. */
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
+        return VINF_SUCCESS;
+    }
+
+    Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
+                                                                    | CPUMCTX_EXTRN_HWVIRT
+                                                                    | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason     = pVmxTransient->uExitReason;
+    ExitInfo.u64Qual     = pVmxTransient->uExitQual;
+    ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
+    ExitInfo.cbInstr     = pVmxTransient->cbExitInstr;
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
+       otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+        if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
+            rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
+    }
+    Assert(rcStrict != VINF_IEM_RAISED_XCPT);
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
+                                                                    | CPUMCTX_EXTRN_HWVIRT
+                                                                    | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason     = pVmxTransient->uExitReason;
+    ExitInfo.u64Qual     = pVmxTransient->uExitQual;
+    ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
+    ExitInfo.cbInstr     = pVmxTransient->cbExitInstr;
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
+                                                                    | CPUMCTX_EXTRN_HWVIRT
+                                                                    | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason     = pVmxTransient->uExitReason;
+    ExitInfo.u64Qual     = pVmxTransient->uExitQual;
+    ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
+    ExitInfo.cbInstr     = pVmxTransient->cbExitInstr;
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /*
+     * Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
+     * thus might not need to import the shadow VMCS state, it's safer just in case
+     * code elsewhere dares look at unsynced VMCS fields.
+     */
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
+                                                                    | CPUMCTX_EXTRN_HWVIRT
+                                                                    | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason     = pVmxTransient->uExitReason;
+    ExitInfo.u64Qual     = pVmxTransient->uExitQual;
+    ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
+    ExitInfo.cbInstr     = pVmxTransient->cbExitInstr;
+    if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
+        HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
+       otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+        if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
+            rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
+    }
+    Assert(rcStrict != VINF_IEM_RAISED_XCPT);
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /*
+     * Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
+     * gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
+     * flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
+     */
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
+                                                                    | CPUMCTX_EXTRN_HWVIRT
+                                                                    | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason     = pVmxTransient->uExitReason;
+    ExitInfo.u64Qual     = pVmxTransient->uExitQual;
+    ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
+    ExitInfo.cbInstr     = pVmxTransient->cbExitInstr;
+    if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
+        HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
+                                                                    | CPUMCTX_EXTRN_HWVIRT
+                                                                    | IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_HWVIRT);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
+                                                                    | CPUMCTX_EXTRN_HWVIRT
+                                                                    | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason     = pVmxTransient->uExitReason;
+    ExitInfo.u64Qual     = pVmxTransient->uExitQual;
+    ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
+    ExitInfo.cbInstr     = pVmxTransient->cbExitInstr;
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
+                                                                    | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason     = pVmxTransient->uExitReason;
+    ExitInfo.u64Qual     = pVmxTransient->uExitQual;
+    ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
+    ExitInfo.cbInstr     = pVmxTransient->cbExitInstr;
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+/** @} */
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/** @name Nested-guest VM-exit handlers.
+ * @{
+ */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+
+/**
+ * Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
+
+    uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
+    uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
+    Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
+
+    switch (uExitIntType)
+    {
+        /*
+         * Physical NMIs:
+         *     We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
+         */
+        case VMX_EXIT_INT_INFO_TYPE_NMI:
+            return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
+
+        /*
+         * Hardware exceptions,
+         * Software exceptions,
+         * Privileged software exceptions:
+         *     Figure out if the exception must be delivered to the guest or the nested-guest.
+         */
+        case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
+        case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
+        case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
+        {
+            hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
+            hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+            hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
+            hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
+
+            PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+            bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
+                                                                   pVmxTransient->uExitIntErrorCode);
+            if (fIntercept)
+            {
+                /* Exit qualification is required for debug and page-fault exceptions. */
+                hmR0VmxReadExitQualVmcs(pVmxTransient);
+
+                /*
+                 * For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
+                 * software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
+                 * length. However, if delivery of a software interrupt, software exception or privileged
+                 * software exception causes a VM-exit, that too provides the VM-exit instruction length.
+                 */
+                VMXVEXITINFO ExitInfo;
+                RT_ZERO(ExitInfo);
+                ExitInfo.uReason = pVmxTransient->uExitReason;
+                ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
+                ExitInfo.u64Qual = pVmxTransient->uExitQual;
+
+                VMXVEXITEVENTINFO ExitEventInfo;
+                RT_ZERO(ExitEventInfo);
+                ExitEventInfo.uExitIntInfo         = pVmxTransient->uExitIntInfo;
+                ExitEventInfo.uExitIntErrCode      = pVmxTransient->uExitIntErrorCode;
+                ExitEventInfo.uIdtVectoringInfo    = pVmxTransient->uIdtVectoringInfo;
+                ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
+
+#ifdef DEBUG_ramshankar
+                hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+                Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n", pVmxTransient->uExitIntInfo,
+                          pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
+                if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
+                {
+                    Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n", pVmxTransient->uIdtVectoringInfo,
+                              pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
+                }
+#endif
+                return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
+            }
+
+            /* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in hmR0VmxExitXcptPF. */
+            Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
+            return hmR0VmxExitXcpt(pVCpu, pVmxTransient);
+        }
+
+        /*
+         * Software interrupts:
+         *    VM-exits cannot be caused by software interrupts.
+         *
+         * External interrupts:
+         *    This should only happen when "acknowledge external interrupts on VM-exit"
+         *    control is set. However, we never set this when executing a guest or
+         *    nested-guest. For nested-guests it is emulated while injecting interrupts into
+         *    the guest.
+         */
+        case VMX_EXIT_INT_INFO_TYPE_SW_INT:
+        case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
+        default:
+        {
+            pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
+            return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
+        }
+    }
+}
+
+
+/**
+ * Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
+ * Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    return IEMExecVmxVmexitTripleFault(pVCpu);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
+ */
+HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
+        return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
+    return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
+ */
+HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
+        return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
+    return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
+ * Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
+    hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
+
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason = pVmxTransient->uExitReason;
+    ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
+    ExitInfo.u64Qual = pVmxTransient->uExitQual;
+
+    VMXVEXITEVENTINFO ExitEventInfo;
+    RT_ZERO(ExitEventInfo);
+    ExitEventInfo.uIdtVectoringInfo    = pVmxTransient->uIdtVectoringInfo;
+    ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
+    return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
+    {
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return hmR0VmxExitHlt(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
+    {
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        hmR0VmxReadExitQualVmcs(pVmxTransient);
+
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason = pVmxTransient->uExitReason;
+        ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
+        ExitInfo.u64Qual = pVmxTransient->uExitQual;
+        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
+    {
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
+ * (VMX_EXIT_VMWRITE). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    Assert(   pVmxTransient->uExitReason == VMX_EXIT_VMREAD
+           || pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
+
+    hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+
+    uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
+    Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
+    uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
+
+    HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
+    if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
+        u64VmcsField &= UINT64_C(0xffffffff);
+
+    if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
+    {
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        hmR0VmxReadExitQualVmcs(pVmxTransient);
+
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason   = pVmxTransient->uExitReason;
+        ExitInfo.cbInstr   = pVmxTransient->cbExitInstr;
+        ExitInfo.u64Qual   = pVmxTransient->uExitQual;
+        ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
+        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
+        return hmR0VmxExitVmread(pVCpu, pVmxTransient);
+    return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
+    {
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+
+    return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+
+    VBOXSTRICTRC rcStrict;
+    uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
+    switch (uAccessType)
+    {
+        case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
+        {
+            uint8_t const iCrReg   = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
+            uint8_t const iGReg    = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
+            Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
+            uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
+
+            bool fIntercept;
+            switch (iCrReg)
+            {
+                case 0:
+                case 4:
+                    fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
+                    break;
+
+                case 3:
+                    fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
+                    break;
+
+                case 8:
+                    fIntercept = CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
+                    break;
+
+                default:
+                    fIntercept = false;
+                    break;
+            }
+            if (fIntercept)
+            {
+                VMXVEXITINFO ExitInfo;
+                RT_ZERO(ExitInfo);
+                ExitInfo.uReason = pVmxTransient->uExitReason;
+                ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
+                ExitInfo.u64Qual = pVmxTransient->uExitQual;
+                rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+            }
+            else
+                rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
+            break;
+        }
+
+        case VMX_EXIT_QUAL_CRX_ACCESS_READ:
+        {
+            /*
+             * CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
+             * CR2 reads do not cause a VM-exit.
+             * CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
+             * CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
+             */
+            uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
+            if (   iCrReg == 3
+                || iCrReg == 8)
+            {
+                static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
+                                                                  0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
+                uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
+                if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uIntercept))
+                {
+                    VMXVEXITINFO ExitInfo;
+                    RT_ZERO(ExitInfo);
+                    ExitInfo.uReason = pVmxTransient->uExitReason;
+                    ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
+                    ExitInfo.u64Qual = pVmxTransient->uExitQual;
+                    rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+                }
+                else
+                {
+                    uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
+                    rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
+                }
+            }
+            else
+            {
+                AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
+                HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
+            }
+            break;
+        }
+
+        case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
+        {
+            PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
+            Assert(pVmcsNstGst);
+            uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
+            uint64_t const uReadShadow  = pVmcsNstGst->u64Cr0ReadShadow.u;
+            if (   (uGstHostMask & X86_CR0_TS)
+                && (uReadShadow  & X86_CR0_TS))
+            {
+                VMXVEXITINFO ExitInfo;
+                RT_ZERO(ExitInfo);
+                ExitInfo.uReason = pVmxTransient->uExitReason;
+                ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
+                ExitInfo.u64Qual = pVmxTransient->uExitQual;
+                rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+            }
+            else
+                rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
+            break;
+        }
+
+        case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:        /* LMSW (Load Machine-Status Word into CR0) */
+        {
+            RTGCPTR        GCPtrEffDst;
+            uint16_t const uNewMsw     = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
+            bool const     fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
+            if (fMemOperand)
+            {
+                hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
+                GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
+            }
+            else
+                GCPtrEffDst = NIL_RTGCPTR;
+
+            if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw))
+            {
+                VMXVEXITINFO ExitInfo;
+                RT_ZERO(ExitInfo);
+                ExitInfo.uReason            = pVmxTransient->uExitReason;
+                ExitInfo.cbInstr            = pVmxTransient->cbExitInstr;
+                ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
+                ExitInfo.u64Qual            = pVmxTransient->uExitQual;
+                rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+            }
+            else
+                rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
+            break;
+        }
+
+        default:
+        {
+            AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
+            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
+        }
+    }
+
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
+    {
+        hmR0VmxReadExitQualVmcs(pVmxTransient);
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason = pVmxTransient->uExitReason;
+        ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
+        ExitInfo.u64Qual = pVmxTransient->uExitQual;
+        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+
+    uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
+    uint8_t  const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
+    AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
+
+    static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 };   /* Size of the I/O accesses in bytes. */
+    uint8_t const cbAccess = s_aIOSizes[uIOSize];
+    if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
+    {
+        /*
+         * IN/OUT instruction:
+         *   - Provides VM-exit instruction length.
+         *
+         * INS/OUTS instruction:
+         *   - Provides VM-exit instruction length.
+         *   - Provides Guest-linear address.
+         *   - Optionally provides VM-exit instruction info (depends on CPU feature).
+         */
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+
+        /* Make sure we don't use stale/uninitialized VMX-transient info. below. */
+        pVmxTransient->ExitInstrInfo.u  = 0;
+        pVmxTransient->uGuestLinearAddr = 0;
+
+        bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
+        bool const fIOString       = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
+        if (fIOString)
+        {
+            hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
+            if (fVmxInsOutsInfo)
+            {
+                Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
+                hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+            }
+        }
+
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason            = pVmxTransient->uExitReason;
+        ExitInfo.cbInstr            = pVmxTransient->cbExitInstr;
+        ExitInfo.u64Qual            = pVmxTransient->uExitQual;
+        ExitInfo.InstrInfo          = pVmxTransient->ExitInstrInfo;
+        ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
+        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
+ */
+HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    uint32_t fMsrpm;
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
+        fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
+    else
+        fMsrpm = VMXMSRPM_EXIT_RD;
+
+    if (fMsrpm & VMXMSRPM_EXIT_RD)
+    {
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
+ */
+HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    uint32_t fMsrpm;
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
+        fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
+    else
+        fMsrpm = VMXMSRPM_EXIT_WR;
+
+    if (fMsrpm & VMXMSRPM_EXIT_WR)
+    {
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
+    {
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return hmR0VmxExitMwait(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
+    hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason                 = pVmxTransient->uExitReason;
+    ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
+    return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
+    {
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
+     *        PAUSE when executing a nested-guest? If it does not, we would not need
+     *        to check for the intercepts here. Just call VM-exit... */
+
+    /* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
+    if (   CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
+        || CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
+    {
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return hmR0VmxExitPause(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for when the TPR value is lowered below the
+ * specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
+ */
+HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
+    {
+        hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason                 = pVmxTransient->uExitReason;
+        ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
+        return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
+    }
+    return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
+    hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+
+    Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
+
+    Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
+              VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
+
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason = pVmxTransient->uExitReason;
+    ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
+    ExitInfo.u64Qual = pVmxTransient->uExitQual;
+
+    VMXVEXITEVENTINFO ExitEventInfo;
+    RT_ZERO(ExitEventInfo);
+    ExitEventInfo.uIdtVectoringInfo    = pVmxTransient->uIdtVectoringInfo;
+    ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
+    return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
+    {
+        Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
+ */
+HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
+    {
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
+    {
+        Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
+        hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+        hmR0VmxReadExitQualVmcs(pVmxTransient);
+        hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+
+        VMXVEXITINFO ExitInfo;
+        RT_ZERO(ExitInfo);
+        ExitInfo.uReason   = pVmxTransient->uExitReason;
+        ExitInfo.cbInstr   = pVmxTransient->cbExitInstr;
+        ExitInfo.u64Qual   = pVmxTransient->uExitQual;
+        ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
+        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for invalid-guest state
+ * (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /*
+     * Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
+     * So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
+     * Handle it like it's in an invalid guest state of the outer guest.
+     *
+     * When the fast path is implemented, this should be changed to cause the corresponding
+     * nested-guest VM-exit.
+     */
+    return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
+ * and only provide the instruction length.
+ *
+ * Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+#ifdef VBOX_STRICT
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    switch (pVmxTransient->uExitReason)
+    {
+        case VMX_EXIT_ENCLS:
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
+            break;
+
+        case VMX_EXIT_VMFUNC:
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_VMFUNC));
+            break;
+    }
+#endif
+
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for instructions that provide instruction length as
+ * well as more information.
+ *
+ * Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+#ifdef VBOX_STRICT
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    switch (pVmxTransient->uExitReason)
+    {
+        case VMX_EXIT_GDTR_IDTR_ACCESS:
+        case VMX_EXIT_LDTR_TR_ACCESS:
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
+            break;
+
+        case VMX_EXIT_RDRAND:
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
+            break;
+
+        case VMX_EXIT_RDSEED:
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
+            break;
+
+        case VMX_EXIT_XSAVES:
+        case VMX_EXIT_XRSTORS:
+            /** @todo NSTVMX: Verify XSS-bitmap. */
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
+            break;
+
+        case VMX_EXIT_UMWAIT:
+        case VMX_EXIT_TPAUSE:
+            Assert(CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
+            break;
+    }
+#endif
+
+    hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
+    hmR0VmxReadExitQualVmcs(pVmxTransient);
+    hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
+
+    VMXVEXITINFO ExitInfo;
+    RT_ZERO(ExitInfo);
+    ExitInfo.uReason   = pVmxTransient->uExitReason;
+    ExitInfo.cbInstr   = pVmxTransient->cbExitInstr;
+    ExitInfo.u64Qual   = pVmxTransient->uExitQual;
+    ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
+    return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+}
+
+/** @} */
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
diff --git a/src/VBox/VMM/VMMR0/HMVMXR0.h b/src/VBox/VMM/VMMR0/HMVMXR0.h
new file mode 100644
index 00000000..a86b4d92
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/HMVMXR0.h
@@ -0,0 +1,56 @@
+/* $Id: HMVMXR0.h $ */
+/** @file
+ * HM VMX (VT-x) - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_VMMR0_HMVMXR0_h
+#define VMM_INCLUDED_SRC_VMMR0_HMVMXR0_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+RT_C_DECLS_BEGIN
+
+/** @defgroup grp_vmx_int   Internal
+ * @ingroup grp_vmx
+ * @internal
+ * @{
+ */
+
+#ifdef IN_RING0
+VMMR0DECL(int)          VMXR0Enter(PVMCPUCC pVCpu);
+VMMR0DECL(void)         VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit);
+VMMR0DECL(int)          VMXR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation);
+VMMR0DECL(int)          VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys,
+                                       bool fEnabledBySystem, PCSUPHWVIRTMSRS pHwvirtMsrs);
+VMMR0DECL(int)          VMXR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys);
+VMMR0DECL(int)          VMXR0GlobalInit(void);
+VMMR0DECL(void)         VMXR0GlobalTerm(void);
+VMMR0DECL(int)          VMXR0InitVM(PVMCC pVM);
+VMMR0DECL(int)          VMXR0TermVM(PVMCC pVM);
+VMMR0DECL(int)          VMXR0SetupVM(PVMCC pVM);
+VMMR0DECL(int)          VMXR0ExportHostState(PVMCPUCC pVCpu);
+VMMR0DECL(int)          VMXR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt);
+VMMR0DECL(int)          VMXR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat);
+VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPUCC pVCpu);
+DECLASM(int)            VMXR0StartVM64(RTHCUINT fResume, PCPUMCTX pCtx, void *pvUnused, PVMCC pVM, PVMCPUCC pVCpu);
+#endif /* IN_RING0 */
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_VMMR0_HMVMXR0_h */
+
diff --git a/src/VBox/VMM/VMMR0/IOMR0.cpp b/src/VBox/VMM/VMMR0/IOMR0.cpp
new file mode 100644
index 00000000..ebba9610
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/IOMR0.cpp
@@ -0,0 +1,57 @@
+/* $Id: IOMR0.cpp $ */
+/** @file
+ * IOM - Host Context Ring 0.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_IOM
+#include <VBox/vmm/iom.h>
+#include "IOMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/log.h>
+#include <iprt/assertcompile.h>
+
+
+
+/**
+ * Initializes the per-VM data for the IOM.
+ *
+ * This is called from under the GVMM lock, so it should only initialize the
+ * data so IOMR0CleanupVM and others will work smoothly.
+ *
+ * @param   pGVM    Pointer to the global VM structure.
+ */
+VMMR0_INT_DECL(void) IOMR0InitPerVMData(PGVM pGVM)
+{
+    AssertCompile(sizeof(pGVM->iom.s) <= sizeof(pGVM->iom.padding));
+    AssertCompile(sizeof(pGVM->iomr0.s) <= sizeof(pGVM->iomr0.padding));
+
+    iomR0IoPortInitPerVMData(pGVM);
+    iomR0MmioInitPerVMData(pGVM);
+}
+
+
+/**
+ * Cleans up any loose ends before the GVM structure is destroyed.
+ */
+VMMR0_INT_DECL(void) IOMR0CleanupVM(PGVM pGVM)
+{
+    iomR0IoPortCleanupVM(pGVM);
+    iomR0MmioCleanupVM(pGVM);
+}
+
diff --git a/src/VBox/VMM/VMMR0/IOMR0IoPort.cpp b/src/VBox/VMM/VMMR0/IOMR0IoPort.cpp
new file mode 100644
index 00000000..9c4fa9a0
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/IOMR0IoPort.cpp
@@ -0,0 +1,382 @@
+/* $Id: IOMR0IoPort.cpp $ */
+/** @file
+ * IOM - Host Context Ring 0, I/O ports.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_IOM_IOPORT
+#include <VBox/vmm/iom.h>
+#include "IOMInternal.h"
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/mem.h>
+#include <iprt/memobj.h>
+#include <iprt/process.h>
+#include <iprt/string.h>
+
+
+
+/**
+ * Initializes the I/O port related members.
+ *
+ * @param   pGVM    Pointer to the global VM structure.
+ */
+void iomR0IoPortInitPerVMData(PGVM pGVM)
+{
+    pGVM->iomr0.s.hIoPortMapObj      = NIL_RTR0MEMOBJ;
+    pGVM->iomr0.s.hIoPortMemObj      = NIL_RTR0MEMOBJ;
+#ifdef VBOX_WITH_STATISTICS
+    pGVM->iomr0.s.hIoPortStatsMapObj = NIL_RTR0MEMOBJ;
+    pGVM->iomr0.s.hIoPortStatsMemObj = NIL_RTR0MEMOBJ;
+#endif
+}
+
+
+/**
+ * Cleans up I/O port related resources.
+ */
+void iomR0IoPortCleanupVM(PGVM pGVM)
+{
+    RTR0MemObjFree(pGVM->iomr0.s.hIoPortMapObj, true /*fFreeMappings*/);
+    pGVM->iomr0.s.hIoPortMapObj      = NIL_RTR0MEMOBJ;
+    RTR0MemObjFree(pGVM->iomr0.s.hIoPortMemObj, true /*fFreeMappings*/);
+    pGVM->iomr0.s.hIoPortMemObj      = NIL_RTR0MEMOBJ;
+#ifdef VBOX_WITH_STATISTICS
+    RTR0MemObjFree(pGVM->iomr0.s.hIoPortStatsMapObj, true /*fFreeMappings*/);
+    pGVM->iomr0.s.hIoPortStatsMapObj = NIL_RTR0MEMOBJ;
+    RTR0MemObjFree(pGVM->iomr0.s.hIoPortStatsMemObj, true /*fFreeMappings*/);
+    pGVM->iomr0.s.hIoPortStatsMemObj = NIL_RTR0MEMOBJ;
+#endif
+}
+
+
+/**
+ * Implements PDMDEVHLPR0::pfnIoPortSetUpContext.
+ *
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   pDevIns         The device instance.
+ * @param   hIoPorts        The I/O port handle (already registered in ring-3).
+ * @param   pfnOut          The OUT handler callback, optional.
+ * @param   pfnIn           The IN handler callback, optional.
+ * @param   pfnOutStr       The REP OUTS handler callback, optional.
+ * @param   pfnInStr        The REP INS handler callback, optional.
+ * @param   pvUser          User argument for the callbacks.
+ * @thread  EMT(0)
+ * @note    Only callable at VM creation time.
+ */
+VMMR0_INT_DECL(int)  IOMR0IoPortSetUpContext(PGVM pGVM, PPDMDEVINS pDevIns, IOMIOPORTHANDLE hIoPorts,
+                                             PFNIOMIOPORTNEWOUT pfnOut,  PFNIOMIOPORTNEWIN pfnIn,
+                                             PFNIOMIOPORTNEWOUTSTRING pfnOutStr, PFNIOMIOPORTNEWINSTRING pfnInStr, void *pvUser)
+{
+    /*
+     * Validate input and state.
+     */
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+    AssertReturn(hIoPorts < pGVM->iomr0.s.cIoPortAlloc, VERR_IOM_INVALID_IOPORT_HANDLE);
+    AssertReturn(hIoPorts < pGVM->iom.s.cIoPortRegs, VERR_IOM_INVALID_IOPORT_HANDLE);
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    AssertReturn(pDevIns->pDevInsForR3 != NIL_RTR3PTR && !(pDevIns->pDevInsForR3 & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(pGVM->iomr0.s.paIoPortRing3Regs[hIoPorts].pDevIns == pDevIns->pDevInsForR3, VERR_IOM_INVALID_IOPORT_HANDLE);
+    AssertReturn(pGVM->iomr0.s.paIoPortRegs[hIoPorts].pDevIns == NULL, VERR_WRONG_ORDER);
+    Assert(pGVM->iomr0.s.paIoPortRegs[hIoPorts].idxSelf == hIoPorts);
+
+    AssertReturn(pfnOut || pfnIn || pfnOutStr || pfnInStr, VERR_INVALID_PARAMETER);
+    AssertPtrNullReturn(pfnOut, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnIn, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnOutStr, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnInStr, VERR_INVALID_POINTER);
+
+    uint16_t const fFlags = pGVM->iomr0.s.paIoPortRing3Regs[hIoPorts].fFlags;
+    RTIOPORT const cPorts = pGVM->iomr0.s.paIoPortRing3Regs[hIoPorts].cPorts;
+    AssertMsgReturn(cPorts > 0 && cPorts <= _8K, ("cPorts=%s\n", cPorts), VERR_IOM_INVALID_IOPORT_HANDLE);
+
+    /*
+     * Do the job.
+     */
+    pGVM->iomr0.s.paIoPortRegs[hIoPorts].pvUser             = pvUser;
+    pGVM->iomr0.s.paIoPortRegs[hIoPorts].pDevIns            = pDevIns;
+    pGVM->iomr0.s.paIoPortRegs[hIoPorts].pfnOutCallback     = pfnOut;
+    pGVM->iomr0.s.paIoPortRegs[hIoPorts].pfnInCallback      = pfnIn;
+    pGVM->iomr0.s.paIoPortRegs[hIoPorts].pfnOutStrCallback  = pfnOutStr;
+    pGVM->iomr0.s.paIoPortRegs[hIoPorts].pfnInStrCallback   = pfnInStr;
+    pGVM->iomr0.s.paIoPortRegs[hIoPorts].cPorts             = cPorts;
+    pGVM->iomr0.s.paIoPortRegs[hIoPorts].fFlags             = fFlags;
+#ifdef VBOX_WITH_STATISTICS
+    uint16_t const idxStats = pGVM->iomr0.s.paIoPortRing3Regs[hIoPorts].idxStats;
+    pGVM->iomr0.s.paIoPortRegs[hIoPorts].idxStats           = (uint32_t)idxStats + cPorts <= pGVM->iomr0.s.cIoPortStatsAllocation
+                                                            ? idxStats : UINT16_MAX;
+#else
+    pGVM->iomr0.s.paIoPortRegs[hIoPorts].idxStats           = UINT16_MAX;
+#endif
+
+    pGVM->iomr0.s.paIoPortRing3Regs[hIoPorts].fRing0 = true;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Grows the I/O port registration (all contexts) and lookup tables.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   cReqMinEntries  The minimum growth (absolute).
+ * @thread  EMT(0)
+ * @note    Only callable at VM creation time.
+ */
+VMMR0_INT_DECL(int) IOMR0IoPortGrowRegistrationTables(PGVM pGVM, uint64_t cReqMinEntries)
+{
+    /*
+     * Validate input and state.
+     */
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+    AssertReturn(cReqMinEntries <= _4K, VERR_IOM_TOO_MANY_IOPORT_REGISTRATIONS);
+    uint32_t cNewEntries = (uint32_t)cReqMinEntries;
+    AssertReturn(cNewEntries >= pGVM->iom.s.cIoPortAlloc, VERR_IOM_IOPORT_IPE_1);
+    uint32_t const cOldEntries = pGVM->iomr0.s.cIoPortAlloc;
+    ASMCompilerBarrier();
+    AssertReturn(cNewEntries >= cOldEntries, VERR_IOM_IOPORT_IPE_2);
+    AssertReturn(pGVM->iom.s.cIoPortRegs >= pGVM->iomr0.s.cIoPortMax, VERR_IOM_IOPORT_IPE_3);
+
+    /*
+     * Allocate the new tables.  We use a single allocation for the three tables (ring-0,
+     * ring-3, lookup) and does a partial mapping of the result to ring-3.
+     */
+    uint32_t const cbRing0  = RT_ALIGN_32(cNewEntries * sizeof(IOMIOPORTENTRYR0),     PAGE_SIZE);
+    uint32_t const cbRing3  = RT_ALIGN_32(cNewEntries * sizeof(IOMIOPORTENTRYR3),     PAGE_SIZE);
+    uint32_t const cbShared = RT_ALIGN_32(cNewEntries * sizeof(IOMIOPORTLOOKUPENTRY), PAGE_SIZE);
+    uint32_t const cbNew    = cbRing0 + cbRing3 + cbShared;
+
+    /* Use the rounded up space as best we can. */
+    cNewEntries = RT_MIN(RT_MIN(cbRing0 / sizeof(IOMIOPORTENTRYR0), cbRing3 / sizeof(IOMIOPORTENTRYR3)),
+                         cbShared / sizeof(IOMIOPORTLOOKUPENTRY));
+
+    RTR0MEMOBJ hMemObj;
+    int rc = RTR0MemObjAllocPage(&hMemObj, cbNew, false /*fExecutable*/);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Zero and map it.
+         */
+        RT_BZERO(RTR0MemObjAddress(hMemObj), cbNew);
+
+        RTR0MEMOBJ hMapObj;
+        rc = RTR0MemObjMapUserEx(&hMapObj, hMemObj, (RTR3PTR)-1, PAGE_SIZE, RTMEM_PROT_READ | RTMEM_PROT_WRITE,
+                                 RTR0ProcHandleSelf(), cbRing0, cbNew - cbRing0);
+        if (RT_SUCCESS(rc))
+        {
+            PIOMIOPORTENTRYR0     const paRing0    = (PIOMIOPORTENTRYR0)RTR0MemObjAddress(hMemObj);
+            PIOMIOPORTENTRYR3     const paRing3    = (PIOMIOPORTENTRYR3)((uintptr_t)paRing0 + cbRing0);
+            PIOMIOPORTLOOKUPENTRY const paLookup   = (PIOMIOPORTLOOKUPENTRY)((uintptr_t)paRing3 + cbRing3);
+            RTR3UINTPTR           const uAddrRing3 = RTR0MemObjAddressR3(hMapObj);
+
+            /*
+             * Copy over the old info and initialize the idxSelf and idxStats members.
+             */
+            if (pGVM->iomr0.s.paIoPortRegs != NULL)
+            {
+                memcpy(paRing0,  pGVM->iomr0.s.paIoPortRegs,      sizeof(paRing0[0])  * cOldEntries);
+                memcpy(paRing3,  pGVM->iomr0.s.paIoPortRing3Regs, sizeof(paRing3[0])  * cOldEntries);
+                memcpy(paLookup, pGVM->iomr0.s.paIoPortLookup,    sizeof(paLookup[0]) * cOldEntries);
+            }
+
+            size_t i = cbRing0 / sizeof(*paRing0);
+            while (i-- > cOldEntries)
+            {
+                paRing0[i].idxSelf  = (uint16_t)i;
+                paRing0[i].idxStats = UINT16_MAX;
+            }
+            i = cbRing3 / sizeof(*paRing3);
+            while (i-- > cOldEntries)
+            {
+                paRing3[i].idxSelf  = (uint16_t)i;
+                paRing3[i].idxStats = UINT16_MAX;
+            }
+
+            /*
+             * Switch the memory handles.
+             */
+            RTR0MEMOBJ hTmp = pGVM->iomr0.s.hIoPortMapObj;
+            pGVM->iomr0.s.hIoPortMapObj = hMapObj;
+            hMapObj = hTmp;
+
+            hTmp = pGVM->iomr0.s.hIoPortMemObj;
+            pGVM->iomr0.s.hIoPortMemObj = hMemObj;
+            hMemObj = hTmp;
+
+            /*
+             * Update the variables.
+             */
+            pGVM->iomr0.s.paIoPortRegs      = paRing0;
+            pGVM->iomr0.s.paIoPortRing3Regs = paRing3;
+            pGVM->iomr0.s.paIoPortLookup    = paLookup;
+            pGVM->iom.s.paIoPortRegs        = uAddrRing3;
+            pGVM->iom.s.paIoPortLookup      = uAddrRing3 + cbRing3;
+            pGVM->iom.s.cIoPortAlloc        = cNewEntries;
+            pGVM->iomr0.s.cIoPortAlloc      = cNewEntries;
+
+            /*
+             * Free the old allocation.
+             */
+            RTR0MemObjFree(hMapObj, true /*fFreeMappings*/);
+        }
+        RTR0MemObjFree(hMemObj, true /*fFreeMappings*/);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Grows the I/O port statistics table.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   cReqMinEntries  The minimum growth (absolute).
+ * @thread  EMT(0)
+ * @note    Only callable at VM creation time.
+ */
+VMMR0_INT_DECL(int) IOMR0IoPortGrowStatisticsTable(PGVM pGVM, uint64_t cReqMinEntries)
+{
+    /*
+     * Validate input and state.
+     */
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+    AssertReturn(cReqMinEntries <= _64K, VERR_IOM_TOO_MANY_IOPORT_REGISTRATIONS);
+    uint32_t cNewEntries = (uint32_t)cReqMinEntries;
+#ifdef VBOX_WITH_STATISTICS
+    uint32_t const cOldEntries = pGVM->iomr0.s.cIoPortStatsAllocation;
+    ASMCompilerBarrier();
+#else
+    uint32_t const cOldEntries = 0;
+#endif
+    AssertReturn(cNewEntries > cOldEntries, VERR_IOM_IOPORT_IPE_1);
+    AssertReturn(pGVM->iom.s.cIoPortStatsAllocation == cOldEntries, VERR_IOM_IOPORT_IPE_1);
+    AssertReturn(pGVM->iom.s.cIoPortStats <= cOldEntries, VERR_IOM_IOPORT_IPE_2);
+#ifdef VBOX_WITH_STATISTICS
+    AssertReturn(!pGVM->iomr0.s.fIoPortStatsFrozen, VERR_WRONG_ORDER);
+#endif
+
+    /*
+     * Allocate a new table, zero it and map it.
+     */
+#ifndef VBOX_WITH_STATISTICS
+    AssertFailedReturn(VERR_NOT_SUPPORTED);
+#else
+    uint32_t const cbNew = RT_ALIGN_32(cNewEntries * sizeof(IOMIOPORTSTATSENTRY), PAGE_SIZE);
+    cNewEntries = cbNew / sizeof(IOMIOPORTSTATSENTRY);
+
+    RTR0MEMOBJ hMemObj;
+    int rc = RTR0MemObjAllocPage(&hMemObj, cbNew, false /*fExecutable*/);
+    if (RT_SUCCESS(rc))
+    {
+        RT_BZERO(RTR0MemObjAddress(hMemObj), cbNew);
+
+        RTR0MEMOBJ hMapObj;
+        rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, PAGE_SIZE, RTMEM_PROT_READ | RTMEM_PROT_WRITE, RTR0ProcHandleSelf());
+        if (RT_SUCCESS(rc))
+        {
+            PIOMIOPORTSTATSENTRY pIoPortStats = (PIOMIOPORTSTATSENTRY)RTR0MemObjAddress(hMemObj);
+
+            /*
+             * Anything to copy over and free up?
+             */
+            if (pGVM->iomr0.s.paIoPortStats)
+                memcpy(pIoPortStats, pGVM->iomr0.s.paIoPortStats, cOldEntries * sizeof(IOMIOPORTSTATSENTRY));
+
+            /*
+             * Switch the memory handles.
+             */
+            RTR0MEMOBJ hTmp = pGVM->iomr0.s.hIoPortStatsMapObj;
+            pGVM->iomr0.s.hIoPortStatsMapObj = hMapObj;
+            hMapObj = hTmp;
+
+            hTmp = pGVM->iomr0.s.hIoPortStatsMemObj;
+            pGVM->iomr0.s.hIoPortStatsMemObj = hMemObj;
+            hMemObj = hTmp;
+
+            /*
+             * Update the variables.
+             */
+            pGVM->iomr0.s.paIoPortStats          = pIoPortStats;
+            pGVM->iom.s.paIoPortStats            = RTR0MemObjAddressR3(pGVM->iomr0.s.hIoPortStatsMapObj);
+            pGVM->iom.s.cIoPortStatsAllocation   = cNewEntries;
+            pGVM->iomr0.s.cIoPortStatsAllocation = cNewEntries;
+
+            /*
+             * Free the old allocation.
+             */
+            RTR0MemObjFree(hMapObj, true /*fFreeMappings*/);
+        }
+        RTR0MemObjFree(hMemObj, true /*fFreeMappings*/);
+    }
+    return rc;
+#endif /* VBOX_WITH_STATISTICS */
+}
+
+/**
+ * Called after all devices has been instantiated to copy over the statistics
+ * indices to the ring-0 I/O port registration table.
+ *
+ * This simplifies keeping statistics for I/O port ranges that are ring-3 only.
+ *
+ * After this call, IOMR0IoPortGrowStatisticsTable() will stop working.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @thread  EMT(0)
+ * @note    Only callable at VM creation time.
+ */
+VMMR0_INT_DECL(int) IOMR0IoPortSyncStatisticsIndices(PGVM pGVM)
+{
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * First, freeze the statistics array:
+     */
+    pGVM->iomr0.s.fIoPortStatsFrozen = true;
+
+    /*
+     * Second, synchronize the indices:
+     */
+    uint32_t const          cRegs        = RT_MIN(pGVM->iom.s.cIoPortRegs, pGVM->iomr0.s.cIoPortAlloc);
+    uint32_t const          cStatsAlloc  = pGVM->iomr0.s.cIoPortStatsAllocation;
+    PIOMIOPORTENTRYR0       paIoPortRegs   = pGVM->iomr0.s.paIoPortRegs;
+    IOMIOPORTENTRYR3 const *paIoPortRegsR3 = pGVM->iomr0.s.paIoPortRing3Regs;
+    AssertReturn((paIoPortRegs && paIoPortRegsR3) || cRegs == 0, VERR_IOM_IOPORT_IPE_3);
+
+    for (uint32_t i = 0 ; i < cRegs; i++)
+    {
+        uint16_t idxStats = paIoPortRegsR3[i].idxStats;
+        paIoPortRegs[i].idxStats = idxStats < cStatsAlloc ? idxStats : UINT16_MAX;
+    }
+
+#else
+    RT_NOREF(pGVM);
+#endif
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMR0/IOMR0Mmio.cpp b/src/VBox/VMM/VMMR0/IOMR0Mmio.cpp
new file mode 100644
index 00000000..45766568
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/IOMR0Mmio.cpp
@@ -0,0 +1,378 @@
+/* $Id: IOMR0Mmio.cpp $ */
+/** @file
+ * IOM - Host Context Ring 0, MMIO.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_IOM_MMIO
+#include <VBox/vmm/iom.h>
+#include "IOMInternal.h"
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/mem.h>
+#include <iprt/memobj.h>
+#include <iprt/process.h>
+#include <iprt/string.h>
+
+
+
+/**
+ * Initializes the MMIO related members.
+ *
+ * @param   pGVM    Pointer to the global VM structure.
+ */
+void iomR0MmioInitPerVMData(PGVM pGVM)
+{
+    pGVM->iomr0.s.hMmioMapObj      = NIL_RTR0MEMOBJ;
+    pGVM->iomr0.s.hMmioMemObj      = NIL_RTR0MEMOBJ;
+#ifdef VBOX_WITH_STATISTICS
+    pGVM->iomr0.s.hMmioStatsMapObj = NIL_RTR0MEMOBJ;
+    pGVM->iomr0.s.hMmioStatsMemObj = NIL_RTR0MEMOBJ;
+#endif
+}
+
+
+/**
+ * Cleans up MMIO related resources.
+ */
+void iomR0MmioCleanupVM(PGVM pGVM)
+{
+    RTR0MemObjFree(pGVM->iomr0.s.hMmioMapObj, true /*fFreeMappings*/);
+    pGVM->iomr0.s.hMmioMapObj      = NIL_RTR0MEMOBJ;
+    RTR0MemObjFree(pGVM->iomr0.s.hMmioMemObj, true /*fFreeMappings*/);
+    pGVM->iomr0.s.hMmioMemObj      = NIL_RTR0MEMOBJ;
+#ifdef VBOX_WITH_STATISTICS
+    RTR0MemObjFree(pGVM->iomr0.s.hMmioStatsMapObj, true /*fFreeMappings*/);
+    pGVM->iomr0.s.hMmioStatsMapObj = NIL_RTR0MEMOBJ;
+    RTR0MemObjFree(pGVM->iomr0.s.hMmioStatsMemObj, true /*fFreeMappings*/);
+    pGVM->iomr0.s.hMmioStatsMemObj = NIL_RTR0MEMOBJ;
+#endif
+}
+
+
+/**
+ * Implements PDMDEVHLPR0::pfnMmioSetUpContext.
+ *
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   pDevIns         The device instance.
+ * @param   hRegion         The MMIO region handle (already registered in
+ *                          ring-3).
+ * @param   pfnWrite        The write handler callback, optional.
+ * @param   pfnRead         The read handler callback, optional.
+ * @param   pfnFill         The fill handler callback, optional.
+ * @param   pvUser          User argument for the callbacks.
+ * @thread  EMT(0)
+ * @note    Only callable at VM creation time.
+ */
+VMMR0_INT_DECL(int)  IOMR0MmioSetUpContext(PGVM pGVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, PFNIOMMMIONEWWRITE pfnWrite,
+                                           PFNIOMMMIONEWREAD pfnRead, PFNIOMMMIONEWFILL pfnFill, void *pvUser)
+{
+    /*
+     * Validate input and state.
+     */
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+    AssertReturn(hRegion < pGVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(hRegion < pGVM->iom.s.cMmioRegs, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    AssertReturn(pDevIns->pDevInsForR3 != NIL_RTR3PTR && !(pDevIns->pDevInsForR3 & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(pGVM->iomr0.s.paMmioRing3Regs[hRegion].pDevIns == pDevIns->pDevInsForR3, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(pGVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL, VERR_WRONG_ORDER);
+    Assert(pGVM->iomr0.s.paMmioRegs[hRegion].idxSelf == hRegion);
+
+    AssertReturn(pfnWrite || pfnRead || pfnFill, VERR_INVALID_PARAMETER);
+    AssertPtrNullReturn(pfnWrite, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnRead, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnFill, VERR_INVALID_POINTER);
+
+    uint32_t const fFlags   = pGVM->iomr0.s.paMmioRing3Regs[hRegion].fFlags;
+    RTGCPHYS const cbRegion = pGVM->iomr0.s.paMmioRing3Regs[hRegion].cbRegion;
+    AssertMsgReturn(cbRegion > 0 && cbRegion <= _1T, ("cbRegion=%#RGp\n", cbRegion), VERR_IOM_INVALID_MMIO_HANDLE);
+
+    /*
+     * Do the job.
+     */
+    pGVM->iomr0.s.paMmioRegs[hRegion].cbRegion          = cbRegion;
+    pGVM->iomr0.s.paMmioRegs[hRegion].pvUser            = pvUser;
+    pGVM->iomr0.s.paMmioRegs[hRegion].pDevIns           = pDevIns;
+    pGVM->iomr0.s.paMmioRegs[hRegion].pfnWriteCallback  = pfnWrite;
+    pGVM->iomr0.s.paMmioRegs[hRegion].pfnReadCallback   = pfnRead;
+    pGVM->iomr0.s.paMmioRegs[hRegion].pfnFillCallback   = pfnFill;
+    pGVM->iomr0.s.paMmioRegs[hRegion].fFlags            = fFlags;
+#ifdef VBOX_WITH_STATISTICS
+    uint16_t const idxStats = pGVM->iomr0.s.paMmioRing3Regs[hRegion].idxStats;
+    pGVM->iomr0.s.paMmioRegs[hRegion].idxStats          = (uint32_t)idxStats < pGVM->iomr0.s.cMmioStatsAllocation
+                                                        ? idxStats : UINT16_MAX;
+#else
+    pGVM->iomr0.s.paMmioRegs[hRegion].idxStats          = UINT16_MAX;
+#endif
+
+    pGVM->iomr0.s.paMmioRing3Regs[hRegion].fRing0 = true;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Grows the MMIO registration (all contexts) and lookup tables.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   cReqMinEntries  The minimum growth (absolute).
+ * @thread  EMT(0)
+ * @note    Only callable at VM creation time.
+ */
+VMMR0_INT_DECL(int) IOMR0MmioGrowRegistrationTables(PGVM pGVM, uint64_t cReqMinEntries)
+{
+    /*
+     * Validate input and state.
+     */
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+    AssertReturn(cReqMinEntries <= _4K, VERR_IOM_TOO_MANY_MMIO_REGISTRATIONS);
+    uint32_t cNewEntries = (uint32_t)cReqMinEntries;
+    AssertReturn(cNewEntries >= pGVM->iom.s.cMmioAlloc, VERR_IOM_MMIO_IPE_1);
+    uint32_t const cOldEntries = pGVM->iomr0.s.cMmioAlloc;
+    ASMCompilerBarrier();
+    AssertReturn(cNewEntries >= cOldEntries, VERR_IOM_MMIO_IPE_2);
+    AssertReturn(pGVM->iom.s.cMmioRegs >= pGVM->iomr0.s.cMmioMax, VERR_IOM_MMIO_IPE_3);
+
+    /*
+     * Allocate the new tables.  We use a single allocation for the three tables (ring-0,
+     * ring-3, lookup) and does a partial mapping of the result to ring-3.
+     */
+    uint32_t const cbRing0  = RT_ALIGN_32(cNewEntries * sizeof(IOMMMIOENTRYR0),     PAGE_SIZE);
+    uint32_t const cbRing3  = RT_ALIGN_32(cNewEntries * sizeof(IOMMMIOENTRYR3),     PAGE_SIZE);
+    uint32_t const cbShared = RT_ALIGN_32(cNewEntries * sizeof(IOMMMIOLOOKUPENTRY), PAGE_SIZE);
+    uint32_t const cbNew    = cbRing0 + cbRing3 + cbShared;
+
+    /* Use the rounded up space as best we can. */
+    cNewEntries = RT_MIN(RT_MIN(cbRing0 / sizeof(IOMMMIOENTRYR0), cbRing3 / sizeof(IOMMMIOENTRYR3)),
+                         cbShared / sizeof(IOMMMIOLOOKUPENTRY));
+
+    RTR0MEMOBJ hMemObj;
+    int rc = RTR0MemObjAllocPage(&hMemObj, cbNew, false /*fExecutable*/);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Zero and map it.
+         */
+        RT_BZERO(RTR0MemObjAddress(hMemObj), cbNew);
+
+        RTR0MEMOBJ hMapObj;
+        rc = RTR0MemObjMapUserEx(&hMapObj, hMemObj, (RTR3PTR)-1, PAGE_SIZE, RTMEM_PROT_READ | RTMEM_PROT_WRITE,
+                                 RTR0ProcHandleSelf(), cbRing0, cbNew - cbRing0);
+        if (RT_SUCCESS(rc))
+        {
+            PIOMMMIOENTRYR0       const paRing0    = (PIOMMMIOENTRYR0)RTR0MemObjAddress(hMemObj);
+            PIOMMMIOENTRYR3       const paRing3    = (PIOMMMIOENTRYR3)((uintptr_t)paRing0 + cbRing0);
+            PIOMMMIOLOOKUPENTRY   const paLookup   = (PIOMMMIOLOOKUPENTRY)((uintptr_t)paRing3 + cbRing3);
+            RTR3UINTPTR           const uAddrRing3 = RTR0MemObjAddressR3(hMapObj);
+
+            /*
+             * Copy over the old info and initialize the idxSelf and idxStats members.
+             */
+            if (pGVM->iomr0.s.paMmioRegs != NULL)
+            {
+                memcpy(paRing0,  pGVM->iomr0.s.paMmioRegs,      sizeof(paRing0[0])  * cOldEntries);
+                memcpy(paRing3,  pGVM->iomr0.s.paMmioRing3Regs, sizeof(paRing3[0])  * cOldEntries);
+                memcpy(paLookup, pGVM->iomr0.s.paMmioLookup,    sizeof(paLookup[0]) * cOldEntries);
+            }
+
+            size_t i = cbRing0 / sizeof(*paRing0);
+            while (i-- > cOldEntries)
+            {
+                paRing0[i].idxSelf  = (uint16_t)i;
+                paRing0[i].idxStats = UINT16_MAX;
+            }
+            i = cbRing3 / sizeof(*paRing3);
+            while (i-- > cOldEntries)
+            {
+                paRing3[i].idxSelf  = (uint16_t)i;
+                paRing3[i].idxStats = UINT16_MAX;
+            }
+
+            /*
+             * Switch the memory handles.
+             */
+            RTR0MEMOBJ hTmp = pGVM->iomr0.s.hMmioMapObj;
+            pGVM->iomr0.s.hMmioMapObj = hMapObj;
+            hMapObj = hTmp;
+
+            hTmp = pGVM->iomr0.s.hMmioMemObj;
+            pGVM->iomr0.s.hMmioMemObj = hMemObj;
+            hMemObj = hTmp;
+
+            /*
+             * Update the variables.
+             */
+            pGVM->iomr0.s.paMmioRegs      = paRing0;
+            pGVM->iomr0.s.paMmioRing3Regs = paRing3;
+            pGVM->iomr0.s.paMmioLookup    = paLookup;
+            pGVM->iom.s.paMmioRegs        = uAddrRing3;
+            pGVM->iom.s.paMmioLookup      = uAddrRing3 + cbRing3;
+            pGVM->iom.s.cMmioAlloc        = cNewEntries;
+            pGVM->iomr0.s.cMmioAlloc      = cNewEntries;
+
+            /*
+             * Free the old allocation.
+             */
+            RTR0MemObjFree(hMapObj, true /*fFreeMappings*/);
+        }
+        RTR0MemObjFree(hMemObj, true /*fFreeMappings*/);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Grows the MMIO statistics table.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   cReqMinEntries  The minimum growth (absolute).
+ * @thread  EMT(0)
+ * @note    Only callable at VM creation time.
+ */
+VMMR0_INT_DECL(int) IOMR0MmioGrowStatisticsTable(PGVM pGVM, uint64_t cReqMinEntries)
+{
+    /*
+     * Validate input and state.
+     */
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+    AssertReturn(cReqMinEntries <= _64K, VERR_IOM_TOO_MANY_MMIO_REGISTRATIONS);
+    uint32_t cNewEntries = (uint32_t)cReqMinEntries;
+#ifdef VBOX_WITH_STATISTICS
+    uint32_t const cOldEntries = pGVM->iomr0.s.cMmioStatsAllocation;
+    ASMCompilerBarrier();
+#else
+    uint32_t const cOldEntries = 0;
+#endif
+    AssertReturn(cNewEntries > cOldEntries, VERR_IOM_MMIO_IPE_1);
+    AssertReturn(pGVM->iom.s.cMmioStatsAllocation == cOldEntries, VERR_IOM_MMIO_IPE_1);
+    AssertReturn(pGVM->iom.s.cMmioStats <= cOldEntries, VERR_IOM_MMIO_IPE_2);
+#ifdef VBOX_WITH_STATISTICS
+    AssertReturn(!pGVM->iomr0.s.fMmioStatsFrozen, VERR_WRONG_ORDER);
+#endif
+
+    /*
+     * Allocate a new table, zero it and map it.
+     */
+#ifndef VBOX_WITH_STATISTICS
+    AssertFailedReturn(VERR_NOT_SUPPORTED);
+#else
+    uint32_t const cbNew = RT_ALIGN_32(cNewEntries * sizeof(IOMMMIOSTATSENTRY), PAGE_SIZE);
+    cNewEntries = cbNew / sizeof(IOMMMIOSTATSENTRY);
+
+    RTR0MEMOBJ hMemObj;
+    int rc = RTR0MemObjAllocPage(&hMemObj, cbNew, false /*fExecutable*/);
+    if (RT_SUCCESS(rc))
+    {
+        RT_BZERO(RTR0MemObjAddress(hMemObj), cbNew);
+
+        RTR0MEMOBJ hMapObj;
+        rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, PAGE_SIZE, RTMEM_PROT_READ | RTMEM_PROT_WRITE, RTR0ProcHandleSelf());
+        if (RT_SUCCESS(rc))
+        {
+            PIOMMMIOSTATSENTRY pMmioStats = (PIOMMMIOSTATSENTRY)RTR0MemObjAddress(hMemObj);
+
+            /*
+             * Anything to copy over and free up?
+             */
+            if (pGVM->iomr0.s.paMmioStats)
+                memcpy(pMmioStats, pGVM->iomr0.s.paMmioStats, cOldEntries * sizeof(IOMMMIOSTATSENTRY));
+
+            /*
+             * Switch the memory handles.
+             */
+            RTR0MEMOBJ hTmp = pGVM->iomr0.s.hMmioStatsMapObj;
+            pGVM->iomr0.s.hMmioStatsMapObj = hMapObj;
+            hMapObj = hTmp;
+
+            hTmp = pGVM->iomr0.s.hMmioStatsMemObj;
+            pGVM->iomr0.s.hMmioStatsMemObj = hMemObj;
+            hMemObj = hTmp;
+
+            /*
+             * Update the variables.
+             */
+            pGVM->iomr0.s.paMmioStats          = pMmioStats;
+            pGVM->iom.s.paMmioStats            = RTR0MemObjAddressR3(pGVM->iomr0.s.hMmioStatsMapObj);
+            pGVM->iom.s.cMmioStatsAllocation   = cNewEntries;
+            pGVM->iomr0.s.cMmioStatsAllocation = cNewEntries;
+
+            /*
+             * Free the old allocation.
+             */
+            RTR0MemObjFree(hMapObj, true /*fFreeMappings*/);
+        }
+        RTR0MemObjFree(hMemObj, true /*fFreeMappings*/);
+    }
+    return rc;
+#endif /* VBOX_WITH_STATISTICS */
+}
+
+
+/**
+ * Called after all devices has been instantiated to copy over the statistics
+ * indices to the ring-0 MMIO registration table.
+ *
+ * This simplifies keeping statistics for MMIO ranges that are ring-3 only.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @thread  EMT(0)
+ * @note    Only callable at VM creation time.
+ */
+VMMR0_INT_DECL(int) IOMR0MmioSyncStatisticsIndices(PGVM pGVM)
+{
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * First, freeze the statistics array:
+     */
+    pGVM->iomr0.s.fMmioStatsFrozen = true;
+
+    /*
+     * Second, synchronize the indices:
+     */
+    uint32_t const          cRegs        = RT_MIN(pGVM->iom.s.cMmioRegs, pGVM->iomr0.s.cMmioAlloc);
+    uint32_t const          cStatsAlloc  = pGVM->iomr0.s.cMmioStatsAllocation;
+    PIOMMMIOENTRYR0         paMmioRegs   = pGVM->iomr0.s.paMmioRegs;
+    IOMMMIOENTRYR3 const   *paMmioRegsR3 = pGVM->iomr0.s.paMmioRing3Regs;
+    AssertReturn((paMmioRegs && paMmioRegsR3) || cRegs == 0, VERR_IOM_MMIO_IPE_3);
+
+    for (uint32_t i = 0 ; i < cRegs; i++)
+    {
+        uint16_t idxStats = paMmioRegsR3[i].idxStats;
+        paMmioRegs[i].idxStats = idxStats < cStatsAlloc ? idxStats : UINT16_MAX;
+    }
+
+#else
+    RT_NOREF(pGVM);
+#endif
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMR0/Makefile.kup b/src/VBox/VMM/VMMR0/Makefile.kup
new file mode 100644
index 00000000..e69de29b
diff --git a/src/VBox/VMM/VMMR0/NEMR0Native-win.cpp b/src/VBox/VMM/VMMR0/NEMR0Native-win.cpp
new file mode 100644
index 00000000..2a858f15
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/NEMR0Native-win.cpp
@@ -0,0 +1,2616 @@
+/* $Id: NEMR0Native-win.cpp $ */
+/** @file
+ * NEM - Native execution manager, native ring-0 Windows backend.
+ */
+
+/*
+ * Copyright (C) 2018-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_NEM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <iprt/nt/nt.h>
+#include <iprt/nt/hyperv.h>
+#include <iprt/nt/vid.h>
+#include <winerror.h>
+
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/dbgftrace.h>
+#include "NEMInternal.h"
+#include <VBox/vmm/gvm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/gvmm.h>
+#include <VBox/param.h>
+
+#include <iprt/dbg.h>
+#include <iprt/memobj.h>
+#include <iprt/string.h>
+#include <iprt/time.h>
+
+
+/* Assert compile context sanity. */
+#ifndef RT_OS_WINDOWS
+# error "Windows only file!"
+#endif
+#ifndef RT_ARCH_AMD64
+# error "AMD64 only file!"
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+typedef uint32_t DWORD; /* for winerror.h constants */
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+static uint64_t (*g_pfnHvlInvokeHypercall)(uint64_t uCallInfo, uint64_t HCPhysInput, uint64_t HCPhysOutput);
+
+/**
+ * WinHvr.sys!WinHvDepositMemory
+ *
+ * This API will try allocates cPages on IdealNode and deposit it to the
+ * hypervisor for use with the given partition.  The memory will be freed when
+ * VID.SYS calls WinHvWithdrawAllMemory when the partition is cleanedup.
+ *
+ * Apparently node numbers above 64 has a different meaning.
+ */
+static NTSTATUS (*g_pfnWinHvDepositMemory)(uintptr_t idPartition, size_t cPages, uintptr_t IdealNode, size_t *pcActuallyAdded);
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+NEM_TMPL_STATIC int  nemR0WinMapPages(PGVM pGVM, PGVMCPU pGVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
+                                      uint32_t cPages, uint32_t fFlags);
+NEM_TMPL_STATIC int  nemR0WinUnmapPages(PGVM pGVM, PGVMCPU pGVCpu, RTGCPHYS GCPhys, uint32_t cPages);
+#if defined(NEM_WIN_WITH_RING0_RUNLOOP) || defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
+NEM_TMPL_STATIC int  nemR0WinExportState(PGVM pGVM, PGVMCPU pGVCpu, PCPUMCTX pCtx);
+NEM_TMPL_STATIC int  nemR0WinImportState(PGVM pGVM, PGVMCPU pGVCpu, PCPUMCTX pCtx, uint64_t fWhat, bool fCanUpdateCr3);
+NEM_TMPL_STATIC int  nemR0WinQueryCpuTick(PGVM pGVM, PGVMCPU pGVCpu, uint64_t *pcTicks, uint32_t *pcAux);
+NEM_TMPL_STATIC int  nemR0WinResumeCpuTickOnAll(PGVM pGVM, PGVMCPU pGVCpu, uint64_t uPausedTscValue);
+#endif
+DECLINLINE(NTSTATUS) nemR0NtPerformIoControl(PGVM pGVM, PGVMCPU pGVCpu, uint32_t uFunction, void *pvInput, uint32_t cbInput,
+                                             void *pvOutput, uint32_t cbOutput);
+
+
+/*
+ * Instantate the code we share with ring-0.
+ */
+#ifdef NEM_WIN_WITH_RING0_RUNLOOP
+# define NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+#else
+# undef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+#endif
+#include "../VMMAll/NEMAllNativeTemplate-win.cpp.h"
+
+
+
+/**
+ * Worker for NEMR0InitVM that allocates a hypercall page.
+ *
+ * @returns VBox status code.
+ * @param   pHypercallData  The hypercall data page to initialize.
+ */
+static int nemR0InitHypercallData(PNEMR0HYPERCALLDATA pHypercallData)
+{
+    int rc = RTR0MemObjAllocPage(&pHypercallData->hMemObj, PAGE_SIZE, false /*fExecutable*/);
+    if (RT_SUCCESS(rc))
+    {
+        pHypercallData->HCPhysPage = RTR0MemObjGetPagePhysAddr(pHypercallData->hMemObj, 0 /*iPage*/);
+        AssertStmt(pHypercallData->HCPhysPage != NIL_RTHCPHYS, rc = VERR_INTERNAL_ERROR_3);
+        pHypercallData->pbPage     = (uint8_t *)RTR0MemObjAddress(pHypercallData->hMemObj);
+        AssertStmt(pHypercallData->pbPage, rc = VERR_INTERNAL_ERROR_3);
+        if (RT_SUCCESS(rc))
+            return VINF_SUCCESS;
+
+        /* bail out */
+        RTR0MemObjFree(pHypercallData->hMemObj, true /*fFreeMappings*/);
+    }
+    pHypercallData->hMemObj     = NIL_RTR0MEMOBJ;
+    pHypercallData->HCPhysPage  = NIL_RTHCPHYS;
+    pHypercallData->pbPage      = NULL;
+    return rc;
+}
+
+/**
+ * Worker for NEMR0CleanupVM and NEMR0InitVM that cleans up a hypercall page.
+ *
+ * @param   pHypercallData  The hypercall data page to uninitialize.
+ */
+static void nemR0DeleteHypercallData(PNEMR0HYPERCALLDATA pHypercallData)
+{
+    /* Check pbPage here since it's NULL, whereas the hMemObj can be either
+       NIL_RTR0MEMOBJ or 0 (they aren't necessarily the same). */
+    if (pHypercallData->pbPage != NULL)
+    {
+        RTR0MemObjFree(pHypercallData->hMemObj, true /*fFreeMappings*/);
+        pHypercallData->pbPage = NULL;
+    }
+    pHypercallData->hMemObj    = NIL_RTR0MEMOBJ;
+    pHypercallData->HCPhysPage = NIL_RTHCPHYS;
+}
+
+
+/**
+ * Called by NEMR3Init to make sure we've got what we need.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The ring-0 VM handle.
+ * @thread  EMT(0)
+ */
+VMMR0_INT_DECL(int) NEMR0InitVM(PGVM pGVM)
+{
+    AssertCompile(sizeof(pGVM->nemr0.s) <= sizeof(pGVM->nemr0.padding));
+    AssertCompile(sizeof(pGVM->aCpus[0].nemr0.s) <= sizeof(pGVM->aCpus[0].nemr0.padding));
+
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, 0);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * We want to perform hypercalls here.  The NT kernel started to expose a very low
+     * level interface to do this thru somewhere between build 14271 and 16299.  Since
+     * we need build 17134 to get anywhere at all, the exact build is not relevant here.
+     *
+     * We also need to deposit memory to the hypervisor for use with partition (page
+     * mapping structures, stuff).
+     */
+    RTDBGKRNLINFO hKrnlInfo;
+    rc = RTR0DbgKrnlInfoOpen(&hKrnlInfo, 0);
+    if (RT_SUCCESS(rc))
+    {
+        rc = RTR0DbgKrnlInfoQuerySymbol(hKrnlInfo, NULL, "HvlInvokeHypercall", (void **)&g_pfnHvlInvokeHypercall);
+        if (RT_FAILURE(rc))
+            rc = VERR_NEM_MISSING_KERNEL_API_1;
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTR0DbgKrnlInfoQuerySymbol(hKrnlInfo, "winhvr.sys", "WinHvDepositMemory", (void **)&g_pfnWinHvDepositMemory);
+            if (RT_FAILURE(rc))
+                rc = rc == VERR_MODULE_NOT_FOUND ? VERR_NEM_MISSING_KERNEL_API_2 : VERR_NEM_MISSING_KERNEL_API_3;
+        }
+        RTR0DbgKrnlInfoRelease(hKrnlInfo);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Allocate a page for non-EMT threads to use for hypercalls (update
+             * statistics and such) and a critical section protecting it.
+             */
+            rc = RTCritSectInit(&pGVM->nemr0.s.HypercallDataCritSect);
+            if (RT_SUCCESS(rc))
+            {
+                rc = nemR0InitHypercallData(&pGVM->nemr0.s.HypercallData);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Allocate a page for each VCPU to place hypercall data on.
+                     */
+                    for (VMCPUID i = 0; i < pGVM->cCpus; i++)
+                    {
+                        rc = nemR0InitHypercallData(&pGVM->aCpus[i].nemr0.s.HypercallData);
+                        if (RT_FAILURE(rc))
+                        {
+                            while (i-- > 0)
+                                nemR0DeleteHypercallData(&pGVM->aCpus[i].nemr0.s.HypercallData);
+                            break;
+                        }
+                    }
+                    if (RT_SUCCESS(rc))
+                    {
+                        /*
+                         * So far, so good.
+                         */
+                        return rc;
+                    }
+
+                    /*
+                     * Bail out.
+                     */
+                    nemR0DeleteHypercallData(&pGVM->nemr0.s.HypercallData);
+                }
+                RTCritSectDelete(&pGVM->nemr0.s.HypercallDataCritSect);
+            }
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Perform an I/O control operation on the partition handle (VID.SYS).
+ *
+ * @returns NT status code.
+ * @param   pGVM            The ring-0 VM structure.
+ * @param   pGVCpu          The global (ring-0) CPU structure of the calling EMT.
+ * @param   uFunction       The function to perform.
+ * @param   pvInput         The input buffer.  This must point within the VM
+ *                          structure so we can easily convert to a ring-3
+ *                          pointer if necessary.
+ * @param   cbInput         The size of the input.  @a pvInput must be NULL when
+ *                          zero.
+ * @param   pvOutput        The output buffer.  This must also point within the
+ *                          VM structure for ring-3 pointer magic.
+ * @param   cbOutput        The size of the output.  @a pvOutput must be NULL
+ *                          when zero.
+ * @thread  EMT(pGVCpu)
+ */
+DECLINLINE(NTSTATUS) nemR0NtPerformIoControl(PGVM pGVM, PGVMCPU pGVCpu, uint32_t uFunction, void *pvInput, uint32_t cbInput,
+                                             void *pvOutput, uint32_t cbOutput)
+{
+#ifdef RT_STRICT
+    /*
+     * Input and output parameters are part of the VM CPU structure.
+     */
+    VMCPU_ASSERT_EMT(pGVCpu);
+    if (pvInput)
+        AssertReturn(((uintptr_t)pvInput + cbInput) - (uintptr_t)pGVCpu <= sizeof(*pGVCpu), VERR_INVALID_PARAMETER);
+    if (pvOutput)
+        AssertReturn(((uintptr_t)pvOutput + cbOutput) - (uintptr_t)pGVCpu <= sizeof(*pGVCpu), VERR_INVALID_PARAMETER);
+#endif
+
+    int32_t rcNt = STATUS_UNSUCCESSFUL;
+    int rc = SUPR0IoCtlPerform(pGVM->nemr0.s.pIoCtlCtx, uFunction,
+                               pvInput,
+                               pvInput  ? (uintptr_t)pvInput  + pGVCpu->nemr0.s.offRing3ConversionDelta : NIL_RTR3PTR,
+                               cbInput,
+                               pvOutput,
+                               pvOutput ? (uintptr_t)pvOutput + pGVCpu->nemr0.s.offRing3ConversionDelta : NIL_RTR3PTR,
+                               cbOutput,
+                               &rcNt);
+    if (RT_SUCCESS(rc) || !NT_SUCCESS((NTSTATUS)rcNt))
+        return (NTSTATUS)rcNt;
+    return STATUS_UNSUCCESSFUL;
+}
+
+
+/**
+ * 2nd part of the initialization, after we've got a partition handle.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The ring-0 VM handle.
+ * @thread  EMT(0)
+ */
+VMMR0_INT_DECL(int) NEMR0InitVMPart2(PGVM pGVM)
+{
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, 0);
+    AssertRCReturn(rc, rc);
+    SUPR0Printf("NEMR0InitVMPart2\n"); LogRel(("2: NEMR0InitVMPart2\n"));
+    Assert(pGVM->nemr0.s.fMayUseRing0Runloop == false);
+
+    /*
+     * Copy and validate the I/O control information from ring-3.
+     */
+    NEMWINIOCTL Copy = pGVM->nem.s.IoCtlGetHvPartitionId;
+    AssertLogRelReturn(Copy.uFunction != 0, VERR_NEM_INIT_FAILED);
+    AssertLogRelReturn(Copy.cbInput == 0, VERR_NEM_INIT_FAILED);
+    AssertLogRelReturn(Copy.cbOutput == sizeof(HV_PARTITION_ID), VERR_NEM_INIT_FAILED);
+    pGVM->nemr0.s.IoCtlGetHvPartitionId = Copy;
+
+    pGVM->nemr0.s.fMayUseRing0Runloop = pGVM->nem.s.fUseRing0Runloop;
+
+    Copy = pGVM->nem.s.IoCtlStartVirtualProcessor;
+    AssertLogRelStmt(Copy.uFunction != 0, rc = VERR_NEM_INIT_FAILED);
+    AssertLogRelStmt(Copy.cbInput == sizeof(HV_VP_INDEX), rc = VERR_NEM_INIT_FAILED);
+    AssertLogRelStmt(Copy.cbOutput == 0, rc = VERR_NEM_INIT_FAILED);
+    AssertLogRelStmt(Copy.uFunction != pGVM->nemr0.s.IoCtlGetHvPartitionId.uFunction, rc = VERR_NEM_INIT_FAILED);
+    if (RT_SUCCESS(rc))
+        pGVM->nemr0.s.IoCtlStartVirtualProcessor = Copy;
+
+    Copy = pGVM->nem.s.IoCtlStopVirtualProcessor;
+    AssertLogRelStmt(Copy.uFunction != 0, rc = VERR_NEM_INIT_FAILED);
+    AssertLogRelStmt(Copy.cbInput == sizeof(HV_VP_INDEX), rc = VERR_NEM_INIT_FAILED);
+    AssertLogRelStmt(Copy.cbOutput == 0, rc = VERR_NEM_INIT_FAILED);
+    AssertLogRelStmt(Copy.uFunction != pGVM->nemr0.s.IoCtlGetHvPartitionId.uFunction, rc = VERR_NEM_INIT_FAILED);
+    AssertLogRelStmt(Copy.uFunction != pGVM->nemr0.s.IoCtlStartVirtualProcessor.uFunction, rc = VERR_NEM_INIT_FAILED);
+    if (RT_SUCCESS(rc))
+        pGVM->nemr0.s.IoCtlStopVirtualProcessor = Copy;
+
+    Copy = pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext;
+    AssertLogRelStmt(Copy.uFunction != 0, rc = VERR_NEM_INIT_FAILED);
+    AssertLogRelStmt(   Copy.cbInput == sizeof(VID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT)
+                     || Copy.cbInput == RT_OFFSETOF(VID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT, cMillies),
+                     rc = VERR_NEM_INIT_FAILED);
+    AssertLogRelStmt(Copy.cbOutput == 0, VERR_NEM_INIT_FAILED);
+    AssertLogRelStmt(Copy.uFunction != pGVM->nemr0.s.IoCtlGetHvPartitionId.uFunction, rc = VERR_NEM_INIT_FAILED);
+    AssertLogRelStmt(Copy.uFunction != pGVM->nemr0.s.IoCtlStartVirtualProcessor.uFunction, rc = VERR_NEM_INIT_FAILED);
+    AssertLogRelStmt(Copy.uFunction != pGVM->nemr0.s.IoCtlStopVirtualProcessor.uFunction, rc = VERR_NEM_INIT_FAILED);
+    if (RT_SUCCESS(rc))
+        pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext = Copy;
+
+    if (   RT_SUCCESS(rc)
+        || !pGVM->nem.s.fUseRing0Runloop)
+    {
+        /*
+         * Setup of an I/O control context for the partition handle for later use.
+         */
+        rc = SUPR0IoCtlSetupForHandle(pGVM->pSession, pGVM->nem.s.hPartitionDevice, 0, &pGVM->nemr0.s.pIoCtlCtx);
+        AssertLogRelRCReturn(rc, rc);
+        for (VMCPUID idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
+        {
+            PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+            pGVCpu->nemr0.s.offRing3ConversionDelta = (uintptr_t)pGVM->aCpus[idCpu].pVCpuR3 - (uintptr_t)pGVCpu;
+        }
+
+        /*
+         * Get the partition ID.
+         */
+        PVMCPUCC pVCpu0 = &pGVM->aCpus[0];
+        NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pVCpu0, pGVM->nemr0.s.IoCtlGetHvPartitionId.uFunction, NULL, 0,
+                                                &pVCpu0->nem.s.uIoCtlBuf.idPartition, sizeof(pVCpu0->nem.s.uIoCtlBuf.idPartition));
+        AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("IoCtlGetHvPartitionId failed: %#x\n", rcNt), VERR_NEM_INIT_FAILED);
+        pGVM->nemr0.s.idHvPartition = pVCpu0->nem.s.uIoCtlBuf.idPartition;
+        AssertLogRelMsgReturn(pGVM->nemr0.s.idHvPartition == pGVM->nem.s.idHvPartition,
+                              ("idHvPartition mismatch: r0=%#RX64, r3=%#RX64\n", pGVM->nemr0.s.idHvPartition, pGVM->nem.s.idHvPartition),
+                              VERR_NEM_INIT_FAILED);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Cleanup the NEM parts of the VM in ring-0.
+ *
+ * This is always called and must deal the state regardless of whether
+ * NEMR0InitVM() was called or not.  So, take care here.
+ *
+ * @param   pGVM            The ring-0 VM handle.
+ */
+VMMR0_INT_DECL(void) NEMR0CleanupVM(PGVM pGVM)
+{
+    pGVM->nemr0.s.idHvPartition = HV_PARTITION_ID_INVALID;
+
+    /* Clean up I/O control context. */
+    if (pGVM->nemr0.s.pIoCtlCtx)
+    {
+        int rc = SUPR0IoCtlCleanup(pGVM->nemr0.s.pIoCtlCtx);
+        AssertRC(rc);
+        pGVM->nemr0.s.pIoCtlCtx = NULL;
+    }
+
+    /* Free the hypercall pages. */
+    VMCPUID i = pGVM->cCpus;
+    while (i-- > 0)
+        nemR0DeleteHypercallData(&pGVM->aCpus[i].nemr0.s.HypercallData);
+
+    /* The non-EMT one too. */
+    if (RTCritSectIsInitialized(&pGVM->nemr0.s.HypercallDataCritSect))
+        RTCritSectDelete(&pGVM->nemr0.s.HypercallDataCritSect);
+    nemR0DeleteHypercallData(&pGVM->nemr0.s.HypercallData);
+}
+
+
+#if 0 /* for debugging GPA unmapping.  */
+static int nemR3WinDummyReadGpa(PGVM pGVM, PGVMCPU pGVCpu, RTGCPHYS GCPhys)
+{
+    PHV_INPUT_READ_GPA  pIn  = (PHV_INPUT_READ_GPA)pGVCpu->nemr0.s.pbHypercallData;
+    PHV_OUTPUT_READ_GPA pOut = (PHV_OUTPUT_READ_GPA)(pIn + 1);
+    pIn->PartitionId            = pGVM->nemr0.s.idHvPartition;
+    pIn->VpIndex                = pGVCpu->idCpu;
+    pIn->ByteCount              = 0x10;
+    pIn->BaseGpa                = GCPhys;
+    pIn->ControlFlags.AsUINT64  = 0;
+    pIn->ControlFlags.CacheType = HvCacheTypeX64WriteCombining;
+    memset(pOut, 0xfe, sizeof(*pOut));
+    uint64_t volatile uResult = g_pfnHvlInvokeHypercall(HvCallReadGpa, pGVCpu->nemr0.s.HCPhysHypercallData,
+                                                        pGVCpu->nemr0.s.HCPhysHypercallData + sizeof(*pIn));
+    LogRel(("nemR3WinDummyReadGpa: %RGp -> %#RX64; code=%u rsvd=%u abData=%.16Rhxs\n",
+            GCPhys, uResult, pOut->AccessResult.ResultCode, pOut->AccessResult.Reserved, pOut->Data));
+    __debugbreak();
+
+    return uResult != 0 ? VERR_READ_ERROR : VINF_SUCCESS;
+}
+#endif
+
+
+/**
+ * Worker for NEMR0MapPages and others.
+ */
+NEM_TMPL_STATIC int nemR0WinMapPages(PGVM pGVM, PGVMCPU pGVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
+                                     uint32_t cPages, uint32_t fFlags)
+{
+    /*
+     * Validate.
+     */
+    AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API_1);
+
+    AssertReturn(cPages > 0, VERR_OUT_OF_RANGE);
+    AssertReturn(cPages <= NEM_MAX_MAP_PAGES, VERR_OUT_OF_RANGE);
+    AssertReturn(!(fFlags & ~(HV_MAP_GPA_MAYBE_ACCESS_MASK & ~HV_MAP_GPA_DUNNO_ACCESS)), VERR_INVALID_FLAGS);
+    AssertMsgReturn(!(GCPhysDst & X86_PAGE_OFFSET_MASK), ("GCPhysDst=%RGp\n", GCPhysDst), VERR_OUT_OF_RANGE);
+    AssertReturn(GCPhysDst < _1E, VERR_OUT_OF_RANGE);
+    if (GCPhysSrc != GCPhysDst)
+    {
+        AssertMsgReturn(!(GCPhysSrc & X86_PAGE_OFFSET_MASK), ("GCPhysSrc=%RGp\n", GCPhysSrc), VERR_OUT_OF_RANGE);
+        AssertReturn(GCPhysSrc < _1E, VERR_OUT_OF_RANGE);
+    }
+
+    /*
+     * Compose and make the hypercall.
+     * Ring-3 is not allowed to fill in the host physical addresses of the call.
+     */
+    for (uint32_t iTries = 0;; iTries++)
+    {
+        RTGCPHYS GCPhysSrcTmp = GCPhysSrc;
+        HV_INPUT_MAP_GPA_PAGES *pMapPages = (HV_INPUT_MAP_GPA_PAGES *)pGVCpu->nemr0.s.HypercallData.pbPage;
+        AssertPtrReturn(pMapPages, VERR_INTERNAL_ERROR_3);
+        pMapPages->TargetPartitionId    = pGVM->nemr0.s.idHvPartition;
+        pMapPages->TargetGpaBase        = GCPhysDst >> X86_PAGE_SHIFT;
+        pMapPages->MapFlags             = fFlags;
+        pMapPages->u32ExplicitPadding   = 0;
+
+        for (uint32_t iPage = 0; iPage < cPages; iPage++, GCPhysSrcTmp += X86_PAGE_SIZE)
+        {
+            RTHCPHYS HCPhys = NIL_RTGCPHYS;
+            int rc = PGMPhysGCPhys2HCPhys(pGVM, GCPhysSrcTmp, &HCPhys);
+            AssertRCReturn(rc, rc);
+            pMapPages->PageList[iPage] = HCPhys >> X86_PAGE_SHIFT;
+        }
+
+        uint64_t uResult = g_pfnHvlInvokeHypercall(HvCallMapGpaPages | ((uint64_t)cPages << 32),
+                                                   pGVCpu->nemr0.s.HypercallData.HCPhysPage, 0);
+        Log6(("NEMR0MapPages: %RGp/%RGp L %u prot %#x -> %#RX64\n",
+              GCPhysDst, GCPhysSrcTmp - cPages * X86_PAGE_SIZE, cPages, fFlags, uResult));
+        if (uResult == ((uint64_t)cPages << 32))
+            return VINF_SUCCESS;
+
+        /*
+         * If the partition is out of memory, try donate another 512 pages to
+         * it (2MB).  VID.SYS does multiples of 512 pages, nothing smaller.
+         */
+        if (   uResult != HV_STATUS_INSUFFICIENT_MEMORY
+            || iTries > 16
+            || g_pfnWinHvDepositMemory == NULL)
+        {
+            LogRel(("g_pfnHvlInvokeHypercall/MapGpaPages -> %#RX64\n", uResult));
+            return VERR_NEM_MAP_PAGES_FAILED;
+        }
+
+        size_t cPagesAdded = 0;
+        NTSTATUS rcNt = g_pfnWinHvDepositMemory(pGVM->nemr0.s.idHvPartition, 512, 0, &cPagesAdded);
+        if (!cPagesAdded)
+        {
+            LogRel(("g_pfnWinHvDepositMemory -> %#x / %#RX64\n", rcNt, uResult));
+            return VERR_NEM_MAP_PAGES_FAILED;
+        }
+    }
+}
+
+
+/**
+ * Maps pages into the guest physical address space.
+ *
+ * Generally the caller will be under the PGM lock already, so no extra effort
+ * is needed to make sure all changes happens under it.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The ring-0 VM handle.
+ * @param   idCpu           The calling EMT.  Necessary for getting the
+ *                          hypercall page and arguments.
+ * @thread  EMT(idCpu)
+ */
+VMMR0_INT_DECL(int) NEMR0MapPages(PGVM pGVM, VMCPUID idCpu)
+{
+    /*
+     * Unpack the call.
+     */
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_SUCCESS(rc))
+    {
+        PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+
+        RTGCPHYS const          GCPhysSrc = pGVCpu->nem.s.Hypercall.MapPages.GCPhysSrc;
+        RTGCPHYS const          GCPhysDst = pGVCpu->nem.s.Hypercall.MapPages.GCPhysDst;
+        uint32_t const          cPages    = pGVCpu->nem.s.Hypercall.MapPages.cPages;
+        HV_MAP_GPA_FLAGS const  fFlags    = pGVCpu->nem.s.Hypercall.MapPages.fFlags;
+
+        /*
+         * Do the work.
+         */
+        rc = nemR0WinMapPages(pGVM, pGVCpu, GCPhysSrc, GCPhysDst, cPages, fFlags);
+    }
+    return rc;
+}
+
+
+/**
+ * Worker for NEMR0UnmapPages and others.
+ */
+NEM_TMPL_STATIC int nemR0WinUnmapPages(PGVM pGVM, PGVMCPU pGVCpu, RTGCPHYS GCPhys, uint32_t cPages)
+{
+    /*
+     * Validate input.
+     */
+    AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API_1);
+
+    AssertReturn(cPages > 0, VERR_OUT_OF_RANGE);
+    AssertReturn(cPages <= NEM_MAX_UNMAP_PAGES, VERR_OUT_OF_RANGE);
+    AssertMsgReturn(!(GCPhys & X86_PAGE_OFFSET_MASK), ("%RGp\n", GCPhys), VERR_OUT_OF_RANGE);
+    AssertReturn(GCPhys < _1E, VERR_OUT_OF_RANGE);
+
+    /*
+     * Compose and make the hypercall.
+     */
+    HV_INPUT_UNMAP_GPA_PAGES *pUnmapPages = (HV_INPUT_UNMAP_GPA_PAGES *)pGVCpu->nemr0.s.HypercallData.pbPage;
+    AssertPtrReturn(pUnmapPages, VERR_INTERNAL_ERROR_3);
+    pUnmapPages->TargetPartitionId    = pGVM->nemr0.s.idHvPartition;
+    pUnmapPages->TargetGpaBase        = GCPhys >> X86_PAGE_SHIFT;
+    pUnmapPages->fFlags               = 0;
+
+    uint64_t uResult = g_pfnHvlInvokeHypercall(HvCallUnmapGpaPages | ((uint64_t)cPages << 32),
+                                               pGVCpu->nemr0.s.HypercallData.HCPhysPage, 0);
+    Log6(("NEMR0UnmapPages: %RGp L %u -> %#RX64\n", GCPhys, cPages, uResult));
+    if (uResult == ((uint64_t)cPages << 32))
+    {
+#if 1       /* Do we need to do this? Hopefully not... */
+        uint64_t volatile uR = g_pfnHvlInvokeHypercall(HvCallUncommitGpaPages | ((uint64_t)cPages << 32),
+                                                       pGVCpu->nemr0.s.HypercallData.HCPhysPage, 0);
+        AssertMsg(uR == ((uint64_t)cPages << 32), ("uR=%#RX64\n", uR)); NOREF(uR);
+#endif
+        return VINF_SUCCESS;
+    }
+
+    LogRel(("g_pfnHvlInvokeHypercall/UnmapGpaPages -> %#RX64\n", uResult));
+    return VERR_NEM_UNMAP_PAGES_FAILED;
+}
+
+
+/**
+ * Unmaps pages from the guest physical address space.
+ *
+ * Generally the caller will be under the PGM lock already, so no extra effort
+ * is needed to make sure all changes happens under it.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The ring-0 VM handle.
+ * @param   idCpu           The calling EMT.  Necessary for getting the
+ *                          hypercall page and arguments.
+ * @thread  EMT(idCpu)
+ */
+VMMR0_INT_DECL(int) NEMR0UnmapPages(PGVM pGVM, VMCPUID idCpu)
+{
+    /*
+     * Unpack the call.
+     */
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_SUCCESS(rc))
+    {
+        PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+
+        RTGCPHYS const GCPhys = pGVCpu->nem.s.Hypercall.UnmapPages.GCPhys;
+        uint32_t const cPages = pGVCpu->nem.s.Hypercall.UnmapPages.cPages;
+
+        /*
+         * Do the work.
+         */
+        rc = nemR0WinUnmapPages(pGVM, pGVCpu, GCPhys, cPages);
+    }
+    return rc;
+}
+
+
+#if defined(NEM_WIN_WITH_RING0_RUNLOOP) || defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
+/**
+ * Worker for NEMR0ExportState.
+ *
+ * Intention is to use it internally later.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The ring-0 VM handle.
+ * @param   pGVCpu      The ring-0 VCPU handle.
+ * @param   pCtx        The CPU context structure to import into.
+ */
+NEM_TMPL_STATIC int nemR0WinExportState(PGVM pGVM, PGVMCPU pGVCpu, PCPUMCTX pCtx)
+{
+    HV_INPUT_SET_VP_REGISTERS *pInput = (HV_INPUT_SET_VP_REGISTERS *)pGVCpu->nemr0.s.HypercallData.pbPage;
+    AssertPtrReturn(pInput, VERR_INTERNAL_ERROR_3);
+    AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API_1);
+
+    pInput->PartitionId = pGVM->nemr0.s.idHvPartition;
+    pInput->VpIndex     = pGVCpu->idCpu;
+    pInput->RsvdZ       = 0;
+
+    uint64_t const fWhat = ~pCtx->fExtrn & (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK);
+    if (   !fWhat
+        && pGVCpu->nem.s.fCurrentInterruptWindows == pGVCpu->nem.s.fDesiredInterruptWindows)
+        return VINF_SUCCESS;
+    uintptr_t iReg = 0;
+
+    /* GPRs */
+    if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_RAX)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterRax;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->rax;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RCX)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterRcx;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->rcx;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RDX)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterRdx;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->rdx;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RBX)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterRbx;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->rbx;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RSP)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterRsp;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->rsp;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RBP)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterRbp;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->rbp;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RSI)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterRsi;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->rsi;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RDI)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterRdi;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->rdi;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_R8_R15)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterR8;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->r8;
+            iReg++;
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterR9;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->r9;
+            iReg++;
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterR10;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->r10;
+            iReg++;
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterR11;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->r11;
+            iReg++;
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterR12;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->r12;
+            iReg++;
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterR13;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->r13;
+            iReg++;
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterR14;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->r14;
+            iReg++;
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                = HvX64RegisterR15;
+            pInput->Elements[iReg].Value.Reg64         = pCtx->r15;
+            iReg++;
+        }
+    }
+
+    /* RIP & Flags */
+    if (fWhat & CPUMCTX_EXTRN_RIP)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                = HvX64RegisterRip;
+        pInput->Elements[iReg].Value.Reg64         = pCtx->rip;
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_RFLAGS)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                = HvX64RegisterRflags;
+        pInput->Elements[iReg].Value.Reg64         = pCtx->rflags.u;
+        iReg++;
+    }
+
+    /* Segments */
+# define COPY_OUT_SEG(a_idx, a_enmName, a_SReg) \
+        do { \
+            HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[a_idx]); \
+            pInput->Elements[a_idx].Name                     = a_enmName; \
+            pInput->Elements[a_idx].Value.Segment.Base       = (a_SReg).u64Base; \
+            pInput->Elements[a_idx].Value.Segment.Limit      = (a_SReg).u32Limit; \
+            pInput->Elements[a_idx].Value.Segment.Selector   = (a_SReg).Sel; \
+            pInput->Elements[a_idx].Value.Segment.Attributes = (a_SReg).Attr.u; \
+        } while (0)
+    if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_CS)
+        {
+            COPY_OUT_SEG(iReg, HvX64RegisterCs,   pCtx->cs);
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_ES)
+        {
+            COPY_OUT_SEG(iReg, HvX64RegisterEs,   pCtx->es);
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_SS)
+        {
+            COPY_OUT_SEG(iReg, HvX64RegisterSs,   pCtx->ss);
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_DS)
+        {
+            COPY_OUT_SEG(iReg, HvX64RegisterDs,   pCtx->ds);
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_FS)
+        {
+            COPY_OUT_SEG(iReg, HvX64RegisterFs,   pCtx->fs);
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_GS)
+        {
+            COPY_OUT_SEG(iReg, HvX64RegisterGs,   pCtx->gs);
+            iReg++;
+        }
+    }
+
+    /* Descriptor tables & task segment. */
+    if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_LDTR)
+        {
+            COPY_OUT_SEG(iReg, HvX64RegisterLdtr, pCtx->ldtr);
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_TR)
+        {
+            COPY_OUT_SEG(iReg, HvX64RegisterTr,   pCtx->tr);
+            iReg++;
+        }
+
+        if (fWhat & CPUMCTX_EXTRN_IDTR)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Value.Table.Pad[0]   = 0;
+            pInput->Elements[iReg].Value.Table.Pad[1]   = 0;
+            pInput->Elements[iReg].Value.Table.Pad[2]   = 0;
+            pInput->Elements[iReg].Name                 = HvX64RegisterIdtr;
+            pInput->Elements[iReg].Value.Table.Limit    = pCtx->idtr.cbIdt;
+            pInput->Elements[iReg].Value.Table.Base     = pCtx->idtr.pIdt;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_GDTR)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Value.Table.Pad[0]   = 0;
+            pInput->Elements[iReg].Value.Table.Pad[1]   = 0;
+            pInput->Elements[iReg].Value.Table.Pad[2]   = 0;
+            pInput->Elements[iReg].Name                 = HvX64RegisterGdtr;
+            pInput->Elements[iReg].Value.Table.Limit    = pCtx->gdtr.cbGdt;
+            pInput->Elements[iReg].Value.Table.Base     = pCtx->gdtr.pGdt;
+            iReg++;
+        }
+    }
+
+    /* Control registers. */
+    if (fWhat & CPUMCTX_EXTRN_CR_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_CR0)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                 = HvX64RegisterCr0;
+            pInput->Elements[iReg].Value.Reg64          = pCtx->cr0;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_CR2)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                 = HvX64RegisterCr2;
+            pInput->Elements[iReg].Value.Reg64          = pCtx->cr2;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_CR3)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                 = HvX64RegisterCr3;
+            pInput->Elements[iReg].Value.Reg64          = pCtx->cr3;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_CR4)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                 = HvX64RegisterCr4;
+            pInput->Elements[iReg].Value.Reg64          = pCtx->cr4;
+            iReg++;
+        }
+    }
+    if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterCr8;
+        pInput->Elements[iReg].Value.Reg64          = CPUMGetGuestCR8(pGVCpu);
+        iReg++;
+    }
+
+    /** @todo does HvX64RegisterXfem mean XCR0? What about the related MSR. */
+
+    /* Debug registers. */
+/** @todo fixme. Figure out what the hyper-v version of KVM_SET_GUEST_DEBUG would be. */
+    if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterDr0;
+        //pInput->Elements[iReg].Value.Reg64        = CPUMGetHyperDR0(pGVCpu);
+        pInput->Elements[iReg].Value.Reg64          = pCtx->dr[0];
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterDr1;
+        //pInput->Elements[iReg].Value.Reg64        = CPUMGetHyperDR1(pGVCpu);
+        pInput->Elements[iReg].Value.Reg64          = pCtx->dr[1];
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterDr2;
+        //pInput->Elements[iReg].Value.Reg64        = CPUMGetHyperDR2(pGVCpu);
+        pInput->Elements[iReg].Value.Reg64          = pCtx->dr[2];
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterDr3;
+        //pInput->Elements[iReg].Value.Reg64        = CPUMGetHyperDR3(pGVCpu);
+        pInput->Elements[iReg].Value.Reg64          = pCtx->dr[3];
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR6)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterDr6;
+        //pInput->Elements[iReg].Value.Reg64        = CPUMGetHyperDR6(pGVCpu);
+        pInput->Elements[iReg].Value.Reg64          = pCtx->dr[6];
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR7)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterDr7;
+        //pInput->Elements[iReg].Value.Reg64        = CPUMGetHyperDR7(pGVCpu);
+        pInput->Elements[iReg].Value.Reg64          = pCtx->dr[7];
+        iReg++;
+    }
+
+    /* Floating point state. */
+    if (fWhat & CPUMCTX_EXTRN_X87)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                      = HvX64RegisterFpMmx0;
+        pInput->Elements[iReg].Value.Fp.AsUINT128.Low64  = pCtx->pXStateR0->x87.aRegs[0].au64[0];
+        pInput->Elements[iReg].Value.Fp.AsUINT128.High64 = pCtx->pXStateR0->x87.aRegs[0].au64[1];
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                      = HvX64RegisterFpMmx1;
+        pInput->Elements[iReg].Value.Fp.AsUINT128.Low64  = pCtx->pXStateR0->x87.aRegs[1].au64[0];
+        pInput->Elements[iReg].Value.Fp.AsUINT128.High64 = pCtx->pXStateR0->x87.aRegs[1].au64[1];
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                      = HvX64RegisterFpMmx2;
+        pInput->Elements[iReg].Value.Fp.AsUINT128.Low64  = pCtx->pXStateR0->x87.aRegs[2].au64[0];
+        pInput->Elements[iReg].Value.Fp.AsUINT128.High64 = pCtx->pXStateR0->x87.aRegs[2].au64[1];
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                      = HvX64RegisterFpMmx3;
+        pInput->Elements[iReg].Value.Fp.AsUINT128.Low64  = pCtx->pXStateR0->x87.aRegs[3].au64[0];
+        pInput->Elements[iReg].Value.Fp.AsUINT128.High64 = pCtx->pXStateR0->x87.aRegs[3].au64[1];
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                      = HvX64RegisterFpMmx4;
+        pInput->Elements[iReg].Value.Fp.AsUINT128.Low64  = pCtx->pXStateR0->x87.aRegs[4].au64[0];
+        pInput->Elements[iReg].Value.Fp.AsUINT128.High64 = pCtx->pXStateR0->x87.aRegs[4].au64[1];
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                      = HvX64RegisterFpMmx5;
+        pInput->Elements[iReg].Value.Fp.AsUINT128.Low64  = pCtx->pXStateR0->x87.aRegs[5].au64[0];
+        pInput->Elements[iReg].Value.Fp.AsUINT128.High64 = pCtx->pXStateR0->x87.aRegs[5].au64[1];
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                      = HvX64RegisterFpMmx6;
+        pInput->Elements[iReg].Value.Fp.AsUINT128.Low64  = pCtx->pXStateR0->x87.aRegs[6].au64[0];
+        pInput->Elements[iReg].Value.Fp.AsUINT128.High64 = pCtx->pXStateR0->x87.aRegs[6].au64[1];
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                      = HvX64RegisterFpMmx7;
+        pInput->Elements[iReg].Value.Fp.AsUINT128.Low64  = pCtx->pXStateR0->x87.aRegs[7].au64[0];
+        pInput->Elements[iReg].Value.Fp.AsUINT128.High64 = pCtx->pXStateR0->x87.aRegs[7].au64[1];
+        iReg++;
+
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                            = HvX64RegisterFpControlStatus;
+        pInput->Elements[iReg].Value.FpControlStatus.FpControl = pCtx->pXStateR0->x87.FCW;
+        pInput->Elements[iReg].Value.FpControlStatus.FpStatus  = pCtx->pXStateR0->x87.FSW;
+        pInput->Elements[iReg].Value.FpControlStatus.FpTag     = pCtx->pXStateR0->x87.FTW;
+        pInput->Elements[iReg].Value.FpControlStatus.Reserved  = pCtx->pXStateR0->x87.FTW >> 8;
+        pInput->Elements[iReg].Value.FpControlStatus.LastFpOp  = pCtx->pXStateR0->x87.FOP;
+        pInput->Elements[iReg].Value.FpControlStatus.LastFpRip = (pCtx->pXStateR0->x87.FPUIP)
+                                                               | ((uint64_t)pCtx->pXStateR0->x87.CS << 32)
+                                                               | ((uint64_t)pCtx->pXStateR0->x87.Rsrvd1 << 48);
+        iReg++;
+/** @todo we've got trouble if if we try write just SSE w/o X87.  */
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                                        = HvX64RegisterXmmControlStatus;
+        pInput->Elements[iReg].Value.XmmControlStatus.LastFpRdp            = (pCtx->pXStateR0->x87.FPUDP)
+                                                                           | ((uint64_t)pCtx->pXStateR0->x87.DS << 32)
+                                                                           | ((uint64_t)pCtx->pXStateR0->x87.Rsrvd2 << 48);
+        pInput->Elements[iReg].Value.XmmControlStatus.XmmStatusControl     = pCtx->pXStateR0->x87.MXCSR;
+        pInput->Elements[iReg].Value.XmmControlStatus.XmmStatusControlMask = pCtx->pXStateR0->x87.MXCSR_MASK; /** @todo ??? (Isn't this an output field?) */
+        iReg++;
+    }
+
+    /* Vector state. */
+    if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm0;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[0].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[0].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm1;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[1].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[1].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm2;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[2].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[2].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm3;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[3].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[3].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm4;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[4].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[4].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm5;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[5].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[5].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm6;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[6].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[6].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm7;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[7].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[7].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm8;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[8].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[8].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm9;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[9].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[9].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm10;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[10].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[10].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm11;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[11].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[11].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm12;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[12].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[12].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm13;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[13].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[13].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm14;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[14].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[14].uXmm.s.Hi;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterXmm15;
+        pInput->Elements[iReg].Value.Reg128.Low64   = pCtx->pXStateR0->x87.aXMM[15].uXmm.s.Lo;
+        pInput->Elements[iReg].Value.Reg128.High64  = pCtx->pXStateR0->x87.aXMM[15].uXmm.s.Hi;
+        iReg++;
+    }
+
+    /* MSRs */
+    // HvX64RegisterTsc - don't touch
+    if (fWhat & CPUMCTX_EXTRN_EFER)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterEfer;
+        pInput->Elements[iReg].Value.Reg64          = pCtx->msrEFER;
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterKernelGsBase;
+        pInput->Elements[iReg].Value.Reg64          = pCtx->msrKERNELGSBASE;
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterSysenterCs;
+        pInput->Elements[iReg].Value.Reg64          = pCtx->SysEnter.cs;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterSysenterEip;
+        pInput->Elements[iReg].Value.Reg64          = pCtx->SysEnter.eip;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterSysenterEsp;
+        pInput->Elements[iReg].Value.Reg64          = pCtx->SysEnter.esp;
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterStar;
+        pInput->Elements[iReg].Value.Reg64          = pCtx->msrSTAR;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterLstar;
+        pInput->Elements[iReg].Value.Reg64          = pCtx->msrLSTAR;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterCstar;
+        pInput->Elements[iReg].Value.Reg64          = pCtx->msrCSTAR;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterSfmask;
+        pInput->Elements[iReg].Value.Reg64          = pCtx->msrSFMASK;
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterApicBase;
+        pInput->Elements[iReg].Value.Reg64          = APICGetBaseMsrNoCheck(pGVCpu);
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterPat;
+        pInput->Elements[iReg].Value.Reg64          = pCtx->msrPAT;
+        iReg++;
+# if 0 /** @todo HvX64RegisterMtrrCap is read only?  Seems it's not even readable. */
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrCap;
+        pInput->Elements[iReg].Value.Reg64          = CPUMGetGuestIa32MtrrCap(pGVCpu);
+        iReg++;
+# endif
+
+        PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pGVCpu);
+
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrDefType;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MtrrDefType;
+        iReg++;
+
+        /** @todo we dont keep state for HvX64RegisterMtrrPhysBaseX and HvX64RegisterMtrrPhysMaskX */
+
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrFix64k00000;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MtrrFix64K_00000;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrFix16k80000;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MtrrFix16K_80000;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrFix16kA0000;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MtrrFix16K_A0000;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrFix4kC0000;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MtrrFix4K_C0000;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrFix4kC8000;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MtrrFix4K_C8000;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrFix4kD0000;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MtrrFix4K_D0000;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrFix4kD8000;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MtrrFix4K_D8000;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrFix4kE0000;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MtrrFix4K_E0000;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrFix4kE8000;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MtrrFix4K_E8000;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrFix4kF0000;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MtrrFix4K_F0000;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterMtrrFix4kF8000;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MtrrFix4K_F8000;
+        iReg++;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvX64RegisterTscAux;
+        pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.TscAux;
+        iReg++;
+
+# if 0 /** @todo Why can't we write these on Intel systems? Not that we really care... */
+        const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pGVM);
+        if (enmCpuVendor != CPUMCPUVENDOR_AMD)
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                 = HvX64RegisterIa32MiscEnable;
+            pInput->Elements[iReg].Value.Reg64          = pCtxMsrs->msr.MiscEnable;
+            iReg++;
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                 = HvX64RegisterIa32FeatureControl;
+            pInput->Elements[iReg].Value.Reg64          = CPUMGetGuestIa32FeatureControl(pGVCpu);
+            iReg++;
+        }
+# endif
+    }
+
+    /* event injection (clear it). */
+    if (fWhat & CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvRegisterPendingInterruption;
+        pInput->Elements[iReg].Value.Reg64          = 0;
+        iReg++;
+    }
+
+    /* Interruptibility state.  This can get a little complicated since we get
+       half of the state via HV_X64_VP_EXECUTION_STATE. */
+    if (   (fWhat & (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
+        ==          (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI) )
+    {
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                 = HvRegisterInterruptState;
+        pInput->Elements[iReg].Value.Reg64          = 0;
+        if (   VMCPU_FF_IS_SET(pGVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+            && EMGetInhibitInterruptsPC(pGVCpu) == pCtx->rip)
+            pInput->Elements[iReg].Value.InterruptState.InterruptShadow = 1;
+        if (VMCPU_FF_IS_SET(pGVCpu, VMCPU_FF_BLOCK_NMIS))
+            pInput->Elements[iReg].Value.InterruptState.NmiMasked = 1;
+        iReg++;
+    }
+    else if (fWhat & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT)
+    {
+        if (   pGVCpu->nem.s.fLastInterruptShadow
+            || (   VMCPU_FF_IS_SET(pGVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+                && EMGetInhibitInterruptsPC(pGVCpu) == pCtx->rip))
+        {
+            HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+            pInput->Elements[iReg].Name                 = HvRegisterInterruptState;
+            pInput->Elements[iReg].Value.Reg64          = 0;
+            if (   VMCPU_FF_IS_SET(pGVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+                && EMGetInhibitInterruptsPC(pGVCpu) == pCtx->rip)
+                pInput->Elements[iReg].Value.InterruptState.InterruptShadow = 1;
+            /** @todo Retrieve NMI state, currently assuming it's zero. (yes this may happen on I/O) */
+            //if (VMCPU_FF_IS_ANY_SET(pGVCpu, VMCPU_FF_BLOCK_NMIS))
+            //    pInput->Elements[iReg].Value.InterruptState.NmiMasked = 1;
+            iReg++;
+        }
+    }
+    else
+        Assert(!(fWhat & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI));
+
+    /* Interrupt windows. Always set if active as Hyper-V seems to be forgetful. */
+    uint8_t const fDesiredIntWin = pGVCpu->nem.s.fDesiredInterruptWindows;
+    if (   fDesiredIntWin
+        || pGVCpu->nem.s.fCurrentInterruptWindows != fDesiredIntWin)
+    {
+        pGVCpu->nem.s.fCurrentInterruptWindows = pGVCpu->nem.s.fDesiredInterruptWindows;
+        HV_REGISTER_ASSOC_ZERO_PADDING_AND_HI64(&pInput->Elements[iReg]);
+        pInput->Elements[iReg].Name                                         = HvX64RegisterDeliverabilityNotifications;
+        pInput->Elements[iReg].Value.DeliverabilityNotifications.AsUINT64   = fDesiredIntWin;
+        Assert(pInput->Elements[iReg].Value.DeliverabilityNotifications.NmiNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_NMI));
+        Assert(pInput->Elements[iReg].Value.DeliverabilityNotifications.InterruptNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_REGULAR));
+        Assert(pInput->Elements[iReg].Value.DeliverabilityNotifications.InterruptPriority == (fDesiredIntWin & NEM_WIN_INTW_F_PRIO_MASK) >> NEM_WIN_INTW_F_PRIO_SHIFT);
+        iReg++;
+    }
+
+    /// @todo HvRegisterPendingEvent0
+    /// @todo HvRegisterPendingEvent1
+
+    /*
+     * Set the registers.
+     */
+    Assert((uintptr_t)&pInput->Elements[iReg] - (uintptr_t)pGVCpu->nemr0.s.HypercallData.pbPage < PAGE_SIZE); /* max is 127 */
+
+    /*
+     * Make the hypercall.
+     */
+    uint64_t uResult = g_pfnHvlInvokeHypercall(HV_MAKE_CALL_INFO(HvCallSetVpRegisters, iReg),
+                                               pGVCpu->nemr0.s.HypercallData.HCPhysPage, 0 /*GCPhysOutput*/);
+    AssertLogRelMsgReturn(uResult == HV_MAKE_CALL_REP_RET(iReg),
+                          ("uResult=%RX64 iRegs=%#x\n", uResult, iReg),
+                          VERR_NEM_SET_REGISTERS_FAILED);
+    //LogFlow(("nemR0WinExportState: uResult=%#RX64 iReg=%zu fWhat=%#018RX64 fExtrn=%#018RX64 -> %#018RX64\n", uResult, iReg, fWhat, pCtx->fExtrn,
+    //         pCtx->fExtrn | CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK | CPUMCTX_EXTRN_KEEPER_NEM ));
+    pCtx->fExtrn |= CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK | CPUMCTX_EXTRN_KEEPER_NEM;
+    return VINF_SUCCESS;
+}
+#endif /* NEM_WIN_WITH_RING0_RUNLOOP || NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
+
+
+/**
+ * Export the state to the native API (out of CPUMCTX).
+ *
+ * @returns VBox status code
+ * @param   pGVM        The ring-0 VM handle.
+ * @param   idCpu       The calling EMT.  Necessary for getting the
+ *                      hypercall page and arguments.
+ */
+VMMR0_INT_DECL(int)  NEMR0ExportState(PGVM pGVM, VMCPUID idCpu)
+{
+#if defined(NEM_WIN_WITH_RING0_RUNLOOP) || defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
+    /*
+     * Validate the call.
+     */
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_SUCCESS(rc))
+    {
+        PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+        AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API_1);
+
+        /*
+         * Call worker.
+         */
+        rc = nemR0WinExportState(pGVM, pGVCpu, &pGVCpu->cpum.GstCtx);
+    }
+    return rc;
+#else
+    RT_NOREF(pGVM, idCpu);
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+#if defined(NEM_WIN_WITH_RING0_RUNLOOP) || defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
+/**
+ * Worker for NEMR0ImportState.
+ *
+ * Intention is to use it internally later.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The ring-0 VM handle.
+ * @param   pGVCpu          The ring-0 VCPU handle.
+ * @param   pCtx            The CPU context structure to import into.
+ * @param   fWhat           What to import, CPUMCTX_EXTRN_XXX.
+ * @param   fCanUpdateCr3   Whether it's safe to update CR3 or not.
+ */
+NEM_TMPL_STATIC int nemR0WinImportState(PGVM pGVM, PGVMCPU pGVCpu, PCPUMCTX pCtx, uint64_t fWhat, bool fCanUpdateCr3)
+{
+    HV_INPUT_GET_VP_REGISTERS *pInput = (HV_INPUT_GET_VP_REGISTERS *)pGVCpu->nemr0.s.HypercallData.pbPage;
+    AssertPtrReturn(pInput, VERR_INTERNAL_ERROR_3);
+    AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API_1);
+    Assert(pCtx == &pGVCpu->cpum.GstCtx);
+
+    fWhat &= pCtx->fExtrn;
+
+    pInput->PartitionId = pGVM->nemr0.s.idHvPartition;
+    pInput->VpIndex     = pGVCpu->idCpu;
+    pInput->fFlags      = 0;
+
+    /* GPRs */
+    uintptr_t iReg = 0;
+    if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_RAX)
+            pInput->Names[iReg++] = HvX64RegisterRax;
+        if (fWhat & CPUMCTX_EXTRN_RCX)
+            pInput->Names[iReg++] = HvX64RegisterRcx;
+        if (fWhat & CPUMCTX_EXTRN_RDX)
+            pInput->Names[iReg++] = HvX64RegisterRdx;
+        if (fWhat & CPUMCTX_EXTRN_RBX)
+            pInput->Names[iReg++] = HvX64RegisterRbx;
+        if (fWhat & CPUMCTX_EXTRN_RSP)
+            pInput->Names[iReg++] = HvX64RegisterRsp;
+        if (fWhat & CPUMCTX_EXTRN_RBP)
+            pInput->Names[iReg++] = HvX64RegisterRbp;
+        if (fWhat & CPUMCTX_EXTRN_RSI)
+            pInput->Names[iReg++] = HvX64RegisterRsi;
+        if (fWhat & CPUMCTX_EXTRN_RDI)
+            pInput->Names[iReg++] = HvX64RegisterRdi;
+        if (fWhat & CPUMCTX_EXTRN_R8_R15)
+        {
+            pInput->Names[iReg++] = HvX64RegisterR8;
+            pInput->Names[iReg++] = HvX64RegisterR9;
+            pInput->Names[iReg++] = HvX64RegisterR10;
+            pInput->Names[iReg++] = HvX64RegisterR11;
+            pInput->Names[iReg++] = HvX64RegisterR12;
+            pInput->Names[iReg++] = HvX64RegisterR13;
+            pInput->Names[iReg++] = HvX64RegisterR14;
+            pInput->Names[iReg++] = HvX64RegisterR15;
+        }
+    }
+
+    /* RIP & Flags */
+    if (fWhat & CPUMCTX_EXTRN_RIP)
+        pInput->Names[iReg++]  = HvX64RegisterRip;
+    if (fWhat & CPUMCTX_EXTRN_RFLAGS)
+        pInput->Names[iReg++] = HvX64RegisterRflags;
+
+    /* Segments */
+    if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_CS)
+            pInput->Names[iReg++] = HvX64RegisterCs;
+        if (fWhat & CPUMCTX_EXTRN_ES)
+            pInput->Names[iReg++] = HvX64RegisterEs;
+        if (fWhat & CPUMCTX_EXTRN_SS)
+            pInput->Names[iReg++] = HvX64RegisterSs;
+        if (fWhat & CPUMCTX_EXTRN_DS)
+            pInput->Names[iReg++] = HvX64RegisterDs;
+        if (fWhat & CPUMCTX_EXTRN_FS)
+            pInput->Names[iReg++] = HvX64RegisterFs;
+        if (fWhat & CPUMCTX_EXTRN_GS)
+            pInput->Names[iReg++] = HvX64RegisterGs;
+    }
+
+    /* Descriptor tables and the task segment. */
+    if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_LDTR)
+            pInput->Names[iReg++] = HvX64RegisterLdtr;
+        if (fWhat & CPUMCTX_EXTRN_TR)
+            pInput->Names[iReg++] = HvX64RegisterTr;
+        if (fWhat & CPUMCTX_EXTRN_IDTR)
+            pInput->Names[iReg++] = HvX64RegisterIdtr;
+        if (fWhat & CPUMCTX_EXTRN_GDTR)
+            pInput->Names[iReg++] = HvX64RegisterGdtr;
+    }
+
+    /* Control registers. */
+    if (fWhat & CPUMCTX_EXTRN_CR_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_CR0)
+            pInput->Names[iReg++] = HvX64RegisterCr0;
+        if (fWhat & CPUMCTX_EXTRN_CR2)
+            pInput->Names[iReg++] = HvX64RegisterCr2;
+        if (fWhat & CPUMCTX_EXTRN_CR3)
+            pInput->Names[iReg++] = HvX64RegisterCr3;
+        if (fWhat & CPUMCTX_EXTRN_CR4)
+            pInput->Names[iReg++] = HvX64RegisterCr4;
+    }
+    if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
+        pInput->Names[iReg++] = HvX64RegisterCr8;
+
+    /* Debug registers. */
+    if (fWhat & CPUMCTX_EXTRN_DR7)
+        pInput->Names[iReg++] = HvX64RegisterDr7;
+    if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
+    {
+        if (!(fWhat & CPUMCTX_EXTRN_DR7) && (pCtx->fExtrn & CPUMCTX_EXTRN_DR7))
+        {
+            fWhat |= CPUMCTX_EXTRN_DR7;
+            pInput->Names[iReg++] = HvX64RegisterDr7;
+        }
+        pInput->Names[iReg++] = HvX64RegisterDr0;
+        pInput->Names[iReg++] = HvX64RegisterDr1;
+        pInput->Names[iReg++] = HvX64RegisterDr2;
+        pInput->Names[iReg++] = HvX64RegisterDr3;
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR6)
+        pInput->Names[iReg++] = HvX64RegisterDr6;
+
+    /* Floating point state. */
+    if (fWhat & CPUMCTX_EXTRN_X87)
+    {
+        pInput->Names[iReg++] = HvX64RegisterFpMmx0;
+        pInput->Names[iReg++] = HvX64RegisterFpMmx1;
+        pInput->Names[iReg++] = HvX64RegisterFpMmx2;
+        pInput->Names[iReg++] = HvX64RegisterFpMmx3;
+        pInput->Names[iReg++] = HvX64RegisterFpMmx4;
+        pInput->Names[iReg++] = HvX64RegisterFpMmx5;
+        pInput->Names[iReg++] = HvX64RegisterFpMmx6;
+        pInput->Names[iReg++] = HvX64RegisterFpMmx7;
+        pInput->Names[iReg++] = HvX64RegisterFpControlStatus;
+    }
+    if (fWhat & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX))
+        pInput->Names[iReg++] = HvX64RegisterXmmControlStatus;
+
+    /* Vector state. */
+    if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
+    {
+        pInput->Names[iReg++] = HvX64RegisterXmm0;
+        pInput->Names[iReg++] = HvX64RegisterXmm1;
+        pInput->Names[iReg++] = HvX64RegisterXmm2;
+        pInput->Names[iReg++] = HvX64RegisterXmm3;
+        pInput->Names[iReg++] = HvX64RegisterXmm4;
+        pInput->Names[iReg++] = HvX64RegisterXmm5;
+        pInput->Names[iReg++] = HvX64RegisterXmm6;
+        pInput->Names[iReg++] = HvX64RegisterXmm7;
+        pInput->Names[iReg++] = HvX64RegisterXmm8;
+        pInput->Names[iReg++] = HvX64RegisterXmm9;
+        pInput->Names[iReg++] = HvX64RegisterXmm10;
+        pInput->Names[iReg++] = HvX64RegisterXmm11;
+        pInput->Names[iReg++] = HvX64RegisterXmm12;
+        pInput->Names[iReg++] = HvX64RegisterXmm13;
+        pInput->Names[iReg++] = HvX64RegisterXmm14;
+        pInput->Names[iReg++] = HvX64RegisterXmm15;
+    }
+
+    /* MSRs */
+    // HvX64RegisterTsc - don't touch
+    if (fWhat & CPUMCTX_EXTRN_EFER)
+        pInput->Names[iReg++] = HvX64RegisterEfer;
+    if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+        pInput->Names[iReg++] = HvX64RegisterKernelGsBase;
+    if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+    {
+        pInput->Names[iReg++] = HvX64RegisterSysenterCs;
+        pInput->Names[iReg++] = HvX64RegisterSysenterEip;
+        pInput->Names[iReg++] = HvX64RegisterSysenterEsp;
+    }
+    if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+    {
+        pInput->Names[iReg++] = HvX64RegisterStar;
+        pInput->Names[iReg++] = HvX64RegisterLstar;
+        pInput->Names[iReg++] = HvX64RegisterCstar;
+        pInput->Names[iReg++] = HvX64RegisterSfmask;
+    }
+
+# ifdef LOG_ENABLED
+    const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pGVM);
+# endif
+    if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
+    {
+        pInput->Names[iReg++] = HvX64RegisterApicBase; /// @todo APIC BASE
+        pInput->Names[iReg++] = HvX64RegisterPat;
+# if 0 /*def LOG_ENABLED*/ /** @todo something's wrong with HvX64RegisterMtrrCap? (AMD) */
+        pInput->Names[iReg++] = HvX64RegisterMtrrCap;
+# endif
+        pInput->Names[iReg++] = HvX64RegisterMtrrDefType;
+        pInput->Names[iReg++] = HvX64RegisterMtrrFix64k00000;
+        pInput->Names[iReg++] = HvX64RegisterMtrrFix16k80000;
+        pInput->Names[iReg++] = HvX64RegisterMtrrFix16kA0000;
+        pInput->Names[iReg++] = HvX64RegisterMtrrFix4kC0000;
+        pInput->Names[iReg++] = HvX64RegisterMtrrFix4kC8000;
+        pInput->Names[iReg++] = HvX64RegisterMtrrFix4kD0000;
+        pInput->Names[iReg++] = HvX64RegisterMtrrFix4kD8000;
+        pInput->Names[iReg++] = HvX64RegisterMtrrFix4kE0000;
+        pInput->Names[iReg++] = HvX64RegisterMtrrFix4kE8000;
+        pInput->Names[iReg++] = HvX64RegisterMtrrFix4kF0000;
+        pInput->Names[iReg++] = HvX64RegisterMtrrFix4kF8000;
+        pInput->Names[iReg++] = HvX64RegisterTscAux;
+# if 0 /** @todo why can't we read HvX64RegisterIa32MiscEnable? */
+        if (enmCpuVendor != CPUMCPUVENDOR_AMD)
+            pInput->Names[iReg++] = HvX64RegisterIa32MiscEnable;
+# endif
+# ifdef LOG_ENABLED
+        if (enmCpuVendor != CPUMCPUVENDOR_AMD && enmCpuVendor != CPUMCPUVENDOR_HYGON)
+            pInput->Names[iReg++] = HvX64RegisterIa32FeatureControl;
+# endif
+    }
+
+    /* Interruptibility. */
+    if (fWhat & (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
+    {
+        pInput->Names[iReg++] = HvRegisterInterruptState;
+        pInput->Names[iReg++] = HvX64RegisterRip;
+    }
+
+    /* event injection */
+    pInput->Names[iReg++] = HvRegisterPendingInterruption;
+    pInput->Names[iReg++] = HvRegisterPendingEvent0;
+    pInput->Names[iReg++] = HvRegisterPendingEvent1;
+    size_t const cRegs   = iReg;
+    size_t const cbInput = RT_ALIGN_Z(RT_UOFFSETOF_DYN(HV_INPUT_GET_VP_REGISTERS, Names[cRegs]), 32);
+
+    HV_REGISTER_VALUE *paValues = (HV_REGISTER_VALUE *)((uint8_t *)pInput + cbInput);
+    Assert((uintptr_t)&paValues[cRegs] - (uintptr_t)pGVCpu->nemr0.s.HypercallData.pbPage < PAGE_SIZE); /* (max is around 168 registers) */
+    RT_BZERO(paValues, cRegs * sizeof(paValues[0]));
+
+    /*
+     * Make the hypercall.
+     */
+    uint64_t uResult = g_pfnHvlInvokeHypercall(HV_MAKE_CALL_INFO(HvCallGetVpRegisters, cRegs),
+                                               pGVCpu->nemr0.s.HypercallData.HCPhysPage,
+                                               pGVCpu->nemr0.s.HypercallData.HCPhysPage + cbInput);
+    AssertLogRelMsgReturn(uResult == HV_MAKE_CALL_REP_RET(cRegs),
+                          ("uResult=%RX64 cRegs=%#x\n", uResult, cRegs),
+                          VERR_NEM_GET_REGISTERS_FAILED);
+    //LogFlow(("nemR0WinImportState: uResult=%#RX64 iReg=%zu fWhat=%#018RX64 fExtr=%#018RX64\n", uResult, cRegs, fWhat, pCtx->fExtrn));
+
+    /*
+     * Copy information to the CPUM context.
+     */
+    iReg = 0;
+
+    /* GPRs */
+    if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_RAX)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterRax);
+            pCtx->rax = paValues[iReg++].Reg64;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RCX)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterRcx);
+            pCtx->rcx = paValues[iReg++].Reg64;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RDX)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterRdx);
+            pCtx->rdx = paValues[iReg++].Reg64;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RBX)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterRbx);
+            pCtx->rbx = paValues[iReg++].Reg64;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RSP)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterRsp);
+            pCtx->rsp = paValues[iReg++].Reg64;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RBP)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterRbp);
+            pCtx->rbp = paValues[iReg++].Reg64;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RSI)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterRsi);
+            pCtx->rsi = paValues[iReg++].Reg64;
+        }
+        if (fWhat & CPUMCTX_EXTRN_RDI)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterRdi);
+            pCtx->rdi = paValues[iReg++].Reg64;
+        }
+        if (fWhat & CPUMCTX_EXTRN_R8_R15)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterR8);
+            Assert(pInput->Names[iReg + 7] == HvX64RegisterR15);
+            pCtx->r8  = paValues[iReg++].Reg64;
+            pCtx->r9  = paValues[iReg++].Reg64;
+            pCtx->r10 = paValues[iReg++].Reg64;
+            pCtx->r11 = paValues[iReg++].Reg64;
+            pCtx->r12 = paValues[iReg++].Reg64;
+            pCtx->r13 = paValues[iReg++].Reg64;
+            pCtx->r14 = paValues[iReg++].Reg64;
+            pCtx->r15 = paValues[iReg++].Reg64;
+        }
+    }
+
+    /* RIP & Flags */
+    if (fWhat & CPUMCTX_EXTRN_RIP)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterRip);
+        pCtx->rip      = paValues[iReg++].Reg64;
+    }
+    if (fWhat & CPUMCTX_EXTRN_RFLAGS)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterRflags);
+        pCtx->rflags.u = paValues[iReg++].Reg64;
+    }
+
+    /* Segments */
+# define COPY_BACK_SEG(a_idx, a_enmName, a_SReg) \
+        do { \
+            Assert(pInput->Names[a_idx] == a_enmName); \
+            (a_SReg).u64Base  = paValues[a_idx].Segment.Base; \
+            (a_SReg).u32Limit = paValues[a_idx].Segment.Limit; \
+            (a_SReg).ValidSel = (a_SReg).Sel = paValues[a_idx].Segment.Selector; \
+            (a_SReg).Attr.u   = paValues[a_idx].Segment.Attributes; \
+            (a_SReg).fFlags   = CPUMSELREG_FLAGS_VALID; \
+        } while (0)
+    if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_CS)
+        {
+            COPY_BACK_SEG(iReg, HvX64RegisterCs,   pCtx->cs);
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_ES)
+        {
+            COPY_BACK_SEG(iReg, HvX64RegisterEs,   pCtx->es);
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_SS)
+        {
+            COPY_BACK_SEG(iReg, HvX64RegisterSs,   pCtx->ss);
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_DS)
+        {
+            COPY_BACK_SEG(iReg, HvX64RegisterDs,   pCtx->ds);
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_FS)
+        {
+            COPY_BACK_SEG(iReg, HvX64RegisterFs,   pCtx->fs);
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_GS)
+        {
+            COPY_BACK_SEG(iReg, HvX64RegisterGs,   pCtx->gs);
+            iReg++;
+        }
+    }
+    /* Descriptor tables and the task segment. */
+    if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_LDTR)
+        {
+            COPY_BACK_SEG(iReg, HvX64RegisterLdtr, pCtx->ldtr);
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_TR)
+        {
+            /* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY.  So,
+               avoid to trigger sanity assertions around the code, always fix this. */
+            COPY_BACK_SEG(iReg, HvX64RegisterTr,   pCtx->tr);
+            switch (pCtx->tr.Attr.n.u4Type)
+            {
+                case X86_SEL_TYPE_SYS_386_TSS_BUSY:
+                case X86_SEL_TYPE_SYS_286_TSS_BUSY:
+                    break;
+                case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
+                    pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
+                    break;
+                case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
+                    pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
+                    break;
+            }
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_IDTR)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterIdtr);
+            pCtx->idtr.cbIdt = paValues[iReg].Table.Limit;
+            pCtx->idtr.pIdt  = paValues[iReg].Table.Base;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_GDTR)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterGdtr);
+            pCtx->gdtr.cbGdt = paValues[iReg].Table.Limit;
+            pCtx->gdtr.pGdt  = paValues[iReg].Table.Base;
+            iReg++;
+        }
+    }
+
+    /* Control registers. */
+    bool fMaybeChangedMode = false;
+    bool fUpdateCr3        = false;
+    if (fWhat & CPUMCTX_EXTRN_CR_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_CR0)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterCr0);
+            if (pCtx->cr0 != paValues[iReg].Reg64)
+            {
+                CPUMSetGuestCR0(pGVCpu, paValues[iReg].Reg64);
+                fMaybeChangedMode = true;
+            }
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_CR2)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterCr2);
+            pCtx->cr2 = paValues[iReg].Reg64;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_CR3)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterCr3);
+            if (pCtx->cr3 != paValues[iReg].Reg64)
+            {
+                CPUMSetGuestCR3(pGVCpu, paValues[iReg].Reg64);
+                fUpdateCr3 = true;
+            }
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_CR4)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterCr4);
+            if (pCtx->cr4 != paValues[iReg].Reg64)
+            {
+                CPUMSetGuestCR4(pGVCpu, paValues[iReg].Reg64);
+                fMaybeChangedMode = true;
+            }
+            iReg++;
+        }
+    }
+    if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterCr8);
+        APICSetTpr(pGVCpu, (uint8_t)paValues[iReg].Reg64 << 4);
+        iReg++;
+    }
+
+    /* Debug registers. */
+    if (fWhat & CPUMCTX_EXTRN_DR7)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterDr7);
+        if (pCtx->dr[7] != paValues[iReg].Reg64)
+            CPUMSetGuestDR7(pGVCpu, paValues[iReg].Reg64);
+        pCtx->fExtrn &= ~CPUMCTX_EXTRN_DR7; /* Hack alert! Avoids asserting when processing CPUMCTX_EXTRN_DR0_DR3. */
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterDr0);
+        Assert(pInput->Names[iReg+3] == HvX64RegisterDr3);
+        if (pCtx->dr[0] != paValues[iReg].Reg64)
+            CPUMSetGuestDR0(pGVCpu, paValues[iReg].Reg64);
+        iReg++;
+        if (pCtx->dr[1] != paValues[iReg].Reg64)
+            CPUMSetGuestDR1(pGVCpu, paValues[iReg].Reg64);
+        iReg++;
+        if (pCtx->dr[2] != paValues[iReg].Reg64)
+            CPUMSetGuestDR2(pGVCpu, paValues[iReg].Reg64);
+        iReg++;
+        if (pCtx->dr[3] != paValues[iReg].Reg64)
+            CPUMSetGuestDR3(pGVCpu, paValues[iReg].Reg64);
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR6)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterDr6);
+        if (pCtx->dr[6] != paValues[iReg].Reg64)
+            CPUMSetGuestDR6(pGVCpu, paValues[iReg].Reg64);
+        iReg++;
+    }
+
+    /* Floating point state. */
+    if (fWhat & CPUMCTX_EXTRN_X87)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterFpMmx0);
+        Assert(pInput->Names[iReg + 7] == HvX64RegisterFpMmx7);
+        pCtx->pXStateR0->x87.aRegs[0].au64[0] = paValues[iReg].Fp.AsUINT128.Low64;
+        pCtx->pXStateR0->x87.aRegs[0].au64[1] = paValues[iReg].Fp.AsUINT128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aRegs[1].au64[0] = paValues[iReg].Fp.AsUINT128.Low64;
+        pCtx->pXStateR0->x87.aRegs[1].au64[1] = paValues[iReg].Fp.AsUINT128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aRegs[2].au64[0] = paValues[iReg].Fp.AsUINT128.Low64;
+        pCtx->pXStateR0->x87.aRegs[2].au64[1] = paValues[iReg].Fp.AsUINT128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aRegs[3].au64[0] = paValues[iReg].Fp.AsUINT128.Low64;
+        pCtx->pXStateR0->x87.aRegs[3].au64[1] = paValues[iReg].Fp.AsUINT128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aRegs[4].au64[0] = paValues[iReg].Fp.AsUINT128.Low64;
+        pCtx->pXStateR0->x87.aRegs[4].au64[1] = paValues[iReg].Fp.AsUINT128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aRegs[5].au64[0] = paValues[iReg].Fp.AsUINT128.Low64;
+        pCtx->pXStateR0->x87.aRegs[5].au64[1] = paValues[iReg].Fp.AsUINT128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aRegs[6].au64[0] = paValues[iReg].Fp.AsUINT128.Low64;
+        pCtx->pXStateR0->x87.aRegs[6].au64[1] = paValues[iReg].Fp.AsUINT128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aRegs[7].au64[0] = paValues[iReg].Fp.AsUINT128.Low64;
+        pCtx->pXStateR0->x87.aRegs[7].au64[1] = paValues[iReg].Fp.AsUINT128.High64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterFpControlStatus);
+        pCtx->pXStateR0->x87.FCW        = paValues[iReg].FpControlStatus.FpControl;
+        pCtx->pXStateR0->x87.FSW        = paValues[iReg].FpControlStatus.FpStatus;
+        pCtx->pXStateR0->x87.FTW        = paValues[iReg].FpControlStatus.FpTag
+                                        /*| (paValues[iReg].FpControlStatus.Reserved << 8)*/;
+        pCtx->pXStateR0->x87.FOP        = paValues[iReg].FpControlStatus.LastFpOp;
+        pCtx->pXStateR0->x87.FPUIP      = (uint32_t)paValues[iReg].FpControlStatus.LastFpRip;
+        pCtx->pXStateR0->x87.CS         = (uint16_t)(paValues[iReg].FpControlStatus.LastFpRip >> 32);
+        pCtx->pXStateR0->x87.Rsrvd1     = (uint16_t)(paValues[iReg].FpControlStatus.LastFpRip >> 48);
+        iReg++;
+    }
+
+    if (fWhat & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX))
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterXmmControlStatus);
+        if (fWhat & CPUMCTX_EXTRN_X87)
+        {
+            pCtx->pXStateR0->x87.FPUDP  = (uint32_t)paValues[iReg].XmmControlStatus.LastFpRdp;
+            pCtx->pXStateR0->x87.DS     = (uint16_t)(paValues[iReg].XmmControlStatus.LastFpRdp >> 32);
+            pCtx->pXStateR0->x87.Rsrvd2 = (uint16_t)(paValues[iReg].XmmControlStatus.LastFpRdp >> 48);
+        }
+        pCtx->pXStateR0->x87.MXCSR      = paValues[iReg].XmmControlStatus.XmmStatusControl;
+        pCtx->pXStateR0->x87.MXCSR_MASK = paValues[iReg].XmmControlStatus.XmmStatusControlMask; /** @todo ??? (Isn't this an output field?) */
+        iReg++;
+    }
+
+    /* Vector state. */
+    if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterXmm0);
+        Assert(pInput->Names[iReg+15] == HvX64RegisterXmm15);
+        pCtx->pXStateR0->x87.aXMM[0].uXmm.s.Lo  = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[0].uXmm.s.Hi  = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[1].uXmm.s.Lo  = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[1].uXmm.s.Hi  = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[2].uXmm.s.Lo  = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[2].uXmm.s.Hi  = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[3].uXmm.s.Lo  = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[3].uXmm.s.Hi  = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[4].uXmm.s.Lo  = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[4].uXmm.s.Hi  = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[5].uXmm.s.Lo  = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[5].uXmm.s.Hi  = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[6].uXmm.s.Lo  = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[6].uXmm.s.Hi  = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[7].uXmm.s.Lo  = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[7].uXmm.s.Hi  = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[8].uXmm.s.Lo  = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[8].uXmm.s.Hi  = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[9].uXmm.s.Lo  = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[9].uXmm.s.Hi  = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[10].uXmm.s.Lo = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[10].uXmm.s.Hi = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[11].uXmm.s.Lo = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[11].uXmm.s.Hi = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[12].uXmm.s.Lo = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[12].uXmm.s.Hi = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[13].uXmm.s.Lo = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[13].uXmm.s.Hi = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[14].uXmm.s.Lo = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[14].uXmm.s.Hi = paValues[iReg].Reg128.High64;
+        iReg++;
+        pCtx->pXStateR0->x87.aXMM[15].uXmm.s.Lo = paValues[iReg].Reg128.Low64;
+        pCtx->pXStateR0->x87.aXMM[15].uXmm.s.Hi = paValues[iReg].Reg128.High64;
+        iReg++;
+    }
+
+
+    /* MSRs */
+    // HvX64RegisterTsc - don't touch
+    if (fWhat & CPUMCTX_EXTRN_EFER)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterEfer);
+        if (paValues[iReg].Reg64 != pCtx->msrEFER)
+        {
+            Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtx->msrEFER, paValues[iReg].Reg64));
+            if ((paValues[iReg].Reg64 ^ pCtx->msrEFER) & MSR_K6_EFER_NXE)
+                PGMNotifyNxeChanged(pGVCpu, RT_BOOL(paValues[iReg].Reg64 & MSR_K6_EFER_NXE));
+            pCtx->msrEFER = paValues[iReg].Reg64;
+            fMaybeChangedMode = true;
+        }
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterKernelGsBase);
+        if (pCtx->msrKERNELGSBASE != paValues[iReg].Reg64)
+            Log7(("NEM/%u: MSR KERNELGSBASE changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtx->msrKERNELGSBASE, paValues[iReg].Reg64));
+        pCtx->msrKERNELGSBASE = paValues[iReg].Reg64;
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterSysenterCs);
+        if (pCtx->SysEnter.cs != paValues[iReg].Reg64)
+            Log7(("NEM/%u: MSR SYSENTER.CS changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtx->SysEnter.cs, paValues[iReg].Reg64));
+        pCtx->SysEnter.cs = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterSysenterEip);
+        if (pCtx->SysEnter.eip != paValues[iReg].Reg64)
+            Log7(("NEM/%u: MSR SYSENTER.EIP changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtx->SysEnter.eip, paValues[iReg].Reg64));
+        pCtx->SysEnter.eip = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterSysenterEsp);
+        if (pCtx->SysEnter.esp != paValues[iReg].Reg64)
+            Log7(("NEM/%u: MSR SYSENTER.ESP changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtx->SysEnter.esp, paValues[iReg].Reg64));
+        pCtx->SysEnter.esp = paValues[iReg].Reg64;
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterStar);
+        if (pCtx->msrSTAR != paValues[iReg].Reg64)
+            Log7(("NEM/%u: MSR STAR changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtx->msrSTAR, paValues[iReg].Reg64));
+        pCtx->msrSTAR   = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterLstar);
+        if (pCtx->msrLSTAR != paValues[iReg].Reg64)
+            Log7(("NEM/%u: MSR LSTAR changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtx->msrLSTAR, paValues[iReg].Reg64));
+        pCtx->msrLSTAR  = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterCstar);
+        if (pCtx->msrCSTAR != paValues[iReg].Reg64)
+            Log7(("NEM/%u: MSR CSTAR changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtx->msrCSTAR, paValues[iReg].Reg64));
+        pCtx->msrCSTAR  = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterSfmask);
+        if (pCtx->msrSFMASK != paValues[iReg].Reg64)
+            Log7(("NEM/%u: MSR SFMASK changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtx->msrSFMASK, paValues[iReg].Reg64));
+        pCtx->msrSFMASK = paValues[iReg].Reg64;
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
+    {
+        Assert(pInput->Names[iReg] == HvX64RegisterApicBase);
+        const uint64_t uOldBase = APICGetBaseMsrNoCheck(pGVCpu);
+        if (paValues[iReg].Reg64 != uOldBase)
+        {
+            Log7(("NEM/%u: MSR APICBase changed %RX64 -> %RX64 (%RX64)\n",
+                  pGVCpu->idCpu, uOldBase, paValues[iReg].Reg64, paValues[iReg].Reg64 ^ uOldBase));
+            int rc2 = APICSetBaseMsr(pGVCpu, paValues[iReg].Reg64);
+            AssertLogRelMsg(rc2 == VINF_SUCCESS, ("rc2=%Rrc [%#RX64]\n", rc2, paValues[iReg].Reg64));
+        }
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterPat);
+        if (pCtx->msrPAT != paValues[iReg].Reg64)
+            Log7(("NEM/%u: MSR PAT changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtx->msrPAT, paValues[iReg].Reg64));
+        pCtx->msrPAT    = paValues[iReg].Reg64;
+        iReg++;
+
+# if 0 /*def LOG_ENABLED*/ /** @todo something's wrong with HvX64RegisterMtrrCap? (AMD) */
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrCap);
+        if (paValues[iReg].Reg64 != CPUMGetGuestIa32MtrrCap(pGVCpu))
+            Log7(("NEM/%u: MSR MTRR_CAP changed %RX64 -> %RX64 (!!)\n", pGVCpu->idCpu, CPUMGetGuestIa32MtrrCap(pGVCpu), paValues[iReg].Reg64));
+        iReg++;
+# endif
+
+        PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pGVCpu);
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrDefType);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.MtrrDefType )
+            Log7(("NEM/%u: MSR MTRR_DEF_TYPE changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MtrrDefType, paValues[iReg].Reg64));
+        pCtxMsrs->msr.MtrrDefType = paValues[iReg].Reg64;
+        iReg++;
+
+        /** @todo we dont keep state for HvX64RegisterMtrrPhysBaseX and HvX64RegisterMtrrPhysMaskX */
+
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrFix64k00000);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.MtrrFix64K_00000 )
+            Log7(("NEM/%u: MSR MTRR_FIX16K_00000 changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MtrrFix64K_00000, paValues[iReg].Reg64));
+        pCtxMsrs->msr.MtrrFix64K_00000 = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrFix16k80000);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.MtrrFix16K_80000 )
+            Log7(("NEM/%u: MSR MTRR_FIX16K_80000 changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MtrrFix16K_80000, paValues[iReg].Reg64));
+        pCtxMsrs->msr.MtrrFix16K_80000 = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrFix16kA0000);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.MtrrFix16K_A0000 )
+            Log7(("NEM/%u: MSR MTRR_FIX16K_A0000 changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MtrrFix16K_A0000, paValues[iReg].Reg64));
+        pCtxMsrs->msr.MtrrFix16K_A0000 = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrFix4kC0000);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.MtrrFix4K_C0000 )
+            Log7(("NEM/%u: MSR MTRR_FIX16K_C0000 changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MtrrFix4K_C0000, paValues[iReg].Reg64));
+        pCtxMsrs->msr.MtrrFix4K_C0000 = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrFix4kC8000);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.MtrrFix4K_C8000 )
+            Log7(("NEM/%u: MSR MTRR_FIX16K_C8000 changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MtrrFix4K_C8000, paValues[iReg].Reg64));
+        pCtxMsrs->msr.MtrrFix4K_C8000 = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrFix4kD0000);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.MtrrFix4K_D0000 )
+            Log7(("NEM/%u: MSR MTRR_FIX16K_D0000 changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MtrrFix4K_D0000, paValues[iReg].Reg64));
+        pCtxMsrs->msr.MtrrFix4K_D0000 = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrFix4kD8000);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.MtrrFix4K_D8000 )
+            Log7(("NEM/%u: MSR MTRR_FIX16K_D8000 changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MtrrFix4K_D8000, paValues[iReg].Reg64));
+        pCtxMsrs->msr.MtrrFix4K_D8000 = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrFix4kE0000);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.MtrrFix4K_E0000 )
+            Log7(("NEM/%u: MSR MTRR_FIX16K_E0000 changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MtrrFix4K_E0000, paValues[iReg].Reg64));
+        pCtxMsrs->msr.MtrrFix4K_E0000 = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrFix4kE8000);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.MtrrFix4K_E8000 )
+            Log7(("NEM/%u: MSR MTRR_FIX16K_E8000 changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MtrrFix4K_E8000, paValues[iReg].Reg64));
+        pCtxMsrs->msr.MtrrFix4K_E8000 = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrFix4kF0000);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.MtrrFix4K_F0000 )
+            Log7(("NEM/%u: MSR MTRR_FIX16K_F0000 changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MtrrFix4K_F0000, paValues[iReg].Reg64));
+        pCtxMsrs->msr.MtrrFix4K_F0000 = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterMtrrFix4kF8000);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.MtrrFix4K_F8000 )
+            Log7(("NEM/%u: MSR MTRR_FIX16K_F8000 changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MtrrFix4K_F8000, paValues[iReg].Reg64));
+        pCtxMsrs->msr.MtrrFix4K_F8000 = paValues[iReg].Reg64;
+        iReg++;
+
+        Assert(pInput->Names[iReg] == HvX64RegisterTscAux);
+        if (paValues[iReg].Reg64 != pCtxMsrs->msr.TscAux )
+            Log7(("NEM/%u: MSR TSC_AUX changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.TscAux, paValues[iReg].Reg64));
+        pCtxMsrs->msr.TscAux = paValues[iReg].Reg64;
+        iReg++;
+
+# if 0 /** @todo why can't we even read HvX64RegisterIa32MiscEnable? */
+        if (enmCpuVendor != CPUMCPUVENDOR_AMD)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterIa32MiscEnable);
+            if (paValues[iReg].Reg64 != pCtxMsrs->msr.MiscEnable)
+                Log7(("NEM/%u: MSR MISC_ENABLE changed %RX64 -> %RX64\n", pGVCpu->idCpu, pCtxMsrs->msr.MiscEnable, paValues[iReg].Reg64));
+            pCtxMsrs->msr.MiscEnable = paValues[iReg].Reg64;
+            iReg++;
+        }
+# endif
+# ifdef LOG_ENABLED
+        if (enmCpuVendor != CPUMCPUVENDOR_AMD && enmCpuVendor != CPUMCPUVENDOR_HYGON)
+        {
+            Assert(pInput->Names[iReg] == HvX64RegisterIa32FeatureControl);
+            if (paValues[iReg].Reg64 != pCtx->hwvirt.vmx.Msrs.u64FeatCtrl)
+                Log7(("NEM/%u: MSR FEATURE_CONTROL changed %RX64 -> %RX64 (!!)\n", pGVCpu->idCpu, pCtx->hwvirt.vmx.Msrs.u64FeatCtrl, paValues[iReg].Reg64));
+            iReg++;
+        }
+# endif
+    }
+
+    /* Interruptibility. */
+    if (fWhat & (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
+    {
+        Assert(pInput->Names[iReg] == HvRegisterInterruptState);
+        Assert(pInput->Names[iReg + 1] == HvX64RegisterRip);
+
+        if (!(pCtx->fExtrn & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT))
+        {
+            pGVCpu->nem.s.fLastInterruptShadow = paValues[iReg].InterruptState.InterruptShadow;
+            if (paValues[iReg].InterruptState.InterruptShadow)
+                EMSetInhibitInterruptsPC(pGVCpu, paValues[iReg + 1].Reg64);
+            else
+                VMCPU_FF_CLEAR(pGVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+        }
+
+        if (!(pCtx->fExtrn & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
+        {
+            if (paValues[iReg].InterruptState.NmiMasked)
+                VMCPU_FF_SET(pGVCpu, VMCPU_FF_BLOCK_NMIS);
+            else
+                VMCPU_FF_CLEAR(pGVCpu, VMCPU_FF_BLOCK_NMIS);
+        }
+
+        fWhat |= CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI;
+        iReg += 2;
+    }
+
+    /* Event injection. */
+    /// @todo HvRegisterPendingInterruption
+    Assert(pInput->Names[iReg] == HvRegisterPendingInterruption);
+    if (paValues[iReg].PendingInterruption.InterruptionPending)
+    {
+        Log7(("PendingInterruption: type=%u vector=%#x errcd=%RTbool/%#x instr-len=%u nested=%u\n",
+              paValues[iReg].PendingInterruption.InterruptionType, paValues[iReg].PendingInterruption.InterruptionVector,
+              paValues[iReg].PendingInterruption.DeliverErrorCode, paValues[iReg].PendingInterruption.ErrorCode,
+              paValues[iReg].PendingInterruption.InstructionLength, paValues[iReg].PendingInterruption.NestedEvent));
+        AssertMsg((paValues[iReg].PendingInterruption.AsUINT64 & UINT64_C(0xfc00)) == 0,
+                  ("%#RX64\n", paValues[iReg].PendingInterruption.AsUINT64));
+    }
+
+    /// @todo HvRegisterPendingEvent0
+    /// @todo HvRegisterPendingEvent1
+
+    /* Almost done, just update extrn flags and maybe change PGM mode. */
+    pCtx->fExtrn &= ~fWhat;
+    if (!(pCtx->fExtrn & (CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT))))
+        pCtx->fExtrn = 0;
+
+    /* Typical. */
+    if (!fMaybeChangedMode && !fUpdateCr3)
+        return VINF_SUCCESS;
+
+    /*
+     * Slow.
+     */
+    int rc = VINF_SUCCESS;
+    if (fMaybeChangedMode)
+    {
+        rc = PGMChangeMode(pGVCpu, pCtx->cr0, pCtx->cr4, pCtx->msrEFER);
+        AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
+    }
+
+    if (fUpdateCr3)
+    {
+        if (fCanUpdateCr3)
+        {
+            LogFlow(("nemR0WinImportState: -> PGMUpdateCR3!\n"));
+            rc = PGMUpdateCR3(pGVCpu, pCtx->cr3);
+            AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
+        }
+        else
+        {
+            LogFlow(("nemR0WinImportState: -> VERR_NEM_FLUSH_TLB!\n"));
+            rc = VERR_NEM_FLUSH_TLB; /* Calling PGMFlushTLB w/o long jump setup doesn't work, ring-3 does it. */
+        }
+    }
+
+    return rc;
+}
+#endif /* NEM_WIN_WITH_RING0_RUNLOOP || NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
+
+
+/**
+ * Import the state from the native API (back to CPUMCTX).
+ *
+ * @returns VBox status code
+ * @param   pGVM        The ring-0 VM handle.
+ * @param   idCpu       The calling EMT.  Necessary for getting the
+ *                      hypercall page and arguments.
+ * @param   fWhat       What to import, CPUMCTX_EXTRN_XXX. Set
+ *                      CPUMCTX_EXTERN_ALL for everything.
+ */
+VMMR0_INT_DECL(int) NEMR0ImportState(PGVM pGVM, VMCPUID idCpu, uint64_t fWhat)
+{
+#if defined(NEM_WIN_WITH_RING0_RUNLOOP) || defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
+    /*
+     * Validate the call.
+     */
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_SUCCESS(rc))
+    {
+        PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+        AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API_1);
+
+        /*
+         * Call worker.
+         */
+        rc = nemR0WinImportState(pGVM, pGVCpu, &pGVCpu->cpum.GstCtx, fWhat, false /*fCanUpdateCr3*/);
+    }
+    return rc;
+#else
+    RT_NOREF(pGVM, idCpu, fWhat);
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+#if defined(NEM_WIN_WITH_RING0_RUNLOOP) || defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
+/**
+ * Worker for NEMR0QueryCpuTick and the ring-0 NEMHCQueryCpuTick.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The ring-0 VM handle.
+ * @param   pGVCpu      The ring-0 VCPU handle.
+ * @param   pcTicks     Where to return the current CPU tick count.
+ * @param   pcAux       Where to return the hyper-V TSC_AUX value.  Optional.
+ */
+NEM_TMPL_STATIC int nemR0WinQueryCpuTick(PGVM pGVM, PGVMCPU pGVCpu, uint64_t *pcTicks, uint32_t *pcAux)
+{
+    /*
+     * Hypercall parameters.
+     */
+    HV_INPUT_GET_VP_REGISTERS *pInput = (HV_INPUT_GET_VP_REGISTERS *)pGVCpu->nemr0.s.HypercallData.pbPage;
+    AssertPtrReturn(pInput, VERR_INTERNAL_ERROR_3);
+    AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API_1);
+
+    pInput->PartitionId = pGVM->nemr0.s.idHvPartition;
+    pInput->VpIndex     = pGVCpu->idCpu;
+    pInput->fFlags      = 0;
+    pInput->Names[0]    = HvX64RegisterTsc;
+    pInput->Names[1]    = HvX64RegisterTscAux;
+
+    size_t const cbInput = RT_ALIGN_Z(RT_UOFFSETOF(HV_INPUT_GET_VP_REGISTERS, Names[2]), 32);
+    HV_REGISTER_VALUE *paValues = (HV_REGISTER_VALUE *)((uint8_t *)pInput + cbInput);
+    RT_BZERO(paValues, sizeof(paValues[0]) * 2);
+
+    /*
+     * Make the hypercall.
+     */
+    uint64_t uResult = g_pfnHvlInvokeHypercall(HV_MAKE_CALL_INFO(HvCallGetVpRegisters, 2),
+                                               pGVCpu->nemr0.s.HypercallData.HCPhysPage,
+                                               pGVCpu->nemr0.s.HypercallData.HCPhysPage + cbInput);
+    AssertLogRelMsgReturn(uResult == HV_MAKE_CALL_REP_RET(2), ("uResult=%RX64 cRegs=%#x\n", uResult, 2),
+                          VERR_NEM_GET_REGISTERS_FAILED);
+
+    /*
+     * Get results.
+     */
+    *pcTicks = paValues[0].Reg64;
+    if (pcAux)
+        *pcAux = paValues[0].Reg32;
+    return VINF_SUCCESS;
+}
+#endif /* NEM_WIN_WITH_RING0_RUNLOOP || NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
+
+
+/**
+ * Queries the TSC and TSC_AUX values, putting the results in .
+ *
+ * @returns VBox status code
+ * @param   pGVM        The ring-0 VM handle.
+ * @param   idCpu       The calling EMT.  Necessary for getting the
+ *                      hypercall page and arguments.
+ */
+VMMR0_INT_DECL(int) NEMR0QueryCpuTick(PGVM pGVM, VMCPUID idCpu)
+{
+#if defined(NEM_WIN_WITH_RING0_RUNLOOP) || defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
+    /*
+     * Validate the call.
+     */
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_SUCCESS(rc))
+    {
+        PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+        AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API_1);
+
+        /*
+         * Call worker.
+         */
+        pGVCpu->nem.s.Hypercall.QueryCpuTick.cTicks = 0;
+        pGVCpu->nem.s.Hypercall.QueryCpuTick.uAux   = 0;
+        rc = nemR0WinQueryCpuTick(pGVM, pGVCpu, &pGVCpu->nem.s.Hypercall.QueryCpuTick.cTicks,
+                                  &pGVCpu->nem.s.Hypercall.QueryCpuTick.uAux);
+    }
+    return rc;
+#else
+    RT_NOREF(pGVM, idCpu);
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+#if defined(NEM_WIN_WITH_RING0_RUNLOOP) || defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
+/**
+ * Worker for NEMR0ResumeCpuTickOnAll and the ring-0 NEMHCResumeCpuTickOnAll.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The ring-0 VM handle.
+ * @param   pGVCpu          The ring-0 VCPU handle.
+ * @param   uPausedTscValue The TSC value at the time of pausing.
+ */
+NEM_TMPL_STATIC int nemR0WinResumeCpuTickOnAll(PGVM pGVM, PGVMCPU pGVCpu, uint64_t uPausedTscValue)
+{
+    AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API_1);
+
+    /*
+     * Set up the hypercall parameters.
+     */
+    HV_INPUT_SET_VP_REGISTERS *pInput = (HV_INPUT_SET_VP_REGISTERS *)pGVCpu->nemr0.s.HypercallData.pbPage;
+    AssertPtrReturn(pInput, VERR_INTERNAL_ERROR_3);
+
+    pInput->PartitionId = pGVM->nemr0.s.idHvPartition;
+    pInput->VpIndex     = 0;
+    pInput->RsvdZ       = 0;
+    pInput->Elements[0].Name  = HvX64RegisterTsc;
+    pInput->Elements[0].Pad0  = 0;
+    pInput->Elements[0].Pad1  = 0;
+    pInput->Elements[0].Value.Reg128.High64 = 0;
+    pInput->Elements[0].Value.Reg64 = uPausedTscValue;
+
+    /*
+     * Disable interrupts and do the first virtual CPU.
+     */
+    RTCCINTREG const fSavedFlags = ASMIntDisableFlags();
+    uint64_t const   uFirstTsc   = ASMReadTSC();
+    uint64_t uResult = g_pfnHvlInvokeHypercall(HV_MAKE_CALL_INFO(HvCallSetVpRegisters, 1),
+                                               pGVCpu->nemr0.s.HypercallData.HCPhysPage, 0 /* no output */);
+    AssertLogRelMsgReturnStmt(uResult == HV_MAKE_CALL_REP_RET(1), ("uResult=%RX64 uTsc=%#RX64\n", uResult, uPausedTscValue),
+                              ASMSetFlags(fSavedFlags), VERR_NEM_SET_TSC);
+
+    /*
+     * Do secondary processors, adjusting for elapsed TSC and keeping finger crossed
+     * that we don't introduce too much drift here.
+     */
+    for (VMCPUID iCpu = 1; iCpu < pGVM->cCpus; iCpu++)
+    {
+        Assert(pInput->PartitionId == pGVM->nemr0.s.idHvPartition);
+        Assert(pInput->RsvdZ       == 0);
+        Assert(pInput->Elements[0].Name == HvX64RegisterTsc);
+        Assert(pInput->Elements[0].Pad0 == 0);
+        Assert(pInput->Elements[0].Pad1 == 0);
+        Assert(pInput->Elements[0].Value.Reg128.High64 == 0);
+
+        pInput->VpIndex = iCpu;
+        const uint64_t offDelta = (ASMReadTSC() - uFirstTsc);
+        pInput->Elements[0].Value.Reg64 = uPausedTscValue + offDelta;
+
+        uResult = g_pfnHvlInvokeHypercall(HV_MAKE_CALL_INFO(HvCallSetVpRegisters, 1),
+                                          pGVCpu->nemr0.s.HypercallData.HCPhysPage, 0 /* no output */);
+        AssertLogRelMsgReturnStmt(uResult == HV_MAKE_CALL_REP_RET(1),
+                                  ("uResult=%RX64 uTsc=%#RX64 + %#RX64\n", uResult, uPausedTscValue, offDelta),
+                                  ASMSetFlags(fSavedFlags), VERR_NEM_SET_TSC);
+    }
+
+    /*
+     * Done.
+     */
+    ASMSetFlags(fSavedFlags);
+    return VINF_SUCCESS;
+}
+#endif /* NEM_WIN_WITH_RING0_RUNLOOP || NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
+
+
+/**
+ * Sets the TSC register to @a uPausedTscValue on all CPUs.
+ *
+ * @returns VBox status code
+ * @param   pGVM            The ring-0 VM handle.
+ * @param   idCpu           The calling EMT.  Necessary for getting the
+ *                          hypercall page and arguments.
+ * @param   uPausedTscValue The TSC value at the time of pausing.
+ */
+VMMR0_INT_DECL(int) NEMR0ResumeCpuTickOnAll(PGVM pGVM, VMCPUID idCpu, uint64_t uPausedTscValue)
+{
+#if defined(NEM_WIN_WITH_RING0_RUNLOOP) || defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
+    /*
+     * Validate the call.
+     */
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_SUCCESS(rc))
+    {
+        PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+        AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API_1);
+
+        /*
+         * Call worker.
+         */
+        pGVCpu->nem.s.Hypercall.QueryCpuTick.cTicks = 0;
+        pGVCpu->nem.s.Hypercall.QueryCpuTick.uAux   = 0;
+        rc = nemR0WinResumeCpuTickOnAll(pGVM, pGVCpu, uPausedTscValue);
+    }
+    return rc;
+#else
+    RT_NOREF(pGVM, idCpu, uPausedTscValue);
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+VMMR0_INT_DECL(VBOXSTRICTRC) NEMR0RunGuestCode(PGVM pGVM, VMCPUID idCpu)
+{
+#ifdef NEM_WIN_WITH_RING0_RUNLOOP
+    if (pGVM->nemr0.s.fMayUseRing0Runloop)
+        return nemHCWinRunGC(pGVM, &pGVM->aCpus[idCpu]);
+    return VERR_NEM_RING3_ONLY;
+#else
+    RT_NOREF(pGVM, idCpu);
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/**
+ * Updates statistics in the VM structure.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The ring-0 VM handle.
+ * @param   idCpu       The calling EMT, or NIL.  Necessary for getting the hypercall
+ *                      page and arguments.
+ */
+VMMR0_INT_DECL(int)  NEMR0UpdateStatistics(PGVM pGVM, VMCPUID idCpu)
+{
+    /*
+     * Validate the call.
+     */
+    int rc;
+    if (idCpu == NIL_VMCPUID)
+        rc = GVMMR0ValidateGVM(pGVM);
+    else
+        rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_SUCCESS(rc))
+    {
+        AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API_1);
+
+        PNEMR0HYPERCALLDATA pHypercallData = idCpu != NIL_VMCPUID
+                                           ? &pGVM->aCpus[idCpu].nemr0.s.HypercallData
+                                           : &pGVM->nemr0.s.HypercallData;
+        if (   RT_VALID_PTR(pHypercallData->pbPage)
+            && pHypercallData->HCPhysPage != NIL_RTHCPHYS)
+        {
+            if (idCpu == NIL_VMCPUID)
+                rc = RTCritSectEnter(&pGVM->nemr0.s.HypercallDataCritSect);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Query the memory statistics for the partition.
+                 */
+                HV_INPUT_GET_MEMORY_BALANCE  *pInput = (HV_INPUT_GET_MEMORY_BALANCE *)pHypercallData->pbPage;
+                pInput->TargetPartitionId                               = pGVM->nemr0.s.idHvPartition;
+                pInput->ProximityDomainInfo.Flags.ProximityPreferred    = 0;
+                pInput->ProximityDomainInfo.Flags.ProxyimityInfoValid   = 0;
+                pInput->ProximityDomainInfo.Flags.Reserved              = 0;
+                pInput->ProximityDomainInfo.Id                          = 0;
+
+                HV_OUTPUT_GET_MEMORY_BALANCE *pOutput = (HV_OUTPUT_GET_MEMORY_BALANCE *)(pInput + 1);
+                RT_ZERO(*pOutput);
+
+                uint64_t uResult = g_pfnHvlInvokeHypercall(HvCallGetMemoryBalance,
+                                                           pHypercallData->HCPhysPage,
+                                                           pHypercallData->HCPhysPage + sizeof(*pInput));
+                if (uResult == HV_STATUS_SUCCESS)
+                {
+                    pGVM->nem.s.R0Stats.cPagesAvailable = pOutput->PagesAvailable;
+                    pGVM->nem.s.R0Stats.cPagesInUse     = pOutput->PagesInUse;
+                    rc = VINF_SUCCESS;
+                }
+                else
+                {
+                    LogRel(("HvCallGetMemoryBalance -> %#RX64 (%#RX64 %#RX64)!!\n",
+                            uResult, pOutput->PagesAvailable, pOutput->PagesInUse));
+                    rc = VERR_NEM_IPE_0;
+                }
+
+                if (idCpu == NIL_VMCPUID)
+                    RTCritSectLeave(&pGVM->nemr0.s.HypercallDataCritSect);
+            }
+        }
+        else
+            rc = VERR_WRONG_ORDER;
+    }
+    return rc;
+}
+
+
+#if 1 && defined(DEBUG_bird)
+/**
+ * Debug only interface for poking around and exploring Hyper-V stuff.
+ *
+ * @param   pGVM        The ring-0 VM handle.
+ * @param   idCpu       The calling EMT.
+ * @param   u64Arg      What to query.  0 == registers.
+ */
+VMMR0_INT_DECL(int) NEMR0DoExperiment(PGVM pGVM, VMCPUID idCpu, uint64_t u64Arg)
+{
+    /*
+     * Resolve CPU structures.
+     */
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    if (RT_SUCCESS(rc))
+    {
+        AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API_1);
+
+        PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+        if (u64Arg == 0)
+        {
+            /*
+             * Query register.
+             */
+            HV_INPUT_GET_VP_REGISTERS *pInput = (HV_INPUT_GET_VP_REGISTERS *)pGVCpu->nemr0.s.HypercallData.pbPage;
+            AssertPtrReturn(pInput, VERR_INTERNAL_ERROR_3);
+
+            size_t const cbInput = RT_ALIGN_Z(RT_UOFFSETOF(HV_INPUT_GET_VP_REGISTERS, Names[1]), 32);
+            HV_REGISTER_VALUE *paValues = (HV_REGISTER_VALUE *)((uint8_t *)pInput + cbInput);
+            RT_BZERO(paValues, sizeof(paValues[0]) * 1);
+
+            pInput->PartitionId = pGVM->nemr0.s.idHvPartition;
+            pInput->VpIndex     = pGVCpu->idCpu;
+            pInput->fFlags      = 0;
+            pInput->Names[0]    = (HV_REGISTER_NAME)pGVCpu->nem.s.Hypercall.Experiment.uItem;
+
+            uint64_t uResult = g_pfnHvlInvokeHypercall(HV_MAKE_CALL_INFO(HvCallGetVpRegisters, 1),
+                                                       pGVCpu->nemr0.s.HypercallData.HCPhysPage,
+                                                       pGVCpu->nemr0.s.HypercallData.HCPhysPage + cbInput);
+            pGVCpu->nem.s.Hypercall.Experiment.fSuccess = uResult == HV_MAKE_CALL_REP_RET(1);
+            pGVCpu->nem.s.Hypercall.Experiment.uStatus  = uResult;
+            pGVCpu->nem.s.Hypercall.Experiment.uLoValue = paValues[0].Reg128.Low64;
+            pGVCpu->nem.s.Hypercall.Experiment.uHiValue = paValues[0].Reg128.High64;
+            rc = VINF_SUCCESS;
+        }
+        else if (u64Arg == 1)
+        {
+            /*
+             * Query partition property.
+             */
+            HV_INPUT_GET_PARTITION_PROPERTY *pInput = (HV_INPUT_GET_PARTITION_PROPERTY *)pGVCpu->nemr0.s.HypercallData.pbPage;
+            AssertPtrReturn(pInput, VERR_INTERNAL_ERROR_3);
+
+            size_t const cbInput = RT_ALIGN_Z(sizeof(*pInput), 32);
+            HV_OUTPUT_GET_PARTITION_PROPERTY *pOutput = (HV_OUTPUT_GET_PARTITION_PROPERTY *)((uint8_t *)pInput + cbInput);
+            pOutput->PropertyValue = 0;
+
+            pInput->PartitionId  = pGVM->nemr0.s.idHvPartition;
+            pInput->PropertyCode = (HV_PARTITION_PROPERTY_CODE)pGVCpu->nem.s.Hypercall.Experiment.uItem;
+            pInput->uPadding     = 0;
+
+            uint64_t uResult = g_pfnHvlInvokeHypercall(HvCallGetPartitionProperty,
+                                                       pGVCpu->nemr0.s.HypercallData.HCPhysPage,
+                                                       pGVCpu->nemr0.s.HypercallData.HCPhysPage + cbInput);
+            pGVCpu->nem.s.Hypercall.Experiment.fSuccess = uResult == HV_STATUS_SUCCESS;
+            pGVCpu->nem.s.Hypercall.Experiment.uStatus  = uResult;
+            pGVCpu->nem.s.Hypercall.Experiment.uLoValue = pOutput->PropertyValue;
+            pGVCpu->nem.s.Hypercall.Experiment.uHiValue = 0;
+            rc = VINF_SUCCESS;
+        }
+        else if (u64Arg == 2)
+        {
+            /*
+             * Set register.
+             */
+            HV_INPUT_SET_VP_REGISTERS *pInput = (HV_INPUT_SET_VP_REGISTERS *)pGVCpu->nemr0.s.HypercallData.pbPage;
+            AssertPtrReturn(pInput, VERR_INTERNAL_ERROR_3);
+            RT_BZERO(pInput, RT_UOFFSETOF(HV_INPUT_SET_VP_REGISTERS, Elements[1]));
+
+            pInput->PartitionId = pGVM->nemr0.s.idHvPartition;
+            pInput->VpIndex     = pGVCpu->idCpu;
+            pInput->RsvdZ      = 0;
+            pInput->Elements[0].Name = (HV_REGISTER_NAME)pGVCpu->nem.s.Hypercall.Experiment.uItem;
+            pInput->Elements[0].Value.Reg128.High64 = pGVCpu->nem.s.Hypercall.Experiment.uHiValue;
+            pInput->Elements[0].Value.Reg128.Low64  = pGVCpu->nem.s.Hypercall.Experiment.uLoValue;
+
+            uint64_t uResult = g_pfnHvlInvokeHypercall(HV_MAKE_CALL_INFO(HvCallSetVpRegisters, 1),
+                                                       pGVCpu->nemr0.s.HypercallData.HCPhysPage, 0);
+            pGVCpu->nem.s.Hypercall.Experiment.fSuccess = uResult == HV_MAKE_CALL_REP_RET(1);
+            pGVCpu->nem.s.Hypercall.Experiment.uStatus  = uResult;
+            rc = VINF_SUCCESS;
+        }
+        else
+            rc = VERR_INVALID_FUNCTION;
+    }
+    return rc;
+}
+#endif /* DEBUG_bird */
+
diff --git a/src/VBox/VMM/VMMR0/PDMR0DevHlp.cpp b/src/VBox/VMM/VMMR0/PDMR0DevHlp.cpp
new file mode 100644
index 00000000..a9790717
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/PDMR0DevHlp.cpp
@@ -0,0 +1,1558 @@
+/* $Id: PDMR0DevHlp.cpp $ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, R0 Device Helper parts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DEVICE
+#define PDMPCIDEV_INCLUDE_PRIVATE  /* Hack to get pdmpcidevint.h included at the right point. */
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/gvm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/gvmm.h>
+
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <VBox/sup.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/string.h>
+
+#include "dtrace/VBoxVMM.h"
+#include "PDMInline.h"
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+extern DECLEXPORT(const PDMDEVHLPR0)    g_pdmR0DevHlp;
+extern DECLEXPORT(const PDMPICHLP)      g_pdmR0PicHlp;
+extern DECLEXPORT(const PDMIOAPICHLP)   g_pdmR0IoApicHlp;
+extern DECLEXPORT(const PDMPCIHLPR0)    g_pdmR0PciHlp;
+extern DECLEXPORT(const PDMHPETHLPR0)   g_pdmR0HpetHlp;
+extern DECLEXPORT(const PDMPCIRAWHLPR0) g_pdmR0PciRawHlp;
+RT_C_DECLS_END
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static bool pdmR0IsaSetIrq(PGVM pGVM, int iIrq, int iLevel, uint32_t uTagSrc);
+
+
+/** @name Ring-0 Device Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnIoPortSetUpContextEx} */
+static DECLCALLBACK(int) pdmR0DevHlp_IoPortSetUpContextEx(PPDMDEVINS pDevIns, IOMIOPORTHANDLE hIoPorts,
+                                                          PFNIOMIOPORTNEWOUT pfnOut, PFNIOMIOPORTNEWIN pfnIn,
+                                                          PFNIOMIOPORTNEWOUTSTRING pfnOutStr, PFNIOMIOPORTNEWINSTRING pfnInStr,
+                                                          void *pvUser)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_IoPortSetUpContextEx: caller='%s'/%d: hIoPorts=%#x pfnOut=%p pfnIn=%p pfnOutStr=%p pfnInStr=%p pvUser=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hIoPorts, pfnOut, pfnIn, pfnOutStr, pfnInStr, pvUser));
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+    int rc = IOMR0IoPortSetUpContext(pGVM, pDevIns, hIoPorts, pfnOut, pfnIn, pfnOutStr, pfnInStr, pvUser);
+
+    LogFlow(("pdmR0DevHlp_IoPortSetUpContextEx: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnMmioSetUpContextEx} */
+static DECLCALLBACK(int) pdmR0DevHlp_MmioSetUpContextEx(PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, PFNIOMMMIONEWWRITE pfnWrite,
+                                                        PFNIOMMMIONEWREAD pfnRead, PFNIOMMMIONEWFILL pfnFill, void *pvUser)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_MmioSetUpContextEx: caller='%s'/%d: hRegion=%#x pfnWrite=%p pfnRead=%p pfnFill=%p pvUser=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hRegion, pfnWrite, pfnRead, pfnFill, pvUser));
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+    int rc = IOMR0MmioSetUpContext(pGVM, pDevIns, hRegion, pfnWrite, pfnRead, pfnFill, pvUser);
+
+    LogFlow(("pdmR0DevHlp_MmioSetUpContextEx: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnMmio2SetUpContext} */
+static DECLCALLBACK(int) pdmR0DevHlp_Mmio2SetUpContext(PPDMDEVINS pDevIns, PGMMMIO2HANDLE hRegion,
+                                                       size_t offSub, size_t cbSub, void **ppvMapping)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_Mmio2SetUpContext: caller='%s'/%d: hRegion=%#x offSub=%#zx cbSub=%#zx ppvMapping=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hRegion, offSub, cbSub, ppvMapping));
+    *ppvMapping = NULL;
+
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+    int rc = PGMR0PhysMMIO2MapKernel(pGVM, pDevIns, hRegion, offSub, cbSub, ppvMapping);
+
+    LogFlow(("pdmR0DevHlp_Mmio2SetUpContext: caller='%s'/%d: returns %Rrc (%p)\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *ppvMapping));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnPCIPhysRead} */
+static DECLCALLBACK(int) pdmR0DevHlp_PCIPhysRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, RTGCPHYS GCPhys,
+                                                 void *pvBuf, size_t cbRead)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    if (!pPciDev) /* NULL is an alias for the default PCI device. */
+        pPciDev = pDevIns->apPciDevs[0];
+    AssertReturn(pPciDev, VERR_PDM_NOT_PCI_DEVICE);
+    PDMPCIDEV_ASSERT_VALID_AND_REGISTERED(pDevIns, pPciDev);
+
+#ifndef PDM_DO_NOT_RESPECT_PCI_BM_BIT
+    /*
+     * Just check the busmaster setting here and forward the request to the generic read helper.
+     */
+    if (PCIDevIsBusmaster(pPciDev))
+    { /* likely */ }
+    else
+    {
+        Log(("pdmRCDevHlp_PCIPhysRead: caller=%p/%d: returns %Rrc - Not bus master! GCPhys=%RGp cbRead=%#zx\n",
+             pDevIns, pDevIns->iInstance, VERR_PDM_NOT_PCI_BUS_MASTER, GCPhys, cbRead));
+        memset(pvBuf, 0xff, cbRead);
+        return VERR_PDM_NOT_PCI_BUS_MASTER;
+    }
+#endif
+
+    return pDevIns->pHlpR0->pfnPhysRead(pDevIns, GCPhys, pvBuf, cbRead);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnPCIPhysWrite} */
+static DECLCALLBACK(int) pdmR0DevHlp_PCIPhysWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, RTGCPHYS GCPhys,
+                                                  const void *pvBuf, size_t cbWrite)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    if (!pPciDev) /* NULL is an alias for the default PCI device. */
+        pPciDev = pDevIns->apPciDevs[0];
+    AssertReturn(pPciDev, VERR_PDM_NOT_PCI_DEVICE);
+    PDMPCIDEV_ASSERT_VALID_AND_REGISTERED(pDevIns, pPciDev);
+
+#ifndef PDM_DO_NOT_RESPECT_PCI_BM_BIT
+    /*
+     * Just check the busmaster setting here and forward the request to the generic read helper.
+     */
+    if (PCIDevIsBusmaster(pPciDev))
+    { /* likely */ }
+    else
+    {
+        Log(("pdmRCDevHlp_PCIPhysWrite: caller=%p/%d: returns %Rrc - Not bus master! GCPhys=%RGp cbWrite=%#zx\n",
+             pDevIns, pDevIns->iInstance, VERR_PDM_NOT_PCI_BUS_MASTER, GCPhys, cbWrite));
+        return VERR_PDM_NOT_PCI_BUS_MASTER;
+    }
+#endif
+
+    return pDevIns->pHlpR0->pfnPhysWrite(pDevIns, GCPhys, pvBuf, cbWrite);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnPCISetIrq} */
+static DECLCALLBACK(void) pdmR0DevHlp_PCISetIrq(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, int iIrq, int iLevel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    if (!pPciDev) /* NULL is an alias for the default PCI device. */
+        pPciDev = pDevIns->apPciDevs[0];
+    AssertReturnVoid(pPciDev);
+    LogFlow(("pdmR0DevHlp_PCISetIrq: caller=%p/%d: pPciDev=%p:{%#x} iIrq=%d iLevel=%d\n",
+             pDevIns, pDevIns->iInstance, pPciDev, pPciDev->uDevFn, iIrq, iLevel));
+    PDMPCIDEV_ASSERT_VALID_AND_REGISTERED(pDevIns, pPciDev);
+
+    PGVM         pGVM      = pDevIns->Internal.s.pGVM;
+    size_t const idxBus    = pPciDev->Int.s.idxPdmBus;
+    AssertReturnVoid(idxBus < RT_ELEMENTS(pGVM->pdmr0.s.aPciBuses));
+    PPDMPCIBUSR0 pPciBusR0 = &pGVM->pdmr0.s.aPciBuses[idxBus];
+
+    pdmLock(pGVM);
+
+    uint32_t uTagSrc;
+    if (iLevel & PDM_IRQ_LEVEL_HIGH)
+    {
+        pDevIns->Internal.s.pIntR3R0->uLastIrqTag = uTagSrc = pdmCalcIrqTag(pGVM, pDevIns->Internal.s.pInsR3R0->idTracing);
+        if (iLevel == PDM_IRQ_LEVEL_HIGH)
+            VBOXVMM_PDM_IRQ_HIGH(VMMGetCpu(pGVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+        else
+            VBOXVMM_PDM_IRQ_HILO(VMMGetCpu(pGVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+    }
+    else
+        uTagSrc = pDevIns->Internal.s.pIntR3R0->uLastIrqTag;
+
+    if (pPciBusR0->pDevInsR0)
+    {
+        pPciBusR0->pfnSetIrqR0(pPciBusR0->pDevInsR0, pPciDev, iIrq, iLevel, uTagSrc);
+
+        pdmUnlock(pGVM);
+
+        if (iLevel == PDM_IRQ_LEVEL_LOW)
+            VBOXVMM_PDM_IRQ_LOW(VMMGetCpu(pGVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+    }
+    else
+    {
+        pdmUnlock(pGVM);
+
+        /* queue for ring-3 execution. */
+        PPDMDEVHLPTASK pTask = (PPDMDEVHLPTASK)PDMQueueAlloc(pGVM->pdm.s.pDevHlpQueueR0);
+        AssertReturnVoid(pTask);
+
+        pTask->enmOp = PDMDEVHLPTASKOP_PCI_SET_IRQ;
+        pTask->pDevInsR3 = PDMDEVINS_2_R3PTR(pDevIns);
+        pTask->u.PciSetIRQ.iIrq = iIrq;
+        pTask->u.PciSetIRQ.iLevel = iLevel;
+        pTask->u.PciSetIRQ.uTagSrc = uTagSrc;
+        pTask->u.PciSetIRQ.pPciDevR3 = MMHyperR0ToR3(pGVM, pPciDev);
+
+        PDMQueueInsertEx(pGVM->pdm.s.pDevHlpQueueR0, &pTask->Core, 0);
+    }
+
+    LogFlow(("pdmR0DevHlp_PCISetIrq: caller=%p/%d: returns void; uTagSrc=%#x\n", pDevIns, pDevIns->iInstance, uTagSrc));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnISASetIrq} */
+static DECLCALLBACK(void) pdmR0DevHlp_ISASetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_ISASetIrq: caller=%p/%d: iIrq=%d iLevel=%d\n", pDevIns, pDevIns->iInstance, iIrq, iLevel));
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+
+    pdmLock(pGVM);
+    uint32_t uTagSrc;
+    if (iLevel & PDM_IRQ_LEVEL_HIGH)
+    {
+        pDevIns->Internal.s.pIntR3R0->uLastIrqTag = uTagSrc = pdmCalcIrqTag(pGVM, pDevIns->Internal.s.pInsR3R0->idTracing);
+        if (iLevel == PDM_IRQ_LEVEL_HIGH)
+            VBOXVMM_PDM_IRQ_HIGH(VMMGetCpu(pGVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+        else
+            VBOXVMM_PDM_IRQ_HILO(VMMGetCpu(pGVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+    }
+    else
+        uTagSrc = pDevIns->Internal.s.pIntR3R0->uLastIrqTag;
+
+    bool fRc = pdmR0IsaSetIrq(pGVM, iIrq, iLevel, uTagSrc);
+
+    if (iLevel == PDM_IRQ_LEVEL_LOW && fRc)
+        VBOXVMM_PDM_IRQ_LOW(VMMGetCpu(pGVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+    pdmUnlock(pGVM);
+    LogFlow(("pdmR0DevHlp_ISASetIrq: caller=%p/%d: returns void; uTagSrc=%#x\n", pDevIns, pDevIns->iInstance, uTagSrc));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnIoApicSendMsi} */
+static DECLCALLBACK(void) pdmR0DevHlp_IoApicSendMsi(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint32_t uValue)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_IoApicSendMsi: caller=%p/%d: GCPhys=%RGp uValue=%#x\n", pDevIns, pDevIns->iInstance, GCPhys, uValue));
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+
+    uint32_t uTagSrc;
+    pDevIns->Internal.s.pIntR3R0->uLastIrqTag = uTagSrc = pdmCalcIrqTag(pGVM, pDevIns->Internal.s.pInsR3R0->idTracing);
+    VBOXVMM_PDM_IRQ_HILO(VMMGetCpu(pGVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+
+    if (pGVM->pdm.s.IoApic.pDevInsR0)
+        pGVM->pdm.s.IoApic.pfnSendMsiR0(pGVM->pdm.s.IoApic.pDevInsR0, GCPhys, uValue, uTagSrc);
+    else
+        AssertFatalMsgFailed(("Lazy bastards!"));
+
+    LogFlow(("pdmR0DevHlp_IoApicSendMsi: caller=%p/%d: returns void; uTagSrc=%#x\n", pDevIns, pDevIns->iInstance, uTagSrc));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnPhysRead} */
+static DECLCALLBACK(int) pdmR0DevHlp_PhysRead(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_PhysRead: caller=%p/%d: GCPhys=%RGp pvBuf=%p cbRead=%#x\n",
+             pDevIns, pDevIns->iInstance, GCPhys, pvBuf, cbRead));
+
+    VBOXSTRICTRC rcStrict = PGMPhysRead(pDevIns->Internal.s.pGVM, GCPhys, pvBuf, cbRead, PGMACCESSORIGIN_DEVICE);
+    AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); /** @todo track down the users for this bugger. */
+
+    Log(("pdmR0DevHlp_PhysRead: caller=%p/%d: returns %Rrc\n", pDevIns, pDevIns->iInstance, VBOXSTRICTRC_VAL(rcStrict) ));
+    return VBOXSTRICTRC_VAL(rcStrict);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnPhysWrite} */
+static DECLCALLBACK(int) pdmR0DevHlp_PhysWrite(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_PhysWrite: caller=%p/%d: GCPhys=%RGp pvBuf=%p cbWrite=%#x\n",
+             pDevIns, pDevIns->iInstance, GCPhys, pvBuf, cbWrite));
+
+    VBOXSTRICTRC rcStrict = PGMPhysWrite(pDevIns->Internal.s.pGVM, GCPhys, pvBuf, cbWrite, PGMACCESSORIGIN_DEVICE);
+    AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); /** @todo track down the users for this bugger. */
+
+    Log(("pdmR0DevHlp_PhysWrite: caller=%p/%d: returns %Rrc\n", pDevIns, pDevIns->iInstance, VBOXSTRICTRC_VAL(rcStrict) ));
+    return VBOXSTRICTRC_VAL(rcStrict);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnA20IsEnabled} */
+static DECLCALLBACK(bool) pdmR0DevHlp_A20IsEnabled(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_A20IsEnabled: caller=%p/%d:\n", pDevIns, pDevIns->iInstance));
+
+    bool fEnabled = PGMPhysIsA20Enabled(VMMGetCpu(pDevIns->Internal.s.pGVM));
+
+    Log(("pdmR0DevHlp_A20IsEnabled: caller=%p/%d: returns %RTbool\n", pDevIns, pDevIns->iInstance, fEnabled));
+    return fEnabled;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnVMState} */
+static DECLCALLBACK(VMSTATE) pdmR0DevHlp_VMState(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+
+    VMSTATE enmVMState = pDevIns->Internal.s.pGVM->enmVMState;
+
+    LogFlow(("pdmR0DevHlp_VMState: caller=%p/%d: returns %d\n", pDevIns, pDevIns->iInstance, enmVMState));
+    return enmVMState;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnVMSetError} */
+static DECLCALLBACK(int) pdmR0DevHlp_VMSetError(PPDMDEVINS pDevIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    va_list args;
+    va_start(args, pszFormat);
+    int rc2 = VMSetErrorV(pDevIns->Internal.s.pGVM, rc, RT_SRC_POS_ARGS, pszFormat, args); Assert(rc2 == rc); NOREF(rc2);
+    va_end(args);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnVMSetErrorV} */
+static DECLCALLBACK(int) pdmR0DevHlp_VMSetErrorV(PPDMDEVINS pDevIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    int rc2 = VMSetErrorV(pDevIns->Internal.s.pGVM, rc, RT_SRC_POS_ARGS, pszFormat, va); Assert(rc2 == rc); NOREF(rc2);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnVMSetRuntimeError} */
+static DECLCALLBACK(int) pdmR0DevHlp_VMSetRuntimeError(PPDMDEVINS pDevIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    va_list va;
+    va_start(va, pszFormat);
+    int rc = VMSetRuntimeErrorV(pDevIns->Internal.s.pGVM, fFlags, pszErrorId, pszFormat, va);
+    va_end(va);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnVMSetRuntimeErrorV} */
+static DECLCALLBACK(int) pdmR0DevHlp_VMSetRuntimeErrorV(PPDMDEVINS pDevIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    int rc = VMSetRuntimeErrorV(pDevIns->Internal.s.pGVM, fFlags, pszErrorId, pszFormat, va);
+    return rc;
+}
+
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnGetVM} */
+static DECLCALLBACK(PVMCC)  pdmR0DevHlp_GetVM(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_GetVM: caller='%p'/%d\n", pDevIns, pDevIns->iInstance));
+    return pDevIns->Internal.s.pGVM;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnGetVMCPU} */
+static DECLCALLBACK(PVMCPUCC) pdmR0DevHlp_GetVMCPU(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_GetVMCPU: caller='%p'/%d\n", pDevIns, pDevIns->iInstance));
+    return VMMGetCpu(pDevIns->Internal.s.pGVM);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPRC,pfnGetCurrentCpuId} */
+static DECLCALLBACK(VMCPUID) pdmR0DevHlp_GetCurrentCpuId(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VMCPUID idCpu = VMMGetCpuId(pDevIns->Internal.s.pGVM);
+    LogFlow(("pdmR0DevHlp_GetCurrentCpuId: caller='%p'/%d for CPU %u\n", pDevIns, pDevIns->iInstance, idCpu));
+    return idCpu;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerToPtr} */
+static DECLCALLBACK(PTMTIMERR0) pdmR0DevHlp_TimerToPtr(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return (PTMTIMERR0)MMHyperR3ToCC(pDevIns->Internal.s.pGVM, hTimer);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerFromMicro} */
+static DECLCALLBACK(uint64_t) pdmR0DevHlp_TimerFromMicro(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cMicroSecs)
+{
+    return TMTimerFromMicro(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer), cMicroSecs);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerFromMilli} */
+static DECLCALLBACK(uint64_t) pdmR0DevHlp_TimerFromMilli(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cMilliSecs)
+{
+    return TMTimerFromMilli(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer), cMilliSecs);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerFromNano} */
+static DECLCALLBACK(uint64_t) pdmR0DevHlp_TimerFromNano(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cNanoSecs)
+{
+    return TMTimerFromNano(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer), cNanoSecs);
+}
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerGet} */
+static DECLCALLBACK(uint64_t) pdmR0DevHlp_TimerGet(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMTimerGet(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerGetFreq} */
+static DECLCALLBACK(uint64_t) pdmR0DevHlp_TimerGetFreq(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMTimerGetFreq(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerGetNano} */
+static DECLCALLBACK(uint64_t) pdmR0DevHlp_TimerGetNano(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMTimerGetNano(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerIsActive} */
+static DECLCALLBACK(bool) pdmR0DevHlp_TimerIsActive(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMTimerIsActive(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerIsLockOwner} */
+static DECLCALLBACK(bool) pdmR0DevHlp_TimerIsLockOwner(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMTimerIsLockOwner(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerLockClock} */
+static DECLCALLBACK(VBOXSTRICTRC) pdmR0DevHlp_TimerLockClock(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, int rcBusy)
+{
+    return TMTimerLock(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer), rcBusy);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerLockClock2} */
+static DECLCALLBACK(VBOXSTRICTRC) pdmR0DevHlp_TimerLockClock2(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer,
+                                                              PPDMCRITSECT pCritSect, int rcBusy)
+{
+    VBOXSTRICTRC rc = TMTimerLock(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer), rcBusy);
+    if (rc == VINF_SUCCESS)
+    {
+        rc = PDMCritSectEnter(pCritSect, rcBusy);
+        if (rc == VINF_SUCCESS)
+            return rc;
+        AssertRC(VBOXSTRICTRC_VAL(rc));
+        TMTimerUnlock(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer));
+    }
+    else
+        AssertRC(VBOXSTRICTRC_VAL(rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerSet} */
+static DECLCALLBACK(int) pdmR0DevHlp_TimerSet(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t uExpire)
+{
+    return TMTimerSet(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer), uExpire);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerSetFrequencyHint} */
+static DECLCALLBACK(int) pdmR0DevHlp_TimerSetFrequencyHint(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint32_t uHz)
+{
+    return TMTimerSetFrequencyHint(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer), uHz);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerSetMicro} */
+static DECLCALLBACK(int) pdmR0DevHlp_TimerSetMicro(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cMicrosToNext)
+{
+    return TMTimerSetMicro(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer), cMicrosToNext);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerSetMillies} */
+static DECLCALLBACK(int) pdmR0DevHlp_TimerSetMillies(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cMilliesToNext)
+{
+    return TMTimerSetMillies(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer), cMilliesToNext);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerSetNano} */
+static DECLCALLBACK(int) pdmR0DevHlp_TimerSetNano(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cNanosToNext)
+{
+    return TMTimerSetNano(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer), cNanosToNext);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerSetRelative} */
+static DECLCALLBACK(int) pdmR0DevHlp_TimerSetRelative(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cTicksToNext, uint64_t *pu64Now)
+{
+    return TMTimerSetRelative(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer), cTicksToNext, pu64Now);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerStop} */
+static DECLCALLBACK(int) pdmR0DevHlp_TimerStop(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMTimerStop(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerUnlockClock} */
+static DECLCALLBACK(void) pdmR0DevHlp_TimerUnlockClock(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    TMTimerUnlock(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTimerUnlockClock2} */
+static DECLCALLBACK(void) pdmR0DevHlp_TimerUnlockClock2(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, PPDMCRITSECT pCritSect)
+{
+    TMTimerUnlock(pdmR0DevHlp_TimerToPtr(pDevIns, hTimer));
+    int rc = PDMCritSectLeave(pCritSect);
+    AssertRC(rc);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTMTimeVirtGet} */
+static DECLCALLBACK(uint64_t) pdmR0DevHlp_TMTimeVirtGet(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_TMTimeVirtGet: caller='%p'/%d\n", pDevIns, pDevIns->iInstance));
+    return TMVirtualGet(pDevIns->Internal.s.pGVM);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTMTimeVirtGetFreq} */
+static DECLCALLBACK(uint64_t) pdmR0DevHlp_TMTimeVirtGetFreq(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_TMTimeVirtGetFreq: caller='%p'/%d\n", pDevIns, pDevIns->iInstance));
+    return TMVirtualGetFreq(pDevIns->Internal.s.pGVM);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTMTimeVirtGetNano} */
+static DECLCALLBACK(uint64_t) pdmR0DevHlp_TMTimeVirtGetNano(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_TMTimeVirtGetNano: caller='%p'/%d\n", pDevIns, pDevIns->iInstance));
+    return TMVirtualToNano(pDevIns->Internal.s.pGVM, TMVirtualGet(pDevIns->Internal.s.pGVM));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnQueueToPtr} */
+static DECLCALLBACK(PPDMQUEUE)  pdmR0DevHlp_QueueToPtr(PPDMDEVINS pDevIns, PDMQUEUEHANDLE hQueue)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return (PPDMQUEUE)MMHyperR3ToCC(pDevIns->Internal.s.pGVM, hQueue);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnQueueAlloc} */
+static DECLCALLBACK(PPDMQUEUEITEMCORE) pdmR0DevHlp_QueueAlloc(PPDMDEVINS pDevIns, PDMQUEUEHANDLE hQueue)
+{
+    return PDMQueueAlloc(pdmR0DevHlp_QueueToPtr(pDevIns, hQueue));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnQueueInsert} */
+static DECLCALLBACK(void) pdmR0DevHlp_QueueInsert(PPDMDEVINS pDevIns, PDMQUEUEHANDLE hQueue, PPDMQUEUEITEMCORE pItem)
+{
+    return PDMQueueInsert(pdmR0DevHlp_QueueToPtr(pDevIns, hQueue), pItem);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnQueueInsertEx} */
+static DECLCALLBACK(void) pdmR0DevHlp_QueueInsertEx(PPDMDEVINS pDevIns, PDMQUEUEHANDLE hQueue, PPDMQUEUEITEMCORE pItem,
+                                                    uint64_t cNanoMaxDelay)
+{
+    return PDMQueueInsertEx(pdmR0DevHlp_QueueToPtr(pDevIns, hQueue), pItem, cNanoMaxDelay);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnQueueFlushIfNecessary} */
+static DECLCALLBACK(bool) pdmR0DevHlp_QueueFlushIfNecessary(PPDMDEVINS pDevIns, PDMQUEUEHANDLE hQueue)
+{
+    return PDMQueueFlushIfNecessary(pdmR0DevHlp_QueueToPtr(pDevIns, hQueue));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnTaskTrigger} */
+static DECLCALLBACK(int) pdmR0DevHlp_TaskTrigger(PPDMDEVINS pDevIns, PDMTASKHANDLE hTask)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_TaskTrigger: caller='%s'/%d: hTask=%RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, hTask));
+
+    int rc = PDMTaskTrigger(pDevIns->Internal.s.pGVM, PDMTASKTYPE_DEV, pDevIns->pDevInsForR3, hTask);
+
+    LogFlow(("pdmR0DevHlp_TaskTrigger: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnSUPSemEventSignal} */
+static DECLCALLBACK(int) pdmR0DevHlp_SUPSemEventSignal(PPDMDEVINS pDevIns, SUPSEMEVENT hEvent)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_SUPSemEventSignal: caller='%s'/%d: hEvent=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, hEvent));
+
+    int rc = SUPSemEventSignal(pDevIns->Internal.s.pGVM->pSession, hEvent);
+
+    LogFlow(("pdmR0DevHlp_SUPSemEventSignal: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnSUPSemEventWaitNoResume} */
+static DECLCALLBACK(int) pdmR0DevHlp_SUPSemEventWaitNoResume(PPDMDEVINS pDevIns, SUPSEMEVENT hEvent, uint32_t cMillies)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_SUPSemEventWaitNoResume: caller='%s'/%d: hEvent=%p cNsTimeout=%RU32\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hEvent, cMillies));
+
+    int rc = SUPSemEventWaitNoResume(pDevIns->Internal.s.pGVM->pSession, hEvent, cMillies);
+
+    LogFlow(("pdmR0DevHlp_SUPSemEventWaitNoResume: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnSUPSemEventWaitNsAbsIntr} */
+static DECLCALLBACK(int) pdmR0DevHlp_SUPSemEventWaitNsAbsIntr(PPDMDEVINS pDevIns, SUPSEMEVENT hEvent, uint64_t uNsTimeout)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_SUPSemEventWaitNsAbsIntr: caller='%s'/%d: hEvent=%p uNsTimeout=%RU64\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hEvent, uNsTimeout));
+
+    int rc = SUPSemEventWaitNsAbsIntr(pDevIns->Internal.s.pGVM->pSession, hEvent, uNsTimeout);
+
+    LogFlow(("pdmR0DevHlp_SUPSemEventWaitNsAbsIntr: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnSUPSemEventWaitNsRelIntr} */
+static DECLCALLBACK(int) pdmR0DevHlp_SUPSemEventWaitNsRelIntr(PPDMDEVINS pDevIns, SUPSEMEVENT hEvent, uint64_t cNsTimeout)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_SUPSemEventWaitNsRelIntr: caller='%s'/%d: hEvent=%p cNsTimeout=%RU64\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hEvent, cNsTimeout));
+
+    int rc = SUPSemEventWaitNsRelIntr(pDevIns->Internal.s.pGVM->pSession, hEvent, cNsTimeout);
+
+    LogFlow(("pdmR0DevHlp_SUPSemEventWaitNsRelIntr: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnSUPSemEventGetResolution} */
+static DECLCALLBACK(uint32_t) pdmR0DevHlp_SUPSemEventGetResolution(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_SUPSemEventGetResolution: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+
+    uint32_t cNsResolution = SUPSemEventGetResolution(pDevIns->Internal.s.pGVM->pSession);
+
+    LogFlow(("pdmR0DevHlp_SUPSemEventGetResolution: caller='%s'/%d: returns %u\n", pDevIns->pReg->szName, pDevIns->iInstance, cNsResolution));
+    return cNsResolution;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnSUPSemEventMultiSignal} */
+static DECLCALLBACK(int) pdmR0DevHlp_SUPSemEventMultiSignal(PPDMDEVINS pDevIns, SUPSEMEVENTMULTI hEventMulti)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_SUPSemEventMultiSignal: caller='%s'/%d: hEventMulti=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, hEventMulti));
+
+    int rc = SUPSemEventMultiSignal(pDevIns->Internal.s.pGVM->pSession, hEventMulti);
+
+    LogFlow(("pdmR0DevHlp_SUPSemEventMultiSignal: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnSUPSemEventMultiReset} */
+static DECLCALLBACK(int) pdmR0DevHlp_SUPSemEventMultiReset(PPDMDEVINS pDevIns, SUPSEMEVENTMULTI hEventMulti)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_SUPSemEventMultiReset: caller='%s'/%d: hEventMulti=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, hEventMulti));
+
+    int rc = SUPSemEventMultiReset(pDevIns->Internal.s.pGVM->pSession, hEventMulti);
+
+    LogFlow(("pdmR0DevHlp_SUPSemEventMultiReset: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnSUPSemEventMultiWaitNoResume} */
+static DECLCALLBACK(int) pdmR0DevHlp_SUPSemEventMultiWaitNoResume(PPDMDEVINS pDevIns, SUPSEMEVENTMULTI hEventMulti,
+                                                                  uint32_t cMillies)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_SUPSemEventMultiWaitNoResume: caller='%s'/%d: hEventMulti=%p cMillies=%RU32\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hEventMulti, cMillies));
+
+    int rc = SUPSemEventMultiWaitNoResume(pDevIns->Internal.s.pGVM->pSession, hEventMulti, cMillies);
+
+    LogFlow(("pdmR0DevHlp_SUPSemEventMultiWaitNoResume: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnSUPSemEventMultiWaitNsAbsIntr} */
+static DECLCALLBACK(int) pdmR0DevHlp_SUPSemEventMultiWaitNsAbsIntr(PPDMDEVINS pDevIns, SUPSEMEVENTMULTI hEventMulti,
+                                                                   uint64_t uNsTimeout)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_SUPSemEventMultiWaitNsAbsIntr: caller='%s'/%d: hEventMulti=%p uNsTimeout=%RU64\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hEventMulti, uNsTimeout));
+
+    int rc = SUPSemEventMultiWaitNsAbsIntr(pDevIns->Internal.s.pGVM->pSession, hEventMulti, uNsTimeout);
+
+    LogFlow(("pdmR0DevHlp_SUPSemEventMultiWaitNsAbsIntr: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnSUPSemEventMultiWaitNsRelIntr} */
+static DECLCALLBACK(int) pdmR0DevHlp_SUPSemEventMultiWaitNsRelIntr(PPDMDEVINS pDevIns, SUPSEMEVENTMULTI hEventMulti,
+                                                                   uint64_t cNsTimeout)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_SUPSemEventMultiWaitNsRelIntr: caller='%s'/%d: hEventMulti=%p cNsTimeout=%RU64\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hEventMulti, cNsTimeout));
+
+    int rc = SUPSemEventMultiWaitNsRelIntr(pDevIns->Internal.s.pGVM->pSession, hEventMulti, cNsTimeout);
+
+    LogFlow(("pdmR0DevHlp_SUPSemEventMultiWaitNsRelIntr: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnSUPSemEventMultiGetResolution} */
+static DECLCALLBACK(uint32_t) pdmR0DevHlp_SUPSemEventMultiGetResolution(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_SUPSemEventMultiGetResolution: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+
+    uint32_t cNsResolution = SUPSemEventMultiGetResolution(pDevIns->Internal.s.pGVM->pSession);
+
+    LogFlow(("pdmR0DevHlp_SUPSemEventMultiGetResolution: caller='%s'/%d: returns %u\n", pDevIns->pReg->szName, pDevIns->iInstance, cNsResolution));
+    return cNsResolution;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnCritSectGetNop} */
+static DECLCALLBACK(PPDMCRITSECT) pdmR0DevHlp_CritSectGetNop(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+
+    PPDMCRITSECT pCritSect = &pGVM->pdm.s.NopCritSect;
+    LogFlow(("pdmR0DevHlp_CritSectGetNop: caller='%s'/%d: return %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pCritSect));
+    return pCritSect;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnSetDeviceCritSect} */
+static DECLCALLBACK(int) pdmR0DevHlp_SetDeviceCritSect(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect)
+{
+    /*
+     * Validate input.
+     *
+     * Note! We only allow the automatically created default critical section
+     *       to be replaced by this API.
+     */
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertPtrReturn(pCritSect, VERR_INVALID_POINTER);
+    LogFlow(("pdmR0DevHlp_SetDeviceCritSect: caller='%s'/%d: pCritSect=%p (%s)\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pCritSect, pCritSect->s.pszName));
+    AssertReturn(PDMCritSectIsInitialized(pCritSect), VERR_INVALID_PARAMETER);
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+    AssertReturn(pCritSect->s.pVMR0 == pGVM, VERR_INVALID_PARAMETER);
+
+    VM_ASSERT_EMT(pGVM);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_WRONG_ORDER);
+
+    /*
+     * Check that ring-3 has already done this, then effect the change.
+     */
+    AssertReturn(pDevIns->pDevInsForR3R0->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_CHANGED_CRITSECT, VERR_WRONG_ORDER);
+    pDevIns->pCritSectRoR0 = pCritSect;
+
+    LogFlow(("pdmR0DevHlp_SetDeviceCritSect: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnCritSectEnter} */
+static DECLCALLBACK(int)      pdmR0DevHlp_CritSectEnter(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, int rcBusy)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns); /** @todo pass pDevIns->Internal.s.pGVM to the crit sect code.   */
+    return PDMCritSectEnter(pCritSect, rcBusy);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnCritSectEnterDebug} */
+static DECLCALLBACK(int)      pdmR0DevHlp_CritSectEnterDebug(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns); /** @todo pass pDevIns->Internal.s.pGVM to the crit sect code.   */
+    return PDMCritSectEnterDebug(pCritSect, rcBusy, uId, RT_SRC_POS_ARGS);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnCritSectTryEnter} */
+static DECLCALLBACK(int)      pdmR0DevHlp_CritSectTryEnter(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns); /** @todo pass pDevIns->Internal.s.pGVM to the crit sect code.   */
+    return PDMCritSectTryEnter(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnCritSectTryEnterDebug} */
+static DECLCALLBACK(int)      pdmR0DevHlp_CritSectTryEnterDebug(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns); /** @todo pass pDevIns->Internal.s.pGVM to the crit sect code.   */
+    return PDMCritSectTryEnterDebug(pCritSect, uId, RT_SRC_POS_ARGS);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnCritSectLeave} */
+static DECLCALLBACK(int)      pdmR0DevHlp_CritSectLeave(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns); /** @todo pass pDevIns->Internal.s.pGVM to the crit sect code.   */
+    return PDMCritSectLeave(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnCritSectIsOwner} */
+static DECLCALLBACK(bool)     pdmR0DevHlp_CritSectIsOwner(PPDMDEVINS pDevIns, PCPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns); /** @todo pass pDevIns->Internal.s.pGVM to the crit sect code.   */
+    return PDMCritSectIsOwner(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnCritSectIsInitialized} */
+static DECLCALLBACK(bool)     pdmR0DevHlp_CritSectIsInitialized(PPDMDEVINS pDevIns, PCPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return PDMCritSectIsInitialized(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnCritSectHasWaiters} */
+static DECLCALLBACK(bool)     pdmR0DevHlp_CritSectHasWaiters(PPDMDEVINS pDevIns, PCPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return PDMCritSectHasWaiters(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnCritSectGetRecursion} */
+static DECLCALLBACK(uint32_t) pdmR0DevHlp_CritSectGetRecursion(PPDMDEVINS pDevIns, PCPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return PDMCritSectGetRecursion(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnCritSectScheduleExitEvent} */
+static DECLCALLBACK(int) pdmR0DevHlp_CritSectScheduleExitEvent(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect,
+                                                               SUPSEMEVENT hEventToSignal)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return PDMHCCritSectScheduleExitEvent(pCritSect, hEventToSignal);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnDBGFTraceBuf} */
+static DECLCALLBACK(RTTRACEBUF) pdmR0DevHlp_DBGFTraceBuf(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RTTRACEBUF hTraceBuf = pDevIns->Internal.s.pGVM->hTraceBufR0;
+    LogFlow(("pdmR0DevHlp_DBGFTraceBuf: caller='%p'/%d: returns %p\n", pDevIns, pDevIns->iInstance, hTraceBuf));
+    return hTraceBuf;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnPCIBusSetUpContext} */
+static DECLCALLBACK(int) pdmR0DevHlp_PCIBusSetUpContext(PPDMDEVINS pDevIns, PPDMPCIBUSREGR0 pPciBusReg, PCPDMPCIHLPR0 *ppPciHlp)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_PCIBusSetUpContext: caller='%p'/%d: pPciBusReg=%p{.u32Version=%#x, .iBus=%#u, .pfnSetIrq=%p, u32EnvVersion=%#x} ppPciHlp=%p\n",
+             pDevIns, pDevIns->iInstance, pPciBusReg, pPciBusReg->u32Version, pPciBusReg->iBus, pPciBusReg->pfnSetIrq,
+             pPciBusReg->u32EndVersion, ppPciHlp));
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pPciBusReg, VERR_INVALID_POINTER);
+    AssertLogRelMsgReturn(pPciBusReg->u32Version == PDM_PCIBUSREGCC_VERSION,
+                          ("%#x vs %#x\n", pPciBusReg->u32Version, PDM_PCIBUSREGCC_VERSION), VERR_VERSION_MISMATCH);
+    AssertPtrReturn(pPciBusReg->pfnSetIrq, VERR_INVALID_POINTER);
+    AssertLogRelMsgReturn(pPciBusReg->u32EndVersion == PDM_PCIBUSREGCC_VERSION,
+                          ("%#x vs %#x\n", pPciBusReg->u32EndVersion, PDM_PCIBUSREGCC_VERSION), VERR_VERSION_MISMATCH);
+
+    AssertPtrReturn(ppPciHlp, VERR_INVALID_POINTER);
+
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_WRONG_ORDER);
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+
+    /* Check the shared bus data (registered earlier from ring-3): */
+    uint32_t iBus = pPciBusReg->iBus;
+    ASMCompilerBarrier();
+    AssertLogRelMsgReturn(iBus < RT_ELEMENTS(pGVM->pdm.s.aPciBuses), ("iBus=%#x\n", iBus), VERR_OUT_OF_RANGE);
+    PPDMPCIBUS pPciBusShared = &pGVM->pdm.s.aPciBuses[iBus];
+    AssertLogRelMsgReturn(pPciBusShared->iBus == iBus, ("%u vs %u\n", pPciBusShared->iBus, iBus), VERR_INVALID_PARAMETER);
+    AssertLogRelMsgReturn(pPciBusShared->pDevInsR3 == pDevIns->pDevInsForR3,
+                          ("%p vs %p (iBus=%u)\n", pPciBusShared->pDevInsR3, pDevIns->pDevInsForR3, iBus), VERR_NOT_OWNER);
+
+    /* Check that the bus isn't already registered in ring-0: */
+    AssertCompile(RT_ELEMENTS(pGVM->pdm.s.aPciBuses) == RT_ELEMENTS(pGVM->pdmr0.s.aPciBuses));
+    PPDMPCIBUSR0 pPciBusR0 = &pGVM->pdmr0.s.aPciBuses[iBus];
+    AssertLogRelMsgReturn(pPciBusR0->pDevInsR0 == NULL,
+                          ("%p (caller pDevIns=%p, iBus=%u)\n", pPciBusR0->pDevInsR0, pDevIns, iBus),
+                          VERR_ALREADY_EXISTS);
+
+    /*
+     * Do the registering.
+     */
+    pPciBusR0->iBus        = iBus;
+    pPciBusR0->uPadding0   = 0xbeefbeef;
+    pPciBusR0->pfnSetIrqR0 = pPciBusReg->pfnSetIrq;
+    pPciBusR0->pDevInsR0   = pDevIns;
+
+    *ppPciHlp = &g_pdmR0PciHlp;
+
+    LogFlow(("pdmR0DevHlp_PCIBusSetUpContext: caller='%p'/%d: returns VINF_SUCCESS\n", pDevIns, pDevIns->iInstance));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnPICSetUpContext} */
+static DECLCALLBACK(int) pdmR0DevHlp_PICSetUpContext(PPDMDEVINS pDevIns, PPDMPICREG pPicReg, PCPDMPICHLP *ppPicHlp)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_PICSetUpContext: caller='%s'/%d: pPicReg=%p:{.u32Version=%#x, .pfnSetIrq=%p, .pfnGetInterrupt=%p, .u32TheEnd=%#x } ppPicHlp=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPicReg, pPicReg->u32Version, pPicReg->pfnSetIrq, pPicReg->pfnGetInterrupt, pPicReg->u32TheEnd, ppPicHlp));
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(pPicReg->u32Version == PDM_PICREG_VERSION,
+                    ("%s/%d: u32Version=%#x expected %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, pPicReg->u32Version, PDM_PICREG_VERSION),
+                    VERR_VERSION_MISMATCH);
+    AssertPtrReturn(pPicReg->pfnSetIrq, VERR_INVALID_POINTER);
+    AssertPtrReturn(pPicReg->pfnGetInterrupt, VERR_INVALID_POINTER);
+    AssertMsgReturn(pPicReg->u32TheEnd == PDM_PICREG_VERSION,
+                    ("%s/%d: u32TheEnd=%#x expected %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, pPicReg->u32TheEnd, PDM_PICREG_VERSION),
+                    VERR_VERSION_MISMATCH);
+    AssertPtrReturn(ppPicHlp, VERR_INVALID_POINTER);
+
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_WRONG_ORDER);
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+
+    /* Check that it's the same device as made the ring-3 registrations: */
+    AssertLogRelMsgReturn(pGVM->pdm.s.Pic.pDevInsR3 == pDevIns->pDevInsForR3,
+                          ("%p vs %p\n", pGVM->pdm.s.Pic.pDevInsR3, pDevIns->pDevInsForR3), VERR_NOT_OWNER);
+
+    /* Check that it isn't already registered in ring-0: */
+    AssertLogRelMsgReturn(pGVM->pdm.s.Pic.pDevInsR0 == NULL, ("%p (caller pDevIns=%p)\n", pGVM->pdm.s.Pic.pDevInsR0, pDevIns),
+                          VERR_ALREADY_EXISTS);
+
+    /*
+     * Take down the callbacks and instance.
+     */
+    pGVM->pdm.s.Pic.pDevInsR0 = pDevIns;
+    pGVM->pdm.s.Pic.pfnSetIrqR0 = pPicReg->pfnSetIrq;
+    pGVM->pdm.s.Pic.pfnGetInterruptR0 = pPicReg->pfnGetInterrupt;
+    Log(("PDM: Registered PIC device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+
+    /* set the helper pointer and return. */
+    *ppPicHlp = &g_pdmR0PicHlp;
+    LogFlow(("pdmR0DevHlp_PICSetUpContext: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnApicSetUpContext} */
+static DECLCALLBACK(int) pdmR0DevHlp_ApicSetUpContext(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_ApicSetUpContext: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_WRONG_ORDER);
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+
+    /* Check that it's the same device as made the ring-3 registrations: */
+    AssertLogRelMsgReturn(pGVM->pdm.s.Apic.pDevInsR3 == pDevIns->pDevInsForR3,
+                          ("%p vs %p\n", pGVM->pdm.s.Apic.pDevInsR3, pDevIns->pDevInsForR3), VERR_NOT_OWNER);
+
+    /* Check that it isn't already registered in ring-0: */
+    AssertLogRelMsgReturn(pGVM->pdm.s.Apic.pDevInsR0 == NULL, ("%p (caller pDevIns=%p)\n", pGVM->pdm.s.Apic.pDevInsR0, pDevIns),
+                          VERR_ALREADY_EXISTS);
+
+    /*
+     * Take down the instance.
+     */
+    pGVM->pdm.s.Apic.pDevInsR0 = pDevIns;
+    Log(("PDM: Registered APIC device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+
+    /* set the helper pointer and return. */
+    LogFlow(("pdmR0DevHlp_ApicSetUpContext: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnIoApicSetUpContext} */
+static DECLCALLBACK(int) pdmR0DevHlp_IoApicSetUpContext(PPDMDEVINS pDevIns, PPDMIOAPICREG pIoApicReg, PCPDMIOAPICHLP *ppIoApicHlp)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_IoApicSetUpContext: caller='%s'/%d: pIoApicReg=%p:{.u32Version=%#x, .pfnSetIrq=%p, .pfnSendMsi=%p, .pfnSetEoi=%p, .u32TheEnd=%#x } ppIoApicHlp=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pIoApicReg, pIoApicReg->u32Version, pIoApicReg->pfnSetIrq, pIoApicReg->pfnSendMsi, pIoApicReg->pfnSetEoi, pIoApicReg->u32TheEnd, ppIoApicHlp));
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(pIoApicReg->u32Version == PDM_IOAPICREG_VERSION,
+                    ("%s/%d: u32Version=%#x expected %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, pIoApicReg->u32Version, PDM_IOAPICREG_VERSION),
+                    VERR_VERSION_MISMATCH);
+    AssertPtrReturn(pIoApicReg->pfnSetIrq, VERR_INVALID_POINTER);
+    AssertPtrReturn(pIoApicReg->pfnSendMsi, VERR_INVALID_POINTER);
+    AssertPtrReturn(pIoApicReg->pfnSetEoi, VERR_INVALID_POINTER);
+    AssertMsgReturn(pIoApicReg->u32TheEnd == PDM_IOAPICREG_VERSION,
+                    ("%s/%d: u32TheEnd=%#x expected %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, pIoApicReg->u32TheEnd, PDM_IOAPICREG_VERSION),
+                    VERR_VERSION_MISMATCH);
+    AssertPtrReturn(ppIoApicHlp, VERR_INVALID_POINTER);
+
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_WRONG_ORDER);
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+
+    /* Check that it's the same device as made the ring-3 registrations: */
+    AssertLogRelMsgReturn(pGVM->pdm.s.IoApic.pDevInsR3 == pDevIns->pDevInsForR3,
+                          ("%p vs %p\n", pGVM->pdm.s.IoApic.pDevInsR3, pDevIns->pDevInsForR3), VERR_NOT_OWNER);
+
+    /* Check that it isn't already registered in ring-0: */
+    AssertLogRelMsgReturn(pGVM->pdm.s.IoApic.pDevInsR0 == NULL, ("%p (caller pDevIns=%p)\n", pGVM->pdm.s.IoApic.pDevInsR0, pDevIns),
+                          VERR_ALREADY_EXISTS);
+
+    /*
+     * Take down the callbacks and instance.
+     */
+    pGVM->pdm.s.IoApic.pDevInsR0    = pDevIns;
+    pGVM->pdm.s.IoApic.pfnSetIrqR0  = pIoApicReg->pfnSetIrq;
+    pGVM->pdm.s.IoApic.pfnSendMsiR0 = pIoApicReg->pfnSendMsi;
+    pGVM->pdm.s.IoApic.pfnSetEoiR0  = pIoApicReg->pfnSetEoi;
+    Log(("PDM: Registered IOAPIC device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+
+    /* set the helper pointer and return. */
+    *ppIoApicHlp = &g_pdmR0IoApicHlp;
+    LogFlow(("pdmR0DevHlp_IoApicSetUpContext: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR0,pfnHpetSetUpContext} */
+static DECLCALLBACK(int) pdmR0DevHlp_HpetSetUpContext(PPDMDEVINS pDevIns, PPDMHPETREG pHpetReg, PCPDMHPETHLPR0 *ppHpetHlp)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR0DevHlp_HpetSetUpContext: caller='%s'/%d: pHpetReg=%p:{.u32Version=%#x, } ppHpetHlp=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pHpetReg, pHpetReg->u32Version, ppHpetHlp));
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(pHpetReg->u32Version == PDM_HPETREG_VERSION,
+                    ("%s/%d: u32Version=%#x expected %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, pHpetReg->u32Version, PDM_HPETREG_VERSION),
+                    VERR_VERSION_MISMATCH);
+    AssertPtrReturn(ppHpetHlp, VERR_INVALID_POINTER);
+
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_WRONG_ORDER);
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+
+    /* Check that it's the same device as made the ring-3 registrations: */
+    AssertLogRelMsgReturn(pGVM->pdm.s.pHpet == pDevIns->pDevInsForR3, ("%p vs %p\n", pGVM->pdm.s.pHpet, pDevIns->pDevInsForR3),
+                          VERR_NOT_OWNER);
+
+    ///* Check that it isn't already registered in ring-0: */
+    //AssertLogRelMsgReturn(pGVM->pdm.s.Hpet.pDevInsR0 == NULL, ("%p (caller pDevIns=%p)\n", pGVM->pdm.s.Hpet.pDevInsR0, pDevIns),
+    //                      VERR_ALREADY_EXISTS);
+
+    /*
+     * Nothing to take down here at present.
+     */
+    Log(("PDM: Registered HPET device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+
+    /* set the helper pointer and return. */
+    *ppHpetHlp = &g_pdmR0HpetHlp;
+    LogFlow(("pdmR0DevHlp_HpetSetUpContext: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * The Ring-0 Device Helper Callbacks.
+ */
+extern DECLEXPORT(const PDMDEVHLPR0) g_pdmR0DevHlp =
+{
+    PDM_DEVHLPR0_VERSION,
+    pdmR0DevHlp_IoPortSetUpContextEx,
+    pdmR0DevHlp_MmioSetUpContextEx,
+    pdmR0DevHlp_Mmio2SetUpContext,
+    pdmR0DevHlp_PCIPhysRead,
+    pdmR0DevHlp_PCIPhysWrite,
+    pdmR0DevHlp_PCISetIrq,
+    pdmR0DevHlp_ISASetIrq,
+    pdmR0DevHlp_IoApicSendMsi,
+    pdmR0DevHlp_PhysRead,
+    pdmR0DevHlp_PhysWrite,
+    pdmR0DevHlp_A20IsEnabled,
+    pdmR0DevHlp_VMState,
+    pdmR0DevHlp_VMSetError,
+    pdmR0DevHlp_VMSetErrorV,
+    pdmR0DevHlp_VMSetRuntimeError,
+    pdmR0DevHlp_VMSetRuntimeErrorV,
+    pdmR0DevHlp_GetVM,
+    pdmR0DevHlp_GetVMCPU,
+    pdmR0DevHlp_GetCurrentCpuId,
+    pdmR0DevHlp_TimerToPtr,
+    pdmR0DevHlp_TimerFromMicro,
+    pdmR0DevHlp_TimerFromMilli,
+    pdmR0DevHlp_TimerFromNano,
+    pdmR0DevHlp_TimerGet,
+    pdmR0DevHlp_TimerGetFreq,
+    pdmR0DevHlp_TimerGetNano,
+    pdmR0DevHlp_TimerIsActive,
+    pdmR0DevHlp_TimerIsLockOwner,
+    pdmR0DevHlp_TimerLockClock,
+    pdmR0DevHlp_TimerLockClock2,
+    pdmR0DevHlp_TimerSet,
+    pdmR0DevHlp_TimerSetFrequencyHint,
+    pdmR0DevHlp_TimerSetMicro,
+    pdmR0DevHlp_TimerSetMillies,
+    pdmR0DevHlp_TimerSetNano,
+    pdmR0DevHlp_TimerSetRelative,
+    pdmR0DevHlp_TimerStop,
+    pdmR0DevHlp_TimerUnlockClock,
+    pdmR0DevHlp_TimerUnlockClock2,
+    pdmR0DevHlp_TMTimeVirtGet,
+    pdmR0DevHlp_TMTimeVirtGetFreq,
+    pdmR0DevHlp_TMTimeVirtGetNano,
+    pdmR0DevHlp_QueueToPtr,
+    pdmR0DevHlp_QueueAlloc,
+    pdmR0DevHlp_QueueInsert,
+    pdmR0DevHlp_QueueInsertEx,
+    pdmR0DevHlp_QueueFlushIfNecessary,
+    pdmR0DevHlp_TaskTrigger,
+    pdmR0DevHlp_SUPSemEventSignal,
+    pdmR0DevHlp_SUPSemEventWaitNoResume,
+    pdmR0DevHlp_SUPSemEventWaitNsAbsIntr,
+    pdmR0DevHlp_SUPSemEventWaitNsRelIntr,
+    pdmR0DevHlp_SUPSemEventGetResolution,
+    pdmR0DevHlp_SUPSemEventMultiSignal,
+    pdmR0DevHlp_SUPSemEventMultiReset,
+    pdmR0DevHlp_SUPSemEventMultiWaitNoResume,
+    pdmR0DevHlp_SUPSemEventMultiWaitNsAbsIntr,
+    pdmR0DevHlp_SUPSemEventMultiWaitNsRelIntr,
+    pdmR0DevHlp_SUPSemEventMultiGetResolution,
+    pdmR0DevHlp_CritSectGetNop,
+    pdmR0DevHlp_SetDeviceCritSect,
+    pdmR0DevHlp_CritSectEnter,
+    pdmR0DevHlp_CritSectEnterDebug,
+    pdmR0DevHlp_CritSectTryEnter,
+    pdmR0DevHlp_CritSectTryEnterDebug,
+    pdmR0DevHlp_CritSectLeave,
+    pdmR0DevHlp_CritSectIsOwner,
+    pdmR0DevHlp_CritSectIsInitialized,
+    pdmR0DevHlp_CritSectHasWaiters,
+    pdmR0DevHlp_CritSectGetRecursion,
+    pdmR0DevHlp_CritSectScheduleExitEvent,
+    pdmR0DevHlp_DBGFTraceBuf,
+    pdmR0DevHlp_PCIBusSetUpContext,
+    pdmR0DevHlp_PICSetUpContext,
+    pdmR0DevHlp_ApicSetUpContext,
+    pdmR0DevHlp_IoApicSetUpContext,
+    pdmR0DevHlp_HpetSetUpContext,
+    NULL /*pfnReserved1*/,
+    NULL /*pfnReserved2*/,
+    NULL /*pfnReserved3*/,
+    NULL /*pfnReserved4*/,
+    NULL /*pfnReserved5*/,
+    NULL /*pfnReserved6*/,
+    NULL /*pfnReserved7*/,
+    NULL /*pfnReserved8*/,
+    NULL /*pfnReserved9*/,
+    NULL /*pfnReserved10*/,
+    PDM_DEVHLPR0_VERSION
+};
+
+/** @} */
+
+
+/** @name PIC Ring-0 Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMPICHLP,pfnSetInterruptFF} */
+static DECLCALLBACK(void) pdmR0PicHlp_SetInterruptFF(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PGVM     pGVM  = (PGVM)pDevIns->Internal.s.pGVM;
+    PVMCPUCC pVCpu = &pGVM->aCpus[0];     /* for PIC we always deliver to CPU 0, MP use APIC */
+    /** @todo r=ramshankar: Propagating rcRZ and make all callers handle it? */
+    APICLocalInterrupt(pVCpu, 0 /* u8Pin */, 1 /* u8Level */, VINF_SUCCESS /* rcRZ */);
+}
+
+
+/** @interface_method_impl{PDMPICHLP,pfnClearInterruptFF} */
+static DECLCALLBACK(void) pdmR0PicHlp_ClearInterruptFF(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PGVM     pGVM  = (PGVM)pDevIns->Internal.s.pGVM;
+    PVMCPUCC pVCpu = &pGVM->aCpus[0];     /* for PIC we always deliver to CPU 0, MP use APIC */
+    /** @todo r=ramshankar: Propagating rcRZ and make all callers handle it? */
+    APICLocalInterrupt(pVCpu, 0 /* u8Pin */, 0 /* u8Level */, VINF_SUCCESS /* rcRZ */);
+}
+
+
+/** @interface_method_impl{PDMPICHLP,pfnLock} */
+static DECLCALLBACK(int) pdmR0PicHlp_Lock(PPDMDEVINS pDevIns, int rc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    return pdmLockEx(pDevIns->Internal.s.pGVM, rc);
+}
+
+
+/** @interface_method_impl{PDMPICHLP,pfnUnlock} */
+static DECLCALLBACK(void) pdmR0PicHlp_Unlock(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    pdmUnlock(pDevIns->Internal.s.pGVM);
+}
+
+
+/**
+ * The Ring-0 PIC Helper Callbacks.
+ */
+extern DECLEXPORT(const PDMPICHLP) g_pdmR0PicHlp =
+{
+    PDM_PICHLP_VERSION,
+    pdmR0PicHlp_SetInterruptFF,
+    pdmR0PicHlp_ClearInterruptFF,
+    pdmR0PicHlp_Lock,
+    pdmR0PicHlp_Unlock,
+    PDM_PICHLP_VERSION
+};
+
+/** @} */
+
+
+/** @name I/O APIC Ring-0 Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMIOAPICHLP,pfnApicBusDeliver} */
+static DECLCALLBACK(int) pdmR0IoApicHlp_ApicBusDeliver(PPDMDEVINS pDevIns, uint8_t u8Dest, uint8_t u8DestMode,
+                                                       uint8_t u8DeliveryMode, uint8_t uVector, uint8_t u8Polarity,
+                                                       uint8_t u8TriggerMode, uint32_t uTagSrc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+    LogFlow(("pdmR0IoApicHlp_ApicBusDeliver: caller=%p/%d: u8Dest=%RX8 u8DestMode=%RX8 u8DeliveryMode=%RX8 uVector=%RX8 u8Polarity=%RX8 u8TriggerMode=%RX8 uTagSrc=%#x\n",
+             pDevIns, pDevIns->iInstance, u8Dest, u8DestMode, u8DeliveryMode, uVector, u8Polarity, u8TriggerMode, uTagSrc));
+    return APICBusDeliver(pGVM, u8Dest, u8DestMode, u8DeliveryMode, uVector, u8Polarity, u8TriggerMode, uTagSrc);
+}
+
+
+/** @interface_method_impl{PDMIOAPICHLP,pfnLock} */
+static DECLCALLBACK(int) pdmR0IoApicHlp_Lock(PPDMDEVINS pDevIns, int rc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    return pdmLockEx(pDevIns->Internal.s.pGVM, rc);
+}
+
+
+/** @interface_method_impl{PDMIOAPICHLP,pfnUnlock} */
+static DECLCALLBACK(void) pdmR0IoApicHlp_Unlock(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    pdmUnlock(pDevIns->Internal.s.pGVM);
+}
+
+
+/**
+ * The Ring-0 I/O APIC Helper Callbacks.
+ */
+extern DECLEXPORT(const PDMIOAPICHLP) g_pdmR0IoApicHlp =
+{
+    PDM_IOAPICHLP_VERSION,
+    pdmR0IoApicHlp_ApicBusDeliver,
+    pdmR0IoApicHlp_Lock,
+    pdmR0IoApicHlp_Unlock,
+    PDM_IOAPICHLP_VERSION
+};
+
+/** @} */
+
+
+
+
+/** @name PCI Bus Ring-0 Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMPCIHLPR0,pfnIsaSetIrq} */
+static DECLCALLBACK(void) pdmR0PciHlp_IsaSetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel, uint32_t uTagSrc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    Log4(("pdmR0PciHlp_IsaSetIrq: iIrq=%d iLevel=%d uTagSrc=%#x\n", iIrq, iLevel, uTagSrc));
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+
+    pdmLock(pGVM);
+    pdmR0IsaSetIrq(pGVM, iIrq, iLevel, uTagSrc);
+    pdmUnlock(pGVM);
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR0,pfnIoApicSetIrq} */
+static DECLCALLBACK(void) pdmR0PciHlp_IoApicSetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel, uint32_t uTagSrc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    Log4(("pdmR0PciHlp_IoApicSetIrq: iIrq=%d iLevel=%d uTagSrc=%#x\n", iIrq, iLevel, uTagSrc));
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+
+    if (pGVM->pdm.s.IoApic.pDevInsR0)
+        pGVM->pdm.s.IoApic.pfnSetIrqR0(pGVM->pdm.s.IoApic.pDevInsR0, iIrq, iLevel, uTagSrc);
+    else if (pGVM->pdm.s.IoApic.pDevInsR3)
+    {
+        /* queue for ring-3 execution. */
+        PPDMDEVHLPTASK pTask = (PPDMDEVHLPTASK)PDMQueueAlloc(pGVM->pdm.s.pDevHlpQueueR0);
+        if (pTask)
+        {
+            pTask->enmOp = PDMDEVHLPTASKOP_IOAPIC_SET_IRQ;
+            pTask->pDevInsR3 = NIL_RTR3PTR; /* not required */
+            pTask->u.IoApicSetIRQ.iIrq = iIrq;
+            pTask->u.IoApicSetIRQ.iLevel = iLevel;
+            pTask->u.IoApicSetIRQ.uTagSrc = uTagSrc;
+
+            PDMQueueInsertEx(pGVM->pdm.s.pDevHlpQueueR0, &pTask->Core, 0);
+        }
+        else
+            AssertMsgFailed(("We're out of devhlp queue items!!!\n"));
+    }
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR0,pfnIoApicSendMsi} */
+static DECLCALLBACK(void) pdmR0PciHlp_IoApicSendMsi(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint32_t uValue, uint32_t uTagSrc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    Log4(("pdmR0PciHlp_IoApicSendMsi: GCPhys=%p uValue=%d uTagSrc=%#x\n", GCPhys, uValue, uTagSrc));
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+    if (pGVM->pdm.s.IoApic.pDevInsR0)
+        pGVM->pdm.s.IoApic.pfnSendMsiR0(pGVM->pdm.s.IoApic.pDevInsR0, GCPhys, uValue, uTagSrc);
+    else
+        AssertFatalMsgFailed(("Lazy bastards!"));
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR0,pfnLock} */
+static DECLCALLBACK(int) pdmR0PciHlp_Lock(PPDMDEVINS pDevIns, int rc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    return pdmLockEx(pDevIns->Internal.s.pGVM, rc);
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR0,pfnUnlock} */
+static DECLCALLBACK(void) pdmR0PciHlp_Unlock(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    pdmUnlock(pDevIns->Internal.s.pGVM);
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR0,pfnGetBusByNo} */
+static DECLCALLBACK(PPDMDEVINS) pdmR0PciHlp_GetBusByNo(PPDMDEVINS pDevIns, uint32_t idxPdmBus)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+    AssertReturn(idxPdmBus < RT_ELEMENTS(pGVM->pdmr0.s.aPciBuses), NULL);
+    PPDMDEVINS pRetDevIns = pGVM->pdmr0.s.aPciBuses[idxPdmBus].pDevInsR0;
+    LogFlow(("pdmR3PciHlp_GetBusByNo: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pRetDevIns));
+    return pRetDevIns;
+}
+
+
+/**
+ * The Ring-0 PCI Bus Helper Callbacks.
+ */
+extern DECLEXPORT(const PDMPCIHLPR0) g_pdmR0PciHlp =
+{
+    PDM_PCIHLPR0_VERSION,
+    pdmR0PciHlp_IsaSetIrq,
+    pdmR0PciHlp_IoApicSetIrq,
+    pdmR0PciHlp_IoApicSendMsi,
+    pdmR0PciHlp_Lock,
+    pdmR0PciHlp_Unlock,
+    pdmR0PciHlp_GetBusByNo,
+    PDM_PCIHLPR0_VERSION, /* the end */
+};
+
+/** @} */
+
+
+/** @name HPET Ring-0 Helpers
+ * @{
+ */
+/* none */
+
+/**
+ * The Ring-0 HPET Helper Callbacks.
+ */
+extern DECLEXPORT(const PDMHPETHLPR0) g_pdmR0HpetHlp =
+{
+    PDM_HPETHLPR0_VERSION,
+    PDM_HPETHLPR0_VERSION, /* the end */
+};
+
+/** @} */
+
+
+/** @name Raw PCI Ring-0 Helpers
+ * @{
+ */
+/* none */
+
+/**
+ * The Ring-0 PCI raw Helper Callbacks.
+ */
+extern DECLEXPORT(const PDMPCIRAWHLPR0) g_pdmR0PciRawHlp =
+{
+    PDM_PCIRAWHLPR0_VERSION,
+    PDM_PCIRAWHLPR0_VERSION, /* the end */
+};
+
+/** @} */
+
+
+
+
+/**
+ * Sets an irq on the PIC and I/O APIC.
+ *
+ * @returns true if delivered, false if postponed.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   iIrq        The irq.
+ * @param   iLevel      The new level.
+ * @param   uTagSrc     The IRQ tag and source.
+ *
+ * @remarks The caller holds the PDM lock.
+ */
+static bool pdmR0IsaSetIrq(PGVM pGVM, int iIrq, int iLevel, uint32_t uTagSrc)
+{
+    if (RT_LIKELY(    (   pGVM->pdm.s.IoApic.pDevInsR0
+                       || !pGVM->pdm.s.IoApic.pDevInsR3)
+                  &&  (   pGVM->pdm.s.Pic.pDevInsR0
+                       || !pGVM->pdm.s.Pic.pDevInsR3)))
+    {
+        if (pGVM->pdm.s.Pic.pDevInsR0)
+            pGVM->pdm.s.Pic.pfnSetIrqR0(pGVM->pdm.s.Pic.pDevInsR0, iIrq, iLevel, uTagSrc);
+        if (pGVM->pdm.s.IoApic.pDevInsR0)
+            pGVM->pdm.s.IoApic.pfnSetIrqR0(pGVM->pdm.s.IoApic.pDevInsR0, iIrq, iLevel, uTagSrc);
+        return true;
+    }
+
+    /* queue for ring-3 execution. */
+    PPDMDEVHLPTASK pTask = (PPDMDEVHLPTASK)PDMQueueAlloc(pGVM->pdm.s.pDevHlpQueueR0);
+    AssertReturn(pTask, false);
+
+    pTask->enmOp = PDMDEVHLPTASKOP_ISA_SET_IRQ;
+    pTask->pDevInsR3 = NIL_RTR3PTR; /* not required */
+    pTask->u.IsaSetIRQ.iIrq = iIrq;
+    pTask->u.IsaSetIRQ.iLevel = iLevel;
+    pTask->u.IsaSetIRQ.uTagSrc = uTagSrc;
+
+    PDMQueueInsertEx(pGVM->pdm.s.pDevHlpQueueR0, &pTask->Core, 0);
+    return false;
+}
+
diff --git a/src/VBox/VMM/VMMR0/PDMR0Device.cpp b/src/VBox/VMM/VMMR0/PDMR0Device.cpp
new file mode 100644
index 00000000..571d0e61
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/PDMR0Device.cpp
@@ -0,0 +1,803 @@
+/* $Id: PDMR0Device.cpp $ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, R0 Device parts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DEVICE
+#define PDMPCIDEV_INCLUDE_PRIVATE  /* Hack to get pdmpcidevint.h included at the right point. */
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/gvm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/gvmm.h>
+
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <VBox/msi.h>
+#include <VBox/sup.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/mem.h>
+#include <iprt/memobj.h>
+#include <iprt/process.h>
+#include <iprt/string.h>
+
+#include "dtrace/VBoxVMM.h"
+#include "PDMInline.h"
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+extern DECLEXPORT(const PDMDEVHLPR0)    g_pdmR0DevHlp;
+extern DECLEXPORT(const PDMPICHLP)      g_pdmR0PicHlp;
+extern DECLEXPORT(const PDMIOAPICHLP)   g_pdmR0IoApicHlp;
+extern DECLEXPORT(const PDMPCIHLPR0)    g_pdmR0PciHlp;
+extern DECLEXPORT(const PDMHPETHLPR0)   g_pdmR0HpetHlp;
+extern DECLEXPORT(const PDMPCIRAWHLPR0) g_pdmR0PciRawHlp;
+RT_C_DECLS_END
+
+/** List of PDMDEVMODREGR0 structures protected by the loader lock. */
+static RTLISTANCHOR g_PDMDevModList;
+
+
+/**
+ * Pointer to the ring-0 device registrations for VMMR0.
+ */
+static const PDMDEVREGR0 *g_apVMM0DevRegs[] =
+{
+    &g_DeviceAPIC,
+};
+
+/**
+ * Module device registration record for VMMR0.
+ */
+static PDMDEVMODREGR0 g_VBoxDDR0ModDevReg =
+{
+    /* .u32Version = */ PDM_DEVMODREGR0_VERSION,
+    /* .cDevRegs = */   RT_ELEMENTS(g_apVMM0DevRegs),
+    /* .papDevRegs = */ &g_apVMM0DevRegs[0],
+    /* .hMod = */       NULL,
+    /* .ListEntry = */  { NULL, NULL },
+};
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+
+
+/**
+ * Initializes the global ring-0 PDM data.
+ */
+VMMR0_INT_DECL(void) PDMR0Init(void *hMod)
+{
+    RTListInit(&g_PDMDevModList);
+    g_VBoxDDR0ModDevReg.hMod = hMod;
+    RTListAppend(&g_PDMDevModList, &g_VBoxDDR0ModDevReg.ListEntry);
+}
+
+
+/**
+ * Used by PDMR0CleanupVM to destroy a device instance.
+ *
+ * This is done during VM cleanup so that we're sure there are no active threads
+ * inside the device code.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   pDevIns     The device instance.
+ * @param   idxR0Device The device instance handle.
+ */
+static int pdmR0DeviceDestroy(PGVM pGVM, PPDMDEVINSR0 pDevIns, uint32_t idxR0Device)
+{
+    /*
+     * Assert sanity.
+     */
+    Assert(idxR0Device < pGVM->pdmr0.s.cDevInstances);
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    Assert(pDevIns->u32Version == PDM_DEVINSR0_VERSION);
+    Assert(pDevIns->Internal.s.idxR0Device == idxR0Device);
+
+    /*
+     * Call the final destructor if there is one.
+     */
+    if (pDevIns->pReg->pfnFinalDestruct)
+        pDevIns->pReg->pfnFinalDestruct(pDevIns);
+    pDevIns->u32Version = ~PDM_DEVINSR0_VERSION;
+
+    /*
+     * Remove the device from the instance table.
+     */
+    Assert(pGVM->pdmr0.s.apDevInstances[idxR0Device] == pDevIns);
+    pGVM->pdmr0.s.apDevInstances[idxR0Device] = NULL;
+    if (idxR0Device + 1 == pGVM->pdmr0.s.cDevInstances)
+        pGVM->pdmr0.s.cDevInstances = idxR0Device;
+
+    /*
+     * Free the ring-3 mapping and instance memory.
+     */
+    RTR0MEMOBJ hMemObj = pDevIns->Internal.s.hMapObj;
+    pDevIns->Internal.s.hMapObj = NIL_RTR0MEMOBJ;
+    RTR0MemObjFree(hMemObj, true);
+
+    hMemObj = pDevIns->Internal.s.hMemObj;
+    pDevIns->Internal.s.hMemObj = NIL_RTR0MEMOBJ;
+    RTR0MemObjFree(hMemObj, true);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Initializes the per-VM data for the PDM.
+ *
+ * This is called from under the GVMM lock, so it only need to initialize the
+ * data so PDMR0CleanupVM and others will work smoothly.
+ *
+ * @param   pGVM    Pointer to the global VM structure.
+ */
+VMMR0_INT_DECL(void) PDMR0InitPerVMData(PGVM pGVM)
+{
+    AssertCompile(sizeof(pGVM->pdm.s) <= sizeof(pGVM->pdm.padding));
+    AssertCompile(sizeof(pGVM->pdmr0.s) <= sizeof(pGVM->pdmr0.padding));
+
+    pGVM->pdmr0.s.cDevInstances = 0;
+}
+
+
+/**
+ * Cleans up any loose ends before the GVM structure is destroyed.
+ */
+VMMR0_INT_DECL(void) PDMR0CleanupVM(PGVM pGVM)
+{
+    uint32_t i = pGVM->pdmr0.s.cDevInstances;
+    while (i-- > 0)
+    {
+        PPDMDEVINSR0 pDevIns = pGVM->pdmr0.s.apDevInstances[i];
+        if (pDevIns)
+            pdmR0DeviceDestroy(pGVM, pDevIns, i);
+    }
+}
+
+
+/**
+ * Worker for PDMR0DeviceCreate that does the actual instantiation.
+ *
+ * Allocates a memory object and divides it up as follows:
+ * @verbatim
+   --------------------------------------
+   ring-0 devins
+   --------------------------------------
+   ring-0 instance data
+   --------------------------------------
+   ring-0 PCI device data (optional) ??
+   --------------------------------------
+   page alignment padding
+   --------------------------------------
+   ring-3 devins
+   --------------------------------------
+   ring-3 instance data
+   --------------------------------------
+   ring-3 PCI device data (optional) ??
+   --------------------------------------
+  [page alignment padding                ] -
+  [--------------------------------------]  \
+  [raw-mode devins                       ]   \
+  [--------------------------------------]   - Optional, only when raw-mode is enabled.
+  [raw-mode instance data                ]   /
+  [--------------------------------------]  /
+  [raw-mode PCI device data (optional)?? ] -
+   --------------------------------------
+   shared instance data
+   --------------------------------------
+   default crit section
+   --------------------------------------
+   shared PCI device data (optional)
+   --------------------------------------
+   @endverbatim
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   pDevReg         The device registration structure.
+ * @param   iInstance       The device instance number.
+ * @param   cbInstanceR3    The size of the ring-3 instance data.
+ * @param   cbInstanceRC    The size of the raw-mode instance data.
+ * @param   hMod            The module implementing the device.  On success, the
+ * @param   RCPtrMapping    The raw-mode context mapping address, NIL_RTGCPTR if
+ *                          not to include raw-mode.
+ * @param   ppDevInsR3      Where to return the ring-3 device instance address.
+ * @thread  EMT(0)
+ */
+static int pdmR0DeviceCreateWorker(PGVM pGVM, PCPDMDEVREGR0 pDevReg, uint32_t iInstance, uint32_t cbInstanceR3,
+                                   uint32_t cbInstanceRC, RTRGPTR RCPtrMapping, void *hMod, PPDMDEVINSR3 *ppDevInsR3)
+{
+    /*
+     * Check that the instance number isn't a duplicate.
+     */
+    for (size_t i = 0; i < pGVM->pdmr0.s.cDevInstances; i++)
+    {
+        PPDMDEVINS pCur = pGVM->pdmr0.s.apDevInstances[i];
+        AssertLogRelReturn(!pCur || pCur->pReg != pDevReg || pCur->iInstance != iInstance, VERR_DUPLICATE);
+    }
+
+    /*
+     * Figure out how much memory we need and allocate it.
+     */
+    uint32_t const cbRing0     = RT_ALIGN_32(RT_UOFFSETOF(PDMDEVINSR0, achInstanceData) + pDevReg->cbInstanceCC, PAGE_SIZE);
+    uint32_t const cbRing3     = RT_ALIGN_32(RT_UOFFSETOF(PDMDEVINSR3, achInstanceData) + cbInstanceR3,
+                                             RCPtrMapping != NIL_RTRGPTR ? PAGE_SIZE : 64);
+    uint32_t const cbRC        = RCPtrMapping != NIL_RTRGPTR ? 0
+                               : RT_ALIGN_32(RT_UOFFSETOF(PDMDEVINSRC, achInstanceData) + cbInstanceRC, 64);
+    uint32_t const cbShared    = RT_ALIGN_32(pDevReg->cbInstanceShared, 64);
+    uint32_t const cbCritSect  = RT_ALIGN_32(sizeof(PDMCRITSECT), 64);
+    uint32_t const cbMsixState = RT_ALIGN_32(pDevReg->cMaxMsixVectors * 16 + (pDevReg->cMaxMsixVectors + 7) / 8, _4K);
+    uint32_t const cbPciDev    = RT_ALIGN_32(RT_UOFFSETOF_DYN(PDMPCIDEV, abMsixState[cbMsixState]), 64);
+    uint32_t const cPciDevs    = RT_MIN(pDevReg->cMaxPciDevices, 8);
+    uint32_t const cbPciDevs   = cbPciDev * cPciDevs;
+    uint32_t const cbTotal     = RT_ALIGN_32(cbRing0 + cbRing3 + cbRC + cbShared + cbCritSect + cbPciDevs, PAGE_SIZE);
+    AssertLogRelMsgReturn(cbTotal <= PDM_MAX_DEVICE_INSTANCE_SIZE,
+                          ("Instance of '%s' is too big: cbTotal=%u, max %u\n",
+                           pDevReg->szName, cbTotal, PDM_MAX_DEVICE_INSTANCE_SIZE),
+                          VERR_OUT_OF_RANGE);
+
+    RTR0MEMOBJ hMemObj;
+    int rc = RTR0MemObjAllocPage(&hMemObj, cbTotal, false /*fExecutable*/);
+    if (RT_FAILURE(rc))
+        return rc;
+    RT_BZERO(RTR0MemObjAddress(hMemObj), cbTotal);
+
+    /* Map it. */
+    RTR0MEMOBJ hMapObj;
+    rc = RTR0MemObjMapUserEx(&hMapObj, hMemObj, (RTR3PTR)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE, RTR0ProcHandleSelf(),
+                             cbRing0, cbTotal - cbRing0);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMDEVINSR0        pDevIns   = (PPDMDEVINSR0)RTR0MemObjAddress(hMemObj);
+        struct PDMDEVINSR3 *pDevInsR3 = (struct PDMDEVINSR3 *)((uint8_t *)pDevIns + cbRing0);
+
+        /*
+         * Initialize the ring-0 instance.
+         */
+        pDevIns->u32Version             = PDM_DEVINSR0_VERSION;
+        pDevIns->iInstance              = iInstance;
+        pDevIns->pHlpR0                 = &g_pdmR0DevHlp;
+        pDevIns->pvInstanceDataR0       = (uint8_t *)pDevIns + cbRing0 + cbRing3 + cbRC;
+        pDevIns->pvInstanceDataForR0    = &pDevIns->achInstanceData[0];
+        pDevIns->pCritSectRoR0          = (PPDMCRITSECT)((uint8_t *)pDevIns->pvInstanceDataR0 + cbShared);
+        pDevIns->pReg                   = pDevReg;
+        pDevIns->pDevInsForR3           = RTR0MemObjAddressR3(hMapObj);
+        pDevIns->pDevInsForR3R0         = pDevInsR3;
+        pDevIns->pvInstanceDataForR3R0  = &pDevInsR3->achInstanceData[0];
+        pDevIns->cbPciDev               = cbPciDev;
+        pDevIns->cPciDevs               = cPciDevs;
+        for (uint32_t iPciDev = 0; iPciDev < cPciDevs; iPciDev++)
+        {
+            /* Note! PDMDevice.cpp has a copy of this code.  Keep in sync. */
+            PPDMPCIDEV pPciDev = (PPDMPCIDEV)((uint8_t *)pDevIns->pCritSectRoR0 + cbCritSect + cbPciDev * iPciDev);
+            if (iPciDev < RT_ELEMENTS(pDevIns->apPciDevs))
+                pDevIns->apPciDevs[iPciDev] = pPciDev;
+            pPciDev->cbConfig           = _4K;
+            pPciDev->cbMsixState        = cbMsixState;
+            pPciDev->idxSubDev          = (uint16_t)iPciDev;
+            pPciDev->Int.s.idxSubDev    = (uint16_t)iPciDev;
+            pPciDev->u32Magic           = PDMPCIDEV_MAGIC;
+        }
+        pDevIns->Internal.s.pGVM        = pGVM;
+        pDevIns->Internal.s.pRegR0      = pDevReg;
+        pDevIns->Internal.s.hMod        = hMod;
+        pDevIns->Internal.s.hMemObj     = hMemObj;
+        pDevIns->Internal.s.hMapObj     = hMapObj;
+        pDevIns->Internal.s.pInsR3R0    = pDevInsR3;
+        pDevIns->Internal.s.pIntR3R0    = &pDevInsR3->Internal.s;
+
+        /*
+         * Initialize the ring-3 instance data as much as we can.
+         * Note! PDMDevice.cpp does this job for ring-3 only devices.  Keep in sync.
+         */
+        pDevInsR3->u32Version           = PDM_DEVINSR3_VERSION;
+        pDevInsR3->iInstance            = iInstance;
+        pDevInsR3->cbRing3              = cbTotal - cbRing0;
+        pDevInsR3->fR0Enabled           = true;
+        pDevInsR3->fRCEnabled           = RCPtrMapping != NIL_RTRGPTR;
+        pDevInsR3->pvInstanceDataR3     = pDevIns->pDevInsForR3 + cbRing3 + cbRC;
+        pDevInsR3->pvInstanceDataForR3  = pDevIns->pDevInsForR3 + RT_UOFFSETOF(PDMDEVINSR3, achInstanceData);
+        pDevInsR3->pCritSectRoR3        = pDevIns->pDevInsForR3 + cbRing3 + cbRC + cbShared;
+        pDevInsR3->pDevInsR0RemoveMe    = pDevIns;
+        pDevInsR3->pvInstanceDataR0     = pDevIns->pvInstanceDataR0;
+        pDevInsR3->pvInstanceDataRC     = RCPtrMapping == NIL_RTRGPTR
+                                        ? NIL_RTRGPTR : pDevIns->pDevInsForRC + RT_UOFFSETOF(PDMDEVINSRC, achInstanceData);
+        pDevInsR3->pDevInsForRC         = pDevIns->pDevInsForRC;
+        pDevInsR3->pDevInsForRCR3       = pDevIns->pDevInsForR3 + cbRing3;
+        pDevInsR3->pDevInsForRCR3       = pDevInsR3->pDevInsForRCR3 + RT_UOFFSETOF(PDMDEVINSRC, achInstanceData);
+        pDevInsR3->cbPciDev             = cbPciDev;
+        pDevInsR3->cPciDevs             = cPciDevs;
+        for (uint32_t i = 0; i < RT_MIN(cPciDevs, RT_ELEMENTS(pDevIns->apPciDevs)); i++)
+            pDevInsR3->apPciDevs[i] = pDevInsR3->pCritSectRoR3 + cbCritSect + cbPciDev * i;
+
+        pDevInsR3->Internal.s.pVMR3     = pGVM->pVMR3;
+        pDevInsR3->Internal.s.fIntFlags = RCPtrMapping == NIL_RTRGPTR ? PDMDEVINSINT_FLAGS_R0_ENABLED
+                                        : PDMDEVINSINT_FLAGS_R0_ENABLED | PDMDEVINSINT_FLAGS_RC_ENABLED;
+
+        /*
+         * Initialize the raw-mode instance data as much as possible.
+         */
+        if (RCPtrMapping != NIL_RTRGPTR)
+        {
+            struct PDMDEVINSRC *pDevInsRC = RCPtrMapping == NIL_RTRGPTR ? NULL
+                                          : (struct PDMDEVINSRC *)((uint8_t *)pDevIns + cbRing0 + cbRing3);
+
+            pDevIns->pDevInsForRC           = RCPtrMapping;
+            pDevIns->pDevInsForRCR0         = pDevInsRC;
+            pDevIns->pvInstanceDataForRCR0  = &pDevInsRC->achInstanceData[0];
+
+            pDevInsRC->u32Version           = PDM_DEVINSRC_VERSION;
+            pDevInsRC->iInstance            = iInstance;
+            pDevInsRC->pvInstanceDataRC     = pDevIns->pDevInsForRC + cbRC;
+            pDevInsRC->pvInstanceDataForRC  = pDevIns->pDevInsForRC + RT_UOFFSETOF(PDMDEVINSRC, achInstanceData);
+            pDevInsRC->pCritSectRoRC        = pDevIns->pDevInsForRC + cbRC + cbShared;
+            pDevInsRC->cbPciDev             = cbPciDev;
+            pDevInsRC->cPciDevs             = cPciDevs;
+            for (uint32_t i = 0; i < RT_MIN(cPciDevs, RT_ELEMENTS(pDevIns->apPciDevs)); i++)
+                pDevInsRC->apPciDevs[i] = pDevInsRC->pCritSectRoRC + cbCritSect + cbPciDev * i;
+
+            pDevInsRC->Internal.s.pVMRC     = pGVM->pVMRC;
+        }
+
+        /*
+         * Add to the device instance array and set its handle value.
+         */
+        AssertCompile(sizeof(pGVM->pdmr0.padding) == sizeof(pGVM->pdmr0));
+        uint32_t idxR0Device = pGVM->pdmr0.s.cDevInstances;
+        if (idxR0Device < RT_ELEMENTS(pGVM->pdmr0.s.apDevInstances))
+        {
+            pGVM->pdmr0.s.apDevInstances[idxR0Device] = pDevIns;
+            pGVM->pdmr0.s.cDevInstances = idxR0Device + 1;
+            pDevIns->Internal.s.idxR0Device   = idxR0Device;
+            pDevInsR3->Internal.s.idxR0Device = idxR0Device;
+
+            /*
+             * Call the early constructor if present.
+             */
+            if (pDevReg->pfnEarlyConstruct)
+                rc = pDevReg->pfnEarlyConstruct(pDevIns);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * We're done.
+                 */
+                *ppDevInsR3 = RTR0MemObjAddressR3(hMapObj);
+                return rc;
+            }
+
+            /*
+             * Bail out.
+             */
+            if (pDevIns->pReg->pfnFinalDestruct)
+                pDevIns->pReg->pfnFinalDestruct(pDevIns);
+
+            pGVM->pdmr0.s.apDevInstances[idxR0Device] = NULL;
+            Assert(pGVM->pdmr0.s.cDevInstances == idxR0Device + 1);
+            pGVM->pdmr0.s.cDevInstances = idxR0Device;
+        }
+
+        RTR0MemObjFree(hMapObj, true);
+    }
+    RTR0MemObjFree(hMemObj, true);
+    return rc;
+}
+
+
+/**
+ * Used by ring-3 PDM to create a device instance that operates both in ring-3
+ * and ring-0.
+ *
+ * Creates an instance of a device (for both ring-3 and ring-0, and optionally
+ * raw-mode context).
+ *
+ * @returns VBox status code.
+ * @param   pGVM    The global (ring-0) VM structure.
+ * @param   pReq    Pointer to the request buffer.
+ * @thread  EMT(0)
+ */
+VMMR0_INT_DECL(int) PDMR0DeviceCreateReqHandler(PGVM pGVM, PPDMDEVICECREATEREQ pReq)
+{
+    LogFlow(("PDMR0DeviceCreateReqHandler: %s in %s\n", pReq->szDevName, pReq->szModName));
+
+    /*
+     * Validate the request.
+     */
+    AssertReturn(pReq->Hdr.cbReq == sizeof(*pReq), VERR_INVALID_PARAMETER);
+    pReq->pDevInsR3 = NIL_RTR3PTR;
+
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, 0);
+    AssertRCReturn(rc, rc);
+
+    AssertReturn(pReq->fFlags           != 0, VERR_INVALID_FLAGS);
+    AssertReturn(pReq->fClass           != 0, VERR_WRONG_TYPE);
+    AssertReturn(pReq->uSharedVersion   != 0, VERR_INVALID_PARAMETER);
+    AssertReturn(pReq->cbInstanceShared != 0, VERR_INVALID_PARAMETER);
+    size_t const cchDevName = RTStrNLen(pReq->szDevName, sizeof(pReq->szDevName));
+    AssertReturn(cchDevName < sizeof(pReq->szDevName), VERR_NO_STRING_TERMINATOR);
+    AssertReturn(cchDevName > 0, VERR_EMPTY_STRING);
+    AssertReturn(cchDevName < RT_SIZEOFMEMB(PDMDEVREG, szName), VERR_NOT_FOUND);
+
+    size_t const cchModName = RTStrNLen(pReq->szModName, sizeof(pReq->szModName));
+    AssertReturn(cchModName < sizeof(pReq->szModName), VERR_NO_STRING_TERMINATOR);
+    AssertReturn(cchModName > 0, VERR_EMPTY_STRING);
+    AssertReturn(pReq->cbInstanceShared <= PDM_MAX_DEVICE_INSTANCE_SIZE, VERR_OUT_OF_RANGE);
+    AssertReturn(pReq->cbInstanceR3 <= PDM_MAX_DEVICE_INSTANCE_SIZE, VERR_OUT_OF_RANGE);
+    AssertReturn(pReq->cbInstanceRC <= PDM_MAX_DEVICE_INSTANCE_SIZE, VERR_OUT_OF_RANGE);
+    AssertReturn(pReq->iInstance < 1024, VERR_OUT_OF_RANGE);
+    AssertReturn(pReq->iInstance < pReq->cMaxInstances, VERR_OUT_OF_RANGE);
+    AssertReturn(pReq->cMaxPciDevices <= 8, VERR_OUT_OF_RANGE);
+    AssertReturn(pReq->cMaxMsixVectors <= VBOX_MSIX_MAX_ENTRIES, VERR_OUT_OF_RANGE);
+
+    /*
+     * Reference the module.
+     */
+    void *hMod = NULL;
+    rc = SUPR0LdrModByName(pGVM->pSession, pReq->szModName, &hMod);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("PDMR0DeviceCreateReqHandler: SUPR0LdrModByName(,%s,) failed: %Rrc\n", pReq->szModName, rc));
+        return rc;
+    }
+
+    /*
+     * Look for the the module and the device registration structure.
+     */
+    int rcLock = SUPR0LdrLock(pGVM->pSession);
+    AssertRC(rc);
+
+    rc = VERR_NOT_FOUND;
+    PPDMDEVMODREGR0 pMod;
+    RTListForEach(&g_PDMDevModList, pMod, PDMDEVMODREGR0, ListEntry)
+    {
+        if (pMod->hMod == hMod)
+        {
+            /*
+             * Found the module. We can drop the loader lock now before we
+             * search the devices it registers.
+             */
+            if (RT_SUCCESS(rcLock))
+            {
+                rcLock = SUPR0LdrUnlock(pGVM->pSession);
+                AssertRC(rcLock);
+            }
+            rcLock = VERR_ALREADY_RESET;
+
+            PCPDMDEVREGR0 *papDevRegs = pMod->papDevRegs;
+            size_t         i          = pMod->cDevRegs;
+            while (i-- > 0)
+            {
+                PCPDMDEVREGR0 pDevReg = papDevRegs[i];
+                LogFlow(("PDMR0DeviceCreateReqHandler: candidate #%u: %s %#x\n", i, pReq->szDevName, pDevReg->u32Version));
+                if (   PDM_VERSION_ARE_COMPATIBLE(pDevReg->u32Version, PDM_DEVREGR0_VERSION)
+                    && pDevReg->szName[cchDevName] == '\0'
+                    && memcmp(pDevReg->szName, pReq->szDevName, cchDevName) == 0)
+                {
+
+                    /*
+                     * Found the device, now check whether it matches the ring-3 registration.
+                     */
+                    if (   pReq->uSharedVersion   == pDevReg->uSharedVersion
+                        && pReq->cbInstanceShared == pDevReg->cbInstanceShared
+                        && pReq->cbInstanceRC     == pDevReg->cbInstanceRC
+                        && pReq->fFlags           == pDevReg->fFlags
+                        && pReq->fClass           == pDevReg->fClass
+                        && pReq->cMaxInstances    == pDevReg->cMaxInstances
+                        && pReq->cMaxPciDevices   == pDevReg->cMaxPciDevices
+                        && pReq->cMaxMsixVectors  == pDevReg->cMaxMsixVectors)
+                    {
+                        rc = pdmR0DeviceCreateWorker(pGVM, pDevReg, pReq->iInstance, pReq->cbInstanceR3, pReq->cbInstanceRC,
+                                                     NIL_RTRCPTR /** @todo new raw-mode */, hMod, &pReq->pDevInsR3);
+                        if (RT_SUCCESS(rc))
+                            hMod = NULL; /* keep the module reference */
+                    }
+                    else
+                    {
+                        LogRel(("PDMR0DeviceCreate: Ring-3 does not match ring-0 device registration (%s):\n"
+                                "    uSharedVersion: %#x vs %#x\n"
+                                "  cbInstanceShared: %#x vs %#x\n"
+                                "      cbInstanceRC: %#x vs %#x\n"
+                                "            fFlags: %#x vs %#x\n"
+                                "            fClass: %#x vs %#x\n"
+                                "     cMaxInstances: %#x vs %#x\n"
+                                "    cMaxPciDevices: %#x vs %#x\n"
+                                "   cMaxMsixVectors: %#x vs %#x\n"
+                                ,
+                                pReq->szDevName,
+                                pReq->uSharedVersion,   pDevReg->uSharedVersion,
+                                pReq->cbInstanceShared, pDevReg->cbInstanceShared,
+                                pReq->cbInstanceRC,     pDevReg->cbInstanceRC,
+                                pReq->fFlags,           pDevReg->fFlags,
+                                pReq->fClass,           pDevReg->fClass,
+                                pReq->cMaxInstances,    pDevReg->cMaxInstances,
+                                pReq->cMaxPciDevices,   pDevReg->cMaxPciDevices,
+                                pReq->cMaxMsixVectors,  pDevReg->cMaxMsixVectors));
+                        rc = VERR_INCOMPATIBLE_CONFIG;
+                    }
+                }
+            }
+            break;
+        }
+    }
+
+    if (RT_SUCCESS_NP(rcLock))
+    {
+        rcLock = SUPR0LdrUnlock(pGVM->pSession);
+        AssertRC(rcLock);
+    }
+    SUPR0LdrModRelease(pGVM->pSession, hMod);
+    return rc;
+}
+
+
+/**
+ * Used by ring-3 PDM to call standard ring-0 device methods.
+ *
+ * @returns VBox status code.
+ * @param   pGVM    The global (ring-0) VM structure.
+ * @param   pReq    Pointer to the request buffer.
+ * @param   idCpu   The ID of the calling EMT.
+ * @thread  EMT(0), except for PDMDEVICEGENCALL_REQUEST which can be any EMT.
+ */
+VMMR0_INT_DECL(int) PDMR0DeviceGenCallReqHandler(PGVM pGVM, PPDMDEVICEGENCALLREQ pReq, VMCPUID idCpu)
+{
+    /*
+     * Validate the request.
+     */
+    AssertReturn(pReq->Hdr.cbReq == sizeof(*pReq), VERR_INVALID_PARAMETER);
+
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    AssertRCReturn(rc, rc);
+
+    AssertReturn(pReq->idxR0Device < pGVM->pdmr0.s.cDevInstances, VERR_INVALID_HANDLE);
+    PPDMDEVINSR0 pDevIns = pGVM->pdmr0.s.apDevInstances[pReq->idxR0Device];
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    AssertReturn(pDevIns->pDevInsForR3 == pReq->pDevInsR3, VERR_INVALID_HANDLE);
+
+    /*
+     * Make the call.
+     */
+    rc = VINF_SUCCESS /*VINF_NOT_IMPLEMENTED*/;
+    switch (pReq->enmCall)
+    {
+        case PDMDEVICEGENCALL_CONSTRUCT:
+            AssertMsgBreakStmt(pGVM->enmVMState < VMSTATE_CREATED, ("enmVMState=%d\n", pGVM->enmVMState), rc = VERR_INVALID_STATE);
+            AssertReturn(idCpu == 0,  VERR_VM_THREAD_NOT_EMT);
+            if (pDevIns->pReg->pfnConstruct)
+                rc = pDevIns->pReg->pfnConstruct(pDevIns);
+            break;
+
+        case PDMDEVICEGENCALL_DESTRUCT:
+            AssertMsgBreakStmt(pGVM->enmVMState < VMSTATE_CREATED || pGVM->enmVMState >= VMSTATE_DESTROYING,
+                               ("enmVMState=%d\n", pGVM->enmVMState), rc = VERR_INVALID_STATE);
+            AssertReturn(idCpu == 0,  VERR_VM_THREAD_NOT_EMT);
+            if (pDevIns->pReg->pfnDestruct)
+            {
+                pDevIns->pReg->pfnDestruct(pDevIns);
+                rc = VINF_SUCCESS;
+            }
+            break;
+
+        case PDMDEVICEGENCALL_REQUEST:
+            if (pDevIns->pReg->pfnRequest)
+                rc = pDevIns->pReg->pfnRequest(pDevIns, pReq->Params.Req.uReq, pReq->Params.Req.uArg);
+            else
+                rc = VERR_INVALID_FUNCTION;
+            break;
+
+        default:
+            AssertMsgFailed(("enmCall=%d\n", pReq->enmCall));
+            rc = VERR_INVALID_FUNCTION;
+            break;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Legacy device mode compatiblity.
+ *
+ * @returns VBox status code.
+ * @param   pGVM    The global (ring-0) VM structure.
+ * @param   pReq    Pointer to the request buffer.
+ * @thread  EMT(0)
+ */
+VMMR0_INT_DECL(int) PDMR0DeviceCompatSetCritSectReqHandler(PGVM pGVM, PPDMDEVICECOMPATSETCRITSECTREQ pReq)
+{
+    /*
+     * Validate the request.
+     */
+    AssertReturn(pReq->Hdr.cbReq == sizeof(*pReq), VERR_INVALID_PARAMETER);
+
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, 0);
+    AssertRCReturn(rc, rc);
+
+    AssertReturn(pReq->idxR0Device < pGVM->pdmr0.s.cDevInstances, VERR_INVALID_HANDLE);
+    PPDMDEVINSR0 pDevIns = pGVM->pdmr0.s.apDevInstances[pReq->idxR0Device];
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    AssertReturn(pDevIns->pDevInsForR3 == pReq->pDevInsR3, VERR_INVALID_HANDLE);
+
+    AssertReturn(pGVM->enmVMState == VMSTATE_CREATING, VERR_INVALID_STATE);
+
+    /*
+     * The critical section address can be in a few different places:
+     *      1. shared data.
+     *      2. nop section.
+     *      3. pdm critsect.
+     */
+    PPDMCRITSECT pCritSect;
+    if (pReq->pCritSectR3 == pGVM->pVMR3 + RT_UOFFSETOF(VM, pdm.s.NopCritSect))
+    {
+        pCritSect = &pGVM->pdm.s.NopCritSect;
+        Log(("PDMR0DeviceCompatSetCritSectReqHandler: Nop - %p %#x\n", pCritSect, pCritSect->s.Core.u32Magic));
+    }
+    else if (pReq->pCritSectR3 == pGVM->pVMR3 + RT_UOFFSETOF(VM, pdm.s.CritSect))
+    {
+        pCritSect = &pGVM->pdm.s.CritSect;
+        Log(("PDMR0DeviceCompatSetCritSectReqHandler: PDM - %p %#x\n", pCritSect, pCritSect->s.Core.u32Magic));
+    }
+    else
+    {
+        size_t offCritSect = pReq->pCritSectR3 - pDevIns->pDevInsForR3R0->pvInstanceDataR3;
+        AssertLogRelMsgReturn(   offCritSect                       <  pDevIns->pReg->cbInstanceShared
+                              && offCritSect + sizeof(PDMCRITSECT) <= pDevIns->pReg->cbInstanceShared,
+                              ("offCritSect=%p pCritSectR3=%p cbInstanceShared=%#x (%s)\n",
+                               offCritSect, pReq->pCritSectR3, pDevIns->pReg->cbInstanceShared, pDevIns->pReg->szName),
+                              VERR_INVALID_POINTER);
+        pCritSect = (PPDMCRITSECT)((uint8_t *)pDevIns->pvInstanceDataR0 + offCritSect);
+        Log(("PDMR0DeviceCompatSetCritSectReqHandler: custom - %#x/%p %#x\n", offCritSect, pCritSect, pCritSect->s.Core.u32Magic));
+    }
+    AssertLogRelMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC,
+                          ("cs=%p magic=%#x dev=%s\n", pCritSect, pCritSect->s.Core.u32Magic, pDevIns->pReg->szName),
+                          VERR_INVALID_MAGIC);
+
+    /*
+     * Make the update.
+     */
+    pDevIns->pCritSectRoR0 = pCritSect;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Registers the device implementations living in a module.
+ *
+ * This should normally only be called during ModuleInit().  The should be a
+ * call to PDMR0DeviceDeregisterModule from the ModuleTerm() function to undo
+ * the effects of this call.
+ *
+ * @returns VBox status code.
+ * @param   hMod            The module handle of the module being registered.
+ * @param   pModReg         The module registration structure.  This will be
+ *                          used directly so it must live as long as the module
+ *                          and be writable.
+ *
+ * @note    Caller must own the loader lock!
+ */
+VMMR0DECL(int) PDMR0DeviceRegisterModule(void *hMod, PPDMDEVMODREGR0 pModReg)
+{
+    /*
+     * Validate the input.
+     */
+    AssertPtrReturn(hMod, VERR_INVALID_HANDLE);
+    Assert(SUPR0LdrIsLockOwnerByMod(hMod, true));
+
+    AssertPtrReturn(pModReg, VERR_INVALID_POINTER);
+    AssertLogRelMsgReturn(PDM_VERSION_ARE_COMPATIBLE(pModReg->u32Version, PDM_DEVMODREGR0_VERSION),
+                          ("pModReg->u32Version=%#x vs %#x\n", pModReg->u32Version, PDM_DEVMODREGR0_VERSION),
+                          VERR_VERSION_MISMATCH);
+    AssertLogRelMsgReturn(pModReg->cDevRegs <= 256 && pModReg->cDevRegs > 0, ("cDevRegs=%u\n", pModReg->cDevRegs),
+                          VERR_OUT_OF_RANGE);
+    AssertLogRelMsgReturn(pModReg->hMod == NULL, ("hMod=%p\n", pModReg->hMod), VERR_INVALID_PARAMETER);
+    AssertLogRelMsgReturn(pModReg->ListEntry.pNext == NULL, ("pNext=%p\n", pModReg->ListEntry.pNext), VERR_INVALID_PARAMETER);
+    AssertLogRelMsgReturn(pModReg->ListEntry.pPrev == NULL, ("pPrev=%p\n", pModReg->ListEntry.pPrev), VERR_INVALID_PARAMETER);
+
+    for (size_t i = 0; i < pModReg->cDevRegs; i++)
+    {
+        PCPDMDEVREGR0 pDevReg = pModReg->papDevRegs[i];
+        AssertLogRelMsgReturn(RT_VALID_PTR(pDevReg), ("[%u]: %p\n", i, pDevReg), VERR_INVALID_POINTER);
+        AssertLogRelMsgReturn(PDM_VERSION_ARE_COMPATIBLE(pDevReg->u32Version, PDM_DEVREGR0_VERSION),
+                              ("pDevReg->u32Version=%#x vs %#x\n", pModReg->u32Version, PDM_DEVREGR0_VERSION), VERR_VERSION_MISMATCH);
+        AssertLogRelMsgReturn(RT_VALID_PTR(pDevReg->pszDescription), ("[%u]: %p\n", i, pDevReg->pszDescription), VERR_INVALID_POINTER);
+        AssertLogRelMsgReturn(pDevReg->uReserved0     == 0, ("[%u]: %#x\n", i, pDevReg->uReserved0),     VERR_INVALID_PARAMETER);
+        AssertLogRelMsgReturn(pDevReg->fClass         != 0, ("[%u]: %#x\n", i, pDevReg->fClass),         VERR_INVALID_PARAMETER);
+        AssertLogRelMsgReturn(pDevReg->fFlags         != 0, ("[%u]: %#x\n", i, pDevReg->fFlags),         VERR_INVALID_PARAMETER);
+        AssertLogRelMsgReturn(pDevReg->cMaxInstances   > 0, ("[%u]: %#x\n", i, pDevReg->cMaxInstances),  VERR_INVALID_PARAMETER);
+        AssertLogRelMsgReturn(pDevReg->cMaxPciDevices <= 8, ("[%u]: %#x\n", i, pDevReg->cMaxPciDevices), VERR_INVALID_PARAMETER);
+        AssertLogRelMsgReturn(pDevReg->cMaxMsixVectors <= VBOX_MSIX_MAX_ENTRIES,
+                              ("[%u]: %#x\n", i, pDevReg->cMaxMsixVectors), VERR_INVALID_PARAMETER);
+
+        /* The name must be printable ascii and correctly terminated. */
+        for (size_t off = 0; off < RT_ELEMENTS(pDevReg->szName); off++)
+        {
+            char ch = pDevReg->szName[off];
+            AssertLogRelMsgReturn(RT_C_IS_PRINT(ch) || (ch == '\0' && off > 0),
+                                  ("[%u]: off=%u  szName: %.*Rhxs\n", i, off, sizeof(pDevReg->szName), &pDevReg->szName[0]),
+                                  VERR_INVALID_NAME);
+            if (ch == '\0')
+                break;
+        }
+    }
+
+    /*
+     * Add it, assuming we're being called at ModuleInit/ModuleTerm time only, or
+     * that the caller has already taken the loader lock.
+     */
+    pModReg->hMod = hMod;
+    RTListAppend(&g_PDMDevModList, &pModReg->ListEntry);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters the device implementations living in a module.
+ *
+ * This should normally only be called during ModuleTerm().
+ *
+ * @returns VBox status code.
+ * @param   hMod            The module handle of the module being registered.
+ * @param   pModReg         The module registration structure.  This will be
+ *                          used directly so it must live as long as the module
+ *                          and be writable.
+ *
+ * @note    Caller must own the loader lock!
+ */
+VMMR0DECL(int) PDMR0DeviceDeregisterModule(void *hMod, PPDMDEVMODREGR0 pModReg)
+{
+    /*
+     * Validate the input.
+     */
+    AssertPtrReturn(hMod, VERR_INVALID_HANDLE);
+    Assert(SUPR0LdrIsLockOwnerByMod(hMod, true));
+
+    AssertPtrReturn(pModReg, VERR_INVALID_POINTER);
+    AssertLogRelMsgReturn(PDM_VERSION_ARE_COMPATIBLE(pModReg->u32Version, PDM_DEVMODREGR0_VERSION),
+                          ("pModReg->u32Version=%#x vs %#x\n", pModReg->u32Version, PDM_DEVMODREGR0_VERSION),
+                          VERR_VERSION_MISMATCH);
+    AssertLogRelMsgReturn(pModReg->hMod == hMod || pModReg->hMod == NULL, ("pModReg->hMod=%p vs %p\n", pModReg->hMod, hMod),
+                          VERR_INVALID_PARAMETER);
+
+    /*
+     * Unlink the registration record and return it to virgin conditions.  Ignore
+     * the call if not registered.
+     */
+    if (pModReg->hMod)
+    {
+        pModReg->hMod = NULL;
+        RTListNodeRemove(&pModReg->ListEntry);
+        pModReg->ListEntry.pNext = NULL;
+        pModReg->ListEntry.pPrev = NULL;
+        return VINF_SUCCESS;
+    }
+    return VWRN_NOT_FOUND;
+}
+
diff --git a/src/VBox/VMM/VMMR0/PDMR0Driver.cpp b/src/VBox/VMM/VMMR0/PDMR0Driver.cpp
new file mode 100644
index 00000000..c22d9805
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/PDMR0Driver.cpp
@@ -0,0 +1,163 @@
+/* $Id: PDMR0Driver.cpp $ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, R0 Driver parts.
+ */
+
+/*
+ * Copyright (C) 2010-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DRIVER
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/gvmm.h>
+
+#include <VBox/log.h>
+#include <iprt/errcore.h>
+#include <iprt/assert.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+extern DECLEXPORT(const PDMDRVHLPR0)    g_pdmR0DrvHlp;
+RT_C_DECLS_END
+
+
+/** @name Ring-0 Context Driver Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMDRVHLPR0,pfnVMSetError} */
+static DECLCALLBACK(int) pdmR0DrvHlp_VMSetError(PPDMDRVINS pDrvIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    va_list args;
+    va_start(args, pszFormat);
+    int rc2 = VMSetErrorV(pDrvIns->Internal.s.pVMR0, rc, RT_SRC_POS_ARGS, pszFormat, args); Assert(rc2 == rc); NOREF(rc2);
+    va_end(args);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR0,pfnVMSetErrorV} */
+static DECLCALLBACK(int) pdmR0DrvHlp_VMSetErrorV(PPDMDRVINS pDrvIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    int rc2 = VMSetErrorV(pDrvIns->Internal.s.pVMR0, rc, RT_SRC_POS_ARGS, pszFormat, va); Assert(rc2 == rc); NOREF(rc2);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR0,pfnVMSetRuntimeError} */
+static DECLCALLBACK(int) pdmR0DrvHlp_VMSetRuntimeError(PPDMDRVINS pDrvIns, uint32_t fFlags, const char *pszErrorId,
+                                                       const char *pszFormat, ...)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    va_list va;
+    va_start(va, pszFormat);
+    int rc = VMSetRuntimeErrorV(pDrvIns->Internal.s.pVMR0, fFlags, pszErrorId, pszFormat, va);
+    va_end(va);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR0,pfnVMSetRuntimeErrorV} */
+static DECLCALLBACK(int) pdmR0DrvHlp_VMSetRuntimeErrorV(PPDMDRVINS pDrvIns, uint32_t fFlags, const char *pszErrorId,
+                                                        const char *pszFormat, va_list va)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    int rc = VMSetRuntimeErrorV(pDrvIns->Internal.s.pVMR0, fFlags, pszErrorId, pszFormat, va);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR0,pfnAssertEMT} */
+static DECLCALLBACK(bool) pdmR0DrvHlp_AssertEMT(PPDMDRVINS pDrvIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    if (VM_IS_EMT(pDrvIns->Internal.s.pVMR0))
+        return true;
+
+    RTAssertMsg1Weak("AssertEMT", iLine, pszFile, pszFunction);
+    RTAssertPanic();
+    return false;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR0,pfnAssertOther} */
+static DECLCALLBACK(bool) pdmR0DrvHlp_AssertOther(PPDMDRVINS pDrvIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    if (!VM_IS_EMT(pDrvIns->Internal.s.pVMR0))
+        return true;
+
+    RTAssertMsg1Weak("AssertOther", iLine, pszFile, pszFunction);
+    RTAssertPanic();
+    return false;
+}
+
+
+/**
+ * The Ring-0 Context Driver Helper Callbacks.
+ */
+extern DECLEXPORT(const PDMDRVHLPR0) g_pdmR0DrvHlp =
+{
+    PDM_DRVHLPRC_VERSION,
+    pdmR0DrvHlp_VMSetError,
+    pdmR0DrvHlp_VMSetErrorV,
+    pdmR0DrvHlp_VMSetRuntimeError,
+    pdmR0DrvHlp_VMSetRuntimeErrorV,
+    pdmR0DrvHlp_AssertEMT,
+    pdmR0DrvHlp_AssertOther,
+    PDM_DRVHLPRC_VERSION
+};
+
+/** @} */
+
+
+
+/**
+ * PDMDrvHlpCallR0 helper.
+ *
+ * @returns See PFNPDMDRVREQHANDLERR0.
+ * @param   pGVM    The global (ring-0) VM structure. (For validation.)
+ * @param   pReq    Pointer to the request buffer.
+ */
+VMMR0_INT_DECL(int) PDMR0DriverCallReqHandler(PGVM pGVM, PPDMDRIVERCALLREQHANDLERREQ pReq)
+{
+    /*
+     * Validate input and make the call.
+     */
+    int rc = GVMMR0ValidateGVM(pGVM);
+    if (RT_SUCCESS(rc))
+    {
+        AssertPtrReturn(pReq, VERR_INVALID_POINTER);
+        AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
+
+        PPDMDRVINS pDrvIns = pReq->pDrvInsR0;
+        AssertPtrReturn(pDrvIns, VERR_INVALID_POINTER);
+        AssertReturn(pDrvIns->Internal.s.pVMR0 == pGVM, VERR_INVALID_PARAMETER);
+
+        PFNPDMDRVREQHANDLERR0 pfnReqHandlerR0 = pDrvIns->Internal.s.pfnReqHandlerR0;
+        AssertPtrReturn(pfnReqHandlerR0, VERR_INVALID_POINTER);
+
+        rc = pfnReqHandlerR0(pDrvIns, pReq->uOperation, pReq->u64Arg);
+    }
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR0/PGMR0.cpp b/src/VBox/VMM/VMMR0/PGMR0.cpp
new file mode 100644
index 00000000..365cd90a
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/PGMR0.cpp
@@ -0,0 +1,807 @@
+/* $Id: PGMR0.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor, Ring-0.
+ */
+
+/*
+ * Copyright (C) 2007-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/rawpci.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/gmm.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/gvm.h>
+#include "PGMInline.h"
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+#include <iprt/mem.h>
+#include <iprt/memobj.h>
+
+
+/*
+ * Instantiate the ring-0 header/code templates.
+ */
+/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_PROT(name)
+#include "PGMR0Bth.h"
+#undef PGM_BTH_NAME
+
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_PROT(name)
+#include "PGMR0Bth.h"
+#undef PGM_BTH_NAME
+
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_AMD64_PROT(name)
+#include "PGMR0Bth.h"
+#undef PGM_BTH_NAME
+
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_PROT(name)
+#include "PGMR0Bth.h"
+#undef PGM_BTH_NAME
+
+
+/**
+ * Initializes the per-VM data for the PGM.
+ *
+ * This is called from under the GVMM lock, so it should only initialize the
+ * data so PGMR0CleanupVM and others will work smoothly.
+ *
+ * @returns VBox status code.
+ * @param   pGVM    Pointer to the global VM structure.
+ */
+VMMR0_INT_DECL(int) PGMR0InitPerVMData(PGVM pGVM)
+{
+    AssertCompile(sizeof(pGVM->pgm.s) <= sizeof(pGVM->pgm.padding));
+    AssertCompile(sizeof(pGVM->pgmr0.s) <= sizeof(pGVM->pgmr0.padding));
+
+    AssertCompile(RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs) == RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMapObjs));
+    for (uint32_t i = 0; i < RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs); i++)
+    {
+        pGVM->pgmr0.s.ahPoolMemObjs[i] = NIL_RTR0MEMOBJ;
+        pGVM->pgmr0.s.ahPoolMapObjs[i] = NIL_RTR0MEMOBJ;
+    }
+    return RTCritSectInit(&pGVM->pgmr0.s.PoolGrowCritSect);
+}
+
+
+/**
+ * Initalize the per-VM PGM for ring-0.
+ *
+ * @returns VBox status code.
+ * @param   pGVM    Pointer to the global VM structure.
+ */
+VMMR0_INT_DECL(int) PGMR0InitVM(PGVM pGVM)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    rc = PGMR0DynMapInitVM(pGVM);
+#endif
+    RT_NOREF(pGVM);
+    return rc;
+}
+
+
+/**
+ * Cleans up any loose ends before the GVM structure is destroyed.
+ */
+VMMR0_INT_DECL(void) PGMR0CleanupVM(PGVM pGVM)
+{
+    for (uint32_t i = 0; i < RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs); i++)
+    {
+        if (pGVM->pgmr0.s.ahPoolMapObjs[i] != NIL_RTR0MEMOBJ)
+        {
+            int rc = RTR0MemObjFree(pGVM->pgmr0.s.ahPoolMapObjs[i], true /*fFreeMappings*/);
+            AssertRC(rc);
+            pGVM->pgmr0.s.ahPoolMapObjs[i] = NIL_RTR0MEMOBJ;
+        }
+
+        if (pGVM->pgmr0.s.ahPoolMemObjs[i] != NIL_RTR0MEMOBJ)
+        {
+            int rc = RTR0MemObjFree(pGVM->pgmr0.s.ahPoolMemObjs[i], true /*fFreeMappings*/);
+            AssertRC(rc);
+            pGVM->pgmr0.s.ahPoolMemObjs[i] = NIL_RTR0MEMOBJ;
+        }
+    }
+
+    if (RTCritSectIsInitialized(&pGVM->pgmr0.s.PoolGrowCritSect))
+        RTCritSectDelete(&pGVM->pgmr0.s.PoolGrowCritSect);
+}
+
+
+/**
+ * Worker function for PGMR3PhysAllocateHandyPages and pgmPhysEnsureHandyPage.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success. FF cleared.
+ * @retval  VINF_EM_NO_MEMORY if we're out of memory. The FF is set in this case.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The ID of the calling EMT.
+ *
+ * @thread  EMT(idCpu)
+ *
+ * @remarks Must be called from within the PGM critical section. The caller
+ *          must clear the new pages.
+ */
+VMMR0_INT_DECL(int) PGMR0PhysAllocateHandyPages(PGVM pGVM, VMCPUID idCpu)
+{
+    /*
+     * Validate inputs.
+     */
+    AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID); /* caller already checked this, but just to be sure. */
+    AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_NOT_OWNER);
+    PGM_LOCK_ASSERT_OWNER_EX(pGVM, &pGVM->aCpus[idCpu]);
+
+    /*
+     * Check for error injection.
+     */
+    if (RT_UNLIKELY(pGVM->pgm.s.fErrInjHandyPages))
+        return VERR_NO_MEMORY;
+
+    /*
+     * Try allocate a full set of handy pages.
+     */
+    uint32_t iFirst = pGVM->pgm.s.cHandyPages;
+    AssertReturn(iFirst <= RT_ELEMENTS(pGVM->pgm.s.aHandyPages), VERR_PGM_HANDY_PAGE_IPE);
+    uint32_t cPages = RT_ELEMENTS(pGVM->pgm.s.aHandyPages) - iFirst;
+    if (!cPages)
+        return VINF_SUCCESS;
+    int rc = GMMR0AllocateHandyPages(pGVM, idCpu, cPages, cPages, &pGVM->pgm.s.aHandyPages[iFirst]);
+    if (RT_SUCCESS(rc))
+    {
+#ifdef VBOX_STRICT
+        for (uint32_t i = 0; i < RT_ELEMENTS(pGVM->pgm.s.aHandyPages); i++)
+        {
+            Assert(pGVM->pgm.s.aHandyPages[i].idPage != NIL_GMM_PAGEID);
+            Assert(pGVM->pgm.s.aHandyPages[i].idPage <= GMM_PAGEID_LAST);
+            Assert(pGVM->pgm.s.aHandyPages[i].idSharedPage == NIL_GMM_PAGEID);
+            Assert(pGVM->pgm.s.aHandyPages[i].HCPhysGCPhys != NIL_RTHCPHYS);
+            Assert(!(pGVM->pgm.s.aHandyPages[i].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
+        }
+#endif
+
+        pGVM->pgm.s.cHandyPages = RT_ELEMENTS(pGVM->pgm.s.aHandyPages);
+    }
+    else if (rc != VERR_GMM_SEED_ME)
+    {
+        if (    (   rc == VERR_GMM_HIT_GLOBAL_LIMIT
+                 || rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT)
+            &&  iFirst < PGM_HANDY_PAGES_MIN)
+        {
+
+#ifdef VBOX_STRICT
+            /* We're ASSUMING that GMM has updated all the entires before failing us. */
+            uint32_t i;
+            for (i = iFirst; i < RT_ELEMENTS(pGVM->pgm.s.aHandyPages); i++)
+            {
+                Assert(pGVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID);
+                Assert(pGVM->pgm.s.aHandyPages[i].idSharedPage == NIL_GMM_PAGEID);
+                Assert(pGVM->pgm.s.aHandyPages[i].HCPhysGCPhys == NIL_RTHCPHYS);
+            }
+#endif
+
+            /*
+             * Reduce the number of pages until we hit the minimum limit.
+             */
+            do
+            {
+                cPages >>= 1;
+                if (cPages + iFirst < PGM_HANDY_PAGES_MIN)
+                    cPages = PGM_HANDY_PAGES_MIN - iFirst;
+                rc = GMMR0AllocateHandyPages(pGVM, idCpu, 0, cPages, &pGVM->pgm.s.aHandyPages[iFirst]);
+            } while (   (   rc == VERR_GMM_HIT_GLOBAL_LIMIT
+                         || rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT)
+                     && cPages + iFirst > PGM_HANDY_PAGES_MIN);
+            if (RT_SUCCESS(rc))
+            {
+#ifdef VBOX_STRICT
+                i = iFirst + cPages;
+                while (i-- > 0)
+                {
+                    Assert(pGVM->pgm.s.aHandyPages[i].idPage != NIL_GMM_PAGEID);
+                    Assert(pGVM->pgm.s.aHandyPages[i].idPage <= GMM_PAGEID_LAST);
+                    Assert(pGVM->pgm.s.aHandyPages[i].idSharedPage == NIL_GMM_PAGEID);
+                    Assert(pGVM->pgm.s.aHandyPages[i].HCPhysGCPhys != NIL_RTHCPHYS);
+                    Assert(!(pGVM->pgm.s.aHandyPages[i].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
+                }
+
+                for (i = cPages + iFirst; i < RT_ELEMENTS(pGVM->pgm.s.aHandyPages); i++)
+                {
+                    Assert(pGVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID);
+                    Assert(pGVM->pgm.s.aHandyPages[i].idSharedPage == NIL_GMM_PAGEID);
+                    Assert(pGVM->pgm.s.aHandyPages[i].HCPhysGCPhys == NIL_RTHCPHYS);
+                }
+#endif
+
+                pGVM->pgm.s.cHandyPages = iFirst + cPages;
+            }
+        }
+
+        if (RT_FAILURE(rc) && rc != VERR_GMM_SEED_ME)
+        {
+            LogRel(("PGMR0PhysAllocateHandyPages: rc=%Rrc iFirst=%d cPages=%d\n", rc, iFirst, cPages));
+            VM_FF_SET(pGVM, VM_FF_PGM_NO_MEMORY);
+        }
+    }
+
+
+    LogFlow(("PGMR0PhysAllocateHandyPages: cPages=%d rc=%Rrc\n", cPages, rc));
+    return rc;
+}
+
+
+/**
+ * Flushes any changes pending in the handy page array.
+ *
+ * It is very important that this gets done when page sharing is enabled.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success. FF cleared.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The ID of the calling EMT.
+ *
+ * @thread  EMT(idCpu)
+ *
+ * @remarks Must be called from within the PGM critical section.
+ */
+VMMR0_INT_DECL(int) PGMR0PhysFlushHandyPages(PGVM pGVM, VMCPUID idCpu)
+{
+    /*
+     * Validate inputs.
+     */
+    AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID); /* caller already checked this, but just to be sure. */
+    AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_NOT_OWNER);
+    PGM_LOCK_ASSERT_OWNER_EX(pGVM, &pGVM->aCpus[idCpu]);
+
+    /*
+     * Try allocate a full set of handy pages.
+     */
+    uint32_t iFirst = pGVM->pgm.s.cHandyPages;
+    AssertReturn(iFirst <= RT_ELEMENTS(pGVM->pgm.s.aHandyPages), VERR_PGM_HANDY_PAGE_IPE);
+    uint32_t cPages = RT_ELEMENTS(pGVM->pgm.s.aHandyPages) - iFirst;
+    if (!cPages)
+        return VINF_SUCCESS;
+    int rc = GMMR0AllocateHandyPages(pGVM, idCpu, cPages, 0, &pGVM->pgm.s.aHandyPages[iFirst]);
+
+    LogFlow(("PGMR0PhysFlushHandyPages: cPages=%d rc=%Rrc\n", cPages, rc));
+    return rc;
+}
+
+
+/**
+ * Worker function for PGMR3PhysAllocateLargeHandyPage
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_EM_NO_MEMORY if we're out of memory.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The ID of the calling EMT.
+ *
+ * @thread  EMT(idCpu)
+ *
+ * @remarks Must be called from within the PGM critical section. The caller
+ *          must clear the new pages.
+ */
+VMMR0_INT_DECL(int) PGMR0PhysAllocateLargeHandyPage(PGVM pGVM, VMCPUID idCpu)
+{
+    /*
+     * Validate inputs.
+     */
+    AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID); /* caller already checked this, but just to be sure. */
+    AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_NOT_OWNER);
+    PGM_LOCK_ASSERT_OWNER_EX(pGVM, &pGVM->aCpus[idCpu]);
+    Assert(!pGVM->pgm.s.cLargeHandyPages);
+
+    /*
+     * Do the job.
+     */
+    int rc = GMMR0AllocateLargePage(pGVM, idCpu, _2M,
+                                    &pGVM->pgm.s.aLargeHandyPage[0].idPage,
+                                    &pGVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys);
+    if (RT_SUCCESS(rc))
+        pGVM->pgm.s.cLargeHandyPages = 1;
+
+    return rc;
+}
+
+
+/**
+ * Locate a MMIO2 range.
+ *
+ * @returns Pointer to the MMIO2 range.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   pDevIns     The device instance owning the region.
+ * @param   hMmio2      Handle to look up.
+ */
+DECLINLINE(PPGMREGMMIO2RANGE) pgmR0PhysMMIOExFind(PGVM pGVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
+{
+    /*
+     * We use the lookup table here as list walking is tedious in ring-0 when using
+     * ring-3 pointers and this probably will require some kind of refactoring anyway.
+     */
+    if (hMmio2 <= RT_ELEMENTS(pGVM->pgm.s.apMmio2RangesR0) && hMmio2 != 0)
+    {
+        PPGMREGMMIO2RANGE pCur = pGVM->pgm.s.apMmio2RangesR0[hMmio2 - 1];
+        if (pCur && pCur->pDevInsR3 == pDevIns->pDevInsForR3)
+        {
+            Assert(pCur->idMmio2 == hMmio2);
+            AssertReturn(pCur->fFlags & PGMREGMMIO2RANGE_F_MMIO2, NULL);
+            return pCur;
+        }
+        Assert(!pCur);
+    }
+    return NULL;
+}
+
+
+/**
+ * Worker for PDMDEVHLPR0::pfnMmio2SetUpContext.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   pDevIns     The device instance.
+ * @param   hMmio2      The MMIO2 region to map into ring-0 address space.
+ * @param   offSub      The offset into the region.
+ * @param   cbSub       The size of the mapping, zero meaning all the rest.
+ * @param   ppvMapping  Where to return the ring-0 mapping address.
+ */
+VMMR0_INT_DECL(int) PGMR0PhysMMIO2MapKernel(PGVM pGVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2,
+                                            size_t offSub, size_t cbSub, void **ppvMapping)
+{
+    AssertReturn(!(offSub & PAGE_OFFSET_MASK), VERR_UNSUPPORTED_ALIGNMENT);
+    AssertReturn(!(cbSub  & PAGE_OFFSET_MASK), VERR_UNSUPPORTED_ALIGNMENT);
+
+    /*
+     * Translate hRegion into a range pointer.
+     */
+    PPGMREGMMIO2RANGE pFirstRegMmio = pgmR0PhysMMIOExFind(pGVM, pDevIns, hMmio2);
+    AssertReturn(pFirstRegMmio, VERR_NOT_FOUND);
+#if defined(VBOX_WITH_RAM_IN_KERNEL) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+    uint8_t * const pvR0  = (uint8_t *)pFirstRegMmio->pvR0;
+#else
+    RTR3PTR const  pvR3   = pFirstRegMmio->pvR3;
+#endif
+    RTGCPHYS const cbReal = pFirstRegMmio->cbReal;
+    pFirstRegMmio = NULL;
+    ASMCompilerBarrier();
+
+    AssertReturn(offSub < cbReal, VERR_OUT_OF_RANGE);
+    if (cbSub == 0)
+        cbSub = cbReal - offSub;
+    else
+        AssertReturn(cbSub < cbReal && cbSub + offSub <= cbReal, VERR_OUT_OF_RANGE);
+
+    /*
+     * Do the mapping.
+     */
+#if defined(VBOX_WITH_RAM_IN_KERNEL) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+    AssertPtr(pvR0);
+    *ppvMapping = pvR0 + offSub;
+    return VINF_SUCCESS;
+#else
+    return SUPR0PageMapKernel(pGVM->pSession, pvR3, (uint32_t)offSub, (uint32_t)cbSub, 0 /*fFlags*/, ppvMapping);
+#endif
+}
+
+
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+/* Interface sketch.  The interface belongs to a global PCI pass-through
+   manager.  It shall use the global VM handle, not the user VM handle to
+   store the per-VM info (domain) since that is all ring-0 stuff, thus
+   passing pGVM here.  I've tentitively prefixed the functions 'GPciRawR0',
+   we can discuss the PciRaw code re-organtization when I'm back from
+   vacation.
+
+   I've implemented the initial IOMMU set up below.  For things to work
+   reliably, we will probably need add a whole bunch of checks and
+   GPciRawR0GuestPageUpdate call to the PGM code.  For the present,
+   assuming nested paging (enforced) and prealloc (enforced), no
+   ballooning (check missing), page sharing (check missing) or live
+   migration (check missing), it might work fine.  At least if some
+   VM power-off hook is present and can tear down the IOMMU page tables. */
+
+/**
+ * Tells the global PCI pass-through manager that we are about to set up the
+ * guest page to host page mappings for the specfied VM.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pGVM                The ring-0 VM structure.
+ */
+VMMR0_INT_DECL(int) GPciRawR0GuestPageBeginAssignments(PGVM pGVM)
+{
+    NOREF(pGVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Assigns a host page mapping for a guest page.
+ *
+ * This is only used when setting up the mappings, i.e. between
+ * GPciRawR0GuestPageBeginAssignments and GPciRawR0GuestPageEndAssignments.
+ *
+ * @returns VBox status code.
+ * @param   pGVM                The ring-0 VM structure.
+ * @param   GCPhys              The address of the guest page (page aligned).
+ * @param   HCPhys              The address of the host page (page aligned).
+ */
+VMMR0_INT_DECL(int) GPciRawR0GuestPageAssign(PGVM pGVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys)
+{
+    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_3);
+    AssertReturn(!(HCPhys & PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_3);
+
+    if (pGVM->rawpci.s.pfnContigMemInfo)
+        /** @todo what do we do on failure? */
+        pGVM->rawpci.s.pfnContigMemInfo(&pGVM->rawpci.s, HCPhys, GCPhys, PAGE_SIZE, PCIRAW_MEMINFO_MAP);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Indicates that the specified guest page doesn't exists but doesn't have host
+ * page mapping we trust PCI pass-through with.
+ *
+ * This is only used when setting up the mappings, i.e. between
+ * GPciRawR0GuestPageBeginAssignments and GPciRawR0GuestPageEndAssignments.
+ *
+ * @returns VBox status code.
+ * @param   pGVM                The ring-0 VM structure.
+ * @param   GCPhys              The address of the guest page (page aligned).
+ * @param   HCPhys              The address of the host page (page aligned).
+ */
+VMMR0_INT_DECL(int) GPciRawR0GuestPageUnassign(PGVM pGVM, RTGCPHYS GCPhys)
+{
+    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_3);
+
+    if (pGVM->rawpci.s.pfnContigMemInfo)
+        /** @todo what do we do on failure? */
+        pGVM->rawpci.s.pfnContigMemInfo(&pGVM->rawpci.s, 0, GCPhys, PAGE_SIZE, PCIRAW_MEMINFO_UNMAP);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Tells the global PCI pass-through manager that we have completed setting up
+ * the guest page to host page mappings for the specfied VM.
+ *
+ * This complements GPciRawR0GuestPageBeginAssignments and will be called even
+ * if some page assignment failed.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pGVM                The ring-0 VM structure.
+ */
+VMMR0_INT_DECL(int) GPciRawR0GuestPageEndAssignments(PGVM pGVM)
+{
+    NOREF(pGVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Tells the global PCI pass-through manager that a guest page mapping has
+ * changed after the initial setup.
+ *
+ * @returns VBox status code.
+ * @param   pGVM                The ring-0 VM structure.
+ * @param   GCPhys              The address of the guest page (page aligned).
+ * @param   HCPhys              The new host page address or NIL_RTHCPHYS if
+ *                              now unassigned.
+ */
+VMMR0_INT_DECL(int) GPciRawR0GuestPageUpdate(PGVM pGVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys)
+{
+    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_4);
+    AssertReturn(!(HCPhys & PAGE_OFFSET_MASK) || HCPhys == NIL_RTHCPHYS, VERR_INTERNAL_ERROR_4);
+    NOREF(pGVM);
+    return VINF_SUCCESS;
+}
+
+#endif /* VBOX_WITH_PCI_PASSTHROUGH */
+
+
+/**
+ * Sets up the IOMMU when raw PCI device is enabled.
+ *
+ * @note    This is a hack that will probably be remodelled and refined later!
+ *
+ * @returns VBox status code.
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ */
+VMMR0_INT_DECL(int) PGMR0PhysSetupIoMmu(PGVM pGVM)
+{
+    int rc = GVMMR0ValidateGVM(pGVM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+    if (pGVM->pgm.s.fPciPassthrough)
+    {
+        /*
+         * The Simplistic Approach - Enumerate all the pages and call tell the
+         * IOMMU about each of them.
+         */
+        pgmLock(pGVM);
+        rc = GPciRawR0GuestPageBeginAssignments(pGVM);
+        if (RT_SUCCESS(rc))
+        {
+            for (PPGMRAMRANGE pRam = pGVM->pgm.s.pRamRangesXR0; RT_SUCCESS(rc) && pRam; pRam = pRam->pNextR0)
+            {
+                PPGMPAGE    pPage  = &pRam->aPages[0];
+                RTGCPHYS    GCPhys = pRam->GCPhys;
+                uint32_t    cLeft  = pRam->cb >> PAGE_SHIFT;
+                while (cLeft-- > 0)
+                {
+                    /* Only expose pages that are 100% safe for now. */
+                    if (   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
+                        && PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED
+                        && !PGM_PAGE_HAS_ANY_HANDLERS(pPage))
+                        rc = GPciRawR0GuestPageAssign(pGVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage));
+                    else
+                        rc = GPciRawR0GuestPageUnassign(pGVM, GCPhys);
+
+                    /* next */
+                    pPage++;
+                    GCPhys += PAGE_SIZE;
+                }
+            }
+
+            int rc2 = GPciRawR0GuestPageEndAssignments(pGVM);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+        }
+        pgmUnlock(pGVM);
+    }
+    else
+#endif
+        rc = VERR_NOT_SUPPORTED;
+    return rc;
+}
+
+
+/**
+ * \#PF Handler for nested paging.
+ *
+ * @returns VBox status code (appropriate for trap handling and GC return).
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   pGVCpu              The global (ring-0) CPU structure of the calling
+ *                              EMT.
+ * @param   enmShwPagingMode    Paging mode for the nested page tables.
+ * @param   uErr                The trap error code.
+ * @param   pRegFrame           Trap register frame.
+ * @param   GCPhysFault         The fault address.
+ */
+VMMR0DECL(int) PGMR0Trap0eHandlerNestedPaging(PGVM pGVM, PGVMCPU pGVCpu, PGMMODE enmShwPagingMode, RTGCUINT uErr,
+                                              PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault)
+{
+    int rc;
+
+    LogFlow(("PGMTrap0eHandler: uErr=%RGx GCPhysFault=%RGp eip=%RGv\n", uErr, GCPhysFault, (RTGCPTR)pRegFrame->rip));
+    STAM_PROFILE_START(&pGVCpu->pgm.s.StatRZTrap0e, a);
+    STAM_STATS({ pGVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = NULL; } );
+
+    /* AMD uses the host's paging mode; Intel has a single mode (EPT). */
+    AssertMsg(   enmShwPagingMode == PGMMODE_32_BIT || enmShwPagingMode == PGMMODE_PAE      || enmShwPagingMode == PGMMODE_PAE_NX
+              || enmShwPagingMode == PGMMODE_AMD64  || enmShwPagingMode == PGMMODE_AMD64_NX || enmShwPagingMode == PGMMODE_EPT,
+              ("enmShwPagingMode=%d\n", enmShwPagingMode));
+
+    /* Reserved shouldn't end up here. */
+    Assert(!(uErr & X86_TRAP_PF_RSVD));
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Error code stats.
+     */
+    if (uErr & X86_TRAP_PF_US)
+    {
+        if (!(uErr & X86_TRAP_PF_P))
+        {
+            if (uErr & X86_TRAP_PF_RW)
+                STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNotPresentWrite);
+            else
+                STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNotPresentRead);
+        }
+        else if (uErr & X86_TRAP_PF_RW)
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSWrite);
+        else if (uErr & X86_TRAP_PF_RSVD)
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSReserved);
+        else if (uErr & X86_TRAP_PF_ID)
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNXE);
+        else
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSRead);
+    }
+    else
+    {   /* Supervisor */
+        if (!(uErr & X86_TRAP_PF_P))
+        {
+            if (uErr & X86_TRAP_PF_RW)
+                STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVNotPresentWrite);
+            else
+                STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVNotPresentRead);
+        }
+        else if (uErr & X86_TRAP_PF_RW)
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVWrite);
+        else if (uErr & X86_TRAP_PF_ID)
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSNXE);
+        else if (uErr & X86_TRAP_PF_RSVD)
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVReserved);
+    }
+#endif
+
+    /*
+     * Call the worker.
+     *
+     * Note! We pretend the guest is in protected mode without paging, so we
+     *       can use existing code to build the nested page tables.
+     */
+/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
+    bool fLockTaken = false;
+    switch (enmShwPagingMode)
+    {
+        case PGMMODE_32_BIT:
+            rc = PGM_BTH_NAME_32BIT_PROT(Trap0eHandler)(pGVCpu, uErr, pRegFrame, GCPhysFault, &fLockTaken);
+            break;
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX:
+            rc = PGM_BTH_NAME_PAE_PROT(Trap0eHandler)(pGVCpu, uErr, pRegFrame, GCPhysFault, &fLockTaken);
+            break;
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:
+            rc = PGM_BTH_NAME_AMD64_PROT(Trap0eHandler)(pGVCpu, uErr, pRegFrame, GCPhysFault, &fLockTaken);
+            break;
+        case PGMMODE_EPT:
+            rc = PGM_BTH_NAME_EPT_PROT(Trap0eHandler)(pGVCpu, uErr, pRegFrame, GCPhysFault, &fLockTaken);
+            break;
+        default:
+            AssertFailed();
+            rc = VERR_INVALID_PARAMETER;
+            break;
+    }
+    if (fLockTaken)
+    {
+        PGM_LOCK_ASSERT_OWNER(pGVM);
+        pgmUnlock(pGVM);
+    }
+
+    if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
+        rc = VINF_SUCCESS;
+    /*
+     * Handle the case where we cannot interpret the instruction because we cannot get the guest physical address
+     * via its page tables, see @bugref{6043}.
+     */
+    else if (   rc == VERR_PAGE_NOT_PRESENT                 /* SMP only ; disassembly might fail. */
+             || rc == VERR_PAGE_TABLE_NOT_PRESENT           /* seen with UNI & SMP */
+             || rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT   /* seen with SMP */
+             || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)     /* precaution */
+    {
+        Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGp error code %x (rip=%RGv)\n", rc, GCPhysFault, uErr, pRegFrame->rip));
+        /* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about
+           single VCPU VMs though. */
+        rc = VINF_SUCCESS;
+    }
+
+    STAM_STATS({ if (!pGVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution))
+                    pGVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Misc; });
+    STAM_PROFILE_STOP_EX(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0e, pGVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution), a);
+    return rc;
+}
+
+
+/**
+ * \#PF Handler for deliberate nested paging misconfiguration (/reserved bit)
+ * employed for MMIO pages.
+ *
+ * @returns VBox status code (appropriate for trap handling and GC return).
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   pGVCpu              The global (ring-0) CPU structure of the calling
+ *                              EMT.
+ * @param   enmShwPagingMode    Paging mode for the nested page tables.
+ * @param   pRegFrame           Trap register frame.
+ * @param   GCPhysFault         The fault address.
+ * @param   uErr                The error code, UINT32_MAX if not available
+ *                              (VT-x).
+ */
+VMMR0DECL(VBOXSTRICTRC) PGMR0Trap0eHandlerNPMisconfig(PGVM pGVM, PGVMCPU pGVCpu, PGMMODE enmShwPagingMode,
+                                                      PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, uint32_t uErr)
+{
+#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
+    STAM_PROFILE_START(&pGVCpu->CTX_SUFF(pStats)->StatR0NpMiscfg, a);
+    VBOXSTRICTRC rc;
+
+    /*
+     * Try lookup the all access physical handler for the address.
+     */
+    pgmLock(pGVM);
+    PPGMPHYSHANDLER         pHandler     = pgmHandlerPhysicalLookup(pGVM, GCPhysFault);
+    PPGMPHYSHANDLERTYPEINT  pHandlerType = RT_LIKELY(pHandler) ? PGMPHYSHANDLER_GET_TYPE(pGVM, pHandler) : NULL;
+    if (RT_LIKELY(pHandler && pHandlerType->enmKind != PGMPHYSHANDLERKIND_WRITE))
+    {
+        /*
+         * If the handle has aliases page or pages that have been temporarily
+         * disabled, we'll have to take a detour to make sure we resync them
+         * to avoid lots of unnecessary exits.
+         */
+        PPGMPAGE pPage;
+        if (   (   pHandler->cAliasedPages
+                || pHandler->cTmpOffPages)
+            && (   (pPage = pgmPhysGetPage(pGVM, GCPhysFault)) == NULL
+                || PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
+           )
+        {
+            Log(("PGMR0Trap0eHandlerNPMisconfig: Resyncing aliases / tmp-off page at %RGp (uErr=%#x) %R[pgmpage]\n", GCPhysFault, uErr, pPage));
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatR0NpMiscfgSyncPage);
+            rc = pgmShwSyncNestedPageLocked(pGVCpu, GCPhysFault, 1 /*cPages*/, enmShwPagingMode);
+            pgmUnlock(pGVM);
+        }
+        else
+        {
+            if (pHandlerType->CTX_SUFF(pfnPfHandler))
+            {
+                void *pvUser = pHandler->CTX_SUFF(pvUser);
+                STAM_PROFILE_START(&pHandler->Stat, h);
+                pgmUnlock(pGVM);
+
+                Log6(("PGMR0Trap0eHandlerNPMisconfig: calling %p(,%#x,,%RGp,%p)\n", pHandlerType->CTX_SUFF(pfnPfHandler), uErr, GCPhysFault, pvUser));
+                rc = pHandlerType->CTX_SUFF(pfnPfHandler)(pGVM, pGVCpu, uErr == UINT32_MAX ? RTGCPTR_MAX : uErr, pRegFrame,
+                                                          GCPhysFault, GCPhysFault, pvUser);
+
+#ifdef VBOX_WITH_STATISTICS
+                pgmLock(pGVM);
+                pHandler = pgmHandlerPhysicalLookup(pGVM, GCPhysFault);
+                if (pHandler)
+                    STAM_PROFILE_STOP(&pHandler->Stat, h);
+                pgmUnlock(pGVM);
+#endif
+            }
+            else
+            {
+                pgmUnlock(pGVM);
+                Log(("PGMR0Trap0eHandlerNPMisconfig: %RGp (uErr=%#x) -> R3\n", GCPhysFault, uErr));
+                rc = VINF_EM_RAW_EMULATE_INSTR;
+            }
+        }
+    }
+    else
+    {
+        /*
+         * Must be out of sync, so do a SyncPage and restart the instruction.
+         *
+         * ASSUMES that ALL handlers are page aligned and covers whole pages
+         * (assumption asserted in PGMHandlerPhysicalRegisterEx).
+         */
+        Log(("PGMR0Trap0eHandlerNPMisconfig: Out of sync page at %RGp (uErr=%#x)\n", GCPhysFault, uErr));
+        STAM_COUNTER_INC(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatR0NpMiscfgSyncPage);
+        rc = pgmShwSyncNestedPageLocked(pGVCpu, GCPhysFault, 1 /*cPages*/, enmShwPagingMode);
+        pgmUnlock(pGVM);
+    }
+
+    STAM_PROFILE_STOP(&pGVCpu->pgm.s.CTX_SUFF(pStats)->StatR0NpMiscfg, a);
+    return rc;
+
+#else
+    AssertLogRelFailed();
+    return VERR_PGM_NOT_USED_IN_MODE;
+#endif
+}
+
diff --git a/src/VBox/VMM/VMMR0/PGMR0Bth.h b/src/VBox/VMM/VMMR0/PGMR0Bth.h
new file mode 100644
index 00000000..4eceac21
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/PGMR0Bth.h
@@ -0,0 +1,25 @@
+/* $Id: PGMR0Bth.h $ */
+/** @file
+ * VBox - Page Manager / Monitor, Shadow+Guest Paging Template.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+RT_C_DECLS_BEGIN
+PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
+RT_C_DECLS_END
+
diff --git a/src/VBox/VMM/VMMR0/PGMR0Pool.cpp b/src/VBox/VMM/VMMR0/PGMR0Pool.cpp
new file mode 100644
index 00000000..bc05b456
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/PGMR0Pool.cpp
@@ -0,0 +1,153 @@
+/* $Id: PGMR0Pool.cpp $ */
+/** @file
+ * PGM Shadow Page Pool, ring-0 specific bits.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_POOL
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/hm.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include "PGMInline.h"
+
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/mem.h>
+#include <iprt/memobj.h>
+
+
+
+/**
+ * Grows the shadow page pool.
+ *
+ * I.e. adds more pages to it, assuming that hasn't reached cMaxPages yet.
+ *
+ * @returns VBox status code.
+ * @param   pGVM    The ring-0 VM structure.
+ */
+VMMR0_INT_DECL(int) PGMR0PoolGrow(PGVM pGVM)
+{
+    PPGMPOOL pPool = pGVM->pgm.s.pPoolR0;
+    AssertReturn(pPool->cCurPages < pPool->cMaxPages, VERR_PGM_POOL_MAXED_OUT_ALREADY);
+    AssertReturn(pPool->pVMR3 == pGVM->pVMR3, VERR_PGM_POOL_IPE);
+    AssertReturn(pPool->pVMR0 == pGVM, VERR_PGM_POOL_IPE);
+
+    /* With 32-bit guests and no EPT, the CR3 limits the root pages to low
+       (below 4 GB) memory. */
+    /** @todo change the pool to handle ROOT page allocations specially when
+     *        required. */
+    bool const fCanUseHighMemory = HMIsNestedPagingActive(pGVM);
+
+    STAM_REL_PROFILE_START(&pPool->StatGrow, a);
+    int rc = RTCritSectEnter(&pGVM->pgmr0.s.PoolGrowCritSect);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Figure out how many pages should allocate.
+     */
+    uint32_t const cMaxPages = RT_MIN(pPool->cMaxPages, PGMPOOL_IDX_LAST);
+    uint32_t const cCurPages = RT_MIN(pPool->cCurPages, cMaxPages);
+    if (cCurPages < cMaxPages)
+    {
+        uint32_t cNewPages = cMaxPages - cCurPages;
+        if (cNewPages > PGMPOOL_CFG_MAX_GROW)
+            cNewPages = PGMPOOL_CFG_MAX_GROW;
+        LogFlow(("PGMR3PoolGrow: Growing the pool by %u (%#x) pages to %u (%#x) pages. fCanUseHighMemory=%RTbool\n",
+                 cNewPages, cNewPages, cCurPages + cNewPages, cCurPages + cNewPages, fCanUseHighMemory));
+
+        /* Check that the handles in the arrays entry are both NIL. */
+        uintptr_t const idxMemHandle = cCurPages / (PGMPOOL_CFG_MAX_GROW);
+        AssertCompile(   (PGMPOOL_IDX_LAST + (PGMPOOL_CFG_MAX_GROW - 1)) / PGMPOOL_CFG_MAX_GROW
+                      <= RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs));
+        AssertCompile(RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs) == RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMapObjs));
+        AssertLogRelMsgReturnStmt(   pGVM->pgmr0.s.ahPoolMemObjs[idxMemHandle] == NIL_RTR0MEMOBJ
+                                  && pGVM->pgmr0.s.ahPoolMapObjs[idxMemHandle] == NIL_RTR0MEMOBJ,
+                                  ("idxMemHandle=%#x\n", idxMemHandle), RTCritSectLeave(&pGVM->pgmr0.s.PoolGrowCritSect),
+                                  VERR_PGM_POOL_IPE);
+
+        /*
+         * Allocate the new pages and map them into ring-3.
+         */
+        RTR0MEMOBJ hMemObj = NIL_RTR0MEMOBJ;
+        if (fCanUseHighMemory)
+            rc = RTR0MemObjAllocPage(&hMemObj, cNewPages * PAGE_SIZE, false /*fExecutable*/);
+        else
+            rc = RTR0MemObjAllocLow(&hMemObj, cNewPages * PAGE_SIZE, false /*fExecutable*/);
+        if (RT_SUCCESS(rc))
+        {
+            RTR0MEMOBJ hMapObj = NIL_RTR0MEMOBJ;
+            rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
+            if (RT_SUCCESS(rc))
+            {
+                pGVM->pgmr0.s.ahPoolMemObjs[idxMemHandle] = hMemObj;
+                pGVM->pgmr0.s.ahPoolMapObjs[idxMemHandle] = hMapObj;
+
+                uint8_t *pbRing0 = (uint8_t *)RTR0MemObjAddress(hMemObj);
+                RTR3PTR  pbRing3 = RTR0MemObjAddressR3(hMapObj);
+                AssertPtr(pbRing0);
+                Assert(((uintptr_t)pbRing0 & PAGE_OFFSET_MASK) == 0);
+                Assert(pbRing3 != NIL_RTR3PTR);
+                Assert((pbRing3 & PAGE_OFFSET_MASK) == 0);
+
+                /*
+                 * Initialize the new pages.
+                 */
+                for (unsigned iNewPage = 0; iNewPage < cNewPages; iNewPage++)
+                {
+                    PPGMPOOLPAGE pPage = &pPool->aPages[cCurPages + iNewPage];
+                    pPage->pvPageR0         = &pbRing0[iNewPage * PAGE_SIZE];
+                    pPage->pvPageR3         = pbRing3 + iNewPage * PAGE_SIZE;
+                    pPage->Core.Key         = RTR0MemObjGetPagePhysAddr(hMemObj, iNewPage);
+                    AssertFatal(pPage->Core.Key < _4G || fCanUseHighMemory);
+                    pPage->GCPhys           = NIL_RTGCPHYS;
+                    pPage->enmKind          = PGMPOOLKIND_FREE;
+                    pPage->idx              = pPage - &pPool->aPages[0];
+                    LogFlow(("PGMR3PoolGrow: insert page #%#x - %RHp\n", pPage->idx, pPage->Core.Key));
+                    pPage->iNext            = pPool->iFreeHead;
+                    pPage->iUserHead        = NIL_PGMPOOL_USER_INDEX;
+                    pPage->iModifiedNext    = NIL_PGMPOOL_IDX;
+                    pPage->iModifiedPrev    = NIL_PGMPOOL_IDX;
+                    pPage->iMonitoredNext   = NIL_PGMPOOL_IDX;
+                    pPage->iMonitoredPrev   = NIL_PGMPOOL_IDX;
+                    pPage->iAgeNext         = NIL_PGMPOOL_IDX;
+                    pPage->iAgePrev         = NIL_PGMPOOL_IDX;
+                    /* commit it */
+                    bool fRc = RTAvloHCPhysInsert(&pPool->HCPhysTree, &pPage->Core); Assert(fRc); NOREF(fRc);
+                    pPool->iFreeHead = cCurPages + iNewPage;
+                    pPool->cCurPages = cCurPages + iNewPage + 1;
+                }
+
+                STAM_REL_PROFILE_STOP(&pPool->StatGrow, a);
+                RTCritSectLeave(&pGVM->pgmr0.s.PoolGrowCritSect);
+                return VINF_SUCCESS;
+            }
+
+            RTR0MemObjFree(hMemObj, true /*fFreeMappings*/);
+        }
+        if (cCurPages > 64)
+            LogRelMax(5, ("PGMR0PoolGrow: rc=%Rrc cNewPages=%#x cCurPages=%#x cMaxPages=%#x fCanUseHighMemory=%d\n",
+                          rc, cNewPages, cCurPages, cMaxPages, fCanUseHighMemory));
+        else
+            LogRel(("PGMR0PoolGrow: rc=%Rrc cNewPages=%#x cCurPages=%#x cMaxPages=%#x fCanUseHighMemory=%d\n",
+                    rc, cNewPages, cCurPages, cMaxPages, fCanUseHighMemory));
+    }
+    RTCritSectLeave(&pGVM->pgmr0.s.PoolGrowCritSect);
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR0/PGMR0SharedPage.cpp b/src/VBox/VMM/VMMR0/PGMR0SharedPage.cpp
new file mode 100644
index 00000000..909bf143
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/PGMR0SharedPage.cpp
@@ -0,0 +1,171 @@
+/* $Id: PGMR0SharedPage.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor, Page Sharing, Ring-0.
+ */
+
+/*
+ * Copyright (C) 2010-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_SHARED
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/gmm.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/gvm.h>
+#include "PGMInline.h"
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+#include <iprt/mem.h>
+
+
+#ifdef VBOX_WITH_PAGE_SHARING
+/**
+ * Check a registered module for shared page changes.
+ *
+ * The PGM lock shall be taken prior to calling this method.
+ *
+ * @returns The following VBox status codes.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pGVM                Pointer to the GVM instance data.
+ * @param   idCpu               The ID of the calling virtual CPU.
+ * @param   pModule             Global module description.
+ * @param   paRegionsGCPtrs     Array parallel to pModules->aRegions with the
+ *                              addresses of the regions in the calling
+ *                              process.
+ */
+VMMR0DECL(int) PGMR0SharedModuleCheck(PVMCC pVM, PGVM pGVM, VMCPUID idCpu, PGMMSHAREDMODULE pModule, PCRTGCPTR64 paRegionsGCPtrs)
+{
+    PVMCPUCC            pVCpu         = &pGVM->aCpus[idCpu];
+    int                 rc            = VINF_SUCCESS;
+    bool                fFlushTLBs    = false;
+    bool                fFlushRemTLBs = false;
+    GMMSHAREDPAGEDESC   PageDesc;
+
+    Log(("PGMR0SharedModuleCheck: check %s %s base=%RGv size=%x\n", pModule->szName, pModule->szVersion, pModule->Core.Key, pModule->cbModule));
+
+    PGM_LOCK_ASSERT_OWNER(pVM);     /* This cannot fail as we grab the lock in pgmR3SharedModuleRegRendezvous before calling into ring-0. */
+
+    /*
+     * Check every region of the shared module.
+     */
+    for (uint32_t idxRegion = 0; idxRegion < pModule->cRegions; idxRegion++)
+    {
+        RTGCPTR  GCPtrPage  = paRegionsGCPtrs[idxRegion] & ~(RTGCPTR)PAGE_OFFSET_MASK;
+        uint32_t cbLeft     = pModule->aRegions[idxRegion].cb; Assert(!(cbLeft & PAGE_OFFSET_MASK));
+        uint32_t idxPage    = 0;
+
+        while (cbLeft)
+        {
+            /** @todo inefficient to fetch each guest page like this... */
+            RTGCPHYS GCPhys;
+            uint64_t fFlags;
+            rc = PGMGstGetPage(pVCpu, GCPtrPage, &fFlags, &GCPhys);
+            if (    rc == VINF_SUCCESS
+                &&  !(fFlags & X86_PTE_RW)) /* important as we make assumptions about this below! */
+            {
+                PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+                Assert(!pPage || !PGM_PAGE_IS_BALLOONED(pPage));
+                if (    pPage
+                    &&  PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED
+                    &&  PGM_PAGE_GET_READ_LOCKS(pPage) == 0
+                    &&  PGM_PAGE_GET_WRITE_LOCKS(pPage) == 0 )
+                {
+                    PageDesc.idPage = PGM_PAGE_GET_PAGEID(pPage);
+                    PageDesc.HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+                    PageDesc.GCPhys = GCPhys;
+
+                    rc = GMMR0SharedModuleCheckPage(pGVM, pModule, idxRegion, idxPage, &PageDesc);
+                    if (RT_FAILURE(rc))
+                        break;
+
+                    /*
+                     * Any change for this page?
+                     */
+                    if (PageDesc.idPage != NIL_GMM_PAGEID)
+                    {
+                        Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+
+                        Log(("PGMR0SharedModuleCheck: shared page gst virt=%RGv phys=%RGp host %RHp->%RHp\n",
+                             GCPtrPage, PageDesc.GCPhys, PGM_PAGE_GET_HCPHYS(pPage), PageDesc.HCPhys));
+
+                        /* Page was either replaced by an existing shared
+                           version of it or converted into a read-only shared
+                           page, so, clear all references. */
+                        bool fFlush = false;
+                        rc = pgmPoolTrackUpdateGCPhys(pVM, PageDesc.GCPhys, pPage, true /* clear the entries */, &fFlush);
+                        Assert(   rc == VINF_SUCCESS
+                               || (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
+                                   && (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)));
+                        if (rc == VINF_SUCCESS)
+                            fFlushTLBs |= fFlush;
+                        fFlushRemTLBs = true;
+
+                        if (PageDesc.HCPhys != PGM_PAGE_GET_HCPHYS(pPage))
+                        {
+                            /* Update the physical address and page id now. */
+                            PGM_PAGE_SET_HCPHYS(pVM, pPage, PageDesc.HCPhys);
+                            PGM_PAGE_SET_PAGEID(pVM, pPage, PageDesc.idPage);
+
+                            /* Invalidate page map TLB entry for this page too. */
+                            pgmPhysInvalidatePageMapTLBEntry(pVM, PageDesc.GCPhys);
+                            pVM->pgm.s.cReusedSharedPages++;
+                        }
+                        /* else: nothing changed (== this page is now a shared
+                           page), so no need to flush anything. */
+
+                        pVM->pgm.s.cSharedPages++;
+                        pVM->pgm.s.cPrivatePages--;
+                        PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_SHARED);
+
+# ifdef VBOX_STRICT /* check sum hack */
+                        pPage->s.u2Unused0 = PageDesc.u32StrictChecksum        & 3;
+                        //pPage->s.u2Unused1 = (PageDesc.u32StrictChecksum >> 8) & 3;
+# endif
+                    }
+                }
+            }
+            else
+            {
+                Assert(    rc == VINF_SUCCESS
+                       ||  rc == VERR_PAGE_NOT_PRESENT
+                       ||  rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT
+                       ||  rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
+                       ||  rc == VERR_PAGE_TABLE_NOT_PRESENT);
+                rc = VINF_SUCCESS;  /* ignore error */
+            }
+
+            idxPage++;
+            GCPtrPage += PAGE_SIZE;
+            cbLeft    -= PAGE_SIZE;
+        }
+    }
+
+    /*
+     * Do TLB flushing if necessary.
+     */
+    if (fFlushTLBs)
+        PGM_INVL_ALL_VCPU_TLBS(pVM);
+
+    if (fFlushRemTLBs)
+        for (VMCPUID idCurCpu = 0; idCurCpu < pGVM->cCpus; idCurCpu++)
+            CPUMSetChangedFlags(&pGVM->aCpus[idCurCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
+
+    return rc;
+}
+#endif /* VBOX_WITH_PAGE_SHARING */
+
diff --git a/src/VBox/VMM/VMMR0/VMMR0.cpp b/src/VBox/VMM/VMMR0/VMMR0.cpp
new file mode 100644
index 00000000..b666391e
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/VMMR0.cpp
@@ -0,0 +1,2753 @@
+/* $Id: VMMR0.cpp $ */
+/** @file
+ * VMM - Host Context Ring 0.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <VBox/vmm/vmm.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pgm.h>
+#ifdef VBOX_WITH_NEM_R0
+# include <VBox/vmm/nem.h>
+#endif
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/tm.h>
+#include "VMMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/gvm.h>
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+# include <VBox/vmm/pdmpci.h>
+#endif
+#include <VBox/vmm/apic.h>
+
+#include <VBox/vmm/gvmm.h>
+#include <VBox/vmm/gmm.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/intnet.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/version.h>
+#include <VBox/log.h>
+
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/assert.h>
+#include <iprt/crc.h>
+#include <iprt/mp.h>
+#include <iprt/once.h>
+#include <iprt/stdarg.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/timer.h>
+#include <iprt/time.h>
+
+#include "dtrace/VBoxVMM.h"
+
+
+#if defined(_MSC_VER) && defined(RT_ARCH_AMD64) /** @todo check this with with VC7! */
+#  pragma intrinsic(_AddressOfReturnAddress)
+#endif
+
+#if defined(RT_OS_DARWIN) && ARCH_BITS == 32
+# error "32-bit darwin is no longer supported. Go back to 4.3 or earlier!"
+#endif
+
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** @def VMM_CHECK_SMAP_SETUP
+ * SMAP check setup. */
+/** @def VMM_CHECK_SMAP_CHECK
+ * Checks that the AC flag is set if SMAP is enabled. If AC is not set,
+ * it will be logged and @a a_BadExpr is executed. */
+/** @def VMM_CHECK_SMAP_CHECK2
+ * Checks that the AC flag is set if SMAP is enabled.  If AC is not set, it will
+ * be logged, written to the VMs assertion text buffer, and @a a_BadExpr is
+ * executed. */
+#if (defined(VBOX_STRICT) || 1) && !defined(VBOX_WITH_RAM_IN_KERNEL)
+# define VMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = SUPR0GetKernelFeatures()
+# define VMM_CHECK_SMAP_CHECK(a_BadExpr) \
+    do { \
+        if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
+        { \
+            RTCCUINTREG fEflCheck = ASMGetFlags(); \
+            if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
+            { /* likely */ } \
+            else \
+            { \
+                SUPR0Printf("%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
+                a_BadExpr; \
+            } \
+        } \
+    } while (0)
+# define VMM_CHECK_SMAP_CHECK2(a_pGVM, a_BadExpr) \
+    do { \
+        if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
+        { \
+            RTCCUINTREG fEflCheck = ASMGetFlags(); \
+            if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
+            { /* likely */ } \
+            else if (a_pGVM) \
+            { \
+                SUPR0BadContext((a_pGVM)->pSession, __FILE__, __LINE__, "EFLAGS.AC is zero!"); \
+                RTStrPrintf((a_pGVM)->vmm.s.szRing0AssertMsg1, sizeof((a_pGVM)->vmm.s.szRing0AssertMsg1), \
+                            "%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
+                a_BadExpr; \
+            } \
+            else \
+            { \
+                SUPR0Printf("%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
+                a_BadExpr; \
+            } \
+        } \
+    } while (0)
+#else
+# define VMM_CHECK_SMAP_SETUP()                         uint32_t const fKernelFeatures = 0
+# define VMM_CHECK_SMAP_CHECK(a_BadExpr)                NOREF(fKernelFeatures)
+# define VMM_CHECK_SMAP_CHECK2(a_pGVM, a_BadExpr)       NOREF(fKernelFeatures)
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) || defined(RT_OS_FREEBSD))
+extern uint64_t __udivdi3(uint64_t, uint64_t);
+extern uint64_t __umoddi3(uint64_t, uint64_t);
+#endif
+RT_C_DECLS_END
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** Drag in necessary library bits.
+ * The runtime lives here (in VMMR0.r0) and VBoxDD*R0.r0 links against us. */
+PFNRT g_VMMR0Deps[] =
+{
+    (PFNRT)RTCrc32,
+    (PFNRT)RTOnce,
+#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) || defined(RT_OS_FREEBSD))
+    (PFNRT)__udivdi3,
+    (PFNRT)__umoddi3,
+#endif
+    NULL
+};
+
+#ifdef RT_OS_SOLARIS
+/* Dependency information for the native solaris loader. */
+extern "C" { char _depends_on[] = "vboxdrv"; }
+#endif
+
+
+/**
+ * Initialize the module.
+ * This is called when we're first loaded.
+ *
+ * @returns 0 on success.
+ * @returns VBox status on failure.
+ * @param   hMod        Image handle for use in APIs.
+ */
+DECLEXPORT(int) ModuleInit(void *hMod)
+{
+    VMM_CHECK_SMAP_SETUP();
+    VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+
+#ifdef VBOX_WITH_DTRACE_R0
+    /*
+     * The first thing to do is register the static tracepoints.
+     * (Deregistration is automatic.)
+     */
+    int rc2 = SUPR0TracerRegisterModule(hMod, &g_VTGObjHeader);
+    if (RT_FAILURE(rc2))
+        return rc2;
+#endif
+    LogFlow(("ModuleInit:\n"));
+
+#ifdef VBOX_WITH_64ON32_CMOS_DEBUG
+    /*
+     * Display the CMOS debug code.
+     */
+    ASMOutU8(0x72, 0x03);
+    uint8_t bDebugCode = ASMInU8(0x73);
+    LogRel(("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode));
+    RTLogComPrintf("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode);
+#endif
+
+    /*
+     * Initialize the VMM, GVMM, GMM, HM, PGM (Darwin) and INTNET.
+     */
+    int rc = vmmInitFormatTypes();
+    if (RT_SUCCESS(rc))
+    {
+        VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+        rc = GVMMR0Init();
+        if (RT_SUCCESS(rc))
+        {
+            VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+            rc = GMMR0Init();
+            if (RT_SUCCESS(rc))
+            {
+                VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+                rc = HMR0Init();
+                if (RT_SUCCESS(rc))
+                {
+                    VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+
+                    PDMR0Init(hMod);
+                    VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+
+                    rc = PGMRegisterStringFormatTypes();
+                    if (RT_SUCCESS(rc))
+                    {
+                        VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+                        rc = PGMR0DynMapInit();
+#endif
+                        if (RT_SUCCESS(rc))
+                        {
+                            VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+                            rc = IntNetR0Init();
+                            if (RT_SUCCESS(rc))
+                            {
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+                                VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+                                rc = PciRawR0Init();
+#endif
+                                if (RT_SUCCESS(rc))
+                                {
+                                    VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+                                    rc = CPUMR0ModuleInit();
+                                    if (RT_SUCCESS(rc))
+                                    {
+#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
+                                        VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+                                        rc = vmmR0TripleFaultHackInit();
+                                        if (RT_SUCCESS(rc))
+#endif
+                                        {
+                                            VMM_CHECK_SMAP_CHECK(rc = VERR_VMM_SMAP_BUT_AC_CLEAR);
+                                            if (RT_SUCCESS(rc))
+                                            {
+                                                LogFlow(("ModuleInit: returns success\n"));
+                                                return VINF_SUCCESS;
+                                            }
+                                        }
+
+                                        /*
+                                         * Bail out.
+                                         */
+#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
+                                        vmmR0TripleFaultHackTerm();
+#endif
+                                    }
+                                    else
+                                        LogRel(("ModuleInit: CPUMR0ModuleInit -> %Rrc\n", rc));
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+                                    PciRawR0Term();
+#endif
+                                }
+                                else
+                                    LogRel(("ModuleInit: PciRawR0Init -> %Rrc\n", rc));
+                                IntNetR0Term();
+                            }
+                            else
+                                LogRel(("ModuleInit: IntNetR0Init -> %Rrc\n", rc));
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+                            PGMR0DynMapTerm();
+#endif
+                        }
+                        else
+                            LogRel(("ModuleInit: PGMR0DynMapInit -> %Rrc\n", rc));
+                        PGMDeregisterStringFormatTypes();
+                    }
+                    else
+                        LogRel(("ModuleInit: PGMRegisterStringFormatTypes -> %Rrc\n", rc));
+                    HMR0Term();
+                }
+                else
+                    LogRel(("ModuleInit: HMR0Init -> %Rrc\n", rc));
+                GMMR0Term();
+            }
+            else
+                LogRel(("ModuleInit: GMMR0Init -> %Rrc\n", rc));
+            GVMMR0Term();
+        }
+        else
+            LogRel(("ModuleInit: GVMMR0Init -> %Rrc\n", rc));
+        vmmTermFormatTypes();
+    }
+    else
+        LogRel(("ModuleInit: vmmInitFormatTypes -> %Rrc\n", rc));
+
+    LogFlow(("ModuleInit: failed %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Terminate the module.
+ * This is called when we're finally unloaded.
+ *
+ * @param   hMod        Image handle for use in APIs.
+ */
+DECLEXPORT(void) ModuleTerm(void *hMod)
+{
+    NOREF(hMod);
+    LogFlow(("ModuleTerm:\n"));
+
+    /*
+     * Terminate the CPUM module (Local APIC cleanup).
+     */
+    CPUMR0ModuleTerm();
+
+    /*
+     * Terminate the internal network service.
+     */
+    IntNetR0Term();
+
+    /*
+     * PGM (Darwin), HM and PciRaw global cleanup.
+     */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    PGMR0DynMapTerm();
+#endif
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+    PciRawR0Term();
+#endif
+    PGMDeregisterStringFormatTypes();
+    HMR0Term();
+#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
+    vmmR0TripleFaultHackTerm();
+#endif
+
+    /*
+     * Destroy the GMM and GVMM instances.
+     */
+    GMMR0Term();
+    GVMMR0Term();
+
+    vmmTermFormatTypes();
+
+    LogFlow(("ModuleTerm: returns\n"));
+}
+
+
+/**
+ * Initiates the R0 driver for a particular VM instance.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   uSvnRev     The SVN revision of the ring-3 part.
+ * @param   uBuildType  Build type indicator.
+ * @thread  EMT(0)
+ */
+static int vmmR0InitVM(PGVM pGVM, uint32_t uSvnRev, uint32_t uBuildType)
+{
+    VMM_CHECK_SMAP_SETUP();
+    VMM_CHECK_SMAP_CHECK(return VERR_VMM_SMAP_BUT_AC_CLEAR);
+
+    /*
+     * Match the SVN revisions and build type.
+     */
+    if (uSvnRev != VMMGetSvnRev())
+    {
+        LogRel(("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev()));
+        SUPR0Printf("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev());
+        return VERR_VMM_R0_VERSION_MISMATCH;
+    }
+    if (uBuildType != vmmGetBuildType())
+    {
+        LogRel(("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType()));
+        SUPR0Printf("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType());
+        return VERR_VMM_R0_VERSION_MISMATCH;
+    }
+
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, 0 /*idCpu*/);
+    if (RT_FAILURE(rc))
+        return rc;
+
+#ifdef LOG_ENABLED
+    /*
+     * Register the EMT R0 logger instance for VCPU 0.
+     */
+    PVMCPUCC pVCpu = VMCC_GET_CPU_0(pGVM);
+
+    PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
+    if (pR0Logger)
+    {
+# if 0 /* testing of the logger. */
+        LogCom(("vmmR0InitVM: before %p\n", RTLogDefaultInstance()));
+        LogCom(("vmmR0InitVM: pfnFlush=%p actual=%p\n", pR0Logger->Logger.pfnFlush, vmmR0LoggerFlush));
+        LogCom(("vmmR0InitVM: pfnLogger=%p actual=%p\n", pR0Logger->Logger.pfnLogger, vmmR0LoggerWrapper));
+        LogCom(("vmmR0InitVM: offScratch=%d fFlags=%#x fDestFlags=%#x\n", pR0Logger->Logger.offScratch, pR0Logger->Logger.fFlags, pR0Logger->Logger.fDestFlags));
+
+        RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pGVM->pSession);
+        LogCom(("vmmR0InitVM: after %p reg\n", RTLogDefaultInstance()));
+        RTLogSetDefaultInstanceThread(NULL, pGVM->pSession);
+        LogCom(("vmmR0InitVM: after %p dereg\n", RTLogDefaultInstance()));
+
+        pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
+        LogCom(("vmmR0InitVM: returned successfully from direct logger call.\n"));
+        pR0Logger->Logger.pfnFlush(&pR0Logger->Logger);
+        LogCom(("vmmR0InitVM: returned successfully from direct flush call.\n"));
+
+        RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pGVM->pSession);
+        LogCom(("vmmR0InitVM: after %p reg2\n", RTLogDefaultInstance()));
+        pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
+        LogCom(("vmmR0InitVM: returned successfully from direct logger call (2). offScratch=%d\n", pR0Logger->Logger.offScratch));
+        RTLogSetDefaultInstanceThread(NULL, pGVM->pSession);
+        LogCom(("vmmR0InitVM: after %p dereg2\n", RTLogDefaultInstance()));
+
+        RTLogLoggerEx(&pR0Logger->Logger, 0, ~0U, "hello ring-0 logger (RTLogLoggerEx)\n");
+        LogCom(("vmmR0InitVM: RTLogLoggerEx returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
+
+        RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pGVM->pSession);
+        RTLogPrintf("hello ring-0 logger (RTLogPrintf)\n");
+        LogCom(("vmmR0InitVM: RTLogPrintf returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
+# endif
+        Log(("Switching to per-thread logging instance %p (key=%p)\n", &pR0Logger->Logger, pGVM->pSession));
+        RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pGVM->pSession);
+        pR0Logger->fRegistered = true;
+    }
+#endif /* LOG_ENABLED */
+SUPR0Printf("VMMR0InitVM: eflags=%x fKernelFeatures=%#x (SUPKERNELFEATURES_SMAP=%d)\n",
+            ASMGetFlags(), fKernelFeatures, RT_BOOL(fKernelFeatures & SUPKERNELFEATURES_SMAP));
+
+    /*
+     * Check if the host supports high resolution timers or not.
+     */
+    if (   pGVM->vmm.s.fUsePeriodicPreemptionTimers
+        && !RTTimerCanDoHighResolution())
+        pGVM->vmm.s.fUsePeriodicPreemptionTimers = false;
+
+    /*
+     * Initialize the per VM data for GVMM and GMM.
+     */
+    VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+    rc = GVMMR0InitVM(pGVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Init HM, CPUM and PGM (Darwin only).
+         */
+        VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+        rc = HMR0InitVM(pGVM);
+        if (RT_SUCCESS(rc))
+            VMM_CHECK_SMAP_CHECK2(pGVM, rc = VERR_VMM_RING0_ASSERTION); /* CPUR0InitVM will otherwise panic the host */
+        if (RT_SUCCESS(rc))
+        {
+            rc = CPUMR0InitVM(pGVM);
+            if (RT_SUCCESS(rc))
+            {
+                VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+                rc = PGMR0InitVM(pGVM);
+                if (RT_SUCCESS(rc))
+                {
+                    VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+                    rc = EMR0InitVM(pGVM);
+                    if (RT_SUCCESS(rc))
+                    {
+                        VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+                        rc = PciRawR0InitVM(pGVM);
+#endif
+                        if (RT_SUCCESS(rc))
+                        {
+                            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+                            rc = GIMR0InitVM(pGVM);
+                            if (RT_SUCCESS(rc))
+                            {
+                                VMM_CHECK_SMAP_CHECK2(pGVM, rc = VERR_VMM_RING0_ASSERTION);
+                                if (RT_SUCCESS(rc))
+                                {
+                                    GVMMR0DoneInitVM(pGVM);
+
+                                    /*
+                                     * Collect a bit of info for the VM release log.
+                                     */
+                                    pGVM->vmm.s.fIsPreemptPendingApiTrusty = RTThreadPreemptIsPendingTrusty();
+                                    pGVM->vmm.s.fIsPreemptPossible         = RTThreadPreemptIsPossible();;
+
+                                    VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+                                    return rc;
+                                }
+
+                                /* bail out*/
+                                GIMR0TermVM(pGVM);
+                            }
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+                            PciRawR0TermVM(pGVM);
+#endif
+                        }
+                    }
+                }
+            }
+            HMR0TermVM(pGVM);
+        }
+    }
+
+    RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pGVM->pSession);
+    return rc;
+}
+
+
+/**
+ * Does EMT specific VM initialization.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The ring-0 VM structure.
+ * @param   idCpu       The EMT that's calling.
+ */
+static int vmmR0InitVMEmt(PGVM pGVM, VMCPUID idCpu)
+{
+    /* Paranoia (caller checked these already). */
+    AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
+    AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_INVALID_CPU_ID);
+
+#ifdef LOG_ENABLED
+    /*
+     * Registration of ring 0 loggers.
+     */
+    PVMCPUCC       pVCpu     = &pGVM->aCpus[idCpu];
+    PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
+    if (   pR0Logger
+        && !pR0Logger->fRegistered)
+    {
+        RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pGVM->pSession);
+        pR0Logger->fRegistered = true;
+    }
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Terminates the R0 bits for a particular VM instance.
+ *
+ * This is normally called by ring-3 as part of the VM termination process, but
+ * may alternatively be called during the support driver session cleanup when
+ * the VM object is destroyed (see GVMM).
+ *
+ * @returns VBox status code.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       Set to 0 if EMT(0) or NIL_VMCPUID if session cleanup
+ *                      thread.
+ * @thread  EMT(0) or session clean up thread.
+ */
+VMMR0_INT_DECL(int) VMMR0TermVM(PGVM pGVM, VMCPUID idCpu)
+{
+    /*
+     * Check EMT(0) claim if we're called from userland.
+     */
+    if (idCpu != NIL_VMCPUID)
+    {
+        AssertReturn(idCpu == 0, VERR_INVALID_CPU_ID);
+        int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+    PciRawR0TermVM(pGVM);
+#endif
+
+    /*
+     * Tell GVMM what we're up to and check that we only do this once.
+     */
+    if (GVMMR0DoingTermVM(pGVM))
+    {
+        GIMR0TermVM(pGVM);
+
+        /** @todo I wish to call PGMR0PhysFlushHandyPages(pGVM, &pGVM->aCpus[idCpu])
+         *        here to make sure we don't leak any shared pages if we crash... */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+        PGMR0DynMapTermVM(pGVM);
+#endif
+        HMR0TermVM(pGVM);
+    }
+
+    /*
+     * Deregister the logger.
+     */
+    RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pGVM->pSession);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * An interrupt or unhalt force flag is set, deal with it.
+ *
+ * @returns VINF_SUCCESS (or VINF_EM_HALT).
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   uMWait                  Result from EMMonitorWaitIsActive().
+ * @param   enmInterruptibility     Guest CPU interruptbility level.
+ */
+static int vmmR0DoHaltInterrupt(PVMCPUCC pVCpu, unsigned uMWait, CPUMINTERRUPTIBILITY enmInterruptibility)
+{
+    Assert(!TRPMHasTrap(pVCpu));
+    Assert(   enmInterruptibility > CPUMINTERRUPTIBILITY_INVALID
+           && enmInterruptibility < CPUMINTERRUPTIBILITY_END);
+
+    /*
+     * Pending interrupts w/o any SMIs or NMIs?  That the usual case.
+     */
+    if (    VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
+        && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI  | VMCPU_FF_INTERRUPT_NMI))
+    {
+        if (enmInterruptibility <= CPUMINTERRUPTIBILITY_UNRESTRAINED)
+        {
+            uint8_t u8Interrupt = 0;
+            int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
+            Log(("vmmR0DoHaltInterrupt: CPU%d u8Interrupt=%d (%#x) rc=%Rrc\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc));
+            if (RT_SUCCESS(rc))
+            {
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_UNHALT);
+
+                rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
+                AssertRCSuccess(rc);
+                STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
+                return rc;
+            }
+        }
+    }
+    /*
+     * SMI is not implemented yet, at least not here.
+     */
+    else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI))
+    {
+        Log12(("vmmR0DoHaltInterrupt: CPU%d failed #3\n", pVCpu->idCpu));
+        STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3);
+        return VINF_EM_HALT;
+    }
+    /*
+     * NMI.
+     */
+    else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
+    {
+        if (enmInterruptibility < CPUMINTERRUPTIBILITY_NMI_INHIBIT)
+        {
+            /** @todo later. */
+            Log12(("vmmR0DoHaltInterrupt: CPU%d failed #2 (uMWait=%u enmInt=%d)\n", pVCpu->idCpu, uMWait, enmInterruptibility));
+            STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3);
+            return VINF_EM_HALT;
+        }
+    }
+    /*
+     * Nested-guest virtual interrupt.
+     */
+    else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
+    {
+        if (enmInterruptibility < CPUMINTERRUPTIBILITY_VIRT_INT_DISABLED)
+        {
+            /** @todo NSTVMX: NSTSVM: Remember, we might have to check and perform VM-exits
+             *        here before injecting the virtual interrupt. See emR3ForcedActions
+             *        for details. */
+            Log12(("vmmR0DoHaltInterrupt: CPU%d failed #1 (uMWait=%u enmInt=%d)\n", pVCpu->idCpu, uMWait, enmInterruptibility));
+            STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3);
+            return VINF_EM_HALT;
+        }
+    }
+
+    if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UNHALT))
+    {
+        STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
+        Log11(("vmmR0DoHaltInterrupt: CPU%d success VINF_SUCCESS (UNHALT)\n", pVCpu->idCpu));
+        return VINF_SUCCESS;
+    }
+    if (uMWait > 1)
+    {
+        STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
+        Log11(("vmmR0DoHaltInterrupt: CPU%d success VINF_SUCCESS (uMWait=%u > 1)\n", pVCpu->idCpu, uMWait));
+        return VINF_SUCCESS;
+    }
+
+    Log12(("vmmR0DoHaltInterrupt: CPU%d failed #0 (uMWait=%u enmInt=%d)\n", pVCpu->idCpu, uMWait, enmInterruptibility));
+    STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3);
+    return VINF_EM_HALT;
+}
+
+
+/**
+ * This does one round of vmR3HaltGlobal1Halt().
+ *
+ * The rational here is that we'll reduce latency in interrupt situations if we
+ * don't go to ring-3 immediately on a VINF_EM_HALT (guest executed HLT or
+ * MWAIT), but do one round of blocking here instead and hope the interrupt is
+ * raised in the meanwhile.
+ *
+ * If we go to ring-3 we'll quit the inner HM/NEM loop in EM and end up in the
+ * outer loop, which will then call VMR3WaitHalted() and that in turn will do a
+ * ring-0 call (unless we're too close to a timer event).  When the interrupt
+ * wakes us up, we'll return from ring-0 and EM will by instinct do a
+ * rescheduling (because of raw-mode) before it resumes the HM/NEM loop and gets
+ * back to VMMR0EntryFast().
+ *
+ * @returns VINF_SUCCESS or VINF_EM_HALT.
+ * @param   pGVM        The ring-0 VM structure.
+ * @param   pGVCpu      The ring-0 virtual CPU structure.
+ *
+ * @todo r=bird: All the blocking/waiting and EMT managment should move out of
+ *       the VM module, probably to VMM.  Then this would be more weird wrt
+ *       parameters and statistics.
+ */
+static int vmmR0DoHalt(PGVM pGVM, PGVMCPU pGVCpu)
+{
+    /*
+     * Do spin stat historization.
+     */
+    if (++pGVCpu->vmm.s.cR0Halts & 0xff)
+    { /* likely */ }
+    else if (pGVCpu->vmm.s.cR0HaltsSucceeded > pGVCpu->vmm.s.cR0HaltsToRing3)
+    {
+        pGVCpu->vmm.s.cR0HaltsSucceeded = 2;
+        pGVCpu->vmm.s.cR0HaltsToRing3   = 0;
+    }
+    else
+    {
+        pGVCpu->vmm.s.cR0HaltsSucceeded = 0;
+        pGVCpu->vmm.s.cR0HaltsToRing3   = 2;
+    }
+
+    /*
+     * Flags that makes us go to ring-3.
+     */
+    uint32_t const fVmFFs  = VM_FF_TM_VIRTUAL_SYNC            | VM_FF_PDM_QUEUES              | VM_FF_PDM_DMA
+                           | VM_FF_DBGF                       | VM_FF_REQUEST                 | VM_FF_CHECK_VM_STATE
+                           | VM_FF_RESET                      | VM_FF_EMT_RENDEZVOUS          | VM_FF_PGM_NEED_HANDY_PAGES
+                           | VM_FF_PGM_NO_MEMORY              | VM_FF_DEBUG_SUSPEND;
+    uint64_t const fCpuFFs = VMCPU_FF_TIMER                   | VMCPU_FF_PDM_CRITSECT         | VMCPU_FF_IEM
+                           | VMCPU_FF_REQUEST                 | VMCPU_FF_DBGF                 | VMCPU_FF_HM_UPDATE_CR3
+                           | VMCPU_FF_HM_UPDATE_PAE_PDPES     | VMCPU_FF_PGM_SYNC_CR3         | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
+                           | VMCPU_FF_TO_R3                   | VMCPU_FF_IOM;
+
+    /*
+     * Check preconditions.
+     */
+    unsigned const             uMWait              = EMMonitorWaitIsActive(pGVCpu);
+    CPUMINTERRUPTIBILITY const enmInterruptibility = CPUMGetGuestInterruptibility(pGVCpu);
+    if (   pGVCpu->vmm.s.fMayHaltInRing0
+        && !TRPMHasTrap(pGVCpu)
+        && (   enmInterruptibility == CPUMINTERRUPTIBILITY_UNRESTRAINED
+            || uMWait > 1))
+    {
+        if (   !VM_FF_IS_ANY_SET(pGVM, fVmFFs)
+            && !VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
+        {
+            /*
+             * Interrupts pending already?
+             */
+            if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
+                APICUpdatePendingInterrupts(pGVCpu);
+
+            /*
+             * Flags that wake up from the halted state.
+             */
+            uint64_t const fIntMask = VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC | VMCPU_FF_INTERRUPT_NESTED_GUEST
+                                    | VMCPU_FF_INTERRUPT_NMI  | VMCPU_FF_INTERRUPT_SMI | VMCPU_FF_UNHALT;
+
+            if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
+                return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
+            ASMNopPause();
+
+            /*
+             * Check out how long till the next timer event.
+             */
+            uint64_t u64Delta;
+            uint64_t u64GipTime = TMTimerPollGIP(pGVM, pGVCpu, &u64Delta);
+
+            if (   !VM_FF_IS_ANY_SET(pGVM, fVmFFs)
+                && !VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
+            {
+                if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
+                    APICUpdatePendingInterrupts(pGVCpu);
+
+                if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
+                    return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
+
+                /*
+                 * Wait if there is enough time to the next timer event.
+                 */
+                if (u64Delta >= pGVCpu->vmm.s.cNsSpinBlockThreshold)
+                {
+                    /* If there are few other CPU cores around, we will procrastinate a
+                       little before going to sleep, hoping for some device raising an
+                       interrupt or similar.   Though, the best thing here would be to
+                       dynamically adjust the spin count according to its usfulness or
+                       something... */
+                    if (   pGVCpu->vmm.s.cR0HaltsSucceeded > pGVCpu->vmm.s.cR0HaltsToRing3
+                        && RTMpGetOnlineCount() >= 4)
+                    {
+                        /** @todo Figure out how we can skip this if it hasn't help recently...
+                         *        @bugref{9172#c12} */
+                        uint32_t cSpinLoops = 42;
+                        while (cSpinLoops-- > 0)
+                        {
+                            ASMNopPause();
+                            if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
+                                APICUpdatePendingInterrupts(pGVCpu);
+                            ASMNopPause();
+                            if (VM_FF_IS_ANY_SET(pGVM, fVmFFs))
+                            {
+                                STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3FromSpin);
+                                STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3);
+                                return VINF_EM_HALT;
+                            }
+                            ASMNopPause();
+                            if (VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
+                            {
+                                STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3FromSpin);
+                                STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3);
+                                return VINF_EM_HALT;
+                            }
+                            ASMNopPause();
+                            if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
+                            {
+                                STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltExecFromSpin);
+                                return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
+                            }
+                            ASMNopPause();
+                        }
+                    }
+
+                    /*
+                     * We have to set the state to VMCPUSTATE_STARTED_HALTED here so ring-3
+                     * knows when to notify us (cannot access VMINTUSERPERVMCPU::fWait from here).
+                     * After changing the state we must recheck the force flags of course.
+                     */
+                    if (VMCPU_CMPXCHG_STATE(pGVCpu, VMCPUSTATE_STARTED_HALTED, VMCPUSTATE_STARTED))
+                    {
+                        if (   !VM_FF_IS_ANY_SET(pGVM, fVmFFs)
+                            && !VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
+                        {
+                            if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
+                                APICUpdatePendingInterrupts(pGVCpu);
+
+                            if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
+                            {
+                                VMCPU_CMPXCHG_STATE(pGVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_HALTED);
+                                return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
+                            }
+
+                            /* Okay, block! */
+                            uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
+                            int rc = GVMMR0SchedHalt(pGVM, pGVCpu, u64GipTime);
+                            uint64_t const u64EndSchedHalt     = RTTimeNanoTS();
+                            uint64_t const cNsElapsedSchedHalt = u64EndSchedHalt - u64StartSchedHalt;
+                            Log10(("vmmR0DoHalt: CPU%d: halted %llu ns\n", pGVCpu->idCpu, cNsElapsedSchedHalt));
+
+                            VMCPU_CMPXCHG_STATE(pGVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_HALTED);
+                            STAM_REL_PROFILE_ADD_PERIOD(&pGVCpu->vmm.s.StatR0HaltBlock, cNsElapsedSchedHalt);
+                            if (   rc == VINF_SUCCESS
+                                || rc == VERR_INTERRUPTED)
+                            {
+                                /* Keep some stats like ring-3 does. */
+                                int64_t const cNsOverslept = u64EndSchedHalt - u64GipTime;
+                                if (cNsOverslept > 50000)
+                                    STAM_REL_PROFILE_ADD_PERIOD(&pGVCpu->vmm.s.StatR0HaltBlockOverslept, cNsOverslept);
+                                else if (cNsOverslept < -50000)
+                                    STAM_REL_PROFILE_ADD_PERIOD(&pGVCpu->vmm.s.StatR0HaltBlockInsomnia,  cNsElapsedSchedHalt);
+                                else
+                                    STAM_REL_PROFILE_ADD_PERIOD(&pGVCpu->vmm.s.StatR0HaltBlockOnTime,    cNsElapsedSchedHalt);
+
+                                /*
+                                 * Recheck whether we can resume execution or have to go to ring-3.
+                                 */
+                                if (   !VM_FF_IS_ANY_SET(pGVM, fVmFFs)
+                                    && !VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
+                                {
+                                    if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
+                                        APICUpdatePendingInterrupts(pGVCpu);
+                                    if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
+                                    {
+                                        STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltExecFromBlock);
+                                        return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
+                                    }
+                                    STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PostNoInt);
+                                    Log12(("vmmR0DoHalt: CPU%d post #2 - No pending interrupt\n", pGVCpu->idCpu));
+                                }
+                                else
+                                {
+                                    STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PostPendingFF);
+                                    Log12(("vmmR0DoHalt: CPU%d post #1 - Pending FF\n", pGVCpu->idCpu));
+                                }
+                            }
+                            else
+                            {
+                                STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3Other);
+                                Log12(("vmmR0DoHalt: CPU%d GVMMR0SchedHalt failed: %Rrc\n", pGVCpu->idCpu, rc));
+                            }
+                        }
+                        else
+                        {
+                            VMCPU_CMPXCHG_STATE(pGVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_HALTED);
+                            STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PendingFF);
+                            Log12(("vmmR0DoHalt: CPU%d failed #5 - Pending FF\n", pGVCpu->idCpu));
+                        }
+                    }
+                    else
+                    {
+                        STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3Other);
+                        Log12(("vmmR0DoHalt: CPU%d failed #4 - enmState=%d\n", pGVCpu->idCpu, VMCPU_GET_STATE(pGVCpu)));
+                    }
+                }
+                else
+                {
+                    STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3SmallDelta);
+                    Log12(("vmmR0DoHalt: CPU%d failed #3 - delta too small: %RU64\n", pGVCpu->idCpu, u64Delta));
+                }
+            }
+            else
+            {
+                STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PendingFF);
+                Log12(("vmmR0DoHalt: CPU%d failed #2 - Pending FF\n", pGVCpu->idCpu));
+            }
+        }
+        else
+        {
+            STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PendingFF);
+            Log12(("vmmR0DoHalt: CPU%d failed #1 - Pending FF\n", pGVCpu->idCpu));
+        }
+    }
+    else
+    {
+        STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3Other);
+        Log12(("vmmR0DoHalt: CPU%d failed #0 - fMayHaltInRing0=%d TRPMHasTrap=%d enmInt=%d uMWait=%u\n",
+               pGVCpu->idCpu, pGVCpu->vmm.s.fMayHaltInRing0, TRPMHasTrap(pGVCpu), enmInterruptibility, uMWait));
+    }
+
+    STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3);
+    return VINF_EM_HALT;
+}
+
+
+/**
+ * VMM ring-0 thread-context callback.
+ *
+ * This does common HM state updating and calls the HM-specific thread-context
+ * callback.
+ *
+ * @param   enmEvent    The thread-context event.
+ * @param   pvUser      Opaque pointer to the VMCPU.
+ *
+ * @thread  EMT(pvUser)
+ */
+static DECLCALLBACK(void) vmmR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, void *pvUser)
+{
+    PVMCPUCC pVCpu = (PVMCPUCC)pvUser;
+
+    switch (enmEvent)
+    {
+        case RTTHREADCTXEVENT_IN:
+        {
+            /*
+             * Linux may call us with preemption enabled (really!) but technically we
+             * cannot get preempted here, otherwise we end up in an infinite recursion
+             * scenario (i.e. preempted in resume hook -> preempt hook -> resume hook...
+             * ad infinitum). Let's just disable preemption for now...
+             */
+            /** @todo r=bird: I don't believe the above. The linux code is clearly enabling
+             *        preemption after doing the callout (one or two functions up the
+             *        call chain). */
+            /** @todo r=ramshankar: See @bugref{5313#c30}. */
+            RTTHREADPREEMPTSTATE ParanoidPreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
+            RTThreadPreemptDisable(&ParanoidPreemptState);
+
+            /* We need to update the VCPU <-> host CPU mapping. */
+            RTCPUID idHostCpu;
+            uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
+            pVCpu->iHostCpuSet   = iHostCpuSet;
+            ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
+
+            /* In the very unlikely event that the GIP delta for the CPU we're
+               rescheduled needs calculating, try force a return to ring-3.
+               We unfortunately cannot do the measurements right here. */
+            if (RT_UNLIKELY(SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
+
+            /* Invoke the HM-specific thread-context callback. */
+            HMR0ThreadCtxCallback(enmEvent, pvUser);
+
+            /* Restore preemption. */
+            RTThreadPreemptRestore(&ParanoidPreemptState);
+            break;
+        }
+
+        case RTTHREADCTXEVENT_OUT:
+        {
+            /* Invoke the HM-specific thread-context callback. */
+            HMR0ThreadCtxCallback(enmEvent, pvUser);
+
+            /*
+             * Sigh. See VMMGetCpu() used by VMCPU_ASSERT_EMT(). We cannot let several VCPUs
+             * have the same host CPU associated with it.
+             */
+            pVCpu->iHostCpuSet = UINT32_MAX;
+            ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
+            break;
+        }
+
+        default:
+            /* Invoke the HM-specific thread-context callback. */
+            HMR0ThreadCtxCallback(enmEvent, pvUser);
+            break;
+    }
+}
+
+
+/**
+ * Creates thread switching hook for the current EMT thread.
+ *
+ * This is called by GVMMR0CreateVM and GVMMR0RegisterVCpu.  If the host
+ * platform does not implement switcher hooks, no hooks will be create and the
+ * member set to NIL_RTTHREADCTXHOOK.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @thread  EMT(pVCpu)
+ */
+VMMR0_INT_DECL(int) VMMR0ThreadCtxHookCreateForEmt(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(pVCpu->vmm.s.hCtxHook == NIL_RTTHREADCTXHOOK);
+
+#if 1 /* To disable this stuff change to zero. */
+    int rc = RTThreadCtxHookCreate(&pVCpu->vmm.s.hCtxHook, 0, vmmR0ThreadCtxCallback, pVCpu);
+    if (RT_SUCCESS(rc))
+        return rc;
+#else
+    RT_NOREF(vmmR0ThreadCtxCallback);
+    int rc = VERR_NOT_SUPPORTED;
+#endif
+
+    pVCpu->vmm.s.hCtxHook = NIL_RTTHREADCTXHOOK;
+    if (rc == VERR_NOT_SUPPORTED)
+        return VINF_SUCCESS;
+
+    LogRelMax(32, ("RTThreadCtxHookCreate failed! rc=%Rrc pVCpu=%p idCpu=%RU32\n", rc, pVCpu, pVCpu->idCpu));
+    return VINF_SUCCESS; /* Just ignore it, we can live without context hooks. */
+}
+
+
+/**
+ * Destroys the thread switching hook for the specified VCPU.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @remarks Can be called from any thread.
+ */
+VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDestroyForEmt(PVMCPUCC pVCpu)
+{
+    int rc = RTThreadCtxHookDestroy(pVCpu->vmm.s.hCtxHook);
+    AssertRC(rc);
+    pVCpu->vmm.s.hCtxHook = NIL_RTTHREADCTXHOOK;
+}
+
+
+/**
+ * Disables the thread switching hook for this VCPU (if we got one).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @thread  EMT(pVCpu)
+ *
+ * @remarks This also clears VMCPU::idHostCpu, so the mapping is invalid after
+ *          this call.  This means you have to be careful with what you do!
+ */
+VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDisable(PVMCPUCC pVCpu)
+{
+    /*
+     * Clear the VCPU <-> host CPU mapping as we've left HM context.
+     * @bugref{7726#c19} explains the need for this trick:
+     *
+     *      VMXR0CallRing3Callback/SVMR0CallRing3Callback &
+     *      hmR0VmxLeaveSession/hmR0SvmLeaveSession disables context hooks during
+     *      longjmp & normal return to ring-3, which opens a window where we may be
+     *      rescheduled without changing VMCPUID::idHostCpu and cause confusion if
+     *      the CPU starts executing a different EMT.  Both functions first disables
+     *      preemption and then calls HMR0LeaveCpu which invalids idHostCpu, leaving
+     *      an opening for getting preempted.
+     */
+    /** @todo Make HM not need this API!  Then we could leave the hooks enabled
+     *        all the time. */
+    /** @todo move this into the context hook disabling if(). */
+    ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
+
+    /*
+     * Disable the context hook, if we got one.
+     */
+    if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
+    {
+        Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+        int rc = RTThreadCtxHookDisable(pVCpu->vmm.s.hCtxHook);
+        AssertRC(rc);
+    }
+}
+
+
+/**
+ * Internal version of VMMR0ThreadCtxHooksAreRegistered.
+ *
+ * @returns true if registered, false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(bool) vmmR0ThreadCtxHookIsEnabled(PVMCPUCC pVCpu)
+{
+    return RTThreadCtxHookIsEnabled(pVCpu->vmm.s.hCtxHook);
+}
+
+
+/**
+ * Whether thread-context hooks are registered for this VCPU.
+ *
+ * @returns true if registered, false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMR0_INT_DECL(bool) VMMR0ThreadCtxHookIsEnabled(PVMCPUCC pVCpu)
+{
+    return vmmR0ThreadCtxHookIsEnabled(pVCpu);
+}
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * Record return code statistics
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   rc          The status code.
+ */
+static void vmmR0RecordRC(PVMCC pVM, PVMCPUCC pVCpu, int rc)
+{
+    /*
+     * Collect statistics.
+     */
+    switch (rc)
+    {
+        case VINF_SUCCESS:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetNormal);
+            break;
+        case VINF_EM_RAW_INTERRUPT:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterrupt);
+            break;
+        case VINF_EM_RAW_INTERRUPT_HYPER:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptHyper);
+            break;
+        case VINF_EM_RAW_GUEST_TRAP:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGuestTrap);
+            break;
+        case VINF_EM_RAW_RING_SWITCH:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitch);
+            break;
+        case VINF_EM_RAW_RING_SWITCH_INT:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitchInt);
+            break;
+        case VINF_EM_RAW_STALE_SELECTOR:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetStaleSelector);
+            break;
+        case VINF_EM_RAW_IRET_TRAP:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIRETTrap);
+            break;
+        case VINF_IOM_R3_IOPORT_READ:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIORead);
+            break;
+        case VINF_IOM_R3_IOPORT_WRITE:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOWrite);
+            break;
+        case VINF_IOM_R3_IOPORT_COMMIT_WRITE:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOCommitWrite);
+            break;
+        case VINF_IOM_R3_MMIO_READ:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIORead);
+            break;
+        case VINF_IOM_R3_MMIO_WRITE:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOWrite);
+            break;
+        case VINF_IOM_R3_MMIO_COMMIT_WRITE:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOCommitWrite);
+            break;
+        case VINF_IOM_R3_MMIO_READ_WRITE:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOReadWrite);
+            break;
+        case VINF_PATM_HC_MMIO_PATCH_READ:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchRead);
+            break;
+        case VINF_PATM_HC_MMIO_PATCH_WRITE:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchWrite);
+            break;
+        case VINF_CPUM_R3_MSR_READ:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRRead);
+            break;
+        case VINF_CPUM_R3_MSR_WRITE:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRWrite);
+            break;
+        case VINF_EM_RAW_EMULATE_INSTR:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetEmulate);
+            break;
+        case VINF_PATCH_EMULATE_INSTR:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchEmulate);
+            break;
+        case VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetLDTFault);
+            break;
+        case VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGDTFault);
+            break;
+        case VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIDTFault);
+            break;
+        case VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTSSFault);
+            break;
+        case VINF_CSAM_PENDING_ACTION:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCSAMTask);
+            break;
+        case VINF_PGM_SYNC_CR3:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetSyncCR3);
+            break;
+        case VINF_PATM_PATCH_INT3:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchInt3);
+            break;
+        case VINF_PATM_PATCH_TRAP_PF:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchPF);
+            break;
+        case VINF_PATM_PATCH_TRAP_GP:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchGP);
+            break;
+        case VINF_PATM_PENDING_IRQ_AFTER_IRET:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchIretIRQ);
+            break;
+        case VINF_EM_RESCHEDULE_REM:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRescheduleREM);
+            break;
+        case VINF_EM_RAW_TO_R3:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Total);
+            if (VM_FF_IS_SET(pVM, VM_FF_TM_VIRTUAL_SYNC))
+                STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3TMVirt);
+            else if (VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES))
+                STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3HandyPages);
+            else if (VM_FF_IS_SET(pVM, VM_FF_PDM_QUEUES))
+                STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3PDMQueues);
+            else if (VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
+                STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Rendezvous);
+            else if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
+                STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3DMA);
+            else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TIMER))
+                STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Timer);
+            else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PDM_CRITSECT))
+                STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3CritSect);
+            else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TO_R3))
+                STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3FF);
+            else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
+                STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iem);
+            else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IOM))
+                STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iom);
+            else
+                STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Unknown);
+            break;
+
+        case VINF_EM_RAW_TIMER_PENDING:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTimerPending);
+            break;
+        case VINF_EM_RAW_INTERRUPT_PENDING:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptPending);
+            break;
+        case VINF_VMM_CALL_HOST:
+            switch (pVCpu->vmm.s.enmCallRing3Operation)
+            {
+                case VMMCALLRING3_PDM_CRIT_SECT_ENTER:
+                    STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPDMCritSectEnter);
+                    break;
+                case VMMCALLRING3_PDM_LOCK:
+                    STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPDMLock);
+                    break;
+                case VMMCALLRING3_PGM_POOL_GROW:
+                    STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMPoolGrow);
+                    break;
+                case VMMCALLRING3_PGM_LOCK:
+                    STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMLock);
+                    break;
+                case VMMCALLRING3_PGM_MAP_CHUNK:
+                    STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMMapChunk);
+                    break;
+                case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
+                    STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMAllocHandy);
+                    break;
+                case VMMCALLRING3_VMM_LOGGER_FLUSH:
+                    STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallLogFlush);
+                    break;
+                case VMMCALLRING3_VM_SET_ERROR:
+                    STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallVMSetError);
+                    break;
+                case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
+                    STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallVMSetRuntimeError);
+                    break;
+                case VMMCALLRING3_VM_R0_ASSERTION:
+                default:
+                    STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCallRing3);
+                    break;
+            }
+            break;
+        case VINF_PATM_DUPLICATE_FUNCTION:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPATMDuplicateFn);
+            break;
+        case VINF_PGM_CHANGE_MODE:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMChangeMode);
+            break;
+        case VINF_PGM_POOL_FLUSH_PENDING:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMFlushPending);
+            break;
+        case VINF_EM_PENDING_REQUEST:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPendingRequest);
+            break;
+        case VINF_EM_HM_PATCH_TPR_INSTR:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchTPR);
+            break;
+        default:
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMisc);
+            break;
+    }
+}
+#endif /* VBOX_WITH_STATISTICS */
+
+
+/**
+ * The Ring 0 entry point, called by the fast-ioctl path.
+ *
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   pVMIgnored      The cross context VM structure. The return code is
+ *                          stored in pVM->vmm.s.iLastGZRc.
+ * @param   idCpu           The Virtual CPU ID of the calling EMT.
+ * @param   enmOperation    Which operation to execute.
+ * @remarks Assume called with interrupts _enabled_.
+ */
+VMMR0DECL(void) VMMR0EntryFast(PGVM pGVM, PVMCC pVMIgnored, VMCPUID idCpu, VMMR0OPERATION enmOperation)
+{
+    RT_NOREF(pVMIgnored);
+
+    /*
+     * Validation.
+     */
+    if (   idCpu < pGVM->cCpus
+        && pGVM->cCpus == pGVM->cCpusUnsafe)
+    { /*likely*/ }
+    else
+    {
+        SUPR0Printf("VMMR0EntryFast: Bad idCpu=%#x cCpus=%#x cCpusUnsafe=%#x\n", idCpu, pGVM->cCpus, pGVM->cCpusUnsafe);
+        return;
+    }
+
+    PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+    RTNATIVETHREAD const hNativeThread = RTThreadNativeSelf();
+    if (RT_LIKELY(   pGVCpu->hEMT            == hNativeThread
+                  && pGVCpu->hNativeThreadR0 == hNativeThread))
+    { /* likely */ }
+    else
+    {
+        SUPR0Printf("VMMR0EntryFast: Bad thread idCpu=%#x hNativeSelf=%p pGVCpu->hEmt=%p pGVCpu->hNativeThreadR0=%p\n",
+                    idCpu, hNativeThread, pGVCpu->hEMT, pGVCpu->hNativeThreadR0);
+        return;
+    }
+
+    /*
+     * SMAP fun.
+     */
+    VMM_CHECK_SMAP_SETUP();
+    VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+
+    /*
+     * Perform requested operation.
+     */
+    switch (enmOperation)
+    {
+        /*
+         * Run guest code using the available hardware acceleration technology.
+         */
+        case VMMR0_DO_HM_RUN:
+        {
+            for (;;) /* hlt loop */
+            {
+                /*
+                 * Disable preemption.
+                 */
+                Assert(!vmmR0ThreadCtxHookIsEnabled(pGVCpu));
+                RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
+                RTThreadPreemptDisable(&PreemptState);
+
+                /*
+                 * Get the host CPU identifiers, make sure they are valid and that
+                 * we've got a TSC delta for the CPU.
+                 */
+                RTCPUID  idHostCpu;
+                uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
+                if (RT_LIKELY(   iHostCpuSet < RTCPUSET_MAX_CPUS
+                              && SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
+                {
+                    pGVCpu->iHostCpuSet = iHostCpuSet;
+                    ASMAtomicWriteU32(&pGVCpu->idHostCpu, idHostCpu);
+
+                    /*
+                     * Update the periodic preemption timer if it's active.
+                     */
+                    if (pGVM->vmm.s.fUsePeriodicPreemptionTimers)
+                        GVMMR0SchedUpdatePeriodicPreemptionTimer(pGVM, pGVCpu->idHostCpu, TMCalcHostTimerFrequency(pGVM, pGVCpu));
+                    VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+
+#ifdef VMM_R0_TOUCH_FPU
+                    /*
+                     * Make sure we've got the FPU state loaded so and we don't need to clear
+                     * CR0.TS and get out of sync with the host kernel when loading the guest
+                     * FPU state.  @ref sec_cpum_fpu (CPUM.cpp) and @bugref{4053}.
+                     */
+                    CPUMR0TouchHostFpu();
+#endif
+                    int  rc;
+                    bool fPreemptRestored = false;
+                    if (!HMR0SuspendPending())
+                    {
+                        /*
+                         * Enable the context switching hook.
+                         */
+                        if (pGVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
+                        {
+                            Assert(!RTThreadCtxHookIsEnabled(pGVCpu->vmm.s.hCtxHook));
+                            int rc2 = RTThreadCtxHookEnable(pGVCpu->vmm.s.hCtxHook); AssertRC(rc2);
+                        }
+
+                        /*
+                         * Enter HM context.
+                         */
+                        rc = HMR0Enter(pGVCpu);
+                        if (RT_SUCCESS(rc))
+                        {
+                            VMCPU_SET_STATE(pGVCpu, VMCPUSTATE_STARTED_HM);
+
+                            /*
+                             * When preemption hooks are in place, enable preemption now that
+                             * we're in HM context.
+                             */
+                            if (vmmR0ThreadCtxHookIsEnabled(pGVCpu))
+                            {
+                                fPreemptRestored = true;
+                                RTThreadPreemptRestore(&PreemptState);
+                            }
+
+                            /*
+                             * Setup the longjmp machinery and execute guest code (calls HMR0RunGuestCode).
+                             */
+                            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+                            rc = vmmR0CallRing3SetJmp(&pGVCpu->vmm.s.CallRing3JmpBufR0, HMR0RunGuestCode, pGVM, pGVCpu);
+                            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+
+                            /*
+                             * Assert sanity on the way out.  Using manual assertions code here as normal
+                             * assertions are going to panic the host since we're outside the setjmp/longjmp zone.
+                             */
+                            if (RT_UNLIKELY(   VMCPU_GET_STATE(pGVCpu) != VMCPUSTATE_STARTED_HM
+                                            && RT_SUCCESS_NP(rc)  && rc !=  VINF_VMM_CALL_HOST ))
+                            {
+                                pGVM->vmm.s.szRing0AssertMsg1[0] = '\0';
+                                RTStrPrintf(pGVM->vmm.s.szRing0AssertMsg2, sizeof(pGVM->vmm.s.szRing0AssertMsg2),
+                                            "Got VMCPU state %d expected %d.\n", VMCPU_GET_STATE(pGVCpu), VMCPUSTATE_STARTED_HM);
+                                rc = VERR_VMM_WRONG_HM_VMCPU_STATE;
+                            }
+                            /** @todo Get rid of this. HM shouldn't disable the context hook. */
+                            else if (RT_UNLIKELY(vmmR0ThreadCtxHookIsEnabled(pGVCpu)))
+                            {
+                                pGVM->vmm.s.szRing0AssertMsg1[0] = '\0';
+                                RTStrPrintf(pGVM->vmm.s.szRing0AssertMsg2, sizeof(pGVM->vmm.s.szRing0AssertMsg2),
+                                            "Thread-context hooks still enabled! VCPU=%p Id=%u rc=%d.\n", pGVCpu, pGVCpu->idCpu, rc);
+                                rc = VERR_INVALID_STATE;
+                            }
+
+                            VMCPU_SET_STATE(pGVCpu, VMCPUSTATE_STARTED);
+                        }
+                        STAM_COUNTER_INC(&pGVM->vmm.s.StatRunGC);
+
+                        /*
+                         * Invalidate the host CPU identifiers before we disable the context
+                         * hook / restore preemption.
+                         */
+                        pGVCpu->iHostCpuSet = UINT32_MAX;
+                        ASMAtomicWriteU32(&pGVCpu->idHostCpu, NIL_RTCPUID);
+
+                        /*
+                         * Disable context hooks.  Due to unresolved cleanup issues, we
+                         * cannot leave the hooks enabled when we return to ring-3.
+                         *
+                         * Note! At the moment HM may also have disabled the hook
+                         *       when we get here, but the IPRT API handles that.
+                         */
+                        if (pGVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
+                        {
+                            ASMAtomicWriteU32(&pGVCpu->idHostCpu, NIL_RTCPUID);
+                            RTThreadCtxHookDisable(pGVCpu->vmm.s.hCtxHook);
+                        }
+                    }
+                    /*
+                     * The system is about to go into suspend mode; go back to ring 3.
+                     */
+                    else
+                    {
+                        rc = VINF_EM_RAW_INTERRUPT;
+                        pGVCpu->iHostCpuSet = UINT32_MAX;
+                        ASMAtomicWriteU32(&pGVCpu->idHostCpu, NIL_RTCPUID);
+                    }
+
+                    /** @todo When HM stops messing with the context hook state, we'll disable
+                     *        preemption again before the RTThreadCtxHookDisable call. */
+                    if (!fPreemptRestored)
+                        RTThreadPreemptRestore(&PreemptState);
+
+                    pGVCpu->vmm.s.iLastGZRc = rc;
+
+                    /* Fire dtrace probe and collect statistics. */
+                    VBOXVMM_R0_VMM_RETURN_TO_RING3_HM(pGVCpu, CPUMQueryGuestCtxPtr(pGVCpu), rc);
+#ifdef VBOX_WITH_STATISTICS
+                    vmmR0RecordRC(pGVM, pGVCpu, rc);
+#endif
+                    /*
+                     * If this is a halt.
+                     */
+                    if (rc != VINF_EM_HALT)
+                    { /* we're not in a hurry for a HLT, so prefer this path */ }
+                    else
+                    {
+                        pGVCpu->vmm.s.iLastGZRc = rc = vmmR0DoHalt(pGVM, pGVCpu);
+                        if (rc == VINF_SUCCESS)
+                        {
+                            pGVCpu->vmm.s.cR0HaltsSucceeded++;
+                            continue;
+                        }
+                        pGVCpu->vmm.s.cR0HaltsToRing3++;
+                    }
+                }
+                /*
+                 * Invalid CPU set index or TSC delta in need of measuring.
+                 */
+                else
+                {
+                    pGVCpu->iHostCpuSet = UINT32_MAX;
+                    ASMAtomicWriteU32(&pGVCpu->idHostCpu, NIL_RTCPUID);
+                    RTThreadPreemptRestore(&PreemptState);
+                    if (iHostCpuSet < RTCPUSET_MAX_CPUS)
+                    {
+                        int rc = SUPR0TscDeltaMeasureBySetIndex(pGVM->pSession, iHostCpuSet, 0 /*fFlags*/,
+                                                                2 /*cMsWaitRetry*/, 5*RT_MS_1SEC /*cMsWaitThread*/,
+                                                                0 /*default cTries*/);
+                        if (RT_SUCCESS(rc) || rc == VERR_CPU_OFFLINE)
+                            pGVCpu->vmm.s.iLastGZRc = VINF_EM_RAW_TO_R3;
+                        else
+                            pGVCpu->vmm.s.iLastGZRc = rc;
+                    }
+                    else
+                        pGVCpu->vmm.s.iLastGZRc = VERR_INVALID_CPU_INDEX;
+                }
+                break;
+
+            } /* halt loop. */
+            break;
+        }
+
+#ifdef VBOX_WITH_NEM_R0
+# if defined(RT_ARCH_AMD64) && defined(RT_OS_WINDOWS)
+        case VMMR0_DO_NEM_RUN:
+        {
+            /*
+             * Setup the longjmp machinery and execute guest code (calls NEMR0RunGuestCode).
+             */
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+#  ifdef VBOXSTRICTRC_STRICT_ENABLED
+            int rc = vmmR0CallRing3SetJmp2(&pGVCpu->vmm.s.CallRing3JmpBufR0, (PFNVMMR0SETJMP2)NEMR0RunGuestCode, pGVM, idCpu);
+#  else
+            int rc = vmmR0CallRing3SetJmp2(&pGVCpu->vmm.s.CallRing3JmpBufR0, NEMR0RunGuestCode, pGVM, idCpu);
+#  endif
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            STAM_COUNTER_INC(&pGVM->vmm.s.StatRunGC);
+
+            pGVCpu->vmm.s.iLastGZRc = rc;
+
+            /*
+             * Fire dtrace probe and collect statistics.
+             */
+            VBOXVMM_R0_VMM_RETURN_TO_RING3_NEM(pGVCpu, CPUMQueryGuestCtxPtr(pGVCpu), rc);
+#  ifdef VBOX_WITH_STATISTICS
+            vmmR0RecordRC(pGVM, pGVCpu, rc);
+#  endif
+            break;
+        }
+# endif
+#endif
+
+        /*
+         * For profiling.
+         */
+        case VMMR0_DO_NOP:
+            pGVCpu->vmm.s.iLastGZRc = VINF_SUCCESS;
+            break;
+
+        /*
+         * Shouldn't happen.
+         */
+        default:
+            AssertMsgFailed(("%#x\n", enmOperation));
+            pGVCpu->vmm.s.iLastGZRc = VERR_NOT_SUPPORTED;
+            break;
+    }
+    VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+}
+
+
+/**
+ * Validates a session or VM session argument.
+ *
+ * @returns true / false accordingly.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   pClaimedSession The session claim to validate.
+ * @param   pSession        The session argument.
+ */
+DECLINLINE(bool) vmmR0IsValidSession(PGVM pGVM, PSUPDRVSESSION pClaimedSession, PSUPDRVSESSION pSession)
+{
+    /* This must be set! */
+    if (!pSession)
+        return false;
+
+    /* Only one out of the two. */
+    if (pGVM && pClaimedSession)
+        return false;
+    if (pGVM)
+        pClaimedSession = pGVM->pSession;
+    return pClaimedSession == pSession;
+}
+
+
+/**
+ * VMMR0EntryEx worker function, either called directly or when ever possible
+ * called thru a longjmp so we can exit safely on failure.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   idCpu           Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
+ *                          is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
+ * @param   enmOperation    Which operation to execute.
+ * @param   pReqHdr         This points to a SUPVMMR0REQHDR packet. Optional.
+ *                          The support driver validates this if it's present.
+ * @param   u64Arg          Some simple constant argument.
+ * @param   pSession        The session of the caller.
+ *
+ * @remarks Assume called with interrupts _enabled_.
+ */
+static int vmmR0EntryExWorker(PGVM pGVM, VMCPUID idCpu, VMMR0OPERATION enmOperation,
+                              PSUPVMMR0REQHDR pReqHdr, uint64_t u64Arg, PSUPDRVSESSION pSession)
+{
+    /*
+     * Validate pGVM and idCpu for consistency and validity.
+     */
+    if (pGVM != NULL)
+    {
+        if (RT_LIKELY(((uintptr_t)pGVM & PAGE_OFFSET_MASK) == 0))
+        { /* likely */ }
+        else
+        {
+            SUPR0Printf("vmmR0EntryExWorker: Invalid pGVM=%p! (op=%d)\n", pGVM, enmOperation);
+            return VERR_INVALID_POINTER;
+        }
+
+        if (RT_LIKELY(idCpu == NIL_VMCPUID || idCpu < pGVM->cCpus))
+        { /* likely */ }
+        else
+        {
+            SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu %#x (cCpus=%#x)\n", idCpu, pGVM->cCpus);
+            return VERR_INVALID_PARAMETER;
+        }
+
+        if (RT_LIKELY(   pGVM->enmVMState >= VMSTATE_CREATING
+                      && pGVM->enmVMState <= VMSTATE_TERMINATED
+                      && pGVM->pSession   == pSession
+                      && pGVM->pSelf      == pGVM))
+        { /* likely */ }
+        else
+        {
+            SUPR0Printf("vmmR0EntryExWorker: Invalid pGVM=%p:{.enmVMState=%d, .cCpus=%#x, .pSession=%p(==%p), .pSelf=%p(==%p)}! (op=%d)\n",
+                        pGVM, pGVM->enmVMState, pGVM->cCpus, pGVM->pSession, pSession, pGVM->pSelf, pGVM, enmOperation);
+            return VERR_INVALID_POINTER;
+        }
+    }
+    else if (RT_LIKELY(idCpu == NIL_VMCPUID))
+    { /* likely */ }
+    else
+    {
+        SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu=%u\n", idCpu);
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * SMAP fun.
+     */
+    VMM_CHECK_SMAP_SETUP();
+    VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+
+    /*
+     * Process the request.
+     */
+    int rc;
+    switch (enmOperation)
+    {
+        /*
+         * GVM requests
+         */
+        case VMMR0_DO_GVMM_CREATE_VM:
+            if (pGVM == NULL && u64Arg == 0 && idCpu == NIL_VMCPUID)
+                rc = GVMMR0CreateVMReq((PGVMMCREATEVMREQ)pReqHdr, pSession);
+            else
+                rc = VERR_INVALID_PARAMETER;
+            VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GVMM_DESTROY_VM:
+            if (pReqHdr == NULL && u64Arg == 0)
+                rc = GVMMR0DestroyVM(pGVM);
+            else
+                rc = VERR_INVALID_PARAMETER;
+            VMM_CHECK_SMAP_CHECK(RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GVMM_REGISTER_VMCPU:
+            if (pGVM != NULL)
+                rc = GVMMR0RegisterVCpu(pGVM, idCpu);
+            else
+                rc = VERR_INVALID_PARAMETER;
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GVMM_DEREGISTER_VMCPU:
+            if (pGVM != NULL)
+                rc = GVMMR0DeregisterVCpu(pGVM, idCpu);
+            else
+                rc = VERR_INVALID_PARAMETER;
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GVMM_SCHED_HALT:
+            if (pReqHdr)
+                return VERR_INVALID_PARAMETER;
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            rc = GVMMR0SchedHaltReq(pGVM, idCpu, u64Arg);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GVMM_SCHED_WAKE_UP:
+            if (pReqHdr || u64Arg)
+                return VERR_INVALID_PARAMETER;
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            rc = GVMMR0SchedWakeUp(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GVMM_SCHED_POKE:
+            if (pReqHdr || u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GVMMR0SchedPoke(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GVMM_SCHED_WAKE_UP_AND_POKE_CPUS:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GVMMR0SchedWakeUpAndPokeCpusReq(pGVM, (PGVMMSCHEDWAKEUPANDPOKECPUSREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GVMM_SCHED_POLL:
+            if (pReqHdr || u64Arg > 1)
+                return VERR_INVALID_PARAMETER;
+            rc = GVMMR0SchedPoll(pGVM, idCpu, !!u64Arg);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GVMM_QUERY_STATISTICS:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GVMMR0QueryStatisticsReq(pGVM, (PGVMMQUERYSTATISTICSSREQ)pReqHdr, pSession);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GVMM_RESET_STATISTICS:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GVMMR0ResetStatisticsReq(pGVM, (PGVMMRESETSTATISTICSSREQ)pReqHdr, pSession);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        /*
+         * Initialize the R0 part of a VM instance.
+         */
+        case VMMR0_DO_VMMR0_INIT:
+            rc = vmmR0InitVM(pGVM, RT_LODWORD(u64Arg), RT_HIDWORD(u64Arg));
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        /*
+         * Does EMT specific ring-0 init.
+         */
+        case VMMR0_DO_VMMR0_INIT_EMT:
+            rc = vmmR0InitVMEmt(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        /*
+         * Terminate the R0 part of a VM instance.
+         */
+        case VMMR0_DO_VMMR0_TERM:
+            rc = VMMR0TermVM(pGVM, 0 /*idCpu*/);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        /*
+         * Attempt to enable hm mode and check the current setting.
+         */
+        case VMMR0_DO_HM_ENABLE:
+            rc = HMR0EnableAllCpus(pGVM);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        /*
+         * Setup the hardware accelerated session.
+         */
+        case VMMR0_DO_HM_SETUP_VM:
+            rc = HMR0SetupVM(pGVM);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        /*
+         * PGM wrappers.
+         */
+        case VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES:
+            if (idCpu == NIL_VMCPUID)
+                return VERR_INVALID_CPU_ID;
+            rc = PGMR0PhysAllocateHandyPages(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_PGM_FLUSH_HANDY_PAGES:
+            if (idCpu == NIL_VMCPUID)
+                return VERR_INVALID_CPU_ID;
+            rc = PGMR0PhysFlushHandyPages(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE:
+            if (idCpu == NIL_VMCPUID)
+                return VERR_INVALID_CPU_ID;
+            rc = PGMR0PhysAllocateLargeHandyPage(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_PGM_PHYS_SETUP_IOMMU:
+            if (idCpu != 0)
+                return VERR_INVALID_CPU_ID;
+            rc = PGMR0PhysSetupIoMmu(pGVM);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_PGM_POOL_GROW:
+            if (idCpu == NIL_VMCPUID)
+                return VERR_INVALID_CPU_ID;
+            rc = PGMR0PoolGrow(pGVM);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        /*
+         * GMM wrappers.
+         */
+        case VMMR0_DO_GMM_INITIAL_RESERVATION:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0InitialReservationReq(pGVM, idCpu, (PGMMINITIALRESERVATIONREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_UPDATE_RESERVATION:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0UpdateReservationReq(pGVM, idCpu, (PGMMUPDATERESERVATIONREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_ALLOCATE_PAGES:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0AllocatePagesReq(pGVM, idCpu, (PGMMALLOCATEPAGESREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_FREE_PAGES:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0FreePagesReq(pGVM, idCpu, (PGMMFREEPAGESREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_FREE_LARGE_PAGE:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0FreeLargePageReq(pGVM, idCpu, (PGMMFREELARGEPAGEREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_QUERY_HYPERVISOR_MEM_STATS:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0QueryHypervisorMemoryStatsReq((PGMMMEMSTATSREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_QUERY_MEM_STATS:
+            if (idCpu == NIL_VMCPUID)
+                return VERR_INVALID_CPU_ID;
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0QueryMemoryStatsReq(pGVM, idCpu, (PGMMMEMSTATSREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_BALLOONED_PAGES:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0BalloonedPagesReq(pGVM, idCpu, (PGMMBALLOONEDPAGESREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_MAP_UNMAP_CHUNK:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0MapUnmapChunkReq(pGVM, (PGMMMAPUNMAPCHUNKREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_SEED_CHUNK:
+            if (pReqHdr)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0SeedChunk(pGVM, idCpu, (RTR3PTR)u64Arg);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_REGISTER_SHARED_MODULE:
+            if (idCpu == NIL_VMCPUID)
+                return VERR_INVALID_CPU_ID;
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0RegisterSharedModuleReq(pGVM, idCpu, (PGMMREGISTERSHAREDMODULEREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_UNREGISTER_SHARED_MODULE:
+            if (idCpu == NIL_VMCPUID)
+                return VERR_INVALID_CPU_ID;
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0UnregisterSharedModuleReq(pGVM, idCpu, (PGMMUNREGISTERSHAREDMODULEREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_RESET_SHARED_MODULES:
+            if (idCpu == NIL_VMCPUID)
+                return VERR_INVALID_CPU_ID;
+            if (    u64Arg
+                ||  pReqHdr)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0ResetSharedModules(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+#ifdef VBOX_WITH_PAGE_SHARING
+        case VMMR0_DO_GMM_CHECK_SHARED_MODULES:
+        {
+            if (idCpu == NIL_VMCPUID)
+                return VERR_INVALID_CPU_ID;
+            if (    u64Arg
+                ||  pReqHdr)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0CheckSharedModules(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+#endif
+
+#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
+        case VMMR0_DO_GMM_FIND_DUPLICATE_PAGE:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0FindDuplicatePageReq(pGVM, (PGMMFINDDUPLICATEPAGEREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+#endif
+
+        case VMMR0_DO_GMM_QUERY_STATISTICS:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0QueryStatisticsReq(pGVM, (PGMMQUERYSTATISTICSSREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_GMM_RESET_STATISTICS:
+            if (u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = GMMR0ResetStatisticsReq(pGVM, (PGMMRESETSTATISTICSSREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        /*
+         * A quick GCFGM mock-up.
+         */
+        /** @todo GCFGM with proper access control, ring-3 management interface and all that. */
+        case VMMR0_DO_GCFGM_SET_VALUE:
+        case VMMR0_DO_GCFGM_QUERY_VALUE:
+        {
+            if (pGVM || !pReqHdr || u64Arg || idCpu != NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            PGCFGMVALUEREQ pReq = (PGCFGMVALUEREQ)pReqHdr;
+            if (pReq->Hdr.cbReq != sizeof(*pReq))
+                return VERR_INVALID_PARAMETER;
+            if (enmOperation == VMMR0_DO_GCFGM_SET_VALUE)
+            {
+                rc = GVMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
+                //if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+                //    rc = GMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
+            }
+            else
+            {
+                rc = GVMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
+                //if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+                //    rc = GMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
+            }
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
+        /*
+         * PDM Wrappers.
+         */
+        case VMMR0_DO_PDM_DRIVER_CALL_REQ_HANDLER:
+        {
+            if (!pReqHdr || u64Arg || idCpu != NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = PDMR0DriverCallReqHandler(pGVM, (PPDMDRIVERCALLREQHANDLERREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
+        case VMMR0_DO_PDM_DEVICE_CREATE:
+        {
+            if (!pReqHdr || u64Arg || idCpu != 0)
+                return VERR_INVALID_PARAMETER;
+            rc = PDMR0DeviceCreateReqHandler(pGVM, (PPDMDEVICECREATEREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
+        case VMMR0_DO_PDM_DEVICE_GEN_CALL:
+        {
+            if (!pReqHdr || u64Arg)
+                return VERR_INVALID_PARAMETER;
+            rc = PDMR0DeviceGenCallReqHandler(pGVM, (PPDMDEVICEGENCALLREQ)pReqHdr, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
+        /** @todo Remove the once all devices has been converted to new style! @bugref{9218} */
+        case VMMR0_DO_PDM_DEVICE_COMPAT_SET_CRITSECT:
+        {
+            if (!pReqHdr || u64Arg || idCpu != 0)
+                return VERR_INVALID_PARAMETER;
+            rc = PDMR0DeviceCompatSetCritSectReqHandler(pGVM, (PPDMDEVICECOMPATSETCRITSECTREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
+        /*
+         * Requests to the internal networking service.
+         */
+        case VMMR0_DO_INTNET_OPEN:
+        {
+            PINTNETOPENREQ pReq = (PINTNETOPENREQ)pReqHdr;
+            if (u64Arg || !pReq || !vmmR0IsValidSession(pGVM, pReq->pSession, pSession) || idCpu != NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = IntNetR0OpenReq(pSession, pReq);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
+        case VMMR0_DO_INTNET_IF_CLOSE:
+            if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFCLOSEREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = IntNetR0IfCloseReq(pSession, (PINTNETIFCLOSEREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+
+        case VMMR0_DO_INTNET_IF_GET_BUFFER_PTRS:
+            if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFGETBUFFERPTRSREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = IntNetR0IfGetBufferPtrsReq(pSession, (PINTNETIFGETBUFFERPTRSREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_INTNET_IF_SET_PROMISCUOUS_MODE:
+            if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = IntNetR0IfSetPromiscuousModeReq(pSession, (PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_INTNET_IF_SET_MAC_ADDRESS:
+            if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFSETMACADDRESSREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = IntNetR0IfSetMacAddressReq(pSession, (PINTNETIFSETMACADDRESSREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_INTNET_IF_SET_ACTIVE:
+            if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFSETACTIVEREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = IntNetR0IfSetActiveReq(pSession, (PINTNETIFSETACTIVEREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_INTNET_IF_SEND:
+            if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFSENDREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = IntNetR0IfSendReq(pSession, (PINTNETIFSENDREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_INTNET_IF_WAIT:
+            if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = IntNetR0IfWaitReq(pSession, (PINTNETIFWAITREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_INTNET_IF_ABORT_WAIT:
+            if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = IntNetR0IfAbortWaitReq(pSession, (PINTNETIFABORTWAITREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+#if 0 //def VBOX_WITH_PCI_PASSTHROUGH
+        /*
+         * Requests to host PCI driver service.
+         */
+        case VMMR0_DO_PCIRAW_REQ:
+            if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PPCIRAWSENDREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = PciRawR0ProcessReq(pGVM, pSession, (PPCIRAWSENDREQ)pReqHdr);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+#endif
+
+        /*
+         * NEM requests.
+         */
+#ifdef VBOX_WITH_NEM_R0
+# if defined(RT_ARCH_AMD64) && defined(RT_OS_WINDOWS)
+        case VMMR0_DO_NEM_INIT_VM:
+            if (u64Arg || pReqHdr || idCpu != 0)
+                return VERR_INVALID_PARAMETER;
+            rc = NEMR0InitVM(pGVM);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_NEM_INIT_VM_PART_2:
+            if (u64Arg || pReqHdr || idCpu != 0)
+                return VERR_INVALID_PARAMETER;
+            rc = NEMR0InitVMPart2(pGVM);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_NEM_MAP_PAGES:
+            if (u64Arg || pReqHdr || idCpu == NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = NEMR0MapPages(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_NEM_UNMAP_PAGES:
+            if (u64Arg || pReqHdr || idCpu == NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = NEMR0UnmapPages(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_NEM_EXPORT_STATE:
+            if (u64Arg || pReqHdr || idCpu == NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = NEMR0ExportState(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_NEM_IMPORT_STATE:
+            if (pReqHdr || idCpu == NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = NEMR0ImportState(pGVM, idCpu, u64Arg);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_NEM_QUERY_CPU_TICK:
+            if (u64Arg || pReqHdr || idCpu == NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = NEMR0QueryCpuTick(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_NEM_RESUME_CPU_TICK_ON_ALL:
+            if (pReqHdr || idCpu == NIL_VMCPUID)
+                return VERR_INVALID_PARAMETER;
+            rc = NEMR0ResumeCpuTickOnAll(pGVM, idCpu, u64Arg);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+        case VMMR0_DO_NEM_UPDATE_STATISTICS:
+            if (u64Arg || pReqHdr)
+                return VERR_INVALID_PARAMETER;
+            rc = NEMR0UpdateStatistics(pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
+#   if 1 && defined(DEBUG_bird)
+        case VMMR0_DO_NEM_EXPERIMENT:
+            if (pReqHdr)
+                return VERR_INVALID_PARAMETER;
+            rc = NEMR0DoExperiment(pGVM, idCpu, u64Arg);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+#   endif
+# endif
+#endif
+
+        /*
+         * IOM requests.
+         */
+        case VMMR0_DO_IOM_GROW_IO_PORTS:
+        {
+            if (pReqHdr || idCpu != 0)
+                return VERR_INVALID_PARAMETER;
+            rc = IOMR0IoPortGrowRegistrationTables(pGVM, u64Arg);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
+        case VMMR0_DO_IOM_GROW_IO_PORT_STATS:
+        {
+            if (pReqHdr || idCpu != 0)
+                return VERR_INVALID_PARAMETER;
+            rc = IOMR0IoPortGrowStatisticsTable(pGVM, u64Arg);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
+        case VMMR0_DO_IOM_GROW_MMIO_REGS:
+        {
+            if (pReqHdr || idCpu != 0)
+                return VERR_INVALID_PARAMETER;
+            rc = IOMR0MmioGrowRegistrationTables(pGVM, u64Arg);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
+        case VMMR0_DO_IOM_GROW_MMIO_STATS:
+        {
+            if (pReqHdr || idCpu != 0)
+                return VERR_INVALID_PARAMETER;
+            rc = IOMR0MmioGrowStatisticsTable(pGVM, u64Arg);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
+        case VMMR0_DO_IOM_SYNC_STATS_INDICES:
+        {
+            if (pReqHdr || idCpu != 0)
+                return VERR_INVALID_PARAMETER;
+            rc = IOMR0IoPortSyncStatisticsIndices(pGVM);
+            if (RT_SUCCESS(rc))
+                rc = IOMR0MmioSyncStatisticsIndices(pGVM);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
+        /*
+         * For profiling.
+         */
+        case VMMR0_DO_NOP:
+        case VMMR0_DO_SLOW_NOP:
+            return VINF_SUCCESS;
+
+        /*
+         * For testing Ring-0 APIs invoked in this environment.
+         */
+        case VMMR0_DO_TESTS:
+            /** @todo make new test */
+            return VINF_SUCCESS;
+
+        default:
+            /*
+             * We're returning VERR_NOT_SUPPORT here so we've got something else
+             * than -1 which the interrupt gate glue code might return.
+             */
+            Log(("operation %#x is not supported\n", enmOperation));
+            return VERR_NOT_SUPPORTED;
+    }
+    return rc;
+}
+
+
+/**
+ * Argument for vmmR0EntryExWrapper containing the arguments for VMMR0EntryEx.
+ */
+typedef struct VMMR0ENTRYEXARGS
+{
+    PGVM                pGVM;
+    VMCPUID             idCpu;
+    VMMR0OPERATION      enmOperation;
+    PSUPVMMR0REQHDR     pReq;
+    uint64_t            u64Arg;
+    PSUPDRVSESSION      pSession;
+} VMMR0ENTRYEXARGS;
+/** Pointer to a vmmR0EntryExWrapper argument package. */
+typedef VMMR0ENTRYEXARGS *PVMMR0ENTRYEXARGS;
+
+/**
+ * This is just a longjmp wrapper function for VMMR0EntryEx calls.
+ *
+ * @returns VBox status code.
+ * @param   pvArgs      The argument package
+ */
+static DECLCALLBACK(int) vmmR0EntryExWrapper(void *pvArgs)
+{
+    return vmmR0EntryExWorker(((PVMMR0ENTRYEXARGS)pvArgs)->pGVM,
+                              ((PVMMR0ENTRYEXARGS)pvArgs)->idCpu,
+                              ((PVMMR0ENTRYEXARGS)pvArgs)->enmOperation,
+                              ((PVMMR0ENTRYEXARGS)pvArgs)->pReq,
+                              ((PVMMR0ENTRYEXARGS)pvArgs)->u64Arg,
+                              ((PVMMR0ENTRYEXARGS)pvArgs)->pSession);
+}
+
+
+/**
+ * The Ring 0 entry point, called by the support library (SUP).
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   pVM             The cross context VM structure.
+ * @param   idCpu           Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
+ *                          is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
+ * @param   enmOperation    Which operation to execute.
+ * @param   pReq            Pointer to the SUPVMMR0REQHDR packet. Optional.
+ * @param   u64Arg          Some simple constant argument.
+ * @param   pSession        The session of the caller.
+ * @remarks Assume called with interrupts _enabled_.
+ */
+VMMR0DECL(int) VMMR0EntryEx(PGVM pGVM, PVMCC pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation,
+                            PSUPVMMR0REQHDR pReq, uint64_t u64Arg, PSUPDRVSESSION pSession)
+{
+    /*
+     * Requests that should only happen on the EMT thread will be
+     * wrapped in a setjmp so we can assert without causing trouble.
+     */
+    if (   pVM  != NULL
+        && pGVM != NULL
+        && pVM  == pGVM /** @todo drop pGVM */
+        && idCpu < pGVM->cCpus
+        && pGVM->pSession == pSession
+        && pGVM->pSelf    == pVM)
+    {
+        switch (enmOperation)
+        {
+            /* These might/will be called before VMMR3Init. */
+            case VMMR0_DO_GMM_INITIAL_RESERVATION:
+            case VMMR0_DO_GMM_UPDATE_RESERVATION:
+            case VMMR0_DO_GMM_ALLOCATE_PAGES:
+            case VMMR0_DO_GMM_FREE_PAGES:
+            case VMMR0_DO_GMM_BALLOONED_PAGES:
+            /* On the mac we might not have a valid jmp buf, so check these as well. */
+            case VMMR0_DO_VMMR0_INIT:
+            case VMMR0_DO_VMMR0_TERM:
+
+            case VMMR0_DO_PDM_DEVICE_CREATE:
+            case VMMR0_DO_PDM_DEVICE_GEN_CALL:
+            case VMMR0_DO_IOM_GROW_IO_PORTS:
+            case VMMR0_DO_IOM_GROW_IO_PORT_STATS:
+            {
+                PGVMCPU        pGVCpu        = &pGVM->aCpus[idCpu];
+                RTNATIVETHREAD hNativeThread = RTThreadNativeSelf();
+                if (RT_LIKELY(   pGVCpu->hEMT            == hNativeThread
+                              && pGVCpu->hNativeThreadR0 == hNativeThread))
+                {
+                    if (!pGVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack)
+                        break;
+
+                    /** @todo validate this EMT claim... GVM knows. */
+                    VMMR0ENTRYEXARGS Args;
+                    Args.pGVM = pGVM;
+                    Args.idCpu = idCpu;
+                    Args.enmOperation = enmOperation;
+                    Args.pReq = pReq;
+                    Args.u64Arg = u64Arg;
+                    Args.pSession = pSession;
+                    return vmmR0CallRing3SetJmpEx(&pGVCpu->vmm.s.CallRing3JmpBufR0, vmmR0EntryExWrapper, &Args);
+                }
+                return VERR_VM_THREAD_NOT_EMT;
+            }
+
+            default:
+            case VMMR0_DO_PGM_POOL_GROW:
+                break;
+        }
+    }
+    return vmmR0EntryExWorker(pGVM, idCpu, enmOperation, pReq, u64Arg, pSession);
+}
+
+
+/**
+ * Checks whether we've armed the ring-0 long jump machinery.
+ *
+ * @returns @c true / @c false
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @thread  EMT
+ * @sa      VMMIsLongJumpArmed
+ */
+VMMR0_INT_DECL(bool) VMMR0IsLongJumpArmed(PVMCPUCC pVCpu)
+{
+#ifdef RT_ARCH_X86
+    return pVCpu->vmm.s.CallRing3JmpBufR0.eip
+        && !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
+#else
+    return pVCpu->vmm.s.CallRing3JmpBufR0.rip
+        && !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
+#endif
+}
+
+
+/**
+ * Checks whether we've done a ring-3 long jump.
+ *
+ * @returns @c true / @c false
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @thread  EMT
+ */
+VMMR0_INT_DECL(bool) VMMR0IsInRing3LongJump(PVMCPUCC pVCpu)
+{
+    return pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
+}
+
+
+/**
+ * Internal R0 logger worker: Flush logger.
+ *
+ * @param   pLogger     The logger instance to flush.
+ * @remark  This function must be exported!
+ */
+VMMR0DECL(void) vmmR0LoggerFlush(PRTLOGGER pLogger)
+{
+#ifdef LOG_ENABLED
+    /*
+     * Convert the pLogger into a VM handle and 'call' back to Ring-3.
+     * (This is a bit paranoid code.)
+     */
+    PVMMR0LOGGER pR0Logger = (PVMMR0LOGGER)((uintptr_t)pLogger - RT_UOFFSETOF(VMMR0LOGGER, Logger));
+    if (    !VALID_PTR(pR0Logger)
+        ||  !VALID_PTR(pR0Logger + 1)
+        ||  pLogger->u32Magic != RTLOGGER_MAGIC)
+    {
+# ifdef DEBUG
+        SUPR0Printf("vmmR0LoggerFlush: pLogger=%p!\n", pLogger);
+# endif
+        return;
+    }
+    if (pR0Logger->fFlushingDisabled)
+        return; /* quietly */
+
+    PVMCC pVM = pR0Logger->pVM;
+    if (   !VALID_PTR(pVM)
+        || pVM->pSelf != pVM)
+    {
+# ifdef DEBUG
+        SUPR0Printf("vmmR0LoggerFlush: pVM=%p! pSelf=%p! pLogger=%p\n", pVM, pVM->pSelf, pLogger);
+# endif
+        return;
+    }
+
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    if (pVCpu)
+    {
+        /*
+         * Check that the jump buffer is armed.
+         */
+# ifdef RT_ARCH_X86
+        if (    !pVCpu->vmm.s.CallRing3JmpBufR0.eip
+            ||  pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
+# else
+        if (    !pVCpu->vmm.s.CallRing3JmpBufR0.rip
+            ||  pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
+# endif
+        {
+# ifdef DEBUG
+            SUPR0Printf("vmmR0LoggerFlush: Jump buffer isn't armed!\n");
+# endif
+            return;
+        }
+        VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VMM_LOGGER_FLUSH, 0);
+    }
+# ifdef DEBUG
+    else
+        SUPR0Printf("vmmR0LoggerFlush: invalid VCPU context!\n");
+# endif
+#else
+    NOREF(pLogger);
+#endif  /* LOG_ENABLED */
+}
+
+#ifdef LOG_ENABLED
+
+/**
+ * Disables flushing of the ring-0 debug log.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMR0_INT_DECL(void) VMMR0LogFlushDisable(PVMCPUCC pVCpu)
+{
+    if (pVCpu->vmm.s.pR0LoggerR0)
+        pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = true;
+    if (pVCpu->vmm.s.pR0RelLoggerR0)
+        pVCpu->vmm.s.pR0RelLoggerR0->fFlushingDisabled = true;
+}
+
+
+/**
+ * Enables flushing of the ring-0 debug log.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMR0_INT_DECL(void) VMMR0LogFlushEnable(PVMCPUCC pVCpu)
+{
+    if (pVCpu->vmm.s.pR0LoggerR0)
+        pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = false;
+    if (pVCpu->vmm.s.pR0RelLoggerR0)
+        pVCpu->vmm.s.pR0RelLoggerR0->fFlushingDisabled = false;
+}
+
+
+/**
+ * Checks if log flushing is disabled or not.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMR0_INT_DECL(bool) VMMR0IsLogFlushDisabled(PVMCPUCC pVCpu)
+{
+    if (pVCpu->vmm.s.pR0LoggerR0)
+        return pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled;
+    if (pVCpu->vmm.s.pR0RelLoggerR0)
+        return pVCpu->vmm.s.pR0RelLoggerR0->fFlushingDisabled;
+    return true;
+}
+
+#endif /* LOG_ENABLED */
+
+/**
+ * Override RTLogRelGetDefaultInstanceEx so we can do LogRel to VBox.log from EMTs in ring-0.
+ */
+DECLEXPORT(PRTLOGGER) RTLogRelGetDefaultInstanceEx(uint32_t fFlagsAndGroup)
+{
+    PGVMCPU pGVCpu = GVMMR0GetGVCpuByEMT(NIL_RTNATIVETHREAD);
+    if (pGVCpu)
+    {
+        PVMCPUCC pVCpu = pGVCpu;
+        if (RT_VALID_PTR(pVCpu))
+        {
+            PVMMR0LOGGER pVmmLogger = pVCpu->vmm.s.pR0RelLoggerR0;
+            if (RT_VALID_PTR(pVmmLogger))
+            {
+                if (   pVmmLogger->fCreated
+                    && pVmmLogger->pVM == pGVCpu->pGVM)
+                {
+                    if (pVmmLogger->Logger.fFlags & RTLOGFLAGS_DISABLED)
+                        return NULL;
+                    uint16_t const fFlags = RT_LO_U16(fFlagsAndGroup);
+                    uint16_t const iGroup = RT_HI_U16(fFlagsAndGroup);
+                    if (   iGroup != UINT16_MAX
+                        && (   (  pVmmLogger->Logger.afGroups[iGroup < pVmmLogger->Logger.cGroups ? iGroup : 0]
+                                & (fFlags | (uint32_t)RTLOGGRPFLAGS_ENABLED))
+                            != (fFlags | (uint32_t)RTLOGGRPFLAGS_ENABLED)))
+                        return NULL;
+                    return &pVmmLogger->Logger;
+                }
+            }
+        }
+    }
+    return SUPR0GetDefaultLogRelInstanceEx(fFlagsAndGroup);
+}
+
+
+/**
+ * Jump back to ring-3 if we're the EMT and the longjmp is armed.
+ *
+ * @returns true if the breakpoint should be hit, false if it should be ignored.
+ */
+DECLEXPORT(bool) RTCALL RTAssertShouldPanic(void)
+{
+#if 0
+    return true;
+#else
+    PVMCC pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
+    if (pVM)
+    {
+        PVMCPUCC pVCpu = VMMGetCpu(pVM);
+
+        if (pVCpu)
+        {
+#ifdef RT_ARCH_X86
+            if (    pVCpu->vmm.s.CallRing3JmpBufR0.eip
+                &&  !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
+#else
+            if (    pVCpu->vmm.s.CallRing3JmpBufR0.rip
+                &&  !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
+#endif
+            {
+                int rc = VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VM_R0_ASSERTION, 0);
+                return RT_FAILURE_NP(rc);
+            }
+        }
+    }
+#ifdef RT_OS_LINUX
+    return true;
+#else
+    return false;
+#endif
+#endif
+}
+
+
+/**
+ * Override this so we can push it up to ring-3.
+ *
+ * @param   pszExpr     Expression. Can be NULL.
+ * @param   uLine       Location line number.
+ * @param   pszFile     Location file name.
+ * @param   pszFunction Location function name.
+ */
+DECLEXPORT(void) RTCALL RTAssertMsg1Weak(const char *pszExpr, unsigned uLine, const char *pszFile, const char *pszFunction)
+{
+    /*
+     * To the log.
+     */
+    LogAlways(("\n!!R0-Assertion Failed!!\n"
+               "Expression: %s\n"
+               "Location  : %s(%d) %s\n",
+               pszExpr, pszFile, uLine, pszFunction));
+
+    /*
+     * To the global VMM buffer.
+     */
+    PVMCC pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
+    if (pVM)
+        RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1),
+                    "\n!!R0-Assertion Failed!!\n"
+                    "Expression: %.*s\n"
+                    "Location  : %s(%d) %s\n",
+                    sizeof(pVM->vmm.s.szRing0AssertMsg1) / 4 * 3, pszExpr,
+                    pszFile, uLine, pszFunction);
+
+    /*
+     * Continue the normal way.
+     */
+    RTAssertMsg1(pszExpr, uLine, pszFile, pszFunction);
+}
+
+
+/**
+ * Callback for RTLogFormatV which writes to the ring-3 log port.
+ * See PFNLOGOUTPUT() for details.
+ */
+static DECLCALLBACK(size_t) rtLogOutput(void *pv, const char *pachChars, size_t cbChars)
+{
+    for (size_t i = 0; i < cbChars; i++)
+    {
+        LogAlways(("%c", pachChars[i])); NOREF(pachChars);
+    }
+
+    NOREF(pv);
+    return cbChars;
+}
+
+
+/**
+ * Override this so we can push it up to ring-3.
+ *
+ * @param   pszFormat   The format string.
+ * @param   va          Arguments.
+ */
+DECLEXPORT(void) RTCALL RTAssertMsg2WeakV(const char *pszFormat, va_list va)
+{
+    va_list vaCopy;
+
+    /*
+     * Push the message to the loggers.
+     */
+    PRTLOGGER pLog = RTLogGetDefaultInstance(); /* Don't initialize it here... */
+    if (pLog)
+    {
+        va_copy(vaCopy, va);
+        RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
+        va_end(vaCopy);
+    }
+    pLog = RTLogRelGetDefaultInstance();
+    if (pLog)
+    {
+        va_copy(vaCopy, va);
+        RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
+        va_end(vaCopy);
+    }
+
+    /*
+     * Push it to the global VMM buffer.
+     */
+    PVMCC pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
+    if (pVM)
+    {
+        va_copy(vaCopy, va);
+        RTStrPrintfV(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2), pszFormat, vaCopy);
+        va_end(vaCopy);
+    }
+
+    /*
+     * Continue the normal way.
+     */
+    RTAssertMsg2V(pszFormat, va);
+}
+
diff --git a/src/VBox/VMM/VMMR0/VMMR0.def b/src/VBox/VMM/VMMR0/VMMR0.def
new file mode 100644
index 00000000..5ef46607
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/VMMR0.def
@@ -0,0 +1,120 @@
+; $Id: VMMR0.def $
+;; @file
+; VMM Ring 0 DLL - Definition file.
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+LIBRARY VMMR0.r0
+EXPORTS
+    ; data
+
+    ; code
+    GIMGetMmio2Regions
+    PDMCritSectEnter
+    PDMCritSectEnterDebug
+    PDMCritSectIsOwner
+    PDMCritSectLeave
+    PDMHCCritSectScheduleExitEvent
+    PDMCritSectTryEnter
+    PDMCritSectTryEnterDebug
+    PDMQueueAlloc
+    PDMQueueInsert
+    PGMHandlerPhysicalPageTempOff
+    PGMShwMakePageWritable
+    PGMPhysSimpleWriteGCPhys
+    PGMPhysSimpleReadGCPtr
+    PGMPhysSimpleWriteGCPtr
+    PGMPhysReadGCPtr
+    PGMPhysWriteGCPtr
+    PGMPhysSimpleDirtyWriteGCPtr
+    PDMR0DeviceRegisterModule
+    PDMR0DeviceDeregisterModule
+    IOMMmioResetRegion
+    IOMMmioMapMmio2Page
+    RTLogDefaultInstance
+    RTLogDefaultInstanceEx
+    RTLogRelGetDefaultInstance
+    RTLogRelGetDefaultInstanceEx
+    RTLogLogger
+    RTLogLoggerEx
+    RTLogLoggerExV
+    RTTimeMilliTS
+    RTTraceBufAddMsgF
+    RTTraceBufAddPos
+    RTTraceBufAddPosMsgF
+    TMTimerFromMilli
+    TMTimerFromMicro
+    TMTimerFromNano
+    TMTimerGet
+    TMTimerGetFreq
+    TMTimerIsActive
+    TMTimerIsLockOwner
+    TMTimerLock
+    TMTimerSet
+    TMTimerSetRelative
+    TMTimerSetMillies
+    TMTimerSetMicro
+    TMTimerSetNano
+    TMTimerSetFrequencyHint
+    TMTimerStop
+    TMTimerUnlock
+    VMMGetSvnRev
+    vmmR0LoggerFlush
+    vmmR0LoggerWrapper
+    VMSetError
+    VMSetErrorV
+
+    ; Internal Networking
+    IntNetR0Open
+    IntNetR0IfClose
+    IntNetR0IfGetBufferPtrs
+    IntNetR0IfSetPromiscuousMode
+    IntNetR0IfSetMacAddress
+    IntNetR0IfSetActive
+    IntNetR0IfSend
+    IntNetR0IfWait
+
+    ; Network Shaper
+    PDMNsAllocateBandwidth
+
+    ; runtime
+    RTAssertMsg1Weak
+    RTAssertMsg2Weak
+    RTAssertShouldPanic
+    RTCrc32
+    RTOnceSlow
+    RTTimeNanoTSLegacySyncInvarNoDelta
+    RTTimeNanoTSLegacySyncInvarWithDelta
+    RTTimeNanoTSLegacyAsync
+    RTTimeNanoTSLFenceSyncInvarNoDelta
+    RTTimeNanoTSLFenceSyncInvarWithDelta
+    RTTimeNanoTSLFenceAsync
+    RTTimeSystemNanoTS
+    RTTimeNanoTS
+    ASMMultU64ByU32DivByU32             ; not-os2
+    ASMAtomicXchgU8                     ; not-x86
+    ASMAtomicXchgU16                    ; not-x86
+    ASMBitFirstSet                      ; not-x86
+    ASMNopPause                         ; not-x86
+    nocrt_memchr
+    nocrt_memcmp
+    nocrt_memcpy
+    memcpy=nocrt_memcpy                 ; not-os2
+    nocrt_memmove
+    nocrt_memset
+    memset=nocrt_memset                 ; not-os2
+    nocrt_strcpy
+    nocrt_strcmp
+    nocrt_strchr
+    nocrt_strlen
+
diff --git a/src/VBox/VMM/VMMR0/VMMR0JmpA-amd64.asm b/src/VBox/VMM/VMMR0/VMMR0JmpA-amd64.asm
new file mode 100644
index 00000000..9d330417
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/VMMR0JmpA-amd64.asm
@@ -0,0 +1,491 @@
+; $Id: VMMR0JmpA-amd64.asm $
+;; @file
+; VMM - R0 SetJmp / LongJmp routines for AMD64.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+;*******************************************************************************
+;* Header Files                                                                *
+;*******************************************************************************
+%define RT_ASM_WITH_SEH64
+%include "VBox/asmdefs.mac"
+%include "VMMInternal.mac"
+%include "VBox/err.mac"
+%include "VBox/param.mac"
+
+
+;*******************************************************************************
+;*  Defined Constants And Macros                                               *
+;*******************************************************************************
+%define RESUME_MAGIC    07eadf00dh
+%define STACK_PADDING   0eeeeeeeeeeeeeeeeh
+
+;; Workaround for linux 4.6 fast/slow syscall stack depth difference.
+%ifdef VMM_R0_SWITCH_STACK
+ %define STACK_FUZZ_SIZE 0
+%else
+ %define STACK_FUZZ_SIZE 128
+%endif
+
+
+BEGINCODE
+
+
+;;
+; The setjmp variant used for calling Ring-3.
+;
+; This differs from the normal setjmp in that it will resume VMMRZCallRing3 if we're
+; in the middle of a ring-3 call. Another differences is the function pointer and
+; argument. This has to do with resuming code and the stack frame of the caller.
+;
+; @returns  VINF_SUCCESS on success or whatever is passed to vmmR0CallRing3LongJmp.
+; @param    pJmpBuf msc:rcx gcc:rdi x86:[esp+0x04]     Our jmp_buf.
+; @param    pfn     msc:rdx gcc:rsi x86:[esp+0x08]     The function to be called when not resuming.
+; @param    pvUser1 msc:r8  gcc:rdx x86:[esp+0x0c]     The argument of that function.
+; @param    pvUser2 msc:r9  gcc:rcx x86:[esp+0x10]     The argument of that function.
+;
+BEGINPROC vmmR0CallRing3SetJmp
+GLOBALNAME vmmR0CallRing3SetJmp2
+GLOBALNAME vmmR0CallRing3SetJmpEx
+    ;
+    ; Save the registers.
+    ;
+    push    rbp
+    SEH64_PUSH_xBP
+    mov     rbp, rsp
+    SEH64_SET_FRAME_xBP 0
+ %ifdef ASM_CALL64_MSC
+    sub     rsp, 30h + STACK_FUZZ_SIZE  ; (10h is used by resume (??), 20h for callee spill area)
+    SEH64_ALLOCATE_STACK 30h + STACK_FUZZ_SIZE
+SEH64_END_PROLOGUE
+    mov     r11, rdx                    ; pfn
+    mov     rdx, rcx                    ; pJmpBuf;
+ %else
+    sub     rsp, 10h + STACK_FUZZ_SIZE  ; (10h is used by resume (??))
+    SEH64_ALLOCATE_STACK 10h + STACK_FUZZ_SIZE
+SEH64_END_PROLOGUE
+    mov     r8, rdx                     ; pvUser1 (save it like MSC)
+    mov     r9, rcx                     ; pvUser2 (save it like MSC)
+    mov     r11, rsi                    ; pfn
+    mov     rdx, rdi                    ; pJmpBuf
+ %endif
+    mov     [xDX + VMMR0JMPBUF.rbx], rbx
+ %ifdef ASM_CALL64_MSC
+    mov     [xDX + VMMR0JMPBUF.rsi], rsi
+    mov     [xDX + VMMR0JMPBUF.rdi], rdi
+ %endif
+    mov     [xDX + VMMR0JMPBUF.rbp], rbp
+    mov     [xDX + VMMR0JMPBUF.r12], r12
+    mov     [xDX + VMMR0JMPBUF.r13], r13
+    mov     [xDX + VMMR0JMPBUF.r14], r14
+    mov     [xDX + VMMR0JMPBUF.r15], r15
+    mov     xAX, [rbp + 8]              ; (not really necessary, except for validity check)
+    mov     [xDX + VMMR0JMPBUF.rip], xAX
+ %ifdef ASM_CALL64_MSC
+    lea     r10, [rsp + 20h]            ; must save the spill area
+ %else
+    lea     r10, [rsp]
+ %endif
+    mov     [xDX + VMMR0JMPBUF.rsp], r10
+ %ifdef RT_OS_WINDOWS
+    movdqa  [xDX + VMMR0JMPBUF.xmm6], xmm6
+    movdqa  [xDX + VMMR0JMPBUF.xmm7], xmm7
+    movdqa  [xDX + VMMR0JMPBUF.xmm8], xmm8
+    movdqa  [xDX + VMMR0JMPBUF.xmm9], xmm9
+    movdqa  [xDX + VMMR0JMPBUF.xmm10], xmm10
+    movdqa  [xDX + VMMR0JMPBUF.xmm11], xmm11
+    movdqa  [xDX + VMMR0JMPBUF.xmm12], xmm12
+    movdqa  [xDX + VMMR0JMPBUF.xmm13], xmm13
+    movdqa  [xDX + VMMR0JMPBUF.xmm14], xmm14
+    movdqa  [xDX + VMMR0JMPBUF.xmm15], xmm15
+ %endif
+    pushf
+    pop     xAX
+    mov     [xDX + VMMR0JMPBUF.rflags], xAX
+
+    ;
+    ; If we're not in a ring-3 call, call pfn and return.
+    ;
+    test    byte [xDX + VMMR0JMPBUF.fInRing3Call], 1
+    jnz     .resume
+
+ %ifdef VMM_R0_SWITCH_STACK
+    mov     r15, [xDX + VMMR0JMPBUF.pvSavedStack]
+    test    r15, r15
+    jz      .entry_error
+  %ifdef VBOX_STRICT
+    cmp     dword [r15], 0h
+    jne     .entry_error
+    mov     rdi, r15
+    mov     rcx, VMM_STACK_SIZE / 8
+    mov     rax, qword 0eeeeeeeffeeeeeeeh
+    repne stosq
+    mov     [rdi - 10h], rbx
+  %endif
+    lea     r15, [r15 + VMM_STACK_SIZE - 40h]
+    mov     rsp, r15                    ; Switch stack!
+ %endif ; VMM_R0_SWITCH_STACK
+
+    mov     r12, rdx                    ; Save pJmpBuf.
+ %ifdef ASM_CALL64_MSC
+    mov     rcx, r8                     ; pvUser -> arg0
+    mov     rdx, r9
+ %else
+    mov     rdi, r8                     ; pvUser -> arg0
+    mov     rsi, r9
+ %endif
+    call    r11
+    mov     rdx, r12                    ; Restore pJmpBuf
+
+ %ifdef VMM_R0_SWITCH_STACK
+  %ifdef VBOX_STRICT
+    mov     r15, [xDX + VMMR0JMPBUF.pvSavedStack]
+    mov     dword [r15], 0h             ; Reset the marker
+  %endif
+ %endif
+
+    ;
+    ; Return like in the long jump but clear eip, no shortcuts here.
+    ;
+.proper_return:
+%ifdef RT_OS_WINDOWS
+    movdqa  xmm6,  [xDX + VMMR0JMPBUF.xmm6 ]
+    movdqa  xmm7,  [xDX + VMMR0JMPBUF.xmm7 ]
+    movdqa  xmm8,  [xDX + VMMR0JMPBUF.xmm8 ]
+    movdqa  xmm9,  [xDX + VMMR0JMPBUF.xmm9 ]
+    movdqa  xmm10, [xDX + VMMR0JMPBUF.xmm10]
+    movdqa  xmm11, [xDX + VMMR0JMPBUF.xmm11]
+    movdqa  xmm12, [xDX + VMMR0JMPBUF.xmm12]
+    movdqa  xmm13, [xDX + VMMR0JMPBUF.xmm13]
+    movdqa  xmm14, [xDX + VMMR0JMPBUF.xmm14]
+    movdqa  xmm15, [xDX + VMMR0JMPBUF.xmm15]
+%endif
+    mov     rbx, [xDX + VMMR0JMPBUF.rbx]
+%ifdef ASM_CALL64_MSC
+    mov     rsi, [xDX + VMMR0JMPBUF.rsi]
+    mov     rdi, [xDX + VMMR0JMPBUF.rdi]
+%endif
+    mov     r12, [xDX + VMMR0JMPBUF.r12]
+    mov     r13, [xDX + VMMR0JMPBUF.r13]
+    mov     r14, [xDX + VMMR0JMPBUF.r14]
+    mov     r15, [xDX + VMMR0JMPBUF.r15]
+    mov     rbp, [xDX + VMMR0JMPBUF.rbp]
+    and     qword [xDX + VMMR0JMPBUF.rip], byte 0 ; used for valid check.
+    mov     rsp, [xDX + VMMR0JMPBUF.rsp]
+    push    qword [xDX + VMMR0JMPBUF.rflags]
+    popf
+    leave
+    ret
+
+.entry_error:
+    mov     eax, VERR_VMM_SET_JMP_ERROR
+    jmp     .proper_return
+
+.stack_overflow:
+    mov     eax, VERR_VMM_SET_JMP_STACK_OVERFLOW
+    jmp     .proper_return
+
+    ;
+    ; Aborting resume.
+    ; Note! No need to restore XMM registers here since we haven't touched them yet.
+    ;
+.bad:
+    and     qword [xDX + VMMR0JMPBUF.rip], byte 0 ; used for valid check.
+    mov     rbx, [xDX + VMMR0JMPBUF.rbx]
+ %ifdef ASM_CALL64_MSC
+    mov     rsi, [xDX + VMMR0JMPBUF.rsi]
+    mov     rdi, [xDX + VMMR0JMPBUF.rdi]
+ %endif
+    mov     r12, [xDX + VMMR0JMPBUF.r12]
+    mov     r13, [xDX + VMMR0JMPBUF.r13]
+    mov     r14, [xDX + VMMR0JMPBUF.r14]
+    mov     r15, [xDX + VMMR0JMPBUF.r15]
+    mov     eax, VERR_VMM_SET_JMP_ABORTED_RESUME
+    leave
+    ret
+
+    ;
+    ; Resume VMMRZCallRing3 the call.
+    ;
+.resume:
+ %ifndef VMM_R0_SWITCH_STACK
+    ; Sanity checks incoming stack, applying fuzz if needed.
+    sub     r10, [xDX + VMMR0JMPBUF.SpCheck]
+    jz      .resume_stack_checked_out
+    add     r10, STACK_FUZZ_SIZE        ; plus/minus STACK_FUZZ_SIZE is fine.
+    cmp     r10, STACK_FUZZ_SIZE * 2
+    ja      .bad
+
+    mov     r10, [xDX + VMMR0JMPBUF.SpCheck]
+    mov     [xDX + VMMR0JMPBUF.rsp], r10 ; Must be update in case of another long jump (used for save calc).
+
+.resume_stack_checked_out:
+    mov     ecx, [xDX + VMMR0JMPBUF.cbSavedStack]
+    cmp     rcx, VMM_STACK_SIZE
+    ja      .bad
+    test    rcx, 7
+    jnz     .bad
+    mov     rdi, [xDX + VMMR0JMPBUF.SpCheck]
+    sub     rdi, [xDX + VMMR0JMPBUF.SpResume]
+    cmp     rcx, rdi
+    jne     .bad
+ %endif
+
+%ifdef VMM_R0_SWITCH_STACK
+    ; Switch stack.
+    mov     rsp, [xDX + VMMR0JMPBUF.SpResume]
+%else
+    ; Restore the stack.
+    mov     ecx, [xDX + VMMR0JMPBUF.cbSavedStack]
+    shr     ecx, 3
+    mov     rsi, [xDX + VMMR0JMPBUF.pvSavedStack]
+    mov     rdi, [xDX + VMMR0JMPBUF.SpResume]
+    mov     rsp, rdi
+    rep movsq
+%endif ; !VMM_R0_SWITCH_STACK
+    mov     byte [xDX + VMMR0JMPBUF.fInRing3Call], 0
+
+    ;
+    ; Continue where we left off.
+    ;
+%ifdef VBOX_STRICT
+    pop     rax                         ; magic
+    cmp     rax, RESUME_MAGIC
+    je      .magic_ok
+    mov     ecx, 0123h
+    mov     [ecx], edx
+.magic_ok:
+%endif
+%ifdef RT_OS_WINDOWS
+    movdqa  xmm6,  [rsp + 000h]
+    movdqa  xmm7,  [rsp + 010h]
+    movdqa  xmm8,  [rsp + 020h]
+    movdqa  xmm9,  [rsp + 030h]
+    movdqa  xmm10, [rsp + 040h]
+    movdqa  xmm11, [rsp + 050h]
+    movdqa  xmm12, [rsp + 060h]
+    movdqa  xmm13, [rsp + 070h]
+    movdqa  xmm14, [rsp + 080h]
+    movdqa  xmm15, [rsp + 090h]
+    add     rsp, 0a0h
+%endif
+    popf
+    pop     rbx
+%ifdef ASM_CALL64_MSC
+    pop     rsi
+    pop     rdi
+%endif
+    pop     r12
+    pop     r13
+    pop     r14
+    pop     r15
+    pop     rbp
+    xor     eax, eax                    ; VINF_SUCCESS
+    ret
+ENDPROC vmmR0CallRing3SetJmp
+
+
+;;
+; Worker for VMMRZCallRing3.
+; This will save the stack and registers.
+;
+; @param    pJmpBuf msc:rcx gcc:rdi x86:[ebp+8]     Pointer to the jump buffer.
+; @param    rc      msc:rdx gcc:rsi x86:[ebp+c]     The return code.
+;
+BEGINPROC vmmR0CallRing3LongJmp
+    ;
+    ; Save the registers on the stack.
+    ;
+    push    rbp
+    SEH64_PUSH_xBP
+    mov     rbp, rsp
+    SEH64_SET_FRAME_xBP 0
+    push    r15
+    SEH64_PUSH_GREG r15
+    push    r14
+    SEH64_PUSH_GREG r14
+    push    r13
+    SEH64_PUSH_GREG r13
+    push    r12
+    SEH64_PUSH_GREG r12
+%ifdef ASM_CALL64_MSC
+    push    rdi
+    SEH64_PUSH_GREG rdi
+    push    rsi
+    SEH64_PUSH_GREG rsi
+%endif
+    push    rbx
+    SEH64_PUSH_GREG rbx
+    pushf
+    SEH64_ALLOCATE_STACK 8
+%ifdef RT_OS_WINDOWS
+    sub     rsp, 0a0h
+    SEH64_ALLOCATE_STACK 0a0h
+    movdqa  [rsp + 000h], xmm6
+    movdqa  [rsp + 010h], xmm7
+    movdqa  [rsp + 020h], xmm8
+    movdqa  [rsp + 030h], xmm9
+    movdqa  [rsp + 040h], xmm10
+    movdqa  [rsp + 050h], xmm11
+    movdqa  [rsp + 060h], xmm12
+    movdqa  [rsp + 070h], xmm13
+    movdqa  [rsp + 080h], xmm14
+    movdqa  [rsp + 090h], xmm15
+%endif
+%ifdef VBOX_STRICT
+    push    RESUME_MAGIC
+    SEH64_ALLOCATE_STACK 8
+%endif
+SEH64_END_PROLOGUE
+
+    ;
+    ; Normalize the parameters.
+    ;
+%ifdef ASM_CALL64_MSC
+    mov     eax, edx                    ; rc
+    mov     rdx, rcx                    ; pJmpBuf
+%else
+    mov     rdx, rdi                    ; pJmpBuf
+    mov     eax, esi                    ; rc
+%endif
+
+    ;
+    ; Is the jump buffer armed?
+    ;
+    cmp     qword [xDX + VMMR0JMPBUF.rip], byte 0
+    je      .nok
+
+    ;
+    ; Sanity checks.
+    ;
+    mov     rdi, [xDX + VMMR0JMPBUF.pvSavedStack]
+    test    rdi, rdi                    ; darwin may set this to 0.
+    jz      .nok
+    mov     [xDX + VMMR0JMPBUF.SpResume], rsp
+ %ifndef VMM_R0_SWITCH_STACK
+    mov     rsi, rsp
+    mov     rcx, [xDX + VMMR0JMPBUF.rsp]
+    sub     rcx, rsi
+
+    ; two sanity checks on the size.
+    cmp     rcx, VMM_STACK_SIZE         ; check max size.
+    jnbe    .nok
+
+    ;
+    ; Copy the stack
+    ;
+    test    ecx, 7                      ; check alignment
+    jnz     .nok
+    mov     [xDX + VMMR0JMPBUF.cbSavedStack], ecx
+    shr     ecx, 3
+    rep movsq
+
+ %endif ; !VMM_R0_SWITCH_STACK
+
+    ; Save a PC and return PC here to assist unwinding.
+.unwind_point:
+    lea     rcx, [.unwind_point wrt RIP]
+    mov     [xDX + VMMR0JMPBUF.SavedEipForUnwind], rcx
+    mov     rcx, [xDX + VMMR0JMPBUF.rbp]
+    lea     rcx, [rcx + 8]
+    mov     [xDX + VMMR0JMPBUF.UnwindRetPcLocation], rcx
+    mov     rcx, [rcx]
+    mov     [xDX + VMMR0JMPBUF.UnwindRetPcValue], rcx
+
+    ; Save RSP & RBP to enable stack dumps
+    mov     rcx, rbp
+    mov     [xDX + VMMR0JMPBUF.SavedEbp], rcx
+    sub     rcx, 8
+    mov     [xDX + VMMR0JMPBUF.SavedEsp], rcx
+
+    ; store the last pieces of info.
+    mov     rcx, [xDX + VMMR0JMPBUF.rsp]
+    mov     [xDX + VMMR0JMPBUF.SpCheck], rcx
+    mov     byte [xDX + VMMR0JMPBUF.fInRing3Call], 1
+
+    ;
+    ; Do the long jump.
+    ;
+%ifdef RT_OS_WINDOWS
+    movdqa  xmm6,  [xDX + VMMR0JMPBUF.xmm6 ]
+    movdqa  xmm7,  [xDX + VMMR0JMPBUF.xmm7 ]
+    movdqa  xmm8,  [xDX + VMMR0JMPBUF.xmm8 ]
+    movdqa  xmm9,  [xDX + VMMR0JMPBUF.xmm9 ]
+    movdqa  xmm10, [xDX + VMMR0JMPBUF.xmm10]
+    movdqa  xmm11, [xDX + VMMR0JMPBUF.xmm11]
+    movdqa  xmm12, [xDX + VMMR0JMPBUF.xmm12]
+    movdqa  xmm13, [xDX + VMMR0JMPBUF.xmm13]
+    movdqa  xmm14, [xDX + VMMR0JMPBUF.xmm14]
+    movdqa  xmm15, [xDX + VMMR0JMPBUF.xmm15]
+%endif
+    mov     rbx, [xDX + VMMR0JMPBUF.rbx]
+%ifdef ASM_CALL64_MSC
+    mov     rsi, [xDX + VMMR0JMPBUF.rsi]
+    mov     rdi, [xDX + VMMR0JMPBUF.rdi]
+%endif
+    mov     r12, [xDX + VMMR0JMPBUF.r12]
+    mov     r13, [xDX + VMMR0JMPBUF.r13]
+    mov     r14, [xDX + VMMR0JMPBUF.r14]
+    mov     r15, [xDX + VMMR0JMPBUF.r15]
+    mov     rbp, [xDX + VMMR0JMPBUF.rbp]
+    mov     rsp, [xDX + VMMR0JMPBUF.rsp]
+    push    qword [xDX + VMMR0JMPBUF.rflags]
+    popf
+    leave
+    ret
+
+    ;
+    ; Failure
+    ;
+.nok:
+%ifdef VBOX_STRICT
+    pop     rax                         ; magic
+    cmp     rax, RESUME_MAGIC
+    je      .magic_ok
+    mov     ecx, 0123h
+    mov     [rcx], edx
+.magic_ok:
+%endif
+    mov     eax, VERR_VMM_LONG_JMP_ERROR
+%ifdef RT_OS_WINDOWS
+    add     rsp, 0a0h                   ; skip XMM registers since they are unmodified.
+%endif
+    popf
+    pop     rbx
+%ifdef ASM_CALL64_MSC
+    pop     rsi
+    pop     rdi
+%endif
+    pop     r12
+    pop     r13
+    pop     r14
+    pop     r15
+    leave
+    ret
+ENDPROC vmmR0CallRing3LongJmp
+
+
+;;
+; Internal R0 logger worker: Logger wrapper.
+;
+; @cproto VMMR0DECL(void) vmmR0LoggerWrapper(const char *pszFormat, ...)
+;
+BEGINPROC_EXPORTED vmmR0LoggerWrapper
+SEH64_END_PROLOGUE
+    int3
+    int3
+    int3
+    ret
+ENDPROC vmmR0LoggerWrapper
+
diff --git a/src/VBox/VMM/VMMR0/VMMR0JmpA-x86.asm b/src/VBox/VMM/VMMR0/VMMR0JmpA-x86.asm
new file mode 100644
index 00000000..8ec64213
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/VMMR0JmpA-x86.asm
@@ -0,0 +1,401 @@
+; $Id: VMMR0JmpA-x86.asm $
+;; @file
+; VMM - R0 SetJmp / LongJmp routines for X86.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+;*******************************************************************************
+;* Header Files                                                                *
+;*******************************************************************************
+%include "VBox/asmdefs.mac"
+%include "VMMInternal.mac"
+%include "VBox/err.mac"
+%include "VBox/param.mac"
+
+
+;*******************************************************************************
+;*  Defined Constants And Macros                                               *
+;*******************************************************************************
+%define RESUME_MAGIC    07eadf00dh
+%define STACK_PADDING   0eeeeeeeeh
+
+
+; For vmmR0LoggerWrapper. (The other architecture(s) use(s) C99 variadic macros.)
+extern NAME(RTLogLogger)
+
+
+BEGINCODE
+
+
+;;
+; The setjmp variant used for calling Ring-3.
+;
+; This differs from the normal setjmp in that it will resume VMMRZCallRing3 if we're
+; in the middle of a ring-3 call. Another differences is the function pointer and
+; argument. This has to do with resuming code and the stack frame of the caller.
+;
+; @returns  VINF_SUCCESS on success or whatever is passed to vmmR0CallRing3LongJmp.
+; @param    pJmpBuf msc:rcx gcc:rdi x86:[esp+0x04]     Our jmp_buf.
+; @param    pfn     msc:rdx gcc:rsi x86:[esp+0x08]     The function to be called when not resuming.
+; @param    pvUser1 msc:r8  gcc:rdx x86:[esp+0x0c]     The argument of that function.
+; @param    pvUser2 msc:r9  gcc:rcx x86:[esp+0x10]     The argument of that function.
+;
+BEGINPROC vmmR0CallRing3SetJmp
+GLOBALNAME vmmR0CallRing3SetJmp2
+GLOBALNAME vmmR0CallRing3SetJmpEx
+    ;
+    ; Save the registers.
+    ;
+    mov     edx, [esp + 4h]             ; pJmpBuf
+    mov     [xDX + VMMR0JMPBUF.ebx], ebx
+    mov     [xDX + VMMR0JMPBUF.esi], esi
+    mov     [xDX + VMMR0JMPBUF.edi], edi
+    mov     [xDX + VMMR0JMPBUF.ebp], ebp
+    mov     xAX, [esp]
+    mov     [xDX + VMMR0JMPBUF.eip], xAX
+    lea     ecx, [esp + 4]              ; (used in resume)
+    mov     [xDX + VMMR0JMPBUF.esp], ecx
+    pushf
+    pop     xAX
+    mov     [xDX + VMMR0JMPBUF.eflags], xAX
+
+    ;
+    ; If we're not in a ring-3 call, call pfn and return.
+    ;
+    test    byte [xDX + VMMR0JMPBUF.fInRing3Call], 1
+    jnz     .resume
+
+    mov     ebx, edx                    ; pJmpBuf -> ebx (persistent reg)
+%ifdef VMM_R0_SWITCH_STACK
+    mov     esi, [ebx + VMMR0JMPBUF.pvSavedStack]
+    test    esi, esi
+    jz      .entry_error
+ %ifdef VBOX_STRICT
+    cmp     dword [esi], 0h
+    jne     .entry_error
+    mov     edx, esi
+    mov     edi, esi
+    mov     ecx, VMM_STACK_SIZE / 4
+    mov     eax, STACK_PADDING
+    repne stosd
+ %endif
+    lea     esi, [esi + VMM_STACK_SIZE - 32]
+    mov     [esi + 1ch], dword 0deadbeefh ; Marker 1.
+    mov     [esi + 18h], ebx            ; Save pJmpBuf pointer.
+    mov     [esi + 14h], dword 00c00ffeeh ; Marker 2.
+    mov     [esi + 10h], dword 0f00dbeefh ; Marker 3.
+    mov     edx, [esp + 10h]            ; pvArg2
+    mov     ecx, [esp + 0ch]            ; pvArg1
+    mov     eax, [esp + 08h]            ; pfn
+ %if 1                                  ; Use this to eat of some extra stack - handy for finding paths using lots of stack.
+  %define FRAME_OFFSET 0
+ %else
+  %define FRAME_OFFSET 1024
+ %endif
+    mov     [esi - FRAME_OFFSET + 04h], edx
+    mov     [esi - FRAME_OFFSET      ], ecx
+    lea     esp, [esi - FRAME_OFFSET]   ; Switch stack!
+    call    eax
+    and     dword [esi + 1ch], byte 0   ; reset marker.
+
+ %ifdef VBOX_STRICT
+    ; Calc stack usage and check for overflows.
+    mov     edi, [ebx + VMMR0JMPBUF.pvSavedStack]
+    cmp     dword [edi], STACK_PADDING  ; Check for obvious stack overflow.
+    jne     .stack_overflow
+    mov     esi, eax                    ; save eax
+    mov     eax, STACK_PADDING
+    mov     ecx, VMM_STACK_SIZE / 4
+    cld
+    repe scasd
+    shl     ecx, 2                      ; *4
+    cmp     ecx, VMM_STACK_SIZE - 64    ; Less than 64 bytes left -> overflow as well.
+    mov     eax, esi                    ; restore eax in case of overflow (esi remains used)
+    jae     .stack_overflow_almost
+
+    ; Update stack usage statistics.
+    cmp     ecx, [ebx + VMMR0JMPBUF.cbUsedMax] ; New max usage?
+    jle     .no_used_max
+    mov     [ebx + VMMR0JMPBUF.cbUsedMax], ecx
+.no_used_max:
+    ; To simplify the average stuff, just historize before we hit div errors.
+    inc     dword [ebx + VMMR0JMPBUF.cUsedTotal]
+    test    [ebx + VMMR0JMPBUF.cUsedTotal], dword 0c0000000h
+    jz      .no_historize
+    mov     dword [ebx + VMMR0JMPBUF.cUsedTotal], 2
+    mov     edi, [ebx + VMMR0JMPBUF.cbUsedAvg]
+    mov     [ebx + VMMR0JMPBUF.cbUsedTotal], edi
+    mov     dword [ebx + VMMR0JMPBUF.cbUsedTotal + 4], 0
+.no_historize:
+    add     [ebx + VMMR0JMPBUF.cbUsedTotal], ecx
+    adc     dword [ebx + VMMR0JMPBUF.cbUsedTotal + 4], 0
+    mov     eax, [ebx + VMMR0JMPBUF.cbUsedTotal]
+    mov     edx, [ebx + VMMR0JMPBUF.cbUsedTotal + 4]
+    mov     edi, [ebx + VMMR0JMPBUF.cUsedTotal]
+    div     edi
+    mov     [ebx + VMMR0JMPBUF.cbUsedAvg], eax
+
+    mov     eax, esi                    ; restore eax (final, esi released)
+
+    mov     edi, [ebx + VMMR0JMPBUF.pvSavedStack]
+    mov     dword [edi], 0h             ; Reset the overflow marker.
+ %endif ; VBOX_STRICT
+
+%else  ; !VMM_R0_SWITCH_STACK
+    mov     ecx, [esp + 0ch]            ; pvArg1
+    mov     edx, [esp + 10h]            ; pvArg2
+    mov     eax, [esp + 08h]            ; pfn
+    sub     esp, 12                     ; align the stack on a 16-byte boundary.
+    mov     [esp      ], ecx
+    mov     [esp + 04h], edx
+    call    eax
+%endif ; !VMM_R0_SWITCH_STACK
+    mov     edx, ebx                    ; pJmpBuf -> edx (volatile reg)
+
+    ;
+    ; Return like in the long jump but clear eip, no short cuts here.
+    ;
+.proper_return:
+    mov     ebx, [xDX + VMMR0JMPBUF.ebx]
+    mov     esi, [xDX + VMMR0JMPBUF.esi]
+    mov     edi, [xDX + VMMR0JMPBUF.edi]
+    mov     ebp, [xDX + VMMR0JMPBUF.ebp]
+    mov     xCX, [xDX + VMMR0JMPBUF.eip]
+    and     dword [xDX + VMMR0JMPBUF.eip], byte 0 ; used for valid check.
+    mov     esp, [xDX + VMMR0JMPBUF.esp]
+    push    dword [xDX + VMMR0JMPBUF.eflags]
+    popf
+    jmp     xCX
+
+.entry_error:
+    mov     eax, VERR_VMM_SET_JMP_ERROR
+    jmp     .proper_return
+
+.stack_overflow:
+    mov     eax, VERR_VMM_SET_JMP_STACK_OVERFLOW
+    mov     edx, ebx
+    jmp     .proper_return
+
+.stack_overflow_almost:
+    mov     eax, VERR_VMM_SET_JMP_STACK_OVERFLOW
+    mov     edx, ebx
+    jmp     .proper_return
+
+    ;
+    ; Aborting resume.
+    ;
+.bad:
+    and     dword [xDX + VMMR0JMPBUF.eip], byte 0 ; used for valid check.
+    mov     edi, [xDX + VMMR0JMPBUF.edi]
+    mov     esi, [xDX + VMMR0JMPBUF.esi]
+    mov     ebx, [xDX + VMMR0JMPBUF.ebx]
+    mov     eax, VERR_VMM_SET_JMP_ABORTED_RESUME
+    ret
+
+    ;
+    ; Resume VMMRZCallRing3 the call.
+    ;
+.resume:
+    ; Sanity checks.
+%ifdef VMM_R0_SWITCH_STACK
+    mov     eax, [xDX + VMMR0JMPBUF.pvSavedStack]
+ %ifdef RT_STRICT
+    cmp     dword [eax], STACK_PADDING
+ %endif
+    lea     eax, [eax + VMM_STACK_SIZE - 32]
+    cmp     dword [eax + 1ch], 0deadbeefh       ; Marker 1.
+    jne     .bad
+ %ifdef RT_STRICT
+    cmp     [esi + 18h], edx                    ; The saved pJmpBuf pointer.
+    jne     .bad
+    cmp     dword [esi + 14h], 00c00ffeeh       ; Marker 2.
+    jne     .bad
+    cmp     dword [esi + 10h], 0f00dbeefh       ; Marker 3.
+    jne     .bad
+ %endif
+%else  ; !VMM_R0_SWITCH_STACK
+    cmp     ecx, [xDX + VMMR0JMPBUF.SpCheck]
+    jne     .bad
+.espCheck_ok:
+    mov     ecx, [xDX + VMMR0JMPBUF.cbSavedStack]
+    cmp     ecx, VMM_STACK_SIZE
+    ja      .bad
+    test    ecx, 3
+    jnz     .bad
+    mov     edi, [xDX + VMMR0JMPBUF.esp]
+    sub     edi, [xDX + VMMR0JMPBUF.SpResume]
+    cmp     ecx, edi
+    jne     .bad
+%endif
+
+%ifdef VMM_R0_SWITCH_STACK
+    ; Switch stack.
+    mov     esp, [xDX + VMMR0JMPBUF.SpResume]
+%else
+    ; Restore the stack.
+    mov     ecx, [xDX + VMMR0JMPBUF.cbSavedStack]
+    shr     ecx, 2
+    mov     esi, [xDX + VMMR0JMPBUF.pvSavedStack]
+    mov     edi, [xDX + VMMR0JMPBUF.SpResume]
+    mov     esp, edi
+    rep movsd
+%endif ; !VMM_R0_SWITCH_STACK
+    mov     byte [xDX + VMMR0JMPBUF.fInRing3Call], 0
+
+    ;
+    ; Continue where we left off.
+    ;
+%ifdef VBOX_STRICT
+    pop     eax                         ; magic
+    cmp     eax, RESUME_MAGIC
+    je      .magic_ok
+    mov     ecx, 0123h
+    mov     [ecx], edx
+.magic_ok:
+%endif
+    popf
+    pop     ebx
+    pop     esi
+    pop     edi
+    pop     ebp
+    xor     eax, eax                    ; VINF_SUCCESS
+    ret
+ENDPROC vmmR0CallRing3SetJmp
+
+
+;;
+; Worker for VMMRZCallRing3.
+; This will save the stack and registers.
+;
+; @param    pJmpBuf msc:rcx gcc:rdi x86:[ebp+8]     Pointer to the jump buffer.
+; @param    rc      msc:rdx gcc:rsi x86:[ebp+c]     The return code.
+;
+BEGINPROC vmmR0CallRing3LongJmp
+    ;
+    ; Save the registers on the stack.
+    ;
+    push    ebp
+    mov     ebp, esp
+    push    edi
+    push    esi
+    push    ebx
+    pushf
+%ifdef VBOX_STRICT
+    push    RESUME_MAGIC
+%endif
+
+    ;
+    ; Load parameters.
+    ;
+    mov     edx, [ebp + 08h]            ; pJmpBuf
+    mov     eax, [ebp + 0ch]            ; rc
+
+    ;
+    ; Is the jump buffer armed?
+    ;
+    cmp     dword [xDX + VMMR0JMPBUF.eip], byte 0
+    je      .nok
+
+    ;
+    ; Sanity checks.
+    ;
+    mov     edi, [xDX + VMMR0JMPBUF.pvSavedStack]
+    test    edi, edi                    ; darwin may set this to 0.
+    jz      .nok
+    mov     [xDX + VMMR0JMPBUF.SpResume], esp
+%ifndef VMM_R0_SWITCH_STACK
+    mov     esi, esp
+    mov     ecx, [xDX + VMMR0JMPBUF.esp]
+    sub     ecx, esi
+
+    ; two sanity checks on the size.
+    cmp     ecx, VMM_STACK_SIZE         ; check max size.
+    jnbe    .nok
+
+    ;
+    ; Copy the stack.
+    ;
+    test    ecx, 3                      ; check alignment
+    jnz     .nok
+    mov     [xDX + VMMR0JMPBUF.cbSavedStack], ecx
+    shr     ecx, 2
+    rep movsd
+%endif ; !VMM_R0_SWITCH_STACK
+
+    ; Save a PC here to assist unwinding.
+.unwind_point:
+    mov     dword [xDX + VMMR0JMPBUF.SavedEipForUnwind], .unwind_point
+    mov     ecx, [xDX + VMMR0JMPBUF.ebp]
+    lea     ecx, [ecx + 4]
+    mov     [xDX + VMMR0JMPBUF.UnwindRetPcLocation], ecx
+
+    ; Save ESP & EBP to enable stack dumps
+    mov     ecx, ebp
+    mov     [xDX + VMMR0JMPBUF.SavedEbp], ecx
+    sub     ecx, 4
+    mov     [xDX + VMMR0JMPBUF.SavedEsp], ecx
+
+    ; store the last pieces of info.
+    mov     ecx, [xDX + VMMR0JMPBUF.esp]
+    mov     [xDX + VMMR0JMPBUF.SpCheck], ecx
+    mov     byte [xDX + VMMR0JMPBUF.fInRing3Call], 1
+
+    ;
+    ; Do the long jump.
+    ;
+    mov     ebx, [xDX + VMMR0JMPBUF.ebx]
+    mov     esi, [xDX + VMMR0JMPBUF.esi]
+    mov     edi, [xDX + VMMR0JMPBUF.edi]
+    mov     ebp, [xDX + VMMR0JMPBUF.ebp]
+    mov     ecx, [xDX + VMMR0JMPBUF.eip]
+    mov     [xDX + VMMR0JMPBUF.UnwindRetPcValue], ecx
+    mov     esp, [xDX + VMMR0JMPBUF.esp]
+    push    dword [xDX + VMMR0JMPBUF.eflags]
+    popf
+    jmp     ecx
+
+    ;
+    ; Failure
+    ;
+.nok:
+%ifdef VBOX_STRICT
+    pop     eax                         ; magic
+    cmp     eax, RESUME_MAGIC
+    je      .magic_ok
+    mov     ecx, 0123h
+    mov     [ecx], edx
+.magic_ok:
+%endif
+    popf
+    pop     ebx
+    pop     esi
+    pop     edi
+    mov     eax, VERR_VMM_LONG_JMP_ERROR
+    leave
+    ret
+ENDPROC vmmR0CallRing3LongJmp
+
+
+;;
+; Internal R0 logger worker: Logger wrapper.
+;
+; @cproto VMMR0DECL(void) vmmR0LoggerWrapper(const char *pszFormat, ...)
+;
+EXPORTEDNAME vmmR0LoggerWrapper
+    push    0                           ; assumes we're the wrapper for a default instance.
+    call    NAME(RTLogLogger)
+    add     esp, byte 4
+    ret
+ENDPROC vmmR0LoggerWrapper
+
diff --git a/src/VBox/VMM/VMMR0/VMMR0TripleFaultHack.cpp b/src/VBox/VMM/VMMR0/VMMR0TripleFaultHack.cpp
new file mode 100644
index 00000000..d286ec70
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/VMMR0TripleFaultHack.cpp
@@ -0,0 +1,209 @@
+/* $Id: VMMR0TripleFaultHack.cpp $ */
+/** @file
+ * VMM - Host Context Ring 0, Triple Fault Debugging Hack.
+ *
+ * Only use this when desperate.  May not work on all systems, esp. newer ones,
+ * since it require BIOS support for the warm reset vector at 0467h.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <VBox/vmm/vmm.h>
+#include "VMMInternal.h"
+#include <VBox/param.h>
+
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/assert.h>
+#include <iprt/memobj.h>
+#include <iprt/mem.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+static RTR0MEMOBJ   g_hMemPage0;
+static RTR0MEMOBJ   g_hMapPage0;
+static uint8_t     *g_pbPage0;
+
+static RTR0MEMOBJ   g_hMemLowCore;
+static RTR0MEMOBJ   g_hMapLowCore;
+static uint8_t     *g_pbLowCore;
+static RTHCPHYS     g_HCPhysLowCore;
+
+/** @name For restoring memory we've overwritten.
+ * @{ */
+static uint32_t     g_u32SavedVector;
+static uint16_t     g_u16SavedCadIndicator;
+static void        *g_pvSavedLowCore;
+/** @}  */
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+/* VMMR0TripleFaultHackA.asm */
+DECLASM(void) vmmR0TripleFaultHackStart(void);
+DECLASM(void) vmmR0TripleFaultHackEnd(void);
+DECLASM(void) vmmR0TripleFaultHackTripleFault(void);
+
+
+/**
+ * Initalizes the triple fault / boot hack.
+ *
+ * Always call vmmR0TripleFaultHackTerm to clean up, even when this call fails.
+ *
+ * @returns VBox status code.
+ */
+int vmmR0TripleFaultHackInit(void)
+{
+    /*
+     * Map the first page.
+     */
+    int rc = RTR0MemObjEnterPhys(&g_hMemPage0, 0, PAGE_SIZE, RTMEM_CACHE_POLICY_DONT_CARE);
+    AssertRCReturn(rc, rc);
+    rc = RTR0MemObjMapKernel(&g_hMapPage0, g_hMemPage0, (void *)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
+    AssertRCReturn(rc, rc);
+    g_pbPage0 = (uint8_t *)RTR0MemObjAddress(g_hMapPage0);
+    LogRel(("0040:0067 = %04x:%04x\n", RT_MAKE_U16(g_pbPage0[0x467+2],  g_pbPage0[0x467+3]),  RT_MAKE_U16(g_pbPage0[0x467+0],  g_pbPage0[0x467+1]) ));
+
+    /*
+     * Allocate some "low core" memory.  If that fails, just grab some memory.
+     */
+    //rc = RTR0MemObjAllocPhys(&g_hMemLowCore, PAGE_SIZE, _1M - 1);
+    //__debugbreak();
+    rc = RTR0MemObjEnterPhys(&g_hMemLowCore, 0x7000, PAGE_SIZE, RTMEM_CACHE_POLICY_DONT_CARE);
+    AssertRCReturn(rc, rc);
+    rc = RTR0MemObjMapKernel(&g_hMapLowCore, g_hMemLowCore, (void *)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
+    AssertRCReturn(rc, rc);
+    g_pbLowCore = (uint8_t *)RTR0MemObjAddress(g_hMapLowCore);
+    g_HCPhysLowCore = RTR0MemObjGetPagePhysAddr(g_hMapLowCore, 0);
+    LogRel(("Low core at %RHp mapped at %p\n", g_HCPhysLowCore, g_pbLowCore));
+
+    /*
+     * Save memory we'll be overwriting.
+     */
+    g_pvSavedLowCore = RTMemAlloc(PAGE_SIZE);
+    AssertReturn(g_pvSavedLowCore, VERR_NO_MEMORY);
+    memcpy(g_pvSavedLowCore, g_pbLowCore, PAGE_SIZE);
+
+    g_u32SavedVector = RT_MAKE_U32_FROM_U8(g_pbPage0[0x467], g_pbPage0[0x467+1], g_pbPage0[0x467+2], g_pbPage0[0x467+3]);
+    g_u16SavedCadIndicator = RT_MAKE_U16(g_pbPage0[0x472], g_pbPage0[0x472+1]);
+
+    /*
+     * Install the code.
+     */
+    size_t cbCode = (uintptr_t)&vmmR0TripleFaultHackEnd - (uintptr_t)&vmmR0TripleFaultHackStart;
+    AssertLogRelReturn(cbCode <= PAGE_SIZE, VERR_OUT_OF_RANGE);
+    memcpy(g_pbLowCore, &vmmR0TripleFaultHackStart, cbCode);
+
+    g_pbPage0[0x467+0] = 0x00;
+    g_pbPage0[0x467+1] = 0x70;
+    g_pbPage0[0x467+2] = 0x00;
+    g_pbPage0[0x467+3] = 0x00;
+
+    g_pbPage0[0x472+0] = 0x34;
+    g_pbPage0[0x472+1] = 0x12;
+
+    /*
+     * Configure the status port and cmos shutdown command.
+     */
+    uint32_t fSaved = ASMIntDisableFlags();
+
+    ASMOutU8(0x70, 0x0f);
+    ASMOutU8(0x71, 0x0a);
+
+    ASMOutU8(0x70, 0x05);
+    ASMInU8(0x71);
+
+    ASMReloadCR3();
+    ASMWriteBackAndInvalidateCaches();
+
+    ASMSetFlags(fSaved);
+
+#if 1 /* For testing & debugging. */
+    vmmR0TripleFaultHackTripleFault();
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Try undo the harm done by the init function.
+ *
+ * This may leave the system in an unstable state since we might have been
+ * hijacking memory below 1MB that is in use by the kernel.
+ */
+void vmmR0TripleFaultHackTerm(void)
+{
+    /*
+     * Restore overwritten memory.
+     */
+    if (   g_pvSavedLowCore
+        && g_pbLowCore)
+        memcpy(g_pbLowCore, g_pvSavedLowCore, PAGE_SIZE);
+
+    if (g_pbPage0)
+    {
+        g_pbPage0[0x467+0] = RT_BYTE1(g_u32SavedVector);
+        g_pbPage0[0x467+1] = RT_BYTE2(g_u32SavedVector);
+        g_pbPage0[0x467+2] = RT_BYTE3(g_u32SavedVector);
+        g_pbPage0[0x467+3] = RT_BYTE4(g_u32SavedVector);
+
+        g_pbPage0[0x472+0] = RT_BYTE1(g_u16SavedCadIndicator);
+        g_pbPage0[0x472+1] = RT_BYTE2(g_u16SavedCadIndicator);
+    }
+
+    /*
+     * Fix the CMOS.
+     */
+    if (g_pvSavedLowCore)
+    {
+        uint32_t fSaved = ASMIntDisableFlags();
+
+        ASMOutU8(0x70, 0x0f);
+        ASMOutU8(0x71, 0x0a);
+
+        ASMOutU8(0x70, 0x00);
+        ASMInU8(0x71);
+
+        ASMReloadCR3();
+        ASMWriteBackAndInvalidateCaches();
+
+        ASMSetFlags(fSaved);
+    }
+
+    /*
+     * Release resources.
+     */
+    RTMemFree(g_pvSavedLowCore);
+    g_pvSavedLowCore = NULL;
+
+    RTR0MemObjFree(g_hMemLowCore, true /*fFreeMappings*/);
+    g_hMemLowCore   = NIL_RTR0MEMOBJ;
+    g_hMapLowCore   = NIL_RTR0MEMOBJ;
+    g_pbLowCore     = NULL;
+    g_HCPhysLowCore = NIL_RTHCPHYS;
+
+    RTR0MemObjFree(g_hMemPage0, true /*fFreeMappings*/);
+    g_hMemPage0     = NIL_RTR0MEMOBJ;
+    g_hMapPage0     = NIL_RTR0MEMOBJ;
+    g_pbPage0       = NULL;
+}
+
diff --git a/src/VBox/VMM/VMMR0/VMMR0TripleFaultHackA.asm b/src/VBox/VMM/VMMR0/VMMR0TripleFaultHackA.asm
new file mode 100644
index 00000000..cca91654
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/VMMR0TripleFaultHackA.asm
@@ -0,0 +1,264 @@
+; $Id: VMMR0TripleFaultHackA.asm $
+;; @file
+; VMM - Host Context Ring 0, Assembly Code for The Triple Fault Debugging Hack.
+;
+
+;
+; Copyright (C) 2011-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+;*******************************************************************************
+;* Header Files                                                                *
+;*******************************************************************************
+%include "VBox/asmdefs.mac"
+
+
+BEGINCODE
+GLOBALNAME vmmR0TripleFaultHackStart
+%define CALC_ADDR(a_Addr) ( (a_Addr) - NAME(vmmR0TripleFaultHackStart) + 07000h )
+
+
+BITS 16
+BEGINPROC vmmR0TripleFaultHack
+    ; Set up stack.
+    cli                                 ; paranoia
+    mov     sp, 0ffffh
+    mov     ax, cs
+    mov     ss, ax
+    mov     ds, ax
+    mov     es, ax
+    cld                                 ; paranoia
+
+    COM_INIT
+
+    ; Beep and say hello to the post-reset world.
+    call    NAME(vmmR0TripleFaultHackBeep)
+    mov     si, CALC_ADDR(.s_szHello)
+    call    NAME(vmmR0TripleFaultHackPrint)
+
+.forever:
+    hlt
+    jmp     .forever
+
+.s_szHello:
+    db      'Hello post-reset world', 0ah, 0dh, 0
+ENDPROC   vmmR0TripleFaultHack
+
+;; ds:si = zero terminated string.
+BEGINPROC   vmmR0TripleFaultHackPrint
+    push    eax
+    push    esi
+
+.outer_loop:
+    lodsb
+    cmp     al, 0
+    je      .done
+    call    NAME(vmmR0TripleFaultHackPrintCh)
+    jmp     .outer_loop
+
+.done:
+    pop     esi
+    pop     eax
+    ret
+ENDPROC     vmmR0TripleFaultHackPrint
+
+
+;; al = char to print
+BEGINPROC   vmmR0TripleFaultHackPrintCh
+    push    eax
+    push    edx
+    push    ecx
+    mov     ah, al                      ; save char.
+
+    ; Wait for status.
+    mov     ecx, _1G
+    mov     dx, VBOX_UART_BASE + 5
+.pre_status:
+    in      al, dx
+    test    al, 20h
+    jnz     .put_char
+    dec     ecx
+    jnz     .pre_status
+
+    ; Write the character.
+.put_char:
+    mov     al, ah
+    mov     dx, VBOX_UART_BASE
+    out     dx, al
+
+    ; Wait for status.
+    mov     ecx, _1G
+    mov     dx, VBOX_UART_BASE + 5
+.post_status:
+    in      al, dx
+    test    al, 20h
+    jnz     .done
+    dec     ecx
+    jnz     .post_status
+
+.done:
+    pop     ecx
+    pop     edx
+    pop     eax
+    ret
+ENDPROC     vmmR0TripleFaultHackPrintCh
+
+;;
+; make a 440 BEEP.
+BEGINPROC   vmmR0TripleFaultHackBeep
+    push    eax
+    push    edx
+    push    ecx
+
+    ; program PIT(1) and stuff.
+    mov     al, 10110110b
+    out     43h, al
+    mov     ax, 0a79h                   ; A = 440
+    out     42h, al
+    shr     ax, 8
+    out     42h, al
+
+    in      al, 61h
+    or      al, 3
+    out     61h, al
+
+    ; delay
+    mov     ecx, _1G
+.delay:
+    inc     ecx
+    dec     ecx
+    dec     ecx
+    jnz     .delay
+
+    ; shut up speaker.
+    in      al, 61h
+    and     al, 11111100b
+    out     61h, al
+
+.done:
+    pop     ecx
+    pop     edx
+    pop     eax
+    ret
+ENDPROC     vmmR0TripleFaultHackBeep
+
+
+GLOBALNAME vmmR0TripleFaultHackEnd
+
+
+
+
+;;;
+;;;
+;;;
+;;;
+;;;
+
+
+
+BITS ARCH_BITS
+
+BEGINPROC vmmR0TripleFaultHackKbdWait
+        push    xAX
+
+.check_status:
+        in      al, 64h
+        test    al, 1                   ; KBD_STAT_OBF
+        jnz     .read_data_and_status
+        test    al, 2                   ; KBD_STAT_IBF
+        jnz     .check_status
+
+        pop     xAX
+        ret
+
+.read_data_and_status:
+        in      al, 60h
+        jmp     .check_status
+ENDPROC vmmR0TripleFaultHackKbdWait
+
+
+BEGINPROC vmmR0TripleFaultHackKbdRead
+        out     64h, al                 ; Write the command.
+
+.check_status:
+        in      al, 64h
+        test    al, 1                   ; KBD_STAT_OBF
+        jz      .check_status
+
+        in      al, 60h                 ; Read the data.
+        ret
+ENDPROC   vmmR0TripleFaultHackKbdRead
+
+
+BEGINPROC vmmR0TripleFaultHackKbdWrite
+        out     64h, al                 ; Write the command.
+        call    NAME(vmmR0TripleFaultHackKbdWait)
+
+        xchg    al, ah
+        out     60h, al                 ; Write the data.
+        call    NAME(vmmR0TripleFaultHackKbdWait)
+        xchg    al, ah
+
+        ret
+ENDPROC   vmmR0TripleFaultHackKbdWrite
+
+
+
+BEGINPROC vmmR0TripleFaultHackTripleFault
+    push    xAX
+    push    xSI
+
+    xor     eax, eax
+    push    xAX
+    push    xAX
+    push    xAX
+    push    xAX
+
+    COM_CHAR 'B'
+    COM_CHAR 'y'
+    COM_CHAR 'e'
+    COM_CHAR '!'
+    COM_CHAR 0ah
+    COM_CHAR 0dh
+
+
+    ;call    NAME(vmmR0TripleFaultHackBeep32)
+%if 1
+    lidt    [xSP]
+%elif 0
+    in      al, 92h
+    or      al, 1
+    out     92h, al
+    in      al, 92h
+    cli
+    hlt
+%else
+    mov     al, 0d0h                ; KBD_CCMD_READ_OUTPORT
+    call    NAME(vmmR0TripleFaultHackKbdRead)
+    mov     ah, 0feh
+    and     ah, al
+    mov     al, 0d1h                ; KBD_CCMD_WRITE_OUTPORT
+    call    NAME(vmmR0TripleFaultHackKbdWrite)
+    cli
+    hlt
+%endif
+    int3
+
+    pop     xAX
+    pop     xAX
+    pop     xAX
+    pop     xAX
+
+    pop     xSI
+    pop     xAX
+    ret
+ENDPROC   vmmR0TripleFaultHackTripleFault
+
diff --git a/src/VBox/VMM/VMMR3/APIC.cpp b/src/VBox/VMM/VMMR3/APIC.cpp
new file mode 100644
index 00000000..5a8dcdfc
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/APIC.cpp
@@ -0,0 +1,1564 @@
+/* $Id: APIC.cpp $ */
+/** @file
+ * APIC - Advanced Programmable Interrupt Controller.
+ */
+
+/*
+ * Copyright (C) 2016-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DEV_APIC
+#include <VBox/log.h>
+#include "APICInternal.h"
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/vm.h>
+
+
+#ifndef VBOX_DEVICE_STRUCT_TESTCASE
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The current APIC saved state version. */
+#define APIC_SAVED_STATE_VERSION                  5
+/** VirtualBox 5.1 beta2 - pre fActiveLintX. */
+#define APIC_SAVED_STATE_VERSION_VBOX_51_BETA2    4
+/** The saved state version used by VirtualBox 5.0 and
+ *  earlier.  */
+#define APIC_SAVED_STATE_VERSION_VBOX_50          3
+/** The saved state version used by VirtualBox v3 and earlier.
+ * This does not include the config.  */
+#define APIC_SAVED_STATE_VERSION_VBOX_30          2
+/** Some ancient version... */
+#define APIC_SAVED_STATE_VERSION_ANCIENT          1
+
+#ifdef VBOX_WITH_STATISTICS
+# define X2APIC_MSRRANGE(a_uFirst, a_uLast, a_szName) \
+    { (a_uFirst), (a_uLast), kCpumMsrRdFn_Ia32X2ApicN, kCpumMsrWrFn_Ia32X2ApicN, 0, 0, 0, 0, 0, a_szName, { 0 }, { 0 }, { 0 }, { 0 } }
+# define X2APIC_MSRRANGE_INVALID(a_uFirst, a_uLast, a_szName) \
+    { (a_uFirst), (a_uLast), kCpumMsrRdFn_WriteOnly, kCpumMsrWrFn_ReadOnly, 0, 0, 0, 0, UINT64_MAX /*fWrGpMask*/, a_szName, { 0 }, { 0 }, { 0 }, { 0 } }
+#else
+# define X2APIC_MSRRANGE(a_uFirst, a_uLast, a_szName) \
+    { (a_uFirst), (a_uLast), kCpumMsrRdFn_Ia32X2ApicN, kCpumMsrWrFn_Ia32X2ApicN, 0, 0, 0, 0, 0, a_szName }
+# define X2APIC_MSRRANGE_INVALID(a_uFirst, a_uLast, a_szName) \
+    { (a_uFirst), (a_uLast), kCpumMsrRdFn_WriteOnly, kCpumMsrWrFn_ReadOnly, 0, 0, 0, 0, UINT64_MAX /*fWrGpMask*/, a_szName }
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/**
+ * MSR range supported by the x2APIC.
+ * See Intel spec. 10.12.2 "x2APIC Register Availability".
+ */
+static CPUMMSRRANGE const g_MsrRange_x2Apic = X2APIC_MSRRANGE(MSR_IA32_X2APIC_START, MSR_IA32_X2APIC_END, "x2APIC range");
+static CPUMMSRRANGE const g_MsrRange_x2Apic_Invalid = X2APIC_MSRRANGE_INVALID(MSR_IA32_X2APIC_START, MSR_IA32_X2APIC_END, "x2APIC range invalid");
+#undef X2APIC_MSRRANGE
+#undef X2APIC_MSRRANGE_GP
+
+/** Saved state field descriptors for XAPICPAGE. */
+static const SSMFIELD g_aXApicPageFields[] =
+{
+    SSMFIELD_ENTRY( XAPICPAGE, id.u8ApicId),
+    SSMFIELD_ENTRY( XAPICPAGE, version.all.u32Version),
+    SSMFIELD_ENTRY( XAPICPAGE, tpr.u8Tpr),
+    SSMFIELD_ENTRY( XAPICPAGE, apr.u8Apr),
+    SSMFIELD_ENTRY( XAPICPAGE, ppr.u8Ppr),
+    SSMFIELD_ENTRY( XAPICPAGE, ldr.all.u32Ldr),
+    SSMFIELD_ENTRY( XAPICPAGE, dfr.all.u32Dfr),
+    SSMFIELD_ENTRY( XAPICPAGE, svr.all.u32Svr),
+    SSMFIELD_ENTRY( XAPICPAGE, isr.u[0].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, isr.u[1].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, isr.u[2].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, isr.u[3].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, isr.u[4].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, isr.u[5].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, isr.u[6].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, isr.u[7].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, tmr.u[0].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, tmr.u[1].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, tmr.u[2].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, tmr.u[3].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, tmr.u[4].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, tmr.u[5].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, tmr.u[6].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, tmr.u[7].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, irr.u[0].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, irr.u[1].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, irr.u[2].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, irr.u[3].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, irr.u[4].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, irr.u[5].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, irr.u[6].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, irr.u[7].u32Reg),
+    SSMFIELD_ENTRY( XAPICPAGE, esr.all.u32Errors),
+    SSMFIELD_ENTRY( XAPICPAGE, icr_lo.all.u32IcrLo),
+    SSMFIELD_ENTRY( XAPICPAGE, icr_hi.all.u32IcrHi),
+    SSMFIELD_ENTRY( XAPICPAGE, lvt_timer.all.u32LvtTimer),
+    SSMFIELD_ENTRY( XAPICPAGE, lvt_thermal.all.u32LvtThermal),
+    SSMFIELD_ENTRY( XAPICPAGE, lvt_perf.all.u32LvtPerf),
+    SSMFIELD_ENTRY( XAPICPAGE, lvt_lint0.all.u32LvtLint0),
+    SSMFIELD_ENTRY( XAPICPAGE, lvt_lint1.all.u32LvtLint1),
+    SSMFIELD_ENTRY( XAPICPAGE, lvt_error.all.u32LvtError),
+    SSMFIELD_ENTRY( XAPICPAGE, timer_icr.u32InitialCount),
+    SSMFIELD_ENTRY( XAPICPAGE, timer_ccr.u32CurrentCount),
+    SSMFIELD_ENTRY( XAPICPAGE, timer_dcr.all.u32DivideValue),
+    SSMFIELD_ENTRY_TERM()
+};
+
+/** Saved state field descriptors for X2APICPAGE. */
+static const SSMFIELD g_aX2ApicPageFields[] =
+{
+    SSMFIELD_ENTRY(X2APICPAGE, id.u32ApicId),
+    SSMFIELD_ENTRY(X2APICPAGE, version.all.u32Version),
+    SSMFIELD_ENTRY(X2APICPAGE, tpr.u8Tpr),
+    SSMFIELD_ENTRY(X2APICPAGE, ppr.u8Ppr),
+    SSMFIELD_ENTRY(X2APICPAGE, ldr.u32LogicalApicId),
+    SSMFIELD_ENTRY(X2APICPAGE, svr.all.u32Svr),
+    SSMFIELD_ENTRY(X2APICPAGE, isr.u[0].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, isr.u[1].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, isr.u[2].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, isr.u[3].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, isr.u[4].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, isr.u[5].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, isr.u[6].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, isr.u[7].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, tmr.u[0].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, tmr.u[1].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, tmr.u[2].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, tmr.u[3].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, tmr.u[4].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, tmr.u[5].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, tmr.u[6].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, tmr.u[7].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, irr.u[0].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, irr.u[1].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, irr.u[2].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, irr.u[3].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, irr.u[4].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, irr.u[5].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, irr.u[6].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, irr.u[7].u32Reg),
+    SSMFIELD_ENTRY(X2APICPAGE, esr.all.u32Errors),
+    SSMFIELD_ENTRY(X2APICPAGE, icr_lo.all.u32IcrLo),
+    SSMFIELD_ENTRY(X2APICPAGE, icr_hi.u32IcrHi),
+    SSMFIELD_ENTRY(X2APICPAGE, lvt_timer.all.u32LvtTimer),
+    SSMFIELD_ENTRY(X2APICPAGE, lvt_thermal.all.u32LvtThermal),
+    SSMFIELD_ENTRY(X2APICPAGE, lvt_perf.all.u32LvtPerf),
+    SSMFIELD_ENTRY(X2APICPAGE, lvt_lint0.all.u32LvtLint0),
+    SSMFIELD_ENTRY(X2APICPAGE, lvt_lint1.all.u32LvtLint1),
+    SSMFIELD_ENTRY(X2APICPAGE, lvt_error.all.u32LvtError),
+    SSMFIELD_ENTRY(X2APICPAGE, timer_icr.u32InitialCount),
+    SSMFIELD_ENTRY(X2APICPAGE, timer_ccr.u32CurrentCount),
+    SSMFIELD_ENTRY(X2APICPAGE, timer_dcr.all.u32DivideValue),
+    SSMFIELD_ENTRY_TERM()
+};
+
+
+/**
+ * Sets the CPUID feature bits for the APIC mode.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   enmMode         The APIC mode.
+ */
+static void apicR3SetCpuIdFeatureLevel(PVM pVM, PDMAPICMODE enmMode)
+{
+    switch (enmMode)
+    {
+        case PDMAPICMODE_NONE:
+            CPUMR3ClearGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_X2APIC);
+            CPUMR3ClearGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_APIC);
+            break;
+
+        case PDMAPICMODE_APIC:
+            CPUMR3ClearGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_X2APIC);
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_APIC);
+            break;
+
+        case PDMAPICMODE_X2APIC:
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_APIC);
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_X2APIC);
+            break;
+
+        default:
+            AssertMsgFailed(("Unknown/invalid APIC mode: %d\n", (int)enmMode));
+    }
+}
+
+
+/**
+ * Receives an INIT IPI.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMR3_INT_DECL(void) APICR3InitIpi(PVMCPU pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    LogFlow(("APIC%u: APICR3InitIpi\n", pVCpu->idCpu));
+    apicInitIpi(pVCpu);
+}
+
+
+/**
+ * Sets whether Hyper-V compatibility mode (MSR interface) is enabled or not.
+ *
+ * This mode is a hybrid of xAPIC and x2APIC modes, some caveats:
+ * 1. MSRs are used even ones that are missing (illegal) in x2APIC like DFR.
+ * 2. A single ICR is used by the guest to send IPIs rather than 2 ICR writes.
+ * 3. It is unclear what the behaviour will be when invalid bits are set,
+ *    currently we follow x2APIC behaviour of causing a \#GP.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   fHyperVCompatMode   Whether the compatibility mode is enabled.
+ */
+VMMR3_INT_DECL(void) APICR3HvSetCompatMode(PVM pVM, bool fHyperVCompatMode)
+{
+    Assert(pVM);
+    PAPIC pApic = VM_TO_APIC(pVM);
+    pApic->fHyperVCompatMode = fHyperVCompatMode;
+
+    if (fHyperVCompatMode)
+        LogRel(("APIC: Enabling Hyper-V x2APIC compatibility mode\n"));
+
+    int rc = CPUMR3MsrRangesInsert(pVM, &g_MsrRange_x2Apic);
+    AssertLogRelRC(rc);
+}
+
+
+/**
+ * Helper for dumping an APIC 256-bit sparse register.
+ *
+ * @param   pApicReg        The APIC 256-bit spare register.
+ * @param   pHlp            The debug output helper.
+ */
+static void apicR3DbgInfo256BitReg(volatile const XAPIC256BITREG *pApicReg, PCDBGFINFOHLP pHlp)
+{
+    ssize_t const  cFragments = RT_ELEMENTS(pApicReg->u);
+    unsigned const cBitsPerFragment = sizeof(pApicReg->u[0].u32Reg) * 8;
+    XAPIC256BITREG ApicReg;
+    RT_ZERO(ApicReg);
+
+    pHlp->pfnPrintf(pHlp, "    ");
+    for (ssize_t i = cFragments - 1; i >= 0; i--)
+    {
+        uint32_t const uFragment = pApicReg->u[i].u32Reg;
+        ApicReg.u[i].u32Reg = uFragment;
+        pHlp->pfnPrintf(pHlp, "%08x", uFragment);
+    }
+    pHlp->pfnPrintf(pHlp, "\n");
+
+    uint32_t cPending = 0;
+    pHlp->pfnPrintf(pHlp, "    Pending:");
+    for (ssize_t i = cFragments - 1; i >= 0; i--)
+    {
+        uint32_t uFragment = ApicReg.u[i].u32Reg;
+        if (uFragment)
+        {
+            do
+            {
+                unsigned idxSetBit = ASMBitLastSetU32(uFragment);
+                --idxSetBit;
+                ASMBitClear(&uFragment, idxSetBit);
+
+                idxSetBit += (i * cBitsPerFragment);
+                pHlp->pfnPrintf(pHlp, " %#02x", idxSetBit);
+                ++cPending;
+            } while (uFragment);
+        }
+    }
+    if (!cPending)
+        pHlp->pfnPrintf(pHlp, " None");
+    pHlp->pfnPrintf(pHlp, "\n");
+}
+
+
+/**
+ * Helper for dumping an APIC pending-interrupt bitmap.
+ *
+ * @param   pApicPib        The pending-interrupt bitmap.
+ * @param   pHlp            The debug output helper.
+ */
+static void apicR3DbgInfoPib(PCAPICPIB pApicPib, PCDBGFINFOHLP pHlp)
+{
+    /* Copy the pending-interrupt bitmap as an APIC 256-bit sparse register. */
+    XAPIC256BITREG ApicReg;
+    RT_ZERO(ApicReg);
+    ssize_t const cFragmentsDst = RT_ELEMENTS(ApicReg.u);
+    ssize_t const cFragmentsSrc = RT_ELEMENTS(pApicPib->au64VectorBitmap);
+    AssertCompile(RT_ELEMENTS(ApicReg.u) == 2 * RT_ELEMENTS(pApicPib->au64VectorBitmap));
+    for (ssize_t idxPib = cFragmentsSrc - 1, idxReg = cFragmentsDst - 1; idxPib >= 0; idxPib--, idxReg -= 2)
+    {
+        uint64_t const uFragment   = pApicPib->au64VectorBitmap[idxPib];
+        uint32_t const uFragmentLo = RT_LO_U32(uFragment);
+        uint32_t const uFragmentHi = RT_HI_U32(uFragment);
+        ApicReg.u[idxReg].u32Reg     = uFragmentHi;
+        ApicReg.u[idxReg - 1].u32Reg = uFragmentLo;
+    }
+
+    /* Dump it. */
+    apicR3DbgInfo256BitReg(&ApicReg, pHlp);
+}
+
+
+/**
+ * Dumps basic APIC state.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) apicR3Info(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    PCAPICCPU    pApicCpu    = VMCPU_TO_APICCPU(pVCpu);
+    PCXAPICPAGE  pXApicPage  = VMCPU_TO_CXAPICPAGE(pVCpu);
+    PCX2APICPAGE pX2ApicPage = VMCPU_TO_CX2APICPAGE(pVCpu);
+
+    uint64_t const uBaseMsr  = pApicCpu->uApicBaseMsr;
+    APICMODE const enmMode   = apicGetMode(uBaseMsr);
+    bool const   fX2ApicMode = XAPIC_IN_X2APIC_MODE(pVCpu);
+
+    pHlp->pfnPrintf(pHlp, "APIC%u:\n", pVCpu->idCpu);
+    pHlp->pfnPrintf(pHlp, "  APIC Base MSR                 = %#RX64 (Addr=%#RX64)\n", uBaseMsr,
+                    MSR_IA32_APICBASE_GET_ADDR(uBaseMsr));
+    pHlp->pfnPrintf(pHlp, "  Mode                          = %u (%s)\n", enmMode, apicGetModeName(enmMode));
+    if (fX2ApicMode)
+    {
+        pHlp->pfnPrintf(pHlp, "  APIC ID                       = %u (%#x)\n", pX2ApicPage->id.u32ApicId,
+                                                                              pX2ApicPage->id.u32ApicId);
+    }
+    else
+        pHlp->pfnPrintf(pHlp, "  APIC ID                       = %u (%#x)\n", pXApicPage->id.u8ApicId, pXApicPage->id.u8ApicId);
+    pHlp->pfnPrintf(pHlp, "  Version                       = %#x\n",      pXApicPage->version.all.u32Version);
+    pHlp->pfnPrintf(pHlp, "    APIC Version                  = %#x\n",      pXApicPage->version.u.u8Version);
+    pHlp->pfnPrintf(pHlp, "    Max LVT entry index (0..N)    = %u\n",       pXApicPage->version.u.u8MaxLvtEntry);
+    pHlp->pfnPrintf(pHlp, "    EOI Broadcast supression      = %RTbool\n",  pXApicPage->version.u.fEoiBroadcastSupression);
+    if (!fX2ApicMode)
+        pHlp->pfnPrintf(pHlp, "  APR                           = %u (%#x)\n", pXApicPage->apr.u8Apr, pXApicPage->apr.u8Apr);
+    pHlp->pfnPrintf(pHlp, "  TPR                           = %u (%#x)\n", pXApicPage->tpr.u8Tpr, pXApicPage->tpr.u8Tpr);
+    pHlp->pfnPrintf(pHlp, "    Task-priority class           = %#x\n",      XAPIC_TPR_GET_TP(pXApicPage->tpr.u8Tpr) >> 4);
+    pHlp->pfnPrintf(pHlp, "    Task-priority subclass        = %#x\n",      XAPIC_TPR_GET_TP_SUBCLASS(pXApicPage->tpr.u8Tpr));
+    pHlp->pfnPrintf(pHlp, "  PPR                           = %u (%#x)\n", pXApicPage->ppr.u8Ppr, pXApicPage->ppr.u8Ppr);
+    pHlp->pfnPrintf(pHlp, "    Processor-priority class      = %#x\n",      XAPIC_PPR_GET_PP(pXApicPage->ppr.u8Ppr) >> 4);
+    pHlp->pfnPrintf(pHlp, "    Processor-priority subclass   = %#x\n",      XAPIC_PPR_GET_PP_SUBCLASS(pXApicPage->ppr.u8Ppr));
+    if (!fX2ApicMode)
+        pHlp->pfnPrintf(pHlp, "  RRD                           = %u (%#x)\n", pXApicPage->rrd.u32Rrd, pXApicPage->rrd.u32Rrd);
+    pHlp->pfnPrintf(pHlp, "  LDR                           = %#x\n",      pXApicPage->ldr.all.u32Ldr);
+    pHlp->pfnPrintf(pHlp, "    Logical APIC ID               = %#x\n",      fX2ApicMode ? pX2ApicPage->ldr.u32LogicalApicId
+                                                                          : pXApicPage->ldr.u.u8LogicalApicId);
+    if (!fX2ApicMode)
+    {
+        pHlp->pfnPrintf(pHlp, "  DFR                           = %#x\n",  pXApicPage->dfr.all.u32Dfr);
+        pHlp->pfnPrintf(pHlp, "    Model                         = %#x (%s)\n", pXApicPage->dfr.u.u4Model,
+                        apicGetDestFormatName((XAPICDESTFORMAT)pXApicPage->dfr.u.u4Model));
+    }
+    pHlp->pfnPrintf(pHlp, "  SVR                           = %#x\n", pXApicPage->svr.all.u32Svr);
+    pHlp->pfnPrintf(pHlp, "    Vector                        = %u (%#x)\n", pXApicPage->svr.u.u8SpuriousVector,
+                                                                          pXApicPage->svr.u.u8SpuriousVector);
+    pHlp->pfnPrintf(pHlp, "    Software Enabled              = %RTbool\n",  RT_BOOL(pXApicPage->svr.u.fApicSoftwareEnable));
+    pHlp->pfnPrintf(pHlp, "    Supress EOI broadcast         = %RTbool\n",  RT_BOOL(pXApicPage->svr.u.fSupressEoiBroadcast));
+    pHlp->pfnPrintf(pHlp, "  ISR\n");
+    apicR3DbgInfo256BitReg(&pXApicPage->isr, pHlp);
+    pHlp->pfnPrintf(pHlp, "  TMR\n");
+    apicR3DbgInfo256BitReg(&pXApicPage->tmr, pHlp);
+    pHlp->pfnPrintf(pHlp, "  IRR\n");
+    apicR3DbgInfo256BitReg(&pXApicPage->irr, pHlp);
+    pHlp->pfnPrintf(pHlp, "  PIB\n");
+    apicR3DbgInfoPib((PCAPICPIB)pApicCpu->pvApicPibR3, pHlp);
+    pHlp->pfnPrintf(pHlp, "  Level PIB\n");
+    apicR3DbgInfoPib(&pApicCpu->ApicPibLevel, pHlp);
+    pHlp->pfnPrintf(pHlp, "  ESR Internal                  = %#x\n",      pApicCpu->uEsrInternal);
+    pHlp->pfnPrintf(pHlp, "  ESR                           = %#x\n",      pXApicPage->esr.all.u32Errors);
+    pHlp->pfnPrintf(pHlp, "    Redirectable IPI              = %RTbool\n",  pXApicPage->esr.u.fRedirectableIpi);
+    pHlp->pfnPrintf(pHlp, "    Send Illegal Vector           = %RTbool\n",  pXApicPage->esr.u.fSendIllegalVector);
+    pHlp->pfnPrintf(pHlp, "    Recv Illegal Vector           = %RTbool\n",  pXApicPage->esr.u.fRcvdIllegalVector);
+    pHlp->pfnPrintf(pHlp, "    Illegal Register Address      = %RTbool\n",  pXApicPage->esr.u.fIllegalRegAddr);
+    pHlp->pfnPrintf(pHlp, "  ICR Low                       = %#x\n",      pXApicPage->icr_lo.all.u32IcrLo);
+    pHlp->pfnPrintf(pHlp, "    Vector                        = %u (%#x)\n", pXApicPage->icr_lo.u.u8Vector,
+                                                                            pXApicPage->icr_lo.u.u8Vector);
+    pHlp->pfnPrintf(pHlp, "    Delivery Mode                 = %#x (%s)\n", pXApicPage->icr_lo.u.u3DeliveryMode,
+                    apicGetDeliveryModeName((XAPICDELIVERYMODE)pXApicPage->icr_lo.u.u3DeliveryMode));
+    pHlp->pfnPrintf(pHlp, "    Destination Mode              = %#x (%s)\n", pXApicPage->icr_lo.u.u1DestMode,
+                    apicGetDestModeName((XAPICDESTMODE)pXApicPage->icr_lo.u.u1DestMode));
+    if (!fX2ApicMode)
+        pHlp->pfnPrintf(pHlp, "    Delivery Status               = %u\n",       pXApicPage->icr_lo.u.u1DeliveryStatus);
+    pHlp->pfnPrintf(pHlp, "    Level                         = %u\n",       pXApicPage->icr_lo.u.u1Level);
+    pHlp->pfnPrintf(pHlp, "    Trigger Mode                  = %u (%s)\n",  pXApicPage->icr_lo.u.u1TriggerMode,
+                    apicGetTriggerModeName((XAPICTRIGGERMODE)pXApicPage->icr_lo.u.u1TriggerMode));
+    pHlp->pfnPrintf(pHlp, "    Destination shorthand         = %#x (%s)\n", pXApicPage->icr_lo.u.u2DestShorthand,
+                    apicGetDestShorthandName((XAPICDESTSHORTHAND)pXApicPage->icr_lo.u.u2DestShorthand));
+    pHlp->pfnPrintf(pHlp, "  ICR High                      = %#x\n",      pXApicPage->icr_hi.all.u32IcrHi);
+    pHlp->pfnPrintf(pHlp, "    Destination field/mask        = %#x\n",      fX2ApicMode ? pX2ApicPage->icr_hi.u32IcrHi
+                                                                          : pXApicPage->icr_hi.u.u8Dest);
+}
+
+
+/**
+ * Helper for dumping the LVT timer.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   pHlp    The debug output helper.
+ */
+static void apicR3InfoLvtTimer(PVMCPU pVCpu, PCDBGFINFOHLP pHlp)
+{
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    uint32_t const uLvtTimer = pXApicPage->lvt_timer.all.u32LvtTimer;
+    pHlp->pfnPrintf(pHlp, "LVT Timer          = %#RX32\n",   uLvtTimer);
+    pHlp->pfnPrintf(pHlp, "  Vector             = %u (%#x)\n", pXApicPage->lvt_timer.u.u8Vector, pXApicPage->lvt_timer.u.u8Vector);
+    pHlp->pfnPrintf(pHlp, "  Delivery status    = %u\n",       pXApicPage->lvt_timer.u.u1DeliveryStatus);
+    pHlp->pfnPrintf(pHlp, "  Masked             = %RTbool\n",  XAPIC_LVT_IS_MASKED(uLvtTimer));
+    pHlp->pfnPrintf(pHlp, "  Timer Mode         = %#x (%s)\n", pXApicPage->lvt_timer.u.u2TimerMode,
+                    apicGetTimerModeName((XAPICTIMERMODE)pXApicPage->lvt_timer.u.u2TimerMode));
+}
+
+
+/**
+ * Dumps APIC Local Vector Table (LVT) information.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) apicR3InfoLvt(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+
+    /*
+     * Delivery modes available in the LVT entries. They're different (more reserved stuff) from the
+     * ICR delivery modes and hence we don't use apicGetDeliveryMode but mostly because we want small,
+     * fixed-length strings to fit our formatting needs here.
+     */
+    static const char * const s_apszLvtDeliveryModes[] =
+    {
+        "Fixed ",
+        "Rsvd  ",
+        "SMI   ",
+        "Rsvd  ",
+        "NMI   ",
+        "INIT  ",
+        "Rsvd  ",
+        "ExtINT"
+    };
+    /* Delivery Status. */
+    static const char * const s_apszLvtDeliveryStatus[] =
+    {
+        "Idle",
+        "Pend"
+    };
+    const char *pszNotApplicable = "";
+
+    pHlp->pfnPrintf(pHlp, "VCPU[%u] APIC Local Vector Table (LVT):\n", pVCpu->idCpu);
+    pHlp->pfnPrintf(pHlp, "lvt     timermode  mask  trigger  rirr  polarity  dlvr_st  dlvr_mode   vector\n");
+    /* Timer. */
+    {
+        /* Timer modes. */
+        static const char * const s_apszLvtTimerModes[] =
+        {
+            "One-shot ",
+            "Periodic ",
+            "TSC-dline"
+        };
+        const uint32_t       uLvtTimer         = pXApicPage->lvt_timer.all.u32LvtTimer;
+        const XAPICTIMERMODE enmTimerMode      = XAPIC_LVT_GET_TIMER_MODE(uLvtTimer);
+        const char          *pszTimerMode      = s_apszLvtTimerModes[enmTimerMode];
+        const uint8_t        uMask             = XAPIC_LVT_IS_MASKED(uLvtTimer);
+        const uint8_t        uDeliveryStatus   = uLvtTimer & XAPIC_LVT_DELIVERY_STATUS;
+        const char          *pszDeliveryStatus = s_apszLvtDeliveryStatus[uDeliveryStatus];
+        const uint8_t        uVector           = XAPIC_LVT_GET_VECTOR(uLvtTimer);
+
+        pHlp->pfnPrintf(pHlp, "%-7s  %9s  %u     %5s     %1s   %8s    %4s     %6s    %3u (%#x)\n",
+                        "Timer",
+                        pszTimerMode,
+                        uMask,
+                        pszNotApplicable, /* TriggerMode */
+                        pszNotApplicable, /* Remote IRR */
+                        pszNotApplicable, /* Polarity */
+                        pszDeliveryStatus,
+                        pszNotApplicable, /* Delivery Mode */
+                        uVector,
+                        uVector);
+    }
+
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+    /* Thermal sensor. */
+    {
+        uint32_t const uLvtThermal = pXApicPage->lvt_thermal.all.u32LvtThermal;
+        const uint8_t           uMask             = XAPIC_LVT_IS_MASKED(uLvtThermal);
+        const uint8_t           uDeliveryStatus   = uLvtThermal & XAPIC_LVT_DELIVERY_STATUS;
+        const char             *pszDeliveryStatus = s_apszLvtDeliveryStatus[uDeliveryStatus];
+        const XAPICDELIVERYMODE enmDeliveryMode   = XAPIC_LVT_GET_DELIVERY_MODE(uLvtThermal);
+        const char             *pszDeliveryMode   = s_apszLvtDeliveryModes[enmDeliveryMode];
+        const uint8_t           uVector           = XAPIC_LVT_GET_VECTOR(uLvtThermal);
+
+        pHlp->pfnPrintf(pHlp, "%-7s  %9s  %u     %5s     %1s   %8s    %4s     %6s    %3u (%#x)\n",
+                        "Thermal",
+                        pszNotApplicable, /* Timer mode */
+                        uMask,
+                        pszNotApplicable, /* TriggerMode */
+                        pszNotApplicable, /* Remote IRR */
+                        pszNotApplicable, /* Polarity */
+                        pszDeliveryStatus,
+                        pszDeliveryMode,
+                        uVector,
+                        uVector);
+    }
+#endif
+
+    /* Performance Monitor Counters. */
+    {
+        uint32_t const uLvtPerf = pXApicPage->lvt_thermal.all.u32LvtThermal;
+        const uint8_t           uMask             = XAPIC_LVT_IS_MASKED(uLvtPerf);
+        const uint8_t           uDeliveryStatus   = uLvtPerf & XAPIC_LVT_DELIVERY_STATUS;
+        const char             *pszDeliveryStatus = s_apszLvtDeliveryStatus[uDeliveryStatus];
+        const XAPICDELIVERYMODE enmDeliveryMode   = XAPIC_LVT_GET_DELIVERY_MODE(uLvtPerf);
+        const char             *pszDeliveryMode   = s_apszLvtDeliveryModes[enmDeliveryMode];
+        const uint8_t           uVector           = XAPIC_LVT_GET_VECTOR(uLvtPerf);
+
+        pHlp->pfnPrintf(pHlp, "%-7s  %9s  %u     %5s     %1s   %8s    %4s     %6s    %3u (%#x)\n",
+                        "Perf",
+                        pszNotApplicable, /* Timer mode */
+                        uMask,
+                        pszNotApplicable, /* TriggerMode */
+                        pszNotApplicable, /* Remote IRR */
+                        pszNotApplicable, /* Polarity */
+                        pszDeliveryStatus,
+                        pszDeliveryMode,
+                        uVector,
+                        uVector);
+    }
+
+    /* LINT0, LINT1. */
+    {
+        /* LINTx name. */
+        static const char * const s_apszLvtLint[] =
+        {
+            "LINT0",
+            "LINT1"
+        };
+        /* Trigger mode. */
+        static const char * const s_apszLvtTriggerModes[] =
+        {
+            "Edge ",
+            "Level"
+        };
+        /* Polarity. */
+        static const char * const s_apszLvtPolarity[] =
+        {
+            "ActiveHi",
+            "ActiveLo"
+        };
+
+        uint32_t aLvtLint[2];
+        aLvtLint[0] = pXApicPage->lvt_lint0.all.u32LvtLint0;
+        aLvtLint[1] = pXApicPage->lvt_lint1.all.u32LvtLint1;
+        for (size_t i = 0; i < RT_ELEMENTS(aLvtLint); i++)
+        {
+            uint32_t const uLvtLint = aLvtLint[i];
+            const char             *pszLint           = s_apszLvtLint[i];
+            const uint8_t           uMask             = XAPIC_LVT_IS_MASKED(uLvtLint);
+            const XAPICTRIGGERMODE  enmTriggerMode    = XAPIC_LVT_GET_TRIGGER_MODE(uLvtLint);
+            const char             *pszTriggerMode    = s_apszLvtTriggerModes[enmTriggerMode];
+            const uint8_t           uRemoteIrr        = XAPIC_LVT_GET_REMOTE_IRR(uLvtLint);
+            const uint8_t           uPolarity         = XAPIC_LVT_GET_POLARITY(uLvtLint);
+            const char             *pszPolarity       = s_apszLvtPolarity[uPolarity];
+            const uint8_t           uDeliveryStatus   = uLvtLint & XAPIC_LVT_DELIVERY_STATUS;
+            const char             *pszDeliveryStatus = s_apszLvtDeliveryStatus[uDeliveryStatus];
+            const XAPICDELIVERYMODE enmDeliveryMode   = XAPIC_LVT_GET_DELIVERY_MODE(uLvtLint);
+            const char             *pszDeliveryMode   = s_apszLvtDeliveryModes[enmDeliveryMode];
+            const uint8_t           uVector           = XAPIC_LVT_GET_VECTOR(uLvtLint);
+
+            pHlp->pfnPrintf(pHlp, "%-7s  %9s  %u     %5s     %u   %8s    %4s     %6s    %3u (%#x)\n",
+                            pszLint,
+                            pszNotApplicable, /* Timer mode */
+                            uMask,
+                            pszTriggerMode,
+                            uRemoteIrr,
+                            pszPolarity,
+                            pszDeliveryStatus,
+                            pszDeliveryMode,
+                            uVector,
+                            uVector);
+        }
+    }
+
+    /* Error. */
+    {
+        uint32_t const uLvtError = pXApicPage->lvt_thermal.all.u32LvtThermal;
+        const uint8_t           uMask             = XAPIC_LVT_IS_MASKED(uLvtError);
+        const uint8_t           uDeliveryStatus   = uLvtError & XAPIC_LVT_DELIVERY_STATUS;
+        const char             *pszDeliveryStatus = s_apszLvtDeliveryStatus[uDeliveryStatus];
+        const XAPICDELIVERYMODE enmDeliveryMode   = XAPIC_LVT_GET_DELIVERY_MODE(uLvtError);
+        const char             *pszDeliveryMode   = s_apszLvtDeliveryModes[enmDeliveryMode];
+        const uint8_t           uVector           = XAPIC_LVT_GET_VECTOR(uLvtError);
+
+        pHlp->pfnPrintf(pHlp, "%-7s  %9s  %u     %5s     %1s   %8s    %4s     %6s    %3u (%#x)\n",
+                        "Error",
+                        pszNotApplicable, /* Timer mode */
+                        uMask,
+                        pszNotApplicable, /* TriggerMode */
+                        pszNotApplicable, /* Remote IRR */
+                        pszNotApplicable, /* Polarity */
+                        pszDeliveryStatus,
+                        pszDeliveryMode,
+                        uVector,
+                        uVector);
+    }
+}
+
+
+/**
+ * Dumps the APIC timer information.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) apicR3InfoTimer(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    PCAPICCPU   pApicCpu   = VMCPU_TO_APICCPU(pVCpu);
+
+    pHlp->pfnPrintf(pHlp, "VCPU[%u] Local APIC timer:\n", pVCpu->idCpu);
+    pHlp->pfnPrintf(pHlp, "  ICR              = %#RX32\n", pXApicPage->timer_icr.u32InitialCount);
+    pHlp->pfnPrintf(pHlp, "  CCR              = %#RX32\n", pXApicPage->timer_ccr.u32CurrentCount);
+    pHlp->pfnPrintf(pHlp, "  DCR              = %#RX32\n", pXApicPage->timer_dcr.all.u32DivideValue);
+    pHlp->pfnPrintf(pHlp, "    Timer shift    = %#x\n",    apicGetTimerShift(pXApicPage));
+    pHlp->pfnPrintf(pHlp, "  Timer initial TS = %#RU64\n", pApicCpu->u64TimerInitial);
+    apicR3InfoLvtTimer(pVCpu, pHlp);
+}
+
+
+#ifdef APIC_FUZZY_SSM_COMPAT_TEST
+
+/**
+ * Reads a 32-bit register at a specified offset.
+ *
+ * @returns The value at the specified offset.
+ * @param   pXApicPage      The xAPIC page.
+ * @param   offReg          The offset of the register being read.
+ *
+ * @remarks Duplicate of apicReadRaw32()!
+ */
+static uint32_t apicR3ReadRawR32(PCXAPICPAGE pXApicPage, uint16_t offReg)
+{
+    Assert(offReg < sizeof(*pXApicPage) - sizeof(uint32_t));
+    uint8_t  const *pbXApic =  (const uint8_t *)pXApicPage;
+    uint32_t const  uValue  = *(const uint32_t *)(pbXApic + offReg);
+    return uValue;
+}
+
+
+/**
+ * Helper for dumping per-VCPU APIC state to the release logger.
+ *
+ * This is primarily concerned about the APIC state relevant for saved-states.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszPrefix   A caller supplied prefix before dumping the state.
+ * @param   uVersion    Data layout version.
+ */
+static void apicR3DumpState(PVMCPU pVCpu, const char *pszPrefix, uint32_t uVersion)
+{
+    PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+
+    LogRel(("APIC%u: %s (version %u):\n", pVCpu->idCpu, pszPrefix, uVersion));
+
+    switch (uVersion)
+    {
+        case APIC_SAVED_STATE_VERSION:
+        case APIC_SAVED_STATE_VERSION_VBOX_51_BETA2:
+        {
+            /* The auxiliary state. */
+            LogRel(("APIC%u: uApicBaseMsr             = %#RX64\n", pVCpu->idCpu, pApicCpu->uApicBaseMsr));
+            LogRel(("APIC%u: uEsrInternal             = %#RX64\n", pVCpu->idCpu, pApicCpu->uEsrInternal));
+
+            /* The timer. */
+            LogRel(("APIC%u: u64TimerInitial          = %#RU64\n", pVCpu->idCpu, pApicCpu->u64TimerInitial));
+            LogRel(("APIC%u: uHintedTimerInitialCount = %#RU64\n", pVCpu->idCpu, pApicCpu->uHintedTimerInitialCount));
+            LogRel(("APIC%u: uHintedTimerShift        = %#RU64\n", pVCpu->idCpu, pApicCpu->uHintedTimerShift));
+
+            PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+            LogRel(("APIC%u: uTimerICR                = %#RX32\n", pVCpu->idCpu, pXApicPage->timer_icr.u32InitialCount));
+            LogRel(("APIC%u: uTimerCCR                = %#RX32\n", pVCpu->idCpu, pXApicPage->timer_ccr.u32CurrentCount));
+
+            /* The PIBs. */
+            LogRel(("APIC%u: Edge PIB : %.*Rhxs\n", pVCpu->idCpu, sizeof(APICPIB), pApicCpu->pvApicPibR3));
+            LogRel(("APIC%u: Level PIB: %.*Rhxs\n", pVCpu->idCpu, sizeof(APICPIB), &pApicCpu->ApicPibLevel));
+
+            /* The LINT0, LINT1 interrupt line active states. */
+            LogRel(("APIC%u: fActiveLint0             = %RTbool\n", pVCpu->idCpu, pApicCpu->fActiveLint0));
+            LogRel(("APIC%u: fActiveLint1             = %RTbool\n", pVCpu->idCpu, pApicCpu->fActiveLint1));
+
+            /* The APIC page. */
+            LogRel(("APIC%u: APIC page: %.*Rhxs\n", pVCpu->idCpu, sizeof(XAPICPAGE), pApicCpu->pvApicPageR3));
+            break;
+        }
+
+        case APIC_SAVED_STATE_VERSION_VBOX_50:
+        case APIC_SAVED_STATE_VERSION_VBOX_30:
+        case APIC_SAVED_STATE_VERSION_ANCIENT:
+        {
+            PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+            LogRel(("APIC%u: uApicBaseMsr             = %#RX32\n", pVCpu->idCpu, RT_LO_U32(pApicCpu->uApicBaseMsr)));
+            LogRel(("APIC%u: uId                      = %#RX32\n", pVCpu->idCpu, pXApicPage->id.u8ApicId));
+            LogRel(("APIC%u: uPhysId                  = N/A\n",    pVCpu->idCpu));
+            LogRel(("APIC%u: uArbId                   = N/A\n",    pVCpu->idCpu));
+            LogRel(("APIC%u: uTpr                     = %#RX32\n", pVCpu->idCpu, pXApicPage->tpr.u8Tpr));
+            LogRel(("APIC%u: uSvr                     = %#RX32\n", pVCpu->idCpu, pXApicPage->svr.all.u32Svr));
+            LogRel(("APIC%u: uLdr                     = %#x\n",    pVCpu->idCpu, pXApicPage->ldr.all.u32Ldr));
+            LogRel(("APIC%u: uDfr                     = %#x\n",    pVCpu->idCpu, pXApicPage->dfr.all.u32Dfr));
+
+            for (size_t i = 0; i < 8; i++)
+            {
+                LogRel(("APIC%u: Isr[%u].u32Reg            = %#RX32\n", pVCpu->idCpu, i, pXApicPage->isr.u[i].u32Reg));
+                LogRel(("APIC%u: Tmr[%u].u32Reg            = %#RX32\n", pVCpu->idCpu, i, pXApicPage->tmr.u[i].u32Reg));
+                LogRel(("APIC%u: Irr[%u].u32Reg            = %#RX32\n", pVCpu->idCpu, i, pXApicPage->irr.u[i].u32Reg));
+            }
+
+            for (size_t i = 0; i < XAPIC_MAX_LVT_ENTRIES_P4; i++)
+            {
+                uint16_t const offReg = XAPIC_OFF_LVT_START + (i << 4);
+                LogRel(("APIC%u: Lvt[%u].u32Reg            = %#RX32\n", pVCpu->idCpu, i, apicR3ReadRawR32(pXApicPage, offReg)));
+            }
+
+            LogRel(("APIC%u: uEsr                     = %#RX32\n", pVCpu->idCpu, pXApicPage->esr.all.u32Errors));
+            LogRel(("APIC%u: uIcr_Lo                  = %#RX32\n", pVCpu->idCpu, pXApicPage->icr_lo.all.u32IcrLo));
+            LogRel(("APIC%u: uIcr_Hi                  = %#RX32\n", pVCpu->idCpu, pXApicPage->icr_hi.all.u32IcrHi));
+            LogRel(("APIC%u: uTimerDcr                = %#RX32\n", pVCpu->idCpu, pXApicPage->timer_dcr.all.u32DivideValue));
+            LogRel(("APIC%u: uCountShift              = %#RX32\n", pVCpu->idCpu, apicGetTimerShift(pXApicPage)));
+            LogRel(("APIC%u: uInitialCount            = %#RX32\n", pVCpu->idCpu, pXApicPage->timer_icr.u32InitialCount));
+            LogRel(("APIC%u: u64InitialCountLoadTime  = %#RX64\n", pVCpu->idCpu, pApicCpu->u64TimerInitial));
+            LogRel(("APIC%u: u64NextTime / TimerCCR   = %#RX64\n", pVCpu->idCpu, pXApicPage->timer_ccr.u32CurrentCount));
+            break;
+        }
+
+        default:
+        {
+            LogRel(("APIC: apicR3DumpState: Invalid/unrecognized saved-state version %u (%#x)\n", uVersion, uVersion));
+            break;
+        }
+    }
+}
+
+#endif  /* APIC_FUZZY_SSM_COMPAT_TEST */
+
+/**
+ * Worker for saving per-VM APIC data.
+ *
+ * @returns VBox status code.
+ * @param   pDevIns     The device instance.
+ * @param   pVM     The cross context VM structure.
+ * @param   pSSM    The SSM handle.
+ */
+static int apicR3SaveVMData(PPDMDEVINS pDevIns, PVM pVM, PSSMHANDLE pSSM)
+{
+    PCPDMDEVHLPR3   pHlp  = pDevIns->pHlpR3;
+    PAPIC           pApic = VM_TO_APIC(pVM);
+    pHlp->pfnSSMPutU32(pSSM,  pVM->cCpus);
+    pHlp->pfnSSMPutBool(pSSM, pApic->fIoApicPresent);
+    return pHlp->pfnSSMPutU32(pSSM, pApic->enmMaxMode);
+}
+
+
+/**
+ * Worker for loading per-VM APIC data.
+ *
+ * @returns VBox status code.
+ * @param   pDevIns     The device instance.
+ * @param   pVM     The cross context VM structure.
+ * @param   pSSM    The SSM handle.
+ */
+static int apicR3LoadVMData(PPDMDEVINS pDevIns, PVM pVM, PSSMHANDLE pSSM)
+{
+    PAPIC           pApic = VM_TO_APIC(pVM);
+    PCPDMDEVHLPR3   pHlp  = pDevIns->pHlpR3;
+
+    /* Load and verify number of CPUs. */
+    uint32_t cCpus;
+    int rc = pHlp->pfnSSMGetU32(pSSM, &cCpus);
+    AssertRCReturn(rc, rc);
+    if (cCpus != pVM->cCpus)
+        return pHlp->pfnSSMSetCfgError(pSSM, RT_SRC_POS, N_("Config mismatch - cCpus: saved=%u config=%u"), cCpus, pVM->cCpus);
+
+    /* Load and verify I/O APIC presence. */
+    bool fIoApicPresent;
+    rc = pHlp->pfnSSMGetBool(pSSM, &fIoApicPresent);
+    AssertRCReturn(rc, rc);
+    if (fIoApicPresent != pApic->fIoApicPresent)
+        return pHlp->pfnSSMSetCfgError(pSSM, RT_SRC_POS, N_("Config mismatch - fIoApicPresent: saved=%RTbool config=%RTbool"),
+                                       fIoApicPresent, pApic->fIoApicPresent);
+
+    /* Load and verify configured max APIC mode. */
+    uint32_t uSavedMaxApicMode;
+    rc = pHlp->pfnSSMGetU32(pSSM, &uSavedMaxApicMode);
+    AssertRCReturn(rc, rc);
+    if (uSavedMaxApicMode != (uint32_t)pApic->enmMaxMode)
+        return pHlp->pfnSSMSetCfgError(pSSM, RT_SRC_POS, N_("Config mismatch - uApicMode: saved=%u config=%u"),
+                                       uSavedMaxApicMode, pApic->enmMaxMode);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for loading per-VCPU APIC data for legacy (old) saved-states.
+ *
+ * @returns VBox status code.
+ * @param   pDevIns     The device instance.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pSSM        The SSM handle.
+ * @param   uVersion    Data layout version.
+ */
+static int apicR3LoadLegacyVCpuData(PPDMDEVINS pDevIns, PVMCPU pVCpu, PSSMHANDLE pSSM, uint32_t uVersion)
+{
+    AssertReturn(uVersion <= APIC_SAVED_STATE_VERSION_VBOX_50, VERR_NOT_SUPPORTED);
+
+    PCPDMDEVHLPR3   pHlp       = pDevIns->pHlpR3;
+    PAPICCPU        pApicCpu   = VMCPU_TO_APICCPU(pVCpu);
+    PXAPICPAGE      pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+
+    uint32_t uApicBaseLo;
+    int rc = pHlp->pfnSSMGetU32(pSSM, &uApicBaseLo);
+    AssertRCReturn(rc, rc);
+    pApicCpu->uApicBaseMsr = uApicBaseLo;
+    Log2(("APIC%u: apicR3LoadLegacyVCpuData: uApicBaseMsr=%#RX64\n", pVCpu->idCpu, pApicCpu->uApicBaseMsr));
+
+    switch (uVersion)
+    {
+        case APIC_SAVED_STATE_VERSION_VBOX_50:
+        case APIC_SAVED_STATE_VERSION_VBOX_30:
+        {
+            uint32_t uApicId, uPhysApicId, uArbId;
+            pHlp->pfnSSMGetU32(pSSM, &uApicId);      pXApicPage->id.u8ApicId = uApicId;
+            pHlp->pfnSSMGetU32(pSSM, &uPhysApicId);  NOREF(uPhysApicId); /* PhysId == pVCpu->idCpu */
+            pHlp->pfnSSMGetU32(pSSM, &uArbId);       NOREF(uArbId);      /* ArbID is & was unused. */
+            break;
+        }
+
+        case APIC_SAVED_STATE_VERSION_ANCIENT:
+        {
+            uint8_t uPhysApicId;
+            pHlp->pfnSSMGetU8(pSSM, &pXApicPage->id.u8ApicId);
+            pHlp->pfnSSMGetU8(pSSM, &uPhysApicId);   NOREF(uPhysApicId); /* PhysId == pVCpu->idCpu */
+            break;
+        }
+
+        default:
+            return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    uint32_t u32Tpr;
+    pHlp->pfnSSMGetU32(pSSM, &u32Tpr);
+    pXApicPage->tpr.u8Tpr = u32Tpr & XAPIC_TPR_VALID;
+
+    pHlp->pfnSSMGetU32(pSSM, &pXApicPage->svr.all.u32Svr);
+    pHlp->pfnSSMGetU8(pSSM,  &pXApicPage->ldr.u.u8LogicalApicId);
+
+    uint8_t uDfr;
+    pHlp->pfnSSMGetU8(pSSM,  &uDfr);
+    pXApicPage->dfr.u.u4Model = uDfr >> 4;
+
+    AssertCompile(RT_ELEMENTS(pXApicPage->isr.u) == 8);
+    AssertCompile(RT_ELEMENTS(pXApicPage->tmr.u) == 8);
+    AssertCompile(RT_ELEMENTS(pXApicPage->irr.u) == 8);
+    for (size_t i = 0; i < 8; i++)
+    {
+        pHlp->pfnSSMGetU32(pSSM, &pXApicPage->isr.u[i].u32Reg);
+        pHlp->pfnSSMGetU32(pSSM, &pXApicPage->tmr.u[i].u32Reg);
+        pHlp->pfnSSMGetU32(pSSM, &pXApicPage->irr.u[i].u32Reg);
+    }
+
+    pHlp->pfnSSMGetU32(pSSM, &pXApicPage->lvt_timer.all.u32LvtTimer);
+    pHlp->pfnSSMGetU32(pSSM, &pXApicPage->lvt_thermal.all.u32LvtThermal);
+    pHlp->pfnSSMGetU32(pSSM, &pXApicPage->lvt_perf.all.u32LvtPerf);
+    pHlp->pfnSSMGetU32(pSSM, &pXApicPage->lvt_lint0.all.u32LvtLint0);
+    pHlp->pfnSSMGetU32(pSSM, &pXApicPage->lvt_lint1.all.u32LvtLint1);
+    pHlp->pfnSSMGetU32(pSSM, &pXApicPage->lvt_error.all.u32LvtError);
+
+    pHlp->pfnSSMGetU32(pSSM, &pXApicPage->esr.all.u32Errors);
+    pHlp->pfnSSMGetU32(pSSM, &pXApicPage->icr_lo.all.u32IcrLo);
+    pHlp->pfnSSMGetU32(pSSM, &pXApicPage->icr_hi.all.u32IcrHi);
+
+    uint32_t u32TimerShift;
+    pHlp->pfnSSMGetU32(pSSM, &pXApicPage->timer_dcr.all.u32DivideValue);
+    pHlp->pfnSSMGetU32(pSSM, &u32TimerShift);
+    /*
+     * Old implementation may have left the timer shift uninitialized until
+     * the timer configuration register was written. Unfortunately zero is
+     * also a valid timer shift value, so we're just going to ignore it
+     * completely. The shift count can always be derived from the DCR.
+     * See @bugref{8245#c98}.
+     */
+    uint8_t const uTimerShift = apicGetTimerShift(pXApicPage);
+
+    pHlp->pfnSSMGetU32(pSSM, &pXApicPage->timer_icr.u32InitialCount);
+    pHlp->pfnSSMGetU64(pSSM, &pApicCpu->u64TimerInitial);
+    uint64_t uNextTS;
+    rc = pHlp->pfnSSMGetU64(pSSM, &uNextTS);       AssertRCReturn(rc, rc);
+    if (uNextTS >= pApicCpu->u64TimerInitial + ((pXApicPage->timer_icr.u32InitialCount + 1) << uTimerShift))
+        pXApicPage->timer_ccr.u32CurrentCount = pXApicPage->timer_icr.u32InitialCount;
+
+    rc = PDMDevHlpTimerLoad(pDevIns, pApicCpu->hTimer, pSSM);
+    AssertRCReturn(rc, rc);
+    Assert(pApicCpu->uHintedTimerInitialCount == 0);
+    Assert(pApicCpu->uHintedTimerShift == 0);
+    if (PDMDevHlpTimerIsActive(pDevIns, pApicCpu->hTimer))
+    {
+        uint32_t const uInitialCount = pXApicPage->timer_icr.u32InitialCount;
+        apicHintTimerFreq(pDevIns, pApicCpu, uInitialCount, uTimerShift);
+    }
+
+    return rc;
+}
+
+
+/**
+ * @copydoc FNSSMDEVSAVEEXEC
+ */
+static DECLCALLBACK(int) apicR3SaveExec(PPDMDEVINS pDevIns, PSSMHANDLE pSSM)
+{
+    PVM             pVM  = PDMDevHlpGetVM(pDevIns);
+    PCPDMDEVHLPR3   pHlp = pDevIns->pHlpR3;
+
+    AssertReturn(pVM, VERR_INVALID_VM_HANDLE);
+
+    LogFlow(("APIC: apicR3SaveExec\n"));
+
+    /* Save per-VM data. */
+    int rc = apicR3SaveVMData(pDevIns, pVM, pSSM);
+    AssertRCReturn(rc, rc);
+
+    /* Save per-VCPU data.*/
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+
+        /* Update interrupts from the pending-interrupts bitmaps to the IRR. */
+        APICUpdatePendingInterrupts(pVCpu);
+
+        /* Save the auxiliary data. */
+        pHlp->pfnSSMPutU64(pSSM, pApicCpu->uApicBaseMsr);
+        pHlp->pfnSSMPutU32(pSSM, pApicCpu->uEsrInternal);
+
+        /* Save the APIC page. */
+        if (XAPIC_IN_X2APIC_MODE(pVCpu))
+            pHlp->pfnSSMPutStruct(pSSM, (const void *)pApicCpu->pvApicPageR3, &g_aX2ApicPageFields[0]);
+        else
+            pHlp->pfnSSMPutStruct(pSSM, (const void *)pApicCpu->pvApicPageR3, &g_aXApicPageFields[0]);
+
+        /* Save the timer. */
+        pHlp->pfnSSMPutU64(pSSM, pApicCpu->u64TimerInitial);
+        PDMDevHlpTimerSave(pDevIns, pApicCpu->hTimer, pSSM);
+
+        /* Save the LINT0, LINT1 interrupt line states. */
+        pHlp->pfnSSMPutBool(pSSM, pApicCpu->fActiveLint0);
+        pHlp->pfnSSMPutBool(pSSM, pApicCpu->fActiveLint1);
+
+#if defined(APIC_FUZZY_SSM_COMPAT_TEST) || defined(DEBUG_ramshankar)
+        apicR3DumpState(pVCpu, "Saved state", APIC_SAVED_STATE_VERSION);
+#endif
+    }
+
+#ifdef APIC_FUZZY_SSM_COMPAT_TEST
+    /* The state is fuzzy, don't even bother trying to load the guest. */
+    return VERR_INVALID_STATE;
+#else
+    return rc;
+#endif
+}
+
+
+/**
+ * @copydoc FNSSMDEVLOADEXEC
+ */
+static DECLCALLBACK(int) apicR3LoadExec(PPDMDEVINS pDevIns, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    PVM             pVM  = PDMDevHlpGetVM(pDevIns);
+    PCPDMDEVHLPR3   pHlp = pDevIns->pHlpR3;
+
+    AssertReturn(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(uPass == SSM_PASS_FINAL, VERR_WRONG_ORDER);
+
+    LogFlow(("APIC: apicR3LoadExec: uVersion=%u uPass=%#x\n", uVersion, uPass));
+
+    /* Weed out invalid versions. */
+    if (   uVersion != APIC_SAVED_STATE_VERSION
+        && uVersion != APIC_SAVED_STATE_VERSION_VBOX_51_BETA2
+        && uVersion != APIC_SAVED_STATE_VERSION_VBOX_50
+        && uVersion != APIC_SAVED_STATE_VERSION_VBOX_30
+        && uVersion != APIC_SAVED_STATE_VERSION_ANCIENT)
+    {
+        LogRel(("APIC: apicR3LoadExec: Invalid/unrecognized saved-state version %u (%#x)\n", uVersion, uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    int rc = VINF_SUCCESS;
+    if (uVersion > APIC_SAVED_STATE_VERSION_VBOX_30)
+    {
+        rc = apicR3LoadVMData(pDevIns, pVM, pSSM);
+        AssertRCReturn(rc, rc);
+
+        if (uVersion == APIC_SAVED_STATE_VERSION)
+        { /* Load any new additional per-VM data. */ }
+    }
+
+    /*
+     * Restore per CPU state.
+     *
+     * Note! PDM will restore the VMCPU_FF_INTERRUPT_APIC flag for us.
+     *       This code doesn't touch it.  No devices should make us touch
+     *       it later during the restore either, only during the 'done' phase.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU   pVCpu    = pVM->apCpusR3[idCpu];
+        PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+
+        if (uVersion > APIC_SAVED_STATE_VERSION_VBOX_50)
+        {
+            /* Load the auxiliary data. */
+            pHlp->pfnSSMGetU64V(pSSM, &pApicCpu->uApicBaseMsr);
+            pHlp->pfnSSMGetU32(pSSM, &pApicCpu->uEsrInternal);
+
+            /* Load the APIC page. */
+            if (XAPIC_IN_X2APIC_MODE(pVCpu))
+                pHlp->pfnSSMGetStruct(pSSM, pApicCpu->pvApicPageR3, &g_aX2ApicPageFields[0]);
+            else
+                pHlp->pfnSSMGetStruct(pSSM, pApicCpu->pvApicPageR3, &g_aXApicPageFields[0]);
+
+            /* Load the timer. */
+            rc = pHlp->pfnSSMGetU64(pSSM, &pApicCpu->u64TimerInitial);     AssertRCReturn(rc, rc);
+            rc = PDMDevHlpTimerLoad(pDevIns, pApicCpu->hTimer, pSSM);      AssertRCReturn(rc, rc);
+            Assert(pApicCpu->uHintedTimerShift == 0);
+            Assert(pApicCpu->uHintedTimerInitialCount == 0);
+            if (PDMDevHlpTimerIsActive(pDevIns, pApicCpu->hTimer))
+            {
+                PCXAPICPAGE    pXApicPage    = VMCPU_TO_CXAPICPAGE(pVCpu);
+                uint32_t const uInitialCount = pXApicPage->timer_icr.u32InitialCount;
+                uint8_t const  uTimerShift   = apicGetTimerShift(pXApicPage);
+                apicHintTimerFreq(pDevIns, pApicCpu, uInitialCount, uTimerShift);
+            }
+
+            /* Load the LINT0, LINT1 interrupt line states. */
+            if (uVersion > APIC_SAVED_STATE_VERSION_VBOX_51_BETA2)
+            {
+                pHlp->pfnSSMGetBoolV(pSSM, &pApicCpu->fActiveLint0);
+                pHlp->pfnSSMGetBoolV(pSSM, &pApicCpu->fActiveLint1);
+            }
+        }
+        else
+        {
+            rc = apicR3LoadLegacyVCpuData(pDevIns, pVCpu, pSSM, uVersion);
+            AssertRCReturn(rc, rc);
+        }
+
+        /*
+         * Check that we're still good wrt restored data, then tell CPUM about the current CPUID[1].EDX[9] visibility.
+         */
+        rc = pHlp->pfnSSMHandleGetStatus(pSSM);
+        AssertRCReturn(rc, rc);
+        CPUMSetGuestCpuIdPerCpuApicFeature(pVCpu, RT_BOOL(pApicCpu->uApicBaseMsr & MSR_IA32_APICBASE_EN));
+
+#if defined(APIC_FUZZY_SSM_COMPAT_TEST) || defined(DEBUG_ramshankar)
+        apicR3DumpState(pVCpu, "Loaded state", uVersion);
+#endif
+    }
+
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNTMTIMERDEV}
+ *
+ * @note    pvUser points to the VMCPU.
+ *
+ * @remarks Currently this function is invoked on the last EMT, see @c
+ *          idTimerCpu in tmR3TimerCallback().  However, the code does -not-
+ *          rely on this and is designed to work with being invoked on any
+ *          thread.
+ */
+static DECLCALLBACK(void) apicR3TimerCallback(PPDMDEVINS pDevIns, PTMTIMER pTimer, void *pvUser)
+{
+    PVMCPU      pVCpu    = (PVMCPU)pvUser;
+    PAPICCPU    pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    Assert(PDMDevHlpTimerIsLockOwner(pDevIns, pApicCpu->hTimer));
+    Assert(pVCpu);
+    LogFlow(("APIC%u: apicR3TimerCallback\n", pVCpu->idCpu));
+    RT_NOREF(pDevIns, pTimer, pApicCpu);
+
+    PXAPICPAGE     pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    uint32_t const uLvtTimer  = pXApicPage->lvt_timer.all.u32LvtTimer;
+#ifdef VBOX_WITH_STATISTICS
+    STAM_COUNTER_INC(&pApicCpu->StatTimerCallback);
+#endif
+    if (!XAPIC_LVT_IS_MASKED(uLvtTimer))
+    {
+        uint8_t uVector = XAPIC_LVT_GET_VECTOR(uLvtTimer);
+        Log2(("APIC%u: apicR3TimerCallback: Raising timer interrupt. uVector=%#x\n", pVCpu->idCpu, uVector));
+        apicPostInterrupt(pVCpu, uVector, XAPICTRIGGERMODE_EDGE, 0 /* uSrcTag */);
+    }
+
+    XAPICTIMERMODE enmTimerMode = XAPIC_LVT_GET_TIMER_MODE(uLvtTimer);
+    switch (enmTimerMode)
+    {
+        case XAPICTIMERMODE_PERIODIC:
+        {
+            /* The initial-count register determines if the periodic timer is re-armed. */
+            uint32_t const uInitialCount = pXApicPage->timer_icr.u32InitialCount;
+            pXApicPage->timer_ccr.u32CurrentCount = uInitialCount;
+            if (uInitialCount)
+            {
+                Log2(("APIC%u: apicR3TimerCallback: Re-arming timer. uInitialCount=%#RX32\n", pVCpu->idCpu, uInitialCount));
+                apicStartTimer(pVCpu, uInitialCount);
+            }
+            break;
+        }
+
+        case XAPICTIMERMODE_ONESHOT:
+        {
+            pXApicPage->timer_ccr.u32CurrentCount = 0;
+            break;
+        }
+
+        case XAPICTIMERMODE_TSC_DEADLINE:
+        {
+            /** @todo implement TSC deadline. */
+            AssertMsgFailed(("APIC: TSC deadline mode unimplemented\n"));
+            break;
+        }
+    }
+}
+
+
+/**
+ * @interface_method_impl{PDMDEVREG,pfnReset}
+ */
+DECLCALLBACK(void) apicR3Reset(PPDMDEVINS pDevIns)
+{
+    PVM pVM = PDMDevHlpGetVM(pDevIns);
+    VM_ASSERT_EMT0(pVM);
+    VM_ASSERT_IS_NOT_RUNNING(pVM);
+
+    LogFlow(("APIC: apicR3Reset\n"));
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU   pVCpuDest = pVM->apCpusR3[idCpu];
+        PAPICCPU pApicCpu  = VMCPU_TO_APICCPU(pVCpuDest);
+
+        if (PDMDevHlpTimerIsActive(pDevIns, pApicCpu->hTimer))
+            PDMDevHlpTimerStop(pDevIns, pApicCpu->hTimer);
+
+        apicResetCpu(pVCpuDest, true /* fResetApicBaseMsr */);
+
+        /* Clear the interrupt pending force flag. */
+        apicClearInterruptFF(pVCpuDest, PDMAPICIRQ_HARDWARE);
+    }
+}
+
+
+/**
+ * @interface_method_impl{PDMDEVREG,pfnRelocate}
+ */
+DECLCALLBACK(void) apicR3Relocate(PPDMDEVINS pDevIns, RTGCINTPTR offDelta)
+{
+    RT_NOREF(pDevIns, offDelta);
+}
+
+
+/**
+ * Terminates the APIC state.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+static void apicR3TermState(PVM pVM)
+{
+    PAPIC pApic = VM_TO_APIC(pVM);
+    LogFlow(("APIC: apicR3TermState: pVM=%p\n", pVM));
+
+    /* Unmap and free the PIB. */
+    if (pApic->pvApicPibR3 != NIL_RTR3PTR)
+    {
+        size_t const cPages = pApic->cbApicPib >> PAGE_SHIFT;
+        if (cPages == 1)
+            SUPR3PageFreeEx(pApic->pvApicPibR3, cPages);
+        else
+            SUPR3ContFree(pApic->pvApicPibR3, cPages);
+        pApic->pvApicPibR3 = NIL_RTR3PTR;
+        pApic->pvApicPibR0 = NIL_RTR0PTR;
+    }
+
+    /* Unmap and free the virtual-APIC pages. */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU   pVCpu    = pVM->apCpusR3[idCpu];
+        PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+
+        pApicCpu->pvApicPibR3 = NIL_RTR3PTR;
+        pApicCpu->pvApicPibR0 = NIL_RTR0PTR;
+
+        if (pApicCpu->pvApicPageR3 != NIL_RTR3PTR)
+        {
+            SUPR3PageFreeEx(pApicCpu->pvApicPageR3, 1 /* cPages */);
+            pApicCpu->pvApicPageR3 = NIL_RTR3PTR;
+            pApicCpu->pvApicPageR0 = NIL_RTR0PTR;
+        }
+    }
+}
+
+
+/**
+ * Initializes the APIC state.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+static int apicR3InitState(PVM pVM)
+{
+    PAPIC pApic = VM_TO_APIC(pVM);
+    LogFlow(("APIC: apicR3InitState: pVM=%p\n", pVM));
+
+    /*
+     * Allocate and map the pending-interrupt bitmap (PIB).
+     *
+     * We allocate all the VCPUs' PIBs contiguously in order to save space as
+     * physically contiguous allocations are rounded to a multiple of page size.
+     */
+    Assert(pApic->pvApicPibR3 == NIL_RTR3PTR);
+    Assert(pApic->pvApicPibR0 == NIL_RTR0PTR);
+    pApic->cbApicPib    = RT_ALIGN_Z(pVM->cCpus * sizeof(APICPIB), PAGE_SIZE);
+    size_t const cPages = pApic->cbApicPib >> PAGE_SHIFT;
+    if (cPages == 1)
+    {
+        SUPPAGE SupApicPib;
+        RT_ZERO(SupApicPib);
+        SupApicPib.Phys = NIL_RTHCPHYS;
+        int rc = SUPR3PageAllocEx(1 /* cPages */, 0 /* fFlags */, &pApic->pvApicPibR3, &pApic->pvApicPibR0, &SupApicPib);
+        if (RT_SUCCESS(rc))
+        {
+            pApic->HCPhysApicPib = SupApicPib.Phys;
+            AssertLogRelReturn(pApic->pvApicPibR3, VERR_INTERNAL_ERROR);
+        }
+        else
+        {
+            LogRel(("APIC: Failed to allocate %u bytes for the pending-interrupt bitmap, rc=%Rrc\n", pApic->cbApicPib, rc));
+            return rc;
+        }
+    }
+    else
+        pApic->pvApicPibR3 = SUPR3ContAlloc(cPages, &pApic->pvApicPibR0, &pApic->HCPhysApicPib);
+
+    if (pApic->pvApicPibR3)
+    {
+        AssertLogRelReturn(pApic->pvApicPibR0   != NIL_RTR0PTR,  VERR_INTERNAL_ERROR);
+        AssertLogRelReturn(pApic->HCPhysApicPib != NIL_RTHCPHYS, VERR_INTERNAL_ERROR);
+
+        /* Initialize the PIB. */
+        RT_BZERO(pApic->pvApicPibR3, pApic->cbApicPib);
+
+        /*
+         * Allocate the map the virtual-APIC pages.
+         */
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PVMCPU   pVCpu    = pVM->apCpusR3[idCpu];
+            PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+
+            SUPPAGE SupApicPage;
+            RT_ZERO(SupApicPage);
+            SupApicPage.Phys = NIL_RTHCPHYS;
+
+            Assert(pVCpu->idCpu == idCpu);
+            Assert(pApicCpu->pvApicPageR3 == NIL_RTR3PTR);
+            Assert(pApicCpu->pvApicPageR0 == NIL_RTR0PTR);
+            AssertCompile(sizeof(XAPICPAGE) == PAGE_SIZE);
+            pApicCpu->cbApicPage = sizeof(XAPICPAGE);
+            int rc = SUPR3PageAllocEx(1 /* cPages */, 0 /* fFlags */, &pApicCpu->pvApicPageR3, &pApicCpu->pvApicPageR0,
+                                      &SupApicPage);
+            if (RT_SUCCESS(rc))
+            {
+                AssertLogRelReturn(pApicCpu->pvApicPageR3   != NIL_RTR3PTR,  VERR_INTERNAL_ERROR);
+                AssertLogRelReturn(pApicCpu->HCPhysApicPage != NIL_RTHCPHYS, VERR_INTERNAL_ERROR);
+                pApicCpu->HCPhysApicPage = SupApicPage.Phys;
+
+                /* Associate the per-VCPU PIB pointers to the per-VM PIB mapping. */
+                uint32_t const offApicPib  = idCpu * sizeof(APICPIB);
+                pApicCpu->pvApicPibR0      = (RTR0PTR)((RTR0UINTPTR)pApic->pvApicPibR0 + offApicPib);
+                pApicCpu->pvApicPibR3      = (RTR3PTR)((RTR3UINTPTR)pApic->pvApicPibR3 + offApicPib);
+
+                /* Initialize the virtual-APIC state. */
+                RT_BZERO(pApicCpu->pvApicPageR3, pApicCpu->cbApicPage);
+                apicResetCpu(pVCpu, true /* fResetApicBaseMsr */);
+
+#ifdef DEBUG_ramshankar
+                Assert(pApicCpu->pvApicPibR3 != NIL_RTR3PTR);
+                Assert(pApicCpu->pvApicPibR0 != NIL_RTR0PTR);
+                Assert(pApicCpu->pvApicPageR3 != NIL_RTR3PTR);
+#endif
+            }
+            else
+            {
+                LogRel(("APIC%u: Failed to allocate %u bytes for the virtual-APIC page, rc=%Rrc\n", idCpu, pApicCpu->cbApicPage, rc));
+                apicR3TermState(pVM);
+                return rc;
+            }
+        }
+
+#ifdef DEBUG_ramshankar
+        Assert(pApic->pvApicPibR3 != NIL_RTR3PTR);
+        Assert(pApic->pvApicPibR0 != NIL_RTR0PTR);
+#endif
+        return VINF_SUCCESS;
+    }
+
+    LogRel(("APIC: Failed to allocate %u bytes of physically contiguous memory for the pending-interrupt bitmap\n",
+            pApic->cbApicPib));
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * @interface_method_impl{PDMDEVREG,pfnDestruct}
+ */
+DECLCALLBACK(int) apicR3Destruct(PPDMDEVINS pDevIns)
+{
+    PDMDEV_CHECK_VERSIONS_RETURN_QUIET(pDevIns);
+    PVM pVM = PDMDevHlpGetVM(pDevIns);
+    LogFlow(("APIC: apicR3Destruct: pVM=%p\n", pVM));
+
+    apicR3TermState(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @interface_method_impl{PDMDEVREG,pfnInitComplete}
+ */
+DECLCALLBACK(int) apicR3InitComplete(PPDMDEVINS pDevIns)
+{
+    PVM   pVM   = PDMDevHlpGetVM(pDevIns);
+    PAPIC pApic = VM_TO_APIC(pVM);
+
+    /*
+     * Init APIC settings that rely on HM and CPUM configurations.
+     */
+    CPUMCPUIDLEAF CpuLeaf;
+    int rc = CPUMR3CpuIdGetLeaf(pVM, &CpuLeaf, 1, 0);
+    AssertRCReturn(rc, rc);
+
+    pApic->fSupportsTscDeadline = RT_BOOL(CpuLeaf.uEcx & X86_CPUID_FEATURE_ECX_TSCDEADL);
+    pApic->fPostedIntrsEnabled  = HMR3IsPostedIntrsEnabled(pVM->pUVM);
+    pApic->fVirtApicRegsEnabled = HMR3IsVirtApicRegsEnabled(pVM->pUVM);
+
+    LogRel(("APIC: fPostedIntrsEnabled=%RTbool fVirtApicRegsEnabled=%RTbool fSupportsTscDeadline=%RTbool\n",
+            pApic->fPostedIntrsEnabled, pApic->fVirtApicRegsEnabled, pApic->fSupportsTscDeadline));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @interface_method_impl{PDMDEVREG,pfnConstruct}
+ */
+DECLCALLBACK(int) apicR3Construct(PPDMDEVINS pDevIns, int iInstance, PCFGMNODE pCfg)
+{
+    PDMDEV_CHECK_VERSIONS_RETURN(pDevIns);
+    PAPICDEV        pApicDev = PDMDEVINS_2_DATA(pDevIns, PAPICDEV);
+    PCPDMDEVHLPR3   pHlp     = pDevIns->pHlpR3;
+    PVM             pVM      = PDMDevHlpGetVM(pDevIns);
+    PAPIC           pApic    = VM_TO_APIC(pVM);
+    Assert(iInstance == 0); NOREF(iInstance);
+
+    /*
+     * Init the data.
+     */
+    pApic->pDevInsR3    = pDevIns;
+    pApic->fR0Enabled   = pDevIns->fR0Enabled;
+    pApic->fRCEnabled   = pDevIns->fRCEnabled;
+
+    /*
+     * Validate APIC settings.
+     */
+    PDMDEV_VALIDATE_CONFIG_RETURN(pDevIns, "Mode|IOAPIC|NumCPUs", "");
+
+    int rc = pHlp->pfnCFGMQueryBoolDef(pCfg, "IOAPIC", &pApic->fIoApicPresent, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    /* Max APIC feature level. */
+    uint8_t uMaxMode;
+    rc = pHlp->pfnCFGMQueryU8Def(pCfg, "Mode", &uMaxMode, PDMAPICMODE_APIC);
+    AssertLogRelRCReturn(rc, rc);
+    switch ((PDMAPICMODE)uMaxMode)
+    {
+        case PDMAPICMODE_NONE:
+            LogRel(("APIC: APIC maximum mode configured as 'None', effectively disabled/not-present!\n"));
+        case PDMAPICMODE_APIC:
+        case PDMAPICMODE_X2APIC:
+            break;
+        default:
+            return VMR3SetError(pVM->pUVM, VERR_INVALID_PARAMETER, RT_SRC_POS, "APIC mode %d unknown.", uMaxMode);
+    }
+    pApic->enmMaxMode = (PDMAPICMODE)uMaxMode;
+
+    /*
+     * Disable automatic PDM locking for this device.
+     */
+    rc = PDMDevHlpSetDeviceCritSect(pDevIns, PDMDevHlpCritSectGetNop(pDevIns));
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Register the APIC with PDM.
+     */
+    rc = PDMDevHlpApicRegister(pDevIns);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Initialize the APIC state.
+     */
+    if (pApic->enmMaxMode == PDMAPICMODE_X2APIC)
+    {
+        rc = CPUMR3MsrRangesInsert(pVM, &g_MsrRange_x2Apic);
+        AssertLogRelRCReturn(rc, rc);
+    }
+    else
+    {
+        /* We currently don't have a function to remove the range, so we register an range which will cause a #GP. */
+        rc = CPUMR3MsrRangesInsert(pVM, &g_MsrRange_x2Apic_Invalid);
+        AssertLogRelRCReturn(rc, rc);
+    }
+
+    /* Tell CPUM about the APIC feature level so it can adjust APICBASE MSR GP mask and CPUID bits. */
+    apicR3SetCpuIdFeatureLevel(pVM, pApic->enmMaxMode);
+
+    /* Finally, initialize the state. */
+    rc = apicR3InitState(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Register the MMIO range.
+     */
+    PAPICCPU pApicCpu0 = VMCPU_TO_APICCPU(pVM->apCpusR3[0]);
+    RTGCPHYS GCPhysApicBase = MSR_IA32_APICBASE_GET_ADDR(pApicCpu0->uApicBaseMsr);
+
+    rc = PDMDevHlpMmioCreateAndMap(pDevIns, GCPhysApicBase, sizeof(XAPICPAGE), apicWriteMmio, apicReadMmio,
+                                   IOMMMIO_FLAGS_READ_DWORD | IOMMMIO_FLAGS_WRITE_DWORD_ZEROED, "APIC", &pApicDev->hMmio);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Create the APIC timers.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU   pVCpu    = pVM->apCpusR3[idCpu];
+        PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+        RTStrPrintf(&pApicCpu->szTimerDesc[0], sizeof(pApicCpu->szTimerDesc), "APIC Timer %u", pVCpu->idCpu);
+        rc = PDMDevHlpTimerCreate(pDevIns, TMCLOCK_VIRTUAL_SYNC, apicR3TimerCallback, pVCpu, TMTIMER_FLAGS_NO_CRIT_SECT,
+                                  pApicCpu->szTimerDesc, &pApicCpu->hTimer);
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Register saved state callbacks.
+     */
+    rc = PDMDevHlpSSMRegister(pDevIns, APIC_SAVED_STATE_VERSION, sizeof(*pApicDev), apicR3SaveExec, apicR3LoadExec);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Register debugger info callbacks.
+     *
+     * We use separate callbacks rather than arguments so they can also be
+     * dumped in an automated fashion while collecting crash diagnostics and
+     * not just used during live debugging via the VM debugger.
+     */
+    DBGFR3InfoRegisterInternalEx(pVM, "apic",      "Dumps APIC basic information.", apicR3Info,      DBGFINFO_FLAGS_ALL_EMTS);
+    DBGFR3InfoRegisterInternalEx(pVM, "apiclvt",   "Dumps APIC LVT information.",   apicR3InfoLvt,   DBGFINFO_FLAGS_ALL_EMTS);
+    DBGFR3InfoRegisterInternalEx(pVM, "apictimer", "Dumps APIC timer information.", apicR3InfoTimer, DBGFINFO_FLAGS_ALL_EMTS);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Statistics.
+     */
+#define APIC_REG_COUNTER(a_pvReg, a_pszNameFmt, a_pszDesc) \
+        PDMDevHlpSTAMRegisterF(pDevIns, a_pvReg, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, \
+                               STAMUNIT_OCCURENCES, a_pszDesc, a_pszNameFmt, idCpu)
+# define APIC_PROF_COUNTER(a_pvReg, a_pszNameFmt, a_pszDesc) \
+        PDMDevHlpSTAMRegisterF(pDevIns, a_pvReg, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, \
+                               STAMUNIT_TICKS_PER_CALL, a_pszDesc, a_pszNameFmt, idCpu)
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU   pVCpu     = pVM->apCpusR3[idCpu];
+        PAPICCPU pApicCpu  = VMCPU_TO_APICCPU(pVCpu);
+
+        APIC_REG_COUNTER(&pApicCpu->StatMmioReadRZ,    "%u/RZ/MmioRead",    "Number of APIC MMIO reads in RZ.");
+        APIC_REG_COUNTER(&pApicCpu->StatMmioWriteRZ,   "%u/RZ/MmioWrite",   "Number of APIC MMIO writes in RZ.");
+        APIC_REG_COUNTER(&pApicCpu->StatMsrReadRZ,     "%u/RZ/MsrRead",     "Number of APIC MSR reads in RZ.");
+        APIC_REG_COUNTER(&pApicCpu->StatMsrWriteRZ,    "%u/RZ/MsrWrite",    "Number of APIC MSR writes in RZ.");
+
+        APIC_REG_COUNTER(&pApicCpu->StatMmioReadR3,    "%u/R3/MmioReadR3",  "Number of APIC MMIO reads in R3.");
+        APIC_REG_COUNTER(&pApicCpu->StatMmioWriteR3,   "%u/R3/MmioWriteR3", "Number of APIC MMIO writes in R3.");
+        APIC_REG_COUNTER(&pApicCpu->StatMsrReadR3,     "%u/R3/MsrReadR3",   "Number of APIC MSR reads in R3.");
+        APIC_REG_COUNTER(&pApicCpu->StatMsrWriteR3,    "%u/R3/MsrWriteR3",  "Number of APIC MSR writes in R3.");
+
+        APIC_REG_COUNTER(&pApicCpu->StatPostIntrAlreadyPending,
+                                                       "%u/PostInterruptAlreadyPending", "Number of times an interrupt is already pending.");
+        APIC_REG_COUNTER(&pApicCpu->StatTimerCallback, "%u/TimerCallback",  "Number of times the timer callback is invoked.");
+
+        APIC_REG_COUNTER(&pApicCpu->StatTprWrite,      "%u/TprWrite",       "Number of TPR writes.");
+        APIC_REG_COUNTER(&pApicCpu->StatTprRead,       "%u/TprRead",        "Number of TPR reads.");
+        APIC_REG_COUNTER(&pApicCpu->StatEoiWrite,      "%u/EoiWrite",       "Number of EOI writes.");
+        APIC_REG_COUNTER(&pApicCpu->StatMaskedByTpr,   "%u/MaskedByTpr",    "Number of times TPR masks an interrupt in apicGetInterrupt.");
+        APIC_REG_COUNTER(&pApicCpu->StatMaskedByPpr,   "%u/MaskedByPpr",    "Number of times PPR masks an interrupt in apicGetInterrupt.");
+        APIC_REG_COUNTER(&pApicCpu->StatTimerIcrWrite, "%u/TimerIcrWrite",  "Number of times the timer ICR is written.");
+        APIC_REG_COUNTER(&pApicCpu->StatIcrLoWrite,    "%u/IcrLoWrite",     "Number of times the ICR Lo (send IPI) is written.");
+        APIC_REG_COUNTER(&pApicCpu->StatIcrHiWrite,    "%u/IcrHiWrite",     "Number of times the ICR Hi is written.");
+        APIC_REG_COUNTER(&pApicCpu->StatIcrFullWrite,  "%u/IcrFullWrite",   "Number of times the ICR full (send IPI, x2APIC) is written.");
+        APIC_REG_COUNTER(&pApicCpu->StatIdMsrRead,     "%u/IdMsrRead",      "Number of times the APIC-ID MSR is read.");
+
+        APIC_PROF_COUNTER(&pApicCpu->StatUpdatePendingIntrs,
+                                                       "/PROF/CPU%u/APIC/UpdatePendingInterrupts", "Profiling of APICUpdatePendingInterrupts");
+        APIC_PROF_COUNTER(&pApicCpu->StatPostIntr,     "/PROF/CPU%u/APIC/PostInterrupt",  "Profiling of APICPostInterrupt");
+    }
+
+# undef APIC_PROF_COUNTER
+# undef APIC_REG_ACCESS_COUNTER
+#endif
+
+    return VINF_SUCCESS;
+}
+
+#endif /* !VBOX_DEVICE_STRUCT_TESTCASE */
+
diff --git a/src/VBox/VMM/VMMR3/CFGM.cpp b/src/VBox/VMM/VMMR3/CFGM.cpp
new file mode 100644
index 00000000..3447029c
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/CFGM.cpp
@@ -0,0 +1,3273 @@
+/* $Id: CFGM.cpp $ */
+/** @file
+ * CFGM - Configuration Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_cfgm       CFGM - The Configuration Manager
+ *
+ * The configuration manager is a directory containing the VM configuration at
+ * run time. It works in a manner similar to the windows registry - it's like a
+ * file system hierarchy, but the files (values) live in a separate name space
+ * and can include the path separators.
+ *
+ * The configuration is normally created via a callback passed to VMR3Create()
+ * via the pfnCFGMConstructor parameter. To make testcase writing a bit simpler,
+ * we allow the callback to be NULL, in which case a simple default
+ * configuration will be created by CFGMR3ConstructDefaultTree(). The
+ * Console::configConstructor() method in Main/ConsoleImpl2.cpp creates the
+ * configuration from the XML.
+ *
+ * Devices, drivers, services and other PDM stuff are given their own subtree
+ * where they are protected from accessing information of any parents. This is
+ * is implemented via the CFGMR3SetRestrictedRoot() API.
+ *
+ * Data validation beyond the basic primitives is left to the caller. The caller
+ * is in a better position to know the proper validation rules of the individual
+ * properties.
+ *
+ * @see grp_cfgm
+ *
+ *
+ * @section sec_cfgm_primitives     Data Primitives
+ *
+ * CFGM supports the following data primitives:
+ *      - Integers. Representation is unsigned 64-bit. Boolean, unsigned and
+ *        small integers, and pointers are all represented using this primitive.
+ *      - Zero terminated character strings. These are of course UTF-8.
+ *      - Variable length byte strings. This can be used to get/put binary
+ *        objects like for instance RTMAC.
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_CFGM
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/mm.h>
+#include "CFGMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/mem.h>
+#include <iprt/param.h>
+#include <iprt/string.h>
+#include <iprt/utf16.h>
+#include <iprt/uuid.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static void cfgmR3DumpPath(PCFGMNODE pNode, PCDBGFINFOHLP pHlp);
+static void cfgmR3Dump(PCFGMNODE pRoot, unsigned iLevel, PCDBGFINFOHLP pHlp);
+static DECLCALLBACK(void) cfgmR3Info(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static int  cfgmR3ResolveNode(PCFGMNODE pNode, const char *pszPath, PCFGMNODE *ppChild);
+static int  cfgmR3ResolveLeaf(PCFGMNODE pNode, const char *pszName, PCFGMLEAF *ppLeaf);
+static int  cfgmR3InsertLeaf(PCFGMNODE pNode, const char *pszName, PCFGMLEAF *ppLeaf);
+static void cfgmR3RemoveLeaf(PCFGMNODE pNode, PCFGMLEAF pLeaf);
+static void cfgmR3FreeValue(PVM pVM, PCFGMLEAF pLeaf);
+
+
+/** @todo replace pVM for pUVM !*/
+
+/**
+ * Allocator wrapper.
+ *
+ * @returns Pointer to the allocated memory, NULL on failure.
+ * @param   pVM         The cross context VM structure, if the tree
+ *                      is associated with one.
+ * @param   enmTag      The allocation tag.
+ * @param   cb          The size of the allocation.
+ */
+static void *cfgmR3MemAlloc(PVM pVM, MMTAG enmTag, size_t cb)
+{
+    if (pVM)
+        return MMR3HeapAlloc(pVM, enmTag, cb);
+    return RTMemAlloc(cb);
+}
+
+
+/**
+ * Free wrapper.
+ *
+ * @returns Pointer to the allocated memory, NULL on failure.
+ * @param   pVM         The cross context VM structure, if the tree
+ *                      is associated with one.
+ * @param   pv          The memory block to free.
+ */
+static void cfgmR3MemFree(PVM pVM, void *pv)
+{
+    if (pVM)
+        MMR3HeapFree(pv);
+    else
+        RTMemFree(pv);
+}
+
+
+/**
+ * String allocator wrapper.
+ *
+ * @returns Pointer to the allocated memory, NULL on failure.
+ * @param   pVM         The cross context VM structure, if the tree
+ *                      is associated with one.
+ * @param   enmTag      The allocation tag.
+ * @param   cbString    The size of the allocation, terminator included.
+ */
+static char *cfgmR3StrAlloc(PVM pVM, MMTAG enmTag,  size_t cbString)
+{
+    if (pVM)
+        return (char *)MMR3HeapAlloc(pVM, enmTag, cbString);
+    return (char *)RTStrAlloc(cbString);
+}
+
+
+/**
+ * String free wrapper.
+ *
+ * @returns Pointer to the allocated memory, NULL on failure.
+ * @param   pVM         The cross context VM structure, if the tree
+ *                      is associated with one.
+ * @param   pszString   The memory block to free.
+ */
+static void cfgmR3StrFree(PVM pVM, char *pszString)
+{
+    if (pVM)
+        MMR3HeapFree(pszString);
+    else
+        RTStrFree(pszString);
+}
+
+
+/**
+ * Frees one node, leaving any children or leaves to the caller.
+ *
+ * @param   pNode               The node structure to free.
+ */
+static void cfgmR3FreeNodeOnly(PCFGMNODE pNode)
+{
+    pNode->pFirstLeaf    = NULL;
+    pNode->pFirstChild   = NULL;
+    pNode->pNext         = NULL;
+    pNode->pPrev         = NULL;
+    if (!pNode->pVM)
+        RTMemFree(pNode);
+    else
+    {
+        pNode->pVM       = NULL;
+        MMR3HeapFree(pNode);
+    }
+}
+
+
+
+
+/**
+ * Constructs the configuration for the VM.
+ *
+ * This should only be called used once.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pfnCFGMConstructor  Pointer to callback function for constructing
+ *                              the VM configuration tree.  This is called on
+ *                              the EMT.
+ * @param   pvUser              The user argument passed to pfnCFGMConstructor.
+ * @thread  EMT.
+ * @internal
+ */
+VMMR3DECL(int) CFGMR3Init(PVM pVM, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUser)
+{
+    LogFlow(("CFGMR3Init: pfnCFGMConstructor=%p pvUser=%p\n", pfnCFGMConstructor, pvUser));
+
+    /*
+     * Init data members.
+     */
+    pVM->cfgm.s.pRoot = NULL;
+
+    /*
+     * Register DBGF into item.
+     */
+    int rc = DBGFR3InfoRegisterInternal(pVM, "cfgm", "Dumps a part of the CFGM tree. The argument indicates where to start.",
+                                        cfgmR3Info);
+    AssertRCReturn(rc,rc);
+
+    /*
+     * Root Node.
+     */
+    PCFGMNODE pRoot = (PCFGMNODE)MMR3HeapAllocZ(pVM, MM_TAG_CFGM, sizeof(*pRoot));
+    if (!pRoot)
+        return VERR_NO_MEMORY;
+    pRoot->pVM        = pVM;
+    pRoot->cchName    = 0;
+    pVM->cfgm.s.pRoot = pRoot;
+
+    /*
+     * Call the constructor if specified, if not use the default one.
+     */
+    if (pfnCFGMConstructor)
+        rc = pfnCFGMConstructor(pVM->pUVM, pVM, pvUser);
+    else
+        rc = CFGMR3ConstructDefaultTree(pVM);
+    if (RT_SUCCESS(rc))
+    {
+        Log(("CFGMR3Init: Successfully constructed the configuration\n"));
+        CFGMR3Dump(CFGMR3GetRoot(pVM));
+    }
+    else
+        LogRel(("Constructor failed with rc=%Rrc pfnCFGMConstructor=%p\n", rc, pfnCFGMConstructor));
+
+    return rc;
+}
+
+
+/**
+ * Terminates the configuration manager.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @internal
+ */
+VMMR3DECL(int) CFGMR3Term(PVM pVM)
+{
+    CFGMR3RemoveNode(pVM->cfgm.s.pRoot);
+    pVM->cfgm.s.pRoot = NULL;
+    return 0;
+}
+
+
+/**
+ * Gets the root node for the VM.
+ *
+ * @returns Pointer to root node.
+ * @param   pVM             The cross context VM structure.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetRoot(PVM pVM)
+{
+    return pVM->cfgm.s.pRoot;
+}
+
+
+/**
+ * Gets the root node for the VM.
+ *
+ * @returns Pointer to root node.
+ * @param   pUVM        The user mode VM structure.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetRootU(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+    PVM pVM = pUVM->pVM;
+    AssertReturn(pVM, NULL);
+    return pVM->cfgm.s.pRoot;
+}
+
+
+/**
+ * Gets the parent of a CFGM node.
+ *
+ * @returns Pointer to the parent node.
+ * @returns NULL if pNode is Root or pNode is the start of a
+ *          restricted subtree (use CFGMR3GetParentEx() for that).
+ *
+ * @param   pNode           The node which parent we query.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetParent(PCFGMNODE pNode)
+{
+    if (pNode && !pNode->fRestrictedRoot)
+        return pNode->pParent;
+    return NULL;
+}
+
+
+/**
+ * Gets the parent of a CFGM node.
+ *
+ * @returns Pointer to the parent node.
+ * @returns NULL if pNode is Root or pVM is not correct.
+ *
+ * @param   pVM             The cross context VM structure.  Used as token that
+ *                          the caller is trusted.
+ * @param   pNode           The node which parent we query.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetParentEx(PVM pVM, PCFGMNODE pNode)
+{
+    if (pNode && pNode->pVM == pVM)
+        return pNode->pParent;
+    return NULL;
+}
+
+
+/**
+ * Query a child node.
+ *
+ * @returns Pointer to the specified node.
+ * @returns NULL if node was not found or pNode is NULL.
+ * @param   pNode           Node pszPath is relative to.
+ * @param   pszPath         Path to the child node or pNode.
+ *                          It's good style to end this with '/'.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetChild(PCFGMNODE pNode, const char *pszPath)
+{
+    PCFGMNODE pChild;
+    int rc = cfgmR3ResolveNode(pNode, pszPath, &pChild);
+    if (RT_SUCCESS(rc))
+        return pChild;
+    return NULL;
+}
+
+
+/**
+ * Query a child node by a format string.
+ *
+ * @returns Pointer to the specified node.
+ * @returns NULL if node was not found or pNode is NULL.
+ * @param   pNode           Node pszPath is relative to.
+ * @param   pszPathFormat   Path to the child node or pNode.
+ *                          It's good style to end this with '/'.
+ * @param   ...             Arguments to pszPathFormat.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetChildF(PCFGMNODE pNode, const char *pszPathFormat, ...)
+{
+    va_list Args;
+    va_start(Args,  pszPathFormat);
+    PCFGMNODE pRet = CFGMR3GetChildFV(pNode, pszPathFormat, Args);
+    va_end(Args);
+    return pRet;
+}
+
+
+/**
+ * Query a child node by a format string.
+ *
+ * @returns Pointer to the specified node.
+ * @returns NULL if node was not found or pNode is NULL.
+ * @param   pNode           Node pszPath is relative to.
+ * @param   pszPathFormat   Path to the child node or pNode.
+ *                          It's good style to end this with '/'.
+ * @param   Args            Arguments to pszPathFormat.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetChildFV(PCFGMNODE pNode, const char *pszPathFormat, va_list Args)
+{
+    char *pszPath;
+    RTStrAPrintfV(&pszPath, pszPathFormat, Args);
+    if (pszPath)
+    {
+        PCFGMNODE pChild;
+        int rc = cfgmR3ResolveNode(pNode, pszPath, &pChild);
+        RTStrFree(pszPath);
+        if (RT_SUCCESS(rc))
+            return pChild;
+    }
+    return NULL;
+}
+
+
+/**
+ * Gets the first child node.
+ * Use this to start an enumeration of child nodes.
+ *
+ * @returns Pointer to the first child.
+ * @returns NULL if no children.
+ * @param   pNode           Node to enumerate children for.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetFirstChild(PCFGMNODE pNode)
+{
+    return pNode ? pNode->pFirstChild : NULL;
+}
+
+
+/**
+ * Gets the next sibling node.
+ * Use this to continue an enumeration.
+ *
+ * @returns Pointer to the first child.
+ * @returns NULL if no children.
+ * @param   pCur            Node to returned by a call to CFGMR3GetFirstChild()
+ *                          or successive calls to this function.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetNextChild(PCFGMNODE pCur)
+{
+    return pCur ? pCur->pNext : NULL;
+}
+
+
+/**
+ * Gets the name of the current node.
+ * (Needed for enumeration.)
+ *
+ * @returns VBox status code.
+ * @param   pCur            Node to returned by a call to CFGMR3GetFirstChild()
+ *                          or successive calls to CFGMR3GetNextChild().
+ * @param   pszName         Where to store the node name.
+ * @param   cchName         Size of the buffer pointed to by pszName (with terminator).
+ */
+VMMR3DECL(int) CFGMR3GetName(PCFGMNODE pCur, char *pszName, size_t cchName)
+{
+    int rc;
+    if (pCur)
+    {
+        if (cchName > pCur->cchName)
+        {
+            rc = VINF_SUCCESS;
+            memcpy(pszName, pCur->szName, pCur->cchName + 1);
+        }
+        else
+            rc = VERR_CFGM_NOT_ENOUGH_SPACE;
+    }
+    else
+        rc = VERR_CFGM_NO_NODE;
+    return rc;
+}
+
+
+/**
+ * Gets the length of the current node's name.
+ * (Needed for enumeration.)
+ *
+ * @returns Node name length in bytes including the terminating null char.
+ * @returns 0 if pCur is NULL.
+ * @param   pCur            Node to returned by a call to CFGMR3GetFirstChild()
+ *                          or successive calls to CFGMR3GetNextChild().
+ */
+VMMR3DECL(size_t) CFGMR3GetNameLen(PCFGMNODE pCur)
+{
+    return pCur ? pCur->cchName + 1 : 0;
+}
+
+
+/**
+ * Validates that the child nodes are within a set of valid names.
+ *
+ * @returns true if all names are found in pszzAllowed.
+ * @returns false if not.
+ * @param   pNode           The node which children should be examined.
+ * @param   pszzValid       List of valid names separated by '\\0' and ending with
+ *                          a double '\\0'.
+ *
+ * @deprecated Use CFGMR3ValidateConfig.
+ */
+VMMR3DECL(bool) CFGMR3AreChildrenValid(PCFGMNODE pNode, const char *pszzValid)
+{
+    if (pNode)
+    {
+        for (PCFGMNODE pChild = pNode->pFirstChild; pChild; pChild = pChild->pNext)
+        {
+            /* search pszzValid for the name */
+            const char *psz = pszzValid;
+            while (*psz)
+            {
+                size_t cch = strlen(psz);
+                if (    cch == pChild->cchName
+                    &&  !memcmp(psz, pChild->szName, cch))
+                    break;
+
+                /* next */
+                psz += cch + 1;
+            }
+
+            /* if at end of pszzValid we didn't find it => failure */
+            if (!*psz)
+            {
+                AssertMsgFailed(("Couldn't find '%s' in the valid values\n", pChild->szName));
+                return false;
+            }
+        }
+    }
+
+    /* all ok. */
+    return true;
+}
+
+
+/**
+ * Gets the first value of a node.
+ * Use this to start an enumeration of values.
+ *
+ * @returns Pointer to the first value.
+ * @param   pCur            The node (Key) which values to enumerate.
+ */
+VMMR3DECL(PCFGMLEAF) CFGMR3GetFirstValue(PCFGMNODE pCur)
+{
+    return pCur ? pCur->pFirstLeaf : NULL;
+}
+
+/**
+ * Gets the next value in enumeration.
+ *
+ * @returns Pointer to the next value.
+ * @param   pCur            The current value as returned by this function or CFGMR3GetFirstValue().
+ */
+VMMR3DECL(PCFGMLEAF) CFGMR3GetNextValue(PCFGMLEAF pCur)
+{
+    return pCur ? pCur->pNext : NULL;
+}
+
+/**
+ * Get the value name.
+ * (Needed for enumeration.)
+ *
+ * @returns VBox status code.
+ * @param   pCur            Value returned by a call to CFGMR3GetFirstValue()
+ *                          or successive calls to CFGMR3GetNextValue().
+ * @param   pszName         Where to store the value name.
+ * @param   cchName         Size of the buffer pointed to by pszName (with terminator).
+ */
+VMMR3DECL(int) CFGMR3GetValueName(PCFGMLEAF pCur, char *pszName, size_t cchName)
+{
+    int rc;
+    if (pCur)
+    {
+        if (cchName > pCur->cchName)
+        {
+            rc = VINF_SUCCESS;
+            memcpy(pszName, pCur->szName, pCur->cchName + 1);
+        }
+        else
+            rc = VERR_CFGM_NOT_ENOUGH_SPACE;
+    }
+    else
+        rc = VERR_CFGM_NO_NODE;
+    return rc;
+}
+
+
+/**
+ * Gets the length of the current node's name.
+ * (Needed for enumeration.)
+ *
+ * @returns Value name length in bytes including the terminating null char.
+ * @returns 0 if pCur is NULL.
+ * @param   pCur            Value returned by a call to CFGMR3GetFirstValue()
+ *                          or successive calls to CFGMR3GetNextValue().
+ */
+VMMR3DECL(size_t) CFGMR3GetValueNameLen(PCFGMLEAF pCur)
+{
+    return pCur ? pCur->cchName + 1 : 0;
+}
+
+
+/**
+ * Gets the value type.
+ * (For enumeration.)
+ *
+ * @returns VBox status code.
+ * @param   pCur            Value returned by a call to CFGMR3GetFirstValue()
+ *                          or successive calls to CFGMR3GetNextValue().
+ */
+VMMR3DECL(CFGMVALUETYPE) CFGMR3GetValueType(PCFGMLEAF pCur)
+{
+    Assert(pCur);
+    return pCur->enmType;
+}
+
+
+/**
+ * Validates that the values are within a set of valid names.
+ *
+ * @returns true if all names are found in pszzValid.
+ * @returns false if not.
+ * @param   pNode           The node which values should be examined.
+ * @param   pszzValid       List of valid names separated by '\\0' and ending with
+ *                          a double '\\0'.
+ * @deprecated Use CFGMR3ValidateConfig.
+ */
+VMMR3DECL(bool) CFGMR3AreValuesValid(PCFGMNODE pNode, const char *pszzValid)
+{
+    if (pNode)
+    {
+        for (PCFGMLEAF pLeaf = pNode->pFirstLeaf; pLeaf; pLeaf = pLeaf->pNext)
+        {
+            /* search pszzValid for the name */
+            const char *psz = pszzValid;
+            while (*psz)
+            {
+                size_t cch = strlen(psz);
+                if (    cch == pLeaf->cchName
+                    &&  !memcmp(psz, pLeaf->szName, cch))
+                    break;
+
+                /* next */
+                psz += cch + 1;
+            }
+
+            /* if at end of pszzValid we didn't find it => failure */
+            if (!*psz)
+            {
+                AssertMsgFailed(("Couldn't find '%s' in the valid values\n", pLeaf->szName));
+                return false;
+            }
+        }
+    }
+
+    /* all ok. */
+    return true;
+}
+
+
+/**
+ * Checks if the given value exists.
+ *
+ * @returns true if it exists, false if not.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         The name of the value we seek.
+ */
+VMMR3DECL(bool) CFGMR3Exists(PCFGMNODE pNode, const char *pszName)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    return RT_SUCCESS_NP(rc);
+}
+
+
+/**
+ * Query value type.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   penmType        Where to store the type.
+ */
+VMMR3DECL(int) CFGMR3QueryType(PCFGMNODE pNode, const char *pszName, PCFGMVALUETYPE penmType)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (penmType)
+            *penmType = pLeaf->enmType;
+    }
+    return rc;
+}
+
+
+/**
+ * Query value size.
+ * This works on all types of values.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pcb             Where to store the value size.
+ */
+VMMR3DECL(int) CFGMR3QuerySize(PCFGMNODE pNode, const char *pszName, size_t *pcb)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        switch (pLeaf->enmType)
+        {
+            case CFGMVALUETYPE_INTEGER:
+                *pcb = sizeof(pLeaf->Value.Integer.u64);
+                break;
+
+            case CFGMVALUETYPE_STRING:
+                *pcb = pLeaf->Value.String.cb;
+                break;
+
+            case CFGMVALUETYPE_BYTES:
+                *pcb = pLeaf->Value.Bytes.cb;
+                break;
+
+            default:
+                rc = VERR_CFGM_IPE_1;
+                AssertMsgFailed(("Invalid value type %d\n", pLeaf->enmType));
+                break;
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Query integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu64            Where to store the integer value.
+ */
+VMMR3DECL(int) CFGMR3QueryInteger(PCFGMNODE pNode, const char *pszName, uint64_t *pu64)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLeaf->enmType == CFGMVALUETYPE_INTEGER)
+            *pu64 = pLeaf->Value.Integer.u64;
+        else
+            rc = VERR_CFGM_NOT_INTEGER;
+    }
+    return rc;
+}
+
+
+/**
+ * Query integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu64            Where to store the integer value. This is set to the default on failure.
+ * @param   u64Def          The default value. This is always set.
+ */
+VMMR3DECL(int) CFGMR3QueryIntegerDef(PCFGMNODE pNode, const char *pszName, uint64_t *pu64, uint64_t u64Def)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLeaf->enmType == CFGMVALUETYPE_INTEGER)
+            *pu64 = pLeaf->Value.Integer.u64;
+        else
+            rc = VERR_CFGM_NOT_INTEGER;
+    }
+
+    if (RT_FAILURE(rc))
+    {
+        *pu64 = u64Def;
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+            rc = VINF_SUCCESS;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Query zero terminated character value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of a zero terminate character value.
+ * @param   pszString       Where to store the string.
+ * @param   cchString       Size of the string buffer. (Includes terminator.)
+ */
+VMMR3DECL(int) CFGMR3QueryString(PCFGMNODE pNode, const char *pszName, char *pszString, size_t cchString)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLeaf->enmType == CFGMVALUETYPE_STRING)
+        {
+            size_t cbSrc = pLeaf->Value.String.cb;
+            if (cchString >= cbSrc)
+            {
+                memcpy(pszString, pLeaf->Value.String.psz, cbSrc);
+                memset(pszString + cbSrc, 0, cchString - cbSrc);
+            }
+            else
+                rc = VERR_CFGM_NOT_ENOUGH_SPACE;
+        }
+        else
+            rc = VERR_CFGM_NOT_STRING;
+    }
+    return rc;
+}
+
+
+/**
+ * Query zero terminated character value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of a zero terminate character value.
+ * @param   pszString       Where to store the string. This will not be set on overflow error.
+ * @param   cchString       Size of the string buffer. (Includes terminator.)
+ * @param   pszDef          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryStringDef(PCFGMNODE pNode, const char *pszName, char *pszString, size_t cchString, const char *pszDef)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLeaf->enmType == CFGMVALUETYPE_STRING)
+        {
+            size_t cbSrc = pLeaf->Value.String.cb;
+            if (cchString >= cbSrc)
+            {
+                memcpy(pszString, pLeaf->Value.String.psz, cbSrc);
+                memset(pszString + cbSrc, 0, cchString - cbSrc);
+            }
+            else
+                rc = VERR_CFGM_NOT_ENOUGH_SPACE;
+        }
+        else
+            rc = VERR_CFGM_NOT_STRING;
+    }
+
+    if (RT_FAILURE(rc) && rc != VERR_CFGM_NOT_ENOUGH_SPACE)
+    {
+        size_t cchDef = strlen(pszDef);
+        if (cchString > cchDef)
+        {
+            memcpy(pszString, pszDef, cchDef);
+            memset(pszString + cchDef, 0, cchString - cchDef);
+            if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+                rc = VINF_SUCCESS;
+        }
+        else if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+            rc = VERR_CFGM_NOT_ENOUGH_SPACE;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Query byte string value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of a byte string value.
+ * @param   pvData          Where to store the binary data.
+ * @param   cbData          Size of buffer pvData points too.
+ */
+VMMR3DECL(int) CFGMR3QueryBytes(PCFGMNODE pNode, const char *pszName, void *pvData, size_t cbData)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLeaf->enmType == CFGMVALUETYPE_BYTES)
+        {
+            if (cbData >= pLeaf->Value.Bytes.cb)
+            {
+                memcpy(pvData, pLeaf->Value.Bytes.pau8, pLeaf->Value.Bytes.cb);
+                memset((char *)pvData + pLeaf->Value.Bytes.cb, 0, cbData - pLeaf->Value.Bytes.cb);
+            }
+            else
+                rc = VERR_CFGM_NOT_ENOUGH_SPACE;
+        }
+        else
+            rc = VERR_CFGM_NOT_BYTES;
+    }
+    return rc;
+}
+
+
+/**
+ * Validate one level of a configuration node.
+ *
+ * This replaces the CFGMR3AreChildrenValid and CFGMR3AreValuesValid APIs.
+ *
+ * @returns VBox status code.
+ *
+ *          When an error is returned, both VMSetError and AssertLogRelMsgFailed
+ *          have been called.  So, all the caller needs to do is to propagate
+ *          the error status up to PDM.
+ *
+ * @param   pNode               The node to validate.
+ * @param   pszNode             The node path, always ends with a slash.  Use
+ *                              "/" for the root config node.
+ * @param   pszValidValues      Patterns describing the valid value names.  See
+ *                              RTStrSimplePatternMultiMatch for details on the
+ *                              pattern syntax.
+ * @param   pszValidNodes       Patterns describing the valid node (key) names.
+ *                              See RTStrSimplePatternMultiMatch for details on
+ *                              the pattern syntax.
+ * @param   pszWho              Who is calling.
+ * @param   uInstance           The instance number of the caller.
+ */
+VMMR3DECL(int) CFGMR3ValidateConfig(PCFGMNODE pNode, const char *pszNode,
+                                    const char *pszValidValues, const char *pszValidNodes,
+                                    const char *pszWho, uint32_t uInstance)
+{
+    /* Input validation. */
+    AssertPtrNullReturn(pNode, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszNode, VERR_INVALID_POINTER);
+    Assert(*pszNode && pszNode[strlen(pszNode) - 1] == '/');
+    AssertPtrReturn(pszValidValues, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszValidNodes, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszWho, VERR_INVALID_POINTER);
+
+    if (pNode)
+    {
+        /*
+         * Enumerate the leaves and check them against pszValidValues.
+         */
+        for (PCFGMLEAF pLeaf = pNode->pFirstLeaf; pLeaf; pLeaf = pLeaf->pNext)
+        {
+            if (!RTStrSimplePatternMultiMatch(pszValidValues, RTSTR_MAX,
+                                              pLeaf->szName, pLeaf->cchName,
+                                              NULL))
+            {
+                AssertLogRelMsgFailed(("%s/%u: Value '%s%s' didn't match '%s'\n",
+                                       pszWho, uInstance, pszNode, pLeaf->szName, pszValidValues));
+                return VMSetError(pNode->pVM, VERR_CFGM_CONFIG_UNKNOWN_VALUE, RT_SRC_POS,
+                                  N_("Unknown configuration value '%s%s' found in the configuration of %s instance #%u"),
+                                  pszNode, pLeaf->szName, pszWho, uInstance);
+            }
+
+        }
+
+        /*
+         * Enumerate the child nodes and check them against pszValidNodes.
+         */
+        for (PCFGMNODE pChild = pNode->pFirstChild; pChild; pChild = pChild->pNext)
+        {
+            if (!RTStrSimplePatternMultiMatch(pszValidNodes, RTSTR_MAX,
+                                              pChild->szName, pChild->cchName,
+                                              NULL))
+            {
+                AssertLogRelMsgFailed(("%s/%u: Node '%s%s' didn't match '%s'\n",
+                                       pszWho, uInstance, pszNode, pChild->szName, pszValidNodes));
+                return VMSetError(pNode->pVM, VERR_CFGM_CONFIG_UNKNOWN_NODE, RT_SRC_POS,
+                                  N_("Unknown configuration node '%s%s' found in the configuration of %s instance #%u"),
+                                  pszNode, pChild->szName, pszWho, uInstance);
+            }
+        }
+    }
+
+    /* All is well. */
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Populates the CFGM tree with the default configuration.
+ *
+ * This assumes an empty tree and is intended for testcases and such that only
+ * need to do very small adjustments to the config.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @internal
+ */
+VMMR3DECL(int) CFGMR3ConstructDefaultTree(PVM pVM)
+{
+    int rc;
+    int rcAll = VINF_SUCCESS;
+#define UPDATERC() do { if (RT_FAILURE(rc) && RT_SUCCESS(rcAll)) rcAll = rc; } while (0)
+
+    PCFGMNODE pRoot = CFGMR3GetRoot(pVM);
+    AssertReturn(pRoot, VERR_WRONG_ORDER);
+
+    /*
+     * Create VM default values.
+     */
+    rc = CFGMR3InsertString(pRoot,  "Name",                 "Default VM");
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "RamSize",              128U * _1M);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "RamHoleSize",          512U * _1M);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "TimerMillies",         10);
+    UPDATERC();
+
+    /*
+     * PDM.
+     */
+    PCFGMNODE pPdm;
+    rc = CFGMR3InsertNode(pRoot, "PDM", &pPdm);
+    UPDATERC();
+    PCFGMNODE pDevices = NULL;
+    rc = CFGMR3InsertNode(pPdm, "Devices", &pDevices);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pDevices, "LoadBuiltin",       1); /* boolean */
+    UPDATERC();
+    PCFGMNODE pDrivers = NULL;
+    rc = CFGMR3InsertNode(pPdm, "Drivers", &pDrivers);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pDrivers, "LoadBuiltin",       1); /* boolean */
+    UPDATERC();
+
+
+    /*
+     * Devices
+     */
+    pDevices = NULL;
+    rc = CFGMR3InsertNode(pRoot, "Devices", &pDevices);
+    UPDATERC();
+    /* device */
+    PCFGMNODE pDev = NULL;
+    PCFGMNODE pInst = NULL;
+    PCFGMNODE pCfg = NULL;
+#if 0
+    PCFGMNODE pLunL0 = NULL;
+    PCFGMNODE pLunL1 = NULL;
+#endif
+
+    /*
+     * PC Arch.
+     */
+    rc = CFGMR3InsertNode(pDevices, "pcarch", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * PC Bios.
+     */
+    rc = CFGMR3InsertNode(pDevices, "pcbios", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "BootDevice0",          "IDE");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "BootDevice1",          "NONE");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "BootDevice2",          "NONE");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "BootDevice3",          "NONE");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "HardDiskDevice",       "piix3ide");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "FloppyDevice",         "");
+    UPDATERC();
+    RTUUID Uuid;
+    RTUuidClear(&Uuid);
+    rc = CFGMR3InsertBytes(pCfg,    "UUID", &Uuid, sizeof(Uuid));
+    UPDATERC();
+
+    /*
+     * PCI bus.
+     */
+    rc = CFGMR3InsertNode(pDevices, "pci", &pDev); /* piix3 */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * PS/2 keyboard & mouse
+     */
+    rc = CFGMR3InsertNode(pDevices, "pckbd", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+#if 0
+    rc = CFGMR3InsertNode(pInst,    "LUN#0", &pLunL0);
+    UPDATERC();
+    rc = CFGMR3InsertString(pLunL0, "Driver",               "KeyboardQueue");
+    UPDATERC();
+    rc = CFGMR3InsertNode(pLunL0,   "Config", &pCfg);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "QueueSize",            64);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pLunL0,   "AttachedDriver", &pLunL1);
+    UPDATERC();
+    rc = CFGMR3InsertString(pLunL1, "Driver",               "MainKeyboard");
+    UPDATERC();
+    rc = CFGMR3InsertNode(pLunL1,   "Config", &pCfg);
+    UPDATERC();
+#endif
+#if 0
+    rc = CFGMR3InsertNode(pInst,    "LUN#1", &pLunL0);
+    UPDATERC();
+    rc = CFGMR3InsertString(pLunL0, "Driver",               "MouseQueue");
+    UPDATERC();
+    rc = CFGMR3InsertNode(pLunL0,   "Config", &pCfg);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "QueueSize",            128);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pLunL0,   "AttachedDriver", &pLunL1);
+    UPDATERC();
+    rc = CFGMR3InsertString(pLunL1, "Driver",               "MainMouse");
+    UPDATERC();
+    rc = CFGMR3InsertNode(pLunL1,   "Config", &pCfg);
+    UPDATERC();
+#endif
+
+    /*
+     * i8254 Programmable Interval Timer And Dummy Speaker
+     */
+    rc = CFGMR3InsertNode(pDevices, "i8254", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+#ifdef DEBUG
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+#endif
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * i8259 Programmable Interrupt Controller.
+     */
+    rc = CFGMR3InsertNode(pDevices, "i8259", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * RTC MC146818.
+     */
+    rc = CFGMR3InsertNode(pDevices, "mc146818", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * VGA.
+     */
+    rc = CFGMR3InsertNode(pDevices, "vga", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "VRamSize",             4 * _1M);
+    UPDATERC();
+
+    /* Bios logo. */
+    rc = CFGMR3InsertInteger(pCfg,  "FadeIn",               1);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "FadeOut",              1);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "LogoTime",             0);
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "LogoFile",             "");
+    UPDATERC();
+
+#if 0
+    rc = CFGMR3InsertNode(pInst,    "LUN#0", &pLunL0);
+    UPDATERC();
+    rc = CFGMR3InsertString(pLunL0, "Driver",               "MainDisplay");
+    UPDATERC();
+#endif
+
+    /*
+     * IDE controller.
+     */
+    rc = CFGMR3InsertNode(pDevices, "piix3ide", &pDev); /* piix3 */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * VMMDev.
+     */
+    rc = CFGMR3InsertNode(pDevices, "VMMDev", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1); /* boolean */
+    UPDATERC();
+
+
+    /*
+     * ...
+     */
+
+#undef UPDATERC
+    return rcAll;
+}
+
+
+
+
+/**
+ * Resolves a path reference to a child node.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszPath         Path to the child node.
+ * @param   ppChild         Where to store the pointer to the child node.
+ */
+static int cfgmR3ResolveNode(PCFGMNODE pNode, const char *pszPath, PCFGMNODE *ppChild)
+{
+    *ppChild = NULL;
+    if (!pNode)
+        return VERR_CFGM_NO_PARENT;
+    PCFGMNODE pChild = NULL;
+    for (;;)
+    {
+        /* skip leading slashes. */
+        while (*pszPath == '/')
+            pszPath++;
+
+        /* End of path? */
+        if (!*pszPath)
+        {
+            if (!pChild)
+                return VERR_CFGM_INVALID_CHILD_PATH;
+            *ppChild = pChild;
+            return VINF_SUCCESS;
+        }
+
+        /* find end of component. */
+        const char *pszNext = strchr(pszPath, '/');
+        if (!pszNext)
+            pszNext = strchr(pszPath,  '\0');
+        RTUINT cchName = pszNext - pszPath;
+
+        /* search child list. */
+        pChild = pNode->pFirstChild;
+        for ( ; pChild; pChild = pChild->pNext)
+            if (pChild->cchName == cchName)
+            {
+                int iDiff = memcmp(pszPath, pChild->szName, cchName);
+                if (iDiff <= 0)
+                {
+                    if (iDiff != 0)
+                        pChild = NULL;
+                    break;
+                }
+            }
+        if (!pChild)
+            return VERR_CFGM_CHILD_NOT_FOUND;
+
+        /* next iteration */
+        pNode = pChild;
+        pszPath = pszNext;
+    }
+
+    /* won't get here */
+}
+
+
+/**
+ * Resolves a path reference to a child node.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of a byte string value.
+ * @param   ppLeaf          Where to store the pointer to the leaf node.
+ */
+static int cfgmR3ResolveLeaf(PCFGMNODE pNode, const char *pszName, PCFGMLEAF *ppLeaf)
+{
+    *ppLeaf = NULL;
+    if (!pNode)
+        return VERR_CFGM_NO_PARENT;
+
+    size_t      cchName = strlen(pszName);
+    PCFGMLEAF   pLeaf   = pNode->pFirstLeaf;
+    while (pLeaf)
+    {
+        if (cchName == pLeaf->cchName)
+        {
+            int iDiff = memcmp(pszName, pLeaf->szName, cchName);
+            if (iDiff <= 0)
+            {
+                if (iDiff != 0)
+                    break;
+                *ppLeaf = pLeaf;
+                return VINF_SUCCESS;
+            }
+        }
+
+        /* next */
+        pLeaf = pLeaf->pNext;
+    }
+    return VERR_CFGM_VALUE_NOT_FOUND;
+}
+
+
+
+/**
+ * Creates a CFGM tree.
+ *
+ * This is intended for creating device/driver configs can be
+ * passed around and later attached to the main tree in the
+ * correct location.
+ *
+ * @returns Pointer to the root node, NULL on error (out of memory or invalid
+ *          VM handle).
+ * @param   pUVM            The user mode VM handle.  For testcase (and other
+ *                          purposes, NULL can be used.  However, the resulting
+ *                          tree cannot be inserted into a tree that has a
+ *                          non-NULL value.  Using NULL can be usedful for
+ *                          testcases and similar, non VMM uses.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3CreateTree(PUVM pUVM)
+{
+    if (pUVM)
+    {
+        UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+        VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, NULL);
+    }
+
+    PCFGMNODE pNew;
+    if (pUVM)
+        pNew = (PCFGMNODE)MMR3HeapAllocU(pUVM, MM_TAG_CFGM, sizeof(*pNew));
+    else
+        pNew = (PCFGMNODE)RTMemAlloc(sizeof(*pNew));
+    if (pNew)
+    {
+        pNew->pPrev         = NULL;
+        pNew->pNext         = NULL;
+        pNew->pParent       = NULL;
+        pNew->pFirstChild   = NULL;
+        pNew->pFirstLeaf    = NULL;
+        pNew->pVM           = pUVM ? pUVM->pVM : NULL;
+        pNew->fRestrictedRoot = false;
+        pNew->cchName       = 0;
+        pNew->szName[0]     = 0;
+    }
+    return pNew;
+}
+
+
+/**
+ * Duplicates a CFGM sub-tree or a full tree.
+ *
+ * @returns Pointer to the root node.  NULL if we run out of memory or the
+ *          input parameter is NULL.
+ * @param   pRoot       The root of the tree to duplicate.
+ * @param   ppCopy      Where to return the root of the duplicate.
+ */
+VMMR3DECL(int) CFGMR3DuplicateSubTree(PCFGMNODE pRoot, PCFGMNODE *ppCopy)
+{
+    AssertPtrReturn(pRoot, VERR_INVALID_POINTER);
+
+    /*
+     * Create a new tree.
+     */
+    PCFGMNODE pNewRoot = CFGMR3CreateTree(pRoot->pVM ? pRoot->pVM->pUVM : NULL);
+    if (!pNewRoot)
+        return VERR_NO_MEMORY;
+
+    /*
+     * Duplicate the content.
+     */
+    int         rc      = VINF_SUCCESS;
+    PCFGMNODE   pSrcCur = pRoot;
+    PCFGMNODE   pDstCur = pNewRoot;
+    for (;;)
+    {
+        if (   !pDstCur->pFirstChild
+            && !pDstCur->pFirstLeaf)
+        {
+            /*
+             * Values first.
+             */
+            /** @todo this isn't the most efficient way to do it. */
+            for (PCFGMLEAF pLeaf = pSrcCur->pFirstLeaf; pLeaf && RT_SUCCESS(rc); pLeaf = pLeaf->pNext)
+                rc = CFGMR3InsertValue(pDstCur, pLeaf);
+
+            /*
+             * Insert immediate child nodes.
+             */
+            /** @todo this isn't the most efficient way to do it. */
+            for (PCFGMNODE pChild = pSrcCur->pFirstChild; pChild && RT_SUCCESS(rc); pChild = pChild->pNext)
+                rc = CFGMR3InsertNode(pDstCur, pChild->szName, NULL);
+
+            AssertLogRelRCBreak(rc);
+        }
+
+        /*
+         * Deep copy of the children.
+         */
+        if (pSrcCur->pFirstChild)
+        {
+            Assert(pDstCur->pFirstChild && !strcmp(pDstCur->pFirstChild->szName, pSrcCur->pFirstChild->szName));
+            pSrcCur = pSrcCur->pFirstChild;
+            pDstCur = pDstCur->pFirstChild;
+        }
+        /*
+         * If it's the root node, we're done.
+         */
+        else if (pSrcCur == pRoot)
+            break;
+        else
+        {
+            /*
+             * Upon reaching the end of a sibling list, we must ascend and
+             * resume the sibiling walk on an previous level.
+             */
+            if (!pSrcCur->pNext)
+            {
+                do
+                {
+                    pSrcCur = pSrcCur->pParent;
+                    pDstCur = pDstCur->pParent;
+                } while (!pSrcCur->pNext && pSrcCur != pRoot);
+                if (pSrcCur == pRoot)
+                    break;
+            }
+
+            /*
+             * Next sibling.
+             */
+            Assert(pDstCur->pNext && !strcmp(pDstCur->pNext->szName, pSrcCur->pNext->szName));
+            pSrcCur = pSrcCur->pNext;
+            pDstCur = pDstCur->pNext;
+        }
+    }
+
+    if (RT_FAILURE(rc))
+    {
+        CFGMR3RemoveNode(pNewRoot);
+        return rc;
+    }
+
+    *ppCopy = pNewRoot;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Insert subtree.
+ *
+ * This function inserts (no duplication) a tree created by CFGMR3CreateTree()
+ * into the main tree.
+ *
+ * The root node of the inserted subtree will need to be reallocated, which
+ * effectually means that the passed in pSubTree handle becomes invalid
+ * upon successful return. Use the value returned in ppChild instead
+ * of pSubTree.
+ *
+ * @returns VBox status code.
+ * @returns VERR_CFGM_NODE_EXISTS if the final child node name component exists.
+ * @param   pNode       Parent node.
+ * @param   pszName     Name or path of the new child node.
+ * @param   pSubTree    The subtree to insert. Must be returned by CFGMR3CreateTree().
+ * @param   ppChild     Where to store the address of the new child node. (optional)
+ */
+VMMR3DECL(int) CFGMR3InsertSubTree(PCFGMNODE pNode, const char *pszName, PCFGMNODE pSubTree, PCFGMNODE *ppChild)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pNode, VERR_INVALID_POINTER);
+    AssertPtrReturn(pSubTree, VERR_INVALID_POINTER);
+    AssertReturn(pNode != pSubTree, VERR_INVALID_PARAMETER);
+    AssertReturn(!pSubTree->pParent, VERR_INVALID_PARAMETER);
+    AssertReturn(pNode->pVM == pSubTree->pVM, VERR_INVALID_PARAMETER);
+    Assert(!pSubTree->pNext);
+    Assert(!pSubTree->pPrev);
+
+    /*
+     * Use CFGMR3InsertNode to create a new node and then
+     * re-attach the children and leaves of the subtree to it.
+     */
+    PCFGMNODE pNewChild;
+    int rc = CFGMR3InsertNode(pNode, pszName, &pNewChild);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(!pNewChild->pFirstChild);
+        Assert(!pNewChild->pFirstLeaf);
+
+        pNewChild->pFirstChild = pSubTree->pFirstChild;
+        pNewChild->pFirstLeaf = pSubTree->pFirstLeaf;
+        for (PCFGMNODE pChild = pNewChild->pFirstChild; pChild; pChild = pChild->pNext)
+            pChild->pParent = pNewChild;
+
+        if (ppChild)
+            *ppChild = pNewChild;
+
+        /* free the old subtree root */
+        cfgmR3FreeNodeOnly(pSubTree);
+    }
+    return rc;
+}
+
+
+/**
+ * Replaces a (sub-)tree with new one.
+ *
+ * This function removes the exiting (sub-)tree, completely freeing it in the
+ * process, and inserts (no duplication) the specified tree.  The tree can
+ * either be created by CFGMR3CreateTree or CFGMR3DuplicateSubTree.
+ *
+ * @returns VBox status code.
+ * @param   pRoot       The sub-tree to replace.  This node will remain valid
+ *                      after the call.
+ * @param   pNewRoot    The tree to replace @a pRoot with.  This not will
+ *                      become invalid after a successful call.
+ */
+VMMR3DECL(int) CFGMR3ReplaceSubTree(PCFGMNODE pRoot, PCFGMNODE pNewRoot)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pRoot, VERR_INVALID_POINTER);
+    AssertPtrReturn(pNewRoot, VERR_INVALID_POINTER);
+    AssertReturn(pRoot != pNewRoot, VERR_INVALID_PARAMETER);
+    AssertReturn(!pNewRoot->pParent, VERR_INVALID_PARAMETER);
+    AssertReturn(pNewRoot->pVM == pRoot->pVM, VERR_INVALID_PARAMETER);
+    AssertReturn(!pNewRoot->pNext, VERR_INVALID_PARAMETER);
+    AssertReturn(!pNewRoot->pPrev, VERR_INVALID_PARAMETER);
+
+    /*
+     * Free the current properties fo pRoot.
+     */
+    while (pRoot->pFirstChild)
+        CFGMR3RemoveNode(pRoot->pFirstChild);
+
+    while (pRoot->pFirstLeaf)
+        cfgmR3RemoveLeaf(pRoot, pRoot->pFirstLeaf);
+
+    /*
+     * Copy all the properties from the new root to the current one.
+     */
+    pRoot->pFirstLeaf       = pNewRoot->pFirstLeaf;
+    pRoot->pFirstChild      = pNewRoot->pFirstChild;
+    for (PCFGMNODE pChild = pRoot->pFirstChild; pChild; pChild = pChild->pNext)
+        pChild->pParent     = pRoot;
+
+    cfgmR3FreeNodeOnly(pNewRoot);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Copies all values and keys from one tree onto another.
+ *
+ * The flags control what happens to keys and values with the same name
+ * existing in both source and destination.
+ *
+ * @returns VBox status code.
+ * @param   pDstTree            The destination tree.
+ * @param   pSrcTree            The source tree.
+ * @param   fFlags              Copy flags, see CFGM_COPY_FLAGS_XXX.
+ */
+VMMR3DECL(int) CFGMR3CopyTree(PCFGMNODE pDstTree, PCFGMNODE pSrcTree, uint32_t fFlags)
+{
+    /*
+     * Input validation.
+     */
+    AssertPtrReturn(pSrcTree, VERR_INVALID_POINTER);
+    AssertPtrReturn(pDstTree, VERR_INVALID_POINTER);
+    AssertReturn(pDstTree != pSrcTree, VERR_INVALID_PARAMETER);
+    AssertReturn(!(fFlags & ~(CFGM_COPY_FLAGS_VALUE_DISP_MASK | CFGM_COPY_FLAGS_KEY_DISP_MASK)), VERR_INVALID_PARAMETER);
+    AssertReturn(   (fFlags & CFGM_COPY_FLAGS_VALUE_DISP_MASK) != CFGM_COPY_FLAGS_RESERVED_VALUE_DISP_0
+                 && (fFlags & CFGM_COPY_FLAGS_VALUE_DISP_MASK) != CFGM_COPY_FLAGS_RESERVED_VALUE_DISP_1,
+                 VERR_INVALID_PARAMETER);
+    AssertReturn((fFlags & CFGM_COPY_FLAGS_KEY_DISP_MASK) != CFGM_COPY_FLAGS_RESERVED_KEY_DISP,
+                 VERR_INVALID_PARAMETER);
+
+    /*
+     * Copy the values.
+     */
+    int rc;
+    for (PCFGMLEAF pValue = CFGMR3GetFirstValue(pSrcTree); pValue; pValue = CFGMR3GetNextValue(pValue))
+    {
+        rc = CFGMR3InsertValue(pDstTree, pValue);
+        if (rc == VERR_CFGM_LEAF_EXISTS)
+        {
+            if ((fFlags & CFGM_COPY_FLAGS_VALUE_DISP_MASK) == CFGM_COPY_FLAGS_REPLACE_VALUES)
+            {
+                rc = CFGMR3RemoveValue(pDstTree, pValue->szName);
+                if (RT_FAILURE(rc))
+                    break;
+                rc = CFGMR3InsertValue(pDstTree, pValue);
+            }
+            else
+                rc = VINF_SUCCESS;
+        }
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Copy/merge the keys - merging results in recursion.
+     */
+    for (PCFGMNODE pSrcChild = CFGMR3GetFirstChild(pSrcTree); pSrcChild; pSrcChild = CFGMR3GetNextChild(pSrcChild))
+    {
+        PCFGMNODE pDstChild = CFGMR3GetChild(pDstTree, pSrcChild->szName);
+        if (   pDstChild
+            && (fFlags & CFGM_COPY_FLAGS_KEY_DISP_MASK) == CFGM_COPY_FLAGS_REPLACE_KEYS)
+        {
+            CFGMR3RemoveNode(pDstChild);
+            pDstChild = NULL;
+        }
+        if (!pDstChild)
+        {
+            PCFGMNODE pChildCopy;
+            rc = CFGMR3DuplicateSubTree(pSrcChild, &pChildCopy);
+            AssertRCReturn(rc, rc);
+            rc = CFGMR3InsertSubTree(pDstTree, pSrcChild->szName, pChildCopy, NULL);
+            AssertRCReturnStmt(rc, CFGMR3RemoveNode(pChildCopy), rc);
+        }
+        else if ((fFlags & CFGM_COPY_FLAGS_KEY_DISP_MASK) == CFGM_COPY_FLAGS_MERGE_KEYS)
+        {
+            rc = CFGMR3CopyTree(pDstChild, pSrcChild, fFlags);
+            AssertRCReturn(rc, rc);
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Compares two names.
+ *
+ * @returns Similar to memcpy.
+ * @param   pszName1            The first name.
+ * @param   cchName1            The length of the first name.
+ * @param   pszName2            The second name.
+ * @param   cchName2            The length of the second name.
+ */
+DECLINLINE(int) cfgmR3CompareNames(const char *pszName1, size_t cchName1, const char *pszName2, size_t cchName2)
+{
+    int iDiff;
+    if (cchName1 <= cchName2)
+    {
+        iDiff = memcmp(pszName1, pszName2, cchName1);
+        if (!iDiff && cchName1 < cchName2)
+            iDiff = -1;
+    }
+    else
+    {
+        iDiff = memcmp(pszName1, pszName2, cchName2);
+        if (!iDiff)
+            iDiff = 1;
+    }
+    return iDiff;
+}
+
+
+/**
+ * Insert a node.
+ *
+ * @returns VBox status code.
+ * @returns VERR_CFGM_NODE_EXISTS if the final child node name component exists.
+ * @param   pNode       Parent node.
+ * @param   pszName     Name or path of the new child node.
+ * @param   ppChild     Where to store the address of the new child node. (optional)
+ */
+VMMR3DECL(int) CFGMR3InsertNode(PCFGMNODE pNode, const char *pszName, PCFGMNODE *ppChild)
+{
+    int rc;
+    if (pNode)
+    {
+        /*
+         * If given a path we have to deal with it component by component.
+         */
+        while (*pszName == '/')
+            pszName++;
+        if (strchr(pszName, '/'))
+        {
+            char *pszDup = RTStrDup(pszName);
+            if (pszDup)
+            {
+                char *psz = pszDup;
+                for (;;)
+                {
+                    /* Terminate at '/' and find the next component. */
+                    char *pszNext = strchr(psz, '/');
+                    if (pszNext)
+                    {
+                        *pszNext++ = '\0';
+                        while (*pszNext == '/')
+                            pszNext++;
+                        if (*pszNext == '\0')
+                            pszNext = NULL;
+                    }
+
+                    /* does it exist? */
+                    PCFGMNODE pChild = CFGMR3GetChild(pNode, psz);
+                    if (!pChild)
+                    {
+                        /* no, insert it */
+                        rc = CFGMR3InsertNode(pNode, psz, &pChild);
+                        if (RT_FAILURE(rc))
+                            break;
+                        if (!pszNext)
+                        {
+                            if (ppChild)
+                                *ppChild = pChild;
+                            break;
+                        }
+
+                    }
+                    /* if last component fail */
+                    else if (!pszNext)
+                    {
+                        rc = VERR_CFGM_NODE_EXISTS;
+                        break;
+                    }
+
+                    /* next */
+                    pNode = pChild;
+                    psz = pszNext;
+                }
+                RTStrFree(pszDup);
+            }
+            else
+                rc = VERR_NO_TMP_MEMORY;
+        }
+        /*
+         * Not multicomponent, just make sure it's a non-zero name.
+         */
+        else if (*pszName)
+        {
+            /*
+             * Check if already exists and find last node in chain.
+             */
+            size_t    cchName = strlen(pszName);
+            PCFGMNODE pPrev   = NULL;
+            PCFGMNODE pNext   = pNode->pFirstChild;
+            if (pNext)
+            {
+                for ( ; pNext; pPrev = pNext, pNext = pNext->pNext)
+                {
+                    int iDiff = cfgmR3CompareNames(pszName, cchName, pNext->szName, pNext->cchName);
+                    if (iDiff <= 0)
+                    {
+                        if (!iDiff)
+                            return VERR_CFGM_NODE_EXISTS;
+                        break;
+                    }
+                }
+            }
+
+            /*
+             * Allocate and init node.
+             */
+            PCFGMNODE pNew = (PCFGMNODE)cfgmR3MemAlloc(pNode->pVM, MM_TAG_CFGM, sizeof(*pNew) + cchName);
+            if (pNew)
+            {
+                pNew->pParent       = pNode;
+                pNew->pFirstChild   = NULL;
+                pNew->pFirstLeaf    = NULL;
+                pNew->pVM           = pNode->pVM;
+                pNew->fRestrictedRoot = false;
+                pNew->cchName       = cchName;
+                memcpy(pNew->szName, pszName, cchName + 1);
+
+                /*
+                 * Insert into child list.
+                 */
+                pNew->pPrev         = pPrev;
+                if (pPrev)
+                    pPrev->pNext    = pNew;
+                else
+                    pNode->pFirstChild = pNew;
+                pNew->pNext         = pNext;
+                if (pNext)
+                    pNext->pPrev    = pNew;
+
+                if (ppChild)
+                    *ppChild = pNew;
+                rc = VINF_SUCCESS;
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+        else
+        {
+            rc = VERR_CFGM_INVALID_NODE_PATH;
+            AssertMsgFailed(("Invalid path %s\n", pszName));
+        }
+    }
+    else
+    {
+        rc = VERR_CFGM_NO_PARENT;
+        AssertMsgFailed(("No parent! path %s\n", pszName));
+    }
+
+    return rc;
+}
+
+
+/**
+ * Insert a node, format string name.
+ *
+ * @returns VBox status     code.
+ * @param   pNode           Parent node.
+ * @param   ppChild         Where to store the address of the new child node. (optional)
+ * @param   pszNameFormat   Name of or path the new child node.
+ * @param   ...             Name format arguments.
+ */
+VMMR3DECL(int) CFGMR3InsertNodeF(PCFGMNODE pNode, PCFGMNODE *ppChild, const char *pszNameFormat, ...)
+{
+    va_list Args;
+    va_start(Args,  pszNameFormat);
+    int rc = CFGMR3InsertNodeFV(pNode, ppChild, pszNameFormat, Args);
+    va_end(Args);
+    return rc;
+}
+
+
+/**
+ * Insert a node, format string name.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   ppChild         Where to store the address of the new child node. (optional)
+ * @param   pszNameFormat   Name or path of the new child node.
+ * @param   Args            Name format arguments.
+ */
+VMMR3DECL(int) CFGMR3InsertNodeFV(PCFGMNODE pNode, PCFGMNODE *ppChild, const char *pszNameFormat, va_list Args)
+{
+    int     rc;
+    char   *pszName;
+    RTStrAPrintfV(&pszName, pszNameFormat, Args);
+    if (pszName)
+    {
+        rc = CFGMR3InsertNode(pNode, pszName, ppChild);
+        RTStrFree(pszName);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+    return rc;
+}
+
+
+/**
+ * Marks the node as the root of a restricted subtree, i.e. the end of
+ * a CFGMR3GetParent() journey.
+ *
+ * @param   pNode       The node to mark.
+ */
+VMMR3DECL(void) CFGMR3SetRestrictedRoot(PCFGMNODE pNode)
+{
+    if (pNode)
+        pNode->fRestrictedRoot = true;
+}
+
+
+/**
+ * Insert a node.
+ *
+ * @returns VBox status code.
+ * @param   pNode       Parent node.
+ * @param   pszName     Name of the new child node.
+ * @param   ppLeaf      Where to store the new leaf.
+ *                      The caller must fill in the enmType and Value fields!
+ */
+static int cfgmR3InsertLeaf(PCFGMNODE pNode, const char *pszName, PCFGMLEAF *ppLeaf)
+{
+    int rc;
+    if (*pszName)
+    {
+        if (pNode)
+        {
+            /*
+             * Check if already exists and find last node in chain.
+             */
+            size_t    cchName  = strlen(pszName);
+            PCFGMLEAF pPrev    = NULL;
+            PCFGMLEAF pNext    = pNode->pFirstLeaf;
+            if (pNext)
+            {
+                for ( ; pNext; pPrev = pNext, pNext = pNext->pNext)
+                {
+                    int iDiff = cfgmR3CompareNames(pszName, cchName, pNext->szName, pNext->cchName);
+                    if (iDiff <= 0)
+                    {
+                        if (!iDiff)
+                            return VERR_CFGM_LEAF_EXISTS;
+                        break;
+                    }
+                }
+            }
+
+            /*
+             * Allocate and init node.
+             */
+            PCFGMLEAF pNew = (PCFGMLEAF)cfgmR3MemAlloc(pNode->pVM, MM_TAG_CFGM, sizeof(*pNew) + cchName);
+            if (pNew)
+            {
+                pNew->cchName       = cchName;
+                memcpy(pNew->szName, pszName, cchName + 1);
+
+                /*
+                 * Insert into child list.
+                 */
+                pNew->pPrev         = pPrev;
+                if (pPrev)
+                    pPrev->pNext    = pNew;
+                else
+                    pNode->pFirstLeaf = pNew;
+                pNew->pNext         = pNext;
+                if (pNext)
+                    pNext->pPrev    = pNew;
+
+                *ppLeaf = pNew;
+                rc = VINF_SUCCESS;
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+        else
+            rc = VERR_CFGM_NO_PARENT;
+    }
+    else
+        rc = VERR_CFGM_INVALID_CHILD_PATH;
+    return rc;
+}
+
+
+/**
+ * Removes a node.
+ *
+ * @param   pNode       The node to remove.
+ */
+VMMR3DECL(void) CFGMR3RemoveNode(PCFGMNODE pNode)
+{
+    if (pNode)
+    {
+        /*
+         * Free children.
+         */
+        while (pNode->pFirstChild)
+            CFGMR3RemoveNode(pNode->pFirstChild);
+
+        /*
+         * Free leaves.
+         */
+        while (pNode->pFirstLeaf)
+            cfgmR3RemoveLeaf(pNode, pNode->pFirstLeaf);
+
+        /*
+         * Unlink ourselves.
+         */
+        if (pNode->pPrev)
+            pNode->pPrev->pNext = pNode->pNext;
+        else
+        {
+            if (pNode->pParent)
+                pNode->pParent->pFirstChild = pNode->pNext;
+            else if (   pNode->pVM                         /* might be a different tree */
+                     && pNode == pNode->pVM->cfgm.s.pRoot)
+                pNode->pVM->cfgm.s.pRoot = NULL;
+        }
+        if (pNode->pNext)
+            pNode->pNext->pPrev = pNode->pPrev;
+
+        /*
+         * Free ourselves.
+         */
+        cfgmR3FreeNodeOnly(pNode);
+    }
+}
+
+
+/**
+ * Removes a leaf.
+ *
+ * @param   pNode   Parent node.
+ * @param   pLeaf   Leaf to remove.
+ */
+static void cfgmR3RemoveLeaf(PCFGMNODE pNode, PCFGMLEAF pLeaf)
+{
+    if (pNode && pLeaf)
+    {
+        /*
+         * Unlink.
+         */
+        if (pLeaf->pPrev)
+            pLeaf->pPrev->pNext = pLeaf->pNext;
+        else
+            pNode->pFirstLeaf = pLeaf->pNext;
+        if (pLeaf->pNext)
+            pLeaf->pNext->pPrev = pLeaf->pPrev;
+
+        /*
+         * Free value and node.
+         */
+        cfgmR3FreeValue(pNode->pVM, pLeaf);
+        pLeaf->pNext = NULL;
+        pLeaf->pPrev = NULL;
+        cfgmR3MemFree(pNode->pVM, pLeaf);
+    }
+}
+
+
+/**
+ * Frees whatever resources the leaf value is owning.
+ *
+ * Use this before assigning a new value to a leaf.
+ * The caller must either free the leaf or assign a new value to it.
+ *
+ * @param   pVM         The cross context VM structure, if the tree
+ *                      is associated with one.
+ * @param   pLeaf       Pointer to the leaf which value should be free.
+ */
+static void cfgmR3FreeValue(PVM pVM, PCFGMLEAF pLeaf)
+{
+    if (pLeaf)
+    {
+        switch (pLeaf->enmType)
+        {
+            case CFGMVALUETYPE_BYTES:
+                cfgmR3MemFree(pVM, pLeaf->Value.Bytes.pau8);
+                pLeaf->Value.Bytes.pau8 = NULL;
+                pLeaf->Value.Bytes.cb = 0;
+                break;
+
+            case CFGMVALUETYPE_STRING:
+                cfgmR3StrFree(pVM, pLeaf->Value.String.psz);
+                pLeaf->Value.String.psz = NULL;
+                pLeaf->Value.String.cb = 0;
+                break;
+
+            case CFGMVALUETYPE_INTEGER:
+                break;
+        }
+        pLeaf->enmType = (CFGMVALUETYPE)0;
+    }
+}
+
+/**
+ * Destroys a tree created with CFGMR3CreateTree or CFGMR3DuplicateSubTree.
+ *
+ * @returns VBox status code.
+ * @param   pRoot       The root node of the tree.
+ */
+VMMR3DECL(int) CFGMR3DestroyTree(PCFGMNODE pRoot)
+{
+    if (!pRoot)
+        return VINF_SUCCESS;
+    AssertReturn(!pRoot->pParent, VERR_INVALID_PARAMETER);
+    AssertReturn(!pRoot->pVM || pRoot != pRoot->pVM->cfgm.s.pRoot, VERR_ACCESS_DENIED);
+
+    CFGMR3RemoveNode(pRoot);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inserts a new integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   u64Integer      The value.
+ */
+VMMR3DECL(int) CFGMR3InsertInteger(PCFGMNODE pNode, const char *pszName, uint64_t u64Integer)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3InsertLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        pLeaf->enmType = CFGMVALUETYPE_INTEGER;
+        pLeaf->Value.Integer.u64 = u64Integer;
+    }
+    return rc;
+}
+
+
+/**
+ * Inserts a new string value. This variant expects that the caller know the length
+ * of the string already so we can avoid calling strlen() here.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   pszString       The value. Must not be NULL.
+ * @param   cchString       The length of the string excluding the
+ *                          terminator.
+ */
+VMMR3DECL(int) CFGMR3InsertStringN(PCFGMNODE pNode, const char *pszName, const char *pszString, size_t cchString)
+{
+    Assert(RTStrNLen(pszString, cchString) == cchString);
+
+    int rc;
+    if (pNode)
+    {
+        /*
+         * Allocate string object first.
+         */
+        char *pszStringCopy = (char *)cfgmR3StrAlloc(pNode->pVM, MM_TAG_CFGM_STRING, cchString + 1);
+        if (pszStringCopy)
+        {
+            memcpy(pszStringCopy, pszString, cchString);
+            pszStringCopy[cchString] = '\0';
+
+            /*
+             * Create value leaf and set it to string type.
+             */
+            PCFGMLEAF pLeaf;
+            rc = cfgmR3InsertLeaf(pNode, pszName, &pLeaf);
+            if (RT_SUCCESS(rc))
+            {
+                pLeaf->enmType = CFGMVALUETYPE_STRING;
+                pLeaf->Value.String.psz = pszStringCopy;
+                pLeaf->Value.String.cb  = cchString + 1;
+            }
+            else
+                cfgmR3StrFree(pNode->pVM, pszStringCopy);
+        }
+        else
+            rc = VERR_NO_MEMORY;
+    }
+    else
+        rc = VERR_CFGM_NO_PARENT;
+
+    return rc;
+}
+
+
+/**
+ * Inserts a new string value. Calls strlen(pszString) internally; if you know the
+ * length of the string, CFGMR3InsertStringLengthKnown() is faster.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   pszString       The value.
+ */
+VMMR3DECL(int) CFGMR3InsertString(PCFGMNODE pNode, const char *pszName, const char *pszString)
+{
+    return CFGMR3InsertStringN(pNode, pszName, pszString, strlen(pszString));
+}
+
+
+/**
+ * Same as CFGMR3InsertString except the string value given in RTStrPrintfV
+ * fashion.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   pszFormat       The value given as a format string.
+ * @param   va              Argument to pszFormat.
+ */
+VMMR3DECL(int) CFGMR3InsertStringFV(PCFGMNODE pNode, const char *pszName, const char *pszFormat, va_list va)
+{
+    int rc;
+    if (pNode)
+    {
+        /*
+         * Allocate string object first.
+         */
+        char *pszString;
+        if (!pNode->pVM)
+            pszString = RTStrAPrintf2(pszFormat, va);
+        else
+            pszString = MMR3HeapAPrintfVU(pNode->pVM->pUVM, MM_TAG_CFGM_STRING, pszFormat, va);
+        if (pszString)
+        {
+            /*
+             * Create value leaf and set it to string type.
+             */
+            PCFGMLEAF pLeaf;
+            rc = cfgmR3InsertLeaf(pNode, pszName, &pLeaf);
+            if (RT_SUCCESS(rc))
+            {
+                pLeaf->enmType = CFGMVALUETYPE_STRING;
+                pLeaf->Value.String.psz = pszString;
+                pLeaf->Value.String.cb  = strlen(pszString) + 1;
+            }
+            else
+                cfgmR3StrFree(pNode->pVM, pszString);
+        }
+        else
+            rc = VERR_NO_MEMORY;
+    }
+    else
+        rc = VERR_CFGM_NO_PARENT;
+
+    return rc;
+}
+
+
+/**
+ * Same as CFGMR3InsertString except the string value given in RTStrPrintf
+ * fashion.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   pszFormat       The value given as a format string.
+ * @param   ...             Argument to pszFormat.
+ */
+VMMR3DECL(int) CFGMR3InsertStringF(PCFGMNODE pNode, const char *pszName, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    int rc = CFGMR3InsertStringFV(pNode, pszName, pszFormat, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Same as CFGMR3InsertString except the string value given as a UTF-16 string.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   pwszValue       The string value (UTF-16).
+ */
+VMMR3DECL(int) CFGMR3InsertStringW(PCFGMNODE pNode, const char *pszName, PCRTUTF16 pwszValue)
+{
+    char *pszValue;
+    int rc = RTUtf16ToUtf8(pwszValue, &pszValue);
+    if (RT_SUCCESS(rc))
+    {
+        rc = CFGMR3InsertString(pNode, pszName, pszValue);
+        RTStrFree(pszValue);
+    }
+    return rc;
+}
+
+
+/**
+ * Inserts a new integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   pvBytes         The value.
+ * @param   cbBytes         The value size.
+ */
+VMMR3DECL(int) CFGMR3InsertBytes(PCFGMNODE pNode, const char *pszName, const void *pvBytes, size_t cbBytes)
+{
+    int rc;
+    if (pNode)
+    {
+        if (cbBytes == (RTUINT)cbBytes)
+        {
+            /*
+             * Allocate string object first.
+             */
+            void *pvCopy = cfgmR3MemAlloc(pNode->pVM, MM_TAG_CFGM_STRING, cbBytes);
+            if (pvCopy || !cbBytes)
+            {
+                memcpy(pvCopy, pvBytes, cbBytes);
+
+                /*
+                 * Create value leaf and set it to string type.
+                 */
+                PCFGMLEAF pLeaf;
+                rc = cfgmR3InsertLeaf(pNode, pszName, &pLeaf);
+                if (RT_SUCCESS(rc))
+                {
+                    pLeaf->enmType = CFGMVALUETYPE_BYTES;
+                    pLeaf->Value.Bytes.cb   = cbBytes;
+                    pLeaf->Value.Bytes.pau8 = (uint8_t *)pvCopy;
+                }
+                else
+                    cfgmR3MemFree(pNode->pVM, pvCopy);
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+        else
+            rc = VERR_OUT_OF_RANGE;
+    }
+    else
+        rc = VERR_CFGM_NO_PARENT;
+
+    return rc;
+}
+
+
+/**
+ * Make a copy of the specified value under the given node.
+ *
+ * @returns VBox status code.
+ * @param   pNode               Parent node.
+ * @param   pValue              The value to copy and insert.
+ */
+VMMR3DECL(int) CFGMR3InsertValue(PCFGMNODE pNode, PCFGMLEAF pValue)
+{
+    int rc;
+    switch (pValue->enmType)
+    {
+        case CFGMVALUETYPE_INTEGER:
+            rc = CFGMR3InsertInteger(pNode, pValue->szName, pValue->Value.Integer.u64);
+            break;
+
+        case CFGMVALUETYPE_BYTES:
+            rc = CFGMR3InsertBytes(pNode, pValue->szName, pValue->Value.Bytes.pau8, pValue->Value.Bytes.cb);
+            break;
+
+        case CFGMVALUETYPE_STRING:
+            rc = CFGMR3InsertStringN(pNode, pValue->szName, pValue->Value.String.psz, pValue->Value.String.cb - 1);
+            break;
+
+        default:
+            rc = VERR_CFGM_IPE_1;
+            AssertMsgFailed(("Invalid value type %d\n", pValue->enmType));
+            break;
+    }
+    return rc;
+}
+
+
+/**
+ * Remove a value.
+ *
+ * @returns VBox status code.
+ * @param   pNode       Parent node.
+ * @param   pszName     Name of the new child node.
+ */
+VMMR3DECL(int) CFGMR3RemoveValue(PCFGMNODE pNode, const char *pszName)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+        cfgmR3RemoveLeaf(pNode, pLeaf);
+    return rc;
+}
+
+
+
+/*
+ *  -+- helper apis -+-
+ */
+
+
+/**
+ * Query unsigned 64-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu64            Where to store the integer value.
+ */
+VMMR3DECL(int) CFGMR3QueryU64(PCFGMNODE pNode, const char *pszName, uint64_t *pu64)
+{
+    return CFGMR3QueryInteger(pNode, pszName, pu64);
+}
+
+
+/**
+ * Query unsigned 64-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu64            Where to store the integer value. Set to default on failure.
+ * @param   u64Def          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryU64Def(PCFGMNODE pNode, const char *pszName, uint64_t *pu64, uint64_t u64Def)
+{
+    return CFGMR3QueryIntegerDef(pNode, pszName, pu64, u64Def);
+}
+
+
+/**
+ * Query signed 64-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi64            Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryS64(PCFGMNODE pNode, const char *pszName, int64_t *pi64)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+        *pi64 = (int64_t)u64;
+    return rc;
+}
+
+
+/**
+ * Query signed 64-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi64            Where to store the value. Set to default on failure.
+ * @param   i64Def          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryS64Def(PCFGMNODE pNode, const char *pszName, int64_t *pi64, int64_t i64Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, i64Def);
+    *pi64 = (int64_t)u64;
+    return rc;
+}
+
+
+/**
+ * Query unsigned 32-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu32            Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryU32(PCFGMNODE pNode, const char *pszName, uint32_t *pu32)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(u64 & UINT64_C(0xffffffff00000000)))
+            *pu32 = (uint32_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query unsigned 32-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu32            Where to store the value. Set to default on failure.
+ * @param   u32Def          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryU32Def(PCFGMNODE pNode, const char *pszName, uint32_t *pu32, uint32_t u32Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, u32Def);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(u64 & UINT64_C(0xffffffff00000000)))
+            *pu32 = (uint32_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pu32 = u32Def;
+    return rc;
+}
+
+
+/**
+ * Query signed 32-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi32            Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryS32(PCFGMNODE pNode, const char *pszName, int32_t *pi32)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        if (   !(u64 & UINT64_C(0xffffffff80000000))
+            ||  (u64 & UINT64_C(0xffffffff80000000)) == UINT64_C(0xffffffff80000000))
+            *pi32 = (int32_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query signed 32-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi32            Where to store the value. Set to default on failure.
+ * @param   i32Def          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryS32Def(PCFGMNODE pNode, const char *pszName, int32_t *pi32, int32_t i32Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, i32Def);
+    if (RT_SUCCESS(rc))
+    {
+        if (   !(u64 & UINT64_C(0xffffffff80000000))
+            ||  (u64 & UINT64_C(0xffffffff80000000)) == UINT64_C(0xffffffff80000000))
+            *pi32 = (int32_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pi32 = i32Def;
+    return rc;
+}
+
+
+/**
+ * Query unsigned 16-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu16            Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryU16(PCFGMNODE pNode, const char *pszName, uint16_t *pu16)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(u64 & UINT64_C(0xffffffffffff0000)))
+            *pu16 = (int16_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query unsigned 16-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu16            Where to store the value. Set to default on failure.
+ * @param   u16Def          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryU16Def(PCFGMNODE pNode, const char *pszName, uint16_t *pu16, uint16_t u16Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, u16Def);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(u64 & UINT64_C(0xffffffffffff0000)))
+            *pu16 = (int16_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pu16 = u16Def;
+    return rc;
+}
+
+
+/**
+ * Query signed 16-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi16            Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryS16(PCFGMNODE pNode, const char *pszName, int16_t *pi16)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        if (   !(u64 & UINT64_C(0xffffffffffff8000))
+            ||  (u64 & UINT64_C(0xffffffffffff8000)) == UINT64_C(0xffffffffffff8000))
+            *pi16 = (int16_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query signed 16-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi16            Where to store the value. Set to default on failure.
+ * @param   i16Def          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryS16Def(PCFGMNODE pNode, const char *pszName, int16_t *pi16, int16_t i16Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, i16Def);
+    if (RT_SUCCESS(rc))
+    {
+        if (   !(u64 & UINT64_C(0xffffffffffff8000))
+            ||  (u64 & UINT64_C(0xffffffffffff8000)) == UINT64_C(0xffffffffffff8000))
+            *pi16 = (int16_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pi16 = i16Def;
+    return rc;
+}
+
+
+/**
+ * Query unsigned 8-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu8             Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryU8(PCFGMNODE pNode, const char *pszName, uint8_t *pu8)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(u64 & UINT64_C(0xffffffffffffff00)))
+            *pu8 = (uint8_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query unsigned 8-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu8             Where to store the value. Set to default on failure.
+ * @param   u8Def           The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryU8Def(PCFGMNODE pNode, const char *pszName, uint8_t *pu8, uint8_t u8Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, u8Def);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(u64 & UINT64_C(0xffffffffffffff00)))
+            *pu8 = (uint8_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pu8 = u8Def;
+    return rc;
+}
+
+
+/**
+ * Query signed 8-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi8             Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryS8(PCFGMNODE pNode, const char *pszName, int8_t *pi8)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        if (   !(u64 & UINT64_C(0xffffffffffffff80))
+            ||  (u64 & UINT64_C(0xffffffffffffff80)) == UINT64_C(0xffffffffffffff80))
+            *pi8 = (int8_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query signed 8-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi8             Where to store the value. Set to default on failure.
+ * @param   i8Def           The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryS8Def(PCFGMNODE pNode, const char *pszName, int8_t *pi8, int8_t i8Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, i8Def);
+    if (RT_SUCCESS(rc))
+    {
+        if (   !(u64 & UINT64_C(0xffffffffffffff80))
+            ||  (u64 & UINT64_C(0xffffffffffffff80)) == UINT64_C(0xffffffffffffff80))
+            *pi8 = (int8_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pi8 = i8Def;
+    return rc;
+}
+
+
+/**
+ * Query boolean integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pf              Where to store the value.
+ * @remark  This function will interpret any non-zero value as true.
+ */
+VMMR3DECL(int) CFGMR3QueryBool(PCFGMNODE pNode, const char *pszName, bool *pf)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+        *pf = u64 ? true : false;
+    return rc;
+}
+
+
+/**
+ * Query boolean integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pf              Where to store the value. Set to default on failure.
+ * @param   fDef            The default value.
+ * @remark  This function will interpret any non-zero value as true.
+ */
+VMMR3DECL(int) CFGMR3QueryBoolDef(PCFGMNODE pNode, const char *pszName, bool *pf, bool fDef)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, fDef);
+    *pf = u64 ? true : false;
+    return rc;
+}
+
+
+/**
+ * Query I/O port address value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pPort           Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryPort(PCFGMNODE pNode, const char *pszName, PRTIOPORT pPort)
+{
+    AssertCompileSize(RTIOPORT, 2);
+    return CFGMR3QueryU16(pNode, pszName, pPort);
+}
+
+
+/**
+ * Query I/O port address value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pPort           Where to store the value. Set to default on failure.
+ * @param   PortDef         The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryPortDef(PCFGMNODE pNode, const char *pszName, PRTIOPORT pPort, RTIOPORT PortDef)
+{
+    AssertCompileSize(RTIOPORT, 2);
+    return CFGMR3QueryU16Def(pNode, pszName, pPort, PortDef);
+}
+
+
+/**
+ * Query unsigned int address value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu              Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryUInt(PCFGMNODE pNode, const char *pszName, unsigned int *pu)
+{
+    AssertCompileSize(unsigned int, 4);
+    return CFGMR3QueryU32(pNode, pszName, (uint32_t *)pu);
+}
+
+
+/**
+ * Query unsigned int address value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu              Where to store the value. Set to default on failure.
+ * @param   uDef            The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryUIntDef(PCFGMNODE pNode, const char *pszName, unsigned int *pu, unsigned int uDef)
+{
+    AssertCompileSize(unsigned int, 4);
+    return CFGMR3QueryU32Def(pNode, pszName, (uint32_t *)pu, uDef);
+}
+
+
+/**
+ * Query signed int address value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi              Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QuerySInt(PCFGMNODE pNode, const char *pszName, signed int *pi)
+{
+    AssertCompileSize(signed int, 4);
+    return CFGMR3QueryS32(pNode, pszName, (int32_t *)pi);
+}
+
+
+/**
+ * Query unsigned int address value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi              Where to store the value. Set to default on failure.
+ * @param   iDef            The default value.
+ */
+VMMR3DECL(int) CFGMR3QuerySIntDef(PCFGMNODE pNode, const char *pszName, signed int *pi, signed int iDef)
+{
+    AssertCompileSize(signed int, 4);
+    return CFGMR3QueryS32Def(pNode, pszName, (int32_t *)pi, iDef);
+}
+
+
+/**
+ * Query pointer integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   ppv             Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryPtr(PCFGMNODE pNode, const char *pszName, void **ppv)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        uintptr_t u = (uintptr_t)u64;
+        if (u64 == u)
+            *ppv = (void *)u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query pointer integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   ppv             Where to store the value. Set to default on failure.
+ * @param   pvDef           The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryPtrDef(PCFGMNODE pNode, const char *pszName, void **ppv, void *pvDef)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, (uintptr_t)pvDef);
+    if (RT_SUCCESS(rc))
+    {
+        uintptr_t u = (uintptr_t)u64;
+        if (u64 == u)
+            *ppv = (void *)u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *ppv = pvDef;
+    return rc;
+}
+
+
+/**
+ * Query Guest Context pointer integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pGCPtr          Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryGCPtr(PCFGMNODE pNode, const char *pszName, PRTGCPTR pGCPtr)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCPTR u = (RTGCPTR)u64;
+        if (u64 == u)
+            *pGCPtr = u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query Guest Context pointer integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pGCPtr          Where to store the value. Set to default on failure.
+ * @param   GCPtrDef        The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryGCPtrDef(PCFGMNODE pNode, const char *pszName, PRTGCPTR pGCPtr, RTGCPTR GCPtrDef)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, GCPtrDef);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCPTR u = (RTGCPTR)u64;
+        if (u64 == u)
+            *pGCPtr = u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pGCPtr = GCPtrDef;
+    return rc;
+}
+
+
+/**
+ * Query Guest Context unsigned pointer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pGCPtr          Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryGCPtrU(PCFGMNODE pNode, const char *pszName, PRTGCUINTPTR pGCPtr)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCUINTPTR u = (RTGCUINTPTR)u64;
+        if (u64 == u)
+            *pGCPtr = u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query Guest Context unsigned pointer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pGCPtr          Where to store the value. Set to default on failure.
+ * @param   GCPtrDef        The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryGCPtrUDef(PCFGMNODE pNode, const char *pszName, PRTGCUINTPTR pGCPtr, RTGCUINTPTR GCPtrDef)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, GCPtrDef);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCUINTPTR u = (RTGCUINTPTR)u64;
+        if (u64 == u)
+            *pGCPtr = u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pGCPtr = GCPtrDef;
+    return rc;
+}
+
+
+/**
+ * Query Guest Context signed pointer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pGCPtr          Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryGCPtrS(PCFGMNODE pNode, const char *pszName, PRTGCINTPTR pGCPtr)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCINTPTR u = (RTGCINTPTR)u64;
+        if (u64 == (uint64_t)u)
+            *pGCPtr = u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query Guest Context signed pointer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pGCPtr          Where to store the value. Set to default on failure.
+ * @param   GCPtrDef        The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryGCPtrSDef(PCFGMNODE pNode, const char *pszName, PRTGCINTPTR pGCPtr, RTGCINTPTR GCPtrDef)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, GCPtrDef);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCINTPTR u = (RTGCINTPTR)u64;
+        if (u64 == (uint64_t)u)
+            *pGCPtr = u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pGCPtr = GCPtrDef;
+    return rc;
+}
+
+
+/**
+ * Query zero terminated character value storing it in a
+ * buffer allocated from the MM heap.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Value name. This value must be of zero terminated character string type.
+ * @param   ppszString      Where to store the string pointer.
+ *                          Free this using MMR3HeapFree() (or RTStrFree if not
+ *                          associated with a pUVM - see CFGMR3CreateTree).
+ */
+VMMR3DECL(int) CFGMR3QueryStringAlloc(PCFGMNODE pNode, const char *pszName, char **ppszString)
+{
+    size_t cbString;
+    int rc = CFGMR3QuerySize(pNode, pszName, &cbString);
+    if (RT_SUCCESS(rc))
+    {
+        char *pszString = cfgmR3StrAlloc(pNode->pVM, MM_TAG_CFGM_USER, cbString);
+        if (pszString)
+        {
+            rc = CFGMR3QueryString(pNode, pszName, pszString, cbString);
+            if (RT_SUCCESS(rc))
+                *ppszString = pszString;
+            else
+                cfgmR3StrFree(pNode->pVM, pszString);
+        }
+        else
+            rc = VERR_NO_MEMORY;
+    }
+    return rc;
+}
+
+
+/**
+ * Query zero terminated character value storing it in a
+ * buffer allocated from the MM heap.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.  This cannot be
+ *                          NULL if @a pszDef is not NULL, because we need
+ *                          somewhere way to get to the VM in order to call
+ *                          MMR3HeapStrDup.
+ * @param   pszName         Value name. This value must be of zero terminated character string type.
+ * @param   ppszString      Where to store the string pointer. Not set on failure.
+ *                          Free this using MMR3HeapFree() (or RTStrFree if not
+ *                          associated with a pUVM - see CFGMR3CreateTree).
+ * @param   pszDef          The default return value.  This can be NULL.
+ */
+VMMR3DECL(int) CFGMR3QueryStringAllocDef(PCFGMNODE pNode, const char *pszName, char **ppszString, const char *pszDef)
+{
+    Assert(pNode || !pszDef); /* We need pVM if we need to duplicate the string later. */
+
+    /*
+     * (Don't call CFGMR3QuerySize and CFGMR3QueryStringDef here as the latter
+     * cannot handle pszDef being NULL.)
+     */
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLeaf->enmType == CFGMVALUETYPE_STRING)
+        {
+            size_t const cbSrc = pLeaf->Value.String.cb;
+            char *pszString = cfgmR3StrAlloc(pNode->pVM, MM_TAG_CFGM_USER, cbSrc);
+            if (pszString)
+            {
+                memcpy(pszString, pLeaf->Value.String.psz, cbSrc);
+                *ppszString = pszString;
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+        else
+            rc = VERR_CFGM_NOT_STRING;
+    }
+    if (RT_FAILURE(rc))
+    {
+        if (!pszDef)
+            *ppszString = NULL;
+        else
+        {
+            size_t const cbDef = strlen(pszDef) + 1;
+            *ppszString = cfgmR3StrAlloc(pNode->pVM, MM_TAG_CFGM_USER, cbDef);
+            memcpy(*ppszString, pszDef, cbDef);
+        }
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+            rc = VINF_SUCCESS;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Dumps the configuration (sub)tree to the release log.
+ *
+ * @param   pRoot   The root node of the dump.
+ */
+VMMR3DECL(void) CFGMR3Dump(PCFGMNODE pRoot)
+{
+    bool fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
+    LogRel(("************************* CFGM dump *************************\n"));
+    cfgmR3Dump(pRoot, 0, DBGFR3InfoLogRelHlp());
+    LogRel(("********************* End of CFGM dump **********************\n"));
+    RTLogRelSetBuffering(fOldBuffered);
+}
+
+
+/**
+ * Info handler, internal version.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        Callback functions for doing output.
+ * @param   pszArgs     Argument string. Optional and specific to the handler.
+ */
+static DECLCALLBACK(void) cfgmR3Info(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /*
+     * Figure where to start.
+     */
+    PCFGMNODE pRoot = pVM->cfgm.s.pRoot;
+    if (pszArgs && *pszArgs)
+    {
+        int rc = cfgmR3ResolveNode(pRoot, pszArgs, &pRoot);
+        if (RT_FAILURE(rc))
+        {
+            pHlp->pfnPrintf(pHlp, "Failed to resolve CFGM path '%s', %Rrc", pszArgs, rc);
+            return;
+        }
+    }
+
+    /*
+     * Dump the specified tree.
+     */
+    pHlp->pfnPrintf(pHlp, "pRoot=%p:{", pRoot);
+    cfgmR3DumpPath(pRoot, pHlp);
+    pHlp->pfnPrintf(pHlp, "}\n");
+    cfgmR3Dump(pRoot, 0, pHlp);
+}
+
+
+/**
+ * Recursively prints a path name.
+ */
+static void cfgmR3DumpPath(PCFGMNODE pNode, PCDBGFINFOHLP pHlp)
+{
+    if (pNode->pParent)
+        cfgmR3DumpPath(pNode->pParent, pHlp);
+    pHlp->pfnPrintf(pHlp, "%s/", pNode->szName);
+}
+
+
+/**
+ * Dumps a branch of a tree.
+ */
+static void cfgmR3Dump(PCFGMNODE pRoot, unsigned iLevel, PCDBGFINFOHLP pHlp)
+{
+    /*
+     * Path.
+     */
+    pHlp->pfnPrintf(pHlp, "[");
+    cfgmR3DumpPath(pRoot, pHlp);
+    pHlp->pfnPrintf(pHlp, "] (level %d)%s\n", iLevel, pRoot->fRestrictedRoot ? " (restricted root)" : "");
+
+    /*
+     * Values.
+     */
+    PCFGMLEAF pLeaf;
+    size_t cchMax = 0;
+    for (pLeaf = CFGMR3GetFirstValue(pRoot); pLeaf; pLeaf = CFGMR3GetNextValue(pLeaf))
+        cchMax = RT_MAX(cchMax, pLeaf->cchName);
+    for (pLeaf = CFGMR3GetFirstValue(pRoot); pLeaf; pLeaf = CFGMR3GetNextValue(pLeaf))
+    {
+        switch (CFGMR3GetValueType(pLeaf))
+        {
+            case CFGMVALUETYPE_INTEGER:
+            {
+                pHlp->pfnPrintf(pHlp, "  %-*s <integer> = %#018llx (%'lld", (int)cchMax, pLeaf->szName, pLeaf->Value.Integer.u64, pLeaf->Value.Integer.u64);
+                if (   (   pLeaf->cchName >= 4
+                        && !RTStrCmp(&pLeaf->szName[pLeaf->cchName - 4], "Size"))
+                    || (   pLeaf->cchName >= 2
+                        && !RTStrNCmp(pLeaf->szName, "cb", 2)) )
+                {
+                    if (pLeaf->Value.Integer.u64 > _2M)
+                        pHlp->pfnPrintf(pHlp, ", %'lld MB", pLeaf->Value.Integer.u64 / _1M);
+                    else if (pLeaf->Value.Integer.u64 > _2K)
+                        pHlp->pfnPrintf(pHlp, ", %'lld KB", pLeaf->Value.Integer.u64 / _1K);
+                    if (pLeaf->Value.Integer.u64 > _2G)
+                        pHlp->pfnPrintf(pHlp, ", %'lld.%lld GB",
+                                              pLeaf->Value.Integer.u64 / _1G,
+                                              (pLeaf->Value.Integer.u64 % _1G) / (_1G / 10));
+                }
+                pHlp->pfnPrintf(pHlp, ")\n");
+                break;
+            }
+
+            case CFGMVALUETYPE_STRING:
+                pHlp->pfnPrintf(pHlp, "  %-*s <string>  = \"%s\" (cb=%zu)\n", (int)cchMax, pLeaf->szName, pLeaf->Value.String.psz, pLeaf->Value.String.cb);
+                break;
+
+            case CFGMVALUETYPE_BYTES:
+                pHlp->pfnPrintf(pHlp, "  %-*s <bytes>   = \"%.*Rhxs\" (cb=%zu)\n", (int)cchMax, pLeaf->szName, pLeaf->Value.Bytes.cb, pLeaf->Value.Bytes.pau8, pLeaf->Value.Bytes.cb);
+                break;
+
+            default:
+                AssertMsgFailed(("bad leaf!\n"));
+                break;
+        }
+    }
+    pHlp->pfnPrintf(pHlp, "\n");
+
+    /*
+     * Children.
+     */
+    for (PCFGMNODE pChild = CFGMR3GetFirstChild(pRoot); pChild; pChild = CFGMR3GetNextChild(pChild))
+    {
+        Assert(pChild->pNext != pChild);
+        Assert(pChild->pPrev != pChild);
+        Assert(pChild->pPrev != pChild->pNext || !pChild->pPrev);
+        Assert(pChild->pFirstChild != pChild);
+        Assert(pChild->pParent == pRoot);
+        cfgmR3Dump(pChild, iLevel + 1, pHlp);
+    }
+}
+
diff --git a/src/VBox/VMM/VMMR3/CPUM.cpp b/src/VBox/VMM/VMMR3/CPUM.cpp
new file mode 100644
index 00000000..f2bc2102
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/CPUM.cpp
@@ -0,0 +1,4627 @@
+/* $Id: CPUM.cpp $ */
+/** @file
+ * CPUM - CPU Monitor / Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_cpum CPUM - CPU Monitor / Manager
+ *
+ * The CPU Monitor / Manager keeps track of all the CPU registers. It is
+ * also responsible for lazy FPU handling and some of the context loading
+ * in raw mode.
+ *
+ * There are three CPU contexts, the most important one is the guest one (GC).
+ * When running in raw-mode (RC) there is a special hyper context for the VMM
+ * part that floats around inside the guest address space. When running in
+ * raw-mode, CPUM also maintains a host context for saving and restoring
+ * registers across world switches. This latter is done in cooperation with the
+ * world switcher (@see pg_vmm).
+ *
+ * @see grp_cpum
+ *
+ * @section sec_cpum_fpu        FPU / SSE / AVX / ++ state.
+ *
+ * TODO: proper write up, currently just some notes.
+ *
+ * The ring-0 FPU handling per OS:
+ *
+ *      - 64-bit Windows uses XMM registers in the kernel as part of the calling
+ *        convention (Visual C++ doesn't seem to have a way to disable
+ *        generating such code either), so CR0.TS/EM are always zero from what I
+ *        can tell.  We are also forced to always load/save the guest XMM0-XMM15
+ *        registers when entering/leaving guest context.  Interrupt handlers
+ *        using FPU/SSE will offically have call save and restore functions
+ *        exported by the kernel, if the really really have to use the state.
+ *
+ *      - 32-bit windows does lazy FPU handling, I think, probably including
+ *        lazying saving.  The Windows Internals book states that it's a bad
+ *        idea to use the FPU in kernel space. However, it looks like it will
+ *        restore the FPU state of the current thread in case of a kernel \#NM.
+ *        Interrupt handlers should be same as for 64-bit.
+ *
+ *      - Darwin allows taking \#NM in kernel space, restoring current thread's
+ *        state if I read the code correctly.  It saves the FPU state of the
+ *        outgoing thread, and uses CR0.TS to lazily load the state of the
+ *        incoming one.  No idea yet how the FPU is treated by interrupt
+ *        handlers, i.e. whether they are allowed to disable the state or
+ *        something.
+ *
+ *      - Linux also allows \#NM in kernel space (don't know since when), and
+ *        uses CR0.TS for lazy loading.  Saves outgoing thread's state, lazy
+ *        loads the incoming unless configured to agressivly load it.  Interrupt
+ *        handlers can ask whether they're allowed to use the FPU, and may
+ *        freely trash the state if Linux thinks it has saved the thread's state
+ *        already.  This is a problem.
+ *
+ *      - Solaris will, from what I can tell, panic if it gets an \#NM in kernel
+ *        context.  When switching threads, the kernel will save the state of
+ *        the outgoing thread and lazy load the incoming one using CR0.TS.
+ *        There are a few routines in seeblk.s which uses the SSE unit in ring-0
+ *        to do stuff, HAT are among the users.  The routines there will
+ *        manually clear CR0.TS and save the XMM registers they use only if
+ *        CR0.TS was zero upon entry.  They will skip it when not, because as
+ *        mentioned above, the FPU state is saved when switching away from a
+ *        thread and CR0.TS set to 1, so when CR0.TS is 1 there is nothing to
+ *        preserve.  This is a problem if we restore CR0.TS to 1 after loading
+ *        the guest state.
+ *
+ *      - FreeBSD - no idea yet.
+ *
+ *      - OS/2 does not allow \#NMs in kernel space IIRC.  Does lazy loading,
+ *        possibly also lazy saving.  Interrupts must preserve the CR0.TS+EM &
+ *        FPU states.
+ *
+ * Up to r107425 (2016-05-24) we would only temporarily modify CR0.TS/EM while
+ * saving and restoring the host and guest states.  The motivation for this
+ * change is that we want to be able to emulate SSE instruction in ring-0 (IEM).
+ *
+ * Starting with that change, we will leave CR0.TS=EM=0 after saving the host
+ * state and only restore it once we've restore the host FPU state. This has the
+ * accidental side effect of triggering Solaris to preserve XMM registers in
+ * sseblk.s. When CR0 was changed by saving the FPU state, CPUM must now inform
+ * the VT-x (HMVMX) code about it as it caches the CR0 value in the VMCS.
+ *
+ *
+ * @section sec_cpum_logging        Logging Level Assignments.
+ *
+ * Following log level assignments:
+ *      - Log6 is used for FPU state management.
+ *      - Log7 is used for FPU state actualization.
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_CPUM
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/cpumdis.h>
+#include <VBox/vmm/cpumctx-v1_6.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/hmvmxinline.h>
+#include <VBox/vmm/ssm.h>
+#include "CPUMInternal.h"
+#include <VBox/vmm/vm.h>
+
+#include <VBox/param.h>
+#include <VBox/dis.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/assert.h>
+#include <iprt/cpuset.h>
+#include <iprt/mem.h>
+#include <iprt/mp.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/**
+ * This was used in the saved state up to the early life of version 14.
+ *
+ * It indicates that we may have some out-of-sync hidden segement registers.
+ * It is only relevant for raw-mode.
+ */
+#define CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID    RT_BIT(12)
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+
+/**
+ * What kind of cpu info dump to perform.
+ */
+typedef enum CPUMDUMPTYPE
+{
+    CPUMDUMPTYPE_TERSE,
+    CPUMDUMPTYPE_DEFAULT,
+    CPUMDUMPTYPE_VERBOSE
+} CPUMDUMPTYPE;
+/** Pointer to a cpu info dump type. */
+typedef CPUMDUMPTYPE *PCPUMDUMPTYPE;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(int)  cpumR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass);
+static DECLCALLBACK(int)  cpumR3SaveExec(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)  cpumR3LoadPrep(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)  cpumR3LoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(int)  cpumR3LoadDone(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(void) cpumR3InfoAll(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) cpumR3InfoGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) cpumR3InfoGuestHwvirt(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) cpumR3InfoGuestInstr(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) cpumR3InfoHyper(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) cpumR3InfoHost(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** Saved state field descriptors for CPUMCTX. */
+static const SSMFIELD g_aCpumCtxFields[] =
+{
+    SSMFIELD_ENTRY(         CPUMCTX, rdi),
+    SSMFIELD_ENTRY(         CPUMCTX, rsi),
+    SSMFIELD_ENTRY(         CPUMCTX, rbp),
+    SSMFIELD_ENTRY(         CPUMCTX, rax),
+    SSMFIELD_ENTRY(         CPUMCTX, rbx),
+    SSMFIELD_ENTRY(         CPUMCTX, rdx),
+    SSMFIELD_ENTRY(         CPUMCTX, rcx),
+    SSMFIELD_ENTRY(         CPUMCTX, rsp),
+    SSMFIELD_ENTRY(         CPUMCTX, rflags),
+    SSMFIELD_ENTRY(         CPUMCTX, rip),
+    SSMFIELD_ENTRY(         CPUMCTX, r8),
+    SSMFIELD_ENTRY(         CPUMCTX, r9),
+    SSMFIELD_ENTRY(         CPUMCTX, r10),
+    SSMFIELD_ENTRY(         CPUMCTX, r11),
+    SSMFIELD_ENTRY(         CPUMCTX, r12),
+    SSMFIELD_ENTRY(         CPUMCTX, r13),
+    SSMFIELD_ENTRY(         CPUMCTX, r14),
+    SSMFIELD_ENTRY(         CPUMCTX, r15),
+    SSMFIELD_ENTRY(         CPUMCTX, es.Sel),
+    SSMFIELD_ENTRY(         CPUMCTX, es.ValidSel),
+    SSMFIELD_ENTRY(         CPUMCTX, es.fFlags),
+    SSMFIELD_ENTRY(         CPUMCTX, es.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, es.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, es.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, cs.Sel),
+    SSMFIELD_ENTRY(         CPUMCTX, cs.ValidSel),
+    SSMFIELD_ENTRY(         CPUMCTX, cs.fFlags),
+    SSMFIELD_ENTRY(         CPUMCTX, cs.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, cs.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, cs.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, ss.Sel),
+    SSMFIELD_ENTRY(         CPUMCTX, ss.ValidSel),
+    SSMFIELD_ENTRY(         CPUMCTX, ss.fFlags),
+    SSMFIELD_ENTRY(         CPUMCTX, ss.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, ss.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, ss.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, ds.Sel),
+    SSMFIELD_ENTRY(         CPUMCTX, ds.ValidSel),
+    SSMFIELD_ENTRY(         CPUMCTX, ds.fFlags),
+    SSMFIELD_ENTRY(         CPUMCTX, ds.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, ds.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, ds.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, fs.Sel),
+    SSMFIELD_ENTRY(         CPUMCTX, fs.ValidSel),
+    SSMFIELD_ENTRY(         CPUMCTX, fs.fFlags),
+    SSMFIELD_ENTRY(         CPUMCTX, fs.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, fs.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, fs.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, gs.Sel),
+    SSMFIELD_ENTRY(         CPUMCTX, gs.ValidSel),
+    SSMFIELD_ENTRY(         CPUMCTX, gs.fFlags),
+    SSMFIELD_ENTRY(         CPUMCTX, gs.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, gs.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, gs.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, cr0),
+    SSMFIELD_ENTRY(         CPUMCTX, cr2),
+    SSMFIELD_ENTRY(         CPUMCTX, cr3),
+    SSMFIELD_ENTRY(         CPUMCTX, cr4),
+    SSMFIELD_ENTRY(         CPUMCTX, dr[0]),
+    SSMFIELD_ENTRY(         CPUMCTX, dr[1]),
+    SSMFIELD_ENTRY(         CPUMCTX, dr[2]),
+    SSMFIELD_ENTRY(         CPUMCTX, dr[3]),
+    SSMFIELD_ENTRY(         CPUMCTX, dr[6]),
+    SSMFIELD_ENTRY(         CPUMCTX, dr[7]),
+    SSMFIELD_ENTRY(         CPUMCTX, gdtr.cbGdt),
+    SSMFIELD_ENTRY(         CPUMCTX, gdtr.pGdt),
+    SSMFIELD_ENTRY(         CPUMCTX, idtr.cbIdt),
+    SSMFIELD_ENTRY(         CPUMCTX, idtr.pIdt),
+    SSMFIELD_ENTRY(         CPUMCTX, SysEnter.cs),
+    SSMFIELD_ENTRY(         CPUMCTX, SysEnter.eip),
+    SSMFIELD_ENTRY(         CPUMCTX, SysEnter.esp),
+    SSMFIELD_ENTRY(         CPUMCTX, msrEFER),
+    SSMFIELD_ENTRY(         CPUMCTX, msrSTAR),
+    SSMFIELD_ENTRY(         CPUMCTX, msrPAT),
+    SSMFIELD_ENTRY(         CPUMCTX, msrLSTAR),
+    SSMFIELD_ENTRY(         CPUMCTX, msrCSTAR),
+    SSMFIELD_ENTRY(         CPUMCTX, msrSFMASK),
+    SSMFIELD_ENTRY(         CPUMCTX, msrKERNELGSBASE),
+    SSMFIELD_ENTRY(         CPUMCTX, ldtr.Sel),
+    SSMFIELD_ENTRY(         CPUMCTX, ldtr.ValidSel),
+    SSMFIELD_ENTRY(         CPUMCTX, ldtr.fFlags),
+    SSMFIELD_ENTRY(         CPUMCTX, ldtr.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, ldtr.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, ldtr.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, tr.Sel),
+    SSMFIELD_ENTRY(         CPUMCTX, tr.ValidSel),
+    SSMFIELD_ENTRY(         CPUMCTX, tr.fFlags),
+    SSMFIELD_ENTRY(         CPUMCTX, tr.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, tr.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, tr.Attr),
+    SSMFIELD_ENTRY_VER(     CPUMCTX, aXcr[0],                           CPUM_SAVED_STATE_VERSION_XSAVE),
+    SSMFIELD_ENTRY_VER(     CPUMCTX, aXcr[1],                           CPUM_SAVED_STATE_VERSION_XSAVE),
+    SSMFIELD_ENTRY_VER(     CPUMCTX, fXStateMask,                       CPUM_SAVED_STATE_VERSION_XSAVE),
+    SSMFIELD_ENTRY_TERM()
+};
+
+/** Saved state field descriptors for SVM nested hardware-virtualization
+ *  Host State. */
+static const SSMFIELD g_aSvmHwvirtHostState[] =
+{
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, uEferMsr),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, uCr0),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, uCr4),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, uCr3),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, uRip),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, uRsp),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, uRax),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, rflags),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, es.Sel),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, es.ValidSel),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, es.fFlags),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, es.u64Base),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, es.u32Limit),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, es.Attr),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, cs.Sel),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, cs.ValidSel),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, cs.fFlags),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, cs.u64Base),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, cs.u32Limit),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, cs.Attr),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, ss.Sel),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, ss.ValidSel),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, ss.fFlags),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, ss.u64Base),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, ss.u32Limit),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, ss.Attr),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, ds.Sel),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, ds.ValidSel),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, ds.fFlags),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, ds.u64Base),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, ds.u32Limit),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, ds.Attr),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, gdtr.cbGdt),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, gdtr.pGdt),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, idtr.cbIdt),
+    SSMFIELD_ENTRY(       SVMHOSTSTATE, idtr.pIdt),
+    SSMFIELD_ENTRY_IGNORE(SVMHOSTSTATE, abPadding),
+    SSMFIELD_ENTRY_TERM()
+};
+
+/** Saved state field descriptors for VMX nested hardware-virtualization
+ *  VMCS. */
+static const SSMFIELD g_aVmxHwvirtVmcs[] =
+{
+    SSMFIELD_ENTRY(       VMXVVMCS, u32VmcsRevId),
+    SSMFIELD_ENTRY(       VMXVVMCS, enmVmxAbort),
+    SSMFIELD_ENTRY(       VMXVVMCS, fVmcsState),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au8Padding0),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au32Reserved0),
+
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, u16Reserved0),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u32RoVmInstrError),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32RoExitReason),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32RoExitIntInfo),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32RoExitIntErrCode),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32RoIdtVectoringInfo),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32RoIdtVectoringErrCode),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32RoExitInstrLen),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32RoExitInstrInfo),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au32RoReserved2),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u64RoGuestPhysAddr),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au64Reserved1),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u64RoExitQual),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64RoIoRcx),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64RoIoRsi),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64RoIoRdi),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64RoIoRip),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64RoGuestLinearAddr),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au64Reserved5),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u16Vpid),
+    SSMFIELD_ENTRY(       VMXVVMCS, u16PostIntNotifyVector),
+    SSMFIELD_ENTRY(       VMXVVMCS, u16EptpIndex),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au16Reserved0),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u32PinCtls),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32ProcCtls),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32XcptBitmap),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32XcptPFMask),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32XcptPFMatch),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32Cr3TargetCount),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32ExitCtls),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32ExitMsrStoreCount),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32ExitMsrLoadCount),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32EntryCtls),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32EntryMsrLoadCount),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32EntryIntInfo),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32EntryXcptErrCode),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32EntryInstrLen),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32TprThreshold),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32ProcCtls2),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32PleGap),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32PleWindow),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au32Reserved1),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrIoBitmapA),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrIoBitmapB),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrMsrBitmap),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrExitMsrStore),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrExitMsrLoad),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrEntryMsrLoad),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64ExecVmcsPtr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrPml),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64TscOffset),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrVirtApic),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrApicAccess),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrPostedIntDesc),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64VmFuncCtls),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64EptpPtr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64EoiExitBitmap0),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64EoiExitBitmap1),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64EoiExitBitmap2),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64EoiExitBitmap3),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrEptpList),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrVmreadBitmap),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrVmwriteBitmap),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64AddrXcptVeInfo),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64XssBitmap),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64EnclsBitmap),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64SpptPtr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64TscMultiplier),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au64Reserved0),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u64Cr0Mask),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64Cr4Mask),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64Cr0ReadShadow),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64Cr4ReadShadow),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64Cr3Target0),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64Cr3Target1),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64Cr3Target2),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64Cr3Target3),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au64Reserved4),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, HostEs),
+    SSMFIELD_ENTRY(       VMXVVMCS, HostCs),
+    SSMFIELD_ENTRY(       VMXVVMCS, HostSs),
+    SSMFIELD_ENTRY(       VMXVVMCS, HostDs),
+    SSMFIELD_ENTRY(       VMXVVMCS, HostFs),
+    SSMFIELD_ENTRY(       VMXVVMCS, HostGs),
+    SSMFIELD_ENTRY(       VMXVVMCS, HostTr),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au16Reserved2),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u32HostSysenterCs),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au32Reserved4),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostPatMsr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostEferMsr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostPerfGlobalCtlMsr),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au64Reserved3),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostCr0),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostCr3),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostCr4),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostFsBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostGsBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostTrBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostGdtrBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostIdtrBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostSysenterEsp),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostSysenterEip),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostRsp),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64HostRip),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au64Reserved7),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, GuestEs),
+    SSMFIELD_ENTRY(       VMXVVMCS, GuestCs),
+    SSMFIELD_ENTRY(       VMXVVMCS, GuestSs),
+    SSMFIELD_ENTRY(       VMXVVMCS, GuestDs),
+    SSMFIELD_ENTRY(       VMXVVMCS, GuestFs),
+    SSMFIELD_ENTRY(       VMXVVMCS, GuestGs),
+    SSMFIELD_ENTRY(       VMXVVMCS, GuestLdtr),
+    SSMFIELD_ENTRY(       VMXVVMCS, GuestTr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u16GuestIntStatus),
+    SSMFIELD_ENTRY(       VMXVVMCS, u16PmlIndex),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au16Reserved1),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestEsLimit),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestCsLimit),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestSsLimit),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestDsLimit),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestFsLimit),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestGsLimit),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestLdtrLimit),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestTrLimit),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestGdtrLimit),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestIdtrLimit),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestEsAttr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestCsAttr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestSsAttr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestDsAttr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestFsAttr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestGsAttr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestLdtrAttr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestTrAttr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestIntrState),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestActivityState),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestSmBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32GuestSysenterCS),
+    SSMFIELD_ENTRY(       VMXVVMCS, u32PreemptTimer),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au32Reserved3),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u64VmcsLinkPtr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestDebugCtlMsr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestPatMsr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestEferMsr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestPerfGlobalCtlMsr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestPdpte0),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestPdpte1),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestPdpte2),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestPdpte3),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestBndcfgsMsr),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestRtitCtlMsr),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au64Reserved2),
+
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestCr0),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestCr3),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestCr4),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestEsBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestCsBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestSsBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestDsBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestFsBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestGsBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestLdtrBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestTrBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestGdtrBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestIdtrBase),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestDr7),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestRsp),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestRip),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestRFlags),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestPendingDbgXcpts),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestSysenterEsp),
+    SSMFIELD_ENTRY(       VMXVVMCS, u64GuestSysenterEip),
+    SSMFIELD_ENTRY_IGNORE(VMXVVMCS, au64Reserved6),
+
+    SSMFIELD_ENTRY_TERM()
+};
+
+/** Saved state field descriptors for CPUMCTX. */
+static const SSMFIELD g_aCpumX87Fields[] =
+{
+    SSMFIELD_ENTRY(         X86FXSTATE, FCW),
+    SSMFIELD_ENTRY(         X86FXSTATE, FSW),
+    SSMFIELD_ENTRY(         X86FXSTATE, FTW),
+    SSMFIELD_ENTRY(         X86FXSTATE, FOP),
+    SSMFIELD_ENTRY(         X86FXSTATE, FPUIP),
+    SSMFIELD_ENTRY(         X86FXSTATE, CS),
+    SSMFIELD_ENTRY(         X86FXSTATE, Rsrvd1),
+    SSMFIELD_ENTRY(         X86FXSTATE, FPUDP),
+    SSMFIELD_ENTRY(         X86FXSTATE, DS),
+    SSMFIELD_ENTRY(         X86FXSTATE, Rsrvd2),
+    SSMFIELD_ENTRY(         X86FXSTATE, MXCSR),
+    SSMFIELD_ENTRY(         X86FXSTATE, MXCSR_MASK),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[0]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[1]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[2]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[3]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[4]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[5]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[6]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[7]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[0]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[1]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[2]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[3]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[4]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[5]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[6]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[7]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[8]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[9]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[10]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[11]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[12]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[13]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[14]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[15]),
+    SSMFIELD_ENTRY_VER(     X86FXSTATE, au32RsrvdForSoftware[0],        CPUM_SAVED_STATE_VERSION_XSAVE), /* 32-bit/64-bit hack */
+    SSMFIELD_ENTRY_TERM()
+};
+
+/** Saved state field descriptors for X86XSAVEHDR. */
+static const SSMFIELD g_aCpumXSaveHdrFields[] =
+{
+    SSMFIELD_ENTRY(         X86XSAVEHDR,  bmXState),
+    SSMFIELD_ENTRY_TERM()
+};
+
+/** Saved state field descriptors for X86XSAVEYMMHI. */
+static const SSMFIELD g_aCpumYmmHiFields[] =
+{
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[0]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[1]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[2]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[3]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[4]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[5]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[6]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[7]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[8]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[9]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[10]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[11]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[12]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[13]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[14]),
+    SSMFIELD_ENTRY(         X86XSAVEYMMHI, aYmmHi[15]),
+    SSMFIELD_ENTRY_TERM()
+};
+
+/** Saved state field descriptors for X86XSAVEBNDREGS. */
+static const SSMFIELD g_aCpumBndRegsFields[] =
+{
+    SSMFIELD_ENTRY(         X86XSAVEBNDREGS, aRegs[0]),
+    SSMFIELD_ENTRY(         X86XSAVEBNDREGS, aRegs[1]),
+    SSMFIELD_ENTRY(         X86XSAVEBNDREGS, aRegs[2]),
+    SSMFIELD_ENTRY(         X86XSAVEBNDREGS, aRegs[3]),
+    SSMFIELD_ENTRY_TERM()
+};
+
+/** Saved state field descriptors for X86XSAVEBNDCFG. */
+static const SSMFIELD g_aCpumBndCfgFields[] =
+{
+    SSMFIELD_ENTRY(         X86XSAVEBNDCFG, fConfig),
+    SSMFIELD_ENTRY(         X86XSAVEBNDCFG, fStatus),
+    SSMFIELD_ENTRY_TERM()
+};
+
+#if 0 /** @todo */
+/** Saved state field descriptors for X86XSAVEOPMASK. */
+static const SSMFIELD g_aCpumOpmaskFields[] =
+{
+    SSMFIELD_ENTRY(         X86XSAVEOPMASK, aKRegs[0]),
+    SSMFIELD_ENTRY(         X86XSAVEOPMASK, aKRegs[1]),
+    SSMFIELD_ENTRY(         X86XSAVEOPMASK, aKRegs[2]),
+    SSMFIELD_ENTRY(         X86XSAVEOPMASK, aKRegs[3]),
+    SSMFIELD_ENTRY(         X86XSAVEOPMASK, aKRegs[4]),
+    SSMFIELD_ENTRY(         X86XSAVEOPMASK, aKRegs[5]),
+    SSMFIELD_ENTRY(         X86XSAVEOPMASK, aKRegs[6]),
+    SSMFIELD_ENTRY(         X86XSAVEOPMASK, aKRegs[7]),
+    SSMFIELD_ENTRY_TERM()
+};
+#endif
+
+/** Saved state field descriptors for X86XSAVEZMMHI256. */
+static const SSMFIELD g_aCpumZmmHi256Fields[] =
+{
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[0]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[1]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[2]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[3]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[4]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[5]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[6]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[7]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[8]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[9]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[10]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[11]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[12]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[13]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[14]),
+    SSMFIELD_ENTRY(         X86XSAVEZMMHI256, aHi256Regs[15]),
+    SSMFIELD_ENTRY_TERM()
+};
+
+/** Saved state field descriptors for X86XSAVEZMM16HI. */
+static const SSMFIELD g_aCpumZmm16HiFields[] =
+{
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[0]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[1]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[2]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[3]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[4]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[5]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[6]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[7]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[8]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[9]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[10]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[11]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[12]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[13]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[14]),
+    SSMFIELD_ENTRY(         X86XSAVEZMM16HI, aRegs[15]),
+    SSMFIELD_ENTRY_TERM()
+};
+
+
+
+/** Saved state field descriptors for CPUMCTX in V4.1 before the hidden selector
+ * registeres changed. */
+static const SSMFIELD g_aCpumX87FieldsMem[] =
+{
+    SSMFIELD_ENTRY(         X86FXSTATE, FCW),
+    SSMFIELD_ENTRY(         X86FXSTATE, FSW),
+    SSMFIELD_ENTRY(         X86FXSTATE, FTW),
+    SSMFIELD_ENTRY(         X86FXSTATE, FOP),
+    SSMFIELD_ENTRY(         X86FXSTATE, FPUIP),
+    SSMFIELD_ENTRY(         X86FXSTATE, CS),
+    SSMFIELD_ENTRY(         X86FXSTATE, Rsrvd1),
+    SSMFIELD_ENTRY(         X86FXSTATE, FPUDP),
+    SSMFIELD_ENTRY(         X86FXSTATE, DS),
+    SSMFIELD_ENTRY(         X86FXSTATE, Rsrvd2),
+    SSMFIELD_ENTRY(         X86FXSTATE, MXCSR),
+    SSMFIELD_ENTRY(         X86FXSTATE, MXCSR_MASK),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[0]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[1]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[2]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[3]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[4]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[5]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[6]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[7]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[0]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[1]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[2]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[3]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[4]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[5]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[6]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[7]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[8]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[9]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[10]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[11]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[12]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[13]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[14]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[15]),
+    SSMFIELD_ENTRY_IGNORE(  X86FXSTATE, au32RsrvdRest),
+    SSMFIELD_ENTRY_IGNORE(  X86FXSTATE, au32RsrvdForSoftware),
+};
+
+/** Saved state field descriptors for CPUMCTX in V4.1 before the hidden selector
+ * registeres changed. */
+static const SSMFIELD g_aCpumCtxFieldsMem[] =
+{
+    SSMFIELD_ENTRY(         CPUMCTX, rdi),
+    SSMFIELD_ENTRY(         CPUMCTX, rsi),
+    SSMFIELD_ENTRY(         CPUMCTX, rbp),
+    SSMFIELD_ENTRY(         CPUMCTX, rax),
+    SSMFIELD_ENTRY(         CPUMCTX, rbx),
+    SSMFIELD_ENTRY(         CPUMCTX, rdx),
+    SSMFIELD_ENTRY(         CPUMCTX, rcx),
+    SSMFIELD_ENTRY(         CPUMCTX, rsp),
+    SSMFIELD_ENTRY_OLD(              lss_esp, sizeof(uint32_t)),
+    SSMFIELD_ENTRY(         CPUMCTX, ss.Sel),
+    SSMFIELD_ENTRY_OLD(              ssPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(         CPUMCTX, gs.Sel),
+    SSMFIELD_ENTRY_OLD(              gsPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(         CPUMCTX, fs.Sel),
+    SSMFIELD_ENTRY_OLD(              fsPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(         CPUMCTX, es.Sel),
+    SSMFIELD_ENTRY_OLD(              esPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(         CPUMCTX, ds.Sel),
+    SSMFIELD_ENTRY_OLD(              dsPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(         CPUMCTX, cs.Sel),
+    SSMFIELD_ENTRY_OLD(              csPadding, sizeof(uint16_t)*3),
+    SSMFIELD_ENTRY(         CPUMCTX, rflags),
+    SSMFIELD_ENTRY(         CPUMCTX, rip),
+    SSMFIELD_ENTRY(         CPUMCTX, r8),
+    SSMFIELD_ENTRY(         CPUMCTX, r9),
+    SSMFIELD_ENTRY(         CPUMCTX, r10),
+    SSMFIELD_ENTRY(         CPUMCTX, r11),
+    SSMFIELD_ENTRY(         CPUMCTX, r12),
+    SSMFIELD_ENTRY(         CPUMCTX, r13),
+    SSMFIELD_ENTRY(         CPUMCTX, r14),
+    SSMFIELD_ENTRY(         CPUMCTX, r15),
+    SSMFIELD_ENTRY(         CPUMCTX, es.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, es.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, es.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, cs.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, cs.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, cs.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, ss.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, ss.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, ss.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, ds.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, ds.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, ds.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, fs.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, fs.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, fs.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, gs.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, gs.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, gs.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, cr0),
+    SSMFIELD_ENTRY(         CPUMCTX, cr2),
+    SSMFIELD_ENTRY(         CPUMCTX, cr3),
+    SSMFIELD_ENTRY(         CPUMCTX, cr4),
+    SSMFIELD_ENTRY(         CPUMCTX, dr[0]),
+    SSMFIELD_ENTRY(         CPUMCTX, dr[1]),
+    SSMFIELD_ENTRY(         CPUMCTX, dr[2]),
+    SSMFIELD_ENTRY(         CPUMCTX, dr[3]),
+    SSMFIELD_ENTRY_OLD(              dr[4], sizeof(uint64_t)),
+    SSMFIELD_ENTRY_OLD(              dr[5], sizeof(uint64_t)),
+    SSMFIELD_ENTRY(         CPUMCTX, dr[6]),
+    SSMFIELD_ENTRY(         CPUMCTX, dr[7]),
+    SSMFIELD_ENTRY(         CPUMCTX, gdtr.cbGdt),
+    SSMFIELD_ENTRY(         CPUMCTX, gdtr.pGdt),
+    SSMFIELD_ENTRY_OLD(              gdtrPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(         CPUMCTX, idtr.cbIdt),
+    SSMFIELD_ENTRY(         CPUMCTX, idtr.pIdt),
+    SSMFIELD_ENTRY_OLD(              idtrPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(         CPUMCTX, ldtr.Sel),
+    SSMFIELD_ENTRY_OLD(              ldtrPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(         CPUMCTX, tr.Sel),
+    SSMFIELD_ENTRY_OLD(              trPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(         CPUMCTX, SysEnter.cs),
+    SSMFIELD_ENTRY(         CPUMCTX, SysEnter.eip),
+    SSMFIELD_ENTRY(         CPUMCTX, SysEnter.esp),
+    SSMFIELD_ENTRY(         CPUMCTX, msrEFER),
+    SSMFIELD_ENTRY(         CPUMCTX, msrSTAR),
+    SSMFIELD_ENTRY(         CPUMCTX, msrPAT),
+    SSMFIELD_ENTRY(         CPUMCTX, msrLSTAR),
+    SSMFIELD_ENTRY(         CPUMCTX, msrCSTAR),
+    SSMFIELD_ENTRY(         CPUMCTX, msrSFMASK),
+    SSMFIELD_ENTRY(         CPUMCTX, msrKERNELGSBASE),
+    SSMFIELD_ENTRY(         CPUMCTX, ldtr.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, ldtr.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, ldtr.Attr),
+    SSMFIELD_ENTRY(         CPUMCTX, tr.u64Base),
+    SSMFIELD_ENTRY(         CPUMCTX, tr.u32Limit),
+    SSMFIELD_ENTRY(         CPUMCTX, tr.Attr),
+    SSMFIELD_ENTRY_TERM()
+};
+
+/** Saved state field descriptors for CPUMCTX_VER1_6. */
+static const SSMFIELD g_aCpumX87FieldsV16[] =
+{
+    SSMFIELD_ENTRY(         X86FXSTATE, FCW),
+    SSMFIELD_ENTRY(         X86FXSTATE, FSW),
+    SSMFIELD_ENTRY(         X86FXSTATE, FTW),
+    SSMFIELD_ENTRY(         X86FXSTATE, FOP),
+    SSMFIELD_ENTRY(         X86FXSTATE, FPUIP),
+    SSMFIELD_ENTRY(         X86FXSTATE, CS),
+    SSMFIELD_ENTRY(         X86FXSTATE, Rsrvd1),
+    SSMFIELD_ENTRY(         X86FXSTATE, FPUDP),
+    SSMFIELD_ENTRY(         X86FXSTATE, DS),
+    SSMFIELD_ENTRY(         X86FXSTATE, Rsrvd2),
+    SSMFIELD_ENTRY(         X86FXSTATE, MXCSR),
+    SSMFIELD_ENTRY(         X86FXSTATE, MXCSR_MASK),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[0]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[1]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[2]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[3]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[4]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[5]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[6]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aRegs[7]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[0]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[1]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[2]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[3]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[4]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[5]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[6]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[7]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[8]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[9]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[10]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[11]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[12]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[13]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[14]),
+    SSMFIELD_ENTRY(         X86FXSTATE, aXMM[15]),
+    SSMFIELD_ENTRY_IGNORE(  X86FXSTATE, au32RsrvdRest),
+    SSMFIELD_ENTRY_IGNORE(  X86FXSTATE, au32RsrvdForSoftware),
+    SSMFIELD_ENTRY_TERM()
+};
+
+/** Saved state field descriptors for CPUMCTX_VER1_6. */
+static const SSMFIELD g_aCpumCtxFieldsV16[] =
+{
+    SSMFIELD_ENTRY(             CPUMCTX, rdi),
+    SSMFIELD_ENTRY(             CPUMCTX, rsi),
+    SSMFIELD_ENTRY(             CPUMCTX, rbp),
+    SSMFIELD_ENTRY(             CPUMCTX, rax),
+    SSMFIELD_ENTRY(             CPUMCTX, rbx),
+    SSMFIELD_ENTRY(             CPUMCTX, rdx),
+    SSMFIELD_ENTRY(             CPUMCTX, rcx),
+    SSMFIELD_ENTRY_U32_ZX_U64(  CPUMCTX, rsp),
+    SSMFIELD_ENTRY(             CPUMCTX, ss.Sel),
+    SSMFIELD_ENTRY_OLD(                  ssPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY_OLD(         CPUMCTX, sizeof(uint64_t) /*rsp_notused*/),
+    SSMFIELD_ENTRY(             CPUMCTX, gs.Sel),
+    SSMFIELD_ENTRY_OLD(                  gsPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(             CPUMCTX, fs.Sel),
+    SSMFIELD_ENTRY_OLD(                  fsPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(             CPUMCTX, es.Sel),
+    SSMFIELD_ENTRY_OLD(                  esPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(             CPUMCTX, ds.Sel),
+    SSMFIELD_ENTRY_OLD(                  dsPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(             CPUMCTX, cs.Sel),
+    SSMFIELD_ENTRY_OLD(                  csPadding, sizeof(uint16_t)*3),
+    SSMFIELD_ENTRY(             CPUMCTX, rflags),
+    SSMFIELD_ENTRY(             CPUMCTX, rip),
+    SSMFIELD_ENTRY(             CPUMCTX, r8),
+    SSMFIELD_ENTRY(             CPUMCTX, r9),
+    SSMFIELD_ENTRY(             CPUMCTX, r10),
+    SSMFIELD_ENTRY(             CPUMCTX, r11),
+    SSMFIELD_ENTRY(             CPUMCTX, r12),
+    SSMFIELD_ENTRY(             CPUMCTX, r13),
+    SSMFIELD_ENTRY(             CPUMCTX, r14),
+    SSMFIELD_ENTRY(             CPUMCTX, r15),
+    SSMFIELD_ENTRY_U32_ZX_U64(  CPUMCTX, es.u64Base),
+    SSMFIELD_ENTRY(             CPUMCTX, es.u32Limit),
+    SSMFIELD_ENTRY(             CPUMCTX, es.Attr),
+    SSMFIELD_ENTRY_U32_ZX_U64(  CPUMCTX, cs.u64Base),
+    SSMFIELD_ENTRY(             CPUMCTX, cs.u32Limit),
+    SSMFIELD_ENTRY(             CPUMCTX, cs.Attr),
+    SSMFIELD_ENTRY_U32_ZX_U64(  CPUMCTX, ss.u64Base),
+    SSMFIELD_ENTRY(             CPUMCTX, ss.u32Limit),
+    SSMFIELD_ENTRY(             CPUMCTX, ss.Attr),
+    SSMFIELD_ENTRY_U32_ZX_U64(  CPUMCTX, ds.u64Base),
+    SSMFIELD_ENTRY(             CPUMCTX, ds.u32Limit),
+    SSMFIELD_ENTRY(             CPUMCTX, ds.Attr),
+    SSMFIELD_ENTRY_U32_ZX_U64(  CPUMCTX, fs.u64Base),
+    SSMFIELD_ENTRY(             CPUMCTX, fs.u32Limit),
+    SSMFIELD_ENTRY(             CPUMCTX, fs.Attr),
+    SSMFIELD_ENTRY_U32_ZX_U64(  CPUMCTX, gs.u64Base),
+    SSMFIELD_ENTRY(             CPUMCTX, gs.u32Limit),
+    SSMFIELD_ENTRY(             CPUMCTX, gs.Attr),
+    SSMFIELD_ENTRY(             CPUMCTX, cr0),
+    SSMFIELD_ENTRY(             CPUMCTX, cr2),
+    SSMFIELD_ENTRY(             CPUMCTX, cr3),
+    SSMFIELD_ENTRY(             CPUMCTX, cr4),
+    SSMFIELD_ENTRY_OLD(                  cr8, sizeof(uint64_t)),
+    SSMFIELD_ENTRY(             CPUMCTX, dr[0]),
+    SSMFIELD_ENTRY(             CPUMCTX, dr[1]),
+    SSMFIELD_ENTRY(             CPUMCTX, dr[2]),
+    SSMFIELD_ENTRY(             CPUMCTX, dr[3]),
+    SSMFIELD_ENTRY_OLD(                  dr[4], sizeof(uint64_t)),
+    SSMFIELD_ENTRY_OLD(                  dr[5], sizeof(uint64_t)),
+    SSMFIELD_ENTRY(             CPUMCTX, dr[6]),
+    SSMFIELD_ENTRY(             CPUMCTX, dr[7]),
+    SSMFIELD_ENTRY(             CPUMCTX, gdtr.cbGdt),
+    SSMFIELD_ENTRY_U32_ZX_U64(  CPUMCTX, gdtr.pGdt),
+    SSMFIELD_ENTRY_OLD(                  gdtrPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY_OLD(                  gdtrPadding64, sizeof(uint64_t)),
+    SSMFIELD_ENTRY(             CPUMCTX, idtr.cbIdt),
+    SSMFIELD_ENTRY_U32_ZX_U64(  CPUMCTX, idtr.pIdt),
+    SSMFIELD_ENTRY_OLD(                  idtrPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY_OLD(                  idtrPadding64, sizeof(uint64_t)),
+    SSMFIELD_ENTRY(             CPUMCTX, ldtr.Sel),
+    SSMFIELD_ENTRY_OLD(                  ldtrPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(             CPUMCTX, tr.Sel),
+    SSMFIELD_ENTRY_OLD(                  trPadding, sizeof(uint16_t)),
+    SSMFIELD_ENTRY(             CPUMCTX, SysEnter.cs),
+    SSMFIELD_ENTRY(             CPUMCTX, SysEnter.eip),
+    SSMFIELD_ENTRY(             CPUMCTX, SysEnter.esp),
+    SSMFIELD_ENTRY(             CPUMCTX, msrEFER),
+    SSMFIELD_ENTRY(             CPUMCTX, msrSTAR),
+    SSMFIELD_ENTRY(             CPUMCTX, msrPAT),
+    SSMFIELD_ENTRY(             CPUMCTX, msrLSTAR),
+    SSMFIELD_ENTRY(             CPUMCTX, msrCSTAR),
+    SSMFIELD_ENTRY(             CPUMCTX, msrSFMASK),
+    SSMFIELD_ENTRY_OLD(                  msrFSBASE, sizeof(uint64_t)),
+    SSMFIELD_ENTRY_OLD(                  msrGSBASE, sizeof(uint64_t)),
+    SSMFIELD_ENTRY(             CPUMCTX, msrKERNELGSBASE),
+    SSMFIELD_ENTRY_U32_ZX_U64(  CPUMCTX, ldtr.u64Base),
+    SSMFIELD_ENTRY(             CPUMCTX, ldtr.u32Limit),
+    SSMFIELD_ENTRY(             CPUMCTX, ldtr.Attr),
+    SSMFIELD_ENTRY_U32_ZX_U64(  CPUMCTX, tr.u64Base),
+    SSMFIELD_ENTRY(             CPUMCTX, tr.u32Limit),
+    SSMFIELD_ENTRY(             CPUMCTX, tr.Attr),
+    SSMFIELD_ENTRY_OLD(                  padding, sizeof(uint32_t)*2),
+    SSMFIELD_ENTRY_TERM()
+};
+
+
+/**
+ * Checks for partial/leaky FXSAVE/FXRSTOR handling on AMD CPUs.
+ *
+ * AMD K7, K8 and newer AMD CPUs do not save/restore the x87 error pointers
+ * (last instruction pointer, last data pointer, last opcode) except when the ES
+ * bit (Exception Summary) in x87 FSW (FPU Status Word) is set. Thus if we don't
+ * clear these registers there is potential, local FPU leakage from a process
+ * using the FPU to another.
+ *
+ * See AMD Instruction Reference for FXSAVE, FXRSTOR.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+static void cpumR3CheckLeakyFpu(PVM pVM)
+{
+    uint32_t u32CpuVersion = ASMCpuId_EAX(1);
+    uint32_t const u32Family = u32CpuVersion >> 8;
+    if (   u32Family >= 6      /* K7 and higher */
+        && (ASMIsAmdCpu() || ASMIsHygonCpu()) )
+    {
+        uint32_t cExt = ASMCpuId_EAX(0x80000000);
+        if (ASMIsValidExtRange(cExt))
+        {
+            uint32_t fExtFeaturesEDX = ASMCpuId_EDX(0x80000001);
+            if (fExtFeaturesEDX & X86_CPUID_AMD_FEATURE_EDX_FFXSR)
+            {
+                for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                {
+                    PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+                    pVCpu->cpum.s.fUseFlags |= CPUM_USE_FFXSR_LEAKY;
+                }
+                Log(("CPUM: Host CPU has leaky fxsave/fxrstor behaviour\n"));
+            }
+        }
+    }
+}
+
+
+/**
+ * Frees memory allocated for the SVM hardware virtualization state.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+static void cpumR3FreeSvmHwVirtState(PVM pVM)
+{
+    Assert(pVM->cpum.s.GuestFeatures.fSvm);
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+        if (pVCpu->cpum.s.Guest.hwvirt.svm.pVmcbR3)
+        {
+            SUPR3PageFreeEx(pVCpu->cpum.s.Guest.hwvirt.svm.pVmcbR3, SVM_VMCB_PAGES);
+            pVCpu->cpum.s.Guest.hwvirt.svm.pVmcbR3 = NULL;
+        }
+        pVCpu->cpum.s.Guest.hwvirt.svm.HCPhysVmcb = NIL_RTHCPHYS;
+
+        if (pVCpu->cpum.s.Guest.hwvirt.svm.pvMsrBitmapR3)
+        {
+            SUPR3PageFreeEx(pVCpu->cpum.s.Guest.hwvirt.svm.pvMsrBitmapR3, SVM_MSRPM_PAGES);
+            pVCpu->cpum.s.Guest.hwvirt.svm.pvMsrBitmapR3 = NULL;
+        }
+
+        if (pVCpu->cpum.s.Guest.hwvirt.svm.pvIoBitmapR3)
+        {
+            SUPR3PageFreeEx(pVCpu->cpum.s.Guest.hwvirt.svm.pvIoBitmapR3, SVM_IOPM_PAGES);
+            pVCpu->cpum.s.Guest.hwvirt.svm.pvIoBitmapR3 = NULL;
+        }
+    }
+}
+
+
+/**
+ * Allocates memory for the SVM hardware virtualization state.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+static int cpumR3AllocSvmHwVirtState(PVM pVM)
+{
+    Assert(pVM->cpum.s.GuestFeatures.fSvm);
+
+    int rc = VINF_SUCCESS;
+    LogRel(("CPUM: Allocating %u pages for the nested-guest SVM MSR and IO permission bitmaps\n",
+            pVM->cCpus * (SVM_MSRPM_PAGES + SVM_IOPM_PAGES)));
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+        pVCpu->cpum.s.Guest.hwvirt.enmHwvirt = CPUMHWVIRT_SVM;
+
+        /*
+         * Allocate the nested-guest VMCB.
+         */
+        SUPPAGE SupNstGstVmcbPage;
+        RT_ZERO(SupNstGstVmcbPage);
+        SupNstGstVmcbPage.Phys = NIL_RTHCPHYS;
+        Assert(SVM_VMCB_PAGES == 1);
+        Assert(!pVCpu->cpum.s.Guest.hwvirt.svm.pVmcbR3);
+        rc = SUPR3PageAllocEx(SVM_VMCB_PAGES, 0 /* fFlags */, (void **)&pVCpu->cpum.s.Guest.hwvirt.svm.pVmcbR3,
+                              &pVCpu->cpum.s.Guest.hwvirt.svm.pVmcbR0, &SupNstGstVmcbPage);
+        if (RT_FAILURE(rc))
+        {
+            Assert(!pVCpu->cpum.s.Guest.hwvirt.svm.pVmcbR3);
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's VMCB\n", pVCpu->idCpu, SVM_VMCB_PAGES));
+            break;
+        }
+        pVCpu->cpum.s.Guest.hwvirt.svm.HCPhysVmcb = SupNstGstVmcbPage.Phys;
+
+        /*
+         * Allocate the MSRPM (MSR Permission bitmap).
+         *
+         * This need not be physically contiguous pages because we use the one from
+         * HMPHYSCPU while executing the nested-guest using hardware-assisted SVM.
+         * This one is just used for caching the bitmap from guest physical memory.
+         */
+        Assert(!pVCpu->cpum.s.Guest.hwvirt.svm.pvMsrBitmapR3);
+        rc = SUPR3PageAllocEx(SVM_MSRPM_PAGES, 0 /* fFlags */, &pVCpu->cpum.s.Guest.hwvirt.svm.pvMsrBitmapR3,
+                              &pVCpu->cpum.s.Guest.hwvirt.svm.pvMsrBitmapR0, NULL /* paPages */);
+        if (RT_FAILURE(rc))
+        {
+            Assert(!pVCpu->cpum.s.Guest.hwvirt.svm.pvMsrBitmapR3);
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's MSR permission bitmap\n", pVCpu->idCpu,
+                    SVM_MSRPM_PAGES));
+            break;
+        }
+
+        /*
+         * Allocate the IOPM (IO Permission bitmap).
+         *
+         * This need not be physically contiguous pages because we re-use the ring-0
+         * allocated IOPM while executing the nested-guest using hardware-assisted SVM
+         * because it's identical (we trap all IO accesses).
+         *
+         * This one is just used for caching the IOPM from guest physical memory in
+         * case the guest hypervisor allows direct access to some IO ports.
+         */
+        Assert(!pVCpu->cpum.s.Guest.hwvirt.svm.pvIoBitmapR3);
+        rc = SUPR3PageAllocEx(SVM_IOPM_PAGES, 0 /* fFlags */, &pVCpu->cpum.s.Guest.hwvirt.svm.pvIoBitmapR3,
+                              &pVCpu->cpum.s.Guest.hwvirt.svm.pvIoBitmapR0, NULL /* paPages */);
+        if (RT_FAILURE(rc))
+        {
+            Assert(!pVCpu->cpum.s.Guest.hwvirt.svm.pvIoBitmapR3);
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's IO permission bitmap\n", pVCpu->idCpu,
+                    SVM_IOPM_PAGES));
+            break;
+        }
+    }
+
+    /* On any failure, cleanup. */
+    if (RT_FAILURE(rc))
+        cpumR3FreeSvmHwVirtState(pVM);
+
+    return rc;
+}
+
+
+/**
+ * Resets per-VCPU SVM hardware virtualization state.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) cpumR3ResetSvmHwVirtState(PVMCPU pVCpu)
+{
+    PCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    Assert(pCtx->hwvirt.enmHwvirt == CPUMHWVIRT_SVM);
+    Assert(pCtx->hwvirt.svm.CTX_SUFF(pVmcb));
+
+    memset(pCtx->hwvirt.svm.CTX_SUFF(pVmcb), 0, SVM_VMCB_PAGES << PAGE_SHIFT);
+    pCtx->hwvirt.svm.uMsrHSavePa    = 0;
+    pCtx->hwvirt.svm.uPrevPauseTick = 0;
+}
+
+
+/**
+ * Frees memory allocated for the VMX hardware virtualization state.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+static void cpumR3FreeVmxHwVirtState(PVM pVM)
+{
+    Assert(pVM->cpum.s.GuestFeatures.fVmx);
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU   pVCpu = pVM->apCpusR3[i];
+        PCPUMCTX pCtx  = &pVCpu->cpum.s.Guest;
+
+        if (pCtx->hwvirt.vmx.pVmcsR3)
+        {
+            SUPR3ContFree(pCtx->hwvirt.vmx.pVmcsR3, VMX_V_VMCS_PAGES);
+            pCtx->hwvirt.vmx.pVmcsR3 = NULL;
+        }
+        if (pCtx->hwvirt.vmx.pShadowVmcsR3)
+        {
+            SUPR3ContFree(pCtx->hwvirt.vmx.pShadowVmcsR3, VMX_V_VMCS_PAGES);
+            pCtx->hwvirt.vmx.pShadowVmcsR3 = NULL;
+        }
+        if (pCtx->hwvirt.vmx.pvVirtApicPageR3)
+        {
+            SUPR3ContFree(pCtx->hwvirt.vmx.pvVirtApicPageR3, VMX_V_VIRT_APIC_PAGES);
+            pCtx->hwvirt.vmx.pvVirtApicPageR3 = NULL;
+        }
+        if (pCtx->hwvirt.vmx.pvVmreadBitmapR3)
+        {
+            SUPR3ContFree(pCtx->hwvirt.vmx.pvVmreadBitmapR3, VMX_V_VMREAD_VMWRITE_BITMAP_PAGES);
+            pCtx->hwvirt.vmx.pvVmreadBitmapR3 = NULL;
+        }
+        if (pCtx->hwvirt.vmx.pvVmwriteBitmapR3)
+        {
+            SUPR3ContFree(pCtx->hwvirt.vmx.pvVmwriteBitmapR3, VMX_V_VMREAD_VMWRITE_BITMAP_PAGES);
+            pCtx->hwvirt.vmx.pvVmwriteBitmapR3 = NULL;
+        }
+        if (pCtx->hwvirt.vmx.pEntryMsrLoadAreaR3)
+        {
+            SUPR3ContFree(pCtx->hwvirt.vmx.pEntryMsrLoadAreaR3, VMX_V_AUTOMSR_AREA_PAGES);
+            pCtx->hwvirt.vmx.pEntryMsrLoadAreaR3 = NULL;
+        }
+        if (pCtx->hwvirt.vmx.pExitMsrStoreAreaR3)
+        {
+            SUPR3ContFree(pCtx->hwvirt.vmx.pExitMsrStoreAreaR3, VMX_V_AUTOMSR_AREA_PAGES);
+            pCtx->hwvirt.vmx.pExitMsrStoreAreaR3 = NULL;
+        }
+        if (pCtx->hwvirt.vmx.pExitMsrLoadAreaR3)
+        {
+            SUPR3ContFree(pCtx->hwvirt.vmx.pExitMsrLoadAreaR3, VMX_V_AUTOMSR_AREA_PAGES);
+            pCtx->hwvirt.vmx.pExitMsrLoadAreaR3 = NULL;
+        }
+        if (pCtx->hwvirt.vmx.pvMsrBitmapR3)
+        {
+            SUPR3ContFree(pCtx->hwvirt.vmx.pvMsrBitmapR3, VMX_V_MSR_BITMAP_PAGES);
+            pCtx->hwvirt.vmx.pvMsrBitmapR3 = NULL;
+        }
+        if (pCtx->hwvirt.vmx.pvIoBitmapR3)
+        {
+            SUPR3ContFree(pCtx->hwvirt.vmx.pvIoBitmapR3, VMX_V_IO_BITMAP_A_PAGES + VMX_V_IO_BITMAP_B_PAGES);
+            pCtx->hwvirt.vmx.pvIoBitmapR3 = NULL;
+        }
+    }
+}
+
+
+/**
+ * Allocates memory for the VMX hardware virtualization state.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+static int cpumR3AllocVmxHwVirtState(PVM pVM)
+{
+    int rc = VINF_SUCCESS;
+    uint32_t const cPages = VMX_V_VMCS_PAGES
+                          + VMX_V_SHADOW_VMCS_PAGES
+                          + VMX_V_VIRT_APIC_PAGES
+                          + (2 * VMX_V_VMREAD_VMWRITE_BITMAP_PAGES)
+                          + (3 * VMX_V_AUTOMSR_AREA_PAGES)
+                          + VMX_V_MSR_BITMAP_PAGES
+                          + (VMX_V_IO_BITMAP_A_PAGES + VMX_V_IO_BITMAP_B_PAGES);
+    LogRel(("CPUM: Allocating %u pages for the nested-guest VMCS and related structures\n", pVM->cCpus * cPages));
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU   pVCpu = pVM->apCpusR3[i];
+        PCPUMCTX pCtx  = &pVCpu->cpum.s.Guest;
+        pCtx->hwvirt.enmHwvirt = CPUMHWVIRT_VMX;
+
+        /*
+         * Allocate the nested-guest current VMCS.
+         */
+        pCtx->hwvirt.vmx.pVmcsR3 = (PVMXVVMCS)SUPR3ContAlloc(VMX_V_VMCS_PAGES,
+                                                             &pCtx->hwvirt.vmx.pVmcsR0,
+                                                             &pCtx->hwvirt.vmx.HCPhysVmcs);
+        if (pCtx->hwvirt.vmx.pVmcsR3)
+        { /* likely */ }
+        else
+        {
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's VMCS\n", pVCpu->idCpu, VMX_V_VMCS_PAGES));
+            break;
+        }
+
+        /*
+         * Allocate the nested-guest shadow VMCS.
+         */
+        pCtx->hwvirt.vmx.pShadowVmcsR3 = (PVMXVVMCS)SUPR3ContAlloc(VMX_V_VMCS_PAGES,
+                                                                   &pCtx->hwvirt.vmx.pShadowVmcsR0,
+                                                                   &pCtx->hwvirt.vmx.HCPhysShadowVmcs);
+        if (pCtx->hwvirt.vmx.pShadowVmcsR3)
+        { /* likely */ }
+        else
+        {
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's shadow VMCS\n", pVCpu->idCpu, VMX_V_VMCS_PAGES));
+            break;
+        }
+
+        /*
+         * Allocate the virtual-APIC page.
+         */
+        pCtx->hwvirt.vmx.pvVirtApicPageR3 = SUPR3ContAlloc(VMX_V_VIRT_APIC_PAGES,
+                                                           &pCtx->hwvirt.vmx.pvVirtApicPageR0,
+                                                           &pCtx->hwvirt.vmx.HCPhysVirtApicPage);
+        if (pCtx->hwvirt.vmx.pvVirtApicPageR3)
+        { /* likely */ }
+        else
+        {
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's virtual-APIC page\n", pVCpu->idCpu,
+                    VMX_V_VIRT_APIC_PAGES));
+            break;
+        }
+
+        /*
+         * Allocate the VMREAD-bitmap.
+         */
+        pCtx->hwvirt.vmx.pvVmreadBitmapR3 = SUPR3ContAlloc(VMX_V_VMREAD_VMWRITE_BITMAP_PAGES,
+                                                           &pCtx->hwvirt.vmx.pvVmreadBitmapR0,
+                                                           &pCtx->hwvirt.vmx.HCPhysVmreadBitmap);
+        if (pCtx->hwvirt.vmx.pvVmreadBitmapR3)
+        { /* likely */ }
+        else
+        {
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's VMREAD-bitmap\n", pVCpu->idCpu,
+                    VMX_V_VMREAD_VMWRITE_BITMAP_PAGES));
+            break;
+        }
+
+        /*
+         * Allocatge the VMWRITE-bitmap.
+         */
+        pCtx->hwvirt.vmx.pvVmwriteBitmapR3 = SUPR3ContAlloc(VMX_V_VMREAD_VMWRITE_BITMAP_PAGES,
+                                                            &pCtx->hwvirt.vmx.pvVmwriteBitmapR0,
+                                                            &pCtx->hwvirt.vmx.HCPhysVmwriteBitmap);
+        if (pCtx->hwvirt.vmx.pvVmwriteBitmapR3)
+        { /* likely */ }
+        else
+        {
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's VMWRITE-bitmap\n", pVCpu->idCpu,
+                    VMX_V_VMREAD_VMWRITE_BITMAP_PAGES));
+            break;
+        }
+
+        /*
+         * Allocate the VM-entry MSR-load area.
+         */
+        pCtx->hwvirt.vmx.pEntryMsrLoadAreaR3 = (PVMXAUTOMSR)SUPR3ContAlloc(VMX_V_AUTOMSR_AREA_PAGES,
+                                                                           &pCtx->hwvirt.vmx.pEntryMsrLoadAreaR0,
+                                                                           &pCtx->hwvirt.vmx.HCPhysEntryMsrLoadArea);
+        if (pCtx->hwvirt.vmx.pEntryMsrLoadAreaR3)
+        { /* likely */ }
+        else
+        {
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's VM-entry MSR-load area\n", pVCpu->idCpu,
+                    VMX_V_AUTOMSR_AREA_PAGES));
+            break;
+        }
+
+        /*
+         * Allocate the VM-exit MSR-store area.
+         */
+        pCtx->hwvirt.vmx.pExitMsrStoreAreaR3 = (PVMXAUTOMSR)SUPR3ContAlloc(VMX_V_AUTOMSR_AREA_PAGES,
+                                                                           &pCtx->hwvirt.vmx.pExitMsrStoreAreaR0,
+                                                                           &pCtx->hwvirt.vmx.HCPhysExitMsrStoreArea);
+        if (pCtx->hwvirt.vmx.pExitMsrStoreAreaR3)
+        { /* likely */ }
+        else
+        {
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's VM-exit MSR-store area\n", pVCpu->idCpu,
+                    VMX_V_AUTOMSR_AREA_PAGES));
+            break;
+        }
+
+        /*
+         * Allocate the VM-exit MSR-load area.
+         */
+        pCtx->hwvirt.vmx.pExitMsrLoadAreaR3 = (PVMXAUTOMSR)SUPR3ContAlloc(VMX_V_AUTOMSR_AREA_PAGES,
+                                                                          &pCtx->hwvirt.vmx.pExitMsrLoadAreaR0,
+                                                                          &pCtx->hwvirt.vmx.HCPhysExitMsrLoadArea);
+        if (pCtx->hwvirt.vmx.pExitMsrLoadAreaR3)
+        { /* likely */ }
+        else
+        {
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's VM-exit MSR-load area\n", pVCpu->idCpu,
+                    VMX_V_AUTOMSR_AREA_PAGES));
+            break;
+        }
+
+        /*
+         * Allocate the MSR bitmap.
+         */
+        pCtx->hwvirt.vmx.pvMsrBitmapR3 = SUPR3ContAlloc(VMX_V_MSR_BITMAP_PAGES,
+                                                        &pCtx->hwvirt.vmx.pvMsrBitmapR0,
+                                                        &pCtx->hwvirt.vmx.HCPhysMsrBitmap);
+        if (pCtx->hwvirt.vmx.pvMsrBitmapR3)
+        { /* likely */ }
+        else
+        {
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's MSR bitmap\n", pVCpu->idCpu,
+                    VMX_V_MSR_BITMAP_PAGES));
+            break;
+        }
+
+        /*
+         * Allocate the I/O bitmaps (A and B).
+         */
+        pCtx->hwvirt.vmx.pvIoBitmapR3 = SUPR3ContAlloc(VMX_V_IO_BITMAP_A_PAGES + VMX_V_IO_BITMAP_B_PAGES,
+                                                       &pCtx->hwvirt.vmx.pvIoBitmapR0,
+                                                       &pCtx->hwvirt.vmx.HCPhysIoBitmap);
+        if (pCtx->hwvirt.vmx.pvIoBitmapR3)
+        { /* likely */ }
+        else
+        {
+            LogRel(("CPUM%u: Failed to alloc %u pages for the nested-guest's I/O bitmaps\n", pVCpu->idCpu,
+                    VMX_V_IO_BITMAP_A_PAGES + VMX_V_IO_BITMAP_B_PAGES));
+            break;
+        }
+
+        /*
+         * Zero out all allocated pages (should compress well for saved-state).
+         */
+        memset(pCtx->hwvirt.vmx.CTX_SUFF(pVmcs),               0, VMX_V_VMCS_SIZE);
+        memset(pCtx->hwvirt.vmx.CTX_SUFF(pShadowVmcs),         0, VMX_V_SHADOW_VMCS_SIZE);
+        memset(pCtx->hwvirt.vmx.CTX_SUFF(pvVirtApicPage),      0, VMX_V_VIRT_APIC_SIZE);
+        memset(pCtx->hwvirt.vmx.CTX_SUFF(pvVmreadBitmap),      0, VMX_V_VMREAD_VMWRITE_BITMAP_SIZE);
+        memset(pCtx->hwvirt.vmx.CTX_SUFF(pvVmwriteBitmap),     0, VMX_V_VMREAD_VMWRITE_BITMAP_SIZE);
+        memset(pCtx->hwvirt.vmx.CTX_SUFF(pEntryMsrLoadArea),   0, VMX_V_AUTOMSR_AREA_SIZE);
+        memset(pCtx->hwvirt.vmx.CTX_SUFF(pExitMsrStoreArea),   0, VMX_V_AUTOMSR_AREA_SIZE);
+        memset(pCtx->hwvirt.vmx.CTX_SUFF(pExitMsrLoadArea),    0, VMX_V_AUTOMSR_AREA_SIZE);
+        memset(pCtx->hwvirt.vmx.CTX_SUFF(pvMsrBitmap),         0, VMX_V_MSR_BITMAP_SIZE);
+        memset(pCtx->hwvirt.vmx.CTX_SUFF(pvIoBitmap),          0, VMX_V_IO_BITMAP_A_SIZE + VMX_V_IO_BITMAP_B_SIZE);
+    }
+
+    /* On any failure, cleanup. */
+    if (RT_FAILURE(rc))
+        cpumR3FreeVmxHwVirtState(pVM);
+
+    return rc;
+}
+
+
+/**
+ * Resets per-VCPU VMX hardware virtualization state.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) cpumR3ResetVmxHwVirtState(PVMCPU pVCpu)
+{
+    PCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    Assert(pCtx->hwvirt.enmHwvirt == CPUMHWVIRT_VMX);
+    Assert(pCtx->hwvirt.vmx.CTX_SUFF(pVmcs));
+    Assert(pCtx->hwvirt.vmx.CTX_SUFF(pShadowVmcs));
+
+    memset(pCtx->hwvirt.vmx.CTX_SUFF(pVmcs),       0, VMX_V_VMCS_SIZE);
+    memset(pCtx->hwvirt.vmx.CTX_SUFF(pShadowVmcs), 0, VMX_V_SHADOW_VMCS_SIZE);
+    pCtx->hwvirt.vmx.GCPhysVmxon       = NIL_RTGCPHYS;
+    pCtx->hwvirt.vmx.GCPhysShadowVmcs  = NIL_RTGCPHYS;
+    pCtx->hwvirt.vmx.GCPhysVmxon       = NIL_RTGCPHYS;
+    pCtx->hwvirt.vmx.fInVmxRootMode    = false;
+    pCtx->hwvirt.vmx.fInVmxNonRootMode = false;
+    /* Don't reset diagnostics here. */
+
+    /* Stop any VMX-preemption timer. */
+    CPUMStopGuestVmxPremptTimer(pVCpu);
+
+    /* Clear all nested-guest FFs. */
+    VMCPU_FF_CLEAR_MASK(pVCpu, VMCPU_FF_VMX_ALL_MASK);
+}
+
+
+/**
+ * Displays the host and guest VMX features.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     "terse", "default" or "verbose".
+ */
+DECLCALLBACK(void) cpumR3InfoVmxFeatures(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    RT_NOREF(pszArgs);
+    PCCPUMFEATURES pHostFeatures  = &pVM->cpum.s.HostFeatures;
+    PCCPUMFEATURES pGuestFeatures = &pVM->cpum.s.GuestFeatures;
+    if (   pHostFeatures->enmCpuVendor == CPUMCPUVENDOR_INTEL
+        || pHostFeatures->enmCpuVendor == CPUMCPUVENDOR_VIA
+        || pHostFeatures->enmCpuVendor == CPUMCPUVENDOR_SHANGHAI)
+    {
+#define VMXFEATDUMP(a_szDesc, a_Var) \
+        pHlp->pfnPrintf(pHlp, "  %s = %u (%u)\n", a_szDesc, pGuestFeatures->a_Var, pHostFeatures->a_Var)
+
+        pHlp->pfnPrintf(pHlp, "Nested hardware virtualization - VMX features\n");
+        pHlp->pfnPrintf(pHlp, "  Mnemonic - Description                                  = guest (host)\n");
+        VMXFEATDUMP("VMX - Virtual-Machine Extensions                       ", fVmx);
+        /* Basic. */
+        VMXFEATDUMP("InsOutInfo - INS/OUTS instruction info.                ", fVmxInsOutInfo);
+        /* Pin-based controls. */
+        VMXFEATDUMP("ExtIntExit - External interrupt exiting                ", fVmxExtIntExit);
+        VMXFEATDUMP("NmiExit - NMI exiting                                  ", fVmxNmiExit);
+        VMXFEATDUMP("VirtNmi - Virtual NMIs                                 ", fVmxVirtNmi);
+        VMXFEATDUMP("PreemptTimer - VMX preemption timer                    ", fVmxPreemptTimer);
+        VMXFEATDUMP("PostedInt - Posted interrupts                          ", fVmxPostedInt);
+        /* Processor-based controls. */
+        VMXFEATDUMP("IntWindowExit - Interrupt-window exiting               ", fVmxIntWindowExit);
+        VMXFEATDUMP("TscOffsetting - TSC offsetting                         ", fVmxTscOffsetting);
+        VMXFEATDUMP("HltExit - HLT exiting                                  ", fVmxHltExit);
+        VMXFEATDUMP("InvlpgExit - INVLPG exiting                            ", fVmxInvlpgExit);
+        VMXFEATDUMP("MwaitExit - MWAIT exiting                              ", fVmxMwaitExit);
+        VMXFEATDUMP("RdpmcExit - RDPMC exiting                              ", fVmxRdpmcExit);
+        VMXFEATDUMP("RdtscExit - RDTSC exiting                              ", fVmxRdtscExit);
+        VMXFEATDUMP("Cr3LoadExit - CR3-load exiting                         ", fVmxCr3LoadExit);
+        VMXFEATDUMP("Cr3StoreExit - CR3-store exiting                       ", fVmxCr3StoreExit);
+        VMXFEATDUMP("Cr8LoadExit  - CR8-load exiting                        ", fVmxCr8LoadExit);
+        VMXFEATDUMP("Cr8StoreExit - CR8-store exiting                       ", fVmxCr8StoreExit);
+        VMXFEATDUMP("UseTprShadow - Use TPR shadow                          ", fVmxUseTprShadow);
+        VMXFEATDUMP("NmiWindowExit - NMI-window exiting                     ", fVmxNmiWindowExit);
+        VMXFEATDUMP("MovDRxExit - Mov-DR exiting                            ", fVmxMovDRxExit);
+        VMXFEATDUMP("UncondIoExit - Unconditional I/O exiting               ", fVmxUncondIoExit);
+        VMXFEATDUMP("UseIoBitmaps - Use I/O bitmaps                         ", fVmxUseIoBitmaps);
+        VMXFEATDUMP("MonitorTrapFlag - Monitor Trap Flag                    ", fVmxMonitorTrapFlag);
+        VMXFEATDUMP("UseMsrBitmaps - MSR bitmaps                            ", fVmxUseMsrBitmaps);
+        VMXFEATDUMP("MonitorExit - MONITOR exiting                          ", fVmxMonitorExit);
+        VMXFEATDUMP("PauseExit - PAUSE exiting                              ", fVmxPauseExit);
+        VMXFEATDUMP("SecondaryExecCtl - Activate secondary controls         ", fVmxSecondaryExecCtls);
+        /* Secondary processor-based controls. */
+        VMXFEATDUMP("VirtApic - Virtualize-APIC accesses                    ", fVmxVirtApicAccess);
+        VMXFEATDUMP("Ept - Extended Page Tables                             ", fVmxEpt);
+        VMXFEATDUMP("DescTableExit - Descriptor-table exiting               ", fVmxDescTableExit);
+        VMXFEATDUMP("Rdtscp - Enable RDTSCP                                 ", fVmxRdtscp);
+        VMXFEATDUMP("VirtX2ApicMode - Virtualize-x2APIC mode                ", fVmxVirtX2ApicMode);
+        VMXFEATDUMP("Vpid - Enable VPID                                     ", fVmxVpid);
+        VMXFEATDUMP("WbinvdExit - WBINVD exiting                            ", fVmxWbinvdExit);
+        VMXFEATDUMP("UnrestrictedGuest - Unrestricted guest                 ", fVmxUnrestrictedGuest);
+        VMXFEATDUMP("ApicRegVirt - APIC-register virtualization             ", fVmxApicRegVirt);
+        VMXFEATDUMP("VirtIntDelivery - Virtual-interrupt delivery           ", fVmxVirtIntDelivery);
+        VMXFEATDUMP("PauseLoopExit - PAUSE-loop exiting                     ", fVmxPauseLoopExit);
+        VMXFEATDUMP("RdrandExit - RDRAND exiting                            ", fVmxRdrandExit);
+        VMXFEATDUMP("Invpcid - Enable INVPCID                               ", fVmxInvpcid);
+        VMXFEATDUMP("VmFuncs - Enable VM Functions                          ", fVmxVmFunc);
+        VMXFEATDUMP("VmcsShadowing - VMCS shadowing                         ", fVmxVmcsShadowing);
+        VMXFEATDUMP("RdseedExiting - RDSEED exiting                         ", fVmxRdseedExit);
+        VMXFEATDUMP("PML - Page-Modification Log (PML)                      ", fVmxPml);
+        VMXFEATDUMP("EptVe - EPT violations can cause #VE                   ", fVmxEptXcptVe);
+        VMXFEATDUMP("XsavesXRstors - Enable XSAVES/XRSTORS                  ", fVmxXsavesXrstors);
+        /* VM-entry controls. */
+        VMXFEATDUMP("EntryLoadDebugCtls - Load debug controls on VM-entry   ", fVmxEntryLoadDebugCtls);
+        VMXFEATDUMP("Ia32eModeGuest - IA-32e mode guest                     ", fVmxIa32eModeGuest);
+        VMXFEATDUMP("EntryLoadEferMsr - Load IA32_EFER MSR on VM-entry      ", fVmxEntryLoadEferMsr);
+        VMXFEATDUMP("EntryLoadPatMsr - Load IA32_PAT MSR on VM-entry        ", fVmxEntryLoadPatMsr);
+        /* VM-exit controls. */
+        VMXFEATDUMP("ExitSaveDebugCtls - Save debug controls on VM-exit     ", fVmxExitSaveDebugCtls);
+        VMXFEATDUMP("HostAddrSpaceSize - Host address-space size            ", fVmxHostAddrSpaceSize);
+        VMXFEATDUMP("ExitAckExtInt - Acknowledge interrupt on VM-exit       ", fVmxExitAckExtInt);
+        VMXFEATDUMP("ExitSavePatMsr - Save IA32_PAT MSR on VM-exit          ", fVmxExitSavePatMsr);
+        VMXFEATDUMP("ExitLoadPatMsr - Load IA32_PAT MSR on VM-exit          ", fVmxExitLoadPatMsr);
+        VMXFEATDUMP("ExitSaveEferMsr - Save IA32_EFER MSR on VM-exit        ", fVmxExitSaveEferMsr);
+        VMXFEATDUMP("ExitLoadEferMsr - Load IA32_EFER MSR on VM-exit        ", fVmxExitLoadEferMsr);
+        VMXFEATDUMP("SavePreemptTimer - Save VMX-preemption timer           ", fVmxSavePreemptTimer);
+        /* Miscellaneous data. */
+        VMXFEATDUMP("ExitSaveEferLma - Save IA32_EFER.LMA on VM-exit        ", fVmxExitSaveEferLma);
+        VMXFEATDUMP("IntelPt - Intel PT (Processor Trace) in VMX operation  ", fVmxIntelPt);
+        VMXFEATDUMP("VmwriteAll - VMWRITE to any supported VMCS field       ", fVmxVmwriteAll);
+        VMXFEATDUMP("EntryInjectSoftInt - Inject softint. with 0-len instr. ", fVmxEntryInjectSoftInt);
+#undef VMXFEATDUMP
+    }
+    else
+        pHlp->pfnPrintf(pHlp, "No VMX features present - requires an Intel or compatible CPU.\n");
+}
+
+
+/**
+ * Checks whether nested-guest execution using hardware-assisted VMX (e.g, using HM
+ * or NEM) is allowed.
+ *
+ * @returns @c true if hardware-assisted nested-guest execution is allowed, @c false
+ *          otherwise.
+ * @param   pVM     The cross context VM structure.
+ */
+static bool cpumR3IsHwAssistNstGstExecAllowed(PVM pVM)
+{
+    AssertMsg(pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NOT_SET, ("Calling this function too early!\n"));
+#ifndef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
+    if (   pVM->bMainExecutionEngine == VM_EXEC_ENGINE_HW_VIRT
+        || pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        return true;
+#else
+    NOREF(pVM);
+#endif
+    return false;
+}
+
+
+/**
+ * Initializes the VMX guest MSRs from guest CPU features based on the host MSRs.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pHostVmxMsrs    The host VMX MSRs. Pass NULL when fully emulating VMX
+ *                          and no hardware-assisted nested-guest execution is
+ *                          possible for this VM.
+ * @param   pGuestFeatures  The guest features to use (only VMX features are
+ *                          accessed).
+ * @param   pGuestVmxMsrs   Where to store the initialized guest VMX MSRs.
+ *
+ * @remarks This function ASSUMES the VMX guest-features are already exploded!
+ */
+static void cpumR3InitVmxGuestMsrs(PVM pVM, PCVMXMSRS pHostVmxMsrs, PCCPUMFEATURES pGuestFeatures, PVMXMSRS pGuestVmxMsrs)
+{
+    bool const fIsNstGstHwExecAllowed = cpumR3IsHwAssistNstGstExecAllowed(pVM);
+
+    Assert(!fIsNstGstHwExecAllowed || pHostVmxMsrs);
+    Assert(pGuestFeatures->fVmx);
+
+    /*
+     * We don't support the following MSRs yet:
+     *   - True Pin-based VM-execution controls.
+     *   - True Processor-based VM-execution controls.
+     *   - True VM-entry VM-execution controls.
+     *   - True VM-exit VM-execution controls.
+     */
+
+    /* Feature control. */
+    pGuestVmxMsrs->u64FeatCtrl = MSR_IA32_FEATURE_CONTROL_LOCK | MSR_IA32_FEATURE_CONTROL_VMXON;
+
+    /* Basic information. */
+    {
+        uint64_t const u64Basic = RT_BF_MAKE(VMX_BF_BASIC_VMCS_ID,         VMX_V_VMCS_REVISION_ID        )
+                                | RT_BF_MAKE(VMX_BF_BASIC_VMCS_SIZE,       VMX_V_VMCS_SIZE               )
+                                | RT_BF_MAKE(VMX_BF_BASIC_PHYSADDR_WIDTH,  !pGuestFeatures->fLongMode    )
+                                | RT_BF_MAKE(VMX_BF_BASIC_DUAL_MON,        0                             )
+                                | RT_BF_MAKE(VMX_BF_BASIC_VMCS_MEM_TYPE,   VMX_BASIC_MEM_TYPE_WB         )
+                                | RT_BF_MAKE(VMX_BF_BASIC_VMCS_INS_OUTS,   pGuestFeatures->fVmxInsOutInfo)
+                                | RT_BF_MAKE(VMX_BF_BASIC_TRUE_CTLS,       0                             );
+        pGuestVmxMsrs->u64Basic = u64Basic;
+    }
+
+    /* Pin-based VM-execution controls. */
+    {
+        uint32_t const fFeatures = (pGuestFeatures->fVmxExtIntExit   << VMX_BF_PIN_CTLS_EXT_INT_EXIT_SHIFT )
+                                 | (pGuestFeatures->fVmxNmiExit      << VMX_BF_PIN_CTLS_NMI_EXIT_SHIFT     )
+                                 | (pGuestFeatures->fVmxVirtNmi      << VMX_BF_PIN_CTLS_VIRT_NMI_SHIFT     )
+                                 | (pGuestFeatures->fVmxPreemptTimer << VMX_BF_PIN_CTLS_PREEMPT_TIMER_SHIFT)
+                                 | (pGuestFeatures->fVmxPostedInt    << VMX_BF_PIN_CTLS_POSTED_INT_SHIFT   );
+        uint32_t const fAllowed0 = VMX_PIN_CTLS_DEFAULT1;
+        uint32_t const fAllowed1 = fFeatures | VMX_PIN_CTLS_DEFAULT1;
+        AssertMsg((fAllowed0 & fAllowed1) == fAllowed0, ("fAllowed0=%#RX32 fAllowed1=%#RX32 fFeatures=%#RX32\n",
+                                                         fAllowed0, fAllowed1, fFeatures));
+        pGuestVmxMsrs->PinCtls.u = RT_MAKE_U64(fAllowed0, fAllowed1);
+    }
+
+    /* Processor-based VM-execution controls. */
+    {
+        uint32_t const fFeatures = (pGuestFeatures->fVmxIntWindowExit     << VMX_BF_PROC_CTLS_INT_WINDOW_EXIT_SHIFT   )
+                                 | (pGuestFeatures->fVmxTscOffsetting     << VMX_BF_PROC_CTLS_USE_TSC_OFFSETTING_SHIFT)
+                                 | (pGuestFeatures->fVmxHltExit           << VMX_BF_PROC_CTLS_HLT_EXIT_SHIFT          )
+                                 | (pGuestFeatures->fVmxInvlpgExit        << VMX_BF_PROC_CTLS_INVLPG_EXIT_SHIFT       )
+                                 | (pGuestFeatures->fVmxMwaitExit         << VMX_BF_PROC_CTLS_MWAIT_EXIT_SHIFT        )
+                                 | (pGuestFeatures->fVmxRdpmcExit         << VMX_BF_PROC_CTLS_RDPMC_EXIT_SHIFT        )
+                                 | (pGuestFeatures->fVmxRdtscExit         << VMX_BF_PROC_CTLS_RDTSC_EXIT_SHIFT        )
+                                 | (pGuestFeatures->fVmxCr3LoadExit       << VMX_BF_PROC_CTLS_CR3_LOAD_EXIT_SHIFT     )
+                                 | (pGuestFeatures->fVmxCr3StoreExit      << VMX_BF_PROC_CTLS_CR3_STORE_EXIT_SHIFT    )
+                                 | (pGuestFeatures->fVmxCr8LoadExit       << VMX_BF_PROC_CTLS_CR8_LOAD_EXIT_SHIFT     )
+                                 | (pGuestFeatures->fVmxCr8StoreExit      << VMX_BF_PROC_CTLS_CR8_STORE_EXIT_SHIFT    )
+                                 | (pGuestFeatures->fVmxUseTprShadow      << VMX_BF_PROC_CTLS_USE_TPR_SHADOW_SHIFT    )
+                                 | (pGuestFeatures->fVmxNmiWindowExit     << VMX_BF_PROC_CTLS_NMI_WINDOW_EXIT_SHIFT   )
+                                 | (pGuestFeatures->fVmxMovDRxExit        << VMX_BF_PROC_CTLS_MOV_DR_EXIT_SHIFT       )
+                                 | (pGuestFeatures->fVmxUncondIoExit      << VMX_BF_PROC_CTLS_UNCOND_IO_EXIT_SHIFT    )
+                                 | (pGuestFeatures->fVmxUseIoBitmaps      << VMX_BF_PROC_CTLS_USE_IO_BITMAPS_SHIFT    )
+                                 | (pGuestFeatures->fVmxMonitorTrapFlag   << VMX_BF_PROC_CTLS_MONITOR_TRAP_FLAG_SHIFT )
+                                 | (pGuestFeatures->fVmxUseMsrBitmaps     << VMX_BF_PROC_CTLS_USE_MSR_BITMAPS_SHIFT   )
+                                 | (pGuestFeatures->fVmxMonitorExit       << VMX_BF_PROC_CTLS_MONITOR_EXIT_SHIFT      )
+                                 | (pGuestFeatures->fVmxPauseExit         << VMX_BF_PROC_CTLS_PAUSE_EXIT_SHIFT        )
+                                 | (pGuestFeatures->fVmxSecondaryExecCtls << VMX_BF_PROC_CTLS_USE_SECONDARY_CTLS_SHIFT);
+        uint32_t const fAllowed0 = VMX_PROC_CTLS_DEFAULT1;
+        uint32_t const fAllowed1 = fFeatures | VMX_PROC_CTLS_DEFAULT1;
+        AssertMsg((fAllowed0 & fAllowed1) == fAllowed0, ("fAllowed0=%#RX32 fAllowed1=%#RX32 fFeatures=%#RX32\n", fAllowed0,
+                                                         fAllowed1, fFeatures));
+        pGuestVmxMsrs->ProcCtls.u = RT_MAKE_U64(fAllowed0, fAllowed1);
+    }
+
+    /* Secondary processor-based VM-execution controls. */
+    if (pGuestFeatures->fVmxSecondaryExecCtls)
+    {
+        uint32_t const fFeatures = (pGuestFeatures->fVmxVirtApicAccess    << VMX_BF_PROC_CTLS2_VIRT_APIC_ACCESS_SHIFT  )
+                                 | (pGuestFeatures->fVmxEpt               << VMX_BF_PROC_CTLS2_EPT_SHIFT               )
+                                 | (pGuestFeatures->fVmxDescTableExit     << VMX_BF_PROC_CTLS2_DESC_TABLE_EXIT_SHIFT   )
+                                 | (pGuestFeatures->fVmxRdtscp            << VMX_BF_PROC_CTLS2_RDTSCP_SHIFT            )
+                                 | (pGuestFeatures->fVmxVirtX2ApicMode    << VMX_BF_PROC_CTLS2_VIRT_X2APIC_MODE_SHIFT  )
+                                 | (pGuestFeatures->fVmxVpid              << VMX_BF_PROC_CTLS2_VPID_SHIFT              )
+                                 | (pGuestFeatures->fVmxWbinvdExit        << VMX_BF_PROC_CTLS2_WBINVD_EXIT_SHIFT       )
+                                 | (pGuestFeatures->fVmxUnrestrictedGuest << VMX_BF_PROC_CTLS2_UNRESTRICTED_GUEST_SHIFT)
+                                 | (pGuestFeatures->fVmxApicRegVirt       << VMX_BF_PROC_CTLS2_APIC_REG_VIRT_SHIFT     )
+                                 | (pGuestFeatures->fVmxVirtIntDelivery   << VMX_BF_PROC_CTLS2_VIRT_INT_DELIVERY_SHIFT )
+                                 | (pGuestFeatures->fVmxPauseLoopExit     << VMX_BF_PROC_CTLS2_PAUSE_LOOP_EXIT_SHIFT   )
+                                 | (pGuestFeatures->fVmxRdrandExit        << VMX_BF_PROC_CTLS2_RDRAND_EXIT_SHIFT       )
+                                 | (pGuestFeatures->fVmxInvpcid           << VMX_BF_PROC_CTLS2_INVPCID_SHIFT           )
+                                 | (pGuestFeatures->fVmxVmFunc            << VMX_BF_PROC_CTLS2_VMFUNC_SHIFT            )
+                                 | (pGuestFeatures->fVmxVmcsShadowing     << VMX_BF_PROC_CTLS2_VMCS_SHADOWING_SHIFT    )
+                                 | (pGuestFeatures->fVmxRdseedExit        << VMX_BF_PROC_CTLS2_RDSEED_EXIT_SHIFT       )
+                                 | (pGuestFeatures->fVmxPml               << VMX_BF_PROC_CTLS2_PML_SHIFT               )
+                                 | (pGuestFeatures->fVmxEptXcptVe         << VMX_BF_PROC_CTLS2_EPT_VE_SHIFT            )
+                                 | (pGuestFeatures->fVmxXsavesXrstors     << VMX_BF_PROC_CTLS2_XSAVES_XRSTORS_SHIFT    )
+                                 | (pGuestFeatures->fVmxUseTscScaling     << VMX_BF_PROC_CTLS2_TSC_SCALING_SHIFT       );
+        uint32_t const fAllowed0 = 0;
+        uint32_t const fAllowed1 = fFeatures;
+        pGuestVmxMsrs->ProcCtls2.u = RT_MAKE_U64(fAllowed0, fAllowed1);
+    }
+
+    /* VM-exit controls. */
+    {
+        uint32_t const fFeatures = (pGuestFeatures->fVmxExitSaveDebugCtls << VMX_BF_EXIT_CTLS_SAVE_DEBUG_SHIFT          )
+                                 | (pGuestFeatures->fVmxHostAddrSpaceSize << VMX_BF_EXIT_CTLS_HOST_ADDR_SPACE_SIZE_SHIFT)
+                                 | (pGuestFeatures->fVmxExitAckExtInt     << VMX_BF_EXIT_CTLS_ACK_EXT_INT_SHIFT         )
+                                 | (pGuestFeatures->fVmxExitSavePatMsr    << VMX_BF_EXIT_CTLS_SAVE_PAT_MSR_SHIFT        )
+                                 | (pGuestFeatures->fVmxExitLoadPatMsr    << VMX_BF_EXIT_CTLS_LOAD_PAT_MSR_SHIFT        )
+                                 | (pGuestFeatures->fVmxExitSaveEferMsr   << VMX_BF_EXIT_CTLS_SAVE_EFER_MSR_SHIFT       )
+                                 | (pGuestFeatures->fVmxExitLoadEferMsr   << VMX_BF_EXIT_CTLS_LOAD_EFER_MSR_SHIFT       )
+                                 | (pGuestFeatures->fVmxSavePreemptTimer  << VMX_BF_EXIT_CTLS_SAVE_PREEMPT_TIMER_SHIFT  );
+        /* Set the default1 class bits. See Intel spec. A.4 "VM-exit Controls". */
+        uint32_t const fAllowed0 = VMX_EXIT_CTLS_DEFAULT1;
+        uint32_t const fAllowed1 = fFeatures | VMX_EXIT_CTLS_DEFAULT1;
+        AssertMsg((fAllowed0 & fAllowed1) == fAllowed0, ("fAllowed0=%#RX32 fAllowed1=%#RX32 fFeatures=%#RX32\n", fAllowed0,
+                                                         fAllowed1, fFeatures));
+        pGuestVmxMsrs->ExitCtls.u = RT_MAKE_U64(fAllowed0, fAllowed1);
+    }
+
+    /* VM-entry controls. */
+    {
+        uint32_t const fFeatures = (pGuestFeatures->fVmxEntryLoadDebugCtls << VMX_BF_ENTRY_CTLS_LOAD_DEBUG_SHIFT      )
+                                 | (pGuestFeatures->fVmxIa32eModeGuest     << VMX_BF_ENTRY_CTLS_IA32E_MODE_GUEST_SHIFT)
+                                 | (pGuestFeatures->fVmxEntryLoadEferMsr   << VMX_BF_ENTRY_CTLS_LOAD_EFER_MSR_SHIFT   )
+                                 | (pGuestFeatures->fVmxEntryLoadPatMsr    << VMX_BF_ENTRY_CTLS_LOAD_PAT_MSR_SHIFT    );
+        uint32_t const fAllowed0 = VMX_ENTRY_CTLS_DEFAULT1;
+        uint32_t const fAllowed1 = fFeatures | VMX_ENTRY_CTLS_DEFAULT1;
+        AssertMsg((fAllowed0 & fAllowed1) == fAllowed0, ("fAllowed0=%#RX32 fAllowed0=%#RX32 fFeatures=%#RX32\n", fAllowed0,
+                                                         fAllowed1, fFeatures));
+        pGuestVmxMsrs->EntryCtls.u = RT_MAKE_U64(fAllowed0, fAllowed1);
+    }
+
+    /* Miscellaneous data. */
+    {
+        uint64_t const uHostMsr = fIsNstGstHwExecAllowed ? pHostVmxMsrs->u64Misc : 0;
+
+        uint8_t const  cMaxMsrs       = RT_MIN(RT_BF_GET(uHostMsr, VMX_BF_MISC_MAX_MSRS), VMX_V_AUTOMSR_COUNT_MAX);
+        uint8_t const  fActivityState = RT_BF_GET(uHostMsr, VMX_BF_MISC_ACTIVITY_STATES) & VMX_V_GUEST_ACTIVITY_STATE_MASK;
+        pGuestVmxMsrs->u64Misc = RT_BF_MAKE(VMX_BF_MISC_PREEMPT_TIMER_TSC,      VMX_V_PREEMPT_TIMER_SHIFT             )
+                               | RT_BF_MAKE(VMX_BF_MISC_EXIT_SAVE_EFER_LMA,     pGuestFeatures->fVmxExitSaveEferLma   )
+                               | RT_BF_MAKE(VMX_BF_MISC_ACTIVITY_STATES,        fActivityState                        )
+                               | RT_BF_MAKE(VMX_BF_MISC_INTEL_PT,               pGuestFeatures->fVmxIntelPt           )
+                               | RT_BF_MAKE(VMX_BF_MISC_SMM_READ_SMBASE_MSR,    0                                     )
+                               | RT_BF_MAKE(VMX_BF_MISC_CR3_TARGET,             VMX_V_CR3_TARGET_COUNT                )
+                               | RT_BF_MAKE(VMX_BF_MISC_MAX_MSRS,               cMaxMsrs                              )
+                               | RT_BF_MAKE(VMX_BF_MISC_VMXOFF_BLOCK_SMI,       0                                     )
+                               | RT_BF_MAKE(VMX_BF_MISC_VMWRITE_ALL,            pGuestFeatures->fVmxVmwriteAll        )
+                               | RT_BF_MAKE(VMX_BF_MISC_ENTRY_INJECT_SOFT_INT,  pGuestFeatures->fVmxEntryInjectSoftInt)
+                               | RT_BF_MAKE(VMX_BF_MISC_MSEG_ID,                VMX_V_MSEG_REV_ID                     );
+    }
+
+    /* CR0 Fixed-0. */
+    pGuestVmxMsrs->u64Cr0Fixed0 = pGuestFeatures->fVmxUnrestrictedGuest ? VMX_V_CR0_FIXED0_UX : VMX_V_CR0_FIXED0;
+
+    /* CR0 Fixed-1. */
+    {
+        /*
+         * All CPUs I've looked at so far report CR0 fixed-1 bits as 0xffffffff.
+         * This is different from CR4 fixed-1 bits which are reported as per the
+         * CPU features and/or micro-architecture/generation. Why? Ask Intel.
+         */
+        uint64_t const uHostMsr = fIsNstGstHwExecAllowed ? pHostVmxMsrs->u64Cr0Fixed1 : 0xffffffff;
+        pGuestVmxMsrs->u64Cr0Fixed1 = uHostMsr | pGuestVmxMsrs->u64Cr0Fixed0;   /* Make sure the CR0 MB1 bits are not clear. */
+    }
+
+    /* CR4 Fixed-0. */
+    pGuestVmxMsrs->u64Cr4Fixed0 = VMX_V_CR4_FIXED0;
+
+    /* CR4 Fixed-1. */
+    {
+        uint64_t const uHostMsr = fIsNstGstHwExecAllowed ? pHostVmxMsrs->u64Cr4Fixed1 : CPUMGetGuestCR4ValidMask(pVM);
+        pGuestVmxMsrs->u64Cr4Fixed1 = uHostMsr | pGuestVmxMsrs->u64Cr4Fixed0;   /* Make sure the CR4 MB1 bits are not clear. */
+    }
+
+    /* VMCS Enumeration. */
+    pGuestVmxMsrs->u64VmcsEnum = VMX_V_VMCS_MAX_INDEX << VMX_BF_VMCS_ENUM_HIGHEST_IDX_SHIFT;
+
+    /* VPID and EPT Capabilities. */
+    {
+        /*
+         * INVVPID instruction always causes a VM-exit unconditionally, so we are free to fake
+         * and emulate any INVVPID flush type. However, it only makes sense to expose the types
+         * when INVVPID instruction is supported just to be more compatible with guest
+         * hypervisors that may make assumptions by only looking at this MSR even though they
+         * are technically supposed to refer to bit 37 of MSR_IA32_VMX_PROC_CTLS2 first.
+         *
+         * See Intel spec. 25.1.2 "Instructions That Cause VM Exits Unconditionally".
+         * See Intel spec. 30.3 "VMX Instructions".
+         */
+        uint8_t const fVpid = pGuestFeatures->fVmxVpid;
+        pGuestVmxMsrs->u64EptVpidCaps = RT_BF_MAKE(VMX_BF_EPT_VPID_CAP_INVVPID,                           fVpid)
+                                      | RT_BF_MAKE(VMX_BF_EPT_VPID_CAP_INVVPID_SINGLE_CTX,                fVpid & 1)
+                                      | RT_BF_MAKE(VMX_BF_EPT_VPID_CAP_INVVPID_ALL_CTX,                   fVpid & 1)
+                                      | RT_BF_MAKE(VMX_BF_EPT_VPID_CAP_INVVPID_SINGLE_CTX_RETAIN_GLOBALS, fVpid & 1);
+    }
+
+    /* VM Functions. */
+    if (pGuestFeatures->fVmxVmFunc)
+        pGuestVmxMsrs->u64VmFunc = RT_BF_MAKE(VMX_BF_VMFUNC_EPTP_SWITCHING, 1);
+}
+
+
+/**
+ * Checks whether the given guest CPU VMX features are compatible with the provided
+ * base features.
+ *
+ * @returns @c true if compatible, @c false otherwise.
+ * @param   pVM     The cross context VM structure.
+ * @param   pBase   The base VMX CPU features.
+ * @param   pGst    The guest VMX CPU features.
+ *
+ * @remarks Only VMX feature bits are examined.
+ */
+static bool cpumR3AreVmxCpuFeaturesCompatible(PVM pVM, PCCPUMFEATURES pBase, PCCPUMFEATURES pGst)
+{
+    if (cpumR3IsHwAssistNstGstExecAllowed(pVM))
+    {
+        uint64_t const fBase = ((uint64_t)pBase->fVmxInsOutInfo         <<  0) | ((uint64_t)pBase->fVmxExtIntExit         <<  1)
+                             | ((uint64_t)pBase->fVmxNmiExit            <<  2) | ((uint64_t)pBase->fVmxVirtNmi            <<  3)
+                             | ((uint64_t)pBase->fVmxPreemptTimer       <<  4) | ((uint64_t)pBase->fVmxPostedInt          <<  5)
+                             | ((uint64_t)pBase->fVmxIntWindowExit      <<  6) | ((uint64_t)pBase->fVmxTscOffsetting      <<  7)
+                             | ((uint64_t)pBase->fVmxHltExit            <<  8) | ((uint64_t)pBase->fVmxInvlpgExit         <<  9)
+                             | ((uint64_t)pBase->fVmxMwaitExit          << 10) | ((uint64_t)pBase->fVmxRdpmcExit          << 11)
+                             | ((uint64_t)pBase->fVmxRdtscExit          << 12) | ((uint64_t)pBase->fVmxCr3LoadExit        << 13)
+                             | ((uint64_t)pBase->fVmxCr3StoreExit       << 14) | ((uint64_t)pBase->fVmxCr8LoadExit        << 15)
+                             | ((uint64_t)pBase->fVmxCr8StoreExit       << 16) | ((uint64_t)pBase->fVmxUseTprShadow       << 17)
+                             | ((uint64_t)pBase->fVmxNmiWindowExit      << 18) | ((uint64_t)pBase->fVmxMovDRxExit         << 19)
+                             | ((uint64_t)pBase->fVmxUncondIoExit       << 20) | ((uint64_t)pBase->fVmxUseIoBitmaps       << 21)
+                             | ((uint64_t)pBase->fVmxMonitorTrapFlag    << 22) | ((uint64_t)pBase->fVmxUseMsrBitmaps      << 23)
+                             | ((uint64_t)pBase->fVmxMonitorExit        << 24) | ((uint64_t)pBase->fVmxPauseExit          << 25)
+                             | ((uint64_t)pBase->fVmxSecondaryExecCtls  << 26) | ((uint64_t)pBase->fVmxVirtApicAccess     << 27)
+                             | ((uint64_t)pBase->fVmxEpt                << 28) | ((uint64_t)pBase->fVmxDescTableExit      << 29)
+                             | ((uint64_t)pBase->fVmxRdtscp             << 30) | ((uint64_t)pBase->fVmxVirtX2ApicMode     << 31)
+                             | ((uint64_t)pBase->fVmxVpid               << 32) | ((uint64_t)pBase->fVmxWbinvdExit         << 33)
+                             | ((uint64_t)pBase->fVmxUnrestrictedGuest  << 34) | ((uint64_t)pBase->fVmxApicRegVirt        << 35)
+                             | ((uint64_t)pBase->fVmxVirtIntDelivery    << 36) | ((uint64_t)pBase->fVmxPauseLoopExit      << 37)
+                             | ((uint64_t)pBase->fVmxRdrandExit         << 38) | ((uint64_t)pBase->fVmxInvpcid            << 39)
+                             | ((uint64_t)pBase->fVmxVmFunc             << 40) | ((uint64_t)pBase->fVmxVmcsShadowing      << 41)
+                             | ((uint64_t)pBase->fVmxRdseedExit         << 42) | ((uint64_t)pBase->fVmxPml                << 43)
+                             | ((uint64_t)pBase->fVmxEptXcptVe          << 44) | ((uint64_t)pBase->fVmxXsavesXrstors      << 45)
+                             | ((uint64_t)pBase->fVmxUseTscScaling      << 46) | ((uint64_t)pBase->fVmxEntryLoadDebugCtls << 47)
+                             | ((uint64_t)pBase->fVmxIa32eModeGuest     << 48) | ((uint64_t)pBase->fVmxEntryLoadEferMsr   << 49)
+                             | ((uint64_t)pBase->fVmxEntryLoadPatMsr    << 50) | ((uint64_t)pBase->fVmxExitSaveDebugCtls  << 51)
+                             | ((uint64_t)pBase->fVmxHostAddrSpaceSize  << 52) | ((uint64_t)pBase->fVmxExitAckExtInt      << 53)
+                             | ((uint64_t)pBase->fVmxExitSavePatMsr     << 54) | ((uint64_t)pBase->fVmxExitLoadPatMsr     << 55)
+                             | ((uint64_t)pBase->fVmxExitSaveEferMsr    << 56) | ((uint64_t)pBase->fVmxExitLoadEferMsr    << 57)
+                             | ((uint64_t)pBase->fVmxSavePreemptTimer   << 58) | ((uint64_t)pBase->fVmxExitSaveEferLma    << 59)
+                             | ((uint64_t)pBase->fVmxIntelPt            << 60) | ((uint64_t)pBase->fVmxVmwriteAll         << 61)
+                             | ((uint64_t)pBase->fVmxEntryInjectSoftInt << 62);
+
+        uint64_t const fGst  = ((uint64_t)pGst->fVmxInsOutInfo          <<  0) | ((uint64_t)pGst->fVmxExtIntExit          <<  1)
+                             | ((uint64_t)pGst->fVmxNmiExit             <<  2) | ((uint64_t)pGst->fVmxVirtNmi             <<  3)
+                             | ((uint64_t)pGst->fVmxPreemptTimer        <<  4) | ((uint64_t)pGst->fVmxPostedInt           <<  5)
+                             | ((uint64_t)pGst->fVmxIntWindowExit       <<  6) | ((uint64_t)pGst->fVmxTscOffsetting       <<  7)
+                             | ((uint64_t)pGst->fVmxHltExit             <<  8) | ((uint64_t)pGst->fVmxInvlpgExit          <<  9)
+                             | ((uint64_t)pGst->fVmxMwaitExit           << 10) | ((uint64_t)pGst->fVmxRdpmcExit           << 11)
+                             | ((uint64_t)pGst->fVmxRdtscExit           << 12) | ((uint64_t)pGst->fVmxCr3LoadExit         << 13)
+                             | ((uint64_t)pGst->fVmxCr3StoreExit        << 14) | ((uint64_t)pGst->fVmxCr8LoadExit         << 15)
+                             | ((uint64_t)pGst->fVmxCr8StoreExit        << 16) | ((uint64_t)pGst->fVmxUseTprShadow        << 17)
+                             | ((uint64_t)pGst->fVmxNmiWindowExit       << 18) | ((uint64_t)pGst->fVmxMovDRxExit          << 19)
+                             | ((uint64_t)pGst->fVmxUncondIoExit        << 20) | ((uint64_t)pGst->fVmxUseIoBitmaps        << 21)
+                             | ((uint64_t)pGst->fVmxMonitorTrapFlag     << 22) | ((uint64_t)pGst->fVmxUseMsrBitmaps       << 23)
+                             | ((uint64_t)pGst->fVmxMonitorExit         << 24) | ((uint64_t)pGst->fVmxPauseExit           << 25)
+                             | ((uint64_t)pGst->fVmxSecondaryExecCtls   << 26) | ((uint64_t)pGst->fVmxVirtApicAccess      << 27)
+                             | ((uint64_t)pGst->fVmxEpt                 << 28) | ((uint64_t)pGst->fVmxDescTableExit       << 29)
+                             | ((uint64_t)pGst->fVmxRdtscp              << 30) | ((uint64_t)pGst->fVmxVirtX2ApicMode      << 31)
+                             | ((uint64_t)pGst->fVmxVpid                << 32) | ((uint64_t)pGst->fVmxWbinvdExit          << 33)
+                             | ((uint64_t)pGst->fVmxUnrestrictedGuest   << 34) | ((uint64_t)pGst->fVmxApicRegVirt         << 35)
+                             | ((uint64_t)pGst->fVmxVirtIntDelivery     << 36) | ((uint64_t)pGst->fVmxPauseLoopExit       << 37)
+                             | ((uint64_t)pGst->fVmxRdrandExit          << 38) | ((uint64_t)pGst->fVmxInvpcid             << 39)
+                             | ((uint64_t)pGst->fVmxVmFunc              << 40) | ((uint64_t)pGst->fVmxVmcsShadowing       << 41)
+                             | ((uint64_t)pGst->fVmxRdseedExit          << 42) | ((uint64_t)pGst->fVmxPml                 << 43)
+                             | ((uint64_t)pGst->fVmxEptXcptVe           << 44) | ((uint64_t)pGst->fVmxXsavesXrstors       << 45)
+                             | ((uint64_t)pGst->fVmxUseTscScaling       << 46) | ((uint64_t)pGst->fVmxEntryLoadDebugCtls  << 47)
+                             | ((uint64_t)pGst->fVmxIa32eModeGuest      << 48) | ((uint64_t)pGst->fVmxEntryLoadEferMsr    << 49)
+                             | ((uint64_t)pGst->fVmxEntryLoadPatMsr     << 50) | ((uint64_t)pGst->fVmxExitSaveDebugCtls   << 51)
+                             | ((uint64_t)pGst->fVmxHostAddrSpaceSize   << 52) | ((uint64_t)pGst->fVmxExitAckExtInt       << 53)
+                             | ((uint64_t)pGst->fVmxExitSavePatMsr      << 54) | ((uint64_t)pGst->fVmxExitLoadPatMsr      << 55)
+                             | ((uint64_t)pGst->fVmxExitSaveEferMsr     << 56) | ((uint64_t)pGst->fVmxExitLoadEferMsr     << 57)
+                             | ((uint64_t)pGst->fVmxSavePreemptTimer    << 58) | ((uint64_t)pGst->fVmxExitSaveEferLma     << 59)
+                             | ((uint64_t)pGst->fVmxIntelPt             << 60) | ((uint64_t)pGst->fVmxVmwriteAll          << 61)
+                             | ((uint64_t)pGst->fVmxEntryInjectSoftInt  << 62);
+
+        if ((fBase | fGst) != fBase)
+        {
+            uint64_t const fDiff = fBase ^ fGst;
+            LogRel(("CPUM: VMX features now exposed to the guest are incompatible with those from the saved state. fBase=%#RX64 fGst=%#RX64 fDiff=%#RX64\n",
+                    fBase, fGst, fDiff));
+            return false;
+        }
+        return true;
+    }
+    return true;
+}
+
+
+/**
+ * Initializes VMX guest features and MSRs.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pHostVmxMsrs    The host VMX MSRs. Pass NULL when fully emulating VMX
+ *                          and no hardware-assisted nested-guest execution is
+ *                          possible for this VM.
+ * @param   pGuestVmxMsrs   Where to store the initialized guest VMX MSRs.
+ */
+void cpumR3InitVmxGuestFeaturesAndMsrs(PVM pVM, PCVMXMSRS pHostVmxMsrs, PVMXMSRS pGuestVmxMsrs)
+{
+    Assert(pVM);
+    Assert(pGuestVmxMsrs);
+
+    /*
+     * Initialize the set of VMX features we emulate.
+     *
+     * Note! Some bits might be reported as 1 always if they fall under the
+     * default1 class bits (e.g. fVmxEntryLoadDebugCtls), see @bugref{9180#c5}.
+     */
+    CPUMFEATURES EmuFeat;
+    RT_ZERO(EmuFeat);
+    EmuFeat.fVmx                      = 1;
+    EmuFeat.fVmxInsOutInfo            = 1;
+    EmuFeat.fVmxExtIntExit            = 1;
+    EmuFeat.fVmxNmiExit               = 1;
+    EmuFeat.fVmxVirtNmi               = 1;
+    EmuFeat.fVmxPreemptTimer          = 0;  /* Currently disabled on purpose, see @bugref{9180#c108}. */
+    EmuFeat.fVmxPostedInt             = 0;
+    EmuFeat.fVmxIntWindowExit         = 1;
+    EmuFeat.fVmxTscOffsetting         = 1;
+    EmuFeat.fVmxHltExit               = 1;
+    EmuFeat.fVmxInvlpgExit            = 1;
+    EmuFeat.fVmxMwaitExit             = 1;
+    EmuFeat.fVmxRdpmcExit             = 1;
+    EmuFeat.fVmxRdtscExit             = 1;
+    EmuFeat.fVmxCr3LoadExit           = 1;
+    EmuFeat.fVmxCr3StoreExit          = 1;
+    EmuFeat.fVmxCr8LoadExit           = 1;
+    EmuFeat.fVmxCr8StoreExit          = 1;
+    EmuFeat.fVmxUseTprShadow          = 1;
+    EmuFeat.fVmxNmiWindowExit         = 0;
+    EmuFeat.fVmxMovDRxExit            = 1;
+    EmuFeat.fVmxUncondIoExit          = 1;
+    EmuFeat.fVmxUseIoBitmaps          = 1;
+    EmuFeat.fVmxMonitorTrapFlag       = 0;
+    EmuFeat.fVmxUseMsrBitmaps         = 1;
+    EmuFeat.fVmxMonitorExit           = 1;
+    EmuFeat.fVmxPauseExit             = 1;
+    EmuFeat.fVmxSecondaryExecCtls     = 1;
+    EmuFeat.fVmxVirtApicAccess        = 1;
+    EmuFeat.fVmxEpt                   = 0;  /* Cannot be disabled if unrestricted guest is enabled. */
+    EmuFeat.fVmxDescTableExit         = 1;
+    EmuFeat.fVmxRdtscp                = 1;
+    EmuFeat.fVmxVirtX2ApicMode        = 0;
+    EmuFeat.fVmxVpid                  = 0;  /** @todo NSTVMX: enable this. */
+    EmuFeat.fVmxWbinvdExit            = 1;
+    EmuFeat.fVmxUnrestrictedGuest     = 0;
+    EmuFeat.fVmxApicRegVirt           = 0;
+    EmuFeat.fVmxVirtIntDelivery       = 0;
+    EmuFeat.fVmxPauseLoopExit         = 0;
+    EmuFeat.fVmxRdrandExit            = 0;
+    EmuFeat.fVmxInvpcid               = 1;
+    EmuFeat.fVmxVmFunc                = 0;
+    EmuFeat.fVmxVmcsShadowing         = 0;
+    EmuFeat.fVmxRdseedExit            = 0;
+    EmuFeat.fVmxPml                   = 0;
+    EmuFeat.fVmxEptXcptVe             = 0;
+    EmuFeat.fVmxXsavesXrstors         = 0;
+    EmuFeat.fVmxUseTscScaling         = 0;
+    EmuFeat.fVmxEntryLoadDebugCtls    = 1;
+    EmuFeat.fVmxIa32eModeGuest        = 1;
+    EmuFeat.fVmxEntryLoadEferMsr      = 1;
+    EmuFeat.fVmxEntryLoadPatMsr       = 0;
+    EmuFeat.fVmxExitSaveDebugCtls     = 1;
+    EmuFeat.fVmxHostAddrSpaceSize     = 1;
+    EmuFeat.fVmxExitAckExtInt         = 1;
+    EmuFeat.fVmxExitSavePatMsr        = 0;
+    EmuFeat.fVmxExitLoadPatMsr        = 0;
+    EmuFeat.fVmxExitSaveEferMsr       = 1;
+    EmuFeat.fVmxExitLoadEferMsr       = 1;
+    EmuFeat.fVmxSavePreemptTimer      = 0;  /* Cannot be enabled if VMX-preemption timer is disabled. */
+    EmuFeat.fVmxExitSaveEferLma       = 1;  /* Cannot be disabled if unrestricted guest is enabled. */
+    EmuFeat.fVmxIntelPt               = 0;
+    EmuFeat.fVmxVmwriteAll            = 0;  /** @todo NSTVMX: enable this when nested VMCS shadowing is enabled. */
+    EmuFeat.fVmxEntryInjectSoftInt    = 1;
+
+    /*
+     * Merge guest features.
+     *
+     * When hardware-assisted VMX may be used, any feature we emulate must also be supported
+     * by the hardware, hence we merge our emulated features with the host features below.
+     */
+    PCCPUMFEATURES pBaseFeat  = cpumR3IsHwAssistNstGstExecAllowed(pVM) ? &pVM->cpum.s.HostFeatures : &EmuFeat;
+    PCPUMFEATURES  pGuestFeat = &pVM->cpum.s.GuestFeatures;
+    Assert(pBaseFeat->fVmx);
+    pGuestFeat->fVmxInsOutInfo            = (pBaseFeat->fVmxInsOutInfo            & EmuFeat.fVmxInsOutInfo           );
+    pGuestFeat->fVmxExtIntExit            = (pBaseFeat->fVmxExtIntExit            & EmuFeat.fVmxExtIntExit           );
+    pGuestFeat->fVmxNmiExit               = (pBaseFeat->fVmxNmiExit               & EmuFeat.fVmxNmiExit              );
+    pGuestFeat->fVmxVirtNmi               = (pBaseFeat->fVmxVirtNmi               & EmuFeat.fVmxVirtNmi              );
+    pGuestFeat->fVmxPreemptTimer          = (pBaseFeat->fVmxPreemptTimer          & EmuFeat.fVmxPreemptTimer         );
+    pGuestFeat->fVmxPostedInt             = (pBaseFeat->fVmxPostedInt             & EmuFeat.fVmxPostedInt            );
+    pGuestFeat->fVmxIntWindowExit         = (pBaseFeat->fVmxIntWindowExit         & EmuFeat.fVmxIntWindowExit        );
+    pGuestFeat->fVmxTscOffsetting         = (pBaseFeat->fVmxTscOffsetting         & EmuFeat.fVmxTscOffsetting        );
+    pGuestFeat->fVmxHltExit               = (pBaseFeat->fVmxHltExit               & EmuFeat.fVmxHltExit              );
+    pGuestFeat->fVmxInvlpgExit            = (pBaseFeat->fVmxInvlpgExit            & EmuFeat.fVmxInvlpgExit           );
+    pGuestFeat->fVmxMwaitExit             = (pBaseFeat->fVmxMwaitExit             & EmuFeat.fVmxMwaitExit            );
+    pGuestFeat->fVmxRdpmcExit             = (pBaseFeat->fVmxRdpmcExit             & EmuFeat.fVmxRdpmcExit            );
+    pGuestFeat->fVmxRdtscExit             = (pBaseFeat->fVmxRdtscExit             & EmuFeat.fVmxRdtscExit            );
+    pGuestFeat->fVmxCr3LoadExit           = (pBaseFeat->fVmxCr3LoadExit           & EmuFeat.fVmxCr3LoadExit          );
+    pGuestFeat->fVmxCr3StoreExit          = (pBaseFeat->fVmxCr3StoreExit          & EmuFeat.fVmxCr3StoreExit         );
+    pGuestFeat->fVmxCr8LoadExit           = (pBaseFeat->fVmxCr8LoadExit           & EmuFeat.fVmxCr8LoadExit          );
+    pGuestFeat->fVmxCr8StoreExit          = (pBaseFeat->fVmxCr8StoreExit          & EmuFeat.fVmxCr8StoreExit         );
+    pGuestFeat->fVmxUseTprShadow          = (pBaseFeat->fVmxUseTprShadow          & EmuFeat.fVmxUseTprShadow         );
+    pGuestFeat->fVmxNmiWindowExit         = (pBaseFeat->fVmxNmiWindowExit         & EmuFeat.fVmxNmiWindowExit        );
+    pGuestFeat->fVmxMovDRxExit            = (pBaseFeat->fVmxMovDRxExit            & EmuFeat.fVmxMovDRxExit           );
+    pGuestFeat->fVmxUncondIoExit          = (pBaseFeat->fVmxUncondIoExit          & EmuFeat.fVmxUncondIoExit         );
+    pGuestFeat->fVmxUseIoBitmaps          = (pBaseFeat->fVmxUseIoBitmaps          & EmuFeat.fVmxUseIoBitmaps         );
+    pGuestFeat->fVmxMonitorTrapFlag       = (pBaseFeat->fVmxMonitorTrapFlag       & EmuFeat.fVmxMonitorTrapFlag      );
+    pGuestFeat->fVmxUseMsrBitmaps         = (pBaseFeat->fVmxUseMsrBitmaps         & EmuFeat.fVmxUseMsrBitmaps        );
+    pGuestFeat->fVmxMonitorExit           = (pBaseFeat->fVmxMonitorExit           & EmuFeat.fVmxMonitorExit          );
+    pGuestFeat->fVmxPauseExit             = (pBaseFeat->fVmxPauseExit             & EmuFeat.fVmxPauseExit            );
+    pGuestFeat->fVmxSecondaryExecCtls     = (pBaseFeat->fVmxSecondaryExecCtls     & EmuFeat.fVmxSecondaryExecCtls    );
+    pGuestFeat->fVmxVirtApicAccess        = (pBaseFeat->fVmxVirtApicAccess        & EmuFeat.fVmxVirtApicAccess       );
+    pGuestFeat->fVmxEpt                   = (pBaseFeat->fVmxEpt                   & EmuFeat.fVmxEpt                  );
+    pGuestFeat->fVmxDescTableExit         = (pBaseFeat->fVmxDescTableExit         & EmuFeat.fVmxDescTableExit        );
+    pGuestFeat->fVmxRdtscp                = (pBaseFeat->fVmxRdtscp                & EmuFeat.fVmxRdtscp               );
+    pGuestFeat->fVmxVirtX2ApicMode        = (pBaseFeat->fVmxVirtX2ApicMode        & EmuFeat.fVmxVirtX2ApicMode       );
+    pGuestFeat->fVmxVpid                  = (pBaseFeat->fVmxVpid                  & EmuFeat.fVmxVpid                 );
+    pGuestFeat->fVmxWbinvdExit            = (pBaseFeat->fVmxWbinvdExit            & EmuFeat.fVmxWbinvdExit           );
+    pGuestFeat->fVmxUnrestrictedGuest     = (pBaseFeat->fVmxUnrestrictedGuest     & EmuFeat.fVmxUnrestrictedGuest    );
+    pGuestFeat->fVmxApicRegVirt           = (pBaseFeat->fVmxApicRegVirt           & EmuFeat.fVmxApicRegVirt          );
+    pGuestFeat->fVmxVirtIntDelivery       = (pBaseFeat->fVmxVirtIntDelivery       & EmuFeat.fVmxVirtIntDelivery      );
+    pGuestFeat->fVmxPauseLoopExit         = (pBaseFeat->fVmxPauseLoopExit         & EmuFeat.fVmxPauseLoopExit        );
+    pGuestFeat->fVmxRdrandExit            = (pBaseFeat->fVmxRdrandExit            & EmuFeat.fVmxRdrandExit           );
+    pGuestFeat->fVmxInvpcid               = (pBaseFeat->fVmxInvpcid               & EmuFeat.fVmxInvpcid              );
+    pGuestFeat->fVmxVmFunc                = (pBaseFeat->fVmxVmFunc                & EmuFeat.fVmxVmFunc               );
+    pGuestFeat->fVmxVmcsShadowing         = (pBaseFeat->fVmxVmcsShadowing         & EmuFeat.fVmxVmcsShadowing        );
+    pGuestFeat->fVmxRdseedExit            = (pBaseFeat->fVmxRdseedExit            & EmuFeat.fVmxRdseedExit           );
+    pGuestFeat->fVmxPml                   = (pBaseFeat->fVmxPml                   & EmuFeat.fVmxPml                  );
+    pGuestFeat->fVmxEptXcptVe             = (pBaseFeat->fVmxEptXcptVe             & EmuFeat.fVmxEptXcptVe            );
+    pGuestFeat->fVmxXsavesXrstors         = (pBaseFeat->fVmxXsavesXrstors         & EmuFeat.fVmxXsavesXrstors        );
+    pGuestFeat->fVmxUseTscScaling         = (pBaseFeat->fVmxUseTscScaling         & EmuFeat.fVmxUseTscScaling        );
+    pGuestFeat->fVmxEntryLoadDebugCtls    = (pBaseFeat->fVmxEntryLoadDebugCtls    & EmuFeat.fVmxEntryLoadDebugCtls   );
+    pGuestFeat->fVmxIa32eModeGuest        = (pBaseFeat->fVmxIa32eModeGuest        & EmuFeat.fVmxIa32eModeGuest       );
+    pGuestFeat->fVmxEntryLoadEferMsr      = (pBaseFeat->fVmxEntryLoadEferMsr      & EmuFeat.fVmxEntryLoadEferMsr     );
+    pGuestFeat->fVmxEntryLoadPatMsr       = (pBaseFeat->fVmxEntryLoadPatMsr       & EmuFeat.fVmxEntryLoadPatMsr      );
+    pGuestFeat->fVmxExitSaveDebugCtls     = (pBaseFeat->fVmxExitSaveDebugCtls     & EmuFeat.fVmxExitSaveDebugCtls    );
+    pGuestFeat->fVmxHostAddrSpaceSize     = (pBaseFeat->fVmxHostAddrSpaceSize     & EmuFeat.fVmxHostAddrSpaceSize    );
+    pGuestFeat->fVmxExitAckExtInt         = (pBaseFeat->fVmxExitAckExtInt         & EmuFeat.fVmxExitAckExtInt        );
+    pGuestFeat->fVmxExitSavePatMsr        = (pBaseFeat->fVmxExitSavePatMsr        & EmuFeat.fVmxExitSavePatMsr       );
+    pGuestFeat->fVmxExitLoadPatMsr        = (pBaseFeat->fVmxExitLoadPatMsr        & EmuFeat.fVmxExitLoadPatMsr       );
+    pGuestFeat->fVmxExitSaveEferMsr       = (pBaseFeat->fVmxExitSaveEferMsr       & EmuFeat.fVmxExitSaveEferMsr      );
+    pGuestFeat->fVmxExitLoadEferMsr       = (pBaseFeat->fVmxExitLoadEferMsr       & EmuFeat.fVmxExitLoadEferMsr      );
+    pGuestFeat->fVmxSavePreemptTimer      = (pBaseFeat->fVmxSavePreemptTimer      & EmuFeat.fVmxSavePreemptTimer     );
+    pGuestFeat->fVmxExitSaveEferLma       = (pBaseFeat->fVmxExitSaveEferLma       & EmuFeat.fVmxExitSaveEferLma      );
+    pGuestFeat->fVmxIntelPt               = (pBaseFeat->fVmxIntelPt               & EmuFeat.fVmxIntelPt              );
+    pGuestFeat->fVmxVmwriteAll            = (pBaseFeat->fVmxVmwriteAll            & EmuFeat.fVmxVmwriteAll           );
+    pGuestFeat->fVmxEntryInjectSoftInt    = (pBaseFeat->fVmxEntryInjectSoftInt    & EmuFeat.fVmxEntryInjectSoftInt   );
+
+    if (   !pVM->cpum.s.fNestedVmxPreemptTimer
+        || HMIsSubjectToVmxPreemptTimerErratum())
+    {
+        LogRel(("CPUM: Warning! VMX-preemption timer not exposed to guest due to forced CFGM setting or CPU erratum.\n"));
+        pGuestFeat->fVmxPreemptTimer     = 0;
+        pGuestFeat->fVmxSavePreemptTimer = 0;
+    }
+
+    /* Paranoia. */
+    if (!pGuestFeat->fVmxSecondaryExecCtls)
+    {
+        Assert(!pGuestFeat->fVmxVirtApicAccess);
+        Assert(!pGuestFeat->fVmxEpt);
+        Assert(!pGuestFeat->fVmxDescTableExit);
+        Assert(!pGuestFeat->fVmxRdtscp);
+        Assert(!pGuestFeat->fVmxVirtX2ApicMode);
+        Assert(!pGuestFeat->fVmxVpid);
+        Assert(!pGuestFeat->fVmxWbinvdExit);
+        Assert(!pGuestFeat->fVmxUnrestrictedGuest);
+        Assert(!pGuestFeat->fVmxApicRegVirt);
+        Assert(!pGuestFeat->fVmxVirtIntDelivery);
+        Assert(!pGuestFeat->fVmxPauseLoopExit);
+        Assert(!pGuestFeat->fVmxRdrandExit);
+        Assert(!pGuestFeat->fVmxInvpcid);
+        Assert(!pGuestFeat->fVmxVmFunc);
+        Assert(!pGuestFeat->fVmxVmcsShadowing);
+        Assert(!pGuestFeat->fVmxRdseedExit);
+        Assert(!pGuestFeat->fVmxPml);
+        Assert(!pGuestFeat->fVmxEptXcptVe);
+        Assert(!pGuestFeat->fVmxXsavesXrstors);
+        Assert(!pGuestFeat->fVmxUseTscScaling);
+    }
+    if (pGuestFeat->fVmxUnrestrictedGuest)
+    {
+        /* See footnote in Intel spec. 27.2 "Recording VM-Exit Information And Updating VM-entry Control Fields". */
+        Assert(pGuestFeat->fVmxExitSaveEferLma);
+    }
+
+    /*
+     * Finally initialize the VMX guest MSRs.
+     */
+    cpumR3InitVmxGuestMsrs(pVM, pHostVmxMsrs, pGuestFeat, pGuestVmxMsrs);
+}
+
+
+/**
+ * Gets the host hardware-virtualization MSRs.
+ *
+ * @returns VBox status code.
+ * @param   pMsrs       Where to store the MSRs.
+ */
+static int cpumR3GetHostHwvirtMsrs(PCPUMMSRS pMsrs)
+{
+    Assert(pMsrs);
+
+    uint32_t fCaps = 0;
+    int rc = SUPR3QueryVTCaps(&fCaps);
+    if (RT_SUCCESS(rc))
+    {
+        if (fCaps & (SUPVTCAPS_VT_X | SUPVTCAPS_AMD_V))
+        {
+            SUPHWVIRTMSRS HwvirtMsrs;
+            rc = SUPR3GetHwvirtMsrs(&HwvirtMsrs, false /* fForceRequery */);
+            if (RT_SUCCESS(rc))
+            {
+                if (fCaps & SUPVTCAPS_VT_X)
+                    HMGetVmxMsrsFromHwvirtMsrs(&HwvirtMsrs, &pMsrs->hwvirt.vmx);
+                else
+                    HMGetSvmMsrsFromHwvirtMsrs(&HwvirtMsrs, &pMsrs->hwvirt.svm);
+                return VINF_SUCCESS;
+            }
+
+            LogRel(("CPUM: Querying hardware-virtualization MSRs failed. rc=%Rrc\n", rc));
+            return rc;
+        }
+        else
+        {
+            LogRel(("CPUM: Querying hardware-virtualization capability succeeded but did not find VT-x or AMD-V\n"));
+            return VERR_INTERNAL_ERROR_5;
+        }
+    }
+    else
+        LogRel(("CPUM: No hardware-virtualization capability detected\n"));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Callback that fires when the nested VMX-preemption timer expired.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   pTimer  Pointer to timer.
+ * @param   pvUser  Opaque pointer to the virtual-CPU.
+ */
+static DECLCALLBACK(void) cpumR3VmxPreemptTimerCallback(PVM pVM, PTMTIMER pTimer, void *pvUser)
+{
+    RT_NOREF2(pVM, pTimer);
+    Assert(pvUser);
+
+    PVMCPU pVCpu = (PVMCPUR3)pvUser;
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER);
+}
+
+
+/**
+ * Initializes the CPUM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(int) CPUMR3Init(PVM pVM)
+{
+    LogFlow(("CPUMR3Init\n"));
+
+    /*
+     * Assert alignment, sizes and tables.
+     */
+    AssertCompileMemberAlignment(VM, cpum.s, 32);
+    AssertCompile(sizeof(pVM->cpum.s) <= sizeof(pVM->cpum.padding));
+    AssertCompileSizeAlignment(CPUMCTX, 64);
+    AssertCompileSizeAlignment(CPUMCTXMSRS, 64);
+    AssertCompileSizeAlignment(CPUMHOSTCTX, 64);
+    AssertCompileMemberAlignment(VM, cpum, 64);
+    AssertCompileMemberAlignment(VMCPU, cpum.s, 64);
+#ifdef VBOX_STRICT
+    int rc2 = cpumR3MsrStrictInitChecks();
+    AssertRCReturn(rc2, rc2);
+#endif
+
+    /*
+     * Gather info about the host CPU.
+     */
+    if (!ASMHasCpuId())
+    {
+        LogRel(("The CPU doesn't support CPUID!\n"));
+        return VERR_UNSUPPORTED_CPU;
+    }
+
+    pVM->cpum.s.fHostMxCsrMask = CPUMR3DeterminHostMxCsrMask();
+
+    CPUMMSRS HostMsrs;
+    RT_ZERO(HostMsrs);
+    int rc = cpumR3GetHostHwvirtMsrs(&HostMsrs);
+    AssertLogRelRCReturn(rc, rc);
+
+    PCPUMCPUIDLEAF  paLeaves;
+    uint32_t        cLeaves;
+    rc = CPUMR3CpuIdCollectLeaves(&paLeaves, &cLeaves);
+    AssertLogRelRCReturn(rc, rc);
+
+    rc = cpumR3CpuIdExplodeFeatures(paLeaves, cLeaves, &HostMsrs, &pVM->cpum.s.HostFeatures);
+    RTMemFree(paLeaves);
+    AssertLogRelRCReturn(rc, rc);
+    pVM->cpum.s.GuestFeatures.enmCpuVendor = pVM->cpum.s.HostFeatures.enmCpuVendor;
+
+    /*
+     * Check that the CPU supports the minimum features we require.
+     */
+    if (!pVM->cpum.s.HostFeatures.fFxSaveRstor)
+        return VMSetError(pVM, VERR_UNSUPPORTED_CPU, RT_SRC_POS, "Host CPU does not support the FXSAVE/FXRSTOR instruction.");
+    if (!pVM->cpum.s.HostFeatures.fMmx)
+        return VMSetError(pVM, VERR_UNSUPPORTED_CPU, RT_SRC_POS, "Host CPU does not support MMX.");
+    if (!pVM->cpum.s.HostFeatures.fTsc)
+        return VMSetError(pVM, VERR_UNSUPPORTED_CPU, RT_SRC_POS, "Host CPU does not support RDTSC.");
+
+    /*
+     * Setup the CR4 AND and OR masks used in the raw-mode switcher.
+     */
+    pVM->cpum.s.CR4.AndMask = X86_CR4_OSXMMEEXCPT | X86_CR4_PVI | X86_CR4_VME;
+    pVM->cpum.s.CR4.OrMask  = X86_CR4_OSFXSR;
+
+    /*
+     * Figure out which XSAVE/XRSTOR features are available on the host.
+     */
+    uint64_t fXcr0Host = 0;
+    uint64_t fXStateHostMask = 0;
+    if (   pVM->cpum.s.HostFeatures.fXSaveRstor
+        && pVM->cpum.s.HostFeatures.fOpSysXSaveRstor)
+    {
+        fXStateHostMask  = fXcr0Host = ASMGetXcr0();
+        fXStateHostMask &= XSAVE_C_X87 | XSAVE_C_SSE | XSAVE_C_YMM | XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI;
+        AssertLogRelMsgStmt((fXStateHostMask & (XSAVE_C_X87 | XSAVE_C_SSE)) == (XSAVE_C_X87 | XSAVE_C_SSE),
+                            ("%#llx\n", fXStateHostMask), fXStateHostMask = 0);
+    }
+    pVM->cpum.s.fXStateHostMask = fXStateHostMask;
+    LogRel(("CPUM: fXStateHostMask=%#llx; initial: %#llx; host XCR0=%#llx\n",
+            pVM->cpum.s.fXStateHostMask, fXStateHostMask, fXcr0Host));
+
+    /*
+     * Allocate memory for the extended CPU state and initialize the host XSAVE/XRSTOR mask.
+     */
+    uint32_t cbMaxXState = pVM->cpum.s.HostFeatures.cbMaxExtendedState;
+    cbMaxXState = RT_ALIGN(cbMaxXState, 128);
+    AssertLogRelReturn(cbMaxXState >= sizeof(X86FXSTATE) && cbMaxXState <= _8K, VERR_CPUM_IPE_2);
+
+    uint8_t *pbXStates;
+    rc = MMR3HyperAllocOnceNoRelEx(pVM, cbMaxXState * 2 * pVM->cCpus, PAGE_SIZE, MM_TAG_CPUM_CTX,
+                                   MMHYPER_AONR_FLAGS_KERNEL_MAPPING, (void **)&pbXStates);
+    AssertLogRelRCReturn(rc, rc);
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+
+        pVCpu->cpum.s.Guest.pXStateR3 = (PX86XSAVEAREA)pbXStates;
+        pVCpu->cpum.s.Guest.pXStateR0 = MMHyperR3ToR0(pVM, pbXStates);
+        pbXStates += cbMaxXState;
+
+        pVCpu->cpum.s.Host.pXStateR3  = (PX86XSAVEAREA)pbXStates;
+        pVCpu->cpum.s.Host.pXStateR0  = MMHyperR3ToR0(pVM, pbXStates);
+        pbXStates += cbMaxXState;
+
+        pVCpu->cpum.s.Host.fXStateMask = fXStateHostMask;
+    }
+
+    /*
+     * Register saved state data item.
+     */
+    rc = SSMR3RegisterInternal(pVM, "cpum", 1, CPUM_SAVED_STATE_VERSION, sizeof(CPUM),
+                               NULL, cpumR3LiveExec, NULL,
+                               NULL, cpumR3SaveExec, NULL,
+                               cpumR3LoadPrep, cpumR3LoadExec, cpumR3LoadDone);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Register info handlers and registers with the debugger facility.
+     */
+    DBGFR3InfoRegisterInternalEx(pVM, "cpum",             "Displays the all the cpu states.",
+                                 &cpumR3InfoAll, DBGFINFO_FLAGS_ALL_EMTS);
+    DBGFR3InfoRegisterInternalEx(pVM, "cpumguest",        "Displays the guest cpu state.",
+                                 &cpumR3InfoGuest, DBGFINFO_FLAGS_ALL_EMTS);
+    DBGFR3InfoRegisterInternalEx(pVM, "cpumguesthwvirt",   "Displays the guest hwvirt. cpu state.",
+                                 &cpumR3InfoGuestHwvirt, DBGFINFO_FLAGS_ALL_EMTS);
+    DBGFR3InfoRegisterInternalEx(pVM, "cpumhyper",        "Displays the hypervisor cpu state.",
+                                 &cpumR3InfoHyper, DBGFINFO_FLAGS_ALL_EMTS);
+    DBGFR3InfoRegisterInternalEx(pVM, "cpumhost",         "Displays the host cpu state.",
+                                 &cpumR3InfoHost, DBGFINFO_FLAGS_ALL_EMTS);
+    DBGFR3InfoRegisterInternalEx(pVM, "cpumguestinstr",   "Displays the current guest instruction.",
+                                 &cpumR3InfoGuestInstr, DBGFINFO_FLAGS_ALL_EMTS);
+    DBGFR3InfoRegisterInternal(  pVM, "cpuid",            "Displays the guest cpuid leaves.",         &cpumR3CpuIdInfo);
+    DBGFR3InfoRegisterInternal(  pVM, "cpumvmxfeat",      "Displays the host and guest VMX hwvirt. features.",
+                               &cpumR3InfoVmxFeatures);
+
+    rc = cpumR3DbgInit(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Check if we need to workaround partial/leaky FPU handling.
+     */
+    cpumR3CheckLeakyFpu(pVM);
+
+    /*
+     * Initialize the Guest CPUID and MSR states.
+     */
+    rc = cpumR3InitCpuIdAndMsrs(pVM, &HostMsrs);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Allocate memory required by the guest hardware-virtualization structures.
+     * This must be done after initializing CPUID/MSR features as we access the
+     * the VMX/SVM guest features below.
+     *
+     * In the case of nested VT-x, we also need to create the per-VCPU
+     * VMX preemption timers.
+     */
+    if (pVM->cpum.s.GuestFeatures.fVmx)
+        rc = cpumR3AllocVmxHwVirtState(pVM);
+    else if (pVM->cpum.s.GuestFeatures.fSvm)
+        rc = cpumR3AllocSvmHwVirtState(pVM);
+    else
+        Assert(pVM->apCpusR3[0]->cpum.s.Guest.hwvirt.enmHwvirt == CPUMHWVIRT_NONE);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    CPUMR3Reset(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * The CPUM will update the addresses used by the switcher.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(void) CPUMR3Relocate(PVM pVM)
+{
+    RT_NOREF(pVM);
+}
+
+
+/**
+ * Terminates the CPUM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(int) CPUMR3Term(PVM pVM)
+{
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        memset(pVCpu->cpum.s.aMagic, 0, sizeof(pVCpu->cpum.s.aMagic));
+        pVCpu->cpum.s.uMagic      = 0;
+        pvCpu->cpum.s.Guest.dr[5] = 0;
+    }
+#endif
+
+    if (pVM->cpum.s.GuestFeatures.fVmx)
+    {
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+            int rc = TMR3TimerDestroy(pVCpu->cpum.s.pNestedVmxPreemptTimerR3); AssertRC(rc);
+            pVCpu->cpum.s.pNestedVmxPreemptTimerR0 = NIL_RTR0PTR;
+        }
+
+        cpumR3FreeVmxHwVirtState(pVM);
+    }
+    else if (pVM->cpum.s.GuestFeatures.fSvm)
+        cpumR3FreeSvmHwVirtState(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resets a virtual CPU.
+ *
+ * Used by CPUMR3Reset and CPU hot plugging.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure of the CPU that is
+ *                  being reset.  This may differ from the current EMT.
+ */
+VMMR3DECL(void) CPUMR3ResetCpu(PVM pVM, PVMCPU pVCpu)
+{
+    /** @todo anything different for VCPU > 0? */
+    PCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+
+    /*
+     * Initialize everything to ZERO first.
+     */
+    uint32_t fUseFlags              =  pVCpu->cpum.s.fUseFlags & ~CPUM_USED_FPU_SINCE_REM;
+
+    AssertCompile(RTASSERT_OFFSET_OF(CPUMCTX, pXStateR0) < RTASSERT_OFFSET_OF(CPUMCTX, pXStateR3));
+    memset(pCtx, 0, RT_UOFFSETOF(CPUMCTX, pXStateR0));
+
+    pVCpu->cpum.s.fUseFlags         = fUseFlags;
+
+    pCtx->cr0                       = X86_CR0_CD | X86_CR0_NW | X86_CR0_ET;  //0x60000010
+    pCtx->eip                       = 0x0000fff0;
+    pCtx->edx                       = 0x00000600;   /* P6 processor */
+    pCtx->eflags.Bits.u1Reserved0   = 1;
+
+    pCtx->cs.Sel                    = 0xf000;
+    pCtx->cs.ValidSel               = 0xf000;
+    pCtx->cs.fFlags                 = CPUMSELREG_FLAGS_VALID;
+    pCtx->cs.u64Base                = UINT64_C(0xffff0000);
+    pCtx->cs.u32Limit               = 0x0000ffff;
+    pCtx->cs.Attr.n.u1DescType      = 1; /* code/data segment */
+    pCtx->cs.Attr.n.u1Present       = 1;
+    pCtx->cs.Attr.n.u4Type          = X86_SEL_TYPE_ER_ACC;
+
+    pCtx->ds.fFlags                 = CPUMSELREG_FLAGS_VALID;
+    pCtx->ds.u32Limit               = 0x0000ffff;
+    pCtx->ds.Attr.n.u1DescType      = 1; /* code/data segment */
+    pCtx->ds.Attr.n.u1Present       = 1;
+    pCtx->ds.Attr.n.u4Type          = X86_SEL_TYPE_RW_ACC;
+
+    pCtx->es.fFlags                 = CPUMSELREG_FLAGS_VALID;
+    pCtx->es.u32Limit               = 0x0000ffff;
+    pCtx->es.Attr.n.u1DescType      = 1; /* code/data segment */
+    pCtx->es.Attr.n.u1Present       = 1;
+    pCtx->es.Attr.n.u4Type          = X86_SEL_TYPE_RW_ACC;
+
+    pCtx->fs.fFlags                 = CPUMSELREG_FLAGS_VALID;
+    pCtx->fs.u32Limit               = 0x0000ffff;
+    pCtx->fs.Attr.n.u1DescType      = 1; /* code/data segment */
+    pCtx->fs.Attr.n.u1Present       = 1;
+    pCtx->fs.Attr.n.u4Type          = X86_SEL_TYPE_RW_ACC;
+
+    pCtx->gs.fFlags                 = CPUMSELREG_FLAGS_VALID;
+    pCtx->gs.u32Limit               = 0x0000ffff;
+    pCtx->gs.Attr.n.u1DescType      = 1; /* code/data segment */
+    pCtx->gs.Attr.n.u1Present       = 1;
+    pCtx->gs.Attr.n.u4Type          = X86_SEL_TYPE_RW_ACC;
+
+    pCtx->ss.fFlags                 = CPUMSELREG_FLAGS_VALID;
+    pCtx->ss.u32Limit               = 0x0000ffff;
+    pCtx->ss.Attr.n.u1Present       = 1;
+    pCtx->ss.Attr.n.u1DescType      = 1; /* code/data segment */
+    pCtx->ss.Attr.n.u4Type          = X86_SEL_TYPE_RW_ACC;
+
+    pCtx->idtr.cbIdt                = 0xffff;
+    pCtx->gdtr.cbGdt                = 0xffff;
+
+    pCtx->ldtr.fFlags               = CPUMSELREG_FLAGS_VALID;
+    pCtx->ldtr.u32Limit             = 0xffff;
+    pCtx->ldtr.Attr.n.u1Present     = 1;
+    pCtx->ldtr.Attr.n.u4Type        = X86_SEL_TYPE_SYS_LDT;
+
+    pCtx->tr.fFlags                 = CPUMSELREG_FLAGS_VALID;
+    pCtx->tr.u32Limit               = 0xffff;
+    pCtx->tr.Attr.n.u1Present       = 1;
+    pCtx->tr.Attr.n.u4Type          = X86_SEL_TYPE_SYS_386_TSS_BUSY;    /* Deduction, not properly documented by Intel. */
+
+    pCtx->dr[6]                     = X86_DR6_INIT_VAL;
+    pCtx->dr[7]                     = X86_DR7_INIT_VAL;
+
+    PX86FXSTATE pFpuCtx = &pCtx->pXStateR3->x87; AssertReleaseMsg(RT_VALID_PTR(pFpuCtx), ("%p\n", pFpuCtx));
+    pFpuCtx->FTW                    = 0x00;         /* All empty (abbridged tag reg edition). */
+    pFpuCtx->FCW                    = 0x37f;
+
+    /* Intel 64 and IA-32 Architectures Software Developer's Manual Volume 3A, Table 8-1.
+       IA-32 Processor States Following Power-up, Reset, or INIT */
+    pFpuCtx->MXCSR                  = 0x1F80;
+    pFpuCtx->MXCSR_MASK             = pVM->cpum.s.GuestInfo.fMxCsrMask; /** @todo check if REM messes this up... */
+
+    pCtx->aXcr[0]                   = XSAVE_C_X87;
+    if (pVM->cpum.s.HostFeatures.cbMaxExtendedState >= RT_UOFFSETOF(X86XSAVEAREA, Hdr))
+    {
+        /* The entire FXSAVE state needs loading when we switch to XSAVE/XRSTOR
+           as we don't know what happened before.  (Bother optimize later?) */
+        pCtx->pXStateR3->Hdr.bmXState = XSAVE_C_X87 | XSAVE_C_SSE;
+    }
+
+    /*
+     * MSRs.
+     */
+    /* Init PAT MSR */
+    pCtx->msrPAT                    = MSR_IA32_CR_PAT_INIT_VAL;
+
+    /* EFER MBZ; see AMD64 Architecture Programmer's Manual Volume 2: Table 14-1. Initial Processor State.
+     * The Intel docs don't mention it. */
+    Assert(!pCtx->msrEFER);
+
+    /* IA32_MISC_ENABLE - not entirely sure what the init/reset state really
+       is supposed to be here, just trying provide useful/sensible values. */
+    PCPUMMSRRANGE pRange = cpumLookupMsrRange(pVM, MSR_IA32_MISC_ENABLE);
+    if (pRange)
+    {
+        pVCpu->cpum.s.GuestMsrs.msr.MiscEnable = MSR_IA32_MISC_ENABLE_BTS_UNAVAIL
+                                               | MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL
+                                               | (pVM->cpum.s.GuestFeatures.fMonitorMWait ? MSR_IA32_MISC_ENABLE_MONITOR : 0)
+                                               | MSR_IA32_MISC_ENABLE_FAST_STRINGS;
+        pRange->fWrIgnMask |= MSR_IA32_MISC_ENABLE_BTS_UNAVAIL
+                            | MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL;
+        pRange->fWrGpMask  &= ~pVCpu->cpum.s.GuestMsrs.msr.MiscEnable;
+    }
+
+    /** @todo Wire IA32_MISC_ENABLE bit 22 to our NT 4 CPUID trick. */
+
+    /** @todo r=ramshankar: Currently broken for SMP as TMCpuTickSet() expects to be
+     *        called from each EMT while we're getting called by CPUMR3Reset()
+     *        iteratively on the same thread. Fix later.  */
+#if 0 /** @todo r=bird: This we will do in TM, not here. */
+    /* TSC must be 0. Intel spec. Table 9-1. "IA-32 Processor States Following Power-up, Reset, or INIT." */
+    CPUMSetGuestMsr(pVCpu, MSR_IA32_TSC, 0);
+#endif
+
+
+    /* C-state control. Guesses. */
+    pVCpu->cpum.s.GuestMsrs.msr.PkgCStateCfgCtrl = 1 /*C1*/ | RT_BIT_32(25) | RT_BIT_32(26) | RT_BIT_32(27) | RT_BIT_32(28);
+    /* For Nehalem+ and Atoms, the 0xE2 MSR (MSR_PKG_CST_CONFIG_CONTROL) is documented. For Core 2,
+     * it's undocumented but exists as MSR_PMG_CST_CONFIG_CONTROL and has similar but not identical
+     * functionality. The default value must be different due to incompatible write mask.
+     */
+    if (CPUMMICROARCH_IS_INTEL_CORE2(pVM->cpum.s.GuestFeatures.enmMicroarch))
+        pVCpu->cpum.s.GuestMsrs.msr.PkgCStateCfgCtrl = 0x202a01;    /* From Mac Pro Harpertown, unlocked. */
+    else if (pVM->cpum.s.GuestFeatures.enmMicroarch == kCpumMicroarch_Intel_Core_Yonah)
+        pVCpu->cpum.s.GuestMsrs.msr.PkgCStateCfgCtrl = 0x26740c;    /* From MacBookPro1,1. */
+
+    /*
+     * Hardware virtualization state.
+     */
+    CPUMSetGuestGif(pCtx, true);
+    Assert(!pVM->cpum.s.GuestFeatures.fVmx || !pVM->cpum.s.GuestFeatures.fSvm);   /* Paranoia. */
+    if (pVM->cpum.s.GuestFeatures.fVmx)
+        cpumR3ResetVmxHwVirtState(pVCpu);
+    else if (pVM->cpum.s.GuestFeatures.fSvm)
+        cpumR3ResetSvmHwVirtState(pVCpu);
+}
+
+
+/**
+ * Resets the CPU.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(void) CPUMR3Reset(PVM pVM)
+{
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        CPUMR3ResetCpu(pVM, pVCpu);
+
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+
+        /* Magic marker for searching in crash dumps. */
+        strcpy((char *)pVCpu->.cpum.s.aMagic, "CPUMCPU Magic");
+        pVCpu->cpum.s.uMagic       = UINT64_C(0xDEADBEEFDEADBEEF);
+        pVCpu->cpum.s.Guest->dr[5] = UINT64_C(0xDEADBEEFDEADBEEF);
+#endif
+    }
+}
+
+
+
+
+/**
+ * Pass 0 live exec callback.
+ *
+ * @returns VINF_SSM_DONT_CALL_AGAIN.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ * @param   uPass               The pass (0).
+ */
+static DECLCALLBACK(int) cpumR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    AssertReturn(uPass == 0, VERR_SSM_UNEXPECTED_PASS);
+    cpumR3SaveCpuId(pVM, pSSM);
+    return VINF_SSM_DONT_CALL_AGAIN;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) cpumR3SaveExec(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Save.
+     */
+    SSMR3PutU32(pSSM, pVM->cCpus);
+    SSMR3PutU32(pSSM, sizeof(pVM->apCpusR3[0]->cpum.s.GuestMsrs.msr));
+    CPUMCTX DummyHyperCtx;
+    RT_ZERO(DummyHyperCtx);
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+
+        SSMR3PutStructEx(pSSM, &DummyHyperCtx,           sizeof(DummyHyperCtx),           0, g_aCpumCtxFields, NULL);
+
+        PCPUMCTX pGstCtx = &pVCpu->cpum.s.Guest;
+        SSMR3PutStructEx(pSSM, pGstCtx,                  sizeof(*pGstCtx),                0, g_aCpumCtxFields, NULL);
+        SSMR3PutStructEx(pSSM, &pGstCtx->pXStateR3->x87, sizeof(pGstCtx->pXStateR3->x87), 0, g_aCpumX87Fields, NULL);
+        if (pGstCtx->fXStateMask != 0)
+            SSMR3PutStructEx(pSSM, &pGstCtx->pXStateR3->Hdr, sizeof(pGstCtx->pXStateR3->Hdr), 0, g_aCpumXSaveHdrFields, NULL);
+        if (pGstCtx->fXStateMask & XSAVE_C_YMM)
+        {
+            PCX86XSAVEYMMHI pYmmHiCtx = CPUMCTX_XSAVE_C_PTR(pGstCtx, XSAVE_C_YMM_BIT, PCX86XSAVEYMMHI);
+            SSMR3PutStructEx(pSSM, pYmmHiCtx, sizeof(*pYmmHiCtx), SSMSTRUCT_FLAGS_FULL_STRUCT, g_aCpumYmmHiFields, NULL);
+        }
+        if (pGstCtx->fXStateMask & XSAVE_C_BNDREGS)
+        {
+            PCX86XSAVEBNDREGS pBndRegs = CPUMCTX_XSAVE_C_PTR(pGstCtx, XSAVE_C_BNDREGS_BIT, PCX86XSAVEBNDREGS);
+            SSMR3PutStructEx(pSSM, pBndRegs, sizeof(*pBndRegs), SSMSTRUCT_FLAGS_FULL_STRUCT, g_aCpumBndRegsFields, NULL);
+        }
+        if (pGstCtx->fXStateMask & XSAVE_C_BNDCSR)
+        {
+            PCX86XSAVEBNDCFG pBndCfg = CPUMCTX_XSAVE_C_PTR(pGstCtx, XSAVE_C_BNDCSR_BIT, PCX86XSAVEBNDCFG);
+            SSMR3PutStructEx(pSSM, pBndCfg, sizeof(*pBndCfg), SSMSTRUCT_FLAGS_FULL_STRUCT, g_aCpumBndCfgFields, NULL);
+        }
+        if (pGstCtx->fXStateMask & XSAVE_C_ZMM_HI256)
+        {
+            PCX86XSAVEZMMHI256 pZmmHi256 = CPUMCTX_XSAVE_C_PTR(pGstCtx, XSAVE_C_ZMM_HI256_BIT, PCX86XSAVEZMMHI256);
+            SSMR3PutStructEx(pSSM, pZmmHi256, sizeof(*pZmmHi256), SSMSTRUCT_FLAGS_FULL_STRUCT, g_aCpumZmmHi256Fields, NULL);
+        }
+        if (pGstCtx->fXStateMask & XSAVE_C_ZMM_16HI)
+        {
+            PCX86XSAVEZMM16HI pZmm16Hi = CPUMCTX_XSAVE_C_PTR(pGstCtx, XSAVE_C_ZMM_16HI_BIT, PCX86XSAVEZMM16HI);
+            SSMR3PutStructEx(pSSM, pZmm16Hi, sizeof(*pZmm16Hi), SSMSTRUCT_FLAGS_FULL_STRUCT, g_aCpumZmm16HiFields, NULL);
+        }
+        if (pVM->cpum.s.GuestFeatures.fSvm)
+        {
+            Assert(pGstCtx->hwvirt.svm.CTX_SUFF(pVmcb));
+            SSMR3PutU64(pSSM,    pGstCtx->hwvirt.svm.uMsrHSavePa);
+            SSMR3PutGCPhys(pSSM, pGstCtx->hwvirt.svm.GCPhysVmcb);
+            SSMR3PutU64(pSSM,    pGstCtx->hwvirt.svm.uPrevPauseTick);
+            SSMR3PutU16(pSSM,    pGstCtx->hwvirt.svm.cPauseFilter);
+            SSMR3PutU16(pSSM,    pGstCtx->hwvirt.svm.cPauseFilterThreshold);
+            SSMR3PutBool(pSSM,   pGstCtx->hwvirt.svm.fInterceptEvents);
+            SSMR3PutStructEx(pSSM, &pGstCtx->hwvirt.svm.HostState, sizeof(pGstCtx->hwvirt.svm.HostState), 0 /* fFlags */,
+                             g_aSvmHwvirtHostState, NULL /* pvUser */);
+            SSMR3PutMem(pSSM,    pGstCtx->hwvirt.svm.pVmcbR3,       SVM_VMCB_PAGES  << X86_PAGE_4K_SHIFT);
+            SSMR3PutMem(pSSM,    pGstCtx->hwvirt.svm.pvMsrBitmapR3, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
+            SSMR3PutMem(pSSM,    pGstCtx->hwvirt.svm.pvIoBitmapR3,  SVM_IOPM_PAGES  << X86_PAGE_4K_SHIFT);
+            SSMR3PutU32(pSSM,    pGstCtx->hwvirt.fLocalForcedActions);
+            SSMR3PutBool(pSSM,   pGstCtx->hwvirt.fGif);
+        }
+        if (pVM->cpum.s.GuestFeatures.fVmx)
+        {
+            Assert(pGstCtx->hwvirt.vmx.CTX_SUFF(pVmcs));
+            SSMR3PutGCPhys(pSSM,   pGstCtx->hwvirt.vmx.GCPhysVmxon);
+            SSMR3PutGCPhys(pSSM,   pGstCtx->hwvirt.vmx.GCPhysVmcs);
+            SSMR3PutGCPhys(pSSM,   pGstCtx->hwvirt.vmx.GCPhysShadowVmcs);
+            SSMR3PutBool(pSSM,     pGstCtx->hwvirt.vmx.fInVmxRootMode);
+            SSMR3PutBool(pSSM,     pGstCtx->hwvirt.vmx.fInVmxNonRootMode);
+            SSMR3PutBool(pSSM,     pGstCtx->hwvirt.vmx.fInterceptEvents);
+            SSMR3PutBool(pSSM,     pGstCtx->hwvirt.vmx.fNmiUnblockingIret);
+            SSMR3PutStructEx(pSSM, pGstCtx->hwvirt.vmx.pVmcsR3, sizeof(VMXVVMCS), 0, g_aVmxHwvirtVmcs, NULL);
+            SSMR3PutStructEx(pSSM, pGstCtx->hwvirt.vmx.pShadowVmcsR3, sizeof(VMXVVMCS), 0, g_aVmxHwvirtVmcs, NULL);
+            SSMR3PutMem(pSSM,      pGstCtx->hwvirt.vmx.pvVmreadBitmapR3, VMX_V_VMREAD_VMWRITE_BITMAP_SIZE);
+            SSMR3PutMem(pSSM,      pGstCtx->hwvirt.vmx.pvVmwriteBitmapR3, VMX_V_VMREAD_VMWRITE_BITMAP_SIZE);
+            SSMR3PutMem(pSSM,      pGstCtx->hwvirt.vmx.pEntryMsrLoadAreaR3, VMX_V_AUTOMSR_AREA_SIZE);
+            SSMR3PutMem(pSSM,      pGstCtx->hwvirt.vmx.pExitMsrStoreAreaR3, VMX_V_AUTOMSR_AREA_SIZE);
+            SSMR3PutMem(pSSM,      pGstCtx->hwvirt.vmx.pExitMsrLoadAreaR3,  VMX_V_AUTOMSR_AREA_SIZE);
+            SSMR3PutMem(pSSM,      pGstCtx->hwvirt.vmx.pvMsrBitmapR3, VMX_V_MSR_BITMAP_SIZE);
+            SSMR3PutMem(pSSM,      pGstCtx->hwvirt.vmx.pvIoBitmapR3, VMX_V_IO_BITMAP_A_SIZE + VMX_V_IO_BITMAP_B_SIZE);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.uFirstPauseLoopTick);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.uPrevPauseTick);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.uEntryTick);
+            SSMR3PutU16(pSSM,      pGstCtx->hwvirt.vmx.offVirtApicWrite);
+            SSMR3PutBool(pSSM,     pGstCtx->hwvirt.vmx.fVirtNmiBlocking);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.u64FeatCtrl);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.u64Basic);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.PinCtls.u);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.ProcCtls.u);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.ProcCtls2.u);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.ExitCtls.u);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.EntryCtls.u);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.TruePinCtls.u);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.TrueProcCtls.u);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.TrueEntryCtls.u);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.TrueExitCtls.u);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.u64Misc);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.u64VmcsEnum);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.u64VmFunc);
+            SSMR3PutU64(pSSM,      pGstCtx->hwvirt.vmx.Msrs.u64EptVpidCaps);
+        }
+        SSMR3PutU32(pSSM, pVCpu->cpum.s.fUseFlags);
+        SSMR3PutU32(pSSM, pVCpu->cpum.s.fChanged);
+        AssertCompileSizeAlignment(pVCpu->cpum.s.GuestMsrs.msr, sizeof(uint64_t));
+        SSMR3PutMem(pSSM, &pVCpu->cpum.s.GuestMsrs, sizeof(pVCpu->cpum.s.GuestMsrs.msr));
+    }
+
+    cpumR3SaveCpuId(pVM, pSSM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTLOADPREP}
+ */
+static DECLCALLBACK(int) cpumR3LoadPrep(PVM pVM, PSSMHANDLE pSSM)
+{
+    NOREF(pSSM);
+    pVM->cpum.s.fPendingRestore = true;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTLOADEXEC}
+ */
+static DECLCALLBACK(int) cpumR3LoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    int rc; /* Only for AssertRCReturn use. */
+
+    /*
+     * Validate version.
+     */
+    if (    uVersion != CPUM_SAVED_STATE_VERSION_HWVIRT_VMX_IEM
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_HWVIRT_SVM
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_XSAVE
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_GOOD_CPUID_COUNT
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_BAD_CPUID_COUNT
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_PUT_STRUCT
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_MEM
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_NO_MSR_SIZE
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER3_2
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER3_0
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER2_1_NOMSR
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER2_0
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER1_6)
+    {
+        AssertMsgFailed(("cpumR3LoadExec: Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    if (uPass == SSM_PASS_FINAL)
+    {
+        /*
+         * Set the size of RTGCPTR for SSMR3GetGCPtr. (Only necessary for
+         * really old SSM file versions.)
+         */
+        if (uVersion == CPUM_SAVED_STATE_VERSION_VER1_6)
+            SSMR3HandleSetGCPtrSize(pSSM, sizeof(RTGCPTR32));
+        else if (uVersion <= CPUM_SAVED_STATE_VERSION_VER3_0)
+            SSMR3HandleSetGCPtrSize(pSSM, sizeof(RTGCPTR));
+
+        /*
+         * Figure x86 and ctx field definitions to use for older states.
+         */
+        uint32_t const  fLoad = uVersion > CPUM_SAVED_STATE_VERSION_MEM ? 0 : SSMSTRUCT_FLAGS_MEM_BAND_AID_RELAXED;
+        PCSSMFIELD      paCpumCtx1Fields = g_aCpumX87Fields;
+        PCSSMFIELD      paCpumCtx2Fields = g_aCpumCtxFields;
+        if (uVersion == CPUM_SAVED_STATE_VERSION_VER1_6)
+        {
+            paCpumCtx1Fields = g_aCpumX87FieldsV16;
+            paCpumCtx2Fields = g_aCpumCtxFieldsV16;
+        }
+        else if (uVersion <= CPUM_SAVED_STATE_VERSION_MEM)
+        {
+            paCpumCtx1Fields = g_aCpumX87FieldsMem;
+            paCpumCtx2Fields = g_aCpumCtxFieldsMem;
+        }
+
+        /*
+         * The hyper state used to preceed the CPU count.  Starting with
+         * XSAVE it was moved down till after we've got the count.
+         */
+        CPUMCTX HyperCtxIgnored;
+        if (uVersion < CPUM_SAVED_STATE_VERSION_XSAVE)
+        {
+            for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+            {
+                X86FXSTATE Ign;
+                SSMR3GetStructEx(pSSM, &Ign, sizeof(Ign), fLoad | SSMSTRUCT_FLAGS_NO_TAIL_MARKER, paCpumCtx1Fields, NULL);
+                SSMR3GetStructEx(pSSM, &HyperCtxIgnored, sizeof(HyperCtxIgnored),
+                                 fLoad | SSMSTRUCT_FLAGS_NO_LEAD_MARKER, paCpumCtx2Fields, NULL);
+            }
+        }
+
+        if (uVersion >= CPUM_SAVED_STATE_VERSION_VER2_1_NOMSR)
+        {
+            uint32_t cCpus;
+            rc = SSMR3GetU32(pSSM, &cCpus); AssertRCReturn(rc, rc);
+            AssertLogRelMsgReturn(cCpus == pVM->cCpus, ("Mismatching CPU counts: saved: %u; configured: %u \n", cCpus, pVM->cCpus),
+                                  VERR_SSM_UNEXPECTED_DATA);
+        }
+        AssertLogRelMsgReturn(   uVersion > CPUM_SAVED_STATE_VERSION_VER2_0
+                              || pVM->cCpus == 1,
+                              ("cCpus=%u\n", pVM->cCpus),
+                              VERR_SSM_UNEXPECTED_DATA);
+
+        uint32_t cbMsrs = 0;
+        if (uVersion > CPUM_SAVED_STATE_VERSION_NO_MSR_SIZE)
+        {
+            rc = SSMR3GetU32(pSSM, &cbMsrs); AssertRCReturn(rc, rc);
+            AssertLogRelMsgReturn(RT_ALIGN(cbMsrs, sizeof(uint64_t)) == cbMsrs, ("Size of MSRs is misaligned: %#x\n", cbMsrs),
+                                  VERR_SSM_UNEXPECTED_DATA);
+            AssertLogRelMsgReturn(cbMsrs <= sizeof(CPUMCTXMSRS) && cbMsrs > 0,  ("Size of MSRs is out of range: %#x\n", cbMsrs),
+                                  VERR_SSM_UNEXPECTED_DATA);
+        }
+
+        /*
+         * Do the per-CPU restoring.
+         */
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PVMCPU   pVCpu   = pVM->apCpusR3[idCpu];
+            PCPUMCTX pGstCtx = &pVCpu->cpum.s.Guest;
+
+            if (uVersion >= CPUM_SAVED_STATE_VERSION_XSAVE)
+            {
+                /*
+                 * The XSAVE saved state layout moved the hyper state down here.
+                 */
+                rc = SSMR3GetStructEx(pSSM, &HyperCtxIgnored,         sizeof(HyperCtxIgnored),         0, g_aCpumCtxFields, NULL);
+                AssertRCReturn(rc, rc);
+
+                /*
+                 * Start by restoring the CPUMCTX structure and the X86FXSAVE bits of the extended state.
+                 */
+                rc = SSMR3GetStructEx(pSSM, pGstCtx,                  sizeof(*pGstCtx),                0, g_aCpumCtxFields, NULL);
+                rc = SSMR3GetStructEx(pSSM, &pGstCtx->pXStateR3->x87, sizeof(pGstCtx->pXStateR3->x87), 0, g_aCpumX87Fields, NULL);
+                AssertRCReturn(rc, rc);
+
+                /* Check that the xsave/xrstor mask is valid (invalid results in #GP). */
+                if (pGstCtx->fXStateMask != 0)
+                {
+                    AssertLogRelMsgReturn(!(pGstCtx->fXStateMask & ~pVM->cpum.s.fXStateGuestMask),
+                                          ("fXStateMask=%#RX64 fXStateGuestMask=%#RX64\n",
+                                           pGstCtx->fXStateMask, pVM->cpum.s.fXStateGuestMask),
+                                          VERR_CPUM_INCOMPATIBLE_XSAVE_COMP_MASK);
+                    AssertLogRelMsgReturn(pGstCtx->fXStateMask & XSAVE_C_X87,
+                                          ("fXStateMask=%#RX64\n", pGstCtx->fXStateMask), VERR_CPUM_INVALID_XSAVE_COMP_MASK);
+                    AssertLogRelMsgReturn((pGstCtx->fXStateMask & (XSAVE_C_SSE | XSAVE_C_YMM)) != XSAVE_C_YMM,
+                                          ("fXStateMask=%#RX64\n", pGstCtx->fXStateMask), VERR_CPUM_INVALID_XSAVE_COMP_MASK);
+                    AssertLogRelMsgReturn(   (pGstCtx->fXStateMask & (XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI)) == 0
+                                          ||    (pGstCtx->fXStateMask & (XSAVE_C_SSE | XSAVE_C_YMM | XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI))
+                                             ==                         (XSAVE_C_SSE | XSAVE_C_YMM | XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI),
+                                          ("fXStateMask=%#RX64\n", pGstCtx->fXStateMask), VERR_CPUM_INVALID_XSAVE_COMP_MASK);
+                }
+
+                /* Check that the XCR0 mask is valid (invalid results in #GP). */
+                AssertLogRelMsgReturn(pGstCtx->aXcr[0] & XSAVE_C_X87, ("xcr0=%#RX64\n", pGstCtx->aXcr[0]), VERR_CPUM_INVALID_XCR0);
+                if (pGstCtx->aXcr[0] != XSAVE_C_X87)
+                {
+                    AssertLogRelMsgReturn(!(pGstCtx->aXcr[0] & ~(pGstCtx->fXStateMask | XSAVE_C_X87)),
+                                          ("xcr0=%#RX64 fXStateMask=%#RX64\n", pGstCtx->aXcr[0], pGstCtx->fXStateMask),
+                                          VERR_CPUM_INVALID_XCR0);
+                    AssertLogRelMsgReturn(pGstCtx->aXcr[0] & XSAVE_C_X87,
+                                          ("xcr0=%#RX64\n", pGstCtx->aXcr[0]), VERR_CPUM_INVALID_XSAVE_COMP_MASK);
+                    AssertLogRelMsgReturn((pGstCtx->aXcr[0] & (XSAVE_C_SSE | XSAVE_C_YMM)) != XSAVE_C_YMM,
+                                          ("xcr0=%#RX64\n", pGstCtx->aXcr[0]), VERR_CPUM_INVALID_XSAVE_COMP_MASK);
+                    AssertLogRelMsgReturn(   (pGstCtx->aXcr[0] & (XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI)) == 0
+                                          ||    (pGstCtx->aXcr[0] & (XSAVE_C_SSE | XSAVE_C_YMM | XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI))
+                                             ==                     (XSAVE_C_SSE | XSAVE_C_YMM | XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI),
+                                          ("xcr0=%#RX64\n", pGstCtx->aXcr[0]), VERR_CPUM_INVALID_XSAVE_COMP_MASK);
+                }
+
+                /* Check that the XCR1 is zero, as we don't implement it yet. */
+                AssertLogRelMsgReturn(!pGstCtx->aXcr[1], ("xcr1=%#RX64\n", pGstCtx->aXcr[1]), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+                /*
+                 * Restore the individual extended state components we support.
+                 */
+                if (pGstCtx->fXStateMask != 0)
+                {
+                    rc = SSMR3GetStructEx(pSSM, &pGstCtx->pXStateR3->Hdr, sizeof(pGstCtx->pXStateR3->Hdr),
+                                          0, g_aCpumXSaveHdrFields, NULL);
+                    AssertRCReturn(rc, rc);
+                    AssertLogRelMsgReturn(!(pGstCtx->pXStateR3->Hdr.bmXState & ~pGstCtx->fXStateMask),
+                                          ("bmXState=%#RX64 fXStateMask=%#RX64\n",
+                                           pGstCtx->pXStateR3->Hdr.bmXState, pGstCtx->fXStateMask),
+                                          VERR_CPUM_INVALID_XSAVE_HDR);
+                }
+                if (pGstCtx->fXStateMask & XSAVE_C_YMM)
+                {
+                    PX86XSAVEYMMHI pYmmHiCtx = CPUMCTX_XSAVE_C_PTR(pGstCtx, XSAVE_C_YMM_BIT, PX86XSAVEYMMHI);
+                    SSMR3GetStructEx(pSSM, pYmmHiCtx, sizeof(*pYmmHiCtx), SSMSTRUCT_FLAGS_FULL_STRUCT, g_aCpumYmmHiFields, NULL);
+                }
+                if (pGstCtx->fXStateMask & XSAVE_C_BNDREGS)
+                {
+                    PX86XSAVEBNDREGS pBndRegs = CPUMCTX_XSAVE_C_PTR(pGstCtx, XSAVE_C_BNDREGS_BIT, PX86XSAVEBNDREGS);
+                    SSMR3GetStructEx(pSSM, pBndRegs, sizeof(*pBndRegs), SSMSTRUCT_FLAGS_FULL_STRUCT, g_aCpumBndRegsFields, NULL);
+                }
+                if (pGstCtx->fXStateMask & XSAVE_C_BNDCSR)
+                {
+                    PX86XSAVEBNDCFG pBndCfg = CPUMCTX_XSAVE_C_PTR(pGstCtx, XSAVE_C_BNDCSR_BIT, PX86XSAVEBNDCFG);
+                    SSMR3GetStructEx(pSSM, pBndCfg, sizeof(*pBndCfg), SSMSTRUCT_FLAGS_FULL_STRUCT, g_aCpumBndCfgFields, NULL);
+                }
+                if (pGstCtx->fXStateMask & XSAVE_C_ZMM_HI256)
+                {
+                    PX86XSAVEZMMHI256 pZmmHi256 = CPUMCTX_XSAVE_C_PTR(pGstCtx, XSAVE_C_ZMM_HI256_BIT, PX86XSAVEZMMHI256);
+                    SSMR3GetStructEx(pSSM, pZmmHi256, sizeof(*pZmmHi256), SSMSTRUCT_FLAGS_FULL_STRUCT, g_aCpumZmmHi256Fields, NULL);
+                }
+                if (pGstCtx->fXStateMask & XSAVE_C_ZMM_16HI)
+                {
+                    PX86XSAVEZMM16HI pZmm16Hi = CPUMCTX_XSAVE_C_PTR(pGstCtx, XSAVE_C_ZMM_16HI_BIT, PX86XSAVEZMM16HI);
+                    SSMR3GetStructEx(pSSM, pZmm16Hi, sizeof(*pZmm16Hi), SSMSTRUCT_FLAGS_FULL_STRUCT, g_aCpumZmm16HiFields, NULL);
+                }
+                if (uVersion >= CPUM_SAVED_STATE_VERSION_HWVIRT_SVM)
+                {
+                    if (pVM->cpum.s.GuestFeatures.fSvm)
+                    {
+                        Assert(pGstCtx->hwvirt.svm.CTX_SUFF(pVmcb));
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.svm.uMsrHSavePa);
+                        SSMR3GetGCPhys(pSSM,   &pGstCtx->hwvirt.svm.GCPhysVmcb);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.svm.uPrevPauseTick);
+                        SSMR3GetU16(pSSM,      &pGstCtx->hwvirt.svm.cPauseFilter);
+                        SSMR3GetU16(pSSM,      &pGstCtx->hwvirt.svm.cPauseFilterThreshold);
+                        SSMR3GetBool(pSSM,     &pGstCtx->hwvirt.svm.fInterceptEvents);
+                        SSMR3GetStructEx(pSSM, &pGstCtx->hwvirt.svm.HostState, sizeof(pGstCtx->hwvirt.svm.HostState),
+                                         0 /* fFlags */, g_aSvmHwvirtHostState, NULL /* pvUser */);
+                        SSMR3GetMem(pSSM,       pGstCtx->hwvirt.svm.pVmcbR3,       SVM_VMCB_PAGES  << X86_PAGE_4K_SHIFT);
+                        SSMR3GetMem(pSSM,       pGstCtx->hwvirt.svm.pvMsrBitmapR3, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
+                        SSMR3GetMem(pSSM,       pGstCtx->hwvirt.svm.pvIoBitmapR3,  SVM_IOPM_PAGES  << X86_PAGE_4K_SHIFT);
+                        SSMR3GetU32(pSSM,      &pGstCtx->hwvirt.fLocalForcedActions);
+                        SSMR3GetBool(pSSM,     &pGstCtx->hwvirt.fGif);
+                    }
+                }
+                if (uVersion >= CPUM_SAVED_STATE_VERSION_HWVIRT_VMX_IEM)
+                {
+                    if (pVM->cpum.s.GuestFeatures.fVmx)
+                    {
+                        Assert(pGstCtx->hwvirt.vmx.CTX_SUFF(pVmcs));
+                        SSMR3GetGCPhys(pSSM,   &pGstCtx->hwvirt.vmx.GCPhysVmxon);
+                        SSMR3GetGCPhys(pSSM,   &pGstCtx->hwvirt.vmx.GCPhysVmcs);
+                        SSMR3GetGCPhys(pSSM,   &pGstCtx->hwvirt.vmx.GCPhysShadowVmcs);
+                        SSMR3GetBool(pSSM,     &pGstCtx->hwvirt.vmx.fInVmxRootMode);
+                        SSMR3GetBool(pSSM,     &pGstCtx->hwvirt.vmx.fInVmxNonRootMode);
+                        SSMR3GetBool(pSSM,     &pGstCtx->hwvirt.vmx.fInterceptEvents);
+                        SSMR3GetBool(pSSM,     &pGstCtx->hwvirt.vmx.fNmiUnblockingIret);
+                        SSMR3GetStructEx(pSSM,  pGstCtx->hwvirt.vmx.pVmcsR3, sizeof(VMXVVMCS), 0, g_aVmxHwvirtVmcs, NULL);
+                        SSMR3GetStructEx(pSSM,  pGstCtx->hwvirt.vmx.pShadowVmcsR3, sizeof(VMXVVMCS), 0, g_aVmxHwvirtVmcs, NULL);
+                        SSMR3GetMem(pSSM,       pGstCtx->hwvirt.vmx.pvVmreadBitmapR3, VMX_V_VMREAD_VMWRITE_BITMAP_SIZE);
+                        SSMR3GetMem(pSSM,       pGstCtx->hwvirt.vmx.pvVmwriteBitmapR3, VMX_V_VMREAD_VMWRITE_BITMAP_SIZE);
+                        SSMR3GetMem(pSSM,       pGstCtx->hwvirt.vmx.pEntryMsrLoadAreaR3, VMX_V_AUTOMSR_AREA_SIZE);
+                        SSMR3GetMem(pSSM,       pGstCtx->hwvirt.vmx.pExitMsrStoreAreaR3, VMX_V_AUTOMSR_AREA_SIZE);
+                        SSMR3GetMem(pSSM,       pGstCtx->hwvirt.vmx.pExitMsrLoadAreaR3,  VMX_V_AUTOMSR_AREA_SIZE);
+                        SSMR3GetMem(pSSM,       pGstCtx->hwvirt.vmx.pvMsrBitmapR3, VMX_V_MSR_BITMAP_SIZE);
+                        SSMR3GetMem(pSSM,       pGstCtx->hwvirt.vmx.pvIoBitmapR3, VMX_V_IO_BITMAP_A_SIZE + VMX_V_IO_BITMAP_B_SIZE);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.uFirstPauseLoopTick);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.uPrevPauseTick);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.uEntryTick);
+                        SSMR3GetU16(pSSM,      &pGstCtx->hwvirt.vmx.offVirtApicWrite);
+                        SSMR3GetBool(pSSM,     &pGstCtx->hwvirt.vmx.fVirtNmiBlocking);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.u64FeatCtrl);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.u64Basic);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.PinCtls.u);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.ProcCtls.u);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.ProcCtls2.u);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.ExitCtls.u);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.EntryCtls.u);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.TruePinCtls.u);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.TrueProcCtls.u);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.TrueEntryCtls.u);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.TrueExitCtls.u);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.u64Misc);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.u64VmcsEnum);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.u64VmFunc);
+                        SSMR3GetU64(pSSM,      &pGstCtx->hwvirt.vmx.Msrs.u64EptVpidCaps);
+                    }
+                }
+            }
+            else
+            {
+                /*
+                 * Pre XSAVE saved state.
+                 */
+                SSMR3GetStructEx(pSSM, &pGstCtx->pXStateR3->x87, sizeof(pGstCtx->pXStateR3->x87),
+                                 fLoad | SSMSTRUCT_FLAGS_NO_TAIL_MARKER, paCpumCtx1Fields, NULL);
+                SSMR3GetStructEx(pSSM, pGstCtx, sizeof(*pGstCtx), fLoad | SSMSTRUCT_FLAGS_NO_LEAD_MARKER, paCpumCtx2Fields, NULL);
+            }
+
+            /*
+             * Restore a couple of flags and the MSRs.
+             */
+            SSMR3GetU32(pSSM, &pVCpu->cpum.s.fUseFlags);
+            SSMR3GetU32(pSSM, &pVCpu->cpum.s.fChanged);
+
+            rc = VINF_SUCCESS;
+            if (uVersion > CPUM_SAVED_STATE_VERSION_NO_MSR_SIZE)
+                rc = SSMR3GetMem(pSSM, &pVCpu->cpum.s.GuestMsrs.au64[0], cbMsrs);
+            else if (uVersion >= CPUM_SAVED_STATE_VERSION_VER3_0)
+            {
+                SSMR3GetMem(pSSM, &pVCpu->cpum.s.GuestMsrs.au64[0], 2 * sizeof(uint64_t)); /* Restore two MSRs. */
+                rc = SSMR3Skip(pSSM, 62 * sizeof(uint64_t));
+            }
+            AssertRCReturn(rc, rc);
+
+            /* REM and other may have cleared must-be-one fields in DR6 and
+               DR7, fix these. */
+            pGstCtx->dr[6] &= ~(X86_DR6_RAZ_MASK | X86_DR6_MBZ_MASK);
+            pGstCtx->dr[6] |= X86_DR6_RA1_MASK;
+            pGstCtx->dr[7] &= ~(X86_DR7_RAZ_MASK | X86_DR7_MBZ_MASK);
+            pGstCtx->dr[7] |= X86_DR7_RA1_MASK;
+        }
+
+        /* Older states does not have the internal selector register flags
+           and valid selector value.  Supply those. */
+        if (uVersion <= CPUM_SAVED_STATE_VERSION_MEM)
+        {
+            for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+            {
+                PVMCPU      pVCpu  = pVM->apCpusR3[idCpu];
+                bool const  fValid = true /*!VM_IS_RAW_MODE_ENABLED(pVM)*/
+                                  || (   uVersion > CPUM_SAVED_STATE_VERSION_VER3_2
+                                      && !(pVCpu->cpum.s.fChanged & CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID));
+                PCPUMSELREG paSelReg = CPUMCTX_FIRST_SREG(&pVCpu->cpum.s.Guest);
+                if (fValid)
+                {
+                    for (uint32_t iSelReg = 0; iSelReg < X86_SREG_COUNT; iSelReg++)
+                    {
+                        paSelReg[iSelReg].fFlags   = CPUMSELREG_FLAGS_VALID;
+                        paSelReg[iSelReg].ValidSel = paSelReg[iSelReg].Sel;
+                    }
+
+                    pVCpu->cpum.s.Guest.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+                    pVCpu->cpum.s.Guest.ldtr.ValidSel = pVCpu->cpum.s.Guest.ldtr.Sel;
+                }
+                else
+                {
+                    for (uint32_t iSelReg = 0; iSelReg < X86_SREG_COUNT; iSelReg++)
+                    {
+                        paSelReg[iSelReg].fFlags   = 0;
+                        paSelReg[iSelReg].ValidSel = 0;
+                    }
+
+                    /* This might not be 104% correct, but I think it's close
+                       enough for all practical purposes...  (REM always loaded
+                       LDTR registers.) */
+                    pVCpu->cpum.s.Guest.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+                    pVCpu->cpum.s.Guest.ldtr.ValidSel = pVCpu->cpum.s.Guest.ldtr.Sel;
+                }
+                pVCpu->cpum.s.Guest.tr.fFlags     = CPUMSELREG_FLAGS_VALID;
+                pVCpu->cpum.s.Guest.tr.ValidSel   = pVCpu->cpum.s.Guest.tr.Sel;
+            }
+        }
+
+        /* Clear CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID. */
+        if (   uVersion >  CPUM_SAVED_STATE_VERSION_VER3_2
+            && uVersion <= CPUM_SAVED_STATE_VERSION_MEM)
+            for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+            {
+                PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+                pVCpu->cpum.s.fChanged &= CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID;
+            }
+
+        /*
+         * A quick sanity check.
+         */
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+            AssertLogRelReturn(!(pVCpu->cpum.s.Guest.es.fFlags & ~CPUMSELREG_FLAGS_VALID_MASK), VERR_SSM_UNEXPECTED_DATA);
+            AssertLogRelReturn(!(pVCpu->cpum.s.Guest.cs.fFlags & ~CPUMSELREG_FLAGS_VALID_MASK), VERR_SSM_UNEXPECTED_DATA);
+            AssertLogRelReturn(!(pVCpu->cpum.s.Guest.ss.fFlags & ~CPUMSELREG_FLAGS_VALID_MASK), VERR_SSM_UNEXPECTED_DATA);
+            AssertLogRelReturn(!(pVCpu->cpum.s.Guest.ds.fFlags & ~CPUMSELREG_FLAGS_VALID_MASK), VERR_SSM_UNEXPECTED_DATA);
+            AssertLogRelReturn(!(pVCpu->cpum.s.Guest.fs.fFlags & ~CPUMSELREG_FLAGS_VALID_MASK), VERR_SSM_UNEXPECTED_DATA);
+            AssertLogRelReturn(!(pVCpu->cpum.s.Guest.gs.fFlags & ~CPUMSELREG_FLAGS_VALID_MASK), VERR_SSM_UNEXPECTED_DATA);
+        }
+    }
+
+    pVM->cpum.s.fPendingRestore = false;
+
+    /*
+     * Guest CPUIDs (and VMX MSR features).
+     */
+    if (uVersion >= CPUM_SAVED_STATE_VERSION_VER3_2)
+    {
+        CPUMMSRS GuestMsrs;
+        RT_ZERO(GuestMsrs);
+
+        CPUMFEATURES BaseFeatures;
+        bool const fVmxGstFeat = pVM->cpum.s.GuestFeatures.fVmx;
+        if (fVmxGstFeat)
+        {
+            /*
+             * At this point the MSRs in the guest CPU-context are loaded with the guest VMX MSRs from the saved state.
+             * However the VMX sub-features have not been exploded yet. So cache the base (host derived) VMX features
+             * here so we can compare them for compatibility after exploding guest features.
+             */
+            BaseFeatures = pVM->cpum.s.GuestFeatures;
+
+            /* Use the VMX MSR features from the saved state while exploding guest features. */
+            GuestMsrs.hwvirt.vmx = pVM->apCpusR3[0]->cpum.s.Guest.hwvirt.vmx.Msrs;
+        }
+
+        /* Load CPUID and explode guest features. */
+        rc = cpumR3LoadCpuId(pVM, pSSM, uVersion, &GuestMsrs);
+        if (fVmxGstFeat)
+        {
+            /*
+             * Check if the exploded VMX features from the saved state are compatible with the host-derived features
+             * we cached earlier (above). The is required if we use hardware-assisted nested-guest execution with
+             * VMX features presented to the guest.
+             */
+            bool const fIsCompat = cpumR3AreVmxCpuFeaturesCompatible(pVM, &BaseFeatures, &pVM->cpum.s.GuestFeatures);
+            if (!fIsCompat)
+                return VERR_CPUM_INVALID_HWVIRT_FEAT_COMBO;
+        }
+        return rc;
+    }
+    return cpumR3LoadCpuIdPre32(pVM, pSSM, uVersion);
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTLOADDONE}
+ */
+static DECLCALLBACK(int) cpumR3LoadDone(PVM pVM, PSSMHANDLE pSSM)
+{
+    if (RT_FAILURE(SSMR3HandleGetStatus(pSSM)))
+        return VINF_SUCCESS;
+
+    /* just check this since we can. */ /** @todo Add a SSM unit flag for indicating that it's mandatory during a restore.  */
+    if (pVM->cpum.s.fPendingRestore)
+    {
+        LogRel(("CPUM: Missing state!\n"));
+        return VERR_INTERNAL_ERROR_2;
+    }
+
+    bool const fSupportsLongMode = VMR3IsLongModeAllowed(pVM);
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+
+        /* Notify PGM of the NXE states in case they've changed. */
+        PGMNotifyNxeChanged(pVCpu, RT_BOOL(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_NXE));
+
+        /* During init. this is done in CPUMR3InitCompleted(). */
+        if (fSupportsLongMode)
+            pVCpu->cpum.s.fUseFlags |= CPUM_USE_SUPPORTS_LONGMODE;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the CPUM state restore is still pending.
+ *
+ * @returns true / false.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMDECL(bool) CPUMR3IsStateRestorePending(PVM pVM)
+{
+    return pVM->cpum.s.fPendingRestore;
+}
+
+
+/**
+ * Formats the EFLAGS value into mnemonics.
+ *
+ * @param   pszEFlags   Where to write the mnemonics. (Assumes sufficient buffer space.)
+ * @param   efl         The EFLAGS value.
+ */
+static void cpumR3InfoFormatFlags(char *pszEFlags, uint32_t efl)
+{
+    /*
+     * Format the flags.
+     */
+    static const struct
+    {
+        const char *pszSet; const char *pszClear; uint32_t fFlag;
+    }   s_aFlags[] =
+    {
+        { "vip",NULL, X86_EFL_VIP },
+        { "vif",NULL, X86_EFL_VIF },
+        { "ac", NULL, X86_EFL_AC },
+        { "vm", NULL, X86_EFL_VM },
+        { "rf", NULL, X86_EFL_RF },
+        { "nt", NULL, X86_EFL_NT },
+        { "ov", "nv", X86_EFL_OF },
+        { "dn", "up", X86_EFL_DF },
+        { "ei", "di", X86_EFL_IF },
+        { "tf", NULL, X86_EFL_TF },
+        { "nt", "pl", X86_EFL_SF },
+        { "nz", "zr", X86_EFL_ZF },
+        { "ac", "na", X86_EFL_AF },
+        { "po", "pe", X86_EFL_PF },
+        { "cy", "nc", X86_EFL_CF },
+    };
+    char *psz = pszEFlags;
+    for (unsigned i = 0; i < RT_ELEMENTS(s_aFlags); i++)
+    {
+        const char *pszAdd = s_aFlags[i].fFlag & efl ? s_aFlags[i].pszSet : s_aFlags[i].pszClear;
+        if (pszAdd)
+        {
+            strcpy(psz, pszAdd);
+            psz += strlen(pszAdd);
+            *psz++ = ' ';
+        }
+    }
+    psz[-1] = '\0';
+}
+
+
+/**
+ * Formats a full register dump.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pCtx        The context to format.
+ * @param   pCtxCore    The context core to format.
+ * @param   pHlp        Output functions.
+ * @param   enmType     The dump type.
+ * @param   pszPrefix   Register name prefix.
+ */
+static void cpumR3InfoOne(PVM pVM, PCPUMCTX pCtx, PCCPUMCTXCORE pCtxCore, PCDBGFINFOHLP pHlp, CPUMDUMPTYPE enmType,
+                          const char *pszPrefix)
+{
+    NOREF(pVM);
+
+    /*
+     * Format the EFLAGS.
+     */
+    uint32_t efl = pCtxCore->eflags.u32;
+    char szEFlags[80];
+    cpumR3InfoFormatFlags(&szEFlags[0], efl);
+
+    /*
+     * Format the registers.
+     */
+    switch (enmType)
+    {
+        case CPUMDUMPTYPE_TERSE:
+            if (CPUMIsGuestIn64BitCodeEx(pCtx))
+                pHlp->pfnPrintf(pHlp,
+                    "%srax=%016RX64 %srbx=%016RX64 %srcx=%016RX64 %srdx=%016RX64\n"
+                    "%srsi=%016RX64 %srdi=%016RX64 %sr8 =%016RX64 %sr9 =%016RX64\n"
+                    "%sr10=%016RX64 %sr11=%016RX64 %sr12=%016RX64 %sr13=%016RX64\n"
+                    "%sr14=%016RX64 %sr15=%016RX64\n"
+                    "%srip=%016RX64 %srsp=%016RX64 %srbp=%016RX64 %siopl=%d %*s\n"
+                    "%scs=%04x %sss=%04x %sds=%04x %ses=%04x %sfs=%04x %sgs=%04x                %seflags=%08x\n",
+                    pszPrefix, pCtxCore->rax, pszPrefix, pCtxCore->rbx, pszPrefix, pCtxCore->rcx, pszPrefix, pCtxCore->rdx, pszPrefix, pCtxCore->rsi, pszPrefix, pCtxCore->rdi,
+                    pszPrefix, pCtxCore->r8, pszPrefix, pCtxCore->r9, pszPrefix, pCtxCore->r10, pszPrefix, pCtxCore->r11, pszPrefix, pCtxCore->r12, pszPrefix, pCtxCore->r13,
+                    pszPrefix, pCtxCore->r14, pszPrefix, pCtxCore->r15,
+                    pszPrefix, pCtxCore->rip, pszPrefix, pCtxCore->rsp, pszPrefix, pCtxCore->rbp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
+                    pszPrefix, pCtxCore->cs.Sel, pszPrefix, pCtxCore->ss.Sel, pszPrefix, pCtxCore->ds.Sel, pszPrefix, pCtxCore->es.Sel,
+                    pszPrefix, pCtxCore->fs.Sel, pszPrefix, pCtxCore->gs.Sel, pszPrefix, efl);
+            else
+                pHlp->pfnPrintf(pHlp,
+                    "%seax=%08x %sebx=%08x %secx=%08x %sedx=%08x %sesi=%08x %sedi=%08x\n"
+                    "%seip=%08x %sesp=%08x %sebp=%08x %siopl=%d %*s\n"
+                    "%scs=%04x %sss=%04x %sds=%04x %ses=%04x %sfs=%04x %sgs=%04x                %seflags=%08x\n",
+                    pszPrefix, pCtxCore->eax, pszPrefix, pCtxCore->ebx, pszPrefix, pCtxCore->ecx, pszPrefix, pCtxCore->edx, pszPrefix, pCtxCore->esi, pszPrefix, pCtxCore->edi,
+                    pszPrefix, pCtxCore->eip, pszPrefix, pCtxCore->esp, pszPrefix, pCtxCore->ebp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
+                    pszPrefix, pCtxCore->cs.Sel, pszPrefix, pCtxCore->ss.Sel, pszPrefix, pCtxCore->ds.Sel, pszPrefix, pCtxCore->es.Sel,
+                    pszPrefix, pCtxCore->fs.Sel, pszPrefix, pCtxCore->gs.Sel, pszPrefix, efl);
+            break;
+
+        case CPUMDUMPTYPE_DEFAULT:
+            if (CPUMIsGuestIn64BitCodeEx(pCtx))
+                pHlp->pfnPrintf(pHlp,
+                    "%srax=%016RX64 %srbx=%016RX64 %srcx=%016RX64 %srdx=%016RX64\n"
+                    "%srsi=%016RX64 %srdi=%016RX64 %sr8 =%016RX64 %sr9 =%016RX64\n"
+                    "%sr10=%016RX64 %sr11=%016RX64 %sr12=%016RX64 %sr13=%016RX64\n"
+                    "%sr14=%016RX64 %sr15=%016RX64\n"
+                    "%srip=%016RX64 %srsp=%016RX64 %srbp=%016RX64 %siopl=%d %*s\n"
+                    "%scs=%04x %sss=%04x %sds=%04x %ses=%04x %sfs=%04x %sgs=%04x %str=%04x      %seflags=%08x\n"
+                    "%scr0=%08RX64 %scr2=%08RX64 %scr3=%08RX64 %scr4=%08RX64 %sgdtr=%016RX64:%04x %sldtr=%04x\n"
+                    ,
+                    pszPrefix, pCtxCore->rax, pszPrefix, pCtxCore->rbx, pszPrefix, pCtxCore->rcx, pszPrefix, pCtxCore->rdx, pszPrefix, pCtxCore->rsi, pszPrefix, pCtxCore->rdi,
+                    pszPrefix, pCtxCore->r8, pszPrefix, pCtxCore->r9, pszPrefix, pCtxCore->r10, pszPrefix, pCtxCore->r11, pszPrefix, pCtxCore->r12, pszPrefix, pCtxCore->r13,
+                    pszPrefix, pCtxCore->r14, pszPrefix, pCtxCore->r15,
+                    pszPrefix, pCtxCore->rip, pszPrefix, pCtxCore->rsp, pszPrefix, pCtxCore->rbp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
+                    pszPrefix, pCtxCore->cs.Sel, pszPrefix, pCtxCore->ss.Sel, pszPrefix, pCtxCore->ds.Sel, pszPrefix, pCtxCore->es.Sel,
+                    pszPrefix, pCtxCore->fs.Sel, pszPrefix, pCtxCore->gs.Sel, pszPrefix, pCtx->tr.Sel, pszPrefix, efl,
+                    pszPrefix, pCtx->cr0, pszPrefix, pCtx->cr2, pszPrefix, pCtx->cr3, pszPrefix, pCtx->cr4,
+                    pszPrefix, pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pszPrefix, pCtx->ldtr.Sel);
+            else
+                pHlp->pfnPrintf(pHlp,
+                    "%seax=%08x %sebx=%08x %secx=%08x %sedx=%08x %sesi=%08x %sedi=%08x\n"
+                    "%seip=%08x %sesp=%08x %sebp=%08x %siopl=%d %*s\n"
+                    "%scs=%04x %sss=%04x %sds=%04x %ses=%04x %sfs=%04x %sgs=%04x %str=%04x      %seflags=%08x\n"
+                    "%scr0=%08RX64 %scr2=%08RX64 %scr3=%08RX64 %scr4=%08RX64 %sgdtr=%08RX64:%04x %sldtr=%04x\n"
+                    ,
+                    pszPrefix, pCtxCore->eax, pszPrefix, pCtxCore->ebx, pszPrefix, pCtxCore->ecx, pszPrefix, pCtxCore->edx, pszPrefix, pCtxCore->esi, pszPrefix, pCtxCore->edi,
+                    pszPrefix, pCtxCore->eip, pszPrefix, pCtxCore->esp, pszPrefix, pCtxCore->ebp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
+                    pszPrefix, pCtxCore->cs.Sel, pszPrefix, pCtxCore->ss.Sel, pszPrefix, pCtxCore->ds.Sel, pszPrefix, pCtxCore->es.Sel,
+                    pszPrefix, pCtxCore->fs.Sel, pszPrefix, pCtxCore->gs.Sel, pszPrefix, pCtx->tr.Sel, pszPrefix, efl,
+                    pszPrefix, pCtx->cr0, pszPrefix, pCtx->cr2, pszPrefix, pCtx->cr3, pszPrefix, pCtx->cr4,
+                    pszPrefix, pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pszPrefix, pCtx->ldtr.Sel);
+            break;
+
+        case CPUMDUMPTYPE_VERBOSE:
+            if (CPUMIsGuestIn64BitCodeEx(pCtx))
+                pHlp->pfnPrintf(pHlp,
+                    "%srax=%016RX64 %srbx=%016RX64 %srcx=%016RX64 %srdx=%016RX64\n"
+                    "%srsi=%016RX64 %srdi=%016RX64 %sr8 =%016RX64 %sr9 =%016RX64\n"
+                    "%sr10=%016RX64 %sr11=%016RX64 %sr12=%016RX64 %sr13=%016RX64\n"
+                    "%sr14=%016RX64 %sr15=%016RX64\n"
+                    "%srip=%016RX64 %srsp=%016RX64 %srbp=%016RX64 %siopl=%d %*s\n"
+                    "%scs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                    "%sds={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                    "%ses={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                    "%sfs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                    "%sgs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                    "%sss={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                    "%scr0=%016RX64 %scr2=%016RX64 %scr3=%016RX64 %scr4=%016RX64\n"
+                    "%sdr0=%016RX64 %sdr1=%016RX64 %sdr2=%016RX64 %sdr3=%016RX64\n"
+                    "%sdr4=%016RX64 %sdr5=%016RX64 %sdr6=%016RX64 %sdr7=%016RX64\n"
+                    "%sgdtr=%016RX64:%04x  %sidtr=%016RX64:%04x  %seflags=%08x\n"
+                    "%sldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
+                    "%str  ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
+                    "%sSysEnter={cs=%04llx eip=%016RX64 esp=%016RX64}\n"
+                    ,
+                    pszPrefix, pCtxCore->rax, pszPrefix, pCtxCore->rbx, pszPrefix, pCtxCore->rcx, pszPrefix, pCtxCore->rdx, pszPrefix, pCtxCore->rsi, pszPrefix, pCtxCore->rdi,
+                    pszPrefix, pCtxCore->r8, pszPrefix, pCtxCore->r9, pszPrefix, pCtxCore->r10, pszPrefix, pCtxCore->r11, pszPrefix, pCtxCore->r12, pszPrefix, pCtxCore->r13,
+                    pszPrefix, pCtxCore->r14, pszPrefix, pCtxCore->r15,
+                    pszPrefix, pCtxCore->rip, pszPrefix, pCtxCore->rsp, pszPrefix, pCtxCore->rbp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
+                    pszPrefix, pCtxCore->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit, pCtx->cs.Attr.u,
+                    pszPrefix, pCtxCore->ds.Sel, pCtx->ds.u64Base, pCtx->ds.u32Limit, pCtx->ds.Attr.u,
+                    pszPrefix, pCtxCore->es.Sel, pCtx->es.u64Base, pCtx->es.u32Limit, pCtx->es.Attr.u,
+                    pszPrefix, pCtxCore->fs.Sel, pCtx->fs.u64Base, pCtx->fs.u32Limit, pCtx->fs.Attr.u,
+                    pszPrefix, pCtxCore->gs.Sel, pCtx->gs.u64Base, pCtx->gs.u32Limit, pCtx->gs.Attr.u,
+                    pszPrefix, pCtxCore->ss.Sel, pCtx->ss.u64Base, pCtx->ss.u32Limit, pCtx->ss.Attr.u,
+                    pszPrefix, pCtx->cr0,  pszPrefix, pCtx->cr2, pszPrefix, pCtx->cr3,  pszPrefix, pCtx->cr4,
+                    pszPrefix, pCtx->dr[0],  pszPrefix, pCtx->dr[1], pszPrefix, pCtx->dr[2],  pszPrefix, pCtx->dr[3],
+                    pszPrefix, pCtx->dr[4],  pszPrefix, pCtx->dr[5], pszPrefix, pCtx->dr[6],  pszPrefix, pCtx->dr[7],
+                    pszPrefix, pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pszPrefix, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, pszPrefix, efl,
+                    pszPrefix, pCtx->ldtr.Sel, pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit, pCtx->ldtr.Attr.u,
+                    pszPrefix, pCtx->tr.Sel, pCtx->tr.u64Base, pCtx->tr.u32Limit, pCtx->tr.Attr.u,
+                    pszPrefix, pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp);
+            else
+                pHlp->pfnPrintf(pHlp,
+                    "%seax=%08x %sebx=%08x %secx=%08x %sedx=%08x %sesi=%08x %sedi=%08x\n"
+                    "%seip=%08x %sesp=%08x %sebp=%08x %siopl=%d %*s\n"
+                    "%scs={%04x base=%016RX64 limit=%08x flags=%08x} %sdr0=%08RX64 %sdr1=%08RX64\n"
+                    "%sds={%04x base=%016RX64 limit=%08x flags=%08x} %sdr2=%08RX64 %sdr3=%08RX64\n"
+                    "%ses={%04x base=%016RX64 limit=%08x flags=%08x} %sdr4=%08RX64 %sdr5=%08RX64\n"
+                    "%sfs={%04x base=%016RX64 limit=%08x flags=%08x} %sdr6=%08RX64 %sdr7=%08RX64\n"
+                    "%sgs={%04x base=%016RX64 limit=%08x flags=%08x} %scr0=%08RX64 %scr2=%08RX64\n"
+                    "%sss={%04x base=%016RX64 limit=%08x flags=%08x} %scr3=%08RX64 %scr4=%08RX64\n"
+                    "%sgdtr=%016RX64:%04x  %sidtr=%016RX64:%04x  %seflags=%08x\n"
+                    "%sldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
+                    "%str  ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
+                    "%sSysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
+                    ,
+                    pszPrefix, pCtxCore->eax, pszPrefix, pCtxCore->ebx, pszPrefix, pCtxCore->ecx, pszPrefix, pCtxCore->edx, pszPrefix, pCtxCore->esi, pszPrefix, pCtxCore->edi,
+                    pszPrefix, pCtxCore->eip, pszPrefix, pCtxCore->esp, pszPrefix, pCtxCore->ebp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
+                    pszPrefix, pCtxCore->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit, pCtx->cs.Attr.u, pszPrefix, pCtx->dr[0],  pszPrefix, pCtx->dr[1],
+                    pszPrefix, pCtxCore->ds.Sel, pCtx->ds.u64Base, pCtx->ds.u32Limit, pCtx->ds.Attr.u, pszPrefix, pCtx->dr[2],  pszPrefix, pCtx->dr[3],
+                    pszPrefix, pCtxCore->es.Sel, pCtx->es.u64Base, pCtx->es.u32Limit, pCtx->es.Attr.u, pszPrefix, pCtx->dr[4],  pszPrefix, pCtx->dr[5],
+                    pszPrefix, pCtxCore->fs.Sel, pCtx->fs.u64Base, pCtx->fs.u32Limit, pCtx->fs.Attr.u, pszPrefix, pCtx->dr[6],  pszPrefix, pCtx->dr[7],
+                    pszPrefix, pCtxCore->gs.Sel, pCtx->gs.u64Base, pCtx->gs.u32Limit, pCtx->gs.Attr.u, pszPrefix, pCtx->cr0,  pszPrefix, pCtx->cr2,
+                    pszPrefix, pCtxCore->ss.Sel, pCtx->ss.u64Base, pCtx->ss.u32Limit, pCtx->ss.Attr.u, pszPrefix, pCtx->cr3,  pszPrefix, pCtx->cr4,
+                    pszPrefix, pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pszPrefix, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, pszPrefix, efl,
+                    pszPrefix, pCtx->ldtr.Sel, pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit, pCtx->ldtr.Attr.u,
+                    pszPrefix, pCtx->tr.Sel, pCtx->tr.u64Base, pCtx->tr.u32Limit, pCtx->tr.Attr.u,
+                    pszPrefix, pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp);
+
+            pHlp->pfnPrintf(pHlp, "%sxcr=%016RX64 %sxcr1=%016RX64 %sxss=%016RX64 (fXStateMask=%016RX64)\n",
+                            pszPrefix, pCtx->aXcr[0], pszPrefix, pCtx->aXcr[1],
+                            pszPrefix, UINT64_C(0) /** @todo XSS */, pCtx->fXStateMask);
+            if (pCtx->CTX_SUFF(pXState))
+            {
+                PX86FXSTATE pFpuCtx = &pCtx->CTX_SUFF(pXState)->x87;
+                pHlp->pfnPrintf(pHlp,
+                    "%sFCW=%04x %sFSW=%04x %sFTW=%04x %sFOP=%04x %sMXCSR=%08x %sMXCSR_MASK=%08x\n"
+                    "%sFPUIP=%08x %sCS=%04x %sRsrvd1=%04x  %sFPUDP=%08x %sDS=%04x %sRsvrd2=%04x\n"
+                    ,
+                    pszPrefix, pFpuCtx->FCW,   pszPrefix, pFpuCtx->FSW, pszPrefix, pFpuCtx->FTW, pszPrefix, pFpuCtx->FOP,
+                    pszPrefix, pFpuCtx->MXCSR, pszPrefix, pFpuCtx->MXCSR_MASK,
+                    pszPrefix, pFpuCtx->FPUIP, pszPrefix, pFpuCtx->CS,  pszPrefix, pFpuCtx->Rsrvd1,
+                    pszPrefix, pFpuCtx->FPUDP, pszPrefix, pFpuCtx->DS,  pszPrefix, pFpuCtx->Rsrvd2
+                    );
+                /*
+                 * The FSAVE style memory image contains ST(0)-ST(7) at increasing addresses,
+                 * not (FP)R0-7 as Intel SDM suggests.
+                 */
+                unsigned iShift = (pFpuCtx->FSW >> 11) & 7;
+                for (unsigned iST = 0; iST < RT_ELEMENTS(pFpuCtx->aRegs); iST++)
+                {
+                    unsigned iFPR        = (iST + iShift) % RT_ELEMENTS(pFpuCtx->aRegs);
+                    unsigned uTag        = (pFpuCtx->FTW >> (2 * iFPR)) & 3;
+                    char     chSign      = pFpuCtx->aRegs[iST].au16[4] & 0x8000 ? '-' : '+';
+                    unsigned iInteger    = (unsigned)(pFpuCtx->aRegs[iST].au64[0] >> 63);
+                    uint64_t u64Fraction = pFpuCtx->aRegs[iST].au64[0] & UINT64_C(0x7fffffffffffffff);
+                    int      iExponent   = pFpuCtx->aRegs[iST].au16[4] & 0x7fff;
+                    iExponent -= 16383; /* subtract bias */
+                    /** @todo This isn't entirenly correct and needs more work! */
+                    pHlp->pfnPrintf(pHlp,
+                                    "%sST(%u)=%sFPR%u={%04RX16'%08RX32'%08RX32} t%d %c%u.%022llu * 2 ^ %d (*)",
+                                    pszPrefix, iST, pszPrefix, iFPR,
+                                    pFpuCtx->aRegs[iST].au16[4], pFpuCtx->aRegs[iST].au32[1], pFpuCtx->aRegs[iST].au32[0],
+                                    uTag, chSign, iInteger, u64Fraction, iExponent);
+                    if (pFpuCtx->aRegs[iST].au16[5] || pFpuCtx->aRegs[iST].au16[6] || pFpuCtx->aRegs[iST].au16[7])
+                        pHlp->pfnPrintf(pHlp, " res={%04RX16,%04RX16,%04RX16}\n",
+                                        pFpuCtx->aRegs[iST].au16[5], pFpuCtx->aRegs[iST].au16[6], pFpuCtx->aRegs[iST].au16[7]);
+                    else
+                        pHlp->pfnPrintf(pHlp, "\n");
+                }
+
+                /* XMM/YMM/ZMM registers. */
+                if (pCtx->fXStateMask & XSAVE_C_YMM)
+                {
+                    PCX86XSAVEYMMHI pYmmHiCtx = CPUMCTX_XSAVE_C_PTR(pCtx, XSAVE_C_YMM_BIT, PCX86XSAVEYMMHI);
+                    if (!(pCtx->fXStateMask & XSAVE_C_ZMM_HI256))
+                        for (unsigned i = 0; i < RT_ELEMENTS(pFpuCtx->aXMM); i++)
+                            pHlp->pfnPrintf(pHlp, "%sYMM%u%s=%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32\n",
+                                            pszPrefix, i, i < 10 ? " " : "",
+                                            pYmmHiCtx->aYmmHi[i].au32[3],
+                                            pYmmHiCtx->aYmmHi[i].au32[2],
+                                            pYmmHiCtx->aYmmHi[i].au32[1],
+                                            pYmmHiCtx->aYmmHi[i].au32[0],
+                                            pFpuCtx->aXMM[i].au32[3],
+                                            pFpuCtx->aXMM[i].au32[2],
+                                            pFpuCtx->aXMM[i].au32[1],
+                                            pFpuCtx->aXMM[i].au32[0]);
+                    else
+                    {
+                        PCX86XSAVEZMMHI256 pZmmHi256 = CPUMCTX_XSAVE_C_PTR(pCtx, XSAVE_C_ZMM_HI256_BIT, PCX86XSAVEZMMHI256);
+                        for (unsigned i = 0; i < RT_ELEMENTS(pFpuCtx->aXMM); i++)
+                            pHlp->pfnPrintf(pHlp,
+                                            "%sZMM%u%s=%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32''%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32\n",
+                                            pszPrefix, i, i < 10 ? " " : "",
+                                            pZmmHi256->aHi256Regs[i].au32[7],
+                                            pZmmHi256->aHi256Regs[i].au32[6],
+                                            pZmmHi256->aHi256Regs[i].au32[5],
+                                            pZmmHi256->aHi256Regs[i].au32[4],
+                                            pZmmHi256->aHi256Regs[i].au32[3],
+                                            pZmmHi256->aHi256Regs[i].au32[2],
+                                            pZmmHi256->aHi256Regs[i].au32[1],
+                                            pZmmHi256->aHi256Regs[i].au32[0],
+                                            pYmmHiCtx->aYmmHi[i].au32[3],
+                                            pYmmHiCtx->aYmmHi[i].au32[2],
+                                            pYmmHiCtx->aYmmHi[i].au32[1],
+                                            pYmmHiCtx->aYmmHi[i].au32[0],
+                                            pFpuCtx->aXMM[i].au32[3],
+                                            pFpuCtx->aXMM[i].au32[2],
+                                            pFpuCtx->aXMM[i].au32[1],
+                                            pFpuCtx->aXMM[i].au32[0]);
+
+                        PCX86XSAVEZMM16HI pZmm16Hi = CPUMCTX_XSAVE_C_PTR(pCtx, XSAVE_C_ZMM_16HI_BIT, PCX86XSAVEZMM16HI);
+                        for (unsigned i = 0; i < RT_ELEMENTS(pZmm16Hi->aRegs); i++)
+                            pHlp->pfnPrintf(pHlp,
+                                            "%sZMM%u=%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32''%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32'%08RX32\n",
+                                            pszPrefix, i + 16,
+                                            pZmm16Hi->aRegs[i].au32[15],
+                                            pZmm16Hi->aRegs[i].au32[14],
+                                            pZmm16Hi->aRegs[i].au32[13],
+                                            pZmm16Hi->aRegs[i].au32[12],
+                                            pZmm16Hi->aRegs[i].au32[11],
+                                            pZmm16Hi->aRegs[i].au32[10],
+                                            pZmm16Hi->aRegs[i].au32[9],
+                                            pZmm16Hi->aRegs[i].au32[8],
+                                            pZmm16Hi->aRegs[i].au32[7],
+                                            pZmm16Hi->aRegs[i].au32[6],
+                                            pZmm16Hi->aRegs[i].au32[5],
+                                            pZmm16Hi->aRegs[i].au32[4],
+                                            pZmm16Hi->aRegs[i].au32[3],
+                                            pZmm16Hi->aRegs[i].au32[2],
+                                            pZmm16Hi->aRegs[i].au32[1],
+                                            pZmm16Hi->aRegs[i].au32[0]);
+                    }
+                }
+                else
+                    for (unsigned i = 0; i < RT_ELEMENTS(pFpuCtx->aXMM); i++)
+                        pHlp->pfnPrintf(pHlp,
+                                        i & 1
+                                        ? "%sXMM%u%s=%08RX32'%08RX32'%08RX32'%08RX32\n"
+                                        : "%sXMM%u%s=%08RX32'%08RX32'%08RX32'%08RX32  ",
+                                        pszPrefix, i, i < 10 ? " " : "",
+                                        pFpuCtx->aXMM[i].au32[3],
+                                        pFpuCtx->aXMM[i].au32[2],
+                                        pFpuCtx->aXMM[i].au32[1],
+                                        pFpuCtx->aXMM[i].au32[0]);
+
+                if (pCtx->fXStateMask & XSAVE_C_OPMASK)
+                {
+                    PCX86XSAVEOPMASK pOpMask = CPUMCTX_XSAVE_C_PTR(pCtx, XSAVE_C_OPMASK_BIT, PCX86XSAVEOPMASK);
+                    for (unsigned i = 0; i < RT_ELEMENTS(pOpMask->aKRegs); i += 4)
+                        pHlp->pfnPrintf(pHlp, "%sK%u=%016RX64  %sK%u=%016RX64  %sK%u=%016RX64  %sK%u=%016RX64\n",
+                                        pszPrefix, i + 0, pOpMask->aKRegs[i + 0],
+                                        pszPrefix, i + 1, pOpMask->aKRegs[i + 1],
+                                        pszPrefix, i + 2, pOpMask->aKRegs[i + 2],
+                                        pszPrefix, i + 3, pOpMask->aKRegs[i + 3]);
+                }
+
+                if (pCtx->fXStateMask & XSAVE_C_BNDREGS)
+                {
+                    PCX86XSAVEBNDREGS pBndRegs = CPUMCTX_XSAVE_C_PTR(pCtx, XSAVE_C_BNDREGS_BIT, PCX86XSAVEBNDREGS);
+                    for (unsigned i = 0; i < RT_ELEMENTS(pBndRegs->aRegs); i += 2)
+                        pHlp->pfnPrintf(pHlp, "%sBNDREG%u=%016RX64/%016RX64  %sBNDREG%u=%016RX64/%016RX64\n",
+                                        pszPrefix, i, pBndRegs->aRegs[i].uLowerBound, pBndRegs->aRegs[i].uUpperBound,
+                                        pszPrefix, i + 1, pBndRegs->aRegs[i + 1].uLowerBound, pBndRegs->aRegs[i + 1].uUpperBound);
+                }
+
+                if (pCtx->fXStateMask & XSAVE_C_BNDCSR)
+                {
+                    PCX86XSAVEBNDCFG pBndCfg = CPUMCTX_XSAVE_C_PTR(pCtx, XSAVE_C_BNDCSR_BIT, PCX86XSAVEBNDCFG);
+                    pHlp->pfnPrintf(pHlp, "%sBNDCFG.CONFIG=%016RX64 %sBNDCFG.STATUS=%016RX64\n",
+                                    pszPrefix, pBndCfg->fConfig, pszPrefix, pBndCfg->fStatus);
+                }
+
+                for (unsigned i = 0; i < RT_ELEMENTS(pFpuCtx->au32RsrvdRest); i++)
+                    if (pFpuCtx->au32RsrvdRest[i])
+                        pHlp->pfnPrintf(pHlp, "%sRsrvdRest[%u]=%RX32 (offset=%#x)\n",
+                                        pszPrefix, i, pFpuCtx->au32RsrvdRest[i], RT_UOFFSETOF_DYN(X86FXSTATE, au32RsrvdRest[i]) );
+            }
+
+            pHlp->pfnPrintf(pHlp,
+                "%sEFER         =%016RX64\n"
+                "%sPAT          =%016RX64\n"
+                "%sSTAR         =%016RX64\n"
+                "%sCSTAR        =%016RX64\n"
+                "%sLSTAR        =%016RX64\n"
+                "%sSFMASK       =%016RX64\n"
+                "%sKERNELGSBASE =%016RX64\n",
+                pszPrefix, pCtx->msrEFER,
+                pszPrefix, pCtx->msrPAT,
+                pszPrefix, pCtx->msrSTAR,
+                pszPrefix, pCtx->msrCSTAR,
+                pszPrefix, pCtx->msrLSTAR,
+                pszPrefix, pCtx->msrSFMASK,
+                pszPrefix, pCtx->msrKERNELGSBASE);
+            break;
+    }
+}
+
+
+/**
+ * Display all cpu states and any other cpum info.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) cpumR3InfoAll(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    cpumR3InfoGuest(pVM, pHlp, pszArgs);
+    cpumR3InfoGuestInstr(pVM, pHlp, pszArgs);
+    cpumR3InfoGuestHwvirt(pVM, pHlp, pszArgs);
+    cpumR3InfoHyper(pVM, pHlp, pszArgs);
+    cpumR3InfoHost(pVM, pHlp, pszArgs);
+}
+
+
+/**
+ * Parses the info argument.
+ *
+ * The argument starts with 'verbose', 'terse' or 'default' and then
+ * continues with the comment string.
+ *
+ * @param   pszArgs         The pointer to the argument string.
+ * @param   penmType        Where to store the dump type request.
+ * @param   ppszComment     Where to store the pointer to the comment string.
+ */
+static void cpumR3InfoParseArg(const char *pszArgs, CPUMDUMPTYPE *penmType, const char **ppszComment)
+{
+    if (!pszArgs)
+    {
+        *penmType = CPUMDUMPTYPE_DEFAULT;
+        *ppszComment = "";
+    }
+    else
+    {
+        if (!strncmp(pszArgs, RT_STR_TUPLE("verbose")))
+        {
+            pszArgs += 7;
+            *penmType = CPUMDUMPTYPE_VERBOSE;
+        }
+        else if (!strncmp(pszArgs, RT_STR_TUPLE("terse")))
+        {
+            pszArgs += 5;
+            *penmType = CPUMDUMPTYPE_TERSE;
+        }
+        else if (!strncmp(pszArgs, RT_STR_TUPLE("default")))
+        {
+            pszArgs += 7;
+            *penmType = CPUMDUMPTYPE_DEFAULT;
+        }
+        else
+            *penmType = CPUMDUMPTYPE_DEFAULT;
+        *ppszComment = RTStrStripL(pszArgs);
+    }
+}
+
+
+/**
+ * Display the guest cpu state.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments.
+ */
+static DECLCALLBACK(void) cpumR3InfoGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    CPUMDUMPTYPE enmType;
+    const char *pszComment;
+    cpumR3InfoParseArg(pszArgs, &enmType, &pszComment);
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    pHlp->pfnPrintf(pHlp, "Guest CPUM (VCPU %d) state: %s\n", pVCpu->idCpu, pszComment);
+
+    PCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    cpumR3InfoOne(pVM, pCtx, CPUMCTX2CORE(pCtx), pHlp, enmType, "");
+}
+
+
+/**
+ * Displays an SVM VMCB control area.
+ *
+ * @param   pHlp        The info helper functions.
+ * @param   pVmcbCtrl   Pointer to a SVM VMCB controls area.
+ * @param   pszPrefix   Caller specified string prefix.
+ */
+static void cpumR3InfoSvmVmcbCtrl(PCDBGFINFOHLP pHlp, PCSVMVMCBCTRL pVmcbCtrl, const char *pszPrefix)
+{
+    AssertReturnVoid(pHlp);
+    AssertReturnVoid(pVmcbCtrl);
+
+    pHlp->pfnPrintf(pHlp, "%sCRX-read intercepts        = %#RX16\n",    pszPrefix, pVmcbCtrl->u16InterceptRdCRx);
+    pHlp->pfnPrintf(pHlp, "%sCRX-write intercepts       = %#RX16\n",    pszPrefix, pVmcbCtrl->u16InterceptWrCRx);
+    pHlp->pfnPrintf(pHlp, "%sDRX-read intercepts        = %#RX16\n",    pszPrefix, pVmcbCtrl->u16InterceptRdDRx);
+    pHlp->pfnPrintf(pHlp, "%sDRX-write intercepts       = %#RX16\n",    pszPrefix, pVmcbCtrl->u16InterceptWrDRx);
+    pHlp->pfnPrintf(pHlp, "%sException intercepts       = %#RX32\n",    pszPrefix, pVmcbCtrl->u32InterceptXcpt);
+    pHlp->pfnPrintf(pHlp, "%sControl intercepts         = %#RX64\n",    pszPrefix, pVmcbCtrl->u64InterceptCtrl);
+    pHlp->pfnPrintf(pHlp, "%sPause-filter threshold     = %#RX16\n",    pszPrefix, pVmcbCtrl->u16PauseFilterThreshold);
+    pHlp->pfnPrintf(pHlp, "%sPause-filter count         = %#RX16\n",    pszPrefix, pVmcbCtrl->u16PauseFilterCount);
+    pHlp->pfnPrintf(pHlp, "%sIOPM bitmap physaddr       = %#RX64\n",    pszPrefix, pVmcbCtrl->u64IOPMPhysAddr);
+    pHlp->pfnPrintf(pHlp, "%sMSRPM bitmap physaddr      = %#RX64\n",    pszPrefix, pVmcbCtrl->u64MSRPMPhysAddr);
+    pHlp->pfnPrintf(pHlp, "%sTSC offset                 = %#RX64\n",    pszPrefix, pVmcbCtrl->u64TSCOffset);
+    pHlp->pfnPrintf(pHlp, "%sTLB Control\n",                            pszPrefix);
+    pHlp->pfnPrintf(pHlp, "  %sASID                       = %#RX32\n",  pszPrefix, pVmcbCtrl->TLBCtrl.n.u32ASID);
+    pHlp->pfnPrintf(pHlp, "  %sTLB-flush type             = %u\n",      pszPrefix, pVmcbCtrl->TLBCtrl.n.u8TLBFlush);
+    pHlp->pfnPrintf(pHlp, "%sInterrupt Control\n",                      pszPrefix);
+    pHlp->pfnPrintf(pHlp, "  %sVTPR                       = %#RX8 (%u)\n", pszPrefix, pVmcbCtrl->IntCtrl.n.u8VTPR, pVmcbCtrl->IntCtrl.n.u8VTPR);
+    pHlp->pfnPrintf(pHlp, "  %sVIRQ (Pending)             = %RTbool\n", pszPrefix, pVmcbCtrl->IntCtrl.n.u1VIrqPending);
+    pHlp->pfnPrintf(pHlp, "  %sVINTR vector               = %#RX8\n",   pszPrefix, pVmcbCtrl->IntCtrl.n.u8VIntrVector);
+    pHlp->pfnPrintf(pHlp, "  %sVGIF                       = %u\n",      pszPrefix, pVmcbCtrl->IntCtrl.n.u1VGif);
+    pHlp->pfnPrintf(pHlp, "  %sVINTR priority             = %#RX8\n",   pszPrefix, pVmcbCtrl->IntCtrl.n.u4VIntrPrio);
+    pHlp->pfnPrintf(pHlp, "  %sIgnore TPR                 = %RTbool\n", pszPrefix, pVmcbCtrl->IntCtrl.n.u1IgnoreTPR);
+    pHlp->pfnPrintf(pHlp, "  %sVINTR masking              = %RTbool\n", pszPrefix, pVmcbCtrl->IntCtrl.n.u1VIntrMasking);
+    pHlp->pfnPrintf(pHlp, "  %sVGIF enable                = %RTbool\n", pszPrefix, pVmcbCtrl->IntCtrl.n.u1VGifEnable);
+    pHlp->pfnPrintf(pHlp, "  %sAVIC enable                = %RTbool\n", pszPrefix, pVmcbCtrl->IntCtrl.n.u1AvicEnable);
+    pHlp->pfnPrintf(pHlp, "%sInterrupt Shadow\n",                       pszPrefix);
+    pHlp->pfnPrintf(pHlp, "  %sInterrupt shadow           = %RTbool\n", pszPrefix, pVmcbCtrl->IntShadow.n.u1IntShadow);
+    pHlp->pfnPrintf(pHlp, "  %sGuest-interrupt Mask       = %RTbool\n", pszPrefix, pVmcbCtrl->IntShadow.n.u1GuestIntMask);
+    pHlp->pfnPrintf(pHlp, "%sExit Code                  = %#RX64\n",    pszPrefix, pVmcbCtrl->u64ExitCode);
+    pHlp->pfnPrintf(pHlp, "%sEXITINFO1                  = %#RX64\n",    pszPrefix, pVmcbCtrl->u64ExitInfo1);
+    pHlp->pfnPrintf(pHlp, "%sEXITINFO2                  = %#RX64\n",    pszPrefix, pVmcbCtrl->u64ExitInfo2);
+    pHlp->pfnPrintf(pHlp, "%sExit Interrupt Info\n",                    pszPrefix);
+    pHlp->pfnPrintf(pHlp, "  %sValid                      = %RTbool\n", pszPrefix, pVmcbCtrl->ExitIntInfo.n.u1Valid);
+    pHlp->pfnPrintf(pHlp, "  %sVector                     = %#RX8 (%u)\n", pszPrefix, pVmcbCtrl->ExitIntInfo.n.u8Vector, pVmcbCtrl->ExitIntInfo.n.u8Vector);
+    pHlp->pfnPrintf(pHlp, "  %sType                       = %u\n",      pszPrefix, pVmcbCtrl->ExitIntInfo.n.u3Type);
+    pHlp->pfnPrintf(pHlp, "  %sError-code valid           = %RTbool\n", pszPrefix, pVmcbCtrl->ExitIntInfo.n.u1ErrorCodeValid);
+    pHlp->pfnPrintf(pHlp, "  %sError-code                 = %#RX32\n",  pszPrefix, pVmcbCtrl->ExitIntInfo.n.u32ErrorCode);
+    pHlp->pfnPrintf(pHlp, "%sNested paging and SEV\n",                  pszPrefix);
+    pHlp->pfnPrintf(pHlp, "  %sNested paging              = %RTbool\n", pszPrefix, pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging);
+    pHlp->pfnPrintf(pHlp, "  %sSEV (Secure Encrypted VM)  = %RTbool\n", pszPrefix, pVmcbCtrl->NestedPagingCtrl.n.u1Sev);
+    pHlp->pfnPrintf(pHlp, "  %sSEV-ES (Encrypted State)   = %RTbool\n", pszPrefix, pVmcbCtrl->NestedPagingCtrl.n.u1SevEs);
+    pHlp->pfnPrintf(pHlp, "%sEvent Inject\n",                           pszPrefix);
+    pHlp->pfnPrintf(pHlp, "  %sValid                      = %RTbool\n", pszPrefix, pVmcbCtrl->EventInject.n.u1Valid);
+    pHlp->pfnPrintf(pHlp, "  %sVector                     = %#RX32 (%u)\n", pszPrefix, pVmcbCtrl->EventInject.n.u8Vector, pVmcbCtrl->EventInject.n.u8Vector);
+    pHlp->pfnPrintf(pHlp, "  %sType                       = %u\n",      pszPrefix, pVmcbCtrl->EventInject.n.u3Type);
+    pHlp->pfnPrintf(pHlp, "  %sError-code valid           = %RTbool\n", pszPrefix, pVmcbCtrl->EventInject.n.u1ErrorCodeValid);
+    pHlp->pfnPrintf(pHlp, "  %sError-code                 = %#RX32\n",  pszPrefix, pVmcbCtrl->EventInject.n.u32ErrorCode);
+    pHlp->pfnPrintf(pHlp, "%sNested-paging CR3          = %#RX64\n",    pszPrefix, pVmcbCtrl->u64NestedPagingCR3);
+    pHlp->pfnPrintf(pHlp, "%sLBR Virtualization\n",                     pszPrefix);
+    pHlp->pfnPrintf(pHlp, "  %sLBR virt                   = %RTbool\n", pszPrefix, pVmcbCtrl->LbrVirt.n.u1LbrVirt);
+    pHlp->pfnPrintf(pHlp, "  %sVirt. VMSAVE/VMLOAD        = %RTbool\n", pszPrefix, pVmcbCtrl->LbrVirt.n.u1VirtVmsaveVmload);
+    pHlp->pfnPrintf(pHlp, "%sVMCB Clean Bits            = %#RX32\n",    pszPrefix, pVmcbCtrl->u32VmcbCleanBits);
+    pHlp->pfnPrintf(pHlp, "%sNext-RIP                   = %#RX64\n",    pszPrefix, pVmcbCtrl->u64NextRIP);
+    pHlp->pfnPrintf(pHlp, "%sInstruction bytes fetched  = %u\n",        pszPrefix, pVmcbCtrl->cbInstrFetched);
+    pHlp->pfnPrintf(pHlp, "%sInstruction bytes          = %.*Rhxs\n",   pszPrefix, sizeof(pVmcbCtrl->abInstr), pVmcbCtrl->abInstr);
+    pHlp->pfnPrintf(pHlp, "%sAVIC\n",                                   pszPrefix);
+    pHlp->pfnPrintf(pHlp, "  %sBar addr                   = %#RX64\n",  pszPrefix, pVmcbCtrl->AvicBar.n.u40Addr);
+    pHlp->pfnPrintf(pHlp, "  %sBacking page addr          = %#RX64\n",  pszPrefix, pVmcbCtrl->AvicBackingPagePtr.n.u40Addr);
+    pHlp->pfnPrintf(pHlp, "  %sLogical table addr         = %#RX64\n",  pszPrefix, pVmcbCtrl->AvicLogicalTablePtr.n.u40Addr);
+    pHlp->pfnPrintf(pHlp, "  %sPhysical table addr        = %#RX64\n",  pszPrefix, pVmcbCtrl->AvicPhysicalTablePtr.n.u40Addr);
+    pHlp->pfnPrintf(pHlp, "  %sLast guest core Id         = %u\n",      pszPrefix, pVmcbCtrl->AvicPhysicalTablePtr.n.u8LastGuestCoreId);
+}
+
+
+/**
+ * Helper for dumping the SVM VMCB selector registers.
+ *
+ * @param   pHlp        The info helper functions.
+ * @param   pSel        Pointer to the SVM selector register.
+ * @param   pszName     Name of the selector.
+ * @param   pszPrefix   Caller specified string prefix.
+ */
+DECLINLINE(void) cpumR3InfoSvmVmcbSelReg(PCDBGFINFOHLP pHlp, PCSVMSELREG pSel, const char *pszName, const char *pszPrefix)
+{
+    /* The string width of 4 used below is to handle 'LDTR'. Change later if longer register names are used. */
+    pHlp->pfnPrintf(pHlp, "%s%-4s                       = {%04x base=%016RX64 limit=%08x flags=%04x}\n", pszPrefix,
+                    pszName, pSel->u16Sel, pSel->u64Base, pSel->u32Limit, pSel->u16Attr);
+}
+
+
+/**
+ * Helper for dumping the SVM VMCB GDTR/IDTR registers.
+ *
+ * @param   pHlp        The info helper functions.
+ * @param   pXdtr       Pointer to the descriptor table register.
+ * @param   pszName     Name of the descriptor table register.
+ * @param   pszPrefix   Caller specified string prefix.
+ */
+DECLINLINE(void) cpumR3InfoSvmVmcbXdtr(PCDBGFINFOHLP pHlp, PCSVMXDTR pXdtr, const char *pszName, const char *pszPrefix)
+{
+    /* The string width of 4 used below is to cover 'GDTR', 'IDTR'. Change later if longer register names are used. */
+    pHlp->pfnPrintf(pHlp, "%s%-4s                       = %016RX64:%04x\n", pszPrefix, pszName, pXdtr->u64Base, pXdtr->u32Limit);
+}
+
+
+/**
+ * Displays an SVM VMCB state-save area.
+ *
+ * @param   pHlp            The info helper functions.
+ * @param   pVmcbStateSave  Pointer to a SVM VMCB controls area.
+ * @param   pszPrefix       Caller specified string prefix.
+ */
+static void cpumR3InfoSvmVmcbStateSave(PCDBGFINFOHLP pHlp, PCSVMVMCBSTATESAVE pVmcbStateSave, const char *pszPrefix)
+{
+    AssertReturnVoid(pHlp);
+    AssertReturnVoid(pVmcbStateSave);
+
+    char szEFlags[80];
+    cpumR3InfoFormatFlags(&szEFlags[0], pVmcbStateSave->u64RFlags);
+
+    cpumR3InfoSvmVmcbSelReg(pHlp, &pVmcbStateSave->CS,   "CS",   pszPrefix);
+    cpumR3InfoSvmVmcbSelReg(pHlp, &pVmcbStateSave->SS,   "SS",   pszPrefix);
+    cpumR3InfoSvmVmcbSelReg(pHlp, &pVmcbStateSave->ES,   "ES",   pszPrefix);
+    cpumR3InfoSvmVmcbSelReg(pHlp, &pVmcbStateSave->DS,   "DS",   pszPrefix);
+    cpumR3InfoSvmVmcbSelReg(pHlp, &pVmcbStateSave->FS,   "FS",   pszPrefix);
+    cpumR3InfoSvmVmcbSelReg(pHlp, &pVmcbStateSave->GS,   "GS",   pszPrefix);
+    cpumR3InfoSvmVmcbSelReg(pHlp, &pVmcbStateSave->LDTR, "LDTR", pszPrefix);
+    cpumR3InfoSvmVmcbSelReg(pHlp, &pVmcbStateSave->TR,   "TR",   pszPrefix);
+    cpumR3InfoSvmVmcbXdtr(pHlp, &pVmcbStateSave->GDTR,   "GDTR", pszPrefix);
+    cpumR3InfoSvmVmcbXdtr(pHlp, &pVmcbStateSave->IDTR,   "IDTR", pszPrefix);
+    pHlp->pfnPrintf(pHlp, "%sCPL                        = %u\n",     pszPrefix, pVmcbStateSave->u8CPL);
+    pHlp->pfnPrintf(pHlp, "%sEFER                       = %#RX64\n", pszPrefix, pVmcbStateSave->u64EFER);
+    pHlp->pfnPrintf(pHlp, "%sCR4                        = %#RX64\n", pszPrefix, pVmcbStateSave->u64CR4);
+    pHlp->pfnPrintf(pHlp, "%sCR3                        = %#RX64\n", pszPrefix, pVmcbStateSave->u64CR3);
+    pHlp->pfnPrintf(pHlp, "%sCR0                        = %#RX64\n", pszPrefix, pVmcbStateSave->u64CR0);
+    pHlp->pfnPrintf(pHlp, "%sDR7                        = %#RX64\n", pszPrefix, pVmcbStateSave->u64DR7);
+    pHlp->pfnPrintf(pHlp, "%sDR6                        = %#RX64\n", pszPrefix, pVmcbStateSave->u64DR6);
+    pHlp->pfnPrintf(pHlp, "%sRFLAGS                     = %#RX64 %31s\n", pszPrefix, pVmcbStateSave->u64RFlags, szEFlags);
+    pHlp->pfnPrintf(pHlp, "%sRIP                        = %#RX64\n", pszPrefix, pVmcbStateSave->u64RIP);
+    pHlp->pfnPrintf(pHlp, "%sRSP                        = %#RX64\n", pszPrefix, pVmcbStateSave->u64RSP);
+    pHlp->pfnPrintf(pHlp, "%sRAX                        = %#RX64\n", pszPrefix, pVmcbStateSave->u64RAX);
+    pHlp->pfnPrintf(pHlp, "%sSTAR                       = %#RX64\n", pszPrefix, pVmcbStateSave->u64STAR);
+    pHlp->pfnPrintf(pHlp, "%sLSTAR                      = %#RX64\n", pszPrefix, pVmcbStateSave->u64LSTAR);
+    pHlp->pfnPrintf(pHlp, "%sCSTAR                      = %#RX64\n", pszPrefix, pVmcbStateSave->u64CSTAR);
+    pHlp->pfnPrintf(pHlp, "%sSFMASK                     = %#RX64\n", pszPrefix, pVmcbStateSave->u64SFMASK);
+    pHlp->pfnPrintf(pHlp, "%sKERNELGSBASE               = %#RX64\n", pszPrefix, pVmcbStateSave->u64KernelGSBase);
+    pHlp->pfnPrintf(pHlp, "%sSysEnter CS                = %#RX64\n", pszPrefix, pVmcbStateSave->u64SysEnterCS);
+    pHlp->pfnPrintf(pHlp, "%sSysEnter EIP               = %#RX64\n", pszPrefix, pVmcbStateSave->u64SysEnterEIP);
+    pHlp->pfnPrintf(pHlp, "%sSysEnter ESP               = %#RX64\n", pszPrefix, pVmcbStateSave->u64SysEnterESP);
+    pHlp->pfnPrintf(pHlp, "%sCR2                        = %#RX64\n", pszPrefix, pVmcbStateSave->u64CR2);
+    pHlp->pfnPrintf(pHlp, "%sPAT                        = %#RX64\n", pszPrefix, pVmcbStateSave->u64PAT);
+    pHlp->pfnPrintf(pHlp, "%sDBGCTL                     = %#RX64\n", pszPrefix, pVmcbStateSave->u64DBGCTL);
+    pHlp->pfnPrintf(pHlp, "%sBR_FROM                    = %#RX64\n", pszPrefix, pVmcbStateSave->u64BR_FROM);
+    pHlp->pfnPrintf(pHlp, "%sBR_TO                      = %#RX64\n", pszPrefix, pVmcbStateSave->u64BR_TO);
+    pHlp->pfnPrintf(pHlp, "%sLASTXCPT_FROM              = %#RX64\n", pszPrefix, pVmcbStateSave->u64LASTEXCPFROM);
+    pHlp->pfnPrintf(pHlp, "%sLASTXCPT_TO                = %#RX64\n", pszPrefix, pVmcbStateSave->u64LASTEXCPTO);
+}
+
+
+/**
+ * Displays a virtual-VMCS.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pVmcs       Pointer to a virtual VMCS.
+ * @param   pszPrefix   Caller specified string prefix.
+ */
+static void cpumR3InfoVmxVmcs(PVMCPU pVCpu, PCDBGFINFOHLP pHlp, PCVMXVVMCS pVmcs, const char *pszPrefix)
+{
+    AssertReturnVoid(pHlp);
+    AssertReturnVoid(pVmcs);
+
+    /* The string width of -4 used in the macros below to cover 'LDTR', 'GDTR', 'IDTR. */
+#define CPUMVMX_DUMP_HOST_XDTR(a_pHlp, a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \
+    do { \
+        (a_pHlp)->pfnPrintf((a_pHlp), "  %s%-4s                       = {base=%016RX64}\n", \
+                            (a_pszPrefix), (a_SegName), (a_pVmcs)->u64Host##a_Seg##Base.u); \
+    } while (0)
+
+#define CPUMVMX_DUMP_HOST_FS_GS_TR(a_pHlp, a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \
+    do { \
+        (a_pHlp)->pfnPrintf((a_pHlp), "  %s%-4s                       = {%04x base=%016RX64}\n", \
+                            (a_pszPrefix), (a_SegName), (a_pVmcs)->Host##a_Seg, (a_pVmcs)->u64Host##a_Seg##Base.u); \
+    } while (0)
+
+#define CPUMVMX_DUMP_GUEST_SEGREG(a_pHlp, a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \
+    do { \
+        (a_pHlp)->pfnPrintf((a_pHlp), "  %s%-4s                       = {%04x base=%016RX64 limit=%08x flags=%04x}\n", \
+                            (a_pszPrefix), (a_SegName), (a_pVmcs)->Guest##a_Seg, (a_pVmcs)->u64Guest##a_Seg##Base.u, \
+                            (a_pVmcs)->u32Guest##a_Seg##Limit, (a_pVmcs)->u32Guest##a_Seg##Attr); \
+    } while (0)
+
+#define CPUMVMX_DUMP_GUEST_XDTR(a_pHlp, a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \
+    do { \
+        (a_pHlp)->pfnPrintf((a_pHlp), "  %s%-4s                       = {base=%016RX64 limit=%08x}\n", \
+                            (a_pszPrefix), (a_SegName), (a_pVmcs)->u64Guest##a_Seg##Base.u, (a_pVmcs)->u32Guest##a_Seg##Limit); \
+    } while (0)
+
+    /* Header. */
+    {
+        pHlp->pfnPrintf(pHlp, "%sHeader:\n", pszPrefix);
+        pHlp->pfnPrintf(pHlp, "  %sVMCS revision id           = %#RX32\n",   pszPrefix, pVmcs->u32VmcsRevId);
+        pHlp->pfnPrintf(pHlp, "  %sVMX-abort id               = %#RX32 (%s)\n", pszPrefix, pVmcs->enmVmxAbort, VMXGetAbortDesc(pVmcs->enmVmxAbort));
+        pHlp->pfnPrintf(pHlp, "  %sVMCS state                 = %#x (%s)\n", pszPrefix, pVmcs->fVmcsState, VMXGetVmcsStateDesc(pVmcs->fVmcsState));
+    }
+
+    /* Control fields. */
+    {
+        /* 16-bit. */
+        pHlp->pfnPrintf(pHlp, "%sControl:\n", pszPrefix);
+        pHlp->pfnPrintf(pHlp, "  %sVPID                       = %#RX16\n",   pszPrefix, pVmcs->u16Vpid);
+        pHlp->pfnPrintf(pHlp, "  %sPosted intr notify vector  = %#RX16\n",   pszPrefix, pVmcs->u16PostIntNotifyVector);
+        pHlp->pfnPrintf(pHlp, "  %sEPTP index                 = %#RX16\n",   pszPrefix, pVmcs->u16EptpIndex);
+
+        /* 32-bit. */
+        pHlp->pfnPrintf(pHlp, "  %sPin ctls                   = %#RX32\n",   pszPrefix, pVmcs->u32PinCtls);
+        pHlp->pfnPrintf(pHlp, "  %sProcessor ctls             = %#RX32\n",   pszPrefix, pVmcs->u32ProcCtls);
+        pHlp->pfnPrintf(pHlp, "  %sSecondary processor ctls   = %#RX32\n",   pszPrefix, pVmcs->u32ProcCtls2);
+        pHlp->pfnPrintf(pHlp, "  %sVM-exit ctls               = %#RX32\n",   pszPrefix, pVmcs->u32ExitCtls);
+        pHlp->pfnPrintf(pHlp, "  %sVM-entry ctls              = %#RX32\n",   pszPrefix, pVmcs->u32EntryCtls);
+        pHlp->pfnPrintf(pHlp, "  %sException bitmap           = %#RX32\n",   pszPrefix, pVmcs->u32XcptBitmap);
+        pHlp->pfnPrintf(pHlp, "  %sPage-fault mask            = %#RX32\n",   pszPrefix, pVmcs->u32XcptPFMask);
+        pHlp->pfnPrintf(pHlp, "  %sPage-fault match           = %#RX32\n",   pszPrefix, pVmcs->u32XcptPFMatch);
+        pHlp->pfnPrintf(pHlp, "  %sCR3-target count           = %RU32\n",    pszPrefix, pVmcs->u32Cr3TargetCount);
+        pHlp->pfnPrintf(pHlp, "  %sVM-exit MSR store count    = %RU32\n",    pszPrefix, pVmcs->u32ExitMsrStoreCount);
+        pHlp->pfnPrintf(pHlp, "  %sVM-exit MSR load count     = %RU32\n",    pszPrefix, pVmcs->u32ExitMsrLoadCount);
+        pHlp->pfnPrintf(pHlp, "  %sVM-entry MSR load count    = %RU32\n",    pszPrefix, pVmcs->u32EntryMsrLoadCount);
+        pHlp->pfnPrintf(pHlp, "  %sVM-entry interruption info = %#RX32\n",   pszPrefix, pVmcs->u32EntryIntInfo);
+        {
+            uint32_t const fInfo = pVmcs->u32EntryIntInfo;
+            uint8_t  const uType = VMX_ENTRY_INT_INFO_TYPE(fInfo);
+            pHlp->pfnPrintf(pHlp, "    %sValid                      = %RTbool\n", pszPrefix, VMX_ENTRY_INT_INFO_IS_VALID(fInfo));
+            pHlp->pfnPrintf(pHlp, "    %sType                       = %#x (%s)\n", pszPrefix, uType, VMXGetEntryIntInfoTypeDesc(uType));
+            pHlp->pfnPrintf(pHlp, "    %sVector                     = %#x\n",     pszPrefix, VMX_ENTRY_INT_INFO_VECTOR(fInfo));
+            pHlp->pfnPrintf(pHlp, "    %sNMI-unblocking-IRET        = %RTbool\n", pszPrefix, VMX_ENTRY_INT_INFO_IS_NMI_UNBLOCK_IRET(fInfo));
+            pHlp->pfnPrintf(pHlp, "    %sError-code valid           = %RTbool\n", pszPrefix, VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(fInfo));
+        }
+        pHlp->pfnPrintf(pHlp, "  %sVM-entry xcpt error-code   = %#RX32\n",   pszPrefix, pVmcs->u32EntryXcptErrCode);
+        pHlp->pfnPrintf(pHlp, "  %sVM-entry instr length      = %u byte(s)\n", pszPrefix, pVmcs->u32EntryInstrLen);
+        pHlp->pfnPrintf(pHlp, "  %sTPR threshold              = %#RX32\n",   pszPrefix, pVmcs->u32TprThreshold);
+        pHlp->pfnPrintf(pHlp, "  %sPLE gap                    = %#RX32\n",   pszPrefix, pVmcs->u32PleGap);
+        pHlp->pfnPrintf(pHlp, "  %sPLE window                 = %#RX32\n",   pszPrefix, pVmcs->u32PleWindow);
+
+        /* 64-bit. */
+        pHlp->pfnPrintf(pHlp, "  %sIO-bitmap A addr           = %#RX64\n",   pszPrefix, pVmcs->u64AddrIoBitmapA.u);
+        pHlp->pfnPrintf(pHlp, "  %sIO-bitmap B addr           = %#RX64\n",   pszPrefix, pVmcs->u64AddrIoBitmapB.u);
+        pHlp->pfnPrintf(pHlp, "  %sMSR-bitmap addr            = %#RX64\n",   pszPrefix, pVmcs->u64AddrMsrBitmap.u);
+        pHlp->pfnPrintf(pHlp, "  %sVM-exit MSR store addr     = %#RX64\n",   pszPrefix, pVmcs->u64AddrExitMsrStore.u);
+        pHlp->pfnPrintf(pHlp, "  %sVM-exit MSR load addr      = %#RX64\n",   pszPrefix, pVmcs->u64AddrExitMsrLoad.u);
+        pHlp->pfnPrintf(pHlp, "  %sVM-entry MSR load addr     = %#RX64\n",   pszPrefix, pVmcs->u64AddrEntryMsrLoad.u);
+        pHlp->pfnPrintf(pHlp, "  %sExecutive VMCS ptr         = %#RX64\n",   pszPrefix, pVmcs->u64ExecVmcsPtr.u);
+        pHlp->pfnPrintf(pHlp, "  %sPML addr                   = %#RX64\n",   pszPrefix, pVmcs->u64AddrPml.u);
+        pHlp->pfnPrintf(pHlp, "  %sTSC offset                 = %#RX64\n",   pszPrefix, pVmcs->u64TscOffset.u);
+        pHlp->pfnPrintf(pHlp, "  %sVirtual-APIC addr          = %#RX64\n",   pszPrefix, pVmcs->u64AddrVirtApic.u);
+        pHlp->pfnPrintf(pHlp, "  %sAPIC-access addr           = %#RX64\n",   pszPrefix, pVmcs->u64AddrApicAccess.u);
+        pHlp->pfnPrintf(pHlp, "  %sPosted-intr desc addr      = %#RX64\n",   pszPrefix, pVmcs->u64AddrPostedIntDesc.u);
+        pHlp->pfnPrintf(pHlp, "  %sVM-functions control       = %#RX64\n",   pszPrefix, pVmcs->u64VmFuncCtls.u);
+        pHlp->pfnPrintf(pHlp, "  %sEPTP ptr                   = %#RX64\n",   pszPrefix, pVmcs->u64EptpPtr.u);
+        pHlp->pfnPrintf(pHlp, "  %sEOI-exit bitmap 0 addr     = %#RX64\n",   pszPrefix, pVmcs->u64EoiExitBitmap0.u);
+        pHlp->pfnPrintf(pHlp, "  %sEOI-exit bitmap 1 addr     = %#RX64\n",   pszPrefix, pVmcs->u64EoiExitBitmap1.u);
+        pHlp->pfnPrintf(pHlp, "  %sEOI-exit bitmap 2 addr     = %#RX64\n",   pszPrefix, pVmcs->u64EoiExitBitmap2.u);
+        pHlp->pfnPrintf(pHlp, "  %sEOI-exit bitmap 3 addr     = %#RX64\n",   pszPrefix, pVmcs->u64EoiExitBitmap3.u);
+        pHlp->pfnPrintf(pHlp, "  %sEPTP-list addr             = %#RX64\n",   pszPrefix, pVmcs->u64AddrEptpList.u);
+        pHlp->pfnPrintf(pHlp, "  %sVMREAD-bitmap addr         = %#RX64\n",   pszPrefix, pVmcs->u64AddrVmreadBitmap.u);
+        pHlp->pfnPrintf(pHlp, "  %sVMWRITE-bitmap addr        = %#RX64\n",   pszPrefix, pVmcs->u64AddrVmwriteBitmap.u);
+        pHlp->pfnPrintf(pHlp, "  %sVirt-Xcpt info addr        = %#RX64\n",   pszPrefix, pVmcs->u64AddrXcptVeInfo.u);
+        pHlp->pfnPrintf(pHlp, "  %sXSS-bitmap                 = %#RX64\n",   pszPrefix, pVmcs->u64XssBitmap.u);
+        pHlp->pfnPrintf(pHlp, "  %sENCLS-exiting bitmap       = %#RX64\n",   pszPrefix, pVmcs->u64EnclsBitmap.u);
+        pHlp->pfnPrintf(pHlp, "  %sSPPT ptr                   = %#RX64\n",   pszPrefix, pVmcs->u64SpptPtr.u);
+        pHlp->pfnPrintf(pHlp, "  %sTSC multiplier             = %#RX64\n",   pszPrefix, pVmcs->u64TscMultiplier.u);
+
+        /* Natural width. */
+        pHlp->pfnPrintf(pHlp, "  %sCR0 guest/host mask        = %#RX64\n",   pszPrefix, pVmcs->u64Cr0Mask.u);
+        pHlp->pfnPrintf(pHlp, "  %sCR4 guest/host mask        = %#RX64\n",   pszPrefix, pVmcs->u64Cr4Mask.u);
+        pHlp->pfnPrintf(pHlp, "  %sCR0 read shadow            = %#RX64\n",   pszPrefix, pVmcs->u64Cr0ReadShadow.u);
+        pHlp->pfnPrintf(pHlp, "  %sCR4 read shadow            = %#RX64\n",   pszPrefix, pVmcs->u64Cr4ReadShadow.u);
+        pHlp->pfnPrintf(pHlp, "  %sCR3-target 0               = %#RX64\n",   pszPrefix, pVmcs->u64Cr3Target0.u);
+        pHlp->pfnPrintf(pHlp, "  %sCR3-target 1               = %#RX64\n",   pszPrefix, pVmcs->u64Cr3Target1.u);
+        pHlp->pfnPrintf(pHlp, "  %sCR3-target 2               = %#RX64\n",   pszPrefix, pVmcs->u64Cr3Target2.u);
+        pHlp->pfnPrintf(pHlp, "  %sCR3-target 3               = %#RX64\n",   pszPrefix, pVmcs->u64Cr3Target3.u);
+    }
+
+    /* Guest state. */
+    {
+        char szEFlags[80];
+        cpumR3InfoFormatFlags(&szEFlags[0], pVmcs->u64GuestRFlags.u);
+        pHlp->pfnPrintf(pHlp, "%sGuest state:\n", pszPrefix);
+
+        /* 16-bit. */
+        CPUMVMX_DUMP_GUEST_SEGREG(pHlp, pVmcs, Cs,   "cs",   pszPrefix);
+        CPUMVMX_DUMP_GUEST_SEGREG(pHlp, pVmcs, Ss,   "ss",   pszPrefix);
+        CPUMVMX_DUMP_GUEST_SEGREG(pHlp, pVmcs, Es,   "es",   pszPrefix);
+        CPUMVMX_DUMP_GUEST_SEGREG(pHlp, pVmcs, Ds,   "ds",   pszPrefix);
+        CPUMVMX_DUMP_GUEST_SEGREG(pHlp, pVmcs, Fs,   "fs",   pszPrefix);
+        CPUMVMX_DUMP_GUEST_SEGREG(pHlp, pVmcs, Gs,   "gs",   pszPrefix);
+        CPUMVMX_DUMP_GUEST_SEGREG(pHlp, pVmcs, Ldtr, "ldtr", pszPrefix);
+        CPUMVMX_DUMP_GUEST_SEGREG(pHlp, pVmcs, Tr,   "tr",   pszPrefix);
+        CPUMVMX_DUMP_GUEST_XDTR(pHlp,   pVmcs, Gdtr, "gdtr", pszPrefix);
+        CPUMVMX_DUMP_GUEST_XDTR(pHlp,   pVmcs, Idtr, "idtr", pszPrefix);
+        pHlp->pfnPrintf(pHlp, "  %sInterrupt status           = %#RX16\n",   pszPrefix, pVmcs->u16GuestIntStatus);
+        pHlp->pfnPrintf(pHlp, "  %sPML index                  = %#RX16\n",   pszPrefix, pVmcs->u16PmlIndex);
+
+        /* 32-bit. */
+        pHlp->pfnPrintf(pHlp, "  %sInterruptibility state     = %#RX32\n",   pszPrefix, pVmcs->u32GuestIntrState);
+        pHlp->pfnPrintf(pHlp, "  %sActivity state             = %#RX32\n",   pszPrefix, pVmcs->u32GuestActivityState);
+        pHlp->pfnPrintf(pHlp, "  %sSMBASE                     = %#RX32\n",   pszPrefix, pVmcs->u32GuestSmBase);
+        pHlp->pfnPrintf(pHlp, "  %sSysEnter CS                = %#RX32\n",   pszPrefix, pVmcs->u32GuestSysenterCS);
+        pHlp->pfnPrintf(pHlp, "  %sVMX-preemption timer value = %#RX32\n",   pszPrefix, pVmcs->u32PreemptTimer);
+
+        /* 64-bit. */
+        pHlp->pfnPrintf(pHlp, "  %sVMCS link ptr              = %#RX64\n",   pszPrefix, pVmcs->u64VmcsLinkPtr.u);
+        pHlp->pfnPrintf(pHlp, "  %sDBGCTL                     = %#RX64\n",   pszPrefix, pVmcs->u64GuestDebugCtlMsr.u);
+        pHlp->pfnPrintf(pHlp, "  %sPAT                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestPatMsr.u);
+        pHlp->pfnPrintf(pHlp, "  %sEFER                       = %#RX64\n",   pszPrefix, pVmcs->u64GuestEferMsr.u);
+        pHlp->pfnPrintf(pHlp, "  %sPERFGLOBALCTRL             = %#RX64\n",   pszPrefix, pVmcs->u64GuestPerfGlobalCtlMsr.u);
+        pHlp->pfnPrintf(pHlp, "  %sPDPTE 0                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestPdpte0.u);
+        pHlp->pfnPrintf(pHlp, "  %sPDPTE 1                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestPdpte1.u);
+        pHlp->pfnPrintf(pHlp, "  %sPDPTE 2                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestPdpte2.u);
+        pHlp->pfnPrintf(pHlp, "  %sPDPTE 3                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestPdpte3.u);
+        pHlp->pfnPrintf(pHlp, "  %sBNDCFGS                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestBndcfgsMsr.u);
+        pHlp->pfnPrintf(pHlp, "  %sRTIT_CTL                   = %#RX64\n",   pszPrefix, pVmcs->u64GuestRtitCtlMsr.u);
+
+        /* Natural width. */
+        pHlp->pfnPrintf(pHlp, "  %scr0                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestCr0.u);
+        pHlp->pfnPrintf(pHlp, "  %scr3                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestCr3.u);
+        pHlp->pfnPrintf(pHlp, "  %scr4                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestCr4.u);
+        pHlp->pfnPrintf(pHlp, "  %sdr7                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestDr7.u);
+        pHlp->pfnPrintf(pHlp, "  %srsp                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestRsp.u);
+        pHlp->pfnPrintf(pHlp, "  %srip                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestRip.u);
+        pHlp->pfnPrintf(pHlp, "  %srflags                     = %#RX64 %31s\n",pszPrefix, pVmcs->u64GuestRFlags.u, szEFlags);
+        pHlp->pfnPrintf(pHlp, "  %sPending debug xcpts        = %#RX64\n",   pszPrefix, pVmcs->u64GuestPendingDbgXcpts.u);
+        pHlp->pfnPrintf(pHlp, "  %sSysEnter ESP               = %#RX64\n",   pszPrefix, pVmcs->u64GuestSysenterEsp.u);
+        pHlp->pfnPrintf(pHlp, "  %sSysEnter EIP               = %#RX64\n",   pszPrefix, pVmcs->u64GuestSysenterEip.u);
+    }
+
+    /* Host state. */
+    {
+        pHlp->pfnPrintf(pHlp, "%sHost state:\n", pszPrefix);
+
+        /* 16-bit. */
+        pHlp->pfnPrintf(pHlp, "  %scs                         = %#RX16\n",   pszPrefix, pVmcs->HostCs);
+        pHlp->pfnPrintf(pHlp, "  %sss                         = %#RX16\n",   pszPrefix, pVmcs->HostSs);
+        pHlp->pfnPrintf(pHlp, "  %sds                         = %#RX16\n",   pszPrefix, pVmcs->HostDs);
+        pHlp->pfnPrintf(pHlp, "  %ses                         = %#RX16\n",   pszPrefix, pVmcs->HostEs);
+        CPUMVMX_DUMP_HOST_FS_GS_TR(pHlp, pVmcs, Fs, "fs", pszPrefix);
+        CPUMVMX_DUMP_HOST_FS_GS_TR(pHlp, pVmcs, Gs, "gs", pszPrefix);
+        CPUMVMX_DUMP_HOST_FS_GS_TR(pHlp, pVmcs, Tr, "tr", pszPrefix);
+        CPUMVMX_DUMP_HOST_XDTR(pHlp, pVmcs, Gdtr, "gdtr", pszPrefix);
+        CPUMVMX_DUMP_HOST_XDTR(pHlp, pVmcs, Idtr, "idtr", pszPrefix);
+
+        /* 32-bit. */
+        pHlp->pfnPrintf(pHlp, "  %sSysEnter CS                = %#RX32\n",   pszPrefix, pVmcs->u32HostSysenterCs);
+
+        /* 64-bit. */
+        pHlp->pfnPrintf(pHlp, "  %sEFER                       = %#RX64\n",   pszPrefix, pVmcs->u64HostEferMsr.u);
+        pHlp->pfnPrintf(pHlp, "  %sPAT                        = %#RX64\n",   pszPrefix, pVmcs->u64HostPatMsr.u);
+        pHlp->pfnPrintf(pHlp, "  %sPERFGLOBALCTRL             = %#RX64\n",   pszPrefix, pVmcs->u64HostPerfGlobalCtlMsr.u);
+
+        /* Natural width. */
+        pHlp->pfnPrintf(pHlp, "  %scr0                        = %#RX64\n",   pszPrefix, pVmcs->u64HostCr0.u);
+        pHlp->pfnPrintf(pHlp, "  %scr3                        = %#RX64\n",   pszPrefix, pVmcs->u64HostCr3.u);
+        pHlp->pfnPrintf(pHlp, "  %scr4                        = %#RX64\n",   pszPrefix, pVmcs->u64HostCr4.u);
+        pHlp->pfnPrintf(pHlp, "  %sSysEnter ESP               = %#RX64\n",   pszPrefix, pVmcs->u64HostSysenterEsp.u);
+        pHlp->pfnPrintf(pHlp, "  %sSysEnter EIP               = %#RX64\n",   pszPrefix, pVmcs->u64HostSysenterEip.u);
+        pHlp->pfnPrintf(pHlp, "  %srsp                        = %#RX64\n",   pszPrefix, pVmcs->u64HostRsp.u);
+        pHlp->pfnPrintf(pHlp, "  %srip                        = %#RX64\n",   pszPrefix, pVmcs->u64HostRip.u);
+    }
+
+    /* Read-only fields. */
+    {
+        pHlp->pfnPrintf(pHlp, "%sRead-only data fields:\n", pszPrefix);
+
+        /* 16-bit (none currently). */
+
+        /* 32-bit. */
+        pHlp->pfnPrintf(pHlp, "  %sExit reason                = %u (%s)\n",  pszPrefix, pVmcs->u32RoExitReason, HMGetVmxExitName(pVmcs->u32RoExitReason));
+        pHlp->pfnPrintf(pHlp, "  %sExit qualification         = %#RX64\n",   pszPrefix, pVmcs->u64RoExitQual.u);
+        pHlp->pfnPrintf(pHlp, "  %sVM-instruction error       = %#RX32\n",   pszPrefix, pVmcs->u32RoVmInstrError);
+        pHlp->pfnPrintf(pHlp, "  %sVM-exit intr info          = %#RX32\n",   pszPrefix, pVmcs->u32RoExitIntInfo);
+        {
+            uint32_t const fInfo = pVmcs->u32RoExitIntInfo;
+            uint8_t  const uType = VMX_EXIT_INT_INFO_TYPE(fInfo);
+            pHlp->pfnPrintf(pHlp, "    %sValid                      = %RTbool\n", pszPrefix, VMX_EXIT_INT_INFO_IS_VALID(fInfo));
+            pHlp->pfnPrintf(pHlp, "    %sType                       = %#x (%s)\n",     pszPrefix, uType, VMXGetExitIntInfoTypeDesc(uType));
+            pHlp->pfnPrintf(pHlp, "    %sVector                     = %#x\n",     pszPrefix, VMX_EXIT_INT_INFO_VECTOR(fInfo));
+            pHlp->pfnPrintf(pHlp, "    %sNMI-unblocking-IRET        = %RTbool\n", pszPrefix, VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(fInfo));
+            pHlp->pfnPrintf(pHlp, "    %sError-code valid           = %RTbool\n", pszPrefix, VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(fInfo));
+        }
+        pHlp->pfnPrintf(pHlp, "  %sVM-exit intr error-code    = %#RX32\n",   pszPrefix, pVmcs->u32RoExitIntErrCode);
+        pHlp->pfnPrintf(pHlp, "  %sIDT-vectoring info         = %#RX32\n",   pszPrefix, pVmcs->u32RoIdtVectoringInfo);
+        {
+            uint32_t const fInfo = pVmcs->u32RoIdtVectoringInfo;
+            uint8_t  const uType = VMX_IDT_VECTORING_INFO_TYPE(fInfo);
+            pHlp->pfnPrintf(pHlp, "    %sValid                      = %RTbool\n", pszPrefix, VMX_IDT_VECTORING_INFO_IS_VALID(fInfo));
+            pHlp->pfnPrintf(pHlp, "    %sType                       = %#x (%s)\n",     pszPrefix, uType, VMXGetIdtVectoringInfoTypeDesc(uType));
+            pHlp->pfnPrintf(pHlp, "    %sVector                     = %#x\n",     pszPrefix, VMX_IDT_VECTORING_INFO_VECTOR(fInfo));
+            pHlp->pfnPrintf(pHlp, "    %sError-code valid           = %RTbool\n", pszPrefix, VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(fInfo));
+        }
+        pHlp->pfnPrintf(pHlp, "  %sIDT-vectoring error-code   = %#RX32\n",   pszPrefix, pVmcs->u32RoIdtVectoringErrCode);
+        pHlp->pfnPrintf(pHlp, "  %sVM-exit instruction length = %u byte(s)\n", pszPrefix, pVmcs->u32RoExitInstrLen);
+        pHlp->pfnPrintf(pHlp, "  %sVM-exit instruction info   = %#RX64\n",   pszPrefix, pVmcs->u32RoExitInstrInfo);
+
+        /* 64-bit. */
+        pHlp->pfnPrintf(pHlp, "  %sGuest-physical addr        = %#RX64\n",   pszPrefix, pVmcs->u64RoGuestPhysAddr.u);
+
+        /* Natural width. */
+        pHlp->pfnPrintf(pHlp, "  %sI/O RCX                    = %#RX64\n",   pszPrefix, pVmcs->u64RoIoRcx.u);
+        pHlp->pfnPrintf(pHlp, "  %sI/O RSI                    = %#RX64\n",   pszPrefix, pVmcs->u64RoIoRsi.u);
+        pHlp->pfnPrintf(pHlp, "  %sI/O RDI                    = %#RX64\n",   pszPrefix, pVmcs->u64RoIoRdi.u);
+        pHlp->pfnPrintf(pHlp, "  %sI/O RIP                    = %#RX64\n",   pszPrefix, pVmcs->u64RoIoRip.u);
+        pHlp->pfnPrintf(pHlp, "  %sGuest-linear addr          = %#RX64\n",   pszPrefix, pVmcs->u64RoGuestLinearAddr.u);
+    }
+
+#ifdef DEBUG_ramshankar
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+    {
+        void *pvPage = RTMemTmpAllocZ(VMX_V_VIRT_APIC_SIZE);
+        Assert(pvPage);
+        RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pvPage, GCPhysVirtApic, VMX_V_VIRT_APIC_SIZE);
+        if (RT_SUCCESS(rc))
+        {
+            pHlp->pfnPrintf(pHlp, "  %sVirtual-APIC page\n", pszPrefix);
+            pHlp->pfnPrintf(pHlp, "%.*Rhxs\n", VMX_V_VIRT_APIC_SIZE, pvPage);
+            pHlp->pfnPrintf(pHlp, "\n");
+        }
+        RTMemTmpFree(pvPage);
+    }
+#else
+    NOREF(pVCpu);
+#endif
+
+#undef CPUMVMX_DUMP_HOST_XDTR
+#undef CPUMVMX_DUMP_HOST_FS_GS_TR
+#undef CPUMVMX_DUMP_GUEST_SEGREG
+#undef CPUMVMX_DUMP_GUEST_XDTR
+}
+
+
+/**
+ * Display the guest's hardware-virtualization cpu state.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) cpumR3InfoGuestHwvirt(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    RT_NOREF(pszArgs);
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    /*
+     * Figure out what to dump.
+     */
+    /** @todo perhaps make this configurable through pszArgs, depending on how much
+     *        noise we wish to accept when nested hwvirt. isn't used. */
+#define CPUMHWVIRTDUMP_NONE     (0)
+#define CPUMHWVIRTDUMP_SVM      RT_BIT(0)
+#define CPUMHWVIRTDUMP_VMX      RT_BIT(1)
+#define CPUMHWVIRTDUMP_COMMON   RT_BIT(2)
+#define CPUMHWVIRTDUMP_LAST     CPUMHWVIRTDUMP_VMX
+
+    PCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    static const char *const s_aHwvirtModes[] = { "No/inactive", "SVM", "VMX", "Common" };
+    bool const fSvm = pVM->cpum.s.GuestFeatures.fSvm;
+    bool const fVmx = pVM->cpum.s.GuestFeatures.fVmx;
+    uint8_t const idxHwvirtState = fSvm ? CPUMHWVIRTDUMP_SVM : (fVmx ? CPUMHWVIRTDUMP_VMX : CPUMHWVIRTDUMP_NONE);
+    AssertCompile(CPUMHWVIRTDUMP_LAST <= RT_ELEMENTS(s_aHwvirtModes));
+    Assert(idxHwvirtState < RT_ELEMENTS(s_aHwvirtModes));
+    const char *pcszHwvirtMode   = s_aHwvirtModes[idxHwvirtState];
+    uint32_t fDumpState          = idxHwvirtState | CPUMHWVIRTDUMP_COMMON;
+
+    /*
+     * Dump it.
+     */
+    pHlp->pfnPrintf(pHlp, "VCPU[%u] hardware virtualization state:\n", pVCpu->idCpu);
+
+    if (fDumpState & CPUMHWVIRTDUMP_COMMON)
+        pHlp->pfnPrintf(pHlp, "fLocalForcedActions          = %#RX32\n",  pCtx->hwvirt.fLocalForcedActions);
+
+    pHlp->pfnPrintf(pHlp, "%s hwvirt state%s\n", pcszHwvirtMode, (fDumpState & (CPUMHWVIRTDUMP_SVM | CPUMHWVIRTDUMP_VMX)) ?
+                                                                 ":" : "");
+    if (fDumpState & CPUMHWVIRTDUMP_SVM)
+    {
+        pHlp->pfnPrintf(pHlp, "  fGif                       = %RTbool\n", pCtx->hwvirt.fGif);
+
+        char szEFlags[80];
+        cpumR3InfoFormatFlags(&szEFlags[0], pCtx->hwvirt.svm.HostState.rflags.u);
+        pHlp->pfnPrintf(pHlp, "  uMsrHSavePa                = %#RX64\n",    pCtx->hwvirt.svm.uMsrHSavePa);
+        pHlp->pfnPrintf(pHlp, "  GCPhysVmcb                 = %#RGp\n",     pCtx->hwvirt.svm.GCPhysVmcb);
+        pHlp->pfnPrintf(pHlp, "  VmcbCtrl:\n");
+        cpumR3InfoSvmVmcbCtrl(pHlp, &pCtx->hwvirt.svm.pVmcbR3->ctrl,       "    " /* pszPrefix */);
+        pHlp->pfnPrintf(pHlp, "  VmcbStateSave:\n");
+        cpumR3InfoSvmVmcbStateSave(pHlp, &pCtx->hwvirt.svm.pVmcbR3->guest, "    " /* pszPrefix */);
+        pHlp->pfnPrintf(pHlp, "  HostState:\n");
+        pHlp->pfnPrintf(pHlp, "    uEferMsr                   = %#RX64\n",  pCtx->hwvirt.svm.HostState.uEferMsr);
+        pHlp->pfnPrintf(pHlp, "    uCr0                       = %#RX64\n",  pCtx->hwvirt.svm.HostState.uCr0);
+        pHlp->pfnPrintf(pHlp, "    uCr4                       = %#RX64\n",  pCtx->hwvirt.svm.HostState.uCr4);
+        pHlp->pfnPrintf(pHlp, "    uCr3                       = %#RX64\n",  pCtx->hwvirt.svm.HostState.uCr3);
+        pHlp->pfnPrintf(pHlp, "    uRip                       = %#RX64\n",  pCtx->hwvirt.svm.HostState.uRip);
+        pHlp->pfnPrintf(pHlp, "    uRsp                       = %#RX64\n",  pCtx->hwvirt.svm.HostState.uRsp);
+        pHlp->pfnPrintf(pHlp, "    uRax                       = %#RX64\n",  pCtx->hwvirt.svm.HostState.uRax);
+        pHlp->pfnPrintf(pHlp, "    rflags                     = %#RX64 %31s\n", pCtx->hwvirt.svm.HostState.rflags.u64, szEFlags);
+        PCPUMSELREG pSel = &pCtx->hwvirt.svm.HostState.es;
+        pHlp->pfnPrintf(pHlp, "    es                         = {%04x base=%016RX64 limit=%08x flags=%08x}\n",
+                        pSel->Sel, pSel->u64Base, pSel->u32Limit, pSel->Attr.u);
+        pSel = &pCtx->hwvirt.svm.HostState.cs;
+        pHlp->pfnPrintf(pHlp, "    cs                         = {%04x base=%016RX64 limit=%08x flags=%08x}\n",
+                        pSel->Sel, pSel->u64Base, pSel->u32Limit, pSel->Attr.u);
+        pSel = &pCtx->hwvirt.svm.HostState.ss;
+        pHlp->pfnPrintf(pHlp, "    ss                         = {%04x base=%016RX64 limit=%08x flags=%08x}\n",
+                        pSel->Sel, pSel->u64Base, pSel->u32Limit, pSel->Attr.u);
+        pSel = &pCtx->hwvirt.svm.HostState.ds;
+        pHlp->pfnPrintf(pHlp, "    ds                         = {%04x base=%016RX64 limit=%08x flags=%08x}\n",
+                        pSel->Sel, pSel->u64Base, pSel->u32Limit, pSel->Attr.u);
+        pHlp->pfnPrintf(pHlp, "    gdtr                       = %016RX64:%04x\n", pCtx->hwvirt.svm.HostState.gdtr.pGdt,
+                        pCtx->hwvirt.svm.HostState.gdtr.cbGdt);
+        pHlp->pfnPrintf(pHlp, "    idtr                       = %016RX64:%04x\n", pCtx->hwvirt.svm.HostState.idtr.pIdt,
+                        pCtx->hwvirt.svm.HostState.idtr.cbIdt);
+        pHlp->pfnPrintf(pHlp, "  cPauseFilter               = %RU16\n",     pCtx->hwvirt.svm.cPauseFilter);
+        pHlp->pfnPrintf(pHlp, "  cPauseFilterThreshold      = %RU32\n",     pCtx->hwvirt.svm.cPauseFilterThreshold);
+        pHlp->pfnPrintf(pHlp, "  fInterceptEvents           = %u\n",        pCtx->hwvirt.svm.fInterceptEvents);
+        pHlp->pfnPrintf(pHlp, "  pvMsrBitmapR3              = %p\n",        pCtx->hwvirt.svm.pvMsrBitmapR3);
+        pHlp->pfnPrintf(pHlp, "  pvMsrBitmapR0              = %RKv\n",      pCtx->hwvirt.svm.pvMsrBitmapR0);
+        pHlp->pfnPrintf(pHlp, "  pvIoBitmapR3               = %p\n",        pCtx->hwvirt.svm.pvIoBitmapR3);
+        pHlp->pfnPrintf(pHlp, "  pvIoBitmapR0               = %RKv\n",      pCtx->hwvirt.svm.pvIoBitmapR0);
+    }
+
+    if (fDumpState & CPUMHWVIRTDUMP_VMX)
+    {
+        pHlp->pfnPrintf(pHlp, "  GCPhysVmxon                = %#RGp\n",     pCtx->hwvirt.vmx.GCPhysVmxon);
+        pHlp->pfnPrintf(pHlp, "  GCPhysVmcs                 = %#RGp\n",     pCtx->hwvirt.vmx.GCPhysVmcs);
+        pHlp->pfnPrintf(pHlp, "  GCPhysShadowVmcs           = %#RGp\n",     pCtx->hwvirt.vmx.GCPhysShadowVmcs);
+        pHlp->pfnPrintf(pHlp, "  enmDiag                    = %u (%s)\n",   pCtx->hwvirt.vmx.enmDiag, HMGetVmxDiagDesc(pCtx->hwvirt.vmx.enmDiag));
+        pHlp->pfnPrintf(pHlp, "  uDiagAux                   = %#RX64\n",    pCtx->hwvirt.vmx.uDiagAux);
+        pHlp->pfnPrintf(pHlp, "  enmAbort                   = %u (%s)\n",   pCtx->hwvirt.vmx.enmAbort, VMXGetAbortDesc(pCtx->hwvirt.vmx.enmAbort));
+        pHlp->pfnPrintf(pHlp, "  uAbortAux                  = %u (%#x)\n",  pCtx->hwvirt.vmx.uAbortAux, pCtx->hwvirt.vmx.uAbortAux);
+        pHlp->pfnPrintf(pHlp, "  fInVmxRootMode             = %RTbool\n",   pCtx->hwvirt.vmx.fInVmxRootMode);
+        pHlp->pfnPrintf(pHlp, "  fInVmxNonRootMode          = %RTbool\n",   pCtx->hwvirt.vmx.fInVmxNonRootMode);
+        pHlp->pfnPrintf(pHlp, "  fInterceptEvents           = %RTbool\n",   pCtx->hwvirt.vmx.fInterceptEvents);
+        pHlp->pfnPrintf(pHlp, "  fNmiUnblockingIret         = %RTbool\n",   pCtx->hwvirt.vmx.fNmiUnblockingIret);
+        pHlp->pfnPrintf(pHlp, "  uFirstPauseLoopTick        = %RX64\n",     pCtx->hwvirt.vmx.uFirstPauseLoopTick);
+        pHlp->pfnPrintf(pHlp, "  uPrevPauseTick             = %RX64\n",     pCtx->hwvirt.vmx.uPrevPauseTick);
+        pHlp->pfnPrintf(pHlp, "  uEntryTick                 = %RX64\n",     pCtx->hwvirt.vmx.uEntryTick);
+        pHlp->pfnPrintf(pHlp, "  offVirtApicWrite           = %#RX16\n",    pCtx->hwvirt.vmx.offVirtApicWrite);
+        pHlp->pfnPrintf(pHlp, "  fVirtNmiBlocking           = %RTbool\n",   pCtx->hwvirt.vmx.fVirtNmiBlocking);
+        pHlp->pfnPrintf(pHlp, "  VMCS cache:\n");
+        cpumR3InfoVmxVmcs(pVCpu, pHlp, pCtx->hwvirt.vmx.pVmcsR3, "  " /* pszPrefix */);
+    }
+
+#undef CPUMHWVIRTDUMP_NONE
+#undef CPUMHWVIRTDUMP_COMMON
+#undef CPUMHWVIRTDUMP_SVM
+#undef CPUMHWVIRTDUMP_VMX
+#undef CPUMHWVIRTDUMP_LAST
+#undef CPUMHWVIRTDUMP_ALL
+}
+
+/**
+ * Display the current guest instruction
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) cpumR3InfoGuestInstr(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    char szInstruction[256];
+    szInstruction[0] = '\0';
+    DBGFR3DisasInstrCurrent(pVCpu, szInstruction, sizeof(szInstruction));
+    pHlp->pfnPrintf(pHlp, "\nCPUM%u: %s\n\n", pVCpu->idCpu, szInstruction);
+}
+
+
+/**
+ * Display the hypervisor cpu state.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) cpumR3InfoHyper(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    CPUMDUMPTYPE enmType;
+    const char *pszComment;
+    cpumR3InfoParseArg(pszArgs, &enmType, &pszComment);
+    pHlp->pfnPrintf(pHlp, "Hypervisor CPUM state: %s\n", pszComment);
+
+    pHlp->pfnPrintf(pHlp,
+                    ".dr0=%016RX64 .dr1=%016RX64 .dr2=%016RX64 .dr3=%016RX64\n"
+                    ".dr4=%016RX64 .dr5=%016RX64 .dr6=%016RX64 .dr7=%016RX64\n",
+                    pVCpu->cpum.s.Hyper.dr[0], pVCpu->cpum.s.Hyper.dr[1], pVCpu->cpum.s.Hyper.dr[2], pVCpu->cpum.s.Hyper.dr[3],
+                    pVCpu->cpum.s.Hyper.dr[4], pVCpu->cpum.s.Hyper.dr[5], pVCpu->cpum.s.Hyper.dr[6], pVCpu->cpum.s.Hyper.dr[7]);
+    pHlp->pfnPrintf(pHlp, "CR4OrMask=%#x CR4AndMask=%#x\n", pVM->cpum.s.CR4.OrMask, pVM->cpum.s.CR4.AndMask);
+}
+
+
+/**
+ * Display the host cpu state.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) cpumR3InfoHost(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    CPUMDUMPTYPE enmType;
+    const char *pszComment;
+    cpumR3InfoParseArg(pszArgs, &enmType, &pszComment);
+    pHlp->pfnPrintf(pHlp, "Host CPUM state: %s\n", pszComment);
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+    PCPUMHOSTCTX pCtx = &pVCpu->cpum.s.Host;
+
+    /*
+     * Format the EFLAGS.
+     */
+    uint64_t efl = pCtx->rflags;
+    char szEFlags[80];
+    cpumR3InfoFormatFlags(&szEFlags[0], efl);
+
+    /*
+     * Format the registers.
+     */
+    pHlp->pfnPrintf(pHlp,
+        "rax=xxxxxxxxxxxxxxxx rbx=%016RX64 rcx=xxxxxxxxxxxxxxxx\n"
+        "rdx=xxxxxxxxxxxxxxxx rsi=%016RX64 rdi=%016RX64\n"
+        "rip=xxxxxxxxxxxxxxxx rsp=%016RX64 rbp=%016RX64\n"
+        " r8=xxxxxxxxxxxxxxxx  r9=xxxxxxxxxxxxxxxx r10=%016RX64\n"
+        "r11=%016RX64 r12=%016RX64 r13=%016RX64\n"
+        "r14=%016RX64 r15=%016RX64\n"
+        "iopl=%d  %31s\n"
+        "cs=%04x  ds=%04x  es=%04x  fs=%04x  gs=%04x                   eflags=%08RX64\n"
+        "cr0=%016RX64 cr2=xxxxxxxxxxxxxxxx cr3=%016RX64\n"
+        "cr4=%016RX64 ldtr=%04x tr=%04x\n"
+        "dr[0]=%016RX64 dr[1]=%016RX64 dr[2]=%016RX64\n"
+        "dr[3]=%016RX64 dr[6]=%016RX64 dr[7]=%016RX64\n"
+        "gdtr=%016RX64:%04x  idtr=%016RX64:%04x\n"
+        "SysEnter={cs=%04x eip=%08x esp=%08x}\n"
+        "FSbase=%016RX64 GSbase=%016RX64 efer=%08RX64\n"
+        ,
+        /*pCtx->rax,*/ pCtx->rbx, /*pCtx->rcx,
+        pCtx->rdx,*/ pCtx->rsi, pCtx->rdi,
+        /*pCtx->rip,*/ pCtx->rsp, pCtx->rbp,
+        /*pCtx->r8,  pCtx->r9,*/  pCtx->r10,
+        pCtx->r11, pCtx->r12, pCtx->r13,
+        pCtx->r14, pCtx->r15,
+        X86_EFL_GET_IOPL(efl), szEFlags,
+        pCtx->cs, pCtx->ds, pCtx->es, pCtx->fs, pCtx->gs, efl,
+        pCtx->cr0, /*pCtx->cr2,*/ pCtx->cr3,
+        pCtx->cr4, pCtx->ldtr, pCtx->tr,
+        pCtx->dr0, pCtx->dr1, pCtx->dr2,
+        pCtx->dr3, pCtx->dr6, pCtx->dr7,
+        pCtx->gdtr.uAddr, pCtx->gdtr.cb, pCtx->idtr.uAddr, pCtx->idtr.cb,
+        pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp,
+        pCtx->FSbase, pCtx->GSbase, pCtx->efer);
+}
+
+/**
+ * Structure used when disassembling and instructions in DBGF.
+ * This is used so the reader function can get the stuff it needs.
+ */
+typedef struct CPUMDISASSTATE
+{
+    /** Pointer to the CPU structure. */
+    PDISCPUSTATE    pCpu;
+    /** Pointer to the VM. */
+    PVM             pVM;
+    /** Pointer to the VMCPU. */
+    PVMCPU          pVCpu;
+    /** Pointer to the first byte in the segment. */
+    RTGCUINTPTR     GCPtrSegBase;
+    /** Pointer to the byte after the end of the segment. (might have wrapped!) */
+    RTGCUINTPTR     GCPtrSegEnd;
+    /** The size of the segment minus 1. */
+    RTGCUINTPTR     cbSegLimit;
+    /** Pointer to the current page - R3 Ptr. */
+    void const     *pvPageR3;
+    /** Pointer to the current page - GC Ptr. */
+    RTGCPTR         pvPageGC;
+    /** The lock information that PGMPhysReleasePageMappingLock needs. */
+    PGMPAGEMAPLOCK  PageMapLock;
+    /** Whether the PageMapLock is valid or not. */
+    bool            fLocked;
+    /** 64 bits mode or not. */
+    bool            f64Bits;
+} CPUMDISASSTATE, *PCPUMDISASSTATE;
+
+
+/**
+ * @callback_method_impl{FNDISREADBYTES}
+ */
+static DECLCALLBACK(int) cpumR3DisasInstrRead(PDISCPUSTATE pDis, uint8_t offInstr, uint8_t cbMinRead, uint8_t cbMaxRead)
+{
+    PCPUMDISASSTATE pState = (PCPUMDISASSTATE)pDis->pvUser;
+    for (;;)
+    {
+        RTGCUINTPTR GCPtr = pDis->uInstrAddr + offInstr + pState->GCPtrSegBase;
+
+        /*
+         * Need to update the page translation?
+         */
+        if (   !pState->pvPageR3
+            || (GCPtr >> PAGE_SHIFT) != (pState->pvPageGC >> PAGE_SHIFT))
+        {
+            /* translate the address */
+            pState->pvPageGC = GCPtr & PAGE_BASE_GC_MASK;
+
+            /* Release mapping lock previously acquired. */
+            if (pState->fLocked)
+                PGMPhysReleasePageMappingLock(pState->pVM, &pState->PageMapLock);
+            int rc = PGMPhysGCPtr2CCPtrReadOnly(pState->pVCpu, pState->pvPageGC, &pState->pvPageR3, &pState->PageMapLock);
+            if (RT_SUCCESS(rc))
+                pState->fLocked  = true;
+            else
+            {
+                pState->fLocked  = false;
+                pState->pvPageR3 = NULL;
+                return rc;
+            }
+        }
+
+        /*
+         * Check the segment limit.
+         */
+        if (!pState->f64Bits && pDis->uInstrAddr + offInstr > pState->cbSegLimit)
+            return VERR_OUT_OF_SELECTOR_BOUNDS;
+
+        /*
+         * Calc how much we can read.
+         */
+        uint32_t cb = PAGE_SIZE - (GCPtr & PAGE_OFFSET_MASK);
+        if (!pState->f64Bits)
+        {
+            RTGCUINTPTR cbSeg = pState->GCPtrSegEnd - GCPtr;
+            if (cb > cbSeg && cbSeg)
+                cb = cbSeg;
+        }
+        if (cb > cbMaxRead)
+            cb = cbMaxRead;
+
+        /*
+         * Read and advance or exit.
+         */
+        memcpy(&pDis->abInstr[offInstr], (uint8_t *)pState->pvPageR3 + (GCPtr & PAGE_OFFSET_MASK), cb);
+        offInstr  += (uint8_t)cb;
+        if (cb >= cbMinRead)
+        {
+            pDis->cbCachedInstr = offInstr;
+            return VINF_SUCCESS;
+        }
+        cbMinRead -= (uint8_t)cb;
+        cbMaxRead -= (uint8_t)cb;
+    }
+}
+
+
+/**
+ * Disassemble an instruction and return the information in the provided structure.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest CPU context.
+ * @param   GCPtrPC     Program counter (relative to CS) to disassemble from.
+ * @param   pCpu        Disassembly state.
+ * @param   pszPrefix   String prefix for logging (debug only).
+ *
+ */
+VMMR3DECL(int) CPUMR3DisasmInstrCPU(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTGCPTR GCPtrPC, PDISCPUSTATE pCpu,
+                                    const char *pszPrefix)
+{
+    CPUMDISASSTATE  State;
+    int             rc;
+
+    const PGMMODE enmMode = PGMGetGuestMode(pVCpu);
+    State.pCpu            = pCpu;
+    State.pvPageGC        = 0;
+    State.pvPageR3        = NULL;
+    State.pVM             = pVM;
+    State.pVCpu           = pVCpu;
+    State.fLocked         = false;
+    State.f64Bits         = false;
+
+    /*
+     * Get selector information.
+     */
+    DISCPUMODE enmDisCpuMode;
+    if (    (pCtx->cr0 & X86_CR0_PE)
+        &&   pCtx->eflags.Bits.u1VM == 0)
+    {
+        if (!CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->cs))
+            return VERR_CPUM_HIDDEN_CS_LOAD_ERROR;
+        State.f64Bits         = enmMode >= PGMMODE_AMD64 && pCtx->cs.Attr.n.u1Long;
+        State.GCPtrSegBase    = pCtx->cs.u64Base;
+        State.GCPtrSegEnd     = pCtx->cs.u32Limit + 1 + (RTGCUINTPTR)pCtx->cs.u64Base;
+        State.cbSegLimit      = pCtx->cs.u32Limit;
+        enmDisCpuMode         = (State.f64Bits)
+                              ? DISCPUMODE_64BIT
+                              : pCtx->cs.Attr.n.u1DefBig
+                              ? DISCPUMODE_32BIT
+                              : DISCPUMODE_16BIT;
+    }
+    else
+    {
+        /* real or V86 mode */
+        enmDisCpuMode         = DISCPUMODE_16BIT;
+        State.GCPtrSegBase    = pCtx->cs.Sel * 16;
+        State.GCPtrSegEnd     = 0xFFFFFFFF;
+        State.cbSegLimit      = 0xFFFFFFFF;
+    }
+
+    /*
+     * Disassemble the instruction.
+     */
+    uint32_t cbInstr;
+#ifndef LOG_ENABLED
+    RT_NOREF_PV(pszPrefix);
+    rc = DISInstrWithReader(GCPtrPC, enmDisCpuMode, cpumR3DisasInstrRead, &State, pCpu, &cbInstr);
+    if (RT_SUCCESS(rc))
+    {
+#else
+    char szOutput[160];
+    rc = DISInstrToStrWithReader(GCPtrPC, enmDisCpuMode, cpumR3DisasInstrRead, &State,
+                                 pCpu, &cbInstr, szOutput, sizeof(szOutput));
+    if (RT_SUCCESS(rc))
+    {
+        /* log it */
+        if (pszPrefix)
+            Log(("%s-CPU%d: %s", pszPrefix, pVCpu->idCpu, szOutput));
+        else
+            Log(("%s", szOutput));
+#endif
+        rc = VINF_SUCCESS;
+    }
+    else
+        Log(("CPUMR3DisasmInstrCPU: DISInstr failed for %04X:%RGv rc=%Rrc\n", pCtx->cs.Sel, GCPtrPC, rc));
+
+    /* Release mapping lock acquired in cpumR3DisasInstrRead. */
+    if (State.fLocked)
+        PGMPhysReleasePageMappingLock(pVM, &State.PageMapLock);
+
+    return rc;
+}
+
+
+
+/**
+ * API for controlling a few of the CPU features found in CR4.
+ *
+ * Currently only X86_CR4_TSD is accepted as input.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   fOr     The CR4 OR mask.
+ * @param   fAnd    The CR4 AND mask.
+ */
+VMMR3DECL(int) CPUMR3SetCR4Feature(PVM pVM, RTHCUINTREG fOr, RTHCUINTREG fAnd)
+{
+    AssertMsgReturn(!(fOr & ~(X86_CR4_TSD)), ("%#x\n", fOr), VERR_INVALID_PARAMETER);
+    AssertMsgReturn((fAnd & ~(X86_CR4_TSD)) == ~(X86_CR4_TSD), ("%#x\n", fAnd), VERR_INVALID_PARAMETER);
+
+    pVM->cpum.s.CR4.OrMask &= fAnd;
+    pVM->cpum.s.CR4.OrMask |= fOr;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Enters REM, gets and resets the changed flags (CPUM_CHANGED_*).
+ *
+ * Only REM should ever call this function!
+ *
+ * @returns The changed flags.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   puCpl       Where to return the current privilege level (CPL).
+ */
+VMMR3DECL(uint32_t) CPUMR3RemEnter(PVMCPU pVCpu, uint32_t *puCpl)
+{
+    Assert(!pVCpu->cpum.s.fRemEntered);
+
+    /*
+     * Get the CPL first.
+     */
+    *puCpl = CPUMGetGuestCPL(pVCpu);
+
+    /*
+     * Get and reset the flags.
+     */
+    uint32_t fFlags = pVCpu->cpum.s.fChanged;
+    pVCpu->cpum.s.fChanged = 0;
+
+    /** @todo change the switcher to use the fChanged flags. */
+    if (pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_SINCE_REM)
+    {
+        fFlags |= CPUM_CHANGED_FPU_REM;
+        pVCpu->cpum.s.fUseFlags &= ~CPUM_USED_FPU_SINCE_REM;
+    }
+
+    pVCpu->cpum.s.fRemEntered = true;
+    return fFlags;
+}
+
+
+/**
+ * Leaves REM.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   fNoOutOfSyncSels    This is @c false if there are out of sync
+ *                              registers.
+ */
+VMMR3DECL(void) CPUMR3RemLeave(PVMCPU pVCpu, bool fNoOutOfSyncSels)
+{
+    Assert(pVCpu->cpum.s.fRemEntered);
+
+    RT_NOREF_PV(fNoOutOfSyncSels);
+
+    pVCpu->cpum.s.fRemEntered = false;
+}
+
+
+/**
+ * Called when the ring-3 init phase completes.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   enmWhat             Which init phase.
+ */
+VMMR3DECL(int) CPUMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    switch (enmWhat)
+    {
+        case VMINITCOMPLETED_RING3:
+        {
+            /*
+             * Figure out if the guest uses 32-bit or 64-bit FPU state at runtime for 64-bit capable VMs.
+             * Only applicable/used on 64-bit hosts, refer CPUMR0A.asm. See @bugref{7138}.
+             */
+            bool const fSupportsLongMode = VMR3IsLongModeAllowed(pVM);
+            for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+            {
+                PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+
+                /* While loading a saved-state we fix it up in, cpumR3LoadDone(). */
+                if (fSupportsLongMode)
+                    pVCpu->cpum.s.fUseFlags |= CPUM_USE_SUPPORTS_LONGMODE;
+            }
+
+            /* Register statistic counters for MSRs. */
+            cpumR3MsrRegStats(pVM);
+
+            /* Create VMX-preemption timer for nested guests if required. */
+            if (pVM->cpum.s.GuestFeatures.fVmx)
+            {
+                for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                {
+                    PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+                    char aszTimerName[128];
+                    RTStrPrintf(&aszTimerName[0], sizeof(aszTimerName), "Nested Guest VMX-preempt. timer %u", idCpu);
+                    int rc = TMR3TimerCreateInternal(pVM, TMCLOCK_VIRTUAL_SYNC, cpumR3VmxPreemptTimerCallback, pVCpu,
+                                                     aszTimerName, &pVCpu->cpum.s.pNestedVmxPreemptTimerR3);
+                    AssertLogRelRCReturn(rc, rc);
+                    pVCpu->cpum.s.pNestedVmxPreemptTimerR0 = TMTimerR0Ptr(pVCpu->cpum.s.pNestedVmxPreemptTimerR3);
+                }
+            }
+            break;
+        }
+
+        default:
+            break;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Called when the ring-0 init phases completed.
+ *
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMR3DECL(void) CPUMR3LogCpuIdAndMsrFeatures(PVM pVM)
+{
+    /*
+     * Enable log buffering as we're going to log a lot of lines.
+     */
+    bool const fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
+
+    /*
+     * Log the cpuid.
+     */
+    RTCPUSET OnlineSet;
+    LogRel(("CPUM: Logical host processors: %u present, %u max, %u online, online mask: %016RX64\n",
+                (unsigned)RTMpGetPresentCount(), (unsigned)RTMpGetCount(), (unsigned)RTMpGetOnlineCount(),
+                RTCpuSetToU64(RTMpGetOnlineSet(&OnlineSet)) ));
+    RTCPUID cCores = RTMpGetCoreCount();
+    if (cCores)
+        LogRel(("CPUM: Physical host cores: %u\n", (unsigned)cCores));
+    LogRel(("************************* CPUID dump ************************\n"));
+    DBGFR3Info(pVM->pUVM, "cpuid", "verbose", DBGFR3InfoLogRelHlp());
+    LogRel(("\n"));
+    DBGFR3_INFO_LOG_SAFE(pVM, "cpuid", "verbose"); /* macro */
+    LogRel(("******************** End of CPUID dump **********************\n"));
+
+    /*
+     * Log VT-x extended features.
+     *
+     * SVM features are currently all covered under CPUID so there is nothing
+     * to do here for SVM.
+     */
+    if (pVM->cpum.s.HostFeatures.fVmx)
+    {
+        LogRel(("*********************** VT-x features ***********************\n"));
+        DBGFR3Info(pVM->pUVM, "cpumvmxfeat", "default", DBGFR3InfoLogRelHlp());
+        LogRel(("\n"));
+        LogRel(("******************* End of VT-x features ********************\n"));
+    }
+
+    /*
+     * Restore the log buffering state to what it was previously.
+     */
+    RTLogRelSetBuffering(fOldBuffered);
+}
+
diff --git a/src/VBox/VMM/VMMR3/CPUMDbg.cpp b/src/VBox/VMM/VMMR3/CPUMDbg.cpp
new file mode 100644
index 00000000..59f63d49
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/CPUMDbg.cpp
@@ -0,0 +1,1260 @@
+/* $Id: CPUMDbg.cpp $ */
+/** @file
+ * CPUM - CPU Monitor / Manager, Debugger & Debugging APIs.
+ */
+
+/*
+ * Copyright (C) 2010-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/apic.h>
+#include "CPUMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/thread.h>
+#include <iprt/string.h>
+#include <iprt/uint128.h>
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGet_Generic(void *pvUser, PCDBGFREGDESC pDesc, PDBGFREGVAL pValue)
+{
+    PVMCPU      pVCpu   = (PVMCPU)pvUser;
+    void const *pv      = (uint8_t const *)&pVCpu->cpum + pDesc->offRegister;
+
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    switch (pDesc->enmType)
+    {
+        case DBGFREGVALTYPE_U8:        pValue->u8   = *(uint8_t  const *)pv; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U16:       pValue->u16  = *(uint16_t const *)pv; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U32:       pValue->u32  = *(uint32_t const *)pv; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U64:       pValue->u64  = *(uint64_t const *)pv; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U128:      pValue->u128 = *(PCRTUINT128U    )pv; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U256:      pValue->u256 = *(PCRTUINT256U    )pv; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U512:      pValue->u512 = *(PCRTUINT512U    )pv; return VINF_SUCCESS;
+        default:
+            AssertMsgFailedReturn(("%d %s\n", pDesc->enmType, pDesc->pszName), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnSet}
+ */
+static DECLCALLBACK(int) cpumR3RegSet_Generic(void *pvUser, PCDBGFREGDESC pDesc, PCDBGFREGVAL pValue, PCDBGFREGVAL pfMask)
+{
+    PVMCPU      pVCpu = (PVMCPU)pvUser;
+    void       *pv    = (uint8_t *)&pVCpu->cpum + pDesc->offRegister;
+
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    switch (pDesc->enmType)
+    {
+        case DBGFREGVALTYPE_U8:
+            *(uint8_t *)pv &= ~pfMask->u8;
+            *(uint8_t *)pv |= pValue->u8 & pfMask->u8;
+            return VINF_SUCCESS;
+
+        case DBGFREGVALTYPE_U16:
+            *(uint16_t *)pv &= ~pfMask->u16;
+            *(uint16_t *)pv |= pValue->u16 & pfMask->u16;
+            return VINF_SUCCESS;
+
+        case DBGFREGVALTYPE_U32:
+            *(uint32_t *)pv &= ~pfMask->u32;
+            *(uint32_t *)pv |= pValue->u32 & pfMask->u32;
+            return VINF_SUCCESS;
+
+        case DBGFREGVALTYPE_U64:
+            *(uint64_t *)pv &= ~pfMask->u64;
+            *(uint64_t *)pv |= pValue->u64 & pfMask->u64;
+            return VINF_SUCCESS;
+
+        case DBGFREGVALTYPE_U128:
+        {
+            RTUINT128U Val;
+            RTUInt128AssignAnd((PRTUINT128U)pv, RTUInt128AssignBitwiseNot(RTUInt128Assign(&Val, &pfMask->u128)));
+            RTUInt128AssignOr((PRTUINT128U)pv, RTUInt128AssignAnd(RTUInt128Assign(&Val, &pValue->u128), &pfMask->u128));
+            return VINF_SUCCESS;
+        }
+
+        default:
+            AssertMsgFailedReturn(("%d %s\n", pDesc->enmType, pDesc->pszName), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGet_XStateGeneric(void *pvUser, PCDBGFREGDESC pDesc, PDBGFREGVAL pValue)
+{
+    PVMCPU      pVCpu   = (PVMCPU)pvUser;
+    void const *pv      = (uint8_t const *)&pVCpu->cpum.s.Guest.pXStateR3 + pDesc->offRegister;
+
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    switch (pDesc->enmType)
+    {
+        case DBGFREGVALTYPE_U8:        pValue->u8   = *(uint8_t  const *)pv; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U16:       pValue->u16  = *(uint16_t const *)pv; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U32:       pValue->u32  = *(uint32_t const *)pv; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U64:       pValue->u64  = *(uint64_t const *)pv; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U128:      pValue->u128 = *(PCRTUINT128U    )pv; return VINF_SUCCESS;
+        default:
+            AssertMsgFailedReturn(("%d %s\n", pDesc->enmType, pDesc->pszName), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnSet}
+ */
+static DECLCALLBACK(int) cpumR3RegSet_XStateGeneric(void *pvUser, PCDBGFREGDESC pDesc, PCDBGFREGVAL pValue, PCDBGFREGVAL pfMask)
+{
+    PVMCPU      pVCpu = (PVMCPU)pvUser;
+    void       *pv    = (uint8_t *)&pVCpu->cpum.s.Guest.pXStateR3 + pDesc->offRegister;
+
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    switch (pDesc->enmType)
+    {
+        case DBGFREGVALTYPE_U8:
+            *(uint8_t *)pv &= ~pfMask->u8;
+            *(uint8_t *)pv |= pValue->u8 & pfMask->u8;
+            return VINF_SUCCESS;
+
+        case DBGFREGVALTYPE_U16:
+            *(uint16_t *)pv &= ~pfMask->u16;
+            *(uint16_t *)pv |= pValue->u16 & pfMask->u16;
+            return VINF_SUCCESS;
+
+        case DBGFREGVALTYPE_U32:
+            *(uint32_t *)pv &= ~pfMask->u32;
+            *(uint32_t *)pv |= pValue->u32 & pfMask->u32;
+            return VINF_SUCCESS;
+
+        case DBGFREGVALTYPE_U64:
+            *(uint64_t *)pv &= ~pfMask->u64;
+            *(uint64_t *)pv |= pValue->u64 & pfMask->u64;
+            return VINF_SUCCESS;
+
+        case DBGFREGVALTYPE_U128:
+        {
+            RTUINT128U Val;
+            RTUInt128AssignAnd((PRTUINT128U)pv, RTUInt128AssignBitwiseNot(RTUInt128Assign(&Val, &pfMask->u128)));
+            RTUInt128AssignOr((PRTUINT128U)pv, RTUInt128AssignAnd(RTUInt128Assign(&Val, &pValue->u128), &pfMask->u128));
+            return VINF_SUCCESS;
+        }
+
+        default:
+            AssertMsgFailedReturn(("%d %s\n", pDesc->enmType, pDesc->pszName), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+}
+
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegSet_seg(void *pvUser, PCDBGFREGDESC pDesc, PCDBGFREGVAL pValue, PCDBGFREGVAL pfMask)
+{
+    /** @todo perform a selector load, updating hidden selectors and stuff. */
+    NOREF(pvUser); NOREF(pDesc); NOREF(pValue); NOREF(pfMask);
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGet_gdtr(void *pvUser, PCDBGFREGDESC pDesc, PDBGFREGVAL pValue)
+{
+    PVMCPU          pVCpu = (PVMCPU)pvUser;
+    VBOXGDTR const *pGdtr = (VBOXGDTR const *)((uint8_t const *)&pVCpu->cpum + pDesc->offRegister);
+
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(pDesc->enmType == DBGFREGVALTYPE_DTR);
+
+    pValue->dtr.u32Limit  = pGdtr->cbGdt;
+    pValue->dtr.u64Base   = pGdtr->pGdt;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegSet_gdtr(void *pvUser, PCDBGFREGDESC pDesc, PCDBGFREGVAL pValue, PCDBGFREGVAL pfMask)
+{
+    NOREF(pvUser); NOREF(pDesc); NOREF(pValue); NOREF(pfMask);
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGet_idtr(void *pvUser, PCDBGFREGDESC pDesc, PDBGFREGVAL pValue)
+{
+    PVMCPU          pVCpu = (PVMCPU)pvUser;
+    VBOXIDTR const *pIdtr = (VBOXIDTR const *)((uint8_t const *)&pVCpu->cpum + pDesc->offRegister);
+
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(pDesc->enmType == DBGFREGVALTYPE_DTR);
+
+    pValue->dtr.u32Limit  = pIdtr->cbIdt;
+    pValue->dtr.u64Base   = pIdtr->pIdt;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegSet_idtr(void *pvUser, PCDBGFREGDESC pDesc, PCDBGFREGVAL pValue, PCDBGFREGVAL pfMask)
+{
+    NOREF(pvUser); NOREF(pDesc); NOREF(pValue); NOREF(pfMask);
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * Determins the tag register value for a CPU register when the FPU state
+ * format is FXSAVE.
+ *
+ * @returns The tag register value.
+ * @param   pFpu                Pointer to the guest FPU.
+ * @param   iReg                The register number (0..7).
+ */
+DECLINLINE(uint16_t) cpumR3RegCalcFpuTagFromFxSave(PCX86FXSTATE pFpu, unsigned iReg)
+{
+    /*
+     * See table 11-1 in the AMD docs.
+     */
+    if (!(pFpu->FTW & RT_BIT_32(iReg)))
+        return 3; /* b11 - empty */
+
+    uint16_t const uExp  = pFpu->aRegs[iReg].au16[4];
+    if (uExp == 0)
+    {
+        if (pFpu->aRegs[iReg].au64[0] == 0) /* J & M == 0 */
+            return 1; /* b01 - zero */
+        return 2; /* b10 - special */
+    }
+
+    if (uExp == UINT16_C(0xffff))
+        return 2; /* b10 - special */
+
+    if (!(pFpu->aRegs[iReg].au64[0] >> 63)) /* J == 0 */
+        return 2; /* b10 - special */
+
+    return 0; /* b00 - valid (normal) */
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGet_ftw(void *pvUser, PCDBGFREGDESC pDesc, PDBGFREGVAL pValue)
+{
+    PVMCPU          pVCpu   = (PVMCPU)pvUser;
+    PCX86FXSTATE    pFpu    = (PCX86FXSTATE)((uint8_t const *)&pVCpu->cpum + pDesc->offRegister);
+
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(pDesc->enmType == DBGFREGVALTYPE_U16);
+
+    pValue->u16 =  cpumR3RegCalcFpuTagFromFxSave(pFpu, 0)
+                | (cpumR3RegCalcFpuTagFromFxSave(pFpu, 1) <<  2)
+                | (cpumR3RegCalcFpuTagFromFxSave(pFpu, 2) <<  4)
+                | (cpumR3RegCalcFpuTagFromFxSave(pFpu, 3) <<  6)
+                | (cpumR3RegCalcFpuTagFromFxSave(pFpu, 4) <<  8)
+                | (cpumR3RegCalcFpuTagFromFxSave(pFpu, 5) << 10)
+                | (cpumR3RegCalcFpuTagFromFxSave(pFpu, 6) << 12)
+                | (cpumR3RegCalcFpuTagFromFxSave(pFpu, 7) << 14);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegSet_ftw(void *pvUser, PCDBGFREGDESC pDesc, PCDBGFREGVAL pValue, PCDBGFREGVAL pfMask)
+{
+    NOREF(pvUser); NOREF(pDesc); NOREF(pValue); NOREF(pfMask);
+    return VERR_DBGF_READ_ONLY_REGISTER;
+}
+
+#if 0 /* unused */
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGet_Dummy(void *pvUser, PCDBGFREGDESC pDesc, PDBGFREGVAL pValue)
+{
+    RT_NOREF_PV(pvUser);
+    switch (pDesc->enmType)
+    {
+        case DBGFREGVALTYPE_U8:        pValue->u8   = 0; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U16:       pValue->u16  = 0; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U32:       pValue->u32  = 0; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U64:       pValue->u64  = 0; return VINF_SUCCESS;
+        case DBGFREGVALTYPE_U128:
+            RT_ZERO(pValue->u128);
+            return VINF_SUCCESS;
+        case DBGFREGVALTYPE_DTR:
+            pValue->dtr.u32Limit = 0;
+            pValue->dtr.u64Base  = 0;
+            return VINF_SUCCESS;
+        case DBGFREGVALTYPE_R80:
+            RT_ZERO(pValue->r80Ex);
+            return VINF_SUCCESS;
+        default:
+            AssertMsgFailedReturn(("%d %s\n", pDesc->enmType, pDesc->pszName), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnSet}
+ */
+static DECLCALLBACK(int) cpumR3RegSet_Dummy(void *pvUser, PCDBGFREGDESC pDesc, PCDBGFREGVAL pValue, PCDBGFREGVAL pfMask)
+{
+    NOREF(pvUser); NOREF(pDesc); NOREF(pValue); NOREF(pfMask);
+    return VERR_DBGF_READ_ONLY_REGISTER;
+}
+
+#endif /* unused */
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGet_ymm(void *pvUser, PCDBGFREGDESC pDesc, PDBGFREGVAL pValue)
+{
+    PVMCPU      pVCpu   = (PVMCPU)pvUser;
+    uint32_t    iReg    = pDesc->offRegister;
+
+    Assert(pDesc->enmType == DBGFREGVALTYPE_U256);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    if (iReg < 16)
+    {
+        pValue->u256.DQWords.dqw0 = pVCpu->cpum.s.Guest.pXStateR3->x87.aXMM[iReg].uXmm;
+        pValue->u256.DQWords.dqw1 = pVCpu->cpum.s.Guest.pXStateR3->u.YmmHi.aYmmHi[iReg].uXmm;
+        return VINF_SUCCESS;
+    }
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnSet}
+ */
+static DECLCALLBACK(int) cpumR3RegSet_ymm(void *pvUser, PCDBGFREGDESC pDesc, PCDBGFREGVAL pValue, PCDBGFREGVAL pfMask)
+{
+    PVMCPU      pVCpu = (PVMCPU)pvUser;
+    uint32_t    iReg  = pDesc->offRegister;
+
+    Assert(pDesc->enmType == DBGFREGVALTYPE_U256);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    if (iReg < 16)
+    {
+        RTUINT128U Val;
+        RTUInt128AssignAnd(&pVCpu->cpum.s.Guest.pXStateR3->x87.aXMM[iReg].uXmm,
+                           RTUInt128AssignBitwiseNot(RTUInt128Assign(&Val, &pfMask->u256.DQWords.dqw0)));
+        RTUInt128AssignOr(&pVCpu->cpum.s.Guest.pXStateR3->u.YmmHi.aYmmHi[iReg].uXmm,
+                          RTUInt128AssignAnd(RTUInt128Assign(&Val, &pValue->u128), &pfMask->u128));
+
+    }
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/*
+ *
+ * Guest register access functions.
+ *
+ */
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGstGet_crX(void *pvUser, PCDBGFREGDESC pDesc, PDBGFREGVAL pValue)
+{
+    PVMCPU pVCpu = (PVMCPU)pvUser;
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    uint64_t u64Value;
+    int rc = CPUMGetGuestCRx(pVCpu, pDesc->offRegister, &u64Value);
+    if (rc == VERR_PDM_NO_APIC_INSTANCE) /* CR8 might not be available, see @bugref{8868}.*/
+        u64Value = 0;
+    else
+        AssertRCReturn(rc, rc);
+    switch (pDesc->enmType)
+    {
+        case DBGFREGVALTYPE_U64:    pValue->u64 = u64Value; break;
+        case DBGFREGVALTYPE_U32:    pValue->u32 = (uint32_t)u64Value; break;
+        default:
+            AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGstSet_crX(void *pvUser, PCDBGFREGDESC pDesc, PCDBGFREGVAL pValue, PCDBGFREGVAL pfMask)
+{
+    int         rc;
+    PVMCPU      pVCpu = (PVMCPU)pvUser;
+
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Calculate the new value.
+     */
+    uint64_t u64Value;
+    uint64_t fMask;
+    uint64_t fMaskMax;
+    switch (pDesc->enmType)
+    {
+        case DBGFREGVALTYPE_U64:
+            u64Value = pValue->u64;
+            fMask    = pfMask->u64;
+            fMaskMax = UINT64_MAX;
+            break;
+        case DBGFREGVALTYPE_U32:
+            u64Value = pValue->u32;
+            fMask    = pfMask->u32;
+            fMaskMax = UINT32_MAX;
+            break;
+        default:
+            AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+    if (fMask != fMaskMax)
+    {
+        uint64_t u64FullValue;
+        rc = CPUMGetGuestCRx(pVCpu, pDesc->offRegister, &u64FullValue);
+        if (RT_FAILURE(rc))
+            return rc;
+        u64Value = (u64FullValue & ~fMask)
+                 | (u64Value     &  fMask);
+    }
+
+    /*
+     * Perform the assignment.
+     */
+    switch (pDesc->offRegister)
+    {
+        case 0: rc = CPUMSetGuestCR0(pVCpu, u64Value); break;
+        case 2: rc = CPUMSetGuestCR2(pVCpu, u64Value); break;
+        case 3: rc = CPUMSetGuestCR3(pVCpu, u64Value); break;
+        case 4: rc = CPUMSetGuestCR4(pVCpu, u64Value); break;
+        case 8: rc = APICSetTpr(pVCpu, (uint8_t)(u64Value << 4)); break;
+        default:
+            AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+    return rc;
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGstGet_drX(void *pvUser, PCDBGFREGDESC pDesc, PDBGFREGVAL pValue)
+{
+    PVMCPU pVCpu = (PVMCPU)pvUser;
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    uint64_t u64Value;
+    int rc = CPUMGetGuestDRx(pVCpu, pDesc->offRegister, &u64Value);
+    AssertRCReturn(rc, rc);
+    switch (pDesc->enmType)
+    {
+        case DBGFREGVALTYPE_U64:    pValue->u64 = u64Value; break;
+        case DBGFREGVALTYPE_U32:    pValue->u32 = (uint32_t)u64Value; break;
+        default:
+            AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGstSet_drX(void *pvUser, PCDBGFREGDESC pDesc, PCDBGFREGVAL pValue, PCDBGFREGVAL pfMask)
+{
+    int         rc;
+    PVMCPU      pVCpu = (PVMCPU)pvUser;
+
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Calculate the new value.
+     */
+    uint64_t u64Value;
+    uint64_t fMask;
+    uint64_t fMaskMax;
+    switch (pDesc->enmType)
+    {
+        case DBGFREGVALTYPE_U64:
+            u64Value = pValue->u64;
+            fMask    = pfMask->u64;
+            fMaskMax = UINT64_MAX;
+            break;
+        case DBGFREGVALTYPE_U32:
+            u64Value = pValue->u32;
+            fMask    = pfMask->u32;
+            fMaskMax = UINT32_MAX;
+            break;
+        default:
+            AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+    if (fMask != fMaskMax)
+    {
+        uint64_t u64FullValue;
+        rc = CPUMGetGuestDRx(pVCpu, pDesc->offRegister, &u64FullValue);
+        if (RT_FAILURE(rc))
+            return rc;
+        u64Value = (u64FullValue & ~fMask)
+                 | (u64Value     &  fMask);
+    }
+
+    /*
+     * Perform the assignment.
+     */
+    return CPUMSetGuestDRx(pVCpu, pDesc->offRegister, u64Value);
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGstGet_msr(void *pvUser, PCDBGFREGDESC pDesc, PDBGFREGVAL pValue)
+{
+    PVMCPU pVCpu = (PVMCPU)pvUser;
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    uint64_t u64Value;
+    VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pDesc->offRegister, &u64Value);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        switch (pDesc->enmType)
+        {
+            case DBGFREGVALTYPE_U64:    pValue->u64 = u64Value; break;
+            case DBGFREGVALTYPE_U32:    pValue->u32 = (uint32_t)u64Value; break;
+            case DBGFREGVALTYPE_U16:    pValue->u16 = (uint16_t)u64Value; break;
+            default:
+                AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+        }
+        return VBOXSTRICTRC_VAL(rcStrict);
+    }
+
+    /** @todo what to do about errors? */
+    Assert(RT_FAILURE_NP(rcStrict));
+    return VBOXSTRICTRC_VAL(rcStrict);
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGstSet_msr(void *pvUser, PCDBGFREGDESC pDesc, PCDBGFREGVAL pValue, PCDBGFREGVAL pfMask)
+{
+    PVMCPU pVCpu = (PVMCPU)pvUser;
+
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Calculate the new value.
+     */
+    uint64_t u64Value;
+    uint64_t fMask;
+    uint64_t fMaskMax;
+    switch (pDesc->enmType)
+    {
+        case DBGFREGVALTYPE_U64:
+            u64Value = pValue->u64;
+            fMask    = pfMask->u64;
+            fMaskMax = UINT64_MAX;
+            break;
+        case DBGFREGVALTYPE_U32:
+            u64Value = pValue->u32;
+            fMask    = pfMask->u32;
+            fMaskMax = UINT32_MAX;
+            break;
+        case DBGFREGVALTYPE_U16:
+            u64Value = pValue->u16;
+            fMask    = pfMask->u16;
+            fMaskMax = UINT16_MAX;
+            break;
+        default:
+            AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+    if (fMask != fMaskMax)
+    {
+        uint64_t u64FullValue;
+        VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pDesc->offRegister, &u64FullValue);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            AssertRC(RT_FAILURE_NP(rcStrict));
+            return VBOXSTRICTRC_VAL(rcStrict);
+        }
+        u64Value = (u64FullValue & ~fMask)
+                 | (u64Value     &  fMask);
+    }
+
+    /*
+     * Perform the assignment.
+     */
+    VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, pDesc->offRegister, u64Value);
+    if (rcStrict == VINF_SUCCESS)
+        return VINF_SUCCESS;
+    AssertRC(RT_FAILURE_NP(rcStrict));
+    return VBOXSTRICTRC_VAL(rcStrict);
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGstGet_stN(void *pvUser, PCDBGFREGDESC pDesc, PDBGFREGVAL pValue)
+{
+    PVMCPU pVCpu = (PVMCPU)pvUser;
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(pDesc->enmType == DBGFREGVALTYPE_R80);
+
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.s.Guest.CTX_SUFF(pXState)->x87;
+    unsigned iReg = (pFpuCtx->FSW >> 11) & 7;
+    iReg += pDesc->offRegister;
+    iReg &= 7;
+    pValue->r80Ex = pFpuCtx->aRegs[iReg].r80Ex;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @interface_method_impl{DBGFREGDESC,pfnGet}
+ */
+static DECLCALLBACK(int) cpumR3RegGstSet_stN(void *pvUser, PCDBGFREGDESC pDesc, PCDBGFREGVAL pValue, PCDBGFREGVAL pfMask)
+{
+    NOREF(pvUser); NOREF(pDesc); NOREF(pValue); NOREF(pfMask);
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+
+/*
+ * Set up aliases.
+ */
+#define CPUMREGALIAS_STD(Name, psz32, psz16, psz8)  \
+    static DBGFREGALIAS const g_aCpumRegAliases_##Name[] = \
+    { \
+        { psz32, DBGFREGVALTYPE_U32     }, \
+        { psz16, DBGFREGVALTYPE_U16     }, \
+        { psz8,  DBGFREGVALTYPE_U8      }, \
+        { NULL,  DBGFREGVALTYPE_INVALID } \
+    }
+CPUMREGALIAS_STD(rax,  "eax",   "ax",   "al");
+CPUMREGALIAS_STD(rcx,  "ecx",   "cx",   "cl");
+CPUMREGALIAS_STD(rdx,  "edx",   "dx",   "dl");
+CPUMREGALIAS_STD(rbx,  "ebx",   "bx",   "bl");
+CPUMREGALIAS_STD(rsp,  "esp",   "sp",   NULL);
+CPUMREGALIAS_STD(rbp,  "ebp",   "bp",   NULL);
+CPUMREGALIAS_STD(rsi,  "esi",   "si",  "sil");
+CPUMREGALIAS_STD(rdi,  "edi",   "di",  "dil");
+CPUMREGALIAS_STD(r8,   "r8d",  "r8w",  "r8b");
+CPUMREGALIAS_STD(r9,   "r9d",  "r9w",  "r9b");
+CPUMREGALIAS_STD(r10, "r10d", "r10w", "r10b");
+CPUMREGALIAS_STD(r11, "r11d", "r11w", "r11b");
+CPUMREGALIAS_STD(r12, "r12d", "r12w", "r12b");
+CPUMREGALIAS_STD(r13, "r13d", "r13w", "r13b");
+CPUMREGALIAS_STD(r14, "r14d", "r14w", "r14b");
+CPUMREGALIAS_STD(r15, "r15d", "r15w", "r15b");
+CPUMREGALIAS_STD(rip, "eip",   "ip",    NULL);
+CPUMREGALIAS_STD(rflags, "eflags", "flags", NULL);
+#undef CPUMREGALIAS_STD
+
+static DBGFREGALIAS const g_aCpumRegAliases_fpuip[] =
+{
+    { "fpuip16", DBGFREGVALTYPE_U16  },
+    { NULL, DBGFREGVALTYPE_INVALID }
+};
+
+static DBGFREGALIAS const g_aCpumRegAliases_fpudp[] =
+{
+    { "fpudp16", DBGFREGVALTYPE_U16  },
+    { NULL, DBGFREGVALTYPE_INVALID }
+};
+
+static DBGFREGALIAS const g_aCpumRegAliases_cr0[] =
+{
+    { "msw", DBGFREGVALTYPE_U16  },
+    { NULL, DBGFREGVALTYPE_INVALID }
+};
+
+/*
+ * Sub fields.
+ */
+/** Sub-fields for the (hidden) segment attribute register. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_seg[] =
+{
+    DBGFREGSUBFIELD_RW("type",   0,   4,  0),
+    DBGFREGSUBFIELD_RW("s",      4,   1,  0),
+    DBGFREGSUBFIELD_RW("dpl",    5,   2,  0),
+    DBGFREGSUBFIELD_RW("p",      7,   1,  0),
+    DBGFREGSUBFIELD_RW("avl",   12,   1,  0),
+    DBGFREGSUBFIELD_RW("l",     13,   1,  0),
+    DBGFREGSUBFIELD_RW("d",     14,   1,  0),
+    DBGFREGSUBFIELD_RW("g",     15,   1,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the flags register. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_rflags[] =
+{
+    DBGFREGSUBFIELD_RW("cf",     0,   1,  0),
+    DBGFREGSUBFIELD_RW("pf",     2,   1,  0),
+    DBGFREGSUBFIELD_RW("af",     4,   1,  0),
+    DBGFREGSUBFIELD_RW("zf",     6,   1,  0),
+    DBGFREGSUBFIELD_RW("sf",     7,   1,  0),
+    DBGFREGSUBFIELD_RW("tf",     8,   1,  0),
+    DBGFREGSUBFIELD_RW("if",     9,   1,  0),
+    DBGFREGSUBFIELD_RW("df",    10,   1,  0),
+    DBGFREGSUBFIELD_RW("of",    11,   1,  0),
+    DBGFREGSUBFIELD_RW("iopl",  12,   2,  0),
+    DBGFREGSUBFIELD_RW("nt",    14,   1,  0),
+    DBGFREGSUBFIELD_RW("rf",    16,   1,  0),
+    DBGFREGSUBFIELD_RW("vm",    17,   1,  0),
+    DBGFREGSUBFIELD_RW("ac",    18,   1,  0),
+    DBGFREGSUBFIELD_RW("vif",   19,   1,  0),
+    DBGFREGSUBFIELD_RW("vip",   20,   1,  0),
+    DBGFREGSUBFIELD_RW("id",    21,   1,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the FPU control word register. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_fcw[] =
+{
+    DBGFREGSUBFIELD_RW("im",     1,   1,  0),
+    DBGFREGSUBFIELD_RW("dm",     2,   1,  0),
+    DBGFREGSUBFIELD_RW("zm",     3,   1,  0),
+    DBGFREGSUBFIELD_RW("om",     4,   1,  0),
+    DBGFREGSUBFIELD_RW("um",     5,   1,  0),
+    DBGFREGSUBFIELD_RW("pm",     6,   1,  0),
+    DBGFREGSUBFIELD_RW("pc",     8,   2,  0),
+    DBGFREGSUBFIELD_RW("rc",    10,   2,  0),
+    DBGFREGSUBFIELD_RW("x",     12,   1,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the FPU status word register. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_fsw[] =
+{
+    DBGFREGSUBFIELD_RW("ie",     0,   1,  0),
+    DBGFREGSUBFIELD_RW("de",     1,   1,  0),
+    DBGFREGSUBFIELD_RW("ze",     2,   1,  0),
+    DBGFREGSUBFIELD_RW("oe",     3,   1,  0),
+    DBGFREGSUBFIELD_RW("ue",     4,   1,  0),
+    DBGFREGSUBFIELD_RW("pe",     5,   1,  0),
+    DBGFREGSUBFIELD_RW("se",     6,   1,  0),
+    DBGFREGSUBFIELD_RW("es",     7,   1,  0),
+    DBGFREGSUBFIELD_RW("c0",     8,   1,  0),
+    DBGFREGSUBFIELD_RW("c1",     9,   1,  0),
+    DBGFREGSUBFIELD_RW("c2",    10,   1,  0),
+    DBGFREGSUBFIELD_RW("top",   11,   3,  0),
+    DBGFREGSUBFIELD_RW("c3",    14,   1,  0),
+    DBGFREGSUBFIELD_RW("b",     15,   1,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the FPU tag word register. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_ftw[] =
+{
+    DBGFREGSUBFIELD_RW("tag0",   0,   2,  0),
+    DBGFREGSUBFIELD_RW("tag1",   2,   2,  0),
+    DBGFREGSUBFIELD_RW("tag2",   4,   2,  0),
+    DBGFREGSUBFIELD_RW("tag3",   6,   2,  0),
+    DBGFREGSUBFIELD_RW("tag4",   8,   2,  0),
+    DBGFREGSUBFIELD_RW("tag5",  10,   2,  0),
+    DBGFREGSUBFIELD_RW("tag6",  12,   2,  0),
+    DBGFREGSUBFIELD_RW("tag7",  14,   2,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the Multimedia Extensions Control and Status Register. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_mxcsr[] =
+{
+    DBGFREGSUBFIELD_RW("ie",     0,   1,  0),
+    DBGFREGSUBFIELD_RW("de",     1,   1,  0),
+    DBGFREGSUBFIELD_RW("ze",     2,   1,  0),
+    DBGFREGSUBFIELD_RW("oe",     3,   1,  0),
+    DBGFREGSUBFIELD_RW("ue",     4,   1,  0),
+    DBGFREGSUBFIELD_RW("pe",     5,   1,  0),
+    DBGFREGSUBFIELD_RW("daz",    6,   1,  0),
+    DBGFREGSUBFIELD_RW("im",     7,   1,  0),
+    DBGFREGSUBFIELD_RW("dm",     8,   1,  0),
+    DBGFREGSUBFIELD_RW("zm",     9,   1,  0),
+    DBGFREGSUBFIELD_RW("om",    10,   1,  0),
+    DBGFREGSUBFIELD_RW("um",    11,   1,  0),
+    DBGFREGSUBFIELD_RW("pm",    12,   1,  0),
+    DBGFREGSUBFIELD_RW("rc",    13,   2,  0),
+    DBGFREGSUBFIELD_RW("fz",    14,   1,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the FPU tag word register. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_stN[] =
+{
+    DBGFREGSUBFIELD_RW("man",    0,  64,  0),
+    DBGFREGSUBFIELD_RW("exp",   64,  15,  0),
+    DBGFREGSUBFIELD_RW("sig",   79,   1,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the MMX registers. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_mmN[] =
+{
+    DBGFREGSUBFIELD_RW("dw0",    0,  32,  0),
+    DBGFREGSUBFIELD_RW("dw1",   32,  32,  0),
+    DBGFREGSUBFIELD_RW("w0",     0,  16,  0),
+    DBGFREGSUBFIELD_RW("w1",    16,  16,  0),
+    DBGFREGSUBFIELD_RW("w2",    32,  16,  0),
+    DBGFREGSUBFIELD_RW("w3",    48,  16,  0),
+    DBGFREGSUBFIELD_RW("b0",     0,   8,  0),
+    DBGFREGSUBFIELD_RW("b1",     8,   8,  0),
+    DBGFREGSUBFIELD_RW("b2",    16,   8,  0),
+    DBGFREGSUBFIELD_RW("b3",    24,   8,  0),
+    DBGFREGSUBFIELD_RW("b4",    32,   8,  0),
+    DBGFREGSUBFIELD_RW("b5",    40,   8,  0),
+    DBGFREGSUBFIELD_RW("b6",    48,   8,  0),
+    DBGFREGSUBFIELD_RW("b7",    56,   8,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the XMM registers. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_xmmN[] =
+{
+    DBGFREGSUBFIELD_RW("r0",      0,     32,  0),
+    DBGFREGSUBFIELD_RW("r0.man",  0+ 0,  23,  0),
+    DBGFREGSUBFIELD_RW("r0.exp",  0+23,   8,  0),
+    DBGFREGSUBFIELD_RW("r0.sig",  0+31,   1,  0),
+    DBGFREGSUBFIELD_RW("r1",     32,     32,  0),
+    DBGFREGSUBFIELD_RW("r1.man", 32+ 0,  23,  0),
+    DBGFREGSUBFIELD_RW("r1.exp", 32+23,   8,  0),
+    DBGFREGSUBFIELD_RW("r1.sig", 32+31,   1,  0),
+    DBGFREGSUBFIELD_RW("r2",     64,     32,  0),
+    DBGFREGSUBFIELD_RW("r2.man", 64+ 0,  23,  0),
+    DBGFREGSUBFIELD_RW("r2.exp", 64+23,   8,  0),
+    DBGFREGSUBFIELD_RW("r2.sig", 64+31,   1,  0),
+    DBGFREGSUBFIELD_RW("r3",     96,     32,  0),
+    DBGFREGSUBFIELD_RW("r3.man", 96+ 0,  23,  0),
+    DBGFREGSUBFIELD_RW("r3.exp", 96+23,   8,  0),
+    DBGFREGSUBFIELD_RW("r3.sig", 96+31,   1,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+#if 0 /* needs special accessor, too lazy for that now. */
+/** Sub-fields for the YMM registers. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_ymmN[] =
+{
+    DBGFREGSUBFIELD_RW("r0",       0,     32,  0),
+    DBGFREGSUBFIELD_RW("r0.man",   0+ 0,  23,  0),
+    DBGFREGSUBFIELD_RW("r0.exp",   0+23,   8,  0),
+    DBGFREGSUBFIELD_RW("r0.sig",   0+31,   1,  0),
+    DBGFREGSUBFIELD_RW("r1",      32,     32,  0),
+    DBGFREGSUBFIELD_RW("r1.man",  32+ 0,  23,  0),
+    DBGFREGSUBFIELD_RW("r1.exp",  32+23,   8,  0),
+    DBGFREGSUBFIELD_RW("r1.sig",  32+31,   1,  0),
+    DBGFREGSUBFIELD_RW("r2",      64,     32,  0),
+    DBGFREGSUBFIELD_RW("r2.man",  64+ 0,  23,  0),
+    DBGFREGSUBFIELD_RW("r2.exp",  64+23,   8,  0),
+    DBGFREGSUBFIELD_RW("r2.sig",  64+31,   1,  0),
+    DBGFREGSUBFIELD_RW("r3",      96,     32,  0),
+    DBGFREGSUBFIELD_RW("r3.man",  96+ 0,  23,  0),
+    DBGFREGSUBFIELD_RW("r3.exp",  96+23,   8,  0),
+    DBGFREGSUBFIELD_RW("r3.sig",  96+31,   1,  0),
+    DBGFREGSUBFIELD_RW("r4",     128,     32,  0),
+    DBGFREGSUBFIELD_RW("r4.man", 128+ 0,  23,  0),
+    DBGFREGSUBFIELD_RW("r4.exp", 128+23,   8,  0),
+    DBGFREGSUBFIELD_RW("r4.sig", 128+31,   1,  0),
+    DBGFREGSUBFIELD_RW("r5",     160,     32,  0),
+    DBGFREGSUBFIELD_RW("r5.man", 160+ 0,  23,  0),
+    DBGFREGSUBFIELD_RW("r5.exp", 160+23,   8,  0),
+    DBGFREGSUBFIELD_RW("r5.sig", 160+31,   1,  0),
+    DBGFREGSUBFIELD_RW("r6",     192,     32,  0),
+    DBGFREGSUBFIELD_RW("r6.man", 192+ 0,  23,  0),
+    DBGFREGSUBFIELD_RW("r6.exp", 192+23,   8,  0),
+    DBGFREGSUBFIELD_RW("r6.sig", 192+31,   1,  0),
+    DBGFREGSUBFIELD_RW("r7",     224,     32,  0),
+    DBGFREGSUBFIELD_RW("r7.man", 224+ 0,  23,  0),
+    DBGFREGSUBFIELD_RW("r7.exp", 224+23,   8,  0),
+    DBGFREGSUBFIELD_RW("r7.sig", 224+31,   1,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+#endif
+
+/** Sub-fields for the CR0 register. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_cr0[] =
+{
+    DBGFREGSUBFIELD_RW("pe",      0,      1,  0),
+    DBGFREGSUBFIELD_RW("mp",      1,      1,  0),
+    DBGFREGSUBFIELD_RW("em",      2,      1,  0),
+    DBGFREGSUBFIELD_RW("ts",      3,      1,  0),
+    DBGFREGSUBFIELD_RO("et",      4,      1,  0),
+    DBGFREGSUBFIELD_RW("ne",      5,      1,  0),
+    DBGFREGSUBFIELD_RW("wp",     16,      1,  0),
+    DBGFREGSUBFIELD_RW("am",     18,      1,  0),
+    DBGFREGSUBFIELD_RW("nw",     29,      1,  0),
+    DBGFREGSUBFIELD_RW("cd",     30,      1,  0),
+    DBGFREGSUBFIELD_RW("pg",     31,      1,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the CR3 register. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_cr3[] =
+{
+    DBGFREGSUBFIELD_RW("pwt",     3,      1,  0),
+    DBGFREGSUBFIELD_RW("pcd",     4,      1,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the CR4 register. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_cr4[] =
+{
+    DBGFREGSUBFIELD_RW("vme",     0,      1,  0),
+    DBGFREGSUBFIELD_RW("pvi",     1,      1,  0),
+    DBGFREGSUBFIELD_RW("tsd",     2,      1,  0),
+    DBGFREGSUBFIELD_RW("de",      3,      1,  0),
+    DBGFREGSUBFIELD_RW("pse",     4,      1,  0),
+    DBGFREGSUBFIELD_RW("pae",     5,      1,  0),
+    DBGFREGSUBFIELD_RW("mce",     6,      1,  0),
+    DBGFREGSUBFIELD_RW("pge",     7,      1,  0),
+    DBGFREGSUBFIELD_RW("pce",     8,      1,  0),
+    DBGFREGSUBFIELD_RW("osfxsr",  9,      1,  0),
+    DBGFREGSUBFIELD_RW("osxmmeexcpt", 10, 1,  0),
+    DBGFREGSUBFIELD_RW("vmxe",   13,      1,  0),
+    DBGFREGSUBFIELD_RW("smxe",   14,      1,  0),
+    DBGFREGSUBFIELD_RW("pcide",  17,      1,  0),
+    DBGFREGSUBFIELD_RW("osxsave", 18,     1,  0),
+    DBGFREGSUBFIELD_RW("smep",   20,      1,  0),
+    DBGFREGSUBFIELD_RW("smap",   21,      1,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the DR6 register. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_dr6[] =
+{
+    DBGFREGSUBFIELD_RW("b0",      0,      1,  0),
+    DBGFREGSUBFIELD_RW("b1",      1,      1,  0),
+    DBGFREGSUBFIELD_RW("b2",      2,      1,  0),
+    DBGFREGSUBFIELD_RW("b3",      3,      1,  0),
+    DBGFREGSUBFIELD_RW("bd",     13,      1,  0),
+    DBGFREGSUBFIELD_RW("bs",     14,      1,  0),
+    DBGFREGSUBFIELD_RW("bt",     15,      1,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the DR7 register. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_dr7[] =
+{
+    DBGFREGSUBFIELD_RW("l0",      0,      1,  0),
+    DBGFREGSUBFIELD_RW("g0",      1,      1,  0),
+    DBGFREGSUBFIELD_RW("l1",      2,      1,  0),
+    DBGFREGSUBFIELD_RW("g1",      3,      1,  0),
+    DBGFREGSUBFIELD_RW("l2",      4,      1,  0),
+    DBGFREGSUBFIELD_RW("g2",      5,      1,  0),
+    DBGFREGSUBFIELD_RW("l3",      6,      1,  0),
+    DBGFREGSUBFIELD_RW("g3",      7,      1,  0),
+    DBGFREGSUBFIELD_RW("le",      8,      1,  0),
+    DBGFREGSUBFIELD_RW("ge",      9,      1,  0),
+    DBGFREGSUBFIELD_RW("gd",     13,      1,  0),
+    DBGFREGSUBFIELD_RW("rw0",    16,      2,  0),
+    DBGFREGSUBFIELD_RW("len0",   18,      2,  0),
+    DBGFREGSUBFIELD_RW("rw1",    20,      2,  0),
+    DBGFREGSUBFIELD_RW("len1",   22,      2,  0),
+    DBGFREGSUBFIELD_RW("rw2",    24,      2,  0),
+    DBGFREGSUBFIELD_RW("len2",   26,      2,  0),
+    DBGFREGSUBFIELD_RW("rw3",    28,      2,  0),
+    DBGFREGSUBFIELD_RW("len3",   30,      2,  0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the CR_PAT MSR. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_apic_base[] =
+{
+    DBGFREGSUBFIELD_RW("bsp",     8,      1,  0),
+    DBGFREGSUBFIELD_RW("ge",      9,      1,  0),
+    DBGFREGSUBFIELD_RW("base",    12,    20, 12),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the CR_PAT MSR. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_cr_pat[] =
+{
+    /** @todo  */
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the PERF_STATUS MSR. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_perf_status[] =
+{
+    /** @todo  */
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the EFER MSR. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_efer[] =
+{
+    /** @todo  */
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the STAR MSR. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_star[] =
+{
+    /** @todo  */
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the CSTAR MSR. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_cstar[] =
+{
+    /** @todo  */
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** Sub-fields for the LSTAR MSR. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_lstar[] =
+{
+    /** @todo  */
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+#if 0 /** @todo */
+/** Sub-fields for the SF_MASK MSR. */
+static DBGFREGSUBFIELD const g_aCpumRegFields_sf_mask[] =
+{
+    /** @todo  */
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+#endif
+
+
+/** @name Macros for producing register descriptor table entries.
+ * @{ */
+#define CPU_REG_EX_AS(a_szName, a_RegSuff, a_TypeSuff, a_offRegister, a_pfnGet, a_pfnSet, a_paAliases, a_paSubFields) \
+    { a_szName, DBGFREG_##a_RegSuff, DBGFREGVALTYPE_##a_TypeSuff, 0 /*fFlags*/, a_offRegister, a_pfnGet, a_pfnSet, a_paAliases, a_paSubFields }
+
+#define CPU_REG_REG(UName, LName) \
+    CPU_REG_RW_AS(#LName,           UName,          U64, LName,                 cpumR3RegGet_Generic, cpumR3RegSet_Generic, g_aCpumRegAliases_##LName,  NULL)
+
+#define CPU_REG_SEG(UName, LName) \
+    CPU_REG_RW_AS(#LName,           UName,          U16, LName.Sel,             cpumR3RegGet_Generic, cpumR3RegSet_seg,     NULL,                       NULL                ), \
+    CPU_REG_RW_AS(#LName "_attr",   UName##_ATTR,   U32, LName.Attr.u,          cpumR3RegGet_Generic, cpumR3RegSet_Generic, NULL,                       g_aCpumRegFields_seg), \
+    CPU_REG_RW_AS(#LName "_base",   UName##_BASE,   U64, LName.u64Base,         cpumR3RegGet_Generic, cpumR3RegSet_Generic, NULL,                       NULL                ), \
+    CPU_REG_RW_AS(#LName "_lim",    UName##_LIMIT,  U32, LName.u32Limit,        cpumR3RegGet_Generic, cpumR3RegSet_Generic, NULL,                       NULL                )
+
+#define CPU_REG_MM(n) \
+    CPU_REG_XS_RW_AS("mm" #n,       MM##n,          U64, x87.aRegs[n].mmx, cpumR3RegGet_XStateGeneric, cpumR3RegSet_XStateGeneric, NULL,                g_aCpumRegFields_mmN)
+
+#define CPU_REG_XMM(n) \
+    CPU_REG_XS_RW_AS("xmm" #n,      XMM##n,         U128, x87.aXMM[n].xmm, cpumR3RegGet_XStateGeneric, cpumR3RegSet_XStateGeneric, NULL,                g_aCpumRegFields_xmmN)
+
+#define CPU_REG_YMM(n) \
+    { "ymm" #n, DBGFREG_YMM##n, DBGFREGVALTYPE_U256, 0 /*fFlags*/, n,   cpumR3RegGet_ymm, cpumR3RegSet_ymm, NULL /*paAliases*/, NULL /*paSubFields*/ }
+
+/** @} */
+
+
+/**
+ * The guest register descriptors.
+ */
+static DBGFREGDESC const g_aCpumRegGstDescs[] =
+{
+#define CPU_REG_RW_AS(a_szName, a_RegSuff, a_TypeSuff, a_CpumCtxMemb, a_pfnGet, a_pfnSet, a_paAliases, a_paSubFields) \
+    { a_szName, DBGFREG_##a_RegSuff, DBGFREGVALTYPE_##a_TypeSuff, 0 /*fFlags*/,            RT_OFFSETOF(CPUMCPU, Guest.a_CpumCtxMemb), a_pfnGet, a_pfnSet, a_paAliases, a_paSubFields }
+#define CPU_REG_RO_AS(a_szName, a_RegSuff, a_TypeSuff, a_CpumCtxMemb, a_pfnGet, a_pfnSet, a_paAliases, a_paSubFields) \
+    { a_szName, DBGFREG_##a_RegSuff, DBGFREGVALTYPE_##a_TypeSuff, DBGFREG_FLAGS_READ_ONLY, RT_OFFSETOF(CPUMCPU, Guest.a_CpumCtxMemb), a_pfnGet, a_pfnSet, a_paAliases, a_paSubFields }
+#define CPU_REG_XS_RW_AS(a_szName, a_RegSuff, a_TypeSuff, a_XStateMemb, a_pfnGet, a_pfnSet, a_paAliases, a_paSubFields) \
+    { a_szName, DBGFREG_##a_RegSuff, DBGFREGVALTYPE_##a_TypeSuff, 0 /*fFlags*/,            RT_OFFSETOF(X86XSAVEAREA, a_XStateMemb),   a_pfnGet, a_pfnSet, a_paAliases, a_paSubFields }
+#define CPU_REG_XS_RO_AS(a_szName, a_RegSuff, a_TypeSuff, a_XStateMemb, a_pfnGet, a_pfnSet, a_paAliases, a_paSubFields) \
+    { a_szName, DBGFREG_##a_RegSuff, DBGFREGVALTYPE_##a_TypeSuff, DBGFREG_FLAGS_READ_ONLY, RT_OFFSETOF(X86XSAVEAREA, a_XStateMemb), a_pfnGet, a_pfnSet, a_paAliases, a_paSubFields }
+#define CPU_REG_MSR(a_szName, UName, a_TypeSuff, a_paSubFields) \
+    CPU_REG_EX_AS(a_szName,         MSR_##UName,    a_TypeSuff, MSR_##UName,    cpumR3RegGstGet_msr,  cpumR3RegGstSet_msr,  NULL,                       a_paSubFields)
+#define CPU_REG_ST(n) \
+    CPU_REG_EX_AS("st" #n,          ST##n,          R80, n,                     cpumR3RegGstGet_stN,  cpumR3RegGstSet_stN,  NULL,                       g_aCpumRegFields_stN)
+
+    CPU_REG_REG(RAX, rax),
+    CPU_REG_REG(RCX, rcx),
+    CPU_REG_REG(RDX, rdx),
+    CPU_REG_REG(RBX, rbx),
+    CPU_REG_REG(RSP, rsp),
+    CPU_REG_REG(RBP, rbp),
+    CPU_REG_REG(RSI, rsi),
+    CPU_REG_REG(RDI, rdi),
+    CPU_REG_REG(R8,   r8),
+    CPU_REG_REG(R9,   r9),
+    CPU_REG_REG(R10, r10),
+    CPU_REG_REG(R11, r11),
+    CPU_REG_REG(R12, r12),
+    CPU_REG_REG(R13, r13),
+    CPU_REG_REG(R14, r14),
+    CPU_REG_REG(R15, r15),
+    CPU_REG_SEG(CS, cs),
+    CPU_REG_SEG(DS, ds),
+    CPU_REG_SEG(ES, es),
+    CPU_REG_SEG(FS, fs),
+    CPU_REG_SEG(GS, gs),
+    CPU_REG_SEG(SS, ss),
+    CPU_REG_REG(RIP, rip),
+    CPU_REG_RW_AS("rflags",         RFLAGS,         U64, rflags,         cpumR3RegGet_Generic,         cpumR3RegSet_Generic,         g_aCpumRegAliases_rflags,   g_aCpumRegFields_rflags ),
+    CPU_REG_XS_RW_AS("fcw",         FCW,            U16, x87.FCW,        cpumR3RegGet_XStateGeneric,   cpumR3RegSet_XStateGeneric,   NULL,                       g_aCpumRegFields_fcw    ),
+    CPU_REG_XS_RW_AS("fsw",         FSW,            U16, x87.FSW,        cpumR3RegGet_XStateGeneric,   cpumR3RegSet_XStateGeneric,   NULL,                       g_aCpumRegFields_fsw    ),
+    CPU_REG_XS_RO_AS("ftw",         FTW,            U16, x87,            cpumR3RegGet_ftw,             cpumR3RegSet_ftw,             NULL,                       g_aCpumRegFields_ftw    ),
+    CPU_REG_XS_RW_AS("fop",         FOP,            U16, x87.FOP,        cpumR3RegGet_XStateGeneric,   cpumR3RegSet_XStateGeneric,   NULL,                       NULL                    ),
+    CPU_REG_XS_RW_AS("fpuip",       FPUIP,          U32, x87.FPUIP,      cpumR3RegGet_XStateGeneric,   cpumR3RegSet_XStateGeneric,   g_aCpumRegAliases_fpuip,    NULL                    ),
+    CPU_REG_XS_RW_AS("fpucs",       FPUCS,          U16, x87.CS,         cpumR3RegGet_XStateGeneric,   cpumR3RegSet_XStateGeneric,   NULL,                       NULL                    ),
+    CPU_REG_XS_RW_AS("fpudp",       FPUDP,          U32, x87.FPUDP,      cpumR3RegGet_XStateGeneric,   cpumR3RegSet_XStateGeneric,   g_aCpumRegAliases_fpudp,    NULL                    ),
+    CPU_REG_XS_RW_AS("fpuds",       FPUDS,          U16, x87.DS,         cpumR3RegGet_XStateGeneric,   cpumR3RegSet_XStateGeneric,   NULL,                       NULL                    ),
+    CPU_REG_XS_RW_AS("mxcsr",       MXCSR,          U32, x87.MXCSR,      cpumR3RegGet_XStateGeneric,   cpumR3RegSet_XStateGeneric,   NULL,                       g_aCpumRegFields_mxcsr  ),
+    CPU_REG_XS_RW_AS("mxcsr_mask",  MXCSR_MASK,     U32, x87.MXCSR_MASK, cpumR3RegGet_XStateGeneric,   cpumR3RegSet_XStateGeneric,   NULL,                       g_aCpumRegFields_mxcsr  ),
+    CPU_REG_ST(0),
+    CPU_REG_ST(1),
+    CPU_REG_ST(2),
+    CPU_REG_ST(3),
+    CPU_REG_ST(4),
+    CPU_REG_ST(5),
+    CPU_REG_ST(6),
+    CPU_REG_ST(7),
+    CPU_REG_MM(0),
+    CPU_REG_MM(1),
+    CPU_REG_MM(2),
+    CPU_REG_MM(3),
+    CPU_REG_MM(4),
+    CPU_REG_MM(5),
+    CPU_REG_MM(6),
+    CPU_REG_MM(7),
+    CPU_REG_XMM(0),
+    CPU_REG_XMM(1),
+    CPU_REG_XMM(2),
+    CPU_REG_XMM(3),
+    CPU_REG_XMM(4),
+    CPU_REG_XMM(5),
+    CPU_REG_XMM(6),
+    CPU_REG_XMM(7),
+    CPU_REG_XMM(8),
+    CPU_REG_XMM(9),
+    CPU_REG_XMM(10),
+    CPU_REG_XMM(11),
+    CPU_REG_XMM(12),
+    CPU_REG_XMM(13),
+    CPU_REG_XMM(14),
+    CPU_REG_XMM(15),
+    CPU_REG_YMM(0),
+    CPU_REG_YMM(1),
+    CPU_REG_YMM(2),
+    CPU_REG_YMM(3),
+    CPU_REG_YMM(4),
+    CPU_REG_YMM(5),
+    CPU_REG_YMM(6),
+    CPU_REG_YMM(7),
+    CPU_REG_YMM(8),
+    CPU_REG_YMM(9),
+    CPU_REG_YMM(10),
+    CPU_REG_YMM(11),
+    CPU_REG_YMM(12),
+    CPU_REG_YMM(13),
+    CPU_REG_YMM(14),
+    CPU_REG_YMM(15),
+    CPU_REG_RW_AS("gdtr_base",      GDTR_BASE,      U64, gdtr.pGdt,             cpumR3RegGet_Generic,   cpumR3RegSet_Generic,   NULL,                       NULL                    ),
+    CPU_REG_RW_AS("gdtr_lim",       GDTR_LIMIT,     U16, gdtr.cbGdt,            cpumR3RegGet_Generic,   cpumR3RegSet_Generic,   NULL,                       NULL                    ),
+    CPU_REG_RW_AS("idtr_base",      IDTR_BASE,      U64, idtr.pIdt,             cpumR3RegGet_Generic,   cpumR3RegSet_Generic,   NULL,                       NULL                    ),
+    CPU_REG_RW_AS("idtr_lim",       IDTR_LIMIT,     U16, idtr.cbIdt,            cpumR3RegGet_Generic,   cpumR3RegSet_Generic,   NULL,                       NULL                    ),
+    CPU_REG_SEG(LDTR, ldtr),
+    CPU_REG_SEG(TR, tr),
+    CPU_REG_EX_AS("cr0",            CR0,            U32, 0,                     cpumR3RegGstGet_crX,    cpumR3RegGstSet_crX,    g_aCpumRegAliases_cr0,      g_aCpumRegFields_cr0    ),
+    CPU_REG_EX_AS("cr2",            CR2,            U64, 2,                     cpumR3RegGstGet_crX,    cpumR3RegGstSet_crX,    NULL,                       NULL                    ),
+    CPU_REG_EX_AS("cr3",            CR3,            U64, 3,                     cpumR3RegGstGet_crX,    cpumR3RegGstSet_crX,    NULL,                       g_aCpumRegFields_cr3    ),
+    CPU_REG_EX_AS("cr4",            CR4,            U32, 4,                     cpumR3RegGstGet_crX,    cpumR3RegGstSet_crX,    NULL,                       g_aCpumRegFields_cr4    ),
+    CPU_REG_EX_AS("cr8",            CR8,            U32, 8,                     cpumR3RegGstGet_crX,    cpumR3RegGstSet_crX,    NULL,                       NULL                    ),
+    CPU_REG_EX_AS("dr0",            DR0,            U64, 0,                     cpumR3RegGstGet_drX,    cpumR3RegGstSet_drX,    NULL,                       NULL                    ),
+    CPU_REG_EX_AS("dr1",            DR1,            U64, 1,                     cpumR3RegGstGet_drX,    cpumR3RegGstSet_drX,    NULL,                       NULL                    ),
+    CPU_REG_EX_AS("dr2",            DR2,            U64, 2,                     cpumR3RegGstGet_drX,    cpumR3RegGstSet_drX,    NULL,                       NULL                    ),
+    CPU_REG_EX_AS("dr3",            DR3,            U64, 3,                     cpumR3RegGstGet_drX,    cpumR3RegGstSet_drX,    NULL,                       NULL                    ),
+    CPU_REG_EX_AS("dr6",            DR6,            U32, 6,                     cpumR3RegGstGet_drX,    cpumR3RegGstSet_drX,    NULL,                       g_aCpumRegFields_dr6    ),
+    CPU_REG_EX_AS("dr7",            DR7,            U32, 7,                     cpumR3RegGstGet_drX,    cpumR3RegGstSet_drX,    NULL,                       g_aCpumRegFields_dr7    ),
+    CPU_REG_MSR("apic_base",     IA32_APICBASE,     U32, g_aCpumRegFields_apic_base  ),
+    CPU_REG_MSR("pat",           IA32_CR_PAT,       U64, g_aCpumRegFields_cr_pat     ),
+    CPU_REG_MSR("perf_status",   IA32_PERF_STATUS,  U64, g_aCpumRegFields_perf_status),
+    CPU_REG_MSR("sysenter_cs",   IA32_SYSENTER_CS,  U16, NULL                        ),
+    CPU_REG_MSR("sysenter_eip",  IA32_SYSENTER_EIP, U32, NULL                        ),
+    CPU_REG_MSR("sysenter_esp",  IA32_SYSENTER_ESP, U32, NULL                        ),
+    CPU_REG_MSR("tsc",           IA32_TSC,          U32, NULL                        ),
+    CPU_REG_MSR("efer",          K6_EFER,           U32, g_aCpumRegFields_efer       ),
+    CPU_REG_MSR("star",          K6_STAR,           U64, g_aCpumRegFields_star       ),
+    CPU_REG_MSR("cstar",         K8_CSTAR,          U64, g_aCpumRegFields_cstar      ),
+    CPU_REG_MSR("msr_fs_base",   K8_FS_BASE,        U64, NULL                        ),
+    CPU_REG_MSR("msr_gs_base",   K8_GS_BASE,        U64, NULL                        ),
+    CPU_REG_MSR("krnl_gs_base",  K8_KERNEL_GS_BASE, U64, NULL                        ),
+    CPU_REG_MSR("lstar",         K8_LSTAR,          U64, g_aCpumRegFields_lstar      ),
+    CPU_REG_MSR("sf_mask",       K8_SF_MASK,        U64, NULL                        ),
+    CPU_REG_MSR("tsc_aux",       K8_TSC_AUX,        U64, NULL                        ),
+    CPU_REG_EX_AS("ah",             AH,             U8,  RT_OFFSETOF(CPUMCPU, Guest.rax) + 1, cpumR3RegGet_Generic, cpumR3RegSet_Generic, NULL,             NULL                    ),
+    CPU_REG_EX_AS("ch",             CH,             U8,  RT_OFFSETOF(CPUMCPU, Guest.rcx) + 1, cpumR3RegGet_Generic, cpumR3RegSet_Generic, NULL,             NULL                    ),
+    CPU_REG_EX_AS("dh",             DH,             U8,  RT_OFFSETOF(CPUMCPU, Guest.rdx) + 1, cpumR3RegGet_Generic, cpumR3RegSet_Generic, NULL,             NULL                    ),
+    CPU_REG_EX_AS("bh",             BH,             U8,  RT_OFFSETOF(CPUMCPU, Guest.rbx) + 1, cpumR3RegGet_Generic, cpumR3RegSet_Generic, NULL,             NULL                    ),
+    CPU_REG_RW_AS("gdtr",           GDTR,           DTR, gdtr,                  cpumR3RegGet_gdtr,        cpumR3RegSet_gdtr,    NULL,                       NULL                    ),
+    CPU_REG_RW_AS("idtr",           IDTR,           DTR, idtr,                  cpumR3RegGet_idtr,        cpumR3RegSet_idtr,    NULL,                       NULL                    ),
+    DBGFREGDESC_TERMINATOR()
+
+#undef CPU_REG_RW_AS
+#undef CPU_REG_RO_AS
+#undef CPU_REG_MSR
+#undef CPU_REG_ST
+};
+
+
+/**
+ * Initializes the debugger related sides of the CPUM component.
+ *
+ * Called by CPUMR3Init.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ */
+int cpumR3DbgInit(PVM pVM)
+{
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        int rc = DBGFR3RegRegisterCpu(pVM, pVM->apCpusR3[idCpu], g_aCpumRegGstDescs, true /*fGuestRegs*/);
+        AssertLogRelRCReturn(rc, rc);
+    }
+
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMR3/CPUMR3CpuId.cpp b/src/VBox/VMM/VMMR3/CPUMR3CpuId.cpp
new file mode 100644
index 00000000..267dc604
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/CPUMR3CpuId.cpp
@@ -0,0 +1,7232 @@
+/* $Id: CPUMR3CpuId.cpp $ */
+/** @file
+ * CPUM - CPU ID part.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_CPUM
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/ssm.h>
+#include "CPUMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/sup.h>
+
+#include <VBox/err.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/ctype.h>
+#include <iprt/mem.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** For sanity and avoid wasting hyper heap on buggy config / saved state. */
+#define CPUM_CPUID_MAX_LEAVES       2048
+/* Max size we accept for the XSAVE area. */
+#define CPUM_MAX_XSAVE_AREA_SIZE    10240
+/* Min size we accept for the XSAVE area. */
+#define CPUM_MIN_XSAVE_AREA_SIZE    0x240
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/**
+ * The intel pentium family.
+ */
+static const CPUMMICROARCH g_aenmIntelFamily06[] =
+{
+    /* [ 0(0x00)] = */ kCpumMicroarch_Intel_P6,           /* Pentium Pro A-step (says sandpile.org). */
+    /* [ 1(0x01)] = */ kCpumMicroarch_Intel_P6,           /* Pentium Pro */
+    /* [ 2(0x02)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [ 3(0x03)] = */ kCpumMicroarch_Intel_P6_II,        /* PII Klamath */
+    /* [ 4(0x04)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [ 5(0x05)] = */ kCpumMicroarch_Intel_P6_II,        /* PII Deschutes */
+    /* [ 6(0x06)] = */ kCpumMicroarch_Intel_P6_II,        /* Celeron Mendocino. */
+    /* [ 7(0x07)] = */ kCpumMicroarch_Intel_P6_III,       /* PIII Katmai. */
+    /* [ 8(0x08)] = */ kCpumMicroarch_Intel_P6_III,       /* PIII Coppermine (includes Celeron). */
+    /* [ 9(0x09)] = */ kCpumMicroarch_Intel_P6_M_Banias,  /* Pentium/Celeron M Banias. */
+    /* [10(0x0a)] = */ kCpumMicroarch_Intel_P6_III,       /* PIII Xeon */
+    /* [11(0x0b)] = */ kCpumMicroarch_Intel_P6_III,       /* PIII Tualatin (includes Celeron). */
+    /* [12(0x0c)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [13(0x0d)] = */ kCpumMicroarch_Intel_P6_M_Dothan,  /* Pentium/Celeron M Dothan. */
+    /* [14(0x0e)] = */ kCpumMicroarch_Intel_Core_Yonah,   /* Core Yonah (Enhanced Pentium M). */
+    /* [15(0x0f)] = */ kCpumMicroarch_Intel_Core2_Merom,  /* Merom */
+    /* [16(0x10)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [17(0x11)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [18(0x12)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [19(0x13)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [20(0x14)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [21(0x15)] = */ kCpumMicroarch_Intel_P6_M_Dothan,  /* Tolapai - System-on-a-chip. */
+    /* [22(0x16)] = */ kCpumMicroarch_Intel_Core2_Merom,
+    /* [23(0x17)] = */ kCpumMicroarch_Intel_Core2_Penryn,
+    /* [24(0x18)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [25(0x19)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [26(0x1a)] = */ kCpumMicroarch_Intel_Core7_Nehalem, /* Nehalem-EP */
+    /* [27(0x1b)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [28(0x1c)] = */ kCpumMicroarch_Intel_Atom_Bonnell, /* Diamonville, Pineview, */
+    /* [29(0x1d)] = */ kCpumMicroarch_Intel_Core2_Penryn,
+    /* [30(0x1e)] = */ kCpumMicroarch_Intel_Core7_Nehalem, /* Clarksfield, Lynnfield, Jasper Forest. */
+    /* [31(0x1f)] = */ kCpumMicroarch_Intel_Core7_Nehalem, /* Only listed by sandpile.org.  2 cores ABD/HVD, whatever that means. */
+    /* [32(0x20)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [33(0x21)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [34(0x22)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [35(0x23)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [36(0x24)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [37(0x25)] = */ kCpumMicroarch_Intel_Core7_Westmere, /* Arrandale, Clarksdale. */
+    /* [38(0x26)] = */ kCpumMicroarch_Intel_Atom_Lincroft,
+    /* [39(0x27)] = */ kCpumMicroarch_Intel_Atom_Saltwell,
+    /* [40(0x28)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [41(0x29)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [42(0x2a)] = */ kCpumMicroarch_Intel_Core7_SandyBridge,
+    /* [43(0x2b)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [44(0x2c)] = */ kCpumMicroarch_Intel_Core7_Westmere, /* Gulftown, Westmere-EP. */
+    /* [45(0x2d)] = */ kCpumMicroarch_Intel_Core7_SandyBridge, /* SandyBridge-E, SandyBridge-EN, SandyBridge-EP. */
+    /* [46(0x2e)] = */ kCpumMicroarch_Intel_Core7_Nehalem,  /* Beckton (Xeon). */
+    /* [47(0x2f)] = */ kCpumMicroarch_Intel_Core7_Westmere, /* Westmere-EX. */
+    /* [48(0x30)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [49(0x31)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [50(0x32)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [51(0x33)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [52(0x34)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [53(0x35)] = */ kCpumMicroarch_Intel_Atom_Saltwell, /* ?? */
+    /* [54(0x36)] = */ kCpumMicroarch_Intel_Atom_Saltwell, /* Cedarview, ++ */
+    /* [55(0x37)] = */ kCpumMicroarch_Intel_Atom_Silvermont,
+    /* [56(0x38)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [57(0x39)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [58(0x3a)] = */ kCpumMicroarch_Intel_Core7_IvyBridge,
+    /* [59(0x3b)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [60(0x3c)] = */ kCpumMicroarch_Intel_Core7_Haswell,
+    /* [61(0x3d)] = */ kCpumMicroarch_Intel_Core7_Broadwell,
+    /* [62(0x3e)] = */ kCpumMicroarch_Intel_Core7_IvyBridge,
+    /* [63(0x3f)] = */ kCpumMicroarch_Intel_Core7_Haswell,
+    /* [64(0x40)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [65(0x41)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [66(0x42)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [67(0x43)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [68(0x44)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [69(0x45)] = */ kCpumMicroarch_Intel_Core7_Haswell,
+    /* [70(0x46)] = */ kCpumMicroarch_Intel_Core7_Haswell,
+    /* [71(0x47)] = */ kCpumMicroarch_Intel_Core7_Broadwell,    /* i7-5775C */
+    /* [72(0x48)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [73(0x49)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [74(0x4a)] = */ kCpumMicroarch_Intel_Atom_Silvermont,
+    /* [75(0x4b)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [76(0x4c)] = */ kCpumMicroarch_Intel_Atom_Airmount,
+    /* [77(0x4d)] = */ kCpumMicroarch_Intel_Atom_Silvermont,
+    /* [78(0x4e)] = */ kCpumMicroarch_Intel_Core7_Skylake,
+    /* [79(0x4f)] = */ kCpumMicroarch_Intel_Core7_Broadwell,    /* Broadwell-E */
+    /* [80(0x50)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [81(0x51)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [82(0x52)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [83(0x53)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [84(0x54)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [85(0x55)] = */ kCpumMicroarch_Intel_Core7_Skylake,      /* server cpu; skylake <= 4, cascade lake > 5 */
+    /* [86(0x56)] = */ kCpumMicroarch_Intel_Core7_Broadwell,    /* Xeon D-1540, Broadwell-DE */
+    /* [87(0x57)] = */ kCpumMicroarch_Intel_Phi_KnightsLanding,
+    /* [88(0x58)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [89(0x59)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [90(0x5a)] = */ kCpumMicroarch_Intel_Atom_Silvermont,    /* Moorefield */
+    /* [91(0x5b)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [92(0x5c)] = */ kCpumMicroarch_Intel_Atom_Goldmont,      /* Apollo Lake */
+    /* [93(0x5d)] = */ kCpumMicroarch_Intel_Atom_Silvermont,    /* x3-C3230 */
+    /* [94(0x5e)] = */ kCpumMicroarch_Intel_Core7_Skylake,      /* i7-6700K */
+    /* [95(0x5f)] = */ kCpumMicroarch_Intel_Atom_Goldmont,      /* Denverton */
+    /* [96(0x60)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [97(0x61)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [98(0x62)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [99(0x63)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[100(0x64)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[101(0x65)] = */ kCpumMicroarch_Intel_Atom_Silvermont,    /* SoFIA */
+    /*[102(0x66)] = */ kCpumMicroarch_Intel_Core7_CannonLake, /* unconfirmed */
+    /*[103(0x67)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[104(0x68)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[105(0x69)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[106(0x6a)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[107(0x6b)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[108(0x6c)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[109(0x6d)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[110(0x6e)] = */ kCpumMicroarch_Intel_Atom_Airmount,      /* or silvermount? */
+    /*[111(0x6f)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[112(0x70)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[113(0x71)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[114(0x72)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[115(0x73)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[116(0x74)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[117(0x75)] = */ kCpumMicroarch_Intel_Atom_Airmount,      /* or silvermount? */
+    /*[118(0x76)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[119(0x77)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[120(0x78)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[121(0x79)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[122(0x7a)] = */ kCpumMicroarch_Intel_Atom_GoldmontPlus,
+    /*[123(0x7b)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[124(0x7c)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[125(0x7d)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[126(0x7e)] = */ kCpumMicroarch_Intel_Core7_IceLake, /* unconfirmed */
+    /*[127(0x7f)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[128(0x80)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[129(0x81)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[130(0x82)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[131(0x83)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[132(0x84)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[133(0x85)] = */ kCpumMicroarch_Intel_Phi_KnightsMill,
+    /*[134(0x86)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[135(0x87)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[136(0x88)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[137(0x89)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[138(0x8a)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[139(0x8b)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[140(0x8c)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[141(0x8d)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[142(0x8e)] = */ kCpumMicroarch_Intel_Core7_KabyLake, /* Stepping >= 0xB is Whiskey Lake, 0xA is CoffeeLake. */
+    /*[143(0x8f)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[144(0x90)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[145(0x91)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[146(0x92)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[147(0x93)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[148(0x94)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[149(0x95)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[150(0x96)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[151(0x97)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[152(0x98)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[153(0x99)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[154(0x9a)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[155(0x9b)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[156(0x9c)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[157(0x9d)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[158(0x9e)] = */ kCpumMicroarch_Intel_Core7_KabyLake, /* Stepping >= 0xB is Whiskey Lake, 0xA is CoffeeLake. */
+    /*[159(0x9f)] = */ kCpumMicroarch_Intel_Unknown,
+};
+AssertCompile(RT_ELEMENTS(g_aenmIntelFamily06) == 0x9f+1);
+
+
+/**
+ * Figures out the (sub-)micro architecture given a bit of CPUID info.
+ *
+ * @returns Micro architecture.
+ * @param   enmVendor           The CPU vendor .
+ * @param   bFamily             The CPU family.
+ * @param   bModel              The CPU model.
+ * @param   bStepping           The CPU stepping.
+ */
+VMMR3DECL(CPUMMICROARCH) CPUMR3CpuIdDetermineMicroarchEx(CPUMCPUVENDOR enmVendor, uint8_t bFamily,
+                                                         uint8_t bModel, uint8_t bStepping)
+{
+    if (enmVendor == CPUMCPUVENDOR_AMD)
+    {
+        switch (bFamily)
+        {
+            case 0x02:  return kCpumMicroarch_AMD_Am286; /* Not really kosher... */
+            case 0x03:  return kCpumMicroarch_AMD_Am386;
+            case 0x23:  return kCpumMicroarch_AMD_Am386; /* SX*/
+            case 0x04:  return bModel < 14 ? kCpumMicroarch_AMD_Am486 : kCpumMicroarch_AMD_Am486Enh;
+            case 0x05:  return bModel <  6 ? kCpumMicroarch_AMD_K5    : kCpumMicroarch_AMD_K6; /* Genode LX is 0x0a, lump it with K6. */
+            case 0x06:
+                switch (bModel)
+                {
+                    case  0: return kCpumMicroarch_AMD_K7_Palomino;
+                    case  1: return kCpumMicroarch_AMD_K7_Palomino;
+                    case  2: return kCpumMicroarch_AMD_K7_Palomino;
+                    case  3: return kCpumMicroarch_AMD_K7_Spitfire;
+                    case  4: return kCpumMicroarch_AMD_K7_Thunderbird;
+                    case  6: return kCpumMicroarch_AMD_K7_Palomino;
+                    case  7: return kCpumMicroarch_AMD_K7_Morgan;
+                    case  8: return kCpumMicroarch_AMD_K7_Thoroughbred;
+                    case 10: return kCpumMicroarch_AMD_K7_Barton; /* Thorton too. */
+                }
+                return kCpumMicroarch_AMD_K7_Unknown;
+            case 0x0f:
+                /*
+                 * This family is a friggin mess. Trying my best to make some
+                 * sense out of it. Too much happened in the 0x0f family to
+                 * lump it all together as K8 (130nm->90nm->65nm, AMD-V, ++).
+                 *
+                 * Emperical CPUID.01h.EAX evidence from revision guides, wikipedia,
+                 * cpu-world.com, and other places:
+                 *  - 130nm:
+                 *     - ClawHammer:    F7A/SH-CG, F5A/-CG, F4A/-CG, F50/-B0, F48/-C0, F58/-C0,
+                 *     - SledgeHammer:  F50/SH-B0, F48/-C0, F58/-C0, F4A/-CG, F5A/-CG, F7A/-CG, F51/-B3
+                 *     - Newcastle:     FC0/DH-CG (erratum #180: FE0/DH-CG), FF0/DH-CG
+                 *     - Dublin:        FC0/-CG, FF0/-CG, F82/CH-CG, F4A/-CG, F48/SH-C0,
+                 *     - Odessa:        FC0/DH-CG (erratum #180: FE0/DH-CG)
+                 *     - Paris:         FF0/DH-CG, FC0/DH-CG (erratum #180: FE0/DH-CG),
+                 *  - 90nm:
+                 *     - Winchester:    10FF0/DH-D0, 20FF0/DH-E3.
+                 *     - Oakville:      10FC0/DH-D0.
+                 *     - Georgetown:    10FC0/DH-D0.
+                 *     - Sonora:        10FC0/DH-D0.
+                 *     - Venus:         20F71/SH-E4
+                 *     - Troy:          20F51/SH-E4
+                 *     - Athens:        20F51/SH-E4
+                 *     - San Diego:     20F71/SH-E4.
+                 *     - Lancaster:     20F42/SH-E5
+                 *     - Newark:        20F42/SH-E5.
+                 *     - Albany:        20FC2/DH-E6.
+                 *     - Roma:          20FC2/DH-E6.
+                 *     - Venice:        20FF0/DH-E3, 20FC2/DH-E6, 20FF2/DH-E6.
+                 *     - Palermo:       10FC0/DH-D0, 20FF0/DH-E3, 20FC0/DH-E3, 20FC2/DH-E6, 20FF2/DH-E6
+                 *  - 90nm introducing Dual core:
+                 *     - Denmark:       20F30/JH-E1, 20F32/JH-E6
+                 *     - Italy:         20F10/JH-E1, 20F12/JH-E6
+                 *     - Egypt:         20F10/JH-E1, 20F12/JH-E6
+                 *     - Toledo:        20F32/JH-E6, 30F72/DH-E6 (single code variant).
+                 *     - Manchester:    20FB1/BH-E4, 30FF2/BH-E4.
+                 *  - 90nm 2nd gen opteron ++, AMD-V introduced (might be missing in some cheaper models):
+                 *     - Santa Ana:     40F32/JH-F2, /-F3
+                 *     - Santa Rosa:    40F12/JH-F2, 40F13/JH-F3
+                 *     - Windsor:       40F32/JH-F2, 40F33/JH-F3, C0F13/JH-F3, 40FB2/BH-F2, ??20FB1/BH-E4??.
+                 *     - Manila:        50FF2/DH-F2, 40FF2/DH-F2
+                 *     - Orleans:       40FF2/DH-F2, 50FF2/DH-F2, 50FF3/DH-F3.
+                 *     - Keene:         40FC2/DH-F2.
+                 *     - Richmond:      40FC2/DH-F2
+                 *     - Taylor:        40F82/BH-F2
+                 *     - Trinidad:      40F82/BH-F2
+                 *
+                 *  - 65nm:
+                 *     - Brisbane:      60FB1/BH-G1, 60FB2/BH-G2.
+                 *     - Tyler:         60F81/BH-G1, 60F82/BH-G2.
+                 *     - Sparta:        70FF1/DH-G1, 70FF2/DH-G2.
+                 *     - Lima:          70FF1/DH-G1, 70FF2/DH-G2.
+                 *     - Sherman:       /-G1, 70FC2/DH-G2.
+                 *     - Huron:         70FF2/DH-G2.
+                 */
+                if (bModel < 0x10)
+                    return kCpumMicroarch_AMD_K8_130nm;
+                if (bModel >= 0x60 && bModel < 0x80)
+                    return kCpumMicroarch_AMD_K8_65nm;
+                if (bModel >= 0x40)
+                    return kCpumMicroarch_AMD_K8_90nm_AMDV;
+                switch (bModel)
+                {
+                    case 0x21:
+                    case 0x23:
+                    case 0x2b:
+                    case 0x2f:
+                    case 0x37:
+                    case 0x3f:
+                        return kCpumMicroarch_AMD_K8_90nm_DualCore;
+                }
+                return kCpumMicroarch_AMD_K8_90nm;
+            case 0x10:
+                return kCpumMicroarch_AMD_K10;
+            case 0x11:
+                return kCpumMicroarch_AMD_K10_Lion;
+            case 0x12:
+                return kCpumMicroarch_AMD_K10_Llano;
+            case 0x14:
+                return kCpumMicroarch_AMD_Bobcat;
+            case 0x15:
+                switch (bModel)
+                {
+                    case 0x00:  return kCpumMicroarch_AMD_15h_Bulldozer;    /* Any? prerelease? */
+                    case 0x01:  return kCpumMicroarch_AMD_15h_Bulldozer;    /* Opteron 4200, FX-81xx. */
+                    case 0x02:  return kCpumMicroarch_AMD_15h_Piledriver;   /* Opteron 4300, FX-83xx. */
+                    case 0x10:  return kCpumMicroarch_AMD_15h_Piledriver;   /* A10-5800K for e.g. */
+                    case 0x11:  /* ?? */
+                    case 0x12:  /* ?? */
+                    case 0x13:  return kCpumMicroarch_AMD_15h_Piledriver;   /* A10-6800K for e.g. */
+                }
+                return kCpumMicroarch_AMD_15h_Unknown;
+            case 0x16:
+                return kCpumMicroarch_AMD_Jaguar;
+            case 0x17:
+                return kCpumMicroarch_AMD_Zen_Ryzen;
+        }
+        return kCpumMicroarch_AMD_Unknown;
+    }
+
+    if (enmVendor == CPUMCPUVENDOR_INTEL)
+    {
+        switch (bFamily)
+        {
+            case 3:
+                return kCpumMicroarch_Intel_80386;
+            case 4:
+                return kCpumMicroarch_Intel_80486;
+            case 5:
+                return kCpumMicroarch_Intel_P5;
+            case 6:
+                if (bModel < RT_ELEMENTS(g_aenmIntelFamily06))
+                {
+                    CPUMMICROARCH enmMicroArch = g_aenmIntelFamily06[bModel];
+                    if (enmMicroArch == kCpumMicroarch_Intel_Core7_KabyLake)
+                    {
+                        if (bStepping >= 0xa && bStepping <= 0xc)
+                            enmMicroArch = kCpumMicroarch_Intel_Core7_CoffeeLake;
+                        else if (bStepping >= 0xc)
+                            enmMicroArch = kCpumMicroarch_Intel_Core7_WhiskeyLake;
+                    }
+                    else if (   enmMicroArch == kCpumMicroarch_Intel_Core7_Skylake
+                             && bModel == 0x55
+                             && bStepping >= 5)
+                        enmMicroArch = kCpumMicroarch_Intel_Core7_CascadeLake;
+                    return enmMicroArch;
+                }
+                return kCpumMicroarch_Intel_Atom_Unknown;
+            case 15:
+                switch (bModel)
+                {
+                    case 0:     return kCpumMicroarch_Intel_NB_Willamette;
+                    case 1:     return kCpumMicroarch_Intel_NB_Willamette;
+                    case 2:     return kCpumMicroarch_Intel_NB_Northwood;
+                    case 3:     return kCpumMicroarch_Intel_NB_Prescott;
+                    case 4:     return kCpumMicroarch_Intel_NB_Prescott2M; /* ?? */
+                    case 5:     return kCpumMicroarch_Intel_NB_Unknown; /*??*/
+                    case 6:     return kCpumMicroarch_Intel_NB_CedarMill;
+                    case 7:     return kCpumMicroarch_Intel_NB_Gallatin;
+                    default:    return kCpumMicroarch_Intel_NB_Unknown;
+                }
+                break;
+            /* The following are not kosher but kind of follow intuitively from 6, 5 & 4. */
+            case 0:
+                return kCpumMicroarch_Intel_8086;
+            case 1:
+                return kCpumMicroarch_Intel_80186;
+            case 2:
+                return kCpumMicroarch_Intel_80286;
+        }
+        return kCpumMicroarch_Intel_Unknown;
+    }
+
+    if (enmVendor == CPUMCPUVENDOR_VIA)
+    {
+        switch (bFamily)
+        {
+            case 5:
+                switch (bModel)
+                {
+                    case 1: return kCpumMicroarch_Centaur_C6;
+                    case 4: return kCpumMicroarch_Centaur_C6;
+                    case 8: return kCpumMicroarch_Centaur_C2;
+                    case 9: return kCpumMicroarch_Centaur_C3;
+                }
+                break;
+
+            case 6:
+                switch (bModel)
+                {
+                    case  5: return kCpumMicroarch_VIA_C3_M2;
+                    case  6: return kCpumMicroarch_VIA_C3_C5A;
+                    case  7: return bStepping < 8 ? kCpumMicroarch_VIA_C3_C5B : kCpumMicroarch_VIA_C3_C5C;
+                    case  8: return kCpumMicroarch_VIA_C3_C5N;
+                    case  9: return bStepping < 8 ? kCpumMicroarch_VIA_C3_C5XL : kCpumMicroarch_VIA_C3_C5P;
+                    case 10: return kCpumMicroarch_VIA_C7_C5J;
+                    case 15: return kCpumMicroarch_VIA_Isaiah;
+                }
+                break;
+        }
+        return kCpumMicroarch_VIA_Unknown;
+    }
+
+    if (enmVendor == CPUMCPUVENDOR_SHANGHAI)
+    {
+        switch (bFamily)
+        {
+            case 6:
+            case 7:
+                return kCpumMicroarch_Shanghai_Wudaokou;
+            default:
+                break;
+        }
+        return kCpumMicroarch_Shanghai_Unknown;
+    }
+
+    if (enmVendor == CPUMCPUVENDOR_CYRIX)
+    {
+        switch (bFamily)
+        {
+            case 4:
+                switch (bModel)
+                {
+                    case 9: return kCpumMicroarch_Cyrix_5x86;
+                }
+                break;
+
+            case 5:
+                switch (bModel)
+                {
+                    case 2: return kCpumMicroarch_Cyrix_M1;
+                    case 4: return kCpumMicroarch_Cyrix_MediaGX;
+                    case 5: return kCpumMicroarch_Cyrix_MediaGXm;
+                }
+                break;
+
+            case 6:
+                switch (bModel)
+                {
+                    case 0: return kCpumMicroarch_Cyrix_M2;
+                }
+                break;
+
+        }
+        return kCpumMicroarch_Cyrix_Unknown;
+    }
+
+    if (enmVendor == CPUMCPUVENDOR_HYGON)
+    {
+        switch (bFamily)
+        {
+            case 0x18:
+                return kCpumMicroarch_Hygon_Dhyana;
+            default:
+                break;
+        }
+        return kCpumMicroarch_Hygon_Unknown;
+    }
+
+    return kCpumMicroarch_Unknown;
+}
+
+
+/**
+ * Translates a microarchitecture enum value to the corresponding string
+ * constant.
+ *
+ * @returns Read-only string constant (omits "kCpumMicroarch_" prefix). Returns
+ *          NULL if the value is invalid.
+ *
+ * @param   enmMicroarch    The enum value to convert.
+ */
+VMMR3DECL(const char *) CPUMR3MicroarchName(CPUMMICROARCH enmMicroarch)
+{
+    switch (enmMicroarch)
+    {
+#define CASE_RET_STR(enmValue)  case enmValue: return #enmValue + (sizeof("kCpumMicroarch_") - 1)
+        CASE_RET_STR(kCpumMicroarch_Intel_8086);
+        CASE_RET_STR(kCpumMicroarch_Intel_80186);
+        CASE_RET_STR(kCpumMicroarch_Intel_80286);
+        CASE_RET_STR(kCpumMicroarch_Intel_80386);
+        CASE_RET_STR(kCpumMicroarch_Intel_80486);
+        CASE_RET_STR(kCpumMicroarch_Intel_P5);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_P6);
+        CASE_RET_STR(kCpumMicroarch_Intel_P6_II);
+        CASE_RET_STR(kCpumMicroarch_Intel_P6_III);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_P6_M_Banias);
+        CASE_RET_STR(kCpumMicroarch_Intel_P6_M_Dothan);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core_Yonah);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_Core2_Merom);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core2_Penryn);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_Nehalem);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_Westmere);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_SandyBridge);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_IvyBridge);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_Haswell);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_Broadwell);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_Skylake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_KabyLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_CoffeeLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_WhiskeyLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_CascadeLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_CannonLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_IceLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_TigerLake);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Bonnell);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Lincroft);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Saltwell);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Silvermont);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Airmount);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Goldmont);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_GoldmontPlus);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsFerry);
+        CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsCorner);
+        CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsLanding);
+        CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsHill);
+        CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsMill);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_Willamette);
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_Northwood);
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_Prescott);
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_Prescott2M);
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_CedarMill);
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_Gallatin);
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_Am286);
+        CASE_RET_STR(kCpumMicroarch_AMD_Am386);
+        CASE_RET_STR(kCpumMicroarch_AMD_Am486);
+        CASE_RET_STR(kCpumMicroarch_AMD_Am486Enh);
+        CASE_RET_STR(kCpumMicroarch_AMD_K5);
+        CASE_RET_STR(kCpumMicroarch_AMD_K6);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Palomino);
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Spitfire);
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Thunderbird);
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Morgan);
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Thoroughbred);
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Barton);
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_K8_130nm);
+        CASE_RET_STR(kCpumMicroarch_AMD_K8_90nm);
+        CASE_RET_STR(kCpumMicroarch_AMD_K8_90nm_DualCore);
+        CASE_RET_STR(kCpumMicroarch_AMD_K8_90nm_AMDV);
+        CASE_RET_STR(kCpumMicroarch_AMD_K8_65nm);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_K10);
+        CASE_RET_STR(kCpumMicroarch_AMD_K10_Lion);
+        CASE_RET_STR(kCpumMicroarch_AMD_K10_Llano);
+        CASE_RET_STR(kCpumMicroarch_AMD_Bobcat);
+        CASE_RET_STR(kCpumMicroarch_AMD_Jaguar);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_15h_Bulldozer);
+        CASE_RET_STR(kCpumMicroarch_AMD_15h_Piledriver);
+        CASE_RET_STR(kCpumMicroarch_AMD_15h_Steamroller);
+        CASE_RET_STR(kCpumMicroarch_AMD_15h_Excavator);
+        CASE_RET_STR(kCpumMicroarch_AMD_15h_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_16h_First);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_Zen_Ryzen);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_Hygon_Dhyana);
+        CASE_RET_STR(kCpumMicroarch_Hygon_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_Centaur_C6);
+        CASE_RET_STR(kCpumMicroarch_Centaur_C2);
+        CASE_RET_STR(kCpumMicroarch_Centaur_C3);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_M2);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_C5A);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_C5B);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_C5C);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_C5N);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_C5XL);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_C5P);
+        CASE_RET_STR(kCpumMicroarch_VIA_C7_C5J);
+        CASE_RET_STR(kCpumMicroarch_VIA_Isaiah);
+        CASE_RET_STR(kCpumMicroarch_VIA_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_Shanghai_Wudaokou);
+        CASE_RET_STR(kCpumMicroarch_Shanghai_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_Cyrix_5x86);
+        CASE_RET_STR(kCpumMicroarch_Cyrix_M1);
+        CASE_RET_STR(kCpumMicroarch_Cyrix_MediaGX);
+        CASE_RET_STR(kCpumMicroarch_Cyrix_MediaGXm);
+        CASE_RET_STR(kCpumMicroarch_Cyrix_M2);
+        CASE_RET_STR(kCpumMicroarch_Cyrix_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_NEC_V20);
+        CASE_RET_STR(kCpumMicroarch_NEC_V30);
+
+        CASE_RET_STR(kCpumMicroarch_Unknown);
+
+#undef CASE_RET_STR
+        case kCpumMicroarch_Invalid:
+        case kCpumMicroarch_Intel_End:
+        case kCpumMicroarch_Intel_Core2_End:
+        case kCpumMicroarch_Intel_Core7_End:
+        case kCpumMicroarch_Intel_Atom_End:
+        case kCpumMicroarch_Intel_P6_Core_Atom_End:
+        case kCpumMicroarch_Intel_Phi_End:
+        case kCpumMicroarch_Intel_NB_End:
+        case kCpumMicroarch_AMD_K7_End:
+        case kCpumMicroarch_AMD_K8_End:
+        case kCpumMicroarch_AMD_15h_End:
+        case kCpumMicroarch_AMD_16h_End:
+        case kCpumMicroarch_AMD_Zen_End:
+        case kCpumMicroarch_AMD_End:
+        case kCpumMicroarch_Hygon_End:
+        case kCpumMicroarch_VIA_End:
+        case kCpumMicroarch_Shanghai_End:
+        case kCpumMicroarch_Cyrix_End:
+        case kCpumMicroarch_NEC_End:
+        case kCpumMicroarch_32BitHack:
+            break;
+        /* no default! */
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Determins the host CPU MXCSR mask.
+ *
+ * @returns MXCSR mask.
+ */
+VMMR3DECL(uint32_t) CPUMR3DeterminHostMxCsrMask(void)
+{
+    if (   ASMHasCpuId()
+        && ASMIsValidStdRange(ASMCpuId_EAX(0))
+        && ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_FXSR)
+    {
+        uint8_t volatile abBuf[sizeof(X86FXSTATE) + 64];
+        PX86FXSTATE      pState = (PX86FXSTATE)&abBuf[64 - ((uintptr_t)&abBuf[0] & 63)];
+        RT_ZERO(*pState);
+        ASMFxSave(pState);
+        if (pState->MXCSR_MASK == 0)
+            return 0xffbf;
+        return pState->MXCSR_MASK;
+    }
+    return 0;
+}
+
+
+/**
+ * Gets a matching leaf in the CPUID leaf array.
+ *
+ * @returns Pointer to the matching leaf, or NULL if not found.
+ * @param   paLeaves            The CPUID leaves to search.  This is sorted.
+ * @param   cLeaves             The number of leaves in the array.
+ * @param   uLeaf               The leaf to locate.
+ * @param   uSubLeaf            The subleaf to locate.  Pass 0 if no sub-leaves.
+ */
+static PCPUMCPUIDLEAF cpumR3CpuIdGetLeaf(PCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, uint32_t uLeaf, uint32_t uSubLeaf)
+{
+    /* Lazy bird does linear lookup here since this is only used for the
+       occational CPUID overrides. */
+    for (uint32_t i = 0; i < cLeaves; i++)
+        if (   paLeaves[i].uLeaf    == uLeaf
+            && paLeaves[i].uSubLeaf == (uSubLeaf & paLeaves[i].fSubLeafMask))
+            return &paLeaves[i];
+    return NULL;
+}
+
+
+#ifndef IN_VBOX_CPU_REPORT
+/**
+ * Gets a matching leaf in the CPUID leaf array, converted to a CPUMCPUID.
+ *
+ * @returns true if found, false it not.
+ * @param   paLeaves            The CPUID leaves to search.  This is sorted.
+ * @param   cLeaves             The number of leaves in the array.
+ * @param   uLeaf               The leaf to locate.
+ * @param   uSubLeaf            The subleaf to locate.  Pass 0 if no sub-leaves.
+ * @param   pLegacy             The legacy output leaf.
+ */
+static bool cpumR3CpuIdGetLeafLegacy(PCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, uint32_t uLeaf, uint32_t uSubLeaf,
+                                     PCPUMCPUID pLegacy)
+{
+    PCPUMCPUIDLEAF pLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, uLeaf, uSubLeaf);
+    if (pLeaf)
+    {
+        pLegacy->uEax = pLeaf->uEax;
+        pLegacy->uEbx = pLeaf->uEbx;
+        pLegacy->uEcx = pLeaf->uEcx;
+        pLegacy->uEdx = pLeaf->uEdx;
+        return true;
+    }
+    return false;
+}
+#endif /* IN_VBOX_CPU_REPORT */
+
+
+/**
+ * Ensures that the CPUID leaf array can hold one more leaf.
+ *
+ * @returns Pointer to the CPUID leaf array (*ppaLeaves) on success.  NULL on
+ *          failure.
+ * @param   pVM         The cross context VM structure.  If NULL, use
+ *                      the process heap, otherwise the VM's hyper heap.
+ * @param   ppaLeaves   Pointer to the variable holding the array pointer
+ *                      (input/output).
+ * @param   cLeaves     The current array size.
+ *
+ * @remarks This function will automatically update the R0 and RC pointers when
+ *          using the hyper heap, which means @a ppaLeaves and @a cLeaves must
+ *          be the corresponding VM's CPUID arrays (which is asserted).
+ */
+static PCPUMCPUIDLEAF cpumR3CpuIdEnsureSpace(PVM pVM, PCPUMCPUIDLEAF *ppaLeaves, uint32_t cLeaves)
+{
+    /*
+     * If pVM is not specified, we're on the regular heap and can waste a
+     * little space to speed things up.
+     */
+    uint32_t cAllocated;
+    if (!pVM)
+    {
+        cAllocated = RT_ALIGN(cLeaves, 16);
+        if (cLeaves + 1 > cAllocated)
+        {
+            void *pvNew = RTMemRealloc(*ppaLeaves, (cAllocated + 16) * sizeof(**ppaLeaves));
+            if (pvNew)
+                *ppaLeaves = (PCPUMCPUIDLEAF)pvNew;
+            else
+            {
+                RTMemFree(*ppaLeaves);
+                *ppaLeaves = NULL;
+            }
+        }
+    }
+    /*
+     * Otherwise, we're on the hyper heap and are probably just inserting
+     * one or two leaves and should conserve space.
+     */
+    else
+    {
+#ifdef IN_VBOX_CPU_REPORT
+        AssertReleaseFailed();
+#else
+        Assert(ppaLeaves == &pVM->cpum.s.GuestInfo.paCpuIdLeavesR3);
+        Assert(cLeaves == pVM->cpum.s.GuestInfo.cCpuIdLeaves);
+
+        size_t cb    = cLeaves       * sizeof(**ppaLeaves);
+        size_t cbNew = (cLeaves + 1) * sizeof(**ppaLeaves);
+        int rc = MMR3HyperRealloc(pVM, *ppaLeaves, cb, 32, MM_TAG_CPUM_CPUID, cbNew, (void **)ppaLeaves);
+        if (RT_SUCCESS(rc))
+            pVM->cpum.s.GuestInfo.paCpuIdLeavesR0 = MMHyperR3ToR0(pVM, *ppaLeaves);
+        else
+        {
+            *ppaLeaves = NULL;
+            pVM->cpum.s.GuestInfo.paCpuIdLeavesR0 = NIL_RTR0PTR;
+            LogRel(("CPUM: cpumR3CpuIdEnsureSpace: MMR3HyperRealloc failed. rc=%Rrc\n", rc));
+        }
+#endif
+    }
+    return *ppaLeaves;
+}
+
+
+/**
+ * Append a CPUID leaf or sub-leaf.
+ *
+ * ASSUMES linear insertion order, so we'll won't need to do any searching or
+ * replace anything.  Use cpumR3CpuIdInsert() for those cases.
+ *
+ * @returns VINF_SUCCESS or VERR_NO_MEMORY.  On error, *ppaLeaves is freed, so
+ *          the caller need do no more work.
+ * @param   ppaLeaves       Pointer to the pointer to the array of sorted
+ *                          CPUID leaves and sub-leaves.
+ * @param   pcLeaves        Where we keep the leaf count for *ppaLeaves.
+ * @param   uLeaf           The leaf we're adding.
+ * @param   uSubLeaf        The sub-leaf number.
+ * @param   fSubLeafMask    The sub-leaf mask.
+ * @param   uEax            The EAX value.
+ * @param   uEbx            The EBX value.
+ * @param   uEcx            The ECX value.
+ * @param   uEdx            The EDX value.
+ * @param   fFlags          The flags.
+ */
+static int cpumR3CollectCpuIdInfoAddOne(PCPUMCPUIDLEAF *ppaLeaves, uint32_t *pcLeaves,
+                                        uint32_t uLeaf, uint32_t uSubLeaf, uint32_t fSubLeafMask,
+                                        uint32_t uEax, uint32_t uEbx, uint32_t uEcx, uint32_t uEdx, uint32_t fFlags)
+{
+    if (!cpumR3CpuIdEnsureSpace(NULL /* pVM */, ppaLeaves, *pcLeaves))
+        return VERR_NO_MEMORY;
+
+    PCPUMCPUIDLEAF pNew = &(*ppaLeaves)[*pcLeaves];
+    Assert(   *pcLeaves == 0
+           || pNew[-1].uLeaf < uLeaf
+           || (pNew[-1].uLeaf == uLeaf && pNew[-1].uSubLeaf < uSubLeaf) );
+
+    pNew->uLeaf        = uLeaf;
+    pNew->uSubLeaf     = uSubLeaf;
+    pNew->fSubLeafMask = fSubLeafMask;
+    pNew->uEax         = uEax;
+    pNew->uEbx         = uEbx;
+    pNew->uEcx         = uEcx;
+    pNew->uEdx         = uEdx;
+    pNew->fFlags       = fFlags;
+
+    *pcLeaves += 1;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks that we've updated the CPUID leaves array correctly.
+ *
+ * This is a no-op in non-strict builds.
+ *
+ * @param   paLeaves            The leaves array.
+ * @param   cLeaves             The number of leaves.
+ */
+static void cpumR3CpuIdAssertOrder(PCPUMCPUIDLEAF paLeaves, uint32_t cLeaves)
+{
+#ifdef VBOX_STRICT
+    for (uint32_t i = 1; i < cLeaves; i++)
+        if (paLeaves[i].uLeaf != paLeaves[i - 1].uLeaf)
+            AssertMsg(paLeaves[i].uLeaf > paLeaves[i - 1].uLeaf, ("%#x vs %#x\n", paLeaves[i].uLeaf, paLeaves[i - 1].uLeaf));
+        else
+        {
+            AssertMsg(paLeaves[i].uSubLeaf > paLeaves[i - 1].uSubLeaf,
+                      ("%#x: %#x vs %#x\n", paLeaves[i].uLeaf, paLeaves[i].uSubLeaf, paLeaves[i - 1].uSubLeaf));
+            AssertMsg(paLeaves[i].fSubLeafMask == paLeaves[i - 1].fSubLeafMask,
+                      ("%#x/%#x: %#x vs %#x\n", paLeaves[i].uLeaf, paLeaves[i].uSubLeaf, paLeaves[i].fSubLeafMask, paLeaves[i - 1].fSubLeafMask));
+            AssertMsg(paLeaves[i].fFlags == paLeaves[i - 1].fFlags,
+                      ("%#x/%#x: %#x vs %#x\n", paLeaves[i].uLeaf, paLeaves[i].uSubLeaf, paLeaves[i].fFlags, paLeaves[i - 1].fFlags));
+        }
+#else
+    NOREF(paLeaves);
+    NOREF(cLeaves);
+#endif
+}
+
+
+/**
+ * Inserts a CPU ID leaf, replacing any existing ones.
+ *
+ * When inserting a simple leaf where we already got a series of sub-leaves with
+ * the same leaf number (eax), the simple leaf will replace the whole series.
+ *
+ * When pVM is NULL, this ASSUMES that the leaves array is still on the normal
+ * host-context heap and has only been allocated/reallocated by the
+ * cpumR3CpuIdEnsureSpace function.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.  If NULL, use
+ *                      the process heap, otherwise the VM's hyper heap.
+ * @param   ppaLeaves   Pointer to the pointer to the array of sorted
+ *                      CPUID leaves and sub-leaves. Must be NULL if using
+ *                      the hyper heap.
+ * @param   pcLeaves    Where we keep the leaf count for *ppaLeaves. Must
+ *                      be NULL if using the hyper heap.
+ * @param   pNewLeaf    Pointer to the data of the new leaf we're about to
+ *                      insert.
+ */
+static int cpumR3CpuIdInsert(PVM pVM, PCPUMCPUIDLEAF *ppaLeaves, uint32_t *pcLeaves, PCPUMCPUIDLEAF pNewLeaf)
+{
+    /*
+     * Validate input parameters if we are using the hyper heap and use the VM's CPUID arrays.
+     */
+    if (pVM)
+    {
+        AssertReturn(!ppaLeaves, VERR_INVALID_PARAMETER);
+        AssertReturn(!pcLeaves, VERR_INVALID_PARAMETER);
+
+        ppaLeaves = &pVM->cpum.s.GuestInfo.paCpuIdLeavesR3;
+        pcLeaves  = &pVM->cpum.s.GuestInfo.cCpuIdLeaves;
+    }
+
+    PCPUMCPUIDLEAF  paLeaves = *ppaLeaves;
+    uint32_t        cLeaves  = *pcLeaves;
+
+    /*
+     * Validate the new leaf a little.
+     */
+    AssertLogRelMsgReturn(!(pNewLeaf->fFlags & ~CPUMCPUIDLEAF_F_VALID_MASK),
+                          ("%#x/%#x: %#x", pNewLeaf->uLeaf, pNewLeaf->uSubLeaf, pNewLeaf->fFlags),
+                          VERR_INVALID_FLAGS);
+    AssertLogRelMsgReturn(pNewLeaf->fSubLeafMask != 0 || pNewLeaf->uSubLeaf == 0,
+                          ("%#x/%#x: %#x", pNewLeaf->uLeaf, pNewLeaf->uSubLeaf, pNewLeaf->fSubLeafMask),
+                          VERR_INVALID_PARAMETER);
+    AssertLogRelMsgReturn(RT_IS_POWER_OF_TWO(pNewLeaf->fSubLeafMask + 1),
+                          ("%#x/%#x: %#x", pNewLeaf->uLeaf, pNewLeaf->uSubLeaf, pNewLeaf->fSubLeafMask),
+                          VERR_INVALID_PARAMETER);
+    AssertLogRelMsgReturn((pNewLeaf->fSubLeafMask & pNewLeaf->uSubLeaf) == pNewLeaf->uSubLeaf,
+                          ("%#x/%#x: %#x", pNewLeaf->uLeaf, pNewLeaf->uSubLeaf, pNewLeaf->fSubLeafMask),
+                          VERR_INVALID_PARAMETER);
+
+    /*
+     * Find insertion point.  The lazy bird uses the same excuse as in
+     * cpumR3CpuIdGetLeaf(), but optimizes for linear insertion (saved state).
+     */
+    uint32_t i;
+    if (   cLeaves > 0
+        && paLeaves[cLeaves - 1].uLeaf < pNewLeaf->uLeaf)
+    {
+        /* Add at end. */
+        i = cLeaves;
+    }
+    else if (   cLeaves > 0
+             && paLeaves[cLeaves - 1].uLeaf == pNewLeaf->uLeaf)
+    {
+        /* Either replacing the last leaf or dealing with sub-leaves. Spool
+           back to the first sub-leaf to pretend we did the linear search. */
+        i = cLeaves - 1;
+        while (   i > 0
+               && paLeaves[i - 1].uLeaf == pNewLeaf->uLeaf)
+            i--;
+    }
+    else
+    {
+        /* Linear search from the start. */
+        i = 0;
+        while (   i < cLeaves
+                  && paLeaves[i].uLeaf < pNewLeaf->uLeaf)
+            i++;
+    }
+    if (   i < cLeaves
+        && paLeaves[i].uLeaf == pNewLeaf->uLeaf)
+    {
+        if (paLeaves[i].fSubLeafMask != pNewLeaf->fSubLeafMask)
+        {
+            /*
+             * The sub-leaf mask differs, replace all existing leaves with the
+             * same leaf number.
+             */
+            uint32_t c = 1;
+            while (   i + c < cLeaves
+                   && paLeaves[i + c].uLeaf == pNewLeaf->uLeaf)
+                c++;
+            if (c > 1 && i + c < cLeaves)
+            {
+                memmove(&paLeaves[i + c], &paLeaves[i + 1], (cLeaves - i - c) * sizeof(paLeaves[0]));
+                *pcLeaves = cLeaves -= c - 1;
+            }
+
+            paLeaves[i] = *pNewLeaf;
+            cpumR3CpuIdAssertOrder(*ppaLeaves, *pcLeaves);
+            return VINF_SUCCESS;
+        }
+
+        /* Find sub-leaf insertion point. */
+        while (   i < cLeaves
+               && paLeaves[i].uSubLeaf < pNewLeaf->uSubLeaf
+               && paLeaves[i].uLeaf == pNewLeaf->uLeaf)
+            i++;
+
+        /*
+         * If we've got an exactly matching leaf, replace it.
+         */
+        if (   i < cLeaves
+            && paLeaves[i].uLeaf    == pNewLeaf->uLeaf
+            && paLeaves[i].uSubLeaf == pNewLeaf->uSubLeaf)
+        {
+            paLeaves[i] = *pNewLeaf;
+            cpumR3CpuIdAssertOrder(*ppaLeaves, *pcLeaves);
+            return VINF_SUCCESS;
+        }
+    }
+
+    /*
+     * Adding a new leaf at 'i'.
+     */
+    AssertLogRelReturn(cLeaves < CPUM_CPUID_MAX_LEAVES, VERR_TOO_MANY_CPUID_LEAVES);
+    paLeaves = cpumR3CpuIdEnsureSpace(pVM, ppaLeaves, cLeaves);
+    if (!paLeaves)
+        return VERR_NO_MEMORY;
+
+    if (i < cLeaves)
+        memmove(&paLeaves[i + 1], &paLeaves[i], (cLeaves - i) * sizeof(paLeaves[0]));
+    *pcLeaves += 1;
+    paLeaves[i] = *pNewLeaf;
+
+    cpumR3CpuIdAssertOrder(*ppaLeaves, *pcLeaves);
+    return VINF_SUCCESS;
+}
+
+
+#ifndef IN_VBOX_CPU_REPORT
+/**
+ * Removes a range of CPUID leaves.
+ *
+ * This will not reallocate the array.
+ *
+ * @param   paLeaves        The array of sorted CPUID leaves and sub-leaves.
+ * @param   pcLeaves        Where we keep the leaf count for @a paLeaves.
+ * @param   uFirst          The first leaf.
+ * @param   uLast           The last leaf.
+ */
+static void cpumR3CpuIdRemoveRange(PCPUMCPUIDLEAF paLeaves, uint32_t *pcLeaves, uint32_t uFirst, uint32_t uLast)
+{
+    uint32_t cLeaves = *pcLeaves;
+
+    Assert(uFirst <= uLast);
+
+    /*
+     * Find the first one.
+     */
+    uint32_t iFirst = 0;
+    while (   iFirst < cLeaves
+           && paLeaves[iFirst].uLeaf < uFirst)
+        iFirst++;
+
+    /*
+     * Find the end (last + 1).
+     */
+    uint32_t iEnd = iFirst;
+    while (   iEnd < cLeaves
+           && paLeaves[iEnd].uLeaf <= uLast)
+        iEnd++;
+
+    /*
+     * Adjust the array if anything needs removing.
+     */
+    if (iFirst < iEnd)
+    {
+        if (iEnd < cLeaves)
+            memmove(&paLeaves[iFirst], &paLeaves[iEnd], (cLeaves - iEnd) * sizeof(paLeaves[0]));
+        *pcLeaves = cLeaves -= (iEnd - iFirst);
+    }
+
+    cpumR3CpuIdAssertOrder(paLeaves, *pcLeaves);
+}
+#endif /* IN_VBOX_CPU_REPORT */
+
+
+/**
+ * Checks if ECX make a difference when reading a given CPUID leaf.
+ *
+ * @returns @c true if it does, @c false if it doesn't.
+ * @param   uLeaf               The leaf we're reading.
+ * @param   pcSubLeaves         Number of sub-leaves accessible via ECX.
+ * @param   pfFinalEcxUnchanged Whether ECX is passed thru when going beyond the
+ *                              final sub-leaf (for leaf 0xb only).
+ */
+static bool cpumR3IsEcxRelevantForCpuIdLeaf(uint32_t uLeaf, uint32_t *pcSubLeaves, bool *pfFinalEcxUnchanged)
+{
+    *pfFinalEcxUnchanged = false;
+
+    uint32_t auCur[4];
+    uint32_t auPrev[4];
+    ASMCpuIdExSlow(uLeaf, 0, 0, 0, &auPrev[0], &auPrev[1], &auPrev[2], &auPrev[3]);
+
+    /* Look for sub-leaves. */
+    uint32_t uSubLeaf = 1;
+    for (;;)
+    {
+        ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &auCur[0], &auCur[1], &auCur[2], &auCur[3]);
+        if (memcmp(auCur, auPrev, sizeof(auCur)))
+            break;
+
+        /* Advance / give up. */
+        uSubLeaf++;
+        if (uSubLeaf >= 64)
+        {
+            *pcSubLeaves = 1;
+            return false;
+        }
+    }
+
+    /* Count sub-leaves. */
+    uint32_t cMinLeaves = uLeaf == 0xd ? 64 : 0;
+    uint32_t cRepeats = 0;
+    uSubLeaf = 0;
+    for (;;)
+    {
+        ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &auCur[0], &auCur[1], &auCur[2], &auCur[3]);
+
+        /* Figuring out when to stop isn't entirely straight forward as we need
+           to cover undocumented behavior up to a point and implementation shortcuts. */
+
+        /* 1. Look for more than 4 repeating value sets. */
+        if (   auCur[0] == auPrev[0]
+            && auCur[1] == auPrev[1]
+            && (    auCur[2] == auPrev[2]
+                || (   auCur[2]  == uSubLeaf
+                    && auPrev[2] == uSubLeaf - 1) )
+            && auCur[3] == auPrev[3])
+        {
+            if (   uLeaf != 0xd
+                || uSubLeaf >= 64
+                || (   auCur[0] == 0
+                    && auCur[1] == 0
+                    && auCur[2] == 0
+                    && auCur[3] == 0
+                    && auPrev[2] == 0) )
+                cRepeats++;
+            if (cRepeats > 4 && uSubLeaf >= cMinLeaves)
+                break;
+        }
+        else
+            cRepeats = 0;
+
+        /* 2. Look for zero values. */
+        if (   auCur[0] == 0
+            && auCur[1] == 0
+            && (auCur[2] == 0 || auCur[2] == uSubLeaf)
+            && (auCur[3] == 0 || uLeaf == 0xb /* edx is fixed */)
+            && uSubLeaf >= cMinLeaves)
+        {
+            cRepeats = 0;
+            break;
+        }
+
+        /* 3. Leaf 0xb level type 0 check. */
+        if (   uLeaf == 0xb
+            && (auCur[2]  & 0xff00) == 0
+            && (auPrev[2] & 0xff00) == 0)
+        {
+            cRepeats = 0;
+            break;
+        }
+
+        /* 99. Give up. */
+        if (uSubLeaf >= 128)
+        {
+#ifndef IN_VBOX_CPU_REPORT
+            /* Ok, limit it according to the documentation if possible just to
+               avoid annoying users with these detection issues. */
+            uint32_t cDocLimit = UINT32_MAX;
+            if (uLeaf == 0x4)
+                cDocLimit = 4;
+            else if (uLeaf == 0x7)
+                cDocLimit = 1;
+            else if (uLeaf == 0xd)
+                cDocLimit = 63;
+            else if (uLeaf == 0xf)
+                cDocLimit = 2;
+            if (cDocLimit != UINT32_MAX)
+            {
+                *pfFinalEcxUnchanged = auCur[2] == uSubLeaf && uLeaf == 0xb;
+                *pcSubLeaves = cDocLimit + 3;
+                return true;
+            }
+#endif
+            *pcSubLeaves = UINT32_MAX;
+            return true;
+        }
+
+        /* Advance. */
+        uSubLeaf++;
+        memcpy(auPrev, auCur, sizeof(auCur));
+    }
+
+    /* Standard exit. */
+    *pfFinalEcxUnchanged = auCur[2] == uSubLeaf && uLeaf == 0xb;
+    *pcSubLeaves = uSubLeaf + 1 - cRepeats;
+    if (*pcSubLeaves == 0)
+        *pcSubLeaves = 1;
+    return true;
+}
+
+
+/**
+ * Gets a CPU ID leaf.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pLeaf       Where to store the found leaf.
+ * @param   uLeaf       The leaf to locate.
+ * @param   uSubLeaf    The subleaf to locate.  Pass 0 if no sub-leaves.
+ */
+VMMR3DECL(int) CPUMR3CpuIdGetLeaf(PVM pVM, PCPUMCPUIDLEAF pLeaf, uint32_t uLeaf, uint32_t uSubLeaf)
+{
+    PCPUMCPUIDLEAF pcLeaf = cpumR3CpuIdGetLeaf(pVM->cpum.s.GuestInfo.paCpuIdLeavesR3, pVM->cpum.s.GuestInfo.cCpuIdLeaves,
+                                               uLeaf, uSubLeaf);
+    if (pcLeaf)
+    {
+        memcpy(pLeaf, pcLeaf, sizeof(*pLeaf));
+        return VINF_SUCCESS;
+    }
+
+    return VERR_NOT_FOUND;
+}
+
+
+/**
+ * Inserts a CPU ID leaf, replacing any existing ones.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pNewLeaf    Pointer to the leaf being inserted.
+ */
+VMMR3DECL(int) CPUMR3CpuIdInsert(PVM pVM, PCPUMCPUIDLEAF pNewLeaf)
+{
+    /*
+     * Validate parameters.
+     */
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+    AssertReturn(pNewLeaf, VERR_INVALID_PARAMETER);
+
+    /*
+     * Disallow replacing CPU ID leaves that this API currently cannot manage.
+     * These leaves have dependencies on saved-states, see PATMCpuidReplacement().
+     * If you want to modify these leaves, use CPUMSetGuestCpuIdFeature().
+     */
+    if (   pNewLeaf->uLeaf == UINT32_C(0x00000000)  /* Standard */
+        || pNewLeaf->uLeaf == UINT32_C(0x00000001)
+        || pNewLeaf->uLeaf == UINT32_C(0x80000000)  /* Extended */
+        || pNewLeaf->uLeaf == UINT32_C(0x80000001)
+        || pNewLeaf->uLeaf == UINT32_C(0xc0000000)  /* Centaur */
+        || pNewLeaf->uLeaf == UINT32_C(0xc0000001) )
+    {
+        return VERR_NOT_SUPPORTED;
+    }
+
+    return cpumR3CpuIdInsert(pVM, NULL /* ppaLeaves */, NULL /* pcLeaves */, pNewLeaf);
+}
+
+/**
+ * Collects CPUID leaves and sub-leaves, returning a sorted array of them.
+ *
+ * @returns VBox status code.
+ * @param   ppaLeaves           Where to return the array pointer on success.
+ *                              Use RTMemFree to release.
+ * @param   pcLeaves            Where to return the size of the array on
+ *                              success.
+ */
+VMMR3DECL(int) CPUMR3CpuIdCollectLeaves(PCPUMCPUIDLEAF *ppaLeaves, uint32_t *pcLeaves)
+{
+    *ppaLeaves = NULL;
+    *pcLeaves = 0;
+
+    /*
+     * Try out various candidates. This must be sorted!
+     */
+    static struct { uint32_t uMsr; bool fSpecial; } const s_aCandidates[] =
+    {
+        { UINT32_C(0x00000000), false },
+        { UINT32_C(0x10000000), false },
+        { UINT32_C(0x20000000), false },
+        { UINT32_C(0x30000000), false },
+        { UINT32_C(0x40000000), false },
+        { UINT32_C(0x50000000), false },
+        { UINT32_C(0x60000000), false },
+        { UINT32_C(0x70000000), false },
+        { UINT32_C(0x80000000), false },
+        { UINT32_C(0x80860000), false },
+        { UINT32_C(0x8ffffffe), true  },
+        { UINT32_C(0x8fffffff), true  },
+        { UINT32_C(0x90000000), false },
+        { UINT32_C(0xa0000000), false },
+        { UINT32_C(0xb0000000), false },
+        { UINT32_C(0xc0000000), false },
+        { UINT32_C(0xd0000000), false },
+        { UINT32_C(0xe0000000), false },
+        { UINT32_C(0xf0000000), false },
+    };
+
+    for (uint32_t iOuter = 0; iOuter < RT_ELEMENTS(s_aCandidates); iOuter++)
+    {
+        uint32_t uLeaf = s_aCandidates[iOuter].uMsr;
+        uint32_t uEax, uEbx, uEcx, uEdx;
+        ASMCpuIdExSlow(uLeaf, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
+
+        /*
+         * Does EAX look like a typical leaf count value?
+         */
+        if (   uEax         > uLeaf
+            && uEax - uLeaf < UINT32_C(0xff)) /* Adjust 0xff limit when exceeded by real HW. */
+        {
+            /* Yes, dump them. */
+            uint32_t cLeaves = uEax - uLeaf + 1;
+            while (cLeaves-- > 0)
+            {
+                ASMCpuIdExSlow(uLeaf, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
+
+                uint32_t fFlags = 0;
+
+                /* There are currently three known leaves containing an APIC ID
+                   that needs EMT specific attention */
+                if (uLeaf == 1)
+                    fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC_ID;
+                else if (uLeaf == 0xb && uEcx != 0)
+                    fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC_ID;
+                else if (   uLeaf == UINT32_C(0x8000001e)
+                         && (   uEax
+                             || uEbx
+                             || uEdx
+                             || ASMIsAmdCpuEx((*ppaLeaves)[0].uEbx, (*ppaLeaves)[0].uEcx, (*ppaLeaves)[0].uEdx)
+                             || ASMIsHygonCpuEx((*ppaLeaves)[0].uEbx, (*ppaLeaves)[0].uEcx, (*ppaLeaves)[0].uEdx)) )
+                    fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC_ID;
+
+                /* The APIC bit is per-VCpu and needs flagging. */
+                if (uLeaf == 1)
+                    fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC;
+                else if (   uLeaf == UINT32_C(0x80000001)
+                         && (   (uEdx & X86_CPUID_AMD_FEATURE_EDX_APIC)
+                             || ASMIsAmdCpuEx((*ppaLeaves)[0].uEbx, (*ppaLeaves)[0].uEcx, (*ppaLeaves)[0].uEdx)
+                             || ASMIsHygonCpuEx((*ppaLeaves)[0].uEbx, (*ppaLeaves)[0].uEcx, (*ppaLeaves)[0].uEdx)) )
+                    fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC;
+
+                /* Check three times here to reduce the chance of CPU migration
+                   resulting in false positives with things like the APIC ID. */
+                uint32_t cSubLeaves;
+                bool fFinalEcxUnchanged;
+                if (   cpumR3IsEcxRelevantForCpuIdLeaf(uLeaf, &cSubLeaves, &fFinalEcxUnchanged)
+                    && cpumR3IsEcxRelevantForCpuIdLeaf(uLeaf, &cSubLeaves, &fFinalEcxUnchanged)
+                    && cpumR3IsEcxRelevantForCpuIdLeaf(uLeaf, &cSubLeaves, &fFinalEcxUnchanged))
+                {
+                    if (cSubLeaves > (uLeaf == 0xd ? 68U : 16U))
+                    {
+                        /* This shouldn't happen.  But in case it does, file all
+                           relevant details in the release log. */
+                        LogRel(("CPUM: VERR_CPUM_TOO_MANY_CPUID_SUBLEAVES! uLeaf=%#x cSubLeaves=%#x\n", uLeaf, cSubLeaves));
+                        LogRel(("------------------ dump of problematic sub-leaves -----------------\n"));
+                        for (uint32_t uSubLeaf = 0; uSubLeaf < 128; uSubLeaf++)
+                        {
+                            uint32_t auTmp[4];
+                            ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &auTmp[0], &auTmp[1], &auTmp[2], &auTmp[3]);
+                            LogRel(("CPUM: %#010x, %#010x => %#010x %#010x %#010x %#010x\n",
+                                    uLeaf, uSubLeaf, auTmp[0], auTmp[1], auTmp[2], auTmp[3]));
+                        }
+                        LogRel(("----------------- dump of what we've found so far -----------------\n"));
+                        for (uint32_t i = 0 ; i < *pcLeaves; i++)
+                            LogRel(("CPUM: %#010x, %#010x/%#010x => %#010x %#010x %#010x %#010x\n",
+                                    (*ppaLeaves)[i].uLeaf, (*ppaLeaves)[i].uSubLeaf,  (*ppaLeaves)[i].fSubLeafMask,
+                                    (*ppaLeaves)[i].uEax, (*ppaLeaves)[i].uEbx, (*ppaLeaves)[i].uEcx, (*ppaLeaves)[i].uEdx));
+                        LogRel(("\nPlease create a defect on virtualbox.org and attach this log file!\n\n"));
+                        return VERR_CPUM_TOO_MANY_CPUID_SUBLEAVES;
+                    }
+
+                    if (fFinalEcxUnchanged)
+                        fFlags |= CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES;
+
+                    for (uint32_t uSubLeaf = 0; uSubLeaf < cSubLeaves; uSubLeaf++)
+                    {
+                        ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &uEax, &uEbx, &uEcx, &uEdx);
+                        int rc = cpumR3CollectCpuIdInfoAddOne(ppaLeaves, pcLeaves,
+                                                              uLeaf, uSubLeaf, UINT32_MAX, uEax, uEbx, uEcx, uEdx, fFlags);
+                        if (RT_FAILURE(rc))
+                            return rc;
+                    }
+                }
+                else
+                {
+                    int rc = cpumR3CollectCpuIdInfoAddOne(ppaLeaves, pcLeaves,
+                                                          uLeaf, 0, 0, uEax, uEbx, uEcx, uEdx, fFlags);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+
+                /* next */
+                uLeaf++;
+            }
+        }
+        /*
+         * Special CPUIDs needs special handling as they don't follow the
+         * leaf count principle used above.
+         */
+        else if (s_aCandidates[iOuter].fSpecial)
+        {
+            bool fKeep = false;
+            if (uLeaf == 0x8ffffffe && uEax == UINT32_C(0x00494544))
+                fKeep = true;
+            else if (   uLeaf == 0x8fffffff
+                     && RT_C_IS_PRINT(RT_BYTE1(uEax))
+                     && RT_C_IS_PRINT(RT_BYTE2(uEax))
+                     && RT_C_IS_PRINT(RT_BYTE3(uEax))
+                     && RT_C_IS_PRINT(RT_BYTE4(uEax))
+                     && RT_C_IS_PRINT(RT_BYTE1(uEbx))
+                     && RT_C_IS_PRINT(RT_BYTE2(uEbx))
+                     && RT_C_IS_PRINT(RT_BYTE3(uEbx))
+                     && RT_C_IS_PRINT(RT_BYTE4(uEbx))
+                     && RT_C_IS_PRINT(RT_BYTE1(uEcx))
+                     && RT_C_IS_PRINT(RT_BYTE2(uEcx))
+                     && RT_C_IS_PRINT(RT_BYTE3(uEcx))
+                     && RT_C_IS_PRINT(RT_BYTE4(uEcx))
+                     && RT_C_IS_PRINT(RT_BYTE1(uEdx))
+                     && RT_C_IS_PRINT(RT_BYTE2(uEdx))
+                     && RT_C_IS_PRINT(RT_BYTE3(uEdx))
+                     && RT_C_IS_PRINT(RT_BYTE4(uEdx)) )
+                fKeep = true;
+            if (fKeep)
+            {
+                int rc = cpumR3CollectCpuIdInfoAddOne(ppaLeaves, pcLeaves,
+                                                      uLeaf, 0, 0, uEax, uEbx, uEcx, uEdx, 0);
+                if (RT_FAILURE(rc))
+                    return rc;
+            }
+        }
+    }
+
+    cpumR3CpuIdAssertOrder(*ppaLeaves, *pcLeaves);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Determines the method the CPU uses to handle unknown CPUID leaves.
+ *
+ * @returns VBox status code.
+ * @param   penmUnknownMethod   Where to return the method.
+ * @param   pDefUnknown         Where to return default unknown values.  This
+ *                              will be set, even if the resulting method
+ *                              doesn't actually needs it.
+ */
+VMMR3DECL(int) CPUMR3CpuIdDetectUnknownLeafMethod(PCPUMUNKNOWNCPUID penmUnknownMethod, PCPUMCPUID pDefUnknown)
+{
+    uint32_t uLastStd = ASMCpuId_EAX(0);
+    uint32_t uLastExt = ASMCpuId_EAX(0x80000000);
+    if (!ASMIsValidExtRange(uLastExt))
+        uLastExt = 0x80000000;
+
+    uint32_t auChecks[] =
+    {
+        uLastStd + 1,
+        uLastStd + 5,
+        uLastStd + 8,
+        uLastStd + 32,
+        uLastStd + 251,
+        uLastExt + 1,
+        uLastExt + 8,
+        uLastExt + 15,
+        uLastExt + 63,
+        uLastExt + 255,
+        0x7fbbffcc,
+        0x833f7872,
+        0xefff2353,
+        0x35779456,
+        0x1ef6d33e,
+    };
+
+    static const uint32_t s_auValues[] =
+    {
+        0xa95d2156,
+        0x00000001,
+        0x00000002,
+        0x00000008,
+        0x00000000,
+        0x55773399,
+        0x93401769,
+        0x12039587,
+    };
+
+    /*
+     * Simple method, all zeros.
+     */
+    *penmUnknownMethod = CPUMUNKNOWNCPUID_DEFAULTS;
+    pDefUnknown->uEax = 0;
+    pDefUnknown->uEbx = 0;
+    pDefUnknown->uEcx = 0;
+    pDefUnknown->uEdx = 0;
+
+    /*
+     * Intel has been observed returning the last standard leaf.
+     */
+    uint32_t auLast[4];
+    ASMCpuIdExSlow(uLastStd, 0, 0, 0, &auLast[0], &auLast[1], &auLast[2], &auLast[3]);
+
+    uint32_t cChecks = RT_ELEMENTS(auChecks);
+    while (cChecks > 0)
+    {
+        uint32_t auCur[4];
+        ASMCpuIdExSlow(auChecks[cChecks - 1], 0, 0, 0, &auCur[0], &auCur[1], &auCur[2], &auCur[3]);
+        if (memcmp(auCur, auLast, sizeof(auCur)))
+            break;
+        cChecks--;
+    }
+    if (cChecks == 0)
+    {
+        /* Now, what happens when the input changes? Esp. ECX. */
+        uint32_t cTotal       = 0;
+        uint32_t cSame        = 0;
+        uint32_t cLastWithEcx = 0;
+        uint32_t cNeither     = 0;
+        uint32_t cValues = RT_ELEMENTS(s_auValues);
+        while (cValues > 0)
+        {
+            uint32_t uValue = s_auValues[cValues - 1];
+            uint32_t auLastWithEcx[4];
+            ASMCpuIdExSlow(uLastStd, uValue, uValue, uValue,
+                           &auLastWithEcx[0], &auLastWithEcx[1], &auLastWithEcx[2], &auLastWithEcx[3]);
+
+            cChecks = RT_ELEMENTS(auChecks);
+            while (cChecks > 0)
+            {
+                uint32_t auCur[4];
+                ASMCpuIdExSlow(auChecks[cChecks - 1], uValue, uValue, uValue, &auCur[0], &auCur[1], &auCur[2], &auCur[3]);
+                if (!memcmp(auCur, auLast, sizeof(auCur)))
+                {
+                    cSame++;
+                    if (!memcmp(auCur, auLastWithEcx, sizeof(auCur)))
+                        cLastWithEcx++;
+                }
+                else if (!memcmp(auCur, auLastWithEcx, sizeof(auCur)))
+                    cLastWithEcx++;
+                else
+                    cNeither++;
+                cTotal++;
+                cChecks--;
+            }
+            cValues--;
+        }
+
+        Log(("CPUM: cNeither=%d cSame=%d cLastWithEcx=%d cTotal=%d\n", cNeither, cSame, cLastWithEcx, cTotal));
+        if (cSame == cTotal)
+            *penmUnknownMethod = CPUMUNKNOWNCPUID_LAST_STD_LEAF;
+        else if (cLastWithEcx == cTotal)
+            *penmUnknownMethod = CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX;
+        else
+            *penmUnknownMethod = CPUMUNKNOWNCPUID_LAST_STD_LEAF;
+        pDefUnknown->uEax = auLast[0];
+        pDefUnknown->uEbx = auLast[1];
+        pDefUnknown->uEcx = auLast[2];
+        pDefUnknown->uEdx = auLast[3];
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Unchanged register values?
+     */
+    cChecks = RT_ELEMENTS(auChecks);
+    while (cChecks > 0)
+    {
+        uint32_t const  uLeaf   = auChecks[cChecks - 1];
+        uint32_t        cValues = RT_ELEMENTS(s_auValues);
+        while (cValues > 0)
+        {
+            uint32_t uValue = s_auValues[cValues - 1];
+            uint32_t auCur[4];
+            ASMCpuIdExSlow(uLeaf, uValue, uValue, uValue, &auCur[0], &auCur[1], &auCur[2], &auCur[3]);
+            if (   auCur[0] != uLeaf
+                || auCur[1] != uValue
+                || auCur[2] != uValue
+                || auCur[3] != uValue)
+                break;
+            cValues--;
+        }
+        if (cValues != 0)
+            break;
+        cChecks--;
+    }
+    if (cChecks == 0)
+    {
+        *penmUnknownMethod = CPUMUNKNOWNCPUID_PASSTHRU;
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Just go with the simple method.
+     */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Translates a unknow CPUID leaf method into the constant name (sans prefix).
+ *
+ * @returns Read only name string.
+ * @param   enmUnknownMethod    The method to translate.
+ */
+VMMR3DECL(const char *) CPUMR3CpuIdUnknownLeafMethodName(CPUMUNKNOWNCPUID enmUnknownMethod)
+{
+    switch (enmUnknownMethod)
+    {
+        case CPUMUNKNOWNCPUID_DEFAULTS:                  return "DEFAULTS";
+        case CPUMUNKNOWNCPUID_LAST_STD_LEAF:             return "LAST_STD_LEAF";
+        case CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX:    return "LAST_STD_LEAF_WITH_ECX";
+        case CPUMUNKNOWNCPUID_PASSTHRU:                  return "PASSTHRU";
+
+        case CPUMUNKNOWNCPUID_INVALID:
+        case CPUMUNKNOWNCPUID_END:
+        case CPUMUNKNOWNCPUID_32BIT_HACK:
+            break;
+    }
+    return "Invalid-unknown-CPUID-method";
+}
+
+
+/**
+ * Detect the CPU vendor give n the
+ *
+ * @returns The vendor.
+ * @param   uEAX                EAX from CPUID(0).
+ * @param   uEBX                EBX from CPUID(0).
+ * @param   uECX                ECX from CPUID(0).
+ * @param   uEDX                EDX from CPUID(0).
+ */
+VMMR3DECL(CPUMCPUVENDOR) CPUMR3CpuIdDetectVendorEx(uint32_t uEAX, uint32_t uEBX, uint32_t uECX, uint32_t uEDX)
+{
+    if (ASMIsValidStdRange(uEAX))
+    {
+        if (ASMIsAmdCpuEx(uEBX, uECX, uEDX))
+            return CPUMCPUVENDOR_AMD;
+
+        if (ASMIsIntelCpuEx(uEBX, uECX, uEDX))
+            return CPUMCPUVENDOR_INTEL;
+
+        if (ASMIsViaCentaurCpuEx(uEBX, uECX, uEDX))
+            return CPUMCPUVENDOR_VIA;
+
+        if (ASMIsShanghaiCpuEx(uEBX, uECX, uEDX))
+            return CPUMCPUVENDOR_SHANGHAI;
+
+        if (   uEBX == UINT32_C(0x69727943) /* CyrixInstead */
+            && uECX == UINT32_C(0x64616574)
+            && uEDX == UINT32_C(0x736E4978))
+            return CPUMCPUVENDOR_CYRIX;
+
+        if (ASMIsHygonCpuEx(uEBX, uECX, uEDX))
+            return CPUMCPUVENDOR_HYGON;
+
+        /* "Geode by NSC", example: family 5, model 9.  */
+
+        /** @todo detect the other buggers... */
+    }
+
+    return CPUMCPUVENDOR_UNKNOWN;
+}
+
+
+/**
+ * Translates a CPU vendor enum value into the corresponding string constant.
+ *
+ * The named can be prefixed with 'CPUMCPUVENDOR_' to construct a valid enum
+ * value name.  This can be useful when generating code.
+ *
+ * @returns Read only name string.
+ * @param   enmVendor           The CPU vendor value.
+ */
+VMMR3DECL(const char *) CPUMR3CpuVendorName(CPUMCPUVENDOR enmVendor)
+{
+    switch (enmVendor)
+    {
+        case CPUMCPUVENDOR_INTEL:       return "INTEL";
+        case CPUMCPUVENDOR_AMD:         return "AMD";
+        case CPUMCPUVENDOR_VIA:         return "VIA";
+        case CPUMCPUVENDOR_CYRIX:       return "CYRIX";
+        case CPUMCPUVENDOR_SHANGHAI:    return "SHANGHAI";
+        case CPUMCPUVENDOR_HYGON:       return "HYGON";
+        case CPUMCPUVENDOR_UNKNOWN:     return "UNKNOWN";
+
+        case CPUMCPUVENDOR_INVALID:
+        case CPUMCPUVENDOR_32BIT_HACK:
+            break;
+    }
+    return "Invalid-cpu-vendor";
+}
+
+
+static PCCPUMCPUIDLEAF cpumR3CpuIdFindLeaf(PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, uint32_t uLeaf)
+{
+    /* Could do binary search, doing linear now because I'm lazy. */
+    PCCPUMCPUIDLEAF pLeaf = paLeaves;
+    while (cLeaves-- > 0)
+    {
+        if (pLeaf->uLeaf == uLeaf)
+            return pLeaf;
+        pLeaf++;
+    }
+    return NULL;
+}
+
+
+static PCCPUMCPUIDLEAF cpumR3CpuIdFindLeafEx(PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, uint32_t uLeaf, uint32_t uSubLeaf)
+{
+    PCCPUMCPUIDLEAF pLeaf = cpumR3CpuIdFindLeaf(paLeaves, cLeaves, uLeaf);
+    if (   !pLeaf
+        || pLeaf->uSubLeaf != (uSubLeaf & pLeaf->fSubLeafMask))
+        return pLeaf;
+
+    /* Linear sub-leaf search. Lazy as usual. */
+    cLeaves -= pLeaf - paLeaves;
+    while (   cLeaves-- > 0
+           && pLeaf->uLeaf == uLeaf)
+    {
+        if (pLeaf->uSubLeaf == (uSubLeaf & pLeaf->fSubLeafMask))
+            return pLeaf;
+        pLeaf++;
+    }
+
+    return NULL;
+}
+
+
+static void cpumR3ExplodeVmxFeatures(PCVMXMSRS pVmxMsrs, PCPUMFEATURES pFeatures)
+{
+    Assert(pVmxMsrs);
+    Assert(pFeatures);
+    Assert(pFeatures->fVmx);
+
+    /* Basic information. */
+    {
+        uint64_t const u64Basic = pVmxMsrs->u64Basic;
+        pFeatures->fVmxInsOutInfo            = RT_BF_GET(u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
+    }
+
+    /* Pin-based VM-execution controls. */
+    {
+        uint32_t const fPinCtls = pVmxMsrs->PinCtls.n.allowed1;
+        pFeatures->fVmxExtIntExit            = RT_BOOL(fPinCtls & VMX_PIN_CTLS_EXT_INT_EXIT);
+        pFeatures->fVmxNmiExit               = RT_BOOL(fPinCtls & VMX_PIN_CTLS_NMI_EXIT);
+        pFeatures->fVmxVirtNmi               = RT_BOOL(fPinCtls & VMX_PIN_CTLS_VIRT_NMI);
+        pFeatures->fVmxPreemptTimer          = RT_BOOL(fPinCtls & VMX_PIN_CTLS_PREEMPT_TIMER);
+        pFeatures->fVmxPostedInt             = RT_BOOL(fPinCtls & VMX_PIN_CTLS_POSTED_INT);
+    }
+
+    /* Processor-based VM-execution controls. */
+    {
+        uint32_t const fProcCtls = pVmxMsrs->ProcCtls.n.allowed1;
+        pFeatures->fVmxIntWindowExit         = RT_BOOL(fProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
+        pFeatures->fVmxTscOffsetting         = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING);
+        pFeatures->fVmxHltExit               = RT_BOOL(fProcCtls & VMX_PROC_CTLS_HLT_EXIT);
+        pFeatures->fVmxInvlpgExit            = RT_BOOL(fProcCtls & VMX_PROC_CTLS_INVLPG_EXIT);
+        pFeatures->fVmxMwaitExit             = RT_BOOL(fProcCtls & VMX_PROC_CTLS_MWAIT_EXIT);
+        pFeatures->fVmxRdpmcExit             = RT_BOOL(fProcCtls & VMX_PROC_CTLS_RDPMC_EXIT);
+        pFeatures->fVmxRdtscExit             = RT_BOOL(fProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
+        pFeatures->fVmxCr3LoadExit           = RT_BOOL(fProcCtls & VMX_PROC_CTLS_CR3_LOAD_EXIT);
+        pFeatures->fVmxCr3StoreExit          = RT_BOOL(fProcCtls & VMX_PROC_CTLS_CR3_STORE_EXIT);
+        pFeatures->fVmxCr8LoadExit           = RT_BOOL(fProcCtls & VMX_PROC_CTLS_CR8_LOAD_EXIT);
+        pFeatures->fVmxCr8StoreExit          = RT_BOOL(fProcCtls & VMX_PROC_CTLS_CR8_STORE_EXIT);
+        pFeatures->fVmxUseTprShadow          = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
+        pFeatures->fVmxNmiWindowExit         = RT_BOOL(fProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
+        pFeatures->fVmxMovDRxExit            = RT_BOOL(fProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
+        pFeatures->fVmxUncondIoExit          = RT_BOOL(fProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
+        pFeatures->fVmxUseIoBitmaps          = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS);
+        pFeatures->fVmxMonitorTrapFlag       = RT_BOOL(fProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
+        pFeatures->fVmxUseMsrBitmaps         = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
+        pFeatures->fVmxMonitorExit           = RT_BOOL(fProcCtls & VMX_PROC_CTLS_MONITOR_EXIT);
+        pFeatures->fVmxPauseExit             = RT_BOOL(fProcCtls & VMX_PROC_CTLS_PAUSE_EXIT);
+        pFeatures->fVmxSecondaryExecCtls     = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
+    }
+
+    /* Secondary processor-based VM-execution controls. */
+    {
+        uint32_t const fProcCtls2 = pFeatures->fVmxSecondaryExecCtls ? pVmxMsrs->ProcCtls2.n.allowed1 : 0;
+        pFeatures->fVmxVirtApicAccess        = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
+        pFeatures->fVmxEpt                   = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_EPT);
+        pFeatures->fVmxDescTableExit         = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT);
+        pFeatures->fVmxRdtscp                = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_RDTSCP);
+        pFeatures->fVmxVirtX2ApicMode        = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE);
+        pFeatures->fVmxVpid                  = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VPID);
+        pFeatures->fVmxWbinvdExit            = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_WBINVD_EXIT);
+        pFeatures->fVmxUnrestrictedGuest     = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
+        pFeatures->fVmxApicRegVirt           = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT);
+        pFeatures->fVmxVirtIntDelivery       = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+        pFeatures->fVmxPauseLoopExit         = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
+        pFeatures->fVmxRdrandExit            = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT);
+        pFeatures->fVmxInvpcid               = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_INVPCID);
+        pFeatures->fVmxVmFunc                = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VMFUNC);
+        pFeatures->fVmxVmcsShadowing         = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING);
+        pFeatures->fVmxRdseedExit            = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_RDSEED_EXIT);
+        pFeatures->fVmxPml                   = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_PML);
+        pFeatures->fVmxEptXcptVe             = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_EPT_VE);
+        pFeatures->fVmxXsavesXrstors         = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_XSAVES_XRSTORS);
+        pFeatures->fVmxUseTscScaling         = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_TSC_SCALING);
+    }
+
+    /* VM-exit controls. */
+    {
+        uint32_t const fExitCtls = pVmxMsrs->ExitCtls.n.allowed1;
+        pFeatures->fVmxExitSaveDebugCtls     = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_SAVE_DEBUG);
+        pFeatures->fVmxHostAddrSpaceSize     = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE);
+        pFeatures->fVmxExitAckExtInt         = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT);
+        pFeatures->fVmxExitSavePatMsr        = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_SAVE_PAT_MSR);
+        pFeatures->fVmxExitLoadPatMsr        = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_LOAD_PAT_MSR);
+        pFeatures->fVmxExitSaveEferMsr       = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_SAVE_EFER_MSR);
+        pFeatures->fVmxExitLoadEferMsr       = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR);
+        pFeatures->fVmxSavePreemptTimer      = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER);
+    }
+
+    /* VM-entry controls. */
+    {
+        uint32_t const fEntryCtls = pVmxMsrs->EntryCtls.n.allowed1;
+        pFeatures->fVmxEntryLoadDebugCtls    = RT_BOOL(fEntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG);
+        pFeatures->fVmxIa32eModeGuest        = RT_BOOL(fEntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+        pFeatures->fVmxEntryLoadEferMsr      = RT_BOOL(fEntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR);
+        pFeatures->fVmxEntryLoadPatMsr       = RT_BOOL(fEntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR);
+    }
+
+    /* Miscellaneous data. */
+    {
+        uint32_t const fMiscData = pVmxMsrs->u64Misc;
+        pFeatures->fVmxExitSaveEferLma       = RT_BOOL(fMiscData & VMX_MISC_EXIT_SAVE_EFER_LMA);
+        pFeatures->fVmxIntelPt               = RT_BOOL(fMiscData & VMX_MISC_INTEL_PT);
+        pFeatures->fVmxVmwriteAll            = RT_BOOL(fMiscData & VMX_MISC_VMWRITE_ALL);
+        pFeatures->fVmxEntryInjectSoftInt    = RT_BOOL(fMiscData & VMX_MISC_ENTRY_INJECT_SOFT_INT);
+    }
+}
+
+
+int cpumR3CpuIdExplodeFeatures(PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, PCCPUMMSRS pMsrs, PCPUMFEATURES pFeatures)
+{
+    Assert(pMsrs);
+    RT_ZERO(*pFeatures);
+    if (cLeaves >= 2)
+    {
+        AssertLogRelReturn(paLeaves[0].uLeaf == 0, VERR_CPUM_IPE_1);
+        AssertLogRelReturn(paLeaves[1].uLeaf == 1, VERR_CPUM_IPE_1);
+        PCCPUMCPUIDLEAF const pStd0Leaf = cpumR3CpuIdFindLeafEx(paLeaves, cLeaves, 0, 0);
+        AssertLogRelReturn(pStd0Leaf, VERR_CPUM_IPE_1);
+        PCCPUMCPUIDLEAF const pStd1Leaf = cpumR3CpuIdFindLeafEx(paLeaves, cLeaves, 1, 0);
+        AssertLogRelReturn(pStd1Leaf, VERR_CPUM_IPE_1);
+
+        pFeatures->enmCpuVendor = CPUMR3CpuIdDetectVendorEx(pStd0Leaf->uEax,
+                                                            pStd0Leaf->uEbx,
+                                                            pStd0Leaf->uEcx,
+                                                            pStd0Leaf->uEdx);
+        pFeatures->uFamily      = ASMGetCpuFamily(pStd1Leaf->uEax);
+        pFeatures->uModel       = ASMGetCpuModel(pStd1Leaf->uEax, pFeatures->enmCpuVendor == CPUMCPUVENDOR_INTEL);
+        pFeatures->uStepping    = ASMGetCpuStepping(pStd1Leaf->uEax);
+        pFeatures->enmMicroarch = CPUMR3CpuIdDetermineMicroarchEx((CPUMCPUVENDOR)pFeatures->enmCpuVendor,
+                                                                  pFeatures->uFamily,
+                                                                  pFeatures->uModel,
+                                                                  pFeatures->uStepping);
+
+        PCCPUMCPUIDLEAF const pExtLeaf8 = cpumR3CpuIdFindLeaf(paLeaves, cLeaves, 0x80000008);
+        if (pExtLeaf8)
+        {
+            pFeatures->cMaxPhysAddrWidth   = pExtLeaf8->uEax & 0xff;
+            pFeatures->cMaxLinearAddrWidth = (pExtLeaf8->uEax >> 8) & 0xff;
+        }
+        else if (pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PSE36)
+        {
+            pFeatures->cMaxPhysAddrWidth   = 36;
+            pFeatures->cMaxLinearAddrWidth = 36;
+        }
+        else
+        {
+            pFeatures->cMaxPhysAddrWidth   = 32;
+            pFeatures->cMaxLinearAddrWidth = 32;
+        }
+
+        /* Standard features. */
+        pFeatures->fMsr                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_MSR);
+        pFeatures->fApic                = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_APIC);
+        pFeatures->fX2Apic              = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_X2APIC);
+        pFeatures->fPse                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PSE);
+        pFeatures->fPse36               = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PSE36);
+        pFeatures->fPae                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PAE);
+        pFeatures->fPat                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PAT);
+        pFeatures->fFxSaveRstor         = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_FXSR);
+        pFeatures->fXSaveRstor          = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_XSAVE);
+        pFeatures->fOpSysXSaveRstor     = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_OSXSAVE);
+        pFeatures->fMmx                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_MMX);
+        pFeatures->fSse                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_SSE);
+        pFeatures->fSse2                = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_SSE2);
+        pFeatures->fSse3                = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_SSE3);
+        pFeatures->fSsse3               = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_SSSE3);
+        pFeatures->fSse41               = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_SSE4_1);
+        pFeatures->fSse42               = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_SSE4_2);
+        pFeatures->fAvx                 = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_AVX);
+        pFeatures->fTsc                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_TSC);
+        pFeatures->fSysEnter            = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_SEP);
+        pFeatures->fHypervisorPresent   = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_HVP);
+        pFeatures->fMonitorMWait        = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_MONITOR);
+        pFeatures->fMovCmpXchg16b       = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_CX16);
+        pFeatures->fClFlush             = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_CLFSH);
+        pFeatures->fPcid                = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_PCID);
+        pFeatures->fVmx                 = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_VMX);
+        if (pFeatures->fVmx)
+            cpumR3ExplodeVmxFeatures(&pMsrs->hwvirt.vmx, pFeatures);
+
+        /* Structured extended features. */
+        PCCPUMCPUIDLEAF const pSxfLeaf0 = cpumR3CpuIdFindLeafEx(paLeaves, cLeaves, 7, 0);
+        if (pSxfLeaf0)
+        {
+            pFeatures->fFsGsBase            = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_FSGSBASE);
+            pFeatures->fAvx2                = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_AVX2);
+            pFeatures->fAvx512Foundation    = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_AVX512F);
+            pFeatures->fClFlushOpt          = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_CLFLUSHOPT);
+            pFeatures->fInvpcid             = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_INVPCID);
+
+            pFeatures->fIbpb                = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_IBRS_IBPB);
+            pFeatures->fIbrs                = pFeatures->fIbpb;
+            pFeatures->fStibp               = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_STIBP);
+            pFeatures->fFlushCmd            = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_FLUSH_CMD);
+            pFeatures->fArchCap             = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_ARCHCAP);
+            pFeatures->fMdsClear            = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_MD_CLEAR);
+        }
+
+        /* MWAIT/MONITOR leaf. */
+        PCCPUMCPUIDLEAF const pMWaitLeaf = cpumR3CpuIdFindLeaf(paLeaves, cLeaves, 5);
+        if (pMWaitLeaf)
+            pFeatures->fMWaitExtensions = (pMWaitLeaf->uEcx & (X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0))
+                                        ==                    (X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0);
+
+        /* Extended features. */
+        PCCPUMCPUIDLEAF const pExtLeaf  = cpumR3CpuIdFindLeaf(paLeaves, cLeaves, 0x80000001);
+        if (pExtLeaf)
+        {
+            pFeatures->fLongMode        = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE);
+            pFeatures->fSysCall         = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_EXT_FEATURE_EDX_SYSCALL);
+            pFeatures->fNoExecute       = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_EXT_FEATURE_EDX_NX);
+            pFeatures->fLahfSahf        = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_EXT_FEATURE_ECX_LAHF_SAHF);
+            pFeatures->fRdTscP          = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_EXT_FEATURE_EDX_RDTSCP);
+            pFeatures->fMovCr8In32Bit   = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_CMPL);
+            pFeatures->f3DNow           = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_3DNOW);
+            pFeatures->f3DNowPrefetch   = (pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF)
+                                       || (pExtLeaf->uEdx & (  X86_CPUID_EXT_FEATURE_EDX_LONG_MODE
+                                                             | X86_CPUID_AMD_FEATURE_EDX_3DNOW));
+        }
+
+        /* VMX (VMXON, VMCS region and related data structures) physical address width (depends on long-mode). */
+        pFeatures->cVmxMaxPhysAddrWidth = pFeatures->fLongMode ? pFeatures->cMaxPhysAddrWidth : 32;
+
+        if (   pExtLeaf
+            && (   pFeatures->enmCpuVendor == CPUMCPUVENDOR_AMD
+                || pFeatures->enmCpuVendor == CPUMCPUVENDOR_HYGON))
+        {
+            /* AMD features. */
+            pFeatures->fMsr            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_MSR);
+            pFeatures->fApic           |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_APIC);
+            pFeatures->fPse            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PSE);
+            pFeatures->fPse36          |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PSE36);
+            pFeatures->fPae            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PAE);
+            pFeatures->fPat            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PAT);
+            pFeatures->fFxSaveRstor    |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_FXSR);
+            pFeatures->fMmx            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_MMX);
+            pFeatures->fTsc            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_TSC);
+            pFeatures->fIbpb           |= pExtLeaf8 && (pExtLeaf8->uEbx & X86_CPUID_AMD_EFEID_EBX_IBPB);
+            pFeatures->fAmdMmxExts      = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_AXMMX);
+            pFeatures->fXop             = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_XOP);
+            pFeatures->fSvm             = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_SVM);
+            if (pFeatures->fSvm)
+            {
+                PCCPUMCPUIDLEAF pSvmLeaf = cpumR3CpuIdFindLeaf(paLeaves, cLeaves, 0x8000000a);
+                AssertLogRelReturn(pSvmLeaf, VERR_CPUM_IPE_1);
+                pFeatures->fSvmNestedPaging         = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING);
+                pFeatures->fSvmLbrVirt              = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
+                pFeatures->fSvmSvmLock              = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_SVM_LOCK);
+                pFeatures->fSvmNextRipSave          = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
+                pFeatures->fSvmTscRateMsr           = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_TSC_RATE_MSR);
+                pFeatures->fSvmVmcbClean            = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
+                pFeatures->fSvmFlusbByAsid          = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID);
+                pFeatures->fSvmDecodeAssists        = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
+                pFeatures->fSvmPauseFilter          = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
+                pFeatures->fSvmPauseFilterThreshold = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
+                pFeatures->fSvmAvic                 = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_AVIC);
+                pFeatures->fSvmVirtVmsaveVmload     = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
+                pFeatures->fSvmVGif                 = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_VGIF);
+                pFeatures->fSvmGmet                 = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_GMET);
+                pFeatures->uSvmMaxAsid              = pSvmLeaf->uEbx;
+            }
+        }
+
+        /*
+         * Quirks.
+         */
+        pFeatures->fLeakyFxSR = pExtLeaf
+                             && (pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_FFXSR)
+                             && (   (   pFeatures->enmCpuVendor == CPUMCPUVENDOR_AMD
+                                     && pFeatures->uFamily >= 6 /* K7 and up */)
+                                 || pFeatures->enmCpuVendor == CPUMCPUVENDOR_HYGON);
+
+        /*
+         * Max extended (/FPU) state.
+         */
+        pFeatures->cbMaxExtendedState = pFeatures->fFxSaveRstor ? sizeof(X86FXSTATE) : sizeof(X86FPUSTATE);
+        if (pFeatures->fXSaveRstor)
+        {
+            PCCPUMCPUIDLEAF const pXStateLeaf0 = cpumR3CpuIdFindLeafEx(paLeaves, cLeaves, 13, 0);
+            if (pXStateLeaf0)
+            {
+                if (   pXStateLeaf0->uEcx >= sizeof(X86FXSTATE)
+                    && pXStateLeaf0->uEcx <= CPUM_MAX_XSAVE_AREA_SIZE
+                    && RT_ALIGN_32(pXStateLeaf0->uEcx, 8) == pXStateLeaf0->uEcx
+                    && pXStateLeaf0->uEbx >= sizeof(X86FXSTATE)
+                    && pXStateLeaf0->uEbx <= pXStateLeaf0->uEcx
+                    && RT_ALIGN_32(pXStateLeaf0->uEbx, 8) == pXStateLeaf0->uEbx)
+                {
+                    pFeatures->cbMaxExtendedState = pXStateLeaf0->uEcx;
+
+                    /* (paranoia:) */
+                    PCCPUMCPUIDLEAF const pXStateLeaf1 = cpumR3CpuIdFindLeafEx(paLeaves, cLeaves, 13, 1);
+                    if (   pXStateLeaf1
+                        && pXStateLeaf1->uEbx > pFeatures->cbMaxExtendedState
+                        && pXStateLeaf1->uEbx <= CPUM_MAX_XSAVE_AREA_SIZE
+                        && (pXStateLeaf1->uEcx || pXStateLeaf1->uEdx) )
+                        pFeatures->cbMaxExtendedState = pXStateLeaf1->uEbx;
+                }
+                else
+                    AssertLogRelMsgFailedStmt(("Unexpected max/cur XSAVE area sizes: %#x/%#x\n", pXStateLeaf0->uEcx, pXStateLeaf0->uEbx),
+                                              pFeatures->fXSaveRstor = 0);
+            }
+            else
+                AssertLogRelMsgFailedStmt(("Expected leaf eax=0xd/ecx=0 with the XSAVE/XRSTOR feature!\n"),
+                                          pFeatures->fXSaveRstor = 0);
+        }
+    }
+    else
+        AssertLogRelReturn(cLeaves == 0, VERR_CPUM_IPE_1);
+    return VINF_SUCCESS;
+}
+
+
+/*
+ *
+ * Init related code.
+ * Init related code.
+ * Init related code.
+ *
+ *
+ */
+#ifndef IN_VBOX_CPU_REPORT
+
+
+/**
+ * Gets an exactly matching leaf + sub-leaf in the CPUID leaf array.
+ *
+ * This ignores the fSubLeafMask.
+ *
+ * @returns Pointer to the matching leaf, or NULL if not found.
+ * @param   pCpum               The CPUM instance data.
+ * @param   uLeaf               The leaf to locate.
+ * @param   uSubLeaf            The subleaf to locate.
+ */
+static PCPUMCPUIDLEAF cpumR3CpuIdGetExactLeaf(PCPUM pCpum, uint32_t uLeaf, uint32_t uSubLeaf)
+{
+    uint64_t        uNeedle   = RT_MAKE_U64(uSubLeaf, uLeaf);
+    PCPUMCPUIDLEAF  paLeaves  = pCpum->GuestInfo.paCpuIdLeavesR3;
+    uint32_t        iEnd      = pCpum->GuestInfo.cCpuIdLeaves;
+    if (iEnd)
+    {
+        uint32_t    iBegin   = 0;
+        for (;;)
+        {
+            uint32_t const i    = (iEnd - iBegin) / 2 + iBegin;
+            uint64_t const uCur = RT_MAKE_U64(paLeaves[i].uSubLeaf, paLeaves[i].uLeaf);
+            if (uNeedle < uCur)
+            {
+                if (i > iBegin)
+                    iEnd = i;
+                else
+                    break;
+            }
+            else if (uNeedle > uCur)
+            {
+                if (i + 1 < iEnd)
+                    iBegin = i + 1;
+                else
+                    break;
+            }
+            else
+                return &paLeaves[i];
+        }
+    }
+    return NULL;
+}
+
+
+/**
+ * Loads MSR range overrides.
+ *
+ * This must be called before the MSR ranges are moved from the normal heap to
+ * the hyper heap!
+ *
+ * @returns VBox status code (VMSetError called).
+ * @param   pVM                 The cross context VM structure.
+ * @param   pMsrNode            The CFGM node with the MSR overrides.
+ */
+static int cpumR3LoadMsrOverrides(PVM pVM, PCFGMNODE pMsrNode)
+{
+    for (PCFGMNODE pNode = CFGMR3GetFirstChild(pMsrNode); pNode; pNode = CFGMR3GetNextChild(pNode))
+    {
+        /*
+         * Assemble a valid MSR range.
+         */
+        CPUMMSRRANGE MsrRange;
+        MsrRange.offCpumCpu = 0;
+        MsrRange.fReserved  = 0;
+
+        int rc = CFGMR3GetName(pNode, MsrRange.szName, sizeof(MsrRange.szName));
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Invalid MSR entry (name is probably too long): %Rrc\n", rc);
+
+        rc = CFGMR3QueryU32(pNode, "First", &MsrRange.uFirst);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Invalid MSR entry '%s': Error querying mandatory 'First' value: %Rrc\n",
+                              MsrRange.szName, rc);
+
+        rc = CFGMR3QueryU32Def(pNode, "Last", &MsrRange.uLast, MsrRange.uFirst);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Invalid MSR entry '%s': Error querying 'Last' value: %Rrc\n",
+                              MsrRange.szName, rc);
+
+        char szType[32];
+        rc = CFGMR3QueryStringDef(pNode, "Type", szType, sizeof(szType), "FixedValue");
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Invalid MSR entry '%s': Error querying 'Type' value: %Rrc\n",
+                              MsrRange.szName, rc);
+        if (!RTStrICmp(szType, "FixedValue"))
+        {
+            MsrRange.enmRdFn = kCpumMsrRdFn_FixedValue;
+            MsrRange.enmWrFn = kCpumMsrWrFn_IgnoreWrite;
+
+            rc = CFGMR3QueryU64Def(pNode, "Value", &MsrRange.uValue, 0);
+            if (RT_FAILURE(rc))
+                return VMSetError(pVM, rc, RT_SRC_POS, "Invalid MSR entry '%s': Error querying 'Value' value: %Rrc\n",
+                                  MsrRange.szName, rc);
+
+            rc = CFGMR3QueryU64Def(pNode, "WrGpMask", &MsrRange.fWrGpMask, 0);
+            if (RT_FAILURE(rc))
+                return VMSetError(pVM, rc, RT_SRC_POS, "Invalid MSR entry '%s': Error querying 'WrGpMask' value: %Rrc\n",
+                                  MsrRange.szName, rc);
+
+            rc = CFGMR3QueryU64Def(pNode, "WrIgnMask", &MsrRange.fWrIgnMask, 0);
+            if (RT_FAILURE(rc))
+                return VMSetError(pVM, rc, RT_SRC_POS, "Invalid MSR entry '%s': Error querying 'WrIgnMask' value: %Rrc\n",
+                                  MsrRange.szName, rc);
+        }
+        else
+            return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS,
+                              "Invalid MSR entry '%s': Unknown type '%s'\n", MsrRange.szName, szType);
+
+        /*
+         * Insert the range into the table (replaces/splits/shrinks existing
+         * MSR ranges).
+         */
+        rc = cpumR3MsrRangesInsert(NULL /* pVM */, &pVM->cpum.s.GuestInfo.paMsrRangesR3, &pVM->cpum.s.GuestInfo.cMsrRanges,
+                                   &MsrRange);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Error adding MSR entry '%s': %Rrc\n", MsrRange.szName, rc);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads CPUID leaf overrides.
+ *
+ * This must be called before the CPUID leaves are moved from the normal
+ * heap to the hyper heap!
+ *
+ * @returns VBox status code (VMSetError called).
+ * @param   pVM             The cross context VM structure.
+ * @param   pParentNode     The CFGM node with the CPUID leaves.
+ * @param   pszLabel        How to label the overrides we're loading.
+ */
+static int cpumR3LoadCpuIdOverrides(PVM pVM, PCFGMNODE pParentNode, const char *pszLabel)
+{
+    for (PCFGMNODE pNode = CFGMR3GetFirstChild(pParentNode); pNode; pNode = CFGMR3GetNextChild(pNode))
+    {
+        /*
+         * Get the leaf and subleaf numbers.
+         */
+        char szName[128];
+        int rc = CFGMR3GetName(pNode, szName, sizeof(szName));
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Invalid %s entry (name is probably too long): %Rrc\n", pszLabel, rc);
+
+        /* The leaf number is either specified directly or thru the node name. */
+        uint32_t uLeaf;
+        rc = CFGMR3QueryU32(pNode, "Leaf", &uLeaf);
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        {
+            rc = RTStrToUInt32Full(szName, 16, &uLeaf);
+            if (rc != VINF_SUCCESS)
+                return VMSetError(pVM, VERR_INVALID_NAME, RT_SRC_POS,
+                                  "Invalid %s entry: Invalid leaf number: '%s' \n", pszLabel, szName);
+        }
+        else if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Invalid %s entry '%s': Error querying 'Leaf' value: %Rrc\n",
+                              pszLabel, szName, rc);
+
+        uint32_t uSubLeaf;
+        rc = CFGMR3QueryU32Def(pNode, "SubLeaf", &uSubLeaf, 0);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Invalid %s entry '%s': Error querying 'SubLeaf' value: %Rrc\n",
+                              pszLabel, szName, rc);
+
+        uint32_t fSubLeafMask;
+        rc = CFGMR3QueryU32Def(pNode, "SubLeafMask", &fSubLeafMask, 0);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Invalid %s entry '%s': Error querying 'SubLeafMask' value: %Rrc\n",
+                              pszLabel, szName, rc);
+
+        /*
+         * Look up the specified leaf, since the output register values
+         * defaults to any existing values.  This allows overriding a single
+         * register, without needing to know the other values.
+         */
+        PCCPUMCPUIDLEAF pLeaf = cpumR3CpuIdGetExactLeaf(&pVM->cpum.s, uLeaf, uSubLeaf);
+        CPUMCPUIDLEAF   Leaf;
+        if (pLeaf)
+            Leaf = *pLeaf;
+        else
+            RT_ZERO(Leaf);
+        Leaf.uLeaf          = uLeaf;
+        Leaf.uSubLeaf       = uSubLeaf;
+        Leaf.fSubLeafMask   = fSubLeafMask;
+
+        rc = CFGMR3QueryU32Def(pNode, "eax", &Leaf.uEax, Leaf.uEax);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Invalid %s entry '%s': Error querying 'eax' value: %Rrc\n",
+                              pszLabel, szName, rc);
+        rc = CFGMR3QueryU32Def(pNode, "ebx", &Leaf.uEbx, Leaf.uEbx);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Invalid %s entry '%s': Error querying 'ebx' value: %Rrc\n",
+                              pszLabel, szName, rc);
+        rc = CFGMR3QueryU32Def(pNode, "ecx", &Leaf.uEcx, Leaf.uEcx);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Invalid %s entry '%s': Error querying 'ecx' value: %Rrc\n",
+                              pszLabel, szName, rc);
+        rc = CFGMR3QueryU32Def(pNode, "edx", &Leaf.uEdx, Leaf.uEdx);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Invalid %s entry '%s': Error querying 'edx' value: %Rrc\n",
+                              pszLabel, szName, rc);
+
+        /*
+         * Insert the leaf into the table (replaces existing ones).
+         */
+        rc = cpumR3CpuIdInsert(NULL /* pVM */, &pVM->cpum.s.GuestInfo.paCpuIdLeavesR3, &pVM->cpum.s.GuestInfo.cCpuIdLeaves,
+                               &Leaf);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Error adding CPUID leaf entry '%s': %Rrc\n", szName, rc);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Fetches overrides for a CPUID leaf.
+ *
+ * @returns VBox status code.
+ * @param   pLeaf               The leaf to load the overrides into.
+ * @param   pCfgNode            The CFGM node containing the overrides
+ *                              (/CPUM/HostCPUID/ or /CPUM/CPUID/).
+ * @param   iLeaf               The CPUID leaf number.
+ */
+static int cpumR3CpuIdFetchLeafOverride(PCPUMCPUID pLeaf, PCFGMNODE pCfgNode, uint32_t iLeaf)
+{
+    PCFGMNODE pLeafNode = CFGMR3GetChildF(pCfgNode, "%RX32", iLeaf);
+    if (pLeafNode)
+    {
+        uint32_t u32;
+        int rc = CFGMR3QueryU32(pLeafNode, "eax", &u32);
+        if (RT_SUCCESS(rc))
+            pLeaf->uEax = u32;
+        else
+            AssertReturn(rc == VERR_CFGM_VALUE_NOT_FOUND, rc);
+
+        rc = CFGMR3QueryU32(pLeafNode, "ebx", &u32);
+        if (RT_SUCCESS(rc))
+            pLeaf->uEbx = u32;
+        else
+            AssertReturn(rc == VERR_CFGM_VALUE_NOT_FOUND, rc);
+
+        rc = CFGMR3QueryU32(pLeafNode, "ecx", &u32);
+        if (RT_SUCCESS(rc))
+            pLeaf->uEcx = u32;
+        else
+            AssertReturn(rc == VERR_CFGM_VALUE_NOT_FOUND, rc);
+
+        rc = CFGMR3QueryU32(pLeafNode, "edx", &u32);
+        if (RT_SUCCESS(rc))
+            pLeaf->uEdx = u32;
+        else
+            AssertReturn(rc == VERR_CFGM_VALUE_NOT_FOUND, rc);
+
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Load the overrides for a set of CPUID leaves.
+ *
+ * @returns VBox status code.
+ * @param   paLeaves            The leaf array.
+ * @param   cLeaves             The number of leaves.
+ * @param   uStart              The start leaf number.
+ * @param   pCfgNode            The CFGM node containing the overrides
+ *                              (/CPUM/HostCPUID/ or /CPUM/CPUID/).
+ */
+static int cpumR3CpuIdInitLoadOverrideSet(uint32_t uStart, PCPUMCPUID paLeaves, uint32_t cLeaves, PCFGMNODE pCfgNode)
+{
+    for (uint32_t i = 0; i < cLeaves; i++)
+    {
+        int rc = cpumR3CpuIdFetchLeafOverride(&paLeaves[i], pCfgNode, uStart + i);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Installs the CPUID leaves and explods the data into structures like
+ * GuestFeatures and CPUMCTX::aoffXState.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pCpum       The CPUM part of @a VM.
+ * @param   paLeaves    The leaves.  These will be copied (but not freed).
+ * @param   cLeaves     The number of leaves.
+ * @param   pMsrs       The MSRs.
+ */
+static int cpumR3CpuIdInstallAndExplodeLeaves(PVM pVM, PCPUM pCpum, PCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, PCCPUMMSRS pMsrs)
+{
+    cpumR3CpuIdAssertOrder(paLeaves, cLeaves);
+
+    /*
+     * Install the CPUID information.
+     */
+    int rc = MMHyperDupMem(pVM, paLeaves, sizeof(paLeaves[0]) * cLeaves, 32,
+                           MM_TAG_CPUM_CPUID, (void **)&pCpum->GuestInfo.paCpuIdLeavesR3);
+
+    AssertLogRelRCReturn(rc, rc);
+    pCpum->GuestInfo.cCpuIdLeaves = cLeaves;
+    pCpum->GuestInfo.paCpuIdLeavesR0 = MMHyperR3ToR0(pVM, pCpum->GuestInfo.paCpuIdLeavesR3);
+    Assert(MMHyperR0ToR3(pVM, pCpum->GuestInfo.paCpuIdLeavesR0) == (void *)pCpum->GuestInfo.paCpuIdLeavesR3);
+
+    /*
+     * Update the default CPUID leaf if necessary.
+     */
+    switch (pCpum->GuestInfo.enmUnknownCpuIdMethod)
+    {
+        case CPUMUNKNOWNCPUID_LAST_STD_LEAF:
+        case CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX:
+        {
+            /* We don't use CPUID(0).eax here because of the NT hack that only
+               changes that value without actually removing any leaves. */
+            uint32_t i = 0;
+            if (   pCpum->GuestInfo.cCpuIdLeaves > 0
+                && pCpum->GuestInfo.paCpuIdLeavesR3[0].uLeaf <= UINT32_C(0xff))
+            {
+                while (   i + 1 < pCpum->GuestInfo.cCpuIdLeaves
+                       && pCpum->GuestInfo.paCpuIdLeavesR3[i + 1].uLeaf <= UINT32_C(0xff))
+                    i++;
+                pCpum->GuestInfo.DefCpuId.uEax = pCpum->GuestInfo.paCpuIdLeavesR3[i].uEax;
+                pCpum->GuestInfo.DefCpuId.uEbx = pCpum->GuestInfo.paCpuIdLeavesR3[i].uEbx;
+                pCpum->GuestInfo.DefCpuId.uEcx = pCpum->GuestInfo.paCpuIdLeavesR3[i].uEcx;
+                pCpum->GuestInfo.DefCpuId.uEdx = pCpum->GuestInfo.paCpuIdLeavesR3[i].uEdx;
+            }
+            break;
+        }
+        default:
+            break;
+    }
+
+    /*
+     * Explode the guest CPU features.
+     */
+    rc = cpumR3CpuIdExplodeFeatures(pCpum->GuestInfo.paCpuIdLeavesR3, pCpum->GuestInfo.cCpuIdLeaves, pMsrs,
+                                    &pCpum->GuestFeatures);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Adjust the scalable bus frequency according to the CPUID information
+     * we're now using.
+     */
+    if (CPUMMICROARCH_IS_INTEL_CORE7(pVM->cpum.s.GuestFeatures.enmMicroarch))
+        pCpum->GuestInfo.uScalableBusFreq = pCpum->GuestFeatures.enmMicroarch >= kCpumMicroarch_Intel_Core7_SandyBridge
+                                          ? UINT64_C(100000000)  /* 100MHz */
+                                          : UINT64_C(133333333); /* 133MHz */
+
+    /*
+     * Populate the legacy arrays.  Currently used for everything, later only
+     * for patch manager.
+     */
+    struct { PCPUMCPUID paCpuIds; uint32_t cCpuIds, uBase; } aOldRanges[] =
+    {
+        { pCpum->aGuestCpuIdPatmStd,        RT_ELEMENTS(pCpum->aGuestCpuIdPatmStd),     0x00000000 },
+        { pCpum->aGuestCpuIdPatmExt,        RT_ELEMENTS(pCpum->aGuestCpuIdPatmExt),     0x80000000 },
+        { pCpum->aGuestCpuIdPatmCentaur,    RT_ELEMENTS(pCpum->aGuestCpuIdPatmCentaur), 0xc0000000 },
+    };
+    for (uint32_t i = 0; i < RT_ELEMENTS(aOldRanges); i++)
+    {
+        uint32_t    cLeft       = aOldRanges[i].cCpuIds;
+        uint32_t    uLeaf       = aOldRanges[i].uBase + cLeft;
+        PCPUMCPUID  pLegacyLeaf = &aOldRanges[i].paCpuIds[cLeft];
+        while (cLeft-- > 0)
+        {
+            uLeaf--;
+            pLegacyLeaf--;
+
+            PCCPUMCPUIDLEAF pLeaf = cpumR3CpuIdGetExactLeaf(pCpum, uLeaf, 0 /* uSubLeaf */);
+            if (pLeaf)
+            {
+                pLegacyLeaf->uEax = pLeaf->uEax;
+                pLegacyLeaf->uEbx = pLeaf->uEbx;
+                pLegacyLeaf->uEcx = pLeaf->uEcx;
+                pLegacyLeaf->uEdx = pLeaf->uEdx;
+            }
+            else
+                *pLegacyLeaf = pCpum->GuestInfo.DefCpuId;
+        }
+    }
+
+    /*
+     * Configure XSAVE offsets according to the CPUID info and set the feature flags.
+     */
+    PVMCPU pVCpu0 = pVM->apCpusR3[0];
+    memset(&pVCpu0->cpum.s.Guest.aoffXState[0], 0xff, sizeof(pVCpu0->cpum.s.Guest.aoffXState));
+    pVCpu0->cpum.s.Guest.aoffXState[XSAVE_C_X87_BIT] = 0;
+    pVCpu0->cpum.s.Guest.aoffXState[XSAVE_C_SSE_BIT] = 0;
+    for (uint32_t iComponent = XSAVE_C_SSE_BIT + 1; iComponent < 63; iComponent++)
+        if (pCpum->fXStateGuestMask & RT_BIT_64(iComponent))
+        {
+            PCPUMCPUIDLEAF pSubLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 0xd, iComponent);
+            AssertLogRelMsgReturn(pSubLeaf, ("iComponent=%#x\n", iComponent), VERR_CPUM_IPE_1);
+            AssertLogRelMsgReturn(pSubLeaf->fSubLeafMask >= iComponent, ("iComponent=%#x\n", iComponent), VERR_CPUM_IPE_1);
+            AssertLogRelMsgReturn(   pSubLeaf->uEax > 0
+                                  && pSubLeaf->uEbx >= CPUM_MIN_XSAVE_AREA_SIZE
+                                  && pSubLeaf->uEax <= pCpum->GuestFeatures.cbMaxExtendedState
+                                  && pSubLeaf->uEbx <= pCpum->GuestFeatures.cbMaxExtendedState
+                                  && pSubLeaf->uEbx + pSubLeaf->uEax <= pCpum->GuestFeatures.cbMaxExtendedState,
+                                  ("iComponent=%#x eax=%#x ebx=%#x cbMax=%#x\n", iComponent, pSubLeaf->uEax, pSubLeaf->uEbx,
+                                   pCpum->GuestFeatures.cbMaxExtendedState),
+                                  VERR_CPUM_IPE_1);
+            pVCpu0->cpum.s.Guest.aoffXState[iComponent] = pSubLeaf->uEbx;
+        }
+
+    /* Copy the CPU #0  data to the other CPUs. */
+    for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        memcpy(&pVCpu->cpum.s.Guest.aoffXState[0], &pVCpu0->cpum.s.Guest.aoffXState[0], sizeof(pVCpu0->cpum.s.Guest.aoffXState));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/** @name Instruction Set Extension Options
+ * @{  */
+/** Configuration option type (extended boolean, really). */
+typedef uint8_t CPUMISAEXTCFG;
+/** Always disable the extension. */
+#define CPUMISAEXTCFG_DISABLED              false
+/** Enable the extension if it's supported by the host CPU. */
+#define CPUMISAEXTCFG_ENABLED_SUPPORTED     true
+/** Enable the extension if it's supported by the host CPU, but don't let
+ * the portable CPUID feature disable it. */
+#define CPUMISAEXTCFG_ENABLED_PORTABLE      UINT8_C(127)
+/** Always enable the extension. */
+#define CPUMISAEXTCFG_ENABLED_ALWAYS        UINT8_C(255)
+/** @} */
+
+/**
+ * CPUID Configuration (from CFGM).
+ *
+ * @remarks  The members aren't document since we would only be duplicating the
+ *           \@cfgm entries in cpumR3CpuIdReadConfig.
+ */
+typedef struct CPUMCPUIDCONFIG
+{
+    bool            fNt4LeafLimit;
+    bool            fInvariantTsc;
+    bool            fForceVme;
+    bool            fNestedHWVirt;
+
+    CPUMISAEXTCFG   enmCmpXchg16b;
+    CPUMISAEXTCFG   enmMonitor;
+    CPUMISAEXTCFG   enmMWaitExtensions;
+    CPUMISAEXTCFG   enmSse41;
+    CPUMISAEXTCFG   enmSse42;
+    CPUMISAEXTCFG   enmAvx;
+    CPUMISAEXTCFG   enmAvx2;
+    CPUMISAEXTCFG   enmXSave;
+    CPUMISAEXTCFG   enmAesNi;
+    CPUMISAEXTCFG   enmPClMul;
+    CPUMISAEXTCFG   enmPopCnt;
+    CPUMISAEXTCFG   enmMovBe;
+    CPUMISAEXTCFG   enmRdRand;
+    CPUMISAEXTCFG   enmRdSeed;
+    CPUMISAEXTCFG   enmCLFlushOpt;
+    CPUMISAEXTCFG   enmFsGsBase;
+    CPUMISAEXTCFG   enmPcid;
+    CPUMISAEXTCFG   enmInvpcid;
+    CPUMISAEXTCFG   enmFlushCmdMsr;
+    CPUMISAEXTCFG   enmMdsClear;
+    CPUMISAEXTCFG   enmArchCapMsr;
+
+    CPUMISAEXTCFG   enmAbm;
+    CPUMISAEXTCFG   enmSse4A;
+    CPUMISAEXTCFG   enmMisAlnSse;
+    CPUMISAEXTCFG   enm3dNowPrf;
+    CPUMISAEXTCFG   enmAmdExtMmx;
+
+    uint32_t        uMaxStdLeaf;
+    uint32_t        uMaxExtLeaf;
+    uint32_t        uMaxCentaurLeaf;
+    uint32_t        uMaxIntelFamilyModelStep;
+    char            szCpuName[128];
+} CPUMCPUIDCONFIG;
+/** Pointer to CPUID config (from CFGM). */
+typedef CPUMCPUIDCONFIG *PCPUMCPUIDCONFIG;
+
+
+/**
+ * Mini CPU selection support for making Mac OS X happy.
+ *
+ * Executes the  /CPUM/MaxIntelFamilyModelStep config.
+ *
+ * @param   pCpum       The CPUM instance data.
+ * @param   pConfig     The CPUID configuration we've read from CFGM.
+ */
+static void cpumR3CpuIdLimitIntelFamModStep(PCPUM pCpum, PCPUMCPUIDCONFIG pConfig)
+{
+    if (pCpum->GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        PCPUMCPUIDLEAF pStdFeatureLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 1, 0);
+        uint32_t uCurIntelFamilyModelStep = RT_MAKE_U32_FROM_U8(ASMGetCpuStepping(pStdFeatureLeaf->uEax),
+                                                                ASMGetCpuModelIntel(pStdFeatureLeaf->uEax),
+                                                                ASMGetCpuFamily(pStdFeatureLeaf->uEax),
+                                                                0);
+        uint32_t uMaxIntelFamilyModelStep = pConfig->uMaxIntelFamilyModelStep;
+        if (pConfig->uMaxIntelFamilyModelStep < uCurIntelFamilyModelStep)
+        {
+            uint32_t uNew = pStdFeatureLeaf->uEax & UINT32_C(0xf0003000);
+            uNew |= RT_BYTE1(uMaxIntelFamilyModelStep) & 0xf; /* stepping */
+            uNew |= (RT_BYTE2(uMaxIntelFamilyModelStep) & 0xf) << 4; /* 4 low model bits */
+            uNew |= (RT_BYTE2(uMaxIntelFamilyModelStep) >> 4) << 16; /* 4 high model bits */
+            uNew |= (RT_BYTE3(uMaxIntelFamilyModelStep) & 0xf) << 8; /* 4 low family bits */
+            if (RT_BYTE3(uMaxIntelFamilyModelStep) > 0xf) /* 8 high family bits, using intel's suggested calculation. */
+                uNew |= ( (RT_BYTE3(uMaxIntelFamilyModelStep) - (RT_BYTE3(uMaxIntelFamilyModelStep) & 0xf)) & 0xff ) << 20;
+            LogRel(("CPU: CPUID(0).EAX %#x -> %#x (uMaxIntelFamilyModelStep=%#x, uCurIntelFamilyModelStep=%#x\n",
+                    pStdFeatureLeaf->uEax, uNew, uMaxIntelFamilyModelStep, uCurIntelFamilyModelStep));
+            pStdFeatureLeaf->uEax = uNew;
+        }
+    }
+}
+
+
+
+/**
+ * Limit it the number of entries, zapping the remainder.
+ *
+ * The limits are masking off stuff about power saving and similar, this
+ * is perhaps a bit crudely done as there is probably some relatively harmless
+ * info too in these leaves (like words about having a constant TSC).
+ *
+ * @param   pCpum       The CPUM instance data.
+ * @param   pConfig     The CPUID configuration we've read from CFGM.
+ */
+static void cpumR3CpuIdLimitLeaves(PCPUM pCpum, PCPUMCPUIDCONFIG pConfig)
+{
+    /*
+     * Standard leaves.
+     */
+    uint32_t       uSubLeaf = 0;
+    PCPUMCPUIDLEAF pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 0, uSubLeaf);
+    if (pCurLeaf)
+    {
+        uint32_t uLimit = pCurLeaf->uEax;
+        if (uLimit <= UINT32_C(0x000fffff))
+        {
+            if (uLimit > pConfig->uMaxStdLeaf)
+            {
+                pCurLeaf->uEax = uLimit = pConfig->uMaxStdLeaf;
+                cpumR3CpuIdRemoveRange(pCpum->GuestInfo.paCpuIdLeavesR3, &pCpum->GuestInfo.cCpuIdLeaves,
+                                       uLimit + 1, UINT32_C(0x000fffff));
+            }
+
+            /* NT4 hack, no zapping of extra leaves here. */
+            if (pConfig->fNt4LeafLimit && uLimit > 3)
+                pCurLeaf->uEax = uLimit = 3;
+
+            while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0x00000000), ++uSubLeaf)) != NULL)
+                pCurLeaf->uEax = uLimit;
+        }
+        else
+        {
+            LogRel(("CPUID: Invalid standard range: %#x\n", uLimit));
+            cpumR3CpuIdRemoveRange(pCpum->GuestInfo.paCpuIdLeavesR3, &pCpum->GuestInfo.cCpuIdLeaves,
+                                   UINT32_C(0x00000000), UINT32_C(0x0fffffff));
+        }
+    }
+
+    /*
+     * Extended leaves.
+     */
+    uSubLeaf = 0;
+    pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0x80000000), uSubLeaf);
+    if (pCurLeaf)
+    {
+        uint32_t uLimit = pCurLeaf->uEax;
+        if (   uLimit >= UINT32_C(0x80000000)
+            && uLimit <= UINT32_C(0x800fffff))
+        {
+            if (uLimit > pConfig->uMaxExtLeaf)
+            {
+                pCurLeaf->uEax = uLimit = pConfig->uMaxExtLeaf;
+                cpumR3CpuIdRemoveRange(pCpum->GuestInfo.paCpuIdLeavesR3, &pCpum->GuestInfo.cCpuIdLeaves,
+                                       uLimit + 1, UINT32_C(0x800fffff));
+                while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0x80000000), ++uSubLeaf)) != NULL)
+                    pCurLeaf->uEax = uLimit;
+            }
+        }
+        else
+        {
+            LogRel(("CPUID: Invalid extended range: %#x\n", uLimit));
+            cpumR3CpuIdRemoveRange(pCpum->GuestInfo.paCpuIdLeavesR3, &pCpum->GuestInfo.cCpuIdLeaves,
+                                   UINT32_C(0x80000000), UINT32_C(0x8ffffffd));
+        }
+    }
+
+    /*
+     * Centaur leaves (VIA).
+     */
+    uSubLeaf = 0;
+    pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0xc0000000), uSubLeaf);
+    if (pCurLeaf)
+    {
+        uint32_t uLimit = pCurLeaf->uEax;
+        if (   uLimit >= UINT32_C(0xc0000000)
+            && uLimit <= UINT32_C(0xc00fffff))
+        {
+            if (uLimit > pConfig->uMaxCentaurLeaf)
+            {
+                pCurLeaf->uEax = uLimit = pConfig->uMaxCentaurLeaf;
+                cpumR3CpuIdRemoveRange(pCpum->GuestInfo.paCpuIdLeavesR3, &pCpum->GuestInfo.cCpuIdLeaves,
+                                       uLimit + 1, UINT32_C(0xcfffffff));
+                while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0xc0000000), ++uSubLeaf)) != NULL)
+                    pCurLeaf->uEax = uLimit;
+            }
+        }
+        else
+        {
+            LogRel(("CPUID: Invalid centaur range: %#x\n", uLimit));
+            cpumR3CpuIdRemoveRange(pCpum->GuestInfo.paCpuIdLeavesR3, &pCpum->GuestInfo.cCpuIdLeaves,
+                                   UINT32_C(0xc0000000), UINT32_C(0xcfffffff));
+        }
+    }
+}
+
+
+/**
+ * Clears a CPUID leaf and all sub-leaves (to zero).
+ *
+ * @param   pCpum       The CPUM instance data.
+ * @param   uLeaf       The leaf to clear.
+ */
+static void cpumR3CpuIdZeroLeaf(PCPUM pCpum, uint32_t uLeaf)
+{
+    uint32_t       uSubLeaf = 0;
+    PCPUMCPUIDLEAF pCurLeaf;
+    while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, uLeaf, uSubLeaf)) != NULL)
+    {
+        pCurLeaf->uEax = 0;
+        pCurLeaf->uEbx = 0;
+        pCurLeaf->uEcx = 0;
+        pCurLeaf->uEdx = 0;
+        uSubLeaf++;
+    }
+}
+
+
+/**
+ * Used by cpumR3CpuIdSanitize to ensure that we don't have any sub-leaves for
+ * the given leaf.
+ *
+ * @returns pLeaf.
+ * @param   pCpum       The CPUM instance data.
+ * @param   pLeaf       The leaf to ensure is alone with it's EAX input value.
+ */
+static PCPUMCPUIDLEAF cpumR3CpuIdMakeSingleLeaf(PCPUM pCpum, PCPUMCPUIDLEAF pLeaf)
+{
+    Assert((uintptr_t)(pLeaf - pCpum->GuestInfo.paCpuIdLeavesR3) < pCpum->GuestInfo.cCpuIdLeaves);
+    if (pLeaf->fSubLeafMask != 0)
+    {
+        /*
+         * Figure out how many sub-leaves in need of removal (we'll keep the first).
+         * Log everything while we're at it.
+         */
+        LogRel(("CPUM:\n"
+                "CPUM: Unexpected CPUID sub-leaves for leaf %#x; fSubLeafMask=%#x\n", pLeaf->uLeaf, pLeaf->fSubLeafMask));
+        PCPUMCPUIDLEAF  pLast    = &pCpum->GuestInfo.paCpuIdLeavesR3[pCpum->GuestInfo.cCpuIdLeaves - 1];
+        PCPUMCPUIDLEAF  pSubLeaf = pLeaf;
+        for (;;)
+        {
+            LogRel(("CPUM: %08x/%08x: %08x %08x %08x %08x; flags=%#x mask=%#x\n",
+                    pSubLeaf->uLeaf, pSubLeaf->uSubLeaf,
+                    pSubLeaf->uEax, pSubLeaf->uEbx, pSubLeaf->uEcx, pSubLeaf->uEdx,
+                    pSubLeaf->fFlags, pSubLeaf->fSubLeafMask));
+            if (pSubLeaf == pLast || pSubLeaf[1].uLeaf != pLeaf->uLeaf)
+                break;
+            pSubLeaf++;
+        }
+        LogRel(("CPUM:\n"));
+
+        /*
+         * Remove the offending sub-leaves.
+         */
+        if (pSubLeaf != pLeaf)
+        {
+            if (pSubLeaf != pLast)
+                memmove(pLeaf + 1, pSubLeaf + 1, (uintptr_t)pLast - (uintptr_t)pSubLeaf);
+            pCpum->GuestInfo.cCpuIdLeaves -= (uint32_t)(pSubLeaf - pLeaf);
+        }
+
+        /*
+         * Convert the first sub-leaf into a single leaf.
+         */
+        pLeaf->uSubLeaf     = 0;
+        pLeaf->fSubLeafMask = 0;
+    }
+    return pLeaf;
+}
+
+
+/**
+ * Sanitizes and adjust the CPUID leaves.
+ *
+ * Drop features that aren't virtualized (or virtualizable).  Adjust information
+ * and capabilities to fit the virtualized hardware.  Remove information the
+ * guest shouldn't have (because it's wrong in the virtual world or because it
+ * gives away host details) or that we don't have documentation for and no idea
+ * what means.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure (for cCpus).
+ * @param   pCpum       The CPUM instance data.
+ * @param   pConfig     The CPUID configuration we've read from CFGM.
+ */
+static int cpumR3CpuIdSanitize(PVM pVM, PCPUM pCpum, PCPUMCPUIDCONFIG pConfig)
+{
+#define PORTABLE_CLEAR_BITS_WHEN(Lvl, a_pLeafReg, FeatNm, fMask, uValue) \
+    if ( pCpum->u8PortableCpuIdLevel >= (Lvl) && ((a_pLeafReg) & (fMask)) == (uValue) ) \
+    { \
+        LogRel(("PortableCpuId: " #a_pLeafReg "[" #FeatNm "]: %#x -> 0\n", (a_pLeafReg) & (fMask))); \
+        (a_pLeafReg) &= ~(uint32_t)(fMask); \
+    }
+#define PORTABLE_DISABLE_FEATURE_BIT(Lvl, a_pLeafReg, FeatNm, fBitMask) \
+    if ( pCpum->u8PortableCpuIdLevel >= (Lvl) && ((a_pLeafReg) & (fBitMask)) ) \
+    { \
+        LogRel(("PortableCpuId: " #a_pLeafReg "[" #FeatNm "]: 1 -> 0\n")); \
+        (a_pLeafReg) &= ~(uint32_t)(fBitMask); \
+    }
+#define PORTABLE_DISABLE_FEATURE_BIT_CFG(Lvl, a_pLeafReg, FeatNm, fBitMask, enmConfig) \
+    if (   pCpum->u8PortableCpuIdLevel >= (Lvl) \
+        && ((a_pLeafReg) & (fBitMask)) \
+        && (enmConfig) != CPUMISAEXTCFG_ENABLED_PORTABLE ) \
+    { \
+        LogRel(("PortableCpuId: " #a_pLeafReg "[" #FeatNm "]: 1 -> 0\n")); \
+        (a_pLeafReg) &= ~(uint32_t)(fBitMask); \
+    }
+    Assert(pCpum->GuestFeatures.enmCpuVendor != CPUMCPUVENDOR_INVALID);
+
+    /* Cpuid 1:
+     * EAX: CPU model, family and stepping.
+     *
+     * ECX + EDX: Supported features.  Only report features we can support.
+     * Note! When enabling new features the Synthetic CPU and Portable CPUID
+     *       options may require adjusting (i.e. stripping what was enabled).
+     *
+     * EBX: Branding, CLFLUSH line size, logical processors per package and
+     *      initial APIC ID.
+     */
+    PCPUMCPUIDLEAF pStdFeatureLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 1, 0); /* Note! Must refetch when used later. */
+    AssertLogRelReturn(pStdFeatureLeaf, VERR_CPUM_IPE_2);
+    pStdFeatureLeaf = cpumR3CpuIdMakeSingleLeaf(pCpum, pStdFeatureLeaf);
+
+    pStdFeatureLeaf->uEdx &= X86_CPUID_FEATURE_EDX_FPU
+                           | X86_CPUID_FEATURE_EDX_VME
+                           | X86_CPUID_FEATURE_EDX_DE
+                           | X86_CPUID_FEATURE_EDX_PSE
+                           | X86_CPUID_FEATURE_EDX_TSC
+                           | X86_CPUID_FEATURE_EDX_MSR
+                           //| X86_CPUID_FEATURE_EDX_PAE   - set later if configured.
+                           | X86_CPUID_FEATURE_EDX_MCE
+                           | X86_CPUID_FEATURE_EDX_CX8
+                           //| X86_CPUID_FEATURE_EDX_APIC  - set by the APIC device if present.
+                           //| RT_BIT_32(10)               - not defined
+                           /* Note! we don't report sysenter/sysexit support due to our inability to keep the IOPL part of eflags in sync while in ring 1 (see @bugref{1757}) */
+                           //| X86_CPUID_FEATURE_EDX_SEP
+                           | X86_CPUID_FEATURE_EDX_MTRR
+                           | X86_CPUID_FEATURE_EDX_PGE
+                           | X86_CPUID_FEATURE_EDX_MCA
+                           | X86_CPUID_FEATURE_EDX_CMOV
+                           | X86_CPUID_FEATURE_EDX_PAT     /* 16 */
+                           | X86_CPUID_FEATURE_EDX_PSE36
+                           //| X86_CPUID_FEATURE_EDX_PSN   - no serial number.
+                           | X86_CPUID_FEATURE_EDX_CLFSH
+                           //| RT_BIT_32(20)               - not defined
+                           //| X86_CPUID_FEATURE_EDX_DS    - no debug store.
+                           //| X86_CPUID_FEATURE_EDX_ACPI  - not supported (not DevAcpi, right?).
+                           | X86_CPUID_FEATURE_EDX_MMX
+                           | X86_CPUID_FEATURE_EDX_FXSR
+                           | X86_CPUID_FEATURE_EDX_SSE
+                           | X86_CPUID_FEATURE_EDX_SSE2
+                           //| X86_CPUID_FEATURE_EDX_SS    - no self snoop.
+                           | X86_CPUID_FEATURE_EDX_HTT
+                           //| X86_CPUID_FEATURE_EDX_TM    - no thermal monitor.
+                           //| RT_BIT_32(30)               - not defined
+                           //| X86_CPUID_FEATURE_EDX_PBE   - no pending break enabled.
+                           ;
+    pStdFeatureLeaf->uEcx &= 0
+                           | X86_CPUID_FEATURE_ECX_SSE3
+                           | (pConfig->enmPClMul ? X86_CPUID_FEATURE_ECX_PCLMUL : 0)
+                           //| X86_CPUID_FEATURE_ECX_DTES64 - not implemented yet.
+                           /* Can't properly emulate monitor & mwait with guest SMP; force the guest to use hlt for idling VCPUs. */
+                           | ((pConfig->enmMonitor && pVM->cCpus == 1) ? X86_CPUID_FEATURE_ECX_MONITOR : 0)
+                           //| X86_CPUID_FEATURE_ECX_CPLDS - no CPL qualified debug store.
+                           | (pConfig->fNestedHWVirt ? X86_CPUID_FEATURE_ECX_VMX : 0)
+                           //| X86_CPUID_FEATURE_ECX_SMX   - not virtualized yet.
+                           //| X86_CPUID_FEATURE_ECX_EST   - no extended speed step.
+                           //| X86_CPUID_FEATURE_ECX_TM2   - no thermal monitor 2.
+                           | X86_CPUID_FEATURE_ECX_SSSE3
+                           //| X86_CPUID_FEATURE_ECX_CNTXID - no L1 context id (MSR++).
+                           //| X86_CPUID_FEATURE_ECX_FMA   - not implemented yet.
+                           | (pConfig->enmCmpXchg16b ? X86_CPUID_FEATURE_ECX_CX16 : 0)
+                           /* ECX Bit 14 - xTPR Update Control. Processor supports changing IA32_MISC_ENABLES[bit 23]. */
+                           //| X86_CPUID_FEATURE_ECX_TPRUPDATE
+                           //| X86_CPUID_FEATURE_ECX_PDCM  - not implemented yet.
+                           | (pConfig->enmPcid ? X86_CPUID_FEATURE_ECX_PCID : 0)
+                           //| X86_CPUID_FEATURE_ECX_DCA   - not implemented yet.
+                           | (pConfig->enmSse41 ? X86_CPUID_FEATURE_ECX_SSE4_1 : 0)
+                           | (pConfig->enmSse42 ? X86_CPUID_FEATURE_ECX_SSE4_2 : 0)
+                           //| X86_CPUID_FEATURE_ECX_X2APIC - turned on later by the device if enabled.
+                           | (pConfig->enmMovBe ? X86_CPUID_FEATURE_ECX_MOVBE : 0)
+                           | (pConfig->enmPopCnt ? X86_CPUID_FEATURE_ECX_POPCNT : 0)
+                           //| X86_CPUID_FEATURE_ECX_TSCDEADL - not implemented yet.
+                           | (pConfig->enmAesNi ? X86_CPUID_FEATURE_ECX_AES : 0)
+                           | (pConfig->enmXSave ? X86_CPUID_FEATURE_ECX_XSAVE : 0 )
+                           //| X86_CPUID_FEATURE_ECX_OSXSAVE - mirrors CR4.OSXSAVE state, set dynamically.
+                           | (pConfig->enmAvx ? X86_CPUID_FEATURE_ECX_AVX : 0)
+                           //| X86_CPUID_FEATURE_ECX_F16C  - not implemented yet.
+                           | (pConfig->enmRdRand ? X86_CPUID_FEATURE_ECX_RDRAND : 0)
+                           //| X86_CPUID_FEATURE_ECX_HVP   - Set explicitly later.
+                           ;
+
+    /* Mask out PCID unless FSGSBASE is exposed due to a bug in Windows 10 SMP guests, see @bugref{9089#c15}. */
+    if (   !pVM->cpum.s.GuestFeatures.fFsGsBase
+        && (pStdFeatureLeaf->uEcx & X86_CPUID_FEATURE_ECX_PCID))
+    {
+        pStdFeatureLeaf->uEcx &= ~X86_CPUID_FEATURE_ECX_PCID;
+        LogRel(("CPUM: Disabled PCID without FSGSBASE to workaround buggy guests\n"));
+    }
+
+    if (pCpum->u8PortableCpuIdLevel > 0)
+    {
+        PORTABLE_CLEAR_BITS_WHEN(1, pStdFeatureLeaf->uEax, ProcessorType, (UINT32_C(3) << 12), (UINT32_C(2) << 12));
+        PORTABLE_DISABLE_FEATURE_BIT(    1, pStdFeatureLeaf->uEcx, SSSE3,  X86_CPUID_FEATURE_ECX_SSSE3);
+        PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pStdFeatureLeaf->uEcx, PCID,   X86_CPUID_FEATURE_ECX_PCID,   pConfig->enmPcid);
+        PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pStdFeatureLeaf->uEcx, SSE4_1, X86_CPUID_FEATURE_ECX_SSE4_1, pConfig->enmSse41);
+        PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pStdFeatureLeaf->uEcx, SSE4_2, X86_CPUID_FEATURE_ECX_SSE4_2, pConfig->enmSse42);
+        PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pStdFeatureLeaf->uEcx, MOVBE,  X86_CPUID_FEATURE_ECX_MOVBE,  pConfig->enmMovBe);
+        PORTABLE_DISABLE_FEATURE_BIT(    1, pStdFeatureLeaf->uEcx, AES,    X86_CPUID_FEATURE_ECX_AES);
+        PORTABLE_DISABLE_FEATURE_BIT(    1, pStdFeatureLeaf->uEcx, VMX,    X86_CPUID_FEATURE_ECX_VMX);
+        PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pStdFeatureLeaf->uEcx, PCLMUL, X86_CPUID_FEATURE_ECX_PCLMUL, pConfig->enmPClMul);
+        PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pStdFeatureLeaf->uEcx, POPCNT, X86_CPUID_FEATURE_ECX_POPCNT, pConfig->enmPopCnt);
+        PORTABLE_DISABLE_FEATURE_BIT(    1, pStdFeatureLeaf->uEcx, F16C,   X86_CPUID_FEATURE_ECX_F16C);
+        PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pStdFeatureLeaf->uEcx, XSAVE,  X86_CPUID_FEATURE_ECX_XSAVE,  pConfig->enmXSave);
+        PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pStdFeatureLeaf->uEcx, AVX,    X86_CPUID_FEATURE_ECX_AVX,    pConfig->enmAvx);
+        PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pStdFeatureLeaf->uEcx, RDRAND, X86_CPUID_FEATURE_ECX_RDRAND, pConfig->enmRdRand);
+        PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pStdFeatureLeaf->uEcx, CX16,   X86_CPUID_FEATURE_ECX_CX16,   pConfig->enmCmpXchg16b);
+        PORTABLE_DISABLE_FEATURE_BIT(    2, pStdFeatureLeaf->uEcx, SSE3,   X86_CPUID_FEATURE_ECX_SSE3);
+        PORTABLE_DISABLE_FEATURE_BIT(    3, pStdFeatureLeaf->uEdx, SSE2,   X86_CPUID_FEATURE_EDX_SSE2);
+        PORTABLE_DISABLE_FEATURE_BIT(    3, pStdFeatureLeaf->uEdx, SSE,    X86_CPUID_FEATURE_EDX_SSE);
+        PORTABLE_DISABLE_FEATURE_BIT(    3, pStdFeatureLeaf->uEdx, CLFSH,  X86_CPUID_FEATURE_EDX_CLFSH);
+        PORTABLE_DISABLE_FEATURE_BIT(    3, pStdFeatureLeaf->uEdx, CMOV,   X86_CPUID_FEATURE_EDX_CMOV);
+
+        Assert(!(pStdFeatureLeaf->uEdx & (  X86_CPUID_FEATURE_EDX_SEP
+                                          | X86_CPUID_FEATURE_EDX_PSN
+                                          | X86_CPUID_FEATURE_EDX_DS
+                                          | X86_CPUID_FEATURE_EDX_ACPI
+                                          | X86_CPUID_FEATURE_EDX_SS
+                                          | X86_CPUID_FEATURE_EDX_TM
+                                          | X86_CPUID_FEATURE_EDX_PBE
+                                          )));
+        Assert(!(pStdFeatureLeaf->uEcx & (  X86_CPUID_FEATURE_ECX_DTES64
+                                          | X86_CPUID_FEATURE_ECX_CPLDS
+                                          | X86_CPUID_FEATURE_ECX_AES
+                                          | X86_CPUID_FEATURE_ECX_VMX
+                                          | X86_CPUID_FEATURE_ECX_SMX
+                                          | X86_CPUID_FEATURE_ECX_EST
+                                          | X86_CPUID_FEATURE_ECX_TM2
+                                          | X86_CPUID_FEATURE_ECX_CNTXID
+                                          | X86_CPUID_FEATURE_ECX_FMA
+                                          | X86_CPUID_FEATURE_ECX_TPRUPDATE
+                                          | X86_CPUID_FEATURE_ECX_PDCM
+                                          | X86_CPUID_FEATURE_ECX_DCA
+                                          | X86_CPUID_FEATURE_ECX_OSXSAVE
+                                          )));
+    }
+
+    /* Set up APIC ID for CPU 0, configure multi core/threaded smp. */
+    pStdFeatureLeaf->uEbx &= UINT32_C(0x0000ffff); /* (APIC-ID := 0 and #LogCpus := 0) */
+
+    /* The HTT bit is architectural and does not directly indicate hyper-threading or multiple cores;
+     * it was set even on single-core/non-HT Northwood P4s for example. The HTT bit only means that the
+     * information in EBX[23:16] (max number of addressable logical processor IDs) is valid.
+     */
+#ifdef VBOX_WITH_MULTI_CORE
+    if (pVM->cCpus > 1)
+        pStdFeatureLeaf->uEdx |= X86_CPUID_FEATURE_EDX_HTT;  /* Force if emulating a multi-core CPU. */
+#endif
+    if (pStdFeatureLeaf->uEdx & X86_CPUID_FEATURE_EDX_HTT)
+    {
+        /* If CPUID Fn0000_0001_EDX[HTT] = 1 then LogicalProcessorCount is the number of threads per CPU
+           core times the number of CPU cores per processor */
+#ifdef VBOX_WITH_MULTI_CORE
+        pStdFeatureLeaf->uEbx |= pVM->cCpus <= 0xff ? (pVM->cCpus << 16) : UINT32_C(0x00ff0000);
+#else
+        /* Single logical processor in a package. */
+        pStdFeatureLeaf->uEbx |= (1 << 16);
+#endif
+    }
+
+    uint32_t uMicrocodeRev;
+    int rc = SUPR3QueryMicrocodeRev(&uMicrocodeRev);
+    if (RT_SUCCESS(rc))
+    {
+        LogRel(("CPUM: Microcode revision 0x%08X\n", uMicrocodeRev));
+    }
+    else
+    {
+        uMicrocodeRev = 0;
+        LogRel(("CPUM: Failed to query microcode revision. rc=%Rrc\n", rc));
+    }
+
+    /* Mask out the VME capability on certain CPUs, unless overridden by fForceVme.
+     * VME bug was fixed in AGESA 1.0.0.6, microcode patch level 8001126.
+     */
+    if (   (   pVM->cpum.s.GuestFeatures.enmMicroarch == kCpumMicroarch_AMD_Zen_Ryzen
+            /** @todo The following ASSUMES that Hygon uses the same version numbering
+             * as AMD and that they shipped buggy firmware. */
+            || pVM->cpum.s.GuestFeatures.enmMicroarch == kCpumMicroarch_Hygon_Dhyana)
+        && uMicrocodeRev < 0x8001126
+        && !pConfig->fForceVme)
+    {
+        /** @todo The above is a very coarse test but at the moment we don't know any better (see @bugref{8852}). */
+        LogRel(("CPUM: Zen VME workaround engaged\n"));
+        pStdFeatureLeaf->uEdx &= ~X86_CPUID_FEATURE_EDX_VME;
+    }
+
+    /* Force standard feature bits. */
+    if (pConfig->enmPClMul == CPUMISAEXTCFG_ENABLED_ALWAYS)
+        pStdFeatureLeaf->uEcx |= X86_CPUID_FEATURE_ECX_PCLMUL;
+    if (pConfig->enmMonitor == CPUMISAEXTCFG_ENABLED_ALWAYS)
+        pStdFeatureLeaf->uEcx |= X86_CPUID_FEATURE_ECX_MONITOR;
+    if (pConfig->enmCmpXchg16b == CPUMISAEXTCFG_ENABLED_ALWAYS)
+        pStdFeatureLeaf->uEcx |= X86_CPUID_FEATURE_ECX_CX16;
+    if (pConfig->enmSse41 == CPUMISAEXTCFG_ENABLED_ALWAYS)
+        pStdFeatureLeaf->uEcx |= X86_CPUID_FEATURE_ECX_SSE4_1;
+    if (pConfig->enmSse42 == CPUMISAEXTCFG_ENABLED_ALWAYS)
+        pStdFeatureLeaf->uEcx |= X86_CPUID_FEATURE_ECX_SSE4_2;
+    if (pConfig->enmMovBe == CPUMISAEXTCFG_ENABLED_ALWAYS)
+        pStdFeatureLeaf->uEcx |= X86_CPUID_FEATURE_ECX_MOVBE;
+    if (pConfig->enmPopCnt == CPUMISAEXTCFG_ENABLED_ALWAYS)
+        pStdFeatureLeaf->uEcx |= X86_CPUID_FEATURE_ECX_POPCNT;
+    if (pConfig->enmAesNi == CPUMISAEXTCFG_ENABLED_ALWAYS)
+        pStdFeatureLeaf->uEcx |= X86_CPUID_FEATURE_ECX_AES;
+    if (pConfig->enmXSave == CPUMISAEXTCFG_ENABLED_ALWAYS)
+        pStdFeatureLeaf->uEcx |= X86_CPUID_FEATURE_ECX_XSAVE;
+    if (pConfig->enmAvx == CPUMISAEXTCFG_ENABLED_ALWAYS)
+        pStdFeatureLeaf->uEcx |= X86_CPUID_FEATURE_ECX_AVX;
+    if (pConfig->enmRdRand == CPUMISAEXTCFG_ENABLED_ALWAYS)
+        pStdFeatureLeaf->uEcx |= X86_CPUID_FEATURE_ECX_RDRAND;
+
+    pStdFeatureLeaf = NULL; /* Must refetch! */
+
+    /* Cpuid 0x80000001: (Similar, but in no way identical to 0x00000001.)
+     * AMD:
+     *  EAX: CPU model, family and stepping.
+     *
+     *  ECX + EDX: Supported features.  Only report features we can support.
+     *  Note! When enabling new features the Synthetic CPU and Portable CPUID
+     *        options may require adjusting (i.e. stripping what was enabled).
+     *  ASSUMES that this is ALWAYS the AMD defined feature set if present.
+     *
+     *  EBX: Branding ID and package type (or reserved).
+     *
+     * Intel and probably most others:
+     *  EAX: 0
+     *  EBX: 0
+     *  ECX + EDX: Subset of AMD features, mainly for AMD64 support.
+     */
+    PCPUMCPUIDLEAF pExtFeatureLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0x80000001), 0);
+    if (pExtFeatureLeaf)
+    {
+        pExtFeatureLeaf = cpumR3CpuIdMakeSingleLeaf(pCpum, pExtFeatureLeaf);
+
+        pExtFeatureLeaf->uEdx &= X86_CPUID_AMD_FEATURE_EDX_FPU
+                               | X86_CPUID_AMD_FEATURE_EDX_VME
+                               | X86_CPUID_AMD_FEATURE_EDX_DE
+                               | X86_CPUID_AMD_FEATURE_EDX_PSE
+                               | X86_CPUID_AMD_FEATURE_EDX_TSC
+                               | X86_CPUID_AMD_FEATURE_EDX_MSR //?? this means AMD MSRs..
+                               //| X86_CPUID_AMD_FEATURE_EDX_PAE    - turned on when necessary
+                               //| X86_CPUID_AMD_FEATURE_EDX_MCE    - not virtualized yet.
+                               | X86_CPUID_AMD_FEATURE_EDX_CX8
+                               //| X86_CPUID_AMD_FEATURE_EDX_APIC   - set by the APIC device if present.
+                               //| RT_BIT_32(10)                    - reserved
+                               /* Note! We don't report sysenter/sysexit support due to our inability to keep the IOPL part of
+                                        eflags in sync while in ring 1 (see @bugref{1757}). HM enables them later. */
+                               //| X86_CPUID_EXT_FEATURE_EDX_SYSCALL
+                               | X86_CPUID_AMD_FEATURE_EDX_MTRR
+                               | X86_CPUID_AMD_FEATURE_EDX_PGE
+                               | X86_CPUID_AMD_FEATURE_EDX_MCA
+                               | X86_CPUID_AMD_FEATURE_EDX_CMOV
+                               | X86_CPUID_AMD_FEATURE_EDX_PAT
+                               | X86_CPUID_AMD_FEATURE_EDX_PSE36
+                               //| RT_BIT_32(18)                    - reserved
+                               //| RT_BIT_32(19)                    - reserved
+                               //| X86_CPUID_EXT_FEATURE_EDX_NX     - enabled later by PGM
+                               //| RT_BIT_32(21)                    - reserved
+                               | (pConfig->enmAmdExtMmx ? X86_CPUID_AMD_FEATURE_EDX_AXMMX : 0)
+                               | X86_CPUID_AMD_FEATURE_EDX_MMX
+                               | X86_CPUID_AMD_FEATURE_EDX_FXSR
+                               | X86_CPUID_AMD_FEATURE_EDX_FFXSR
+                               //| X86_CPUID_EXT_FEATURE_EDX_PAGE1GB
+                               | X86_CPUID_EXT_FEATURE_EDX_RDTSCP
+                               //| RT_BIT_32(28)                    - reserved
+                               //| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE - turned on when necessary
+                               | X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX
+                               | X86_CPUID_AMD_FEATURE_EDX_3DNOW
+                               ;
+        pExtFeatureLeaf->uEcx &= X86_CPUID_EXT_FEATURE_ECX_LAHF_SAHF
+                               //| X86_CPUID_AMD_FEATURE_ECX_CMPL   - set below if applicable.
+                               | (pConfig->fNestedHWVirt ? X86_CPUID_AMD_FEATURE_ECX_SVM : 0)
+                               //| X86_CPUID_AMD_FEATURE_ECX_EXT_APIC
+                               /* Note: This could prevent teleporting from AMD to Intel CPUs! */
+                               | X86_CPUID_AMD_FEATURE_ECX_CR8L         /* expose lock mov cr0 = mov cr8 hack for guests that can use this feature to access the TPR. */
+                               | (pConfig->enmAbm       ? X86_CPUID_AMD_FEATURE_ECX_ABM : 0)
+                               | (pConfig->enmSse4A     ? X86_CPUID_AMD_FEATURE_ECX_SSE4A : 0)
+                               | (pConfig->enmMisAlnSse ? X86_CPUID_AMD_FEATURE_ECX_MISALNSSE : 0)
+                               | (pConfig->enm3dNowPrf  ? X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF : 0)
+                               //| X86_CPUID_AMD_FEATURE_ECX_OSVW
+                               //| X86_CPUID_AMD_FEATURE_ECX_IBS
+                               //| X86_CPUID_AMD_FEATURE_ECX_XOP
+                               //| X86_CPUID_AMD_FEATURE_ECX_SKINIT
+                               //| X86_CPUID_AMD_FEATURE_ECX_WDT
+                               //| RT_BIT_32(14)                    - reserved
+                               //| X86_CPUID_AMD_FEATURE_ECX_LWP    - not supported
+                               //| X86_CPUID_AMD_FEATURE_ECX_FMA4   - not yet virtualized.
+                               //| RT_BIT_32(17)                    - reserved
+                               //| RT_BIT_32(18)                    - reserved
+                               //| X86_CPUID_AMD_FEATURE_ECX_NODEID - not yet virtualized.
+                               //| RT_BIT_32(20)                    - reserved
+                               //| X86_CPUID_AMD_FEATURE_ECX_TBM    - not yet virtualized.
+                               //| X86_CPUID_AMD_FEATURE_ECX_TOPOEXT - not yet virtualized.
+                               //| RT_BIT_32(23)                    - reserved
+                               //| RT_BIT_32(24)                    - reserved
+                               //| RT_BIT_32(25)                    - reserved
+                               //| RT_BIT_32(26)                    - reserved
+                               //| RT_BIT_32(27)                    - reserved
+                               //| RT_BIT_32(28)                    - reserved
+                               //| RT_BIT_32(29)                    - reserved
+                               //| RT_BIT_32(30)                    - reserved
+                               //| RT_BIT_32(31)                    - reserved
+                               ;
+#ifdef VBOX_WITH_MULTI_CORE
+        if (   pVM->cCpus > 1
+            && (   pCpum->GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_AMD
+                || pCpum->GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_HYGON))
+            pExtFeatureLeaf->uEcx |= X86_CPUID_AMD_FEATURE_ECX_CMPL; /* CmpLegacy */
+#endif
+
+        if (pCpum->u8PortableCpuIdLevel > 0)
+        {
+            PORTABLE_DISABLE_FEATURE_BIT(    1, pExtFeatureLeaf->uEcx, CR8L,       X86_CPUID_AMD_FEATURE_ECX_CR8L);
+            PORTABLE_DISABLE_FEATURE_BIT(    1, pExtFeatureLeaf->uEcx, SVM,        X86_CPUID_AMD_FEATURE_ECX_SVM);
+            PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pExtFeatureLeaf->uEcx, ABM,        X86_CPUID_AMD_FEATURE_ECX_ABM,       pConfig->enmAbm);
+            PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pExtFeatureLeaf->uEcx, SSE4A,      X86_CPUID_AMD_FEATURE_ECX_SSE4A,     pConfig->enmSse4A);
+            PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pExtFeatureLeaf->uEcx, MISALNSSE,  X86_CPUID_AMD_FEATURE_ECX_MISALNSSE, pConfig->enmMisAlnSse);
+            PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pExtFeatureLeaf->uEcx, 3DNOWPRF,   X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF,  pConfig->enm3dNowPrf);
+            PORTABLE_DISABLE_FEATURE_BIT(    1, pExtFeatureLeaf->uEcx, XOP,        X86_CPUID_AMD_FEATURE_ECX_XOP);
+            PORTABLE_DISABLE_FEATURE_BIT(    1, pExtFeatureLeaf->uEcx, TBM,        X86_CPUID_AMD_FEATURE_ECX_TBM);
+            PORTABLE_DISABLE_FEATURE_BIT(    1, pExtFeatureLeaf->uEcx, FMA4,       X86_CPUID_AMD_FEATURE_ECX_FMA4);
+            PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pExtFeatureLeaf->uEdx, AXMMX,      X86_CPUID_AMD_FEATURE_EDX_AXMMX,     pConfig->enmAmdExtMmx);
+            PORTABLE_DISABLE_FEATURE_BIT(    1, pExtFeatureLeaf->uEdx, 3DNOW,      X86_CPUID_AMD_FEATURE_EDX_3DNOW);
+            PORTABLE_DISABLE_FEATURE_BIT(    1, pExtFeatureLeaf->uEdx, 3DNOW_EX,   X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX);
+            PORTABLE_DISABLE_FEATURE_BIT(    1, pExtFeatureLeaf->uEdx, FFXSR,      X86_CPUID_AMD_FEATURE_EDX_FFXSR);
+            PORTABLE_DISABLE_FEATURE_BIT(    1, pExtFeatureLeaf->uEdx, RDTSCP,     X86_CPUID_EXT_FEATURE_EDX_RDTSCP);
+            PORTABLE_DISABLE_FEATURE_BIT(    2, pExtFeatureLeaf->uEcx, LAHF_SAHF,  X86_CPUID_EXT_FEATURE_ECX_LAHF_SAHF);
+            PORTABLE_DISABLE_FEATURE_BIT(    3, pExtFeatureLeaf->uEcx, CMOV,       X86_CPUID_AMD_FEATURE_EDX_CMOV);
+
+            Assert(!(pExtFeatureLeaf->uEcx & (  X86_CPUID_AMD_FEATURE_ECX_SVM
+                                              | X86_CPUID_AMD_FEATURE_ECX_EXT_APIC
+                                              | X86_CPUID_AMD_FEATURE_ECX_OSVW
+                                              | X86_CPUID_AMD_FEATURE_ECX_IBS
+                                              | X86_CPUID_AMD_FEATURE_ECX_SKINIT
+                                              | X86_CPUID_AMD_FEATURE_ECX_WDT
+                                              | X86_CPUID_AMD_FEATURE_ECX_LWP
+                                              | X86_CPUID_AMD_FEATURE_ECX_NODEID
+                                              | X86_CPUID_AMD_FEATURE_ECX_TOPOEXT
+                                              | UINT32_C(0xff964000)
+                                              )));
+            Assert(!(pExtFeatureLeaf->uEdx & (  RT_BIT(10)
+                                              | X86_CPUID_EXT_FEATURE_EDX_SYSCALL
+                                              | RT_BIT(18)
+                                              | RT_BIT(19)
+                                              | RT_BIT(21)
+                                              | X86_CPUID_AMD_FEATURE_EDX_AXMMX
+                                              | X86_CPUID_EXT_FEATURE_EDX_PAGE1GB
+                                              | RT_BIT(28)
+                                              )));
+        }
+
+        /* Force extended feature bits. */
+        if (pConfig->enmAbm       == CPUMISAEXTCFG_ENABLED_ALWAYS)
+            pExtFeatureLeaf->uEcx |= X86_CPUID_AMD_FEATURE_ECX_ABM;
+        if (pConfig->enmSse4A     == CPUMISAEXTCFG_ENABLED_ALWAYS)
+            pExtFeatureLeaf->uEcx |= X86_CPUID_AMD_FEATURE_ECX_SSE4A;
+        if (pConfig->enmMisAlnSse == CPUMISAEXTCFG_ENABLED_ALWAYS)
+            pExtFeatureLeaf->uEcx |= X86_CPUID_AMD_FEATURE_ECX_MISALNSSE;
+        if (pConfig->enm3dNowPrf  == CPUMISAEXTCFG_ENABLED_ALWAYS)
+            pExtFeatureLeaf->uEcx |= X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF;
+        if (pConfig->enmAmdExtMmx  == CPUMISAEXTCFG_ENABLED_ALWAYS)
+            pExtFeatureLeaf->uEdx |= X86_CPUID_AMD_FEATURE_EDX_AXMMX;
+    }
+    pExtFeatureLeaf = NULL; /* Must refetch! */
+
+
+    /* Cpuid 2:
+     * Intel: (Nondeterministic) Cache and TLB information
+     * AMD:   Reserved
+     * VIA:   Reserved
+     * Safe to expose.
+     */
+    uint32_t        uSubLeaf = 0;
+    PCPUMCPUIDLEAF  pCurLeaf;
+    while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 2, uSubLeaf)) != NULL)
+    {
+        if ((pCurLeaf->uEax & 0xff) > 1)
+        {
+            LogRel(("CpuId: Std[2].al: %d -> 1\n", pCurLeaf->uEax & 0xff));
+            pCurLeaf->uEax &= UINT32_C(0xffffff01);
+        }
+        uSubLeaf++;
+    }
+
+    /* Cpuid 3:
+     * Intel: EAX, EBX - reserved (transmeta uses these)
+     *        ECX, EDX - Processor Serial Number if available, otherwise reserved
+     * AMD:   Reserved
+     * VIA:   Reserved
+     * Safe to expose
+     */
+    pStdFeatureLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 1, 0);
+    if (!(pStdFeatureLeaf->uEdx & X86_CPUID_FEATURE_EDX_PSN))
+    {
+        uSubLeaf = 0;
+        while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 3, uSubLeaf)) != NULL)
+        {
+            pCurLeaf->uEcx = pCurLeaf->uEdx = 0;
+            if (pCpum->u8PortableCpuIdLevel > 0)
+                pCurLeaf->uEax = pCurLeaf->uEbx = 0;
+            uSubLeaf++;
+        }
+    }
+
+    /* Cpuid 4 + ECX:
+     * Intel: Deterministic Cache Parameters Leaf.
+     * AMD:   Reserved
+     * VIA:   Reserved
+     * Safe to expose, except for EAX:
+     *      Bits 25-14: Maximum number of addressable IDs for logical processors sharing this cache (see note)**
+     *      Bits 31-26: Maximum number of processor cores in this physical package**
+     * Note: These SMP values are constant regardless of ECX
+     */
+    uSubLeaf = 0;
+    while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 4, uSubLeaf)) != NULL)
+    {
+        pCurLeaf->uEax &= UINT32_C(0x00003fff); /* Clear the #maxcores, #threads-sharing-cache (both are #-1).*/
+#ifdef VBOX_WITH_MULTI_CORE
+        if (   pVM->cCpus > 1
+            && pCpum->GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+        {
+            AssertReturn(pVM->cCpus <= 64, VERR_TOO_MANY_CPUS);
+            /* One logical processor with possibly multiple cores. */
+            /* See  http://www.intel.com/Assets/PDF/appnote/241618.pdf p. 29 */
+            pCurLeaf->uEax |= pVM->cCpus <= 0x40 ? ((pVM->cCpus - 1) << 26) : UINT32_C(0xfc000000); /* 6 bits only -> 64 cores! */
+        }
+#endif
+        uSubLeaf++;
+    }
+
+    /* Cpuid 5:     Monitor/mwait Leaf
+     * Intel: ECX, EDX - reserved
+     *        EAX, EBX - Smallest and largest monitor line size
+     * AMD:   EDX - reserved
+     *        EAX, EBX - Smallest and largest monitor line size
+     *        ECX - extensions (ignored for now)
+     * VIA:   Reserved
+     * Safe to expose
+     */
+    uSubLeaf = 0;
+    while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 5, uSubLeaf)) != NULL)
+    {
+        pStdFeatureLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 1, 0);
+        if (!(pStdFeatureLeaf->uEcx & X86_CPUID_FEATURE_ECX_MONITOR))
+            pCurLeaf->uEax = pCurLeaf->uEbx = 0;
+
+        pCurLeaf->uEcx = pCurLeaf->uEdx = 0;
+        if (pConfig->enmMWaitExtensions)
+        {
+            pCurLeaf->uEcx = X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0;
+            /** @todo for now we just expose host's MWAIT C-states, although conceptually
+               it shall be part of our power management virtualization model */
+#if 0
+            /* MWAIT sub C-states */
+            pCurLeaf->uEdx =
+                    (0 << 0)  /* 0 in C0 */ |
+                    (2 << 4)  /* 2 in C1 */ |
+                    (2 << 8)  /* 2 in C2 */ |
+                    (2 << 12) /* 2 in C3 */ |
+                    (0 << 16) /* 0 in C4 */
+                    ;
+#endif
+        }
+        else
+            pCurLeaf->uEcx = pCurLeaf->uEdx = 0;
+        uSubLeaf++;
+    }
+
+    /* Cpuid 6: Digital Thermal Sensor and Power Management Paramenters.
+     * Intel: Various stuff.
+     * AMD: EAX, EBX, EDX - reserved.
+     *      ECX - Bit zero is EffFreq, indicating MSR_0000_00e7 and MSR_0000_00e8
+     *            present.  Same as intel.
+     * VIA: ??
+     *
+     * We clear everything here for now.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 6);
+
+    /* Cpuid 7 + ECX: Structured Extended Feature Flags Enumeration
+     * EAX: Number of sub leaves.
+     * EBX+ECX+EDX: Feature flags
+     *
+     * We only have documentation for one sub-leaf, so clear all other (no need
+     * to remove them as such, just set them to zero).
+     *
+     * Note! When enabling new features the Synthetic CPU and Portable CPUID
+     *       options may require adjusting (i.e. stripping what was enabled).
+     */
+    uSubLeaf = 0;
+    while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 7, uSubLeaf)) != NULL)
+    {
+        switch (uSubLeaf)
+        {
+            case 0:
+            {
+                pCurLeaf->uEax  = 0;    /* Max ECX input is 0. */
+                pCurLeaf->uEbx &= 0
+                               | (pConfig->enmFsGsBase ? X86_CPUID_STEXT_FEATURE_EBX_FSGSBASE : 0)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_TSC_ADJUST        RT_BIT(1)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_SGX               RT_BIT(2)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_BMI1              RT_BIT(3)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_HLE               RT_BIT(4)
+                               | (pConfig->enmAvx2 ? X86_CPUID_STEXT_FEATURE_EBX_AVX2 : 0)
+                               | X86_CPUID_STEXT_FEATURE_EBX_FDP_EXCPTN_ONLY
+                               //| X86_CPUID_STEXT_FEATURE_EBX_SMEP              RT_BIT(7)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_BMI2              RT_BIT(8)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_ERMS              RT_BIT(9)
+                               | (pConfig->enmInvpcid ? X86_CPUID_STEXT_FEATURE_EBX_INVPCID : 0)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_RTM               RT_BIT(11)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_PQM               RT_BIT(12)
+                               | X86_CPUID_STEXT_FEATURE_EBX_DEPR_FPU_CS_DS
+                               //| X86_CPUID_STEXT_FEATURE_EBX_MPE               RT_BIT(14)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_PQE               RT_BIT(15)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_AVX512F           RT_BIT(16)
+                               //| RT_BIT(17) - reserved
+                               | (pConfig->enmRdSeed ? X86_CPUID_STEXT_FEATURE_EBX_RDSEED : 0)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_ADX               RT_BIT(19)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_SMAP              RT_BIT(20)
+                               //| RT_BIT(21) - reserved
+                               //| RT_BIT(22) - reserved
+                               | (pConfig->enmCLFlushOpt ? X86_CPUID_STEXT_FEATURE_EBX_CLFLUSHOPT : 0)
+                               //| RT_BIT(24) - reserved
+                               //| X86_CPUID_STEXT_FEATURE_EBX_INTEL_PT          RT_BIT(25)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_AVX512PF          RT_BIT(26)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_AVX512ER          RT_BIT(27)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_AVX512CD          RT_BIT(28)
+                               //| X86_CPUID_STEXT_FEATURE_EBX_SHA               RT_BIT(29)
+                               //| RT_BIT(30) - reserved
+                               //| RT_BIT(31) - reserved
+                               ;
+                pCurLeaf->uEcx &= 0
+                               //| X86_CPUID_STEXT_FEATURE_ECX_PREFETCHWT1 - we do not do vector functions yet.
+                               ;
+                pCurLeaf->uEdx &= 0
+                               | (pConfig->enmMdsClear ? X86_CPUID_STEXT_FEATURE_EDX_MD_CLEAR : 0)
+                               //| X86_CPUID_STEXT_FEATURE_EDX_IBRS_IBPB         RT_BIT(26)
+                               //| X86_CPUID_STEXT_FEATURE_EDX_STIBP             RT_BIT(27)
+                               | (pConfig->enmFlushCmdMsr ? X86_CPUID_STEXT_FEATURE_EDX_FLUSH_CMD : 0)
+                               | (pConfig->enmArchCapMsr ? X86_CPUID_STEXT_FEATURE_EDX_ARCHCAP : 0)
+                               ;
+
+                /* Mask out INVPCID unless FSGSBASE is exposed due to a bug in Windows 10 SMP guests, see @bugref{9089#c15}. */
+                if (  !pVM->cpum.s.GuestFeatures.fFsGsBase
+                   && (pCurLeaf->uEbx & X86_CPUID_STEXT_FEATURE_EBX_INVPCID))
+                {
+                    pCurLeaf->uEbx &= ~X86_CPUID_STEXT_FEATURE_EBX_INVPCID;
+                    LogRel(("CPUM: Disabled INVPCID without FSGSBASE to work around buggy guests\n"));
+                }
+
+                if (pCpum->u8PortableCpuIdLevel > 0)
+                {
+                    PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pCurLeaf->uEbx, FSGSBASE,   X86_CPUID_STEXT_FEATURE_EBX_FSGSBASE, pConfig->enmFsGsBase);
+                    PORTABLE_DISABLE_FEATURE_BIT(    1, pCurLeaf->uEbx, SGX,        X86_CPUID_STEXT_FEATURE_EBX_SGX);
+                    PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pCurLeaf->uEbx, AVX2,       X86_CPUID_STEXT_FEATURE_EBX_AVX2, pConfig->enmAvx2);
+                    PORTABLE_DISABLE_FEATURE_BIT(    1, pCurLeaf->uEbx, SMEP,       X86_CPUID_STEXT_FEATURE_EBX_SMEP);
+                    PORTABLE_DISABLE_FEATURE_BIT(    1, pCurLeaf->uEbx, BMI2,       X86_CPUID_STEXT_FEATURE_EBX_BMI2);
+                    PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pCurLeaf->uEbx, INVPCID,    X86_CPUID_STEXT_FEATURE_EBX_INVPCID, pConfig->enmInvpcid);
+                    PORTABLE_DISABLE_FEATURE_BIT(    1, pCurLeaf->uEbx, AVX512F,    X86_CPUID_STEXT_FEATURE_EBX_AVX512F);
+                    PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pCurLeaf->uEbx, RDSEED,     X86_CPUID_STEXT_FEATURE_EBX_RDSEED, pConfig->enmRdSeed);
+                    PORTABLE_DISABLE_FEATURE_BIT_CFG(1, pCurLeaf->uEbx, CLFLUSHOPT, X86_CPUID_STEXT_FEATURE_EBX_RDSEED, pConfig->enmCLFlushOpt);
+                    PORTABLE_DISABLE_FEATURE_BIT(    1, pCurLeaf->uEbx, AVX512PF,   X86_CPUID_STEXT_FEATURE_EBX_AVX512PF);
+                    PORTABLE_DISABLE_FEATURE_BIT(    1, pCurLeaf->uEbx, AVX512ER,   X86_CPUID_STEXT_FEATURE_EBX_AVX512ER);
+                    PORTABLE_DISABLE_FEATURE_BIT(    1, pCurLeaf->uEbx, AVX512CD,   X86_CPUID_STEXT_FEATURE_EBX_AVX512CD);
+                    PORTABLE_DISABLE_FEATURE_BIT(    1, pCurLeaf->uEbx, SMAP,       X86_CPUID_STEXT_FEATURE_EBX_SMAP);
+                    PORTABLE_DISABLE_FEATURE_BIT(    1, pCurLeaf->uEbx, SHA,        X86_CPUID_STEXT_FEATURE_EBX_SHA);
+                    PORTABLE_DISABLE_FEATURE_BIT(    1, pCurLeaf->uEcx, PREFETCHWT1, X86_CPUID_STEXT_FEATURE_ECX_PREFETCHWT1);
+                    PORTABLE_DISABLE_FEATURE_BIT_CFG(3, pCurLeaf->uEdx, FLUSH_CMD,  X86_CPUID_STEXT_FEATURE_EDX_FLUSH_CMD, pConfig->enmFlushCmdMsr);
+                    PORTABLE_DISABLE_FEATURE_BIT_CFG(3, pCurLeaf->uEdx, MD_CLEAR,   X86_CPUID_STEXT_FEATURE_EDX_MD_CLEAR, pConfig->enmMdsClear);
+                    PORTABLE_DISABLE_FEATURE_BIT_CFG(3, pCurLeaf->uEdx, ARCHCAP,    X86_CPUID_STEXT_FEATURE_EDX_ARCHCAP, pConfig->enmArchCapMsr);
+                }
+
+                /* Dependencies. */
+                if (!(pCurLeaf->uEdx & X86_CPUID_STEXT_FEATURE_EDX_FLUSH_CMD))
+                    pCurLeaf->uEdx &= ~X86_CPUID_STEXT_FEATURE_EDX_MD_CLEAR;
+
+                /* Force standard feature bits. */
+                if (pConfig->enmFsGsBase == CPUMISAEXTCFG_ENABLED_ALWAYS)
+                    pCurLeaf->uEbx |= X86_CPUID_STEXT_FEATURE_EBX_FSGSBASE;
+                if (pConfig->enmAvx2 == CPUMISAEXTCFG_ENABLED_ALWAYS)
+                    pCurLeaf->uEbx |= X86_CPUID_STEXT_FEATURE_EBX_AVX2;
+                if (pConfig->enmRdSeed == CPUMISAEXTCFG_ENABLED_ALWAYS)
+                    pCurLeaf->uEbx |= X86_CPUID_STEXT_FEATURE_EBX_RDSEED;
+                if (pConfig->enmCLFlushOpt == CPUMISAEXTCFG_ENABLED_ALWAYS)
+                    pCurLeaf->uEbx |= X86_CPUID_STEXT_FEATURE_EBX_CLFLUSHOPT;
+                if (pConfig->enmInvpcid == CPUMISAEXTCFG_ENABLED_ALWAYS)
+                    pCurLeaf->uEbx |= X86_CPUID_STEXT_FEATURE_EBX_INVPCID;
+                if (pConfig->enmFlushCmdMsr == CPUMISAEXTCFG_ENABLED_ALWAYS)
+                    pCurLeaf->uEdx |= X86_CPUID_STEXT_FEATURE_EDX_FLUSH_CMD;
+                if (pConfig->enmMdsClear == CPUMISAEXTCFG_ENABLED_ALWAYS)
+                    pCurLeaf->uEdx |= X86_CPUID_STEXT_FEATURE_EDX_MD_CLEAR;
+                if (pConfig->enmArchCapMsr == CPUMISAEXTCFG_ENABLED_ALWAYS)
+                    pCurLeaf->uEdx |= X86_CPUID_STEXT_FEATURE_EDX_ARCHCAP;
+                break;
+            }
+
+            default:
+                /* Invalid index, all values are zero. */
+                pCurLeaf->uEax = 0;
+                pCurLeaf->uEbx = 0;
+                pCurLeaf->uEcx = 0;
+                pCurLeaf->uEdx = 0;
+                break;
+        }
+        uSubLeaf++;
+    }
+
+    /* Cpuid 8: Marked as reserved by Intel and AMD.
+     * We zero this since we don't know what it may have been used for.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 8);
+
+    /* Cpuid 9: Direct Cache Access (DCA) Parameters
+     * Intel: EAX - Value of PLATFORM_DCA_CAP bits.
+     *        EBX, ECX, EDX - reserved.
+     * AMD:   Reserved
+     * VIA:   ??
+     *
+     * We zero this.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 9);
+
+    /* Cpuid 0xa: Architectural Performance Monitor Features
+     * Intel: EAX - Value of PLATFORM_DCA_CAP bits.
+     *        EBX, ECX, EDX - reserved.
+     * AMD:   Reserved
+     * VIA:   ??
+     *
+     * We zero this, for now at least.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 10);
+
+    /* Cpuid 0xb+ECX: x2APIC Features / Processor Topology.
+     * Intel: EAX - APCI ID shift right for next level.
+     *        EBX - Factory configured cores/threads at this level.
+     *        ECX - Level number (same as input) and level type (1,2,0).
+     *        EDX - Extended initial APIC ID.
+     * AMD:   Reserved
+     * VIA:   ??
+     */
+    uSubLeaf = 0;
+    while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 11, uSubLeaf)) != NULL)
+    {
+        if (pCurLeaf->fFlags & CPUMCPUIDLEAF_F_CONTAINS_APIC_ID)
+        {
+            uint8_t bLevelType = RT_BYTE2(pCurLeaf->uEcx);
+            if (bLevelType == 1)
+            {
+                /* Thread level - we don't do threads at the moment. */
+                pCurLeaf->uEax = 0; /** @todo is this correct? Real CPUs never do 0 here, I think... */
+                pCurLeaf->uEbx = 1;
+            }
+            else if (bLevelType == 2)
+            {
+                /* Core level. */
+                pCurLeaf->uEax = 1; /** @todo real CPUs are supposed to be in the 4-6 range, not 1.  Our APIC ID assignments are a little special... */
+#ifdef VBOX_WITH_MULTI_CORE
+                while (RT_BIT_32(pCurLeaf->uEax) < pVM->cCpus)
+                    pCurLeaf->uEax++;
+#endif
+                pCurLeaf->uEbx = pVM->cCpus;
+            }
+            else
+            {
+                AssertLogRelMsg(bLevelType == 0, ("bLevelType=%#x uSubLeaf=%#x\n", bLevelType, uSubLeaf));
+                pCurLeaf->uEax = 0;
+                pCurLeaf->uEbx = 0;
+                pCurLeaf->uEcx = 0;
+            }
+            pCurLeaf->uEcx = (pCurLeaf->uEcx & UINT32_C(0xffffff00)) | (uSubLeaf & 0xff);
+            pCurLeaf->uEdx = 0;  /* APIC ID is filled in by CPUMGetGuestCpuId() at runtime.  Init for EMT(0) as usual. */
+        }
+        else
+        {
+            pCurLeaf->uEax = 0;
+            pCurLeaf->uEbx = 0;
+            pCurLeaf->uEcx = 0;
+            pCurLeaf->uEdx = 0;
+        }
+        uSubLeaf++;
+    }
+
+    /* Cpuid 0xc: Marked as reserved by Intel and AMD.
+     * We zero this since we don't know what it may have been used for.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 12);
+
+    /* Cpuid 0xd + ECX: Processor Extended State Enumeration
+     * ECX=0:   EAX - Valid bits in XCR0[31:0].
+     *          EBX - Maximum state size as per current XCR0 value.
+     *          ECX - Maximum state size for all supported features.
+     *          EDX - Valid bits in XCR0[63:32].
+     * ECX=1:   EAX - Various X-features.
+     *          EBX - Maximum state size as per current XCR0|IA32_XSS value.
+     *          ECX - Valid bits in IA32_XSS[31:0].
+     *          EDX - Valid bits in IA32_XSS[63:32].
+     * ECX=N, where N in 2..63 and indicates a bit in XCR0 and/or IA32_XSS,
+     *        if the bit invalid all four registers are set to zero.
+     *          EAX - The state size for this feature.
+     *          EBX - The state byte offset of this feature.
+     *          ECX - Bit 0 indicates whether this sub-leaf maps to a valid IA32_XSS bit (=1) or a valid XCR0 bit (=0).
+     *          EDX - Reserved, but is set to zero if invalid sub-leaf index.
+     *
+     * Clear them all as we don't currently implement extended CPU state.
+     */
+    /* Figure out the supported XCR0/XSS mask component and make sure CPUID[1].ECX[27] = CR4.OSXSAVE. */
+    uint64_t fGuestXcr0Mask = 0;
+    pStdFeatureLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 1, 0);
+    if (pStdFeatureLeaf && (pStdFeatureLeaf->uEcx & X86_CPUID_FEATURE_ECX_XSAVE))
+    {
+        fGuestXcr0Mask = XSAVE_C_X87 | XSAVE_C_SSE;
+        if (pStdFeatureLeaf && (pStdFeatureLeaf->uEcx & X86_CPUID_FEATURE_ECX_AVX))
+            fGuestXcr0Mask |= XSAVE_C_YMM;
+        pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 7, 0);
+        if (pCurLeaf && (pCurLeaf->uEbx & X86_CPUID_STEXT_FEATURE_EBX_AVX512F))
+            fGuestXcr0Mask |= XSAVE_C_ZMM_16HI | XSAVE_C_ZMM_HI256 | XSAVE_C_OPMASK;
+        fGuestXcr0Mask &= pCpum->fXStateHostMask;
+
+        pStdFeatureLeaf->fFlags |= CPUMCPUIDLEAF_F_CONTAINS_OSXSAVE;
+    }
+    pStdFeatureLeaf = NULL;
+    pCpum->fXStateGuestMask = fGuestXcr0Mask;
+
+    /* Work the sub-leaves. */
+    uint32_t cbXSaveMaxActual = CPUM_MIN_XSAVE_AREA_SIZE;
+    uint32_t cbXSaveMaxReport = CPUM_MIN_XSAVE_AREA_SIZE;
+    for (uSubLeaf = 0; uSubLeaf < 63; uSubLeaf++)
+    {
+        pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 13, uSubLeaf);
+        if (pCurLeaf)
+        {
+            if (fGuestXcr0Mask)
+            {
+                switch (uSubLeaf)
+                {
+                    case 0:
+                        pCurLeaf->uEax &= RT_LO_U32(fGuestXcr0Mask);
+                        pCurLeaf->uEdx &= RT_HI_U32(fGuestXcr0Mask);
+                        AssertLogRelMsgReturn((pCurLeaf->uEax & (XSAVE_C_X87 | XSAVE_C_SSE)) == (XSAVE_C_X87 | XSAVE_C_SSE),
+                                              ("CPUID(0xd/0).EAX missing mandatory X87 or SSE bits: %#RX32", pCurLeaf->uEax),
+                                              VERR_CPUM_IPE_1);
+                        cbXSaveMaxActual = pCurLeaf->uEcx;
+                        AssertLogRelMsgReturn(cbXSaveMaxActual <= CPUM_MAX_XSAVE_AREA_SIZE && cbXSaveMaxActual >= CPUM_MIN_XSAVE_AREA_SIZE,
+                                              ("%#x max=%#x\n", cbXSaveMaxActual, CPUM_MAX_XSAVE_AREA_SIZE), VERR_CPUM_IPE_2);
+                        AssertLogRelMsgReturn(pCurLeaf->uEbx >= CPUM_MIN_XSAVE_AREA_SIZE && pCurLeaf->uEbx <= cbXSaveMaxActual,
+                                              ("ebx=%#x cbXSaveMaxActual=%#x\n", pCurLeaf->uEbx, cbXSaveMaxActual),
+                                              VERR_CPUM_IPE_2);
+                        continue;
+                    case 1:
+                        pCurLeaf->uEax &= 0;
+                        pCurLeaf->uEcx &= 0;
+                        pCurLeaf->uEdx &= 0;
+                        /** @todo what about checking ebx? */
+                        continue;
+                    default:
+                        if (fGuestXcr0Mask & RT_BIT_64(uSubLeaf))
+                        {
+                            AssertLogRelMsgReturn(   pCurLeaf->uEax <= cbXSaveMaxActual
+                                                  && pCurLeaf->uEax >  0
+                                                  && pCurLeaf->uEbx < cbXSaveMaxActual
+                                                  && pCurLeaf->uEbx >= CPUM_MIN_XSAVE_AREA_SIZE
+                                                  && pCurLeaf->uEbx + pCurLeaf->uEax <= cbXSaveMaxActual,
+                                                  ("%#x: eax=%#x ebx=%#x cbMax=%#x\n",
+                                                   uSubLeaf, pCurLeaf->uEax, pCurLeaf->uEbx, cbXSaveMaxActual),
+                                                  VERR_CPUM_IPE_2);
+                            AssertLogRel(!(pCurLeaf->uEcx & 1));
+                            pCurLeaf->uEcx = 0; /* Bit 0 should be zero (XCR0), the reset are reserved... */
+                            pCurLeaf->uEdx = 0; /* it's reserved... */
+                            if (pCurLeaf->uEbx + pCurLeaf->uEax > cbXSaveMaxReport)
+                                cbXSaveMaxReport = pCurLeaf->uEbx + pCurLeaf->uEax;
+                            continue;
+                        }
+                        break;
+                }
+            }
+
+            /* Clear the leaf. */
+            pCurLeaf->uEax = 0;
+            pCurLeaf->uEbx = 0;
+            pCurLeaf->uEcx = 0;
+            pCurLeaf->uEdx = 0;
+        }
+    }
+
+    /* Update the max and current feature sizes to shut up annoying Linux kernels. */
+    if (cbXSaveMaxReport != cbXSaveMaxActual && fGuestXcr0Mask)
+    {
+        pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 13, 0);
+        if (pCurLeaf)
+        {
+            LogRel(("CPUM: Changing leaf 13[0]: EBX=%#RX32 -> %#RX32, ECX=%#RX32 -> %#RX32\n",
+                    pCurLeaf->uEbx, cbXSaveMaxReport, pCurLeaf->uEcx, cbXSaveMaxReport));
+            pCurLeaf->uEbx = cbXSaveMaxReport;
+            pCurLeaf->uEcx = cbXSaveMaxReport;
+        }
+    }
+
+    /* Cpuid 0xe: Marked as reserved by Intel and AMD.
+     * We zero this since we don't know what it may have been used for.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 14);
+
+    /* Cpuid 0xf + ECX: Platform quality of service monitoring (PQM),
+     * also known as Intel Resource Director Technology (RDT) Monitoring
+     * We zero this as we don't currently virtualize PQM.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 15);
+
+    /* Cpuid 0x10 + ECX: Platform quality of service enforcement (PQE),
+     * also known as Intel Resource Director Technology (RDT) Allocation
+     * We zero this as we don't currently virtualize PQE.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 16);
+
+    /* Cpuid 0x11: Marked as reserved by Intel and AMD.
+     * We zero this since we don't know what it may have been used for.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 17);
+
+    /* Cpuid 0x12 + ECX: SGX resource enumeration.
+     * We zero this as we don't currently virtualize this.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 18);
+
+    /* Cpuid 0x13: Marked as reserved by Intel and AMD.
+     * We zero this since we don't know what it may have been used for.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 19);
+
+    /* Cpuid 0x14 + ECX: Processor Trace (PT) capability enumeration.
+     * We zero this as we don't currently virtualize this.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 20);
+
+    /* Cpuid 0x15: Timestamp Counter / Core Crystal Clock info.
+     * Intel: uTscFrequency = uCoreCrystalClockFrequency * EBX / EAX.
+     *        EAX - denominator (unsigned).
+     *        EBX - numerator (unsigned).
+     *        ECX, EDX - reserved.
+     * AMD:   Reserved / undefined / not implemented.
+     * VIA:   Reserved / undefined / not implemented.
+     * We zero this as we don't currently virtualize this.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 21);
+
+    /* Cpuid 0x16: Processor frequency info
+     * Intel: EAX - Core base frequency in MHz.
+     *        EBX - Core maximum frequency in MHz.
+     *        ECX - Bus (reference) frequency in MHz.
+     *        EDX - Reserved.
+     * AMD:   Reserved / undefined / not implemented.
+     * VIA:   Reserved / undefined / not implemented.
+     * We zero this as we don't currently virtualize this.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, 22);
+
+    /* Cpuid 0x17..0x10000000: Unknown.
+     * We don't know these and what they mean, so remove them. */
+    cpumR3CpuIdRemoveRange(pCpum->GuestInfo.paCpuIdLeavesR3, &pCpum->GuestInfo.cCpuIdLeaves,
+                           UINT32_C(0x00000017), UINT32_C(0x0fffffff));
+
+
+    /* CpuId 0x40000000..0x4fffffff: Reserved for hypervisor/emulator.
+     * We remove all these as we're a hypervisor and must provide our own.
+     */
+    cpumR3CpuIdRemoveRange(pCpum->GuestInfo.paCpuIdLeavesR3, &pCpum->GuestInfo.cCpuIdLeaves,
+                           UINT32_C(0x40000000), UINT32_C(0x4fffffff));
+
+
+    /* Cpuid 0x80000000 is harmless. */
+
+    /* Cpuid 0x80000001 is handled with cpuid 1 way up above. */
+
+    /* Cpuid 0x80000002...0x80000004 contains the processor name and is considered harmless. */
+
+    /* Cpuid 0x800000005 & 0x800000006 contain information about L1, L2 & L3 cache and TLB identifiers.
+     * Safe to pass on to the guest.
+     *
+     * AMD:   0x800000005 L1 cache information
+     *        0x800000006 L2/L3 cache information
+     * Intel: 0x800000005 reserved
+     *        0x800000006 L2 cache information
+     * VIA:   0x800000005 TLB and L1 cache information
+     *        0x800000006 L2 cache information
+     */
+
+    /* Cpuid 0x800000007: Advanced Power Management Information.
+     * AMD:   EAX: Processor feedback capabilities.
+     *        EBX: RAS capabilites.
+     *        ECX: Advanced power monitoring interface.
+     *        EDX: Enhanced power management capabilities.
+     * Intel: EAX, EBX, ECX - reserved.
+     *        EDX - Invariant TSC indicator supported (bit 8), the rest is reserved.
+     * VIA:   Reserved
+     * We let the guest see EDX_TSCINVAR (and later maybe EDX_EFRO). Actually, we should set EDX_TSCINVAR.
+     */
+    uSubLeaf = 0;
+    while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0x80000007), uSubLeaf)) != NULL)
+    {
+        pCurLeaf->uEax = pCurLeaf->uEbx = pCurLeaf->uEcx = 0;
+        if (   pCpum->GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_AMD
+            || pCpum->GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_HYGON)
+        {
+            /*
+             * Older 64-bit linux kernels blindly assume that the AMD performance counters work
+             * if X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR is set, see @bugref{7243#c85}. Exposing this
+             * bit is now configurable.
+             */
+            pCurLeaf->uEdx &= 0
+                           //| X86_CPUID_AMD_ADVPOWER_EDX_TS
+                           //| X86_CPUID_AMD_ADVPOWER_EDX_FID
+                           //| X86_CPUID_AMD_ADVPOWER_EDX_VID
+                           //| X86_CPUID_AMD_ADVPOWER_EDX_TTP
+                           //| X86_CPUID_AMD_ADVPOWER_EDX_TM
+                           //| X86_CPUID_AMD_ADVPOWER_EDX_STC
+                           //| X86_CPUID_AMD_ADVPOWER_EDX_MC
+                           //| X86_CPUID_AMD_ADVPOWER_EDX_HWPSTATE
+                             | X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR
+                           //| X86_CPUID_AMD_ADVPOWER_EDX_CPB       RT_BIT(9)
+                           //| X86_CPUID_AMD_ADVPOWER_EDX_EFRO      RT_BIT(10)
+                           //| X86_CPUID_AMD_ADVPOWER_EDX_PFI       RT_BIT(11)
+                           //| X86_CPUID_AMD_ADVPOWER_EDX_PA        RT_BIT(12)
+                           | 0;
+        }
+        else
+            pCurLeaf->uEdx &= X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR;
+        if (!pConfig->fInvariantTsc)
+            pCurLeaf->uEdx &= ~X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR;
+        uSubLeaf++;
+    }
+
+    /* Cpuid 0x80000008:
+     * AMD:               EBX, EDX - reserved
+     *                    EAX: Virtual/Physical/Guest address Size
+     *                    ECX: Number of cores + APICIdCoreIdSize
+     * Intel:             EAX: Virtual/Physical address Size
+     *                    EBX, ECX, EDX - reserved
+     * VIA:               EAX: Virtual/Physical address Size
+     *                    EBX, ECX, EDX - reserved
+     *
+     * We only expose the virtual+pysical address size to the guest atm.
+     * On AMD we set the core count, but not the apic id stuff as we're
+     * currently not doing the apic id assignments in a complatible manner.
+     */
+    uSubLeaf = 0;
+    while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0x80000008), uSubLeaf)) != NULL)
+    {
+        pCurLeaf->uEax &= UINT32_C(0x0000ffff); /* Virtual & physical address sizes only. */
+        pCurLeaf->uEbx  = 0;  /* reserved - [12] == IBPB */
+        pCurLeaf->uEdx  = 0;  /* reserved */
+
+        /* Set APICIdCoreIdSize to zero (use legacy method to determine the number of cores per cpu).
+         * Set core count to 0, indicating 1 core. Adjust if we're in multi core mode on AMD. */
+        pCurLeaf->uEcx = 0;
+#ifdef VBOX_WITH_MULTI_CORE
+        if (   pVM->cCpus > 1
+            && (   pCpum->GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_AMD
+                || pCpum->GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_HYGON))
+            pCurLeaf->uEcx |= (pVM->cCpus - 1) & UINT32_C(0xff);
+#endif
+        uSubLeaf++;
+    }
+
+    /* Cpuid 0x80000009: Reserved
+     * We zero this since we don't know what it may have been used for.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, UINT32_C(0x80000009));
+
+    /* Cpuid 0x8000000a: SVM information on AMD, invalid on Intel.
+     * AMD:   EAX - SVM revision.
+     *        EBX - Number of ASIDs.
+     *        ECX - Reserved.
+     *        EDX - SVM Feature identification.
+     */
+    if (   pCpum->GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_AMD
+        || pCpum->GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_HYGON)
+    {
+        pExtFeatureLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0x80000001), 0);
+        if (   pExtFeatureLeaf
+            && (pExtFeatureLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_SVM))
+        {
+            PCPUMCPUIDLEAF pSvmFeatureLeaf = cpumR3CpuIdGetExactLeaf(pCpum, 0x8000000a, 0);
+            if (pSvmFeatureLeaf)
+            {
+                pSvmFeatureLeaf->uEax  = 0x1;
+                pSvmFeatureLeaf->uEbx  = 0x8000;                                        /** @todo figure out virtual NASID. */
+                pSvmFeatureLeaf->uEcx  = 0;
+                pSvmFeatureLeaf->uEdx &= (  X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE         /** @todo Support other SVM features */
+                                          | X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID
+                                          | X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
+            }
+            else
+            {
+                /* Should never happen. */
+                LogRel(("CPUM: Warning! Expected CPUID leaf 0x8000000a not present! SVM features not exposed to the guest\n"));
+                cpumR3CpuIdZeroLeaf(pCpum, UINT32_C(0x8000000a));
+            }
+        }
+        else
+        {
+            /* If SVM is not supported, this is reserved, zero out. */
+            cpumR3CpuIdZeroLeaf(pCpum, UINT32_C(0x8000000a));
+        }
+    }
+    else
+    {
+        /* Cpuid 0x8000000a: Reserved on Intel.
+         * We zero this since we don't know what it may have been used for.
+         */
+        cpumR3CpuIdZeroLeaf(pCpum, UINT32_C(0x8000000a));
+    }
+
+    /* Cpuid 0x8000000b thru 0x80000018: Reserved
+     * We clear these as we don't know what purpose they might have. */
+    for (uint32_t uLeaf = UINT32_C(0x8000000b); uLeaf <= UINT32_C(0x80000018); uLeaf++)
+        cpumR3CpuIdZeroLeaf(pCpum, uLeaf);
+
+    /* Cpuid 0x80000019: TLB configuration
+     * Seems to be harmless, pass them thru as is. */
+
+    /* Cpuid 0x8000001a: Peformance optimization identifiers.
+     * Strip anything we don't know what is or addresses feature we don't implement. */
+    uSubLeaf = 0;
+    while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0x8000001a), uSubLeaf)) != NULL)
+    {
+        pCurLeaf->uEax &= RT_BIT_32(0) /* FP128 - use 1x128-bit instead of 2x64-bit. */
+                        | RT_BIT_32(1) /* MOVU - Prefere unaligned MOV over MOVL + MOVH. */
+                        //| RT_BIT_32(2) /* FP256 - use 1x256-bit instead of 2x128-bit. */
+                        ;
+        pCurLeaf->uEbx  = 0;  /* reserved */
+        pCurLeaf->uEcx  = 0;  /* reserved */
+        pCurLeaf->uEdx  = 0;  /* reserved */
+        uSubLeaf++;
+    }
+
+    /* Cpuid 0x8000001b: Instruct based sampling (IBS) information.
+     * Clear this as we don't currently virtualize this feature. */
+    cpumR3CpuIdZeroLeaf(pCpum, UINT32_C(0x8000001b));
+
+    /* Cpuid 0x8000001c: Lightweight profiling (LWP) information.
+     * Clear this as we don't currently virtualize this feature. */
+    cpumR3CpuIdZeroLeaf(pCpum, UINT32_C(0x8000001c));
+
+    /* Cpuid 0x8000001d+ECX: Get cache configuration descriptors.
+     * We need to sanitize the cores per cache (EAX[25:14]).
+     *
+     * This is very much the same as Intel's CPUID(4) leaf, except EAX[31:26]
+     * and EDX[2] are reserved here, and EAX[14:25] is documented having a
+     * slightly different meaning.
+     */
+    uSubLeaf = 0;
+    while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0x8000001d), uSubLeaf)) != NULL)
+    {
+#ifdef VBOX_WITH_MULTI_CORE
+        uint32_t cCores = ((pCurLeaf->uEax >> 14) & 0xfff) + 1;
+        if (cCores > pVM->cCpus)
+            cCores = pVM->cCpus;
+        pCurLeaf->uEax &= UINT32_C(0x00003fff);
+        pCurLeaf->uEax |= ((cCores - 1) & 0xfff) << 14;
+#else
+        pCurLeaf->uEax &= UINT32_C(0x00003fff);
+#endif
+        uSubLeaf++;
+    }
+
+    /* Cpuid 0x8000001e: Get APIC / unit / node information.
+     * If AMD, we configure it for our layout (on EMT(0)).  In the multi-core
+     * setup, we have one compute unit with all the cores in it.  Single node.
+     */
+    uSubLeaf = 0;
+    while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0x8000001e), uSubLeaf)) != NULL)
+    {
+        pCurLeaf->uEax = 0; /* Extended APIC ID = EMT(0).idApic (== 0).  */
+        if (pCurLeaf->fFlags & CPUMCPUIDLEAF_F_CONTAINS_APIC_ID)
+        {
+#ifdef VBOX_WITH_MULTI_CORE
+            pCurLeaf->uEbx = pVM->cCpus < 0x100
+                           ? (pVM->cCpus - 1) << 8 : UINT32_C(0x0000ff00); /* Compute unit ID 0, core per unit. */
+#else
+            pCurLeaf->uEbx = 0; /* Compute unit ID 0, 1 core per unit. */
+#endif
+            pCurLeaf->uEcx = 0; /* Node ID 0, 1 node per CPU. */
+        }
+        else
+        {
+            Assert(pCpum->GuestFeatures.enmCpuVendor != CPUMCPUVENDOR_AMD);
+            Assert(pCpum->GuestFeatures.enmCpuVendor != CPUMCPUVENDOR_HYGON);
+            pCurLeaf->uEbx = 0; /* Reserved. */
+            pCurLeaf->uEcx = 0; /* Reserved. */
+        }
+        pCurLeaf->uEdx = 0; /* Reserved. */
+        uSubLeaf++;
+    }
+
+    /* Cpuid 0x8000001f...0x8ffffffd: Unknown.
+     * We don't know these and what they mean, so remove them. */
+    cpumR3CpuIdRemoveRange(pCpum->GuestInfo.paCpuIdLeavesR3, &pCpum->GuestInfo.cCpuIdLeaves,
+                           UINT32_C(0x8000001f), UINT32_C(0x8ffffffd));
+
+    /* Cpuid 0x8ffffffe: Mystery AMD K6 leaf.
+     * Just pass it thru for now. */
+
+    /* Cpuid 0x8fffffff: Mystery hammer time leaf!
+     * Just pass it thru for now. */
+
+    /* Cpuid 0xc0000000: Centaur stuff.
+     * Harmless, pass it thru. */
+
+    /* Cpuid 0xc0000001: Centaur features.
+     * VIA: EAX - Family, model, stepping.
+     *      EDX - Centaur extended feature flags.  Nothing interesting, except may
+     *            FEMMS (bit 5), but VIA marks it as 'reserved', so never mind.
+     *      EBX, ECX - reserved.
+     * We keep EAX but strips the rest.
+     */
+    uSubLeaf = 0;
+    while ((pCurLeaf = cpumR3CpuIdGetExactLeaf(pCpum, UINT32_C(0xc0000001), uSubLeaf)) != NULL)
+    {
+        pCurLeaf->uEbx = 0;
+        pCurLeaf->uEcx = 0;
+        pCurLeaf->uEdx = 0; /* Bits 0 thru 9 are documented on sandpil.org, but we don't want them, except maybe 5 (FEMMS). */
+        uSubLeaf++;
+    }
+
+    /* Cpuid 0xc0000002: Old Centaur Current Performance Data.
+     * We only have fixed stale values, but should be harmless. */
+
+    /* Cpuid 0xc0000003: Reserved.
+     * We zero this since we don't know what it may have been used for.
+     */
+    cpumR3CpuIdZeroLeaf(pCpum, UINT32_C(0xc0000003));
+
+    /* Cpuid 0xc0000004: Centaur Performance Info.
+     * We only have fixed stale values, but should be harmless. */
+
+
+    /* Cpuid 0xc0000005...0xcfffffff: Unknown.
+     * We don't know these and what they mean, so remove them. */
+    cpumR3CpuIdRemoveRange(pCpum->GuestInfo.paCpuIdLeavesR3, &pCpum->GuestInfo.cCpuIdLeaves,
+                           UINT32_C(0xc0000005), UINT32_C(0xcfffffff));
+
+    return VINF_SUCCESS;
+#undef PORTABLE_DISABLE_FEATURE_BIT
+#undef PORTABLE_CLEAR_BITS_WHEN
+}
+
+
+/**
+ * Reads a value in /CPUM/IsaExts/ node.
+ *
+ * @returns VBox status code (error message raised).
+ * @param   pVM             The cross context VM structure. (For errors.)
+ * @param   pIsaExts        The /CPUM/IsaExts node (can be NULL).
+ * @param   pszValueName    The value / extension name.
+ * @param   penmValue       Where to return the choice.
+ * @param   enmDefault      The default choice.
+ */
+static int cpumR3CpuIdReadIsaExtCfg(PVM pVM, PCFGMNODE pIsaExts, const char *pszValueName,
+                                    CPUMISAEXTCFG *penmValue, CPUMISAEXTCFG enmDefault)
+{
+    /*
+     * Try integer encoding first.
+     */
+    uint64_t uValue;
+    int rc = CFGMR3QueryInteger(pIsaExts, pszValueName, &uValue);
+    if (RT_SUCCESS(rc))
+        switch (uValue)
+        {
+            case 0: *penmValue = CPUMISAEXTCFG_DISABLED; break;
+            case 1: *penmValue = CPUMISAEXTCFG_ENABLED_SUPPORTED; break;
+            case 2: *penmValue = CPUMISAEXTCFG_ENABLED_ALWAYS; break;
+            case 9: *penmValue = CPUMISAEXTCFG_ENABLED_PORTABLE; break;
+            default:
+                return VMSetError(pVM, VERR_CPUM_INVALID_CONFIG_VALUE, RT_SRC_POS,
+                                  "Invalid config value for '/CPUM/IsaExts/%s': %llu (expected 0/'disabled', 1/'enabled', 2/'portable', or 9/'forced')",
+                                  pszValueName, uValue);
+        }
+    /*
+     * If missing, use default.
+     */
+    else if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+        *penmValue = enmDefault;
+    else
+    {
+        if (rc == VERR_CFGM_NOT_INTEGER)
+        {
+            /*
+             * Not an integer, try read it as a string.
+             */
+            char szValue[32];
+            rc = CFGMR3QueryString(pIsaExts, pszValueName, szValue, sizeof(szValue));
+            if (RT_SUCCESS(rc))
+            {
+                RTStrToLower(szValue);
+                size_t cchValue = strlen(szValue);
+#define EQ(a_str) (cchValue == sizeof(a_str) - 1U && memcmp(szValue, a_str, sizeof(a_str) - 1))
+                if (     EQ("disabled") || EQ("disable") || EQ("off") || EQ("no"))
+                    *penmValue = CPUMISAEXTCFG_DISABLED;
+                else if (EQ("enabled")  || EQ("enable")  || EQ("on")  || EQ("yes"))
+                    *penmValue = CPUMISAEXTCFG_ENABLED_SUPPORTED;
+                else if (EQ("forced")   || EQ("force")   || EQ("always"))
+                    *penmValue = CPUMISAEXTCFG_ENABLED_ALWAYS;
+                else if (EQ("portable"))
+                    *penmValue = CPUMISAEXTCFG_ENABLED_PORTABLE;
+                else if (EQ("default")  || EQ("def"))
+                    *penmValue = enmDefault;
+                else
+                    return VMSetError(pVM, VERR_CPUM_INVALID_CONFIG_VALUE, RT_SRC_POS,
+                                      "Invalid config value for '/CPUM/IsaExts/%s': '%s' (expected 0/'disabled', 1/'enabled', 2/'portable', or 9/'forced')",
+                                      pszValueName, uValue);
+#undef EQ
+            }
+        }
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, "Error reading config value '/CPUM/IsaExts/%s': %Rrc", pszValueName, rc);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Reads a value in /CPUM/IsaExts/ node, forcing it to DISABLED if wanted.
+ *
+ * @returns VBox status code (error message raised).
+ * @param   pVM             The cross context VM structure. (For errors.)
+ * @param   pIsaExts        The /CPUM/IsaExts node (can be NULL).
+ * @param   pszValueName    The value / extension name.
+ * @param   penmValue       Where to return the choice.
+ * @param   enmDefault      The default choice.
+ * @param   fAllowed        Allowed choice.  Applied both to the result and to
+ *                          the default value.
+ */
+static int cpumR3CpuIdReadIsaExtCfgEx(PVM pVM, PCFGMNODE pIsaExts, const char *pszValueName,
+                                      CPUMISAEXTCFG *penmValue, CPUMISAEXTCFG enmDefault, bool fAllowed)
+{
+    int rc;
+    if (fAllowed)
+        rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, pszValueName, penmValue, enmDefault);
+    else
+    {
+        rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, pszValueName, penmValue, false /*enmDefault*/);
+        if (RT_SUCCESS(rc) && *penmValue == CPUMISAEXTCFG_ENABLED_ALWAYS)
+            LogRel(("CPUM: Ignoring forced '%s'\n", pszValueName));
+        *penmValue = CPUMISAEXTCFG_DISABLED;
+    }
+    return rc;
+}
+
+
+/**
+ * Reads a value in /CPUM/IsaExts/ node that used to be located in /CPUM/.
+ *
+ * @returns VBox status code (error message raised).
+ * @param   pVM             The cross context VM structure. (For errors.)
+ * @param   pIsaExts        The /CPUM/IsaExts node (can be NULL).
+ * @param   pCpumCfg        The /CPUM node (can be NULL).
+ * @param   pszValueName    The value / extension name.
+ * @param   penmValue       Where to return the choice.
+ * @param   enmDefault      The default choice.
+ */
+static int cpumR3CpuIdReadIsaExtCfgLegacy(PVM pVM, PCFGMNODE pIsaExts, PCFGMNODE pCpumCfg, const char *pszValueName,
+                                          CPUMISAEXTCFG *penmValue, CPUMISAEXTCFG enmDefault)
+{
+    if (CFGMR3Exists(pCpumCfg, pszValueName))
+    {
+        if (!CFGMR3Exists(pIsaExts, pszValueName))
+            LogRel(("Warning: /CPUM/%s is deprecated, use /CPUM/IsaExts/%s instead.\n", pszValueName, pszValueName));
+        else
+            return VMSetError(pVM, VERR_DUPLICATE, RT_SRC_POS,
+                              "Duplicate config values '/CPUM/%s' and '/CPUM/IsaExts/%s' - please remove the former!",
+                              pszValueName, pszValueName);
+
+        bool fLegacy;
+        int rc = CFGMR3QueryBoolDef(pCpumCfg, pszValueName, &fLegacy, enmDefault != CPUMISAEXTCFG_DISABLED);
+        if (RT_SUCCESS(rc))
+        {
+            *penmValue = fLegacy;
+            return VINF_SUCCESS;
+        }
+        return VMSetError(pVM, VERR_DUPLICATE, RT_SRC_POS, "Error querying '/CPUM/%s': %Rrc", pszValueName, rc);
+    }
+
+    return cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, pszValueName, penmValue, enmDefault);
+}
+
+
+static int cpumR3CpuIdReadConfig(PVM pVM, PCPUMCPUIDCONFIG pConfig, PCFGMNODE pCpumCfg, bool fNestedPagingAndFullGuestExec)
+{
+    int rc;
+
+    /** @cfgm{/CPUM/PortableCpuIdLevel, 8-bit, 0, 3, 0}
+     * When non-zero CPUID features that could cause portability issues will be
+     * stripped.  The higher the value the more features gets stripped.  Higher
+     * values should only be used when older CPUs are involved since it may
+     * harm performance and maybe also cause problems with specific guests. */
+    rc = CFGMR3QueryU8Def(pCpumCfg, "PortableCpuIdLevel", &pVM->cpum.s.u8PortableCpuIdLevel, 0);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/GuestCpuName, string}
+     * The name of the CPU we're to emulate.  The default is the host CPU.
+     * Note! CPUs other than "host" one is currently unsupported. */
+    rc = CFGMR3QueryStringDef(pCpumCfg, "GuestCpuName", pConfig->szCpuName, sizeof(pConfig->szCpuName), "host");
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/NT4LeafLimit, boolean, false}
+     * Limit the number of standard CPUID leaves to 0..3 to prevent NT4 from
+     * bugchecking with MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED (0x3e).
+     * This option corresponds somewhat to IA32_MISC_ENABLES.BOOT_NT4[bit 22].
+     */
+    rc = CFGMR3QueryBoolDef(pCpumCfg, "NT4LeafLimit", &pConfig->fNt4LeafLimit, false);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/InvariantTsc, boolean, true}
+     * Pass-through the invariant TSC flag in 0x80000007 if available on the host
+     * CPU. On AMD CPUs, users may wish to suppress it to avoid trouble from older
+     * 64-bit linux guests which assume the presence of AMD performance counters
+     * that we do not virtualize.
+     */
+    rc = CFGMR3QueryBoolDef(pCpumCfg, "InvariantTsc", &pConfig->fInvariantTsc, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/ForceVme, boolean, false}
+     * Always expose the VME (Virtual-8086 Mode Extensions) capability if true.
+     * By default the flag is passed thru as is from the host CPU, except
+     * on AMD Ryzen CPUs where it's masked to avoid trouble with XP/Server 2003
+     * guests and DOS boxes in general.
+     */
+    rc = CFGMR3QueryBoolDef(pCpumCfg, "ForceVme", &pConfig->fForceVme, false);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/MaxIntelFamilyModelStep, uint32_t, UINT32_MAX}
+     * Restrict the reported CPU family+model+stepping of intel CPUs.  This is
+     * probably going to be a temporary hack, so don't depend on this.
+     * The 1st byte of the value is the stepping, the 2nd byte value is the model
+     * number and the 3rd byte value is the family, and the 4th value must be zero.
+     */
+    rc = CFGMR3QueryU32Def(pCpumCfg, "MaxIntelFamilyModelStep", &pConfig->uMaxIntelFamilyModelStep, UINT32_MAX);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/MaxStdLeaf, uint32_t, 0x00000016}
+     * The last standard leaf to keep.  The actual last value that is stored in EAX
+     * is RT_MAX(CPUID[0].EAX,/CPUM/MaxStdLeaf).  Leaves beyond the max leaf are
+     * removed.  (This works independently of and differently from NT4LeafLimit.)
+     * The default is usually set to what we're able to reasonably sanitize.
+     */
+    rc = CFGMR3QueryU32Def(pCpumCfg, "MaxStdLeaf", &pConfig->uMaxStdLeaf, UINT32_C(0x00000016));
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/MaxExtLeaf, uint32_t, 0x8000001e}
+     * The last extended leaf to keep.  The actual last value that is stored in EAX
+     * is RT_MAX(CPUID[0x80000000].EAX,/CPUM/MaxStdLeaf).  Leaves beyond the max
+     * leaf are removed.  The default is set to what we're able to sanitize.
+     */
+    rc = CFGMR3QueryU32Def(pCpumCfg, "MaxExtLeaf", &pConfig->uMaxExtLeaf, UINT32_C(0x8000001e));
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/MaxCentaurLeaf, uint32_t, 0xc0000004}
+     * The last extended leaf to keep.  The actual last value that is stored in EAX
+     * is RT_MAX(CPUID[0xc0000000].EAX,/CPUM/MaxCentaurLeaf).  Leaves beyond the max
+     * leaf are removed.  The default is set to what we're able to sanitize.
+     */
+    rc = CFGMR3QueryU32Def(pCpumCfg, "MaxCentaurLeaf", &pConfig->uMaxCentaurLeaf, UINT32_C(0xc0000004));
+    AssertLogRelRCReturn(rc, rc);
+
+    bool fQueryNestedHwvirt = false
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+                           || pVM->cpum.s.HostFeatures.enmCpuVendor == CPUMCPUVENDOR_AMD
+                           || pVM->cpum.s.HostFeatures.enmCpuVendor == CPUMCPUVENDOR_HYGON
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                           || pVM->cpum.s.HostFeatures.enmCpuVendor == CPUMCPUVENDOR_INTEL
+                           || pVM->cpum.s.HostFeatures.enmCpuVendor == CPUMCPUVENDOR_VIA
+#endif
+                           ;
+    if (fQueryNestedHwvirt)
+    {
+        /** @cfgm{/CPUM/NestedHWVirt, bool, false}
+         * Whether to expose the hardware virtualization (VMX/SVM) feature to the guest.
+         * The default is false, and when enabled requires a 64-bit CPU with support for
+         * nested-paging and AMD-V or unrestricted guest mode.
+         */
+        rc = CFGMR3QueryBoolDef(pCpumCfg, "NestedHWVirt", &pConfig->fNestedHWVirt, false);
+        AssertLogRelRCReturn(rc, rc);
+        if (   pConfig->fNestedHWVirt
+            && !fNestedPagingAndFullGuestExec)
+            return VMSetError(pVM, VERR_CPUM_INVALID_HWVIRT_CONFIG, RT_SRC_POS,
+                              "Cannot enable nested VT-x/AMD-V without nested-paging and unresricted guest execution!\n");
+
+        /** @todo Think about enabling this later with NEM/KVM. */
+        if (   pConfig->fNestedHWVirt
+            && VM_IS_NEM_ENABLED(pVM))
+        {
+            LogRel(("CPUM: WARNING! Can't turn on nested VT-x/AMD-V when NEM is used!\n"));
+            pConfig->fNestedHWVirt = false;
+        }
+
+        /** @cfgm{/CPUM/NestedVmxPreemptTimer, bool, true}
+         * Whether to expose the VMX-preemption timer feature to the guest (if also
+         * supported by the host hardware). The default is true, and when disabled will
+         * prevent exposing the VMX-preemption timer feature to the guest even if the host
+         * supports it.
+         */
+        rc = CFGMR3QueryBoolDef(pCpumCfg, "NestedVmxPreemptTimer", &pVM->cpum.s.fNestedVmxPreemptTimer, true);
+        AssertLogRelRCReturn(rc, rc);
+    }
+
+    /*
+     * Instruction Set Architecture (ISA) Extensions.
+     */
+    PCFGMNODE pIsaExts = CFGMR3GetChild(pCpumCfg, "IsaExts");
+    if (pIsaExts)
+    {
+        rc = CFGMR3ValidateConfig(pIsaExts, "/CPUM/IsaExts/",
+                                   "CMPXCHG16B"
+                                  "|MONITOR"
+                                  "|MWaitExtensions"
+                                  "|SSE4.1"
+                                  "|SSE4.2"
+                                  "|XSAVE"
+                                  "|AVX"
+                                  "|AVX2"
+                                  "|AESNI"
+                                  "|PCLMUL"
+                                  "|POPCNT"
+                                  "|MOVBE"
+                                  "|RDRAND"
+                                  "|RDSEED"
+                                  "|CLFLUSHOPT"
+                                  "|FSGSBASE"
+                                  "|PCID"
+                                  "|INVPCID"
+                                  "|FlushCmdMsr"
+                                  "|ABM"
+                                  "|SSE4A"
+                                  "|MISALNSSE"
+                                  "|3DNOWPRF"
+                                  "|AXMMX"
+                                  , "" /*pszValidNodes*/, "CPUM" /*pszWho*/, 0 /*uInstance*/);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /** @cfgm{/CPUM/IsaExts/CMPXCHG16B, boolean, depends}
+     * Expose CMPXCHG16B to the guest if supported by the host. For the time
+     * being the default is to only do this for VMs with nested paging and AMD-V or
+     * unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfgLegacy(pVM, pIsaExts, pCpumCfg, "CMPXCHG16B", &pConfig->enmCmpXchg16b, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/MONITOR, boolean, true}
+     * Expose MONITOR/MWAIT instructions to the guest.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfgLegacy(pVM, pIsaExts, pCpumCfg, "MONITOR", &pConfig->enmMonitor, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/MWaitExtensions, boolean, false}
+     * Expose MWAIT extended features to the guest.  For now we expose just MWAIT
+     * break on interrupt feature (bit 1).
+     */
+    rc = cpumR3CpuIdReadIsaExtCfgLegacy(pVM, pIsaExts, pCpumCfg, "MWaitExtensions", &pConfig->enmMWaitExtensions, false);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/SSE4.1, boolean, true}
+     * Expose SSE4.1 to the guest if available.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfgLegacy(pVM, pIsaExts, pCpumCfg, "SSE4.1", &pConfig->enmSse41, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/SSE4.2, boolean, true}
+     * Expose SSE4.2 to the guest if available.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfgLegacy(pVM, pIsaExts, pCpumCfg, "SSE4.2", &pConfig->enmSse42, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    bool const fMayHaveXSave = fNestedPagingAndFullGuestExec
+                            && pVM->cpum.s.HostFeatures.fXSaveRstor
+                            && pVM->cpum.s.HostFeatures.fOpSysXSaveRstor;
+    uint64_t const fXStateHostMask = pVM->cpum.s.fXStateHostMask;
+
+    /** @cfgm{/CPUM/IsaExts/XSAVE, boolean, depends}
+     * Expose XSAVE/XRSTOR to the guest if available.  For the time being the
+     * default is to only expose this to VMs with nested paging and AMD-V or
+     * unrestricted guest execution mode.  Not possible to force this one without
+     * host support at the moment.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfgEx(pVM, pIsaExts, "XSAVE", &pConfig->enmXSave, fNestedPagingAndFullGuestExec,
+                                    fMayHaveXSave /*fAllowed*/);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/AVX, boolean, depends}
+     * Expose the AVX instruction set extensions to the guest if available and
+     * XSAVE is exposed too.  For the time being the default is to only expose this
+     * to VMs with nested paging and AMD-V or unrestricted guest execution mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfgEx(pVM, pIsaExts, "AVX", &pConfig->enmAvx, fNestedPagingAndFullGuestExec,
+                                    fMayHaveXSave && pConfig->enmXSave && (fXStateHostMask & XSAVE_C_YMM)  /*fAllowed*/);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/AVX2, boolean, depends}
+     * Expose the AVX2 instruction set extensions to the guest if available and
+     * XSAVE is exposed too. For the time being the default is to only expose this
+     * to VMs with nested paging and AMD-V or unrestricted guest execution mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfgEx(pVM, pIsaExts, "AVX2", &pConfig->enmAvx2, fNestedPagingAndFullGuestExec /* temporarily */,
+                                    fMayHaveXSave && pConfig->enmXSave && (fXStateHostMask & XSAVE_C_YMM) /*fAllowed*/);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/AESNI, isaextcfg, depends}
+     * Whether to expose the AES instructions to the guest.  For the time being the
+     * default is to only do this for VMs with nested paging and AMD-V or
+     * unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "AESNI", &pConfig->enmAesNi, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/PCLMUL, isaextcfg, depends}
+     * Whether to expose the PCLMULQDQ instructions to the guest.  For the time
+     * being the default is to only do this for VMs with nested paging and AMD-V or
+     * unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "PCLMUL", &pConfig->enmPClMul, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/POPCNT, isaextcfg, depends}
+     * Whether to expose the POPCNT instructions to the guest.  For the time
+     * being the default is to only do this for VMs with nested paging and AMD-V or
+     * unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "POPCNT", &pConfig->enmPopCnt, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/MOVBE, isaextcfg, depends}
+     * Whether to expose the MOVBE instructions to the guest.  For the time
+     * being the default is to only do this for VMs with nested paging and AMD-V or
+     * unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "MOVBE", &pConfig->enmMovBe, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/RDRAND, isaextcfg, depends}
+     * Whether to expose the RDRAND instructions to the guest.  For the time being
+     * the default is to only do this for VMs with nested paging and AMD-V or
+     * unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "RDRAND", &pConfig->enmRdRand, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/RDSEED, isaextcfg, depends}
+     * Whether to expose the RDSEED instructions to the guest.  For the time being
+     * the default is to only do this for VMs with nested paging and AMD-V or
+     * unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "RDSEED", &pConfig->enmRdSeed, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/CLFLUSHOPT, isaextcfg, depends}
+     * Whether to expose the CLFLUSHOPT instructions to the guest.  For the time
+     * being the default is to only do this for VMs with nested paging and AMD-V or
+     * unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "CLFLUSHOPT", &pConfig->enmCLFlushOpt, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/FSGSBASE, isaextcfg, true}
+     * Whether to expose the read/write FSGSBASE instructions to the guest.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "FSGSBASE", &pConfig->enmFsGsBase, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/PCID, isaextcfg, true}
+     * Whether to expose the PCID feature to the guest.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "PCID", &pConfig->enmPcid, pConfig->enmFsGsBase);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/INVPCID, isaextcfg, true}
+     * Whether to expose the INVPCID instruction to the guest.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "INVPCID", &pConfig->enmInvpcid, pConfig->enmFsGsBase);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/FlushCmdMsr, isaextcfg, true}
+     * Whether to expose the IA32_FLUSH_CMD MSR to the guest.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "FlushCmdMsr", &pConfig->enmFlushCmdMsr, CPUMISAEXTCFG_ENABLED_SUPPORTED);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/MdsClear, isaextcfg, true}
+     * Whether to advertise the VERW and MDS related IA32_FLUSH_CMD MSR bits to
+     * the guest.  Requires FlushCmdMsr to be present too.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "MdsClear", &pConfig->enmMdsClear, CPUMISAEXTCFG_ENABLED_SUPPORTED);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/ArchCapMSr, isaextcfg, true}
+     * Whether to expose the MSR_IA32_ARCH_CAPABILITIES MSR to the guest.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "ArchCapMsr", &pConfig->enmArchCapMsr, CPUMISAEXTCFG_ENABLED_SUPPORTED);
+    AssertLogRelRCReturn(rc, rc);
+
+
+    /* AMD: */
+
+    /** @cfgm{/CPUM/IsaExts/ABM, isaextcfg, depends}
+     * Whether to expose the AMD ABM instructions to the guest.  For the time
+     * being the default is to only do this for VMs with nested paging and AMD-V or
+     * unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "ABM", &pConfig->enmAbm, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/SSE4A, isaextcfg, depends}
+     * Whether to expose the AMD SSE4A instructions to the guest.  For the time
+     * being the default is to only do this for VMs with nested paging and AMD-V or
+     * unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "SSE4A", &pConfig->enmSse4A, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/MISALNSSE, isaextcfg, depends}
+     * Whether to expose the AMD MisAlSse feature (MXCSR flag 17) to the guest.  For
+     * the time being the default is to only do this for VMs with nested paging and
+     * AMD-V or unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "MISALNSSE", &pConfig->enmMisAlnSse, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/3DNOWPRF, isaextcfg, depends}
+     * Whether to expose the AMD 3D Now! prefetch instructions to the guest.
+     * For the time being the default is to only do this for VMs with nested paging
+     * and AMD-V or unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "3DNOWPRF", &pConfig->enm3dNowPrf, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/CPUM/IsaExts/AXMMX, isaextcfg, depends}
+     * Whether to expose the AMD's MMX Extensions to the guest.  For the time being
+     * the default is to only do this for VMs with nested paging and AMD-V or
+     * unrestricted guest mode.
+     */
+    rc = cpumR3CpuIdReadIsaExtCfg(pVM, pIsaExts, "AXMMX", &pConfig->enmAmdExtMmx, fNestedPagingAndFullGuestExec);
+    AssertLogRelRCReturn(rc, rc);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Initializes the emulated CPU's CPUID & MSR information.
+ *
+ * @returns VBox status code.
+ * @param   pVM          The cross context VM structure.
+ * @param   pHostMsrs    Pointer to the host MSRs.
+ */
+int cpumR3InitCpuIdAndMsrs(PVM pVM, PCCPUMMSRS pHostMsrs)
+{
+    Assert(pHostMsrs);
+
+    PCPUM       pCpum    = &pVM->cpum.s;
+    PCFGMNODE   pCpumCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "CPUM");
+
+    /*
+     * Set the fCpuIdApicFeatureVisible flags so the APIC can assume visibility
+     * on construction and manage everything from here on.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        pVCpu->cpum.s.fCpuIdApicFeatureVisible = true;
+    }
+
+    /*
+     * Read the configuration.
+     */
+    CPUMCPUIDCONFIG Config;
+    RT_ZERO(Config);
+
+    int rc = cpumR3CpuIdReadConfig(pVM, &Config, pCpumCfg, HMAreNestedPagingAndFullGuestExecEnabled(pVM));
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Get the guest CPU data from the database and/or the host.
+     *
+     * The CPUID and MSRs are currently living on the regular heap to avoid
+     * fragmenting the hyper heap (and because there isn't/wasn't any realloc
+     * API for the hyper heap).  This means special cleanup considerations.
+     */
+    rc = cpumR3DbGetCpuInfo(Config.szCpuName, &pCpum->GuestInfo);
+    if (RT_FAILURE(rc))
+        return rc == VERR_CPUM_DB_CPU_NOT_FOUND
+             ? VMSetError(pVM, rc, RT_SRC_POS,
+                          "Info on guest CPU '%s' could not be found. Please, select a different CPU.", Config.szCpuName)
+             : rc;
+
+    if (pCpum->GuestInfo.fMxCsrMask & ~pVM->cpum.s.fHostMxCsrMask)
+    {
+        LogRel(("Stripping unsupported MXCSR bits from guest mask: %#x -> %#x (host: %#x)\n", pCpum->GuestInfo.fMxCsrMask,
+                pCpum->GuestInfo.fMxCsrMask & pVM->cpum.s.fHostMxCsrMask, pVM->cpum.s.fHostMxCsrMask));
+        pCpum->GuestInfo.fMxCsrMask &= pVM->cpum.s.fHostMxCsrMask;
+    }
+    LogRel(("CPUM: MXCSR_MASK=%#x (host: %#x)\n", pCpum->GuestInfo.fMxCsrMask, pVM->cpum.s.fHostMxCsrMask));
+
+    /** @cfgm{/CPUM/MSRs/[Name]/[First|Last|Type|Value|...],}
+     * Overrides the guest MSRs.
+     */
+    rc = cpumR3LoadMsrOverrides(pVM, CFGMR3GetChild(pCpumCfg, "MSRs"));
+
+    /** @cfgm{/CPUM/HostCPUID/[000000xx|800000xx|c000000x]/[eax|ebx|ecx|edx],32-bit}
+     * Overrides the CPUID leaf values (from the host CPU usually) used for
+     * calculating the guest CPUID leaves.  This can be used to preserve the CPUID
+     * values when moving a VM to a different machine.  Another use is restricting
+     * (or extending) the feature set exposed to the guest. */
+    if (RT_SUCCESS(rc))
+        rc = cpumR3LoadCpuIdOverrides(pVM, CFGMR3GetChild(pCpumCfg, "HostCPUID"), "HostCPUID");
+
+    if (RT_SUCCESS(rc) && CFGMR3GetChild(pCpumCfg, "CPUID")) /* 2nd override, now discontinued. */
+        rc = VMSetError(pVM, VERR_CFGM_CONFIG_UNKNOWN_NODE, RT_SRC_POS,
+                        "Found unsupported configuration node '/CPUM/CPUID/'. "
+                        "Please use IMachine::setCPUIDLeaf() instead.");
+
+    CPUMMSRS GuestMsrs;
+    RT_ZERO(GuestMsrs);
+
+    /*
+     * Pre-explode the CPUID info.
+     */
+    if (RT_SUCCESS(rc))
+    {
+        rc = cpumR3CpuIdExplodeFeatures(pCpum->GuestInfo.paCpuIdLeavesR3, pCpum->GuestInfo.cCpuIdLeaves, &GuestMsrs,
+                                        &pCpum->GuestFeatures);
+    }
+
+    /*
+     * Sanitize the cpuid information passed on to the guest.
+     */
+    if (RT_SUCCESS(rc))
+    {
+        rc = cpumR3CpuIdSanitize(pVM, pCpum, &Config);
+        if (RT_SUCCESS(rc))
+        {
+            cpumR3CpuIdLimitLeaves(pCpum, &Config);
+            cpumR3CpuIdLimitIntelFamModStep(pCpum, &Config);
+        }
+    }
+
+    /*
+     * Setup MSRs introduced in microcode updates or that are otherwise not in
+     * the CPU profile, but are advertised in the CPUID info we just sanitized.
+     */
+    if (RT_SUCCESS(rc))
+        rc = cpumR3MsrReconcileWithCpuId(pVM);
+    /*
+     * MSR fudging.
+     */
+    if (RT_SUCCESS(rc))
+    {
+        /** @cfgm{/CPUM/FudgeMSRs, boolean, true}
+         * Fudges some common MSRs if not present in the selected CPU database entry.
+         * This is for trying to keep VMs running when moved between different hosts
+         * and different CPU vendors. */
+        bool fEnable;
+        rc = CFGMR3QueryBoolDef(pCpumCfg, "FudgeMSRs", &fEnable, true); AssertRC(rc);
+        if (RT_SUCCESS(rc) && fEnable)
+        {
+            rc = cpumR3MsrApplyFudge(pVM);
+            AssertLogRelRC(rc);
+        }
+    }
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Move the MSR and CPUID arrays over on the hypervisor heap, and explode
+         * guest CPU features again.
+         */
+        void *pvFree = pCpum->GuestInfo.paCpuIdLeavesR3;
+        int rc1 = cpumR3CpuIdInstallAndExplodeLeaves(pVM, pCpum, pCpum->GuestInfo.paCpuIdLeavesR3,
+                                                     pCpum->GuestInfo.cCpuIdLeaves, &GuestMsrs);
+        RTMemFree(pvFree);
+
+        pvFree = pCpum->GuestInfo.paMsrRangesR3;
+        int rc2 = MMHyperDupMem(pVM, pvFree,
+                                sizeof(pCpum->GuestInfo.paMsrRangesR3[0]) * pCpum->GuestInfo.cMsrRanges, 32,
+                                MM_TAG_CPUM_MSRS, (void **)&pCpum->GuestInfo.paMsrRangesR3);
+        RTMemFree(pvFree);
+        AssertLogRelRCReturn(rc1, rc1);
+        AssertLogRelRCReturn(rc2, rc2);
+
+        pCpum->GuestInfo.paMsrRangesR0 = MMHyperR3ToR0(pVM, pCpum->GuestInfo.paMsrRangesR3);
+
+        /*
+         * Finally, initialize guest VMX MSRs.
+         *
+         * This needs to be done -after- exploding guest features and sanitizing CPUID leaves
+         * as constructing VMX capabilities MSRs rely on CPU feature bits like long mode,
+         * unrestricted-guest execution, CR4 feature bits and possibly more in the future.
+         */
+        if (pVM->cpum.s.GuestFeatures.fVmx)
+        {
+            Assert(Config.fNestedHWVirt);
+            cpumR3InitVmxGuestFeaturesAndMsrs(pVM, &pHostMsrs->hwvirt.vmx, &GuestMsrs.hwvirt.vmx);
+
+            /* Copy MSRs to all VCPUs */
+            PCVMXMSRS pVmxMsrs = &GuestMsrs.hwvirt.vmx;
+            for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+            {
+                PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+                memcpy(&pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs, pVmxMsrs, sizeof(*pVmxMsrs));
+            }
+        }
+
+        /*
+         * Some more configuration that we're applying at the end of everything
+         * via the CPUMR3SetGuestCpuIdFeature API.
+         */
+
+        /* Check if PAE was explicitely enabled by the user. */
+        bool fEnable;
+        rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "EnablePAE", &fEnable, false);
+        AssertRCReturn(rc, rc);
+        if (fEnable)
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE);
+
+        /* We don't normally enable NX for raw-mode, so give the user a chance to force it on. */
+        rc = CFGMR3QueryBoolDef(pCpumCfg, "EnableNX", &fEnable, false);
+        AssertRCReturn(rc, rc);
+        if (fEnable)
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
+
+        /* Check if speculation control is enabled. */
+        rc = CFGMR3QueryBoolDef(pCpumCfg, "SpecCtrl", &fEnable, false);
+        AssertRCReturn(rc, rc);
+        if (fEnable)
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SPEC_CTRL);
+
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Failed before switching to hyper heap.
+     */
+    RTMemFree(pCpum->GuestInfo.paCpuIdLeavesR3);
+    pCpum->GuestInfo.paCpuIdLeavesR3 = NULL;
+    RTMemFree(pCpum->GuestInfo.paMsrRangesR3);
+    pCpum->GuestInfo.paMsrRangesR3 = NULL;
+    return rc;
+}
+
+
+/**
+ * Sets a CPUID feature bit during VM initialization.
+ *
+ * Since the CPUID feature bits are generally related to CPU features, other
+ * CPUM configuration like MSRs can also be modified by calls to this API.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   enmFeature      The feature to set.
+ */
+VMMR3_INT_DECL(void) CPUMR3SetGuestCpuIdFeature(PVM pVM, CPUMCPUIDFEATURE enmFeature)
+{
+    PCPUMCPUIDLEAF pLeaf;
+    PCPUMMSRRANGE  pMsrRange;
+
+    switch (enmFeature)
+    {
+        /*
+         * Set the APIC bit in both feature masks.
+         */
+        case CPUMCPUIDFEATURE_APIC:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000001));
+            if (pLeaf && (pLeaf->fFlags & CPUMCPUIDLEAF_F_CONTAINS_APIC))
+                pVM->cpum.s.aGuestCpuIdPatmStd[1].uEdx = pLeaf->uEdx |= X86_CPUID_FEATURE_EDX_APIC;
+
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (pLeaf && (pLeaf->fFlags & CPUMCPUIDLEAF_F_CONTAINS_APIC))
+                pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx = pLeaf->uEdx |= X86_CPUID_AMD_FEATURE_EDX_APIC;
+
+            pVM->cpum.s.GuestFeatures.fApic = 1;
+
+            /* Make sure we've got the APICBASE MSR present. */
+            pMsrRange = cpumLookupMsrRange(pVM, MSR_IA32_APICBASE);
+            if (!pMsrRange)
+            {
+                static CPUMMSRRANGE const s_ApicBase =
+                {
+                    /*.uFirst =*/ MSR_IA32_APICBASE, /*.uLast =*/ MSR_IA32_APICBASE,
+                    /*.enmRdFn =*/ kCpumMsrRdFn_Ia32ApicBase, /*.enmWrFn =*/ kCpumMsrWrFn_Ia32ApicBase,
+                    /*.offCpumCpu =*/ UINT16_MAX, /*.fReserved =*/ 0, /*.uValue =*/ 0, /*.fWrIgnMask =*/ 0, /*.fWrGpMask =*/ 0,
+                    /*.szName = */ "IA32_APIC_BASE"
+                };
+                int rc = CPUMR3MsrRangesInsert(pVM, &s_ApicBase);
+                AssertLogRelRC(rc);
+            }
+
+            LogRel(("CPUM: SetGuestCpuIdFeature: Enabled xAPIC\n"));
+            break;
+
+        /*
+         * Set the x2APIC bit in the standard feature mask.
+         * Note! ASSUMES CPUMCPUIDFEATURE_APIC is called first.
+         */
+        case CPUMCPUIDFEATURE_X2APIC:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmStd[1].uEcx = pLeaf->uEcx |= X86_CPUID_FEATURE_ECX_X2APIC;
+            pVM->cpum.s.GuestFeatures.fX2Apic = 1;
+
+            /* Make sure the MSR doesn't GP or ignore the EXTD bit. */
+            pMsrRange = cpumLookupMsrRange(pVM, MSR_IA32_APICBASE);
+            if (pMsrRange)
+            {
+                pMsrRange->fWrGpMask  &= ~MSR_IA32_APICBASE_EXTD;
+                pMsrRange->fWrIgnMask &= ~MSR_IA32_APICBASE_EXTD;
+            }
+
+            LogRel(("CPUM: SetGuestCpuIdFeature: Enabled x2APIC\n"));
+            break;
+
+        /*
+         * Set the sysenter/sysexit bit in the standard feature mask.
+         * Assumes the caller knows what it's doing! (host must support these)
+         */
+        case CPUMCPUIDFEATURE_SEP:
+            if (!pVM->cpum.s.HostFeatures.fSysEnter)
+            {
+                AssertMsgFailed(("ERROR: Can't turn on SEP when the host doesn't support it!!\n"));
+                return;
+            }
+
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmStd[1].uEdx = pLeaf->uEdx |= X86_CPUID_FEATURE_EDX_SEP;
+            pVM->cpum.s.GuestFeatures.fSysEnter = 1;
+            LogRel(("CPUM: SetGuestCpuIdFeature: Enabled SYSENTER/EXIT\n"));
+            break;
+
+        /*
+         * Set the syscall/sysret bit in the extended feature mask.
+         * Assumes the caller knows what it's doing! (host must support these)
+         */
+        case CPUMCPUIDFEATURE_SYSCALL:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (   !pLeaf
+                || !pVM->cpum.s.HostFeatures.fSysCall)
+            {
+                LogRel(("CPUM: WARNING! Can't turn on SYSCALL/SYSRET when the host doesn't support it!\n"));
+                return;
+            }
+
+            /* Valid for both Intel and AMD CPUs, although only in 64 bits mode for Intel. */
+            pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx = pLeaf->uEdx |= X86_CPUID_EXT_FEATURE_EDX_SYSCALL;
+            pVM->cpum.s.GuestFeatures.fSysCall = 1;
+            LogRel(("CPUM: SetGuestCpuIdFeature: Enabled SYSCALL/RET\n"));
+            break;
+
+        /*
+         * Set the PAE bit in both feature masks.
+         * Assumes the caller knows what it's doing! (host must support these)
+         */
+        case CPUMCPUIDFEATURE_PAE:
+            if (!pVM->cpum.s.HostFeatures.fPae)
+            {
+                LogRel(("CPUM: WARNING! Can't turn on PAE when the host doesn't support it!\n"));
+                return;
+            }
+
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmStd[1].uEdx = pLeaf->uEdx |= X86_CPUID_FEATURE_EDX_PAE;
+
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (    pLeaf
+                &&  (   pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_AMD
+                     || pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_HYGON))
+                pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx = pLeaf->uEdx |= X86_CPUID_AMD_FEATURE_EDX_PAE;
+
+            pVM->cpum.s.GuestFeatures.fPae = 1;
+            LogRel(("CPUM: SetGuestCpuIdFeature: Enabled PAE\n"));
+            break;
+
+        /*
+         * Set the LONG MODE bit in the extended feature mask.
+         * Assumes the caller knows what it's doing! (host must support these)
+         */
+        case CPUMCPUIDFEATURE_LONG_MODE:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (   !pLeaf
+                || !pVM->cpum.s.HostFeatures.fLongMode)
+            {
+                LogRel(("CPUM: WARNING! Can't turn on LONG MODE when the host doesn't support it!\n"));
+                return;
+            }
+
+            /* Valid for both Intel and AMD. */
+            pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx = pLeaf->uEdx |= X86_CPUID_EXT_FEATURE_EDX_LONG_MODE;
+            pVM->cpum.s.GuestFeatures.fLongMode = 1;
+            pVM->cpum.s.GuestFeatures.cVmxMaxPhysAddrWidth = pVM->cpum.s.GuestFeatures.cMaxPhysAddrWidth;
+            if (pVM->cpum.s.GuestFeatures.fVmx)
+                for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                {
+                    PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+                    pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Basic &= ~VMX_BASIC_PHYSADDR_WIDTH_32BIT;
+                }
+            LogRel(("CPUM: SetGuestCpuIdFeature: Enabled LONG MODE\n"));
+            break;
+
+        /*
+         * Set the NX/XD bit in the extended feature mask.
+         * Assumes the caller knows what it's doing! (host must support these)
+         */
+        case CPUMCPUIDFEATURE_NX:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (   !pLeaf
+                || !pVM->cpum.s.HostFeatures.fNoExecute)
+            {
+                LogRel(("CPUM: WARNING! Can't turn on NX/XD when the host doesn't support it!\n"));
+                return;
+            }
+
+            /* Valid for both Intel and AMD. */
+            pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx = pLeaf->uEdx |= X86_CPUID_EXT_FEATURE_EDX_NX;
+            pVM->cpum.s.GuestFeatures.fNoExecute = 1;
+            LogRel(("CPUM: SetGuestCpuIdFeature: Enabled NX\n"));
+            break;
+
+
+        /*
+         * Set the LAHF/SAHF support in 64-bit mode.
+         * Assumes the caller knows what it's doing! (host must support this)
+         */
+        case CPUMCPUIDFEATURE_LAHF:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (   !pLeaf
+                || !pVM->cpum.s.HostFeatures.fLahfSahf)
+            {
+                LogRel(("CPUM: WARNING! Can't turn on LAHF/SAHF when the host doesn't support it!\n"));
+                return;
+            }
+
+            /* Valid for both Intel and AMD. */
+            pVM->cpum.s.aGuestCpuIdPatmExt[1].uEcx = pLeaf->uEcx |= X86_CPUID_EXT_FEATURE_ECX_LAHF_SAHF;
+            pVM->cpum.s.GuestFeatures.fLahfSahf = 1;
+            LogRel(("CPUM: SetGuestCpuIdFeature: Enabled LAHF/SAHF\n"));
+            break;
+
+        /*
+         * Set the page attribute table bit.  This is alternative page level
+         * cache control that doesn't much matter when everything is
+         * virtualized, though it may when passing thru device memory.
+         */
+        case CPUMCPUIDFEATURE_PAT:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmStd[1].uEdx = pLeaf->uEdx |= X86_CPUID_FEATURE_EDX_PAT;
+
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (   pLeaf
+                && (   pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_AMD
+                    || pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_HYGON))
+                pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx = pLeaf->uEdx |= X86_CPUID_AMD_FEATURE_EDX_PAT;
+
+            pVM->cpum.s.GuestFeatures.fPat = 1;
+            LogRel(("CPUM: SetGuestCpuIdFeature: Enabled PAT\n"));
+            break;
+
+        /*
+         * Set the RDTSCP support bit.
+         * Assumes the caller knows what it's doing! (host must support this)
+         */
+        case CPUMCPUIDFEATURE_RDTSCP:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (   !pLeaf
+                || !pVM->cpum.s.HostFeatures.fRdTscP
+                || pVM->cpum.s.u8PortableCpuIdLevel > 0)
+            {
+                if (!pVM->cpum.s.u8PortableCpuIdLevel)
+                    LogRel(("CPUM: WARNING! Can't turn on RDTSCP when the host doesn't support it!\n"));
+                return;
+            }
+
+            /* Valid for both Intel and AMD. */
+            pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx = pLeaf->uEdx |= X86_CPUID_EXT_FEATURE_EDX_RDTSCP;
+            pVM->cpum.s.HostFeatures.fRdTscP = 1;
+            LogRel(("CPUM: SetGuestCpuIdFeature: Enabled RDTSCP.\n"));
+            break;
+
+       /*
+        * Set the Hypervisor Present bit in the standard feature mask.
+        */
+        case CPUMCPUIDFEATURE_HVP:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmStd[1].uEcx = pLeaf->uEcx |= X86_CPUID_FEATURE_ECX_HVP;
+            pVM->cpum.s.GuestFeatures.fHypervisorPresent = 1;
+            LogRel(("CPUM: SetGuestCpuIdFeature: Enabled Hypervisor Present bit\n"));
+            break;
+
+        /*
+         * Set the MWAIT Extensions Present bit in the MWAIT/MONITOR leaf.
+         * This currently includes the Present bit and MWAITBREAK bit as well.
+         */
+        case CPUMCPUIDFEATURE_MWAIT_EXTS:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000005));
+            if (   !pLeaf
+                || !pVM->cpum.s.HostFeatures.fMWaitExtensions)
+            {
+                LogRel(("CPUM: WARNING! Can't turn on MWAIT Extensions when the host doesn't support it!\n"));
+                return;
+            }
+
+            /* Valid for both Intel and AMD. */
+            pVM->cpum.s.aGuestCpuIdPatmStd[5].uEcx = pLeaf->uEcx |= X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0;
+            pVM->cpum.s.GuestFeatures.fMWaitExtensions = 1;
+            LogRel(("CPUM: SetGuestCpuIdFeature: Enabled MWAIT Extensions.\n"));
+            break;
+
+        /*
+         * Set up the speculation control CPUID bits and MSRs. This is quite complicated
+         * on Intel CPUs, and different on AMDs.
+         */
+        case CPUMCPUIDFEATURE_SPEC_CTRL:
+            if (pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+            {
+                pLeaf = cpumR3CpuIdGetExactLeaf(&pVM->cpum.s, UINT32_C(0x00000007), 0);
+                if (   !pLeaf
+                    || !(pVM->cpum.s.HostFeatures.fIbpb || pVM->cpum.s.HostFeatures.fIbrs))
+                {
+                    LogRel(("CPUM: WARNING! Can't turn on Speculation Control when the host doesn't support it!\n"));
+                    return;
+                }
+
+                /* The feature can be enabled. Let's see what we can actually do. */
+                pVM->cpum.s.GuestFeatures.fSpeculationControl = 1;
+
+                /* We will only expose STIBP if IBRS is present to keep things simpler (simple is not an option). */
+                if (pVM->cpum.s.HostFeatures.fIbrs)
+                {
+                    pLeaf->uEdx |= X86_CPUID_STEXT_FEATURE_EDX_IBRS_IBPB;
+                    pVM->cpum.s.GuestFeatures.fIbrs = 1;
+                    if (pVM->cpum.s.HostFeatures.fStibp)
+                    {
+                        pLeaf->uEdx |= X86_CPUID_STEXT_FEATURE_EDX_STIBP;
+                        pVM->cpum.s.GuestFeatures.fStibp = 1;
+                    }
+
+                    /* Make sure we have the speculation control MSR... */
+                    pMsrRange = cpumLookupMsrRange(pVM, MSR_IA32_SPEC_CTRL);
+                    if (!pMsrRange)
+                    {
+                        static CPUMMSRRANGE const s_SpecCtrl =
+                        {
+                            /*.uFirst =*/ MSR_IA32_SPEC_CTRL, /*.uLast =*/ MSR_IA32_SPEC_CTRL,
+                            /*.enmRdFn =*/ kCpumMsrRdFn_Ia32SpecCtrl, /*.enmWrFn =*/ kCpumMsrWrFn_Ia32SpecCtrl,
+                            /*.offCpumCpu =*/ UINT16_MAX, /*.fReserved =*/ 0, /*.uValue =*/ 0, /*.fWrIgnMask =*/ 0, /*.fWrGpMask =*/ 0,
+                            /*.szName = */ "IA32_SPEC_CTRL"
+                        };
+                        int rc = CPUMR3MsrRangesInsert(pVM, &s_SpecCtrl);
+                        AssertLogRelRC(rc);
+                    }
+
+                    /* ... and the predictor command MSR. */
+                    pMsrRange = cpumLookupMsrRange(pVM, MSR_IA32_PRED_CMD);
+                    if (!pMsrRange)
+                    {
+                        /** @todo incorrect fWrGpMask. */
+                        static CPUMMSRRANGE const s_SpecCtrl =
+                        {
+                            /*.uFirst =*/ MSR_IA32_PRED_CMD, /*.uLast =*/ MSR_IA32_PRED_CMD,
+                            /*.enmRdFn =*/ kCpumMsrRdFn_WriteOnly, /*.enmWrFn =*/ kCpumMsrWrFn_Ia32PredCmd,
+                            /*.offCpumCpu =*/ UINT16_MAX, /*.fReserved =*/ 0, /*.uValue =*/ 0, /*.fWrIgnMask =*/ 0, /*.fWrGpMask =*/ 0,
+                            /*.szName = */ "IA32_PRED_CMD"
+                        };
+                        int rc = CPUMR3MsrRangesInsert(pVM, &s_SpecCtrl);
+                        AssertLogRelRC(rc);
+                    }
+
+                }
+
+                if (pVM->cpum.s.HostFeatures.fArchCap)
+                {
+                    /* Install the architectural capabilities MSR. */
+                    pMsrRange = cpumLookupMsrRange(pVM, MSR_IA32_ARCH_CAPABILITIES);
+                    if (!pMsrRange)
+                    {
+                        static CPUMMSRRANGE const s_ArchCaps =
+                        {
+                            /*.uFirst =*/ MSR_IA32_ARCH_CAPABILITIES, /*.uLast =*/ MSR_IA32_ARCH_CAPABILITIES,
+                            /*.enmRdFn =*/ kCpumMsrRdFn_Ia32ArchCapabilities, /*.enmWrFn =*/ kCpumMsrWrFn_ReadOnly,
+                            /*.offCpumCpu =*/ UINT16_MAX, /*.fReserved =*/ 0, /*.uValue =*/ 0, /*.fWrIgnMask =*/ 0, /*.fWrGpMask =*/ UINT64_MAX,
+                            /*.szName = */ "IA32_ARCH_CAPABILITIES"
+                        };
+                        int rc = CPUMR3MsrRangesInsert(pVM, &s_ArchCaps);
+                        AssertLogRelRC(rc);
+                    }
+
+                    /* Advertise IBRS_ALL if present at this point... */
+                    if (pVM->cpum.s.HostFeatures.fArchCap & MSR_IA32_ARCH_CAP_F_IBRS_ALL)
+                        VMCC_FOR_EACH_VMCPU_STMT(pVM, pVCpu->cpum.s.GuestMsrs.msr.ArchCaps |= MSR_IA32_ARCH_CAP_F_IBRS_ALL);
+                }
+
+                LogRel(("CPUM: SetGuestCpuIdFeature: Enabled Speculation Control.\n"));
+            }
+            else if (   pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_AMD
+                     || pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_HYGON)
+            {
+                /* The precise details of AMD's implementation are not yet clear. */
+            }
+            break;
+
+        default:
+            AssertMsgFailed(("enmFeature=%d\n", enmFeature));
+            break;
+    }
+
+    /** @todo can probably kill this as this API is now init time only... */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        pVCpu->cpum.s.fChanged |= CPUM_CHANGED_CPUID;
+    }
+}
+
+
+/**
+ * Queries a CPUID feature bit.
+ *
+ * @returns boolean for feature presence
+ * @param   pVM             The cross context VM structure.
+ * @param   enmFeature      The feature to query.
+ * @deprecated Use the cpum.ro.GuestFeatures directly instead.
+ */
+VMMR3_INT_DECL(bool) CPUMR3GetGuestCpuIdFeature(PVM pVM, CPUMCPUIDFEATURE enmFeature)
+{
+    switch (enmFeature)
+    {
+        case CPUMCPUIDFEATURE_APIC:         return pVM->cpum.s.GuestFeatures.fApic;
+        case CPUMCPUIDFEATURE_X2APIC:       return pVM->cpum.s.GuestFeatures.fX2Apic;
+        case CPUMCPUIDFEATURE_SYSCALL:      return pVM->cpum.s.GuestFeatures.fSysCall;
+        case CPUMCPUIDFEATURE_SEP:          return pVM->cpum.s.GuestFeatures.fSysEnter;
+        case CPUMCPUIDFEATURE_PAE:          return pVM->cpum.s.GuestFeatures.fPae;
+        case CPUMCPUIDFEATURE_NX:           return pVM->cpum.s.GuestFeatures.fNoExecute;
+        case CPUMCPUIDFEATURE_LAHF:         return pVM->cpum.s.GuestFeatures.fLahfSahf;
+        case CPUMCPUIDFEATURE_LONG_MODE:    return pVM->cpum.s.GuestFeatures.fLongMode;
+        case CPUMCPUIDFEATURE_PAT:          return pVM->cpum.s.GuestFeatures.fPat;
+        case CPUMCPUIDFEATURE_RDTSCP:       return pVM->cpum.s.GuestFeatures.fRdTscP;
+        case CPUMCPUIDFEATURE_HVP:          return pVM->cpum.s.GuestFeatures.fHypervisorPresent;
+        case CPUMCPUIDFEATURE_MWAIT_EXTS:   return pVM->cpum.s.GuestFeatures.fMWaitExtensions;
+        case CPUMCPUIDFEATURE_SPEC_CTRL:    return pVM->cpum.s.GuestFeatures.fSpeculationControl;
+
+        case CPUMCPUIDFEATURE_INVALID:
+        case CPUMCPUIDFEATURE_32BIT_HACK:
+            break;
+    }
+    AssertFailed();
+    return false;
+}
+
+
+/**
+ * Clears a CPUID feature bit.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   enmFeature      The feature to clear.
+ *
+ * @deprecated Probably better to default the feature to disabled and only allow
+ *             setting (enabling) it during construction.
+ */
+VMMR3_INT_DECL(void) CPUMR3ClearGuestCpuIdFeature(PVM pVM, CPUMCPUIDFEATURE enmFeature)
+{
+    PCPUMCPUIDLEAF pLeaf;
+    switch (enmFeature)
+    {
+        case CPUMCPUIDFEATURE_APIC:
+            Assert(!pVM->cpum.s.GuestFeatures.fApic); /* We only expect this call during init. No MSR adjusting needed. */
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmStd[1].uEdx = pLeaf->uEdx &= ~X86_CPUID_FEATURE_EDX_APIC;
+
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (pLeaf && (pLeaf->fFlags & CPUMCPUIDLEAF_F_CONTAINS_APIC))
+                pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx = pLeaf->uEdx &= ~X86_CPUID_AMD_FEATURE_EDX_APIC;
+
+            pVM->cpum.s.GuestFeatures.fApic = 0;
+            Log(("CPUM: ClearGuestCpuIdFeature: Disabled xAPIC\n"));
+            break;
+
+        case CPUMCPUIDFEATURE_X2APIC:
+            Assert(!pVM->cpum.s.GuestFeatures.fX2Apic); /* We only expect this call during init. No MSR adjusting needed. */
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmStd[1].uEcx = pLeaf->uEcx &= ~X86_CPUID_FEATURE_ECX_X2APIC;
+            pVM->cpum.s.GuestFeatures.fX2Apic = 0;
+            Log(("CPUM: ClearGuestCpuIdFeature: Disabled x2APIC\n"));
+            break;
+
+        case CPUMCPUIDFEATURE_PAE:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmStd[1].uEdx = pLeaf->uEdx &= ~X86_CPUID_FEATURE_EDX_PAE;
+
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (   pLeaf
+                && (   pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_AMD
+                    || pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_HYGON))
+                pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx = pLeaf->uEdx &= ~X86_CPUID_AMD_FEATURE_EDX_PAE;
+
+            pVM->cpum.s.GuestFeatures.fPae = 0;
+            Log(("CPUM: ClearGuestCpuIdFeature: Disabled PAE!\n"));
+            break;
+
+        case CPUMCPUIDFEATURE_PAT:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmStd[1].uEdx = pLeaf->uEdx &= ~X86_CPUID_FEATURE_EDX_PAT;
+
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (   pLeaf
+                && (   pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_AMD
+                    || pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_HYGON))
+                pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx = pLeaf->uEdx &= ~X86_CPUID_AMD_FEATURE_EDX_PAT;
+
+            pVM->cpum.s.GuestFeatures.fPat = 0;
+            Log(("CPUM: ClearGuestCpuIdFeature: Disabled PAT!\n"));
+            break;
+
+        case CPUMCPUIDFEATURE_LONG_MODE:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx = pLeaf->uEdx &= ~X86_CPUID_EXT_FEATURE_EDX_LONG_MODE;
+            pVM->cpum.s.GuestFeatures.fLongMode = 0;
+            pVM->cpum.s.GuestFeatures.cVmxMaxPhysAddrWidth = 32;
+            if (pVM->cpum.s.GuestFeatures.fVmx)
+                for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                {
+                    PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+                    pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Basic |= VMX_BASIC_PHYSADDR_WIDTH_32BIT;
+                }
+            break;
+
+        case CPUMCPUIDFEATURE_LAHF:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmExt[1].uEcx = pLeaf->uEcx &= ~X86_CPUID_EXT_FEATURE_ECX_LAHF_SAHF;
+            pVM->cpum.s.GuestFeatures.fLahfSahf = 0;
+            break;
+
+        case CPUMCPUIDFEATURE_RDTSCP:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x80000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx = pLeaf->uEdx &= ~X86_CPUID_EXT_FEATURE_EDX_RDTSCP;
+            pVM->cpum.s.GuestFeatures.fRdTscP = 0;
+            Log(("CPUM: ClearGuestCpuIdFeature: Disabled RDTSCP!\n"));
+            break;
+
+        case CPUMCPUIDFEATURE_HVP:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000001));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmStd[1].uEcx = pLeaf->uEcx &= ~X86_CPUID_FEATURE_ECX_HVP;
+            pVM->cpum.s.GuestFeatures.fHypervisorPresent = 0;
+            break;
+
+        case CPUMCPUIDFEATURE_MWAIT_EXTS:
+            pLeaf = cpumCpuIdGetLeaf(pVM, UINT32_C(0x00000005));
+            if (pLeaf)
+                pVM->cpum.s.aGuestCpuIdPatmStd[5].uEcx = pLeaf->uEcx &= ~(X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0);
+            pVM->cpum.s.GuestFeatures.fMWaitExtensions = 0;
+            Log(("CPUM: ClearGuestCpuIdFeature: Disabled MWAIT Extensions!\n"));
+            break;
+
+        case CPUMCPUIDFEATURE_SPEC_CTRL:
+            pLeaf = cpumR3CpuIdGetExactLeaf(&pVM->cpum.s, UINT32_C(0x00000007), 0);
+            if (pLeaf)
+                pLeaf->uEdx &= ~(X86_CPUID_STEXT_FEATURE_EDX_IBRS_IBPB | X86_CPUID_STEXT_FEATURE_EDX_STIBP);
+            VMCC_FOR_EACH_VMCPU_STMT(pVM, pVCpu->cpum.s.GuestMsrs.msr.ArchCaps &= ~MSR_IA32_ARCH_CAP_F_IBRS_ALL);
+            Log(("CPUM: ClearGuestCpuIdFeature: Disabled speculation control!\n"));
+            break;
+
+        default:
+            AssertMsgFailed(("enmFeature=%d\n", enmFeature));
+            break;
+    }
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        pVCpu->cpum.s.fChanged |= CPUM_CHANGED_CPUID;
+    }
+}
+
+
+
+/*
+ *
+ *
+ * Saved state related code.
+ * Saved state related code.
+ * Saved state related code.
+ *
+ *
+ */
+
+/**
+ * Called both in pass 0 and the final pass.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+void cpumR3SaveCpuId(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Save all the CPU ID leaves.
+     */
+    SSMR3PutU32(pSSM, sizeof(pVM->cpum.s.GuestInfo.paCpuIdLeavesR3[0]));
+    SSMR3PutU32(pSSM, pVM->cpum.s.GuestInfo.cCpuIdLeaves);
+    SSMR3PutMem(pSSM, pVM->cpum.s.GuestInfo.paCpuIdLeavesR3,
+                sizeof(pVM->cpum.s.GuestInfo.paCpuIdLeavesR3[0]) * pVM->cpum.s.GuestInfo.cCpuIdLeaves);
+
+    SSMR3PutMem(pSSM, &pVM->cpum.s.GuestInfo.DefCpuId, sizeof(pVM->cpum.s.GuestInfo.DefCpuId));
+
+    /*
+     * Save a good portion of the raw CPU IDs as well as they may come in
+     * handy when validating features for raw mode.
+     */
+    CPUMCPUID   aRawStd[16];
+    for (unsigned i = 0; i < RT_ELEMENTS(aRawStd); i++)
+        ASMCpuIdExSlow(i, 0, 0, 0, &aRawStd[i].uEax, &aRawStd[i].uEbx, &aRawStd[i].uEcx, &aRawStd[i].uEdx);
+    SSMR3PutU32(pSSM, RT_ELEMENTS(aRawStd));
+    SSMR3PutMem(pSSM, &aRawStd[0], sizeof(aRawStd));
+
+    CPUMCPUID   aRawExt[32];
+    for (unsigned i = 0; i < RT_ELEMENTS(aRawExt); i++)
+        ASMCpuIdExSlow(i | UINT32_C(0x80000000), 0, 0, 0, &aRawExt[i].uEax, &aRawExt[i].uEbx, &aRawExt[i].uEcx, &aRawExt[i].uEdx);
+    SSMR3PutU32(pSSM, RT_ELEMENTS(aRawExt));
+    SSMR3PutMem(pSSM, &aRawExt[0], sizeof(aRawExt));
+}
+
+
+static int cpumR3LoadOneOldGuestCpuIdArray(PSSMHANDLE pSSM, uint32_t uBase, PCPUMCPUIDLEAF *ppaLeaves, uint32_t *pcLeaves)
+{
+    uint32_t cCpuIds;
+    int rc = SSMR3GetU32(pSSM, &cCpuIds);
+    if (RT_SUCCESS(rc))
+    {
+        if (cCpuIds < 64)
+        {
+            for (uint32_t i = 0; i < cCpuIds; i++)
+            {
+                CPUMCPUID CpuId;
+                rc = SSMR3GetMem(pSSM, &CpuId, sizeof(CpuId));
+                if (RT_FAILURE(rc))
+                    break;
+
+                CPUMCPUIDLEAF NewLeaf;
+                NewLeaf.uLeaf           = uBase + i;
+                NewLeaf.uSubLeaf        = 0;
+                NewLeaf.fSubLeafMask    = 0;
+                NewLeaf.uEax            = CpuId.uEax;
+                NewLeaf.uEbx            = CpuId.uEbx;
+                NewLeaf.uEcx            = CpuId.uEcx;
+                NewLeaf.uEdx            = CpuId.uEdx;
+                NewLeaf.fFlags          = 0;
+                rc = cpumR3CpuIdInsert(NULL /* pVM */, ppaLeaves, pcLeaves, &NewLeaf);
+            }
+        }
+        else
+            rc = VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    }
+    if (RT_FAILURE(rc))
+    {
+        RTMemFree(*ppaLeaves);
+        *ppaLeaves = NULL;
+        *pcLeaves = 0;
+    }
+    return rc;
+}
+
+
+static int cpumR3LoadGuestCpuIdArray(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, PCPUMCPUIDLEAF *ppaLeaves, uint32_t *pcLeaves)
+{
+    *ppaLeaves = NULL;
+    *pcLeaves = 0;
+
+    int rc;
+    if (uVersion > CPUM_SAVED_STATE_VERSION_PUT_STRUCT)
+    {
+        /*
+         * The new format. Starts by declaring the leave size and count.
+         */
+        uint32_t cbLeaf;
+        SSMR3GetU32(pSSM, &cbLeaf);
+        uint32_t cLeaves;
+        rc = SSMR3GetU32(pSSM, &cLeaves);
+        if (RT_SUCCESS(rc))
+        {
+            if (cbLeaf == sizeof(**ppaLeaves))
+            {
+                if (cLeaves <= CPUM_CPUID_MAX_LEAVES)
+                {
+                    /*
+                     * Load the leaves one by one.
+                     *
+                     * The uPrev stuff is a kludge for working around a week worth of bad saved
+                     * states during the CPUID revamp in March 2015.  We saved too many leaves
+                     * due to a bug in cpumR3CpuIdInstallAndExplodeLeaves, thus ending up with
+                     * garbage entires at the end of the array when restoring.  We also had
+                     * a subleaf insertion bug that triggered with the leaf 4 stuff below,
+                     * this kludge doesn't deal correctly with that, but who cares...
+                     */
+                    uint32_t uPrev = 0;
+                    for (uint32_t i = 0; i < cLeaves && RT_SUCCESS(rc); i++)
+                    {
+                        CPUMCPUIDLEAF Leaf;
+                        rc = SSMR3GetMem(pSSM, &Leaf, sizeof(Leaf));
+                        if (RT_SUCCESS(rc))
+                        {
+                            if (   uVersion != CPUM_SAVED_STATE_VERSION_BAD_CPUID_COUNT
+                                || Leaf.uLeaf >= uPrev)
+                            {
+                                rc = cpumR3CpuIdInsert(NULL /* pVM */, ppaLeaves, pcLeaves, &Leaf);
+                                uPrev = Leaf.uLeaf;
+                            }
+                            else
+                                uPrev = UINT32_MAX;
+                        }
+                    }
+                }
+                else
+                    rc = SSMR3SetLoadError(pSSM, VERR_TOO_MANY_CPUID_LEAVES, RT_SRC_POS,
+                                           "Too many CPUID leaves: %#x, max %#x", cLeaves, CPUM_CPUID_MAX_LEAVES);
+            }
+            else
+                rc = SSMR3SetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
+                                       "CPUMCPUIDLEAF size differs: saved=%#x, our=%#x", cbLeaf, sizeof(**ppaLeaves));
+        }
+    }
+    else
+    {
+        /*
+         * The old format with its three inflexible arrays.
+         */
+        rc = cpumR3LoadOneOldGuestCpuIdArray(pSSM, UINT32_C(0x00000000), ppaLeaves, pcLeaves);
+        if (RT_SUCCESS(rc))
+            rc = cpumR3LoadOneOldGuestCpuIdArray(pSSM, UINT32_C(0x80000000), ppaLeaves, pcLeaves);
+        if (RT_SUCCESS(rc))
+            rc = cpumR3LoadOneOldGuestCpuIdArray(pSSM, UINT32_C(0xc0000000), ppaLeaves, pcLeaves);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Fake up leaf 4 on intel like we used to do in CPUMGetGuestCpuId earlier.
+             */
+            PCPUMCPUIDLEAF pLeaf = cpumR3CpuIdGetLeaf(*ppaLeaves, *pcLeaves, 0, 0);
+            if (   pLeaf
+                && ASMIsIntelCpuEx(pLeaf->uEbx, pLeaf->uEcx, pLeaf->uEdx))
+            {
+                CPUMCPUIDLEAF Leaf;
+                Leaf.uLeaf        = 4;
+                Leaf.fSubLeafMask = UINT32_MAX;
+                Leaf.uSubLeaf     = 0;
+                Leaf.uEdx = UINT32_C(0);          /* 3 flags, 0 is fine. */
+                Leaf.uEcx = UINT32_C(63);         /* sets - 1 */
+                Leaf.uEbx = (UINT32_C(7) << 22)   /* associativity -1 */
+                          | (UINT32_C(0) << 12)   /* phys line partitions - 1 */
+                          | UINT32_C(63);         /* system coherency line size - 1 */
+                Leaf.uEax = (RT_MIN(pVM->cCpus - 1, UINT32_C(0x3f)) << 26)  /* cores per package - 1 */
+                          | (UINT32_C(0) << 14)   /* threads per cache - 1 */
+                          | (UINT32_C(1) << 5)    /* cache level */
+                          | UINT32_C(1);          /* cache type (data) */
+                Leaf.fFlags       = 0;
+                rc = cpumR3CpuIdInsert(NULL /* pVM */, ppaLeaves, pcLeaves, &Leaf);
+                if (RT_SUCCESS(rc))
+                {
+                    Leaf.uSubLeaf = 1; /* Should've been cache type 2 (code), but buggy code made it data. */
+                    rc = cpumR3CpuIdInsert(NULL /* pVM */, ppaLeaves, pcLeaves, &Leaf);
+                }
+                if (RT_SUCCESS(rc))
+                {
+                    Leaf.uSubLeaf = 2; /* Should've been cache type 3 (unified), but buggy code made it data. */
+                    Leaf.uEcx = 4095;                   /* sets - 1 */
+                    Leaf.uEbx &= UINT32_C(0x003fffff);  /* associativity - 1 */
+                    Leaf.uEbx |= UINT32_C(23) << 22;
+                    Leaf.uEax &= UINT32_C(0xfc003fff);  /* threads per cache - 1 */
+                    Leaf.uEax |= RT_MIN(pVM->cCpus - 1, UINT32_C(0xfff)) << 14;
+                    Leaf.uEax &= UINT32_C(0xffffff1f);  /* level */
+                    Leaf.uEax |= UINT32_C(2) << 5;
+                    rc = cpumR3CpuIdInsert(NULL /* pVM */, ppaLeaves, pcLeaves, &Leaf);
+                }
+            }
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Loads the CPU ID leaves saved by pass 0, inner worker.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ * @param   uVersion            The format version.
+ * @param   paLeaves            Guest CPUID leaves loaded from the state.
+ * @param   cLeaves             The number of leaves in @a paLeaves.
+ * @param   pMsrs               The guest MSRs.
+ */
+int cpumR3LoadCpuIdInner(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, PCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, PCCPUMMSRS pMsrs)
+{
+    AssertMsgReturn(uVersion >= CPUM_SAVED_STATE_VERSION_VER3_2, ("%u\n", uVersion), VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION);
+
+    /*
+     * Continue loading the state into stack buffers.
+     */
+    CPUMCPUID   GuestDefCpuId;
+    int rc = SSMR3GetMem(pSSM, &GuestDefCpuId, sizeof(GuestDefCpuId));
+    AssertRCReturn(rc, rc);
+
+    CPUMCPUID   aRawStd[16];
+    uint32_t    cRawStd;
+    rc = SSMR3GetU32(pSSM, &cRawStd); AssertRCReturn(rc, rc);
+    if (cRawStd > RT_ELEMENTS(aRawStd))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    rc = SSMR3GetMem(pSSM, &aRawStd[0], cRawStd * sizeof(aRawStd[0]));
+    AssertRCReturn(rc, rc);
+    for (uint32_t i = cRawStd; i < RT_ELEMENTS(aRawStd); i++)
+        ASMCpuIdExSlow(i, 0, 0, 0, &aRawStd[i].uEax, &aRawStd[i].uEbx, &aRawStd[i].uEcx, &aRawStd[i].uEdx);
+
+    CPUMCPUID   aRawExt[32];
+    uint32_t    cRawExt;
+    rc = SSMR3GetU32(pSSM, &cRawExt); AssertRCReturn(rc, rc);
+    if (cRawExt > RT_ELEMENTS(aRawExt))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    rc = SSMR3GetMem(pSSM, &aRawExt[0], cRawExt * sizeof(aRawExt[0]));
+    AssertRCReturn(rc, rc);
+    for (uint32_t i = cRawExt; i < RT_ELEMENTS(aRawExt); i++)
+        ASMCpuIdExSlow(i | UINT32_C(0x80000000), 0, 0, 0, &aRawExt[i].uEax, &aRawExt[i].uEbx, &aRawExt[i].uEcx, &aRawExt[i].uEdx);
+
+    /*
+     * Get the raw CPU IDs for the current host.
+     */
+    CPUMCPUID   aHostRawStd[16];
+    for (unsigned i = 0; i < RT_ELEMENTS(aHostRawStd); i++)
+        ASMCpuIdExSlow(i, 0, 0, 0, &aHostRawStd[i].uEax, &aHostRawStd[i].uEbx, &aHostRawStd[i].uEcx, &aHostRawStd[i].uEdx);
+
+    CPUMCPUID   aHostRawExt[32];
+    for (unsigned i = 0; i < RT_ELEMENTS(aHostRawExt); i++)
+        ASMCpuIdExSlow(i | UINT32_C(0x80000000), 0, 0, 0,
+                       &aHostRawExt[i].uEax, &aHostRawExt[i].uEbx, &aHostRawExt[i].uEcx, &aHostRawExt[i].uEdx);
+
+    /*
+     * Get the host and guest overrides so we don't reject the state because
+     * some feature was enabled thru these interfaces.
+     * Note! We currently only need the feature leaves, so skip rest.
+     */
+    PCFGMNODE   pOverrideCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "CPUM/HostCPUID");
+    CPUMCPUID   aHostOverrideStd[2];
+    memcpy(&aHostOverrideStd[0], &aHostRawStd[0], sizeof(aHostOverrideStd));
+    cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x00000000), &aHostOverrideStd[0], RT_ELEMENTS(aHostOverrideStd), pOverrideCfg);
+
+    CPUMCPUID   aHostOverrideExt[2];
+    memcpy(&aHostOverrideExt[0], &aHostRawExt[0], sizeof(aHostOverrideExt));
+    cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x80000000), &aHostOverrideExt[0], RT_ELEMENTS(aHostOverrideExt), pOverrideCfg);
+
+    /*
+     * This can be skipped.
+     */
+    bool fStrictCpuIdChecks;
+    CFGMR3QueryBoolDef(CFGMR3GetChild(CFGMR3GetRoot(pVM), "CPUM"), "StrictCpuIdChecks", &fStrictCpuIdChecks, true);
+
+    /*
+     * Define a bunch of macros for simplifying the santizing/checking code below.
+     */
+    /* Generic expression + failure message. */
+#define CPUID_CHECK_RET(expr, fmt) \
+    do { \
+        if (!(expr)) \
+        { \
+            char *pszMsg = RTStrAPrintf2 fmt; /* lack of variadic macros sucks */ \
+            if (fStrictCpuIdChecks) \
+            { \
+                int rcCpuid = SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, "%s", pszMsg); \
+                RTStrFree(pszMsg); \
+                return rcCpuid; \
+            } \
+            LogRel(("CPUM: %s\n", pszMsg)); \
+            RTStrFree(pszMsg); \
+        } \
+    } while (0)
+#define CPUID_CHECK_WRN(expr, fmt) \
+    do { \
+        if (!(expr)) \
+            LogRel(fmt); \
+    } while (0)
+
+    /* For comparing two values and bitch if they differs. */
+#define CPUID_CHECK2_RET(what, host, saved) \
+    do { \
+        if ((host) != (saved)) \
+        { \
+            if (fStrictCpuIdChecks) \
+                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
+                                         N_(#what " mismatch: host=%#x saved=%#x"), (host), (saved)); \
+            LogRel(("CPUM: " #what " differs: host=%#x saved=%#x\n", (host), (saved))); \
+        } \
+    } while (0)
+#define CPUID_CHECK2_WRN(what, host, saved) \
+    do { \
+        if ((host) != (saved)) \
+            LogRel(("CPUM: " #what " differs: host=%#x saved=%#x\n", (host), (saved))); \
+    } while (0)
+
+    /* For checking raw cpu features (raw mode). */
+#define CPUID_RAW_FEATURE_RET(set, reg, bit) \
+    do { \
+        if ((aHostRaw##set [1].reg & bit) != (aRaw##set [1].reg & bit)) \
+        { \
+            if (fStrictCpuIdChecks) \
+                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
+                                         N_(#bit " mismatch: host=%d saved=%d"), \
+                                         !!(aHostRaw##set [1].reg & (bit)), !!(aRaw##set [1].reg & (bit)) ); \
+            LogRel(("CPUM: " #bit" differs: host=%d saved=%d\n", \
+                    !!(aHostRaw##set [1].reg & (bit)), !!(aRaw##set [1].reg & (bit)) )); \
+        } \
+    } while (0)
+#define CPUID_RAW_FEATURE_WRN(set, reg, bit) \
+    do { \
+        if ((aHostRaw##set [1].reg & bit) != (aRaw##set [1].reg & bit)) \
+            LogRel(("CPUM: " #bit" differs: host=%d saved=%d\n", \
+                    !!(aHostRaw##set [1].reg & (bit)), !!(aRaw##set [1].reg & (bit)) )); \
+    } while (0)
+#define CPUID_RAW_FEATURE_IGN(set, reg, bit) do { } while (0)
+
+    /* For checking guest features. */
+#define CPUID_GST_FEATURE_RET(set, reg, bit) \
+    do { \
+        if (    (aGuestCpuId##set [1].reg & bit) \
+            && !(aHostRaw##set [1].reg & bit) \
+            && !(aHostOverride##set [1].reg & bit) \
+           ) \
+        { \
+            if (fStrictCpuIdChecks) \
+                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
+                                         N_(#bit " is not supported by the host but has already exposed to the guest")); \
+            LogRel(("CPUM: " #bit " is not supported by the host but has already exposed to the guest\n")); \
+        } \
+    } while (0)
+#define CPUID_GST_FEATURE_WRN(set, reg, bit) \
+    do { \
+        if (    (aGuestCpuId##set [1].reg & bit) \
+            && !(aHostRaw##set [1].reg & bit) \
+            && !(aHostOverride##set [1].reg & bit) \
+           ) \
+            LogRel(("CPUM: " #bit " is not supported by the host but has already exposed to the guest\n")); \
+    } while (0)
+#define CPUID_GST_FEATURE_EMU(set, reg, bit) \
+    do { \
+        if (    (aGuestCpuId##set [1].reg & bit) \
+            && !(aHostRaw##set [1].reg & bit) \
+            && !(aHostOverride##set [1].reg & bit) \
+           ) \
+            LogRel(("CPUM: Warning - " #bit " is not supported by the host but already exposed to the guest. This may impact performance.\n")); \
+    } while (0)
+#define CPUID_GST_FEATURE_IGN(set, reg, bit) do { } while (0)
+
+    /* For checking guest features if AMD guest CPU. */
+#define CPUID_GST_AMD_FEATURE_RET(set, reg, bit) \
+    do { \
+        if (    (aGuestCpuId##set [1].reg & bit) \
+            &&  fGuestAmd \
+            && (!fGuestAmd || !(aHostRaw##set [1].reg & bit)) \
+            && !(aHostOverride##set [1].reg & bit) \
+           ) \
+        { \
+            if (fStrictCpuIdChecks) \
+                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
+                                         N_(#bit " is not supported by the host but has already exposed to the guest")); \
+            LogRel(("CPUM: " #bit " is not supported by the host but has already exposed to the guest\n")); \
+        } \
+    } while (0)
+#define CPUID_GST_AMD_FEATURE_WRN(set, reg, bit) \
+    do { \
+        if (    (aGuestCpuId##set [1].reg & bit) \
+            &&  fGuestAmd \
+            && (!fGuestAmd || !(aHostRaw##set [1].reg & bit)) \
+            && !(aHostOverride##set [1].reg & bit) \
+           ) \
+            LogRel(("CPUM: " #bit " is not supported by the host but has already exposed to the guest\n")); \
+    } while (0)
+#define CPUID_GST_AMD_FEATURE_EMU(set, reg, bit) \
+    do { \
+        if (    (aGuestCpuId##set [1].reg & bit) \
+            &&  fGuestAmd \
+            && (!fGuestAmd || !(aHostRaw##set [1].reg & bit)) \
+            && !(aHostOverride##set [1].reg & bit) \
+           ) \
+            LogRel(("CPUM: Warning - " #bit " is not supported by the host but already exposed to the guest. This may impact performance.\n")); \
+    } while (0)
+#define CPUID_GST_AMD_FEATURE_IGN(set, reg, bit) do { } while (0)
+
+    /* For checking AMD features which have a corresponding bit in the standard
+       range.  (Intel defines very few bits in the extended feature sets.) */
+#define CPUID_GST_FEATURE2_RET(reg, ExtBit, StdBit) \
+    do { \
+        if (    (aGuestCpuIdExt [1].reg    & (ExtBit)) \
+            && !(fHostAmd  \
+                 ? aHostRawExt[1].reg      & (ExtBit) \
+                 : aHostRawStd[1].reg      & (StdBit)) \
+            && !(aHostOverrideExt[1].reg   & (ExtBit)) \
+           ) \
+        { \
+            if (fStrictCpuIdChecks) \
+                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
+                                         N_(#ExtBit " is not supported by the host but has already exposed to the guest")); \
+            LogRel(("CPUM: " #ExtBit " is not supported by the host but has already exposed to the guest\n")); \
+        } \
+    } while (0)
+#define CPUID_GST_FEATURE2_WRN(reg, ExtBit, StdBit) \
+    do { \
+        if (    (aGuestCpuId[1].reg        & (ExtBit)) \
+            && !(fHostAmd  \
+                 ? aHostRawExt[1].reg      & (ExtBit) \
+                 : aHostRawStd[1].reg      & (StdBit)) \
+            && !(aHostOverrideExt[1].reg   & (ExtBit)) \
+           ) \
+            LogRel(("CPUM: " #ExtBit " is not supported by the host but has already exposed to the guest\n")); \
+    } while (0)
+#define CPUID_GST_FEATURE2_EMU(reg, ExtBit, StdBit) \
+    do { \
+        if (    (aGuestCpuIdExt [1].reg    & (ExtBit)) \
+            && !(fHostAmd  \
+                 ? aHostRawExt[1].reg      & (ExtBit) \
+                 : aHostRawStd[1].reg      & (StdBit)) \
+            && !(aHostOverrideExt[1].reg   & (ExtBit)) \
+           ) \
+            LogRel(("CPUM: Warning - " #ExtBit " is not supported by the host but already exposed to the guest. This may impact performance.\n")); \
+    } while (0)
+#define CPUID_GST_FEATURE2_IGN(reg, ExtBit, StdBit) do { } while (0)
+
+
+    /*
+     * Verify that we can support the features already exposed to the guest on
+     * this host.
+     *
+     * Most of the features we're emulating requires intercepting instruction
+     * and doing it the slow way, so there is no need to warn when they aren't
+     * present in the host CPU.  Thus we use IGN instead of EMU on these.
+     *
+     * Trailing comments:
+     *      "EMU"  - Possible to emulate, could be lots of work and very slow.
+     *      "EMU?" - Can this be emulated?
+     */
+    CPUMCPUID aGuestCpuIdStd[2];
+    RT_ZERO(aGuestCpuIdStd);
+    cpumR3CpuIdGetLeafLegacy(paLeaves, cLeaves, 1, 0, &aGuestCpuIdStd[1]);
+
+    /* CPUID(1).ecx */
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_SSE3);    // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_PCLMUL);  // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_DTES64);  // -> EMU?
+    CPUID_GST_FEATURE_IGN(Std, uEcx, X86_CPUID_FEATURE_ECX_MONITOR);
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_CPLDS);   // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_VMX);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_SMX);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_EST);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_TM2);     // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_SSSE3);   // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_CNTXID);  // -> EMU
+    CPUID_GST_FEATURE_IGN(Std, uEcx, X86_CPUID_FEATURE_ECX_SDBG);
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_FMA);     // -> EMU? what's this?
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_CX16);    // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_TPRUPDATE);//-> EMU
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_PDCM);    // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEcx, RT_BIT_32(16) /*reserved*/);
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_PCID);
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_DCA);     // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_SSE4_1);  // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_SSE4_2);  // -> EMU
+    CPUID_GST_FEATURE_IGN(Std, uEcx, X86_CPUID_FEATURE_ECX_X2APIC);
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_MOVBE);   // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_POPCNT);  // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_TSCDEADL);
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_AES);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_XSAVE);   // -> EMU
+    CPUID_GST_FEATURE_IGN(Std, uEcx, X86_CPUID_FEATURE_ECX_OSXSAVE);
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_AVX);     // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_F16C);
+    CPUID_GST_FEATURE_RET(Std, uEcx, X86_CPUID_FEATURE_ECX_RDRAND);
+    CPUID_GST_FEATURE_IGN(Std, uEcx, X86_CPUID_FEATURE_ECX_HVP);     // Normally not set by host
+
+    /* CPUID(1).edx */
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_FPU);
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_VME);
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_DE);      // -> EMU?
+    CPUID_GST_FEATURE_IGN(Std, uEdx, X86_CPUID_FEATURE_EDX_PSE);
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_TSC);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_MSR);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_PAE);
+    CPUID_GST_FEATURE_IGN(Std, uEdx, X86_CPUID_FEATURE_EDX_MCE);
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_CX8);     // -> EMU?
+    CPUID_GST_FEATURE_IGN(Std, uEdx, X86_CPUID_FEATURE_EDX_APIC);
+    CPUID_GST_FEATURE_RET(Std, uEdx, RT_BIT_32(10) /*reserved*/);
+    CPUID_GST_FEATURE_IGN(Std, uEdx, X86_CPUID_FEATURE_EDX_SEP);
+    CPUID_GST_FEATURE_IGN(Std, uEdx, X86_CPUID_FEATURE_EDX_MTRR);
+    CPUID_GST_FEATURE_IGN(Std, uEdx, X86_CPUID_FEATURE_EDX_PGE);
+    CPUID_GST_FEATURE_IGN(Std, uEdx, X86_CPUID_FEATURE_EDX_MCA);
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_CMOV);    // -> EMU
+    CPUID_GST_FEATURE_IGN(Std, uEdx, X86_CPUID_FEATURE_EDX_PAT);
+    CPUID_GST_FEATURE_IGN(Std, uEdx, X86_CPUID_FEATURE_EDX_PSE36);
+    CPUID_GST_FEATURE_IGN(Std, uEdx, X86_CPUID_FEATURE_EDX_PSN);
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_CLFSH);   // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEdx, RT_BIT_32(20) /*reserved*/);
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_DS);      // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_ACPI);    // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_MMX);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_FXSR);    // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_SSE);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_SSE2);    // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_SS);      // -> EMU?
+    CPUID_GST_FEATURE_IGN(Std, uEdx, X86_CPUID_FEATURE_EDX_HTT);     // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_TM);      // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, uEdx, RT_BIT_32(30) /*JMPE/IA64*/);   // -> EMU
+    CPUID_GST_FEATURE_RET(Std, uEdx, X86_CPUID_FEATURE_EDX_PBE);     // -> EMU?
+
+    /* CPUID(0x80000000). */
+    CPUMCPUID aGuestCpuIdExt[2];
+    RT_ZERO(aGuestCpuIdExt);
+    if (cpumR3CpuIdGetLeafLegacy(paLeaves, cLeaves, UINT32_C(0x80000001), 0, &aGuestCpuIdExt[1]))
+    {
+        /** @todo deal with no 0x80000001 on the host. */
+        bool const fHostAmd  = ASMIsAmdCpuEx(aHostRawStd[0].uEbx, aHostRawStd[0].uEcx, aHostRawStd[0].uEdx)
+                            || ASMIsHygonCpuEx(aHostRawStd[0].uEbx, aHostRawStd[0].uEcx, aHostRawStd[0].uEdx);
+        bool const fGuestAmd = ASMIsAmdCpuEx(aGuestCpuIdExt[0].uEbx, aGuestCpuIdExt[0].uEcx, aGuestCpuIdExt[0].uEdx)
+                            || ASMIsHygonCpuEx(aGuestCpuIdExt[0].uEbx, aGuestCpuIdExt[0].uEcx, aGuestCpuIdExt[0].uEdx);
+
+        /* CPUID(0x80000001).ecx */
+        CPUID_GST_FEATURE_WRN(Ext, uEcx, X86_CPUID_EXT_FEATURE_ECX_LAHF_SAHF);   // -> EMU
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_CMPL);    // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_SVM);     // -> EMU
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_EXT_APIC);// ???
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_CR8L);    // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_ABM);     // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_SSE4A);   // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_MISALNSSE);//-> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF);// -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_OSVW);    // -> EMU?
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_IBS);     // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_XOP);     // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_SKINIT);  // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEcx, X86_CPUID_AMD_FEATURE_ECX_WDT);     // -> EMU
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(14));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(15));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(16));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(17));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(18));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(19));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(20));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(21));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(22));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(23));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(24));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(25));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(26));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(27));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(28));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(29));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(30));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEcx, RT_BIT_32(31));
+
+        /* CPUID(0x80000001).edx */
+        CPUID_GST_FEATURE2_RET(        uEdx, X86_CPUID_AMD_FEATURE_EDX_FPU,   X86_CPUID_FEATURE_EDX_FPU);     // -> EMU
+        CPUID_GST_FEATURE2_RET(        uEdx, X86_CPUID_AMD_FEATURE_EDX_VME,   X86_CPUID_FEATURE_EDX_VME);     // -> EMU
+        CPUID_GST_FEATURE2_RET(        uEdx, X86_CPUID_AMD_FEATURE_EDX_DE,    X86_CPUID_FEATURE_EDX_DE);      // -> EMU
+        CPUID_GST_FEATURE2_IGN(        uEdx, X86_CPUID_AMD_FEATURE_EDX_PSE,   X86_CPUID_FEATURE_EDX_PSE);
+        CPUID_GST_FEATURE2_RET(        uEdx, X86_CPUID_AMD_FEATURE_EDX_TSC,   X86_CPUID_FEATURE_EDX_TSC);     // -> EMU
+        CPUID_GST_FEATURE2_RET(        uEdx, X86_CPUID_AMD_FEATURE_EDX_MSR,   X86_CPUID_FEATURE_EDX_MSR);     // -> EMU
+        CPUID_GST_FEATURE2_RET(        uEdx, X86_CPUID_AMD_FEATURE_EDX_PAE,   X86_CPUID_FEATURE_EDX_PAE);
+        CPUID_GST_FEATURE2_IGN(        uEdx, X86_CPUID_AMD_FEATURE_EDX_MCE,   X86_CPUID_FEATURE_EDX_MCE);
+        CPUID_GST_FEATURE2_RET(        uEdx, X86_CPUID_AMD_FEATURE_EDX_CX8,   X86_CPUID_FEATURE_EDX_CX8);     // -> EMU?
+        CPUID_GST_FEATURE2_IGN(        uEdx, X86_CPUID_AMD_FEATURE_EDX_APIC,  X86_CPUID_FEATURE_EDX_APIC);
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEdx, RT_BIT_32(10) /*reserved*/);
+        CPUID_GST_FEATURE_IGN(    Ext, uEdx, X86_CPUID_EXT_FEATURE_EDX_SYSCALL);                              // On Intel: long mode only.
+        CPUID_GST_FEATURE2_IGN(        uEdx, X86_CPUID_AMD_FEATURE_EDX_MTRR,  X86_CPUID_FEATURE_EDX_MTRR);
+        CPUID_GST_FEATURE2_IGN(        uEdx, X86_CPUID_AMD_FEATURE_EDX_PGE,   X86_CPUID_FEATURE_EDX_PGE);
+        CPUID_GST_FEATURE2_IGN(        uEdx, X86_CPUID_AMD_FEATURE_EDX_MCA,   X86_CPUID_FEATURE_EDX_MCA);
+        CPUID_GST_FEATURE2_RET(        uEdx, X86_CPUID_AMD_FEATURE_EDX_CMOV,  X86_CPUID_FEATURE_EDX_CMOV);    // -> EMU
+        CPUID_GST_FEATURE2_IGN(        uEdx, X86_CPUID_AMD_FEATURE_EDX_PAT,   X86_CPUID_FEATURE_EDX_PAT);
+        CPUID_GST_FEATURE2_IGN(        uEdx, X86_CPUID_AMD_FEATURE_EDX_PSE36, X86_CPUID_FEATURE_EDX_PSE36);
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEdx, RT_BIT_32(18) /*reserved*/);
+        CPUID_GST_AMD_FEATURE_WRN(Ext, uEdx, RT_BIT_32(19) /*reserved*/);
+        CPUID_GST_FEATURE_RET(    Ext, uEdx, X86_CPUID_EXT_FEATURE_EDX_NX);
+        CPUID_GST_FEATURE_WRN(    Ext, uEdx, RT_BIT_32(21) /*reserved*/);
+        CPUID_GST_FEATURE_RET(    Ext, uEdx, X86_CPUID_AMD_FEATURE_EDX_AXMMX);
+        CPUID_GST_FEATURE2_RET(        uEdx, X86_CPUID_AMD_FEATURE_EDX_MMX,   X86_CPUID_FEATURE_EDX_MMX);     // -> EMU
+        CPUID_GST_FEATURE2_RET(        uEdx, X86_CPUID_AMD_FEATURE_EDX_FXSR,  X86_CPUID_FEATURE_EDX_FXSR);    // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEdx, X86_CPUID_AMD_FEATURE_EDX_FFXSR);
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEdx, X86_CPUID_EXT_FEATURE_EDX_PAGE1GB);
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEdx, X86_CPUID_EXT_FEATURE_EDX_RDTSCP);
+        CPUID_GST_FEATURE_IGN(    Ext, uEdx, RT_BIT_32(28) /*reserved*/);
+        CPUID_GST_FEATURE_RET(    Ext, uEdx, X86_CPUID_EXT_FEATURE_EDX_LONG_MODE);
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEdx, X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX);
+        CPUID_GST_AMD_FEATURE_RET(Ext, uEdx, X86_CPUID_AMD_FEATURE_EDX_3DNOW);
+    }
+
+    /** @todo check leaf 7   */
+
+    /* CPUID(d) - XCR0 stuff - takes ECX as input.
+     * ECX=0:   EAX - Valid bits in XCR0[31:0].
+     *          EBX - Maximum state size as per current XCR0 value.
+     *          ECX - Maximum state size for all supported features.
+     *          EDX - Valid bits in XCR0[63:32].
+     * ECX=1:   EAX - Various X-features.
+     *          EBX - Maximum state size as per current XCR0|IA32_XSS value.
+     *          ECX - Valid bits in IA32_XSS[31:0].
+     *          EDX - Valid bits in IA32_XSS[63:32].
+     * ECX=N, where N in 2..63 and indicates a bit in XCR0 and/or IA32_XSS,
+     *        if the bit invalid all four registers are set to zero.
+     *          EAX - The state size for this feature.
+     *          EBX - The state byte offset of this feature.
+     *          ECX - Bit 0 indicates whether this sub-leaf maps to a valid IA32_XSS bit (=1) or a valid XCR0 bit (=0).
+     *          EDX - Reserved, but is set to zero if invalid sub-leaf index.
+     */
+    uint64_t fGuestXcr0Mask = 0;
+    PCPUMCPUIDLEAF pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x0000000d), 0);
+    if (   pCurLeaf
+        && (aGuestCpuIdStd[1].uEcx & X86_CPUID_FEATURE_ECX_XSAVE)
+        && (   pCurLeaf->uEax
+            || pCurLeaf->uEbx
+            || pCurLeaf->uEcx
+            || pCurLeaf->uEdx) )
+    {
+        fGuestXcr0Mask = RT_MAKE_U64(pCurLeaf->uEax, pCurLeaf->uEdx);
+        if (fGuestXcr0Mask & ~pVM->cpum.s.fXStateHostMask)
+            return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS,
+                                     N_("CPUID(0xd/0).EDX:EAX mismatch: %#llx saved, %#llx supported by the current host (XCR0 bits)"),
+                                     fGuestXcr0Mask, pVM->cpum.s.fXStateHostMask);
+        if ((fGuestXcr0Mask & (XSAVE_C_X87 | XSAVE_C_SSE)) != (XSAVE_C_X87 | XSAVE_C_SSE))
+            return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS,
+                                     N_("CPUID(0xd/0).EDX:EAX missing mandatory X87 or SSE bits: %#RX64"), fGuestXcr0Mask);
+
+        /* We don't support any additional features yet. */
+        pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x0000000d), 1);
+        if (pCurLeaf && pCurLeaf->uEax)
+            return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS,
+                                     N_("CPUID(0xd/1).EAX=%#x, expected zero"), pCurLeaf->uEax);
+        if (pCurLeaf && (pCurLeaf->uEcx || pCurLeaf->uEdx))
+            return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS,
+                                     N_("CPUID(0xd/1).EDX:ECX=%#llx, expected zero"),
+                                     RT_MAKE_U64(pCurLeaf->uEdx, pCurLeaf->uEcx));
+
+
+        for (uint32_t uSubLeaf = 2; uSubLeaf < 64; uSubLeaf++)
+        {
+            pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x0000000d), uSubLeaf);
+            if (pCurLeaf)
+            {
+                /* If advertised, the state component offset and size must match the one used by host. */
+                if (pCurLeaf->uEax || pCurLeaf->uEbx || pCurLeaf->uEcx || pCurLeaf->uEdx)
+                {
+                    CPUMCPUID RawHost;
+                    ASMCpuIdExSlow(UINT32_C(0x0000000d), 0, uSubLeaf, 0,
+                                   &RawHost.uEax, &RawHost.uEbx, &RawHost.uEcx, &RawHost.uEdx);
+                    if (   RawHost.uEbx != pCurLeaf->uEbx
+                        || RawHost.uEax != pCurLeaf->uEax)
+                        return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS,
+                                                 N_("CPUID(0xd/%#x).EBX/EAX=%#x/%#x, current host uses %#x/%#x (offset/size)"),
+                                                 uSubLeaf, pCurLeaf->uEbx, pCurLeaf->uEax, RawHost.uEbx, RawHost.uEax);
+                }
+            }
+        }
+    }
+    /* Clear leaf 0xd just in case we're loading an old state... */
+    else if (pCurLeaf)
+    {
+        for (uint32_t uSubLeaf = 0; uSubLeaf < 64; uSubLeaf++)
+        {
+            pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x0000000d), uSubLeaf);
+            if (pCurLeaf)
+            {
+                AssertLogRelMsg(   uVersion <= CPUM_SAVED_STATE_VERSION_PUT_STRUCT
+                                || (   pCurLeaf->uEax == 0
+                                    && pCurLeaf->uEbx == 0
+                                    && pCurLeaf->uEcx == 0
+                                    && pCurLeaf->uEdx == 0),
+                                ("uVersion=%#x; %#x %#x %#x %#x\n",
+                                 uVersion, pCurLeaf->uEax, pCurLeaf->uEbx, pCurLeaf->uEcx, pCurLeaf->uEdx));
+                pCurLeaf->uEax = pCurLeaf->uEbx = pCurLeaf->uEcx = pCurLeaf->uEdx = 0;
+            }
+        }
+    }
+
+    /* Update the fXStateGuestMask value for the VM. */
+    if (pVM->cpum.s.fXStateGuestMask != fGuestXcr0Mask)
+    {
+        LogRel(("CPUM: fXStateGuestMask=%#llx -> %#llx\n", pVM->cpum.s.fXStateGuestMask, fGuestXcr0Mask));
+        pVM->cpum.s.fXStateGuestMask = fGuestXcr0Mask;
+        if (!fGuestXcr0Mask && (aGuestCpuIdStd[1].uEcx & X86_CPUID_FEATURE_ECX_XSAVE))
+            return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS,
+                                     N_("Internal Processing Error: XSAVE feature bit enabled, but leaf 0xd is empty."));
+    }
+
+#undef CPUID_CHECK_RET
+#undef CPUID_CHECK_WRN
+#undef CPUID_CHECK2_RET
+#undef CPUID_CHECK2_WRN
+#undef CPUID_RAW_FEATURE_RET
+#undef CPUID_RAW_FEATURE_WRN
+#undef CPUID_RAW_FEATURE_IGN
+#undef CPUID_GST_FEATURE_RET
+#undef CPUID_GST_FEATURE_WRN
+#undef CPUID_GST_FEATURE_EMU
+#undef CPUID_GST_FEATURE_IGN
+#undef CPUID_GST_FEATURE2_RET
+#undef CPUID_GST_FEATURE2_WRN
+#undef CPUID_GST_FEATURE2_EMU
+#undef CPUID_GST_FEATURE2_IGN
+#undef CPUID_GST_AMD_FEATURE_RET
+#undef CPUID_GST_AMD_FEATURE_WRN
+#undef CPUID_GST_AMD_FEATURE_EMU
+#undef CPUID_GST_AMD_FEATURE_IGN
+
+    /*
+     * We're good, commit the CPU ID leaves.
+     */
+    MMHyperFree(pVM, pVM->cpum.s.GuestInfo.paCpuIdLeavesR3);
+    pVM->cpum.s.GuestInfo.paCpuIdLeavesR3 = NULL;
+    pVM->cpum.s.GuestInfo.paCpuIdLeavesR0 = NIL_RTR0PTR;
+    pVM->cpum.s.GuestInfo.DefCpuId = GuestDefCpuId;
+    rc = cpumR3CpuIdInstallAndExplodeLeaves(pVM, &pVM->cpum.s, paLeaves, cLeaves, pMsrs);
+    AssertLogRelRCReturn(rc, rc);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads the CPU ID leaves saved by pass 0.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ * @param   uVersion            The format version.
+ * @param   pMsrs               The guest MSRs.
+ */
+int cpumR3LoadCpuId(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, PCCPUMMSRS pMsrs)
+{
+    AssertMsgReturn(uVersion >= CPUM_SAVED_STATE_VERSION_VER3_2, ("%u\n", uVersion), VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION);
+
+    /*
+     * Load the CPUID leaves array first and call worker to do the rest, just so
+     * we can free the memory when we need to without ending up in column 1000.
+     */
+    PCPUMCPUIDLEAF paLeaves;
+    uint32_t       cLeaves;
+    int rc = cpumR3LoadGuestCpuIdArray(pVM, pSSM, uVersion, &paLeaves, &cLeaves);
+    AssertRC(rc);
+    if (RT_SUCCESS(rc))
+    {
+        rc = cpumR3LoadCpuIdInner(pVM, pSSM, uVersion, paLeaves, cLeaves, pMsrs);
+        RTMemFree(paLeaves);
+    }
+    return rc;
+}
+
+
+
+/**
+ * Loads the CPU ID leaves saved by pass 0 in an pre 3.2 saved state.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ * @param   uVersion            The format version.
+ */
+int cpumR3LoadCpuIdPre32(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion)
+{
+    AssertMsgReturn(uVersion < CPUM_SAVED_STATE_VERSION_VER3_2, ("%u\n", uVersion), VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION);
+
+    /*
+     * Restore the CPUID leaves.
+     *
+     * Note that we support restoring less than the current amount of standard
+     * leaves because we've been allowed more is newer version of VBox.
+     */
+    uint32_t cElements;
+    int rc = SSMR3GetU32(pSSM, &cElements); AssertRCReturn(rc, rc);
+    if (cElements > RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdPatmStd))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    SSMR3GetMem(pSSM, &pVM->cpum.s.aGuestCpuIdPatmStd[0], cElements*sizeof(pVM->cpum.s.aGuestCpuIdPatmStd[0]));
+
+    rc = SSMR3GetU32(pSSM, &cElements); AssertRCReturn(rc, rc);
+    if (cElements != RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdPatmExt))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    SSMR3GetMem(pSSM, &pVM->cpum.s.aGuestCpuIdPatmExt[0], sizeof(pVM->cpum.s.aGuestCpuIdPatmExt));
+
+    rc = SSMR3GetU32(pSSM, &cElements); AssertRCReturn(rc, rc);
+    if (cElements != RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdPatmCentaur))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    SSMR3GetMem(pSSM, &pVM->cpum.s.aGuestCpuIdPatmCentaur[0], sizeof(pVM->cpum.s.aGuestCpuIdPatmCentaur));
+
+    SSMR3GetMem(pSSM, &pVM->cpum.s.GuestInfo.DefCpuId, sizeof(pVM->cpum.s.GuestInfo.DefCpuId));
+
+    /*
+     * Check that the basic cpuid id information is unchanged.
+     */
+    /** @todo we should check the 64 bits capabilities too! */
+    uint32_t au32CpuId[8] = {0,0,0,0, 0,0,0,0};
+    ASMCpuIdExSlow(0, 0, 0, 0, &au32CpuId[0], &au32CpuId[1], &au32CpuId[2], &au32CpuId[3]);
+    ASMCpuIdExSlow(1, 0, 0, 0, &au32CpuId[4], &au32CpuId[5], &au32CpuId[6], &au32CpuId[7]);
+    uint32_t au32CpuIdSaved[8];
+    rc = SSMR3GetMem(pSSM, &au32CpuIdSaved[0], sizeof(au32CpuIdSaved));
+    if (RT_SUCCESS(rc))
+    {
+        /* Ignore CPU stepping. */
+        au32CpuId[4]      &=  0xfffffff0;
+        au32CpuIdSaved[4] &=  0xfffffff0;
+
+        /* Ignore APIC ID (AMD specs). */
+        au32CpuId[5]      &= ~0xff000000;
+        au32CpuIdSaved[5] &= ~0xff000000;
+
+        /* Ignore the number of Logical CPUs (AMD specs). */
+        au32CpuId[5]      &= ~0x00ff0000;
+        au32CpuIdSaved[5] &= ~0x00ff0000;
+
+        /* Ignore some advanced capability bits, that we don't expose to the guest. */
+        au32CpuId[6]      &= ~(   X86_CPUID_FEATURE_ECX_DTES64
+                               |  X86_CPUID_FEATURE_ECX_VMX
+                               |  X86_CPUID_FEATURE_ECX_SMX
+                               |  X86_CPUID_FEATURE_ECX_EST
+                               |  X86_CPUID_FEATURE_ECX_TM2
+                               |  X86_CPUID_FEATURE_ECX_CNTXID
+                               |  X86_CPUID_FEATURE_ECX_TPRUPDATE
+                               |  X86_CPUID_FEATURE_ECX_PDCM
+                               |  X86_CPUID_FEATURE_ECX_DCA
+                               |  X86_CPUID_FEATURE_ECX_X2APIC
+                              );
+        au32CpuIdSaved[6] &= ~(   X86_CPUID_FEATURE_ECX_DTES64
+                               |  X86_CPUID_FEATURE_ECX_VMX
+                               |  X86_CPUID_FEATURE_ECX_SMX
+                               |  X86_CPUID_FEATURE_ECX_EST
+                               |  X86_CPUID_FEATURE_ECX_TM2
+                               |  X86_CPUID_FEATURE_ECX_CNTXID
+                               |  X86_CPUID_FEATURE_ECX_TPRUPDATE
+                               |  X86_CPUID_FEATURE_ECX_PDCM
+                               |  X86_CPUID_FEATURE_ECX_DCA
+                               |  X86_CPUID_FEATURE_ECX_X2APIC
+                              );
+
+        /* Make sure we don't forget to update the masks when enabling
+         * features in the future.
+         */
+        AssertRelease(!(pVM->cpum.s.aGuestCpuIdPatmStd[1].uEcx &
+                              (   X86_CPUID_FEATURE_ECX_DTES64
+                               |  X86_CPUID_FEATURE_ECX_VMX
+                               |  X86_CPUID_FEATURE_ECX_SMX
+                               |  X86_CPUID_FEATURE_ECX_EST
+                               |  X86_CPUID_FEATURE_ECX_TM2
+                               |  X86_CPUID_FEATURE_ECX_CNTXID
+                               |  X86_CPUID_FEATURE_ECX_TPRUPDATE
+                               |  X86_CPUID_FEATURE_ECX_PDCM
+                               |  X86_CPUID_FEATURE_ECX_DCA
+                               |  X86_CPUID_FEATURE_ECX_X2APIC
+                              )));
+        /* do the compare */
+        if (memcmp(au32CpuIdSaved, au32CpuId, sizeof(au32CpuIdSaved)))
+        {
+            if (SSMR3HandleGetAfter(pSSM) == SSMAFTER_DEBUG_IT)
+                LogRel(("cpumR3LoadExec: CpuId mismatch! (ignored due to SSMAFTER_DEBUG_IT)\n"
+                        "Saved=%.*Rhxs\n"
+                        "Real =%.*Rhxs\n",
+                        sizeof(au32CpuIdSaved), au32CpuIdSaved,
+                        sizeof(au32CpuId), au32CpuId));
+            else
+            {
+                LogRel(("cpumR3LoadExec: CpuId mismatch!\n"
+                        "Saved=%.*Rhxs\n"
+                        "Real =%.*Rhxs\n",
+                        sizeof(au32CpuIdSaved), au32CpuIdSaved,
+                        sizeof(au32CpuId), au32CpuId));
+                rc = VERR_SSM_LOAD_CPUID_MISMATCH;
+            }
+        }
+    }
+
+    return rc;
+}
+
+
+
+/*
+ *
+ *
+ * CPUID Info Handler.
+ * CPUID Info Handler.
+ * CPUID Info Handler.
+ *
+ *
+ */
+
+
+
+/**
+ * Get L1 cache / TLS associativity.
+ */
+static const char *getCacheAss(unsigned u, char *pszBuf)
+{
+    if (u == 0)
+        return "res0  ";
+    if (u == 1)
+        return "direct";
+    if (u == 255)
+        return "fully";
+    if (u >= 256)
+        return "???";
+
+    RTStrPrintf(pszBuf, 16, "%d way", u);
+    return pszBuf;
+}
+
+
+/**
+ * Get L2 cache associativity.
+ */
+const char *getL2CacheAss(unsigned u)
+{
+    switch (u)
+    {
+        case 0:  return "off   ";
+        case 1:  return "direct";
+        case 2:  return "2 way ";
+        case 3:  return "res3  ";
+        case 4:  return "4 way ";
+        case 5:  return "res5  ";
+        case 6:  return "8 way ";
+        case 7:  return "res7  ";
+        case 8:  return "16 way";
+        case 9:  return "res9  ";
+        case 10: return "res10 ";
+        case 11: return "res11 ";
+        case 12: return "res12 ";
+        case 13: return "res13 ";
+        case 14: return "res14 ";
+        case 15: return "fully ";
+        default: return "????";
+    }
+}
+
+
+/** CPUID(1).EDX field descriptions. */
+static DBGFREGSUBFIELD const g_aLeaf1EdxSubFields[] =
+{
+    DBGFREGSUBFIELD_RO("FPU\0"         "x87 FPU on Chip",                            0, 1, 0),
+    DBGFREGSUBFIELD_RO("VME\0"         "Virtual 8086 Mode Enhancements",             1, 1, 0),
+    DBGFREGSUBFIELD_RO("DE\0"          "Debugging extensions",                       2, 1, 0),
+    DBGFREGSUBFIELD_RO("PSE\0"         "Page Size Extension",                        3, 1, 0),
+    DBGFREGSUBFIELD_RO("TSC\0"         "Time Stamp Counter",                         4, 1, 0),
+    DBGFREGSUBFIELD_RO("MSR\0"         "Model Specific Registers",                   5, 1, 0),
+    DBGFREGSUBFIELD_RO("PAE\0"         "Physical Address Extension",                 6, 1, 0),
+    DBGFREGSUBFIELD_RO("MCE\0"         "Machine Check Exception",                    7, 1, 0),
+    DBGFREGSUBFIELD_RO("CX8\0"         "CMPXCHG8B instruction",                      8, 1, 0),
+    DBGFREGSUBFIELD_RO("APIC\0"        "APIC On-Chip",                               9, 1, 0),
+    DBGFREGSUBFIELD_RO("SEP\0"         "SYSENTER and SYSEXIT Present",              11, 1, 0),
+    DBGFREGSUBFIELD_RO("MTRR\0"        "Memory Type Range Registers",               12, 1, 0),
+    DBGFREGSUBFIELD_RO("PGE\0"         "PTE Global Bit",                            13, 1, 0),
+    DBGFREGSUBFIELD_RO("MCA\0"         "Machine Check Architecture",                14, 1, 0),
+    DBGFREGSUBFIELD_RO("CMOV\0"        "Conditional Move instructions",             15, 1, 0),
+    DBGFREGSUBFIELD_RO("PAT\0"         "Page Attribute Table",                      16, 1, 0),
+    DBGFREGSUBFIELD_RO("PSE-36\0"      "36-bit Page Size Extension",                17, 1, 0),
+    DBGFREGSUBFIELD_RO("PSN\0"         "Processor Serial Number",                   18, 1, 0),
+    DBGFREGSUBFIELD_RO("CLFSH\0"       "CLFLUSH instruction",                       19, 1, 0),
+    DBGFREGSUBFIELD_RO("DS\0"          "Debug Store",                               21, 1, 0),
+    DBGFREGSUBFIELD_RO("ACPI\0"        "Thermal Mon. & Soft. Clock Ctrl.",          22, 1, 0),
+    DBGFREGSUBFIELD_RO("MMX\0"         "Intel MMX Technology",                      23, 1, 0),
+    DBGFREGSUBFIELD_RO("FXSR\0"        "FXSAVE and FXRSTOR instructions",           24, 1, 0),
+    DBGFREGSUBFIELD_RO("SSE\0"         "SSE support",                               25, 1, 0),
+    DBGFREGSUBFIELD_RO("SSE2\0"        "SSE2 support",                              26, 1, 0),
+    DBGFREGSUBFIELD_RO("SS\0"          "Self Snoop",                                27, 1, 0),
+    DBGFREGSUBFIELD_RO("HTT\0"         "Hyper-Threading Technology",                28, 1, 0),
+    DBGFREGSUBFIELD_RO("TM\0"          "Therm. Monitor",                            29, 1, 0),
+    DBGFREGSUBFIELD_RO("PBE\0"         "Pending Break Enabled",                     31, 1, 0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** CPUID(1).ECX field descriptions. */
+static DBGFREGSUBFIELD const g_aLeaf1EcxSubFields[] =
+{
+    DBGFREGSUBFIELD_RO("SSE3\0"       "SSE3 support",                                    0, 1, 0),
+    DBGFREGSUBFIELD_RO("PCLMUL\0"     "PCLMULQDQ support (for AES-GCM)",                 1, 1, 0),
+    DBGFREGSUBFIELD_RO("DTES64\0"     "DS Area 64-bit Layout",                           2, 1, 0),
+    DBGFREGSUBFIELD_RO("MONITOR\0"    "MONITOR/MWAIT instructions",                      3, 1, 0),
+    DBGFREGSUBFIELD_RO("CPL-DS\0"     "CPL Qualified Debug Store",                       4, 1, 0),
+    DBGFREGSUBFIELD_RO("VMX\0"        "Virtual Machine Extensions",                      5, 1, 0),
+    DBGFREGSUBFIELD_RO("SMX\0"        "Safer Mode Extensions",                           6, 1, 0),
+    DBGFREGSUBFIELD_RO("EST\0"        "Enhanced SpeedStep Technology",                   7, 1, 0),
+    DBGFREGSUBFIELD_RO("TM2\0"        "Terminal Monitor 2",                              8, 1, 0),
+    DBGFREGSUBFIELD_RO("SSSE3\0"      "Supplemental Streaming SIMD Extensions 3",        9, 1, 0),
+    DBGFREGSUBFIELD_RO("CNTX-ID\0"    "L1 Context ID",                                  10, 1, 0),
+    DBGFREGSUBFIELD_RO("SDBG\0"       "Silicon Debug interface",                        11, 1, 0),
+    DBGFREGSUBFIELD_RO("FMA\0"        "Fused Multiply Add extensions",                  12, 1, 0),
+    DBGFREGSUBFIELD_RO("CX16\0"       "CMPXCHG16B instruction",                         13, 1, 0),
+    DBGFREGSUBFIELD_RO("TPRUPDATE\0"  "xTPR Update Control",                            14, 1, 0),
+    DBGFREGSUBFIELD_RO("PDCM\0"       "Perf/Debug Capability MSR",                      15, 1, 0),
+    DBGFREGSUBFIELD_RO("PCID\0"       "Process Context Identifiers",                    17, 1, 0),
+    DBGFREGSUBFIELD_RO("DCA\0"        "Direct Cache Access",                            18, 1, 0),
+    DBGFREGSUBFIELD_RO("SSE4_1\0"     "SSE4_1 support",                                 19, 1, 0),
+    DBGFREGSUBFIELD_RO("SSE4_2\0"     "SSE4_2 support",                                 20, 1, 0),
+    DBGFREGSUBFIELD_RO("X2APIC\0"     "x2APIC support",                                 21, 1, 0),
+    DBGFREGSUBFIELD_RO("MOVBE\0"      "MOVBE instruction",                              22, 1, 0),
+    DBGFREGSUBFIELD_RO("POPCNT\0"     "POPCNT instruction",                             23, 1, 0),
+    DBGFREGSUBFIELD_RO("TSCDEADL\0"   "Time Stamp Counter Deadline",                    24, 1, 0),
+    DBGFREGSUBFIELD_RO("AES\0"        "AES instructions",                               25, 1, 0),
+    DBGFREGSUBFIELD_RO("XSAVE\0"      "XSAVE instruction",                              26, 1, 0),
+    DBGFREGSUBFIELD_RO("OSXSAVE\0"    "OSXSAVE instruction",                            27, 1, 0),
+    DBGFREGSUBFIELD_RO("AVX\0"        "AVX support",                                    28, 1, 0),
+    DBGFREGSUBFIELD_RO("F16C\0"       "16-bit floating point conversion instructions",  29, 1, 0),
+    DBGFREGSUBFIELD_RO("RDRAND\0"     "RDRAND instruction",                             30, 1, 0),
+    DBGFREGSUBFIELD_RO("HVP\0"        "Hypervisor Present (we're a guest)",             31, 1, 0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** CPUID(7,0).EBX field descriptions. */
+static DBGFREGSUBFIELD const g_aLeaf7Sub0EbxSubFields[] =
+{
+    DBGFREGSUBFIELD_RO("FSGSBASE\0"         "RDFSBASE/RDGSBASE/WRFSBASE/WRGSBASE instr.",    0, 1, 0),
+    DBGFREGSUBFIELD_RO("TSCADJUST\0"        "Supports MSR_IA32_TSC_ADJUST",                  1, 1, 0),
+    DBGFREGSUBFIELD_RO("SGX\0"              "Supports Software Guard Extensions",            2, 1, 0),
+    DBGFREGSUBFIELD_RO("BMI1\0"             "Advanced Bit Manipulation extension 1",         3, 1, 0),
+    DBGFREGSUBFIELD_RO("HLE\0"              "Hardware Lock Elision",                         4, 1, 0),
+    DBGFREGSUBFIELD_RO("AVX2\0"             "Advanced Vector Extensions 2",                  5, 1, 0),
+    DBGFREGSUBFIELD_RO("FDP_EXCPTN_ONLY\0"  "FPU DP only updated on exceptions",             6, 1, 0),
+    DBGFREGSUBFIELD_RO("SMEP\0"             "Supervisor Mode Execution Prevention",          7, 1, 0),
+    DBGFREGSUBFIELD_RO("BMI2\0"             "Advanced Bit Manipulation extension 2",         8, 1, 0),
+    DBGFREGSUBFIELD_RO("ERMS\0"             "Enhanced REP MOVSB/STOSB instructions",         9, 1, 0),
+    DBGFREGSUBFIELD_RO("INVPCID\0"          "INVPCID instruction",                          10, 1, 0),
+    DBGFREGSUBFIELD_RO("RTM\0"              "Restricted Transactional Memory",              11, 1, 0),
+    DBGFREGSUBFIELD_RO("PQM\0"              "Platform Quality of Service Monitoring",       12, 1, 0),
+    DBGFREGSUBFIELD_RO("DEPFPU_CS_DS\0"     "Deprecates FPU CS, FPU DS values if set",      13, 1, 0),
+    DBGFREGSUBFIELD_RO("MPE\0"              "Intel Memory Protection Extensions",           14, 1, 0),
+    DBGFREGSUBFIELD_RO("PQE\0"              "Platform Quality of Service Enforcement",      15, 1, 0),
+    DBGFREGSUBFIELD_RO("AVX512F\0"          "AVX512 Foundation instructions",               16, 1, 0),
+    DBGFREGSUBFIELD_RO("RDSEED\0"           "RDSEED instruction",                           18, 1, 0),
+    DBGFREGSUBFIELD_RO("ADX\0"              "ADCX/ADOX instructions",                       19, 1, 0),
+    DBGFREGSUBFIELD_RO("SMAP\0"             "Supervisor Mode Access Prevention",            20, 1, 0),
+    DBGFREGSUBFIELD_RO("CLFLUSHOPT\0"       "CLFLUSHOPT (Cache Line Flush) instruction",    23, 1, 0),
+    DBGFREGSUBFIELD_RO("INTEL_PT\0"         "Intel Processor Trace",                        25, 1, 0),
+    DBGFREGSUBFIELD_RO("AVX512PF\0"         "AVX512 Prefetch instructions",                 26, 1, 0),
+    DBGFREGSUBFIELD_RO("AVX512ER\0"         "AVX512 Exponential & Reciprocal instructions", 27, 1, 0),
+    DBGFREGSUBFIELD_RO("AVX512CD\0"         "AVX512 Conflict Detection instructions",       28, 1, 0),
+    DBGFREGSUBFIELD_RO("SHA\0"              "Secure Hash Algorithm extensions",             29, 1, 0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** CPUID(7,0).ECX field descriptions.   */
+static DBGFREGSUBFIELD const g_aLeaf7Sub0EcxSubFields[] =
+{
+    DBGFREGSUBFIELD_RO("PREFETCHWT1\0" "PREFETCHWT1 instruction",                        0, 1, 0),
+    DBGFREGSUBFIELD_RO("UMIP\0"         "User mode insturction prevention",              2, 1, 0),
+    DBGFREGSUBFIELD_RO("PKU\0"          "Protection Key for Usermode pages",             3, 1, 0),
+    DBGFREGSUBFIELD_RO("OSPKE\0"        "CR4.PKU mirror",                                4, 1, 0),
+    DBGFREGSUBFIELD_RO("MAWAU\0"        "Value used by BNDLDX & BNDSTX",                17, 5, 0),
+    DBGFREGSUBFIELD_RO("RDPID\0"        "Read processor ID support",                    22, 1, 0),
+    DBGFREGSUBFIELD_RO("SGX_LC\0"       "Supports SGX Launch Configuration",            30, 1, 0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** CPUID(7,0).EDX field descriptions.   */
+static DBGFREGSUBFIELD const g_aLeaf7Sub0EdxSubFields[] =
+{
+    DBGFREGSUBFIELD_RO("MD_CLEAR\0"     "Supports MDS related buffer clearing",         10, 1, 0),
+    DBGFREGSUBFIELD_RO("IBRS_IBPB\0"    "IA32_SPEC_CTRL.IBRS and IA32_PRED_CMD.IBPB",   26, 1, 0),
+    DBGFREGSUBFIELD_RO("STIBP\0"        "Supports IA32_SPEC_CTRL.STIBP",                27, 1, 0),
+    DBGFREGSUBFIELD_RO("FLUSH_CMD\0"    "Supports IA32_FLUSH_CMD",                      28, 1, 0),
+    DBGFREGSUBFIELD_RO("ARCHCAP\0"      "Supports IA32_ARCH_CAP",                       29, 1, 0),
+    DBGFREGSUBFIELD_RO("CORECAP\0"      "Supports IA32_CORE_CAP",                       30, 1, 0),
+    DBGFREGSUBFIELD_RO("SSBD\0"         "Supports IA32_SPEC_CTRL.SSBD",                 31, 1, 0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+
+/** CPUID(13,0).EAX+EDX, XCR0, ++ bit descriptions. */
+static DBGFREGSUBFIELD const g_aXSaveStateBits[] =
+{
+    DBGFREGSUBFIELD_RO("x87\0"       "Legacy FPU state",                                 0, 1, 0),
+    DBGFREGSUBFIELD_RO("SSE\0"       "128-bit SSE state",                                1, 1, 0),
+    DBGFREGSUBFIELD_RO("YMM_Hi128\0" "Upper 128 bits of YMM0-15 (AVX)",                  2, 1, 0),
+    DBGFREGSUBFIELD_RO("BNDREGS\0"   "MPX bound register state",                         3, 1, 0),
+    DBGFREGSUBFIELD_RO("BNDCSR\0"    "MPX bound config and status state",                4, 1, 0),
+    DBGFREGSUBFIELD_RO("Opmask\0"    "opmask state",                                     5, 1, 0),
+    DBGFREGSUBFIELD_RO("ZMM_Hi256\0" "Upper 256 bits of ZMM0-15 (AVX-512)",              6, 1, 0),
+    DBGFREGSUBFIELD_RO("Hi16_ZMM\0"  "512-bits ZMM16-31 state (AVX-512)",                7, 1, 0),
+    DBGFREGSUBFIELD_RO("LWP\0"       "Lightweight Profiling (AMD)",                     62, 1, 0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** CPUID(13,1).EAX field descriptions.   */
+static DBGFREGSUBFIELD const g_aLeaf13Sub1EaxSubFields[] =
+{
+    DBGFREGSUBFIELD_RO("XSAVEOPT\0"  "XSAVEOPT is available",                            0, 1, 0),
+    DBGFREGSUBFIELD_RO("XSAVEC\0"    "XSAVEC and compacted XRSTOR supported",            1, 1, 0),
+    DBGFREGSUBFIELD_RO("XGETBC1\0"   "XGETBV with ECX=1 supported",                      2, 1, 0),
+    DBGFREGSUBFIELD_RO("XSAVES\0"    "XSAVES/XRSTORS and IA32_XSS supported",            3, 1, 0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+
+/** CPUID(0x80000001,0).EDX field descriptions.   */
+static DBGFREGSUBFIELD const g_aExtLeaf1EdxSubFields[] =
+{
+    DBGFREGSUBFIELD_RO("FPU\0"          "x87 FPU on Chip",                               0, 1, 0),
+    DBGFREGSUBFIELD_RO("VME\0"          "Virtual 8086 Mode Enhancements",                1, 1, 0),
+    DBGFREGSUBFIELD_RO("DE\0"           "Debugging extensions",                          2, 1, 0),
+    DBGFREGSUBFIELD_RO("PSE\0"          "Page Size Extension",                           3, 1, 0),
+    DBGFREGSUBFIELD_RO("TSC\0"          "Time Stamp Counter",                            4, 1, 0),
+    DBGFREGSUBFIELD_RO("MSR\0"          "K86 Model Specific Registers",                  5, 1, 0),
+    DBGFREGSUBFIELD_RO("PAE\0"          "Physical Address Extension",                    6, 1, 0),
+    DBGFREGSUBFIELD_RO("MCE\0"          "Machine Check Exception",                       7, 1, 0),
+    DBGFREGSUBFIELD_RO("CX8\0"          "CMPXCHG8B instruction",                         8, 1, 0),
+    DBGFREGSUBFIELD_RO("APIC\0"         "APIC On-Chip",                                  9, 1, 0),
+    DBGFREGSUBFIELD_RO("SEP\0"          "SYSCALL/SYSRET",                               11, 1, 0),
+    DBGFREGSUBFIELD_RO("MTRR\0"         "Memory Type Range Registers",                  12, 1, 0),
+    DBGFREGSUBFIELD_RO("PGE\0"          "PTE Global Bit",                               13, 1, 0),
+    DBGFREGSUBFIELD_RO("MCA\0"          "Machine Check Architecture",                   14, 1, 0),
+    DBGFREGSUBFIELD_RO("CMOV\0"         "Conditional Move instructions",                15, 1, 0),
+    DBGFREGSUBFIELD_RO("PAT\0"          "Page Attribute Table",                         16, 1, 0),
+    DBGFREGSUBFIELD_RO("PSE-36\0"       "36-bit Page Size Extension",                   17, 1, 0),
+    DBGFREGSUBFIELD_RO("NX\0"           "No-Execute/Execute-Disable",                   20, 1, 0),
+    DBGFREGSUBFIELD_RO("AXMMX\0"        "AMD Extensions to MMX instructions",           22, 1, 0),
+    DBGFREGSUBFIELD_RO("MMX\0"          "Intel MMX Technology",                         23, 1, 0),
+    DBGFREGSUBFIELD_RO("FXSR\0"         "FXSAVE and FXRSTOR Instructions",              24, 1, 0),
+    DBGFREGSUBFIELD_RO("FFXSR\0"        "AMD fast FXSAVE and FXRSTOR instructions",     25, 1, 0),
+    DBGFREGSUBFIELD_RO("Page1GB\0"      "1 GB large page",                              26, 1, 0),
+    DBGFREGSUBFIELD_RO("RDTSCP\0"       "RDTSCP instruction",                           27, 1, 0),
+    DBGFREGSUBFIELD_RO("LM\0"           "AMD64 Long Mode",                              29, 1, 0),
+    DBGFREGSUBFIELD_RO("3DNOWEXT\0"     "AMD Extensions to 3DNow",                      30, 1, 0),
+    DBGFREGSUBFIELD_RO("3DNOW\0"        "AMD 3DNow",                                    31, 1, 0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** CPUID(0x80000001,0).ECX field descriptions.   */
+static DBGFREGSUBFIELD const g_aExtLeaf1EcxSubFields[] =
+{
+    DBGFREGSUBFIELD_RO("LahfSahf\0"     "LAHF/SAHF support in 64-bit mode",              0, 1, 0),
+    DBGFREGSUBFIELD_RO("CmpLegacy\0"    "Core multi-processing legacy mode",             1, 1, 0),
+    DBGFREGSUBFIELD_RO("SVM\0"          "AMD Secure Virtual Machine extensions",         2, 1, 0),
+    DBGFREGSUBFIELD_RO("EXTAPIC\0"      "AMD Extended APIC registers",                   3, 1, 0),
+    DBGFREGSUBFIELD_RO("CR8L\0"         "AMD LOCK MOV CR0 means MOV CR8",                4, 1, 0),
+    DBGFREGSUBFIELD_RO("ABM\0"          "AMD Advanced Bit Manipulation",                 5, 1, 0),
+    DBGFREGSUBFIELD_RO("SSE4A\0"        "SSE4A instructions",                            6, 1, 0),
+    DBGFREGSUBFIELD_RO("MISALIGNSSE\0"  "AMD Misaligned SSE mode",                       7, 1, 0),
+    DBGFREGSUBFIELD_RO("3DNOWPRF\0"     "AMD PREFETCH and PREFETCHW instructions",       8, 1, 0),
+    DBGFREGSUBFIELD_RO("OSVW\0"         "AMD OS Visible Workaround",                     9, 1, 0),
+    DBGFREGSUBFIELD_RO("IBS\0"          "Instruct Based Sampling",                      10, 1, 0),
+    DBGFREGSUBFIELD_RO("XOP\0"          "Extended Operation support",                   11, 1, 0),
+    DBGFREGSUBFIELD_RO("SKINIT\0"       "SKINIT, STGI, and DEV support",                12, 1, 0),
+    DBGFREGSUBFIELD_RO("WDT\0"          "AMD Watchdog Timer support",                   13, 1, 0),
+    DBGFREGSUBFIELD_RO("LWP\0"          "Lightweight Profiling support",                15, 1, 0),
+    DBGFREGSUBFIELD_RO("FMA4\0"         "Four operand FMA instruction support",         16, 1, 0),
+    DBGFREGSUBFIELD_RO("NodeId\0"       "NodeId in MSR C001_100C",                      19, 1, 0),
+    DBGFREGSUBFIELD_RO("TBM\0"          "Trailing Bit Manipulation instructions",       21, 1, 0),
+    DBGFREGSUBFIELD_RO("TOPOEXT\0"      "Topology Extensions",                          22, 1, 0),
+    DBGFREGSUBFIELD_RO("PRFEXTCORE\0"   "Performance Counter Extensions support",       23, 1, 0),
+    DBGFREGSUBFIELD_RO("PRFEXTNB\0"     "NB Performance Counter Extensions support",    24, 1, 0),
+    DBGFREGSUBFIELD_RO("DATABPEXT\0"    "Data-access Breakpoint Extension",             26, 1, 0),
+    DBGFREGSUBFIELD_RO("PERFTSC\0"      "Performance Time Stamp Counter",               27, 1, 0),
+    DBGFREGSUBFIELD_RO("PCX_L2I\0"      "L2I/L3 Performance Counter Extensions",        28, 1, 0),
+    DBGFREGSUBFIELD_RO("MWAITX\0"       "MWAITX and MONITORX instructions",             29, 1, 0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** CPUID(0x8000000a,0).EDX field descriptions.   */
+static DBGFREGSUBFIELD const g_aExtLeafAEdxSubFields[] =
+{
+    DBGFREGSUBFIELD_RO("NP\0"             "Nested Paging",                               0, 1, 0),
+    DBGFREGSUBFIELD_RO("LbrVirt\0"        "Last Branch Record Virtualization",           1, 1, 0),
+    DBGFREGSUBFIELD_RO("SVML\0"           "SVM Lock",                                    2, 1, 0),
+    DBGFREGSUBFIELD_RO("NRIPS\0"          "NextRIP Save",                                3, 1, 0),
+    DBGFREGSUBFIELD_RO("TscRateMsr\0"     "MSR based TSC rate control",                  4, 1, 0),
+    DBGFREGSUBFIELD_RO("VmcbClean\0"      "VMCB clean bits",                             5, 1, 0),
+    DBGFREGSUBFIELD_RO("FlushByASID\0"    "Flush by ASID",                               6, 1, 0),
+    DBGFREGSUBFIELD_RO("DecodeAssists\0"  "Decode Assists",                              7, 1, 0),
+    DBGFREGSUBFIELD_RO("PauseFilter\0"    "Pause intercept filter",                     10, 1, 0),
+    DBGFREGSUBFIELD_RO("PauseFilterThreshold\0" "Pause filter threshold",               12, 1, 0),
+    DBGFREGSUBFIELD_RO("AVIC\0"           "Advanced Virtual Interrupt Controller",      13, 1, 0),
+    DBGFREGSUBFIELD_RO("VMSAVEVirt\0"     "VMSAVE and VMLOAD Virtualization",           15, 1, 0),
+    DBGFREGSUBFIELD_RO("VGIF\0"           "Virtual Global-Interrupt Flag",              16, 1, 0),
+    DBGFREGSUBFIELD_RO("GMET\0"           "Guest Mode Execute Trap Extension",          17, 1, 0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+
+/** CPUID(0x80000007,0).EDX field descriptions.   */
+static DBGFREGSUBFIELD const g_aExtLeaf7EdxSubFields[] =
+{
+    DBGFREGSUBFIELD_RO("TS\0"           "Temperature Sensor",                            0, 1, 0),
+    DBGFREGSUBFIELD_RO("FID\0"          "Frequency ID control",                          1, 1, 0),
+    DBGFREGSUBFIELD_RO("VID\0"          "Voltage ID control",                            2, 1, 0),
+    DBGFREGSUBFIELD_RO("VID\0"          "Voltage ID control",                            2, 1, 0),
+    DBGFREGSUBFIELD_RO("TTP\0"          "Thermal Trip",                                  3, 1, 0),
+    DBGFREGSUBFIELD_RO("TM\0"           "Hardware Thermal Control (HTC)",                4, 1, 0),
+    DBGFREGSUBFIELD_RO("100MHzSteps\0"  "100 MHz Multiplier control",                    6, 1, 0),
+    DBGFREGSUBFIELD_RO("HwPstate\0"     "Hardware P-state control",                      7, 1, 0),
+    DBGFREGSUBFIELD_RO("TscInvariant\0" "Invariant Time Stamp Counter",                  8, 1, 0),
+    DBGFREGSUBFIELD_RO("CBP\0"          "Core Performance Boost",                        9, 1, 0),
+    DBGFREGSUBFIELD_RO("EffFreqRO\0"    "Read-only Effective Frequency Interface",      10, 1, 0),
+    DBGFREGSUBFIELD_RO("ProcFdbkIf\0"   "Processor Feedback Interface",                 11, 1, 0),
+    DBGFREGSUBFIELD_RO("ProcPwrRep\0"   "Core power reporting interface support",       12, 1, 0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+/** CPUID(0x80000008,0).EBX field descriptions.   */
+static DBGFREGSUBFIELD const g_aExtLeaf8EbxSubFields[] =
+{
+    DBGFREGSUBFIELD_RO("CLZERO\0"       "Clear zero instruction (cacheline)",            0, 1, 0),
+    DBGFREGSUBFIELD_RO("IRPerf\0"       "Instructions retired count support",            1, 1, 0),
+    DBGFREGSUBFIELD_RO("XSaveErPtr\0"   "Save/restore error pointers (FXSAVE/RSTOR*)",   2, 1, 0),
+    DBGFREGSUBFIELD_RO("RDPRU\0"        "RDPRU instruction",                             4, 1, 0),
+    DBGFREGSUBFIELD_RO("MCOMMIT\0"      "MCOMMIT instruction",                           8, 1, 0),
+    DBGFREGSUBFIELD_RO("IBPB\0"         "Supports the IBPB command in IA32_PRED_CMD",   12, 1, 0),
+    DBGFREGSUBFIELD_TERMINATOR()
+};
+
+
+static void cpumR3CpuIdInfoMnemonicListU32(PCDBGFINFOHLP pHlp, uint32_t uVal, PCDBGFREGSUBFIELD pDesc,
+                                           const char *pszLeadIn, uint32_t cchWidth)
+{
+    if (pszLeadIn)
+        pHlp->pfnPrintf(pHlp, "%*s", cchWidth, pszLeadIn);
+
+    for (uint32_t iBit = 0; iBit < 32; iBit++)
+        if (RT_BIT_32(iBit) & uVal)
+        {
+            while (   pDesc->pszName != NULL
+                   && iBit >= (uint32_t)pDesc->iFirstBit + pDesc->cBits)
+                pDesc++;
+            if (   pDesc->pszName != NULL
+                && iBit - (uint32_t)pDesc->iFirstBit < (uint32_t)pDesc->cBits)
+            {
+                if (pDesc->cBits == 1)
+                    pHlp->pfnPrintf(pHlp, " %s", pDesc->pszName);
+                else
+                {
+                    uint32_t uFieldValue = uVal >> pDesc->iFirstBit;
+                    if (pDesc->cBits < 32)
+                        uFieldValue &= RT_BIT_32(pDesc->cBits) - UINT32_C(1);
+                    pHlp->pfnPrintf(pHlp, pDesc->cBits < 4 ? " %s=%u" : " %s=%#x", pDesc->pszName, uFieldValue);
+                    iBit = pDesc->iFirstBit + pDesc->cBits - 1;
+                }
+            }
+            else
+                pHlp->pfnPrintf(pHlp, " %u", iBit);
+        }
+    if (pszLeadIn)
+        pHlp->pfnPrintf(pHlp, "\n");
+}
+
+
+static void cpumR3CpuIdInfoMnemonicListU64(PCDBGFINFOHLP pHlp, uint64_t uVal, PCDBGFREGSUBFIELD pDesc,
+                                           const char *pszLeadIn, uint32_t cchWidth)
+{
+    if (pszLeadIn)
+        pHlp->pfnPrintf(pHlp, "%*s", cchWidth, pszLeadIn);
+
+    for (uint32_t iBit = 0; iBit < 64; iBit++)
+        if (RT_BIT_64(iBit) & uVal)
+        {
+            while (   pDesc->pszName != NULL
+                   && iBit >= (uint32_t)pDesc->iFirstBit + pDesc->cBits)
+                pDesc++;
+            if (   pDesc->pszName != NULL
+                && iBit - (uint32_t)pDesc->iFirstBit < (uint32_t)pDesc->cBits)
+            {
+                if (pDesc->cBits == 1)
+                    pHlp->pfnPrintf(pHlp, " %s", pDesc->pszName);
+                else
+                {
+                    uint64_t uFieldValue = uVal >> pDesc->iFirstBit;
+                    if (pDesc->cBits < 64)
+                        uFieldValue &= RT_BIT_64(pDesc->cBits) - UINT64_C(1);
+                    pHlp->pfnPrintf(pHlp, pDesc->cBits < 4 ? " %s=%llu" : " %s=%#llx", pDesc->pszName, uFieldValue);
+                    iBit = pDesc->iFirstBit + pDesc->cBits - 1;
+                }
+            }
+            else
+                pHlp->pfnPrintf(pHlp, " %u", iBit);
+        }
+    if (pszLeadIn)
+        pHlp->pfnPrintf(pHlp, "\n");
+}
+
+
+static void cpumR3CpuIdInfoValueWithMnemonicListU64(PCDBGFINFOHLP pHlp, uint64_t uVal, PCDBGFREGSUBFIELD pDesc,
+                                                    const char *pszLeadIn, uint32_t cchWidth)
+{
+    if (!uVal)
+        pHlp->pfnPrintf(pHlp, "%*s %#010x`%08x\n", cchWidth, pszLeadIn, RT_HI_U32(uVal), RT_LO_U32(uVal));
+    else
+    {
+        pHlp->pfnPrintf(pHlp, "%*s %#010x`%08x (", cchWidth, pszLeadIn, RT_HI_U32(uVal), RT_LO_U32(uVal));
+        cpumR3CpuIdInfoMnemonicListU64(pHlp, uVal, pDesc, NULL, 0);
+        pHlp->pfnPrintf(pHlp, " )\n");
+    }
+}
+
+
+static void cpumR3CpuIdInfoVerboseCompareListU32(PCDBGFINFOHLP pHlp, uint32_t uVal1, uint32_t uVal2, PCDBGFREGSUBFIELD pDesc,
+                                                 uint32_t cchWidth)
+{
+    uint32_t uCombined = uVal1 | uVal2;
+    for (uint32_t iBit = 0; iBit < 32; iBit++)
+        if (   (RT_BIT_32(iBit) & uCombined)
+            || (iBit == pDesc->iFirstBit && pDesc->pszName) )
+        {
+            while (   pDesc->pszName != NULL
+                   && iBit >= (uint32_t)pDesc->iFirstBit + pDesc->cBits)
+                pDesc++;
+
+            if (   pDesc->pszName != NULL
+                && iBit - (uint32_t)pDesc->iFirstBit < (uint32_t)pDesc->cBits)
+            {
+                size_t      cchMnemonic  = strlen(pDesc->pszName);
+                const char *pszDesc      = pDesc->pszName + cchMnemonic + 1;
+                size_t      cchDesc      = strlen(pszDesc);
+                uint32_t    uFieldValue1 = uVal1 >> pDesc->iFirstBit;
+                uint32_t    uFieldValue2 = uVal2 >> pDesc->iFirstBit;
+                if (pDesc->cBits < 32)
+                {
+                    uFieldValue1 &= RT_BIT_32(pDesc->cBits) - UINT32_C(1);
+                    uFieldValue2 &= RT_BIT_32(pDesc->cBits) - UINT32_C(1);
+                }
+
+                pHlp->pfnPrintf(pHlp,  pDesc->cBits < 4 ? "  %s - %s%*s= %u (%u)\n" : "  %s - %s%*s= %#x (%#x)\n",
+                                pDesc->pszName, pszDesc,
+                                cchMnemonic + 3 + cchDesc < cchWidth ? cchWidth - (cchMnemonic + 3 + cchDesc) : 1, "",
+                                uFieldValue1, uFieldValue2);
+
+                iBit = pDesc->iFirstBit + pDesc->cBits - 1U;
+                pDesc++;
+            }
+            else
+                pHlp->pfnPrintf(pHlp, "  %2u - Reserved%*s= %u (%u)\n", iBit, 13 < cchWidth ? cchWidth - 13 : 1, "",
+                                RT_BOOL(uVal1 & RT_BIT_32(iBit)), RT_BOOL(uVal2 & RT_BIT_32(iBit)));
+        }
+}
+
+
+/**
+ * Produces a detailed summary of standard leaf 0x00000001.
+ *
+ * @param   pHlp        The info helper functions.
+ * @param   pCurLeaf    The 0x00000001 leaf.
+ * @param   fVerbose    Whether to be very verbose or not.
+ * @param   fIntel      Set if intel CPU.
+ */
+static void cpumR3CpuIdInfoStdLeaf1Details(PCDBGFINFOHLP pHlp, PCCPUMCPUIDLEAF pCurLeaf, bool fVerbose, bool fIntel)
+{
+    Assert(pCurLeaf); Assert(pCurLeaf->uLeaf == 1);
+    static const char * const s_apszTypes[4] = { "primary", "overdrive", "MP", "reserved" };
+    uint32_t uEAX = pCurLeaf->uEax;
+    uint32_t uEBX = pCurLeaf->uEbx;
+
+    pHlp->pfnPrintf(pHlp,
+                    "%36s %2d \tExtended: %d \tEffective: %d\n"
+                    "%36s %2d \tExtended: %d \tEffective: %d\n"
+                    "%36s %d\n"
+                    "%36s %d (%s)\n"
+                    "%36s %#04x\n"
+                    "%36s %d\n"
+                    "%36s %d\n"
+                    "%36s %#04x\n"
+                    ,
+                    "Family:",      (uEAX >> 8) & 0xf, (uEAX >> 20) & 0x7f, ASMGetCpuFamily(uEAX),
+                    "Model:",       (uEAX >> 4) & 0xf, (uEAX >> 16) & 0x0f, ASMGetCpuModel(uEAX, fIntel),
+                    "Stepping:",    ASMGetCpuStepping(uEAX),
+                    "Type:",        (uEAX >> 12) & 3, s_apszTypes[(uEAX >> 12) & 3],
+                    "APIC ID:",     (uEBX >> 24) & 0xff,
+                    "Logical CPUs:",(uEBX >> 16) & 0xff,
+                    "CLFLUSH Size:",(uEBX >>  8) & 0xff,
+                    "Brand ID:",    (uEBX >>  0) & 0xff);
+    if (fVerbose)
+    {
+        CPUMCPUID Host;
+        ASMCpuIdExSlow(1, 0, 0, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+        pHlp->pfnPrintf(pHlp, "Features\n");
+        pHlp->pfnPrintf(pHlp, "  Mnemonic - Description                                  = guest (host)\n");
+        cpumR3CpuIdInfoVerboseCompareListU32(pHlp, pCurLeaf->uEdx, Host.uEdx, g_aLeaf1EdxSubFields, 56);
+        cpumR3CpuIdInfoVerboseCompareListU32(pHlp, pCurLeaf->uEcx, Host.uEcx, g_aLeaf1EcxSubFields, 56);
+    }
+    else
+    {
+        cpumR3CpuIdInfoMnemonicListU32(pHlp, pCurLeaf->uEdx, g_aLeaf1EdxSubFields, "Features EDX:", 36);
+        cpumR3CpuIdInfoMnemonicListU32(pHlp, pCurLeaf->uEcx, g_aLeaf1EcxSubFields, "Features ECX:", 36);
+    }
+}
+
+
+/**
+ * Produces a detailed summary of standard leaf 0x00000007.
+ *
+ * @param   pHlp        The info helper functions.
+ * @param   paLeaves    The CPUID leaves array.
+ * @param   cLeaves     The number of leaves in the array.
+ * @param   pCurLeaf    The first 0x00000007 leaf.
+ * @param   fVerbose    Whether to be very verbose or not.
+ */
+static void cpumR3CpuIdInfoStdLeaf7Details(PCDBGFINFOHLP pHlp, PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves,
+                                           PCCPUMCPUIDLEAF pCurLeaf, bool fVerbose)
+{
+    Assert(pCurLeaf); Assert(pCurLeaf->uLeaf == 7);
+    pHlp->pfnPrintf(pHlp, "Structured Extended Feature Flags Enumeration (leaf 7):\n");
+    for (;;)
+    {
+        CPUMCPUID Host;
+        ASMCpuIdExSlow(pCurLeaf->uLeaf, 0, pCurLeaf->uSubLeaf, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+
+        switch (pCurLeaf->uSubLeaf)
+        {
+            case 0:
+                if (fVerbose)
+                {
+                    pHlp->pfnPrintf(pHlp, "  Mnemonic - Description                                  = guest (host)\n");
+                    cpumR3CpuIdInfoVerboseCompareListU32(pHlp, pCurLeaf->uEbx, Host.uEbx, g_aLeaf7Sub0EbxSubFields, 56);
+                    cpumR3CpuIdInfoVerboseCompareListU32(pHlp, pCurLeaf->uEcx, Host.uEcx, g_aLeaf7Sub0EcxSubFields, 56);
+                    if (pCurLeaf->uEdx || Host.uEdx)
+                        cpumR3CpuIdInfoVerboseCompareListU32(pHlp, pCurLeaf->uEdx, Host.uEdx, g_aLeaf7Sub0EdxSubFields, 56);
+                }
+                else
+                {
+                    cpumR3CpuIdInfoMnemonicListU32(pHlp, pCurLeaf->uEbx, g_aLeaf7Sub0EbxSubFields, "Ext Features EBX:", 36);
+                    cpumR3CpuIdInfoMnemonicListU32(pHlp, pCurLeaf->uEcx, g_aLeaf7Sub0EcxSubFields, "Ext Features ECX:", 36);
+                    if (pCurLeaf->uEdx)
+                        cpumR3CpuIdInfoMnemonicListU32(pHlp, pCurLeaf->uEdx, g_aLeaf7Sub0EdxSubFields, "Ext Features EDX:", 36);
+                }
+                break;
+
+            default:
+                if (pCurLeaf->uEdx || pCurLeaf->uEcx || pCurLeaf->uEbx)
+                    pHlp->pfnPrintf(pHlp, "Unknown extended feature sub-leaf #%u: EAX=%#x EBX=%#x ECX=%#x EDX=%#x\n",
+                                    pCurLeaf->uSubLeaf, pCurLeaf->uEax, pCurLeaf->uEbx, pCurLeaf->uEcx, pCurLeaf->uEdx);
+                break;
+
+        }
+
+        /* advance. */
+        pCurLeaf++;
+        if (   (uintptr_t)(pCurLeaf - paLeaves) >= cLeaves
+            || pCurLeaf->uLeaf != 0x7)
+            break;
+    }
+}
+
+
+/**
+ * Produces a detailed summary of standard leaf 0x0000000d.
+ *
+ * @param   pHlp        The info helper functions.
+ * @param   paLeaves    The CPUID leaves array.
+ * @param   cLeaves     The number of leaves in the array.
+ * @param   pCurLeaf    The first 0x00000007 leaf.
+ * @param   fVerbose    Whether to be very verbose or not.
+ */
+static void cpumR3CpuIdInfoStdLeaf13Details(PCDBGFINFOHLP pHlp, PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves,
+                                            PCCPUMCPUIDLEAF pCurLeaf, bool fVerbose)
+{
+    RT_NOREF_PV(fVerbose);
+    Assert(pCurLeaf); Assert(pCurLeaf->uLeaf == 13);
+    pHlp->pfnPrintf(pHlp, "Processor Extended State Enumeration (leaf 0xd):\n");
+    for (uint32_t uSubLeaf = 0; uSubLeaf < 64; uSubLeaf++)
+    {
+        CPUMCPUID Host;
+        ASMCpuIdExSlow(UINT32_C(0x0000000d), 0, uSubLeaf, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+
+        switch (uSubLeaf)
+        {
+            case 0:
+                if (pCurLeaf && pCurLeaf->uSubLeaf == uSubLeaf)
+                    pHlp->pfnPrintf(pHlp, "%42s %#x/%#x\n", "XSAVE area cur/max size by XCR0, guest:",
+                                    pCurLeaf->uEbx, pCurLeaf->uEcx);
+                pHlp->pfnPrintf(pHlp, "%42s %#x/%#x\n", "XSAVE area cur/max size by XCR0, host:",  Host.uEbx, Host.uEcx);
+
+                if (pCurLeaf && pCurLeaf->uSubLeaf == uSubLeaf)
+                    cpumR3CpuIdInfoValueWithMnemonicListU64(pHlp, RT_MAKE_U64(pCurLeaf->uEax, pCurLeaf->uEdx), g_aXSaveStateBits,
+                                                            "Valid XCR0 bits, guest:", 42);
+                cpumR3CpuIdInfoValueWithMnemonicListU64(pHlp, RT_MAKE_U64(Host.uEax, Host.uEdx), g_aXSaveStateBits,
+                                                        "Valid XCR0 bits, host:", 42);
+                break;
+
+            case 1:
+                if (pCurLeaf && pCurLeaf->uSubLeaf == uSubLeaf)
+                    cpumR3CpuIdInfoMnemonicListU32(pHlp, pCurLeaf->uEax, g_aLeaf13Sub1EaxSubFields, "XSAVE features, guest:", 42);
+                cpumR3CpuIdInfoMnemonicListU32(pHlp, Host.uEax, g_aLeaf13Sub1EaxSubFields, "XSAVE features, host:", 42);
+
+                if (pCurLeaf && pCurLeaf->uSubLeaf == uSubLeaf)
+                    pHlp->pfnPrintf(pHlp, "%42s %#x\n", "XSAVE area cur size XCR0|XSS, guest:", pCurLeaf->uEbx);
+                pHlp->pfnPrintf(pHlp, "%42s %#x\n", "XSAVE area cur size XCR0|XSS, host:", Host.uEbx);
+
+                if (pCurLeaf && pCurLeaf->uSubLeaf == uSubLeaf)
+                    cpumR3CpuIdInfoValueWithMnemonicListU64(pHlp, RT_MAKE_U64(pCurLeaf->uEcx, pCurLeaf->uEdx), g_aXSaveStateBits,
+                                                            "  Valid IA32_XSS bits, guest:", 42);
+                cpumR3CpuIdInfoValueWithMnemonicListU64(pHlp, RT_MAKE_U64(Host.uEdx, Host.uEcx), g_aXSaveStateBits,
+                                                        "  Valid IA32_XSS bits, host:", 42);
+                break;
+
+            default:
+                if (   pCurLeaf
+                    && pCurLeaf->uSubLeaf == uSubLeaf
+                    && (pCurLeaf->uEax || pCurLeaf->uEbx || pCurLeaf->uEcx || pCurLeaf->uEdx) )
+                {
+                    pHlp->pfnPrintf(pHlp, "  State #%u, guest: off=%#06x, cb=%#06x %s", uSubLeaf, pCurLeaf->uEbx,
+                                    pCurLeaf->uEax, pCurLeaf->uEcx & RT_BIT_32(0) ? "XCR0-bit" : "IA32_XSS-bit");
+                    if (pCurLeaf->uEcx & ~RT_BIT_32(0))
+                        pHlp->pfnPrintf(pHlp, " ECX[reserved]=%#x\n", pCurLeaf->uEcx & ~RT_BIT_32(0));
+                    if (pCurLeaf->uEdx)
+                        pHlp->pfnPrintf(pHlp, " EDX[reserved]=%#x\n", pCurLeaf->uEdx);
+                    pHlp->pfnPrintf(pHlp, " --");
+                    cpumR3CpuIdInfoMnemonicListU64(pHlp, RT_BIT_64(uSubLeaf), g_aXSaveStateBits, NULL, 0);
+                    pHlp->pfnPrintf(pHlp, "\n");
+                }
+                if (Host.uEax || Host.uEbx || Host.uEcx || Host.uEdx)
+                {
+                    pHlp->pfnPrintf(pHlp, "  State #%u, host:  off=%#06x, cb=%#06x %s", uSubLeaf, Host.uEbx,
+                                    Host.uEax, Host.uEcx & RT_BIT_32(0) ? "XCR0-bit" : "IA32_XSS-bit");
+                    if (Host.uEcx & ~RT_BIT_32(0))
+                        pHlp->pfnPrintf(pHlp, " ECX[reserved]=%#x\n", Host.uEcx & ~RT_BIT_32(0));
+                    if (Host.uEdx)
+                        pHlp->pfnPrintf(pHlp, " EDX[reserved]=%#x\n", Host.uEdx);
+                    pHlp->pfnPrintf(pHlp, " --");
+                    cpumR3CpuIdInfoMnemonicListU64(pHlp, RT_BIT_64(uSubLeaf), g_aXSaveStateBits, NULL, 0);
+                    pHlp->pfnPrintf(pHlp, "\n");
+                }
+                break;
+
+        }
+
+        /* advance. */
+        if (pCurLeaf)
+        {
+            while (   (uintptr_t)(pCurLeaf - paLeaves) < cLeaves
+                   && pCurLeaf->uSubLeaf <= uSubLeaf
+                   && pCurLeaf->uLeaf == UINT32_C(0x0000000d))
+                pCurLeaf++;
+            if (   (uintptr_t)(pCurLeaf - paLeaves) >= cLeaves
+                || pCurLeaf->uLeaf != UINT32_C(0x0000000d))
+                pCurLeaf = NULL;
+        }
+    }
+}
+
+
+static PCCPUMCPUIDLEAF cpumR3CpuIdInfoRawRange(PCDBGFINFOHLP pHlp, PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves,
+                                               PCCPUMCPUIDLEAF pCurLeaf, uint32_t uUpToLeaf, const char *pszTitle)
+{
+    if (   (uintptr_t)(pCurLeaf - paLeaves) < cLeaves
+        && pCurLeaf->uLeaf <= uUpToLeaf)
+    {
+        pHlp->pfnPrintf(pHlp,
+                        "         %s\n"
+                        "     Leaf/sub-leaf  eax      ebx      ecx      edx\n", pszTitle);
+        while (   (uintptr_t)(pCurLeaf - paLeaves) < cLeaves
+               && pCurLeaf->uLeaf <= uUpToLeaf)
+        {
+            CPUMCPUID Host;
+            ASMCpuIdExSlow(pCurLeaf->uLeaf, 0, pCurLeaf->uSubLeaf, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+            pHlp->pfnPrintf(pHlp,
+                            "Gst: %08x/%04x  %08x %08x %08x %08x\n"
+                            "Hst:                %08x %08x %08x %08x\n",
+                            pCurLeaf->uLeaf, pCurLeaf->uSubLeaf, pCurLeaf->uEax, pCurLeaf->uEbx, pCurLeaf->uEcx, pCurLeaf->uEdx,
+                            Host.uEax, Host.uEbx, Host.uEcx, Host.uEdx);
+            pCurLeaf++;
+        }
+    }
+
+    return pCurLeaf;
+}
+
+
+/**
+ * Display the guest CpuId leaves.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     "terse", "default" or "verbose".
+ */
+DECLCALLBACK(void) cpumR3CpuIdInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /*
+     * Parse the argument.
+     */
+    unsigned iVerbosity = 1;
+    if (pszArgs)
+    {
+        pszArgs = RTStrStripL(pszArgs);
+        if (!strcmp(pszArgs, "terse"))
+            iVerbosity--;
+        else if (!strcmp(pszArgs, "verbose"))
+            iVerbosity++;
+    }
+
+    uint32_t        uLeaf;
+    CPUMCPUID       Host;
+    uint32_t        cLeaves  = pVM->cpum.s.GuestInfo.cCpuIdLeaves;
+    PCPUMCPUIDLEAF  paLeaves = pVM->cpum.s.GuestInfo.paCpuIdLeavesR3;
+    PCCPUMCPUIDLEAF pCurLeaf;
+    PCCPUMCPUIDLEAF pNextLeaf;
+    bool const      fIntel = ASMIsIntelCpuEx(pVM->cpum.s.aGuestCpuIdPatmStd[0].uEbx,
+                                             pVM->cpum.s.aGuestCpuIdPatmStd[0].uEcx,
+                                             pVM->cpum.s.aGuestCpuIdPatmStd[0].uEdx);
+
+    /*
+     * Standard leaves.  Custom raw dump here due to ECX sub-leaves host handling.
+     */
+    uint32_t        cHstMax = ASMCpuId_EAX(0);
+    uint32_t        cGstMax = paLeaves[0].uLeaf == 0 ? paLeaves[0].uEax : 0;
+    uint32_t        cMax    = RT_MAX(cGstMax, cHstMax);
+    pHlp->pfnPrintf(pHlp,
+                    "         Raw Standard CPUID Leaves\n"
+                    "     Leaf/sub-leaf  eax      ebx      ecx      edx\n");
+    for (uLeaf = 0, pCurLeaf = paLeaves; uLeaf <= cMax; uLeaf++)
+    {
+        uint32_t cMaxSubLeaves = 1;
+        if (uLeaf == 4 || uLeaf == 7 || uLeaf == 0xb)
+            cMaxSubLeaves = 16;
+        else if (uLeaf == 0xd)
+            cMaxSubLeaves = 128;
+
+        for (uint32_t uSubLeaf = 0; uSubLeaf < cMaxSubLeaves; uSubLeaf++)
+        {
+            ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+            if (   (uintptr_t)(pCurLeaf - paLeaves) < cLeaves
+                && pCurLeaf->uLeaf    == uLeaf
+                && pCurLeaf->uSubLeaf == uSubLeaf)
+            {
+                pHlp->pfnPrintf(pHlp,
+                                "Gst: %08x/%04x  %08x %08x %08x %08x\n"
+                                "Hst:                %08x %08x %08x %08x\n",
+                                uLeaf, uSubLeaf, pCurLeaf->uEax, pCurLeaf->uEbx, pCurLeaf->uEcx, pCurLeaf->uEdx,
+                                Host.uEax, Host.uEbx, Host.uEcx, Host.uEdx);
+                pCurLeaf++;
+            }
+            else if (   uLeaf != 0xd
+                     || uSubLeaf <= 1
+                     || Host.uEbx != 0 )
+                pHlp->pfnPrintf(pHlp,
+                                "Hst: %08x/%04x  %08x %08x %08x %08x\n",
+                                uLeaf, uSubLeaf, Host.uEax, Host.uEbx, Host.uEcx, Host.uEdx);
+
+            /* Done? */
+            if (   (   (uintptr_t)(pCurLeaf - paLeaves) >= cLeaves
+                    || pCurLeaf->uLeaf != uLeaf)
+                && (   (uLeaf == 0x4 && ((Host.uEax & 0x000f) == 0 || (Host.uEax & 0x000f) >= 8))
+                    || (uLeaf == 0x7 && Host.uEax == 0)
+                    || (uLeaf == 0xb && ((Host.uEcx & 0xff00) == 0 || (Host.uEcx & 0xff00) >= 8))
+                    || (uLeaf == 0xb && (Host.uEcx & 0xff) != uSubLeaf)
+                    || (uLeaf == 0xd && uSubLeaf >= 128)
+                   )
+               )
+                break;
+        }
+    }
+    pNextLeaf = pCurLeaf;
+
+    /*
+     * If verbose, decode it.
+     */
+    if (iVerbosity && paLeaves[0].uLeaf == 0)
+        pHlp->pfnPrintf(pHlp,
+                        "%36s %.04s%.04s%.04s\n"
+                        "%36s 0x00000000-%#010x\n"
+                        ,
+                        "Name:", &paLeaves[0].uEbx, &paLeaves[0].uEdx, &paLeaves[0].uEcx,
+                        "Supports:", paLeaves[0].uEax);
+
+    if (iVerbosity && (pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x00000001), 0)) != NULL)
+        cpumR3CpuIdInfoStdLeaf1Details(pHlp, pCurLeaf, iVerbosity > 1, fIntel);
+
+    if (iVerbosity && (pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x00000007), 0)) != NULL)
+        cpumR3CpuIdInfoStdLeaf7Details(pHlp, paLeaves, cLeaves, pCurLeaf, iVerbosity > 1);
+
+    if (iVerbosity && (pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x0000000d), 0)) != NULL)
+        cpumR3CpuIdInfoStdLeaf13Details(pHlp, paLeaves, cLeaves, pCurLeaf, iVerbosity > 1);
+
+    pCurLeaf = pNextLeaf;
+
+    /*
+     * Hypervisor leaves.
+     *
+     * Unlike most of the other leaves reported, the guest hypervisor leaves
+     * aren't a subset of the host CPUID bits.
+     */
+    pCurLeaf = cpumR3CpuIdInfoRawRange(pHlp, paLeaves, cLeaves, pCurLeaf, UINT32_C(0x3fffffff), "Unknown CPUID Leaves");
+
+    ASMCpuIdExSlow(UINT32_C(0x40000000), 0, 0, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+    cHstMax  = Host.uEax >= UINT32_C(0x40000001) && Host.uEax <= UINT32_C(0x40000fff) ? Host.uEax : 0;
+    cGstMax  = (uintptr_t)(pCurLeaf - paLeaves) < cLeaves && pCurLeaf->uLeaf == UINT32_C(0x40000000)
+             ? RT_MIN(pCurLeaf->uEax, UINT32_C(0x40000fff)) : 0;
+    cMax     = RT_MAX(cHstMax, cGstMax);
+    if (cMax >= UINT32_C(0x40000000))
+    {
+        pNextLeaf = cpumR3CpuIdInfoRawRange(pHlp, paLeaves, cLeaves, pCurLeaf, cMax, "Raw Hypervisor CPUID Leaves");
+
+        /** @todo dump these in more detail. */
+
+        pCurLeaf = pNextLeaf;
+    }
+
+
+    /*
+     * Extended.  Custom raw dump here due to ECX sub-leaves host handling.
+     * Implemented after AMD specs.
+     */
+    pCurLeaf = cpumR3CpuIdInfoRawRange(pHlp, paLeaves, cLeaves, pCurLeaf, UINT32_C(0x7fffffff), "Unknown CPUID Leaves");
+
+    ASMCpuIdExSlow(UINT32_C(0x80000000), 0, 0, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+    cHstMax  = ASMIsValidExtRange(Host.uEax) ? RT_MIN(Host.uEax, UINT32_C(0x80000fff)) : 0;
+    cGstMax  = (uintptr_t)(pCurLeaf - paLeaves) < cLeaves && pCurLeaf->uLeaf == UINT32_C(0x80000000)
+             ? RT_MIN(pCurLeaf->uEax, UINT32_C(0x80000fff)) : 0;
+    cMax     = RT_MAX(cHstMax, cGstMax);
+    if (cMax >= UINT32_C(0x80000000))
+    {
+
+        pHlp->pfnPrintf(pHlp,
+                        "         Raw Extended CPUID Leaves\n"
+                        "     Leaf/sub-leaf  eax      ebx      ecx      edx\n");
+        PCCPUMCPUIDLEAF pExtLeaf = pCurLeaf;
+        for (uLeaf = UINT32_C(0x80000000); uLeaf <= cMax; uLeaf++)
+        {
+            uint32_t cMaxSubLeaves = 1;
+            if (uLeaf == UINT32_C(0x8000001d))
+                cMaxSubLeaves = 16;
+
+            for (uint32_t uSubLeaf = 0; uSubLeaf < cMaxSubLeaves; uSubLeaf++)
+            {
+                ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+                if (   (uintptr_t)(pCurLeaf - paLeaves) < cLeaves
+                    && pCurLeaf->uLeaf    == uLeaf
+                    && pCurLeaf->uSubLeaf == uSubLeaf)
+                {
+                    pHlp->pfnPrintf(pHlp,
+                                    "Gst: %08x/%04x  %08x %08x %08x %08x\n"
+                                    "Hst:                %08x %08x %08x %08x\n",
+                                    uLeaf, uSubLeaf, pCurLeaf->uEax, pCurLeaf->uEbx, pCurLeaf->uEcx, pCurLeaf->uEdx,
+                                    Host.uEax, Host.uEbx, Host.uEcx, Host.uEdx);
+                    pCurLeaf++;
+                }
+                else if (   uLeaf != 0xd
+                         || uSubLeaf <= 1
+                         || Host.uEbx != 0 )
+                    pHlp->pfnPrintf(pHlp,
+                                    "Hst: %08x/%04x  %08x %08x %08x %08x\n",
+                                    uLeaf, uSubLeaf, Host.uEax, Host.uEbx, Host.uEcx, Host.uEdx);
+
+                /* Done? */
+                if (   (   (uintptr_t)(pCurLeaf - paLeaves) >= cLeaves
+                        || pCurLeaf->uLeaf != uLeaf)
+                    && (uLeaf == UINT32_C(0x8000001d) && ((Host.uEax & 0x000f) == 0 || (Host.uEax & 0x000f) >= 8)) )
+                    break;
+            }
+        }
+        pNextLeaf = pCurLeaf;
+
+        /*
+         * Understandable output
+         */
+        if (iVerbosity)
+            pHlp->pfnPrintf(pHlp,
+                            "Ext Name:                        %.4s%.4s%.4s\n"
+                            "Ext Supports:                    0x80000000-%#010x\n",
+                            &pExtLeaf->uEbx, &pExtLeaf->uEdx, &pExtLeaf->uEcx, pExtLeaf->uEax);
+
+        pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x80000001), 0);
+        if (iVerbosity && pCurLeaf)
+        {
+            uint32_t uEAX = pCurLeaf->uEax;
+            pHlp->pfnPrintf(pHlp,
+                            "Family:                          %d  \tExtended: %d \tEffective: %d\n"
+                            "Model:                           %d  \tExtended: %d \tEffective: %d\n"
+                            "Stepping:                        %d\n"
+                            "Brand ID:                        %#05x\n",
+                            (uEAX >> 8) & 0xf, (uEAX >> 20) & 0x7f, ASMGetCpuFamily(uEAX),
+                            (uEAX >> 4) & 0xf, (uEAX >> 16) & 0x0f, ASMGetCpuModel(uEAX, fIntel),
+                            ASMGetCpuStepping(uEAX),
+                            pCurLeaf->uEbx & 0xfff);
+
+            if (iVerbosity == 1)
+            {
+                cpumR3CpuIdInfoMnemonicListU32(pHlp, pCurLeaf->uEdx, g_aExtLeaf1EdxSubFields, "Ext Features EDX:", 34);
+                cpumR3CpuIdInfoMnemonicListU32(pHlp, pCurLeaf->uEcx, g_aExtLeaf1EdxSubFields, "Ext Features ECX:", 34);
+            }
+            else
+            {
+                ASMCpuIdExSlow(0x80000001, 0, 0, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+                pHlp->pfnPrintf(pHlp, "Ext Features\n");
+                pHlp->pfnPrintf(pHlp, "  Mnemonic - Description                                  = guest (host)\n");
+                cpumR3CpuIdInfoVerboseCompareListU32(pHlp, pCurLeaf->uEdx, Host.uEdx, g_aExtLeaf1EdxSubFields, 56);
+                cpumR3CpuIdInfoVerboseCompareListU32(pHlp, pCurLeaf->uEcx, Host.uEcx, g_aExtLeaf1EcxSubFields, 56);
+                if (Host.uEcx & X86_CPUID_AMD_FEATURE_ECX_SVM)
+                {
+                    pHlp->pfnPrintf(pHlp, "SVM Feature Identification (leaf A):\n");
+                    ASMCpuIdExSlow(0x8000000a, 0, 0, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+                    pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x8000000a), 0);
+                    uint32_t const uGstEdx = pCurLeaf ? pCurLeaf->uEdx : 0;
+                    cpumR3CpuIdInfoVerboseCompareListU32(pHlp, uGstEdx, Host.uEdx, g_aExtLeafAEdxSubFields, 56);
+                }
+            }
+        }
+
+        if (iVerbosity && (pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x80000002), 0)) != NULL)
+        {
+            char szString[4*4*3+1] = {0};
+            uint32_t *pu32 = (uint32_t *)szString;
+            *pu32++ = pCurLeaf->uEax;
+            *pu32++ = pCurLeaf->uEbx;
+            *pu32++ = pCurLeaf->uEcx;
+            *pu32++ = pCurLeaf->uEdx;
+            pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x80000003), 0);
+            if (pCurLeaf)
+            {
+                *pu32++ = pCurLeaf->uEax;
+                *pu32++ = pCurLeaf->uEbx;
+                *pu32++ = pCurLeaf->uEcx;
+                *pu32++ = pCurLeaf->uEdx;
+            }
+            pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x80000004), 0);
+            if (pCurLeaf)
+            {
+                *pu32++ = pCurLeaf->uEax;
+                *pu32++ = pCurLeaf->uEbx;
+                *pu32++ = pCurLeaf->uEcx;
+                *pu32++ = pCurLeaf->uEdx;
+            }
+            pHlp->pfnPrintf(pHlp, "Full Name:                       \"%s\"\n", szString);
+        }
+
+        if (iVerbosity && (pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x80000005), 0)) != NULL)
+        {
+            uint32_t uEAX = pCurLeaf->uEax;
+            uint32_t uEBX = pCurLeaf->uEbx;
+            uint32_t uECX = pCurLeaf->uEcx;
+            uint32_t uEDX = pCurLeaf->uEdx;
+            char sz1[32];
+            char sz2[32];
+
+            pHlp->pfnPrintf(pHlp,
+                            "TLB 2/4M Instr/Uni:              %s %3d entries\n"
+                            "TLB 2/4M Data:                   %s %3d entries\n",
+                            getCacheAss((uEAX >>  8) & 0xff, sz1), (uEAX >>  0) & 0xff,
+                            getCacheAss((uEAX >> 24) & 0xff, sz2), (uEAX >> 16) & 0xff);
+            pHlp->pfnPrintf(pHlp,
+                            "TLB 4K Instr/Uni:                %s %3d entries\n"
+                            "TLB 4K Data:                     %s %3d entries\n",
+                            getCacheAss((uEBX >>  8) & 0xff, sz1), (uEBX >>  0) & 0xff,
+                            getCacheAss((uEBX >> 24) & 0xff, sz2), (uEBX >> 16) & 0xff);
+            pHlp->pfnPrintf(pHlp, "L1 Instr Cache Line Size:        %d bytes\n"
+                            "L1 Instr Cache Lines Per Tag:    %d\n"
+                            "L1 Instr Cache Associativity:    %s\n"
+                            "L1 Instr Cache Size:             %d KB\n",
+                            (uEDX >> 0) & 0xff,
+                            (uEDX >> 8) & 0xff,
+                            getCacheAss((uEDX >> 16) & 0xff, sz1),
+                            (uEDX >> 24) & 0xff);
+            pHlp->pfnPrintf(pHlp,
+                            "L1 Data Cache Line Size:         %d bytes\n"
+                            "L1 Data Cache Lines Per Tag:     %d\n"
+                            "L1 Data Cache Associativity:     %s\n"
+                            "L1 Data Cache Size:              %d KB\n",
+                            (uECX >> 0) & 0xff,
+                            (uECX >> 8) & 0xff,
+                            getCacheAss((uECX >> 16) & 0xff, sz1),
+                            (uECX >> 24) & 0xff);
+        }
+
+        if (iVerbosity && (pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x80000006), 0)) != NULL)
+        {
+            uint32_t uEAX = pCurLeaf->uEax;
+            uint32_t uEBX = pCurLeaf->uEbx;
+            uint32_t uEDX = pCurLeaf->uEdx;
+
+            pHlp->pfnPrintf(pHlp,
+                            "L2 TLB 2/4M Instr/Uni:           %s %4d entries\n"
+                            "L2 TLB 2/4M Data:                %s %4d entries\n",
+                            getL2CacheAss((uEAX >> 12) & 0xf),  (uEAX >>  0) & 0xfff,
+                            getL2CacheAss((uEAX >> 28) & 0xf),  (uEAX >> 16) & 0xfff);
+            pHlp->pfnPrintf(pHlp,
+                            "L2 TLB 4K Instr/Uni:             %s %4d entries\n"
+                            "L2 TLB 4K Data:                  %s %4d entries\n",
+                            getL2CacheAss((uEBX >> 12) & 0xf),  (uEBX >>  0) & 0xfff,
+                            getL2CacheAss((uEBX >> 28) & 0xf),  (uEBX >> 16) & 0xfff);
+            pHlp->pfnPrintf(pHlp,
+                            "L2 Cache Line Size:              %d bytes\n"
+                            "L2 Cache Lines Per Tag:          %d\n"
+                            "L2 Cache Associativity:          %s\n"
+                            "L2 Cache Size:                   %d KB\n",
+                            (uEDX >> 0) & 0xff,
+                            (uEDX >> 8) & 0xf,
+                            getL2CacheAss((uEDX >> 12) & 0xf),
+                            (uEDX >> 16) & 0xffff);
+        }
+
+        if (iVerbosity && (pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x80000007), 0)) != NULL)
+        {
+            ASMCpuIdExSlow(UINT32_C(0x80000007), 0, 0, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+            if (pCurLeaf->uEdx || (Host.uEdx && iVerbosity))
+            {
+                if (iVerbosity < 1)
+                    cpumR3CpuIdInfoMnemonicListU32(pHlp, pCurLeaf->uEdx, g_aExtLeaf7EdxSubFields, "APM Features EDX:", 34);
+                else
+                    cpumR3CpuIdInfoVerboseCompareListU32(pHlp, pCurLeaf->uEdx, Host.uEdx, g_aExtLeaf7EdxSubFields, 56);
+            }
+        }
+
+        pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0x80000008), 0);
+        if (pCurLeaf != NULL)
+        {
+            ASMCpuIdExSlow(UINT32_C(0x80000008), 0, 0, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+            if (pCurLeaf->uEbx || (Host.uEbx && iVerbosity))
+            {
+                if (iVerbosity < 1)
+                    cpumR3CpuIdInfoMnemonicListU32(pHlp, pCurLeaf->uEbx, g_aExtLeaf8EbxSubFields, "Ext Features ext IDs EBX:", 34);
+                else
+                    cpumR3CpuIdInfoVerboseCompareListU32(pHlp, pCurLeaf->uEbx, Host.uEbx, g_aExtLeaf8EbxSubFields, 56);
+            }
+
+            if (iVerbosity)
+            {
+                uint32_t uEAX = pCurLeaf->uEax;
+                uint32_t uECX = pCurLeaf->uEcx;
+
+                pHlp->pfnPrintf(pHlp,
+                                "Physical Address Width:          %d bits\n"
+                                "Virtual Address Width:           %d bits\n"
+                                "Guest Physical Address Width:    %d bits\n",
+                                (uEAX >> 0) & 0xff,
+                                (uEAX >> 8) & 0xff,
+                                (uEAX >> 16) & 0xff);
+                pHlp->pfnPrintf(pHlp,
+                                "Physical Core Count:             %d\n",
+                                ((uECX >> 0) & 0xff) + 1);
+            }
+        }
+
+        pCurLeaf = pNextLeaf;
+    }
+
+
+
+    /*
+     * Centaur.
+     */
+    pCurLeaf = cpumR3CpuIdInfoRawRange(pHlp, paLeaves, cLeaves, pCurLeaf, UINT32_C(0xbfffffff), "Unknown CPUID Leaves");
+
+    ASMCpuIdExSlow(UINT32_C(0xc0000000), 0, 0, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+    cHstMax  = Host.uEax >= UINT32_C(0xc0000001) && Host.uEax <= UINT32_C(0xc0000fff)
+             ? RT_MIN(Host.uEax,      UINT32_C(0xc0000fff)) : 0;
+    cGstMax  = (uintptr_t)(pCurLeaf - paLeaves) < cLeaves && pCurLeaf->uLeaf == UINT32_C(0xc0000000)
+             ? RT_MIN(pCurLeaf->uEax, UINT32_C(0xc0000fff)) : 0;
+    cMax     = RT_MAX(cHstMax, cGstMax);
+    if (cMax >= UINT32_C(0xc0000000))
+    {
+        pNextLeaf = cpumR3CpuIdInfoRawRange(pHlp, paLeaves, cLeaves, pCurLeaf, cMax, "Raw Centaur CPUID Leaves");
+
+        /*
+         * Understandable output
+         */
+        if (iVerbosity && (pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0xc0000000), 0)) != NULL)
+            pHlp->pfnPrintf(pHlp,
+                            "Centaur Supports:                0xc0000000-%#010x\n",
+                            pCurLeaf->uEax);
+
+        if (iVerbosity && (pCurLeaf = cpumR3CpuIdGetLeaf(paLeaves, cLeaves, UINT32_C(0xc0000001), 0)) != NULL)
+        {
+            ASMCpuIdExSlow(0xc0000001, 0, 0, 0, &Host.uEax, &Host.uEbx, &Host.uEcx, &Host.uEdx);
+            uint32_t uEdxGst = pCurLeaf->uEdx;
+            uint32_t uEdxHst = Host.uEdx;
+
+            if (iVerbosity == 1)
+            {
+                pHlp->pfnPrintf(pHlp, "Centaur Features EDX:           ");
+                if (uEdxGst & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " AIS");
+                if (uEdxGst & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " AIS-E");
+                if (uEdxGst & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " RNG");
+                if (uEdxGst & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " RNG-E");
+                if (uEdxGst & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " LH");
+                if (uEdxGst & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " FEMMS");
+                if (uEdxGst & RT_BIT(6))   pHlp->pfnPrintf(pHlp, " ACE");
+                if (uEdxGst & RT_BIT(7))   pHlp->pfnPrintf(pHlp, " ACE-E");
+                /* possibly indicating MM/HE and MM/HE-E on older chips... */
+                if (uEdxGst & RT_BIT(8))   pHlp->pfnPrintf(pHlp, " ACE2");
+                if (uEdxGst & RT_BIT(9))   pHlp->pfnPrintf(pHlp, " ACE2-E");
+                if (uEdxGst & RT_BIT(10))  pHlp->pfnPrintf(pHlp, " PHE");
+                if (uEdxGst & RT_BIT(11))  pHlp->pfnPrintf(pHlp, " PHE-E");
+                if (uEdxGst & RT_BIT(12))  pHlp->pfnPrintf(pHlp, " PMM");
+                if (uEdxGst & RT_BIT(13))  pHlp->pfnPrintf(pHlp, " PMM-E");
+                for (unsigned iBit = 14; iBit < 32; iBit++)
+                    if (uEdxGst & RT_BIT(iBit))
+                        pHlp->pfnPrintf(pHlp, " %d", iBit);
+                pHlp->pfnPrintf(pHlp, "\n");
+            }
+            else
+            {
+                pHlp->pfnPrintf(pHlp, "Mnemonic - Description                 = guest (host)\n");
+                pHlp->pfnPrintf(pHlp, "AIS - Alternate Instruction Set        = %d (%d)\n",  !!(uEdxGst & RT_BIT( 0)),  !!(uEdxHst & RT_BIT( 0)));
+                pHlp->pfnPrintf(pHlp, "AIS-E - AIS enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT( 1)),  !!(uEdxHst & RT_BIT( 1)));
+                pHlp->pfnPrintf(pHlp, "RNG - Random Number Generator          = %d (%d)\n",  !!(uEdxGst & RT_BIT( 2)),  !!(uEdxHst & RT_BIT( 2)));
+                pHlp->pfnPrintf(pHlp, "RNG-E - RNG enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT( 3)),  !!(uEdxHst & RT_BIT( 3)));
+                pHlp->pfnPrintf(pHlp, "LH - LongHaul MSR 0000_110Ah           = %d (%d)\n",  !!(uEdxGst & RT_BIT( 4)),  !!(uEdxHst & RT_BIT( 4)));
+                pHlp->pfnPrintf(pHlp, "FEMMS - FEMMS                          = %d (%d)\n",  !!(uEdxGst & RT_BIT( 5)),  !!(uEdxHst & RT_BIT( 5)));
+                pHlp->pfnPrintf(pHlp, "ACE - Advanced Cryptography Engine     = %d (%d)\n",  !!(uEdxGst & RT_BIT( 6)),  !!(uEdxHst & RT_BIT( 6)));
+                pHlp->pfnPrintf(pHlp, "ACE-E - ACE enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT( 7)),  !!(uEdxHst & RT_BIT( 7)));
+                /* possibly indicating MM/HE and MM/HE-E on older chips... */
+                pHlp->pfnPrintf(pHlp, "ACE2 - Advanced Cryptography Engine 2  = %d (%d)\n",  !!(uEdxGst & RT_BIT( 8)),  !!(uEdxHst & RT_BIT( 8)));
+                pHlp->pfnPrintf(pHlp, "ACE2-E - ACE enabled                   = %d (%d)\n",  !!(uEdxGst & RT_BIT( 9)),  !!(uEdxHst & RT_BIT( 9)));
+                pHlp->pfnPrintf(pHlp, "PHE - Padlock Hash Engine              = %d (%d)\n",  !!(uEdxGst & RT_BIT(10)),  !!(uEdxHst & RT_BIT(10)));
+                pHlp->pfnPrintf(pHlp, "PHE-E - PHE enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT(11)),  !!(uEdxHst & RT_BIT(11)));
+                pHlp->pfnPrintf(pHlp, "PMM - Montgomery Multiplier            = %d (%d)\n",  !!(uEdxGst & RT_BIT(12)),  !!(uEdxHst & RT_BIT(12)));
+                pHlp->pfnPrintf(pHlp, "PMM-E - PMM enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT(13)),  !!(uEdxHst & RT_BIT(13)));
+                pHlp->pfnPrintf(pHlp, "14 - Reserved                          = %d (%d)\n",  !!(uEdxGst & RT_BIT(14)),  !!(uEdxHst & RT_BIT(14)));
+                pHlp->pfnPrintf(pHlp, "15 - Reserved                          = %d (%d)\n",  !!(uEdxGst & RT_BIT(15)),  !!(uEdxHst & RT_BIT(15)));
+                pHlp->pfnPrintf(pHlp, "Parallax                               = %d (%d)\n",  !!(uEdxGst & RT_BIT(16)),  !!(uEdxHst & RT_BIT(16)));
+                pHlp->pfnPrintf(pHlp, "Parallax enabled                       = %d (%d)\n",  !!(uEdxGst & RT_BIT(17)),  !!(uEdxHst & RT_BIT(17)));
+                pHlp->pfnPrintf(pHlp, "Overstress                             = %d (%d)\n",  !!(uEdxGst & RT_BIT(18)),  !!(uEdxHst & RT_BIT(18)));
+                pHlp->pfnPrintf(pHlp, "Overstress enabled                     = %d (%d)\n",  !!(uEdxGst & RT_BIT(19)),  !!(uEdxHst & RT_BIT(19)));
+                pHlp->pfnPrintf(pHlp, "TM3 - Temperature Monitoring 3         = %d (%d)\n",  !!(uEdxGst & RT_BIT(20)),  !!(uEdxHst & RT_BIT(20)));
+                pHlp->pfnPrintf(pHlp, "TM3-E - TM3 enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT(21)),  !!(uEdxHst & RT_BIT(21)));
+                pHlp->pfnPrintf(pHlp, "RNG2 - Random Number Generator 2       = %d (%d)\n",  !!(uEdxGst & RT_BIT(22)),  !!(uEdxHst & RT_BIT(22)));
+                pHlp->pfnPrintf(pHlp, "RNG2-E - RNG2 enabled                  = %d (%d)\n",  !!(uEdxGst & RT_BIT(23)),  !!(uEdxHst & RT_BIT(23)));
+                pHlp->pfnPrintf(pHlp, "24 - Reserved                          = %d (%d)\n",  !!(uEdxGst & RT_BIT(24)),  !!(uEdxHst & RT_BIT(24)));
+                pHlp->pfnPrintf(pHlp, "PHE2 - Padlock Hash Engine 2           = %d (%d)\n",  !!(uEdxGst & RT_BIT(25)),  !!(uEdxHst & RT_BIT(25)));
+                pHlp->pfnPrintf(pHlp, "PHE2-E - PHE2 enabled                  = %d (%d)\n",  !!(uEdxGst & RT_BIT(26)),  !!(uEdxHst & RT_BIT(26)));
+                for (unsigned iBit = 27; iBit < 32; iBit++)
+                    if ((uEdxGst | uEdxHst) & RT_BIT(iBit))
+                        pHlp->pfnPrintf(pHlp, "Bit %d                                 = %d (%d)\n", iBit, !!(uEdxGst & RT_BIT(iBit)), !!(uEdxHst & RT_BIT(iBit)));
+                pHlp->pfnPrintf(pHlp, "\n");
+            }
+        }
+
+        pCurLeaf = pNextLeaf;
+    }
+
+    /*
+     * The remainder.
+     */
+    pCurLeaf = cpumR3CpuIdInfoRawRange(pHlp, paLeaves, cLeaves, pCurLeaf, UINT32_C(0xffffffff), "Unknown CPUID Leaves");
+}
+
+#endif /* !IN_VBOX_CPU_REPORT */
+
diff --git a/src/VBox/VMM/VMMR3/CPUMR3Db.cpp b/src/VBox/VMM/VMMR3/CPUMR3Db.cpp
new file mode 100644
index 00000000..f09ed843
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/CPUMR3Db.cpp
@@ -0,0 +1,1162 @@
+/* $Id: CPUMR3Db.cpp $ */
+/** @file
+ * CPUM - CPU database part.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_CPUM
+#include <VBox/vmm/cpum.h>
+#include "CPUMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/mm.h>
+
+#include <VBox/err.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/mem.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+typedef struct CPUMDBENTRY
+{
+    /** The CPU name. */
+    const char     *pszName;
+    /** The full CPU name. */
+    const char     *pszFullName;
+    /** The CPU vendor (CPUMCPUVENDOR). */
+    uint8_t         enmVendor;
+    /** The CPU family. */
+    uint8_t         uFamily;
+    /** The CPU model. */
+    uint8_t         uModel;
+    /** The CPU stepping. */
+    uint8_t         uStepping;
+    /** The microarchitecture. */
+    CPUMMICROARCH   enmMicroarch;
+    /** Scalable bus frequency used for reporting other frequencies. */
+    uint64_t        uScalableBusFreq;
+    /** Flags - CPUDB_F_XXX. */
+    uint32_t        fFlags;
+    /** The maximum physical address with of the CPU.  This should correspond to
+     * the value in CPUID leaf 0x80000008 when present. */
+    uint8_t         cMaxPhysAddrWidth;
+    /** The MXCSR mask. */
+    uint32_t        fMxCsrMask;
+    /** Pointer to an array of CPUID leaves.  */
+    PCCPUMCPUIDLEAF paCpuIdLeaves;
+    /** The number of CPUID leaves in the array paCpuIdLeaves points to. */
+    uint32_t        cCpuIdLeaves;
+    /** The method used to deal with unknown CPUID leaves. */
+    CPUMUNKNOWNCPUID enmUnknownCpuId;
+    /** The default unknown CPUID value. */
+    CPUMCPUID       DefUnknownCpuId;
+
+    /** MSR mask.  Several microarchitectures ignore the higher bits of ECX in
+     *  the RDMSR and WRMSR instructions. */
+    uint32_t        fMsrMask;
+
+    /** The number of ranges in the table pointed to b paMsrRanges. */
+    uint32_t        cMsrRanges;
+    /** MSR ranges for this CPU. */
+    PCCPUMMSRRANGE  paMsrRanges;
+} CPUMDBENTRY;
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** @name CPUDB_F_XXX - CPUDBENTRY::fFlags
+ * @{ */
+/** Should execute all in IEM.
+ * @todo Implement this - currently done in Main...  */
+#define CPUDB_F_EXECUTE_ALL_IN_IEM          RT_BIT_32(0)
+/** @} */
+
+
+/** @def NULL_ALONE
+ * For eliminating an unnecessary data dependency in standalone builds (for
+ * VBoxSVC). */
+/** @def ZERO_ALONE
+ * For eliminating an unnecessary data size dependency in standalone builds (for
+ * VBoxSVC). */
+#ifndef CPUM_DB_STANDALONE
+# define NULL_ALONE(a_aTable)    a_aTable
+# define ZERO_ALONE(a_cTable)    a_cTable
+#else
+# define NULL_ALONE(a_aTable)    NULL
+# define ZERO_ALONE(a_cTable)    0
+#endif
+
+
+/** @name Short macros for the MSR range entries.
+ *
+ * These are rather cryptic, but this is to reduce the attack on the right
+ * margin.
+ *
+ * @{ */
+/** Alias one MSR onto another (a_uTarget). */
+#define MAL(a_uMsr, a_szName, a_uTarget) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_MsrAlias, kCpumMsrWrFn_MsrAlias, 0, a_uTarget, 0, 0, a_szName)
+/** Functions handles everything. */
+#define MFN(a_uMsr, a_szName, a_enmRdFnSuff, a_enmWrFnSuff) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_##a_enmRdFnSuff, kCpumMsrWrFn_##a_enmWrFnSuff, 0, 0, 0, 0, a_szName)
+/** Functions handles everything, with GP mask. */
+#define MFG(a_uMsr, a_szName, a_enmRdFnSuff, a_enmWrFnSuff, a_fWrGpMask) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_##a_enmRdFnSuff, kCpumMsrWrFn_##a_enmWrFnSuff, 0, 0, 0, a_fWrGpMask, a_szName)
+/** Function handlers, read-only. */
+#define MFO(a_uMsr, a_szName, a_enmRdFnSuff) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_##a_enmRdFnSuff, kCpumMsrWrFn_ReadOnly, 0, 0, 0, UINT64_MAX, a_szName)
+/** Function handlers, ignore all writes. */
+#define MFI(a_uMsr, a_szName, a_enmRdFnSuff) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_##a_enmRdFnSuff, kCpumMsrWrFn_IgnoreWrite, 0, 0, UINT64_MAX, 0, a_szName)
+/** Function handlers, with value. */
+#define MFV(a_uMsr, a_szName, a_enmRdFnSuff, a_enmWrFnSuff, a_uValue) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_##a_enmRdFnSuff, kCpumMsrWrFn_##a_enmWrFnSuff, 0, a_uValue, 0, 0, a_szName)
+/** Function handlers, with write ignore mask. */
+#define MFW(a_uMsr, a_szName, a_enmRdFnSuff, a_enmWrFnSuff, a_fWrIgnMask) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_##a_enmRdFnSuff, kCpumMsrWrFn_##a_enmWrFnSuff, 0, 0, a_fWrIgnMask, 0, a_szName)
+/** Function handlers, extended version. */
+#define MFX(a_uMsr, a_szName, a_enmRdFnSuff, a_enmWrFnSuff, a_uValue, a_fWrIgnMask, a_fWrGpMask) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_##a_enmRdFnSuff, kCpumMsrWrFn_##a_enmWrFnSuff, 0, a_uValue, a_fWrIgnMask, a_fWrGpMask, a_szName)
+/** Function handlers, with CPUMCPU storage variable. */
+#define MFS(a_uMsr, a_szName, a_enmRdFnSuff, a_enmWrFnSuff, a_CpumCpuMember) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_##a_enmRdFnSuff, kCpumMsrWrFn_##a_enmWrFnSuff, \
+         RT_OFFSETOF(CPUMCPU, a_CpumCpuMember), 0, 0, 0, a_szName)
+/** Function handlers, with CPUMCPU storage variable, ignore mask and GP mask. */
+#define MFZ(a_uMsr, a_szName, a_enmRdFnSuff, a_enmWrFnSuff, a_CpumCpuMember, a_fWrIgnMask, a_fWrGpMask) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_##a_enmRdFnSuff, kCpumMsrWrFn_##a_enmWrFnSuff, \
+         RT_OFFSETOF(CPUMCPU, a_CpumCpuMember), 0, a_fWrIgnMask, a_fWrGpMask, a_szName)
+/** Read-only fixed value. */
+#define MVO(a_uMsr, a_szName, a_uValue) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_FixedValue, kCpumMsrWrFn_ReadOnly, 0, a_uValue, 0, UINT64_MAX, a_szName)
+/** Read-only fixed value, ignores all writes. */
+#define MVI(a_uMsr, a_szName, a_uValue) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_FixedValue, kCpumMsrWrFn_IgnoreWrite, 0, a_uValue, UINT64_MAX, 0, a_szName)
+/** Read fixed value, ignore writes outside GP mask. */
+#define MVG(a_uMsr, a_szName, a_uValue, a_fWrGpMask) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_FixedValue, kCpumMsrWrFn_IgnoreWrite, 0, a_uValue, 0, a_fWrGpMask, a_szName)
+/** Read fixed value, extended version with both GP and ignore masks. */
+#define MVX(a_uMsr, a_szName, a_uValue, a_fWrIgnMask, a_fWrGpMask) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_FixedValue, kCpumMsrWrFn_IgnoreWrite, 0, a_uValue, a_fWrIgnMask, a_fWrGpMask, a_szName)
+/** The short form, no CPUM backing. */
+#define MSN(a_uMsr, a_szName, a_enmRdFnSuff, a_enmWrFnSuff, a_uInitOrReadValue, a_fWrIgnMask, a_fWrGpMask) \
+    RINT(a_uMsr, a_uMsr, kCpumMsrRdFn_##a_enmRdFnSuff, kCpumMsrWrFn_##a_enmWrFnSuff, 0, \
+         a_uInitOrReadValue, a_fWrIgnMask, a_fWrGpMask, a_szName)
+
+/** Range: Functions handles everything. */
+#define RFN(a_uFirst, a_uLast, a_szName, a_enmRdFnSuff, a_enmWrFnSuff) \
+    RINT(a_uFirst, a_uLast, kCpumMsrRdFn_##a_enmRdFnSuff, kCpumMsrWrFn_##a_enmWrFnSuff, 0, 0, 0, 0, a_szName)
+/** Range: Read fixed value, read-only. */
+#define RVO(a_uFirst, a_uLast, a_szName, a_uValue) \
+    RINT(a_uFirst, a_uLast, kCpumMsrRdFn_FixedValue, kCpumMsrWrFn_ReadOnly, 0, a_uValue, 0, UINT64_MAX, a_szName)
+/** Range: Read fixed value, ignore writes. */
+#define RVI(a_uFirst, a_uLast, a_szName, a_uValue) \
+    RINT(a_uFirst, a_uLast, kCpumMsrRdFn_FixedValue, kCpumMsrWrFn_IgnoreWrite, 0, a_uValue, UINT64_MAX, 0, a_szName)
+/** Range: The short form, no CPUM backing. */
+#define RSN(a_uFirst, a_uLast, a_szName, a_enmRdFnSuff, a_enmWrFnSuff, a_uInitOrReadValue, a_fWrIgnMask, a_fWrGpMask) \
+    RINT(a_uFirst, a_uLast, kCpumMsrRdFn_##a_enmRdFnSuff, kCpumMsrWrFn_##a_enmWrFnSuff, 0, \
+         a_uInitOrReadValue, a_fWrIgnMask, a_fWrGpMask, a_szName)
+
+/** Internal form used by the macros. */
+#ifdef VBOX_WITH_STATISTICS
+# define RINT(a_uFirst, a_uLast, a_enmRdFn, a_enmWrFn, a_offCpumCpu, a_uInitOrReadValue, a_fWrIgnMask, a_fWrGpMask, a_szName) \
+    { a_uFirst, a_uLast, a_enmRdFn, a_enmWrFn, a_offCpumCpu, 0, a_uInitOrReadValue, a_fWrIgnMask, a_fWrGpMask, a_szName, \
+      { 0 }, { 0 }, { 0 }, { 0 } }
+#else
+# define RINT(a_uFirst, a_uLast, a_enmRdFn, a_enmWrFn, a_offCpumCpu, a_uInitOrReadValue, a_fWrIgnMask, a_fWrGpMask, a_szName) \
+    { a_uFirst, a_uLast, a_enmRdFn, a_enmWrFn, a_offCpumCpu, 0, a_uInitOrReadValue, a_fWrIgnMask, a_fWrGpMask, a_szName }
+#endif
+/** @} */
+
+#ifndef CPUM_DB_STANDALONE
+
+#include "cpus/Intel_Core_i7_6700K.h"
+#include "cpus/Intel_Core_i7_5600U.h"
+#include "cpus/Intel_Core_i7_3960X.h"
+#include "cpus/Intel_Core_i5_3570.h"
+#include "cpus/Intel_Core_i7_2635QM.h"
+#include "cpus/Intel_Xeon_X5482_3_20GHz.h"
+#include "cpus/Intel_Core2_X6800_2_93GHz.h"
+#include "cpus/Intel_Core2_T7600_2_33GHz.h"
+#include "cpus/Intel_Core_Duo_T2600_2_16GHz.h"
+#include "cpus/Intel_Pentium_M_processor_2_00GHz.h"
+#include "cpus/Intel_Pentium_4_3_00GHz.h"
+#include "cpus/Intel_Pentium_N3530_2_16GHz.h"
+#include "cpus/Intel_Atom_330_1_60GHz.h"
+#include "cpus/Intel_80486.h"
+#include "cpus/Intel_80386.h"
+#include "cpus/Intel_80286.h"
+#include "cpus/Intel_80186.h"
+#include "cpus/Intel_8086.h"
+
+#include "cpus/AMD_FX_8150_Eight_Core.h"
+#include "cpus/AMD_Phenom_II_X6_1100T.h"
+#include "cpus/Quad_Core_AMD_Opteron_2384.h"
+#include "cpus/AMD_Athlon_64_X2_Dual_Core_4200.h"
+#include "cpus/AMD_Athlon_64_3200.h"
+
+#include "cpus/VIA_QuadCore_L4700_1_2_GHz.h"
+
+#include "cpus/ZHAOXIN_KaiXian_KX_U5581_1_8GHz.h"
+
+#include "cpus/Hygon_C86_7185_32_core.h"
+
+
+/**
+ * The database entries.
+ *
+ * 1. The first entry is special.  It is the fallback for unknown
+ *    processors.  Thus, it better be pretty representative.
+ *
+ * 2. The first entry for a CPU vendor is likewise important as it is
+ *    the default entry for that vendor.
+ *
+ * Generally we put the most recent CPUs first, since these tend to have the
+ * most complicated and backwards compatible list of MSRs.
+ */
+static CPUMDBENTRY const * const g_apCpumDbEntries[] =
+{
+#ifdef VBOX_CPUDB_Intel_Core_i7_6700K_h
+    &g_Entry_Intel_Core_i7_6700K,
+#endif
+#ifdef VBOX_CPUDB_Intel_Core_i7_5600U_h
+    &g_Entry_Intel_Core_i7_5600U,
+#endif
+#ifdef VBOX_CPUDB_Intel_Core_i5_3570_h
+    &g_Entry_Intel_Core_i5_3570,
+#endif
+#ifdef VBOX_CPUDB_Intel_Core_i7_3960X_h
+    &g_Entry_Intel_Core_i7_3960X,
+#endif
+#ifdef VBOX_CPUDB_Intel_Core_i7_2635QM_h
+    &g_Entry_Intel_Core_i7_2635QM,
+#endif
+#ifdef VBOX_CPUDB_Intel_Pentium_N3530_2_16GHz_h
+    &g_Entry_Intel_Pentium_N3530_2_16GHz,
+#endif
+#ifdef VBOX_CPUDB_Intel_Atom_330_1_60GHz_h
+    &g_Entry_Intel_Atom_330_1_60GHz,
+#endif
+#ifdef VBOX_CPUDB_Intel_Pentium_M_processor_2_00GHz_h
+    &g_Entry_Intel_Pentium_M_processor_2_00GHz,
+#endif
+#ifdef VBOX_CPUDB_Intel_Xeon_X5482_3_20GHz_h
+    &g_Entry_Intel_Xeon_X5482_3_20GHz,
+#endif
+#ifdef VBOX_CPUDB_Intel_Core2_X6800_2_93GHz_h
+    &g_Entry_Intel_Core2_X6800_2_93GHz,
+#endif
+#ifdef VBOX_CPUDB_Intel_Core2_T7600_2_33GHz_h
+    &g_Entry_Intel_Core2_T7600_2_33GHz,
+#endif
+#ifdef VBOX_CPUDB_Intel_Core_Duo_T2600_2_16GHz_h
+    &g_Entry_Intel_Core_Duo_T2600_2_16GHz,
+#endif
+#ifdef VBOX_CPUDB_Intel_Pentium_4_3_00GHz_h
+    &g_Entry_Intel_Pentium_4_3_00GHz,
+#endif
+#ifdef VBOX_CPUDB_Intel_Pentium_4_3_00GHz_h
+    &g_Entry_Intel_Pentium_4_3_00GHz,
+#endif
+/** @todo pentium, pentium mmx, pentium pro, pentium II, pentium III */
+#ifdef VBOX_CPUDB_Intel_80486_h
+    &g_Entry_Intel_80486,
+#endif
+#ifdef VBOX_CPUDB_Intel_80386_h
+    &g_Entry_Intel_80386,
+#endif
+#ifdef VBOX_CPUDB_Intel_80286_h
+    &g_Entry_Intel_80286,
+#endif
+#ifdef VBOX_CPUDB_Intel_80186_h
+    &g_Entry_Intel_80186,
+#endif
+#ifdef VBOX_CPUDB_Intel_8086_h
+    &g_Entry_Intel_8086,
+#endif
+
+#ifdef VBOX_CPUDB_AMD_FX_8150_Eight_Core_h
+    &g_Entry_AMD_FX_8150_Eight_Core,
+#endif
+#ifdef VBOX_CPUDB_AMD_Phenom_II_X6_1100T_h
+    &g_Entry_AMD_Phenom_II_X6_1100T,
+#endif
+#ifdef VBOX_CPUDB_Quad_Core_AMD_Opteron_2384_h
+    &g_Entry_Quad_Core_AMD_Opteron_2384,
+#endif
+#ifdef VBOX_CPUDB_AMD_Athlon_64_X2_Dual_Core_4200_h
+    &g_Entry_AMD_Athlon_64_X2_Dual_Core_4200,
+#endif
+#ifdef VBOX_CPUDB_AMD_Athlon_64_3200_h
+    &g_Entry_AMD_Athlon_64_3200,
+#endif
+
+#ifdef VBOX_CPUDB_ZHAOXIN_KaiXian_KX_U5581_1_8GHz_h
+    &g_Entry_ZHAOXIN_KaiXian_KX_U5581_1_8GHz,
+#endif
+
+#ifdef VBOX_CPUDB_VIA_QuadCore_L4700_1_2_GHz_h
+    &g_Entry_VIA_QuadCore_L4700_1_2_GHz,
+#endif
+
+#ifdef VBOX_CPUDB_NEC_V20_h
+    &g_Entry_NEC_V20,
+#endif
+
+#ifdef VBOX_CPUDB_Hygon_C86_7185_32_core_h
+    &g_Entry_Hygon_C86_7185_32_core,
+#endif
+};
+
+
+
+/**
+ * Binary search used by cpumR3MsrRangesInsert and has some special properties
+ * wrt to mismatches.
+ *
+ * @returns Insert location.
+ * @param   paMsrRanges         The MSR ranges to search.
+ * @param   cMsrRanges          The number of MSR ranges.
+ * @param   uMsr                What to search for.
+ */
+static uint32_t cpumR3MsrRangesBinSearch(PCCPUMMSRRANGE paMsrRanges, uint32_t cMsrRanges, uint32_t uMsr)
+{
+    if (!cMsrRanges)
+        return 0;
+
+    uint32_t iStart = 0;
+    uint32_t iLast  = cMsrRanges - 1;
+    for (;;)
+    {
+        uint32_t i = iStart + (iLast - iStart + 1) / 2;
+        if (   uMsr >= paMsrRanges[i].uFirst
+            && uMsr <= paMsrRanges[i].uLast)
+            return i;
+        if (uMsr < paMsrRanges[i].uFirst)
+        {
+            if (i <= iStart)
+                return i;
+            iLast = i - 1;
+        }
+        else
+        {
+            if (i >= iLast)
+            {
+                if (i < cMsrRanges)
+                    i++;
+                return i;
+            }
+            iStart = i + 1;
+        }
+    }
+}
+
+
+/**
+ * Ensures that there is space for at least @a cNewRanges in the table,
+ * reallocating the table if necessary.
+ *
+ * @returns Pointer to the MSR ranges on success, NULL on failure.  On failure
+ *          @a *ppaMsrRanges is freed and set to NULL.
+ * @param   pVM             The cross context VM structure.  If NULL,
+ *                          use the process heap, otherwise the VM's hyper heap.
+ * @param   ppaMsrRanges    The variable pointing to the ranges (input/output).
+ * @param   cMsrRanges      The current number of ranges.
+ * @param   cNewRanges      The number of ranges to be added.
+ */
+static PCPUMMSRRANGE cpumR3MsrRangesEnsureSpace(PVM pVM, PCPUMMSRRANGE *ppaMsrRanges, uint32_t cMsrRanges, uint32_t cNewRanges)
+{
+    uint32_t cMsrRangesAllocated;
+    if (!pVM)
+        cMsrRangesAllocated = RT_ALIGN_32(cMsrRanges, 16);
+    else
+    {
+        /*
+         * We're using the hyper heap now, but when the range array was copied over to it from
+         * the host-context heap, we only copy the exact size and not the ensured size.
+         * See @bugref{7270}.
+         */
+        cMsrRangesAllocated = cMsrRanges;
+    }
+    if (cMsrRangesAllocated < cMsrRanges + cNewRanges)
+    {
+        void    *pvNew;
+        uint32_t cNew = RT_ALIGN_32(cMsrRanges + cNewRanges, 16);
+        if (pVM)
+        {
+            Assert(ppaMsrRanges == &pVM->cpum.s.GuestInfo.paMsrRangesR3);
+            Assert(cMsrRanges   == pVM->cpum.s.GuestInfo.cMsrRanges);
+
+            size_t cb    = cMsrRangesAllocated * sizeof(**ppaMsrRanges);
+            size_t cbNew = cNew * sizeof(**ppaMsrRanges);
+            int rc = MMR3HyperRealloc(pVM, *ppaMsrRanges, cb, 32, MM_TAG_CPUM_MSRS, cbNew, &pvNew);
+            if (RT_FAILURE(rc))
+            {
+                *ppaMsrRanges = NULL;
+                pVM->cpum.s.GuestInfo.paMsrRangesR0 = NIL_RTR0PTR;
+                LogRel(("CPUM: cpumR3MsrRangesEnsureSpace: MMR3HyperRealloc failed. rc=%Rrc\n", rc));
+                return NULL;
+            }
+            *ppaMsrRanges = (PCPUMMSRRANGE)pvNew;
+        }
+        else
+        {
+            pvNew = RTMemRealloc(*ppaMsrRanges, cNew * sizeof(**ppaMsrRanges));
+            if (!pvNew)
+            {
+                RTMemFree(*ppaMsrRanges);
+                *ppaMsrRanges = NULL;
+                return NULL;
+            }
+        }
+        *ppaMsrRanges = (PCPUMMSRRANGE)pvNew;
+    }
+
+    if (pVM)
+    {
+        /* Update the R0 pointer. */
+        Assert(ppaMsrRanges == &pVM->cpum.s.GuestInfo.paMsrRangesR3);
+        pVM->cpum.s.GuestInfo.paMsrRangesR0 = MMHyperR3ToR0(pVM, *ppaMsrRanges);
+    }
+
+    return *ppaMsrRanges;
+}
+
+
+/**
+ * Inserts a new MSR range in into an sorted MSR range array.
+ *
+ * If the new MSR range overlaps existing ranges, the existing ones will be
+ * adjusted/removed to fit in the new one.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_NO_MEMORY
+ *
+ * @param   pVM             The cross context VM structure.  If NULL,
+ *                          use the process heap, otherwise the VM's hyper heap.
+ * @param   ppaMsrRanges    The variable pointing to the ranges (input/output).
+ *                          Must be NULL if using the hyper heap.
+ * @param   pcMsrRanges     The variable holding number of ranges. Must be NULL
+ *                          if using the hyper heap.
+ * @param   pNewRange       The new range.
+ */
+int cpumR3MsrRangesInsert(PVM pVM, PCPUMMSRRANGE *ppaMsrRanges, uint32_t *pcMsrRanges, PCCPUMMSRRANGE pNewRange)
+{
+    Assert(pNewRange->uLast >= pNewRange->uFirst);
+    Assert(pNewRange->enmRdFn > kCpumMsrRdFn_Invalid && pNewRange->enmRdFn < kCpumMsrRdFn_End);
+    Assert(pNewRange->enmWrFn > kCpumMsrWrFn_Invalid && pNewRange->enmWrFn < kCpumMsrWrFn_End);
+
+    /*
+     * Validate and use the VM's MSR ranges array if we are using the hyper heap.
+     */
+    if (pVM)
+    {
+        AssertReturn(!ppaMsrRanges, VERR_INVALID_PARAMETER);
+        AssertReturn(!pcMsrRanges,  VERR_INVALID_PARAMETER);
+
+        ppaMsrRanges = &pVM->cpum.s.GuestInfo.paMsrRangesR3;
+        pcMsrRanges  = &pVM->cpum.s.GuestInfo.cMsrRanges;
+    }
+    else
+    {
+        AssertReturn(ppaMsrRanges, VERR_INVALID_POINTER);
+        AssertReturn(pcMsrRanges, VERR_INVALID_POINTER);
+    }
+
+    uint32_t        cMsrRanges  = *pcMsrRanges;
+    PCPUMMSRRANGE   paMsrRanges = *ppaMsrRanges;
+
+    /*
+     * Optimize the linear insertion case where we add new entries at the end.
+     */
+    if (   cMsrRanges > 0
+        && paMsrRanges[cMsrRanges - 1].uLast < pNewRange->uFirst)
+    {
+        paMsrRanges = cpumR3MsrRangesEnsureSpace(pVM, ppaMsrRanges, cMsrRanges, 1);
+        if (!paMsrRanges)
+            return VERR_NO_MEMORY;
+        paMsrRanges[cMsrRanges] = *pNewRange;
+        *pcMsrRanges += 1;
+    }
+    else
+    {
+        uint32_t i = cpumR3MsrRangesBinSearch(paMsrRanges, cMsrRanges, pNewRange->uFirst);
+        Assert(i == cMsrRanges || pNewRange->uFirst <= paMsrRanges[i].uLast);
+        Assert(i == 0 || pNewRange->uFirst > paMsrRanges[i - 1].uLast);
+
+        /*
+         * Adding an entirely new entry?
+         */
+        if (   i >= cMsrRanges
+            || pNewRange->uLast < paMsrRanges[i].uFirst)
+        {
+            paMsrRanges = cpumR3MsrRangesEnsureSpace(pVM, ppaMsrRanges, cMsrRanges, 1);
+            if (!paMsrRanges)
+                return VERR_NO_MEMORY;
+            if (i < cMsrRanges)
+                memmove(&paMsrRanges[i + 1], &paMsrRanges[i], (cMsrRanges - i) * sizeof(paMsrRanges[0]));
+            paMsrRanges[i] = *pNewRange;
+            *pcMsrRanges += 1;
+        }
+        /*
+         * Replace existing entry?
+         */
+        else if (   pNewRange->uFirst == paMsrRanges[i].uFirst
+                 && pNewRange->uLast  == paMsrRanges[i].uLast)
+            paMsrRanges[i] = *pNewRange;
+        /*
+         * Splitting an existing entry?
+         */
+        else if (   pNewRange->uFirst > paMsrRanges[i].uFirst
+                 && pNewRange->uLast  < paMsrRanges[i].uLast)
+        {
+            paMsrRanges = cpumR3MsrRangesEnsureSpace(pVM, ppaMsrRanges, cMsrRanges, 2);
+            if (!paMsrRanges)
+                return VERR_NO_MEMORY;
+            if (i < cMsrRanges)
+                memmove(&paMsrRanges[i + 2], &paMsrRanges[i], (cMsrRanges - i) * sizeof(paMsrRanges[0]));
+            paMsrRanges[i + 1] = *pNewRange;
+            paMsrRanges[i + 2] = paMsrRanges[i];
+            paMsrRanges[i    ].uLast  = pNewRange->uFirst - 1;
+            paMsrRanges[i + 2].uFirst = pNewRange->uLast  + 1;
+            *pcMsrRanges += 2;
+        }
+        /*
+         * Complicated scenarios that can affect more than one range.
+         *
+         * The current code does not optimize memmove calls when replacing
+         * one or more existing ranges, because it's tedious to deal with and
+         * not expected to be a frequent usage scenario.
+         */
+        else
+        {
+            /* Adjust start of first match? */
+            if (   pNewRange->uFirst <= paMsrRanges[i].uFirst
+                && pNewRange->uLast  <  paMsrRanges[i].uLast)
+                paMsrRanges[i].uFirst = pNewRange->uLast + 1;
+            else
+            {
+                /* Adjust end of first match? */
+                if (pNewRange->uFirst > paMsrRanges[i].uFirst)
+                {
+                    Assert(paMsrRanges[i].uLast >= pNewRange->uFirst);
+                    paMsrRanges[i].uLast = pNewRange->uFirst - 1;
+                    i++;
+                }
+                /* Replace the whole first match (lazy bird). */
+                else
+                {
+                    if (i + 1 < cMsrRanges)
+                        memmove(&paMsrRanges[i], &paMsrRanges[i + 1], (cMsrRanges - i - 1) * sizeof(paMsrRanges[0]));
+                    cMsrRanges = *pcMsrRanges -= 1;
+                }
+
+                /* Do the new range affect more ranges? */
+                while (   i < cMsrRanges
+                       && pNewRange->uLast >= paMsrRanges[i].uFirst)
+                {
+                    if (pNewRange->uLast < paMsrRanges[i].uLast)
+                    {
+                        /* Adjust the start of it, then we're done. */
+                        paMsrRanges[i].uFirst = pNewRange->uLast + 1;
+                        break;
+                    }
+
+                    /* Remove it entirely. */
+                    if (i + 1 < cMsrRanges)
+                        memmove(&paMsrRanges[i], &paMsrRanges[i + 1], (cMsrRanges - i - 1) * sizeof(paMsrRanges[0]));
+                    cMsrRanges = *pcMsrRanges -= 1;
+                }
+            }
+
+            /* Now, perform a normal insertion. */
+            paMsrRanges = cpumR3MsrRangesEnsureSpace(pVM, ppaMsrRanges, cMsrRanges, 1);
+            if (!paMsrRanges)
+                return VERR_NO_MEMORY;
+            if (i < cMsrRanges)
+                memmove(&paMsrRanges[i + 1], &paMsrRanges[i], (cMsrRanges - i) * sizeof(paMsrRanges[0]));
+            paMsrRanges[i] = *pNewRange;
+            *pcMsrRanges += 1;
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Reconciles CPUID info with MSRs (selected ones).
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ */
+int cpumR3MsrReconcileWithCpuId(PVM pVM)
+{
+    PCCPUMMSRRANGE papToAdd[10];
+    uint32_t       cToAdd = 0;
+
+    /*
+     * The IA32_FLUSH_CMD MSR was introduced in MCUs for CVS-2018-3646 and associates.
+     */
+    if (pVM->cpum.s.GuestFeatures.fFlushCmd && !cpumLookupMsrRange(pVM, MSR_IA32_FLUSH_CMD))
+    {
+        static CPUMMSRRANGE const s_FlushCmd =
+        {
+            /*.uFirst =*/       MSR_IA32_FLUSH_CMD,
+            /*.uLast =*/        MSR_IA32_FLUSH_CMD,
+            /*.enmRdFn =*/      kCpumMsrRdFn_WriteOnly,
+            /*.enmWrFn =*/      kCpumMsrWrFn_Ia32FlushCmd,
+            /*.offCpumCpu =*/   UINT16_MAX,
+            /*.fReserved =*/    0,
+            /*.uValue =*/       0,
+            /*.fWrIgnMask =*/   0,
+            /*.fWrGpMask =*/    ~MSR_IA32_FLUSH_CMD_F_L1D,
+            /*.szName = */      "IA32_FLUSH_CMD"
+        };
+        papToAdd[cToAdd++] = &s_FlushCmd;
+    }
+
+    /*
+     * The MSR_IA32_ARCH_CAPABILITIES was introduced in various spectre MCUs, or at least
+     * documented in relation to such.
+     */
+    if (pVM->cpum.s.GuestFeatures.fArchCap && !cpumLookupMsrRange(pVM, MSR_IA32_ARCH_CAPABILITIES))
+    {
+        static CPUMMSRRANGE const s_ArchCaps =
+        {
+            /*.uFirst =*/       MSR_IA32_ARCH_CAPABILITIES,
+            /*.uLast =*/        MSR_IA32_ARCH_CAPABILITIES,
+            /*.enmRdFn =*/      kCpumMsrRdFn_Ia32ArchCapabilities,
+            /*.enmWrFn =*/      kCpumMsrWrFn_ReadOnly,
+            /*.offCpumCpu =*/   UINT16_MAX,
+            /*.fReserved =*/    0,
+            /*.uValue =*/       0,
+            /*.fWrIgnMask =*/   0,
+            /*.fWrGpMask =*/    UINT64_MAX,
+            /*.szName = */      "IA32_ARCH_CAPABILITIES"
+        };
+        papToAdd[cToAdd++] = &s_ArchCaps;
+    }
+
+    /*
+     * Do the adding.
+     */
+    for (uint32_t i = 0; i < cToAdd; i++)
+    {
+        PCCPUMMSRRANGE pRange = papToAdd[i];
+        LogRel(("CPUM: MSR/CPUID reconciliation insert: %#010x %s\n", pRange->uFirst, pRange->szName));
+        int rc = cpumR3MsrRangesInsert(NULL /* pVM */, &pVM->cpum.s.GuestInfo.paMsrRangesR3, &pVM->cpum.s.GuestInfo.cMsrRanges,
+                                       pRange);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for cpumR3MsrApplyFudge that applies one table.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   paRanges            Array of MSRs to fudge.
+ * @param   cRanges             Number of MSRs in the array.
+ */
+static int cpumR3MsrApplyFudgeTable(PVM pVM, PCCPUMMSRRANGE paRanges, size_t cRanges)
+{
+    for (uint32_t i = 0; i < cRanges; i++)
+        if (!cpumLookupMsrRange(pVM, paRanges[i].uFirst))
+        {
+            LogRel(("CPUM: MSR fudge: %#010x %s\n", paRanges[i].uFirst, paRanges[i].szName));
+            int rc = cpumR3MsrRangesInsert(NULL /* pVM */, &pVM->cpum.s.GuestInfo.paMsrRangesR3, &pVM->cpum.s.GuestInfo.cMsrRanges,
+                                           &paRanges[i]);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Fudges the MSRs that guest are known to access in some odd cases.
+ *
+ * A typical example is a VM that has been moved between different hosts where
+ * for instance the cpu vendor differs.
+ *
+ * Another example is older CPU profiles (e.g. Atom Bonnet) for newer CPUs (e.g.
+ * Atom Silvermont), where features reported thru CPUID aren't present in the
+ * MSRs (e.g. AMD64_TSC_AUX).
+ *
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ */
+int cpumR3MsrApplyFudge(PVM pVM)
+{
+    /*
+     * Basic.
+     */
+    static CPUMMSRRANGE const s_aFudgeMsrs[] =
+    {
+        MFO(0x00000000, "IA32_P5_MC_ADDR",          Ia32P5McAddr),
+        MFX(0x00000001, "IA32_P5_MC_TYPE",          Ia32P5McType,   Ia32P5McType,   0, 0, UINT64_MAX),
+        MVO(0x00000017, "IA32_PLATFORM_ID",         0),
+        MFN(0x0000001b, "IA32_APIC_BASE",           Ia32ApicBase,   Ia32ApicBase),
+        MVI(0x0000008b, "BIOS_SIGN",                0),
+        MFX(0x000000fe, "IA32_MTRRCAP",             Ia32MtrrCap,    ReadOnly,       0x508, 0, 0),
+        MFX(0x00000179, "IA32_MCG_CAP",             Ia32McgCap,     ReadOnly,       0x005, 0, 0),
+        MFX(0x0000017a, "IA32_MCG_STATUS",          Ia32McgStatus,  Ia32McgStatus,  0, ~(uint64_t)UINT32_MAX, 0),
+        MFN(0x000001a0, "IA32_MISC_ENABLE",         Ia32MiscEnable, Ia32MiscEnable),
+        MFN(0x000001d9, "IA32_DEBUGCTL",            Ia32DebugCtl,   Ia32DebugCtl),
+        MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP",   P6LastBranchFromIp),
+        MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP",     P6LastBranchToIp),
+        MFO(0x000001dd, "P6_LAST_INT_FROM_IP",      P6LastIntFromIp),
+        MFO(0x000001de, "P6_LAST_INT_TO_IP",        P6LastIntToIp),
+        MFS(0x00000277, "IA32_PAT",                 Ia32Pat, Ia32Pat, Guest.msrPAT),
+        MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE",       Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, ~(uint64_t)0xc07),
+        MFN(0x00000400, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    };
+    int rc = cpumR3MsrApplyFudgeTable(pVM, &s_aFudgeMsrs[0], RT_ELEMENTS(s_aFudgeMsrs));
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * XP might mistake opterons and other newer CPUs for P4s.
+     */
+    if (pVM->cpum.s.GuestFeatures.uFamily >= 0xf)
+    {
+        static CPUMMSRRANGE const s_aP4FudgeMsrs[] =
+        {
+            MFX(0x0000002c, "P4_EBC_FREQUENCY_ID", IntelP4EbcFrequencyId, IntelP4EbcFrequencyId, 0xf12010f, UINT64_MAX, 0),
+        };
+        rc = cpumR3MsrApplyFudgeTable(pVM, &s_aP4FudgeMsrs[0], RT_ELEMENTS(s_aP4FudgeMsrs));
+        AssertLogRelRCReturn(rc, rc);
+    }
+
+    if (pVM->cpum.s.GuestFeatures.fRdTscP)
+    {
+        static CPUMMSRRANGE const s_aRdTscPFudgeMsrs[] =
+        {
+            MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, 0, ~(uint64_t)UINT32_MAX),
+        };
+        rc = cpumR3MsrApplyFudgeTable(pVM, &s_aRdTscPFudgeMsrs[0], RT_ELEMENTS(s_aRdTscPFudgeMsrs));
+        AssertLogRelRCReturn(rc, rc);
+    }
+
+    /*
+     * Windows 10 incorrectly writes to MSR_IA32_TSX_CTRL without checking
+     * CPUID.ARCH_CAP(EAX=7h,ECX=0):EDX[bit 29] or the MSR feature bits in
+     * MSR_IA32_ARCH_CAPABILITIES[bit 7], see @bugref{9630}.
+     * Ignore writes to this MSR and return 0 on reads.
+     */
+    if (pVM->cpum.s.GuestFeatures.fArchCap)
+    {
+        static CPUMMSRRANGE const s_aTsxCtrl[] =
+        {
+            MVI(MSR_IA32_TSX_CTRL, "IA32_TSX_CTRL", 0),
+        };
+        rc = cpumR3MsrApplyFudgeTable(pVM, &s_aTsxCtrl[0], RT_ELEMENTS(s_aTsxCtrl));
+        AssertLogRelRCReturn(rc, rc);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Do we consider @a enmConsider a better match for @a enmTarget than
+ * @a enmFound?
+ *
+ * Only called when @a enmConsider isn't exactly what we're looking for.
+ *
+ * @returns true/false.
+ * @param   enmConsider         The new microarch to consider.
+ * @param   enmTarget           The target microarch.
+ * @param   enmFound            The best microarch match we've found thus far.
+ */
+DECLINLINE(bool) cpumR3DbIsBetterMarchMatch(CPUMMICROARCH enmConsider, CPUMMICROARCH enmTarget, CPUMMICROARCH enmFound)
+{
+    Assert(enmConsider != enmTarget);
+
+    /*
+     * If we've got an march match, don't bother with enmConsider.
+     */
+    if (enmFound == enmTarget)
+        return false;
+
+    /*
+     * Found is below: Pick 'consider' if it's closer to the target or above it.
+     */
+    if (enmFound < enmTarget)
+        return enmConsider > enmFound;
+
+    /*
+     * Found is above: Pick 'consider' if it's also above (paranoia: or equal)
+     *                 and but closer to the target.
+     */
+    return enmConsider >= enmTarget && enmConsider < enmFound;
+}
+
+
+/**
+ * Do we consider @a enmConsider a better match for @a enmTarget than
+ * @a enmFound?
+ *
+ * Only called for intel family 06h CPUs.
+ *
+ * @returns true/false.
+ * @param   enmConsider         The new microarch to consider.
+ * @param   enmTarget           The target microarch.
+ * @param   enmFound            The best microarch match we've found thus far.
+ */
+static bool cpumR3DbIsBetterIntelFam06Match(CPUMMICROARCH enmConsider, CPUMMICROARCH enmTarget, CPUMMICROARCH enmFound)
+{
+    /* Check intel family 06h claims. */
+    AssertReturn(enmConsider >= kCpumMicroarch_Intel_P6_Core_Atom_First && enmConsider <= kCpumMicroarch_Intel_P6_Core_Atom_End,
+                 false);
+    AssertReturn(enmTarget   >= kCpumMicroarch_Intel_P6_Core_Atom_First && enmTarget   <= kCpumMicroarch_Intel_P6_Core_Atom_End,
+                 false);
+
+    /* Put matches out of the way. */
+    if (enmConsider == enmTarget)
+        return true;
+    if (enmFound == enmTarget)
+        return false;
+
+    /* If found isn't a family 06h march, whatever we're considering must be a better choice. */
+    if (   enmFound < kCpumMicroarch_Intel_P6_Core_Atom_First
+        || enmFound > kCpumMicroarch_Intel_P6_Core_Atom_End)
+        return true;
+
+    /*
+     * The family 06h stuff is split into three categories:
+     *      - Common P6 heritage
+     *      - Core
+     *      - Atom
+     *
+     * Determin which of the three arguments are Atom marchs, because that's
+     * all we need to make the right choice.
+     */
+    bool const fConsiderAtom = enmConsider >= kCpumMicroarch_Intel_Atom_First;
+    bool const fTargetAtom   = enmTarget   >= kCpumMicroarch_Intel_Atom_First;
+    bool const fFoundAtom    = enmFound    >= kCpumMicroarch_Intel_Atom_First;
+
+    /*
+     * Want atom:
+     */
+    if (fTargetAtom)
+    {
+        /* Pick the atom if we've got one of each.*/
+        if (fConsiderAtom != fFoundAtom)
+            return fConsiderAtom;
+        /* If we haven't got any atoms under consideration, pick a P6 or the earlier core.
+           Note! Not entirely sure Dothan is the best choice, but it'll do for now. */
+        if (!fConsiderAtom)
+        {
+            if (enmConsider > enmFound)
+                return enmConsider <= kCpumMicroarch_Intel_P6_M_Dothan;
+            return enmFound > kCpumMicroarch_Intel_P6_M_Dothan;
+        }
+        /* else: same category, default comparison rules. */
+        Assert(fConsiderAtom && fFoundAtom);
+    }
+    /*
+     * Want non-atom:
+     */
+    /* Pick the non-atom if we've got one of each. */
+    else if (fConsiderAtom != fFoundAtom)
+        return fFoundAtom;
+    /* If we've only got atoms under consideration, pick the older one just to pick something. */
+    else if (fConsiderAtom)
+        return enmConsider < enmFound;
+    else
+        Assert(!fConsiderAtom && !fFoundAtom);
+
+    /*
+     * Same basic category.  Do same compare as caller.
+     */
+    return cpumR3DbIsBetterMarchMatch(enmConsider, enmTarget, enmFound);
+}
+
+
+int cpumR3DbGetCpuInfo(const char *pszName, PCPUMINFO pInfo)
+{
+    CPUMDBENTRY const *pEntry = NULL;
+    int                rc;
+
+    if (!strcmp(pszName, "host"))
+    {
+        /*
+         * Create a CPU database entry for the host CPU.  This means getting
+         * the CPUID bits from the real CPU and grabbing the closest matching
+         * database entry for MSRs.
+         */
+        rc = CPUMR3CpuIdDetectUnknownLeafMethod(&pInfo->enmUnknownCpuIdMethod, &pInfo->DefCpuId);
+        if (RT_FAILURE(rc))
+            return rc;
+        rc = CPUMR3CpuIdCollectLeaves(&pInfo->paCpuIdLeavesR3, &pInfo->cCpuIdLeaves);
+        if (RT_FAILURE(rc))
+            return rc;
+        pInfo->fMxCsrMask = CPUMR3DeterminHostMxCsrMask();
+
+        /* Lookup database entry for MSRs. */
+        CPUMCPUVENDOR const enmVendor    = CPUMR3CpuIdDetectVendorEx(pInfo->paCpuIdLeavesR3[0].uEax,
+                                                                     pInfo->paCpuIdLeavesR3[0].uEbx,
+                                                                     pInfo->paCpuIdLeavesR3[0].uEcx,
+                                                                     pInfo->paCpuIdLeavesR3[0].uEdx);
+        uint32_t      const uStd1Eax     = pInfo->paCpuIdLeavesR3[1].uEax;
+        uint8_t       const uFamily      = ASMGetCpuFamily(uStd1Eax);
+        uint8_t       const uModel       = ASMGetCpuModel(uStd1Eax, enmVendor == CPUMCPUVENDOR_INTEL);
+        uint8_t       const uStepping    = ASMGetCpuStepping(uStd1Eax);
+        CPUMMICROARCH const enmMicroarch = CPUMR3CpuIdDetermineMicroarchEx(enmVendor, uFamily, uModel, uStepping);
+
+        for (unsigned i = 0; i < RT_ELEMENTS(g_apCpumDbEntries); i++)
+        {
+            CPUMDBENTRY const *pCur = g_apCpumDbEntries[i];
+            if ((CPUMCPUVENDOR)pCur->enmVendor == enmVendor)
+            {
+                /* Match against Family, Microarch, model and stepping.  Except
+                   for family, always match the closer with preference given to
+                   the later/older ones. */
+                if (pCur->uFamily == uFamily)
+                {
+                    if (pCur->enmMicroarch == enmMicroarch)
+                    {
+                        if (pCur->uModel == uModel)
+                        {
+                            if (pCur->uStepping == uStepping)
+                            {
+                                /* Perfect match. */
+                                pEntry = pCur;
+                                break;
+                            }
+
+                            if (   !pEntry
+                                || pEntry->uModel       != uModel
+                                || pEntry->enmMicroarch != enmMicroarch
+                                || pEntry->uFamily      != uFamily)
+                                pEntry = pCur;
+                            else if (  pCur->uStepping >= uStepping
+                                     ? pCur->uStepping < pEntry->uStepping || pEntry->uStepping < uStepping
+                                     : pCur->uStepping > pEntry->uStepping)
+                                     pEntry = pCur;
+                        }
+                        else if (   !pEntry
+                                 || pEntry->enmMicroarch != enmMicroarch
+                                 || pEntry->uFamily      != uFamily)
+                            pEntry = pCur;
+                        else if (  pCur->uModel >= uModel
+                                 ? pCur->uModel < pEntry->uModel || pEntry->uModel < uModel
+                                 : pCur->uModel > pEntry->uModel)
+                            pEntry = pCur;
+                    }
+                    else if (   !pEntry
+                             || pEntry->uFamily != uFamily)
+                        pEntry = pCur;
+                    /* Special march matching rules applies to intel family 06h. */
+                    else if (     enmVendor == CPUMCPUVENDOR_INTEL
+                               && uFamily   == 6
+                             ? cpumR3DbIsBetterIntelFam06Match(pCur->enmMicroarch, enmMicroarch, pEntry->enmMicroarch)
+                             : cpumR3DbIsBetterMarchMatch(pCur->enmMicroarch, enmMicroarch, pEntry->enmMicroarch))
+                        pEntry = pCur;
+                }
+                /* We don't do closeness matching on family, we use the first
+                   entry for the CPU vendor instead. (P4 workaround.) */
+                else if (!pEntry)
+                    pEntry = pCur;
+            }
+        }
+
+        if (pEntry)
+            LogRel(("CPUM: Matched host CPU %s %#x/%#x/%#x %s with CPU DB entry '%s' (%s %#x/%#x/%#x %s)\n",
+                    CPUMR3CpuVendorName(enmVendor), uFamily, uModel, uStepping, CPUMR3MicroarchName(enmMicroarch),
+                    pEntry->pszName,  CPUMR3CpuVendorName((CPUMCPUVENDOR)pEntry->enmVendor), pEntry->uFamily, pEntry->uModel,
+                    pEntry->uStepping, CPUMR3MicroarchName(pEntry->enmMicroarch) ));
+        else
+        {
+            pEntry = g_apCpumDbEntries[0];
+            LogRel(("CPUM: No matching processor database entry %s %#x/%#x/%#x %s, falling back on '%s'\n",
+                    CPUMR3CpuVendorName(enmVendor), uFamily, uModel, uStepping, CPUMR3MicroarchName(enmMicroarch),
+                    pEntry->pszName));
+        }
+    }
+    else
+    {
+        /*
+         * We're supposed to be emulating a specific CPU that is included in
+         * our CPU database.  The CPUID tables needs to be copied onto the
+         * heap so the caller can modify them and so they can be freed like
+         * in the host case above.
+         */
+        for (unsigned i = 0; i < RT_ELEMENTS(g_apCpumDbEntries); i++)
+            if (!strcmp(pszName, g_apCpumDbEntries[i]->pszName))
+            {
+                pEntry = g_apCpumDbEntries[i];
+                break;
+            }
+        if (!pEntry)
+        {
+            LogRel(("CPUM: Cannot locate any CPU by the name '%s'\n", pszName));
+            return VERR_CPUM_DB_CPU_NOT_FOUND;
+        }
+
+        pInfo->cCpuIdLeaves = pEntry->cCpuIdLeaves;
+        if (pEntry->cCpuIdLeaves)
+        {
+            /* Must allocate a multiple of 16 here, matching cpumR3CpuIdEnsureSpace. */
+            size_t cbExtra = sizeof(pEntry->paCpuIdLeaves[0]) * (RT_ALIGN(pEntry->cCpuIdLeaves, 16) - pEntry->cCpuIdLeaves);
+            pInfo->paCpuIdLeavesR3 = (PCPUMCPUIDLEAF)RTMemDupEx(pEntry->paCpuIdLeaves,
+                                                                sizeof(pEntry->paCpuIdLeaves[0]) * pEntry->cCpuIdLeaves,
+                                                                cbExtra);
+            if (!pInfo->paCpuIdLeavesR3)
+                return VERR_NO_MEMORY;
+        }
+        else
+            pInfo->paCpuIdLeavesR3 = NULL;
+
+        pInfo->enmUnknownCpuIdMethod = pEntry->enmUnknownCpuId;
+        pInfo->DefCpuId              = pEntry->DefUnknownCpuId;
+        pInfo->fMxCsrMask            = pEntry->fMxCsrMask;
+
+        LogRel(("CPUM: Using CPU DB entry '%s' (%s %#x/%#x/%#x %s)\n",
+                pEntry->pszName, CPUMR3CpuVendorName((CPUMCPUVENDOR)pEntry->enmVendor),
+                pEntry->uFamily, pEntry->uModel, pEntry->uStepping, CPUMR3MicroarchName(pEntry->enmMicroarch) ));
+    }
+
+    pInfo->fMsrMask             = pEntry->fMsrMask;
+    pInfo->iFirstExtCpuIdLeaf   = 0; /* Set by caller. */
+    pInfo->uScalableBusFreq     = pEntry->uScalableBusFreq;
+    pInfo->paCpuIdLeavesR0      = NIL_RTR0PTR;
+    pInfo->paMsrRangesR0        = NIL_RTR0PTR;
+
+    /*
+     * Copy the MSR range.
+     */
+    uint32_t        cMsrs   = 0;
+    PCPUMMSRRANGE   paMsrs  = NULL;
+
+    PCCPUMMSRRANGE  pCurMsr = pEntry->paMsrRanges;
+    uint32_t        cLeft   = pEntry->cMsrRanges;
+    while (cLeft-- > 0)
+    {
+        rc = cpumR3MsrRangesInsert(NULL /* pVM */, &paMsrs, &cMsrs, pCurMsr);
+        if (RT_FAILURE(rc))
+        {
+            Assert(!paMsrs); /* The above function frees this. */
+            RTMemFree(pInfo->paCpuIdLeavesR3);
+            pInfo->paCpuIdLeavesR3 = NULL;
+            return rc;
+        }
+        pCurMsr++;
+    }
+
+    pInfo->paMsrRangesR3   = paMsrs;
+    pInfo->cMsrRanges      = cMsrs;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Insert an MSR range into the VM.
+ *
+ * If the new MSR range overlaps existing ranges, the existing ones will be
+ * adjusted/removed to fit in the new one.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pNewRange           Pointer to the MSR range being inserted.
+ */
+VMMR3DECL(int) CPUMR3MsrRangesInsert(PVM pVM, PCCPUMMSRRANGE pNewRange)
+{
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+    AssertReturn(pNewRange, VERR_INVALID_PARAMETER);
+
+    return cpumR3MsrRangesInsert(pVM, NULL /* ppaMsrRanges */, NULL /* pcMsrRanges */, pNewRange);
+}
+
+
+/**
+ * Register statistics for the MSRs.
+ *
+ * This must not be called before the MSRs have been finalized and moved to the
+ * hyper heap.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ */
+int cpumR3MsrRegStats(PVM pVM)
+{
+    /*
+     * Global statistics.
+     */
+    PCPUM pCpum = &pVM->cpum.s;
+    STAM_REL_REG(pVM, &pCpum->cMsrReads,                STAMTYPE_COUNTER,   "/CPUM/MSR-Totals/Reads",
+                 STAMUNIT_OCCURENCES, "All RDMSRs making it to CPUM.");
+    STAM_REL_REG(pVM, &pCpum->cMsrReadsRaiseGp,         STAMTYPE_COUNTER,   "/CPUM/MSR-Totals/ReadsRaisingGP",
+                 STAMUNIT_OCCURENCES, "RDMSR raising #GPs, except unknown MSRs.");
+    STAM_REL_REG(pVM, &pCpum->cMsrReadsUnknown,         STAMTYPE_COUNTER,   "/CPUM/MSR-Totals/ReadsUnknown",
+                 STAMUNIT_OCCURENCES, "RDMSR on unknown MSRs (raises #GP).");
+    STAM_REL_REG(pVM, &pCpum->cMsrWrites,               STAMTYPE_COUNTER,   "/CPUM/MSR-Totals/Writes",
+                 STAMUNIT_OCCURENCES, "All WRMSRs making it to CPUM.");
+    STAM_REL_REG(pVM, &pCpum->cMsrWritesRaiseGp,        STAMTYPE_COUNTER,   "/CPUM/MSR-Totals/WritesRaisingGP",
+                 STAMUNIT_OCCURENCES, "WRMSR raising #GPs, except unknown MSRs.");
+    STAM_REL_REG(pVM, &pCpum->cMsrWritesToIgnoredBits,  STAMTYPE_COUNTER,   "/CPUM/MSR-Totals/WritesToIgnoredBits",
+                 STAMUNIT_OCCURENCES, "Writing of ignored bits.");
+    STAM_REL_REG(pVM, &pCpum->cMsrWritesUnknown,        STAMTYPE_COUNTER,   "/CPUM/MSR-Totals/WritesUnknown",
+                 STAMUNIT_OCCURENCES, "WRMSR on unknown MSRs (raises #GP).");
+
+
+# ifdef VBOX_WITH_STATISTICS
+    /*
+     * Per range.
+     */
+    PCPUMMSRRANGE   paRanges = pVM->cpum.s.GuestInfo.paMsrRangesR3;
+    uint32_t        cRanges  = pVM->cpum.s.GuestInfo.cMsrRanges;
+    for (uint32_t i = 0; i < cRanges; i++)
+    {
+        char    szName[160];
+        ssize_t cchName;
+
+        if (paRanges[i].uFirst == paRanges[i].uLast)
+            cchName = RTStrPrintf(szName, sizeof(szName), "/CPUM/MSRs/%#010x-%s",
+                                  paRanges[i].uFirst, paRanges[i].szName);
+        else
+            cchName = RTStrPrintf(szName, sizeof(szName), "/CPUM/MSRs/%#010x-%#010x-%s",
+                                  paRanges[i].uFirst, paRanges[i].uLast, paRanges[i].szName);
+
+        RTStrCopy(&szName[cchName], sizeof(szName) - cchName, "-reads");
+        STAMR3Register(pVM, &paRanges[i].cReads, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_OCCURENCES, "RDMSR");
+
+        RTStrCopy(&szName[cchName], sizeof(szName) - cchName, "-writes");
+        STAMR3Register(pVM, &paRanges[i].cWrites, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, "WRMSR");
+
+        RTStrCopy(&szName[cchName], sizeof(szName) - cchName, "-GPs");
+        STAMR3Register(pVM, &paRanges[i].cGps, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, "#GPs");
+
+        RTStrCopy(&szName[cchName], sizeof(szName) - cchName, "-ign-bits-writes");
+        STAMR3Register(pVM, &paRanges[i].cIgnoredBits, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, "WRMSR w/ ignored bits");
+    }
+# endif /* VBOX_WITH_STATISTICS */
+
+    return VINF_SUCCESS;
+}
+
+#endif /* !CPUM_DB_STANDALONE */
+
diff --git a/src/VBox/VMM/VMMR3/DBGF.cpp b/src/VBox/VMM/VMMR3/DBGF.cpp
new file mode 100644
index 00000000..f40880c3
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGF.cpp
@@ -0,0 +1,2106 @@
+/* $Id: DBGF.cpp $ */
+/** @file
+ * DBGF - Debugger Facility.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page   pg_dbgf     DBGF - The Debugger Facility
+ *
+ * The purpose of the DBGF is to provide an interface for debuggers to
+ * manipulate the VMM without having to mess up the source code for each of
+ * them. The DBGF is always built in and will always work when a debugger
+ * attaches to the VM. The DBGF provides the basic debugger features, such as
+ * halting execution, handling breakpoints, single step execution, instruction
+ * disassembly, info querying, OS specific diggers, symbol and module
+ * management.
+ *
+ * The interface is working in a manner similar to the win32, linux and os2
+ * debugger interfaces. The interface has an asynchronous nature. This comes
+ * from the fact that the VMM and the Debugger are running in different threads.
+ * They are referred to as the "emulation thread" and the "debugger thread", or
+ * as the "ping thread" and the "pong thread, respectivly. (The last set of
+ * names comes from the use of the Ping-Pong synchronization construct from the
+ * RTSem API.)
+ *
+ * @see grp_dbgf
+ *
+ *
+ * @section sec_dbgf_scenario   Usage Scenario
+ *
+ * The debugger starts by attaching to the VM. For practical reasons we limit the
+ * number of concurrently attached debuggers to 1 per VM. The action of
+ * attaching to the VM causes the VM to check and generate debug events.
+ *
+ * The debugger then will wait/poll for debug events and issue commands.
+ *
+ * The waiting and polling is done by the DBGFEventWait() function. It will wait
+ * for the emulation thread to send a ping, thus indicating that there is an
+ * event waiting to be processed.
+ *
+ * An event can be a response to a command issued previously, the hitting of a
+ * breakpoint, or running into a bad/fatal VMM condition. The debugger now has
+ * the ping and must respond to the event at hand - the VMM is waiting. This
+ * usually means that the user of the debugger must do something, but it doesn't
+ * have to. The debugger is free to call any DBGF function (nearly at least)
+ * while processing the event.
+ *
+ * Typically the user will issue a request for the execution to be resumed, so
+ * the debugger calls DBGFResume() and goes back to waiting/polling for events.
+ *
+ * When the user eventually terminates the debugging session or selects another
+ * VM, the debugger detaches from the VM. This means that breakpoints are
+ * disabled and that the emulation thread no longer polls for debugger commands.
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/semaphore.h>
+#include <iprt/thread.h>
+#include <iprt/asm.h>
+#include <iprt/time.h>
+#include <iprt/assert.h>
+#include <iprt/stream.h>
+#include <iprt/env.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Instruction type returned by dbgfStepGetCurInstrType.
+ */
+typedef enum DBGFSTEPINSTRTYPE
+{
+    DBGFSTEPINSTRTYPE_INVALID = 0,
+    DBGFSTEPINSTRTYPE_OTHER,
+    DBGFSTEPINSTRTYPE_RET,
+    DBGFSTEPINSTRTYPE_CALL,
+    DBGFSTEPINSTRTYPE_END,
+    DBGFSTEPINSTRTYPE_32BIT_HACK = 0x7fffffff
+} DBGFSTEPINSTRTYPE;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int dbgfR3VMMWait(PVM pVM);
+static int dbgfR3VMMCmd(PVM pVM, DBGFCMD enmCmd, PDBGFCMDDATA pCmdData, bool *pfResumeExecution);
+static DECLCALLBACK(int) dbgfR3Attach(PVM pVM);
+static DBGFSTEPINSTRTYPE dbgfStepGetCurInstrType(PVM pVM, PVMCPU pVCpu);
+static bool dbgfStepAreWeThereYet(PVM pVM,  PVMCPU pVCpu);
+
+
+/**
+ * Sets the VMM Debug Command variable.
+ *
+ * @returns Previous command.
+ * @param   pVM     The cross context VM structure.
+ * @param   enmCmd  The command.
+ */
+DECLINLINE(DBGFCMD) dbgfR3SetCmd(PVM pVM, DBGFCMD enmCmd)
+{
+    DBGFCMD rc;
+    if (enmCmd == DBGFCMD_NO_COMMAND)
+    {
+        Log2(("DBGF: Setting command to %d (DBGFCMD_NO_COMMAND)\n", enmCmd));
+        rc = (DBGFCMD)ASMAtomicXchgU32((uint32_t volatile *)(void *)&pVM->dbgf.s.enmVMMCmd, enmCmd);
+        VM_FF_CLEAR(pVM, VM_FF_DBGF);
+    }
+    else
+    {
+        Log2(("DBGF: Setting command to %d\n", enmCmd));
+        AssertMsg(pVM->dbgf.s.enmVMMCmd == DBGFCMD_NO_COMMAND, ("enmCmd=%d enmVMMCmd=%d\n", enmCmd, pVM->dbgf.s.enmVMMCmd));
+        rc = (DBGFCMD)ASMAtomicXchgU32((uint32_t volatile *)(void *)&pVM->dbgf.s.enmVMMCmd, enmCmd);
+        VM_FF_SET(pVM, VM_FF_DBGF);
+        VMR3NotifyGlobalFFU(pVM->pUVM, 0 /* didn't notify REM */);
+    }
+    return rc;
+}
+
+
+/**
+ * Initializes the DBGF.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) DBGFR3Init(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+    AssertCompile(sizeof(pUVM->dbgf.s)          <= sizeof(pUVM->dbgf.padding));
+    AssertCompile(sizeof(pUVM->aCpus[0].dbgf.s) <= sizeof(pUVM->aCpus[0].dbgf.padding));
+
+    pVM->dbgf.s.SteppingFilter.idCpu = NIL_VMCPUID;
+
+    /*
+     * The usual sideways mountain climbing style of init:
+     */
+    int rc = dbgfR3InfoInit(pUVM); /* (First, initalizes the shared critical section.) */
+    if (RT_SUCCESS(rc))
+    {
+        rc = dbgfR3TraceInit(pVM);
+        if (RT_SUCCESS(rc))
+        {
+            rc = dbgfR3RegInit(pUVM);
+            if (RT_SUCCESS(rc))
+            {
+                rc = dbgfR3AsInit(pUVM);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = dbgfR3BpInit(pVM);
+                    if (RT_SUCCESS(rc))
+                    {
+                        rc = dbgfR3OSInit(pUVM);
+                        if (RT_SUCCESS(rc))
+                        {
+                            rc = dbgfR3PlugInInit(pUVM);
+                            if (RT_SUCCESS(rc))
+                            {
+                                rc = dbgfR3BugCheckInit(pVM);
+                                if (RT_SUCCESS(rc))
+                                {
+                                    return VINF_SUCCESS;
+                                }
+                                dbgfR3PlugInTerm(pUVM);
+                            }
+                            dbgfR3OSTermPart1(pUVM);
+                            dbgfR3OSTermPart2(pUVM);
+                        }
+                    }
+                    dbgfR3AsTerm(pUVM);
+                }
+                dbgfR3RegTerm(pUVM);
+            }
+            dbgfR3TraceTerm(pVM);
+        }
+        dbgfR3InfoTerm(pUVM);
+    }
+    return rc;
+}
+
+
+/**
+ * Terminates and cleans up resources allocated by the DBGF.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) DBGFR3Term(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+
+    dbgfR3OSTermPart1(pUVM);
+    dbgfR3PlugInTerm(pUVM);
+    dbgfR3OSTermPart2(pUVM);
+    dbgfR3AsTerm(pUVM);
+    dbgfR3RegTerm(pUVM);
+    dbgfR3TraceTerm(pVM);
+    dbgfR3InfoTerm(pUVM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Called when the VM is powered off to detach debuggers.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) DBGFR3PowerOff(PVM pVM)
+{
+
+    /*
+     * Send a termination event to any attached debugger.
+     */
+    /* wait to become the speaker (we should already be that). */
+    if (    pVM->dbgf.s.fAttached
+        &&  RTSemPingShouldWait(&pVM->dbgf.s.PingPong))
+        RTSemPingWait(&pVM->dbgf.s.PingPong, 5000);
+
+    if (pVM->dbgf.s.fAttached)
+    {
+        /* Just mark it as detached if we're not in a position to send a power
+           off event.  It should fail later on. */
+        if (!RTSemPingIsSpeaker(&pVM->dbgf.s.PingPong))
+        {
+            ASMAtomicWriteBool(&pVM->dbgf.s.fAttached, false);
+            if (RTSemPingIsSpeaker(&pVM->dbgf.s.PingPong))
+                ASMAtomicWriteBool(&pVM->dbgf.s.fAttached, true);
+        }
+
+        if (RTSemPingIsSpeaker(&pVM->dbgf.s.PingPong))
+        {
+            /* Try send the power off event. */
+            int rc;
+            DBGFCMD enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_NO_COMMAND);
+            if (enmCmd == DBGFCMD_DETACH_DEBUGGER)
+                /* the debugger beat us to initiating the detaching. */
+                rc = VINF_SUCCESS;
+            else
+            {
+                /* ignore the command (if any). */
+                enmCmd = DBGFCMD_NO_COMMAND;
+                pVM->dbgf.s.DbgEvent.enmType = DBGFEVENT_POWERING_OFF;
+                pVM->dbgf.s.DbgEvent.enmCtx  = DBGFEVENTCTX_OTHER;
+                rc = RTSemPing(&pVM->dbgf.s.PingPong);
+            }
+
+            /*
+             * Process commands and priority requests until we get a command
+             * indicating that the debugger has detached.
+             */
+            uint32_t cPollHack = 1;
+            PVMCPU   pVCpu     = VMMGetCpu(pVM);
+            while (RT_SUCCESS(rc))
+            {
+                if (enmCmd != DBGFCMD_NO_COMMAND)
+                {
+                    /* process command */
+                    bool fResumeExecution;
+                    DBGFCMDDATA CmdData = pVM->dbgf.s.VMMCmdData;
+                    rc = dbgfR3VMMCmd(pVM, enmCmd, &CmdData, &fResumeExecution);
+                    if (enmCmd == DBGFCMD_DETACHED_DEBUGGER)
+                        break;
+                    enmCmd = DBGFCMD_NO_COMMAND;
+                }
+                else
+                {
+                    /* Wait for new command, processing pending priority requests
+                       first.  The request processing is a bit crazy, but
+                       unfortunately required by plugin unloading. */
+                    if (   VM_FF_IS_SET(pVM, VM_FF_REQUEST)
+                        || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
+                    {
+                        LogFlow(("DBGFR3PowerOff: Processes priority requests...\n"));
+                        rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY, true /*fPriorityOnly*/);
+                        if (rc == VINF_SUCCESS)
+                            rc = VMR3ReqProcessU(pVM->pUVM, pVCpu->idCpu, true /*fPriorityOnly*/);
+                        LogFlow(("DBGFR3PowerOff: VMR3ReqProcess -> %Rrc\n", rc));
+                        cPollHack = 1;
+                    }
+                    /* Need to handle rendezvous too, for generic debug event management. */
+                    else if (VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
+                    {
+                        rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
+                        AssertLogRel(rc == VINF_SUCCESS);
+                        cPollHack = 1;
+                    }
+                    else if (cPollHack < 120)
+                        cPollHack++;
+
+                    rc = RTSemPingWait(&pVM->dbgf.s.PingPong, cPollHack);
+                    if (RT_SUCCESS(rc))
+                        enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_NO_COMMAND);
+                    else if (rc == VERR_TIMEOUT)
+                        rc = VINF_SUCCESS;
+                }
+            }
+
+            /*
+             * Clear the FF so we won't get confused later on.
+             */
+            VM_FF_CLEAR(pVM, VM_FF_DBGF);
+        }
+    }
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3_INT_DECL(void) DBGFR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    dbgfR3TraceRelocate(pVM);
+    dbgfR3AsRelocate(pVM->pUVM, offDelta);
+}
+
+
+/**
+ * Waits a little while for a debuggger to attach.
+ *
+ * @returns True is a debugger have attached.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context per CPU structure.
+ * @param   enmEvent    Event.
+ *
+ * @thread  EMT(pVCpu)
+ */
+bool dbgfR3WaitForAttach(PVM pVM, PVMCPU pVCpu, DBGFEVENTTYPE enmEvent)
+{
+    /*
+     * First a message.
+     */
+#ifndef RT_OS_L4
+
+# if !defined(DEBUG) || defined(DEBUG_sandervl) || defined(DEBUG_frank)
+    int cWait = 10;
+# else
+    int cWait = !VM_IS_RAW_MODE_ENABLED(pVM)
+             && (   enmEvent == DBGFEVENT_ASSERTION_HYPER
+                 || enmEvent == DBGFEVENT_FATAL_ERROR)
+             && !RTEnvExist("VBOX_DBGF_WAIT_FOR_ATTACH")
+              ? 10
+              : 150;
+# endif
+    RTStrmPrintf(g_pStdErr, "DBGF: No debugger attached, waiting %d second%s for one to attach (event=%d)\n",
+                 cWait / 10, cWait != 10 ? "s" : "", enmEvent);
+    RTStrmFlush(g_pStdErr);
+    while (cWait > 0)
+    {
+        RTThreadSleep(100);
+        if (pVM->dbgf.s.fAttached)
+        {
+            RTStrmPrintf(g_pStdErr, "Attached!\n");
+            RTStrmFlush(g_pStdErr);
+            return true;
+        }
+
+        /* Process priority stuff. */
+        if (   VM_FF_IS_SET(pVM, VM_FF_REQUEST)
+            || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
+        {
+            int rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY, true /*fPriorityOnly*/);
+            if (rc == VINF_SUCCESS)
+                rc = VMR3ReqProcessU(pVM->pUVM, pVCpu->idCpu, true /*fPriorityOnly*/);
+            if (rc != VINF_SUCCESS)
+            {
+                RTStrmPrintf(g_pStdErr, "[rcReq=%Rrc, ignored!]", rc);
+                RTStrmFlush(g_pStdErr);
+            }
+        }
+
+        /* next */
+        if (!(cWait % 10))
+        {
+            RTStrmPrintf(g_pStdErr, "%d.", cWait / 10);
+            RTStrmFlush(g_pStdErr);
+        }
+        cWait--;
+    }
+#endif
+
+    RTStrmPrintf(g_pStdErr, "Stopping the VM!\n");
+    RTStrmFlush(g_pStdErr);
+    return false;
+}
+
+
+/**
+ * Forced action callback.
+ *
+ * The VMM will call this from it's main loop when either VM_FF_DBGF or
+ * VMCPU_FF_DBGF are set.
+ *
+ * The function checks for and executes pending commands from the debugger.
+ * Then it checks for pending debug events and serves these.
+ *
+ * @returns VINF_SUCCESS normally.
+ * @returns VERR_DBGF_RAISE_FATAL_ERROR to pretend a fatal error happened.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context per CPU structure.
+ */
+VMMR3_INT_DECL(int) DBGFR3VMMForcedAction(PVM pVM, PVMCPU pVCpu)
+{
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+
+    if (VM_FF_TEST_AND_CLEAR(pVM, VM_FF_DBGF))
+    {
+        /*
+         * Command pending? Process it.
+         */
+        if (pVM->dbgf.s.enmVMMCmd != DBGFCMD_NO_COMMAND)
+        {
+            bool            fResumeExecution;
+            DBGFCMDDATA     CmdData = pVM->dbgf.s.VMMCmdData;
+            DBGFCMD         enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_NO_COMMAND);
+            rcStrict = dbgfR3VMMCmd(pVM, enmCmd, &CmdData, &fResumeExecution);
+            if (!fResumeExecution)
+                rcStrict = dbgfR3VMMWait(pVM);
+        }
+    }
+
+    /*
+     * Dispatch pending events.
+     */
+    if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_DBGF))
+    {
+        if (   pVCpu->dbgf.s.cEvents > 0
+            && pVCpu->dbgf.s.aEvents[pVCpu->dbgf.s.cEvents - 1].enmState == DBGFEVENTSTATE_CURRENT)
+        {
+            VBOXSTRICTRC rcStrict2 = DBGFR3EventHandlePending(pVM, pVCpu);
+            if (   rcStrict2 != VINF_SUCCESS
+                && (   rcStrict == VINF_SUCCESS
+                    || RT_FAILURE(rcStrict2)
+                    || rcStrict2 < rcStrict) ) /** @todo oversimplified? */
+                rcStrict = rcStrict2;
+        }
+    }
+
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * Flag whether the event implies that we're stopped in the hypervisor code
+ * and have to block certain operations.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   enmEvent    The event.
+ */
+static void dbgfR3EventSetStoppedInHyperFlag(PVM pVM, DBGFEVENTTYPE enmEvent)
+{
+    switch (enmEvent)
+    {
+        case DBGFEVENT_STEPPED_HYPER:
+        case DBGFEVENT_ASSERTION_HYPER:
+        case DBGFEVENT_BREAKPOINT_HYPER:
+            pVM->dbgf.s.fStoppedInHyper = true;
+            break;
+        default:
+            pVM->dbgf.s.fStoppedInHyper = false;
+            break;
+    }
+}
+
+
+/**
+ * Try to determine the event context.
+ *
+ * @returns debug event context.
+ * @param   pVM         The cross context VM structure.
+ */
+static DBGFEVENTCTX dbgfR3FigureEventCtx(PVM pVM)
+{
+    /** @todo SMP support! */
+    PVMCPU pVCpu = pVM->apCpusR3[0];
+
+    switch (EMGetState(pVCpu))
+    {
+        case EMSTATE_RAW:
+        case EMSTATE_DEBUG_GUEST_RAW:
+            return DBGFEVENTCTX_RAW;
+
+        case EMSTATE_REM:
+        case EMSTATE_DEBUG_GUEST_REM:
+            return DBGFEVENTCTX_REM;
+
+        case EMSTATE_DEBUG_HYPER:
+        case EMSTATE_GURU_MEDITATION:
+            return DBGFEVENTCTX_HYPER;
+
+        default:
+            return DBGFEVENTCTX_OTHER;
+    }
+}
+
+/**
+ * The common event prologue code.
+ * It will set the 'stopped-in-hyper' flag, make sure someone is attached,
+ * and perhaps process any high priority pending actions (none yet).
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmEvent    The event to be sent.
+ */
+static int dbgfR3EventPrologue(PVM pVM, DBGFEVENTTYPE enmEvent)
+{
+    /** @todo SMP */
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    /*
+     * Check if a debugger is attached.
+     */
+    if (    !pVM->dbgf.s.fAttached
+        &&  !dbgfR3WaitForAttach(pVM, pVCpu, enmEvent))
+    {
+        Log(("DBGFR3VMMEventSrc: enmEvent=%d - debugger not attached\n", enmEvent));
+        return VERR_DBGF_NOT_ATTACHED;
+    }
+
+    /*
+     * Set flag.
+     */
+    dbgfR3EventSetStoppedInHyperFlag(pVM, enmEvent);
+
+    /*
+     * Look thru pending commands and finish those which make sense now.
+     */
+    /** @todo Process/purge pending commands. */
+    //int rc = DBGFR3VMMForcedAction(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sends the event in the event buffer.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+static int dbgfR3SendEvent(PVM pVM)
+{
+    pVM->dbgf.s.SteppingFilter.idCpu = NIL_VMCPUID;
+
+    int rc = RTSemPing(&pVM->dbgf.s.PingPong);
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3VMMWait(pVM);
+
+    pVM->dbgf.s.fStoppedInHyper = false;
+    /** @todo sync VMM -> REM after exitting the debugger. everything may change while in the debugger! */
+    return rc;
+}
+
+
+/**
+ * Processes a pending event on the current CPU.
+ *
+ * This is called by EM in response to VINF_EM_DBG_EVENT.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context per CPU structure.
+ *
+ * @thread  EMT(pVCpu)
+ */
+VMMR3_INT_DECL(VBOXSTRICTRC) DBGFR3EventHandlePending(PVM pVM, PVMCPU pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_DBGF);
+
+    /*
+     * Check that we've got an event first.
+     */
+    AssertReturn(pVCpu->dbgf.s.cEvents > 0, VINF_SUCCESS);
+    AssertReturn(pVCpu->dbgf.s.aEvents[pVCpu->dbgf.s.cEvents - 1].enmState == DBGFEVENTSTATE_CURRENT, VINF_SUCCESS);
+    PDBGFEVENT pEvent = &pVCpu->dbgf.s.aEvents[pVCpu->dbgf.s.cEvents - 1].Event;
+
+    /*
+     * Make sure we've got a debugger and is allowed to speak to it.
+     */
+    int rc = dbgfR3EventPrologue(pVM, pEvent->enmType);
+    if (RT_FAILURE(rc))
+    {
+        /** @todo drop them events?   */
+        return rc;
+    }
+
+/** @todo SMP + debugger speaker logic  */
+    /*
+     * Copy the event over and mark it as ignore.
+     */
+    pVM->dbgf.s.DbgEvent = *pEvent;
+    pVCpu->dbgf.s.aEvents[pVCpu->dbgf.s.cEvents - 1].enmState = DBGFEVENTSTATE_IGNORE;
+    return dbgfR3SendEvent(pVM);
+}
+
+
+/**
+ * Send a generic debugger event which takes no data.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmEvent    The event to send.
+ * @internal
+ */
+VMMR3DECL(int) DBGFR3Event(PVM pVM, DBGFEVENTTYPE enmEvent)
+{
+    /*
+     * Do stepping filtering.
+     */
+    /** @todo Would be better if we did some of this inside the execution
+     *        engines. */
+    if (   enmEvent == DBGFEVENT_STEPPED
+        || enmEvent == DBGFEVENT_STEPPED_HYPER)
+    {
+        if (!dbgfStepAreWeThereYet(pVM, VMMGetCpu(pVM)))
+            return VINF_EM_DBG_STEP;
+    }
+
+    int rc = dbgfR3EventPrologue(pVM, enmEvent);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Send the event and process the reply communication.
+     */
+    pVM->dbgf.s.DbgEvent.enmType = enmEvent;
+    pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
+    return dbgfR3SendEvent(pVM);
+}
+
+
+/**
+ * Send a debugger event which takes the full source file location.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmEvent    The event to send.
+ * @param   pszFile     Source file.
+ * @param   uLine       Line number in source file.
+ * @param   pszFunction Function name.
+ * @param   pszFormat   Message which accompanies the event.
+ * @param   ...         Message arguments.
+ * @internal
+ */
+VMMR3DECL(int) DBGFR3EventSrc(PVM pVM, DBGFEVENTTYPE enmEvent, const char *pszFile, unsigned uLine, const char *pszFunction, const char *pszFormat, ...)
+{
+    va_list args;
+    va_start(args, pszFormat);
+    int rc = DBGFR3EventSrcV(pVM, enmEvent, pszFile, uLine, pszFunction, pszFormat, args);
+    va_end(args);
+    return rc;
+}
+
+
+/**
+ * Send a debugger event which takes the full source file location.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmEvent    The event to send.
+ * @param   pszFile     Source file.
+ * @param   uLine       Line number in source file.
+ * @param   pszFunction Function name.
+ * @param   pszFormat   Message which accompanies the event.
+ * @param   args        Message arguments.
+ * @internal
+ */
+VMMR3DECL(int) DBGFR3EventSrcV(PVM pVM, DBGFEVENTTYPE enmEvent, const char *pszFile, unsigned uLine, const char *pszFunction, const char *pszFormat, va_list args)
+{
+    int rc = dbgfR3EventPrologue(pVM, enmEvent);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Format the message.
+     */
+    char   *pszMessage = NULL;
+    char    szMessage[8192];
+    if (pszFormat && *pszFormat)
+    {
+        pszMessage = &szMessage[0];
+        RTStrPrintfV(szMessage, sizeof(szMessage), pszFormat, args);
+    }
+
+    /*
+     * Send the event and process the reply communication.
+     */
+    pVM->dbgf.s.DbgEvent.enmType = enmEvent;
+    pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
+    pVM->dbgf.s.DbgEvent.u.Src.pszFile      = pszFile;
+    pVM->dbgf.s.DbgEvent.u.Src.uLine        = uLine;
+    pVM->dbgf.s.DbgEvent.u.Src.pszFunction  = pszFunction;
+    pVM->dbgf.s.DbgEvent.u.Src.pszMessage   = pszMessage;
+    return dbgfR3SendEvent(pVM);
+}
+
+
+/**
+ * Send a debugger event which takes the two assertion messages.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmEvent    The event to send.
+ * @param   pszMsg1     First assertion message.
+ * @param   pszMsg2     Second assertion message.
+ */
+VMMR3_INT_DECL(int) DBGFR3EventAssertion(PVM pVM, DBGFEVENTTYPE enmEvent, const char *pszMsg1, const char *pszMsg2)
+{
+    int rc = dbgfR3EventPrologue(pVM, enmEvent);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Send the event and process the reply communication.
+     */
+    pVM->dbgf.s.DbgEvent.enmType = enmEvent;
+    pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
+    pVM->dbgf.s.DbgEvent.u.Assert.pszMsg1 = pszMsg1;
+    pVM->dbgf.s.DbgEvent.u.Assert.pszMsg2 = pszMsg2;
+    return dbgfR3SendEvent(pVM);
+}
+
+
+/**
+ * Breakpoint was hit somewhere.
+ * Figure out which breakpoint it is and notify the debugger.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmEvent    DBGFEVENT_BREAKPOINT_HYPER or DBGFEVENT_BREAKPOINT.
+ */
+VMMR3_INT_DECL(int) DBGFR3EventBreakpoint(PVM pVM, DBGFEVENTTYPE enmEvent)
+{
+    int rc = dbgfR3EventPrologue(pVM, enmEvent);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Send the event and process the reply communication.
+     */
+    /** @todo SMP */
+    PVMCPU pVCpu = VMMGetCpu0(pVM);
+
+    pVM->dbgf.s.DbgEvent.enmType = enmEvent;
+    RTUINT iBp = pVM->dbgf.s.DbgEvent.u.Bp.iBp = pVCpu->dbgf.s.iActiveBp;
+    pVCpu->dbgf.s.iActiveBp = ~0U;
+    if (iBp != ~0U)
+        pVM->dbgf.s.DbgEvent.enmCtx = DBGFEVENTCTX_RAW;
+    else
+    {
+        /* REM breakpoints has be been searched for. */
+#if 0   /** @todo get flat PC api! */
+        uint32_t eip = CPUMGetGuestEIP(pVM);
+#else
+        /** @todo SMP support!! */
+        PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(VMMGetCpu(pVM));
+        RTGCPTR  eip = pCtx->rip + pCtx->cs.u64Base;
+#endif
+        for (size_t i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints); i++)
+            if (    pVM->dbgf.s.aBreakpoints[i].enmType == DBGFBPTYPE_REM
+                &&  pVM->dbgf.s.aBreakpoints[i].u.Rem.GCPtr == eip)
+            {
+                pVM->dbgf.s.DbgEvent.u.Bp.iBp = pVM->dbgf.s.aBreakpoints[i].iBp;
+                break;
+            }
+        AssertMsg(pVM->dbgf.s.DbgEvent.u.Bp.iBp != ~0U, ("eip=%08x\n", eip));
+        pVM->dbgf.s.DbgEvent.enmCtx = DBGFEVENTCTX_REM;
+    }
+    return dbgfR3SendEvent(pVM);
+}
+
+
+/**
+ * Waits for the debugger to respond.
+ *
+ * @returns VBox status code. (clearify)
+ * @param   pVM     The cross context VM structure.
+ */
+static int dbgfR3VMMWait(PVM pVM)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    LogFlow(("dbgfR3VMMWait:\n"));
+    int rcRet = VINF_SUCCESS;
+
+    /*
+     * Waits for the debugger to reply (i.e. issue an command).
+     */
+    for (;;)
+    {
+        /*
+         * Wait.
+         */
+        uint32_t cPollHack = 1; /** @todo this interface is horrible now that we're using lots of VMR3ReqCall stuff all over DBGF. */
+        for (;;)
+        {
+            int rc;
+            if (    !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_REQUEST)
+                &&  !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
+            {
+                rc = RTSemPingWait(&pVM->dbgf.s.PingPong, cPollHack);
+                if (RT_SUCCESS(rc))
+                    break;
+                if (rc != VERR_TIMEOUT)
+                {
+                    LogFlow(("dbgfR3VMMWait: returns %Rrc\n", rc));
+                    return rc;
+                }
+            }
+
+            if (VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
+            {
+                rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
+                cPollHack = 1;
+            }
+            else if (   VM_FF_IS_SET(pVM, VM_FF_REQUEST)
+                     || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
+            {
+                LogFlow(("dbgfR3VMMWait: Processes requests...\n"));
+                rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY, false /*fPriorityOnly*/);
+                if (rc == VINF_SUCCESS)
+                    rc = VMR3ReqProcessU(pVM->pUVM, pVCpu->idCpu, false /*fPriorityOnly*/);
+                LogFlow(("dbgfR3VMMWait: VMR3ReqProcess -> %Rrc rcRet=%Rrc\n", rc, rcRet));
+                cPollHack = 1;
+            }
+            else
+            {
+                rc = VINF_SUCCESS;
+                if (cPollHack < 120)
+                    cPollHack++;
+            }
+
+            if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
+            {
+                switch (rc)
+                {
+                    case VINF_EM_DBG_BREAKPOINT:
+                    case VINF_EM_DBG_STEPPED:
+                    case VINF_EM_DBG_STEP:
+                    case VINF_EM_DBG_STOP:
+                    case VINF_EM_DBG_EVENT:
+                        AssertMsgFailed(("rc=%Rrc\n", rc));
+                        break;
+
+                    /* return straight away */
+                    case VINF_EM_TERMINATE:
+                    case VINF_EM_OFF:
+                        LogFlow(("dbgfR3VMMWait: returns %Rrc\n", rc));
+                        return rc;
+
+                    /* remember return code. */
+                    default:
+                        AssertReleaseMsgFailed(("rc=%Rrc is not in the switch!\n", rc));
+                        RT_FALL_THRU();
+                    case VINF_EM_RESET:
+                    case VINF_EM_SUSPEND:
+                    case VINF_EM_HALT:
+                    case VINF_EM_RESUME:
+                    case VINF_EM_RESCHEDULE:
+                    case VINF_EM_RESCHEDULE_REM:
+                    case VINF_EM_RESCHEDULE_RAW:
+                        if (rc < rcRet || rcRet == VINF_SUCCESS)
+                            rcRet = rc;
+                        break;
+                }
+            }
+            else if (RT_FAILURE(rc))
+            {
+                LogFlow(("dbgfR3VMMWait: returns %Rrc\n", rc));
+                return rc;
+            }
+        }
+
+        /*
+         * Process the command.
+         */
+        bool            fResumeExecution;
+        DBGFCMDDATA     CmdData = pVM->dbgf.s.VMMCmdData;
+        DBGFCMD         enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_NO_COMMAND);
+        int rc = dbgfR3VMMCmd(pVM, enmCmd, &CmdData, &fResumeExecution);
+        if (fResumeExecution)
+        {
+            if (RT_FAILURE(rc))
+                rcRet = rc;
+            else if (    rc >= VINF_EM_FIRST
+                     &&  rc <= VINF_EM_LAST
+                     &&  (rc < rcRet || rcRet == VINF_SUCCESS))
+                rcRet = rc;
+            LogFlow(("dbgfR3VMMWait: returns %Rrc\n", rcRet));
+            return rcRet;
+        }
+    }
+}
+
+
+/**
+ * Executes command from debugger.
+ *
+ * The caller is responsible for waiting or resuming execution based on the
+ * value returned in the *pfResumeExecution indicator.
+ *
+ * @returns VBox status code. (clearify!)
+ * @param   pVM                 The cross context VM structure.
+ * @param   enmCmd              The command in question.
+ * @param   pCmdData            Pointer to the command data.
+ * @param   pfResumeExecution   Where to store the resume execution / continue waiting indicator.
+ */
+static int dbgfR3VMMCmd(PVM pVM, DBGFCMD enmCmd, PDBGFCMDDATA pCmdData, bool *pfResumeExecution)
+{
+    bool    fSendEvent;
+    bool    fResume;
+    int     rc = VINF_SUCCESS;
+
+    NOREF(pCmdData); /* for later */
+
+    switch (enmCmd)
+    {
+        /*
+         * Halt is answered by an event say that we've halted.
+         */
+        case DBGFCMD_HALT:
+        {
+            pVM->dbgf.s.DbgEvent.enmType = DBGFEVENT_HALT_DONE;
+            pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
+            fSendEvent = true;
+            fResume = false;
+            break;
+        }
+
+
+        /*
+         * Resume is not answered we'll just resume execution.
+         */
+        case DBGFCMD_GO:
+        {
+            /** @todo SMP */
+            PVMCPU pVCpu = VMMGetCpu0(pVM);
+            pVCpu->dbgf.s.fSingleSteppingRaw = false;
+            fSendEvent = false;
+            fResume = true;
+            break;
+        }
+
+        /** @todo implement (and define) the rest of the commands. */
+
+        /*
+         * Disable breakpoints and stuff.
+         * Send an everythings cool event to the debugger thread and resume execution.
+         */
+        case DBGFCMD_DETACH_DEBUGGER:
+        {
+            ASMAtomicWriteBool(&pVM->dbgf.s.fAttached, false);
+            pVM->dbgf.s.DbgEvent.enmType = DBGFEVENT_DETACH_DONE;
+            pVM->dbgf.s.DbgEvent.enmCtx  = DBGFEVENTCTX_OTHER;
+            pVM->dbgf.s.SteppingFilter.idCpu = NIL_VMCPUID;
+            fSendEvent = true;
+            fResume = true;
+            break;
+        }
+
+        /*
+         * The debugger has detached successfully.
+         * There is no reply to this event.
+         */
+        case DBGFCMD_DETACHED_DEBUGGER:
+        {
+            fSendEvent = false;
+            fResume = true;
+            break;
+        }
+
+        /*
+         * Single step, with trace into.
+         */
+        case DBGFCMD_SINGLE_STEP:
+        {
+            Log2(("Single step\n"));
+            /** @todo SMP */
+            PVMCPU pVCpu = VMMGetCpu0(pVM);
+            if (pVM->dbgf.s.SteppingFilter.fFlags & DBGF_STEP_F_OVER)
+            {
+                if (dbgfStepGetCurInstrType(pVM, pVCpu) == DBGFSTEPINSTRTYPE_CALL)
+                    pVM->dbgf.s.SteppingFilter.uCallDepth++;
+            }
+            if (pVM->dbgf.s.SteppingFilter.cMaxSteps > 0)
+            {
+                pVCpu->dbgf.s.fSingleSteppingRaw = true;
+                fSendEvent = false;
+                fResume = true;
+                rc = VINF_EM_DBG_STEP;
+            }
+            else
+            {
+                /* Stop after zero steps. Nonsense, but whatever. */
+                pVM->dbgf.s.SteppingFilter.idCpu = NIL_VMCPUID;
+                pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
+                pVM->dbgf.s.DbgEvent.enmType = pVM->dbgf.s.DbgEvent.enmCtx != DBGFEVENTCTX_HYPER
+                                             ? DBGFEVENT_STEPPED : DBGFEVENT_STEPPED_HYPER;
+                fSendEvent = false;
+                fResume = false;
+            }
+            break;
+        }
+
+        /*
+         * Default is to send an invalid command event.
+         */
+        default:
+        {
+            pVM->dbgf.s.DbgEvent.enmType = DBGFEVENT_INVALID_COMMAND;
+            pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
+            fSendEvent = true;
+            fResume = false;
+            break;
+        }
+    }
+
+    /*
+     * Send pending event.
+     */
+    if (fSendEvent)
+    {
+        Log2(("DBGF: Emulation thread: sending event %d\n", pVM->dbgf.s.DbgEvent.enmType));
+        int rc2 = RTSemPing(&pVM->dbgf.s.PingPong);
+        if (RT_FAILURE(rc2))
+        {
+            AssertRC(rc2);
+            *pfResumeExecution = true;
+            return rc2;
+        }
+    }
+
+    /*
+     * Return.
+     */
+    *pfResumeExecution = fResume;
+    return rc;
+}
+
+
+/**
+ * Attaches a debugger to the specified VM.
+ *
+ * Only one debugger at a time.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(int) DBGFR3Attach(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Call the VM, use EMT for serialization.
+     *
+     * Using a priority call here so we can actually attach a debugger during
+     * the countdown in dbgfR3WaitForAttach.
+     */
+    /** @todo SMP */
+    return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3Attach, 1, pVM);
+}
+
+
+/**
+ * EMT worker for DBGFR3Attach.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+static DECLCALLBACK(int) dbgfR3Attach(PVM pVM)
+{
+    if (pVM->dbgf.s.fAttached)
+    {
+        Log(("dbgR3Attach: Debugger already attached\n"));
+        return VERR_DBGF_ALREADY_ATTACHED;
+    }
+
+    /*
+     * Create the Ping-Pong structure.
+     */
+    int rc = RTSemPingPongInit(&pVM->dbgf.s.PingPong);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Set the attached flag.
+     */
+    ASMAtomicWriteBool(&pVM->dbgf.s.fAttached, true);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Detaches a debugger from the specified VM.
+ *
+ * Caller must be attached to the VM.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(int) DBGFR3Detach(PUVM pUVM)
+{
+    LogFlow(("DBGFR3Detach:\n"));
+    int rc;
+
+    /*
+     * Validate input. The UVM handle shall be valid, the VM handle might be
+     * in the processes of being destroyed already, so deal quietly with that.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    if (!VM_IS_VALID_EXT(pVM))
+        return VERR_INVALID_VM_HANDLE;
+
+    /*
+     * Check if attached.
+     */
+    if (!pVM->dbgf.s.fAttached)
+        return VERR_DBGF_NOT_ATTACHED;
+
+    /*
+     * Try send the detach command.
+     * Keep in mind that we might be racing EMT, so, be extra careful.
+     */
+    DBGFCMD enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_DETACH_DEBUGGER);
+    if (RTSemPongIsSpeaker(&pVM->dbgf.s.PingPong))
+    {
+        rc = RTSemPong(&pVM->dbgf.s.PingPong);
+        AssertMsgRCReturn(rc, ("Failed to signal emulation thread. rc=%Rrc\n", rc), rc);
+        LogRel(("DBGFR3Detach: enmCmd=%d (pong -> ping)\n", enmCmd));
+    }
+
+    /*
+     * Wait for the OK event.
+     */
+    rc = RTSemPongWait(&pVM->dbgf.s.PingPong, RT_INDEFINITE_WAIT);
+    AssertLogRelMsgRCReturn(rc, ("Wait on detach command failed, rc=%Rrc\n", rc), rc);
+
+    /*
+     * Send the notification command indicating that we're really done.
+     */
+    enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_DETACHED_DEBUGGER);
+    rc = RTSemPong(&pVM->dbgf.s.PingPong);
+    AssertMsgRCReturn(rc, ("Failed to signal emulation thread. rc=%Rrc\n", rc), rc);
+
+    LogFlowFunc(("returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Wait for a debug event.
+ *
+ * @returns VBox status code. Will not return VBOX_INTERRUPTED.
+ * @param   pUVM        The user mode VM handle.
+ * @param   cMillies    Number of millis to wait.
+ * @param   ppEvent     Where to store the event pointer.
+ */
+VMMR3DECL(int) DBGFR3EventWait(PUVM pUVM, RTMSINTERVAL cMillies, PCDBGFEVENT *ppEvent)
+{
+    /*
+     * Check state.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
+    *ppEvent = NULL;
+
+    /*
+     * Wait.
+     */
+    int rc = RTSemPongWait(&pVM->dbgf.s.PingPong, cMillies);
+    if (RT_SUCCESS(rc))
+    {
+        *ppEvent = &pVM->dbgf.s.DbgEvent;
+        Log2(("DBGF: Debugger thread: receiving event %d\n", (*ppEvent)->enmType));
+        return VINF_SUCCESS;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Halts VM execution.
+ *
+ * After calling this the VM isn't actually halted till an DBGFEVENT_HALT_DONE
+ * arrives. Until that time it's not possible to issue any new commands.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(int) DBGFR3Halt(PUVM pUVM)
+{
+    /*
+     * Check state.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
+    RTPINGPONGSPEAKER enmSpeaker = pVM->dbgf.s.PingPong.enmSpeaker;
+    if (   enmSpeaker == RTPINGPONGSPEAKER_PONG
+        || enmSpeaker == RTPINGPONGSPEAKER_PONG_SIGNALED)
+        return VWRN_DBGF_ALREADY_HALTED;
+
+    /*
+     * Send command.
+     */
+    dbgfR3SetCmd(pVM, DBGFCMD_HALT);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the VM is halted by the debugger.
+ *
+ * @returns True if halted.
+ * @returns False if not halted.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(bool) DBGFR3IsHalted(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    AssertReturn(pVM->dbgf.s.fAttached, false);
+
+    RTPINGPONGSPEAKER enmSpeaker = pVM->dbgf.s.PingPong.enmSpeaker;
+    return enmSpeaker == RTPINGPONGSPEAKER_PONG_SIGNALED
+        || enmSpeaker == RTPINGPONGSPEAKER_PONG;
+}
+
+
+/**
+ * Checks if the debugger can wait for events or not.
+ *
+ * This function is only used by lazy, multiplexing debuggers. :-)
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if waitable.
+ * @retval  VERR_SEM_OUT_OF_TURN if not waitable.
+ * @retval  VERR_INVALID_VM_HANDLE if the VM is being (/ has been) destroyed
+ *          (not asserted) or if the handle is invalid (asserted).
+ * @retval  VERR_DBGF_NOT_ATTACHED if not attached.
+ *
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(int) DBGFR3QueryWaitable(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /* Note! There is a slight race here, unfortunately. */
+    PVM pVM = pUVM->pVM;
+    if (!RT_VALID_PTR(pVM))
+        return VERR_INVALID_VM_HANDLE;
+    if (pVM->enmVMState >= VMSTATE_DESTROYING)
+        return VERR_INVALID_VM_HANDLE;
+    if (!pVM->dbgf.s.fAttached)
+        return VERR_DBGF_NOT_ATTACHED;
+
+    if (!RTSemPongShouldWait(&pVM->dbgf.s.PingPong))
+        return VERR_SEM_OUT_OF_TURN;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resumes VM execution.
+ *
+ * There is no receipt event on this command.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(int) DBGFR3Resume(PUVM pUVM)
+{
+    /*
+     * Check state.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
+    if (RT_LIKELY(RTSemPongIsSpeaker(&pVM->dbgf.s.PingPong)))
+    { /* likely */ }
+    else
+        return VERR_SEM_OUT_OF_TURN;
+
+    /*
+     * Send the ping back to the emulation thread telling it to run.
+     */
+    dbgfR3SetCmd(pVM, DBGFCMD_GO);
+    int rc = RTSemPong(&pVM->dbgf.s.PingPong);
+    AssertRC(rc);
+
+    return rc;
+}
+
+
+/**
+ * Classifies the current instruction.
+ *
+ * @returns Type of instruction.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The current CPU.
+ * @thread  EMT(pVCpu)
+ */
+static DBGFSTEPINSTRTYPE dbgfStepGetCurInstrType(PVM pVM, PVMCPU pVCpu)
+{
+    /*
+     * Read the instruction.
+     */
+    size_t   cbRead   = 0;
+    uint8_t  abOpcode[16] = { 0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0 };
+    int rc = PGMR3DbgReadGCPtr(pVM, abOpcode, CPUMGetGuestFlatPC(pVCpu), sizeof(abOpcode) - 1, 0 /*fFlags*/, &cbRead);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Do minimal parsing.  No real need to involve the disassembler here.
+         */
+        uint8_t *pb = abOpcode;
+        for (;;)
+        {
+            switch (*pb++)
+            {
+                default:
+                    return DBGFSTEPINSTRTYPE_OTHER;
+
+                case 0xe8: /* call rel16/32 */
+                case 0x9a: /* call farptr */
+                case 0xcc: /* int3 */
+                case 0xcd: /* int xx */
+                // case 0xce: /* into */
+                    return DBGFSTEPINSTRTYPE_CALL;
+
+                case 0xc2: /* ret xx */
+                case 0xc3: /* ret */
+                case 0xca: /* retf xx */
+                case 0xcb: /* retf */
+                case 0xcf: /* iret */
+                    return DBGFSTEPINSTRTYPE_RET;
+
+                case 0xff:
+                    if (   ((*pb >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) == 2  /* call indir */
+                        || ((*pb >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) == 3) /* call indir-farptr */
+                        return DBGFSTEPINSTRTYPE_CALL;
+                    return DBGFSTEPINSTRTYPE_OTHER;
+
+                case 0x0f:
+                    switch (*pb++)
+                    {
+                        case 0x05: /* syscall */
+                        case 0x34: /* sysenter */
+                            return DBGFSTEPINSTRTYPE_CALL;
+                        case 0x07: /* sysret */
+                        case 0x35: /* sysexit */
+                            return DBGFSTEPINSTRTYPE_RET;
+                    }
+                    break;
+
+                /* Must handle some REX prefixes. So we do all normal prefixes. */
+                case 0x40: case 0x41: case 0x42: case 0x43: case 0x44: case 0x45: case 0x46: case 0x47:
+                case 0x48: case 0x49: case 0x4a: case 0x4b: case 0x4c: case 0x4d: case 0x4e: case 0x4f:
+                    if (!CPUMIsGuestIn64BitCode(pVCpu))
+                        return DBGFSTEPINSTRTYPE_OTHER;
+                    break;
+
+                case 0x2e: /* CS */
+                case 0x36: /* SS */
+                case 0x3e: /* DS */
+                case 0x26: /* ES */
+                case 0x64: /* FS */
+                case 0x65: /* GS */
+                case 0x66: /* op size */
+                case 0x67: /* addr size */
+                case 0xf0: /* lock */
+                case 0xf2: /* REPNZ */
+                case 0xf3: /* REPZ */
+                    break;
+            }
+        }
+    }
+
+    return DBGFSTEPINSTRTYPE_INVALID;
+}
+
+
+/**
+ * Checks if the stepping has reached a stop point.
+ *
+ * Called when raising a stepped event.
+ *
+ * @returns true if the event should be raised, false if we should take one more
+ *          step first.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context per CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+static bool dbgfStepAreWeThereYet(PVM pVM, PVMCPU pVCpu)
+{
+    /*
+     * Check valid pVCpu and that it matches the CPU one stepping.
+     */
+    if (pVCpu)
+    {
+        if (pVCpu->idCpu == pVM->dbgf.s.SteppingFilter.idCpu)
+        {
+            /*
+             * Increase the number of steps and see if we've reached the max.
+             */
+            pVM->dbgf.s.SteppingFilter.cSteps++;
+            if (pVM->dbgf.s.SteppingFilter.cSteps < pVM->dbgf.s.SteppingFilter.cMaxSteps)
+            {
+                /*
+                 * Check PC and SP address filtering.
+                 */
+                if (pVM->dbgf.s.SteppingFilter.fFlags & (DBGF_STEP_F_STOP_ON_ADDRESS | DBGF_STEP_F_STOP_ON_STACK_POP))
+                {
+                    if (   (pVM->dbgf.s.SteppingFilter.fFlags & DBGF_STEP_F_STOP_ON_ADDRESS)
+                        && pVM->dbgf.s.SteppingFilter.AddrPc == CPUMGetGuestFlatPC(pVCpu))
+                        return true;
+                    if (   (pVM->dbgf.s.SteppingFilter.fFlags & DBGF_STEP_F_STOP_ON_STACK_POP)
+                        &&   CPUMGetGuestFlatSP(pVCpu) - pVM->dbgf.s.SteppingFilter.AddrStackPop
+                           < pVM->dbgf.s.SteppingFilter.cbStackPop)
+                        return true;
+                }
+
+                /*
+                 * Do step-over filtering separate from the step-into one.
+                 */
+                if (pVM->dbgf.s.SteppingFilter.fFlags & DBGF_STEP_F_OVER)
+                {
+                    DBGFSTEPINSTRTYPE enmType = dbgfStepGetCurInstrType(pVM, pVCpu);
+                    switch (enmType)
+                    {
+                        default:
+                            if (   pVM->dbgf.s.SteppingFilter.uCallDepth != 0
+                                || (pVM->dbgf.s.SteppingFilter.fFlags & DBGF_STEP_F_STOP_FILTER_MASK))
+                                break;
+                            return true;
+                        case DBGFSTEPINSTRTYPE_CALL:
+                            if (   (pVM->dbgf.s.SteppingFilter.fFlags & DBGF_STEP_F_STOP_ON_CALL)
+                                && pVM->dbgf.s.SteppingFilter.uCallDepth == 0)
+                                return true;
+                            pVM->dbgf.s.SteppingFilter.uCallDepth++;
+                            break;
+                        case DBGFSTEPINSTRTYPE_RET:
+                            if (pVM->dbgf.s.SteppingFilter.uCallDepth == 0)
+                            {
+                                if (pVM->dbgf.s.SteppingFilter.fFlags & DBGF_STEP_F_STOP_ON_RET)
+                                    return true;
+                                /* If after return, we use the cMaxStep limit to stop the next time. */
+                                if (pVM->dbgf.s.SteppingFilter.fFlags & DBGF_STEP_F_STOP_AFTER_RET)
+                                    pVM->dbgf.s.SteppingFilter.cMaxSteps = pVM->dbgf.s.SteppingFilter.cSteps + 1;
+                            }
+                            else if (pVM->dbgf.s.SteppingFilter.uCallDepth > 0)
+                                pVM->dbgf.s.SteppingFilter.uCallDepth--;
+                            break;
+                    }
+                    return false;
+                }
+                /*
+                 * Filtered step-into.
+                 */
+                else if (  pVM->dbgf.s.SteppingFilter.fFlags
+                         & (DBGF_STEP_F_STOP_ON_CALL | DBGF_STEP_F_STOP_ON_RET | DBGF_STEP_F_STOP_AFTER_RET))
+                {
+                    DBGFSTEPINSTRTYPE enmType = dbgfStepGetCurInstrType(pVM, pVCpu);
+                    switch (enmType)
+                    {
+                        default:
+                            break;
+                        case DBGFSTEPINSTRTYPE_CALL:
+                            if (pVM->dbgf.s.SteppingFilter.fFlags & DBGF_STEP_F_STOP_ON_CALL)
+                                return true;
+                            break;
+                        case DBGFSTEPINSTRTYPE_RET:
+                            if (pVM->dbgf.s.SteppingFilter.fFlags & DBGF_STEP_F_STOP_ON_RET)
+                                return true;
+                            /* If after return, we use the cMaxStep limit to stop the next time. */
+                            if (pVM->dbgf.s.SteppingFilter.fFlags & DBGF_STEP_F_STOP_AFTER_RET)
+                                pVM->dbgf.s.SteppingFilter.cMaxSteps = pVM->dbgf.s.SteppingFilter.cSteps + 1;
+                            break;
+                    }
+                    return false;
+                }
+            }
+        }
+    }
+
+    return true;
+}
+
+
+/**
+ * Step Into.
+ *
+ * A single step event is generated from this command.
+ * The current implementation is not reliable, so don't rely on the event coming.
+ *
+ * @returns VBox status code.
+ * @param   pUVM    The user mode VM handle.
+ * @param   idCpu   The ID of the CPU to single step on.
+ */
+VMMR3DECL(int) DBGFR3Step(PUVM pUVM, VMCPUID idCpu)
+{
+    return DBGFR3StepEx(pUVM, idCpu, DBGF_STEP_F_INTO, NULL, NULL, 0, 1);
+}
+
+
+/**
+ * Full fleged step.
+ *
+ * This extended stepping API allows for doing multiple steps before raising an
+ * event, helping implementing step over, step out and other more advanced
+ * features.
+ *
+ * Like the DBGFR3Step() API, this will normally generate a DBGFEVENT_STEPPED or
+ * DBGFEVENT_STEPPED_EVENT.  However the stepping may be interrupted by other
+ * events, which will abort the stepping.
+ *
+ * The stop on pop area feature is for safeguarding step out.
+ *
+ * Please note though, that it will always use stepping and never breakpoints.
+ * While this allows for a much greater flexibility it can at times be rather
+ * slow.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   idCpu           The ID of the CPU to single step on.
+ * @param   fFlags          Flags controlling the stepping, DBGF_STEP_F_XXX.
+ *                          Either DBGF_STEP_F_INTO or DBGF_STEP_F_OVER must
+ *                          always be specified.
+ * @param   pStopPcAddr     Address to stop executing at.  Completely ignored
+ *                          unless DBGF_STEP_F_STOP_ON_ADDRESS is specified.
+ * @param   pStopPopAddr    Stack address that SP must be lower than when
+ *                          performing DBGF_STEP_F_STOP_ON_STACK_POP filtering.
+ * @param   cbStopPop       The range starting at @a pStopPopAddr which is
+ *                          considered to be within the same thread stack. Note
+ *                          that the API allows @a pStopPopAddr and @a cbStopPop
+ *                          to form an area that wraps around and it will
+ *                          consider the part starting at 0 as included.
+ * @param   cMaxSteps       The maximum number of steps to take.  This is to
+ *                          prevent stepping for ever, so passing UINT32_MAX is
+ *                          not recommended.
+ *
+ * @remarks The two address arguments must be guest context virtual addresses,
+ *          or HMA.  The code doesn't make much of a point of out HMA, though.
+ */
+VMMR3DECL(int) DBGFR3StepEx(PUVM pUVM, VMCPUID idCpu, uint32_t fFlags, PCDBGFADDRESS pStopPcAddr,
+                            PCDBGFADDRESS pStopPopAddr, RTGCUINTPTR cbStopPop, uint32_t cMaxSteps)
+{
+    /*
+     * Check state.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
+    AssertReturn(!(fFlags & ~DBGF_STEP_F_VALID_MASK), VERR_INVALID_FLAGS);
+    AssertReturn(RT_BOOL(fFlags & DBGF_STEP_F_INTO) != RT_BOOL(fFlags & DBGF_STEP_F_OVER), VERR_INVALID_FLAGS);
+    if (fFlags & DBGF_STEP_F_STOP_ON_ADDRESS)
+    {
+        AssertReturn(RT_VALID_PTR(pStopPcAddr), VERR_INVALID_POINTER);
+        AssertReturn(DBGFADDRESS_IS_VALID(pStopPcAddr), VERR_INVALID_PARAMETER);
+        AssertReturn(DBGFADDRESS_IS_VIRT_GC(pStopPcAddr), VERR_INVALID_PARAMETER);
+    }
+    AssertReturn(!(fFlags & DBGF_STEP_F_STOP_ON_STACK_POP) || RT_VALID_PTR(pStopPopAddr), VERR_INVALID_POINTER);
+    if (fFlags & DBGF_STEP_F_STOP_ON_STACK_POP)
+    {
+        AssertReturn(RT_VALID_PTR(pStopPopAddr), VERR_INVALID_POINTER);
+        AssertReturn(DBGFADDRESS_IS_VALID(pStopPopAddr), VERR_INVALID_PARAMETER);
+        AssertReturn(DBGFADDRESS_IS_VIRT_GC(pStopPopAddr), VERR_INVALID_PARAMETER);
+        AssertReturn(cbStopPop > 0, VERR_INVALID_PARAMETER);
+    }
+
+    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
+    if (RT_LIKELY(RTSemPongIsSpeaker(&pVM->dbgf.s.PingPong)))
+    { /* likely */ }
+    else
+        return VERR_SEM_OUT_OF_TURN;
+    Assert(pVM->dbgf.s.SteppingFilter.idCpu == NIL_VMCPUID);
+
+    /*
+     * Send the ping back to the emulation thread telling it to run.
+     */
+    if (fFlags == DBGF_STEP_F_INTO)
+        pVM->dbgf.s.SteppingFilter.idCpu = NIL_VMCPUID;
+    else
+        pVM->dbgf.s.SteppingFilter.idCpu = idCpu;
+    pVM->dbgf.s.SteppingFilter.fFlags = fFlags;
+    if (fFlags & DBGF_STEP_F_STOP_ON_ADDRESS)
+        pVM->dbgf.s.SteppingFilter.AddrPc = pStopPcAddr->FlatPtr;
+    else
+        pVM->dbgf.s.SteppingFilter.AddrPc = 0;
+    if (fFlags & DBGF_STEP_F_STOP_ON_STACK_POP)
+    {
+        pVM->dbgf.s.SteppingFilter.AddrStackPop = pStopPopAddr->FlatPtr;
+        pVM->dbgf.s.SteppingFilter.cbStackPop   = cbStopPop;
+    }
+    else
+    {
+        pVM->dbgf.s.SteppingFilter.AddrStackPop = 0;
+        pVM->dbgf.s.SteppingFilter.cbStackPop   = RTGCPTR_MAX;
+    }
+
+    pVM->dbgf.s.SteppingFilter.cMaxSteps    = cMaxSteps;
+    pVM->dbgf.s.SteppingFilter.cSteps       = 0;
+    pVM->dbgf.s.SteppingFilter.uCallDepth   = 0;
+
+/** @todo SMP (idCpu) */
+    dbgfR3SetCmd(pVM, DBGFCMD_SINGLE_STEP);
+    int rc = RTSemPong(&pVM->dbgf.s.PingPong);
+    AssertRC(rc);
+    return rc;
+}
+
+
+
+/**
+ * dbgfR3EventConfigEx argument packet.
+ */
+typedef struct DBGFR3EVENTCONFIGEXARGS
+{
+    PCDBGFEVENTCONFIG   paConfigs;
+    size_t              cConfigs;
+    int                 rc;
+} DBGFR3EVENTCONFIGEXARGS;
+/** Pointer to a dbgfR3EventConfigEx argument packet. */
+typedef DBGFR3EVENTCONFIGEXARGS *PDBGFR3EVENTCONFIGEXARGS;
+
+
+/**
+ * @callback_method_impl{FNVMMEMTRENDEZVOUS, Worker for DBGFR3EventConfigEx.}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) dbgfR3EventConfigEx(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    if (pVCpu->idCpu == 0)
+    {
+        PDBGFR3EVENTCONFIGEXARGS        pArgs = (PDBGFR3EVENTCONFIGEXARGS)pvUser;
+        DBGFEVENTCONFIG volatile const *paConfigs = pArgs->paConfigs;
+        size_t                          cConfigs  = pArgs->cConfigs;
+
+        /*
+         * Apply the changes.
+         */
+        unsigned cChanges = 0;
+        for (uint32_t i = 0; i < cConfigs; i++)
+        {
+            DBGFEVENTTYPE enmType = paConfigs[i].enmType;
+            AssertReturn(enmType >= DBGFEVENT_FIRST_SELECTABLE && enmType < DBGFEVENT_END, VERR_INVALID_PARAMETER);
+            if (paConfigs[i].fEnabled)
+                cChanges += ASMAtomicBitTestAndSet(&pVM->dbgf.s.bmSelectedEvents, enmType) == false;
+            else
+                cChanges += ASMAtomicBitTestAndClear(&pVM->dbgf.s.bmSelectedEvents, enmType) == true;
+        }
+
+        /*
+         * Inform HM about changes.
+         */
+        if (cChanges > 0 && HMIsEnabled(pVM))
+        {
+            HMR3NotifyDebugEventChanged(pVM);
+            HMR3NotifyDebugEventChangedPerCpu(pVM, pVCpu);
+        }
+    }
+    else if (HMIsEnabled(pVM))
+        HMR3NotifyDebugEventChangedPerCpu(pVM, pVCpu);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Configures (enables/disables) multiple selectable debug events.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   paConfigs   The event to configure and their new state.
+ * @param   cConfigs    Number of entries in @a paConfigs.
+ */
+VMMR3DECL(int) DBGFR3EventConfigEx(PUVM pUVM, PCDBGFEVENTCONFIG paConfigs, size_t cConfigs)
+{
+    /*
+     * Validate input.
+     */
+    size_t i = cConfigs;
+    while (i-- > 0)
+    {
+        AssertReturn(paConfigs[i].enmType >= DBGFEVENT_FIRST_SELECTABLE, VERR_INVALID_PARAMETER);
+        AssertReturn(paConfigs[i].enmType <  DBGFEVENT_END, VERR_INVALID_PARAMETER);
+    }
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Apply the changes in EMT(0) and rendezvous with the other CPUs so they
+     * can sync their data and execution with new debug state.
+     */
+    DBGFR3EVENTCONFIGEXARGS Args = { paConfigs, cConfigs, VINF_SUCCESS };
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING | VMMEMTRENDEZVOUS_FLAGS_PRIORITY,
+                                dbgfR3EventConfigEx, &Args);
+    if (RT_SUCCESS(rc))
+        rc = Args.rc;
+    return rc;
+}
+
+
+/**
+ * Enables or disables a selectable debug event.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   enmEvent    The selectable debug event.
+ * @param   fEnabled    The new state.
+ */
+VMMR3DECL(int) DBGFR3EventConfig(PUVM pUVM, DBGFEVENTTYPE enmEvent, bool fEnabled)
+{
+    /*
+     * Convert to an array call.
+     */
+    DBGFEVENTCONFIG EvtCfg = { enmEvent, fEnabled };
+    return DBGFR3EventConfigEx(pUVM, &EvtCfg, 1);
+}
+
+
+/**
+ * Checks if the given selectable event is enabled.
+ *
+ * @returns true if enabled, false if not or invalid input.
+ * @param   pUVM        The user mode VM handle.
+ * @param   enmEvent    The selectable debug event.
+ * @sa      DBGFR3EventQuery
+ */
+VMMR3DECL(bool) DBGFR3EventIsEnabled(PUVM pUVM, DBGFEVENTTYPE enmEvent)
+{
+    /*
+     * Validate input.
+     */
+    AssertReturn(   enmEvent >= DBGFEVENT_HALT_DONE
+                 && enmEvent <  DBGFEVENT_END, false);
+    Assert(   enmEvent >= DBGFEVENT_FIRST_SELECTABLE
+           || enmEvent == DBGFEVENT_BREAKPOINT
+           || enmEvent == DBGFEVENT_BREAKPOINT_IO
+           || enmEvent == DBGFEVENT_BREAKPOINT_MMIO);
+
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+
+    /*
+     * Check the event status.
+     */
+    return ASMBitTest(&pVM->dbgf.s.bmSelectedEvents, enmEvent);
+}
+
+
+/**
+ * Queries the status of a set of events.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   paConfigs   The events to query and where to return the state.
+ * @param   cConfigs    The number of elements in @a paConfigs.
+ * @sa      DBGFR3EventIsEnabled, DBGF_IS_EVENT_ENABLED
+ */
+VMMR3DECL(int) DBGFR3EventQuery(PUVM pUVM, PDBGFEVENTCONFIG paConfigs, size_t cConfigs)
+{
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    for (size_t i = 0; i < cConfigs; i++)
+    {
+        DBGFEVENTTYPE enmType = paConfigs[i].enmType;
+        AssertReturn(   enmType >= DBGFEVENT_HALT_DONE
+                     && enmType <  DBGFEVENT_END, VERR_INVALID_PARAMETER);
+        Assert(   enmType >= DBGFEVENT_FIRST_SELECTABLE
+               || enmType == DBGFEVENT_BREAKPOINT
+               || enmType == DBGFEVENT_BREAKPOINT_IO
+               || enmType == DBGFEVENT_BREAKPOINT_MMIO);
+        paConfigs[i].fEnabled = ASMBitTest(&pVM->dbgf.s.bmSelectedEvents, paConfigs[i].enmType);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * dbgfR3InterruptConfigEx argument packet.
+ */
+typedef struct DBGFR3INTERRUPTCONFIGEXARGS
+{
+    PCDBGFINTERRUPTCONFIG   paConfigs;
+    size_t                  cConfigs;
+    int                     rc;
+} DBGFR3INTERRUPTCONFIGEXARGS;
+/** Pointer to a dbgfR3InterruptConfigEx argument packet. */
+typedef DBGFR3INTERRUPTCONFIGEXARGS *PDBGFR3INTERRUPTCONFIGEXARGS;
+
+/**
+ * @callback_method_impl{FNVMMEMTRENDEZVOUS,
+ *      Worker for DBGFR3InterruptConfigEx.}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) dbgfR3InterruptConfigEx(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    if (pVCpu->idCpu == 0)
+    {
+        PDBGFR3INTERRUPTCONFIGEXARGS    pArgs = (PDBGFR3INTERRUPTCONFIGEXARGS)pvUser;
+        PCDBGFINTERRUPTCONFIG           paConfigs = pArgs->paConfigs;
+        size_t                          cConfigs  = pArgs->cConfigs;
+
+        /*
+         * Apply the changes.
+         */
+        bool fChanged = false;
+        bool fThis;
+        for (uint32_t i = 0; i < cConfigs; i++)
+        {
+            /*
+             * Hardware interrupts.
+             */
+            if (paConfigs[i].enmHardState == DBGFINTERRUPTSTATE_ENABLED)
+            {
+                fChanged |= fThis = ASMAtomicBitTestAndSet(&pVM->dbgf.s.bmHardIntBreakpoints, paConfigs[i].iInterrupt) == false;
+                if (fThis)
+                {
+                    Assert(pVM->dbgf.s.cHardIntBreakpoints < 256);
+                    pVM->dbgf.s.cHardIntBreakpoints++;
+                }
+            }
+            else if (paConfigs[i].enmHardState == DBGFINTERRUPTSTATE_DISABLED)
+            {
+                fChanged |= fThis = ASMAtomicBitTestAndClear(&pVM->dbgf.s.bmHardIntBreakpoints, paConfigs[i].iInterrupt) == true;
+                if (fThis)
+                {
+                    Assert(pVM->dbgf.s.cHardIntBreakpoints > 0);
+                    pVM->dbgf.s.cHardIntBreakpoints--;
+                }
+            }
+
+            /*
+             * Software interrupts.
+             */
+            if (paConfigs[i].enmHardState == DBGFINTERRUPTSTATE_ENABLED)
+            {
+                fChanged |= fThis = ASMAtomicBitTestAndSet(&pVM->dbgf.s.bmSoftIntBreakpoints, paConfigs[i].iInterrupt) == false;
+                if (fThis)
+                {
+                    Assert(pVM->dbgf.s.cSoftIntBreakpoints < 256);
+                    pVM->dbgf.s.cSoftIntBreakpoints++;
+                }
+            }
+            else if (paConfigs[i].enmSoftState == DBGFINTERRUPTSTATE_DISABLED)
+            {
+                fChanged |= fThis = ASMAtomicBitTestAndClear(&pVM->dbgf.s.bmSoftIntBreakpoints, paConfigs[i].iInterrupt) == true;
+                if (fThis)
+                {
+                    Assert(pVM->dbgf.s.cSoftIntBreakpoints > 0);
+                    pVM->dbgf.s.cSoftIntBreakpoints--;
+                }
+            }
+        }
+
+        /*
+         * Update the event bitmap entries.
+         */
+        if (pVM->dbgf.s.cHardIntBreakpoints > 0)
+            fChanged |= ASMAtomicBitTestAndSet(&pVM->dbgf.s.bmSelectedEvents, DBGFEVENT_INTERRUPT_HARDWARE) == false;
+        else
+            fChanged |= ASMAtomicBitTestAndClear(&pVM->dbgf.s.bmSelectedEvents, DBGFEVENT_INTERRUPT_HARDWARE) == true;
+
+        if (pVM->dbgf.s.cSoftIntBreakpoints > 0)
+            fChanged |= ASMAtomicBitTestAndSet(&pVM->dbgf.s.bmSelectedEvents, DBGFEVENT_INTERRUPT_SOFTWARE) == false;
+        else
+            fChanged |= ASMAtomicBitTestAndClear(&pVM->dbgf.s.bmSelectedEvents, DBGFEVENT_INTERRUPT_SOFTWARE) == true;
+
+        /*
+         * Inform HM about changes.
+         */
+        if (fChanged && HMIsEnabled(pVM))
+        {
+            HMR3NotifyDebugEventChanged(pVM);
+            HMR3NotifyDebugEventChangedPerCpu(pVM, pVCpu);
+        }
+    }
+    else if (HMIsEnabled(pVM))
+        HMR3NotifyDebugEventChangedPerCpu(pVM, pVCpu);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Changes
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   paConfigs   The events to query and where to return the state.
+ * @param   cConfigs    The number of elements in @a paConfigs.
+ * @sa      DBGFR3InterruptConfigHardware, DBGFR3InterruptConfigSoftware
+ */
+VMMR3DECL(int) DBGFR3InterruptConfigEx(PUVM pUVM, PCDBGFINTERRUPTCONFIG paConfigs, size_t cConfigs)
+{
+    /*
+     * Validate input.
+     */
+    size_t i = cConfigs;
+    while (i-- > 0)
+    {
+        AssertReturn(paConfigs[i].enmHardState <= DBGFINTERRUPTSTATE_DONT_TOUCH, VERR_INVALID_PARAMETER);
+        AssertReturn(paConfigs[i].enmSoftState <= DBGFINTERRUPTSTATE_DONT_TOUCH, VERR_INVALID_PARAMETER);
+    }
+
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Apply the changes in EMT(0) and rendezvous with the other CPUs so they
+     * can sync their data and execution with new debug state.
+     */
+    DBGFR3INTERRUPTCONFIGEXARGS Args = { paConfigs, cConfigs, VINF_SUCCESS };
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING | VMMEMTRENDEZVOUS_FLAGS_PRIORITY,
+                                dbgfR3InterruptConfigEx, &Args);
+    if (RT_SUCCESS(rc))
+        rc = Args.rc;
+    return rc;
+}
+
+
+/**
+ * Configures interception of a hardware interrupt.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   iInterrupt  The interrupt number.
+ * @param   fEnabled    Whether interception is enabled or not.
+ * @sa      DBGFR3InterruptSoftwareConfig, DBGFR3InterruptConfigEx
+ */
+VMMR3DECL(int) DBGFR3InterruptHardwareConfig(PUVM pUVM, uint8_t iInterrupt, bool fEnabled)
+{
+    /*
+     * Convert to DBGFR3InterruptConfigEx call.
+     */
+    DBGFINTERRUPTCONFIG IntCfg = { iInterrupt, (uint8_t)fEnabled, DBGFINTERRUPTSTATE_DONT_TOUCH };
+    return DBGFR3InterruptConfigEx(pUVM, &IntCfg, 1);
+}
+
+
+/**
+ * Configures interception of a software interrupt.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   iInterrupt  The interrupt number.
+ * @param   fEnabled    Whether interception is enabled or not.
+ * @sa      DBGFR3InterruptHardwareConfig, DBGFR3InterruptConfigEx
+ */
+VMMR3DECL(int) DBGFR3InterruptSoftwareConfig(PUVM pUVM, uint8_t iInterrupt, bool fEnabled)
+{
+    /*
+     * Convert to DBGFR3InterruptConfigEx call.
+     */
+    DBGFINTERRUPTCONFIG IntCfg = { iInterrupt, DBGFINTERRUPTSTATE_DONT_TOUCH, (uint8_t)fEnabled };
+    return DBGFR3InterruptConfigEx(pUVM, &IntCfg, 1);
+}
+
+
+/**
+ * Checks whether interception is enabled for a hardware interrupt.
+ *
+ * @returns true if enabled, false if not or invalid input.
+ * @param   pUVM        The user mode VM handle.
+ * @param   iInterrupt  The interrupt number.
+ * @sa      DBGFR3InterruptSoftwareIsEnabled, DBGF_IS_HARDWARE_INT_ENABLED,
+ *          DBGF_IS_SOFTWARE_INT_ENABLED
+ */
+VMMR3DECL(int) DBGFR3InterruptHardwareIsEnabled(PUVM pUVM, uint8_t iInterrupt)
+{
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+
+    /*
+     * Check it.
+     */
+    return ASMBitTest(&pVM->dbgf.s.bmHardIntBreakpoints, iInterrupt);
+}
+
+
+/**
+ * Checks whether interception is enabled for a software interrupt.
+ *
+ * @returns true if enabled, false if not or invalid input.
+ * @param   pUVM        The user mode VM handle.
+ * @param   iInterrupt  The interrupt number.
+ * @sa      DBGFR3InterruptHardwareIsEnabled, DBGF_IS_SOFTWARE_INT_ENABLED,
+ *          DBGF_IS_HARDWARE_INT_ENABLED,
+ */
+VMMR3DECL(int) DBGFR3InterruptSoftwareIsEnabled(PUVM pUVM, uint8_t iInterrupt)
+{
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+
+    /*
+     * Check it.
+     */
+    return ASMBitTest(&pVM->dbgf.s.bmSoftIntBreakpoints, iInterrupt);
+}
+
+
+
+/**
+ * Call this to single step programmatically.
+ *
+ * You must pass down the return code to the EM loop! That's
+ * where the actual single stepping take place (at least in the
+ * current implementation).
+ *
+ * @returns VINF_EM_DBG_STEP
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @thread  VCpu EMT
+ * @internal
+ */
+VMMR3_INT_DECL(int) DBGFR3PrgStep(PVMCPU pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    pVCpu->dbgf.s.fSingleSteppingRaw = true;
+    return VINF_EM_DBG_STEP;
+}
+
+
+/**
+ * Inject an NMI into a running VM (only VCPU 0!)
+ *
+ * @returns VBox status code.
+ * @param   pUVM    The user mode VM structure.
+ * @param   idCpu   The ID of the CPU to inject the NMI on.
+ */
+VMMR3DECL(int) DBGFR3InjectNMI(PUVM pUVM, VMCPUID idCpu)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+
+    /** @todo Implement generic NMI injection. */
+    /** @todo NEM: NMI injection   */
+    if (!HMIsEnabled(pVM))
+        return VERR_NOT_SUP_BY_NEM;
+
+    VMCPU_FF_SET(pVM->apCpusR3[idCpu], VMCPU_FF_INTERRUPT_NMI);
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFAddr.cpp b/src/VBox/VMM/VMMR3/DBGFAddr.cpp
new file mode 100644
index 00000000..8edb7998
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFAddr.cpp
@@ -0,0 +1,485 @@
+/* $Id: DBGFAddr.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Mixed Address Methods.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/hm.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+
+
+/**
+ * Common worker for DBGFR3AddrFromSelOff and DBGFR3AddrFromSelInfoOff.
+ */
+static int dbgfR3AddrFromSelInfoOffWorker(PDBGFADDRESS pAddress, PCDBGFSELINFO pSelInfo, RTUINTPTR off)
+{
+    if (pSelInfo->fFlags & (DBGFSELINFO_FLAGS_INVALID | DBGFSELINFO_FLAGS_NOT_PRESENT))
+        return pSelInfo->fFlags & DBGFSELINFO_FLAGS_NOT_PRESENT
+             ? VERR_SELECTOR_NOT_PRESENT
+             : VERR_INVALID_SELECTOR;
+
+    /** @todo This all goes voodoo in long mode. */
+    /* check limit. */
+    if (DBGFSelInfoIsExpandDown(pSelInfo))
+    {
+        if (    !pSelInfo->u.Raw.Gen.u1Granularity
+            &&  off > UINT32_C(0xffff))
+            return VERR_OUT_OF_SELECTOR_BOUNDS;
+        if (off <= pSelInfo->cbLimit)
+            return VERR_OUT_OF_SELECTOR_BOUNDS;
+    }
+    else if (off > pSelInfo->cbLimit)
+        return VERR_OUT_OF_SELECTOR_BOUNDS;
+
+    pAddress->FlatPtr = pSelInfo->GCPtrBase + off;
+
+    /** @todo fix all these selector tests! */
+    if (    !pSelInfo->GCPtrBase
+        &&  pSelInfo->u.Raw.Gen.u1Granularity
+        &&  pSelInfo->u.Raw.Gen.u1DefBig)
+        pAddress->fFlags = DBGFADDRESS_FLAGS_FLAT;
+    else if (pSelInfo->cbLimit <= UINT32_C(0xffff))
+        pAddress->fFlags = DBGFADDRESS_FLAGS_FAR16;
+    else if (pSelInfo->cbLimit <= UINT32_C(0xffffffff))
+        pAddress->fFlags = DBGFADDRESS_FLAGS_FAR32;
+    else
+        pAddress->fFlags = DBGFADDRESS_FLAGS_FAR64;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates a mixed address from a Sel:off pair.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   idCpu           The CPU ID.
+ * @param   pAddress        Where to store the mixed address.
+ * @param   Sel             The selector part.
+ * @param   off             The offset part.
+ */
+VMMR3DECL(int) DBGFR3AddrFromSelOff(PUVM pUVM, VMCPUID idCpu, PDBGFADDRESS pAddress, RTSEL Sel, RTUINTPTR off)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_PARAMETER);
+
+    pAddress->Sel = Sel;
+    pAddress->off = off;
+    if (Sel != DBGF_SEL_FLAT)
+    {
+        DBGFSELINFO SelInfo;
+        int rc = DBGFR3SelQueryInfo(pUVM, idCpu, Sel, DBGFSELQI_FLAGS_DT_GUEST | DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE, &SelInfo);
+        if (RT_FAILURE(rc))
+            return rc;
+        rc = dbgfR3AddrFromSelInfoOffWorker(pAddress, &SelInfo, off);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+    else
+    {
+        pAddress->FlatPtr = off;
+        pAddress->fFlags = DBGFADDRESS_FLAGS_FLAT;
+    }
+    pAddress->fFlags |= DBGFADDRESS_FLAGS_VALID;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates a mixed address from selector info and an offset into the segment
+ * described by it.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pAddress        Where to store the mixed address.
+ * @param   pSelInfo        The selector info.
+ * @param   off             The offset part.
+ */
+VMMR3DECL(int) DBGFR3AddrFromSelInfoOff(PUVM pUVM, PDBGFADDRESS pAddress, PCDBGFSELINFO pSelInfo, RTUINTPTR off)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+
+    pAddress->Sel = pSelInfo->Sel;
+    pAddress->off = off;
+    int rc = dbgfR3AddrFromSelInfoOffWorker(pAddress, pSelInfo, off);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    pAddress->fFlags |= DBGFADDRESS_FLAGS_VALID;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates a mixed address from a flat address.
+ *
+ * @returns pAddress.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pAddress        Where to store the mixed address.
+ * @param   FlatPtr         The flat pointer.
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3AddrFromFlat(PUVM pUVM, PDBGFADDRESS pAddress, RTGCUINTPTR FlatPtr)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, NULL);
+    pAddress->Sel     = DBGF_SEL_FLAT;
+    pAddress->off     = FlatPtr;
+    pAddress->FlatPtr = FlatPtr;
+    pAddress->fFlags  = DBGFADDRESS_FLAGS_FLAT | DBGFADDRESS_FLAGS_VALID;
+    return pAddress;
+}
+
+
+/**
+ * Creates a mixed address from a guest physical address.
+ *
+ * @returns pAddress.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pAddress        Where to store the mixed address.
+ * @param   PhysAddr        The guest physical address.
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3AddrFromPhys(PUVM pUVM, PDBGFADDRESS pAddress, RTGCPHYS PhysAddr)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+    pAddress->Sel     = DBGF_SEL_FLAT;
+    pAddress->off     = PhysAddr;
+    pAddress->FlatPtr = PhysAddr;
+    pAddress->fFlags  = DBGFADDRESS_FLAGS_PHYS | DBGFADDRESS_FLAGS_VALID;
+    return pAddress;
+}
+
+
+/**
+ * Creates a mixed address from a flat host ring-0 address.
+ *
+ * @returns pAddress
+ * @param   pAddress        Where to store the mixed address.
+ * @param   R0Ptr           The host ring-0 address.
+ */
+VMMR3_INT_DECL(PDBGFADDRESS) DBGFR3AddrFromHostR0(PDBGFADDRESS pAddress, RTR0UINTPTR R0Ptr)
+{
+    pAddress->FlatPtr = R0Ptr;
+    pAddress->off     = R0Ptr;
+    pAddress->fFlags  = DBGFADDRESS_FLAGS_RING0 | DBGFADDRESS_FLAGS_VALID;
+    pAddress->Sel     = DBGF_SEL_FLAT;
+    return pAddress;
+}
+
+
+/**
+ * Checks if the specified address is valid (checks the structure pointer too).
+ *
+ * @returns true if valid.
+ * @returns false if invalid.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pAddress        The address to validate.
+ */
+VMMR3DECL(bool) DBGFR3AddrIsValid(PUVM pUVM, PCDBGFADDRESS pAddress)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    if (!VALID_PTR(pAddress))
+        return false;
+    if (!DBGFADDRESS_IS_VALID(pAddress))
+        return false;
+    /* more? */
+    return true;
+}
+
+
+/**
+ * Called on the EMT for the VCpu.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pAddress        The address.
+ * @param   pGCPhys         Where to return the physical address.
+ */
+static DECLCALLBACK(int) dbgfR3AddrToPhysOnVCpu(PVMCPU pVCpu, PCDBGFADDRESS pAddress, PRTGCPHYS pGCPhys)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    /* This is just a wrapper because we cannot pass FlatPtr thru VMR3ReqCall directly. */
+    return PGMGstGetPage(pVCpu, pAddress->FlatPtr, NULL, pGCPhys);
+}
+
+
+/**
+ * Converts an address to a guest physical address.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_PARAMETER if the address is invalid.
+ * @retval  VERR_INVALID_STATE if the VM is being terminated or if the virtual
+ *          CPU handle is invalid.
+ * @retval  VERR_NOT_SUPPORTED is the type of address cannot be converted.
+ * @retval  VERR_PAGE_NOT_PRESENT
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT
+ * @retval  VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
+ * @retval  VERR_PAGE_MAP_LEVEL4_NOT_PRESENT
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   idCpu           The ID of the CPU context to convert virtual
+ *                          addresses.
+ * @param   pAddress        The address.
+ * @param   pGCPhys         Where to return the physical address.
+ */
+VMMR3DECL(int)  DBGFR3AddrToPhys(PUVM pUVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, PRTGCPHYS pGCPhys)
+{
+    /*
+     * Parameter validation.
+     */
+    AssertPtr(pGCPhys);
+    *pGCPhys = NIL_RTGCPHYS;
+    AssertPtr(pAddress);
+    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), VERR_INVALID_PARAMETER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_STATE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_PARAMETER);
+
+    /*
+     * Convert by address type.
+     */
+    int rc;
+    if (pAddress->fFlags & DBGFADDRESS_FLAGS_PHYS)
+    {
+        *pGCPhys = pAddress->FlatPtr;
+        rc = VINF_SUCCESS;
+    }
+    else
+    {
+        PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+        if (VMCPU_IS_EMT(pVCpu))
+            rc = dbgfR3AddrToPhysOnVCpu(pVCpu, pAddress, pGCPhys);
+        else
+            rc = VMR3ReqPriorityCallWaitU(pUVM, pVCpu->idCpu,
+                                          (PFNRT)dbgfR3AddrToPhysOnVCpu, 3, pVCpu, pAddress, pGCPhys);
+    }
+    return rc;
+}
+
+
+/**
+ * Converts an address to a host physical address.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_PARAMETER if the address is invalid.
+ * @retval  VERR_INVALID_STATE if the VM is being terminated or if the virtual
+ *          CPU handle is invalid.
+ * @retval  VERR_NOT_SUPPORTED is the type of address cannot be converted.
+ * @retval  VERR_PAGE_NOT_PRESENT
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT
+ * @retval  VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
+ * @retval  VERR_PAGE_MAP_LEVEL4_NOT_PRESENT
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   idCpu           The ID of the CPU context to convert virtual
+ *                          addresses.
+ * @param   pAddress        The address.
+ * @param   pHCPhys         Where to return the physical address.
+ */
+VMMR3DECL(int)  DBGFR3AddrToHostPhys(PUVM pUVM, VMCPUID idCpu, PDBGFADDRESS pAddress, PRTHCPHYS pHCPhys)
+{
+    /*
+     * Parameter validation.
+     */
+    AssertPtr(pHCPhys);
+    *pHCPhys = NIL_RTHCPHYS;
+    AssertPtr(pAddress);
+    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), VERR_INVALID_PARAMETER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_STATE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_PARAMETER);
+
+    /*
+     * Convert it.
+     */
+    RTGCPHYS GCPhys;
+    int rc = DBGFR3AddrToPhys(pUVM, idCpu, pAddress, &GCPhys);
+    if (RT_SUCCESS(rc))
+        rc = PGMPhysGCPhys2HCPhys(pVM, pAddress->FlatPtr, pHCPhys);
+    return rc;
+}
+
+
+/**
+ * Called on the EMT for the VCpu.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   idCpu           The ID of the CPU context.
+ * @param   pAddress        The address.
+ * @param   fReadOnly       Whether returning a read-only page is fine or not.
+ * @param   ppvR3Ptr        Where to return the address.
+ */
+static DECLCALLBACK(int) dbgfR3AddrToVolatileR3PtrOnVCpu(PUVM pUVM, VMCPUID idCpu, PDBGFADDRESS pAddress, bool fReadOnly,
+                                                         void **ppvR3Ptr)
+{
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    Assert(idCpu == VMMGetCpuId(pVM));
+
+    /*
+     * This is a tad ugly, but it gets the job done.
+     */
+    int rc;
+    PGMPAGEMAPLOCK Lock;
+    if (pAddress->fFlags & DBGFADDRESS_FLAGS_PHYS)
+    {
+        if (fReadOnly)
+            rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, pAddress->FlatPtr, (void const **)ppvR3Ptr, &Lock);
+        else
+            rc = PGMPhysGCPhys2CCPtr(pVM, pAddress->FlatPtr, ppvR3Ptr, &Lock);
+    }
+    else
+    {
+        PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+        if (fReadOnly)
+            rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, pAddress->FlatPtr, (void const **)ppvR3Ptr, &Lock);
+        else
+            rc = PGMPhysGCPtr2CCPtr(pVCpu, pAddress->FlatPtr, ppvR3Ptr, &Lock);
+    }
+    if (RT_SUCCESS(rc))
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+    return rc;
+}
+
+
+
+
+/**
+ * Converts an address to a volatile host virtual address.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_PARAMETER if the address is invalid.
+ * @retval  VERR_INVALID_STATE if the VM is being terminated or if the virtual
+ *          CPU handle is invalid.
+ * @retval  VERR_NOT_SUPPORTED is the type of address cannot be converted.
+ * @retval  VERR_PAGE_NOT_PRESENT
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT
+ * @retval  VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
+ * @retval  VERR_PAGE_MAP_LEVEL4_NOT_PRESENT
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   idCpu           The ID of the CPU context to convert virtual
+ *                          addresses.
+ * @param   pAddress        The address.
+ * @param   fReadOnly       Whether returning a read-only page is fine or not.
+ *                          If set to thru the page may have to be made writable
+ *                          before we return.
+ * @param   ppvR3Ptr        Where to return the address.
+ */
+VMMR3DECL(int)  DBGFR3AddrToVolatileR3Ptr(PUVM pUVM, VMCPUID idCpu, PDBGFADDRESS pAddress, bool fReadOnly, void **ppvR3Ptr)
+{
+    /*
+     * Parameter validation.
+     */
+    AssertPtr(ppvR3Ptr);
+    *ppvR3Ptr = NULL;
+    AssertPtr(pAddress);
+    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), VERR_INVALID_PARAMETER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_STATE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_PARAMETER);
+
+    /*
+     * Convert it.
+     */
+    return VMR3ReqPriorityCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3AddrToVolatileR3PtrOnVCpu, 5,
+                                    pUVM, idCpu, pAddress, fReadOnly, ppvR3Ptr);
+}
+
+
+/**
+ * Adds an offset to an address.
+ *
+ * @returns pAddress.
+ *
+ * @param   pAddress        The address.
+ * @param   uAddend         How much to add.
+ *
+ * @remarks No address space or segment limit checks are performed,
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3AddrAdd(PDBGFADDRESS pAddress, RTGCUINTPTR uAddend)
+{
+    /*
+     * Parameter validation.
+     */
+    AssertPtrReturn(pAddress, NULL);
+    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), NULL);
+
+    /*
+     * Add the stuff.
+     */
+    pAddress->off     += uAddend;
+    pAddress->FlatPtr += uAddend;
+
+    return pAddress;
+}
+
+
+/**
+ * Subtracts an offset from an address.
+ *
+ * @returns VINF_SUCCESS on success.
+ *
+ * @param   pAddress        The address.
+ * @param   uSubtrahend     How much to subtract.
+ *
+ * @remarks No address space or segment limit checks are performed,
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3AddrSub(PDBGFADDRESS pAddress, RTGCUINTPTR uSubtrahend)
+{
+    /*
+     * Parameter validation.
+     */
+    AssertPtrReturn(pAddress, NULL);
+    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), NULL);
+
+    /*
+     * Add the stuff.
+     */
+    pAddress->off     -= uSubtrahend;
+    pAddress->FlatPtr -= uSubtrahend;
+
+    return pAddress;
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFAddrSpace.cpp b/src/VBox/VMM/VMMR3/DBGFAddrSpace.cpp
new file mode 100644
index 00000000..1937d422
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFAddrSpace.cpp
@@ -0,0 +1,1351 @@
+/* $Id: DBGFAddrSpace.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Address Space Management.
+ */
+
+/*
+ * Copyright (C) 2008-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_dbgf_addr_space     DBGFAddrSpace - Address Space Management
+ *
+ * What's an address space? It's mainly a convenient way of stuffing
+ * module segments and ad-hoc symbols together. It will also help out
+ * when the debugger gets extended to deal with user processes later.
+ *
+ * There are two standard address spaces that will always be present:
+ *   - The physical address space.
+ *   - The global virtual address space.
+ *
+ * Additional address spaces will be added and removed at runtime for
+ * guest processes. The global virtual address space will be used to
+ * track the kernel parts of the OS, or at least the bits of the kernel
+ * that is part of all address spaces (mac os x and 4G/4G patched linux).
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/mm.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/env.h>
+#include <iprt/mem.h>
+#include <iprt/path.h>
+#include <iprt/param.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Address space database node.
+ */
+typedef struct DBGFASDBNODE
+{
+    /** The node core for DBGF::AsHandleTree, the key is the address space handle. */
+    AVLPVNODECORE   HandleCore;
+    /** The node core for DBGF::AsPidTree, the key is the process id. */
+    AVLU32NODECORE  PidCore;
+    /** The node core for DBGF::AsNameSpace, the string is the address space name. */
+    RTSTRSPACECORE  NameCore;
+
+} DBGFASDBNODE;
+/** Pointer to an address space database node. */
+typedef DBGFASDBNODE *PDBGFASDBNODE;
+
+
+/**
+ * For dbgfR3AsLoadImageOpenData and dbgfR3AsLoadMapOpenData.
+ */
+typedef struct DBGFR3ASLOADOPENDATA
+{
+    const char     *pszModName;
+    RTGCUINTPTR     uSubtrahend;
+    uint32_t        fFlags;
+    RTDBGMOD        hMod;
+} DBGFR3ASLOADOPENDATA;
+
+#if 0 /* unused */
+/**
+ * Callback for dbgfR3AsSearchPath and dbgfR3AsSearchEnvPath.
+ *
+ * @returns VBox status code. If success, then the search is completed.
+ * @param   pszFilename     The file name under evaluation.
+ * @param   pvUser          The user argument.
+ */
+typedef int FNDBGFR3ASSEARCHOPEN(const char *pszFilename, void *pvUser);
+/** Pointer to a FNDBGFR3ASSEARCHOPEN. */
+typedef FNDBGFR3ASSEARCHOPEN *PFNDBGFR3ASSEARCHOPEN;
+#endif
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** Locks the address space database for writing. */
+#define DBGF_AS_DB_LOCK_WRITE(pUVM) \
+    do { \
+        int rcSem = RTSemRWRequestWrite((pUVM)->dbgf.s.hAsDbLock, RT_INDEFINITE_WAIT); \
+        AssertRC(rcSem); \
+    } while (0)
+
+/** Unlocks the address space database after writing. */
+#define DBGF_AS_DB_UNLOCK_WRITE(pUVM) \
+    do { \
+        int rcSem = RTSemRWReleaseWrite((pUVM)->dbgf.s.hAsDbLock); \
+        AssertRC(rcSem); \
+    } while (0)
+
+/** Locks the address space database for reading. */
+#define DBGF_AS_DB_LOCK_READ(pUVM) \
+    do { \
+        int rcSem = RTSemRWRequestRead((pUVM)->dbgf.s.hAsDbLock, RT_INDEFINITE_WAIT); \
+        AssertRC(rcSem); \
+    } while (0)
+
+/** Unlocks the address space database after reading. */
+#define DBGF_AS_DB_UNLOCK_READ(pUVM) \
+    do { \
+        int rcSem = RTSemRWReleaseRead((pUVM)->dbgf.s.hAsDbLock); \
+        AssertRC(rcSem); \
+    } while (0)
+
+
+
+/**
+ * Initializes the address space parts of DBGF.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ */
+int dbgfR3AsInit(PUVM pUVM)
+{
+    Assert(pUVM->pVM);
+
+    /*
+     * Create the semaphore.
+     */
+    int rc = RTSemRWCreate(&pUVM->dbgf.s.hAsDbLock);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Create the debugging config instance and set it up, defaulting to
+     * deferred loading in order to keep things fast.
+     */
+    rc = RTDbgCfgCreate(&pUVM->dbgf.s.hDbgCfg, "VBOXDBG_", true /*fNativePaths*/);
+    AssertRCReturn(rc, rc);
+    rc = RTDbgCfgChangeUInt(pUVM->dbgf.s.hDbgCfg, RTDBGCFGPROP_FLAGS, RTDBGCFGOP_PREPEND,
+                            RTDBGCFG_FLAGS_DEFERRED);
+    AssertRCReturn(rc, rc);
+
+    static struct
+    {
+        RTDBGCFGPROP    enmProp;
+        const char     *pszEnvName;
+        const char     *pszCfgName;
+    } const s_aProps[] =
+    {
+        { RTDBGCFGPROP_FLAGS,               "VBOXDBG_FLAGS",            "Flags"             },
+        { RTDBGCFGPROP_PATH,                "VBOXDBG_PATH",             "Path"              },
+        { RTDBGCFGPROP_SUFFIXES,            "VBOXDBG_SUFFIXES",         "Suffixes"          },
+        { RTDBGCFGPROP_SRC_PATH,            "VBOXDBG_SRC_PATH",         "SrcPath"           },
+    };
+    PCFGMNODE pCfgDbgf = CFGMR3GetChild(CFGMR3GetRootU(pUVM), "/DBGF");
+    for (unsigned i = 0; i < RT_ELEMENTS(s_aProps); i++)
+    {
+        char szEnvValue[8192];
+        rc = RTEnvGetEx(RTENV_DEFAULT, s_aProps[i].pszEnvName, szEnvValue, sizeof(szEnvValue), NULL);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTDbgCfgChangeString(pUVM->dbgf.s.hDbgCfg, s_aProps[i].enmProp, RTDBGCFGOP_PREPEND, szEnvValue);
+            if (RT_FAILURE(rc))
+                return VMR3SetError(pUVM, rc, RT_SRC_POS,
+                                    "DBGF Config Error: %s=%s -> %Rrc", s_aProps[i].pszEnvName, szEnvValue, rc);
+        }
+        else if (rc != VERR_ENV_VAR_NOT_FOUND)
+            return VMR3SetError(pUVM, rc, RT_SRC_POS,
+                                "DBGF Config Error: Error querying env.var. %s: %Rrc", s_aProps[i].pszEnvName, rc);
+
+        char *pszCfgValue;
+        rc = CFGMR3QueryStringAllocDef(pCfgDbgf, s_aProps[i].pszCfgName, &pszCfgValue, NULL);
+        if (RT_FAILURE(rc))
+            return VMR3SetError(pUVM, rc, RT_SRC_POS,
+                                "DBGF Config Error: Querying /DBGF/%s -> %Rrc", s_aProps[i].pszCfgName, rc);
+        if (pszCfgValue)
+        {
+            rc = RTDbgCfgChangeString(pUVM->dbgf.s.hDbgCfg, s_aProps[i].enmProp, RTDBGCFGOP_PREPEND, pszCfgValue);
+            if (RT_FAILURE(rc))
+                return VMR3SetError(pUVM, rc, RT_SRC_POS,
+                                    "DBGF Config Error: /DBGF/%s=%s -> %Rrc", s_aProps[i].pszCfgName, pszCfgValue, rc);
+            MMR3HeapFree(pszCfgValue);
+        }
+    }
+
+    /*
+     * Prepend the NoArch and VBoxDbgSyms directories to the path.
+     */
+    char szPath[RTPATH_MAX];
+    rc = RTPathAppPrivateNoArch(szPath, sizeof(szPath));
+    AssertRCReturn(rc, rc);
+#ifdef RT_OS_DARWIN
+    rc = RTPathAppend(szPath, sizeof(szPath), "../Resources/VBoxDbgSyms/");
+#else
+    rc = RTDbgCfgChangeString(pUVM->dbgf.s.hDbgCfg, RTDBGCFGPROP_PATH, RTDBGCFGOP_PREPEND, szPath);
+    AssertRCReturn(rc, rc);
+
+    rc = RTPathAppend(szPath, sizeof(szPath), "VBoxDbgSyms/");
+#endif
+    AssertRCReturn(rc, rc);
+    rc = RTDbgCfgChangeString(pUVM->dbgf.s.hDbgCfg, RTDBGCFGPROP_PATH, RTDBGCFGOP_PREPEND, szPath);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Create the standard address spaces.
+     */
+    RTDBGAS hDbgAs;
+    rc = RTDbgAsCreate(&hDbgAs, 0, RTGCPTR_MAX, "Global");
+    AssertRCReturn(rc, rc);
+    rc = DBGFR3AsAdd(pUVM, hDbgAs, NIL_RTPROCESS);
+    AssertRCReturn(rc, rc);
+    pUVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_GLOBAL)] = hDbgAs;
+
+    RTDbgAsRetain(hDbgAs);
+    pUVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_KERNEL)] = hDbgAs;
+
+    rc = RTDbgAsCreate(&hDbgAs, 0, RTGCPHYS_MAX, "Physical");
+    AssertRCReturn(rc, rc);
+    rc = DBGFR3AsAdd(pUVM, hDbgAs, NIL_RTPROCESS);
+    AssertRCReturn(rc, rc);
+    pUVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_PHYS)] = hDbgAs;
+
+    rc = RTDbgAsCreate(&hDbgAs, 0, RTRCPTR_MAX, "HyperRawMode");
+    AssertRCReturn(rc, rc);
+    rc = DBGFR3AsAdd(pUVM, hDbgAs, NIL_RTPROCESS);
+    AssertRCReturn(rc, rc);
+    pUVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_RC)] = hDbgAs;
+    RTDbgAsRetain(hDbgAs);
+    pUVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_RC_AND_GC_GLOBAL)] = hDbgAs;
+
+    rc = RTDbgAsCreate(&hDbgAs, 0, RTR0PTR_MAX, "HyperRing0");
+    AssertRCReturn(rc, rc);
+    rc = DBGFR3AsAdd(pUVM, hDbgAs, NIL_RTPROCESS);
+    AssertRCReturn(rc, rc);
+    pUVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_R0)] = hDbgAs;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Callback used by dbgfR3AsTerm / RTAvlPVDestroy to release an address space.
+ *
+ * @returns 0.
+ * @param   pNode           The address space database node.
+ * @param   pvIgnore        NULL.
+ */
+static DECLCALLBACK(int) dbgfR3AsTermDestroyNode(PAVLPVNODECORE pNode, void *pvIgnore)
+{
+    PDBGFASDBNODE pDbNode = (PDBGFASDBNODE)pNode;
+    RTDbgAsRelease((RTDBGAS)pDbNode->HandleCore.Key);
+    pDbNode->HandleCore.Key = NIL_RTDBGAS;
+    /* Don't bother freeing it here as MM will free it soon and MM is much at
+       it when doing it wholesale instead of piecemeal. */
+    NOREF(pvIgnore);
+    return 0;
+}
+
+
+/**
+ * Terminates the address space parts of DBGF.
+ *
+ * @param   pUVM        The user mode VM handle.
+ */
+void dbgfR3AsTerm(PUVM pUVM)
+{
+    /*
+     * Create the semaphore.
+     */
+    int rc = RTSemRWDestroy(pUVM->dbgf.s.hAsDbLock);
+    AssertRC(rc);
+    pUVM->dbgf.s.hAsDbLock = NIL_RTSEMRW;
+
+    /*
+     * Release all the address spaces.
+     */
+    RTAvlPVDestroy(&pUVM->dbgf.s.AsHandleTree, dbgfR3AsTermDestroyNode, NULL);
+    for (size_t i = 0; i < RT_ELEMENTS(pUVM->dbgf.s.ahAsAliases); i++)
+    {
+        RTDbgAsRelease(pUVM->dbgf.s.ahAsAliases[i]);
+        pUVM->dbgf.s.ahAsAliases[i] = NIL_RTDBGAS;
+    }
+
+    /*
+     * Release the reference to the debugging config.
+     */
+    rc = RTDbgCfgRelease(pUVM->dbgf.s.hDbgCfg);
+    AssertRC(rc);
+}
+
+
+/**
+ * Relocates the RC address space.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   offDelta    The relocation delta.
+ */
+void dbgfR3AsRelocate(PUVM pUVM, RTGCUINTPTR offDelta)
+{
+    /*
+     * We will relocate the raw-mode context modules by offDelta if they have
+     * been injected into the DBGF_AS_RC map.
+     */
+    if (   pUVM->dbgf.s.afAsAliasPopuplated[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_RC)]
+        && offDelta != 0)
+    {
+        RTDBGAS hAs = pUVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_RC)];
+
+        /* Take a snapshot of the modules as we might have overlapping
+           addresses between the previous and new mapping. */
+        RTDbgAsLockExcl(hAs);
+        uint32_t cModules = RTDbgAsModuleCount(hAs);
+        if (cModules > 0 && cModules < _4K)
+        {
+            struct DBGFASRELOCENTRY
+            {
+                RTDBGMOD    hDbgMod;
+                RTRCPTR     uOldAddr;
+            } *paEntries = (struct DBGFASRELOCENTRY *)RTMemTmpAllocZ(sizeof(paEntries[0]) * cModules);
+            if (paEntries)
+            {
+                /* Snapshot. */
+                for (uint32_t i = 0; i < cModules; i++)
+                {
+                    paEntries[i].hDbgMod = RTDbgAsModuleByIndex(hAs, i);
+                    AssertLogRelMsg(paEntries[i].hDbgMod != NIL_RTDBGMOD, ("iModule=%#x\n", i));
+
+                    RTDBGASMAPINFO  aMappings[1] = { { 0, 0 } };
+                    uint32_t        cMappings = 1;
+                    int rc = RTDbgAsModuleQueryMapByIndex(hAs, i, &aMappings[0], &cMappings, 0 /*fFlags*/);
+                    if (RT_SUCCESS(rc) && cMappings == 1 && aMappings[0].iSeg == NIL_RTDBGSEGIDX)
+                        paEntries[i].uOldAddr = (RTRCPTR)aMappings[0].Address;
+                    else
+                        AssertLogRelMsgFailed(("iModule=%#x rc=%Rrc cMappings=%#x.\n", i, rc, cMappings));
+                }
+
+                /* Unlink them. */
+                for (uint32_t i = 0; i < cModules; i++)
+                {
+                    int rc = RTDbgAsModuleUnlink(hAs, paEntries[i].hDbgMod);
+                    AssertLogRelMsg(RT_SUCCESS(rc), ("iModule=%#x rc=%Rrc hDbgMod=%p\n", i, rc, paEntries[i].hDbgMod));
+                }
+
+                /* Link them at the new locations. */
+                for (uint32_t i = 0; i < cModules; i++)
+                {
+                    RTRCPTR uNewAddr = paEntries[i].uOldAddr + offDelta;
+                    int rc = RTDbgAsModuleLink(hAs, paEntries[i].hDbgMod, uNewAddr,
+                                               RTDBGASLINK_FLAGS_REPLACE);
+                    AssertLogRelMsg(RT_SUCCESS(rc),
+                                    ("iModule=%#x rc=%Rrc hDbgMod=%p %RRv -> %RRv\n", i, rc, paEntries[i].hDbgMod,
+                                     paEntries[i].uOldAddr, uNewAddr));
+                    RTDbgModRelease(paEntries[i].hDbgMod);
+                }
+
+                RTMemTmpFree(paEntries);
+            }
+            else
+                AssertLogRelMsgFailed(("No memory for %#x modules.\n", cModules));
+        }
+        else
+            AssertLogRelMsgFailed(("cModules=%#x\n", cModules));
+        RTDbgAsUnlockExcl(hAs);
+    }
+}
+
+
+/**
+ * Gets the IPRT debugging configuration handle (no refs retained).
+ *
+ * @returns Config handle or NIL_RTDBGCFG.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(RTDBGCFG)     DBGFR3AsGetConfig(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NIL_RTDBGCFG);
+    return pUVM->dbgf.s.hDbgCfg;
+}
+
+
+/**
+ * Adds the address space to the database.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   hDbgAs      The address space handle. The reference of the caller
+ *                      will NOT be consumed.
+ * @param   ProcId      The process id or NIL_RTPROCESS.
+ */
+VMMR3DECL(int) DBGFR3AsAdd(PUVM pUVM, RTDBGAS hDbgAs, RTPROCESS ProcId)
+{
+    /*
+     * Input validation.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    const char *pszName = RTDbgAsName(hDbgAs);
+    if (!pszName)
+        return VERR_INVALID_HANDLE;
+    uint32_t cRefs = RTDbgAsRetain(hDbgAs);
+    if (cRefs == UINT32_MAX)
+        return VERR_INVALID_HANDLE;
+
+    /*
+     * Allocate a tracking node.
+     */
+    int rc = VERR_NO_MEMORY;
+    PDBGFASDBNODE pDbNode = (PDBGFASDBNODE)MMR3HeapAllocU(pUVM, MM_TAG_DBGF_AS, sizeof(*pDbNode));
+    if (pDbNode)
+    {
+        pDbNode->HandleCore.Key     = hDbgAs;
+        pDbNode->PidCore.Key        = ProcId;
+        pDbNode->NameCore.pszString = pszName;
+        pDbNode->NameCore.cchString = strlen(pszName);
+        DBGF_AS_DB_LOCK_WRITE(pUVM);
+        if (RTStrSpaceInsert(&pUVM->dbgf.s.AsNameSpace, &pDbNode->NameCore))
+        {
+            if (RTAvlPVInsert(&pUVM->dbgf.s.AsHandleTree, &pDbNode->HandleCore))
+            {
+                DBGF_AS_DB_UNLOCK_WRITE(pUVM);
+                return VINF_SUCCESS;
+            }
+
+            /* bail out */
+            RTStrSpaceRemove(&pUVM->dbgf.s.AsNameSpace, pszName);
+        }
+        DBGF_AS_DB_UNLOCK_WRITE(pUVM);
+        MMR3HeapFree(pDbNode);
+    }
+    RTDbgAsRelease(hDbgAs);
+    return rc;
+}
+
+
+/**
+ * Delete an address space from the database.
+ *
+ * The address space must not be engaged as any of the standard aliases.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_SHARING_VIOLATION if in use as an alias.
+ * @retval  VERR_NOT_FOUND if not found in the address space database.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   hDbgAs      The address space handle. Aliases are not allowed.
+ */
+VMMR3DECL(int) DBGFR3AsDelete(PUVM pUVM, RTDBGAS hDbgAs)
+{
+    /*
+     * Input validation. Retain the address space so it can be released outside
+     * the lock as well as validated.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    if (hDbgAs == NIL_RTDBGAS)
+        return VINF_SUCCESS;
+    uint32_t cRefs = RTDbgAsRetain(hDbgAs);
+    if (cRefs == UINT32_MAX)
+        return VERR_INVALID_HANDLE;
+    RTDbgAsRelease(hDbgAs);
+
+    DBGF_AS_DB_LOCK_WRITE(pUVM);
+
+    /*
+     * You cannot delete any of the aliases.
+     */
+    for (size_t i = 0; i < RT_ELEMENTS(pUVM->dbgf.s.ahAsAliases); i++)
+        if (pUVM->dbgf.s.ahAsAliases[i] == hDbgAs)
+        {
+            DBGF_AS_DB_UNLOCK_WRITE(pUVM);
+            return VERR_SHARING_VIOLATION;
+        }
+
+    /*
+     * Ok, try remove it from the database.
+     */
+    PDBGFASDBNODE pDbNode = (PDBGFASDBNODE)RTAvlPVRemove(&pUVM->dbgf.s.AsHandleTree, hDbgAs);
+    if (!pDbNode)
+    {
+        DBGF_AS_DB_UNLOCK_WRITE(pUVM);
+        return VERR_NOT_FOUND;
+    }
+    RTStrSpaceRemove(&pUVM->dbgf.s.AsNameSpace, pDbNode->NameCore.pszString);
+    if (pDbNode->PidCore.Key != NIL_RTPROCESS)
+        RTAvlU32Remove(&pUVM->dbgf.s.AsPidTree, pDbNode->PidCore.Key);
+
+    DBGF_AS_DB_UNLOCK_WRITE(pUVM);
+
+    /*
+     * Free the resources.
+     */
+    RTDbgAsRelease(hDbgAs);
+    MMR3HeapFree(pDbNode);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Changes an alias to point to a new address space.
+ *
+ * Not all the aliases can be changed, currently it's only DBGF_AS_GLOBAL
+ * and DBGF_AS_KERNEL.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   hAlias      The alias to change.
+ * @param   hAliasFor   The address space hAlias should be an alias for.  This
+ *                      can be an alias. The caller's reference to this address
+ *                      space will NOT be consumed.
+ */
+VMMR3DECL(int) DBGFR3AsSetAlias(PUVM pUVM, RTDBGAS hAlias, RTDBGAS hAliasFor)
+{
+    /*
+     * Input validation.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertMsgReturn(DBGF_AS_IS_ALIAS(hAlias), ("%p\n", hAlias), VERR_INVALID_PARAMETER);
+    AssertMsgReturn(!DBGF_AS_IS_FIXED_ALIAS(hAlias), ("%p\n", hAlias), VERR_INVALID_PARAMETER);
+    RTDBGAS hRealAliasFor = DBGFR3AsResolveAndRetain(pUVM, hAliasFor);
+    if (hRealAliasFor == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+    /*
+     * Make sure the handle is already in the database.
+     */
+    int rc = VERR_NOT_FOUND;
+    DBGF_AS_DB_LOCK_WRITE(pUVM);
+    if (RTAvlPVGet(&pUVM->dbgf.s.AsHandleTree, hRealAliasFor))
+    {
+        /*
+         * Update the alias table and release the current address space.
+         */
+        RTDBGAS hAsOld;
+        ASMAtomicXchgHandle(&pUVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(hAlias)], hRealAliasFor, &hAsOld);
+        uint32_t cRefs = RTDbgAsRelease(hAsOld);
+        Assert(cRefs > 0); Assert(cRefs != UINT32_MAX); NOREF(cRefs);
+        rc = VINF_SUCCESS;
+    }
+    else
+        RTDbgAsRelease(hRealAliasFor);
+    DBGF_AS_DB_UNLOCK_WRITE(pUVM);
+
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNPDMR3ENUM}
+ */
+static DECLCALLBACK(int) dbgfR3AsLazyPopulateR0Callback(PVM pVM, const char *pszFilename, const char *pszName,
+                                                        RTUINTPTR ImageBase, size_t cbImage, PDMLDRCTX enmCtx, void *pvArg)
+{
+    NOREF(pVM); NOREF(cbImage);
+
+    /* Only ring-0 modules. */
+    if (enmCtx == PDMLDRCTX_RING_0)
+    {
+        RTDBGMOD hDbgMod;
+        int rc = RTDbgModCreateFromImage(&hDbgMod, pszFilename, pszName, RTLDRARCH_HOST, pVM->pUVM->dbgf.s.hDbgCfg);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTDbgAsModuleLink((RTDBGAS)pvArg, hDbgMod, ImageBase, 0 /*fFlags*/);
+            if (RT_FAILURE(rc))
+                LogRel(("DBGF: Failed to link module \"%s\" into DBGF_AS_R0 at %RTptr: %Rrc\n",
+                        pszName, ImageBase, rc));
+        }
+        else
+            LogRel(("DBGF: RTDbgModCreateFromImage failed with rc=%Rrc for module \"%s\" (%s)\n",
+                    rc, pszName, pszFilename));
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNPDMR3ENUM}
+ */
+static DECLCALLBACK(int) dbgfR3AsLazyPopulateRCCallback(PVM pVM, const char *pszFilename, const char *pszName,
+                                                        RTUINTPTR ImageBase, size_t cbImage, PDMLDRCTX enmCtx, void *pvArg)
+{
+    NOREF(pVM); NOREF(cbImage);
+
+    /* Only raw-mode modules. */
+    if (enmCtx == PDMLDRCTX_RAW_MODE)
+    {
+        RTDBGMOD hDbgMod;
+        int rc = RTDbgModCreateFromImage(&hDbgMod, pszFilename, pszName, RTLDRARCH_X86_32, pVM->pUVM->dbgf.s.hDbgCfg);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTDbgAsModuleLink((RTDBGAS)pvArg, hDbgMod, ImageBase, 0 /*fFlags*/);
+            if (RT_FAILURE(rc))
+                LogRel(("DBGF: Failed to link module \"%s\" into DBGF_AS_RC at %RTptr: %Rrc\n",
+                        pszName, ImageBase, rc));
+        }
+        else
+            LogRel(("DBGF: RTDbgModCreateFromImage failed with rc=%Rrc for module \"%s\" (%s)\n",
+                    rc, pszName, pszFilename));
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Lazily populates the specified address space.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   hAlias      The alias.
+ */
+static void dbgfR3AsLazyPopulate(PUVM pUVM, RTDBGAS hAlias)
+{
+    DBGF_AS_DB_LOCK_WRITE(pUVM);
+    uintptr_t iAlias = DBGF_AS_ALIAS_2_INDEX(hAlias);
+    if (!pUVM->dbgf.s.afAsAliasPopuplated[iAlias])
+    {
+        RTDBGAS hDbgAs = pUVM->dbgf.s.ahAsAliases[iAlias];
+        if (hAlias == DBGF_AS_R0 && pUVM->pVM)
+            PDMR3LdrEnumModules(pUVM->pVM, dbgfR3AsLazyPopulateR0Callback, hDbgAs);
+        else if (hAlias == DBGF_AS_RC && pUVM->pVM && VM_IS_RAW_MODE_ENABLED(pUVM->pVM))
+        {
+            LogRel(("DBGF: Lazy init of RC address space\n"));
+            PDMR3LdrEnumModules(pUVM->pVM, dbgfR3AsLazyPopulateRCCallback, hDbgAs);
+        }
+        else if (hAlias == DBGF_AS_PHYS && pUVM->pVM)
+        {
+            /** @todo Lazy load pc and vga bios symbols or the EFI stuff. */
+        }
+
+        pUVM->dbgf.s.afAsAliasPopuplated[iAlias] = true;
+    }
+    DBGF_AS_DB_UNLOCK_WRITE(pUVM);
+}
+
+
+/**
+ * Resolves the address space handle into a real handle if it's an alias.
+ *
+ * @returns Real address space handle. NIL_RTDBGAS if invalid handle.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   hAlias      The possibly address space alias.
+ *
+ * @remarks Doesn't take any locks.
+ */
+VMMR3DECL(RTDBGAS) DBGFR3AsResolve(PUVM pUVM, RTDBGAS hAlias)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+    AssertCompileNS(NIL_RTDBGAS == (RTDBGAS)0);
+
+    uintptr_t   iAlias = DBGF_AS_ALIAS_2_INDEX(hAlias);
+    if (iAlias < DBGF_AS_COUNT)
+        ASMAtomicReadHandle(&pUVM->dbgf.s.ahAsAliases[iAlias], &hAlias);
+    return hAlias;
+}
+
+
+/**
+ * Resolves the address space handle into a real handle if it's an alias,
+ * and retains whatever it is.
+ *
+ * @returns Real address space handle. NIL_RTDBGAS if invalid handle.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   hAlias      The possibly address space alias.
+ */
+VMMR3DECL(RTDBGAS) DBGFR3AsResolveAndRetain(PUVM pUVM, RTDBGAS hAlias)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+    AssertCompileNS(NIL_RTDBGAS == (RTDBGAS)0);
+
+    uint32_t    cRefs;
+    uintptr_t   iAlias = DBGF_AS_ALIAS_2_INDEX(hAlias);
+    if (iAlias < DBGF_AS_COUNT)
+    {
+        if (DBGF_AS_IS_FIXED_ALIAS(hAlias))
+        {
+            /* Perform lazy address space population. */
+            if (!pUVM->dbgf.s.afAsAliasPopuplated[iAlias])
+                dbgfR3AsLazyPopulate(pUVM, hAlias);
+
+            /* Won't ever change, no need to grab the lock. */
+            hAlias = pUVM->dbgf.s.ahAsAliases[iAlias];
+            cRefs = RTDbgAsRetain(hAlias);
+        }
+        else
+        {
+            /* May change, grab the lock so we can read it safely. */
+            DBGF_AS_DB_LOCK_READ(pUVM);
+            hAlias = pUVM->dbgf.s.ahAsAliases[iAlias];
+            cRefs = RTDbgAsRetain(hAlias);
+            DBGF_AS_DB_UNLOCK_READ(pUVM);
+        }
+    }
+    else
+        /* Not an alias, just retain it. */
+        cRefs = RTDbgAsRetain(hAlias);
+
+    return cRefs != UINT32_MAX ? hAlias : NIL_RTDBGAS;
+}
+
+
+/**
+ * Query an address space by name.
+ *
+ * @returns Retained address space handle if found, NIL_RTDBGAS if not.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   pszName     The name.
+ */
+VMMR3DECL(RTDBGAS) DBGFR3AsQueryByName(PUVM pUVM, const char *pszName)
+{
+    /*
+     * Validate the input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NIL_RTDBGAS);
+    AssertPtrReturn(pszName, NIL_RTDBGAS);
+    AssertReturn(*pszName, NIL_RTDBGAS);
+
+    /*
+     * Look it up in the string space and retain the result.
+     */
+    RTDBGAS hDbgAs = NIL_RTDBGAS;
+    DBGF_AS_DB_LOCK_READ(pUVM);
+
+    PRTSTRSPACECORE pNode = RTStrSpaceGet(&pUVM->dbgf.s.AsNameSpace, pszName);
+    if (pNode)
+    {
+        PDBGFASDBNODE pDbNode = RT_FROM_MEMBER(pNode, DBGFASDBNODE, NameCore);
+        hDbgAs = (RTDBGAS)pDbNode->HandleCore.Key;
+        uint32_t cRefs = RTDbgAsRetain(hDbgAs);
+        if (RT_UNLIKELY(cRefs == UINT32_MAX))
+            hDbgAs = NIL_RTDBGAS;
+    }
+
+    DBGF_AS_DB_UNLOCK_READ(pUVM);
+    return hDbgAs;
+}
+
+
+/**
+ * Query an address space by process ID.
+ *
+ * @returns Retained address space handle if found, NIL_RTDBGAS if not.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   ProcId      The process ID.
+ */
+VMMR3DECL(RTDBGAS) DBGFR3AsQueryByPid(PUVM pUVM, RTPROCESS ProcId)
+{
+    /*
+     * Validate the input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NIL_RTDBGAS);
+    AssertReturn(ProcId != NIL_RTPROCESS, NIL_RTDBGAS);
+
+    /*
+     * Look it up in the PID tree and retain the result.
+     */
+    RTDBGAS hDbgAs = NIL_RTDBGAS;
+    DBGF_AS_DB_LOCK_READ(pUVM);
+
+    PAVLU32NODECORE pNode = RTAvlU32Get(&pUVM->dbgf.s.AsPidTree, ProcId);
+    if (pNode)
+    {
+        PDBGFASDBNODE pDbNode = RT_FROM_MEMBER(pNode, DBGFASDBNODE, PidCore);
+        hDbgAs = (RTDBGAS)pDbNode->HandleCore.Key;
+        uint32_t cRefs = RTDbgAsRetain(hDbgAs);
+        if (RT_UNLIKELY(cRefs == UINT32_MAX))
+            hDbgAs = NIL_RTDBGAS;
+    }
+    DBGF_AS_DB_UNLOCK_READ(pUVM);
+
+    return hDbgAs;
+}
+
+#if 0 /* unused */
+
+/**
+ * Searches for the file in the path.
+ *
+ * The file is first tested without any path modification, then we walk the path
+ * looking in each directory.
+ *
+ * @returns VBox status code.
+ * @param   pszFilename     The file to search for.
+ * @param   pszPath         The search path.
+ * @param   pfnOpen         The open callback function.
+ * @param   pvUser          User argument for the callback.
+ */
+static int dbgfR3AsSearchPath(const char *pszFilename, const char *pszPath, PFNDBGFR3ASSEARCHOPEN pfnOpen, void *pvUser)
+{
+    char szFound[RTPATH_MAX];
+
+    /* Check the filename length. */
+    size_t const    cchFilename = strlen(pszFilename);
+    if (cchFilename >= sizeof(szFound))
+        return VERR_FILENAME_TOO_LONG;
+    const char     *pszName = RTPathFilename(pszFilename);
+    if (!pszName)
+        return VERR_IS_A_DIRECTORY;
+    size_t const    cchName = strlen(pszName);
+
+    /*
+     * Try default location first.
+     */
+    memcpy(szFound, pszFilename, cchFilename + 1);
+    int rc = pfnOpen(szFound, pvUser);
+    if (RT_SUCCESS(rc))
+        return rc;
+
+    /*
+     * Walk the search path.
+     */
+    const char *psz = pszPath;
+    while (*psz)
+    {
+        /* Skip leading blanks - no directories with leading spaces, thank you. */
+        while (RT_C_IS_BLANK(*psz))
+            psz++;
+
+        /* Find the end of this element. */
+        const char *pszNext;
+        const char *pszEnd = strchr(psz, ';');
+        if (!pszEnd)
+            pszEnd = pszNext = strchr(psz, '\0');
+        else
+            pszNext = pszEnd + 1;
+        if (pszEnd != psz)
+        {
+            size_t const cch = pszEnd - psz;
+            if (cch + 1 + cchName < sizeof(szFound))
+            {
+                /** @todo RTPathCompose, RTPathComposeN(). This code isn't right
+                 * for 'E:' on DOS systems. It may also create unwanted double slashes. */
+                memcpy(szFound, psz, cch);
+                szFound[cch] = '/';
+                memcpy(szFound + cch + 1, pszName, cchName + 1);
+                int rc2 = pfnOpen(szFound, pvUser);
+                if (RT_SUCCESS(rc2))
+                    return rc2;
+                if (    rc2 != rc
+                    &&  (   rc == VERR_FILE_NOT_FOUND
+                         || rc == VERR_OPEN_FAILED))
+                    rc = rc2;
+            }
+        }
+
+        /* advance */
+        psz = pszNext;
+    }
+
+    /*
+     * Walk the path once again, this time do a depth search.
+     */
+    /** @todo do a depth search using the specified path. */
+
+    /* failed */
+    return rc;
+}
+
+
+/**
+ * Same as dbgfR3AsSearchEnv, except that the path is taken from the environment.
+ *
+ * If the environment variable doesn't exist, the current directory is searched
+ * instead.
+ *
+ * @returns VBox status code.
+ * @param   pszFilename     The filename.
+ * @param   pszEnvVar       The environment variable name.
+ * @param   pfnOpen         The open callback function.
+ * @param   pvUser          User argument for the callback.
+ */
+static int dbgfR3AsSearchEnvPath(const char *pszFilename, const char *pszEnvVar, PFNDBGFR3ASSEARCHOPEN pfnOpen, void *pvUser)
+{
+    int     rc;
+    char   *pszPath = RTEnvDupEx(RTENV_DEFAULT, pszEnvVar);
+    if (pszPath)
+    {
+        rc = dbgfR3AsSearchPath(pszFilename, pszPath, pfnOpen, pvUser);
+        RTStrFree(pszPath);
+    }
+    else
+        rc = dbgfR3AsSearchPath(pszFilename, ".", pfnOpen, pvUser);
+    return rc;
+}
+
+
+/**
+ * Same as dbgfR3AsSearchEnv, except that the path is taken from the DBGF config
+ * (CFGM).
+ *
+ * Nothing is done if the CFGM variable isn't set.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszFilename     The filename.
+ * @param   pszCfgValue     The name of the config variable (under /DBGF/).
+ * @param   pfnOpen         The open callback function.
+ * @param   pvUser          User argument for the callback.
+ */
+static int dbgfR3AsSearchCfgPath(PUVM pUVM, const char *pszFilename, const char *pszCfgValue,
+                                 PFNDBGFR3ASSEARCHOPEN pfnOpen, void *pvUser)
+{
+    char *pszPath;
+    int rc = CFGMR3QueryStringAllocDef(CFGMR3GetChild(CFGMR3GetRootU(pUVM), "/DBGF"), pszCfgValue, &pszPath, NULL);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (!pszPath)
+        return VERR_FILE_NOT_FOUND;
+    rc = dbgfR3AsSearchPath(pszFilename, pszPath, pfnOpen, pvUser);
+    MMR3HeapFree(pszPath);
+    return rc;
+}
+
+#endif /* unused */
+
+
+/**
+ * Load symbols from an executable module into the specified address space.
+ *
+ * If an module exist at the specified address it will be replaced by this
+ * call, otherwise a new module is created.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   hDbgAs          The address space.
+ * @param   pszFilename     The filename of the executable module.
+ * @param   pszModName      The module name. If NULL, then then the file name
+ *                          base is used (no extension or nothing).
+ * @param   enmArch         The desired architecture, use RTLDRARCH_WHATEVER if
+ *                          it's not relevant or known.
+ * @param   pModAddress     The load address of the module.
+ * @param   iModSeg         The segment to load, pass NIL_RTDBGSEGIDX to load
+ *                          the whole image.
+ * @param   fFlags          For DBGFR3AsLinkModule, see RTDBGASLINK_FLAGS_*.
+ */
+VMMR3DECL(int) DBGFR3AsLoadImage(PUVM pUVM, RTDBGAS hDbgAs, const char *pszFilename, const char *pszModName, RTLDRARCH enmArch,
+                                 PCDBGFADDRESS pModAddress, RTDBGSEGIDX iModSeg, uint32_t fFlags)
+{
+    /*
+     * Validate input
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszFilename, VERR_INVALID_POINTER);
+    AssertReturn(*pszFilename, VERR_INVALID_PARAMETER);
+    AssertReturn(DBGFR3AddrIsValid(pUVM, pModAddress), VERR_INVALID_PARAMETER);
+    AssertReturn(!(fFlags & ~RTDBGASLINK_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
+    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pUVM, hDbgAs);
+    if (hRealAS == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+    RTDBGMOD hDbgMod;
+    int rc = RTDbgModCreateFromImage(&hDbgMod, pszFilename, pszModName, enmArch, pUVM->dbgf.s.hDbgCfg);
+    if (RT_SUCCESS(rc))
+    {
+        rc = DBGFR3AsLinkModule(pUVM, hRealAS, hDbgMod, pModAddress, iModSeg, fFlags & RTDBGASLINK_FLAGS_VALID_MASK);
+        if (RT_FAILURE(rc))
+            RTDbgModRelease(hDbgMod);
+    }
+
+    RTDbgAsRelease(hRealAS);
+    return rc;
+}
+
+
+/**
+ * Load symbols from a map file into a module at the specified address space.
+ *
+ * If an module exist at the specified address it will be replaced by this
+ * call, otherwise a new module is created.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   hDbgAs          The address space.
+ * @param   pszFilename     The map file.
+ * @param   pszModName      The module name. If NULL, then then the file name
+ *                          base is used (no extension or nothing).
+ * @param   pModAddress     The load address of the module.
+ * @param   iModSeg         The segment to load, pass NIL_RTDBGSEGIDX to load
+ *                          the whole image.
+ * @param   uSubtrahend     Value to to subtract from the symbols in the map
+ *                          file. This is useful for the linux System.map and
+ *                          /proc/kallsyms.
+ * @param   fFlags          Flags reserved for future extensions, must be 0.
+ */
+VMMR3DECL(int) DBGFR3AsLoadMap(PUVM pUVM, RTDBGAS hDbgAs, const char *pszFilename, const char *pszModName,
+                               PCDBGFADDRESS pModAddress, RTDBGSEGIDX iModSeg, RTGCUINTPTR uSubtrahend, uint32_t fFlags)
+{
+    /*
+     * Validate input
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszFilename, VERR_INVALID_POINTER);
+    AssertReturn(*pszFilename, VERR_INVALID_PARAMETER);
+    AssertReturn(DBGFR3AddrIsValid(pUVM, pModAddress), VERR_INVALID_PARAMETER);
+    AssertReturn(fFlags == 0, VERR_INVALID_PARAMETER);
+    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pUVM, hDbgAs);
+    if (hRealAS == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+    RTDBGMOD hDbgMod;
+    int rc = RTDbgModCreateFromMap(&hDbgMod, pszFilename, pszModName, uSubtrahend, pUVM->dbgf.s.hDbgCfg);
+    if (RT_SUCCESS(rc))
+    {
+        rc = DBGFR3AsLinkModule(pUVM, hRealAS, hDbgMod, pModAddress, iModSeg, 0);
+        if (RT_FAILURE(rc))
+            RTDbgModRelease(hDbgMod);
+    }
+
+    RTDbgAsRelease(hRealAS);
+    return rc;
+}
+
+
+/**
+ * Wrapper around RTDbgAsModuleLink, RTDbgAsModuleLinkSeg and DBGFR3AsResolve.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   hDbgAs          The address space handle.
+ * @param   hMod            The module handle.
+ * @param   pModAddress     The link address.
+ * @param   iModSeg         The segment to link, NIL_RTDBGSEGIDX for the entire image.
+ * @param   fFlags          Flags to pass to the link functions, see RTDBGASLINK_FLAGS_*.
+ */
+VMMR3DECL(int) DBGFR3AsLinkModule(PUVM pUVM, RTDBGAS hDbgAs, RTDBGMOD hMod, PCDBGFADDRESS pModAddress,
+                                  RTDBGSEGIDX iModSeg, uint32_t fFlags)
+{
+    /*
+     * Input validation.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(DBGFR3AddrIsValid(pUVM, pModAddress), VERR_INVALID_PARAMETER);
+    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pUVM, hDbgAs);
+    if (hRealAS == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+    /*
+     * Do the job.
+     */
+    int rc;
+    if (iModSeg == NIL_RTDBGSEGIDX)
+        rc = RTDbgAsModuleLink(hRealAS, hMod, pModAddress->FlatPtr, fFlags);
+    else
+        rc = RTDbgAsModuleLinkSeg(hRealAS, hMod, iModSeg, pModAddress->FlatPtr, fFlags);
+
+    RTDbgAsRelease(hRealAS);
+    return rc;
+}
+
+
+/**
+ * Wrapper around RTDbgAsModuleByName and RTDbgAsModuleUnlink.
+ *
+ * Unlinks all mappings matching the given module name.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   hDbgAs          The address space handle.
+ * @param   pszModName      The name of the module to unlink.
+ */
+VMMR3DECL(int) DBGFR3AsUnlinkModuleByName(PUVM pUVM, RTDBGAS hDbgAs, const char *pszModName)
+{
+    /*
+     * Input validation.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pUVM, hDbgAs);
+    if (hRealAS == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+    /*
+     * Do the job.
+     */
+    RTDBGMOD hMod;
+    int rc = RTDbgAsModuleByName(hRealAS, pszModName, 0, &hMod);
+    if (RT_SUCCESS(rc))
+    {
+        for (;;)
+        {
+            rc = RTDbgAsModuleUnlink(hRealAS, hMod);
+            RTDbgModRelease(hMod);
+            if (RT_FAILURE(rc))
+                break;
+            rc = RTDbgAsModuleByName(hRealAS, pszModName, 0, &hMod);
+            if (RT_FAILURE_NP(rc))
+            {
+                if (rc == VERR_NOT_FOUND)
+                    rc = VINF_SUCCESS;
+                break;
+            }
+        }
+    }
+
+    RTDbgAsRelease(hRealAS);
+    return rc;
+}
+
+
+/**
+ * Adds the module name to the symbol name.
+ *
+ * @param   pSymbol         The symbol info (in/out).
+ * @param   hMod            The module handle.
+ */
+static void dbgfR3AsSymbolJoinNames(PRTDBGSYMBOL pSymbol, RTDBGMOD hMod)
+{
+    /* Figure the lengths, adjust them if the result is too long. */
+    const char *pszModName = RTDbgModName(hMod);
+    size_t      cchModName = strlen(pszModName);
+    size_t      cchSymbol  = strlen(pSymbol->szName);
+    if (cchModName + 1 + cchSymbol >= sizeof(pSymbol->szName))
+    {
+        if (cchModName >= sizeof(pSymbol->szName) / 4)
+            cchModName = sizeof(pSymbol->szName) / 4;
+        if (cchModName + 1 + cchSymbol >= sizeof(pSymbol->szName))
+            cchSymbol = sizeof(pSymbol->szName) - cchModName - 2;
+        Assert(cchModName + 1 + cchSymbol < sizeof(pSymbol->szName));
+    }
+
+    /* Do the moving and copying. */
+    memmove(&pSymbol->szName[cchModName + 1], &pSymbol->szName[0], cchSymbol + 1);
+    memcpy(&pSymbol->szName[0], pszModName, cchModName);
+    pSymbol->szName[cchModName] = '!';
+}
+
+
+/**
+ * Query a symbol by address.
+ *
+ * The returned symbol is the one we consider closes to the specified address.
+ *
+ * @returns VBox status code. See RTDbgAsSymbolByAddr.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   hDbgAs          The address space handle.
+ * @param   pAddress        The address to lookup.
+ * @param   fFlags          One of the RTDBGSYMADDR_FLAGS_XXX flags.
+ * @param   poffDisp        Where to return the distance between the returned
+ *                          symbol and pAddress. Optional.
+ * @param   pSymbol         Where to return the symbol information. The returned
+ *                          symbol name will be prefixed by the module name as
+ *                          far as space allows.
+ * @param   phMod           Where to return the module handle. Optional.
+ */
+VMMR3DECL(int) DBGFR3AsSymbolByAddr(PUVM pUVM, RTDBGAS hDbgAs, PCDBGFADDRESS pAddress, uint32_t fFlags,
+                                    PRTGCINTPTR poffDisp, PRTDBGSYMBOL pSymbol, PRTDBGMOD phMod)
+{
+    /*
+     * Implement the special address space aliases the lazy way.
+     */
+    if (hDbgAs == DBGF_AS_RC_AND_GC_GLOBAL)
+    {
+        int rc = DBGFR3AsSymbolByAddr(pUVM, DBGF_AS_RC, pAddress, fFlags, poffDisp, pSymbol, phMod);
+        if (RT_FAILURE(rc))
+            rc = DBGFR3AsSymbolByAddr(pUVM, DBGF_AS_GLOBAL, pAddress, fFlags, poffDisp, pSymbol, phMod);
+        return rc;
+    }
+
+    /*
+     * Input validation.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(DBGFR3AddrIsValid(pUVM, pAddress), VERR_INVALID_PARAMETER);
+    AssertPtrNullReturn(poffDisp, VERR_INVALID_POINTER);
+    AssertPtrReturn(pSymbol, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(phMod, VERR_INVALID_POINTER);
+    if (poffDisp)
+        *poffDisp = 0;
+    if (phMod)
+        *phMod = NIL_RTDBGMOD;
+    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pUVM, hDbgAs);
+    if (hRealAS == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+    /*
+     * Do the lookup.
+     */
+    RTDBGMOD hMod;
+    int rc = RTDbgAsSymbolByAddr(hRealAS, pAddress->FlatPtr, fFlags, poffDisp, pSymbol, &hMod);
+    if (RT_SUCCESS(rc))
+    {
+        dbgfR3AsSymbolJoinNames(pSymbol, hMod);
+        if (!phMod)
+            RTDbgModRelease(hMod);
+    }
+
+    RTDbgAsRelease(hRealAS);
+    return rc;
+}
+
+
+/**
+ * Convenience function that combines RTDbgSymbolDup and DBGFR3AsSymbolByAddr.
+ *
+ * @returns Pointer to the symbol on success. This must be free using
+ *          RTDbgSymbolFree(). NULL is returned if not found or any error
+ *          occurs.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   hDbgAs          See DBGFR3AsSymbolByAddr.
+ * @param   pAddress        See DBGFR3AsSymbolByAddr.
+ * @param   fFlags          See DBGFR3AsSymbolByAddr.
+ * @param   poffDisp        See DBGFR3AsSymbolByAddr.
+ * @param   phMod           See DBGFR3AsSymbolByAddr.
+ */
+VMMR3DECL(PRTDBGSYMBOL) DBGFR3AsSymbolByAddrA(PUVM pUVM, RTDBGAS hDbgAs, PCDBGFADDRESS pAddress, uint32_t fFlags,
+                                              PRTGCINTPTR poffDisp, PRTDBGMOD phMod)
+{
+    RTDBGSYMBOL SymInfo;
+    int rc = DBGFR3AsSymbolByAddr(pUVM, hDbgAs, pAddress, fFlags, poffDisp, &SymInfo, phMod);
+    if (RT_SUCCESS(rc))
+        return RTDbgSymbolDup(&SymInfo);
+    return NULL;
+}
+
+
+/**
+ * Query a symbol by name.
+ *
+ * The symbol can be prefixed by a module name pattern to scope the search. The
+ * pattern is a simple string pattern with '*' and '?' as wild chars. See
+ * RTStrSimplePatternMatch().
+ *
+ * @returns VBox status code. See RTDbgAsSymbolByAddr.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   hDbgAs          The address space handle.
+ * @param   pszSymbol       The symbol to search for, maybe prefixed by a
+ *                          module pattern.
+ * @param   pSymbol         Where to return the symbol information.
+ *                          The returned symbol name will be prefixed by
+ *                          the module name as far as space allows.
+ * @param   phMod           Where to return the module handle. Optional.
+ */
+VMMR3DECL(int) DBGFR3AsSymbolByName(PUVM pUVM, RTDBGAS hDbgAs, const char *pszSymbol,
+                                    PRTDBGSYMBOL pSymbol, PRTDBGMOD phMod)
+{
+    /*
+     * Implement the special address space aliases the lazy way.
+     */
+    if (hDbgAs == DBGF_AS_RC_AND_GC_GLOBAL)
+    {
+        int rc = DBGFR3AsSymbolByName(pUVM, DBGF_AS_RC, pszSymbol, pSymbol, phMod);
+        if (RT_FAILURE(rc))
+            rc = DBGFR3AsSymbolByName(pUVM, DBGF_AS_GLOBAL, pszSymbol, pSymbol, phMod);
+        return rc;
+    }
+
+    /*
+     * Input validation.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pSymbol, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(phMod, VERR_INVALID_POINTER);
+    if (phMod)
+        *phMod = NIL_RTDBGMOD;
+    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pUVM, hDbgAs);
+    if (hRealAS == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+
+    /*
+     * Do the lookup.
+     */
+    RTDBGMOD hMod;
+    int rc = RTDbgAsSymbolByName(hRealAS, pszSymbol, pSymbol, &hMod);
+    if (RT_SUCCESS(rc))
+    {
+        dbgfR3AsSymbolJoinNames(pSymbol, hMod);
+        if (!phMod)
+            RTDbgModRelease(hMod);
+    }
+
+    RTDbgAsRelease(hRealAS);
+    return rc;
+}
+
+
+VMMR3DECL(int)          DBGFR3AsLineByAddr(PUVM pUVM, RTDBGAS hDbgAs, PCDBGFADDRESS pAddress,
+                                           PRTGCINTPTR poffDisp, PRTDBGLINE pLine, PRTDBGMOD phMod)
+{
+    /*
+     * Implement the special address space aliases the lazy way.
+     */
+    if (hDbgAs == DBGF_AS_RC_AND_GC_GLOBAL)
+    {
+        int rc = DBGFR3AsLineByAddr(pUVM, DBGF_AS_RC, pAddress, poffDisp, pLine, phMod);
+        if (RT_FAILURE(rc))
+            rc = DBGFR3AsLineByAddr(pUVM, DBGF_AS_GLOBAL, pAddress, poffDisp, pLine, phMod);
+        return rc;
+    }
+
+    /*
+     * Input validation.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(DBGFR3AddrIsValid(pUVM, pAddress), VERR_INVALID_PARAMETER);
+    AssertPtrNullReturn(poffDisp, VERR_INVALID_POINTER);
+    AssertPtrReturn(pLine, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(phMod, VERR_INVALID_POINTER);
+    if (poffDisp)
+        *poffDisp = 0;
+    if (phMod)
+        *phMod = NIL_RTDBGMOD;
+    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pUVM, hDbgAs);
+    if (hRealAS == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+    /*
+     * Do the lookup.
+     */
+    int rc = RTDbgAsLineByAddr(hRealAS, pAddress->FlatPtr, poffDisp, pLine, phMod);
+
+    RTDbgAsRelease(hRealAS);
+    return rc;
+}
+
+
+VMMR3DECL(PRTDBGLINE)   DBGFR3AsLineByAddrA(PUVM pUVM, RTDBGAS hDbgAs, PCDBGFADDRESS pAddress,
+                                            PRTGCINTPTR poffDisp, PRTDBGMOD phMod)
+{
+    RTDBGLINE Line;
+    int rc = DBGFR3AsLineByAddr(pUVM, hDbgAs, pAddress, poffDisp, &Line, phMod);
+    if (RT_SUCCESS(rc))
+        return RTDbgLineDup(&Line);
+    return NULL;
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFBp.cpp b/src/VBox/VMM/VMMR3/DBGFBp.cpp
new file mode 100644
index 00000000..8c8f8539
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFBp.cpp
@@ -0,0 +1,1426 @@
+/* $Id: DBGFBp.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Breakpoint Management.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/hm.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * DBGF INT3-breakpoint set callback arguments.
+ */
+typedef struct DBGFBPINT3ARGS
+{
+    /** The source virtual CPU ID (used for breakpoint address resolution). */
+    VMCPUID         idSrcCpu;
+    /** The breakpoint address. */
+    PCDBGFADDRESS   pAddress;
+    /** The hit count at which the breakpoint starts triggering. */
+    uint64_t        iHitTrigger;
+    /** The hit count at which disables the breakpoint. */
+    uint64_t        iHitDisable;
+    /** Where to store the breakpoint Id (optional). */
+    uint32_t       *piBp;
+} DBGFBPINT3ARGS;
+/** Pointer to a DBGF INT3 breakpoint set callback argument. */
+typedef DBGFBPINT3ARGS *PDBGFBPINT3ARGS;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+static int dbgfR3BpRegArm(PVM pVM, PDBGFBP pBp);
+static int dbgfR3BpInt3Arm(PVM pVM, PDBGFBP pBp);
+RT_C_DECLS_END
+
+
+
+/**
+ * Initialize the breakpoint stuff.
+ *
+ * @returns VINF_SUCCESS
+ * @param   pVM     The cross context VM structure.
+ */
+int dbgfR3BpInit(PVM pVM)
+{
+    /*
+     * Init structures.
+     */
+    unsigned i;
+    for (i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); i++)
+    {
+        pVM->dbgf.s.aHwBreakpoints[i].iBp = i;
+        pVM->dbgf.s.aHwBreakpoints[i].enmType = DBGFBPTYPE_FREE;
+        pVM->dbgf.s.aHwBreakpoints[i].u.Reg.iReg = i;
+    }
+
+    for (i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints); i++)
+    {
+        pVM->dbgf.s.aBreakpoints[i].iBp = i + RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
+        pVM->dbgf.s.aBreakpoints[i].enmType = DBGFBPTYPE_FREE;
+    }
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        pVCpu->dbgf.s.iActiveBp = ~0U;
+    }
+
+    /*
+     * Register saved state.
+     */
+    /** @todo */
+
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Allocate a breakpoint.
+ *
+ * @returns Pointer to the allocated breakpoint.
+ * @returns NULL if we're out of breakpoints.
+ * @param   pVM     The cross context VM structure.
+ * @param   enmType The type to allocate.
+ */
+static PDBGFBP dbgfR3BpAlloc(PVM pVM, DBGFBPTYPE enmType)
+{
+    /*
+     * Determine which array to search and where in the array to start
+     * searching (latter for grouping similar BPs, reducing runtime overhead).
+     */
+    unsigned    iStart;
+    unsigned    cBps;
+    PDBGFBP     paBps;
+    switch (enmType)
+    {
+        case DBGFBPTYPE_REG:
+            cBps = RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
+            paBps = &pVM->dbgf.s.aHwBreakpoints[0];
+            iStart = 0;
+            break;
+
+        case DBGFBPTYPE_INT3:
+        case DBGFBPTYPE_REM:
+        case DBGFBPTYPE_PORT_IO:
+        case DBGFBPTYPE_MMIO:
+            cBps = RT_ELEMENTS(pVM->dbgf.s.aBreakpoints);
+            paBps = &pVM->dbgf.s.aBreakpoints[0];
+            if (enmType == DBGFBPTYPE_PORT_IO)
+                iStart = cBps / 4 * 2;
+            else if (enmType == DBGFBPTYPE_MMIO)
+                iStart = cBps / 4 * 1;
+            else if (enmType == DBGFBPTYPE_REM)
+                iStart = cBps / 4 * 3;
+            else
+                iStart = 0;
+            break;
+
+        default:
+            AssertMsgFailed(("enmType=%d\n", enmType));
+            return NULL;
+    }
+
+    /*
+     * Search for a free breakpoint entry.
+     */
+    unsigned iBp;
+    for (iBp = iStart; iBp < cBps; iBp++)
+        if (paBps[iBp].enmType == DBGFBPTYPE_FREE)
+            break;
+    if (iBp >= cBps && iStart != 0)
+        for (iBp = 0; iBp < cBps; iBp++)
+            if (paBps[iBp].enmType == DBGFBPTYPE_FREE)
+                break;
+    if (iBp < cBps)
+    {
+        /*
+         * Return what we found.
+         */
+        paBps[iBp].fEnabled = false;
+        paBps[iBp].cHits    = 0;
+        paBps[iBp].enmType  = enmType;
+        return &paBps[iBp];
+    }
+
+    LogFlow(("dbgfR3BpAlloc: returns NULL - we're out of breakpoint slots! cBps=%u\n", cBps));
+    return NULL;
+}
+
+
+/**
+ * Updates the search optimization structure for enabled breakpoints of the
+ * specified type.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmType     The breakpoint type.
+ * @param   pOpt        The breakpoint optimization structure to update.
+ */
+static int dbgfR3BpUpdateSearchOptimizations(PVM pVM, DBGFBPTYPE enmType, PDBGFBPSEARCHOPT pOpt)
+{
+    DBGFBPSEARCHOPT Opt  = { UINT32_MAX, 0 };
+
+    for (uint32_t iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints); iBp++)
+        if (   pVM->dbgf.s.aBreakpoints[iBp].enmType == enmType
+            && pVM->dbgf.s.aBreakpoints[iBp].fEnabled)
+        {
+            if (Opt.iStartSearch > iBp)
+                Opt.iStartSearch = iBp;
+            Opt.cToSearch = iBp - Opt.iStartSearch + 1;
+        }
+
+    *pOpt = Opt;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get a breakpoint give by breakpoint id.
+ *
+ * @returns Pointer to the allocated breakpoint.
+ * @returns NULL if the breakpoint is invalid.
+ * @param   pVM     The cross context VM structure.
+ * @param   iBp     The breakpoint id.
+ */
+static PDBGFBP dbgfR3BpGet(PVM pVM, uint32_t iBp)
+{
+    /* Find it. */
+    PDBGFBP pBp;
+    if (iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints))
+        pBp = &pVM->dbgf.s.aHwBreakpoints[iBp];
+    else
+    {
+        iBp -= RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
+        if (iBp >= RT_ELEMENTS(pVM->dbgf.s.aBreakpoints))
+            return NULL;
+        pBp = &pVM->dbgf.s.aBreakpoints[iBp];
+    }
+
+    /* check if it's valid. */
+    switch (pBp->enmType)
+    {
+        case DBGFBPTYPE_FREE:
+            return NULL;
+
+        case DBGFBPTYPE_REG:
+        case DBGFBPTYPE_INT3:
+        case DBGFBPTYPE_REM:
+        case DBGFBPTYPE_PORT_IO:
+        case DBGFBPTYPE_MMIO:
+            break;
+
+        default:
+            AssertMsgFailed(("Invalid enmType=%d!\n", pBp->enmType));
+            return NULL;
+    }
+
+    return pBp;
+}
+
+
+/**
+ * Get a breakpoint give by address.
+ *
+ * @returns Pointer to the allocated breakpoint.
+ * @returns NULL if the breakpoint is invalid.
+ * @param   pVM     The cross context VM structure.
+ * @param   enmType The breakpoint type.
+ * @param   GCPtr   The breakpoint address.
+ */
+static PDBGFBP dbgfR3BpGetByAddr(PVM pVM, DBGFBPTYPE enmType, RTGCUINTPTR GCPtr)
+{
+    /*
+     * Determine which array to search.
+     */
+    unsigned cBps;
+    PDBGFBP  paBps;
+    switch (enmType)
+    {
+        case DBGFBPTYPE_REG:
+            cBps = RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
+            paBps = &pVM->dbgf.s.aHwBreakpoints[0];
+            break;
+
+        case DBGFBPTYPE_INT3:
+        case DBGFBPTYPE_REM:
+            cBps = RT_ELEMENTS(pVM->dbgf.s.aBreakpoints);
+            paBps = &pVM->dbgf.s.aBreakpoints[0];
+            break;
+
+        default:
+            AssertMsgFailed(("enmType=%d\n", enmType));
+            return NULL;
+    }
+
+    /*
+     * Search.
+     */
+    for (unsigned iBp = 0; iBp < cBps; iBp++)
+        if (   paBps[iBp].enmType == enmType
+            && paBps[iBp].u.GCPtr == GCPtr)
+            return &paBps[iBp];
+
+    return NULL;
+}
+
+
+/**
+ * Frees a breakpoint.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   pBp     The breakpoint to free.
+ */
+static void dbgfR3BpFree(PVM pVM, PDBGFBP pBp)
+{
+    switch (pBp->enmType)
+    {
+        case DBGFBPTYPE_FREE:
+            AssertMsgFailed(("Already freed!\n"));
+            return;
+
+        case DBGFBPTYPE_REG:
+            Assert((uintptr_t)(pBp - &pVM->dbgf.s.aHwBreakpoints[0]) < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints));
+            break;
+
+        case DBGFBPTYPE_INT3:
+        case DBGFBPTYPE_REM:
+        case DBGFBPTYPE_PORT_IO:
+        case DBGFBPTYPE_MMIO:
+            Assert((uintptr_t)(pBp - &pVM->dbgf.s.aBreakpoints[0]) < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints));
+            break;
+
+        default:
+            AssertMsgFailed(("Invalid enmType=%d!\n", pBp->enmType));
+            return;
+
+    }
+    pBp->enmType = DBGFBPTYPE_FREE;
+    NOREF(pVM);
+}
+
+
+/**
+ * @callback_method_impl{FNVMMEMTRENDEZVOUS}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) dbgfR3BpEnableInt3OnCpu(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    /*
+     * Validate input.
+     */
+    PDBGFBP pBp = (PDBGFBP)pvUser;
+    AssertReturn(pBp, VERR_INVALID_PARAMETER);
+    Assert(pBp->enmType == DBGFBPTYPE_INT3);
+    VMCPU_ASSERT_EMT(pVCpu); RT_NOREF(pVCpu);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Arm the breakpoint.
+     */
+    return dbgfR3BpInt3Arm(pVM, pBp);
+}
+
+
+/**
+ * @callback_method_impl{FNVMMEMTRENDEZVOUS}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) dbgfR3BpSetInt3OnCpu(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    /*
+     * Validate input.
+     */
+    PDBGFBPINT3ARGS pBpArgs = (PDBGFBPINT3ARGS)pvUser;
+    AssertReturn(pBpArgs, VERR_INVALID_PARAMETER);
+    VMCPU_ASSERT_EMT(pVCpu);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    AssertMsgReturn(!pBpArgs->piBp || VALID_PTR(pBpArgs->piBp), ("piBp=%p\n", pBpArgs->piBp), VERR_INVALID_POINTER);
+    PCDBGFADDRESS pAddress = pBpArgs->pAddress;
+    if (!DBGFR3AddrIsValid(pVM->pUVM, pAddress))
+        return VERR_INVALID_PARAMETER;
+
+    if (pBpArgs->iHitTrigger > pBpArgs->iHitDisable)
+        return VERR_INVALID_PARAMETER;
+
+    /*
+     * Check if we're on the source CPU where we can resolve the breakpoint address.
+     */
+    if (pVCpu->idCpu == pBpArgs->idSrcCpu)
+    {
+        if (pBpArgs->piBp)
+            *pBpArgs->piBp = UINT32_MAX;
+
+        /*
+         * Check if the breakpoint already exists.
+         */
+        PDBGFBP pBp = dbgfR3BpGetByAddr(pVM, DBGFBPTYPE_INT3, pAddress->FlatPtr);
+        if (pBp)
+        {
+            int rc = VINF_SUCCESS;
+            if (!pBp->fEnabled)
+                rc = dbgfR3BpInt3Arm(pVM, pBp);
+            if (RT_SUCCESS(rc))
+            {
+                if (pBpArgs->piBp)
+                    *pBpArgs->piBp = pBp->iBp;
+
+                /*
+                 * Returning VINF_DBGF_BP_ALREADY_EXIST here causes a VBOXSTRICTRC out-of-range assertion
+                 * in VMMR3EmtRendezvous(). Re-setting of an existing breakpoint shouldn't cause an assertion
+                 * killing the VM (and debugging session), so for now we'll pretend success.
+                 */
+#if 0
+                rc = VINF_DBGF_BP_ALREADY_EXIST;
+#endif
+            }
+            else
+                dbgfR3BpFree(pVM, pBp);
+            return rc;
+        }
+
+        /*
+         * Allocate the breakpoint.
+         */
+        pBp = dbgfR3BpAlloc(pVM, DBGFBPTYPE_INT3);
+        if (!pBp)
+            return VERR_DBGF_NO_MORE_BP_SLOTS;
+
+        /*
+         * Translate & save the breakpoint address into a guest-physical address.
+         */
+        int rc = DBGFR3AddrToPhys(pVM->pUVM, pBpArgs->idSrcCpu, pAddress, &pBp->u.Int3.PhysAddr);
+        if (RT_SUCCESS(rc))
+        {
+            /* The physical address from DBGFR3AddrToPhys() is the start of the page,
+               we need the exact byte offset into the page while writing to it in dbgfR3BpInt3Arm(). */
+            pBp->u.Int3.PhysAddr |= (pAddress->FlatPtr & X86_PAGE_OFFSET_MASK);
+            pBp->u.Int3.GCPtr = pAddress->FlatPtr;
+            pBp->iHitTrigger  = pBpArgs->iHitTrigger;
+            pBp->iHitDisable  = pBpArgs->iHitDisable;
+
+            /*
+             * Now set the breakpoint in guest memory.
+             */
+            rc = dbgfR3BpInt3Arm(pVM, pBp);
+            if (RT_SUCCESS(rc))
+            {
+                if (pBpArgs->piBp)
+                    *pBpArgs->piBp = pBp->iBp;
+                return VINF_SUCCESS;
+            }
+        }
+
+        dbgfR3BpFree(pVM, pBp);
+        return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets a breakpoint (int 3 based).
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   idSrcCpu    The ID of the virtual CPU used for the
+ *                      breakpoint address resolution.
+ * @param   pAddress    The address of the breakpoint.
+ * @param   iHitTrigger The hit count at which the breakpoint start triggering.
+ *                      Use 0 (or 1) if it's gonna trigger at once.
+ * @param   iHitDisable The hit count which disables the breakpoint.
+ *                      Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   piBp        Where to store the breakpoint id. (optional)
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3BpSetInt3(PUVM pUVM, VMCPUID idSrcCpu, PCDBGFADDRESS pAddress, uint64_t iHitTrigger, uint64_t iHitDisable,
+                               uint32_t *piBp)
+{
+    AssertReturn(idSrcCpu <= pUVM->cCpus, VERR_INVALID_CPU_ID);
+
+    DBGFBPINT3ARGS BpArgs;
+    RT_ZERO(BpArgs);
+    BpArgs.idSrcCpu    = idSrcCpu;
+    BpArgs.iHitTrigger = iHitTrigger;
+    BpArgs.iHitDisable = iHitDisable;
+    BpArgs.pAddress    = pAddress;
+    BpArgs.piBp        = piBp;
+
+    int rc = VMMR3EmtRendezvous(pUVM->pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, dbgfR3BpSetInt3OnCpu, &BpArgs);
+    LogFlow(("DBGFR3BpSet: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Arms an int 3 breakpoint.
+ *
+ * This is used to implement both DBGFR3BpSetInt3() and
+ * DBGFR3BpEnable().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pBp         The breakpoint.
+ */
+static int dbgfR3BpInt3Arm(PVM pVM, PDBGFBP pBp)
+{
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Save current byte and write the int3 instruction byte.
+     */
+    int rc = PGMPhysSimpleReadGCPhys(pVM, &pBp->u.Int3.bOrg, pBp->u.Int3.PhysAddr, sizeof(pBp->u.Int3.bOrg));
+    if (RT_SUCCESS(rc))
+    {
+        static const uint8_t s_bInt3 = 0xcc;
+        rc = PGMPhysSimpleWriteGCPhys(pVM, pBp->u.Int3.PhysAddr, &s_bInt3, sizeof(s_bInt3));
+        if (RT_SUCCESS(rc))
+        {
+            pBp->fEnabled = true;
+            dbgfR3BpUpdateSearchOptimizations(pVM, DBGFBPTYPE_INT3, &pVM->dbgf.s.Int3);
+            pVM->dbgf.s.cEnabledInt3Breakpoints = pVM->dbgf.s.Int3.cToSearch;
+            Log(("DBGF: Set breakpoint at %RGv (Phys %RGp) cEnabledInt3Breakpoints=%u\n", pBp->u.Int3.GCPtr,
+                 pBp->u.Int3.PhysAddr, pVM->dbgf.s.cEnabledInt3Breakpoints));
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Disarms an int 3 breakpoint.
+ *
+ * This is used to implement both DBGFR3BpClear() and DBGFR3BpDisable().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pBp         The breakpoint.
+ */
+static int dbgfR3BpInt3Disarm(PVM pVM, PDBGFBP pBp)
+{
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Check that the current byte is the int3 instruction, and restore the original one.
+     * We currently ignore invalid bytes.
+     */
+    uint8_t bCurrent = 0;
+    int rc = PGMPhysSimpleReadGCPhys(pVM, &bCurrent, pBp->u.Int3.PhysAddr, sizeof(bCurrent));
+    if (   RT_SUCCESS(rc)
+        && bCurrent == 0xcc)
+    {
+        rc = PGMPhysSimpleWriteGCPhys(pVM, pBp->u.Int3.PhysAddr, &pBp->u.Int3.bOrg, sizeof(pBp->u.Int3.bOrg));
+        if (RT_SUCCESS(rc))
+        {
+            pBp->fEnabled = false;
+            dbgfR3BpUpdateSearchOptimizations(pVM, DBGFBPTYPE_INT3, &pVM->dbgf.s.Int3);
+            pVM->dbgf.s.cEnabledInt3Breakpoints = pVM->dbgf.s.Int3.cToSearch;
+            Log(("DBGF: Removed breakpoint at %RGv (Phys %RGp) cEnabledInt3Breakpoints=%u\n", pBp->u.Int3.GCPtr,
+                 pBp->u.Int3.PhysAddr, pVM->dbgf.s.cEnabledInt3Breakpoints));
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNVMMEMTRENDEZVOUS}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) dbgfR3BpDisableInt3OnCpu(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    /*
+     * Validate input.
+     */
+    PDBGFBP pBp = (PDBGFBP)pvUser;
+    AssertReturn(pBp, VERR_INVALID_PARAMETER);
+    Assert(pBp->enmType == DBGFBPTYPE_INT3);
+    VMCPU_ASSERT_EMT(pVCpu); RT_NOREF(pVCpu);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Disarm the breakpoint.
+     */
+    return dbgfR3BpInt3Disarm(pVM, pBp);
+}
+
+
+/**
+ * Sets a register breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pAddress        The address of the breakpoint.
+ * @param   piHitTrigger    The hit count at which the breakpoint start triggering.
+ *                          Use 0 (or 1) if it's gonna trigger at once.
+ * @param   piHitDisable    The hit count which disables the breakpoint.
+ *                          Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   fType           The access type (one of the X86_DR7_RW_* defines).
+ * @param   cb              The access size - 1,2,4 or 8 (the latter is AMD64 long mode only.
+ *                          Must be 1 if fType is X86_DR7_RW_EO.
+ * @param   piBp            Where to store the breakpoint id. (optional)
+ * @thread  EMT
+ * @internal
+ */
+static DECLCALLBACK(int) dbgfR3BpSetReg(PUVM pUVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable,
+                                        uint8_t fType, uint8_t cb, uint32_t *piBp)
+{
+    /*
+     * Validate input.
+     */
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    if (!DBGFR3AddrIsValid(pUVM, pAddress))
+        return VERR_INVALID_PARAMETER;
+    if (*piHitTrigger > *piHitDisable)
+        return VERR_INVALID_PARAMETER;
+    AssertMsgReturn(!piBp || VALID_PTR(piBp), ("piBp=%p\n", piBp), VERR_INVALID_POINTER);
+    if (piBp)
+        *piBp = UINT32_MAX;
+    switch (fType)
+    {
+        case X86_DR7_RW_EO:
+            if (cb == 1)
+                break;
+            AssertMsgFailed(("fType=%#x cb=%d != 1\n", fType, cb));
+            return VERR_INVALID_PARAMETER;
+        case X86_DR7_RW_IO:
+        case X86_DR7_RW_RW:
+        case X86_DR7_RW_WO:
+            break;
+        default:
+            AssertMsgFailed(("fType=%#x\n", fType));
+            return VERR_INVALID_PARAMETER;
+    }
+    switch (cb)
+    {
+        case 1:
+        case 2:
+        case 4:
+            break;
+        default:
+            AssertMsgFailed(("cb=%#x\n", cb));
+            return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Check if the breakpoint already exists.
+     */
+    PDBGFBP pBp = dbgfR3BpGetByAddr(pVM, DBGFBPTYPE_REG, pAddress->FlatPtr);
+    if (    pBp
+        &&  pBp->u.Reg.cb == cb
+        &&  pBp->u.Reg.fType == fType)
+    {
+        int rc = VINF_SUCCESS;
+        if (!pBp->fEnabled)
+            rc = dbgfR3BpRegArm(pVM, pBp);
+        if (RT_SUCCESS(rc))
+        {
+            rc = VINF_DBGF_BP_ALREADY_EXIST;
+            if (piBp)
+                *piBp = pBp->iBp;
+        }
+        return rc;
+    }
+
+    /*
+     * Allocate and initialize the bp.
+     */
+    pBp = dbgfR3BpAlloc(pVM, DBGFBPTYPE_REG);
+    if (!pBp)
+        return VERR_DBGF_NO_MORE_BP_SLOTS;
+    pBp->iHitTrigger = *piHitTrigger;
+    pBp->iHitDisable = *piHitDisable;
+    Assert(pBp->iBp == pBp->u.Reg.iReg);
+    pBp->u.Reg.GCPtr = pAddress->FlatPtr;
+    pBp->u.Reg.fType = fType;
+    pBp->u.Reg.cb    = cb;
+    ASMCompilerBarrier();
+    pBp->fEnabled    = true;
+
+    /*
+     * Arm the breakpoint.
+     */
+    int rc = dbgfR3BpRegArm(pVM, pBp);
+    if (RT_SUCCESS(rc))
+    {
+        if (piBp)
+            *piBp = pBp->iBp;
+    }
+    else
+        dbgfR3BpFree(pVM, pBp);
+
+    return rc;
+}
+
+
+/**
+ * Sets a register breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pAddress    The address of the breakpoint.
+ * @param   iHitTrigger The hit count at which the breakpoint start triggering.
+ *                      Use 0 (or 1) if it's gonna trigger at once.
+ * @param   iHitDisable The hit count which disables the breakpoint.
+ *                      Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   fType       The access type (one of the X86_DR7_RW_* defines).
+ * @param   cb          The access size - 1,2,4 or 8 (the latter is AMD64 long mode only.
+ *                      Must be 1 if fType is X86_DR7_RW_EO.
+ * @param   piBp        Where to store the breakpoint id. (optional)
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3BpSetReg(PUVM pUVM, PCDBGFADDRESS pAddress, uint64_t iHitTrigger, uint64_t iHitDisable,
+                              uint8_t fType, uint8_t cb, uint32_t *piBp)
+{
+    /*
+     * This must be done on EMT.
+     */
+    int rc = VMR3ReqCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3BpSetReg, 7,
+                              pUVM, pAddress, &iHitTrigger, &iHitDisable, fType, cb, piBp);
+    LogFlow(("DBGFR3BpSetReg: returns %Rrc\n", rc));
+    return rc;
+
+}
+
+
+/**
+ * @callback_method_impl{FNVMMEMTRENDEZVOUS}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) dbgfR3BpRegRecalcOnCpu(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    NOREF(pVM); NOREF(pvUser);
+
+    /*
+     * CPU 0 updates the enabled hardware breakpoint counts.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        pVM->dbgf.s.cEnabledHwBreakpoints   = 0;
+        pVM->dbgf.s.cEnabledHwIoBreakpoints = 0;
+
+        for (uint32_t iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); iBp++)
+            if (   pVM->dbgf.s.aHwBreakpoints[iBp].fEnabled
+                && pVM->dbgf.s.aHwBreakpoints[iBp].enmType == DBGFBPTYPE_REG)
+            {
+                pVM->dbgf.s.cEnabledHwBreakpoints   += 1;
+                pVM->dbgf.s.cEnabledHwIoBreakpoints += pVM->dbgf.s.aHwBreakpoints[iBp].u.Reg.fType == X86_DR7_RW_IO;
+            }
+    }
+
+    return CPUMRecalcHyperDRx(pVCpu, UINT8_MAX, false);
+}
+
+
+/**
+ * Arms a debug register breakpoint.
+ *
+ * This is used to implement both DBGFR3BpSetReg() and DBGFR3BpEnable().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pBp         The breakpoint.
+ * @thread  EMT(0)
+ */
+static int dbgfR3BpRegArm(PVM pVM, PDBGFBP pBp)
+{
+    RT_NOREF_PV(pBp);
+    Assert(pBp->fEnabled);
+    return VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, dbgfR3BpRegRecalcOnCpu, NULL);
+}
+
+
+/**
+ * Disarms a debug register breakpoint.
+ *
+ * This is used to implement both DBGFR3BpClear() and DBGFR3BpDisable().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pBp         The breakpoint.
+ * @thread  EMT(0)
+ */
+static int dbgfR3BpRegDisarm(PVM pVM, PDBGFBP pBp)
+{
+    RT_NOREF_PV(pBp);
+    Assert(!pBp->fEnabled);
+    return VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, dbgfR3BpRegRecalcOnCpu, NULL);
+}
+
+
+/**
+ * EMT worker for DBGFR3BpSetREM().
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pAddress        The address of the breakpoint.
+ * @param   piHitTrigger    The hit count at which the breakpoint start triggering.
+ *                          Use 0 (or 1) if it's gonna trigger at once.
+ * @param   piHitDisable    The hit count which disables the breakpoint.
+ *                          Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   piBp            Where to store the breakpoint id. (optional)
+ * @thread  EMT(0)
+ * @internal
+ */
+static DECLCALLBACK(int) dbgfR3BpSetREM(PUVM pUVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger,
+                                        uint64_t *piHitDisable, uint32_t *piBp)
+{
+    /*
+     * Validate input.
+     */
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    if (!DBGFR3AddrIsValid(pUVM, pAddress))
+        return VERR_INVALID_PARAMETER;
+    if (*piHitTrigger > *piHitDisable)
+        return VERR_INVALID_PARAMETER;
+    AssertMsgReturn(!piBp || VALID_PTR(piBp), ("piBp=%p\n", piBp), VERR_INVALID_POINTER);
+    if (piBp)
+        *piBp = UINT32_MAX;
+
+    /*
+     * Check if the breakpoint already exists.
+     */
+    PDBGFBP pBp = dbgfR3BpGetByAddr(pVM, DBGFBPTYPE_REM, pAddress->FlatPtr);
+    if (pBp)
+    {
+        int rc = VINF_SUCCESS;
+        if (!pBp->fEnabled)
+            rc = IEMBreakpointSet(pVM, pBp->u.Rem.GCPtr);
+        if (RT_SUCCESS(rc))
+        {
+            rc = VINF_DBGF_BP_ALREADY_EXIST;
+            if (piBp)
+                *piBp = pBp->iBp;
+        }
+        return rc;
+    }
+
+    /*
+     * Allocate and initialize the bp.
+     */
+    pBp = dbgfR3BpAlloc(pVM, DBGFBPTYPE_REM);
+    if (!pBp)
+        return VERR_DBGF_NO_MORE_BP_SLOTS;
+    pBp->u.Rem.GCPtr = pAddress->FlatPtr;
+    pBp->iHitTrigger = *piHitTrigger;
+    pBp->iHitDisable = *piHitDisable;
+    ASMCompilerBarrier();
+    pBp->fEnabled    = true;
+
+    /*
+     * Now ask REM to set the breakpoint.
+     */
+    int rc = IEMBreakpointSet(pVM, pAddress->FlatPtr);
+    if (RT_SUCCESS(rc))
+    {
+        if (piBp)
+            *piBp = pBp->iBp;
+    }
+    else
+        dbgfR3BpFree(pVM, pBp);
+
+    return rc;
+}
+
+
+/**
+ * Sets a recompiler breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pAddress    The address of the breakpoint.
+ * @param   iHitTrigger The hit count at which the breakpoint start triggering.
+ *                      Use 0 (or 1) if it's gonna trigger at once.
+ * @param   iHitDisable The hit count which disables the breakpoint.
+ *                      Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   piBp        Where to store the breakpoint id. (optional)
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3BpSetREM(PUVM pUVM, PCDBGFADDRESS pAddress, uint64_t iHitTrigger, uint64_t iHitDisable, uint32_t *piBp)
+{
+    /*
+     * This must be done on EMT.
+     */
+    int rc = VMR3ReqCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3BpSetREM, 5,
+                              pUVM, pAddress, &iHitTrigger, &iHitDisable, piBp);
+    LogFlow(("DBGFR3BpSetREM: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Updates IOM on whether we've got any armed I/O port or MMIO breakpoints.
+ *
+ * @returns VINF_SUCCESS
+ * @param   pVM         The cross context VM structure.
+ * @thread  EMT(0)
+ */
+static int dbgfR3BpUpdateIom(PVM pVM)
+{
+    dbgfR3BpUpdateSearchOptimizations(pVM, DBGFBPTYPE_PORT_IO, &pVM->dbgf.s.PortIo);
+    if (pVM->dbgf.s.PortIo.cToSearch)
+        ASMAtomicBitSet(&pVM->dbgf.s.bmSelectedEvents, DBGFEVENT_BREAKPOINT_IO);
+    else
+        ASMAtomicBitClear(&pVM->dbgf.s.bmSelectedEvents, DBGFEVENT_BREAKPOINT_IO);
+
+    dbgfR3BpUpdateSearchOptimizations(pVM, DBGFBPTYPE_MMIO, &pVM->dbgf.s.Mmio);
+    if (pVM->dbgf.s.Mmio.cToSearch)
+        ASMAtomicBitSet(&pVM->dbgf.s.bmSelectedEvents, DBGFEVENT_BREAKPOINT_MMIO);
+    else
+        ASMAtomicBitClear(&pVM->dbgf.s.bmSelectedEvents, DBGFEVENT_BREAKPOINT_MMIO);
+
+    IOMR3NotifyBreakpointCountChange(pVM, pVM->dbgf.s.PortIo.cToSearch != 0, pVM->dbgf.s.Mmio.cToSearch != 0);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * EMT worker for DBGFR3BpSetPortIo.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   uPort           The first I/O port.
+ * @param   cPorts          The number of I/O ports.
+ * @param   fAccess         The access we want to break on.
+ * @param   piHitTrigger    The hit count at which the breakpoint start triggering.
+ *                          Use 0 (or 1) if it's gonna trigger at once.
+ * @param   piHitDisable    The hit count which disables the breakpoint.
+ *                          Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   piBp            Where to store the breakpoint ID.
+ * @thread  EMT(0)
+ */
+static DECLCALLBACK(int) dbgfR3BpSetPortIo(PUVM pUVM, RTIOPORT uPort, RTIOPORT cPorts, uint32_t fAccess,
+                                           uint64_t const *piHitTrigger, uint64_t const *piHitDisable, uint32_t *piBp)
+{
+    /*
+     * Validate input.
+     */
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    *piBp = UINT32_MAX;
+
+    /*
+     * Check if the breakpoint already exists.
+     */
+    for (uint32_t i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints); i++)
+        if (   pVM->dbgf.s.aBreakpoints[i].enmType == DBGFBPTYPE_PORT_IO
+            && pVM->dbgf.s.aBreakpoints[i].u.PortIo.uPort == uPort
+            && pVM->dbgf.s.aBreakpoints[i].u.PortIo.cPorts == cPorts
+            && pVM->dbgf.s.aBreakpoints[i].u.PortIo.fAccess == fAccess)
+        {
+            if (!pVM->dbgf.s.aBreakpoints[i].fEnabled)
+            {
+                pVM->dbgf.s.aBreakpoints[i].fEnabled = true;
+                dbgfR3BpUpdateIom(pVM);
+            }
+            *piBp = pVM->dbgf.s.aBreakpoints[i].iBp;
+            return VINF_DBGF_BP_ALREADY_EXIST;
+        }
+
+    /*
+     * Allocate and initialize the breakpoint.
+     */
+    PDBGFBP pBp = dbgfR3BpAlloc(pVM, DBGFBPTYPE_PORT_IO);
+    if (!pBp)
+        return VERR_DBGF_NO_MORE_BP_SLOTS;
+    pBp->iHitTrigger        = *piHitTrigger;
+    pBp->iHitDisable        = *piHitDisable;
+    pBp->u.PortIo.uPort     = uPort;
+    pBp->u.PortIo.cPorts    = cPorts;
+    pBp->u.PortIo.fAccess   = fAccess;
+    ASMCompilerBarrier();
+    pBp->fEnabled           = true;
+
+    /*
+     * Tell IOM.
+     */
+    dbgfR3BpUpdateIom(pVM);
+    *piBp = pBp->iBp;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets an I/O port breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   uPort           The first I/O port.
+ * @param   cPorts          The number of I/O ports, see DBGFBPIOACCESS_XXX.
+ * @param   fAccess         The access we want to break on.
+ * @param   iHitTrigger     The hit count at which the breakpoint start
+ *                          triggering. Use 0 (or 1) if it's gonna trigger at
+ *                          once.
+ * @param   iHitDisable     The hit count which disables the breakpoint.
+ *                          Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   piBp            Where to store the breakpoint ID. Optional.
+ * @thread  Any thread.
+ */
+VMMR3DECL(int)  DBGFR3BpSetPortIo(PUVM pUVM, RTIOPORT uPort, RTIOPORT cPorts, uint32_t fAccess,
+                                  uint64_t iHitTrigger, uint64_t iHitDisable, uint32_t *piBp)
+{
+    AssertReturn(!(fAccess & ~DBGFBPIOACCESS_VALID_MASK_PORT_IO), VERR_INVALID_FLAGS);
+    AssertReturn(fAccess, VERR_INVALID_FLAGS);
+    if (iHitTrigger > iHitDisable)
+        return VERR_INVALID_PARAMETER;
+    AssertPtrNullReturn(piBp, VERR_INVALID_POINTER);
+    AssertReturn(cPorts > 0, VERR_OUT_OF_RANGE);
+    AssertReturn((RTIOPORT)(uPort + cPorts) < uPort, VERR_OUT_OF_RANGE);
+
+    /*
+     * This must be done on EMT.
+     */
+    uint32_t iBp = UINT32_MAX;
+    int rc = VMR3ReqCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3BpSetPortIo, 7,
+                              pUVM, uPort, cPorts, fAccess, &iHitTrigger, &iHitDisable, piBp);
+    if (piBp)
+        *piBp = iBp;
+    LogFlow(("DBGFR3BpSetPortIo: returns %Rrc iBp=%d\n", rc, iBp));
+    return rc;
+}
+
+
+/**
+ * EMT worker for DBGFR3BpSetMmio.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pGCPhys         The start of the MMIO range to break on.
+ * @param   cb              The size of the MMIO range.
+ * @param   fAccess         The access we want to break on.
+ * @param   piHitTrigger    The hit count at which the breakpoint start triggering.
+ *                          Use 0 (or 1) if it's gonna trigger at once.
+ * @param   piHitDisable    The hit count which disables the breakpoint.
+ *                          Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   piBp            Where to store the breakpoint ID.
+ * @thread  EMT(0)
+ */
+static DECLCALLBACK(int) dbgfR3BpSetMmio(PUVM pUVM, PCRTGCPHYS pGCPhys, uint32_t cb, uint32_t fAccess,
+                                         uint64_t const *piHitTrigger, uint64_t const *piHitDisable, uint32_t *piBp)
+{
+    RTGCPHYS const GCPhys = *pGCPhys;
+
+    /*
+     * Validate input.
+     */
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    *piBp = UINT32_MAX;
+
+    /*
+     * Check if the breakpoint already exists.
+     */
+    for (uint32_t i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints); i++)
+        if (   pVM->dbgf.s.aBreakpoints[i].enmType == DBGFBPTYPE_MMIO
+            && pVM->dbgf.s.aBreakpoints[i].u.Mmio.PhysAddr == GCPhys
+            && pVM->dbgf.s.aBreakpoints[i].u.Mmio.cb == cb
+            && pVM->dbgf.s.aBreakpoints[i].u.Mmio.fAccess == fAccess)
+        {
+            if (!pVM->dbgf.s.aBreakpoints[i].fEnabled)
+            {
+                pVM->dbgf.s.aBreakpoints[i].fEnabled = true;
+                dbgfR3BpUpdateIom(pVM);
+            }
+            *piBp = pVM->dbgf.s.aBreakpoints[i].iBp;
+            return VINF_DBGF_BP_ALREADY_EXIST;
+        }
+
+    /*
+     * Allocate and initialize the breakpoint.
+     */
+    PDBGFBP pBp = dbgfR3BpAlloc(pVM, DBGFBPTYPE_MMIO);
+    if (!pBp)
+        return VERR_DBGF_NO_MORE_BP_SLOTS;
+    pBp->iHitTrigger        = *piHitTrigger;
+    pBp->iHitDisable        = *piHitDisable;
+    pBp->u.Mmio.PhysAddr    = GCPhys;
+    pBp->u.Mmio.cb          = cb;
+    pBp->u.Mmio.fAccess     = fAccess;
+    ASMCompilerBarrier();
+    pBp->fEnabled           = true;
+
+    /*
+     * Tell IOM.
+     */
+    dbgfR3BpUpdateIom(pVM);
+    *piBp = pBp->iBp;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets a memory mapped I/O breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   GCPhys          The first MMIO address.
+ * @param   cb              The size of the MMIO range to break on.
+ * @param   fAccess         The access we want to break on.
+ * @param   iHitTrigger     The hit count at which the breakpoint start
+ *                          triggering. Use 0 (or 1) if it's gonna trigger at
+ *                          once.
+ * @param   iHitDisable     The hit count which disables the breakpoint.
+ *                          Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   piBp            Where to store the breakpoint ID. Optional.
+ * @thread  Any thread.
+ */
+VMMR3DECL(int)  DBGFR3BpSetMmio(PUVM pUVM, RTGCPHYS GCPhys, uint32_t cb, uint32_t fAccess,
+                                uint64_t iHitTrigger, uint64_t iHitDisable, uint32_t *piBp)
+{
+    AssertReturn(!(fAccess & ~DBGFBPIOACCESS_VALID_MASK_MMIO), VERR_INVALID_FLAGS);
+    AssertReturn(fAccess, VERR_INVALID_FLAGS);
+    if (iHitTrigger > iHitDisable)
+        return VERR_INVALID_PARAMETER;
+    AssertPtrNullReturn(piBp, VERR_INVALID_POINTER);
+    AssertReturn(cb, VERR_OUT_OF_RANGE);
+    AssertReturn(GCPhys + cb < GCPhys, VERR_OUT_OF_RANGE);
+
+    /*
+     * This must be done on EMT.
+     */
+    uint32_t iBp = UINT32_MAX;
+    int rc = VMR3ReqCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3BpSetMmio, 7,
+                              pUVM, &GCPhys, cb, fAccess, &iHitTrigger, &iHitDisable, piBp);
+    if (piBp)
+        *piBp = iBp;
+    LogFlow(("DBGFR3BpSetMmio: returns %Rrc iBp=%d\n", rc, iBp));
+    return rc;
+}
+
+
+/**
+ * EMT worker for DBGFR3BpClear().
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   iBp         The id of the breakpoint which should be removed (cleared).
+ * @thread  EMT(0)
+ * @internal
+ */
+static DECLCALLBACK(int) dbgfR3BpClear(PUVM pUVM, uint32_t iBp)
+{
+    /*
+     * Validate input.
+     */
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_EMT(pVM);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    PDBGFBP pBp = dbgfR3BpGet(pVM, iBp);
+    if (!pBp)
+        return VERR_DBGF_BP_NOT_FOUND;
+
+    /*
+     * Disarm the breakpoint if it's enabled.
+     */
+    if (pBp->fEnabled)
+    {
+        pBp->fEnabled = false;
+        int rc;
+        switch (pBp->enmType)
+        {
+            case DBGFBPTYPE_REG:
+                rc = dbgfR3BpRegDisarm(pVM, pBp);
+                break;
+
+            case DBGFBPTYPE_INT3:
+                rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, dbgfR3BpDisableInt3OnCpu, pBp);
+                break;
+
+            case DBGFBPTYPE_REM:
+                rc = IEMBreakpointClear(pVM, pBp->u.Rem.GCPtr);
+                break;
+
+            case DBGFBPTYPE_PORT_IO:
+            case DBGFBPTYPE_MMIO:
+                rc = dbgfR3BpUpdateIom(pVM);
+                break;
+
+            default:
+                AssertMsgFailedReturn(("Invalid enmType=%d!\n", pBp->enmType), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+        }
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Free the breakpoint.
+     */
+    dbgfR3BpFree(pVM, pBp);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Clears a breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   iBp         The id of the breakpoint which should be removed (cleared).
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3BpClear(PUVM pUVM, uint32_t iBp)
+{
+    /*
+     * This must be done on EMT.
+     */
+    int rc = VMR3ReqCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3BpClear, 2, pUVM, iBp);
+    LogFlow(("DBGFR3BpClear: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * EMT worker for DBGFR3BpEnable().
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   iBp         The id of the breakpoint which should be enabled.
+ * @thread  EMT(0)
+ * @internal
+ */
+static DECLCALLBACK(int) dbgfR3BpEnable(PUVM pUVM, uint32_t iBp)
+{
+    /*
+     * Validate input.
+     */
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    PDBGFBP pBp = dbgfR3BpGet(pVM, iBp);
+    if (!pBp)
+        return VERR_DBGF_BP_NOT_FOUND;
+
+    /*
+     * Already enabled?
+     */
+    if (pBp->fEnabled)
+        return VINF_DBGF_BP_ALREADY_ENABLED;
+
+    /*
+     * Arm the breakpoint.
+     */
+    int rc;
+    pBp->fEnabled = true;
+    switch (pBp->enmType)
+    {
+        case DBGFBPTYPE_REG:
+            rc = dbgfR3BpRegArm(pVM, pBp);
+            break;
+
+        case DBGFBPTYPE_INT3:
+            rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, dbgfR3BpEnableInt3OnCpu, pBp);
+            break;
+
+        case DBGFBPTYPE_REM:
+            rc = IEMBreakpointSet(pVM, pBp->u.Rem.GCPtr);
+            break;
+
+        case DBGFBPTYPE_PORT_IO:
+        case DBGFBPTYPE_MMIO:
+            rc = dbgfR3BpUpdateIom(pVM);
+            break;
+
+        default:
+            AssertMsgFailedReturn(("Invalid enmType=%d!\n", pBp->enmType), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+    if (RT_FAILURE(rc))
+        pBp->fEnabled = false;
+
+    return rc;
+}
+
+
+/**
+ * Enables a breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   iBp         The id of the breakpoint which should be enabled.
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3BpEnable(PUVM pUVM, uint32_t iBp)
+{
+    /*
+     * This must be done on EMT.
+     */
+    int rc = VMR3ReqCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3BpEnable, 2, pUVM, iBp);
+    LogFlow(("DBGFR3BpEnable: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * EMT worker for DBGFR3BpDisable().
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   iBp         The id of the breakpoint which should be disabled.
+ * @thread  EMT(0)
+ * @internal
+ */
+static DECLCALLBACK(int) dbgfR3BpDisable(PUVM pUVM, uint32_t iBp)
+{
+    /*
+     * Validate input.
+     */
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    PDBGFBP pBp = dbgfR3BpGet(pVM, iBp);
+    if (!pBp)
+        return VERR_DBGF_BP_NOT_FOUND;
+
+    /*
+     * Already enabled?
+     */
+    if (!pBp->fEnabled)
+        return VINF_DBGF_BP_ALREADY_DISABLED;
+
+    /*
+     * Remove the breakpoint.
+     */
+    pBp->fEnabled = false;
+    int rc;
+    switch (pBp->enmType)
+    {
+        case DBGFBPTYPE_REG:
+            rc = dbgfR3BpRegDisarm(pVM, pBp);
+            break;
+
+        case DBGFBPTYPE_INT3:
+            rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, dbgfR3BpDisableInt3OnCpu, pBp);
+            break;
+
+        case DBGFBPTYPE_REM:
+            rc = IEMBreakpointClear(pVM, pBp->u.Rem.GCPtr);
+            break;
+
+        case DBGFBPTYPE_PORT_IO:
+        case DBGFBPTYPE_MMIO:
+            rc = dbgfR3BpUpdateIom(pVM);
+            break;
+
+        default:
+            AssertMsgFailedReturn(("Invalid enmType=%d!\n", pBp->enmType), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Disables a breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   iBp         The id of the breakpoint which should be disabled.
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3BpDisable(PUVM pUVM, uint32_t iBp)
+{
+    /*
+     * This must be done on EMT.
+     */
+    int rc = VMR3ReqCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3BpDisable, 2, pUVM, iBp);
+    LogFlow(("DBGFR3BpDisable: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * EMT worker for DBGFR3BpEnum().
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pfnCallback The callback function.
+ * @param   pvUser      The user argument to pass to the callback.
+ * @thread  EMT
+ * @internal
+ */
+static DECLCALLBACK(int) dbgfR3BpEnum(PUVM pUVM, PFNDBGFBPENUM pfnCallback, void *pvUser)
+{
+    /*
+     * Validate input.
+     */
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pfnCallback, VERR_INVALID_POINTER);
+
+    /*
+     * Enumerate the hardware breakpoints.
+     */
+    unsigned i;
+    for (i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); i++)
+        if (pVM->dbgf.s.aHwBreakpoints[i].enmType != DBGFBPTYPE_FREE)
+        {
+            int rc = pfnCallback(pUVM, pvUser, &pVM->dbgf.s.aHwBreakpoints[i]);
+            if (RT_FAILURE(rc) || rc == VINF_CALLBACK_RETURN)
+                return rc;
+        }
+
+    /*
+     * Enumerate the other breakpoints.
+     */
+    for (i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints); i++)
+        if (pVM->dbgf.s.aBreakpoints[i].enmType != DBGFBPTYPE_FREE)
+        {
+            int rc = pfnCallback(pUVM, pvUser, &pVM->dbgf.s.aBreakpoints[i]);
+            if (RT_FAILURE(rc) || rc == VINF_CALLBACK_RETURN)
+                return rc;
+        }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Enumerate the breakpoints.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pfnCallback The callback function.
+ * @param   pvUser      The user argument to pass to the callback.
+ * @thread  Any thread but the callback will be called from EMT.
+ */
+VMMR3DECL(int) DBGFR3BpEnum(PUVM pUVM, PFNDBGFBPENUM pfnCallback, void *pvUser)
+{
+    /*
+     * This must be done on EMT.
+     */
+    int rc = VMR3ReqPriorityCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3BpEnum, 3, pUVM, pfnCallback, pvUser);
+    LogFlow(("DBGFR3BpEnum: returns %Rrc\n", rc));
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFCoreWrite.cpp b/src/VBox/VMM/VMMR3/DBGFCoreWrite.cpp
new file mode 100644
index 00000000..f49413e7
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFCoreWrite.cpp
@@ -0,0 +1,664 @@
+/* $Id: DBGFCoreWrite.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Guest Core Dump.
+ */
+
+/*
+ * Copyright (C) 2010-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_dbgf_vmcore    VMCore Format
+ *
+ * The VirtualBox VMCore Format:
+ * [ ELF 64 Header]  -- Only 1
+ *
+ * [ PT_NOTE ]       -- Only 1
+ *    - Offset into CoreDescriptor followed by list of Notes (Note Hdr + data) of VBox CPUs.
+ *    - (Any Additional custom Note sections).
+ *
+ * [ PT_LOAD ]       -- One for each contiguous memory chunk
+ *    - Memory offset (physical).
+ *    - File offset.
+ *
+ * CoreDescriptor
+ *    - Magic, VBox version.
+ *    - Number of CPus.
+ *
+ * Per-CPU register dump
+ *    - CPU 1 Note Hdr + Data.
+ *    - CPU 2 Note Hdr + Data.
+ *    ...
+ * (Additional custom notes Hdr+data)
+ *    - VBox 1 Note Hdr + Data.
+ *    - VBox 2 Note Hdr + Data.
+ *    ...
+ * Memory dump
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <iprt/param.h>
+#include <iprt/file.h>
+#include <iprt/mem.h>
+#include <iprt/formats/elf64.h>
+
+#include "DBGFInternal.h"
+
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/dbgfcorefmt.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/version.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#define DBGFLOG_NAME           "DBGFCoreWrite"
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+static const int g_NoteAlign  = 8;
+static const int g_cbNoteName = 16;
+
+/* The size of these strings (incl. NULL terminator) must align to 8 bytes (g_NoteAlign) and -not- 4 bytes. */
+static const char *g_pcszCoreVBoxCore = "VBCORE";
+static const char *g_pcszCoreVBoxCpu  = "VBCPU";
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Guest core writer data.
+ *
+ * Used to pass parameters from DBGFR3CoreWrite to dbgfR3CoreWriteRendezvous().
+ */
+typedef struct DBGFCOREDATA
+{
+    /** The name of the file to write the file to. */
+    const char *pszFilename;
+    /** Whether to replace (/overwrite) any existing file. */
+    bool        fReplaceFile;
+} DBGFCOREDATA;
+/** Pointer to the guest core writer data.  */
+typedef DBGFCOREDATA *PDBGFCOREDATA;
+
+
+
+/**
+ * ELF function to write 64-bit ELF header.
+ *
+ * @param   hFile           The file to write to.
+ * @param   cProgHdrs       Number of program headers.
+ * @param   cSecHdrs        Number of section headers.
+ *
+ * @return IPRT status code.
+ */
+static int Elf64WriteElfHdr(RTFILE hFile, uint16_t cProgHdrs, uint16_t cSecHdrs)
+{
+    Elf64_Ehdr ElfHdr;
+    RT_ZERO(ElfHdr);
+    ElfHdr.e_ident[EI_MAG0]  = ELFMAG0;
+    ElfHdr.e_ident[EI_MAG1]  = ELFMAG1;
+    ElfHdr.e_ident[EI_MAG2]  = ELFMAG2;
+    ElfHdr.e_ident[EI_MAG3]  = ELFMAG3;
+    ElfHdr.e_ident[EI_DATA]  = ELFDATA2LSB;
+    ElfHdr.e_type            = ET_CORE;
+    ElfHdr.e_version         = EV_CURRENT;
+    ElfHdr.e_ident[EI_CLASS] = ELFCLASS64;
+    /* 32-bit builds will produce cores with e_machine EM_386. */
+#ifdef RT_ARCH_AMD64
+    ElfHdr.e_machine         = EM_X86_64;
+#else
+    ElfHdr.e_machine         = EM_386;
+#endif
+    ElfHdr.e_phnum           = cProgHdrs;
+    ElfHdr.e_shnum           = cSecHdrs;
+    ElfHdr.e_ehsize          = sizeof(ElfHdr);
+    ElfHdr.e_phoff           = sizeof(ElfHdr);
+    ElfHdr.e_phentsize       = sizeof(Elf64_Phdr);
+    ElfHdr.e_shentsize       = sizeof(Elf64_Shdr);
+
+    return RTFileWrite(hFile, &ElfHdr, sizeof(ElfHdr), NULL /* all */);
+}
+
+
+/**
+ * ELF function to write 64-bit program header.
+ *
+ * @param   hFile           The file to write to.
+ * @param   Type            Type of program header (PT_*).
+ * @param   fFlags          Flags (access permissions, PF_*).
+ * @param   offFileData     File offset of contents.
+ * @param   cbFileData      Size of contents in the file.
+ * @param   cbMemData       Size of contents in memory.
+ * @param   Phys            Physical address, pass zero if not applicable.
+ *
+ * @return IPRT status code.
+ */
+static int Elf64WriteProgHdr(RTFILE hFile, uint32_t Type, uint32_t fFlags, uint64_t offFileData, uint64_t cbFileData,
+                             uint64_t cbMemData, RTGCPHYS Phys)
+{
+    Elf64_Phdr ProgHdr;
+    RT_ZERO(ProgHdr);
+    ProgHdr.p_type   = Type;
+    ProgHdr.p_flags  = fFlags;
+    ProgHdr.p_offset = offFileData;
+    ProgHdr.p_filesz = cbFileData;
+    ProgHdr.p_memsz  = cbMemData;
+    ProgHdr.p_paddr  = Phys;
+
+    return RTFileWrite(hFile, &ProgHdr, sizeof(ProgHdr), NULL /* all */);
+}
+
+
+/**
+ * Returns the size of the NOTE section given the name and size of the data.
+ *
+ * @param   pszName         Name of the note section.
+ * @param   cbData          Size of the data portion of the note section.
+ *
+ * @return The size of the NOTE section as rounded to the file alignment.
+ */
+static uint64_t Elf64NoteSectionSize(const char *pszName, uint64_t cbData)
+{
+    uint64_t cbNote = sizeof(Elf64_Nhdr);
+
+    size_t cbName      = strlen(pszName) + 1;
+    size_t cbNameAlign = RT_ALIGN_Z(cbName, g_NoteAlign);
+
+    cbNote += cbNameAlign;
+    cbNote += RT_ALIGN_64(cbData, g_NoteAlign);
+    return cbNote;
+}
+
+
+/**
+ * Elf function to write 64-bit note header.
+ *
+ * @param   hFile       The file to write to.
+ * @param   Type        Type of this section.
+ * @param   pszName     Name of this section.
+ * @param   pvData      Opaque pointer to the data, if NULL only computes size.
+ * @param   cbData      Size of the data.
+ *
+ * @returns IPRT status code.
+ */
+static int Elf64WriteNoteHdr(RTFILE hFile, uint16_t Type, const char *pszName, const void *pvData, uint64_t cbData)
+{
+    AssertReturn(pvData, VERR_INVALID_POINTER);
+    AssertReturn(cbData > 0, VERR_NO_DATA);
+
+    char szNoteName[g_cbNoteName];
+    RT_ZERO(szNoteName);
+    RTStrCopy(szNoteName, sizeof(szNoteName), pszName);
+
+    size_t   cbName      = strlen(szNoteName) + 1;
+    size_t   cbNameAlign = RT_ALIGN_Z(cbName, g_NoteAlign);
+    uint64_t cbDataAlign = RT_ALIGN_64(cbData, g_NoteAlign);
+
+    /*
+     * Yell loudly and bail if we are going to be writing a core file that is not compatible with
+     * both Solaris and the 64-bit ELF spec. which dictates 8-byte alignment. See @bugref{5211#c3}.
+     */
+    if (cbNameAlign - cbName > 3)
+    {
+        LogRel((DBGFLOG_NAME ": Elf64WriteNoteHdr pszName=%s cbName=%u cbNameAlign=%u, cbName aligns to 4 not 8-bytes!\n",
+                pszName, cbName, cbNameAlign));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (cbDataAlign - cbData > 3)
+    {
+        LogRel((DBGFLOG_NAME ": Elf64WriteNoteHdr pszName=%s cbData=%u cbDataAlign=%u, cbData aligns to 4 not 8-bytes!\n",
+                pszName, cbData, cbDataAlign));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    static const char s_achPad[7] = { 0, 0, 0, 0, 0, 0, 0 };
+    AssertCompile(sizeof(s_achPad) >= g_NoteAlign - 1);
+
+    Elf64_Nhdr ElfNoteHdr;
+    RT_ZERO(ElfNoteHdr);
+    ElfNoteHdr.n_namesz = (Elf64_Word)cbName - 1;    /* Again, a discrepancy between ELF-64 and Solaris,
+                                                        we will follow ELF-64, see @bugref{5211#c3}. */
+    ElfNoteHdr.n_type   = Type;
+    ElfNoteHdr.n_descsz = (Elf64_Word)cbDataAlign;
+
+    /*
+     * Write note header.
+     */
+    int rc = RTFileWrite(hFile, &ElfNoteHdr, sizeof(ElfNoteHdr), NULL /* all */);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Write note name.
+         */
+        rc = RTFileWrite(hFile, szNoteName, cbName, NULL /* all */);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Write note name padding if required.
+             */
+            if (cbNameAlign > cbName)
+                rc = RTFileWrite(hFile, s_achPad, cbNameAlign - cbName, NULL);
+
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Write note data.
+                 */
+                rc = RTFileWrite(hFile, pvData, cbData, NULL /* all */);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Write note data padding if required.
+                     */
+                    if (cbDataAlign > cbData)
+                        rc = RTFileWrite(hFile, s_achPad, cbDataAlign - cbData, NULL /* all*/);
+                }
+            }
+        }
+    }
+
+    if (RT_FAILURE(rc))
+        LogRel((DBGFLOG_NAME ": RTFileWrite failed. rc=%Rrc pszName=%s cbName=%u cbNameAlign=%u cbData=%u cbDataAlign=%u\n",
+                rc, pszName, cbName, cbNameAlign, cbData, cbDataAlign));
+
+    return rc;
+}
+
+
+/**
+ * Count the number of memory ranges that go into the core file.
+ *
+ * We cannot do a page-by-page dump of the entire guest memory as there will be
+ * way too many program header entries. Also we don't want to dump MMIO regions
+ * which means we cannot have a 1:1 mapping between core file offset and memory
+ * offset. Instead we dump the memory in ranges. A memory range is a contiguous
+ * memory area suitable for dumping to a core file.
+ *
+ * @param   pVM             The cross context VM structure.
+ *
+ * @return Number of memory ranges
+ */
+static uint32_t dbgfR3GetRamRangeCount(PVM pVM)
+{
+    return PGMR3PhysGetRamRangeCount(pVM);
+}
+
+
+/**
+ * Gets the guest-CPU context suitable for dumping into the core file.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pDbgfCpu    Where to dump the guest-CPU data.
+ */
+static void dbgfR3GetCoreCpu(PVMCPU pVCpu, PDBGFCORECPU pDbgfCpu)
+{
+#define DBGFCOPYSEL(a_dbgfsel, a_cpumselreg) \
+    do { \
+        (a_dbgfsel).uBase  = (a_cpumselreg).u64Base; \
+        (a_dbgfsel).uLimit = (a_cpumselreg).u32Limit; \
+        (a_dbgfsel).uAttr  = (a_cpumselreg).Attr.u; \
+        (a_dbgfsel).uSel   = (a_cpumselreg).Sel; \
+    } while (0)
+
+    PVM       pVM  = pVCpu->CTX_SUFF(pVM);
+    PCCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
+    pDbgfCpu->rax             = pCtx->rax;
+    pDbgfCpu->rbx             = pCtx->rbx;
+    pDbgfCpu->rcx             = pCtx->rcx;
+    pDbgfCpu->rdx             = pCtx->rdx;
+    pDbgfCpu->rsi             = pCtx->rsi;
+    pDbgfCpu->rdi             = pCtx->rdi;
+    pDbgfCpu->r8              = pCtx->r8;
+    pDbgfCpu->r9              = pCtx->r9;
+    pDbgfCpu->r10             = pCtx->r10;
+    pDbgfCpu->r11             = pCtx->r11;
+    pDbgfCpu->r12             = pCtx->r12;
+    pDbgfCpu->r13             = pCtx->r13;
+    pDbgfCpu->r14             = pCtx->r14;
+    pDbgfCpu->r15             = pCtx->r15;
+    pDbgfCpu->rip             = pCtx->rip;
+    pDbgfCpu->rsp             = pCtx->rsp;
+    pDbgfCpu->rbp             = pCtx->rbp;
+    pDbgfCpu->rflags          = pCtx->rflags.u;
+    DBGFCOPYSEL(pDbgfCpu->cs, pCtx->cs);
+    DBGFCOPYSEL(pDbgfCpu->ds, pCtx->ds);
+    DBGFCOPYSEL(pDbgfCpu->es, pCtx->es);
+    DBGFCOPYSEL(pDbgfCpu->fs, pCtx->fs);
+    DBGFCOPYSEL(pDbgfCpu->gs, pCtx->gs);
+    DBGFCOPYSEL(pDbgfCpu->ss, pCtx->ss);
+    pDbgfCpu->cr0             = pCtx->cr0;
+    pDbgfCpu->cr2             = pCtx->cr2;
+    pDbgfCpu->cr3             = pCtx->cr3;
+    pDbgfCpu->cr4             = pCtx->cr4;
+    AssertCompile(RT_ELEMENTS(pDbgfCpu->dr) == RT_ELEMENTS(pCtx->dr));
+    for (unsigned i = 0; i < RT_ELEMENTS(pDbgfCpu->dr); i++)
+        pDbgfCpu->dr[i] = pCtx->dr[i];
+    pDbgfCpu->gdtr.uAddr      = pCtx->gdtr.pGdt;
+    pDbgfCpu->gdtr.cb         = pCtx->gdtr.cbGdt;
+    pDbgfCpu->idtr.uAddr      = pCtx->idtr.pIdt;
+    pDbgfCpu->idtr.cb         = pCtx->idtr.cbIdt;
+    DBGFCOPYSEL(pDbgfCpu->ldtr, pCtx->ldtr);
+    DBGFCOPYSEL(pDbgfCpu->tr,   pCtx->tr);
+    pDbgfCpu->sysenter.cs     = pCtx->SysEnter.cs;
+    pDbgfCpu->sysenter.eip    = pCtx->SysEnter.eip;
+    pDbgfCpu->sysenter.esp    = pCtx->SysEnter.esp;
+    pDbgfCpu->msrEFER         = pCtx->msrEFER;
+    pDbgfCpu->msrSTAR         = pCtx->msrSTAR;
+    pDbgfCpu->msrPAT          = pCtx->msrPAT;
+    pDbgfCpu->msrLSTAR        = pCtx->msrLSTAR;
+    pDbgfCpu->msrCSTAR        = pCtx->msrCSTAR;
+    pDbgfCpu->msrSFMASK       = pCtx->msrSFMASK;
+    pDbgfCpu->msrKernelGSBase = pCtx->msrKERNELGSBASE;
+    pDbgfCpu->msrApicBase     = APICGetBaseMsrNoCheck(pVCpu);
+    pDbgfCpu->aXcr[0]         = pCtx->aXcr[0];
+    pDbgfCpu->aXcr[1]         = pCtx->aXcr[1];
+    AssertCompile(sizeof(pDbgfCpu->ext) == sizeof(*pCtx->pXStateR3));
+    pDbgfCpu->cbExt = pVM->cpum.ro.GuestFeatures.cbMaxExtendedState;
+    if (RT_LIKELY(pDbgfCpu->cbExt))
+        memcpy(&pDbgfCpu->ext, pCtx->pXStateR3, pDbgfCpu->cbExt);
+
+#undef DBGFCOPYSEL
+}
+
+
+/**
+ * Worker function for dbgfR3CoreWrite() which does the writing.
+ *
+ * @returns VBox status code
+ * @param   pVM                 The cross context VM structure.
+ * @param   hFile               The file to write to.  Caller closes this.
+ */
+static int dbgfR3CoreWriteWorker(PVM pVM, RTFILE hFile)
+{
+    /*
+     * Collect core information.
+     */
+    uint32_t const cu32MemRanges = dbgfR3GetRamRangeCount(pVM);
+    uint16_t const cMemRanges    = cu32MemRanges < UINT16_MAX - 1 ? cu32MemRanges : UINT16_MAX - 1; /* One PT_NOTE Program header */
+    uint16_t const cProgHdrs     = cMemRanges + 1;
+
+    DBGFCOREDESCRIPTOR CoreDescriptor;
+    RT_ZERO(CoreDescriptor);
+    CoreDescriptor.u32Magic           = DBGFCORE_MAGIC;
+    CoreDescriptor.u32FmtVersion      = DBGFCORE_FMT_VERSION;
+    CoreDescriptor.cbSelf             = sizeof(CoreDescriptor);
+    CoreDescriptor.u32VBoxVersion     = VBOX_FULL_VERSION;
+    CoreDescriptor.u32VBoxRevision    = VMMGetSvnRev();
+    CoreDescriptor.cCpus              = pVM->cCpus;
+
+    Log((DBGFLOG_NAME ": CoreDescriptor Version=%u Revision=%u\n", CoreDescriptor.u32VBoxVersion, CoreDescriptor.u32VBoxRevision));
+
+    /*
+     * Compute the file layout (see pg_dbgf_vmcore).
+     */
+    uint64_t const offElfHdr          = RTFileTell(hFile);
+    uint64_t const offNoteSection     = offElfHdr         + sizeof(Elf64_Ehdr);
+    uint64_t const offLoadSections    = offNoteSection    + sizeof(Elf64_Phdr);
+    uint64_t const cbLoadSections     = cMemRanges * sizeof(Elf64_Phdr);
+    uint64_t const offCoreDescriptor  = offLoadSections   + cbLoadSections;
+    uint64_t const cbCoreDescriptor   = Elf64NoteSectionSize(g_pcszCoreVBoxCore, sizeof(CoreDescriptor));
+    uint64_t const offCpuDumps        = offCoreDescriptor + cbCoreDescriptor;
+    uint64_t const cbCpuDumps         = pVM->cCpus * Elf64NoteSectionSize(g_pcszCoreVBoxCpu, sizeof(DBGFCORECPU));
+    uint64_t const offMemory          = offCpuDumps       + cbCpuDumps;
+
+    uint64_t const offNoteSectionData = offCoreDescriptor;
+    uint64_t const cbNoteSectionData  = cbCoreDescriptor + cbCpuDumps;
+
+    /*
+     * Write ELF header.
+     */
+    int rc = Elf64WriteElfHdr(hFile, cProgHdrs, 0 /* cSecHdrs */);
+    if (RT_FAILURE(rc))
+    {
+        LogRel((DBGFLOG_NAME ": Elf64WriteElfHdr failed. rc=%Rrc\n", rc));
+        return rc;
+    }
+
+    /*
+     * Write PT_NOTE program header.
+     */
+    Assert(RTFileTell(hFile) == offNoteSection);
+    rc = Elf64WriteProgHdr(hFile, PT_NOTE, PF_R,
+                           offNoteSectionData,  /* file offset to contents */
+                           cbNoteSectionData,   /* size in core file */
+                           cbNoteSectionData,   /* size in memory */
+                           0);                  /* physical address */
+    if (RT_FAILURE(rc))
+    {
+        LogRel((DBGFLOG_NAME ": Elf64WritreProgHdr failed for PT_NOTE. rc=%Rrc\n", rc));
+        return rc;
+    }
+
+    /*
+     * Write PT_LOAD program header for each memory range.
+     */
+    Assert(RTFileTell(hFile) == offLoadSections);
+    uint64_t offMemRange = offMemory;
+    for (uint16_t iRange = 0; iRange < cMemRanges; iRange++)
+    {
+        RTGCPHYS    GCPhysStart;
+        RTGCPHYS    GCPhysEnd;
+        bool        fIsMmio;
+        rc = PGMR3PhysGetRange(pVM, iRange, &GCPhysStart, &GCPhysEnd, NULL /* pszDesc */, &fIsMmio);
+        if (RT_FAILURE(rc))
+        {
+            LogRel((DBGFLOG_NAME ": PGMR3PhysGetRange failed for iRange(%u) rc=%Rrc\n", iRange, rc));
+            return rc;
+        }
+
+        uint64_t cbMemRange  = GCPhysEnd - GCPhysStart + 1;
+        uint64_t cbFileRange = fIsMmio ? 0 : cbMemRange;
+
+        Log((DBGFLOG_NAME ": PGMR3PhysGetRange iRange=%u GCPhysStart=%#x GCPhysEnd=%#x cbMemRange=%u\n",
+             iRange, GCPhysStart, GCPhysEnd, cbMemRange));
+
+        rc = Elf64WriteProgHdr(hFile, PT_LOAD, PF_R,
+                               offMemRange,                         /* file offset to contents */
+                               cbFileRange,                         /* size in core file */
+                               cbMemRange,                          /* size in memory */
+                               GCPhysStart);                        /* physical address */
+        if (RT_FAILURE(rc))
+        {
+            LogRel((DBGFLOG_NAME ": Elf64WriteProgHdr failed for memory range(%u) cbFileRange=%u cbMemRange=%u rc=%Rrc\n",
+                    iRange, cbFileRange, cbMemRange, rc));
+            return rc;
+        }
+
+        offMemRange += cbFileRange;
+    }
+
+    /*
+     * Write the Core descriptor note header and data.
+     */
+    Assert(RTFileTell(hFile) == offCoreDescriptor);
+    rc = Elf64WriteNoteHdr(hFile, NT_VBOXCORE, g_pcszCoreVBoxCore, &CoreDescriptor, sizeof(CoreDescriptor));
+    if (RT_FAILURE(rc))
+    {
+        LogRel((DBGFLOG_NAME ": Elf64WriteNoteHdr failed for Note '%s' rc=%Rrc\n", g_pcszCoreVBoxCore, rc));
+        return rc;
+    }
+
+    /*
+     * Write the CPU context note headers and data.
+     * We allocate the DBGFCORECPU struct. rather than using the stack as it can be pretty large due to X86XSAVEAREA.
+     */
+    Assert(RTFileTell(hFile) == offCpuDumps);
+    PDBGFCORECPU pDbgfCoreCpu = (PDBGFCORECPU)RTMemAlloc(sizeof(*pDbgfCoreCpu));
+    if (RT_UNLIKELY(!pDbgfCoreCpu))
+    {
+        LogRel((DBGFLOG_NAME ": Failed to alloc %u bytes for DBGFCORECPU\n", sizeof(*pDbgfCoreCpu)));
+        return VERR_NO_MEMORY;
+    }
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        RT_BZERO(pDbgfCoreCpu, sizeof(*pDbgfCoreCpu));
+        dbgfR3GetCoreCpu(pVCpu, pDbgfCoreCpu);
+
+        rc = Elf64WriteNoteHdr(hFile, NT_VBOXCPU, g_pcszCoreVBoxCpu, pDbgfCoreCpu, sizeof(*pDbgfCoreCpu));
+        if (RT_FAILURE(rc))
+        {
+            LogRel((DBGFLOG_NAME ": Elf64WriteNoteHdr failed for vCPU[%u] rc=%Rrc\n", idCpu, rc));
+            RTMemFree(pDbgfCoreCpu);
+            return rc;
+        }
+    }
+    RTMemFree(pDbgfCoreCpu);
+    pDbgfCoreCpu = NULL;
+
+    /*
+     * Write memory ranges.
+     */
+    Assert(RTFileTell(hFile) == offMemory);
+    for (uint16_t iRange = 0; iRange < cMemRanges; iRange++)
+    {
+        RTGCPHYS GCPhysStart;
+        RTGCPHYS GCPhysEnd;
+        bool     fIsMmio;
+        rc = PGMR3PhysGetRange(pVM, iRange, &GCPhysStart, &GCPhysEnd, NULL /* pszDesc */, &fIsMmio);
+        if (RT_FAILURE(rc))
+        {
+            LogRel((DBGFLOG_NAME ": PGMR3PhysGetRange(2) failed for iRange(%u) rc=%Rrc\n", iRange, rc));
+            return rc;
+        }
+
+        if (fIsMmio)
+            continue;
+
+        /*
+         * Write page-by-page of this memory range.
+         *
+         * The read function may fail on MMIO ranges, we write these as zero
+         * pages for now (would be nice to have the VGA bits there though).
+         */
+        uint64_t cbMemRange  = GCPhysEnd - GCPhysStart + 1;
+        uint64_t cPages      = cbMemRange >> PAGE_SHIFT;
+        for (uint64_t iPage = 0; iPage < cPages; iPage++)
+        {
+            uint8_t abPage[PAGE_SIZE];
+            rc = PGMPhysSimpleReadGCPhys(pVM, abPage, GCPhysStart + (iPage << PAGE_SHIFT),  sizeof(abPage));
+            if (RT_FAILURE(rc))
+            {
+                if (rc != VERR_PGM_PHYS_PAGE_RESERVED)
+                    LogRel((DBGFLOG_NAME ": PGMPhysRead failed for iRange=%u iPage=%u. rc=%Rrc. Ignoring...\n", iRange, iPage, rc));
+                RT_ZERO(abPage);
+            }
+
+            rc = RTFileWrite(hFile, abPage, sizeof(abPage), NULL /* all */);
+            if (RT_FAILURE(rc))
+            {
+                LogRel((DBGFLOG_NAME ": RTFileWrite failed. iRange=%u iPage=%u rc=%Rrc\n", iRange, iPage, rc));
+                return rc;
+            }
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * EMT Rendezvous worker function for DBGFR3CoreWrite().
+ *
+ * @param   pVM              The cross context VM structure.
+ * @param   pVCpu            The cross context virtual CPU structure of the calling EMT.
+ * @param   pvData           Opaque data.
+ *
+ * @return VBox status code.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) dbgfR3CoreWriteRendezvous(PVM pVM, PVMCPU pVCpu, void *pvData)
+{
+    /*
+     * Validate input.
+     */
+    AssertReturn(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(pVCpu, VERR_INVALID_VMCPU_HANDLE);
+    AssertReturn(pvData, VERR_INVALID_POINTER);
+
+    PDBGFCOREDATA pDbgfData = (PDBGFCOREDATA)pvData;
+
+    /*
+     * Create the core file.
+     */
+    uint32_t fFlags = (pDbgfData->fReplaceFile ? RTFILE_O_CREATE_REPLACE : RTFILE_O_CREATE)
+                    | RTFILE_O_WRITE
+                    | RTFILE_O_DENY_ALL
+                    | (0600 << RTFILE_O_CREATE_MODE_SHIFT);
+    RTFILE   hFile;
+    int rc = RTFileOpen(&hFile, pDbgfData->pszFilename, fFlags);
+    if (RT_SUCCESS(rc))
+    {
+        rc = dbgfR3CoreWriteWorker(pVM, hFile);
+        RTFileClose(hFile);
+    }
+    else
+        LogRel((DBGFLOG_NAME ": RTFileOpen failed for '%s' rc=%Rrc\n", pDbgfData->pszFilename, rc));
+    return rc;
+}
+
+
+/**
+ * Write core dump of the guest.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszFilename         The name of the file to which the guest core
+ *                              dump should be written.
+ * @param   fReplaceFile        Whether to replace the file or not.
+ *
+ * @remarks The VM may need to be suspended before calling this function in
+ *          order to truly stop all device threads and drivers. This function
+ *          only synchronizes EMTs.
+ */
+VMMR3DECL(int) DBGFR3CoreWrite(PUVM pUVM, const char *pszFilename, bool fReplaceFile)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(pszFilename, VERR_INVALID_HANDLE);
+
+    /*
+     * Pass the core write request down to EMT rendezvous which makes sure
+     * other EMTs, if any, are not running. IO threads could still be running
+     * but we don't care about them.
+     */
+    DBGFCOREDATA CoreData;
+    RT_ZERO(CoreData);
+    CoreData.pszFilename  = pszFilename;
+    CoreData.fReplaceFile = fReplaceFile;
+
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, dbgfR3CoreWriteRendezvous, &CoreData);
+    if (RT_SUCCESS(rc))
+        LogRel((DBGFLOG_NAME ": Successfully wrote guest core dump '%s'\n", pszFilename));
+    else
+        LogRel((DBGFLOG_NAME ": Failed to write guest core dump '%s'. rc=%Rrc\n", pszFilename, rc));
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFCpu.cpp b/src/VBox/VMM/VMMR3/DBGFCpu.cpp
new file mode 100644
index 00000000..98b8cbb9
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFCpu.cpp
@@ -0,0 +1,163 @@
+/* $Id: DBGFCpu.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, CPU State Accessors.
+ */
+
+/*
+ * Copyright (C) 2009-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#define VMCPU_INCL_CPUM_GST_CTX /* For CPUM_IMPORT_EXTRN_RET(). */
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/cpum.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <iprt/errcore.h>
+#include <VBox/log.h>
+#include <VBox/param.h>
+#include <iprt/assert.h>
+
+
+/**
+ * Wrapper around CPUMGetGuestMode.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVM         The cross context VM structure.
+ * @param   idCpu       The current CPU ID.
+ * @param   penmMode    Where to return the mode.
+ */
+static DECLCALLBACK(int) dbgfR3CpuGetMode(PVM pVM, VMCPUID idCpu, CPUMMODE *penmMode)
+{
+    Assert(idCpu == VMMGetCpuId(pVM));
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER);
+    *penmMode = CPUMGetGuestMode(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get the current CPU mode.
+ *
+ * @returns The CPU mode on success, CPUMMODE_INVALID on failure.
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The target CPU ID.
+ */
+VMMR3DECL(CPUMMODE) DBGFR3CpuGetMode(PUVM pUVM, VMCPUID idCpu)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, CPUMMODE_INVALID);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, CPUMMODE_INVALID);
+    AssertReturn(idCpu < pUVM->pVM->cCpus, CPUMMODE_INVALID);
+
+    CPUMMODE enmMode;
+    int rc = VMR3ReqPriorityCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3CpuGetMode, 3, pUVM->pVM, idCpu, &enmMode);
+    if (RT_FAILURE(rc))
+        return CPUMMODE_INVALID;
+    return enmMode;
+}
+
+
+/**
+ * Wrapper around CPUMIsGuestIn64BitCode.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVM             The cross context VM structure.
+ * @param   idCpu           The current CPU ID.
+ * @param   pfIn64BitCode   Where to return the result.
+ */
+static DECLCALLBACK(int) dbgfR3CpuIn64BitCode(PVM pVM, VMCPUID idCpu, bool *pfIn64BitCode)
+{
+    Assert(idCpu == VMMGetCpuId(pVM));
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
+    *pfIn64BitCode = CPUMIsGuestIn64BitCode(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the given CPU is executing 64-bit code or not.
+ *
+ * @returns true / false accordingly.
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The target CPU ID.
+ */
+VMMR3DECL(bool) DBGFR3CpuIsIn64BitCode(PUVM pUVM, VMCPUID idCpu)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, false);
+    AssertReturn(idCpu < pUVM->pVM->cCpus, false);
+
+    bool fIn64BitCode;
+    int rc = VMR3ReqPriorityCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3CpuIn64BitCode, 3, pUVM->pVM, idCpu, &fIn64BitCode);
+    if (RT_FAILURE(rc))
+        return false;
+    return fIn64BitCode;
+}
+
+
+/**
+ * Wrapper around CPUMIsGuestInV86Code.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVM             The cross context VM structure.
+ * @param   idCpu           The current CPU ID.
+ * @param   pfInV86Code     Where to return the result.
+ */
+static DECLCALLBACK(int) dbgfR3CpuInV86Code(PVM pVM, VMCPUID idCpu, bool *pfInV86Code)
+{
+    Assert(idCpu == VMMGetCpuId(pVM));
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_RFLAGS);
+    *pfInV86Code = CPUMIsGuestInV86ModeEx(CPUMQueryGuestCtxPtr(pVCpu));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the given CPU is executing V8086 code or not.
+ *
+ * @returns true / false accordingly.
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The target CPU ID.
+ */
+VMMR3DECL(bool) DBGFR3CpuIsInV86Code(PUVM pUVM, VMCPUID idCpu)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, false);
+    AssertReturn(idCpu < pUVM->pVM->cCpus, false);
+
+    bool fInV86Code;
+    int rc = VMR3ReqPriorityCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3CpuInV86Code, 3, pUVM->pVM, idCpu, &fInV86Code);
+    if (RT_FAILURE(rc))
+        return false;
+    return fInV86Code;
+}
+
+
+/**
+ * Get the number of CPUs (or threads if you insist).
+ *
+ * @returns The number of CPUs
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(VMCPUID) DBGFR3CpuGetCount(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, 1);
+    return pUVM->cCpus;
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFDisas.cpp b/src/VBox/VMM/VMMR3/DBGFDisas.cpp
new file mode 100644
index 00000000..065b63c4
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFDisas.cpp
@@ -0,0 +1,790 @@
+/* $Id: DBGFDisas.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Disassembler.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/cpum.h>
+#include "DBGFInternal.h"
+#include <VBox/dis.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/alloca.h>
+#include <iprt/ctype.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Structure used when disassembling and instructions in DBGF.
+ * This is used so the reader function can get the stuff it needs.
+ */
+typedef struct
+{
+    /** The core structure. */
+    DISCPUSTATE     Cpu;
+    /** The cross context VM structure. */
+    PVM             pVM;
+    /** The cross context virtual CPU structure. */
+    PVMCPU          pVCpu;
+    /** The address space for resolving symbol. */
+    RTDBGAS         hDbgAs;
+    /** Pointer to the first byte in the segment. */
+    RTGCUINTPTR     GCPtrSegBase;
+    /** Pointer to the byte after the end of the segment. (might have wrapped!) */
+    RTGCUINTPTR     GCPtrSegEnd;
+    /** The size of the segment minus 1. */
+    RTGCUINTPTR     cbSegLimit;
+    /** The guest paging mode. */
+    PGMMODE         enmMode;
+    /** Pointer to the current page - R3 Ptr. */
+    void const     *pvPageR3;
+    /** Pointer to the current page - GC Ptr. */
+    RTGCPTR         GCPtrPage;
+    /** Pointer to the next instruction (relative to GCPtrSegBase). */
+    RTGCUINTPTR     GCPtrNext;
+    /** The lock information that PGMPhysReleasePageMappingLock needs. */
+    PGMPAGEMAPLOCK  PageMapLock;
+    /** Whether the PageMapLock is valid or not. */
+    bool            fLocked;
+    /** 64 bits mode or not. */
+    bool            f64Bits;
+} DBGFDISASSTATE, *PDBGFDISASSTATE;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static FNDISREADBYTES dbgfR3DisasInstrRead;
+
+
+
+/**
+ * Calls the disassembler with the proper reader functions and such for disa
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pSelInfo    The selector info.
+ * @param   enmMode     The guest paging mode.
+ * @param   fFlags      DBGF_DISAS_FLAGS_XXX.
+ * @param   GCPtr       The GC pointer (selector offset).
+ * @param   pState      The disas CPU state.
+ */
+static int dbgfR3DisasInstrFirst(PVM pVM, PVMCPU pVCpu, PDBGFSELINFO pSelInfo, PGMMODE enmMode,
+                                 RTGCPTR GCPtr, uint32_t fFlags, PDBGFDISASSTATE pState)
+{
+    pState->GCPtrSegBase    = pSelInfo->GCPtrBase;
+    pState->GCPtrSegEnd     = pSelInfo->cbLimit + 1 + (RTGCUINTPTR)pSelInfo->GCPtrBase;
+    pState->cbSegLimit      = pSelInfo->cbLimit;
+    pState->enmMode         = enmMode;
+    pState->GCPtrPage       = 0;
+    pState->pvPageR3        = NULL;
+    pState->hDbgAs          = DBGF_AS_GLOBAL;
+    pState->pVM             = pVM;
+    pState->pVCpu           = pVCpu;
+    pState->fLocked         = false;
+    pState->f64Bits         = enmMode >= PGMMODE_AMD64 && pSelInfo->u.Raw.Gen.u1Long;
+
+    DISCPUMODE enmCpuMode;
+    switch (fFlags & DBGF_DISAS_FLAGS_MODE_MASK)
+    {
+        default:
+            AssertFailed();
+            RT_FALL_THRU();
+        case DBGF_DISAS_FLAGS_DEFAULT_MODE:
+            enmCpuMode   = pState->f64Bits
+                         ? DISCPUMODE_64BIT
+                         : pSelInfo->u.Raw.Gen.u1DefBig
+                         ? DISCPUMODE_32BIT
+                         : DISCPUMODE_16BIT;
+            break;
+        case DBGF_DISAS_FLAGS_16BIT_MODE:
+        case DBGF_DISAS_FLAGS_16BIT_REAL_MODE:
+            enmCpuMode = DISCPUMODE_16BIT;
+            break;
+        case DBGF_DISAS_FLAGS_32BIT_MODE:
+            enmCpuMode = DISCPUMODE_32BIT;
+            break;
+        case DBGF_DISAS_FLAGS_64BIT_MODE:
+            enmCpuMode = DISCPUMODE_64BIT;
+            break;
+    }
+
+    uint32_t cbInstr;
+    int rc = DISInstrWithReader(GCPtr,
+                                enmCpuMode,
+                                dbgfR3DisasInstrRead,
+                                &pState->Cpu,
+                                &pState->Cpu,
+                                &cbInstr);
+    if (RT_SUCCESS(rc))
+    {
+        pState->GCPtrNext = GCPtr + cbInstr;
+        return VINF_SUCCESS;
+    }
+
+    /* cleanup */
+    if (pState->fLocked)
+    {
+        PGMPhysReleasePageMappingLock(pVM, &pState->PageMapLock);
+        pState->fLocked = false;
+    }
+    return rc;
+}
+
+
+#if 0
+/**
+ * Calls the disassembler for disassembling the next instruction.
+ *
+ * @returns VBox status code.
+ * @param   pState      The disas CPU state.
+ */
+static int dbgfR3DisasInstrNext(PDBGFDISASSTATE pState)
+{
+    uint32_t cbInstr;
+    int rc = DISInstr(&pState->Cpu, (void *)pState->GCPtrNext, 0, &cbInstr, NULL);
+    if (RT_SUCCESS(rc))
+    {
+        pState->GCPtrNext = GCPtr + cbInstr;
+        return VINF_SUCCESS;
+    }
+    return rc;
+}
+#endif
+
+
+/**
+ * Done with the disassembler state, free associated resources.
+ *
+ * @param   pState      The disas CPU state ++.
+ */
+static void dbgfR3DisasInstrDone(PDBGFDISASSTATE pState)
+{
+    if (pState->fLocked)
+    {
+        PGMPhysReleasePageMappingLock(pState->pVM, &pState->PageMapLock);
+        pState->fLocked = false;
+    }
+}
+
+
+/**
+ * @callback_method_impl{FNDISREADBYTES}
+ *
+ * @remarks The source is relative to the base address indicated by
+ *          DBGFDISASSTATE::GCPtrSegBase.
+ */
+static DECLCALLBACK(int) dbgfR3DisasInstrRead(PDISCPUSTATE pDis, uint8_t offInstr, uint8_t cbMinRead, uint8_t cbMaxRead)
+{
+    PDBGFDISASSTATE pState = (PDBGFDISASSTATE)pDis;
+    for (;;)
+    {
+        RTGCUINTPTR GCPtr = pDis->uInstrAddr + offInstr + pState->GCPtrSegBase;
+
+        /*
+         * Need to update the page translation?
+         */
+        if (    !pState->pvPageR3
+            ||  (GCPtr >> PAGE_SHIFT) != (pState->GCPtrPage >> PAGE_SHIFT))
+        {
+            int rc = VINF_SUCCESS;
+
+            /* translate the address */
+            pState->GCPtrPage = GCPtr & PAGE_BASE_GC_MASK;
+            if (pState->fLocked)
+                PGMPhysReleasePageMappingLock(pState->pVM, &pState->PageMapLock);
+            if (pState->enmMode <= PGMMODE_PROTECTED)
+                rc = PGMPhysGCPhys2CCPtrReadOnly(pState->pVM, pState->GCPtrPage, &pState->pvPageR3, &pState->PageMapLock);
+            else
+                rc = PGMPhysGCPtr2CCPtrReadOnly(pState->pVCpu, pState->GCPtrPage, &pState->pvPageR3, &pState->PageMapLock);
+            if (RT_SUCCESS(rc))
+                pState->fLocked = true;
+            else
+            {
+                pState->fLocked  = false;
+                pState->pvPageR3 = NULL;
+                return rc;
+            }
+        }
+
+        /*
+         * Check the segment limit.
+         */
+        if (!pState->f64Bits && pDis->uInstrAddr + offInstr > pState->cbSegLimit)
+            return VERR_OUT_OF_SELECTOR_BOUNDS;
+
+        /*
+         * Calc how much we can read, maxing out the read.
+         */
+        uint32_t cb = PAGE_SIZE - (GCPtr & PAGE_OFFSET_MASK);
+        if (!pState->f64Bits)
+        {
+            RTGCUINTPTR cbSeg = pState->GCPtrSegEnd - GCPtr;
+            if (cb > cbSeg && cbSeg)
+                cb = cbSeg;
+        }
+        if (cb > cbMaxRead)
+            cb = cbMaxRead;
+
+        /*
+         * Read and advance,
+         */
+        memcpy(&pDis->abInstr[offInstr], (char *)pState->pvPageR3 + (GCPtr & PAGE_OFFSET_MASK), cb);
+        offInstr  += (uint8_t)cb;
+        if (cb >= cbMinRead)
+        {
+            pDis->cbCachedInstr = offInstr;
+            return VINF_SUCCESS;
+        }
+        cbMaxRead -= (uint8_t)cb;
+        cbMinRead -= (uint8_t)cb;
+    }
+}
+
+
+/**
+ * @callback_method_impl{FNDISGETSYMBOL}
+ */
+static DECLCALLBACK(int) dbgfR3DisasGetSymbol(PCDISCPUSTATE pDis, uint32_t u32Sel, RTUINTPTR uAddress,
+                                              char *pszBuf, size_t cchBuf, RTINTPTR *poff, void *pvUser)
+{
+    PDBGFDISASSTATE pState   = (PDBGFDISASSTATE)pDis;
+    PCDBGFSELINFO   pSelInfo = (PCDBGFSELINFO)pvUser;
+
+    /*
+     * Address conversion
+     */
+    DBGFADDRESS     Addr;
+    int             rc;
+    /* Start with CS. */
+    if (   DIS_FMT_SEL_IS_REG(u32Sel)
+        ?  DIS_FMT_SEL_GET_REG(u32Sel) == DISSELREG_CS
+        :  pSelInfo->Sel == DIS_FMT_SEL_GET_VALUE(u32Sel))
+        rc = DBGFR3AddrFromSelInfoOff(pState->pVM->pUVM, &Addr, pSelInfo, uAddress);
+    /* In long mode everything but FS and GS is easy. */
+    else if (   pState->Cpu.uCpuMode == DISCPUMODE_64BIT
+             && DIS_FMT_SEL_IS_REG(u32Sel)
+             && DIS_FMT_SEL_GET_REG(u32Sel) != DISSELREG_GS
+             && DIS_FMT_SEL_GET_REG(u32Sel) != DISSELREG_FS)
+    {
+        DBGFR3AddrFromFlat(pState->pVM->pUVM, &Addr, uAddress);
+        rc = VINF_SUCCESS;
+    }
+    /* Here's a quick hack to catch patch manager SS relative access. */
+    else if (   DIS_FMT_SEL_IS_REG(u32Sel)
+             && DIS_FMT_SEL_GET_REG(u32Sel) == DISSELREG_SS
+             && pSelInfo->GCPtrBase == 0
+             && pSelInfo->cbLimit   >= UINT32_MAX)
+    {
+        DBGFR3AddrFromFlat(pState->pVM->pUVM, &Addr, uAddress);
+        rc = VINF_SUCCESS;
+    }
+    else
+    {
+        /** @todo implement a generic solution here. */
+        rc = VERR_SYMBOL_NOT_FOUND;
+    }
+
+    /*
+     * If we got an address, try resolve it into a symbol.
+     */
+    if (RT_SUCCESS(rc))
+    {
+        RTDBGSYMBOL     Sym;
+        RTGCINTPTR      off;
+        rc = DBGFR3AsSymbolByAddr(pState->pVM->pUVM, pState->hDbgAs, &Addr,
+                                  RTDBGSYMADDR_FLAGS_LESS_OR_EQUAL | RTDBGSYMADDR_FLAGS_SKIP_ABS_IN_DEFERRED,
+                                  &off, &Sym, NULL /*phMod*/);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Return the symbol and offset.
+             */
+            size_t cchName = strlen(Sym.szName);
+            if (cchName >= cchBuf)
+                cchName = cchBuf - 1;
+            memcpy(pszBuf, Sym.szName, cchName);
+            pszBuf[cchName] = '\0';
+
+            *poff = off;
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Disassembles the one instruction according to the specified flags and
+ * address, internal worker executing on the EMT of the specified virtual CPU.
+ *
+ * @returns VBox status code.
+ * @param       pVM             The cross context VM structure.
+ * @param       pVCpu           The cross context virtual CPU structure.
+ * @param       Sel             The code selector. This used to determine the 32/16 bit ness and
+ *                              calculation of the actual instruction address.
+ * @param       pGCPtr          Pointer to the variable holding the code address
+ *                              relative to the base of Sel.
+ * @param       fFlags          Flags controlling where to start and how to format.
+ *                              A combination of the DBGF_DISAS_FLAGS_* \#defines.
+ * @param       pszOutput       Output buffer.
+ * @param       cbOutput        Size of the output buffer.
+ * @param       pcbInstr        Where to return the size of the instruction.
+ * @param       pDisState       Where to store the disassembler state into.
+ */
+static DECLCALLBACK(int)
+dbgfR3DisasInstrExOnVCpu(PVM pVM, PVMCPU pVCpu, RTSEL Sel, PRTGCPTR pGCPtr, uint32_t fFlags,
+                         char *pszOutput, uint32_t cbOutput, uint32_t *pcbInstr, PDBGFDISSTATE pDisState)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    RTGCPTR GCPtr = *pGCPtr;
+    int     rc;
+
+    /*
+     * Get the Sel and GCPtr if fFlags requests that.
+     */
+    PCCPUMCTXCORE  pCtxCore   = NULL;
+    PCCPUMSELREG   pSRegCS    = NULL;
+    if (fFlags & DBGF_DISAS_FLAGS_CURRENT_GUEST)
+    {
+        pCtxCore   = CPUMGetGuestCtxCore(pVCpu);
+        Sel        = pCtxCore->cs.Sel;
+        pSRegCS    = &pCtxCore->cs;
+        GCPtr      = pCtxCore->rip;
+    }
+    /*
+     * Check if the selector matches the guest CS, use the hidden
+     * registers from that if they are valid. Saves time and effort.
+     */
+    else
+    {
+        pCtxCore = CPUMGetGuestCtxCore(pVCpu);
+        if (pCtxCore->cs.Sel == Sel && Sel != DBGF_SEL_FLAT)
+            pSRegCS = &pCtxCore->cs;
+        else
+            pCtxCore = NULL;
+    }
+
+    /*
+     * Read the selector info - assume no stale selectors and nasty stuff like that.
+     *
+     * Note! We CANNOT load invalid hidden selector registers since that would
+     *       mean that log/debug statements or the debug will influence the
+     *       guest state and make things behave differently.
+     */
+    DBGFSELINFO     SelInfo;
+    const PGMMODE   enmMode          = PGMGetGuestMode(pVCpu);
+    bool            fRealModeAddress = false;
+
+    if (   pSRegCS
+        && CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSRegCS))
+    {
+        SelInfo.Sel                     = Sel;
+        SelInfo.SelGate                 = 0;
+        SelInfo.GCPtrBase               = pSRegCS->u64Base;
+        SelInfo.cbLimit                 = pSRegCS->u32Limit;
+        SelInfo.fFlags                  = PGMMODE_IS_LONG_MODE(enmMode)
+                                        ? DBGFSELINFO_FLAGS_LONG_MODE
+                                        : enmMode != PGMMODE_REAL && !pCtxCore->eflags.Bits.u1VM
+                                        ? DBGFSELINFO_FLAGS_PROT_MODE
+                                        : DBGFSELINFO_FLAGS_REAL_MODE;
+
+        SelInfo.u.Raw.au32[0]           = 0;
+        SelInfo.u.Raw.au32[1]           = 0;
+        SelInfo.u.Raw.Gen.u16LimitLow   = 0xffff;
+        SelInfo.u.Raw.Gen.u4LimitHigh   = 0xf;
+        SelInfo.u.Raw.Gen.u1Present     = pSRegCS->Attr.n.u1Present;
+        SelInfo.u.Raw.Gen.u1Granularity = pSRegCS->Attr.n.u1Granularity;;
+        SelInfo.u.Raw.Gen.u1DefBig      = pSRegCS->Attr.n.u1DefBig;
+        SelInfo.u.Raw.Gen.u1Long        = pSRegCS->Attr.n.u1Long;
+        SelInfo.u.Raw.Gen.u1DescType    = pSRegCS->Attr.n.u1DescType;
+        SelInfo.u.Raw.Gen.u4Type        = pSRegCS->Attr.n.u4Type;
+        fRealModeAddress                = !!(SelInfo.fFlags & DBGFSELINFO_FLAGS_REAL_MODE);
+    }
+    else if (Sel == DBGF_SEL_FLAT)
+    {
+        SelInfo.Sel                     = Sel;
+        SelInfo.SelGate                 = 0;
+        SelInfo.GCPtrBase               = 0;
+        SelInfo.cbLimit                 = ~(RTGCUINTPTR)0;
+        SelInfo.fFlags                  = PGMMODE_IS_LONG_MODE(enmMode)
+                                        ? DBGFSELINFO_FLAGS_LONG_MODE
+                                        : enmMode != PGMMODE_REAL
+                                        ? DBGFSELINFO_FLAGS_PROT_MODE
+                                        : DBGFSELINFO_FLAGS_REAL_MODE;
+        SelInfo.u.Raw.au32[0]           = 0;
+        SelInfo.u.Raw.au32[1]           = 0;
+        SelInfo.u.Raw.Gen.u16LimitLow   = 0xffff;
+        SelInfo.u.Raw.Gen.u4LimitHigh   = 0xf;
+
+        pSRegCS = &CPUMGetGuestCtxCore(pVCpu)->cs;
+        if (CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSRegCS))
+        {
+            /* Assume the current CS defines the execution mode. */
+            SelInfo.u.Raw.Gen.u1Present     = pSRegCS->Attr.n.u1Present;
+            SelInfo.u.Raw.Gen.u1Granularity = pSRegCS->Attr.n.u1Granularity;;
+            SelInfo.u.Raw.Gen.u1DefBig      = pSRegCS->Attr.n.u1DefBig;
+            SelInfo.u.Raw.Gen.u1Long        = pSRegCS->Attr.n.u1Long;
+            SelInfo.u.Raw.Gen.u1DescType    = pSRegCS->Attr.n.u1DescType;
+            SelInfo.u.Raw.Gen.u4Type        = pSRegCS->Attr.n.u4Type;
+        }
+        else
+        {
+            pSRegCS  = NULL;
+            SelInfo.u.Raw.Gen.u1Present     = 1;
+            SelInfo.u.Raw.Gen.u1Granularity = 1;
+            SelInfo.u.Raw.Gen.u1DefBig      = 1;
+            SelInfo.u.Raw.Gen.u1DescType    = 1;
+            SelInfo.u.Raw.Gen.u4Type        = X86_SEL_TYPE_EO;
+        }
+    }
+    else if (   (pCtxCore && pCtxCore->eflags.Bits.u1VM)
+             || enmMode == PGMMODE_REAL
+             || (fFlags & DBGF_DISAS_FLAGS_MODE_MASK) == DBGF_DISAS_FLAGS_16BIT_REAL_MODE)
+    {   /* V86 mode or real mode - real mode addressing */
+        SelInfo.Sel                     = Sel;
+        SelInfo.SelGate                 = 0;
+        SelInfo.GCPtrBase               = Sel * 16;
+        SelInfo.cbLimit                 = ~(RTGCUINTPTR)0;
+        SelInfo.fFlags                  = DBGFSELINFO_FLAGS_REAL_MODE;
+        SelInfo.u.Raw.au32[0]           = 0;
+        SelInfo.u.Raw.au32[1]           = 0;
+        SelInfo.u.Raw.Gen.u16LimitLow   = 0xffff;
+        SelInfo.u.Raw.Gen.u4LimitHigh   = 0xf;
+        SelInfo.u.Raw.Gen.u1Present     = 1;
+        SelInfo.u.Raw.Gen.u1Granularity = 1;
+        SelInfo.u.Raw.Gen.u1DefBig      = 0; /* 16 bits */
+        SelInfo.u.Raw.Gen.u1DescType    = 1;
+        SelInfo.u.Raw.Gen.u4Type        = X86_SEL_TYPE_EO;
+        fRealModeAddress                = true;
+    }
+    else
+    {
+        rc = SELMR3GetSelectorInfo(pVCpu, Sel, &SelInfo);
+        if (RT_FAILURE(rc))
+        {
+            RTStrPrintf(pszOutput, cbOutput, "Sel=%04x -> %Rrc\n", Sel, rc);
+            return rc;
+        }
+    }
+
+    /*
+     * Disassemble it.
+     */
+    DBGFDISASSTATE State;
+    rc = dbgfR3DisasInstrFirst(pVM, pVCpu, &SelInfo, enmMode, GCPtr, fFlags, &State);
+    if (RT_FAILURE(rc))
+    {
+        if (State.Cpu.cbCachedInstr)
+            RTStrPrintf(pszOutput, cbOutput, "Disas -> %Rrc; %.*Rhxs\n", rc, (size_t)State.Cpu.cbCachedInstr, State.Cpu.abInstr);
+        else
+            RTStrPrintf(pszOutput, cbOutput, "Disas -> %Rrc\n", rc);
+        return rc;
+    }
+
+    /*
+     * Format it.
+     */
+    char szBuf[512];
+    DISFormatYasmEx(&State.Cpu, szBuf, sizeof(szBuf),
+                    DIS_FMT_FLAGS_RELATIVE_BRANCH,
+                    fFlags & DBGF_DISAS_FLAGS_NO_SYMBOLS ? NULL : dbgfR3DisasGetSymbol,
+                    &SelInfo);
+
+    /*
+     * Print it to the user specified buffer.
+     */
+    size_t cch;
+    if (fFlags & DBGF_DISAS_FLAGS_NO_BYTES)
+    {
+        if (fFlags & DBGF_DISAS_FLAGS_NO_ADDRESS)
+            cch = RTStrPrintf(pszOutput, cbOutput, "%s", szBuf);
+        else if (fRealModeAddress)
+            cch = RTStrPrintf(pszOutput, cbOutput, "%04x:%04x  %s", Sel, (unsigned)GCPtr, szBuf);
+        else if (Sel == DBGF_SEL_FLAT)
+        {
+            if (enmMode >= PGMMODE_AMD64)
+                cch = RTStrPrintf(pszOutput, cbOutput, "%RGv  %s", GCPtr, szBuf);
+            else
+                cch = RTStrPrintf(pszOutput, cbOutput, "%08RX32  %s", (uint32_t)GCPtr, szBuf);
+        }
+        else
+        {
+            if (enmMode >= PGMMODE_AMD64)
+                cch = RTStrPrintf(pszOutput, cbOutput, "%04x:%RGv  %s", Sel, GCPtr, szBuf);
+            else
+                cch = RTStrPrintf(pszOutput, cbOutput, "%04x:%08RX32  %s", Sel, (uint32_t)GCPtr, szBuf);
+        }
+    }
+    else
+    {
+        uint32_t        cbInstr  = State.Cpu.cbInstr;
+        uint8_t const  *pabInstr = State.Cpu.abInstr;
+        if (fFlags & DBGF_DISAS_FLAGS_NO_ADDRESS)
+            cch = RTStrPrintf(pszOutput, cbOutput, "%.*Rhxs%*s %s",
+                              cbInstr, pabInstr, cbInstr < 8 ? (8 - cbInstr) * 3 : 0, "",
+                              szBuf);
+        else if (fRealModeAddress)
+            cch = RTStrPrintf(pszOutput, cbOutput, "%04x:%04x %.*Rhxs%*s %s",
+                              Sel, (unsigned)GCPtr,
+                              cbInstr, pabInstr, cbInstr < 8 ? (8 - cbInstr) * 3 : 0, "",
+                              szBuf);
+        else if (Sel == DBGF_SEL_FLAT)
+        {
+            if (enmMode >= PGMMODE_AMD64)
+                cch = RTStrPrintf(pszOutput, cbOutput, "%RGv %.*Rhxs%*s %s",
+                                  GCPtr,
+                                  cbInstr, pabInstr, cbInstr < 8 ? (8 - cbInstr) * 3 : 0, "",
+                                  szBuf);
+            else
+                cch = RTStrPrintf(pszOutput, cbOutput, "%08RX32 %.*Rhxs%*s %s",
+                                  (uint32_t)GCPtr,
+                                  cbInstr, pabInstr, cbInstr < 8 ? (8 - cbInstr) * 3 : 0, "",
+                                  szBuf);
+        }
+        else
+        {
+            if (enmMode >= PGMMODE_AMD64)
+                cch = RTStrPrintf(pszOutput, cbOutput, "%04x:%RGv %.*Rhxs%*s %s",
+                                  Sel, GCPtr,
+                                  cbInstr, pabInstr, cbInstr < 8 ? (8 - cbInstr) * 3 : 0, "",
+                                  szBuf);
+            else
+                cch = RTStrPrintf(pszOutput, cbOutput, "%04x:%08RX32 %.*Rhxs%*s %s",
+                                  Sel, (uint32_t)GCPtr,
+                                  cbInstr, pabInstr, cbInstr < 8 ? (8 - cbInstr) * 3 : 0, "",
+                                  szBuf);
+        }
+    }
+
+    if (pcbInstr)
+        *pcbInstr = State.Cpu.cbInstr;
+
+    if (pDisState)
+    {
+        pDisState->pCurInstr = State.Cpu.pCurInstr;
+        pDisState->cbInstr   = State.Cpu.cbInstr;
+        pDisState->Param1    = State.Cpu.Param1;
+        pDisState->Param2    = State.Cpu.Param2;
+        pDisState->Param3    = State.Cpu.Param3;
+        pDisState->Param4    = State.Cpu.Param4;
+    }
+
+    dbgfR3DisasInstrDone(&State);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Disassembles the one instruction according to the specified flags and address
+ * returning part of the disassembler state.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   idCpu           The ID of virtual CPU.
+ * @param   pAddr           The code address.
+ * @param   fFlags          Flags controlling where to start and how to format.
+ *                          A combination of the DBGF_DISAS_FLAGS_* \#defines.
+ * @param   pszOutput       Output buffer.  This will always be properly
+ *                          terminated if @a cbOutput is greater than zero.
+ * @param   cbOutput        Size of the output buffer.
+ * @param   pDisState       The disassembler state to fill in.
+ *
+ * @remarks May have to switch to the EMT of the virtual CPU in order to do
+ *          address conversion.
+ */
+DECLHIDDEN(int) dbgfR3DisasInstrStateEx(PUVM pUVM, VMCPUID idCpu, PDBGFADDRESS pAddr, uint32_t fFlags,
+                                        char *pszOutput, uint32_t cbOutput, PDBGFDISSTATE pDisState)
+{
+    AssertReturn(cbOutput > 0, VERR_INVALID_PARAMETER);
+    *pszOutput = '\0';
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_CPU_ID);
+    AssertReturn(!(fFlags & ~DBGF_DISAS_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn((fFlags & DBGF_DISAS_FLAGS_MODE_MASK) <= DBGF_DISAS_FLAGS_64BIT_MODE, VERR_INVALID_PARAMETER);
+
+    /*
+     * Optimize the common case where we're called on the EMT of idCpu since
+     * we're using this all the time when logging.
+     */
+    int     rc;
+    PVMCPU  pVCpu = VMMGetCpu(pVM);
+    if (    pVCpu
+        &&  pVCpu->idCpu == idCpu)
+        rc = dbgfR3DisasInstrExOnVCpu(pVM, pVCpu, pAddr->Sel, &pAddr->off, fFlags, pszOutput, cbOutput, NULL, pDisState);
+    else
+        rc = VMR3ReqPriorityCallWait(pVM, idCpu, (PFNRT)dbgfR3DisasInstrExOnVCpu, 9,
+                                     pVM, VMMGetCpuById(pVM, idCpu), pAddr->Sel, &pAddr->off, fFlags, pszOutput, cbOutput, NULL, pDisState);
+    return rc;
+}
+
+/**
+ * Disassembles the one instruction according to the specified flags and address.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   idCpu           The ID of virtual CPU.
+ * @param   Sel             The code selector. This used to determine the 32/16 bit ness and
+ *                          calculation of the actual instruction address.
+ * @param   GCPtr           The code address relative to the base of Sel.
+ * @param   fFlags          Flags controlling where to start and how to format.
+ *                          A combination of the DBGF_DISAS_FLAGS_* \#defines.
+ * @param   pszOutput       Output buffer.  This will always be properly
+ *                          terminated if @a cbOutput is greater than zero.
+ * @param   cbOutput        Size of the output buffer.
+ * @param   pcbInstr        Where to return the size of the instruction.
+ *
+ * @remarks May have to switch to the EMT of the virtual CPU in order to do
+ *          address conversion.
+ */
+VMMR3DECL(int) DBGFR3DisasInstrEx(PUVM pUVM, VMCPUID idCpu, RTSEL Sel, RTGCPTR GCPtr, uint32_t fFlags,
+                                  char *pszOutput, uint32_t cbOutput, uint32_t *pcbInstr)
+{
+    AssertReturn(cbOutput > 0, VERR_INVALID_PARAMETER);
+    *pszOutput = '\0';
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_CPU_ID);
+    AssertReturn(!(fFlags & ~DBGF_DISAS_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn((fFlags & DBGF_DISAS_FLAGS_MODE_MASK) <= DBGF_DISAS_FLAGS_64BIT_MODE, VERR_INVALID_PARAMETER);
+
+    /*
+     * Optimize the common case where we're called on the EMT of idCpu since
+     * we're using this all the time when logging.
+     */
+    int     rc;
+    PVMCPU  pVCpu = VMMGetCpu(pVM);
+    if (    pVCpu
+        &&  pVCpu->idCpu == idCpu)
+        rc = dbgfR3DisasInstrExOnVCpu(pVM, pVCpu, Sel, &GCPtr, fFlags, pszOutput, cbOutput, pcbInstr, NULL);
+    else
+        rc = VMR3ReqPriorityCallWait(pVM, idCpu, (PFNRT)dbgfR3DisasInstrExOnVCpu, 9,
+                                     pVM, VMMGetCpuById(pVM, idCpu), Sel, &GCPtr, fFlags, pszOutput, cbOutput, pcbInstr, NULL);
+    return rc;
+}
+
+
+/**
+ * Disassembles the current guest context instruction.
+ * All registers and data will be displayed. Addresses will be attempted resolved to symbols.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pszOutput       Output buffer.  This will always be properly
+ *                          terminated if @a cbOutput is greater than zero.
+ * @param   cbOutput        Size of the output buffer.
+ * @thread  EMT(pVCpu)
+ */
+VMMR3_INT_DECL(int) DBGFR3DisasInstrCurrent(PVMCPU pVCpu, char *pszOutput, uint32_t cbOutput)
+{
+    AssertReturn(cbOutput > 0, VERR_INVALID_PARAMETER);
+    *pszOutput = '\0';
+    Assert(VMCPU_IS_EMT(pVCpu));
+
+    RTGCPTR GCPtr = 0;
+    return dbgfR3DisasInstrExOnVCpu(pVCpu->pVMR3, pVCpu, 0, &GCPtr,
+                                    DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE
+                                    | DBGF_DISAS_FLAGS_ANNOTATE_PATCHED,
+                                    pszOutput, cbOutput, NULL, NULL);
+}
+
+
+/**
+ * Disassembles the current guest context instruction and writes it to the log.
+ * All registers and data will be displayed. Addresses will be attempted resolved to symbols.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pszPrefix       Short prefix string to the disassembly string. (optional)
+ * @thread  EMT(pVCpu)
+ */
+VMMR3DECL(int) DBGFR3DisasInstrCurrentLogInternal(PVMCPU pVCpu, const char *pszPrefix)
+{
+    char szBuf[256];
+    szBuf[0] = '\0';
+    int rc = DBGFR3DisasInstrCurrent(pVCpu, &szBuf[0], sizeof(szBuf));
+    if (RT_FAILURE(rc))
+        RTStrPrintf(szBuf, sizeof(szBuf), "DBGFR3DisasInstrCurrentLog failed with rc=%Rrc\n", rc);
+    if (pszPrefix && *pszPrefix)
+    {
+        if (pVCpu->CTX_SUFF(pVM)->cCpus > 1)
+            RTLogPrintf("%s-CPU%u: %s\n", pszPrefix, pVCpu->idCpu, szBuf);
+        else
+            RTLogPrintf("%s: %s\n", pszPrefix, szBuf);
+    }
+    else
+        RTLogPrintf("%s\n", szBuf);
+    return rc;
+}
+
+
+
+/**
+ * Disassembles the specified guest context instruction and writes it to the log.
+ * Addresses will be attempted resolved to symbols.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling
+ *                      EMT.
+ * @param   Sel         The code selector. This used to determine the 32/16
+ *                      bit-ness and calculation of the actual instruction
+ *                      address.
+ * @param   GCPtr       The code address relative to the base of Sel.
+ * @param   pszPrefix   Short prefix string to the disassembly string.
+ *                      (optional)
+ * @thread  EMT(pVCpu)
+ */
+VMMR3DECL(int) DBGFR3DisasInstrLogInternal(PVMCPU pVCpu, RTSEL Sel, RTGCPTR GCPtr, const char *pszPrefix)
+{
+    Assert(VMCPU_IS_EMT(pVCpu));
+
+    char szBuf[256];
+    RTGCPTR GCPtrTmp = GCPtr;
+    int rc = dbgfR3DisasInstrExOnVCpu(pVCpu->pVMR3, pVCpu, Sel, &GCPtrTmp, DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                                      &szBuf[0], sizeof(szBuf), NULL, NULL);
+    if (RT_FAILURE(rc))
+        RTStrPrintf(szBuf, sizeof(szBuf), "DBGFR3DisasInstrLog(, %RTsel, %RGv) failed with rc=%Rrc\n", Sel, GCPtr, rc);
+    if (pszPrefix && *pszPrefix)
+    {
+        if (pVCpu->CTX_SUFF(pVM)->cCpus > 1)
+            RTLogPrintf("%s-CPU%u: %s\n", pszPrefix, pVCpu->idCpu, szBuf);
+        else
+            RTLogPrintf("%s: %s\n", pszPrefix, szBuf);
+    }
+    else
+        RTLogPrintf("%s\n", szBuf);
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFInfo.cpp b/src/VBox/VMM/VMMR3/DBGFInfo.cpp
new file mode 100644
index 00000000..2e4cc581
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFInfo.cpp
@@ -0,0 +1,1464 @@
+/* $Id: DBGFInfo.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Info.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF_INFO
+#include <VBox/vmm/dbgf.h>
+
+#include <VBox/vmm/mm.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/getopt.h>
+#include <iprt/param.h>
+#include <iprt/semaphore.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(void) dbgfR3InfoLog_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...);
+static DECLCALLBACK(void) dbgfR3InfoLog_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args);
+static DECLCALLBACK(void) dbgfR3InfoLogRel_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...);
+static DECLCALLBACK(void) dbgfR3InfoLogRel_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args);
+static DECLCALLBACK(void) dbgfR3InfoStdErr_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...);
+static DECLCALLBACK(void) dbgfR3InfoStdErr_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args);
+static DECLCALLBACK(void) dbgfR3InfoHelp(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** Logger output. */
+static const DBGFINFOHLP g_dbgfR3InfoLogHlp =
+{
+    dbgfR3InfoLog_Printf,
+    dbgfR3InfoLog_PrintfV,
+    DBGFR3InfoGenricGetOptError,
+};
+
+/** Release logger output. */
+static const DBGFINFOHLP g_dbgfR3InfoLogRelHlp =
+{
+    dbgfR3InfoLogRel_Printf,
+    dbgfR3InfoLogRel_PrintfV,
+    DBGFR3InfoGenricGetOptError
+};
+
+/** Standard error output. */
+static const DBGFINFOHLP g_dbgfR3InfoStdErrHlp =
+{
+    dbgfR3InfoStdErr_Printf,
+    dbgfR3InfoStdErr_PrintfV,
+    DBGFR3InfoGenricGetOptError
+};
+
+
+/**
+ * Initialize the info handlers.
+ *
+ * This is called first during the DBGF init process and thus does the shared
+ * critsect init.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ */
+int dbgfR3InfoInit(PUVM pUVM)
+{
+    /*
+     * Make sure we already didn't initialized in the lazy manner.
+     */
+    if (RTCritSectRwIsInitialized(&pUVM->dbgf.s.CritSect))
+        return VINF_SUCCESS;
+
+    /*
+     * Initialize the crit sect.
+     */
+    int rc = RTCritSectRwInit(&pUVM->dbgf.s.CritSect);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Register the 'info help' item.
+     */
+    rc = DBGFR3InfoRegisterInternal(pUVM->pVM, "help", "List of info items.", dbgfR3InfoHelp);
+    AssertRCReturn(rc, rc);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminate the info handlers.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ */
+int dbgfR3InfoTerm(PUVM pUVM)
+{
+    /*
+     * Delete the crit sect.
+     */
+    int rc = RTCritSectRwDelete(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * @interface_method_impl{DBGFINFOHLP,pfnGetOptError}
+ */
+VMMR3DECL(void) DBGFR3InfoGenricGetOptError(PCDBGFINFOHLP pHlp, int rc, PRTGETOPTUNION pValueUnion, PRTGETOPTSTATE pState)
+{
+    RT_NOREF(pState);
+    char szMsg[1024];
+    RTGetOptFormatError(szMsg, sizeof(szMsg), rc, pValueUnion);
+    pHlp->pfnPrintf(pHlp, "syntax error: %s\n", szMsg);
+}
+
+
+/**
+ * @interface_method_impl{DBGFINFOHLP,pfnPrintf, Logger output.}
+ */
+static DECLCALLBACK(void) dbgfR3InfoLog_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...)
+{
+    NOREF(pHlp);
+    va_list args;
+    va_start(args, pszFormat);
+    RTLogPrintfV(pszFormat, args);
+    va_end(args);
+}
+
+
+/**
+ * @interface_method_impl{DBGFINFOHLP,pfnPrintfV, Logger output.}
+ */
+static DECLCALLBACK(void) dbgfR3InfoLog_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args)
+{
+    NOREF(pHlp);
+    RTLogPrintfV(pszFormat, args);
+}
+
+
+/**
+ * Gets the logger info helper.
+ * The returned info helper will unconditionally write all output to the log.
+ *
+ * @returns Pointer to the logger info helper.
+ */
+VMMR3DECL(PCDBGFINFOHLP) DBGFR3InfoLogHlp(void)
+{
+    return &g_dbgfR3InfoLogHlp;
+}
+
+
+/**
+ * @interface_method_impl{DBGFINFOHLP,pfnPrintf, Release logger output.}
+ */
+static DECLCALLBACK(void) dbgfR3InfoLogRel_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...)
+{
+    NOREF(pHlp);
+    va_list args;
+    va_start(args, pszFormat);
+    RTLogRelPrintfV(pszFormat, args);
+    va_end(args);
+}
+
+
+/**
+ * @interface_method_impl{DBGFINFOHLP,pfnPrintfV, Release logger output.}
+ */
+static DECLCALLBACK(void) dbgfR3InfoLogRel_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args)
+{
+    NOREF(pHlp);
+    RTLogRelPrintfV(pszFormat, args);
+}
+
+
+/**
+ * @interface_method_impl{DBGFINFOHLP,pfnPrintf, Stdandard error output.}
+ */
+static DECLCALLBACK(void) dbgfR3InfoStdErr_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...)
+{
+    NOREF(pHlp);
+    va_list args;
+    va_start(args, pszFormat);
+    RTStrmPrintfV(g_pStdErr, pszFormat, args);
+    va_end(args);
+}
+
+
+/**
+ * @interface_method_impl{DBGFINFOHLP,pfnPrintfV, Stdandard error output.}
+ */
+static DECLCALLBACK(void) dbgfR3InfoStdErr_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args)
+{
+    NOREF(pHlp);
+    RTStrmPrintfV(g_pStdErr, pszFormat, args);
+}
+
+
+/**
+ * Gets the release logger info helper.
+ * The returned info helper will unconditionally write all output to the release log.
+ *
+ * @returns Pointer to the release logger info helper.
+ */
+VMMR3DECL(PCDBGFINFOHLP) DBGFR3InfoLogRelHlp(void)
+{
+    return &g_dbgfR3InfoLogRelHlp;
+}
+
+
+/**
+ * Handle registration worker.
+ *
+ * This allocates the structure, initializes the common fields and inserts into the list.
+ * Upon successful return the we're inside the crit sect and the caller must leave it.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   fFlags      The flags.
+ * @param   ppInfo      Where to store the created
+ */
+static int dbgfR3InfoRegister(PUVM pUVM, const char *pszName, const char *pszDesc, uint32_t fFlags, PDBGFINFO *ppInfo)
+{
+    /*
+     * Validate.
+     */
+    AssertPtrReturn(pszName, VERR_INVALID_POINTER);
+    AssertReturn(*pszName, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertMsgReturn(!(fFlags & ~(DBGFINFO_FLAGS_RUN_ON_EMT | DBGFINFO_FLAGS_ALL_EMTS)),
+                    ("fFlags=%#x\n", fFlags), VERR_INVALID_FLAGS);
+
+    /*
+     * Allocate and initialize.
+     */
+    int rc;
+    size_t cchName = strlen(pszName) + 1;
+    PDBGFINFO pInfo = (PDBGFINFO)MMR3HeapAllocU(pUVM, MM_TAG_DBGF_INFO, RT_UOFFSETOF_DYN(DBGFINFO, szName[cchName]));
+    if (pInfo)
+    {
+        pInfo->enmType = DBGFINFOTYPE_INVALID;
+        pInfo->fFlags = fFlags;
+        pInfo->pszDesc = pszDesc;
+        pInfo->cchName = cchName - 1;
+        memcpy(pInfo->szName, pszName, cchName);
+
+        /* lazy init */
+        rc = VINF_SUCCESS;
+        if (!RTCritSectRwIsInitialized(&pUVM->dbgf.s.CritSect))
+            rc = dbgfR3InfoInit(pUVM);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Insert in alphabetical order.
+             */
+            rc = RTCritSectRwEnterExcl(&pUVM->dbgf.s.CritSect);
+            AssertRC(rc);
+            PDBGFINFO pPrev = NULL;
+            PDBGFINFO pCur;
+            for (pCur = pUVM->dbgf.s.pInfoFirst; pCur; pPrev = pCur, pCur = pCur->pNext)
+                if (strcmp(pszName, pCur->szName) < 0)
+                    break;
+            pInfo->pNext = pCur;
+            if (pPrev)
+                pPrev->pNext = pInfo;
+            else
+                pUVM->dbgf.s.pInfoFirst = pInfo;
+
+            *ppInfo = pInfo;
+            return VINF_SUCCESS;
+        }
+        MMR3HeapFree(pInfo);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by a device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   pDevIns     The device instance owning the info.
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoRegisterDevice(PVM pVM, const char *pszName, const char *pszDesc,
+                                             PFNDBGFHANDLERDEV pfnHandler, PPDMDEVINS pDevIns)
+{
+    LogFlow(("DBGFR3InfoRegisterDevice: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pDevIns=%p\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, pDevIns));
+
+    /*
+     * Validate the specific stuff.
+     */
+    AssertPtrReturn(pfnHandler, VERR_INVALID_POINTER);
+    AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pVM->pUVM, pszName, pszDesc, 0, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_DEV;
+        pInfo->u.Dev.pfnHandler = pfnHandler;
+        pInfo->u.Dev.pDevIns = pDevIns;
+        RTCritSectRwLeaveExcl(&pVM->pUVM->dbgf.s.CritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by a driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   pDrvIns     The driver instance owning the info.
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoRegisterDriver(PVM pVM, const char *pszName, const char *pszDesc, PFNDBGFHANDLERDRV pfnHandler, PPDMDRVINS pDrvIns)
+{
+    LogFlow(("DBGFR3InfoRegisterDriver: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pDrvIns=%p\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, pDrvIns));
+
+    /*
+     * Validate the specific stuff.
+     */
+    AssertPtrReturn(pfnHandler, VERR_INVALID_POINTER);
+    AssertPtrReturn(pDrvIns, VERR_INVALID_POINTER);
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pVM->pUVM, pszName, pszDesc, 0, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_DRV;
+        pInfo->u.Drv.pfnHandler = pfnHandler;
+        pInfo->u.Drv.pDrvIns = pDrvIns;
+        RTCritSectRwLeaveExcl(&pVM->pUVM->dbgf.s.CritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by an internal component.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoRegisterInternal(PVM pVM, const char *pszName, const char *pszDesc, PFNDBGFHANDLERINT pfnHandler)
+{
+    return DBGFR3InfoRegisterInternalEx(pVM, pszName, pszDesc, pfnHandler, 0);
+}
+
+
+/**
+ * Register a info handler owned by an internal component.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   fFlags      Flags, see the DBGFINFO_FLAGS_*.
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoRegisterInternalEx(PVM pVM, const char *pszName, const char *pszDesc,
+                                                 PFNDBGFHANDLERINT pfnHandler, uint32_t fFlags)
+{
+    LogFlow(("DBGFR3InfoRegisterInternalEx: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p fFlags=%x\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, fFlags));
+
+    /*
+     * Validate the specific stuff.
+     */
+    AssertPtrReturn(pfnHandler, VERR_INVALID_POINTER);
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pVM->pUVM, pszName, pszDesc, fFlags, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_INT;
+        pInfo->u.Int.pfnHandler = pfnHandler;
+        RTCritSectRwLeaveExcl(&pVM->pUVM->dbgf.s.CritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by an external component.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   pvUser      User argument to be passed to the handler.
+ */
+VMMR3DECL(int) DBGFR3InfoRegisterExternal(PUVM pUVM, const char *pszName, const char *pszDesc,
+                                          PFNDBGFHANDLEREXT pfnHandler, void *pvUser)
+{
+    LogFlow(("DBGFR3InfoRegisterExternal: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pvUser=%p\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, pvUser));
+
+    /*
+     * Validate the specific stuff.
+     */
+    AssertPtrReturn(pfnHandler, VERR_INVALID_POINTER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pUVM, pszName, pszDesc, 0, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_EXT;
+        pInfo->u.Ext.pfnHandler = pfnHandler;
+        pInfo->u.Ext.pvUser = pvUser;
+        RTCritSectRwLeaveExcl(&pUVM->dbgf.s.CritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by a device, argv style.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   pDevIns     The device instance owning the info.
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoRegisterDeviceArgv(PVM pVM, const char *pszName, const char *pszDesc,
+                                                 PFNDBGFINFOARGVDEV pfnHandler, PPDMDEVINS pDevIns)
+{
+    LogFlow(("DBGFR3InfoRegisterDeviceArgv: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pDevIns=%p\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, pDevIns));
+
+    /*
+     * Validate the specific stuff.
+     */
+    AssertPtrReturn(pfnHandler, VERR_INVALID_POINTER);
+    AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pVM->pUVM, pszName, pszDesc, 0, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_DEV_ARGV;
+        pInfo->u.DevArgv.pfnHandler = pfnHandler;
+        pInfo->u.DevArgv.pDevIns = pDevIns;
+        RTCritSectRwLeaveExcl(&pVM->pUVM->dbgf.s.CritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by a driver, argv style.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   pDrvIns     The driver instance owning the info.
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoRegisterDriverArgv(PVM pVM, const char *pszName, const char *pszDesc,
+                                                 PFNDBGFINFOARGVDRV pfnHandler, PPDMDRVINS pDrvIns)
+{
+    LogFlow(("DBGFR3InfoRegisterDriverArgv: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pDrvIns=%p\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, pDrvIns));
+
+    /*
+     * Validate the specific stuff.
+     */
+    AssertPtrReturn(pfnHandler, VERR_INVALID_POINTER);
+    AssertPtrReturn(pDrvIns, VERR_INVALID_POINTER);
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pVM->pUVM, pszName, pszDesc, 0, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_DRV_ARGV;
+        pInfo->u.DrvArgv.pfnHandler = pfnHandler;
+        pInfo->u.DrvArgv.pDrvIns = pDrvIns;
+        RTCritSectRwLeaveExcl(&pVM->pUVM->dbgf.s.CritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by a USB device, argv style.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   pUsbIns     The USB device instance owning the info.
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoRegisterUsbArgv(PVM pVM, const char *pszName, const char *pszDesc,
+                                                 PFNDBGFINFOARGVUSB pfnHandler, PPDMUSBINS pUsbIns)
+{
+    LogFlow(("DBGFR3InfoRegisterDriverArgv: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pUsbIns=%p\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, pUsbIns));
+
+    /*
+     * Validate the specific stuff.
+     */
+    AssertPtrReturn(pfnHandler, VERR_INVALID_POINTER);
+    AssertPtrReturn(pUsbIns, VERR_INVALID_POINTER);
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pVM->pUVM, pszName, pszDesc, 0, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_USB_ARGV;
+        pInfo->u.UsbArgv.pfnHandler = pfnHandler;
+        pInfo->u.UsbArgv.pUsbIns = pUsbIns;
+        RTCritSectRwLeaveExcl(&pVM->pUVM->dbgf.s.CritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by an internal component, argv style.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   fFlags      Flags, see the DBGFINFO_FLAGS_*.
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoRegisterInternalArgv(PVM pVM, const char *pszName, const char *pszDesc,
+                                                   PFNDBGFINFOARGVINT pfnHandler, uint32_t fFlags)
+{
+    LogFlow(("DBGFR3InfoRegisterInternalArgv: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p fFlags=%x\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, fFlags));
+
+    /*
+     * Validate the specific stuff.
+     */
+    AssertPtrReturn(pfnHandler, VERR_INVALID_POINTER);
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pVM->pUVM, pszName, pszDesc, fFlags, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_INT_ARGV;
+        pInfo->u.IntArgv.pfnHandler = pfnHandler;
+        RTCritSectRwLeaveExcl(&pVM->pUVM->dbgf.s.CritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by an external component.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   pvUser      User argument to be passed to the handler.
+ */
+VMMR3DECL(int) DBGFR3InfoRegisterExternalArgv(PUVM pUVM, const char *pszName, const char *pszDesc,
+                                              PFNDBGFINFOARGVEXT pfnHandler, void *pvUser)
+{
+    LogFlow(("DBGFR3InfoRegisterExternalArgv: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pvUser=%p\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, pvUser));
+
+    /*
+     * Validate the specific stuff.
+     */
+    AssertPtrReturn(pfnHandler, VERR_INVALID_POINTER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pUVM, pszName, pszDesc, 0, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_EXT_ARGV;
+        pInfo->u.ExtArgv.pfnHandler = pfnHandler;
+        pInfo->u.ExtArgv.pvUser = pvUser;
+        RTCritSectRwLeaveExcl(&pUVM->dbgf.s.CritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Deregister one(/all) info handler(s) owned by a device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     Device instance.
+ * @param   pszName     The identifier of the info. If NULL all owned by the device.
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoDeregisterDevice(PVM pVM, PPDMDEVINS pDevIns, const char *pszName)
+{
+    LogFlow(("DBGFR3InfoDeregisterDevice: pDevIns=%p pszName=%p:{%s}\n", pDevIns, pszName, pszName));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pszName, VERR_INVALID_POINTER);
+    size_t cchName = pszName ? strlen(pszName) : 0;
+    PUVM pUVM = pVM->pUVM;
+
+    /*
+     * Enumerate the info handlers and free the requested entries.
+     */
+    int rc = RTCritSectRwEnterExcl(&pUVM->dbgf.s.CritSect); AssertRC(rc);
+    rc = VERR_FILE_NOT_FOUND;
+    PDBGFINFO pPrev = NULL;
+    PDBGFINFO pInfo = pUVM->dbgf.s.pInfoFirst;
+    if (pszName)
+    {
+        /*
+         * Free a specific one.
+         */
+        for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
+            if (   (   (pInfo->enmType == DBGFINFOTYPE_DEV      && pInfo->u.Dev.pDevIns     == pDevIns)
+                    || (pInfo->enmType == DBGFINFOTYPE_DEV_ARGV && pInfo->u.DevArgv.pDevIns == pDevIns))
+                && pInfo->cchName == cchName
+                && memcmp(pInfo->szName, pszName, cchName) == 0)
+            {
+                if (pPrev)
+                    pPrev->pNext = pInfo->pNext;
+                else
+                    pUVM->dbgf.s.pInfoFirst = pInfo->pNext;
+                MMR3HeapFree(pInfo);
+                rc = VINF_SUCCESS;
+                break;
+            }
+    }
+    else
+    {
+        /*
+         * Free all owned by the device.
+         */
+        while (pInfo != NULL)
+            if (   (pInfo->enmType == DBGFINFOTYPE_DEV      && pInfo->u.Dev.pDevIns     == pDevIns)
+                || (pInfo->enmType == DBGFINFOTYPE_DEV_ARGV && pInfo->u.DevArgv.pDevIns == pDevIns))
+            {
+                PDBGFINFO volatile pFree = pInfo;
+                pInfo = pInfo->pNext;
+                if (pPrev)
+                    pPrev->pNext = pInfo;
+                else
+                    pUVM->dbgf.s.pInfoFirst = pInfo;
+                MMR3HeapFree(pFree);
+            }
+            else
+            {
+                pPrev = pInfo;
+                pInfo = pInfo->pNext;
+            }
+        rc = VINF_SUCCESS;
+    }
+    int rc2 = RTCritSectRwLeaveExcl(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc2);
+    AssertRC(rc);
+    LogFlow(("DBGFR3InfoDeregisterDevice: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Deregister one(/all) info handler(s) owned by a driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDrvIns     Driver instance.
+ * @param   pszName     The identifier of the info. If NULL all owned by the driver.
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoDeregisterDriver(PVM pVM, PPDMDRVINS pDrvIns, const char *pszName)
+{
+    LogFlow(("DBGFR3InfoDeregisterDriver: pDrvIns=%p pszName=%p:{%s}\n", pDrvIns, pszName, pszName));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pDrvIns, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pszName, VERR_INVALID_POINTER);
+    size_t cchName = pszName ? strlen(pszName) : 0;
+    PUVM pUVM = pVM->pUVM;
+
+    /*
+     * Enumerate the info handlers and free the requested entries.
+     */
+    int rc = RTCritSectRwEnterExcl(&pUVM->dbgf.s.CritSect); AssertRC(rc);
+    rc = VERR_FILE_NOT_FOUND;
+    PDBGFINFO pPrev = NULL;
+    PDBGFINFO pInfo = pUVM->dbgf.s.pInfoFirst;
+    if (pszName)
+    {
+        /*
+         * Free a specific one.
+         */
+        for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
+            if (   (   (pInfo->enmType == DBGFINFOTYPE_DRV      && pInfo->u.Drv.pDrvIns     == pDrvIns)
+                    || (pInfo->enmType == DBGFINFOTYPE_DRV_ARGV && pInfo->u.DrvArgv.pDrvIns == pDrvIns))
+                && pInfo->cchName == cchName
+                && memcmp(pInfo->szName, pszName, cchName) == 0)
+            {
+                if (pPrev)
+                    pPrev->pNext = pInfo->pNext;
+                else
+                    pUVM->dbgf.s.pInfoFirst = pInfo->pNext;
+                MMR3HeapFree(pInfo);
+                rc = VINF_SUCCESS;
+                break;
+            }
+    }
+    else
+    {
+        /*
+         * Free all owned by the driver.
+         */
+        while (pInfo != NULL)
+            if (   (pInfo->enmType == DBGFINFOTYPE_DRV      && pInfo->u.Drv.pDrvIns     == pDrvIns)
+                || (pInfo->enmType == DBGFINFOTYPE_DRV_ARGV && pInfo->u.DrvArgv.pDrvIns == pDrvIns))
+            {
+                PDBGFINFO volatile pFree = pInfo;
+                pInfo = pInfo->pNext;
+                if (pPrev)
+                    pPrev->pNext = pInfo;
+                else
+                    pUVM->dbgf.s.pInfoFirst = pInfo;
+                MMR3HeapFree(pFree);
+            }
+            else
+            {
+                pPrev = pInfo;
+                pInfo = pInfo->pNext;
+            }
+        rc = VINF_SUCCESS;
+    }
+    int rc2 = RTCritSectRwLeaveExcl(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc2);
+    AssertRC(rc);
+    LogFlow(("DBGFR3InfoDeregisterDriver: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Deregister one(/all) info handler(s) owned by a USB device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pUsbIns     USB device instance.
+ * @param   pszName     The identifier of the info. If NULL all owned by the driver.
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoDeregisterUsb(PVM pVM, PPDMUSBINS pUsbIns, const char *pszName)
+{
+    LogFlow(("DBGFR3InfoDeregisterUsb: pUsbIns=%p pszName=%p:{%s}\n", pUsbIns, pszName, pszName));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pUsbIns, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pszName, VERR_INVALID_POINTER);
+    size_t cchName = pszName ? strlen(pszName) : 0;
+    PUVM pUVM = pVM->pUVM;
+
+    /*
+     * Enumerate the info handlers and free the requested entries.
+     */
+    int rc = RTCritSectRwEnterExcl(&pUVM->dbgf.s.CritSect); AssertRC(rc);
+    rc = VERR_FILE_NOT_FOUND;
+    PDBGFINFO pPrev = NULL;
+    PDBGFINFO pInfo = pUVM->dbgf.s.pInfoFirst;
+    if (pszName)
+    {
+        /*
+         * Free a specific one.
+         */
+        for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
+            if (   pInfo->enmType == DBGFINFOTYPE_USB_ARGV
+                && pInfo->u.UsbArgv.pUsbIns == pUsbIns
+                && pInfo->cchName == cchName
+                && memcmp(pInfo->szName, pszName, cchName) == 0)
+            {
+                if (pPrev)
+                    pPrev->pNext = pInfo->pNext;
+                else
+                    pUVM->dbgf.s.pInfoFirst = pInfo->pNext;
+                MMR3HeapFree(pInfo);
+                rc = VINF_SUCCESS;
+                break;
+            }
+    }
+    else
+    {
+        /*
+         * Free all owned by the driver.
+         */
+        while (pInfo != NULL)
+            if (   pInfo->enmType == DBGFINFOTYPE_USB_ARGV
+                && pInfo->u.UsbArgv.pUsbIns == pUsbIns)
+            {
+                PDBGFINFO volatile pFree = pInfo;
+                pInfo = pInfo->pNext;
+                if (pPrev)
+                    pPrev->pNext = pInfo;
+                else
+                    pUVM->dbgf.s.pInfoFirst = pInfo;
+                MMR3HeapFree(pFree);
+            }
+            else
+            {
+                pPrev = pInfo;
+                pInfo = pInfo->pNext;
+            }
+        rc = VINF_SUCCESS;
+    }
+    int rc2 = RTCritSectRwLeaveExcl(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc2);
+    AssertRC(rc);
+    LogFlow(("DBGFR3InfoDeregisterDriver: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Internal deregistration helper.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the VM.
+ * @param   pszName     The identifier of the info.
+ * @param   enmType1    The first info owner type (old style).
+ * @param   enmType2    The second info owner type (argv).
+ */
+static int dbgfR3InfoDeregister(PUVM pUVM, const char *pszName, DBGFINFOTYPE enmType1, DBGFINFOTYPE enmType2)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pszName, VERR_INVALID_POINTER);
+
+    /*
+     * Find the info handler.
+     */
+    size_t cchName = strlen(pszName);
+    int rc = RTCritSectRwEnterExcl(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc);
+    rc = VERR_FILE_NOT_FOUND;
+    PDBGFINFO pPrev = NULL;
+    PDBGFINFO pInfo = pUVM->dbgf.s.pInfoFirst;
+    for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
+        if (   pInfo->cchName == cchName
+            && memcmp(pInfo->szName, pszName, cchName) == 0
+            && (pInfo->enmType == enmType1 || pInfo->enmType == enmType2))
+        {
+            if (pPrev)
+                pPrev->pNext = pInfo->pNext;
+            else
+                pUVM->dbgf.s.pInfoFirst = pInfo->pNext;
+            MMR3HeapFree(pInfo);
+            rc = VINF_SUCCESS;
+            break;
+        }
+    int rc2 = RTCritSectRwLeaveExcl(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc2);
+    AssertRC(rc);
+    LogFlow(("dbgfR3InfoDeregister: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Deregister a info handler owned by an internal component.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pszName     The identifier of the info. If NULL all owned by the device.
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoDeregisterInternal(PVM pVM, const char *pszName)
+{
+    LogFlow(("DBGFR3InfoDeregisterInternal: pszName=%p:{%s}\n", pszName, pszName));
+    return dbgfR3InfoDeregister(pVM->pUVM, pszName, DBGFINFOTYPE_INT, DBGFINFOTYPE_INT_ARGV);
+}
+
+
+/**
+ * Deregister a info handler owned by an external component.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pszName     The identifier of the info. If NULL all owned by the device.
+ */
+VMMR3DECL(int) DBGFR3InfoDeregisterExternal(PUVM pUVM, const char *pszName)
+{
+    LogFlow(("DBGFR3InfoDeregisterExternal: pszName=%p:{%s}\n", pszName, pszName));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    return dbgfR3InfoDeregister(pUVM, pszName, DBGFINFOTYPE_EXT, DBGFINFOTYPE_EXT_ARGV);
+}
+
+
+/**
+ * Worker for DBGFR3InfoEx.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       Which CPU to run EMT bound handlers on.  VMCPUID_ANY or
+ *                      a valid CPU ID.
+ * @param   pszName     What to dump.
+ * @param   pszArgs     Arguments, optional.
+ * @param   pHlp        Output helper, optional.
+ */
+static DECLCALLBACK(int) dbgfR3Info(PUVM pUVM, VMCPUID idCpu, const char *pszName, const char *pszArgs, PCDBGFINFOHLP pHlp)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pszName, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pszArgs, VERR_INVALID_POINTER);
+    if (pHlp)
+    {
+        AssertPtrReturn(pHlp, VERR_INVALID_PARAMETER);
+        AssertPtrReturn(pHlp->pfnPrintf, VERR_INVALID_PARAMETER);
+        AssertPtrReturn(pHlp->pfnPrintfV, VERR_INVALID_PARAMETER);
+    }
+    else
+        pHlp = &g_dbgfR3InfoLogHlp;
+    Assert(idCpu == NIL_VMCPUID || idCpu < pUVM->cCpus); /* if not nil, we're on that EMT already. */
+
+    /*
+     * Find the info handler.
+     */
+    size_t cchName = strlen(pszName);
+    int rc = RTCritSectRwEnterShared(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc);
+    PDBGFINFO pInfo = pUVM->dbgf.s.pInfoFirst;
+    for (; pInfo; pInfo = pInfo->pNext)
+        if (    pInfo->cchName == cchName
+            &&  !memcmp(pInfo->szName, pszName, cchName))
+            break;
+    if (pInfo)
+    {
+        /*
+         * Found it.
+         */
+        VMCPUID idDstCpu = NIL_VMCPUID;
+        if ((pInfo->fFlags & (DBGFINFO_FLAGS_RUN_ON_EMT | DBGFINFO_FLAGS_ALL_EMTS)) && idCpu == NIL_VMCPUID)
+            idDstCpu = pInfo->fFlags & DBGFINFO_FLAGS_ALL_EMTS ? VMCPUID_ALL : VMCPUID_ANY;
+
+        rc = VINF_SUCCESS;
+        switch (pInfo->enmType)
+        {
+            case DBGFINFOTYPE_DEV:
+                if (idDstCpu != NIL_VMCPUID)
+                    rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.Dev.pfnHandler, 3,
+                                                  pInfo->u.Dev.pDevIns, pHlp, pszArgs);
+                else
+                    pInfo->u.Dev.pfnHandler(pInfo->u.Dev.pDevIns, pHlp, pszArgs);
+                break;
+
+            case DBGFINFOTYPE_DRV:
+                if (idDstCpu != NIL_VMCPUID)
+                    rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.Drv.pfnHandler, 3,
+                                                  pInfo->u.Drv.pDrvIns, pHlp, pszArgs);
+                else
+                    pInfo->u.Drv.pfnHandler(pInfo->u.Drv.pDrvIns, pHlp, pszArgs);
+                break;
+
+            case DBGFINFOTYPE_INT:
+                if (RT_VALID_PTR(pUVM->pVM))
+                {
+                    if (idDstCpu != NIL_VMCPUID)
+                        rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.Int.pfnHandler, 3,
+                                                      pUVM->pVM, pHlp, pszArgs);
+                    else
+                        pInfo->u.Int.pfnHandler(pUVM->pVM, pHlp, pszArgs);
+                }
+                else
+                    rc = VERR_INVALID_VM_HANDLE;
+                break;
+
+            case DBGFINFOTYPE_EXT:
+                if (idDstCpu != NIL_VMCPUID)
+                    rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.Ext.pfnHandler, 3,
+                                                  pInfo->u.Ext.pvUser, pHlp, pszArgs);
+                else
+                    pInfo->u.Ext.pfnHandler(pInfo->u.Ext.pvUser, pHlp, pszArgs);
+                break;
+
+            case DBGFINFOTYPE_DEV_ARGV:
+            case DBGFINFOTYPE_DRV_ARGV:
+            case DBGFINFOTYPE_USB_ARGV:
+            case DBGFINFOTYPE_INT_ARGV:
+            case DBGFINFOTYPE_EXT_ARGV:
+            {
+                char **papszArgv;
+                int    cArgs;
+                rc = RTGetOptArgvFromString(&papszArgv, &cArgs, pszArgs ? pszArgs : "", RTGETOPTARGV_CNV_QUOTE_BOURNE_SH, NULL);
+                if (RT_SUCCESS(rc))
+                {
+                    switch (pInfo->enmType)
+                    {
+                        case DBGFINFOTYPE_DEV_ARGV:
+                            if (idDstCpu != NIL_VMCPUID)
+                                rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.DevArgv.pfnHandler, 4,
+                                                              pInfo->u.DevArgv.pDevIns, pHlp, cArgs, papszArgv);
+                            else
+                                pInfo->u.DevArgv.pfnHandler(pInfo->u.DevArgv.pDevIns, pHlp, cArgs, papszArgv);
+                            break;
+
+                        case DBGFINFOTYPE_DRV_ARGV:
+                            if (idDstCpu != NIL_VMCPUID)
+                                rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.DrvArgv.pfnHandler, 4,
+                                                              pInfo->u.DrvArgv.pDrvIns, pHlp, cArgs, papszArgv);
+                            else
+                                pInfo->u.DrvArgv.pfnHandler(pInfo->u.DrvArgv.pDrvIns, pHlp, cArgs, papszArgv);
+                            break;
+
+                        case DBGFINFOTYPE_USB_ARGV:
+                            if (idDstCpu != NIL_VMCPUID)
+                                rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.UsbArgv.pfnHandler, 4,
+                                                              pInfo->u.UsbArgv.pUsbIns, pHlp, cArgs, papszArgv);
+                            else
+                                pInfo->u.UsbArgv.pfnHandler(pInfo->u.UsbArgv.pUsbIns, pHlp, cArgs, papszArgv);
+                            break;
+
+                        case DBGFINFOTYPE_INT_ARGV:
+                            if (RT_VALID_PTR(pUVM->pVM))
+                            {
+                                if (idDstCpu != NIL_VMCPUID)
+                                    rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.IntArgv.pfnHandler, 4,
+                                                                  pUVM->pVM, pHlp, cArgs, papszArgv);
+                                else
+                                    pInfo->u.IntArgv.pfnHandler(pUVM->pVM, pHlp, cArgs, papszArgv);
+                            }
+                            else
+                                rc = VERR_INVALID_VM_HANDLE;
+                            break;
+
+                        case DBGFINFOTYPE_EXT_ARGV:
+                            if (idDstCpu != NIL_VMCPUID)
+                                rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.ExtArgv.pfnHandler, 4,
+                                                              pInfo->u.ExtArgv.pvUser, pHlp, cArgs, papszArgv);
+                            else
+                                pInfo->u.ExtArgv.pfnHandler(pInfo->u.ExtArgv.pvUser, pHlp, cArgs, papszArgv);
+                            break;
+
+                        default:
+                            AssertFailedBreakStmt(rc = VERR_INTERNAL_ERROR);
+                    }
+
+                    RTGetOptArgvFree(papszArgv);
+                }
+                break;
+            }
+
+            default:
+                AssertMsgFailedReturn(("Invalid info type enmType=%d\n", pInfo->enmType), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+        }
+
+        int rc2 = RTCritSectRwLeaveShared(&pUVM->dbgf.s.CritSect);
+        AssertRC(rc2);
+    }
+    else
+    {
+        rc = RTCritSectRwLeaveShared(&pUVM->dbgf.s.CritSect);
+        AssertRC(rc);
+        rc = VERR_FILE_NOT_FOUND;
+    }
+    return rc;
+}
+
+
+/**
+ * Display a piece of info writing to the supplied handler.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pszName     The identifier of the info to display.
+ * @param   pszArgs     Arguments to the info handler.
+ * @param   pHlp        The output helper functions. If NULL the logger will be used.
+ */
+VMMR3DECL(int) DBGFR3Info(PUVM pUVM, const char *pszName, const char *pszArgs, PCDBGFINFOHLP pHlp)
+{
+    return DBGFR3InfoEx(pUVM, NIL_VMCPUID, pszName, pszArgs, pHlp);
+}
+
+
+/**
+ * Display a piece of info writing to the supplied handler.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The CPU to exectue the request on.  Pass NIL_VMCPUID
+ *                      to not involve any EMT unless necessary.
+ * @param   pszName     The identifier of the info to display.
+ * @param   pszArgs     Arguments to the info handler.
+ * @param   pHlp        The output helper functions. If NULL the logger will be used.
+ */
+VMMR3DECL(int) DBGFR3InfoEx(PUVM pUVM, VMCPUID idCpu, const char *pszName, const char *pszArgs, PCDBGFINFOHLP pHlp)
+{
+    /*
+     * Some input validation.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(   idCpu != VMCPUID_ANY_QUEUE
+                 && idCpu != VMCPUID_ALL
+                 && idCpu != VMCPUID_ALL_REVERSE, VERR_INVALID_PARAMETER);
+
+    /*
+     * Run on any specific EMT?
+     */
+    if (idCpu == NIL_VMCPUID)
+        return dbgfR3Info(pUVM, NIL_VMCPUID, pszName, pszArgs, pHlp);
+    return VMR3ReqPriorityCallWaitU(pUVM, idCpu,
+                                    (PFNRT)dbgfR3Info, 5, pUVM, idCpu, pszName, pszArgs, pHlp);
+}
+
+
+/**
+ * Wrapper for DBGFR3Info that outputs to the release log.
+ *
+ * @returns See DBGFR3Info.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pszName     See DBGFR3Info.
+ * @param   pszArgs     See DBGFR3Info.
+ */
+VMMR3DECL(int) DBGFR3InfoLogRel(PUVM pUVM, const char *pszName, const char *pszArgs)
+{
+    return DBGFR3InfoEx(pUVM, NIL_VMCPUID, pszName, pszArgs, &g_dbgfR3InfoLogRelHlp);
+}
+
+
+/**
+ * Wrapper for DBGFR3Info that outputs to standard error.
+ *
+ * @returns See DBGFR3Info.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pszName     See DBGFR3Info.
+ * @param   pszArgs     See DBGFR3Info.
+ */
+VMMR3DECL(int) DBGFR3InfoStdErr(PUVM pUVM, const char *pszName, const char *pszArgs)
+{
+    return DBGFR3InfoEx(pUVM, NIL_VMCPUID, pszName, pszArgs, &g_dbgfR3InfoStdErrHlp);
+}
+
+
+/**
+ * Display several info items.
+ *
+ * This is intended used by the fatal error dump only.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pszIncludePat   Simple string pattern of info items to include.
+ * @param   pszExcludePat   Simple string pattern of info items to exclude.
+ * @param   pszSepFmt       Item separator format string.  The item name will be
+ *                          given as parameter.
+ * @param   pHlp            The output helper functions.  If NULL the logger
+ *                          will be used.
+ *
+ * @thread  EMT
+ */
+VMMR3_INT_DECL(int) DBGFR3InfoMulti(PVM pVM, const char *pszIncludePat, const char *pszExcludePat, const char *pszSepFmt,
+                                    PCDBGFINFOHLP pHlp)
+{
+    /*
+     * Validate input.
+     */
+    PUVM pUVM = pVM->pUVM;
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pszIncludePat, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszExcludePat, VERR_INVALID_POINTER);
+    if (pHlp)
+    {
+        AssertPtrReturn(pHlp->pfnPrintf, VERR_INVALID_POINTER);
+        AssertPtrReturn(pHlp->pfnPrintfV, VERR_INVALID_POINTER);
+    }
+    else
+        pHlp = &g_dbgfR3InfoLogHlp;
+
+    size_t const cchIncludePat = strlen(pszIncludePat);
+    size_t const cchExcludePat = strlen(pszExcludePat);
+    const char *pszArgs = "";
+
+    /*
+     * Enumerate the info handlers and call the ones matching.
+     * Note! We won't leave the critical section here...
+     */
+    char *apszArgs[2] = { NULL, NULL };
+    int rc = RTCritSectRwEnterShared(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc);
+    rc = VWRN_NOT_FOUND;
+    for (PDBGFINFO pInfo = pUVM->dbgf.s.pInfoFirst; pInfo; pInfo = pInfo->pNext)
+    {
+        if (   RTStrSimplePatternMultiMatch(pszIncludePat, cchIncludePat, pInfo->szName, pInfo->cchName, NULL)
+            && !RTStrSimplePatternMultiMatch(pszExcludePat, cchExcludePat, pInfo->szName, pInfo->cchName, NULL))
+        {
+            pHlp->pfnPrintf(pHlp, pszSepFmt, pInfo->szName);
+
+            VMCPUID idDstCpu = NIL_VMCPUID;
+            if (pInfo->fFlags & (DBGFINFO_FLAGS_RUN_ON_EMT | DBGFINFO_FLAGS_ALL_EMTS))
+                idDstCpu = pInfo->fFlags & DBGFINFO_FLAGS_ALL_EMTS ? VMCPUID_ALL : VMCPUID_ANY;
+
+            rc = VINF_SUCCESS;
+            switch (pInfo->enmType)
+            {
+                case DBGFINFOTYPE_DEV:
+                    if (idDstCpu != NIL_VMCPUID)
+                        rc = VMR3ReqPriorityCallVoidWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.Dev.pfnHandler, 3,
+                                                          pInfo->u.Dev.pDevIns, pHlp, pszArgs);
+                    else
+                        pInfo->u.Dev.pfnHandler(pInfo->u.Dev.pDevIns, pHlp, pszArgs);
+                    break;
+
+                case DBGFINFOTYPE_DRV:
+                    if (idDstCpu != NIL_VMCPUID)
+                        rc = VMR3ReqPriorityCallVoidWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.Drv.pfnHandler, 3,
+                                                          pInfo->u.Drv.pDrvIns, pHlp, pszArgs);
+                    else
+                        pInfo->u.Drv.pfnHandler(pInfo->u.Drv.pDrvIns, pHlp, pszArgs);
+                    break;
+
+                case DBGFINFOTYPE_INT:
+                    if (idDstCpu != NIL_VMCPUID)
+                        rc = VMR3ReqPriorityCallVoidWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.Int.pfnHandler, 3,
+                                                          pVM, pHlp, pszArgs);
+                    else
+                        pInfo->u.Int.pfnHandler(pVM, pHlp, pszArgs);
+                    break;
+
+                case DBGFINFOTYPE_EXT:
+                    if (idDstCpu != NIL_VMCPUID)
+                        rc = VMR3ReqPriorityCallVoidWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.Ext.pfnHandler, 3,
+                                                          pInfo->u.Ext.pvUser, pHlp, pszArgs);
+                    else
+                        pInfo->u.Ext.pfnHandler(pInfo->u.Ext.pvUser, pHlp, pszArgs);
+                    break;
+
+                case DBGFINFOTYPE_DEV_ARGV:
+                    if (idDstCpu != NIL_VMCPUID)
+                        rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.DevArgv.pfnHandler, 4,
+                                                      pInfo->u.DevArgv.pDevIns, pHlp, 0, &apszArgs[0]);
+                    else
+                        pInfo->u.DevArgv.pfnHandler(pInfo->u.DevArgv.pDevIns, pHlp, 0, &apszArgs[0]);
+                    break;
+
+                case DBGFINFOTYPE_DRV_ARGV:
+                    if (idDstCpu != NIL_VMCPUID)
+                        rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.DrvArgv.pfnHandler, 4,
+                                                      pInfo->u.DrvArgv.pDrvIns, pHlp, 0, &apszArgs[0]);
+                    else
+                        pInfo->u.DrvArgv.pfnHandler(pInfo->u.DrvArgv.pDrvIns, pHlp, 0, &apszArgs[0]);
+                    break;
+
+                case DBGFINFOTYPE_USB_ARGV:
+                    if (idDstCpu != NIL_VMCPUID)
+                        rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.UsbArgv.pfnHandler, 4,
+                                                      pInfo->u.UsbArgv.pUsbIns, pHlp, 0, &apszArgs[0]);
+                    else
+                        pInfo->u.UsbArgv.pfnHandler(pInfo->u.UsbArgv.pUsbIns, pHlp, 0, &apszArgs[0]);
+                    break;
+
+                case DBGFINFOTYPE_INT_ARGV:
+                    if (RT_VALID_PTR(pUVM->pVM))
+                    {
+                        if (idDstCpu != NIL_VMCPUID)
+                            rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.IntArgv.pfnHandler, 4,
+                                                          pUVM->pVM, pHlp, 0, &apszArgs[0]);
+                        else
+                            pInfo->u.IntArgv.pfnHandler(pUVM->pVM, pHlp, 0, &apszArgs[0]);
+                    }
+                    else
+                        rc = VERR_INVALID_VM_HANDLE;
+                    break;
+
+                case DBGFINFOTYPE_EXT_ARGV:
+                    if (idDstCpu != NIL_VMCPUID)
+                        rc = VMR3ReqPriorityCallWaitU(pUVM, idDstCpu, (PFNRT)pInfo->u.ExtArgv.pfnHandler, 4,
+                                                      pInfo->u.ExtArgv.pvUser, pHlp, 0, &apszArgs[0]);
+                    else
+                        pInfo->u.ExtArgv.pfnHandler(pInfo->u.ExtArgv.pvUser, pHlp, 0, &apszArgs[0]);
+                    break;
+
+                default:
+                    AssertMsgFailedReturn(("Invalid info type enmType=%d\n", pInfo->enmType), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+            }
+        }
+    }
+    int rc2 = RTCritSectRwLeaveShared(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc2);
+
+    return rc;
+}
+
+
+/**
+ * Enumerate all the register info handlers.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pfnCallback     Pointer to callback function.
+ * @param   pvUser          User argument to pass to the callback.
+ */
+VMMR3DECL(int) DBGFR3InfoEnum(PUVM pUVM, PFNDBGFINFOENUM pfnCallback, void *pvUser)
+{
+    LogFlow(("DBGFR3InfoLog: pfnCallback=%p pvUser=%p\n", pfnCallback, pvUser));
+
+    /*
+     * Validate input.
+     */
+    if (!pfnCallback)
+    {
+        AssertMsgFailed(("!pfnCallback\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Enter and enumerate.
+     */
+    int rc = RTCritSectRwEnterShared(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc);
+
+    rc = VINF_SUCCESS;
+    for (PDBGFINFO pInfo = pUVM->dbgf.s.pInfoFirst; RT_SUCCESS(rc) && pInfo; pInfo = pInfo->pNext)
+        rc = pfnCallback(pUVM, pInfo->szName, pInfo->pszDesc, pvUser);
+
+    /*
+     * Leave and exit.
+     */
+    int rc2 = RTCritSectRwLeaveShared(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc2);
+
+    LogFlow(("DBGFR3InfoLog: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Info handler, internal version.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        Callback functions for doing output.
+ * @param   pszArgs     Argument string. Optional and specific to the handler.
+ */
+static DECLCALLBACK(void) dbgfR3InfoHelp(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    LogFlow(("dbgfR3InfoHelp: pszArgs=%s\n", pszArgs));
+
+    /*
+     * Enter and enumerate.
+     */
+    PUVM pUVM = pVM->pUVM;
+    int rc = RTCritSectRwEnterShared(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc);
+
+    if (pszArgs && *pszArgs)
+    {
+        for (PDBGFINFO pInfo = pUVM->dbgf.s.pInfoFirst; pInfo; pInfo = pInfo->pNext)
+        {
+            const char *psz = strstr(pszArgs, pInfo->szName);
+            if (    psz
+                &&  (   psz == pszArgs
+                     || RT_C_IS_SPACE(psz[-1]))
+                &&  (   !psz[pInfo->cchName]
+                     || RT_C_IS_SPACE(psz[pInfo->cchName])))
+                pHlp->pfnPrintf(pHlp, "%-16s  %s\n",
+                                pInfo->szName, pInfo->pszDesc);
+        }
+    }
+    else
+    {
+        for (PDBGFINFO pInfo = pUVM->dbgf.s.pInfoFirst; pInfo; pInfo = pInfo->pNext)
+            pHlp->pfnPrintf(pHlp, "%-16s  %s\n",
+                            pInfo->szName, pInfo->pszDesc);
+    }
+
+    /*
+     * Leave and exit.
+     */
+    rc = RTCritSectRwLeaveShared(&pUVM->dbgf.s.CritSect);
+    AssertRC(rc);
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFLog.cpp b/src/VBox/VMM/VMMR3/DBGFLog.cpp
new file mode 100644
index 00000000..9e6d48e5
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFLog.cpp
@@ -0,0 +1,187 @@
+/* $Id: DBGFLog.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Log Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/vmapi.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+#include <iprt/param.h>
+#include <iprt/string.h>
+
+
+/**
+ * Checkes for logger prefixes and selects the right logger.
+ *
+ * @returns Target logger.
+ * @param   ppsz                Pointer to the string pointer.
+ */
+static PRTLOGGER dbgfR3LogResolvedLogger(const char **ppsz)
+{
+    PRTLOGGER   pLogger;
+    const char *psz = *ppsz;
+    if (!strncmp(psz, RT_STR_TUPLE("release:")))
+    {
+        *ppsz += sizeof("release:") - 1;
+        pLogger = RTLogRelGetDefaultInstance();
+    }
+    else
+    {
+        if (!strncmp(psz, RT_STR_TUPLE("debug:")))
+            *ppsz += sizeof("debug:") - 1;
+        pLogger = RTLogDefaultInstance();
+    }
+    return pLogger;
+}
+
+
+/**
+ * EMT worker for DBGFR3LogModifyGroups.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszGroupSettings    The group settings string. (VBOX_LOG)
+ */
+static DECLCALLBACK(int) dbgfR3LogModifyGroups(PUVM pUVM, const char *pszGroupSettings)
+{
+    PRTLOGGER pLogger = dbgfR3LogResolvedLogger(&pszGroupSettings);
+    if (!pLogger)
+        return VINF_SUCCESS;
+
+    int rc = RTLogGroupSettings(pLogger, pszGroupSettings);
+    if (RT_SUCCESS(rc) && pUVM->pVM)
+    {
+        VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+        rc = VMMR3UpdateLoggers(pUVM->pVM);
+    }
+    return rc;
+}
+
+
+/**
+ * Changes the logger group settings.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszGroupSettings    The group settings string. (VBOX_LOG)
+ *                              By prefixing the string with \"release:\" the
+ *                              changes will be applied to the release log
+ *                              instead of the debug log.  The prefix \"debug:\"
+ *                              is also recognized.
+ */
+VMMR3DECL(int) DBGFR3LogModifyGroups(PUVM pUVM, const char *pszGroupSettings)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszGroupSettings, VERR_INVALID_POINTER);
+
+    return VMR3ReqPriorityCallWaitU(pUVM, VMCPUID_ANY, (PFNRT)dbgfR3LogModifyGroups, 2, pUVM, pszGroupSettings);
+}
+
+
+/**
+ * EMT worker for DBGFR3LogModifyFlags.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszFlagSettings     The group settings string. (VBOX_LOG_FLAGS)
+ */
+static DECLCALLBACK(int) dbgfR3LogModifyFlags(PUVM pUVM, const char *pszFlagSettings)
+{
+    PRTLOGGER pLogger = dbgfR3LogResolvedLogger(&pszFlagSettings);
+    if (!pLogger)
+        return VINF_SUCCESS;
+
+    int rc = RTLogFlags(pLogger, pszFlagSettings);
+    if (RT_SUCCESS(rc) && pUVM->pVM)
+    {
+        VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+        rc = VMMR3UpdateLoggers(pUVM->pVM);
+    }
+    return rc;
+}
+
+
+/**
+ * Changes the logger flag settings.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszFlagSettings     The group settings string. (VBOX_LOG_FLAGS)
+ *                              By prefixing the string with \"release:\" the
+ *                              changes will be applied to the release log
+ *                              instead of the debug log.  The prefix \"debug:\"
+ *                              is also recognized.
+ */
+VMMR3DECL(int) DBGFR3LogModifyFlags(PUVM pUVM, const char *pszFlagSettings)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszFlagSettings, VERR_INVALID_POINTER);
+
+    return VMR3ReqPriorityCallWaitU(pUVM, VMCPUID_ANY, (PFNRT)dbgfR3LogModifyFlags, 2, pUVM, pszFlagSettings);
+}
+
+
+/**
+ * EMT worker for DBGFR3LogModifyFlags.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszDestSettings     The destination settings string. (VBOX_LOG_DEST)
+ */
+static DECLCALLBACK(int) dbgfR3LogModifyDestinations(PUVM pUVM, const char *pszDestSettings)
+{
+    PRTLOGGER pLogger = dbgfR3LogResolvedLogger(&pszDestSettings);
+    if (!pLogger)
+        return VINF_SUCCESS;
+
+    int rc = RTLogDestinations(NULL, pszDestSettings);
+    if (RT_SUCCESS(rc) && pUVM->pVM)
+    {
+        VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+        rc = VMMR3UpdateLoggers(pUVM->pVM);
+    }
+    return rc;
+}
+
+
+/**
+ * Changes the logger destination settings.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszDestSettings     The destination settings string. (VBOX_LOG_DEST)
+ *                              By prefixing the string with \"release:\" the
+ *                              changes will be applied to the release log
+ *                              instead of the debug log.  The prefix \"debug:\"
+ *                              is also recognized.
+ */
+VMMR3DECL(int) DBGFR3LogModifyDestinations(PUVM pUVM, const char *pszDestSettings)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszDestSettings, VERR_INVALID_POINTER);
+
+    return VMR3ReqPriorityCallWaitU(pUVM, VMCPUID_ANY, (PFNRT)dbgfR3LogModifyDestinations, 2, pUVM, pszDestSettings);
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFMem.cpp b/src/VBox/VMM/VMMR3/DBGFMem.cpp
new file mode 100644
index 00000000..babd88ab
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFMem.cpp
@@ -0,0 +1,643 @@
+/* $Id: DBGFMem.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Memory Methods.
+ */
+
+/*
+ * Copyright (C) 2007-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/hm.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/vmm/mm.h>
+
+
+
+/**
+ * Scan guest memory for an exact byte string.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The ID of the CPU context to search in.
+ * @param   pAddress    Where to store the mixed address.
+ * @param   puAlign     The alignment restriction imposed on the search result.
+ * @param   pcbRange    The number of bytes to scan. Passed as a pointer because
+ *                      it may be 64-bit.
+ * @param   pabNeedle   What to search for - exact search.
+ * @param   cbNeedle    Size of the search byte string.
+ * @param   pHitAddress Where to put the address of the first hit.
+ */
+static DECLCALLBACK(int) dbgfR3MemScan(PUVM pUVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, PCRTGCUINTPTR pcbRange,
+                                       RTGCUINTPTR *puAlign, const uint8_t *pabNeedle, size_t cbNeedle, PDBGFADDRESS pHitAddress)
+{
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    Assert(idCpu == VMMGetCpuId(pVM));
+
+    /*
+     * Validate the input we use, PGM does the rest.
+     */
+    RTGCUINTPTR cbRange = *pcbRange;
+    if (!DBGFR3AddrIsValid(pUVM, pAddress))
+        return VERR_INVALID_POINTER;
+    if (!VALID_PTR(pHitAddress))
+        return VERR_INVALID_POINTER;
+
+    /*
+     * Select DBGF worker by addressing mode.
+     */
+    int     rc;
+    PVMCPU  pVCpu   = VMMGetCpuById(pVM, idCpu);
+    PGMMODE enmMode = PGMGetGuestMode(pVCpu);
+    if (    enmMode == PGMMODE_REAL
+        ||  enmMode == PGMMODE_PROTECTED
+        ||  DBGFADDRESS_IS_PHYS(pAddress)
+        )
+    {
+        RTGCPHYS GCPhysAlign = *puAlign;
+        if (GCPhysAlign != *puAlign)
+            return VERR_OUT_OF_RANGE;
+        RTGCPHYS PhysHit;
+        rc = PGMR3DbgScanPhysical(pVM, pAddress->FlatPtr, cbRange, GCPhysAlign, pabNeedle, cbNeedle, &PhysHit);
+        if (RT_SUCCESS(rc))
+            DBGFR3AddrFromPhys(pUVM, pHitAddress, PhysHit);
+    }
+    else
+    {
+#if GC_ARCH_BITS > 32
+        if (    (   pAddress->FlatPtr >= _4G
+                 || pAddress->FlatPtr + cbRange > _4G)
+            &&  enmMode != PGMMODE_AMD64
+            &&  enmMode != PGMMODE_AMD64_NX)
+            return VERR_DBGF_MEM_NOT_FOUND;
+#endif
+        RTGCUINTPTR GCPtrHit;
+        rc = PGMR3DbgScanVirtual(pVM, pVCpu, pAddress->FlatPtr, cbRange, *puAlign, pabNeedle, cbNeedle, &GCPtrHit);
+        if (RT_SUCCESS(rc))
+            DBGFR3AddrFromFlat(pUVM, pHitAddress, GCPtrHit);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Scan guest memory for an exact byte string.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS and *pGCPtrHit on success.
+ * @retval  VERR_DBGF_MEM_NOT_FOUND if not found.
+ * @retval  VERR_INVALID_POINTER if any of the pointer arguments are invalid.
+ * @retval  VERR_INVALID_ARGUMENT if any other arguments are invalid.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The ID of the CPU context to search in.
+ * @param   pAddress    Where to store the mixed address.
+ * @param   cbRange     The number of bytes to scan.
+ * @param   uAlign      The alignment restriction imposed on the result.
+ *                      Usually set to 1.
+ * @param   pvNeedle    What to search for - exact search.
+ * @param   cbNeedle    Size of the search byte string.
+ * @param   pHitAddress Where to put the address of the first hit.
+ *
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3MemScan(PUVM pUVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, RTGCUINTPTR cbRange, RTGCUINTPTR uAlign,
+                             const void *pvNeedle, size_t cbNeedle, PDBGFADDRESS pHitAddress)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_CPU_ID);
+    return VMR3ReqPriorityCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3MemScan, 8,
+                                    pUVM, idCpu, pAddress, &cbRange, &uAlign, pvNeedle, cbNeedle, pHitAddress);
+
+}
+
+
+/**
+ * Read guest memory.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The ID of the CPU context to read memory from.
+ * @param   pAddress    Where to start reading.
+ * @param   pvBuf       Where to store the data we've read.
+ * @param   cbRead      The number of bytes to read.
+ */
+static DECLCALLBACK(int) dbgfR3MemRead(PUVM pUVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, void *pvBuf, size_t cbRead)
+{
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    Assert(idCpu == VMMGetCpuId(pVM));
+
+    /*
+     * Validate the input we use, PGM does the rest.
+     */
+    if (!DBGFR3AddrIsValid(pUVM, pAddress))
+        return VERR_INVALID_POINTER;
+    if (!VALID_PTR(pvBuf))
+        return VERR_INVALID_POINTER;
+
+    /*
+     * Select PGM worker by addressing mode.
+     */
+    int rc;
+    PVMCPU  pVCpu   = VMMGetCpuById(pVM, idCpu);
+    PGMMODE enmMode = PGMGetGuestMode(pVCpu);
+    if (    enmMode == PGMMODE_REAL
+        ||  enmMode == PGMMODE_PROTECTED
+        ||  DBGFADDRESS_IS_PHYS(pAddress) )
+        rc = PGMPhysSimpleReadGCPhys(pVM, pvBuf, pAddress->FlatPtr, cbRead);
+    else
+    {
+#if GC_ARCH_BITS > 32
+        if (    (   pAddress->FlatPtr >= _4G
+                 || pAddress->FlatPtr + cbRead > _4G)
+            &&  enmMode != PGMMODE_AMD64
+            &&  enmMode != PGMMODE_AMD64_NX)
+            return VERR_PAGE_TABLE_NOT_PRESENT;
+#endif
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, pvBuf, pAddress->FlatPtr, cbRead);
+    }
+    return rc;
+}
+
+
+/**
+ * Read guest memory.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The ID of the source CPU context (for the address).
+ * @param   pAddress    Where to start reading.
+ * @param   pvBuf       Where to store the data we've read.
+ * @param   cbRead      The number of bytes to read.
+ */
+VMMR3DECL(int) DBGFR3MemRead(PUVM pUVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, void *pvBuf, size_t cbRead)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_CPU_ID);
+
+    if ((pAddress->fFlags & DBGFADDRESS_FLAGS_TYPE_MASK) == DBGFADDRESS_FLAGS_RING0)
+    {
+        AssertCompile(sizeof(RTHCUINTPTR) <= sizeof(pAddress->FlatPtr));
+        VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+        return VMMR3ReadR0Stack(pUVM->pVM, idCpu, (RTHCUINTPTR)pAddress->FlatPtr, pvBuf, cbRead);
+    }
+    return VMR3ReqPriorityCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3MemRead, 5, pUVM, idCpu, pAddress, pvBuf, cbRead);
+}
+
+
+/**
+ * Read a zero terminated string from guest memory.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The ID of the source CPU context (for the address).
+ * @param   pAddress    Where to start reading.
+ * @param   pszBuf      Where to store the string.
+ * @param   cchBuf      The size of the buffer.
+ */
+static DECLCALLBACK(int) dbgfR3MemReadString(PUVM pUVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, char *pszBuf, size_t cchBuf)
+{
+    /*
+     * Validate the input we use, PGM does the rest.
+     */
+    if (!DBGFR3AddrIsValid(pUVM, pAddress))
+        return VERR_INVALID_POINTER;
+    if (!VALID_PTR(pszBuf))
+        return VERR_INVALID_POINTER;
+
+    /*
+     * Let dbgfR3MemRead do the job.
+     */
+    int rc = dbgfR3MemRead(pUVM, idCpu, pAddress, pszBuf, cchBuf);
+
+    /*
+     * Make sure the result is terminated and that overflow is signaled.
+     * This may look a bit reckless with the rc but, it should be fine.
+     */
+    if (!RTStrEnd(pszBuf, cchBuf))
+    {
+        pszBuf[cchBuf - 1] = '\0';
+        rc = VINF_BUFFER_OVERFLOW;
+    }
+    /*
+     * Handle partial reads (not perfect).
+     */
+    else if (RT_FAILURE(rc))
+    {
+        if (pszBuf[0])
+            rc = VINF_SUCCESS;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Read a zero terminated string from guest memory.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The ID of the source CPU context (for the address).
+ * @param   pAddress    Where to start reading.
+ * @param   pszBuf      Where to store the string.
+ * @param   cchBuf      The size of the buffer.
+ */
+VMMR3DECL(int) DBGFR3MemReadString(PUVM pUVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, char *pszBuf, size_t cchBuf)
+{
+    /*
+     * Validate and zero output.
+     */
+    if (!VALID_PTR(pszBuf))
+        return VERR_INVALID_POINTER;
+    if (cchBuf <= 0)
+        return VERR_INVALID_PARAMETER;
+    memset(pszBuf, 0, cchBuf);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_CPU_ID);
+
+    /*
+     * Pass it on to the EMT.
+     */
+    return VMR3ReqPriorityCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3MemReadString, 5, pUVM, idCpu, pAddress, pszBuf, cchBuf);
+}
+
+
+/**
+ * Writes guest memory.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The ID of the target CPU context (for the address).
+ * @param   pAddress    Where to start writing.
+ * @param   pvBuf       The data to write.
+ * @param   cbWrite     The number of bytes to write.
+ */
+static DECLCALLBACK(int) dbgfR3MemWrite(PUVM pUVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, void const *pvBuf, size_t cbWrite)
+{
+    /*
+     * Validate the input we use, PGM does the rest.
+     */
+    if (!DBGFR3AddrIsValid(pUVM, pAddress))
+        return VERR_INVALID_POINTER;
+    if (!VALID_PTR(pvBuf))
+        return VERR_INVALID_POINTER;
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Select PGM function by addressing mode.
+     */
+    int rc;
+    PVMCPU  pVCpu   = VMMGetCpuById(pVM, idCpu);
+    PGMMODE enmMode = PGMGetGuestMode(pVCpu);
+    if (    enmMode == PGMMODE_REAL
+        ||  enmMode == PGMMODE_PROTECTED
+        ||  DBGFADDRESS_IS_PHYS(pAddress) )
+        rc = PGMPhysSimpleWriteGCPhys(pVM, pAddress->FlatPtr, pvBuf, cbWrite);
+    else
+    {
+#if GC_ARCH_BITS > 32
+        if (    (   pAddress->FlatPtr >= _4G
+                 || pAddress->FlatPtr + cbWrite > _4G)
+            &&  enmMode != PGMMODE_AMD64
+            &&  enmMode != PGMMODE_AMD64_NX)
+            return VERR_PAGE_TABLE_NOT_PRESENT;
+#endif
+        rc = PGMPhysSimpleWriteGCPtr(pVCpu, pAddress->FlatPtr, pvBuf, cbWrite);
+    }
+    return rc;
+}
+
+
+/**
+ * Read guest memory.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The ID of the target CPU context (for the address).
+ * @param   pAddress    Where to start writing.
+ * @param   pvBuf       The data to write.
+ * @param   cbWrite     The number of bytes to write.
+ */
+VMMR3DECL(int) DBGFR3MemWrite(PUVM pUVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, void const *pvBuf, size_t cbWrite)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_CPU_ID);
+    return VMR3ReqPriorityCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3MemWrite, 5, pUVM, idCpu, pAddress, pvBuf, cbWrite);
+}
+
+
+/**
+ * Worker for DBGFR3SelQueryInfo that calls into SELM.
+ */
+static DECLCALLBACK(int) dbgfR3SelQueryInfo(PUVM pUVM, VMCPUID idCpu, RTSEL Sel, uint32_t fFlags, PDBGFSELINFO pSelInfo)
+{
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Make the query.
+     */
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+    int rc = SELMR3GetSelectorInfo(pVCpu, Sel, pSelInfo);
+
+    /*
+     * 64-bit mode HACKS for making data and stack selectors wide open when
+     * queried. This is voodoo magic.
+     */
+    if (fFlags & DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE)
+    {
+        /* Expand 64-bit data and stack selectors. The check is a bit bogus... */
+        if (    RT_SUCCESS(rc)
+            &&  (pSelInfo->fFlags & (  DBGFSELINFO_FLAGS_LONG_MODE | DBGFSELINFO_FLAGS_REAL_MODE | DBGFSELINFO_FLAGS_PROT_MODE
+                                     | DBGFSELINFO_FLAGS_GATE      | DBGFSELINFO_FLAGS_HYPER
+                                     | DBGFSELINFO_FLAGS_INVALID   | DBGFSELINFO_FLAGS_NOT_PRESENT))
+                 == DBGFSELINFO_FLAGS_LONG_MODE
+            &&  pSelInfo->cbLimit != ~(RTGCPTR)0
+            &&  CPUMIsGuestIn64BitCode(pVCpu) )
+        {
+            pSelInfo->GCPtrBase = 0;
+            pSelInfo->cbLimit   = ~(RTGCPTR)0;
+        }
+        else if (   Sel == 0
+                 && CPUMIsGuestIn64BitCode(pVCpu))
+        {
+            pSelInfo->GCPtrBase = 0;
+            pSelInfo->cbLimit   = ~(RTGCPTR)0;
+            pSelInfo->Sel       = 0;
+            pSelInfo->SelGate   = 0;
+            pSelInfo->fFlags    = DBGFSELINFO_FLAGS_LONG_MODE;
+            pSelInfo->u.Raw64.Gen.u1Present  = 1;
+            pSelInfo->u.Raw64.Gen.u1Long     = 1;
+            pSelInfo->u.Raw64.Gen.u1DescType = 1;
+            rc = VINF_SUCCESS;
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Gets information about a selector.
+ *
+ * Intended for the debugger mostly and will prefer the guest
+ * descriptor tables over the shadow ones.
+ *
+ * @returns VBox status code, the following are the common ones.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_INVALID_SELECTOR if the selector isn't fully inside the
+ *          descriptor table.
+ * @retval  VERR_SELECTOR_NOT_PRESENT if the LDT is invalid or not present. This
+ *          is not returned if the selector itself isn't present, you have to
+ *          check that for yourself (see DBGFSELINFO::fFlags).
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT or VERR_PAGE_NOT_PRESENT if the
+ *          pagetable or page backing the selector table wasn't present.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The ID of the virtual CPU context.
+ * @param   Sel         The selector to get info about.
+ * @param   fFlags      Flags, see DBGFQSEL_FLAGS_*.
+ * @param   pSelInfo    Where to store the information. This will always be
+ *                      updated.
+ *
+ * @remarks This is a wrapper around SELMR3GetSelectorInfo and
+ *          SELMR3GetShadowSelectorInfo.
+ */
+VMMR3DECL(int) DBGFR3SelQueryInfo(PUVM pUVM, VMCPUID idCpu, RTSEL Sel, uint32_t fFlags, PDBGFSELINFO pSelInfo)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_CPU_ID);
+    AssertReturn(!(fFlags & ~(DBGFSELQI_FLAGS_DT_GUEST | DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE)), VERR_INVALID_PARAMETER);
+
+    /* Clear the return data here on this thread. */
+    memset(pSelInfo, 0, sizeof(*pSelInfo));
+
+    /*
+     * Dispatch the request to a worker running on the target CPU.
+     */
+    return VMR3ReqPriorityCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3SelQueryInfo, 5, pUVM, idCpu, Sel, fFlags, pSelInfo);
+}
+
+
+/**
+ * Validates a CS selector.
+ *
+ * @returns VBox status code.
+ * @param   pSelInfo    Pointer to the selector information for the CS selector.
+ * @param   SelCPL      The selector defining the CPL (SS).
+ */
+VMMDECL(int) DBGFR3SelInfoValidateCS(PCDBGFSELINFO pSelInfo, RTSEL SelCPL)
+{
+    /*
+     * Check if present.
+     */
+    if (pSelInfo->u.Raw.Gen.u1Present)
+    {
+        /*
+         * Type check.
+         */
+        if (    pSelInfo->u.Raw.Gen.u1DescType == 1
+            &&  (pSelInfo->u.Raw.Gen.u4Type & X86_SEL_TYPE_CODE))
+        {
+            /*
+             * Check level.
+             */
+            unsigned uLevel = RT_MAX(SelCPL & X86_SEL_RPL, pSelInfo->Sel & X86_SEL_RPL);
+            if (    !(pSelInfo->u.Raw.Gen.u4Type & X86_SEL_TYPE_CONF)
+                ?   uLevel <= pSelInfo->u.Raw.Gen.u2Dpl
+                :   uLevel >= pSelInfo->u.Raw.Gen.u2Dpl /* hope I got this right now... */
+                    )
+                return VINF_SUCCESS;
+            return VERR_INVALID_RPL;
+        }
+        return VERR_NOT_CODE_SELECTOR;
+    }
+    return VERR_SELECTOR_NOT_PRESENT;
+}
+
+
+/**
+ * Converts a PGM paging mode to a set of DBGFPGDMP_XXX flags.
+ *
+ * @returns Flags. UINT32_MAX if the mode is invalid (asserted).
+ * @param   enmMode             The mode.
+ */
+static uint32_t dbgfR3PagingDumpModeToFlags(PGMMODE enmMode)
+{
+    switch (enmMode)
+    {
+        case PGMMODE_32_BIT:
+            return DBGFPGDMP_FLAGS_PSE;
+        case PGMMODE_PAE:
+            return DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE;
+        case PGMMODE_PAE_NX:
+            return DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_NXE;
+        case PGMMODE_AMD64:
+            return DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME;
+        case PGMMODE_AMD64_NX:
+            return DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME | DBGFPGDMP_FLAGS_NXE;
+        case PGMMODE_NESTED_32BIT:
+            return DBGFPGDMP_FLAGS_NP | DBGFPGDMP_FLAGS_PSE;
+        case PGMMODE_NESTED_PAE:
+            return DBGFPGDMP_FLAGS_NP | DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_NXE;
+        case PGMMODE_NESTED_AMD64:
+            return DBGFPGDMP_FLAGS_NP | DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME | DBGFPGDMP_FLAGS_NXE;
+        case PGMMODE_EPT:
+            return DBGFPGDMP_FLAGS_EPT;
+        case PGMMODE_NONE:
+            return 0;
+        default:
+            AssertFailedReturn(UINT32_MAX);
+    }
+}
+
+
+/**
+ * EMT worker for DBGFR3PagingDumpEx.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The shared VM handle.
+ * @param   idCpu           The current CPU ID.
+ * @param   fFlags          The flags, DBGFPGDMP_FLAGS_XXX.  Valid.
+ * @param   pcr3            The CR3 to use (unless we're getting the current
+ *                          state, see @a fFlags).
+ * @param   pu64FirstAddr   The first address.
+ * @param   pu64LastAddr    The last address.
+ * @param   cMaxDepth       The depth.
+ * @param   pHlp            The output callbacks.
+ */
+static DECLCALLBACK(int) dbgfR3PagingDumpEx(PUVM pUVM, VMCPUID idCpu, uint32_t fFlags, uint64_t *pcr3,
+                                            uint64_t *pu64FirstAddr, uint64_t *pu64LastAddr,
+                                            uint32_t cMaxDepth, PCDBGFINFOHLP pHlp)
+{
+    /*
+     * Implement dumping both context by means of recursion.
+     */
+    if ((fFlags & (DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_SHADOW)) == (DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_SHADOW))
+    {
+        int rc1 = dbgfR3PagingDumpEx(pUVM, idCpu, fFlags & ~DBGFPGDMP_FLAGS_GUEST,
+                                     pcr3, pu64FirstAddr, pu64LastAddr, cMaxDepth, pHlp);
+        int rc2 = dbgfR3PagingDumpEx(pUVM, idCpu, fFlags & ~DBGFPGDMP_FLAGS_SHADOW,
+                                     pcr3, pu64FirstAddr, pu64LastAddr, cMaxDepth, pHlp);
+        return RT_FAILURE(rc1) ? rc1 : rc2;
+    }
+
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Get the current CR3/mode if required.
+     */
+    uint64_t cr3 = *pcr3;
+    if (fFlags & (DBGFPGDMP_FLAGS_CURRENT_CR3 | DBGFPGDMP_FLAGS_CURRENT_MODE))
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        if (fFlags & DBGFPGDMP_FLAGS_SHADOW)
+        {
+            if (PGMGetShadowMode(pVCpu) == PGMMODE_NONE)
+            {
+                pHlp->pfnPrintf(pHlp, "Shadow paging mode is 'none' (NEM)\n");
+                return VINF_SUCCESS;
+            }
+
+            if (fFlags & DBGFPGDMP_FLAGS_CURRENT_CR3)
+                cr3 = PGMGetHyperCR3(pVCpu);
+            if (fFlags & DBGFPGDMP_FLAGS_CURRENT_MODE)
+                fFlags |= dbgfR3PagingDumpModeToFlags(PGMGetShadowMode(pVCpu));
+        }
+        else
+        {
+            if (fFlags & DBGFPGDMP_FLAGS_CURRENT_CR3)
+                cr3 = CPUMGetGuestCR3(pVCpu);
+            if (fFlags & DBGFPGDMP_FLAGS_CURRENT_MODE)
+            {
+                AssertCompile(DBGFPGDMP_FLAGS_PSE == X86_CR4_PSE);      AssertCompile(DBGFPGDMP_FLAGS_PAE == X86_CR4_PAE);
+                fFlags |= CPUMGetGuestCR4(pVCpu)  & (X86_CR4_PSE | X86_CR4_PAE);
+                AssertCompile(DBGFPGDMP_FLAGS_LME == MSR_K6_EFER_LME);  AssertCompile(DBGFPGDMP_FLAGS_NXE == MSR_K6_EFER_NXE);
+                fFlags |= CPUMGetGuestEFER(pVCpu) & (MSR_K6_EFER_LME | MSR_K6_EFER_NXE);
+            }
+        }
+    }
+    fFlags &= ~(DBGFPGDMP_FLAGS_CURRENT_MODE | DBGFPGDMP_FLAGS_CURRENT_CR3);
+
+    /*
+     * Call PGM to do the real work.
+     */
+    int rc;
+    if (fFlags & DBGFPGDMP_FLAGS_SHADOW)
+        rc = PGMR3DumpHierarchyShw(pVM, cr3, fFlags, *pu64FirstAddr, *pu64LastAddr, cMaxDepth, pHlp);
+    else
+        rc = PGMR3DumpHierarchyGst(pVM, cr3, fFlags, *pu64FirstAddr, *pu64LastAddr, cMaxDepth, pHlp);
+    return rc;
+}
+
+
+/**
+ * Dump paging structures.
+ *
+ * This API can be used to dump both guest and shadow structures.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   idCpu           The current CPU ID.
+ * @param   fFlags          The flags, DBGFPGDMP_FLAGS_XXX.
+ * @param   cr3             The CR3 to use (unless we're getting the current
+ *                          state, see @a fFlags).
+ * @param   u64FirstAddr    The address to start dumping at.
+ * @param   u64LastAddr     The address to end dumping after.
+ * @param   cMaxDepth       The depth.
+ * @param   pHlp            The output callbacks.  Defaults to the debug log if
+ *                          NULL.
+ */
+VMMDECL(int) DBGFR3PagingDumpEx(PUVM pUVM, VMCPUID idCpu, uint32_t fFlags, uint64_t cr3, uint64_t u64FirstAddr,
+                                uint64_t u64LastAddr, uint32_t cMaxDepth, PCDBGFINFOHLP pHlp)
+{
+    /*
+     * Input validation.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_CPU_ID);
+    AssertReturn(!(fFlags & ~DBGFPGDMP_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(fFlags & (DBGFPGDMP_FLAGS_SHADOW | DBGFPGDMP_FLAGS_GUEST), VERR_INVALID_PARAMETER);
+    AssertReturn((fFlags & DBGFPGDMP_FLAGS_CURRENT_MODE) || !(fFlags & DBGFPGDMP_FLAGS_MODE_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(   !(fFlags & DBGFPGDMP_FLAGS_EPT)
+                 || !(fFlags & (DBGFPGDMP_FLAGS_LME | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_NXE))
+                 , VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pHlp, VERR_INVALID_POINTER);
+    AssertReturn(cMaxDepth, VERR_INVALID_PARAMETER);
+
+    /*
+     * Forward the request to the target CPU.
+     */
+    return VMR3ReqPriorityCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3PagingDumpEx, 8,
+                                    pUVM, idCpu, fFlags, &cr3, &u64FirstAddr, &u64LastAddr, cMaxDepth, pHlp);
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFModule.cpp b/src/VBox/VMM/VMMR3/DBGFModule.cpp
new file mode 100644
index 00000000..0c33a720
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFModule.cpp
@@ -0,0 +1,290 @@
+/* $Id: DBGFModule.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Module & Segment Management.
+ */
+
+/*
+ * Copyright (C) 2008-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_dbgf_module    DBGFModule - Module & Segment Management
+ *
+ * A module is our representation of an executable binary. It's main purpose
+ * is to provide segments that can be mapped into address spaces and thereby
+ * provide debug info for those parts for the guest code or data.
+ *
+ * This module will not deal directly with debug info, it will only serve
+ * as an interface between the debugger / symbol lookup and the debug info
+ * readers.
+ *
+ * An executable binary doesn't need to have a file, or that is, we don't
+ * need the file to create a module for it. There will be interfaces for
+ * ROMs to register themselves so we can get to their symbols, and there
+ * will be interfaces for the guest OS plugins (@see pg_dbgf_os) to
+ * register kernel, drivers and other global modules.
+ */
+
+#if 0
+#include <VBox/vmm/dbgf.h>
+
+
+/** Special segment number that indicates that the offset is a relative
+ * virtual address (RVA). I.e. an offset from the start of the module. */
+#define DBGF_SEG_RVA UINT32_C(0xfffffff0)
+
+/** @defgroup grp_dbgf_dbginfo Debug Info Types
+ * @{ */
+/** Other format. */
+#define DBGF_DBGINFO_OTHER      RT_BIT_32(0)
+/** Stabs. */
+#define DBGF_DBGINFO_STABS      RT_BIT_32(1)
+/** Debug With Arbitrary Record Format (DWARF). */
+#define DBGF_DBGINFO_DWARF      RT_BIT_32(2)
+/** Microsoft Codeview debug info. */
+#define DBGF_DBGINFO_CODEVIEW   RT_BIT_32(3)
+/** Watcom debug info. */
+#define DBGF_DBGINFO_WATCOM     RT_BIT_32(4)
+/** IBM High Level Language debug info. */
+#define DBGF_DBGINFO_HLL        RT_BIT_32(5)
+/** Old OS/2 and Windows symbol file. */
+#define DBGF_DBGINFO_SYM        RT_BIT_32(6)
+/** Map file. */
+#define DBGF_DBGINFO_MAP        RT_BIT_32(7)
+/** @} */
+
+/** @defgroup grp_dbgf_exeimg Executable Image Types
+ * @{ */
+/** Some other format. */
+#define DBGF_EXEIMG_OTHER       RT_BIT_32(0)
+/** Portable Executable. */
+#define DBGF_EXEIMG_PE          RT_BIT_32(1)
+/** Linear eXecutable. */
+#define DBGF_EXEIMG_LX          RT_BIT_32(2)
+/** Linear Executable. */
+#define DBGF_EXEIMG_LE          RT_BIT_32(3)
+/** New Executable. */
+#define DBGF_EXEIMG_NE          RT_BIT_32(4)
+/** DOS Executable (Mark Zbikowski). */
+#define DBGF_EXEIMG_MZ          RT_BIT_32(5)
+/** COM Executable. */
+#define DBGF_EXEIMG_COM         RT_BIT_32(6)
+/** a.out Executable. */
+#define DBGF_EXEIMG_AOUT        RT_BIT_32(7)
+/** Executable and Linkable Format. */
+#define DBGF_EXEIMG_ELF         RT_BIT_32(8)
+/** Mach-O Executable (including FAT ones). */
+#define DBGF_EXEIMG_MACHO       RT_BIT_32(9)
+/** @} */
+
+/** Pointer to a module. */
+typedef struct DBGFMOD *PDBGFMOD;
+
+
+/**
+ * Virtual method table for executable image interpreters.
+ */
+typedef struct DBGFMODVTIMG
+{
+    /** Magic number (DBGFMODVTIMG_MAGIC). */
+    uint32_t    u32Magic;
+    /** Mask of supported debug info types, see grp_dbgf_exeimg.
+     * Used to speed up the search for a suitable interpreter. */
+    uint32_t    fSupports;
+    /** The name of the interpreter. */
+    const char *pszName;
+
+    /**
+     * Try open the image.
+     *
+     * This combines probing and opening.
+     *
+     * @returns VBox status code. No informational returns defined.
+     *
+     * @param   pMod        Pointer to the module that is being opened.
+     *
+     *                      The DBGFMOD::pszDbgFile member will point to
+     *                      the filename of any debug info we're aware of
+     *                      on input. Also, or alternatively, it is expected
+     *                      that the interpreter will look for debug info in
+     *                      the executable image file when present and that it
+     *                      may ask the image interpreter for this when it's
+     *                      around.
+     *
+     *                      Upon successful return the method is expected to
+     *                      initialize pDbgOps and pvDbgPriv.
+     */
+    DECLCALLBACKMEMBER(int, pfnTryOpen)(PDBGFMOD pMod);
+
+    /**
+     * Close the interpreter, freeing all associated resources.
+     *
+     * The caller sets the pDbgOps and pvDbgPriv DBGFMOD members
+     * to NULL upon return.
+     *
+     * @param   pMod        Pointer to the module structure.
+     */
+    DECLCALLBACKMEMBER(int, pfnClose)(PDBGFMOD pMod);
+
+} DBGFMODVTIMG
+
+/**
+ * Virtual method table for debug info interpreters.
+ */
+typedef struct DBGFMODVTDBG
+{
+    /** Magic number (DBGFMODVTDBG_MAGIC). */
+    uint32_t    u32Magic;
+    /** Mask of supported debug info types, see grp_dbgf_dbginfo.
+     * Used to speed up the search for a suitable interpreter. */
+    uint32_t    fSupports;
+    /** The name of the interpreter. */
+    const char *pszName;
+
+    /**
+     * Try open the image.
+     *
+     * This combines probing and opening.
+     *
+     * @returns VBox status code. No informational returns defined.
+     *
+     * @param   pMod        Pointer to the module that is being opened.
+     *
+     *                      The DBGFMOD::pszDbgFile member will point to
+     *                      the filename of any debug info we're aware of
+     *                      on input. Also, or alternatively, it is expected
+     *                      that the interpreter will look for debug info in
+     *                      the executable image file when present and that it
+     *                      may ask the image interpreter for this when it's
+     *                      around.
+     *
+     *                      Upon successful return the method is expected to
+     *                      initialize pDbgOps and pvDbgPriv.
+     */
+    DECLCALLBACKMEMBER(int, pfnTryOpen)(PDBGFMOD pMod);
+
+    /**
+     * Close the interpreter, freeing all associated resources.
+     *
+     * The caller sets the pDbgOps and pvDbgPriv DBGFMOD members
+     * to NULL upon return.
+     *
+     * @param   pMod        Pointer to the module structure.
+     */
+    DECLCALLBACKMEMBER(int, pfnClose)(PDBGFMOD pMod);
+
+    /**
+     * Queries symbol information by symbol name.
+     *
+     * @returns VBox status code.
+     * @retval  VINF_SUCCESS on success, no informational status code.
+     * @retval  VERR_DBGF_NO_SYMBOLS if there aren't any symbols.
+     * @retval  VERR_SYMBOL_NOT_FOUND if no suitable symbol was found.
+     *
+     * @param   pMod        Pointer to the module structure.
+     * @param   pszSymbol   The symbol name.
+     * @para    pSymbol     Where to store the symbol information.
+     */
+    DECLCALLBACKMEMBER(int, pfnSymbolByName)(PDBGFMOD pMod, const char *pszSymbol, PDBGFSYMBOL pSymbol);
+
+    /**
+     * Queries symbol information by address.
+     *
+     * The returned symbol is what the debug info interpreter considers the symbol
+     * most applicable to the specified address. This usually means a symbol with an
+     * address equal or lower than the requested.
+     *
+     * @returns VBox status code.
+     * @retval  VINF_SUCCESS on success, no informational status code.
+     * @retval  VERR_DBGF_NO_SYMBOLS if there aren't any symbols.
+     * @retval  VERR_SYMBOL_NOT_FOUND if no suitable symbol was found.
+     *
+     * @param   pMod        Pointer to the module structure.
+     * @param   iSeg        The segment number (0-based). DBGF_SEG_RVA can be used.
+     * @param   off         The offset into the segment.
+     * @param   poffDisp    Where to store the distance between the specified address
+     *                      and the returned symbol. Optional.
+     * @param   pSymbol     Where to store the symbol information.
+     */
+    DECLCALLBACKMEMBER(int, pfnSymbolByAddr)(PDBGFMOD pMod, uint32_t iSeg, RTGCUINTPTR off, PRTGCINTPTR poffDisp, PDBGFSYMBOL pSymbol);
+
+    /**
+     * Queries line number information by address.
+     *
+     * @returns VBox status code.
+     * @retval  VINF_SUCCESS on success, no informational status code.
+     * @retval  VERR_DBGF_NO_LINE_NUMBERS if there aren't any line numbers.
+     * @retval  VERR_DBGF_LINE_NOT_FOUND if no suitable line number was found.
+     *
+     * @param   pMod        Pointer to the module structure.
+     * @param   iSeg        The segment number (0-based). DBGF_SEG_RVA can be used.
+     * @param   off         The offset into the segment.
+     * @param   poffDisp    Where to store the distance between the specified address
+     *                      and the returned line number. Optional.
+     * @param   pLine       Where to store the information about the closest line number.
+     */
+    DECLCALLBACKMEMBER(int, pfnLineByAddr)(PDBGFMOD pMod, uint32_t iSeg, RTGCUINTPTR off, PRTGCINTPTR poffDisp, PDBGFLINE pLine);
+
+    /**
+     * Adds a symbol to the module (optional).
+     *
+     * This method is used to implement DBGFR3SymbolAdd.
+     *
+     * @returns VBox status code.
+     * @retval  VERR_NOT_SUPPORTED if the interpreter doesn't support this feature.
+     *
+     * @param   pMod        Pointer to the module structure.
+     * @param   pszSymbol   The symbol name.
+     * @param   iSeg        The segment number (0-based). DBGF_SEG_RVA can be used.
+     * @param   off         The offset into the segment.
+     * @param   cbSymbol    The area covered by the symbol. 0 is fine.
+     */
+    DECLCALLBACKMEMBER(int, pfnSymbolAdd)(PDBGFMOD pMod, const char *pszSymbol, uint32_t iSeg, RTGCUINTPTR off, RTUINT cbSymbol);
+
+    /** For catching initialization errors (DBGFMODVTDBG_MAGIC). */
+    uint32_t    u32EndMagic;
+} DBGFMODVTDBG;
+
+#define DBGFMODVTDBG_MAGIC 123
+
+/**
+ * Module.
+ */
+typedef struct DBGFMOD
+{
+    /** Magic value (DBGFMOD_MAGIC). */
+    uint32_t        u32Magic;
+    /** The number of address spaces this module is currently linked into.
+     * This is used to perform automatic cleanup and sharing. */
+    uint32_t        cLinks;
+    /** The module name (short). */
+    const char     *pszName;
+    /** The module filename. Can be NULL. */
+    const char     *pszImgFile;
+    /** The debug info file (if external). Can be NULL. */
+    const char     *pszDbgFile;
+
+    /** The method table for the executable image interpreter. */
+    PCDBGFMODVTIMG  pImgVt;
+    /** Pointer to the private data of the executable image interpreter. */
+    void           *pvImgPriv;
+
+    /** The method table for the debug info interpreter. */
+    PCDBGFMODVTDBG  pDbgVt;
+    /** Pointer to the private data of the debug info interpreter. */
+    void           *pvDbgPriv;
+
+} DBGFMOD;
+
+#define DBGFMOD_MAGIC   0x12345678
+
+#endif
+
diff --git a/src/VBox/VMM/VMMR3/DBGFOS.cpp b/src/VBox/VMM/VMMR3/DBGFOS.cpp
new file mode 100644
index 00000000..82a297ae
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFOS.cpp
@@ -0,0 +1,661 @@
+/* $Id: DBGFOS.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Guest OS Diggers.
+ */
+
+/*
+ * Copyright (C) 2008-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/mm.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/param.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+
+#define DBGF_OS_READ_LOCK(pUVM) \
+    do { int rcLock = RTCritSectRwEnterShared(&pUVM->dbgf.s.CritSect); AssertRC(rcLock); } while (0)
+#define DBGF_OS_READ_UNLOCK(pUVM) \
+    do { int rcLock = RTCritSectRwLeaveShared(&pUVM->dbgf.s.CritSect); AssertRC(rcLock); } while (0)
+
+#define DBGF_OS_WRITE_LOCK(pUVM) \
+    do { int rcLock = RTCritSectRwEnterExcl(&pUVM->dbgf.s.CritSect); AssertRC(rcLock); } while (0)
+#define DBGF_OS_WRITE_UNLOCK(pUVM) \
+    do { int rcLock = RTCritSectRwLeaveExcl(&pUVM->dbgf.s.CritSect); AssertRC(rcLock); } while (0)
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * EMT interface wrappers.
+ *
+ * The diggers expects to be called on an EMT.  To avoid the debugger+Main having
+ *
+ * Since the user (debugger/Main) shouldn't be calling directly into the digger code, but rather
+ */
+typedef struct DBGFOSEMTWRAPPER
+{
+    /** Pointer to the next list entry. */
+    struct DBGFOSEMTWRAPPER    *pNext;
+    /** The interface type.   */
+    DBGFOSINTERFACE             enmIf;
+    /** The digger interface pointer. */
+    union
+    {
+        /** Generic void pointer. */
+        void                    *pv;
+        /** DBGFOSINTERFACE_DMESG.*/
+        PDBGFOSIDMESG           pDmesg;
+    } uDigger;
+    /** The user mode VM handle. */
+    PUVM                        pUVM;
+    /** The wrapper interface union (consult enmIf). */
+    union
+    {
+        /** DBGFOSINTERFACE_DMESG.*/
+        DBGFOSIDMESG            Dmesg;
+    } uWrapper;
+} DBGFOSEMTWRAPPER;
+/** Pointer to an EMT interface wrapper.   */
+typedef DBGFOSEMTWRAPPER *PDBGFOSEMTWRAPPER;
+
+
+/**
+ * Internal init routine called by DBGFR3Init().
+ *
+ * @returns VBox status code.
+ * @param   pUVM    The user mode VM handle.
+ */
+int dbgfR3OSInit(PUVM pUVM)
+{
+    RT_NOREF_PV(pUVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal cleanup routine called by DBGFR3Term(), part 1.
+ *
+ * @param   pUVM    The user mode VM handle.
+ */
+void dbgfR3OSTermPart1(PUVM pUVM)
+{
+    DBGF_OS_WRITE_LOCK(pUVM);
+
+    /*
+     * Terminate the current one.
+     */
+    if (pUVM->dbgf.s.pCurOS)
+    {
+        pUVM->dbgf.s.pCurOS->pReg->pfnTerm(pUVM, pUVM->dbgf.s.pCurOS->abData);
+        pUVM->dbgf.s.pCurOS = NULL;
+    }
+
+    DBGF_OS_WRITE_UNLOCK(pUVM);
+}
+
+
+/**
+ * Internal cleanup routine called by DBGFR3Term(), part 2.
+ *
+ * @param   pUVM    The user mode VM handle.
+ */
+void dbgfR3OSTermPart2(PUVM pUVM)
+{
+    DBGF_OS_WRITE_LOCK(pUVM);
+
+    /* This shouldn't happen. */
+    AssertStmt(!pUVM->dbgf.s.pCurOS, dbgfR3OSTermPart1(pUVM));
+
+    /*
+     * Destroy all the instances.
+     */
+    while (pUVM->dbgf.s.pOSHead)
+    {
+        PDBGFOS pOS = pUVM->dbgf.s.pOSHead;
+        pUVM->dbgf.s.pOSHead = pOS->pNext;
+        if (pOS->pReg->pfnDestruct)
+            pOS->pReg->pfnDestruct(pUVM, pOS->abData);
+
+        PDBGFOSEMTWRAPPER pFree = pOS->pWrapperHead;
+        while ((pFree = pOS->pWrapperHead) != NULL)
+        {
+            pOS->pWrapperHead = pFree->pNext;
+            pFree->pNext = NULL;
+            MMR3HeapFree(pFree);
+        }
+
+        MMR3HeapFree(pOS);
+    }
+
+    DBGF_OS_WRITE_UNLOCK(pUVM);
+}
+
+
+/**
+ * EMT worker function for DBGFR3OSRegister.
+ *
+ * @returns VBox status code.
+ * @param   pUVM    The user mode VM handle.
+ * @param   pReg    The registration structure.
+ */
+static DECLCALLBACK(int) dbgfR3OSRegister(PUVM pUVM, PDBGFOSREG pReg)
+{
+    /* more validations. */
+    DBGF_OS_READ_LOCK(pUVM);
+    PDBGFOS pOS;
+    for (pOS = pUVM->dbgf.s.pOSHead; pOS; pOS = pOS->pNext)
+        if (!strcmp(pOS->pReg->szName, pReg->szName))
+        {
+            DBGF_OS_READ_UNLOCK(pUVM);
+            Log(("dbgfR3OSRegister: %s -> VERR_ALREADY_LOADED\n", pReg->szName));
+            return VERR_ALREADY_LOADED;
+        }
+    DBGF_OS_READ_UNLOCK(pUVM);
+
+    /*
+     * Allocate a new structure, call the constructor and link it into the list.
+     */
+    pOS = (PDBGFOS)MMR3HeapAllocZU(pUVM, MM_TAG_DBGF_OS, RT_UOFFSETOF_DYN(DBGFOS, abData[pReg->cbData]));
+    AssertReturn(pOS, VERR_NO_MEMORY);
+    pOS->pReg = pReg;
+
+    int rc = pOS->pReg->pfnConstruct(pUVM, pOS->abData);
+    if (RT_SUCCESS(rc))
+    {
+        DBGF_OS_WRITE_LOCK(pUVM);
+        pOS->pNext = pUVM->dbgf.s.pOSHead;
+        pUVM->dbgf.s.pOSHead = pOS;
+        DBGF_OS_WRITE_UNLOCK(pUVM);
+    }
+    else
+    {
+        if (pOS->pReg->pfnDestruct)
+            pOS->pReg->pfnDestruct(pUVM, pOS->abData);
+        MMR3HeapFree(pOS);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Registers a guest OS digger.
+ *
+ * This will instantiate an instance of the digger and add it
+ * to the list for us in the next call to DBGFR3OSDetect().
+ *
+ * @returns VBox status code.
+ * @param   pUVM    The user mode VM handle.
+ * @param   pReg    The registration structure.
+ * @thread  Any.
+ */
+VMMR3DECL(int) DBGFR3OSRegister(PUVM pUVM, PCDBGFOSREG pReg)
+{
+    /*
+     * Validate intput.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    AssertPtrReturn(pReg, VERR_INVALID_POINTER);
+    AssertReturn(pReg->u32Magic == DBGFOSREG_MAGIC, VERR_INVALID_MAGIC);
+    AssertReturn(pReg->u32EndMagic == DBGFOSREG_MAGIC, VERR_INVALID_MAGIC);
+    AssertReturn(!pReg->fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(pReg->cbData < _2G, VERR_INVALID_PARAMETER);
+    AssertReturn(pReg->szName[0], VERR_INVALID_NAME);
+    AssertReturn(RTStrEnd(&pReg->szName[0], sizeof(pReg->szName)), VERR_INVALID_NAME);
+    AssertPtrReturn(pReg->pfnConstruct, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pReg->pfnDestruct, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReg->pfnProbe, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReg->pfnInit, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReg->pfnRefresh, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReg->pfnTerm, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReg->pfnQueryVersion, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReg->pfnQueryInterface, VERR_INVALID_POINTER);
+
+    /*
+     * Pass it on to EMT(0).
+     */
+    return VMR3ReqPriorityCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3OSRegister, 2, pUVM, pReg);
+}
+
+
+/**
+ * EMT worker function for DBGFR3OSDeregister.
+ *
+ * @returns VBox status code.
+ * @param   pUVM    The user mode VM handle.
+ * @param   pReg    The registration structure.
+ */
+static DECLCALLBACK(int) dbgfR3OSDeregister(PUVM pUVM, PDBGFOSREG pReg)
+{
+    /*
+     * Unlink it.
+     */
+    bool    fWasCurOS = false;
+    PDBGFOS pOSPrev   = NULL;
+    PDBGFOS pOS;
+    DBGF_OS_WRITE_LOCK(pUVM);
+    for (pOS = pUVM->dbgf.s.pOSHead; pOS; pOSPrev = pOS, pOS = pOS->pNext)
+        if (pOS->pReg == pReg)
+        {
+            if (pOSPrev)
+                pOSPrev->pNext = pOS->pNext;
+            else
+                pUVM->dbgf.s.pOSHead = pOS->pNext;
+            if (pUVM->dbgf.s.pCurOS == pOS)
+            {
+                pUVM->dbgf.s.pCurOS = NULL;
+                fWasCurOS = true;
+            }
+            break;
+        }
+    DBGF_OS_WRITE_UNLOCK(pUVM);
+    if (!pOS)
+    {
+        Log(("DBGFR3OSDeregister: %s -> VERR_NOT_FOUND\n", pReg->szName));
+        return VERR_NOT_FOUND;
+    }
+
+    /*
+     * Terminate it if it was the current OS, then invoke the
+     * destructor and clean up.
+     */
+    if (fWasCurOS)
+        pOS->pReg->pfnTerm(pUVM, pOS->abData);
+    if (pOS->pReg->pfnDestruct)
+        pOS->pReg->pfnDestruct(pUVM, pOS->abData);
+
+    PDBGFOSEMTWRAPPER pFree = pOS->pWrapperHead;
+    while ((pFree = pOS->pWrapperHead) != NULL)
+    {
+        pOS->pWrapperHead = pFree->pNext;
+        pFree->pNext = NULL;
+        MMR3HeapFree(pFree);
+    }
+
+    MMR3HeapFree(pOS);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters a guest OS digger previously registered by DBGFR3OSRegister.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM    The user mode VM handle.
+ * @param   pReg    The registration structure.
+ * @thread  Any.
+ */
+VMMR3DECL(int)  DBGFR3OSDeregister(PUVM pUVM, PCDBGFOSREG pReg)
+{
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pReg, VERR_INVALID_POINTER);
+    AssertReturn(pReg->u32Magic == DBGFOSREG_MAGIC, VERR_INVALID_MAGIC);
+    AssertReturn(pReg->u32EndMagic == DBGFOSREG_MAGIC, VERR_INVALID_MAGIC);
+    AssertReturn(RTStrEnd(&pReg->szName[0], sizeof(pReg->szName)), VERR_INVALID_NAME);
+
+    DBGF_OS_READ_LOCK(pUVM);
+    PDBGFOS pOS;
+    for (pOS = pUVM->dbgf.s.pOSHead; pOS; pOS = pOS->pNext)
+        if (pOS->pReg == pReg)
+            break;
+    DBGF_OS_READ_UNLOCK(pUVM);
+
+    if (!pOS)
+    {
+        Log(("DBGFR3OSDeregister: %s -> VERR_NOT_FOUND\n", pReg->szName));
+        return VERR_NOT_FOUND;
+    }
+
+    /*
+     * Pass it on to EMT(0).
+     */
+    return VMR3ReqPriorityCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3OSDeregister, 2, pUVM, pReg);
+}
+
+
+/**
+ * EMT worker function for DBGFR3OSDetect.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully detected.
+ * @retval  VINF_DBGF_OS_NOT_DETCTED if we cannot figure it out.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   pszName     Where to store the OS name. Empty string if not detected.
+ * @param   cchName     Size of the buffer.
+ */
+static DECLCALLBACK(int) dbgfR3OSDetect(PUVM pUVM, char *pszName, size_t cchName)
+{
+    /*
+     * Cycle thru the detection routines.
+     */
+    DBGF_OS_WRITE_LOCK(pUVM);
+
+    PDBGFOS const pOldOS = pUVM->dbgf.s.pCurOS;
+    pUVM->dbgf.s.pCurOS = NULL;
+
+    for (PDBGFOS pNewOS = pUVM->dbgf.s.pOSHead; pNewOS; pNewOS = pNewOS->pNext)
+        if (pNewOS->pReg->pfnProbe(pUVM, pNewOS->abData))
+        {
+            int rc;
+            pUVM->dbgf.s.pCurOS = pNewOS;
+            if (pOldOS == pNewOS)
+                rc = pNewOS->pReg->pfnRefresh(pUVM, pNewOS->abData);
+            else
+            {
+                if (pOldOS)
+                    pOldOS->pReg->pfnTerm(pUVM, pNewOS->abData);
+                rc = pNewOS->pReg->pfnInit(pUVM, pNewOS->abData);
+            }
+            if (pszName && cchName)
+                strncat(pszName, pNewOS->pReg->szName, cchName);
+
+            DBGF_OS_WRITE_UNLOCK(pUVM);
+            return rc;
+        }
+
+    /* not found */
+    if (pOldOS)
+        pOldOS->pReg->pfnTerm(pUVM, pOldOS->abData);
+
+    DBGF_OS_WRITE_UNLOCK(pUVM);
+    return VINF_DBGF_OS_NOT_DETCTED;
+}
+
+
+/**
+ * Detects the guest OS and try dig out symbols and useful stuff.
+ *
+ * When called the 2nd time, symbols will be updated that if the OS
+ * is the same.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully detected.
+ * @retval  VINF_DBGF_OS_NOT_DETCTED if we cannot figure it out.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   pszName     Where to store the OS name. Empty string if not detected.
+ * @param   cchName     Size of the buffer.
+ * @thread  Any.
+ */
+VMMR3DECL(int) DBGFR3OSDetect(PUVM pUVM, char *pszName, size_t cchName)
+{
+    AssertPtrNullReturn(pszName, VERR_INVALID_POINTER);
+    if (pszName && cchName)
+        *pszName = '\0';
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Pass it on to EMT(0).
+     */
+    return VMR3ReqPriorityCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3OSDetect, 3, pUVM, pszName, cchName);
+}
+
+
+/**
+ * EMT worker function for DBGFR3OSQueryNameAndVersion
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszName         Where to store the OS name. Optional.
+ * @param   cchName         The size of the name buffer.
+ * @param   pszVersion      Where to store the version string. Optional.
+ * @param   cchVersion      The size of the version buffer.
+ */
+static DECLCALLBACK(int) dbgfR3OSQueryNameAndVersion(PUVM pUVM, char *pszName, size_t cchName, char *pszVersion, size_t cchVersion)
+{
+    /*
+     * Any known OS?
+     */
+    DBGF_OS_READ_LOCK(pUVM);
+
+    if (pUVM->dbgf.s.pCurOS)
+    {
+        int rc = VINF_SUCCESS;
+        if (pszName && cchName)
+        {
+            size_t cch = strlen(pUVM->dbgf.s.pCurOS->pReg->szName);
+            if (cchName > cch)
+                memcpy(pszName, pUVM->dbgf.s.pCurOS->pReg->szName, cch + 1);
+            else
+            {
+                memcpy(pszName, pUVM->dbgf.s.pCurOS->pReg->szName, cchName - 1);
+                pszName[cchName - 1] = '\0';
+                rc = VINF_BUFFER_OVERFLOW;
+            }
+        }
+
+        if (pszVersion && cchVersion)
+        {
+            int rc2 = pUVM->dbgf.s.pCurOS->pReg->pfnQueryVersion(pUVM, pUVM->dbgf.s.pCurOS->abData, pszVersion, cchVersion);
+            if (RT_FAILURE(rc2) || rc == VINF_SUCCESS)
+                rc = rc2;
+        }
+
+        DBGF_OS_READ_UNLOCK(pUVM);
+        return rc;
+    }
+
+    DBGF_OS_READ_UNLOCK(pUVM);
+    return VERR_DBGF_OS_NOT_DETCTED;
+}
+
+
+/**
+ * Queries the name and/or version string for the guest OS.
+ *
+ * It goes without saying that this querying is done using the current
+ * guest OS digger and not additions or user configuration.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszName         Where to store the OS name. Optional.
+ * @param   cchName         The size of the name buffer.
+ * @param   pszVersion      Where to store the version string. Optional.
+ * @param   cchVersion      The size of the version buffer.
+ * @thread  Any.
+ */
+VMMR3DECL(int) DBGFR3OSQueryNameAndVersion(PUVM pUVM, char *pszName, size_t cchName, char *pszVersion, size_t cchVersion)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrNullReturn(pszName, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pszVersion, VERR_INVALID_POINTER);
+
+    /*
+     * Initialize the output up front.
+     */
+    if (pszName && cchName)
+        *pszName = '\0';
+    if (pszVersion && cchVersion)
+        *pszVersion = '\0';
+
+    /*
+     * Pass it on to EMT(0).
+     */
+    return VMR3ReqPriorityCallWaitU(pUVM, 0 /*idDstCpu*/,
+                                   (PFNRT)dbgfR3OSQueryNameAndVersion, 5, pUVM, pszName, cchName, pszVersion, cchVersion);
+}
+
+
+/**
+ * @interface_method_impl{DBGFOSIDMESG,pfnQueryKernelLog, Generic EMT wrapper.}
+ */
+static DECLCALLBACK(int) dbgfR3OSEmtIDmesg_QueryKernelLog(PDBGFOSIDMESG pThis, PUVM pUVM, uint32_t fFlags, uint32_t cMessages,
+                                                          char *pszBuf, size_t cbBuf, size_t *pcbActual)
+{
+    PDBGFOSEMTWRAPPER pWrapper = RT_FROM_MEMBER(pThis, DBGFOSEMTWRAPPER, uWrapper.Dmesg);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(pUVM == pWrapper->pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(!fFlags, VERR_INVALID_FLAGS);
+    AssertReturn(cMessages > 0, VERR_INVALID_PARAMETER);
+    if (cbBuf)
+        AssertPtrReturn(pszBuf, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pcbActual, VERR_INVALID_POINTER);
+
+    return VMR3ReqPriorityCallWaitU(pWrapper->pUVM, 0 /*idDstCpu*/,
+                                   (PFNRT)pWrapper->uDigger.pDmesg->pfnQueryKernelLog, 7,
+                                    pWrapper->uDigger.pDmesg, pUVM, fFlags, cMessages, pszBuf, cbBuf, pcbActual);
+
+}
+
+
+/**
+ * EMT worker for DBGFR3OSQueryInterface.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   enmIf           The interface identifier.
+ * @param   ppvIf           Where to store the interface pointer on success.
+ */
+static DECLCALLBACK(void) dbgfR3OSQueryInterface(PUVM pUVM, DBGFOSINTERFACE enmIf, void **ppvIf)
+{
+    AssertPtrReturnVoid(ppvIf);
+    *ppvIf = NULL;
+    AssertReturnVoid(enmIf > DBGFOSINTERFACE_INVALID && enmIf < DBGFOSINTERFACE_END);
+    UVM_ASSERT_VALID_EXT_RETURN_VOID(pUVM);
+
+    /*
+     * Forward the query to the current OS.
+     */
+    DBGF_OS_READ_LOCK(pUVM);
+    PDBGFOS pOS = pUVM->dbgf.s.pCurOS;
+    if (pOS)
+    {
+        void *pvDiggerIf;
+        pvDiggerIf = pOS->pReg->pfnQueryInterface(pUVM, pUVM->dbgf.s.pCurOS->abData, enmIf);
+        if (pvDiggerIf)
+        {
+            /*
+             * Do we have an EMT wrapper for this interface already?
+             *
+             * We ASSUME the interfaces are static and not dynamically allocated
+             * for each QueryInterface call.
+             */
+            PDBGFOSEMTWRAPPER pWrapper = pOS->pWrapperHead;
+            while (   pWrapper != NULL
+                   && (   pWrapper->uDigger.pv != pvDiggerIf
+                       && pWrapper->enmIf      != enmIf) )
+                pWrapper = pWrapper->pNext;
+            if (pWrapper)
+            {
+                *ppvIf = &pWrapper->uWrapper;
+                DBGF_OS_READ_UNLOCK(pUVM);
+                return;
+            }
+            DBGF_OS_READ_UNLOCK(pUVM);
+
+            /*
+             * Create a wrapper.
+             */
+            int rc = MMR3HeapAllocExU(pUVM, MM_TAG_DBGF_OS, sizeof(*pWrapper), (void **)&pWrapper);
+            if (RT_FAILURE(rc))
+                return;
+            pWrapper->uDigger.pv = pvDiggerIf;
+            pWrapper->pUVM       = pUVM;
+            pWrapper->enmIf      = enmIf;
+            switch (enmIf)
+            {
+                case DBGFOSINTERFACE_DMESG:
+                    pWrapper->uWrapper.Dmesg.u32Magic          = DBGFOSIDMESG_MAGIC;
+                    pWrapper->uWrapper.Dmesg.pfnQueryKernelLog = dbgfR3OSEmtIDmesg_QueryKernelLog;
+                    pWrapper->uWrapper.Dmesg.u32EndMagic       = DBGFOSIDMESG_MAGIC;
+                    break;
+                default:
+                    AssertFailed();
+                    MMR3HeapFree(pWrapper);
+                    return;
+            }
+
+            DBGF_OS_WRITE_LOCK(pUVM);
+            if (pUVM->dbgf.s.pCurOS == pOS)
+            {
+                pWrapper->pNext = pOS->pWrapperHead;
+                pOS->pWrapperHead = pWrapper;
+                *ppvIf = &pWrapper->uWrapper;
+                DBGF_OS_WRITE_UNLOCK(pUVM);
+            }
+            else
+            {
+                DBGF_OS_WRITE_UNLOCK(pUVM);
+                MMR3HeapFree(pWrapper);
+            }
+            return;
+        }
+    }
+    DBGF_OS_READ_UNLOCK(pUVM);
+}
+
+
+/**
+ * Query an optional digger interface.
+ *
+ * @returns Pointer to the digger interface on success, NULL if the interfaces isn't
+ *          available or no active guest OS digger.
+ * @param   pUVM            The user mode VM handle.
+ * @param   enmIf           The interface identifier.
+ * @thread  Any.
+ */
+VMMR3DECL(void *) DBGFR3OSQueryInterface(PUVM pUVM, DBGFOSINTERFACE enmIf)
+{
+    AssertMsgReturn(enmIf > DBGFOSINTERFACE_INVALID && enmIf < DBGFOSINTERFACE_END, ("%d\n", enmIf), NULL);
+
+    /*
+     * Pass it on to an EMT.
+     */
+    void *pvIf = NULL;
+    VMR3ReqPriorityCallVoidWaitU(pUVM, VMCPUID_ANY, (PFNRT)dbgfR3OSQueryInterface, 3, pUVM, enmIf, &pvIf);
+    return pvIf;
+}
+
+
+
+/**
+ * Internal wrapper for calling DBGFOSREG::pfnStackUnwindAssist.
+ */
+int dbgfR3OSStackUnwindAssist(PUVM pUVM, VMCPUID idCpu, PDBGFSTACKFRAME pFrame, PRTDBGUNWINDSTATE pState,
+                              PCCPUMCTX pInitialCtx, RTDBGAS hAs, uint64_t *puScratch)
+{
+    int rc = VINF_SUCCESS;
+    if (pUVM->dbgf.s.pCurOS)
+    {
+        ASMCompilerBarrier();
+        DBGF_OS_READ_LOCK(pUVM);
+        PDBGFOS pOS = pUVM->dbgf.s.pCurOS;
+        if (pOS)
+            rc = pOS->pReg->pfnStackUnwindAssist(pUVM, pUVM->dbgf.s.pCurOS->abData, idCpu, pFrame,
+                                                 pState, pInitialCtx, hAs, puScratch);
+        DBGF_OS_READ_UNLOCK(pUVM);
+    }
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFR3BugCheck.cpp b/src/VBox/VMM/VMMR3/DBGFR3BugCheck.cpp
new file mode 100644
index 00000000..54b5a823
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFR3BugCheck.cpp
@@ -0,0 +1,920 @@
+/* $Id: DBGFR3BugCheck.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, NT Bug Checks.
+ */
+
+/*
+ * Copyright (C) 2018-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/tm.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static FNDBGFHANDLERINT dbgfR3BugCheckInfo;
+
+
+/**
+ * Initializes the bug check state and registers the info callback.
+ *
+ * No termination function needed.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ */
+int dbgfR3BugCheckInit(PVM pVM)
+{
+    pVM->dbgf.s.BugCheck.idCpu    = NIL_VMCPUID;
+    pVM->dbgf.s.BugCheck.enmEvent = DBGFEVENT_END;
+
+    return DBGFR3InfoRegisterInternal(pVM, "bugcheck",
+                                      "Show bugcheck info.  Can specify bug check code and parameters to lookup info.",
+                                      dbgfR3BugCheckInfo);
+}
+
+
+/**
+ * Names a few common NT status codes for DBGFR3FormatBugCheck.
+ */
+static const char *dbgfR3GetNtStatusName(uint32_t uNtStatus)
+{
+    switch (uNtStatus)
+    {
+        case 0x80000001: return " - STATUS_GUARD_PAGE_VIOLATION";
+        case 0x80000002: return " - STATUS_DATATYPE_MISALIGNMENT";
+        case 0x80000003: return " - STATUS_BREAKPOINT";
+        case 0x80000004: return " - STATUS_SINGLE_STEP";
+        case 0xc0000008: return " - STATUS_INVALID_HANDLE";
+        case 0xc0000005: return " - STATUS_ACCESS_VIOLATION";
+        case 0xc0000027: return " - STATUS_UNWIND";
+        case 0xc0000028: return " - STATUS_BAD_STACK";
+        case 0xc0000029: return " - STATUS_INVALID_UNWIND_TARGET";
+        default:         return "";
+    }
+}
+
+
+/**
+ * Formats a symbol for DBGFR3FormatBugCheck.
+ */
+static const char *dbgfR3FormatSymbol(PUVM pUVM, char *pszSymbol, size_t cchSymbol, const char *pszPrefix, uint64_t uFlatAddr)
+{
+    DBGFADDRESS  Addr;
+    RTGCINTPTR   offDisp = 0;
+    PRTDBGSYMBOL pSym = DBGFR3AsSymbolByAddrA(pUVM, DBGF_AS_GLOBAL, DBGFR3AddrFromFlat(pUVM, &Addr, uFlatAddr),
+                                              RTDBGSYMADDR_FLAGS_LESS_OR_EQUAL | RTDBGSYMADDR_FLAGS_SKIP_ABS_IN_DEFERRED,
+                                              &offDisp, NULL /*phMod*/);
+    if (pSym)
+    {
+        if (!offDisp)
+            RTStrPrintf(pszSymbol, cchSymbol, "%s%s", pszPrefix, pSym->szName);
+        else if (offDisp > 0)
+            RTStrPrintf(pszSymbol, cchSymbol, "%s%s + %#RX64", pszPrefix, pSym->szName, (uint64_t)offDisp);
+        else
+            RTStrPrintf(pszSymbol, cchSymbol, "%s%s - %#RX64", pszPrefix, pSym->szName, (uint64_t)-offDisp);
+        RTDbgSymbolFree(pSym);
+    }
+    else
+        *pszSymbol = '\0';
+    return pszSymbol;
+}
+
+
+/**
+ * Formats a windows bug check (BSOD).
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_BUFFER_OVERFLOW if there is more data than the buffer can handle.
+ *
+ * @param   pUVM                The usermode VM handle.
+ * @param   pszDetails          The output buffer.
+ * @param   cbDetails           The size of the output buffer.
+ * @param   uBugCheck           The bugheck code.
+ * @param   uP1                 Bug check parameter 1.
+ * @param   uP2                 Bug check parameter 2.
+ * @param   uP3                 Bug check parameter 3.
+ * @param   uP4                 Bug check parameter 4.
+ */
+VMMR3DECL(int) DBGFR3FormatBugCheck(PUVM pUVM, char *pszDetails, size_t cbDetails,
+                                    uint64_t uBugCheck, uint64_t uP1, uint64_t uP2, uint64_t uP3, uint64_t uP4)
+{
+    /*
+     * Start with bug check line typically seen in windbg.
+     */
+    size_t cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                 "BugCheck %RX64 {%RX64, %RX64, %RX64, %RX64}\n", uBugCheck, uP1, uP2, uP3, uP4);
+    if (cchUsed >= cbDetails)
+        return VINF_BUFFER_OVERFLOW;
+    pszDetails += cchUsed;
+    cbDetails  -= cchUsed;
+
+    /*
+     * Try name the bugcheck and format parameters if we can/care.
+     */
+    char szSym[512];
+    switch (uBugCheck)
+    {
+        case 0x00000001: cchUsed = RTStrPrintf(pszDetails, cbDetails, "APC_INDEX_MISMATCH\n"); break;
+        case 0x00000002: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DEVICE_QUEUE_NOT_BUSY\n"); break;
+        case 0x00000003: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_AFFINITY_SET\n"); break;
+        case 0x00000004: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_DATA_ACCESS_TRAP\n"); break;
+        case 0x00000005: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_PROCESS_ATTACH_ATTEMPT\n"); break;
+        case 0x00000006: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_PROCESS_DETACH_ATTEMPT\n"); break;
+        case 0x00000007: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_SOFTWARE_INTERRUPT\n"); break;
+        case 0x00000008: cchUsed = RTStrPrintf(pszDetails, cbDetails, "IRQL_NOT_DISPATCH_LEVEL\n"); break;
+        case 0x00000009: cchUsed = RTStrPrintf(pszDetails, cbDetails, "IRQL_NOT_GREATER_OR_EQUAL\n"); break;
+        case 0x0000000a:
+            cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                  "IRQL_NOT_LESS_OR_EQUAL\n"
+                                  "P1: %016RX64 - memory referenced\n"
+                                  "P2: %016RX64 - IRQL\n"
+                                  "P3: %016RX64 - bitfield\n"
+                                  "    b0: %u - %s operation\n"
+                                  "    b3: %u - %sexecute operation\n"
+                                  "P4: %016RX64 - EIP/RIP%s\n",
+                                  uP1, uP2, uP3,
+                                  RT_BOOL(uP3 & RT_BIT_64(0)), uP3 & RT_BIT_64(0) ? "write" : "read",
+                                  RT_BOOL(uP3 & RT_BIT_64(3)), uP3 & RT_BIT_64(3) ? "not-" : "",
+                                  uP4, dbgfR3FormatSymbol(pUVM, szSym, sizeof(szSym), ": ", uP4));
+            break;
+        case 0x0000000b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NO_EXCEPTION_HANDLING_SUPPORT\n"); break;
+        case 0x0000000c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MAXIMUM_WAIT_OBJECTS_EXCEEDED\n"); break;
+        case 0x0000000d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MUTEX_LEVEL_NUMBER_VIOLATION\n"); break;
+        case 0x0000000e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NO_USER_MODE_CONTEXT\n"); break;
+        case 0x0000000f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SPIN_LOCK_ALREADY_OWNED\n"); break;
+        case 0x00000010: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SPIN_LOCK_NOT_OWNED\n"); break;
+        case 0x00000011: cchUsed = RTStrPrintf(pszDetails, cbDetails, "THREAD_NOT_MUTEX_OWNER\n"); break;
+        case 0x00000012: cchUsed = RTStrPrintf(pszDetails, cbDetails, "TRAP_CAUSE_UNKNOWN\n"); break;
+        case 0x00000013: cchUsed = RTStrPrintf(pszDetails, cbDetails, "EMPTY_THREAD_REAPER_LIST\n"); break;
+        case 0x00000014: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CREATE_DELETE_LOCK_NOT_LOCKED\n"); break;
+        case 0x00000015: cchUsed = RTStrPrintf(pszDetails, cbDetails, "LAST_CHANCE_CALLED_FROM_KMODE\n"); break;
+        case 0x00000016: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CID_HANDLE_CREATION\n"); break;
+        case 0x00000017: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CID_HANDLE_DELETION\n"); break;
+        case 0x00000018: cchUsed = RTStrPrintf(pszDetails, cbDetails, "REFERENCE_BY_POINTER\n"); break;
+        case 0x00000019: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BAD_POOL_HEADER\n"); break;
+        case 0x0000001a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MEMORY_MANAGEMENT\n"); break;
+        case 0x0000001b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PFN_SHARE_COUNT\n"); break;
+        case 0x0000001c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PFN_REFERENCE_COUNT\n"); break;
+        case 0x0000001d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NO_SPIN_LOCK_AVAILABLE\n"); break;
+        case 0x0000001e:
+            cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                  "KMODE_EXCEPTION_NOT_HANDLED\n"
+                                  "P1: %016RX64 - exception code%s\n"
+                                  "P2: %016RX64 - EIP/RIP%s\n"
+                                  "P3: %016RX64 - Xcpt param #0\n"
+                                  "P4: %016RX64 - Xcpt param #1\n",
+                                  uP1, dbgfR3GetNtStatusName((uint32_t)uP1),
+                                  uP2, dbgfR3FormatSymbol(pUVM, szSym, sizeof(szSym), ": ", uP2),
+                                  uP3, uP4);
+            break;
+        case 0x0000001f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SHARED_RESOURCE_CONV_ERROR\n"); break;
+        case 0x00000020: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_APC_PENDING_DURING_EXIT\n"); break;
+        case 0x00000021: cchUsed = RTStrPrintf(pszDetails, cbDetails, "QUOTA_UNDERFLOW\n"); break;
+        case 0x00000022: cchUsed = RTStrPrintf(pszDetails, cbDetails, "FILE_SYSTEM\n"); break;
+        case 0x00000023: cchUsed = RTStrPrintf(pszDetails, cbDetails, "FAT_FILE_SYSTEM\n"); break;
+        case 0x00000024: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NTFS_FILE_SYSTEM\n"); break;
+        case 0x00000025: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NPFS_FILE_SYSTEM\n"); break;
+        case 0x00000026: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CDFS_FILE_SYSTEM\n"); break;
+        case 0x00000027: cchUsed = RTStrPrintf(pszDetails, cbDetails, "RDR_FILE_SYSTEM\n"); break;
+        case 0x00000028: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CORRUPT_ACCESS_TOKEN\n"); break;
+        case 0x00000029: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SECURITY_SYSTEM\n"); break;
+        case 0x0000002a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INCONSISTENT_IRP\n"); break;
+        case 0x0000002b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PANIC_STACK_SWITCH\n"); break;
+        case 0x0000002c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PORT_DRIVER_INTERNAL\n"); break;
+        case 0x0000002d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SCSI_DISK_DRIVER_INTERNAL\n"); break;
+        case 0x0000002e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DATA_BUS_ERROR\n"); break;
+        case 0x0000002f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INSTRUCTION_BUS_ERROR\n"); break;
+        case 0x00000030: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SET_OF_INVALID_CONTEXT\n"); break;
+        case 0x00000031: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PHASE0_INITIALIZATION_FAILED\n"); break;
+        case 0x00000032: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PHASE1_INITIALIZATION_FAILED\n"); break;
+        case 0x00000033: cchUsed = RTStrPrintf(pszDetails, cbDetails, "UNEXPECTED_INITIALIZATION_CALL\n"); break;
+        case 0x00000034: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CACHE_MANAGER\n"); break;
+        case 0x00000035: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NO_MORE_IRP_STACK_LOCATIONS\n"); break;
+        case 0x00000036: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DEVICE_REFERENCE_COUNT_NOT_ZERO\n"); break;
+        case 0x00000037: cchUsed = RTStrPrintf(pszDetails, cbDetails, "FLOPPY_INTERNAL_ERROR\n"); break;
+        case 0x00000038: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SERIAL_DRIVER_INTERNAL\n"); break;
+        case 0x00000039: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SYSTEM_EXIT_OWNED_MUTEX\n"); break;
+        case 0x0000003a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SYSTEM_UNWIND_PREVIOUS_USER\n"); break;
+        case 0x0000003b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SYSTEM_SERVICE_EXCEPTION\n"); break;
+        case 0x0000003c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INTERRUPT_UNWIND_ATTEMPTED\n"); break;
+        case 0x0000003d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INTERRUPT_EXCEPTION_NOT_HANDLED\n"); break;
+        case 0x0000003e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED\n"); break;
+        case 0x0000003f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NO_MORE_SYSTEM_PTES\n"); break;
+        case 0x00000040: cchUsed = RTStrPrintf(pszDetails, cbDetails, "TARGET_MDL_TOO_SMALL\n"); break;
+        case 0x00000041: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MUST_SUCCEED_POOL_EMPTY\n"); break;
+        case 0x00000042: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ATDISK_DRIVER_INTERNAL\n"); break;
+        case 0x00000043: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NO_SUCH_PARTITION\n"); break;
+        case 0x00000044: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MULTIPLE_IRP_COMPLETE_REQUESTS\n"); break;
+        case 0x00000045: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INSUFFICIENT_SYSTEM_MAP_REGS\n"); break;
+        case 0x00000046: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DEREF_UNKNOWN_LOGON_SESSION\n"); break;
+        case 0x00000047: cchUsed = RTStrPrintf(pszDetails, cbDetails, "REF_UNKNOWN_LOGON_SESSION\n"); break;
+        case 0x00000048: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CANCEL_STATE_IN_COMPLETED_IRP\n"); break;
+        case 0x00000049: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PAGE_FAULT_WITH_INTERRUPTS_OFF\n"); break;
+        case 0x0000004a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "IRQL_GT_ZERO_AT_SYSTEM_SERVICE\n"); break;
+        case 0x0000004b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "STREAMS_INTERNAL_ERROR\n"); break;
+        case 0x0000004c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "FATAL_UNHANDLED_HARD_ERROR\n"); break;
+        case 0x0000004d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NO_PAGES_AVAILABLE\n"); break;
+        case 0x0000004e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PFN_LIST_CORRUPT\n"); break;
+        case 0x0000004f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NDIS_INTERNAL_ERROR\n"); break;
+        case 0x00000050: /* PAGE_FAULT_IN_NONPAGED_AREA */
+        case 0x10000050: /* PAGE_FAULT_IN_NONPAGED_AREA_M */
+            cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                  "PAGE_FAULT_IN_NONPAGED_AREA%s\n"
+                                  "P1: %016RX64 - memory referenced\n"
+                                  "P2: %016RX64 - IRQL\n"
+                                  "P3: %016RX64 - %s\n"
+                                  "P4: %016RX64 - reserved\n",
+                                  uBugCheck & 0x10000000 ? "_M" : "", uP1, uP2, uP3, uP3 & RT_BIT_64(0) ? "write" : "read", uP4);
+            break;
+        case 0x00000051: cchUsed = RTStrPrintf(pszDetails, cbDetails, "REGISTRY_ERROR\n"); break;
+        case 0x00000052: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MAILSLOT_FILE_SYSTEM\n"); break;
+        case 0x00000053: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NO_BOOT_DEVICE\n"); break;
+        case 0x00000054: cchUsed = RTStrPrintf(pszDetails, cbDetails, "LM_SERVER_INTERNAL_ERROR\n"); break;
+        case 0x00000055: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DATA_COHERENCY_EXCEPTION\n"); break;
+        case 0x00000056: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INSTRUCTION_COHERENCY_EXCEPTION\n"); break;
+        case 0x00000057: cchUsed = RTStrPrintf(pszDetails, cbDetails, "XNS_INTERNAL_ERROR\n"); break;
+        case 0x00000058: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VOLMGRX_INTERNAL_ERROR\n"); break;
+        case 0x00000059: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PINBALL_FILE_SYSTEM\n"); break;
+        case 0x0000005a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CRITICAL_SERVICE_FAILED\n"); break;
+        case 0x0000005b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SET_ENV_VAR_FAILED\n"); break;
+        case 0x0000005c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "HAL_INITIALIZATION_FAILED\n"); break;
+        case 0x0000005d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "UNSUPPORTED_PROCESSOR\n"); break;
+        case 0x0000005e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "OBJECT_INITIALIZATION_FAILED\n"); break;
+        case 0x0000005f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SECURITY_INITIALIZATION_FAILED\n"); break;
+        case 0x00000060: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PROCESS_INITIALIZATION_FAILED\n"); break;
+        case 0x00000061: cchUsed = RTStrPrintf(pszDetails, cbDetails, "HAL1_INITIALIZATION_FAILED\n"); break;
+        case 0x00000062: cchUsed = RTStrPrintf(pszDetails, cbDetails, "OBJECT1_INITIALIZATION_FAILED\n"); break;
+        case 0x00000063: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SECURITY1_INITIALIZATION_FAILED\n"); break;
+        case 0x00000064: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SYMBOLIC_INITIALIZATION_FAILED\n"); break;
+        case 0x00000065: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MEMORY1_INITIALIZATION_FAILED\n"); break;
+        case 0x00000066: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CACHE_INITIALIZATION_FAILED\n"); break;
+        case 0x00000067: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CONFIG_INITIALIZATION_FAILED\n"); break;
+        case 0x00000068: cchUsed = RTStrPrintf(pszDetails, cbDetails, "FILE_INITIALIZATION_FAILED\n"); break;
+        case 0x00000069: cchUsed = RTStrPrintf(pszDetails, cbDetails, "IO1_INITIALIZATION_FAILED\n"); break;
+        case 0x0000006a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "LPC_INITIALIZATION_FAILED\n"); break;
+        case 0x0000006b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PROCESS1_INITIALIZATION_FAILED\n"); break;
+        case 0x0000006c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "REFMON_INITIALIZATION_FAILED\n"); break;
+        case 0x0000006d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SESSION1_INITIALIZATION_FAILED\n"); break;
+        case 0x0000006e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BOOTPROC_INITIALIZATION_FAILED\n"); break;
+        case 0x0000006f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VSL_INITIALIZATION_FAILED\n"); break;
+        case 0x00000070: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SOFT_RESTART_FATAL_ERROR\n"); break;
+        case 0x00000072: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ASSIGN_DRIVE_LETTERS_FAILED\n"); break;
+        case 0x00000073: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CONFIG_LIST_FAILED\n"); break;
+        case 0x00000074: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BAD_SYSTEM_CONFIG_INFO\n"); break;
+        case 0x00000075: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CANNOT_WRITE_CONFIGURATION\n"); break;
+        case 0x00000076: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PROCESS_HAS_LOCKED_PAGES\n"); break;
+        case 0x00000077: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_STACK_INPAGE_ERROR\n"); break;
+        case 0x00000078: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PHASE0_EXCEPTION\n"); break;
+        case 0x00000079: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MISMATCHED_HAL\n"); break;
+        case 0x0000007a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_DATA_INPAGE_ERROR\n"); break;
+        case 0x0000007b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INACCESSIBLE_BOOT_DEVICE\n"); break;
+        case 0x0000007c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BUGCODE_NDIS_DRIVER\n"); break;
+        case 0x0000007d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INSTALL_MORE_MEMORY\n"); break;
+        case 0x0000007e: /* SYSTEM_THREAD_EXCEPTION_NOT_HANDLED */
+        case 0x1000007e: /* SYSTEM_THREAD_EXCEPTION_NOT_HANDLED_M */
+            cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                  "SYSTEM_THREAD_EXCEPTION_NOT_HANDLED%s\n"
+                                  "P1: %016RX64 - exception code%s\n"
+                                  "P2: %016RX64 - EIP/RIP%s\n"
+                                  "P3: %016RX64 - Xcpt address\n"
+                                  "P4: %016RX64 - Context address\n",
+                                  uBugCheck & 0x10000000 ? "_M" : "", uP1, dbgfR3GetNtStatusName((uint32_t)uP1),
+                                  uP2, dbgfR3FormatSymbol(pUVM, szSym, sizeof(szSym), ": ", uP2),
+                                  uP3, uP4);
+            break;
+        case 0x0000007f: /* UNEXPECTED_KERNEL_MODE_TRAP */
+        case 0x1000007f: /* UNEXPECTED_KERNEL_MODE_TRAP_M */
+            cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                  "UNEXPECTED_KERNEL_MODE_TRAP%s\n"
+                                  "P1: %016RX64 - x86 trap number\n"
+                                  "P2: %016RX64 - reserved/errorcode?\n"
+                                  "P3: %016RX64 - reserved\n"
+                                  "P4: %016RX64 - reserved\n",
+                                  uBugCheck & 0x10000000 ? "_M" : "", uP1, uP2, uP3, uP4);
+            break;
+        case 0x00000080: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NMI_HARDWARE_FAILURE\n"); break;
+        case 0x00000081: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SPIN_LOCK_INIT_FAILURE\n"); break;
+        case 0x00000082: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DFS_FILE_SYSTEM\n"); break;
+        case 0x00000083: cchUsed = RTStrPrintf(pszDetails, cbDetails, "OFS_FILE_SYSTEM\n"); break;
+        case 0x00000084: cchUsed = RTStrPrintf(pszDetails, cbDetails, "RECOM_DRIVER\n"); break;
+        case 0x00000085: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SETUP_FAILURE\n"); break;
+        case 0x00000086: cchUsed = RTStrPrintf(pszDetails, cbDetails, "AUDIT_FAILURE\n"); break;
+        case 0x0000008b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MBR_CHECKSUM_MISMATCH\n"); break;
+        case 0x0000008e: /* KERNEL_MODE_EXCEPTION_NOT_HANDLED */
+        case 0x1000008e: /* KERNEL_MODE_EXCEPTION_NOT_HANDLED_M */
+            cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                  "KERNEL_MODE_EXCEPTION_NOT_HANDLED%s\n"
+                                  "P1: %016RX64 - exception code%s\n"
+                                  "P2: %016RX64 - EIP/RIP%s\n"
+                                  "P3: %016RX64 - Trap frame address\n"
+                                  "P4: %016RX64 - reserved\n",
+                                  uBugCheck & 0x10000000 ? "_M" : "", uP1, dbgfR3GetNtStatusName((uint32_t)uP1),
+                                  uP2, dbgfR3FormatSymbol(pUVM, szSym, sizeof(szSym), ": ", uP2),
+                                  uP3, uP4);
+            break;
+        case 0x0000008f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PP0_INITIALIZATION_FAILED\n"); break;
+        case 0x00000090: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PP1_INITIALIZATION_FAILED\n"); break;
+        case 0x00000091: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WIN32K_INIT_OR_RIT_FAILURE\n"); break;
+        case 0x00000092: cchUsed = RTStrPrintf(pszDetails, cbDetails, "UP_DRIVER_ON_MP_SYSTEM\n"); break;
+        case 0x00000093: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_KERNEL_HANDLE\n"); break;
+        case 0x00000094: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_STACK_LOCKED_AT_EXIT\n"); break;
+        case 0x00000095: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PNP_INTERNAL_ERROR\n"); break;
+        case 0x00000096: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_WORK_QUEUE_ITEM\n"); break;
+        case 0x00000097: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BOUND_IMAGE_UNSUPPORTED\n"); break;
+        case 0x00000098: cchUsed = RTStrPrintf(pszDetails, cbDetails, "END_OF_NT_EVALUATION_PERIOD\n"); break;
+        case 0x00000099: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_REGION_OR_SEGMENT\n"); break;
+        case 0x0000009a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SYSTEM_LICENSE_VIOLATION\n"); break;
+        case 0x0000009b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "UDFS_FILE_SYSTEM\n"); break;
+        case 0x0000009c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MACHINE_CHECK_EXCEPTION\n"); break;
+        case 0x0000009e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "USER_MODE_HEALTH_MONITOR\n"); break;
+        case 0x0000009f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_POWER_STATE_FAILURE\n"); break;
+        case 0x000000a0: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INTERNAL_POWER_ERROR\n"); break;
+        case 0x000000a1: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PCI_BUS_DRIVER_INTERNAL\n"); break;
+        case 0x000000a2: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MEMORY_IMAGE_CORRUPT\n"); break;
+        case 0x000000a3: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ACPI_DRIVER_INTERNAL\n"); break;
+        case 0x000000a4: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CNSS_FILE_SYSTEM_FILTER\n"); break;
+        case 0x000000a5: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ACPI_BIOS_ERROR\n"); break;
+        case 0x000000a6: cchUsed = RTStrPrintf(pszDetails, cbDetails, "FP_EMULATION_ERROR\n"); break;
+        case 0x000000a7: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BAD_EXHANDLE\n"); break;
+        case 0x000000a8: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BOOTING_IN_SAFEMODE_MINIMAL\n"); break;
+        case 0x000000a9: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BOOTING_IN_SAFEMODE_NETWORK\n"); break;
+        case 0x000000aa: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BOOTING_IN_SAFEMODE_DSREPAIR\n"); break;
+        case 0x000000ab: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SESSION_HAS_VALID_POOL_ON_EXIT\n"); break;
+        case 0x000000ac: cchUsed = RTStrPrintf(pszDetails, cbDetails, "HAL_MEMORY_ALLOCATION\n"); break;
+        case 0x000000b1: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BGI_DETECTED_VIOLATION\n"); break;
+        case 0x000000b4: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VIDEO_DRIVER_INIT_FAILURE\n"); break;
+        case 0x000000b5: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BOOTLOG_LOADED\n"); break;
+        case 0x000000b6: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BOOTLOG_NOT_LOADED\n"); break;
+        case 0x000000b7: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BOOTLOG_ENABLED\n"); break;
+        case 0x000000b8: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ATTEMPTED_SWITCH_FROM_DPC\n"); break;
+        case 0x000000b9: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CHIPSET_DETECTED_ERROR\n"); break;
+        case 0x000000ba: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SESSION_HAS_VALID_VIEWS_ON_EXIT\n"); break;
+        case 0x000000bb: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NETWORK_BOOT_INITIALIZATION_FAILED\n"); break;
+        case 0x000000bc: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NETWORK_BOOT_DUPLICATE_ADDRESS\n"); break;
+        case 0x000000bd: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_HIBERNATED_STATE\n"); break;
+        case 0x000000be: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ATTEMPTED_WRITE_TO_READONLY_MEMORY\n"); break;
+        case 0x000000bf: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MUTEX_ALREADY_OWNED\n"); break;
+        case 0x000000c0: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PCI_CONFIG_SPACE_ACCESS_FAILURE\n"); break;
+        case 0x000000c1: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SPECIAL_POOL_DETECTED_MEMORY_CORRUPTION\n"); break;
+
+        case 0x000000c2:
+            cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                  "BAD_POOL_CALLER\n"
+                                  "P1: %016RX64 - ", uP1);
+            if (cchUsed >= cbDetails)
+                return VINF_BUFFER_OVERFLOW;
+            cbDetails  -= cchUsed;
+            pszDetails += cchUsed;
+            switch (uP1)
+            {
+                case 1:
+                case 2:
+                case 4:
+                    cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                          "Pool header corrupted!\n"
+                                          "P2: %016RX64 - Pool header address\n"
+                                          "P3: %016RX64 - Pool header contents\n"
+                                          "P4: %016RX64 - reserved\n", uP2, uP3, uP4);
+                    break;
+                case 6:
+                    cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                          "Double free w/o tag!\n"
+                                          "P2: %016RX64 - reserved\n"
+                                          "P3: %016RX64 - Pool header address\n"
+                                          "P4: %016RX64 - Pool header contents\n", uP2, uP3, uP4);
+                    break;
+                case 7:
+                    cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                          "Double free w/ tag!\n"
+                                          "P2: %016RX64 - tag %c%c%c%c\n"
+                                          "P3: %016RX64 - Pool header contents\n"
+                                          "P4: %016RX64 - Free address\n",
+                                          uP2,
+                                          RT_C_IS_PRINT(RT_BYTE1(uP2)) ? RT_BYTE1(uP2) : '.',
+                                          RT_C_IS_PRINT(RT_BYTE2(uP2)) ? RT_BYTE2(uP2) : '.',
+                                          RT_C_IS_PRINT(RT_BYTE3(uP2)) ? RT_BYTE3(uP2) : '.',
+                                          RT_C_IS_PRINT(RT_BYTE4(uP2)) ? RT_BYTE4(uP2) : '.',
+                                          uP3, uP4);
+                    break;
+                case 8:
+                    cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                          "Wrong IRQL for allocation!\n"
+                                          "P2: %016RX64 - IRQL\n"
+                                          "P3: %016RX64 - Pool type\n"
+                                          "P4: %016RX64 - Allocation size\n",
+                                          uP2, uP3, uP4);
+                    break;
+                case 9:
+                    cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                          "Wrong IRQL for free!\n"
+                                          "P2: %016RX64 - IRQL\n"
+                                          "P3: %016RX64 - Pool type\n"
+                                          "P4: %016RX64 - Pool address\n",
+                                          uP2, uP3, uP4);
+                    break;
+                /** @todo fill in more BAD_POOL_CALLER types here as needed.*/
+                default:
+                    cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                          "Unknown pool violation type\n"
+                                          "P2: %016RX64 - type specific\n"
+                                          "P3: %016RX64 - type specific\n"
+                                          "P4: %016RX64 - type specific\n",
+                                          uP2, uP3, uP4);
+                    break;
+            }
+            break;
+
+        case 0x000000c3: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SYSTEM_IMAGE_BAD_SIGNATURE\n"); break;
+        case 0x000000c4: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_VERIFIER_DETECTED_VIOLATION\n"); break;
+        case 0x000000c5: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_CORRUPTED_EXPOOL\n"); break;
+        case 0x000000c6: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_CAUGHT_MODIFYING_FREED_POOL\n"); break;
+        case 0x000000c7: cchUsed = RTStrPrintf(pszDetails, cbDetails, "TIMER_OR_DPC_INVALID\n"); break;
+        case 0x000000c8: cchUsed = RTStrPrintf(pszDetails, cbDetails, "IRQL_UNEXPECTED_VALUE\n"); break;
+        case 0x000000c9: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_VERIFIER_IOMANAGER_VIOLATION\n"); break;
+        case 0x000000ca: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PNP_DETECTED_FATAL_ERROR\n"); break;
+        case 0x000000cb: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_LEFT_LOCKED_PAGES_IN_PROCESS\n"); break;
+        case 0x000000cc: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PAGE_FAULT_IN_FREED_SPECIAL_POOL\n"); break;
+        case 0x000000cd: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PAGE_FAULT_BEYOND_END_OF_ALLOCATION\n"); break;
+        case 0x000000ce: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_UNLOADED_WITHOUT_CANCELLING_PENDING_OPERATIONS\n"); break;
+        case 0x000000cf: cchUsed = RTStrPrintf(pszDetails, cbDetails, "TERMINAL_SERVER_DRIVER_MADE_INCORRECT_MEMORY_REFERENCE\n"); break;
+        case 0x000000d0: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_CORRUPTED_MMPOOL\n"); break;
+        case 0x000000d1:
+            cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                  "DRIVER_IRQL_NOT_LESS_OR_EQUAL\n"
+                                  "P1: %016RX64 - memory referenced\n"
+                                  "P2: %016RX64 - IRQL\n"
+                                  "P3: %016RX64 - %s\n"
+                                  "P4: %016RX64 - EIP/RIP%s\n",
+                                  uP1, uP2, uP3, uP3 & RT_BIT_64(0) ? "write" : "read",
+                                  uP4, dbgfR3FormatSymbol(pUVM, szSym, sizeof(szSym), ": ", uP4));
+            break;
+        case 0x000000d2: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BUGCODE_ID_DRIVER\n"); break;
+        case 0x000000d3: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_PORTION_MUST_BE_NONPAGED\n"); break;
+        case 0x000000d4: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SYSTEM_SCAN_AT_RAISED_IRQL_CAUGHT_IMPROPER_DRIVER_UNLOAD\n"); break;
+        case 0x000000d5: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_PAGE_FAULT_IN_FREED_SPECIAL_POOL\n"); break;
+        case 0x000000d6: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_PAGE_FAULT_BEYOND_END_OF_ALLOCATION\n"); break;
+        case 0x100000d6: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_PAGE_FAULT_BEYOND_END_OF_ALLOCATION_M\n"); break;
+        case 0x000000d7: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_UNMAPPING_INVALID_VIEW\n"); break;
+        case 0x000000d8:
+            cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                  "DRIVER_USED_EXCESSIVE_PTES\n"
+                                  "P1: %016RX64 - Driver name pointer\n"
+                                  "P2: %016RX64 - Number of PTEs\n"
+                                  "P3: %016RX64 - Free system PTEs\n"
+                                  "P4: %016RX64 - System PTEs\n",
+                                  uP1, uP2, uP3, uP4);
+            break;
+        case 0x000000d9: cchUsed = RTStrPrintf(pszDetails, cbDetails, "LOCKED_PAGES_TRACKER_CORRUPTION\n"); break;
+        case 0x000000da: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SYSTEM_PTE_MISUSE\n"); break;
+        case 0x000000db: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_CORRUPTED_SYSPTES\n"); break;
+        case 0x000000dc: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_INVALID_STACK_ACCESS\n"); break;
+        case 0x000000de: cchUsed = RTStrPrintf(pszDetails, cbDetails, "POOL_CORRUPTION_IN_FILE_AREA\n"); break;
+        case 0x000000df: cchUsed = RTStrPrintf(pszDetails, cbDetails, "IMPERSONATING_WORKER_THREAD\n"); break;
+        case 0x000000e0: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ACPI_BIOS_FATAL_ERROR\n"); break;
+        case 0x000000e1: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WORKER_THREAD_RETURNED_AT_BAD_IRQL\n"); break;
+        case 0x000000e2: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MANUALLY_INITIATED_CRASH\n"); break;
+        case 0x000000e3: cchUsed = RTStrPrintf(pszDetails, cbDetails, "RESOURCE_NOT_OWNED\n"); break;
+        case 0x000000e4: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WORKER_INVALID\n"); break;
+        case 0x000000e5: cchUsed = RTStrPrintf(pszDetails, cbDetails, "POWER_FAILURE_SIMULATE\n"); break;
+        case 0x000000e6: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_VERIFIER_DMA_VIOLATION\n"); break;
+        case 0x000000e7: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_FLOATING_POINT_STATE\n"); break;
+        case 0x000000e8: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_CANCEL_OF_FILE_OPEN\n"); break;
+        case 0x000000e9: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ACTIVE_EX_WORKER_THREAD_TERMINATION\n"); break;
+        case 0x000000ea: cchUsed = RTStrPrintf(pszDetails, cbDetails, "THREAD_STUCK_IN_DEVICE_DRIVER\n"); break;
+        case 0x100000ea: cchUsed = RTStrPrintf(pszDetails, cbDetails, "THREAD_STUCK_IN_DEVICE_DRIVER_M\n"); break;
+        case 0x000000eb: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DIRTY_MAPPED_PAGES_CONGESTION\n"); break;
+        case 0x000000ec: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SESSION_HAS_VALID_SPECIAL_POOL_ON_EXIT\n"); break;
+        case 0x000000ed: cchUsed = RTStrPrintf(pszDetails, cbDetails, "UNMOUNTABLE_BOOT_VOLUME\n"); break;
+        case 0x000000ef: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CRITICAL_PROCESS_DIED\n"); break;
+        case 0x000000f0: cchUsed = RTStrPrintf(pszDetails, cbDetails, "STORAGE_MINIPORT_ERROR\n"); break;
+        case 0x000000f1: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SCSI_VERIFIER_DETECTED_VIOLATION\n"); break;
+        case 0x000000f2: cchUsed = RTStrPrintf(pszDetails, cbDetails, "HARDWARE_INTERRUPT_STORM\n"); break;
+        case 0x000000f3: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DISORDERLY_SHUTDOWN\n"); break;
+        case 0x000000f4: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CRITICAL_OBJECT_TERMINATION\n"); break;
+        case 0x000000f5: cchUsed = RTStrPrintf(pszDetails, cbDetails, "FLTMGR_FILE_SYSTEM\n"); break;
+        case 0x000000f6: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PCI_VERIFIER_DETECTED_VIOLATION\n"); break;
+        case 0x000000f7: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_OVERRAN_STACK_BUFFER\n"); break;
+        case 0x000000f8: cchUsed = RTStrPrintf(pszDetails, cbDetails, "RAMDISK_BOOT_INITIALIZATION_FAILED\n"); break;
+        case 0x000000f9: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_RETURNED_STATUS_REPARSE_FOR_VOLUME_OPEN\n"); break;
+        case 0x000000fa: cchUsed = RTStrPrintf(pszDetails, cbDetails, "HTTP_DRIVER_CORRUPTED\n"); break;
+        case 0x000000fb: cchUsed = RTStrPrintf(pszDetails, cbDetails, "RECURSIVE_MACHINE_CHECK\n"); break;
+        case 0x000000fc: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ATTEMPTED_EXECUTE_OF_NOEXECUTE_MEMORY\n"); break;
+        case 0x000000fd: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DIRTY_NOWRITE_PAGES_CONGESTION\n"); break;
+        case 0x000000fe: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BUGCODE_USB_DRIVER\n"); break;
+        case 0x000000ff: cchUsed = RTStrPrintf(pszDetails, cbDetails, "RESERVE_QUEUE_OVERFLOW\n"); break;
+        case 0x00000100: cchUsed = RTStrPrintf(pszDetails, cbDetails, "LOADER_BLOCK_MISMATCH\n"); break;
+        case 0x00000101: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLOCK_WATCHDOG_TIMEOUT\n"); break;
+        case 0x00000102: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DPC_WATCHDOG_TIMEOUT\n"); break;
+        case 0x00000103: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MUP_FILE_SYSTEM\n"); break;
+        case 0x00000104: cchUsed = RTStrPrintf(pszDetails, cbDetails, "AGP_INVALID_ACCESS\n"); break;
+        case 0x00000105: cchUsed = RTStrPrintf(pszDetails, cbDetails, "AGP_GART_CORRUPTION\n"); break;
+        case 0x00000106: cchUsed = RTStrPrintf(pszDetails, cbDetails, "AGP_ILLEGALLY_REPROGRAMMED\n"); break;
+        case 0x00000107: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_EXPAND_STACK_ACTIVE\n"); break;
+        case 0x00000108: cchUsed = RTStrPrintf(pszDetails, cbDetails, "THIRD_PARTY_FILE_SYSTEM_FAILURE\n"); break;
+        case 0x00000109: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CRITICAL_STRUCTURE_CORRUPTION\n"); break;
+        case 0x0000010a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "APP_TAGGING_INITIALIZATION_FAILED\n"); break;
+        case 0x0000010b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DFSC_FILE_SYSTEM\n"); break;
+        case 0x0000010c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "FSRTL_EXTRA_CREATE_PARAMETER_VIOLATION\n"); break;
+        case 0x0000010d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WDF_VIOLATION\n"); break;
+        case 0x0000010e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VIDEO_MEMORY_MANAGEMENT_INTERNAL\n"); break;
+        case 0x00000110: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_INVALID_CRUNTIME_PARAMETER\n"); break;
+        case 0x00000111: cchUsed = RTStrPrintf(pszDetails, cbDetails, "RECURSIVE_NMI\n"); break;
+        case 0x00000112: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MSRPC_STATE_VIOLATION\n"); break;
+        case 0x00000113: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VIDEO_DXGKRNL_FATAL_ERROR\n"); break;
+        case 0x00000114: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VIDEO_SHADOW_DRIVER_FATAL_ERROR\n"); break;
+        case 0x00000115: cchUsed = RTStrPrintf(pszDetails, cbDetails, "AGP_INTERNAL\n"); break;
+        case 0x00000116: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VIDEO_TDR_FAILURE\n"); break;
+        case 0x00000117: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VIDEO_TDR_TIMEOUT_DETECTED\n"); break;
+        case 0x00000118: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NTHV_GUEST_ERROR\n"); break;
+        case 0x00000119: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VIDEO_SCHEDULER_INTERNAL_ERROR\n"); break;
+        case 0x0000011a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "EM_INITIALIZATION_ERROR\n"); break;
+        case 0x0000011b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_RETURNED_HOLDING_CANCEL_LOCK\n"); break;
+        case 0x0000011c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ATTEMPTED_WRITE_TO_CM_PROTECTED_STORAGE\n"); break;
+        case 0x0000011d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "EVENT_TRACING_FATAL_ERROR\n"); break;
+        case 0x0000011e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "TOO_MANY_RECURSIVE_FAULTS\n"); break;
+        case 0x0000011f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_DRIVER_HANDLE\n"); break;
+        case 0x00000120: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BITLOCKER_FATAL_ERROR\n"); break;
+        case 0x00000121: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_VIOLATION\n"); break;
+        case 0x00000122: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WHEA_INTERNAL_ERROR\n"); break;
+        case 0x00000123: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CRYPTO_SELF_TEST_FAILURE\n"); break;
+        case 0x00000124: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WHEA_UNCORRECTABLE_ERROR\n"); break;
+        case 0x00000125: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NMR_INVALID_STATE\n"); break;
+        case 0x00000126: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NETIO_INVALID_POOL_CALLER\n"); break;
+        case 0x00000127: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PAGE_NOT_ZERO\n"); break;
+        case 0x00000128: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WORKER_THREAD_RETURNED_WITH_BAD_IO_PRIORITY\n"); break;
+        case 0x00000129: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WORKER_THREAD_RETURNED_WITH_BAD_PAGING_IO_PRIORITY\n"); break;
+        case 0x0000012a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MUI_NO_VALID_SYSTEM_LANGUAGE\n"); break;
+        case 0x0000012b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "FAULTY_HARDWARE_CORRUPTED_PAGE\n"); break;
+        case 0x0000012c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "EXFAT_FILE_SYSTEM\n"); break;
+        case 0x0000012d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VOLSNAP_OVERLAPPED_TABLE_ACCESS\n"); break;
+        case 0x0000012e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_MDL_RANGE\n"); break;
+        case 0x0000012f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VHD_BOOT_INITIALIZATION_FAILED\n"); break;
+        case 0x00000130: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DYNAMIC_ADD_PROCESSOR_MISMATCH\n"); break;
+        case 0x00000131: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_EXTENDED_PROCESSOR_STATE\n"); break;
+        case 0x00000132: cchUsed = RTStrPrintf(pszDetails, cbDetails, "RESOURCE_OWNER_POINTER_INVALID\n"); break;
+        case 0x00000133: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DPC_WATCHDOG_VIOLATION\n"); break;
+        case 0x00000134: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVE_EXTENDER\n"); break;
+        case 0x00000135: cchUsed = RTStrPrintf(pszDetails, cbDetails, "REGISTRY_FILTER_DRIVER_EXCEPTION\n"); break;
+        case 0x00000136: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VHD_BOOT_HOST_VOLUME_NOT_ENOUGH_SPACE\n"); break;
+        case 0x00000137: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WIN32K_HANDLE_MANAGER\n"); break;
+        case 0x00000138: cchUsed = RTStrPrintf(pszDetails, cbDetails, "GPIO_CONTROLLER_DRIVER_ERROR\n"); break;
+
+        case 0x00000139:
+        {
+            const char *pszCheck;
+            switch (uP1)
+            {
+                case 0x00:  pszCheck = "Stack buffer overrun (/GS)"; break;
+                case 0x01:  pszCheck = "Illegal virtual function table use (VTGuard)"; break;
+                case 0x02:  pszCheck = "Stack buffer overrun (via cookie)"; break;
+                case 0x03:  pszCheck = "Correupt LIST_ENTRY"; break;
+                case 0x04:  pszCheck = "Out of bounds stack pointer"; break;
+                case 0x05:  pszCheck = "Invalid parameter (fatal)"; break;
+                case 0x06:  pszCheck = "Uninitialized stack cookie (by loader prior to Win8)"; break;
+                case 0x07:  pszCheck = "Fatal program exit request"; break;
+                case 0x08:  pszCheck = "Compiler bounds check violation"; break;
+                case 0x09:  pszCheck = "Direct RtlQueryRegistryValues w/o typechecking on untrusted hive"; break;
+                /* https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/bug-check---bug-check-0x139-kernel-security-check-failure
+                   and !analyze -show differs on the following: */
+                case 0x0a: case 0x0b: case 0x0c: case 0x0d: case 0x0e:
+                case 0x0f:  pszCheck = "Memory safety violation [?]"; break;
+                case 0x10:  pszCheck = "Invalid indirect call (indirect call guard) [?]"; break;
+                case 0x11:  pszCheck = "Invalid memory write (write guard) [?]"; break;
+                case 0x12:  pszCheck = "Invalid target context for fiber switch [?]"; break;
+                /** @todo there are lots more... */
+                default:    pszCheck = "Todo/Unknown"; break;
+            }
+            cchUsed = RTStrPrintf(pszDetails, cbDetails,
+                                  "KERNEL_SECURITY_CHECK_FAILURE\n"
+                                  "P1: %016RX64 - %s!\n"
+                                  "P2: %016RX64 - Trap frame address\n"
+                                  "P3: %016RX64 - Exception record\n"
+                                  "P4: %016RX64 - reserved\n", uP1, pszCheck, uP2, uP3, uP4);
+            break;
+        }
+
+        case 0x0000013a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_MODE_HEAP_CORRUPTION\n"); break;
+        case 0x0000013b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PASSIVE_INTERRUPT_ERROR\n"); break;
+        case 0x0000013c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_IO_BOOST_STATE\n"); break;
+        case 0x0000013d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CRITICAL_INITIALIZATION_FAILURE\n"); break;
+        case 0x0000013e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ERRATA_WORKAROUND_UNSUCCESSFUL\n"); break;
+        case 0x00000140: cchUsed = RTStrPrintf(pszDetails, cbDetails, "STORAGE_DEVICE_ABNORMALITY_DETECTED\n"); break;
+        case 0x00000141: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VIDEO_ENGINE_TIMEOUT_DETECTED\n"); break;
+        case 0x00000142: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VIDEO_TDR_APPLICATION_BLOCKED\n"); break;
+        case 0x00000143: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PROCESSOR_DRIVER_INTERNAL\n"); break;
+        case 0x00000144: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BUGCODE_USB3_DRIVER\n"); break;
+        case 0x00000145: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SECURE_BOOT_VIOLATION\n"); break;
+        case 0x00000146: cchUsed = RTStrPrintf(pszDetails, cbDetails, "NDIS_NET_BUFFER_LIST_INFO_ILLEGALLY_TRANSFERRED\n"); break;
+        case 0x00000147: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ABNORMAL_RESET_DETECTED\n"); break;
+        case 0x00000148: cchUsed = RTStrPrintf(pszDetails, cbDetails, "IO_OBJECT_INVALID\n"); break;
+        case 0x00000149: cchUsed = RTStrPrintf(pszDetails, cbDetails, "REFS_FILE_SYSTEM\n"); break;
+        case 0x0000014a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_WMI_INTERNAL\n"); break;
+        case 0x0000014b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SOC_SUBSYSTEM_FAILURE\n"); break;
+        case 0x0000014c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "FATAL_ABNORMAL_RESET_ERROR\n"); break;
+        case 0x0000014d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "EXCEPTION_SCOPE_INVALID\n"); break;
+        case 0x0000014e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SOC_CRITICAL_DEVICE_REMOVED\n"); break;
+        case 0x0000014f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PDC_WATCHDOG_TIMEOUT\n"); break;
+        case 0x00000150: cchUsed = RTStrPrintf(pszDetails, cbDetails, "TCPIP_AOAC_NIC_ACTIVE_REFERENCE_LEAK\n"); break;
+        case 0x00000151: cchUsed = RTStrPrintf(pszDetails, cbDetails, "UNSUPPORTED_INSTRUCTION_MODE\n"); break;
+        case 0x00000152: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_PUSH_LOCK_FLAGS\n"); break;
+        case 0x00000153: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_LOCK_ENTRY_LEAKED_ON_THREAD_TERMINATION\n"); break;
+        case 0x00000154: cchUsed = RTStrPrintf(pszDetails, cbDetails, "UNEXPECTED_STORE_EXCEPTION\n"); break;
+        case 0x00000155: cchUsed = RTStrPrintf(pszDetails, cbDetails, "OS_DATA_TAMPERING\n"); break;
+        case 0x00000156: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WINSOCK_DETECTED_HUNG_CLOSESOCKET_LIVEDUMP\n"); break;
+        case 0x00000157: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_THREAD_PRIORITY_FLOOR_VIOLATION\n"); break;
+        case 0x00000158: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ILLEGAL_IOMMU_PAGE_FAULT\n"); break;
+        case 0x00000159: cchUsed = RTStrPrintf(pszDetails, cbDetails, "HAL_ILLEGAL_IOMMU_PAGE_FAULT\n"); break;
+        case 0x0000015a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SDBUS_INTERNAL_ERROR\n"); break;
+        case 0x0000015b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WORKER_THREAD_RETURNED_WITH_SYSTEM_PAGE_PRIORITY_ACTIVE\n"); break;
+        case 0x0000015c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PDC_WATCHDOG_TIMEOUT_LIVEDUMP\n"); break;
+        case 0x0000015d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SOC_SUBSYSTEM_FAILURE_LIVEDUMP\n"); break;
+        case 0x0000015e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BUGCODE_NDIS_DRIVER_LIVE_DUMP\n"); break;
+        case 0x0000015f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CONNECTED_STANDBY_WATCHDOG_TIMEOUT_LIVEDUMP\n"); break;
+        case 0x00000160: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WIN32K_ATOMIC_CHECK_FAILURE\n"); break;
+        case 0x00000161: cchUsed = RTStrPrintf(pszDetails, cbDetails, "LIVE_SYSTEM_DUMP\n"); break;
+        case 0x00000162: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_AUTO_BOOST_INVALID_LOCK_RELEASE\n"); break;
+        case 0x00000163: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WORKER_THREAD_TEST_CONDITION\n"); break;
+        case 0x00000164: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WIN32K_CRITICAL_FAILURE\n"); break;
+        case 0x00000165: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLUSTER_CSV_STATUS_IO_TIMEOUT_LIVEDUMP\n"); break;
+        case 0x00000166: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLUSTER_RESOURCE_CALL_TIMEOUT_LIVEDUMP\n"); break;
+        case 0x00000167: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLUSTER_CSV_SNAPSHOT_DEVICE_INFO_TIMEOUT_LIVEDUMP\n"); break;
+        case 0x00000168: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLUSTER_CSV_STATE_TRANSITION_TIMEOUT_LIVEDUMP\n"); break;
+        case 0x00000169: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLUSTER_CSV_VOLUME_ARRIVAL_LIVEDUMP\n"); break;
+        case 0x0000016a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLUSTER_CSV_VOLUME_REMOVAL_LIVEDUMP\n"); break;
+        case 0x0000016b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLUSTER_CSV_CLUSTER_WATCHDOG_LIVEDUMP\n"); break;
+        case 0x0000016c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_RUNDOWN_PROTECTION_FLAGS\n"); break;
+        case 0x0000016d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_SLOT_ALLOCATOR_FLAGS\n"); break;
+        case 0x0000016e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ERESOURCE_INVALID_RELEASE\n"); break;
+        case 0x0000016f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLUSTER_CSV_STATE_TRANSITION_INTERVAL_TIMEOUT_LIVEDUMP\n"); break;
+        case 0x00000170: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLUSTER_CSV_CLUSSVC_DISCONNECT_WATCHDOG\n"); break;
+        case 0x00000171: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CRYPTO_LIBRARY_INTERNAL_ERROR\n"); break;
+        case 0x00000173: cchUsed = RTStrPrintf(pszDetails, cbDetails, "COREMSGCALL_INTERNAL_ERROR\n"); break;
+        case 0x00000174: cchUsed = RTStrPrintf(pszDetails, cbDetails, "COREMSG_INTERNAL_ERROR\n"); break;
+        case 0x00000175: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PREVIOUS_FATAL_ABNORMAL_RESET_ERROR\n"); break;
+        case 0x00000178: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ELAM_DRIVER_DETECTED_FATAL_ERROR\n"); break;
+        case 0x00000179: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLUSTER_CLUSPORT_STATUS_IO_TIMEOUT_LIVEDUMP\n"); break;
+        case 0x0000017b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PROFILER_CONFIGURATION_ILLEGAL\n"); break;
+        case 0x0000017c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PDC_LOCK_WATCHDOG_LIVEDUMP\n"); break;
+        case 0x0000017d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PDC_UNEXPECTED_REVOCATION_LIVEDUMP\n"); break;
+        case 0x00000180: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WVR_LIVEDUMP_REPLICATION_IOCONTEXT_TIMEOUT\n"); break;
+        case 0x00000181: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WVR_LIVEDUMP_STATE_TRANSITION_TIMEOUT\n"); break;
+        case 0x00000182: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WVR_LIVEDUMP_RECOVERY_IOCONTEXT_TIMEOUT\n"); break;
+        case 0x00000183: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WVR_LIVEDUMP_APP_IO_TIMEOUT\n"); break;
+        case 0x00000184: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WVR_LIVEDUMP_MANUALLY_INITIATED\n"); break;
+        case 0x00000185: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WVR_LIVEDUMP_STATE_FAILURE\n"); break;
+        case 0x00000186: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WVR_LIVEDUMP_CRITICAL_ERROR\n"); break;
+        case 0x00000187: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VIDEO_DWMINIT_TIMEOUT_FALLBACK_BDD\n"); break;
+        case 0x00000188: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLUSTER_CSVFS_LIVEDUMP\n"); break;
+        case 0x00000189: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BAD_OBJECT_HEADER\n"); break;
+        case 0x0000018a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SILO_CORRUPT\n"); break;
+        case 0x0000018b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SECURE_KERNEL_ERROR\n"); break;
+        case 0x0000018c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "HYPERGUARD_VIOLATION\n"); break;
+        case 0x0000018d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SECURE_FAULT_UNHANDLED\n"); break;
+        case 0x0000018e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_PARTITION_REFERENCE_VIOLATION\n"); break;
+        case 0x00000190: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WIN32K_CRITICAL_FAILURE_LIVEDUMP\n"); break;
+        case 0x00000191: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PF_DETECTED_CORRUPTION\n"); break;
+        case 0x00000192: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_AUTO_BOOST_LOCK_ACQUISITION_WITH_RAISED_IRQL\n"); break;
+        case 0x00000193: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VIDEO_DXGKRNL_LIVEDUMP\n"); break;
+        case 0x00000194: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_NONRESPONSIVEPROCESS\n"); break;
+        case 0x00000195: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SMB_SERVER_LIVEDUMP\n"); break;
+        case 0x00000196: cchUsed = RTStrPrintf(pszDetails, cbDetails, "LOADER_ROLLBACK_DETECTED\n"); break;
+        case 0x00000197: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WIN32K_SECURITY_FAILURE\n"); break;
+        case 0x00000198: cchUsed = RTStrPrintf(pszDetails, cbDetails, "UFX_LIVEDUMP\n"); break;
+        case 0x00000199: cchUsed = RTStrPrintf(pszDetails, cbDetails, "KERNEL_STORAGE_SLOT_IN_USE\n"); break;
+        case 0x0000019a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WORKER_THREAD_RETURNED_WHILE_ATTACHED_TO_SILO\n"); break;
+        case 0x0000019b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "TTM_FATAL_ERROR\n"); break;
+        case 0x0000019c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WIN32K_POWER_WATCHDOG_TIMEOUT\n"); break;
+        case 0x0000019d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CLUSTER_SVHDX_LIVEDUMP\n"); break;
+        case 0x0000019e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BUGCODE_NETADAPTER_DRIVER\n"); break;
+        case 0x0000019f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "PDC_PRIVILEGE_CHECK_LIVEDUMP\n"); break;
+        case 0x000001a0: cchUsed = RTStrPrintf(pszDetails, cbDetails, "TTM_WATCHDOG_TIMEOUT\n"); break;
+        case 0x000001a1: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WIN32K_CALLOUT_WATCHDOG_LIVEDUMP\n"); break;
+        case 0x000001a2: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WIN32K_CALLOUT_WATCHDOG_BUGCHECK\n"); break;
+        case 0x000001a3: cchUsed = RTStrPrintf(pszDetails, cbDetails, "CALL_HAS_NOT_RETURNED_WATCHDOG_TIMEOUT_LIVEDUMP\n"); break;
+        case 0x000001a4: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIPS_SW_HW_DIVERGENCE_LIVEDUMP\n"); break;
+        case 0x000001a5: cchUsed = RTStrPrintf(pszDetails, cbDetails, "USB_DRIPS_BLOCKER_SURPRISE_REMOVAL_LIVEDUMP\n"); break;
+        case 0x000001c4: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_VERIFIER_DETECTED_VIOLATION_LIVEDUMP\n"); break;
+        case 0x000001c5: cchUsed = RTStrPrintf(pszDetails, cbDetails, "IO_THREADPOOL_DEADLOCK_LIVEDUMP\n"); break;
+        case 0x000001c6: cchUsed = RTStrPrintf(pszDetails, cbDetails, "FAST_ERESOURCE_PRECONDITION_VIOLATION\n"); break;
+        case 0x000001c7: cchUsed = RTStrPrintf(pszDetails, cbDetails, "STORE_DATA_STRUCTURE_CORRUPTION\n"); break;
+        case 0x000001c8: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MANUALLY_INITIATED_POWER_BUTTON_HOLD\n"); break;
+        case 0x000001c9: cchUsed = RTStrPrintf(pszDetails, cbDetails, "USER_MODE_HEALTH_MONITOR_LIVEDUMP\n"); break;
+        case 0x000001ca: cchUsed = RTStrPrintf(pszDetails, cbDetails, "HYPERVISOR_WATCHDOG_TIMEOUT\n"); break;
+        case 0x000001cb: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_SILO_DETACH\n"); break;
+        case 0x000001cc: cchUsed = RTStrPrintf(pszDetails, cbDetails, "EXRESOURCE_TIMEOUT_LIVEDUMP\n"); break;
+        case 0x000001cd: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_CALLBACK_STACK_ADDRESS\n"); break;
+        case 0x000001ce: cchUsed = RTStrPrintf(pszDetails, cbDetails, "INVALID_KERNEL_STACK_ADDRESS\n"); break;
+        case 0x000001cf: cchUsed = RTStrPrintf(pszDetails, cbDetails, "HARDWARE_WATCHDOG_TIMEOUT\n"); break;
+        case 0x000001d0: cchUsed = RTStrPrintf(pszDetails, cbDetails, "ACPI_FIRMWARE_WATCHDOG_TIMEOUT\n"); break;
+        case 0x000001d1: cchUsed = RTStrPrintf(pszDetails, cbDetails, "TELEMETRY_ASSERTS_LIVEDUMP\n"); break;
+        case 0x000001d2: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WORKER_THREAD_INVALID_STATE\n"); break;
+        case 0x000001d3: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WFP_INVALID_OPERATION\n"); break;
+        case 0x000001d4: cchUsed = RTStrPrintf(pszDetails, cbDetails, "UCMUCSI_LIVEDUMP\n"); break;
+        case 0x000001d5: cchUsed = RTStrPrintf(pszDetails, cbDetails, "DRIVER_PNP_WATCHDOG\n"); break;
+        case 0x00000315: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_MTBFCOMMANDTIMEOUT\n"); break;
+        case 0x00000356: cchUsed = RTStrPrintf(pszDetails, cbDetails, "XBOX_ERACTRL_CS_TIMEOUT\n"); break;
+        case 0x00000357: cchUsed = RTStrPrintf(pszDetails, cbDetails, "XBOX_CORRUPTED_IMAGE\n"); break;
+        case 0x00000358: cchUsed = RTStrPrintf(pszDetails, cbDetails, "XBOX_INVERTED_FUNCTION_TABLE_OVERFLOW\n"); break;
+        case 0x00000359: cchUsed = RTStrPrintf(pszDetails, cbDetails, "XBOX_CORRUPTED_IMAGE_BASE\n"); break;
+        case 0x00000360: cchUsed = RTStrPrintf(pszDetails, cbDetails, "XBOX_360_SYSTEM_CRASH\n"); break;
+        case 0x00000420: cchUsed = RTStrPrintf(pszDetails, cbDetails, "XBOX_360_SYSTEM_CRASH_RESERVED\n"); break;
+        case 0x00000bfe: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BC_BLUETOOTH_VERIFIER_FAULT\n"); break;
+        case 0x00000bff: cchUsed = RTStrPrintf(pszDetails, cbDetails, "BC_BTHMINI_VERIFIER_FAULT\n"); break;
+        case 0x00008866: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_SICKAPPLICATION\n"); break;
+        case 0x0000f000: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_UNSPECIFIED\n"); break;
+        case 0x0000f002: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_BLANKSCREEN\n"); break;
+        case 0x0000f003: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_INPUT\n"); break;
+        case 0x0000f004: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_WATCHDOG\n"); break;
+        case 0x0000f005: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_STARTNOTVISIBLE\n"); break;
+        case 0x0000f006: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_NAVIGATIONMODEL\n"); break;
+        case 0x0000f007: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_OUTOFMEMORY\n"); break;
+        case 0x0000f008: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_GRAPHICS\n"); break;
+        case 0x0000f009: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_NAVSERVERTIMEOUT\n"); break;
+        case 0x0000f00a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_CHROMEPROCESSCRASH\n"); break;
+        case 0x0000f00b: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_NOTIFICATIONDISMISSAL\n"); break;
+        case 0x0000f00c: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_SPEECHDISMISSAL\n"); break;
+        case 0x0000f00d: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_CALLDISMISSAL\n"); break;
+        case 0x0000f00e: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_APPBARDISMISSAL\n"); break;
+        case 0x0000f00f: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_RILADAPTATIONCRASH\n"); break;
+        case 0x0000f010: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_APPLISTUNREACHABLE\n"); break;
+        case 0x0000f011: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_REPORTNOTIFICATIONFAILURE\n"); break;
+        case 0x0000f012: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_UNEXPECTEDSHUTDOWN\n"); break;
+        case 0x0000f013: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_RPCFAILURE\n"); break;
+        case 0x0000f014: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_AUXILIARYFULLDUMP\n"); break;
+        case 0x0000f015: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_ACCOUNTPROVSVCINITFAILURE\n"); break;
+        case 0x0000f101: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_MTBFCOMMANDHANG\n"); break;
+        case 0x0000f102: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_MTBFPASSBUGCHECK\n"); break;
+        case 0x0000f103: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_MTBFIOERROR\n"); break;
+        case 0x0000f200: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_RENDERTHREADHANG\n"); break;
+        case 0x0000f201: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_RENDERMOBILEUIOOM\n"); break;
+        case 0x0000f300: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_DEVICEUPDATEUNSPECIFIED\n"); break;
+        case 0x0000f400: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_AUDIODRIVERHANG\n"); break;
+        case 0x0000f500: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_BATTERYPULLOUT\n"); break;
+        case 0x0000f600: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_MEDIACORETESTHANG\n"); break;
+        case 0x0000f700: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_RESOURCEMANAGEMENT\n"); break;
+        case 0x0000f800: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_CAPTURESERVICE\n"); break;
+        case 0x0000f900: cchUsed = RTStrPrintf(pszDetails, cbDetails, "SAVER_WAITFORSHELLREADY\n"); break;
+        case 0x00020001: cchUsed = RTStrPrintf(pszDetails, cbDetails, "HYPERVISOR_ERROR\n"); break;
+        case 0x4000008a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "THREAD_TERMINATE_HELD_MUTEX\n"); break;
+        case 0x400000ad: cchUsed = RTStrPrintf(pszDetails, cbDetails, "VIDEO_DRIVER_DEBUG_REPORT_REQUEST\n"); break;
+        case 0xc000021a: cchUsed = RTStrPrintf(pszDetails, cbDetails, "WINLOGON_FATAL_ERROR\n"); break;
+        case 0xdeaddead: cchUsed = RTStrPrintf(pszDetails, cbDetails, "MANUALLY_INITIATED_CRASH1\n"); break;
+        default: cchUsed = 0; break;
+    }
+    if (cchUsed < cbDetails)
+        return VINF_SUCCESS;
+    return VINF_BUFFER_OVERFLOW;
+}
+
+
+/**
+ * Report a bug check.
+ *
+ * @returns
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per virtual CPU structure.
+ * @param   enmEvent        The kind of BSOD event this is.
+ * @param   uBugCheck       The bug check number.
+ * @param   uP1             The bug check parameter \#1.
+ * @param   uP2             The bug check parameter \#2.
+ * @param   uP3             The bug check parameter \#3.
+ * @param   uP4             The bug check parameter \#4.
+ */
+VMMR3DECL(VBOXSTRICTRC) DBGFR3ReportBugCheck(PVM pVM, PVMCPU pVCpu, DBGFEVENTTYPE enmEvent, uint64_t uBugCheck,
+                                             uint64_t uP1, uint64_t uP2, uint64_t uP3, uint64_t uP4)
+{
+    /*
+     * Be careful.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_INVALID_VMCPU_HANDLE);
+    const char *pszSource;
+    switch (enmEvent)
+    {
+        case DBGFEVENT_BSOD_MSR:        pszSource = "GIMHv"; break;
+        case DBGFEVENT_BSOD_EFI:        pszSource = "EFI"; break;
+        case DBGFEVENT_BSOD_VMMDEV:     pszSource = "VMMDev"; break;
+        default:
+            AssertMsgFailedReturn(("enmEvent=%d\n", enmEvent), VERR_INVALID_PARAMETER);
+    }
+
+    /*
+     * Note it down.
+     */
+    pVM->dbgf.s.BugCheck.enmEvent        = enmEvent;
+    pVM->dbgf.s.BugCheck.uBugCheck       = uBugCheck;
+    pVM->dbgf.s.BugCheck.auParameters[0] = uP1;
+    pVM->dbgf.s.BugCheck.auParameters[1] = uP2;
+    pVM->dbgf.s.BugCheck.auParameters[2] = uP3;
+    pVM->dbgf.s.BugCheck.auParameters[3] = uP4;
+    pVM->dbgf.s.BugCheck.idCpu           = pVCpu->idCpu;
+    pVM->dbgf.s.BugCheck.uTimestamp      = TMVirtualGet(pVM);
+    pVM->dbgf.s.BugCheck.uResetNo        = VMGetResetCount(pVM);
+
+    /*
+     * Log the details.
+     */
+    char szDetails[2048];
+    DBGFR3FormatBugCheck(pVM->pUVM, szDetails, sizeof(szDetails), uBugCheck, uP1, uP2, uP3, uP4);
+    LogRel(("%s: %s", pszSource, szDetails));
+
+    /*
+     * Raise debugger event.
+     */
+    VBOXSTRICTRC rc = VINF_SUCCESS;
+    if (DBGF_IS_EVENT_ENABLED(pVM, enmEvent))
+        rc = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent, DBGFEVENTCTX_OTHER, 5 /*cArgs*/, uBugCheck, uP1, uP2, uP3, uP4);
+
+    /*
+     * Take actions.
+     */
+    /** @todo Take actions on BSOD, like notifying main or stopping the VM...
+     * For testing it makes little sense to continue after a BSOD. */
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNDBGFHANDLERINT, bugcheck}
+ */
+static DECLCALLBACK(void) dbgfR3BugCheckInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    char szDetails[2048];
+
+    /*
+     * Any arguments for bug check formatting?
+     */
+    if (pszArgs && *pszArgs)
+        pszArgs = RTStrStripL(pszArgs);
+    if (pszArgs && *pszArgs)
+    {
+        uint64_t auData[5] = { 0, 0, 0, 0, 0 };
+        unsigned iData = 0;
+        do
+        {
+            /* Find the next hex digit  */
+            char ch;
+            while ((ch = *pszArgs) != '\0' && !RT_C_IS_XDIGIT(ch))
+                pszArgs++;
+            if (ch == '\0')
+                break;
+
+            /* Extract the number. */
+            char *pszNext = (char *)pszArgs + 1;
+            RTStrToUInt64Ex(pszArgs, &pszNext, 16, &auData[iData]);
+
+            /* Advance. */
+            pszArgs = pszNext;
+            iData++;
+        } while (iData < RT_ELEMENTS(auData) && *pszArgs);
+
+        /* Format it. */
+        DBGFR3FormatBugCheck(pVM->pUVM, szDetails, sizeof(szDetails), auData[0], auData[1], auData[2], auData[3], auData[4]);
+        pHlp->pfnPrintf(pHlp, "%s", szDetails);
+    }
+    /*
+     * Format what's been reported (if any).
+     */
+    else if (pVM->dbgf.s.BugCheck.enmEvent != DBGFEVENT_END)
+    {
+        DBGFR3FormatBugCheck(pVM->pUVM, szDetails, sizeof(szDetails), pVM->dbgf.s.BugCheck.uBugCheck,
+                             pVM->dbgf.s.BugCheck.auParameters[0], pVM->dbgf.s.BugCheck.auParameters[1],
+                             pVM->dbgf.s.BugCheck.auParameters[2], pVM->dbgf.s.BugCheck.auParameters[3]);
+        const char *pszSource  = pVM->dbgf.s.BugCheck.enmEvent == DBGFEVENT_BSOD_MSR    ? "GIMHv"
+                               : pVM->dbgf.s.BugCheck.enmEvent == DBGFEVENT_BSOD_EFI    ? "EFI"
+                               : pVM->dbgf.s.BugCheck.enmEvent == DBGFEVENT_BSOD_VMMDEV ? "VMMDev" : "<unknown>";
+        uint32_t const uFreq   = TMVirtualGetFreq(pVM);
+        uint64_t const cSecs   = pVM->dbgf.s.BugCheck.uTimestamp / uFreq;
+        uint32_t const cMillis = (pVM->dbgf.s.BugCheck.uTimestamp - cSecs * uFreq) * 1000 / uFreq;
+        pHlp->pfnPrintf(pHlp, "BugCheck on CPU #%u after %RU64.%03u s VM uptime, %u resets ago (src: %s)\n%s",
+                        pVM->dbgf.s.BugCheck.idCpu, cSecs, cMillis,  VMGetResetCount(pVM) - pVM->dbgf.s.BugCheck.uResetNo,
+                        pszSource, szDetails);
+    }
+    else
+        pHlp->pfnPrintf(pHlp, "No bug check reported.\n");
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFR3Flow.cpp b/src/VBox/VMM/VMMR3/DBGFR3Flow.cpp
new file mode 100644
index 00000000..9365aebd
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFR3Flow.cpp
@@ -0,0 +1,2260 @@
+/* $Id: DBGFR3Flow.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Control Flow Graph Interface (CFG).
+ */
+
+/*
+ * Copyright (C) 2016-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_dbgf_cfg    DBGFR3Flow - Control Flow Graph Interface
+ *
+ * The control flow graph interface provides an API to disassemble
+ * guest code providing the result in a control flow graph.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/param.h>
+#include <iprt/list.h>
+#include <iprt/mem.h>
+#include <iprt/sort.h>
+#include <iprt/strcache.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+
+/**
+ * Internal control flow graph state.
+ */
+typedef struct DBGFFLOWINT
+{
+    /** Reference counter. */
+    uint32_t volatile       cRefs;
+    /** Internal reference counter for basic blocks. */
+    uint32_t volatile       cRefsBb;
+    /** Flags during creation. */
+    uint32_t                fFlags;
+    /** List of all basic blocks. */
+    RTLISTANCHOR            LstFlowBb;
+    /** List of identified branch tables. */
+    RTLISTANCHOR            LstBranchTbl;
+    /** Number of basic blocks in this control flow graph. */
+    uint32_t                cBbs;
+    /** Number of branch tables in this control flow graph. */
+    uint32_t                cBranchTbls;
+    /** The lowest addres of a basic block. */
+    DBGFADDRESS             AddrLowest;
+    /** The highest address of a basic block. */
+    DBGFADDRESS             AddrHighest;
+    /** String cache for disassembled instructions. */
+    RTSTRCACHE              hStrCacheInstr;
+} DBGFFLOWINT;
+/** Pointer to an internal control flow graph state. */
+typedef DBGFFLOWINT *PDBGFFLOWINT;
+
+/**
+ * Instruction record
+ */
+typedef struct DBGFFLOWBBINSTR
+{
+    /** Instruction address. */
+    DBGFADDRESS             AddrInstr;
+    /** Size of instruction. */
+    uint32_t                cbInstr;
+    /** Disassembled instruction string. */
+    const char              *pszInstr;
+} DBGFFLOWBBINSTR;
+/** Pointer to an instruction record. */
+typedef DBGFFLOWBBINSTR *PDBGFFLOWBBINSTR;
+
+
+/**
+ * A branch table identified by the graph processor.
+ */
+typedef struct DBGFFLOWBRANCHTBLINT
+{
+    /** Node for the list of branch tables. */
+    RTLISTNODE              NdBranchTbl;
+    /** The owning control flow graph. */
+    PDBGFFLOWINT            pFlow;
+    /** Reference counter. */
+    uint32_t volatile       cRefs;
+    /** The general register index holding the bracnh table base. */
+    uint8_t                 idxGenRegBase;
+    /** Start address of the branch table. */
+    DBGFADDRESS             AddrStart;
+    /** Number of valid entries in the branch table. */
+    uint32_t                cSlots;
+    /** The addresses contained in the branch table - variable in size. */
+    DBGFADDRESS             aAddresses[1];
+} DBGFFLOWBRANCHTBLINT;
+/** Pointer to a branch table structure. */
+typedef DBGFFLOWBRANCHTBLINT *PDBGFFLOWBRANCHTBLINT;
+
+
+/**
+ * Internal control flow graph basic block state.
+ */
+typedef struct DBGFFLOWBBINT
+{
+    /** Node for the list of all basic blocks. */
+    RTLISTNODE               NdFlowBb;
+    /** The control flow graph the basic block belongs to. */
+    PDBGFFLOWINT             pFlow;
+    /** Reference counter. */
+    uint32_t volatile        cRefs;
+    /** Basic block end type. */
+    DBGFFLOWBBENDTYPE        enmEndType;
+    /** Start address of this basic block. */
+    DBGFADDRESS              AddrStart;
+    /** End address of this basic block. */
+    DBGFADDRESS              AddrEnd;
+    /** Address of the block succeeding.
+     *  This is valid for conditional jumps
+     * (the other target is referenced by AddrEnd+1) and
+     * unconditional jumps (not ret, iret, etc.) except
+     * if we can't infer the jump target (jmp *eax for example). */
+    DBGFADDRESS              AddrTarget;
+    /** The indirect branch table identified for indirect branches. */
+    PDBGFFLOWBRANCHTBLINT    pFlowBranchTbl;
+    /** Last status error code if DBGF_FLOW_BB_F_INCOMPLETE_ERR is set. */
+    int                      rcError;
+    /** Error message if DBGF_FLOW_BB_F_INCOMPLETE_ERR is set. */
+    char                     *pszErr;
+    /** Flags for this basic block. */
+    uint32_t                 fFlags;
+    /** Number of instructions in this basic block. */
+    uint32_t                 cInstr;
+    /** Maximum number of instruction records for this basic block. */
+    uint32_t                 cInstrMax;
+    /** Instruction records, variable in size. */
+    DBGFFLOWBBINSTR          aInstr[1];
+} DBGFFLOWBBINT;
+/** Pointer to an internal control flow graph basic block state. */
+typedef DBGFFLOWBBINT *PDBGFFLOWBBINT;
+
+
+/**
+ * Control flow graph iterator state.
+ */
+typedef struct DBGFFLOWITINT
+{
+    /** Pointer to the control flow graph (holding a reference). */
+    PDBGFFLOWINT             pFlow;
+    /** Next basic block to return. */
+    uint32_t                 idxBbNext;
+    /** Array of basic blocks sorted by the specified order - variable in size. */
+    PDBGFFLOWBBINT           apBb[1];
+} DBGFFLOWITINT;
+/** Pointer to the internal control flow graph iterator state. */
+typedef DBGFFLOWITINT *PDBGFFLOWITINT;
+
+
+/**
+ * Control flow graph branch table iterator state.
+ */
+typedef struct DBGFFLOWBRANCHTBLITINT
+{
+    /** Pointer to the control flow graph (holding a reference). */
+    PDBGFFLOWINT             pFlow;
+    /** Next branch table to return. */
+    uint32_t                 idxTblNext;
+    /** Array of branch table pointers sorted by the specified order - variable in size. */
+    PDBGFFLOWBRANCHTBLINT    apBranchTbl[1];
+} DBGFFLOWBRANCHTBLITINT;
+/** Pointer to the internal control flow graph branch table iterator state. */
+typedef DBGFFLOWBRANCHTBLITINT *PDBGFFLOWBRANCHTBLITINT;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+
+static uint32_t dbgfR3FlowBbReleaseInt(PDBGFFLOWBBINT pFlowBb, bool fMayDestroyFlow);
+static void dbgfR3FlowBranchTblDestroy(PDBGFFLOWBRANCHTBLINT pFlowBranchTbl);
+
+
+/**
+ * Checks whether both addresses are equal.
+ *
+ * @returns true if both addresses point to the same location, false otherwise.
+ * @param   pAddr1              First address.
+ * @param   pAddr2              Second address.
+ */
+static bool dbgfR3FlowAddrEqual(PDBGFADDRESS pAddr1, PDBGFADDRESS pAddr2)
+{
+    return    pAddr1->Sel == pAddr2->Sel
+           && pAddr1->off == pAddr2->off;
+}
+
+
+/**
+ * Checks whether the first given address is lower than the second one.
+ *
+ * @returns true if both addresses point to the same location, false otherwise.
+ * @param   pAddr1              First address.
+ * @param   pAddr2              Second address.
+ */
+static bool dbgfR3FlowAddrLower(PDBGFADDRESS pAddr1, PDBGFADDRESS pAddr2)
+{
+    return    pAddr1->Sel == pAddr2->Sel
+           && pAddr1->off < pAddr2->off;
+}
+
+
+/**
+ * Checks whether the given basic block and address intersect.
+ *
+ * @returns true if they intersect, false otherwise.
+ * @param   pFlowBb             The basic block to check.
+ * @param   pAddr               The address to check for.
+ */
+static bool dbgfR3FlowAddrIntersect(PDBGFFLOWBBINT pFlowBb, PDBGFADDRESS pAddr)
+{
+    return    (pFlowBb->AddrStart.Sel == pAddr->Sel)
+           && (pFlowBb->AddrStart.off <= pAddr->off)
+           && (pFlowBb->AddrEnd.off >= pAddr->off);
+}
+
+
+/**
+ * Returns the distance of the two given addresses.
+ *
+ * @returns Distance of the addresses.
+ * @param   pAddr1              The first address.
+ * @param   pAddr2              The second address.
+ */
+static RTGCUINTPTR dbgfR3FlowAddrGetDistance(PDBGFADDRESS pAddr1, PDBGFADDRESS pAddr2)
+{
+    if (pAddr1->Sel == pAddr2->Sel)
+    {
+        if (pAddr1->off >= pAddr2->off)
+            return pAddr1->off - pAddr2->off;
+        else
+            return pAddr2->off - pAddr1->off;
+    }
+    else
+        AssertFailed();
+
+    return 0;
+}
+
+
+/**
+ * Creates a new basic block.
+ *
+ * @returns Pointer to the basic block on success or NULL if out of memory.
+ * @param   pThis               The control flow graph.
+ * @param   pAddrStart          The start of the basic block.
+ * @param   fFlowBbFlags        Additional flags for this bascic block.
+ * @param   cInstrMax           Maximum number of instructions this block can hold initially.
+ */
+static PDBGFFLOWBBINT dbgfR3FlowBbCreate(PDBGFFLOWINT pThis, PDBGFADDRESS pAddrStart, uint32_t fFlowBbFlags,
+                                         uint32_t cInstrMax)
+{
+    PDBGFFLOWBBINT pFlowBb = (PDBGFFLOWBBINT)RTMemAllocZ(RT_UOFFSETOF_DYN(DBGFFLOWBBINT, aInstr[cInstrMax]));
+    if (RT_LIKELY(pFlowBb))
+    {
+        RTListInit(&pFlowBb->NdFlowBb);
+        pFlowBb->cRefs          = 1;
+        pFlowBb->enmEndType     = DBGFFLOWBBENDTYPE_INVALID;
+        pFlowBb->pFlow          = pThis;
+        pFlowBb->fFlags         = DBGF_FLOW_BB_F_EMPTY | fFlowBbFlags;
+        pFlowBb->AddrStart      = *pAddrStart;
+        pFlowBb->AddrEnd        = *pAddrStart;
+        pFlowBb->rcError        = VINF_SUCCESS;
+        pFlowBb->pszErr         = NULL;
+        pFlowBb->cInstr         = 0;
+        pFlowBb->cInstrMax      = cInstrMax;
+        pFlowBb->pFlowBranchTbl = NULL;
+        ASMAtomicIncU32(&pThis->cRefsBb);
+    }
+
+    return pFlowBb;
+}
+
+
+/**
+ * Creates an empty branch table with the given size.
+ *
+ * @returns Pointer to the empty branch table on success or NULL if out of memory.
+ * @param   pThis               The control flow graph.
+ * @param   pAddrStart          The start of the branch table.
+ * @param   idxGenRegBase       The general register index holding the base address.
+ * @param   cSlots              Number of slots the table has.
+ */
+static PDBGFFLOWBRANCHTBLINT
+dbgfR3FlowBranchTblCreate(PDBGFFLOWINT pThis, PDBGFADDRESS pAddrStart, uint8_t idxGenRegBase,  uint32_t cSlots)
+{
+    PDBGFFLOWBRANCHTBLINT pBranchTbl = (PDBGFFLOWBRANCHTBLINT)RTMemAllocZ(RT_UOFFSETOF_DYN(DBGFFLOWBRANCHTBLINT,
+                                                                                           aAddresses[cSlots]));
+    if (RT_LIKELY(pBranchTbl))
+    {
+        RTListInit(&pBranchTbl->NdBranchTbl);
+        pBranchTbl->pFlow         = pThis;
+        pBranchTbl->idxGenRegBase = idxGenRegBase;
+        pBranchTbl->AddrStart     = *pAddrStart;
+        pBranchTbl->cSlots        = cSlots;
+        pBranchTbl->cRefs         = 1;
+    }
+
+    return pBranchTbl;
+}
+
+
+/**
+ * Destroys a control flow graph.
+ *
+ * @returns nothing.
+ * @param   pThis               The control flow graph to destroy.
+ */
+static void dbgfR3FlowDestroy(PDBGFFLOWINT pThis)
+{
+    /* Defer destruction if there are still basic blocks referencing us. */
+    PDBGFFLOWBBINT pFlowBb;
+    PDBGFFLOWBBINT pFlowBbNext;
+    RTListForEachSafe(&pThis->LstFlowBb, pFlowBb, pFlowBbNext, DBGFFLOWBBINT, NdFlowBb)
+    {
+        dbgfR3FlowBbReleaseInt(pFlowBb, false /*fMayDestroyFlow*/);
+    }
+
+    Assert(!pThis->cRefs);
+    if (!pThis->cRefsBb)
+    {
+        /* Destroy the branch tables. */
+        PDBGFFLOWBRANCHTBLINT pTbl = NULL;
+        PDBGFFLOWBRANCHTBLINT pTblNext = NULL;
+        RTListForEachSafe(&pThis->LstBranchTbl, pTbl, pTblNext, DBGFFLOWBRANCHTBLINT, NdBranchTbl)
+        {
+            dbgfR3FlowBranchTblDestroy(pTbl);
+        }
+
+        RTStrCacheDestroy(pThis->hStrCacheInstr);
+        RTMemFree(pThis);
+    }
+}
+
+
+/**
+ * Destroys a basic block.
+ *
+ * @returns nothing.
+ * @param   pFlowBb              The basic block to destroy.
+ * @param   fMayDestroyFlow      Flag whether the control flow graph container
+ *                               should be destroyed when there is nothing referencing it.
+ */
+static void dbgfR3FlowBbDestroy(PDBGFFLOWBBINT pFlowBb, bool fMayDestroyFlow)
+{
+    PDBGFFLOWINT pThis = pFlowBb->pFlow;
+
+    RTListNodeRemove(&pFlowBb->NdFlowBb);
+    pThis->cBbs--;
+    for (uint32_t idxInstr = 0; idxInstr < pFlowBb->cInstr; idxInstr++)
+        RTStrCacheRelease(pThis->hStrCacheInstr, pFlowBb->aInstr[idxInstr].pszInstr);
+    uint32_t cRefsBb = ASMAtomicDecU32(&pThis->cRefsBb);
+    RTMemFree(pFlowBb);
+
+    if (!cRefsBb && !pThis->cRefs && fMayDestroyFlow)
+        dbgfR3FlowDestroy(pThis);
+}
+
+
+/**
+ * Destroys a given branch table.
+ *
+ * @returns nothing.
+ * @param   pFlowBranchTbl      The flow branch table to destroy.
+ */
+static void dbgfR3FlowBranchTblDestroy(PDBGFFLOWBRANCHTBLINT pFlowBranchTbl)
+{
+    RTListNodeRemove(&pFlowBranchTbl->NdBranchTbl);
+    RTMemFree(pFlowBranchTbl);
+}
+
+
+/**
+ * Internal basic block release worker.
+ *
+ * @returns New reference count of the released basic block, on 0
+ *          it is destroyed.
+ * @param   pFlowBb              The basic block to release.
+ * @param   fMayDestroyFlow      Flag whether the control flow graph container
+ *                               should be destroyed when there is nothing referencing it.
+ */
+static uint32_t dbgfR3FlowBbReleaseInt(PDBGFFLOWBBINT pFlowBb, bool fMayDestroyFlow)
+{
+    uint32_t cRefs = ASMAtomicDecU32(&pFlowBb->cRefs);
+    AssertMsg(cRefs < _1M, ("%#x %p %d\n", cRefs, pFlowBb, pFlowBb->enmEndType));
+    if (cRefs == 0)
+        dbgfR3FlowBbDestroy(pFlowBb, fMayDestroyFlow);
+    return cRefs;
+}
+
+
+/**
+ * Links the given basic block into the control flow graph.
+ *
+ * @returns nothing.
+ * @param   pThis               The control flow graph to link into.
+ * @param   pFlowBb             The basic block to link.
+ */
+DECLINLINE(void) dbgfR3FlowLink(PDBGFFLOWINT pThis, PDBGFFLOWBBINT pFlowBb)
+{
+    RTListAppend(&pThis->LstFlowBb, &pFlowBb->NdFlowBb);
+    pThis->cBbs++;
+}
+
+
+/**
+ * Links the given branch table into the control flow graph.
+ *
+ * @returns nothing.
+ * @param   pThis               The control flow graph to link into.
+ * @param   pBranchTbl          The branch table to link.
+ */
+DECLINLINE(void) dbgfR3FlowBranchTblLink(PDBGFFLOWINT pThis, PDBGFFLOWBRANCHTBLINT pBranchTbl)
+{
+    RTListAppend(&pThis->LstBranchTbl, &pBranchTbl->NdBranchTbl);
+    pThis->cBranchTbls++;
+}
+
+
+/**
+ * Returns the first unpopulated basic block of the given control flow graph.
+ *
+ * @returns The first unpopulated control flow graph or NULL if not found.
+ * @param   pThis               The control flow graph.
+ */
+DECLINLINE(PDBGFFLOWBBINT) dbgfR3FlowGetUnpopulatedBb(PDBGFFLOWINT pThis)
+{
+    PDBGFFLOWBBINT pFlowBb;
+    RTListForEach(&pThis->LstFlowBb, pFlowBb, DBGFFLOWBBINT, NdFlowBb)
+    {
+        if (pFlowBb->fFlags & DBGF_FLOW_BB_F_EMPTY)
+            return pFlowBb;
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Returns the branch table with the given address if it exists.
+ *
+ * @returns Pointer to the branch table record or NULL if not found.
+ * @param   pThis               The control flow graph.
+ * @param   pAddrTbl            The branch table address.
+ */
+DECLINLINE(PDBGFFLOWBRANCHTBLINT) dbgfR3FlowBranchTblFindByAddr(PDBGFFLOWINT pThis, PDBGFADDRESS pAddrTbl)
+{
+    PDBGFFLOWBRANCHTBLINT pTbl;
+    RTListForEach(&pThis->LstBranchTbl, pTbl, DBGFFLOWBRANCHTBLINT, NdBranchTbl)
+    {
+        if (dbgfR3FlowAddrEqual(&pTbl->AddrStart, pAddrTbl))
+            return pTbl;
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Sets the given error status for the basic block.
+ *
+ * @returns nothing.
+ * @param   pFlowBb              The basic block causing the error.
+ * @param   rcError             The error to set.
+ * @param   pszFmt              Format string of the error description.
+ * @param   ...                 Arguments for the format string.
+ */
+static void dbgfR3FlowBbSetError(PDBGFFLOWBBINT pFlowBb, int rcError, const char *pszFmt, ...)
+{
+    va_list va;
+    va_start(va, pszFmt);
+
+    Assert(!(pFlowBb->fFlags & DBGF_FLOW_BB_F_INCOMPLETE_ERR));
+    pFlowBb->fFlags |= DBGF_FLOW_BB_F_INCOMPLETE_ERR;
+    pFlowBb->fFlags &= ~DBGF_FLOW_BB_F_EMPTY;
+    pFlowBb->rcError = rcError;
+    pFlowBb->pszErr = RTStrAPrintf2V(pszFmt, va);
+    va_end(va);
+}
+
+
+/**
+ * Checks whether the given control flow graph contains a basic block
+ * with the given start address.
+ *
+ * @returns true if there is a basic block with the start address, false otherwise.
+ * @param   pThis               The control flow graph.
+ * @param   pAddr               The address to check for.
+ */
+static bool dbgfR3FlowHasBbWithStartAddr(PDBGFFLOWINT pThis, PDBGFADDRESS pAddr)
+{
+    PDBGFFLOWBBINT pFlowBb;
+    RTListForEach(&pThis->LstFlowBb, pFlowBb, DBGFFLOWBBINT, NdFlowBb)
+    {
+        if (dbgfR3FlowAddrEqual(&pFlowBb->AddrStart, pAddr))
+            return true;
+    }
+    return false;
+}
+
+
+/**
+ * Splits a given basic block into two at the given address.
+ *
+ * @returns VBox status code.
+ * @param   pThis               The control flow graph.
+ * @param   pFlowBb              The basic block to split.
+ * @param   pAddr               The address to split at.
+ */
+static int dbgfR3FlowBbSplit(PDBGFFLOWINT pThis, PDBGFFLOWBBINT pFlowBb, PDBGFADDRESS pAddr)
+{
+    int rc = VINF_SUCCESS;
+    uint32_t idxInstrSplit;
+
+    /* If the block is empty it will get populated later so there is nothing to split,
+     * same if the start address equals. */
+    if (   pFlowBb->fFlags & DBGF_FLOW_BB_F_EMPTY
+        || dbgfR3FlowAddrEqual(&pFlowBb->AddrStart, pAddr))
+        return VINF_SUCCESS;
+
+    /* Find the instruction to split at. */
+    for (idxInstrSplit = 1; idxInstrSplit < pFlowBb->cInstr; idxInstrSplit++)
+        if (dbgfR3FlowAddrEqual(&pFlowBb->aInstr[idxInstrSplit].AddrInstr, pAddr))
+            break;
+
+    Assert(idxInstrSplit > 0);
+
+    /*
+     * Given address might not be on instruction boundary, this is not supported
+     * so far and results in an error.
+     */
+    if (idxInstrSplit < pFlowBb->cInstr)
+    {
+        /* Create new basic block. */
+        uint32_t cInstrNew = pFlowBb->cInstr - idxInstrSplit;
+        PDBGFFLOWBBINT pFlowBbNew = dbgfR3FlowBbCreate(pThis, &pFlowBb->aInstr[idxInstrSplit].AddrInstr,
+                                                       0 /*fFlowBbFlags*/, cInstrNew);
+        if (pFlowBbNew)
+        {
+            /* Move instructions over. */
+            pFlowBbNew->cInstr         = cInstrNew;
+            pFlowBbNew->AddrEnd        = pFlowBb->AddrEnd;
+            pFlowBbNew->enmEndType     = pFlowBb->enmEndType;
+            pFlowBbNew->AddrTarget     = pFlowBb->AddrTarget;
+            pFlowBbNew->fFlags         = pFlowBb->fFlags & ~DBGF_FLOW_BB_F_ENTRY;
+            pFlowBbNew->pFlowBranchTbl = pFlowBb->pFlowBranchTbl;
+            pFlowBb->pFlowBranchTbl    = NULL;
+
+            /* Move any error to the new basic block and clear them in the old basic block. */
+            pFlowBbNew->rcError    = pFlowBb->rcError;
+            pFlowBbNew->pszErr     = pFlowBb->pszErr;
+            pFlowBb->rcError       = VINF_SUCCESS;
+            pFlowBb->pszErr        = NULL;
+            pFlowBb->fFlags       &= ~DBGF_FLOW_BB_F_INCOMPLETE_ERR;
+
+            memcpy(&pFlowBbNew->aInstr[0], &pFlowBb->aInstr[idxInstrSplit], cInstrNew * sizeof(DBGFFLOWBBINSTR));
+            pFlowBb->cInstr     = idxInstrSplit;
+            pFlowBb->enmEndType = DBGFFLOWBBENDTYPE_UNCOND;
+            pFlowBb->AddrEnd    = pFlowBb->aInstr[idxInstrSplit-1].AddrInstr;
+            pFlowBb->AddrTarget = pFlowBbNew->AddrStart;
+            DBGFR3AddrAdd(&pFlowBb->AddrEnd, pFlowBb->aInstr[idxInstrSplit-1].cbInstr - 1);
+            RT_BZERO(&pFlowBb->aInstr[idxInstrSplit], cInstrNew * sizeof(DBGFFLOWBBINSTR));
+
+            dbgfR3FlowLink(pThis, pFlowBbNew);
+        }
+        else
+            rc = VERR_NO_MEMORY;
+    }
+    else
+        AssertFailedStmt(rc = VERR_INVALID_STATE); /** @todo Proper status code. */
+
+    return rc;
+}
+
+
+/**
+ * Makes sure there is an successor at the given address splitting already existing
+ * basic blocks if they intersect.
+ *
+ * @returns VBox status code.
+ * @param   pThis               The control flow graph.
+ * @param   pAddrSucc           The guest address the new successor should start at.
+ * @param   fNewBbFlags         Flags for the new basic block.
+ * @param   pBranchTbl          Branch table candidate for this basic block.
+ */
+static int dbgfR3FlowBbSuccessorAdd(PDBGFFLOWINT pThis, PDBGFADDRESS pAddrSucc,
+                                    uint32_t fNewBbFlags, PDBGFFLOWBRANCHTBLINT pBranchTbl)
+{
+    PDBGFFLOWBBINT pFlowBb;
+    RTListForEach(&pThis->LstFlowBb, pFlowBb, DBGFFLOWBBINT, NdFlowBb)
+    {
+        /*
+         * The basic block must be split if it intersects with the given address
+         * and the start address does not equal the given one.
+         */
+        if (dbgfR3FlowAddrIntersect(pFlowBb, pAddrSucc))
+            return dbgfR3FlowBbSplit(pThis, pFlowBb, pAddrSucc);
+    }
+
+    int rc = VINF_SUCCESS;
+    pFlowBb = dbgfR3FlowBbCreate(pThis, pAddrSucc, fNewBbFlags, 10);
+    if (pFlowBb)
+    {
+        pFlowBb->pFlowBranchTbl = pBranchTbl;
+        dbgfR3FlowLink(pThis, pFlowBb);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    return rc;
+}
+
+
+/**
+ * Returns whether the parameter indicates an indirect branch.
+ *
+ * @returns Flag whether this is an indirect branch.
+ * @param   pDisParam           The parameter from the disassembler.
+ */
+DECLINLINE(bool) dbgfR3FlowBranchTargetIsIndirect(PDISOPPARAM pDisParam)
+{
+    bool fIndirect = true;
+
+    if (   pDisParam->fUse & (DISUSE_IMMEDIATE8 | DISUSE_IMMEDIATE16 | DISUSE_IMMEDIATE32 | DISUSE_IMMEDIATE64)
+        || pDisParam->fUse & (DISUSE_IMMEDIATE8_REL | DISUSE_IMMEDIATE16_REL | DISUSE_IMMEDIATE32_REL | DISUSE_IMMEDIATE64_REL))
+        fIndirect = false;
+
+    return fIndirect;
+}
+
+
+/**
+ * Resolves the direct branch target address if possible from the given instruction address
+ * and instruction parameter.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The usermode VM handle.
+ * @param   idCpu               CPU id for resolving the address.
+ * @param   pDisParam           The parameter from the disassembler.
+ * @param   pAddrInstr          The instruction address.
+ * @param   cbInstr             Size of instruction in bytes.
+ * @param   fRelJmp             Flag whether this is a reltive jump.
+ * @param   pAddrJmpTarget      Where to store the address to the jump target on success.
+ */
+static int dbgfR3FlowQueryDirectBranchTarget(PUVM pUVM, VMCPUID idCpu, PDISOPPARAM pDisParam, PDBGFADDRESS pAddrInstr,
+                                             uint32_t cbInstr, bool fRelJmp, PDBGFADDRESS pAddrJmpTarget)
+{
+    int rc = VINF_SUCCESS;
+
+    Assert(!dbgfR3FlowBranchTargetIsIndirect(pDisParam));
+
+    /* Relative jumps are always from the beginning of the next instruction. */
+    *pAddrJmpTarget = *pAddrInstr;
+    DBGFR3AddrAdd(pAddrJmpTarget, cbInstr);
+
+    if (fRelJmp)
+    {
+        RTGCINTPTR iRel = 0;
+        if (pDisParam->fUse & DISUSE_IMMEDIATE8_REL)
+            iRel = (int8_t)pDisParam->uValue;
+        else if (pDisParam->fUse & DISUSE_IMMEDIATE16_REL)
+            iRel = (int16_t)pDisParam->uValue;
+        else if (pDisParam->fUse & DISUSE_IMMEDIATE32_REL)
+            iRel = (int32_t)pDisParam->uValue;
+        else if (pDisParam->fUse & DISUSE_IMMEDIATE64_REL)
+            iRel = (int64_t)pDisParam->uValue;
+        else
+            AssertFailedStmt(rc = VERR_NOT_SUPPORTED);
+
+        if (iRel < 0)
+            DBGFR3AddrSub(pAddrJmpTarget, -iRel);
+        else
+            DBGFR3AddrAdd(pAddrJmpTarget, iRel);
+    }
+    else
+    {
+        if (pDisParam->fUse & (DISUSE_IMMEDIATE8 | DISUSE_IMMEDIATE16 | DISUSE_IMMEDIATE32 | DISUSE_IMMEDIATE64))
+        {
+            if (DBGFADDRESS_IS_FLAT(pAddrInstr))
+                DBGFR3AddrFromFlat(pUVM, pAddrJmpTarget, pDisParam->uValue);
+            else
+                DBGFR3AddrFromSelOff(pUVM, idCpu, pAddrJmpTarget, pAddrInstr->Sel, pDisParam->uValue);
+        }
+        else
+            AssertFailedStmt(rc = VERR_INVALID_STATE);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Returns the CPU mode based on the given assembler flags.
+ *
+ * @returns CPU mode.
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               CPU id for disassembling.
+ * @param   fFlagsDisasm        The flags used for disassembling.
+ */
+static CPUMMODE dbgfR3FlowGetDisasCpuMode(PUVM pUVM, VMCPUID idCpu, uint32_t fFlagsDisasm)
+{
+    CPUMMODE enmMode = CPUMMODE_INVALID;
+    uint32_t fDisasMode = fFlagsDisasm & DBGF_DISAS_FLAGS_MODE_MASK;
+    if (fDisasMode == DBGF_DISAS_FLAGS_DEFAULT_MODE)
+        enmMode = DBGFR3CpuGetMode(pUVM, idCpu);
+    else if (   fDisasMode == DBGF_DISAS_FLAGS_16BIT_MODE
+             || fDisasMode == DBGF_DISAS_FLAGS_16BIT_REAL_MODE)
+        enmMode = CPUMMODE_REAL;
+    else if (fDisasMode == DBGF_DISAS_FLAGS_32BIT_MODE)
+        enmMode = CPUMMODE_PROTECTED;
+    else if (fDisasMode == DBGF_DISAS_FLAGS_64BIT_MODE)
+        enmMode = CPUMMODE_LONG;
+    else
+        AssertFailed();
+
+    return enmMode;
+}
+
+
+/**
+ * Searches backwards in the given basic block starting the given instruction index for
+ * a mov instruction with the given register as the target where the constant looks like
+ * a pointer.
+ *
+ * @returns Flag whether a candidate was found.
+ * @param   pFlowBb             The basic block containing the indirect branch.
+ * @param   idxRegTgt           The general register the mov targets.
+ * @param   cbPtr               The pointer size to look for.
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               CPU id for disassembling.
+ * @param   fFlagsDisasm        The flags to use for disassembling.
+ * @param   pidxInstrStart      The instruction index to start searching for on input,
+ *                              The last instruction evaluated on output.
+ * @param   pAddrDest           Where to store the candidate address on success.
+ */
+static bool dbgfR3FlowSearchMovWithConstantPtrSizeBackwards(PDBGFFLOWBBINT pFlowBb, uint8_t idxRegTgt, uint32_t cbPtr,
+                                                            PUVM pUVM, VMCPUID idCpu, uint32_t fFlagsDisasm,
+                                                            uint32_t *pidxInstrStart, PDBGFADDRESS pAddrDest)
+{
+    bool fFound = false;
+    uint32_t idxInstrCur = *pidxInstrStart;
+    uint32_t cInstrCheck = idxInstrCur + 1;
+
+    for (;;)
+    {
+        /** @todo Avoid to disassemble again. */
+        PDBGFFLOWBBINSTR pInstr = &pFlowBb->aInstr[idxInstrCur];
+        DBGFDISSTATE DisState;
+        char szOutput[_4K];
+
+        int rc = dbgfR3DisasInstrStateEx(pUVM, idCpu, &pInstr->AddrInstr, fFlagsDisasm,
+                                         &szOutput[0], sizeof(szOutput), &DisState);
+        if (RT_SUCCESS(rc))
+        {
+            if (   DisState.pCurInstr->uOpcode == OP_MOV
+                && (DisState.Param1.fUse & (DISUSE_REG_GEN16 | DISUSE_REG_GEN32 | DISUSE_REG_GEN64))
+                && DisState.Param1.Base.idxGenReg == idxRegTgt
+                /*&& DisState.Param1.cb == cbPtr*/
+                && DisState.Param2.cb == cbPtr
+                && (DisState.Param2.fUse & (DISUSE_IMMEDIATE16 | DISUSE_IMMEDIATE32 | DISUSE_IMMEDIATE64)))
+            {
+                /* Found possible candidate. */
+                fFound = true;
+                if (DBGFADDRESS_IS_FLAT(&pInstr->AddrInstr))
+                    DBGFR3AddrFromFlat(pUVM, pAddrDest, DisState.Param2.uValue);
+                else
+                    DBGFR3AddrFromSelOff(pUVM, idCpu, pAddrDest, pInstr->AddrInstr.Sel, DisState.Param2.uValue);
+                break;
+            }
+        }
+        else
+            break;
+
+        cInstrCheck--;
+        if (!cInstrCheck)
+            break;
+
+        idxInstrCur--;
+    }
+
+    *pidxInstrStart = idxInstrCur;
+    return fFound;
+}
+
+
+/**
+ * Verifies the given branch table candidate and adds it to the control flow graph on success.
+ *
+ * @returns VBox status code.
+ * @param   pThis               The flow control graph.
+ * @param   pFlowBb             The basic block causing the indirect branch.
+ * @param   pAddrBranchTbl      Address of the branch table location.
+ * @param   idxGenRegBase       The general register holding the base address.
+ * @param   cbPtr               Guest pointer size.
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               CPU id for disassembling.
+ *
+ * @todo Handle branch tables greater than 4KB (lazy coder).
+ */
+static int dbgfR3FlowBranchTblVerifyAdd(PDBGFFLOWINT pThis, PDBGFFLOWBBINT pFlowBb, PDBGFADDRESS pAddrBranchTbl,
+                                        uint8_t idxGenRegBase, uint32_t cbPtr, PUVM pUVM, VMCPUID idCpu)
+{
+    int rc = VINF_SUCCESS;
+    PDBGFFLOWBRANCHTBLINT pBranchTbl = dbgfR3FlowBranchTblFindByAddr(pThis, pAddrBranchTbl);
+
+    if (!pBranchTbl)
+    {
+        uint32_t cSlots = 0;
+        uint8_t abBuf[_4K];
+
+        rc = DBGFR3MemRead(pUVM, idCpu, pAddrBranchTbl, &abBuf[0], sizeof(abBuf));
+        if (RT_SUCCESS(rc))
+        {
+            uint8_t *pbBuf = &abBuf[0];
+            while (pbBuf < &abBuf[0] + sizeof(abBuf))
+            {
+                DBGFADDRESS AddrDest;
+                RTGCUINTPTR GCPtr =   cbPtr == sizeof(uint64_t)
+                                    ? *(uint64_t *)pbBuf
+                                    : cbPtr == sizeof(uint32_t)
+                                    ? *(uint32_t *)pbBuf
+                                    : *(uint16_t *)pbBuf;
+                pbBuf += cbPtr;
+
+                if (DBGFADDRESS_IS_FLAT(pAddrBranchTbl))
+                    DBGFR3AddrFromFlat(pUVM, &AddrDest, GCPtr);
+                else
+                    DBGFR3AddrFromSelOff(pUVM, idCpu, &AddrDest, pAddrBranchTbl->Sel, GCPtr);
+
+                if (dbgfR3FlowAddrGetDistance(&AddrDest, &pFlowBb->AddrEnd) > _512K)
+                    break;
+
+                cSlots++;
+            }
+
+            /* If there are any slots use it. */
+            if (cSlots)
+            {
+                pBranchTbl = dbgfR3FlowBranchTblCreate(pThis, pAddrBranchTbl, idxGenRegBase, cSlots);
+                if (pBranchTbl)
+                {
+                    /* Get the addresses. */
+                    for (unsigned i = 0; i < cSlots && RT_SUCCESS(rc); i++)
+                    {
+                        RTGCUINTPTR GCPtr =   cbPtr == sizeof(uint64_t)
+                                            ? *(uint64_t *)&abBuf[i * cbPtr]
+                                            : cbPtr == sizeof(uint32_t)
+                                            ? *(uint32_t *)&abBuf[i * cbPtr]
+                                            : *(uint16_t *)&abBuf[i * cbPtr];
+
+                        if (DBGFADDRESS_IS_FLAT(pAddrBranchTbl))
+                            DBGFR3AddrFromFlat(pUVM, &pBranchTbl->aAddresses[i], GCPtr);
+                        else
+                            DBGFR3AddrFromSelOff(pUVM, idCpu, &pBranchTbl->aAddresses[i],
+                                                 pAddrBranchTbl->Sel, GCPtr);
+                        rc = dbgfR3FlowBbSuccessorAdd(pThis, &pBranchTbl->aAddresses[i], DBGF_FLOW_BB_F_BRANCH_TABLE,
+                                                      pBranchTbl);
+                    }
+                    dbgfR3FlowBranchTblLink(pThis, pBranchTbl);
+                }
+                else
+                    rc = VERR_NO_MEMORY;
+            }
+        }
+    }
+
+    if (pBranchTbl)
+        pFlowBb->pFlowBranchTbl = pBranchTbl;
+
+    return rc;
+}
+
+
+/**
+ * Checks whether the location for the branch target candidate contains a valid code address.
+ *
+ * @returns VBox status code.
+ * @param   pThis               The flow control graph.
+ * @param   pFlowBb             The basic block causing the indirect branch.
+ * @param   pAddrBranchTgt      Address of the branch target location.
+ * @param   idxGenRegBase       The general register holding the address of the location.
+ * @param   cbPtr               Guest pointer size.
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               CPU id for disassembling.
+ * @param   fBranchTbl          Flag whether this is a possible branch table containing multiple
+ *                              targets.
+ */
+static int dbgfR3FlowCheckBranchTargetLocation(PDBGFFLOWINT pThis, PDBGFFLOWBBINT pFlowBb, PDBGFADDRESS pAddrBranchTgt,
+                                               uint8_t idxGenRegBase, uint32_t cbPtr, PUVM pUVM, VMCPUID idCpu, bool fBranchTbl)
+{
+    int rc = VINF_SUCCESS;
+
+    if (!fBranchTbl)
+    {
+        union { uint16_t u16Val; uint32_t u32Val; uint64_t u64Val; } uVal;
+        rc = DBGFR3MemRead(pUVM, idCpu, pAddrBranchTgt, &uVal, cbPtr);
+        if (RT_SUCCESS(rc))
+        {
+            DBGFADDRESS AddrTgt;
+            RTGCUINTPTR GCPtr =   cbPtr == sizeof(uint64_t)
+                                ? uVal.u64Val
+                                : cbPtr == sizeof(uint32_t)
+                                ? uVal.u32Val
+                                : uVal.u16Val;
+            if (DBGFADDRESS_IS_FLAT(pAddrBranchTgt))
+                DBGFR3AddrFromFlat(pUVM, &AddrTgt, GCPtr);
+            else
+                DBGFR3AddrFromSelOff(pUVM, idCpu, &AddrTgt, pAddrBranchTgt->Sel, GCPtr);
+
+            if (dbgfR3FlowAddrGetDistance(&AddrTgt, &pFlowBb->AddrEnd) <= _128K)
+            {
+                /* Finish the basic block. */
+                pFlowBb->AddrTarget = AddrTgt;
+                rc = dbgfR3FlowBbSuccessorAdd(pThis, &AddrTgt,
+                                              (pFlowBb->fFlags & DBGF_FLOW_BB_F_BRANCH_TABLE),
+                                              pFlowBb->pFlowBranchTbl);
+            }
+            else
+                rc = VERR_NOT_FOUND;
+        }
+    }
+    else
+        rc = dbgfR3FlowBranchTblVerifyAdd(pThis, pFlowBb, pAddrBranchTgt,
+                                          idxGenRegBase, cbPtr, pUVM, idCpu);
+
+    return rc;
+}
+
+
+/**
+ * Tries to resolve the indirect branch.
+ *
+ * @returns VBox status code.
+ * @param   pThis               The flow control graph.
+ * @param   pFlowBb             The basic block causing the indirect branch.
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               CPU id for disassembling.
+ * @param   pDisParam           The parameter from the disassembler.
+ * @param   fFlagsDisasm        Flags for the disassembler.
+ */
+static int dbgfR3FlowTryResolveIndirectBranch(PDBGFFLOWINT pThis, PDBGFFLOWBBINT pFlowBb, PUVM pUVM,
+                                              VMCPUID idCpu, PDISOPPARAM pDisParam, uint32_t fFlagsDisasm)
+{
+    Assert(dbgfR3FlowBranchTargetIsIndirect(pDisParam));
+
+    uint32_t cbPtr = 0;
+    CPUMMODE enmMode = dbgfR3FlowGetDisasCpuMode(pUVM, idCpu, fFlagsDisasm);
+
+    switch (enmMode)
+    {
+        case CPUMMODE_REAL:
+            cbPtr = sizeof(uint16_t);
+            break;
+        case CPUMMODE_PROTECTED:
+            cbPtr = sizeof(uint32_t);
+            break;
+        case CPUMMODE_LONG:
+            cbPtr = sizeof(uint64_t);
+            break;
+        default:
+            AssertMsgFailed(("Invalid CPU mode %u\n", enmMode));
+    }
+
+    if (pDisParam->fUse & DISUSE_BASE)
+    {
+        uint8_t idxRegBase = pDisParam->Base.idxGenReg;
+
+        /* Check that the used register size and the pointer size match. */
+        if (   ((pDisParam->fUse & DISUSE_REG_GEN16) && cbPtr == sizeof(uint16_t))
+            || ((pDisParam->fUse & DISUSE_REG_GEN32) && cbPtr == sizeof(uint32_t))
+            || ((pDisParam->fUse & DISUSE_REG_GEN64) && cbPtr == sizeof(uint64_t)))
+        {
+            /*
+             * Search all instructions backwards until a move to the used general register
+             * is detected with a constant using the pointer size.
+             */
+            uint32_t idxInstrStart = pFlowBb->cInstr - 1 - 1; /* Don't look at the branch. */
+            bool fCandidateFound = false;
+            bool fBranchTbl = RT_BOOL(pDisParam->fUse & DISUSE_INDEX);
+            DBGFADDRESS AddrBranchTgt;
+            do
+            {
+                fCandidateFound = dbgfR3FlowSearchMovWithConstantPtrSizeBackwards(pFlowBb, idxRegBase, cbPtr,
+                                                                                  pUVM, idCpu, fFlagsDisasm,
+                                                                                  &idxInstrStart, &AddrBranchTgt);
+                if (fCandidateFound)
+                {
+                    /* Check that the address is not too far away from the instruction address. */
+                    RTGCUINTPTR offPtr = dbgfR3FlowAddrGetDistance(&AddrBranchTgt, &pFlowBb->AddrEnd);
+                    if (offPtr <= 20 * _1M)
+                    {
+                        /* Read the content at the address and check that it is near this basic block too. */
+                        int rc = dbgfR3FlowCheckBranchTargetLocation(pThis, pFlowBb, &AddrBranchTgt, idxRegBase,
+                                                                     cbPtr, pUVM, idCpu, fBranchTbl);
+                        if (RT_SUCCESS(rc))
+                            break;
+                        fCandidateFound = false;
+                    }
+
+                    if (idxInstrStart > 0)
+                        idxInstrStart--;
+                }
+            } while (idxInstrStart > 0 && !fCandidateFound);
+        }
+        else
+            dbgfR3FlowBbSetError(pFlowBb, VERR_INVALID_STATE,
+                                 "The base register size and selected pointer size do not match (fUse=%#x cbPtr=%u)",
+                                 pDisParam->fUse, cbPtr);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Tries to resolve the indirect branch.
+ *
+ * @returns VBox status code.
+ * @param   pThis               The flow control graph.
+ * @param   pFlowBb             The basic block causing the indirect branch.
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               CPU id for disassembling.
+ * @param   pDisParam           The parameter from the disassembler.
+ * @param   fFlagsDisasm        Flags for the disassembler.
+ */
+static int dbgfR3FlowBbCheckBranchTblCandidate(PDBGFFLOWINT pThis, PDBGFFLOWBBINT pFlowBb, PUVM pUVM,
+                                               VMCPUID idCpu, PDISOPPARAM pDisParam, uint32_t fFlagsDisasm)
+{
+    int rc = VINF_SUCCESS;
+
+    Assert(pFlowBb->fFlags & DBGF_FLOW_BB_F_BRANCH_TABLE && pFlowBb->pFlowBranchTbl);
+
+    uint32_t cbPtr = 0;
+    CPUMMODE enmMode = dbgfR3FlowGetDisasCpuMode(pUVM, idCpu, fFlagsDisasm);
+
+    switch (enmMode)
+    {
+        case CPUMMODE_REAL:
+            cbPtr = sizeof(uint16_t);
+            break;
+        case CPUMMODE_PROTECTED:
+            cbPtr = sizeof(uint32_t);
+            break;
+        case CPUMMODE_LONG:
+            cbPtr = sizeof(uint64_t);
+            break;
+        default:
+            AssertMsgFailed(("Invalid CPU mode %u\n", enmMode));
+    }
+
+    if (pDisParam->fUse & DISUSE_BASE)
+    {
+        uint8_t idxRegBase = pDisParam->Base.idxGenReg;
+
+        /* Check that the used register size and the pointer size match. */
+        if (   ((pDisParam->fUse & DISUSE_REG_GEN16) && cbPtr == sizeof(uint16_t))
+            || ((pDisParam->fUse & DISUSE_REG_GEN32) && cbPtr == sizeof(uint32_t))
+            || ((pDisParam->fUse & DISUSE_REG_GEN64) && cbPtr == sizeof(uint64_t)))
+        {
+            if (idxRegBase != pFlowBb->pFlowBranchTbl->idxGenRegBase)
+            {
+                /* Try to find the new branch table. */
+                pFlowBb->pFlowBranchTbl = NULL;
+                rc = dbgfR3FlowTryResolveIndirectBranch(pThis, pFlowBb, pUVM, idCpu, pDisParam, fFlagsDisasm);
+            }
+            /** @todo else check that the base register is not modified in this basic block. */
+        }
+        else
+            dbgfR3FlowBbSetError(pFlowBb, VERR_INVALID_STATE,
+                                 "The base register size and selected pointer size do not match (fUse=%#x cbPtr=%u)",
+                                 pDisParam->fUse, cbPtr);
+    }
+    else
+        dbgfR3FlowBbSetError(pFlowBb, VERR_INVALID_STATE,
+                             "The instruction does not use a register");
+
+    return rc;
+}
+
+
+/**
+ * Processes and fills one basic block.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               CPU id for disassembling.
+ * @param   pThis               The control flow graph to populate.
+ * @param   pFlowBb             The basic block to fill.
+ * @param   cbDisasmMax         The maximum amount to disassemble.
+ * @param   fFlags              Combination of DBGF_DISAS_FLAGS_*.
+ */
+static int dbgfR3FlowBbProcess(PUVM pUVM, VMCPUID idCpu, PDBGFFLOWINT pThis, PDBGFFLOWBBINT pFlowBb,
+                              uint32_t cbDisasmMax, uint32_t fFlags)
+{
+    int rc = VINF_SUCCESS;
+    uint32_t cbDisasmLeft = cbDisasmMax ? cbDisasmMax : UINT32_MAX;
+    DBGFADDRESS AddrDisasm = pFlowBb->AddrEnd;
+
+    Assert(pFlowBb->fFlags & DBGF_FLOW_BB_F_EMPTY);
+
+    /*
+     * Disassemble instruction by instruction until we get a conditional or
+     * unconditional jump or some sort of return.
+     */
+    while (   cbDisasmLeft
+           && RT_SUCCESS(rc))
+    {
+        DBGFDISSTATE DisState;
+        char szOutput[_4K];
+
+        /*
+         * Before disassembling we have to check whether the address belongs
+         * to another basic block and stop here.
+         */
+        if (   !(pFlowBb->fFlags & DBGF_FLOW_BB_F_EMPTY)
+            && dbgfR3FlowHasBbWithStartAddr(pThis, &AddrDisasm))
+        {
+            pFlowBb->AddrTarget = AddrDisasm;
+            pFlowBb->enmEndType = DBGFFLOWBBENDTYPE_UNCOND;
+            break;
+        }
+
+        pFlowBb->fFlags &= ~DBGF_FLOW_BB_F_EMPTY;
+
+        rc = dbgfR3DisasInstrStateEx(pUVM, idCpu, &AddrDisasm, fFlags,
+                                     &szOutput[0], sizeof(szOutput), &DisState);
+        if (RT_SUCCESS(rc))
+        {
+            cbDisasmLeft -= DisState.cbInstr;
+
+            if (pFlowBb->cInstr == pFlowBb->cInstrMax)
+            {
+                /* Reallocate. */
+                RTListNodeRemove(&pFlowBb->NdFlowBb);
+                PDBGFFLOWBBINT pFlowBbNew = (PDBGFFLOWBBINT)RTMemRealloc(pFlowBb,
+                                                                         RT_UOFFSETOF_DYN(DBGFFLOWBBINT, aInstr[pFlowBb->cInstrMax + 10]));
+                if (pFlowBbNew)
+                {
+                    pFlowBbNew->cInstrMax += 10;
+                    pFlowBb = pFlowBbNew;
+                }
+                else
+                    rc = VERR_NO_MEMORY;
+                RTListAppend(&pThis->LstFlowBb, &pFlowBb->NdFlowBb);
+            }
+
+            if (RT_SUCCESS(rc))
+            {
+                PDBGFFLOWBBINSTR pInstr = &pFlowBb->aInstr[pFlowBb->cInstr];
+
+                pInstr->AddrInstr = AddrDisasm;
+                pInstr->cbInstr   = DisState.cbInstr;
+                pInstr->pszInstr  = RTStrCacheEnter(pThis->hStrCacheInstr, &szOutput[0]);
+                pFlowBb->cInstr++;
+
+                pFlowBb->AddrEnd = AddrDisasm;
+                DBGFR3AddrAdd(&pFlowBb->AddrEnd, pInstr->cbInstr - 1);
+                DBGFR3AddrAdd(&AddrDisasm, pInstr->cbInstr);
+
+                /*
+                 * Check control flow instructions and create new basic blocks
+                 * marking the current one as complete.
+                 */
+                if (DisState.pCurInstr->fOpType & DISOPTYPE_CONTROLFLOW)
+                {
+                    uint16_t uOpc = DisState.pCurInstr->uOpcode;
+
+                    if (   uOpc == OP_RETN || uOpc == OP_RETF || uOpc == OP_IRET
+                        || uOpc == OP_SYSEXIT || uOpc == OP_SYSRET)
+                        pFlowBb->enmEndType = DBGFFLOWBBENDTYPE_EXIT;
+                    else if (uOpc == OP_JMP)
+                    {
+                        Assert(DisState.pCurInstr->fOpType & DISOPTYPE_UNCOND_CONTROLFLOW);
+
+                        if (dbgfR3FlowBranchTargetIsIndirect(&DisState.Param1))
+                        {
+                            pFlowBb->enmEndType = DBGFFLOWBBENDTYPE_UNCOND_INDIRECT_JMP;
+
+                            if (pFlowBb->fFlags & DBGF_FLOW_BB_F_BRANCH_TABLE)
+                            {
+                                Assert(pThis->fFlags & DBGF_FLOW_CREATE_F_TRY_RESOLVE_INDIRECT_BRANCHES);
+
+                                /*
+                                 * This basic block was already discovered by parsing a jump table and
+                                 * there should be a candidate for the branch table. Check whether it uses the
+                                 * same branch table.
+                                 */
+                                rc = dbgfR3FlowBbCheckBranchTblCandidate(pThis, pFlowBb, pUVM, idCpu,
+                                                                         &DisState.Param1, fFlags);
+                            }
+                            else
+                            {
+                                if (pThis->fFlags & DBGF_FLOW_CREATE_F_TRY_RESOLVE_INDIRECT_BRANCHES)
+                                    rc = dbgfR3FlowTryResolveIndirectBranch(pThis, pFlowBb, pUVM, idCpu,
+                                                                            &DisState.Param1, fFlags);
+                                else
+                                    dbgfR3FlowBbSetError(pFlowBb, VERR_NOT_SUPPORTED,
+                                                         "Detected indirect branch and resolving it not being enabled");
+                            }
+                        }
+                        else
+                        {
+                            pFlowBb->enmEndType = DBGFFLOWBBENDTYPE_UNCOND_JMP;
+
+                            /* Create one new basic block with the jump target address. */
+                            rc = dbgfR3FlowQueryDirectBranchTarget(pUVM, idCpu, &DisState.Param1, &pInstr->AddrInstr, pInstr->cbInstr,
+                                                                   RT_BOOL(DisState.pCurInstr->fOpType & DISOPTYPE_RELATIVE_CONTROLFLOW),
+                                                                   &pFlowBb->AddrTarget);
+                            if (RT_SUCCESS(rc))
+                                rc = dbgfR3FlowBbSuccessorAdd(pThis, &pFlowBb->AddrTarget,
+                                                              (pFlowBb->fFlags & DBGF_FLOW_BB_F_BRANCH_TABLE),
+                                                              pFlowBb->pFlowBranchTbl);
+                        }
+                    }
+                    else if (uOpc != OP_CALL)
+                    {
+                        Assert(DisState.pCurInstr->fOpType & DISOPTYPE_COND_CONTROLFLOW);
+                        pFlowBb->enmEndType = DBGFFLOWBBENDTYPE_COND;
+
+                        /*
+                         * Create two new basic blocks, one with the jump target address
+                         * and one starting after the current instruction.
+                         */
+                        rc = dbgfR3FlowBbSuccessorAdd(pThis, &AddrDisasm,
+                                                      (pFlowBb->fFlags & DBGF_FLOW_BB_F_BRANCH_TABLE),
+                                                      pFlowBb->pFlowBranchTbl);
+                        if (RT_SUCCESS(rc))
+                        {
+                            rc = dbgfR3FlowQueryDirectBranchTarget(pUVM, idCpu, &DisState.Param1, &pInstr->AddrInstr, pInstr->cbInstr,
+                                                                   RT_BOOL(DisState.pCurInstr->fOpType & DISOPTYPE_RELATIVE_CONTROLFLOW),
+                                                                   &pFlowBb->AddrTarget);
+                            if (RT_SUCCESS(rc))
+                                rc = dbgfR3FlowBbSuccessorAdd(pThis, &pFlowBb->AddrTarget,
+                                                              (pFlowBb->fFlags & DBGF_FLOW_BB_F_BRANCH_TABLE),
+                                                              pFlowBb->pFlowBranchTbl);
+                        }
+                    }
+
+                    if (RT_FAILURE(rc))
+                        dbgfR3FlowBbSetError(pFlowBb, rc, "Adding successor blocks failed with %Rrc", rc);
+
+                    /* Quit disassembling. */
+                    if (   uOpc != OP_CALL
+                        || RT_FAILURE(rc))
+                        break;
+                }
+            }
+            else
+                dbgfR3FlowBbSetError(pFlowBb, rc, "Increasing basic block failed with %Rrc", rc);
+        }
+        else
+            dbgfR3FlowBbSetError(pFlowBb, rc, "Disassembling the instruction failed with %Rrc", rc);
+    }
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Populate all empty basic blocks.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               CPU id for disassembling.
+ * @param   pThis               The control flow graph to populate.
+ * @param   pAddrStart          The start address to disassemble at.
+ * @param   cbDisasmMax         The maximum amount to disassemble.
+ * @param   fFlags              Combination of DBGF_DISAS_FLAGS_*.
+ */
+static int dbgfR3FlowPopulate(PUVM pUVM, VMCPUID idCpu, PDBGFFLOWINT pThis, PDBGFADDRESS pAddrStart,
+                             uint32_t cbDisasmMax, uint32_t fFlags)
+{
+    int rc = VINF_SUCCESS;
+    PDBGFFLOWBBINT pFlowBb = dbgfR3FlowGetUnpopulatedBb(pThis);
+    DBGFADDRESS AddrEnd = *pAddrStart;
+    DBGFR3AddrAdd(&AddrEnd, cbDisasmMax);
+
+    while (VALID_PTR(pFlowBb))
+    {
+        rc = dbgfR3FlowBbProcess(pUVM, idCpu, pThis, pFlowBb, cbDisasmMax, fFlags);
+        if (RT_FAILURE(rc))
+            break;
+
+        pFlowBb = dbgfR3FlowGetUnpopulatedBb(pThis);
+    }
+
+    return rc;
+}
+
+/**
+ * Creates a new control flow graph from the given start address.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               CPU id for disassembling.
+ * @param   pAddressStart       Where to start creating the control flow graph.
+ * @param   cbDisasmMax         Limit the amount of bytes to disassemble, 0 for no limit.
+ * @param   fFlagsFlow          Combination of DBGF_FLOW_CREATE_F_* to control the creation of the flow graph.
+ * @param   fFlagsDisasm        Combination of DBGF_DISAS_FLAGS_* controlling the style of the disassembled
+ *                              instructions.
+ * @param   phFlow              Where to store the handle to the control flow graph on success.
+ */
+VMMR3DECL(int) DBGFR3FlowCreate(PUVM pUVM, VMCPUID idCpu, PDBGFADDRESS pAddressStart, uint32_t cbDisasmMax,
+                                uint32_t fFlagsFlow, uint32_t fFlagsDisasm, PDBGFFLOW phFlow)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_CPU_ID);
+    AssertPtrReturn(pAddressStart, VERR_INVALID_POINTER);
+    AssertReturn(!(fFlagsDisasm & ~DBGF_DISAS_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn((fFlagsDisasm & DBGF_DISAS_FLAGS_MODE_MASK) <= DBGF_DISAS_FLAGS_64BIT_MODE, VERR_INVALID_PARAMETER);
+
+    /* Create the control flow graph container. */
+    int rc = VINF_SUCCESS;
+    PDBGFFLOWINT pThis = (PDBGFFLOWINT)RTMemAllocZ(sizeof(DBGFFLOWINT));
+    if (RT_LIKELY(pThis))
+    {
+        rc = RTStrCacheCreate(&pThis->hStrCacheInstr, "DBGFFLOW");
+        if (RT_SUCCESS(rc))
+        {
+            pThis->cRefs       = 1;
+            pThis->cRefsBb     = 0;
+            pThis->cBbs        = 0;
+            pThis->cBranchTbls = 0;
+            pThis->fFlags      = fFlagsFlow;
+            RTListInit(&pThis->LstFlowBb);
+            RTListInit(&pThis->LstBranchTbl);
+            /* Create the entry basic block and start the work. */
+
+            PDBGFFLOWBBINT pFlowBb = dbgfR3FlowBbCreate(pThis, pAddressStart, DBGF_FLOW_BB_F_ENTRY, 10);
+            if (RT_LIKELY(pFlowBb))
+            {
+                dbgfR3FlowLink(pThis, pFlowBb);
+                rc = dbgfR3FlowPopulate(pUVM, idCpu, pThis, pAddressStart, cbDisasmMax, fFlagsDisasm);
+                if (RT_SUCCESS(rc))
+                {
+                    *phFlow = pThis;
+                    return VINF_SUCCESS;
+                }
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+
+        ASMAtomicDecU32(&pThis->cRefs);
+        dbgfR3FlowDestroy(pThis);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    return rc;
+}
+
+
+/**
+ * Retains the control flow graph handle.
+ *
+ * @returns Current reference count.
+ * @param   hFlow                The control flow graph handle to retain.
+ */
+VMMR3DECL(uint32_t) DBGFR3FlowRetain(DBGFFLOW hFlow)
+{
+    PDBGFFLOWINT pThis = hFlow;
+    AssertPtrReturn(pThis, UINT32_MAX);
+
+    uint32_t cRefs = ASMAtomicIncU32(&pThis->cRefs);
+    AssertMsg(cRefs > 1 && cRefs < _1M, ("%#x %p\n", cRefs, pThis));
+    return cRefs;
+}
+
+
+/**
+ * Releases the control flow graph handle.
+ *
+ * @returns Current reference count, on 0 the control flow graph will be destroyed.
+ * @param   hFlow                The control flow graph handle to release.
+ */
+VMMR3DECL(uint32_t) DBGFR3FlowRelease(DBGFFLOW hFlow)
+{
+    PDBGFFLOWINT pThis = hFlow;
+    if (!pThis)
+        return 0;
+    AssertPtrReturn(pThis, UINT32_MAX);
+
+    uint32_t cRefs = ASMAtomicDecU32(&pThis->cRefs);
+    AssertMsg(cRefs < _1M, ("%#x %p\n", cRefs, pThis));
+    if (cRefs == 0)
+        dbgfR3FlowDestroy(pThis);
+    return cRefs;
+}
+
+
+/**
+ * Queries the basic block denoting the entry point into the control flow graph.
+ *
+ * @returns VBox status code.
+ * @param   hFlow                The control flow graph handle.
+ * @param   phFlowBb             Where to store the basic block handle on success.
+ */
+VMMR3DECL(int) DBGFR3FlowQueryStartBb(DBGFFLOW hFlow, PDBGFFLOWBB phFlowBb)
+{
+    PDBGFFLOWINT pThis = hFlow;
+    AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
+
+    PDBGFFLOWBBINT pFlowBb;
+    RTListForEach(&pThis->LstFlowBb, pFlowBb, DBGFFLOWBBINT, NdFlowBb)
+    {
+        if (pFlowBb->fFlags & DBGF_FLOW_BB_F_ENTRY)
+        {
+            *phFlowBb = pFlowBb;
+            return VINF_SUCCESS;
+        }
+    }
+
+    AssertFailed(); /* Should never get here. */
+    return VERR_INTERNAL_ERROR;
+}
+
+
+/**
+ * Queries a basic block in the given control flow graph which covers the given
+ * address.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_NOT_FOUND if there is no basic block intersecting with the address.
+ * @param   hFlow               The control flow graph handle.
+ * @param   pAddr               The address to look for.
+ * @param   phFlowBb            Where to store the basic block handle on success.
+ */
+VMMR3DECL(int) DBGFR3FlowQueryBbByAddress(DBGFFLOW hFlow, PDBGFADDRESS pAddr, PDBGFFLOWBB phFlowBb)
+{
+    PDBGFFLOWINT pThis = hFlow;
+    AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
+    AssertPtrReturn(pAddr, VERR_INVALID_POINTER);
+    AssertPtrReturn(phFlowBb, VERR_INVALID_POINTER);
+
+    PDBGFFLOWBBINT pFlowBb;
+    RTListForEach(&pThis->LstFlowBb, pFlowBb, DBGFFLOWBBINT, NdFlowBb)
+    {
+        if (dbgfR3FlowAddrIntersect(pFlowBb, pAddr))
+        {
+            DBGFR3FlowBbRetain(pFlowBb);
+            *phFlowBb = pFlowBb;
+            return VINF_SUCCESS;
+        }
+    }
+
+    return VERR_NOT_FOUND;
+}
+
+
+/**
+ * Queries a branch table in the given control flow graph by the given address.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_NOT_FOUND if there is no branch table with the given address.
+ * @param   hFlow               The control flow graph handle.
+ * @param   pAddr               The address of the branch table.
+ * @param   phFlowBranchTbl     Where to store the handle to branch table on success.
+ *
+ * @note Call DBGFR3FlowBranchTblRelease() when the handle is not required anymore.
+ */
+VMMR3DECL(int) DBGFR3FlowQueryBranchTblByAddress(DBGFFLOW hFlow, PDBGFADDRESS pAddr, PDBGFFLOWBRANCHTBL phFlowBranchTbl)
+{
+    PDBGFFLOWINT pThis = hFlow;
+    AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
+    AssertPtrReturn(pAddr, VERR_INVALID_POINTER);
+    AssertPtrReturn(phFlowBranchTbl, VERR_INVALID_POINTER);
+
+    PDBGFFLOWBRANCHTBLINT pBranchTbl = dbgfR3FlowBranchTblFindByAddr(pThis, pAddr);
+    if (pBranchTbl)
+    {
+        DBGFR3FlowBranchTblRetain(pBranchTbl);
+        *phFlowBranchTbl = pBranchTbl;
+        return VINF_SUCCESS;
+    }
+
+    return VERR_NOT_FOUND;
+}
+
+
+/**
+ * Returns the number of basic blcoks inside the control flow graph.
+ *
+ * @returns Number of basic blocks.
+ * @param   hFlow                The control flow graph handle.
+ */
+VMMR3DECL(uint32_t) DBGFR3FlowGetBbCount(DBGFFLOW hFlow)
+{
+    PDBGFFLOWINT pThis = hFlow;
+    AssertPtrReturn(pThis, 0);
+
+    return pThis->cBbs;
+}
+
+
+/**
+ * Returns the number of branch tables inside the control flow graph.
+ *
+ * @returns Number of basic blocks.
+ * @param   hFlow                The control flow graph handle.
+ */
+VMMR3DECL(uint32_t) DBGFR3FlowGetBranchTblCount(DBGFFLOW hFlow)
+{
+    PDBGFFLOWINT pThis = hFlow;
+    AssertPtrReturn(pThis, 0);
+
+    return pThis->cBranchTbls;
+}
+
+
+/**
+ * Retains the basic block handle.
+ *
+ * @returns Current reference count.
+ * @param   hFlowBb              The basic block handle to retain.
+ */
+VMMR3DECL(uint32_t) DBGFR3FlowBbRetain(DBGFFLOWBB hFlowBb)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, UINT32_MAX);
+
+    uint32_t cRefs = ASMAtomicIncU32(&pFlowBb->cRefs);
+    AssertMsg(cRefs > 1 && cRefs < _1M, ("%#x %p %d\n", cRefs, pFlowBb, pFlowBb->enmEndType));
+    return cRefs;
+}
+
+
+/**
+ * Releases the basic block handle.
+ *
+ * @returns Current reference count, on 0 the basic block will be destroyed.
+ * @param   hFlowBb              The basic block handle to release.
+ */
+VMMR3DECL(uint32_t) DBGFR3FlowBbRelease(DBGFFLOWBB hFlowBb)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    if (!pFlowBb)
+        return 0;
+
+    return dbgfR3FlowBbReleaseInt(pFlowBb, true /* fMayDestroyFlow */);
+}
+
+
+/**
+ * Returns the start address of the basic block.
+ *
+ * @returns Pointer to DBGF adress containing the start address of the basic block.
+ * @param   hFlowBb              The basic block handle.
+ * @param   pAddrStart          Where to store the start address of the basic block.
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3FlowBbGetStartAddress(DBGFFLOWBB hFlowBb, PDBGFADDRESS pAddrStart)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, NULL);
+    AssertPtrReturn(pAddrStart, NULL);
+
+    *pAddrStart = pFlowBb->AddrStart;
+    return pAddrStart;
+}
+
+
+/**
+ * Returns the end address of the basic block (inclusive).
+ *
+ * @returns Pointer to DBGF adress containing the end address of the basic block.
+ * @param   hFlowBb              The basic block handle.
+ * @param   pAddrEnd            Where to store the end address of the basic block.
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3FlowBbGetEndAddress(DBGFFLOWBB hFlowBb, PDBGFADDRESS pAddrEnd)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, NULL);
+    AssertPtrReturn(pAddrEnd, NULL);
+
+    *pAddrEnd = pFlowBb->AddrEnd;
+    return pAddrEnd;
+}
+
+
+/**
+ * Returns the address the last instruction in the basic block branches to.
+ *
+ * @returns Pointer to DBGF adress containing the branch address of the basic block.
+ * @param   hFlowBb             The basic block handle.
+ * @param   pAddrTarget         Where to store the branch address of the basic block.
+ *
+ * @note This is only valid for unconditional or conditional branches and will assert
+ *       for every other basic block type.
+ * @note For indirect unconditional branches using a branch table this will return the start address
+ *       of the branch table.
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3FlowBbGetBranchAddress(DBGFFLOWBB hFlowBb, PDBGFADDRESS pAddrTarget)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, NULL);
+    AssertPtrReturn(pAddrTarget, NULL);
+    AssertReturn(   pFlowBb->enmEndType == DBGFFLOWBBENDTYPE_UNCOND_JMP
+                 || pFlowBb->enmEndType == DBGFFLOWBBENDTYPE_COND
+                 || pFlowBb->enmEndType == DBGFFLOWBBENDTYPE_UNCOND_INDIRECT_JMP,
+                 NULL);
+
+    if (   pFlowBb->enmEndType == DBGFFLOWBBENDTYPE_UNCOND_INDIRECT_JMP
+        && pFlowBb->pFlowBranchTbl)
+        *pAddrTarget = pFlowBb->pFlowBranchTbl->AddrStart;
+    else
+        *pAddrTarget = pFlowBb->AddrTarget;
+    return pAddrTarget;
+}
+
+
+/**
+ * Returns the address of the next block following this one in the instruction stream.
+ * (usually end address + 1).
+ *
+ * @returns Pointer to DBGF adress containing the following address of the basic block.
+ * @param   hFlowBb              The basic block handle.
+ * @param   pAddrFollow         Where to store the following address of the basic block.
+ *
+ * @note This is only valid for conditional branches and if the last instruction in the
+ *       given basic block doesn't change the control flow but the blocks were split
+ *       because the successor is referenced by multiple other blocks as an entry point.
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3FlowBbGetFollowingAddress(DBGFFLOWBB hFlowBb, PDBGFADDRESS pAddrFollow)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, NULL);
+    AssertPtrReturn(pAddrFollow, NULL);
+    AssertReturn(   pFlowBb->enmEndType == DBGFFLOWBBENDTYPE_UNCOND
+                 || pFlowBb->enmEndType == DBGFFLOWBBENDTYPE_COND,
+                 NULL);
+
+    *pAddrFollow = pFlowBb->AddrEnd;
+    DBGFR3AddrAdd(pAddrFollow, 1);
+    return pAddrFollow;
+}
+
+
+/**
+ * Returns the type of the last instruction in the basic block.
+ *
+ * @returns Last instruction type.
+ * @param   hFlowBb              The basic block handle.
+ */
+VMMR3DECL(DBGFFLOWBBENDTYPE) DBGFR3FlowBbGetType(DBGFFLOWBB hFlowBb)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, DBGFFLOWBBENDTYPE_INVALID);
+
+    return pFlowBb->enmEndType;
+}
+
+
+/**
+ * Get the number of instructions contained in the basic block.
+ *
+ * @returns Number of instructions in the basic block.
+ * @param   hFlowBb              The basic block handle.
+ */
+VMMR3DECL(uint32_t) DBGFR3FlowBbGetInstrCount(DBGFFLOWBB hFlowBb)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, 0);
+
+    return pFlowBb->cInstr;
+}
+
+
+/**
+ * Get flags for the given basic block.
+ *
+ * @returns Combination of DBGF_FLOW_BB_F_*
+ * @param   hFlowBb              The basic block handle.
+ */
+VMMR3DECL(uint32_t) DBGFR3FlowBbGetFlags(DBGFFLOWBB hFlowBb)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, 0);
+
+    return pFlowBb->fFlags;
+}
+
+
+/**
+ * Queries the branch table used if the given basic block ends with an indirect branch
+ * and has a branch table referenced.
+ *
+ * @returns VBox status code.
+ * @param   hFlowBb              The basic block handle.
+ * @param   phBranchTbl          Where to store the branch table handle on success.
+ *
+ * @note Release the branch table reference with DBGFR3FlowBranchTblRelease() when not required
+ *       anymore.
+ */
+VMMR3DECL(int) DBGFR3FlowBbQueryBranchTbl(DBGFFLOWBB hFlowBb, PDBGFFLOWBRANCHTBL phBranchTbl)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, VERR_INVALID_HANDLE);
+    AssertReturn(pFlowBb->enmEndType == DBGFFLOWBBENDTYPE_UNCOND_INDIRECT_JMP, VERR_INVALID_STATE);
+    AssertPtrReturn(pFlowBb->pFlowBranchTbl, VERR_INVALID_STATE);
+    AssertPtrReturn(phBranchTbl, VERR_INVALID_POINTER);
+
+    DBGFR3FlowBranchTblRetain(pFlowBb->pFlowBranchTbl);
+    *phBranchTbl = pFlowBb->pFlowBranchTbl;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Returns the error status and message if the given basic block has an error.
+ *
+ * @returns VBox status code of the error for the basic block.
+ * @param   hFlowBb              The basic block handle.
+ * @param   ppszErr             Where to store the pointer to the error message - optional.
+ */
+VMMR3DECL(int) DBGFR3FlowBbQueryError(DBGFFLOWBB hFlowBb, const char **ppszErr)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, VERR_INVALID_HANDLE);
+
+    if (ppszErr)
+        *ppszErr = pFlowBb->pszErr;
+
+    return pFlowBb->rcError;
+}
+
+
+/**
+ * Store the disassembled instruction as a string in the given output buffer.
+ *
+ * @returns VBox status code.
+ * @param   hFlowBb              The basic block handle.
+ * @param   idxInstr            The instruction to query.
+ * @param   pAddrInstr          Where to store the guest instruction address on success, optional.
+ * @param   pcbInstr            Where to store the instruction size on success, optional.
+ * @param   ppszInstr           Where to store the pointer to the disassembled instruction string, optional.
+ */
+VMMR3DECL(int) DBGFR3FlowBbQueryInstr(DBGFFLOWBB hFlowBb, uint32_t idxInstr, PDBGFADDRESS pAddrInstr,
+                                     uint32_t *pcbInstr, const char **ppszInstr)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, VERR_INVALID_POINTER);
+    AssertReturn(idxInstr < pFlowBb->cInstr, VERR_INVALID_PARAMETER);
+
+    if (pAddrInstr)
+        *pAddrInstr = pFlowBb->aInstr[idxInstr].AddrInstr;
+    if (pcbInstr)
+        *pcbInstr = pFlowBb->aInstr[idxInstr].cbInstr;
+    if (ppszInstr)
+        *ppszInstr = pFlowBb->aInstr[idxInstr].pszInstr;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Queries the successors of the basic block.
+ *
+ * @returns VBox status code.
+ * @param   hFlowBb              The basic block handle.
+ * @param   phFlowBbFollow       Where to store the handle to the basic block following
+ *                              this one (optional).
+ * @param   phFlowBbTarget       Where to store the handle to the basic block being the
+ *                              branch target for this one (optional).
+ */
+VMMR3DECL(int) DBGFR3FlowBbQuerySuccessors(DBGFFLOWBB hFlowBb, PDBGFFLOWBB phFlowBbFollow, PDBGFFLOWBB phFlowBbTarget)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, VERR_INVALID_POINTER);
+
+    if (   phFlowBbFollow
+        && (   pFlowBb->enmEndType == DBGFFLOWBBENDTYPE_UNCOND
+            || pFlowBb->enmEndType == DBGFFLOWBBENDTYPE_COND))
+    {
+        DBGFADDRESS AddrStart = pFlowBb->AddrEnd;
+        DBGFR3AddrAdd(&AddrStart, 1);
+        int rc = DBGFR3FlowQueryBbByAddress(pFlowBb->pFlow, &AddrStart, phFlowBbFollow);
+        AssertRC(rc);
+    }
+
+    if (   phFlowBbTarget
+        && (   pFlowBb->enmEndType == DBGFFLOWBBENDTYPE_UNCOND_JMP
+            || pFlowBb->enmEndType == DBGFFLOWBBENDTYPE_COND))
+    {
+        int rc = DBGFR3FlowQueryBbByAddress(pFlowBb->pFlow, &pFlowBb->AddrTarget, phFlowBbTarget);
+        AssertRC(rc);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Returns the number of basic blocks referencing this basic block as a target.
+ *
+ * @returns Number of other basic blocks referencing this one.
+ * @param   hFlowBb              The basic block handle.
+ *
+ * @note If the given basic block references itself (loop, etc.) this will be counted as well.
+ */
+VMMR3DECL(uint32_t) DBGFR3FlowBbGetRefBbCount(DBGFFLOWBB hFlowBb)
+{
+    PDBGFFLOWBBINT pFlowBb = hFlowBb;
+    AssertPtrReturn(pFlowBb, 0);
+
+    uint32_t cRefsBb = 0;
+    PDBGFFLOWBBINT pFlowBbCur;
+    RTListForEach(&pFlowBb->pFlow->LstFlowBb, pFlowBbCur, DBGFFLOWBBINT, NdFlowBb)
+    {
+        if (pFlowBbCur->fFlags & DBGF_FLOW_BB_F_INCOMPLETE_ERR)
+            continue;
+
+        if (   pFlowBbCur->enmEndType == DBGFFLOWBBENDTYPE_UNCOND
+            || pFlowBbCur->enmEndType == DBGFFLOWBBENDTYPE_COND)
+        {
+            DBGFADDRESS AddrStart = pFlowBb->AddrEnd;
+            DBGFR3AddrAdd(&AddrStart, 1);
+            if (dbgfR3FlowAddrEqual(&pFlowBbCur->AddrStart, &AddrStart))
+                cRefsBb++;
+        }
+
+        if (   (   pFlowBbCur->enmEndType == DBGFFLOWBBENDTYPE_UNCOND_JMP
+                || pFlowBbCur->enmEndType == DBGFFLOWBBENDTYPE_COND)
+            && dbgfR3FlowAddrEqual(&pFlowBbCur->AddrStart, &pFlowBb->AddrTarget))
+            cRefsBb++;
+    }
+    return cRefsBb;
+}
+
+
+/**
+ * Returns the basic block handles referencing the given basic block.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_BUFFER_OVERFLOW if the array can't hold all the basic blocks.
+ * @param   hFlowBb             The basic block handle.
+ * @param   paFlowBbRef         Pointer to the array containing the referencing basic block handles on success.
+ * @param   cRef                Number of entries in the given array.
+ */
+VMMR3DECL(int) DBGFR3FlowBbGetRefBb(DBGFFLOWBB hFlowBb, PDBGFFLOWBB paFlowBbRef, uint32_t cRef)
+{
+    RT_NOREF3(hFlowBb, paFlowBbRef, cRef);
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * Retains a reference for the given control flow graph branch table.
+ *
+ * @returns new reference count.
+ * @param   hFlowBranchTbl      The branch table handle.
+ */
+VMMR3DECL(uint32_t) DBGFR3FlowBranchTblRetain(DBGFFLOWBRANCHTBL hFlowBranchTbl)
+{
+    PDBGFFLOWBRANCHTBLINT pFlowBranchTbl = hFlowBranchTbl;
+    AssertPtrReturn(pFlowBranchTbl, UINT32_MAX);
+
+    uint32_t cRefs = ASMAtomicIncU32(&pFlowBranchTbl->cRefs);
+    AssertMsg(cRefs > 1 && cRefs < _1M, ("%#x %p\n", cRefs, pFlowBranchTbl));
+    return cRefs;
+}
+
+
+/**
+ * Releases a given branch table handle.
+ *
+ * @returns the new reference count of the given branch table, on 0 it is destroyed.
+ * @param   hFlowBranchTbl      The branch table handle.
+ */
+VMMR3DECL(uint32_t) DBGFR3FlowBranchTblRelease(DBGFFLOWBRANCHTBL hFlowBranchTbl)
+{
+    PDBGFFLOWBRANCHTBLINT pFlowBranchTbl = hFlowBranchTbl;
+    if (!pFlowBranchTbl)
+        return 0;
+    AssertPtrReturn(pFlowBranchTbl, UINT32_MAX);
+
+    uint32_t cRefs = ASMAtomicDecU32(&pFlowBranchTbl->cRefs);
+    AssertMsg(cRefs < _1M, ("%#x %p\n", cRefs, pFlowBranchTbl));
+    if (cRefs == 0)
+        dbgfR3FlowBranchTblDestroy(pFlowBranchTbl);
+    return cRefs;
+}
+
+
+/**
+ * Return the number of slots the branch table has.
+ *
+ * @returns Number of slots in the branch table.
+ * @param   hFlowBranchTbl      The branch table handle.
+ */
+VMMR3DECL(uint32_t) DBGFR3FlowBranchTblGetSlots(DBGFFLOWBRANCHTBL hFlowBranchTbl)
+{
+    PDBGFFLOWBRANCHTBLINT pFlowBranchTbl = hFlowBranchTbl;
+    AssertPtrReturn(pFlowBranchTbl, 0);
+
+    return pFlowBranchTbl->cSlots;
+}
+
+
+/**
+ * Returns the start address of the branch table in the guest.
+ *
+ * @returns Pointer to start address of the branch table (pAddrStart).
+ * @param   hFlowBranchTbl      The branch table handle.
+ * @param   pAddrStart          Where to store the branch table address.
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3FlowBranchTblGetStartAddress(DBGFFLOWBRANCHTBL hFlowBranchTbl, PDBGFADDRESS pAddrStart)
+{
+    PDBGFFLOWBRANCHTBLINT pFlowBranchTbl = hFlowBranchTbl;
+    AssertPtrReturn(pFlowBranchTbl, NULL);
+    AssertPtrReturn(pAddrStart, NULL);
+
+    *pAddrStart = pFlowBranchTbl->AddrStart;
+    return pAddrStart;
+}
+
+
+/**
+ * Returns one address in the branch table at the given slot index.
+ *
+ * @return Pointer to the address at the given slot in the given branch table.
+ * @param  hFlowBranchTbl       The branch table handle.
+ * @param  idxSlot              The slot the address should be returned from.
+ * @param  pAddrSlot            Where to store the address.
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3FlowBranchTblGetAddrAtSlot(DBGFFLOWBRANCHTBL hFlowBranchTbl, uint32_t idxSlot, PDBGFADDRESS pAddrSlot)
+{
+    PDBGFFLOWBRANCHTBLINT pFlowBranchTbl = hFlowBranchTbl;
+    AssertPtrReturn(pFlowBranchTbl, NULL);
+    AssertPtrReturn(pAddrSlot, NULL);
+    AssertReturn(idxSlot < pFlowBranchTbl->cSlots, NULL);
+
+    *pAddrSlot = pFlowBranchTbl->aAddresses[idxSlot];
+    return pAddrSlot;
+}
+
+
+/**
+ * Query all addresses contained in the given branch table.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_BUFFER_OVERFLOW if there is not enough space in the array to hold all addresses.
+ * @param   hFlowBranchTbl      The branch table handle.
+ * @param   paAddrs             Where to store the addresses on success.
+ * @param   cAddrs              Number of entries the array can hold.
+ */
+VMMR3DECL(int) DBGFR3FlowBranchTblQueryAddresses(DBGFFLOWBRANCHTBL hFlowBranchTbl, PDBGFADDRESS paAddrs, uint32_t cAddrs)
+{
+    PDBGFFLOWBRANCHTBLINT pFlowBranchTbl = hFlowBranchTbl;
+    AssertPtrReturn(pFlowBranchTbl, VERR_INVALID_HANDLE);
+    AssertPtrReturn(paAddrs, VERR_INVALID_POINTER);
+    AssertReturn(cAddrs > 0, VERR_INVALID_PARAMETER);
+
+    if (cAddrs < pFlowBranchTbl->cSlots)
+        return VERR_BUFFER_OVERFLOW;
+
+    memcpy(paAddrs, &pFlowBranchTbl->aAddresses[0], pFlowBranchTbl->cSlots * sizeof(DBGFADDRESS));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNRTSORTCMP}
+ */
+static DECLCALLBACK(int) dbgfR3FlowItSortCmp(void const *pvElement1, void const *pvElement2, void *pvUser)
+{
+    PDBGFFLOWITORDER penmOrder = (PDBGFFLOWITORDER)pvUser;
+    PDBGFFLOWBBINT pFlowBb1 = *(PDBGFFLOWBBINT *)pvElement1;
+    PDBGFFLOWBBINT pFlowBb2 = *(PDBGFFLOWBBINT *)pvElement2;
+
+    if (dbgfR3FlowAddrEqual(&pFlowBb1->AddrStart, &pFlowBb2->AddrStart))
+        return 0;
+
+    if (*penmOrder == DBGFFLOWITORDER_BY_ADDR_LOWEST_FIRST)
+    {
+        if (dbgfR3FlowAddrLower(&pFlowBb1->AddrStart, &pFlowBb2->AddrStart))
+            return -1;
+        else
+            return 1;
+    }
+    else
+    {
+        if (dbgfR3FlowAddrLower(&pFlowBb1->AddrStart, &pFlowBb2->AddrStart))
+            return 1;
+        else
+            return -1;
+    }
+}
+
+
+/**
+ * Creates a new iterator for the given control flow graph.
+ *
+ * @returns VBox status code.
+ * @param   hFlow               The control flow graph handle.
+ * @param   enmOrder            The order in which the basic blocks are enumerated.
+ * @param   phFlowIt            Where to store the handle to the iterator on success.
+ */
+VMMR3DECL(int) DBGFR3FlowItCreate(DBGFFLOW hFlow, DBGFFLOWITORDER enmOrder, PDBGFFLOWIT phFlowIt)
+{
+    int rc = VINF_SUCCESS;
+    PDBGFFLOWINT pFlow = hFlow;
+    AssertPtrReturn(pFlow, VERR_INVALID_POINTER);
+    AssertPtrReturn(phFlowIt, VERR_INVALID_POINTER);
+    AssertReturn(enmOrder > DBGFFLOWITORDER_INVALID && enmOrder < DBGFFLOWITORDER_BREADTH_FIRST,
+                 VERR_INVALID_PARAMETER);
+    AssertReturn(enmOrder < DBGFFLOWITORDER_DEPTH_FRIST, VERR_NOT_IMPLEMENTED); /** @todo */
+
+    PDBGFFLOWITINT pIt = (PDBGFFLOWITINT)RTMemAllocZ(RT_UOFFSETOF_DYN(DBGFFLOWITINT, apBb[pFlow->cBbs]));
+    if (RT_LIKELY(pIt))
+    {
+        DBGFR3FlowRetain(hFlow);
+        pIt->pFlow      = pFlow;
+        pIt->idxBbNext = 0;
+        /* Fill the list and then sort. */
+        uint32_t idxBb = 0;
+        PDBGFFLOWBBINT pFlowBb;
+        RTListForEach(&pFlow->LstFlowBb, pFlowBb, DBGFFLOWBBINT, NdFlowBb)
+        {
+            DBGFR3FlowBbRetain(pFlowBb);
+            pIt->apBb[idxBb++] = pFlowBb;
+        }
+
+        /* Sort the blocks by address. */
+        RTSortShell(&pIt->apBb[0], pFlow->cBbs, sizeof(PDBGFFLOWBBINT), dbgfR3FlowItSortCmp, &enmOrder);
+
+        *phFlowIt = pIt;
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    return rc;
+}
+
+
+/**
+ * Destroys a given control flow graph iterator.
+ *
+ * @returns nothing.
+ * @param   hFlowIt              The control flow graph iterator handle.
+ */
+VMMR3DECL(void) DBGFR3FlowItDestroy(DBGFFLOWIT hFlowIt)
+{
+    PDBGFFLOWITINT pIt = hFlowIt;
+    AssertPtrReturnVoid(pIt);
+
+    for (unsigned i = 0; i < pIt->pFlow->cBbs; i++)
+        DBGFR3FlowBbRelease(pIt->apBb[i]);
+
+    DBGFR3FlowRelease(pIt->pFlow);
+    RTMemFree(pIt);
+}
+
+
+/**
+ * Returns the next basic block in the iterator or NULL if there is no
+ * basic block left.
+ *
+ * @returns Handle to the next basic block in the iterator or NULL if the end
+ *          was reached.
+ * @param   hFlowIt              The iterator handle.
+ *
+ * @note If a valid handle is returned it must be release with DBGFR3FlowBbRelease()
+ *       when not required anymore.
+ */
+VMMR3DECL(DBGFFLOWBB) DBGFR3FlowItNext(DBGFFLOWIT hFlowIt)
+{
+    PDBGFFLOWITINT pIt = hFlowIt;
+    AssertPtrReturn(pIt, NULL);
+
+    PDBGFFLOWBBINT pFlowBb = NULL;
+    if (pIt->idxBbNext < pIt->pFlow->cBbs)
+    {
+        pFlowBb = pIt->apBb[pIt->idxBbNext++];
+        DBGFR3FlowBbRetain(pFlowBb);
+    }
+
+    return pFlowBb;
+}
+
+
+/**
+ * Resets the given iterator to the beginning.
+ *
+ * @returns VBox status code.
+ * @param   hFlowIt              The iterator handle.
+ */
+VMMR3DECL(int) DBGFR3FlowItReset(DBGFFLOWIT hFlowIt)
+{
+    PDBGFFLOWITINT pIt = hFlowIt;
+    AssertPtrReturn(pIt, VERR_INVALID_HANDLE);
+
+    pIt->idxBbNext = 0;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNRTSORTCMP}
+ */
+static DECLCALLBACK(int) dbgfR3FlowBranchTblItSortCmp(void const *pvElement1, void const *pvElement2, void *pvUser)
+{
+    PDBGFFLOWITORDER penmOrder = (PDBGFFLOWITORDER)pvUser;
+    PDBGFFLOWBRANCHTBLINT pTbl1 = *(PDBGFFLOWBRANCHTBLINT *)pvElement1;
+    PDBGFFLOWBRANCHTBLINT pTbl2 = *(PDBGFFLOWBRANCHTBLINT *)pvElement2;
+
+    if (dbgfR3FlowAddrEqual(&pTbl1->AddrStart, &pTbl2->AddrStart))
+        return 0;
+
+    if (*penmOrder == DBGFFLOWITORDER_BY_ADDR_LOWEST_FIRST)
+    {
+        if (dbgfR3FlowAddrLower(&pTbl1->AddrStart, &pTbl2->AddrStart))
+            return -1;
+        else
+            return 1;
+    }
+    else
+    {
+        if (dbgfR3FlowAddrLower(&pTbl1->AddrStart, &pTbl2->AddrStart))
+            return 1;
+        else
+            return -1;
+    }
+}
+
+
+/**
+ * Creates a new branch table iterator for the given control flow graph.
+ *
+ * @returns VBox status code.
+ * @param   hFlow               The control flow graph handle.
+ * @param   enmOrder            The order in which the basic blocks are enumerated.
+ * @param   phFlowBranchTblIt   Where to store the handle to the iterator on success.
+ */
+VMMR3DECL(int) DBGFR3FlowBranchTblItCreate(DBGFFLOW hFlow, DBGFFLOWITORDER enmOrder,
+                                           PDBGFFLOWBRANCHTBLIT phFlowBranchTblIt)
+{
+    int rc = VINF_SUCCESS;
+    PDBGFFLOWINT pFlow = hFlow;
+    AssertPtrReturn(pFlow, VERR_INVALID_POINTER);
+    AssertPtrReturn(phFlowBranchTblIt, VERR_INVALID_POINTER);
+    AssertReturn(enmOrder > DBGFFLOWITORDER_INVALID && enmOrder < DBGFFLOWITORDER_BREADTH_FIRST,
+                 VERR_INVALID_PARAMETER);
+    AssertReturn(enmOrder < DBGFFLOWITORDER_DEPTH_FRIST, VERR_NOT_SUPPORTED);
+
+    PDBGFFLOWBRANCHTBLITINT pIt = (PDBGFFLOWBRANCHTBLITINT)RTMemAllocZ(RT_UOFFSETOF_DYN(DBGFFLOWBRANCHTBLITINT,
+                                                                                        apBranchTbl[pFlow->cBranchTbls]));
+    if (RT_LIKELY(pIt))
+    {
+        DBGFR3FlowRetain(hFlow);
+        pIt->pFlow      = pFlow;
+        pIt->idxTblNext = 0;
+        /* Fill the list and then sort. */
+        uint32_t idxTbl = 0;
+        PDBGFFLOWBRANCHTBLINT pFlowBranchTbl;
+        RTListForEach(&pFlow->LstBranchTbl, pFlowBranchTbl, DBGFFLOWBRANCHTBLINT, NdBranchTbl)
+        {
+            DBGFR3FlowBranchTblRetain(pFlowBranchTbl);
+            pIt->apBranchTbl[idxTbl++] = pFlowBranchTbl;
+        }
+
+        /* Sort the blocks by address. */
+        RTSortShell(&pIt->apBranchTbl[0], pFlow->cBranchTbls, sizeof(PDBGFFLOWBRANCHTBLINT), dbgfR3FlowBranchTblItSortCmp, &enmOrder);
+
+        *phFlowBranchTblIt = pIt;
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    return rc;
+}
+
+
+/**
+ * Destroys a given control flow graph branch table iterator.
+ *
+ * @returns nothing.
+ * @param   hFlowBranchTblIt              The control flow graph branch table iterator handle.
+ */
+VMMR3DECL(void) DBGFR3FlowBranchTblItDestroy(DBGFFLOWBRANCHTBLIT hFlowBranchTblIt)
+{
+    PDBGFFLOWBRANCHTBLITINT pIt = hFlowBranchTblIt;
+    AssertPtrReturnVoid(pIt);
+
+    for (unsigned i = 0; i < pIt->pFlow->cBranchTbls; i++)
+        DBGFR3FlowBranchTblRelease(pIt->apBranchTbl[i]);
+
+    DBGFR3FlowRelease(pIt->pFlow);
+    RTMemFree(pIt);
+}
+
+
+/**
+ * Returns the next branch table in the iterator or NULL if there is no
+ * branch table left.
+ *
+ * @returns Handle to the next basic block in the iterator or NULL if the end
+ *          was reached.
+ * @param   hFlowBranchTblIt    The iterator handle.
+ *
+ * @note If a valid handle is returned it must be release with DBGFR3FlowBranchTblRelease()
+ *       when not required anymore.
+ */
+VMMR3DECL(DBGFFLOWBRANCHTBL) DBGFR3FlowBranchTblItNext(DBGFFLOWBRANCHTBLIT hFlowBranchTblIt)
+{
+    PDBGFFLOWBRANCHTBLITINT pIt = hFlowBranchTblIt;
+    AssertPtrReturn(pIt, NULL);
+
+    PDBGFFLOWBRANCHTBLINT pTbl = NULL;
+    if (pIt->idxTblNext < pIt->pFlow->cBranchTbls)
+    {
+        pTbl = pIt->apBranchTbl[pIt->idxTblNext++];
+        DBGFR3FlowBranchTblRetain(pTbl);
+    }
+
+    return pTbl;
+}
+
+
+/**
+ * Resets the given iterator to the beginning.
+ *
+ * @returns VBox status code.
+ * @param   hFlowBranchTblIt    The iterator handle.
+ */
+VMMR3DECL(int) DBGFR3FlowBranchTblItReset(DBGFFLOWBRANCHTBLIT hFlowBranchTblIt)
+{
+    PDBGFFLOWBRANCHTBLITINT pIt = hFlowBranchTblIt;
+    AssertPtrReturn(pIt, VERR_INVALID_HANDLE);
+
+    pIt->idxTblNext = 0;
+    return VINF_SUCCESS;
+}
diff --git a/src/VBox/VMM/VMMR3/DBGFR3ModInMem.cpp b/src/VBox/VMM/VMMR3/DBGFR3ModInMem.cpp
new file mode 100644
index 00000000..9ec3696e
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFR3ModInMem.cpp
@@ -0,0 +1,1098 @@
+/* $Id: DBGFR3ModInMem.cpp $ */
+/** @file
+ * DBGFR3ModInMemPe - In memory PE module 'loader'.
+ */
+
+/*
+ * Copyright (C) 2009-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+
+#include <VBox/err.h>
+#include <iprt/ctype.h>
+#include <iprt/ldr.h>
+#include <iprt/mem.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+#include <iprt/sort.h>
+#include <iprt/formats/pecoff.h>
+#include <iprt/formats/mz.h>
+#include <iprt/formats/elf.h>
+#include <iprt/formats/mach-o.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/** Entry for mapping file offset to memory location. */
+typedef struct DBGFMODINMEMMAPPING
+{
+    /** The file offset. */
+    uint32_t        offFile;
+    /** The file size of this mapping. */
+    uint32_t        cbFile;
+    /** The size of this mapping. */
+    uint32_t        cbMem;
+    /** The offset to the memory from the start of the image.
+     * @note This can be negative (for mach_kernel).  */
+    int32_t         offMem;
+} DBGFMODINMEMMAPPING;
+typedef DBGFMODINMEMMAPPING *PDBGFMODINMEMMAPPING;
+typedef DBGFMODINMEMMAPPING const *PCDBGFMODINMEMMAPPING;
+
+/**
+ * Common in-memory reader instance data.
+ */
+typedef struct DBGFMODINMEMRDR
+{
+    /** The VM handle (referenced). */
+    PUVM                pUVM;
+    /** The image base. */
+    DBGFADDRESS         ImageAddr;
+    /** The file size, based on the offFile and cbFile of the last  mapping. */
+    uint32_t            cbFile;
+    /** Number of entries in the aMappings table. */
+    uint32_t            cMappings;
+    /** Mapping hint. */
+    uint32_t            iHint;
+    /** Mapping file offset to memory offsets, ordered by file offset. */
+    DBGFMODINMEMMAPPING aMappings[RT_FLEXIBLE_ARRAY_NESTED];
+} DBGFMODINMEMRDR;
+/** Pointer to the common instance data for an in-memory file reader. */
+typedef DBGFMODINMEMRDR *PDBGFMODINMEMRDR;
+
+/**
+ * The WinNT digger's loader reader instance data.
+ */
+typedef struct DBGFMODPERDR
+{
+    /** The VM handle (referenced). */
+    PUVM                pUVM;
+    /** The image base. */
+    DBGFADDRESS         ImageAddr;
+    /** The image size. */
+    uint32_t            cbImage;
+    /** The file offset of the SizeOfImage field in the optional header if it
+     *  needs patching, otherwise set to UINT32_MAX. */
+    uint32_t            offSizeOfImage;
+    /** The correct image size. */
+    uint32_t            cbCorrectImageSize;
+    /** Number of entries in the aMappings table. */
+    uint32_t            cMappings;
+    /** Mapping hint. */
+    uint32_t            iHint;
+    /** Mapping file offset to memory offsets, ordered by file offset. */
+    struct
+    {
+        /** The file offset. */
+        uint32_t        offFile;
+        /** The size of this mapping. */
+        uint32_t        cbMem;
+        /** The offset to the memory from the start of the image.  */
+        uint32_t        offMem;
+    } aMappings[1];
+} DBGFMODPERDR;
+/** Pointer a WinNT loader reader instance data. */
+typedef DBGFMODPERDR *PDBGFMODPERDR;
+
+/**
+ * Stack buffer.
+ */
+typedef union DBGFMODINMEMBUF
+{
+    uint8_t             ab[0x2000];
+    IMAGE_DOS_HEADER    DosHdr;
+    IMAGE_NT_HEADERS32  Nt32;
+    IMAGE_NT_HEADERS64  Nt64;
+    mach_header_64      MachoHdr;
+    DBGFMODINMEMMAPPING aMappings[0x2000 / sizeof(DBGFMODINMEMMAPPING)];
+} DBGFMODINMEMBUF;
+/** Pointer to stack buffer. */
+typedef DBGFMODINMEMBUF *PDBGFMODINMEMBUF;
+
+
+
+/**
+ * Normalizes a debug module name.
+ *
+ * @returns Normalized debug module name.
+ * @param   pszName         The name.
+ * @param   pszBuf          Buffer to use if work is needed.
+ * @param   cbBuf           Size of buffer.
+ */
+const char *dbgfR3ModNormalizeName(const char *pszName, char *pszBuf, size_t cbBuf)
+{
+    /*
+     * Skip to the filename in case someone gave us a full filename path.
+     */
+    pszName = RTPathFilenameEx(pszName, RTPATH_STR_F_STYLE_DOS);
+
+    /*
+     * Is it okay?
+     */
+    size_t cchName = strlen(pszName);
+    size_t off = 0;
+    for (;; off++)
+    {
+        char ch = pszName[off];
+        if (ch == '\0')
+            return pszName;
+        if (!RT_C_IS_ALNUM(ch) && ch != '_')
+            break;
+    }
+
+    /*
+     * It's no okay, so morph it.
+     */
+    if (cchName >= cbBuf)
+        cchName = cbBuf - 1;
+    for (off = 0; off < cchName; off++)
+    {
+        char ch = pszName[off];
+        if (!RT_C_IS_ALNUM(ch))
+            ch = '_';
+        pszBuf[off] = ch;
+    }
+    pszBuf[off] = '\0';
+
+    return pszBuf;
+}
+
+
+/**
+ * @callback_method_impl{PFNRTLDRRDRMEMREAD}
+ */
+static DECLCALLBACK(int) dbgfModInMemCommon_Read(void *pvBuf, size_t cb, size_t off, void *pvUser)
+{
+    PDBGFMODINMEMRDR pThis   = (PDBGFMODINMEMRDR)pvUser;
+    uint32_t         offFile = (uint32_t)off;
+    AssertReturn(offFile == off, VERR_INVALID_PARAMETER);
+
+    /*
+     * Set i to a mapping that starts at or before the specified offset.
+     * ASSUMING aMappings are sorted by offFile.
+     */
+    uint32_t i = pThis->iHint;
+    if (pThis->aMappings[i].offFile > offFile)
+    {
+        i = pThis->cMappings; /** @todo doesn't need to start from the end here... */
+        while (i-- > 0)
+            if (offFile >= pThis->aMappings[i].offFile)
+                break;
+        pThis->iHint = i;
+    }
+
+    while (cb > 0)
+    {
+        uint32_t offNextMap =  i + 1 < pThis->cMappings ? pThis->aMappings[i + 1].offFile
+                            : pThis->aMappings[i].offFile + RT_MAX(pThis->aMappings[i].cbFile, pThis->aMappings[i].cbMem);
+        uint32_t offMap     = offFile - pThis->aMappings[i].offFile;
+
+        /* Read file bits backed by memory. */
+        if (offMap < pThis->aMappings[i].cbMem)
+        {
+            uint32_t cbToRead = pThis->aMappings[i].cbMem - offMap;
+            if (cbToRead > cb)
+                cbToRead = (uint32_t)cb;
+
+            DBGFADDRESS Addr = pThis->ImageAddr;
+            DBGFR3AddrAdd(&Addr, pThis->aMappings[i].offMem + offMap);
+
+            int rc = DBGFR3MemRead(pThis->pUVM, 0 /*idCpu*/, &Addr, pvBuf, cbToRead);
+            if (RT_FAILURE(rc))
+                return rc;
+
+            /* Done? */
+            if (cbToRead == cb)
+                break;
+
+            offFile += cbToRead;
+            cb      -= cbToRead;
+            pvBuf    = (char *)pvBuf + cbToRead;
+        }
+
+        /* Mind the gap. */
+        if (offNextMap > offFile)
+        {
+            uint32_t cbZero = offNextMap - offFile;
+            if (cbZero > cb)
+            {
+                RT_BZERO(pvBuf, cb);
+                break;
+            }
+
+            RT_BZERO(pvBuf, cbZero);
+            offFile += cbZero;
+            cb      -= cbZero;
+            pvBuf   = (char *)pvBuf + cbZero;
+        }
+
+        pThis->iHint = ++i;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{PFNRTLDRRDRMEMDTOR}
+ */
+static DECLCALLBACK(void) dbgfModInMemCommon_Dtor(void *pvUser, size_t cbImage)
+{
+    PDBGFMODINMEMRDR pThis = (PDBGFMODINMEMRDR)pvUser;
+    RT_NOREF(cbImage);
+
+    VMR3ReleaseUVM(pThis->pUVM);
+    pThis->pUVM = NULL;
+
+    RTMemFree(pThis);
+}
+
+
+/**
+ * @callback_method_impl{FNRTSORTCMP}
+ */
+static DECLCALLBACK(int) dbgfModInMemCompMappings(void const *pvElement1, void const *pvElement2, void *pvUser)
+{
+    RT_NOREF(pvUser);
+    PCDBGFMODINMEMMAPPING pElement1 = (PCDBGFMODINMEMMAPPING)pvElement1;
+    PCDBGFMODINMEMMAPPING pElement2 = (PCDBGFMODINMEMMAPPING)pvElement2;
+    if (pElement1->offFile < pElement2->offFile)
+        return -1;
+    if (pElement1->offFile > pElement2->offFile)
+        return 1;
+    if (pElement1->cbFile < pElement2->cbFile)
+        return -1;
+    if (pElement1->cbFile > pElement2->cbFile)
+        return 1;
+    if (pElement1->offMem < pElement2->offMem)
+        return -1;
+    if (pElement1->offMem > pElement2->offMem)
+        return 1;
+    if (pElement1->cbMem < pElement2->cbMem)
+        return -1;
+    if (pElement1->cbMem > pElement2->cbMem)
+        return 1;
+    return 0;
+}
+
+
+static int dbgfModInMemCommon_Init(PDBGFMODINMEMRDR pThis, PUVM pUVM, PCDBGFADDRESS pImageAddr,PCDBGFMODINMEMMAPPING paMappings,
+                                   uint32_t cMappings, const char *pszName, RTLDRARCH enmArch,
+                                   PRTLDRMOD phLdrMod, PRTERRINFO pErrInfo)
+{
+    /*
+     * Initialize the reader instance.
+     */
+    VMR3RetainUVM(pUVM);
+    pThis->pUVM      = pUVM;
+    pThis->ImageAddr = *pImageAddr;
+    pThis->cMappings = cMappings;
+    pThis->iHint     = 0;
+    memcpy(pThis->aMappings, paMappings, cMappings * sizeof(pThis->aMappings[0]));
+    RTSortShell(pThis->aMappings, cMappings, sizeof(pThis->aMappings[0]), dbgfModInMemCompMappings, NULL);
+    pThis->cbFile    = pThis->aMappings[cMappings - 1].offFile + pThis->aMappings[cMappings - 1].cbFile;
+
+    /*
+     * Call the loader to open it.
+     * Note! destructore is always called.
+     */
+
+    RTLDRMOD hLdrMod;
+    int rc = RTLdrOpenInMemory(pszName, RTLDR_O_FOR_DEBUG, enmArch, pThis->cbFile,
+                               dbgfModInMemCommon_Read, dbgfModInMemCommon_Dtor, pThis,
+                               &hLdrMod, pErrInfo);
+    if (RT_SUCCESS(rc))
+        *phLdrMod = hLdrMod;
+    else
+        *phLdrMod = NIL_RTLDRMOD;
+    return rc;
+}
+
+
+/**
+ * Handles in-memory ELF images.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pImageAddr      The image address.
+ * @param   fFlags          Flags, DBGFMODINMEM_F_XXX.
+ * @param   pszName         The module name, optional.
+ * @param   pszFilename     The image filename, optional.
+ * @param   enmArch         The image arch if we force it, pass
+ *                          RTLDRARCH_WHATEVER if you don't care.
+ * @param   cbImage         Image size.  Pass 0 if not known.
+ * @param   puBuf           The header buffer.
+ * @param   phDbgMod        Where to return the resulting debug module on success.
+ * @param   pErrInfo        Where to return extended error info on failure.
+ */
+static int dbgfR3ModInMemElf(PUVM pUVM, PCDBGFADDRESS pImageAddr, uint32_t fFlags, const char *pszName, const char *pszFilename,
+                             RTLDRARCH enmArch, uint32_t cbImage, PDBGFMODINMEMBUF puBuf,
+                             PRTDBGMOD phDbgMod, PRTERRINFO pErrInfo)
+{
+    RT_NOREF(pUVM, fFlags, pszName, pszFilename, enmArch, cbImage, puBuf, phDbgMod);
+    return RTERRINFO_LOG_SET_F(pErrInfo, VERR_INVALID_EXE_SIGNATURE, "Found ELF magic at %RGv", pImageAddr->FlatPtr);
+}
+
+
+/**
+ * Handles in-memory Mach-O images.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pImageAddr      The image address.
+ * @param   fFlags          Flags, DBGFMODINMEM_F_XXX.
+ * @param   pszName         The module name, optional.
+ * @param   pszFilename     The image filename, optional.
+ * @param   enmArch         The image arch if we force it, pass
+ *                          RTLDRARCH_WHATEVER if you don't care.
+ * @param   cbImage         Image size.  Pass 0 if not known.
+ * @param   puBuf           The header buffer.
+ * @param   phDbgMod        Where to return the resulting debug module on success.
+ * @param   pErrInfo        Where to return extended error info on failure.
+ */
+static int dbgfR3ModInMemMachO(PUVM pUVM, PCDBGFADDRESS pImageAddr, uint32_t fFlags, const char *pszName, const char *pszFilename,
+                               RTLDRARCH enmArch, uint32_t cbImage, PDBGFMODINMEMBUF puBuf,
+                               PRTDBGMOD phDbgMod, PRTERRINFO pErrInfo)
+{
+    RT_NOREF(cbImage, fFlags);
+
+    /*
+     * Match up enmArch.
+     */
+    if (enmArch == RTLDRARCH_AMD64)
+    {
+        if (puBuf->MachoHdr.magic != IMAGE_MACHO64_SIGNATURE)
+            return RTERRINFO_LOG_SET_F(pErrInfo, VERR_LDR_ARCH_MISMATCH, "Wanted AMD64 but header is not 64-bit");
+        if (puBuf->MachoHdr.cputype != CPU_TYPE_X86_64)
+            return RTERRINFO_LOG_SET_F(pErrInfo, VERR_LDR_ARCH_MISMATCH, "Wanted AMD64 but cpu type is %#x instead of %#x",
+                                       puBuf->MachoHdr.cputype, CPU_TYPE_X86_64);
+    }
+    else if (enmArch == RTLDRARCH_X86_32)
+    {
+        if (puBuf->MachoHdr.magic != IMAGE_MACHO32_SIGNATURE)
+            return RTERRINFO_LOG_SET_F(pErrInfo, VERR_LDR_ARCH_MISMATCH, "Wanted X86_32 but header is not 32-bit");
+        if (puBuf->MachoHdr.cputype != CPU_TYPE_X86)
+            return RTERRINFO_LOG_SET_F(pErrInfo, VERR_LDR_ARCH_MISMATCH, "Wanted X86_32 but cpu type is %#x instead of %#x",
+                                       puBuf->MachoHdr.cputype, CPU_TYPE_X86);
+    }
+    else if (enmArch != RTLDRARCH_WHATEVER)
+        return RTERRINFO_LOG_SET_F(pErrInfo, VERR_LDR_ARCH_MISMATCH, "Unsupported enmArch value %s (%d)",
+                                   RTLdrArchName(enmArch), enmArch);
+
+    /*
+     * Guess the module name if not specified and make sure it conforms to DBGC expectations.
+     */
+    char szNormalized[128];
+    if (!pszName)
+    {
+        if (pszFilename)
+            pszName = RTPathFilenameEx(pszFilename, RTPATH_STR_F_STYLE_DOS /*whatever*/);
+        if (!pszName)
+        {
+            RTStrPrintf(szNormalized, sizeof(szNormalized), "image_%#llx", (uint64_t)pImageAddr->FlatPtr);
+            pszName = szNormalized;
+        }
+    }
+    if (pszName != szNormalized)
+        pszName = dbgfR3ModNormalizeName(pszName, szNormalized, sizeof(szNormalized));
+
+    /*
+     * Read the load commands into memory, they follow the header.  Refuse
+     * if there appear to be too many or too much of these.
+     */
+    uint32_t const cLoadCmds  = puBuf->MachoHdr.ncmds;
+    uint32_t const cbLoadCmds = puBuf->MachoHdr.sizeofcmds;
+    if (cLoadCmds > _8K || cLoadCmds < 2)
+        return RTERRINFO_LOG_SET_F(pErrInfo, VERR_LDRMACHO_BAD_HEADER,
+                                   "ncmds=%u is out of sensible range (2..8192)", cLoadCmds);
+    if (cbLoadCmds > _2M || cbLoadCmds < sizeof(load_command_t) * 2)
+        return RTERRINFO_LOG_SET_F(pErrInfo, VERR_LDRMACHO_BAD_HEADER,
+                                   "cbLoadCmds=%#x is out of sensible range (8..2MiB)", cbLoadCmds);
+
+    uint8_t *pbLoadCmds = (uint8_t *)RTMemTmpAllocZ(cbLoadCmds);
+    AssertReturn(pbLoadCmds, VERR_NO_TMP_MEMORY);
+
+    uint32_t const cbHdr = puBuf->MachoHdr.magic == IMAGE_MACHO64_SIGNATURE ? sizeof(mach_header_64) : sizeof(mach_header_32);
+    DBGFADDRESS Addr = *pImageAddr;
+    int rc = DBGFR3MemRead(pUVM, 0 /*idCpu*/, DBGFR3AddrAdd(&Addr, cbHdr), pbLoadCmds, cbLoadCmds);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Scan it for segments so we can tranlate file offsets to virtual
+         * memory locations.
+         */
+        RTUUID   Uuid = RTUUID_INITIALIZE_NULL;
+        uint32_t cMappings = 0;
+        uint32_t offCmd = 0;
+        for (uint32_t iCmd = 0; iCmd < cLoadCmds; iCmd++)
+        {
+            load_command_t const *pCurCmd = (load_command_t const *)&pbLoadCmds[offCmd];
+            uint32_t       const  cbCurCmd = offCmd + sizeof(*pCurCmd) <= cbLoadCmds ? pCurCmd->cmdsize : sizeof(*pCurCmd);
+            if (offCmd + cbCurCmd > cbLoadCmds)
+                rc = RTERRINFO_LOG_SET_F(pErrInfo, VERR_LDRMACHO_BAD_LOAD_COMMAND,
+                                         "Load command #%u @ %#x is out of bounds: size %#x, left %#x", iCmd, offCmd, cbCurCmd,
+                                         cbLoadCmds - offCmd);
+            else if (pCurCmd->cmd == LC_SEGMENT_64)
+            {
+                segment_command_64 const *pSeg = (segment_command_64 const *)pCurCmd;
+                if (cbCurCmd >= sizeof(*pSeg))
+                {
+                    if (cMappings >= RT_ELEMENTS(puBuf->aMappings))
+                        rc = RTERRINFO_LOG_SET_F(pErrInfo, VERR_OUT_OF_RANGE, "Too many segments!");
+                    else
+                    {
+                        puBuf->aMappings[cMappings].offFile = pSeg->fileoff;
+                        puBuf->aMappings[cMappings].cbFile  = pSeg->filesize;
+                        puBuf->aMappings[cMappings].offMem  = pSeg->vmaddr - pImageAddr->FlatPtr;
+                        puBuf->aMappings[cMappings].cbMem   = pSeg->vmsize;
+                        cMappings++;
+                    }
+                }
+                else
+                    rc = RTERRINFO_LOG_SET_F(pErrInfo, VERR_LDRMACHO_BAD_LOAD_COMMAND,
+                                             "Load command #%u @ %#x is too small for a 64-bit segment: %#x", iCmd, offCmd, cbCurCmd);
+            }
+            else if (pCurCmd->cmd == LC_SEGMENT_32)
+            {
+                segment_command_32 const *pSeg = (segment_command_32 const *)pCurCmd;
+                if (cbCurCmd >= sizeof(*pSeg))
+                {
+                    if (cMappings >= RT_ELEMENTS(puBuf->aMappings))
+                        rc = RTERRINFO_LOG_SET_F(pErrInfo, VERR_OUT_OF_RANGE, "Too many segments!");
+                    else
+                    {
+                        puBuf->aMappings[cMappings].offFile = pSeg->fileoff;
+                        puBuf->aMappings[cMappings].cbFile  = pSeg->filesize;
+                        puBuf->aMappings[cMappings].offMem  = pSeg->vmaddr - pImageAddr->FlatPtr;
+                        puBuf->aMappings[cMappings].cbMem   = pSeg->vmsize;
+                        cMappings++;
+                    }
+                }
+                else
+                    rc = RTERRINFO_LOG_SET_F(pErrInfo, VERR_LDRMACHO_BAD_LOAD_COMMAND,
+                                             "Load command #%u @ %#x is too small for a 32-bit segment: %#x", iCmd, offCmd, cbCurCmd);
+            }
+            else if (pCurCmd->cmd == LC_UUID && cbCurCmd == sizeof(uuid_command_t))
+                memcpy(&Uuid, ((uuid_command_t const *)pCurCmd)->uuid, sizeof(Uuid));
+
+            if (RT_SUCCESS(rc))
+                offCmd += cbCurCmd;
+            else
+                break;
+        } /* for each command */
+
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Create generic loader module instance (pThis is tied to it
+             * come rain come shine).
+             */
+            PDBGFMODINMEMRDR pThis = (PDBGFMODINMEMRDR)RTMemAllocZVar(RT_UOFFSETOF_DYN(DBGFMODINMEMRDR, aMappings[cMappings]));
+            if (pThis)
+            {
+                RTLDRMOD hLdrMod;
+                rc = dbgfModInMemCommon_Init(pThis, pUVM, pImageAddr, puBuf->aMappings, cMappings,
+                                             pszName, enmArch, &hLdrMod, pErrInfo);
+                if (RT_FAILURE(rc))
+                    hLdrMod = NIL_RTLDRMOD;
+
+                RTDBGMOD hMod;
+                rc = RTDbgModCreateFromMachOImage(&hMod, pszFilename ? pszFilename : pszName, pszName, enmArch,
+                                                  &hLdrMod, 0 /*cbImage*/, 0, NULL, &Uuid, DBGFR3AsGetConfig(pUVM), fFlags);
+                if (RT_SUCCESS(rc))
+                    *phDbgMod = hMod;
+#if 0 /** @todo later */
+                else if (!(fFlags & DBGFMODINMEM_F_NO_CONTAINER_FALLBACK))
+                {
+                    /*
+                     * Fallback is a container module.
+                     */
+                    rc = RTDbgModCreate(&hMod, pszName, cbImage, 0);
+                    if (RT_SUCCESS(rc))
+                    {
+                        rc = RTDbgModSymbolAdd(hMod, "Headers", 0 /*iSeg*/, 0, cbImage, 0 /*fFlags*/, NULL);
+                        AssertRC(rc);
+                    }
+                }
+#endif
+                if (hLdrMod != NIL_RTLDRMOD)
+                    RTLdrClose(hLdrMod);
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+    }
+    else
+        RTERRINFO_LOG_SET_F(pErrInfo, rc, "Failed to read %#x bytes of load commands", cbLoadCmds);
+    RTMemTmpFree(pbLoadCmds);
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{PFNRTLDRRDRMEMREAD}
+ */
+static DECLCALLBACK(int) dbgfModInMemPeRdr_Read(void *pvBuf, size_t cb, size_t off, void *pvUser)
+{
+    PDBGFMODPERDR pThis   = (PDBGFMODPERDR)pvUser;
+    uint32_t      offFile = (uint32_t)off;
+    AssertReturn(offFile == off, VERR_INVALID_PARAMETER);
+
+    uint32_t i = pThis->iHint;
+    if (pThis->aMappings[i].offFile > offFile)
+    {
+        i = pThis->cMappings;
+        while (i-- > 0)
+            if (offFile >= pThis->aMappings[i].offFile)
+                break;
+        pThis->iHint = i;
+    }
+
+    while (cb > 0)
+    {
+        uint32_t offNextMap =  i + 1 < pThis->cMappings ? pThis->aMappings[i + 1].offFile : pThis->cbImage;
+        uint32_t offMap     = offFile - pThis->aMappings[i].offFile;
+
+        /* Read file bits backed by memory. */
+        if (offMap < pThis->aMappings[i].cbMem)
+        {
+            uint32_t cbToRead = pThis->aMappings[i].cbMem - offMap;
+            if (cbToRead > cb)
+                cbToRead = (uint32_t)cb;
+
+            DBGFADDRESS Addr = pThis->ImageAddr;
+            DBGFR3AddrAdd(&Addr, pThis->aMappings[i].offMem + offMap);
+
+            int rc = DBGFR3MemRead(pThis->pUVM, 0 /*idCpu*/, &Addr, pvBuf, cbToRead);
+            if (RT_FAILURE(rc))
+                return rc;
+
+            /* Apply SizeOfImage patch? */
+            if (   pThis->offSizeOfImage != UINT32_MAX
+                && offFile            < pThis->offSizeOfImage + 4
+                && offFile + cbToRead > pThis->offSizeOfImage)
+            {
+                uint32_t SizeOfImage = pThis->cbCorrectImageSize;
+                uint32_t cbPatch     = sizeof(SizeOfImage);
+                int32_t  offPatch    = pThis->offSizeOfImage - offFile;
+                uint8_t *pbPatch     = (uint8_t *)pvBuf + offPatch;
+                if (offFile + cbToRead < pThis->offSizeOfImage + cbPatch)
+                    cbPatch = offFile + cbToRead - pThis->offSizeOfImage;
+                while (cbPatch-- > 0)
+                {
+                    if (offPatch >= 0)
+                        *pbPatch = (uint8_t)SizeOfImage;
+                    offPatch++;
+                    pbPatch++;
+                    SizeOfImage >>= 8;
+                }
+            }
+
+            /* Done? */
+            if (cbToRead == cb)
+                break;
+
+            offFile += cbToRead;
+            cb      -= cbToRead;
+            pvBuf    = (char *)pvBuf + cbToRead;
+        }
+
+        /* Mind the gap. */
+        if (offNextMap > offFile)
+        {
+            uint32_t cbZero = offNextMap - offFile;
+            if (cbZero > cb)
+            {
+                RT_BZERO(pvBuf, cb);
+                break;
+            }
+
+            RT_BZERO(pvBuf, cbZero);
+            offFile += cbZero;
+            cb      -= cbZero;
+            pvBuf   = (char *)pvBuf + cbZero;
+        }
+
+        pThis->iHint = ++i;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{PFNRTLDRRDRMEMDTOR}
+ */
+static DECLCALLBACK(void) dbgfModInMemPeRdr_Dtor(void *pvUser, size_t cbImage)
+{
+    PDBGFMODPERDR pThis = (PDBGFMODPERDR)pvUser;
+    RT_NOREF(cbImage);
+
+    VMR3ReleaseUVM(pThis->pUVM);
+    pThis->pUVM = NULL;
+    RTMemFree(pvUser);
+}
+
+
+/**
+ * Checks if the section headers look okay.
+ *
+ * @returns VBox status code.
+ * @param   paShdrs             Pointer to the section headers.
+ * @param   cShdrs              Number of headers.
+ * @param   cbImage             The image size reported by NT.
+ * @param   cbImageFromHdr      The image size by the linker in the header.
+ * @param   uRvaRsrc            The RVA of the resource directory. UINT32_MAX if
+ *                              no resource directory.
+ * @param   cbSectAlign         The section alignment specified in the header.
+ * @param   fNt31               Set if NT 3.1.  Needed for chopped off HAL.
+ * @param   pcbImageCorrect     The corrected image size.  This is derived from
+ *                              cbImage and virtual range of the section tables.
+ *
+ *                              The problem is that NT may choose to drop the
+ *                              last pages in images it loads early, starting at
+ *                              the resource directory.  These images will have
+ *                              a page aligned cbImage.
+ *
+ * @param   pErrInfo            Where to return more error details.
+ */
+static int dbgfR3ModPeCheckSectHdrsAndImgSize(PCIMAGE_SECTION_HEADER paShdrs, uint32_t cShdrs, uint32_t cbImage,
+                                              uint32_t cbImageFromHdr, uint32_t uRvaRsrc, uint32_t cbSectAlign,
+                                              bool fNt31, uint32_t *pcbImageCorrect, PRTERRINFO pErrInfo)
+{
+    *pcbImageCorrect = cbImage;
+
+    for (uint32_t i = 0; i < cShdrs; i++)
+    {
+        if (!paShdrs[i].Name[0])
+            return RTERRINFO_LOG_SET_F(pErrInfo, VERR_BAD_EXE_FORMAT, "Section header #%u has no name", i);
+
+        if (paShdrs[i].Characteristics & IMAGE_SCN_TYPE_NOLOAD)
+            continue;
+
+        /* Tweak to determine the virtual size if the linker didn't set it (NT 3.1). */
+        /** @todo this isn't really perfect. cbImage is kind of wrong...   */
+        uint32_t cbVirtual = paShdrs[i].Misc.VirtualSize;
+        if (cbVirtual == 0)
+        {
+            for (uint32_t j = i + 1; j < cShdrs; j++)
+                if (   !(paShdrs[j].Characteristics & IMAGE_SCN_TYPE_NOLOAD)
+                    && paShdrs[j].VirtualAddress > paShdrs[i].VirtualAddress)
+                {
+                    cbVirtual = paShdrs[j].VirtualAddress - paShdrs[i].VirtualAddress;
+                    break;
+                }
+            if (!cbVirtual)
+            {
+                if (paShdrs[i].VirtualAddress < cbImageFromHdr)
+                    cbVirtual = cbImageFromHdr - paShdrs[i].VirtualAddress;
+                else if (paShdrs[i].SizeOfRawData > 0)
+                    cbVirtual = RT_ALIGN(paShdrs[i].SizeOfRawData, _4K);
+            }
+        }
+
+        /* Check that sizes are within the same range and that both sizes and
+           addresses are within reasonable limits. */
+        if (   RT_ALIGN(cbVirtual, _64K) < RT_ALIGN(paShdrs[i].SizeOfRawData, _64K)
+            || cbVirtual                >= _1G
+            || paShdrs[i].SizeOfRawData >= _1G)
+            return RTERRINFO_LOG_SET_F(pErrInfo, VERR_BAD_EXE_FORMAT,
+                                       "Section header #%u (%.8s) has a VirtualSize=%#x (%#x) and SizeOfRawData=%#x, that's too much data!",
+                                       i, paShdrs[i].Name, cbVirtual, paShdrs[i].Misc.VirtualSize, paShdrs[i].SizeOfRawData);
+        uint32_t uRvaEnd = paShdrs[i].VirtualAddress + cbVirtual;
+        if (uRvaEnd >= _1G || uRvaEnd < paShdrs[i].VirtualAddress)
+            return RTERRINFO_LOG_SET_F(pErrInfo, VERR_BAD_EXE_FORMAT,
+                                       "Section header #%u (%.8s) has a VirtualSize=%#x (%#x) and VirtualAddr=%#x, %#x in total, that's too much!",
+                                       i, paShdrs[i].Name, cbVirtual, paShdrs[i].Misc.VirtualSize, paShdrs[i].VirtualAddress, uRvaEnd);
+
+        /* Check for images chopped off around '.rsrc'. */
+        if (    cbImage < uRvaEnd
+            &&  uRvaEnd >= uRvaRsrc)
+            cbImage = RT_ALIGN(uRvaEnd, cbSectAlign);
+
+        /* Check that the section is within the image. */
+        if (uRvaEnd > cbImage && fNt31)
+            return RTERRINFO_LOG_SET_F(pErrInfo, VERR_BAD_EXE_FORMAT,
+                                       "Section header #%u has a virtual address range beyond the image: %#x TO %#x cbImage=%#x",
+                                       i, paShdrs[i].VirtualAddress, uRvaEnd, cbImage);
+    }
+
+    Assert(*pcbImageCorrect == cbImage || !(*pcbImageCorrect & 0xfff));
+    *pcbImageCorrect = cbImage;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Create a loader module for the in-guest-memory PE module.
+ */
+static int dbgfR3ModInMemPeCreateLdrMod(PUVM pUVM, uint32_t fFlags, const char *pszName, PCDBGFADDRESS pImageAddr,
+                                        uint32_t cbImage, uint32_t cbImageFromHdr, bool f32Bit,
+                                        uint32_t cShdrs, PCIMAGE_SECTION_HEADER paShdrs, uint32_t cbSectAlign,
+                                        uint32_t cDataDir, PCIMAGE_DATA_DIRECTORY paDataDir, uint32_t offHdrs,
+                                        PRTLDRMOD phLdrMod, PRTERRINFO pErrInfo)
+{
+    /*
+     * Allocate and create a reader instance.
+     */
+    PDBGFMODPERDR pRdr = (PDBGFMODPERDR)RTMemAlloc(RT_UOFFSETOF_DYN(DBGFMODPERDR, aMappings[cShdrs + 2]));
+    if (!pRdr)
+        return VERR_NO_MEMORY;
+
+    VMR3RetainUVM(pUVM);
+    pRdr->pUVM               = pUVM;
+    pRdr->ImageAddr          = *pImageAddr;
+    pRdr->cbImage            = cbImage;
+    pRdr->cbCorrectImageSize = cbImage;
+    pRdr->offSizeOfImage     = UINT32_MAX;
+    pRdr->iHint              = 0;
+
+    /*
+     * Use the section table to construct a more accurate view of the file/image.
+     */
+    uint32_t uRvaRsrc = UINT32_MAX;
+    if (   cDataDir > IMAGE_DIRECTORY_ENTRY_RESOURCE
+        && paDataDir[IMAGE_DIRECTORY_ENTRY_RESOURCE].Size > 0)
+        uRvaRsrc = paDataDir[IMAGE_DIRECTORY_ENTRY_RESOURCE].VirtualAddress;
+
+    int rc = dbgfR3ModPeCheckSectHdrsAndImgSize(paShdrs, cShdrs, cbImage, cbImageFromHdr, uRvaRsrc, cbSectAlign,
+                                                RT_BOOL(fFlags & DBGFMODINMEM_F_PE_NT31), &pRdr->cbCorrectImageSize, pErrInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pRdr->cMappings = 0;
+
+        for (uint32_t i = 0; i < cShdrs; i++)
+            if (   paShdrs[i].SizeOfRawData    > 0
+                && paShdrs[i].PointerToRawData > 0)
+            {
+                uint32_t j = 1;
+                if (!pRdr->cMappings)
+                    pRdr->cMappings++;
+                else
+                {
+                    while (j < pRdr->cMappings && pRdr->aMappings[j].offFile < paShdrs[i].PointerToRawData)
+                        j++;
+                    if (j < pRdr->cMappings)
+                        memmove(&pRdr->aMappings[j + 1], &pRdr->aMappings[j], (pRdr->cMappings - j) * sizeof(pRdr->aMappings));
+                }
+                pRdr->aMappings[j].offFile = paShdrs[i].PointerToRawData;
+                pRdr->aMappings[j].offMem  = paShdrs[i].VirtualAddress;
+                pRdr->aMappings[j].cbMem   = i + 1 < cShdrs
+                                           ? paShdrs[i + 1].VirtualAddress - paShdrs[i].VirtualAddress
+                                           : paShdrs[i].Misc.VirtualSize;
+                if (j == pRdr->cMappings)
+                    pRdr->cbImage = paShdrs[i].PointerToRawData + paShdrs[i].SizeOfRawData;
+                pRdr->cMappings++;
+            }
+
+        /* Insert the mapping of the headers that isn't covered by the section table. */
+        pRdr->aMappings[0].offFile = 0;
+        pRdr->aMappings[0].offMem  = 0;
+        pRdr->aMappings[0].cbMem   = pRdr->cMappings ? pRdr->aMappings[1].offFile : pRdr->cbImage;
+
+        int j = pRdr->cMappings - 1;
+        while (j-- > 0)
+        {
+            uint32_t cbFile = pRdr->aMappings[j + 1].offFile - pRdr->aMappings[j].offFile;
+            if (pRdr->aMappings[j].cbMem > cbFile)
+                pRdr->aMappings[j].cbMem = cbFile;
+        }
+    }
+    else if (fFlags & DBGFMODINMEM_F_NO_READER_FALLBACK)
+        return rc;
+    else
+    {
+        /*
+         * Fallback, fake identity mapped file data.
+         */
+        pRdr->cMappings = 1;
+        pRdr->aMappings[0].offFile = 0;
+        pRdr->aMappings[0].offMem  = 0;
+        pRdr->aMappings[0].cbMem   = pRdr->cbImage;
+    }
+
+    /* Enable the SizeOfImage patching if necessary. */
+    if (pRdr->cbCorrectImageSize != cbImage)
+    {
+        Log(("dbgfR3ModInMemPeCreateLdrMod: The image is really %#x bytes long, not %#x as mapped by NT!\n",
+             pRdr->cbCorrectImageSize, cbImage));
+        pRdr->offSizeOfImage = f32Bit
+                             ? offHdrs + RT_OFFSETOF(IMAGE_NT_HEADERS32, OptionalHeader.SizeOfImage)
+                             : offHdrs + RT_OFFSETOF(IMAGE_NT_HEADERS64, OptionalHeader.SizeOfImage);
+    }
+
+    /*
+     * Call the loader to open the PE image for debugging.
+     * Note! It always calls pfnDtor.
+     */
+    RTLDRMOD hLdrMod;
+    rc = RTLdrOpenInMemory(pszName, RTLDR_O_FOR_DEBUG, RTLDRARCH_WHATEVER, pRdr->cbImage,
+                           dbgfModInMemPeRdr_Read, dbgfModInMemPeRdr_Dtor, pRdr,
+                           &hLdrMod, pErrInfo);
+    if (RT_SUCCESS(rc))
+        *phLdrMod = hLdrMod;
+    else
+        *phLdrMod = NIL_RTLDRMOD;
+    return rc;
+}
+
+
+/**
+ * Handles in-memory PE images.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pImageAddr      The image address.
+ * @param   fFlags          Flags, DBGFMODINMEM_F_XXX.
+ * @param   pszName         The module name, optional.
+ * @param   pszFilename     The image filename, optional.
+ * @param   enmArch         The image arch if we force it, pass
+ *                          RTLDRARCH_WHATEVER if you don't care.
+ * @param   cbImage         Image size.  Pass 0 if not known.
+ * @param   offPeHdrs       Offset of the PE header.
+ * @param   cbPeHdrsPart1   How read into uBuf at @a offPeHdrs.
+ * @param   puBuf           The header buffer.
+ * @param   phDbgMod        Where to return the resulting debug module on success.
+ * @param   pErrInfo        Where to return extended error info on failure.
+ */
+static int dbgfR3ModInMemPe(PUVM pUVM, PCDBGFADDRESS pImageAddr, uint32_t fFlags, const char *pszName, const char *pszFilename,
+                            RTLDRARCH enmArch, uint32_t cbImage, uint32_t offPeHdrs, uint32_t cbPeHdrsPart1,
+                            PDBGFMODINMEMBUF puBuf, PRTDBGMOD phDbgMod, PRTERRINFO pErrInfo)
+{
+    /*
+     * Read the optional header and the section table after validating the
+     * info we need from the file header.
+     */
+    /* Check the opt hdr size and number of sections as these are used to determine how much to read next. */
+    if (   puBuf->Nt32.FileHeader.SizeOfOptionalHeader < sizeof(IMAGE_OPTIONAL_HEADER32)
+        || puBuf->Nt32.FileHeader.SizeOfOptionalHeader > sizeof(IMAGE_OPTIONAL_HEADER64) + 128)
+        return RTERRINFO_LOG_SET_F(pErrInfo, VERR_BAD_EXE_FORMAT, "Invalid SizeOfOptionalHeader value: %#RX32",
+                                   puBuf->Nt32.FileHeader.SizeOfOptionalHeader);
+
+    if (   puBuf->Nt32.FileHeader.NumberOfSections < 1
+        || puBuf->Nt32.FileHeader.NumberOfSections > 190 /* what fits in our 8K buffer */)
+        return RTERRINFO_LOG_SET_F(pErrInfo, VERR_BAD_EXE_FORMAT, "NumberOfSections is out of range: %#RX32 (1..190)",
+                                   puBuf->Nt32.FileHeader.NumberOfSections);
+
+    /* Read the optional header and section table. */
+    uint32_t const cbHdrs = RT_UOFFSETOF(IMAGE_NT_HEADERS32, OptionalHeader)
+                          + puBuf->Nt32.FileHeader.SizeOfOptionalHeader
+                          + puBuf->Nt32.FileHeader.NumberOfSections * sizeof(IMAGE_SECTION_HEADER);
+    AssertReturn(cbHdrs <= sizeof(*puBuf), RTERRINFO_LOG_SET_F(pErrInfo, VERR_INTERNAL_ERROR_2, "cbHdrs=%#x", cbHdrs));
+
+    DBGFADDRESS PeHdrPart2Addr = *pImageAddr;
+    DBGFR3AddrAdd(&PeHdrPart2Addr, offPeHdrs + cbPeHdrsPart1);
+    int rc = DBGFR3MemRead(pUVM, 0 /*idCpu*/, &PeHdrPart2Addr, &puBuf->ab[cbPeHdrsPart1], cbHdrs - cbPeHdrsPart1);
+    if (RT_FAILURE(rc))
+        return RTERRINFO_LOG_SET_F(pErrInfo, rc,
+                                   "Failed to read the second part of the PE headers at %RGv (off=%#RX32 + %#RX32): %Rrc",
+                                   PeHdrPart2Addr.FlatPtr, offPeHdrs, cbPeHdrsPart1, rc);
+
+    /*
+     * Check the image architecture and determine the bitness.
+     */
+    RTLDRARCH enmArchActual;
+    bool f32Bit;
+    switch (puBuf->Nt32.FileHeader.Machine)
+    {
+        case IMAGE_FILE_MACHINE_I386:
+            enmArchActual = RTLDRARCH_X86_32;
+            f32Bit = true;
+            break;
+        case IMAGE_FILE_MACHINE_AMD64:
+            enmArchActual = RTLDRARCH_AMD64;
+            f32Bit = false;
+            break;
+        case IMAGE_FILE_MACHINE_ARM:
+        case IMAGE_FILE_MACHINE_THUMB:
+        case IMAGE_FILE_MACHINE_ARMNT:
+            enmArchActual = RTLDRARCH_ARM32;
+            f32Bit = true;
+            break;
+        case IMAGE_FILE_MACHINE_ARM64:
+            enmArchActual = RTLDRARCH_ARM64;
+            f32Bit = false;
+            break;
+        default:
+            return RTERRINFO_LOG_SET_F(pErrInfo, VERR_LDR_ARCH_MISMATCH, "Unknown machine: %#x", puBuf->Nt32.FileHeader.Machine);
+    }
+    if (   enmArch != RTLDRARCH_WHATEVER
+        && enmArch != enmArchActual)
+        return RTERRINFO_LOG_SET_F(pErrInfo, VERR_LDR_ARCH_MISMATCH, "Found %s expected %s",
+                                   RTLdrArchName(enmArchActual), RTLdrArchName(enmArch));
+
+    /*
+     * Check optional header magic and size.
+     */
+    uint16_t const uOptMagic = f32Bit ? IMAGE_NT_OPTIONAL_HDR32_MAGIC : IMAGE_NT_OPTIONAL_HDR64_MAGIC;
+    if (puBuf->Nt32.OptionalHeader.Magic != uOptMagic)
+        return RTERRINFO_LOG_SET_F(pErrInfo, VERR_BAD_EXE_FORMAT, "Unexpected optional header magic: %#x (expected %#x)",
+                                   puBuf->Nt32.OptionalHeader.Magic, uOptMagic);
+
+    uint32_t const cDataDir = f32Bit ? puBuf->Nt32.OptionalHeader.NumberOfRvaAndSizes : puBuf->Nt64.OptionalHeader.NumberOfRvaAndSizes;
+    if (   cDataDir <= IMAGE_DIRECTORY_ENTRY_BASERELOC /* a bit random */
+        || cDataDir > 32 /* also random */)
+        return RTERRINFO_LOG_SET_F(pErrInfo, VERR_BAD_EXE_FORMAT, "Unexpected data directory size: %#x", cDataDir);
+
+    uint32_t cbOptHdr = f32Bit ? sizeof(IMAGE_OPTIONAL_HEADER32) : sizeof(IMAGE_OPTIONAL_HEADER64);
+    cbOptHdr -= sizeof(IMAGE_DATA_DIRECTORY) * IMAGE_NUMBEROF_DIRECTORY_ENTRIES;
+    cbOptHdr += sizeof(IMAGE_DATA_DIRECTORY) * cDataDir;
+    if (puBuf->Nt32.FileHeader.SizeOfOptionalHeader != cbOptHdr)
+        return RTERRINFO_LOG_SET_F(pErrInfo, VERR_BAD_EXE_FORMAT, "Unexpected optional header size: %#x (expected %#x)",
+                                   puBuf->Nt32.FileHeader.SizeOfOptionalHeader, cbOptHdr);
+
+    uint32_t const cbSectAlign = f32Bit ? puBuf->Nt32.OptionalHeader.SectionAlignment : puBuf->Nt64.OptionalHeader.SectionAlignment;
+    PCIMAGE_SECTION_HEADER pSHdrs    = (PCIMAGE_SECTION_HEADER)((uintptr_t)&puBuf->Nt32.OptionalHeader + cbOptHdr);
+    PCIMAGE_DATA_DIRECTORY paDataDir = (PCIMAGE_DATA_DIRECTORY)((uintptr_t)pSHdrs - cDataDir * sizeof(IMAGE_DATA_DIRECTORY));
+
+    /*
+     * Establish the image size.
+     */
+    uint32_t cbImageFromHdr = f32Bit ? puBuf->Nt32.OptionalHeader.SizeOfImage : puBuf->Nt64.OptionalHeader.SizeOfImage;
+    if (   !cbImage
+        || (fFlags & DBGFMODINMEM_F_PE_NT31))
+        cbImage = RT_ALIGN(cbImageFromHdr, _4K);
+    else if (RT_ALIGN(cbImageFromHdr, _4K) != RT_ALIGN(cbImage, _4K))
+        return RTERRINFO_LOG_SET_F(pErrInfo, VERR_MISMATCH, "Image size mismatch: input=%#x header=%#x", cbImage, cbImageFromHdr);
+
+    /*
+     * Guess the module name if not specified and make sure it conforms to DBGC expectations.
+     */
+    if (!pszName)
+    {
+        if (pszFilename)
+            pszName = RTPathFilenameEx(pszFilename, RTPATH_STR_F_STYLE_DOS);
+        /** @todo */
+    }
+
+    char szNormalized[128];
+    pszName = dbgfR3ModNormalizeName(pszName, szNormalized, sizeof(szNormalized));
+
+    /*
+     * Create the module using the in memory image first, falling back on cached image.
+     */
+    RTLDRMOD hLdrMod;
+    rc = dbgfR3ModInMemPeCreateLdrMod(pUVM, fFlags, pszName, pImageAddr, cbImage, cbImageFromHdr, f32Bit,
+                                      puBuf->Nt32.FileHeader.NumberOfSections, pSHdrs, cbSectAlign, cDataDir, paDataDir,
+                                      offPeHdrs, &hLdrMod, pErrInfo);
+    if (RT_FAILURE(rc))
+        hLdrMod = NIL_RTLDRMOD;
+
+    RTDBGMOD hMod;
+    rc = RTDbgModCreateFromPeImage(&hMod, pszFilename, pszName, &hLdrMod, cbImageFromHdr,
+                                   puBuf->Nt32.FileHeader.TimeDateStamp, DBGFR3AsGetConfig(pUVM));
+    if (RT_SUCCESS(rc))
+        *phDbgMod = hMod;
+    else if (!(fFlags & DBGFMODINMEM_F_NO_CONTAINER_FALLBACK))
+    {
+        /*
+         * Fallback is a container module.
+         */
+        rc = RTDbgModCreate(&hMod, pszName, cbImage, 0);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTDbgModSymbolAdd(hMod, "Headers", 0 /*iSeg*/, 0, cbImage, 0 /*fFlags*/, NULL);
+            AssertRC(rc);
+        }
+    }
+    return rc;
+}
+
+
+
+/**
+ * Process a PE image found in guest memory.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   pImageAddr      The image address.
+ * @param   fFlags          Flags, DBGFMODINMEM_F_XXX.
+ * @param   pszName         The module name, optional.
+ * @param   pszFilename     The image filename, optional.
+ * @param   enmArch         The image arch if we force it, pass
+ *                          RTLDRARCH_WHATEVER if you don't care.
+ * @param   cbImage         Image size.  Pass 0 if not known.
+ * @param   phDbgMod        Where to return the resulting debug module on success.
+ * @param   pErrInfo        Where to return extended error info on failure.
+ */
+VMMR3DECL(int) DBGFR3ModInMem(PUVM pUVM, PCDBGFADDRESS pImageAddr, uint32_t fFlags, const char *pszName, const char *pszFilename,
+                              RTLDRARCH enmArch, uint32_t cbImage, PRTDBGMOD phDbgMod, PRTERRINFO pErrInfo)
+{
+    /*
+     * Validate and adjust.
+     */
+    AssertPtrReturn(phDbgMod, VERR_INVALID_POINTER);
+    *phDbgMod = NIL_RTDBGMOD;
+    AssertPtrReturn(pImageAddr, VERR_INVALID_POINTER);
+    AssertMsgReturn(cbImage == 0 || cbImage >= sizeof(IMAGE_NT_HEADERS32) + sizeof(IMAGE_DOS_HEADER),
+                    ("cbImage=%#x\n", cbImage), VERR_INVALID_PARAMETER);
+    AssertMsgReturn(!(fFlags & ~DBGFMODINMEM_F_VALID_MASK), ("%#x\n", fFlags), VERR_INVALID_FLAGS);
+    if (enmArch == RTLDRARCH_HOST)
+        enmArch = RTLdrGetHostArch();
+
+    /*
+     * Look for an image header we can work with.
+     */
+    DBGFMODINMEMBUF uBuf;
+    RT_ZERO(uBuf);
+
+    int rc = DBGFR3MemRead(pUVM, 0 /*idCpu*/, pImageAddr, &uBuf, sizeof(uBuf.DosHdr));
+    if (RT_FAILURE(rc))
+        return RTERRINFO_LOG_SET_F(pErrInfo, rc, "Failed to read DOS header at %RGv: %Rrc", pImageAddr->FlatPtr, rc);
+
+    if (uBuf.ab[0] == ELFMAG0 && uBuf.ab[1] == ELFMAG1 && uBuf.ab[2] == ELFMAG2 && uBuf.ab[3] == ELFMAG3)
+        return dbgfR3ModInMemElf(pUVM, pImageAddr, fFlags, pszName, pszFilename, enmArch, cbImage, &uBuf, phDbgMod, pErrInfo);
+
+    if (   uBuf.MachoHdr.magic == IMAGE_MACHO64_SIGNATURE
+        || uBuf.MachoHdr.magic == IMAGE_MACHO32_SIGNATURE)
+        return dbgfR3ModInMemMachO(pUVM, pImageAddr, fFlags, pszName, pszFilename, enmArch, cbImage, &uBuf, phDbgMod, pErrInfo);
+
+    uint32_t offNewHdrs;
+    if (uBuf.DosHdr.e_magic == IMAGE_DOS_SIGNATURE)
+    {
+        offNewHdrs = uBuf.DosHdr.e_lfanew;
+        if (   offNewHdrs < 16
+            || offNewHdrs > (cbImage ? _2M : cbImage - sizeof(IMAGE_NT_HEADERS32)))
+            return RTERRINFO_LOG_SET_F(pErrInfo, rc, "e_lfanew value is out of range: %RX32 (16..%u)",
+                                       offNewHdrs, (cbImage ? _2M : cbImage - sizeof(IMAGE_NT_HEADERS32)));
+    }
+    else if (uBuf.Nt32.Signature == IMAGE_NT_SIGNATURE)
+        offNewHdrs = 0;
+    else
+        return RTERRINFO_LOG_SET_F(pErrInfo, VERR_INVALID_EXE_SIGNATURE, "Unknown image magic at %RGv: %.8Rhxs",
+                                   pImageAddr->FlatPtr, uBuf.ab);
+
+    /*
+     * Read the next bit of header, assuming PE so stop at the end of
+     * the COFF file header.
+     */
+    DBGFADDRESS PeHdrAddr = *pImageAddr;
+    DBGFR3AddrAdd(&PeHdrAddr, offNewHdrs);
+    uint32_t const cbPeHdrsPart1 = RT_UOFFSETOF(IMAGE_NT_HEADERS32, OptionalHeader);
+    rc = DBGFR3MemRead(pUVM, 0 /*idCpu*/, &PeHdrAddr, &uBuf, cbPeHdrsPart1);
+    if (RT_FAILURE(rc))
+        return RTERRINFO_LOG_SET_F(pErrInfo, rc, "Failed to read PE/LX/NE headers at %RGv (off=%#RX32): %Rrc",
+                                   PeHdrAddr.FlatPtr, offNewHdrs, rc);
+
+    if (uBuf.Nt32.Signature == IMAGE_NT_SIGNATURE)
+        return dbgfR3ModInMemPe(pUVM, pImageAddr, fFlags, pszName, pszFilename, enmArch, cbImage, offNewHdrs, cbPeHdrsPart1,
+                                &uBuf, phDbgMod, pErrInfo);
+
+    return RTERRINFO_LOG_SET_F(pErrInfo, VERR_INVALID_EXE_SIGNATURE, "No PE/LX/NE header at %RGv (off=%#RX32): %.8Rhxs",
+                               PeHdrAddr.FlatPtr, offNewHdrs, uBuf.ab);
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFR3PlugIn.cpp b/src/VBox/VMM/VMMR3/DBGFR3PlugIn.cpp
new file mode 100644
index 00000000..fd6c8698
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFR3PlugIn.cpp
@@ -0,0 +1,616 @@
+/* $Id: DBGFR3PlugIn.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Plug-In Support.
+ */
+
+/*
+ * Copyright (C) 2008-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/mm.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/version.h>
+
+#include <iprt/alloca.h>
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/env.h>
+#include <iprt/dir.h>
+#include <iprt/ldr.h>
+#include <iprt/param.h>
+#include <iprt/path.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+
+#define DBGF_PLUG_IN_READ_LOCK(pUVM) \
+    do { int rcLock = RTCritSectRwEnterShared(&pUVM->dbgf.s.CritSect); AssertRC(rcLock); } while (0)
+#define DBGF_PLUG_IN_READ_UNLOCK(pUVM) \
+    do { int rcLock = RTCritSectRwLeaveShared(&pUVM->dbgf.s.CritSect); AssertRC(rcLock); } while (0)
+
+#define DBGF_PLUG_IN_WRITE_LOCK(pUVM) \
+    do { int rcLock = RTCritSectRwEnterExcl(&pUVM->dbgf.s.CritSect); AssertRC(rcLock); } while (0)
+#define DBGF_PLUG_IN_WRITE_UNLOCK(pUVM) \
+    do { int rcLock = RTCritSectRwLeaveExcl(&pUVM->dbgf.s.CritSect); AssertRC(rcLock); } while (0)
+
+/** Max allowed length of a plug-in name (excludes the path and suffix). */
+#define DBGFPLUGIN_MAX_NAME     64
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Plug-in tracking record.
+ */
+typedef struct DBGFPLUGIN
+{
+    /** Pointer to the next plug-in. */
+    struct DBGFPLUGIN  *pNext;
+    /** The loader handle.  */
+    RTLDRMOD            hLdrMod;
+    /** The plug-in entry point. */
+    PFNDBGFPLUGIN       pfnEntry;
+    /** The name length. */
+    uint8_t             cchName;
+    /** The plug-in name (variable length).  */
+    char                szName[1];
+} DBGFPLUGIN;
+/** Pointer to plug-in tracking record. */
+typedef DBGFPLUGIN *PDBGFPLUGIN;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(void) dbgfPlugInUnloadAll(PUVM pUVM);
+static FNDBGFHANDLERINT dbgfR3PlugInInfoList;
+
+
+/**
+ * Internal init routine called by DBGFR3Init().
+ *
+ * @returns VBox status code.
+ * @param   pUVM    The user mode VM handle.
+ */
+int dbgfR3PlugInInit(PUVM pUVM)
+{
+    return DBGFR3InfoRegisterInternal(pUVM->pVM, "plugins", "Lists the debugger plug-ins.", dbgfR3PlugInInfoList);
+}
+
+
+/**
+ * Internal cleanup routine called by DBGFR3Term().
+ *
+ * @param   pUVM    The user mode VM handle.
+ */
+void dbgfR3PlugInTerm(PUVM pUVM)
+{
+    dbgfPlugInUnloadAll(pUVM);
+}
+
+
+/**
+ * Extracts the plug-in name from a plug-in specifier that may or may not
+ * include path and/or suffix.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pszDst      Where to return the name. At least DBGFPLUGIN_MAX_NAME
+ *                      worth of buffer space.
+ * @param   pszPlugIn   The plug-in module specifier to parse.
+ * @param   pErrInfo    Optional error information structure.
+ */
+static int dbgfPlugInExtractName(char *pszDst, const char *pszPlugIn, PRTERRINFO pErrInfo)
+{
+    /*
+     * Parse out the name stopping at the extension.
+     */
+    const char *pszName = RTPathFilename(pszPlugIn);
+    if (!pszName || !*pszName)
+        return VERR_INVALID_NAME;
+    if (!RTStrNICmp(pszName, RT_STR_TUPLE(DBGF_PLUG_IN_PREFIX)))
+    {
+        pszName += sizeof(DBGF_PLUG_IN_PREFIX) - 1;
+        if (!*pszName)
+            return RTErrInfoSetF(pErrInfo, VERR_INVALID_NAME, "Invalid plug-in name: nothing after the prefix");
+    }
+
+    int     ch;
+    size_t  cchName = 0;
+    while (   (ch = pszName[cchName]) != '\0'
+           && ch != '.')
+    {
+        if (   RT_C_IS_ALPHA(ch)
+            || (RT_C_IS_DIGIT(ch) && cchName != 0))
+            cchName++;
+        else
+        {
+            if (!RT_C_IS_DIGIT(ch))
+                return RTErrInfoSetF(pErrInfo, VERR_INVALID_NAME, "Invalid plug-in name: '%c' is not alphanumeric", ch);
+            return RTErrInfoSetF(pErrInfo, VERR_INVALID_NAME,
+                                 "Invalid plug-in name: Cannot start with a digit (after the prefix)");
+        }
+    }
+
+    if (cchName >= DBGFPLUGIN_MAX_NAME)
+        return RTErrInfoSetF(pErrInfo, VERR_INVALID_NAME, "Invalid plug-in name: too long (max %u)", DBGFPLUGIN_MAX_NAME);
+
+    /*
+     * We're very picky about the extension when present.
+     */
+    if (   ch == '.'
+        && RTStrICmp(&pszName[cchName], RTLdrGetSuff()))
+        return RTErrInfoSetF(pErrInfo, VERR_INVALID_NAME,
+                             "Invalid plug-in name: Suffix isn't the default dll/so/dylib one (%s): '%s'",
+                             RTLdrGetSuff(), &pszName[cchName]);
+
+    /*
+     * Copy it.
+     */
+    memcpy(pszDst, pszName, cchName);
+    pszDst[cchName] = '\0';
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Locate a loaded plug-in.
+ *
+ * @returns Pointer to the plug-in tracking structure.
+ * @param   pUVM                Pointer to the user-mode VM structure.
+ * @param   pszName             The name of the plug-in we're looking for.
+ * @param   ppPrev              Where to optionally return the pointer to the
+ *                              previous list member.
+ */
+static PDBGFPLUGIN dbgfR3PlugInLocate(PUVM pUVM, const char *pszName, PDBGFPLUGIN *ppPrev)
+{
+    PDBGFPLUGIN pPrev = NULL;
+    PDBGFPLUGIN pCur  = pUVM->dbgf.s.pPlugInHead;
+    while (pCur)
+    {
+        if (!RTStrICmp(pCur->szName, pszName))
+        {
+            if (ppPrev)
+                *ppPrev = pPrev;
+            return pCur;
+        }
+
+        /* advance */
+        pPrev = pCur;
+        pCur  = pCur->pNext;
+    }
+    return NULL;
+}
+
+
+/**
+ * Try load the specified plug-in module.
+ *
+ * @returns VINF_SUCCESS on success, path error or loader error on failure.
+ *
+ * @param   pPlugIn     The plug-in tracing record.
+ * @param   pszModule   Module name.
+ * @param   pErrInfo    Optional error information structure.
+ */
+static int dbgfR3PlugInTryLoad(PDBGFPLUGIN pPlugIn, const char *pszModule, PRTERRINFO pErrInfo)
+{
+    /*
+     * Load it and try resolve the entry point.
+     */
+    int rc = SUPR3HardenedVerifyPlugIn(pszModule, pErrInfo);
+    if (RT_SUCCESS(rc))
+        rc = RTLdrLoadEx(pszModule, &pPlugIn->hLdrMod, RTLDRLOAD_FLAGS_LOCAL, pErrInfo);
+    if (RT_SUCCESS(rc))
+    {
+        rc = RTLdrGetSymbol(pPlugIn->hLdrMod, DBGF_PLUG_IN_ENTRYPOINT, (void **)&pPlugIn->pfnEntry);
+        if (RT_SUCCESS(rc))
+        {
+            LogRel(("DBGF: Loaded Plug-In '%s' (%s)\n", pPlugIn->szName, pszModule));
+            return VINF_SUCCESS;
+        }
+
+        RTErrInfoSet(pErrInfo, rc, "Failed to locate plug-in entrypoint (" DBGF_PLUG_IN_ENTRYPOINT ")" );
+        LogRel(("DBGF: RTLdrGetSymbol('%s', '%s',) -> %Rrc\n", pszModule, DBGF_PLUG_IN_ENTRYPOINT, rc));
+
+        RTLdrClose(pPlugIn->hLdrMod);
+        pPlugIn->hLdrMod = NIL_RTLDRMOD;
+    }
+    return rc;
+}
+
+
+/**
+ * RTPathTraverseList callback.
+ *
+ * @returns See FNRTPATHTRAVERSER.
+ *
+ * @param   pchPath     See FNRTPATHTRAVERSER.
+ * @param   cchPath     See FNRTPATHTRAVERSER.
+ * @param   pvUser1     The plug-in specifier.
+ * @param   pvUser2     The plug-in tracking record.
+ */
+static DECLCALLBACK(int) dbgfR3PlugInLoadCallback(const char *pchPath, size_t cchPath, void *pvUser1, void *pvUser2)
+{
+    PDBGFPLUGIN pPlugIn   = (PDBGFPLUGIN)pvUser1;
+    PRTERRINFO  pErrInfo  = (PRTERRINFO)pvUser2;
+
+    /*
+     * Join the path and the specified plug-in name, adding prefix and suffix.
+     */
+    const char  *pszSuff   = RTLdrGetSuff();
+    size_t const cchSuff   = strlen(pszSuff);
+    size_t const cchModule = cchPath + sizeof(RTPATH_SLASH_STR) + sizeof(DBGF_PLUG_IN_PREFIX) + pPlugIn->cchName + cchSuff + 4;
+    char        *pszModule = (char *)alloca(cchModule);
+    AssertReturn(pszModule, VERR_TRY_AGAIN);
+    memcpy(pszModule, pchPath, cchPath);
+    pszModule[cchPath] = '\0';
+
+    int rc = RTPathAppend(pszModule, cchModule, DBGF_PLUG_IN_PREFIX);
+    AssertRCReturn(rc, VERR_TRY_AGAIN);
+    strcat(&pszModule[cchPath], pPlugIn->szName);
+    strcat(&pszModule[cchPath + sizeof(DBGF_PLUG_IN_PREFIX) - 1 + pPlugIn->cchName], pszSuff);
+    Assert(strlen(pszModule) < cchModule - 4);
+
+    if (RTPathExists(pszModule))
+    {
+        rc = dbgfR3PlugInTryLoad(pPlugIn, pszModule, pErrInfo);
+        if (RT_SUCCESS(rc))
+            return VINF_SUCCESS;
+    }
+
+    return VERR_TRY_AGAIN;
+}
+
+
+/**
+ * Loads a plug-in.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                Pointer to the user-mode VM structure.
+ * @param   pszName             The plug-in name.
+ * @param   pszMaybeModule      Path to the plug-in, or just the
+ *                              plug-in name as specified by the user.  Ignored
+ *                              if no path.
+ * @param   pErrInfo            Optional error information structure.
+ */
+static DECLCALLBACK(int) dbgfR3PlugInLoad(PUVM pUVM, const char *pszName, const char *pszMaybeModule, PRTERRINFO pErrInfo)
+{
+    DBGF_PLUG_IN_WRITE_LOCK(pUVM);
+
+    /*
+     * Check if a plug-in by the given name already exists.
+     */
+    PDBGFPLUGIN pPlugIn = dbgfR3PlugInLocate(pUVM, pszName, NULL);
+    if (pPlugIn)
+    {
+        DBGF_PLUG_IN_WRITE_UNLOCK(pUVM);
+        return RTErrInfoSetF(pErrInfo, VERR_ALREADY_EXISTS, "A plug-in by the name '%s' already exists", pszName);
+    }
+
+    /*
+     * Create a module structure and we can pass around via RTPathTraverseList if needed.
+     */
+    size_t cbName = strlen(pszName) + 1;
+    pPlugIn = (PDBGFPLUGIN)MMR3HeapAllocZU(pUVM, MM_TAG_DBGF, RT_UOFFSETOF_DYN(DBGFPLUGIN, szName[cbName]));
+    if (RT_UNLIKELY(!pPlugIn))
+    {
+        DBGF_PLUG_IN_WRITE_UNLOCK(pUVM);
+        return VERR_NO_MEMORY;
+    }
+    memcpy(pPlugIn->szName, pszName, cbName);
+    pPlugIn->cchName = (uint8_t)cbName - 1;
+    Assert(pPlugIn->cchName == cbName - 1);
+
+    /*
+     * If the caller specified a path, try load exactly what was specified.
+     */
+    int rc;
+    if (RTPathHavePath(pszMaybeModule))
+        rc = dbgfR3PlugInTryLoad(pPlugIn, pszMaybeModule, pErrInfo);
+    else
+    {
+        /*
+         * No path specified, search for the plug-in using the canonical
+         * module name for it.
+         */
+        RTErrInfoClear(pErrInfo);
+
+        /* 1. The private architecture directory. */
+        char szPath[_4K];
+        rc = RTPathAppPrivateArch(szPath, sizeof(szPath));
+        if (RT_SUCCESS(rc))
+            rc = RTPathTraverseList(szPath, '\0', dbgfR3PlugInLoadCallback, pPlugIn, pErrInfo);
+        if (RT_FAILURE_NP(rc))
+        {
+            /* 2. The config value 'PlugInPath' */
+            int rc2 = CFGMR3QueryString(CFGMR3GetChild(CFGMR3GetRootU(pUVM), "/DBGF"), "PlugInPath", szPath, sizeof(szPath));
+            if (RT_SUCCESS(rc2))
+                rc = RTPathTraverseList(szPath, ';', dbgfR3PlugInLoadCallback, pPlugIn, pErrInfo);
+            if (RT_FAILURE_NP(rc))
+            {
+                /* 3. The VBOXDBG_PLUG_IN_PATH environment variable. */
+                rc2 = RTEnvGetEx(RTENV_DEFAULT, "VBOXDBG_PLUG_IN_PATH", szPath, sizeof(szPath), NULL);
+                if (RT_SUCCESS(rc2))
+                    rc = RTPathTraverseList(szPath, ';', dbgfR3PlugInLoadCallback, pPlugIn, pErrInfo);
+            }
+        }
+
+        if (rc == VERR_END_OF_STRING)
+            rc = VERR_FILE_NOT_FOUND;
+        if (pErrInfo && !RTErrInfoIsSet(pErrInfo))
+            RTErrInfoSetF(pErrInfo, rc, "Failed to locate '%s'", pPlugIn->szName);
+    }
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Try initialize it.
+         */
+        rc = pPlugIn->pfnEntry(DBGFPLUGINOP_INIT, pUVM, VBOX_VERSION);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Link it and we're good.
+             */
+            pPlugIn->pNext = pUVM->dbgf.s.pPlugInHead;
+            pUVM->dbgf.s.pPlugInHead = pPlugIn;
+
+            DBGF_PLUG_IN_WRITE_UNLOCK(pUVM);
+            return VINF_SUCCESS;
+        }
+
+        RTErrInfoSet(pErrInfo, rc, "Plug-in init failed");
+        LogRel(("DBGF: Plug-in '%s' failed during init: %Rrc\n", pPlugIn->szName, rc));
+        RTLdrClose(pPlugIn->hLdrMod);
+    }
+    MMR3HeapFree(pPlugIn);
+
+    DBGF_PLUG_IN_WRITE_UNLOCK(pUVM);
+    return rc;
+}
+
+
+/**
+ * Load a debugging plug-in.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_ALREADY_EXISTS if the module was already loaded.
+ * @retval  VINF_BUFFER_OVERFLOW if the actual plug-in name buffer was too small
+ *          (the plug-in was still successfully loaded).
+ * @param   pUVM        Pointer to the user-mode VM structure.
+ * @param   pszPlugIn   The plug-in name.  This may specify the exact path to
+ *                      the plug-in module, or it may just specify the core name
+ *                      of the plug-in without prefix, suffix and path.
+ * @param   pszActual   Buffer to return the actual plug-in name in. Optional.
+ *                      This will be returned on VERR_ALREADY_EXSIST too.
+ * @param   cbActual    The size of @a pszActual.
+ * @param   pErrInfo    Optional error information structure.
+ */
+VMMR3DECL(int) DBGFR3PlugInLoad(PUVM pUVM, const char *pszPlugIn, char *pszActual, size_t cbActual, PRTERRINFO pErrInfo)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszPlugIn, VERR_INVALID_PARAMETER);
+
+    /*
+     * Extract the plug-in name.  Copy it to the return buffer as we'll want to
+     * return it in the VERR_ALREADY_EXISTS case too.
+     */
+    char szName[DBGFPLUGIN_MAX_NAME];
+    int rc = dbgfPlugInExtractName(szName, pszPlugIn, pErrInfo);
+    if (RT_SUCCESS(rc))
+    {
+        int rc2 = VINF_SUCCESS;
+        if (pszActual)
+            rc2 = RTStrCopy(pszActual, cbActual, szName);
+
+        /*
+         * Write lock releated DBGF bits and try load it.
+         */
+        rc = VMR3ReqPriorityCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3PlugInLoad, 4, pUVM, szName, pszPlugIn, pErrInfo);
+        if (rc2 != VINF_SUCCESS && RT_SUCCESS(rc))
+            rc = VINF_BUFFER_OVERFLOW;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Load all plug-ins from the architechture private directory of VBox.
+ *
+ * @param   pUVM    Pointer to the user-mode VM structure.
+ */
+VMMR3DECL(void) DBGFR3PlugInLoadAll(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN_VOID(pUVM);
+
+    /*
+     * Pass it on to EMT(0) if necessary (thanks to DBGFR3Os*).
+     */
+    if (VMR3GetVMCPUId(pUVM->pVM) != 0)
+    {
+        VMR3ReqPriorityCallVoidWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)DBGFR3PlugInLoadAll, 1, pUVM);
+        return;
+    }
+
+
+    /*
+     * Open the architecture specific directory with a filter on our prefix
+     * and names including a dot.
+     */
+    const char *pszSuff = RTLdrGetSuff();
+    size_t      cchSuff = strlen(pszSuff);
+
+    char szPath[RTPATH_MAX];
+    int rc = RTPathAppPrivateArch(szPath, sizeof(szPath) - cchSuff);
+    AssertRCReturnVoid(rc);
+    size_t offDir = strlen(szPath);
+
+    rc = RTPathAppend(szPath, sizeof(szPath) - cchSuff, DBGF_PLUG_IN_PREFIX "*");
+    AssertRCReturnVoid(rc);
+    strcat(szPath, pszSuff);
+
+    RTDIR hDir;
+    rc = RTDirOpenFiltered(&hDir, szPath, RTDIRFILTER_WINNT, 0 /*fFlags*/);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Now read it and try load each of the plug-in modules.
+         */
+        RTDIRENTRY DirEntry;
+        while (RT_SUCCESS(RTDirRead(hDir, &DirEntry, NULL)))
+        {
+            szPath[offDir] = '\0';
+            rc = RTPathAppend(szPath, sizeof(szPath), DirEntry.szName);
+            if (RT_SUCCESS(rc))
+            {
+                char szName[DBGFPLUGIN_MAX_NAME];
+                rc = dbgfPlugInExtractName(szName, DirEntry.szName, NULL);
+                if (RT_SUCCESS(rc))
+                {
+                    DBGF_PLUG_IN_WRITE_LOCK(pUVM);
+                    dbgfR3PlugInLoad(pUVM, szName, szPath, NULL);
+                    DBGF_PLUG_IN_WRITE_UNLOCK(pUVM);
+                }
+            }
+        }
+
+        RTDirClose(hDir);
+    }
+}
+
+
+/**
+ * Unloads a plug-in by name (no path, prefix or suffix).
+ *
+ * @returns VBox status code.
+ * @retval  VERR_NOT_FOUND if the specified plug-in wasn't found.
+ * @param   pUVM        Pointer to the user-mode VM structure.
+ * @param   pszName     The name of the plug-in to unload.
+ */
+VMMR3DECL(int) DBGFR3PlugInUnload(PUVM pUVM, const char *pszName)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Pass it on to EMT(0) if necessary (thanks to DBGFR3Os*).
+     */
+    if (VMR3GetVMCPUId(pUVM->pVM) != 0)
+        return VMR3ReqPriorityCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)DBGFR3PlugInUnload, 2, pUVM, pszName);
+
+
+    /*
+     * Find the plug-in.
+     */
+    DBGF_PLUG_IN_WRITE_LOCK(pUVM);
+
+    int rc;
+    PDBGFPLUGIN pPrevPlugIn;
+    PDBGFPLUGIN pPlugIn = dbgfR3PlugInLocate(pUVM, pszName, &pPrevPlugIn);
+    if (pPlugIn)
+    {
+        /*
+         * Unlink, terminate, unload and free the plug-in.
+         */
+        if (pPrevPlugIn)
+            pPrevPlugIn->pNext = pPlugIn->pNext;
+        else
+            pUVM->dbgf.s.pPlugInHead = pPlugIn->pNext;
+
+        pPlugIn->pfnEntry(DBGFPLUGINOP_TERM, pUVM, 0);
+        RTLdrClose(pPlugIn->hLdrMod);
+
+        pPlugIn->pfnEntry = NULL;
+        pPlugIn->hLdrMod  = NIL_RTLDRMOD;
+        MMR3HeapFree(pPlugIn->pNext);
+        rc = VINF_SUCCESS;
+    }
+    else
+        rc = VERR_NOT_FOUND;
+
+    DBGF_PLUG_IN_WRITE_UNLOCK(pUVM);
+    return rc;
+}
+
+
+/**
+ * Unload all plug-ins.
+ *
+ * @param   pUVM    Pointer to the user-mode VM structure.
+ */
+static DECLCALLBACK(void) dbgfPlugInUnloadAll(PUVM pUVM)
+{
+    DBGF_PLUG_IN_WRITE_LOCK(pUVM);
+
+    while (pUVM->dbgf.s.pPlugInHead)
+    {
+        PDBGFPLUGIN pPlugin = pUVM->dbgf.s.pPlugInHead;
+        pUVM->dbgf.s.pPlugInHead = pPlugin->pNext;
+
+        pPlugin->pfnEntry(DBGFPLUGINOP_TERM, pUVM, 0);
+
+        int rc2 = RTLdrClose(pPlugin->hLdrMod);
+        AssertRC(rc2);
+
+        pPlugin->pfnEntry = NULL;
+        pPlugin->hLdrMod  = NIL_RTLDRMOD;
+        MMR3HeapFree(pPlugin);
+    }
+
+    DBGF_PLUG_IN_WRITE_UNLOCK(pUVM);
+}
+
+
+/**
+ * Unloads all plug-ins.
+ *
+ * @param   pUVM    Pointer to the user-mode VM structure.
+ */
+VMMR3DECL(void) DBGFR3PlugInUnloadAll(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN_VOID(pUVM);
+    /* Thanks to DBGFR3Os, this must be done on EMT(0). */
+    VMR3ReqPriorityCallVoidWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfPlugInUnloadAll, 1, pUVM);
+}
+
+
+
+/**
+ * @callback_method_impl{FNDBGFHANDLERINT, The 'plugins' info item.}
+ */
+static DECLCALLBACK(void) dbgfR3PlugInInfoList(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    PDBGFPLUGIN pPlugIn = pVM->pUVM->dbgf.s.pPlugInHead;
+    RT_NOREF_PV(pszArgs);
+    if (pPlugIn)
+    {
+        pHlp->pfnPrintf(pHlp, "Debugging plug-in%s: %s", pPlugIn->pNext ? "s" : "", pPlugIn->szName);
+        while ((pPlugIn = pPlugIn->pNext) != NULL)
+            pHlp->pfnPrintf(pHlp, ", %s", pPlugIn->szName);
+        pHlp->pfnPrintf(pHlp, "\n");
+
+    }
+    else
+        pHlp->pfnPrintf(pHlp, "No plug-ins loaded\n");
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFR3Trace.cpp b/src/VBox/VMM/VMMR3/DBGFR3Trace.cpp
new file mode 100644
index 00000000..de84b6fb
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFR3Trace.cpp
@@ -0,0 +1,447 @@
+/* $Id: DBGFR3Trace.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Tracing.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgftrace.h>
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pdmapi.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/vm.h>
+#include "VMMTracing.h"
+
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/param.h>
+
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/trace.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(void) dbgfR3TraceInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/**
+ * VMM trace point group translation table.
+ */
+static const struct
+{
+    /** The group name. */
+    const char *pszName;
+    /** The name length. */
+    uint32_t    cchName;
+    /** The mask. */
+    uint32_t    fMask;
+}   g_aVmmTpGroups[] =
+{
+    {  RT_STR_TUPLE("em"), VMMTPGROUP_EM },
+    {  RT_STR_TUPLE("hm"), VMMTPGROUP_HM },
+    {  RT_STR_TUPLE("tm"), VMMTPGROUP_TM },
+};
+
+
+/**
+ * Initializes the tracing.
+ *
+ * @returns VBox status code
+ * @param   pVM         The cross context VM structure.
+ * @param   cbEntry     The trace entry size.
+ * @param   cEntries    The number of entries.
+ */
+static int dbgfR3TraceEnable(PVM pVM, uint32_t cbEntry, uint32_t cEntries)
+{
+    /*
+     * Don't enable it twice.
+     */
+    if (pVM->hTraceBufR3 != NIL_RTTRACEBUF)
+        return VERR_ALREADY_EXISTS;
+
+    /*
+     * Resolve default parameter values.
+     */
+    int rc;
+    if (!cbEntry)
+    {
+        rc = CFGMR3QueryU32Def(CFGMR3GetChild(CFGMR3GetRoot(pVM), "DBGF"), "TraceBufEntrySize", &cbEntry, 128);
+        AssertRCReturn(rc, rc);
+    }
+    if (!cEntries)
+    {
+        rc = CFGMR3QueryU32Def(CFGMR3GetChild(CFGMR3GetRoot(pVM), "DBGF"), "TraceBufEntries", &cEntries, 4096);
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Figure the required size.
+     */
+    RTTRACEBUF  hTraceBuf;
+    size_t      cbBlock = 0;
+    rc = RTTraceBufCarve(&hTraceBuf, cEntries, cbEntry, 0 /*fFlags*/, NULL, &cbBlock);
+    if (rc != VERR_BUFFER_OVERFLOW)
+    {
+        AssertReturn(!RT_SUCCESS_NP(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+        return rc;
+    }
+
+    /*
+     * Allocate a hyper heap block and carve a trace buffer out of it.
+     *
+     * Note! We ASSUME that the returned trace buffer handle has the same value
+     *       as the heap block.
+     */
+    cbBlock = RT_ALIGN_Z(cbBlock, PAGE_SIZE);
+    void *pvBlock;
+    rc = MMR3HyperAllocOnceNoRel(pVM, cbBlock, PAGE_SIZE, MM_TAG_DBGF, &pvBlock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    rc = RTTraceBufCarve(&hTraceBuf, cEntries, cbEntry, 0 /*fFlags*/, pvBlock, &cbBlock);
+    AssertRCReturn(rc, rc);
+    AssertRelease(hTraceBuf == (RTTRACEBUF)pvBlock);
+    AssertRelease((void *)hTraceBuf == pvBlock);
+
+    pVM->hTraceBufR3 = hTraceBuf;
+    pVM->hTraceBufR0 = MMHyperCCToR0(pVM, hTraceBuf);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Initializes the tracing.
+ *
+ * @returns VBox status code
+ * @param   pVM                 The cross context VM structure.
+ */
+int dbgfR3TraceInit(PVM pVM)
+{
+    /*
+     * Initialize the trace buffer handles.
+     */
+    Assert(NIL_RTTRACEBUF == (RTTRACEBUF)NULL);
+    pVM->hTraceBufR3 = NIL_RTTRACEBUF;
+    pVM->hTraceBufR0 = NIL_RTR0PTR;
+
+    /*
+     * Check the config and enable tracing if requested.
+     */
+    PCFGMNODE pDbgfNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "DBGF");
+#if defined(DEBUG) || defined(RTTRACE_ENABLED)
+    bool const          fDefault        = false;
+    const char * const  pszConfigDefault = "";
+#else
+    bool const          fDefault        = false;
+    const char * const  pszConfigDefault = "";
+#endif
+    bool                fTracingEnabled;
+    int rc = CFGMR3QueryBoolDef(pDbgfNode, "TracingEnabled", &fTracingEnabled, fDefault);
+    AssertRCReturn(rc, rc);
+    if (fTracingEnabled)
+    {
+        rc = dbgfR3TraceEnable(pVM, 0, 0);
+        if (RT_SUCCESS(rc))
+        {
+            if (pDbgfNode)
+            {
+                char       *pszTracingConfig;
+                rc = CFGMR3QueryStringAllocDef(pDbgfNode, "TracingConfig", &pszTracingConfig, pszConfigDefault);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = DBGFR3TraceConfig(pVM, pszTracingConfig);
+                    if (RT_FAILURE(rc))
+                        rc = VMSetError(pVM, rc, RT_SRC_POS, "TracingConfig=\"%s\" -> %Rrc", pszTracingConfig, rc);
+                    MMR3HeapFree(pszTracingConfig);
+                }
+            }
+            else
+            {
+                rc = DBGFR3TraceConfig(pVM, pszConfigDefault);
+                if (RT_FAILURE(rc))
+                    rc = VMSetError(pVM, rc, RT_SRC_POS, "TracingConfig=\"%s\" (default) -> %Rrc", pszConfigDefault, rc);
+            }
+        }
+    }
+
+    /*
+     * Register a debug info item that will dump the trace buffer content.
+     */
+    if (RT_SUCCESS(rc))
+        rc = DBGFR3InfoRegisterInternal(pVM, "tracebuf", "Display the trace buffer content. No arguments.", dbgfR3TraceInfo);
+
+    return rc;
+}
+
+
+/**
+ * Terminates the tracing.
+ *
+ * @param   pVM                 The cross context VM structure.
+ */
+void dbgfR3TraceTerm(PVM pVM)
+{
+    /* nothing to do */
+    NOREF(pVM);
+}
+
+
+/**
+ * Relocates the trace buffer handle in RC.
+ *
+ * @param   pVM                 The cross context VM structure.
+ */
+void dbgfR3TraceRelocate(PVM pVM)
+{
+    RT_NOREF(pVM);
+}
+
+
+/**
+ * Change the traceing configuration of the VM.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_NOT_FOUND if any of the trace point groups mentioned in the
+ *          config string cannot be found. (Or if the string cannot be made
+ *          sense of.)  No change made.
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_POINTER
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pszConfig   The configuration change specification.
+ *
+ *                      Trace point group names, optionally prefixed by a '-' to
+ *                      indicate that the group is being disabled. A special
+ *                      group 'all' can be used to enable or disable all trace
+ *                      points.
+ *
+ *                      Drivers, devices and USB devices each have their own
+ *                      trace point group which can be accessed by prefixing
+ *                      their official PDM name by 'drv', 'dev' or 'usb'
+ *                      respectively.
+ */
+VMMDECL(int) DBGFR3TraceConfig(PVM pVM, const char *pszConfig)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszConfig, VERR_INVALID_POINTER);
+    if (pVM->hTraceBufR3 == NIL_RTTRACEBUF)
+        return VERR_DBGF_NO_TRACE_BUFFER;
+
+    /*
+     * We do this in two passes, the first pass just validates the input string
+     * and the second applies the changes.
+     */
+    for (uint32_t uPass = 0; uPass < 1; uPass++)
+    {
+        char ch;
+        while ((ch = *pszConfig) != '\0')
+        {
+            if (RT_C_IS_SPACE(ch))
+                continue;
+
+            /*
+             * Operation prefix.
+             */
+            bool fNo = false;
+            do
+            {
+                if (ch == 'n' && pszConfig[1] == 'o')
+                {
+                    fNo = !fNo;
+                    pszConfig++;
+                }
+                else if (ch == '+')
+                    fNo = false;
+                else if (ch == '-' || ch == '!' || ch == '~')
+                    fNo = !fNo;
+                else
+                    break;
+            } while ((ch = *++pszConfig) != '\0');
+            if (ch == '\0')
+                break;
+
+            /*
+             * Extract the name.
+             */
+            const char *pszName = pszConfig;
+            while (   ch != '\0'
+                   && !RT_C_IS_SPACE(ch)
+                   && !RT_C_IS_PUNCT(ch))
+                ch = *++pszConfig;
+            size_t const cchName = pszConfig - pszName;
+
+            /*
+             * 'all' - special group that enables or disables all trace points.
+             */
+            if (cchName == 3 && !strncmp(pszName, "all", 3))
+            {
+                if (uPass != 0)
+                {
+                    uint32_t iCpu = pVM->cCpus;
+                    if (!fNo)
+                        while (iCpu-- > 0)
+                            pVM->apCpusR3[iCpu]->fTraceGroups = UINT32_MAX;
+                    else
+                        while (iCpu-- > 0)
+                            pVM->apCpusR3[iCpu]->fTraceGroups = 0;
+                    PDMR3TracingConfig(pVM, NULL, 0, !fNo, uPass > 0);
+                }
+            }
+            else
+            {
+                /*
+                 * A specific group, try the VMM first then PDM.
+                 */
+                uint32_t i = RT_ELEMENTS(g_aVmmTpGroups);
+                while (i-- > 0)
+                    if (   g_aVmmTpGroups[i].cchName == cchName
+                        && !strncmp(g_aVmmTpGroups[i].pszName, pszName, cchName))
+                    {
+                        if (uPass != 0)
+                        {
+                            uint32_t iCpu = pVM->cCpus;
+                            if (!fNo)
+                                while (iCpu-- > 0)
+                                    pVM->apCpusR3[iCpu]->fTraceGroups |= g_aVmmTpGroups[i].fMask;
+                            else
+                                while (iCpu-- > 0)
+                                    pVM->apCpusR3[iCpu]->fTraceGroups &= ~g_aVmmTpGroups[i].fMask;
+                        }
+                        break;
+                    }
+
+                if (i == UINT32_MAX)
+                {
+                    int rc = PDMR3TracingConfig(pVM, pszName, cchName, !fNo, uPass > 0);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Query the trace configuration specification string.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_POINTER
+ * @retval  VERR_BUFFER_OVERFLOW if the buffer is too small. Buffer will be
+ *          empty.
+
+ * @param   pVM                 The cross context VM structure.
+ * @param   pszConfig           Pointer to the output buffer.
+ * @param   cbConfig            The size of the output buffer.
+ */
+VMMDECL(int) DBGFR3TraceQueryConfig(PVM pVM, char *pszConfig, size_t cbConfig)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszConfig, VERR_INVALID_POINTER);
+    if (cbConfig < 1)
+        return VERR_BUFFER_OVERFLOW;
+    *pszConfig = '\0';
+
+    if (pVM->hTraceBufR3 == NIL_RTTRACEBUF)
+        return VERR_DBGF_NO_TRACE_BUFFER;
+
+    int             rc           = VINF_SUCCESS;
+    uint32_t const  fTraceGroups = pVM->apCpusR3[0]->fTraceGroups;
+    if (   fTraceGroups == UINT32_MAX
+        && PDMR3TracingAreAll(pVM, true /*fEnabled*/))
+        rc = RTStrCopy(pszConfig, cbConfig, "all");
+    else if (   fTraceGroups == 0
+             && PDMR3TracingAreAll(pVM, false /*fEnabled*/))
+        rc = RTStrCopy(pszConfig, cbConfig, "-all");
+    else
+    {
+        char   *pszDst = pszConfig;
+        size_t  cbDst  = cbConfig;
+        uint32_t i = RT_ELEMENTS(g_aVmmTpGroups);
+        while (i-- > 0)
+            if (g_aVmmTpGroups[i].fMask & fTraceGroups)
+            {
+                size_t cchThis = g_aVmmTpGroups[i].cchName + (pszDst != pszConfig);
+                if (cchThis >= cbDst)
+                {
+                    rc = VERR_BUFFER_OVERFLOW;
+                    break;
+                }
+                if (pszDst != pszConfig)
+                {
+                    *pszDst = ' ';
+                    memcpy(pszDst + 1, g_aVmmTpGroups[i].pszName, g_aVmmTpGroups[i].cchName + 1);
+                }
+                else
+                    memcpy(pszDst, g_aVmmTpGroups[i].pszName, g_aVmmTpGroups[i].cchName + 1);
+                pszDst += cchThis;
+                cbDst  -= cchThis;
+            }
+
+        if (RT_SUCCESS(rc))
+            rc = PDMR3TracingQueryConfig(pVM, pszDst, cbDst);
+    }
+
+    if (RT_FAILURE(rc))
+        *pszConfig = '\0';
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNRTTRACEBUFCALLBACK}
+ */
+static DECLCALLBACK(int)
+dbgfR3TraceInfoDumpEntry(RTTRACEBUF hTraceBuf, uint32_t iEntry, uint64_t NanoTS, RTCPUID idCpu, const char *pszMsg, void *pvUser)
+{
+    PCDBGFINFOHLP pHlp = (PCDBGFINFOHLP)pvUser;
+    pHlp->pfnPrintf(pHlp, "#%04u/%'llu/%02x: %s\n", iEntry, NanoTS, idCpu, pszMsg);
+    NOREF(hTraceBuf);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNDBGFHANDLERINT, Info handler for displaying the trace buffer content.}
+ */
+static DECLCALLBACK(void) dbgfR3TraceInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    RTTRACEBUF hTraceBuf = pVM->hTraceBufR3;
+    if (hTraceBuf == NIL_RTTRACEBUF)
+        pHlp->pfnPrintf(pHlp, "Tracing is disabled\n");
+    else
+    {
+        pHlp->pfnPrintf(pHlp, "Trace buffer %p - %u entries of %u bytes\n",
+                        hTraceBuf, RTTraceBufGetEntryCount(hTraceBuf), RTTraceBufGetEntrySize(hTraceBuf));
+        RTTraceBufEnumEntries(hTraceBuf, dbgfR3TraceInfoDumpEntry, (void *)pHlp);
+    }
+    NOREF(pszArgs);
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFR3Type.cpp b/src/VBox/VMM/VMMR3/DBGFR3Type.cpp
new file mode 100644
index 00000000..327489a9
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFR3Type.cpp
@@ -0,0 +1,1278 @@
+/* $Id: DBGFR3Type.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Type Management.
+ */
+
+/*
+ * Copyright (C) 2016-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_dbgf_type    DBGFType - Type Management
+ *
+ * The type management system is intended to ease retrieval of values from
+ * structures in the guest OS without having to take care of the size of pointers.
+ *
+ * @todo r=bird: We need to join this up with modules and address spaces.  It
+ *       cannot be standalone like this.  Also, it must be comming from IPRT as
+ *       there is no point in duplicating code (been there, done that with
+ *       symbols and debug info already).  This unfortunately means we need to
+ *       find some common way of abstracting DWARF and Codeview type info so we
+ *       can extend those debug info parsers to make type information available.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/param.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+
+/** Locks the type database for writing. */
+#define DBGF_TYPE_DB_LOCK_WRITE(pUVM) \
+    do { \
+        int rcSem = RTSemRWRequestWrite((pUVM)->dbgf.s.hTypeDbLock, RT_INDEFINITE_WAIT); \
+        AssertRC(rcSem); \
+    } while (0)
+
+/** Unlocks the type database after writing. */
+#define DBGF_TYPE_DB_UNLOCK_WRITE(pUVM) \
+    do { \
+        int rcSem = RTSemRWReleaseWrite((pUVM)->dbgf.s.hTypeDbLock); \
+        AssertRC(rcSem); \
+    } while (0)
+
+/** Locks the type database for reading. */
+#define DBGF_TYPE_DB_LOCK_READ(pUVM) \
+    do { \
+        int rcSem = RTSemRWRequestRead((pUVM)->dbgf.s.hTypeDbLock, RT_INDEFINITE_WAIT); \
+        AssertRC(rcSem); \
+    } while (0)
+
+/** Unlocks the type database after reading. */
+#define DBGF_TYPE_DB_UNLOCK_READ(pUVM) \
+    do { \
+        int rcSem = RTSemRWReleaseRead((pUVM)->dbgf.s.hTypeDbLock); \
+        AssertRC(rcSem); \
+    } while (0)
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * DBGF registered type.
+ */
+typedef struct DBGFTYPE
+{
+    /** String space core. */
+    RTSTRSPACECORE          Core;
+    /** Pointer to the registration structure, NULL means builtin type. */
+    PCDBGFTYPEREG           pReg;
+    /** How often the type is referenced by other types. */
+    volatile uint32_t       cRefs;
+    /** Size of the type. */
+    size_t                  cbType;
+    /** Builtin type if pReg is NULL (otherwise it is invalid). */
+    DBGFTYPEBUILTIN         enmTypeBuiltin;
+} DBGFTYPE;
+/** Pointer to a DBGF type. */
+typedef DBGFTYPE *PDBGFTYPE;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int dbgfR3TypeParseBufferByType(PUVM pUVM, PDBGFTYPE pType, uint8_t *pbBuf, size_t cbBuf,
+                                       PDBGFTYPEVAL *ppVal, size_t *pcbParsed);
+
+
+/**
+ * Looks up a type by the identifier.
+ *
+ * @returns Pointer to the type structure on success, NULL otherwise.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszType             The type identifier.
+ */
+static PDBGFTYPE dbgfR3TypeLookup(PUVM pUVM, const char *pszType)
+{
+    PRTSTRSPACE pTypeSpace = &pUVM->dbgf.s.TypeSpace;
+    return (PDBGFTYPE)RTStrSpaceGet(pTypeSpace, pszType);
+}
+
+
+/**
+ * Calculate the size of the given type.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pType               The type to calculate the size for.
+ * @param   fCalcNested         Flag whether to calculate the size for nested
+ *                              structs if the sizes are 0.
+ */
+static int dbgfR3TypeCalcSize(PUVM pUVM, PDBGFTYPE pType, bool fCalcNested)
+{
+    int rc = VINF_SUCCESS;
+
+    /* Builtin types are never recalculated. */
+    if (pType->pReg)
+    {
+        switch (pType->pReg->enmVariant)
+        {
+            case DBGFTYPEVARIANT_STRUCT:
+            {
+                size_t cbType = 0;
+
+                /* Go through the members and update size. */
+                for (uint32_t i = 0; i < pType->pReg->cMembers && RT_SUCCESS(rc); i++)
+                {
+                    PCDBGFTYPEREGMEMBER pMember = &pType->pReg->paMembers[i];
+
+                    if (pMember->fFlags & DBGFTYPEREGMEMBER_F_POINTER)
+                    {
+                        /* Use the current pointer size. */
+                        PDBGFTYPE pTypeMember = dbgfR3TypeLookup(pUVM, "ptr_t");
+                        if (RT_LIKELY(pTypeMember))
+                        {
+                            if (pMember->fFlags & DBGFTYPEREGMEMBER_F_ARRAY)
+                                cbType += pMember->cElements * pTypeMember->cbType;
+                            else
+                                cbType += pTypeMember->cbType;
+                        }
+                    }
+                    else
+                    {
+                        PDBGFTYPE pTypeMember = dbgfR3TypeLookup(pUVM, pMember->pszType);
+                        if (RT_LIKELY(pTypeMember))
+                        {
+                            if (   pTypeMember->cbType == 0
+                                && fCalcNested)
+                                rc = dbgfR3TypeCalcSize(pUVM, pTypeMember, fCalcNested);
+
+                            if (RT_SUCCESS(rc))
+                            {
+                                if (pMember->fFlags & DBGFTYPEREGMEMBER_F_ARRAY)
+                                    cbType += pMember->cElements * pTypeMember->cbType;
+                                else
+                                    cbType += pTypeMember->cbType;
+                            }
+                        }
+                        else
+                            rc = VERR_INVALID_STATE;
+                    }
+                }
+
+                if (RT_SUCCESS(rc))
+                    pType->cbType = cbType;
+                break;
+            }
+
+            case DBGFTYPEVARIANT_UNION:
+            {
+                /* Get size of the biggest member and use that one. */
+                size_t cbType = 0;
+
+                for (uint32_t i = 0; i < pType->pReg->cMembers && RT_SUCCESS(rc); i++)
+                {
+                    PCDBGFTYPEREGMEMBER pMember = &pType->pReg->paMembers[i];
+
+                    if (pMember->fFlags & DBGFTYPEREGMEMBER_F_POINTER)
+                    {
+                        /* Use the current pointer size. */
+                        PDBGFTYPE pTypeMember = dbgfR3TypeLookup(pUVM, "ptr_t");
+                        if (RT_LIKELY(pTypeMember))
+                        {
+                            if (pMember->fFlags & DBGFTYPEREGMEMBER_F_ARRAY)
+                                cbType = RT_MAX(cbType, pMember->cElements * pTypeMember->cbType);
+                            else
+                                cbType = RT_MAX(cbType, pTypeMember->cbType);
+                        }
+                    }
+                    else
+                    {
+                        PDBGFTYPE pTypeMember = dbgfR3TypeLookup(pUVM, pMember->pszType);
+                        if (RT_LIKELY(pTypeMember))
+                        {
+                            if (   pTypeMember->cbType == 0
+                                && fCalcNested)
+                                rc = dbgfR3TypeCalcSize(pUVM, pTypeMember, fCalcNested);
+
+                            if (RT_SUCCESS(rc))
+                            {
+                                if (pMember->fFlags & DBGFTYPEREGMEMBER_F_ARRAY)
+                                    cbType = RT_MAX(cbType, pMember->cElements * pTypeMember->cbType);
+                                else
+                                    cbType = RT_MAX(cbType, pTypeMember->cbType);
+                            }
+                        }
+                        else
+                            rc = VERR_INVALID_STATE;
+                    }
+                }
+
+                if (RT_SUCCESS(rc))
+                    pType->cbType = cbType;
+                break;
+            }
+
+            case DBGFTYPEVARIANT_ALIAS:
+            {
+                /* Get the size of the alias. */
+                PDBGFTYPE pAliased = dbgfR3TypeLookup(pUVM, pType->pReg->pszAliasedType);
+                if (RT_LIKELY(pAliased))
+                {
+                    if (   pAliased->cbType == 0
+                        && fCalcNested)
+                        rc = dbgfR3TypeCalcSize(pUVM, pAliased, fCalcNested);
+
+                    if (RT_SUCCESS(rc))
+                        pType->cbType = pAliased->cbType;
+                }
+                else
+                    rc = VERR_INVALID_STATE;
+                break;
+            }
+
+            default:
+                AssertMsgFailedReturn(("Invalid type variant: %d", pType->pReg->enmVariant), VERR_INVALID_STATE);
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Callback for clearing the size of all non built-in types.
+ *
+ * @returns VBox status code.
+ * @param   pStr                The type structure.
+ * @param   pvUser              The user mode VM handle.
+ */
+static DECLCALLBACK(int) dbgfR3TypeTraverseClearSize(PRTSTRSPACECORE pStr, void *pvUser)
+{
+    PDBGFTYPE pType = (PDBGFTYPE)pStr;
+
+    if (pType->pReg)
+        pType->cbType = 0;
+
+    NOREF(pvUser);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Callback for calculating the size of all non built-in types.
+ *
+ * @returns VBox status code.
+ * @param   pStr                The type structure.
+ * @param   pvUser              The user mode VM handle.
+ */
+static DECLCALLBACK(int) dbgfR3TypeTraverseCalcSize(PRTSTRSPACECORE pStr, void *pvUser)
+{
+    PDBGFTYPE pType = (PDBGFTYPE)pStr;
+
+    if (    pType->pReg
+        && !pType->cbType)
+        dbgfR3TypeCalcSize((PUVM)pvUser, pType, true /* fCalcNested */);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Recalculate the sizes of all registered non builtin types.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ */
+static int dbgfR3TypeRecalculateAllSizes(PUVM pUVM)
+{
+    int rc = VINF_SUCCESS;
+
+    /*
+     * Clear the sizes of all non builtin types to 0 first so we know which type we
+     * visited later on.
+     */
+    rc = RTStrSpaceEnumerate(&pUVM->dbgf.s.TypeSpace, dbgfR3TypeTraverseClearSize, pUVM);
+    if (RT_SUCCESS(rc))
+    {
+        /* Now recalculate the size. */
+        rc = RTStrSpaceEnumerate(&pUVM->dbgf.s.TypeSpace, dbgfR3TypeTraverseCalcSize, pUVM);
+    }
+
+    return rc;
+}
+
+/**
+ * Validates a given type registration.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pReg                The type registration structure.
+ */
+static int dbgfR3TypeValidate(PUVM pUVM, PCDBGFTYPEREG pReg)
+{
+    int rc = VINF_SUCCESS;
+
+    switch (pReg->enmVariant)
+    {
+        case DBGFTYPEVARIANT_ALIAS:
+            if (   pReg->cMembers > 0
+                || pReg->paMembers
+                || !pReg->pszAliasedType)
+                rc = VERR_INVALID_PARAMETER;
+            else
+            {
+                PDBGFTYPE pAlias = dbgfR3TypeLookup(pUVM, pReg->pszAliasedType);
+                if (RT_UNLIKELY(!pAlias))
+                    rc = VERR_NOT_FOUND;
+            }
+            break;
+        case DBGFTYPEVARIANT_STRUCT:
+        case DBGFTYPEVARIANT_UNION:
+            if (!pReg->pszAliasedType)
+            {
+                for (uint32_t i = 0; i < pReg->cMembers; i++)
+                {
+                    PCDBGFTYPEREGMEMBER pMember = &pReg->paMembers[i];
+
+                    /* Use the current pointer size. */
+                    PDBGFTYPE pTypeMember = dbgfR3TypeLookup(pUVM, pMember->pszType);
+                    if (RT_UNLIKELY(!pTypeMember))
+                    {
+                        rc = VERR_NOT_FOUND;
+                        break;
+                    }
+
+                    if (pMember->fFlags & DBGFTYPEREGMEMBER_F_ARRAY)
+                    {
+                        if (pMember->cElements == 0)
+                            rc = VERR_INVALID_PARAMETER;
+                    }
+                    else if (pMember->cElements != 0)
+                        rc = VERR_INVALID_PARAMETER;
+                }
+            }
+            else
+                rc = VERR_INVALID_PARAMETER;
+            break;
+        default:
+            AssertMsgFailedBreakStmt(("Invalid type variant: %d", pReg->enmVariant),
+                                     rc = VERR_INVALID_PARAMETER);
+    }
+
+    return rc;
+}
+
+/**
+ * Retains or releases the reference counters to referenced types for the given
+ * type registration structure.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pReg                The type registration structure.
+ * @param   fRetain             Flag whether to retain or release references.
+ */
+static int dbgfR3TypeUpdateRefCnts(PUVM pUVM, PCDBGFTYPEREG pReg, bool fRetain)
+{
+    int rc = VINF_SUCCESS;
+
+    switch (pReg->enmVariant)
+    {
+        case DBGFTYPEVARIANT_ALIAS:
+        {
+            AssertPtr(pReg->pszAliasedType);
+
+            PDBGFTYPE pAlias = dbgfR3TypeLookup(pUVM, pReg->pszAliasedType);
+            AssertPtr(pAlias);
+
+            if (fRetain)
+                pAlias->cRefs++;
+            else
+                pAlias->cRefs--;
+            break;
+        }
+        case DBGFTYPEVARIANT_STRUCT:
+        case DBGFTYPEVARIANT_UNION:
+        {
+            for (uint32_t i = 0; i < pReg->cMembers; i++)
+            {
+                PCDBGFTYPEREGMEMBER pMember = &pReg->paMembers[i];
+
+                /* Use the current pointer size. */
+                PDBGFTYPE pTypeMember = dbgfR3TypeLookup(pUVM, pMember->pszType);
+                AssertPtr(pTypeMember);
+
+                if (fRetain)
+                    pTypeMember->cRefs++;
+                else
+                    pTypeMember->cRefs--;
+            }
+            break;
+        }
+        default:
+            AssertMsgFailedBreakStmt(("Invalid type variant: %d", pReg->enmVariant),
+                                     rc = VERR_INVALID_PARAMETER);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Registers a single type in the database.
+ *
+ * @returns VBox status code.
+ * @retval VERR_ALREADY_EXISTS if the type exists already.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pReg                The type registration structure.
+ */
+static int dbgfR3TypeRegister(PUVM pUVM, PCDBGFTYPEREG pReg)
+{
+    int rc = VINF_SUCCESS;
+
+    LogFlowFunc(("pUVM=%#p pReg=%#p{%s}\n", pUVM, pReg, pReg->pszType));
+
+    if (dbgfR3TypeLookup(pUVM, pReg->pszType) == NULL)
+    {
+        rc = dbgfR3TypeValidate(pUVM, pReg);
+        if (RT_SUCCESS(rc))
+        {
+            PDBGFTYPE pType = (PDBGFTYPE)MMR3HeapAllocZU(pUVM, MM_TAG_DBGF_TYPE, sizeof(DBGFTYPE));
+            if (RT_LIKELY(pType))
+            {
+                pType->Core.pszString = pReg->pszType;
+                pType->pReg           = pReg;
+                pType->cRefs          = 0;
+                pType->enmTypeBuiltin = DBGFTYPEBUILTIN_INVALID;
+                rc = dbgfR3TypeCalcSize(pUVM, pType, false /* fCalcNested */);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = dbgfR3TypeUpdateRefCnts(pUVM, pReg, true /* fRetain */);
+                    if (RT_SUCCESS(rc))
+                    {
+                        bool fSucc = RTStrSpaceInsert(&pUVM->dbgf.s.TypeSpace, &pType->Core);
+                        Assert(fSucc);
+                        if (!fSucc)
+                        {
+                            dbgfR3TypeUpdateRefCnts(pUVM, pReg, false /* fRetain */);
+                            rc = VERR_ALREADY_EXISTS;
+                        }
+                    }
+                }
+
+                if (RT_FAILURE(rc))
+                    MMR3HeapFree(pType);
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+    }
+    else
+        rc = VERR_ALREADY_EXISTS;
+
+    LogFlowFunc(("-> rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Registers a new built-in type
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   enmTypeBuiltin      The builtin type enum.
+ * @param   cbType              Size of the type in bytes.
+ * @param   pszType             The type identifier for the builtin type.
+ */
+static int dbgfR3TypeRegisterBuiltin(PUVM pUVM, DBGFTYPEBUILTIN enmTypeBuiltin,
+                                     size_t cbType, const char *pszType)
+{
+    LogFlowFunc(("pUVM=%#p enmBuiltin=%d pszType=%s\n", pUVM, enmTypeBuiltin, pszType));
+
+    AssertReturn(!dbgfR3TypeLookup(pUVM, pszType), VERR_INVALID_STATE);
+
+    int rc = VINF_SUCCESS;
+    PDBGFTYPE pType = (PDBGFTYPE)MMR3HeapAllocZU(pUVM, MM_TAG_DBGF_TYPE, sizeof(DBGFTYPE));
+    if (RT_LIKELY(pType))
+    {
+        pType->Core.pszString = pszType;
+        pType->pReg           = NULL;
+        pType->cRefs          = 0;
+        pType->cbType         = cbType;
+        pType->enmTypeBuiltin = enmTypeBuiltin;
+        bool fSucc = RTStrSpaceInsert(&pUVM->dbgf.s.TypeSpace, &pType->Core);
+        Assert(fSucc);
+        if (!fSucc)
+            rc = VERR_ALREADY_EXISTS;
+
+        if (RT_FAILURE(rc))
+            MMR3HeapFree(pType);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    return rc;
+}
+
+
+/**
+ * Registers builtin types.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ */
+static int dbgfTypeRegisterBuiltinTypes(PUVM pUVM)
+{
+    int rc = dbgfR3TypeRegisterBuiltin(pUVM, DBGFTYPEBUILTIN_UINT8,  sizeof(uint8_t),  "uint8_t");
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3TypeRegisterBuiltin(pUVM, DBGFTYPEBUILTIN_INT8,   sizeof(int8_t),   "int8_t");
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3TypeRegisterBuiltin(pUVM, DBGFTYPEBUILTIN_UINT16, sizeof(uint16_t), "uint16_t");
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3TypeRegisterBuiltin(pUVM, DBGFTYPEBUILTIN_INT16,  sizeof(int16_t),  "int16_t");
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3TypeRegisterBuiltin(pUVM, DBGFTYPEBUILTIN_UINT32, sizeof(uint32_t), "uint32_t");
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3TypeRegisterBuiltin(pUVM, DBGFTYPEBUILTIN_INT32,  sizeof(int32_t),  "int32_t");
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3TypeRegisterBuiltin(pUVM, DBGFTYPEBUILTIN_UINT64, sizeof(uint64_t), "uint64_t");
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3TypeRegisterBuiltin(pUVM, DBGFTYPEBUILTIN_INT64,  sizeof(int64_t),  "int64_t");
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3TypeRegisterBuiltin(pUVM, DBGFTYPEBUILTIN_PTR32,  sizeof(uint32_t), "ptr32_t");
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3TypeRegisterBuiltin(pUVM, DBGFTYPEBUILTIN_PTR64,  sizeof(uint64_t), "ptr64_t");
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3TypeRegisterBuiltin(pUVM, DBGFTYPEBUILTIN_PTR,                   0, "ptr_t");
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3TypeRegisterBuiltin(pUVM, DBGFTYPEBUILTIN_SIZE,                  0, "size_t");
+
+    return rc;
+}
+
+
+/**
+ * Parses a single entry for a given type and assigns the value from the byte buffer
+ * to the value entry.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pMember             The type member.
+ * @param   pValEntry           The value entry holding the value on success.
+ * @param   pbBuf               The raw byte buffer.
+ * @param   cbBuf               Size of the byte buffer.
+ * @param   pcbParsed           Where to store the amount of consumed bytes on success.
+ */
+static int dbgfR3TypeParseEntry(PUVM pUVM, PCDBGFTYPEREGMEMBER pMember, PDBGFTYPEVALENTRY pValEntry,
+                                uint8_t *pbBuf, size_t cbBuf, size_t *pcbParsed)
+{
+    int rc = VINF_SUCCESS;
+    PDBGFTYPE pTypeMember = dbgfR3TypeLookup(pUVM, pMember->pszType);
+    uint32_t cValBufs = 1;
+    size_t cbParsed = 0;
+    PDBGFTYPEVALBUF pValBuf = &pValEntry->Buf.Val;
+
+    AssertPtrReturn(pTypeMember, VERR_INVALID_STATE);
+
+    if (pMember->fFlags & DBGFTYPEREGMEMBER_F_ARRAY)
+    {
+        cValBufs = pMember->cElements;
+        pValBuf = (PDBGFTYPEVALBUF)MMR3HeapAllocZU(pUVM, MM_TAG_DBGF_TYPE, cValBufs * sizeof(DBGFTYPEVALBUF));
+        if (RT_UNLIKELY(!pValBuf))
+            rc = VERR_NO_MEMORY;
+
+        pValEntry->Buf.pVal = pValBuf;
+        pValEntry->cEntries = cValBufs;
+        pValEntry->cbType   = pTypeMember->cbType;
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        for (uint32_t iValBuf = 0; iValBuf < cValBufs && RT_SUCCESS(rc); iValBuf++)
+        {
+            size_t cbThisParsed = 0;
+
+            if (pTypeMember->pReg)
+            {
+                /* Compound or aliased type */
+                rc = dbgfR3TypeParseBufferByType(pUVM, pTypeMember, pbBuf, cbBuf,
+                                                 &pValBuf->pVal, &cbThisParsed);
+                if (RT_SUCCESS(rc))
+                    pValEntry->enmType = DBGFTYPEBUILTIN_COMPOUND;
+            }
+            else
+            {
+                void *pvVal = NULL;
+
+                switch (pTypeMember->enmTypeBuiltin)
+                {
+                    case DBGFTYPEBUILTIN_UINT8:
+                        pvVal = &pValBuf->u8;
+                        cbThisParsed = 1;
+                        break;
+                    case DBGFTYPEBUILTIN_INT8:
+                        pvVal = &pValBuf->i8;
+                        cbThisParsed = 1;
+                        break;
+                    case DBGFTYPEBUILTIN_UINT16:
+                        pvVal = &pValBuf->u16;
+                        cbThisParsed = 2;
+                        break;
+                    case DBGFTYPEBUILTIN_INT16:
+                        pvVal = &pValBuf->i16;
+                        cbThisParsed = 2;
+                        break;
+                    case DBGFTYPEBUILTIN_UINT32:
+                        pvVal = &pValBuf->u32;
+                        cbThisParsed = 4;
+                        break;
+                    case DBGFTYPEBUILTIN_INT32:
+                        pvVal = &pValBuf->i32;
+                        cbThisParsed = 4;
+                        break;
+                    case DBGFTYPEBUILTIN_UINT64:
+                        pvVal = &pValBuf->u64;
+                        cbThisParsed = 8;
+                        break;
+                    case DBGFTYPEBUILTIN_INT64:
+                        pvVal = &pValBuf->i64;
+                        cbThisParsed = 8;
+                        break;
+                    case DBGFTYPEBUILTIN_PTR32:
+                        pvVal = &pValBuf->GCPtr;
+                        cbThisParsed = 4;
+                        break;
+                    case DBGFTYPEBUILTIN_PTR64:
+                        pvVal = &pValBuf->GCPtr;
+                        cbThisParsed = 8;
+                        break;
+                    case DBGFTYPEBUILTIN_PTR:
+                        pvVal = &pValBuf->GCPtr;
+                        cbThisParsed = pTypeMember->cbType;
+                        break;
+                    case DBGFTYPEBUILTIN_SIZE:
+                        pvVal = &pValBuf->size;
+                        cbThisParsed = pTypeMember->cbType;
+                        break;
+                    case DBGFTYPEBUILTIN_FLOAT32:
+                    case DBGFTYPEBUILTIN_FLOAT64:
+                    case DBGFTYPEBUILTIN_COMPOUND:
+                    default:
+                        AssertMsgFailedBreakStmt(("Invalid built-in type specified: %d\n", pTypeMember->enmTypeBuiltin),
+                                                 rc = VERR_INVALID_STATE);
+                }
+
+                if (RT_SUCCESS(rc))
+                {
+                    pValEntry->enmType = pTypeMember->enmTypeBuiltin;
+                    if (cbBuf >= cbThisParsed)
+                        memcpy(pvVal, pbBuf, cbThisParsed);
+                    else
+                        rc = VERR_BUFFER_OVERFLOW;
+                }
+            }
+
+            pValBuf++;
+
+            cbParsed += cbThisParsed;
+            pbBuf    += cbThisParsed;
+            cbBuf    -= cbThisParsed;
+        }
+    }
+
+    if (   RT_FAILURE(rc)
+        && cValBufs > 1)
+        MMR3HeapFree(pValBuf);
+
+    if (RT_SUCCESS(rc))
+    {
+        pValEntry->cEntries = cValBufs;
+        *pcbParsed = cbParsed;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Parses the given byte buffer and returns the value based no the type information.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pType               The type information.
+ * @param   pbBuf               The byte buffer to parse.
+ * @param   cbBuf               Size of the buffer.
+ * @param   ppVal               Where to store the pointer to the value on success.
+ * @param   pcbParsed           How many bytes of the buffer we consumed.
+ */
+static int dbgfR3TypeParseBufferByType(PUVM pUVM, PDBGFTYPE pType, uint8_t *pbBuf, size_t cbBuf,
+                                       PDBGFTYPEVAL *ppVal, size_t *pcbParsed)
+{
+    int rc = VINF_SUCCESS;
+    uint32_t cEntries = pType->pReg ? pType->pReg->cMembers : 1;
+    PDBGFTYPEVAL pVal = (PDBGFTYPEVAL)MMR3HeapAllocZU(pUVM, MM_TAG_DBGF_TYPE,
+                                                      RT_UOFFSETOF_DYN(DBGFTYPEVAL, aEntries[cEntries]));
+    if (RT_LIKELY(pVal))
+    {
+        size_t cbParsed = 0;
+
+        pVal->pTypeReg = pType->pReg;
+        for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+        {
+            PCDBGFTYPEREGMEMBER pMember = &pType->pReg->paMembers[i];
+            PDBGFTYPEVALENTRY pValEntry = &pVal->aEntries[i];
+            rc = dbgfR3TypeParseEntry(pUVM, pMember, pValEntry, pbBuf, cbBuf, &cbParsed);
+            if (RT_SUCCESS(rc))
+            {
+                pbBuf += cbParsed;
+                cbBuf -= cbParsed;
+            }
+        }
+
+        if (RT_SUCCESS(rc))
+        {
+            pVal->cEntries = cEntries;
+            *pcbParsed = cbParsed;
+            *ppVal     = pVal;
+        }
+        else
+            MMR3HeapFree(pVal); /** @todo Leak for embedded structs. */
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    return rc;
+}
+
+
+/**
+ * Dumps one level of a typed value.
+ *
+ * @returns VBox status code.
+ * @param   pVal                The value to dump.
+ * @param   iLvl                The current level.
+ * @param   cLvlMax             The maximum level.
+ * @param   pfnDump             The dumper callback.
+ * @param   pvUser              The opaque user data to pass to the dumper callback.
+ */
+static int dbgfR3TypeValDump(PDBGFTYPEVAL pVal, uint32_t iLvl, uint32_t cLvlMax,
+                             PFNDBGFR3TYPEVALDUMP pfnDump, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    PCDBGFTYPEREG pType = pVal->pTypeReg;
+
+    for (uint32_t i = 0; i < pVal->cEntries && rc == VINF_SUCCESS; i++)
+    {
+        PCDBGFTYPEREGMEMBER pTypeMember = &pType->paMembers[i];
+        PDBGFTYPEVALENTRY pValEntry = &pVal->aEntries[i];
+        PDBGFTYPEVALBUF pValBuf = pValEntry->cEntries > 1 ? pValEntry->Buf.pVal : &pValEntry->Buf.Val;
+
+        rc = pfnDump(0 /* off */, pTypeMember->pszName, iLvl, pValEntry->enmType, pValEntry->cbType,
+                     pValBuf, pValEntry->cEntries, pvUser);
+        if (   rc == VINF_SUCCESS
+            && pValEntry->enmType == DBGFTYPEBUILTIN_COMPOUND
+            && iLvl < cLvlMax)
+        {
+            /* Print embedded structs. */
+            for (uint32_t iValBuf = 0; iValBuf < pValEntry->cEntries && rc == VINF_SUCCESS; iValBuf++)
+                rc = dbgfR3TypeValDump(pValBuf[iValBuf].pVal, iLvl + 1, cLvlMax, pfnDump, pvUser);
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Dumps one level of a type.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pType               The type to dump.
+ * @param   iLvl                The current level.
+ * @param   cLvlMax             The maximum level.
+ * @param   pfnDump             The dumper callback.
+ * @param   pvUser              The opaque user data to pass to the dumper callback.
+ */
+static int dbgfR3TypeDump(PUVM pUVM, PDBGFTYPE pType, uint32_t iLvl, uint32_t cLvlMax,
+                          PFNDBGFR3TYPEDUMP pfnDump, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    PCDBGFTYPEREG pTypeReg = pType->pReg;
+
+    switch (pTypeReg->enmVariant)
+    {
+        case DBGFTYPEVARIANT_ALIAS:
+            rc = VERR_NOT_IMPLEMENTED;
+            break;
+        case DBGFTYPEVARIANT_STRUCT:
+        case DBGFTYPEVARIANT_UNION:
+            for (uint32_t i = 0; i < pTypeReg->cMembers && rc == VINF_SUCCESS; i++)
+            {
+                PCDBGFTYPEREGMEMBER pTypeMember = &pTypeReg->paMembers[i];
+                PDBGFTYPE pTypeResolved = dbgfR3TypeLookup(pUVM, pTypeMember->pszType);
+
+                rc = pfnDump(0 /* off */, pTypeMember->pszName, iLvl, pTypeMember->pszType,
+                             pTypeMember->fFlags, pTypeMember->cElements, pvUser);
+                if (   rc == VINF_SUCCESS
+                    && pTypeResolved->pReg
+                    && iLvl < cLvlMax)
+                {
+                    /* Print embedded structs. */
+                    rc = dbgfR3TypeDump(pUVM, pTypeResolved, iLvl + 1, cLvlMax, pfnDump, pvUser);
+                }
+            }
+            break;
+        default:
+            AssertMsgFailed(("Invalid type variant: %u\n", pTypeReg->enmVariant));
+            rc = VERR_INVALID_STATE;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Initializes the type database.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ */
+DECLHIDDEN(int)  dbgfR3TypeInit(PUVM pUVM)
+{
+    int  rc   = VINF_SUCCESS;
+    if (!pUVM->dbgf.s.fTypeDbInitialized)
+    {
+        rc = RTSemRWCreate(&pUVM->dbgf.s.hTypeDbLock);
+        if (RT_SUCCESS(rc))
+        {
+            rc = dbgfTypeRegisterBuiltinTypes(pUVM);
+            if (RT_FAILURE(rc))
+            {
+                RTSemRWDestroy(pUVM->dbgf.s.hTypeDbLock);
+                pUVM->dbgf.s.hTypeDbLock = NIL_RTSEMRW;
+            }
+        }
+        pUVM->dbgf.s.fTypeDbInitialized = RT_SUCCESS(rc);
+    }
+    return rc;
+}
+
+
+/**
+ * Terminates the type database.
+ *
+ * @param   pUVM                The user mode VM handle.
+ */
+DECLHIDDEN(void) dbgfR3TypeTerm(PUVM pUVM)
+{
+    RTSemRWDestroy(pUVM->dbgf.s.hTypeDbLock);
+    pUVM->dbgf.s.hTypeDbLock = NIL_RTSEMRW;
+    pUVM->dbgf.s.fTypeDbInitialized = false;
+}
+
+
+/**
+ * Registers a new type for lookup.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_ALREADY_EXISTS if the type exists already.
+ * @param   pUVM                The user mode VM handle.
+ * @param   cTypes              Number of types to register.
+ * @param   paTypes             The array of type registration structures to register.
+ */
+VMMR3DECL(int) DBGFR3TypeRegister(PUVM pUVM, uint32_t cTypes, PCDBGFTYPEREG paTypes)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(cTypes > 0, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(paTypes, VERR_INVALID_POINTER);
+
+    int rc = VINF_SUCCESS;
+    if (!pUVM->dbgf.s.fTypeDbInitialized)
+    {
+        rc = dbgfR3TypeInit(pUVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    DBGF_TYPE_DB_LOCK_WRITE(pUVM);
+    for (uint32_t i = 0; i < cTypes && RT_SUCCESS(rc); i++)
+    {
+        rc = dbgfR3TypeRegister(pUVM, &paTypes[i]);
+        if (   RT_FAILURE(rc)
+            && i > 0)
+        {
+            /* Deregister types in reverse order. */
+            do
+            {
+                int rc2 = DBGFR3TypeDeregister(pUVM, paTypes[i].pszType);
+                AssertRC(rc2);
+                i--;
+            } while (i > 0);
+
+            break;
+        }
+    }
+    DBGF_TYPE_DB_UNLOCK_WRITE(pUVM);
+
+    return rc;
+}
+
+
+/**
+ * Deregisters a previously registered type.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_NOT_FOUND if the type is not known.
+ * @retval  VERR_RESOURCE_IN_USE if the type is used by another type.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszType             The type identifier to deregister.
+ */
+VMMR3DECL(int) DBGFR3TypeDeregister(PUVM pUVM, const char *pszType)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszType, VERR_INVALID_POINTER);
+
+    int rc = VINF_SUCCESS;
+    if (!pUVM->dbgf.s.fTypeDbInitialized)
+    {
+        rc = dbgfR3TypeInit(pUVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    DBGF_TYPE_DB_LOCK_WRITE(pUVM);
+    PDBGFTYPE pType = dbgfR3TypeLookup(pUVM, pszType);
+    if (pType)
+    {
+        if (!pType->cRefs)
+        {
+
+        }
+        else
+            rc = VERR_RESOURCE_IN_USE;
+    }
+    else
+        rc = VERR_NOT_FOUND;
+    DBGF_TYPE_DB_UNLOCK_WRITE(pUVM);
+
+    return rc;
+}
+
+
+/**
+ * Return the type registration structure for the given type identifier.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_NOT_FOUND if the type is not known.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszType             The type identifier to get the registration structure from.
+ * @param   ppTypeReg           Where to store the type registration structure on success.
+ */
+VMMR3DECL(int) DBGFR3TypeQueryReg(PUVM pUVM, const char *pszType, PCDBGFTYPEREG *ppTypeReg)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszType, VERR_INVALID_POINTER);
+    AssertPtrReturn(ppTypeReg, VERR_INVALID_POINTER);
+
+    int rc = VINF_SUCCESS;
+    if (!pUVM->dbgf.s.fTypeDbInitialized)
+    {
+        rc = dbgfR3TypeInit(pUVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    DBGF_TYPE_DB_LOCK_READ(pUVM);
+    PDBGFTYPE pType = dbgfR3TypeLookup(pUVM, pszType);
+    if (pType)
+        *ppTypeReg = pType->pReg;
+    else
+        rc = VERR_NOT_FOUND;
+    DBGF_TYPE_DB_UNLOCK_READ(pUVM);
+
+    LogFlowFunc(("-> rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Queries the size a given type would occupy in memory.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_NOT_FOUND if the type is not known.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszType             The type identifier.
+ * @param   pcbType             Where to store the amount of memory occupied in bytes.
+ */
+VMMR3DECL(int) DBGFR3TypeQuerySize(PUVM pUVM, const char *pszType, size_t *pcbType)
+{
+    LogFlowFunc(("pUVM=%#p pszType=%s pcbType=%#p\n", pUVM, pszType, pcbType));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszType, VERR_INVALID_POINTER);
+    AssertPtrReturn(pcbType, VERR_INVALID_POINTER);
+
+    int rc = VINF_SUCCESS;
+    if (!pUVM->dbgf.s.fTypeDbInitialized)
+    {
+        rc = dbgfR3TypeInit(pUVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    DBGF_TYPE_DB_LOCK_READ(pUVM);
+    PDBGFTYPE pType = dbgfR3TypeLookup(pUVM, pszType);
+    if (pType)
+        *pcbType = pType->cbType;
+    else
+        rc = VERR_NOT_FOUND;
+    DBGF_TYPE_DB_UNLOCK_READ(pUVM);
+
+    LogFlowFunc(("-> rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Sets the size of the given type in bytes.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_NOT_FOUND if the type is not known.
+ * @retval  VERR_NOT_SUPPORTED if changing the size of this type is not supported.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszType             The type identifier.
+ * @param   cbType              The size of the type in bytes.
+ *
+ * @note: This currently works only for the builtin pointer type without the explicit
+ *        size (ptr_t or DBGFTYPEBUILTIN_PTR).
+ */
+VMMR3DECL(int) DBGFR3TypeSetSize(PUVM pUVM, const char *pszType, size_t cbType)
+{
+    LogFlowFunc(("pUVM=%#p pszType=%s cbType=%zu\n", pUVM, pszType, cbType));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszType, VERR_INVALID_POINTER);
+    AssertReturn(cbType > 0, VERR_INVALID_PARAMETER);
+
+    int rc = VINF_SUCCESS;
+    if (!pUVM->dbgf.s.fTypeDbInitialized)
+    {
+        rc = dbgfR3TypeInit(pUVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    DBGF_TYPE_DB_LOCK_WRITE(pUVM);
+    PDBGFTYPE pType = dbgfR3TypeLookup(pUVM, pszType);
+    if (pType)
+    {
+        if (   !pType->pReg
+            && (   pType->enmTypeBuiltin == DBGFTYPEBUILTIN_PTR
+                || pType->enmTypeBuiltin == DBGFTYPEBUILTIN_SIZE))
+        {
+            if (pType->cbType != cbType)
+            {
+                pType->cbType = cbType;
+                rc = dbgfR3TypeRecalculateAllSizes(pUVM);
+            }
+        }
+        else
+            rc = VERR_NOT_SUPPORTED;
+    }
+    else
+        rc = VERR_NOT_FOUND;
+    DBGF_TYPE_DB_UNLOCK_WRITE(pUVM);
+
+    LogFlowFunc(("-> rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Dumps the type information of the given type.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszType             The type identifier.
+ * @param   fFlags              Flags to control the dumping (reserved, MBZ).
+ * @param   cLvlMax             Maximum levels to nest.
+ * @param   pfnDump             The dumper callback.
+ * @param   pvUser              Opaque user data.
+ */
+VMMR3DECL(int) DBGFR3TypeDumpEx(PUVM pUVM, const char *pszType, uint32_t fFlags,
+                                uint32_t cLvlMax, PFNDBGFR3TYPEDUMP pfnDump, void *pvUser)
+{
+    LogFlowFunc(("pUVM=%#p pszType=%s fFlags=%#x cLvlMax=%u pfnDump=%#p pvUser=%#p\n",
+                 pUVM, pszType, fFlags, cLvlMax, pfnDump, pvUser));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszType, VERR_INVALID_POINTER);
+    AssertPtrReturn(pfnDump, VERR_INVALID_POINTER);
+    RT_NOREF_PV(fFlags);
+
+    int rc = VINF_SUCCESS;
+    if (!pUVM->dbgf.s.fTypeDbInitialized)
+    {
+        rc = dbgfR3TypeInit(pUVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    DBGF_TYPE_DB_LOCK_READ(pUVM);
+    PDBGFTYPE pType = dbgfR3TypeLookup(pUVM, pszType);
+    if (pType)
+        rc = dbgfR3TypeDump(pUVM, pType, 0 /* iLvl */, cLvlMax, pfnDump, pvUser);
+    else
+        rc = VERR_NOT_FOUND;
+    DBGF_TYPE_DB_UNLOCK_READ(pUVM);
+
+    LogFlowFunc(("-> rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Returns the value of a memory buffer at the given address formatted for the given
+ * type.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_NOT_FOUND if the type is not known.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pAddress            The address to start reading from.
+ * @param   pszType             The type identifier.
+ * @param   ppVal               Where to store the pointer to the value structure
+ *                              on success.
+ */
+VMMR3DECL(int) DBGFR3TypeQueryValByType(PUVM pUVM, PCDBGFADDRESS pAddress, const char *pszType,
+                                        PDBGFTYPEVAL *ppVal)
+{
+    LogFlowFunc(("pUVM=%#p pAddress=%#p pszType=%s ppVal=%#p\n", pUVM, pAddress, pszType, ppVal));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pAddress, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszType, VERR_INVALID_POINTER);
+    AssertPtrReturn(ppVal, VERR_INVALID_POINTER);
+
+    int rc = VINF_SUCCESS;
+    if (!pUVM->dbgf.s.fTypeDbInitialized)
+    {
+        rc = dbgfR3TypeInit(pUVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    DBGF_TYPE_DB_LOCK_READ(pUVM);
+    PDBGFTYPE pType = dbgfR3TypeLookup(pUVM, pszType);
+    if (pType)
+    {
+        uint8_t *pbBuf = (uint8_t *)MMR3HeapAllocZU(pUVM, MM_TAG_DBGF_TYPE, pType->cbType);
+        if (RT_LIKELY(pbBuf))
+        {
+            rc = DBGFR3MemRead(pUVM, 0 /*idCpu*/, pAddress, pbBuf, pType->cbType);
+            if (RT_SUCCESS(rc))
+            {
+                /* Parse the buffer based on the type. */
+                size_t cbParsed = 0;
+                rc = dbgfR3TypeParseBufferByType(pUVM, pType, pbBuf, pType->cbType,
+                                                 ppVal, &cbParsed);
+            }
+
+            MMR3HeapFree(pbBuf);
+        }
+        else
+            rc = VERR_NO_MEMORY;
+    }
+    else
+        rc = VERR_NOT_FOUND;
+    DBGF_TYPE_DB_UNLOCK_READ(pUVM);
+
+    LogFlowFunc(("-> rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Frees all acquired resources of a value previously obtained with
+ * DBGFR3TypeQueryValByType().
+ *
+ * @returns nothing.
+ * @param   pVal                The value to free.
+ */
+VMMR3DECL(void) DBGFR3TypeValFree(PDBGFTYPEVAL pVal)
+{
+    AssertPtrReturnVoid(pVal);
+
+    for (uint32_t i = 0; i < pVal->cEntries; i++)
+    {
+        PDBGFTYPEVALENTRY pValEntry = &pVal->aEntries[i];
+        PDBGFTYPEVALBUF pValBuf = pValEntry->cEntries > 1 ? pValEntry->Buf.pVal : &pValEntry->Buf.Val;
+
+        if (pValEntry->enmType == DBGFTYPEBUILTIN_COMPOUND)
+            for (uint32_t iBuf = 0; iBuf < pValEntry->cEntries; iBuf++)
+                DBGFR3TypeValFree(pValBuf->pVal);
+
+        if (pValEntry->cEntries > 1)
+            MMR3HeapFree(pValEntry->Buf.pVal);
+    }
+
+    MMR3HeapFree(pVal);
+}
+
+
+/**
+ * Reads the guest memory with the given type and dumps the content of the type.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pAddress            The address to start reading from.
+ * @param   pszType             The type identifier.
+ * @param   fFlags              Flags for tweaking (reserved, must be zero).
+ * @param   cLvlMax             Maximum number of levels to expand embedded structs.
+ * @param   pfnDump             The dumper callback.
+ * @param   pvUser              The opaque user data to pass to the callback.
+ */
+VMMR3DECL(int) DBGFR3TypeValDumpEx(PUVM pUVM, PCDBGFADDRESS pAddress, const char *pszType, uint32_t fFlags,
+                                   uint32_t cLvlMax, FNDBGFR3TYPEVALDUMP pfnDump, void *pvUser)
+{
+    LogFlowFunc(("pUVM=%#p pAddress=%#p pszType=%s fFlags=%#x pfnDump=%#p pvUser=%#p\n",
+                 pUVM, pAddress, pszType, fFlags,pfnDump, pvUser));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pAddress, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszType, VERR_INVALID_POINTER);
+    AssertPtrReturn(pfnDump, VERR_INVALID_POINTER);
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(cLvlMax >= 1, VERR_INVALID_PARAMETER);
+
+    PDBGFTYPEVAL pVal = NULL;
+    int rc = DBGFR3TypeQueryValByType(pUVM, pAddress, pszType, &pVal);
+    if (RT_SUCCESS(rc))
+    {
+        rc = dbgfR3TypeValDump(pVal, 0 /* iLvl */, cLvlMax, pfnDump, pvUser);
+        DBGFR3TypeValFree(pVal);
+    }
+
+    LogFlowFunc(("-> rc=%Rrc\n", rc));
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFReg.cpp b/src/VBox/VMM/VMMR3/DBGFReg.cpp
new file mode 100644
index 00000000..fab4c1af
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFReg.cpp
@@ -0,0 +1,2719 @@
+/* $Id: DBGFReg.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Register Methods.
+ */
+
+/*
+ * Copyright (C) 2010-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/ctype.h>
+#include <iprt/string.h>
+#include <iprt/uint128.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** Locks the register database for writing. */
+#define DBGF_REG_DB_LOCK_WRITE(pUVM) \
+    do { \
+        int rcSem = RTSemRWRequestWrite((pUVM)->dbgf.s.hRegDbLock, RT_INDEFINITE_WAIT); \
+        AssertRC(rcSem); \
+    } while (0)
+
+/** Unlocks the register database after writing. */
+#define DBGF_REG_DB_UNLOCK_WRITE(pUVM) \
+    do { \
+        int rcSem = RTSemRWReleaseWrite((pUVM)->dbgf.s.hRegDbLock); \
+        AssertRC(rcSem); \
+    } while (0)
+
+/** Locks the register database for reading. */
+#define DBGF_REG_DB_LOCK_READ(pUVM) \
+    do { \
+        int rcSem = RTSemRWRequestRead((pUVM)->dbgf.s.hRegDbLock, RT_INDEFINITE_WAIT); \
+        AssertRC(rcSem); \
+    } while (0)
+
+/** Unlocks the register database after reading. */
+#define DBGF_REG_DB_UNLOCK_READ(pUVM) \
+    do { \
+        int rcSem = RTSemRWReleaseRead((pUVM)->dbgf.s.hRegDbLock); \
+        AssertRC(rcSem); \
+    } while (0)
+
+
+/** The max length of a set, register or sub-field name. */
+#define DBGF_REG_MAX_NAME       40
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Register set registration record type.
+ */
+typedef enum DBGFREGSETTYPE
+{
+    /** Invalid zero value. */
+    DBGFREGSETTYPE_INVALID = 0,
+    /** CPU record. */
+    DBGFREGSETTYPE_CPU,
+    /** Device record. */
+    DBGFREGSETTYPE_DEVICE,
+    /** End of valid record types. */
+    DBGFREGSETTYPE_END
+} DBGFREGSETTYPE;
+
+
+/**
+ * Register set registration record.
+ */
+typedef struct DBGFREGSET
+{
+    /** String space core. */
+    RTSTRSPACECORE          Core;
+    /** The registration record type. */
+    DBGFREGSETTYPE          enmType;
+    /** The user argument for the callbacks. */
+    union
+    {
+        /** The CPU view. */
+        PVMCPU              pVCpu;
+        /** The device view. */
+        PPDMDEVINS          pDevIns;
+        /** The general view. */
+        void               *pv;
+    } uUserArg;
+
+    /** The register descriptors. */
+    PCDBGFREGDESC           paDescs;
+    /** The number of register descriptors. */
+    uint32_t                cDescs;
+
+    /** Array of lookup records.
+     * The first part of the array runs parallel to paDescs, the rest are
+     * covering for aliases and bitfield variations.  It's done this way to
+     * simplify the query all operations. */
+    struct DBGFREGLOOKUP   *paLookupRecs;
+    /** The number of lookup records. */
+    uint32_t                cLookupRecs;
+
+    /** The register name prefix. */
+    char                    szPrefix[1];
+} DBGFREGSET;
+/** Pointer to a register registration record. */
+typedef DBGFREGSET *PDBGFREGSET;
+/** Pointer to a const register registration record. */
+typedef DBGFREGSET const *PCDBGFREGSET;
+
+
+/**
+ * Register lookup record.
+ */
+typedef struct DBGFREGLOOKUP
+{
+    /** The string space core. */
+    RTSTRSPACECORE      Core;
+    /** Pointer to the set. */
+    PCDBGFREGSET        pSet;
+    /** Pointer to the register descriptor. */
+    PCDBGFREGDESC       pDesc;
+    /** If an alias this points to the alias descriptor, NULL if not. */
+    PCDBGFREGALIAS      pAlias;
+    /** If a sub-field this points to the sub-field descriptor, NULL if not. */
+    PCDBGFREGSUBFIELD   pSubField;
+} DBGFREGLOOKUP;
+/** Pointer to a register lookup record. */
+typedef DBGFREGLOOKUP *PDBGFREGLOOKUP;
+/** Pointer to a const register lookup record. */
+typedef DBGFREGLOOKUP const *PCDBGFREGLOOKUP;
+
+
+/**
+ * Argument packet from DBGFR3RegNmQueryAll to dbgfR3RegNmQueryAllWorker.
+ */
+typedef struct DBGFR3REGNMQUERYALLARGS
+{
+    /** The output register array. */
+    PDBGFREGENTRYNM paRegs;
+    /** The number of entries in the output array. */
+    size_t          cRegs;
+    /** The current register number when enumerating the string space.
+     * @remarks Only used by EMT(0). */
+    size_t          iReg;
+} DBGFR3REGNMQUERYALLARGS;
+/** Pointer to a dbgfR3RegNmQueryAllWorker argument packet. */
+typedef DBGFR3REGNMQUERYALLARGS *PDBGFR3REGNMQUERYALLARGS;
+
+
+/**
+ * Argument packet passed by DBGFR3RegPrintfV to dbgfR3RegPrintfCbOutput and
+ * dbgfR3RegPrintfCbFormat.
+ */
+typedef struct DBGFR3REGPRINTFARGS
+{
+    /** The user mode VM handle. */
+    PUVM        pUVM;
+    /** The target CPU. */
+    VMCPUID     idCpu;
+    /** Set if we're looking at guest registers. */
+    bool        fGuestRegs;
+    /** The output buffer. */
+    char       *pszBuf;
+    /** The format string. */
+    const char *pszFormat;
+    /** The va list with format arguments. */
+    va_list     va;
+
+    /** The current buffer offset. */
+    size_t      offBuf;
+    /** The amount of buffer space left, not counting the terminator char. */
+    size_t      cchLeftBuf;
+    /** The status code of the whole operation.  First error is return,
+     * subsequent ones are suppressed. */
+    int         rc;
+} DBGFR3REGPRINTFARGS;
+/** Pointer to a DBGFR3RegPrintfV argument packet. */
+typedef DBGFR3REGPRINTFARGS *PDBGFR3REGPRINTFARGS;
+
+
+
+/**
+ * Initializes the register database.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ */
+int dbgfR3RegInit(PUVM pUVM)
+{
+    int  rc   = VINF_SUCCESS;
+    if (!pUVM->dbgf.s.fRegDbInitialized)
+    {
+        rc = RTSemRWCreate(&pUVM->dbgf.s.hRegDbLock);
+        pUVM->dbgf.s.fRegDbInitialized = RT_SUCCESS(rc);
+    }
+    return rc;
+}
+
+
+/**
+ * Terminates the register database.
+ *
+ * @param   pUVM                The user mode VM handle.
+ */
+void dbgfR3RegTerm(PUVM pUVM)
+{
+    RTSemRWDestroy(pUVM->dbgf.s.hRegDbLock);
+    pUVM->dbgf.s.hRegDbLock = NIL_RTSEMRW;
+    pUVM->dbgf.s.fRegDbInitialized = false;
+}
+
+
+/**
+ * Validates a register name.
+ *
+ * This is used for prefixes, aliases and field names.
+ *
+ * @returns true if valid, false if not.
+ * @param   pszName             The register name to validate.
+ * @param   chDot               Set to '.' if accepted, otherwise 0.
+ */
+static bool dbgfR3RegIsNameValid(const char *pszName, char chDot)
+{
+    const char *psz = pszName;
+    if (!RT_C_IS_ALPHA(*psz))
+        return false;
+    char ch;
+    while ((ch = *++psz))
+        if (   !RT_C_IS_LOWER(ch)
+            && !RT_C_IS_DIGIT(ch)
+            && ch != '_'
+            && ch != chDot)
+            return false;
+    if (psz - pszName > DBGF_REG_MAX_NAME)
+        return false;
+    return true;
+}
+
+
+/**
+ * Common worker for registering a register set.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   paRegisters         The register descriptors.
+ * @param   enmType             The set type.
+ * @param   pvUserArg           The user argument for the callbacks.
+ * @param   pszPrefix           The name prefix.
+ * @param   iInstance           The instance number to be appended to @a
+ *                              pszPrefix when creating the set name.
+ */
+static int dbgfR3RegRegisterCommon(PUVM pUVM, PCDBGFREGDESC paRegisters, DBGFREGSETTYPE enmType, void *pvUserArg,
+                                   const char *pszPrefix, uint32_t iInstance)
+{
+    /*
+     * Validate input.
+     */
+    /* The name components. */
+    AssertMsgReturn(dbgfR3RegIsNameValid(pszPrefix, 0), ("%s\n", pszPrefix), VERR_INVALID_NAME);
+    const char  *psz             = RTStrEnd(pszPrefix, RTSTR_MAX);
+    bool const   fNeedUnderscore = RT_C_IS_DIGIT(psz[-1]);
+    size_t const cchPrefix       = psz - pszPrefix + fNeedUnderscore;
+    AssertMsgReturn(cchPrefix < RT_SIZEOFMEMB(DBGFREGSET, szPrefix) - 4 - 1, ("%s\n", pszPrefix), VERR_INVALID_NAME);
+
+    AssertMsgReturn(iInstance <= 9999, ("%d\n", iInstance), VERR_INVALID_NAME);
+
+    /* The descriptors. */
+    uint32_t cLookupRecs = 0;
+    uint32_t iDesc;
+    for (iDesc = 0; paRegisters[iDesc].pszName != NULL; iDesc++)
+    {
+        AssertMsgReturn(dbgfR3RegIsNameValid(paRegisters[iDesc].pszName, 0), ("%s (#%u)\n", paRegisters[iDesc].pszName, iDesc), VERR_INVALID_NAME);
+
+        if (enmType == DBGFREGSETTYPE_CPU)
+            AssertMsgReturn(iDesc < (unsigned)DBGFREG_END && (unsigned)paRegisters[iDesc].enmReg == iDesc,
+                            ("%d iDesc=%d\n", paRegisters[iDesc].enmReg, iDesc),
+                            VERR_INVALID_PARAMETER);
+        else
+            AssertReturn(paRegisters[iDesc].enmReg == DBGFREG_END, VERR_INVALID_PARAMETER);
+        AssertReturn(   paRegisters[iDesc].enmType > DBGFREGVALTYPE_INVALID
+                     && paRegisters[iDesc].enmType < DBGFREGVALTYPE_END, VERR_INVALID_PARAMETER);
+        AssertMsgReturn(!(paRegisters[iDesc].fFlags & ~DBGFREG_FLAGS_READ_ONLY),
+                        ("%#x (#%u)\n", paRegisters[iDesc].fFlags, iDesc),
+                        VERR_INVALID_PARAMETER);
+        AssertPtrReturn(paRegisters[iDesc].pfnGet, VERR_INVALID_PARAMETER);
+        AssertReturn(RT_VALID_PTR(paRegisters[iDesc].pfnSet) || (paRegisters[iDesc].fFlags & DBGFREG_FLAGS_READ_ONLY),
+                     VERR_INVALID_PARAMETER);
+
+        uint32_t        iAlias    = 0;
+        PCDBGFREGALIAS  paAliases = paRegisters[iDesc].paAliases;
+        if (paAliases)
+        {
+            AssertPtrReturn(paAliases, VERR_INVALID_PARAMETER);
+            for (; paAliases[iAlias].pszName; iAlias++)
+            {
+                AssertMsgReturn(dbgfR3RegIsNameValid(paAliases[iAlias].pszName, 0), ("%s (%s)\n", paAliases[iAlias].pszName, paRegisters[iDesc].pszName), VERR_INVALID_NAME);
+                AssertReturn(   paAliases[iAlias].enmType > DBGFREGVALTYPE_INVALID
+                             && paAliases[iAlias].enmType < DBGFREGVALTYPE_END, VERR_INVALID_PARAMETER);
+            }
+        }
+
+        uint32_t          iSubField   = 0;
+        PCDBGFREGSUBFIELD paSubFields = paRegisters[iDesc].paSubFields;
+        if (paSubFields)
+        {
+            AssertPtrReturn(paSubFields, VERR_INVALID_PARAMETER);
+            for (; paSubFields[iSubField].pszName; iSubField++)
+            {
+                AssertMsgReturn(dbgfR3RegIsNameValid(paSubFields[iSubField].pszName, '.'), ("%s (%s)\n", paSubFields[iSubField].pszName, paRegisters[iDesc].pszName), VERR_INVALID_NAME);
+                AssertReturn(paSubFields[iSubField].iFirstBit + paSubFields[iSubField].cBits <= 128, VERR_INVALID_PARAMETER);
+                AssertReturn(paSubFields[iSubField].cBits + paSubFields[iSubField].cShift <= 128, VERR_INVALID_PARAMETER);
+                AssertPtrNullReturn(paSubFields[iSubField].pfnGet, VERR_INVALID_POINTER);
+                AssertPtrNullReturn(paSubFields[iSubField].pfnSet, VERR_INVALID_POINTER);
+            }
+        }
+
+        cLookupRecs += (1 + iAlias) * (1 + iSubField);
+    }
+
+    /* Check the instance number of the CPUs. */
+    AssertReturn(enmType != DBGFREGSETTYPE_CPU || iInstance < pUVM->cCpus, VERR_INVALID_CPU_ID);
+
+    /*
+     * Allocate a new record and all associated lookup records.
+     */
+    size_t cbRegSet = RT_UOFFSETOF_DYN(DBGFREGSET, szPrefix[cchPrefix + 4 + 1]);
+    cbRegSet = RT_ALIGN_Z(cbRegSet, 32);
+    size_t const offLookupRecArray = cbRegSet;
+    cbRegSet += cLookupRecs * sizeof(DBGFREGLOOKUP);
+
+    PDBGFREGSET pRegSet = (PDBGFREGSET)MMR3HeapAllocZU(pUVM, MM_TAG_DBGF_REG, cbRegSet);
+    if (!pRegSet)
+        return VERR_NO_MEMORY;
+
+    /*
+     * Initialize the new record.
+     */
+    pRegSet->Core.pszString = pRegSet->szPrefix;
+    pRegSet->enmType        = enmType;
+    pRegSet->uUserArg.pv    = pvUserArg;
+    pRegSet->paDescs        = paRegisters;
+    pRegSet->cDescs         = iDesc;
+    pRegSet->cLookupRecs    = cLookupRecs;
+    pRegSet->paLookupRecs   = (PDBGFREGLOOKUP)((uintptr_t)pRegSet + offLookupRecArray);
+    if (fNeedUnderscore)
+        RTStrPrintf(pRegSet->szPrefix, cchPrefix + 4 + 1, "%s_%u", pszPrefix, iInstance);
+    else
+        RTStrPrintf(pRegSet->szPrefix, cchPrefix + 4 + 1, "%s%u", pszPrefix, iInstance);
+
+
+    /*
+     * Initialize the lookup records. See DBGFREGSET::paLookupRecs.
+     */
+    char szName[DBGF_REG_MAX_NAME * 3 + 16];
+    strcpy(szName, pRegSet->szPrefix);
+    char *pszReg = strchr(szName, '\0');
+    *pszReg++ = '.';
+
+    /* Array parallel to the descriptors. */
+    int             rc = VINF_SUCCESS;
+    PDBGFREGLOOKUP  pLookupRec = &pRegSet->paLookupRecs[0];
+    for (iDesc = 0; paRegisters[iDesc].pszName != NULL && RT_SUCCESS(rc); iDesc++)
+    {
+        strcpy(pszReg, paRegisters[iDesc].pszName);
+        pLookupRec->Core.pszString = MMR3HeapStrDupU(pUVM, MM_TAG_DBGF_REG, szName);
+        if (!pLookupRec->Core.pszString)
+            rc = VERR_NO_STR_MEMORY;
+        pLookupRec->pSet      = pRegSet;
+        pLookupRec->pDesc     = &paRegisters[iDesc];
+        pLookupRec->pAlias    = NULL;
+        pLookupRec->pSubField = NULL;
+        pLookupRec++;
+    }
+
+    /* Aliases and sub-fields. */
+    for (iDesc = 0; paRegisters[iDesc].pszName != NULL && RT_SUCCESS(rc); iDesc++)
+    {
+        PCDBGFREGALIAS  pCurAlias  = NULL; /* first time we add sub-fields for the real name. */
+        PCDBGFREGALIAS  pNextAlias = paRegisters[iDesc].paAliases;
+        const char     *pszRegName = paRegisters[iDesc].pszName;
+        while (RT_SUCCESS(rc))
+        {
+            /* Add sub-field records. */
+            PCDBGFREGSUBFIELD paSubFields = paRegisters[iDesc].paSubFields;
+            if (paSubFields)
+            {
+                size_t cchReg = strlen(pszRegName);
+                memcpy(pszReg, pszRegName, cchReg);
+                char *pszSub = &pszReg[cchReg];
+                *pszSub++ = '.';
+                for (uint32_t iSubField = 0; paSubFields[iSubField].pszName && RT_SUCCESS(rc); iSubField++)
+                {
+                    strcpy(pszSub, paSubFields[iSubField].pszName);
+                    pLookupRec->Core.pszString = MMR3HeapStrDupU(pUVM, MM_TAG_DBGF_REG, szName);
+                    if (!pLookupRec->Core.pszString)
+                        rc = VERR_NO_STR_MEMORY;
+                    pLookupRec->pSet      = pRegSet;
+                    pLookupRec->pDesc     = &paRegisters[iDesc];
+                    pLookupRec->pAlias    = pCurAlias;
+                    pLookupRec->pSubField = &paSubFields[iSubField];
+                    pLookupRec++;
+                }
+            }
+
+            /* Advance to the next alias. */
+            pCurAlias = pNextAlias++;
+            if (!pCurAlias)
+                break;
+            pszRegName = pCurAlias->pszName;
+            if (!pszRegName)
+                break;
+
+            /* The alias record. */
+            strcpy(pszReg, pszRegName);
+            pLookupRec->Core.pszString = MMR3HeapStrDupU(pUVM, MM_TAG_DBGF_REG, szName);
+            if (!pLookupRec->Core.pszString)
+                rc = VERR_NO_STR_MEMORY;
+            pLookupRec->pSet      = pRegSet;
+            pLookupRec->pDesc     = &paRegisters[iDesc];
+            pLookupRec->pAlias    = pCurAlias;
+            pLookupRec->pSubField = NULL;
+            pLookupRec++;
+        }
+    }
+    Assert(pLookupRec == &pRegSet->paLookupRecs[pRegSet->cLookupRecs]);
+
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Insert the record into the register set string space and optionally into
+         * the CPU register set cache.
+         */
+        DBGF_REG_DB_LOCK_WRITE(pUVM);
+
+        bool fInserted = RTStrSpaceInsert(&pUVM->dbgf.s.RegSetSpace, &pRegSet->Core);
+        if (fInserted)
+        {
+            pUVM->dbgf.s.cRegs += pRegSet->cDescs;
+            if (enmType == DBGFREGSETTYPE_CPU)
+            {
+                if (pRegSet->cDescs > DBGFREG_ALL_COUNT)
+                    pUVM->dbgf.s.cRegs -= pRegSet->cDescs - DBGFREG_ALL_COUNT;
+                if (!strcmp(pszPrefix, "cpu"))
+                    pUVM->aCpus[iInstance].dbgf.s.pGuestRegSet = pRegSet;
+                else
+                    pUVM->aCpus[iInstance].dbgf.s.pHyperRegSet = pRegSet;
+            }
+
+            PDBGFREGLOOKUP  paLookupRecs = pRegSet->paLookupRecs;
+            uint32_t        iLookupRec   = pRegSet->cLookupRecs;
+            while (iLookupRec-- > 0)
+            {
+                bool fInserted2 = RTStrSpaceInsert(&pUVM->dbgf.s.RegSpace, &paLookupRecs[iLookupRec].Core);
+                AssertMsg(fInserted2, ("'%s'", paLookupRecs[iLookupRec].Core.pszString)); NOREF(fInserted2);
+            }
+
+            DBGF_REG_DB_UNLOCK_WRITE(pUVM);
+            return VINF_SUCCESS;
+        }
+
+        DBGF_REG_DB_UNLOCK_WRITE(pUVM);
+        rc = VERR_DUPLICATE;
+    }
+
+    /*
+     * Bail out.
+     */
+    for (uint32_t i = 0; i < pRegSet->cLookupRecs; i++)
+        MMR3HeapFree((char *)pRegSet->paLookupRecs[i].Core.pszString);
+    MMR3HeapFree(pRegSet);
+
+    return rc;
+}
+
+
+/**
+ * Registers a set of registers for a CPU.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   paRegisters     The register descriptors.
+ * @param   fGuestRegs      Set if it's the guest registers, clear if
+ *                          hypervisor registers.
+ */
+VMMR3_INT_DECL(int) DBGFR3RegRegisterCpu(PVM pVM, PVMCPU pVCpu, PCDBGFREGDESC paRegisters, bool fGuestRegs)
+{
+    PUVM pUVM = pVM->pUVM;
+    if (!pUVM->dbgf.s.fRegDbInitialized)
+    {
+        int rc = dbgfR3RegInit(pUVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return dbgfR3RegRegisterCommon(pUVM, paRegisters, DBGFREGSETTYPE_CPU, pVCpu,
+                                   fGuestRegs ? "cpu" : "hypercpu", pVCpu->idCpu);
+}
+
+
+/**
+ * Registers a set of registers for a device.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   paRegisters     The register descriptors.
+ * @param   pDevIns         The device instance. This will be the callback user
+ *                          argument.
+ * @param   pszPrefix       The device name.
+ * @param   iInstance       The device instance.
+ */
+VMMR3_INT_DECL(int) DBGFR3RegRegisterDevice(PVM pVM, PCDBGFREGDESC paRegisters, PPDMDEVINS pDevIns,
+                                            const char *pszPrefix, uint32_t iInstance)
+{
+    AssertPtrReturn(paRegisters, VERR_INVALID_POINTER);
+    AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszPrefix, VERR_INVALID_POINTER);
+
+    return dbgfR3RegRegisterCommon(pVM->pUVM, paRegisters, DBGFREGSETTYPE_DEVICE, pDevIns, pszPrefix, iInstance);
+}
+
+
+/**
+ * Clears the register value variable.
+ *
+ * @param   pValue              The variable to clear.
+ */
+DECLINLINE(void) dbgfR3RegValClear(PDBGFREGVAL pValue)
+{
+    pValue->au64[0] = 0;
+    pValue->au64[1] = 0;
+    pValue->au64[2] = 0;
+    pValue->au64[3] = 0;
+    pValue->au64[4] = 0;
+    pValue->au64[5] = 0;
+    pValue->au64[6] = 0;
+    pValue->au64[7] = 0;
+}
+
+
+/**
+ * Sets a 80-bit floating point variable to a 64-bit unsigned interger value.
+ *
+ * @param   pValue              The value.
+ * @param   u64                 The integer value.
+ */
+DECLINLINE(void) dbgfR3RegValR80SetU64(PDBGFREGVAL pValue, uint64_t u64)
+{
+    /** @todo fixme  */
+    pValue->r80.s.fSign       = 0;
+    pValue->r80.s.uExponent   = 16383;
+    pValue->r80.s.u64Mantissa = u64;
+}
+
+
+/**
+ * Sets a 80-bit floating point variable to a 64-bit unsigned interger value.
+ *
+ * @param   pValue              The value.
+ * @param   u128                The integer value.
+ */
+DECLINLINE(void) dbgfR3RegValR80SetU128(PDBGFREGVAL pValue, RTUINT128U u128)
+{
+    /** @todo fixme  */
+    pValue->r80.s.fSign       = 0;
+    pValue->r80.s.uExponent   = 16383;
+    pValue->r80.s.u64Mantissa = u128.s.Lo;
+}
+
+
+/**
+ * Get a 80-bit floating point variable as a 64-bit unsigned integer.
+ *
+ * @returns 64-bit unsigned integer.
+ * @param   pValue              The value.
+ */
+DECLINLINE(uint64_t) dbgfR3RegValR80GetU64(PCDBGFREGVAL pValue)
+{
+    /** @todo stupid, stupid MSC. */
+    return pValue->r80.s.u64Mantissa;
+}
+
+
+/**
+ * Get a 80-bit floating point variable as a 128-bit unsigned integer.
+ *
+ * @returns 128-bit unsigned integer.
+ * @param   pValue              The value.
+ */
+DECLINLINE(RTUINT128U) dbgfR3RegValR80GetU128(PCDBGFREGVAL pValue)
+{
+    /** @todo stupid, stupid MSC. */
+    RTUINT128U uRet;
+#if 0
+    uRet.s.Lo = (uint64_t)InVal.lrd;
+    uRet.s.Hi = (uint64_t)InVal.lrd / _4G / _4G;
+#else
+    uRet.s.Lo = pValue->r80.s.u64Mantissa;
+    uRet.s.Hi = 0;
+#endif
+    return uRet;
+}
+
+
+/**
+ * Performs a cast between register value types.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ *
+ * @param   pValue              The value to cast (input + output).
+ * @param   enmFromType         The input value.
+ * @param   enmToType           The desired output value.
+ */
+static int dbgfR3RegValCast(PDBGFREGVAL pValue, DBGFREGVALTYPE enmFromType, DBGFREGVALTYPE enmToType)
+{
+    DBGFREGVAL const InVal = *pValue;
+    dbgfR3RegValClear(pValue);
+
+    /* Note! No default cases here as gcc warnings about missing enum values
+             are desired. */
+    switch (enmFromType)
+    {
+        case DBGFREGVALTYPE_U8:
+            switch (enmToType)
+            {
+                case DBGFREGVALTYPE_U8:     pValue->u8        = InVal.u8; return VINF_SUCCESS;
+                case DBGFREGVALTYPE_U16:    pValue->u16       = InVal.u8; return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_U32:    pValue->u32       = InVal.u8; return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_U64:    pValue->u64       = InVal.u8; return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_U128:   pValue->u128.s.Lo = InVal.u8; return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_U256:   pValue->u256.Words.w0 = InVal.u8; return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_U512:   pValue->u512.Words.w0 = InVal.u8; return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_R80:    dbgfR3RegValR80SetU64(pValue, InVal.u8); return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_DTR:                                  return VERR_DBGF_UNSUPPORTED_CAST;
+
+                case DBGFREGVALTYPE_32BIT_HACK:
+                case DBGFREGVALTYPE_END:
+                case DBGFREGVALTYPE_INVALID:
+                    break;
+            }
+            break;
+
+        case DBGFREGVALTYPE_U16:
+            switch (enmToType)
+            {
+                case DBGFREGVALTYPE_U8:     pValue->u8        = InVal.u16;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U16:    pValue->u16       = InVal.u16;  return VINF_SUCCESS;
+                case DBGFREGVALTYPE_U32:    pValue->u32       = InVal.u16;  return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_U64:    pValue->u64       = InVal.u16;  return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_U128:   pValue->u128.s.Lo = InVal.u16;  return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_U256:   pValue->u256.Words.w0 = InVal.u16;  return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_U512:   pValue->u512.Words.w0 = InVal.u16;  return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_R80:    dbgfR3RegValR80SetU64(pValue, InVal.u16); return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_DTR:                                    return VERR_DBGF_UNSUPPORTED_CAST;
+
+                case DBGFREGVALTYPE_32BIT_HACK:
+                case DBGFREGVALTYPE_END:
+                case DBGFREGVALTYPE_INVALID:
+                    break;
+            }
+            break;
+
+        case DBGFREGVALTYPE_U32:
+            switch (enmToType)
+            {
+                case DBGFREGVALTYPE_U8:     pValue->u8        = InVal.u32;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U16:    pValue->u16       = InVal.u32;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U32:    pValue->u32       = InVal.u32;  return VINF_SUCCESS;
+                case DBGFREGVALTYPE_U64:    pValue->u64       = InVal.u32;  return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_U128:   pValue->u128.s.Lo = InVal.u32;  return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_U256:   pValue->u256.DWords.dw0 = InVal.u32;  return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_U512:   pValue->u512.DWords.dw0 = InVal.u32;  return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_R80:    dbgfR3RegValR80SetU64(pValue, InVal.u32); return VINF_DBGF_ZERO_EXTENDED_REGISTER;
+                case DBGFREGVALTYPE_DTR:                                    return VERR_DBGF_UNSUPPORTED_CAST;
+
+                case DBGFREGVALTYPE_32BIT_HACK:
+                case DBGFREGVALTYPE_END:
+                case DBGFREGVALTYPE_INVALID:
+                    break;
+            }
+            break;
+
+        case DBGFREGVALTYPE_U64:
+            switch (enmToType)
+            {
+                case DBGFREGVALTYPE_U8:     pValue->u8        = InVal.u64;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U16:    pValue->u16       = InVal.u64;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U32:    pValue->u32       = InVal.u64;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U64:    pValue->u64       = InVal.u64;  return VINF_SUCCESS;
+                case DBGFREGVALTYPE_U128:   pValue->u128.s.Lo = InVal.u64;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U256:   pValue->u256.QWords.qw0 = InVal.u64;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U512:   pValue->u512.QWords.qw0 = InVal.u64;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_R80:    dbgfR3RegValR80SetU64(pValue, InVal.u64); return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_DTR:                                    return VERR_DBGF_UNSUPPORTED_CAST;
+
+                case DBGFREGVALTYPE_32BIT_HACK:
+                case DBGFREGVALTYPE_END:
+                case DBGFREGVALTYPE_INVALID:
+                    break;
+            }
+            break;
+
+        case DBGFREGVALTYPE_U128:
+            switch (enmToType)
+            {
+                case DBGFREGVALTYPE_U8:     pValue->u8        = InVal.u128.s.Lo;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U16:    pValue->u16       = InVal.u128.s.Lo;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U32:    pValue->u32       = InVal.u128.s.Lo;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U64:    pValue->u64       = InVal.u128.s.Lo;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U128:   pValue->u128      = InVal.u128;       return VINF_SUCCESS;
+                case DBGFREGVALTYPE_U256:   pValue->u256.DQWords.dqw0 = InVal.u128; return VINF_SUCCESS;
+                case DBGFREGVALTYPE_U512:   pValue->u512.DQWords.dqw0 = InVal.u128; return VINF_SUCCESS;
+                case DBGFREGVALTYPE_R80:    dbgfR3RegValR80SetU128(pValue, InVal.u128); return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_DTR:                                          return VERR_DBGF_UNSUPPORTED_CAST;
+
+                case DBGFREGVALTYPE_32BIT_HACK:
+                case DBGFREGVALTYPE_END:
+                case DBGFREGVALTYPE_INVALID:
+                    break;
+            }
+            break;
+
+        case DBGFREGVALTYPE_U256:
+            switch (enmToType)
+            {
+                case DBGFREGVALTYPE_U8:     pValue->u8        = InVal.u256.Words.w0;        return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U16:    pValue->u16       = InVal.u256.Words.w0;        return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U32:    pValue->u32       = InVal.u256.DWords.dw0;      return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U64:    pValue->u64       = InVal.u256.QWords.qw0;      return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U128:   pValue->u128      = InVal.u256.DQWords.dqw0;    return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U256:   pValue->u256      = InVal.u256;                 return VINF_SUCCESS;
+                case DBGFREGVALTYPE_U512:   pValue->u512.OWords.ow0 = InVal.u256;           return VINF_SUCCESS;
+                case DBGFREGVALTYPE_R80:    dbgfR3RegValR80SetU128(pValue, InVal.u256.DQWords.dqw0); return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_DTR:                                                    return VERR_DBGF_UNSUPPORTED_CAST;
+
+                case DBGFREGVALTYPE_32BIT_HACK:
+                case DBGFREGVALTYPE_END:
+                case DBGFREGVALTYPE_INVALID:
+                    break;
+            }
+            break;
+
+        case DBGFREGVALTYPE_U512:
+            switch (enmToType)
+            {
+                case DBGFREGVALTYPE_U8:     pValue->u8        = InVal.u512.Words.w0;        return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U16:    pValue->u16       = InVal.u512.Words.w0;        return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U32:    pValue->u32       = InVal.u512.DWords.dw0;      return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U64:    pValue->u64       = InVal.u512.QWords.qw0;      return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U128:   pValue->u128      = InVal.u512.DQWords.dqw0;    return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U256:   pValue->u256      = InVal.u512.OWords.ow0;      return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U512:   pValue->u512      = InVal.u512;                 return VINF_SUCCESS;
+                case DBGFREGVALTYPE_R80:    dbgfR3RegValR80SetU128(pValue, InVal.u512.DQWords.dqw0); return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_DTR:                                                    return VERR_DBGF_UNSUPPORTED_CAST;
+
+                case DBGFREGVALTYPE_32BIT_HACK:
+                case DBGFREGVALTYPE_END:
+                case DBGFREGVALTYPE_INVALID:
+                    break;
+            }
+            break;
+
+        case DBGFREGVALTYPE_R80:
+            switch (enmToType)
+            {
+                case DBGFREGVALTYPE_U8:     pValue->u8        = (uint8_t )dbgfR3RegValR80GetU64(&InVal);  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U16:    pValue->u16       = (uint16_t)dbgfR3RegValR80GetU64(&InVal);  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U32:    pValue->u32       = (uint32_t)dbgfR3RegValR80GetU64(&InVal);  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U64:    pValue->u64       = (uint64_t)dbgfR3RegValR80GetU64(&InVal);  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U128:   pValue->u128      = dbgfR3RegValR80GetU128(&InVal);           return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U256:   pValue->u256.DQWords.dqw0 = dbgfR3RegValR80GetU128(&InVal);   return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U512:   pValue->u512.DQWords.dqw0 = dbgfR3RegValR80GetU128(&InVal);   return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_R80:    pValue->r80       = InVal.r80;          return VINF_SUCCESS;
+                case DBGFREGVALTYPE_DTR:                                            return VERR_DBGF_UNSUPPORTED_CAST;
+
+                case DBGFREGVALTYPE_32BIT_HACK:
+                case DBGFREGVALTYPE_END:
+                case DBGFREGVALTYPE_INVALID:
+                    break;
+            }
+            break;
+
+        case DBGFREGVALTYPE_DTR:
+            switch (enmToType)
+            {
+                case DBGFREGVALTYPE_U8:     pValue->u8        = InVal.dtr.u64Base;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U16:    pValue->u16       = InVal.dtr.u64Base;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U32:    pValue->u32       = InVal.dtr.u64Base;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U64:    pValue->u64       = InVal.dtr.u64Base;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U128:   pValue->u128.s.Lo = InVal.dtr.u64Base;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U256:   pValue->u256.QWords.qw0 = InVal.dtr.u64Base;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_U512:   pValue->u512.QWords.qw0 = InVal.dtr.u64Base;  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_R80:    dbgfR3RegValR80SetU64(pValue, InVal.dtr.u64Base);  return VINF_DBGF_TRUNCATED_REGISTER;
+                case DBGFREGVALTYPE_DTR:    pValue->dtr       = InVal.dtr;          return VINF_SUCCESS;
+
+                case DBGFREGVALTYPE_32BIT_HACK:
+                case DBGFREGVALTYPE_END:
+                case DBGFREGVALTYPE_INVALID:
+                    break;
+            }
+            break;
+
+        case DBGFREGVALTYPE_INVALID:
+        case DBGFREGVALTYPE_END:
+        case DBGFREGVALTYPE_32BIT_HACK:
+            break;
+    }
+
+    AssertMsgFailed(("%d / %d\n", enmFromType, enmToType));
+    return VERR_DBGF_UNSUPPORTED_CAST;
+}
+
+
+/**
+ * Worker for the CPU register queries.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               The virtual CPU ID.
+ * @param   enmReg              The register to query.
+ * @param   enmType             The desired return type.
+ * @param   fGuestRegs          Query guest CPU registers if set (true),
+ *                              hypervisor CPU registers if clear (false).
+ * @param   pValue              Where to return the register value.
+ */
+static DECLCALLBACK(int) dbgfR3RegCpuQueryWorkerOnCpu(PUVM pUVM, VMCPUID idCpu, DBGFREG enmReg, DBGFREGVALTYPE enmType,
+                                                      bool fGuestRegs, PDBGFREGVAL pValue)
+{
+    int rc = VINF_SUCCESS;
+    DBGF_REG_DB_LOCK_READ(pUVM);
+
+    /*
+     * Look up the register set of the specified CPU.
+     */
+    PDBGFREGSET pSet = fGuestRegs
+                     ? pUVM->aCpus[idCpu].dbgf.s.pGuestRegSet
+                     : pUVM->aCpus[idCpu].dbgf.s.pHyperRegSet;
+    if (RT_LIKELY(pSet))
+    {
+        /*
+         * Look up the register and get the register value.
+         */
+        if (RT_LIKELY(pSet->cDescs > (size_t)enmReg))
+        {
+            PCDBGFREGDESC pDesc = &pSet->paDescs[enmReg];
+
+            pValue->au64[0] = pValue->au64[1] = 0;
+            rc = pDesc->pfnGet(pSet->uUserArg.pv, pDesc, pValue);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Do the cast if the desired return type doesn't match what
+                 * the getter returned.
+                 */
+                if (pDesc->enmType == enmType)
+                    rc = VINF_SUCCESS;
+                else
+                    rc = dbgfR3RegValCast(pValue, pDesc->enmType, enmType);
+            }
+        }
+        else
+            rc = VERR_DBGF_REGISTER_NOT_FOUND;
+    }
+    else
+        rc = VERR_INVALID_CPU_ID;
+
+    DBGF_REG_DB_UNLOCK_READ(pUVM);
+    return rc;
+}
+
+
+/**
+ * Internal worker for the CPU register query functions.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               The virtual CPU ID.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   enmReg              The register to query.
+ * @param   enmType             The desired return type.
+ * @param   pValue              Where to return the register value.
+ */
+static int dbgfR3RegCpuQueryWorker(PUVM pUVM, VMCPUID idCpu, DBGFREG enmReg, DBGFREGVALTYPE enmType, PDBGFREGVAL pValue)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+    AssertMsgReturn(enmReg >= DBGFREG_AL && enmReg <= DBGFREG_END, ("%d\n", enmReg), VERR_INVALID_PARAMETER);
+
+    bool const fGuestRegs = !(idCpu & DBGFREG_HYPER_VMCPUID);
+    idCpu &= ~DBGFREG_HYPER_VMCPUID;
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_CPU_ID);
+
+    return VMR3ReqPriorityCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3RegCpuQueryWorkerOnCpu, 6,
+                                    pUVM, idCpu, enmReg, enmType, fGuestRegs, pValue);
+}
+
+
+/**
+ * Queries a 8-bit CPU register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               The target CPU ID. Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   enmReg              The register that's being queried.
+ * @param   pu8                 Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegCpuQueryU8(PUVM pUVM, VMCPUID idCpu, DBGFREG enmReg, uint8_t *pu8)
+{
+    DBGFREGVAL Value;
+    int rc = dbgfR3RegCpuQueryWorker(pUVM, idCpu, enmReg, DBGFREGVALTYPE_U8, &Value);
+    if (RT_SUCCESS(rc))
+        *pu8 = Value.u8;
+    else
+        *pu8 = 0;
+    return rc;
+}
+
+
+/**
+ * Queries a 16-bit CPU register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               The target CPU ID.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   enmReg              The register that's being queried.
+ * @param   pu16                Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegCpuQueryU16(PUVM pUVM, VMCPUID idCpu, DBGFREG enmReg, uint16_t *pu16)
+{
+    DBGFREGVAL Value;
+    int rc = dbgfR3RegCpuQueryWorker(pUVM, idCpu, enmReg, DBGFREGVALTYPE_U16, &Value);
+    if (RT_SUCCESS(rc))
+        *pu16 = Value.u16;
+    else
+        *pu16 = 0;
+    return rc;
+}
+
+
+/**
+ * Queries a 32-bit CPU register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               The target CPU ID.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   enmReg              The register that's being queried.
+ * @param   pu32                Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegCpuQueryU32(PUVM pUVM, VMCPUID idCpu, DBGFREG enmReg, uint32_t *pu32)
+{
+    DBGFREGVAL Value;
+    int rc = dbgfR3RegCpuQueryWorker(pUVM, idCpu, enmReg, DBGFREGVALTYPE_U32, &Value);
+    if (RT_SUCCESS(rc))
+        *pu32 = Value.u32;
+    else
+        *pu32 = 0;
+    return rc;
+}
+
+
+/**
+ * Queries a 64-bit CPU register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               The target CPU ID.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   enmReg              The register that's being queried.
+ * @param   pu64                Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegCpuQueryU64(PUVM pUVM, VMCPUID idCpu, DBGFREG enmReg, uint64_t *pu64)
+{
+    DBGFREGVAL Value;
+    int rc = dbgfR3RegCpuQueryWorker(pUVM, idCpu, enmReg, DBGFREGVALTYPE_U64, &Value);
+    if (RT_SUCCESS(rc))
+        *pu64 = Value.u64;
+    else
+        *pu64 = 0;
+    return rc;
+}
+
+
+/**
+ * Queries a descriptor table register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               The target CPU ID.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   enmReg              The register that's being queried.
+ * @param   pu64Base            Where to store the register base value.
+ * @param   pu16Limit           Where to store the register limit value.
+ */
+VMMR3DECL(int) DBGFR3RegCpuQueryXdtr(PUVM pUVM, VMCPUID idCpu, DBGFREG enmReg, uint64_t *pu64Base, uint16_t *pu16Limit)
+{
+    DBGFREGVAL Value;
+    int rc = dbgfR3RegCpuQueryWorker(pUVM, idCpu, enmReg, DBGFREGVALTYPE_DTR, &Value);
+    if (RT_SUCCESS(rc))
+    {
+        *pu64Base  = Value.dtr.u64Base;
+        *pu16Limit = Value.dtr.u32Limit;
+    }
+    else
+    {
+        *pu64Base  = 0;
+        *pu16Limit = 0;
+    }
+    return rc;
+}
+
+
+#if 0 /* rewrite / remove */
+
+/**
+ * Wrapper around CPUMQueryGuestMsr for dbgfR3RegCpuQueryBatchWorker.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   pReg                The where to store the register value and
+ *                              size.
+ * @param   idMsr               The MSR to get.
+ */
+static void dbgfR3RegGetMsrBatch(PVMCPU pVCpu, PDBGFREGENTRY pReg, uint32_t idMsr)
+{
+    pReg->enmType = DBGFREGVALTYPE_U64;
+    int rc = CPUMQueryGuestMsr(pVCpu, idMsr, &pReg->Val.u64);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsg(rc == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", rc));
+        pReg->Val.u64 = 0;
+    }
+}
+
+
+static DECLCALLBACK(int) dbgfR3RegCpuQueryBatchWorker(PUVM pUVM, VMCPUID idCpu, PDBGFREGENTRY paRegs, size_t cRegs)
+{
+#if 0
+    PVMCPU    pVCpu = &pUVM->pVM->aCpus[idCpu];
+    PCCPUMCTX pCtx  = CPUMQueryGuestCtxPtr(pVCpu);
+
+    PDBGFREGENTRY pReg = paRegs - 1;
+    while (cRegs-- > 0)
+    {
+        pReg++;
+        pReg->Val.au64[0] = 0;
+        pReg->Val.au64[1] = 0;
+
+        DBGFREG const enmReg = pReg->enmReg;
+        AssertMsgReturn(enmReg >= 0 && enmReg <= DBGFREG_END, ("%d (%#x)\n", enmReg, enmReg), VERR_DBGF_REGISTER_NOT_FOUND);
+        if (enmReg != DBGFREG_END)
+        {
+            PCDBGFREGDESC pDesc = &g_aDbgfRegDescs[enmReg];
+            if (!pDesc->pfnGet)
+            {
+                PCRTUINT128U pu = (PCRTUINT128U)((uintptr_t)pCtx + pDesc->offCtx);
+                pReg->enmType = pDesc->enmType;
+                switch (pDesc->enmType)
+                {
+                    case DBGFREGVALTYPE_U8:     pReg->Val.u8   = pu->au8[0];   break;
+                    case DBGFREGVALTYPE_U16:    pReg->Val.u16  = pu->au16[0];  break;
+                    case DBGFREGVALTYPE_U32:    pReg->Val.u32  = pu->au32[0];  break;
+                    case DBGFREGVALTYPE_U64:    pReg->Val.u64  = pu->au64[0];  break;
+                    case DBGFREGVALTYPE_U128:
+                        pReg->Val.au64[0] = pu->au64[0];
+                        pReg->Val.au64[1] = pu->au64[1];
+                        break;
+                    case DBGFREGVALTYPE_R80:
+                        pReg->Val.au64[0] = pu->au64[0];
+                        pReg->Val.au16[5] = pu->au16[5];
+                        break;
+                    default:
+                        AssertMsgFailedReturn(("%s %d\n", pDesc->pszName, pDesc->enmType), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+                }
+            }
+            else
+            {
+                int rc = pDesc->pfnGet(pVCpu, pDesc, pCtx, &pReg->Val.u);
+                if (RT_FAILURE(rc))
+                    return rc;
+            }
+        }
+    }
+    return VINF_SUCCESS;
+#else
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/**
+ * Query a batch of registers.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               The target CPU ID.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   paRegs              Pointer to an array of @a cRegs elements.  On
+ *                              input the enmReg members indicates which
+ *                              registers to query.  On successful return the
+ *                              other members are set.  DBGFREG_END can be used
+ *                              as a filler.
+ * @param   cRegs               The number of entries in @a paRegs.
+ */
+VMMR3DECL(int) DBGFR3RegCpuQueryBatch(PUVM pUVM, VMCPUID idCpu, PDBGFREGENTRY paRegs, size_t cRegs)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, NULL);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_CPU_ID);
+    if (!cRegs)
+        return VINF_SUCCESS;
+    AssertReturn(cRegs < _1M, VERR_OUT_OF_RANGE);
+    AssertPtrReturn(paRegs, VERR_INVALID_POINTER);
+    size_t iReg = cRegs;
+    while (iReg-- > 0)
+    {
+        DBGFREG enmReg = paRegs[iReg].enmReg;
+        AssertMsgReturn(enmReg < DBGFREG_END && enmReg >= DBGFREG_AL, ("%d (%#x)", enmReg, enmReg), VERR_DBGF_REGISTER_NOT_FOUND);
+    }
+
+    return VMR3ReqCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3RegCpuQueryBatchWorker, 4, pUVM, idCpu, paRegs, cRegs);
+}
+
+
+/**
+ * Query all registers for a Virtual CPU.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               The target CPU ID.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   paRegs              Pointer to an array of @a cRegs elements.
+ *                              These will be filled with the CPU register
+ *                              values. Overflowing entries will be set to
+ *                              DBGFREG_END.  The returned registers can be
+ *                              accessed by using the DBGFREG values as index.
+ * @param   cRegs               The number of entries in @a paRegs.  The
+ *                              recommended value is DBGFREG_ALL_COUNT.
+ */
+VMMR3DECL(int) DBGFR3RegCpuQueryAll(PUVM pUVM, VMCPUID idCpu, PDBGFREGENTRY paRegs, size_t cRegs)
+{
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, NULL);
+    AssertReturn(idCpu < pUVM->cCpus, VERR_INVALID_CPU_ID);
+    if (!cRegs)
+        return VINF_SUCCESS;
+    AssertReturn(cRegs < _1M, VERR_OUT_OF_RANGE);
+    AssertPtrReturn(paRegs, VERR_INVALID_POINTER);
+
+    /*
+     * Convert it into a batch query (lazy bird).
+     */
+    unsigned iReg = 0;
+    while (iReg < cRegs && iReg < DBGFREG_ALL_COUNT)
+    {
+        paRegs[iReg].enmReg = (DBGFREG)iReg;
+        iReg++;
+    }
+    while (iReg < cRegs)
+        paRegs[iReg++].enmReg = DBGFREG_END;
+
+    return VMR3ReqCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3RegCpuQueryBatchWorker, 4, pUVM, idCpu, paRegs, cRegs);
+}
+
+#endif /* rewrite or remove? */
+
+/**
+ * Gets the name of a register.
+ *
+ * @returns Pointer to read-only register name (lower case).  NULL if the
+ *          parameters are invalid.
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   enmReg              The register identifier.
+ * @param   enmType             The register type.  This is for sort out
+ *                              aliases.  Pass DBGFREGVALTYPE_INVALID to get
+ *                              the standard name.
+ */
+VMMR3DECL(const char *) DBGFR3RegCpuName(PUVM pUVM, DBGFREG enmReg, DBGFREGVALTYPE enmType)
+{
+    AssertReturn(enmReg >= DBGFREG_AL && enmReg < DBGFREG_END, NULL);
+    AssertReturn(enmType >= DBGFREGVALTYPE_INVALID && enmType < DBGFREGVALTYPE_END, NULL);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, NULL);
+
+    PCDBGFREGSET    pSet    = pUVM->aCpus[0].dbgf.s.pGuestRegSet;
+    if (RT_UNLIKELY(!pSet))
+        return NULL;
+
+    PCDBGFREGDESC   pDesc   = &pSet->paDescs[enmReg];
+    PCDBGFREGALIAS  pAlias  = pDesc->paAliases;
+    if (   pAlias
+        && pDesc->enmType != enmType
+        && enmType != DBGFREGVALTYPE_INVALID)
+    {
+        while (pAlias->pszName)
+        {
+            if (pAlias->enmType == enmType)
+                return pAlias->pszName;
+            pAlias++;
+        }
+    }
+
+    return pDesc->pszName;
+}
+
+
+/**
+ * Fold the string to lower case and copy it into the destination buffer.
+ *
+ * @returns Number of folder characters, -1 on overflow.
+ * @param   pszSrc              The source string.
+ * @param   cchSrc              How much to fold and copy.
+ * @param   pszDst              The output buffer.
+ * @param   cbDst               The size of the output buffer.
+ */
+static ssize_t dbgfR3RegCopyToLower(const char *pszSrc, size_t cchSrc, char *pszDst, size_t cbDst)
+{
+    ssize_t cchFolded = 0;
+    char    ch;
+    while (cchSrc-- > 0 && (ch = *pszSrc++))
+    {
+        if (RT_UNLIKELY(cbDst <= 1))
+            return -1;
+        cbDst--;
+
+        char chLower = RT_C_TO_LOWER(ch);
+        cchFolded += chLower != ch;
+        *pszDst++ = chLower;
+    }
+    if (RT_UNLIKELY(!cbDst))
+        return -1;
+    *pszDst = '\0';
+    return cchFolded;
+}
+
+
+/**
+ * Resolves the register name.
+ *
+ * @returns Lookup record.
+ * @param   pUVM                The user mode VM handle.
+ * @param   idDefCpu            The default CPU ID set.
+ * @param   pszReg              The register name.
+ * @param   fGuestRegs          Default to guest CPU registers if set, the
+ *                              hypervisor CPU registers if clear.
+ */
+static PCDBGFREGLOOKUP dbgfR3RegResolve(PUVM pUVM, VMCPUID idDefCpu, const char *pszReg, bool fGuestRegs)
+{
+    DBGF_REG_DB_LOCK_READ(pUVM);
+
+    /* Try looking up the name without any case folding or cpu prefixing. */
+    PRTSTRSPACE pRegSpace = &pUVM->dbgf.s.RegSpace;
+    PCDBGFREGLOOKUP pLookupRec = (PCDBGFREGLOOKUP)RTStrSpaceGet(pRegSpace, pszReg);
+    if (!pLookupRec)
+    {
+        char szName[DBGF_REG_MAX_NAME * 4 + 16];
+
+        /* Lower case it and try again. */
+        ssize_t cchFolded = dbgfR3RegCopyToLower(pszReg, RTSTR_MAX, szName, sizeof(szName) - DBGF_REG_MAX_NAME);
+        if (cchFolded > 0)
+            pLookupRec = (PCDBGFREGLOOKUP)RTStrSpaceGet(pRegSpace, szName);
+        if (   !pLookupRec
+            && cchFolded >= 0
+            && idDefCpu != VMCPUID_ANY)
+        {
+            /* Prefix it with the specified CPU set. */
+            size_t cchCpuSet = RTStrPrintf(szName, sizeof(szName), fGuestRegs ? "cpu%u." : "hypercpu%u.", idDefCpu);
+            dbgfR3RegCopyToLower(pszReg, RTSTR_MAX, &szName[cchCpuSet], sizeof(szName) - cchCpuSet);
+            pLookupRec = (PCDBGFREGLOOKUP)RTStrSpaceGet(pRegSpace, szName);
+        }
+    }
+
+    DBGF_REG_DB_UNLOCK_READ(pUVM);
+    return pLookupRec;
+}
+
+
+/**
+ * Validates the register name.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if the register was found.
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND if not found.
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idDefCpu            The default CPU.
+ * @param   pszReg              The registe name.
+ */
+VMMR3DECL(int) DBGFR3RegNmValidate(PUVM pUVM, VMCPUID idDefCpu, const char *pszReg)
+{
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn((idDefCpu & ~DBGFREG_HYPER_VMCPUID) < pUVM->cCpus || idDefCpu == VMCPUID_ANY, VERR_INVALID_CPU_ID);
+    AssertPtrReturn(pszReg, VERR_INVALID_POINTER);
+
+    /*
+     * Resolve the register.
+     */
+    bool fGuestRegs = true;
+    if ((idDefCpu & DBGFREG_HYPER_VMCPUID) && idDefCpu != VMCPUID_ANY)
+    {
+        fGuestRegs = false;
+        idDefCpu &= ~DBGFREG_HYPER_VMCPUID;
+    }
+
+    PCDBGFREGLOOKUP pLookupRec = dbgfR3RegResolve(pUVM, idDefCpu, pszReg, fGuestRegs);
+    if (!pLookupRec)
+        return VERR_DBGF_REGISTER_NOT_FOUND;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * On CPU worker for the register queries, used by dbgfR3RegNmQueryWorker and
+ * dbgfR3RegPrintfCbFormatNormal.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   pLookupRec          The register lookup record.
+ * @param   enmType             The desired return type.
+ * @param   pValue              Where to return the register value.
+ * @param   penmType            Where to store the register value type.
+ *                              Optional.
+ */
+static DECLCALLBACK(int) dbgfR3RegNmQueryWorkerOnCpu(PUVM pUVM, PCDBGFREGLOOKUP pLookupRec, DBGFREGVALTYPE enmType,
+                                                     PDBGFREGVAL pValue, PDBGFREGVALTYPE penmType)
+{
+    PCDBGFREGDESC       pDesc        = pLookupRec->pDesc;
+    PCDBGFREGSET        pSet         = pLookupRec->pSet;
+    PCDBGFREGSUBFIELD   pSubField    = pLookupRec->pSubField;
+    DBGFREGVALTYPE      enmValueType = pDesc->enmType;
+    int                 rc;
+
+    NOREF(pUVM);
+
+    /*
+     * Get the register or sub-field value.
+     */
+    dbgfR3RegValClear(pValue);
+    if (!pSubField)
+    {
+        rc = pDesc->pfnGet(pSet->uUserArg.pv, pDesc, pValue);
+        if (   pLookupRec->pAlias
+            && pLookupRec->pAlias->enmType != enmValueType
+            && RT_SUCCESS(rc))
+        {
+            rc = dbgfR3RegValCast(pValue, enmValueType, pLookupRec->pAlias->enmType);
+            enmValueType = pLookupRec->pAlias->enmType;
+        }
+    }
+    else
+    {
+        if (pSubField->pfnGet)
+        {
+            rc = pSubField->pfnGet(pSet->uUserArg.pv, pSubField, &pValue->u128);
+            enmValueType = DBGFREGVALTYPE_U128;
+        }
+        else
+        {
+            rc = pDesc->pfnGet(pSet->uUserArg.pv, pDesc, pValue);
+            if (   pLookupRec->pAlias
+                && pLookupRec->pAlias->enmType != enmValueType
+                && RT_SUCCESS(rc))
+            {
+                rc = dbgfR3RegValCast(pValue, enmValueType, pLookupRec->pAlias->enmType);
+                enmValueType = pLookupRec->pAlias->enmType;
+            }
+            if (RT_SUCCESS(rc))
+            {
+                rc = dbgfR3RegValCast(pValue, enmValueType, DBGFREGVALTYPE_U128);
+                if (RT_SUCCESS(rc))
+                {
+                    RTUInt128AssignShiftLeft(&pValue->u128, -pSubField->iFirstBit);
+                    RTUInt128AssignAndNFirstBits(&pValue->u128, pSubField->cBits);
+                    if (pSubField->cShift)
+                        RTUInt128AssignShiftLeft(&pValue->u128, pSubField->cShift);
+                }
+            }
+        }
+        if (RT_SUCCESS(rc))
+        {
+            unsigned const cBits = pSubField->cBits + pSubField->cShift;
+            if (cBits <= 8)
+                enmValueType = DBGFREGVALTYPE_U8;
+            else if (cBits <= 16)
+                enmValueType = DBGFREGVALTYPE_U16;
+            else if (cBits <= 32)
+                enmValueType = DBGFREGVALTYPE_U32;
+            else if (cBits <= 64)
+                enmValueType = DBGFREGVALTYPE_U64;
+            else
+                enmValueType = DBGFREGVALTYPE_U128;
+            rc = dbgfR3RegValCast(pValue, DBGFREGVALTYPE_U128, enmValueType);
+        }
+    }
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Do the cast if the desired return type doesn't match what
+         * the getter returned.
+         */
+        if (   enmValueType == enmType
+            || enmType == DBGFREGVALTYPE_END)
+        {
+            rc = VINF_SUCCESS;
+            if (penmType)
+                *penmType = enmValueType;
+        }
+        else
+        {
+            rc = dbgfR3RegValCast(pValue, enmValueType, enmType);
+            if (penmType)
+                *penmType = RT_SUCCESS(rc) ? enmType : enmValueType;
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Worker for the register queries.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idDefCpu            The virtual CPU ID for the default CPU register
+ *                              set.  Can be OR'ed with DBGFREG_HYPER_VMCPUID.
+ * @param   pszReg              The register to query.
+ * @param   enmType             The desired return type.
+ * @param   pValue              Where to return the register value.
+ * @param   penmType            Where to store the register value type.
+ *                              Optional.
+ */
+static int dbgfR3RegNmQueryWorker(PUVM pUVM, VMCPUID idDefCpu, const char *pszReg, DBGFREGVALTYPE enmType,
+                                  PDBGFREGVAL pValue, PDBGFREGVALTYPE penmType)
+{
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn((idDefCpu & ~DBGFREG_HYPER_VMCPUID) < pUVM->cCpus || idDefCpu == VMCPUID_ANY, VERR_INVALID_CPU_ID);
+    AssertPtrReturn(pszReg, VERR_INVALID_POINTER);
+
+    Assert(enmType > DBGFREGVALTYPE_INVALID && enmType <= DBGFREGVALTYPE_END);
+    AssertPtr(pValue);
+
+    /*
+     * Resolve the register and call the getter on the relevant CPU.
+     */
+    bool fGuestRegs = true;
+    if ((idDefCpu & DBGFREG_HYPER_VMCPUID) && idDefCpu != VMCPUID_ANY)
+    {
+        fGuestRegs = false;
+        idDefCpu &= ~DBGFREG_HYPER_VMCPUID;
+    }
+    PCDBGFREGLOOKUP pLookupRec = dbgfR3RegResolve(pUVM, idDefCpu, pszReg, fGuestRegs);
+    if (pLookupRec)
+    {
+        if (pLookupRec->pSet->enmType == DBGFREGSETTYPE_CPU)
+            idDefCpu = pLookupRec->pSet->uUserArg.pVCpu->idCpu;
+        else if (idDefCpu != VMCPUID_ANY)
+            idDefCpu &= ~DBGFREG_HYPER_VMCPUID;
+        return VMR3ReqPriorityCallWaitU(pUVM, idDefCpu, (PFNRT)dbgfR3RegNmQueryWorkerOnCpu, 5,
+                                        pUVM, pLookupRec, enmType, pValue, penmType);
+    }
+    return VERR_DBGF_REGISTER_NOT_FOUND;
+}
+
+
+/**
+ * Queries a descriptor table register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idDefCpu            The default target CPU ID, VMCPUID_ANY if not
+ *                              applicable. Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   pszReg              The register that's being queried.  Except for
+ *                              CPU registers, this must be on the form
+ *                              "set.reg[.sub]".
+ * @param   pValue              Where to store the register value.
+ * @param   penmType            Where to store the register value type.
+ */
+VMMR3DECL(int) DBGFR3RegNmQuery(PUVM pUVM, VMCPUID idDefCpu, const char *pszReg, PDBGFREGVAL pValue, PDBGFREGVALTYPE penmType)
+{
+    return dbgfR3RegNmQueryWorker(pUVM, idDefCpu, pszReg, DBGFREGVALTYPE_END, pValue, penmType);
+}
+
+
+/**
+ * Queries a 8-bit register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idDefCpu            The default target CPU ID, VMCPUID_ANY if not
+ *                              applicable. Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   pszReg              The register that's being queried.  Except for
+ *                              CPU registers, this must be on the form
+ *                              "set.reg[.sub]".
+ * @param   pu8                 Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegNmQueryU8(PUVM pUVM, VMCPUID idDefCpu, const char *pszReg, uint8_t *pu8)
+{
+    DBGFREGVAL Value;
+    int rc = dbgfR3RegNmQueryWorker(pUVM, idDefCpu, pszReg, DBGFREGVALTYPE_U8, &Value, NULL);
+    if (RT_SUCCESS(rc))
+        *pu8 = Value.u8;
+    else
+        *pu8 = 0;
+    return rc;
+}
+
+
+/**
+ * Queries a 16-bit register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idDefCpu            The default target CPU ID, VMCPUID_ANY if not
+ *                              applicable.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   pszReg              The register that's being queried.  Except for
+ *                              CPU registers, this must be on the form
+ *                              "set.reg[.sub]".
+ * @param   pu16                Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegNmQueryU16(PUVM pUVM, VMCPUID idDefCpu, const char *pszReg, uint16_t *pu16)
+{
+    DBGFREGVAL Value;
+    int rc = dbgfR3RegNmQueryWorker(pUVM, idDefCpu, pszReg, DBGFREGVALTYPE_U16, &Value, NULL);
+    if (RT_SUCCESS(rc))
+        *pu16 = Value.u16;
+    else
+        *pu16 = 0;
+    return rc;
+}
+
+
+/**
+ * Queries a 32-bit register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idDefCpu            The default target CPU ID, VMCPUID_ANY if not
+ *                              applicable.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   pszReg              The register that's being queried.  Except for
+ *                              CPU registers, this must be on the form
+ *                              "set.reg[.sub]".
+ * @param   pu32                Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegNmQueryU32(PUVM pUVM, VMCPUID idDefCpu, const char *pszReg, uint32_t *pu32)
+{
+    DBGFREGVAL Value;
+    int rc = dbgfR3RegNmQueryWorker(pUVM, idDefCpu, pszReg, DBGFREGVALTYPE_U32, &Value, NULL);
+    if (RT_SUCCESS(rc))
+        *pu32 = Value.u32;
+    else
+        *pu32 = 0;
+    return rc;
+}
+
+
+/**
+ * Queries a 64-bit register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idDefCpu            The default target CPU ID, VMCPUID_ANY if not
+ *                              applicable.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   pszReg              The register that's being queried.  Except for
+ *                              CPU registers, this must be on the form
+ *                              "set.reg[.sub]".
+ * @param   pu64                Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegNmQueryU64(PUVM pUVM, VMCPUID idDefCpu, const char *pszReg, uint64_t *pu64)
+{
+    DBGFREGVAL Value;
+    int rc = dbgfR3RegNmQueryWorker(pUVM, idDefCpu, pszReg, DBGFREGVALTYPE_U64, &Value, NULL);
+    if (RT_SUCCESS(rc))
+        *pu64 = Value.u64;
+    else
+        *pu64 = 0;
+    return rc;
+}
+
+
+/**
+ * Queries a 128-bit register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idDefCpu            The default target CPU ID, VMCPUID_ANY if not
+ *                              applicable.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   pszReg              The register that's being queried.  Except for
+ *                              CPU registers, this must be on the form
+ *                              "set.reg[.sub]".
+ * @param   pu128               Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegNmQueryU128(PUVM pUVM, VMCPUID idDefCpu, const char *pszReg, PRTUINT128U pu128)
+{
+    DBGFREGVAL Value;
+    int rc = dbgfR3RegNmQueryWorker(pUVM, idDefCpu, pszReg, DBGFREGVALTYPE_U128, &Value, NULL);
+    if (RT_SUCCESS(rc))
+        *pu128 = Value.u128;
+    else
+        pu128->s.Hi = pu128->s.Lo = 0;
+    return rc;
+}
+
+
+#if 0
+/**
+ * Queries a long double register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idDefCpu            The default target CPU ID, VMCPUID_ANY if not
+ *                              applicable.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   pszReg              The register that's being queried.  Except for
+ *                              CPU registers, this must be on the form
+ *                              "set.reg[.sub]".
+ * @param   plrd                Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegNmQueryLrd(PUVM pUVM, VMCPUID idDefCpu, const char *pszReg, long double *plrd)
+{
+    DBGFREGVAL Value;
+    int rc = dbgfR3RegNmQueryWorker(pUVM, idDefCpu, pszReg, DBGFREGVALTYPE_R80, &Value, NULL);
+    if (RT_SUCCESS(rc))
+        *plrd = Value.lrd;
+    else
+        *plrd = 0;
+    return rc;
+}
+#endif
+
+
+/**
+ * Queries a descriptor table register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idDefCpu            The default target CPU ID, VMCPUID_ANY if not
+ *                              applicable.  Can be OR'ed with
+ *                              DBGFREG_HYPER_VMCPUID.
+ * @param   pszReg              The register that's being queried.  Except for
+ *                              CPU registers, this must be on the form
+ *                              "set.reg[.sub]".
+ * @param   pu64Base            Where to store the register base value.
+ * @param   pu16Limit           Where to store the register limit value.
+ */
+VMMR3DECL(int) DBGFR3RegNmQueryXdtr(PUVM pUVM, VMCPUID idDefCpu, const char *pszReg, uint64_t *pu64Base, uint16_t *pu16Limit)
+{
+    DBGFREGVAL Value;
+    int rc = dbgfR3RegNmQueryWorker(pUVM, idDefCpu, pszReg, DBGFREGVALTYPE_DTR, &Value, NULL);
+    if (RT_SUCCESS(rc))
+    {
+        *pu64Base  = Value.dtr.u64Base;
+        *pu16Limit = Value.dtr.u32Limit;
+    }
+    else
+    {
+        *pu64Base  = 0;
+        *pu16Limit = 0;
+    }
+    return rc;
+}
+
+
+/// @todo VMMR3DECL(int) DBGFR3RegNmQueryBatch(PUVM pUVM,VMCPUID idDefCpu, DBGFREGENTRYNM paRegs, size_t cRegs);
+
+
+/**
+ * Gets the number of registers returned by DBGFR3RegNmQueryAll.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pcRegs              Where to return the register count.
+ */
+VMMR3DECL(int) DBGFR3RegNmQueryAllCount(PUVM pUVM, size_t *pcRegs)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    *pcRegs = pUVM->dbgf.s.cRegs;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Pad register entries.
+ *
+ * @param   paRegs          The output array.
+ * @param   cRegs           The size of the output array.
+ * @param   iReg            The first register to pad.
+ * @param   cRegsToPad      The number of registers to pad.
+ */
+static void dbgfR3RegNmQueryAllPadEntries(PDBGFREGENTRYNM paRegs, size_t cRegs, size_t iReg, size_t cRegsToPad)
+{
+    if (iReg < cRegs)
+    {
+        size_t iEndReg = iReg + cRegsToPad;
+        if (iEndReg > cRegs)
+            iEndReg = cRegs;
+        while (iReg < iEndReg)
+        {
+            paRegs[iReg].pszName = NULL;
+            paRegs[iReg].enmType = DBGFREGVALTYPE_END;
+            dbgfR3RegValClear(&paRegs[iReg].Val);
+            iReg++;
+        }
+    }
+}
+
+
+/**
+ * Query all registers in a set.
+ *
+ * @param   pSet            The set.
+ * @param   cRegsToQuery    The number of registers to query.
+ * @param   paRegs          The output array.
+ * @param   cRegs           The size of the output array.
+ */
+static void dbgfR3RegNmQueryAllInSet(PCDBGFREGSET pSet, size_t cRegsToQuery, PDBGFREGENTRYNM paRegs, size_t cRegs)
+{
+    if (cRegsToQuery > pSet->cDescs)
+        cRegsToQuery = pSet->cDescs;
+    if (cRegsToQuery > cRegs)
+        cRegsToQuery = cRegs;
+
+    for (size_t iReg = 0; iReg < cRegsToQuery; iReg++)
+    {
+        paRegs[iReg].enmType = pSet->paDescs[iReg].enmType;
+        paRegs[iReg].pszName = pSet->paLookupRecs[iReg].Core.pszString;
+        dbgfR3RegValClear(&paRegs[iReg].Val);
+        int rc2 = pSet->paDescs[iReg].pfnGet(pSet->uUserArg.pv, &pSet->paDescs[iReg], &paRegs[iReg].Val);
+        AssertRCSuccess(rc2);
+        if (RT_FAILURE(rc2))
+            dbgfR3RegValClear(&paRegs[iReg].Val);
+    }
+}
+
+
+/**
+ * @callback_method_impl{FNRTSTRSPACECALLBACK, Worker used by
+ *                      dbgfR3RegNmQueryAllWorker}
+ */
+static DECLCALLBACK(int)  dbgfR3RegNmQueryAllEnum(PRTSTRSPACECORE pStr, void *pvUser)
+{
+    PCDBGFREGSET pSet = (PCDBGFREGSET)pStr;
+    if (pSet->enmType != DBGFREGSETTYPE_CPU)
+    {
+        PDBGFR3REGNMQUERYALLARGS pArgs  = (PDBGFR3REGNMQUERYALLARGS)pvUser;
+        if (pArgs->iReg < pArgs->cRegs)
+            dbgfR3RegNmQueryAllInSet(pSet, pSet->cDescs, &pArgs->paRegs[pArgs->iReg], pArgs->cRegs - pArgs->iReg);
+        pArgs->iReg += pSet->cDescs;
+    }
+
+    return 0;
+}
+
+
+/**
+ * @callback_method_impl{FNVMMEMTRENDEZVOUS, Worker used by DBGFR3RegNmQueryAll}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) dbgfR3RegNmQueryAllWorker(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    PDBGFR3REGNMQUERYALLARGS    pArgs  = (PDBGFR3REGNMQUERYALLARGS)pvUser;
+    PDBGFREGENTRYNM             paRegs = pArgs->paRegs;
+    size_t const                cRegs  = pArgs->cRegs;
+    PUVM                        pUVM   = pVM->pUVM;
+    PUVMCPU                     pUVCpu = pVCpu->pUVCpu;
+
+    DBGF_REG_DB_LOCK_READ(pUVM);
+
+    /*
+     * My guest CPU registers.
+     */
+    size_t iCpuReg = pVCpu->idCpu * DBGFREG_ALL_COUNT;
+    if (pUVCpu->dbgf.s.pGuestRegSet)
+    {
+        if (iCpuReg < cRegs)
+            dbgfR3RegNmQueryAllInSet(pUVCpu->dbgf.s.pGuestRegSet, DBGFREG_ALL_COUNT, &paRegs[iCpuReg], cRegs - iCpuReg);
+    }
+    else
+        dbgfR3RegNmQueryAllPadEntries(paRegs, cRegs, iCpuReg, DBGFREG_ALL_COUNT);
+
+    /*
+     * My hypervisor CPU registers.
+     */
+    iCpuReg = pUVM->cCpus * DBGFREG_ALL_COUNT + pUVCpu->idCpu * DBGFREG_ALL_COUNT;
+    if (pUVCpu->dbgf.s.pHyperRegSet)
+    {
+        if (iCpuReg < cRegs)
+            dbgfR3RegNmQueryAllInSet(pUVCpu->dbgf.s.pHyperRegSet, DBGFREG_ALL_COUNT, &paRegs[iCpuReg], cRegs - iCpuReg);
+    }
+    else
+        dbgfR3RegNmQueryAllPadEntries(paRegs, cRegs, iCpuReg, DBGFREG_ALL_COUNT);
+
+    /*
+     * The primary CPU does all the other registers.
+     */
+    if (pUVCpu->idCpu == 0)
+    {
+        pArgs->iReg = pUVM->cCpus * DBGFREG_ALL_COUNT * 2;
+        RTStrSpaceEnumerate(&pUVM->dbgf.s.RegSetSpace, dbgfR3RegNmQueryAllEnum, pArgs);
+        dbgfR3RegNmQueryAllPadEntries(paRegs, cRegs, pArgs->iReg, cRegs);
+    }
+
+    DBGF_REG_DB_UNLOCK_READ(pUVM);
+    return VINF_SUCCESS; /* Ignore errors. */
+}
+
+
+/**
+ * Queries all register.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   paRegs              The output register value array.  The register
+ *                              name string is read only and shall not be freed
+ *                              or modified.
+ * @param   cRegs               The number of entries in @a paRegs.  The
+ *                              correct size can be obtained by calling
+ *                              DBGFR3RegNmQueryAllCount.
+ */
+VMMR3DECL(int) DBGFR3RegNmQueryAll(PUVM pUVM, PDBGFREGENTRYNM paRegs, size_t cRegs)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(paRegs, VERR_INVALID_POINTER);
+    AssertReturn(cRegs > 0, VERR_OUT_OF_RANGE);
+
+    DBGFR3REGNMQUERYALLARGS Args;
+    Args.paRegs = paRegs;
+    Args.cRegs  = cRegs;
+
+    return VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, dbgfR3RegNmQueryAllWorker, &Args);
+}
+
+
+/**
+ * On CPU worker for the register modifications, used by DBGFR3RegNmSet.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   pLookupRec          The register lookup record. Maybe be modified,
+ *                              so please pass a copy of the user's one.
+ * @param   pValue              The new register value.
+ * @param   pMask               Indicate which bits to modify.
+ */
+static DECLCALLBACK(int) dbgfR3RegNmSetWorkerOnCpu(PUVM pUVM, PDBGFREGLOOKUP pLookupRec,
+                                                   PCDBGFREGVAL pValue, PCDBGFREGVAL pMask)
+{
+    RT_NOREF_PV(pUVM);
+    PCDBGFREGSUBFIELD pSubField = pLookupRec->pSubField;
+    if (pSubField && pSubField->pfnSet)
+        return pSubField->pfnSet(pLookupRec->pSet->uUserArg.pv, pSubField, pValue->u128, pMask->u128);
+    return pLookupRec->pDesc->pfnSet(pLookupRec->pSet->uUserArg.pv, pLookupRec->pDesc, pValue, pMask);
+}
+
+
+/**
+ * Worker for the register setting.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_REGISTER_NOT_FOUND
+ * @retval  VERR_DBGF_UNSUPPORTED_CAST
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   idDefCpu            The virtual CPU ID for the default CPU register
+ *                              set.  Can be OR'ed with DBGFREG_HYPER_VMCPUID.
+ * @param   pszReg              The register to query.
+ * @param   pValue              The value to set
+ * @param   enmType             How to interpret the value in @a pValue.
+ */
+VMMR3DECL(int) DBGFR3RegNmSet(PUVM pUVM, VMCPUID idDefCpu, const char *pszReg, PCDBGFREGVAL pValue, DBGFREGVALTYPE enmType)
+{
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn((idDefCpu & ~DBGFREG_HYPER_VMCPUID) < pUVM->cCpus || idDefCpu == VMCPUID_ANY, VERR_INVALID_CPU_ID);
+    AssertPtrReturn(pszReg, VERR_INVALID_POINTER);
+    AssertReturn(enmType > DBGFREGVALTYPE_INVALID && enmType < DBGFREGVALTYPE_END, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pValue, VERR_INVALID_PARAMETER);
+
+    /*
+     * Resolve the register and check that it is writable.
+     */
+    bool fGuestRegs = true;
+    if ((idDefCpu & DBGFREG_HYPER_VMCPUID) && idDefCpu != VMCPUID_ANY)
+    {
+        fGuestRegs = false;
+        idDefCpu &= ~DBGFREG_HYPER_VMCPUID;
+    }
+    PCDBGFREGLOOKUP pLookupRec = dbgfR3RegResolve(pUVM, idDefCpu, pszReg, fGuestRegs);
+    if (pLookupRec)
+    {
+        PCDBGFREGDESC       pDesc        = pLookupRec->pDesc;
+        PCDBGFREGSET        pSet         = pLookupRec->pSet;
+        PCDBGFREGSUBFIELD   pSubField    = pLookupRec->pSubField;
+
+        if (  !(pDesc->fFlags & DBGFREG_FLAGS_READ_ONLY)
+            && (pSubField
+                ?    !(pSubField->fFlags & DBGFREGSUBFIELD_FLAGS_READ_ONLY)
+                  && (pSubField->pfnSet != NULL || pDesc->pfnSet != NULL)
+                : pDesc->pfnSet != NULL) )
+        {
+            /*
+             * Calculate the modification mask and cast the input value to the
+             * type of the target register.
+             */
+            DBGFREGVAL Mask  = DBGFREGVAL_INITIALIZE_ZERO;
+            DBGFREGVAL Value = DBGFREGVAL_INITIALIZE_ZERO;
+            switch (enmType)
+            {
+                case DBGFREGVALTYPE_U8:
+                    Value.u8  = pValue->u8;
+                    Mask.u8   = UINT8_MAX;
+                    break;
+                case DBGFREGVALTYPE_U16:
+                    Value.u16 = pValue->u16;
+                    Mask.u16  = UINT16_MAX;
+                    break;
+                case DBGFREGVALTYPE_U32:
+                    Value.u32 = pValue->u32;
+                    Mask.u32  = UINT32_MAX;
+                    break;
+                case DBGFREGVALTYPE_U64:
+                    Value.u64 = pValue->u64;
+                    Mask.u64  = UINT64_MAX;
+                    break;
+                case DBGFREGVALTYPE_U128:
+                    Value.u128 = pValue->u128;
+                    Mask.u128.s.Lo = UINT64_MAX;
+                    Mask.u128.s.Hi = UINT64_MAX;
+                    break;
+                case DBGFREGVALTYPE_U256:
+                    Value.u256 = pValue->u256;
+                    Mask.u256.QWords.qw0 = UINT64_MAX;
+                    Mask.u256.QWords.qw1 = UINT64_MAX;
+                    Mask.u256.QWords.qw2 = UINT64_MAX;
+                    Mask.u256.QWords.qw3 = UINT64_MAX;
+                    break;
+                case DBGFREGVALTYPE_U512:
+                    Value.u512 = pValue->u512;
+                    Mask.u512.QWords.qw0 = UINT64_MAX;
+                    Mask.u512.QWords.qw1 = UINT64_MAX;
+                    Mask.u512.QWords.qw2 = UINT64_MAX;
+                    Mask.u512.QWords.qw3 = UINT64_MAX;
+                    Mask.u512.QWords.qw4 = UINT64_MAX;
+                    Mask.u512.QWords.qw5 = UINT64_MAX;
+                    Mask.u512.QWords.qw6 = UINT64_MAX;
+                    Mask.u512.QWords.qw7 = UINT64_MAX;
+                    break;
+                case DBGFREGVALTYPE_R80:
+#ifdef RT_COMPILER_WITH_80BIT_LONG_DOUBLE
+                    Value.r80Ex.lrd = pValue->r80Ex.lrd;
+#else
+                    Value.r80Ex.au64[0] = pValue->r80Ex.au64[0];
+                    Value.r80Ex.au16[4] = pValue->r80Ex.au16[4];
+#endif
+                    Value.r80Ex.au64[0] = UINT64_MAX;
+                    Value.r80Ex.au16[4] = UINT16_MAX;
+                    break;
+                case DBGFREGVALTYPE_DTR:
+                    Value.dtr.u32Limit = pValue->dtr.u32Limit;
+                    Value.dtr.u64Base  = pValue->dtr.u64Base;
+                    Mask.dtr.u32Limit  = UINT32_MAX;
+                    Mask.dtr.u64Base   = UINT64_MAX;
+                    break;
+                case DBGFREGVALTYPE_32BIT_HACK:
+                case DBGFREGVALTYPE_END:
+                case DBGFREGVALTYPE_INVALID:
+                    AssertFailedReturn(VERR_INTERNAL_ERROR_3);
+            }
+
+            int rc = VINF_SUCCESS;
+            DBGFREGVALTYPE enmRegType = pDesc->enmType;
+            if (pSubField)
+            {
+                unsigned const cBits = pSubField->cBits + pSubField->cShift;
+                if (cBits <= 8)
+                    enmRegType = DBGFREGVALTYPE_U8;
+                else if (cBits <= 16)
+                    enmRegType = DBGFREGVALTYPE_U16;
+                else if (cBits <= 32)
+                    enmRegType = DBGFREGVALTYPE_U32;
+                else if (cBits <= 64)
+                    enmRegType = DBGFREGVALTYPE_U64;
+                else if (cBits <= 128)
+                    enmRegType = DBGFREGVALTYPE_U128;
+                else if (cBits <= 256)
+                    enmRegType = DBGFREGVALTYPE_U256;
+                else
+                    enmRegType = DBGFREGVALTYPE_U512;
+            }
+            else if (pLookupRec->pAlias)
+            {
+                /* Restrict the input to the size of the alias register. */
+                DBGFREGVALTYPE enmAliasType = pLookupRec->pAlias->enmType;
+                if (enmAliasType != enmType)
+                {
+                    rc = dbgfR3RegValCast(&Value, enmType, enmAliasType);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                    dbgfR3RegValCast(&Mask, enmType, enmAliasType);
+                    enmType = enmAliasType;
+                }
+            }
+
+            if (enmType != enmRegType)
+            {
+                int rc2 = dbgfR3RegValCast(&Value, enmType, enmRegType);
+                if (RT_FAILURE(rc2))
+                    return rc2;
+                if (rc2 != VINF_SUCCESS && rc == VINF_SUCCESS)
+                    rc2 = VINF_SUCCESS;
+                dbgfR3RegValCast(&Mask, enmType, enmRegType);
+            }
+
+            /*
+             * Subfields needs some extra processing if there is no subfield
+             * setter, since we'll be feeding it to the normal register setter
+             * instead.  The mask and value must be shifted and truncated to the
+             * subfield position.
+             */
+            if (pSubField && !pSubField->pfnSet)
+            {
+                /* The shift factor is for displaying a subfield value
+                   2**cShift times larger than the stored value.  We have
+                   to undo this before adjusting value and mask.  */
+                if (pSubField->cShift)
+                {
+                    /* Warn about trunction of the lower bits that get
+                       shifted out below. */
+                    if (rc == VINF_SUCCESS)
+                    {
+                        DBGFREGVAL Value2 = Value;
+                        RTUInt128AssignAndNFirstBits(&Value2.u128, -pSubField->cShift);
+                        if (!RTUInt128BitAreAllClear(&Value2.u128))
+                            rc = VINF_DBGF_TRUNCATED_REGISTER;
+                    }
+                    RTUInt128AssignShiftRight(&Value.u128, pSubField->cShift);
+                }
+
+                RTUInt128AssignAndNFirstBits(&Value.u128, pSubField->cBits);
+                if (rc == VINF_SUCCESS && RTUInt128IsNotEqual(&Value.u128, &Value.u128))
+                    rc = VINF_DBGF_TRUNCATED_REGISTER;
+                RTUInt128AssignAndNFirstBits(&Mask.u128,  pSubField->cBits);
+
+                RTUInt128AssignShiftLeft(&Value.u128, pSubField->iFirstBit);
+                RTUInt128AssignShiftLeft(&Mask.u128,  pSubField->iFirstBit);
+            }
+
+            /*
+             * Do the actual work on an EMT.
+             */
+            if (pSet->enmType == DBGFREGSETTYPE_CPU)
+                idDefCpu = pSet->uUserArg.pVCpu->idCpu;
+            else if (idDefCpu != VMCPUID_ANY)
+                idDefCpu &= ~DBGFREG_HYPER_VMCPUID;
+
+            int rc2 = VMR3ReqPriorityCallWaitU(pUVM, idDefCpu, (PFNRT)dbgfR3RegNmSetWorkerOnCpu, 4,
+                                               pUVM, pLookupRec, &Value, &Mask);
+
+            if (rc == VINF_SUCCESS || RT_FAILURE(rc2))
+                rc = rc2;
+            return rc;
+        }
+        return VERR_DBGF_READ_ONLY_REGISTER;
+    }
+    return VERR_DBGF_REGISTER_NOT_FOUND;
+}
+
+
+/**
+ * Internal worker for DBGFR3RegFormatValue, cbBuf is sufficent.
+ *
+ * @copydoc DBGFR3RegFormatValueEx
+ */
+DECLINLINE(ssize_t) dbgfR3RegFormatValueInt(char *pszBuf, size_t cbBuf, PCDBGFREGVAL pValue, DBGFREGVALTYPE enmType,
+                                            unsigned uBase, signed int cchWidth, signed int cchPrecision, uint32_t fFlags)
+{
+    switch (enmType)
+    {
+        case DBGFREGVALTYPE_U8:
+            return RTStrFormatU8(pszBuf, cbBuf, pValue->u8, uBase, cchWidth, cchPrecision, fFlags);
+        case DBGFREGVALTYPE_U16:
+            return RTStrFormatU16(pszBuf, cbBuf, pValue->u16, uBase, cchWidth, cchPrecision, fFlags);
+        case DBGFREGVALTYPE_U32:
+            return RTStrFormatU32(pszBuf, cbBuf, pValue->u32, uBase, cchWidth, cchPrecision, fFlags);
+        case DBGFREGVALTYPE_U64:
+            return RTStrFormatU64(pszBuf, cbBuf, pValue->u64, uBase, cchWidth, cchPrecision, fFlags);
+        case DBGFREGVALTYPE_U128:
+            return RTStrFormatU128(pszBuf, cbBuf, &pValue->u128, uBase, cchWidth, cchPrecision, fFlags);
+        case DBGFREGVALTYPE_U256:
+            return RTStrFormatU256(pszBuf, cbBuf, &pValue->u256, uBase, cchWidth, cchPrecision, fFlags);
+        case DBGFREGVALTYPE_U512:
+            return RTStrFormatU512(pszBuf, cbBuf, &pValue->u512, uBase, cchWidth, cchPrecision, fFlags);
+        case DBGFREGVALTYPE_R80:
+            return RTStrFormatR80u2(pszBuf, cbBuf, &pValue->r80Ex, cchWidth, cchPrecision, fFlags);
+        case DBGFREGVALTYPE_DTR:
+        {
+            ssize_t cch = RTStrFormatU64(pszBuf, cbBuf, pValue->dtr.u64Base,
+                                         16, 2+16, 0, RTSTR_F_SPECIAL | RTSTR_F_ZEROPAD);
+            AssertReturn(cch > 0, VERR_DBGF_REG_IPE_1);
+            pszBuf[cch++] = ':';
+            cch += RTStrFormatU64(&pszBuf[cch], cbBuf - cch, pValue->dtr.u32Limit,
+                                  16, 4, 0, RTSTR_F_ZEROPAD | RTSTR_F_32BIT);
+            return cch;
+        }
+
+        case DBGFREGVALTYPE_32BIT_HACK:
+        case DBGFREGVALTYPE_END:
+        case DBGFREGVALTYPE_INVALID:
+            break;
+        /* no default, want gcc warnings */
+    }
+
+    RTStrPrintf(pszBuf, cbBuf, "!enmType=%d!", enmType);
+    return VERR_DBGF_REG_IPE_2;
+}
+
+
+/**
+ * Format a register value, extended version.
+ *
+ * @returns The number of bytes returned, VERR_BUFFER_OVERFLOW on failure.
+ * @param   pszBuf          The output buffer.
+ * @param   cbBuf           The size of the output buffer.
+ * @param   pValue          The value to format.
+ * @param   enmType         The value type.
+ * @param   uBase           The base (ignored if not applicable).
+ * @param   cchWidth        The width if RTSTR_F_WIDTH is set, otherwise
+ *                          ignored.
+ * @param   cchPrecision    The width if RTSTR_F_PRECISION is set, otherwise
+ *                          ignored.
+ * @param   fFlags          String formatting flags, RTSTR_F_XXX.
+ */
+VMMR3DECL(ssize_t) DBGFR3RegFormatValueEx(char *pszBuf, size_t cbBuf, PCDBGFREGVAL pValue, DBGFREGVALTYPE enmType,
+                                          unsigned uBase, signed int cchWidth, signed int cchPrecision, uint32_t fFlags)
+{
+    /*
+     * Format to temporary buffer using worker shared with dbgfR3RegPrintfCbFormatNormal.
+     */
+    char szTmp[160];
+    ssize_t cchOutput = dbgfR3RegFormatValueInt(szTmp, sizeof(szTmp), pValue, enmType, uBase, cchWidth, cchPrecision, fFlags);
+    if (cchOutput > 0)
+    {
+        if ((size_t)cchOutput < cbBuf)
+            memcpy(pszBuf, szTmp, cchOutput + 1);
+        else
+        {
+            if (cbBuf)
+            {
+                memcpy(pszBuf, szTmp, cbBuf - 1);
+                pszBuf[cbBuf - 1] = '\0';
+            }
+            cchOutput = VERR_BUFFER_OVERFLOW;
+        }
+    }
+    return cchOutput;
+}
+
+
+/**
+ * Format a register value as hexadecimal and with default width according to
+ * the type.
+ *
+ * @returns The number of bytes returned, VERR_BUFFER_OVERFLOW on failure.
+ * @param   pszBuf          The output buffer.
+ * @param   cbBuf           The size of the output buffer.
+ * @param   pValue          The value to format.
+ * @param   enmType         The value type.
+ * @param   fSpecial        Same as RTSTR_F_SPECIAL.
+ */
+VMMR3DECL(ssize_t) DBGFR3RegFormatValue(char *pszBuf, size_t cbBuf, PCDBGFREGVAL pValue, DBGFREGVALTYPE enmType, bool fSpecial)
+{
+    int cchWidth = 0;
+    switch (enmType)
+    {
+        case DBGFREGVALTYPE_U8:     cchWidth = 2   + fSpecial*2; break;
+        case DBGFREGVALTYPE_U16:    cchWidth = 4   + fSpecial*2; break;
+        case DBGFREGVALTYPE_U32:    cchWidth = 8   + fSpecial*2; break;
+        case DBGFREGVALTYPE_U64:    cchWidth = 16  + fSpecial*2; break;
+        case DBGFREGVALTYPE_U128:   cchWidth = 32  + fSpecial*2; break;
+        case DBGFREGVALTYPE_U256:   cchWidth = 64  + fSpecial*2; break;
+        case DBGFREGVALTYPE_U512:   cchWidth = 128 + fSpecial*2; break;
+        case DBGFREGVALTYPE_R80:    cchWidth = 0; break;
+        case DBGFREGVALTYPE_DTR:    cchWidth = 16+1+4 + fSpecial*2; break;
+
+        case DBGFREGVALTYPE_32BIT_HACK:
+        case DBGFREGVALTYPE_END:
+        case DBGFREGVALTYPE_INVALID:
+            break;
+        /* no default, want gcc warnings */
+    }
+    uint32_t fFlags = RTSTR_F_ZEROPAD;
+    if (fSpecial)
+        fFlags |= RTSTR_F_SPECIAL;
+    if (cchWidth != 0)
+        fFlags |= RTSTR_F_WIDTH;
+    return DBGFR3RegFormatValueEx(pszBuf, cbBuf, pValue, enmType, 16, cchWidth, 0, fFlags);
+}
+
+
+/**
+ * Format a register using special hacks as well as sub-field specifications
+ * (the latter isn't implemented yet).
+ */
+static size_t
+dbgfR3RegPrintfCbFormatField(PDBGFR3REGPRINTFARGS pThis, PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
+                             PCDBGFREGLOOKUP pLookupRec, int cchWidth, int cchPrecision, unsigned fFlags)
+{
+    char szTmp[160];
+
+    NOREF(cchWidth); NOREF(cchPrecision); NOREF(fFlags);
+
+    /*
+     * Retrieve the register value.
+     */
+    DBGFREGVAL      Value;
+    DBGFREGVALTYPE  enmType;
+    int rc = dbgfR3RegNmQueryWorkerOnCpu(pThis->pUVM, pLookupRec, DBGFREGVALTYPE_END, &Value, &enmType);
+    if (RT_FAILURE(rc))
+    {
+        PCRTSTATUSMSG pErr = RTErrGet(rc);
+        if (pErr)
+            return pfnOutput(pvArgOutput, pErr->pszDefine, strlen(pErr->pszDefine));
+        return pfnOutput(pvArgOutput, szTmp, RTStrPrintf(szTmp, sizeof(szTmp), "rc=%d", rc));
+    }
+
+    char *psz = szTmp;
+
+    /*
+     * Special case: Format eflags.
+     */
+    if (   pLookupRec->pSet->enmType == DBGFREGSETTYPE_CPU
+        && pLookupRec->pDesc->enmReg == DBGFREG_RFLAGS
+        && pLookupRec->pSubField     == NULL)
+    {
+        rc = dbgfR3RegValCast(&Value, enmType, DBGFREGVALTYPE_U32);
+        AssertRC(rc);
+        uint32_t const efl = Value.u32;
+
+        /* the iopl */
+        psz += RTStrPrintf(psz, sizeof(szTmp) / 2, "iopl=%u ", X86_EFL_GET_IOPL(efl));
+
+        /* add flags */
+        static const struct
+        {
+            const char *pszSet;
+            const char *pszClear;
+            uint32_t fFlag;
+        } aFlags[] =
+        {
+            { "vip",NULL, X86_EFL_VIP },
+            { "vif",NULL, X86_EFL_VIF },
+            { "ac", NULL, X86_EFL_AC },
+            { "vm", NULL, X86_EFL_VM },
+            { "rf", NULL, X86_EFL_RF },
+            { "nt", NULL, X86_EFL_NT },
+            { "ov", "nv", X86_EFL_OF },
+            { "dn", "up", X86_EFL_DF },
+            { "ei", "di", X86_EFL_IF },
+            { "tf", NULL, X86_EFL_TF },
+            { "ng", "pl", X86_EFL_SF },
+            { "zr", "nz", X86_EFL_ZF },
+            { "ac", "na", X86_EFL_AF },
+            { "po", "pe", X86_EFL_PF },
+            { "cy", "nc", X86_EFL_CF },
+        };
+        for (unsigned i = 0; i < RT_ELEMENTS(aFlags); i++)
+        {
+            const char *pszAdd = aFlags[i].fFlag & efl ? aFlags[i].pszSet : aFlags[i].pszClear;
+            if (pszAdd)
+            {
+                *psz++ = *pszAdd++;
+                *psz++ = *pszAdd++;
+                if (*pszAdd)
+                    *psz++ = *pszAdd++;
+                *psz++ = ' ';
+            }
+        }
+
+        /* drop trailing space */
+        psz--;
+    }
+    else
+    {
+        /*
+         * General case.
+         */
+        AssertMsgFailed(("Not implemented: %s\n", pLookupRec->Core.pszString));
+        return pfnOutput(pvArgOutput, pLookupRec->Core.pszString, pLookupRec->Core.cchString);
+    }
+
+    /* Output the string. */
+    return pfnOutput(pvArgOutput, szTmp, psz - &szTmp[0]);
+}
+
+
+/**
+ * Formats a register having parsed up to the register name.
+ */
+static size_t
+dbgfR3RegPrintfCbFormatNormal(PDBGFR3REGPRINTFARGS pThis, PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
+                              PCDBGFREGLOOKUP pLookupRec, unsigned uBase, int cchWidth, int cchPrecision, unsigned fFlags)
+{
+    char szTmp[160];
+
+    /*
+     * Get the register value.
+     */
+    DBGFREGVAL      Value;
+    DBGFREGVALTYPE  enmType;
+    int rc = dbgfR3RegNmQueryWorkerOnCpu(pThis->pUVM, pLookupRec, DBGFREGVALTYPE_END, &Value, &enmType);
+    if (RT_FAILURE(rc))
+    {
+        PCRTSTATUSMSG pErr = RTErrGet(rc);
+        if (pErr)
+            return pfnOutput(pvArgOutput, pErr->pszDefine, strlen(pErr->pszDefine));
+        return pfnOutput(pvArgOutput, szTmp, RTStrPrintf(szTmp, sizeof(szTmp), "rc=%d", rc));
+    }
+
+    /*
+     * Format the value.
+     */
+    ssize_t cchOutput = dbgfR3RegFormatValueInt(szTmp, sizeof(szTmp), &Value, enmType, uBase, cchWidth, cchPrecision, fFlags);
+    if (RT_UNLIKELY(cchOutput <= 0))
+    {
+        AssertFailed();
+        return pfnOutput(pvArgOutput, "internal-error", sizeof("internal-error") - 1);
+    }
+    return pfnOutput(pvArgOutput, szTmp, cchOutput);
+}
+
+
+/**
+ * @callback_method_impl{FNSTRFORMAT}
+ */
+static DECLCALLBACK(size_t)
+dbgfR3RegPrintfCbFormat(void *pvArg, PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
+                        const char **ppszFormat, va_list *pArgs, int cchWidth,
+                        int cchPrecision, unsigned fFlags, char chArgSize)
+{
+    NOREF(pArgs); NOREF(chArgSize);
+
+    /*
+     * Parse the format type and hand the job to the appropriate worker.
+     */
+    PDBGFR3REGPRINTFARGS pThis = (PDBGFR3REGPRINTFARGS)pvArg;
+    const char *pszFormat = *ppszFormat;
+    if (    pszFormat[0] != 'V'
+        ||  pszFormat[1] != 'R')
+    {
+        AssertMsgFailed(("'%s'\n", pszFormat));
+        return 0;
+    }
+    unsigned offCurly = 2;
+    if (pszFormat[offCurly] != '{')
+    {
+        AssertMsgReturn(pszFormat[offCurly], ("'%s'\n", pszFormat), 0);
+        offCurly++;
+        AssertMsgReturn(pszFormat[offCurly] == '{', ("'%s'\n", pszFormat), 0);
+    }
+    const char *pachReg = &pszFormat[offCurly + 1];
+
+    /*
+     * The end and length of the register.
+     */
+    const char *pszEnd = strchr(pachReg, '}');
+    AssertMsgReturn(pszEnd, ("Missing closing curly bracket: '%s'\n", pszFormat), 0);
+    size_t const cchReg = pszEnd - pachReg;
+
+    /*
+     * Look up the register - same as dbgfR3RegResolve, except for locking and
+     * input string termination.
+     */
+    PRTSTRSPACE pRegSpace = &pThis->pUVM->dbgf.s.RegSpace;
+    /* Try looking up the name without any case folding or cpu prefixing. */
+    PCDBGFREGLOOKUP pLookupRec = (PCDBGFREGLOOKUP)RTStrSpaceGetN(pRegSpace, pachReg, cchReg);
+    if (!pLookupRec)
+    {
+        /* Lower case it and try again. */
+        char szName[DBGF_REG_MAX_NAME * 4 + 16];
+        ssize_t cchFolded = dbgfR3RegCopyToLower(pachReg, cchReg, szName, sizeof(szName) - DBGF_REG_MAX_NAME);
+        if (cchFolded > 0)
+            pLookupRec = (PCDBGFREGLOOKUP)RTStrSpaceGet(pRegSpace, szName);
+        if (   !pLookupRec
+            && cchFolded >= 0
+            && pThis->idCpu != VMCPUID_ANY)
+        {
+            /* Prefix it with the specified CPU set. */
+            size_t cchCpuSet = RTStrPrintf(szName, sizeof(szName), pThis->fGuestRegs ? "cpu%u." : "hypercpu%u.", pThis->idCpu);
+            dbgfR3RegCopyToLower(pachReg, cchReg, &szName[cchCpuSet], sizeof(szName) - cchCpuSet);
+            pLookupRec = (PCDBGFREGLOOKUP)RTStrSpaceGet(pRegSpace, szName);
+        }
+    }
+    AssertMsgReturn(pLookupRec, ("'%s'\n", pszFormat), 0);
+    AssertMsgReturn(   pLookupRec->pSet->enmType != DBGFREGSETTYPE_CPU
+                    || pLookupRec->pSet->uUserArg.pVCpu->idCpu == pThis->idCpu,
+                    ("'%s' idCpu=%u, pSet/cpu=%u\n", pszFormat, pThis->idCpu, pLookupRec->pSet->uUserArg.pVCpu->idCpu),
+                    0);
+
+    /*
+     * Commit the parsed format string.  Up to this point it is nice to know
+     * what register lookup failed and such, so we've delayed comitting.
+     */
+    *ppszFormat = pszEnd + 1;
+
+    /*
+     * Call the responsible worker.
+     */
+    switch (pszFormat[offCurly - 1])
+    {
+        case 'R': /* %VR{} */
+        case 'X': /* %VRX{} */
+            return dbgfR3RegPrintfCbFormatNormal(pThis, pfnOutput, pvArgOutput, pLookupRec,
+                                                 16, cchWidth, cchPrecision, fFlags);
+        case 'U':
+            return dbgfR3RegPrintfCbFormatNormal(pThis, pfnOutput, pvArgOutput, pLookupRec,
+                                                 10, cchWidth, cchPrecision, fFlags);
+        case 'O':
+            return dbgfR3RegPrintfCbFormatNormal(pThis, pfnOutput, pvArgOutput, pLookupRec,
+                                                 8, cchWidth, cchPrecision, fFlags);
+        case 'B':
+            return dbgfR3RegPrintfCbFormatNormal(pThis, pfnOutput, pvArgOutput, pLookupRec,
+                                                 2, cchWidth, cchPrecision, fFlags);
+        case 'F':
+            return dbgfR3RegPrintfCbFormatField(pThis, pfnOutput, pvArgOutput, pLookupRec, cchWidth, cchPrecision, fFlags);
+        default:
+            AssertFailed();
+            return 0;
+    }
+}
+
+
+
+/**
+ * @callback_method_impl{FNRTSTROUTPUT}
+ */
+static DECLCALLBACK(size_t)
+dbgfR3RegPrintfCbOutput(void *pvArg, const char *pachChars, size_t cbChars)
+{
+    PDBGFR3REGPRINTFARGS    pArgs    = (PDBGFR3REGPRINTFARGS)pvArg;
+    size_t                  cbToCopy = cbChars;
+    if (cbToCopy >= pArgs->cchLeftBuf)
+    {
+        if (RT_SUCCESS(pArgs->rc))
+            pArgs->rc = VERR_BUFFER_OVERFLOW;
+        cbToCopy = pArgs->cchLeftBuf;
+    }
+    if (cbToCopy > 0)
+    {
+        memcpy(&pArgs->pszBuf[pArgs->offBuf], pachChars, cbToCopy);
+        pArgs->offBuf     += cbToCopy;
+        pArgs->cchLeftBuf -= cbToCopy;
+        pArgs->pszBuf[pArgs->offBuf] = '\0';
+    }
+    return cbToCopy;
+}
+
+
+/**
+ * On CPU worker for the register formatting, used by DBGFR3RegPrintfV.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pArgs               The argument package and state.
+ */
+static DECLCALLBACK(int) dbgfR3RegPrintfWorkerOnCpu(PDBGFR3REGPRINTFARGS pArgs)
+{
+    DBGF_REG_DB_LOCK_READ(pArgs->pUVM);
+    RTStrFormatV(dbgfR3RegPrintfCbOutput, pArgs, dbgfR3RegPrintfCbFormat, pArgs, pArgs->pszFormat, pArgs->va);
+    DBGF_REG_DB_UNLOCK_READ(pArgs->pUVM);
+    return pArgs->rc;
+}
+
+
+/**
+ * Format a registers.
+ *
+ * This is restricted to registers from one CPU, that specified by @a idCpu.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               The CPU ID of any CPU registers that may be
+ *                              printed, pass VMCPUID_ANY if not applicable.
+ * @param   pszBuf              The output buffer.
+ * @param   cbBuf               The size of the output buffer.
+ * @param   pszFormat           The format string.  Register names are given by
+ *                              %VR{name}, they take no arguments.
+ * @param   va                  Other format arguments.
+ */
+VMMR3DECL(int) DBGFR3RegPrintfV(PUVM pUVM, VMCPUID idCpu, char *pszBuf, size_t cbBuf, const char *pszFormat, va_list va)
+{
+    AssertPtrReturn(pszBuf, VERR_INVALID_POINTER);
+    AssertReturn(cbBuf > 0, VERR_BUFFER_OVERFLOW);
+    *pszBuf = '\0';
+
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn((idCpu & ~DBGFREG_HYPER_VMCPUID) < pUVM->cCpus || idCpu == VMCPUID_ANY, VERR_INVALID_CPU_ID);
+    AssertPtrReturn(pszFormat, VERR_INVALID_POINTER);
+
+    /*
+     * Set up an argument package and execute the formatting on the
+     * specified CPU.
+     */
+    DBGFR3REGPRINTFARGS Args;
+    Args.pUVM       = pUVM;
+    Args.idCpu      = idCpu != VMCPUID_ANY ? idCpu & ~DBGFREG_HYPER_VMCPUID : idCpu;
+    Args.fGuestRegs = idCpu != VMCPUID_ANY && !(idCpu & DBGFREG_HYPER_VMCPUID);
+    Args.pszBuf     = pszBuf;
+    Args.pszFormat  = pszFormat;
+    va_copy(Args.va, va);
+    Args.offBuf     = 0;
+    Args.cchLeftBuf = cbBuf - 1;
+    Args.rc         = VINF_SUCCESS;
+    int rc = VMR3ReqPriorityCallWaitU(pUVM, Args.idCpu, (PFNRT)dbgfR3RegPrintfWorkerOnCpu, 1, &Args);
+    va_end(Args.va);
+    return rc;
+}
+
+
+/**
+ * Format a registers.
+ *
+ * This is restricted to registers from one CPU, that specified by @a idCpu.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   idCpu               The CPU ID of any CPU registers that may be
+ *                              printed, pass VMCPUID_ANY if not applicable.
+ * @param   pszBuf              The output buffer.
+ * @param   cbBuf               The size of the output buffer.
+ * @param   pszFormat           The format string.  Register names are given by
+ *                              %VR{name}, %VRU{name}, %VRO{name} and
+ *                              %VRB{name}, which are hexadecimal, (unsigned)
+ *                              decimal, octal and binary representation.  None
+ *                              of these types takes any arguments.
+ * @param   ...                 Other format arguments.
+ */
+VMMR3DECL(int) DBGFR3RegPrintf(PUVM pUVM, VMCPUID idCpu, char *pszBuf, size_t cbBuf, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    int rc = DBGFR3RegPrintfV(pUVM, idCpu, pszBuf, cbBuf, pszFormat, va);
+    va_end(va);
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/DBGFStack.cpp b/src/VBox/VMM/VMMR3/DBGFStack.cpp
new file mode 100644
index 00000000..62a4609a
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/DBGFStack.cpp
@@ -0,0 +1,1153 @@
+/* $Id: DBGFStack.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, Call Stack Analyser.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/mm.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/param.h>
+#include <iprt/assert.h>
+#include <iprt/alloca.h>
+#include <iprt/mem.h>
+#include <iprt/string.h>
+#include <iprt/formats/pecoff.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+static DECLCALLBACK(int) dbgfR3StackReadCallback(PRTDBGUNWINDSTATE pThis, RTUINTPTR uSp, size_t cbToRead, void *pvDst);
+
+/**
+ * Unwind context.
+ *
+ * @note Using a constructor and destructor here for simple+safe cleanup.
+ */
+typedef struct DBGFUNWINDCTX
+{
+    PUVM        m_pUVM;
+    VMCPUID     m_idCpu;
+    RTDBGAS     m_hAs;
+    PCCPUMCTX   m_pInitialCtx;
+    bool        m_fIsHostRing0;
+    uint64_t    m_uOsScratch; /**< For passing to DBGFOSREG::pfnStackUnwindAssist. */
+
+    RTDBGMOD    m_hCached;
+    RTUINTPTR   m_uCachedMapping;
+    RTUINTPTR   m_cbCachedMapping;
+    RTDBGSEGIDX m_idxCachedSegMapping;
+
+    RTDBGUNWINDSTATE m_State;
+
+    DBGFUNWINDCTX(PUVM pUVM, VMCPUID idCpu, PCCPUMCTX pInitialCtx, RTDBGAS hAs)
+    {
+        m_State.u32Magic     = RTDBGUNWINDSTATE_MAGIC;
+        m_State.enmArch      = RTLDRARCH_AMD64;
+        m_State.pfnReadStack = dbgfR3StackReadCallback;
+        m_State.pvUser       = this;
+        RT_ZERO(m_State.u);
+        if (pInitialCtx)
+        {
+            m_State.u.x86.auRegs[X86_GREG_xAX] = pInitialCtx->rax;
+            m_State.u.x86.auRegs[X86_GREG_xCX] = pInitialCtx->rcx;
+            m_State.u.x86.auRegs[X86_GREG_xDX] = pInitialCtx->rdx;
+            m_State.u.x86.auRegs[X86_GREG_xBX] = pInitialCtx->rbx;
+            m_State.u.x86.auRegs[X86_GREG_xSP] = pInitialCtx->rsp;
+            m_State.u.x86.auRegs[X86_GREG_xBP] = pInitialCtx->rbp;
+            m_State.u.x86.auRegs[X86_GREG_xSI] = pInitialCtx->rsi;
+            m_State.u.x86.auRegs[X86_GREG_xDI] = pInitialCtx->rdi;
+            m_State.u.x86.auRegs[X86_GREG_x8 ] = pInitialCtx->r8;
+            m_State.u.x86.auRegs[X86_GREG_x9 ] = pInitialCtx->r9;
+            m_State.u.x86.auRegs[X86_GREG_x10] = pInitialCtx->r10;
+            m_State.u.x86.auRegs[X86_GREG_x11] = pInitialCtx->r11;
+            m_State.u.x86.auRegs[X86_GREG_x12] = pInitialCtx->r12;
+            m_State.u.x86.auRegs[X86_GREG_x13] = pInitialCtx->r13;
+            m_State.u.x86.auRegs[X86_GREG_x14] = pInitialCtx->r14;
+            m_State.u.x86.auRegs[X86_GREG_x15] = pInitialCtx->r15;
+            m_State.uPc                        = pInitialCtx->rip;
+            m_State.u.x86.uRFlags              = pInitialCtx->rflags.u;
+            m_State.u.x86.auSegs[X86_SREG_ES]  = pInitialCtx->es.Sel;
+            m_State.u.x86.auSegs[X86_SREG_CS]  = pInitialCtx->cs.Sel;
+            m_State.u.x86.auSegs[X86_SREG_SS]  = pInitialCtx->ss.Sel;
+            m_State.u.x86.auSegs[X86_SREG_DS]  = pInitialCtx->ds.Sel;
+            m_State.u.x86.auSegs[X86_SREG_GS]  = pInitialCtx->gs.Sel;
+            m_State.u.x86.auSegs[X86_SREG_FS]  = pInitialCtx->fs.Sel;
+            m_State.u.x86.fRealOrV86           = CPUMIsGuestInRealOrV86ModeEx(pInitialCtx);
+        }
+        else if (hAs == DBGF_AS_R0)
+            VMMR3InitR0StackUnwindState(pUVM, idCpu, &m_State);
+
+        m_pUVM            = pUVM;
+        m_idCpu           = idCpu;
+        m_hAs             = DBGFR3AsResolveAndRetain(pUVM, hAs);
+        m_pInitialCtx     = pInitialCtx;
+        m_fIsHostRing0    = hAs == DBGF_AS_R0;
+        m_uOsScratch      = 0;
+
+        m_hCached         = NIL_RTDBGMOD;
+        m_uCachedMapping  = 0;
+        m_cbCachedMapping = 0;
+        m_idxCachedSegMapping = NIL_RTDBGSEGIDX;
+    }
+
+    ~DBGFUNWINDCTX();
+
+} DBGFUNWINDCTX;
+/** Pointer to unwind context. */
+typedef DBGFUNWINDCTX *PDBGFUNWINDCTX;
+
+
+static void dbgfR3UnwindCtxFlushCache(PDBGFUNWINDCTX pUnwindCtx)
+{
+    if (pUnwindCtx->m_hCached != NIL_RTDBGMOD)
+    {
+        RTDbgModRelease(pUnwindCtx->m_hCached);
+        pUnwindCtx->m_hCached = NIL_RTDBGMOD;
+    }
+    pUnwindCtx->m_cbCachedMapping     = 0;
+    pUnwindCtx->m_idxCachedSegMapping = NIL_RTDBGSEGIDX;
+}
+
+
+DBGFUNWINDCTX::~DBGFUNWINDCTX()
+{
+    dbgfR3UnwindCtxFlushCache(this);
+    if (m_hAs != NIL_RTDBGAS)
+    {
+        RTDbgAsRelease(m_hAs);
+        m_hAs = NIL_RTDBGAS;
+    }
+}
+
+
+/**
+ * @interface_method_impl{RTDBGUNWINDSTATE,pfnReadStack}
+ */
+static DECLCALLBACK(int) dbgfR3StackReadCallback(PRTDBGUNWINDSTATE pThis, RTUINTPTR uSp, size_t cbToRead, void *pvDst)
+{
+    Assert(   pThis->enmArch == RTLDRARCH_AMD64
+           || pThis->enmArch == RTLDRARCH_X86_32);
+
+    PDBGFUNWINDCTX pUnwindCtx = (PDBGFUNWINDCTX)pThis->pvUser;
+    DBGFADDRESS SrcAddr;
+    int rc = VINF_SUCCESS;
+    if (pUnwindCtx->m_fIsHostRing0)
+        DBGFR3AddrFromHostR0(&SrcAddr, uSp);
+    else
+    {
+        if (   pThis->enmArch == RTLDRARCH_X86_32
+            || pThis->enmArch == RTLDRARCH_X86_16)
+        {
+            if (!pThis->u.x86.fRealOrV86)
+                rc = DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &SrcAddr, pThis->u.x86.auSegs[X86_SREG_SS], uSp);
+            else
+                DBGFR3AddrFromFlat(pUnwindCtx->m_pUVM, &SrcAddr, uSp + ((uint32_t)pThis->u.x86.auSegs[X86_SREG_SS] << 4));
+        }
+        else
+            DBGFR3AddrFromFlat(pUnwindCtx->m_pUVM, &SrcAddr, uSp);
+    }
+    if (RT_SUCCESS(rc))
+        rc = DBGFR3MemRead(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &SrcAddr, pvDst, cbToRead);
+    if (RT_SUCCESS(rc))
+        return rc;
+    return -rc; /* Ignore read errors. */
+}
+
+
+/**
+ * Sets PC and SP.
+ *
+ * @returns true.
+ * @param   pUnwindCtx          The unwind context.
+ * @param   pAddrPC             The program counter (PC) value to set.
+ * @param   pAddrStack          The stack pointer (SP) value to set.
+ */
+static bool dbgfR3UnwindCtxSetPcAndSp(PDBGFUNWINDCTX pUnwindCtx, PCDBGFADDRESS pAddrPC, PCDBGFADDRESS pAddrStack)
+{
+    Assert(   pUnwindCtx->m_State.enmArch == RTLDRARCH_AMD64
+           || pUnwindCtx->m_State.enmArch == RTLDRARCH_X86_32);
+
+    if (!DBGFADDRESS_IS_FAR(pAddrPC))
+        pUnwindCtx->m_State.uPc = pAddrPC->FlatPtr;
+    else
+    {
+        pUnwindCtx->m_State.uPc                       = pAddrPC->off;
+        pUnwindCtx->m_State.u.x86.auSegs[X86_SREG_CS] = pAddrPC->Sel;
+    }
+    if (!DBGFADDRESS_IS_FAR(pAddrStack))
+        pUnwindCtx->m_State.u.x86.auRegs[X86_GREG_xSP] = pAddrStack->FlatPtr;
+    else
+    {
+        pUnwindCtx->m_State.u.x86.auRegs[X86_GREG_xSP] = pAddrStack->off;
+        pUnwindCtx->m_State.u.x86.auSegs[X86_SREG_SS]  = pAddrStack->Sel;
+    }
+    return true;
+}
+
+
+/**
+ * Tries to unwind one frame using unwind info.
+ *
+ * @returns true on success, false on failure.
+ * @param   pUnwindCtx      The unwind context.
+ */
+static bool dbgfR3UnwindCtxDoOneFrame(PDBGFUNWINDCTX pUnwindCtx)
+{
+    /*
+     * Need to load it into the cache?
+     */
+    RTUINTPTR offCache = pUnwindCtx->m_State.uPc - pUnwindCtx->m_uCachedMapping;
+    if (offCache >= pUnwindCtx->m_cbCachedMapping)
+    {
+        RTDBGMOD        hDbgMod = NIL_RTDBGMOD;
+        RTUINTPTR       uBase   = 0;
+        RTDBGSEGIDX     idxSeg  = NIL_RTDBGSEGIDX;
+        int rc = RTDbgAsModuleByAddr(pUnwindCtx->m_hAs, pUnwindCtx->m_State.uPc, &hDbgMod, &uBase, &idxSeg);
+        if (RT_SUCCESS(rc))
+        {
+            dbgfR3UnwindCtxFlushCache(pUnwindCtx);
+            pUnwindCtx->m_hCached             = hDbgMod;
+            pUnwindCtx->m_uCachedMapping      = uBase;
+            pUnwindCtx->m_idxCachedSegMapping = idxSeg;
+            pUnwindCtx->m_cbCachedMapping     = idxSeg == NIL_RTDBGSEGIDX ? RTDbgModImageSize(hDbgMod)
+                                              : RTDbgModSegmentSize(hDbgMod, idxSeg);
+            offCache = pUnwindCtx->m_State.uPc - uBase;
+        }
+        else
+            return false;
+    }
+
+    /*
+     * Do the lookup.
+     */
+    AssertCompile(UINT32_MAX == NIL_RTDBGSEGIDX);
+    int rc = RTDbgModUnwindFrame(pUnwindCtx->m_hCached, pUnwindCtx->m_idxCachedSegMapping, offCache, &pUnwindCtx->m_State);
+    if (RT_SUCCESS(rc))
+        return true;
+    return false;
+}
+
+
+/**
+ * Read stack memory, will init entire buffer.
+ */
+DECLINLINE(int) dbgfR3StackRead(PUVM pUVM, VMCPUID idCpu, void *pvBuf, PCDBGFADDRESS pSrcAddr, size_t cb, size_t *pcbRead)
+{
+    int rc = DBGFR3MemRead(pUVM, idCpu, pSrcAddr, pvBuf, cb);
+    if (RT_FAILURE(rc))
+    {
+        /* fallback: byte by byte and zero the ones we fail to read. */
+        size_t cbRead;
+        for (cbRead = 0; cbRead < cb; cbRead++)
+        {
+            DBGFADDRESS Addr = *pSrcAddr;
+            rc = DBGFR3MemRead(pUVM, idCpu, DBGFR3AddrAdd(&Addr, cbRead), (uint8_t *)pvBuf + cbRead, 1);
+            if (RT_FAILURE(rc))
+                break;
+        }
+        if (cbRead)
+            rc = VINF_SUCCESS;
+        memset((char *)pvBuf + cbRead, 0, cb - cbRead);
+        *pcbRead = cbRead;
+    }
+    else
+        *pcbRead = cb;
+    return rc;
+}
+
+/**
+ * Collects sure registers on frame exit.
+ *
+ * @returns VINF_SUCCESS or VERR_NO_MEMORY.
+ * @param   pUVM        The user mode VM handle for the allocation.
+ * @param   pFrame      The frame in question.
+ * @param   pState      The unwind state.
+ */
+static int dbgfR3StackWalkCollectRegisterChanges(PUVM pUVM, PDBGFSTACKFRAME pFrame, PRTDBGUNWINDSTATE pState)
+{
+    pFrame->cSureRegs  = 0;
+    pFrame->paSureRegs = NULL;
+
+    if (   pState->enmArch == RTLDRARCH_AMD64
+        || pState->enmArch == RTLDRARCH_X86_32
+        || pState->enmArch == RTLDRARCH_X86_16)
+    {
+        if (pState->u.x86.Loaded.fAll)
+        {
+            /*
+             * Count relevant registers.
+             */
+            uint32_t cRegs = 0;
+            if (pState->u.x86.Loaded.s.fRegs)
+                for (uint32_t f = 1; f < RT_BIT_32(RT_ELEMENTS(pState->u.x86.auRegs)); f <<= 1)
+                    if (pState->u.x86.Loaded.s.fRegs & f)
+                        cRegs++;
+            if (pState->u.x86.Loaded.s.fSegs)
+                for (uint32_t f = 1; f < RT_BIT_32(RT_ELEMENTS(pState->u.x86.auSegs)); f <<= 1)
+                    if (pState->u.x86.Loaded.s.fSegs & f)
+                        cRegs++;
+            if (pState->u.x86.Loaded.s.fRFlags)
+                cRegs++;
+            if (pState->u.x86.Loaded.s.fErrCd)
+                cRegs++;
+            if (cRegs > 0)
+            {
+                /*
+                 * Allocate the arrays.
+                 */
+                PDBGFREGVALEX paSureRegs = (PDBGFREGVALEX)MMR3HeapAllocZU(pUVM, MM_TAG_DBGF_STACK, sizeof(DBGFREGVALEX) * cRegs);
+                AssertReturn(paSureRegs, VERR_NO_MEMORY);
+                pFrame->paSureRegs = paSureRegs;
+                pFrame->cSureRegs  = cRegs;
+
+                /*
+                 * Popuplate the arrays.
+                 */
+                uint32_t iReg = 0;
+                if (pState->u.x86.Loaded.s.fRegs)
+                    for (uint32_t i = 0; i < RT_ELEMENTS(pState->u.x86.auRegs); i++)
+                        if (pState->u.x86.Loaded.s.fRegs & RT_BIT(i))
+                        {
+                            paSureRegs[iReg].Value.u64 = pState->u.x86.auRegs[i];
+                            paSureRegs[iReg].enmType   = DBGFREGVALTYPE_U64;
+                            paSureRegs[iReg].enmReg    = (DBGFREG)(DBGFREG_RAX + i);
+                            iReg++;
+                        }
+
+                if (pState->u.x86.Loaded.s.fSegs)
+                    for (uint32_t i = 0; i < RT_ELEMENTS(pState->u.x86.auSegs); i++)
+                        if (pState->u.x86.Loaded.s.fSegs & RT_BIT(i))
+                        {
+                            paSureRegs[iReg].Value.u16 = pState->u.x86.auSegs[i];
+                            paSureRegs[iReg].enmType   = DBGFREGVALTYPE_U16;
+                            switch (i)
+                            {
+                                case X86_SREG_ES: paSureRegs[iReg].enmReg = DBGFREG_ES; break;
+                                case X86_SREG_CS: paSureRegs[iReg].enmReg = DBGFREG_CS; break;
+                                case X86_SREG_SS: paSureRegs[iReg].enmReg = DBGFREG_SS; break;
+                                case X86_SREG_DS: paSureRegs[iReg].enmReg = DBGFREG_DS; break;
+                                case X86_SREG_FS: paSureRegs[iReg].enmReg = DBGFREG_FS; break;
+                                case X86_SREG_GS: paSureRegs[iReg].enmReg = DBGFREG_GS; break;
+                                default:          AssertFailedBreak();
+                            }
+                            iReg++;
+                        }
+
+                if (iReg < cRegs)
+                {
+                    if (pState->u.x86.Loaded.s.fRFlags)
+                    {
+                        paSureRegs[iReg].Value.u64 = pState->u.x86.uRFlags;
+                        paSureRegs[iReg].enmType   = DBGFREGVALTYPE_U64;
+                        paSureRegs[iReg].enmReg    = DBGFREG_RFLAGS;
+                        iReg++;
+                    }
+                    if (pState->u.x86.Loaded.s.fErrCd)
+                    {
+                        paSureRegs[iReg].Value.u64 = pState->u.x86.uErrCd;
+                        paSureRegs[iReg].enmType   = DBGFREGVALTYPE_U64;
+                        paSureRegs[iReg].enmReg    = DBGFREG_END;
+                        paSureRegs[iReg].pszName   = "trap-errcd";
+                        iReg++;
+                    }
+                }
+                Assert(iReg == cRegs);
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal worker routine.
+ *
+ * On x86 the typical stack frame layout is like this:
+ *     ..  ..
+ *     16  parameter 2
+ *     12  parameter 1
+ *      8  parameter 0
+ *      4  return address
+ *      0  old ebp; current ebp points here
+ */
+DECL_NO_INLINE(static, int) dbgfR3StackWalk(PDBGFUNWINDCTX pUnwindCtx, PDBGFSTACKFRAME pFrame, bool fFirst)
+{
+    /*
+     * Stop if we got a read error in the previous run.
+     */
+    if (pFrame->fFlags & DBGFSTACKFRAME_FLAGS_LAST)
+        return VERR_NO_MORE_FILES;
+
+    /*
+     * Advance the frame (except for the first).
+     */
+    if (!fFirst) /** @todo we can probably eliminate this fFirst business... */
+    {
+        /* frame, pc and stack is taken from the existing frames return members. */
+        pFrame->AddrFrame = pFrame->AddrReturnFrame;
+        pFrame->AddrPC    = pFrame->AddrReturnPC;
+        pFrame->pSymPC    = pFrame->pSymReturnPC;
+        pFrame->pLinePC   = pFrame->pLineReturnPC;
+
+        /* increment the frame number. */
+        pFrame->iFrame++;
+
+        /* UNWIND_INFO_RET -> USED_UNWIND; return type */
+        if (!(pFrame->fFlags & DBGFSTACKFRAME_FLAGS_UNWIND_INFO_RET))
+            pFrame->fFlags &= ~DBGFSTACKFRAME_FLAGS_USED_UNWIND_INFO;
+        else
+        {
+            pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_USED_UNWIND_INFO;
+            pFrame->fFlags &= ~DBGFSTACKFRAME_FLAGS_UNWIND_INFO_RET;
+            if (pFrame->enmReturnFrameReturnType != RTDBGRETURNTYPE_INVALID)
+            {
+                pFrame->enmReturnType = pFrame->enmReturnFrameReturnType;
+                pFrame->enmReturnFrameReturnType = RTDBGRETURNTYPE_INVALID;
+            }
+        }
+        pFrame->fFlags &= ~DBGFSTACKFRAME_FLAGS_TRAP_FRAME;
+    }
+
+    /*
+     * Figure the return address size and use the old PC to guess stack item size.
+     */
+    /** @todo this is bogus... */
+    unsigned cbRetAddr = RTDbgReturnTypeSize(pFrame->enmReturnType);
+    unsigned cbStackItem;
+    switch (pFrame->AddrPC.fFlags & DBGFADDRESS_FLAGS_TYPE_MASK)
+    {
+        case DBGFADDRESS_FLAGS_FAR16: cbStackItem = 2; break;
+        case DBGFADDRESS_FLAGS_FAR32: cbStackItem = 4; break;
+        case DBGFADDRESS_FLAGS_FAR64: cbStackItem = 8; break;
+        case DBGFADDRESS_FLAGS_RING0: cbStackItem = sizeof(RTHCUINTPTR); break;
+        default:
+            switch (pFrame->enmReturnType)
+            {
+                case RTDBGRETURNTYPE_FAR16:
+                case RTDBGRETURNTYPE_IRET16:
+                case RTDBGRETURNTYPE_IRET32_V86:
+                case RTDBGRETURNTYPE_NEAR16: cbStackItem = 2; break;
+
+                case RTDBGRETURNTYPE_FAR32:
+                case RTDBGRETURNTYPE_IRET32:
+                case RTDBGRETURNTYPE_IRET32_PRIV:
+                case RTDBGRETURNTYPE_NEAR32: cbStackItem = 4; break;
+
+                case RTDBGRETURNTYPE_FAR64:
+                case RTDBGRETURNTYPE_IRET64:
+                case RTDBGRETURNTYPE_NEAR64: cbStackItem = 8; break;
+
+                default:
+                    AssertMsgFailed(("%d\n", pFrame->enmReturnType));
+                    cbStackItem = 4;
+                    break;
+            }
+    }
+
+    /*
+     * Read the raw frame data.
+     * We double cbRetAddr in case we have a far return.
+     */
+    union
+    {
+        uint64_t *pu64;
+        uint32_t *pu32;
+        uint16_t *pu16;
+        uint8_t  *pb;
+        void     *pv;
+    } u, uRet, uArgs, uBp;
+    size_t cbRead = cbRetAddr*2 + cbStackItem + sizeof(pFrame->Args);
+    u.pv = alloca(cbRead);
+    uBp = u;
+    uRet.pb = u.pb + cbStackItem;
+    uArgs.pb = u.pb + cbStackItem + cbRetAddr;
+
+    Assert(DBGFADDRESS_IS_VALID(&pFrame->AddrFrame));
+    int rc = dbgfR3StackRead(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, u.pv, &pFrame->AddrFrame, cbRead, &cbRead);
+    if (   RT_FAILURE(rc)
+        || cbRead < cbRetAddr + cbStackItem)
+        pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_LAST;
+
+    /*
+     * Return Frame address.
+     *
+     * If we used unwind info to get here, the unwind register context will be
+     * positioned after the return instruction has been executed.  We start by
+     * picking up the rBP register here for return frame and will try improve
+     * on it further down by using unwind info.
+     */
+    pFrame->AddrReturnFrame = pFrame->AddrFrame;
+    if (pFrame->fFlags & DBGFSTACKFRAME_FLAGS_USED_UNWIND_INFO)
+    {
+        if (   pFrame->enmReturnType == RTDBGRETURNTYPE_IRET32_PRIV
+            || pFrame->enmReturnType == RTDBGRETURNTYPE_IRET64)
+            DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnFrame,
+                                 pUnwindCtx->m_State.u.x86.auSegs[X86_SREG_SS], pUnwindCtx->m_State.u.x86.auRegs[X86_GREG_xBP]);
+        else if (pFrame->enmReturnType == RTDBGRETURNTYPE_IRET32_V86)
+            DBGFR3AddrFromFlat(pUnwindCtx->m_pUVM, &pFrame->AddrReturnFrame,
+                                 ((uint32_t)pUnwindCtx->m_State.u.x86.auSegs[X86_SREG_SS] << 4)
+                               + pUnwindCtx->m_State.u.x86.auRegs[X86_GREG_xBP]);
+        else
+        {
+            pFrame->AddrReturnFrame.off      = pUnwindCtx->m_State.u.x86.auRegs[X86_GREG_xBP];
+            pFrame->AddrReturnFrame.FlatPtr += pFrame->AddrReturnFrame.off - pFrame->AddrFrame.off;
+        }
+    }
+    else
+    {
+        switch (cbStackItem)
+        {
+            case 2:     pFrame->AddrReturnFrame.off = *uBp.pu16; break;
+            case 4:     pFrame->AddrReturnFrame.off = *uBp.pu32; break;
+            case 8:     pFrame->AddrReturnFrame.off = *uBp.pu64; break;
+            default:    AssertMsgFailedReturn(("cbStackItem=%d\n", cbStackItem), VERR_DBGF_STACK_IPE_1);
+        }
+
+        /* Watcom tries to keep the frame pointer odd for far returns. */
+        if (   cbStackItem <= 4
+            && !(pFrame->fFlags & DBGFSTACKFRAME_FLAGS_USED_UNWIND_INFO))
+        {
+            if (pFrame->AddrReturnFrame.off & 1)
+            {
+                pFrame->AddrReturnFrame.off &= ~(RTGCUINTPTR)1;
+                if (pFrame->enmReturnType == RTDBGRETURNTYPE_NEAR16)
+                {
+                    pFrame->fFlags       |= DBGFSTACKFRAME_FLAGS_USED_ODD_EVEN;
+                    pFrame->enmReturnType = RTDBGRETURNTYPE_FAR16;
+                    cbRetAddr = 4;
+                }
+                else if (pFrame->enmReturnType == RTDBGRETURNTYPE_NEAR32)
+                {
+#if 1
+                    /* Assumes returning 32-bit code. */
+                    pFrame->fFlags       |= DBGFSTACKFRAME_FLAGS_USED_ODD_EVEN;
+                    pFrame->enmReturnType = RTDBGRETURNTYPE_FAR32;
+                    cbRetAddr = 8;
+#else
+                    /* Assumes returning 16-bit code. */
+                    pFrame->fFlags       |= DBGFSTACKFRAME_FLAGS_USED_ODD_EVEN;
+                    pFrame->enmReturnType = RTDBGRETURNTYPE_FAR16;
+                    cbRetAddr = 4;
+#endif
+                }
+            }
+            else if (pFrame->fFlags & DBGFSTACKFRAME_FLAGS_USED_ODD_EVEN)
+            {
+                if (pFrame->enmReturnType == RTDBGRETURNTYPE_FAR16)
+                {
+                    pFrame->enmReturnType = RTDBGRETURNTYPE_NEAR16;
+                    cbRetAddr = 2;
+                }
+                else if (pFrame->enmReturnType == RTDBGRETURNTYPE_NEAR32)
+                {
+                    pFrame->enmReturnType = RTDBGRETURNTYPE_FAR32;
+                    cbRetAddr = 4;
+                }
+                pFrame->fFlags &= ~DBGFSTACKFRAME_FLAGS_USED_ODD_EVEN;
+            }
+            uArgs.pb = u.pb + cbStackItem + cbRetAddr;
+        }
+
+        pFrame->AddrReturnFrame.FlatPtr += pFrame->AddrReturnFrame.off - pFrame->AddrFrame.off;
+    }
+
+    /*
+     * Return Stack Address.
+     */
+    pFrame->AddrReturnStack = pFrame->AddrReturnFrame;
+    if (pFrame->fFlags & DBGFSTACKFRAME_FLAGS_USED_UNWIND_INFO)
+    {
+        if (   pFrame->enmReturnType == RTDBGRETURNTYPE_IRET32_PRIV
+            || pFrame->enmReturnType == RTDBGRETURNTYPE_IRET64)
+            DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnStack,
+                                 pUnwindCtx->m_State.u.x86.auSegs[X86_SREG_SS], pUnwindCtx->m_State.u.x86.auRegs[X86_GREG_xSP]);
+        else if (pFrame->enmReturnType == RTDBGRETURNTYPE_IRET32_V86)
+            DBGFR3AddrFromFlat(pUnwindCtx->m_pUVM, &pFrame->AddrReturnStack,
+                                 ((uint32_t)pUnwindCtx->m_State.u.x86.auSegs[X86_SREG_SS] << 4)
+                               + pUnwindCtx->m_State.u.x86.auRegs[X86_GREG_xSP]);
+        else
+        {
+            pFrame->AddrReturnStack.off      = pUnwindCtx->m_State.u.x86.auRegs[X86_GREG_xSP];
+            pFrame->AddrReturnStack.FlatPtr += pFrame->AddrReturnStack.off - pFrame->AddrStack.off;
+        }
+    }
+    else
+    {
+        pFrame->AddrReturnStack.off     += cbStackItem + cbRetAddr;
+        pFrame->AddrReturnStack.FlatPtr += cbStackItem + cbRetAddr;
+    }
+
+    /*
+     * Return PC.
+     */
+    pFrame->AddrReturnPC = pFrame->AddrPC;
+    if (pFrame->fFlags & DBGFSTACKFRAME_FLAGS_USED_UNWIND_INFO)
+    {
+        if (RTDbgReturnTypeIsNear(pFrame->enmReturnType))
+        {
+            pFrame->AddrReturnPC.off      = pUnwindCtx->m_State.uPc;
+            pFrame->AddrReturnPC.FlatPtr += pFrame->AddrReturnPC.off - pFrame->AddrPC.off;
+        }
+        else
+            DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC,
+                                 pUnwindCtx->m_State.u.x86.auSegs[X86_SREG_CS], pUnwindCtx->m_State.uPc);
+    }
+    else
+    {
+        int rc2;
+        switch (pFrame->enmReturnType)
+        {
+            case RTDBGRETURNTYPE_NEAR16:
+                if (DBGFADDRESS_IS_VALID(&pFrame->AddrReturnPC))
+                {
+                    pFrame->AddrReturnPC.FlatPtr += *uRet.pu16 - pFrame->AddrReturnPC.off;
+                    pFrame->AddrReturnPC.off      = *uRet.pu16;
+                }
+                else
+                    DBGFR3AddrFromFlat(pUnwindCtx->m_pUVM, &pFrame->AddrReturnPC, *uRet.pu16);
+                break;
+            case RTDBGRETURNTYPE_NEAR32:
+                if (DBGFADDRESS_IS_VALID(&pFrame->AddrReturnPC))
+                {
+                    pFrame->AddrReturnPC.FlatPtr += *uRet.pu32 - pFrame->AddrReturnPC.off;
+                    pFrame->AddrReturnPC.off      = *uRet.pu32;
+                }
+                else
+                    DBGFR3AddrFromFlat(pUnwindCtx->m_pUVM, &pFrame->AddrReturnPC, *uRet.pu32);
+                break;
+            case RTDBGRETURNTYPE_NEAR64:
+                if (DBGFADDRESS_IS_VALID(&pFrame->AddrReturnPC))
+                {
+                    pFrame->AddrReturnPC.FlatPtr += *uRet.pu64 - pFrame->AddrReturnPC.off;
+                    pFrame->AddrReturnPC.off      = *uRet.pu64;
+                }
+                else
+                    DBGFR3AddrFromFlat(pUnwindCtx->m_pUVM, &pFrame->AddrReturnPC, *uRet.pu64);
+                break;
+            case RTDBGRETURNTYPE_FAR16:
+                rc2 = DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC, uRet.pu16[1], uRet.pu16[0]);
+                if (RT_SUCCESS(rc2))
+                    break;
+                rc2 = DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC, pFrame->AddrPC.Sel, uRet.pu16[0]);
+                if (RT_SUCCESS(rc2))
+                    pFrame->enmReturnType = RTDBGRETURNTYPE_NEAR16;
+                else
+                    DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC, uRet.pu16[1], uRet.pu16[0]);
+                break;
+            case RTDBGRETURNTYPE_FAR32:
+                rc2 = DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
+                if (RT_SUCCESS(rc2))
+                    break;
+                rc2 = DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC, pFrame->AddrPC.Sel, uRet.pu32[0]);
+                if (RT_SUCCESS(rc2))
+                    pFrame->enmReturnType = RTDBGRETURNTYPE_NEAR32;
+                else
+                    DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
+                break;
+            case RTDBGRETURNTYPE_FAR64:
+                DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC, uRet.pu16[4], uRet.pu64[0]);
+                break;
+            case RTDBGRETURNTYPE_IRET16:
+                DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC, uRet.pu16[1], uRet.pu16[0]);
+                break;
+            case RTDBGRETURNTYPE_IRET32:
+                DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
+                break;
+            case RTDBGRETURNTYPE_IRET32_PRIV:
+                DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
+                break;
+            case RTDBGRETURNTYPE_IRET32_V86:
+                DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
+                break;
+            case RTDBGRETURNTYPE_IRET64:
+                DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &pFrame->AddrReturnPC, uRet.pu16[4], uRet.pu64[0]);
+                break;
+            default:
+                AssertMsgFailed(("enmReturnType=%d\n", pFrame->enmReturnType));
+                return VERR_INVALID_PARAMETER;
+        }
+    }
+
+
+    pFrame->pSymReturnPC  = DBGFR3AsSymbolByAddrA(pUnwindCtx->m_pUVM, pUnwindCtx->m_hAs, &pFrame->AddrReturnPC,
+                                                  RTDBGSYMADDR_FLAGS_LESS_OR_EQUAL | RTDBGSYMADDR_FLAGS_SKIP_ABS_IN_DEFERRED,
+                                                  NULL /*poffDisp*/, NULL /*phMod*/);
+    pFrame->pLineReturnPC = DBGFR3AsLineByAddrA(pUnwindCtx->m_pUVM, pUnwindCtx->m_hAs, &pFrame->AddrReturnPC,
+                                                NULL /*poffDisp*/, NULL /*phMod*/);
+
+    /*
+     * Frame bitness flag.
+     */
+    /** @todo use previous return type for this? */
+    pFrame->fFlags &= ~(DBGFSTACKFRAME_FLAGS_16BIT | DBGFSTACKFRAME_FLAGS_32BIT | DBGFSTACKFRAME_FLAGS_64BIT);
+    switch (cbStackItem)
+    {
+        case 2: pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_16BIT; break;
+        case 4: pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_32BIT; break;
+        case 8: pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_64BIT; break;
+        default:    AssertMsgFailedReturn(("cbStackItem=%d\n", cbStackItem), VERR_DBGF_STACK_IPE_2);
+    }
+
+    /*
+     * The arguments.
+     */
+    memcpy(&pFrame->Args, uArgs.pv, sizeof(pFrame->Args));
+
+    /*
+     * Collect register changes.
+     * Then call the OS layer to assist us (e.g. NT trap frames).
+     */
+    if (pFrame->fFlags & DBGFSTACKFRAME_FLAGS_USED_UNWIND_INFO)
+    {
+        rc = dbgfR3StackWalkCollectRegisterChanges(pUnwindCtx->m_pUVM, pFrame, &pUnwindCtx->m_State);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        if (   pUnwindCtx->m_pInitialCtx
+            && pUnwindCtx->m_hAs != NIL_RTDBGAS)
+        {
+            rc = dbgfR3OSStackUnwindAssist(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, pFrame, &pUnwindCtx->m_State,
+                                           pUnwindCtx->m_pInitialCtx, pUnwindCtx->m_hAs, &pUnwindCtx->m_uOsScratch);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+    }
+
+    /*
+     * Try use unwind information to locate the return frame pointer (for the
+     * next loop iteration).
+     */
+    Assert(!(pFrame->fFlags & DBGFSTACKFRAME_FLAGS_UNWIND_INFO_RET));
+    pFrame->enmReturnFrameReturnType = RTDBGRETURNTYPE_INVALID;
+    if (!(pFrame->fFlags & DBGFSTACKFRAME_FLAGS_LAST))
+    {
+        /* Set PC and SP if we didn't unwind our way here (context will then point
+           and the return PC and SP already). */
+        if (!(pFrame->fFlags & DBGFSTACKFRAME_FLAGS_USED_UNWIND_INFO))
+        {
+            dbgfR3UnwindCtxSetPcAndSp(pUnwindCtx, &pFrame->AddrReturnPC, &pFrame->AddrReturnStack);
+            pUnwindCtx->m_State.u.x86.auRegs[X86_GREG_xBP] = pFrame->AddrReturnFrame.off;
+        }
+        /** @todo Reevaluate CS if the previous frame return type isn't near. */
+        if (   pUnwindCtx->m_State.enmArch == RTLDRARCH_AMD64
+            || pUnwindCtx->m_State.enmArch == RTLDRARCH_X86_32
+            || pUnwindCtx->m_State.enmArch == RTLDRARCH_X86_16)
+            pUnwindCtx->m_State.u.x86.Loaded.fAll = 0;
+        else
+            AssertFailed();
+        if (dbgfR3UnwindCtxDoOneFrame(pUnwindCtx))
+        {
+            if (pUnwindCtx->m_fIsHostRing0)
+                DBGFR3AddrFromHostR0(&pFrame->AddrReturnFrame, pUnwindCtx->m_State.u.x86.FrameAddr.off);
+            else
+            {
+                DBGFADDRESS AddrReturnFrame = pFrame->AddrReturnFrame;
+                rc = DBGFR3AddrFromSelOff(pUnwindCtx->m_pUVM, pUnwindCtx->m_idCpu, &AddrReturnFrame,
+                                          pUnwindCtx->m_State.u.x86.FrameAddr.sel, pUnwindCtx->m_State.u.x86.FrameAddr.off);
+                if (RT_SUCCESS(rc))
+                    pFrame->AddrReturnFrame = AddrReturnFrame;
+            }
+            pFrame->enmReturnFrameReturnType = pUnwindCtx->m_State.enmRetType;
+            pFrame->fFlags                  |= DBGFSTACKFRAME_FLAGS_UNWIND_INFO_RET;
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Walks the entire stack allocating memory as we walk.
+ */
+static DECLCALLBACK(int) dbgfR3StackWalkCtxFull(PUVM pUVM, VMCPUID idCpu, PCCPUMCTX pCtx, RTDBGAS hAs,
+                                                DBGFCODETYPE enmCodeType,
+                                                PCDBGFADDRESS pAddrFrame,
+                                                PCDBGFADDRESS pAddrStack,
+                                                PCDBGFADDRESS pAddrPC,
+                                                RTDBGRETURNTYPE enmReturnType,
+                                                PCDBGFSTACKFRAME *ppFirstFrame)
+{
+    DBGFUNWINDCTX UnwindCtx(pUVM, idCpu, pCtx, hAs);
+
+    /* alloc first frame. */
+    PDBGFSTACKFRAME pCur = (PDBGFSTACKFRAME)MMR3HeapAllocZU(pUVM, MM_TAG_DBGF_STACK, sizeof(*pCur));
+    if (!pCur)
+        return VERR_NO_MEMORY;
+
+    /*
+     * Initialize the frame.
+     */
+    pCur->pNextInternal = NULL;
+    pCur->pFirstInternal = pCur;
+
+    int rc = VINF_SUCCESS;
+    if (pAddrPC)
+        pCur->AddrPC = *pAddrPC;
+    else if (enmCodeType != DBGFCODETYPE_GUEST)
+        DBGFR3AddrFromFlat(pUVM, &pCur->AddrPC, pCtx->rip);
+    else
+        rc = DBGFR3AddrFromSelOff(pUVM, idCpu, &pCur->AddrPC, pCtx->cs.Sel, pCtx->rip);
+    if (RT_SUCCESS(rc))
+    {
+        uint64_t fAddrMask;
+        if (enmCodeType == DBGFCODETYPE_RING0)
+            fAddrMask = HC_ARCH_BITS == 64 ? UINT64_MAX : UINT32_MAX;
+        else if (enmCodeType == DBGFCODETYPE_HYPER)
+            fAddrMask = UINT32_MAX;
+        else if (DBGFADDRESS_IS_FAR16(&pCur->AddrPC))
+            fAddrMask = UINT16_MAX;
+        else if (DBGFADDRESS_IS_FAR32(&pCur->AddrPC))
+            fAddrMask = UINT32_MAX;
+        else if (DBGFADDRESS_IS_FAR64(&pCur->AddrPC))
+            fAddrMask = UINT64_MAX;
+        else
+        {
+            PVMCPU pVCpu = VMMGetCpuById(pUVM->pVM, idCpu);
+            CPUMMODE enmCpuMode = CPUMGetGuestMode(pVCpu);
+            if (enmCpuMode == CPUMMODE_REAL)
+            {
+                fAddrMask = UINT16_MAX;
+                if (enmReturnType == RTDBGRETURNTYPE_INVALID)
+                    pCur->enmReturnType = RTDBGRETURNTYPE_NEAR16;
+            }
+            else if (   enmCpuMode == CPUMMODE_PROTECTED
+                     || !CPUMIsGuestIn64BitCode(pVCpu))
+            {
+                fAddrMask = UINT32_MAX;
+                if (enmReturnType == RTDBGRETURNTYPE_INVALID)
+                    pCur->enmReturnType = RTDBGRETURNTYPE_NEAR32;
+            }
+            else
+            {
+                fAddrMask = UINT64_MAX;
+                if (enmReturnType == RTDBGRETURNTYPE_INVALID)
+                    pCur->enmReturnType = RTDBGRETURNTYPE_NEAR64;
+            }
+        }
+
+        if (enmReturnType == RTDBGRETURNTYPE_INVALID)
+            switch (pCur->AddrPC.fFlags & DBGFADDRESS_FLAGS_TYPE_MASK)
+            {
+                case DBGFADDRESS_FLAGS_FAR16: pCur->enmReturnType = RTDBGRETURNTYPE_NEAR16; break;
+                case DBGFADDRESS_FLAGS_FAR32: pCur->enmReturnType = RTDBGRETURNTYPE_NEAR32; break;
+                case DBGFADDRESS_FLAGS_FAR64: pCur->enmReturnType = RTDBGRETURNTYPE_NEAR64; break;
+                case DBGFADDRESS_FLAGS_RING0:
+                    pCur->enmReturnType = HC_ARCH_BITS == 64 ? RTDBGRETURNTYPE_NEAR64 : RTDBGRETURNTYPE_NEAR32;
+                    break;
+                default:
+                    pCur->enmReturnType = RTDBGRETURNTYPE_NEAR32;
+                    break;
+            }
+
+
+        if (pAddrStack)
+            pCur->AddrStack = *pAddrStack;
+        else if (enmCodeType != DBGFCODETYPE_GUEST)
+            DBGFR3AddrFromFlat(pUVM, &pCur->AddrStack, pCtx->rsp & fAddrMask);
+        else
+            rc = DBGFR3AddrFromSelOff(pUVM, idCpu, &pCur->AddrStack, pCtx->ss.Sel, pCtx->rsp & fAddrMask);
+
+        Assert(!(pCur->fFlags & DBGFSTACKFRAME_FLAGS_USED_UNWIND_INFO));
+        if (pAddrFrame)
+            pCur->AddrFrame = *pAddrFrame;
+        else if (enmCodeType != DBGFCODETYPE_GUEST)
+            DBGFR3AddrFromFlat(pUVM, &pCur->AddrFrame, pCtx->rbp & fAddrMask);
+        else if (RT_SUCCESS(rc))
+            rc = DBGFR3AddrFromSelOff(pUVM, idCpu, &pCur->AddrFrame, pCtx->ss.Sel, pCtx->rbp & fAddrMask);
+
+        /*
+         * Try unwind and get a better frame pointer and state.
+         */
+        if (   RT_SUCCESS(rc)
+            && dbgfR3UnwindCtxSetPcAndSp(&UnwindCtx, &pCur->AddrPC, &pCur->AddrStack)
+            && dbgfR3UnwindCtxDoOneFrame(&UnwindCtx))
+        {
+            pCur->enmReturnType = UnwindCtx.m_State.enmRetType;
+            pCur->fFlags |= DBGFSTACKFRAME_FLAGS_USED_UNWIND_INFO;
+            if (!UnwindCtx.m_fIsHostRing0)
+                rc = DBGFR3AddrFromSelOff(UnwindCtx.m_pUVM, UnwindCtx.m_idCpu, &pCur->AddrFrame,
+                                          UnwindCtx.m_State.u.x86.FrameAddr.sel, UnwindCtx.m_State.u.x86.FrameAddr.off);
+            else
+                DBGFR3AddrFromHostR0(&pCur->AddrFrame, UnwindCtx.m_State.u.x86.FrameAddr.off);
+        }
+        /*
+         * The first frame.
+         */
+        if (RT_SUCCESS(rc))
+        {
+            if (DBGFADDRESS_IS_VALID(&pCur->AddrPC))
+            {
+                pCur->pSymPC  = DBGFR3AsSymbolByAddrA(pUVM, hAs, &pCur->AddrPC,
+                                                      RTDBGSYMADDR_FLAGS_LESS_OR_EQUAL | RTDBGSYMADDR_FLAGS_SKIP_ABS_IN_DEFERRED,
+                                                      NULL /*poffDisp*/, NULL /*phMod*/);
+                pCur->pLinePC = DBGFR3AsLineByAddrA(pUVM, hAs, &pCur->AddrPC, NULL /*poffDisp*/, NULL /*phMod*/);
+            }
+
+            rc = dbgfR3StackWalk(&UnwindCtx, pCur, true /*fFirst*/);
+        }
+    }
+    else
+        pCur->enmReturnType = enmReturnType;
+    if (RT_FAILURE(rc))
+    {
+        DBGFR3StackWalkEnd(pCur);
+        return rc;
+    }
+
+    /*
+     * The other frames.
+     */
+    DBGFSTACKFRAME Next = *pCur;
+    while (!(pCur->fFlags & (DBGFSTACKFRAME_FLAGS_LAST | DBGFSTACKFRAME_FLAGS_MAX_DEPTH | DBGFSTACKFRAME_FLAGS_LOOP)))
+    {
+        Next.cSureRegs  = 0;
+        Next.paSureRegs = NULL;
+
+        /* try walk. */
+        rc = dbgfR3StackWalk(&UnwindCtx, &Next, false /*fFirst*/);
+        if (RT_FAILURE(rc))
+            break;
+
+        /* add the next frame to the chain. */
+        PDBGFSTACKFRAME pNext = (PDBGFSTACKFRAME)MMR3HeapAllocU(pUVM, MM_TAG_DBGF_STACK, sizeof(*pNext));
+        if (!pNext)
+        {
+            DBGFR3StackWalkEnd(pCur);
+            return VERR_NO_MEMORY;
+        }
+        *pNext = Next;
+        pCur->pNextInternal = pNext;
+        pCur = pNext;
+        Assert(pCur->pNextInternal == NULL);
+
+        /* check for loop */
+        for (PCDBGFSTACKFRAME pLoop = pCur->pFirstInternal;
+             pLoop && pLoop != pCur;
+             pLoop = pLoop->pNextInternal)
+            if (pLoop->AddrFrame.FlatPtr == pCur->AddrFrame.FlatPtr)
+            {
+                pCur->fFlags |= DBGFSTACKFRAME_FLAGS_LOOP;
+                break;
+            }
+
+        /* check for insane recursion */
+        if (pCur->iFrame >= 2048)
+            pCur->fFlags |= DBGFSTACKFRAME_FLAGS_MAX_DEPTH;
+    }
+
+    *ppFirstFrame = pCur->pFirstInternal;
+    return rc;
+}
+
+
+/**
+ * Common worker for DBGFR3StackWalkBeginGuestEx, DBGFR3StackWalkBeginHyperEx,
+ * DBGFR3StackWalkBeginGuest and DBGFR3StackWalkBeginHyper.
+ */
+static int dbgfR3StackWalkBeginCommon(PUVM pUVM,
+                                      VMCPUID idCpu,
+                                      DBGFCODETYPE enmCodeType,
+                                      PCDBGFADDRESS pAddrFrame,
+                                      PCDBGFADDRESS pAddrStack,
+                                      PCDBGFADDRESS pAddrPC,
+                                      RTDBGRETURNTYPE enmReturnType,
+                                      PCDBGFSTACKFRAME *ppFirstFrame)
+{
+    /*
+     * Validate parameters.
+     */
+    *ppFirstFrame = NULL;
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+    if (pAddrFrame)
+        AssertReturn(DBGFR3AddrIsValid(pUVM, pAddrFrame), VERR_INVALID_PARAMETER);
+    if (pAddrStack)
+        AssertReturn(DBGFR3AddrIsValid(pUVM, pAddrStack), VERR_INVALID_PARAMETER);
+    if (pAddrPC)
+        AssertReturn(DBGFR3AddrIsValid(pUVM, pAddrPC), VERR_INVALID_PARAMETER);
+    AssertReturn(enmReturnType >= RTDBGRETURNTYPE_INVALID && enmReturnType < RTDBGRETURNTYPE_END, VERR_INVALID_PARAMETER);
+
+    /*
+     * Get the CPUM context pointer and pass it on the specified EMT.
+     */
+    RTDBGAS     hAs;
+    PCCPUMCTX   pCtx;
+    switch (enmCodeType)
+    {
+        case DBGFCODETYPE_GUEST:
+            pCtx = CPUMQueryGuestCtxPtr(VMMGetCpuById(pVM, idCpu));
+            hAs  = DBGF_AS_GLOBAL;
+            break;
+        case DBGFCODETYPE_HYPER:
+            pCtx = CPUMQueryGuestCtxPtr(VMMGetCpuById(pVM, idCpu));
+            hAs  = DBGF_AS_RC_AND_GC_GLOBAL;
+            break;
+        case DBGFCODETYPE_RING0:
+            pCtx = NULL;    /* No valid context present. */
+            hAs  = DBGF_AS_R0;
+            break;
+        default:
+            AssertFailedReturn(VERR_INVALID_PARAMETER);
+    }
+    return VMR3ReqPriorityCallWaitU(pUVM, idCpu, (PFNRT)dbgfR3StackWalkCtxFull, 10,
+                                    pUVM, idCpu, pCtx, hAs, enmCodeType,
+                                    pAddrFrame, pAddrStack, pAddrPC, enmReturnType, ppFirstFrame);
+}
+
+
+/**
+ * Begins a guest stack walk, extended version.
+ *
+ * This will walk the current stack, constructing a list of info frames which is
+ * returned to the caller. The caller uses DBGFR3StackWalkNext to traverse the
+ * list and DBGFR3StackWalkEnd to release it.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VERR_NO_MEMORY if we're out of memory.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   idCpu           The ID of the virtual CPU which stack we want to walk.
+ * @param   enmCodeType     Code type
+ * @param   pAddrFrame      Frame address to start at. (Optional)
+ * @param   pAddrStack      Stack address to start at. (Optional)
+ * @param   pAddrPC         Program counter to start at. (Optional)
+ * @param   enmReturnType   The return address type. (Optional)
+ * @param   ppFirstFrame    Where to return the pointer to the first info frame.
+ */
+VMMR3DECL(int) DBGFR3StackWalkBeginEx(PUVM pUVM,
+                                      VMCPUID idCpu,
+                                      DBGFCODETYPE enmCodeType,
+                                      PCDBGFADDRESS pAddrFrame,
+                                      PCDBGFADDRESS pAddrStack,
+                                      PCDBGFADDRESS pAddrPC,
+                                      RTDBGRETURNTYPE enmReturnType,
+                                      PCDBGFSTACKFRAME *ppFirstFrame)
+{
+    return dbgfR3StackWalkBeginCommon(pUVM, idCpu, enmCodeType, pAddrFrame, pAddrStack, pAddrPC, enmReturnType, ppFirstFrame);
+}
+
+
+/**
+ * Begins a guest stack walk.
+ *
+ * This will walk the current stack, constructing a list of info frames which is
+ * returned to the caller. The caller uses DBGFR3StackWalkNext to traverse the
+ * list and DBGFR3StackWalkEnd to release it.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VERR_NO_MEMORY if we're out of memory.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   idCpu           The ID of the virtual CPU which stack we want to walk.
+ * @param   enmCodeType     Code type
+ * @param   ppFirstFrame    Where to return the pointer to the first info frame.
+ */
+VMMR3DECL(int) DBGFR3StackWalkBegin(PUVM pUVM, VMCPUID idCpu, DBGFCODETYPE enmCodeType, PCDBGFSTACKFRAME *ppFirstFrame)
+{
+    return dbgfR3StackWalkBeginCommon(pUVM, idCpu, enmCodeType, NULL, NULL, NULL, RTDBGRETURNTYPE_INVALID, ppFirstFrame);
+}
+
+/**
+ * Gets the next stack frame.
+ *
+ * @returns Pointer to the info for the next stack frame.
+ *          NULL if no more frames.
+ *
+ * @param   pCurrent    Pointer to the current stack frame.
+ *
+ */
+VMMR3DECL(PCDBGFSTACKFRAME) DBGFR3StackWalkNext(PCDBGFSTACKFRAME pCurrent)
+{
+    return pCurrent
+         ? pCurrent->pNextInternal
+         : NULL;
+}
+
+
+/**
+ * Ends a stack walk process.
+ *
+ * This *must* be called after a successful first call to any of the stack
+ * walker functions. If not called we will leak memory or other resources.
+ *
+ * @param   pFirstFrame     The frame returned by one of the begin functions.
+ */
+VMMR3DECL(void) DBGFR3StackWalkEnd(PCDBGFSTACKFRAME pFirstFrame)
+{
+    if (    !pFirstFrame
+        ||  !pFirstFrame->pFirstInternal)
+        return;
+
+    PDBGFSTACKFRAME pFrame = (PDBGFSTACKFRAME)pFirstFrame->pFirstInternal;
+    while (pFrame)
+    {
+        PDBGFSTACKFRAME pCur = pFrame;
+        pFrame = (PDBGFSTACKFRAME)pCur->pNextInternal;
+        if (pFrame)
+        {
+            if (pCur->pSymReturnPC == pFrame->pSymPC)
+                pFrame->pSymPC = NULL;
+            if (pCur->pSymReturnPC == pFrame->pSymReturnPC)
+                pFrame->pSymReturnPC = NULL;
+
+            if (pCur->pSymPC == pFrame->pSymPC)
+                pFrame->pSymPC = NULL;
+            if (pCur->pSymPC == pFrame->pSymReturnPC)
+                pFrame->pSymReturnPC = NULL;
+
+            if (pCur->pLineReturnPC == pFrame->pLinePC)
+                pFrame->pLinePC = NULL;
+            if (pCur->pLineReturnPC == pFrame->pLineReturnPC)
+                pFrame->pLineReturnPC = NULL;
+
+            if (pCur->pLinePC == pFrame->pLinePC)
+                pFrame->pLinePC = NULL;
+            if (pCur->pLinePC == pFrame->pLineReturnPC)
+                pFrame->pLineReturnPC = NULL;
+        }
+
+        RTDbgSymbolFree(pCur->pSymPC);
+        RTDbgSymbolFree(pCur->pSymReturnPC);
+        RTDbgLineFree(pCur->pLinePC);
+        RTDbgLineFree(pCur->pLineReturnPC);
+
+        if (pCur->paSureRegs)
+        {
+            MMR3HeapFree(pCur->paSureRegs);
+            pCur->paSureRegs = NULL;
+            pCur->cSureRegs = 0;
+        }
+
+        pCur->pNextInternal = NULL;
+        pCur->pFirstInternal = NULL;
+        pCur->fFlags = 0;
+        MMR3HeapFree(pCur);
+    }
+}
+
diff --git a/src/VBox/VMM/VMMR3/EM.cpp b/src/VBox/VMM/VMMR3/EM.cpp
new file mode 100644
index 00000000..1a356777
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/EM.cpp
@@ -0,0 +1,2869 @@
+/* $Id: EM.cpp $ */
+/** @file
+ * EM - Execution Monitor / Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_em         EM - The Execution Monitor / Manager
+ *
+ * The Execution Monitor/Manager is responsible for running the VM, scheduling
+ * the right kind of execution (Raw-mode, Hardware Assisted, Recompiled or
+ * Interpreted), and keeping the CPU states in sync. The function
+ * EMR3ExecuteVM() is the 'main-loop' of the VM, while each of the execution
+ * modes has different inner loops (emR3RawExecute, emR3HmExecute, and
+ * emR3RemExecute).
+ *
+ * The interpreted execution is only used to avoid switching between
+ * raw-mode/hm and the recompiler when fielding virtualization traps/faults.
+ * The interpretation is thus implemented as part of EM.
+ *
+ * @see grp_em
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_EM
+#define VMCPU_INCL_CPUM_GST_CTX /* for CPUM_IMPORT_GUEST_STATE_RET */
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/pdmqueue.h>
+#include <VBox/vmm/hm.h>
+#include "EMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/cpumdis.h>
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <VBox/err.h>
+#include "VMMTracing.h"
+
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <iprt/stream.h>
+#include <iprt/thread.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(int) emR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int) emR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+#if defined(LOG_ENABLED) || defined(VBOX_STRICT)
+static const char *emR3GetStateName(EMSTATE enmState);
+#endif
+static VBOXSTRICTRC emR3Debug(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc);
+#if defined(VBOX_WITH_REM) || defined(DEBUG)
+static int emR3RemStep(PVM pVM, PVMCPU pVCpu);
+#endif
+static int emR3RemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
+
+
+/**
+ * Initializes the EM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) EMR3Init(PVM pVM)
+{
+    LogFlow(("EMR3Init\n"));
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompileMemberAlignment(VM, em.s, 32);
+    AssertCompile(sizeof(pVM->em.s) <= sizeof(pVM->em.padding));
+    AssertCompile(RT_SIZEOFMEMB(VMCPU, em.s.u.FatalLongJump) <= RT_SIZEOFMEMB(VMCPU, em.s.u.achPaddingFatalLongJump));
+    AssertCompile(RT_SIZEOFMEMB(VMCPU, em.s) <= RT_SIZEOFMEMB(VMCPU, em.padding));
+
+    /*
+     * Init the structure.
+     */
+    PCFGMNODE pCfgRoot = CFGMR3GetRoot(pVM);
+    PCFGMNODE pCfgEM = CFGMR3GetChild(pCfgRoot, "EM");
+
+    int rc = CFGMR3QueryBoolDef(pCfgEM, "IemExecutesAll", &pVM->em.s.fIemExecutesAll, false);
+    AssertLogRelRCReturn(rc, rc);
+
+    bool fEnabled;
+    rc = CFGMR3QueryBoolDef(pCfgEM, "TripleFaultReset", &fEnabled, false);
+    AssertLogRelRCReturn(rc, rc);
+    pVM->em.s.fGuruOnTripleFault = !fEnabled;
+    if (!pVM->em.s.fGuruOnTripleFault && pVM->cCpus > 1)
+    {
+        LogRel(("EM: Overriding /EM/TripleFaultReset, must be false on SMP.\n"));
+        pVM->em.s.fGuruOnTripleFault = true;
+    }
+
+    LogRel(("EMR3Init: fIemExecutesAll=%RTbool fGuruOnTripleFault=%RTbool\n", pVM->em.s.fIemExecutesAll, pVM->em.s.fGuruOnTripleFault));
+
+    /** @cfgm{/EM/ExitOptimizationEnabled, bool, true}
+     * Whether to try correlate exit history in any context, detect hot spots and
+     * try optimize these using IEM if there are other exits close by.  This
+     * overrides the context specific settings. */
+    bool fExitOptimizationEnabled = true;
+    rc = CFGMR3QueryBoolDef(pCfgEM, "ExitOptimizationEnabled", &fExitOptimizationEnabled, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/EM/ExitOptimizationEnabledR0, bool, true}
+     * Whether to optimize exits in ring-0.  Setting this to false will also disable
+     * the /EM/ExitOptimizationEnabledR0PreemptDisabled setting.  Depending on preemption
+     * capabilities of the host kernel, this optimization may be unavailable. */
+    bool fExitOptimizationEnabledR0 = true;
+    rc = CFGMR3QueryBoolDef(pCfgEM, "ExitOptimizationEnabledR0", &fExitOptimizationEnabledR0, true);
+    AssertLogRelRCReturn(rc, rc);
+    fExitOptimizationEnabledR0 &= fExitOptimizationEnabled;
+
+    /** @cfgm{/EM/ExitOptimizationEnabledR0PreemptDisabled, bool, false}
+     * Whether to optimize exits in ring-0 when preemption is disable (or preemption
+     * hooks are in effect). */
+    /** @todo change the default to true here */
+    bool fExitOptimizationEnabledR0PreemptDisabled = true;
+    rc = CFGMR3QueryBoolDef(pCfgEM, "ExitOptimizationEnabledR0PreemptDisabled", &fExitOptimizationEnabledR0PreemptDisabled, false);
+    AssertLogRelRCReturn(rc, rc);
+    fExitOptimizationEnabledR0PreemptDisabled &= fExitOptimizationEnabledR0;
+
+    /** @cfgm{/EM/HistoryExecMaxInstructions, integer, 16, 65535, 8192}
+     * Maximum number of instruction to let EMHistoryExec execute in one go. */
+    uint16_t cHistoryExecMaxInstructions = 8192;
+    rc = CFGMR3QueryU16Def(pCfgEM, "HistoryExecMaxInstructions", &cHistoryExecMaxInstructions, cHistoryExecMaxInstructions);
+    AssertLogRelRCReturn(rc, rc);
+    if (cHistoryExecMaxInstructions < 16)
+        return VMSetError(pVM, VERR_OUT_OF_RANGE, RT_SRC_POS, "/EM/HistoryExecMaxInstructions value is too small, min 16");
+
+    /** @cfgm{/EM/HistoryProbeMaxInstructionsWithoutExit, integer, 2, 65535, 24 for HM, 32 for NEM}
+     * Maximum number of instruction between exits during probing. */
+    uint16_t cHistoryProbeMaxInstructionsWithoutExit = 24;
+#ifdef RT_OS_WINDOWS
+    if (VM_IS_NEM_ENABLED(pVM))
+        cHistoryProbeMaxInstructionsWithoutExit = 32;
+#endif
+    rc = CFGMR3QueryU16Def(pCfgEM, "HistoryProbeMaxInstructionsWithoutExit", &cHistoryProbeMaxInstructionsWithoutExit,
+                           cHistoryProbeMaxInstructionsWithoutExit);
+    AssertLogRelRCReturn(rc, rc);
+    if (cHistoryProbeMaxInstructionsWithoutExit < 2)
+        return VMSetError(pVM, VERR_OUT_OF_RANGE, RT_SRC_POS,
+                          "/EM/HistoryProbeMaxInstructionsWithoutExit value is too small, min 16");
+
+    /** @cfgm{/EM/HistoryProbMinInstructions, integer, 0, 65535, depends}
+     * The default is (/EM/HistoryProbeMaxInstructionsWithoutExit + 1) * 3. */
+    uint16_t cHistoryProbeMinInstructions = cHistoryProbeMaxInstructionsWithoutExit < 0x5554
+                                          ? (cHistoryProbeMaxInstructionsWithoutExit + 1) * 3 : 0xffff;
+    rc = CFGMR3QueryU16Def(pCfgEM, "HistoryProbMinInstructions", &cHistoryProbeMinInstructions,
+                           cHistoryProbeMinInstructions);
+    AssertLogRelRCReturn(rc, rc);
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        pVCpu->em.s.fExitOptimizationEnabled                  = fExitOptimizationEnabled;
+        pVCpu->em.s.fExitOptimizationEnabledR0                = fExitOptimizationEnabledR0;
+        pVCpu->em.s.fExitOptimizationEnabledR0PreemptDisabled = fExitOptimizationEnabledR0PreemptDisabled;
+        pVCpu->em.s.cHistoryExecMaxInstructions               = cHistoryExecMaxInstructions;
+        pVCpu->em.s.cHistoryProbeMinInstructions              = cHistoryProbeMinInstructions;
+        pVCpu->em.s.cHistoryProbeMaxInstructionsWithoutExit   = cHistoryProbeMaxInstructionsWithoutExit;
+    }
+
+    /*
+     * Saved state.
+     */
+    rc = SSMR3RegisterInternal(pVM, "em", 0, EM_SAVED_STATE_VERSION, 16,
+                               NULL, NULL, NULL,
+                               NULL, emR3Save, NULL,
+                               NULL, emR3Load, NULL);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+
+        pVCpu->em.s.enmState            = idCpu == 0 ? EMSTATE_NONE : EMSTATE_WAIT_SIPI;
+        pVCpu->em.s.enmPrevState        = EMSTATE_NONE;
+        pVCpu->em.s.u64TimeSliceStart   = 0; /* paranoia */
+        pVCpu->em.s.idxContinueExitRec  = UINT16_MAX;
+
+# define EM_REG_COUNTER(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b, idCpu); \
+        AssertRC(rc);
+
+# define EM_REG_COUNTER_USED(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, c, b, idCpu); \
+        AssertRC(rc);
+
+# define EM_REG_PROFILE(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, idCpu); \
+        AssertRC(rc);
+
+# define EM_REG_PROFILE_ADV(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, idCpu); \
+        AssertRC(rc);
+
+        /*
+         * Statistics.
+         */
+#ifdef VBOX_WITH_STATISTICS
+        PEMSTATS pStats;
+        rc = MMHyperAlloc(pVM, sizeof(*pStats), 0, MM_TAG_EM, (void **)&pStats);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        pVCpu->em.s.pStatsR3 = pStats;
+        pVCpu->em.s.pStatsR0 = MMHyperR3ToR0(pVM, pStats);
+
+# if 1 /* rawmode only? */
+        EM_REG_COUNTER_USED(&pStats->StatIoRestarted,       "/EM/CPU%u/R3/PrivInst/IoRestarted",        "I/O instructions restarted in ring-3.");
+        EM_REG_COUNTER_USED(&pStats->StatIoIem,             "/EM/CPU%u/R3/PrivInst/IoIem",              "I/O instructions end to IEM in ring-3.");
+        EM_REG_COUNTER_USED(&pStats->StatCli,               "/EM/CPU%u/R3/PrivInst/Cli",                "Number of cli instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatSti,               "/EM/CPU%u/R3/PrivInst/Sti",                "Number of sli instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatHlt,               "/EM/CPU%u/R3/PrivInst/Hlt",                "Number of hlt instructions not handled in GC because of PATM.");
+        EM_REG_COUNTER_USED(&pStats->StatInvlpg,            "/EM/CPU%u/R3/PrivInst/Invlpg",             "Number of invlpg instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMisc,              "/EM/CPU%u/R3/PrivInst/Misc",               "Number of misc. instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[0],     "/EM/CPU%u/R3/PrivInst/Mov CR0, X",         "Number of mov CR0 write instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[1],     "/EM/CPU%u/R3/PrivInst/Mov CR1, X",         "Number of mov CR1 write instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[2],     "/EM/CPU%u/R3/PrivInst/Mov CR2, X",         "Number of mov CR2 write instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[3],     "/EM/CPU%u/R3/PrivInst/Mov CR3, X",         "Number of mov CR3 write instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[4],     "/EM/CPU%u/R3/PrivInst/Mov CR4, X",         "Number of mov CR4 write instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[0],      "/EM/CPU%u/R3/PrivInst/Mov X, CR0",         "Number of mov CR0 read instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[1],      "/EM/CPU%u/R3/PrivInst/Mov X, CR1",         "Number of mov CR1 read instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[2],      "/EM/CPU%u/R3/PrivInst/Mov X, CR2",         "Number of mov CR2 read instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[3],      "/EM/CPU%u/R3/PrivInst/Mov X, CR3",         "Number of mov CR3 read instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[4],      "/EM/CPU%u/R3/PrivInst/Mov X, CR4",         "Number of mov CR4 read instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovDRx,            "/EM/CPU%u/R3/PrivInst/MovDRx",             "Number of mov DRx instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatIret,              "/EM/CPU%u/R3/PrivInst/Iret",               "Number of iret instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovLgdt,           "/EM/CPU%u/R3/PrivInst/Lgdt",               "Number of lgdt instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovLidt,           "/EM/CPU%u/R3/PrivInst/Lidt",               "Number of lidt instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovLldt,           "/EM/CPU%u/R3/PrivInst/Lldt",               "Number of lldt instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatSysEnter,          "/EM/CPU%u/R3/PrivInst/Sysenter",           "Number of sysenter instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatSysExit,           "/EM/CPU%u/R3/PrivInst/Sysexit",            "Number of sysexit instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatSysCall,           "/EM/CPU%u/R3/PrivInst/Syscall",            "Number of syscall instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatSysRet,            "/EM/CPU%u/R3/PrivInst/Sysret",             "Number of sysret instructions.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatTotalClis,          "/EM/CPU%u/Cli/Total",                      "Total number of cli instructions executed.");
+#endif
+        pVCpu->em.s.pCliStatTree = 0;
+
+        /* these should be considered for release statistics. */
+        EM_REG_COUNTER(&pVCpu->em.s.StatIOEmu,              "/PROF/CPU%u/EM/Emulation/IO",      "Profiling of emR3RawExecuteIOInstruction.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatPrivEmu,            "/PROF/CPU%u/EM/Emulation/Priv",    "Profiling of emR3RawPrivileged.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatHMEntry,            "/PROF/CPU%u/EM/HMEnter",           "Profiling Hardware Accelerated Mode entry overhead.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatHMExec,             "/PROF/CPU%u/EM/HMExec",            "Profiling Hardware Accelerated Mode execution.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatHMExecuteCalled,    "/PROF/CPU%u/EM/HMExecuteCalled",   "Number of times enmR3HMExecute is called.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatIEMEmu,             "/PROF/CPU%u/EM/IEMEmuSingle",      "Profiling single instruction IEM execution.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatIEMThenREM,         "/PROF/CPU%u/EM/IEMThenRem",        "Profiling IEM-then-REM instruction execution (by IEM).");
+        EM_REG_PROFILE(&pVCpu->em.s.StatNEMEntry,           "/PROF/CPU%u/EM/NEMEnter",          "Profiling NEM entry overhead.");
+#endif /* VBOX_WITH_STATISTICS */
+        EM_REG_PROFILE(&pVCpu->em.s.StatNEMExec,            "/PROF/CPU%u/EM/NEMExec",           "Profiling NEM execution.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatNEMExecuteCalled,   "/PROF/CPU%u/EM/NEMExecuteCalled",  "Number of times enmR3NEMExecute is called.");
+#ifdef VBOX_WITH_STATISTICS
+        EM_REG_PROFILE(&pVCpu->em.s.StatREMEmu,             "/PROF/CPU%u/EM/REMEmuSingle",      "Profiling single instruction REM execution.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatREMExec,            "/PROF/CPU%u/EM/REMExec",           "Profiling REM execution.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatREMSync,            "/PROF/CPU%u/EM/REMSync",           "Profiling REM context syncing.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatRAWEntry,           "/PROF/CPU%u/EM/RAWEnter",          "Profiling Raw Mode entry overhead.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatRAWExec,            "/PROF/CPU%u/EM/RAWExec",           "Profiling Raw Mode execution.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatRAWTail,            "/PROF/CPU%u/EM/RAWTail",           "Profiling Raw Mode tail overhead.");
+#endif /* VBOX_WITH_STATISTICS */
+
+        EM_REG_COUNTER(&pVCpu->em.s.StatForcedActions,      "/PROF/CPU%u/EM/ForcedActions",     "Profiling forced action execution.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatHalted,             "/PROF/CPU%u/EM/Halted",            "Profiling halted state (VMR3WaitHalted).");
+        EM_REG_PROFILE_ADV(&pVCpu->em.s.StatCapped,         "/PROF/CPU%u/EM/Capped",            "Profiling capped state (sleep).");
+        EM_REG_COUNTER(&pVCpu->em.s.StatREMTotal,           "/PROF/CPU%u/EM/REMTotal",          "Profiling emR3RemExecute (excluding FFs).");
+        EM_REG_COUNTER(&pVCpu->em.s.StatRAWTotal,           "/PROF/CPU%u/EM/RAWTotal",          "Profiling emR3RawExecute (excluding FFs).");
+
+        EM_REG_PROFILE_ADV(&pVCpu->em.s.StatTotal,          "/PROF/CPU%u/EM/Total",             "Profiling EMR3ExecuteVM.");
+
+        rc = STAMR3RegisterF(pVM, &pVCpu->em.s.iNextExit, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of recorded exits.", "/PROF/CPU%u/EM/RecordedExits", idCpu);
+        AssertRC(rc);
+
+        /* History record statistics */
+        rc = STAMR3RegisterF(pVM, &pVCpu->em.s.cExitRecordUsed, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of used hash table entries.", "/EM/CPU%u/ExitHashing/Used", idCpu);
+        AssertRC(rc);
+
+        for (uint32_t iStep = 0; iStep < RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits); iStep++)
+        {
+            rc = STAMR3RegisterF(pVM, &pVCpu->em.s.aStatHistoryRecHits[iStep], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                                 "Number of hits at this step.",             "/EM/CPU%u/ExitHashing/Step%02u-Hits", idCpu, iStep);
+            AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pVCpu->em.s.aStatHistoryRecTypeChanged[iStep], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                                 "Number of type changes at this step.",     "/EM/CPU%u/ExitHashing/Step%02u-TypeChanges", idCpu, iStep);
+            AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pVCpu->em.s.aStatHistoryRecTypeChanged[iStep], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                                 "Number of replacments at this step.",     "/EM/CPU%u/ExitHashing/Step%02u-Replacments", idCpu, iStep);
+            AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pVCpu->em.s.aStatHistoryRecNew[iStep], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                                 "Number of new inserts at this step.",     "/EM/CPU%u/ExitHashing/Step%02u-NewInserts", idCpu, iStep);
+            AssertRC(rc);
+        }
+
+        EM_REG_PROFILE(&pVCpu->em.s.StatHistoryExec,              "/EM/CPU%u/ExitOpt/Exec",              "Profiling normal EMHistoryExec operation.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatHistoryExecSavedExits,    "/EM/CPU%u/ExitOpt/ExecSavedExit",     "Net number of saved exits.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatHistoryExecInstructions,  "/EM/CPU%u/ExitOpt/ExecInstructions",  "Number of instructions executed during normal operation.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatHistoryProbe,             "/EM/CPU%u/ExitOpt/Probe",             "Profiling EMHistoryExec when probing.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatHistoryProbeInstructions, "/EM/CPU%u/ExitOpt/ProbeInstructions", "Number of instructions executed during probing.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatHistoryProbedNormal,      "/EM/CPU%u/ExitOpt/ProbedNormal",      "Number of EMEXITACTION_NORMAL_PROBED results.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatHistoryProbedExecWithMax, "/EM/CPU%u/ExitOpt/ProbedExecWithMax", "Number of EMEXITACTION_EXEC_WITH_MAX results.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatHistoryProbedToRing3,     "/EM/CPU%u/ExitOpt/ProbedToRing3",     "Number of ring-3 probe continuations.");
+    }
+
+    emR3InitDbg(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Called when a VM initialization stage is completed.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   enmWhat         The initialization state that was completed.
+ */
+VMMR3_INT_DECL(int) EMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    if (enmWhat == VMINITCOMPLETED_RING0)
+        LogRel(("EM: Exit history optimizations: enabled=%RTbool enabled-r0=%RTbool enabled-r0-no-preemption=%RTbool\n",
+                pVM->apCpusR3[0]->em.s.fExitOptimizationEnabled, pVM->apCpusR3[0]->em.s.fExitOptimizationEnabledR0,
+                pVM->apCpusR3[0]->em.s.fExitOptimizationEnabledR0PreemptDisabled));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) EMR3Relocate(PVM pVM)
+{
+    LogFlow(("EMR3Relocate\n"));
+    RT_NOREF(pVM);
+}
+
+
+/**
+ * Reset the EM state for a CPU.
+ *
+ * Called by EMR3Reset and hot plugging.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMR3_INT_DECL(void) EMR3ResetCpu(PVMCPU pVCpu)
+{
+    /* Reset scheduling state. */
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_UNHALT);
+
+    /* VMR3ResetFF may return VINF_EM_RESET or VINF_EM_SUSPEND, so transition
+       out of the HALTED state here so that enmPrevState doesn't end up as
+       HALTED when EMR3Execute returns. */
+    if (pVCpu->em.s.enmState == EMSTATE_HALTED)
+    {
+        Log(("EMR3ResetCpu: Cpu#%u %s -> %s\n", pVCpu->idCpu, emR3GetStateName(pVCpu->em.s.enmState), pVCpu->idCpu == 0 ? "EMSTATE_NONE" : "EMSTATE_WAIT_SIPI"));
+        pVCpu->em.s.enmState = pVCpu->idCpu == 0 ? EMSTATE_NONE : EMSTATE_WAIT_SIPI;
+    }
+}
+
+
+/**
+ * Reset notification.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) EMR3Reset(PVM pVM)
+{
+    Log(("EMR3Reset: \n"));
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        EMR3ResetCpu(pVM->apCpusR3[idCpu]);
+}
+
+
+/**
+ * Terminates the EM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) EMR3Term(PVM pVM)
+{
+    RT_NOREF(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) emR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+
+        SSMR3PutBool(pSSM, false /*fForceRAW*/);
+
+        Assert(pVCpu->em.s.enmState     == EMSTATE_SUSPENDED);
+        Assert(pVCpu->em.s.enmPrevState != EMSTATE_SUSPENDED);
+        SSMR3PutU32(pSSM, pVCpu->em.s.enmPrevState);
+
+        /* Save mwait state. */
+        SSMR3PutU32(pSSM, pVCpu->em.s.MWait.fWait);
+        SSMR3PutGCPtr(pSSM, pVCpu->em.s.MWait.uMWaitRAX);
+        SSMR3PutGCPtr(pSSM, pVCpu->em.s.MWait.uMWaitRCX);
+        SSMR3PutGCPtr(pSSM, pVCpu->em.s.MWait.uMonitorRAX);
+        SSMR3PutGCPtr(pSSM, pVCpu->em.s.MWait.uMonitorRCX);
+        int rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.MWait.uMonitorRDX);
+        AssertRCReturn(rc, rc);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) emR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    /*
+     * Validate version.
+     */
+    if (    uVersion > EM_SAVED_STATE_VERSION
+        ||  uVersion < EM_SAVED_STATE_VERSION_PRE_SMP)
+    {
+        AssertMsgFailed(("emR3Load: Invalid version uVersion=%d (current %d)!\n", uVersion, EM_SAVED_STATE_VERSION));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+
+    /*
+     * Load the saved state.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+
+        bool fForceRAWIgnored;
+        int rc = SSMR3GetBool(pSSM, &fForceRAWIgnored);
+        AssertRCReturn(rc, rc);
+
+        if (uVersion > EM_SAVED_STATE_VERSION_PRE_SMP)
+        {
+            SSM_GET_ENUM32_RET(pSSM, pVCpu->em.s.enmPrevState, EMSTATE);
+            Assert(pVCpu->em.s.enmPrevState != EMSTATE_SUSPENDED);
+
+            pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
+        }
+        if (uVersion > EM_SAVED_STATE_VERSION_PRE_MWAIT)
+        {
+            /* Load mwait state. */
+            rc = SSMR3GetU32(pSSM, &pVCpu->em.s.MWait.fWait);
+            AssertRCReturn(rc, rc);
+            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.MWait.uMWaitRAX);
+            AssertRCReturn(rc, rc);
+            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.MWait.uMWaitRCX);
+            AssertRCReturn(rc, rc);
+            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.MWait.uMonitorRAX);
+            AssertRCReturn(rc, rc);
+            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.MWait.uMonitorRCX);
+            AssertRCReturn(rc, rc);
+            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.MWait.uMonitorRDX);
+            AssertRCReturn(rc, rc);
+        }
+
+        Assert(!pVCpu->em.s.pCliStatTree);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Argument packet for emR3SetExecutionPolicy.
+ */
+struct EMR3SETEXECPOLICYARGS
+{
+    EMEXECPOLICY    enmPolicy;
+    bool            fEnforce;
+};
+
+
+/**
+ * @callback_method_impl{FNVMMEMTRENDEZVOUS, Rendezvous callback for EMR3SetExecutionPolicy.}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) emR3SetExecutionPolicy(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    /*
+     * Only the first CPU changes the variables.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        struct EMR3SETEXECPOLICYARGS *pArgs = (struct EMR3SETEXECPOLICYARGS *)pvUser;
+        switch (pArgs->enmPolicy)
+        {
+            case EMEXECPOLICY_RECOMPILE_RING0:
+            case EMEXECPOLICY_RECOMPILE_RING3:
+                break;
+            case EMEXECPOLICY_IEM_ALL:
+                pVM->em.s.fIemExecutesAll = pArgs->fEnforce;
+                break;
+            default:
+                AssertFailedReturn(VERR_INVALID_PARAMETER);
+        }
+        Log(("EM: Set execution policy (fIemExecutesAll=%RTbool)\n", pVM->em.s.fIemExecutesAll));
+    }
+
+    /*
+     * Force rescheduling if in RAW, HM, NEM, IEM, or REM.
+     */
+    return    pVCpu->em.s.enmState == EMSTATE_RAW
+           || pVCpu->em.s.enmState == EMSTATE_HM
+           || pVCpu->em.s.enmState == EMSTATE_NEM
+           || pVCpu->em.s.enmState == EMSTATE_IEM
+           || pVCpu->em.s.enmState == EMSTATE_REM
+           || pVCpu->em.s.enmState == EMSTATE_IEM_THEN_REM
+         ? VINF_EM_RESCHEDULE
+         : VINF_SUCCESS;
+}
+
+
+/**
+ * Changes an execution scheduling policy parameter.
+ *
+ * This is used to enable or disable raw-mode / hardware-virtualization
+ * execution of user and supervisor code.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VINF_RESCHEDULE if a rescheduling might be required.
+ * @returns VERR_INVALID_PARAMETER on an invalid enmMode value.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   enmPolicy       The scheduling policy to change.
+ * @param   fEnforce        Whether to enforce the policy or not.
+ */
+VMMR3DECL(int) EMR3SetExecutionPolicy(PUVM pUVM, EMEXECPOLICY enmPolicy, bool fEnforce)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(enmPolicy > EMEXECPOLICY_INVALID && enmPolicy < EMEXECPOLICY_END, VERR_INVALID_PARAMETER);
+
+    struct EMR3SETEXECPOLICYARGS Args = { enmPolicy, fEnforce };
+    return VMMR3EmtRendezvous(pUVM->pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING, emR3SetExecutionPolicy, &Args);
+}
+
+
+/**
+ * Queries an execution scheduling policy parameter.
+ *
+ * @returns VBox status code
+ * @param   pUVM            The user mode VM handle.
+ * @param   enmPolicy       The scheduling policy to query.
+ * @param   pfEnforced      Where to return the current value.
+ */
+VMMR3DECL(int) EMR3QueryExecutionPolicy(PUVM pUVM, EMEXECPOLICY enmPolicy, bool *pfEnforced)
+{
+    AssertReturn(enmPolicy > EMEXECPOLICY_INVALID && enmPolicy < EMEXECPOLICY_END, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pfEnforced, VERR_INVALID_POINTER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /* No need to bother EMTs with a query. */
+    switch (enmPolicy)
+    {
+        case EMEXECPOLICY_RECOMPILE_RING0:
+        case EMEXECPOLICY_RECOMPILE_RING3:
+            *pfEnforced = false;
+            break;
+        case EMEXECPOLICY_IEM_ALL:
+            *pfEnforced = pVM->em.s.fIemExecutesAll;
+            break;
+        default:
+            AssertFailedReturn(VERR_INTERNAL_ERROR_2);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Queries the main execution engine of the VM.
+ *
+ * @returns VBox status code
+ * @param   pUVM                    The user mode VM handle.
+ * @param   pbMainExecutionEngine   Where to return the result, VM_EXEC_ENGINE_XXX.
+ */
+VMMR3DECL(int) EMR3QueryMainExecutionEngine(PUVM pUVM, uint8_t *pbMainExecutionEngine)
+{
+    AssertPtrReturn(pbMainExecutionEngine, VERR_INVALID_POINTER);
+    *pbMainExecutionEngine = VM_EXEC_ENGINE_NOT_SET;
+
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    *pbMainExecutionEngine = pVM->bMainExecutionEngine;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Raise a fatal error.
+ *
+ * Safely terminate the VM with full state report and stuff. This function
+ * will naturally never return.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   rc          VBox status code.
+ */
+VMMR3DECL(void) EMR3FatalError(PVMCPU pVCpu, int rc)
+{
+    pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
+    longjmp(pVCpu->em.s.u.FatalLongJump, rc);
+}
+
+
+#if defined(LOG_ENABLED) || defined(VBOX_STRICT)
+/**
+ * Gets the EM state name.
+ *
+ * @returns pointer to read only state name,
+ * @param   enmState    The state.
+ */
+static const char *emR3GetStateName(EMSTATE enmState)
+{
+    switch (enmState)
+    {
+        case EMSTATE_NONE:              return "EMSTATE_NONE";
+        case EMSTATE_RAW:               return "EMSTATE_RAW";
+        case EMSTATE_HM:                return "EMSTATE_HM";
+        case EMSTATE_IEM:               return "EMSTATE_IEM";
+        case EMSTATE_REM:               return "EMSTATE_REM";
+        case EMSTATE_HALTED:            return "EMSTATE_HALTED";
+        case EMSTATE_WAIT_SIPI:         return "EMSTATE_WAIT_SIPI";
+        case EMSTATE_SUSPENDED:         return "EMSTATE_SUSPENDED";
+        case EMSTATE_TERMINATING:       return "EMSTATE_TERMINATING";
+        case EMSTATE_DEBUG_GUEST_RAW:   return "EMSTATE_DEBUG_GUEST_RAW";
+        case EMSTATE_DEBUG_GUEST_HM:    return "EMSTATE_DEBUG_GUEST_HM";
+        case EMSTATE_DEBUG_GUEST_IEM:   return "EMSTATE_DEBUG_GUEST_IEM";
+        case EMSTATE_DEBUG_GUEST_REM:   return "EMSTATE_DEBUG_GUEST_REM";
+        case EMSTATE_DEBUG_HYPER:       return "EMSTATE_DEBUG_HYPER";
+        case EMSTATE_GURU_MEDITATION:   return "EMSTATE_GURU_MEDITATION";
+        case EMSTATE_IEM_THEN_REM:      return "EMSTATE_IEM_THEN_REM";
+        case EMSTATE_NEM:               return "EMSTATE_NEM";
+        case EMSTATE_DEBUG_GUEST_NEM:   return "EMSTATE_DEBUG_GUEST_NEM";
+        default:                        return "Unknown!";
+    }
+}
+#endif /* LOG_ENABLED || VBOX_STRICT */
+
+
+/**
+ * Handle pending ring-3 I/O port write.
+ *
+ * This is in response to a VINF_EM_PENDING_R3_IOPORT_WRITE status code returned
+ * by EMRZSetPendingIoPortWrite() in ring-0 or raw-mode context.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VBOXSTRICTRC emR3ExecutePendingIoPortWrite(PVM pVM, PVMCPU pVCpu)
+{
+    CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
+
+    /* Get and clear the pending data. */
+    RTIOPORT const uPort   = pVCpu->em.s.PendingIoPortAccess.uPort;
+    uint32_t const uValue  = pVCpu->em.s.PendingIoPortAccess.uValue;
+    uint8_t  const cbValue = pVCpu->em.s.PendingIoPortAccess.cbValue;
+    uint8_t  const cbInstr = pVCpu->em.s.PendingIoPortAccess.cbInstr;
+    pVCpu->em.s.PendingIoPortAccess.cbValue = 0;
+
+    /* Assert sanity. */
+    switch (cbValue)
+    {
+        case 1:     Assert(!(cbValue & UINT32_C(0xffffff00))); break;
+        case 2:     Assert(!(cbValue & UINT32_C(0xffff0000))); break;
+        case 4:     break;
+        default:    AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_EM_INTERNAL_ERROR);
+    }
+    AssertReturn(cbInstr <= 15 && cbInstr >= 1, VERR_EM_INTERNAL_ERROR);
+
+    /* Do the work.*/
+    VBOXSTRICTRC rcStrict = IOMIOPortWrite(pVM, pVCpu, uPort, uValue, cbValue);
+    LogFlow(("EM/OUT: %#x, %#x LB %u -> %Rrc\n", uPort, uValue, cbValue, VBOXSTRICTRC_VAL(rcStrict) ));
+    if (IOM_SUCCESS(rcStrict))
+    {
+        pVCpu->cpum.GstCtx.rip += cbInstr;
+        pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Handle pending ring-3 I/O port write.
+ *
+ * This is in response to a VINF_EM_PENDING_R3_IOPORT_WRITE status code returned
+ * by EMRZSetPendingIoPortRead() in ring-0 or raw-mode context.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VBOXSTRICTRC emR3ExecutePendingIoPortRead(PVM pVM, PVMCPU pVCpu)
+{
+    CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_RAX);
+
+    /* Get and clear the pending data. */
+    RTIOPORT const uPort   = pVCpu->em.s.PendingIoPortAccess.uPort;
+    uint8_t  const cbValue = pVCpu->em.s.PendingIoPortAccess.cbValue;
+    uint8_t  const cbInstr = pVCpu->em.s.PendingIoPortAccess.cbInstr;
+    pVCpu->em.s.PendingIoPortAccess.cbValue = 0;
+
+    /* Assert sanity. */
+    switch (cbValue)
+    {
+        case 1:     break;
+        case 2:     break;
+        case 4:     break;
+        default:    AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_EM_INTERNAL_ERROR);
+    }
+    AssertReturn(pVCpu->em.s.PendingIoPortAccess.uValue == UINT32_C(0x52454144) /* READ*/, VERR_EM_INTERNAL_ERROR);
+    AssertReturn(cbInstr <= 15 && cbInstr >= 1, VERR_EM_INTERNAL_ERROR);
+
+    /* Do the work.*/
+    uint32_t uValue = 0;
+    VBOXSTRICTRC rcStrict = IOMIOPortRead(pVM, pVCpu, uPort, &uValue, cbValue);
+    LogFlow(("EM/IN: %#x LB %u -> %Rrc, %#x\n", uPort, cbValue, VBOXSTRICTRC_VAL(rcStrict), uValue ));
+    if (IOM_SUCCESS(rcStrict))
+    {
+        if (cbValue == 4)
+            pVCpu->cpum.GstCtx.rax = uValue;
+        else if (cbValue == 2)
+            pVCpu->cpum.GstCtx.ax = (uint16_t)uValue;
+        else
+            pVCpu->cpum.GstCtx.al = (uint8_t)uValue;
+        pVCpu->cpum.GstCtx.rip += cbInstr;
+        pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Debug loop.
+ *
+ * @returns VBox status code for EM.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   rc      Current EM VBox status code.
+ */
+static VBOXSTRICTRC emR3Debug(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc)
+{
+    for (;;)
+    {
+        Log(("emR3Debug: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
+        const VBOXSTRICTRC rcLast = rc;
+
+        /*
+         * Debug related RC.
+         */
+        switch (VBOXSTRICTRC_VAL(rc))
+        {
+            /*
+             * Single step an instruction.
+             */
+            case VINF_EM_DBG_STEP:
+                if (   pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_RAW
+                    || pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER)
+                    AssertLogRelMsgFailedStmt(("Bad EM state."), VERR_EM_INTERNAL_ERROR);
+                else if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HM)
+                    rc = EMR3HmSingleInstruction(pVM, pVCpu, 0 /*fFlags*/);
+                else if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_NEM)
+                    rc = VBOXSTRICTRC_TODO(emR3NemSingleInstruction(pVM, pVCpu, 0 /*fFlags*/));
+#ifdef VBOX_WITH_REM /** @todo fix me? */
+                else if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_REM)
+                    rc = emR3RemStep(pVM, pVCpu);
+#endif
+                else
+                {
+                    rc = IEMExecOne(pVCpu); /** @todo add dedicated interface... */
+                    if (rc == VINF_SUCCESS || rc == VINF_EM_RESCHEDULE)
+                        rc = VINF_EM_DBG_STEPPED;
+                }
+                break;
+
+            /*
+             * Simple events: stepped, breakpoint, stop/assertion.
+             */
+            case VINF_EM_DBG_STEPPED:
+                rc = DBGFR3Event(pVM, DBGFEVENT_STEPPED);
+                break;
+
+            case VINF_EM_DBG_BREAKPOINT:
+                rc = DBGFR3EventBreakpoint(pVM, DBGFEVENT_BREAKPOINT);
+                break;
+
+            case VINF_EM_DBG_STOP:
+                rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, NULL, 0, NULL, NULL);
+                break;
+
+            case VINF_EM_DBG_EVENT:
+                rc = DBGFR3EventHandlePending(pVM, pVCpu);
+                break;
+
+            case VINF_EM_DBG_HYPER_STEPPED:
+                rc = DBGFR3Event(pVM, DBGFEVENT_STEPPED_HYPER);
+                break;
+
+            case VINF_EM_DBG_HYPER_BREAKPOINT:
+                rc = DBGFR3EventBreakpoint(pVM, DBGFEVENT_BREAKPOINT_HYPER);
+                break;
+
+            case VINF_EM_DBG_HYPER_ASSERTION:
+                RTPrintf("\nVINF_EM_DBG_HYPER_ASSERTION:\n%s%s\n", VMMR3GetRZAssertMsg1(pVM), VMMR3GetRZAssertMsg2(pVM));
+                RTLogFlush(NULL);
+                rc = DBGFR3EventAssertion(pVM, DBGFEVENT_ASSERTION_HYPER, VMMR3GetRZAssertMsg1(pVM), VMMR3GetRZAssertMsg2(pVM));
+                break;
+
+            /*
+             * Guru meditation.
+             */
+            case VERR_VMM_RING0_ASSERTION: /** @todo Make a guru meditation event! */
+                rc = DBGFR3EventSrc(pVM, DBGFEVENT_FATAL_ERROR, "VERR_VMM_RING0_ASSERTION", 0, NULL, NULL);
+                break;
+            case VERR_REM_TOO_MANY_TRAPS: /** @todo Make a guru meditation event! */
+                rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, "VERR_REM_TOO_MANY_TRAPS", 0, NULL, NULL);
+                break;
+            case VINF_EM_TRIPLE_FAULT:    /** @todo Make a guru meditation event! */
+                rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, "VINF_EM_TRIPLE_FAULT", 0, NULL, NULL);
+                break;
+
+            default: /** @todo don't use default for guru, but make special errors code! */
+            {
+                LogRel(("emR3Debug: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
+                rc = DBGFR3Event(pVM, DBGFEVENT_FATAL_ERROR);
+                break;
+            }
+        }
+
+        /*
+         * Process the result.
+         */
+        switch (VBOXSTRICTRC_VAL(rc))
+        {
+            /*
+             * Continue the debugging loop.
+             */
+            case VINF_EM_DBG_STEP:
+            case VINF_EM_DBG_STOP:
+            case VINF_EM_DBG_EVENT:
+            case VINF_EM_DBG_STEPPED:
+            case VINF_EM_DBG_BREAKPOINT:
+            case VINF_EM_DBG_HYPER_STEPPED:
+            case VINF_EM_DBG_HYPER_BREAKPOINT:
+            case VINF_EM_DBG_HYPER_ASSERTION:
+                break;
+
+            /*
+             * Resuming execution (in some form) has to be done here if we got
+             * a hypervisor debug event.
+             */
+            case VINF_SUCCESS:
+            case VINF_EM_RESUME:
+            case VINF_EM_SUSPEND:
+            case VINF_EM_RESCHEDULE:
+            case VINF_EM_RESCHEDULE_RAW:
+            case VINF_EM_RESCHEDULE_REM:
+            case VINF_EM_HALT:
+                if (pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER)
+                    AssertLogRelMsgFailedReturn(("Not implemented\n"), VERR_EM_INTERNAL_ERROR);
+                if (rc == VINF_SUCCESS)
+                    rc = VINF_EM_RESCHEDULE;
+                return rc;
+
+            /*
+             * The debugger isn't attached.
+             * We'll simply turn the thing off since that's the easiest thing to do.
+             */
+            case VERR_DBGF_NOT_ATTACHED:
+                switch (VBOXSTRICTRC_VAL(rcLast))
+                {
+                    case VINF_EM_DBG_HYPER_STEPPED:
+                    case VINF_EM_DBG_HYPER_BREAKPOINT:
+                    case VINF_EM_DBG_HYPER_ASSERTION:
+                    case VERR_TRPM_PANIC:
+                    case VERR_TRPM_DONT_PANIC:
+                    case VERR_VMM_RING0_ASSERTION:
+                    case VERR_VMM_HYPER_CR3_MISMATCH:
+                    case VERR_VMM_RING3_CALL_DISABLED:
+                        return rcLast;
+                }
+                return VINF_EM_OFF;
+
+            /*
+             * Status codes terminating the VM in one or another sense.
+             */
+            case VINF_EM_TERMINATE:
+            case VINF_EM_OFF:
+            case VINF_EM_RESET:
+            case VINF_EM_NO_MEMORY:
+            case VINF_EM_RAW_STALE_SELECTOR:
+            case VINF_EM_RAW_IRET_TRAP:
+            case VERR_TRPM_PANIC:
+            case VERR_TRPM_DONT_PANIC:
+            case VERR_IEM_INSTR_NOT_IMPLEMENTED:
+            case VERR_IEM_ASPECT_NOT_IMPLEMENTED:
+            case VERR_VMM_RING0_ASSERTION:
+            case VERR_VMM_HYPER_CR3_MISMATCH:
+            case VERR_VMM_RING3_CALL_DISABLED:
+            case VERR_INTERNAL_ERROR:
+            case VERR_INTERNAL_ERROR_2:
+            case VERR_INTERNAL_ERROR_3:
+            case VERR_INTERNAL_ERROR_4:
+            case VERR_INTERNAL_ERROR_5:
+            case VERR_IPE_UNEXPECTED_STATUS:
+            case VERR_IPE_UNEXPECTED_INFO_STATUS:
+            case VERR_IPE_UNEXPECTED_ERROR_STATUS:
+                return rc;
+
+            /*
+             * The rest is unexpected, and will keep us here.
+             */
+            default:
+                AssertMsgFailed(("Unexpected rc %Rrc!\n", VBOXSTRICTRC_VAL(rc)));
+                break;
+        }
+    } /* debug for ever */
+}
+
+
+#if defined(VBOX_WITH_REM) || defined(DEBUG)
+/**
+ * Steps recompiled code.
+ *
+ * @returns VBox status code. The most important ones are: VINF_EM_STEP_EVENT,
+ *          VINF_EM_RESCHEDULE, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+static int emR3RemStep(PVM pVM, PVMCPU pVCpu)
+{
+    Log3(("emR3RemStep: cs:eip=%04x:%08x\n", CPUMGetGuestCS(pVCpu),  CPUMGetGuestEIP(pVCpu)));
+
+    int rc = VBOXSTRICTRC_TODO(IEMExecOne(pVCpu)); NOREF(pVM);
+
+    Log3(("emR3RemStep: returns %Rrc cs:eip=%04x:%08x\n", rc, CPUMGetGuestCS(pVCpu),  CPUMGetGuestEIP(pVCpu)));
+    return rc;
+}
+#endif /* VBOX_WITH_REM || DEBUG */
+
+
+/**
+ * Executes recompiled code.
+ *
+ * This function contains the recompiler version of the inner
+ * execution loop (the outer loop being in EMR3ExecuteVM()).
+ *
+ * @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE,
+ *          VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pfFFDone    Where to store an indicator telling whether or not
+ *                      FFs were done before returning.
+ *
+ */
+static int emR3RemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
+{
+#ifdef LOG_ENABLED
+    uint32_t cpl = CPUMGetGuestCPL(pVCpu);
+
+    if (pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
+        Log(("EMV86: %04X:%08X IF=%d\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.Bits.u1IF));
+    else
+        Log(("EMR%d: %04X:%08X ESP=%08X IF=%d CR0=%x eflags=%x\n", cpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.eflags.Bits.u1IF, (uint32_t)pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.eflags.u));
+#endif
+    STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatREMTotal, a);
+
+#if defined(VBOX_STRICT) && defined(DEBUG_bird)
+    AssertMsg(   VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
+              || !MMHyperIsInsideArea(pVM, CPUMGetGuestEIP(pVCpu)),  /** @todo @bugref{1419} - get flat address. */
+              ("cs:eip=%RX16:%RX32\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
+#endif
+
+    /*
+     * Spin till we get a forced action which returns anything but VINF_SUCCESS
+     * or the REM suggests raw-mode execution.
+     */
+    *pfFFDone = false;
+    uint32_t cLoops     = 0;
+    int     rc          = VINF_SUCCESS;
+    for (;;)
+    {
+        /*
+         * Execute REM.
+         */
+        if (RT_LIKELY(emR3IsExecutionAllowed(pVM, pVCpu)))
+        {
+            STAM_PROFILE_START(&pVCpu->em.s.StatREMExec, c);
+            rc = VBOXSTRICTRC_TODO(IEMExecLots(pVCpu, 8192 /*cMaxInstructions*/, 4095 /*cPollRate*/, NULL /*pcInstructions*/));
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatREMExec, c);
+        }
+        else
+        {
+            /* Give up this time slice; virtual time continues */
+            STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatCapped, u);
+            RTThreadSleep(5);
+            STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatCapped, u);
+            rc = VINF_SUCCESS;
+        }
+
+        /*
+         * Deal with high priority post execution FFs before doing anything
+         * else.  Sync back the state and leave the lock to be on the safe side.
+         */
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
+            rc = VBOXSTRICTRC_TODO(emR3HighPriorityPostForcedActions(pVM, pVCpu, rc));
+
+        /*
+         * Process the returned status code.
+         */
+        if (rc != VINF_SUCCESS)
+        {
+            if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
+                break;
+            if (rc != VINF_REM_INTERRUPED_FF)
+            {
+                /* Try dodge unimplemented IEM trouble by reschduling. */
+                if (   rc == VERR_IEM_ASPECT_NOT_IMPLEMENTED
+                    || rc == VERR_IEM_INSTR_NOT_IMPLEMENTED)
+                {
+                    EMSTATE enmNewState = emR3Reschedule(pVM, pVCpu);
+                    if (enmNewState != EMSTATE_REM && enmNewState != EMSTATE_IEM_THEN_REM)
+                    {
+                        rc = VINF_EM_RESCHEDULE;
+                        break;
+                    }
+                }
+
+                /*
+                 * Anything which is not known to us means an internal error
+                 * and the termination of the VM!
+                 */
+                AssertMsg(rc == VERR_REM_TOO_MANY_TRAPS, ("Unknown GC return code: %Rra\n", rc));
+                break;
+            }
+        }
+
+
+        /*
+         * Check and execute forced actions.
+         *
+         * Sync back the VM state and leave the lock  before calling any of
+         * these, you never know what's going to happen here.
+         */
+#ifdef VBOX_HIGH_RES_TIMERS_HACK
+        TMTimerPollVoid(pVM, pVCpu);
+#endif
+        AssertCompile(VMCPU_FF_ALL_REM_MASK & VMCPU_FF_TIMER);
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_REM_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_ALL_REM_MASK) )
+        {
+            STAM_REL_PROFILE_ADV_SUSPEND(&pVCpu->em.s.StatREMTotal, a);
+            rc = emR3ForcedActions(pVM, pVCpu, rc);
+            VBOXVMM_EM_FF_ALL_RET(pVCpu, rc);
+            STAM_REL_PROFILE_ADV_RESUME(&pVCpu->em.s.StatREMTotal, a);
+            if (    rc != VINF_SUCCESS
+                &&  rc != VINF_EM_RESCHEDULE_REM)
+            {
+                *pfFFDone = true;
+                break;
+            }
+        }
+
+        /*
+         * Have to check if we can get back to fast execution mode every so often.
+         */
+        if (!(++cLoops & 7))
+        {
+            EMSTATE enmCheck = emR3Reschedule(pVM, pVCpu);
+            if (   enmCheck != EMSTATE_REM
+                && enmCheck != EMSTATE_IEM_THEN_REM)
+                return VINF_EM_RESCHEDULE;
+        }
+
+    } /* The Inner Loop, recompiled execution mode version. */
+
+    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatREMTotal, a);
+    return rc;
+}
+
+
+#ifdef DEBUG
+
+int emR3SingleStepExecRem(PVM pVM, PVMCPU pVCpu, uint32_t cIterations)
+{
+    EMSTATE  enmOldState = pVCpu->em.s.enmState;
+
+    pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_REM;
+
+    Log(("Single step BEGIN:\n"));
+    for (uint32_t i = 0; i < cIterations; i++)
+    {
+        DBGFR3PrgStep(pVCpu);
+        DBGFR3_DISAS_INSTR_CUR_LOG(pVCpu, "RSS");
+        emR3RemStep(pVM, pVCpu);
+        if (emR3Reschedule(pVM, pVCpu) != EMSTATE_REM)
+            break;
+    }
+    Log(("Single step END:\n"));
+    CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
+    pVCpu->em.s.enmState = enmOldState;
+    return VINF_EM_RESCHEDULE;
+}
+
+#endif /* DEBUG */
+
+
+/**
+ * Try execute the problematic code in IEM first, then fall back on REM if there
+ * is too much of it or if IEM doesn't implement something.
+ *
+ * @returns Strict VBox status code from IEMExecLots.
+ * @param   pVM        The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pfFFDone    Force flags done indicator.
+ *
+ * @thread  EMT(pVCpu)
+ */
+static VBOXSTRICTRC emR3ExecuteIemThenRem(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
+{
+    LogFlow(("emR3ExecuteIemThenRem: %04x:%RGv\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestRIP(pVCpu)));
+    *pfFFDone = false;
+
+    /*
+     * Execute in IEM for a while.
+     */
+    while (pVCpu->em.s.cIemThenRemInstructions < 1024)
+    {
+        uint32_t     cInstructions;
+        VBOXSTRICTRC rcStrict = IEMExecLots(pVCpu, 1024 - pVCpu->em.s.cIemThenRemInstructions /*cMaxInstructions*/,
+                                            UINT32_MAX/2 /*cPollRate*/, &cInstructions);
+        pVCpu->em.s.cIemThenRemInstructions += cInstructions;
+        if (rcStrict != VINF_SUCCESS)
+        {
+            if (   rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
+                || rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
+                break;
+
+            Log(("emR3ExecuteIemThenRem: returns %Rrc after %u instructions\n",
+                 VBOXSTRICTRC_VAL(rcStrict), pVCpu->em.s.cIemThenRemInstructions));
+            return rcStrict;
+        }
+
+        EMSTATE enmNewState = emR3Reschedule(pVM, pVCpu);
+        if (enmNewState != EMSTATE_REM && enmNewState != EMSTATE_IEM_THEN_REM)
+        {
+            LogFlow(("emR3ExecuteIemThenRem: -> %d (%s) after %u instructions\n",
+                     enmNewState, emR3GetStateName(enmNewState), pVCpu->em.s.cIemThenRemInstructions));
+            pVCpu->em.s.enmPrevState = pVCpu->em.s.enmState;
+            pVCpu->em.s.enmState     = enmNewState;
+            return VINF_SUCCESS;
+        }
+
+        /*
+         * Check for pending actions.
+         */
+        if (   VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_REM_MASK)
+            || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_ALL_REM_MASK & ~VMCPU_FF_UNHALT))
+            return VINF_SUCCESS;
+    }
+
+    /*
+     * Switch to REM.
+     */
+    Log(("emR3ExecuteIemThenRem: -> EMSTATE_REM (after %u instructions)\n", pVCpu->em.s.cIemThenRemInstructions));
+    pVCpu->em.s.enmState = EMSTATE_REM;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Decides whether to execute RAW, HWACC or REM.
+ *
+ * @returns new EM state
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+EMSTATE emR3Reschedule(PVM pVM, PVMCPU pVCpu)
+{
+    /*
+     * We stay in the wait for SIPI state unless explicitly told otherwise.
+     */
+    if (pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI)
+        return EMSTATE_WAIT_SIPI;
+
+    /*
+     * Execute everything in IEM?
+     */
+    if (pVM->em.s.fIemExecutesAll)
+        return EMSTATE_IEM;
+
+    /* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
+    /* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
+    /* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
+
+    X86EFLAGS EFlags = pVCpu->cpum.GstCtx.eflags;
+    if (!VM_IS_RAW_MODE_ENABLED(pVM))
+    {
+        if (VM_IS_HM_ENABLED(pVM))
+        {
+            if (HMCanExecuteGuest(pVM, pVCpu, &pVCpu->cpum.GstCtx))
+                return EMSTATE_HM;
+        }
+        else if (NEMR3CanExecuteGuest(pVM, pVCpu))
+            return EMSTATE_NEM;
+
+        /*
+         * Note! Raw mode and hw accelerated mode are incompatible. The latter
+         *       turns off monitoring features essential for raw mode!
+         */
+        return EMSTATE_IEM_THEN_REM;
+    }
+
+    /*
+     * Standard raw-mode:
+     *
+     * Here we only support 16 & 32 bits protected mode ring 3 code that has no IO privileges
+     * or 32 bits protected mode ring 0 code
+     *
+     * The tests are ordered by the likelihood of being true during normal execution.
+     */
+    if (EFlags.u32 & (X86_EFL_TF /* | HF_INHIBIT_IRQ_MASK*/))
+    {
+        Log2(("raw mode refused: EFlags=%#x\n", EFlags.u32));
+        return EMSTATE_REM;
+    }
+
+# ifndef VBOX_RAW_V86
+    if (EFlags.u32 & X86_EFL_VM) {
+        Log2(("raw mode refused: VM_MASK\n"));
+        return EMSTATE_REM;
+    }
+# endif
+
+    /** @todo check up the X86_CR0_AM flag in respect to raw mode!!! We're probably not emulating it right! */
+    uint32_t u32CR0 = pVCpu->cpum.GstCtx.cr0;
+    if ((u32CR0 & (X86_CR0_PG | X86_CR0_PE)) != (X86_CR0_PG | X86_CR0_PE))
+    {
+        //Log2(("raw mode refused: %s%s%s\n", (u32CR0 & X86_CR0_PG) ? "" : " !PG", (u32CR0 & X86_CR0_PE) ? "" : " !PE", (u32CR0 & X86_CR0_AM) ? "" : " !AM"));
+        return EMSTATE_REM;
+    }
+
+    if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
+    {
+        uint32_t u32Dummy, u32Features;
+
+        CPUMGetGuestCpuId(pVCpu, 1, 0, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
+        if (!(u32Features & X86_CPUID_FEATURE_EDX_PAE))
+            return EMSTATE_REM;
+    }
+
+    unsigned uSS = pVCpu->cpum.GstCtx.ss.Sel;
+    if (    pVCpu->cpum.GstCtx.eflags.Bits.u1VM
+        ||  (uSS & X86_SEL_RPL) == 3)
+    {
+        if (!(EFlags.u32 & X86_EFL_IF))
+        {
+            Log2(("raw mode refused: IF (RawR3)\n"));
+            return EMSTATE_REM;
+        }
+
+        if (!(u32CR0 & X86_CR0_WP))
+        {
+            Log2(("raw mode refused: CR0.WP + RawR0\n"));
+            return EMSTATE_REM;
+        }
+    }
+    else
+    {
+        /* Only ring 0 supervisor code. */
+        if ((uSS & X86_SEL_RPL) != 0)
+        {
+            Log2(("raw r0 mode refused: CPL %d\n", uSS & X86_SEL_RPL));
+            return EMSTATE_REM;
+        }
+
+        // Let's start with pure 32 bits ring 0 code first
+        /** @todo What's pure 32-bit mode? flat? */
+        if (    !(pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+            ||  !(pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig))
+        {
+            Log2(("raw r0 mode refused: SS/CS not 32bit\n"));
+            return EMSTATE_REM;
+        }
+
+        /* Write protection must be turned on, or else the guest can overwrite our hypervisor code and data. */
+        if (!(u32CR0 & X86_CR0_WP))
+        {
+            Log2(("raw r0 mode refused: CR0.WP=0!\n"));
+            return EMSTATE_REM;
+        }
+
+# if !defined(VBOX_ALLOW_IF0) && !defined(VBOX_RUN_INTERRUPT_GATE_HANDLERS)
+        if (!(EFlags.u32 & X86_EFL_IF))
+        {
+            ////Log2(("R0: IF=0 VIF=%d %08X\n", eip, pVMeflags));
+            //Log2(("RR0: Interrupts turned off; fall back to emulation\n"));
+            return EMSTATE_REM;
+        }
+# endif
+
+# ifndef VBOX_WITH_RAW_RING1
+        /** @todo still necessary??? */
+        if (EFlags.Bits.u2IOPL != 0)
+        {
+            Log2(("raw r0 mode refused: IOPL %d\n", EFlags.Bits.u2IOPL));
+            return EMSTATE_REM;
+        }
+# endif
+    }
+
+    /*
+     * Stale hidden selectors means raw-mode is unsafe (being very careful).
+     */
+    if (pVCpu->cpum.GstCtx.cs.fFlags & CPUMSELREG_FLAGS_STALE)
+    {
+        Log2(("raw mode refused: stale CS\n"));
+        return EMSTATE_REM;
+    }
+    if (pVCpu->cpum.GstCtx.ss.fFlags & CPUMSELREG_FLAGS_STALE)
+    {
+        Log2(("raw mode refused: stale SS\n"));
+        return EMSTATE_REM;
+    }
+    if (pVCpu->cpum.GstCtx.ds.fFlags & CPUMSELREG_FLAGS_STALE)
+    {
+        Log2(("raw mode refused: stale DS\n"));
+        return EMSTATE_REM;
+    }
+    if (pVCpu->cpum.GstCtx.es.fFlags & CPUMSELREG_FLAGS_STALE)
+    {
+        Log2(("raw mode refused: stale ES\n"));
+        return EMSTATE_REM;
+    }
+    if (pVCpu->cpum.GstCtx.fs.fFlags & CPUMSELREG_FLAGS_STALE)
+    {
+        Log2(("raw mode refused: stale FS\n"));
+        return EMSTATE_REM;
+    }
+    if (pVCpu->cpum.GstCtx.gs.fFlags & CPUMSELREG_FLAGS_STALE)
+    {
+        Log2(("raw mode refused: stale GS\n"));
+        return EMSTATE_REM;
+    }
+
+# ifdef VBOX_WITH_SAFE_STR
+    if (pVCpu->cpum.GstCtx.tr.Sel == 0)
+    {
+        Log(("Raw mode refused -> TR=0\n"));
+        return EMSTATE_REM;
+    }
+# endif
+
+    /*Assert(PGMPhysIsA20Enabled(pVCpu));*/
+    return EMSTATE_RAW;
+}
+
+
+/**
+ * Executes all high priority post execution force actions.
+ *
+ * @returns Strict VBox status code.  Typically @a rc, but may be upgraded to
+ *          fatal error status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   rc          The current strict VBox status code rc.
+ */
+VBOXSTRICTRC emR3HighPriorityPostForcedActions(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc)
+{
+    VBOXVMM_EM_FF_HIGH(pVCpu, pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions, VBOXSTRICTRC_VAL(rc));
+
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PDM_CRITSECT))
+        PDMCritSectBothFF(pVCpu);
+
+    /* Update CR3 (Nested Paging case for HM). */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
+    {
+        CPUM_IMPORT_EXTRN_RCSTRICT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER, rc);
+        int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
+        if (RT_FAILURE(rc2))
+            return rc2;
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
+    }
+
+    /* Update PAE PDPEs. This must be done *after* PGMUpdateCR3() and used only by the Nested Paging case for HM. */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
+    {
+        CPUM_IMPORT_EXTRN_RCSTRICT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER, rc);
+        if (CPUMIsGuestInPAEMode(pVCpu))
+        {
+            PX86PDPE pPdpes = HMGetPaePdpes(pVCpu);
+            AssertPtr(pPdpes);
+
+            PGMGstUpdatePaePdpes(pVCpu, pPdpes);
+            Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
+        }
+        else
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
+    }
+
+    /* IEM has pending work (typically memory write after INS instruction). */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
+        rc = IEMR3ProcessForceFlag(pVM, pVCpu, rc);
+
+    /* IOM has pending work (comitting an I/O or MMIO write). */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IOM))
+    {
+        rc = IOMR3ProcessForceFlag(pVM, pVCpu, rc);
+        if (pVCpu->em.s.idxContinueExitRec >= RT_ELEMENTS(pVCpu->em.s.aExitRecords))
+        { /* half likely, or at least it's a line shorter. */ }
+        else if (rc == VINF_SUCCESS)
+            rc = VINF_EM_RESUME_R3_HISTORY_EXEC;
+        else
+            pVCpu->em.s.idxContinueExitRec = UINT16_MAX;
+    }
+
+    if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+    {
+        if (    rc > VINF_EM_NO_MEMORY
+            &&  rc <= VINF_EM_LAST)
+            rc = VINF_EM_NO_MEMORY;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Helper for emR3ForcedActions() for VMX external interrupt VM-exit.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_NO_CHANGE if the VMX external interrupt intercept was not active.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+static int emR3VmxNstGstIntrIntercept(PVMCPU pVCpu)
+{
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /* Handle the "external interrupt" VM-exit intercept. */
+    if (CPUMIsGuestVmxPinCtlsSet(&pVCpu->cpum.GstCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
+    {
+        VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector */, true /* fIntPending */);
+        AssertMsg(   rcStrict != VINF_PGM_CHANGE_MODE
+                  && rcStrict != VINF_VMX_VMEXIT
+                  && rcStrict != VINF_NO_CHANGE, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return VBOXSTRICTRC_TODO(rcStrict);
+    }
+#else
+    RT_NOREF(pVCpu);
+#endif
+    return VINF_NO_CHANGE;
+}
+
+
+/**
+ * Helper for emR3ForcedActions() for SVM interrupt intercept.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_NO_CHANGE if the SVM external interrupt intercept was not active.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+static int emR3SvmNstGstIntrIntercept(PVMCPU pVCpu)
+{
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    /* Handle the physical interrupt intercept (can be masked by the nested hypervisor). */
+    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_INTR))
+    {
+        CPUM_ASSERT_NOT_EXTRN(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK);
+        VBOXSTRICTRC rcStrict = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
+        if (RT_SUCCESS(rcStrict))
+        {
+            AssertMsg(   rcStrict != VINF_PGM_CHANGE_MODE
+                      && rcStrict != VINF_SVM_VMEXIT
+                      && rcStrict != VINF_NO_CHANGE, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            return VBOXSTRICTRC_VAL(rcStrict);
+        }
+
+        AssertMsgFailed(("INTR #VMEXIT failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+        return VINF_EM_TRIPLE_FAULT;
+    }
+#else
+    NOREF(pVCpu);
+#endif
+    return VINF_NO_CHANGE;
+}
+
+
+/**
+ * Helper for emR3ForcedActions() for SVM virtual interrupt intercept.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_NO_CHANGE if the SVM virtual interrupt intercept was not active.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+static int emR3SvmNstGstVirtIntrIntercept(PVMCPU pVCpu)
+{
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_VINTR))
+    {
+        CPUM_ASSERT_NOT_EXTRN(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK);
+        VBOXSTRICTRC rcStrict = IEMExecSvmVmexit(pVCpu, SVM_EXIT_VINTR, 0, 0);
+        if (RT_SUCCESS(rcStrict))
+        {
+            Assert(rcStrict != VINF_PGM_CHANGE_MODE);
+            Assert(rcStrict != VINF_SVM_VMEXIT);
+            return VBOXSTRICTRC_VAL(rcStrict);
+        }
+        AssertMsgFailed(("VINTR #VMEXIT failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+        return VINF_EM_TRIPLE_FAULT;
+    }
+#else
+    NOREF(pVCpu);
+#endif
+    return VINF_NO_CHANGE;
+}
+
+
+/**
+ * Executes all pending forced actions.
+ *
+ * Forced actions can cause execution delays and execution
+ * rescheduling. The first we deal with using action priority, so
+ * that for instance pending timers aren't scheduled and ran until
+ * right before execution. The rescheduling we deal with using
+ * return codes. The same goes for VM termination, only in that case
+ * we exit everything.
+ *
+ * @returns VBox status code of equal or greater importance/severity than rc.
+ *          The most important ones are: VINF_EM_RESCHEDULE,
+ *          VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   rc          The current rc.
+ *
+ */
+int emR3ForcedActions(PVM pVM, PVMCPU pVCpu, int rc)
+{
+    STAM_REL_PROFILE_START(&pVCpu->em.s.StatForcedActions, a);
+#ifdef VBOX_STRICT
+    int rcIrq = VINF_SUCCESS;
+#endif
+    int rc2;
+#define UPDATE_RC() \
+        do { \
+            AssertMsg(rc2 <= 0 || (rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST), ("Invalid FF return code: %Rra\n", rc2)); \
+            if (rc2 == VINF_SUCCESS || rc < VINF_SUCCESS) \
+                break; \
+            if (!rc || rc2 < rc) \
+                rc = rc2; \
+        } while (0)
+    VBOXVMM_EM_FF_ALL(pVCpu, pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions, rc);
+
+    /*
+     * Post execution chunk first.
+     */
+    if (    VM_FF_IS_ANY_SET(pVM, VM_FF_NORMAL_PRIORITY_POST_MASK)
+        ||  (VMCPU_FF_NORMAL_PRIORITY_POST_MASK && VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_NORMAL_PRIORITY_POST_MASK)) )
+    {
+        /*
+         * EMT Rendezvous (must be serviced before termination).
+         */
+        if (VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
+        {
+            CPUM_IMPORT_EXTRN_RCSTRICT(pVCpu, ~CPUMCTX_EXTRN_KEEPER_MASK, rc);
+            rc2 = VMMR3EmtRendezvousFF(pVM, pVCpu);
+            UPDATE_RC();
+            /** @todo HACK ALERT! The following test is to make sure EM+TM
+             * thinks the VM is stopped/reset before the next VM state change
+             * is made. We need a better solution for this, or at least make it
+             * possible to do: (rc >= VINF_EM_FIRST && rc <=
+             * VINF_EM_SUSPEND). */
+            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc;
+            }
+        }
+
+        /*
+         * State change request (cleared by vmR3SetStateLocked).
+         */
+        if (VM_FF_IS_SET(pVM, VM_FF_CHECK_VM_STATE))
+        {
+            VMSTATE enmState = VMR3GetState(pVM);
+            switch (enmState)
+            {
+                case VMSTATE_FATAL_ERROR:
+                case VMSTATE_FATAL_ERROR_LS:
+                case VMSTATE_GURU_MEDITATION:
+                case VMSTATE_GURU_MEDITATION_LS:
+                    Log2(("emR3ForcedActions: %s -> VINF_EM_SUSPEND\n", VMGetStateName(enmState) ));
+                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                    return VINF_EM_SUSPEND;
+
+                case VMSTATE_DESTROYING:
+                    Log2(("emR3ForcedActions: %s -> VINF_EM_TERMINATE\n", VMGetStateName(enmState) ));
+                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                    return VINF_EM_TERMINATE;
+
+                default:
+                    AssertMsgFailed(("%s\n", VMGetStateName(enmState)));
+            }
+        }
+
+        /*
+         * Debugger Facility polling.
+         */
+        if (   VM_FF_IS_SET(pVM, VM_FF_DBGF)
+            || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_DBGF) )
+        {
+            CPUM_IMPORT_EXTRN_RCSTRICT(pVCpu, ~CPUMCTX_EXTRN_KEEPER_MASK, rc);
+            rc2 = DBGFR3VMMForcedAction(pVM, pVCpu);
+            UPDATE_RC();
+        }
+
+        /*
+         * Postponed reset request.
+         */
+        if (VM_FF_TEST_AND_CLEAR(pVM, VM_FF_RESET))
+        {
+            CPUM_IMPORT_EXTRN_RCSTRICT(pVCpu, ~CPUMCTX_EXTRN_KEEPER_MASK, rc);
+            rc2 = VBOXSTRICTRC_TODO(VMR3ResetFF(pVM));
+            UPDATE_RC();
+        }
+
+        /*
+         * Out of memory? Putting this after CSAM as it may in theory cause us to run out of memory.
+         */
+        if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+        {
+            rc2 = PGMR3PhysAllocateHandyPages(pVM);
+            UPDATE_RC();
+            if (rc == VINF_EM_NO_MEMORY)
+                return rc;
+        }
+
+        /* check that we got them all  */
+        AssertCompile(VM_FF_NORMAL_PRIORITY_POST_MASK == (VM_FF_CHECK_VM_STATE | VM_FF_DBGF | VM_FF_RESET | VM_FF_PGM_NO_MEMORY | VM_FF_EMT_RENDEZVOUS));
+        AssertCompile(VMCPU_FF_NORMAL_PRIORITY_POST_MASK == VMCPU_FF_DBGF);
+    }
+
+    /*
+     * Normal priority then.
+     * (Executed in no particular order.)
+     */
+    if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_NORMAL_PRIORITY_MASK, VM_FF_PGM_NO_MEMORY))
+    {
+        /*
+         * PDM Queues are pending.
+         */
+        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PDM_QUEUES, VM_FF_PGM_NO_MEMORY))
+            PDMR3QueueFlushAll(pVM);
+
+        /*
+         * PDM DMA transfers are pending.
+         */
+        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PDM_DMA, VM_FF_PGM_NO_MEMORY))
+            PDMR3DmaRun(pVM);
+
+        /*
+         * EMT Rendezvous (make sure they are handled before the requests).
+         */
+        if (VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
+        {
+            CPUM_IMPORT_EXTRN_RCSTRICT(pVCpu, ~CPUMCTX_EXTRN_KEEPER_MASK, rc);
+            rc2 = VMMR3EmtRendezvousFF(pVM, pVCpu);
+            UPDATE_RC();
+            /** @todo HACK ALERT! The following test is to make sure EM+TM
+             * thinks the VM is stopped/reset before the next VM state change
+             * is made. We need a better solution for this, or at least make it
+             * possible to do: (rc >= VINF_EM_FIRST && rc <=
+             * VINF_EM_SUSPEND). */
+            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc;
+            }
+        }
+
+        /*
+         * Requests from other threads.
+         */
+        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_REQUEST, VM_FF_PGM_NO_MEMORY))
+        {
+            CPUM_IMPORT_EXTRN_RCSTRICT(pVCpu, ~CPUMCTX_EXTRN_KEEPER_MASK, rc);
+            rc2 = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY, false /*fPriorityOnly*/);
+            if (rc2 == VINF_EM_OFF || rc2 == VINF_EM_TERMINATE) /** @todo this shouldn't be necessary */
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc2));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc2;
+            }
+            UPDATE_RC();
+            /** @todo HACK ALERT! The following test is to make sure EM+TM
+             * thinks the VM is stopped/reset before the next VM state change
+             * is made. We need a better solution for this, or at least make it
+             * possible to do: (rc >= VINF_EM_FIRST && rc <=
+             * VINF_EM_SUSPEND). */
+            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc;
+            }
+        }
+
+        /* check that we got them all  */
+        AssertCompile(VM_FF_NORMAL_PRIORITY_MASK == (VM_FF_REQUEST | VM_FF_PDM_QUEUES | VM_FF_PDM_DMA | VM_FF_EMT_RENDEZVOUS));
+    }
+
+    /*
+     * Normal priority then. (per-VCPU)
+     * (Executed in no particular order.)
+     */
+    if (    !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)
+        &&  VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_NORMAL_PRIORITY_MASK))
+    {
+        /*
+         * Requests from other threads.
+         */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
+        {
+            CPUM_IMPORT_EXTRN_RCSTRICT(pVCpu, ~CPUMCTX_EXTRN_KEEPER_MASK, rc);
+            rc2 = VMR3ReqProcessU(pVM->pUVM, pVCpu->idCpu, false /*fPriorityOnly*/);
+            if (rc2 == VINF_EM_OFF || rc2 == VINF_EM_TERMINATE || rc2 == VINF_EM_RESET)
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc2));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc2;
+            }
+            UPDATE_RC();
+            /** @todo HACK ALERT! The following test is to make sure EM+TM
+             * thinks the VM is stopped/reset before the next VM state change
+             * is made. We need a better solution for this, or at least make it
+             * possible to do: (rc >= VINF_EM_FIRST && rc <=
+             * VINF_EM_SUSPEND). */
+            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc;
+            }
+        }
+
+        /* check that we got them all  */
+        Assert(!(VMCPU_FF_NORMAL_PRIORITY_MASK & ~VMCPU_FF_REQUEST));
+    }
+
+    /*
+     * High priority pre execution chunk last.
+     * (Executed in ascending priority order.)
+     */
+    if (    VM_FF_IS_ANY_SET(pVM, VM_FF_HIGH_PRIORITY_PRE_MASK)
+        ||  VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_MASK))
+    {
+        /*
+         * Timers before interrupts.
+         */
+        if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TIMER)
+            && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+            TMR3TimerQueuesDo(pVM);
+
+        /*
+         * Pick up asynchronously posted interrupts into the APIC.
+         */
+        if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
+            APICUpdatePendingInterrupts(pVCpu);
+
+        /*
+         * The instruction following an emulated STI should *always* be executed!
+         *
+         * Note! We intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here if
+         *       the eip is the same as the inhibited instr address.  Before we
+         *       are able to execute this instruction in raw mode (iret to
+         *       guest code) an external interrupt might force a world switch
+         *       again.  Possibly allowing a guest interrupt to be dispatched
+         *       in the process.  This could break the guest.  Sounds very
+         *       unlikely, but such timing sensitive problem are not as rare as
+         *       you might think.
+         */
+        if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+            && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+        {
+            CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP);
+            if (CPUMGetGuestRIP(pVCpu) != EMGetInhibitInterruptsPC(pVCpu))
+            {
+                Log(("Clearing VMCPU_FF_INHIBIT_INTERRUPTS at %RGv - successor %RGv\n", (RTGCPTR)CPUMGetGuestRIP(pVCpu), EMGetInhibitInterruptsPC(pVCpu)));
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+            }
+            else
+                Log(("Leaving VMCPU_FF_INHIBIT_INTERRUPTS set at %RGv\n", (RTGCPTR)CPUMGetGuestRIP(pVCpu)));
+        }
+
+        /** @todo SMIs. If we implement SMIs, this is where they will have to be
+         *        delivered. */
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER))
+        {
+            /*
+             * VMX Nested-guest APIC-write pending (can cause VM-exits).
+             * Takes priority over even SMI and INIT signals.
+             * See Intel spec. 29.4.3.2 "APIC-Write Emulation".
+             */
+            if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
+            {
+                rc2 = VBOXSTRICTRC_VAL(IEMExecVmxVmexitApicWrite(pVCpu));
+                if (rc2 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+                    UPDATE_RC();
+            }
+
+            /*
+             * VMX Nested-guest monitor-trap flag (MTF) VM-exit.
+             * Takes priority over "Traps on the previous instruction".
+             * See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts".
+             */
+            if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF))
+            {
+                rc2 = VBOXSTRICTRC_VAL(IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */));
+                Assert(rc2 != VINF_VMX_INTERCEPT_NOT_ACTIVE);
+                UPDATE_RC();
+            }
+
+            /*
+             * VMX Nested-guest preemption timer VM-exit.
+             * Takes priority over NMI-window VM-exits.
+             */
+            if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
+            {
+                rc2 = VBOXSTRICTRC_VAL(IEMExecVmxVmexitPreemptTimer(pVCpu));
+                Assert(rc2 != VINF_VMX_INTERCEPT_NOT_ACTIVE);
+                UPDATE_RC();
+            }
+        }
+#endif
+
+        /*
+         * Guest event injection.
+         */
+        bool fWakeupPending = false;
+        if (   !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)
+            && (!rc || rc >= VINF_EM_RESCHEDULE_HM)
+            && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)  /* Interrupt shadows block both NMIs and interrupts. */
+            && !TRPMHasTrap(pVCpu))                                  /* An event could already be scheduled for dispatching. */
+        {
+            bool fInVmxNonRootMode;
+            bool fInSvmHwvirtMode;
+            bool const fInNestedGuest = CPUMIsGuestInNestedHwvirtMode(&pVCpu->cpum.GstCtx);
+            if (fInNestedGuest)
+            {
+                fInVmxNonRootMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
+                fInSvmHwvirtMode  = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
+            }
+            else
+            {
+                fInVmxNonRootMode = false;
+                fInSvmHwvirtMode  = false;
+            }
+
+            bool fGif = CPUMGetGuestGif(&pVCpu->cpum.GstCtx);
+            if (fGif)
+            {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                /*
+                 * VMX NMI-window VM-exit.
+                 * Takes priority over non-maskable interrupts (NMIs).
+                 * Interrupt shadows block NMI-window VM-exits.
+                 * Any event that is already in TRPM (e.g. injected during VM-entry) takes priority.
+                 *
+                 * See Intel spec. 25.2 "Other Causes Of VM Exits".
+                 * See Intel spec. 26.7.6 "NMI-Window Exiting".
+                 */
+                if (    VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)
+                    && !CPUMIsGuestVmxVirtNmiBlocking(&pVCpu->cpum.GstCtx))
+                {
+                    Assert(CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT));
+                    Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
+                    rc2 = VBOXSTRICTRC_VAL(IEMExecVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* uExitQual */));
+                    AssertMsg(   rc2 != VINF_VMX_INTERCEPT_NOT_ACTIVE
+                              && rc2 != VINF_PGM_CHANGE_MODE
+                              && rc2 != VINF_VMX_VMEXIT
+                              && rc2 != VINF_NO_CHANGE, ("%Rrc\n", rc2));
+                    UPDATE_RC();
+                }
+                else
+#endif
+                /*
+                 * NMIs (take priority over external interrupts).
+                 */
+                if (    VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
+                    && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+                {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                    if (   fInVmxNonRootMode
+                        && CPUMIsGuestVmxPinCtlsSet(&pVCpu->cpum.GstCtx, VMX_PIN_CTLS_NMI_EXIT))
+                    {
+                        rc2 = VBOXSTRICTRC_VAL(IEMExecVmxVmexitXcptNmi(pVCpu));
+                        Assert(rc2 != VINF_VMX_INTERCEPT_NOT_ACTIVE);
+                        UPDATE_RC();
+                    }
+                    else
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+                    if (   fInSvmHwvirtMode
+                        && CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_NMI))
+                    {
+                        rc2 = VBOXSTRICTRC_VAL(IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0 /* uExitInfo1 */,  0 /* uExitInfo2 */));
+                        AssertMsg(   rc2 != VINF_PGM_CHANGE_MODE
+                                  && rc2 != VINF_SVM_VMEXIT
+                                  && rc2 != VINF_NO_CHANGE, ("%Rrc\n", rc2));
+                        UPDATE_RC();
+                    }
+                    else
+#endif
+                    {
+                        rc2 = TRPMAssertTrap(pVCpu, X86_XCPT_NMI, TRPM_TRAP);
+                        if (rc2 == VINF_SUCCESS)
+                        {
+                            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+                            fWakeupPending = true;
+                            if (pVM->em.s.fIemExecutesAll)
+                                rc2 = VINF_EM_RESCHEDULE;
+                            else
+                            {
+                                rc2 = HMR3IsActive(pVCpu)    ? VINF_EM_RESCHEDULE_HM
+                                    : VM_IS_NEM_ENABLED(pVM) ? VINF_EM_RESCHEDULE
+                                    :                          VINF_EM_RESCHEDULE_REM;
+                            }
+                        }
+                        UPDATE_RC();
+                    }
+                }
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                /*
+                 * VMX Interrupt-window VM-exits.
+                 * Takes priority over external interrupts.
+                 */
+                else if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)
+                         && CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx))
+                {
+                    Assert(CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT));
+                    Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
+                    rc2 = VBOXSTRICTRC_VAL(IEMExecVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* uExitQual */));
+                    AssertMsg(   rc2 != VINF_VMX_INTERCEPT_NOT_ACTIVE
+                              && rc2 != VINF_PGM_CHANGE_MODE
+                              && rc2 != VINF_VMX_VMEXIT
+                              && rc2 != VINF_NO_CHANGE, ("%Rrc\n", rc2));
+                    UPDATE_RC();
+                }
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+                /** @todo NSTSVM: Handle this for SVM here too later not when an interrupt is
+                 *        actually pending like we currently do. */
+#endif
+                /*
+                 * External interrupts.
+                 */
+                else
+                {
+                    /*
+                     * VMX: virtual interrupts takes priority over physical interrupts.
+                     * SVM: physical interrupts takes priority over virtual interrupts.
+                     */
+                    if (   fInVmxNonRootMode
+                        && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST)
+                        && CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx))
+                    {
+                        /** @todo NSTVMX: virtual-interrupt delivery. */
+                        rc2 = VINF_SUCCESS;
+                    }
+                    else if (   VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
+                             && CPUMIsGuestPhysIntrEnabled(pVCpu))
+                    {
+                        Assert(pVCpu->em.s.enmState != EMSTATE_WAIT_SIPI);
+                        if (fInVmxNonRootMode)
+                            rc2 = emR3VmxNstGstIntrIntercept(pVCpu);
+                        else if (fInSvmHwvirtMode)
+                            rc2 = emR3SvmNstGstIntrIntercept(pVCpu);
+                        else
+                            rc2 = VINF_NO_CHANGE;
+
+                        if (rc2 == VINF_NO_CHANGE)
+                        {
+                            bool fInjected = false;
+                            CPUM_IMPORT_EXTRN_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+                            /** @todo this really isn't nice, should properly handle this */
+                            /* Note! This can still cause a VM-exit (on Intel). */
+                            rc2 = TRPMR3InjectEvent(pVM, pVCpu, TRPM_HARDWARE_INT, &fInjected);
+                            fWakeupPending = true;
+                            if (   pVM->em.s.fIemExecutesAll
+                                && (   rc2 == VINF_EM_RESCHEDULE_REM
+                                    || rc2 == VINF_EM_RESCHEDULE_HM
+                                    || rc2 == VINF_EM_RESCHEDULE_RAW))
+                            {
+                                rc2 = VINF_EM_RESCHEDULE;
+                            }
+#ifdef VBOX_STRICT
+                            if (fInjected)
+                                rcIrq = rc2;
+#endif
+                        }
+                        UPDATE_RC();
+                    }
+                    else if (   fInSvmHwvirtMode
+                             && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST)
+                             && CPUMIsGuestSvmVirtIntrEnabled(pVCpu,  &pVCpu->cpum.GstCtx))
+                    {
+                        rc2 = emR3SvmNstGstVirtIntrIntercept(pVCpu);
+                        if (rc2 == VINF_NO_CHANGE)
+                        {
+                            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
+                            uint8_t const uNstGstVector = CPUMGetGuestSvmVirtIntrVector(&pVCpu->cpum.GstCtx);
+                            AssertMsg(uNstGstVector > 0 && uNstGstVector <= X86_XCPT_LAST, ("Invalid VINTR %#x\n", uNstGstVector));
+                            TRPMAssertTrap(pVCpu, uNstGstVector, TRPM_HARDWARE_INT);
+                            Log(("EM: Asserting nested-guest virt. hardware intr: %#x\n", uNstGstVector));
+                            rc2 = VINF_EM_RESCHEDULE;
+#ifdef VBOX_STRICT
+                            rcIrq = rc2;
+#endif
+                        }
+                        UPDATE_RC();
+                    }
+                }
+            }
+        }
+
+        /*
+         * Allocate handy pages.
+         */
+        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PGM_NEED_HANDY_PAGES, VM_FF_PGM_NO_MEMORY))
+        {
+            rc2 = PGMR3PhysAllocateHandyPages(pVM);
+            UPDATE_RC();
+        }
+
+        /*
+         * Debugger Facility request.
+         */
+        if (   (   VM_FF_IS_SET(pVM, VM_FF_DBGF)
+                || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_DBGF) )
+            && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY) )
+        {
+            CPUM_IMPORT_EXTRN_RCSTRICT(pVCpu, ~CPUMCTX_EXTRN_KEEPER_MASK, rc);
+            rc2 = DBGFR3VMMForcedAction(pVM, pVCpu);
+            UPDATE_RC();
+        }
+
+        /*
+         * EMT Rendezvous (must be serviced before termination).
+         */
+        if (   !fWakeupPending /* don't miss the wakeup from EMSTATE_HALTED! */
+            && VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
+        {
+            CPUM_IMPORT_EXTRN_RCSTRICT(pVCpu, ~CPUMCTX_EXTRN_KEEPER_MASK, rc);
+            rc2 = VMMR3EmtRendezvousFF(pVM, pVCpu);
+            UPDATE_RC();
+            /** @todo HACK ALERT! The following test is to make sure EM+TM thinks the VM is
+             * stopped/reset before the next VM state change is made. We need a better
+             * solution for this, or at least make it possible to do: (rc >= VINF_EM_FIRST
+             * && rc >= VINF_EM_SUSPEND). */
+            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc;
+            }
+        }
+
+        /*
+         * State change request (cleared by vmR3SetStateLocked).
+         */
+        if (   !fWakeupPending /* don't miss the wakeup from EMSTATE_HALTED! */
+            && VM_FF_IS_SET(pVM, VM_FF_CHECK_VM_STATE))
+        {
+            VMSTATE enmState = VMR3GetState(pVM);
+            switch (enmState)
+            {
+                case VMSTATE_FATAL_ERROR:
+                case VMSTATE_FATAL_ERROR_LS:
+                case VMSTATE_GURU_MEDITATION:
+                case VMSTATE_GURU_MEDITATION_LS:
+                    Log2(("emR3ForcedActions: %s -> VINF_EM_SUSPEND\n", VMGetStateName(enmState) ));
+                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                    return VINF_EM_SUSPEND;
+
+                case VMSTATE_DESTROYING:
+                    Log2(("emR3ForcedActions: %s -> VINF_EM_TERMINATE\n", VMGetStateName(enmState) ));
+                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                    return VINF_EM_TERMINATE;
+
+                default:
+                    AssertMsgFailed(("%s\n", VMGetStateName(enmState)));
+            }
+        }
+
+        /*
+         * Out of memory? Since most of our fellow high priority actions may cause us
+         * to run out of memory, we're employing VM_FF_IS_PENDING_EXCEPT and putting this
+         * at the end rather than the start. Also, VM_FF_TERMINATE has higher priority
+         * than us since we can terminate without allocating more memory.
+         */
+        if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+        {
+            rc2 = PGMR3PhysAllocateHandyPages(pVM);
+            UPDATE_RC();
+            if (rc == VINF_EM_NO_MEMORY)
+                return rc;
+        }
+
+        /*
+         * If the virtual sync clock is still stopped, make TM restart it.
+         */
+        if (VM_FF_IS_SET(pVM, VM_FF_TM_VIRTUAL_SYNC))
+            TMR3VirtualSyncFF(pVM, pVCpu);
+
+#ifdef DEBUG
+        /*
+         * Debug, pause the VM.
+         */
+        if (VM_FF_IS_SET(pVM, VM_FF_DEBUG_SUSPEND))
+        {
+            VM_FF_CLEAR(pVM, VM_FF_DEBUG_SUSPEND);
+            Log(("emR3ForcedActions: returns VINF_EM_SUSPEND\n"));
+            return VINF_EM_SUSPEND;
+        }
+#endif
+
+        /* check that we got them all  */
+        AssertCompile(VM_FF_HIGH_PRIORITY_PRE_MASK == (VM_FF_TM_VIRTUAL_SYNC | VM_FF_DBGF | VM_FF_CHECK_VM_STATE | VM_FF_DEBUG_SUSPEND | VM_FF_PGM_NEED_HANDY_PAGES | VM_FF_PGM_NO_MEMORY | VM_FF_EMT_RENDEZVOUS));
+        AssertCompile(VMCPU_FF_HIGH_PRIORITY_PRE_MASK == (VMCPU_FF_TIMER | VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_UPDATE_APIC | VMCPU_FF_INTERRUPT_PIC | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_INHIBIT_INTERRUPTS | VMCPU_FF_DBGF | VMCPU_FF_INTERRUPT_NESTED_GUEST | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_PREEMPT_TIMER | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW));
+    }
+
+#undef UPDATE_RC
+    Log2(("emR3ForcedActions: returns %Rrc\n", rc));
+    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+    Assert(rcIrq == VINF_SUCCESS || rcIrq == rc);
+    return rc;
+}
+
+
+/**
+ * Check if the preset execution time cap restricts guest execution scheduling.
+ *
+ * @returns true if allowed, false otherwise
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+bool emR3IsExecutionAllowed(PVM pVM, PVMCPU pVCpu)
+{
+    uint64_t u64UserTime, u64KernelTime;
+
+    if (    pVM->uCpuExecutionCap != 100
+        &&  RT_SUCCESS(RTThreadGetExecutionTimeMilli(&u64KernelTime, &u64UserTime)))
+    {
+        uint64_t u64TimeNow = RTTimeMilliTS();
+        if (pVCpu->em.s.u64TimeSliceStart + EM_TIME_SLICE < u64TimeNow)
+        {
+            /* New time slice. */
+            pVCpu->em.s.u64TimeSliceStart     = u64TimeNow;
+            pVCpu->em.s.u64TimeSliceStartExec = u64KernelTime + u64UserTime;
+            pVCpu->em.s.u64TimeSliceExec      = 0;
+        }
+        pVCpu->em.s.u64TimeSliceExec = u64KernelTime + u64UserTime - pVCpu->em.s.u64TimeSliceStartExec;
+
+        Log2(("emR3IsExecutionAllowed: start=%RX64 startexec=%RX64 exec=%RX64 (cap=%x)\n", pVCpu->em.s.u64TimeSliceStart, pVCpu->em.s.u64TimeSliceStartExec, pVCpu->em.s.u64TimeSliceExec, (EM_TIME_SLICE * pVM->uCpuExecutionCap) / 100));
+        if (pVCpu->em.s.u64TimeSliceExec >= (EM_TIME_SLICE * pVM->uCpuExecutionCap) / 100)
+            return false;
+    }
+    return true;
+}
+
+
+/**
+ * Execute VM.
+ *
+ * This function is the main loop of the VM. The emulation thread
+ * calls this function when the VM has been successfully constructed
+ * and we're ready for executing the VM.
+ *
+ * Returning from this function means that the VM is turned off or
+ * suspended (state already saved) and deconstruction is next in line.
+ *
+ * All interaction from other thread are done using forced actions
+ * and signalling of the wait object.
+ *
+ * @returns VBox status code, informational status codes may indicate failure.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMR3_INT_DECL(int) EMR3ExecuteVM(PVM pVM, PVMCPU pVCpu)
+{
+    Log(("EMR3ExecuteVM: pVM=%p enmVMState=%d (%s)  enmState=%d (%s) enmPrevState=%d (%s)\n",
+         pVM,
+         pVM->enmVMState,          VMR3GetStateName(pVM->enmVMState),
+         pVCpu->em.s.enmState,     emR3GetStateName(pVCpu->em.s.enmState),
+         pVCpu->em.s.enmPrevState, emR3GetStateName(pVCpu->em.s.enmPrevState) ));
+    VM_ASSERT_EMT(pVM);
+    AssertMsg(   pVCpu->em.s.enmState == EMSTATE_NONE
+              || pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI
+              || pVCpu->em.s.enmState == EMSTATE_SUSPENDED,
+              ("%s\n", emR3GetStateName(pVCpu->em.s.enmState)));
+
+    int rc = setjmp(pVCpu->em.s.u.FatalLongJump);
+    if (rc == 0)
+    {
+        /*
+         * Start the virtual time.
+         */
+        TMR3NotifyResume(pVM, pVCpu);
+
+        /*
+         * The Outer Main Loop.
+         */
+        bool fFFDone = false;
+
+        /* Reschedule right away to start in the right state. */
+        rc = VINF_SUCCESS;
+
+        /* If resuming after a pause or a state load, restore the previous
+           state or else we'll start executing code. Else, just reschedule. */
+        if (    pVCpu->em.s.enmState == EMSTATE_SUSPENDED
+            &&  (   pVCpu->em.s.enmPrevState == EMSTATE_WAIT_SIPI
+                 || pVCpu->em.s.enmPrevState == EMSTATE_HALTED))
+            pVCpu->em.s.enmState = pVCpu->em.s.enmPrevState;
+        else
+            pVCpu->em.s.enmState = emR3Reschedule(pVM, pVCpu);
+        pVCpu->em.s.cIemThenRemInstructions = 0;
+        Log(("EMR3ExecuteVM: enmState=%s\n", emR3GetStateName(pVCpu->em.s.enmState)));
+
+        STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
+        for (;;)
+        {
+            /*
+             * Before we can schedule anything (we're here because
+             * scheduling is required) we must service any pending
+             * forced actions to avoid any pending action causing
+             * immediate rescheduling upon entering an inner loop
+             *
+             * Do forced actions.
+             */
+            if (   !fFFDone
+                && RT_SUCCESS(rc)
+                && rc != VINF_EM_TERMINATE
+                && rc != VINF_EM_OFF
+                && (   VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_REM_MASK)
+                    || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_ALL_REM_MASK & ~VMCPU_FF_UNHALT)))
+            {
+                rc = emR3ForcedActions(pVM, pVCpu, rc);
+                VBOXVMM_EM_FF_ALL_RET(pVCpu, rc);
+            }
+            else if (fFFDone)
+                fFFDone = false;
+
+            /*
+             * Now what to do?
+             */
+            Log2(("EMR3ExecuteVM: rc=%Rrc\n", rc));
+            EMSTATE const enmOldState = pVCpu->em.s.enmState;
+            switch (rc)
+            {
+                /*
+                 * Keep doing what we're currently doing.
+                 */
+                case VINF_SUCCESS:
+                    break;
+
+                /*
+                 * Reschedule - to raw-mode execution.
+                 */
+/** @todo r=bird: consider merging VINF_EM_RESCHEDULE_RAW with VINF_EM_RESCHEDULE_HM, they serve the same purpose here at least. */
+                case VINF_EM_RESCHEDULE_RAW:
+                    Assert(!pVM->em.s.fIemExecutesAll || pVCpu->em.s.enmState != EMSTATE_IEM);
+                    if (VM_IS_RAW_MODE_ENABLED(pVM))
+                    {
+                        Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_RAW: %d -> %d (EMSTATE_RAW)\n", enmOldState, EMSTATE_RAW));
+                        pVCpu->em.s.enmState = EMSTATE_RAW;
+                    }
+                    else
+                    {
+                        AssertLogRelFailed();
+                        pVCpu->em.s.enmState = EMSTATE_NONE;
+                    }
+                    break;
+
+                /*
+                 * Reschedule - to HM or NEM.
+                 */
+                case VINF_EM_RESCHEDULE_HM:
+                    Assert(!pVM->em.s.fIemExecutesAll || pVCpu->em.s.enmState != EMSTATE_IEM);
+                    if (VM_IS_HM_ENABLED(pVM))
+                    {
+                        Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_HM: %d -> %d (EMSTATE_HM)\n", enmOldState, EMSTATE_HM));
+                        pVCpu->em.s.enmState = EMSTATE_HM;
+                    }
+                    else if (VM_IS_NEM_ENABLED(pVM))
+                    {
+                        Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_HM: %d -> %d (EMSTATE_NEM)\n", enmOldState, EMSTATE_NEM));
+                        pVCpu->em.s.enmState = EMSTATE_NEM;
+                    }
+                    else
+                    {
+                        AssertLogRelFailed();
+                        pVCpu->em.s.enmState = EMSTATE_NONE;
+                    }
+                    break;
+
+                /*
+                 * Reschedule - to recompiled execution.
+                 */
+                case VINF_EM_RESCHEDULE_REM:
+                    Assert(!pVM->em.s.fIemExecutesAll || pVCpu->em.s.enmState != EMSTATE_IEM);
+                    if (!VM_IS_RAW_MODE_ENABLED(pVM))
+                    {
+                        Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_REM: %d -> %d (EMSTATE_IEM_THEN_REM)\n",
+                              enmOldState, EMSTATE_IEM_THEN_REM));
+                        if (pVCpu->em.s.enmState != EMSTATE_IEM_THEN_REM)
+                        {
+                            pVCpu->em.s.enmState = EMSTATE_IEM_THEN_REM;
+                            pVCpu->em.s.cIemThenRemInstructions = 0;
+                        }
+                    }
+                    else
+                    {
+                        Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_REM: %d -> %d (EMSTATE_REM)\n", enmOldState, EMSTATE_REM));
+                        pVCpu->em.s.enmState = EMSTATE_REM;
+                    }
+                    break;
+
+                /*
+                 * Resume.
+                 */
+                case VINF_EM_RESUME:
+                    Log2(("EMR3ExecuteVM: VINF_EM_RESUME: %d -> VINF_EM_RESCHEDULE\n", enmOldState));
+                    /* Don't reschedule in the halted or wait for SIPI case. */
+                    if (    pVCpu->em.s.enmPrevState == EMSTATE_WAIT_SIPI
+                        ||  pVCpu->em.s.enmPrevState == EMSTATE_HALTED)
+                    {
+                        pVCpu->em.s.enmState = pVCpu->em.s.enmPrevState;
+                        break;
+                    }
+                    /* fall through and get scheduled. */
+                    RT_FALL_THRU();
+
+                /*
+                 * Reschedule.
+                 */
+                case VINF_EM_RESCHEDULE:
+                {
+                    EMSTATE enmState = emR3Reschedule(pVM, pVCpu);
+                    Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE: %d -> %d (%s)\n", enmOldState, enmState, emR3GetStateName(enmState)));
+                    if (pVCpu->em.s.enmState != enmState && enmState == EMSTATE_IEM_THEN_REM)
+                        pVCpu->em.s.cIemThenRemInstructions = 0;
+                    pVCpu->em.s.enmState = enmState;
+                    break;
+                }
+
+                /*
+                 * Halted.
+                 */
+                case VINF_EM_HALT:
+                    Log2(("EMR3ExecuteVM: VINF_EM_HALT: %d -> %d\n", enmOldState, EMSTATE_HALTED));
+                    pVCpu->em.s.enmState = EMSTATE_HALTED;
+                    break;
+
+                /*
+                 * Switch to the wait for SIPI state (application processor only)
+                 */
+                case VINF_EM_WAIT_SIPI:
+                    Assert(pVCpu->idCpu != 0);
+                    Log2(("EMR3ExecuteVM: VINF_EM_WAIT_SIPI: %d -> %d\n", enmOldState, EMSTATE_WAIT_SIPI));
+                    pVCpu->em.s.enmState = EMSTATE_WAIT_SIPI;
+                    break;
+
+
+                /*
+                 * Suspend.
+                 */
+                case VINF_EM_SUSPEND:
+                    Log2(("EMR3ExecuteVM: VINF_EM_SUSPEND: %d -> %d\n", enmOldState, EMSTATE_SUSPENDED));
+                    Assert(enmOldState != EMSTATE_SUSPENDED);
+                    pVCpu->em.s.enmPrevState = enmOldState;
+                    pVCpu->em.s.enmState     = EMSTATE_SUSPENDED;
+                    break;
+
+                /*
+                 * Reset.
+                 * We might end up doing a double reset for now, we'll have to clean up the mess later.
+                 */
+                case VINF_EM_RESET:
+                {
+                    if (pVCpu->idCpu == 0)
+                    {
+                        EMSTATE enmState = emR3Reschedule(pVM, pVCpu);
+                        Log2(("EMR3ExecuteVM: VINF_EM_RESET: %d -> %d (%s)\n", enmOldState, enmState, emR3GetStateName(enmState)));
+                        if (pVCpu->em.s.enmState != enmState && enmState == EMSTATE_IEM_THEN_REM)
+                            pVCpu->em.s.cIemThenRemInstructions = 0;
+                        pVCpu->em.s.enmState = enmState;
+                    }
+                    else
+                    {
+                        /* All other VCPUs go into the wait for SIPI state. */
+                        pVCpu->em.s.enmState = EMSTATE_WAIT_SIPI;
+                    }
+                    break;
+                }
+
+                /*
+                 * Power Off.
+                 */
+                case VINF_EM_OFF:
+                    pVCpu->em.s.enmState = EMSTATE_TERMINATING;
+                    Log2(("EMR3ExecuteVM: returns VINF_EM_OFF (%d -> %d)\n", enmOldState, EMSTATE_TERMINATING));
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+                    return rc;
+
+                /*
+                 * Terminate the VM.
+                 */
+                case VINF_EM_TERMINATE:
+                    pVCpu->em.s.enmState = EMSTATE_TERMINATING;
+                    Log(("EMR3ExecuteVM returns VINF_EM_TERMINATE (%d -> %d)\n", enmOldState, EMSTATE_TERMINATING));
+                    if (pVM->enmVMState < VMSTATE_DESTROYING) /* ugly */
+                        TMR3NotifySuspend(pVM, pVCpu);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+                    return rc;
+
+
+                /*
+                 * Out of memory, suspend the VM and stuff.
+                 */
+                case VINF_EM_NO_MEMORY:
+                    Log2(("EMR3ExecuteVM: VINF_EM_NO_MEMORY: %d -> %d\n", enmOldState, EMSTATE_SUSPENDED));
+                    Assert(enmOldState != EMSTATE_SUSPENDED);
+                    pVCpu->em.s.enmPrevState = enmOldState;
+                    pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+
+                    rc = VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_SUSPEND, "HostMemoryLow",
+                                           N_("Unable to allocate and lock memory. The virtual machine will be paused. Please close applications to free up memory or close the VM"));
+                    if (rc != VINF_EM_SUSPEND)
+                    {
+                        if (RT_SUCCESS_NP(rc))
+                        {
+                            AssertLogRelMsgFailed(("%Rrc\n", rc));
+                            rc = VERR_EM_INTERNAL_ERROR;
+                        }
+                        pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
+                    }
+                    return rc;
+
+                /*
+                 * Guest debug events.
+                 */
+                case VINF_EM_DBG_STEPPED:
+                case VINF_EM_DBG_STOP:
+                case VINF_EM_DBG_EVENT:
+                case VINF_EM_DBG_BREAKPOINT:
+                case VINF_EM_DBG_STEP:
+                    if (enmOldState == EMSTATE_RAW)
+                    {
+                        Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_RAW));
+                        pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_RAW;
+                    }
+                    else if (enmOldState == EMSTATE_HM)
+                    {
+                        Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_HM));
+                        pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_HM;
+                    }
+                    else if (enmOldState == EMSTATE_NEM)
+                    {
+                        Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_NEM));
+                        pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_NEM;
+                    }
+                    else if (enmOldState == EMSTATE_REM)
+                    {
+                        Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_REM));
+                        pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_REM;
+                    }
+                    else
+                    {
+                        Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_IEM));
+                        pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_IEM;
+                    }
+                    break;
+
+                /*
+                 * Hypervisor debug events.
+                 */
+                case VINF_EM_DBG_HYPER_STEPPED:
+                case VINF_EM_DBG_HYPER_BREAKPOINT:
+                case VINF_EM_DBG_HYPER_ASSERTION:
+                    Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_HYPER));
+                    pVCpu->em.s.enmState = EMSTATE_DEBUG_HYPER;
+                    break;
+
+                /*
+                 * Triple fault.
+                 */
+                case VINF_EM_TRIPLE_FAULT:
+                    if (!pVM->em.s.fGuruOnTripleFault)
+                    {
+                        Log(("EMR3ExecuteVM: VINF_EM_TRIPLE_FAULT: CPU reset...\n"));
+                        rc = VBOXSTRICTRC_TODO(VMR3ResetTripleFault(pVM));
+                        Log2(("EMR3ExecuteVM: VINF_EM_TRIPLE_FAULT: %d -> %d (rc=%Rrc)\n", enmOldState, pVCpu->em.s.enmState, rc));
+                        continue;
+                    }
+                    /* Else fall through and trigger a guru. */
+                    RT_FALL_THRU();
+
+                case VERR_VMM_RING0_ASSERTION:
+                    Log(("EMR3ExecuteVM: %Rrc: %d -> %d (EMSTATE_GURU_MEDITATION)\n", rc, enmOldState, EMSTATE_GURU_MEDITATION));
+                    pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
+                    break;
+
+                /*
+                 * Any error code showing up here other than the ones we
+                 * know and process above are considered to be FATAL.
+                 *
+                 * Unknown warnings and informational status codes are also
+                 * included in this.
+                 */
+                default:
+                    if (RT_SUCCESS_NP(rc))
+                    {
+                        AssertMsgFailed(("Unexpected warning or informational status code %Rra!\n", rc));
+                        rc = VERR_EM_INTERNAL_ERROR;
+                    }
+                    Log(("EMR3ExecuteVM: %Rrc: %d -> %d (EMSTATE_GURU_MEDITATION)\n", rc, enmOldState, EMSTATE_GURU_MEDITATION));
+                    pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
+                    break;
+            }
+
+            /*
+             * Act on state transition.
+             */
+            EMSTATE const enmNewState = pVCpu->em.s.enmState;
+            if (enmOldState != enmNewState)
+            {
+                VBOXVMM_EM_STATE_CHANGED(pVCpu, enmOldState, enmNewState, rc);
+
+                /* Clear MWait flags and the unhalt FF. */
+                if (   enmOldState == EMSTATE_HALTED
+                    && (   (pVCpu->em.s.MWait.fWait & EMMWAIT_FLAG_ACTIVE)
+                        || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_UNHALT))
+                    && (   enmNewState == EMSTATE_RAW
+                        || enmNewState == EMSTATE_HM
+                        || enmNewState == EMSTATE_NEM
+                        || enmNewState == EMSTATE_REM
+                        || enmNewState == EMSTATE_IEM_THEN_REM
+                        || enmNewState == EMSTATE_DEBUG_GUEST_RAW
+                        || enmNewState == EMSTATE_DEBUG_GUEST_HM
+                        || enmNewState == EMSTATE_DEBUG_GUEST_NEM
+                        || enmNewState == EMSTATE_DEBUG_GUEST_IEM
+                        || enmNewState == EMSTATE_DEBUG_GUEST_REM) )
+                {
+                    if (pVCpu->em.s.MWait.fWait & EMMWAIT_FLAG_ACTIVE)
+                    {
+                        LogFlow(("EMR3ExecuteVM: Clearing MWAIT\n"));
+                        pVCpu->em.s.MWait.fWait &= ~(EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0);
+                    }
+                    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_UNHALT))
+                    {
+                        LogFlow(("EMR3ExecuteVM: Clearing UNHALT\n"));
+                        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_UNHALT);
+                    }
+                }
+            }
+            else
+                VBOXVMM_EM_STATE_UNCHANGED(pVCpu, enmNewState, rc);
+
+            STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x); /* (skip this in release) */
+            STAM_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
+
+            /*
+             * Act on the new state.
+             */
+            switch (enmNewState)
+            {
+                /*
+                 * Execute raw.
+                 */
+                case EMSTATE_RAW:
+                    AssertLogRelMsgFailed(("%Rrc\n", rc));
+                    rc = VERR_EM_INTERNAL_ERROR;
+                    break;
+
+                /*
+                 * Execute hardware accelerated raw.
+                 */
+                case EMSTATE_HM:
+                    rc = emR3HmExecute(pVM, pVCpu, &fFFDone);
+                    break;
+
+                /*
+                 * Execute hardware accelerated raw.
+                 */
+                case EMSTATE_NEM:
+                    rc = VBOXSTRICTRC_TODO(emR3NemExecute(pVM, pVCpu, &fFFDone));
+                    break;
+
+                /*
+                 * Execute recompiled.
+                 */
+                case EMSTATE_REM:
+                    rc = emR3RemExecute(pVM, pVCpu, &fFFDone);
+                    Log2(("EMR3ExecuteVM: emR3RemExecute -> %Rrc\n", rc));
+                    break;
+
+                /*
+                 * Execute in the interpreter.
+                 */
+                case EMSTATE_IEM:
+                {
+                    uint32_t cInstructions = 0;
+#if 0 /* For testing purposes. */
+                    STAM_PROFILE_START(&pVCpu->em.s.StatHmExec, x1);
+                    rc = VBOXSTRICTRC_TODO(EMR3HmSingleInstruction(pVM, pVCpu, EM_ONE_INS_FLAGS_RIP_CHANGE));
+                    STAM_PROFILE_STOP(&pVCpu->em.s.StatHmExec, x1);
+                    if (rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_RESCHEDULE_HM || rc == VINF_EM_RESCHEDULE_REM || rc == VINF_EM_RESCHEDULE_RAW)
+                        rc = VINF_SUCCESS;
+                    else if (rc == VERR_EM_CANNOT_EXEC_GUEST)
+#endif
+                        rc = VBOXSTRICTRC_TODO(IEMExecLots(pVCpu, 4096 /*cMaxInstructions*/, 2047 /*cPollRate*/, &cInstructions));
+                    if (pVM->em.s.fIemExecutesAll)
+                    {
+                        Assert(rc != VINF_EM_RESCHEDULE_REM);
+                        Assert(rc != VINF_EM_RESCHEDULE_RAW);
+                        Assert(rc != VINF_EM_RESCHEDULE_HM);
+#ifdef VBOX_HIGH_RES_TIMERS_HACK
+                        if (cInstructions < 2048)
+                            TMTimerPollVoid(pVM, pVCpu);
+#endif
+                    }
+                    fFFDone = false;
+                    break;
+                }
+
+                /*
+                 * Execute in IEM, hoping we can quickly switch aback to HM
+                 * or RAW execution.  If our hopes fail, we go to REM.
+                 */
+                case EMSTATE_IEM_THEN_REM:
+                {
+                    STAM_PROFILE_START(&pVCpu->em.s.StatIEMThenREM, pIemThenRem);
+                    rc = VBOXSTRICTRC_TODO(emR3ExecuteIemThenRem(pVM, pVCpu, &fFFDone));
+                    STAM_PROFILE_STOP(&pVCpu->em.s.StatIEMThenREM, pIemThenRem);
+                    break;
+                }
+
+                /*
+                 * Application processor execution halted until SIPI.
+                 */
+                case EMSTATE_WAIT_SIPI:
+                    /* no break */
+                /*
+                 * hlt - execution halted until interrupt.
+                 */
+                case EMSTATE_HALTED:
+                {
+                    STAM_REL_PROFILE_START(&pVCpu->em.s.StatHalted, y);
+                    /* If HM (or someone else) store a pending interrupt in
+                       TRPM, it must be dispatched ASAP without any halting.
+                       Anything pending in TRPM has been accepted and the CPU
+                       should already be the right state to receive it. */
+                    if (TRPMHasTrap(pVCpu))
+                        rc = VINF_EM_RESCHEDULE;
+                    /* MWAIT has a special extension where it's woken up when
+                       an interrupt is pending even when IF=0. */
+                    else if (   (pVCpu->em.s.MWait.fWait & (EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0))
+                             ==                            (EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0))
+                    {
+                        rc = VMR3WaitHalted(pVM, pVCpu, false /*fIgnoreInterrupts*/);
+                        if (rc == VINF_SUCCESS)
+                        {
+                            if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
+                                APICUpdatePendingInterrupts(pVCpu);
+
+                            if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC
+                                                         | VMCPU_FF_INTERRUPT_NESTED_GUEST
+                                                         | VMCPU_FF_INTERRUPT_NMI  | VMCPU_FF_INTERRUPT_SMI | VMCPU_FF_UNHALT))
+                            {
+                                Log(("EMR3ExecuteVM: Triggering reschedule on pending IRQ after MWAIT\n"));
+                                rc = VINF_EM_RESCHEDULE;
+                            }
+                        }
+                    }
+                    else
+                    {
+                        rc = VMR3WaitHalted(pVM, pVCpu, !(CPUMGetGuestEFlags(pVCpu) & X86_EFL_IF));
+                        /* We're only interested in NMI/SMIs here which have their own FFs, so we don't need to
+                           check VMCPU_FF_UPDATE_APIC here. */
+                        if (   rc == VINF_SUCCESS
+                            && VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI | VMCPU_FF_INTERRUPT_SMI | VMCPU_FF_UNHALT))
+                        {
+                            Log(("EMR3ExecuteVM: Triggering reschedule on pending NMI/SMI/UNHALT after HLT\n"));
+                            rc = VINF_EM_RESCHEDULE;
+                        }
+                    }
+
+                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatHalted, y);
+                    break;
+                }
+
+                /*
+                 * Suspended - return to VM.cpp.
+                 */
+                case EMSTATE_SUSPENDED:
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+                    Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(enmOldState)));
+                    return VINF_EM_SUSPEND;
+
+                /*
+                 * Debugging in the guest.
+                 */
+                case EMSTATE_DEBUG_GUEST_RAW:
+                case EMSTATE_DEBUG_GUEST_HM:
+                case EMSTATE_DEBUG_GUEST_NEM:
+                case EMSTATE_DEBUG_GUEST_IEM:
+                case EMSTATE_DEBUG_GUEST_REM:
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    rc = VBOXSTRICTRC_TODO(emR3Debug(pVM, pVCpu, rc));
+                    TMR3NotifyResume(pVM, pVCpu);
+                    Log2(("EMR3ExecuteVM: emR3Debug -> %Rrc (state %d)\n", rc, pVCpu->em.s.enmState));
+                    break;
+
+                /*
+                 * Debugging in the hypervisor.
+                 */
+                case EMSTATE_DEBUG_HYPER:
+                {
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+
+                    rc = VBOXSTRICTRC_TODO(emR3Debug(pVM, pVCpu, rc));
+                    Log2(("EMR3ExecuteVM: emR3Debug -> %Rrc (state %d)\n", rc, pVCpu->em.s.enmState));
+                    if (rc != VINF_SUCCESS)
+                    {
+                        if (rc == VINF_EM_OFF || rc == VINF_EM_TERMINATE)
+                            pVCpu->em.s.enmState = EMSTATE_TERMINATING;
+                        else
+                        {
+                            /* switch to guru meditation mode */
+                            pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
+                            VMR3SetGuruMeditation(pVM); /* This notifies the other EMTs. */
+                            VMMR3FatalDump(pVM, pVCpu, rc);
+                        }
+                        Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(enmOldState)));
+                        return rc;
+                    }
+
+                    STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
+                    TMR3NotifyResume(pVM, pVCpu);
+                    break;
+                }
+
+                /*
+                 * Guru meditation takes place in the debugger.
+                 */
+                case EMSTATE_GURU_MEDITATION:
+                {
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    VMR3SetGuruMeditation(pVM); /* This notifies the other EMTs. */
+                    VMMR3FatalDump(pVM, pVCpu, rc);
+                    emR3Debug(pVM, pVCpu, rc);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+                    Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(enmOldState)));
+                    return rc;
+                }
+
+                /*
+                 * The states we don't expect here.
+                 */
+                case EMSTATE_NONE:
+                case EMSTATE_TERMINATING:
+                default:
+                    AssertMsgFailed(("EMR3ExecuteVM: Invalid state %d!\n", pVCpu->em.s.enmState));
+                    pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+                    Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(enmOldState)));
+                    return VERR_EM_INTERNAL_ERROR;
+            }
+        } /* The Outer Main Loop */
+    }
+    else
+    {
+        /*
+         * Fatal error.
+         */
+        Log(("EMR3ExecuteVM: returns %Rrc because of longjmp / fatal error; (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(pVCpu->em.s.enmPrevState)));
+        TMR3NotifySuspend(pVM, pVCpu);
+        VMR3SetGuruMeditation(pVM); /* This notifies the other EMTs. */
+        VMMR3FatalDump(pVM, pVCpu, rc);
+        emR3Debug(pVM, pVCpu, rc);
+        STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+        /** @todo change the VM state! */
+        return rc;
+    }
+
+    /* not reached */
+}
+
diff --git a/src/VBox/VMM/VMMR3/EMHM.cpp b/src/VBox/VMM/VMMR3/EMHM.cpp
new file mode 100644
index 00000000..378b40f8
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/EMHM.cpp
@@ -0,0 +1,477 @@
+/* $Id: EMHM.cpp $ */
+/** @file
+ * EM - Execution Monitor / Manager - hardware virtualization
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_EM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/pdmqueue.h>
+#include <VBox/vmm/hm.h>
+#include "EMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/cpumdis.h>
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <VBox/err.h>
+#include <VBox/vmm/dbgf.h>
+#include "VMMTracing.h"
+
+#include <iprt/asm.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int      emR3HmHandleRC(PVM pVM, PVMCPU pVCpu, int rc);
+DECLINLINE(int) emR3HmExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC = VINF_SUCCESS);
+static int      emR3HmExecuteIOInstruction(PVM pVM, PVMCPU pVCpu);
+static int      emR3HmForcedActions(PVM pVM, PVMCPU pVCpu);
+
+#define EMHANDLERC_WITH_HM
+#define emR3ExecuteInstruction   emR3HmExecuteInstruction
+#define emR3ExecuteIOInstruction emR3HmExecuteIOInstruction
+#include "EMHandleRCTmpl.h"
+
+
+/**
+ * Executes instruction in HM mode if we can.
+ *
+ * This is somewhat comparable to REMR3EmulateInstruction.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_EM_DBG_STEPPED on success.
+ * @retval  VERR_EM_CANNOT_EXEC_GUEST if we cannot execute guest instructions in
+ *          HM right now.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure for the calling EMT.
+ * @param   fFlags  Combinations of EM_ONE_INS_FLAGS_XXX.
+ * @thread  EMT.
+ */
+VMMR3_INT_DECL(VBOXSTRICTRC) EMR3HmSingleInstruction(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
+{
+    Assert(!(fFlags & ~EM_ONE_INS_FLAGS_MASK));
+
+    if (!HMCanExecuteGuest(pVM, pVCpu, &pVCpu->cpum.GstCtx))
+        return VINF_EM_RESCHEDULE;
+
+    uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip;
+    for (;;)
+    {
+        /*
+         * Service necessary FFs before going into HM.
+         */
+        if (   VM_FF_IS_ANY_SET(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
+            || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
+        {
+            VBOXSTRICTRC rcStrict = emR3HmForcedActions(pVM, pVCpu);
+            if (rcStrict != VINF_SUCCESS)
+            {
+                Log(("EMR3HmSingleInstruction: FFs before -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+                return rcStrict;
+            }
+        }
+
+        /*
+         * Go execute it.
+         */
+        bool fOld = HMSetSingleInstruction(pVM, pVCpu, true);
+        VBOXSTRICTRC rcStrict = VMMR3HmRunGC(pVM, pVCpu);
+        HMSetSingleInstruction(pVM, pVCpu, fOld);
+        LogFlow(("EMR3HmSingleInstruction: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+        /*
+         * Handle high priority FFs and informational status codes.  We don't do
+         * normal FF processing the caller or the next call can deal with them.
+         */
+        VMCPU_FF_CLEAR_MASK(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
+        if (   VM_FF_IS_ANY_SET(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
+            || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
+        {
+            rcStrict = emR3HighPriorityPostForcedActions(pVM, pVCpu, rcStrict);
+            LogFlow(("EMR3HmSingleInstruction: FFs after -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+        }
+
+        if (rcStrict != VINF_SUCCESS && (rcStrict < VINF_EM_FIRST || rcStrict > VINF_EM_LAST))
+        {
+            rcStrict = emR3HmHandleRC(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict));
+            Log(("EMR3HmSingleInstruction: emR3HmHandleRC -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+        }
+
+        /*
+         * Done?
+         */
+        if (   (rcStrict != VINF_SUCCESS && rcStrict != VINF_EM_DBG_STEPPED)
+            || !(fFlags & EM_ONE_INS_FLAGS_RIP_CHANGE)
+            || pVCpu->cpum.GstCtx.rip != uOldRip)
+        {
+            if (rcStrict == VINF_SUCCESS && pVCpu->cpum.GstCtx.rip != uOldRip)
+                rcStrict = VINF_EM_DBG_STEPPED;
+            Log(("EMR3HmSingleInstruction: returns %Rrc (rip %llx -> %llx)\n", VBOXSTRICTRC_VAL(rcStrict), uOldRip, pVCpu->cpum.GstCtx.rip));
+            CPUM_IMPORT_EXTRN_RET(pVCpu, ~CPUMCTX_EXTRN_KEEPER_MASK);
+            return rcStrict;
+        }
+    }
+}
+
+
+/**
+ * Executes one (or perhaps a few more) instruction(s).
+ *
+ * @returns VBox status code suitable for EM.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   rcRC        Return code from RC.
+ * @param   pszPrefix   Disassembly prefix. If not NULL we'll disassemble the
+ *                      instruction and prefix the log output with this text.
+ */
+#if defined(LOG_ENABLED) || defined(DOXYGEN_RUNNING)
+static int emR3HmExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcRC, const char *pszPrefix)
+#else
+static int emR3HmExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcRC)
+#endif
+{
+    RT_NOREF(rcRC, pVM);
+
+#ifdef LOG_ENABLED
+    /*
+     * Log it.
+     */
+    Log(("EMINS: %04x:%RGv RSP=%RGv\n", pVCpu->cpum.GstCtx.cs.Sel, (RTGCPTR)pVCpu->cpum.GstCtx.rip, (RTGCPTR)pVCpu->cpum.GstCtx.rsp));
+    if (pszPrefix)
+    {
+        DBGFR3_INFO_LOG(pVM, pVCpu, "cpumguest", pszPrefix);
+        DBGFR3_DISAS_INSTR_CUR_LOG(pVCpu, pszPrefix);
+    }
+#endif
+
+    /*
+     * Use IEM and fallback on REM if the functionality is missing.
+     * Once IEM gets mature enough, nothing should ever fall back.
+     */
+    STAM_PROFILE_START(&pVCpu->em.s.StatIEMEmu, a);
+    VBOXSTRICTRC rcStrict;
+    uint32_t     idxContinueExitRec = pVCpu->em.s.idxContinueExitRec;
+    RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
+    if (idxContinueExitRec >= RT_ELEMENTS(pVCpu->em.s.aExitRecords))
+    {
+        CPUM_IMPORT_EXTRN_RET(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = VBOXSTRICTRC_TODO(IEMExecOne(pVCpu));
+    }
+    else
+    {
+        RT_UNTRUSTED_VALIDATED_FENCE();
+        rcStrict = EMHistoryExec(pVCpu, &pVCpu->em.s.aExitRecords[idxContinueExitRec], 0);
+        LogFlow(("emR3HmExecuteInstruction: %Rrc (EMHistoryExec)\n", VBOXSTRICTRC_VAL(rcStrict)));
+    }
+    STAM_PROFILE_STOP(&pVCpu->em.s.StatIEMEmu, a);
+
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * Executes one (or perhaps a few more) instruction(s).
+ * This is just a wrapper for discarding pszPrefix in non-logging builds.
+ *
+ * @returns VBox status code suitable for EM.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszPrefix   Disassembly prefix. If not NULL we'll disassemble the
+ *                      instruction and prefix the log output with this text.
+ * @param   rcGC        GC return code
+ */
+DECLINLINE(int) emR3HmExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC)
+{
+#ifdef LOG_ENABLED
+    return emR3HmExecuteInstructionWorker(pVM, pVCpu, rcGC, pszPrefix);
+#else
+    RT_NOREF_PV(pszPrefix);
+    return emR3HmExecuteInstructionWorker(pVM, pVCpu, rcGC);
+#endif
+}
+
+
+/**
+ * Executes one (or perhaps a few more) IO instruction(s).
+ *
+ * @returns VBox status code suitable for EM.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+static int emR3HmExecuteIOInstruction(PVM pVM, PVMCPU pVCpu)
+{
+    RT_NOREF(pVM);
+    STAM_PROFILE_START(&pVCpu->em.s.StatIOEmu, a);
+
+    VBOXSTRICTRC rcStrict;
+    uint32_t     idxContinueExitRec = pVCpu->em.s.idxContinueExitRec;
+    RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
+    if (idxContinueExitRec >= RT_ELEMENTS(pVCpu->em.s.aExitRecords))
+    {
+        /*
+         * Hand it over to the interpreter.
+         */
+        CPUM_IMPORT_EXTRN_RET(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+        LogFlow(("emR3HmExecuteIOInstruction: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    }
+    else
+    {
+        RT_UNTRUSTED_VALIDATED_FENCE();
+        CPUM_IMPORT_EXTRN_RET(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = EMHistoryExec(pVCpu, &pVCpu->em.s.aExitRecords[idxContinueExitRec], 0);
+        LogFlow(("emR3HmExecuteIOInstruction: %Rrc (EMHistoryExec)\n", VBOXSTRICTRC_VAL(rcStrict)));
+        STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIoRestarted);
+    }
+
+    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIoIem);
+    STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * Process HM specific forced actions.
+ *
+ * This function is called when any FFs in the VM_FF_HIGH_PRIORITY_PRE_RAW_MASK
+ * or/and VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK are pending.
+ *
+ * @returns VBox status code. May return VINF_EM_NO_MEMORY but none of the other
+ *          EM statuses.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+static int emR3HmForcedActions(PVM pVM, PVMCPU pVCpu)
+{
+    /*
+     * Sync page directory.
+     */
+    if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
+    {
+        CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4);
+        Assert(pVCpu->em.s.enmState != EMSTATE_WAIT_SIPI);
+        int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* Prefetch pages for EIP and ESP. */
+        /** @todo This is rather expensive. Should investigate if it really helps at all. */
+        /** @todo this should be skipped! */
+        CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS);
+        rc = PGMPrefetchPage(pVCpu, SELMToFlat(pVM, DISSELREG_CS, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pVCpu->cpum.GstCtx.rip));
+        if (rc == VINF_SUCCESS)
+            rc = PGMPrefetchPage(pVCpu, SELMToFlat(pVM, DISSELREG_SS, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pVCpu->cpum.GstCtx.rsp));
+        if (rc != VINF_SUCCESS)
+        {
+            if (rc != VINF_PGM_SYNC_CR3)
+            {
+                AssertLogRelMsgReturn(RT_FAILURE(rc), ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                return rc;
+            }
+            rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+        /** @todo maybe prefetch the supervisor stack page as well */
+    }
+
+    /*
+     * Allocate handy pages (just in case the above actions have consumed some pages).
+     */
+    if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PGM_NEED_HANDY_PAGES, VM_FF_PGM_NO_MEMORY))
+    {
+        int rc = PGMR3PhysAllocateHandyPages(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Check whether we're out of memory now.
+     *
+     * This may stem from some of the above actions or operations that has been executed
+     * since we ran FFs. The allocate handy pages must for instance always be followed by
+     * this check.
+     */
+    if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+        return VINF_EM_NO_MEMORY;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Executes hardware accelerated raw code. (Intel VT-x & AMD-V)
+ *
+ * This function contains the raw-mode version of the inner
+ * execution loop (the outer loop being in EMR3ExecuteVM()).
+ *
+ * @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE, VINF_EM_RESCHEDULE_RAW,
+ *          VINF_EM_RESCHEDULE_REM, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pfFFDone    Where to store an indicator telling whether or not
+ *                      FFs were done before returning.
+ */
+int emR3HmExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
+{
+    int      rc = VERR_IPE_UNINITIALIZED_STATUS;
+
+    LogFlow(("emR3HmExecute%d: (cs:eip=%04x:%RGv)\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, (RTGCPTR)pVCpu->cpum.GstCtx.rip));
+    *pfFFDone = false;
+
+    STAM_COUNTER_INC(&pVCpu->em.s.StatHMExecuteCalled);
+
+    /*
+     * Spin till we get a forced action which returns anything but VINF_SUCCESS.
+     */
+    for (;;)
+    {
+        STAM_PROFILE_ADV_START(&pVCpu->em.s.StatHMEntry, a);
+
+        /* Check if a forced reschedule is pending. */
+        if (HMR3IsRescheduleRequired(pVM, &pVCpu->cpum.GstCtx))
+        {
+            rc = VINF_EM_RESCHEDULE;
+            break;
+        }
+
+        /*
+         * Process high priority pre-execution raw-mode FFs.
+         */
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
+        {
+            rc = emR3HmForcedActions(pVM, pVCpu);
+            if (rc != VINF_SUCCESS)
+                break;
+        }
+
+#ifdef LOG_ENABLED
+        /*
+         * Log important stuff before entering GC.
+         */
+        if (TRPMHasTrap(pVCpu))
+            Log(("CPU%d: Pending hardware interrupt=0x%x cs:rip=%04X:%RGv\n", pVCpu->idCpu, TRPMGetTrapNo(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, (RTGCPTR)pVCpu->cpum.GstCtx.rip));
+
+        uint32_t cpl = CPUMGetGuestCPL(pVCpu);
+        if (pVM->cCpus == 1)
+        {
+            if (pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
+                Log(("HWV86: %08X IF=%d\n", pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.Bits.u1IF));
+            else if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
+                Log(("HWR%d: %04X:%RGv ESP=%RGv IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->cpum.GstCtx.cs.Sel, (RTGCPTR)pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.Bits.u1IF, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, (uint32_t)pVCpu->cpum.GstCtx.cr0, (uint32_t)pVCpu->cpum.GstCtx.cr4, (uint32_t)pVCpu->cpum.GstCtx.msrEFER));
+            else
+                Log(("HWR%d: %04X:%08X ESP=%08X IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->cpum.GstCtx.cs.Sel,          pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.eflags.Bits.u1IF, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, (uint32_t)pVCpu->cpum.GstCtx.cr0, (uint32_t)pVCpu->cpum.GstCtx.cr4, (uint32_t)pVCpu->cpum.GstCtx.msrEFER));
+        }
+        else
+        {
+            if (pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
+                Log(("HWV86-CPU%d: %08X IF=%d\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.Bits.u1IF));
+            else if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
+                Log(("HWR%d-CPU%d: %04X:%RGv ESP=%RGv IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, (RTGCPTR)pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.Bits.u1IF, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, (uint32_t)pVCpu->cpum.GstCtx.cr0, (uint32_t)pVCpu->cpum.GstCtx.cr4, (uint32_t)pVCpu->cpum.GstCtx.msrEFER));
+            else
+                Log(("HWR%d-CPU%d: %04X:%08X ESP=%08X IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel,          pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.eflags.Bits.u1IF, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, (uint32_t)pVCpu->cpum.GstCtx.cr0, (uint32_t)pVCpu->cpum.GstCtx.cr4, (uint32_t)pVCpu->cpum.GstCtx.msrEFER));
+        }
+#endif /* LOG_ENABLED */
+
+        /*
+         * Execute the code.
+         */
+        STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatHMEntry, a);
+
+        if (RT_LIKELY(emR3IsExecutionAllowed(pVM, pVCpu)))
+        {
+            STAM_PROFILE_START(&pVCpu->em.s.StatHMExec, x);
+            rc = VMMR3HmRunGC(pVM, pVCpu);
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatHMExec, x);
+        }
+        else
+        {
+            /* Give up this time slice; virtual time continues */
+            STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatCapped, u);
+            RTThreadSleep(5);
+            STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatCapped, u);
+            rc = VINF_SUCCESS;
+        }
+
+
+        /*
+         * Deal with high priority post execution FFs before doing anything else.
+         */
+        VMCPU_FF_CLEAR_MASK(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
+            rc = VBOXSTRICTRC_TODO(emR3HighPriorityPostForcedActions(pVM, pVCpu, rc));
+
+        /*
+         * Process the returned status code.
+         */
+        if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
+            break;
+
+        rc = emR3HmHandleRC(pVM, pVCpu, rc);
+        if (rc != VINF_SUCCESS)
+            break;
+
+        /*
+         * Check and execute forced actions.
+         */
+#ifdef VBOX_HIGH_RES_TIMERS_HACK
+        TMTimerPollVoid(pVM, pVCpu);
+#endif
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_ALL_MASK))
+        {
+            rc = emR3ForcedActions(pVM, pVCpu, rc);
+            VBOXVMM_EM_FF_ALL_RET(pVCpu, rc);
+            if (    rc != VINF_SUCCESS
+                &&  rc != VINF_EM_RESCHEDULE_HM)
+            {
+                *pfFFDone = true;
+                break;
+            }
+        }
+    }
+
+    /*
+     * Return to outer loop.
+     */
+#if defined(LOG_ENABLED) && defined(DEBUG)
+    RTLogFlush(NULL);
+#endif
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/EMR3Dbg.cpp b/src/VBox/VMM/VMMR3/EMR3Dbg.cpp
new file mode 100644
index 00000000..ec83bce4
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/EMR3Dbg.cpp
@@ -0,0 +1,338 @@
+/* $Id: EMR3Dbg.cpp $ */
+/** @file
+ * EM - Execution Monitor / Manager, Debugger Related Bits.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_EM
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/dbg.h>
+#include "EMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <iprt/string.h>
+#include <iprt/ctype.h>
+
+
+/** @callback_method_impl{FNDBGCCMD,
+ * Implements the '.alliem' command. }
+ */
+static DECLCALLBACK(int) enmR3DbgCmdAllIem(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
+{
+    int  rc;
+    bool f;
+
+    if (cArgs == 0)
+    {
+        rc = EMR3QueryExecutionPolicy(pUVM, EMEXECPOLICY_IEM_ALL, &f);
+        if (RT_FAILURE(rc))
+            return DBGCCmdHlpFailRc(pCmdHlp, pCmd, rc, "EMR3QueryExecutionPolicy(,EMEXECPOLICY_IEM_ALL,");
+        DBGCCmdHlpPrintf(pCmdHlp, f ? "alliem: enabled\n" : "alliem: disabled\n");
+    }
+    else
+    {
+        rc = DBGCCmdHlpVarToBool(pCmdHlp, &paArgs[0], &f);
+        if (RT_FAILURE(rc))
+            return DBGCCmdHlpFailRc(pCmdHlp, pCmd, rc, "DBGCCmdHlpVarToBool");
+        rc = EMR3SetExecutionPolicy(pUVM, EMEXECPOLICY_IEM_ALL, f);
+        if (RT_FAILURE(rc))
+            return DBGCCmdHlpFailRc(pCmdHlp, pCmd, rc, "EMR3SetExecutionPolicy(,EMEXECPOLICY_IEM_ALL,%RTbool)", f);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Describes a optional boolean argument. */
+static DBGCVARDESC const g_BoolArg = { 0, 1, DBGCVAR_CAT_ANY, 0, "boolean", "Boolean value." };
+
+/** Commands.  */
+static DBGCCMD const g_aCmds[] =
+{
+    {
+        "alliem", 0, 1, &g_BoolArg, 1, 0, enmR3DbgCmdAllIem, "[boolean]",
+        "Enables or disabled executing ALL code in IEM, if no arguments are given it displays the current status."
+    },
+};
+
+
+/**
+ * Translates EMEXITTYPE into a name.
+ *
+ * @returns Pointer to read-only name, NULL if unknown type.
+ * @param   enmExitType     The exit type to name.
+ */
+VMM_INT_DECL(const char *) EMR3GetExitTypeName(EMEXITTYPE enmExitType)
+{
+    switch (enmExitType)
+    {
+        case EMEXITTYPE_INVALID:            return "invalid";
+        case EMEXITTYPE_IO_PORT_READ:       return "I/O port read";
+        case EMEXITTYPE_IO_PORT_WRITE:      return "I/O port write";
+        case EMEXITTYPE_IO_PORT_STR_READ:   return "I/O port string read";
+        case EMEXITTYPE_IO_PORT_STR_WRITE:  return "I/O port string write";
+        case EMEXITTYPE_MMIO:               return "MMIO access";
+        case EMEXITTYPE_MMIO_READ:          return "MMIO read";
+        case EMEXITTYPE_MMIO_WRITE:         return "MMIO write";
+        case EMEXITTYPE_MSR_READ:           return "MSR read";
+        case EMEXITTYPE_MSR_WRITE:          return "MSR write";
+        case EMEXITTYPE_CPUID:              return "CPUID";
+        case EMEXITTYPE_RDTSC:              return "RDTSC";
+        case EMEXITTYPE_MOV_CRX:            return "MOV CRx";
+        case EMEXITTYPE_MOV_DRX:            return "MOV DRx";
+
+        /* Raw-mode only: */
+        case EMEXITTYPE_INVLPG:             return "INVLPG";
+        case EMEXITTYPE_LLDT:               return "LLDT";
+        case EMEXITTYPE_RDPMC:              return "RDPMC";
+        case EMEXITTYPE_CLTS:               return "CLTS";
+        case EMEXITTYPE_STI:                return "STI";
+        case EMEXITTYPE_INT:                return "INT";
+        case EMEXITTYPE_SYSCALL:            return "SYSCALL";
+        case EMEXITTYPE_SYSENTER:           return "SYSENTER";
+        case EMEXITTYPE_HLT:                return "HLT";
+    }
+    return NULL;
+}
+
+
+/**
+ * Translates flags+type into an exit name.
+ *
+ * @returns Exit name.
+ * @param   uFlagsAndType   The exit to name.
+ * @param   pszFallback     Buffer for formatting a numeric fallback.
+ * @param   cbFallback      Size of fallback buffer.
+ */
+static const char *emR3HistoryGetExitName(uint32_t uFlagsAndType, char *pszFallback, size_t cbFallback)
+{
+    const char *pszExitName;
+    switch (uFlagsAndType & EMEXIT_F_KIND_MASK)
+    {
+        case EMEXIT_F_KIND_EM:
+            pszExitName = EMR3GetExitTypeName((EMEXITTYPE)(uFlagsAndType & EMEXIT_F_TYPE_MASK));
+            break;
+
+        case EMEXIT_F_KIND_VMX:
+            pszExitName = HMGetVmxExitName( uFlagsAndType & EMEXIT_F_TYPE_MASK);
+            break;
+
+        case EMEXIT_F_KIND_SVM:
+            pszExitName = HMGetSvmExitName( uFlagsAndType & EMEXIT_F_TYPE_MASK);
+            break;
+
+        case EMEXIT_F_KIND_NEM:
+            pszExitName = NEMR3GetExitName(   uFlagsAndType & EMEXIT_F_TYPE_MASK);
+            break;
+
+        case EMEXIT_F_KIND_XCPT:
+            switch (uFlagsAndType & EMEXIT_F_TYPE_MASK)
+            {
+                case X86_XCPT_DE:               return "Xcpt #DE";
+                case X86_XCPT_DB:               return "Xcpt #DB";
+                case X86_XCPT_NMI:              return "Xcpt #NMI";
+                case X86_XCPT_BP:               return "Xcpt #BP";
+                case X86_XCPT_OF:               return "Xcpt #OF";
+                case X86_XCPT_BR:               return "Xcpt #BR";
+                case X86_XCPT_UD:               return "Xcpt #UD";
+                case X86_XCPT_NM:               return "Xcpt #NM";
+                case X86_XCPT_DF:               return "Xcpt #DF";
+                case X86_XCPT_CO_SEG_OVERRUN:   return "Xcpt #CO_SEG_OVERRUN";
+                case X86_XCPT_TS:               return "Xcpt #TS";
+                case X86_XCPT_NP:               return "Xcpt #NP";
+                case X86_XCPT_SS:               return "Xcpt #SS";
+                case X86_XCPT_GP:               return "Xcpt #GP";
+                case X86_XCPT_PF:               return "Xcpt #PF";
+                case X86_XCPT_MF:               return "Xcpt #MF";
+                case X86_XCPT_AC:               return "Xcpt #AC";
+                case X86_XCPT_MC:               return "Xcpt #MC";
+                case X86_XCPT_XF:               return "Xcpt #XF";
+                case X86_XCPT_VE:               return "Xcpt #VE";
+                case X86_XCPT_SX:               return "Xcpt #SX";
+                default:
+                    pszExitName = NULL;
+                    break;
+            }
+            break;
+
+        default:
+            AssertFailed();
+            pszExitName = NULL;
+            break;
+    }
+    if (pszExitName)
+        return pszExitName;
+    RTStrPrintf(pszFallback, cbFallback, "%#06x", uFlagsAndType & (EMEXIT_F_KIND_MASK | EMEXIT_F_TYPE_MASK));
+    return pszFallback;
+}
+
+
+/**
+ * Displays the VM-exit history.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) emR3InfoExitHistory(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+
+    /*
+     * Figure out target cpu and parse arguments.
+     */
+    PVMCPU   pVCpu    = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+    bool     fReverse = true;
+    uint32_t cLeft    = RT_ELEMENTS(pVCpu->em.s.aExitHistory);
+
+    while (pszArgs && *pszArgs)
+    {
+        pszArgs = RTStrStripL(pszArgs);
+        if (!*pszArgs)
+            break;
+        if (RT_C_IS_DIGIT(*pszArgs))
+        {
+            /* The number to dump. */
+            uint32_t uValue = cLeft;
+            RTStrToUInt32Ex(pszArgs, (char **)&pszArgs, 0, &uValue);
+            if (uValue > 0)
+                cLeft = RT_MIN(uValue, RT_ELEMENTS(pVCpu->em.s.aExitHistory));
+        }
+        else if (RTStrCmp(pszArgs, "reverse") == 0)
+        {
+            pszArgs += 7;
+            fReverse = true;
+        }
+        else if (RTStrCmp(pszArgs, "ascending") == 0)
+        {
+            pszArgs += 9;
+            fReverse = false;
+        }
+        else if (RTStrCmp(pszArgs, "asc") == 0)
+        {
+            pszArgs += 3;
+            fReverse = false;
+        }
+        else
+        {
+            const char *pszStart = pszArgs;
+            while (*pszArgs && !RT_C_IS_SPACE(*pszArgs))
+                pszArgs++;
+            pHlp->pfnPrintf(pHlp, "Unknown option: %.*s\n", pszArgs - pszStart, pszArgs);
+        }
+    }
+
+    /*
+     * Do the job.
+     */
+    uint64_t idx = pVCpu->em.s.iNextExit;
+    if (idx == 0)
+        pHlp->pfnPrintf(pHlp, "CPU[%u]: VM-exit history: empty\n", pVCpu->idCpu);
+    else
+    {
+        /*
+         * Print header.
+         */
+        pHlp->pfnPrintf(pHlp,
+                        "CPU[%u]: VM-exit history:\n"
+                        "   Exit No.:     TSC timestamp / delta    RIP (Flat/*)      Exit    Name\n"
+                        , pVCpu->idCpu);
+
+        /*
+         * Adjust bounds if ascending order.
+         */
+        if (!fReverse)
+        {
+            if (idx > cLeft)
+                idx -= cLeft;
+            else
+            {
+                cLeft = idx;
+                idx = 0;
+            }
+        }
+
+        /*
+         * Print the entries.
+         */
+        uint64_t uPrevTimestamp = 0;
+        do
+        {
+            if (fReverse)
+                idx -= 1;
+            PCEMEXITENTRY const pEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)idx & 0xff];
+
+            /* Get the exit name. */
+            char        szExitName[16];
+            const char *pszExitName = emR3HistoryGetExitName(pEntry->uFlagsAndType, szExitName, sizeof(szExitName));
+
+            /* Calc delta (negative if reverse order, positive ascending). */
+            int64_t offDelta = uPrevTimestamp != 0 && pEntry->uTimestamp != 0 ? pEntry->uTimestamp - uPrevTimestamp : 0;
+            uPrevTimestamp = pEntry->uTimestamp;
+
+            char szPC[32];
+            if (!(pEntry->uFlagsAndType & (EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC)))
+                RTStrPrintf(szPC, sizeof(szPC), "%016RX64 ", pEntry->uFlatPC);
+            else if (pEntry->uFlagsAndType & EMEXIT_F_UNFLATTENED_PC)
+                RTStrPrintf(szPC, sizeof(szPC), "%016RX64*", pEntry->uFlatPC);
+            else
+                RTStrPrintf(szPC, sizeof(szPC), "%04x:%08RX32*   ", (uint32_t)(pEntry->uFlatPC >> 32), (uint32_t)pEntry->uFlatPC);
+
+            /* Do the printing. */
+            if (pEntry->idxSlot == UINT32_MAX)
+                pHlp->pfnPrintf(pHlp, " %10RU64: %#018RX64/%+-9RI64 %s %#07x %s\n",
+                                idx, pEntry->uTimestamp, offDelta, szPC, pEntry->uFlagsAndType, pszExitName);
+            else
+            {
+                /** @todo more on this later */
+                pHlp->pfnPrintf(pHlp, " %10RU64: %#018RX64/%+-9RI64 %s %#07x %s slot=%#x\n",
+                                idx, pEntry->uTimestamp, offDelta, szPC, pEntry->uFlagsAndType, pszExitName, pEntry->idxSlot);
+            }
+
+            /* Advance if ascending. */
+            if (!fReverse)
+                idx += 1;
+        } while (--cLeft > 0 && idx > 0);
+    }
+}
+
+
+int emR3InitDbg(PVM pVM)
+{
+    /*
+     * Register info dumpers.
+     */
+    const char *pszExitsDesc = "Dumps the VM-exit history. Arguments: Number of entries; 'asc', 'ascending' or 'reverse'.";
+    int rc = DBGFR3InfoRegisterInternalEx(pVM, "exits", pszExitsDesc, emR3InfoExitHistory, DBGFINFO_FLAGS_ALL_EMTS);
+    AssertLogRelRCReturn(rc, rc);
+    rc = DBGFR3InfoRegisterInternalEx(pVM, "exithistory", pszExitsDesc, emR3InfoExitHistory, DBGFINFO_FLAGS_ALL_EMTS);
+    AssertLogRelRCReturn(rc, rc);
+
+#ifdef VBOX_WITH_DEBUGGER
+    /*
+     * Register debugger commands.
+     */
+    rc = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
+    AssertLogRelRCReturn(rc, rc);
+#endif
+
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMR3/EMR3Nem.cpp b/src/VBox/VMM/VMMR3/EMR3Nem.cpp
new file mode 100644
index 00000000..dd0f3f99
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/EMR3Nem.cpp
@@ -0,0 +1,475 @@
+/* $Id: EMR3Nem.cpp $ */
+/** @file
+ * EM - Execution Monitor / Manager - NEM interface.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_EM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/pdmqueue.h>
+#include "EMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/cpumdis.h>
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <VBox/err.h>
+#include <VBox/vmm/dbgf.h>
+#include "VMMTracing.h"
+
+#include <iprt/asm.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int      emR3NemHandleRC(PVM pVM, PVMCPU pVCpu, int rc);
+DECLINLINE(int) emR3NemExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC = VINF_SUCCESS);
+static int      emR3NemExecuteIOInstruction(PVM pVM, PVMCPU pVCpu);
+static int      emR3NemForcedActions(PVM pVM, PVMCPU pVCpu);
+
+#define EMHANDLERC_WITH_NEM
+#define emR3ExecuteInstruction   emR3NemExecuteInstruction
+#define emR3ExecuteIOInstruction emR3NemExecuteIOInstruction
+#include "EMHandleRCTmpl.h"
+
+
+/**
+ * Executes instruction in NEM mode if we can.
+ *
+ * This is somewhat comparable to REMR3EmulateInstruction.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_EM_DBG_STEPPED on success.
+ * @retval  VERR_EM_CANNOT_EXEC_GUEST if we cannot execute guest instructions in
+ *          HM right now.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure for the calling EMT.
+ * @param   fFlags  Combinations of EM_ONE_INS_FLAGS_XXX.
+ * @thread  EMT.
+ */
+VBOXSTRICTRC emR3NemSingleInstruction(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
+{
+    Assert(!(fFlags & ~EM_ONE_INS_FLAGS_MASK));
+
+    if (!NEMR3CanExecuteGuest(pVM, pVCpu))
+        return VINF_EM_RESCHEDULE;
+
+    uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip;
+    for (;;)
+    {
+        /*
+         * Service necessary FFs before going into HM.
+         */
+        if (   VM_FF_IS_ANY_SET(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
+            || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
+        {
+            VBOXSTRICTRC rcStrict = emR3NemForcedActions(pVM, pVCpu);
+            if (rcStrict != VINF_SUCCESS)
+            {
+                Log(("emR3NemSingleInstruction: FFs before -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+                return rcStrict;
+            }
+        }
+
+        /*
+         * Go execute it.
+         */
+        bool fOld = NEMR3SetSingleInstruction(pVM, pVCpu, true);
+        VBOXSTRICTRC rcStrict = NEMR3RunGC(pVM, pVCpu);
+        NEMR3SetSingleInstruction(pVM, pVCpu, fOld);
+        LogFlow(("emR3NemSingleInstruction: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+        /*
+         * Handle high priority FFs and informational status codes.  We don't do
+         * normal FF processing the caller or the next call can deal with them.
+         */
+        VMCPU_FF_CLEAR_MASK(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
+        if (   VM_FF_IS_ANY_SET(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
+            || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
+        {
+            rcStrict = emR3HighPriorityPostForcedActions(pVM, pVCpu, rcStrict);
+            LogFlow(("emR3NemSingleInstruction: FFs after -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+        }
+
+        if (rcStrict != VINF_SUCCESS && (rcStrict < VINF_EM_FIRST || rcStrict > VINF_EM_LAST))
+        {
+            rcStrict = emR3NemHandleRC(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict));
+            Log(("emR3NemSingleInstruction: emR3NemHandleRC -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+        }
+
+        /*
+         * Done?
+         */
+        CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP);
+        if (   (rcStrict != VINF_SUCCESS && rcStrict != VINF_EM_DBG_STEPPED)
+            || !(fFlags & EM_ONE_INS_FLAGS_RIP_CHANGE)
+            || pVCpu->cpum.GstCtx.rip != uOldRip)
+        {
+            if (rcStrict == VINF_SUCCESS && pVCpu->cpum.GstCtx.rip != uOldRip)
+                rcStrict = VINF_EM_DBG_STEPPED;
+            Log(("emR3NemSingleInstruction: returns %Rrc (rip %llx -> %llx)\n",
+                 VBOXSTRICTRC_VAL(rcStrict), uOldRip, pVCpu->cpum.GstCtx.rip));
+            CPUM_IMPORT_EXTRN_RET(pVCpu, ~CPUMCTX_EXTRN_KEEPER_MASK);
+            return rcStrict;
+        }
+    }
+}
+
+
+/**
+ * Executes one (or perhaps a few more) instruction(s).
+ *
+ * @returns VBox status code suitable for EM.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   rcRC        Return code from RC.
+ * @param   pszPrefix   Disassembly prefix. If not NULL we'll disassemble the
+ *                      instruction and prefix the log output with this text.
+ */
+#if defined(LOG_ENABLED) || defined(DOXYGEN_RUNNING)
+static int emR3NemExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcRC, const char *pszPrefix)
+#else
+static int emR3NemExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcRC)
+#endif
+{
+    NOREF(rcRC);
+
+#ifdef LOG_ENABLED
+    /*
+     * Log it.
+     */
+    Log(("EMINS: %04x:%RGv RSP=%RGv\n", pVCpu->cpum.GstCtx.cs.Sel, (RTGCPTR)pVCpu->cpum.GstCtx.rip, (RTGCPTR)pVCpu->cpum.GstCtx.rsp));
+    if (pszPrefix)
+    {
+        DBGFR3_INFO_LOG(pVM, pVCpu, "cpumguest", pszPrefix);
+        DBGFR3_DISAS_INSTR_CUR_LOG(pVCpu, pszPrefix);
+    }
+#endif
+
+    /*
+     * Use IEM and fallback on REM if the functionality is missing.
+     * Once IEM gets mature enough, nothing should ever fall back.
+     */
+    STAM_PROFILE_START(&pVCpu->em.s.StatIEMEmu, a);
+
+    VBOXSTRICTRC rcStrict;
+    uint32_t     idxContinueExitRec = pVCpu->em.s.idxContinueExitRec;
+    RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
+    if (idxContinueExitRec >= RT_ELEMENTS(pVCpu->em.s.aExitRecords))
+    {
+        CPUM_IMPORT_EXTRN_RET(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+        rcStrict = IEMExecOne(pVCpu);
+    }
+    else
+    {
+        RT_UNTRUSTED_VALIDATED_FENCE();
+        rcStrict = EMHistoryExec(pVCpu, &pVCpu->em.s.aExitRecords[idxContinueExitRec], 0);
+        LogFlow(("emR3NemExecuteInstruction: %Rrc (EMHistoryExec)\n", VBOXSTRICTRC_VAL(rcStrict)));
+    }
+
+    STAM_PROFILE_STOP(&pVCpu->em.s.StatIEMEmu, a);
+
+    NOREF(pVM);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * Executes one (or perhaps a few more) instruction(s).
+ * This is just a wrapper for discarding pszPrefix in non-logging builds.
+ *
+ * @returns VBox status code suitable for EM.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszPrefix   Disassembly prefix. If not NULL we'll disassemble the
+ *                      instruction and prefix the log output with this text.
+ * @param   rcGC        GC return code
+ */
+DECLINLINE(int) emR3NemExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC)
+{
+#ifdef LOG_ENABLED
+    return emR3NemExecuteInstructionWorker(pVM, pVCpu, rcGC, pszPrefix);
+#else
+    RT_NOREF_PV(pszPrefix);
+    return emR3NemExecuteInstructionWorker(pVM, pVCpu, rcGC);
+#endif
+}
+
+/**
+ * Executes one (or perhaps a few more) IO instruction(s).
+ *
+ * @returns VBox status code suitable for EM.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+static int emR3NemExecuteIOInstruction(PVM pVM, PVMCPU pVCpu)
+{
+    RT_NOREF_PV(pVM);
+    STAM_PROFILE_START(&pVCpu->em.s.StatIOEmu, a);
+
+    /*
+     * Hand it over to the interpreter.
+     */
+    CPUM_IMPORT_EXTRN_RET(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+    VBOXSTRICTRC rcStrict;
+    uint32_t     idxContinueExitRec = pVCpu->em.s.idxContinueExitRec;
+    RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
+    if (idxContinueExitRec >= RT_ELEMENTS(pVCpu->em.s.aExitRecords))
+    {
+        rcStrict = IEMExecOne(pVCpu);
+        LogFlow(("emR3NemExecuteIOInstruction: %Rrc (IEMExecOne)\n", VBOXSTRICTRC_VAL(rcStrict)));
+        STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIoIem);
+    }
+    else
+    {
+        RT_UNTRUSTED_VALIDATED_FENCE();
+        rcStrict = EMHistoryExec(pVCpu, &pVCpu->em.s.aExitRecords[idxContinueExitRec], 0);
+        LogFlow(("emR3NemExecuteIOInstruction: %Rrc (EMHistoryExec)\n", VBOXSTRICTRC_VAL(rcStrict)));
+        STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIoRestarted);
+    }
+
+    STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * Process NEM specific forced actions.
+ *
+ * This function is called when any FFs in VM_FF_HIGH_PRIORITY_PRE_RAW_MASK
+ * or/and VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK are pending.
+ *
+ * @returns VBox status code. May return VINF_EM_NO_MEMORY but none of the other
+ *          EM statuses.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+static int emR3NemForcedActions(PVM pVM, PVMCPU pVCpu)
+{
+    /*
+     * Sync page directory should not happen in NEM mode.
+     */
+    if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
+    {
+        Log(("NEM: TODO: Make VMCPU_FF_PGM_SYNC_CR3 / VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL quiet! (%#RX64)\n", (uint64_t)pVCpu->fLocalForcedActions));
+        VMCPU_FF_CLEAR_MASK(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+    }
+
+    /*
+     * Allocate handy pages (just in case the above actions have consumed some pages).
+     */
+    if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PGM_NEED_HANDY_PAGES, VM_FF_PGM_NO_MEMORY))
+    {
+        int rc = PGMR3PhysAllocateHandyPages(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Check whether we're out of memory now.
+     *
+     * This may stem from some of the above actions or operations that has been executed
+     * since we ran FFs. The allocate handy pages must for instance always be followed by
+     * this check.
+     */
+    if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+        return VINF_EM_NO_MEMORY;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Executes hardware accelerated raw code. (Intel VT-x & AMD-V)
+ *
+ * This function contains the raw-mode version of the inner
+ * execution loop (the outer loop being in EMR3ExecuteVM()).
+ *
+ * @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE, VINF_EM_RESCHEDULE_RAW,
+ *          VINF_EM_RESCHEDULE_REM, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pfFFDone    Where to store an indicator telling whether or not
+ *                      FFs were done before returning.
+ */
+VBOXSTRICTRC emR3NemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
+{
+    VBOXSTRICTRC rcStrict = VERR_IPE_UNINITIALIZED_STATUS;
+
+    LogFlow(("emR3NemExecute%d: (cs:eip=%04x:%RGv)\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, (RTGCPTR)pVCpu->cpum.GstCtx.rip));
+    *pfFFDone = false;
+
+    STAM_REL_COUNTER_INC(&pVCpu->em.s.StatNEMExecuteCalled);
+
+    /*
+     * Spin till we get a forced action which returns anything but VINF_SUCCESS.
+     */
+    for (;;)
+    {
+        STAM_PROFILE_ADV_START(&pVCpu->em.s.StatNEMEntry, a);
+
+#if 0
+        /* Check if a forced reschedule is pending. */
+        if (NEMR3IsRescheduleRequired(pVCpu))
+        {
+            rcStrict = VINF_EM_RESCHEDULE;
+            break;
+        }
+#endif
+
+        /*
+         * Process high priority pre-execution raw-mode FFs.
+         */
+        if (   VM_FF_IS_ANY_SET(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
+            || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
+        {
+            rcStrict = emR3NemForcedActions(pVM, pVCpu);
+            if (rcStrict != VINF_SUCCESS)
+                break;
+        }
+
+#ifdef LOG_ENABLED
+        /*
+         * Log important stuff before entering GC.
+         */
+        if (TRPMHasTrap(pVCpu))
+            Log(("CPU%d: Pending hardware interrupt=0x%x cs:rip=%04X:%RGv\n", pVCpu->idCpu, TRPMGetTrapNo(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, (RTGCPTR)pVCpu->cpum.GstCtx.rip));
+
+        if (!(pVCpu->cpum.GstCtx.fExtrn & (  CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_CS  | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
+                                           | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4    | CPUMCTX_EXTRN_EFER)))
+        {
+            uint32_t cpl = CPUMGetGuestCPL(pVCpu);
+            if (pVM->cCpus == 1)
+            {
+                if (pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
+                    Log(("NEMV86: %08x IF=%d\n", pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.Bits.u1IF));
+                else if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
+                    Log(("NEMR%d: %04x:%RGv ESP=%RGv IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->cpum.GstCtx.cs.Sel, (RTGCPTR)pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.Bits.u1IF, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, (uint32_t)pVCpu->cpum.GstCtx.cr0, (uint32_t)pVCpu->cpum.GstCtx.cr4, (uint32_t)pVCpu->cpum.GstCtx.msrEFER));
+                else
+                    Log(("NEMR%d: %04x:%08x ESP=%08X IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->cpum.GstCtx.cs.Sel,          pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.eflags.Bits.u1IF, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, (uint32_t)pVCpu->cpum.GstCtx.cr0, (uint32_t)pVCpu->cpum.GstCtx.cr4, (uint32_t)pVCpu->cpum.GstCtx.msrEFER));
+            }
+            else
+            {
+                if (pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
+                    Log(("NEMV86-CPU%d: %08x IF=%d\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.Bits.u1IF));
+                else if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
+                    Log(("NEMR%d-CPU%d: %04x:%RGv ESP=%RGv IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, (RTGCPTR)pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.Bits.u1IF, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, (uint32_t)pVCpu->cpum.GstCtx.cr0, (uint32_t)pVCpu->cpum.GstCtx.cr4, (uint32_t)pVCpu->cpum.GstCtx.msrEFER));
+                else
+                    Log(("NEMR%d-CPU%d: %04x:%08x ESP=%08X IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel,          pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.eflags.Bits.u1IF, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, (uint32_t)pVCpu->cpum.GstCtx.cr0, (uint32_t)pVCpu->cpum.GstCtx.cr4, (uint32_t)pVCpu->cpum.GstCtx.msrEFER));
+            }
+        }
+        else if (pVM->cCpus == 1)
+            Log(("NEMRx: -> NEMR3RunGC\n"));
+        else
+            Log(("NEMRx-CPU%u: -> NEMR3RunGC\n",  pVCpu->idCpu));
+#endif /* LOG_ENABLED */
+
+        /*
+         * Execute the code.
+         */
+        if (RT_LIKELY(emR3IsExecutionAllowed(pVM, pVCpu)))
+        {
+            STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatNEMEntry, a);
+            STAM_REL_PROFILE_START(&pVCpu->em.s.StatNEMExec, x);
+            rcStrict = NEMR3RunGC(pVM, pVCpu);
+            STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatNEMExec, x);
+        }
+        else
+        {
+            /* Give up this time slice; virtual time continues */
+            STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatNEMEntry, a);
+            STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatCapped, u);
+            RTThreadSleep(5);
+            STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatCapped, u);
+            rcStrict = VINF_SUCCESS;
+        }
+
+
+        /*
+         * Deal with high priority post execution FFs before doing anything else.
+         */
+        VMCPU_FF_CLEAR_MASK(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
+        if (   VM_FF_IS_ANY_SET(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
+            || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
+            rcStrict = emR3HighPriorityPostForcedActions(pVM, pVCpu, rcStrict);
+
+        /*
+         * Process the returned status code.
+         */
+        if (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
+            break;
+
+        rcStrict = emR3NemHandleRC(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict));
+        if (rcStrict != VINF_SUCCESS)
+            break;
+
+        /*
+         * Check and execute forced actions.
+         */
+#ifdef VBOX_HIGH_RES_TIMERS_HACK
+        TMTimerPollVoid(pVM, pVCpu);
+#endif
+        if (   VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK)
+            || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_ALL_MASK))
+        {
+            rcStrict = emR3ForcedActions(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict));
+            VBOXVMM_EM_FF_ALL_RET(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+            if (   rcStrict != VINF_SUCCESS
+                && rcStrict != VINF_EM_RESCHEDULE_HM)
+            {
+                *pfFFDone = true;
+                break;
+            }
+        }
+    }
+
+    /*
+     * Return to outer loop, making sure the fetch all state as we leave.
+     *
+     * Note! Not using CPUM_IMPORT_EXTRN_RET here, to prioritize an rcStrict error
+     *       status over import errors.
+     */
+    if (pVCpu->cpum.GstCtx.fExtrn)
+    {
+        int rcImport = NEMImportStateOnDemand(pVCpu, pVCpu->cpum.GstCtx.fExtrn);
+        AssertReturn(RT_SUCCESS(rcImport) || RT_FAILURE_NP(rcStrict), rcImport);
+    }
+#if defined(LOG_ENABLED) && defined(DEBUG)
+    RTLogFlush(NULL);
+#endif
+    return rcStrict;
+}
+
diff --git a/src/VBox/VMM/VMMR3/GIM.cpp b/src/VBox/VMM/VMMR3/GIM.cpp
new file mode 100644
index 00000000..2be825cf
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/GIM.cpp
@@ -0,0 +1,696 @@
+/* $Id: GIM.cpp $ */
+/** @file
+ * GIM - Guest Interface Manager.
+ */
+
+/*
+ * Copyright (C) 2014-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_gim        GIM - The Guest Interface Manager
+ *
+ * The Guest Interface Manager abstracts an interface provider through which
+ * guests may interact with the hypervisor.
+ *
+ * @see grp_gim
+ *
+ *
+ * @section sec_gim_provider   Providers
+ *
+ * A GIM provider implements a particular hypervisor interface such as Microsoft
+ * Hyper-V, Linux KVM and so on. It hooks into various components in the VMM to
+ * ease the guest in running under a recognized, virtualized environment.
+ *
+ * The GIM provider configured for the VM needs to be recognized by the guest OS
+ * in order to make use of features supported by the interface. Since it
+ * requires co-operation from the guest OS, a GIM provider may also be referred to
+ * as a paravirtualization interface.
+ *
+ * One of the goals of having a paravirtualized interface is for enabling guests
+ * to be more accurate and efficient when operating in a virtualized
+ * environment. For instance, a guest OS which interfaces to VirtualBox through
+ * a GIM provider may rely on the provider for supplying the correct TSC
+ * frequency of the host processor. The guest can then avoid caliberating the
+ * TSC itself, resulting in higher accuracy and better performance.
+ *
+ * At most, only one GIM provider can be active for a running VM and cannot be
+ * changed during the lifetime of the VM.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/pdmdev.h>
+#include "GIMInternal.h"
+#include <VBox/vmm/vm.h>
+
+#include <VBox/log.h>
+
+#include <iprt/err.h>
+#include <iprt/semaphore.h>
+#include <iprt/string.h>
+
+/* Include all GIM providers. */
+#include "GIMMinimalInternal.h"
+#include "GIMHvInternal.h"
+#include "GIMKvmInternal.h"
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static FNSSMINTSAVEEXEC  gimR3Save;
+static FNSSMINTLOADEXEC  gimR3Load;
+static FNSSMINTLOADDONE  gimR3LoadDone;
+
+
+/**
+ * Initializes the GIM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) GIMR3Init(PVM pVM)
+{
+    LogFlow(("GIMR3Init\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompile(sizeof(pVM->gim.s) <= sizeof(pVM->gim.padding));
+    AssertCompile(sizeof(pVM->apCpusR3[0]->gim.s) <= sizeof(pVM->apCpusR3[0]->gim.padding));
+
+    /*
+     * Initialize members.
+     */
+    pVM->gim.s.hSemiReadOnlyMmio2Handler = NIL_PGMPHYSHANDLERTYPE;
+
+    /*
+     * Register the saved state data unit.
+     */
+    int rc = SSMR3RegisterInternal(pVM, "GIM", 0 /* uInstance */, GIM_SAVED_STATE_VERSION, sizeof(GIM),
+                                   NULL /* pfnLivePrep */, NULL /* pfnLiveExec */, NULL /* pfnLiveVote*/,
+                                   NULL /* pfnSavePrep */, gimR3Save,              NULL /* pfnSaveDone */,
+                                   NULL /* pfnLoadPrep */, gimR3Load,              gimR3LoadDone);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Read configuration.
+     */
+    PCFGMNODE pCfgNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "GIM/");
+
+    /*
+     * Validate the GIM settings.
+     */
+    rc = CFGMR3ValidateConfig(pCfgNode, "/GIM/", /* pszNode */
+                              "Provider"         /* pszValidValues */
+                              "|Version",
+                              "HyperV",          /* pszValidNodes */
+                              "GIM",             /* pszWho */
+                              0);                /* uInstance */
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /** @cfgm{/GIM/Provider, string}
+     * The name of the GIM provider. The default is "none". */
+    char szProvider[64];
+    rc = CFGMR3QueryStringDef(pCfgNode, "Provider", szProvider, sizeof(szProvider), "None");
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/GIM/Version, uint32_t}
+     * The interface version. The default is 0, which means "provide the most
+     * up-to-date implementation". */
+    uint32_t uVersion;
+    rc = CFGMR3QueryU32Def(pCfgNode, "Version", &uVersion, 0 /* default */);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Setup the GIM provider for this VM.
+     */
+    LogRel(("GIM: Using provider '%s' (Implementation version: %u)\n", szProvider, uVersion));
+    if (!RTStrCmp(szProvider, "None"))
+        pVM->gim.s.enmProviderId = GIMPROVIDERID_NONE;
+    else
+    {
+        pVM->gim.s.u32Version = uVersion;
+        /** @todo r=bird: Because u32Version is saved, it should be translated to the
+         *        'most up-to-date implementation' version number when 0. Otherwise,
+         *        we'll have abiguities when loading the state of older VMs. */
+        if (!RTStrCmp(szProvider, "Minimal"))
+        {
+            pVM->gim.s.enmProviderId = GIMPROVIDERID_MINIMAL;
+            rc = gimR3MinimalInit(pVM);
+        }
+        else if (!RTStrCmp(szProvider, "HyperV"))
+        {
+            pVM->gim.s.enmProviderId = GIMPROVIDERID_HYPERV;
+            rc = gimR3HvInit(pVM, pCfgNode);
+        }
+        else if (!RTStrCmp(szProvider, "KVM"))
+        {
+            pVM->gim.s.enmProviderId = GIMPROVIDERID_KVM;
+            rc = gimR3KvmInit(pVM);
+        }
+        else
+            rc = VMR3SetError(pVM->pUVM, VERR_GIM_INVALID_PROVIDER, RT_SRC_POS, "Provider '%s' unknown.", szProvider);
+    }
+
+    /*
+     * Statistics.
+     */
+    STAM_REL_REG_USED(pVM, &pVM->gim.s.StatDbgXmit,      STAMTYPE_COUNTER, "/GIM/Debug/Transmit",      STAMUNIT_OCCURENCES, "Debug packets sent.");
+    STAM_REL_REG_USED(pVM, &pVM->gim.s.StatDbgXmitBytes, STAMTYPE_COUNTER, "/GIM/Debug/TransmitBytes", STAMUNIT_OCCURENCES, "Debug bytes sent.");
+    STAM_REL_REG_USED(pVM, &pVM->gim.s.StatDbgRecv,      STAMTYPE_COUNTER, "/GIM/Debug/Receive",       STAMUNIT_OCCURENCES, "Debug packets received.");
+    STAM_REL_REG_USED(pVM, &pVM->gim.s.StatDbgRecvBytes, STAMTYPE_COUNTER, "/GIM/Debug/ReceiveBytes",  STAMUNIT_OCCURENCES, "Debug bytes received.");
+
+    STAM_REL_REG_USED(pVM, &pVM->gim.s.StatHypercalls,   STAMTYPE_COUNTER, "/GIM/Hypercalls",          STAMUNIT_OCCURENCES, "Number of hypercalls initiated.");
+    return rc;
+}
+
+
+/**
+ * Initializes the remaining bits of the GIM provider.
+ *
+ * This is called after initializing HM and most other VMM components.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @thread  EMT(0)
+ */
+VMMR3_INT_DECL(int) GIMR3InitCompleted(PVM pVM)
+{
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_MINIMAL:
+            return gimR3MinimalInitCompleted(pVM);
+
+        case GIMPROVIDERID_HYPERV:
+            return gimR3HvInitCompleted(pVM);
+
+        case GIMPROVIDERID_KVM:
+            return gimR3KvmInitCompleted(pVM);
+
+        default:
+            break;
+    }
+
+    if (!TMR3CpuTickIsFixedRateMonotonic(pVM, true /* fWithParavirtEnabled */))
+        LogRel(("GIM: Warning!!! Host TSC is unstable. The guest may behave unpredictably with a paravirtualized clock.\n"));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTSAVEEXEC}
+ */
+static DECLCALLBACK(int) gimR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    AssertReturn(pVM,  VERR_INVALID_PARAMETER);
+    AssertReturn(pSSM, VERR_SSM_INVALID_STATE);
+
+    int rc = VINF_SUCCESS;
+#if 0
+    /* Save per-CPU data. */
+    SSMR3PutU32(pSSM, pVM->cCpus);
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        rc = SSMR3PutXYZ(pSSM, pVCpu->gim.s.XYZ);
+    }
+#endif
+
+    /*
+     * Save per-VM data.
+     */
+    SSMR3PutU32(pSSM, pVM->gim.s.enmProviderId);
+    SSMR3PutU32(pSSM, pVM->gim.s.u32Version);
+
+    /*
+     * Save provider-specific data.
+     */
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            rc = gimR3HvSave(pVM, pSSM);
+            AssertRCReturn(rc, rc);
+            break;
+
+        case GIMPROVIDERID_KVM:
+            rc = gimR3KvmSave(pVM, pSSM);
+            AssertRCReturn(rc, rc);
+            break;
+
+        default:
+            break;
+    }
+
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTLOADEXEC}
+ */
+static DECLCALLBACK(int) gimR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    if (uPass != SSM_PASS_FINAL)
+        return VINF_SUCCESS;
+    if (uVersion != GIM_SAVED_STATE_VERSION)
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+
+    int rc;
+#if 0
+    /* Load per-CPU data. */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        rc = SSMR3PutXYZ(pSSM, pVCpu->gim.s.XYZ);
+    }
+#endif
+
+    /*
+     * Load per-VM data.
+     */
+    uint32_t uProviderId;
+    uint32_t uProviderVersion;
+
+    SSMR3GetU32(pSSM, &uProviderId);
+    rc = SSMR3GetU32(pSSM, &uProviderVersion);
+    AssertRCReturn(rc, rc);
+
+    if ((GIMPROVIDERID)uProviderId != pVM->gim.s.enmProviderId)
+        return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Saved GIM provider %u differs from the configured one (%u)."),
+                                uProviderId, pVM->gim.s.enmProviderId);
+#if 0 /** @todo r=bird: Figure out what you mean to do here with the version. */
+    if (uProviderVersion != pVM->gim.s.u32Version)
+        return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Saved GIM provider version %u differs from the configured one (%u)."),
+                                uProviderVersion, pVM->gim.s.u32Version);
+#else
+    pVM->gim.s.u32Version = uProviderVersion;
+#endif
+
+    /*
+     * Load provider-specific data.
+     */
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            rc = gimR3HvLoad(pVM, pSSM);
+            AssertRCReturn(rc, rc);
+            break;
+
+        case GIMPROVIDERID_KVM:
+            rc = gimR3KvmLoad(pVM, pSSM);
+            AssertRCReturn(rc, rc);
+            break;
+
+        default:
+            break;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTLOADDONE}
+ */
+static DECLCALLBACK(int) gimR3LoadDone(PVM pVM, PSSMHANDLE pSSM)
+{
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimR3HvLoadDone(pVM, pSSM);
+
+        default:
+            return VINF_SUCCESS;
+    }
+}
+
+
+/**
+ * Terminates the GIM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM itself is, at this point, powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) GIMR3Term(PVM pVM)
+{
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimR3HvTerm(pVM);
+
+        case GIMPROVIDERID_KVM:
+            return gimR3KvmTerm(pVM);
+
+        default:
+            break;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3_INT_DECL(void) GIMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            gimR3HvRelocate(pVM, offDelta);
+            break;
+
+        default:
+            break;
+    }
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * For the GIM component this means unmapping and unregistering MMIO2 regions
+ * and other provider-specific resets.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) GIMR3Reset(PVM pVM)
+{
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimR3HvReset(pVM);
+
+        case GIMPROVIDERID_KVM:
+            return gimR3KvmReset(pVM);
+
+        default:
+            break;
+    }
+}
+
+
+/**
+ * Registers the GIM device with VMM.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         Pointer to the GIM device instance.
+ * @param   pDbg            Pointer to the GIM device debug structure, can be
+ *                          NULL.
+ */
+VMMR3DECL(void) GIMR3GimDeviceRegister(PVM pVM, PPDMDEVINS pDevIns, PGIMDEBUG pDbg)
+{
+    pVM->gim.s.pDevInsR3 = pDevIns;
+    pVM->gim.s.pDbgR3    = pDbg;
+}
+
+
+/**
+ * Gets debug setup specified by the provider.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDbgSetup       Where to store the debug setup details.
+ */
+VMMR3DECL(int) GIMR3GetDebugSetup(PVM pVM, PGIMDEBUGSETUP pDbgSetup)
+{
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+    AssertReturn(pDbgSetup, VERR_INVALID_PARAMETER);
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimR3HvGetDebugSetup(pVM, pDbgSetup);
+        default:
+            break;
+    }
+    return VERR_GIM_NO_DEBUG_CONNECTION;
+}
+
+
+/**
+ * Read data from a host debug session.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pvRead              The read buffer.
+ * @param   pcbRead             The size of the read buffer as well as where to store
+ *                              the number of bytes read.
+ * @param   pfnReadComplete     Callback when the buffer has been read and
+ *                              before signalling reading of the next buffer.
+ *                              Optional, can be NULL.
+ * @thread  EMT.
+ */
+VMMR3_INT_DECL(int) gimR3DebugRead(PVM pVM, void *pvRead, size_t *pcbRead, PFNGIMDEBUGBUFREADCOMPLETED pfnReadComplete)
+{
+    PGIMDEBUG pDbg = pVM->gim.s.pDbgR3;
+    if (pDbg)
+    {
+        if (ASMAtomicReadBool(&pDbg->fDbgRecvBufRead) == true)
+        {
+            STAM_REL_COUNTER_INC(&pVM->gim.s.StatDbgRecv);
+            STAM_REL_COUNTER_ADD(&pVM->gim.s.StatDbgRecvBytes, pDbg->cbDbgRecvBufRead);
+
+            memcpy(pvRead, pDbg->pvDbgRecvBuf, pDbg->cbDbgRecvBufRead);
+            *pcbRead = pDbg->cbDbgRecvBufRead;
+            if (pfnReadComplete)
+                pfnReadComplete(pVM);
+            RTSemEventMultiSignal(pDbg->hDbgRecvThreadSem);
+            ASMAtomicWriteBool(&pDbg->fDbgRecvBufRead, false);
+            return VINF_SUCCESS;
+        }
+        else
+            *pcbRead = 0;
+        return VERR_NO_DATA;
+    }
+    return VERR_GIM_NO_DEBUG_CONNECTION;
+}
+
+
+/**
+ * Write data to a host debug session.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pvWrite     The write buffer.
+ * @param   pcbWrite    The size of the write buffer as well as where to store
+ *                      the number of bytes written.
+ * @thread  EMT.
+ */
+VMMR3_INT_DECL(int) gimR3DebugWrite(PVM pVM, void *pvWrite, size_t *pcbWrite)
+{
+    PGIMDEBUG pDbg = pVM->gim.s.pDbgR3;
+    if (pDbg)
+    {
+        PPDMISTREAM pDbgStream = pDbg->pDbgDrvStream;
+        if (pDbgStream)
+        {
+            size_t cbWrite = *pcbWrite;
+            int rc = pDbgStream->pfnWrite(pDbgStream, pvWrite, pcbWrite);
+            if (   RT_SUCCESS(rc)
+                && *pcbWrite == cbWrite)
+            {
+                STAM_REL_COUNTER_INC(&pVM->gim.s.StatDbgXmit);
+                STAM_REL_COUNTER_ADD(&pVM->gim.s.StatDbgXmitBytes, *pcbWrite);
+            }
+            return rc;
+        }
+    }
+    return VERR_GIM_NO_DEBUG_CONNECTION;
+}
+
+#if 0 /* ??? */
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER,
+ *      Write access handler for mapped MMIO2 pages.  Currently ignores writes.}
+ *
+ * @todo In the future we might want to let the GIM provider decide what the
+ *       handler should do (like throwing \#GP faults).
+ */
+static DECLCALLBACK(VBOXSTRICTRC) gimR3Mmio2WriteHandler(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf,
+                                                         size_t cbBuf, PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin,
+                                                         void *pvUser)
+{
+    RT_NOREF6(pVM, pVCpu, GCPhys, pvPhys, pvBuf, cbBuf);
+    RT_NOREF3(enmAccessType, enmOrigin, pvUser);
+
+    /*
+     * Ignore writes to the mapped MMIO2 page.
+     */
+    Assert(enmAccessType == PGMACCESSTYPE_WRITE);
+    return VINF_SUCCESS;        /** @todo Hyper-V says we should \#GP(0) fault for writes to the Hypercall and TSC page. */
+}
+
+
+/**
+ * Unmaps a registered MMIO2 region in the guest address space and removes any
+ * access handlers for it.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pRegion     Pointer to the GIM MMIO2 region.
+ */
+VMMR3_INT_DECL(int) gimR3Mmio2Unmap(PVM pVM, PGIMMMIO2REGION pRegion)
+{
+    AssertPtr(pVM);
+    AssertPtr(pRegion);
+
+    PPDMDEVINS pDevIns = pVM->gim.s.pDevInsR3;
+    AssertPtr(pDevIns);
+    if (pRegion->fMapped)
+    {
+        int rc = PGMHandlerPhysicalDeregister(pVM, pRegion->GCPhysPage);
+        AssertRC(rc);
+
+        rc = PDMDevHlpMMIO2Unmap(pDevIns, pRegion->iRegion, pRegion->GCPhysPage);
+        if (RT_SUCCESS(rc))
+        {
+            pRegion->fMapped    = false;
+            pRegion->GCPhysPage = NIL_RTGCPHYS;
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Maps a registered MMIO2 region in the guest address space.
+ *
+ * The region will be made read-only and writes from the guest will be ignored.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pRegion         Pointer to the GIM MMIO2 region.
+ * @param   GCPhysRegion    Where in the guest address space to map the region.
+ */
+VMMR3_INT_DECL(int) GIMR3Mmio2Map(PVM pVM, PGIMMMIO2REGION pRegion, RTGCPHYS GCPhysRegion)
+{
+    PPDMDEVINS pDevIns = pVM->gim.s.pDevInsR3;
+    AssertPtr(pDevIns);
+
+    /* The guest-physical address must be page-aligned. */
+    if (GCPhysRegion & PAGE_OFFSET_MASK)
+    {
+        LogFunc(("%s: %#RGp not paging aligned\n", pRegion->szDescription, GCPhysRegion));
+        return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+    }
+
+    /* Allow only normal pages to be overlaid using our MMIO2 pages (disallow MMIO, ROM, reserved pages). */
+    /** @todo Hyper-V doesn't seem to be very strict about this, may be relax
+     *        later if some guest really requires it. */
+    if (!PGMPhysIsGCPhysNormal(pVM, GCPhysRegion))
+    {
+        LogFunc(("%s: %#RGp is not normal memory\n", pRegion->szDescription, GCPhysRegion));
+        return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+    }
+
+    if (!pRegion->fRegistered)
+    {
+        LogFunc(("%s: Region has not been registered.\n", pRegion->szDescription));
+        return VERR_GIM_IPE_1;
+    }
+
+    /*
+     * Map the MMIO2 region over the specified guest-physical address.
+     */
+    int rc = PDMDevHlpMMIOExMap(pDevIns, NULL, pRegion->iRegion, GCPhysRegion);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Install access-handlers for the mapped page to prevent (ignore) writes to it
+         * from the guest.
+         */
+        if (pVM->gim.s.hSemiReadOnlyMmio2Handler == NIL_PGMPHYSHANDLERTYPE)
+            rc = PGMR3HandlerPhysicalTypeRegister(pVM, PGMPHYSHANDLERKIND_WRITE,
+                                                  gimR3Mmio2WriteHandler,
+                                                  NULL /* pszModR0 */, NULL /* pszHandlerR0 */, NULL /* pszPfHandlerR0 */,
+                                                  NULL /* pszModRC */, NULL /* pszHandlerRC */, NULL /* pszPfHandlerRC */,
+                                                  "GIM read-only MMIO2 handler",
+                                                  &pVM->gim.s.hSemiReadOnlyMmio2Handler);
+        if (RT_SUCCESS(rc))
+        {
+            rc = PGMHandlerPhysicalRegister(pVM,  GCPhysRegion, GCPhysRegion + (pRegion->cbRegion - 1),
+                                            pVM->gim.s.hSemiReadOnlyMmio2Handler,
+                                            NULL /* pvUserR3 */, NIL_RTR0PTR /* pvUserR0 */, NIL_RTRCPTR /* pvUserRC */,
+                                            pRegion->szDescription);
+            if (RT_SUCCESS(rc))
+            {
+                pRegion->fMapped    = true;
+                pRegion->GCPhysPage = GCPhysRegion;
+                return rc;
+            }
+        }
+
+        PDMDevHlpMMIO2Unmap(pDevIns, pRegion->iRegion, GCPhysRegion);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Registers the physical handler for the registered and mapped MMIO2 region.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pRegion     Pointer to the GIM MMIO2 region.
+ */
+VMMR3_INT_DECL(int) gimR3Mmio2HandlerPhysicalRegister(PVM pVM, PGIMMMIO2REGION pRegion)
+{
+    AssertPtr(pRegion);
+    AssertReturn(pRegion->fRegistered, VERR_GIM_IPE_2);
+    AssertReturn(pRegion->fMapped, VERR_GIM_IPE_3);
+
+    return PGMR3HandlerPhysicalRegister(pVM,
+                                        PGMPHYSHANDLERKIND_WRITE,
+                                        pRegion->GCPhysPage, pRegion->GCPhysPage + (pRegion->cbRegion - 1),
+                                        gimR3Mmio2WriteHandler,  NULL /* pvUserR3 */,
+                                        NULL /* pszModR0 */, NULL /* pszHandlerR0 */, NIL_RTR0PTR /* pvUserR0 */,
+                                        NULL /* pszModRC */, NULL /* pszHandlerRC */, NIL_RTRCPTR /* pvUserRC */,
+                                        pRegion->szDescription);
+}
+
+
+/**
+ * Deregisters the physical handler for the MMIO2 region.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pRegion     Pointer to the GIM MMIO2 region.
+ */
+VMMR3_INT_DECL(int) gimR3Mmio2HandlerPhysicalDeregister(PVM pVM, PGIMMMIO2REGION pRegion)
+{
+    return PGMHandlerPhysicalDeregister(pVM, pRegion->GCPhysPage);
+}
+
+#endif
+
diff --git a/src/VBox/VMM/VMMR3/GIMHv.cpp b/src/VBox/VMM/VMMR3/GIMHv.cpp
new file mode 100644
index 00000000..d2fb506e
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/GIMHv.cpp
@@ -0,0 +1,2287 @@
+/* $Id: GIMHv.cpp $ */
+/** @file
+ * GIM - Guest Interface Manager, Hyper-V implementation.
+ */
+
+/*
+ * Copyright (C) 2014-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/em.h>
+#include "GIMInternal.h"
+#include <VBox/vmm/vm.h>
+
+#include <VBox/err.h>
+#include <VBox/version.h>
+
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/mem.h>
+#include <iprt/semaphore.h>
+#include <iprt/spinlock.h>
+#ifdef DEBUG_ramshankar
+# include <iprt/udp.h>
+#endif
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/**
+ * GIM Hyper-V saved-state version.
+ */
+#define GIM_HV_SAVED_STATE_VERSION                      UINT32_C(4)
+/** Saved states, priot to saving debug UDP source/destination ports.  */
+#define GIM_HV_SAVED_STATE_VERSION_PRE_DEBUG_UDP_PORTS  UINT32_C(3)
+/** Saved states, prior to any synthetic interrupt controller support. */
+#define GIM_HV_SAVED_STATE_VERSION_PRE_SYNIC            UINT32_C(2)
+/** Vanilla saved states, prior to any debug support. */
+#define GIM_HV_SAVED_STATE_VERSION_PRE_DEBUG            UINT32_C(1)
+
+#ifdef VBOX_WITH_STATISTICS
+# define GIMHV_MSRRANGE(a_uFirst, a_uLast, a_szName) \
+    { (a_uFirst), (a_uLast), kCpumMsrRdFn_Gim, kCpumMsrWrFn_Gim, 0, 0, 0, 0, 0, a_szName, { 0 }, { 0 }, { 0 }, { 0 } }
+#else
+# define GIMHV_MSRRANGE(a_uFirst, a_uLast, a_szName) \
+    { (a_uFirst), (a_uLast), kCpumMsrRdFn_Gim, kCpumMsrWrFn_Gim, 0, 0, 0, 0, 0, a_szName }
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/**
+ * Array of MSR ranges supported by Hyper-V.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_HyperV[] =
+{
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE0_FIRST,  MSR_GIM_HV_RANGE0_LAST,  "Hyper-V range 0"),
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE1_FIRST,  MSR_GIM_HV_RANGE1_LAST,  "Hyper-V range 1"),
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE2_FIRST,  MSR_GIM_HV_RANGE2_LAST,  "Hyper-V range 2"),
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE3_FIRST,  MSR_GIM_HV_RANGE3_LAST,  "Hyper-V range 3"),
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE4_FIRST,  MSR_GIM_HV_RANGE4_LAST,  "Hyper-V range 4"),
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE5_FIRST,  MSR_GIM_HV_RANGE5_LAST,  "Hyper-V range 5"),
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE6_FIRST,  MSR_GIM_HV_RANGE6_LAST,  "Hyper-V range 6"),
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE7_FIRST,  MSR_GIM_HV_RANGE7_LAST,  "Hyper-V range 7"),
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE8_FIRST,  MSR_GIM_HV_RANGE8_LAST,  "Hyper-V range 8"),
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE9_FIRST,  MSR_GIM_HV_RANGE9_LAST,  "Hyper-V range 9"),
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE10_FIRST, MSR_GIM_HV_RANGE10_LAST, "Hyper-V range 10"),
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE11_FIRST, MSR_GIM_HV_RANGE11_LAST, "Hyper-V range 11"),
+    GIMHV_MSRRANGE(MSR_GIM_HV_RANGE12_FIRST, MSR_GIM_HV_RANGE12_LAST, "Hyper-V range 12")
+};
+#undef GIMHV_MSRRANGE
+
+/**
+ * DHCP OFFER packet response to the guest (client) over the Hyper-V debug
+ * transport.
+ *
+ * - MAC: Destination: broadcast.
+ * - MAC: Source: 00:00:00:00:01 (hypervisor). It's important that it's
+ *   different from the client's MAC address which is all 0's.
+ * - IP: Source: 10.0.5.1 (hypervisor)
+ * - IP: Destination: broadcast.
+ * - IP: Checksum included.
+ * - BOOTP: Client IP address: 10.0.5.5.
+ * - BOOTP: Server IP address: 10.0.5.1.
+ * - DHCP options: Subnet mask, router, lease-time, DHCP server identifier.
+ *   Options are kept to a minimum required for making Windows guests happy.
+ */
+#define GIMHV_DEBUGCLIENT_IPV4          RT_H2N_U32_C(0x0a000505)    /* 10.0.5.5 */
+#define GIMHV_DEBUGSERVER_IPV4          RT_H2N_U32_C(0x0a000501)    /* 10.0.5.1 */
+static const uint8_t g_abDhcpOffer[] =
+{
+    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x45, 0x10,
+    0x01, 0x28, 0x00, 0x00, 0x00, 0x00, 0x40, 0x11, 0x6a, 0xb5, 0x0a, 0x00, 0x05, 0x01, 0xff, 0xff,
+    0xff, 0xff, 0x00, 0x43, 0x00, 0x44, 0x01, 0x14, 0x00, 0x00, 0x02, 0x01, 0x06, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x05, 0x05, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x82, 0x53, 0x63, 0x35, 0x01, 0x02, 0x01, 0x04, 0xff,
+    0xff, 0xff, 0x00, 0x03, 0x04, 0x0a, 0x00, 0x05, 0x01, 0x33, 0x04, 0xff, 0xff, 0xff, 0xff, 0x36,
+    0x04, 0x0a, 0x00, 0x05, 0x01, 0xff
+};
+
+/**
+ * DHCP ACK packet response to the guest (client) over the Hyper-V debug
+ * transport.
+ *
+ * - MAC: Destination: 00:00:00:00:00 (client).
+ * - IP: Destination: 10.0.5.5 (client).
+ * - Rest are mostly similar to the DHCP offer.
+ */
+static const uint8_t g_abDhcpAck[] =
+{
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x45, 0x10,
+    0x01, 0x28, 0x00, 0x00, 0x00, 0x00, 0x40, 0x11, 0x5b, 0xb0, 0x0a, 0x00, 0x05, 0x01, 0x0a, 0x00,
+    0x05, 0x05, 0x00, 0x43, 0x00, 0x44, 0x01, 0x14, 0x00, 0x00, 0x02, 0x01, 0x06, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x05, 0x05, 0x0a, 0x00, 0x05, 0x05, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x82, 0x53, 0x63, 0x35, 0x01, 0x05, 0x01, 0x04, 0xff,
+    0xff, 0xff, 0x00, 0x03, 0x04, 0x0a, 0x00, 0x05, 0x01, 0x33, 0x04, 0xff, 0xff, 0xff, 0xff, 0x36,
+    0x04, 0x0a, 0x00, 0x05, 0x01, 0xff
+};
+
+/**
+ * ARP reply to the guest (client) over the Hyper-V debug transport.
+ *
+ * - MAC: Destination: 00:00:00:00:00 (client)
+ * - MAC: Source: 00:00:00:00:01 (hypervisor)
+ * - ARP: Reply: 10.0.5.1 is at Source MAC address.
+ */
+static const uint8_t g_abArpReply[] =
+{
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x01,
+    0x08, 0x00, 0x06, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x0a, 0x00, 0x05, 0x01,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x05, 0x05
+};
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int    gimR3HvInitHypercallSupport(PVM pVM);
+static void   gimR3HvTermHypercallSupport(PVM pVM);
+static int    gimR3HvInitDebugSupport(PVM pVM);
+static void   gimR3HvTermDebugSupport(PVM pVM);
+static DECLCALLBACK(void) gimR3HvTimerCallback(PVM pVM, PTMTIMER pTimer, void *pvUser);
+
+/**
+ * Initializes the Hyper-V GIM provider.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pGimCfg     The GIM CFGM node.
+ */
+VMMR3_INT_DECL(int) gimR3HvInit(PVM pVM, PCFGMNODE pGimCfg)
+{
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+    AssertReturn(pVM->gim.s.enmProviderId == GIMPROVIDERID_HYPERV, VERR_INTERNAL_ERROR_5);
+
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+
+    /*
+     * Read configuration.
+     */
+    PCFGMNODE pCfgHv = CFGMR3GetChild(pGimCfg, "HyperV");
+    if (pCfgHv)
+    {
+        /*
+         * Validate the Hyper-V settings.
+         */
+        int rc2 = CFGMR3ValidateConfig(pCfgHv, "/HyperV/",
+                                  "VendorID"
+                                  "|VSInterface"
+                                  "|HypercallDebugInterface",
+                                  "" /* pszValidNodes */, "GIM/HyperV" /* pszWho */, 0 /* uInstance */);
+        if (RT_FAILURE(rc2))
+            return rc2;
+    }
+
+    /** @cfgm{/GIM/HyperV/VendorID, string, 'VBoxVBoxVBox'}
+     * The Hyper-V vendor signature, must be 12 characters. */
+    char szVendor[13];
+    int rc = CFGMR3QueryStringDef(pCfgHv, "VendorID", szVendor, sizeof(szVendor), "VBoxVBoxVBox");
+    AssertLogRelRCReturn(rc, rc);
+    AssertLogRelMsgReturn(strlen(szVendor) == 12,
+                          ("The VendorID config value must be exactly 12 chars, '%s' isn't!\n", szVendor),
+                          VERR_INVALID_PARAMETER);
+
+    LogRel(("GIM: HyperV: Reporting vendor as '%s'\n", szVendor));
+    /** @todo r=bird: GIM_HV_VENDOR_MICROSOFT is 12 char and the string is max
+     *        12+terminator, so the NCmp is a little bit misleading. */
+    if (!RTStrNCmp(szVendor, GIM_HV_VENDOR_MICROSOFT, sizeof(GIM_HV_VENDOR_MICROSOFT) - 1))
+    {
+        LogRel(("GIM: HyperV: Warning! Posing as the Microsoft vendor may alter guest behaviour!\n"));
+        pHv->fIsVendorMsHv = true;
+    }
+
+    /** @cfgm{/GIM/HyperV/VSInterface, bool, true}
+     * The Microsoft virtualization service interface (debugging). */
+    rc = CFGMR3QueryBoolDef(pCfgHv, "VSInterface", &pHv->fIsInterfaceVs, false);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/GIM/HyperV/HypercallDebugInterface, bool, false}
+     * Whether we specify the guest to use hypercalls for debugging rather than MSRs. */
+    rc = CFGMR3QueryBoolDef(pCfgHv, "HypercallDebugInterface", &pHv->fDbgHypercallInterface, false);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Determine interface capabilities based on the version.
+     */
+    if (!pVM->gim.s.u32Version)
+    {
+        /* Basic features. */
+        pHv->uBaseFeat = 0
+                     //| GIM_HV_BASE_FEAT_VP_RUNTIME_MSR
+                       | GIM_HV_BASE_FEAT_PART_TIME_REF_COUNT_MSR
+                     //| GIM_HV_BASE_FEAT_BASIC_SYNIC_MSRS          // Both required for synethetic timers
+                     //| GIM_HV_BASE_FEAT_STIMER_MSRS               // Both required for synethetic timers
+                       | GIM_HV_BASE_FEAT_APIC_ACCESS_MSRS
+                       | GIM_HV_BASE_FEAT_HYPERCALL_MSRS
+                       | GIM_HV_BASE_FEAT_VP_ID_MSR
+                       | GIM_HV_BASE_FEAT_VIRT_SYS_RESET_MSR
+                     //| GIM_HV_BASE_FEAT_STAT_PAGES_MSR
+                       | GIM_HV_BASE_FEAT_PART_REF_TSC_MSR
+                     //| GIM_HV_BASE_FEAT_GUEST_IDLE_STATE_MSR
+                       | GIM_HV_BASE_FEAT_TIMER_FREQ_MSRS
+                     //| GIM_HV_BASE_FEAT_DEBUG_MSRS
+                       ;
+
+        /* Miscellaneous features. */
+        pHv->uMiscFeat = 0
+                       //| GIM_HV_MISC_FEAT_GUEST_DEBUGGING
+                       //| GIM_HV_MISC_FEAT_XMM_HYPERCALL_INPUT
+                         | GIM_HV_MISC_FEAT_TIMER_FREQ
+                         | GIM_HV_MISC_FEAT_GUEST_CRASH_MSRS
+                       //| GIM_HV_MISC_FEAT_DEBUG_MSRS
+                         ;
+
+        /* Hypervisor recommendations to the guest. */
+        pHv->uHyperHints = GIM_HV_HINT_MSR_FOR_SYS_RESET
+                         | GIM_HV_HINT_RELAX_TIME_CHECKS
+                         | GIM_HV_HINT_X2APIC_MSRS
+                         ;
+
+        /* Partition features. */
+        pHv->uPartFlags |= GIM_HV_PART_FLAGS_EXTENDED_HYPERCALLS;
+
+        /* Expose more if we're posing as Microsoft. We can, if needed, force MSR-based Hv
+           debugging by not exposing these bits while exposing the VS interface. The better
+           way is what we do currently, via the GIM_HV_DEBUG_OPTIONS_USE_HYPERCALLS bit. */
+        if (pHv->fIsVendorMsHv)
+        {
+            pHv->uMiscFeat  |= GIM_HV_MISC_FEAT_GUEST_DEBUGGING
+                            |  GIM_HV_MISC_FEAT_DEBUG_MSRS;
+
+            pHv->uPartFlags |= GIM_HV_PART_FLAGS_DEBUGGING;
+        }
+    }
+
+    /*
+     * Populate the required fields in MMIO2 region records for registering.
+     */
+    for (size_t i = 0; i < RT_ELEMENTS(pHv->aMmio2Regions); i++)
+        pHv->aMmio2Regions[i].hMmio2 = NIL_PGMMMIO2HANDLE;
+
+    AssertCompile(GIM_HV_PAGE_SIZE == PAGE_SIZE);
+    PGIMMMIO2REGION pRegion = &pHv->aMmio2Regions[GIM_HV_HYPERCALL_PAGE_REGION_IDX];
+    pRegion->iRegion    = GIM_HV_HYPERCALL_PAGE_REGION_IDX;
+    pRegion->fRCMapping = false;
+    pRegion->cbRegion   = PAGE_SIZE;  /* Sanity checked in gimR3HvLoad(), gimR3HvEnableTscPage() & gimR3HvEnableHypercallPage() */
+    pRegion->GCPhysPage = NIL_RTGCPHYS;
+    RTStrCopy(pRegion->szDescription, sizeof(pRegion->szDescription), "Hyper-V hypercall page");
+
+    pRegion = &pHv->aMmio2Regions[GIM_HV_REF_TSC_PAGE_REGION_IDX];
+    pRegion->iRegion    = GIM_HV_REF_TSC_PAGE_REGION_IDX;
+    pRegion->fRCMapping = false;
+    pRegion->cbRegion   = PAGE_SIZE;  /* Sanity checked in gimR3HvLoad(), gimR3HvEnableTscPage() & gimR3HvEnableHypercallPage() */
+    pRegion->GCPhysPage = NIL_RTGCPHYS;
+    RTStrCopy(pRegion->szDescription, sizeof(pRegion->szDescription), "Hyper-V TSC page");
+
+    /*
+     * Make sure the CPU ID bit are in accordance with the Hyper-V
+     * requirement and other paranoia checks.
+     * See "Requirements for implementing the Microsoft hypervisor interface" spec.
+     */
+    Assert(!(pHv->uPartFlags & (  GIM_HV_PART_FLAGS_CREATE_PART
+                                | GIM_HV_PART_FLAGS_ACCESS_MEMORY_POOL
+                                | GIM_HV_PART_FLAGS_ACCESS_PART_ID
+                                | GIM_HV_PART_FLAGS_ADJUST_MSG_BUFFERS
+                                | GIM_HV_PART_FLAGS_CREATE_PORT
+                                | GIM_HV_PART_FLAGS_ACCESS_STATS
+                                | GIM_HV_PART_FLAGS_CPU_MGMT
+                                | GIM_HV_PART_FLAGS_CPU_PROFILER)));
+    Assert((pHv->uBaseFeat & (GIM_HV_BASE_FEAT_HYPERCALL_MSRS | GIM_HV_BASE_FEAT_VP_ID_MSR))
+                          == (GIM_HV_BASE_FEAT_HYPERCALL_MSRS | GIM_HV_BASE_FEAT_VP_ID_MSR));
+#ifdef VBOX_STRICT
+    for (unsigned i = 0; i < RT_ELEMENTS(pHv->aMmio2Regions); i++)
+    {
+        PCGIMMMIO2REGION pCur = &pHv->aMmio2Regions[i];
+        Assert(!pCur->fRCMapping);
+        Assert(!pCur->fMapped);
+        Assert(pCur->GCPhysPage == NIL_RTGCPHYS);
+    }
+#endif
+
+    /*
+     * Expose HVP (Hypervisor Present) bit to the guest.
+     */
+    CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_HVP);
+
+    /*
+     * Modify the standard hypervisor leaves for Hyper-V.
+     */
+    CPUMCPUIDLEAF HyperLeaf;
+    RT_ZERO(HyperLeaf);
+    HyperLeaf.uLeaf        = UINT32_C(0x40000000);
+    if (   pHv->fIsVendorMsHv
+        && pHv->fIsInterfaceVs)
+        HyperLeaf.uEax     = UINT32_C(0x40000082); /* Since we expose 0x40000082 below for the Hyper-V PV-debugging case. */
+    else
+        HyperLeaf.uEax     = UINT32_C(0x40000006); /* Minimum value for Hyper-V default is 0x40000005. */
+    /*
+     * Don't report vendor as 'Microsoft Hv'[1] by default, see @bugref{7270#c152}.
+     * [1]: ebx=0x7263694d ('rciM') ecx=0x666f736f ('foso') edx=0x76482074 ('vH t')
+     */
+    {
+        uint32_t uVendorEbx;
+        uint32_t uVendorEcx;
+        uint32_t uVendorEdx;
+        uVendorEbx = ((uint32_t)szVendor[ 3]) << 24 | ((uint32_t)szVendor[ 2]) << 16 | ((uint32_t)szVendor[1]) << 8
+                    | (uint32_t)szVendor[ 0];
+        uVendorEcx = ((uint32_t)szVendor[ 7]) << 24 | ((uint32_t)szVendor[ 6]) << 16 | ((uint32_t)szVendor[5]) << 8
+                    | (uint32_t)szVendor[ 4];
+        uVendorEdx = ((uint32_t)szVendor[11]) << 24 | ((uint32_t)szVendor[10]) << 16 | ((uint32_t)szVendor[9]) << 8
+                    | (uint32_t)szVendor[ 8];
+        HyperLeaf.uEbx         = uVendorEbx;
+        HyperLeaf.uEcx         = uVendorEcx;
+        HyperLeaf.uEdx         = uVendorEdx;
+    }
+    rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+    AssertLogRelRCReturn(rc, rc);
+
+    HyperLeaf.uLeaf        = UINT32_C(0x40000001);
+    HyperLeaf.uEax         = 0x31237648;           /* 'Hv#1' */
+    HyperLeaf.uEbx         = 0;                    /* Reserved */
+    HyperLeaf.uEcx         = 0;                    /* Reserved */
+    HyperLeaf.uEdx         = 0;                    /* Reserved */
+    rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Add Hyper-V specific leaves.
+     */
+    HyperLeaf.uLeaf        = UINT32_C(0x40000002); /* MBZ until MSR_GIM_HV_GUEST_OS_ID is set by the guest. */
+    HyperLeaf.uEax         = 0;
+    HyperLeaf.uEbx         = 0;
+    HyperLeaf.uEcx         = 0;
+    HyperLeaf.uEdx         = 0;
+    rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+    AssertLogRelRCReturn(rc, rc);
+
+    HyperLeaf.uLeaf        = UINT32_C(0x40000003);
+    HyperLeaf.uEax         = pHv->uBaseFeat;
+    HyperLeaf.uEbx         = pHv->uPartFlags;
+    HyperLeaf.uEcx         = pHv->uPowMgmtFeat;
+    HyperLeaf.uEdx         = pHv->uMiscFeat;
+    rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+    AssertLogRelRCReturn(rc, rc);
+
+    HyperLeaf.uLeaf        = UINT32_C(0x40000004);
+    HyperLeaf.uEax         = pHv->uHyperHints;
+    HyperLeaf.uEbx         = 0xffffffff;
+    HyperLeaf.uEcx         = 0;
+    HyperLeaf.uEdx         = 0;
+    rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+    AssertLogRelRCReturn(rc, rc);
+
+    RT_ZERO(HyperLeaf);
+    HyperLeaf.uLeaf        = UINT32_C(0x40000005);
+    rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+    AssertLogRelRCReturn(rc, rc);
+
+    /* Leaf 0x40000006 is inserted in gimR3HvInitCompleted(). */
+
+    if (   pHv->fIsVendorMsHv
+        && pHv->fIsInterfaceVs)
+    {
+        HyperLeaf.uLeaf        = UINT32_C(0x40000080);
+        HyperLeaf.uEax         = 0;
+        HyperLeaf.uEbx         = 0x7263694d;        /* 'rciM' */
+        HyperLeaf.uEcx         = 0x666f736f;        /* 'foso'*/
+        HyperLeaf.uEdx         = 0x53562074;        /* 'SV t' */
+        rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+        AssertLogRelRCReturn(rc, rc);
+
+        HyperLeaf.uLeaf        = UINT32_C(0x40000081);
+        HyperLeaf.uEax         = 0x31235356;        /* '1#SV' */
+        HyperLeaf.uEbx         = 0;
+        HyperLeaf.uEcx         = 0;
+        HyperLeaf.uEdx         = 0;
+        rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+        AssertLogRelRCReturn(rc, rc);
+
+        HyperLeaf.uLeaf        = UINT32_C(0x40000082);
+        HyperLeaf.uEax         = RT_BIT_32(1);
+        HyperLeaf.uEbx         = 0;
+        HyperLeaf.uEcx         = 0;
+        HyperLeaf.uEdx         = 0;
+        rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+        AssertLogRelRCReturn(rc, rc);
+    }
+
+    /*
+     * Insert all MSR ranges of Hyper-V.
+     */
+    for (unsigned i = 0; i < RT_ELEMENTS(g_aMsrRanges_HyperV); i++)
+    {
+        int rc2 = CPUMR3MsrRangesInsert(pVM, &g_aMsrRanges_HyperV[i]);
+        AssertLogRelRCReturn(rc2, rc2);
+    }
+
+    /*
+     * Setup non-zero MSRs.
+     */
+    if (pHv->uMiscFeat & GIM_HV_MISC_FEAT_GUEST_CRASH_MSRS)
+        pHv->uCrashCtlMsr = MSR_GIM_HV_CRASH_CTL_NOTIFY;
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PGIMHVCPU pHvCpu = &pVM->apCpusR3[idCpu]->gim.s.u.HvCpu;
+        for (uint8_t idxSintMsr = 0; idxSintMsr < RT_ELEMENTS(pHvCpu->auSintMsrs); idxSintMsr++)
+            pHvCpu->auSintMsrs[idxSintMsr] = MSR_GIM_HV_SINT_MASKED;
+    }
+
+    /*
+     * Setup hypercall support.
+     */
+    rc = gimR3HvInitHypercallSupport(pVM);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Setup debug support.
+     */
+    rc = gimR3HvInitDebugSupport(pVM);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Setup up the per-VCPU synthetic timers.
+     */
+    if (   (pHv->uBaseFeat & GIM_HV_BASE_FEAT_STIMER_MSRS)
+        || (pHv->uBaseFeat & GIM_HV_BASE_FEAT_BASIC_SYNIC_MSRS))
+    {
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PVMCPU       pVCpu     = pVM->apCpusR3[idCpu];
+            PGIMHVCPU    pHvCpu    = &pVCpu->gim.s.u.HvCpu;
+
+            for (uint8_t idxStimer = 0; idxStimer < RT_ELEMENTS(pHvCpu->aStimers); idxStimer++)
+            {
+                PGIMHVSTIMER pHvStimer = &pHvCpu->aStimers[idxStimer];
+
+                /* Associate the synthetic timer with its corresponding VCPU. */
+                pHvStimer->idCpu     = pVCpu->idCpu;
+                pHvStimer->idxStimer = idxStimer;
+
+                /* Create the timer and associate the context pointers. */
+                RTStrPrintf(&pHvStimer->szTimerDesc[0], sizeof(pHvStimer->szTimerDesc), "Hyper-V[%u] Timer%u", pVCpu->idCpu,
+                            idxStimer);
+                rc = TMR3TimerCreateInternal(pVM, TMCLOCK_VIRTUAL_SYNC, gimR3HvTimerCallback, pHvStimer /* pvUser */,
+                                             pHvStimer->szTimerDesc, &pHvStimer->pTimerR3);
+                AssertLogRelRCReturn(rc, rc);
+                pHvStimer->pTimerR0 = TMTimerR0Ptr(pHvStimer->pTimerR3);
+            }
+        }
+    }
+
+    /*
+     * Register statistics.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU    pVCpu  = pVM->apCpusR3[idCpu];
+        PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+
+        for (size_t idxStimer = 0; idxStimer < RT_ELEMENTS(pHvCpu->aStatStimerFired); idxStimer++)
+        {
+            int rc2 = STAMR3RegisterF(pVM, &pHvCpu->aStatStimerFired[idxStimer], STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                                     STAMUNIT_OCCURENCES, "Number of times the synthetic timer fired.",
+                                     "/GIM/HyperV/%u/Stimer%u_Fired", idCpu, idxStimer);
+            AssertLogRelRCReturn(rc2, rc2);
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Initializes remaining bits of the Hyper-V provider.
+ *
+ * This is called after initializing HM and almost all other VMM components.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) gimR3HvInitCompleted(PVM pVM)
+{
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    pHv->cTscTicksPerSecond = TMCpuTicksPerSecond(pVM);
+
+    /*
+     * Determine interface capabilities based on the version.
+     */
+    if (!pVM->gim.s.u32Version)
+    {
+        /* Hypervisor capabilities; features used by the hypervisor. */
+        pHv->uHyperCaps  = HMIsNestedPagingActive(pVM) ? GIM_HV_HOST_FEAT_NESTED_PAGING : 0;
+        pHv->uHyperCaps |= HMIsMsrBitmapActive(pVM)    ? GIM_HV_HOST_FEAT_MSR_BITMAP    : 0;
+    }
+
+    CPUMCPUIDLEAF HyperLeaf;
+    RT_ZERO(HyperLeaf);
+    HyperLeaf.uLeaf        = UINT32_C(0x40000006);
+    HyperLeaf.uEax         = pHv->uHyperCaps;
+    HyperLeaf.uEbx         = 0;
+    HyperLeaf.uEcx         = 0;
+    HyperLeaf.uEdx         = 0;
+    int rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Inform APIC whether Hyper-V compatibility mode is enabled or not.
+     * Do this here rather than on gimR3HvInit() as it gets called after APIC
+     * has finished inserting/removing the x2APIC MSR range.
+     */
+    if (pHv->uHyperHints & GIM_HV_HINT_X2APIC_MSRS)
+        APICR3HvSetCompatMode(pVM, true);
+
+    return rc;
+}
+
+
+/**
+ * Terminates the Hyper-V GIM provider.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) gimR3HvTerm(PVM pVM)
+{
+    gimR3HvReset(pVM);
+    gimR3HvTermHypercallSupport(pVM);
+    gimR3HvTermDebugSupport(pVM);
+
+    PCGIMHV pHv = &pVM->gim.s.u.Hv;
+    if (   (pHv->uBaseFeat & GIM_HV_BASE_FEAT_STIMER_MSRS)
+        || (pHv->uBaseFeat & GIM_HV_BASE_FEAT_BASIC_SYNIC_MSRS))
+    {
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PGIMHVCPU pHvCpu = &pVM->apCpusR3[idCpu]->gim.s.u.HvCpu;
+            for (uint8_t idxStimer = 0; idxStimer < RT_ELEMENTS(pHvCpu->aStimers); idxStimer++)
+            {
+                PGIMHVSTIMER pHvStimer = &pHvCpu->aStimers[idxStimer];
+                TMR3TimerDestroy(pHvStimer->pTimerR3);
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3_INT_DECL(void) gimR3HvRelocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    RT_NOREF(pVM, offDelta);
+}
+
+
+/**
+ * This resets Hyper-V provider MSRs and unmaps whatever Hyper-V regions that
+ * the guest may have mapped.
+ *
+ * This is called when the VM is being reset.
+ *
+ * @param   pVM     The cross context VM structure.
+ *
+ * @thread  EMT(0)
+ */
+VMMR3_INT_DECL(void) gimR3HvReset(PVM pVM)
+{
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Unmap MMIO2 pages that the guest may have setup.
+     */
+    LogRel(("GIM: HyperV: Resetting MMIO2 regions and MSRs\n"));
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    for (unsigned i = 0; i < RT_ELEMENTS(pHv->aMmio2Regions); i++)
+    {
+        PGIMMMIO2REGION pRegion = &pHv->aMmio2Regions[i];
+#if 0
+        gimR3Mmio2Unmap(pVM, pRegion);
+#else
+        pRegion->fMapped    = false;
+        pRegion->GCPhysPage = NIL_RTGCPHYS;
+#endif
+    }
+
+    /*
+     * Reset MSRs.
+     */
+    pHv->u64GuestOsIdMsr      = 0;
+    pHv->u64HypercallMsr      = 0;
+    pHv->u64TscPageMsr        = 0;
+    pHv->uCrashP0Msr          = 0;
+    pHv->uCrashP1Msr          = 0;
+    pHv->uCrashP2Msr          = 0;
+    pHv->uCrashP3Msr          = 0;
+    pHv->uCrashP4Msr          = 0;
+    pHv->uDbgStatusMsr        = 0;
+    pHv->uDbgPendingBufferMsr = 0;
+    pHv->uDbgSendBufferMsr    = 0;
+    pHv->uDbgRecvBufferMsr    = 0;
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PGIMHVCPU pHvCpu = &pVM->apCpusR3[idCpu]->gim.s.u.HvCpu;
+        pHvCpu->uSControlMsr = 0;
+        pHvCpu->uSimpMsr  = 0;
+        pHvCpu->uSiefpMsr = 0;
+        pHvCpu->uApicAssistPageMsr = 0;
+
+        for (uint8_t idxSint = 0; idxSint < RT_ELEMENTS(pHvCpu->auSintMsrs); idxSint++)
+            pHvCpu->auSintMsrs[idxSint] = MSR_GIM_HV_SINT_MASKED;
+
+        for (uint8_t idxStimer = 0; idxStimer < RT_ELEMENTS(pHvCpu->aStimers); idxStimer++)
+        {
+            PGIMHVSTIMER pHvStimer = &pHvCpu->aStimers[idxStimer];
+            pHvStimer->uStimerConfigMsr = 0;
+            pHvStimer->uStimerCountMsr  = 0;
+        }
+    }
+}
+
+
+/**
+ * Callback for when debug data is available over the debugger connection.
+ *
+ * @param pVM       The cross context VM structure.
+ */
+static DECLCALLBACK(void) gimR3HvDebugBufAvail(PVM pVM)
+{
+    PGIMHV   pHv = &pVM->gim.s.u.Hv;
+    RTGCPHYS GCPhysPendingBuffer = pHv->uDbgPendingBufferMsr;
+    if (   GCPhysPendingBuffer
+        && PGMPhysIsGCPhysNormal(pVM, GCPhysPendingBuffer))
+    {
+        uint8_t bPendingData = 1;
+        int rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysPendingBuffer, &bPendingData, sizeof(bPendingData));
+        if (RT_FAILURE(rc))
+        {
+            LogRelMax(5, ("GIM: HyperV: Failed to set pending debug receive buffer at %#RGp, rc=%Rrc\n", GCPhysPendingBuffer,
+                          rc));
+        }
+    }
+}
+
+
+/**
+ * Callback for when debug data has been read from the debugger connection.
+ *
+ * This will be invoked before signalling read of the next debug buffer.
+ *
+ * @param pVM       The cross context VM structure.
+ */
+static DECLCALLBACK(void) gimR3HvDebugBufReadCompleted(PVM pVM)
+{
+    PGIMHV   pHv = &pVM->gim.s.u.Hv;
+    RTGCPHYS GCPhysPendingBuffer = pHv->uDbgPendingBufferMsr;
+    if (   GCPhysPendingBuffer
+        && PGMPhysIsGCPhysNormal(pVM, GCPhysPendingBuffer))
+    {
+        uint8_t bPendingData = 0;
+        int rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysPendingBuffer, &bPendingData, sizeof(bPendingData));
+        if (RT_FAILURE(rc))
+        {
+            LogRelMax(5, ("GIM: HyperV: Failed to clear pending debug receive buffer at %#RGp, rc=%Rrc\n", GCPhysPendingBuffer,
+                          rc));
+        }
+    }
+}
+
+
+/**
+ * Get Hyper-V debug setup parameters.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDbgSetup   Where to store the debug setup details.
+ */
+VMMR3_INT_DECL(int) gimR3HvGetDebugSetup(PVM pVM, PGIMDEBUGSETUP pDbgSetup)
+{
+    Assert(pDbgSetup);
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    if (pHv->fDbgEnabled)
+    {
+        pDbgSetup->pfnDbgRecvBufAvail = gimR3HvDebugBufAvail;
+        pDbgSetup->cbDbgRecvBuf       = GIM_HV_PAGE_SIZE;
+        return VINF_SUCCESS;
+    }
+    return VERR_GIM_NO_DEBUG_CONNECTION;
+}
+
+
+/**
+ * Hyper-V state-save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   pSSM    The saved state handle.
+ */
+VMMR3_INT_DECL(int) gimR3HvSave(PVM pVM, PSSMHANDLE pSSM)
+{
+    PCGIMHV pHv = &pVM->gim.s.u.Hv;
+
+    /*
+     * Save the Hyper-V SSM version.
+     */
+    SSMR3PutU32(pSSM, GIM_HV_SAVED_STATE_VERSION);
+
+    /*
+     * Save per-VM MSRs.
+     */
+    SSMR3PutU64(pSSM, pHv->u64GuestOsIdMsr);
+    SSMR3PutU64(pSSM, pHv->u64HypercallMsr);
+    SSMR3PutU64(pSSM, pHv->u64TscPageMsr);
+
+    /*
+     * Save Hyper-V features / capabilities.
+     */
+    SSMR3PutU32(pSSM, pHv->uBaseFeat);
+    SSMR3PutU32(pSSM, pHv->uPartFlags);
+    SSMR3PutU32(pSSM, pHv->uPowMgmtFeat);
+    SSMR3PutU32(pSSM, pHv->uMiscFeat);
+    SSMR3PutU32(pSSM, pHv->uHyperHints);
+    SSMR3PutU32(pSSM, pHv->uHyperCaps);
+
+    /*
+     * Save the Hypercall region.
+     */
+    PCGIMMMIO2REGION pRegion = &pHv->aMmio2Regions[GIM_HV_HYPERCALL_PAGE_REGION_IDX];
+    SSMR3PutU8(pSSM,     pRegion->iRegion);
+    SSMR3PutBool(pSSM,   pRegion->fRCMapping);
+    SSMR3PutU32(pSSM,    pRegion->cbRegion);
+    SSMR3PutGCPhys(pSSM, pRegion->GCPhysPage);
+    SSMR3PutStrZ(pSSM,   pRegion->szDescription);
+
+    /*
+     * Save the reference TSC region.
+     */
+    pRegion = &pHv->aMmio2Regions[GIM_HV_REF_TSC_PAGE_REGION_IDX];
+    SSMR3PutU8(pSSM,     pRegion->iRegion);
+    SSMR3PutBool(pSSM,   pRegion->fRCMapping);
+    SSMR3PutU32(pSSM,    pRegion->cbRegion);
+    SSMR3PutGCPhys(pSSM, pRegion->GCPhysPage);
+    SSMR3PutStrZ(pSSM,   pRegion->szDescription);
+    /* Save the TSC sequence so we can bump it on restore (as the CPU frequency/offset may change). */
+    uint32_t uTscSequence = 0;
+    if (   pRegion->fMapped
+        && MSR_GIM_HV_REF_TSC_IS_ENABLED(pHv->u64TscPageMsr))
+    {
+        PCGIMHVREFTSC pRefTsc = (PCGIMHVREFTSC)pRegion->pvPageR3;
+        uTscSequence = pRefTsc->u32TscSequence;
+    }
+    SSMR3PutU32(pSSM, uTscSequence);
+
+    /*
+     * Save debug support data.
+     */
+    SSMR3PutU64(pSSM, pHv->uDbgPendingBufferMsr);
+    SSMR3PutU64(pSSM, pHv->uDbgSendBufferMsr);
+    SSMR3PutU64(pSSM, pHv->uDbgRecvBufferMsr);
+    SSMR3PutU64(pSSM, pHv->uDbgStatusMsr);
+    SSMR3PutU32(pSSM, pHv->enmDbgReply);
+    SSMR3PutU32(pSSM, pHv->uDbgBootpXId);
+    SSMR3PutU32(pSSM, pHv->DbgGuestIp4Addr.u);
+    SSMR3PutU16(pSSM, pHv->uUdpGuestDstPort);
+    SSMR3PutU16(pSSM, pHv->uUdpGuestSrcPort);
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PGIMHVCPU pHvCpu = &pVM->apCpusR3[idCpu]->gim.s.u.HvCpu;
+        SSMR3PutU64(pSSM, pHvCpu->uSimpMsr);
+        for (size_t idxSintMsr = 0; idxSintMsr < RT_ELEMENTS(pHvCpu->auSintMsrs); idxSintMsr++)
+            SSMR3PutU64(pSSM, pHvCpu->auSintMsrs[idxSintMsr]);
+    }
+
+    return SSMR3PutU8(pSSM, UINT8_MAX);
+}
+
+
+/**
+ * Hyper-V state-load operation, final pass.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The saved state handle.
+ */
+VMMR3_INT_DECL(int) gimR3HvLoad(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Load the Hyper-V SSM version first.
+     */
+    uint32_t uHvSavedStateVersion;
+    int rc = SSMR3GetU32(pSSM, &uHvSavedStateVersion);
+    AssertRCReturn(rc, rc);
+    if (   uHvSavedStateVersion != GIM_HV_SAVED_STATE_VERSION
+        && uHvSavedStateVersion != GIM_HV_SAVED_STATE_VERSION_PRE_DEBUG_UDP_PORTS
+        && uHvSavedStateVersion != GIM_HV_SAVED_STATE_VERSION_PRE_SYNIC
+        && uHvSavedStateVersion != GIM_HV_SAVED_STATE_VERSION_PRE_DEBUG)
+        return SSMR3SetLoadError(pSSM, VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION, RT_SRC_POS,
+                                 N_("Unsupported Hyper-V saved-state version %u (current %u)!"),
+                                 uHvSavedStateVersion, GIM_HV_SAVED_STATE_VERSION);
+
+    /*
+     * Update the TSC frequency from TM.
+     */
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    pHv->cTscTicksPerSecond = TMCpuTicksPerSecond(pVM);
+
+    /*
+     * Load per-VM MSRs.
+     */
+    SSMR3GetU64(pSSM, &pHv->u64GuestOsIdMsr);
+    SSMR3GetU64(pSSM, &pHv->u64HypercallMsr);
+    SSMR3GetU64(pSSM, &pHv->u64TscPageMsr);
+
+    /*
+     * Load Hyper-V features / capabilities.
+     */
+    SSMR3GetU32(pSSM, &pHv->uBaseFeat);
+    SSMR3GetU32(pSSM, &pHv->uPartFlags);
+    SSMR3GetU32(pSSM, &pHv->uPowMgmtFeat);
+    SSMR3GetU32(pSSM, &pHv->uMiscFeat);
+    SSMR3GetU32(pSSM, &pHv->uHyperHints);
+    SSMR3GetU32(pSSM, &pHv->uHyperCaps);
+
+    /*
+     * Load and enable the Hypercall region.
+     */
+    PGIMMMIO2REGION pRegion = &pHv->aMmio2Regions[GIM_HV_HYPERCALL_PAGE_REGION_IDX];
+    SSMR3GetU8(pSSM,     &pRegion->iRegion);
+    SSMR3GetBool(pSSM,   &pRegion->fRCMapping);
+    SSMR3GetU32(pSSM,    &pRegion->cbRegion);
+    SSMR3GetGCPhys(pSSM, &pRegion->GCPhysPage);
+    rc = SSMR3GetStrZ(pSSM, pRegion->szDescription, sizeof(pRegion->szDescription));
+    AssertRCReturn(rc, rc);
+
+    if (pRegion->cbRegion != PAGE_SIZE)
+        return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Hypercall page region size %u invalid, expected %u"),
+                                pRegion->cbRegion, PAGE_SIZE);
+
+    if (MSR_GIM_HV_HYPERCALL_PAGE_IS_ENABLED(pHv->u64HypercallMsr))
+    {
+        Assert(pRegion->GCPhysPage != NIL_RTGCPHYS);
+        if (RT_LIKELY(pRegion->fRegistered))
+        {
+            rc = gimR3HvEnableHypercallPage(pVM, pRegion->GCPhysPage);
+            if (RT_FAILURE(rc))
+                return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Failed to enable the hypercall page. GCPhys=%#RGp rc=%Rrc"),
+                                        pRegion->GCPhysPage, rc);
+        }
+        else
+            return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Hypercall MMIO2 region not registered. Missing GIM device?!"));
+    }
+
+    /*
+     * Load and enable the reference TSC region.
+     */
+    uint32_t uTscSequence;
+    pRegion = &pHv->aMmio2Regions[GIM_HV_REF_TSC_PAGE_REGION_IDX];
+    SSMR3GetU8(pSSM,     &pRegion->iRegion);
+    SSMR3GetBool(pSSM,   &pRegion->fRCMapping);
+    SSMR3GetU32(pSSM,    &pRegion->cbRegion);
+    SSMR3GetGCPhys(pSSM, &pRegion->GCPhysPage);
+    SSMR3GetStrZ(pSSM,    pRegion->szDescription, sizeof(pRegion->szDescription));
+    rc = SSMR3GetU32(pSSM, &uTscSequence);
+    AssertRCReturn(rc, rc);
+
+    if (pRegion->cbRegion != PAGE_SIZE)
+        return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("TSC page region size %u invalid, expected %u"),
+                                pRegion->cbRegion, PAGE_SIZE);
+
+    if (MSR_GIM_HV_REF_TSC_IS_ENABLED(pHv->u64TscPageMsr))
+    {
+        Assert(pRegion->GCPhysPage != NIL_RTGCPHYS);
+        if (pRegion->fRegistered)
+        {
+            rc = gimR3HvEnableTscPage(pVM, pRegion->GCPhysPage, true /* fUseThisTscSeq */, uTscSequence);
+            if (RT_FAILURE(rc))
+                return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Failed to enable the TSC page. GCPhys=%#RGp rc=%Rrc"),
+                                        pRegion->GCPhysPage, rc);
+        }
+        else
+            return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("TSC-page MMIO2 region not registered. Missing GIM device?!"));
+    }
+
+    /*
+     * Load the debug support data.
+     */
+    if (uHvSavedStateVersion > GIM_HV_SAVED_STATE_VERSION_PRE_DEBUG)
+    {
+        SSMR3GetU64(pSSM, &pHv->uDbgPendingBufferMsr);
+        SSMR3GetU64(pSSM, &pHv->uDbgSendBufferMsr);
+        SSMR3GetU64(pSSM, &pHv->uDbgRecvBufferMsr);
+        SSMR3GetU64(pSSM, &pHv->uDbgStatusMsr);
+        SSM_GET_ENUM32_RET(pSSM, pHv->enmDbgReply, GIMHVDEBUGREPLY);
+        SSMR3GetU32(pSSM, &pHv->uDbgBootpXId);
+        rc = SSMR3GetU32(pSSM, &pHv->DbgGuestIp4Addr.u);
+        AssertRCReturn(rc, rc);
+        if (uHvSavedStateVersion > GIM_HV_SAVED_STATE_VERSION_PRE_DEBUG_UDP_PORTS)
+        {
+            rc = SSMR3GetU16(pSSM, &pHv->uUdpGuestDstPort);     AssertRCReturn(rc, rc);
+            rc = SSMR3GetU16(pSSM, &pHv->uUdpGuestSrcPort);     AssertRCReturn(rc, rc);
+        }
+
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PGIMHVCPU pHvCpu = &pVM->apCpusR3[idCpu]->gim.s.u.HvCpu;
+            SSMR3GetU64(pSSM, &pHvCpu->uSimpMsr);
+            if (uHvSavedStateVersion <= GIM_HV_SAVED_STATE_VERSION_PRE_SYNIC)
+                SSMR3GetU64(pSSM, &pHvCpu->auSintMsrs[GIM_HV_VMBUS_MSG_SINT]);
+            else
+            {
+                for (uint8_t idxSintMsr = 0; idxSintMsr < RT_ELEMENTS(pHvCpu->auSintMsrs); idxSintMsr++)
+                    SSMR3GetU64(pSSM, &pHvCpu->auSintMsrs[idxSintMsr]);
+            }
+        }
+
+        uint8_t bDelim;
+        rc = SSMR3GetU8(pSSM, &bDelim);
+    }
+    else
+        rc = VINF_SUCCESS;
+
+    return rc;
+}
+
+
+/**
+ * Hyper-V load-done callback.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The saved state handle.
+ */
+VMMR3_INT_DECL(int) gimR3HvLoadDone(PVM pVM, PSSMHANDLE pSSM)
+{
+    if (RT_SUCCESS(SSMR3HandleGetStatus(pSSM)))
+    {
+        /*
+         * Update EM on whether MSR_GIM_HV_GUEST_OS_ID allows hypercall instructions.
+         */
+        if (pVM->gim.s.u.Hv.u64GuestOsIdMsr)
+            for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                EMSetHypercallInstructionsEnabled(pVM->apCpusR3[idCpu], true);
+        else
+            for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                EMSetHypercallInstructionsEnabled(pVM->apCpusR3[idCpu], false);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Enables the Hyper-V APIC-assist page.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   GCPhysApicAssistPage    Where to map the APIC-assist page.
+ */
+VMMR3_INT_DECL(int) gimR3HvEnableApicAssistPage(PVMCPU pVCpu, RTGCPHYS GCPhysApicAssistPage)
+{
+    PVM             pVM     = pVCpu->CTX_SUFF(pVM);
+    PPDMDEVINSR3    pDevIns = pVM->gim.s.pDevInsR3;
+    AssertPtrReturn(pDevIns, VERR_GIM_DEVICE_NOT_REGISTERED);
+
+    /*
+     * Map the APIC-assist-page at the specified address.
+     */
+    /** @todo this is buggy when large pages are used due to a PGM limitation, see
+     *        @bugref{7532}. Instead of the overlay style mapping, we just
+     *        rewrite guest memory directly. */
+    size_t const cbApicAssistPage = PAGE_SIZE;
+    void *pvApicAssist = RTMemAllocZ(cbApicAssistPage);
+    if (RT_LIKELY(pvApicAssist))
+    {
+        int rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysApicAssistPage, pvApicAssist, cbApicAssistPage);
+        if (RT_SUCCESS(rc))
+        {
+            /** @todo Inform APIC. */
+            LogRel(("GIM%u: HyperV: Enabled APIC-assist page at %#RGp\n", pVCpu->idCpu, GCPhysApicAssistPage));
+        }
+        else
+        {
+            LogRelFunc(("GIM%u: HyperV: PGMPhysSimpleWriteGCPhys failed. rc=%Rrc\n", pVCpu->idCpu, rc));
+            rc = VERR_GIM_OPERATION_FAILED;
+        }
+
+        RTMemFree(pvApicAssist);
+        return rc;
+    }
+
+    LogRelFunc(("GIM%u: HyperV: Failed to alloc %u bytes\n", pVCpu->idCpu, cbApicAssistPage));
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Disables the Hyper-V APIC-assist page.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMR3_INT_DECL(int) gimR3HvDisableApicAssistPage(PVMCPU pVCpu)
+{
+    LogRel(("GIM%u: HyperV: Disabled APIC-assist page\n", pVCpu->idCpu));
+    /** @todo inform APIC */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Hyper-V synthetic timer callback.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pTimer      Pointer to timer.
+ * @param   pvUser      Pointer to the synthetic timer.
+ */
+static DECLCALLBACK(void) gimR3HvTimerCallback(PVM pVM, PTMTIMER pTimer, void *pvUser)
+{
+    PGIMHVSTIMER pHvStimer = (PGIMHVSTIMER)pvUser;
+    Assert(pHvStimer);
+    Assert(TMTimerIsLockOwner(pTimer)); RT_NOREF(pTimer);
+    Assert(pHvStimer->idCpu < pVM->cCpus);
+
+    PVMCPU    pVCpu  = pVM->apCpusR3[pHvStimer->idCpu];
+    PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+    Assert(pHvStimer->idxStimer < RT_ELEMENTS(pHvCpu->aStatStimerFired));
+
+    STAM_COUNTER_INC(&pHvCpu->aStatStimerFired[pHvStimer->idxStimer]);
+
+    uint64_t const uStimerConfig = pHvStimer->uStimerConfigMsr;
+    uint16_t const idxSint       = MSR_GIM_HV_STIMER_GET_SINTX(uStimerConfig);
+    if (RT_LIKELY(idxSint < RT_ELEMENTS(pHvCpu->auSintMsrs)))
+    {
+        uint64_t const uSint = pHvCpu->auSintMsrs[idxSint];
+        if (!MSR_GIM_HV_SINT_IS_MASKED(uSint))
+        {
+            uint8_t const uVector  = MSR_GIM_HV_SINT_GET_VECTOR(uSint);
+            bool const    fAutoEoi = MSR_GIM_HV_SINT_IS_AUTOEOI(uSint);
+            APICHvSendInterrupt(pVCpu, uVector, fAutoEoi, XAPICTRIGGERMODE_EDGE);
+        }
+    }
+
+    /* Re-arm the timer if it's periodic. */
+    if (MSR_GIM_HV_STIMER_IS_PERIODIC(uStimerConfig))
+        gimHvStartStimer(pVCpu, pHvStimer);
+}
+
+
+/**
+ * Enables the Hyper-V SIEF page.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhysSiefPage  Where to map the SIEF page.
+ */
+VMMR3_INT_DECL(int) gimR3HvEnableSiefPage(PVMCPU pVCpu, RTGCPHYS GCPhysSiefPage)
+{
+    PVM             pVM     = pVCpu->CTX_SUFF(pVM);
+    PPDMDEVINSR3    pDevIns = pVM->gim.s.pDevInsR3;
+    AssertPtrReturn(pDevIns, VERR_GIM_DEVICE_NOT_REGISTERED);
+
+    /*
+     * Map the SIEF page at the specified address.
+     */
+    /** @todo this is buggy when large pages are used due to a PGM limitation, see
+     *        @bugref{7532}. Instead of the overlay style mapping, we just
+     *        rewrite guest memory directly. */
+    size_t const cbSiefPage = PAGE_SIZE;
+    void *pvSiefPage = RTMemAllocZ(cbSiefPage);
+    if (RT_LIKELY(pvSiefPage))
+    {
+        int rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysSiefPage, pvSiefPage, cbSiefPage);
+        if (RT_SUCCESS(rc))
+        {
+            /** @todo SIEF setup. */
+            LogRel(("GIM%u: HyperV: Enabled SIEF page at %#RGp\n", pVCpu->idCpu, GCPhysSiefPage));
+        }
+        else
+        {
+            LogRelFunc(("GIM%u: HyperV: PGMPhysSimpleWriteGCPhys failed. rc=%Rrc\n", pVCpu->idCpu, rc));
+            rc = VERR_GIM_OPERATION_FAILED;
+        }
+
+        RTMemFree(pvSiefPage);
+        return rc;
+    }
+
+    LogRelFunc(("GIM%u: HyperV: Failed to alloc %u bytes\n", pVCpu->idCpu, cbSiefPage));
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Disables the Hyper-V SIEF page.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMR3_INT_DECL(int) gimR3HvDisableSiefPage(PVMCPU pVCpu)
+{
+    LogRel(("GIM%u: HyperV: Disabled APIC-assist page\n", pVCpu->idCpu));
+    /** @todo SIEF teardown. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Enables the Hyper-V TSC page.
+ *
+ * @returns VBox status code.
+ * @param   pVM                The cross context VM structure.
+ * @param   GCPhysTscPage      Where to map the TSC page.
+ * @param   fUseThisTscSeq     Whether to set the TSC sequence number to the one
+ *                             specified in @a uTscSeq.
+ * @param   uTscSeq            The TSC sequence value to use. Ignored if
+ *                             @a fUseThisTscSeq is false.
+ */
+VMMR3_INT_DECL(int) gimR3HvEnableTscPage(PVM pVM, RTGCPHYS GCPhysTscPage, bool fUseThisTscSeq, uint32_t uTscSeq)
+{
+    PPDMDEVINSR3    pDevIns = pVM->gim.s.pDevInsR3;
+    PGIMMMIO2REGION pRegion = &pVM->gim.s.u.Hv.aMmio2Regions[GIM_HV_REF_TSC_PAGE_REGION_IDX];
+    AssertPtrReturn(pDevIns, VERR_GIM_DEVICE_NOT_REGISTERED);
+
+    int rc;
+    if (pRegion->fMapped)
+    {
+        /*
+         * Is it already enabled at the given guest-address?
+         */
+        if (pRegion->GCPhysPage == GCPhysTscPage)
+            return VINF_SUCCESS;
+
+        /*
+         * If it's mapped at a different address, unmap the previous address.
+         */
+        rc = gimR3HvDisableTscPage(pVM);
+        AssertRC(rc);
+    }
+
+    /*
+     * Map the TSC-page at the specified address.
+     */
+    Assert(!pRegion->fMapped);
+
+    /** @todo this is buggy when large pages are used due to a PGM limitation, see
+     *        @bugref{7532}. Instead of the overlay style mapping, we just
+     *        rewrite guest memory directly. */
+#if 0
+    rc = gimR3Mmio2Map(pVM, pRegion, GCPhysTscPage);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pRegion->GCPhysPage == GCPhysTscPage);
+
+        /*
+         * Update the TSC scale. Windows guests expect a non-zero TSC sequence, otherwise
+         * they fallback to using the reference count MSR which is not ideal in terms of VM-exits.
+         *
+         * Also, Hyper-V normalizes the time in 10 MHz, see:
+         * http://technet.microsoft.com/it-it/sysinternals/dn553408%28v=vs.110%29
+         */
+        PGIMHVREFTSC pRefTsc = (PGIMHVREFTSC)pRegion->pvPageR3;
+        Assert(pRefTsc);
+
+        PGIMHV pHv = &pVM->gim.s.u.Hv;
+        uint64_t const u64TscKHz = pHv->cTscTicksPerSecond / UINT64_C(1000);
+        uint32_t       u32TscSeq = 1;
+        if (   fUseThisTscSeq
+            && uTscSeq < UINT32_C(0xfffffffe))
+            u32TscSeq = uTscSeq + 1;
+        pRefTsc->u32TscSequence  = u32TscSeq;
+        pRefTsc->u64TscScale     = ((INT64_C(10000) << 32) / u64TscKHz) << 32;
+        pRefTsc->i64TscOffset    = 0;
+
+        LogRel(("GIM: HyperV: Enabled TSC page at %#RGp - u64TscScale=%#RX64 u64TscKHz=%#RX64 (%'RU64) Seq=%#RU32\n",
+                GCPhysTscPage, pRefTsc->u64TscScale, u64TscKHz, u64TscKHz, pRefTsc->u32TscSequence));
+
+        TMR3CpuTickParavirtEnable(pVM);
+        return VINF_SUCCESS;
+    }
+    else
+        LogRelFunc(("gimR3Mmio2Map failed. rc=%Rrc\n", rc));
+    return VERR_GIM_OPERATION_FAILED;
+#else
+    AssertReturn(pRegion->cbRegion == PAGE_SIZE, VERR_GIM_IPE_2);
+    PGIMHVREFTSC pRefTsc = (PGIMHVREFTSC)RTMemAllocZ(PAGE_SIZE);
+    if (RT_UNLIKELY(!pRefTsc))
+    {
+        LogRelFunc(("Failed to alloc %u bytes\n", PAGE_SIZE));
+        return VERR_NO_MEMORY;
+    }
+
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    uint64_t const u64TscKHz = pHv->cTscTicksPerSecond / UINT64_C(1000);
+    uint32_t       u32TscSeq = 1;
+    if (   fUseThisTscSeq
+        && uTscSeq < UINT32_C(0xfffffffe))
+        u32TscSeq = uTscSeq + 1;
+    pRefTsc->u32TscSequence  = u32TscSeq;
+    pRefTsc->u64TscScale     = ((INT64_C(10000) << 32) / u64TscKHz) << 32;
+    pRefTsc->i64TscOffset    = 0;
+
+    rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysTscPage, pRefTsc, sizeof(*pRefTsc));
+    if (RT_SUCCESS(rc))
+    {
+        LogRel(("GIM: HyperV: Enabled TSC page at %#RGp - u64TscScale=%#RX64 u64TscKHz=%#RX64 (%'RU64) Seq=%#RU32\n",
+                GCPhysTscPage, pRefTsc->u64TscScale, u64TscKHz, u64TscKHz, pRefTsc->u32TscSequence));
+
+        pRegion->GCPhysPage = GCPhysTscPage;
+        pRegion->fMapped = true;
+        TMR3CpuTickParavirtEnable(pVM);
+    }
+    else
+    {
+        LogRelFunc(("GIM: HyperV: PGMPhysSimpleWriteGCPhys failed. rc=%Rrc\n", rc));
+        rc = VERR_GIM_OPERATION_FAILED;
+    }
+    RTMemFree(pRefTsc);
+    return rc;
+#endif
+}
+
+
+/**
+ * Enables the Hyper-V SIM page.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhysSimPage   Where to map the SIM page.
+ */
+VMMR3_INT_DECL(int) gimR3HvEnableSimPage(PVMCPU pVCpu, RTGCPHYS GCPhysSimPage)
+{
+    PVM             pVM     = pVCpu->CTX_SUFF(pVM);
+    PPDMDEVINSR3    pDevIns = pVM->gim.s.pDevInsR3;
+    AssertPtrReturn(pDevIns, VERR_GIM_DEVICE_NOT_REGISTERED);
+
+    /*
+     * Map the SIMP page at the specified address.
+     */
+    /** @todo this is buggy when large pages are used due to a PGM limitation, see
+     *        @bugref{7532}. Instead of the overlay style mapping, we just
+     *        rewrite guest memory directly. */
+    size_t const cbSimPage = PAGE_SIZE;
+    void *pvSimPage = RTMemAllocZ(cbSimPage);
+    if (RT_LIKELY(pvSimPage))
+    {
+        int rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysSimPage, pvSimPage, cbSimPage);
+        if (RT_SUCCESS(rc))
+        {
+            /** @todo SIM setup. */
+            LogRel(("GIM%u: HyperV: Enabled SIM page at %#RGp\n", pVCpu->idCpu, GCPhysSimPage));
+        }
+        else
+        {
+            LogRelFunc(("GIM%u: HyperV: PGMPhysSimpleWriteGCPhys failed. rc=%Rrc\n", pVCpu->idCpu, rc));
+            rc = VERR_GIM_OPERATION_FAILED;
+        }
+
+        RTMemFree(pvSimPage);
+        return rc;
+    }
+
+    LogRelFunc(("GIM%u: HyperV: Failed to alloc %u bytes\n", pVCpu->idCpu, cbSimPage));
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Disables the Hyper-V SIM page.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMR3_INT_DECL(int) gimR3HvDisableSimPage(PVMCPU pVCpu)
+{
+    LogRel(("GIM%u: HyperV: Disabled SIM page\n", pVCpu->idCpu));
+    /** @todo SIM teardown. */
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Disables the Hyper-V TSC page.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) gimR3HvDisableTscPage(PVM pVM)
+{
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    PGIMMMIO2REGION pRegion = &pHv->aMmio2Regions[GIM_HV_REF_TSC_PAGE_REGION_IDX];
+    if (pRegion->fMapped)
+    {
+#if 0
+        gimR3Mmio2Unmap(pVM, pRegion);
+        Assert(!pRegion->fMapped);
+#else
+        pRegion->fMapped = false;
+#endif
+        LogRel(("GIM: HyperV: Disabled TSC page\n"));
+
+        TMR3CpuTickParavirtDisable(pVM);
+        return VINF_SUCCESS;
+    }
+    return VERR_GIM_PVTSC_NOT_ENABLED;
+}
+
+
+/**
+ * Disables the Hyper-V Hypercall page.
+ *
+ * @returns VBox status code.
+ */
+VMMR3_INT_DECL(int) gimR3HvDisableHypercallPage(PVM pVM)
+{
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    PGIMMMIO2REGION pRegion = &pHv->aMmio2Regions[GIM_HV_HYPERCALL_PAGE_REGION_IDX];
+    if (pRegion->fMapped)
+    {
+#if 0
+        gimR3Mmio2Unmap(pVM, pRegion);
+        Assert(!pRegion->fMapped);
+#else
+        pRegion->fMapped = false;
+#endif
+        LogRel(("GIM: HyperV: Disabled Hypercall-page\n"));
+        return VINF_SUCCESS;
+    }
+    return VERR_GIM_HYPERCALLS_NOT_ENABLED;
+}
+
+
+/**
+ * Enables the Hyper-V Hypercall page.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The cross context VM structure.
+ * @param   GCPhysHypercallPage     Where to map the hypercall page.
+ */
+VMMR3_INT_DECL(int) gimR3HvEnableHypercallPage(PVM pVM, RTGCPHYS GCPhysHypercallPage)
+{
+    PPDMDEVINSR3    pDevIns = pVM->gim.s.pDevInsR3;
+    PGIMMMIO2REGION pRegion = &pVM->gim.s.u.Hv.aMmio2Regions[GIM_HV_HYPERCALL_PAGE_REGION_IDX];
+    AssertPtrReturn(pDevIns, VERR_GIM_DEVICE_NOT_REGISTERED);
+
+    if (pRegion->fMapped)
+    {
+        /*
+         * Is it already enabled at the given guest-address?
+         */
+        if (pRegion->GCPhysPage == GCPhysHypercallPage)
+            return VINF_SUCCESS;
+
+        /*
+         * If it's mapped at a different address, unmap the previous address.
+         */
+        int rc2 = gimR3HvDisableHypercallPage(pVM);
+        AssertRC(rc2);
+    }
+
+    /*
+     * Map the hypercall-page at the specified address.
+     */
+    Assert(!pRegion->fMapped);
+
+    /** @todo this is buggy when large pages are used due to a PGM limitation, see
+     *        @bugref{7532}. Instead of the overlay style mapping, we just
+     *        rewrite guest memory directly. */
+#if 0
+    int rc = gimR3Mmio2Map(pVM, pRegion, GCPhysHypercallPage);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pRegion->GCPhysPage == GCPhysHypercallPage);
+
+        /*
+         * Patch the hypercall-page.
+         */
+        size_t cbWritten = 0;
+        rc = VMMPatchHypercall(pVM, pRegion->pvPageR3, PAGE_SIZE, &cbWritten);
+        if (   RT_SUCCESS(rc)
+            && cbWritten < PAGE_SIZE)
+        {
+            uint8_t *pbLast = (uint8_t *)pRegion->pvPageR3 + cbWritten;
+            *pbLast = 0xc3;  /* RET */
+
+            /*
+             * Notify VMM that hypercalls are now enabled for all VCPUs.
+             */
+            for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                VMMHypercallsEnable(pVM->apCpusR3[idCpu]);
+
+            LogRel(("GIM: HyperV: Enabled hypercall page at %#RGp\n", GCPhysHypercallPage));
+            return VINF_SUCCESS;
+        }
+        else
+        {
+            if (rc == VINF_SUCCESS)
+                rc = VERR_GIM_OPERATION_FAILED;
+            LogRel(("GIM: HyperV: VMMPatchHypercall failed. rc=%Rrc cbWritten=%u\n", rc, cbWritten));
+        }
+
+        gimR3Mmio2Unmap(pVM, pRegion);
+    }
+
+    LogRel(("GIM: HyperV: gimR3Mmio2Map failed. rc=%Rrc\n", rc));
+    return rc;
+#else
+    AssertReturn(pRegion->cbRegion == PAGE_SIZE, VERR_GIM_IPE_3);
+    void *pvHypercallPage = RTMemAllocZ(PAGE_SIZE);
+    if (RT_UNLIKELY(!pvHypercallPage))
+    {
+        LogRelFunc(("Failed to alloc %u bytes\n", PAGE_SIZE));
+        return VERR_NO_MEMORY;
+    }
+
+    /*
+     * Patch the hypercall-page.
+     */
+    size_t cbHypercall = 0;
+    int rc = GIMQueryHypercallOpcodeBytes(pVM, pvHypercallPage, PAGE_SIZE, &cbHypercall, NULL /*puDisOpcode*/);
+    if (   RT_SUCCESS(rc)
+        && cbHypercall < PAGE_SIZE)
+    {
+        uint8_t *pbLast = (uint8_t *)pvHypercallPage + cbHypercall;
+        *pbLast = 0xc3;  /* RET */
+
+        rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysHypercallPage, pvHypercallPage, PAGE_SIZE);
+        if (RT_SUCCESS(rc))
+        {
+            pRegion->GCPhysPage = GCPhysHypercallPage;
+            pRegion->fMapped = true;
+            LogRel(("GIM: HyperV: Enabled hypercall page at %#RGp\n", GCPhysHypercallPage));
+        }
+        else
+            LogRel(("GIM: HyperV: PGMPhysSimpleWriteGCPhys failed during hypercall page setup. rc=%Rrc\n", rc));
+    }
+    else
+    {
+        if (rc == VINF_SUCCESS)
+            rc = VERR_GIM_OPERATION_FAILED;
+        LogRel(("GIM: HyperV: VMMPatchHypercall failed. rc=%Rrc cbHypercall=%u\n", rc, cbHypercall));
+    }
+
+    RTMemFree(pvHypercallPage);
+    return rc;
+#endif
+}
+
+
+/**
+ * Initializes Hyper-V guest hypercall support.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+static int gimR3HvInitHypercallSupport(PVM pVM)
+{
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    pHv->pbHypercallIn = (uint8_t *)RTMemAllocZ(GIM_HV_PAGE_SIZE);
+    if (RT_LIKELY(pHv->pbHypercallIn))
+    {
+        pHv->pbHypercallOut = (uint8_t *)RTMemAllocZ(GIM_HV_PAGE_SIZE);
+        if (RT_LIKELY(pHv->pbHypercallOut))
+            return VINF_SUCCESS;
+        RTMemFree(pHv->pbHypercallIn);
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Terminates Hyper-V guest hypercall support.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+static void gimR3HvTermHypercallSupport(PVM pVM)
+{
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    RTMemFree(pHv->pbHypercallIn);
+    pHv->pbHypercallIn = NULL;
+
+    RTMemFree(pHv->pbHypercallOut);
+    pHv->pbHypercallOut = NULL;
+}
+
+
+/**
+ * Initializes Hyper-V guest debug support.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+static int gimR3HvInitDebugSupport(PVM pVM)
+{
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    if (   (pHv->uPartFlags & GIM_HV_PART_FLAGS_DEBUGGING)
+        || pHv->fIsInterfaceVs)
+    {
+        pHv->fDbgEnabled = true;
+        pHv->pvDbgBuffer = RTMemAllocZ(PAGE_SIZE);
+        if (!pHv->pvDbgBuffer)
+            return VERR_NO_MEMORY;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminates Hyper-V guest debug support.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+static void gimR3HvTermDebugSupport(PVM pVM)
+{
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    if (pHv->pvDbgBuffer)
+    {
+        RTMemFree(pHv->pvDbgBuffer);
+        pHv->pvDbgBuffer = NULL;
+    }
+}
+
+
+/**
+ * Reads data from a debugger connection, asynchronous.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvBuf       Where to read the data.
+ * @param   cbBuf       Size of the read buffer @a pvBuf, must be >= @a cbRead.
+ * @param   cbRead      Number of bytes to read.
+ * @param   pcbRead     Where to store how many bytes were really read.
+ * @param   cMsTimeout  Timeout of the read operation in milliseconds.
+ * @param   fUdpPkt     Whether the debug data returned in @a pvBuf needs to be
+ *                      encapsulated in a UDP frame.
+ *
+ * @thread  EMT.
+ */
+VMMR3_INT_DECL(int) gimR3HvDebugRead(PVM pVM, void *pvBuf, uint32_t cbBuf, uint32_t cbRead, uint32_t *pcbRead,
+                                     uint32_t cMsTimeout, bool fUdpPkt)
+{
+    NOREF(cMsTimeout);      /** @todo implement timeout. */
+    AssertCompile(sizeof(size_t) >= sizeof(uint32_t));
+    AssertReturn(cbBuf >= cbRead, VERR_INVALID_PARAMETER);
+
+    int rc;
+    if (!fUdpPkt)
+    {
+        /*
+         * Read the raw debug data.
+         */
+        size_t cbReallyRead = cbRead;
+        rc = gimR3DebugRead(pVM, pvBuf, &cbReallyRead, gimR3HvDebugBufReadCompleted);
+        *pcbRead = (uint32_t)cbReallyRead;
+    }
+    else
+    {
+        /*
+         * Guest requires UDP encapsulated frames.
+         */
+        PGIMHV pHv = &pVM->gim.s.u.Hv;
+        rc = VERR_GIM_IPE_1;
+        switch (pHv->enmDbgReply)
+        {
+            case GIMHVDEBUGREPLY_UDP:
+            {
+                size_t cbReallyRead = cbRead;
+                rc = gimR3DebugRead(pVM, pvBuf, &cbReallyRead, gimR3HvDebugBufReadCompleted);
+                if (   RT_SUCCESS(rc)
+                    && cbReallyRead > 0)
+                {
+                    uint8_t abFrame[sizeof(RTNETETHERHDR) + RTNETIPV4_MIN_LEN + sizeof(RTNETUDP)];
+                    if (cbReallyRead + sizeof(abFrame) <= cbBuf)
+                    {
+                        /*
+                         * Windows guests pumps ethernet frames over the Hyper-V debug connection as
+                         * explained in gimR3HvHypercallPostDebugData(). Here, we reconstruct the packet
+                         * with the guest's self-chosen IP ARP address we saved in pHv->DbgGuestAddr.
+                         *
+                         * Note! We really need to pass the minimum IPv4 header length. The Windows 10 guest
+                         * is -not- happy if we include the IPv4 options field, i.e. using sizeof(RTNETIPV4)
+                         * instead of RTNETIPV4_MIN_LEN.
+                         */
+                        RT_ZERO(abFrame);
+                        PRTNETETHERHDR pEthHdr = (PRTNETETHERHDR)&abFrame[0];
+                        PRTNETIPV4     pIpHdr  = (PRTNETIPV4)    (pEthHdr + 1);
+                        PRTNETUDP      pUdpHdr = (PRTNETUDP)     ((uint8_t *)pIpHdr + RTNETIPV4_MIN_LEN);
+
+                        /* Ethernet */
+                        pEthHdr->EtherType = RT_H2N_U16_C(RTNET_ETHERTYPE_IPV4);
+                        /* IPv4 */
+                        pIpHdr->ip_v       = 4;
+                        pIpHdr->ip_hl      = RTNETIPV4_MIN_LEN / sizeof(uint32_t);
+                        pIpHdr->ip_tos     = 0;
+                        pIpHdr->ip_len     = RT_H2N_U16((uint16_t)cbReallyRead + sizeof(RTNETUDP) + RTNETIPV4_MIN_LEN);
+                        pIpHdr->ip_id      = 0;
+                        pIpHdr->ip_off     = 0;
+                        pIpHdr->ip_ttl     = 255;
+                        pIpHdr->ip_p       = RTNETIPV4_PROT_UDP;
+                        pIpHdr->ip_sum     = 0;
+                        pIpHdr->ip_src.u   = 0;
+                        pIpHdr->ip_dst.u   = pHv->DbgGuestIp4Addr.u;
+                        pIpHdr->ip_sum     = RTNetIPv4HdrChecksum(pIpHdr);
+                        /* UDP */
+                        pUdpHdr->uh_dport  = pHv->uUdpGuestSrcPort;
+                        pUdpHdr->uh_sport  = pHv->uUdpGuestDstPort;
+                        pUdpHdr->uh_ulen   = RT_H2N_U16_C((uint16_t)cbReallyRead + sizeof(*pUdpHdr));
+
+                        /* Make room by moving the payload and prepending the headers. */
+                        uint8_t *pbData = (uint8_t *)pvBuf;
+                        memmove(pbData + sizeof(abFrame), pbData, cbReallyRead);
+                        memcpy(pbData, &abFrame[0], sizeof(abFrame));
+
+                        /* Update the adjusted sizes. */
+                        cbReallyRead += sizeof(abFrame);
+                    }
+                    else
+                        rc = VERR_BUFFER_UNDERFLOW;
+                }
+                *pcbRead = (uint32_t)cbReallyRead;
+                break;
+            }
+
+            case GIMHVDEBUGREPLY_ARP_REPLY:
+            {
+                uint32_t const cbArpReplyPkt =  sizeof(g_abArpReply);
+                if (cbBuf >= cbArpReplyPkt)
+                {
+                    memcpy(pvBuf, g_abArpReply, cbArpReplyPkt);
+                    rc = VINF_SUCCESS;
+                    *pcbRead = cbArpReplyPkt;
+                    pHv->enmDbgReply = GIMHVDEBUGREPLY_ARP_REPLY_SENT;
+                }
+                else
+                {
+                    rc = VERR_BUFFER_UNDERFLOW;
+                    *pcbRead = 0;
+                }
+                break;
+            }
+
+            case GIMHVDEBUGREPLY_DHCP_OFFER:
+            {
+                uint32_t const cbDhcpOfferPkt = sizeof(g_abDhcpOffer);
+                if (cbBuf >= cbDhcpOfferPkt)
+                {
+                    memcpy(pvBuf, g_abDhcpOffer, cbDhcpOfferPkt);
+                    PRTNETETHERHDR pEthHdr   = (PRTNETETHERHDR)pvBuf;
+                    PRTNETIPV4     pIpHdr    = (PRTNETIPV4)    (pEthHdr + 1);
+                    PRTNETUDP      pUdpHdr   = (PRTNETUDP)     ((uint8_t *)pIpHdr + RTNETIPV4_MIN_LEN);
+                    PRTNETBOOTP    pBootpHdr = (PRTNETBOOTP)   (pUdpHdr + 1);
+                    pBootpHdr->bp_xid = pHv->uDbgBootpXId;
+
+                    rc = VINF_SUCCESS;
+                    *pcbRead = cbDhcpOfferPkt;
+                    pHv->enmDbgReply = GIMHVDEBUGREPLY_DHCP_OFFER_SENT;
+                    LogRel(("GIM: HyperV: Debug DHCP offered IP address %RTnaipv4, transaction Id %#x\n", pBootpHdr->bp_yiaddr,
+                            RT_N2H_U32(pHv->uDbgBootpXId)));
+                }
+                else
+                {
+                    rc = VERR_BUFFER_UNDERFLOW;
+                    *pcbRead = 0;
+                }
+                break;
+            }
+
+            case GIMHVDEBUGREPLY_DHCP_ACK:
+            {
+                uint32_t const cbDhcpAckPkt = sizeof(g_abDhcpAck);
+                if (cbBuf >= cbDhcpAckPkt)
+                {
+                    memcpy(pvBuf, g_abDhcpAck, cbDhcpAckPkt);
+                    PRTNETETHERHDR pEthHdr   = (PRTNETETHERHDR)pvBuf;
+                    PRTNETIPV4     pIpHdr    = (PRTNETIPV4)    (pEthHdr + 1);
+                    PRTNETUDP      pUdpHdr   = (PRTNETUDP)     ((uint8_t *)pIpHdr + RTNETIPV4_MIN_LEN);
+                    PRTNETBOOTP    pBootpHdr = (PRTNETBOOTP)   (pUdpHdr + 1);
+                    pBootpHdr->bp_xid = pHv->uDbgBootpXId;
+
+                    rc = VINF_SUCCESS;
+                    *pcbRead = cbDhcpAckPkt;
+                    pHv->enmDbgReply = GIMHVDEBUGREPLY_DHCP_ACK_SENT;
+                    LogRel(("GIM: HyperV: Debug DHCP acknowledged IP address %RTnaipv4, transaction Id %#x\n",
+                            pBootpHdr->bp_yiaddr, RT_N2H_U32(pHv->uDbgBootpXId)));
+                }
+                else
+                {
+                    rc = VERR_BUFFER_UNDERFLOW;
+                    *pcbRead = 0;
+                }
+                break;
+            }
+
+            case GIMHVDEBUGREPLY_ARP_REPLY_SENT:
+            case GIMHVDEBUGREPLY_DHCP_OFFER_SENT:
+            case GIMHVDEBUGREPLY_DHCP_ACK_SENT:
+            {
+                rc = VINF_SUCCESS;
+                *pcbRead = 0;
+                break;
+            }
+
+            default:
+            {
+                AssertMsgFailed(("GIM: HyperV: Invalid/unimplemented debug reply type %u\n", pHv->enmDbgReply));
+                rc = VERR_INTERNAL_ERROR_2;
+            }
+        }
+        Assert(rc != VERR_GIM_IPE_1);
+
+#ifdef DEBUG_ramshankar
+        if (   rc == VINF_SUCCESS
+            && *pcbRead > 0)
+        {
+            RTSOCKET hSocket;
+            int rc2 = RTUdpCreateClientSocket("localhost", 52000, NULL, &hSocket);
+            if (RT_SUCCESS(rc2))
+            {
+                size_t cbTmpWrite = *pcbRead;
+                RTSocketWriteNB(hSocket, pvBuf, *pcbRead, &cbTmpWrite); NOREF(cbTmpWrite);
+                RTSocketClose(hSocket);
+            }
+        }
+#endif
+    }
+
+    return rc;
+}
+
+
+/**
+ * Writes data to the debugger connection, asynchronous.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvData      Pointer to the data to be written.
+ * @param   cbWrite     Size of the write buffer @a pvData.
+ * @param   pcbWritten  Where to store the number of bytes written.
+ * @param   fUdpPkt     Whether the debug data in @a pvData is encapsulated in a
+ *                      UDP frame.
+ *
+ * @thread  EMT.
+ */
+VMMR3_INT_DECL(int) gimR3HvDebugWrite(PVM pVM, void *pvData, uint32_t cbWrite, uint32_t *pcbWritten, bool fUdpPkt)
+{
+    Assert(cbWrite > 0);
+
+    PGIMHV    pHv        = &pVM->gim.s.u.Hv;
+    bool      fIgnorePkt = false;
+    uint8_t  *pbData     = (uint8_t *)pvData;
+    if (fUdpPkt)
+    {
+#ifdef DEBUG_ramshankar
+        RTSOCKET hSocket;
+        int rc2 = RTUdpCreateClientSocket("localhost", 52000, NULL, &hSocket);
+        if (RT_SUCCESS(rc2))
+        {
+            size_t cbTmpWrite = cbWrite;
+            RTSocketWriteNB(hSocket, pbData, cbWrite, &cbTmpWrite);  NOREF(cbTmpWrite);
+            RTSocketClose(hSocket);
+        }
+#endif
+        /*
+         * Windows guests sends us ethernet frames over the Hyper-V debug connection.
+         * It sends DHCP/ARP queries with zero'd out MAC addresses and requires fudging up the
+         * packets somewhere.
+         *
+         * The Microsoft WinDbg debugger talks UDP and thus only expects the actual debug
+         * protocol payload.
+         *
+         * If the guest is configured with the "nodhcp" option it sends ARP queries with
+         * a self-chosen IP and after a couple of attempts of receiving no replies, the guest
+         * picks its own IP address. After this, the guest starts sending the UDP packets
+         * we require. We thus ignore the initial ARP packets until the guest eventually
+         * starts talking UDP. Then we can finally feed the UDP payload over the debug
+         * connection.
+         *
+         * When 'kdvm.dll' is the debug transport in the guest (Windows 7), it doesn't bother
+         * with this DHCP/ARP phase. It starts sending debug data in a UDP frame right away.
+         */
+        if (cbWrite > sizeof(RTNETETHERHDR))
+        {
+            PCRTNETETHERHDR pEtherHdr = (PCRTNETETHERHDR)pbData;
+            if (pEtherHdr->EtherType == RT_H2N_U16_C(RTNET_ETHERTYPE_IPV4))
+            {
+                if (cbWrite > sizeof(RTNETETHERHDR) + RTNETIPV4_MIN_LEN + RTNETUDP_MIN_LEN)
+                {
+                    size_t const cbMaxIpHdr = cbWrite - sizeof(RTNETETHERHDR) - sizeof(RTNETUDP) - 1;
+                    size_t const cbMaxIpPkt = cbWrite - sizeof(RTNETETHERHDR);
+                    PCRTNETIPV4  pIp4Hdr    = (PCRTNETIPV4)(pbData + sizeof(RTNETETHERHDR));
+                    bool const   fValidIp4  = RTNetIPv4IsHdrValid(pIp4Hdr, cbMaxIpHdr, cbMaxIpPkt, false /*fChecksum*/);
+                    if (   fValidIp4
+                        && pIp4Hdr->ip_p == RTNETIPV4_PROT_UDP)
+                    {
+                        uint32_t const cbIpHdr     = pIp4Hdr->ip_hl * 4;
+                        uint32_t const cbMaxUdpPkt = cbWrite - sizeof(RTNETETHERHDR) - cbIpHdr;
+                        PCRTNETUDP pUdpHdr       = (PCRTNETUDP)((uint8_t *)pIp4Hdr + cbIpHdr);
+                        if (   pUdpHdr->uh_ulen >  RT_H2N_U16(sizeof(RTNETUDP))
+                            && pUdpHdr->uh_ulen <= RT_H2N_U16((uint16_t)cbMaxUdpPkt))
+                        {
+                            /*
+                             * Check for DHCP.
+                             */
+                            bool fBuggyPkt = false;
+                            size_t const cbUdpPkt = cbMaxIpPkt - cbIpHdr;
+                            if (   pUdpHdr->uh_dport == RT_N2H_U16_C(RTNETIPV4_PORT_BOOTPS)
+                                && pUdpHdr->uh_sport == RT_N2H_U16_C(RTNETIPV4_PORT_BOOTPC))
+                            {
+                                PCRTNETBOOTP pDhcpPkt = (PCRTNETBOOTP)(pUdpHdr + 1);
+                                uint8_t bMsgType;
+                                if (   cbMaxIpPkt >= cbIpHdr + RTNETUDP_MIN_LEN + RTNETBOOTP_DHCP_MIN_LEN
+                                    && RTNetIPv4IsDHCPValid(pUdpHdr, pDhcpPkt, cbUdpPkt - sizeof(*pUdpHdr), &bMsgType))
+                                {
+                                    switch (bMsgType)
+                                    {
+                                        case RTNET_DHCP_MT_DISCOVER:
+                                            pHv->enmDbgReply = GIMHVDEBUGREPLY_DHCP_OFFER;
+                                            pHv->uDbgBootpXId = pDhcpPkt->bp_xid;
+                                            break;
+                                        case RTNET_DHCP_MT_REQUEST:
+                                            pHv->enmDbgReply = GIMHVDEBUGREPLY_DHCP_ACK;
+                                            pHv->uDbgBootpXId = pDhcpPkt->bp_xid;
+                                            break;
+                                        default:
+                                            LogRelMax(5, ("GIM: HyperV: Debug DHCP MsgType %#x not implemented! Packet dropped\n",
+                                                          bMsgType));
+                                            break;
+                                    }
+                                    fIgnorePkt = true;
+                                }
+                                else if (   pIp4Hdr->ip_src.u == GIMHV_DEBUGCLIENT_IPV4
+                                         && pIp4Hdr->ip_dst.u == 0)
+                                {
+                                    /*
+                                     * Windows 8.1 seems to be sending malformed BOOTP packets at the final stage of the
+                                     * debugger sequence. It appears that a previously sent DHCP request buffer wasn't cleared
+                                     * in the guest and they re-use it instead of sending a zero destination+source port packet
+                                     * as expected below.
+                                     *
+                                     * We workaround Microsoft's bug here, or at least, I'm classifying it as a bug to
+                                     * preserve my own sanity, see @bugref{8006#c54}.
+                                     */
+                                    fBuggyPkt = true;
+                                }
+                            }
+
+                            if (  (   !pUdpHdr->uh_dport
+                                   && !pUdpHdr->uh_sport)
+                                || fBuggyPkt)
+                            {
+                                /*
+                                 * Extract the UDP payload and pass it to the debugger and record the guest IP address.
+                                 *
+                                 * Hyper-V sends UDP debugger packets with source and destination port as 0 except in the
+                                 * aforementioned buggy case. The buggy packet case requires us to remember the ports and
+                                 * reply to them, otherwise the guest won't receive the replies we sent with port 0.
+                                 */
+                                uint32_t const cbFrameHdr = sizeof(RTNETETHERHDR) + cbIpHdr + sizeof(RTNETUDP);
+                                pbData  += cbFrameHdr;
+                                cbWrite -= cbFrameHdr;
+                                pHv->DbgGuestIp4Addr.u = pIp4Hdr->ip_src.u;
+                                pHv->uUdpGuestDstPort  = pUdpHdr->uh_dport;
+                                pHv->uUdpGuestSrcPort  = pUdpHdr->uh_sport;
+                                pHv->enmDbgReply       = GIMHVDEBUGREPLY_UDP;
+                            }
+                            else
+                            {
+                                LogFlow(("GIM: HyperV: Ignoring UDP packet SourcePort=%u DstPort=%u\n", pUdpHdr->uh_sport,
+                                         pUdpHdr->uh_dport));
+                                fIgnorePkt = true;
+                            }
+                        }
+                        else
+                        {
+                            LogFlow(("GIM: HyperV: Ignoring malformed UDP packet. cbMaxUdpPkt=%u UdpPkt.len=%u\n", cbMaxUdpPkt,
+                                     RT_N2H_U16(pUdpHdr->uh_ulen)));
+                            fIgnorePkt = true;
+                        }
+                    }
+                    else
+                    {
+                        LogFlow(("GIM: HyperV: Ignoring non-IP / non-UDP packet. fValidIp4=%RTbool Proto=%u\n", fValidIp4,
+                                  pIp4Hdr->ip_p));
+                        fIgnorePkt = true;
+                    }
+                }
+                else
+                {
+                    LogFlow(("GIM: HyperV: Ignoring IPv4 packet; too short to be valid UDP. cbWrite=%u\n", cbWrite));
+                    fIgnorePkt = true;
+                }
+            }
+            else if (pEtherHdr->EtherType == RT_H2N_U16_C(RTNET_ETHERTYPE_ARP))
+            {
+                /*
+                 * Check for targetted ARP query.
+                 */
+                PCRTNETARPHDR pArpHdr = (PCRTNETARPHDR)(pbData + sizeof(RTNETETHERHDR));
+                if (   pArpHdr->ar_hlen  == sizeof(RTMAC)
+                    && pArpHdr->ar_plen  == sizeof(RTNETADDRIPV4)
+                    && pArpHdr->ar_htype == RT_H2N_U16(RTNET_ARP_ETHER)
+                    && pArpHdr->ar_ptype == RT_H2N_U16(RTNET_ETHERTYPE_IPV4))
+                {
+                    uint16_t uArpOp = pArpHdr->ar_oper;
+                    if (uArpOp == RT_H2N_U16_C(RTNET_ARPOP_REQUEST))
+                    {
+                        PCRTNETARPIPV4 pArpPkt = (PCRTNETARPIPV4)pArpHdr;
+                        bool fGratuitous = pArpPkt->ar_spa.u == pArpPkt->ar_tpa.u;
+                        if (   !fGratuitous
+                            &&  pArpPkt->ar_spa.u == GIMHV_DEBUGCLIENT_IPV4
+                            &&  pArpPkt->ar_tpa.u == GIMHV_DEBUGSERVER_IPV4)
+                        {
+                            pHv->enmDbgReply = GIMHVDEBUGREPLY_ARP_REPLY;
+                        }
+                    }
+                }
+                fIgnorePkt = true;
+            }
+            else
+            {
+                LogFlow(("GIM: HyperV: Ignoring non-IP packet. Ethertype=%#x\n", RT_N2H_U16(pEtherHdr->EtherType)));
+                fIgnorePkt = true;
+            }
+        }
+    }
+
+    if (!fIgnorePkt)
+    {
+        AssertCompile(sizeof(size_t) >= sizeof(uint32_t));
+        size_t cbWriteBuf = cbWrite;
+        int rc = gimR3DebugWrite(pVM, pbData, &cbWriteBuf);
+        if (   RT_SUCCESS(rc)
+            && cbWriteBuf == cbWrite)
+            *pcbWritten = (uint32_t)cbWriteBuf;
+        else
+            *pcbWritten = 0;
+    }
+    else
+        *pcbWritten = cbWrite;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Performs the HvPostDebugData hypercall.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   prcHv       Where to store the result of the hypercall operation.
+ *
+ * @thread  EMT.
+ */
+VMMR3_INT_DECL(int) gimR3HvHypercallPostDebugData(PVM pVM, int *prcHv)
+{
+    AssertPtr(pVM);
+    AssertPtr(prcHv);
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    int    rcHv = GIM_HV_STATUS_OPERATION_DENIED;
+
+    /*
+     * Grab the parameters.
+     */
+    PGIMHVDEBUGPOSTIN pIn = (PGIMHVDEBUGPOSTIN)pHv->pbHypercallIn;
+    AssertPtrReturn(pIn, VERR_GIM_IPE_1);
+    uint32_t   cbWrite = pIn->cbWrite;
+    uint32_t   fFlags = pIn->fFlags;
+    uint8_t   *pbData = ((uint8_t *)pIn) + sizeof(PGIMHVDEBUGPOSTIN);
+
+    PGIMHVDEBUGPOSTOUT pOut = (PGIMHVDEBUGPOSTOUT)pHv->pbHypercallOut;
+
+    /*
+     * Perform the hypercall.
+     */
+#if 0
+    /* Currently disabled as Windows 10 guest passes us undocumented flags. */
+    if (fFlags & ~GIM_HV_DEBUG_POST_OPTIONS_MASK))
+        rcHv = GIM_HV_STATUS_INVALID_PARAMETER;
+#else
+    RT_NOREF1(fFlags);
+#endif
+    if (cbWrite > GIM_HV_DEBUG_MAX_DATA_SIZE)
+        rcHv = GIM_HV_STATUS_INVALID_PARAMETER;
+    else if (!cbWrite)
+    {
+        rcHv = GIM_HV_STATUS_SUCCESS;
+        pOut->cbPending = 0;
+    }
+    else if (cbWrite > 0)
+    {
+        uint32_t cbWritten = 0;
+        int rc2 = gimR3HvDebugWrite(pVM, pbData, cbWrite, &cbWritten, pHv->fIsVendorMsHv /*fUdpPkt*/);
+        if (   RT_SUCCESS(rc2)
+            && cbWritten == cbWrite)
+        {
+            pOut->cbPending = 0;
+            rcHv = GIM_HV_STATUS_SUCCESS;
+        }
+        else
+            rcHv = GIM_HV_STATUS_INSUFFICIENT_BUFFER;
+    }
+
+    /*
+     * Update the guest memory with result.
+     */
+    int rc = PGMPhysSimpleWriteGCPhys(pVM, pHv->GCPhysHypercallOut, pHv->pbHypercallOut, sizeof(GIMHVDEBUGPOSTOUT));
+    if (RT_FAILURE(rc))
+    {
+        LogRelMax(10, ("GIM: HyperV: HvPostDebugData failed to update guest memory. rc=%Rrc\n", rc));
+        rc = VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED;
+    }
+    else
+        Assert(rc == VINF_SUCCESS);
+
+    *prcHv = rcHv;
+    return rc;
+}
+
+
+/**
+ * Performs the HvRetrieveDebugData hypercall.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   prcHv       Where to store the result of the hypercall operation.
+ *
+ * @thread  EMT.
+ */
+VMMR3_INT_DECL(int) gimR3HvHypercallRetrieveDebugData(PVM pVM, int *prcHv)
+{
+    AssertPtr(pVM);
+    AssertPtr(prcHv);
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+    int    rcHv = GIM_HV_STATUS_OPERATION_DENIED;
+
+    /*
+     * Grab the parameters.
+     */
+    PGIMHVDEBUGRETRIEVEIN pIn = (PGIMHVDEBUGRETRIEVEIN)pHv->pbHypercallIn;
+    AssertPtrReturn(pIn, VERR_GIM_IPE_1);
+    uint32_t   cbRead = pIn->cbRead;
+    uint32_t   fFlags = pIn->fFlags;
+    uint64_t   uTimeout = pIn->u64Timeout;
+    uint32_t   cMsTimeout = (fFlags & GIM_HV_DEBUG_RETREIVE_LOOP) ? (uTimeout * 100) / RT_NS_1MS_64 : 0;
+
+    PGIMHVDEBUGRETRIEVEOUT pOut = (PGIMHVDEBUGRETRIEVEOUT)pHv->pbHypercallOut;
+    AssertPtrReturn(pOut, VERR_GIM_IPE_2);
+    uint32_t   *pcbReallyRead = &pOut->cbRead;
+    uint32_t   *pcbRemainingRead = &pOut->cbRemaining;
+    void       *pvData = ((uint8_t *)pOut) + sizeof(GIMHVDEBUGRETRIEVEOUT);
+
+    /*
+     * Perform the hypercall.
+     */
+    *pcbReallyRead    = 0;
+    *pcbRemainingRead = cbRead;
+#if 0
+    /* Currently disabled as Windows 10 guest passes us undocumented flags. */
+    if (fFlags & ~GIM_HV_DEBUG_RETREIVE_OPTIONS_MASK)
+        rcHv = GIM_HV_STATUS_INVALID_PARAMETER;
+#endif
+    if (cbRead > GIM_HV_DEBUG_MAX_DATA_SIZE)
+        rcHv = GIM_HV_STATUS_INVALID_PARAMETER;
+    else if (fFlags & GIM_HV_DEBUG_RETREIVE_TEST_ACTIVITY)
+        rcHv = GIM_HV_STATUS_SUCCESS; /** @todo implement this. */
+    else if (!cbRead)
+        rcHv = GIM_HV_STATUS_SUCCESS;
+    else if (cbRead > 0)
+    {
+        int rc2 = gimR3HvDebugRead(pVM, pvData, GIM_HV_PAGE_SIZE, cbRead, pcbReallyRead, cMsTimeout,
+                                   pHv->fIsVendorMsHv /*fUdpPkt*/);
+        Assert(*pcbReallyRead <= cbRead);
+        if (   RT_SUCCESS(rc2)
+            && *pcbReallyRead > 0)
+        {
+            *pcbRemainingRead = cbRead -  *pcbReallyRead;
+            rcHv = GIM_HV_STATUS_SUCCESS;
+        }
+        else
+            rcHv = GIM_HV_STATUS_NO_DATA;
+    }
+
+    /*
+     * Update the guest memory with result.
+     */
+    int rc = PGMPhysSimpleWriteGCPhys(pVM, pHv->GCPhysHypercallOut, pHv->pbHypercallOut,
+                                      sizeof(GIMHVDEBUGRETRIEVEOUT) + *pcbReallyRead);
+    if (RT_FAILURE(rc))
+    {
+        LogRelMax(10, ("GIM: HyperV: HvRetrieveDebugData failed to update guest memory. rc=%Rrc\n", rc));
+        rc = VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED;
+    }
+    else
+        Assert(rc == VINF_SUCCESS);
+
+    *prcHv = rcHv;
+    return rc;
+}
+
+
+/**
+ * Performs the HvExtCallQueryCapabilities extended hypercall.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   prcHv       Where to store the result of the hypercall operation.
+ *
+ * @thread  EMT.
+ */
+VMMR3_INT_DECL(int) gimR3HvHypercallExtQueryCap(PVM pVM, int *prcHv)
+{
+    AssertPtr(pVM);
+    AssertPtr(prcHv);
+    PGIMHV pHv  = &pVM->gim.s.u.Hv;
+
+    /*
+     * Grab the parameters.
+     */
+   PGIMHVEXTQUERYCAP pOut = (PGIMHVEXTQUERYCAP)pHv->pbHypercallOut;
+
+    /*
+     * Perform the hypercall.
+     */
+    pOut->fCapabilities = GIM_HV_EXT_HYPERCALL_CAP_ZERO_MEM;
+
+    /*
+     * Update the guest memory with result.
+     */
+    int rcHv;
+    int rc = PGMPhysSimpleWriteGCPhys(pVM, pHv->GCPhysHypercallOut, pHv->pbHypercallOut, sizeof(GIMHVEXTQUERYCAP));
+    if (RT_SUCCESS(rc))
+    {
+        rcHv = GIM_HV_STATUS_SUCCESS;
+        LogRel(("GIM: HyperV: Queried extended hypercall capabilities %#RX64 at %#RGp\n", pOut->fCapabilities,
+                pHv->GCPhysHypercallOut));
+    }
+    else
+    {
+        rcHv = GIM_HV_STATUS_OPERATION_DENIED;
+        LogRelMax(10, ("GIM: HyperV: HvHypercallExtQueryCap failed to update guest memory. rc=%Rrc\n", rc));
+        rc = VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED;
+    }
+
+    *prcHv = rcHv;
+    return rc;
+}
+
+
+/**
+ * Performs the HvExtCallGetBootZeroedMemory extended hypercall.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   prcHv       Where to store the result of the hypercall operation.
+ *
+ * @thread  EMT.
+ */
+VMMR3_INT_DECL(int) gimR3HvHypercallExtGetBootZeroedMem(PVM pVM, int *prcHv)
+{
+    AssertPtr(pVM);
+    AssertPtr(prcHv);
+    PGIMHV pHv  = &pVM->gim.s.u.Hv;
+
+    /*
+     * Grab the parameters.
+     */
+    PGIMHVEXTGETBOOTZEROMEM pOut = (PGIMHVEXTGETBOOTZEROMEM)pHv->pbHypercallOut;
+
+    /*
+     * Perform the hypercall.
+     */
+    uint32_t const cRanges = PGMR3PhysGetRamRangeCount(pVM);
+    pOut->cPages = 0;
+    for (uint32_t iRange = 0; iRange < cRanges; iRange++)
+    {
+        RTGCPHYS GCPhysStart;
+        RTGCPHYS GCPhysEnd;
+        int rc = PGMR3PhysGetRange(pVM, iRange, &GCPhysStart, &GCPhysEnd, NULL /* pszDesc */, NULL /* fIsMmio */);
+        if (RT_FAILURE(rc))
+        {
+            LogRelMax(10, ("GIM: HyperV: HvHypercallExtGetBootZeroedMem: PGMR3PhysGetRange failed for iRange(%u) rc=%Rrc\n",
+                           iRange, rc));
+            *prcHv = GIM_HV_STATUS_OPERATION_DENIED;
+            return rc;
+        }
+
+        RTGCPHYS const cbRange = RT_ALIGN(GCPhysEnd - GCPhysStart + 1, PAGE_SIZE);
+        pOut->cPages += cbRange >> GIM_HV_PAGE_SHIFT;
+        if (iRange == 0)
+            pOut->GCPhysStart = GCPhysStart;
+    }
+
+    /*
+     * Update the guest memory with result.
+     */
+    int rcHv;
+    int rc = PGMPhysSimpleWriteGCPhys(pVM, pHv->GCPhysHypercallOut, pHv->pbHypercallOut, sizeof(GIMHVEXTGETBOOTZEROMEM));
+    if (RT_SUCCESS(rc))
+    {
+        LogRel(("GIM: HyperV: Queried boot zeroed guest memory range (starting at %#RGp spanning %u pages) at %#RGp\n",
+                pOut->GCPhysStart, pOut->cPages, pHv->GCPhysHypercallOut));
+        rcHv = GIM_HV_STATUS_SUCCESS;
+    }
+    else
+    {
+        rcHv = GIM_HV_STATUS_OPERATION_DENIED;
+        LogRelMax(10, ("GIM: HyperV: HvHypercallExtGetBootZeroedMem failed to update guest memory. rc=%Rrc\n", rc));
+        rc = VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED;
+    }
+
+    *prcHv = rcHv;
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/GIMKvm.cpp b/src/VBox/VMM/VMMR3/GIMKvm.cpp
new file mode 100644
index 00000000..1a764e7e
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/GIMKvm.cpp
@@ -0,0 +1,620 @@
+/* $Id: GIMKvm.cpp $ */
+/** @file
+ * GIM - Guest Interface Manager, KVM implementation.
+ */
+
+/*
+ * Copyright (C) 2015-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/em.h>
+#include "GIMInternal.h"
+#include <VBox/vmm/vm.h>
+
+#include <VBox/disopcode.h>
+#include <VBox/err.h>
+#include <VBox/version.h>
+
+#include <iprt/asm-math.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/mem.h>
+
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+
+/**
+ * GIM KVM saved-state version.
+ */
+#define GIM_KVM_SAVED_STATE_VERSION         UINT32_C(1)
+
+/**
+ * VBox internal struct. to passback to EMT rendezvous callback while enabling
+ * the KVM wall-clock.
+ */
+typedef struct KVMWALLCLOCKINFO
+{
+    /** Guest physical address of the wall-clock struct.  */
+    RTGCPHYS GCPhysWallClock;
+} KVMWALLCLOCKINFO;
+/** Pointer to the wall-clock info. struct. */
+typedef KVMWALLCLOCKINFO *PKVMWALLCLOCKINFO;
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_STATISTICS
+# define GIMKVM_MSRRANGE(a_uFirst, a_uLast, a_szName) \
+    { (a_uFirst), (a_uLast), kCpumMsrRdFn_Gim, kCpumMsrWrFn_Gim, 0, 0, 0, 0, 0, a_szName, { 0 }, { 0 }, { 0 }, { 0 } }
+#else
+# define GIMKVM_MSRRANGE(a_uFirst, a_uLast, a_szName) \
+    { (a_uFirst), (a_uLast), kCpumMsrRdFn_Gim, kCpumMsrWrFn_Gim, 0, 0, 0, 0, 0, a_szName }
+#endif
+
+/**
+ * Array of MSR ranges supported by KVM.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Kvm[] =
+{
+    GIMKVM_MSRRANGE(MSR_GIM_KVM_RANGE0_FIRST, MSR_GIM_KVM_RANGE0_LAST, "KVM range 0"),
+    GIMKVM_MSRRANGE(MSR_GIM_KVM_RANGE1_FIRST, MSR_GIM_KVM_RANGE1_LAST, "KVM range 1")
+};
+#undef GIMKVM_MSRRANGE
+
+
+/**
+ * Updates the KVM VCPU system-time structure in guest memory.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks This must be called after the system time MSR value has been updated.
+ */
+static int gimR3KvmUpdateSystemTime(PVM pVM, PVMCPU pVCpu)
+{
+    PGIMKVM    pKvm    = &pVM->gim.s.u.Kvm;
+    PGIMKVMCPU pKvmCpu = &pVCpu->gim.s.u.KvmCpu;
+
+    /*
+     * Validate the MSR has the enable bit and the guest's system time struct. address.
+     */
+    MSR_GIM_KVM_SYSTEM_TIME_IS_ENABLED(pKvmCpu->u64SystemTimeMsr);
+    if (!PGMPhysIsGCPhysNormal(pVM, pKvmCpu->GCPhysSystemTime))
+    {
+        LogRel(("GIM: KVM: VCPU%3d: Invalid physical addr requested for mapping system-time struct. GCPhysSystemTime=%#RGp\n",
+               pVCpu->idCpu, pKvmCpu->GCPhysSystemTime));
+        return VERR_GIM_OPERATION_FAILED;
+    }
+
+    VMSTATE const enmVMState = pVM->enmVMState;
+    bool const fRunning = VMSTATE_IS_RUNNING(enmVMState);
+    Assert(!(pKvmCpu->u32SystemTimeVersion & UINT32_C(1)));
+
+    /*
+     * Construct a system-time struct.
+     */
+    GIMKVMSYSTEMTIME SystemTime;
+    RT_ZERO(SystemTime);
+    SystemTime.u32Version  = pKvmCpu->u32SystemTimeVersion + !!fRunning;
+    SystemTime.u64NanoTS   = pKvmCpu->uVirtNanoTS;
+    SystemTime.u64Tsc      = pKvmCpu->uTsc;
+    SystemTime.fFlags      = pKvmCpu->fSystemTimeFlags | GIM_KVM_SYSTEM_TIME_FLAGS_TSC_STABLE;
+
+    /*
+     * How the guest calculates the system time (nanoseconds):
+     *
+     * tsc = rdtsc - SysTime.u64Tsc
+     * if (SysTime.i8TscShift >= 0)
+     *     tsc <<= i8TscShift;
+     * else
+     *     tsc >>= -i8TscShift;
+     * time = ((tsc * SysTime.u32TscScale) >> 32) + SysTime.u64NanoTS
+     */
+    uint64_t u64TscFreq   = pKvm->cTscTicksPerSecond;
+    SystemTime.i8TscShift = 0;
+    while (u64TscFreq > 2 * RT_NS_1SEC_64)
+    {
+        u64TscFreq >>= 1;
+        SystemTime.i8TscShift--;
+    }
+    uint32_t uTscFreqLo = (uint32_t)u64TscFreq;
+    while (uTscFreqLo <= RT_NS_1SEC)
+    {
+        uTscFreqLo <<= 1;
+        SystemTime.i8TscShift++;
+    }
+    SystemTime.u32TscScale = ASMDivU64ByU32RetU32(RT_NS_1SEC_64 << 32, uTscFreqLo);
+
+    /*
+     * For informational purposes, back-calculate the exact TSC frequency the guest will see.
+     * Note that the frequency is in kHz, not Hz, since that's what Linux uses.
+     */
+    uint64_t uTscKHz = (RT_NS_1MS_64 << 32) / SystemTime.u32TscScale;
+    if (SystemTime.i8TscShift < 0)
+        uTscKHz <<= -SystemTime.i8TscShift;
+    else
+        uTscKHz >>=  SystemTime.i8TscShift;
+
+    /*
+     * Update guest memory with the system-time struct.
+     *
+     * We update the struct with an incremented, odd version field to indicate to the guest
+     * that the memory is being updated concurrently by the host and it should discard any
+     * data from this struct when it reads an odd version.
+     *
+     * When the VM is not running, we don't need to do this two step update for obvious
+     * reasons and so we skip it.
+     */
+    if (fRunning)
+        Assert(SystemTime.u32Version & UINT32_C(1));
+    else
+        Assert(!(SystemTime.u32Version & UINT32_C(1)));
+
+    int rc = PGMPhysSimpleWriteGCPhys(pVM, pKvmCpu->GCPhysSystemTime, &SystemTime, sizeof(GIMKVMSYSTEMTIME));
+    if (RT_SUCCESS(rc))
+    {
+        LogRel(("GIM: KVM: VCPU%3d: Enabled system-time struct. at %#RGp - u32TscScale=%#RX32 i8TscShift=%d uVersion=%#RU32 "
+                "fFlags=%#x uTsc=%#RX64 uVirtNanoTS=%#RX64 TscKHz=%RU64\n", pVCpu->idCpu, pKvmCpu->GCPhysSystemTime,
+                SystemTime.u32TscScale, SystemTime.i8TscShift, SystemTime.u32Version + !!fRunning, SystemTime.fFlags,
+                pKvmCpu->uTsc, pKvmCpu->uVirtNanoTS, uTscKHz));
+        TMR3CpuTickParavirtEnable(pVM);
+    }
+    else
+    {
+        LogRel(("GIM: KVM: VCPU%3d: Failed to write system-time struct. at %#RGp. rc=%Rrc\n", pVCpu->idCpu,
+                pKvmCpu->GCPhysSystemTime, rc));
+    }
+
+    if (fRunning)
+    {
+        ++SystemTime.u32Version;
+        Assert(!(SystemTime.u32Version & UINT32_C(1)));
+        rc = PGMPhysSimpleWriteGCPhys(pVM, pKvmCpu->GCPhysSystemTime + RT_UOFFSETOF(GIMKVMSYSTEMTIME, u32Version),
+                                          &SystemTime.u32Version, sizeof(SystemTime.u32Version));
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("GIM: KVM: VCPU%3d: Failed to write system-time struct. while updating version field at %#RGp. rc=%Rrc\n",
+                    pVCpu->idCpu, pKvmCpu->GCPhysSystemTime, rc));
+            return rc;
+        }
+
+        /* Update the version so our next write will start with an even value. */
+        pKvmCpu->u32SystemTimeVersion += 2;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Initializes the KVM GIM provider.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) gimR3KvmInit(PVM pVM)
+{
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+    AssertReturn(pVM->gim.s.enmProviderId == GIMPROVIDERID_KVM, VERR_INTERNAL_ERROR_5);
+
+    int rc;
+    PGIMKVM pKvm = &pVM->gim.s.u.Kvm;
+
+    /*
+     * Determine interface capabilities based on the version.
+     */
+    if (!pVM->gim.s.u32Version)
+    {
+        /* Basic features. */
+        pKvm->uBaseFeat = 0
+                        | GIM_KVM_BASE_FEAT_CLOCK_OLD
+                        //| GIM_KVM_BASE_FEAT_NOP_IO_DELAY
+                        //|  GIM_KVM_BASE_FEAT_MMU_OP
+                        | GIM_KVM_BASE_FEAT_CLOCK
+                        //| GIM_KVM_BASE_FEAT_ASYNC_PF
+                        //| GIM_KVM_BASE_FEAT_STEAL_TIME
+                        //| GIM_KVM_BASE_FEAT_PV_EOI
+                        | GIM_KVM_BASE_FEAT_PV_UNHALT
+                        ;
+        /* Rest of the features are determined in gimR3KvmInitCompleted(). */
+    }
+
+    /*
+     * Expose HVP (Hypervisor Present) bit to the guest.
+     */
+    CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_HVP);
+
+    /*
+     * Modify the standard hypervisor leaves for KVM.
+     */
+    CPUMCPUIDLEAF HyperLeaf;
+    RT_ZERO(HyperLeaf);
+    HyperLeaf.uLeaf        = UINT32_C(0x40000000);
+    HyperLeaf.uEax         = UINT32_C(0x40000001); /* Minimum value for KVM is 0x40000001. */
+    HyperLeaf.uEbx         = 0x4B4D564B;           /* 'KVMK' */
+    HyperLeaf.uEcx         = 0x564B4D56;           /* 'VMKV' */
+    HyperLeaf.uEdx         = 0x0000004D;           /* 'M000' */
+    rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Add KVM specific leaves.
+     */
+    HyperLeaf.uLeaf        = UINT32_C(0x40000001);
+    HyperLeaf.uEax         = pKvm->uBaseFeat;
+    HyperLeaf.uEbx         = 0;                    /* Reserved */
+    HyperLeaf.uEcx         = 0;                    /* Reserved */
+    HyperLeaf.uEdx         = 0;                    /* Reserved */
+    rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Insert all MSR ranges of KVM.
+     */
+    for (unsigned i = 0; i < RT_ELEMENTS(g_aMsrRanges_Kvm); i++)
+    {
+        rc = CPUMR3MsrRangesInsert(pVM, &g_aMsrRanges_Kvm[i]);
+        AssertLogRelRCReturn(rc, rc);
+    }
+
+    /*
+     * Setup hypercall and #UD handling.
+     * Note! We always need to trap VMCALL/VMMCALL hypercall using #UDs for raw-mode VMs.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        EMSetHypercallInstructionsEnabled(pVM->apCpusR3[idCpu], true);
+
+    size_t cbHypercall = 0;
+    rc = GIMQueryHypercallOpcodeBytes(pVM, pKvm->abOpcodeNative, sizeof(pKvm->abOpcodeNative), &cbHypercall, &pKvm->uOpcodeNative);
+    AssertLogRelRCReturn(rc, rc);
+    AssertLogRelReturn(cbHypercall == sizeof(pKvm->abOpcodeNative), VERR_GIM_IPE_1);
+    pKvm->fTrapXcptUD = pKvm->uOpcodeNative != OP_VMCALL;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Initializes remaining bits of the KVM provider.
+ *
+ * This is called after initializing HM and almost all other VMM components.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) gimR3KvmInitCompleted(PVM pVM)
+{
+    PGIMKVM pKvm = &pVM->gim.s.u.Kvm;
+    pKvm->cTscTicksPerSecond = TMCpuTicksPerSecond(pVM);
+
+    if (TMR3CpuTickIsFixedRateMonotonic(pVM, true /* fWithParavirtEnabled */))
+    {
+        /** @todo We might want to consider just enabling this bit *always*. As far
+         *        as I can see in the Linux guest, the "TSC_STABLE" bit is only
+         *        translated as a "monotonic" bit which even in Async systems we
+         *        -should- be reporting a strictly monotonic TSC to the guest.  */
+        pKvm->uBaseFeat |= GIM_KVM_BASE_FEAT_TSC_STABLE;
+
+        CPUMCPUIDLEAF HyperLeaf;
+        RT_ZERO(HyperLeaf);
+        HyperLeaf.uLeaf        = UINT32_C(0x40000001);
+        HyperLeaf.uEax         = pKvm->uBaseFeat;
+        HyperLeaf.uEbx         = 0;
+        HyperLeaf.uEcx         = 0;
+        HyperLeaf.uEdx         = 0;
+        int rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+        AssertLogRelRCReturn(rc, rc);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminates the KVM GIM provider.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) gimR3KvmTerm(PVM pVM)
+{
+    gimR3KvmReset(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * This resets KVM provider MSRs and unmaps whatever KVM regions that
+ * the guest may have mapped.
+ *
+ * This is called when the VM is being reset.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @thread EMT(0)
+ */
+VMMR3_INT_DECL(void) gimR3KvmReset(PVM pVM)
+{
+    VM_ASSERT_EMT0(pVM);
+    LogRel(("GIM: KVM: Resetting MSRs\n"));
+
+    /*
+     * Reset MSRs.
+     */
+    PGIMKVM pKvm = &pVM->gim.s.u.Kvm;
+    pKvm->u64WallClockMsr = 0;
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PGIMKVMCPU pKvmCpu = &pVM->apCpusR3[idCpu]->gim.s.u.KvmCpu;
+        pKvmCpu->u64SystemTimeMsr = 0;
+        pKvmCpu->u32SystemTimeVersion = 0;
+        pKvmCpu->fSystemTimeFlags = 0;
+        pKvmCpu->GCPhysSystemTime = 0;
+        pKvmCpu->uTsc = 0;
+        pKvmCpu->uVirtNanoTS = 0;
+    }
+}
+
+
+/**
+ * KVM state-save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   pSSM    The saved state handle.
+ */
+VMMR3_INT_DECL(int) gimR3KvmSave(PVM pVM, PSSMHANDLE pSSM)
+{
+    PCGIMKVM pKvm = &pVM->gim.s.u.Kvm;
+
+    /*
+     * Save the KVM SSM version.
+     */
+    SSMR3PutU32(pSSM, GIM_KVM_SAVED_STATE_VERSION);
+
+    /*
+     * Save per-VCPU data.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PCGIMKVMCPU pKvmCpu = &pVM->apCpusR3[idCpu]->gim.s.u.KvmCpu;
+        SSMR3PutU64(pSSM, pKvmCpu->u64SystemTimeMsr);
+        SSMR3PutU64(pSSM, pKvmCpu->uTsc);
+        SSMR3PutU64(pSSM, pKvmCpu->uVirtNanoTS);
+        SSMR3PutGCPhys(pSSM, pKvmCpu->GCPhysSystemTime);
+        SSMR3PutU32(pSSM, pKvmCpu->u32SystemTimeVersion);
+        SSMR3PutU8(pSSM, pKvmCpu->fSystemTimeFlags);
+    }
+
+    /*
+     * Save per-VM data.
+     */
+    SSMR3PutU64(pSSM, pKvm->u64WallClockMsr);
+    return SSMR3PutU32(pSSM, pKvm->uBaseFeat);
+}
+
+
+/**
+ * KVM state-load operation, final pass.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The saved state handle.
+ */
+VMMR3_INT_DECL(int) gimR3KvmLoad(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Load the KVM SSM version first.
+     */
+    uint32_t uKvmSavedStatVersion;
+    int rc = SSMR3GetU32(pSSM, &uKvmSavedStatVersion);
+    AssertRCReturn(rc, rc);
+    if (uKvmSavedStatVersion != GIM_KVM_SAVED_STATE_VERSION)
+        return SSMR3SetLoadError(pSSM, VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION, RT_SRC_POS,
+                                 N_("Unsupported KVM saved-state version %u (expected %u)."),
+                                 uKvmSavedStatVersion, GIM_KVM_SAVED_STATE_VERSION);
+
+    /*
+     * Update the TSC frequency from TM.
+     */
+    PGIMKVM pKvm = &pVM->gim.s.u.Kvm;
+    pKvm->cTscTicksPerSecond = TMCpuTicksPerSecond(pVM);
+
+    /*
+     * Load per-VCPU data.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU     pVCpu   = pVM->apCpusR3[idCpu];
+        PGIMKVMCPU pKvmCpu = &pVCpu->gim.s.u.KvmCpu;
+
+        SSMR3GetU64(pSSM, &pKvmCpu->u64SystemTimeMsr);
+        SSMR3GetU64(pSSM, &pKvmCpu->uTsc);
+        SSMR3GetU64(pSSM, &pKvmCpu->uVirtNanoTS);
+        SSMR3GetGCPhys(pSSM, &pKvmCpu->GCPhysSystemTime);
+        SSMR3GetU32(pSSM, &pKvmCpu->u32SystemTimeVersion);
+        rc = SSMR3GetU8(pSSM, &pKvmCpu->fSystemTimeFlags);
+        AssertRCReturn(rc, rc);
+
+        /* Enable the system-time struct. if necessary. */
+        /** @todo update guest struct only if cTscTicksPerSecond doesn't match host
+         *        anymore. */
+        if (MSR_GIM_KVM_SYSTEM_TIME_IS_ENABLED(pKvmCpu->u64SystemTimeMsr))
+        {
+            Assert(!TMVirtualIsTicking(pVM));       /* paranoia. */
+            Assert(!TMCpuTickIsTicking(pVCpu));
+            gimR3KvmUpdateSystemTime(pVM, pVCpu);
+        }
+    }
+
+    /*
+     * Load per-VM data.
+     */
+    SSMR3GetU64(pSSM, &pKvm->u64WallClockMsr);
+    rc = SSMR3GetU32(pSSM, &pKvm->uBaseFeat);
+    AssertRCReturn(rc, rc);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Disables the KVM system-time struct.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) gimR3KvmDisableSystemTime(PVM pVM)
+{
+    TMR3CpuTickParavirtDisable(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{PFNVMMEMTRENDEZVOUS,
+ *      Worker for gimR3KvmEnableWallClock}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) gimR3KvmEnableWallClockCallback(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    PKVMWALLCLOCKINFO pWallClockInfo  = (PKVMWALLCLOCKINFO)pvUser; AssertPtr(pWallClockInfo);
+    RTGCPHYS          GCPhysWallClock = pWallClockInfo->GCPhysWallClock;
+    RT_NOREF1(pVCpu);
+
+    /*
+     * Read the wall-clock version (sequence) from the guest.
+     */
+    uint32_t uVersion;
+    Assert(PGMPhysIsGCPhysNormal(pVM, GCPhysWallClock));
+    int rc = PGMPhysSimpleReadGCPhys(pVM, &uVersion, GCPhysWallClock, sizeof(uVersion));
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("GIM: KVM: Failed to read wall-clock struct. version at %#RGp. rc=%Rrc\n", GCPhysWallClock, rc));
+        return rc;
+    }
+
+    /*
+     * Ensure the version is incrementally even.
+     */
+    /* faster: uVersion = (uVersion | 1) + 1; */
+    if (!(uVersion & 1))
+        ++uVersion;
+    ++uVersion;
+
+    /*
+     * Update wall-clock guest struct. with UTC information.
+     */
+    RTTIMESPEC TimeSpec;
+    int32_t    iSec;
+    int32_t    iNano;
+    TMR3UtcNow(pVM, &TimeSpec);
+    RTTimeSpecGetSecondsAndNano(&TimeSpec, &iSec, &iNano);
+
+    GIMKVMWALLCLOCK WallClock;
+    RT_ZERO(WallClock);
+    AssertCompile(sizeof(uVersion) == sizeof(WallClock.u32Version));
+    WallClock.u32Version = uVersion;
+    WallClock.u32Sec     = iSec;
+    WallClock.u32Nano    = iNano;
+
+    /*
+     * Write out the wall-clock struct. to guest memory.
+     */
+    Assert(!(WallClock.u32Version & 1));
+    rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysWallClock, &WallClock, sizeof(GIMKVMWALLCLOCK));
+    if (RT_SUCCESS(rc))
+        LogRel(("GIM: KVM: Enabled wall-clock struct. at %#RGp - u32Sec=%u u32Nano=%u uVersion=%#RU32\n", GCPhysWallClock,
+                WallClock.u32Sec, WallClock.u32Nano, WallClock.u32Version));
+    else
+        LogRel(("GIM: KVM: Failed to write wall-clock struct. at %#RGp. rc=%Rrc\n", GCPhysWallClock, rc));
+    return rc;
+}
+
+
+/**
+ * Enables the KVM wall-clock structure.
+ *
+ * Since the wall-clock can be read by any VCPU but it is a global struct. in
+ * guest-memory, we do an EMT rendezvous here to be on the safe side. The
+ * alternative is to use an MMIO2 region and use the WallClock.u32Version field
+ * for transactional update. However, this MSR is rarely written to (typically
+ * once during bootup) it's currently not a performance issue especially since
+ * we're already in ring-3. If we really wanted better performance in this code
+ * path, we should be doing it in ring-0 with transactional update while make
+ * sure there is only 1 writer as well.
+ *
+ * @returns VBox status code.
+ * @param   pVM                The cross context VM structure.
+ * @param   GCPhysWallClock    Where the guest wall-clock structure is located.
+ *
+ * @remarks Don't do any release assertions here, these can be triggered by
+ *          guest R0 code.
+ */
+VMMR3_INT_DECL(int) gimR3KvmEnableWallClock(PVM pVM, RTGCPHYS GCPhysWallClock)
+{
+    KVMWALLCLOCKINFO WallClockInfo;
+    WallClockInfo.GCPhysWallClock = GCPhysWallClock;
+    return VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, gimR3KvmEnableWallClockCallback, &WallClockInfo);
+}
+
+
+/**
+ * Enables the KVM system time structure.
+ *
+ * This can be done concurrently because the guest memory being updated is per-VCPU
+ * and the struct even has a "version" field which needs to be incremented
+ * before/after altering guest memory to allow concurrent updates from the host.
+ * Hence this is not being done in an EMT rendezvous. It -is- done in ring-3 since
+ * we call into ring-3 only TM code in the end.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uMsrSystemTime  The system time MSR value being written.
+ */
+VMMR3_INT_DECL(int) gimR3KvmEnableSystemTime(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uMsrSystemTime)
+{
+    Assert(uMsrSystemTime & MSR_GIM_KVM_SYSTEM_TIME_ENABLE_BIT);
+    PGIMKVM    pKvm    = &pVM->gim.s.u.Kvm;
+    PGIMKVMCPU pKvmCpu = &pVCpu->gim.s.u.KvmCpu;
+
+    /*
+     * Update the system-time struct.
+     * The system-time structs are usually placed at a different guest address for each VCPU.
+     */
+    pKvmCpu->uTsc                  = TMCpuTickGetNoCheck(pVCpu);
+    pKvmCpu->uVirtNanoTS           = ASMMultU64ByU32DivByU32(pKvmCpu->uTsc, RT_NS_1SEC, pKvm->cTscTicksPerSecond);
+    pKvmCpu->u64SystemTimeMsr      = uMsrSystemTime;
+    pKvmCpu->GCPhysSystemTime      = MSR_GIM_KVM_SYSTEM_TIME_GUEST_GPA(uMsrSystemTime);
+
+    int rc = gimR3KvmUpdateSystemTime(pVM, pVCpu);
+    if (RT_FAILURE(rc))
+    {
+        pKvmCpu->u64SystemTimeMsr = 0;
+        /* We shouldn't throw a #GP(0) here for buggy guests (neither does KVM apparently), see @bugref{8627}. */
+    }
+
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/GIMMinimal.cpp b/src/VBox/VMM/VMMR3/GIMMinimal.cpp
new file mode 100644
index 00000000..d8160143
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/GIMMinimal.cpp
@@ -0,0 +1,131 @@
+/* $Id: GIMMinimal.cpp $ */
+/** @file
+ * GIM - Guest Interface Manager, Minimal implementation.
+ */
+
+/*
+ * Copyright (C) 2014-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/apic.h>
+#include "GIMInternal.h"
+#include <VBox/vmm/vm.h>
+
+#include <iprt/assert.h>
+#include <iprt/err.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+
+/**
+ * Initializes the Minimal provider.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) gimR3MinimalInit(PVM pVM)
+{
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+    AssertReturn(pVM->gim.s.enmProviderId == GIMPROVIDERID_MINIMAL, VERR_INTERNAL_ERROR_5);
+
+    /*
+     * Expose HVP (Hypervisor Present) bit to the guest.
+     */
+    CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_HVP);
+
+    /*
+     * Insert the hypervisor leaf range.
+     */
+    CPUMCPUIDLEAF HyperLeaf;
+    RT_ZERO(HyperLeaf);
+    HyperLeaf.uLeaf = UINT32_C(0x40000000);
+    HyperLeaf.uEax  = UINT32_C(0x40000010); /* Maximum leaf we implement. */
+    int rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Insert missing zero leaves (you never know what missing leaves are
+         * going to return when read).
+         */
+        RT_ZERO(HyperLeaf);
+        for (uint32_t uLeaf = UINT32_C(0x40000001); uLeaf <= UINT32_C(0x40000010); uLeaf++)
+        {
+            HyperLeaf.uLeaf = uLeaf;
+            rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+            AssertLogRelRCReturn(rc, rc);
+        }
+    }
+    else
+        LogRel(("GIM: Minimal: Failed to insert hypervisor leaf %#RX32. rc=%Rrc\n", HyperLeaf.uLeaf, rc));
+
+    return rc;
+}
+
+
+/**
+ * Initializes remaining bits of the Minimal provider.
+ * This is called after initializing HM and almost all other VMM components.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) gimR3MinimalInitCompleted(PVM pVM)
+{
+    /*
+     * Expose a generic hypervisor-agnostic leaf (originally defined by VMware).
+     * The leaves range from  0x40000010 to 0x400000FF.
+     *
+     * This is done in the init. completed routine as we need PDM to be
+     * initialized (otherwise APICGetTimerFreq() would fail).
+     */
+    CPUMCPUIDLEAF HyperLeaf;
+    int rc = CPUMR3CpuIdGetLeaf(pVM, &HyperLeaf, 0x40000000, 0 /* uSubLeaf */);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(HyperLeaf.uEax >= 0x40000010);
+
+        /*
+         * Add the timing information hypervisor leaf.
+         * MacOS X uses this to determine the TSC, bus frequency. See @bugref{7270}.
+         *
+         * EAX - TSC frequency in KHz.
+         * EBX - APIC frequency in KHz.
+         * ECX, EDX - Reserved.
+         */
+        uint64_t uApicFreq;
+        rc = APICGetTimerFreq(pVM, &uApicFreq);
+        AssertLogRelRCReturn(rc, rc);
+
+        RT_ZERO(HyperLeaf);
+        HyperLeaf.uLeaf        = UINT32_C(0x40000010);
+        HyperLeaf.uEax         = TMCpuTicksPerSecond(pVM) / UINT64_C(1000);
+        HyperLeaf.uEbx         = (uApicFreq + 500) / UINT64_C(1000);
+        rc = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+        AssertLogRelRCReturn(rc, rc);
+    }
+    else
+        LogRel(("GIM: Minimal: failed to get hypervisor leaf 0x40000000. rc=%Rrc\n", rc));
+
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMR3/GMM.cpp b/src/VBox/VMM/VMMR3/GMM.cpp
new file mode 100644
index 00000000..4be4e20e
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/GMM.cpp
@@ -0,0 +1,451 @@
+/* $Id: GMM.cpp $ */
+/** @file
+ * GMM - Global Memory Manager, ring-3 request wrappers.
+ */
+
+/*
+ * Copyright (C) 2008-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GMM
+#include <VBox/vmm/gmm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/sup.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+
+#include <iprt/assert.h>
+#include <VBox/log.h>
+#include <iprt/mem.h>
+#include <iprt/string.h>
+
+
+/**
+ * @see GMMR0InitialReservation
+ */
+GMMR3DECL(int)  GMMR3InitialReservation(PVM pVM, uint64_t cBasePages, uint32_t cShadowPages, uint32_t cFixedPages,
+                                        GMMOCPOLICY enmPolicy, GMMPRIORITY enmPriority)
+{
+    GMMINITIALRESERVATIONREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq = sizeof(Req);
+    Req.cBasePages = cBasePages;
+    Req.cShadowPages = cShadowPages;
+    Req.cFixedPages = cFixedPages;
+    Req.enmPolicy = enmPolicy;
+    Req.enmPriority = enmPriority;
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_INITIAL_RESERVATION, 0, &Req.Hdr);
+}
+
+
+/**
+ * @see GMMR0UpdateReservation
+ */
+GMMR3DECL(int)  GMMR3UpdateReservation(PVM pVM, uint64_t cBasePages, uint32_t cShadowPages, uint32_t cFixedPages)
+{
+    GMMUPDATERESERVATIONREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq = sizeof(Req);
+    Req.cBasePages = cBasePages;
+    Req.cShadowPages = cShadowPages;
+    Req.cFixedPages = cFixedPages;
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_UPDATE_RESERVATION, 0, &Req.Hdr);
+}
+
+
+/**
+ * Prepares a GMMR0AllocatePages request.
+ *
+ * @returns VINF_SUCCESS or VERR_NO_TMP_MEMORY.
+ * @param       pVM         The cross context VM structure.
+ * @param[out]  ppReq       Where to store the pointer to the request packet.
+ * @param       cPages      The number of pages that's to be allocated.
+ * @param       enmAccount  The account to charge.
+ */
+GMMR3DECL(int) GMMR3AllocatePagesPrepare(PVM pVM, PGMMALLOCATEPAGESREQ *ppReq, uint32_t cPages, GMMACCOUNT enmAccount)
+{
+    uint32_t cb = RT_UOFFSETOF_DYN(GMMALLOCATEPAGESREQ, aPages[cPages]);
+    PGMMALLOCATEPAGESREQ pReq = (PGMMALLOCATEPAGESREQ)RTMemTmpAllocZ(cb);
+    if (!pReq)
+        return VERR_NO_TMP_MEMORY;
+
+    pReq->Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    pReq->Hdr.cbReq = cb;
+    pReq->enmAccount = enmAccount;
+    pReq->cPages = cPages;
+    NOREF(pVM);
+    *ppReq = pReq;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Performs a GMMR0AllocatePages request.
+ *
+ * This will call VMSetError on failure.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pReq        Pointer to the request (returned by GMMR3AllocatePagesPrepare).
+ */
+GMMR3DECL(int) GMMR3AllocatePagesPerform(PVM pVM, PGMMALLOCATEPAGESREQ pReq)
+{
+    for (unsigned i = 0; ; i++)
+    {
+        int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_ALLOCATE_PAGES, 0, &pReq->Hdr);
+        if (RT_SUCCESS(rc))
+        {
+#ifdef LOG_ENABLED
+            for (uint32_t iPage = 0; iPage < pReq->cPages; iPage++)
+                Log3(("GMMR3AllocatePagesPerform: idPage=%#x HCPhys=%RHp\n",
+                      pReq->aPages[iPage].idPage, pReq->aPages[iPage].HCPhysGCPhys));
+#endif
+            return rc;
+        }
+        if (rc != VERR_GMM_SEED_ME)
+            return VMSetError(pVM, rc, RT_SRC_POS,
+                              N_("GMMR0AllocatePages failed to allocate %u pages"),
+                              pReq->cPages);
+        Assert(i < pReq->cPages);
+
+        /*
+         * Seed another chunk.
+         */
+        void *pvChunk;
+        rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS,
+                              N_("Out of memory (SUPR3PageAlloc) seeding a %u pages allocation request"),
+                              pReq->cPages);
+
+        rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, N_("GMM seeding failed"));
+    }
+}
+
+
+/**
+ * Cleans up a GMMR0AllocatePages request.
+ * @param   pReq        Pointer to the request (returned by GMMR3AllocatePagesPrepare).
+ */
+GMMR3DECL(void) GMMR3AllocatePagesCleanup(PGMMALLOCATEPAGESREQ pReq)
+{
+    RTMemTmpFree(pReq);
+}
+
+
+/**
+ * Prepares a GMMR0FreePages request.
+ *
+ * @returns VINF_SUCCESS or VERR_NO_TMP_MEMORY.
+ * @param       pVM         The cross context VM structure.
+ * @param[out]  ppReq       Where to store the pointer to the request packet.
+ * @param       cPages      The number of pages that's to be freed.
+ * @param       enmAccount  The account to charge.
+ */
+GMMR3DECL(int) GMMR3FreePagesPrepare(PVM pVM, PGMMFREEPAGESREQ *ppReq, uint32_t cPages, GMMACCOUNT enmAccount)
+{
+    uint32_t cb = RT_UOFFSETOF_DYN(GMMFREEPAGESREQ, aPages[cPages]);
+    PGMMFREEPAGESREQ pReq = (PGMMFREEPAGESREQ)RTMemTmpAllocZ(cb);
+    if (!pReq)
+        return VERR_NO_TMP_MEMORY;
+
+    pReq->Hdr.u32Magic  = SUPVMMR0REQHDR_MAGIC;
+    pReq->Hdr.cbReq     = cb;
+    pReq->enmAccount    = enmAccount;
+    pReq->cPages        = cPages;
+    NOREF(pVM);
+    *ppReq = pReq;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Re-prepares a GMMR0FreePages request.
+ *
+ * @returns VINF_SUCCESS or VERR_NO_TMP_MEMORY.
+ * @param       pVM         The cross context VM structure.
+ * @param       pReq        A request buffer previously returned by
+ *                          GMMR3FreePagesPrepare().
+ * @param       cPages      The number of pages originally passed to
+ *                          GMMR3FreePagesPrepare().
+ * @param       enmAccount  The account to charge.
+ */
+GMMR3DECL(void) GMMR3FreePagesRePrep(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t cPages, GMMACCOUNT enmAccount)
+{
+    Assert(pReq->Hdr.u32Magic == SUPVMMR0REQHDR_MAGIC);
+    pReq->Hdr.cbReq     = RT_UOFFSETOF_DYN(GMMFREEPAGESREQ, aPages[cPages]);
+    pReq->enmAccount    = enmAccount;
+    pReq->cPages        = cPages;
+    NOREF(pVM);
+}
+
+
+/**
+ * Performs a GMMR0FreePages request.
+ * This will call VMSetError on failure.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pReq            Pointer to the request (returned by GMMR3FreePagesPrepare).
+ * @param   cActualPages    The number of pages actually freed.
+ */
+GMMR3DECL(int) GMMR3FreePagesPerform(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t cActualPages)
+{
+    /*
+     * Adjust the request if we ended up with fewer pages than anticipated.
+     */
+    if (cActualPages != pReq->cPages)
+    {
+        AssertReturn(cActualPages < pReq->cPages, VERR_GMM_ACTUAL_PAGES_IPE);
+        if (!cActualPages)
+            return VINF_SUCCESS;
+        pReq->cPages = cActualPages;
+        pReq->Hdr.cbReq = RT_UOFFSETOF_DYN(GMMFREEPAGESREQ, aPages[cActualPages]);
+    }
+
+    /*
+     * Do the job.
+     */
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_FREE_PAGES, 0, &pReq->Hdr);
+    if (RT_SUCCESS(rc))
+        return rc;
+    AssertRC(rc);
+    return VMSetError(pVM, rc, RT_SRC_POS,
+                      N_("GMMR0FreePages failed to free %u pages"),
+                      pReq->cPages);
+}
+
+
+/**
+ * Cleans up a GMMR0FreePages request.
+ * @param   pReq        Pointer to the request (returned by GMMR3FreePagesPrepare).
+ */
+GMMR3DECL(void) GMMR3FreePagesCleanup(PGMMFREEPAGESREQ pReq)
+{
+    RTMemTmpFree(pReq);
+}
+
+
+/**
+ * Frees allocated pages, for bailing out on failure.
+ *
+ * This will not call VMSetError on failure but will use AssertLogRel instead.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pAllocReq   The allocation request to undo.
+ */
+GMMR3DECL(void) GMMR3FreeAllocatedPages(PVM pVM, GMMALLOCATEPAGESREQ const *pAllocReq)
+{
+    uint32_t cb = RT_UOFFSETOF_DYN(GMMFREEPAGESREQ, aPages[pAllocReq->cPages]);
+    PGMMFREEPAGESREQ pReq = (PGMMFREEPAGESREQ)RTMemTmpAllocZ(cb);
+    AssertLogRelReturnVoid(pReq);
+
+    pReq->Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    pReq->Hdr.cbReq = cb;
+    pReq->enmAccount = pAllocReq->enmAccount;
+    pReq->cPages = pAllocReq->cPages;
+    uint32_t iPage = pAllocReq->cPages;
+    while (iPage-- > 0)
+    {
+        Assert(pAllocReq->aPages[iPage].idPage != NIL_GMM_PAGEID);
+        pReq->aPages[iPage].idPage = pAllocReq->aPages[iPage].idPage;
+    }
+
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_FREE_PAGES, 0, &pReq->Hdr);
+    AssertLogRelRC(rc);
+
+    RTMemTmpFree(pReq);
+}
+
+
+/**
+ * @see GMMR0BalloonedPages
+ */
+GMMR3DECL(int)  GMMR3BalloonedPages(PVM pVM, GMMBALLOONACTION enmAction, uint32_t cBalloonedPages)
+{
+    GMMBALLOONEDPAGESREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq = sizeof(Req);
+    Req.enmAction = enmAction;
+    Req.cBalloonedPages = cBalloonedPages;
+
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_BALLOONED_PAGES, 0, &Req.Hdr);
+}
+
+
+/**
+ * @see GMMR0QueryVMMMemoryStatsReq
+ */
+GMMR3DECL(int)  GMMR3QueryHypervisorMemoryStats(PVM pVM, uint64_t *pcTotalAllocPages, uint64_t *pcTotalFreePages, uint64_t *pcTotalBalloonPages, uint64_t *puTotalBalloonSize)
+{
+    GMMMEMSTATSREQ Req;
+    Req.Hdr.u32Magic     = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq        = sizeof(Req);
+    Req.cAllocPages      = 0;
+    Req.cFreePages       = 0;
+    Req.cBalloonedPages  = 0;
+    Req.cSharedPages     = 0;
+
+    *pcTotalAllocPages   = 0;
+    *pcTotalFreePages    = 0;
+    *pcTotalBalloonPages = 0;
+    *puTotalBalloonSize  = 0;
+
+    /* Must be callable from any thread, so can't use VMMR3CallR0. */
+    int rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_GMM_QUERY_HYPERVISOR_MEM_STATS, 0, &Req.Hdr);
+    if (rc == VINF_SUCCESS)
+    {
+        *pcTotalAllocPages   = Req.cAllocPages;
+        *pcTotalFreePages    = Req.cFreePages;
+        *pcTotalBalloonPages = Req.cBalloonedPages;
+        *puTotalBalloonSize  = Req.cSharedPages;
+    }
+    return rc;
+}
+
+
+/**
+ * @see GMMR0QueryMemoryStatsReq
+ */
+GMMR3DECL(int)  GMMR3QueryMemoryStats(PVM pVM, uint64_t *pcAllocPages, uint64_t *pcMaxPages, uint64_t *pcBalloonPages)
+{
+    GMMMEMSTATSREQ Req;
+    Req.Hdr.u32Magic    = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq       = sizeof(Req);
+    Req.cAllocPages     = 0;
+    Req.cFreePages      = 0;
+    Req.cBalloonedPages = 0;
+
+    *pcAllocPages      = 0;
+    *pcMaxPages         = 0;
+    *pcBalloonPages     = 0;
+
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_QUERY_MEM_STATS, 0, &Req.Hdr);
+    if (rc == VINF_SUCCESS)
+    {
+        *pcAllocPages   = Req.cAllocPages;
+        *pcMaxPages     = Req.cMaxPages;
+        *pcBalloonPages = Req.cBalloonedPages;
+    }
+    return rc;
+}
+
+
+/**
+ * @see GMMR0MapUnmapChunk
+ */
+GMMR3DECL(int)  GMMR3MapUnmapChunk(PVM pVM, uint32_t idChunkMap, uint32_t idChunkUnmap, PRTR3PTR ppvR3)
+{
+    GMMMAPUNMAPCHUNKREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq = sizeof(Req);
+    Req.idChunkMap = idChunkMap;
+    Req.idChunkUnmap = idChunkUnmap;
+    Req.pvR3 = NULL;
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
+    if (RT_SUCCESS(rc) && ppvR3)
+        *ppvR3 = Req.pvR3;
+    return rc;
+}
+
+
+/**
+ * @see GMMR0FreeLargePage
+ */
+GMMR3DECL(int)  GMMR3FreeLargePage(PVM pVM,  uint32_t idPage)
+{
+    GMMFREELARGEPAGEREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq = sizeof(Req);
+    Req.idPage = idPage;
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_FREE_LARGE_PAGE, 0, &Req.Hdr);
+}
+
+
+/**
+ * @see GMMR0SeedChunk
+ */
+GMMR3DECL(int)  GMMR3SeedChunk(PVM pVM, RTR3PTR pvR3)
+{
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvR3, NULL);
+}
+
+
+/**
+ * @see GMMR0RegisterSharedModule
+ */
+GMMR3DECL(int) GMMR3RegisterSharedModule(PVM pVM, PGMMREGISTERSHAREDMODULEREQ pReq)
+{
+    pReq->Hdr.u32Magic  = SUPVMMR0REQHDR_MAGIC;
+    pReq->Hdr.cbReq     = RT_UOFFSETOF_DYN(GMMREGISTERSHAREDMODULEREQ, aRegions[pReq->cRegions]);
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_REGISTER_SHARED_MODULE, 0, &pReq->Hdr);
+    if (rc == VINF_SUCCESS)
+        rc = pReq->rc;
+    return rc;
+}
+
+
+/**
+ * @see GMMR0RegisterSharedModule
+ */
+GMMR3DECL(int) GMMR3UnregisterSharedModule(PVM pVM, PGMMUNREGISTERSHAREDMODULEREQ pReq)
+{
+    pReq->Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    pReq->Hdr.cbReq = sizeof(*pReq);
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_UNREGISTER_SHARED_MODULE, 0, &pReq->Hdr);
+}
+
+
+/**
+ * @see GMMR0ResetSharedModules
+ */
+GMMR3DECL(int) GMMR3ResetSharedModules(PVM pVM)
+{
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_RESET_SHARED_MODULES, 0, NULL);
+}
+
+
+/**
+ * @see GMMR0CheckSharedModules
+ */
+GMMR3DECL(int)  GMMR3CheckSharedModules(PVM pVM)
+{
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_CHECK_SHARED_MODULES, 0, NULL);
+}
+
+
+#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
+/**
+ * @see GMMR0FindDuplicatePage
+ */
+GMMR3DECL(bool) GMMR3IsDuplicatePage(PVM pVM, uint32_t idPage)
+{
+    GMMFINDDUPLICATEPAGEREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq    = sizeof(Req);
+    Req.idPage       = idPage;
+    Req.fDuplicate   = false;
+
+    /* Must be callable from any thread, so can't use VMMR3CallR0. */
+    int rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_GMM_FIND_DUPLICATE_PAGE, 0, &Req.Hdr);
+    if (rc == VINF_SUCCESS)
+        return Req.fDuplicate;
+    return false;
+}
+#endif /* VBOX_STRICT && HC_ARCH_BITS == 64 */
+
diff --git a/src/VBox/VMM/VMMR3/HM.cpp b/src/VBox/VMM/VMMR3/HM.cpp
new file mode 100644
index 00000000..c90e3a8c
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/HM.cpp
@@ -0,0 +1,3446 @@
+/* $Id: HM.cpp $ */
+/** @file
+ * HM - Intel/AMD VM Hardware Support Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_hm     HM - Hardware Assisted Virtualization Manager
+ *
+ * The HM manages guest execution using the VT-x and AMD-V CPU hardware
+ * extensions.
+ *
+ * {summary of what HM does}
+ *
+ * Hardware assisted virtualization manager was originally abbreviated HWACCM,
+ * however that was cumbersome to write and parse for such a central component,
+ * so it was shortened to HM when refactoring the code in the 4.3 development
+ * cycle.
+ *
+ * {add sections with more details}
+ *
+ * @sa @ref grp_hm
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_HM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/hm_vmx.h>
+#include <VBox/vmm/hm_svm.h>
+#include "HMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+
+#include <iprt/assert.h>
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/env.h>
+#include <iprt/thread.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** @def HMVMX_REPORT_FEAT
+ * Reports VT-x feature to the release log.
+ *
+ * @param   a_uAllowed1       Mask of allowed-1 feature bits.
+ * @param   a_uAllowed0       Mask of allowed-0 feature bits.
+ * @param   a_StrDesc         The description string to report.
+ * @param   a_Featflag        Mask of the feature to report.
+ */
+#define HMVMX_REPORT_FEAT(a_uAllowed1, a_uAllowed0, a_StrDesc, a_Featflag) \
+    do { \
+        if ((a_uAllowed1) & (a_Featflag)) \
+        { \
+            if ((a_uAllowed0) & (a_Featflag)) \
+                LogRel(("HM:   " a_StrDesc " (must be set)\n")); \
+            else \
+                LogRel(("HM:   " a_StrDesc "\n")); \
+        } \
+        else \
+            LogRel(("HM:   " a_StrDesc " (must be cleared)\n")); \
+    } while (0)
+
+/** @def HMVMX_REPORT_ALLOWED_FEAT
+ * Reports an allowed VT-x feature to the release log.
+ *
+ * @param   a_uAllowed1     Mask of allowed-1 feature bits.
+ * @param   a_StrDesc       The description string to report.
+ * @param   a_FeatFlag      Mask of the feature to report.
+ */
+#define HMVMX_REPORT_ALLOWED_FEAT(a_uAllowed1, a_StrDesc, a_FeatFlag) \
+    do { \
+        if ((a_uAllowed1) & (a_FeatFlag)) \
+            LogRel(("HM:   " a_StrDesc "\n")); \
+        else \
+            LogRel(("HM:   " a_StrDesc " not supported\n")); \
+    } while (0)
+
+/** @def HMVMX_REPORT_MSR_CAP
+ * Reports MSR feature capability.
+ *
+ * @param   a_MsrCaps           Mask of MSR feature bits.
+ * @param   a_StrDesc           The description string to report.
+ * @param   a_fCap              Mask of the feature to report.
+ */
+#define HMVMX_REPORT_MSR_CAP(a_MsrCaps, a_StrDesc, a_fCap) \
+    do { \
+        if ((a_MsrCaps) & (a_fCap)) \
+            LogRel(("HM:   " a_StrDesc "\n")); \
+    } while (0)
+
+/** @def HMVMX_LOGREL_FEAT
+ * Dumps a feature flag from a bitmap of features to the release log.
+ *
+ * @param   a_fVal         The value of all the features.
+ * @param   a_fMask        The specific bitmask of the feature.
+ */
+#define HMVMX_LOGREL_FEAT(a_fVal, a_fMask) \
+    do { \
+        if ((a_fVal) & (a_fMask)) \
+            LogRel(("HM:   %s\n",  #a_fMask)); \
+    } while (0)
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(int)  hmR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)  hmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(void) hmR3InfoSvmNstGstVmcbCache(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) hmR3Info(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) hmR3InfoEventPending(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) hmR3InfoLbr(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static int                hmR3InitFinalizeR3(PVM pVM);
+static int                hmR3InitFinalizeR0(PVM pVM);
+static int                hmR3InitFinalizeR0Intel(PVM pVM);
+static int                hmR3InitFinalizeR0Amd(PVM pVM);
+static int                hmR3TermCPU(PVM pVM);
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * Returns the name of the hardware exception.
+ *
+ * @returns The name of the hardware exception.
+ * @param   uVector     The exception vector.
+ */
+static const char *hmR3GetXcptName(uint8_t uVector)
+{
+    switch (uVector)
+    {
+        case X86_XCPT_DE:             return "#DE";
+        case X86_XCPT_DB:             return "#DB";
+        case X86_XCPT_NMI:            return "#NMI";
+        case X86_XCPT_BP:             return "#BP";
+        case X86_XCPT_OF:             return "#OF";
+        case X86_XCPT_BR:             return "#BR";
+        case X86_XCPT_UD:             return "#UD";
+        case X86_XCPT_NM:             return "#NM";
+        case X86_XCPT_DF:             return "#DF";
+        case X86_XCPT_CO_SEG_OVERRUN: return "#CO_SEG_OVERRUN";
+        case X86_XCPT_TS:             return "#TS";
+        case X86_XCPT_NP:             return "#NP";
+        case X86_XCPT_SS:             return "#SS";
+        case X86_XCPT_GP:             return "#GP";
+        case X86_XCPT_PF:             return "#PF";
+        case X86_XCPT_MF:             return "#MF";
+        case X86_XCPT_AC:             return "#AC";
+        case X86_XCPT_MC:             return "#MC";
+        case X86_XCPT_XF:             return "#XF";
+        case X86_XCPT_VE:             return "#VE";
+        case X86_XCPT_CP:             return "#CP";
+        case X86_XCPT_VC:             return "#VC";
+        case X86_XCPT_SX:             return "#SX";
+    }
+    return "Reserved";
+}
+#endif /* VBOX_WITH_STATISTICS */
+
+
+/**
+ * Initializes the HM.
+ *
+ * This is the very first component to really do init after CFGM so that we can
+ * establish the predominant execution engine for the VM prior to initializing
+ * other modules.  It takes care of NEM initialization if needed (HM disabled or
+ * not available in HW).
+ *
+ * If VT-x or AMD-V hardware isn't available, HM will try fall back on a native
+ * hypervisor API via NEM, and then back on raw-mode if that isn't available
+ * either.  The fallback to raw-mode will not happen if /HM/HMForced is set
+ * (like for guest using SMP or 64-bit as well as for complicated guest like OS
+ * X, OS/2 and others).
+ *
+ * Note that a lot of the set up work is done in ring-0 and thus postponed till
+ * the ring-3 and ring-0 callback to HMR3InitCompleted.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks Be careful with what we call here, since most of the VMM components
+ *          are uninitialized.
+ */
+VMMR3_INT_DECL(int) HMR3Init(PVM pVM)
+{
+    LogFlowFunc(("\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompileMemberAlignment(VM, hm.s, 32);
+    AssertCompile(sizeof(pVM->hm.s) <= sizeof(pVM->hm.padding));
+
+    /*
+     * Register the saved state data unit.
+     */
+    int rc = SSMR3RegisterInternal(pVM, "HWACCM", 0, HM_SAVED_STATE_VERSION, sizeof(HM),
+                                   NULL, NULL, NULL,
+                                   NULL, hmR3Save, NULL,
+                                   NULL, hmR3Load, NULL);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Register info handlers.
+     */
+    rc = DBGFR3InfoRegisterInternalEx(pVM, "hm", "Dumps HM info.", hmR3Info, DBGFINFO_FLAGS_ALL_EMTS);
+    AssertRCReturn(rc, rc);
+
+    rc = DBGFR3InfoRegisterInternalEx(pVM, "hmeventpending", "Dumps the pending HM event.", hmR3InfoEventPending,
+                                      DBGFINFO_FLAGS_ALL_EMTS);
+    AssertRCReturn(rc, rc);
+
+    rc = DBGFR3InfoRegisterInternalEx(pVM, "svmvmcbcache", "Dumps the HM SVM nested-guest VMCB cache.",
+                                      hmR3InfoSvmNstGstVmcbCache, DBGFINFO_FLAGS_ALL_EMTS);
+    AssertRCReturn(rc, rc);
+
+    rc = DBGFR3InfoRegisterInternalEx(pVM, "lbr", "Dumps the HM LBR info.", hmR3InfoLbr, DBGFINFO_FLAGS_ALL_EMTS);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Read configuration.
+     */
+    PCFGMNODE pCfgHm = CFGMR3GetChild(CFGMR3GetRoot(pVM), "HM/");
+
+    /*
+     * Validate the HM settings.
+     */
+    rc = CFGMR3ValidateConfig(pCfgHm, "/HM/",
+                              "HMForced"  /* implied 'true' these days */
+                              "|UseNEMInstead"
+                              "|FallbackToNEM"
+                              "|EnableNestedPaging"
+                              "|EnableUX"
+                              "|EnableLargePages"
+                              "|EnableVPID"
+                              "|IBPBOnVMExit"
+                              "|IBPBOnVMEntry"
+                              "|SpecCtrlByHost"
+                              "|L1DFlushOnSched"
+                              "|L1DFlushOnVMEntry"
+                              "|MDSClearOnSched"
+                              "|MDSClearOnVMEntry"
+                              "|TPRPatchingEnabled"
+                              "|64bitEnabled"
+                              "|Exclusive"
+                              "|MaxResumeLoops"
+                              "|VmxPleGap"
+                              "|VmxPleWindow"
+                              "|VmxLbr"
+                              "|UseVmxPreemptTimer"
+                              "|SvmPauseFilter"
+                              "|SvmPauseFilterThreshold"
+                              "|SvmVirtVmsaveVmload"
+                              "|SvmVGif"
+                              "|LovelyMesaDrvWorkaround",
+                              "" /* pszValidNodes */, "HM" /* pszWho */, 0 /* uInstance */);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /** @cfgm{/HM/HMForced, bool, false}
+     * Forces hardware virtualization, no falling back on raw-mode. HM must be
+     * enabled, i.e. /HMEnabled must be true. */
+    bool fHMForced;
+    AssertRelease(pVM->fHMEnabled);
+    fHMForced = true;
+
+    /** @cfgm{/HM/UseNEMInstead, bool, true}
+     * Don't use HM, use NEM instead. */
+    bool fUseNEMInstead = false;
+    rc = CFGMR3QueryBoolDef(pCfgHm, "UseNEMInstead", &fUseNEMInstead, false);
+    AssertRCReturn(rc, rc);
+    if (fUseNEMInstead && pVM->fHMEnabled)
+    {
+        LogRel(("HM: Setting fHMEnabled to false because fUseNEMInstead is set.\n"));
+        pVM->fHMEnabled = false;
+    }
+
+    /** @cfgm{/HM/FallbackToNEM, bool, true}
+     * Enables fallback on NEM. */
+    bool fFallbackToNEM = true;
+    rc = CFGMR3QueryBoolDef(pCfgHm, "FallbackToNEM", &fFallbackToNEM, true);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/EnableNestedPaging, bool, false}
+     * Enables nested paging (aka extended page tables). */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "EnableNestedPaging", &pVM->hm.s.fAllowNestedPaging, false);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/EnableUX, bool, true}
+     * Enables the VT-x unrestricted execution feature. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "EnableUX", &pVM->hm.s.vmx.fAllowUnrestricted, true);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/EnableLargePages, bool, false}
+     * Enables using large pages (2 MB) for guest memory, thus saving on (nested)
+     * page table walking and maybe better TLB hit rate in some cases. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "EnableLargePages", &pVM->hm.s.fLargePages, false);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/EnableVPID, bool, false}
+     * Enables the VT-x VPID feature. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "EnableVPID", &pVM->hm.s.vmx.fAllowVpid, false);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/TPRPatchingEnabled, bool, false}
+     * Enables TPR patching for 32-bit windows guests with IO-APIC. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "TPRPatchingEnabled", &pVM->hm.s.fTprPatchingAllowed, false);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/64bitEnabled, bool, 32-bit:false, 64-bit:true}
+     * Enables AMD64 cpu features.
+     * On 32-bit hosts this isn't default and require host CPU support. 64-bit hosts
+     * already have the support. */
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    rc = CFGMR3QueryBoolDef(pCfgHm, "64bitEnabled", &pVM->hm.s.fAllow64BitGuests, HC_ARCH_BITS == 64);
+    AssertLogRelRCReturn(rc, rc);
+#else
+    pVM->hm.s.fAllow64BitGuests = false;
+#endif
+
+    /** @cfgm{/HM/VmxPleGap, uint32_t, 0}
+     * The pause-filter exiting gap in TSC ticks. When the number of ticks between
+     * two successive PAUSE instructions exceeds VmxPleGap, the CPU considers the
+     * latest PAUSE instruction to be start of a new PAUSE loop.
+     */
+    rc = CFGMR3QueryU32Def(pCfgHm, "VmxPleGap", &pVM->hm.s.vmx.cPleGapTicks, 0);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/VmxPleWindow, uint32_t, 0}
+     * The pause-filter exiting window in TSC ticks. When the number of ticks
+     * between the current PAUSE instruction and first PAUSE of a loop exceeds
+     * VmxPleWindow, a VM-exit is triggered.
+     *
+     * Setting VmxPleGap and VmxPleGap to 0 disables pause-filter exiting.
+     */
+    rc = CFGMR3QueryU32Def(pCfgHm, "VmxPleWindow", &pVM->hm.s.vmx.cPleWindowTicks, 0);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/VmxLbr, bool, false}
+     * Whether to enable LBR for the guest. This is disabled by default as it's only
+     * useful while debugging and enabling it causes a noticeable performance hit. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "VmxLbr", &pVM->hm.s.vmx.fLbr, false);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/SvmPauseFilterCount, uint16_t, 0}
+     * A counter that is decrement each time a PAUSE instruction is executed by the
+     * guest. When the counter is 0, a \#VMEXIT is triggered.
+     *
+     * Setting SvmPauseFilterCount to 0 disables pause-filter exiting.
+     */
+    rc = CFGMR3QueryU16Def(pCfgHm, "SvmPauseFilter", &pVM->hm.s.svm.cPauseFilter, 0);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/SvmPauseFilterThreshold, uint16_t, 0}
+     * The pause filter threshold in ticks. When the elapsed time (in ticks) between
+     * two successive PAUSE instructions exceeds SvmPauseFilterThreshold, the
+     * PauseFilter count is reset to its initial value. However, if PAUSE is
+     * executed PauseFilter times within PauseFilterThreshold ticks, a VM-exit will
+     * be triggered.
+     *
+     * Requires SvmPauseFilterCount to be non-zero for pause-filter threshold to be
+     * activated.
+     */
+    rc = CFGMR3QueryU16Def(pCfgHm, "SvmPauseFilterThreshold", &pVM->hm.s.svm.cPauseFilterThresholdTicks, 0);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/SvmVirtVmsaveVmload, bool, true}
+     * Whether to make use of virtualized VMSAVE/VMLOAD feature of the CPU if it's
+     * available. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "SvmVirtVmsaveVmload", &pVM->hm.s.svm.fVirtVmsaveVmload, true);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/SvmVGif, bool, true}
+     * Whether to make use of Virtual GIF (Global Interrupt Flag) feature of the CPU
+     * if it's available. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "SvmVGif", &pVM->hm.s.svm.fVGif, true);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/SvmLbrVirt, bool, false}
+     * Whether to make use of the LBR virtualization feature of the CPU if it's
+     * available. This is disabled by default as it's only useful while debugging
+     * and enabling it causes a small hit to performance. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "SvmLbrVirt", &pVM->hm.s.svm.fLbrVirt, false);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{/HM/Exclusive, bool}
+     * Determines the init method for AMD-V and VT-x. If set to true, HM will do a
+     * global init for each host CPU.  If false, we do local init each time we wish
+     * to execute guest code.
+     *
+     * On Windows, default is false due to the higher risk of conflicts with other
+     * hypervisors.
+     *
+     * On Mac OS X, this setting is ignored since the code does not handle local
+     * init when it utilizes the OS provided VT-x function, SUPR0EnableVTx().
+     */
+#if defined(RT_OS_DARWIN)
+    pVM->hm.s.fGlobalInit = true;
+#else
+    rc = CFGMR3QueryBoolDef(pCfgHm, "Exclusive", &pVM->hm.s.fGlobalInit,
+# if defined(RT_OS_WINDOWS)
+                            false
+# else
+                            true
+# endif
+                           );
+    AssertLogRelRCReturn(rc, rc);
+#endif
+
+    /** @cfgm{/HM/MaxResumeLoops, uint32_t}
+     * The number of times to resume guest execution before we forcibly return to
+     * ring-3.  The return value of RTThreadPreemptIsPendingTrusty in ring-0
+     * determines the default value. */
+    rc = CFGMR3QueryU32Def(pCfgHm, "MaxResumeLoops", &pVM->hm.s.cMaxResumeLoops, 0 /* set by R0 later */);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/HM/UseVmxPreemptTimer, bool}
+     * Whether to make use of the VMX-preemption timer feature of the CPU if it's
+     * available. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "UseVmxPreemptTimer", &pVM->hm.s.vmx.fUsePreemptTimer, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/HM/IBPBOnVMExit, bool}
+     * Costly paranoia setting. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "IBPBOnVMExit", &pVM->hm.s.fIbpbOnVmExit, false);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/HM/IBPBOnVMEntry, bool}
+     * Costly paranoia setting. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "IBPBOnVMEntry", &pVM->hm.s.fIbpbOnVmEntry, false);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/HM/L1DFlushOnSched, bool, true}
+     * CVE-2018-3646 workaround, ignored on CPUs that aren't affected. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "L1DFlushOnSched", &pVM->hm.s.fL1dFlushOnSched, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/HM/L1DFlushOnVMEntry, bool}
+     * CVE-2018-3646 workaround, ignored on CPUs that aren't affected. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "L1DFlushOnVMEntry", &pVM->hm.s.fL1dFlushOnVmEntry, false);
+    AssertLogRelRCReturn(rc, rc);
+
+    /* Disable L1DFlushOnSched if L1DFlushOnVMEntry is enabled. */
+    if (pVM->hm.s.fL1dFlushOnVmEntry)
+        pVM->hm.s.fL1dFlushOnSched = false;
+
+    /** @cfgm{/HM/SpecCtrlByHost, bool}
+     * Another expensive paranoia setting. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "SpecCtrlByHost", &pVM->hm.s.fSpecCtrlByHost, false);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/HM/MDSClearOnSched, bool, true}
+     * CVE-2018-12126, CVE-2018-12130, CVE-2018-12127, CVE-2019-11091 workaround,
+     * ignored on CPUs that aren't affected. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "MDSClearOnSched", &pVM->hm.s.fMdsClearOnSched, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/HM/MDSClearOnVmEntry, bool, false}
+     * CVE-2018-12126, CVE-2018-12130, CVE-2018-12127, CVE-2019-11091 workaround,
+     * ignored on CPUs that aren't affected. */
+    rc = CFGMR3QueryBoolDef(pCfgHm, "MDSClearOnVmEntry", &pVM->hm.s.fMdsClearOnVmEntry, false);
+    AssertLogRelRCReturn(rc, rc);
+
+    /* Disable MDSClearOnSched if MDSClearOnVmEntry is enabled. */
+    if (pVM->hm.s.fMdsClearOnVmEntry)
+        pVM->hm.s.fMdsClearOnSched = false;
+
+    /** @cfgm{/HM/LovelyMesaDrvWorkaround,bool}
+     * Workaround for mesa vmsvga 3d driver making incorrect assumptions about
+     * the hypervisor it is running under. */
+    bool f;
+    rc = CFGMR3QueryBoolDef(pCfgHm, "LovelyMesaDrvWorkaround", &f, false);
+    AssertLogRelRCReturn(rc, rc);
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv = f;
+    }
+
+    /*
+     * Check if VT-x or AMD-v support according to the users wishes.
+     */
+    /** @todo SUPR3QueryVTCaps won't catch VERR_VMX_IN_VMX_ROOT_MODE or
+     *        VERR_SVM_IN_USE. */
+    if (pVM->fHMEnabled)
+    {
+        uint32_t fCaps;
+        rc = SUPR3QueryVTCaps(&fCaps);
+        if (RT_SUCCESS(rc))
+        {
+            if (fCaps & SUPVTCAPS_AMD_V)
+            {
+                pVM->hm.s.svm.fSupported = true;
+                LogRel(("HM: HMR3Init: AMD-V%s\n", fCaps & SUPVTCAPS_NESTED_PAGING ? " w/ nested paging" : ""));
+                VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_HW_VIRT);
+            }
+            else if (fCaps & SUPVTCAPS_VT_X)
+            {
+                const char *pszWhy;
+                rc = SUPR3QueryVTxSupported(&pszWhy);
+                if (RT_SUCCESS(rc))
+                {
+                    pVM->hm.s.vmx.fSupported = true;
+                    LogRel(("HM: HMR3Init: VT-x%s%s%s\n",
+                            fCaps & SUPVTCAPS_NESTED_PAGING ? " w/ nested paging" : "",
+                            fCaps & SUPVTCAPS_VTX_UNRESTRICTED_GUEST ? " and unrestricted guest execution" : "",
+                            (fCaps & (SUPVTCAPS_NESTED_PAGING | SUPVTCAPS_VTX_UNRESTRICTED_GUEST)) ? " hw support" : ""));
+                    VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_HW_VIRT);
+                }
+                else
+                {
+                    /*
+                     * Before failing, try fallback to NEM if we're allowed to do that.
+                     */
+                    pVM->fHMEnabled = false;
+                    Assert(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NOT_SET);
+                    if (fFallbackToNEM)
+                    {
+                        LogRel(("HM: HMR3Init: Attempting fall back to NEM: The host kernel does not support VT-x - %s\n", pszWhy));
+                        int rc2 = NEMR3Init(pVM, true /*fFallback*/, fHMForced);
+
+                        ASMCompilerBarrier(); /* NEMR3Init may have changed bMainExecutionEngine. */
+                        if (   RT_SUCCESS(rc2)
+                            && pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NOT_SET)
+                            rc = VINF_SUCCESS;
+                    }
+                    if (RT_FAILURE(rc))
+                        return VMSetError(pVM, rc, RT_SRC_POS, "The host kernel does not support VT-x: %s\n", pszWhy);
+                }
+            }
+            else
+                AssertLogRelMsgFailedReturn(("SUPR3QueryVTCaps didn't return either AMD-V or VT-x flag set (%#x)!\n", fCaps),
+                                            VERR_INTERNAL_ERROR_5);
+
+            /*
+             * Disable nested paging and unrestricted guest execution now if they're
+             * configured so that CPUM can make decisions based on our configuration.
+             */
+            Assert(!pVM->hm.s.fNestedPaging);
+            if (pVM->hm.s.fAllowNestedPaging)
+            {
+                if (fCaps & SUPVTCAPS_NESTED_PAGING)
+                    pVM->hm.s.fNestedPaging = true;
+                else
+                    pVM->hm.s.fAllowNestedPaging = false;
+            }
+
+            if (fCaps & SUPVTCAPS_VT_X)
+            {
+                Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
+                if (pVM->hm.s.vmx.fAllowUnrestricted)
+                {
+                    if (   (fCaps & SUPVTCAPS_VTX_UNRESTRICTED_GUEST)
+                        && pVM->hm.s.fNestedPaging)
+                        pVM->hm.s.vmx.fUnrestrictedGuest = true;
+                    else
+                        pVM->hm.s.vmx.fAllowUnrestricted = false;
+                }
+            }
+        }
+        else
+        {
+            const char *pszMsg;
+            switch (rc)
+            {
+                case VERR_UNSUPPORTED_CPU:          pszMsg = "Unknown CPU, VT-x or AMD-v features cannot be ascertained"; break;
+                case VERR_VMX_NO_VMX:               pszMsg = "VT-x is not available"; break;
+                case VERR_VMX_MSR_VMX_DISABLED:     pszMsg = "VT-x is disabled in the BIOS"; break;
+                case VERR_VMX_MSR_ALL_VMX_DISABLED: pszMsg = "VT-x is disabled in the BIOS for all CPU modes"; break;
+                case VERR_VMX_MSR_LOCKING_FAILED:   pszMsg = "Failed to enable and lock VT-x features"; break;
+                case VERR_SVM_NO_SVM:               pszMsg = "AMD-V is not available"; break;
+                case VERR_SVM_DISABLED:             pszMsg = "AMD-V is disabled in the BIOS (or by the host OS)"; break;
+                default:
+                    return VMSetError(pVM, rc, RT_SRC_POS, "SUPR3QueryVTCaps failed with %Rrc", rc);
+            }
+
+            /*
+             * Before failing, try fallback to NEM if we're allowed to do that.
+             */
+            pVM->fHMEnabled = false;
+            if (fFallbackToNEM)
+            {
+                LogRel(("HM: HMR3Init: Attempting fall back to NEM: %s\n", pszMsg));
+                int rc2 = NEMR3Init(pVM, true /*fFallback*/, fHMForced);
+                ASMCompilerBarrier(); /* NEMR3Init may have changed bMainExecutionEngine. */
+                if (   RT_SUCCESS(rc2)
+                    && pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NOT_SET)
+                    rc = VINF_SUCCESS;
+            }
+            if (RT_FAILURE(rc))
+                return VM_SET_ERROR(pVM, rc, pszMsg);
+        }
+    }
+    else
+    {
+        /*
+         * Disabled HM mean raw-mode, unless NEM is supposed to be used.
+         */
+        if (fUseNEMInstead)
+        {
+            rc = NEMR3Init(pVM, false /*fFallback*/, true);
+            ASMCompilerBarrier(); /* NEMR3Init may have changed bMainExecutionEngine. */
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+        if (   pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NOT_SET
+            || pVM->bMainExecutionEngine == VM_EXEC_ENGINE_RAW_MODE
+            || pVM->bMainExecutionEngine == VM_EXEC_ENGINE_HW_VIRT /* paranoia */)
+            return VM_SET_ERROR(pVM, rc, "Misconfigured VM: No guest execution engine available!");
+    }
+
+    Assert(pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NOT_SET);
+    Assert(pVM->bMainExecutionEngine != VM_EXEC_ENGINE_RAW_MODE);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Initializes HM components after ring-3 phase has been fully initialized.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+static int hmR3InitFinalizeR3(PVM pVM)
+{
+    LogFlowFunc(("\n"));
+
+    if (!HMIsEnabled(pVM))
+        return VINF_SUCCESS;
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        pVCpu->hm.s.fActive = false;
+        pVCpu->hm.s.fGIMTrapXcptUD = GIMShouldTrapXcptUD(pVCpu);    /* Is safe to call now since GIMR3Init() has completed. */
+    }
+
+#ifdef VBOX_WITH_STATISTICS
+    STAM_REG(pVM, &pVM->hm.s.StatTprPatchSuccess,      STAMTYPE_COUNTER, "/HM/TPR/Patch/Success",      STAMUNIT_OCCURENCES, "Number of times an instruction was successfully patched.");
+    STAM_REG(pVM, &pVM->hm.s.StatTprPatchFailure,      STAMTYPE_COUNTER, "/HM/TPR/Patch/Failed",       STAMUNIT_OCCURENCES, "Number of unsuccessful patch attempts.");
+    STAM_REG(pVM, &pVM->hm.s.StatTprReplaceSuccessCr8, STAMTYPE_COUNTER, "/HM/TPR/Replace/SuccessCR8", STAMUNIT_OCCURENCES, "Number of instruction replacements by MOV CR8.");
+    STAM_REG(pVM, &pVM->hm.s.StatTprReplaceSuccessVmc, STAMTYPE_COUNTER, "/HM/TPR/Replace/SuccessVMC", STAMUNIT_OCCURENCES, "Number of instruction replacements by VMMCALL.");
+    STAM_REG(pVM, &pVM->hm.s.StatTprReplaceFailure,    STAMTYPE_COUNTER, "/HM/TPR/Replace/Failed",     STAMUNIT_OCCURENCES, "Number of unsuccessful replace attempts.");
+#endif
+
+    /*
+     * Statistics.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu  = pVM->apCpusR3[idCpu];
+        PHMCPU pHmCpu = &pVCpu->hm.s;
+        int    rc;
+
+# define HM_REG_STAT(a_pVar, a_enmType, s_enmVisibility, a_enmUnit, a_szNmFmt, a_szDesc) do { \
+                rc = STAMR3RegisterF(pVM, a_pVar, a_enmType, s_enmVisibility, a_enmUnit, a_szDesc, a_szNmFmt, idCpu); \
+                AssertRC(rc); \
+            } while (0)
+# define HM_REG_PROFILE(a_pVar, a_szNmFmt, a_szDesc) \
+            HM_REG_STAT(a_pVar, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, a_szNmFmt, a_szDesc)
+
+#ifdef VBOX_WITH_STATISTICS
+
+        HM_REG_PROFILE(&pHmCpu->StatPoke,                   "/PROF/CPU%u/HM/Poke", "Profiling of RTMpPokeCpu.");
+        HM_REG_PROFILE(&pHmCpu->StatSpinPoke,               "/PROF/CPU%u/HM/PokeWait", "Profiling of poke wait.");
+        HM_REG_PROFILE(&pHmCpu->StatSpinPokeFailed,         "/PROF/CPU%u/HM/PokeWaitFailed", "Profiling of poke wait when RTMpPokeCpu fails.");
+        HM_REG_PROFILE(&pHmCpu->StatEntry,                  "/PROF/CPU%u/HM/Entry", "Profiling of entry until entering GC.");
+        HM_REG_PROFILE(&pHmCpu->StatPreExit,                "/PROF/CPU%u/HM/SwitchFromGC_1", "Profiling of pre-exit processing after returning from GC.");
+        HM_REG_PROFILE(&pHmCpu->StatExitHandling,           "/PROF/CPU%u/HM/SwitchFromGC_2", "Profiling of exit handling (longjmps not included!)");
+        HM_REG_PROFILE(&pHmCpu->StatExitIO,                 "/PROF/CPU%u/HM/SwitchFromGC_2/IO", "I/O.");
+        HM_REG_PROFILE(&pHmCpu->StatExitMovCRx,             "/PROF/CPU%u/HM/SwitchFromGC_2/MovCRx", "MOV CRx.");
+        HM_REG_PROFILE(&pHmCpu->StatExitXcptNmi,            "/PROF/CPU%u/HM/SwitchFromGC_2/XcptNmi", "Exceptions, NMIs.");
+        HM_REG_PROFILE(&pHmCpu->StatExitVmentry,            "/PROF/CPU%u/HM/SwitchFromGC_2/Vmentry", "VMLAUNCH/VMRESUME on Intel or VMRUN on AMD.");
+        HM_REG_PROFILE(&pHmCpu->StatImportGuestState,       "/PROF/CPU%u/HM/ImportGuestState",  "Profiling of importing guest state from hardware after VM-exit.");
+        HM_REG_PROFILE(&pHmCpu->StatExportGuestState,       "/PROF/CPU%u/HM/ExportGuestState",  "Profiling of exporting guest state to hardware before VM-entry.");
+        HM_REG_PROFILE(&pHmCpu->StatLoadGuestFpuState,      "/PROF/CPU%u/HM/LoadGuestFpuState", "Profiling of CPUMR0LoadGuestFPU.");
+        HM_REG_PROFILE(&pHmCpu->StatInGC,                   "/PROF/CPU%u/HM/InGC", "Profiling of execution of guest-code in hardware.");
+# ifdef HM_PROFILE_EXIT_DISPATCH
+        HM_REG_STAT(&pHmCpu->StatExitDispatch, STAMTYPE_PROFILE_ADV, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL,
+                    "/PROF/CPU%u/HM/ExitDispatch", "Profiling the dispatching of exit handlers.");
+# endif
+#endif
+# define HM_REG_COUNTER(a, b, desc) HM_REG_STAT(a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, b, desc)
+
+#ifdef VBOX_WITH_STATISTICS
+        HM_REG_COUNTER(&pHmCpu->StatExitAll,                "/HM/CPU%u/Exit/All", "Total exits (including nested-guest exits).");
+        HM_REG_COUNTER(&pHmCpu->StatNestedExitAll,          "/HM/CPU%u/Exit/NestedGuest/All", "Total nested-guest exits.");
+        HM_REG_COUNTER(&pHmCpu->StatExitShadowNM,           "/HM/CPU%u/Exit/Trap/Shw/#NM", "Shadow #NM (device not available, no math co-processor) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestNM,            "/HM/CPU%u/Exit/Trap/Gst/#NM", "Guest #NM (device not available, no math co-processor) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitShadowPF,           "/HM/CPU%u/Exit/Trap/Shw/#PF", "Shadow #PF (page fault) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitShadowPFEM,         "/HM/CPU%u/Exit/Trap/Shw/#PF-EM", "#PF (page fault) exception going back to ring-3 for emulating the instruction.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestPF,            "/HM/CPU%u/Exit/Trap/Gst/#PF", "Guest #PF (page fault) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestUD,            "/HM/CPU%u/Exit/Trap/Gst/#UD", "Guest #UD (undefined opcode) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestSS,            "/HM/CPU%u/Exit/Trap/Gst/#SS", "Guest #SS (stack-segment fault) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestNP,            "/HM/CPU%u/Exit/Trap/Gst/#NP", "Guest #NP (segment not present) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestTS,            "/HM/CPU%u/Exit/Trap/Gst/#TS", "Guest #TS (task switch) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestOF,            "/HM/CPU%u/Exit/Trap/Gst/#OF", "Guest #OF (overflow) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestGP,            "/HM/CPU%u/Exit/Trap/Gst/#GP", "Guest #GP (general protection) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestDE,            "/HM/CPU%u/Exit/Trap/Gst/#DE", "Guest #DE (divide error) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestDF,            "/HM/CPU%u/Exit/Trap/Gst/#DF", "Guest #DF (double fault) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestBR,            "/HM/CPU%u/Exit/Trap/Gst/#BR", "Guest #BR (boundary range exceeded) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestAC,            "/HM/CPU%u/Exit/Trap/Gst/#AC", "Guest #AC (alignment check) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestDB,            "/HM/CPU%u/Exit/Trap/Gst/#DB", "Guest #DB (debug) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestMF,            "/HM/CPU%u/Exit/Trap/Gst/#MF", "Guest #MF (x87 FPU error, math fault) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestBP,            "/HM/CPU%u/Exit/Trap/Gst/#BP", "Guest #BP (breakpoint) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestXF,            "/HM/CPU%u/Exit/Trap/Gst/#XF", "Guest #XF (extended math fault, SIMD FPU) exception.");
+        HM_REG_COUNTER(&pHmCpu->StatExitGuestXcpUnk,        "/HM/CPU%u/Exit/Trap/Gst/Other", "Other guest exceptions.");
+        HM_REG_COUNTER(&pHmCpu->StatExitRdmsr,              "/HM/CPU%u/Exit/Instr/Rdmsr", "MSR read.");
+        HM_REG_COUNTER(&pHmCpu->StatExitWrmsr,              "/HM/CPU%u/Exit/Instr/Wrmsr", "MSR write.");
+        HM_REG_COUNTER(&pHmCpu->StatExitDRxWrite,           "/HM/CPU%u/Exit/Instr/DR-Write", "Debug register write.");
+        HM_REG_COUNTER(&pHmCpu->StatExitDRxRead,            "/HM/CPU%u/Exit/Instr/DR-Read", "Debug register read.");
+        HM_REG_COUNTER(&pHmCpu->StatExitCR0Read,            "/HM/CPU%u/Exit/Instr/CR-Read/CR0", "CR0 read.");
+        HM_REG_COUNTER(&pHmCpu->StatExitCR2Read,            "/HM/CPU%u/Exit/Instr/CR-Read/CR2", "CR2 read.");
+        HM_REG_COUNTER(&pHmCpu->StatExitCR3Read,            "/HM/CPU%u/Exit/Instr/CR-Read/CR3", "CR3 read.");
+        HM_REG_COUNTER(&pHmCpu->StatExitCR4Read,            "/HM/CPU%u/Exit/Instr/CR-Read/CR4", "CR4 read.");
+        HM_REG_COUNTER(&pHmCpu->StatExitCR8Read,            "/HM/CPU%u/Exit/Instr/CR-Read/CR8", "CR8 read.");
+        HM_REG_COUNTER(&pHmCpu->StatExitCR0Write,           "/HM/CPU%u/Exit/Instr/CR-Write/CR0", "CR0 write.");
+        HM_REG_COUNTER(&pHmCpu->StatExitCR2Write,           "/HM/CPU%u/Exit/Instr/CR-Write/CR2", "CR2 write.");
+        HM_REG_COUNTER(&pHmCpu->StatExitCR3Write,           "/HM/CPU%u/Exit/Instr/CR-Write/CR3", "CR3 write.");
+        HM_REG_COUNTER(&pHmCpu->StatExitCR4Write,           "/HM/CPU%u/Exit/Instr/CR-Write/CR4", "CR4 write.");
+        HM_REG_COUNTER(&pHmCpu->StatExitCR8Write,           "/HM/CPU%u/Exit/Instr/CR-Write/CR8", "CR8 write.");
+        HM_REG_COUNTER(&pHmCpu->StatExitClts,               "/HM/CPU%u/Exit/Instr/CLTS", "CLTS instruction.");
+        HM_REG_COUNTER(&pHmCpu->StatExitLmsw,               "/HM/CPU%u/Exit/Instr/LMSW", "LMSW instruction.");
+        HM_REG_COUNTER(&pHmCpu->StatExitXdtrAccess,         "/HM/CPU%u/Exit/Instr/XdtrAccess", "GDTR, IDTR, LDTR access.");
+        HM_REG_COUNTER(&pHmCpu->StatExitIOWrite,            "/HM/CPU%u/Exit/Instr/IO/Write", "I/O write.");
+        HM_REG_COUNTER(&pHmCpu->StatExitIORead,             "/HM/CPU%u/Exit/Instr/IO/Read", "I/O read.");
+        HM_REG_COUNTER(&pHmCpu->StatExitIOStringWrite,      "/HM/CPU%u/Exit/Instr/IO/WriteString", "String I/O write.");
+        HM_REG_COUNTER(&pHmCpu->StatExitIOStringRead,       "/HM/CPU%u/Exit/Instr/IO/ReadString", "String I/O read.");
+        HM_REG_COUNTER(&pHmCpu->StatExitIntWindow,          "/HM/CPU%u/Exit/IntWindow", "Interrupt-window exit. Guest is ready to receive interrupts.");
+        HM_REG_COUNTER(&pHmCpu->StatExitExtInt,             "/HM/CPU%u/Exit/ExtInt", "Physical maskable interrupt (host).");
+#endif
+        HM_REG_COUNTER(&pHmCpu->StatExitHostNmiInGC,        "/HM/CPU%u/Exit/HostNmiInGC", "Host NMI received while in guest context.");
+        HM_REG_COUNTER(&pHmCpu->StatExitHostNmiInGCIpi,     "/HM/CPU%u/Exit/HostNmiInGCIpi", "Host NMI received while in guest context dispatched using IPIs.");
+#ifdef VBOX_WITH_STATISTICS
+        HM_REG_COUNTER(&pHmCpu->StatExitPreemptTimer,       "/HM/CPU%u/Exit/PreemptTimer", "VMX-preemption timer expired.");
+        HM_REG_COUNTER(&pHmCpu->StatExitTprBelowThreshold,  "/HM/CPU%u/Exit/TprBelowThreshold", "TPR lowered below threshold by the guest.");
+        HM_REG_COUNTER(&pHmCpu->StatExitTaskSwitch,         "/HM/CPU%u/Exit/TaskSwitch", "Task switch caused through task gate in IDT.");
+        HM_REG_COUNTER(&pHmCpu->StatExitApicAccess,         "/HM/CPU%u/Exit/ApicAccess", "APIC access. Guest attempted to access memory at a physical address on the APIC-access page.");
+
+        HM_REG_COUNTER(&pHmCpu->StatSwitchTprMaskedIrq,     "/HM/CPU%u/Switch/TprMaskedIrq", "PDMGetInterrupt() signals TPR masks pending Irq.");
+        HM_REG_COUNTER(&pHmCpu->StatSwitchGuestIrq,         "/HM/CPU%u/Switch/IrqPending", "PDMGetInterrupt() cleared behind our back!?!.");
+        HM_REG_COUNTER(&pHmCpu->StatSwitchPendingHostIrq,   "/HM/CPU%u/Switch/PendingHostIrq", "Exit to ring-3 due to pending host interrupt before executing guest code.");
+        HM_REG_COUNTER(&pHmCpu->StatSwitchHmToR3FF,         "/HM/CPU%u/Switch/HmToR3FF", "Exit to ring-3 due to pending timers, EMT rendezvous, critical section etc.");
+        HM_REG_COUNTER(&pHmCpu->StatSwitchVmReq,            "/HM/CPU%u/Switch/VmReq", "Exit to ring-3 due to pending VM requests.");
+        HM_REG_COUNTER(&pHmCpu->StatSwitchPgmPoolFlush,     "/HM/CPU%u/Switch/PgmPoolFlush", "Exit to ring-3 due to pending PGM pool flush.");
+        HM_REG_COUNTER(&pHmCpu->StatSwitchDma,              "/HM/CPU%u/Switch/PendingDma", "Exit to ring-3 due to pending DMA requests.");
+        HM_REG_COUNTER(&pHmCpu->StatSwitchExitToR3,         "/HM/CPU%u/Switch/ExitToR3", "Exit to ring-3 (total).");
+        HM_REG_COUNTER(&pHmCpu->StatSwitchLongJmpToR3,      "/HM/CPU%u/Switch/LongJmpToR3", "Longjump to ring-3.");
+        HM_REG_COUNTER(&pHmCpu->StatSwitchMaxResumeLoops,   "/HM/CPU%u/Switch/MaxResumeLoops", "Maximum VMRESUME inner-loop counter reached.");
+        HM_REG_COUNTER(&pHmCpu->StatSwitchHltToR3,          "/HM/CPU%u/Switch/HltToR3", "HLT causing us to go to ring-3.");
+        HM_REG_COUNTER(&pHmCpu->StatSwitchApicAccessToR3,   "/HM/CPU%u/Switch/ApicAccessToR3", "APIC access causing us to go to ring-3.");
+#endif
+        HM_REG_COUNTER(&pHmCpu->StatSwitchPreempt,          "/HM/CPU%u/Switch/Preempting", "EMT has been preempted while in HM context.");
+#ifdef VBOX_WITH_STATISTICS
+        HM_REG_COUNTER(&pHmCpu->StatSwitchNstGstVmexit,     "/HM/CPU%u/Switch/NstGstVmexit", "Nested-guest VM-exit occurred.");
+
+        HM_REG_COUNTER(&pHmCpu->StatInjectInterrupt,        "/HM/CPU%u/EventInject/Interrupt", "Injected an external interrupt into the guest.");
+        HM_REG_COUNTER(&pHmCpu->StatInjectXcpt,             "/HM/CPU%u/EventInject/Trap", "Injected an exception into the guest.");
+        HM_REG_COUNTER(&pHmCpu->StatInjectReflect,          "/HM/CPU%u/EventInject/Reflect", "Reflecting an exception caused due to event injection.");
+        HM_REG_COUNTER(&pHmCpu->StatInjectConvertDF,        "/HM/CPU%u/EventInject/ReflectDF", "Injected a converted #DF caused due to event injection.");
+        HM_REG_COUNTER(&pHmCpu->StatInjectInterpret,        "/HM/CPU%u/EventInject/Interpret", "Falling back to interpreter for handling exception caused due to event injection.");
+        HM_REG_COUNTER(&pHmCpu->StatInjectReflectNPF,       "/HM/CPU%u/EventInject/ReflectNPF", "Reflecting event that caused an EPT violation / nested #PF.");
+
+        HM_REG_COUNTER(&pHmCpu->StatFlushPage,              "/HM/CPU%u/Flush/Page", "Invalidating a guest page on all guest CPUs.");
+        HM_REG_COUNTER(&pHmCpu->StatFlushPageManual,        "/HM/CPU%u/Flush/Page/Virt", "Invalidating a guest page using guest-virtual address.");
+        HM_REG_COUNTER(&pHmCpu->StatFlushPhysPageManual,    "/HM/CPU%u/Flush/Page/Phys", "Invalidating a guest page using guest-physical address.");
+        HM_REG_COUNTER(&pHmCpu->StatFlushTlb,               "/HM/CPU%u/Flush/TLB", "Forcing a full guest-TLB flush (ring-0).");
+        HM_REG_COUNTER(&pHmCpu->StatFlushTlbManual,         "/HM/CPU%u/Flush/TLB/Manual", "Request a full guest-TLB flush.");
+        HM_REG_COUNTER(&pHmCpu->StatFlushTlbNstGst,         "/HM/CPU%u/Flush/TLB/NestedGuest", "Request a nested-guest-TLB flush.");
+        HM_REG_COUNTER(&pHmCpu->StatFlushTlbWorldSwitch,    "/HM/CPU%u/Flush/TLB/CpuSwitch", "Forcing a full guest-TLB flush due to host-CPU reschedule or ASID-limit hit by another guest-VCPU.");
+        HM_REG_COUNTER(&pHmCpu->StatNoFlushTlbWorldSwitch,  "/HM/CPU%u/Flush/TLB/Skipped", "No TLB flushing required.");
+        HM_REG_COUNTER(&pHmCpu->StatFlushEntire,            "/HM/CPU%u/Flush/TLB/Entire", "Flush the entire TLB (host + guest).");
+        HM_REG_COUNTER(&pHmCpu->StatFlushAsid,              "/HM/CPU%u/Flush/TLB/ASID", "Flushed guest-TLB entries for the current VPID.");
+        HM_REG_COUNTER(&pHmCpu->StatFlushNestedPaging,      "/HM/CPU%u/Flush/TLB/NestedPaging", "Flushed guest-TLB entries for the current EPT.");
+        HM_REG_COUNTER(&pHmCpu->StatFlushTlbInvlpgVirt,     "/HM/CPU%u/Flush/TLB/InvlpgVirt", "Invalidated a guest-TLB entry for a guest-virtual address.");
+        HM_REG_COUNTER(&pHmCpu->StatFlushTlbInvlpgPhys,     "/HM/CPU%u/Flush/TLB/InvlpgPhys", "Currently not possible, flushes entire guest-TLB.");
+        HM_REG_COUNTER(&pHmCpu->StatTlbShootdown,           "/HM/CPU%u/Flush/Shootdown/Page", "Inter-VCPU request to flush queued guest page.");
+        HM_REG_COUNTER(&pHmCpu->StatTlbShootdownFlush,      "/HM/CPU%u/Flush/Shootdown/TLB", "Inter-VCPU request to flush entire guest-TLB.");
+
+        HM_REG_COUNTER(&pHmCpu->StatTscParavirt,            "/HM/CPU%u/TSC/Paravirt", "Paravirtualized TSC in effect.");
+        HM_REG_COUNTER(&pHmCpu->StatTscOffset,              "/HM/CPU%u/TSC/Offset", "TSC offsetting is in effect.");
+        HM_REG_COUNTER(&pHmCpu->StatTscIntercept,           "/HM/CPU%u/TSC/Intercept", "Intercept TSC accesses.");
+
+        HM_REG_COUNTER(&pHmCpu->StatDRxArmed,               "/HM/CPU%u/Debug/Armed", "Loaded guest-debug state while loading guest-state.");
+        HM_REG_COUNTER(&pHmCpu->StatDRxContextSwitch,       "/HM/CPU%u/Debug/ContextSwitch", "Loaded guest-debug state on MOV DRx.");
+        HM_REG_COUNTER(&pHmCpu->StatDRxIoCheck,             "/HM/CPU%u/Debug/IOCheck", "Checking for I/O breakpoint.");
+
+        HM_REG_COUNTER(&pHmCpu->StatExportMinimal,          "/HM/CPU%u/Export/Minimal", "VM-entry exporting minimal guest-state.");
+        HM_REG_COUNTER(&pHmCpu->StatExportFull,             "/HM/CPU%u/Export/Full", "VM-entry exporting the full guest-state.");
+        HM_REG_COUNTER(&pHmCpu->StatLoadGuestFpu,           "/HM/CPU%u/Export/GuestFpu", "VM-entry loading the guest-FPU state.");
+        HM_REG_COUNTER(&pHmCpu->StatExportHostState,        "/HM/CPU%u/Export/HostState", "VM-entry exporting host-state.");
+
+        HM_REG_COUNTER(&pHmCpu->StatVmxCheckBadRmSelBase,   "/HM/CPU%u/VMXCheck/RMSelBase", "Could not use VMX due to unsuitable real-mode selector base.");
+        HM_REG_COUNTER(&pHmCpu->StatVmxCheckBadRmSelLimit,  "/HM/CPU%u/VMXCheck/RMSelLimit", "Could not use VMX due to unsuitable real-mode selector limit.");
+        HM_REG_COUNTER(&pHmCpu->StatVmxCheckBadRmSelAttr,   "/HM/CPU%u/VMXCheck/RMSelAttrs", "Could not use VMX due to unsuitable real-mode selector attributes.");
+
+        HM_REG_COUNTER(&pHmCpu->StatVmxCheckBadV86SelBase,  "/HM/CPU%u/VMXCheck/V86SelBase",  "Could not use VMX due to unsuitable v8086-mode selector base.");
+        HM_REG_COUNTER(&pHmCpu->StatVmxCheckBadV86SelLimit, "/HM/CPU%u/VMXCheck/V86SelLimit", "Could not use VMX due to unsuitable v8086-mode selector limit.");
+        HM_REG_COUNTER(&pHmCpu->StatVmxCheckBadV86SelAttr,  "/HM/CPU%u/VMXCheck/V86SelAttrs", "Could not use VMX due to unsuitable v8086-mode selector attributes.");
+
+        HM_REG_COUNTER(&pHmCpu->StatVmxCheckRmOk,           "/HM/CPU%u/VMXCheck/VMX_RM", "VMX execution in real (V86) mode OK.");
+        HM_REG_COUNTER(&pHmCpu->StatVmxCheckBadSel,         "/HM/CPU%u/VMXCheck/Selector", "Could not use VMX due to unsuitable selector.");
+        HM_REG_COUNTER(&pHmCpu->StatVmxCheckBadRpl,         "/HM/CPU%u/VMXCheck/RPL", "Could not use VMX due to unsuitable RPL.");
+        HM_REG_COUNTER(&pHmCpu->StatVmxCheckPmOk,           "/HM/CPU%u/VMXCheck/VMX_PM", "VMX execution in protected mode OK.");
+
+        bool const fCpuSupportsVmx = ASMIsIntelCpu() || ASMIsViaCentaurCpu() || ASMIsShanghaiCpu();
+
+        /*
+         * Guest Exit reason stats.
+         */
+        pHmCpu->paStatExitReason = NULL;
+        rc = MMHyperAlloc(pVM, MAX_EXITREASON_STAT * sizeof(*pHmCpu->paStatExitReason), 0 /* uAlignment */, MM_TAG_HM,
+                          (void **)&pHmCpu->paStatExitReason);
+        AssertRCReturn(rc, rc);
+
+        if (fCpuSupportsVmx)
+        {
+            for (int j = 0; j < MAX_EXITREASON_STAT; j++)
+            {
+                const char *pszExitName = HMGetVmxExitName(j);
+                if (pszExitName)
+                {
+                    rc = STAMR3RegisterF(pVM, &pHmCpu->paStatExitReason[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED,
+                                         STAMUNIT_OCCURENCES, pszExitName, "/HM/CPU%u/Exit/Reason/%02x", idCpu, j);
+                    AssertRCReturn(rc, rc);
+                }
+            }
+        }
+        else
+        {
+            for (int j = 0; j < MAX_EXITREASON_STAT; j++)
+            {
+                const char *pszExitName = HMGetSvmExitName(j);
+                if (pszExitName)
+                {
+                    rc = STAMR3RegisterF(pVM, &pHmCpu->paStatExitReason[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED,
+                                         STAMUNIT_OCCURENCES, pszExitName, "/HM/CPU%u/Exit/Reason/%02x", idCpu, j);
+                    AssertRC(rc);
+                }
+            }
+        }
+        HM_REG_COUNTER(&pHmCpu->StatExitReasonNpf, "/HM/CPU%u/Exit/Reason/#NPF", "Nested page faults");
+
+        pHmCpu->paStatExitReasonR0 = MMHyperR3ToR0(pVM, pHmCpu->paStatExitReason);
+        Assert(pHmCpu->paStatExitReasonR0 != NIL_RTR0PTR);
+
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+        /*
+         * Nested-guest VM-exit reason stats.
+         */
+        pHmCpu->paStatNestedExitReason = NULL;
+        rc = MMHyperAlloc(pVM, MAX_EXITREASON_STAT * sizeof(*pHmCpu->paStatNestedExitReason), 0 /* uAlignment */, MM_TAG_HM,
+                          (void **)&pHmCpu->paStatNestedExitReason);
+        AssertRCReturn(rc, rc);
+        if (fCpuSupportsVmx)
+        {
+            for (int j = 0; j < MAX_EXITREASON_STAT; j++)
+            {
+                const char *pszExitName = HMGetVmxExitName(j);
+                if (pszExitName)
+                {
+                    rc = STAMR3RegisterF(pVM, &pHmCpu->paStatNestedExitReason[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED,
+                                         STAMUNIT_OCCURENCES, pszExitName, "/HM/CPU%u/Exit/NestedGuest/Reason/%02x", idCpu, j);
+                    AssertRC(rc);
+                }
+            }
+        }
+        else
+        {
+            for (int j = 0; j < MAX_EXITREASON_STAT; j++)
+            {
+                const char *pszExitName = HMGetSvmExitName(j);
+                if (pszExitName)
+                {
+                    rc = STAMR3RegisterF(pVM, &pHmCpu->paStatNestedExitReason[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED,
+                                         STAMUNIT_OCCURENCES, pszExitName, "/HM/CPU%u/Exit/NestedGuest/Reason/%02x", idCpu, j);
+                    AssertRC(rc);
+                }
+            }
+        }
+        HM_REG_COUNTER(&pHmCpu->StatNestedExitReasonNpf, "/HM/CPU%u/Exit/NestedGuest/Reason/#NPF", "Nested page faults");
+        pHmCpu->paStatNestedExitReasonR0 = MMHyperR3ToR0(pVM, pHmCpu->paStatNestedExitReason);
+        Assert(pHmCpu->paStatNestedExitReasonR0 != NIL_RTR0PTR);
+#endif
+
+        /*
+         * Injected interrupts stats.
+         */
+        {
+            uint32_t const cInterrupts = 0xff + 1;
+            rc = MMHyperAlloc(pVM, sizeof(STAMCOUNTER) * cInterrupts, 8, MM_TAG_HM, (void **)&pHmCpu->paStatInjectedIrqs);
+            AssertRCReturn(rc, rc);
+            pHmCpu->paStatInjectedIrqsR0 = MMHyperR3ToR0(pVM, pHmCpu->paStatInjectedIrqs);
+            Assert(pHmCpu->paStatInjectedIrqsR0 != NIL_RTR0PTR);
+            for (unsigned j = 0; j < cInterrupts; j++)
+            {
+                char aszIntrName[64];
+                RTStrPrintf(&aszIntrName[0], sizeof(aszIntrName),  "Interrupt %u", j);
+                rc = STAMR3RegisterF(pVM, &pHmCpu->paStatInjectedIrqs[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED,
+                                     STAMUNIT_OCCURENCES, aszIntrName,
+                                     "/HM/CPU%u/EventInject/InjectIntr/%02X", idCpu, j);
+                AssertRC(rc);
+            }
+        }
+
+        /*
+         * Injected exception stats.
+         */
+        {
+            uint32_t const cXcpts = X86_XCPT_LAST + 1;
+            rc = MMHyperAlloc(pVM, sizeof(STAMCOUNTER) * cXcpts, 8, MM_TAG_HM, (void **)&pHmCpu->paStatInjectedXcpts);
+            AssertRCReturn(rc, rc);
+            pHmCpu->paStatInjectedXcptsR0 = MMHyperR3ToR0(pVM, pHmCpu->paStatInjectedXcpts);
+            Assert(pHmCpu->paStatInjectedXcptsR0 != NIL_RTR0PTR);
+            for (unsigned j = 0; j < cXcpts; j++)
+            {
+                char aszXcptName[64];
+                RTStrPrintf(&aszXcptName[0], sizeof(aszXcptName),  "%s exception", hmR3GetXcptName(j));
+                rc = STAMR3RegisterF(pVM, &pHmCpu->paStatInjectedXcpts[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED,
+                                     STAMUNIT_OCCURENCES, aszXcptName,
+                                     "/HM/CPU%u/EventInject/InjectXcpt/%02X", idCpu, j);
+                AssertRC(rc);
+            }
+        }
+
+#endif /* VBOX_WITH_STATISTICS */
+#undef HM_REG_COUNTER
+#undef HM_REG_PROFILE
+#undef HM_REG_STAT
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Called when a init phase has completed.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   enmWhat             The phase that completed.
+ */
+VMMR3_INT_DECL(int) HMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    switch (enmWhat)
+    {
+        case VMINITCOMPLETED_RING3:
+            return hmR3InitFinalizeR3(pVM);
+        case VMINITCOMPLETED_RING0:
+            return hmR3InitFinalizeR0(pVM);
+        default:
+            return VINF_SUCCESS;
+    }
+}
+
+
+/**
+ * Turns off normal raw mode features.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+static void hmR3DisableRawMode(PVM pVM)
+{
+/** @todo r=bird: HM shouldn't be doing this crap. */
+    /* Reinit the paging mode to force the new shadow mode. */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        PGMHCChangeMode(pVM, pVCpu, PGMMODE_REAL);
+    }
+}
+
+
+/**
+ * Initialize VT-x or AMD-V.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+static int hmR3InitFinalizeR0(PVM pVM)
+{
+    int rc;
+
+    if (!HMIsEnabled(pVM))
+        return VINF_SUCCESS;
+
+    /*
+     * Hack to allow users to work around broken BIOSes that incorrectly set
+     * EFER.SVME, which makes us believe somebody else is already using AMD-V.
+     */
+    if (   !pVM->hm.s.vmx.fSupported
+        && !pVM->hm.s.svm.fSupported
+        &&  pVM->hm.s.rcInit == VERR_SVM_IN_USE /* implies functional AMD-V */
+        &&  RTEnvExist("VBOX_HWVIRTEX_IGNORE_SVM_IN_USE"))
+    {
+        LogRel(("HM: VBOX_HWVIRTEX_IGNORE_SVM_IN_USE active!\n"));
+        pVM->hm.s.svm.fSupported        = true;
+        pVM->hm.s.svm.fIgnoreInUseError = true;
+        pVM->hm.s.rcInit = VINF_SUCCESS;
+    }
+
+    /*
+     * Report ring-0 init errors.
+     */
+    if (   !pVM->hm.s.vmx.fSupported
+        && !pVM->hm.s.svm.fSupported)
+    {
+        LogRel(("HM: Failed to initialize VT-x / AMD-V: %Rrc\n", pVM->hm.s.rcInit));
+        LogRel(("HM: VMX MSR_IA32_FEATURE_CONTROL=%RX64\n", pVM->hm.s.vmx.Msrs.u64FeatCtrl));
+        switch (pVM->hm.s.rcInit)
+        {
+            case VERR_VMX_IN_VMX_ROOT_MODE:
+                return VM_SET_ERROR(pVM, VERR_VMX_IN_VMX_ROOT_MODE, "VT-x is being used by another hypervisor");
+            case VERR_VMX_NO_VMX:
+                return VM_SET_ERROR(pVM, VERR_VMX_NO_VMX, "VT-x is not available");
+            case VERR_VMX_MSR_VMX_DISABLED:
+                return VM_SET_ERROR(pVM, VERR_VMX_MSR_VMX_DISABLED, "VT-x is disabled in the BIOS");
+            case VERR_VMX_MSR_ALL_VMX_DISABLED:
+                return VM_SET_ERROR(pVM, VERR_VMX_MSR_ALL_VMX_DISABLED, "VT-x is disabled in the BIOS for all CPU modes");
+            case VERR_VMX_MSR_LOCKING_FAILED:
+                return VM_SET_ERROR(pVM, VERR_VMX_MSR_LOCKING_FAILED, "Failed to lock VT-x features while trying to enable VT-x");
+            case VERR_VMX_MSR_VMX_ENABLE_FAILED:
+                return VM_SET_ERROR(pVM, VERR_VMX_MSR_VMX_ENABLE_FAILED, "Failed to enable VT-x features");
+            case VERR_VMX_MSR_SMX_VMX_ENABLE_FAILED:
+                return VM_SET_ERROR(pVM, VERR_VMX_MSR_SMX_VMX_ENABLE_FAILED, "Failed to enable VT-x features in SMX mode");
+
+            case VERR_SVM_IN_USE:
+                return VM_SET_ERROR(pVM, VERR_SVM_IN_USE, "AMD-V is being used by another hypervisor");
+            case VERR_SVM_NO_SVM:
+                return VM_SET_ERROR(pVM, VERR_SVM_NO_SVM, "AMD-V is not available");
+            case VERR_SVM_DISABLED:
+                return VM_SET_ERROR(pVM, VERR_SVM_DISABLED, "AMD-V is disabled in the BIOS");
+        }
+        return VMSetError(pVM, pVM->hm.s.rcInit, RT_SRC_POS, "HM ring-0 init failed: %Rrc", pVM->hm.s.rcInit);
+    }
+
+    /*
+     * Enable VT-x or AMD-V on all host CPUs.
+     */
+    rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), 0 /*idCpu*/, VMMR0_DO_HM_ENABLE, 0, NULL);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("HM: Failed to enable, error %Rrc\n", rc));
+        HMR3CheckError(pVM, rc);
+        return rc;
+    }
+
+    /*
+     * No TPR patching is required when the IO-APIC is not enabled for this VM.
+     * (Main should have taken care of this already)
+     */
+    if (!PDMHasIoApic(pVM))
+    {
+        Assert(!pVM->hm.s.fTprPatchingAllowed); /* paranoia */
+        pVM->hm.s.fTprPatchingAllowed = false;
+    }
+
+    /*
+     * Check if L1D flush is needed/possible.
+     */
+    if (   !pVM->cpum.ro.HostFeatures.fFlushCmd
+        || pVM->cpum.ro.HostFeatures.enmMicroarch <  kCpumMicroarch_Intel_Core7_Nehalem
+        || pVM->cpum.ro.HostFeatures.enmMicroarch >= kCpumMicroarch_Intel_Core7_End
+        || pVM->cpum.ro.HostFeatures.fArchVmmNeedNotFlushL1d
+        || pVM->cpum.ro.HostFeatures.fArchRdclNo)
+        pVM->hm.s.fL1dFlushOnSched = pVM->hm.s.fL1dFlushOnVmEntry = false;
+
+    /*
+     * Check if MDS flush is needed/possible.
+     * On atoms and knight family CPUs, we will only allow clearing on scheduling.
+     */
+    if (   !pVM->cpum.ro.HostFeatures.fMdsClear
+        || pVM->cpum.ro.HostFeatures.fArchMdsNo)
+        pVM->hm.s.fMdsClearOnSched = pVM->hm.s.fMdsClearOnVmEntry = false;
+    else if (   (   pVM->cpum.ro.HostFeatures.enmMicroarch >=  kCpumMicroarch_Intel_Atom_Airmount
+                 && pVM->cpum.ro.HostFeatures.enmMicroarch <   kCpumMicroarch_Intel_Atom_End)
+             || (   pVM->cpum.ro.HostFeatures.enmMicroarch >=  kCpumMicroarch_Intel_Phi_KnightsLanding
+                 && pVM->cpum.ro.HostFeatures.enmMicroarch <   kCpumMicroarch_Intel_Phi_End))
+    {
+        if (!pVM->hm.s.fMdsClearOnSched)
+             pVM->hm.s.fMdsClearOnSched = pVM->hm.s.fMdsClearOnVmEntry;
+        pVM->hm.s.fMdsClearOnVmEntry = false;
+    }
+    else if (   pVM->cpum.ro.HostFeatures.enmMicroarch <  kCpumMicroarch_Intel_Core7_Nehalem
+             || pVM->cpum.ro.HostFeatures.enmMicroarch >= kCpumMicroarch_Intel_Core7_End)
+        pVM->hm.s.fMdsClearOnSched = pVM->hm.s.fMdsClearOnVmEntry = false;
+
+    /*
+     * Sync options.
+     */
+    /** @todo Move this out of of CPUMCTX and into some ring-0 only HM structure.
+     *        That will require a little bit of work, of course. */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU   pVCpu   = pVM->apCpusR3[idCpu];
+        PCPUMCTX pCpuCtx = &pVCpu->cpum.GstCtx;
+        pCpuCtx->fWorldSwitcher &= ~(CPUMCTX_WSF_IBPB_EXIT | CPUMCTX_WSF_IBPB_ENTRY);
+        if (pVM->cpum.ro.HostFeatures.fIbpb)
+        {
+            if (pVM->hm.s.fIbpbOnVmExit)
+                pCpuCtx->fWorldSwitcher |= CPUMCTX_WSF_IBPB_EXIT;
+            if (pVM->hm.s.fIbpbOnVmEntry)
+                pCpuCtx->fWorldSwitcher |= CPUMCTX_WSF_IBPB_ENTRY;
+        }
+        if (pVM->cpum.ro.HostFeatures.fFlushCmd && pVM->hm.s.fL1dFlushOnVmEntry)
+            pCpuCtx->fWorldSwitcher |= CPUMCTX_WSF_L1D_ENTRY;
+        if (pVM->cpum.ro.HostFeatures.fMdsClear && pVM->hm.s.fMdsClearOnVmEntry)
+            pCpuCtx->fWorldSwitcher |= CPUMCTX_WSF_MDS_ENTRY;
+        if (idCpu == 0)
+            LogRel(("HM: fWorldSwitcher=%#x (fIbpbOnVmExit=%RTbool fIbpbOnVmEntry=%RTbool fL1dFlushOnVmEntry=%RTbool); fL1dFlushOnSched=%RTbool fMdsClearOnVmEntry=%RTbool\n",
+                    pCpuCtx->fWorldSwitcher, pVM->hm.s.fIbpbOnVmExit, pVM->hm.s.fIbpbOnVmEntry, pVM->hm.s.fL1dFlushOnVmEntry,
+                    pVM->hm.s.fL1dFlushOnSched, pVM->hm.s.fMdsClearOnVmEntry));
+    }
+
+    /*
+     * Do the vendor specific initialization
+     *
+     * Note! We disable release log buffering here since we're doing relatively
+     *       lot of logging and doesn't want to hit the disk with each LogRel
+     *       statement.
+     */
+    AssertLogRelReturn(!pVM->hm.s.fInitialized, VERR_HM_IPE_5);
+    bool fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
+    if (pVM->hm.s.vmx.fSupported)
+        rc = hmR3InitFinalizeR0Intel(pVM);
+    else
+        rc = hmR3InitFinalizeR0Amd(pVM);
+    LogRel((pVM->hm.s.fGlobalInit ? "HM: VT-x/AMD-V init method: Global\n"
+                                  : "HM: VT-x/AMD-V init method: Local\n"));
+    RTLogRelSetBuffering(fOldBuffered);
+    pVM->hm.s.fInitialized = true;
+
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNPDMVMMDEVHEAPNOTIFY}
+ */
+static DECLCALLBACK(void) hmR3VmmDevHeapNotify(PVM pVM, void *pvAllocation, RTGCPHYS GCPhysAllocation)
+{
+    NOREF(pVM);
+    NOREF(pvAllocation);
+    NOREF(GCPhysAllocation);
+}
+
+
+/**
+ * Returns a description of the VMCS (and associated regions') memory type given the
+ * IA32_VMX_BASIC MSR.
+ *
+ * @returns The descriptive memory type.
+ * @param   uMsrVmxBasic        IA32_VMX_BASIC MSR value.
+ */
+static const char *hmR3VmxGetMemTypeDesc(uint64_t uMsrVmxBasic)
+{
+    uint8_t const uMemType = RT_BF_GET(uMsrVmxBasic, VMX_BF_BASIC_VMCS_MEM_TYPE);
+    switch (uMemType)
+    {
+        case VMX_BASIC_MEM_TYPE_WB: return "Write Back (WB)";
+        case VMX_BASIC_MEM_TYPE_UC: return "Uncacheable (UC)";
+    }
+    return "Unknown";
+}
+
+
+/**
+ * Returns a single-line description of all the activity-states supported by the CPU
+ * given the IA32_VMX_MISC MSR.
+ *
+ * @returns All supported activity states.
+ * @param   uMsrMisc            IA32_VMX_MISC MSR value.
+ */
+static const char *hmR3VmxGetActivityStateAllDesc(uint64_t uMsrMisc)
+{
+    static const char * const s_apszActStates[] =
+    {
+        "",
+        " ( HLT )",
+        " ( SHUTDOWN )",
+        " ( HLT SHUTDOWN )",
+        " ( SIPI_WAIT )",
+        " ( HLT SIPI_WAIT )",
+        " ( SHUTDOWN SIPI_WAIT )",
+        " ( HLT SHUTDOWN SIPI_WAIT )"
+    };
+    uint8_t const idxActStates = RT_BF_GET(uMsrMisc, VMX_BF_MISC_ACTIVITY_STATES);
+    Assert(idxActStates < RT_ELEMENTS(s_apszActStates));
+    return s_apszActStates[idxActStates];
+}
+
+
+/**
+ * Reports MSR_IA32_FEATURE_CONTROL MSR to the log.
+ *
+ * @param   fFeatMsr    The feature control MSR value.
+ */
+static void hmR3VmxReportFeatCtlMsr(uint64_t fFeatMsr)
+{
+    uint64_t const val = fFeatMsr;
+    LogRel(("HM: MSR_IA32_FEATURE_CONTROL          = %#RX64\n", val));
+    HMVMX_REPORT_MSR_CAP(val, "LOCK",             MSR_IA32_FEATURE_CONTROL_LOCK);
+    HMVMX_REPORT_MSR_CAP(val, "SMX_VMXON",        MSR_IA32_FEATURE_CONTROL_SMX_VMXON);
+    HMVMX_REPORT_MSR_CAP(val, "VMXON",            MSR_IA32_FEATURE_CONTROL_VMXON);
+    HMVMX_REPORT_MSR_CAP(val, "SENTER_LOCAL_FN0", MSR_IA32_FEATURE_CONTROL_SENTER_LOCAL_FN_0);
+    HMVMX_REPORT_MSR_CAP(val, "SENTER_LOCAL_FN1", MSR_IA32_FEATURE_CONTROL_SENTER_LOCAL_FN_1);
+    HMVMX_REPORT_MSR_CAP(val, "SENTER_LOCAL_FN2", MSR_IA32_FEATURE_CONTROL_SENTER_LOCAL_FN_2);
+    HMVMX_REPORT_MSR_CAP(val, "SENTER_LOCAL_FN3", MSR_IA32_FEATURE_CONTROL_SENTER_LOCAL_FN_3);
+    HMVMX_REPORT_MSR_CAP(val, "SENTER_LOCAL_FN4", MSR_IA32_FEATURE_CONTROL_SENTER_LOCAL_FN_4);
+    HMVMX_REPORT_MSR_CAP(val, "SENTER_LOCAL_FN5", MSR_IA32_FEATURE_CONTROL_SENTER_LOCAL_FN_5);
+    HMVMX_REPORT_MSR_CAP(val, "SENTER_LOCAL_FN6", MSR_IA32_FEATURE_CONTROL_SENTER_LOCAL_FN_6);
+    HMVMX_REPORT_MSR_CAP(val, "SENTER_GLOBAL_EN", MSR_IA32_FEATURE_CONTROL_SENTER_GLOBAL_EN);
+    HMVMX_REPORT_MSR_CAP(val, "SGX_LAUNCH_EN",    MSR_IA32_FEATURE_CONTROL_SGX_LAUNCH_EN);
+    HMVMX_REPORT_MSR_CAP(val, "SGX_GLOBAL_EN",    MSR_IA32_FEATURE_CONTROL_SGX_GLOBAL_EN);
+    HMVMX_REPORT_MSR_CAP(val, "LMCE",             MSR_IA32_FEATURE_CONTROL_LMCE);
+    if (!(val & MSR_IA32_FEATURE_CONTROL_LOCK))
+        LogRel(("HM:   MSR_IA32_FEATURE_CONTROL lock bit not set, possibly bad hardware!\n"));
+}
+
+
+/**
+ * Reports MSR_IA32_VMX_BASIC MSR to the log.
+ *
+ * @param   uBasicMsr    The VMX basic MSR value.
+ */
+static void hmR3VmxReportBasicMsr(uint64_t uBasicMsr)
+{
+    LogRel(("HM: MSR_IA32_VMX_BASIC                = %#RX64\n",     uBasicMsr));
+    LogRel(("HM:   VMCS id                           = %#x\n",      RT_BF_GET(uBasicMsr, VMX_BF_BASIC_VMCS_ID)));
+    LogRel(("HM:   VMCS size                         = %u bytes\n", RT_BF_GET(uBasicMsr, VMX_BF_BASIC_VMCS_SIZE)));
+    LogRel(("HM:   VMCS physical address limit       = %s\n",       RT_BF_GET(uBasicMsr, VMX_BF_BASIC_PHYSADDR_WIDTH) ?
+                                                                    "< 4 GB" : "None"));
+    LogRel(("HM:   VMCS memory type                  = %s\n",       hmR3VmxGetMemTypeDesc(uBasicMsr)));
+    LogRel(("HM:   Dual-monitor treatment support    = %RTbool\n",  RT_BF_GET(uBasicMsr, VMX_BF_BASIC_DUAL_MON)));
+    LogRel(("HM:   OUTS & INS instruction-info       = %RTbool\n",  RT_BF_GET(uBasicMsr, VMX_BF_BASIC_VMCS_INS_OUTS)));
+    LogRel(("HM:   Supports true-capability MSRs     = %RTbool\n",  RT_BF_GET(uBasicMsr, VMX_BF_BASIC_TRUE_CTLS)));
+    LogRel(("HM:   VM-entry Xcpt error-code optional = %RTbool\n",  RT_BF_GET(uBasicMsr, VMX_BF_BASIC_XCPT_ERRCODE)));
+}
+
+
+/**
+ * Reports MSR_IA32_PINBASED_CTLS to the log.
+ *
+ * @param   pVmxMsr    Pointer to the VMX MSR.
+ */
+static void hmR3VmxReportPinBasedCtlsMsr(PCVMXCTLSMSR pVmxMsr)
+{
+    uint64_t const fAllowed1 = pVmxMsr->n.allowed1;
+    uint64_t const fAllowed0 = pVmxMsr->n.allowed0;
+    LogRel(("HM: MSR_IA32_VMX_PINBASED_CTLS        = %#RX64\n", pVmxMsr->u));
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "EXT_INT_EXIT",  VMX_PIN_CTLS_EXT_INT_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "NMI_EXIT",      VMX_PIN_CTLS_NMI_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "VIRTUAL_NMI",   VMX_PIN_CTLS_VIRT_NMI);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "PREEMPT_TIMER", VMX_PIN_CTLS_PREEMPT_TIMER);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "POSTED_INT",    VMX_PIN_CTLS_POSTED_INT);
+}
+
+
+/**
+ * Reports MSR_IA32_VMX_PROCBASED_CTLS MSR to the log.
+ *
+ * @param   pVmxMsr    Pointer to the VMX MSR.
+ */
+static void hmR3VmxReportProcBasedCtlsMsr(PCVMXCTLSMSR pVmxMsr)
+{
+    uint64_t const fAllowed1 = pVmxMsr->n.allowed1;
+    uint64_t const fAllowed0 = pVmxMsr->n.allowed0;
+    LogRel(("HM: MSR_IA32_VMX_PROCBASED_CTLS       = %#RX64\n", pVmxMsr->u));
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "INT_WINDOW_EXIT",    VMX_PROC_CTLS_INT_WINDOW_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "USE_TSC_OFFSETTING", VMX_PROC_CTLS_USE_TSC_OFFSETTING);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "HLT_EXIT",           VMX_PROC_CTLS_HLT_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "INVLPG_EXIT",        VMX_PROC_CTLS_INVLPG_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "MWAIT_EXIT",         VMX_PROC_CTLS_MWAIT_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "RDPMC_EXIT",         VMX_PROC_CTLS_RDPMC_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "RDTSC_EXIT",         VMX_PROC_CTLS_RDTSC_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "CR3_LOAD_EXIT",      VMX_PROC_CTLS_CR3_LOAD_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "CR3_STORE_EXIT",     VMX_PROC_CTLS_CR3_STORE_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "CR8_LOAD_EXIT",      VMX_PROC_CTLS_CR8_LOAD_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "CR8_STORE_EXIT",     VMX_PROC_CTLS_CR8_STORE_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "USE_TPR_SHADOW",     VMX_PROC_CTLS_USE_TPR_SHADOW);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "NMI_WINDOW_EXIT",    VMX_PROC_CTLS_NMI_WINDOW_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "MOV_DR_EXIT",        VMX_PROC_CTLS_MOV_DR_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "UNCOND_IO_EXIT",     VMX_PROC_CTLS_UNCOND_IO_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "USE_IO_BITMAPS",     VMX_PROC_CTLS_USE_IO_BITMAPS);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "MONITOR_TRAP_FLAG",  VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "USE_MSR_BITMAPS",    VMX_PROC_CTLS_USE_MSR_BITMAPS);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "MONITOR_EXIT",       VMX_PROC_CTLS_MONITOR_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "PAUSE_EXIT",         VMX_PROC_CTLS_PAUSE_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "USE_SECONDARY_CTLS", VMX_PROC_CTLS_USE_SECONDARY_CTLS);
+}
+
+
+/**
+ * Reports MSR_IA32_VMX_PROCBASED_CTLS2 MSR to the log.
+ *
+ * @param   pVmxMsr    Pointer to the VMX MSR.
+ */
+static void hmR3VmxReportProcBasedCtls2Msr(PCVMXCTLSMSR pVmxMsr)
+{
+    uint64_t const fAllowed1 = pVmxMsr->n.allowed1;
+    uint64_t const fAllowed0 = pVmxMsr->n.allowed0;
+    LogRel(("HM: MSR_IA32_VMX_PROCBASED_CTLS2      = %#RX64\n", pVmxMsr->u));
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "VIRT_APIC_ACCESS",    VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "EPT",                 VMX_PROC_CTLS2_EPT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "DESC_TABLE_EXIT",     VMX_PROC_CTLS2_DESC_TABLE_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "RDTSCP",              VMX_PROC_CTLS2_RDTSCP);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "VIRT_X2APIC_MODE",    VMX_PROC_CTLS2_VIRT_X2APIC_MODE);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "VPID",                VMX_PROC_CTLS2_VPID);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "WBINVD_EXIT",         VMX_PROC_CTLS2_WBINVD_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "UNRESTRICTED_GUEST",  VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "APIC_REG_VIRT",       VMX_PROC_CTLS2_APIC_REG_VIRT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "VIRT_INT_DELIVERY",   VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "PAUSE_LOOP_EXIT",     VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "RDRAND_EXIT",         VMX_PROC_CTLS2_RDRAND_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "INVPCID",             VMX_PROC_CTLS2_INVPCID);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "VMFUNC",              VMX_PROC_CTLS2_VMFUNC);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "VMCS_SHADOWING",      VMX_PROC_CTLS2_VMCS_SHADOWING);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "ENCLS_EXIT",          VMX_PROC_CTLS2_ENCLS_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "RDSEED_EXIT",         VMX_PROC_CTLS2_RDSEED_EXIT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "PML",                 VMX_PROC_CTLS2_PML);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "EPT_VE",              VMX_PROC_CTLS2_EPT_VE);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "CONCEAL_VMX_FROM_PT", VMX_PROC_CTLS2_CONCEAL_VMX_FROM_PT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "XSAVES_XRSTORS",      VMX_PROC_CTLS2_XSAVES_XRSTORS);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "MODE_BASED_EPT_PERM", VMX_PROC_CTLS2_MODE_BASED_EPT_PERM);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "SPPTP_EPT",           VMX_PROC_CTLS2_SPPTP_EPT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "PT_EPT",              VMX_PROC_CTLS2_PT_EPT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "TSC_SCALING",         VMX_PROC_CTLS2_TSC_SCALING);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "USER_WAIT_PAUSE",     VMX_PROC_CTLS2_USER_WAIT_PAUSE);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "ENCLV_EXIT",          VMX_PROC_CTLS2_ENCLV_EXIT);
+}
+
+
+/**
+ * Reports MSR_IA32_VMX_ENTRY_CTLS to the log.
+ *
+ * @param   pVmxMsr    Pointer to the VMX MSR.
+ */
+static void hmR3VmxReportEntryCtlsMsr(PCVMXCTLSMSR pVmxMsr)
+{
+    uint64_t const fAllowed1 = pVmxMsr->n.allowed1;
+    uint64_t const fAllowed0 = pVmxMsr->n.allowed0;
+    LogRel(("HM: MSR_IA32_VMX_ENTRY_CTLS           = %#RX64\n", pVmxMsr->u));
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "LOAD_DEBUG",          VMX_ENTRY_CTLS_LOAD_DEBUG);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "IA32E_MODE_GUEST",    VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "ENTRY_TO_SMM",        VMX_ENTRY_CTLS_ENTRY_TO_SMM);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "DEACTIVATE_DUAL_MON", VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "LOAD_PERF_MSR",       VMX_ENTRY_CTLS_LOAD_PERF_MSR);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "LOAD_PAT_MSR",        VMX_ENTRY_CTLS_LOAD_PAT_MSR);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "LOAD_EFER_MSR",       VMX_ENTRY_CTLS_LOAD_EFER_MSR);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "LOAD_BNDCFGS_MSR",    VMX_ENTRY_CTLS_LOAD_BNDCFGS_MSR);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "CONCEAL_VMX_FROM_PT", VMX_ENTRY_CTLS_CONCEAL_VMX_FROM_PT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "LOAD_RTIT_CTL_MSR",   VMX_ENTRY_CTLS_LOAD_RTIT_CTL_MSR);
+}
+
+
+/**
+ * Reports MSR_IA32_VMX_EXIT_CTLS to the log.
+ *
+ * @param   pVmxMsr    Pointer to the VMX MSR.
+ */
+static void hmR3VmxReportExitCtlsMsr(PCVMXCTLSMSR pVmxMsr)
+{
+    uint64_t const fAllowed1 = pVmxMsr->n.allowed1;
+    uint64_t const fAllowed0 = pVmxMsr->n.allowed0;
+    LogRel(("HM: MSR_IA32_VMX_EXIT_CTLS            = %#RX64\n", pVmxMsr->u));
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "SAVE_DEBUG",           VMX_EXIT_CTLS_SAVE_DEBUG);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "HOST_ADDR_SPACE_SIZE", VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "LOAD_PERF_MSR",        VMX_EXIT_CTLS_LOAD_PERF_MSR);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "ACK_EXT_INT",          VMX_EXIT_CTLS_ACK_EXT_INT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "SAVE_PAT_MSR",         VMX_EXIT_CTLS_SAVE_PAT_MSR);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "LOAD_PAT_MSR",         VMX_EXIT_CTLS_LOAD_PAT_MSR);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "SAVE_EFER_MSR",        VMX_EXIT_CTLS_SAVE_EFER_MSR);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "LOAD_EFER_MSR",        VMX_EXIT_CTLS_LOAD_EFER_MSR);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "SAVE_PREEMPT_TIMER",   VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "CLEAR_BNDCFGS_MSR",    VMX_EXIT_CTLS_CLEAR_BNDCFGS_MSR);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "CONCEAL_VMX_FROM_PT",  VMX_EXIT_CTLS_CONCEAL_VMX_FROM_PT);
+    HMVMX_REPORT_FEAT(fAllowed1, fAllowed0, "CLEAR_RTIT_CTL_MSR",   VMX_EXIT_CTLS_CLEAR_RTIT_CTL_MSR);
+}
+
+
+/**
+ * Reports MSR_IA32_VMX_EPT_VPID_CAP MSR to the log.
+ *
+ * @param   fCaps    The VMX EPT/VPID capability MSR value.
+ */
+static void hmR3VmxReportEptVpidCapsMsr(uint64_t fCaps)
+{
+    LogRel(("HM: MSR_IA32_VMX_EPT_VPID_CAP         = %#RX64\n", fCaps));
+    HMVMX_REPORT_MSR_CAP(fCaps, "RWX_X_ONLY",                            MSR_IA32_VMX_EPT_VPID_CAP_RWX_X_ONLY);
+    HMVMX_REPORT_MSR_CAP(fCaps, "PAGE_WALK_LENGTH_4",                    MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4);
+    HMVMX_REPORT_MSR_CAP(fCaps, "PAGE_WALK_LENGTH_5",                    MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_5);
+    HMVMX_REPORT_MSR_CAP(fCaps, "EMT_UC",                                MSR_IA32_VMX_EPT_VPID_CAP_EMT_UC);
+    HMVMX_REPORT_MSR_CAP(fCaps, "EMT_WB",                                MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB);
+    HMVMX_REPORT_MSR_CAP(fCaps, "PDE_2M",                                MSR_IA32_VMX_EPT_VPID_CAP_PDE_2M);
+    HMVMX_REPORT_MSR_CAP(fCaps, "PDPTE_1G",                              MSR_IA32_VMX_EPT_VPID_CAP_PDPTE_1G);
+    HMVMX_REPORT_MSR_CAP(fCaps, "INVEPT",                                MSR_IA32_VMX_EPT_VPID_CAP_INVEPT);
+    HMVMX_REPORT_MSR_CAP(fCaps, "EPT_ACCESS_DIRTY",                      MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY);
+    HMVMX_REPORT_MSR_CAP(fCaps, "ADVEXITINFO_EPT",                       MSR_IA32_VMX_EPT_VPID_CAP_ADVEXITINFO_EPT);
+    HMVMX_REPORT_MSR_CAP(fCaps, "SSS",                                   MSR_IA32_VMX_EPT_VPID_CAP_SSS);
+    HMVMX_REPORT_MSR_CAP(fCaps, "INVEPT_SINGLE_CONTEXT",                 MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT);
+    HMVMX_REPORT_MSR_CAP(fCaps, "INVEPT_ALL_CONTEXTS",                   MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
+    HMVMX_REPORT_MSR_CAP(fCaps, "INVVPID",                               MSR_IA32_VMX_EPT_VPID_CAP_INVVPID);
+    HMVMX_REPORT_MSR_CAP(fCaps, "INVVPID_INDIV_ADDR",                    MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
+    HMVMX_REPORT_MSR_CAP(fCaps, "INVVPID_SINGLE_CONTEXT",                MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT);
+    HMVMX_REPORT_MSR_CAP(fCaps, "INVVPID_ALL_CONTEXTS",                  MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS);
+    HMVMX_REPORT_MSR_CAP(fCaps, "INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS", MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS);
+}
+
+
+/**
+ * Reports MSR_IA32_VMX_MISC MSR to the log.
+ *
+ * @param   pVM      Pointer to the VM.
+ * @param   fMisc    The VMX misc. MSR value.
+ */
+static void hmR3VmxReportMiscMsr(PVM pVM, uint64_t fMisc)
+{
+    LogRel(("HM: MSR_IA32_VMX_MISC                 = %#RX64\n", fMisc));
+    uint8_t const cPreemptTimerShift = RT_BF_GET(fMisc, VMX_BF_MISC_PREEMPT_TIMER_TSC);
+    if (cPreemptTimerShift == pVM->hm.s.vmx.cPreemptTimerShift)
+        LogRel(("HM:   PREEMPT_TIMER_TSC                 = %#x\n",    cPreemptTimerShift));
+    else
+    {
+        LogRel(("HM:   PREEMPT_TIMER_TSC                 = %#x - erratum detected, using %#x instead\n", cPreemptTimerShift,
+                pVM->hm.s.vmx.cPreemptTimerShift));
+    }
+    LogRel(("HM:   EXIT_SAVE_EFER_LMA                = %RTbool\n",    RT_BF_GET(fMisc, VMX_BF_MISC_EXIT_SAVE_EFER_LMA)));
+    LogRel(("HM:   ACTIVITY_STATES                   = %#x%s\n",      RT_BF_GET(fMisc, VMX_BF_MISC_ACTIVITY_STATES),
+                                                                      hmR3VmxGetActivityStateAllDesc(fMisc)));
+    LogRel(("HM:   INTEL_PT                          = %RTbool\n",    RT_BF_GET(fMisc, VMX_BF_MISC_INTEL_PT)));
+    LogRel(("HM:   SMM_READ_SMBASE_MSR               = %RTbool\n",    RT_BF_GET(fMisc, VMX_BF_MISC_SMM_READ_SMBASE_MSR)));
+    LogRel(("HM:   CR3_TARGET                        = %#x\n",        RT_BF_GET(fMisc, VMX_BF_MISC_CR3_TARGET)));
+    LogRel(("HM:   MAX_MSR                           = %#x ( %u )\n", RT_BF_GET(fMisc, VMX_BF_MISC_MAX_MSRS),
+                                                                      VMX_MISC_MAX_MSRS(fMisc)));
+    LogRel(("HM:   VMXOFF_BLOCK_SMI                  = %RTbool\n",    RT_BF_GET(fMisc, VMX_BF_MISC_VMXOFF_BLOCK_SMI)));
+    LogRel(("HM:   VMWRITE_ALL                       = %RTbool\n",    RT_BF_GET(fMisc, VMX_BF_MISC_VMWRITE_ALL)));
+    LogRel(("HM:   ENTRY_INJECT_SOFT_INT             = %#x\n",        RT_BF_GET(fMisc, VMX_BF_MISC_ENTRY_INJECT_SOFT_INT)));
+    LogRel(("HM:   MSEG_ID                           = %#x\n",        RT_BF_GET(fMisc, VMX_BF_MISC_MSEG_ID)));
+}
+
+
+/**
+ * Reports MSR_IA32_VMX_VMCS_ENUM MSR to the log.
+ *
+ * @param   uVmcsEnum    The VMX VMCS enum MSR value.
+ */
+static void hmR3VmxReportVmcsEnumMsr(uint64_t uVmcsEnum)
+{
+    LogRel(("HM: MSR_IA32_VMX_VMCS_ENUM            = %#RX64\n", uVmcsEnum));
+    LogRel(("HM:   HIGHEST_IDX                       = %#x\n", RT_BF_GET(uVmcsEnum, VMX_BF_VMCS_ENUM_HIGHEST_IDX)));
+}
+
+
+/**
+ * Reports MSR_IA32_VMX_VMFUNC MSR to the log.
+ *
+ * @param   uVmFunc    The VMX VMFUNC MSR value.
+ */
+static void hmR3VmxReportVmFuncMsr(uint64_t uVmFunc)
+{
+    LogRel(("HM: MSR_IA32_VMX_VMFUNC               = %#RX64\n", uVmFunc));
+    HMVMX_REPORT_ALLOWED_FEAT(uVmFunc, "EPTP_SWITCHING", RT_BF_GET(uVmFunc, VMX_BF_VMFUNC_EPTP_SWITCHING));
+}
+
+
+/**
+ * Reports VMX CR0, CR4 fixed MSRs.
+ *
+ * @param   pMsrs    Pointer to the VMX MSRs.
+ */
+static void hmR3VmxReportCrFixedMsrs(PVMXMSRS pMsrs)
+{
+    LogRel(("HM: MSR_IA32_VMX_CR0_FIXED0           = %#RX64\n", pMsrs->u64Cr0Fixed0));
+    LogRel(("HM: MSR_IA32_VMX_CR0_FIXED1           = %#RX64\n", pMsrs->u64Cr0Fixed1));
+    LogRel(("HM: MSR_IA32_VMX_CR4_FIXED0           = %#RX64\n", pMsrs->u64Cr4Fixed0));
+    LogRel(("HM: MSR_IA32_VMX_CR4_FIXED1           = %#RX64\n", pMsrs->u64Cr4Fixed1));
+}
+
+
+/**
+ * Finish VT-x initialization (after ring-0 init).
+ *
+ * @returns VBox status code.
+ * @param   pVM                The cross context VM structure.
+ */
+static int hmR3InitFinalizeR0Intel(PVM pVM)
+{
+    int rc;
+
+    LogFunc(("pVM->hm.s.vmx.fSupported = %d\n", pVM->hm.s.vmx.fSupported));
+    AssertLogRelReturn(pVM->hm.s.vmx.Msrs.u64FeatCtrl != 0, VERR_HM_IPE_4);
+
+    LogRel(("HM: Using VT-x implementation 3.0\n"));
+    LogRel(("HM: Max resume loops                  = %u\n",     pVM->hm.s.cMaxResumeLoops));
+    LogRel(("HM: Host CR4                          = %#RX64\n", pVM->hm.s.vmx.u64HostCr4));
+    LogRel(("HM: Host EFER                         = %#RX64\n", pVM->hm.s.vmx.u64HostMsrEfer));
+    LogRel(("HM: MSR_IA32_SMM_MONITOR_CTL          = %#RX64\n", pVM->hm.s.vmx.u64HostSmmMonitorCtl));
+
+    hmR3VmxReportFeatCtlMsr(pVM->hm.s.vmx.Msrs.u64FeatCtrl);
+    hmR3VmxReportBasicMsr(pVM->hm.s.vmx.Msrs.u64Basic);
+
+    hmR3VmxReportPinBasedCtlsMsr(&pVM->hm.s.vmx.Msrs.PinCtls);
+    hmR3VmxReportProcBasedCtlsMsr(&pVM->hm.s.vmx.Msrs.ProcCtls);
+    if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
+        hmR3VmxReportProcBasedCtls2Msr(&pVM->hm.s.vmx.Msrs.ProcCtls2);
+
+    hmR3VmxReportEntryCtlsMsr(&pVM->hm.s.vmx.Msrs.EntryCtls);
+    hmR3VmxReportExitCtlsMsr(&pVM->hm.s.vmx.Msrs.ExitCtls);
+
+    if (RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_TRUE_CTLS))
+    {
+        /* We don't extensively dump the true capability MSRs as we don't use them, see @bugref{9180#c5}. */
+        LogRel(("HM: MSR_IA32_VMX_TRUE_PINBASED_CTLS   = %#RX64\n", pVM->hm.s.vmx.Msrs.TruePinCtls));
+        LogRel(("HM: MSR_IA32_VMX_TRUE_PROCBASED_CTLS  = %#RX64\n", pVM->hm.s.vmx.Msrs.TrueProcCtls));
+        LogRel(("HM: MSR_IA32_VMX_TRUE_ENTRY_CTLS      = %#RX64\n", pVM->hm.s.vmx.Msrs.TrueEntryCtls));
+        LogRel(("HM: MSR_IA32_VMX_TRUE_EXIT_CTLS       = %#RX64\n", pVM->hm.s.vmx.Msrs.TrueExitCtls));
+    }
+
+    hmR3VmxReportMiscMsr(pVM, pVM->hm.s.vmx.Msrs.u64Misc);
+    hmR3VmxReportVmcsEnumMsr(pVM->hm.s.vmx.Msrs.u64VmcsEnum);
+    if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps)
+        hmR3VmxReportEptVpidCapsMsr(pVM->hm.s.vmx.Msrs.u64EptVpidCaps);
+    if (pVM->hm.s.vmx.Msrs.u64VmFunc)
+        hmR3VmxReportVmFuncMsr(pVM->hm.s.vmx.Msrs.u64VmFunc);
+    hmR3VmxReportCrFixedMsrs(&pVM->hm.s.vmx.Msrs);
+
+    LogRel(("HM: APIC-access page physaddr         = %#RHp\n",  pVM->hm.s.vmx.HCPhysApicAccess));
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PCVMXVMCSINFO pVmcsInfo = &pVM->apCpusR3[idCpu]->hm.s.vmx.VmcsInfo;
+        LogRel(("HM: VCPU%3d: MSR bitmap physaddr      = %#RHp\n", idCpu, pVmcsInfo->HCPhysMsrBitmap));
+        LogRel(("HM: VCPU%3d: VMCS physaddr            = %#RHp\n", idCpu, pVmcsInfo->HCPhysVmcs));
+    }
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (pVM->cpum.ro.GuestFeatures.fVmx)
+    {
+        LogRel(("HM: Nested-guest:\n"));
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PCVMXVMCSINFO pVmcsInfoNstGst = &pVM->apCpusR3[idCpu]->hm.s.vmx.VmcsInfoNstGst;
+            LogRel(("HM: VCPU%3d: MSR bitmap physaddr      = %#RHp\n", idCpu, pVmcsInfoNstGst->HCPhysMsrBitmap));
+            LogRel(("HM: VCPU%3d: VMCS physaddr            = %#RHp\n", idCpu, pVmcsInfoNstGst->HCPhysVmcs));
+        }
+    }
+#endif
+
+    /*
+     * EPT and unrestricted guest execution are determined in HMR3Init, verify the sanity of that.
+     */
+    AssertLogRelReturn(   !pVM->hm.s.fNestedPaging
+                       || (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_EPT),
+                       VERR_HM_IPE_1);
+    AssertLogRelReturn(   !pVM->hm.s.vmx.fUnrestrictedGuest
+                       || (   (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST)
+                           && pVM->hm.s.fNestedPaging),
+                       VERR_HM_IPE_1);
+
+    /*
+     * Disallow RDTSCP in the guest if there is no secondary process-based VM execution controls as otherwise
+     * RDTSCP would cause a #UD. There might be no CPUs out there where this happens, as RDTSCP was introduced
+     * in Nehalems and secondary VM exec. controls should be supported in all of them, but nonetheless it's Intel...
+     */
+    if (   !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
+        && CPUMR3GetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP))
+    {
+        CPUMR3ClearGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP);
+        LogRel(("HM: Disabled RDTSCP\n"));
+    }
+
+    if (!pVM->hm.s.vmx.fUnrestrictedGuest)
+    {
+        /* Allocate three pages for the TSS we need for real mode emulation. (2 pages for the IO bitmap) */
+        rc = PDMR3VmmDevHeapAlloc(pVM, HM_VTX_TOTAL_DEVHEAP_MEM, hmR3VmmDevHeapNotify, (RTR3PTR *)&pVM->hm.s.vmx.pRealModeTSS);
+        if (RT_SUCCESS(rc))
+        {
+            /* The IO bitmap starts right after the virtual interrupt redirection bitmap.
+               Refer Intel spec. 20.3.3 "Software Interrupt Handling in Virtual-8086 mode"
+               esp. Figure 20-5.*/
+            ASMMemZero32(pVM->hm.s.vmx.pRealModeTSS, sizeof(*pVM->hm.s.vmx.pRealModeTSS));
+            pVM->hm.s.vmx.pRealModeTSS->offIoBitmap = sizeof(*pVM->hm.s.vmx.pRealModeTSS);
+
+            /* Bit set to 0 means software interrupts are redirected to the
+               8086 program interrupt handler rather than switching to
+               protected-mode handler. */
+            memset(pVM->hm.s.vmx.pRealModeTSS->IntRedirBitmap, 0, sizeof(pVM->hm.s.vmx.pRealModeTSS->IntRedirBitmap));
+
+            /* Allow all port IO, so that port IO instructions do not cause
+               exceptions and would instead cause a VM-exit (based on VT-x's
+               IO bitmap which we currently configure to always cause an exit). */
+            memset(pVM->hm.s.vmx.pRealModeTSS + 1, 0, PAGE_SIZE * 2);
+            *((unsigned char *)pVM->hm.s.vmx.pRealModeTSS + HM_VTX_TSS_SIZE - 2) = 0xff;
+
+            /*
+             * Construct a 1024 element page directory with 4 MB pages for the identity mapped
+             * page table used in real and protected mode without paging with EPT.
+             */
+            pVM->hm.s.vmx.pNonPagingModeEPTPageTable = (PX86PD)((char *)pVM->hm.s.vmx.pRealModeTSS + PAGE_SIZE * 3);
+            for (uint32_t i = 0; i < X86_PG_ENTRIES; i++)
+            {
+                pVM->hm.s.vmx.pNonPagingModeEPTPageTable->a[i].u  = _4M * i;
+                pVM->hm.s.vmx.pNonPagingModeEPTPageTable->a[i].u |= X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US
+                                                                 |  X86_PDE4M_A | X86_PDE4M_D | X86_PDE4M_PS
+                                                                 |  X86_PDE4M_G;
+            }
+
+            /* We convert it here every time as PCI regions could be reconfigured. */
+            if (PDMVmmDevHeapIsEnabled(pVM))
+            {
+                RTGCPHYS GCPhys;
+                rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
+                AssertRCReturn(rc, rc);
+                LogRel(("HM: Real Mode TSS guest physaddr      = %#RGp\n", GCPhys));
+
+                rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
+                AssertRCReturn(rc, rc);
+                LogRel(("HM: Non-Paging Mode EPT CR3           = %#RGp\n", GCPhys));
+            }
+        }
+        else
+        {
+            LogRel(("HM: No real mode VT-x support (PDMR3VMMDevHeapAlloc returned %Rrc)\n", rc));
+            pVM->hm.s.vmx.pRealModeTSS = NULL;
+            pVM->hm.s.vmx.pNonPagingModeEPTPageTable = NULL;
+            return VMSetError(pVM, rc, RT_SRC_POS,
+                              "HM failure: No real mode VT-x support (PDMR3VMMDevHeapAlloc returned %Rrc)", rc);
+        }
+    }
+
+    LogRel((pVM->hm.s.fAllow64BitGuests ? "HM: Guest support: 32-bit and 64-bit\n"
+                                        : "HM: Guest support: 32-bit only\n"));
+
+    /*
+     * Call ring-0 to set up the VM.
+     */
+    rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), 0 /* idCpu */, VMMR0_DO_HM_SETUP_VM, 0 /* u64Arg */, NULL /* pReqHdr */);
+    if (rc != VINF_SUCCESS)
+    {
+        LogRel(("HM: VMX setup failed with rc=%Rrc!\n", rc));
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+            LogRel(("HM: CPU[%u] Last instruction error  %#x\n", idCpu, pVCpu->hm.s.vmx.LastError.u32InstrError));
+            LogRel(("HM: CPU[%u] HM error                %#x (%u)\n", idCpu, pVCpu->hm.s.u32HMError, pVCpu->hm.s.u32HMError));
+        }
+        HMR3CheckError(pVM, rc);
+        return VMSetError(pVM, rc, RT_SRC_POS, "VT-x setup failed: %Rrc", rc);
+    }
+
+    LogRel(("HM: Supports VMCS EFER fields         = %RTbool\n", pVM->hm.s.vmx.fSupportsVmcsEfer));
+    LogRel(("HM: Enabled VMX\n"));
+    pVM->hm.s.vmx.fEnabled = true;
+
+    hmR3DisableRawMode(pVM); /** @todo make this go away! */
+
+    /*
+     * Change the CPU features.
+     */
+    CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SEP);
+    if (pVM->hm.s.fAllow64BitGuests)
+    {
+        CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE);
+        CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
+        CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SYSCALL);            /* 64 bits only on Intel CPUs */
+        CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LAHF);
+        CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
+    }
+    /* Turn on NXE if PAE has been enabled *and* the host has turned on NXE
+       (we reuse the host EFER in the switcher). */
+    /** @todo this needs to be fixed properly!! */
+    else if (CPUMR3GetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE))
+    {
+        if (pVM->hm.s.vmx.u64HostMsrEfer & MSR_K6_EFER_NXE)
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
+        else
+            LogRel(("HM: NX not enabled on the host, unavailable to PAE guest\n"));
+    }
+
+    /*
+     * Log configuration details.
+     */
+    if (pVM->hm.s.fNestedPaging)
+    {
+        LogRel(("HM: Enabled nested paging\n"));
+        if (pVM->hm.s.vmx.enmTlbFlushEpt == VMXTLBFLUSHEPT_SINGLE_CONTEXT)
+            LogRel(("HM:   EPT flush type                  = Single context\n"));
+        else if (pVM->hm.s.vmx.enmTlbFlushEpt == VMXTLBFLUSHEPT_ALL_CONTEXTS)
+            LogRel(("HM:   EPT flush type                  = All contexts\n"));
+        else if (pVM->hm.s.vmx.enmTlbFlushEpt == VMXTLBFLUSHEPT_NOT_SUPPORTED)
+            LogRel(("HM:   EPT flush type                  = Not supported\n"));
+        else
+            LogRel(("HM:   EPT flush type                  = %#x\n", pVM->hm.s.vmx.enmTlbFlushEpt));
+
+        if (pVM->hm.s.vmx.fUnrestrictedGuest)
+            LogRel(("HM: Enabled unrestricted guest execution\n"));
+
+        if (pVM->hm.s.fLargePages)
+        {
+            /* Use large (2 MB) pages for our EPT PDEs where possible. */
+            PGMSetLargePageUsage(pVM, true);
+            LogRel(("HM: Enabled large page support\n"));
+        }
+    }
+    else
+        Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
+
+    if (pVM->hm.s.vmx.fVpid)
+    {
+        LogRel(("HM: Enabled VPID\n"));
+        if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_INDIV_ADDR)
+            LogRel(("HM:   VPID flush type                 = Individual addresses\n"));
+        else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
+            LogRel(("HM:   VPID flush type                 = Single context\n"));
+        else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
+            LogRel(("HM:   VPID flush type                 = All contexts\n"));
+        else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
+            LogRel(("HM:   VPID flush type                 = Single context retain globals\n"));
+        else
+            LogRel(("HM:   VPID flush type                 = %#x\n", pVM->hm.s.vmx.enmTlbFlushVpid));
+    }
+    else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_NOT_SUPPORTED)
+        LogRel(("HM: Ignoring VPID capabilities of CPU\n"));
+
+    if (pVM->hm.s.vmx.fUsePreemptTimer)
+        LogRel(("HM: Enabled VMX-preemption timer (cPreemptTimerShift=%u)\n", pVM->hm.s.vmx.cPreemptTimerShift));
+    else
+        LogRel(("HM: Disabled VMX-preemption timer\n"));
+
+    if (pVM->hm.s.fVirtApicRegs)
+        LogRel(("HM: Enabled APIC-register virtualization support\n"));
+
+    if (pVM->hm.s.fPostedIntrs)
+        LogRel(("HM: Enabled posted-interrupt processing support\n"));
+
+    if (pVM->hm.s.vmx.fUseVmcsShadowing)
+    {
+        bool const fFullVmcsShadow = RT_BOOL(pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL);
+        LogRel(("HM: Enabled %s VMCS shadowing\n", fFullVmcsShadow ? "full" : "partial"));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Finish AMD-V initialization (after ring-0 init).
+ *
+ * @returns VBox status code.
+ * @param   pVM                The cross context VM structure.
+ */
+static int hmR3InitFinalizeR0Amd(PVM pVM)
+{
+    LogFunc(("pVM->hm.s.svm.fSupported = %d\n", pVM->hm.s.svm.fSupported));
+
+    LogRel(("HM: Using AMD-V implementation 2.0\n"));
+
+    uint32_t u32Family;
+    uint32_t u32Model;
+    uint32_t u32Stepping;
+    if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
+        LogRel(("HM: AMD Cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
+    LogRel(("HM: Max resume loops                  = %u\n",     pVM->hm.s.cMaxResumeLoops));
+    LogRel(("HM: AMD HWCR MSR                      = %#RX64\n", pVM->hm.s.svm.u64MsrHwcr));
+    LogRel(("HM: AMD-V revision                    = %#x\n",    pVM->hm.s.svm.u32Rev));
+    LogRel(("HM: AMD-V max ASID                    = %RU32\n",  pVM->hm.s.uMaxAsid));
+    LogRel(("HM: AMD-V features                    = %#x\n",    pVM->hm.s.svm.u32Features));
+
+    /*
+     * Enumerate AMD-V features.
+     */
+    static const struct { uint32_t fFlag; const char *pszName; } s_aSvmFeatures[] =
+    {
+#define HMSVM_REPORT_FEATURE(a_StrDesc, a_Define) { a_Define, a_StrDesc }
+        HMSVM_REPORT_FEATURE("NESTED_PAGING",          X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING),
+        HMSVM_REPORT_FEATURE("LBR_VIRT",               X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT),
+        HMSVM_REPORT_FEATURE("SVM_LOCK",               X86_CPUID_SVM_FEATURE_EDX_SVM_LOCK),
+        HMSVM_REPORT_FEATURE("NRIP_SAVE",              X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE),
+        HMSVM_REPORT_FEATURE("TSC_RATE_MSR",           X86_CPUID_SVM_FEATURE_EDX_TSC_RATE_MSR),
+        HMSVM_REPORT_FEATURE("VMCB_CLEAN",             X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN),
+        HMSVM_REPORT_FEATURE("FLUSH_BY_ASID",          X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID),
+        HMSVM_REPORT_FEATURE("DECODE_ASSISTS",         X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS),
+        HMSVM_REPORT_FEATURE("PAUSE_FILTER",           X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER),
+        HMSVM_REPORT_FEATURE("PAUSE_FILTER_THRESHOLD", X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD),
+        HMSVM_REPORT_FEATURE("AVIC",                   X86_CPUID_SVM_FEATURE_EDX_AVIC),
+        HMSVM_REPORT_FEATURE("VIRT_VMSAVE_VMLOAD",     X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD),
+        HMSVM_REPORT_FEATURE("VGIF",                   X86_CPUID_SVM_FEATURE_EDX_VGIF),
+        HMSVM_REPORT_FEATURE("GMET",                   X86_CPUID_SVM_FEATURE_EDX_GMET),
+#undef HMSVM_REPORT_FEATURE
+    };
+
+    uint32_t fSvmFeatures = pVM->hm.s.svm.u32Features;
+    for (unsigned i = 0; i < RT_ELEMENTS(s_aSvmFeatures); i++)
+        if (fSvmFeatures & s_aSvmFeatures[i].fFlag)
+        {
+            LogRel(("HM:   %s\n", s_aSvmFeatures[i].pszName));
+            fSvmFeatures &= ~s_aSvmFeatures[i].fFlag;
+        }
+    if (fSvmFeatures)
+        for (unsigned iBit = 0; iBit < 32; iBit++)
+            if (RT_BIT_32(iBit) & fSvmFeatures)
+                LogRel(("HM:   Reserved bit %u\n", iBit));
+
+    /*
+     * Nested paging is determined in HMR3Init, verify the sanity of that.
+     */
+    AssertLogRelReturn(   !pVM->hm.s.fNestedPaging
+                       || (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING),
+                       VERR_HM_IPE_1);
+
+#if 0
+    /** @todo Add and query IPRT API for host OS support for posted-interrupt IPI
+     *        here. */
+    if (RTR0IsPostIpiSupport())
+        pVM->hm.s.fPostedIntrs = true;
+#endif
+
+    /*
+     * Call ring-0 to set up the VM.
+     */
+    int rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), 0 /*idCpu*/, VMMR0_DO_HM_SETUP_VM, 0, NULL);
+    if (rc != VINF_SUCCESS)
+    {
+        AssertMsgFailed(("%Rrc\n", rc));
+        LogRel(("HM: AMD-V setup failed with rc=%Rrc!\n", rc));
+        return VMSetError(pVM, rc, RT_SRC_POS, "AMD-V setup failed: %Rrc", rc);
+    }
+
+    LogRel(("HM: Enabled SVM\n"));
+    pVM->hm.s.svm.fEnabled = true;
+
+    if (pVM->hm.s.fNestedPaging)
+    {
+        LogRel(("HM:   Enabled nested paging\n"));
+
+        /*
+         * Enable large pages (2 MB) if applicable.
+         */
+        if (pVM->hm.s.fLargePages)
+        {
+            PGMSetLargePageUsage(pVM, true);
+            LogRel(("HM:   Enabled large page support\n"));
+        }
+    }
+
+    if (pVM->hm.s.fVirtApicRegs)
+        LogRel(("HM:   Enabled APIC-register virtualization support\n"));
+
+    if (pVM->hm.s.fPostedIntrs)
+        LogRel(("HM:   Enabled posted-interrupt processing support\n"));
+
+    hmR3DisableRawMode(pVM);
+
+    /*
+     * Change the CPU features.
+     */
+    CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SEP);
+    CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SYSCALL);
+    if (pVM->hm.s.fAllow64BitGuests)
+    {
+        CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE);
+        CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
+        CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
+        CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LAHF);
+    }
+    /* Turn on NXE if PAE has been enabled. */
+    else if (CPUMR3GetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE))
+        CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
+
+    LogRel((pVM->hm.s.fTprPatchingAllowed ? "HM: Enabled TPR patching\n"
+                                          : "HM: Disabled TPR patching\n"));
+
+    LogRel((pVM->hm.s.fAllow64BitGuests ? "HM: Guest support: 32-bit and 64-bit\n"
+                                        : "HM: Guest support: 32-bit only\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) HMR3Relocate(PVM pVM)
+{
+    Log(("HMR3Relocate to %RGv\n", MMHyperGetArea(pVM, 0)));
+
+    /* Fetch the current paging mode during the relocate callback during state loading. */
+    if (VMR3GetState(pVM) == VMSTATE_LOADING)
+    {
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+            pVCpu->hm.s.enmShadowMode = PGMGetShadowMode(pVCpu);
+        }
+    }
+}
+
+
+/**
+ * Terminates the HM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM itself is, at this point, powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) HMR3Term(PVM pVM)
+{
+    if (pVM->hm.s.vmx.pRealModeTSS)
+    {
+        PDMR3VmmDevHeapFree(pVM, pVM->hm.s.vmx.pRealModeTSS);
+        pVM->hm.s.vmx.pRealModeTSS       = 0;
+    }
+    hmR3TermCPU(pVM);
+    return 0;
+}
+
+
+/**
+ * Terminates the per-VCPU HM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+static int hmR3TermCPU(PVM pVM)
+{
+#ifdef VBOX_WITH_STATISTICS
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu]; NOREF(pVCpu);
+        if (pVCpu->hm.s.paStatExitReason)
+        {
+            MMHyperFree(pVM, pVCpu->hm.s.paStatExitReason);
+            pVCpu->hm.s.paStatExitReason   = NULL;
+            pVCpu->hm.s.paStatExitReasonR0 = NIL_RTR0PTR;
+        }
+        if (pVCpu->hm.s.paStatInjectedIrqs)
+        {
+            MMHyperFree(pVM, pVCpu->hm.s.paStatInjectedIrqs);
+            pVCpu->hm.s.paStatInjectedIrqs   = NULL;
+            pVCpu->hm.s.paStatInjectedIrqsR0 = NIL_RTR0PTR;
+        }
+        if (pVCpu->hm.s.paStatInjectedXcpts)
+        {
+            MMHyperFree(pVM, pVCpu->hm.s.paStatInjectedXcpts);
+            pVCpu->hm.s.paStatInjectedXcpts   = NULL;
+            pVCpu->hm.s.paStatInjectedXcptsR0 = NIL_RTR0PTR;
+        }
+# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+        if (pVCpu->hm.s.paStatNestedExitReason)
+        {
+            MMHyperFree(pVM, pVCpu->hm.s.paStatNestedExitReason);
+            pVCpu->hm.s.paStatNestedExitReason   = NULL;
+            pVCpu->hm.s.paStatNestedExitReasonR0 = NIL_RTR0PTR;
+        }
+# endif
+    }
+#else
+    RT_NOREF(pVM);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resets a virtual CPU.
+ *
+ * Used by HMR3Reset and CPU hot plugging.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure to reset.
+ */
+VMMR3_INT_DECL(void) HMR3ResetCpu(PVMCPU pVCpu)
+{
+    /* Sync. entire state on VM reset ring-0 re-entry. It's safe to reset
+       the HM flags here, all other EMTs are in ring-3. See VMR3Reset(). */
+    pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_ALL_GUEST;
+
+    pVCpu->hm.s.fActive                        = false;
+    pVCpu->hm.s.Event.fPending                 = false;
+    pVCpu->hm.s.vmx.u64GstMsrApicBase          = 0;
+    pVCpu->hm.s.vmx.VmcsInfo.fSwitchedTo64on32Obsolete = false;
+    pVCpu->hm.s.vmx.VmcsInfo.fWasInRealMode    = true;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmx)
+    {
+        pVCpu->hm.s.vmx.VmcsInfoNstGst.fSwitchedTo64on32Obsolete = false;
+        pVCpu->hm.s.vmx.VmcsInfoNstGst.fWasInRealMode    = true;
+    }
+#endif
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * For the HM component this means that any GDT/LDT/TSS monitors
+ * needs to be removed.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) HMR3Reset(PVM pVM)
+{
+    LogFlow(("HMR3Reset:\n"));
+
+    if (HMIsEnabled(pVM))
+        hmR3DisableRawMode(pVM);
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        HMR3ResetCpu(pVM->apCpusR3[idCpu]);
+
+    /* Clear all patch information. */
+    pVM->hm.s.pGuestPatchMem     = 0;
+    pVM->hm.s.pFreeGuestPatchMem = 0;
+    pVM->hm.s.cbGuestPatchMem    = 0;
+    pVM->hm.s.cPatches           = 0;
+    pVM->hm.s.PatchTree          = 0;
+    pVM->hm.s.fTPRPatchingActive = false;
+    ASMMemZero32(pVM->hm.s.aPatches, sizeof(pVM->hm.s.aPatches));
+}
+
+
+/**
+ * Callback to patch a TPR instruction (vmmcall or mov cr8).
+ *
+ * @returns VBox strict status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser  Unused.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) hmR3RemovePatches(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    VMCPUID idCpu = (VMCPUID)(uintptr_t)pvUser;
+
+    /* Only execute the handler on the VCPU the original patch request was issued. */
+    if (pVCpu->idCpu != idCpu)
+        return VINF_SUCCESS;
+
+    Log(("hmR3RemovePatches\n"));
+    for (unsigned i = 0; i < pVM->hm.s.cPatches; i++)
+    {
+        uint8_t         abInstr[15];
+        PHMTPRPATCH pPatch = &pVM->hm.s.aPatches[i];
+        RTGCPTR         pInstrGC = (RTGCPTR)pPatch->Core.Key;
+        int             rc;
+
+#ifdef LOG_ENABLED
+        char            szOutput[256];
+        rc = DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, CPUMGetGuestCS(pVCpu), pInstrGC, DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                                szOutput, sizeof(szOutput), NULL);
+        if (RT_SUCCESS(rc))
+            Log(("Patched instr: %s\n", szOutput));
+#endif
+
+        /* Check if the instruction is still the same. */
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pInstrGC, pPatch->cbNewOp);
+        if (rc != VINF_SUCCESS)
+        {
+            Log(("Patched code removed? (rc=%Rrc0\n", rc));
+            continue;   /* swapped out or otherwise removed; skip it. */
+        }
+
+        if (memcmp(abInstr, pPatch->aNewOpcode, pPatch->cbNewOp))
+        {
+            Log(("Patched instruction was changed! (rc=%Rrc0\n", rc));
+            continue;   /* skip it. */
+        }
+
+        rc = PGMPhysSimpleWriteGCPtr(pVCpu, pInstrGC, pPatch->aOpcode, pPatch->cbOp);
+        AssertRC(rc);
+
+#ifdef LOG_ENABLED
+        rc = DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, CPUMGetGuestCS(pVCpu), pInstrGC, DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                                szOutput, sizeof(szOutput), NULL);
+        if (RT_SUCCESS(rc))
+            Log(("Original instr: %s\n", szOutput));
+#endif
+    }
+    pVM->hm.s.cPatches           = 0;
+    pVM->hm.s.PatchTree          = 0;
+    pVM->hm.s.pFreeGuestPatchMem = pVM->hm.s.pGuestPatchMem;
+    pVM->hm.s.fTPRPatchingActive = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for enabling patching in a VT-x/AMD-V guest.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   idCpu       VCPU to execute hmR3RemovePatches on.
+ * @param   pPatchMem   Patch memory range.
+ * @param   cbPatchMem  Size of the memory range.
+ */
+static int hmR3EnablePatching(PVM pVM, VMCPUID idCpu, RTRCPTR pPatchMem, unsigned cbPatchMem)
+{
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE, hmR3RemovePatches, (void *)(uintptr_t)idCpu);
+    AssertRC(rc);
+
+    pVM->hm.s.pGuestPatchMem      = pPatchMem;
+    pVM->hm.s.pFreeGuestPatchMem  = pPatchMem;
+    pVM->hm.s.cbGuestPatchMem     = cbPatchMem;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Enable patching in a VT-x/AMD-V guest
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pPatchMem   Patch memory range.
+ * @param   cbPatchMem  Size of the memory range.
+ */
+VMMR3_INT_DECL(int)  HMR3EnablePatching(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
+{
+    VM_ASSERT_EMT(pVM);
+    Log(("HMR3EnablePatching %RGv size %x\n", pPatchMem, cbPatchMem));
+    if (pVM->cCpus > 1)
+    {
+        /* We own the IOM lock here and could cause a deadlock by waiting for a VCPU that is blocking on the IOM lock. */
+        int rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE,
+                                   (PFNRT)hmR3EnablePatching, 4, pVM, VMMGetCpuId(pVM), (RTRCPTR)pPatchMem, cbPatchMem);
+        AssertRC(rc);
+        return rc;
+    }
+    return hmR3EnablePatching(pVM, VMMGetCpuId(pVM), (RTRCPTR)pPatchMem, cbPatchMem);
+}
+
+
+/**
+ * Disable patching in a VT-x/AMD-V guest.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pPatchMem   Patch memory range.
+ * @param   cbPatchMem  Size of the memory range.
+ */
+VMMR3_INT_DECL(int)  HMR3DisablePatching(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
+{
+    Log(("HMR3DisablePatching %RGv size %x\n", pPatchMem, cbPatchMem));
+    RT_NOREF2(pPatchMem, cbPatchMem);
+
+    Assert(pVM->hm.s.pGuestPatchMem == pPatchMem);
+    Assert(pVM->hm.s.cbGuestPatchMem == cbPatchMem);
+
+    /** @todo Potential deadlock when other VCPUs are waiting on the IOM lock (we own it)!! */
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE, hmR3RemovePatches,
+                                (void *)(uintptr_t)VMMGetCpuId(pVM));
+    AssertRC(rc);
+
+    pVM->hm.s.pGuestPatchMem      = 0;
+    pVM->hm.s.pFreeGuestPatchMem  = 0;
+    pVM->hm.s.cbGuestPatchMem     = 0;
+    pVM->hm.s.fTPRPatchingActive  = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Callback to patch a TPR instruction (vmmcall or mov cr8).
+ *
+ * @returns VBox strict status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser  User specified CPU context.
+ *
+ */
+static DECLCALLBACK(VBOXSTRICTRC) hmR3ReplaceTprInstr(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    /*
+     * Only execute the handler on the VCPU the original patch request was
+     * issued. (The other CPU(s) might not yet have switched to protected
+     * mode, nor have the correct memory context.)
+     */
+    VMCPUID idCpu = (VMCPUID)(uintptr_t)pvUser;
+    if (pVCpu->idCpu != idCpu)
+        return VINF_SUCCESS;
+
+    /*
+     * We're racing other VCPUs here, so don't try patch the instruction twice
+     * and make sure there is still room for our patch record.
+     */
+    PCPUMCTX    pCtx   = &pVCpu->cpum.GstCtx;
+    PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
+    if (pPatch)
+    {
+        Log(("hmR3ReplaceTprInstr: already patched %RGv\n", pCtx->rip));
+        return VINF_SUCCESS;
+    }
+    uint32_t const idx = pVM->hm.s.cPatches;
+    if (idx >= RT_ELEMENTS(pVM->hm.s.aPatches))
+    {
+        Log(("hmR3ReplaceTprInstr: no available patch slots (%RGv)\n", pCtx->rip));
+        return VINF_SUCCESS;
+    }
+    pPatch = &pVM->hm.s.aPatches[idx];
+
+    Log(("hmR3ReplaceTprInstr: rip=%RGv idxPatch=%u\n", pCtx->rip, idx));
+
+    /*
+     * Disassembler the instruction and get cracking.
+     */
+    DBGFR3_DISAS_INSTR_CUR_LOG(pVCpu, "hmR3ReplaceTprInstr");
+    PDISCPUSTATE    pDis = &pVCpu->hm.s.DisState;
+    uint32_t        cbOp;
+    int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
+    AssertRC(rc);
+    if (    rc == VINF_SUCCESS
+        &&  pDis->pCurInstr->uOpcode == OP_MOV
+        &&  cbOp >= 3)
+    {
+        static uint8_t const s_abVMMCall[3] = { 0x0f, 0x01, 0xd9 };
+
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, pPatch->aOpcode, pCtx->rip, cbOp);
+        AssertRC(rc);
+
+        pPatch->cbOp = cbOp;
+
+        if (pDis->Param1.fUse == DISUSE_DISPLACEMENT32)
+        {
+            /* write. */
+            if (pDis->Param2.fUse == DISUSE_REG_GEN32)
+            {
+                pPatch->enmType     = HMTPRINSTR_WRITE_REG;
+                pPatch->uSrcOperand = pDis->Param2.Base.idxGenReg;
+                Log(("hmR3ReplaceTprInstr: HMTPRINSTR_WRITE_REG %u\n", pDis->Param2.Base.idxGenReg));
+            }
+            else
+            {
+                Assert(pDis->Param2.fUse == DISUSE_IMMEDIATE32);
+                pPatch->enmType     = HMTPRINSTR_WRITE_IMM;
+                pPatch->uSrcOperand = pDis->Param2.uValue;
+                Log(("hmR3ReplaceTprInstr: HMTPRINSTR_WRITE_IMM %#llx\n", pDis->Param2.uValue));
+            }
+            rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, s_abVMMCall, sizeof(s_abVMMCall));
+            AssertRC(rc);
+
+            memcpy(pPatch->aNewOpcode, s_abVMMCall, sizeof(s_abVMMCall));
+            pPatch->cbNewOp = sizeof(s_abVMMCall);
+            STAM_COUNTER_INC(&pVM->hm.s.StatTprReplaceSuccessVmc);
+        }
+        else
+        {
+            /*
+             * TPR Read.
+             *
+             * Found:
+             *   mov eax, dword [fffe0080]        (5 bytes)
+             * Check if next instruction is:
+             *   shr eax, 4
+             */
+            Assert(pDis->Param1.fUse == DISUSE_REG_GEN32);
+
+            uint8_t  const idxMmioReg = pDis->Param1.Base.idxGenReg;
+            uint8_t  const cbOpMmio   = cbOp;
+            uint64_t const uSavedRip  = pCtx->rip;
+
+            pCtx->rip += cbOp;
+            rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
+            DBGFR3_DISAS_INSTR_CUR_LOG(pVCpu, "Following read");
+            pCtx->rip = uSavedRip;
+
+            if (    rc == VINF_SUCCESS
+                &&  pDis->pCurInstr->uOpcode == OP_SHR
+                &&  pDis->Param1.fUse == DISUSE_REG_GEN32
+                &&  pDis->Param1.Base.idxGenReg == idxMmioReg
+                &&  pDis->Param2.fUse == DISUSE_IMMEDIATE8
+                &&  pDis->Param2.uValue == 4
+                &&  cbOpMmio + cbOp < sizeof(pVM->hm.s.aPatches[idx].aOpcode))
+            {
+                uint8_t abInstr[15];
+
+                /* Replacing the two instructions above with an AMD-V specific lock-prefixed 32-bit MOV CR8 instruction so as to
+                   access CR8 in 32-bit mode and not cause a #VMEXIT. */
+                rc = PGMPhysSimpleReadGCPtr(pVCpu, &pPatch->aOpcode, pCtx->rip, cbOpMmio + cbOp);
+                AssertRC(rc);
+
+                pPatch->cbOp = cbOpMmio + cbOp;
+
+                /* 0xf0, 0x0f, 0x20, 0xc0 = mov eax, cr8 */
+                abInstr[0] = 0xf0;
+                abInstr[1] = 0x0f;
+                abInstr[2] = 0x20;
+                abInstr[3] = 0xc0 | pDis->Param1.Base.idxGenReg;
+                for (unsigned i = 4; i < pPatch->cbOp; i++)
+                    abInstr[i] = 0x90;  /* nop */
+
+                rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, abInstr, pPatch->cbOp);
+                AssertRC(rc);
+
+                memcpy(pPatch->aNewOpcode, abInstr, pPatch->cbOp);
+                pPatch->cbNewOp = pPatch->cbOp;
+                STAM_COUNTER_INC(&pVM->hm.s.StatTprReplaceSuccessCr8);
+
+                Log(("Acceptable read/shr candidate!\n"));
+                pPatch->enmType = HMTPRINSTR_READ_SHR4;
+            }
+            else
+            {
+                pPatch->enmType     = HMTPRINSTR_READ;
+                pPatch->uDstOperand = idxMmioReg;
+
+                rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, s_abVMMCall, sizeof(s_abVMMCall));
+                AssertRC(rc);
+
+                memcpy(pPatch->aNewOpcode, s_abVMMCall, sizeof(s_abVMMCall));
+                pPatch->cbNewOp = sizeof(s_abVMMCall);
+                STAM_COUNTER_INC(&pVM->hm.s.StatTprReplaceSuccessVmc);
+                Log(("hmR3ReplaceTprInstr: HMTPRINSTR_READ %u\n", pPatch->uDstOperand));
+            }
+        }
+
+        pPatch->Core.Key = pCtx->eip;
+        rc = RTAvloU32Insert(&pVM->hm.s.PatchTree, &pPatch->Core);
+        AssertRC(rc);
+
+        pVM->hm.s.cPatches++;
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Save invalid patch, so we will not try again.
+     */
+    Log(("hmR3ReplaceTprInstr: Failed to patch instr!\n"));
+    pPatch->Core.Key = pCtx->eip;
+    pPatch->enmType  = HMTPRINSTR_INVALID;
+    rc = RTAvloU32Insert(&pVM->hm.s.PatchTree, &pPatch->Core);
+    AssertRC(rc);
+    pVM->hm.s.cPatches++;
+    STAM_COUNTER_INC(&pVM->hm.s.StatTprReplaceFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Callback to patch a TPR instruction (jump to generated code).
+ *
+ * @returns VBox strict status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser  User specified CPU context.
+ *
+ */
+static DECLCALLBACK(VBOXSTRICTRC) hmR3PatchTprInstr(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    /*
+     * Only execute the handler on the VCPU the original patch request was
+     * issued. (The other CPU(s) might not yet have switched to protected
+     * mode, nor have the correct memory context.)
+     */
+    VMCPUID         idCpu  = (VMCPUID)(uintptr_t)pvUser;
+    if (pVCpu->idCpu != idCpu)
+        return VINF_SUCCESS;
+
+    /*
+     * We're racing other VCPUs here, so don't try patch the instruction twice
+     * and make sure there is still room for our patch record.
+     */
+    PCPUMCTX    pCtx   = &pVCpu->cpum.GstCtx;
+    PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
+    if (pPatch)
+    {
+        Log(("hmR3PatchTprInstr: already patched %RGv\n", pCtx->rip));
+        return VINF_SUCCESS;
+    }
+    uint32_t const  idx = pVM->hm.s.cPatches;
+    if (idx >= RT_ELEMENTS(pVM->hm.s.aPatches))
+    {
+        Log(("hmR3PatchTprInstr: no available patch slots (%RGv)\n", pCtx->rip));
+        return VINF_SUCCESS;
+    }
+    pPatch = &pVM->hm.s.aPatches[idx];
+
+    Log(("hmR3PatchTprInstr: rip=%RGv idxPatch=%u\n", pCtx->rip, idx));
+    DBGFR3_DISAS_INSTR_CUR_LOG(pVCpu, "hmR3PatchTprInstr");
+
+    /*
+     * Disassemble the instruction and get cracking.
+     */
+    PDISCPUSTATE    pDis   = &pVCpu->hm.s.DisState;
+    uint32_t        cbOp;
+    int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
+    AssertRC(rc);
+    if (    rc == VINF_SUCCESS
+        &&  pDis->pCurInstr->uOpcode == OP_MOV
+        &&  cbOp >= 5)
+    {
+        uint8_t         aPatch[64];
+        uint32_t        off = 0;
+
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, pPatch->aOpcode, pCtx->rip, cbOp);
+        AssertRC(rc);
+
+        pPatch->cbOp    = cbOp;
+        pPatch->enmType = HMTPRINSTR_JUMP_REPLACEMENT;
+
+        if (pDis->Param1.fUse == DISUSE_DISPLACEMENT32)
+        {
+            /*
+             * TPR write:
+             *
+             * push ECX                      [51]
+             * push EDX                      [52]
+             * push EAX                      [50]
+             * xor EDX,EDX                   [31 D2]
+             * mov EAX,EAX                   [89 C0]
+             *  or
+             * mov EAX,0000000CCh            [B8 CC 00 00 00]
+             * mov ECX,0C0000082h            [B9 82 00 00 C0]
+             * wrmsr                         [0F 30]
+             * pop EAX                       [58]
+             * pop EDX                       [5A]
+             * pop ECX                       [59]
+             * jmp return_address            [E9 return_address]
+             */
+            bool fUsesEax = (pDis->Param2.fUse == DISUSE_REG_GEN32 && pDis->Param2.Base.idxGenReg == DISGREG_EAX);
+
+            aPatch[off++] = 0x51;    /* push ecx */
+            aPatch[off++] = 0x52;    /* push edx */
+            if (!fUsesEax)
+                aPatch[off++] = 0x50;    /* push eax */
+            aPatch[off++] = 0x31;    /* xor edx, edx */
+            aPatch[off++] = 0xd2;
+            if (pDis->Param2.fUse == DISUSE_REG_GEN32)
+            {
+                if (!fUsesEax)
+                {
+                    aPatch[off++] = 0x89;    /* mov eax, src_reg */
+                    aPatch[off++] = MAKE_MODRM(3, pDis->Param2.Base.idxGenReg, DISGREG_EAX);
+                }
+            }
+            else
+            {
+                Assert(pDis->Param2.fUse == DISUSE_IMMEDIATE32);
+                aPatch[off++] = 0xb8;    /* mov eax, immediate */
+                *(uint32_t *)&aPatch[off] = pDis->Param2.uValue;
+                off += sizeof(uint32_t);
+            }
+            aPatch[off++] = 0xb9;    /* mov ecx, 0xc0000082 */
+            *(uint32_t *)&aPatch[off] = MSR_K8_LSTAR;
+            off += sizeof(uint32_t);
+
+            aPatch[off++] = 0x0f;    /* wrmsr */
+            aPatch[off++] = 0x30;
+            if (!fUsesEax)
+                aPatch[off++] = 0x58;    /* pop eax */
+            aPatch[off++] = 0x5a;    /* pop edx */
+            aPatch[off++] = 0x59;    /* pop ecx */
+        }
+        else
+        {
+            /*
+             * TPR read:
+             *
+             * push ECX                      [51]
+             * push EDX                      [52]
+             * push EAX                      [50]
+             * mov ECX,0C0000082h            [B9 82 00 00 C0]
+             * rdmsr                         [0F 32]
+             * mov EAX,EAX                   [89 C0]
+             * pop EAX                       [58]
+             * pop EDX                       [5A]
+             * pop ECX                       [59]
+             * jmp return_address            [E9 return_address]
+             */
+            Assert(pDis->Param1.fUse == DISUSE_REG_GEN32);
+
+            if (pDis->Param1.Base.idxGenReg != DISGREG_ECX)
+                aPatch[off++] = 0x51;    /* push ecx */
+            if (pDis->Param1.Base.idxGenReg != DISGREG_EDX )
+                aPatch[off++] = 0x52;    /* push edx */
+            if (pDis->Param1.Base.idxGenReg != DISGREG_EAX)
+                aPatch[off++] = 0x50;    /* push eax */
+
+            aPatch[off++] = 0x31;    /* xor edx, edx */
+            aPatch[off++] = 0xd2;
+
+            aPatch[off++] = 0xb9;    /* mov ecx, 0xc0000082 */
+            *(uint32_t *)&aPatch[off] = MSR_K8_LSTAR;
+            off += sizeof(uint32_t);
+
+            aPatch[off++] = 0x0f;    /* rdmsr */
+            aPatch[off++] = 0x32;
+
+            if (pDis->Param1.Base.idxGenReg != DISGREG_EAX)
+            {
+                aPatch[off++] = 0x89;    /* mov dst_reg, eax */
+                aPatch[off++] = MAKE_MODRM(3, DISGREG_EAX, pDis->Param1.Base.idxGenReg);
+            }
+
+            if (pDis->Param1.Base.idxGenReg != DISGREG_EAX)
+                aPatch[off++] = 0x58;    /* pop eax */
+            if (pDis->Param1.Base.idxGenReg != DISGREG_EDX )
+                aPatch[off++] = 0x5a;    /* pop edx */
+            if (pDis->Param1.Base.idxGenReg != DISGREG_ECX)
+                aPatch[off++] = 0x59;    /* pop ecx */
+        }
+        aPatch[off++] = 0xe9;    /* jmp return_address */
+        *(RTRCUINTPTR *)&aPatch[off] = ((RTRCUINTPTR)pCtx->eip + cbOp) - ((RTRCUINTPTR)pVM->hm.s.pFreeGuestPatchMem + off + 4);
+        off += sizeof(RTRCUINTPTR);
+
+        if (pVM->hm.s.pFreeGuestPatchMem + off <= pVM->hm.s.pGuestPatchMem + pVM->hm.s.cbGuestPatchMem)
+        {
+            /* Write new code to the patch buffer. */
+            rc = PGMPhysSimpleWriteGCPtr(pVCpu, pVM->hm.s.pFreeGuestPatchMem, aPatch, off);
+            AssertRC(rc);
+
+#ifdef LOG_ENABLED
+            uint32_t cbCurInstr;
+            for (RTGCPTR GCPtrInstr = pVM->hm.s.pFreeGuestPatchMem;
+                 GCPtrInstr < pVM->hm.s.pFreeGuestPatchMem + off;
+                 GCPtrInstr += RT_MAX(cbCurInstr, 1))
+            {
+                char     szOutput[256];
+                rc = DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, pCtx->cs.Sel, GCPtrInstr, DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                                        szOutput, sizeof(szOutput), &cbCurInstr);
+                if (RT_SUCCESS(rc))
+                    Log(("Patch instr %s\n", szOutput));
+                else
+                    Log(("%RGv: rc=%Rrc\n", GCPtrInstr, rc));
+            }
+#endif
+
+            pPatch->aNewOpcode[0] = 0xE9;
+            *(RTRCUINTPTR *)&pPatch->aNewOpcode[1] = ((RTRCUINTPTR)pVM->hm.s.pFreeGuestPatchMem) - ((RTRCUINTPTR)pCtx->eip + 5);
+
+            /* Overwrite the TPR instruction with a jump. */
+            rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->eip, pPatch->aNewOpcode, 5);
+            AssertRC(rc);
+
+            DBGFR3_DISAS_INSTR_CUR_LOG(pVCpu, "Jump");
+
+            pVM->hm.s.pFreeGuestPatchMem += off;
+            pPatch->cbNewOp = 5;
+
+            pPatch->Core.Key = pCtx->eip;
+            rc = RTAvloU32Insert(&pVM->hm.s.PatchTree, &pPatch->Core);
+            AssertRC(rc);
+
+            pVM->hm.s.cPatches++;
+            pVM->hm.s.fTPRPatchingActive = true;
+            STAM_COUNTER_INC(&pVM->hm.s.StatTprPatchSuccess);
+            return VINF_SUCCESS;
+        }
+
+        Log(("Ran out of space in our patch buffer!\n"));
+    }
+    else
+        Log(("hmR3PatchTprInstr: Failed to patch instr!\n"));
+
+
+    /*
+     * Save invalid patch, so we will not try again.
+     */
+    pPatch = &pVM->hm.s.aPatches[idx];
+    pPatch->Core.Key = pCtx->eip;
+    pPatch->enmType  = HMTPRINSTR_INVALID;
+    rc = RTAvloU32Insert(&pVM->hm.s.PatchTree, &pPatch->Core);
+    AssertRC(rc);
+    pVM->hm.s.cPatches++;
+    STAM_COUNTER_INC(&pVM->hm.s.StatTprPatchFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Attempt to patch TPR mmio instructions.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMR3_INT_DECL(int) HMR3PatchTprInstr(PVM pVM, PVMCPU pVCpu)
+{
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE,
+                                pVM->hm.s.pGuestPatchMem ? hmR3PatchTprInstr : hmR3ReplaceTprInstr,
+                                (void *)(uintptr_t)pVCpu->idCpu);
+    AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * Checks if we need to reschedule due to VMM device heap changes.
+ *
+ * @returns true if a reschedule is required, otherwise false.
+ * @param   pVM         The cross context VM structure.
+ * @param   pCtx        VM execution context.
+ */
+VMMR3_INT_DECL(bool) HMR3IsRescheduleRequired(PVM pVM, PCCPUMCTX pCtx)
+{
+    /*
+     * The VMM device heap is a requirement for emulating real-mode or protected-mode without paging
+     * when the unrestricted guest execution feature is missing (VT-x only).
+     */
+    if (    pVM->hm.s.vmx.fEnabled
+        && !pVM->hm.s.vmx.fUnrestrictedGuest
+        &&  CPUMIsGuestInRealModeEx(pCtx)
+        && !PDMVmmDevHeapIsEnabled(pVM))
+        return true;
+
+    return false;
+}
+
+
+/**
+ * Noticiation callback from DBGF when interrupt breakpoints or generic debug
+ * event settings changes.
+ *
+ * DBGF will call HMR3NotifyDebugEventChangedPerCpu on each CPU afterwards, this
+ * function is just updating the VM globals.
+ *
+ * @param   pVM         The VM cross context VM structure.
+ * @thread  EMT(0)
+ */
+VMMR3_INT_DECL(void) HMR3NotifyDebugEventChanged(PVM pVM)
+{
+    /* Interrupts. */
+    bool fUseDebugLoop = pVM->dbgf.ro.cSoftIntBreakpoints > 0
+                      || pVM->dbgf.ro.cHardIntBreakpoints > 0;
+
+    /* CPU Exceptions. */
+    for (DBGFEVENTTYPE enmEvent = DBGFEVENT_XCPT_FIRST;
+         !fUseDebugLoop && enmEvent <= DBGFEVENT_XCPT_LAST;
+         enmEvent = (DBGFEVENTTYPE)(enmEvent + 1))
+        fUseDebugLoop = DBGF_IS_EVENT_ENABLED(pVM, enmEvent);
+
+    /* Common VM exits. */
+    for (DBGFEVENTTYPE enmEvent = DBGFEVENT_EXIT_FIRST;
+         !fUseDebugLoop && enmEvent <= DBGFEVENT_EXIT_LAST_COMMON;
+         enmEvent = (DBGFEVENTTYPE)(enmEvent + 1))
+        fUseDebugLoop = DBGF_IS_EVENT_ENABLED(pVM, enmEvent);
+
+    /* Vendor specific VM exits. */
+    if (HMR3IsVmxEnabled(pVM->pUVM))
+        for (DBGFEVENTTYPE enmEvent = DBGFEVENT_EXIT_VMX_FIRST;
+             !fUseDebugLoop && enmEvent <= DBGFEVENT_EXIT_VMX_LAST;
+             enmEvent = (DBGFEVENTTYPE)(enmEvent + 1))
+            fUseDebugLoop = DBGF_IS_EVENT_ENABLED(pVM, enmEvent);
+    else
+        for (DBGFEVENTTYPE enmEvent = DBGFEVENT_EXIT_SVM_FIRST;
+             !fUseDebugLoop && enmEvent <= DBGFEVENT_EXIT_SVM_LAST;
+             enmEvent = (DBGFEVENTTYPE)(enmEvent + 1))
+            fUseDebugLoop = DBGF_IS_EVENT_ENABLED(pVM, enmEvent);
+
+    /* Done. */
+    pVM->hm.s.fUseDebugLoop = fUseDebugLoop;
+}
+
+
+/**
+ * Follow up notification callback to HMR3NotifyDebugEventChanged for each CPU.
+ *
+ * HM uses this to combine the decision made by HMR3NotifyDebugEventChanged with
+ * per CPU settings.
+ *
+ * @param   pVM         The VM cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMR3_INT_DECL(void) HMR3NotifyDebugEventChangedPerCpu(PVM pVM, PVMCPU pVCpu)
+{
+    pVCpu->hm.s.fUseDebugLoop = pVCpu->hm.s.fSingleInstruction | pVM->hm.s.fUseDebugLoop;
+}
+
+
+/**
+ * Checks if we are currently using hardware acceleration.
+ *
+ * @returns true if hardware acceleration is being used, otherwise false.
+ * @param   pVCpu        The cross context virtual CPU structure.
+ */
+VMMR3_INT_DECL(bool) HMR3IsActive(PCVMCPU pVCpu)
+{
+    return pVCpu->hm.s.fActive;
+}
+
+
+/**
+ * External interface for querying whether hardware acceleration is enabled.
+ *
+ * @returns true if VT-x or AMD-V is being used, otherwise false.
+ * @param   pUVM        The user mode VM handle.
+ * @sa      HMIsEnabled, HMIsEnabledNotMacro.
+ */
+VMMR3DECL(bool) HMR3IsEnabled(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->fHMEnabled; /* Don't use the macro as the GUI may query us very very early. */
+}
+
+
+/**
+ * External interface for querying whether VT-x is being used.
+ *
+ * @returns true if VT-x is being used, otherwise false.
+ * @param   pUVM        The user mode VM handle.
+ * @sa      HMR3IsSvmEnabled, HMIsEnabled
+ */
+VMMR3DECL(bool) HMR3IsVmxEnabled(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->hm.s.vmx.fEnabled
+        && pVM->hm.s.vmx.fSupported
+        && pVM->fHMEnabled;
+}
+
+
+/**
+ * External interface for querying whether AMD-V is being used.
+ *
+ * @returns true if VT-x is being used, otherwise false.
+ * @param   pUVM        The user mode VM handle.
+ * @sa      HMR3IsVmxEnabled, HMIsEnabled
+ */
+VMMR3DECL(bool) HMR3IsSvmEnabled(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->hm.s.svm.fEnabled
+        && pVM->hm.s.svm.fSupported
+        && pVM->fHMEnabled;
+}
+
+
+/**
+ * Checks if we are currently using nested paging.
+ *
+ * @returns true if nested paging is being used, otherwise false.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(bool) HMR3IsNestedPagingActive(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->hm.s.fNestedPaging;
+}
+
+
+/**
+ * Checks if virtualized APIC registers is enabled.
+ *
+ * When enabled this feature allows the hardware to access most of the
+ * APIC registers in the virtual-APIC page without causing VM-exits. See
+ * Intel spec. 29.1.1 "Virtualized APIC Registers".
+ *
+ * @returns true if virtualized APIC registers is enabled, otherwise
+ *          false.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(bool) HMR3IsVirtApicRegsEnabled(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->hm.s.fVirtApicRegs;
+}
+
+
+/**
+ * Checks if APIC posted-interrupt processing is enabled.
+ *
+ * This returns whether we can deliver interrupts to the guest without
+ * leaving guest-context by updating APIC state from host-context.
+ *
+ * @returns true if APIC posted-interrupt processing is enabled,
+ *          otherwise false.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(bool) HMR3IsPostedIntrsEnabled(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->hm.s.fPostedIntrs;
+}
+
+
+/**
+ * Checks if we are currently using VPID in VT-x mode.
+ *
+ * @returns true if VPID is being used, otherwise false.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(bool) HMR3IsVpidActive(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->hm.s.vmx.fVpid;
+}
+
+
+/**
+ * Checks if we are currently using VT-x unrestricted execution,
+ * aka UX.
+ *
+ * @returns true if UX is being used, otherwise false.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(bool) HMR3IsUXActive(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->hm.s.vmx.fUnrestrictedGuest
+        || pVM->hm.s.svm.fSupported;
+}
+
+
+/**
+ * Checks if the VMX-preemption timer is being used.
+ *
+ * @returns true if the VMX-preemption timer is being used, otherwise false.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(bool) HMR3IsVmxPreemptionTimerUsed(PVM pVM)
+{
+    return HMIsEnabled(pVM)
+        && pVM->hm.s.vmx.fEnabled
+        && pVM->hm.s.vmx.fUsePreemptTimer;
+}
+
+
+/**
+ * Helper for HMR3CheckError to log VMCS controls to the release log.
+ *
+ * @param   idCpu           The Virtual CPU ID.
+ * @param   pVmcsInfo       The VMCS info. object.
+ */
+static void hmR3CheckErrorLogVmcsCtls(VMCPUID idCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+    LogRel(("HM: CPU[%u] PinCtls              %#RX32\n", idCpu, pVmcsInfo->u32PinCtls));
+    {
+        uint32_t const u32Val = pVmcsInfo->u32PinCtls;
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PIN_CTLS_EXT_INT_EXIT );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PIN_CTLS_NMI_EXIT     );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PIN_CTLS_VIRT_NMI     );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PIN_CTLS_PREEMPT_TIMER);
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PIN_CTLS_POSTED_INT   );
+    }
+    LogRel(("HM: CPU[%u] ProcCtls             %#RX32\n", idCpu, pVmcsInfo->u32ProcCtls));
+    {
+        uint32_t const u32Val = pVmcsInfo->u32ProcCtls;
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_INT_WINDOW_EXIT   );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_USE_TSC_OFFSETTING);
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_HLT_EXIT          );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_INVLPG_EXIT       );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_MWAIT_EXIT        );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_RDPMC_EXIT        );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_RDTSC_EXIT        );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_CR3_LOAD_EXIT     );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_CR3_STORE_EXIT    );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_CR8_LOAD_EXIT     );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_CR8_STORE_EXIT    );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_USE_TPR_SHADOW    );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_NMI_WINDOW_EXIT   );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_MOV_DR_EXIT       );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_UNCOND_IO_EXIT    );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_USE_IO_BITMAPS    );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_MONITOR_TRAP_FLAG );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_USE_MSR_BITMAPS   );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_MONITOR_EXIT      );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_PAUSE_EXIT        );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS_USE_SECONDARY_CTLS);
+    }
+    LogRel(("HM: CPU[%u] ProcCtls2            %#RX32\n", idCpu, pVmcsInfo->u32ProcCtls2));
+    {
+        uint32_t const u32Val = pVmcsInfo->u32ProcCtls2;
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_VIRT_APIC_ACCESS   );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_EPT                );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_DESC_TABLE_EXIT    );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_RDTSCP             );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_VIRT_X2APIC_MODE   );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_VPID               );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_WBINVD_EXIT        );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_UNRESTRICTED_GUEST );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_APIC_REG_VIRT      );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_VIRT_INT_DELIVERY  );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT    );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_RDRAND_EXIT        );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_INVPCID            );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_VMFUNC             );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_VMCS_SHADOWING     );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_ENCLS_EXIT         );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_RDSEED_EXIT        );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_PML                );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_EPT_VE             );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_CONCEAL_VMX_FROM_PT);
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_XSAVES_XRSTORS     );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_MODE_BASED_EPT_PERM);
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_SPPTP_EPT          );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_PT_EPT             );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_TSC_SCALING        );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_USER_WAIT_PAUSE    );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_PROC_CTLS2_ENCLV_EXIT         );
+    }
+    LogRel(("HM: CPU[%u] EntryCtls            %#RX32\n", idCpu, pVmcsInfo->u32EntryCtls));
+    {
+        uint32_t const u32Val = pVmcsInfo->u32EntryCtls;
+        HMVMX_LOGREL_FEAT(u32Val, VMX_ENTRY_CTLS_LOAD_DEBUG         );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_ENTRY_CTLS_IA32E_MODE_GUEST   );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_ENTRY_CTLS_ENTRY_TO_SMM       );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON);
+        HMVMX_LOGREL_FEAT(u32Val, VMX_ENTRY_CTLS_LOAD_PERF_MSR      );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_ENTRY_CTLS_LOAD_PAT_MSR       );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_ENTRY_CTLS_LOAD_EFER_MSR      );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_ENTRY_CTLS_LOAD_BNDCFGS_MSR   );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_ENTRY_CTLS_CONCEAL_VMX_FROM_PT);
+        HMVMX_LOGREL_FEAT(u32Val, VMX_ENTRY_CTLS_LOAD_RTIT_CTL_MSR  );
+    }
+    LogRel(("HM: CPU[%u] ExitCtls             %#RX32\n", idCpu, pVmcsInfo->u32ExitCtls));
+    {
+        uint32_t const u32Val = pVmcsInfo->u32ExitCtls;
+        HMVMX_LOGREL_FEAT(u32Val, VMX_EXIT_CTLS_SAVE_DEBUG            );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE  );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_EXIT_CTLS_LOAD_PERF_MSR         );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_EXIT_CTLS_ACK_EXT_INT           );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_EXIT_CTLS_SAVE_PAT_MSR          );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_EXIT_CTLS_LOAD_PAT_MSR          );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_EXIT_CTLS_SAVE_EFER_MSR         );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_EXIT_CTLS_LOAD_EFER_MSR         );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER    );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_EXIT_CTLS_CLEAR_BNDCFGS_MSR     );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_EXIT_CTLS_CONCEAL_VMX_FROM_PT   );
+        HMVMX_LOGREL_FEAT(u32Val, VMX_EXIT_CTLS_CLEAR_RTIT_CTL_MSR    );
+    }
+}
+
+
+/**
+ * Check fatal VT-x/AMD-V error and produce some meaningful
+ * log release message.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   iStatusCode VBox status code.
+ */
+VMMR3_INT_DECL(void) HMR3CheckError(PVM pVM, int iStatusCode)
+{
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        /** @todo r=ramshankar: Are all EMTs out of ring-0 at this point!? If not, we
+         *  might be getting inaccurate values for non-guru'ing EMTs. */
+        PVMCPU        pVCpu = pVM->apCpusR3[idCpu];
+        PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+        bool const    fNstGstVmcsActive = pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs;
+        switch (iStatusCode)
+        {
+            case VERR_VMX_INVALID_VMCS_PTR:
+            {
+                LogRel(("HM: VERR_VMX_INVALID_VMCS_PTR:\n"));
+                LogRel(("HM: CPU[%u] %s VMCS active\n", idCpu, fNstGstVmcsActive ? "Nested-guest" : "Guest"));
+                LogRel(("HM: CPU[%u] Current pointer      %#RHp vs %#RHp\n", idCpu, pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs,
+                                                                                pVmcsInfo->HCPhysVmcs));
+                LogRel(("HM: CPU[%u] Current VMCS version %#x\n", idCpu, pVCpu->hm.s.vmx.LastError.u32VmcsRev));
+                LogRel(("HM: CPU[%u] Entered Host Cpu     %u\n",  idCpu, pVCpu->hm.s.vmx.LastError.idEnteredCpu));
+                LogRel(("HM: CPU[%u] Current Host Cpu     %u\n",  idCpu, pVCpu->hm.s.vmx.LastError.idCurrentCpu));
+                break;
+            }
+
+            case VERR_VMX_UNABLE_TO_START_VM:
+            {
+                LogRel(("HM: VERR_VMX_UNABLE_TO_START_VM:\n"));
+                LogRel(("HM: CPU[%u] %s VMCS active\n", idCpu, fNstGstVmcsActive ? "Nested-guest" : "Guest"));
+                LogRel(("HM: CPU[%u] Instruction error    %#x\n", idCpu, pVCpu->hm.s.vmx.LastError.u32InstrError));
+                LogRel(("HM: CPU[%u] Exit reason          %#x\n", idCpu, pVCpu->hm.s.vmx.LastError.u32ExitReason));
+
+                if (   pVCpu->hm.s.vmx.LastError.u32InstrError == VMXINSTRERR_VMLAUNCH_NON_CLEAR_VMCS
+                    || pVCpu->hm.s.vmx.LastError.u32InstrError == VMXINSTRERR_VMRESUME_NON_LAUNCHED_VMCS)
+                {
+                    LogRel(("HM: CPU[%u] Entered Host Cpu     %u\n",  idCpu, pVCpu->hm.s.vmx.LastError.idEnteredCpu));
+                    LogRel(("HM: CPU[%u] Current Host Cpu     %u\n",  idCpu, pVCpu->hm.s.vmx.LastError.idCurrentCpu));
+                }
+                else if (pVCpu->hm.s.vmx.LastError.u32InstrError == VMXINSTRERR_VMENTRY_INVALID_CTLS)
+                {
+                    hmR3CheckErrorLogVmcsCtls(idCpu, pVmcsInfo);
+                    LogRel(("HM: CPU[%u] HCPhysMsrBitmap      %#RHp\n",  idCpu, pVmcsInfo->HCPhysMsrBitmap));
+                    LogRel(("HM: CPU[%u] HCPhysGuestMsrLoad   %#RHp\n",  idCpu, pVmcsInfo->HCPhysGuestMsrLoad));
+                    LogRel(("HM: CPU[%u] HCPhysGuestMsrStore  %#RHp\n",  idCpu, pVmcsInfo->HCPhysGuestMsrStore));
+                    LogRel(("HM: CPU[%u] HCPhysHostMsrLoad    %#RHp\n",  idCpu, pVmcsInfo->HCPhysHostMsrLoad));
+                    LogRel(("HM: CPU[%u] cEntryMsrLoad        %u\n",     idCpu, pVmcsInfo->cEntryMsrLoad));
+                    LogRel(("HM: CPU[%u] cExitMsrStore        %u\n",     idCpu, pVmcsInfo->cExitMsrStore));
+                    LogRel(("HM: CPU[%u] cExitMsrLoad         %u\n",     idCpu, pVmcsInfo->cExitMsrLoad));
+                }
+                /** @todo Log VM-entry event injection control fields
+                 *        VMX_VMCS_CTRL_ENTRY_IRQ_INFO, VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE
+                 *        and VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH from the VMCS. */
+                break;
+            }
+
+            case VERR_VMX_INVALID_GUEST_STATE:
+            {
+                LogRel(("HM: VERR_VMX_INVALID_GUEST_STATE:\n"));
+                LogRel(("HM: CPU[%u] HM error = %#RX32\n", idCpu, pVCpu->hm.s.u32HMError));
+                LogRel(("HM: CPU[%u] Guest-intr. state = %#RX32\n", idCpu, pVCpu->hm.s.vmx.LastError.u32GuestIntrState));
+                hmR3CheckErrorLogVmcsCtls(idCpu, pVmcsInfo);
+                break;
+            }
+
+            /* The guru will dump the HM error and exit history. Nothing extra to report for these errors. */
+            case VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO:
+            case VERR_VMX_INVALID_VMXON_PTR:
+            case VERR_VMX_UNEXPECTED_EXIT:
+            case VERR_VMX_INVALID_VMCS_FIELD:
+            case VERR_SVM_UNKNOWN_EXIT:
+            case VERR_SVM_UNEXPECTED_EXIT:
+            case VERR_SVM_UNEXPECTED_PATCH_TYPE:
+            case VERR_SVM_UNEXPECTED_XCPT_EXIT:
+            case VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE:
+                break;
+        }
+    }
+
+    if (iStatusCode == VERR_VMX_UNABLE_TO_START_VM)
+    {
+        LogRel(("HM: VERR_VMX_UNABLE_TO_START_VM: VM-entry allowed-1  %#RX32\n", pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1));
+        LogRel(("HM: VERR_VMX_UNABLE_TO_START_VM: VM-entry allowed-0  %#RX32\n", pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0));
+    }
+    else if (iStatusCode == VERR_VMX_INVALID_VMXON_PTR)
+        LogRel(("HM: HCPhysVmxEnableError         = %#RHp\n", pVM->hm.s.vmx.HCPhysVmxEnableError));
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * Save only data that cannot be re-loaded while entering HM ring-0 code. This
+ * is because we always save the VM state from ring-3 and thus most HM state
+ * will be re-synced dynamically at runtime and don't need to be part of the VM
+ * saved state.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) hmR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    Log(("hmR3Save:\n"));
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        Assert(!pVCpu->hm.s.Event.fPending);
+        if (pVM->cpum.ro.GuestFeatures.fSvm)
+        {
+            PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+            SSMR3PutBool(pSSM, pVmcbNstGstCache->fCacheValid);
+            SSMR3PutU16(pSSM,  pVmcbNstGstCache->u16InterceptRdCRx);
+            SSMR3PutU16(pSSM,  pVmcbNstGstCache->u16InterceptWrCRx);
+            SSMR3PutU16(pSSM,  pVmcbNstGstCache->u16InterceptRdDRx);
+            SSMR3PutU16(pSSM,  pVmcbNstGstCache->u16InterceptWrDRx);
+            SSMR3PutU16(pSSM,  pVmcbNstGstCache->u16PauseFilterThreshold);
+            SSMR3PutU16(pSSM,  pVmcbNstGstCache->u16PauseFilterCount);
+            SSMR3PutU32(pSSM,  pVmcbNstGstCache->u32InterceptXcpt);
+            SSMR3PutU64(pSSM,  pVmcbNstGstCache->u64InterceptCtrl);
+            SSMR3PutU64(pSSM,  pVmcbNstGstCache->u64TSCOffset);
+            SSMR3PutBool(pSSM, pVmcbNstGstCache->fVIntrMasking);
+            SSMR3PutBool(pSSM, pVmcbNstGstCache->fNestedPaging);
+            SSMR3PutBool(pSSM, pVmcbNstGstCache->fLbrVirt);
+        }
+    }
+
+    /* Save the guest patch data. */
+    SSMR3PutGCPtr(pSSM, pVM->hm.s.pGuestPatchMem);
+    SSMR3PutGCPtr(pSSM, pVM->hm.s.pFreeGuestPatchMem);
+    SSMR3PutU32(pSSM, pVM->hm.s.cbGuestPatchMem);
+
+    /* Store all the guest patch records too. */
+    int rc = SSMR3PutU32(pSSM, pVM->hm.s.cPatches);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    for (uint32_t i = 0; i < pVM->hm.s.cPatches; i++)
+    {
+        AssertCompileSize(HMTPRINSTR, 4);
+        PCHMTPRPATCH pPatch = &pVM->hm.s.aPatches[i];
+        SSMR3PutU32(pSSM, pPatch->Core.Key);
+        SSMR3PutMem(pSSM, pPatch->aOpcode, sizeof(pPatch->aOpcode));
+        SSMR3PutU32(pSSM, pPatch->cbOp);
+        SSMR3PutMem(pSSM, pPatch->aNewOpcode, sizeof(pPatch->aNewOpcode));
+        SSMR3PutU32(pSSM, pPatch->cbNewOp);
+        SSMR3PutU32(pSSM, (uint32_t)pPatch->enmType);
+        SSMR3PutU32(pSSM, pPatch->uSrcOperand);
+        SSMR3PutU32(pSSM, pPatch->uDstOperand);
+        SSMR3PutU32(pSSM, pPatch->pJumpTarget);
+        rc = SSMR3PutU32(pSSM, pPatch->cFaults);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) hmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    int rc;
+
+    LogFlowFunc(("uVersion=%u\n", uVersion));
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+
+    /*
+     * Validate version.
+     */
+    if (   uVersion != HM_SAVED_STATE_VERSION_SVM_NESTED_HWVIRT
+        && uVersion != HM_SAVED_STATE_VERSION_TPR_PATCHING
+        && uVersion != HM_SAVED_STATE_VERSION_NO_TPR_PATCHING
+        && uVersion != HM_SAVED_STATE_VERSION_2_0_X)
+    {
+        AssertMsgFailed(("hmR3Load: Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    /*
+     * Load per-VCPU state.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        if (uVersion >= HM_SAVED_STATE_VERSION_SVM_NESTED_HWVIRT)
+        {
+            /* Load the SVM nested hw.virt state if the VM is configured for it. */
+            if (pVM->cpum.ro.GuestFeatures.fSvm)
+            {
+                PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+                SSMR3GetBool(pSSM, &pVmcbNstGstCache->fCacheValid);
+                SSMR3GetU16(pSSM,  &pVmcbNstGstCache->u16InterceptRdCRx);
+                SSMR3GetU16(pSSM,  &pVmcbNstGstCache->u16InterceptWrCRx);
+                SSMR3GetU16(pSSM,  &pVmcbNstGstCache->u16InterceptRdDRx);
+                SSMR3GetU16(pSSM,  &pVmcbNstGstCache->u16InterceptWrDRx);
+                SSMR3GetU16(pSSM,  &pVmcbNstGstCache->u16PauseFilterThreshold);
+                SSMR3GetU16(pSSM,  &pVmcbNstGstCache->u16PauseFilterCount);
+                SSMR3GetU32(pSSM,  &pVmcbNstGstCache->u32InterceptXcpt);
+                SSMR3GetU64(pSSM,  &pVmcbNstGstCache->u64InterceptCtrl);
+                SSMR3GetU64(pSSM,  &pVmcbNstGstCache->u64TSCOffset);
+                SSMR3GetBool(pSSM, &pVmcbNstGstCache->fVIntrMasking);
+                SSMR3GetBool(pSSM, &pVmcbNstGstCache->fNestedPaging);
+                rc = SSMR3GetBool(pSSM, &pVmcbNstGstCache->fLbrVirt);
+                AssertRCReturn(rc, rc);
+            }
+        }
+        else
+        {
+            /* Pending HM event (obsolete for a long time since TPRM holds the info.) */
+            SSMR3GetU32(pSSM, &pVCpu->hm.s.Event.fPending);
+            SSMR3GetU32(pSSM, &pVCpu->hm.s.Event.u32ErrCode);
+            SSMR3GetU64(pSSM, &pVCpu->hm.s.Event.u64IntInfo);
+
+            /* VMX fWasInRealMode related data. */
+            uint32_t uDummy;
+            SSMR3GetU32(pSSM, &uDummy);
+            SSMR3GetU32(pSSM, &uDummy);
+            rc = SSMR3GetU32(pSSM, &uDummy);
+            AssertRCReturn(rc, rc);
+        }
+    }
+
+    /*
+     * Load TPR patching data.
+     */
+    if (uVersion >= HM_SAVED_STATE_VERSION_TPR_PATCHING)
+    {
+        SSMR3GetGCPtr(pSSM, &pVM->hm.s.pGuestPatchMem);
+        SSMR3GetGCPtr(pSSM, &pVM->hm.s.pFreeGuestPatchMem);
+        SSMR3GetU32(pSSM, &pVM->hm.s.cbGuestPatchMem);
+
+        /* Fetch all TPR patch records. */
+        rc = SSMR3GetU32(pSSM, &pVM->hm.s.cPatches);
+        AssertRCReturn(rc, rc);
+        for (uint32_t i = 0; i < pVM->hm.s.cPatches; i++)
+        {
+            PHMTPRPATCH pPatch = &pVM->hm.s.aPatches[i];
+            SSMR3GetU32(pSSM, &pPatch->Core.Key);
+            SSMR3GetMem(pSSM, pPatch->aOpcode, sizeof(pPatch->aOpcode));
+            SSMR3GetU32(pSSM, &pPatch->cbOp);
+            SSMR3GetMem(pSSM, pPatch->aNewOpcode, sizeof(pPatch->aNewOpcode));
+            SSMR3GetU32(pSSM, &pPatch->cbNewOp);
+            SSM_GET_ENUM32_RET(pSSM, pPatch->enmType, HMTPRINSTR);
+
+            if (pPatch->enmType == HMTPRINSTR_JUMP_REPLACEMENT)
+                pVM->hm.s.fTPRPatchingActive = true;
+            Assert(pPatch->enmType == HMTPRINSTR_JUMP_REPLACEMENT || pVM->hm.s.fTPRPatchingActive == false);
+
+            SSMR3GetU32(pSSM, &pPatch->uSrcOperand);
+            SSMR3GetU32(pSSM, &pPatch->uDstOperand);
+            SSMR3GetU32(pSSM, &pPatch->cFaults);
+            rc = SSMR3GetU32(pSSM, &pPatch->pJumpTarget);
+            AssertRCReturn(rc, rc);
+
+            LogFlow(("hmR3Load: patch %d\n", i));
+            LogFlow(("Key       = %x\n", pPatch->Core.Key));
+            LogFlow(("cbOp      = %d\n", pPatch->cbOp));
+            LogFlow(("cbNewOp   = %d\n", pPatch->cbNewOp));
+            LogFlow(("type      = %d\n", pPatch->enmType));
+            LogFlow(("srcop     = %d\n", pPatch->uSrcOperand));
+            LogFlow(("dstop     = %d\n", pPatch->uDstOperand));
+            LogFlow(("cFaults   = %d\n", pPatch->cFaults));
+            LogFlow(("target    = %x\n", pPatch->pJumpTarget));
+
+            rc = RTAvloU32Insert(&pVM->hm.s.PatchTree, &pPatch->Core);
+            AssertRCReturn(rc, rc);
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Displays HM info.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) hmR3Info(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    if (HMIsEnabled(pVM))
+    {
+        if (pVM->hm.s.vmx.fSupported)
+            pHlp->pfnPrintf(pHlp, "CPU[%u]: VT-x info:\n", pVCpu->idCpu);
+        else
+            pHlp->pfnPrintf(pHlp, "CPU[%u]: AMD-V info:\n", pVCpu->idCpu);
+        pHlp->pfnPrintf(pHlp, "  HM error           = %#x (%u)\n", pVCpu->hm.s.u32HMError, pVCpu->hm.s.u32HMError);
+        pHlp->pfnPrintf(pHlp, "  rcLastExitToR3     = %Rrc\n", pVCpu->hm.s.rcLastExitToR3);
+        if (pVM->hm.s.vmx.fSupported)
+        {
+            PCVMXVMCSINFO pVmcsInfo         = hmGetVmxActiveVmcsInfo(pVCpu);
+            bool const    fRealOnV86Active  = pVmcsInfo->RealMode.fRealOnV86Active;
+            bool const    fNstGstVmcsActive = pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs;
+
+            pHlp->pfnPrintf(pHlp, "  %s VMCS active\n", fNstGstVmcsActive ? "Nested-guest" : "Guest");
+            pHlp->pfnPrintf(pHlp, "    Real-on-v86 active = %RTbool\n", fRealOnV86Active);
+            if (fRealOnV86Active)
+            {
+                pHlp->pfnPrintf(pHlp, "      EFlags  = %#x\n", pVmcsInfo->RealMode.Eflags.u32);
+                pHlp->pfnPrintf(pHlp, "      Attr CS = %#x\n", pVmcsInfo->RealMode.AttrCS.u);
+                pHlp->pfnPrintf(pHlp, "      Attr SS = %#x\n", pVmcsInfo->RealMode.AttrSS.u);
+                pHlp->pfnPrintf(pHlp, "      Attr DS = %#x\n", pVmcsInfo->RealMode.AttrDS.u);
+                pHlp->pfnPrintf(pHlp, "      Attr ES = %#x\n", pVmcsInfo->RealMode.AttrES.u);
+                pHlp->pfnPrintf(pHlp, "      Attr FS = %#x\n", pVmcsInfo->RealMode.AttrFS.u);
+                pHlp->pfnPrintf(pHlp, "      Attr GS = %#x\n", pVmcsInfo->RealMode.AttrGS.u);
+            }
+        }
+    }
+    else
+        pHlp->pfnPrintf(pHlp, "HM is not enabled for this VM!\n");
+}
+
+
+/**
+ * Displays the HM Last-Branch-Record info. for the guest.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) hmR3InfoLbr(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    if (!HMIsEnabled(pVM))
+        pHlp->pfnPrintf(pHlp, "HM is not enabled for this VM!\n");
+
+    if (HMIsVmxActive(pVM))
+    {
+        if (pVM->hm.s.vmx.fLbr)
+        {
+            PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
+            uint32_t const cLbrStack = pVM->hm.s.vmx.idLbrFromIpMsrLast - pVM->hm.s.vmx.idLbrFromIpMsrFirst + 1;
+
+            /** @todo r=ramshankar: The index technically varies depending on the CPU, but
+             *        0xf should cover everything we support thus far. Fix if necessary
+             *        later. */
+            uint32_t const idxTopOfStack = pVmcsInfo->u64LbrTosMsr & 0xf;
+            if (idxTopOfStack > cLbrStack)
+            {
+                pHlp->pfnPrintf(pHlp, "Top-of-stack LBR MSR seems corrupt (index=%u, msr=%#RX64) expected index < %u\n",
+                                idxTopOfStack, pVmcsInfo->u64LbrTosMsr, cLbrStack);
+                return;
+            }
+
+            /*
+             * Dump the circular buffer of LBR records starting from the most recent record (contained in idxTopOfStack).
+             */
+            pHlp->pfnPrintf(pHlp, "CPU[%u]: LBRs (most-recent first)\n", pVCpu->idCpu);
+            uint32_t idxCurrent = idxTopOfStack;
+            Assert(idxTopOfStack < cLbrStack);
+            Assert(RT_ELEMENTS(pVmcsInfo->au64LbrFromIpMsr) <= cLbrStack);
+            Assert(RT_ELEMENTS(pVmcsInfo->au64LbrToIpMsr) <= cLbrStack);
+            for (;;)
+            {
+                if (pVM->hm.s.vmx.idLbrToIpMsrFirst)
+                {
+                    pHlp->pfnPrintf(pHlp, "  Branch (%2u): From IP=%#016RX64 - To IP=%#016RX64\n", idxCurrent,
+                                    pVmcsInfo->au64LbrFromIpMsr[idxCurrent], pVmcsInfo->au64LbrToIpMsr[idxCurrent]);
+                }
+                else
+                    pHlp->pfnPrintf(pHlp, "  Branch (%2u): LBR=%#RX64\n", idxCurrent, pVmcsInfo->au64LbrFromIpMsr[idxCurrent]);
+
+                idxCurrent = (idxCurrent - 1) % cLbrStack;
+                if (idxCurrent == idxTopOfStack)
+                    break;
+            }
+        }
+        else
+            pHlp->pfnPrintf(pHlp, "VM not configured to record LBRs for the guest\n");
+    }
+    else
+    {
+        Assert(HMIsSvmActive(pVM));
+        /** @todo SVM: LBRs (get them from VMCB if possible). */
+        pHlp->pfnPrintf(pHlp, "SVM LBR not implemented in VM debugger yet\n");
+    }
+}
+
+
+/**
+ * Displays the HM pending event.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) hmR3InfoEventPending(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    if (HMIsEnabled(pVM))
+    {
+        pHlp->pfnPrintf(pHlp, "CPU[%u]: HM event (fPending=%RTbool)\n", pVCpu->idCpu, pVCpu->hm.s.Event.fPending);
+        if (pVCpu->hm.s.Event.fPending)
+        {
+            pHlp->pfnPrintf(pHlp, "  u64IntInfo        = %#RX64\n", pVCpu->hm.s.Event.u64IntInfo);
+            pHlp->pfnPrintf(pHlp, "  u32ErrCode        = %#RX64\n", pVCpu->hm.s.Event.u32ErrCode);
+            pHlp->pfnPrintf(pHlp, "  cbInstr           = %u bytes\n", pVCpu->hm.s.Event.cbInstr);
+            pHlp->pfnPrintf(pHlp, "  GCPtrFaultAddress = %#RGp\n", pVCpu->hm.s.Event.GCPtrFaultAddress);
+        }
+    }
+    else
+        pHlp->pfnPrintf(pHlp, "HM is not enabled for this VM!\n");
+}
+
+
+/**
+ * Displays the SVM nested-guest VMCB cache.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) hmR3InfoSvmNstGstVmcbCache(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    bool const fSvmEnabled = HMR3IsSvmEnabled(pVM->pUVM);
+    if (   fSvmEnabled
+        && pVM->cpum.ro.GuestFeatures.fSvm)
+    {
+        PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+        pHlp->pfnPrintf(pHlp, "CPU[%u]: HM SVM nested-guest VMCB cache\n", pVCpu->idCpu);
+        pHlp->pfnPrintf(pHlp, "  fCacheValid             = %#RTbool\n", pVmcbNstGstCache->fCacheValid);
+        pHlp->pfnPrintf(pHlp, "  u16InterceptRdCRx       = %#RX16\n",   pVmcbNstGstCache->u16InterceptRdCRx);
+        pHlp->pfnPrintf(pHlp, "  u16InterceptWrCRx       = %#RX16\n",   pVmcbNstGstCache->u16InterceptWrCRx);
+        pHlp->pfnPrintf(pHlp, "  u16InterceptRdDRx       = %#RX16\n",   pVmcbNstGstCache->u16InterceptRdDRx);
+        pHlp->pfnPrintf(pHlp, "  u16InterceptWrDRx       = %#RX16\n",   pVmcbNstGstCache->u16InterceptWrDRx);
+        pHlp->pfnPrintf(pHlp, "  u16PauseFilterThreshold = %#RX16\n",   pVmcbNstGstCache->u16PauseFilterThreshold);
+        pHlp->pfnPrintf(pHlp, "  u16PauseFilterCount     = %#RX16\n",   pVmcbNstGstCache->u16PauseFilterCount);
+        pHlp->pfnPrintf(pHlp, "  u32InterceptXcpt        = %#RX32\n",   pVmcbNstGstCache->u32InterceptXcpt);
+        pHlp->pfnPrintf(pHlp, "  u64InterceptCtrl        = %#RX64\n",   pVmcbNstGstCache->u64InterceptCtrl);
+        pHlp->pfnPrintf(pHlp, "  u64TSCOffset            = %#RX64\n",   pVmcbNstGstCache->u64TSCOffset);
+        pHlp->pfnPrintf(pHlp, "  fVIntrMasking           = %RTbool\n",  pVmcbNstGstCache->fVIntrMasking);
+        pHlp->pfnPrintf(pHlp, "  fNestedPaging           = %RTbool\n",  pVmcbNstGstCache->fNestedPaging);
+        pHlp->pfnPrintf(pHlp, "  fLbrVirt                = %RTbool\n",  pVmcbNstGstCache->fLbrVirt);
+    }
+    else
+    {
+        if (!fSvmEnabled)
+            pHlp->pfnPrintf(pHlp, "HM SVM is not enabled for this VM!\n");
+        else
+            pHlp->pfnPrintf(pHlp, "SVM feature is not exposed to the guest!\n");
+    }
+}
+
diff --git a/src/VBox/VMM/VMMR3/IEMR3.cpp b/src/VBox/VMM/VMMR3/IEMR3.cpp
new file mode 100644
index 00000000..5859a26b
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/IEMR3.cpp
@@ -0,0 +1,211 @@
+/* $Id: IEMR3.cpp $ */
+/** @file
+ * IEM - Interpreted Execution Manager.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_EM
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/mm.h>
+#include "IEMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/err.h>
+
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/assert.h>
+
+static const char *iemGetTargetCpuName(uint32_t enmTargetCpu)
+{
+    switch (enmTargetCpu)
+    {
+#define CASE_RET_STR(enmValue) case enmValue: return #enmValue + (sizeof("IEMTARGETCPU_") - 1)
+        CASE_RET_STR(IEMTARGETCPU_8086);
+        CASE_RET_STR(IEMTARGETCPU_V20);
+        CASE_RET_STR(IEMTARGETCPU_186);
+        CASE_RET_STR(IEMTARGETCPU_286);
+        CASE_RET_STR(IEMTARGETCPU_386);
+        CASE_RET_STR(IEMTARGETCPU_486);
+        CASE_RET_STR(IEMTARGETCPU_PENTIUM);
+        CASE_RET_STR(IEMTARGETCPU_PPRO);
+        CASE_RET_STR(IEMTARGETCPU_CURRENT);
+#undef CASE_RET_STR
+        default: return "Unknown";
+    }
+}
+
+/**
+ * Initializes the interpreted execution manager.
+ *
+ * This must be called after CPUM as we're quering information from CPUM about
+ * the guest and host CPUs.
+ *
+ * @returns VBox status code.
+ * @param   pVM                The cross context VM structure.
+ */
+VMMR3DECL(int)      IEMR3Init(PVM pVM)
+{
+    uint64_t const uInitialTlbRevision = UINT64_C(0) - (IEMTLB_REVISION_INCR * 200U);
+    uint64_t const uInitialTlbPhysRev  = UINT64_C(0) - (IEMTLB_PHYS_REV_INCR * 100U);
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+
+        pVCpu->iem.s.CodeTlb.uTlbRevision = pVCpu->iem.s.DataTlb.uTlbRevision = uInitialTlbRevision;
+        pVCpu->iem.s.CodeTlb.uTlbPhysRev  = pVCpu->iem.s.DataTlb.uTlbPhysRev  = uInitialTlbPhysRev;
+
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cInstructions,               STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Instructions interpreted",                     "/IEM/CPU%u/cInstructions", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cLongJumps,                  STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
+                        "Number of longjmp calls",                      "/IEM/CPU%u/cLongJumps", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cPotentialExits,             STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Potential exits",                              "/IEM/CPU%u/cPotentialExits", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cRetAspectNotImplemented,    STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "VERR_IEM_ASPECT_NOT_IMPLEMENTED",              "/IEM/CPU%u/cRetAspectNotImplemented", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cRetInstrNotImplemented,     STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "VERR_IEM_INSTR_NOT_IMPLEMENTED",               "/IEM/CPU%u/cRetInstrNotImplemented", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cRetInfStatuses,             STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Informational statuses returned",              "/IEM/CPU%u/cRetInfStatuses", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cRetErrStatuses,             STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Error statuses returned",                      "/IEM/CPU%u/cRetErrStatuses", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cbWritten,                   STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
+                        "Approx bytes written",                         "/IEM/CPU%u/cbWritten", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cPendingCommit,              STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
+                        "Times RC/R0 had to postpone instruction committing to ring-3", "/IEM/CPU%u/cPendingCommit", idCpu);
+
+#ifdef VBOX_WITH_STATISTICS
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbHits,            STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Code TLB hits",                            "/IEM/CPU%u/CodeTlb-Hits", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbHits,            STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Data TLB hits",                            "/IEM/CPU%u/DataTlb-Hits", idCpu);
+#endif
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbMisses,          STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Code TLB misses",                          "/IEM/CPU%u/CodeTlb-Misses", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.uTlbRevision,        STAMTYPE_X64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Code TLB revision",                        "/IEM/CPU%u/CodeTlb-Revision", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.CodeTlb.uTlbPhysRev, STAMTYPE_X64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Code TLB physical revision",               "/IEM/CPU%u/CodeTlb-PhysRev", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbSlowReadPath,    STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Code TLB slow read path",                  "/IEM/CPU%u/CodeTlb-SlowReads", idCpu);
+
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbMisses,          STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Data TLB misses",                          "/IEM/CPU%u/DataTlb-Misses", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.uTlbRevision,        STAMTYPE_X64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Data TLB revision",                        "/IEM/CPU%u/DataTlb-Revision", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.DataTlb.uTlbPhysRev, STAMTYPE_X64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Data TLB physical revision",               "/IEM/CPU%u/DataTlb-PhysRev", idCpu);
+
+#if defined(VBOX_WITH_STATISTICS) && !defined(DOXYGEN_RUNNING)
+        /* Allocate instruction statistics and register them. */
+        pVCpu->iem.s.pStatsR3 = (PIEMINSTRSTATS)MMR3HeapAllocZ(pVM, MM_TAG_IEM, sizeof(IEMINSTRSTATS));
+        AssertLogRelReturn(pVCpu->iem.s.pStatsR3, VERR_NO_MEMORY);
+        int rc = MMHyperAlloc(pVM, sizeof(IEMINSTRSTATS), sizeof(uint64_t), MM_TAG_IEM, (void **)&pVCpu->iem.s.pStatsCCR3);
+        AssertLogRelRCReturn(rc, rc);
+        pVCpu->iem.s.pStatsR0 = MMHyperR3ToR0(pVM, pVCpu->iem.s.pStatsCCR3);
+# define IEM_DO_INSTR_STAT(a_Name, a_szDesc) \
+            STAMR3RegisterF(pVM, &pVCpu->iem.s.pStatsCCR3->a_Name, STAMTYPE_U32_RESET, STAMVISIBILITY_USED, \
+                            STAMUNIT_COUNT, a_szDesc, "/IEM/CPU%u/instr-RZ/" #a_Name, idCpu); \
+            STAMR3RegisterF(pVM, &pVCpu->iem.s.pStatsR3->a_Name, STAMTYPE_U32_RESET, STAMVISIBILITY_USED, \
+                            STAMUNIT_COUNT, a_szDesc, "/IEM/CPU%u/instr-R3/" #a_Name, idCpu);
+# include "IEMInstructionStatisticsTmpl.h"
+# undef IEM_DO_INSTR_STAT
+#endif
+
+        /*
+         * Host and guest CPU information.
+         */
+        if (idCpu == 0)
+        {
+            pVCpu->iem.s.enmCpuVendor             = CPUMGetGuestCpuVendor(pVM);
+            pVCpu->iem.s.enmHostCpuVendor         = CPUMGetHostCpuVendor(pVM);
+#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
+            switch (pVM->cpum.ro.GuestFeatures.enmMicroarch)
+            {
+                case kCpumMicroarch_Intel_8086:     pVCpu->iem.s.uTargetCpu = IEMTARGETCPU_8086; break;
+                case kCpumMicroarch_Intel_80186:    pVCpu->iem.s.uTargetCpu = IEMTARGETCPU_186; break;
+                case kCpumMicroarch_Intel_80286:    pVCpu->iem.s.uTargetCpu = IEMTARGETCPU_286; break;
+                case kCpumMicroarch_Intel_80386:    pVCpu->iem.s.uTargetCpu = IEMTARGETCPU_386; break;
+                case kCpumMicroarch_Intel_80486:    pVCpu->iem.s.uTargetCpu = IEMTARGETCPU_486; break;
+                case kCpumMicroarch_Intel_P5:       pVCpu->iem.s.uTargetCpu = IEMTARGETCPU_PENTIUM; break;
+                case kCpumMicroarch_Intel_P6:       pVCpu->iem.s.uTargetCpu = IEMTARGETCPU_PPRO; break;
+                case kCpumMicroarch_NEC_V20:        pVCpu->iem.s.uTargetCpu = IEMTARGETCPU_V20; break;
+                case kCpumMicroarch_NEC_V30:        pVCpu->iem.s.uTargetCpu = IEMTARGETCPU_V20; break;
+                default:                            pVCpu->iem.s.uTargetCpu = IEMTARGETCPU_CURRENT; break;
+            }
+            LogRel(("IEM: TargetCpu=%s, Microarch=%s\n", iemGetTargetCpuName(pVCpu->iem.s.uTargetCpu), CPUMR3MicroarchName(pVM->cpum.ro.GuestFeatures.enmMicroarch)));
+#endif
+        }
+        else
+        {
+            pVCpu->iem.s.enmCpuVendor             = pVM->apCpusR3[0]->iem.s.enmCpuVendor;
+            pVCpu->iem.s.enmHostCpuVendor         = pVM->apCpusR3[0]->iem.s.enmHostCpuVendor;
+#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
+            pVCpu->iem.s.uTargetCpu               = pVM->apCpusR3[0]->iem.s.uTargetCpu;
+#endif
+        }
+
+        /*
+         * Mark all buffers free.
+         */
+        uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
+        while (iMemMap-- > 0)
+            pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+    }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * Register the per-VM VMX APIC-access page handler type.
+     */
+    if (pVM->cpum.ro.GuestFeatures.fVmx)
+    {
+        PVMCPU pVCpu0 = pVM->apCpusR3[0];
+        int rc = PGMR3HandlerPhysicalTypeRegister(pVM, PGMPHYSHANDLERKIND_ALL, iemVmxApicAccessPageHandler,
+                                                  NULL /* pszModR0 */,
+                                                  "iemVmxApicAccessPageHandler", NULL /* pszPfHandlerR0 */,
+                                                  NULL /* pszModRC */,
+                                                  NULL /* pszHandlerRC */, NULL /* pszPfHandlerRC */,
+                                                  "VMX APIC-access page", &pVCpu0->iem.s.hVmxApicAccessPage);
+        AssertLogRelRCReturn(rc, rc);
+    }
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+VMMR3DECL(int)      IEMR3Term(PVM pVM)
+{
+    NOREF(pVM);
+#if defined(VBOX_WITH_STATISTICS) && !defined(DOXYGEN_RUNNING)
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        MMR3HeapFree(pVCpu->iem.s.pStatsR3);
+        pVCpu->iem.s.pStatsR3 = NULL;
+    }
+#endif
+    return VINF_SUCCESS;
+}
+
+
+VMMR3DECL(void)     IEMR3Relocate(PVM pVM)
+{
+    RT_NOREF(pVM);
+}
+
diff --git a/src/VBox/VMM/VMMR3/IOM.cpp b/src/VBox/VMM/VMMR3/IOM.cpp
new file mode 100644
index 00000000..6f029e71
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/IOM.cpp
@@ -0,0 +1,460 @@
+/* $Id: IOM.cpp $ */
+/** @file
+ * IOM - Input / Output Monitor.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_iom        IOM - The Input / Output Monitor
+ *
+ * The input/output monitor will handle I/O exceptions routing them to the
+ * appropriate device. It implements an API to register and deregister virtual
+ * I/0 port handlers and memory mapped I/O handlers. A handler is PDM devices
+ * and a set of callback functions.
+ *
+ * @see grp_iom
+ *
+ *
+ * @section sec_iom_rawmode     Raw-Mode
+ *
+ * In raw-mode I/O port access is trapped (\#GP(0)) by ensuring that the actual
+ * IOPL is 0 regardless of what the guest IOPL is. The \#GP handler use the
+ * disassembler (DIS) to figure which instruction caused it (there are a number
+ * of instructions in addition to the I/O ones) and if it's an I/O port access
+ * it will hand it to IOMRCIOPortHandler (via EMInterpretPortIO).
+ * IOMRCIOPortHandler will lookup the port in the AVL tree of registered
+ * handlers. If found, the handler will be called otherwise default action is
+ * taken. (Default action is to write into the void and read all set bits.)
+ *
+ * Memory Mapped I/O (MMIO) is implemented as a slightly special case of PGM
+ * access handlers. An MMIO range is registered with IOM which then registers it
+ * with the PGM access handler sub-system. The access handler catches all
+ * access and will be called in the context of a \#PF handler. In RC and R0 this
+ * handler is iomMmioPfHandler while in ring-3 it's iomR3MmioHandler (although
+ * in ring-3 there can be alternative ways). iomMmioPfHandler will attempt to
+ * emulate the instruction that is doing the access and pass the corresponding
+ * reads / writes to the device.
+ *
+ * Emulating I/O port access is less complex and should be slightly faster than
+ * emulating MMIO, so in most cases we should encourage the OS to use port I/O.
+ * Devices which are frequently accessed should register GC handlers to speed up
+ * execution.
+ *
+ *
+ * @section sec_iom_hm     Hardware Assisted Virtualization Mode
+ *
+ * When running in hardware assisted virtualization mode we'll be doing much the
+ * same things as in raw-mode. The main difference is that we're running in the
+ * host ring-0 context and that we don't get faults (\#GP(0) and \#PG) but
+ * exits.
+ *
+ *
+ * @section sec_iom_rem         Recompiled Execution Mode
+ *
+ * When running in the recompiler things are different. I/O port access is
+ * handled by calling IOMIOPortRead and IOMIOPortWrite directly. While MMIO can
+ * be handled in one of two ways. The normal way is that we have a registered a
+ * special RAM range with the recompiler and in the three callbacks (for byte,
+ * word and dword access) we call IOMMMIORead and IOMMMIOWrite directly. The
+ * alternative ways that the physical memory access which goes via PGM will take
+ * care of it by calling iomR3MmioHandler via the PGM access handler machinery
+ * - this shouldn't happen but it is an alternative...
+ *
+ *
+ * @section sec_iom_other       Other Accesses
+ *
+ * I/O ports aren't really exposed in any other way, unless you count the
+ * instruction interpreter in EM, but that's just what we're doing in the
+ * raw-mode \#GP(0) case really. Now, it's possible to call IOMIOPortRead and
+ * IOMIOPortWrite directly to talk to a device, but this is really bad behavior
+ * and should only be done as temporary hacks (the PC BIOS device used to setup
+ * the CMOS this way back in the dark ages).
+ *
+ * MMIO has similar direct routes as the I/O ports and these shouldn't be used
+ * for the same reasons and with the same restrictions. OTOH since MMIO is
+ * mapped into the physical memory address space, it can be accessed in a number
+ * of ways thru PGM.
+ *
+ *
+ * @section sec_iom_logging     Logging Levels
+ *
+ * Following assignments:
+ *      - Level 5 is used for defering I/O port and MMIO writes to ring-3.
+ *
+ */
+
+/** @todo MMIO - simplifying the device end.
+ * - Add a return status for doing DBGFSTOP on access where there are no known
+ *   registers.
+ * -
+ *
+ *   */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_IOM
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pdmdev.h>
+#include "IOMInternal.h"
+#include <VBox/vmm/vm.h>
+
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/string.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+
+
+
+/**
+ * Initializes the IOM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) IOMR3Init(PVM pVM)
+{
+    LogFlow(("IOMR3Init:\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompileMemberAlignment(VM, iom.s, 32);
+    AssertCompile(sizeof(pVM->iom.s) <= sizeof(pVM->iom.padding));
+    AssertCompileMemberAlignment(IOM, CritSect, sizeof(uintptr_t));
+
+    /*
+     * Initialize the REM critical section.
+     */
+#ifdef IOM_WITH_CRIT_SECT_RW
+    int rc = PDMR3CritSectRwInit(pVM, &pVM->iom.s.CritSect, RT_SRC_POS, "IOM Lock");
+#else
+    int rc = PDMR3CritSectInit(pVM, &pVM->iom.s.CritSect, RT_SRC_POS, "IOM Lock");
+#endif
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Register the MMIO access handler type.
+     */
+    rc = PGMR3HandlerPhysicalTypeRegister(pVM, PGMPHYSHANDLERKIND_MMIO,
+                                          iomMmioHandlerNew,
+                                          NULL, "iomMmioHandlerNew", "iomMmioPfHandlerNew",
+                                          NULL, "iomMmioHandlerNew", "iomMmioPfHandlerNew",
+                                          "MMIO New", &pVM->iom.s.hNewMmioHandlerType);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Info.
+     */
+    DBGFR3InfoRegisterInternal(pVM, "ioport", "Dumps all IOPort ranges. No arguments.", &iomR3IoPortInfo);
+    DBGFR3InfoRegisterInternal(pVM, "mmio", "Dumps all MMIO ranges. No arguments.", &iomR3MmioInfo);
+
+    /*
+     * Statistics (names are somewhat contorted to make the registration
+     * sub-trees appear at the end of each group).
+     */
+    STAM_REG(pVM, &pVM->iom.s.StatIoPortCommits,    STAMTYPE_COUNTER, "/IOM/IoPortCommits",     STAMUNIT_OCCURENCES, "Number of ring-3 I/O port commits.");
+    STAM_REG(pVM, &pVM->iom.s.StatIoPortIn,         STAMTYPE_PROFILE, "/IOM/IoPortIN",          STAMUNIT_OCCURENCES, "Number of IN instructions (attempts)");
+    STAM_REG(pVM, &pVM->iom.s.StatIoPortInS,        STAMTYPE_PROFILE, "/IOM/IoPortINS",         STAMUNIT_OCCURENCES, "Number of INS instructions (attempts)");
+    STAM_REG(pVM, &pVM->iom.s.StatIoPortOutS,       STAMTYPE_PROFILE, "/IOM/IoPortOUT",         STAMUNIT_OCCURENCES, "Number of OUT instructions (attempts)");
+    STAM_REG(pVM, &pVM->iom.s.StatIoPortOutS,       STAMTYPE_PROFILE, "/IOM/IoPortOUTS",        STAMUNIT_OCCURENCES, "Number of OUTS instructions (attempts)");
+
+    STAM_REG(pVM, &pVM->iom.s.StatMmioHandlerR3,    STAMTYPE_COUNTER, "/IOM/MmioHandlerR3",     STAMUNIT_OCCURENCES, "Number of calls to iomMmioHandlerNew from ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatMmioHandlerR0,    STAMTYPE_COUNTER, "/IOM/MmioHandlerR0",     STAMUNIT_OCCURENCES, "Number of calls to iomMmioHandlerNew from ring-0.");
+    STAM_REG(pVM, &pVM->iom.s.StatMmioReadsR0ToR3,  STAMTYPE_COUNTER, "/IOM/MmioR0ToR3Reads",   STAMUNIT_OCCURENCES, "Number of reads deferred to ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatMmioWritesR0ToR3, STAMTYPE_COUNTER, "/IOM/MmioR0ToR3Writes",  STAMUNIT_OCCURENCES, "Number of writes deferred to ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatMmioCommitsR0ToR3,STAMTYPE_COUNTER, "/IOM/MmioR0ToR3Commits", STAMUNIT_OCCURENCES, "Number of commits deferred to ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatMmioPfHandler,    STAMTYPE_PROFILE, "/IOM/MmioPfHandler",     STAMUNIT_OCCURENCES, "Number of calls to iomMmioPfHandlerNew.");
+    STAM_REG(pVM, &pVM->iom.s.StatMmioPhysHandler,  STAMTYPE_PROFILE, "/IOM/MmioPhysHandler",   STAMUNIT_OCCURENCES, "Number of calls to IOMR0MmioPhysHandler.");
+    STAM_REG(pVM, &pVM->iom.s.StatMmioCommitsDirect,STAMTYPE_COUNTER, "/IOM/MmioCommitsDirect", STAMUNIT_OCCURENCES, "Number of ring-3 MMIO commits direct to handler via handle hint.");
+    STAM_REG(pVM, &pVM->iom.s.StatMmioCommitsPgm,   STAMTYPE_COUNTER, "/IOM/MmioCommitsPgm",    STAMUNIT_OCCURENCES, "Number of ring-3 MMIO commits via PGM.");
+    STAM_REL_REG(pVM, &pVM->iom.s.StatMmioStaleMappings,   STAMTYPE_PROFILE, "/IOM/MmioMappingsStale",              STAMUNIT_TICKS_PER_CALL, "Number of times iomMmioHandlerNew got a call for a remapped range at the old mapping.");
+    STAM_REG(pVM, &pVM->iom.s.StatMmioDevLockContentionR0, STAMTYPE_COUNTER, "/IOM/MmioDevLockContentionR0",        STAMUNIT_OCCURENCES,     "Number of device lock contention force return to ring-3.");
+
+    LogFlow(("IOMR3Init: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Called when a VM initialization stage is completed.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   enmWhat         The initialization state that was completed.
+ */
+VMMR3_INT_DECL(int) IOMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+#ifdef VBOX_WITH_STATISTICS
+    if (enmWhat == VMINITCOMPLETED_RING0)
+    {
+        /*
+         * Synchronize the ring-3 I/O port and MMIO statistics indices into the
+         * ring-0 tables to simplify ring-0 code.  This also make sure that any
+         * later calls to grow the statistics tables will fail.
+         */
+        int rc = VMMR3CallR0Emt(pVM, pVM->apCpusR3[0], VMMR0_DO_IOM_SYNC_STATS_INDICES, 0, NULL);
+        AssertLogRelRCReturn(rc, rc);
+
+        /*
+         * Register I/O port and MMIO stats now that we're done registering MMIO
+         * regions and won't grow the table again.
+         */
+        for (uint32_t i = 0; i < pVM->iom.s.cIoPortRegs; i++)
+        {
+            PIOMIOPORTENTRYR3 pRegEntry = &pVM->iom.s.paIoPortRegs[i];
+            if (   pRegEntry->fMapped
+                && pRegEntry->idxStats != UINT16_MAX)
+                iomR3IoPortRegStats(pVM, pRegEntry);
+        }
+
+        for (uint32_t i = 0; i < pVM->iom.s.cMmioRegs; i++)
+        {
+            PIOMMMIOENTRYR3 pRegEntry = &pVM->iom.s.paMmioRegs[i];
+            if (   pRegEntry->fMapped
+                && pRegEntry->idxStats != UINT16_MAX)
+                iomR3MmioRegStats(pVM, pRegEntry);
+        }
+    }
+#else
+    RT_NOREF(pVM, enmWhat);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) IOMR3Reset(PVM pVM)
+{
+    RT_NOREF(pVM);
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * The IOM will update the addresses used by the switcher.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3_INT_DECL(void) IOMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    RT_NOREF(pVM, offDelta);
+}
+
+/**
+ * Terminates the IOM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) IOMR3Term(PVM pVM)
+{
+    /*
+     * IOM is not owning anything but automatically freed resources,
+     * so there's nothing to do here.
+     */
+    NOREF(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Handles the unlikely and probably fatal merge cases.
+ *
+ * @returns Merged status code.
+ * @param   rcStrict        Current EM status code.
+ * @param   rcStrictCommit  The IOM I/O or MMIO write commit status to merge
+ *                          with @a rcStrict.
+ * @param   rcIom           For logging purposes only.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.  For logging purposes.
+ */
+DECL_NO_INLINE(static, VBOXSTRICTRC) iomR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit,
+                                                          int rcIom, PVMCPU pVCpu)
+{
+    if (RT_FAILURE_NP(rcStrict))
+        return rcStrict;
+
+    if (RT_FAILURE_NP(rcStrictCommit))
+        return rcStrictCommit;
+
+    if (rcStrict == rcStrictCommit)
+        return rcStrictCommit;
+
+    AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc IOPort={%#06x<-%#xx/%u} MMIO={%RGp<-%.*Rhxs} (rcIom=%Rrc)\n",
+                           VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict),
+                           pVCpu->iom.s.PendingIOPortWrite.IOPort,
+                           pVCpu->iom.s.PendingIOPortWrite.u32Value, pVCpu->iom.s.PendingIOPortWrite.cbValue,
+                           pVCpu->iom.s.PendingMmioWrite.GCPhys,
+                           pVCpu->iom.s.PendingMmioWrite.cbValue, &pVCpu->iom.s.PendingMmioWrite.abValue[0], rcIom));
+    return VERR_IOM_FF_STATUS_IPE;
+}
+
+
+/**
+ * Helper for IOMR3ProcessForceFlag.
+ *
+ * @returns Merged status code.
+ * @param   rcStrict        Current EM status code.
+ * @param   rcStrictCommit  The IOM I/O or MMIO write commit status to merge
+ *                          with @a rcStrict.
+ * @param   rcIom           Either VINF_IOM_R3_IOPORT_COMMIT_WRITE or
+ *                          VINF_IOM_R3_MMIO_COMMIT_WRITE.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.
+ */
+DECLINLINE(VBOXSTRICTRC) iomR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, int rcIom, PVMCPU pVCpu)
+{
+    /* Simple. */
+    if (RT_LIKELY(rcStrict == rcIom || rcStrict == VINF_EM_RAW_TO_R3 || rcStrict == VINF_SUCCESS))
+        return rcStrictCommit;
+
+    if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS))
+        return rcStrict;
+
+    /* EM scheduling status codes. */
+    if (RT_LIKELY(   rcStrict >= VINF_EM_FIRST
+                  && rcStrict <= VINF_EM_LAST))
+    {
+        if (RT_LIKELY(   rcStrictCommit >= VINF_EM_FIRST
+                      && rcStrictCommit <= VINF_EM_LAST))
+            return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit;
+    }
+
+    /* Unlikely */
+    return iomR3MergeStatusSlow(rcStrict, rcStrictCommit, rcIom, pVCpu);
+}
+
+
+/**
+ * Called by force-flag handling code when VMCPU_FF_IOM is set.
+ *
+ * @returns Merge between @a rcStrict and what the commit operation returned.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   rcStrict    The status code returned by ring-0 or raw-mode.
+ * @thread  EMT(pVCpu)
+ *
+ * @remarks The VMCPU_FF_IOM flag is handled before the status codes by EM, so
+ *          we're very likely to see @a rcStrict set to
+ *          VINF_IOM_R3_IOPORT_COMMIT_WRITE and VINF_IOM_R3_MMIO_COMMIT_WRITE
+ *          here.
+ */
+VMMR3_INT_DECL(VBOXSTRICTRC) IOMR3ProcessForceFlag(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rcStrict)
+{
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IOM);
+    Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue || pVCpu->iom.s.PendingMmioWrite.cbValue);
+
+    if (pVCpu->iom.s.PendingIOPortWrite.cbValue)
+    {
+        Log5(("IOM: Dispatching pending I/O port write: %#x LB %u -> %RTiop\n", pVCpu->iom.s.PendingIOPortWrite.u32Value,
+              pVCpu->iom.s.PendingIOPortWrite.cbValue, pVCpu->iom.s.PendingIOPortWrite.IOPort));
+        STAM_COUNTER_INC(&pVM->iom.s.StatIoPortCommits);
+        VBOXSTRICTRC rcStrictCommit = IOMIOPortWrite(pVM, pVCpu, pVCpu->iom.s.PendingIOPortWrite.IOPort,
+                                                     pVCpu->iom.s.PendingIOPortWrite.u32Value,
+                                                     pVCpu->iom.s.PendingIOPortWrite.cbValue);
+        pVCpu->iom.s.PendingIOPortWrite.cbValue = 0;
+        rcStrict = iomR3MergeStatus(rcStrict, rcStrictCommit, VINF_IOM_R3_IOPORT_COMMIT_WRITE, pVCpu);
+    }
+
+
+    if (pVCpu->iom.s.PendingMmioWrite.cbValue)
+    {
+        Log5(("IOM: Dispatching pending MMIO write: %RGp LB %#x\n",
+              pVCpu->iom.s.PendingMmioWrite.GCPhys, pVCpu->iom.s.PendingMmioWrite.cbValue));
+
+        /* Use new MMIO handle hint and bypass PGM if it still looks right. */
+        size_t idxMmioRegionHint = pVCpu->iom.s.PendingMmioWrite.idxMmioRegionHint;
+        if (idxMmioRegionHint < pVM->iom.s.cMmioRegs)
+        {
+            PIOMMMIOENTRYR3 pRegEntry    = &pVM->iom.s.paMmioRegs[idxMmioRegionHint];
+            RTGCPHYS const GCPhysMapping = pRegEntry->GCPhysMapping;
+            RTGCPHYS const offRegion     = pVCpu->iom.s.PendingMmioWrite.GCPhys - GCPhysMapping;
+            if (offRegion < pRegEntry->cbRegion && GCPhysMapping != NIL_RTGCPHYS)
+            {
+                STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsDirect);
+                VBOXSTRICTRC rcStrictCommit = iomR3MmioCommitWorker(pVM, pVCpu, pRegEntry, offRegion);
+                pVCpu->iom.s.PendingMmioWrite.cbValue = 0;
+                return iomR3MergeStatus(rcStrict, rcStrictCommit, VINF_IOM_R3_MMIO_COMMIT_WRITE, pVCpu);
+            }
+        }
+
+        /* Fall back on PGM. */
+        STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsPgm);
+        VBOXSTRICTRC rcStrictCommit = PGMPhysWrite(pVM, pVCpu->iom.s.PendingMmioWrite.GCPhys,
+                                                   pVCpu->iom.s.PendingMmioWrite.abValue, pVCpu->iom.s.PendingMmioWrite.cbValue,
+                                                   PGMACCESSORIGIN_IOM);
+        pVCpu->iom.s.PendingMmioWrite.cbValue = 0;
+        rcStrict = iomR3MergeStatus(rcStrict, rcStrictCommit, VINF_IOM_R3_MMIO_COMMIT_WRITE, pVCpu);
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Notification from DBGF that the number of active I/O port or MMIO
+ * breakpoints has change.
+ *
+ * For performance reasons, IOM will only call DBGF before doing I/O and MMIO
+ * accesses where there are armed breakpoints.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   fPortIo     True if there are armed I/O port breakpoints.
+ * @param   fMmio       True if there are armed MMIO breakpoints.
+ */
+VMMR3_INT_DECL(void) IOMR3NotifyBreakpointCountChange(PVM pVM, bool fPortIo, bool fMmio)
+{
+    /** @todo I/O breakpoints. */
+    RT_NOREF3(pVM, fPortIo, fMmio);
+}
+
+
+/**
+ * Notification from DBGF that an event has been enabled or disabled.
+ *
+ * For performance reasons, IOM may cache the state of events it implements.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   enmEvent    The event.
+ * @param   fEnabled    The new state.
+ */
+VMMR3_INT_DECL(void) IOMR3NotifyDebugEventChange(PVM pVM, DBGFEVENT enmEvent, bool fEnabled)
+{
+    /** @todo IOM debug events. */
+    RT_NOREF3(pVM, enmEvent, fEnabled);
+}
+
diff --git a/src/VBox/VMM/VMMR3/IOMR3IoPort.cpp b/src/VBox/VMM/VMMR3/IOMR3IoPort.cpp
new file mode 100644
index 00000000..f1be2a33
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/IOMR3IoPort.cpp
@@ -0,0 +1,629 @@
+/* $Id: IOMR3IoPort.cpp $ */
+/** @file
+ * IOM - Input / Output Monitor, I/O port related APIs.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_IOM_IOPORT
+#include <VBox/vmm/iom.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pdmdev.h>
+#include "IOMInternal.h"
+#include <VBox/vmm/vm.h>
+
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+
+#include "IOMInline.h"
+
+
+#ifdef VBOX_WITH_STATISTICS
+
+/**
+ * Register statistics for an I/O port entry.
+ */
+void iomR3IoPortRegStats(PVM pVM, PIOMIOPORTENTRYR3 pRegEntry)
+{
+    bool const           fDoRZ      = pRegEntry->fRing0 || pRegEntry->fRawMode;
+    PIOMIOPORTSTATSENTRY pStats     = &pVM->iom.s.paIoPortStats[pRegEntry->idxStats];
+    PCIOMIOPORTDESC      pExtDesc   = pRegEntry->paExtDescs;
+    unsigned             uPort      = pRegEntry->uPort;
+    unsigned const       uFirstPort = uPort;
+    unsigned const       uEndPort   = uPort + pRegEntry->cPorts;
+
+    /* Register a dummy statistics for the prefix. */
+    char                 szName[80];
+    size_t cchPrefix;
+    if (uFirstPort < uEndPort - 1)
+        cchPrefix = RTStrPrintf(szName, sizeof(szName), "/IOM/IoPorts/%04x-%04x", uFirstPort, uEndPort - 1);
+    else
+        cchPrefix = RTStrPrintf(szName, sizeof(szName), "/IOM/IoPorts/%04x", uPort);
+    const char *pszDesc     = pRegEntry->pszDesc;
+    char       *pszFreeDesc = NULL;
+    if (pRegEntry->pDevIns && pRegEntry->pDevIns->iInstance > 0 && pszDesc)
+        pszDesc = pszFreeDesc = RTStrAPrintf2("%u / %s", pRegEntry->pDevIns->iInstance, pszDesc);
+    int rc = STAMR3Register(pVM, &pStats->Total, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, szName,
+                            STAMUNIT_NONE, pRegEntry->pszDesc);
+    AssertRC(rc);
+    RTStrFree(pszFreeDesc);
+
+    /* Register stats for each port under it */
+    do
+    {
+        size_t cchBaseNm;
+        if (uFirstPort < uEndPort - 1)
+            cchBaseNm = cchPrefix + RTStrPrintf(&szName[cchPrefix], sizeof(szName) - cchPrefix, "/%04x-", uPort);
+        else
+        {
+            szName[cchPrefix] = '/';
+            cchBaseNm = cchPrefix + 1;
+        }
+
+# define SET_NM_SUFFIX(a_sz) memcpy(&szName[cchBaseNm], a_sz, sizeof(a_sz));
+        const char * const pszInDesc  = pExtDesc ? pExtDesc->pszIn  : NULL;
+        const char * const pszOutDesc = pExtDesc ? pExtDesc->pszOut : NULL;
+
+        /* register the statistics counters. */
+        SET_NM_SUFFIX("In-R3");
+        rc = STAMR3Register(pVM, &pStats->InR3,      STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, pszInDesc); AssertRC(rc);
+        SET_NM_SUFFIX("Out-R3");
+        rc = STAMR3Register(pVM, &pStats->OutR3,     STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, pszOutDesc); AssertRC(rc);
+        if (fDoRZ)
+        {
+            SET_NM_SUFFIX("In-RZ");
+            rc = STAMR3Register(pVM, &pStats->InRZ,      STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, pszInDesc); AssertRC(rc);
+            SET_NM_SUFFIX("Out-RZ");
+            rc = STAMR3Register(pVM, &pStats->OutRZ,     STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, pszOutDesc); AssertRC(rc);
+            SET_NM_SUFFIX("In-RZtoR3");
+            rc = STAMR3Register(pVM, &pStats->InRZToR3,  STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, NULL); AssertRC(rc);
+            SET_NM_SUFFIX("Out-RZtoR3");
+            rc = STAMR3Register(pVM, &pStats->OutRZToR3, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, NULL); AssertRC(rc);
+        }
+
+        /* Profiling */
+        SET_NM_SUFFIX("In-R3-Prof");
+        rc = STAMR3Register(pVM, &pStats->ProfInR3,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, pszInDesc); AssertRC(rc);
+        SET_NM_SUFFIX("Out-R3-Prof");
+        rc = STAMR3Register(pVM, &pStats->ProfOutR3, STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, pszOutDesc); AssertRC(rc);
+        if (fDoRZ)
+        {
+            SET_NM_SUFFIX("In-RZ-Prof");
+            rc = STAMR3Register(pVM, &pStats->ProfInRZ,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, pszInDesc); AssertRC(rc);
+            SET_NM_SUFFIX("Out-RZ-Prof");
+            rc = STAMR3Register(pVM, &pStats->ProfOutRZ, STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, pszOutDesc); AssertRC(rc);
+        }
+
+        pStats++;
+        uPort++;
+        if (pExtDesc)
+            pExtDesc = pszInDesc || pszOutDesc ? pExtDesc + 1 : NULL;
+    } while (uPort < uEndPort);
+}
+
+
+/**
+ * Deregister statistics for an I/O port entry.
+ */
+static void iomR3IoPortDeregStats(PVM pVM, PIOMIOPORTENTRYR3 pRegEntry, unsigned uPort)
+{
+    char   szPrefix[80];
+    size_t cchPrefix;
+    if (pRegEntry->cPorts > 1)
+        cchPrefix = RTStrPrintf(szPrefix, sizeof(szPrefix), "/IOM/IoPorts/%04x-%04x", uPort, uPort + pRegEntry->cPorts - 1);
+    else
+        cchPrefix = RTStrPrintf(szPrefix, sizeof(szPrefix), "/IOM/IoPorts/%04x", uPort);
+    STAMR3DeregisterByPrefix(pVM->pUVM, szPrefix);
+}
+
+#endif /* VBOX_WITH_STATISTICS */
+
+
+/**
+ * @callback_method_impl{FNIOMIOPORTNEWIN,
+ *      Dummy Port I/O Handler for IN operations.}
+ */
+static DECLCALLBACK(VBOXSTRICTRC)
+iomR3IOPortDummyNewIn(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, uint32_t *pu32, unsigned cb)
+{
+    NOREF(pDevIns); NOREF(pvUser); NOREF(Port);
+    switch (cb)
+    {
+        case 1: *pu32 = 0xff; break;
+        case 2: *pu32 = 0xffff; break;
+        case 4: *pu32 = UINT32_C(0xffffffff); break;
+        default:
+            AssertReleaseMsgFailed(("cb=%d\n", cb));
+            return VERR_IOM_IOPORT_IPE_2;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNIOMIOPORTNEWINSTRING,
+ *      Dummy Port I/O Handler for string IN operations.}
+ */
+static DECLCALLBACK(VBOXSTRICTRC)
+iomR3IOPortDummyNewInStr(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, uint8_t *pbDst, uint32_t *pcTransfer, unsigned cb)
+{
+    NOREF(pDevIns); NOREF(pvUser); NOREF(Port); NOREF(pbDst); NOREF(pcTransfer); NOREF(cb);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNIOMIOPORTNEWOUT,
+ *      Dummy Port I/O Handler for OUT operations.}
+ */
+static DECLCALLBACK(VBOXSTRICTRC)
+iomR3IOPortDummyNewOut(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, uint32_t u32, unsigned cb)
+{
+    NOREF(pDevIns); NOREF(pvUser); NOREF(Port); NOREF(u32); NOREF(cb);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNIOMIOPORTNEWOUTSTRING,
+ *      Dummy Port I/O Handler for string OUT operations.}
+ */
+static DECLCALLBACK(VBOXSTRICTRC)
+iomR3IOPortDummyNewOutStr(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, uint8_t const *pbSrc, uint32_t *pcTransfer, unsigned cb)
+{
+    NOREF(pDevIns); NOREF(pvUser); NOREF(Port); NOREF(pbSrc); NOREF(pcTransfer); NOREF(cb);
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Worker for PDMDEVHLPR3::pfnIoPortCreateEx.
+ */
+VMMR3_INT_DECL(int)  IOMR3IoPortCreate(PVM pVM, PPDMDEVINS pDevIns, RTIOPORT cPorts, uint32_t fFlags, PPDMPCIDEV pPciDev,
+                                       uint32_t iPciRegion, PFNIOMIOPORTNEWOUT pfnOut, PFNIOMIOPORTNEWIN pfnIn,
+                                       PFNIOMIOPORTNEWOUTSTRING pfnOutStr, PFNIOMIOPORTNEWINSTRING pfnInStr, RTR3PTR pvUser,
+                                       const char *pszDesc, PCIOMIOPORTDESC paExtDescs, PIOMIOPORTHANDLE phIoPorts)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(phIoPorts, VERR_INVALID_POINTER);
+    *phIoPorts = UINT32_MAX;
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+    AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
+
+    AssertMsgReturn(cPorts > 0 && cPorts <= _8K, ("cPorts=%#x\n", cPorts), VERR_OUT_OF_RANGE);
+    AssertReturn(!(fFlags & ~IOM_IOPORT_F_VALID_MASK), VERR_INVALID_FLAGS);
+
+    AssertReturn(pfnOut || pfnIn || pfnOutStr || pfnInStr, VERR_INVALID_PARAMETER);
+    AssertPtrNullReturn(pfnOut, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnIn, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnOutStr, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnInStr, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertReturn(*pszDesc != '\0', VERR_INVALID_POINTER);
+    AssertReturn(strlen(pszDesc) < 128, VERR_INVALID_POINTER);
+    if (paExtDescs)
+    {
+        AssertPtrReturn(paExtDescs, VERR_INVALID_POINTER);
+        for (size_t i = 0;; i++)
+        {
+            const char *pszIn  = paExtDescs[i].pszIn;
+            const char *pszOut = paExtDescs[i].pszIn;
+            if (!pszIn && !pszOut)
+                break;
+            AssertReturn(i < _8K, VERR_OUT_OF_RANGE);
+            AssertReturn(!pszIn  || strlen(pszIn)  < 128, VERR_INVALID_POINTER);
+            AssertReturn(!pszOut || strlen(pszOut) < 128, VERR_INVALID_POINTER);
+        }
+    }
+
+    /*
+     * Ensure that we've got table space for it.
+     */
+#ifndef VBOX_WITH_STATISTICS
+    uint16_t const idxStats        = UINT16_MAX;
+#else
+    uint32_t const idxStats        = pVM->iom.s.cIoPortStats;
+    uint32_t const cNewIoPortStats = idxStats + cPorts;
+    AssertReturn(cNewIoPortStats <= _64K, VERR_IOM_TOO_MANY_IOPORT_REGISTRATIONS);
+    if (cNewIoPortStats > pVM->iom.s.cIoPortStatsAllocation)
+    {
+        int rc = VMMR3CallR0Emt(pVM, pVM->apCpusR3[0], VMMR0_DO_IOM_GROW_IO_PORT_STATS, cNewIoPortStats, NULL);
+        AssertLogRelRCReturn(rc, rc);
+        AssertReturn(idxStats == pVM->iom.s.cIoPortStats, VERR_IOM_IOPORT_IPE_1);
+        AssertReturn(cNewIoPortStats <= pVM->iom.s.cIoPortStatsAllocation, VERR_IOM_IOPORT_IPE_2);
+    }
+#endif
+
+    uint32_t idx = pVM->iom.s.cIoPortRegs;
+    if (idx >= pVM->iom.s.cIoPortAlloc)
+    {
+        int rc = VMMR3CallR0Emt(pVM, pVM->apCpusR3[0], VMMR0_DO_IOM_GROW_IO_PORTS, pVM->iom.s.cIoPortAlloc + 1, NULL);
+        AssertLogRelRCReturn(rc, rc);
+        AssertReturn(idx == pVM->iom.s.cIoPortRegs, VERR_IOM_IOPORT_IPE_1);
+        AssertReturn(idx < pVM->iom.s.cIoPortAlloc, VERR_IOM_IOPORT_IPE_2);
+    }
+
+    /*
+     * Enter it.
+     */
+    pVM->iom.s.paIoPortRegs[idx].pvUser             = pvUser;
+    pVM->iom.s.paIoPortRegs[idx].pDevIns            = pDevIns;
+    pVM->iom.s.paIoPortRegs[idx].pfnOutCallback     = pfnOut    ? pfnOut    : iomR3IOPortDummyNewOut;
+    pVM->iom.s.paIoPortRegs[idx].pfnInCallback      = pfnIn     ? pfnIn     : iomR3IOPortDummyNewIn;
+    pVM->iom.s.paIoPortRegs[idx].pfnOutStrCallback  = pfnOutStr ? pfnOutStr : iomR3IOPortDummyNewOutStr;
+    pVM->iom.s.paIoPortRegs[idx].pfnInStrCallback   = pfnInStr  ? pfnInStr  : iomR3IOPortDummyNewInStr;
+    pVM->iom.s.paIoPortRegs[idx].pszDesc            = pszDesc;
+    pVM->iom.s.paIoPortRegs[idx].paExtDescs         = paExtDescs;
+    pVM->iom.s.paIoPortRegs[idx].pPciDev            = pPciDev;
+    pVM->iom.s.paIoPortRegs[idx].iPciRegion         = iPciRegion;
+    pVM->iom.s.paIoPortRegs[idx].cPorts             = cPorts;
+    pVM->iom.s.paIoPortRegs[idx].uPort              = UINT16_MAX;
+    pVM->iom.s.paIoPortRegs[idx].idxStats           = (uint16_t)idxStats;
+    pVM->iom.s.paIoPortRegs[idx].fMapped            = false;
+    pVM->iom.s.paIoPortRegs[idx].fFlags             = (uint8_t)fFlags;
+    pVM->iom.s.paIoPortRegs[idx].idxSelf            = idx;
+
+    pVM->iom.s.cIoPortRegs = idx + 1;
+#ifdef VBOX_WITH_STATISTICS
+    pVM->iom.s.cIoPortStats = cNewIoPortStats;
+#endif
+    *phIoPorts = idx;
+    LogFlow(("IOMR3IoPortCreate: idx=%#x cPorts=%u %s\n", idx, cPorts, pszDesc));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMDEVHLPR3::pfnIoPortMap.
+ */
+VMMR3_INT_DECL(int)  IOMR3IoPortMap(PVM pVM, PPDMDEVINS pDevIns, IOMIOPORTHANDLE hIoPorts, RTIOPORT uPort)
+{
+    /*
+     * Validate input and state.
+     */
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    AssertReturn(hIoPorts < pVM->iom.s.cIoPortRegs, VERR_IOM_INVALID_IOPORT_HANDLE);
+    PIOMIOPORTENTRYR3 const pRegEntry = &pVM->iom.s.paIoPortRegs[hIoPorts];
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_IOM_INVALID_IOPORT_HANDLE);
+
+    RTIOPORT const cPorts = pRegEntry->cPorts;
+    AssertMsgReturn(cPorts > 0 && cPorts <= _8K, ("cPorts=%s\n", cPorts), VERR_IOM_IOPORT_IPE_1);
+    AssertReturn((uint32_t)uPort + cPorts <= _64K, VERR_OUT_OF_RANGE);
+    RTIOPORT const uLastPort = uPort + cPorts - 1;
+    LogFlow(("IOMR3IoPortMap: hIoPorts=%#RX64 %RTiop..%RTiop (%u ports)\n", hIoPorts, uPort, uLastPort, cPorts));
+
+    /*
+     * Do the mapping.
+     */
+    int rc = VINF_SUCCESS;
+    IOM_LOCK_EXCL(pVM);
+
+    if (!pRegEntry->fMapped)
+    {
+        uint32_t const cEntries = RT_MIN(pVM->iom.s.cIoPortLookupEntries, pVM->iom.s.cIoPortRegs);
+        Assert(pVM->iom.s.cIoPortLookupEntries == cEntries);
+
+        PIOMIOPORTLOOKUPENTRY paEntries = pVM->iom.s.paIoPortLookup;
+        PIOMIOPORTLOOKUPENTRY pEntry;
+        if (cEntries > 0)
+        {
+            uint32_t iFirst = 0;
+            uint32_t iEnd   = cEntries;
+            uint32_t i      = cEntries / 2;
+            for (;;)
+            {
+                pEntry = &paEntries[i];
+                if (pEntry->uLastPort < uPort)
+                {
+                    i += 1;
+                    if (i < iEnd)
+                        iFirst = i;
+                    else
+                    {
+                        /* Insert after the entry we just considered: */
+                        pEntry += 1;
+                        if (i < cEntries)
+                            memmove(pEntry + 1, pEntry, sizeof(*pEntry) * (cEntries - i));
+                        break;
+                    }
+                }
+                else if (pEntry->uFirstPort > uLastPort)
+                {
+                    if (i > iFirst)
+                        iEnd = i;
+                    else
+                    {
+                        /* Insert at the entry we just considered: */
+                        if (i < cEntries)
+                            memmove(pEntry + 1, pEntry, sizeof(*pEntry) * (cEntries - i));
+                        break;
+                    }
+                }
+                else
+                {
+                    /* Oops! We've got a conflict. */
+                    AssertLogRelMsgFailed(("%x..%x (%s) conflicts with existing mapping %x..%x (%s)\n",
+                                           uPort, uLastPort, pRegEntry->pszDesc,
+                                           pEntry->uFirstPort, pEntry->uLastPort, pVM->iom.s.paIoPortRegs[pEntry->idx].pszDesc));
+                    IOM_UNLOCK_EXCL(pVM);
+                    return VERR_IOM_IOPORT_RANGE_CONFLICT;
+                }
+
+                i = iFirst + (iEnd - iFirst) / 2;
+            }
+        }
+        else
+            pEntry = paEntries;
+
+        /*
+         * Fill in the entry and bump the table size.
+         */
+        pEntry->idx        = hIoPorts;
+        pEntry->uFirstPort = uPort;
+        pEntry->uLastPort  = uLastPort;
+        pVM->iom.s.cIoPortLookupEntries = cEntries + 1;
+
+        pRegEntry->uPort   = uPort;
+        pRegEntry->fMapped = true;
+
+#ifdef VBOX_WITH_STATISTICS
+        /* Don't register stats here when we're creating the VM as the
+           statistics table may still be reallocated. */
+        if (pVM->enmVMState >= VMSTATE_CREATED)
+            iomR3IoPortRegStats(pVM, pRegEntry);
+#endif
+
+#ifdef VBOX_STRICT
+        /*
+         * Assert table sanity.
+         */
+        AssertMsg(paEntries[0].uLastPort >= paEntries[0].uFirstPort, ("%#x %#x\n", paEntries[0].uLastPort, paEntries[0].uFirstPort));
+        AssertMsg(paEntries[0].idx < pVM->iom.s.cIoPortRegs, ("%#x %#x\n", paEntries[0].idx, pVM->iom.s.cIoPortRegs));
+
+        RTIOPORT uPortPrev = paEntries[0].uLastPort;
+        for (size_t i = 1; i <= cEntries; i++)
+        {
+            AssertMsg(paEntries[i].uLastPort >= paEntries[i].uFirstPort, ("%u: %#x %#x\n", i, paEntries[i].uLastPort, paEntries[i].uFirstPort));
+            AssertMsg(paEntries[i].idx < pVM->iom.s.cIoPortRegs, ("%u: %#x %#x\n", i, paEntries[i].idx, pVM->iom.s.cIoPortRegs));
+            AssertMsg(uPortPrev < paEntries[i].uFirstPort, ("%u: %#x %#x\n", i, uPortPrev, paEntries[i].uFirstPort));
+            AssertMsg(paEntries[i].uLastPort - paEntries[i].uFirstPort + 1 == pVM->iom.s.paIoPortRegs[paEntries[i].idx].cPorts,
+                      ("%u: %#x %#x..%#x -> %u, expected %u\n", i, uPortPrev, paEntries[i].uFirstPort, paEntries[i].uLastPort,
+                       paEntries[i].uLastPort - paEntries[i].uFirstPort + 1, pVM->iom.s.paIoPortRegs[paEntries[i].idx].cPorts));
+            uPortPrev = paEntries[i].uLastPort;
+        }
+#endif
+    }
+    else
+    {
+        AssertFailed();
+        rc = VERR_IOM_IOPORTS_ALREADY_MAPPED;
+    }
+
+    IOM_UNLOCK_EXCL(pVM);
+    return rc;
+}
+
+
+/**
+ * Worker for PDMDEVHLPR3::pfnIoPortUnmap.
+ */
+VMMR3_INT_DECL(int)  IOMR3IoPortUnmap(PVM pVM, PPDMDEVINS pDevIns, IOMIOPORTHANDLE hIoPorts)
+{
+    /*
+     * Validate input and state.
+     */
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    AssertReturn(hIoPorts < pVM->iom.s.cIoPortRegs, VERR_IOM_INVALID_IOPORT_HANDLE);
+    PIOMIOPORTENTRYR3 const pRegEntry = &pVM->iom.s.paIoPortRegs[hIoPorts];
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_IOM_INVALID_IOPORT_HANDLE);
+
+    /*
+     * Do the mapping.
+     */
+    int rc;
+    IOM_LOCK_EXCL(pVM);
+
+    if (pRegEntry->fMapped)
+    {
+        RTIOPORT const uPort     = pRegEntry->uPort;
+        RTIOPORT const uLastPort = uPort + pRegEntry->cPorts - 1;
+        uint32_t const cEntries  = RT_MIN(pVM->iom.s.cIoPortLookupEntries, pVM->iom.s.cIoPortRegs);
+        Assert(pVM->iom.s.cIoPortLookupEntries == cEntries);
+        Assert(cEntries > 0);
+        LogFlow(("IOMR3IoPortUnmap: hIoPorts=%#RX64 %RTiop..%RTiop (%u ports)\n", hIoPorts, uPort, uLastPort, pRegEntry->cPorts));
+
+        PIOMIOPORTLOOKUPENTRY paEntries = pVM->iom.s.paIoPortLookup;
+        uint32_t iFirst = 0;
+        uint32_t iEnd   = cEntries;
+        uint32_t i      = cEntries / 2;
+        for (;;)
+        {
+            PIOMIOPORTLOOKUPENTRY pEntry = &paEntries[i];
+            if (pEntry->uLastPort < uPort)
+            {
+                i += 1;
+                if (i < iEnd)
+                    iFirst = i;
+                else
+                {
+                    rc = VERR_IOM_IOPORT_IPE_1;
+                    AssertLogRelMsgFailedBreak(("%x..%x (%s) not found!\n", uPort, uLastPort, pRegEntry->pszDesc));
+                }
+            }
+            else if (pEntry->uFirstPort > uLastPort)
+            {
+                if (i > iFirst)
+                    iEnd = i;
+                else
+                {
+                    rc = VERR_IOM_IOPORT_IPE_1;
+                    AssertLogRelMsgFailedBreak(("%x..%x (%s) not found!\n", uPort, uLastPort, pRegEntry->pszDesc));
+                }
+            }
+            else if (pEntry->idx == hIoPorts)
+            {
+                Assert(pEntry->uFirstPort == uPort);
+                Assert(pEntry->uLastPort == uLastPort);
+#ifdef VBOX_WITH_STATISTICS
+                iomR3IoPortDeregStats(pVM, pRegEntry, uPort);
+#endif
+                if (i + 1 < cEntries)
+                    memmove(pEntry, pEntry + 1, sizeof(*pEntry) * (cEntries - i - 1));
+                pVM->iom.s.cIoPortLookupEntries = cEntries - 1;
+                pRegEntry->uPort   = UINT16_MAX;
+                pRegEntry->fMapped = false;
+                rc = VINF_SUCCESS;
+                break;
+            }
+            else
+            {
+                AssertLogRelMsgFailed(("Lookig for %x..%x (%s), found %x..%x (%s) instead!\n",
+                                       uPort, uLastPort, pRegEntry->pszDesc,
+                                       pEntry->uFirstPort, pEntry->uLastPort, pVM->iom.s.paIoPortRegs[pEntry->idx].pszDesc));
+                rc = VERR_IOM_IOPORT_IPE_1;
+                break;
+            }
+
+            i = iFirst + (iEnd - iFirst) / 2;
+        }
+
+#ifdef VBOX_STRICT
+        /*
+         * Assert table sanity.
+         */
+        AssertMsg(paEntries[0].uLastPort >= paEntries[0].uFirstPort, ("%#x %#x\n", paEntries[0].uLastPort, paEntries[0].uFirstPort));
+        AssertMsg(paEntries[0].idx < pVM->iom.s.cIoPortRegs, ("%#x %#x\n", paEntries[0].idx, pVM->iom.s.cIoPortRegs));
+
+        RTIOPORT uPortPrev = paEntries[0].uLastPort;
+        for (i = 1; i < cEntries - 1; i++)
+        {
+            AssertMsg(paEntries[i].uLastPort >= paEntries[i].uFirstPort, ("%u: %#x %#x\n", i, paEntries[i].uLastPort, paEntries[i].uFirstPort));
+            AssertMsg(paEntries[i].idx < pVM->iom.s.cIoPortRegs, ("%u: %#x %#x\n", i, paEntries[i].idx, pVM->iom.s.cIoPortRegs));
+            AssertMsg(uPortPrev < paEntries[i].uFirstPort, ("%u: %#x %#x\n", i, uPortPrev, paEntries[i].uFirstPort));
+            AssertMsg(paEntries[i].uLastPort - paEntries[i].uFirstPort + 1 == pVM->iom.s.paIoPortRegs[paEntries[i].idx].cPorts,
+                      ("%u: %#x %#x..%#x -> %u, expected %u\n", i, uPortPrev, paEntries[i].uFirstPort, paEntries[i].uLastPort,
+                       paEntries[i].uLastPort - paEntries[i].uFirstPort + 1, pVM->iom.s.paIoPortRegs[paEntries[i].idx].cPorts));
+            uPortPrev = paEntries[i].uLastPort;
+        }
+#endif
+    }
+    else
+    {
+        AssertFailed();
+        rc = VERR_IOM_IOPORTS_NOT_MAPPED;
+    }
+
+    IOM_UNLOCK_EXCL(pVM);
+    return rc;
+}
+
+
+/**
+ * Validates @a hIoPorts, making sure it belongs to @a pDevIns.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     The device which allegedly owns @a hIoPorts.
+ * @param   hIoPorts    The handle to validate.
+ */
+VMMR3_INT_DECL(int)  IOMR3IoPortValidateHandle(PVM pVM, PPDMDEVINS pDevIns, IOMIOPORTHANDLE hIoPorts)
+{
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    AssertReturn(hIoPorts < RT_MIN(pVM->iom.s.cIoPortRegs, pVM->iom.s.cIoPortAlloc), VERR_IOM_INVALID_IOPORT_HANDLE);
+    PIOMIOPORTENTRYR3 const pRegEntry = &pVM->iom.s.paIoPortRegs[hIoPorts];
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_IOM_INVALID_IOPORT_HANDLE);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the mapping address of I/O ports @a hIoPorts.
+ *
+ * @returns Mapping address if mapped, UINT32_MAX if not mapped or invalid
+ *          input.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     The device which allegedly owns @a hRegion.
+ * @param   hIoPorts    The handle to I/O port region.
+ */
+VMMR3_INT_DECL(uint32_t) IOMR3IoPortGetMappingAddress(PVM pVM, PPDMDEVINS pDevIns, IOMIOPORTHANDLE hIoPorts)
+{
+    AssertPtrReturn(pDevIns, UINT32_MAX);
+    AssertReturn(hIoPorts < RT_MIN(pVM->iom.s.cIoPortRegs, pVM->iom.s.cIoPortAlloc), UINT32_MAX);
+    IOMIOPORTENTRYR3 volatile * const pRegEntry = &pVM->iom.s.paIoPortRegs[hIoPorts];
+    AssertReturn(pRegEntry->pDevIns == pDevIns, UINT32_MAX);
+    for (uint32_t iTry = 0; ; iTry++)
+    {
+        bool        fMapped = pRegEntry->fMapped;
+        RTIOPORT    uPort   = pRegEntry->uPort;
+        if (   (   ASMAtomicReadBool(&pRegEntry->fMapped) == fMapped
+                && uPort == pRegEntry->uPort)
+            || iTry > 1024)
+            return fMapped ? uPort : UINT32_MAX;
+        ASMNopPause();
+    }
+}
+
+
+/**
+ * Display all registered I/O port ranges.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+DECLCALLBACK(void) iomR3IoPortInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    RT_NOREF(pszArgs);
+
+    /* No locking needed here as registerations are only happening during VMSTATE_CREATING. */
+    pHlp->pfnPrintf(pHlp,
+                    "I/O port registrations: %u (%u allocated)\n"
+                    " ## Ctx    Ports Mapping   PCI    Description\n",
+                    pVM->iom.s.cIoPortRegs, pVM->iom.s.cIoPortAlloc);
+    PIOMIOPORTENTRYR3 paRegs = pVM->iom.s.paIoPortRegs;
+    for (uint32_t i = 0; i < pVM->iom.s.cIoPortRegs; i++)
+    {
+        const char * const pszRing = paRegs[i].fRing0 ? paRegs[i].fRawMode ? "+0+C" : "+0  "
+                                   : paRegs[i].fRawMode ? "+C  " : "    ";
+        if (paRegs[i].fMapped && paRegs[i].pPciDev)
+            pHlp->pfnPrintf(pHlp, "%3u R3%s %04x  %04x-%04x pci%u/%u %s\n", paRegs[i].idxSelf, pszRing, paRegs[i].cPorts,
+                            paRegs[i].uPort, paRegs[i].uPort + paRegs[i].cPorts - 1,
+                            paRegs[i].pPciDev->idxSubDev, paRegs[i].iPciRegion, paRegs[i].pszDesc);
+        else if (paRegs[i].fMapped && !paRegs[i].pPciDev)
+            pHlp->pfnPrintf(pHlp, "%3u R3%s %04x  %04x-%04x        %s\n", paRegs[i].idxSelf, pszRing, paRegs[i].cPorts,
+                            paRegs[i].uPort, paRegs[i].uPort + paRegs[i].cPorts - 1, paRegs[i].pszDesc);
+        else if (paRegs[i].pPciDev)
+            pHlp->pfnPrintf(pHlp, "%3u R3%s %04x  unmapped  pci%u/%u %s\n", paRegs[i].idxSelf, pszRing, paRegs[i].cPorts,
+                            paRegs[i].pPciDev->idxSubDev, paRegs[i].iPciRegion, paRegs[i].pszDesc);
+        else
+            pHlp->pfnPrintf(pHlp, "%3u R3%s %04x  unmapped         %s\n",
+                            paRegs[i].idxSelf, pszRing, paRegs[i].cPorts, paRegs[i].pszDesc);
+    }
+}
+
diff --git a/src/VBox/VMM/VMMR3/IOMR3Mmio.cpp b/src/VBox/VMM/VMMR3/IOMR3Mmio.cpp
new file mode 100644
index 00000000..20b3bc3d
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/IOMR3Mmio.cpp
@@ -0,0 +1,553 @@
+/* $Id: IOMR3Mmio.cpp $ */
+/** @file
+ * IOM - Input / Output Monitor, MMIO related APIs.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_IOM_MMIO
+#include <VBox/vmm/iom.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pdmdev.h>
+#include "IOMInternal.h"
+#include <VBox/vmm/vm.h>
+
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+
+#include "IOMInline.h"
+
+
+#ifdef VBOX_WITH_STATISTICS
+
+/**
+ * Register statistics for a MMIO entry.
+ */
+void iomR3MmioRegStats(PVM pVM, PIOMMMIOENTRYR3 pRegEntry)
+{
+    bool const           fDoRZ  = pRegEntry->fRing0 || pRegEntry->fRawMode;
+    PIOMMMIOSTATSENTRY   pStats = &pVM->iom.s.paMmioStats[pRegEntry->idxStats];
+
+    /* Format the prefix: */
+    char                 szName[80];
+    size_t               cchPrefix = RTStrPrintf(szName, sizeof(szName), "/IOM/MmioRegions/%RGp-%RGp",
+                                                 pRegEntry->GCPhysMapping, pRegEntry->GCPhysMapping + pRegEntry->cbRegion - 1);
+
+    /* Mangle the description if this isn't the first device instance: */
+    const char          *pszDesc     = pRegEntry->pszDesc;
+    char                *pszFreeDesc = NULL;
+    if (pRegEntry->pDevIns && pRegEntry->pDevIns->iInstance > 0 && pszDesc)
+        pszDesc = pszFreeDesc = RTStrAPrintf2("%u / %s", pRegEntry->pDevIns->iInstance, pszDesc);
+
+    /* Register statistics: */
+    int rc = STAMR3Register(pVM, &pRegEntry->idxSelf, STAMTYPE_U16, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_NONE, pszDesc); AssertRC(rc);
+    RTStrFree(pszFreeDesc);
+
+# define SET_NM_SUFFIX(a_sz) memcpy(&szName[cchPrefix], a_sz, sizeof(a_sz))
+    SET_NM_SUFFIX("/Read-Complicated");
+    rc = STAMR3Register(pVM, &pStats->ComplicatedReads, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Read-FFor00");
+    rc = STAMR3Register(pVM, &pStats->FFor00Reads, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Read-R3");
+    rc = STAMR3Register(pVM, &pStats->ProfReadR3,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
+    if (fDoRZ)
+    {
+        SET_NM_SUFFIX("/Read-RZ");
+        rc = STAMR3Register(pVM, &pStats->ProfReadRZ,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
+        SET_NM_SUFFIX("/Read-RZtoR3");
+        rc = STAMR3Register(pVM, &pStats->ReadRZToR3,  STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
+    }
+    SET_NM_SUFFIX("/Read-Total");
+    rc = STAMR3Register(pVM, &pStats->Reads,       STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
+
+    SET_NM_SUFFIX("/Write-Complicated");
+    rc = STAMR3Register(pVM, &pStats->ComplicatedWrites, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Write-R3");
+    rc = STAMR3Register(pVM, &pStats->ProfWriteR3,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
+    if (fDoRZ)
+    {
+        SET_NM_SUFFIX("/Write-RZ");
+        rc = STAMR3Register(pVM, &pStats->ProfWriteRZ, STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
+        SET_NM_SUFFIX("/Write-RZtoR3");
+        rc = STAMR3Register(pVM, &pStats->WriteRZToR3, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
+        SET_NM_SUFFIX("/Write-RZtoR3-Commit");
+        rc = STAMR3Register(pVM, &pStats->CommitRZToR3, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,    NULL); AssertRC(rc);
+    }
+    SET_NM_SUFFIX("/Write-Total");
+    rc = STAMR3Register(pVM, &pStats->Writes,      STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
+}
+
+
+/**
+ * Deregister statistics for a MMIO entry.
+ */
+static void iomR3MmioDeregStats(PVM pVM, PIOMMMIOENTRYR3 pRegEntry, RTGCPHYS GCPhys)
+{
+    char szPrefix[80];
+    RTStrPrintf(szPrefix, sizeof(szPrefix), "/IOM/MmioRegions/%RGp-%RGp", GCPhys, GCPhys + pRegEntry->cbRegion - 1);
+    STAMR3DeregisterByPrefix(pVM->pUVM, szPrefix);
+}
+
+#endif /* VBOX_WITH_STATISTICS */
+
+
+/**
+ * Worker for PDMDEVHLPR3::pfnMmioCreateEx.
+ */
+VMMR3_INT_DECL(int)  IOMR3MmioCreate(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS cbRegion, uint32_t fFlags, PPDMPCIDEV pPciDev,
+                                     uint32_t iPciRegion, PFNIOMMMIONEWWRITE pfnWrite, PFNIOMMMIONEWREAD pfnRead,
+                                     PFNIOMMMIONEWFILL pfnFill, void *pvUser, const char *pszDesc, PIOMMMIOHANDLE phRegion)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(phRegion, VERR_INVALID_POINTER);
+    *phRegion = UINT32_MAX;
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+    AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
+
+    AssertMsgReturn(cbRegion > 0 && cbRegion <= MM_MMIO_64_MAX, ("cbRegion=%#RGp (max %#RGp)\n", cbRegion, MM_MMIO_64_MAX),
+                    VERR_OUT_OF_RANGE);
+    AssertMsgReturn(!(cbRegion & PAGE_OFFSET_MASK), ("cbRegion=%#RGp\n", cbRegion), VERR_UNSUPPORTED_ALIGNMENT);
+
+    AssertMsgReturn(   !(fFlags & ~IOMMMIO_FLAGS_VALID_MASK)
+                    && (fFlags & IOMMMIO_FLAGS_READ_MODE)  <= IOMMMIO_FLAGS_READ_DWORD_QWORD
+                    && (fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD,
+                    ("%#x\n", fFlags),
+                    VERR_INVALID_FLAGS);
+
+    AssertReturn(pfnWrite || pfnRead, VERR_INVALID_PARAMETER);
+    AssertPtrNullReturn(pfnWrite, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnRead, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnFill, VERR_INVALID_POINTER);
+
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertReturn(*pszDesc != '\0', VERR_INVALID_POINTER);
+    AssertReturn(strlen(pszDesc) < 128, VERR_INVALID_POINTER);
+
+    /*
+     * Ensure that we've got table space for it.
+     */
+#ifndef VBOX_WITH_STATISTICS
+    uint16_t const idxStats      = UINT16_MAX;
+#else
+    uint32_t const idxStats      = pVM->iom.s.cMmioStats;
+    uint32_t const cNewMmioStats = idxStats + 1;
+    AssertReturn(cNewMmioStats <= _64K, VERR_IOM_TOO_MANY_MMIO_REGISTRATIONS);
+    if (cNewMmioStats > pVM->iom.s.cMmioStatsAllocation)
+    {
+        int rc = VMMR3CallR0Emt(pVM, pVM->apCpusR3[0], VMMR0_DO_IOM_GROW_MMIO_STATS, cNewMmioStats, NULL);
+        AssertLogRelRCReturn(rc, rc);
+        AssertReturn(idxStats == pVM->iom.s.cMmioStats, VERR_IOM_MMIO_IPE_1);
+        AssertReturn(cNewMmioStats <= pVM->iom.s.cMmioStatsAllocation, VERR_IOM_MMIO_IPE_2);
+    }
+#endif
+
+    uint32_t idx = pVM->iom.s.cMmioRegs;
+    if (idx >= pVM->iom.s.cMmioAlloc)
+    {
+        int rc = VMMR3CallR0Emt(pVM, pVM->apCpusR3[0], VMMR0_DO_IOM_GROW_MMIO_REGS, pVM->iom.s.cMmioAlloc + 1, NULL);
+        AssertLogRelRCReturn(rc, rc);
+        AssertReturn(idx == pVM->iom.s.cMmioRegs, VERR_IOM_MMIO_IPE_1);
+        AssertReturn(idx < pVM->iom.s.cMmioAlloc, VERR_IOM_MMIO_IPE_2);
+    }
+
+    /*
+     * Enter it.
+     */
+    pVM->iom.s.paMmioRegs[idx].cbRegion           = cbRegion;
+    pVM->iom.s.paMmioRegs[idx].GCPhysMapping      = NIL_RTGCPHYS;
+    pVM->iom.s.paMmioRegs[idx].pvUser             = pvUser;
+    pVM->iom.s.paMmioRegs[idx].pDevIns            = pDevIns;
+    pVM->iom.s.paMmioRegs[idx].pfnWriteCallback   = pfnWrite;
+    pVM->iom.s.paMmioRegs[idx].pfnReadCallback    = pfnRead;
+    pVM->iom.s.paMmioRegs[idx].pfnFillCallback    = pfnFill;
+    pVM->iom.s.paMmioRegs[idx].pszDesc            = pszDesc;
+    pVM->iom.s.paMmioRegs[idx].pPciDev            = pPciDev;
+    pVM->iom.s.paMmioRegs[idx].iPciRegion         = iPciRegion;
+    pVM->iom.s.paMmioRegs[idx].idxStats           = (uint16_t)idxStats;
+    pVM->iom.s.paMmioRegs[idx].fMapped            = false;
+    pVM->iom.s.paMmioRegs[idx].fFlags             = fFlags;
+    pVM->iom.s.paMmioRegs[idx].idxSelf            = idx;
+
+    pVM->iom.s.cMmioRegs = idx + 1;
+#ifdef VBOX_WITH_STATISTICS
+    pVM->iom.s.cMmioStats = cNewMmioStats;
+#endif
+    *phRegion = idx;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMDEVHLPR3::pfnMmioMap.
+ */
+VMMR3_INT_DECL(int)  IOMR3MmioMap(PVM pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS GCPhys)
+{
+    /*
+     * Validate input and state.
+     */
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    AssertReturn(hRegion < pVM->iom.s.cMmioRegs, VERR_IOM_INVALID_MMIO_HANDLE);
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_IOM_INVALID_MMIO_HANDLE);
+
+    RTGCPHYS const cbRegion = pRegEntry->cbRegion;
+    AssertMsgReturn(cbRegion > 0 && cbRegion <= MM_MMIO_64_MAX, ("cbRegion=%RGp\n", cbRegion), VERR_IOM_MMIO_IPE_1);
+    RTGCPHYS const GCPhysLast = GCPhys + cbRegion - 1;
+
+    AssertLogRelMsgReturn(!(GCPhys & PAGE_OFFSET_MASK),
+                          ("Misaligned! GCPhys=%RGp LB %RGp %s (%s[#%u])\n",
+                           GCPhys, cbRegion, pRegEntry->pszDesc, pDevIns->pReg->szName, pDevIns->iInstance),
+                          VERR_IOM_INVALID_MMIO_RANGE);
+    AssertLogRelMsgReturn(GCPhysLast > GCPhys,
+                          ("Wrapped! GCPhys=%RGp LB %RGp %s (%s[#%u])\n",
+                           GCPhys, cbRegion, pRegEntry->pszDesc, pDevIns->pReg->szName, pDevIns->iInstance),
+                          VERR_IOM_INVALID_MMIO_RANGE);
+
+    /*
+     * Do the mapping.
+     */
+    int rc = VINF_SUCCESS;
+    IOM_LOCK_EXCL(pVM);
+
+    if (!pRegEntry->fMapped)
+    {
+        uint32_t const cEntries = RT_MIN(pVM->iom.s.cMmioLookupEntries, pVM->iom.s.cMmioRegs);
+        Assert(pVM->iom.s.cMmioLookupEntries == cEntries);
+
+        PIOMMMIOLOOKUPENTRY paEntries = pVM->iom.s.paMmioLookup;
+        PIOMMMIOLOOKUPENTRY pEntry;
+        if (cEntries > 0)
+        {
+            uint32_t iFirst = 0;
+            uint32_t iEnd   = cEntries;
+            uint32_t i      = cEntries / 2;
+            for (;;)
+            {
+                pEntry = &paEntries[i];
+                if (pEntry->GCPhysLast < GCPhys)
+                {
+                    i += 1;
+                    if (i < iEnd)
+                        iFirst = i;
+                    else
+                    {
+                        /* Register with PGM before we shuffle the array: */
+                        ASMAtomicWriteU64(&pRegEntry->GCPhysMapping, GCPhys);
+                        rc = PGMR3PhysMMIORegister(pVM, GCPhys, cbRegion, pVM->iom.s.hNewMmioHandlerType,
+                                                   (void *)(uintptr_t)hRegion, hRegion, hRegion, pRegEntry->pszDesc);
+                        AssertRCReturnStmt(rc, ASMAtomicWriteU64(&pRegEntry->GCPhysMapping, NIL_RTGCPHYS); IOM_UNLOCK_EXCL(pVM), rc);
+
+                        /* Insert after the entry we just considered: */
+                        pEntry += 1;
+                        if (i < cEntries)
+                            memmove(pEntry + 1, pEntry, sizeof(*pEntry) * (cEntries - i));
+                        break;
+                    }
+                }
+                else if (pEntry->GCPhysFirst > GCPhysLast)
+                {
+                    if (i > iFirst)
+                        iEnd = i;
+                    else
+                    {
+                        /* Register with PGM before we shuffle the array: */
+                        ASMAtomicWriteU64(&pRegEntry->GCPhysMapping, GCPhys);
+                        rc = PGMR3PhysMMIORegister(pVM, GCPhys, cbRegion, pVM->iom.s.hNewMmioHandlerType,
+                                                   (void *)(uintptr_t)hRegion, hRegion, hRegion, pRegEntry->pszDesc);
+                        AssertRCReturnStmt(rc, ASMAtomicWriteU64(&pRegEntry->GCPhysMapping, NIL_RTGCPHYS); IOM_UNLOCK_EXCL(pVM), rc);
+
+                        /* Insert at the entry we just considered: */
+                        if (i < cEntries)
+                            memmove(pEntry + 1, pEntry, sizeof(*pEntry) * (cEntries - i));
+                        break;
+                    }
+                }
+                else
+                {
+                    /* Oops! We've got a conflict. */
+                    AssertLogRelMsgFailed(("%RGp..%RGp (%s) conflicts with existing mapping %RGp..%RGp (%s)\n",
+                                           GCPhys, GCPhysLast, pRegEntry->pszDesc,
+                                           pEntry->GCPhysFirst, pEntry->GCPhysLast, pVM->iom.s.paMmioRegs[pEntry->idx].pszDesc));
+                    IOM_UNLOCK_EXCL(pVM);
+                    return VERR_IOM_MMIO_RANGE_CONFLICT;
+                }
+
+                i = iFirst + (iEnd - iFirst) / 2;
+            }
+        }
+        else
+        {
+            /* First entry in the lookup table: */
+            ASMAtomicWriteU64(&pRegEntry->GCPhysMapping, GCPhys);
+            rc = PGMR3PhysMMIORegister(pVM, GCPhys, cbRegion, pVM->iom.s.hNewMmioHandlerType,
+                                       (void *)(uintptr_t)hRegion, hRegion, hRegion, pRegEntry->pszDesc);
+            AssertRCReturnStmt(rc, ASMAtomicWriteU64(&pRegEntry->GCPhysMapping, NIL_RTGCPHYS); IOM_UNLOCK_EXCL(pVM), rc);
+
+            pEntry = paEntries;
+        }
+
+        /*
+         * Fill in the entry and bump the table size.
+         */
+        pRegEntry->fMapped  = true;
+        pEntry->idx         = hRegion;
+        pEntry->GCPhysFirst = GCPhys;
+        pEntry->GCPhysLast  = GCPhysLast;
+        pVM->iom.s.cMmioLookupEntries = cEntries + 1;
+
+#ifdef VBOX_WITH_STATISTICS
+        /* Don't register stats here when we're creating the VM as the
+           statistics table may still be reallocated. */
+        if (pVM->enmVMState >= VMSTATE_CREATED)
+            iomR3MmioRegStats(pVM, pRegEntry);
+#endif
+
+#ifdef VBOX_STRICT
+        /*
+         * Assert table sanity.
+         */
+        AssertMsg(paEntries[0].GCPhysLast >= paEntries[0].GCPhysFirst, ("%RGp %RGp\n", paEntries[0].GCPhysLast, paEntries[0].GCPhysFirst));
+        AssertMsg(paEntries[0].idx < pVM->iom.s.cMmioRegs, ("%#x %#x\n", paEntries[0].idx, pVM->iom.s.cMmioRegs));
+
+        RTGCPHYS GCPhysPrev = paEntries[0].GCPhysLast;
+        for (size_t i = 1; i <= cEntries; i++)
+        {
+            AssertMsg(paEntries[i].GCPhysLast >= paEntries[i].GCPhysFirst, ("%u: %RGp %RGp\n", i, paEntries[i].GCPhysLast, paEntries[i].GCPhysFirst));
+            AssertMsg(paEntries[i].idx < pVM->iom.s.cMmioRegs, ("%u: %#x %#x\n", i, paEntries[i].idx, pVM->iom.s.cMmioRegs));
+            AssertMsg(GCPhysPrev < paEntries[i].GCPhysFirst, ("%u: %RGp %RGp\n", i, GCPhysPrev, paEntries[i].GCPhysFirst));
+            GCPhysPrev = paEntries[i].GCPhysLast;
+        }
+#endif
+    }
+    else
+    {
+        AssertFailed();
+        rc = VERR_IOM_MMIO_REGION_ALREADY_MAPPED;
+    }
+
+    IOM_UNLOCK_EXCL(pVM);
+    return rc;
+}
+
+
+/**
+ * Worker for PDMDEVHLPR3::pfnMmioUnmap.
+ */
+VMMR3_INT_DECL(int)  IOMR3MmioUnmap(PVM pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
+{
+    /*
+     * Validate input and state.
+     */
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    AssertReturn(hRegion < pVM->iom.s.cMmioRegs, VERR_IOM_INVALID_MMIO_HANDLE);
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_IOM_INVALID_MMIO_HANDLE);
+
+    /*
+     * Do the mapping.
+     */
+    int rc;
+    IOM_LOCK_EXCL(pVM);
+
+    if (pRegEntry->fMapped)
+    {
+        RTGCPHYS const GCPhys     = pRegEntry->GCPhysMapping;
+        RTGCPHYS const GCPhysLast = GCPhys + pRegEntry->cbRegion - 1;
+        uint32_t const cEntries   = RT_MIN(pVM->iom.s.cMmioLookupEntries, pVM->iom.s.cMmioRegs);
+        Assert(pVM->iom.s.cMmioLookupEntries == cEntries);
+        Assert(cEntries > 0);
+
+        PIOMMMIOLOOKUPENTRY paEntries = pVM->iom.s.paMmioLookup;
+        uint32_t iFirst = 0;
+        uint32_t iEnd   = cEntries;
+        uint32_t i      = cEntries / 2;
+        for (;;)
+        {
+            PIOMMMIOLOOKUPENTRY pEntry = &paEntries[i];
+            if (pEntry->GCPhysLast < GCPhys)
+            {
+                i += 1;
+                if (i < iEnd)
+                    iFirst = i;
+                else
+                {
+                    rc = VERR_IOM_MMIO_IPE_1;
+                    AssertLogRelMsgFailedBreak(("%RGp..%RGp (%s) not found!\n", GCPhys, GCPhysLast, pRegEntry->pszDesc));
+                }
+            }
+            else if (pEntry->GCPhysFirst > GCPhysLast)
+            {
+                if (i > iFirst)
+                    iEnd = i;
+                else
+                {
+                    rc = VERR_IOM_MMIO_IPE_1;
+                    AssertLogRelMsgFailedBreak(("%RGp..%RGp (%s) not found!\n", GCPhys, GCPhysLast, pRegEntry->pszDesc));
+                }
+            }
+            else if (pEntry->idx == hRegion)
+            {
+                Assert(pEntry->GCPhysFirst == GCPhys);
+                Assert(pEntry->GCPhysLast == GCPhysLast);
+#ifdef VBOX_WITH_STATISTICS
+                iomR3MmioDeregStats(pVM, pRegEntry, GCPhys);
+#endif
+                if (i + 1 < cEntries)
+                    memmove(pEntry, pEntry + 1, sizeof(*pEntry) * (cEntries - i - 1));
+                pVM->iom.s.cMmioLookupEntries = cEntries - 1;
+
+                rc = PGMR3PhysMMIODeregister(pVM, GCPhys, pRegEntry->cbRegion);
+                AssertRC(rc);
+
+                pRegEntry->fMapped = false;
+                ASMAtomicWriteU64(&pRegEntry->GCPhysMapping, NIL_RTGCPHYS);
+                break;
+            }
+            else
+            {
+                AssertLogRelMsgFailed(("Lookig for %RGp..%RGp (%s), found %RGp..%RGp (%s) instead!\n",
+                                       GCPhys, GCPhysLast, pRegEntry->pszDesc,
+                                       pEntry->GCPhysFirst, pEntry->GCPhysLast, pVM->iom.s.paMmioRegs[pEntry->idx].pszDesc));
+                rc = VERR_IOM_MMIO_IPE_1;
+                break;
+            }
+
+            i = iFirst + (iEnd - iFirst) / 2;
+        }
+
+#ifdef VBOX_STRICT
+        /*
+         * Assert table sanity.
+         */
+        AssertMsg(paEntries[0].GCPhysLast >= paEntries[0].GCPhysFirst, ("%RGp %RGp\n", paEntries[0].GCPhysLast, paEntries[0].GCPhysFirst));
+        AssertMsg(paEntries[0].idx < pVM->iom.s.cMmioRegs, ("%#x %#x\n", paEntries[0].idx, pVM->iom.s.cMmioRegs));
+
+        RTGCPHYS GCPhysPrev = paEntries[0].GCPhysLast;
+        for (i = 1; i < cEntries - 1; i++)
+        {
+            AssertMsg(paEntries[i].GCPhysLast >= paEntries[i].GCPhysFirst, ("%u: %RGp %RGp\n", i, paEntries[i].GCPhysLast, paEntries[i].GCPhysFirst));
+            AssertMsg(paEntries[i].idx < pVM->iom.s.cMmioRegs, ("%u: %#x %#x\n", i, paEntries[i].idx, pVM->iom.s.cMmioRegs));
+            AssertMsg(GCPhysPrev < paEntries[i].GCPhysFirst, ("%u: %RGp %RGp\n", i, GCPhysPrev, paEntries[i].GCPhysFirst));
+            GCPhysPrev = paEntries[i].GCPhysLast;
+        }
+#endif
+    }
+    else
+    {
+        AssertFailed();
+        rc = VERR_IOM_MMIO_REGION_NOT_MAPPED;
+    }
+
+    IOM_UNLOCK_EXCL(pVM);
+    return rc;
+}
+
+
+VMMR3_INT_DECL(int)  IOMR3MmioReduce(PVM pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS cbRegion)
+{
+    RT_NOREF(pVM, pDevIns, hRegion, cbRegion);
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * Validates @a hRegion, making sure it belongs to @a pDevIns.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     The device which allegedly owns @a hRegion.
+ * @param   hRegion     The handle to validate.
+ */
+VMMR3_INT_DECL(int)  IOMR3MmioValidateHandle(PVM pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
+{
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_IOM_INVALID_MMIO_HANDLE);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the mapping address of MMIO region @a hRegion.
+ *
+ * @returns Mapping address if mapped, NIL_RTGCPHYS if not mapped or invalid
+ *          input.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     The device which allegedly owns @a hRegion.
+ * @param   hRegion     The handle to validate.
+ */
+VMMR3_INT_DECL(RTGCPHYS) IOMR3MmioGetMappingAddress(PVM pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
+{
+    AssertPtrReturn(pDevIns, NIL_RTGCPHYS);
+    AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), NIL_RTGCPHYS);
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
+    AssertReturn(pRegEntry->pDevIns == pDevIns, NIL_RTGCPHYS);
+    return pRegEntry->GCPhysMapping;
+}
+
+
+/**
+ * Display all registered MMIO ranges.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+DECLCALLBACK(void) iomR3MmioInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    RT_NOREF(pszArgs);
+
+    /* No locking needed here as registerations are only happening during VMSTATE_CREATING. */
+    pHlp->pfnPrintf(pHlp,
+                    "MMIO registrations: %u (%u allocated)\n"
+                    " ## Ctx    %.*s %.*s   PCI    Description\n",
+                    pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc,
+                    sizeof(RTGCPHYS) * 2, "Size",
+                    sizeof(RTGCPHYS) * 2 * 2 + 1, "Mapping");
+    PIOMMMIOENTRYR3 paRegs = pVM->iom.s.paMmioRegs;
+    for (uint32_t i = 0; i < pVM->iom.s.cMmioRegs; i++)
+    {
+        const char * const pszRing = paRegs[i].fRing0 ? paRegs[i].fRawMode ? "+0+C" : "+0  "
+                                   : paRegs[i].fRawMode ? "+C  " : "    ";
+        if (paRegs[i].fMapped && paRegs[i].pPciDev)
+            pHlp->pfnPrintf(pHlp, "%3u R3%s %RGp  %RGp-%RGp pci%u/%u %s\n", paRegs[i].idxSelf, pszRing, paRegs[i].cbRegion,
+                            paRegs[i].GCPhysMapping, paRegs[i].GCPhysMapping + paRegs[i].cbRegion - 1,
+                            paRegs[i].pPciDev->idxSubDev, paRegs[i].iPciRegion, paRegs[i].pszDesc);
+        else if (paRegs[i].fMapped && !paRegs[i].pPciDev)
+            pHlp->pfnPrintf(pHlp, "%3u R3%s %RGp  %RGp-%RGp        %s\n", paRegs[i].idxSelf, pszRing, paRegs[i].cbRegion,
+                            paRegs[i].GCPhysMapping, paRegs[i].GCPhysMapping + paRegs[i].cbRegion - 1, paRegs[i].pszDesc);
+        else if (paRegs[i].pPciDev)
+            pHlp->pfnPrintf(pHlp, "%3u R3%s %RGp  %.*s pci%u/%u %s\n", paRegs[i].idxSelf, pszRing, paRegs[i].cbRegion,
+                            sizeof(RTGCPHYS) * 2, "unmapped", paRegs[i].pPciDev->idxSubDev, paRegs[i].iPciRegion, paRegs[i].pszDesc);
+        else
+            pHlp->pfnPrintf(pHlp, "%3u R3%s %RGp  %.*s        %s\n", paRegs[i].idxSelf, pszRing, paRegs[i].cbRegion,
+                            sizeof(RTGCPHYS) * 2, "unmapped", paRegs[i].pszDesc);
+    }
+}
+
diff --git a/src/VBox/VMM/VMMR3/MM.cpp b/src/VBox/VMM/VMMR3/MM.cpp
new file mode 100644
index 00000000..2696ce61
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/MM.cpp
@@ -0,0 +1,849 @@
+/* $Id: MM.cpp $ */
+/** @file
+ * MM - Memory Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_mm     MM - The Memory Manager
+ *
+ * The memory manager is in charge of the following memory:
+ *      - Hypervisor Memory Area (HMA) - Address space management (obsolete in 6.1).
+ *      - Hypervisor Heap - A memory heap that lives in all contexts.
+ *      - User-Kernel Heap - A memory heap lives in both host context.
+ *      - Tagged ring-3 heap.
+ *      - Page pools - Primarily used by PGM for shadow page tables.
+ *      - Locked process memory - Guest RAM and other. (reduce/obsolete this)
+ *      - Physical guest memory (RAM & ROM) - Moving to PGM. (obsolete this)
+ *
+ * The global memory manager (GMM) is the global counter part / partner of MM.
+ * MM will provide therefore ring-3 callable interfaces for some of the GMM APIs
+ * related to resource tracking (PGM is the user).
+ *
+ * @see grp_mm
+ *
+ *
+ * @section sec_mm_hma  Hypervisor Memory Area - Obsolete in 6.1
+ *
+ * The HMA is used when executing in raw-mode. We borrow, with the help of
+ * PGMMap, some unused space (one or more page directory entries to be precise)
+ * in the guest's virtual memory context. PGM will monitor the guest's virtual
+ * address space for changes and relocate the HMA when required.
+ *
+ * To give some idea what's in the HMA, study the 'info hma' output:
+ * @verbatim
+VBoxDbg> info hma
+Hypervisor Memory Area (HMA) Layout: Base 00000000a0000000, 0x00800000 bytes
+00000000a05cc000-00000000a05cd000                  DYNAMIC                  fence
+00000000a05c4000-00000000a05cc000                  DYNAMIC                  Dynamic mapping
+00000000a05c3000-00000000a05c4000                  DYNAMIC                  fence
+00000000a05b8000-00000000a05c3000                  DYNAMIC                  Paging
+00000000a05b6000-00000000a05b8000                  MMIO2   0000000000000000 PCNetShMem
+00000000a0536000-00000000a05b6000                  MMIO2   0000000000000000 VGA VRam
+00000000a0523000-00000000a0536000 00002aaab3d0c000 LOCKED  autofree         alloc once (PDM_DEVICE)
+00000000a0522000-00000000a0523000                  DYNAMIC                  fence
+00000000a051e000-00000000a0522000 00002aaab36f5000 LOCKED  autofree         VBoxDD2RC.rc
+00000000a051d000-00000000a051e000                  DYNAMIC                  fence
+00000000a04eb000-00000000a051d000 00002aaab36c3000 LOCKED  autofree         VBoxDDRC.rc
+00000000a04ea000-00000000a04eb000                  DYNAMIC                  fence
+00000000a04e9000-00000000a04ea000 00002aaab36c2000 LOCKED  autofree         ram range (High ROM Region)
+00000000a04e8000-00000000a04e9000                  DYNAMIC                  fence
+00000000a040e000-00000000a04e8000 00002aaab2e6d000 LOCKED  autofree         VMMRC.rc
+00000000a0208000-00000000a040e000 00002aaab2c67000 LOCKED  autofree         alloc once (PATM)
+00000000a01f7000-00000000a0208000 00002aaaab92d000 LOCKED  autofree         alloc once (SELM)
+00000000a01e7000-00000000a01f7000 00002aaaab5e8000 LOCKED  autofree         alloc once (SELM)
+00000000a01e6000-00000000a01e7000                  DYNAMIC                  fence
+00000000a01e5000-00000000a01e6000 00002aaaab5e7000 HCPHYS  00000000c363c000 Core Code
+00000000a01e4000-00000000a01e5000                  DYNAMIC                  fence
+00000000a01e3000-00000000a01e4000 00002aaaaab26000 HCPHYS  00000000619cf000 GIP
+00000000a01a2000-00000000a01e3000 00002aaaabf32000 LOCKED  autofree         alloc once (PGM_PHYS)
+00000000a016b000-00000000a01a2000 00002aaab233f000 LOCKED  autofree         alloc once (PGM_POOL)
+00000000a016a000-00000000a016b000                  DYNAMIC                  fence
+00000000a0165000-00000000a016a000                  DYNAMIC                  CR3 mapping
+00000000a0164000-00000000a0165000                  DYNAMIC                  fence
+00000000a0024000-00000000a0164000 00002aaab215f000 LOCKED  autofree         Heap
+00000000a0023000-00000000a0024000                  DYNAMIC                  fence
+00000000a0001000-00000000a0023000 00002aaab1d24000 LOCKED  pages            VM
+00000000a0000000-00000000a0001000                  DYNAMIC                  fence
+ @endverbatim
+ *
+ *
+ * @section sec_mm_hyperheap    Hypervisor Heap
+ *
+ * The heap is accessible from ring-3, ring-0 and the raw-mode context. That
+ * said, it's not necessarily mapped into ring-0 on if that's possible since we
+ * don't wish to waste kernel address space without a good reason.
+ *
+ * Allocations within the heap are always in the same relative position in all
+ * contexts, so, it's possible to use offset based linking. In fact, the heap is
+ * internally using offset based linked lists tracking heap blocks. We use
+ * offset linked AVL trees and lists in a lot of places where share structures
+ * between RC, R3 and R0, so this is a strict requirement of the heap. However
+ * this means that we cannot easily extend the heap since the extension won't
+ * necessarily be in the continuation of the current heap memory in all (or any)
+ * context.
+ *
+ * All allocations are tagged. Per tag allocation statistics will be maintaining
+ * and exposed thru STAM when VBOX_WITH_STATISTICS is defined.
+ *
+ *
+ * @section sec_mm_r3heap   Tagged Ring-3 Heap
+ *
+ * The ring-3 heap is a wrapper around the RTMem API adding allocation
+ * statistics and automatic cleanup on VM destruction.
+ *
+ * Per tag allocation statistics will be maintaining and exposed thru STAM when
+ * VBOX_WITH_STATISTICS is defined.
+ *
+ *
+ * @section sec_mm_page     Page Pool
+ *
+ * The MM manages a page pool from which other components can allocate locked,
+ * page aligned and page sized memory objects. The pool provides facilities to
+ * convert back and forth between (host) physical and virtual addresses (within
+ * the pool of course). Several specialized interfaces are provided for the most
+ * common allocations and conversions to save the caller from bothersome casting
+ * and extra parameter passing.
+ *
+ *
+ * @section sec_mm_locked   Locked Process Memory
+ *
+ * MM manages the locked process memory. This is used for a bunch of things
+ * (count the LOCKED entries in the 'info hma' output found in @ref sec_mm_hma),
+ * but the main consumer of memory is currently for guest RAM. There is an
+ * ongoing rewrite that will move all the guest RAM allocation to PGM and
+ * GMM.
+ *
+ * The locking of memory is something doing in cooperation with the VirtualBox
+ * support driver, SUPDrv (aka. VBoxDrv), thru the support library API,
+ * SUPR3 (aka. SUPLib).
+ *
+ *
+ * @section sec_mm_phys     Physical Guest Memory
+ *
+ * MM is currently managing the physical memory for the guest. It relies heavily
+ * on PGM for this. There is an ongoing rewrite that will move this to PGM. (The
+ * rewrite is driven by the need for more flexible guest ram allocation, but
+ * also motivated by the fact that MMPhys is just adding stupid bureaucracy and
+ * that MMR3PhysReserve is a totally weird artifact that must go away.)
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/gmm.h>
+#include "MMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+
+#include <VBox/log.h>
+#include <iprt/alloc.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The current saved state version of MM. */
+#define MM_SAVED_STATE_VERSION      2
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(int) mmR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int) mmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+
+
+
+
+/**
+ * Initializes the MM members of the UVM.
+ *
+ * This is currently only the ring-3 heap.
+ *
+ * @returns VBox status code.
+ * @param   pUVM    Pointer to the user mode VM structure.
+ */
+VMMR3DECL(int) MMR3InitUVM(PUVM pUVM)
+{
+    /*
+     * Assert sizes and order.
+     */
+    AssertCompile(sizeof(pUVM->mm.s) <= sizeof(pUVM->mm.padding));
+    AssertRelease(sizeof(pUVM->mm.s) <= sizeof(pUVM->mm.padding));
+    Assert(!pUVM->mm.s.pHeap);
+
+    /*
+     * Init the heap.
+     */
+    int rc = mmR3HeapCreateU(pUVM, &pUVM->mm.s.pHeap);
+    if (RT_SUCCESS(rc))
+    {
+        rc = mmR3UkHeapCreateU(pUVM, &pUVM->mm.s.pUkHeap);
+        if (RT_SUCCESS(rc))
+            return VINF_SUCCESS;
+        mmR3HeapDestroy(pUVM->mm.s.pHeap);
+        pUVM->mm.s.pHeap = NULL;
+    }
+    return rc;
+}
+
+
+/**
+ * Initializes the MM.
+ *
+ * MM is managing the virtual address space (among other things) and
+ * setup the hypervisor memory area mapping in the VM structure and
+ * the hypervisor alloc-only-heap. Assuming the current init order
+ * and components the hypervisor memory area looks like this:
+ *      -# VM Structure.
+ *      -# Hypervisor alloc only heap (also call Hypervisor memory region).
+ *      -# Core code.
+ *
+ * MM determines the virtual address of the hypervisor memory area by
+ * checking for location at previous run. If that property isn't available
+ * it will choose a default starting location, currently 0xa0000000.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(int) MMR3Init(PVM pVM)
+{
+    LogFlow(("MMR3Init\n"));
+
+    /*
+     * Assert alignment, sizes and order.
+     */
+    AssertRelease(!(RT_UOFFSETOF(VM, mm.s) & 31));
+    AssertRelease(sizeof(pVM->mm.s) <= sizeof(pVM->mm.padding));
+    AssertMsg(pVM->mm.s.offVM == 0, ("Already initialized!\n"));
+
+    /*
+     * Init the structure.
+     */
+    pVM->mm.s.offVM = RT_UOFFSETOF(VM, mm);
+    pVM->mm.s.offLookupHyper = NIL_OFFSET;
+
+    /*
+     * Init the hypervisor related stuff.
+     */
+    int rc = mmR3HyperInit(pVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Register the saved state data unit.
+         */
+        rc = SSMR3RegisterInternal(pVM, "mm", 1, MM_SAVED_STATE_VERSION, sizeof(uint32_t) * 2,
+                                   NULL, NULL, NULL,
+                                   NULL, mmR3Save, NULL,
+                                   NULL, mmR3Load, NULL);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Statistics.
+             */
+            STAM_REG(pVM, &pVM->mm.s.cBasePages,   STAMTYPE_U64, "/MM/Reserved/cBasePages",   STAMUNIT_PAGES, "Reserved number of base pages, ROM and Shadow ROM included.");
+            STAM_REG(pVM, &pVM->mm.s.cHandyPages,  STAMTYPE_U32, "/MM/Reserved/cHandyPages",  STAMUNIT_PAGES, "Reserved number of handy pages.");
+            STAM_REG(pVM, &pVM->mm.s.cShadowPages, STAMTYPE_U32, "/MM/Reserved/cShadowPages", STAMUNIT_PAGES, "Reserved number of shadow paging pages.");
+            STAM_REG(pVM, &pVM->mm.s.cFixedPages,  STAMTYPE_U32, "/MM/Reserved/cFixedPages",  STAMUNIT_PAGES, "Reserved number of fixed pages (MMIO2).");
+            STAM_REG(pVM, &pVM->mm.s.cbRamBase,    STAMTYPE_U64, "/MM/cbRamBase",             STAMUNIT_BYTES, "Size of the base RAM.");
+
+            return rc;
+        }
+
+        /* .... failure .... */
+    }
+    MMR3Term(pVM);
+    return rc;
+}
+
+
+/**
+ * Initializes the MM parts which depends on PGM being initialized.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @remark  No cleanup necessary since MMR3Term() will be called on failure.
+ */
+VMMR3DECL(int) MMR3InitPaging(PVM pVM)
+{
+    LogFlow(("MMR3InitPaging:\n"));
+
+    /*
+     * Query the CFGM values.
+     */
+    int rc;
+    PCFGMNODE pMMCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "MM");
+    if (!pMMCfg)
+    {
+        rc = CFGMR3InsertNode(CFGMR3GetRoot(pVM), "MM", &pMMCfg);
+        AssertRCReturn(rc, rc);
+    }
+
+    /** @cfgm{/RamSize, uint64_t, 0, 16TB, 0}
+     * Specifies the size of the base RAM that is to be set up during
+     * VM initialization.
+     */
+    uint64_t cbRam;
+    rc = CFGMR3QueryU64(CFGMR3GetRoot(pVM), "RamSize", &cbRam);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        cbRam = 0;
+    else
+        AssertMsgRCReturn(rc, ("Configuration error: Failed to query integer \"RamSize\", rc=%Rrc.\n", rc), rc);
+    AssertLogRelMsg(!(cbRam & ~X86_PTE_PAE_PG_MASK), ("%RGp X86_PTE_PAE_PG_MASK=%RX64\n", cbRam, X86_PTE_PAE_PG_MASK));
+    AssertLogRelMsgReturn(cbRam <= GMM_GCPHYS_LAST, ("cbRam=%RGp GMM_GCPHYS_LAST=%RX64\n", cbRam, GMM_GCPHYS_LAST), VERR_OUT_OF_RANGE);
+    cbRam &= X86_PTE_PAE_PG_MASK;
+    pVM->mm.s.cbRamBase = cbRam;
+
+    /** @cfgm{/RamHoleSize, uint32_t, 0, 4032MB, 512MB}
+     * Specifies the size of the memory hole. The memory hole is used
+     * to avoid mapping RAM to the range normally used for PCI memory regions.
+     * Must be aligned on a 4MB boundary. */
+    uint32_t cbRamHole;
+    rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "RamHoleSize", &cbRamHole, MM_RAM_HOLE_SIZE_DEFAULT);
+    AssertLogRelMsgRCReturn(rc, ("Configuration error: Failed to query integer \"RamHoleSize\", rc=%Rrc.\n", rc), rc);
+    AssertLogRelMsgReturn(cbRamHole <= 4032U * _1M,
+                          ("Configuration error: \"RamHoleSize\"=%#RX32 is too large.\n", cbRamHole), VERR_OUT_OF_RANGE);
+    AssertLogRelMsgReturn(cbRamHole > 16 * _1M,
+                          ("Configuration error: \"RamHoleSize\"=%#RX32 is too large.\n", cbRamHole), VERR_OUT_OF_RANGE);
+    AssertLogRelMsgReturn(!(cbRamHole & (_4M - 1)),
+                          ("Configuration error: \"RamHoleSize\"=%#RX32 is misaligned.\n", cbRamHole), VERR_OUT_OF_RANGE);
+    uint64_t const offRamHole = _4G - cbRamHole;
+    if (cbRam < offRamHole)
+        Log(("MM: %RU64 bytes of RAM\n", cbRam));
+    else
+        Log(("MM: %RU64 bytes of RAM with a hole at %RU64 up to 4GB.\n", cbRam, offRamHole));
+
+    /** @cfgm{/MM/Policy, string, no overcommitment}
+     * Specifies the policy to use when reserving memory for this VM. The recognized
+     * value is 'no overcommitment' (default). See GMMPOLICY.
+     */
+    GMMOCPOLICY enmOcPolicy;
+    char sz[64];
+    rc = CFGMR3QueryString(CFGMR3GetRoot(pVM), "Policy", sz, sizeof(sz));
+    if (RT_SUCCESS(rc))
+    {
+        if (    !RTStrICmp(sz, "no_oc")
+            ||  !RTStrICmp(sz, "no overcommitment"))
+            enmOcPolicy = GMMOCPOLICY_NO_OC;
+        else
+            return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, "Unknown \"MM/Policy\" value \"%s\"", sz);
+    }
+    else if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        enmOcPolicy = GMMOCPOLICY_NO_OC;
+    else
+        AssertMsgFailedReturn(("Configuration error: Failed to query string \"MM/Policy\", rc=%Rrc.\n", rc), rc);
+
+    /** @cfgm{/MM/Priority, string, normal}
+     * Specifies the memory priority of this VM. The priority comes into play when the
+     * system is overcommitted and the VMs needs to be milked for memory. The recognized
+     * values are 'low', 'normal' (default) and 'high'. See GMMPRIORITY.
+     */
+    GMMPRIORITY enmPriority;
+    rc = CFGMR3QueryString(CFGMR3GetRoot(pVM), "Priority", sz, sizeof(sz));
+    if (RT_SUCCESS(rc))
+    {
+        if (!RTStrICmp(sz, "low"))
+            enmPriority = GMMPRIORITY_LOW;
+        else if (!RTStrICmp(sz, "normal"))
+            enmPriority = GMMPRIORITY_NORMAL;
+        else if (!RTStrICmp(sz, "high"))
+            enmPriority = GMMPRIORITY_HIGH;
+        else
+            return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, "Unknown \"MM/Priority\" value \"%s\"", sz);
+    }
+    else if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        enmPriority = GMMPRIORITY_NORMAL;
+    else
+        AssertMsgFailedReturn(("Configuration error: Failed to query string \"MM/Priority\", rc=%Rrc.\n", rc), rc);
+
+    /*
+     * Make the initial memory reservation with GMM.
+     */
+    uint64_t cBasePages = (cbRam >> PAGE_SHIFT) + pVM->mm.s.cBasePages;
+    rc = GMMR3InitialReservation(pVM,
+                                 RT_MAX(cBasePages + pVM->mm.s.cHandyPages, 1),
+                                 RT_MAX(pVM->mm.s.cShadowPages, 1),
+                                 RT_MAX(pVM->mm.s.cFixedPages, 1),
+                                 enmOcPolicy,
+                                 enmPriority);
+    if (RT_FAILURE(rc))
+    {
+        if (rc == VERR_GMM_MEMORY_RESERVATION_DECLINED)
+            return VMSetError(pVM, rc, RT_SRC_POS,
+                              N_("Insufficient free memory to start the VM (cbRam=%#RX64 enmOcPolicy=%d enmPriority=%d)"),
+                              cbRam, enmOcPolicy, enmPriority);
+        return VMSetError(pVM, rc, RT_SRC_POS, "GMMR3InitialReservation(,%#RX64,0,0,%d,%d)",
+                          cbRam >> PAGE_SHIFT, enmOcPolicy, enmPriority);
+    }
+
+    /*
+     * If RamSize is 0 we're done now.
+     */
+    if (cbRam < PAGE_SIZE)
+    {
+        Log(("MM: No RAM configured\n"));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Setup the base ram (PGM).
+     */
+    pVM->mm.s.cbRamHole = cbRamHole;
+    if (cbRam > offRamHole)
+    {
+        pVM->mm.s.cbRamBelow4GB = offRamHole;
+        rc = PGMR3PhysRegisterRam(pVM, 0, offRamHole, "Base RAM");
+        if (RT_SUCCESS(rc))
+        {
+            pVM->mm.s.cbRamAbove4GB = cbRam - offRamHole;
+            rc = PGMR3PhysRegisterRam(pVM, _4G, cbRam - offRamHole, "Above 4GB Base RAM");
+        }
+    }
+    else
+    {
+        pVM->mm.s.cbRamBelow4GB = cbRam;
+        pVM->mm.s.cbRamAbove4GB = 0;
+        rc = PGMR3PhysRegisterRam(pVM, 0, cbRam, "Base RAM");
+    }
+
+    /*
+     * Enabled mmR3UpdateReservation here since we don't want the
+     * PGMR3PhysRegisterRam calls above mess things up.
+     */
+    pVM->mm.s.fDoneMMR3InitPaging = true;
+    AssertMsg(pVM->mm.s.cBasePages == cBasePages || RT_FAILURE(rc), ("%RX64 != %RX64\n", pVM->mm.s.cBasePages, cBasePages));
+
+    LogFlow(("MMR3InitPaging: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Terminates the MM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(int) MMR3Term(PVM pVM)
+{
+    /*
+     * Clean up the hypervisor heap.
+     */
+    mmR3HyperTerm(pVM);
+
+    /*
+     * Zero stuff to detect after termination use of the MM interface
+     */
+    pVM->mm.s.offLookupHyper = NIL_OFFSET;
+    pVM->mm.s.pHyperHeapR3   = NULL;        /* freed above. */
+    pVM->mm.s.pHyperHeapR0   = NIL_RTR0PTR; /* freed above. */
+    pVM->mm.s.pHyperHeapRC   = NIL_RTRCPTR; /* freed above. */
+    pVM->mm.s.offVM          = 0;           /* init assertion on this */
+
+    /*
+     * Destroy the User-kernel heap here since the support driver session
+     * may have been terminated by the time we get to MMR3TermUVM.
+     */
+    mmR3UkHeapDestroy(pVM->pUVM->mm.s.pUkHeap);
+    pVM->pUVM->mm.s.pUkHeap = NULL;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminates the UVM part of MM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+VMMR3DECL(void) MMR3TermUVM(PUVM pUVM)
+{
+    /*
+     * Destroy the heaps.
+     */
+    if (pUVM->mm.s.pUkHeap)
+    {
+        mmR3UkHeapDestroy(pUVM->mm.s.pUkHeap);
+        pUVM->mm.s.pUkHeap = NULL;
+    }
+    mmR3HeapDestroy(pUVM->mm.s.pHeap);
+    pUVM->mm.s.pHeap = NULL;
+}
+
+
+/**
+ * Checks if the both VM and UVM parts of MM have been initialized.
+ *
+ * @returns true if initialized, false if not.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(bool) MMR3IsInitialized(PVM pVM)
+{
+    return pVM->mm.s.pHyperHeapR3 != NULL;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) mmR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("mmR3Save:\n"));
+
+    /* (PGM saves the physical memory.) */
+    SSMR3PutU64(pSSM, pVM->mm.s.cBasePages);
+    return SSMR3PutU64(pSSM, pVM->mm.s.cbRamBase);
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion       Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) mmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    LogFlow(("mmR3Load:\n"));
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+
+    /*
+     * Validate version.
+     */
+    if (    SSM_VERSION_MAJOR_CHANGED(uVersion, MM_SAVED_STATE_VERSION)
+        ||  !uVersion)
+    {
+        AssertMsgFailed(("mmR3Load: Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    /*
+     * Check the cBasePages and cbRamBase values.
+     */
+    int rc;
+    RTUINT cb1;
+
+    /* cBasePages (ignored) */
+    uint64_t cPages;
+    if (uVersion >= 2)
+        rc = SSMR3GetU64(pSSM, &cPages);
+    else
+    {
+        rc = SSMR3GetUInt(pSSM, &cb1);
+        cPages = cb1 >> PAGE_SHIFT;
+    }
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* cbRamBase */
+    uint64_t cb;
+    if (uVersion != 1)
+        rc = SSMR3GetU64(pSSM, &cb);
+    else
+    {
+        rc = SSMR3GetUInt(pSSM, &cb1);
+        cb = cb1;
+    }
+    if (RT_FAILURE(rc))
+        return rc;
+    AssertLogRelMsgReturn(cb == pVM->mm.s.cbRamBase,
+                          ("Memory configuration has changed. cbRamBase=%#RX64 save=%#RX64\n", pVM->mm.s.cbRamBase, cb),
+                          VERR_SSM_LOAD_MEMORY_SIZE_MISMATCH);
+
+    /* (PGM restores the physical memory.) */
+    return rc;
+}
+
+
+/**
+ * Updates GMM with memory reservation changes.
+ *
+ * Called when MM::cbRamRegistered, MM::cShadowPages or MM::cFixedPages changes.
+ *
+ * @returns VBox status code - see GMMR0UpdateReservation.
+ * @param   pVM             The cross context VM structure.
+ */
+int mmR3UpdateReservation(PVM pVM)
+{
+    VM_ASSERT_EMT(pVM);
+    if (pVM->mm.s.fDoneMMR3InitPaging)
+        return GMMR3UpdateReservation(pVM,
+                                      RT_MAX(pVM->mm.s.cBasePages + pVM->mm.s.cHandyPages, 1),
+                                      RT_MAX(pVM->mm.s.cShadowPages, 1),
+                                      RT_MAX(pVM->mm.s.cFixedPages, 1));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Interface for PGM to increase the reservation of RAM and ROM pages.
+ *
+ * This can be called before MMR3InitPaging.
+ *
+ * @returns VBox status code. Will set VM error on failure.
+ * @param   pVM             The cross context VM structure.
+ * @param   cAddBasePages   The number of pages to add.
+ */
+VMMR3DECL(int) MMR3IncreaseBaseReservation(PVM pVM, uint64_t cAddBasePages)
+{
+    uint64_t cOld = pVM->mm.s.cBasePages;
+    pVM->mm.s.cBasePages += cAddBasePages;
+    LogFlow(("MMR3IncreaseBaseReservation: +%RU64 (%RU64 -> %RU64\n", cAddBasePages, cOld, pVM->mm.s.cBasePages));
+    int rc = mmR3UpdateReservation(pVM);
+    if (RT_FAILURE(rc))
+    {
+        VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to reserved physical memory for the RAM (%#RX64 -> %#RX64 + %#RX32)"),
+                   cOld, pVM->mm.s.cBasePages, pVM->mm.s.cHandyPages);
+        pVM->mm.s.cBasePages = cOld;
+    }
+    return rc;
+}
+
+
+/**
+ * Interface for PGM to make reservations for handy pages in addition to the
+ * base memory.
+ *
+ * This can be called before MMR3InitPaging.
+ *
+ * @returns VBox status code. Will set VM error on failure.
+ * @param   pVM             The cross context VM structure.
+ * @param   cHandyPages     The number of handy pages.
+ */
+VMMR3DECL(int) MMR3ReserveHandyPages(PVM pVM, uint32_t cHandyPages)
+{
+    AssertReturn(!pVM->mm.s.cHandyPages, VERR_WRONG_ORDER);
+
+    pVM->mm.s.cHandyPages = cHandyPages;
+    LogFlow(("MMR3ReserveHandyPages: %RU32 (base %RU64)\n", pVM->mm.s.cHandyPages, pVM->mm.s.cBasePages));
+    int rc = mmR3UpdateReservation(pVM);
+    if (RT_FAILURE(rc))
+    {
+        VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to reserved physical memory for the RAM (%#RX64 + %#RX32)"),
+                   pVM->mm.s.cBasePages, pVM->mm.s.cHandyPages);
+        pVM->mm.s.cHandyPages = 0;
+    }
+    return rc;
+}
+
+
+/**
+ * Interface for PGM to adjust the reservation of fixed pages.
+ *
+ * This can be called before MMR3InitPaging.
+ *
+ * @returns VBox status code. Will set VM error on failure.
+ * @param   pVM                 The cross context VM structure.
+ * @param   cDeltaFixedPages    The number of pages to add (positive) or subtract (negative).
+ * @param   pszDesc             Some description associated with the reservation.
+ */
+VMMR3DECL(int) MMR3AdjustFixedReservation(PVM pVM, int32_t cDeltaFixedPages, const char *pszDesc)
+{
+    const uint32_t cOld = pVM->mm.s.cFixedPages;
+    pVM->mm.s.cFixedPages += cDeltaFixedPages;
+    LogFlow(("MMR3AdjustFixedReservation: %d (%u -> %u)\n", cDeltaFixedPages, cOld, pVM->mm.s.cFixedPages));
+    int rc = mmR3UpdateReservation(pVM);
+    if (RT_FAILURE(rc))
+    {
+        VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to reserve physical memory (%#x -> %#x; %s)"),
+                   cOld, pVM->mm.s.cFixedPages, pszDesc);
+        pVM->mm.s.cFixedPages = cOld;
+    }
+    return rc;
+}
+
+
+/**
+ * Interface for PGM to update the reservation of shadow pages.
+ *
+ * This can be called before MMR3InitPaging.
+ *
+ * @returns VBox status code. Will set VM error on failure.
+ * @param   pVM             The cross context VM structure.
+ * @param   cShadowPages    The new page count.
+ */
+VMMR3DECL(int) MMR3UpdateShadowReservation(PVM pVM, uint32_t cShadowPages)
+{
+    const uint32_t cOld = pVM->mm.s.cShadowPages;
+    pVM->mm.s.cShadowPages = cShadowPages;
+    LogFlow(("MMR3UpdateShadowReservation: %u -> %u\n", cOld, pVM->mm.s.cShadowPages));
+    int rc = mmR3UpdateReservation(pVM);
+    if (RT_FAILURE(rc))
+    {
+        VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to reserve physical memory for shadow page tables (%#x -> %#x)"), cOld, pVM->mm.s.cShadowPages);
+        pVM->mm.s.cShadowPages = cOld;
+    }
+    return rc;
+}
+
+
+/**
+ * Convert HC Physical address to HC Virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   HCPhys      The host context virtual address.
+ * @param   ppv         Where to store the resulting address.
+ * @thread  The Emulation Thread.
+ *
+ * @remarks Avoid whenever possible.
+ *          Intended for the debugger facility only.
+ * @todo    Rename to indicate the special usage.
+ */
+VMMR3DECL(int) MMR3HCPhys2HCVirt(PVM pVM, RTHCPHYS HCPhys, void **ppv)
+{
+#if 0
+    /*
+     * Try page tables.
+     */
+    int rc = MMPagePhys2PageTry(pVM, HCPhys, ppv);
+    if (RT_SUCCESS(rc))
+        return rc;
+#endif
+
+    /*
+     * Iterate thru the lookup records for HMA.
+     */
+    uint32_t off = HCPhys & PAGE_OFFSET_MASK;
+    HCPhys &= X86_PTE_PAE_PG_MASK;
+    PMMLOOKUPHYPER pCur = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.CTX_SUFF(pHyperHeap) + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        switch (pCur->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+            {
+                PCRTHCPHYS  paHCPhysPages = pCur->u.Locked.paHCPhysPages;
+                size_t      iPage         = pCur->cb >> PAGE_SHIFT;
+                while (iPage-- > 0)
+                    if (paHCPhysPages[iPage] == HCPhys)
+                    {
+                        *ppv = (char *)pCur->u.Locked.pvR3 + (iPage << PAGE_SHIFT) + off;
+                        return VINF_SUCCESS;
+                    }
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+                if (pCur->u.HCPhys.HCPhys - HCPhys < pCur->cb)
+                {
+                    *ppv = (uint8_t *)pCur->u.HCPhys.pvR3 + pCur->u.HCPhys.HCPhys - HCPhys + off;
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case MMLOOKUPHYPERTYPE_GCPHYS:  /* (for now we'll not allow these kind of conversions) */
+            case MMLOOKUPHYPERTYPE_MMIO2:
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+                break;
+
+            default:
+                AssertMsgFailed(("enmType=%d\n", pCur->enmType));
+                break;
+        }
+
+        /* next */
+        if (pCur->offNext ==  (int32_t)NIL_OFFSET)
+            break;
+        pCur = (PMMLOOKUPHYPER)((uint8_t *)pCur + pCur->offNext);
+    }
+    /* give up */
+    return VERR_INVALID_POINTER;
+}
+
+
+
+/**
+ * Get the size of the base RAM.
+ * This usually means the size of the first contiguous block of physical memory.
+ *
+ * @returns The guest base RAM size.
+ * @param   pVM         The cross context VM structure.
+ * @thread  Any.
+ *
+ * @deprecated
+ */
+VMMR3DECL(uint64_t) MMR3PhysGetRamSize(PVM pVM)
+{
+    return pVM->mm.s.cbRamBase;
+}
+
+
+/**
+ * Get the size of RAM below 4GB (starts at address 0x00000000).
+ *
+ * @returns The amount of RAM below 4GB in bytes.
+ * @param   pVM         The cross context VM structure.
+ * @thread  Any.
+ */
+VMMR3DECL(uint32_t) MMR3PhysGetRamSizeBelow4GB(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
+    return pVM->mm.s.cbRamBelow4GB;
+}
+
+
+/**
+ * Get the size of RAM above 4GB (starts at address 0x000100000000).
+ *
+ * @returns The amount of RAM above 4GB in bytes.
+ * @param   pVM         The cross context VM structure.
+ * @thread  Any.
+ */
+VMMR3DECL(uint64_t) MMR3PhysGetRamSizeAbove4GB(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT64_MAX);
+    return pVM->mm.s.cbRamAbove4GB;
+}
+
+
+/**
+ * Get the size of the RAM hole below 4GB.
+ *
+ * @returns Size in bytes.
+ * @param   pVM         The cross context VM structure.
+ * @thread  Any.
+ */
+VMMR3DECL(uint32_t) MMR3PhysGet4GBRamHoleSize(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
+    return pVM->mm.s.cbRamHole;
+}
+
diff --git a/src/VBox/VMM/VMMR3/MMHeap.cpp b/src/VBox/VMM/VMMR3/MMHeap.cpp
new file mode 100644
index 00000000..b33b376f
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/MMHeap.cpp
@@ -0,0 +1,751 @@
+/* $Id: MMHeap.cpp $ */
+/** @file
+ * MM - Memory Manager - Heap.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM_HEAP
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/pgm.h>
+#include "MMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <iprt/errcore.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+
+#include <iprt/alloc.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static void *mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero);
+
+
+
+/**
+ * Allocate and initialize a heap structure and it's associated substructures.
+ *
+ * @returns VBox status code.
+ * @param   pUVM    Pointer to the user mode VM structure.
+ * @param   ppHeap  Where to store the heap pointer.
+ */
+int mmR3HeapCreateU(PUVM pUVM, PMMHEAP *ppHeap)
+{
+    PMMHEAP pHeap = (PMMHEAP)RTMemAllocZ(sizeof(MMHEAP) + sizeof(MMHEAPSTAT));
+    if (pHeap)
+    {
+        int rc = RTCritSectInit(&pHeap->Lock);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Initialize the global stat record.
+             */
+            pHeap->pUVM = pUVM;
+            pHeap->Stat.pHeap = pHeap;
+#ifdef MMR3HEAP_WITH_STATISTICS
+            PMMHEAPSTAT pStat = &pHeap->Stat;
+            STAMR3RegisterU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cAllocations",     STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cReallocations",   STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cFrees",           STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cFailures",        STAMUNIT_COUNT, "Number of failures.");
+            STAMR3RegisterU(pUVM, &pStat->cbCurAllocated, sizeof(pStat->cbCurAllocated) == sizeof(uint32_t) ? STAMTYPE_U32 : STAMTYPE_U64,
+                                                                        STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbCurAllocated",   STAMUNIT_BYTES, "Number of bytes currently allocated.");
+            STAMR3RegisterU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbAllocated",      STAMUNIT_BYTES, "Total number of bytes allocated.");
+            STAMR3RegisterU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbFreed",          STAMUNIT_BYTES, "Total number of bytes freed.");
+#endif
+            *ppHeap = pHeap;
+            return VINF_SUCCESS;
+        }
+        AssertRC(rc);
+        RTMemFree(pHeap);
+    }
+    AssertMsgFailed(("failed to allocate heap structure\n"));
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * MM heap statistics tree destroy callback.
+ */
+static DECLCALLBACK(int) mmR3HeapStatTreeDestroy(PAVLULNODECORE pCore, void *pvParam)
+{
+    RT_NOREF(pvParam);
+
+    /* Don't bother deregistering the stat samples as they get destroyed by STAM. */
+    RTMemFree(pCore);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroy a heap.
+ *
+ * @param   pHeap   Heap handle.
+ */
+void mmR3HeapDestroy(PMMHEAP pHeap)
+{
+    /*
+     * Start by deleting the lock, that'll trap anyone
+     * attempting to use the heap.
+     */
+    RTCritSectDelete(&pHeap->Lock);
+
+    /*
+     * Walk the node list and free all the memory.
+     */
+    PMMHEAPHDR  pHdr = pHeap->pHead;
+    while (pHdr)
+    {
+        void *pv = pHdr;
+        pHdr = pHdr->pNext;
+        RTMemFree(pv);
+    }
+
+    /*
+     * Free the stat nodes.
+     */
+    RTAvlULDestroy(&pHeap->pStatTree, mmR3HeapStatTreeDestroy, NULL);
+    RTMemFree(pHeap);
+}
+
+
+/**
+ * Allocate memory associating it with the VM for collective cleanup.
+ *
+ * The memory will be allocated from the default heap but a header
+ * is added in which we keep track of which VM it belongs to and chain
+ * all the allocations together so they can be freed in one go.
+ *
+ * This interface is typically used for memory block which will not be
+ * freed during the life of the VM.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ */
+VMMR3DECL(void *) MMR3HeapAllocU(PUVM pUVM, MMTAG enmTag, size_t cbSize)
+{
+    Assert(pUVM->mm.s.pHeap);
+    return mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, false);
+}
+
+
+/**
+ * Allocate memory associating it with the VM for collective cleanup.
+ *
+ * The memory will be allocated from the default heap but a header
+ * is added in which we keep track of which VM it belongs to and chain
+ * all the allocations together so they can be freed in one go.
+ *
+ * This interface is typically used for memory block which will not be
+ * freed during the life of the VM.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ */
+VMMR3DECL(void *) MMR3HeapAlloc(PVM pVM, MMTAG enmTag, size_t cbSize)
+{
+    return mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, false);
+}
+
+
+/**
+ * Same as MMR3HeapAllocU().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ */
+VMMR3DECL(int) MMR3HeapAllocExU(PUVM pUVM, MMTAG enmTag, size_t cbSize, void **ppv)
+{
+    Assert(pUVM->mm.s.pHeap);
+    void *pv = mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, false);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Same as MMR3HeapAlloc().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ */
+VMMR3DECL(int) MMR3HeapAllocEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv)
+{
+    void *pv = mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, false);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Same as MMR3HeapAlloc() only the memory is zeroed.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ */
+VMMR3DECL(void *) MMR3HeapAllocZU(PUVM pUVM, MMTAG enmTag, size_t cbSize)
+{
+    return mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, true);
+}
+
+
+/**
+ * Same as MMR3HeapAlloc() only the memory is zeroed.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ */
+VMMR3DECL(void *) MMR3HeapAllocZ(PVM pVM, MMTAG enmTag, size_t cbSize)
+{
+    return mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, true);
+}
+
+
+/**
+ * Same as MMR3HeapAllocZ().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ */
+VMMR3DECL(int) MMR3HeapAllocZExU(PUVM pUVM, MMTAG enmTag, size_t cbSize, void **ppv)
+{
+    Assert(pUVM->mm.s.pHeap);
+    void *pv = mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, true);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Same as MMR3HeapAllocZ().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ */
+VMMR3DECL(int) MMR3HeapAllocZEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv)
+{
+    void *pv = mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, true);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Links @a pHdr into the heap block list (tail).
+ *
+ * @param   pHeap       Heap handle.
+ * @param   pHdr        The block to link.
+ *
+ * @note    Caller has locked the heap!
+ */
+DECLINLINE(void) mmR3HeapLink(PMMHEAP pHeap, PMMHEAPHDR pHdr)
+{
+    /* Tail insertion: */
+    pHdr->pNext = NULL;
+    PMMHEAPHDR pTail = pHeap->pTail;
+    pHdr->pPrev = pTail;
+    if (pTail)
+    {
+        Assert(!pTail->pNext);
+        pTail->pNext = pHdr;
+    }
+    else
+    {
+        Assert(!pHeap->pHead);
+        pHeap->pHead = pHdr;
+    }
+    pHeap->pTail = pHdr;
+}
+
+
+/**
+ * Unlinks @a pHdr from the heal block list.
+ *
+ * @param   pHeap       Heap handle.
+ * @param   pHdr        The block to unlink.
+ *
+ * @note    Caller has locked the heap!
+ */
+DECLINLINE(void) mmR3HeapUnlink(PMMHEAP pHeap, PMMHEAPHDR pHdr)
+{
+    PMMHEAPHDR const pPrev = pHdr->pPrev;
+    PMMHEAPHDR const pNext = pHdr->pNext;
+    if (pPrev)
+        pPrev->pNext = pNext;
+    else
+        pHeap->pHead = pNext;
+
+    if (pNext)
+        pNext->pPrev = pPrev;
+    else
+        pHeap->pTail = pHdr->pPrev;
+}
+
+
+/**
+ * Allocate memory from the heap.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pHeap       Heap handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ * @param   fZero       Whether or not to zero the memory block.
+ */
+void *mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero)
+{
+#ifdef MMR3HEAP_WITH_STATISTICS
+    RTCritSectEnter(&pHeap->Lock);
+
+    /*
+     * Find/alloc statistics nodes.
+     */
+    pHeap->Stat.cAllocations++;
+    PMMHEAPSTAT pStat = (PMMHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
+    if (pStat)
+    {
+        pStat->cAllocations++;
+
+        RTCritSectLeave(&pHeap->Lock);
+    }
+    else
+    {
+        pStat = (PMMHEAPSTAT)RTMemAllocZ(sizeof(MMHEAPSTAT));
+        if (!pStat)
+        {
+            pHeap->Stat.cFailures++;
+            AssertMsgFailed(("Failed to allocate heap stat record.\n"));
+            RTCritSectLeave(&pHeap->Lock);
+            return NULL;
+        }
+        pStat->Core.Key = (AVLULKEY)enmTag;
+        pStat->pHeap    = pHeap;
+        RTAvlULInsert(&pHeap->pStatTree, &pStat->Core);
+
+        pStat->cAllocations++;
+        RTCritSectLeave(&pHeap->Lock);
+
+        /* register the statistics */
+        PUVM pUVM = pHeap->pUVM;
+        const char *pszTag = mmGetTagName(enmTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Number of bytes currently allocated.",    "/MM/R3Heap/%s", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.",        "/MM/R3Heap/%s/cAllocations", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.",      "/MM/R3Heap/%s/cReallocations", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.",         "/MM/R3Heap/%s/cFrees", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_COUNT, "Number of failures.",                     "/MM/R3Heap/%s/cFailures", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes allocated.",        "/MM/R3Heap/%s/cbAllocated", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes freed.",            "/MM/R3Heap/%s/cbFreed", pszTag);
+    }
+#else
+    RT_NOREF_PV(enmTag);
+#endif
+
+    /*
+     * Validate input.
+     */
+    if (cbSize == 0)
+    {
+#ifdef MMR3HEAP_WITH_STATISTICS
+        RTCritSectEnter(&pHeap->Lock);
+        pStat->cFailures++;
+        pHeap->Stat.cFailures++;
+        RTCritSectLeave(&pHeap->Lock);
+#endif
+        AssertFailed();
+        return NULL;
+    }
+
+    /*
+     * Allocate heap block.
+     */
+    cbSize = RT_ALIGN_Z(cbSize, MMR3HEAP_SIZE_ALIGNMENT) + sizeof(MMHEAPHDR);
+    PMMHEAPHDR const pHdr = (PMMHEAPHDR)(fZero ? RTMemAllocZ(cbSize) : RTMemAlloc(cbSize));
+    if (pHdr)
+    { /* likely */ }
+    else
+    {
+        AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cbSize, enmTag, &enmTag));
+#ifdef MMR3HEAP_WITH_STATISTICS
+        RTCritSectEnter(&pHeap->Lock);
+        pStat->cFailures++;
+        pHeap->Stat.cFailures++;
+        RTCritSectLeave(&pHeap->Lock);
+#endif
+        return NULL;
+    }
+    Assert(!((uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1)));
+
+    /*
+     * Init and link in the header.
+     */
+#ifdef MMR3HEAP_WITH_STATISTICS
+    pHdr->pStat  = pStat;
+#else
+    pHdr->pStat  = &pHeap->Stat;
+#endif
+    pHdr->cbSize = cbSize;
+
+    RTCritSectEnter(&pHeap->Lock);
+
+    mmR3HeapLink(pHeap, pHdr);
+
+    /*
+     * Update statistics
+     */
+#ifdef MMR3HEAP_WITH_STATISTICS
+    pStat->cbAllocated          += cbSize;
+    pStat->cbCurAllocated       += cbSize;
+    pHeap->Stat.cbAllocated     += cbSize;
+    pHeap->Stat.cbCurAllocated  += cbSize;
+#endif
+
+    RTCritSectLeave(&pHeap->Lock);
+
+    return pHdr + 1;
+}
+
+
+/**
+ * Reallocate memory allocated with MMR3HeapAlloc(), MMR3HeapAllocZ() or
+ * MMR3HeapRealloc().
+ *
+ * Any additional memory is zeroed (only reliable if the initial allocation was
+ * also of the zeroing kind).
+ *
+ * @returns Pointer to reallocated memory.
+ * @param   pv          Pointer to the memory block to reallocate.
+ *                      Must not be NULL!
+ * @param   cbNewSize   New block size.
+ */
+VMMR3DECL(void *) MMR3HeapRealloc(void *pv, size_t cbNewSize)
+{
+    AssertMsg(pv, ("Invalid pointer pv=%p\n", pv));
+    if (!pv)
+        return NULL;
+
+    /*
+     * If newsize is zero then this is a free.
+     */
+    if (!cbNewSize)
+    {
+        MMR3HeapFree(pv);
+        return NULL;
+    }
+
+    /*
+     * Validate header.
+     */
+    PMMHEAPHDR const pHdr      = (PMMHEAPHDR)pv - 1;
+    size_t const     cbOldSize = pHdr->cbSize;
+    AssertMsgReturn(   !(cbOldSize & (MMR3HEAP_SIZE_ALIGNMENT - 1))
+                    && !((uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1)),
+                    ("Invalid heap header! pv=%p, size=%#x\n", pv, cbOldSize),
+                    NULL);
+    Assert(pHdr->pStat != NULL);
+    Assert(!((uintptr_t)pHdr->pNext & (RTMEM_ALIGNMENT - 1)));
+    Assert(!((uintptr_t)pHdr->pPrev & (RTMEM_ALIGNMENT - 1)));
+
+    PMMHEAP pHeap = pHdr->pStat->pHeap;
+
+    /*
+     * Unlink the header before we reallocate the block.
+     */
+    RTCritSectEnter(&pHeap->Lock);
+#ifdef MMR3HEAP_WITH_STATISTICS
+    pHdr->pStat->cReallocations++;
+    pHeap->Stat.cReallocations++;
+#endif
+    mmR3HeapUnlink(pHeap, pHdr);
+    RTCritSectLeave(&pHeap->Lock);
+
+    /*
+     * Reallocate the block.  Clear added space.
+     */
+    cbNewSize = RT_ALIGN_Z(cbNewSize, MMR3HEAP_SIZE_ALIGNMENT) + sizeof(MMHEAPHDR);
+    PMMHEAPHDR pHdrNew = (PMMHEAPHDR)RTMemReallocZ(pHdr, cbOldSize, cbNewSize);
+    if (pHdrNew)
+        pHdrNew->cbSize = cbNewSize;
+    else
+    {
+        RTCritSectEnter(&pHeap->Lock);
+        mmR3HeapLink(pHeap, pHdr);
+#ifdef MMR3HEAP_WITH_STATISTICS
+        pHdr->pStat->cFailures++;
+        pHeap->Stat.cFailures++;
+#endif
+        RTCritSectLeave(&pHeap->Lock);
+        return NULL;
+    }
+
+    RTCritSectEnter(&pHeap->Lock);
+
+    /*
+     * Relink the header.
+     */
+    mmR3HeapLink(pHeap, pHdrNew);
+
+    /*
+     * Update statistics.
+     */
+#ifdef MMR3HEAP_WITH_STATISTICS
+    pHdrNew->pStat->cbAllocated += cbNewSize - pHdrNew->cbSize;
+    pHeap->Stat.cbAllocated += cbNewSize - pHdrNew->cbSize;
+#endif
+
+    RTCritSectLeave(&pHeap->Lock);
+
+    return pHdrNew + 1;
+}
+
+
+/**
+ * Duplicates the specified string.
+ *
+ * @returns Pointer to the duplicate.
+ * @returns NULL on failure or when input NULL.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   psz         The string to duplicate. NULL is allowed.
+ */
+VMMR3DECL(char *) MMR3HeapStrDupU(PUVM pUVM, MMTAG enmTag, const char *psz)
+{
+    if (!psz)
+        return NULL;
+    AssertPtr(psz);
+
+    size_t cch = strlen(psz) + 1;
+    char *pszDup = (char *)MMR3HeapAllocU(pUVM, enmTag, cch);
+    if (pszDup)
+        memcpy(pszDup, psz, cch);
+    return pszDup;
+}
+
+
+/**
+ * Duplicates the specified string.
+ *
+ * @returns Pointer to the duplicate.
+ * @returns NULL on failure or when input NULL.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   psz         The string to duplicate. NULL is allowed.
+ */
+VMMR3DECL(char *) MMR3HeapStrDup(PVM pVM, MMTAG enmTag, const char *psz)
+{
+    return MMR3HeapStrDupU(pVM->pUVM, enmTag, psz);
+}
+
+
+/**
+ * Allocating string printf.
+ *
+ * @returns Pointer to the string.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmTag      The statistics tag.
+ * @param   pszFormat   The format string.
+ * @param   ...         Format arguments.
+ */
+VMMR3DECL(char *)    MMR3HeapAPrintf(PVM pVM, MMTAG enmTag, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    char *psz = MMR3HeapAPrintfVU(pVM->pUVM, enmTag, pszFormat, va);
+    va_end(va);
+    return psz;
+}
+
+
+/**
+ * Allocating string printf.
+ *
+ * @returns Pointer to the string.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      The statistics tag.
+ * @param   pszFormat   The format string.
+ * @param   ...         Format arguments.
+ */
+VMMR3DECL(char *)    MMR3HeapAPrintfU(PUVM pUVM, MMTAG enmTag, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    char *psz = MMR3HeapAPrintfVU(pUVM, enmTag, pszFormat, va);
+    va_end(va);
+    return psz;
+}
+
+
+/**
+ * Allocating string printf.
+ *
+ * @returns Pointer to the string.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmTag      The statistics tag.
+ * @param   pszFormat   The format string.
+ * @param   va          Format arguments.
+ */
+VMMR3DECL(char *)    MMR3HeapAPrintfV(PVM pVM, MMTAG enmTag, const char *pszFormat, va_list va)
+{
+    return MMR3HeapAPrintfVU(pVM->pUVM, enmTag, pszFormat, va);
+}
+
+
+/**
+ * Allocating string printf.
+ *
+ * @returns Pointer to the string.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      The statistics tag.
+ * @param   pszFormat   The format string.
+ * @param   va          Format arguments.
+ */
+VMMR3DECL(char *)    MMR3HeapAPrintfVU(PUVM pUVM, MMTAG enmTag, const char *pszFormat, va_list va)
+{
+    /*
+     * The lazy bird way.
+     */
+    char *psz;
+    int cch = RTStrAPrintfV(&psz, pszFormat, va);
+    if (cch < 0)
+        return NULL;
+    Assert(psz[cch] == '\0');
+    char *pszRet = (char *)MMR3HeapAllocU(pUVM, enmTag, cch + 1);
+    if (pszRet)
+        memcpy(pszRet, psz, cch + 1);
+    RTStrFree(psz);
+    return pszRet;
+}
+
+
+/**
+ * Releases memory allocated with MMR3HeapAlloc() or MMR3HeapRealloc().
+ *
+ * The memory is cleared/filled before freeing to prevent heap spraying, info
+ * leaks, and help detect use after free trouble.
+ *
+ * @param   pv          Pointer to the memory block to free.
+ */
+VMMR3DECL(void) MMR3HeapFree(void *pv)
+{
+    /* Ignore NULL pointers. */
+    if (!pv)
+        return;
+
+    /*
+     * Validate header.
+     */
+    PMMHEAPHDR const pHdr         = (PMMHEAPHDR)pv - 1;
+    size_t const     cbAllocation = pHdr->cbSize;
+    AssertMsgReturnVoid(   !(pHdr->cbSize & (MMR3HEAP_SIZE_ALIGNMENT - 1))
+                        && !((uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1)),
+                        ("Invalid heap header! pv=%p, size=%#x\n", pv, pHdr->cbSize));
+    AssertPtr(pHdr->pStat);
+    Assert(!((uintptr_t)pHdr->pNext & (RTMEM_ALIGNMENT - 1)));
+    Assert(!((uintptr_t)pHdr->pPrev & (RTMEM_ALIGNMENT - 1)));
+
+    /*
+     * Update statistics
+     */
+    PMMHEAP pHeap = pHdr->pStat->pHeap;
+    RTCritSectEnter(&pHeap->Lock);
+
+#ifdef MMR3HEAP_WITH_STATISTICS
+    pHdr->pStat->cFrees++;
+    pHeap->Stat.cFrees++;
+    pHdr->pStat->cbFreed            += cbAllocation;
+    pHeap->Stat.cbFreed             += cbAllocation;
+    pHdr->pStat->cbCurAllocated     -= cbAllocation;
+    pHeap->Stat.cbCurAllocated      -= cbAllocation;
+#endif
+
+    /*
+     * Unlink it.
+     */
+    mmR3HeapUnlink(pHeap, pHdr);
+
+    RTCritSectLeave(&pHeap->Lock);
+
+    /*
+     * Free the memory.  We clear just to be on the safe size wrt
+     * heap spraying and leaking sensitive info (also helps detecting
+     * double freeing).
+     */
+    RTMemFreeZ(pHdr, cbAllocation);
+}
+
diff --git a/src/VBox/VMM/VMMR3/MMHyper.cpp b/src/VBox/VMM/VMMR3/MMHyper.cpp
new file mode 100644
index 00000000..ed037ffc
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/MMHyper.cpp
@@ -0,0 +1,1516 @@
+/* $Id: MMHyper.cpp $ */
+/** @file
+ * MM - Memory Manager - Hypervisor Memory Area.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM_HYPER
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/dbgf.h>
+#include "MMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/gvm.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+#include <iprt/alloc.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+#ifndef PGM_WITHOUT_MAPPINGS
+static DECLCALLBACK(bool) mmR3HyperRelocateCallback(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode,
+                                                    void *pvUser);
+#endif
+static int mmR3HyperMap(PVM pVM, const size_t cb, const char *pszDesc, PRTGCPTR pGCPtr, PMMLOOKUPHYPER *ppLookup);
+static int mmR3HyperHeapCreate(PVM pVM, const size_t cb, PMMHYPERHEAP *ppHeap, PRTR0PTR pR0PtrHeap);
+static int mmR3HyperHeapMap(PVM pVM, PMMHYPERHEAP pHeap, PRTGCPTR ppHeapGC);
+static DECLCALLBACK(void) mmR3HyperInfoHma(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+/**
+ * Determin the default heap size.
+ *
+ * @returns The heap size in bytes.
+ * @param   pVM     The cross context VM structure.
+ */
+static uint32_t mmR3HyperComputeHeapSize(PVM pVM)
+{
+    /** @todo Redo after moving allocations off the hyper heap. */
+
+    /*
+     * Gather parameters.
+     */
+    bool        fCanUseLargerHeap = true;
+    //bool        fCanUseLargerHeap;
+    //int rc = CFGMR3QueryBoolDef(CFGMR3GetChild(CFGMR3GetRoot(pVM), "MM"), "CanUseLargerHeap", &fCanUseLargerHeap, false);
+    //AssertStmt(RT_SUCCESS(rc), fCanUseLargerHeap = false);
+
+    uint64_t    cbRam;
+    int rc = CFGMR3QueryU64(CFGMR3GetRoot(pVM), "RamSize", &cbRam);
+    AssertStmt(RT_SUCCESS(rc), cbRam = _1G);
+
+    /*
+     * We need to keep saved state compatibility if raw-mode is an option,
+     * so lets filter out that case first.
+     */
+    if (   !fCanUseLargerHeap
+        && VM_IS_RAW_MODE_ENABLED(pVM)
+        && cbRam < 16*_1G64)
+        return 1280 * _1K;
+
+    /*
+     * Calculate the heap size.
+     */
+    uint32_t cbHeap = _1M;
+
+    /* The newer chipset may have more devices attached, putting additional
+       pressure on the heap. */
+    if (fCanUseLargerHeap)
+        cbHeap += _1M;
+
+    /* More CPUs means some extra memory usage. */
+    if (pVM->cCpus > 1)
+        cbHeap += pVM->cCpus * _64K;
+
+    /* Lots of memory means extra memory consumption as well (pool). */
+    if (cbRam > 16*_1G64)
+        cbHeap += _2M; /** @todo figure out extactly how much */
+
+    return RT_ALIGN(cbHeap, _256K);
+}
+
+
+/**
+ * Initializes the hypervisor related MM stuff without
+ * calling down to PGM.
+ *
+ * PGM is not initialized at this  point, PGM relies on
+ * the heap to initialize.
+ *
+ * @returns VBox status code.
+ */
+int mmR3HyperInit(PVM pVM)
+{
+    LogFlow(("mmR3HyperInit:\n"));
+
+    /*
+     * Decide Hypervisor mapping in the guest context
+     * And setup various hypervisor area and heap parameters.
+     */
+    pVM->mm.s.pvHyperAreaGC = (RTGCPTR)MM_HYPER_AREA_ADDRESS;
+    pVM->mm.s.cbHyperArea   = MM_HYPER_AREA_MAX_SIZE;
+    AssertRelease(RT_ALIGN_T(pVM->mm.s.pvHyperAreaGC, 1 << X86_PD_SHIFT, RTGCPTR) == pVM->mm.s.pvHyperAreaGC);
+    Assert(pVM->mm.s.pvHyperAreaGC < 0xff000000);
+
+    /** @todo @bugref{1865}, @bugref{3202}: Change the cbHyperHeap default
+     *        depending on whether VT-x/AMD-V is enabled or not! Don't waste
+     *        precious kernel space on heap for the PATM.
+     */
+    PCFGMNODE pMM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "MM");
+    uint32_t cbHyperHeap;
+    int rc = CFGMR3QueryU32Def(pMM, "cbHyperHeap", &cbHyperHeap, mmR3HyperComputeHeapSize(pVM));
+    AssertLogRelRCReturn(rc, rc);
+
+    cbHyperHeap = RT_ALIGN_32(cbHyperHeap, PAGE_SIZE);
+    LogRel(("MM: cbHyperHeap=%#x (%u)\n", cbHyperHeap, cbHyperHeap));
+
+    /*
+     * Allocate the hypervisor heap.
+     *
+     * (This must be done before we start adding memory to the
+     * hypervisor static area because lookup records are allocated from it.)
+     */
+    rc = mmR3HyperHeapCreate(pVM, cbHyperHeap, &pVM->mm.s.pHyperHeapR3, &pVM->mm.s.pHyperHeapR0);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Make a small head fence to fend of accidental sequential access.
+         */
+        MMR3HyperReserveFence(pVM);
+
+        /*
+         * Map the VM structure into the hypervisor space.
+         * Note! Keeping the mappings here for now in case someone is using
+         *       MMHyperR3ToR0 or similar.
+         */
+        AssertCompileSizeAlignment(VM, PAGE_SIZE);
+        AssertCompileSizeAlignment(VMCPU, PAGE_SIZE);
+        AssertCompileSizeAlignment(GVM, PAGE_SIZE);
+        AssertCompileSizeAlignment(GVMCPU, PAGE_SIZE);
+        AssertRelease(pVM->cbSelf == sizeof(VM));
+        AssertRelease(pVM->cbVCpu == sizeof(VMCPU));
+/** @todo get rid of this   */
+        RTGCPTR GCPtr;
+        rc = MMR3HyperMapPages(pVM, pVM, pVM->pVMR0ForCall, sizeof(VM) >> PAGE_SHIFT, pVM->paVMPagesR3, "VM", &GCPtr);
+        uint32_t offPages = RT_UOFFSETOF_DYN(GVM, aCpus) >> PAGE_SHIFT; /* (Using the _DYN variant avoids -Winvalid-offset) */
+        for (uint32_t idCpu = 0; idCpu < pVM->cCpus && RT_SUCCESS(rc); idCpu++, offPages += sizeof(GVMCPU) >> PAGE_SHIFT)
+        {
+            PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+            RTGCPTR GCPtrIgn;
+            rc = MMR3HyperMapPages(pVM, pVCpu, pVM->pVMR0ForCall + offPages * PAGE_SIZE,
+                                   sizeof(VMCPU) >> PAGE_SHIFT, &pVM->paVMPagesR3[offPages], "VMCPU", &GCPtrIgn);
+        }
+        if (RT_SUCCESS(rc))
+        {
+            pVM->pVMRC = (RTRCPTR)GCPtr;
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+                pVM->apCpusR3[i]->pVMRC = pVM->pVMRC;
+
+            /* Reserve a page for fencing. */
+            MMR3HyperReserveFence(pVM);
+
+            /*
+             * Map the heap into the hypervisor space.
+             */
+            rc = mmR3HyperHeapMap(pVM, pVM->mm.s.pHyperHeapR3, &GCPtr);
+            if (RT_SUCCESS(rc))
+            {
+                pVM->mm.s.pHyperHeapRC = (RTRCPTR)GCPtr;
+                Assert(pVM->mm.s.pHyperHeapRC == GCPtr);
+
+                /*
+                 * Register info handlers.
+                 */
+                DBGFR3InfoRegisterInternal(pVM, "hma", "Show the layout of the Hypervisor Memory Area.", mmR3HyperInfoHma);
+
+                LogFlow(("mmR3HyperInit: returns VINF_SUCCESS\n"));
+                return VINF_SUCCESS;
+            }
+            /* Caller will do proper cleanup. */
+        }
+    }
+
+    LogFlow(("mmR3HyperInit: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Cleans up the hypervisor heap.
+ *
+ * @returns VBox status code.
+ */
+int mmR3HyperTerm(PVM pVM)
+{
+    if (pVM->mm.s.pHyperHeapR3)
+        PDMR3CritSectDelete(&pVM->mm.s.pHyperHeapR3->Lock);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Finalizes the HMA mapping.
+ *
+ * This is called later during init, most (all) HMA allocations should be done
+ * by the time this function is called.
+ *
+ * @returns VBox status code.
+ */
+VMMR3DECL(int) MMR3HyperInitFinalize(PVM pVM)
+{
+    LogFlow(("MMR3HyperInitFinalize:\n"));
+
+    /*
+     * Initialize the hyper heap critical section.
+     */
+    int rc = PDMR3CritSectInit(pVM, &pVM->mm.s.pHyperHeapR3->Lock, RT_SRC_POS, "MM-HYPER");
+    AssertRC(rc);
+
+#ifndef PGM_WITHOUT_MAPPINGS
+    /*
+     * Adjust and create the HMA mapping.
+     */
+    while ((RTINT)pVM->mm.s.offHyperNextStatic + 64*_1K < (RTINT)pVM->mm.s.cbHyperArea - _4M)
+        pVM->mm.s.cbHyperArea -= _4M;
+    rc = PGMR3MapPT(pVM, pVM->mm.s.pvHyperAreaGC, pVM->mm.s.cbHyperArea, 0 /*fFlags*/,
+                    mmR3HyperRelocateCallback, NULL, "Hypervisor Memory Area");
+    if (RT_FAILURE(rc))
+        return rc;
+#endif
+    pVM->mm.s.fPGMInitialized = true;
+
+#ifndef PGM_WITHOUT_MAPPINGS
+    /*
+     * Do all the delayed mappings.
+     */
+    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uintptr_t)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        RTGCPTR     GCPtr = pVM->mm.s.pvHyperAreaGC + pLookup->off;
+        uint32_t    cPages = pLookup->cb >> PAGE_SHIFT;
+        switch (pLookup->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+            {
+                PCRTHCPHYS paHCPhysPages = pLookup->u.Locked.paHCPhysPages;
+                for (uint32_t i = 0; i < cPages; i++)
+                {
+                    rc = PGMMap(pVM, GCPtr + (i << PAGE_SHIFT), paHCPhysPages[i], PAGE_SIZE, 0);
+                    AssertRCReturn(rc, rc);
+                }
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+                rc = PGMMap(pVM, GCPtr, pLookup->u.HCPhys.HCPhys, pLookup->cb, 0);
+                break;
+
+            case MMLOOKUPHYPERTYPE_GCPHYS:
+            {
+                const RTGCPHYS  GCPhys = pLookup->u.GCPhys.GCPhys;
+                const uint32_t  cb = pLookup->cb;
+                for (uint32_t off = 0; off < cb; off += PAGE_SIZE)
+                {
+                    RTHCPHYS HCPhys;
+                    rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys + off, &HCPhys);
+                    if (RT_FAILURE(rc))
+                        break;
+                    rc = PGMMap(pVM, GCPtr + off, HCPhys, PAGE_SIZE, 0);
+                    if (RT_FAILURE(rc))
+                        break;
+                }
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_MMIO2:
+            {
+                const RTGCPHYS offEnd = pLookup->u.MMIO2.off + pLookup->cb;
+                for (RTGCPHYS offCur = pLookup->u.MMIO2.off; offCur < offEnd; offCur += PAGE_SIZE)
+                {
+                    RTHCPHYS HCPhys;
+                    rc = PGMR3PhysMMIO2GetHCPhys(pVM, pLookup->u.MMIO2.pDevIns, pLookup->u.MMIO2.iSubDev,
+                                                 pLookup->u.MMIO2.iRegion, offCur, &HCPhys);
+                    if (RT_FAILURE(rc))
+                        break;
+                    rc = PGMMap(pVM, GCPtr + (offCur - pLookup->u.MMIO2.off), HCPhys, PAGE_SIZE, 0);
+                    if (RT_FAILURE(rc))
+                        break;
+                }
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+                /* do nothing here since these are either fences or managed by someone else using PGM. */
+                break;
+
+            default:
+                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+                break;
+        }
+
+        if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("rc=%Rrc cb=%d off=%#RX32 enmType=%d pszDesc=%s\n",
+                             rc, pLookup->cb, pLookup->off, pLookup->enmType, pLookup->pszDesc));
+            return rc;
+        }
+
+        /* next */
+        if (pLookup->offNext == (int32_t)NIL_OFFSET)
+            break;
+        pLookup = (PMMLOOKUPHYPER)((uintptr_t)pLookup + pLookup->offNext);
+    }
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
+    LogFlow(("MMR3HyperInitFinalize: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+#ifndef PGM_WITHOUT_MAPPINGS
+/**
+ * Callback function which will be called when PGM is trying to find a new
+ * location for the mapping.
+ *
+ * The callback is called in two modes, 1) the check mode and 2) the relocate mode.
+ * In 1) the callback should say if it objects to a suggested new location. If it
+ * accepts the new location, it is called again for doing it's relocation.
+ *
+ *
+ * @returns true if the location is ok.
+ * @returns false if another location should be found.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPtrOld    The old virtual address.
+ * @param   GCPtrNew    The new virtual address.
+ * @param   enmMode     Used to indicate the callback mode.
+ * @param   pvUser      User argument. Ignored.
+ * @remark  The return value is no a failure indicator, it's an acceptance
+ *          indicator. Relocation can not fail!
+ */
+static DECLCALLBACK(bool) mmR3HyperRelocateCallback(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew,
+                                                    PGMRELOCATECALL enmMode, void *pvUser)
+{
+    NOREF(pvUser);
+    switch (enmMode)
+    {
+        /*
+         * Verify location - all locations are good for us.
+         */
+        case PGMRELOCATECALL_SUGGEST:
+            return true;
+
+        /*
+         * Execute the relocation.
+         */
+        case PGMRELOCATECALL_RELOCATE:
+        {
+            /*
+             * Accepted!
+             */
+            AssertMsg(GCPtrOld == pVM->mm.s.pvHyperAreaGC,
+                      ("GCPtrOld=%RGv pVM->mm.s.pvHyperAreaGC=%RGv\n", GCPtrOld, pVM->mm.s.pvHyperAreaGC));
+            Log(("Relocating the hypervisor from %RGv to %RGv\n", GCPtrOld, GCPtrNew));
+
+            /*
+             * Relocate the VM structure and ourselves.
+             */
+            RTGCINTPTR offDelta = GCPtrNew - GCPtrOld;
+            pVM->pVMRC                          += offDelta;
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+                pVM->aCpus[i].pVMRC              = pVM->pVMRC;
+
+            pVM->mm.s.pvHyperAreaGC             += offDelta;
+            Assert(pVM->mm.s.pvHyperAreaGC < _4G);
+            pVM->mm.s.pHyperHeapRC              += offDelta;
+            pVM->mm.s.pHyperHeapR3->pbHeapRC    += offDelta;
+            pVM->mm.s.pHyperHeapR3->pVMRC        = pVM->pVMRC;
+
+            /*
+             * Relocate the rest.
+             */
+            VMR3Relocate(pVM, offDelta);
+            return true;
+        }
+
+        default:
+            AssertMsgFailed(("Invalid relocation mode %d\n", enmMode));
+    }
+
+    return false;
+}
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
+
+/**
+ * Service a VMMCALLRING3_MMHYPER_LOCK call.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(int) MMR3LockCall(PVM pVM)
+{
+    PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
+
+    int rc = PDMR3CritSectEnterEx(&pHeap->Lock, true /* fHostCall */);
+    AssertRC(rc);
+    return rc;
+}
+
+
+#ifndef PGM_WITHOUT_MAPPINGS
+
+/**
+ * Maps contiguous HC physical memory into the hypervisor region in the GC.
+ *
+ * @return VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pvR3        Ring-3 address of the memory. Must be page aligned!
+ * @param   pvR0        Optional ring-0 address of the memory.
+ * @param   HCPhys      Host context physical address of the memory to be
+ *                      mapped. Must be page aligned!
+ * @param   cb          Size of the memory. Will be rounded up to nearest page.
+ * @param   pszDesc     Description.
+ * @param   pGCPtr      Where to store the GC address.
+ */
+VMMR3DECL(int) MMR3HyperMapHCPhys(PVM pVM, void *pvR3, RTR0PTR pvR0, RTHCPHYS HCPhys, size_t cb,
+                                  const char *pszDesc, PRTGCPTR pGCPtr)
+{
+    LogFlow(("MMR3HyperMapHCPhys: pvR3=%p pvR0=%p HCPhys=%RHp cb=%d pszDesc=%p:{%s} pGCPtr=%p\n",
+             pvR3, pvR0, HCPhys, (int)cb, pszDesc, pszDesc, pGCPtr));
+
+    /*
+     * Validate input.
+     */
+    AssertReturn(RT_ALIGN_P(pvR3, PAGE_SIZE) == pvR3, VERR_INVALID_PARAMETER);
+    AssertReturn(RT_ALIGN_T(pvR0, PAGE_SIZE, RTR0PTR) == pvR0, VERR_INVALID_PARAMETER);
+    AssertReturn(RT_ALIGN_T(HCPhys, PAGE_SIZE, RTHCPHYS) == HCPhys, VERR_INVALID_PARAMETER);
+    AssertReturn(pszDesc && *pszDesc, VERR_INVALID_PARAMETER);
+
+    /*
+     * Add the memory to the hypervisor area.
+     */
+    uint32_t cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
+    AssertReturn(cbAligned >= cb, VERR_INVALID_PARAMETER);
+    RTGCPTR         GCPtr;
+    PMMLOOKUPHYPER  pLookup;
+    int rc = mmR3HyperMap(pVM, cbAligned, pszDesc, &GCPtr, &pLookup);
+    if (RT_SUCCESS(rc))
+    {
+        pLookup->enmType = MMLOOKUPHYPERTYPE_HCPHYS;
+        pLookup->u.HCPhys.pvR3   = pvR3;
+        pLookup->u.HCPhys.pvR0   = pvR0;
+        pLookup->u.HCPhys.HCPhys = HCPhys;
+
+        /*
+         * Update the page table.
+         */
+        if (pVM->mm.s.fPGMInitialized)
+            rc = PGMMap(pVM, GCPtr, HCPhys, cbAligned, 0);
+        if (RT_SUCCESS(rc))
+            *pGCPtr = GCPtr;
+    }
+    return rc;
+}
+
+
+/**
+ * Maps contiguous GC physical memory into the hypervisor region in the GC.
+ *
+ * @return VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      Guest context physical address of the memory to be mapped. Must be page aligned!
+ * @param   cb          Size of the memory. Will be rounded up to nearest page.
+ * @param   pszDesc     Mapping description.
+ * @param   pGCPtr      Where to store the GC address.
+ */
+VMMR3DECL(int) MMR3HyperMapGCPhys(PVM pVM, RTGCPHYS GCPhys, size_t cb, const char *pszDesc, PRTGCPTR pGCPtr)
+{
+    LogFlow(("MMR3HyperMapGCPhys: GCPhys=%RGp cb=%d pszDesc=%p:{%s} pGCPtr=%p\n", GCPhys, (int)cb, pszDesc, pszDesc, pGCPtr));
+
+    /*
+     * Validate input.
+     */
+    AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
+    AssertReturn(pszDesc && *pszDesc, VERR_INVALID_PARAMETER);
+
+    /*
+     * Add the memory to the hypervisor area.
+     */
+    cb = RT_ALIGN_Z(cb, PAGE_SIZE);
+    RTGCPTR         GCPtr;
+    PMMLOOKUPHYPER  pLookup;
+    int rc = mmR3HyperMap(pVM, cb, pszDesc, &GCPtr, &pLookup);
+    if (RT_SUCCESS(rc))
+    {
+        pLookup->enmType = MMLOOKUPHYPERTYPE_GCPHYS;
+        pLookup->u.GCPhys.GCPhys = GCPhys;
+
+        /*
+         * Update the page table.
+         */
+        for (unsigned off = 0; off < cb; off += PAGE_SIZE)
+        {
+            RTHCPHYS HCPhys;
+            rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys + off, &HCPhys);
+            AssertRC(rc);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("rc=%Rrc GCPhys=%RGp off=%#x %s\n", rc, GCPhys, off, pszDesc));
+                break;
+            }
+            if (pVM->mm.s.fPGMInitialized)
+            {
+                rc = PGMMap(pVM, GCPtr + off, HCPhys, PAGE_SIZE, 0);
+                AssertRC(rc);
+                if (RT_FAILURE(rc))
+                {
+                    AssertMsgFailed(("rc=%Rrc GCPhys=%RGp off=%#x %s\n", rc, GCPhys, off, pszDesc));
+                    break;
+                }
+            }
+        }
+
+        if (RT_SUCCESS(rc) && pGCPtr)
+            *pGCPtr = GCPtr;
+    }
+    return rc;
+}
+
+
+/**
+ * Maps a portion of an MMIO2 region into the hypervisor region.
+ *
+ * Callers of this API must never deregister the MMIO2 region before the
+ * VM is powered off.  If this becomes a requirement MMR3HyperUnmapMMIO2
+ * API will be needed to perform cleanups.
+ *
+ * @return VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     The device owning the MMIO2 memory.
+ * @param   iSubDev     The sub-device number.
+ * @param   iRegion     The region.
+ * @param   off         The offset into the region. Will be rounded down to closest page boundary.
+ * @param   cb          The number of bytes to map. Will be rounded up to the closest page boundary.
+ * @param   pszDesc     Mapping description.
+ * @param   pRCPtr      Where to store the RC address.
+ */
+VMMR3DECL(int) MMR3HyperMapMMIO2(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
+                                const char *pszDesc, PRTRCPTR pRCPtr)
+{
+    LogFlow(("MMR3HyperMapMMIO2: pDevIns=%p iSubDev=%#x iRegion=%#x off=%RGp cb=%RGp pszDesc=%p:{%s} pRCPtr=%p\n",
+             pDevIns, iSubDev, iRegion, off, cb, pszDesc, pszDesc, pRCPtr));
+    int rc;
+
+    /*
+     * Validate input.
+     */
+    AssertReturn(pszDesc && *pszDesc, VERR_INVALID_PARAMETER);
+    AssertReturn(off + cb > off, VERR_INVALID_PARAMETER);
+    uint32_t const offPage = off & PAGE_OFFSET_MASK;
+    off &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
+    cb += offPage;
+    cb = RT_ALIGN_Z(cb, PAGE_SIZE);
+    const RTGCPHYS offEnd = off + cb;
+    AssertReturn(offEnd > off, VERR_INVALID_PARAMETER);
+    for (RTGCPHYS offCur = off; offCur < offEnd; offCur += PAGE_SIZE)
+    {
+        RTHCPHYS HCPhys;
+        rc = PGMR3PhysMMIO2GetHCPhys(pVM, pDevIns, iSubDev, iRegion, offCur, &HCPhys);
+        AssertMsgRCReturn(rc, ("rc=%Rrc - iSubDev=%#x iRegion=%#x off=%RGp\n", rc, iSubDev, iRegion, off), rc);
+    }
+
+    /*
+     * Add the memory to the hypervisor area.
+     */
+    RTGCPTR         GCPtr;
+    PMMLOOKUPHYPER  pLookup;
+    rc = mmR3HyperMap(pVM, cb, pszDesc, &GCPtr, &pLookup);
+    if (RT_SUCCESS(rc))
+    {
+        pLookup->enmType = MMLOOKUPHYPERTYPE_MMIO2;
+        pLookup->u.MMIO2.pDevIns = pDevIns;
+        pLookup->u.MMIO2.iSubDev = iSubDev;
+        pLookup->u.MMIO2.iRegion = iRegion;
+        pLookup->u.MMIO2.off = off;
+
+        /*
+         * Update the page table.
+         */
+        if (pVM->mm.s.fPGMInitialized)
+        {
+            for (RTGCPHYS offCur = off; offCur < offEnd; offCur += PAGE_SIZE)
+            {
+                RTHCPHYS HCPhys;
+                rc = PGMR3PhysMMIO2GetHCPhys(pVM, pDevIns, iSubDev, iRegion, offCur, &HCPhys);
+                AssertRCReturn(rc, rc);
+                rc = PGMMap(pVM, GCPtr + (offCur - off), HCPhys, PAGE_SIZE, 0);
+                if (RT_FAILURE(rc))
+                {
+                    AssertMsgFailed(("rc=%Rrc offCur=%RGp %s\n", rc, offCur, pszDesc));
+                    break;
+                }
+            }
+        }
+
+        if (RT_SUCCESS(rc))
+        {
+            GCPtr |= offPage;
+            *pRCPtr = GCPtr;
+            AssertLogRelReturn(*pRCPtr == GCPtr, VERR_INTERNAL_ERROR);
+        }
+    }
+    return rc;
+}
+
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
+/**
+ * Maps locked R3 virtual memory into the hypervisor region in the GC.
+ *
+ * @return VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pvR3        The ring-3 address of the memory, must be page aligned.
+ * @param   pvR0        The ring-0 address of the memory, must be page aligned. (optional)
+ * @param   cPages      The number of pages.
+ * @param   paPages     The page descriptors.
+ * @param   pszDesc     Mapping description.
+ * @param   pGCPtr      Where to store the GC address corresponding to pvR3.
+ */
+VMMR3DECL(int) MMR3HyperMapPages(PVM pVM, void *pvR3, RTR0PTR pvR0, size_t cPages, PCSUPPAGE paPages,
+                                 const char *pszDesc, PRTGCPTR pGCPtr)
+{
+    LogFlow(("MMR3HyperMapPages: pvR3=%p pvR0=%p cPages=%zu paPages=%p pszDesc=%p:{%s} pGCPtr=%p\n",
+             pvR3, pvR0, cPages, paPages, pszDesc, pszDesc, pGCPtr));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pvR3, VERR_INVALID_POINTER);
+    AssertPtrReturn(paPages, VERR_INVALID_POINTER);
+    AssertReturn(cPages > 0, VERR_PAGE_COUNT_OUT_OF_RANGE);
+    AssertReturn(cPages <= VBOX_MAX_ALLOC_PAGE_COUNT, VERR_PAGE_COUNT_OUT_OF_RANGE);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pGCPtr, VERR_INVALID_PARAMETER);
+
+    /*
+     * Add the memory to the hypervisor area.
+     */
+    RTGCPTR         GCPtr;
+    PMMLOOKUPHYPER  pLookup;
+    int rc = mmR3HyperMap(pVM, cPages << PAGE_SHIFT, pszDesc, &GCPtr, &pLookup);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Copy the physical page addresses and tell PGM about them.
+         */
+        PRTHCPHYS paHCPhysPages = (PRTHCPHYS)MMR3HeapAlloc(pVM, MM_TAG_MM, sizeof(RTHCPHYS) * cPages);
+        if (paHCPhysPages)
+        {
+            for (size_t i = 0; i < cPages; i++)
+            {
+                AssertReleaseMsgReturn(   paPages[i].Phys != 0
+                                       && paPages[i].Phys != NIL_RTHCPHYS
+                                       && !(paPages[i].Phys & PAGE_OFFSET_MASK),
+                                       ("i=%#zx Phys=%RHp %s\n", i, paPages[i].Phys, pszDesc),
+                                       VERR_INTERNAL_ERROR);
+                paHCPhysPages[i] = paPages[i].Phys;
+            }
+
+#ifndef PGM_WITHOUT_MAPPINGS
+            if (pVM->mm.s.fPGMInitialized)
+            {
+                for (size_t i = 0; i < cPages; i++)
+                {
+                    rc = PGMMap(pVM, GCPtr + (i << PAGE_SHIFT), paHCPhysPages[i], PAGE_SIZE, 0);
+                    AssertRCBreak(rc);
+                }
+            }
+#endif
+            if (RT_SUCCESS(rc))
+            {
+                pLookup->enmType = MMLOOKUPHYPERTYPE_LOCKED;
+                pLookup->u.Locked.pvR3          = pvR3;
+                pLookup->u.Locked.pvR0          = pvR0;
+                pLookup->u.Locked.paHCPhysPages = paHCPhysPages;
+
+                /* done. */
+                *pGCPtr   = GCPtr;
+                return rc;
+            }
+            /* Don't care about failure clean, we're screwed if this fails anyway. */
+        }
+    }
+
+    return rc;
+}
+
+
+#ifndef PGM_WITHOUT_MAPPINGS
+/**
+ * Reserves a hypervisor memory area.
+ * Most frequent usage is fence pages and dynamically mappings like the guest PD and PDPT.
+ *
+ * @return VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   cb          Size of the memory. Will be rounded up to nearest page.
+ * @param   pszDesc     Mapping description.
+ * @param   pGCPtr      Where to store the assigned GC address. Optional.
+ */
+VMMR3DECL(int) MMR3HyperReserve(PVM pVM, unsigned cb, const char *pszDesc, PRTGCPTR pGCPtr)
+{
+    LogFlow(("MMR3HyperMapHCRam: cb=%d pszDesc=%p:{%s} pGCPtr=%p\n", (int)cb, pszDesc, pszDesc, pGCPtr));
+
+    /*
+     * Validate input.
+     */
+    if (    cb <= 0
+        ||  !pszDesc
+        ||  !*pszDesc)
+    {
+        AssertMsgFailed(("Invalid parameter\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Add the memory to the hypervisor area.
+     */
+    RTGCPTR         GCPtr;
+    PMMLOOKUPHYPER  pLookup;
+    int rc = mmR3HyperMap(pVM, cb, pszDesc, &GCPtr, &pLookup);
+    if (RT_SUCCESS(rc))
+    {
+        pLookup->enmType = MMLOOKUPHYPERTYPE_DYNAMIC;
+        if (pGCPtr)
+            *pGCPtr = GCPtr;
+        return VINF_SUCCESS;
+    }
+    return rc;
+}
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
+
+/**
+ * Reserves an electric fence page.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(int) MMR3HyperReserveFence(PVM pVM)
+{
+#ifndef PGM_WITHOUT_MAPPINGS
+    return MMR3HyperReserve(pVM, cb, "fence", NULL);
+#else
+    RT_NOREF(pVM);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/**
+ * Adds memory to the hypervisor memory arena.
+ *
+ * @return VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   cb          Size of the memory. Will be rounded up to nearest page.
+ * @param   pszDesc     The description of the memory.
+ * @param   pGCPtr      Where to store the GC address.
+ * @param   ppLookup    Where to store the pointer to the lookup record.
+ * @remark  We assume the threading structure of VBox imposes natural
+ *          serialization of most functions, this one included.
+ */
+static int mmR3HyperMap(PVM pVM, const size_t cb, const char *pszDesc, PRTGCPTR pGCPtr, PMMLOOKUPHYPER *ppLookup)
+{
+    /*
+     * Validate input.
+     */
+    const uint32_t cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
+    AssertReturn(cbAligned >= cb, VERR_INVALID_PARAMETER);
+    if (pVM->mm.s.offHyperNextStatic + cbAligned >= pVM->mm.s.cbHyperArea) /* don't use the last page, it's a fence. */
+    {
+        AssertMsgFailed(("Out of static mapping space in the HMA! offHyperAreaGC=%x cbAligned=%x cbHyperArea=%x\n",
+                         pVM->mm.s.offHyperNextStatic, cbAligned, pVM->mm.s.cbHyperArea));
+        return VERR_NO_MEMORY;
+    }
+
+    /*
+     * Allocate lookup record.
+     */
+    PMMLOOKUPHYPER  pLookup;
+    int rc = MMHyperAlloc(pVM, sizeof(*pLookup), 1, MM_TAG_MM, (void **)&pLookup);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Initialize it and insert it.
+         */
+        pLookup->offNext        = pVM->mm.s.offLookupHyper;
+        pLookup->cb             = cbAligned;
+        pLookup->off            = pVM->mm.s.offHyperNextStatic;
+        pVM->mm.s.offLookupHyper = (uint8_t *)pLookup - (uint8_t *)pVM->mm.s.pHyperHeapR3;
+        if (pLookup->offNext != (int32_t)NIL_OFFSET)
+            pLookup->offNext   -= pVM->mm.s.offLookupHyper;
+        pLookup->enmType        = MMLOOKUPHYPERTYPE_INVALID;
+        memset(&pLookup->u, 0xff, sizeof(pLookup->u));
+        pLookup->pszDesc        = pszDesc;
+
+        /* Mapping. */
+        *pGCPtr = pVM->mm.s.pvHyperAreaGC + pVM->mm.s.offHyperNextStatic;
+        pVM->mm.s.offHyperNextStatic += cbAligned;
+
+        /* Return pointer. */
+        *ppLookup = pLookup;
+    }
+
+    AssertRC(rc);
+    LogFlow(("mmR3HyperMap: returns %Rrc *pGCPtr=%RGv\n", rc, *pGCPtr));
+    return rc;
+}
+
+
+/**
+ * Allocates a new heap.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   cb          The size of the new heap.
+ * @param   ppHeap      Where to store the heap pointer on successful return.
+ * @param   pR0PtrHeap  Where to store the ring-0 address of the heap on
+ *                      success.
+ */
+static int mmR3HyperHeapCreate(PVM pVM, const size_t cb, PMMHYPERHEAP *ppHeap, PRTR0PTR pR0PtrHeap)
+{
+    /*
+     * Allocate the hypervisor heap.
+     */
+    const uint32_t  cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
+    AssertReturn(cbAligned >= cb, VERR_INVALID_PARAMETER);
+    uint32_t const  cPages = cbAligned >> PAGE_SHIFT;
+    PSUPPAGE        paPages = (PSUPPAGE)MMR3HeapAlloc(pVM, MM_TAG_MM, cPages * sizeof(paPages[0]));
+    if (!paPages)
+        return VERR_NO_MEMORY;
+    void           *pv;
+    RTR0PTR         pvR0 = NIL_RTR0PTR;
+    int rc = SUPR3PageAllocEx(cPages,
+                              0 /*fFlags*/,
+                              &pv,
+                              &pvR0,
+                              paPages);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pvR0 != NIL_RTR0PTR && !(PAGE_OFFSET_MASK & pvR0));
+        memset(pv, 0, cbAligned);
+
+        /*
+         * Initialize the heap and first free chunk.
+         */
+        PMMHYPERHEAP pHeap = (PMMHYPERHEAP)pv;
+        pHeap->u32Magic             = MMHYPERHEAP_MAGIC;
+        pHeap->pbHeapR3             = (uint8_t *)pHeap + MMYPERHEAP_HDR_SIZE;
+        pHeap->pbHeapR0             = pvR0 + MMYPERHEAP_HDR_SIZE;
+        //pHeap->pbHeapRC           = 0; // set by mmR3HyperHeapMap()
+        pHeap->pVMR3                = pVM;
+        pHeap->pVMR0                = pVM->pVMR0ForCall;
+        pHeap->pVMRC                = pVM->pVMRC;
+        pHeap->cbHeap               = cbAligned - MMYPERHEAP_HDR_SIZE;
+        pHeap->cbFree               = pHeap->cbHeap - sizeof(MMHYPERCHUNK);
+        //pHeap->offFreeHead        = 0;
+        //pHeap->offFreeTail        = 0;
+        pHeap->offPageAligned       = pHeap->cbHeap;
+        //pHeap->HyperHeapStatTree  = 0;
+        pHeap->paPages              = paPages;
+
+        PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pHeap->pbHeapR3;
+        pFree->cb                   = pHeap->cbFree;
+        //pFree->core.offNext       = 0;
+        MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
+        pFree->core.offHeap         = -(int32_t)MMYPERHEAP_HDR_SIZE;
+        //pFree->offNext            = 0;
+        //pFree->offPrev            = 0;
+
+        STAMR3Register(pVM, &pHeap->cbHeap, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, "/MM/HyperHeap/cbHeap",  STAMUNIT_BYTES, "The heap size.");
+        STAMR3Register(pVM, &pHeap->cbFree, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, "/MM/HyperHeap/cbFree",  STAMUNIT_BYTES, "The free space.");
+
+        *ppHeap = pHeap;
+        *pR0PtrHeap = pvR0;
+        return VINF_SUCCESS;
+    }
+    AssertMsgFailed(("SUPR3PageAllocEx(%d,,,,) -> %Rrc\n", cbAligned >> PAGE_SHIFT, rc));
+
+    *ppHeap = NULL;
+    return rc;
+}
+
+
+/**
+ * Allocates a new heap.
+ */
+static int mmR3HyperHeapMap(PVM pVM, PMMHYPERHEAP pHeap, PRTGCPTR ppHeapGC)
+{
+    Assert(RT_ALIGN_Z(pHeap->cbHeap + MMYPERHEAP_HDR_SIZE, PAGE_SIZE) == pHeap->cbHeap + MMYPERHEAP_HDR_SIZE);
+    Assert(pHeap->pbHeapR0);
+    Assert(pHeap->paPages);
+    int rc = MMR3HyperMapPages(pVM,
+                               pHeap,
+                               pHeap->pbHeapR0 - MMYPERHEAP_HDR_SIZE,
+                               (pHeap->cbHeap + MMYPERHEAP_HDR_SIZE) >> PAGE_SHIFT,
+                               pHeap->paPages,
+                               "Heap", ppHeapGC);
+    if (RT_SUCCESS(rc))
+    {
+        pHeap->pVMRC    = pVM->pVMRC;
+        pHeap->pbHeapRC = *ppHeapGC + MMYPERHEAP_HDR_SIZE;
+        /* Reserve a page for fencing. */
+        MMR3HyperReserveFence(pVM);
+
+        /* We won't need these any more. */
+        MMR3HeapFree(pHeap->paPages);
+        pHeap->paPages = NULL;
+    }
+    return rc;
+}
+
+
+/**
+ * Allocates memory in the Hypervisor (GC VMM) area which never will
+ * be freed and doesn't have any offset based relation to other heap blocks.
+ *
+ * The latter means that two blocks allocated by this API will not have the
+ * same relative position to each other in GC and HC. In short, never use
+ * this API for allocating nodes for an offset based AVL tree!
+ *
+ * The returned memory is of course zeroed.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   cb          Number of bytes to allocate.
+ * @param   uAlignment  Required memory alignment in bytes.
+ *                      Values are 0,8,16,32 and PAGE_SIZE.
+ *                      0 -> default alignment, i.e. 8 bytes.
+ * @param   enmTag      The statistics tag.
+ * @param   ppv         Where to store the address to the allocated
+ *                      memory.
+ * @remark  This is assumed not to be used at times when serialization is required.
+ */
+VMMR3DECL(int) MMR3HyperAllocOnceNoRel(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
+{
+    return MMR3HyperAllocOnceNoRelEx(pVM, cb, uAlignment, enmTag, 0/*fFlags*/, ppv);
+}
+
+
+/**
+ * Allocates memory in the Hypervisor (GC VMM) area which never will
+ * be freed and doesn't have any offset based relation to other heap blocks.
+ *
+ * The latter means that two blocks allocated by this API will not have the
+ * same relative position to each other in GC and HC. In short, never use
+ * this API for allocating nodes for an offset based AVL tree!
+ *
+ * The returned memory is of course zeroed.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   cb          Number of bytes to allocate.
+ * @param   uAlignment  Required memory alignment in bytes.
+ *                      Values are 0,8,16,32 and PAGE_SIZE.
+ *                      0 -> default alignment, i.e. 8 bytes.
+ * @param   enmTag      The statistics tag.
+ * @param   fFlags      Flags, see MMHYPER_AONR_FLAGS_KERNEL_MAPPING.
+ * @param   ppv         Where to store the address to the allocated memory.
+ * @remark  This is assumed not to be used at times when serialization is required.
+ */
+VMMR3DECL(int) MMR3HyperAllocOnceNoRelEx(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, uint32_t fFlags, void **ppv)
+{
+    AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
+    Assert(!(fFlags & ~(MMHYPER_AONR_FLAGS_KERNEL_MAPPING)));
+
+    /*
+     * Choose between allocating a new chunk of HMA memory
+     * and the heap. We will only do BIG allocations from HMA and
+     * only at creation time.
+     */
+    if (   (   cb < _64K
+            && (   uAlignment != PAGE_SIZE
+                || cb < 48*_1K)
+            && !(fFlags & MMHYPER_AONR_FLAGS_KERNEL_MAPPING)
+           )
+        ||  VMR3GetState(pVM) != VMSTATE_CREATING
+       )
+    {
+        Assert(!(fFlags & MMHYPER_AONR_FLAGS_KERNEL_MAPPING));
+        int rc = MMHyperAlloc(pVM, cb, uAlignment, enmTag, ppv);
+        if (    rc != VERR_MM_HYPER_NO_MEMORY
+            ||  cb <= 8*_1K)
+        {
+            Log2(("MMR3HyperAllocOnceNoRel: cb=%#zx uAlignment=%#x returns %Rrc and *ppv=%p\n",
+                  cb, uAlignment, rc, *ppv));
+            return rc;
+        }
+    }
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    /*
+     * Set MMHYPER_AONR_FLAGS_KERNEL_MAPPING if we're in going to execute in ring-0.
+     */
+    if (VM_IS_HM_OR_NEM_ENABLED(pVM))
+        fFlags |= MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
+#endif
+
+    /*
+     * Validate alignment.
+     */
+    switch (uAlignment)
+    {
+        case 0:
+        case 8:
+        case 16:
+        case 32:
+        case PAGE_SIZE:
+            break;
+        default:
+            AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
+            return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Allocate the pages and map them into HMA space.
+     */
+    uint32_t const  cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
+    AssertReturn(cbAligned >= cb, VERR_INVALID_PARAMETER);
+    uint32_t const  cPages    = cbAligned >> PAGE_SHIFT;
+    PSUPPAGE        paPages   = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(paPages[0]));
+    if (!paPages)
+        return VERR_NO_TMP_MEMORY;
+    void           *pvPages;
+    RTR0PTR         pvR0 = NIL_RTR0PTR;
+    int rc = SUPR3PageAllocEx(cPages,
+                              0 /*fFlags*/,
+                              &pvPages,
+                              &pvR0,
+                              paPages);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pvR0 != NIL_RTR0PTR);
+        memset(pvPages, 0, cbAligned);
+
+        RTGCPTR GCPtr;
+        rc = MMR3HyperMapPages(pVM,
+                               pvPages,
+                               pvR0,
+                               cPages,
+                               paPages,
+                               MMR3HeapAPrintf(pVM, MM_TAG_MM, "alloc once (%s)", mmGetTagName(enmTag)),
+                               &GCPtr);
+        /* not needed anymore */
+        RTMemTmpFree(paPages);
+        if (RT_SUCCESS(rc))
+        {
+            *ppv = pvPages;
+            Log2(("MMR3HyperAllocOnceNoRel: cbAligned=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n",
+                  cbAligned, uAlignment, *ppv));
+            MMR3HyperReserveFence(pVM);
+            return rc;
+        }
+        AssertMsgFailed(("Failed to allocate %zd bytes! %Rrc\n", cbAligned, rc));
+        SUPR3PageFreeEx(pvPages, cPages);
+
+
+        /*
+         * HACK ALERT! Try allocate it off the heap so that we don't freak
+         * out during vga/vmmdev mmio2 allocation with certain ram sizes.
+         */
+        /** @todo make a proper fix for this so we will never end up in this kind of situation! */
+        Log(("MMR3HyperAllocOnceNoRel: MMR3HyperMapHCRam failed with rc=%Rrc, try MMHyperAlloc(,%#x,,) instead\n",  rc, cb));
+        int rc2 = MMHyperAlloc(pVM, cb, uAlignment, enmTag, ppv);
+        if (RT_SUCCESS(rc2))
+        {
+            Log2(("MMR3HyperAllocOnceNoRel: cb=%#x uAlignment=%#x returns %Rrc and *ppv=%p\n",
+                  cb, uAlignment, rc, *ppv));
+            return rc;
+        }
+    }
+    else
+        AssertMsgFailed(("Failed to allocate %zd bytes! %Rrc\n", cbAligned, rc));
+
+    if (rc == VERR_NO_MEMORY)
+        rc = VERR_MM_HYPER_NO_MEMORY;
+    LogRel(("MMR3HyperAllocOnceNoRel: cb=%#zx uAlignment=%#x returns %Rrc\n", cb, uAlignment, rc));
+    return rc;
+}
+
+
+/**
+ * Lookus up a ring-3 pointer to HMA.
+ *
+ * @returns The lookup record on success, NULL on failure.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pvR3                The ring-3 address to look up.
+ */
+DECLINLINE(PMMLOOKUPHYPER) mmR3HyperLookupR3(PVM pVM, void *pvR3)
+{
+    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        switch (pLookup->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+            {
+                unsigned off = (uint8_t *)pvR3 - (uint8_t *)pLookup->u.Locked.pvR3;
+                if (off < pLookup->cb)
+                    return pLookup;
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+            {
+                unsigned off = (uint8_t *)pvR3 - (uint8_t *)pLookup->u.HCPhys.pvR3;
+                if (off < pLookup->cb)
+                    return pLookup;
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_GCPHYS:
+            case MMLOOKUPHYPERTYPE_MMIO2:
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+                /** @todo ?    */
+                break;
+
+            default:
+                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+                return NULL;
+        }
+
+        /* next */
+        if ((unsigned)pLookup->offNext == NIL_OFFSET)
+            return NULL;
+        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
+    }
+}
+
+
+/**
+ * Set / unset guard status on one or more hyper heap pages.
+ *
+ * @returns VBox status code (first failure).
+ * @param   pVM                 The cross context VM structure.
+ * @param   pvStart             The hyper heap page address. Must be page
+ *                              aligned.
+ * @param   cb                  The number of bytes. Must be page aligned.
+ * @param   fSet                Whether to set or unset guard page status.
+ */
+VMMR3DECL(int) MMR3HyperSetGuard(PVM pVM, void *pvStart, size_t cb, bool fSet)
+{
+    /*
+     * Validate input.
+     */
+    AssertReturn(!((uintptr_t)pvStart & PAGE_OFFSET_MASK), VERR_INVALID_POINTER);
+    AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(cb <= UINT32_MAX, VERR_INVALID_PARAMETER);
+    PMMLOOKUPHYPER pLookup = mmR3HyperLookupR3(pVM, pvStart);
+    AssertReturn(pLookup, VERR_INVALID_PARAMETER);
+    AssertReturn(pLookup->enmType == MMLOOKUPHYPERTYPE_LOCKED, VERR_INVALID_PARAMETER);
+
+    /*
+     * Get down to business.
+     * Note! We quietly ignore errors from the support library since the
+     *       protection stuff isn't possible to implement on all platforms.
+     */
+    uint8_t    *pbR3  = (uint8_t *)pLookup->u.Locked.pvR3;
+    RTR0PTR     R0Ptr = pLookup->u.Locked.pvR0 != (uintptr_t)pLookup->u.Locked.pvR3
+                      ? pLookup->u.Locked.pvR0
+                      : NIL_RTR0PTR;
+    uint32_t    off   = (uint32_t)((uint8_t *)pvStart - pbR3);
+    int         rc;
+    if (fSet)
+    {
+#ifndef PGM_WITHOUT_MAPPINGS
+        rc = PGMMapSetPage(pVM, MMHyperR3ToRC(pVM, pvStart), cb, 0);
+#else
+        rc = VINF_SUCCESS;
+#endif
+        SUPR3PageProtect(pbR3, R0Ptr, off, (uint32_t)cb, RTMEM_PROT_NONE);
+    }
+    else
+    {
+#ifndef PGM_WITHOUT_MAPPINGS
+        rc = PGMMapSetPage(pVM, MMHyperR3ToRC(pVM, pvStart), cb, X86_PTE_P | X86_PTE_A | X86_PTE_D | X86_PTE_RW);
+#else
+        rc = VINF_SUCCESS;
+#endif
+        SUPR3PageProtect(pbR3, R0Ptr, off, (uint32_t)cb, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
+    }
+    return rc;
+}
+
+
+/**
+ * Convert hypervisor HC virtual address to HC physical address.
+ *
+ * @returns HC physical address.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvR3        Host context virtual address.
+ */
+VMMR3DECL(RTHCPHYS) MMR3HyperHCVirt2HCPhys(PVM pVM, void *pvR3)
+{
+    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        switch (pLookup->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+            {
+                unsigned off = (uint8_t *)pvR3 - (uint8_t *)pLookup->u.Locked.pvR3;
+                if (off < pLookup->cb)
+                    return pLookup->u.Locked.paHCPhysPages[off >> PAGE_SHIFT] | (off & PAGE_OFFSET_MASK);
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+            {
+                unsigned off = (uint8_t *)pvR3 - (uint8_t *)pLookup->u.HCPhys.pvR3;
+                if (off < pLookup->cb)
+                    return pLookup->u.HCPhys.HCPhys + off;
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_GCPHYS:
+            case MMLOOKUPHYPERTYPE_MMIO2:
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+                /* can (or don't want to) convert these kind of records. */
+                break;
+
+            default:
+                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+                break;
+        }
+
+        /* next */
+        if ((unsigned)pLookup->offNext == NIL_OFFSET)
+            break;
+        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
+    }
+
+    AssertMsgFailed(("pvR3=%p is not inside the hypervisor memory area!\n", pvR3));
+    return NIL_RTHCPHYS;
+}
+
+#ifndef PGM_WITHOUT_MAPPINGS
+
+/**
+ * Implements the hcphys-not-found return case of MMR3HyperQueryInfoFromHCPhys.
+ *
+ * @returns VINF_SUCCESS, VINF_BUFFER_OVERFLOW.
+ * @param   pVM                 The cross context VM structure.
+ * @param   HCPhys              The host physical address to look for.
+ * @param   pLookup             The HMA lookup entry corresponding to HCPhys.
+ * @param   pszWhat             Where to return the description.
+ * @param   cbWhat              Size of the return buffer.
+ * @param   pcbAlloc            Where to return the size of whatever it is.
+ */
+static int mmR3HyperQueryInfoFromHCPhysFound(PVM pVM, RTHCPHYS HCPhys, PMMLOOKUPHYPER pLookup,
+                                             char *pszWhat, size_t cbWhat, uint32_t *pcbAlloc)
+{
+    NOREF(pVM); NOREF(HCPhys);
+    *pcbAlloc = pLookup->cb;
+    int rc = RTStrCopy(pszWhat, cbWhat, pLookup->pszDesc);
+    return rc == VERR_BUFFER_OVERFLOW ? VINF_BUFFER_OVERFLOW : rc;
+}
+
+
+/**
+ * Scans the HMA for the physical page and reports back a description if found.
+ *
+ * @returns VINF_SUCCESS, VINF_BUFFER_OVERFLOW, VERR_NOT_FOUND.
+ * @param   pVM                 The cross context VM structure.
+ * @param   HCPhys              The host physical address to look for.
+ * @param   pszWhat             Where to return the description.
+ * @param   cbWhat              Size of the return buffer.
+ * @param   pcbAlloc            Where to return the size of whatever it is.
+ */
+VMMR3_INT_DECL(int) MMR3HyperQueryInfoFromHCPhys(PVM pVM, RTHCPHYS HCPhys, char *pszWhat, size_t cbWhat, uint32_t *pcbAlloc)
+{
+    RTHCPHYS        HCPhysPage = HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK;
+    PMMLOOKUPHYPER  pLookup    = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        switch (pLookup->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+            {
+                uint32_t i = pLookup->cb >> PAGE_SHIFT;
+                while (i-- > 0)
+                    if (pLookup->u.Locked.paHCPhysPages[i] == HCPhysPage)
+                        return mmR3HyperQueryInfoFromHCPhysFound(pVM, HCPhys, pLookup, pszWhat, cbWhat, pcbAlloc);
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+            {
+                if (pLookup->u.HCPhys.HCPhys - HCPhysPage < pLookup->cb)
+                    return mmR3HyperQueryInfoFromHCPhysFound(pVM, HCPhys, pLookup, pszWhat, cbWhat, pcbAlloc);
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_MMIO2:
+            case MMLOOKUPHYPERTYPE_GCPHYS:
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+            {
+                /* brute force. */
+                uint32_t i = pLookup->cb >> PAGE_SHIFT;
+                while (i-- > 0)
+                {
+                    RTGCPTR     GCPtr = pLookup->off + pVM->mm.s.pvHyperAreaGC;
+                    RTHCPHYS    HCPhysCur;
+                    int rc = PGMMapGetPage(pVM, GCPtr, NULL, &HCPhysCur);
+                    if (RT_SUCCESS(rc) && HCPhysCur == HCPhysPage)
+                        return mmR3HyperQueryInfoFromHCPhysFound(pVM, HCPhys, pLookup, pszWhat, cbWhat, pcbAlloc);
+                }
+                break;
+            }
+            default:
+                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+                break;
+        }
+
+        /* next */
+        if ((unsigned)pLookup->offNext == NIL_OFFSET)
+            break;
+        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
+    }
+    return VERR_NOT_FOUND;
+}
+
+
+/**
+ * Read hypervisor memory from GC virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvDst       Destination address (HC of course).
+ * @param   GCPtr       GC virtual address.
+ * @param   cb          Number of bytes to read.
+ *
+ * @remarks For DBGF only.
+ */
+VMMR3DECL(int) MMR3HyperReadGCVirt(PVM pVM, void *pvDst, RTGCPTR GCPtr, size_t cb)
+{
+    if (GCPtr - pVM->mm.s.pvHyperAreaGC >= pVM->mm.s.cbHyperArea)
+        return VERR_INVALID_POINTER;
+    return PGMR3MapRead(pVM, pvDst, GCPtr, cb);
+}
+
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
+/**
+ * Info handler for 'hma', it dumps the list of lookup records for the hypervisor memory area.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        Callback functions for doing output.
+ * @param   pszArgs     Argument string. Optional and specific to the handler.
+ */
+static DECLCALLBACK(void) mmR3HyperInfoHma(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+
+    pHlp->pfnPrintf(pHlp, "Hypervisor Memory Area (HMA) Layout: Base %RGv, 0x%08x bytes\n",
+                    pVM->mm.s.pvHyperAreaGC, pVM->mm.s.cbHyperArea);
+
+    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        switch (pLookup->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %RHv %RHv LOCKED  %-*s %s\n",
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
+                                pLookup->u.Locked.pvR3,
+                                pLookup->u.Locked.pvR0,
+                                sizeof(RTHCPTR) * 2, "",
+                                pLookup->pszDesc);
+                break;
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %RHv %RHv HCPHYS  %RHp %s\n",
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
+                                pLookup->u.HCPhys.pvR3,
+                                pLookup->u.HCPhys.pvR0,
+                                pLookup->u.HCPhys.HCPhys,
+                                pLookup->pszDesc);
+                break;
+
+            case MMLOOKUPHYPERTYPE_GCPHYS:
+                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %*s GCPHYS  %RGp%*s %s\n",
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
+                                sizeof(RTHCPTR) * 2 * 2 + 1, "",
+                                pLookup->u.GCPhys.GCPhys, RT_ABS((int)(sizeof(RTHCPHYS) - sizeof(RTGCPHYS))) * 2, "",
+                                pLookup->pszDesc);
+                break;
+
+            case MMLOOKUPHYPERTYPE_MMIO2:
+                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %*s MMIO2   %RGp%*s %s\n",
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
+                                sizeof(RTHCPTR) * 2 * 2 + 1, "",
+                                pLookup->u.MMIO2.off, RT_ABS((int)(sizeof(RTHCPHYS) - sizeof(RTGCPHYS))) * 2, "",
+                                pLookup->pszDesc);
+                break;
+
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %*s DYNAMIC %*s %s\n",
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
+                                sizeof(RTHCPTR) * 2 * 2 + 1, "",
+                                sizeof(RTHCPTR) * 2, "",
+                                pLookup->pszDesc);
+                break;
+
+            default:
+                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+                break;
+        }
+
+        /* next */
+        if ((unsigned)pLookup->offNext == NIL_OFFSET)
+            break;
+        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
+    }
+}
+
+
+/**
+ * Re-allocates memory from the hyper heap.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pvOld           The existing block of memory in the hyper heap to
+ *                          re-allocate (can be NULL).
+ * @param   cbOld           Size of the existing block.
+ * @param   uAlignmentNew   Required memory alignment in bytes. Values are
+ *                          0,8,16,32 and PAGE_SIZE. 0 -> default alignment,
+ *                          i.e. 8 bytes.
+ * @param   enmTagNew       The statistics tag.
+ * @param   cbNew           The required size of the new block.
+ * @param   ppv             Where to store the address to the re-allocated
+ *                          block.
+ *
+ * @remarks This does not work like normal realloc() on failure, the memory
+ *          pointed to by @a pvOld is lost if there isn't sufficient space on
+ *          the hyper heap for the re-allocation to succeed.
+*/
+VMMR3DECL(int) MMR3HyperRealloc(PVM pVM, void *pvOld, size_t cbOld, unsigned uAlignmentNew, MMTAG enmTagNew, size_t cbNew,
+                                void **ppv)
+{
+    if (!pvOld)
+        return MMHyperAlloc(pVM, cbNew, uAlignmentNew, enmTagNew, ppv);
+
+    if (!cbNew && pvOld)
+        return MMHyperFree(pVM, pvOld);
+
+    if (cbOld == cbNew)
+        return VINF_SUCCESS;
+
+    size_t cbData = RT_MIN(cbNew, cbOld);
+    void *pvTmp = RTMemTmpAlloc(cbData);
+    if (RT_UNLIKELY(!pvTmp))
+    {
+        MMHyperFree(pVM, pvOld);
+        return VERR_NO_TMP_MEMORY;
+    }
+    memcpy(pvTmp, pvOld, cbData);
+
+    int rc = MMHyperFree(pVM, pvOld);
+    if (RT_SUCCESS(rc))
+    {
+        rc = MMHyperAlloc(pVM, cbNew, uAlignmentNew, enmTagNew, ppv);
+        if (RT_SUCCESS(rc))
+        {
+            Assert(cbData <= cbNew);
+            memcpy(*ppv, pvTmp, cbData);
+        }
+    }
+    else
+        AssertMsgFailed(("Failed to free hyper heap block pvOld=%p cbOld=%u\n", pvOld, cbOld));
+
+    RTMemTmpFree(pvTmp);
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/MMPagePool.cpp b/src/VBox/VMM/VMMR3/MMPagePool.cpp
new file mode 100644
index 00000000..8566bdc9
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/MMPagePool.cpp
@@ -0,0 +1,72 @@
+/* $Id: MMPagePool.cpp $ */
+/** @file
+ * MM - Memory Manager - Page Pool (what's left of it).
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM_POOL
+#include <VBox/vmm/mm.h>
+#include "MMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <iprt/errcore.h>
+#include <VBox/log.h>
+
+
+/**
+ * Gets the HC pointer to the dummy page.
+ *
+ * The dummy page is used as a place holder to prevent potential bugs
+ * from doing really bad things to the system.
+ *
+ * @returns Pointer to the dummy page.
+ * @param   pVM         The cross context VM structure.
+ * @thread  The Emulation Thread.
+ */
+VMMR3DECL(void *) MMR3PageDummyHCPtr(PVM pVM)
+{
+    VM_ASSERT_EMT(pVM);
+    if (!pVM->mm.s.pvDummyPage)
+    {
+        int rc = MMHyperAlloc(pVM, PAGE_SIZE, PAGE_SIZE, MM_TAG_PGM, &pVM->mm.s.pvDummyPage);
+        AssertRelease(RT_SUCCESS(rc));
+        AssertRelease(pVM->mm.s.pvDummyPage);
+        pVM->mm.s.HCPhysDummyPage = MMR3HyperHCVirt2HCPhys(pVM, pVM->mm.s.pvDummyPage);
+        AssertRelease(!(pVM->mm.s.HCPhysDummyPage & ~X86_PTE_PAE_PG_MASK));
+    }
+    return pVM->mm.s.pvDummyPage;
+}
+
+
+/**
+ * Gets the HC Phys to the dummy page.
+ *
+ * The dummy page is used as a place holder to prevent potential bugs
+ * from doing really bad things to the system.
+ *
+ * @returns Pointer to the dummy page.
+ * @param   pVM         The cross context VM structure.
+ * @thread  The Emulation Thread.
+ */
+VMMR3DECL(RTHCPHYS) MMR3PageDummyHCPhys(PVM pVM)
+{
+    VM_ASSERT_EMT(pVM);
+    if (!pVM->mm.s.pvDummyPage)
+        MMR3PageDummyHCPtr(pVM);
+    return pVM->mm.s.HCPhysDummyPage;
+}
+
diff --git a/src/VBox/VMM/VMMR3/MMUkHeap.cpp b/src/VBox/VMM/VMMR3/MMUkHeap.cpp
new file mode 100644
index 00000000..38e50be7
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/MMUkHeap.cpp
@@ -0,0 +1,427 @@
+/* $Id: MMUkHeap.cpp $ */
+/** @file
+ * MM - Memory Manager - Ring-3 Heap with kernel accessible mapping.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM_HEAP
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/stam.h>
+#include "MMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <iprt/errcore.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/heap.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static void *mmR3UkHeapAlloc(PMMUKHEAP pHeap, MMTAG enmTag, size_t cb, bool fZero, PRTR0PTR pR0Ptr);
+
+
+
+/**
+ * Create a User-kernel heap.
+ *
+ * This does not require SUPLib to be initialized as we'll lazily allocate the
+ * kernel accessible memory on the first alloc call.
+ *
+ * @returns VBox status code.
+ * @param   pUVM    Pointer to the user mode VM structure.
+ * @param   ppHeap  Where to store the heap pointer.
+ */
+int mmR3UkHeapCreateU(PUVM pUVM, PMMUKHEAP *ppHeap)
+{
+    PMMUKHEAP pHeap = (PMMUKHEAP)MMR3HeapAllocZU(pUVM, MM_TAG_MM, sizeof(MMUKHEAP));
+    if (pHeap)
+    {
+        int rc = RTCritSectInit(&pHeap->Lock);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Initialize the global stat record.
+             */
+            pHeap->pUVM = pUVM;
+#ifdef MMUKHEAP_WITH_STATISTICS
+            PMMUKHEAPSTAT pStat = &pHeap->Stat;
+            STAMR3RegisterU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cAllocations",     STAMUNIT_CALLS, "Number or MMR3UkHeapAlloc() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cReallocations",   STAMUNIT_CALLS, "Number of MMR3UkHeapRealloc() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cFrees",           STAMUNIT_CALLS, "Number of MMR3UkHeapFree() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cFailures",        STAMUNIT_COUNT, "Number of failures.");
+            STAMR3RegisterU(pUVM, &pStat->cbCurAllocated, sizeof(pStat->cbCurAllocated) == sizeof(uint32_t) ? STAMTYPE_U32 : STAMTYPE_U64,
+                                                                        STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbCurAllocated",   STAMUNIT_BYTES, "Number of bytes currently allocated.");
+            STAMR3RegisterU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbAllocated",      STAMUNIT_BYTES, "Total number of bytes allocated.");
+            STAMR3RegisterU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbFreed",          STAMUNIT_BYTES, "Total number of bytes freed.");
+#endif
+            *ppHeap = pHeap;
+            return VINF_SUCCESS;
+        }
+        AssertRC(rc);
+        MMR3HeapFree(pHeap);
+    }
+    AssertMsgFailed(("failed to allocate heap structure\n"));
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Destroy a User-kernel heap.
+ *
+ * @param   pHeap   Heap handle.
+ */
+void mmR3UkHeapDestroy(PMMUKHEAP pHeap)
+{
+    /*
+     * Start by deleting the lock, that'll trap anyone
+     * attempting to use the heap.
+     */
+    RTCritSectDelete(&pHeap->Lock);
+
+    /*
+     * Walk the sub-heaps and free them.
+     */
+    while (pHeap->pSubHeapHead)
+    {
+        PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
+        pHeap->pSubHeapHead = pSubHeap->pNext;
+        SUPR3PageFreeEx(pSubHeap->pv, pSubHeap->cb >> PAGE_SHIFT);
+        //MMR3HeapFree(pSubHeap); - rely on the automatic cleanup.
+    }
+    //MMR3HeapFree(pHeap->stats);
+    //MMR3HeapFree(pHeap);
+}
+
+
+/**
+ * Allocate memory associating it with the VM for collective cleanup.
+ *
+ * The memory will be allocated from the default heap but a header
+ * is added in which we keep track of which VM it belongs to and chain
+ * all the allocations together so they can be freed in one go.
+ *
+ * This interface is typically used for memory block which will not be
+ * freed during the life of the VM.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ * @param   pR0Ptr      Where to return the ring-0 address of the memory.
+ */
+VMMR3DECL(void *) MMR3UkHeapAlloc(PVM pVM, MMTAG enmTag, size_t cbSize, PRTR0PTR pR0Ptr)
+{
+    return mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, false, pR0Ptr);
+}
+
+
+/**
+ * Same as MMR3UkHeapAlloc().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ * @param   pR0Ptr      Where to return the ring-0 address of the memory.
+ */
+VMMR3DECL(int) MMR3UkHeapAllocEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv, PRTR0PTR pR0Ptr)
+{
+    void *pv = mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, false, pR0Ptr);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Same as MMR3UkHeapAlloc() only the memory is zeroed.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ * @param   pR0Ptr      Where to return the ring-0 address of the memory.
+ */
+VMMR3DECL(void *) MMR3UkHeapAllocZ(PVM pVM, MMTAG enmTag, size_t cbSize, PRTR0PTR pR0Ptr)
+{
+    return mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, true, pR0Ptr);
+}
+
+
+/**
+ * Same as MMR3UkHeapAllocZ().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ * @param   pR0Ptr      Where to return the ring-0 address of the memory.
+ */
+VMMR3DECL(int) MMR3UkHeapAllocZEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv, PRTR0PTR pR0Ptr)
+{
+    void *pv = mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, true, pR0Ptr);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/***
+ * Worker for mmR3UkHeapAlloc that creates and adds a new sub-heap.
+ *
+ * @returns Pointer to the new sub-heap.
+ * @param   pHeap       The heap
+ * @param   cbSubHeap   The size of the sub-heap.
+ */
+static PMMUKHEAPSUB mmR3UkHeapAddSubHeap(PMMUKHEAP pHeap, size_t cbSubHeap)
+{
+    PMMUKHEAPSUB pSubHeap = (PMMUKHEAPSUB)MMR3HeapAllocU(pHeap->pUVM, MM_TAG_MM/*_UK_HEAP*/, sizeof(*pSubHeap));
+    if (pSubHeap)
+    {
+        pSubHeap->cb = cbSubHeap;
+        int rc = SUPR3PageAllocEx(pSubHeap->cb >> PAGE_SHIFT, 0, &pSubHeap->pv, &pSubHeap->pvR0, NULL);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTHeapSimpleInit(&pSubHeap->hSimple, pSubHeap->pv, pSubHeap->cb);
+            if (RT_SUCCESS(rc))
+            {
+                pSubHeap->pNext = pHeap->pSubHeapHead;
+                pHeap->pSubHeapHead = pSubHeap;
+                return pSubHeap;
+            }
+
+            /* bail out */
+            SUPR3PageFreeEx(pSubHeap->pv, pSubHeap->cb >> PAGE_SHIFT);
+        }
+        MMR3HeapFree(pSubHeap);
+    }
+    return NULL;
+}
+
+
+/**
+ * Allocate memory from the heap.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pHeap       Heap handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cb          Size of the block.
+ * @param   fZero       Whether or not to zero the memory block.
+ * @param   pR0Ptr      Where to return the ring-0 pointer.
+ */
+static void *mmR3UkHeapAlloc(PMMUKHEAP pHeap, MMTAG enmTag, size_t cb, bool fZero, PRTR0PTR pR0Ptr)
+{
+    if (pR0Ptr)
+        *pR0Ptr = NIL_RTR0PTR;
+    RTCritSectEnter(&pHeap->Lock);
+
+#ifdef MMUKHEAP_WITH_STATISTICS
+    /*
+     * Find/alloc statistics nodes.
+     */
+    pHeap->Stat.cAllocations++;
+    PMMUKHEAPSTAT pStat = (PMMUKHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
+    if (pStat)
+        pStat->cAllocations++;
+    else
+    {
+        pStat = (PMMUKHEAPSTAT)MMR3HeapAllocZU(pHeap->pUVM, MM_TAG_MM, sizeof(MMUKHEAPSTAT));
+        if (!pStat)
+        {
+            pHeap->Stat.cFailures++;
+            AssertMsgFailed(("Failed to allocate heap stat record.\n"));
+            RTCritSectLeave(&pHeap->Lock);
+            return NULL;
+        }
+        pStat->Core.Key = (AVLULKEY)enmTag;
+        RTAvlULInsert(&pHeap->pStatTree, &pStat->Core);
+
+        pStat->cAllocations++;
+
+        /* register the statistics */
+        PUVM pUVM = pHeap->pUVM;
+        const char *pszTag = mmGetTagName(enmTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Number of bytes currently allocated.",    "/MM/UkHeap/%s", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number or MMR3UkHeapAlloc() calls.",      "/MM/UkHeap/%s/cAllocations", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3UkHeapRealloc() calls.",    "/MM/UkHeap/%s/cReallocations", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3UkHeapFree() calls.",       "/MM/UkHeap/%s/cFrees", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_COUNT, "Number of failures.",                     "/MM/UkHeap/%s/cFailures", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes allocated.",        "/MM/UkHeap/%s/cbAllocated", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes freed.",            "/MM/UkHeap/%s/cbFreed", pszTag);
+    }
+#else
+    RT_NOREF_PV(enmTag);
+#endif
+
+    /*
+     * Validate input.
+     */
+    if (cb == 0)
+    {
+#ifdef MMUKHEAP_WITH_STATISTICS
+        pStat->cFailures++;
+        pHeap->Stat.cFailures++;
+#endif
+        RTCritSectLeave(&pHeap->Lock);
+        return NULL;
+    }
+
+    /*
+     * Allocate heap block.
+     */
+    cb = RT_ALIGN_Z(cb, MMUKHEAP_SIZE_ALIGNMENT);
+    void *pv = NULL;
+    PMMUKHEAPSUB pSubHeapPrev = NULL;
+    PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
+    while (pSubHeap)
+    {
+        if (fZero)
+            pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
+        else
+            pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
+        if (pv)
+        {
+            /* Move the sub-heap with free memory to the head. */
+            if (pSubHeapPrev)
+            {
+                pSubHeapPrev->pNext = pSubHeap->pNext;
+                pSubHeap->pNext = pHeap->pSubHeapHead;
+                pHeap->pSubHeapHead = pSubHeap;
+            }
+            break;
+        }
+        pSubHeapPrev = pSubHeap;
+        pSubHeap = pSubHeap->pNext;
+    }
+    if (RT_UNLIKELY(!pv))
+    {
+        /*
+         * Add another sub-heap.
+         */
+        pSubHeap = mmR3UkHeapAddSubHeap(pHeap, RT_MAX(RT_ALIGN_Z(cb, PAGE_SIZE) + PAGE_SIZE * 16, _256K));
+        if (pSubHeap)
+        {
+            if (fZero)
+                pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
+            else
+                pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
+        }
+        if (RT_UNLIKELY(!pv))
+        {
+            AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cb, enmTag, &enmTag));
+#ifdef MMUKHEAP_WITH_STATISTICS
+            pStat->cFailures++;
+            pHeap->Stat.cFailures++;
+#endif
+            RTCritSectLeave(&pHeap->Lock);
+            return NULL;
+        }
+    }
+
+    /*
+     * Update statistics
+     */
+#ifdef MMUKHEAP_WITH_STATISTICS
+    size_t cbActual = RTHeapSimpleSize(pSubHeap->hSimple, pv);
+    pStat->cbAllocated          += cbActual;
+    pStat->cbCurAllocated       += cbActual;
+    pHeap->Stat.cbAllocated     += cbActual;
+    pHeap->Stat.cbCurAllocated  += cbActual;
+#endif
+
+    if (pR0Ptr)
+        *pR0Ptr = (uintptr_t)pv - (uintptr_t)pSubHeap->pv + pSubHeap->pvR0;
+    RTCritSectLeave(&pHeap->Lock);
+    return pv;
+}
+
+
+/**
+ * Releases memory allocated with MMR3UkHeapAlloc() and MMR3UkHeapAllocZ()
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pv          Pointer to the memory block to free.
+ * @param   enmTag      The allocation accounting tag.
+ */
+VMMR3DECL(void) MMR3UkHeapFree(PVM pVM, void *pv, MMTAG enmTag)
+{
+    /* Ignore NULL pointers. */
+    if (!pv)
+        return;
+
+    PMMUKHEAP pHeap = pVM->pUVM->mm.s.pUkHeap;
+    RTCritSectEnter(&pHeap->Lock);
+
+    /*
+     * Find the sub-heap and block
+     */
+#ifdef MMUKHEAP_WITH_STATISTICS
+    size_t cbActual = 0;
+#endif
+    PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
+    while (pSubHeap)
+    {
+        if ((uintptr_t)pv - (uintptr_t)pSubHeap->pv < pSubHeap->cb)
+        {
+#ifdef MMUKHEAP_WITH_STATISTICS
+            cbActual = RTHeapSimpleSize(pSubHeap->hSimple, pv);
+            PMMUKHEAPSTAT pStat = (PMMUKHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
+            if (pStat)
+            {
+                pStat->cFrees++;
+                pStat->cbCurAllocated     -= cbActual;
+                pStat->cbFreed            += cbActual;
+            }
+            pHeap->Stat.cFrees++;
+            pHeap->Stat.cbFreed           += cbActual;
+            pHeap->Stat.cbCurAllocated    -= cbActual;
+#else
+            RT_NOREF_PV(enmTag);
+#endif
+            RTHeapSimpleFree(pSubHeap->hSimple, pv);
+
+            RTCritSectLeave(&pHeap->Lock);
+            return;
+        }
+    }
+    AssertMsgFailed(("pv=%p\n", pv));
+}
+
diff --git a/src/VBox/VMM/VMMR3/Makefile.kup b/src/VBox/VMM/VMMR3/Makefile.kup
new file mode 100644
index 00000000..e69de29b
diff --git a/src/VBox/VMM/VMMR3/NEMR3.cpp b/src/VBox/VMM/VMMR3/NEMR3.cpp
new file mode 100644
index 00000000..5026342e
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/NEMR3.cpp
@@ -0,0 +1,531 @@
+/* $Id: NEMR3.cpp $ */
+/** @file
+ * NEM - Native execution manager.
+ */
+
+/*
+ * Copyright (C) 2018-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_nem NEM - Native Execution Manager.
+ *
+ * This is an alternative execution manage to HM and raw-mode.   On one host
+ * (Windows) we're forced to use this, on the others we just do it because we
+ * can.   Since this is host specific in nature, information about an
+ * implementation is contained in the NEMR3Native-xxxx.cpp files.
+ *
+ * @ref pg_nem_win
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_NEM
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/gim.h>
+#include "NEMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+
+#include <iprt/asm.h>
+
+
+
+/**
+ * Basic init and configuration reading.
+ *
+ * Always call NEMR3Term after calling this.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) NEMR3InitConfig(PVM pVM)
+{
+    LogFlow(("NEMR3Init\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompileMemberAlignment(VM, nem.s, 64);
+    AssertCompile(sizeof(pVM->nem.s) <= sizeof(pVM->nem.padding));
+
+    /*
+     * Initialize state info so NEMR3Term will always be happy.
+     * No returning prior to setting magics!
+     */
+    pVM->nem.s.u32Magic = NEM_MAGIC;
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        pVCpu->nem.s.u32Magic = NEMCPU_MAGIC;
+    }
+
+    /*
+     * Read configuration.
+     */
+    PCFGMNODE pCfgNem = CFGMR3GetChild(CFGMR3GetRoot(pVM), "NEM/");
+
+    /*
+     * Validate the NEM settings.
+     */
+    int rc = CFGMR3ValidateConfig(pCfgNem,
+                                  "/NEM/",
+                                  "Enabled"
+                                  "|Allow64BitGuests"
+                                  "|LovelyMesaDrvWorkaround"
+#ifdef RT_OS_WINDOWS
+                                  "|UseRing0Runloop"
+#endif
+                                  ,
+                                  "" /* pszValidNodes */, "NEM" /* pszWho */, 0 /* uInstance */);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /** @cfgm{/NEM/NEMEnabled, bool, true}
+     * Whether NEM is enabled. */
+    rc = CFGMR3QueryBoolDef(pCfgNem, "Enabled", &pVM->nem.s.fEnabled, true);
+    AssertLogRelRCReturn(rc, rc);
+
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    /** @cfgm{/NEM/Allow64BitGuests, bool, 32-bit:false, 64-bit:true}
+     * Enables AMD64 CPU features.
+     * On 32-bit hosts this isn't default and require host CPU support. 64-bit hosts
+     * already have the support. */
+    rc = CFGMR3QueryBoolDef(pCfgNem, "Allow64BitGuests", &pVM->nem.s.fAllow64BitGuests, HC_ARCH_BITS == 64);
+    AssertLogRelRCReturn(rc, rc);
+#else
+    pVM->nem.s.fAllow64BitGuests = false;
+#endif
+
+    /** @cfgm{/NEM/LovelyMesaDrvWorkaround, bool, false}
+     * Workaround for mesa vmsvga 3d driver making incorrect assumptions about
+     * the hypervisor it is running under. */
+    bool f;
+    rc = CFGMR3QueryBoolDef(pCfgNem, "LovelyMesaDrvWorkaround", &f, false);
+    AssertLogRelRCReturn(rc, rc);
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        pVCpu->nem.s.fTrapXcptGpForLovelyMesaDrv = f;
+    }
+
+#ifdef RT_OS_WINDOWS
+    /** @cfgm{/NEM/UseRing0Runloop, bool, true}
+     * Whether to use the ring-0 runloop (if enabled in the build) or the ring-3 one.
+     * The latter is generally slower.  This option serves as a way out in case
+     * something breaks in the ring-0 loop. */
+# ifdef NEM_WIN_USE_RING0_RUNLOOP_BY_DEFAULT
+    bool fUseRing0Runloop = true;
+# else
+    bool fUseRing0Runloop = false;
+# endif
+    rc = CFGMR3QueryBoolDef(pCfgNem, "UseRing0Runloop", &fUseRing0Runloop, fUseRing0Runloop);
+    AssertLogRelRCReturn(rc, rc);
+    pVM->nem.s.fUseRing0Runloop = fUseRing0Runloop;
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * This is called by HMR3Init() when HM cannot be used.
+ *
+ * Sets VM::bMainExecutionEngine to VM_EXEC_ENGINE_NATIVE_API if we can use a
+ * native hypervisor API to execute the VM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   fFallback   Whether this is a fallback call.  Cleared if the VM is
+ *                      configured to use NEM instead of HM.
+ * @param   fForced     Whether /HM/HMForced was set.  If set and we fail to
+ *                      enable NEM, we'll return a failure status code.
+ *                      Otherwise we'll assume HMR3Init falls back on raw-mode.
+ */
+VMMR3_INT_DECL(int) NEMR3Init(PVM pVM, bool fFallback, bool fForced)
+{
+    Assert(pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API);
+    int rc;
+    if (pVM->nem.s.fEnabled)
+    {
+#ifdef VBOX_WITH_NATIVE_NEM
+        rc = nemR3NativeInit(pVM, fFallback, fForced);
+        ASMCompilerBarrier(); /* May have changed bMainExecutionEngine. */
+#else
+        RT_NOREF(fFallback);
+        rc = VINF_SUCCESS;
+#endif
+        if (RT_SUCCESS(rc))
+        {
+            if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+                LogRel(("NEM: NEMR3Init: Active.\n"));
+            else
+            {
+                LogRel(("NEM: NEMR3Init: Not available.\n"));
+                if (fForced)
+                    rc = VERR_NEM_NOT_AVAILABLE;
+            }
+        }
+        else
+            LogRel(("NEM: NEMR3Init: Native init failed: %Rrc.\n", rc));
+    }
+    else
+    {
+        LogRel(("NEM: NEMR3Init: Disabled.\n"));
+        rc = fForced ? VERR_NEM_NOT_ENABLED : VINF_SUCCESS;
+    }
+    return rc;
+}
+
+
+/**
+ * Perform initialization that depends on CPUM working.
+ *
+ * This is a noop if NEM wasn't activated by a previous NEMR3Init() call.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) NEMR3InitAfterCPUM(PVM pVM)
+{
+    int rc = VINF_SUCCESS;
+    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+    {
+        /*
+         * Enable CPU features making general ASSUMPTIONS (there are two similar
+         * blocks of code in HM.cpp), to avoid duplicating this code.  The
+         * native backend can make check capabilities and adjust as needed.
+         */
+        CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SEP);
+        if (   CPUMGetGuestCpuVendor(pVM) == CPUMCPUVENDOR_AMD
+            || CPUMGetGuestCpuVendor(pVM) == CPUMCPUVENDOR_HYGON)
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SYSCALL);            /* 64 bits only on Intel CPUs */
+        if (pVM->nem.s.fAllow64BitGuests)
+        {
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SYSCALL);
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE);
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LAHF);
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
+        }
+        /* Turn on NXE if PAE has been enabled. */
+        else if (CPUMR3GetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE))
+            CPUMR3SetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
+
+        /*
+         * Do native after-CPUM init.
+         */
+#ifdef VBOX_WITH_NATIVE_NEM
+        rc = nemR3NativeInitAfterCPUM(pVM);
+#else
+        RT_NOREF(pVM);
+#endif
+    }
+    return rc;
+}
+
+
+/**
+ * Called when a init phase has completed.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmWhat     The phase that completed.
+ */
+VMMR3_INT_DECL(int) NEMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    /*
+     * Check if GIM needs #UD, since that applies to everyone.
+     */
+    if (enmWhat == VMINITCOMPLETED_RING3)
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+            pVCpu->nem.s.fGIMTrapXcptUD = GIMShouldTrapXcptUD(pVCpu);
+        }
+
+    /*
+     * Call native code.
+     */
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        rc = nemR3NativeInitCompleted(pVM, enmWhat);
+#else
+    RT_NOREF(pVM, enmWhat);
+#endif
+    return rc;
+}
+
+
+/**
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) NEMR3Term(PVM pVM)
+{
+    AssertReturn(pVM->nem.s.u32Magic == NEM_MAGIC, VERR_WRONG_ORDER);
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        AssertReturn(pVM->apCpusR3[idCpu]->nem.s.u32Magic == NEMCPU_MAGIC, VERR_WRONG_ORDER);
+
+    /* Do native termination. */
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        rc = nemR3NativeTerm(pVM);
+#endif
+
+    /* Mark it as terminated. */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        pVCpu->nem.s.u32Magic = NEMCPU_MAGIC_DEAD;
+    }
+    pVM->nem.s.u32Magic = NEM_MAGIC_DEAD;
+    return rc;
+}
+
+/**
+ * External interface for querying whether native execution API is used.
+ *
+ * @returns true if NEM is being used, otherwise false.
+ * @param   pUVM        The user mode VM handle.
+ * @sa      HMR3IsEnabled
+ */
+VMMR3DECL(bool) NEMR3IsEnabled(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return VM_IS_NEM_ENABLED(pVM);
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) NEMR3Reset(PVM pVM)
+{
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        nemR3NativeReset(pVM);
+#else
+    RT_NOREF(pVM);
+#endif
+}
+
+
+/**
+ * Resets a virtual CPU.
+ *
+ * Used to bring up secondary CPUs on SMP as well as CPU hot plugging.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure to reset.
+ * @param   fInitIpi    Set if being reset due to INIT IPI.
+ */
+VMMR3_INT_DECL(void) NEMR3ResetCpu(PVMCPU pVCpu, bool fInitIpi)
+{
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVCpu->pVMR3->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        nemR3NativeResetCpu(pVCpu, fInitIpi);
+#else
+    RT_NOREF(pVCpu, fInitIpi);
+#endif
+}
+
+
+/**
+ * Indicates to TM that TMTSCMODE_NATIVE_API should be used for TSC.
+ *
+ * @returns true if TMTSCMODE_NATIVE_API must be used, otherwise @c false.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(bool) NEMR3NeedSpecialTscMode(PVM pVM)
+{
+#ifdef VBOX_WITH_NATIVE_NEM
+# ifdef RT_OS_WINDOWS
+    if (VM_IS_NEM_ENABLED(pVM))
+        return true;
+# endif
+#else
+    RT_NOREF(pVM);
+#endif
+    return false;
+}
+
+
+/**
+ * Gets the name of a generic NEM exit code.
+ *
+ * @returns Pointer to read only string if @a uExit is known, otherwise NULL.
+ * @param   uExit               The NEM exit to name.
+ */
+VMMR3DECL(const char *) NEMR3GetExitName(uint32_t uExit)
+{
+    switch ((NEMEXITTYPE)uExit)
+    {
+        case NEMEXITTYPE_UNRECOVERABLE_EXCEPTION:       return "NEM unrecoverable exception";
+        case NEMEXITTYPE_INVALID_VP_REGISTER_VALUE:     return "NEM invalid vp register value";
+        case NEMEXITTYPE_INTTERRUPT_WINDOW:             return "NEM interrupt window";
+        case NEMEXITTYPE_HALT:                          return "NEM halt";
+        case NEMEXITTYPE_XCPT_UD:                       return "NEM #UD";
+        case NEMEXITTYPE_XCPT_DB:                       return "NEM #DB";
+        case NEMEXITTYPE_XCPT_BP:                       return "NEM #BP";
+        case NEMEXITTYPE_CANCELED:                      return "NEM canceled";
+        case NEMEXITTYPE_MEMORY_ACCESS:                 return "NEM memory access";
+    }
+
+    return NULL;
+}
+
+
+VMMR3_INT_DECL(VBOXSTRICTRC) NEMR3RunGC(PVM pVM, PVMCPU pVCpu)
+{
+    Assert(VM_IS_NEM_ENABLED(pVM));
+#ifdef VBOX_WITH_NATIVE_NEM
+    return nemR3NativeRunGC(pVM, pVCpu);
+#else
+    NOREF(pVM); NOREF(pVCpu);
+    return VERR_INTERNAL_ERROR_3;
+#endif
+}
+
+
+VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
+{
+    Assert(VM_IS_NEM_ENABLED(pVM));
+#ifdef VBOX_WITH_NATIVE_NEM
+    return nemR3NativeCanExecuteGuest(pVM, pVCpu);
+#else
+    NOREF(pVM); NOREF(pVCpu);
+    return false;
+#endif
+}
+
+
+VMMR3_INT_DECL(bool) NEMR3SetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
+{
+    Assert(VM_IS_NEM_ENABLED(pVM));
+#ifdef VBOX_WITH_NATIVE_NEM
+    return nemR3NativeSetSingleInstruction(pVM, pVCpu, fEnable);
+#else
+    NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
+    return false;
+#endif
+}
+
+
+VMMR3_INT_DECL(void) NEMR3NotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
+{
+    AssertLogRelReturnVoid(VM_IS_NEM_ENABLED(pVM));
+#ifdef VBOX_WITH_NATIVE_NEM
+    nemR3NativeNotifyFF(pVM, pVCpu, fFlags);
+#else
+    RT_NOREF(pVM, pVCpu, fFlags);
+#endif
+}
+
+
+
+
+VMMR3_INT_DECL(int)  NEMR3NotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        rc = nemR3NativeNotifyPhysRamRegister(pVM, GCPhys, cb);
+#else
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb);
+#endif
+    return rc;
+}
+
+
+VMMR3_INT_DECL(int)  NEMR3NotifyPhysMmioExMap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvMmio2)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        rc = nemR3NativeNotifyPhysMmioExMap(pVM, GCPhys, cb, fFlags, pvMmio2);
+#else
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags); NOREF(pvMmio2);
+#endif
+    return rc;
+}
+
+
+VMMR3_INT_DECL(int)  NEMR3NotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        rc = nemR3NativeNotifyPhysMmioExUnmap(pVM, GCPhys, cb, fFlags);
+#else
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
+#endif
+    return rc;
+}
+
+
+VMMR3_INT_DECL(int)  NEMR3NotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        rc = nemR3NativeNotifyPhysRomRegisterEarly(pVM, GCPhys, cb, fFlags);
+#else
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
+#endif
+    return rc;
+}
+
+
+/**
+ * Called after the ROM range has been fully completed.
+ *
+ * This will be preceeded by a NEMR3NotifyPhysRomRegisterEarly() call as well a
+ * number of NEMHCNotifyPhysPageProtChanged calls.
+ *
+ * @returns VBox status code
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The ROM address (page aligned).
+ * @param   cb              The size (page aligned).
+ * @param   fFlags          NEM_NOTIFY_PHYS_ROM_F_XXX.
+ */
+VMMR3_INT_DECL(int)  NEMR3NotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        rc = nemR3NativeNotifyPhysRomRegisterLate(pVM, GCPhys, cb, fFlags);
+#else
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
+#endif
+    return rc;
+}
+
+
+VMMR3_INT_DECL(void) NEMR3NotifySetA20(PVMCPU pVCpu, bool fEnabled)
+{
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVCpu->pVMR3->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API)
+        nemR3NativeNotifySetA20(pVCpu, fEnabled);
+#else
+    NOREF(pVCpu); NOREF(fEnabled);
+#endif
+}
+
diff --git a/src/VBox/VMM/VMMR3/NEMR3Native-win.cpp b/src/VBox/VMM/VMMR3/NEMR3Native-win.cpp
new file mode 100644
index 00000000..39d0cf56
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/NEMR3Native-win.cpp
@@ -0,0 +1,2790 @@
+/* $Id: NEMR3Native-win.cpp $ */
+/** @file
+ * NEM - Native execution manager, native ring-3 Windows backend.
+ *
+ * Log group 2: Exit logging.
+ * Log group 3: Log context on exit.
+ * Log group 5: Ring-3 memory management
+ * Log group 6: Ring-0 memory management
+ * Log group 12: API intercepts.
+ */
+
+/*
+ * Copyright (C) 2018-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_NEM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <iprt/nt/nt-and-windows.h>
+#include <iprt/nt/hyperv.h>
+#include <iprt/nt/vid.h>
+#include <WinHvPlatform.h>
+
+#ifndef _WIN32_WINNT_WIN10
+# error "Missing _WIN32_WINNT_WIN10"
+#endif
+#ifndef _WIN32_WINNT_WIN10_RS1 /* Missing define, causing trouble for us. */
+# define _WIN32_WINNT_WIN10_RS1 (_WIN32_WINNT_WIN10 + 1)
+#endif
+#include <sysinfoapi.h>
+#include <debugapi.h>
+#include <errhandlingapi.h>
+#include <fileapi.h>
+#include <winerror.h> /* no api header for this. */
+
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/dbgftrace.h>
+#include "NEMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <iprt/ldr.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+#include <iprt/system.h>
+#include <iprt/utf16.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#ifdef LOG_ENABLED
+# define NEM_WIN_INTERCEPT_NT_IO_CTLS
+#endif
+
+/** VID I/O control detection: Fake partition handle input. */
+#define NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE          ((HANDLE)(uintptr_t)38479125)
+/** VID I/O control detection: Fake partition ID return. */
+#define NEM_WIN_IOCTL_DETECTOR_FAKE_PARTITION_ID    UINT64_C(0xfa1e000042424242)
+/** VID I/O control detection: Fake CPU index input. */
+#define NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX        UINT32_C(42)
+/** VID I/O control detection: Fake timeout input. */
+#define NEM_WIN_IOCTL_DETECTOR_FAKE_TIMEOUT         UINT32_C(0x00080286)
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** @name APIs imported from WinHvPlatform.dll
+ * @{ */
+static decltype(WHvGetCapability) *                 g_pfnWHvGetCapability;
+static decltype(WHvCreatePartition) *               g_pfnWHvCreatePartition;
+static decltype(WHvSetupPartition) *                g_pfnWHvSetupPartition;
+static decltype(WHvDeletePartition) *               g_pfnWHvDeletePartition;
+static decltype(WHvGetPartitionProperty) *          g_pfnWHvGetPartitionProperty;
+static decltype(WHvSetPartitionProperty) *          g_pfnWHvSetPartitionProperty;
+static decltype(WHvMapGpaRange) *                   g_pfnWHvMapGpaRange;
+static decltype(WHvUnmapGpaRange) *                 g_pfnWHvUnmapGpaRange;
+static decltype(WHvTranslateGva) *                  g_pfnWHvTranslateGva;
+#ifndef NEM_WIN_USE_OUR_OWN_RUN_API
+static decltype(WHvCreateVirtualProcessor) *        g_pfnWHvCreateVirtualProcessor;
+static decltype(WHvDeleteVirtualProcessor) *        g_pfnWHvDeleteVirtualProcessor;
+static decltype(WHvRunVirtualProcessor) *           g_pfnWHvRunVirtualProcessor;
+static decltype(WHvCancelRunVirtualProcessor) *     g_pfnWHvCancelRunVirtualProcessor;
+static decltype(WHvGetVirtualProcessorRegisters) *  g_pfnWHvGetVirtualProcessorRegisters;
+static decltype(WHvSetVirtualProcessorRegisters) *  g_pfnWHvSetVirtualProcessorRegisters;
+#endif
+/** @} */
+
+/** @name APIs imported from Vid.dll
+ * @{ */
+static decltype(VidGetHvPartitionId)               *g_pfnVidGetHvPartitionId;
+static decltype(VidStartVirtualProcessor)          *g_pfnVidStartVirtualProcessor;
+static decltype(VidStopVirtualProcessor)           *g_pfnVidStopVirtualProcessor;
+static decltype(VidMessageSlotMap)                 *g_pfnVidMessageSlotMap;
+static decltype(VidMessageSlotHandleAndGetNext)    *g_pfnVidMessageSlotHandleAndGetNext;
+#ifdef LOG_ENABLED
+static decltype(VidGetVirtualProcessorState)       *g_pfnVidGetVirtualProcessorState;
+static decltype(VidSetVirtualProcessorState)       *g_pfnVidSetVirtualProcessorState;
+static decltype(VidGetVirtualProcessorRunningStatus) *g_pfnVidGetVirtualProcessorRunningStatus;
+#endif
+/** @} */
+
+/** The Windows build number. */
+static uint32_t g_uBuildNo = 17134;
+
+
+
+/**
+ * Import instructions.
+ */
+static const struct
+{
+    uint8_t     idxDll;     /**< 0 for WinHvPlatform.dll, 1 for vid.dll. */
+    bool        fOptional;  /**< Set if import is optional. */
+    PFNRT      *ppfn;       /**< The function pointer variable. */
+    const char *pszName;    /**< The function name. */
+} g_aImports[] =
+{
+#define NEM_WIN_IMPORT(a_idxDll, a_fOptional, a_Name) { (a_idxDll), (a_fOptional), (PFNRT *)&RT_CONCAT(g_pfn,a_Name), #a_Name }
+    NEM_WIN_IMPORT(0, false, WHvGetCapability),
+    NEM_WIN_IMPORT(0, false, WHvCreatePartition),
+    NEM_WIN_IMPORT(0, false, WHvSetupPartition),
+    NEM_WIN_IMPORT(0, false, WHvDeletePartition),
+    NEM_WIN_IMPORT(0, false, WHvGetPartitionProperty),
+    NEM_WIN_IMPORT(0, false, WHvSetPartitionProperty),
+    NEM_WIN_IMPORT(0, false, WHvMapGpaRange),
+    NEM_WIN_IMPORT(0, false, WHvUnmapGpaRange),
+    NEM_WIN_IMPORT(0, false, WHvTranslateGva),
+#ifndef NEM_WIN_USE_OUR_OWN_RUN_API
+    NEM_WIN_IMPORT(0, false, WHvCreateVirtualProcessor),
+    NEM_WIN_IMPORT(0, false, WHvDeleteVirtualProcessor),
+    NEM_WIN_IMPORT(0, false, WHvRunVirtualProcessor),
+    NEM_WIN_IMPORT(0, false, WHvCancelRunVirtualProcessor),
+    NEM_WIN_IMPORT(0, false, WHvGetVirtualProcessorRegisters),
+    NEM_WIN_IMPORT(0, false, WHvSetVirtualProcessorRegisters),
+#endif
+    NEM_WIN_IMPORT(1, false, VidGetHvPartitionId),
+    NEM_WIN_IMPORT(1, false, VidMessageSlotMap),
+    NEM_WIN_IMPORT(1, false, VidMessageSlotHandleAndGetNext),
+    NEM_WIN_IMPORT(1, false, VidStartVirtualProcessor),
+    NEM_WIN_IMPORT(1, false, VidStopVirtualProcessor),
+#ifdef LOG_ENABLED
+    NEM_WIN_IMPORT(1, false, VidGetVirtualProcessorState),
+    NEM_WIN_IMPORT(1, false, VidSetVirtualProcessorState),
+    NEM_WIN_IMPORT(1, false, VidGetVirtualProcessorRunningStatus),
+#endif
+#undef NEM_WIN_IMPORT
+};
+
+
+/** The real NtDeviceIoControlFile API in NTDLL.   */
+static decltype(NtDeviceIoControlFile) *g_pfnNtDeviceIoControlFile;
+/** Pointer to the NtDeviceIoControlFile import table entry. */
+static decltype(NtDeviceIoControlFile) **g_ppfnVidNtDeviceIoControlFile;
+/** Info about the VidGetHvPartitionId I/O control interface. */
+static NEMWINIOCTL g_IoCtlGetHvPartitionId;
+/** Info about the VidStartVirtualProcessor I/O control interface. */
+static NEMWINIOCTL g_IoCtlStartVirtualProcessor;
+/** Info about the VidStopVirtualProcessor I/O control interface. */
+static NEMWINIOCTL g_IoCtlStopVirtualProcessor;
+/** Info about the VidMessageSlotHandleAndGetNext I/O control interface. */
+static NEMWINIOCTL g_IoCtlMessageSlotHandleAndGetNext;
+#ifdef LOG_ENABLED
+/** Info about the VidMessageSlotMap I/O control interface - for logging. */
+static NEMWINIOCTL g_IoCtlMessageSlotMap;
+/** Info about the VidGetVirtualProcessorState I/O control interface - for logging. */
+static NEMWINIOCTL g_IoCtlGetVirtualProcessorState;
+/** Info about the VidSetVirtualProcessorState I/O control interface - for logging. */
+static NEMWINIOCTL g_IoCtlSetVirtualProcessorState;
+/** Pointer to what nemR3WinIoctlDetector_ForLogging should fill in. */
+static NEMWINIOCTL *g_pIoCtlDetectForLogging;
+#endif
+
+#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
+/** Mapping slot for CPU #0.
+ * @{  */
+static VID_MESSAGE_MAPPING_HEADER            *g_pMsgSlotMapping = NULL;
+static const HV_MESSAGE_HEADER               *g_pHvMsgHdr;
+static const HV_X64_INTERCEPT_MESSAGE_HEADER *g_pX64MsgHdr;
+/** @} */
+#endif
+
+
+/*
+ * Let the preprocessor alias the APIs to import variables for better autocompletion.
+ */
+#ifndef IN_SLICKEDIT
+# define WHvGetCapability                           g_pfnWHvGetCapability
+# define WHvCreatePartition                         g_pfnWHvCreatePartition
+# define WHvSetupPartition                          g_pfnWHvSetupPartition
+# define WHvDeletePartition                         g_pfnWHvDeletePartition
+# define WHvGetPartitionProperty                    g_pfnWHvGetPartitionProperty
+# define WHvSetPartitionProperty                    g_pfnWHvSetPartitionProperty
+# define WHvMapGpaRange                             g_pfnWHvMapGpaRange
+# define WHvUnmapGpaRange                           g_pfnWHvUnmapGpaRange
+# define WHvTranslateGva                            g_pfnWHvTranslateGva
+# define WHvCreateVirtualProcessor                  g_pfnWHvCreateVirtualProcessor
+# define WHvDeleteVirtualProcessor                  g_pfnWHvDeleteVirtualProcessor
+# define WHvRunVirtualProcessor                     g_pfnWHvRunVirtualProcessor
+# define WHvGetRunExitContextSize                   g_pfnWHvGetRunExitContextSize
+# define WHvCancelRunVirtualProcessor               g_pfnWHvCancelRunVirtualProcessor
+# define WHvGetVirtualProcessorRegisters            g_pfnWHvGetVirtualProcessorRegisters
+# define WHvSetVirtualProcessorRegisters            g_pfnWHvSetVirtualProcessorRegisters
+
+# define VidMessageSlotHandleAndGetNext             g_pfnVidMessageSlotHandleAndGetNext
+# define VidStartVirtualProcessor                   g_pfnVidStartVirtualProcessor
+# define VidStopVirtualProcessor                    g_pfnVidStopVirtualProcessor
+
+#endif
+
+/** WHV_MEMORY_ACCESS_TYPE names */
+static const char * const g_apszWHvMemAccesstypes[4] = { "read", "write", "exec", "!undefined!" };
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+
+/*
+ * Instantate the code we share with ring-0.
+ */
+#ifdef NEM_WIN_USE_OUR_OWN_RUN_API
+# define NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+#else
+# undef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
+#endif
+#include "../VMMAll/NEMAllNativeTemplate-win.cpp.h"
+
+
+
+#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
+/**
+ * Wrapper that logs the call from VID.DLL.
+ *
+ * This is very handy for figuring out why an API call fails.
+ */
+static NTSTATUS WINAPI
+nemR3WinLogWrapper_NtDeviceIoControlFile(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
+                                         PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
+                                         PVOID pvOutput, ULONG cbOutput)
+{
+
+    char szFunction[32];
+    const char *pszFunction;
+    if (uFunction == g_IoCtlMessageSlotHandleAndGetNext.uFunction)
+        pszFunction = "VidMessageSlotHandleAndGetNext";
+    else if (uFunction == g_IoCtlStartVirtualProcessor.uFunction)
+        pszFunction = "VidStartVirtualProcessor";
+    else if (uFunction == g_IoCtlStopVirtualProcessor.uFunction)
+        pszFunction = "VidStopVirtualProcessor";
+    else if (uFunction == g_IoCtlMessageSlotMap.uFunction)
+        pszFunction = "VidMessageSlotMap";
+    else if (uFunction == g_IoCtlGetVirtualProcessorState.uFunction)
+        pszFunction = "VidGetVirtualProcessorState";
+    else if (uFunction == g_IoCtlSetVirtualProcessorState.uFunction)
+        pszFunction = "VidSetVirtualProcessorState";
+    else
+    {
+        RTStrPrintf(szFunction, sizeof(szFunction), "%#x", uFunction);
+        pszFunction = szFunction;
+    }
+
+    if (cbInput > 0 && pvInput)
+        Log12(("VID!NtDeviceIoControlFile: %s/input: %.*Rhxs\n", pszFunction, RT_MIN(cbInput, 32), pvInput));
+    NTSTATUS rcNt = g_pfnNtDeviceIoControlFile(hFile, hEvt, pfnApcCallback, pvApcCtx, pIos, uFunction,
+                                               pvInput, cbInput, pvOutput, cbOutput);
+    if (!hEvt && !pfnApcCallback && !pvApcCtx)
+        Log12(("VID!NtDeviceIoControlFile: hFile=%#zx pIos=%p->{s:%#x, i:%#zx} uFunction=%s Input=%p LB %#x Output=%p LB %#x) -> %#x; Caller=%p\n",
+               hFile, pIos, pIos->Status, pIos->Information, pszFunction, pvInput, cbInput, pvOutput, cbOutput, rcNt, ASMReturnAddress()));
+    else
+        Log12(("VID!NtDeviceIoControlFile: hFile=%#zx hEvt=%#zx Apc=%p/%p pIos=%p->{s:%#x, i:%#zx} uFunction=%s Input=%p LB %#x Output=%p LB %#x) -> %#x; Caller=%p\n",
+               hFile, hEvt, pfnApcCallback, pvApcCtx, pIos, pIos->Status, pIos->Information, pszFunction,
+               pvInput, cbInput, pvOutput, cbOutput, rcNt, ASMReturnAddress()));
+    if (cbOutput > 0 && pvOutput)
+    {
+        Log12(("VID!NtDeviceIoControlFile: %s/output: %.*Rhxs\n", pszFunction, RT_MIN(cbOutput, 32), pvOutput));
+        if (uFunction == 0x2210cc && g_pMsgSlotMapping == NULL && cbOutput >= sizeof(void *))
+        {
+            g_pMsgSlotMapping = *(VID_MESSAGE_MAPPING_HEADER **)pvOutput;
+            g_pHvMsgHdr       = (const HV_MESSAGE_HEADER               *)(g_pMsgSlotMapping + 1);
+            g_pX64MsgHdr      = (const HV_X64_INTERCEPT_MESSAGE_HEADER *)(g_pHvMsgHdr + 1);
+            Log12(("VID!NtDeviceIoControlFile: Message slot mapping: %p\n", g_pMsgSlotMapping));
+        }
+    }
+    if (   g_pMsgSlotMapping
+        && (   uFunction == g_IoCtlMessageSlotHandleAndGetNext.uFunction
+            || uFunction == g_IoCtlStopVirtualProcessor.uFunction
+            || uFunction == g_IoCtlMessageSlotMap.uFunction
+               ))
+        Log12(("VID!NtDeviceIoControlFile: enmVidMsgType=%#x cb=%#x msg=%#x payload=%u cs:rip=%04x:%08RX64 (%s)\n",
+               g_pMsgSlotMapping->enmVidMsgType, g_pMsgSlotMapping->cbMessage,
+               g_pHvMsgHdr->MessageType, g_pHvMsgHdr->PayloadSize,
+               g_pX64MsgHdr->CsSegment.Selector, g_pX64MsgHdr->Rip, pszFunction));
+
+    return rcNt;
+}
+#endif /* NEM_WIN_INTERCEPT_NT_IO_CTLS */
+
+
+/**
+ * Patches the call table of VID.DLL so we can intercept NtDeviceIoControlFile.
+ *
+ * This is for used to figure out the I/O control codes and in logging builds
+ * for logging API calls that WinHvPlatform.dll does.
+ *
+ * @returns VBox status code.
+ * @param   hLdrModVid      The VID module handle.
+ * @param   pErrInfo        Where to return additional error information.
+ */
+static int nemR3WinInitVidIntercepts(RTLDRMOD hLdrModVid, PRTERRINFO pErrInfo)
+{
+    /*
+     * Locate the real API.
+     */
+    g_pfnNtDeviceIoControlFile = (decltype(NtDeviceIoControlFile) *)RTLdrGetSystemSymbol("NTDLL.DLL", "NtDeviceIoControlFile");
+    AssertReturn(g_pfnNtDeviceIoControlFile != NULL,
+                 RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Failed to resolve NtDeviceIoControlFile from NTDLL.DLL"));
+
+    /*
+     * Locate the PE header and get what we need from it.
+     */
+    uint8_t const *pbImage = (uint8_t const *)RTLdrGetNativeHandle(hLdrModVid);
+    IMAGE_DOS_HEADER const *pMzHdr  = (IMAGE_DOS_HEADER const *)pbImage;
+    AssertReturn(pMzHdr->e_magic == IMAGE_DOS_SIGNATURE,
+                 RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL mapping doesn't start with MZ signature: %#x", pMzHdr->e_magic));
+    IMAGE_NT_HEADERS const *pNtHdrs = (IMAGE_NT_HEADERS const *)&pbImage[pMzHdr->e_lfanew];
+    AssertReturn(pNtHdrs->Signature == IMAGE_NT_SIGNATURE,
+                 RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL has invalid PE signaturre: %#x @%#x",
+                               pNtHdrs->Signature, pMzHdr->e_lfanew));
+
+    uint32_t const             cbImage   = pNtHdrs->OptionalHeader.SizeOfImage;
+    IMAGE_DATA_DIRECTORY const ImportDir = pNtHdrs->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT];
+
+    /*
+     * Walk the import descriptor table looking for NTDLL.DLL.
+     */
+    AssertReturn(   ImportDir.Size > 0
+                 && ImportDir.Size < cbImage,
+                 RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad import directory size: %#x", ImportDir.Size));
+    AssertReturn(   ImportDir.VirtualAddress > 0
+                 && ImportDir.VirtualAddress <= cbImage - ImportDir.Size,
+                 RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad import directory RVA: %#x", ImportDir.VirtualAddress));
+
+    for (PIMAGE_IMPORT_DESCRIPTOR pImps = (PIMAGE_IMPORT_DESCRIPTOR)&pbImage[ImportDir.VirtualAddress];
+         pImps->Name != 0 && pImps->FirstThunk != 0;
+         pImps++)
+    {
+        AssertReturn(pImps->Name < cbImage,
+                     RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad import directory entry name: %#x", pImps->Name));
+        const char *pszModName = (const char *)&pbImage[pImps->Name];
+        if (RTStrICmpAscii(pszModName, "ntdll.dll"))
+            continue;
+        AssertReturn(pImps->FirstThunk < cbImage,
+                     RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad FirstThunk: %#x", pImps->FirstThunk));
+        AssertReturn(pImps->OriginalFirstThunk < cbImage,
+                     RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad FirstThunk: %#x", pImps->FirstThunk));
+
+        /*
+         * Walk the thunks table(s) looking for NtDeviceIoControlFile.
+         */
+        PIMAGE_THUNK_DATA pFirstThunk = (PIMAGE_THUNK_DATA)&pbImage[pImps->FirstThunk]; /* update this. */
+        PIMAGE_THUNK_DATA pThunk      = pImps->OriginalFirstThunk == 0                  /* read from this. */
+                                      ? (PIMAGE_THUNK_DATA)&pbImage[pImps->FirstThunk]
+                                      : (PIMAGE_THUNK_DATA)&pbImage[pImps->OriginalFirstThunk];
+        while (pThunk->u1.Ordinal != 0)
+        {
+            if (!(pThunk->u1.Ordinal & IMAGE_ORDINAL_FLAG32))
+            {
+                AssertReturn(pThunk->u1.Ordinal > 0 && pThunk->u1.Ordinal < cbImage,
+                             RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad FirstThunk: %#x", pImps->FirstThunk));
+
+                const char *pszSymbol = (const char *)&pbImage[(uintptr_t)pThunk->u1.AddressOfData + 2];
+                if (strcmp(pszSymbol, "NtDeviceIoControlFile") == 0)
+                {
+                    DWORD fOldProt = PAGE_READONLY;
+                    VirtualProtect(&pFirstThunk->u1.Function, sizeof(uintptr_t), PAGE_EXECUTE_READWRITE, &fOldProt);
+                    g_ppfnVidNtDeviceIoControlFile = (decltype(NtDeviceIoControlFile) **)&pFirstThunk->u1.Function;
+                    /* Don't restore the protection here, so we modify the NtDeviceIoControlFile pointer later. */
+                }
+            }
+
+            pThunk++;
+            pFirstThunk++;
+        }
+    }
+
+    if (*g_ppfnVidNtDeviceIoControlFile)
+    {
+#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
+        *g_ppfnVidNtDeviceIoControlFile = nemR3WinLogWrapper_NtDeviceIoControlFile;
+#endif
+        return VINF_SUCCESS;
+    }
+    return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Failed to patch NtDeviceIoControlFile import in VID.DLL!");
+}
+
+
+/**
+ * Worker for nemR3NativeInit that probes and load the native API.
+ *
+ * @returns VBox status code.
+ * @param   fForced             Whether the HMForced flag is set and we should
+ *                              fail if we cannot initialize.
+ * @param   pErrInfo            Where to always return error info.
+ */
+static int nemR3WinInitProbeAndLoad(bool fForced, PRTERRINFO pErrInfo)
+{
+    /*
+     * Check that the DLL files we need are present, but without loading them.
+     * We'd like to avoid loading them unnecessarily.
+     */
+    WCHAR wszPath[MAX_PATH + 64];
+    UINT  cwcPath = GetSystemDirectoryW(wszPath, MAX_PATH);
+    if (cwcPath >= MAX_PATH || cwcPath < 2)
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "GetSystemDirectoryW failed (%#x / %u)", cwcPath, GetLastError());
+
+    if (wszPath[cwcPath - 1] != '\\' || wszPath[cwcPath - 1] != '/')
+        wszPath[cwcPath++] = '\\';
+    RTUtf16CopyAscii(&wszPath[cwcPath], RT_ELEMENTS(wszPath) - cwcPath, "WinHvPlatform.dll");
+    if (GetFileAttributesW(wszPath) == INVALID_FILE_ATTRIBUTES)
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "The native API dll was not found (%ls)", wszPath);
+
+    /*
+     * Check that we're in a VM and that the hypervisor identifies itself as Hyper-V.
+     */
+    if (!ASMHasCpuId())
+        return RTErrInfoSet(pErrInfo, VERR_NEM_NOT_AVAILABLE, "No CPUID support");
+    if (!ASMIsValidStdRange(ASMCpuId_EAX(0)))
+        return RTErrInfoSet(pErrInfo, VERR_NEM_NOT_AVAILABLE, "No CPUID leaf #1");
+    if (!(ASMCpuId_ECX(1) & X86_CPUID_FEATURE_ECX_HVP))
+        return RTErrInfoSet(pErrInfo, VERR_NEM_NOT_AVAILABLE, "Not in a hypervisor partition (HVP=0)");
+
+    uint32_t cMaxHyperLeaf = 0;
+    uint32_t uEbx = 0;
+    uint32_t uEcx = 0;
+    uint32_t uEdx = 0;
+    ASMCpuIdExSlow(0x40000000, 0, 0, 0, &cMaxHyperLeaf, &uEbx, &uEcx, &uEdx);
+    if (!ASMIsValidHypervisorRange(cMaxHyperLeaf))
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "Invalid hypervisor CPUID range (%#x %#x %#x %#x)",
+                             cMaxHyperLeaf, uEbx, uEcx, uEdx);
+    if (   uEbx != UINT32_C(0x7263694d) /* Micr */
+        || uEcx != UINT32_C(0x666f736f) /* osof */
+        || uEdx != UINT32_C(0x76482074) /* t Hv */)
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE,
+                             "Not Hyper-V CPUID signature: %#x %#x %#x (expected %#x %#x %#x)",
+                             uEbx, uEcx, uEdx, UINT32_C(0x7263694d), UINT32_C(0x666f736f), UINT32_C(0x76482074));
+    if (cMaxHyperLeaf < UINT32_C(0x40000005))
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "Too narrow hypervisor CPUID range (%#x)", cMaxHyperLeaf);
+
+    /** @todo would be great if we could recognize a root partition from the
+     *        CPUID info, but I currently don't dare do that. */
+
+    /*
+     * Now try load the DLLs and resolve the APIs.
+     */
+    static const char * const s_apszDllNames[2] = { "WinHvPlatform.dll",  "vid.dll" };
+    RTLDRMOD                  ahMods[2]         = { NIL_RTLDRMOD,          NIL_RTLDRMOD };
+    int                       rc = VINF_SUCCESS;
+    for (unsigned i = 0; i < RT_ELEMENTS(s_apszDllNames); i++)
+    {
+        int rc2 = RTLdrLoadSystem(s_apszDllNames[i], true /*fNoUnload*/, &ahMods[i]);
+        if (RT_FAILURE(rc2))
+        {
+            if (!RTErrInfoIsSet(pErrInfo))
+                RTErrInfoSetF(pErrInfo, rc2, "Failed to load API DLL: %s: %Rrc", s_apszDllNames[i], rc2);
+            else
+                RTErrInfoAddF(pErrInfo, rc2, "; %s: %Rrc", s_apszDllNames[i], rc2);
+            ahMods[i] = NIL_RTLDRMOD;
+            rc = VERR_NEM_INIT_FAILED;
+        }
+    }
+    if (RT_SUCCESS(rc))
+        rc = nemR3WinInitVidIntercepts(ahMods[1], pErrInfo);
+    if (RT_SUCCESS(rc))
+    {
+        for (unsigned i = 0; i < RT_ELEMENTS(g_aImports); i++)
+        {
+            int rc2 = RTLdrGetSymbol(ahMods[g_aImports[i].idxDll], g_aImports[i].pszName, (void **)g_aImports[i].ppfn);
+            if (RT_FAILURE(rc2))
+            {
+                *g_aImports[i].ppfn = NULL;
+
+                LogRel(("NEM:  %s: Failed to import %s!%s: %Rrc",
+                        g_aImports[i].fOptional ? "info" : fForced ? "fatal" : "error",
+                        s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName, rc2));
+                if (!g_aImports[i].fOptional)
+                {
+                    if (RTErrInfoIsSet(pErrInfo))
+                        RTErrInfoAddF(pErrInfo, rc2, ", %s!%s",
+                                      s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName);
+                    else
+                        rc = RTErrInfoSetF(pErrInfo, rc2, "Failed to import: %s!%s",
+                                           s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName);
+                    Assert(RT_FAILURE(rc));
+                }
+            }
+        }
+        if (RT_SUCCESS(rc))
+        {
+            Assert(!RTErrInfoIsSet(pErrInfo));
+        }
+    }
+
+    for (unsigned i = 0; i < RT_ELEMENTS(ahMods); i++)
+        RTLdrClose(ahMods[i]);
+    return rc;
+}
+
+
+/**
+ * Wrapper for different WHvGetCapability signatures.
+ */
+DECLINLINE(HRESULT) WHvGetCapabilityWrapper(WHV_CAPABILITY_CODE enmCap, WHV_CAPABILITY *pOutput, uint32_t cbOutput)
+{
+    return g_pfnWHvGetCapability(enmCap, pOutput, cbOutput, NULL);
+}
+
+
+/**
+ * Worker for nemR3NativeInit that gets the hypervisor capabilities.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pErrInfo            Where to always return error info.
+ */
+static int nemR3WinInitCheckCapabilities(PVM pVM, PRTERRINFO pErrInfo)
+{
+#define NEM_LOG_REL_CAP_EX(a_szField, a_szFmt, a_Value)     LogRel(("NEM: %-38s= " a_szFmt "\n", a_szField, a_Value))
+#define NEM_LOG_REL_CAP_SUB_EX(a_szField, a_szFmt, a_Value) LogRel(("NEM:   %36s: " a_szFmt "\n", a_szField, a_Value))
+#define NEM_LOG_REL_CAP_SUB(a_szField, a_Value)             NEM_LOG_REL_CAP_SUB_EX(a_szField, "%d", a_Value)
+
+    /*
+     * Is the hypervisor present with the desired capability?
+     *
+     * In build 17083 this translates into:
+     *      - CPUID[0x00000001].HVP is set
+     *      - CPUID[0x40000000] == "Microsoft Hv"
+     *      - CPUID[0x40000001].eax == "Hv#1"
+     *      - CPUID[0x40000003].ebx[12] is set.
+     *      - VidGetExoPartitionProperty(INVALID_HANDLE_VALUE, 0x60000, &Ignored) returns
+     *        a non-zero value.
+     */
+    /**
+     * @todo Someone at Microsoft please explain weird API design:
+     *   1. Pointless CapabilityCode duplication int the output;
+     *   2. No output size.
+     */
+    WHV_CAPABILITY Caps;
+    RT_ZERO(Caps);
+    SetLastError(0);
+    HRESULT hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeHypervisorPresent, &Caps, sizeof(Caps));
+    DWORD   rcWin = GetLastError();
+    if (FAILED(hrc))
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                             "WHvGetCapability/WHvCapabilityCodeHypervisorPresent failed: %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+    if (!Caps.HypervisorPresent)
+    {
+        if (!RTPathExists(RTPATH_NT_PASSTHRU_PREFIX "Device\\VidExo"))
+            return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE,
+                                 "WHvCapabilityCodeHypervisorPresent is FALSE! Make sure you have enabled the 'Windows Hypervisor Platform' feature.");
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "WHvCapabilityCodeHypervisorPresent is FALSE! (%u)", rcWin);
+    }
+    LogRel(("NEM: WHvCapabilityCodeHypervisorPresent is TRUE, so this might work...\n"));
+
+
+    /*
+     * Check what extended VM exits are supported.
+     */
+    RT_ZERO(Caps);
+    hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeExtendedVmExits, &Caps, sizeof(Caps));
+    if (FAILED(hrc))
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                             "WHvGetCapability/WHvCapabilityCodeExtendedVmExits failed: %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+    NEM_LOG_REL_CAP_EX("WHvCapabilityCodeExtendedVmExits", "%'#018RX64", Caps.ExtendedVmExits.AsUINT64);
+    pVM->nem.s.fExtendedMsrExit   = RT_BOOL(Caps.ExtendedVmExits.X64MsrExit);
+    pVM->nem.s.fExtendedCpuIdExit = RT_BOOL(Caps.ExtendedVmExits.X64CpuidExit);
+    pVM->nem.s.fExtendedXcptExit  = RT_BOOL(Caps.ExtendedVmExits.ExceptionExit);
+    NEM_LOG_REL_CAP_SUB("fExtendedMsrExit",   pVM->nem.s.fExtendedMsrExit);
+    NEM_LOG_REL_CAP_SUB("fExtendedCpuIdExit", pVM->nem.s.fExtendedCpuIdExit);
+    NEM_LOG_REL_CAP_SUB("fExtendedXcptExit",  pVM->nem.s.fExtendedXcptExit);
+    if (Caps.ExtendedVmExits.AsUINT64 & ~(uint64_t)7)
+        LogRel(("NEM: Warning! Unknown VM exit definitions: %#RX64\n", Caps.ExtendedVmExits.AsUINT64));
+    /** @todo RECHECK: WHV_EXTENDED_VM_EXITS typedef. */
+
+    /*
+     * Check features in case they end up defining any.
+     */
+    RT_ZERO(Caps);
+    hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeFeatures, &Caps, sizeof(Caps));
+    if (FAILED(hrc))
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                             "WHvGetCapability/WHvCapabilityCodeFeatures failed: %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+    if (Caps.Features.AsUINT64 & ~(uint64_t)0)
+        LogRel(("NEM: Warning! Unknown feature definitions: %#RX64\n", Caps.Features.AsUINT64));
+    /** @todo RECHECK: WHV_CAPABILITY_FEATURES typedef. */
+
+    /*
+     * Check supported exception exit bitmap bits.
+     * We don't currently require this, so we just log failure.
+     */
+    RT_ZERO(Caps);
+    hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeExceptionExitBitmap, &Caps, sizeof(Caps));
+    if (SUCCEEDED(hrc))
+        LogRel(("NEM: Supported exception exit bitmap: %#RX64\n", Caps.ExceptionExitBitmap));
+    else
+        LogRel(("NEM: Warning! WHvGetCapability/WHvCapabilityCodeExceptionExitBitmap failed: %Rhrc (Last=%#x/%u)",
+                hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+
+    /*
+     * Check that the CPU vendor is supported.
+     */
+    RT_ZERO(Caps);
+    hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeProcessorVendor, &Caps, sizeof(Caps));
+    if (FAILED(hrc))
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                             "WHvGetCapability/WHvCapabilityCodeProcessorVendor failed: %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+    switch (Caps.ProcessorVendor)
+    {
+        /** @todo RECHECK: WHV_PROCESSOR_VENDOR typedef. */
+        case WHvProcessorVendorIntel:
+            NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorVendor", "%d - Intel", Caps.ProcessorVendor);
+            pVM->nem.s.enmCpuVendor = CPUMCPUVENDOR_INTEL;
+            break;
+        case WHvProcessorVendorAmd:
+            NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorVendor", "%d - AMD", Caps.ProcessorVendor);
+            pVM->nem.s.enmCpuVendor = CPUMCPUVENDOR_AMD;
+            break;
+        default:
+            NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorVendor", "%d", Caps.ProcessorVendor);
+            return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Unknown processor vendor: %d", Caps.ProcessorVendor);
+    }
+
+    /*
+     * CPU features, guessing these are virtual CPU features?
+     */
+    RT_ZERO(Caps);
+    hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeProcessorFeatures, &Caps, sizeof(Caps));
+    if (FAILED(hrc))
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                             "WHvGetCapability/WHvCapabilityCodeProcessorFeatures failed: %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+    NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorFeatures", "%'#018RX64", Caps.ProcessorFeatures.AsUINT64);
+#define NEM_LOG_REL_CPU_FEATURE(a_Field)    NEM_LOG_REL_CAP_SUB(#a_Field, Caps.ProcessorFeatures.a_Field)
+    NEM_LOG_REL_CPU_FEATURE(Sse3Support);
+    NEM_LOG_REL_CPU_FEATURE(LahfSahfSupport);
+    NEM_LOG_REL_CPU_FEATURE(Ssse3Support);
+    NEM_LOG_REL_CPU_FEATURE(Sse4_1Support);
+    NEM_LOG_REL_CPU_FEATURE(Sse4_2Support);
+    NEM_LOG_REL_CPU_FEATURE(Sse4aSupport);
+    NEM_LOG_REL_CPU_FEATURE(XopSupport);
+    NEM_LOG_REL_CPU_FEATURE(PopCntSupport);
+    NEM_LOG_REL_CPU_FEATURE(Cmpxchg16bSupport);
+    NEM_LOG_REL_CPU_FEATURE(Altmovcr8Support);
+    NEM_LOG_REL_CPU_FEATURE(LzcntSupport);
+    NEM_LOG_REL_CPU_FEATURE(MisAlignSseSupport);
+    NEM_LOG_REL_CPU_FEATURE(MmxExtSupport);
+    NEM_LOG_REL_CPU_FEATURE(Amd3DNowSupport);
+    NEM_LOG_REL_CPU_FEATURE(ExtendedAmd3DNowSupport);
+    NEM_LOG_REL_CPU_FEATURE(Page1GbSupport);
+    NEM_LOG_REL_CPU_FEATURE(AesSupport);
+    NEM_LOG_REL_CPU_FEATURE(PclmulqdqSupport);
+    NEM_LOG_REL_CPU_FEATURE(PcidSupport);
+    NEM_LOG_REL_CPU_FEATURE(Fma4Support);
+    NEM_LOG_REL_CPU_FEATURE(F16CSupport);
+    NEM_LOG_REL_CPU_FEATURE(RdRandSupport);
+    NEM_LOG_REL_CPU_FEATURE(RdWrFsGsSupport);
+    NEM_LOG_REL_CPU_FEATURE(SmepSupport);
+    NEM_LOG_REL_CPU_FEATURE(EnhancedFastStringSupport);
+    NEM_LOG_REL_CPU_FEATURE(Bmi1Support);
+    NEM_LOG_REL_CPU_FEATURE(Bmi2Support);
+    /* two reserved bits here, see below */
+    NEM_LOG_REL_CPU_FEATURE(MovbeSupport);
+    NEM_LOG_REL_CPU_FEATURE(Npiep1Support);
+    NEM_LOG_REL_CPU_FEATURE(DepX87FPUSaveSupport);
+    NEM_LOG_REL_CPU_FEATURE(RdSeedSupport);
+    NEM_LOG_REL_CPU_FEATURE(AdxSupport);
+    NEM_LOG_REL_CPU_FEATURE(IntelPrefetchSupport);
+    NEM_LOG_REL_CPU_FEATURE(SmapSupport);
+    NEM_LOG_REL_CPU_FEATURE(HleSupport);
+    NEM_LOG_REL_CPU_FEATURE(RtmSupport);
+    NEM_LOG_REL_CPU_FEATURE(RdtscpSupport);
+    NEM_LOG_REL_CPU_FEATURE(ClflushoptSupport);
+    NEM_LOG_REL_CPU_FEATURE(ClwbSupport);
+    NEM_LOG_REL_CPU_FEATURE(ShaSupport);
+    NEM_LOG_REL_CPU_FEATURE(X87PointersSavedSupport);
+#undef NEM_LOG_REL_CPU_FEATURE
+    if (Caps.ProcessorFeatures.AsUINT64 & (~(RT_BIT_64(43) - 1) | RT_BIT_64(27) | RT_BIT_64(28)))
+        LogRel(("NEM: Warning! Unknown CPU features: %#RX64\n", Caps.ProcessorFeatures.AsUINT64));
+    pVM->nem.s.uCpuFeatures.u64 = Caps.ProcessorFeatures.AsUINT64;
+    /** @todo RECHECK: WHV_PROCESSOR_FEATURES typedef. */
+
+    /*
+     * The cache line flush size.
+     */
+    RT_ZERO(Caps);
+    hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeProcessorClFlushSize, &Caps, sizeof(Caps));
+    if (FAILED(hrc))
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                             "WHvGetCapability/WHvCapabilityCodeProcessorClFlushSize failed: %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+    NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorClFlushSize", "2^%u", Caps.ProcessorClFlushSize);
+    if (Caps.ProcessorClFlushSize < 8 && Caps.ProcessorClFlushSize > 9)
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Unsupported cache line flush size: %u", Caps.ProcessorClFlushSize);
+    pVM->nem.s.cCacheLineFlushShift = Caps.ProcessorClFlushSize;
+
+    /*
+     * See if they've added more properties that we're not aware of.
+     */
+    /** @todo RECHECK: WHV_CAPABILITY_CODE typedef. */
+    if (!IsDebuggerPresent()) /* Too noisy when in debugger, so skip. */
+    {
+        static const struct
+        {
+            uint32_t iMin, iMax; } s_aUnknowns[] =
+        {
+            { 0x0004, 0x000f },
+            { 0x1003, 0x100f },
+            { 0x2000, 0x200f },
+            { 0x3000, 0x300f },
+            { 0x4000, 0x400f },
+        };
+        for (uint32_t j = 0; j < RT_ELEMENTS(s_aUnknowns); j++)
+            for (uint32_t i = s_aUnknowns[j].iMin; i <= s_aUnknowns[j].iMax; i++)
+            {
+                RT_ZERO(Caps);
+                hrc = WHvGetCapabilityWrapper((WHV_CAPABILITY_CODE)i, &Caps, sizeof(Caps));
+                if (SUCCEEDED(hrc))
+                    LogRel(("NEM: Warning! Unknown capability %#x returning: %.*Rhxs\n", i, sizeof(Caps), &Caps));
+            }
+    }
+
+    /*
+     * For proper operation, we require CPUID exits.
+     */
+    if (!pVM->nem.s.fExtendedCpuIdExit)
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Missing required extended CPUID exit support");
+    if (!pVM->nem.s.fExtendedMsrExit)
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Missing required extended MSR exit support");
+    if (!pVM->nem.s.fExtendedXcptExit)
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Missing required extended exception exit support");
+
+#undef NEM_LOG_REL_CAP_EX
+#undef NEM_LOG_REL_CAP_SUB_EX
+#undef NEM_LOG_REL_CAP_SUB
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Used to fill in g_IoCtlGetHvPartitionId.
+ */
+static NTSTATUS WINAPI
+nemR3WinIoctlDetector_GetHvPartitionId(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
+                                       PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
+                                       PVOID pvOutput, ULONG cbOutput)
+{
+    AssertLogRelMsgReturn(hFile == NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, ("hFile=%p\n", hFile), STATUS_INVALID_PARAMETER_1);
+    RT_NOREF(hEvt); RT_NOREF(pfnApcCallback); RT_NOREF(pvApcCtx);
+    AssertLogRelMsgReturn(RT_VALID_PTR(pIos), ("pIos=%p\n", pIos), STATUS_INVALID_PARAMETER_5);
+    AssertLogRelMsgReturn(cbInput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_8);
+    RT_NOREF(pvInput);
+
+    AssertLogRelMsgReturn(RT_VALID_PTR(pvOutput), ("pvOutput=%p\n", pvOutput), STATUS_INVALID_PARAMETER_9);
+    AssertLogRelMsgReturn(cbOutput == sizeof(HV_PARTITION_ID), ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
+    *(HV_PARTITION_ID *)pvOutput = NEM_WIN_IOCTL_DETECTOR_FAKE_PARTITION_ID;
+
+    g_IoCtlGetHvPartitionId.cbInput   = cbInput;
+    g_IoCtlGetHvPartitionId.cbOutput  = cbOutput;
+    g_IoCtlGetHvPartitionId.uFunction = uFunction;
+
+    return STATUS_SUCCESS;
+}
+
+
+/**
+ * Used to fill in g_IoCtlStartVirtualProcessor.
+ */
+static NTSTATUS WINAPI
+nemR3WinIoctlDetector_StartVirtualProcessor(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
+                                            PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
+                                            PVOID pvOutput, ULONG cbOutput)
+{
+    AssertLogRelMsgReturn(hFile == NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, ("hFile=%p\n", hFile), STATUS_INVALID_PARAMETER_1);
+    RT_NOREF(hEvt); RT_NOREF(pfnApcCallback); RT_NOREF(pvApcCtx);
+    AssertLogRelMsgReturn(RT_VALID_PTR(pIos), ("pIos=%p\n", pIos), STATUS_INVALID_PARAMETER_5);
+    AssertLogRelMsgReturn(cbInput == sizeof(HV_VP_INDEX), ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_8);
+    AssertLogRelMsgReturn(RT_VALID_PTR(pvInput), ("pvInput=%p\n", pvInput), STATUS_INVALID_PARAMETER_9);
+    AssertLogRelMsgReturn(*(HV_VP_INDEX *)pvInput == NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX,
+                          ("*piCpu=%u\n", *(HV_VP_INDEX *)pvInput), STATUS_INVALID_PARAMETER_9);
+    AssertLogRelMsgReturn(cbOutput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
+    RT_NOREF(pvOutput);
+
+    g_IoCtlStartVirtualProcessor.cbInput   = cbInput;
+    g_IoCtlStartVirtualProcessor.cbOutput  = cbOutput;
+    g_IoCtlStartVirtualProcessor.uFunction = uFunction;
+
+    return STATUS_SUCCESS;
+}
+
+
+/**
+ * Used to fill in g_IoCtlStartVirtualProcessor.
+ */
+static NTSTATUS WINAPI
+nemR3WinIoctlDetector_StopVirtualProcessor(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
+                                           PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
+                                           PVOID pvOutput, ULONG cbOutput)
+{
+    AssertLogRelMsgReturn(hFile == NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, ("hFile=%p\n", hFile), STATUS_INVALID_PARAMETER_1);
+    RT_NOREF(hEvt); RT_NOREF(pfnApcCallback); RT_NOREF(pvApcCtx);
+    AssertLogRelMsgReturn(RT_VALID_PTR(pIos), ("pIos=%p\n", pIos), STATUS_INVALID_PARAMETER_5);
+    AssertLogRelMsgReturn(cbInput == sizeof(HV_VP_INDEX), ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_8);
+    AssertLogRelMsgReturn(RT_VALID_PTR(pvInput), ("pvInput=%p\n", pvInput), STATUS_INVALID_PARAMETER_9);
+    AssertLogRelMsgReturn(*(HV_VP_INDEX *)pvInput == NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX,
+                          ("*piCpu=%u\n", *(HV_VP_INDEX *)pvInput), STATUS_INVALID_PARAMETER_9);
+    AssertLogRelMsgReturn(cbOutput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
+    RT_NOREF(pvOutput);
+
+    g_IoCtlStopVirtualProcessor.cbInput   = cbInput;
+    g_IoCtlStopVirtualProcessor.cbOutput  = cbOutput;
+    g_IoCtlStopVirtualProcessor.uFunction = uFunction;
+
+    return STATUS_SUCCESS;
+}
+
+
+/**
+ * Used to fill in g_IoCtlMessageSlotHandleAndGetNext
+ */
+static NTSTATUS WINAPI
+nemR3WinIoctlDetector_MessageSlotHandleAndGetNext(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
+                                                  PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
+                                                  PVOID pvOutput, ULONG cbOutput)
+{
+    AssertLogRelMsgReturn(hFile == NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, ("hFile=%p\n", hFile), STATUS_INVALID_PARAMETER_1);
+    RT_NOREF(hEvt); RT_NOREF(pfnApcCallback); RT_NOREF(pvApcCtx);
+    AssertLogRelMsgReturn(RT_VALID_PTR(pIos), ("pIos=%p\n", pIos), STATUS_INVALID_PARAMETER_5);
+
+    if (g_uBuildNo >= 17758)
+    {
+        /* No timeout since about build 17758, it's now always an infinite wait.  So, a somewhat compatible change.  */
+        AssertLogRelMsgReturn(cbInput == RT_UOFFSETOF(VID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT, cMillies),
+                              ("cbInput=%#x\n", cbInput),
+                              STATUS_INVALID_PARAMETER_8);
+        AssertLogRelMsgReturn(RT_VALID_PTR(pvInput), ("pvInput=%p\n", pvInput), STATUS_INVALID_PARAMETER_9);
+        PCVID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT pVidIn = (PCVID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT)pvInput;
+        AssertLogRelMsgReturn(   pVidIn->iCpu == NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX
+                              && pVidIn->fFlags == VID_MSHAGN_F_HANDLE_MESSAGE,
+                              ("iCpu=%u fFlags=%#x cMillies=%#x\n", pVidIn->iCpu, pVidIn->fFlags, pVidIn->cMillies),
+                              STATUS_INVALID_PARAMETER_9);
+        AssertLogRelMsgReturn(cbOutput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
+    }
+    else
+    {
+        AssertLogRelMsgReturn(cbInput == sizeof(VID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT), ("cbInput=%#x\n", cbInput),
+                              STATUS_INVALID_PARAMETER_8);
+        AssertLogRelMsgReturn(RT_VALID_PTR(pvInput), ("pvInput=%p\n", pvInput), STATUS_INVALID_PARAMETER_9);
+        PCVID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT pVidIn = (PCVID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT)pvInput;
+        AssertLogRelMsgReturn(   pVidIn->iCpu == NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX
+                              && pVidIn->fFlags == VID_MSHAGN_F_HANDLE_MESSAGE
+                              && pVidIn->cMillies == NEM_WIN_IOCTL_DETECTOR_FAKE_TIMEOUT,
+                              ("iCpu=%u fFlags=%#x cMillies=%#x\n", pVidIn->iCpu, pVidIn->fFlags, pVidIn->cMillies),
+                              STATUS_INVALID_PARAMETER_9);
+        AssertLogRelMsgReturn(cbOutput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
+        RT_NOREF(pvOutput);
+    }
+
+    g_IoCtlMessageSlotHandleAndGetNext.cbInput   = cbInput;
+    g_IoCtlMessageSlotHandleAndGetNext.cbOutput  = cbOutput;
+    g_IoCtlMessageSlotHandleAndGetNext.uFunction = uFunction;
+
+    return STATUS_SUCCESS;
+}
+
+
+#ifdef LOG_ENABLED
+/**
+ * Used to fill in what g_pIoCtlDetectForLogging points to.
+ */
+static NTSTATUS WINAPI nemR3WinIoctlDetector_ForLogging(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
+                                                        PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
+                                                        PVOID pvOutput, ULONG cbOutput)
+{
+    RT_NOREF(hFile, hEvt, pfnApcCallback, pvApcCtx, pIos, pvInput, pvOutput);
+
+    g_pIoCtlDetectForLogging->cbInput   = cbInput;
+    g_pIoCtlDetectForLogging->cbOutput  = cbOutput;
+    g_pIoCtlDetectForLogging->uFunction = uFunction;
+
+    return STATUS_SUCCESS;
+}
+#endif
+
+
+/**
+ * Worker for nemR3NativeInit that detect I/O control function numbers for VID.
+ *
+ * We use the function numbers directly in ring-0 and to name functions when
+ * logging NtDeviceIoControlFile calls.
+ *
+ * @note    We could alternatively do this by disassembling the respective
+ *          functions, but hooking NtDeviceIoControlFile and making fake calls
+ *          more easily provides the desired information.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.  Will set I/O
+ *                              control info members.
+ * @param   pErrInfo            Where to always return error info.
+ */
+static int nemR3WinInitDiscoverIoControlProperties(PVM pVM, PRTERRINFO pErrInfo)
+{
+    /*
+     * Probe the I/O control information for select VID APIs so we can use
+     * them directly from ring-0 and better log them.
+     *
+     */
+    decltype(NtDeviceIoControlFile) * const pfnOrg = *g_ppfnVidNtDeviceIoControlFile;
+
+    /* VidGetHvPartitionId - must work due to memory. */
+    *g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_GetHvPartitionId;
+    HV_PARTITION_ID idHvPartition = HV_PARTITION_ID_INVALID;
+    BOOL fRet = g_pfnVidGetHvPartitionId(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, &idHvPartition);
+    *g_ppfnVidNtDeviceIoControlFile = pfnOrg;
+    AssertReturn(fRet && idHvPartition == NEM_WIN_IOCTL_DETECTOR_FAKE_PARTITION_ID && g_IoCtlGetHvPartitionId.uFunction != 0,
+                 RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
+                               "Problem figuring out VidGetHvPartitionId: fRet=%u idHvPartition=%#x dwErr=%u",
+                               fRet, idHvPartition, GetLastError()) );
+    LogRel(("NEM: VidGetHvPartitionId            -> fun:%#x in:%#x out:%#x\n",
+            g_IoCtlGetHvPartitionId.uFunction, g_IoCtlGetHvPartitionId.cbInput, g_IoCtlGetHvPartitionId.cbOutput));
+
+    int rcRet = VINF_SUCCESS;
+    /* VidStartVirtualProcessor */
+    *g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_StartVirtualProcessor;
+    fRet = g_pfnVidStartVirtualProcessor(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX);
+    *g_ppfnVidNtDeviceIoControlFile = pfnOrg;
+    AssertStmt(fRet && g_IoCtlStartVirtualProcessor.uFunction != 0,
+               rcRet = RTERRINFO_LOG_REL_SET_F(pErrInfo, VERR_NEM_RING3_ONLY,
+                                               "Problem figuring out VidStartVirtualProcessor: fRet=%u dwErr=%u",
+                                               fRet, GetLastError()) );
+    LogRel(("NEM: VidStartVirtualProcessor       -> fun:%#x in:%#x out:%#x\n", g_IoCtlStartVirtualProcessor.uFunction,
+            g_IoCtlStartVirtualProcessor.cbInput, g_IoCtlStartVirtualProcessor.cbOutput));
+
+    /* VidStopVirtualProcessor */
+    *g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_StopVirtualProcessor;
+    fRet = g_pfnVidStopVirtualProcessor(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX);
+    *g_ppfnVidNtDeviceIoControlFile = pfnOrg;
+    AssertStmt(fRet && g_IoCtlStopVirtualProcessor.uFunction != 0,
+               rcRet = RTERRINFO_LOG_REL_SET_F(pErrInfo, VERR_NEM_RING3_ONLY,
+                                               "Problem figuring out VidStopVirtualProcessor: fRet=%u dwErr=%u",
+                                               fRet, GetLastError()) );
+    LogRel(("NEM: VidStopVirtualProcessor        -> fun:%#x in:%#x out:%#x\n", g_IoCtlStopVirtualProcessor.uFunction,
+            g_IoCtlStopVirtualProcessor.cbInput, g_IoCtlStopVirtualProcessor.cbOutput));
+
+    /* VidMessageSlotHandleAndGetNext */
+    *g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_MessageSlotHandleAndGetNext;
+    fRet = g_pfnVidMessageSlotHandleAndGetNext(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE,
+                                               NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX, VID_MSHAGN_F_HANDLE_MESSAGE,
+                                               NEM_WIN_IOCTL_DETECTOR_FAKE_TIMEOUT);
+    *g_ppfnVidNtDeviceIoControlFile = pfnOrg;
+    AssertStmt(fRet && g_IoCtlMessageSlotHandleAndGetNext.uFunction != 0,
+               rcRet = RTERRINFO_LOG_REL_SET_F(pErrInfo, VERR_NEM_RING3_ONLY,
+                                               "Problem figuring out VidMessageSlotHandleAndGetNext: fRet=%u dwErr=%u",
+                                               fRet, GetLastError()) );
+    LogRel(("NEM: VidMessageSlotHandleAndGetNext -> fun:%#x in:%#x out:%#x\n",
+            g_IoCtlMessageSlotHandleAndGetNext.uFunction, g_IoCtlMessageSlotHandleAndGetNext.cbInput,
+            g_IoCtlMessageSlotHandleAndGetNext.cbOutput));
+
+#ifdef LOG_ENABLED
+    /* The following are only for logging: */
+    union
+    {
+        VID_MAPPED_MESSAGE_SLOT MapSlot;
+        HV_REGISTER_NAME        Name;
+        HV_REGISTER_VALUE       Value;
+    } uBuf;
+
+    /* VidMessageSlotMap */
+    g_pIoCtlDetectForLogging = &g_IoCtlMessageSlotMap;
+    *g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_ForLogging;
+    fRet = g_pfnVidMessageSlotMap(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, &uBuf.MapSlot, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX);
+    *g_ppfnVidNtDeviceIoControlFile = pfnOrg;
+    Assert(fRet);
+    LogRel(("NEM: VidMessageSlotMap              -> fun:%#x in:%#x out:%#x\n", g_pIoCtlDetectForLogging->uFunction,
+            g_pIoCtlDetectForLogging->cbInput, g_pIoCtlDetectForLogging->cbOutput));
+
+    /* VidGetVirtualProcessorState */
+    uBuf.Name = HvRegisterExplicitSuspend;
+    g_pIoCtlDetectForLogging = &g_IoCtlGetVirtualProcessorState;
+    *g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_ForLogging;
+    fRet = g_pfnVidGetVirtualProcessorState(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX,
+                                            &uBuf.Name, 1, &uBuf.Value);
+    *g_ppfnVidNtDeviceIoControlFile = pfnOrg;
+    Assert(fRet);
+    LogRel(("NEM: VidGetVirtualProcessorState    -> fun:%#x in:%#x out:%#x\n", g_pIoCtlDetectForLogging->uFunction,
+            g_pIoCtlDetectForLogging->cbInput, g_pIoCtlDetectForLogging->cbOutput));
+
+    /* VidSetVirtualProcessorState */
+    uBuf.Name = HvRegisterExplicitSuspend;
+    g_pIoCtlDetectForLogging = &g_IoCtlSetVirtualProcessorState;
+    *g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_ForLogging;
+    fRet = g_pfnVidSetVirtualProcessorState(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX,
+                                            &uBuf.Name, 1, &uBuf.Value);
+    *g_ppfnVidNtDeviceIoControlFile = pfnOrg;
+    Assert(fRet);
+    LogRel(("NEM: VidSetVirtualProcessorState    -> fun:%#x in:%#x out:%#x\n", g_pIoCtlDetectForLogging->uFunction,
+            g_pIoCtlDetectForLogging->cbInput, g_pIoCtlDetectForLogging->cbOutput));
+
+    g_pIoCtlDetectForLogging = NULL;
+#endif
+
+    /* Done. */
+    pVM->nem.s.IoCtlGetHvPartitionId            = g_IoCtlGetHvPartitionId;
+    pVM->nem.s.IoCtlStartVirtualProcessor       = g_IoCtlStartVirtualProcessor;
+    pVM->nem.s.IoCtlStopVirtualProcessor        = g_IoCtlStopVirtualProcessor;
+    pVM->nem.s.IoCtlMessageSlotHandleAndGetNext = g_IoCtlMessageSlotHandleAndGetNext;
+    return rcRet;
+}
+
+
+/**
+ * Creates and sets up a Hyper-V (exo) partition.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pErrInfo            Where to always return error info.
+ */
+static int nemR3WinInitCreatePartition(PVM pVM, PRTERRINFO pErrInfo)
+{
+    AssertReturn(!pVM->nem.s.hPartition,       RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
+    AssertReturn(!pVM->nem.s.hPartitionDevice, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
+
+    /*
+     * Create the partition.
+     */
+    WHV_PARTITION_HANDLE hPartition;
+    HRESULT hrc = WHvCreatePartition(&hPartition);
+    if (FAILED(hrc))
+        return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "WHvCreatePartition failed with %Rhrc (Last=%#x/%u)",
+                             hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+
+    int rc;
+
+    /*
+     * Set partition properties, most importantly the CPU count.
+     */
+    /**
+     * @todo Someone at Microsoft please explain another weird API:
+     *  - Why this API doesn't take the WHV_PARTITION_PROPERTY_CODE value as an
+     *    argument rather than as part of the struct.  That is so weird if you've
+     *    used any other NT or windows API,  including WHvGetCapability().
+     *  - Why use PVOID when WHV_PARTITION_PROPERTY is what's expected.  We
+     *    technically only need 9 bytes for setting/getting
+     *    WHVPartitionPropertyCodeProcessorClFlushSize, but the API insists on 16. */
+    WHV_PARTITION_PROPERTY Property;
+    RT_ZERO(Property);
+    Property.ProcessorCount = pVM->cCpus;
+    hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeProcessorCount, &Property, sizeof(Property));
+    if (SUCCEEDED(hrc))
+    {
+        RT_ZERO(Property);
+        Property.ExtendedVmExits.X64CpuidExit  = pVM->nem.s.fExtendedCpuIdExit; /** @todo Register fixed results and restrict cpuid exits */
+        Property.ExtendedVmExits.X64MsrExit    = pVM->nem.s.fExtendedMsrExit;
+        Property.ExtendedVmExits.ExceptionExit = pVM->nem.s.fExtendedXcptExit;
+        hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeExtendedVmExits, &Property, sizeof(Property));
+        if (SUCCEEDED(hrc))
+        {
+            /*
+             * We'll continue setup in nemR3NativeInitAfterCPUM.
+             */
+            pVM->nem.s.fCreatedEmts     = false;
+            pVM->nem.s.hPartition       = hPartition;
+            LogRel(("NEM: Created partition %p.\n", hPartition));
+            return VINF_SUCCESS;
+        }
+
+        rc = RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED,
+                           "Failed setting WHvPartitionPropertyCodeExtendedVmExits to %'#RX64: %Rhrc",
+                           Property.ExtendedVmExits.AsUINT64, hrc);
+    }
+    else
+        rc = RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED,
+                           "Failed setting WHvPartitionPropertyCodeProcessorCount to %u: %Rhrc (Last=%#x/%u)",
+                           pVM->cCpus, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+    WHvDeletePartition(hPartition);
+
+    Assert(!pVM->nem.s.hPartitionDevice);
+    Assert(!pVM->nem.s.hPartition);
+    return rc;
+}
+
+
+/**
+ * Makes sure APIC and firmware will not allow X2APIC mode.
+ *
+ * This is rather ugly.
+ *
+ * @returns VBox status code
+ * @param   pVM             The cross context VM structure.
+ */
+static int nemR3WinDisableX2Apic(PVM pVM)
+{
+    /*
+     * First make sure the 'Mode' config value of the APIC isn't set to X2APIC.
+     * This defaults to APIC, so no need to change unless it's X2APIC.
+     */
+    PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/Devices/apic/0/Config");
+    if (pCfg)
+    {
+        uint8_t bMode = 0;
+        int rc = CFGMR3QueryU8(pCfg, "Mode", &bMode);
+        AssertLogRelMsgReturn(RT_SUCCESS(rc) || rc == VERR_CFGM_VALUE_NOT_FOUND, ("%Rrc\n", rc), rc);
+        if (RT_SUCCESS(rc) && bMode == PDMAPICMODE_X2APIC)
+        {
+            LogRel(("NEM: Adjusting APIC configuration from X2APIC to APIC max mode.  X2APIC is not supported by the WinHvPlatform API!\n"));
+            LogRel(("NEM: Disable Hyper-V if you need X2APIC for your guests!\n"));
+            rc = CFGMR3RemoveValue(pCfg, "Mode");
+            rc = CFGMR3InsertInteger(pCfg, "Mode", PDMAPICMODE_APIC);
+            AssertLogRelRCReturn(rc, rc);
+        }
+    }
+
+    /*
+     * Now the firmwares.
+     * These also defaults to APIC and only needs adjusting if configured to X2APIC (2).
+     */
+    static const char * const s_apszFirmwareConfigs[] =
+    {
+        "/Devices/efi/0/Config",
+        "/Devices/pcbios/0/Config",
+    };
+    for (unsigned i = 0; i < RT_ELEMENTS(s_apszFirmwareConfigs); i++)
+    {
+        pCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/Devices/APIC/0/Config");
+        if (pCfg)
+        {
+            uint8_t bMode = 0;
+            int rc = CFGMR3QueryU8(pCfg, "APIC", &bMode);
+            AssertLogRelMsgReturn(RT_SUCCESS(rc) || rc == VERR_CFGM_VALUE_NOT_FOUND, ("%Rrc\n", rc), rc);
+            if (RT_SUCCESS(rc) && bMode == 2)
+            {
+                LogRel(("NEM: Adjusting %s/Mode from 2 (X2APIC) to 1 (APIC).\n", s_apszFirmwareConfigs[i]));
+                rc = CFGMR3RemoveValue(pCfg, "APIC");
+                rc = CFGMR3InsertInteger(pCfg, "APIC", 1);
+                AssertLogRelRCReturn(rc, rc);
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Try initialize the native API.
+ *
+ * This may only do part of the job, more can be done in
+ * nemR3NativeInitAfterCPUM() and nemR3NativeInitCompleted().
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   fFallback       Whether we're in fallback mode or use-NEM mode. In
+ *                          the latter we'll fail if we cannot initialize.
+ * @param   fForced         Whether the HMForced flag is set and we should
+ *                          fail if we cannot initialize.
+ */
+int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
+{
+    g_uBuildNo = RTSystemGetNtBuildNo();
+
+    /*
+     * Some state init.
+     */
+    pVM->nem.s.fA20Enabled = true;
+#if 0
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
+    }
+#endif
+
+    /*
+     * Error state.
+     * The error message will be non-empty on failure and 'rc' will be set too.
+     */
+    RTERRINFOSTATIC ErrInfo;
+    PRTERRINFO pErrInfo = RTErrInfoInitStatic(&ErrInfo);
+    int rc = nemR3WinInitProbeAndLoad(fForced, pErrInfo);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Check the capabilties of the hypervisor, starting with whether it's present.
+         */
+        rc = nemR3WinInitCheckCapabilities(pVM, pErrInfo);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Discover the VID I/O control function numbers we need.
+             */
+            rc = nemR3WinInitDiscoverIoControlProperties(pVM, pErrInfo);
+            if (rc == VERR_NEM_RING3_ONLY)
+            {
+                if (pVM->nem.s.fUseRing0Runloop)
+                {
+                    LogRel(("NEM: Disabling UseRing0Runloop.\n"));
+                    pVM->nem.s.fUseRing0Runloop = false;
+                }
+                rc = VINF_SUCCESS;
+            }
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Check out our ring-0 capabilities.
+                 */
+                rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), 0 /*idCpu*/, VMMR0_DO_NEM_INIT_VM, 0, NULL);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Create and initialize a partition.
+                     */
+                    rc = nemR3WinInitCreatePartition(pVM, pErrInfo);
+                    if (RT_SUCCESS(rc))
+                    {
+                        VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_NATIVE_API);
+                        Log(("NEM: Marked active!\n"));
+                        nemR3WinDisableX2Apic(pVM);
+
+                        /* Register release statistics */
+                        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                        {
+                            PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitPortIo,          STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of port I/O exits",               "/NEM/CPU%u/ExitPortIo", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitMemUnmapped,     STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of unmapped memory exits",        "/NEM/CPU%u/ExitMemUnmapped", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitMemIntercept,    STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of intercepted memory exits",     "/NEM/CPU%u/ExitMemIntercept", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitHalt,            STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of HLT exits",                    "/NEM/CPU%u/ExitHalt", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitInterruptWindow, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of HLT exits",                    "/NEM/CPU%u/ExitInterruptWindow", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitCpuId,           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of CPUID exits",                  "/NEM/CPU%u/ExitCpuId", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitMsr,             STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of MSR access exits",             "/NEM/CPU%u/ExitMsr", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitException,       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of exception exits",              "/NEM/CPU%u/ExitException", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionBp,     STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #BP exits",                    "/NEM/CPU%u/ExitExceptionBp", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionDb,     STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #DB exits",                    "/NEM/CPU%u/ExitExceptionDb", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionGp,     STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #GP exits",                    "/NEM/CPU%u/ExitExceptionGp", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionGpMesa, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #GP exits from mesa driver",   "/NEM/CPU%u/ExitExceptionGpMesa", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionUd,     STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #UD exits",                    "/NEM/CPU%u/ExitExceptionUd", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionUdHandled, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of handled #UD exits",         "/NEM/CPU%u/ExitExceptionUdHandled", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatExitUnrecoverable,   STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of unrecoverable exits",          "/NEM/CPU%u/ExitUnrecoverable", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatGetMsgTimeout,       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of get message timeouts/alerts",  "/NEM/CPU%u/GetMsgTimeout", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuSuccess,      STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of successful CPU stops",         "/NEM/CPU%u/StopCpuSuccess", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuPending,      STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of pending CPU stops",            "/NEM/CPU%u/StopCpuPending", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuPendingAlerts,STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of pending CPU stop alerts",      "/NEM/CPU%u/StopCpuPendingAlerts", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuPendingOdd,   STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of odd pending CPU stops (see code)", "/NEM/CPU%u/StopCpuPendingOdd", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatCancelChangedState,  STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of cancel changed state",         "/NEM/CPU%u/CancelChangedState", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatCancelAlertedThread, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of cancel alerted EMT",           "/NEM/CPU%u/CancelAlertedEMT", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnFFPre,        STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of pre execution FF breaks",      "/NEM/CPU%u/BreakOnFFPre", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnFFPost,       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of post execution FF breaks",     "/NEM/CPU%u/BreakOnFFPost", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnCancel,       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of cancel execution breaks",      "/NEM/CPU%u/BreakOnCancel", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnStatus,       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of status code breaks",           "/NEM/CPU%u/BreakOnStatus", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatImportOnDemand,      STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of on-demand state imports",      "/NEM/CPU%u/ImportOnDemand", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturn,      STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of state imports on loop return", "/NEM/CPU%u/ImportOnReturn", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturnSkipped, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of skipped state imports on loop return", "/NEM/CPU%u/ImportOnReturnSkipped", idCpu);
+                            STAMR3RegisterF(pVM, &pNemCpu->StatQueryCpuTick,        STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of TSC queries",                  "/NEM/CPU%u/QueryCpuTick", idCpu);
+                        }
+
+                        PUVM pUVM = pVM->pUVM;
+                        STAMR3RegisterRefresh(pUVM, &pVM->nem.s.R0Stats.cPagesAvailable, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,
+                                              STAMUNIT_PAGES, STAM_REFRESH_GRP_NEM, "Free pages available to the hypervisor",
+                                              "/NEM/R0Stats/cPagesAvailable");
+                        STAMR3RegisterRefresh(pUVM, &pVM->nem.s.R0Stats.cPagesInUse,     STAMTYPE_U64, STAMVISIBILITY_ALWAYS,
+                                              STAMUNIT_PAGES, STAM_REFRESH_GRP_NEM, "Pages in use by hypervisor",
+                                              "/NEM/R0Stats/cPagesInUse");
+                    }
+                }
+                else
+                    rc = RTErrInfoSetF(pErrInfo, rc, "VMMR0_DO_NEM_INIT_VM failed: %Rrc", rc);
+            }
+        }
+    }
+
+    /*
+     * We only fail if in forced mode, otherwise just log the complaint and return.
+     */
+    Assert(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API || RTErrInfoIsSet(pErrInfo));
+    if (   (fForced || !fFallback)
+        && pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API)
+        return VMSetError(pVM, RT_SUCCESS_NP(rc) ? VERR_NEM_NOT_AVAILABLE : rc, RT_SRC_POS, "%s", pErrInfo->pszMsg);
+
+    if (RTErrInfoIsSet(pErrInfo))
+        LogRel(("NEM: Not available: %s\n", pErrInfo->pszMsg));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * This is called after CPUMR3Init is done.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle..
+ */
+int nemR3NativeInitAfterCPUM(PVM pVM)
+{
+    /*
+     * Validate sanity.
+     */
+    WHV_PARTITION_HANDLE hPartition = pVM->nem.s.hPartition;
+    AssertReturn(hPartition != NULL, VERR_WRONG_ORDER);
+    AssertReturn(!pVM->nem.s.hPartitionDevice, VERR_WRONG_ORDER);
+    AssertReturn(!pVM->nem.s.fCreatedEmts, VERR_WRONG_ORDER);
+    AssertReturn(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API, VERR_WRONG_ORDER);
+
+    /*
+     * Continue setting up the partition now that we've got most of the CPUID feature stuff.
+     */
+    WHV_PARTITION_PROPERTY Property;
+    HRESULT                hrc;
+
+#if 0
+    /* Not sure if we really need to set the vendor.
+       Update: Apparently we don't. WHvPartitionPropertyCodeProcessorVendor was removed in 17110. */
+    RT_ZERO(Property);
+    Property.ProcessorVendor = pVM->nem.s.enmCpuVendor == CPUMCPUVENDOR_AMD ? WHvProcessorVendorAmd
+                             : WHvProcessorVendorIntel;
+    hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeProcessorVendor, &Property, sizeof(Property));
+    if (FAILED(hrc))
+        return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
+                          "Failed to set WHvPartitionPropertyCodeProcessorVendor to %u: %Rhrc (Last=%#x/%u)",
+                          Property.ProcessorVendor, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+#endif
+
+    /* Not sure if we really need to set the cache line flush size. */
+    RT_ZERO(Property);
+    Property.ProcessorClFlushSize = pVM->nem.s.cCacheLineFlushShift;
+    hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeProcessorClFlushSize, &Property, sizeof(Property));
+    if (FAILED(hrc))
+        return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
+                          "Failed to set WHvPartitionPropertyCodeProcessorClFlushSize to %u: %Rhrc (Last=%#x/%u)",
+                          pVM->nem.s.cCacheLineFlushShift, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+
+    /* Intercept #DB, #BP and #UD exceptions. */
+    RT_ZERO(Property);
+    Property.ExceptionExitBitmap = RT_BIT_64(WHvX64ExceptionTypeDebugTrapOrFault)
+                                 | RT_BIT_64(WHvX64ExceptionTypeBreakpointTrap)
+                                 | RT_BIT_64(WHvX64ExceptionTypeInvalidOpcodeFault);
+
+    /* Intercept #GP to workaround the buggy mesa vmwgfx driver. */
+    PVMCPU pVCpu = pVM->apCpusR3[0]; /** @todo In theory per vCPU, in practice same for all. */
+    if (pVCpu->nem.s.fTrapXcptGpForLovelyMesaDrv)
+        Property.ExceptionExitBitmap |= RT_BIT_64(WHvX64ExceptionTypeGeneralProtectionFault);
+
+    hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeExceptionExitBitmap, &Property, sizeof(Property));
+    if (FAILED(hrc))
+        return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
+                          "Failed to set WHvPartitionPropertyCodeExceptionExitBitmap to %#RX64: %Rhrc (Last=%#x/%u)",
+                          Property.ExceptionExitBitmap, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+
+
+    /*
+     * Sync CPU features with CPUM.
+     */
+    /** @todo sync CPU features with CPUM. */
+
+    /* Set the partition property. */
+    RT_ZERO(Property);
+    Property.ProcessorFeatures.AsUINT64 = pVM->nem.s.uCpuFeatures.u64;
+    hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeProcessorFeatures, &Property, sizeof(Property));
+    if (FAILED(hrc))
+        return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
+                          "Failed to set WHvPartitionPropertyCodeProcessorFeatures to %'#RX64: %Rhrc (Last=%#x/%u)",
+                          pVM->nem.s.uCpuFeatures.u64, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+
+    /*
+     * Set up the partition and create EMTs.
+     *
+     * Seems like this is where the partition is actually instantiated and we get
+     * a handle to it.
+     */
+    hrc = WHvSetupPartition(hPartition);
+    if (FAILED(hrc))
+        return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
+                          "Call to WHvSetupPartition failed: %Rhrc (Last=%#x/%u)",
+                          hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
+
+    /* Get the handle. */
+    HANDLE hPartitionDevice;
+    __try
+    {
+        hPartitionDevice = ((HANDLE *)hPartition)[1];
+    }
+    __except(EXCEPTION_EXECUTE_HANDLER)
+    {
+        hrc = GetExceptionCode();
+        hPartitionDevice = NULL;
+    }
+    if (   hPartitionDevice == NULL
+        || hPartitionDevice == (HANDLE)(intptr_t)-1)
+        return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
+                          "Failed to get device handle for partition %p: %Rhrc", hPartition, hrc);
+
+    HV_PARTITION_ID idHvPartition = HV_PARTITION_ID_INVALID;
+    if (!g_pfnVidGetHvPartitionId(hPartitionDevice, &idHvPartition))
+        return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
+                          "Failed to get device handle and/or partition ID for %p (hPartitionDevice=%p, Last=%#x/%u)",
+                          hPartition, hPartitionDevice, RTNtLastStatusValue(), RTNtLastErrorValue());
+    pVM->nem.s.hPartitionDevice = hPartitionDevice;
+    pVM->nem.s.idHvPartition    = idHvPartition;
+
+    /*
+     * Setup the EMTs.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        pVCpu = pVM->apCpusR3[idCpu];
+
+        pVCpu->nem.s.hNativeThreadHandle = (RTR3PTR)RTThreadGetNativeHandle(VMR3GetThreadHandle(pVCpu->pUVCpu));
+        Assert((HANDLE)pVCpu->nem.s.hNativeThreadHandle != INVALID_HANDLE_VALUE);
+
+#ifndef NEM_WIN_USE_OUR_OWN_RUN_API
+# ifdef NEM_WIN_WITH_RING0_RUNLOOP
+        if (!pVM->nem.s.fUseRing0Runloop)
+# endif
+        {
+            hrc = WHvCreateVirtualProcessor(hPartition, idCpu, 0 /*fFlags*/);
+            if (FAILED(hrc))
+            {
+                NTSTATUS const rcNtLast  = RTNtLastStatusValue();
+                DWORD const    dwErrLast = RTNtLastErrorValue();
+                while (idCpu-- > 0)
+                {
+                    HRESULT hrc2 = WHvDeleteVirtualProcessor(hPartition, idCpu);
+                    AssertLogRelMsg(SUCCEEDED(hrc2), ("WHvDeleteVirtualProcessor(%p, %u) -> %Rhrc (Last=%#x/%u)\n",
+                                                      hPartition, idCpu, hrc2, RTNtLastStatusValue(),
+                                                      RTNtLastErrorValue()));
+                }
+                return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
+                                  "Call to WHvCreateVirtualProcessor failed: %Rhrc (Last=%#x/%u)", hrc, rcNtLast, dwErrLast);
+            }
+        }
+# ifdef NEM_WIN_WITH_RING0_RUNLOOP
+        else
+# endif
+#endif /* !NEM_WIN_USE_OUR_OWN_RUN_API */
+#if defined(NEM_WIN_WITH_RING0_RUNLOOP) || defined(NEM_WIN_USE_OUR_OWN_RUN_API)
+        {
+            VID_MAPPED_MESSAGE_SLOT MappedMsgSlot = { NULL, UINT32_MAX, UINT32_MAX };
+            if (g_pfnVidMessageSlotMap(hPartitionDevice, &MappedMsgSlot, idCpu))
+            {
+                AssertLogRelMsg(MappedMsgSlot.iCpu == idCpu && MappedMsgSlot.uParentAdvisory == UINT32_MAX,
+                                ("%#x %#x (iCpu=%#x)\n", MappedMsgSlot.iCpu, MappedMsgSlot.uParentAdvisory, idCpu));
+                pVCpu->nem.s.pvMsgSlotMapping = MappedMsgSlot.pMsgBlock;
+            }
+            else
+            {
+                NTSTATUS const rcNtLast  = RTNtLastStatusValue();
+                DWORD const    dwErrLast = RTNtLastErrorValue();
+                return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
+                                  "Call to VidMessageSlotMap failed: Last=%#x/%u", rcNtLast, dwErrLast);
+            }
+        }
+#endif
+    }
+    pVM->nem.s.fCreatedEmts = true;
+
+    /*
+     * Do some more ring-0 initialization now that we've got the partition handle.
+     */
+    int rc = VMMR3CallR0Emt(pVM, pVM->apCpusR3[0], VMMR0_DO_NEM_INIT_VM_PART_2, 0, NULL);
+    if (RT_SUCCESS(rc))
+    {
+        LogRel(("NEM: Successfully set up partition (device handle %p, partition ID %#llx)\n", hPartitionDevice, idHvPartition));
+
+#if 1
+        VMMR3CallR0Emt(pVM, pVM->apCpusR3[0], VMMR0_DO_NEM_UPDATE_STATISTICS, 0, NULL);
+        LogRel(("NEM: Memory balance: %#RX64 out of %#RX64 pages in use\n",
+                pVM->nem.s.R0Stats.cPagesInUse, pVM->nem.s.R0Stats.cPagesAvailable));
+#endif
+
+        /*
+         * Register statistics on shared pages.
+         */
+        /** @todo HvCallMapStatsPage */
+
+        /*
+         * Adjust features.
+         * Note! We've already disabled X2APIC via CFGM during the first init call.
+         */
+
+#if 0 && defined(DEBUG_bird)
+        /*
+         * Poke and probe a little.
+         */
+        PVMCPU              pVCpu = pVM->apCpusR3[0];
+        uint32_t            aRegNames[1024];
+        HV_REGISTER_VALUE   aRegValues[1024];
+        uint32_t            aPropCodes[128];
+        uint64_t            aPropValues[128];
+        for (int iOuter = 0; iOuter < 5; iOuter++)
+        {
+            LogRel(("\niOuter %d\n", iOuter));
+# if 1
+            /* registers */
+            uint32_t iRegValue = 0;
+            uint32_t cRegChanges = 0;
+            for (uint32_t iReg = 0; iReg < 0x001101ff; iReg++)
+            {
+                if (iOuter != 0 && aRegNames[iRegValue] > iReg)
+                    continue;
+                RT_ZERO(pVCpu->nem.s.Hypercall.Experiment);
+                pVCpu->nem.s.Hypercall.Experiment.uItem = iReg;
+                int rc2 = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_EXPERIMENT, 0, NULL);
+                AssertLogRelRCBreak(rc2);
+                if (pVCpu->nem.s.Hypercall.Experiment.fSuccess)
+                {
+                    LogRel(("Register %#010x = %#18RX64, %#18RX64\n", iReg,
+                            pVCpu->nem.s.Hypercall.Experiment.uLoValue, pVCpu->nem.s.Hypercall.Experiment.uHiValue));
+                    if (iReg == HvX64RegisterTsc)
+                    {
+                        uint64_t uTsc = ASMReadTSC();
+                        LogRel(("TSC = %#18RX64; Delta %#18RX64 or %#18RX64\n",
+                                uTsc, pVCpu->nem.s.Hypercall.Experiment.uLoValue - uTsc, uTsc - pVCpu->nem.s.Hypercall.Experiment.uLoValue));
+                    }
+
+                    if (iOuter == 0)
+                        aRegNames[iRegValue] = iReg;
+                    else if(   aRegValues[iRegValue].Reg128.Low64  != pVCpu->nem.s.Hypercall.Experiment.uLoValue
+                            || aRegValues[iRegValue].Reg128.High64 != pVCpu->nem.s.Hypercall.Experiment.uHiValue)
+                    {
+                        LogRel(("Changed from          %#18RX64, %#18RX64 !!\n",
+                                aRegValues[iRegValue].Reg128.Low64, aRegValues[iRegValue].Reg128.High64));
+                        LogRel(("Delta                 %#18RX64, %#18RX64 !!\n",
+                                pVCpu->nem.s.Hypercall.Experiment.uLoValue - aRegValues[iRegValue].Reg128.Low64,
+                                pVCpu->nem.s.Hypercall.Experiment.uHiValue - aRegValues[iRegValue].Reg128.High64));
+                        cRegChanges++;
+                    }
+                    aRegValues[iRegValue].Reg128.Low64  = pVCpu->nem.s.Hypercall.Experiment.uLoValue;
+                    aRegValues[iRegValue].Reg128.High64 = pVCpu->nem.s.Hypercall.Experiment.uHiValue;
+                    iRegValue++;
+                    AssertBreak(iRegValue < RT_ELEMENTS(aRegValues));
+                }
+            }
+            LogRel(("Found %u registers, %u changed\n", iRegValue, cRegChanges));
+# endif
+# if 1
+            /* partition properties */
+            uint32_t iPropValue = 0;
+            uint32_t cPropChanges = 0;
+            for (uint32_t iProp = 0; iProp < 0xc11ff; iProp++)
+            {
+                if (iProp == HvPartitionPropertyDebugChannelId /* hangs host */)
+                    continue;
+                if (iOuter != 0 && aPropCodes[iPropValue] > iProp)
+                    continue;
+                RT_ZERO(pVCpu->nem.s.Hypercall.Experiment);
+                pVCpu->nem.s.Hypercall.Experiment.uItem = iProp;
+                int rc2 = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_EXPERIMENT, 1, NULL);
+                AssertLogRelRCBreak(rc2);
+                if (pVCpu->nem.s.Hypercall.Experiment.fSuccess)
+                {
+                    LogRel(("Property %#010x = %#18RX64\n", iProp, pVCpu->nem.s.Hypercall.Experiment.uLoValue));
+                    if (iOuter == 0)
+                        aPropCodes[iPropValue] = iProp;
+                    else if (aPropValues[iPropValue] != pVCpu->nem.s.Hypercall.Experiment.uLoValue)
+                    {
+                        LogRel(("Changed from          %#18RX64, delta %#18RX64!!\n",
+                                aPropValues[iPropValue], pVCpu->nem.s.Hypercall.Experiment.uLoValue - aPropValues[iPropValue]));
+                        cRegChanges++;
+                    }
+                    aPropValues[iPropValue] = pVCpu->nem.s.Hypercall.Experiment.uLoValue;
+                    iPropValue++;
+                    AssertBreak(iPropValue < RT_ELEMENTS(aPropValues));
+                }
+            }
+            LogRel(("Found %u properties, %u changed\n", iPropValue, cPropChanges));
+# endif
+
+            /* Modify the TSC register value and see what changes. */
+            if (iOuter != 0)
+            {
+                RT_ZERO(pVCpu->nem.s.Hypercall.Experiment);
+                pVCpu->nem.s.Hypercall.Experiment.uItem = HvX64RegisterTsc;
+                pVCpu->nem.s.Hypercall.Experiment.uHiValue = UINT64_C(0x00000fffffffffff) >> iOuter;
+                pVCpu->nem.s.Hypercall.Experiment.uLoValue = UINT64_C(0x0011100000000000) << iOuter;
+                VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_EXPERIMENT, 2, NULL);
+                LogRel(("Setting HvX64RegisterTsc -> %RTbool (%#RX64)\n", pVCpu->nem.s.Hypercall.Experiment.fSuccess, pVCpu->nem.s.Hypercall.Experiment.uStatus));
+            }
+
+            RT_ZERO(pVCpu->nem.s.Hypercall.Experiment);
+            pVCpu->nem.s.Hypercall.Experiment.uItem = HvX64RegisterTsc;
+            VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_EXPERIMENT, 0, NULL);
+            LogRel(("HvX64RegisterTsc = %#RX64, %#RX64\n", pVCpu->nem.s.Hypercall.Experiment.uLoValue, pVCpu->nem.s.Hypercall.Experiment.uHiValue));
+        }
+
+#endif
+        return VINF_SUCCESS;
+    }
+    return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Call to NEMR0InitVMPart2 failed: %Rrc", rc);
+}
+
+
+int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    //BOOL fRet = SetThreadPriority(GetCurrentThread(), 0);
+    //AssertLogRel(fRet);
+
+    NOREF(pVM); NOREF(enmWhat);
+    return VINF_SUCCESS;
+}
+
+
+int nemR3NativeTerm(PVM pVM)
+{
+    /*
+     * Delete the partition.
+     */
+    WHV_PARTITION_HANDLE hPartition = pVM->nem.s.hPartition;
+    pVM->nem.s.hPartition       = NULL;
+    pVM->nem.s.hPartitionDevice = NULL;
+    if (hPartition != NULL)
+    {
+        VMCPUID idCpu = pVM->nem.s.fCreatedEmts ? pVM->cCpus : 0;
+        LogRel(("NEM: Destroying partition %p with its %u VCpus...\n", hPartition, idCpu));
+        while (idCpu-- > 0)
+        {
+            PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+            pVCpu->nem.s.pvMsgSlotMapping = NULL;
+#ifndef NEM_WIN_USE_OUR_OWN_RUN_API
+# ifdef NEM_WIN_WITH_RING0_RUNLOOP
+            if (!pVM->nem.s.fUseRing0Runloop)
+# endif
+            {
+                HRESULT hrc = WHvDeleteVirtualProcessor(hPartition, idCpu);
+                AssertLogRelMsg(SUCCEEDED(hrc), ("WHvDeleteVirtualProcessor(%p, %u) -> %Rhrc (Last=%#x/%u)\n",
+                                                 hPartition, idCpu, hrc, RTNtLastStatusValue(),
+                                                 RTNtLastErrorValue()));
+            }
+#endif
+        }
+        WHvDeletePartition(hPartition);
+    }
+    pVM->nem.s.fCreatedEmts = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM reset notification.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+void nemR3NativeReset(PVM pVM)
+{
+    /* Unfix the A20 gate. */
+    pVM->nem.s.fA20Fixed = false;
+}
+
+
+/**
+ * Reset CPU due to INIT IPI or hot (un)plugging.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the CPU being
+ *                      reset.
+ * @param   fInitIpi    Whether this is the INIT IPI or hot (un)plugging case.
+ */
+void nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi)
+{
+    /* Lock the A20 gate if INIT IPI, make sure it's enabled.  */
+    if (fInitIpi && pVCpu->idCpu > 0)
+    {
+        PVM pVM = pVCpu->CTX_SUFF(pVM);
+        if (!pVM->nem.s.fA20Enabled)
+            nemR3NativeNotifySetA20(pVCpu, true);
+        pVM->nem.s.fA20Enabled = true;
+        pVM->nem.s.fA20Fixed   = true;
+    }
+}
+
+
+VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
+{
+#ifdef NEM_WIN_WITH_RING0_RUNLOOP
+    if (pVM->nem.s.fUseRing0Runloop)
+    {
+        for (;;)
+        {
+            VBOXSTRICTRC rcStrict = VMMR3CallR0EmtFast(pVM, pVCpu, VMMR0_DO_NEM_RUN);
+            if (RT_SUCCESS(rcStrict))
+            {
+                /*
+                 * We deal with VINF_NEM_FLUSH_TLB here, since we're running the risk of
+                 * getting these while we already got another RC (I/O ports).
+                 */
+                /* Status codes: */
+                VBOXSTRICTRC rcPending = pVCpu->nem.s.rcPending;
+                pVCpu->nem.s.rcPending = VINF_SUCCESS;
+                if (rcStrict == VINF_NEM_FLUSH_TLB || rcPending == VINF_NEM_FLUSH_TLB)
+                {
+                    LogFlow(("nemR3NativeRunGC: calling PGMFlushTLB...\n"));
+                    int rc = PGMFlushTLB(pVCpu, CPUMGetGuestCR3(pVCpu), true);
+                    AssertRCReturn(rc, rc);
+                    if (rcStrict == VINF_NEM_FLUSH_TLB)
+                    {
+                        if (   !VM_FF_IS_ANY_SET(pVM, VM_FF_HIGH_PRIORITY_POST_MASK | VM_FF_HP_R0_PRE_HM_MASK)
+                            && !VMCPU_FF_IS_ANY_SET(pVCpu,   (VMCPU_FF_HIGH_PRIORITY_POST_MASK | VMCPU_FF_HP_R0_PRE_HM_MASK)
+                                                           & ~VMCPU_FF_RESUME_GUEST_MASK))
+                        {
+                            VMCPU_FF_CLEAR_MASK(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
+                            continue;
+                        }
+                        rcStrict = VINF_SUCCESS;
+                    }
+                }
+                else
+                    AssertMsg(rcPending == VINF_SUCCESS, ("rcPending=%Rrc\n", VBOXSTRICTRC_VAL(rcPending) ));
+            }
+            LogFlow(("nemR3NativeRunGC: returns %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
+            return rcStrict;
+        }
+    }
+#endif
+    return nemHCWinRunGC(pVM, pVCpu);
+}
+
+
+bool nemR3NativeCanExecuteGuest(PVM pVM, PVMCPU pVCpu)
+{
+    NOREF(pVM); NOREF(pVCpu);
+    return true;
+}
+
+
+bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
+{
+    NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
+    return false;
+}
+
+
+/**
+ * Forced flag notification call from VMEmt.h.
+ *
+ * This is only called when pVCpu is in the VMCPUSTATE_STARTED_EXEC_NEM state.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the CPU
+ *                          to be notified.
+ * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_XXX.
+ */
+void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
+{
+#ifdef NEM_WIN_USE_OUR_OWN_RUN_API
+    nemHCWinCancelRunVirtualProcessor(pVM, pVCpu);
+#else
+# ifdef NEM_WIN_WITH_RING0_RUNLOOP
+    if (pVM->nem.s.fUseRing0Runloop)
+        nemHCWinCancelRunVirtualProcessor(pVM, pVCpu);
+    else
+# endif
+    {
+        Log8(("nemR3NativeNotifyFF: canceling %u\n", pVCpu->idCpu));
+        HRESULT hrc = WHvCancelRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, 0);
+        AssertMsg(SUCCEEDED(hrc), ("WHvCancelRunVirtualProcessor -> hrc=%Rhrc\n", hrc));
+        RT_NOREF_PV(hrc);
+    }
+#endif
+    RT_NOREF_PV(fFlags);
+}
+
+
+DECLINLINE(int) nemR3NativeGCPhys2R3PtrReadOnly(PVM pVM, RTGCPHYS GCPhys, const void **ppv)
+{
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, ppv, &Lock);
+    if (RT_SUCCESS(rc))
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+    return rc;
+}
+
+
+DECLINLINE(int) nemR3NativeGCPhys2R3PtrWriteable(PVM pVM, RTGCPHYS GCPhys, void **ppv)
+{
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, ppv, &Lock);
+    if (RT_SUCCESS(rc))
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+    return rc;
+}
+
+
+int nemR3NativeNotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
+{
+    Log5(("nemR3NativeNotifyPhysRamRegister: %RGp LB %RGp\n", GCPhys, cb));
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb);
+    return VINF_SUCCESS;
+}
+
+
+int nemR3NativeNotifyPhysMmioExMap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvMmio2)
+{
+    Log5(("nemR3NativeNotifyPhysMmioExMap: %RGp LB %RGp fFlags=%#x pvMmio2=%p\n", GCPhys, cb, fFlags, pvMmio2));
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags); NOREF(pvMmio2);
+    return VINF_SUCCESS;
+}
+
+
+int nemR3NativeNotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
+{
+    Log5(("nemR3NativeNotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags));
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Called early during ROM registration, right after the pages have been
+ * allocated and the RAM range updated.
+ *
+ * This will be succeeded by a number of NEMHCNotifyPhysPageProtChanged() calls
+ * and finally a NEMR3NotifyPhysRomRegisterEarly().
+ *
+ * @returns VBox status code
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The ROM address (page aligned).
+ * @param   cb              The size (page aligned).
+ * @param   fFlags          NEM_NOTIFY_PHYS_ROM_F_XXX.
+ */
+int nemR3NativeNotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
+{
+    Log5(("nemR3NativeNotifyPhysRomRegisterEarly: %RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags));
+#if 0 /* Let's not do this after all.  We'll protection change notifications for each page and if not we'll map them lazily. */
+    RTGCPHYS const cPages = cb >> X86_PAGE_SHIFT;
+    for (RTGCPHYS iPage = 0; iPage < cPages; iPage++, GCPhys += X86_PAGE_SIZE)
+    {
+        const void *pvPage;
+        int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhys, &pvPage);
+        if (RT_SUCCESS(rc))
+        {
+            HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhys, X86_PAGE_SIZE,
+                                         WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute);
+            if (SUCCEEDED(hrc))
+            { /* likely */ }
+            else
+            {
+                LogRel(("nemR3NativeNotifyPhysRomRegisterEarly: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                        GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+                return VERR_NEM_INIT_FAILED;
+            }
+        }
+        else
+        {
+            LogRel(("nemR3NativeNotifyPhysRomRegisterEarly: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
+            return rc;
+        }
+    }
+#else
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb);
+#endif
+    RT_NOREF_PV(fFlags);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Called after the ROM range has been fully completed.
+ *
+ * This will be preceeded by a NEMR3NotifyPhysRomRegisterEarly() call as well a
+ * number of NEMHCNotifyPhysPageProtChanged calls.
+ *
+ * @returns VBox status code
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The ROM address (page aligned).
+ * @param   cb              The size (page aligned).
+ * @param   fFlags          NEM_NOTIFY_PHYS_ROM_F_XXX.
+ */
+int nemR3NativeNotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
+{
+    Log5(("nemR3NativeNotifyPhysRomRegisterLate: %RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags));
+    NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNPGMPHYSNEMCHECKPAGE}
+ */
+static DECLCALLBACK(int) nemR3WinUnsetForA20CheckerCallback(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys,
+                                                            PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
+{
+    /* We'll just unmap the memory. */
+    if (pInfo->u2NemState > NEM_WIN_PAGE_STATE_UNMAPPED)
+    {
+#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+        int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
+        AssertRC(rc);
+        if (RT_SUCCESS(rc))
+#else
+        HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
+        if (SUCCEEDED(hrc))
+#endif
+        {
+            uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+            Log5(("NEM GPA unmapped/A20: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[pInfo->u2NemState], cMappedPages));
+            pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
+        }
+        else
+        {
+#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+            LogRel(("nemR3WinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
+            return rc;
+#else
+            LogRel(("nemR3WinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                    GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+            return VERR_INTERNAL_ERROR_2;
+#endif
+        }
+    }
+    RT_NOREF(pVCpu, pvUser);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Unmaps a page from Hyper-V for the purpose of emulating A20 gate behavior.
+ *
+ * @returns The PGMPhysNemQueryPageInfo result.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhys          The page to unmap.
+ */
+static int nemR3WinUnmapPageForA20Gate(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
+{
+    PGMPHYSNEMPAGEINFO Info;
+    return PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhys, false /*fMakeWritable*/, &Info,
+                                     nemR3WinUnsetForA20CheckerCallback, NULL);
+}
+
+
+/**
+ * Called when the A20 state changes.
+ *
+ * Hyper-V doesn't seem to offer a simple way of implementing the A20 line
+ * features of PCs.  So, we do a very minimal emulation of the HMA to make DOS
+ * happy.
+ *
+ * @param   pVCpu           The CPU the A20 state changed on.
+ * @param   fEnabled        Whether it was enabled (true) or disabled.
+ */
+void nemR3NativeNotifySetA20(PVMCPU pVCpu, bool fEnabled)
+{
+    Log(("nemR3NativeNotifySetA20: fEnabled=%RTbool\n", fEnabled));
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    if (!pVM->nem.s.fA20Fixed)
+    {
+        pVM->nem.s.fA20Enabled = fEnabled;
+        for (RTGCPHYS GCPhys = _1M; GCPhys < _1M + _64K; GCPhys += X86_PAGE_SIZE)
+            nemR3WinUnmapPageForA20Gate(pVM, pVCpu, GCPhys);
+    }
+}
+
+
+/** @page pg_nem_win NEM/win - Native Execution Manager, Windows.
+ *
+ * On Windows the Hyper-V root partition (dom0 in zen terminology) does not have
+ * nested VT-x or AMD-V capabilities.  Early on raw-mode worked inside it, but
+ * for a while now we've been getting \#GPs when trying to modify CR4 in the
+ * world switcher.  So, when Hyper-V is active on Windows we have little choice
+ * but to use Hyper-V to run our VMs.
+ *
+ *
+ * @section sub_nem_win_whv   The WinHvPlatform API
+ *
+ * Since Windows 10 build 17083 there is a documented API for managing Hyper-V
+ * VMs: header file WinHvPlatform.h and implementation in WinHvPlatform.dll.
+ * This interface is a wrapper around the undocumented Virtualization
+ * Infrastructure Driver (VID) API - VID.DLL and VID.SYS.  The wrapper is
+ * written in C++, namespaced, early versions (at least) was using standard C++
+ * container templates in several places.
+ *
+ * When creating a VM using WHvCreatePartition, it will only create the
+ * WinHvPlatform structures for it, to which you get an abstract pointer.  The
+ * VID API that actually creates the partition is first engaged when you call
+ * WHvSetupPartition after first setting a lot of properties using
+ * WHvSetPartitionProperty.  Since the VID API is just a very thin wrapper
+ * around CreateFile and NtDeviceIoControlFile, it returns an actual HANDLE for
+ * the partition to WinHvPlatform.  We fish this HANDLE out of the WinHvPlatform
+ * partition structures because we need to talk directly to VID for reasons
+ * we'll get to in a bit.  (Btw. we could also intercept the CreateFileW or
+ * NtDeviceIoControlFile calls from VID.DLL to get the HANDLE should fishing in
+ * the partition structures become difficult.)
+ *
+ * The WinHvPlatform API requires us to both set the number of guest CPUs before
+ * setting up the partition and call WHvCreateVirtualProcessor for each of them.
+ * The CPU creation function boils down to a VidMessageSlotMap call that sets up
+ * and maps a message buffer into ring-3 for async communication with hyper-V
+ * and/or the VID.SYS thread actually running the CPU thru
+ * WinHvRunVpDispatchLoop().  When for instance a VMEXIT is encountered, hyper-V
+ * sends a message that the WHvRunVirtualProcessor API retrieves (and later
+ * acknowledges) via VidMessageSlotHandleAndGetNext.   Since or about build
+ * 17757 a register page is also mapped into user space when creating the
+ * virtual CPU.  It should be noteded that WHvDeleteVirtualProcessor doesn't do
+ * much as there seems to be no partner function VidMessagesSlotMap that
+ * reverses what it did.
+ *
+ * Memory is managed thru calls to WHvMapGpaRange and WHvUnmapGpaRange (GPA does
+ * not mean grade point average here, but rather guest physical addressspace),
+ * which corresponds to VidCreateVaGpaRangeSpecifyUserVa and VidDestroyGpaRange
+ * respectively.  As 'UserVa' indicates, the functions works on user process
+ * memory.  The mappings are also subject to quota restrictions, so the number
+ * of ranges are limited and probably their total size as well.  Obviously
+ * VID.SYS keeps track of the ranges, but so does WinHvPlatform, which means
+ * there is a bit of overhead involved and quota restrctions makes sense.
+ *
+ * Running guest code is done through the WHvRunVirtualProcessor function.  It
+ * asynchronously starts or resumes hyper-V CPU execution and then waits for an
+ * VMEXIT message.  Hyper-V / VID.SYS will return information about the message
+ * in the message buffer mapping, and WHvRunVirtualProcessor will convert that
+ * finto it's own WHV_RUN_VP_EXIT_CONTEXT format.
+ *
+ * Other threads can interrupt the execution by using WHvCancelVirtualProcessor,
+ * which since or about build 17757 uses VidMessageSlotHandleAndGetNext to do
+ * the work (earlier builds would open the waiting thread, do a dummy
+ * QueueUserAPC on it, and let it upon return use VidStopVirtualProcessor to
+ * do the actual stopping).  While there is certainly a race between cancelation
+ * and the CPU causing a natural VMEXIT, it is not known whether this still
+ * causes extra work on subsequent WHvRunVirtualProcessor calls (it did in and
+ * earlier 17134).
+ *
+ * Registers are retrieved and set via WHvGetVirtualProcessorRegisters and
+ * WHvSetVirtualProcessorRegisters.  In addition, several VMEXITs include
+ * essential register state in the exit context information, potentially making
+ * it possible to emulate the instruction causing the exit without involving
+ * WHvGetVirtualProcessorRegisters.
+ *
+ *
+ * @subsection subsec_nem_win_whv_cons  Issues & Feedback
+ *
+ * Here are some observations (mostly against build 17101):
+ *
+ * - The VMEXIT performance is dismal (build 17134).
+ *
+ *   Our proof of concept implementation with a kernel runloop (i.e. not using
+ *   WHvRunVirtualProcessor and friends, but calling VID.SYS fast I/O control
+ *   entry point directly) delivers 9-10% of the port I/O performance and only
+ *   6-7% of the MMIO performance that we have with our own hypervisor.
+ *
+ *   When using the offical WinHvPlatform API, the numbers are %3 for port I/O
+ *   and 5% for MMIO.
+ *
+ *   While the tests we've done are using tight tight loops only doing port I/O
+ *   and MMIO, the problem is clearly visible when running regular guest OSes.
+ *   Anything that hammers the VGA device would be suffering, for example:
+ *
+ *       - Windows 2000 boot screen animation overloads us with MMIO exits
+ *         and won't even boot because all the time is spent in interrupt
+ *         handlers and redrawin the screen.
+ *
+ *       - DSL 4.4 and its bootmenu logo is slower than molasses in january.
+ *
+ *   We have not found a workaround for this yet.
+ *
+ *   Something that might improve the issue a little is to detect blocks with
+ *   excessive MMIO and port I/O exits and emulate instructions to cover
+ *   multiple exits before letting Hyper-V have a go at the guest execution
+ *   again.  This will only improve the situation under some circumstances,
+ *   since emulating instructions without recompilation can be expensive, so
+ *   there will only be real gains if the exitting instructions are tightly
+ *   packed.
+ *
+ *   Update: Security fixes during the summer of 2018 caused the performance to
+ *   dropped even more.
+ *
+ *   Update [build 17757]: Some performance improvements here, but they don't
+ *   yet make up for what was lost this summer.
+ *
+ *
+ * - We need a way to directly modify the TSC offset (or bias if you like).
+ *
+ *   The current approach of setting the WHvX64RegisterTsc register one by one
+ *   on each virtual CPU in sequence will introduce random inaccuracies,
+ *   especially if the thread doing the job is reschduled at a bad time.
+ *
+ *
+ * - Unable to access WHvX64RegisterMsrMtrrCap (build 17134).
+ *
+ *
+ * - On AMD Ryzen grub/debian 9.0 ends up with a unrecoverable exception
+ *   when IA32_MTRR_PHYSMASK0 is written.
+ *
+ *
+ * - The IA32_APIC_BASE register does not work right:
+ *
+ *      - Attempts by the guest to clear bit 11 (EN) are ignored, both the
+ *        guest and the VMM reads back the old value.
+ *
+ *      - Attempts to modify the base address (bits NN:12) seems to be ignored
+ *        in the same way.
+ *
+ *      - The VMM can modify both the base address as well as the the EN and
+ *        BSP bits, however this is useless if we cannot intercept the WRMSR.
+ *
+ *      - Attempts by the guest to set the EXTD bit (X2APIC) result in \#GP(0),
+ *        while the VMM ends up with with ERROR_HV_INVALID_PARAMETER.  Seems
+ *        there is no way to support X2APIC.
+ *
+ *
+ * - Not sure if this is a thing, but WHvCancelVirtualProcessor seems to cause
+ *   cause a lot more spurious WHvRunVirtualProcessor returns that what we get
+ *   with the replacement code.  By spurious returns we mean that the
+ *   subsequent call to WHvRunVirtualProcessor would return immediately.
+ *
+ *   Update [build 17757]: New cancelation code might have addressed this, but
+ *   haven't had time to test it yet.
+ *
+ *
+ * - There is no API for modifying protection of a page within a GPA range.
+ *
+ *   From what we can tell, the only way to modify the protection (like readonly
+ *   -> writable, or vice versa) is to first unmap the range and then remap it
+ *   with the new protection.
+ *
+ *   We are for instance doing this quite a bit in order to track dirty VRAM
+ *   pages.  VRAM pages starts out as readonly, when the guest writes to a page
+ *   we take an exit, notes down which page it is, makes it writable and restart
+ *   the instruction.  After refreshing the display, we reset all the writable
+ *   pages to readonly again, bulk fashion.
+ *
+ *   Now to work around this issue, we do page sized GPA ranges.  In addition to
+ *   add a lot of tracking overhead to WinHvPlatform and VID.SYS, this also
+ *   causes us to exceed our quota before we've even mapped a default sized
+ *   (128MB) VRAM page-by-page.  So, to work around this quota issue we have to
+ *   lazily map pages and actively restrict the number of mappings.
+ *
+ *   Our best workaround thus far is bypassing WinHvPlatform and VID entirely
+ *   when in comes to guest memory management and instead use the underlying
+ *   hypercalls (HvCallMapGpaPages, HvCallUnmapGpaPages) to do it ourselves.
+ *   (This also maps a whole lot better into our own guest page management
+ *   infrastructure.)
+ *
+ *   Update [build 17757]: Introduces a KVM like dirty logging API which could
+ *   help tracking dirty VGA pages, while being useless for shadow ROM and
+ *   devices trying catch the guest updating descriptors and such.
+ *
+ *
+ * - Observed problems doing WHvUnmapGpaRange immediately followed by
+ *   WHvMapGpaRange.
+ *
+ *   As mentioned above, we've been forced to use this sequence when modifying
+ *   page protection.   However, when transitioning from readonly to writable,
+ *   we've ended up looping forever with the same write to readonly memory
+ *   VMEXIT.  We're wondering if this issue might be related to the lazy mapping
+ *   logic in WinHvPlatform.
+ *
+ *   Workaround: Insert a WHvRunVirtualProcessor call and make sure to get a GPA
+ *   unmapped exit between the two calls.  Not entirely great performance wise
+ *   (or the santity of our code).
+ *
+ *
+ * - Implementing A20 gate behavior is tedious, where as correctly emulating the
+ *   A20M# pin (present on 486 and later) is near impossible for SMP setups
+ *   (e.g. possiblity of two CPUs with different A20 status).
+ *
+ *   Workaround: Only do A20 on CPU 0, restricting the emulation to HMA.  We
+ *   unmap all pages related to HMA (0x100000..0x10ffff) when the A20 state
+ *   changes, lazily syncing the right pages back when accessed.
+ *
+ *
+ * - WHVRunVirtualProcessor wastes time converting VID/Hyper-V messages to its
+ *   own format (WHV_RUN_VP_EXIT_CONTEXT).
+ *
+ *   We understand this might be because Microsoft wishes to remain free to
+ *   modify the VID/Hyper-V messages, but it's still rather silly and does slow
+ *   things down a little.  We'd much rather just process the messages directly.
+ *
+ *
+ * - WHVRunVirtualProcessor would've benefited from using a callback interface:
+ *
+ *      - The potential size changes of the exit context structure wouldn't be
+ *        an issue, since the function could manage that itself.
+ *
+ *      - State handling could probably be simplified (like cancelation).
+ *
+ *
+ * - WHvGetVirtualProcessorRegisters and WHvSetVirtualProcessorRegisters
+ *   internally converts register names, probably using temporary heap buffers.
+ *
+ *   From the looks of things, they are converting from WHV_REGISTER_NAME to
+ *   HV_REGISTER_NAME from in the "Virtual Processor Register Names" section in
+ *   the "Hypervisor Top-Level Functional Specification" document.  This feels
+ *   like an awful waste of time.
+ *
+ *   We simply cannot understand why HV_REGISTER_NAME isn't used directly here,
+ *   or at least the same values, making any conversion reduntant.  Restricting
+ *   access to certain registers could easily be implement by scanning the
+ *   inputs.
+ *
+ *   To avoid the heap + conversion overhead, we're currently using the
+ *   HvCallGetVpRegisters and HvCallSetVpRegisters calls directly, at least for
+ *   the ring-0 code.
+ *
+ *   Update [build 17757]: Register translation has been very cleverly
+ *   optimized and made table driven (2 top level tables, 4 + 1 leaf tables).
+ *   Register information consists of the 32-bit HV register name, register page
+ *   offset, and flags (giving valid offset, size and more).  Register
+ *   getting/settings seems to be done by hoping that the register page provides
+ *   it all, and falling back on the VidSetVirtualProcessorState if one or more
+ *   registers are not available there.
+ *
+ *   Note! We have currently not updated our ring-0 code to take the register
+ *   page into account, so it's suffering a little compared to the ring-3 code
+ *   that now uses the offical APIs for registers.
+ *
+ *
+ * - The YMM and XCR0 registers are not yet named (17083).  This probably
+ *   wouldn't be a problem if HV_REGISTER_NAME was used, see previous point.
+ *
+ *   Update [build 17757]: XCR0 is added. YMM register values seems to be put
+ *   into a yet undocumented XsaveState interface.  Approach is a little bulky,
+ *   but saves number of enums and dispenses with register transation.  Also,
+ *   the underlying Vid setter API duplicates the input buffer on the heap,
+ *   adding a 16 byte header.
+ *
+ *
+ * - Why does VID.SYS only query/set 32 registers at the time thru the
+ *   HvCallGetVpRegisters and HvCallSetVpRegisters hypercalls?
+ *
+ *   We've not trouble getting/setting all the registers defined by
+ *   WHV_REGISTER_NAME in one hypercall (around 80).  Some kind of stack
+ *   buffering or similar?
+ *
+ *
+ * - To handle the VMMCALL / VMCALL instructions, it seems we need to intercept
+ *   \#UD exceptions and inspect the opcodes.  A dedicated exit for hypercalls
+ *   would be more efficient, esp. for guests using \#UD for other purposes..
+ *
+ *
+ * - Wrong instruction length in the VpContext with unmapped GPA memory exit
+ *   contexts on 17115/AMD.
+ *
+ *   One byte "PUSH CS" was reported as 2 bytes, while a two byte
+ *   "MOV [EBX],EAX" was reported with a 1 byte instruction length.  Problem
+ *   naturally present in untranslated hyper-v messages.
+ *
+ *
+ * - The I/O port exit context information seems to be missing the address size
+ *   information needed for correct string I/O emulation.
+ *
+ *   VT-x provides this information in bits 7:9 in the instruction information
+ *   field on newer CPUs.  AMD-V in bits 7:9 in the EXITINFO1 field in the VMCB.
+ *
+ *   We can probably work around this by scanning the instruction bytes for
+ *   address size prefixes.  Haven't investigated it any further yet.
+ *
+ *
+ * - Querying WHvCapabilityCodeExceptionExitBitmap returns zero even when
+ *   intercepts demonstrably works (17134).
+ *
+ *
+ * - Querying HvPartitionPropertyDebugChannelId via HvCallGetPartitionProperty
+ *   (hypercall) hangs the host (17134).
+ *
+ *
+ *
+ * Old concerns that have been addressed:
+ *
+ * - The WHvCancelVirtualProcessor API schedules a dummy usermode APC callback
+ *   in order to cancel any current or future alertable wait in VID.SYS during
+ *   the VidMessageSlotHandleAndGetNext call.
+ *
+ *   IIRC this will make the kernel schedule the specified callback thru
+ *   NTDLL!KiUserApcDispatcher by modifying the thread context and quite
+ *   possibly the userland thread stack.  When the APC callback returns to
+ *   KiUserApcDispatcher, it will call NtContinue to restore the old thread
+ *   context and resume execution from there.  This naturally adds up to some
+ *   CPU cycles, ring transitions aren't for free, especially after Spectre &
+ *   Meltdown mitigations.
+ *
+ *   Using NtAltertThread call could do the same without the thread context
+ *   modifications and the extra kernel call.
+ *
+ *   Update: All concerns have addressed in or about build 17757.
+ *
+ *   The WHvCancelVirtualProcessor API is now implemented using a new
+ *   VidMessageSlotHandleAndGetNext() flag (4).  Codepath is slightly longer
+ *   than NtAlertThread, but has the added benefit that spurious wakeups can be
+ *   more easily reduced.
+ *
+ *
+ * - When WHvRunVirtualProcessor returns without a message, or on a terse
+ *   VID message like HLT, it will make a kernel call to get some registers.
+ *   This is potentially inefficient if the caller decides he needs more
+ *   register state.
+ *
+ *   It would be better to just return what's available and let the caller fetch
+ *   what is missing from his point of view in a single kernel call.
+ *
+ *   Update: All concerns have been addressed in or about build 17757.  Selected
+ *   registers are now available via shared memory and thus HLT should (not
+ *   verified) no longer require a system call to compose the exit context data.
+ *
+ *
+ * - The WHvRunVirtualProcessor implementation does lazy GPA range mappings when
+ *   a unmapped GPA message is received from hyper-V.
+ *
+ *   Since MMIO is currently realized as unmapped GPA, this will slow down all
+ *   MMIO accesses a tiny little bit as WHvRunVirtualProcessor looks up the
+ *   guest physical address to check if it is a pending lazy mapping.
+ *
+ *   The lazy mapping feature makes no sense to us.  We as API user have all the
+ *   information and can do lazy mapping ourselves if we want/have to (see next
+ *   point).
+ *
+ *   Update: All concerns have been addressed in or about build 17757.
+ *
+ *
+ * - The WHvGetCapability function has a weird design:
+ *      - The CapabilityCode parameter is pointlessly duplicated in the output
+ *        structure (WHV_CAPABILITY).
+ *
+ *      - API takes void pointer, but everyone will probably be using
+ *        WHV_CAPABILITY due to WHV_CAPABILITY::CapabilityCode making it
+ *        impractical to use anything else.
+ *
+ *      - No output size.
+ *
+ *      - See GetFileAttributesEx, GetFileInformationByHandleEx,
+ *        FindFirstFileEx, and others for typical pattern for generic
+ *        information getters.
+ *
+ *   Update: All concerns have been addressed in build 17110.
+ *
+ *
+ * - The WHvGetPartitionProperty function uses the same weird design as
+ *   WHvGetCapability, see above.
+ *
+ *   Update: All concerns have been addressed in build 17110.
+ *
+ *
+ * - The WHvSetPartitionProperty function has a totally weird design too:
+ *      - In contrast to its partner WHvGetPartitionProperty, the property code
+ *        is not a separate input parameter here but part of the input
+ *        structure.
+ *
+ *      - The input structure is a void pointer rather than a pointer to
+ *        WHV_PARTITION_PROPERTY which everyone probably will be using because
+ *        of the WHV_PARTITION_PROPERTY::PropertyCode field.
+ *
+ *      - Really, why use PVOID for the input when the function isn't accepting
+ *        minimal sizes.  E.g. WHVPartitionPropertyCodeProcessorClFlushSize only
+ *        requires a 9 byte input, but the function insists on 16 bytes (17083).
+ *
+ *      - See GetFileAttributesEx, SetFileInformationByHandle, FindFirstFileEx,
+ *        and others for typical pattern for generic information setters and
+ *        getters.
+ *
+ *   Update: All concerns have been addressed in build 17110.
+ *
+ *
+ *
+ * @section sec_nem_win_impl    Our implementation.
+ *
+ * We set out with the goal of wanting to run as much as possible in ring-0,
+ * reasoning that this would give use the best performance.
+ *
+ * This goal was approached gradually, starting out with a pure WinHvPlatform
+ * implementation, gradually replacing parts: register access, guest memory
+ * handling, running virtual processors.  Then finally moving it all into
+ * ring-0, while keeping most of it configurable so that we could make
+ * comparisons (see NEMInternal.h and nemR3NativeRunGC()).
+ *
+ *
+ * @subsection subsect_nem_win_impl_ioctl       VID.SYS I/O control calls
+ *
+ * To run things in ring-0 we need to talk directly to VID.SYS thru its I/O
+ * control interface.  Looking at changes between like build 17083 and 17101 (if
+ * memory serves) a set of the VID I/O control numbers shifted a little, which
+ * means we need to determin them dynamically.  We currently do this by hooking
+ * the NtDeviceIoControlFile API call from VID.DLL and snooping up the
+ * parameters when making dummy calls to relevant APIs.  (We could also
+ * disassemble the relevant APIs and try fish out the information from that, but
+ * this is way simpler.)
+ *
+ * Issuing I/O control calls from ring-0 is facing a small challenge with
+ * respect to direct buffering.  When using direct buffering the device will
+ * typically check that the buffer is actually in the user address space range
+ * and reject kernel addresses.  Fortunately, we've got the cross context VM
+ * structure that is mapped into both kernel and user space, it's also locked
+ * and safe to access from kernel space.  So, we place the I/O control buffers
+ * in the per-CPU part of it (NEMCPU::uIoCtlBuf) and give the driver the user
+ * address if direct access buffering or kernel address if not.
+ *
+ * The I/O control calls are 'abstracted' in the support driver, see
+ * SUPR0IoCtlSetupForHandle(), SUPR0IoCtlPerform() and SUPR0IoCtlCleanup().
+ *
+ *
+ * @subsection subsect_nem_win_impl_cpumctx     CPUMCTX
+ *
+ * Since the CPU state needs to live in Hyper-V when executing, we probably
+ * should not transfer more than necessary when handling VMEXITs.  To help us
+ * manage this CPUMCTX got a new field CPUMCTX::fExtrn that to indicate which
+ * part of the state is currently externalized (== in Hyper-V).
+ *
+ *
+ * @subsection sec_nem_win_benchmarks           Benchmarks.
+ *
+ * @subsubsection subsect_nem_win_benchmarks_bs2t1      17134/2018-06-22: Bootsector2-test1
+ *
+ * This is ValidationKit/bootsectors/bootsector2-test1.asm as of 2018-06-22
+ * (internal r123172) running a the release build of VirtualBox from the same
+ * source, though with exit optimizations disabled.  Host is AMD Threadripper 1950X
+ * running out an up to date 64-bit Windows 10 build 17134.
+ *
+ * The base line column is using the official WinHv API for everything but physical
+ * memory mapping.  The 2nd column is the default NEM/win configuration where we
+ * put the main execution loop in ring-0, using hypercalls when we can and VID for
+ * managing execution.  The 3rd column is regular VirtualBox using AMD-V directly,
+ * hyper-V is disabled, main execution loop in ring-0.
+ *
+ * @verbatim
+TESTING...                                                           WinHv API           Hypercalls + VID    VirtualBox AMD-V
+  32-bit paged protected mode, CPUID                        :          108 874 ins/sec   113% / 123 602      1198% / 1 305 113
+  32-bit pae protected mode, CPUID                          :          106 722 ins/sec   115% / 122 740      1232% / 1 315 201
+  64-bit long mode, CPUID                                   :          106 798 ins/sec   114% / 122 111      1198% / 1 280 404
+  16-bit unpaged protected mode, CPUID                      :          106 835 ins/sec   114% / 121 994      1216% / 1 299 665
+  32-bit unpaged protected mode, CPUID                      :          105 257 ins/sec   115% / 121 772      1235% / 1 300 860
+  real mode, CPUID                                          :          104 507 ins/sec   116% / 121 800      1228% / 1 283 848
+CPUID EAX=1                                                 : PASSED
+  32-bit paged protected mode, RDTSC                        :       99 581 834 ins/sec   100% / 100 323 307    93% / 93 473 299
+  32-bit pae protected mode, RDTSC                          :       99 620 585 ins/sec   100% / 99 960 952     84% / 83 968 839
+  64-bit long mode, RDTSC                                   :      100 540 009 ins/sec   100% / 100 946 372    93% / 93 652 826
+  16-bit unpaged protected mode, RDTSC                      :       99 688 473 ins/sec   100% / 100 097 751    76% / 76 281 287
+  32-bit unpaged protected mode, RDTSC                      :       98 385 857 ins/sec   102% / 100 510 404    94% / 93 379 536
+  real mode, RDTSC                                          :      100 087 967 ins/sec   101% / 101 386 138    93% / 93 234 999
+RDTSC                                                       : PASSED
+  32-bit paged protected mode, Read CR4                     :        2 156 102 ins/sec    98% / 2 121 967   17114% / 369 009 009
+  32-bit pae protected mode, Read CR4                       :        2 163 820 ins/sec    98% / 2 133 804   17469% / 377 999 261
+  64-bit long mode, Read CR4                                :        2 164 822 ins/sec    98% / 2 128 698   18875% / 408 619 313
+  16-bit unpaged protected mode, Read CR4                   :        2 162 367 ins/sec   100% / 2 168 508   17132% / 370 477 568
+  32-bit unpaged protected mode, Read CR4                   :        2 163 189 ins/sec   100% / 2 169 808   16768% / 362 734 679
+  real mode, Read CR4                                       :        2 162 436 ins/sec   100% / 2 164 914   15551% / 336 288 998
+Read CR4                                                    : PASSED
+  real mode, 32-bit IN                                      :          104 649 ins/sec   118% / 123 513      1028% / 1 075 831
+  real mode, 32-bit OUT                                     :          107 102 ins/sec   115% / 123 660       982% / 1 052 259
+  real mode, 32-bit IN-to-ring-3                            :          105 697 ins/sec    98% / 104 471       201% / 213 216
+  real mode, 32-bit OUT-to-ring-3                           :          105 830 ins/sec    98% / 104 598       198% / 210 495
+  16-bit unpaged protected mode, 32-bit IN                  :          104 855 ins/sec   117% / 123 174      1029% / 1 079 591
+  16-bit unpaged protected mode, 32-bit OUT                 :          107 529 ins/sec   115% / 124 250       992% / 1 067 053
+  16-bit unpaged protected mode, 32-bit IN-to-ring-3        :          106 337 ins/sec   103% / 109 565       196% / 209 367
+  16-bit unpaged protected mode, 32-bit OUT-to-ring-3       :          107 558 ins/sec   100% / 108 237       191% / 206 387
+  32-bit unpaged protected mode, 32-bit IN                  :          106 351 ins/sec   116% / 123 584      1016% / 1 081 325
+  32-bit unpaged protected mode, 32-bit OUT                 :          106 424 ins/sec   116% / 124 252       995% / 1 059 408
+  32-bit unpaged protected mode, 32-bit IN-to-ring-3        :          104 035 ins/sec   101% / 105 305       202% / 210 750
+  32-bit unpaged protected mode, 32-bit OUT-to-ring-3       :          103 831 ins/sec   102% / 106 919       205% / 213 198
+  32-bit paged protected mode, 32-bit IN                    :          103 356 ins/sec   119% / 123 870      1041% / 1 076 463
+  32-bit paged protected mode, 32-bit OUT                   :          107 177 ins/sec   115% / 124 302       998% / 1 069 655
+  32-bit paged protected mode, 32-bit IN-to-ring-3          :          104 491 ins/sec   100% / 104 744       200% / 209 264
+  32-bit paged protected mode, 32-bit OUT-to-ring-3         :          106 603 ins/sec    97% / 103 849       197% / 210 219
+  32-bit pae protected mode, 32-bit IN                      :          105 923 ins/sec   115% / 122 759      1041% / 1 103 261
+  32-bit pae protected mode, 32-bit OUT                     :          107 083 ins/sec   117% / 126 057      1024% / 1 096 667
+  32-bit pae protected mode, 32-bit IN-to-ring-3            :          106 114 ins/sec    97% / 103 496       199% / 211 312
+  32-bit pae protected mode, 32-bit OUT-to-ring-3           :          105 675 ins/sec    96% / 102 096       198% / 209 890
+  64-bit long mode, 32-bit IN                               :          105 800 ins/sec   113% / 120 006      1013% / 1 072 116
+  64-bit long mode, 32-bit OUT                              :          105 635 ins/sec   113% / 120 375       997% / 1 053 655
+  64-bit long mode, 32-bit IN-to-ring-3                     :          105 274 ins/sec    95% / 100 763       197% / 208 026
+  64-bit long mode, 32-bit OUT-to-ring-3                    :          106 262 ins/sec    94% / 100 749       196% / 209 288
+NOP I/O Port Access                                         : PASSED
+  32-bit paged protected mode, 32-bit read                  :           57 687 ins/sec   119% / 69 136       1197% / 690 548
+  32-bit paged protected mode, 32-bit write                 :           57 957 ins/sec   118% / 68 935       1183% / 685 930
+  32-bit paged protected mode, 32-bit read-to-ring-3        :           57 958 ins/sec    95% / 55 432        276% / 160 505
+  32-bit paged protected mode, 32-bit write-to-ring-3       :           57 922 ins/sec   100% / 58 340        304% / 176 464
+  32-bit pae protected mode, 32-bit read                    :           57 478 ins/sec   119% / 68 453       1141% / 656 159
+  32-bit pae protected mode, 32-bit write                   :           57 226 ins/sec   118% / 68 097       1157% / 662 504
+  32-bit pae protected mode, 32-bit read-to-ring-3          :           57 582 ins/sec    94% / 54 651        268% / 154 867
+  32-bit pae protected mode, 32-bit write-to-ring-3         :           57 697 ins/sec   100% / 57 750        299% / 173 030
+  64-bit long mode, 32-bit read                             :           57 128 ins/sec   118% / 67 779       1071% / 611 949
+  64-bit long mode, 32-bit write                            :           57 127 ins/sec   118% / 67 632       1084% / 619 395
+  64-bit long mode, 32-bit read-to-ring-3                   :           57 181 ins/sec    94% / 54 123        265% / 151 937
+  64-bit long mode, 32-bit write-to-ring-3                  :           57 297 ins/sec    99% / 57 286        294% / 168 694
+  16-bit unpaged protected mode, 32-bit read                :           58 827 ins/sec   118% / 69 545       1185% / 697 602
+  16-bit unpaged protected mode, 32-bit write               :           58 678 ins/sec   118% / 69 442       1183% / 694 387
+  16-bit unpaged protected mode, 32-bit read-to-ring-3      :           57 841 ins/sec    96% / 55 730        275% / 159 163
+  16-bit unpaged protected mode, 32-bit write-to-ring-3     :           57 855 ins/sec   101% / 58 834        304% / 176 169
+  32-bit unpaged protected mode, 32-bit read                :           58 063 ins/sec   120% / 69 690       1233% / 716 444
+  32-bit unpaged protected mode, 32-bit write               :           57 936 ins/sec   120% / 69 633       1199% / 694 753
+  32-bit unpaged protected mode, 32-bit read-to-ring-3      :           58 451 ins/sec    96% / 56 183        273% / 159 972
+  32-bit unpaged protected mode, 32-bit write-to-ring-3     :           58 962 ins/sec    99% / 58 955        298% / 175 936
+  real mode, 32-bit read                                    :           58 571 ins/sec   118% / 69 478       1160% / 679 917
+  real mode, 32-bit write                                   :           58 418 ins/sec   118% / 69 320       1185% / 692 513
+  real mode, 32-bit read-to-ring-3                          :           58 072 ins/sec    96% / 55 751        274% / 159 145
+  real mode, 32-bit write-to-ring-3                         :           57 870 ins/sec   101% / 58 755        307% / 178 042
+NOP MMIO Access                                             : PASSED
+SUCCESS
+ * @endverbatim
+ *
+ * What we see here is:
+ *
+ *  - The WinHv API approach is 10 to 12 times slower for exits we can
+ *    handle directly in ring-0 in the VBox AMD-V code.
+ *
+ *  - The WinHv API approach is 2 to 3 times slower for exits we have to
+ *    go to ring-3 to handle with the VBox AMD-V code.
+ *
+ *  - By using hypercalls and VID.SYS from ring-0 we gain between
+ *    13% and 20% over the WinHv API on exits handled in ring-0.
+ *
+ *  - For exits requiring ring-3 handling are between 6% slower and 3% faster
+ *    than the WinHv API.
+ *
+ *
+ * As a side note, it looks like Hyper-V doesn't let the guest read CR4 but
+ * triggers exits all the time.  This isn't all that important these days since
+ * OSes like Linux cache the CR4 value specifically to avoid these kinds of exits.
+ *
+ *
+ * @subsubsection subsect_nem_win_benchmarks_bs2t1u1   17134/2018-10-02: Bootsector2-test1
+ *
+ * Update on 17134.  While expectantly testing a couple of newer builds (17758,
+ * 17763) hoping for some increases in performance, the numbers turned out
+ * altogether worse than the June test run.  So, we went back to the 1803
+ * (17134) installation, made sure it was fully up to date (as per 2018-10-02)
+ * and re-tested.
+ *
+ * The numbers had somehow turned significantly worse over the last 3-4 months,
+ * dropping around  70%  for the WinHv API test, more for Hypercalls + VID.
+ *
+ * @verbatim
+TESTING...                                                           WinHv API           Hypercalls + VID    VirtualBox AMD-V *
+  32-bit paged protected mode, CPUID                        :           33 270 ins/sec        33 154
+  real mode, CPUID                                          :           33 534 ins/sec        32 711
+  [snip]
+  32-bit paged protected mode, RDTSC                        :      102 216 011 ins/sec    98 225 419
+  real mode, RDTSC                                          :      102 492 243 ins/sec    98 225 419
+  [snip]
+  32-bit paged protected mode, Read CR4                     :        2 096 165 ins/sec     2 123 815
+  real mode, Read CR4                                       :        2 081 047 ins/sec     2 075 151
+  [snip]
+  32-bit paged protected mode, 32-bit IN                    :           32 739 ins/sec        33 655
+  32-bit paged protected mode, 32-bit OUT                   :           32 702 ins/sec        33 777
+  32-bit paged protected mode, 32-bit IN-to-ring-3          :           32 579 ins/sec        29 985
+  32-bit paged protected mode, 32-bit OUT-to-ring-3         :           32 750 ins/sec        29 757
+  [snip]
+  32-bit paged protected mode, 32-bit read                  :           20 042 ins/sec        21 489
+  32-bit paged protected mode, 32-bit write                 :           20 036 ins/sec        21 493
+  32-bit paged protected mode, 32-bit read-to-ring-3        :           19 985 ins/sec        19 143
+  32-bit paged protected mode, 32-bit write-to-ring-3       :           19 972 ins/sec        19 595
+
+ * @endverbatim
+ *
+ * Suspects are security updates and/or microcode updates installed since then.
+ * Given that the RDTSC and CR4 numbers are reasonably unchanges, it seems that
+ * the Hyper-V core loop (in hvax64.exe) aren't affected.  Our ring-0 runloop
+ * is equally affected as the ring-3 based runloop, so it cannot be ring
+ * switching as such (unless the ring-0 loop is borked and we didn't notice yet).
+ *
+ * The issue is probably in the thread / process switching area, could be
+ * something special for hyper-V interrupt delivery or worker thread switching.
+ *
+ * Really wish this thread ping-pong going on in VID.SYS could be eliminated!
+ *
+ *
+ * @subsubsection subsect_nem_win_benchmarks_bs2t1u2   17763: Bootsector2-test1
+ *
+ * Some preliminary numbers for build 17763 on the 3.4 GHz AMD 1950X, the second
+ * column will improve we get time to have a look the register page.
+ *
+ * There is a  50%  performance loss here compared to the June numbers with
+ * build 17134.  The RDTSC numbers hits that it isn't in the Hyper-V core
+ * (hvax64.exe), but something on the NT side.
+ *
+ * Clearing bit 20 in nt!KiSpeculationFeatures speeds things up (i.e. changing
+ * the dword from 0x00300065 to 0x00200065 in windbg).  This is checked by
+ * nt!KePrepareToDispatchVirtualProcessor, making it a no-op if the flag is
+ * clear.  winhvr!WinHvpVpDispatchLoop call that function before making
+ * hypercall 0xc2, which presumably does the heavy VCpu lifting in hvcax64.exe.
+ *
+ * @verbatim
+TESTING...                                                           WinHv API           Hypercalls + VID  clr(bit-20) + WinHv API
+  32-bit paged protected mode, CPUID                        :           54 145 ins/sec        51 436               130 076
+  real mode, CPUID                                          :           54 178 ins/sec        51 713               130 449
+  [snip]
+  32-bit paged protected mode, RDTSC                        :       98 927 639 ins/sec   100 254 552           100 549 882
+  real mode, RDTSC                                          :       99 601 206 ins/sec   100 886 699           100 470 957
+  [snip]
+  32-bit paged protected mode, 32-bit IN                    :           54 621 ins/sec        51 524               128 294
+  32-bit paged protected mode, 32-bit OUT                   :           54 870 ins/sec        51 671               129 397
+  32-bit paged protected mode, 32-bit IN-to-ring-3          :           54 624 ins/sec        43 964               127 874
+  32-bit paged protected mode, 32-bit OUT-to-ring-3         :           54 803 ins/sec        44 087               129 443
+  [snip]
+  32-bit paged protected mode, 32-bit read                  :           28 230 ins/sec        34 042                48 113
+  32-bit paged protected mode, 32-bit write                 :           27 962 ins/sec        34 050                48 069
+  32-bit paged protected mode, 32-bit read-to-ring-3        :           27 841 ins/sec        28 397                48 146
+  32-bit paged protected mode, 32-bit write-to-ring-3       :           27 896 ins/sec        29 455                47 970
+ * @endverbatim
+ *
+ *
+ * @subsubsection subsect_nem_win_benchmarks_w2k    17134/2018-06-22: Windows 2000 Boot & Shutdown
+ *
+ * Timing the startup and automatic shutdown of a Windows 2000 SP4 guest serves
+ * as a real world benchmark and example of why exit performance is import.  When
+ * Windows 2000 boots up is doing a lot of VGA redrawing of the boot animation,
+ * which is very costly.  Not having installed guest additions leaves it in a VGA
+ * mode after the bootup sequence is done, keep up the screen access expenses,
+ * though the graphics driver more economical than the bootvid code.
+ *
+ * The VM was configured to automatically logon.  A startup script was installed
+ * to perform the automatic shuting down and powering off the VM (thru
+ * vts_shutdown.exe -f -p).  An offline snapshot of the VM was taken an restored
+ * before each test run.  The test time run time is calculated from the monotonic
+ * VBox.log timestamps, starting with the state change to 'RUNNING' and stopping
+ * at 'POWERING_OFF'.
+ *
+ * The host OS and VirtualBox build is the same as for the bootsector2-test1
+ * scenario.
+ *
+ * Results:
+ *
+ *  - WinHv API for all but physical page mappings:
+ *          32 min 12.19 seconds
+ *
+ *  - The default NEM/win configuration where we put the main execution loop
+ *    in ring-0, using hypercalls when we can and VID for managing execution:
+ *          3 min 23.18 seconds
+ *
+ *  - Regular VirtualBox using AMD-V directly, hyper-V is disabled, main
+ *    execution loop in ring-0:
+ *          58.09 seconds
+ *
+ *  - WinHv API with exit history based optimizations:
+ *          58.66 seconds
+ *
+ *  - Hypercall + VID.SYS with exit history base optimizations:
+ *          58.94 seconds
+ *
+ * With a well above average machine needing over half an hour for booting a
+ * nearly 20 year old guest kind of says it all.  The 13%-20% exit performance
+ * increase we get by using hypercalls and VID.SYS directly pays off a lot here.
+ * The 3m23s is almost acceptable in comparison to the half an hour.
+ *
+ * The similarity between the last three results strongly hits at windows 2000
+ * doing a lot of waiting during boot and shutdown and isn't the best testcase
+ * once a basic performance level is reached.
+ *
+ *
+ * @subsubsection subsection_iem_win_benchmarks_deb9_nat    Debian 9 NAT performance
+ *
+ * This benchmark is about network performance over NAT from a 64-bit Debian 9
+ * VM with a single CPU.  For network performance measurements, we use our own
+ * NetPerf tool (ValidationKit/utils/network/NetPerf.cpp) to measure latency
+ * and throughput.
+ *
+ * The setups, builds and configurations are as in the previous benchmarks
+ * (release r123172 on 1950X running 64-bit W10/17134 (2016-06-xx).  Please note
+ * that the exit optimizations hasn't yet been in tuned with NetPerf in mind.
+ *
+ * The NAT network setup was selected here since it's the default one and the
+ * slowest one.  There is quite a bit of IPC with worker threads and packet
+ * processing involved.
+ *
+ * Latency test is first up.  This is a classic back and forth between the two
+ * NetPerf instances, where the key measurement is the roundrip latency.  The
+ * values here are the lowest result over 3-6 runs.
+ *
+ * Against host system:
+ *   - 152 258 ns/roundtrip - 100% - regular VirtualBox SVM
+ *   - 271 059 ns/roundtrip - 178% - Hypercalls + VID.SYS in ring-0 with exit optimizations.
+ *   - 280 149 ns/roundtrip - 184% - Hypercalls + VID.SYS in ring-0
+ *   - 317 735 ns/roundtrip - 209% - Win HV API with exit optimizations.
+ *   - 342 440 ns/roundtrip - 225% - Win HV API
+ *
+ * Against a remote Windows 10 system over a 10Gbps link:
+ *   - 243 969 ns/roundtrip - 100% - regular VirtualBox SVM
+ *   - 384 427 ns/roundtrip - 158% - Win HV API with exit optimizations.
+ *   - 402 411 ns/roundtrip - 165% - Hypercalls + VID.SYS in ring-0
+ *   - 406 313 ns/roundtrip - 167% - Win HV API
+ *   - 413 160 ns/roundtrip - 169% - Hypercalls + VID.SYS in ring-0 with exit optimizations.
+ *
+ * What we see here is:
+ *
+ *   - Consistent and signficant latency increase using Hyper-V compared
+ *     to directly harnessing AMD-V ourselves.
+ *
+ *   - When talking to the host, it's clear that the hypercalls + VID.SYS
+ *     in ring-0 method pays off.
+ *
+ *   - When talking to a different host, the numbers are closer and it
+ *     is not longer clear which Hyper-V execution method is better.
+ *
+ *
+ * Throughput benchmarks are performed by one side pushing data full throttle
+ * for 10 seconds (minus a 1 second at each end of the test), then reversing
+ * the roles and measuring it in the other direction.  The tests ran 3-5 times
+ * and below are the highest and lowest results in each direction.
+ *
+ * Receiving from host system:
+ *   - Regular VirtualBox SVM:
+ *      Max: 96 907 549 bytes/s - 100%
+ *      Min: 86 912 095 bytes/s - 100%
+ *   - Hypercalls + VID.SYS in ring-0:
+ *      Max: 84 036 544 bytes/s - 87%
+ *      Min: 64 978 112 bytes/s - 75%
+ *   - Hypercalls + VID.SYS in ring-0 with exit optimizations:
+ *      Max: 77 760 699 bytes/s - 80%
+ *      Min: 72 677 171 bytes/s - 84%
+ *   - Win HV API with exit optimizations:
+ *      Max: 64 465 905 bytes/s - 67%
+ *      Min: 62 286 369 bytes/s - 72%
+ *   - Win HV API:
+ *      Max: 62 466 631 bytes/s - 64%
+ *      Min: 61 362 782 bytes/s - 70%
+ *
+ * Sending to the host system:
+ *   - Regular VirtualBox SVM:
+ *      Max: 87 728 652 bytes/s - 100%
+ *      Min: 86 923 198 bytes/s - 100%
+ *   - Hypercalls + VID.SYS in ring-0:
+ *      Max: 84 280 749 bytes/s - 96%
+ *      Min: 78 369 842 bytes/s - 90%
+ *   - Hypercalls + VID.SYS in ring-0 with exit optimizations:
+ *      Max: 84 119 932 bytes/s - 96%
+ *      Min: 77 396 811 bytes/s - 89%
+ *   - Win HV API:
+ *      Max: 81 714 377 bytes/s - 93%
+ *      Min: 78 697 419 bytes/s - 91%
+ *   - Win HV API with exit optimizations:
+ *      Max: 80 502 488 bytes/s - 91%
+ *      Min: 71 164 978 bytes/s - 82%
+ *
+ * Receiving from a remote Windows 10 system over a 10Gbps link:
+ *   - Hypercalls + VID.SYS in ring-0:
+ *      Max: 115 346 922 bytes/s - 136%
+ *      Min: 112 912 035 bytes/s - 137%
+ *   - Regular VirtualBox SVM:
+ *      Max:  84 517 504 bytes/s - 100%
+ *      Min:  82 597 049 bytes/s - 100%
+ *   - Hypercalls + VID.SYS in ring-0 with exit optimizations:
+ *      Max:  77 736 251 bytes/s - 92%
+ *      Min:  73 813 784 bytes/s - 89%
+ *   - Win HV API with exit optimizations:
+ *      Max:  63 035 587 bytes/s - 75%
+ *      Min:  57 538 380 bytes/s - 70%
+ *   - Win HV API:
+ *      Max:  62 279 185 bytes/s - 74%
+ *      Min:  56 813 866 bytes/s - 69%
+ *
+ * Sending to a remote Windows 10 system over a 10Gbps link:
+ *   - Win HV API with exit optimizations:
+ *      Max: 116 502 357 bytes/s - 103%
+ *      Min:  49 046 550 bytes/s - 59%
+ *   - Regular VirtualBox SVM:
+ *      Max: 113 030 991 bytes/s - 100%
+ *      Min:  83 059 511 bytes/s - 100%
+ *   - Hypercalls + VID.SYS in ring-0:
+ *      Max: 106 435 031 bytes/s - 94%
+ *      Min:  47 253 510 bytes/s - 57%
+ *   - Hypercalls + VID.SYS in ring-0 with exit optimizations:
+ *      Max:  94 842 287 bytes/s - 84%
+ *      Min:  68 362 172 bytes/s - 82%
+ *   - Win HV API:
+ *      Max:  65 165 225 bytes/s - 58%
+ *      Min:  47 246 573 bytes/s - 57%
+ *
+ * What we see here is:
+ *
+ *   - Again consistent numbers when talking to the host.  Showing that the
+ *     ring-0 approach is preferable to the ring-3 one.
+ *
+ *   - Again when talking to a remote host, things get more difficult to
+ *     make sense of.  The spread is larger and direct AMD-V gets beaten by
+ *     a different the Hyper-V approaches in each direction.
+ *
+ *   - However, if we treat the first entry (remote host) as weird spikes, the
+ *     other entries are consistently worse compared to direct AMD-V.  For the
+ *     send case we get really bad results for WinHV.
+ *
+ */
+
diff --git a/src/VBox/VMM/VMMR3/PDM.cpp b/src/VBox/VMM/VMMR3/PDM.cpp
new file mode 100644
index 00000000..7a762e6a
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDM.cpp
@@ -0,0 +1,3079 @@
+/* $Id: PDM.cpp $ */
+/** @file
+ * PDM - Pluggable Device Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page   pg_pdm      PDM - The Pluggable Device & Driver Manager
+ *
+ * The PDM handles devices and their drivers in a flexible and dynamic manner.
+ *
+ * VirtualBox is designed to be very configurable, i.e. the ability to select
+ * virtual devices and configure them uniquely for a VM.  For this reason
+ * virtual devices are not statically linked with the VMM but loaded, linked and
+ * instantiated at runtime by PDM using the information found in the
+ * Configuration Manager (CFGM).
+ *
+ * While the chief purpose of PDM is to manager of devices their drivers, it
+ * also serves as somewhere to put usful things like cross context queues, cross
+ * context synchronization (like critsect), VM centric thread management,
+ * asynchronous I/O framework, and so on.
+ *
+ * @sa  @ref grp_pdm
+ *      @subpage pg_pdm_block_cache
+ *      @subpage pg_pdm_audio
+ *
+ *
+ * @section sec_pdm_dev     The Pluggable Devices
+ *
+ * Devices register themselves when the module containing them is loaded.  PDM
+ * will call the entry point 'VBoxDevicesRegister' when loading a device module.
+ * The device module will then use the supplied callback table to check the VMM
+ * version and to register its devices.  Each device has an unique name (within
+ * the VM configuration anyway).  The name is not only used in PDM, but also in
+ * CFGM to organize device and device instance settings, and by anyone who wants
+ * to talk to a specific device instance.
+ *
+ * When all device modules have been successfully loaded PDM will instantiate
+ * those devices which are configured for the VM.  Note that a device may have
+ * more than one instance, take network adaptors as an example.  When
+ * instantiating a device PDM provides device instance memory and a callback
+ * table (aka Device Helpers / DevHlp) with the VM APIs which the device
+ * instance is trusted with.
+ *
+ * Some devices are trusted devices, most are not.  The trusted devices are an
+ * integrated part of the VM and can obtain the VM handle, thus enabling them to
+ * call any VM API.  Untrusted devices can only use the callbacks provided
+ * during device instantiation.
+ *
+ * The main purpose in having DevHlps rather than just giving all the devices
+ * the VM handle and let them call the internal VM APIs directly, is both to
+ * create a binary interface that can be supported across releases and to
+ * create a barrier between devices and the VM.  (The trusted / untrusted bit
+ * hasn't turned out to be of much use btw., but it's easy to maintain so there
+ * isn't any point in removing it.)
+ *
+ * A device can provide a ring-0 and/or a raw-mode context extension to improve
+ * the VM performance by handling exits and traps (respectively) without
+ * requiring context switches (to ring-3).  Callbacks for MMIO and I/O ports
+ * need to be registered specifically for the additional contexts for this to
+ * make sense.  Also, the device has to be trusted to be loaded into R0/RC
+ * because of the extra privilege it entails.  Note that raw-mode code and data
+ * will be subject to relocation.
+ *
+ *
+ * @subsection sec_pdm_dev_pci          PCI Devices
+ *
+ * A PDM device usually registers one a PCI device during it's instantiation,
+ * legacy devices may register zero, while a few (currently none) more
+ * complicated devices may register multiple PCI functions or devices.
+ *
+ * The bus, device and function assignments can either be done explictly via the
+ * configuration or the registration call, or it can be left up to the PCI bus.
+ * The typical VBox configuration construct (ConsoleImpl2.cpp) will do explict
+ * assignments for all devices it's BusAssignmentManager class knows about.
+ *
+ * For explict CFGM style configuration, the "PCIBusNo", "PCIDeviceNo", and
+ * "PCIFunctionNo" values in the PDM device instance configuration (not the
+ * "config" subkey, but the top level one) will be picked up for the primary PCI
+ * device.  The primary PCI configuration is by default the first one, but this
+ * can be controlled using the @a idxDevCfg parameter of the
+ * PDMDEVHLPR3::pfnPCIRegister method.  For subsequent configuration (@a
+ * idxDevCfg > 0) the values are taken from the "PciDevNN" subkey, where "NN" is
+ * replaced by the @a idxDevCfg value.
+ *
+ * There's currently a limit of 256 PCI devices per PDM device.
+ *
+ *
+ * @subsection sec_pdm_dev_new          New Style (6.1)
+ *
+ * VBox 6.1 changes the PDM interface for devices and they have to be converted
+ * to the new style to continue working (see @bugref{9218}).
+ *
+ * Steps for converting a PDM device to the new style:
+ *
+ * - State data needs to be split into shared, ring-3, ring-0 and raw-mode
+ *   structures.  The shared structure shall contains absolutely no pointers.
+ *
+ * - Context specific typedefs ending in CC for the structure and pointer to
+ *   it are required (copy & edit the PRTCSTATECC stuff).
+ *   The pointer to a context specific structure is obtained using the
+ *   PDMINS_2_DATA_CC macro.  The PDMINS_2_DATA macro gets the shared one.
+ *
+ * - Update the registration structure with sizeof the new structures.
+ *
+ * - MMIO handlers to FNIOMMMIONEWREAD and FNIOMMMIONEWRITE form, take care renaming
+ *   GCPhys to off and really treat it as an offset.   Return status is VBOXSTRICTRC,
+ *   which should be propagated to worker functions as far as possible.
+ *
+ * - I/O handlers to FNIOMIOPORTNEWIN and FNIOMIOPORTNEWOUT form, take care renaming
+ *   uPort/Port to offPort and really treat it as an offset.   Return status is
+ *   VBOXSTRICTRC, which should be propagated to worker functions as far as possible.
+ *
+ * - MMIO and I/O port registration must be converted, handles stored in the shared structure.
+ *
+ * - PCI devices must also update the I/O region registration and corresponding
+ *   mapping callback.   The latter is generally not needed any more, as the PCI
+ *   bus does the mapping and unmapping using the handle passed to it during registration.
+ *
+ * - If the device contains ring-0 or raw-mode optimizations:
+ *    - Make sure to replace any R0Enabled, GCEnabled, and RZEnabled with
+ *      pDevIns->fR0Enabled and pDevIns->fRCEnabled.  Removing CFGM reading and
+ *      validation of such options as well as state members for them.
+ *    - Callbacks for ring-0 and raw-mode are registered in a context contructor.
+ *      Setting up of non-default critical section handling needs to be repeated
+ *      in the ring-0/raw-mode context constructor too.   See for instance
+ *      e1kRZConstruct().
+ *
+ * - Convert all PDMCritSect calls to PDMDevHlpCritSect.
+ *   Note! pDevIns should be passed as parameter rather than put in pThisCC.
+ *
+ * - Convert all timers to the handle based ones.
+ *
+ * - Convert all queues to the handle based ones or tasks.
+ *
+ * - Set the PDM_DEVREG_FLAGS_NEW_STYLE in the registration structure.
+ *   (Functionally, this only makes a difference for PDMDevHlpSetDeviceCritSect
+ *   behavior, but it will become mandatory once all devices has been
+ *   converted.)
+ *
+ * - Convert all CFGMR3Xxxx calls to pHlp->pfnCFGMXxxx.
+ *
+ * - Convert all SSMR3Xxxx calls to pHlp->pfnSSMXxxx.
+ *
+ * - Ensure that CFGM values and nodes are validated using PDMDEV_VALIDATE_CONFIG_RETURN()
+ *
+ * - Ensure that the first statement in the constructors is
+ *   @code
+           PDMDEV_CHECK_VERSIONS_RETURN(pDevIns);
+     @endcode
+ *   There shall be absolutely nothing preceeding that and it is mandatory.
+ *
+ * - Ensure that the first statement in the destructors is
+ *   @code
+           PDMDEV_CHECK_VERSIONS_RETURN_QUIET(pDevIns);
+     @endcode
+ *   There shall be absolutely nothing preceeding that and it is mandatory.
+ *
+ * - Use 'nm -u' (tools/win.amd64/mingw-w64/r1/bin/nm.exe on windows) to check
+ *   for VBoxVMM and VMMR0 function you forgot to convert to device help calls
+ *   or would need adding as device helpers or something.
+ *
+ *
+ * @section sec_pdm_special_devs    Special Devices
+ *
+ * Several kinds of devices interacts with the VMM and/or other device and PDM
+ * will work like a mediator for these. The typical pattern is that the device
+ * calls a special registration device helper with a set of callbacks, PDM
+ * responds by copying this and providing a pointer to a set helper callbacks
+ * for that particular kind of device. Unlike interfaces where the callback
+ * table pointer is used a 'this' pointer, these arrangements will use the
+ * device instance pointer (PPDMDEVINS) as a kind of 'this' pointer.
+ *
+ * For an example of this kind of setup, see the PIC. The PIC registers itself
+ * by calling PDMDEVHLPR3::pfnPICRegister.  PDM saves the device instance,
+ * copies the callback tables (PDMPICREG), resolving the ring-0 and raw-mode
+ * addresses in the process, and hands back the pointer to a set of helper
+ * methods (PDMPICHLPR3).  The PCI device then queries the ring-0 and raw-mode
+ * helpers using PDMPICHLPR3::pfnGetR0Helpers and PDMPICHLPR3::pfnGetRCHelpers.
+ * The PCI device repeats ths pfnGetRCHelpers call in it's relocation method
+ * since the address changes when RC is relocated.
+ *
+ * @see grp_pdm_device
+ *
+ * @section sec_pdm_usbdev  The Pluggable USB Devices
+ *
+ * USB devices are handled a little bit differently than other devices.  The
+ * general concepts wrt. pluggability are mostly the same, but the details
+ * varies.  The registration entry point is 'VBoxUsbRegister', the device
+ * instance is PDMUSBINS and the callbacks helpers are different.  Also, USB
+ * device are restricted to ring-3 and cannot have any ring-0 or raw-mode
+ * extensions (at least not yet).
+ *
+ * The way USB devices work differs greatly from other devices though since they
+ * aren't attaches directly to the PCI/ISA/whatever system buses but via a
+ * USB host control (OHCI, UHCI or EHCI).  USB devices handle USB requests
+ * (URBs) and does not register I/O ports, MMIO ranges or PCI bus
+ * devices/functions.
+ *
+ * @see grp_pdm_usbdev
+ *
+ *
+ * @section sec_pdm_drv     The Pluggable Drivers
+ *
+ * The VM devices are often accessing host hardware or OS facilities.  For most
+ * devices these facilities can be abstracted in one or more levels.  These
+ * abstractions are called drivers.
+ *
+ * For instance take a DVD/CD drive.  This can be connected to a SCSI
+ * controller, an ATA controller or a SATA controller.  The basics of the DVD/CD
+ * drive implementation remains the same - eject, insert, read, seek, and such.
+ * (For the scsi SCSCI, you might want to speak SCSI directly to, but that can of
+ * course be fixed - see SCSI passthru.)  So, it
+ * makes much sense to have a generic CD/DVD driver which implements this.
+ *
+ * Then the media 'inserted' into the DVD/CD drive can be a ISO image, or it can
+ * be read from a real CD or DVD drive (there are probably other custom formats
+ * someone could desire to read or construct too).  So, it would make sense to
+ * have abstracted interfaces for dealing with this in a generic way so the
+ * cdrom unit doesn't have to implement it all.  Thus we have created the
+ * CDROM/DVD media driver family.
+ *
+ * So, for this example the IDE controller #1 (i.e. secondary) will have
+ * the DVD/CD Driver attached to it's LUN #0 (master).  When a media is mounted
+ * the DVD/CD Driver will have a ISO, HostDVD or RAW (media) Driver attached.
+ *
+ * It is possible to configure many levels of drivers inserting filters, loggers,
+ * or whatever you desire into the chain.  We're using this for network sniffing,
+ * for instance.
+ *
+ * The drivers are loaded in a similar manner to that of a device, namely by
+ * iterating a keyspace in CFGM, load the modules listed there and call
+ * 'VBoxDriversRegister' with a callback table.
+ *
+ * @see grp_pdm_driver
+ *
+ *
+ * @section sec_pdm_ifs     Interfaces
+ *
+ * The pluggable drivers and devices expose one standard interface (callback
+ * table) which is used to construct, destruct, attach, detach,( ++,) and query
+ * other interfaces. A device will query the interfaces required for it's
+ * operation during init and hot-plug.  PDM may query some interfaces during
+ * runtime mounting too.
+ *
+ * An interface here means a function table contained within the device or
+ * driver instance data. Its methods are invoked with the function table pointer
+ * as the first argument and they will calculate the address of the device or
+ * driver instance data from it. (This is one of the aspects which *might* have
+ * been better done in C++.)
+ *
+ * @see grp_pdm_interfaces
+ *
+ *
+ * @section sec_pdm_utils   Utilities
+ *
+ * As mentioned earlier, PDM is the location of any usful constructs that doesn't
+ * quite fit into IPRT. The next subsections will discuss these.
+ *
+ * One thing these APIs all have in common is that resources will be associated
+ * with a device / driver and automatically freed after it has been destroyed if
+ * the destructor didn't do this.
+ *
+ *
+ * @subsection sec_pdm_async_completion     Async I/O
+ *
+ * The PDM Async I/O API provides a somewhat platform agnostic interface for
+ * asynchronous I/O.  For reasons of performance and complexity this does not
+ * build upon any IPRT API.
+ *
+ * @todo more details.
+ *
+ * @see grp_pdm_async_completion
+ *
+ *
+ * @subsection sec_pdm_async_task   Async Task - not implemented
+ *
+ * @todo implement and describe
+ *
+ * @see grp_pdm_async_task
+ *
+ *
+ * @subsection sec_pdm_critsect     Critical Section
+ *
+ * The PDM Critical Section API is currently building on the IPRT API with the
+ * same name.  It adds the possibility to use critical sections in ring-0 and
+ * raw-mode as well as in ring-3.  There are certain restrictions on the RC and
+ * R0 usage though since we're not able to wait on it, nor wake up anyone that
+ * is waiting on it.  These restrictions origins with the use of a ring-3 event
+ * semaphore.  In a later incarnation we plan to replace the ring-3 event
+ * semaphore with a ring-0 one, thus enabling us to wake up waiters while
+ * exectuing in ring-0 and making the hardware assisted execution mode more
+ * efficient. (Raw-mode won't benefit much from this, naturally.)
+ *
+ * @see grp_pdm_critsect
+ *
+ *
+ * @subsection sec_pdm_queue        Queue
+ *
+ * The PDM Queue API is for queuing one or more tasks for later consumption in
+ * ring-3 by EMT, and optionally forcing a delayed or ASAP return to ring-3.  The
+ * queues can also be run on a timer basis as an alternative to the ASAP thing.
+ * The queue will be flushed at forced action time.
+ *
+ * A queue can also be used by another thread (a I/O worker for instance) to
+ * send work / events over to the EMT.
+ *
+ * @see grp_pdm_queue
+ *
+ *
+ * @subsection sec_pdm_task        Task - not implemented yet
+ *
+ * The PDM Task API is for flagging a task for execution at a later point when
+ * we're back in ring-3, optionally forcing the ring-3 return to happen ASAP.
+ * As you can see the concept is similar to queues only simpler.
+ *
+ * A task can also be scheduled by another thread (a I/O worker for instance) as
+ * a mean of getting something done in EMT.
+ *
+ * @see grp_pdm_task
+ *
+ *
+ * @subsection sec_pdm_thread       Thread
+ *
+ * The PDM Thread API is there to help devices and drivers manage their threads
+ * correctly wrt. power on, suspend, resume, power off and destruction.
+ *
+ * The general usage pattern for threads in the employ of devices and drivers is
+ * that they shuffle data or requests while the VM is running and stop doing
+ * this when the VM is paused or powered down. Rogue threads running while the
+ * VM is paused can cause the state to change during saving or have other
+ * unwanted side effects. The PDM Threads API ensures that this won't happen.
+ *
+ * @see grp_pdm_thread
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM
+#define PDMPCIDEV_INCLUDE_PRIVATE  /* Hack to get pdmpcidevint.h included at the right point. */
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/sup.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/ctype.h>
+#include <iprt/ldr.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The PDM saved state version. */
+#define PDM_SAVED_STATE_VERSION               5
+/** Before the PDM audio architecture was introduced there was an "AudioSniffer"
+ *  device which took care of multiplexing input/output audio data from/to various places.
+ *  Thus this device is not needed/used anymore. */
+#define PDM_SAVED_STATE_VERSION_PRE_PDM_AUDIO 4
+#define PDM_SAVED_STATE_VERSION_PRE_NMI_FF    3
+
+/** The number of nanoseconds a suspend callback needs to take before
+ * PDMR3Suspend warns about it taking too long. */
+#define PDMSUSPEND_WARN_AT_NS               UINT64_C(1200000000)
+
+/** The number of nanoseconds a suspend callback needs to take before
+ * PDMR3PowerOff warns about it taking too long. */
+#define PDMPOWEROFF_WARN_AT_NS              UINT64_C( 900000000)
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Statistics of asynchronous notification tasks - used by reset, suspend and
+ * power off.
+ */
+typedef struct PDMNOTIFYASYNCSTATS
+{
+    /** The start timestamp. */
+    uint64_t        uStartNsTs;
+    /** When to log the next time. */
+    uint64_t        cNsElapsedNextLog;
+    /** The loop counter. */
+    uint32_t        cLoops;
+    /** The number of pending asynchronous notification tasks. */
+    uint32_t        cAsync;
+    /** The name of the operation (log prefix). */
+    const char     *pszOp;
+    /** The current list buffer position. */
+    size_t          offList;
+    /** String containing a list of the pending tasks. */
+    char            szList[1024];
+} PDMNOTIFYASYNCSTATS;
+/** Pointer to the stats of pending asynchronous notification tasks. */
+typedef PDMNOTIFYASYNCSTATS *PPDMNOTIFYASYNCSTATS;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(int) pdmR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass);
+static DECLCALLBACK(int) pdmR3SaveExec(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int) pdmR3LoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(int) pdmR3LoadPrep(PVM pVM, PSSMHANDLE pSSM);
+
+static FNDBGFHANDLERINT pdmR3InfoTracingIds;
+
+
+/**
+ * Initializes the PDM part of the UVM.
+ *
+ * This doesn't really do much right now but has to be here for the sake
+ * of completeness.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+VMMR3_INT_DECL(int) PDMR3InitUVM(PUVM pUVM)
+{
+    AssertCompile(sizeof(pUVM->pdm.s) <= sizeof(pUVM->pdm.padding));
+    AssertRelease(sizeof(pUVM->pdm.s) <= sizeof(pUVM->pdm.padding));
+    pUVM->pdm.s.pModules   = NULL;
+    pUVM->pdm.s.pCritSects = NULL;
+    pUVM->pdm.s.pRwCritSects = NULL;
+    return RTCritSectInit(&pUVM->pdm.s.ListCritSect);
+}
+
+
+/**
+ * Initializes the PDM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) PDMR3Init(PVM pVM)
+{
+    LogFlow(("PDMR3Init\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertRelease(!(RT_UOFFSETOF(VM, pdm.s) & 31));
+    AssertRelease(sizeof(pVM->pdm.s) <= sizeof(pVM->pdm.padding));
+    AssertCompileMemberAlignment(PDM, CritSect, sizeof(uintptr_t));
+
+    /*
+     * Init the structure.
+     */
+    pVM->pdm.s.GCPhysVMMDevHeap = NIL_RTGCPHYS;
+    //pVM->pdm.s.idTracingDev = 0;
+    pVM->pdm.s.idTracingOther = 1024;
+
+    /*
+     * Initialize critical sections first.
+     */
+    int rc = pdmR3CritSectBothInitStats(pVM);
+    if (RT_SUCCESS(rc))
+        rc = PDMR3CritSectInit(pVM, &pVM->pdm.s.CritSect, RT_SRC_POS, "PDM");
+    if (RT_SUCCESS(rc))
+    {
+        rc = PDMR3CritSectInit(pVM, &pVM->pdm.s.NopCritSect, RT_SRC_POS, "NOP");
+        if (RT_SUCCESS(rc))
+            pVM->pdm.s.NopCritSect.s.Core.fFlags |= RTCRITSECT_FLAGS_NOP;
+    }
+
+    /*
+     * Initialize sub components.
+     */
+    if (RT_SUCCESS(rc))
+        rc = pdmR3TaskInit(pVM);
+    if (RT_SUCCESS(rc))
+        rc = pdmR3LdrInitU(pVM->pUVM);
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+    if (RT_SUCCESS(rc))
+        rc = pdmR3AsyncCompletionInit(pVM);
+#endif
+#ifdef VBOX_WITH_NETSHAPER
+    if (RT_SUCCESS(rc))
+        rc = pdmR3NetShaperInit(pVM);
+#endif
+    if (RT_SUCCESS(rc))
+        rc = pdmR3BlkCacheInit(pVM);
+    if (RT_SUCCESS(rc))
+        rc = pdmR3DrvInit(pVM);
+    if (RT_SUCCESS(rc))
+        rc = pdmR3DevInit(pVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Register the saved state data unit.
+         */
+        rc = SSMR3RegisterInternal(pVM, "pdm", 1, PDM_SAVED_STATE_VERSION, 128,
+                                   NULL, pdmR3LiveExec, NULL,
+                                   NULL, pdmR3SaveExec, NULL,
+                                   pdmR3LoadPrep, pdmR3LoadExec, NULL);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Register the info handlers.
+             */
+            DBGFR3InfoRegisterInternal(pVM, "pdmtracingids",
+                                       "Displays the tracing IDs assigned by PDM to devices, USB device, drivers and more.",
+                                       pdmR3InfoTracingIds);
+
+            LogFlow(("PDM: Successfully initialized\n"));
+            return rc;
+        }
+    }
+
+    /*
+     * Cleanup and return failure.
+     */
+    PDMR3Term(pVM);
+    LogFlow(("PDMR3Init: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Init phase completed callback.
+ *
+ * We use this for calling PDMDEVREG::pfnInitComplete callback after everything
+ * else has been initialized.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmWhat     The phase that was completed.
+ */
+VMMR3_INT_DECL(int) PDMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    if (enmWhat == VMINITCOMPLETED_RING0)
+        return pdmR3DevInitComplete(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   offDelta    Relocation delta relative to old location.
+ * @remark  The loader subcomponent is relocated by PDMR3LdrRelocate() very
+ *          early in the relocation phase.
+ */
+VMMR3_INT_DECL(void) PDMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    LogFlow(("PDMR3Relocate\n"));
+
+    /*
+     * Queues.
+     */
+    pdmR3QueueRelocate(pVM, offDelta);
+    pVM->pdm.s.pDevHlpQueueRC = PDMQueueRCPtr(pVM->pdm.s.pDevHlpQueueR3);
+
+    /*
+     * Critical sections.
+     */
+    pdmR3CritSectBothRelocate(pVM);
+
+    /*
+     * The registered PIC.
+     */
+    if (pVM->pdm.s.Pic.pDevInsRC)
+    {
+        pVM->pdm.s.Pic.pDevInsRC            += offDelta;
+        pVM->pdm.s.Pic.pfnSetIrqRC          += offDelta;
+        pVM->pdm.s.Pic.pfnGetInterruptRC    += offDelta;
+    }
+
+    /*
+     * The registered APIC.
+     */
+    if (pVM->pdm.s.Apic.pDevInsRC)
+        pVM->pdm.s.Apic.pDevInsRC           += offDelta;
+
+    /*
+     * The registered I/O APIC.
+     */
+    if (pVM->pdm.s.IoApic.pDevInsRC)
+    {
+        pVM->pdm.s.IoApic.pDevInsRC         += offDelta;
+        pVM->pdm.s.IoApic.pfnSetIrqRC       += offDelta;
+        if (pVM->pdm.s.IoApic.pfnSendMsiRC)
+            pVM->pdm.s.IoApic.pfnSendMsiRC      += offDelta;
+        if (pVM->pdm.s.IoApic.pfnSetEoiRC)
+            pVM->pdm.s.IoApic.pfnSetEoiRC       += offDelta;
+    }
+
+    /*
+     * Devices & Drivers.
+     */
+#ifdef VBOX_WITH_RAW_MODE_KEEP /* needs fixing */
+    int rc;
+    PCPDMDEVHLPRC pDevHlpRC = NIL_RTRCPTR;
+    if (VM_IS_RAW_MODE_ENABLED(pVM))
+    {
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_pdmRCDevHlp", &pDevHlpRC);
+        AssertReleaseMsgRC(rc, ("rc=%Rrc when resolving g_pdmRCDevHlp\n", rc));
+    }
+
+    PCPDMDRVHLPRC pDrvHlpRC = NIL_RTRCPTR;
+    if (VM_IS_RAW_MODE_ENABLED(pVM))
+    {
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_pdmRCDevHlp", &pDrvHlpRC);
+        AssertReleaseMsgRC(rc, ("rc=%Rrc when resolving g_pdmRCDevHlp\n", rc));
+    }
+
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC)
+        {
+            pDevIns->pHlpRC             = pDevHlpRC;
+            pDevIns->pvInstanceDataRC   = MMHyperR3ToRC(pVM, pDevIns->pvInstanceDataR3);
+            if (pDevIns->pCritSectRoR3)
+                pDevIns->pCritSectRoRC  = MMHyperR3ToRC(pVM, pDevIns->pCritSectRoR3);
+            pDevIns->Internal.s.pVMRC   = pVM->pVMRC;
+
+            PPDMPCIDEV pPciDev = pDevIns->Internal.s.pHeadPciDevR3;
+            if (pPciDev)
+            {
+                pDevIns->Internal.s.pHeadPciDevRC = MMHyperR3ToRC(pVM, pPciDev);
+                do
+                {
+                    pPciDev->Int.s.pDevInsRC = MMHyperR3ToRC(pVM, pPciDev->Int.s.pDevInsR3);
+                    pPciDev->Int.s.pPdmBusRC = MMHyperR3ToRC(pVM, pPciDev->Int.s.pPdmBusR3);
+                    if (pPciDev->Int.s.pNextR3)
+                        pPciDev->Int.s.pNextRC = MMHyperR3ToRC(pVM, pPciDev->Int.s.pNextR3);
+                    pPciDev = pPciDev->Int.s.pNextR3;
+                } while (pPciDev);
+            }
+
+            if (pDevIns->pReg->pfnRelocate)
+            {
+                LogFlow(("PDMR3Relocate: Relocating device '%s'/%d\n",
+                         pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->pReg->pfnRelocate(pDevIns, offDelta);
+            }
+        }
+
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+        {
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+            {
+                if (pDrvIns->pReg->fFlags & PDM_DRVREG_FLAGS_RC)
+                {
+                    pDrvIns->pHlpRC = pDrvHlpRC;
+                    pDrvIns->pvInstanceDataRC = MMHyperR3ToRC(pVM, pDrvIns->pvInstanceDataR3);
+                    pDrvIns->Internal.s.pVMRC = pVM->pVMRC;
+                    if (pDrvIns->pReg->pfnRelocate)
+                    {
+                        LogFlow(("PDMR3Relocate: Relocating driver '%s'/%u attached to '%s'/%d/%u\n",
+                                 pDrvIns->pReg->szName, pDrvIns->iInstance,
+                                 pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun));
+                        pDrvIns->pReg->pfnRelocate(pDrvIns, offDelta);
+                    }
+                }
+            }
+        }
+
+    }
+#endif
+}
+
+
+/**
+ * Worker for pdmR3Term that terminates a LUN chain.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pLun        The head of the chain.
+ * @param   pszDevice   The name of the device (for logging).
+ * @param   iInstance   The device instance number (for logging).
+ */
+static void pdmR3TermLuns(PVM pVM, PPDMLUN pLun, const char *pszDevice, unsigned iInstance)
+{
+    RT_NOREF2(pszDevice, iInstance);
+
+    for (; pLun; pLun = pLun->pNext)
+    {
+        /*
+         * Destroy them one at a time from the bottom up.
+         * (The serial device/drivers depends on this - bad.)
+         */
+        PPDMDRVINS pDrvIns = pLun->pBottom;
+        pLun->pBottom = pLun->pTop = NULL;
+        while (pDrvIns)
+        {
+            PPDMDRVINS pDrvNext = pDrvIns->Internal.s.pUp;
+
+            if (pDrvIns->pReg->pfnDestruct)
+            {
+                LogFlow(("pdmR3DevTerm: Destroying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, pLun->iLun, pszDevice, iInstance));
+                pDrvIns->pReg->pfnDestruct(pDrvIns);
+            }
+            pDrvIns->Internal.s.pDrv->cInstances--;
+
+            /* Order of resource freeing like in pdmR3DrvDestroyChain, but
+             * not all need to be done as they are done globally later. */
+            //PDMR3QueueDestroyDriver(pVM, pDrvIns);
+            TMR3TimerDestroyDriver(pVM, pDrvIns);
+            SSMR3DeregisterDriver(pVM, pDrvIns, NULL, 0);
+            //pdmR3ThreadDestroyDriver(pVM, pDrvIns);
+            //DBGFR3InfoDeregisterDriver(pVM, pDrvIns, NULL);
+            //pdmR3CritSectBothDeleteDriver(pVM, pDrvIns);
+            //PDMR3BlkCacheReleaseDriver(pVM, pDrvIns);
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+            //pdmR3AsyncCompletionTemplateDestroyDriver(pVM, pDrvIns);
+#endif
+
+            /* Clear the driver struture to catch sloppy code. */
+            ASMMemFill32(pDrvIns, RT_UOFFSETOF_DYN(PDMDRVINS, achInstanceData[pDrvIns->pReg->cbInstance]), 0xdeadd0d0);
+
+            pDrvIns = pDrvNext;
+        }
+    }
+}
+
+
+/**
+ * Terminates the PDM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) PDMR3Term(PVM pVM)
+{
+    LogFlow(("PDMR3Term:\n"));
+    AssertMsg(PDMCritSectIsInitialized(&pVM->pdm.s.CritSect), ("bad init order!\n"));
+
+    /*
+     * Iterate the device instances and attach drivers, doing
+     * relevant destruction processing.
+     *
+     * N.B. There is no need to mess around freeing memory allocated
+     *      from any MM heap since MM will do that in its Term function.
+     */
+    /* usb ones first. */
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        pdmR3TermLuns(pVM, pUsbIns->Internal.s.pLuns, pUsbIns->pReg->szName, pUsbIns->iInstance);
+
+        /*
+         * Detach it from the HUB (if it's actually attached to one) so the HUB has
+         * a chance to stop accessing any data.
+         */
+        PPDMUSBHUB pHub = pUsbIns->Internal.s.pHub;
+        if (pHub)
+        {
+            int rc = pHub->Reg.pfnDetachDevice(pHub->pDrvIns, pUsbIns, pUsbIns->Internal.s.iPort);
+            if (RT_FAILURE(rc))
+            {
+                LogRel(("PDM: Failed to detach USB device '%s' instance %d from %p: %Rrc\n",
+                        pUsbIns->pReg->szName, pUsbIns->iInstance, pHub, rc));
+            }
+            else
+            {
+                pHub->cAvailablePorts++;
+                Assert(pHub->cAvailablePorts > 0 && pHub->cAvailablePorts <= pHub->cPorts);
+                pUsbIns->Internal.s.pHub = NULL;
+            }
+        }
+
+        if (pUsbIns->pReg->pfnDestruct)
+        {
+            LogFlow(("pdmR3DevTerm: Destroying - device '%s'/%d\n",
+                     pUsbIns->pReg->szName, pUsbIns->iInstance));
+            pUsbIns->pReg->pfnDestruct(pUsbIns);
+        }
+
+        //TMR3TimerDestroyUsb(pVM, pUsbIns);
+        //SSMR3DeregisterUsb(pVM, pUsbIns, NULL, 0);
+        pdmR3ThreadDestroyUsb(pVM, pUsbIns);
+    }
+
+    /* then the 'normal' ones. */
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        pdmR3TermLuns(pVM, pDevIns->Internal.s.pLunsR3, pDevIns->pReg->szName, pDevIns->iInstance);
+
+        if (pDevIns->pReg->pfnDestruct)
+        {
+            LogFlow(("pdmR3DevTerm: Destroying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+            pDevIns->pReg->pfnDestruct(pDevIns);
+        }
+
+        if (pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_R0_CONTRUCT)
+        {
+            LogFlow(("pdmR3DevTerm: Destroying (ring-0) - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+            PDMDEVICEGENCALLREQ Req;
+            RT_ZERO(Req.Params);
+            Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+            Req.Hdr.cbReq    = sizeof(Req);
+            Req.enmCall      = PDMDEVICEGENCALL_DESTRUCT;
+            Req.idxR0Device  = pDevIns->Internal.s.idxR0Device;
+            Req.pDevInsR3    = pDevIns;
+            int rc2 = VMMR3CallR0(pVM, VMMR0_DO_PDM_DEVICE_GEN_CALL, 0, &Req.Hdr);
+            AssertRC(rc2);
+        }
+
+        TMR3TimerDestroyDevice(pVM, pDevIns);
+        SSMR3DeregisterDevice(pVM, pDevIns, NULL, 0);
+        pdmR3CritSectBothDeleteDevice(pVM, pDevIns);
+        pdmR3ThreadDestroyDevice(pVM, pDevIns);
+        PDMR3QueueDestroyDevice(pVM, pDevIns);
+        PGMR3PhysMmio2Deregister(pVM, pDevIns, NIL_PGMMMIO2HANDLE);
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+        pdmR3AsyncCompletionTemplateDestroyDevice(pVM, pDevIns);
+#endif
+        DBGFR3InfoDeregisterDevice(pVM, pDevIns, NULL);
+    }
+
+    /*
+     * Destroy all threads.
+     */
+    pdmR3ThreadDestroyAll(pVM);
+
+    /*
+     * Destroy the block cache.
+     */
+    pdmR3BlkCacheTerm(pVM);
+
+#ifdef VBOX_WITH_NETSHAPER
+    /*
+     * Destroy network bandwidth groups.
+     */
+    pdmR3NetShaperTerm(pVM);
+#endif
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+    /*
+     * Free async completion managers.
+     */
+    pdmR3AsyncCompletionTerm(pVM);
+#endif
+
+    /*
+     * Free modules.
+     */
+    pdmR3LdrTermU(pVM->pUVM);
+
+    /*
+     * Stop task threads.
+     */
+    pdmR3TaskTerm(pVM);
+
+    /*
+     * Destroy the PDM lock.
+     */
+    PDMR3CritSectDelete(&pVM->pdm.s.CritSect);
+    /* The MiscCritSect is deleted by PDMR3CritSectBothTerm later. */
+
+    LogFlow(("PDMR3Term: returns %Rrc\n", VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminates the PDM part of the UVM.
+ *
+ * This will unload any modules left behind.
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+VMMR3_INT_DECL(void) PDMR3TermUVM(PUVM pUVM)
+{
+    /*
+     * In the normal cause of events we will now call pdmR3LdrTermU for
+     * the second time. In the case of init failure however, this might
+     * the first time, which is why we do it.
+     */
+    pdmR3LdrTermU(pUVM);
+
+    Assert(pUVM->pdm.s.pCritSects == NULL);
+    Assert(pUVM->pdm.s.pRwCritSects == NULL);
+    RTCritSectDelete(&pUVM->pdm.s.ListCritSect);
+}
+
+
+/**
+ * For APIC assertions.
+ *
+ * @returns true if we've loaded state.
+ * @param   pVM             The cross context VM structure.
+ */
+VMMR3_INT_DECL(bool)    PDMR3HasLoadedState(PVM pVM)
+{
+    return pVM->pdm.s.fStateLoaded;
+}
+
+
+/**
+ * Bits that are saved in pass 0 and in the final pass.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The saved state handle.
+ */
+static void pdmR3SaveBoth(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Save the list of device instances so we can check that they're all still
+     * there when we load the state and that nothing new has been added.
+     */
+    uint32_t i = 0;
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3, i++)
+    {
+        SSMR3PutU32(pSSM, i);
+        SSMR3PutStrZ(pSSM, pDevIns->pReg->szName);
+        SSMR3PutU32(pSSM, pDevIns->iInstance);
+    }
+    SSMR3PutU32(pSSM, UINT32_MAX); /* terminator */
+}
+
+
+/**
+ * Live save.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The saved state handle.
+ * @param   uPass           The pass.
+ */
+static DECLCALLBACK(int) pdmR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    LogFlow(("pdmR3LiveExec:\n"));
+    AssertReturn(uPass == 0, VERR_SSM_UNEXPECTED_PASS);
+    pdmR3SaveBoth(pVM, pSSM);
+    return VINF_SSM_DONT_CALL_AGAIN;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The saved state handle.
+ */
+static DECLCALLBACK(int) pdmR3SaveExec(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("pdmR3SaveExec:\n"));
+
+    /*
+     * Save interrupt and DMA states.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        SSMR3PutU32(pSSM, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC));
+        SSMR3PutU32(pSSM, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC));
+        SSMR3PutU32(pSSM, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI));
+        SSMR3PutU32(pSSM, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI));
+    }
+    SSMR3PutU32(pSSM, VM_FF_IS_SET(pVM, VM_FF_PDM_DMA));
+
+    pdmR3SaveBoth(pVM, pSSM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Prepare state load operation.
+ *
+ * This will dispatch pending operations and clear the FFs governed by PDM and its devices.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pSSM        The SSM handle.
+ */
+static DECLCALLBACK(int) pdmR3LoadPrep(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("pdmR3LoadPrep: %s%s\n",
+             VM_FF_IS_SET(pVM, VM_FF_PDM_QUEUES)     ? " VM_FF_PDM_QUEUES" : "",
+             VM_FF_IS_SET(pVM, VM_FF_PDM_DMA)        ? " VM_FF_PDM_DMA" : ""));
+#ifdef LOG_ENABLED
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        LogFlow(("pdmR3LoadPrep: VCPU %u %s%s\n", idCpu,
+                VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC) ? " VMCPU_FF_INTERRUPT_APIC" : "",
+                VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC)  ? " VMCPU_FF_INTERRUPT_PIC" : ""));
+    }
+#endif
+    NOREF(pSSM);
+
+    /*
+     * In case there is work pending that will raise an interrupt,
+     * start a DMA transfer, or release a lock. (unlikely)
+     */
+    if (VM_FF_IS_SET(pVM, VM_FF_PDM_QUEUES))
+        PDMR3QueueFlushAll(pVM);
+
+    /* Clear the FFs. */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_SMI);
+    }
+    VM_FF_CLEAR(pVM, VM_FF_PDM_DMA);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) pdmR3LoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    int rc;
+
+    LogFlow(("pdmR3LoadExec: uPass=%#x\n", uPass));
+
+    /*
+     * Validate version.
+     */
+    if (    uVersion != PDM_SAVED_STATE_VERSION
+        &&  uVersion != PDM_SAVED_STATE_VERSION_PRE_NMI_FF
+        &&  uVersion != PDM_SAVED_STATE_VERSION_PRE_PDM_AUDIO)
+    {
+        AssertMsgFailed(("Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    if (uPass == SSM_PASS_FINAL)
+    {
+        /*
+         * Load the interrupt and DMA states.
+         *
+         * The APIC, PIC and DMA devices does not restore these, we do.  In the
+         * APIC and PIC cases, it is possible that some devices is incorrectly
+         * setting IRQs during restore.  We'll warn when this happens.  (There
+         * are debug assertions in PDMDevMiscHlp.cpp and APICAll.cpp for
+         * catching the buggy device.)
+         */
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+
+            /* APIC interrupt */
+            uint32_t fInterruptPending = 0;
+            rc = SSMR3GetU32(pSSM, &fInterruptPending);
+            if (RT_FAILURE(rc))
+                return rc;
+            if (fInterruptPending & ~1)
+            {
+                AssertMsgFailed(("fInterruptPending=%#x (APIC)\n", fInterruptPending));
+                return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+            }
+            AssertLogRelMsg(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC),
+                            ("VCPU%03u: VMCPU_FF_INTERRUPT_APIC set! Devices shouldn't set interrupts during state restore...\n", idCpu));
+            if (fInterruptPending)
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC);
+
+            /* PIC interrupt */
+            fInterruptPending = 0;
+            rc = SSMR3GetU32(pSSM, &fInterruptPending);
+            if (RT_FAILURE(rc))
+                return rc;
+            if (fInterruptPending & ~1)
+            {
+                AssertMsgFailed(("fInterruptPending=%#x (PIC)\n", fInterruptPending));
+                return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+            }
+            AssertLogRelMsg(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC),
+                            ("VCPU%03u: VMCPU_FF_INTERRUPT_PIC set!  Devices shouldn't set interrupts during state restore...\n", idCpu));
+            if (fInterruptPending)
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC);
+
+            if (uVersion > PDM_SAVED_STATE_VERSION_PRE_NMI_FF)
+            {
+                /* NMI interrupt */
+                fInterruptPending = 0;
+                rc = SSMR3GetU32(pSSM, &fInterruptPending);
+                if (RT_FAILURE(rc))
+                    return rc;
+                if (fInterruptPending & ~1)
+                {
+                    AssertMsgFailed(("fInterruptPending=%#x (NMI)\n", fInterruptPending));
+                    return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+                }
+                AssertLogRelMsg(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI), ("VCPU%3u: VMCPU_FF_INTERRUPT_NMI set!\n", idCpu));
+                if (fInterruptPending)
+                    VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+
+                /* SMI interrupt */
+                fInterruptPending = 0;
+                rc = SSMR3GetU32(pSSM, &fInterruptPending);
+                if (RT_FAILURE(rc))
+                    return rc;
+                if (fInterruptPending & ~1)
+                {
+                    AssertMsgFailed(("fInterruptPending=%#x (SMI)\n", fInterruptPending));
+                    return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+                }
+                AssertLogRelMsg(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI), ("VCPU%3u: VMCPU_FF_INTERRUPT_SMI set!\n", idCpu));
+                if (fInterruptPending)
+                    VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI);
+            }
+        }
+
+        /* DMA pending */
+        uint32_t fDMAPending = 0;
+        rc = SSMR3GetU32(pSSM, &fDMAPending);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (fDMAPending & ~1)
+        {
+            AssertMsgFailed(("fDMAPending=%#x\n", fDMAPending));
+            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+        }
+        if (fDMAPending)
+            VM_FF_SET(pVM, VM_FF_PDM_DMA);
+        Log(("pdmR3LoadExec: VM_FF_PDM_DMA=%RTbool\n", VM_FF_IS_SET(pVM, VM_FF_PDM_DMA)));
+    }
+
+    /*
+     * Load the list of devices and verify that they are all there.
+     */
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_FOUND;
+
+    for (uint32_t i = 0; ; i++)
+    {
+        /* Get the sequence number / terminator. */
+        uint32_t    u32Sep;
+        rc = SSMR3GetU32(pSSM, &u32Sep);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u32Sep == UINT32_MAX)
+            break;
+        if (u32Sep != i)
+            AssertMsgFailedReturn(("Out of sequence. u32Sep=%#x i=%#x\n", u32Sep, i), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        /* Get the name and instance number. */
+        char szName[RT_SIZEOFMEMB(PDMDEVREG, szName)];
+        rc = SSMR3GetStrZ(pSSM, szName, sizeof(szName));
+        if (RT_FAILURE(rc))
+            return rc;
+        uint32_t iInstance;
+        rc = SSMR3GetU32(pSSM, &iInstance);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* Try locate it. */
+        PPDMDEVINS pDevIns;
+        for (pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+            if (   !RTStrCmp(szName, pDevIns->pReg->szName)
+                && pDevIns->iInstance == iInstance)
+            {
+                AssertLogRelMsgReturn(!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_FOUND),
+                                      ("%s/#%u\n", pDevIns->pReg->szName, pDevIns->iInstance),
+                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+                pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_FOUND;
+                break;
+            }
+
+        if (!pDevIns)
+        {
+            bool fSkip = false;
+
+            /* Skip the non-existing (deprecated) "AudioSniffer" device stored in the saved state. */
+            if (   uVersion <= PDM_SAVED_STATE_VERSION_PRE_PDM_AUDIO
+                && !RTStrCmp(szName, "AudioSniffer"))
+                fSkip = true;
+
+            if (!fSkip)
+            {
+                LogRel(("Device '%s'/%d not found in current config\n", szName, iInstance));
+                if (SSMR3HandleGetAfter(pSSM) != SSMAFTER_DEBUG_IT)
+                    return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Device '%s'/%d not found in current config"), szName, iInstance);
+            }
+        }
+    }
+
+    /*
+     * Check that no additional devices were configured.
+     */
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_FOUND))
+        {
+            LogRel(("Device '%s'/%d not found in the saved state\n", pDevIns->pReg->szName, pDevIns->iInstance));
+            if (SSMR3HandleGetAfter(pSSM) != SSMAFTER_DEBUG_IT)
+                return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Device '%s'/%d not found in the saved state"),
+                                        pDevIns->pReg->szName, pDevIns->iInstance);
+        }
+
+
+    /*
+     * Indicate that we've been called (for assertions).
+     */
+    pVM->pdm.s.fStateLoaded = true;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3PowerOn that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pszDevName          The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(int) pdmR3PowerOnDrv(PPDMDRVINS pDrvIns, const char *pszDevName, uint32_t iDevInstance, uint32_t iLun)
+{
+    Assert(pDrvIns->Internal.s.fVMSuspended);
+    if (pDrvIns->pReg->pfnPowerOn)
+    {
+        LogFlow(("PDMR3PowerOn: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                 pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance));
+        int rc = VINF_SUCCESS; pDrvIns->pReg->pfnPowerOn(pDrvIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3PowerOn: Driver '%s'/%d on LUN#%d of device '%s'/%d -> %Rrc\n",
+                    pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance, rc));
+            return rc;
+        }
+    }
+    pDrvIns->Internal.s.fVMSuspended = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3PowerOn that deals with one USB device instance.
+ *
+ * @returns VBox status code.
+ * @param   pUsbIns             The USB device instance.
+ */
+DECLINLINE(int) pdmR3PowerOnUsb(PPDMUSBINS pUsbIns)
+{
+    Assert(pUsbIns->Internal.s.fVMSuspended);
+    if (pUsbIns->pReg->pfnVMPowerOn)
+    {
+        LogFlow(("PDMR3PowerOn: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+        int rc = VINF_SUCCESS; pUsbIns->pReg->pfnVMPowerOn(pUsbIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3PowerOn: Device '%s'/%d -> %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+            return rc;
+        }
+    }
+    pUsbIns->Internal.s.fVMSuspended = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3PowerOn that deals with one device instance.
+ *
+ * @returns VBox status code.
+ * @param   pDevIns             The device instance.
+ */
+DECLINLINE(int) pdmR3PowerOnDev(PPDMDEVINS pDevIns)
+{
+    Assert(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED);
+    if (pDevIns->pReg->pfnPowerOn)
+    {
+        LogFlow(("PDMR3PowerOn: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+        PDMCritSectEnter(pDevIns->pCritSectRoR3, VERR_IGNORED);
+        int rc = VINF_SUCCESS; pDevIns->pReg->pfnPowerOn(pDevIns);
+        PDMCritSectLeave(pDevIns->pCritSectRoR3);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3PowerOn: Device '%s'/%d -> %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+    }
+    pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * This function will notify all the devices and their
+ * attached drivers about the VM now being powered on.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(void) PDMR3PowerOn(PVM pVM)
+{
+    LogFlow(("PDMR3PowerOn:\n"));
+
+    /*
+     * Iterate thru the device instances and USB device instances,
+     * processing the drivers associated with those.
+     */
+    int rc = VINF_SUCCESS;
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances;  pDevIns && RT_SUCCESS(rc);  pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3;  pLun && RT_SUCCESS(rc);  pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
+                rc = pdmR3PowerOnDrv(pDrvIns, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun);
+        if (RT_SUCCESS(rc))
+            rc = pdmR3PowerOnDev(pDevIns);
+    }
+
+#ifdef VBOX_WITH_USB
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances;  pUsbIns && RT_SUCCESS(rc);  pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns;  pLun && RT_SUCCESS(rc);  pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
+                rc = pdmR3PowerOnDrv(pDrvIns, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun);
+        if (RT_SUCCESS(rc))
+            rc = pdmR3PowerOnUsb(pUsbIns);
+    }
+#endif
+
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+    pdmR3AsyncCompletionResume(pVM);
+#endif
+
+    /*
+     * Resume all threads.
+     */
+    if (RT_SUCCESS(rc))
+        pdmR3ThreadResumeAll(pVM);
+
+    /*
+     * On failure, clean up via PDMR3Suspend.
+     */
+    if (RT_FAILURE(rc))
+        PDMR3Suspend(pVM);
+
+    LogFlow(("PDMR3PowerOn: returns %Rrc\n", rc));
+    return /*rc*/;
+}
+
+
+/**
+ * Initializes the asynchronous notifi stats structure.
+ *
+ * @param   pThis               The asynchronous notifification stats.
+ * @param   pszOp               The name of the operation.
+ */
+static void pdmR3NotifyAsyncInit(PPDMNOTIFYASYNCSTATS pThis, const char *pszOp)
+{
+    pThis->uStartNsTs           = RTTimeNanoTS();
+    pThis->cNsElapsedNextLog    = 0;
+    pThis->cLoops               = 0;
+    pThis->cAsync               = 0;
+    pThis->pszOp                = pszOp;
+    pThis->offList              = 0;
+    pThis->szList[0]            = '\0';
+}
+
+
+/**
+ * Begin a new loop, prepares to gather new stats.
+ *
+ * @param   pThis               The asynchronous notifification stats.
+ */
+static void pdmR3NotifyAsyncBeginLoop(PPDMNOTIFYASYNCSTATS pThis)
+{
+    pThis->cLoops++;
+    pThis->cAsync       = 0;
+    pThis->offList      = 0;
+    pThis->szList[0]    = '\0';
+}
+
+
+/**
+ * Records a device or USB device with a pending asynchronous notification.
+ *
+ * @param   pThis               The asynchronous notifification stats.
+ * @param   pszName             The name of the thing.
+ * @param   iInstance           The instance number.
+ */
+static void pdmR3NotifyAsyncAdd(PPDMNOTIFYASYNCSTATS pThis, const char *pszName, uint32_t iInstance)
+{
+    pThis->cAsync++;
+    if (pThis->offList < sizeof(pThis->szList) - 4)
+        pThis->offList += RTStrPrintf(&pThis->szList[pThis->offList], sizeof(pThis->szList) - pThis->offList,
+                                      pThis->offList == 0 ? "%s/%u" : ", %s/%u",
+                                      pszName, iInstance);
+}
+
+
+/**
+ * Records the asynchronous completition of a reset, suspend or power off.
+ *
+ * @param   pThis               The asynchronous notifification stats.
+ * @param   pszDrvName          The driver name.
+ * @param   iDrvInstance        The driver instance number.
+ * @param   pszDevName          The device or USB device name.
+ * @param   iDevInstance        The device or USB device instance number.
+ * @param   iLun                The LUN.
+ */
+static void pdmR3NotifyAsyncAddDrv(PPDMNOTIFYASYNCSTATS pThis, const char *pszDrvName, uint32_t iDrvInstance,
+                                   const char *pszDevName, uint32_t iDevInstance, uint32_t iLun)
+{
+    pThis->cAsync++;
+    if (pThis->offList < sizeof(pThis->szList) - 8)
+        pThis->offList += RTStrPrintf(&pThis->szList[pThis->offList], sizeof(pThis->szList) - pThis->offList,
+                                      pThis->offList == 0 ? "%s/%u/%u/%s/%u" : ", %s/%u/%u/%s/%u",
+                                      pszDevName, iDevInstance, iLun, pszDrvName, iDrvInstance);
+}
+
+
+/**
+ * Log the stats.
+ *
+ * @param   pThis               The asynchronous notifification stats.
+ */
+static void pdmR3NotifyAsyncLog(PPDMNOTIFYASYNCSTATS pThis)
+{
+    /*
+     * Return if we shouldn't log at this point.
+     * We log with an internval increasing from 0 sec to 60 sec.
+     */
+    if (!pThis->cAsync)
+        return;
+
+    uint64_t cNsElapsed = RTTimeNanoTS() - pThis->uStartNsTs;
+    if (cNsElapsed < pThis->cNsElapsedNextLog)
+        return;
+
+    if (pThis->cNsElapsedNextLog == 0)
+        pThis->cNsElapsedNextLog = RT_NS_1SEC;
+    else if (pThis->cNsElapsedNextLog >= RT_NS_1MIN / 2)
+        pThis->cNsElapsedNextLog = RT_NS_1MIN;
+    else
+        pThis->cNsElapsedNextLog *= 2;
+
+    /*
+     * Do the logging.
+     */
+    LogRel(("%s: after %5llu ms, %u loops: %u async tasks - %s\n",
+            pThis->pszOp, cNsElapsed / RT_NS_1MS, pThis->cLoops, pThis->cAsync, pThis->szList));
+}
+
+
+/**
+ * Wait for events and process pending requests.
+ *
+ * @param   pThis               The asynchronous notifification stats.
+ * @param   pVM                 The cross context VM structure.
+ */
+static void pdmR3NotifyAsyncWaitAndProcessRequests(PPDMNOTIFYASYNCSTATS pThis, PVM pVM)
+{
+    VM_ASSERT_EMT0(pVM);
+    int rc = VMR3AsyncPdmNotificationWaitU(&pVM->pUVM->aCpus[0]);
+    AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc - %s - %s\n", rc, pThis->pszOp, pThis->szList));
+
+    rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY, true /*fPriorityOnly*/);
+    AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc - %s - %s\n", rc, pThis->pszOp, pThis->szList));
+    rc = VMR3ReqProcessU(pVM->pUVM, 0/*idDstCpu*/, true /*fPriorityOnly*/);
+    AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc - %s - %s\n", rc, pThis->pszOp, pThis->szList));
+}
+
+
+/**
+ * Worker for PDMR3Reset that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pAsync              The structure for recording asynchronous
+ *                              notification tasks.
+ * @param   pszDevName          The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(bool) pdmR3ResetDrv(PPDMDRVINS pDrvIns, PPDMNOTIFYASYNCSTATS pAsync,
+                               const char *pszDevName, uint32_t iDevInstance, uint32_t iLun)
+{
+    if (!pDrvIns->Internal.s.fVMReset)
+    {
+        pDrvIns->Internal.s.fVMReset = true;
+        if (pDrvIns->pReg->pfnReset)
+        {
+            if (!pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Reset: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance));
+                pDrvIns->pReg->pfnReset(pDrvIns);
+                if (pDrvIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3Reset: Async notification started - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                             pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance));
+            }
+            else if (pDrvIns->Internal.s.pfnAsyncNotify(pDrvIns))
+            {
+                LogFlow(("PDMR3Reset: Async notification completed - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance));
+                pDrvIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                pDrvIns->Internal.s.fVMReset = false;
+                pdmR3NotifyAsyncAddDrv(pAsync, pDrvIns->Internal.s.pDrv->pReg->szName, pDrvIns->iInstance,
+                                       pszDevName, iDevInstance, iLun);
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+
+/**
+ * Worker for PDMR3Reset that deals with one USB device instance.
+ *
+ * @param   pUsbIns             The USB device instance.
+ * @param   pAsync              The structure for recording asynchronous
+ *                              notification tasks.
+ */
+DECLINLINE(void) pdmR3ResetUsb(PPDMUSBINS pUsbIns, PPDMNOTIFYASYNCSTATS pAsync)
+{
+    if (!pUsbIns->Internal.s.fVMReset)
+    {
+        pUsbIns->Internal.s.fVMReset = true;
+        if (pUsbIns->pReg->pfnVMReset)
+        {
+            if (!pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Reset: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+                pUsbIns->pReg->pfnVMReset(pUsbIns);
+                if (pUsbIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3Reset: Async notification started - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+            }
+            else if (pUsbIns->Internal.s.pfnAsyncNotify(pUsbIns))
+            {
+                LogFlow(("PDMR3Reset: Async notification completed - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+                pUsbIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                pUsbIns->Internal.s.fVMReset = false;
+                pdmR3NotifyAsyncAdd(pAsync, pUsbIns->Internal.s.pUsbDev->pReg->szName, pUsbIns->iInstance);
+            }
+        }
+    }
+}
+
+
+/**
+ * Worker for PDMR3Reset that deals with one device instance.
+ *
+ * @param   pDevIns             The device instance.
+ * @param   pAsync              The structure for recording asynchronous
+ *                              notification tasks.
+ */
+DECLINLINE(void) pdmR3ResetDev(PPDMDEVINS pDevIns, PPDMNOTIFYASYNCSTATS pAsync)
+{
+    if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_RESET))
+    {
+        pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_RESET;
+        if (pDevIns->pReg->pfnReset)
+        {
+            uint64_t cNsElapsed = RTTimeNanoTS();
+            PDMCritSectEnter(pDevIns->pCritSectRoR3, VERR_IGNORED);
+
+            if (!pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Reset: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->pReg->pfnReset(pDevIns);
+                if (pDevIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3Reset: Async notification started - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+            }
+            else if (pDevIns->Internal.s.pfnAsyncNotify(pDevIns))
+            {
+                LogFlow(("PDMR3Reset: Async notification completed - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_RESET;
+                pdmR3NotifyAsyncAdd(pAsync, pDevIns->Internal.s.pDevR3->pReg->szName, pDevIns->iInstance);
+            }
+
+            PDMCritSectLeave(pDevIns->pCritSectRoR3);
+            cNsElapsed = RTTimeNanoTS() - cNsElapsed;
+            if (cNsElapsed >= PDMSUSPEND_WARN_AT_NS)
+                LogRel(("PDMR3Reset: Device '%s'/%d took %'llu ns to reset\n",
+                        pDevIns->pReg->szName, pDevIns->iInstance, cNsElapsed));
+        }
+    }
+}
+
+
+/**
+ * Resets a virtual CPU.
+ *
+ * Used by PDMR3Reset and CPU hot plugging.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMR3_INT_DECL(void) PDMR3ResetCpu(PVMCPU pVCpu)
+{
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_SMI);
+}
+
+
+/**
+ * This function will notify all the devices and their attached drivers about
+ * the VM now being reset.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) PDMR3Reset(PVM pVM)
+{
+    LogFlow(("PDMR3Reset:\n"));
+
+    /*
+     * Clear all the reset flags.
+     */
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_RESET;
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                pDrvIns->Internal.s.fVMReset = false;
+    }
+#ifdef VBOX_WITH_USB
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        pUsbIns->Internal.s.fVMReset = false;
+        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                pDrvIns->Internal.s.fVMReset = false;
+    }
+#endif
+
+    /*
+     * The outer loop repeats until there are no more async requests.
+     */
+    PDMNOTIFYASYNCSTATS     Async;
+    pdmR3NotifyAsyncInit(&Async, "PDMR3Reset");
+    for (;;)
+    {
+        pdmR3NotifyAsyncBeginLoop(&Async);
+
+        /*
+         * Iterate thru the device instances and USB device instances,
+         * processing the drivers associated with those.
+         */
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        {
+            unsigned const cAsyncStart = Async.cAsync;
+
+            if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_RESET_NOTIFICATION)
+                pdmR3ResetDev(pDevIns, &Async);
+
+            if (Async.cAsync == cAsyncStart)
+                for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+                    for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                        if (!pdmR3ResetDrv(pDrvIns, &Async, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun))
+                            break;
+
+            if (   Async.cAsync == cAsyncStart
+                && !(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_RESET_NOTIFICATION))
+                pdmR3ResetDev(pDevIns, &Async);
+        }
+
+#ifdef VBOX_WITH_USB
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        {
+            unsigned const cAsyncStart = Async.cAsync;
+
+            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                    if (!pdmR3ResetDrv(pDrvIns, &Async, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun))
+                        break;
+
+            if (Async.cAsync == cAsyncStart)
+                pdmR3ResetUsb(pUsbIns, &Async);
+        }
+#endif
+        if (!Async.cAsync)
+            break;
+        pdmR3NotifyAsyncLog(&Async);
+        pdmR3NotifyAsyncWaitAndProcessRequests(&Async, pVM);
+    }
+
+    /*
+     * Clear all pending interrupts and DMA operations.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        PDMR3ResetCpu(pVM->apCpusR3[idCpu]);
+    VM_FF_CLEAR(pVM, VM_FF_PDM_DMA);
+
+    LogFlow(("PDMR3Reset: returns void\n"));
+}
+
+
+/**
+ * This function will tell all the devices to setup up their memory structures
+ * after VM construction and after VM reset.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   fAtReset    Indicates the context, after reset if @c true or after
+ *                      construction if @c false.
+ */
+VMMR3_INT_DECL(void) PDMR3MemSetup(PVM pVM, bool fAtReset)
+{
+    LogFlow(("PDMR3MemSetup: fAtReset=%RTbool\n", fAtReset));
+    PDMDEVMEMSETUPCTX const enmCtx = fAtReset ? PDMDEVMEMSETUPCTX_AFTER_RESET : PDMDEVMEMSETUPCTX_AFTER_CONSTRUCTION;
+
+    /*
+     * Iterate thru the device instances and work the callback.
+     */
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        if (pDevIns->pReg->pfnMemSetup)
+        {
+            PDMCritSectEnter(pDevIns->pCritSectRoR3, VERR_IGNORED);
+            pDevIns->pReg->pfnMemSetup(pDevIns, enmCtx);
+            PDMCritSectLeave(pDevIns->pCritSectRoR3);
+        }
+
+    LogFlow(("PDMR3MemSetup: returns void\n"));
+}
+
+
+/**
+ * Retrieves and resets the info left behind by PDMDevHlpVMReset.
+ *
+ * @returns True if hard reset, false if soft reset.
+ * @param   pVM             The cross context VM structure.
+ * @param   fOverride       If non-zero, the override flags will be used instead
+ *                          of the reset flags kept by PDM. (For triple faults.)
+ * @param   pfResetFlags    Where to return the reset flags (PDMVMRESET_F_XXX).
+ * @thread  EMT
+ */
+VMMR3_INT_DECL(bool) PDMR3GetResetInfo(PVM pVM, uint32_t fOverride, uint32_t *pfResetFlags)
+{
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Get the reset flags.
+     */
+    uint32_t fResetFlags;
+    fResetFlags = ASMAtomicXchgU32(&pVM->pdm.s.fResetFlags, 0);
+    if (fOverride)
+        fResetFlags = fOverride;
+    *pfResetFlags = fResetFlags;
+
+    /*
+     * To try avoid trouble, we never ever do soft/warm resets on SMP systems
+     * with more than CPU #0 active.  However, if only one CPU is active we
+     * will ask the firmware what it wants us to do (because the firmware may
+     * depend on the VMM doing a lot of what is normally its responsibility,
+     * like clearing memory).
+     */
+    bool     fOtherCpusActive = false;
+    VMCPUID  idCpu            = pVM->cCpus;
+    while (idCpu-- > 1)
+    {
+        EMSTATE enmState = EMGetState(pVM->apCpusR3[idCpu]);
+        if (   enmState != EMSTATE_WAIT_SIPI
+            && enmState != EMSTATE_NONE)
+        {
+            fOtherCpusActive = true;
+            break;
+        }
+    }
+
+    bool fHardReset = fOtherCpusActive
+                   || (fResetFlags & PDMVMRESET_F_SRC_MASK) < PDMVMRESET_F_LAST_ALWAYS_HARD
+                   || !pVM->pdm.s.pFirmware
+                   || pVM->pdm.s.pFirmware->Reg.pfnIsHardReset(pVM->pdm.s.pFirmware->pDevIns, fResetFlags);
+
+    Log(("PDMR3GetResetInfo: returns fHardReset=%RTbool fResetFlags=%#x\n", fHardReset, fResetFlags));
+    return fHardReset;
+}
+
+
+/**
+ * Performs a soft reset of devices.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fResetFlags     PDMVMRESET_F_XXX.
+ */
+VMMR3_INT_DECL(void) PDMR3SoftReset(PVM pVM, uint32_t fResetFlags)
+{
+    LogFlow(("PDMR3SoftReset: fResetFlags=%#x\n", fResetFlags));
+
+    /*
+     * Iterate thru the device instances and work the callback.
+     */
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        if (pDevIns->pReg->pfnSoftReset)
+        {
+            PDMCritSectEnter(pDevIns->pCritSectRoR3, VERR_IGNORED);
+            pDevIns->pReg->pfnSoftReset(pDevIns, fResetFlags);
+            PDMCritSectLeave(pDevIns->pCritSectRoR3);
+        }
+
+    LogFlow(("PDMR3SoftReset: returns void\n"));
+}
+
+
+/**
+ * Worker for PDMR3Suspend that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pAsync              The structure for recording asynchronous
+ *                              notification tasks.
+ * @param   pszDevName          The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(bool) pdmR3SuspendDrv(PPDMDRVINS pDrvIns, PPDMNOTIFYASYNCSTATS pAsync,
+                                 const char *pszDevName, uint32_t iDevInstance, uint32_t iLun)
+{
+    if (!pDrvIns->Internal.s.fVMSuspended)
+    {
+        pDrvIns->Internal.s.fVMSuspended = true;
+        if (pDrvIns->pReg->pfnSuspend)
+        {
+            uint64_t cNsElapsed = RTTimeNanoTS();
+
+            if (!pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Suspend: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance));
+                pDrvIns->pReg->pfnSuspend(pDrvIns);
+                if (pDrvIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3Suspend: Async notification started - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                             pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance));
+            }
+            else if (pDrvIns->Internal.s.pfnAsyncNotify(pDrvIns))
+            {
+                LogFlow(("PDMR3Suspend: Async notification completed - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance));
+                pDrvIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+
+            cNsElapsed = RTTimeNanoTS() - cNsElapsed;
+            if (cNsElapsed >= PDMSUSPEND_WARN_AT_NS)
+                LogRel(("PDMR3Suspend: Driver '%s'/%d on LUN#%d of device '%s'/%d took %'llu ns to suspend\n",
+                        pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance, cNsElapsed));
+
+            if (pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                pDrvIns->Internal.s.fVMSuspended = false;
+                pdmR3NotifyAsyncAddDrv(pAsync, pDrvIns->Internal.s.pDrv->pReg->szName, pDrvIns->iInstance, pszDevName, iDevInstance, iLun);
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+
+/**
+ * Worker for PDMR3Suspend that deals with one USB device instance.
+ *
+ * @param   pUsbIns             The USB device instance.
+ * @param   pAsync              The structure for recording asynchronous
+ *                              notification tasks.
+ */
+DECLINLINE(void) pdmR3SuspendUsb(PPDMUSBINS pUsbIns, PPDMNOTIFYASYNCSTATS pAsync)
+{
+    if (!pUsbIns->Internal.s.fVMSuspended)
+    {
+        pUsbIns->Internal.s.fVMSuspended = true;
+        if (pUsbIns->pReg->pfnVMSuspend)
+        {
+            uint64_t cNsElapsed = RTTimeNanoTS();
+
+            if (!pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Suspend: Notifying - USB device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+                pUsbIns->pReg->pfnVMSuspend(pUsbIns);
+                if (pUsbIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3Suspend: Async notification started - USB device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+            }
+            else if (pUsbIns->Internal.s.pfnAsyncNotify(pUsbIns))
+            {
+                LogFlow(("PDMR3Suspend: Async notification completed - USB device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+                pUsbIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                pUsbIns->Internal.s.fVMSuspended = false;
+                pdmR3NotifyAsyncAdd(pAsync, pUsbIns->Internal.s.pUsbDev->pReg->szName, pUsbIns->iInstance);
+            }
+
+            cNsElapsed = RTTimeNanoTS() - cNsElapsed;
+            if (cNsElapsed >= PDMSUSPEND_WARN_AT_NS)
+                LogRel(("PDMR3Suspend: USB device '%s'/%d took %'llu ns to suspend\n",
+                        pUsbIns->pReg->szName, pUsbIns->iInstance, cNsElapsed));
+        }
+    }
+}
+
+
+/**
+ * Worker for PDMR3Suspend that deals with one device instance.
+ *
+ * @param   pDevIns             The device instance.
+ * @param   pAsync              The structure for recording asynchronous
+ *                              notification tasks.
+ */
+DECLINLINE(void) pdmR3SuspendDev(PPDMDEVINS pDevIns, PPDMNOTIFYASYNCSTATS pAsync)
+{
+    if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED))
+    {
+        pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_SUSPENDED;
+        if (pDevIns->pReg->pfnSuspend)
+        {
+            uint64_t cNsElapsed = RTTimeNanoTS();
+            PDMCritSectEnter(pDevIns->pCritSectRoR3, VERR_IGNORED);
+
+            if (!pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Suspend: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->pReg->pfnSuspend(pDevIns);
+                if (pDevIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3Suspend: Async notification started - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+            }
+            else if (pDevIns->Internal.s.pfnAsyncNotify(pDevIns))
+            {
+                LogFlow(("PDMR3Suspend: Async notification completed - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+                pdmR3NotifyAsyncAdd(pAsync, pDevIns->Internal.s.pDevR3->pReg->szName, pDevIns->iInstance);
+            }
+
+            PDMCritSectLeave(pDevIns->pCritSectRoR3);
+            cNsElapsed = RTTimeNanoTS() - cNsElapsed;
+            if (cNsElapsed >= PDMSUSPEND_WARN_AT_NS)
+                LogRel(("PDMR3Suspend: Device '%s'/%d took %'llu ns to suspend\n",
+                        pDevIns->pReg->szName, pDevIns->iInstance, cNsElapsed));
+        }
+    }
+}
+
+
+/**
+ * This function will notify all the devices and their attached drivers about
+ * the VM now being suspended.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @thread  EMT(0)
+ */
+VMMR3_INT_DECL(void) PDMR3Suspend(PVM pVM)
+{
+    LogFlow(("PDMR3Suspend:\n"));
+    VM_ASSERT_EMT0(pVM);
+    uint64_t cNsElapsed = RTTimeNanoTS();
+
+    /*
+     * The outer loop repeats until there are no more async requests.
+     *
+     * Note! We depend on the suspended indicators to be in the desired state
+     *       and we do not reset them before starting because this allows
+     *       PDMR3PowerOn and PDMR3Resume to use PDMR3Suspend for cleaning up
+     *       on failure.
+     */
+    PDMNOTIFYASYNCSTATS Async;
+    pdmR3NotifyAsyncInit(&Async, "PDMR3Suspend");
+    for (;;)
+    {
+        pdmR3NotifyAsyncBeginLoop(&Async);
+
+        /*
+         * Iterate thru the device instances and USB device instances,
+         * processing the drivers associated with those.
+         *
+         * The attached drivers are normally processed first.  Some devices
+         * (like DevAHCI) though needs to be notified before the drivers so
+         * that it doesn't kick off any new requests after the drivers stopped
+         * taking any. (DrvVD changes to read-only in this particular case.)
+         */
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        {
+            unsigned const cAsyncStart = Async.cAsync;
+
+            if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_SUSPEND_NOTIFICATION)
+                pdmR3SuspendDev(pDevIns, &Async);
+
+            if (Async.cAsync == cAsyncStart)
+                for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+                    for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                        if (!pdmR3SuspendDrv(pDrvIns, &Async, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun))
+                            break;
+
+            if (    Async.cAsync == cAsyncStart
+                && !(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_SUSPEND_NOTIFICATION))
+                pdmR3SuspendDev(pDevIns, &Async);
+        }
+
+#ifdef VBOX_WITH_USB
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        {
+            unsigned const cAsyncStart = Async.cAsync;
+
+            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                    if (!pdmR3SuspendDrv(pDrvIns, &Async, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun))
+                        break;
+
+            if (Async.cAsync == cAsyncStart)
+                pdmR3SuspendUsb(pUsbIns, &Async);
+        }
+#endif
+        if (!Async.cAsync)
+            break;
+        pdmR3NotifyAsyncLog(&Async);
+        pdmR3NotifyAsyncWaitAndProcessRequests(&Async, pVM);
+    }
+
+    /*
+     * Suspend all threads.
+     */
+    pdmR3ThreadSuspendAll(pVM);
+
+    cNsElapsed = RTTimeNanoTS() - cNsElapsed;
+    LogRel(("PDMR3Suspend: %'llu ns run time\n", cNsElapsed));
+}
+
+
+/**
+ * Worker for PDMR3Resume that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pszDevName          The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(int) pdmR3ResumeDrv(PPDMDRVINS pDrvIns, const char *pszDevName, uint32_t iDevInstance, uint32_t iLun)
+{
+    Assert(pDrvIns->Internal.s.fVMSuspended);
+    if (pDrvIns->pReg->pfnResume)
+    {
+        LogFlow(("PDMR3Resume: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                 pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance));
+        int rc = VINF_SUCCESS; pDrvIns->pReg->pfnResume(pDrvIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3Resume: Driver '%s'/%d on LUN#%d of device '%s'/%d -> %Rrc\n",
+                    pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance, rc));
+            return rc;
+        }
+    }
+    pDrvIns->Internal.s.fVMSuspended = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3Resume that deals with one USB device instance.
+ *
+ * @returns VBox status code.
+ * @param   pUsbIns             The USB device instance.
+ */
+DECLINLINE(int) pdmR3ResumeUsb(PPDMUSBINS pUsbIns)
+{
+    Assert(pUsbIns->Internal.s.fVMSuspended);
+    if (pUsbIns->pReg->pfnVMResume)
+    {
+        LogFlow(("PDMR3Resume: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+        int rc = VINF_SUCCESS; pUsbIns->pReg->pfnVMResume(pUsbIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3Resume: Device '%s'/%d -> %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+            return rc;
+        }
+    }
+    pUsbIns->Internal.s.fVMSuspended = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3Resume that deals with one device instance.
+ *
+ * @returns VBox status code.
+ * @param   pDevIns             The device instance.
+ */
+DECLINLINE(int) pdmR3ResumeDev(PPDMDEVINS pDevIns)
+{
+    Assert(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED);
+    if (pDevIns->pReg->pfnResume)
+    {
+        LogFlow(("PDMR3Resume: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+        PDMCritSectEnter(pDevIns->pCritSectRoR3, VERR_IGNORED);
+        int rc = VINF_SUCCESS; pDevIns->pReg->pfnResume(pDevIns);
+        PDMCritSectLeave(pDevIns->pCritSectRoR3);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3Resume: Device '%s'/%d -> %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+    }
+    pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * This function will notify all the devices and their
+ * attached drivers about the VM now being resumed.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) PDMR3Resume(PVM pVM)
+{
+    LogFlow(("PDMR3Resume:\n"));
+
+    /*
+     * Iterate thru the device instances and USB device instances,
+     * processing the drivers associated with those.
+     */
+    int rc = VINF_SUCCESS;
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances;  pDevIns && RT_SUCCESS(rc);  pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3;  pLun && RT_SUCCESS(rc);  pLun    = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
+                rc = pdmR3ResumeDrv(pDrvIns, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun);
+        if (RT_SUCCESS(rc))
+            rc = pdmR3ResumeDev(pDevIns);
+    }
+
+#ifdef VBOX_WITH_USB
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances;  pUsbIns && RT_SUCCESS(rc);  pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns;  pLun && RT_SUCCESS(rc);  pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
+                rc = pdmR3ResumeDrv(pDrvIns, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun);
+        if (RT_SUCCESS(rc))
+            rc = pdmR3ResumeUsb(pUsbIns);
+    }
+#endif
+
+    /*
+     * Resume all threads.
+     */
+    if (RT_SUCCESS(rc))
+        pdmR3ThreadResumeAll(pVM);
+
+    /*
+     * Resume the block cache.
+     */
+    if (RT_SUCCESS(rc))
+        pdmR3BlkCacheResume(pVM);
+
+    /*
+     * On failure, clean up via PDMR3Suspend.
+     */
+    if (RT_FAILURE(rc))
+        PDMR3Suspend(pVM);
+
+    LogFlow(("PDMR3Resume: returns %Rrc\n", rc));
+    return /*rc*/;
+}
+
+
+/**
+ * Worker for PDMR3PowerOff that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pAsync              The structure for recording asynchronous
+ *                              notification tasks.
+ * @param   pszDevName          The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(bool) pdmR3PowerOffDrv(PPDMDRVINS pDrvIns, PPDMNOTIFYASYNCSTATS pAsync,
+                                  const char *pszDevName, uint32_t iDevInstance, uint32_t iLun)
+{
+    if (!pDrvIns->Internal.s.fVMSuspended)
+    {
+        pDrvIns->Internal.s.fVMSuspended = true;
+        if (pDrvIns->pReg->pfnPowerOff)
+        {
+            uint64_t cNsElapsed = RTTimeNanoTS();
+
+            if (!pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3PowerOff: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance));
+                pDrvIns->pReg->pfnPowerOff(pDrvIns);
+                if (pDrvIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3PowerOff: Async notification started - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                             pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance));
+            }
+            else if (pDrvIns->Internal.s.pfnAsyncNotify(pDrvIns))
+            {
+                LogFlow(("PDMR3PowerOff: Async notification completed - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance));
+                pDrvIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+
+            cNsElapsed = RTTimeNanoTS() - cNsElapsed;
+            if (cNsElapsed >= PDMPOWEROFF_WARN_AT_NS)
+                LogRel(("PDMR3PowerOff: Driver '%s'/%d on LUN#%d of device '%s'/%d took %'llu ns to power off\n",
+                        pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDevName, iDevInstance, cNsElapsed));
+
+            if (pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                pDrvIns->Internal.s.fVMSuspended = false;
+                pdmR3NotifyAsyncAddDrv(pAsync, pDrvIns->Internal.s.pDrv->pReg->szName, pDrvIns->iInstance,
+                                       pszDevName, iDevInstance, iLun);
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+
+/**
+ * Worker for PDMR3PowerOff that deals with one USB device instance.
+ *
+ * @param   pUsbIns             The USB device instance.
+ * @param   pAsync              The structure for recording asynchronous
+ *                              notification tasks.
+ */
+DECLINLINE(void) pdmR3PowerOffUsb(PPDMUSBINS pUsbIns, PPDMNOTIFYASYNCSTATS pAsync)
+{
+    if (!pUsbIns->Internal.s.fVMSuspended)
+    {
+        pUsbIns->Internal.s.fVMSuspended = true;
+        if (pUsbIns->pReg->pfnVMPowerOff)
+        {
+            uint64_t cNsElapsed = RTTimeNanoTS();
+
+            if (!pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3PowerOff: Notifying - USB device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+                pUsbIns->pReg->pfnVMPowerOff(pUsbIns);
+                if (pUsbIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3PowerOff: Async notification started - USB device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+            }
+            else if (pUsbIns->Internal.s.pfnAsyncNotify(pUsbIns))
+            {
+                LogFlow(("PDMR3PowerOff: Async notification completed - USB device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+                pUsbIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                pUsbIns->Internal.s.fVMSuspended = false;
+                pdmR3NotifyAsyncAdd(pAsync, pUsbIns->Internal.s.pUsbDev->pReg->szName, pUsbIns->iInstance);
+            }
+
+            cNsElapsed = RTTimeNanoTS() - cNsElapsed;
+            if (cNsElapsed >= PDMPOWEROFF_WARN_AT_NS)
+                LogRel(("PDMR3PowerOff: USB device '%s'/%d took %'llu ns to power off\n",
+                        pUsbIns->pReg->szName, pUsbIns->iInstance, cNsElapsed));
+
+        }
+    }
+}
+
+
+/**
+ * Worker for PDMR3PowerOff that deals with one device instance.
+ *
+ * @param   pDevIns             The device instance.
+ * @param   pAsync              The structure for recording asynchronous
+ *                              notification tasks.
+ */
+DECLINLINE(void) pdmR3PowerOffDev(PPDMDEVINS pDevIns, PPDMNOTIFYASYNCSTATS pAsync)
+{
+    if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED))
+    {
+        pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_SUSPENDED;
+        if (pDevIns->pReg->pfnPowerOff)
+        {
+            uint64_t cNsElapsed = RTTimeNanoTS();
+            PDMCritSectEnter(pDevIns->pCritSectRoR3, VERR_IGNORED);
+
+            if (!pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3PowerOff: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->pReg->pfnPowerOff(pDevIns);
+                if (pDevIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3PowerOff: Async notification started - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+            }
+            else if (pDevIns->Internal.s.pfnAsyncNotify(pDevIns))
+            {
+                LogFlow(("PDMR3PowerOff: Async notification completed - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+                pdmR3NotifyAsyncAdd(pAsync, pDevIns->Internal.s.pDevR3->pReg->szName, pDevIns->iInstance);
+            }
+
+            PDMCritSectLeave(pDevIns->pCritSectRoR3);
+            cNsElapsed = RTTimeNanoTS() - cNsElapsed;
+            if (cNsElapsed >= PDMPOWEROFF_WARN_AT_NS)
+                LogFlow(("PDMR3PowerOff: Device '%s'/%d took %'llu ns to power off\n",
+                         pDevIns->pReg->szName, pDevIns->iInstance, cNsElapsed));
+        }
+    }
+}
+
+
+/**
+ * This function will notify all the devices and their
+ * attached drivers about the VM being powered off.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(void) PDMR3PowerOff(PVM pVM)
+{
+    LogFlow(("PDMR3PowerOff:\n"));
+    uint64_t cNsElapsed = RTTimeNanoTS();
+
+    /*
+     * Clear the suspended flags on all devices and drivers first because they
+     * might have been set during a suspend but the power off callbacks should
+     * be called in any case.
+     */
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                pDrvIns->Internal.s.fVMSuspended = false;
+    }
+
+#ifdef VBOX_WITH_USB
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        pUsbIns->Internal.s.fVMSuspended = false;
+
+        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                pDrvIns->Internal.s.fVMSuspended = false;
+    }
+#endif
+
+    /*
+     * The outer loop repeats until there are no more async requests.
+     */
+    PDMNOTIFYASYNCSTATS Async;
+    pdmR3NotifyAsyncInit(&Async, "PDMR3PowerOff");
+    for (;;)
+    {
+        pdmR3NotifyAsyncBeginLoop(&Async);
+
+        /*
+         * Iterate thru the device instances and USB device instances,
+         * processing the drivers associated with those.
+         *
+         * The attached drivers are normally processed first.  Some devices
+         * (like DevAHCI) though needs to be notified before the drivers so
+         * that it doesn't kick off any new requests after the drivers stopped
+         * taking any. (DrvVD changes to read-only in this particular case.)
+         */
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        {
+            unsigned const cAsyncStart = Async.cAsync;
+
+            if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_POWEROFF_NOTIFICATION)
+                pdmR3PowerOffDev(pDevIns, &Async);
+
+            if (Async.cAsync == cAsyncStart)
+                for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+                    for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                        if (!pdmR3PowerOffDrv(pDrvIns, &Async, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun))
+                            break;
+
+            if (    Async.cAsync == cAsyncStart
+                && !(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_POWEROFF_NOTIFICATION))
+                pdmR3PowerOffDev(pDevIns, &Async);
+        }
+
+#ifdef VBOX_WITH_USB
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        {
+            unsigned const cAsyncStart = Async.cAsync;
+
+            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                    if (!pdmR3PowerOffDrv(pDrvIns, &Async, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun))
+                        break;
+
+            if (Async.cAsync == cAsyncStart)
+                pdmR3PowerOffUsb(pUsbIns, &Async);
+        }
+#endif
+        if (!Async.cAsync)
+            break;
+        pdmR3NotifyAsyncLog(&Async);
+        pdmR3NotifyAsyncWaitAndProcessRequests(&Async, pVM);
+    }
+
+    /*
+     * Suspend all threads.
+     */
+    pdmR3ThreadSuspendAll(pVM);
+
+    cNsElapsed = RTTimeNanoTS() - cNsElapsed;
+    LogRel(("PDMR3PowerOff: %'llu ns run time\n", cNsElapsed));
+}
+
+
+/**
+ * Queries the base interface of a device instance.
+ *
+ * The caller can use this to query other interfaces the device implements
+ * and use them to talk to the device.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   ppBase          Where to store the pointer to the base device interface on success.
+ * @remark  We're not doing any locking ATM, so don't try call this at times when the
+ *          device chain is known to be updated.
+ */
+VMMR3DECL(int) PDMR3QueryDevice(PUVM pUVM, const char *pszDevice, unsigned iInstance, PPDMIBASE *ppBase)
+{
+    LogFlow(("PDMR3DeviceQuery: pszDevice=%p:{%s} iInstance=%u ppBase=%p\n", pszDevice, pszDevice, iInstance, ppBase));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Iterate registered devices looking for the device.
+     */
+    size_t cchDevice = strlen(pszDevice);
+    for (PPDMDEV pDev = pUVM->pVM->pdm.s.pDevs; pDev; pDev = pDev->pNext)
+    {
+        if (    pDev->cchName == cchDevice
+            &&  !memcmp(pDev->pReg->szName, pszDevice, cchDevice))
+        {
+            /*
+             * Iterate device instances.
+             */
+            for (PPDMDEVINS pDevIns = pDev->pInstances; pDevIns; pDevIns = pDevIns->Internal.s.pPerDeviceNextR3)
+            {
+                if (pDevIns->iInstance == iInstance)
+                {
+                    if (pDevIns->IBase.pfnQueryInterface)
+                    {
+                        *ppBase = &pDevIns->IBase;
+                        LogFlow(("PDMR3DeviceQuery: return VINF_SUCCESS and *ppBase=%p\n", *ppBase));
+                        return VINF_SUCCESS;
+                    }
+
+                    LogFlow(("PDMR3DeviceQuery: returns VERR_PDM_DEVICE_INSTANCE_NO_IBASE\n"));
+                    return VERR_PDM_DEVICE_INSTANCE_NO_IBASE;
+                }
+            }
+
+            LogFlow(("PDMR3DeviceQuery: returns VERR_PDM_DEVICE_INSTANCE_NOT_FOUND\n"));
+            return VERR_PDM_DEVICE_INSTANCE_NOT_FOUND;
+        }
+    }
+
+    LogFlow(("PDMR3QueryDevice: returns VERR_PDM_DEVICE_NOT_FOUND\n"));
+    return VERR_PDM_DEVICE_NOT_FOUND;
+}
+
+
+/**
+ * Queries the base interface of a device LUN.
+ *
+ * This differs from PDMR3QueryLun by that it returns the interface on the
+ * device and not the top level driver.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   ppBase          Where to store the base interface pointer.
+ * @remark  We're not doing any locking ATM, so don't try call this at times when the
+ *          device chain is known to be updated.
+ */
+VMMR3DECL(int) PDMR3QueryDeviceLun(PUVM pUVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPDMIBASE *ppBase)
+{
+    LogFlow(("PDMR3QueryDeviceLun: pszDevice=%p:{%s} iInstance=%u iLun=%u ppBase=%p\n",
+             pszDevice, pszDevice, iInstance, iLun, ppBase));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Find the LUN.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3DevFindLun(pUVM->pVM, pszDevice, iInstance, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        *ppBase = pLun->pBase;
+        LogFlow(("PDMR3QueryDeviceLun: return VINF_SUCCESS and *ppBase=%p\n", *ppBase));
+        return VINF_SUCCESS;
+    }
+    LogFlow(("PDMR3QueryDeviceLun: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Query the interface of the top level driver on a LUN.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   ppBase          Where to store the base interface pointer.
+ * @remark  We're not doing any locking ATM, so don't try call this at times when the
+ *          device chain is known to be updated.
+ */
+VMMR3DECL(int) PDMR3QueryLun(PUVM pUVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPDMIBASE *ppBase)
+{
+    LogFlow(("PDMR3QueryLun: pszDevice=%p:{%s} iInstance=%u iLun=%u ppBase=%p\n",
+             pszDevice, pszDevice, iInstance, iLun, ppBase));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Find the LUN.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLun->pTop)
+        {
+            *ppBase = &pLun->pTop->IBase;
+            LogFlow(("PDMR3QueryLun: return %Rrc and *ppBase=%p\n", VINF_SUCCESS, *ppBase));
+            return VINF_SUCCESS;
+        }
+        rc = VERR_PDM_NO_DRIVER_ATTACHED_TO_LUN;
+    }
+    LogFlow(("PDMR3QueryLun: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Query the interface of a named driver on a LUN.
+ *
+ * If the driver appears more than once in the driver chain, the first instance
+ * is returned.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   pszDriver       The driver name.
+ * @param   ppBase          Where to store the base interface pointer.
+ *
+ * @remark  We're not doing any locking ATM, so don't try call this at times when the
+ *          device chain is known to be updated.
+ */
+VMMR3DECL(int) PDMR3QueryDriverOnLun(PUVM pUVM, const char *pszDevice, unsigned iInstance, unsigned iLun, const char *pszDriver, PPPDMIBASE ppBase)
+{
+    LogFlow(("PDMR3QueryDriverOnLun: pszDevice=%p:{%s} iInstance=%u iLun=%u pszDriver=%p:{%s} ppBase=%p\n",
+             pszDevice, pszDevice, iInstance, iLun, pszDriver, pszDriver, ppBase));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Find the LUN.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3DevFindLun(pUVM->pVM, pszDevice, iInstance, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLun->pTop)
+        {
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                if (!strcmp(pDrvIns->pReg->szName, pszDriver))
+                {
+                    *ppBase = &pDrvIns->IBase;
+                    LogFlow(("PDMR3QueryDriverOnLun: return %Rrc and *ppBase=%p\n", VINF_SUCCESS, *ppBase));
+                    return VINF_SUCCESS;
+
+                }
+            rc = VERR_PDM_DRIVER_NOT_FOUND;
+        }
+        else
+            rc = VERR_PDM_NO_DRIVER_ATTACHED_TO_LUN;
+    }
+    LogFlow(("PDMR3QueryDriverOnLun: returns %Rrc\n", rc));
+    return rc;
+}
+
+/**
+ * Executes pending DMA transfers.
+ * Forced Action handler.
+ *
+ * @param   pVM             The cross context VM structure.
+ */
+VMMR3DECL(void) PDMR3DmaRun(PVM pVM)
+{
+    /* Note! Not really SMP safe; restrict it to VCPU 0. */
+    if (VMMGetCpuId(pVM) != 0)
+        return;
+
+    if (VM_FF_TEST_AND_CLEAR(pVM, VM_FF_PDM_DMA))
+    {
+        if (pVM->pdm.s.pDmac)
+        {
+            bool fMore = pVM->pdm.s.pDmac->Reg.pfnRun(pVM->pdm.s.pDmac->pDevIns);
+            if (fMore)
+                VM_FF_SET(pVM, VM_FF_PDM_DMA);
+        }
+    }
+}
+
+
+/**
+ * Service a VMMCALLRING3_PDM_LOCK call.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) PDMR3LockCall(PVM pVM)
+{
+    return PDMR3CritSectEnterEx(&pVM->pdm.s.CritSect, true /* fHostCall */);
+}
+
+
+/**
+ * Allocates memory from the VMM device heap.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   cbSize          Allocation size.
+ * @param   pfnNotify       Mapping/unmapping notification callback.
+ * @param   ppv             Ring-3 pointer. (out)
+ */
+VMMR3_INT_DECL(int) PDMR3VmmDevHeapAlloc(PVM pVM, size_t cbSize, PFNPDMVMMDEVHEAPNOTIFY pfnNotify, RTR3PTR *ppv)
+{
+#ifdef DEBUG_bird
+    if (!cbSize || cbSize > pVM->pdm.s.cbVMMDevHeapLeft)
+        return VERR_NO_MEMORY;
+#else
+    AssertReturn(cbSize && cbSize <= pVM->pdm.s.cbVMMDevHeapLeft, VERR_NO_MEMORY);
+#endif
+
+    Log(("PDMR3VMMDevHeapAlloc: %#zx\n", cbSize));
+
+    /** @todo Not a real heap as there's currently only one user. */
+    *ppv = pVM->pdm.s.pvVMMDevHeap;
+    pVM->pdm.s.cbVMMDevHeapLeft = 0;
+    pVM->pdm.s.pfnVMMDevHeapNotify = pfnNotify;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Frees memory from the VMM device heap
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pv              Ring-3 pointer.
+ */
+VMMR3_INT_DECL(int) PDMR3VmmDevHeapFree(PVM pVM, RTR3PTR pv)
+{
+    Log(("PDMR3VmmDevHeapFree: %RHv\n", pv)); RT_NOREF_PV(pv);
+
+    /** @todo not a real heap as there's currently only one user. */
+    pVM->pdm.s.cbVMMDevHeapLeft = pVM->pdm.s.cbVMMDevHeap;
+    pVM->pdm.s.pfnVMMDevHeapNotify = NULL;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for DBGFR3TraceConfig that checks if the given tracing group name
+ * matches a device or driver name and applies the tracing config change.
+ *
+ * @returns VINF_SUCCESS or VERR_NOT_FOUND.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pszName             The tracing config group name.  This is NULL if
+ *                              the operation applies to every device and
+ *                              driver.
+ * @param   cchName             The length to match.
+ * @param   fEnable             Whether to enable or disable the corresponding
+ *                              trace points.
+ * @param   fApply              Whether to actually apply the changes or just do
+ *                              existence checks.
+ */
+VMMR3_INT_DECL(int) PDMR3TracingConfig(PVM pVM, const char *pszName, size_t cchName, bool fEnable, bool fApply)
+{
+    /** @todo This code is potentially racing driver attaching and detaching. */
+
+    /*
+     * Applies to all.
+     */
+    if (pszName == NULL)
+    {
+        AssertReturn(fApply, VINF_SUCCESS);
+
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        {
+            pDevIns->fTracing = fEnable;
+            for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                    pDrvIns->fTracing = fEnable;
+        }
+
+#ifdef VBOX_WITH_USB
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        {
+            pUsbIns->fTracing = fEnable;
+            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                    pDrvIns->fTracing = fEnable;
+
+        }
+#endif
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Specific devices, USB devices or drivers.
+     * Decode prefix to figure which of these it applies to.
+     */
+    if (cchName <= 3)
+        return VERR_NOT_FOUND;
+
+    uint32_t cMatches = 0;
+    if (!strncmp("dev", pszName, 3))
+    {
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        {
+            const char *pszDevName = pDevIns->Internal.s.pDevR3->pReg->szName;
+            size_t      cchDevName = strlen(pszDevName);
+            if (    (   cchDevName == cchName
+                     && RTStrNICmp(pszName, pszDevName, cchDevName))
+                ||  (   cchDevName == cchName - 3
+                     && RTStrNICmp(pszName + 3, pszDevName, cchDevName)) )
+            {
+                cMatches++;
+                if (fApply)
+                    pDevIns->fTracing = fEnable;
+            }
+        }
+    }
+    else if (!strncmp("usb", pszName, 3))
+    {
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        {
+            const char *pszUsbName = pUsbIns->Internal.s.pUsbDev->pReg->szName;
+            size_t      cchUsbName = strlen(pszUsbName);
+            if (    (   cchUsbName == cchName
+                     && RTStrNICmp(pszName, pszUsbName, cchUsbName))
+                ||  (   cchUsbName == cchName - 3
+                     && RTStrNICmp(pszName + 3, pszUsbName, cchUsbName)) )
+            {
+                cMatches++;
+                if (fApply)
+                    pUsbIns->fTracing = fEnable;
+            }
+        }
+    }
+    else if (!strncmp("drv", pszName, 3))
+    {
+        AssertReturn(fApply, VINF_SUCCESS);
+
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+            for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                {
+                    const char *pszDrvName = pDrvIns->Internal.s.pDrv->pReg->szName;
+                    size_t      cchDrvName = strlen(pszDrvName);
+                    if (    (   cchDrvName == cchName
+                             && RTStrNICmp(pszName, pszDrvName, cchDrvName))
+                        ||  (   cchDrvName == cchName - 3
+                             && RTStrNICmp(pszName + 3, pszDrvName, cchDrvName)) )
+                    {
+                        cMatches++;
+                        if (fApply)
+                            pDrvIns->fTracing = fEnable;
+                    }
+                }
+
+#ifdef VBOX_WITH_USB
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                {
+                    const char *pszDrvName = pDrvIns->Internal.s.pDrv->pReg->szName;
+                    size_t      cchDrvName = strlen(pszDrvName);
+                    if (    (   cchDrvName == cchName
+                             && RTStrNICmp(pszName, pszDrvName, cchDrvName))
+                        ||  (   cchDrvName == cchName - 3
+                             && RTStrNICmp(pszName + 3, pszDrvName, cchDrvName)) )
+                    {
+                        cMatches++;
+                        if (fApply)
+                            pDrvIns->fTracing = fEnable;
+                    }
+                }
+#endif
+    }
+    else
+        return VERR_NOT_FOUND;
+
+    return cMatches > 0 ? VINF_SUCCESS : VERR_NOT_FOUND;
+}
+
+
+/**
+ * Worker for DBGFR3TraceQueryConfig that checks whether all drivers, devices,
+ * and USB device have the same tracing settings.
+ *
+ * @returns true / false.
+ * @param   pVM                 The cross context VM structure.
+ * @param   fEnabled            The tracing setting to check for.
+ */
+VMMR3_INT_DECL(bool) PDMR3TracingAreAll(PVM pVM, bool fEnabled)
+{
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        if (pDevIns->fTracing != (uint32_t)fEnabled)
+            return false;
+
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                if (pDrvIns->fTracing != (uint32_t)fEnabled)
+                    return false;
+    }
+
+#ifdef VBOX_WITH_USB
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        if (pUsbIns->fTracing != (uint32_t)fEnabled)
+            return false;
+
+        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                if (pDrvIns->fTracing != (uint32_t)fEnabled)
+                    return false;
+    }
+#endif
+
+    return true;
+}
+
+
+/**
+ * Worker for PDMR3TracingQueryConfig that adds a prefixed name to the output
+ * string.
+ *
+ * @returns VINF_SUCCESS or VERR_BUFFER_OVERFLOW
+ * @param   ppszDst             The pointer to the output buffer pointer.
+ * @param   pcbDst              The pointer to the output buffer size.
+ * @param   fSpace              Whether to add a space before the name.
+ * @param   pszPrefix           The name prefix.
+ * @param   pszName             The name.
+ */
+static int pdmR3TracingAdd(char **ppszDst, size_t *pcbDst, bool fSpace, const char *pszPrefix, const char *pszName)
+{
+    size_t const cchPrefix = strlen(pszPrefix);
+    if (!RTStrNICmp(pszPrefix, pszName, cchPrefix))
+        pszName += cchPrefix;
+    size_t const cchName = strlen(pszName);
+
+    size_t const cchThis = cchName + cchPrefix + fSpace;
+    if (cchThis >= *pcbDst)
+        return VERR_BUFFER_OVERFLOW;
+    if (fSpace)
+    {
+        **ppszDst = ' ';
+        memcpy(*ppszDst + 1, pszPrefix, cchPrefix);
+        memcpy(*ppszDst + 1 + cchPrefix, pszName, cchName + 1);
+    }
+    else
+    {
+        memcpy(*ppszDst, pszPrefix, cchPrefix);
+        memcpy(*ppszDst + cchPrefix, pszName, cchName + 1);
+    }
+    *ppszDst += cchThis;
+    *pcbDst  -= cchThis;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for DBGFR3TraceQueryConfig use when not everything is either enabled
+ * or disabled.
+ *
+ * @returns VINF_SUCCESS or VERR_BUFFER_OVERFLOW
+ * @param   pVM                 The cross context VM structure.
+ * @param   pszConfig           Where to store the config spec.
+ * @param   cbConfig            The size of the output buffer.
+ */
+VMMR3_INT_DECL(int) PDMR3TracingQueryConfig(PVM pVM, char *pszConfig, size_t cbConfig)
+{
+    int     rc;
+    char   *pszDst = pszConfig;
+    size_t  cbDst  = cbConfig;
+
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        if (pDevIns->fTracing)
+        {
+            rc = pdmR3TracingAdd(&pszDst, &cbDst, pszDst != pszConfig, "dev", pDevIns->Internal.s.pDevR3->pReg->szName);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                if (pDrvIns->fTracing)
+                {
+                    rc = pdmR3TracingAdd(&pszDst, &cbDst, pszDst != pszConfig, "drv", pDrvIns->Internal.s.pDrv->pReg->szName);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+    }
+
+#ifdef VBOX_WITH_USB
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        if (pUsbIns->fTracing)
+        {
+            rc = pdmR3TracingAdd(&pszDst, &cbDst, pszDst != pszConfig, "usb", pUsbIns->Internal.s.pUsbDev->pReg->szName);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+
+        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                if (pDrvIns->fTracing)
+                {
+                    rc = pdmR3TracingAdd(&pszDst, &cbDst, pszDst != pszConfig, "drv", pDrvIns->Internal.s.pDrv->pReg->szName);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+    }
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks that a PDMDRVREG::szName, PDMDEVREG::szName or PDMUSBREG::szName
+ * field contains only a limited set of ASCII characters.
+ *
+ * @returns true / false.
+ * @param   pszName             The name to validate.
+ */
+bool pdmR3IsValidName(const char *pszName)
+{
+    char ch;
+    while (   (ch = *pszName) != '\0'
+           && (   RT_C_IS_ALNUM(ch)
+               || ch == '-'
+               || ch == ' ' /** @todo disallow this! */
+               || ch == '_') )
+        pszName++;
+    return ch == '\0';
+}
+
+
+/**
+ * Info handler for 'pdmtracingids'.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The output helpers.
+ * @param   pszArgs     The optional user arguments.
+ *
+ * @remarks Can be called on most threads.
+ */
+static DECLCALLBACK(void) pdmR3InfoTracingIds(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /*
+     * Parse the argument (optional).
+     */
+    if (   pszArgs
+        && *pszArgs
+        && strcmp(pszArgs, "all")
+        && strcmp(pszArgs, "devices")
+        && strcmp(pszArgs, "drivers")
+        && strcmp(pszArgs, "usb"))
+    {
+        pHlp->pfnPrintf(pHlp, "Unable to grok '%s'\n", pszArgs);
+        return;
+    }
+    bool fAll     = !pszArgs || !*pszArgs || !strcmp(pszArgs, "all");
+    bool fDevices = fAll || !strcmp(pszArgs, "devices");
+    bool fUsbDevs = fAll || !strcmp(pszArgs, "usb");
+    bool fDrivers = fAll || !strcmp(pszArgs, "drivers");
+
+    /*
+     * Produce the requested output.
+     */
+/** @todo lock PDM lists! */
+    /* devices */
+    if (fDevices)
+    {
+        pHlp->pfnPrintf(pHlp, "Device tracing IDs:\n");
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+            pHlp->pfnPrintf(pHlp, "%05u  %s\n", pDevIns->idTracing, pDevIns->Internal.s.pDevR3->pReg->szName);
+    }
+
+    /* USB devices */
+    if (fUsbDevs)
+    {
+        pHlp->pfnPrintf(pHlp, "USB device tracing IDs:\n");
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+            pHlp->pfnPrintf(pHlp, "%05u  %s\n", pUsbIns->idTracing, pUsbIns->Internal.s.pUsbDev->pReg->szName);
+    }
+
+    /* Drivers */
+    if (fDrivers)
+    {
+        pHlp->pfnPrintf(pHlp, "Driver tracing IDs:\n");
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        {
+            for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+            {
+                uint32_t iLevel = 0;
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown, iLevel++)
+                    pHlp->pfnPrintf(pHlp, "%05u  %s (level %u, lun %u, dev %s)\n",
+                                    pDrvIns->idTracing, pDrvIns->Internal.s.pDrv->pReg->szName,
+                                    iLevel, pLun->iLun, pDevIns->Internal.s.pDevR3->pReg->szName);
+            }
+        }
+
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        {
+            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+            {
+                uint32_t iLevel = 0;
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown, iLevel++)
+                    pHlp->pfnPrintf(pHlp, "%05u  %s (level %u, lun %u, dev %s)\n",
+                                    pDrvIns->idTracing, pDrvIns->Internal.s.pDrv->pReg->szName,
+                                    iLevel, pLun->iLun, pUsbIns->Internal.s.pUsbDev->pReg->szName);
+            }
+        }
+    }
+}
+
diff --git a/src/VBox/VMM/VMMR3/PDMAsyncCompletion.cpp b/src/VBox/VMM/VMMR3/PDMAsyncCompletion.cpp
new file mode 100644
index 00000000..322756a2
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMAsyncCompletion.cpp
@@ -0,0 +1,1805 @@
+/* $Id: PDMAsyncCompletion.cpp $ */
+/** @file
+ * PDM Async I/O - Transport data asynchronous in R3 using EMT.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/mem.h>
+#include <iprt/critsect.h>
+#include <iprt/tcp.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+
+#include <VBox/vmm/pdmasynccompletion.h>
+#include "PDMAsyncCompletionInternal.h"
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Async I/O type.
+ */
+typedef enum PDMASYNCCOMPLETIONTEMPLATETYPE
+{
+    /** Device . */
+    PDMASYNCCOMPLETIONTEMPLATETYPE_DEV = 1,
+    /** Driver consumer. */
+    PDMASYNCCOMPLETIONTEMPLATETYPE_DRV,
+    /** Internal consumer. */
+    PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL,
+    /** Usb consumer. */
+    PDMASYNCCOMPLETIONTEMPLATETYPE_USB
+} PDMASYNCTEMPLATETYPE;
+
+/**
+ * PDM Async I/O template.
+ */
+typedef struct PDMASYNCCOMPLETIONTEMPLATE
+{
+    /** Pointer to the next template in the list. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONTEMPLATE)    pNext;
+    /** Pointer to the previous template in the list. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONTEMPLATE)    pPrev;
+    /** Type specific data. */
+    union
+    {
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_DEV */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMASYNCCOMPLETEDEV)   pfnCompleted;
+            /** Pointer to the device instance owning the template. */
+            R3PTRTYPE(PPDMDEVINS)               pDevIns;
+        } Dev;
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_DRV */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMASYNCCOMPLETEDRV)   pfnCompleted;
+            /** Pointer to the driver instance owning the template. */
+            R3PTRTYPE(PPDMDRVINS)               pDrvIns;
+            /** User argument given during template creation.
+             *  This is only here to make things much easier
+             *  for DrVVD. */
+            void                               *pvTemplateUser;
+        } Drv;
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMASYNCCOMPLETEINT)   pfnCompleted;
+            /** Pointer to user data. */
+            R3PTRTYPE(void *)                   pvUser;
+        } Int;
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_USB */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMASYNCCOMPLETEUSB)   pfnCompleted;
+            /** Pointer to the usb instance owning the template. */
+            R3PTRTYPE(PPDMUSBINS)               pUsbIns;
+        } Usb;
+    } u;
+    /** Template type. */
+    PDMASYNCCOMPLETIONTEMPLATETYPE          enmType;
+    /** Pointer to the VM. */
+    R3PTRTYPE(PVM)                          pVM;
+    /** Use count of the template. */
+    volatile uint32_t                       cUsed;
+} PDMASYNCCOMPLETIONTEMPLATE;
+
+/**
+ * Bandwidth control manager instance data
+ */
+typedef struct PDMACBWMGR
+{
+    /** Pointer to the next manager in the list. */
+    struct PDMACBWMGR                          *pNext;
+    /** Pointer to the shared UVM structure. */
+    PPDMASYNCCOMPLETIONEPCLASS                  pEpClass;
+    /** Identifier of the manager. */
+    char                                       *pszId;
+    /** Maximum number of bytes the endpoints are allowed to transfer (Max is 4GB/s currently) */
+    volatile uint32_t                           cbTransferPerSecMax;
+    /** Number of bytes we start with */
+    volatile uint32_t                           cbTransferPerSecStart;
+    /** Step after each update */
+    volatile uint32_t                           cbTransferPerSecStep;
+    /** Number of bytes we are allowed to transfer till the next update.
+     * Reset by the refresh timer. */
+    volatile uint32_t                           cbTransferAllowed;
+    /** Timestamp of the last update */
+    volatile uint64_t                           tsUpdatedLast;
+    /** Reference counter - How many endpoints are associated with this manager. */
+    volatile uint32_t                           cRefs;
+} PDMACBWMGR;
+/** Pointer to a bandwidth control manager pointer. */
+typedef PPDMACBWMGR *PPPDMACBWMGR;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static void pdmR3AsyncCompletionPutTask(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, PPDMASYNCCOMPLETIONTASK pTask);
+
+
+/**
+ * Internal worker for the creation apis
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   ppTemplate      Where to store the template handle.
+ * @param   enmType         Async completion template type (dev, drv, usb, int).
+ */
+static int pdmR3AsyncCompletionTemplateCreate(PVM pVM, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate,
+                                              PDMASYNCCOMPLETIONTEMPLATETYPE enmType)
+{
+    PUVM pUVM = pVM->pUVM;
+
+    AssertPtrReturn(ppTemplate, VERR_INVALID_POINTER);
+
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
+    int rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_ASYNC_COMPLETION, sizeof(PDMASYNCCOMPLETIONTEMPLATE), (void **)&pTemplate);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Initialize fields.
+     */
+    pTemplate->pVM = pVM;
+    pTemplate->cUsed = 0;
+    pTemplate->enmType = enmType;
+
+    /*
+     * Add template to the global VM template list.
+     */
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    pTemplate->pNext = pUVM->pdm.s.pAsyncCompletionTemplates;
+    if (pUVM->pdm.s.pAsyncCompletionTemplates)
+        pUVM->pdm.s.pAsyncCompletionTemplates->pPrev = pTemplate;
+    pUVM->pdm.s.pAsyncCompletionTemplates = pTemplate;
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+    *ppTemplate = pTemplate;
+    return VINF_SUCCESS;
+}
+
+
+#ifdef SOME_UNUSED_FUNCTION
+/**
+ * Creates a async completion template for a device instance.
+ *
+ * The template is used when creating new completion tasks.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance.
+ * @param   ppTemplate      Where to store the template pointer on success.
+ * @param   pfnCompleted    The completion callback routine.
+ * @param   pszDesc         Description.
+ */
+int pdmR3AsyncCompletionTemplateCreateDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate,
+                                             PFNPDMASYNCCOMPLETEDEV pfnCompleted, const char *pszDesc)
+{
+    LogFlow(("%s: pDevIns=%p ppTemplate=%p pfnCompleted=%p pszDesc=%s\n",
+              __FUNCTION__, pDevIns, ppTemplate, pfnCompleted, pszDesc));
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT(pVM);
+    AssertPtrReturn(pfnCompleted, VERR_INVALID_POINTER);
+    AssertPtrReturn(ppTemplate, VERR_INVALID_POINTER);
+
+    /*
+     * Create the template.
+     */
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
+    int rc = pdmR3AsyncCompletionTemplateCreate(pVM, &pTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE_DEV);
+    if (RT_SUCCESS(rc))
+    {
+        pTemplate->u.Dev.pDevIns = pDevIns;
+        pTemplate->u.Dev.pfnCompleted = pfnCompleted;
+
+        *ppTemplate = pTemplate;
+        Log(("PDM: Created device template %p: pfnCompleted=%p pDevIns=%p\n",
+             pTemplate, pfnCompleted, pDevIns));
+    }
+
+    return rc;
+}
+#endif /* SOME_UNUSED_FUNCTION */
+
+
+/**
+ * Creates a async completion template for a driver instance.
+ *
+ * The template is used when creating new completion tasks.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDrvIns         The driver instance.
+ * @param   ppTemplate      Where to store the template pointer on success.
+ * @param   pfnCompleted    The completion callback routine.
+ * @param   pvTemplateUser  Template user argument
+ * @param   pszDesc         Description.
+ */
+int pdmR3AsyncCompletionTemplateCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate,
+                                             PFNPDMASYNCCOMPLETEDRV pfnCompleted, void *pvTemplateUser,
+                                             const char *pszDesc)
+{
+    LogFlow(("PDMR3AsyncCompletionTemplateCreateDriver: pDrvIns=%p ppTemplate=%p pfnCompleted=%p pszDesc=%s\n",
+             pDrvIns, ppTemplate, pfnCompleted, pszDesc));
+    RT_NOREF_PV(pszDesc); /** @todo async template description */
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnCompleted, VERR_INVALID_POINTER);
+    AssertPtrReturn(ppTemplate, VERR_INVALID_POINTER);
+
+    /*
+     * Create the template.
+     */
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
+    int rc = pdmR3AsyncCompletionTemplateCreate(pVM, &pTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE_DRV);
+    if (RT_SUCCESS(rc))
+    {
+        pTemplate->u.Drv.pDrvIns        = pDrvIns;
+        pTemplate->u.Drv.pfnCompleted   = pfnCompleted;
+        pTemplate->u.Drv.pvTemplateUser = pvTemplateUser;
+
+        *ppTemplate = pTemplate;
+        Log(("PDM: Created driver template %p: pfnCompleted=%p pDrvIns=%p\n",
+             pTemplate, pfnCompleted, pDrvIns));
+    }
+
+    return rc;
+}
+
+
+#ifdef SOME_UNUSED_FUNCTION
+/**
+ * Creates a async completion template for a USB device instance.
+ *
+ * The template is used when creating new completion tasks.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pUsbIns         The USB device instance.
+ * @param   ppTemplate      Where to store the template pointer on success.
+ * @param   pfnCompleted    The completion callback routine.
+ * @param   pszDesc         Description.
+ */
+int pdmR3AsyncCompletionTemplateCreateUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate,
+                                          PFNPDMASYNCCOMPLETEUSB pfnCompleted, const char *pszDesc)
+{
+    LogFlow(("pdmR3AsyncCompletionTemplateCreateUsb: pUsbIns=%p ppTemplate=%p pfnCompleted=%p pszDesc=%s\n", pUsbIns, ppTemplate, pfnCompleted, pszDesc));
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT(pVM);
+    AssertPtrReturn(pfnCompleted, VERR_INVALID_POINTER);
+    AssertPtrReturn(ppTemplate, VERR_INVALID_POINTER);
+
+    /*
+     * Create the template.
+     */
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
+    int rc = pdmR3AsyncCompletionTemplateCreate(pVM, &pTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE_USB);
+    if (RT_SUCCESS(rc))
+    {
+        pTemplate->u.Usb.pUsbIns = pUsbIns;
+        pTemplate->u.Usb.pfnCompleted = pfnCompleted;
+
+        *ppTemplate = pTemplate;
+        Log(("PDM: Created usb template %p: pfnCompleted=%p pDevIns=%p\n",
+             pTemplate, pfnCompleted, pUsbIns));
+    }
+
+    return rc;
+}
+#endif
+
+
+/**
+ * Creates a async completion template for internally by the VMM.
+ *
+ * The template is used when creating new completion tasks.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   ppTemplate      Where to store the template pointer on success.
+ * @param   pfnCompleted    The completion callback routine.
+ * @param   pvUser2         The 2nd user argument for the callback.
+ * @param   pszDesc         Description.
+ * @internal
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionTemplateCreateInternal(PVM pVM, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate,
+                                                          PFNPDMASYNCCOMPLETEINT pfnCompleted, void *pvUser2, const char *pszDesc)
+{
+    LogFlow(("PDMR3AsyncCompletionTemplateCreateInternal: ppTemplate=%p pfnCompleted=%p pvUser2=%p pszDesc=%s\n",
+              ppTemplate, pfnCompleted, pvUser2, pszDesc));
+    RT_NOREF_PV(pszDesc); /** @todo async template description */
+
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT(pVM);
+    AssertPtrReturn(pfnCompleted, VERR_INVALID_POINTER);
+    AssertPtrReturn(ppTemplate, VERR_INVALID_POINTER);
+
+    /*
+     * Create the template.
+     */
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
+    int rc = pdmR3AsyncCompletionTemplateCreate(pVM, &pTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL);
+    if (RT_SUCCESS(rc))
+    {
+        pTemplate->u.Int.pvUser = pvUser2;
+        pTemplate->u.Int.pfnCompleted = pfnCompleted;
+
+        *ppTemplate = pTemplate;
+        Log(("PDM: Created internal template %p: pfnCompleted=%p pvUser2=%p\n",
+             pTemplate, pfnCompleted, pvUser2));
+    }
+
+    return rc;
+}
+
+
+/**
+ * Destroys the specified async completion template.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PDM_ASYNC_TEMPLATE_BUSY if the template is still in use.
+ *
+ * @param   pTemplate       The template in question.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionTemplateDestroy(PPDMASYNCCOMPLETIONTEMPLATE pTemplate)
+{
+    LogFlow(("%s: pTemplate=%p\n", __FUNCTION__, pTemplate));
+
+    if (!pTemplate)
+    {
+        AssertMsgFailed(("pTemplate is NULL!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Check if the template is still used.
+     */
+    if (pTemplate->cUsed > 0)
+    {
+        AssertMsgFailed(("Template is still in use\n"));
+        return VERR_PDM_ASYNC_TEMPLATE_BUSY;
+    }
+
+    /*
+     * Unlink the template from the list.
+     */
+    PUVM pUVM = pTemplate->pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+
+    PPDMASYNCCOMPLETIONTEMPLATE pPrev = pTemplate->pPrev;
+    PPDMASYNCCOMPLETIONTEMPLATE pNext = pTemplate->pNext;
+
+    if (pPrev)
+        pPrev->pNext = pNext;
+    else
+        pUVM->pdm.s.pAsyncCompletionTemplates = pNext;
+
+    if (pNext)
+        pNext->pPrev = pPrev;
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+    /*
+     * Free the template.
+     */
+    MMR3HeapFree(pTemplate);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroys all the specified async completion templates for the given device instance.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PDM_ASYNC_TEMPLATE_BUSY if one or more of the templates are still in use.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance.
+ */
+int pdmR3AsyncCompletionTemplateDestroyDevice(PVM pVM, PPDMDEVINS pDevIns)
+{
+    LogFlow(("pdmR3AsyncCompletionTemplateDestroyDevice: pDevIns=%p\n", pDevIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pDevIns)
+        return VERR_INVALID_PARAMETER;
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Unlink it.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate = pUVM->pdm.s.pAsyncCompletionTemplates;
+    while (pTemplate)
+    {
+        if (    pTemplate->enmType == PDMASYNCCOMPLETIONTEMPLATETYPE_DEV
+            &&  pTemplate->u.Dev.pDevIns == pDevIns)
+        {
+            PPDMASYNCCOMPLETIONTEMPLATE pTemplateDestroy = pTemplate;
+            pTemplate = pTemplate->pNext;
+            int rc = PDMR3AsyncCompletionTemplateDestroy(pTemplateDestroy);
+            if (RT_FAILURE(rc))
+            {
+                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+                return rc;
+            }
+        }
+        else
+            pTemplate = pTemplate->pNext;
+    }
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroys all the specified async completion templates for the given driver instance.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PDM_ASYNC_TEMPLATE_BUSY if one or more of the templates are still in use.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDrvIns         The driver instance.
+ */
+int pdmR3AsyncCompletionTemplateDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns)
+{
+    LogFlow(("pdmR3AsyncCompletionTemplateDestroyDriver: pDevIns=%p\n", pDrvIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pDrvIns)
+        return VERR_INVALID_PARAMETER;
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Unlink it.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate = pUVM->pdm.s.pAsyncCompletionTemplates;
+    while (pTemplate)
+    {
+        if (    pTemplate->enmType == PDMASYNCCOMPLETIONTEMPLATETYPE_DRV
+            &&  pTemplate->u.Drv.pDrvIns == pDrvIns)
+        {
+            PPDMASYNCCOMPLETIONTEMPLATE pTemplateDestroy = pTemplate;
+            pTemplate = pTemplate->pNext;
+            int rc = PDMR3AsyncCompletionTemplateDestroy(pTemplateDestroy);
+            if (RT_FAILURE(rc))
+            {
+                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+                return rc;
+            }
+        }
+        else
+            pTemplate = pTemplate->pNext;
+    }
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroys all the specified async completion templates for the given USB device instance.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PDM_ASYNC_TEMPLATE_BUSY if one or more of the templates are still in use.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pUsbIns         The USB device instance.
+ */
+int pdmR3AsyncCompletionTemplateDestroyUsb(PVM pVM, PPDMUSBINS pUsbIns)
+{
+    LogFlow(("pdmR3AsyncCompletionTemplateDestroyUsb: pUsbIns=%p\n", pUsbIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pUsbIns)
+        return VERR_INVALID_PARAMETER;
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Unlink it.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate = pUVM->pdm.s.pAsyncCompletionTemplates;
+    while (pTemplate)
+    {
+        if (    pTemplate->enmType == PDMASYNCCOMPLETIONTEMPLATETYPE_USB
+            &&  pTemplate->u.Usb.pUsbIns == pUsbIns)
+        {
+            PPDMASYNCCOMPLETIONTEMPLATE pTemplateDestroy = pTemplate;
+            pTemplate = pTemplate->pNext;
+            int rc = PDMR3AsyncCompletionTemplateDestroy(pTemplateDestroy);
+            if (RT_FAILURE(rc))
+            {
+                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+                return rc;
+            }
+        }
+        else
+            pTemplate = pTemplate->pNext;
+    }
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return VINF_SUCCESS;
+}
+
+
+/** Lazy coder. */
+static PPDMACBWMGR pdmacBwMgrFindById(PPDMASYNCCOMPLETIONEPCLASS pEpClass, const char *pszId)
+{
+    PPDMACBWMGR pBwMgr = NULL;
+
+    if (pszId)
+    {
+        int rc = RTCritSectEnter(&pEpClass->CritSect); AssertRC(rc);
+
+        pBwMgr = pEpClass->pBwMgrsHead;
+        while (   pBwMgr
+               && RTStrCmp(pBwMgr->pszId, pszId))
+            pBwMgr = pBwMgr->pNext;
+
+        rc = RTCritSectLeave(&pEpClass->CritSect); AssertRC(rc);
+    }
+
+    return pBwMgr;
+}
+
+
+/** Lazy coder. */
+static void pdmacBwMgrLink(PPDMACBWMGR pBwMgr)
+{
+    PPDMASYNCCOMPLETIONEPCLASS pEpClass = pBwMgr->pEpClass;
+    int rc = RTCritSectEnter(&pEpClass->CritSect); AssertRC(rc);
+
+    pBwMgr->pNext = pEpClass->pBwMgrsHead;
+    pEpClass->pBwMgrsHead = pBwMgr;
+
+    rc = RTCritSectLeave(&pEpClass->CritSect); AssertRC(rc);
+}
+
+
+#ifdef SOME_UNUSED_FUNCTION
+/** Lazy coder. */
+static void pdmacBwMgrUnlink(PPDMACBWMGR pBwMgr)
+{
+    PPDMASYNCCOMPLETIONEPCLASS pEpClass = pBwMgr->pEpClass;
+    int rc = RTCritSectEnter(&pEpClass->CritSect); AssertRC(rc);
+
+    if (pBwMgr == pEpClass->pBwMgrsHead)
+        pEpClass->pBwMgrsHead = pBwMgr->pNext;
+    else
+    {
+        PPDMACBWMGR pPrev = pEpClass->pBwMgrsHead;
+        while (   pPrev
+               && pPrev->pNext != pBwMgr)
+            pPrev = pPrev->pNext;
+
+        AssertPtr(pPrev);
+        pPrev->pNext = pBwMgr->pNext;
+    }
+
+    rc = RTCritSectLeave(&pEpClass->CritSect); AssertRC(rc);
+}
+#endif /* SOME_UNUSED_FUNCTION */
+
+
+/** Lazy coder. */
+static int pdmacAsyncCompletionBwMgrCreate(PPDMASYNCCOMPLETIONEPCLASS pEpClass, const char *pszBwMgr, uint32_t cbTransferPerSecMax,
+                                           uint32_t cbTransferPerSecStart, uint32_t cbTransferPerSecStep)
+{
+    LogFlowFunc(("pEpClass=%#p pszBwMgr=%#p{%s} cbTransferPerSecMax=%u cbTransferPerSecStart=%u cbTransferPerSecStep=%u\n",
+                 pEpClass, pszBwMgr, pszBwMgr, cbTransferPerSecMax, cbTransferPerSecStart, cbTransferPerSecStep));
+
+    AssertPtrReturn(pEpClass, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszBwMgr, VERR_INVALID_POINTER);
+    AssertReturn(*pszBwMgr != '\0', VERR_INVALID_PARAMETER);
+
+    int         rc;
+    PPDMACBWMGR pBwMgr = pdmacBwMgrFindById(pEpClass, pszBwMgr);
+    if (!pBwMgr)
+    {
+        rc = MMR3HeapAllocZEx(pEpClass->pVM, MM_TAG_PDM_ASYNC_COMPLETION,
+                              sizeof(PDMACBWMGR),
+                              (void **)&pBwMgr);
+        if (RT_SUCCESS(rc))
+        {
+            pBwMgr->pszId = RTStrDup(pszBwMgr);
+            if (pBwMgr->pszId)
+            {
+                pBwMgr->pEpClass              = pEpClass;
+                pBwMgr->cRefs                 = 0;
+
+                /* Init I/O flow control. */
+                pBwMgr->cbTransferPerSecMax   = cbTransferPerSecMax;
+                pBwMgr->cbTransferPerSecStart = cbTransferPerSecStart;
+                pBwMgr->cbTransferPerSecStep  = cbTransferPerSecStep;
+
+                pBwMgr->cbTransferAllowed     = pBwMgr->cbTransferPerSecStart;
+                pBwMgr->tsUpdatedLast         = RTTimeSystemNanoTS();
+
+                pdmacBwMgrLink(pBwMgr);
+                rc = VINF_SUCCESS;
+            }
+            else
+            {
+                rc = VERR_NO_MEMORY;
+                MMR3HeapFree(pBwMgr);
+            }
+        }
+    }
+    else
+        rc = VERR_ALREADY_EXISTS;
+
+    LogFlowFunc(("returns rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/** Lazy coder. */
+DECLINLINE(void) pdmacBwMgrRetain(PPDMACBWMGR pBwMgr)
+{
+    ASMAtomicIncU32(&pBwMgr->cRefs);
+}
+
+
+/** Lazy coder. */
+DECLINLINE(void) pdmacBwMgrRelease(PPDMACBWMGR pBwMgr)
+{
+    Assert(pBwMgr->cRefs > 0);
+    ASMAtomicDecU32(&pBwMgr->cRefs);
+}
+
+
+/**
+ * Checks if the endpoint is allowed to transfer the given amount of bytes.
+ *
+ * @returns true if the endpoint is allowed to transfer the data.
+ *          false otherwise
+ * @param   pEndpoint                 The endpoint.
+ * @param   cbTransfer                The number of bytes to transfer.
+ * @param   pmsWhenNext               Where to store the number of milliseconds
+ *                                    until the bandwidth is refreshed.
+ *                                    Only set if false is returned.
+ */
+bool pdmacEpIsTransferAllowed(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint32_t cbTransfer, RTMSINTERVAL *pmsWhenNext)
+{
+    bool        fAllowed = true;
+    PPDMACBWMGR pBwMgr = ASMAtomicReadPtrT(&pEndpoint->pBwMgr, PPDMACBWMGR);
+
+    LogFlowFunc(("pEndpoint=%p pBwMgr=%p cbTransfer=%u\n", pEndpoint, pBwMgr, cbTransfer));
+
+    if (pBwMgr)
+    {
+        uint32_t cbOld = ASMAtomicSubU32(&pBwMgr->cbTransferAllowed, cbTransfer);
+        if (RT_LIKELY(cbOld >= cbTransfer))
+            fAllowed = true;
+        else
+        {
+            fAllowed = false;
+
+            /* We are out of resources  Check if we can update again. */
+            uint64_t tsNow          = RTTimeSystemNanoTS();
+            uint64_t tsUpdatedLast  = ASMAtomicUoReadU64(&pBwMgr->tsUpdatedLast);
+
+            if (tsNow - tsUpdatedLast >= (1000*1000*1000))
+            {
+                if (ASMAtomicCmpXchgU64(&pBwMgr->tsUpdatedLast, tsNow, tsUpdatedLast))
+                {
+                    if (pBwMgr->cbTransferPerSecStart < pBwMgr->cbTransferPerSecMax)
+                    {
+                       pBwMgr->cbTransferPerSecStart = RT_MIN(pBwMgr->cbTransferPerSecMax, pBwMgr->cbTransferPerSecStart + pBwMgr->cbTransferPerSecStep);
+                       LogFlow(("AIOMgr: Increasing maximum bandwidth to %u bytes/sec\n", pBwMgr->cbTransferPerSecStart));
+                    }
+
+                    /* Update */
+                    uint32_t cbTransferAllowedNew =   pBwMgr->cbTransferPerSecStart > cbTransfer
+                                                    ? pBwMgr->cbTransferPerSecStart - cbTransfer
+                                                    : 0;
+                    ASMAtomicWriteU32(&pBwMgr->cbTransferAllowed, cbTransferAllowedNew);
+                    fAllowed = true;
+                    LogFlow(("AIOMgr: Refreshed bandwidth\n"));
+                }
+            }
+            else
+            {
+                ASMAtomicAddU32(&pBwMgr->cbTransferAllowed, cbTransfer);
+                *pmsWhenNext = ((1000*1000*1000) - (tsNow - tsUpdatedLast)) / (1000*1000);
+            }
+        }
+    }
+
+    LogFlowFunc(("fAllowed=%RTbool\n", fAllowed));
+    return fAllowed;
+}
+
+
+/**
+ * Called by the endpoint if a task has finished.
+ *
+ * @returns nothing
+ * @param   pTask                     Pointer to the finished task.
+ * @param   rc                        Status code of the completed request.
+ * @param   fCallCompletionHandler    Flag whether the completion handler should be called to
+ *                                    inform the owner of the task that it has completed.
+ */
+void pdmR3AsyncCompletionCompleteTask(PPDMASYNCCOMPLETIONTASK pTask, int rc, bool fCallCompletionHandler)
+{
+    LogFlow(("%s: pTask=%#p fCallCompletionHandler=%RTbool\n", __FUNCTION__, pTask, fCallCompletionHandler));
+
+    if (fCallCompletionHandler)
+    {
+        PPDMASYNCCOMPLETIONTEMPLATE pTemplate = pTask->pEndpoint->pTemplate;
+
+        switch (pTemplate->enmType)
+        {
+            case PDMASYNCCOMPLETIONTEMPLATETYPE_DEV:
+                pTemplate->u.Dev.pfnCompleted(pTemplate->u.Dev.pDevIns, pTask->pvUser, rc);
+                break;
+
+            case PDMASYNCCOMPLETIONTEMPLATETYPE_DRV:
+                pTemplate->u.Drv.pfnCompleted(pTemplate->u.Drv.pDrvIns, pTemplate->u.Drv.pvTemplateUser, pTask->pvUser, rc);
+                break;
+
+            case PDMASYNCCOMPLETIONTEMPLATETYPE_USB:
+                pTemplate->u.Usb.pfnCompleted(pTemplate->u.Usb.pUsbIns, pTask->pvUser, rc);
+                break;
+
+            case PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL:
+                pTemplate->u.Int.pfnCompleted(pTemplate->pVM, pTask->pvUser, pTemplate->u.Int.pvUser, rc);
+                break;
+
+            default:
+                AssertMsgFailed(("Unknown template type!\n"));
+        }
+    }
+
+    pdmR3AsyncCompletionPutTask(pTask->pEndpoint, pTask);
+}
+
+
+/**
+ * Worker initializing a endpoint class.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pEpClassOps Pointer to the endpoint class structure.
+ * @param   pCfgHandle  Pointer to the CFGM tree.
+ */
+int pdmR3AsyncCompletionEpClassInit(PVM pVM, PCPDMASYNCCOMPLETIONEPCLASSOPS pEpClassOps, PCFGMNODE pCfgHandle)
+{
+    /* Validate input. */
+    AssertPtrReturn(pEpClassOps, VERR_INVALID_POINTER);
+    AssertReturn(pEpClassOps->u32Version == PDMAC_EPCLASS_OPS_VERSION, VERR_VERSION_MISMATCH);
+    AssertReturn(pEpClassOps->u32VersionEnd == PDMAC_EPCLASS_OPS_VERSION, VERR_VERSION_MISMATCH);
+
+    LogFlow(("pdmR3AsyncCompletionEpClassInit: pVM=%p pEpClassOps=%p{%s}\n", pVM, pEpClassOps, pEpClassOps->pszName));
+
+    /* Allocate global class data. */
+    PPDMASYNCCOMPLETIONEPCLASS pEndpointClass = NULL;
+
+    int rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_ASYNC_COMPLETION,
+                              pEpClassOps->cbEndpointClassGlobal,
+                              (void **)&pEndpointClass);
+    if (RT_SUCCESS(rc))
+    {
+        /* Initialize common data. */
+        pEndpointClass->pVM          = pVM;
+        pEndpointClass->pEndpointOps = pEpClassOps;
+
+        rc = RTCritSectInit(&pEndpointClass->CritSect);
+        if (RT_SUCCESS(rc))
+        {
+            PCFGMNODE pCfgNodeClass = CFGMR3GetChild(pCfgHandle, pEpClassOps->pszName);
+
+            /* Create task cache */
+            rc = RTMemCacheCreate(&pEndpointClass->hMemCacheTasks, pEpClassOps->cbTask,
+                                  0, UINT32_MAX, NULL, NULL, NULL, 0);
+            if (RT_SUCCESS(rc))
+            {
+                /* Call the specific endpoint class initializer. */
+                rc = pEpClassOps->pfnInitialize(pEndpointClass, pCfgNodeClass);
+                if (RT_SUCCESS(rc))
+                {
+                    /* Create all bandwidth groups for resource control. */
+                    PCFGMNODE pCfgBwGrp = CFGMR3GetChild(pCfgNodeClass, "BwGroups");
+                    if (pCfgBwGrp)
+                    {
+                        for (PCFGMNODE pCur = CFGMR3GetFirstChild(pCfgBwGrp); pCur; pCur = CFGMR3GetNextChild(pCur))
+                        {
+                            size_t cbName = CFGMR3GetNameLen(pCur) + 1;
+                            char *pszBwGrpId = (char *)RTMemAllocZ(cbName);
+                            if (pszBwGrpId)
+                            {
+                                rc = CFGMR3GetName(pCur, pszBwGrpId, cbName);
+                                if (RT_SUCCESS(rc))
+                                {
+                                    uint32_t cbMax;
+                                    rc = CFGMR3QueryU32(pCur, "Max", &cbMax);
+                                    if (RT_SUCCESS(rc))
+                                    {
+                                        uint32_t cbStart;
+                                        rc = CFGMR3QueryU32Def(pCur, "Start", &cbStart, cbMax);
+                                        if (RT_SUCCESS(rc))
+                                        {
+                                            uint32_t cbStep;
+                                            rc = CFGMR3QueryU32Def(pCur, "Step", &cbStep, 0);
+                                            if (RT_SUCCESS(rc))
+                                                rc = pdmacAsyncCompletionBwMgrCreate(pEndpointClass, pszBwGrpId,
+                                                                                     cbMax, cbStart, cbStep);
+                                        }
+                                    }
+                                }
+                                RTMemFree(pszBwGrpId);
+                            }
+                            else
+                                rc = VERR_NO_MEMORY;
+                            if (RT_FAILURE(rc))
+                                break;
+                        }
+                    }
+                    if (RT_SUCCESS(rc))
+                    {
+                        PUVM pUVM = pVM->pUVM;
+                        AssertMsg(!pUVM->pdm.s.apAsyncCompletionEndpointClass[pEpClassOps->enmClassType],
+                                  ("Endpoint class was already initialized\n"));
+
+#ifdef VBOX_WITH_STATISTICS
+                        CFGMR3QueryBoolDef(pCfgNodeClass, "AdvancedStatistics", &pEndpointClass->fGatherAdvancedStatistics, true);
+#else
+                        CFGMR3QueryBoolDef(pCfgNodeClass, "AdvancedStatistics", &pEndpointClass->fGatherAdvancedStatistics, false);
+#endif
+
+                        pUVM->pdm.s.apAsyncCompletionEndpointClass[pEpClassOps->enmClassType] = pEndpointClass;
+                        LogFlowFunc((": Initialized endpoint class \"%s\" rc=%Rrc\n", pEpClassOps->pszName, rc));
+                        return VINF_SUCCESS;
+                    }
+                }
+                RTMemCacheDestroy(pEndpointClass->hMemCacheTasks);
+            }
+            RTCritSectDelete(&pEndpointClass->CritSect);
+        }
+        MMR3HeapFree(pEndpointClass);
+    }
+
+    LogFlowFunc((": Failed to initialize endpoint class rc=%Rrc\n", rc));
+
+    return rc;
+}
+
+
+/**
+ * Worker terminating all endpoint classes.
+ *
+ * @returns nothing
+ * @param   pEndpointClass    Pointer to the endpoint class to terminate.
+ *
+ * @remarks This method ensures that any still open endpoint is closed.
+ */
+static void pdmR3AsyncCompletionEpClassTerminate(PPDMASYNCCOMPLETIONEPCLASS pEndpointClass)
+{
+    PVM pVM = pEndpointClass->pVM;
+
+    /* Close all still open endpoints. */
+    while (pEndpointClass->pEndpointsHead)
+        PDMR3AsyncCompletionEpClose(pEndpointClass->pEndpointsHead);
+
+    /* Destroy the bandwidth managers. */
+    PPDMACBWMGR pBwMgr = pEndpointClass->pBwMgrsHead;
+    while (pBwMgr)
+    {
+        PPDMACBWMGR pFree = pBwMgr;
+        pBwMgr = pBwMgr->pNext;
+        MMR3HeapFree(pFree);
+    }
+
+    /* Call the termination callback of the class. */
+    pEndpointClass->pEndpointOps->pfnTerminate(pEndpointClass);
+
+    RTMemCacheDestroy(pEndpointClass->hMemCacheTasks);
+    RTCritSectDelete(&pEndpointClass->CritSect);
+
+    /* Free the memory of the class finally and clear the entry in the class array. */
+    pVM->pUVM->pdm.s.apAsyncCompletionEndpointClass[pEndpointClass->pEndpointOps->enmClassType] = NULL;
+    MMR3HeapFree(pEndpointClass);
+}
+
+
+/**
+ * Records the size of the request in the statistics.
+ *
+ * @returns nothing.
+ * @param   pEndpoint    The endpoint to register the request size for.
+ * @param   cbReq        Size of the request.
+ */
+static void pdmR3AsyncCompletionStatisticsRecordSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, size_t cbReq)
+{
+    if (cbReq < 512)
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqSizeSmaller512);
+    else if (cbReq < _1K)
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqSize512To1K);
+    else if (cbReq < _2K)
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqSize1KTo2K);
+    else if (cbReq < _4K)
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqSize2KTo4K);
+    else if (cbReq < _8K)
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqSize4KTo8K);
+    else if (cbReq < _16K)
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqSize8KTo16K);
+    else if (cbReq < _32K)
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqSize16KTo32K);
+    else if (cbReq < _64K)
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqSize32KTo64K);
+    else if (cbReq < _128K)
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqSize64KTo128K);
+    else if (cbReq < _256K)
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqSize128KTo256K);
+    else if (cbReq < _512K)
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqSize256KTo512K);
+    else
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqSizeOver512K);
+
+    if (cbReq & ((size_t)512 - 1))
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqsUnaligned512);
+    else if (cbReq & ((size_t)_4K - 1))
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqsUnaligned4K);
+    else if (cbReq & ((size_t)_8K - 1))
+        STAM_REL_COUNTER_INC(&pEndpoint->StatReqsUnaligned8K);
+}
+
+
+/**
+ * Records the required processing time of a request.
+ *
+ * @returns nothing.
+ * @param   pEndpoint    The endpoint.
+ * @param   cNsRun       The request time in nanoseconds.
+ */
+static void pdmR3AsyncCompletionStatisticsRecordCompletionTime(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint64_t cNsRun)
+{
+    PSTAMCOUNTER pStatCounter;
+    if (cNsRun < RT_NS_1US)
+        pStatCounter = &pEndpoint->StatTaskRunTimesNs[cNsRun / (RT_NS_1US / 10)];
+    else if (cNsRun < RT_NS_1MS)
+        pStatCounter = &pEndpoint->StatTaskRunTimesUs[cNsRun / (RT_NS_1MS / 10)];
+    else if (cNsRun < RT_NS_1SEC)
+        pStatCounter = &pEndpoint->StatTaskRunTimesMs[cNsRun / (RT_NS_1SEC / 10)];
+    else if (cNsRun < RT_NS_1SEC_64*100)
+        pStatCounter = &pEndpoint->StatTaskRunTimesSec[cNsRun / (RT_NS_1SEC_64*100 / 10)];
+    else
+        pStatCounter = &pEndpoint->StatTaskRunOver100Sec;
+    STAM_REL_COUNTER_INC(pStatCounter);
+
+    STAM_REL_COUNTER_INC(&pEndpoint->StatIoOpsCompleted);
+    pEndpoint->cIoOpsCompleted++;
+    uint64_t tsMsCur = RTTimeMilliTS();
+    uint64_t tsInterval = tsMsCur - pEndpoint->tsIntervalStartMs;
+    if (tsInterval >= 1000)
+    {
+        pEndpoint->StatIoOpsPerSec.c = pEndpoint->cIoOpsCompleted / (tsInterval / 1000);
+        pEndpoint->tsIntervalStartMs = tsMsCur;
+        pEndpoint->cIoOpsCompleted = 0;
+    }
+}
+
+
+/**
+ * Registers advanced statistics for the given endpoint.
+ *
+ * @returns VBox status code.
+ * @param   pEndpoint    The endpoint to register the advanced statistics for.
+ */
+static int pdmR3AsyncCompletionStatisticsRegister(PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
+{
+    int rc = VINF_SUCCESS;
+    PVM pVM = pEndpoint->pEpClass->pVM;
+
+    pEndpoint->tsIntervalStartMs = RTTimeMilliTS();
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesNs) && RT_SUCCESS(rc); i++)
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunTimesNs[i], STAMTYPE_COUNTER,
+                             STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                             "Nanosecond resolution runtime statistics",
+                             "/PDM/AsyncCompletion/File/%s/%d/TaskRun1Ns-%u-%u",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId, i*100, i*100+100-1);
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesUs) && RT_SUCCESS(rc); i++)
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunTimesUs[i], STAMTYPE_COUNTER,
+                             STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                             "Microsecond resolution runtime statistics",
+                             "/PDM/AsyncCompletion/File/%s/%d/TaskRun2MicroSec-%u-%u",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId, i*100, i*100+100-1);
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMs) && RT_SUCCESS(rc); i++)
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunTimesMs[i], STAMTYPE_COUNTER,
+                             STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                             "Milliseconds resolution runtime statistics",
+                             "/PDM/AsyncCompletion/File/%s/%d/TaskRun3Ms-%u-%u",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId, i*100, i*100+100-1);
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMs) && RT_SUCCESS(rc); i++)
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunTimesSec[i], STAMTYPE_COUNTER,
+                             STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                             "Second resolution runtime statistics",
+                             "/PDM/AsyncCompletion/File/%s/%d/TaskRun4Sec-%u-%u",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId, i*10, i*10+10-1);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunOver100Sec, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                             "Tasks which ran more than 100sec",
+                             "/PDM/AsyncCompletion/File/%s/%d/TaskRunSecGreater100Sec",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatIoOpsPerSec, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Processed I/O operations per second",
+                             "/PDM/AsyncCompletion/File/%s/%d/IoOpsPerSec",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatIoOpsStarted, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Started I/O operations for this endpoint",
+                             "/PDM/AsyncCompletion/File/%s/%d/IoOpsStarted",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatIoOpsCompleted, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Completed I/O operations for this endpoint",
+                             "/PDM/AsyncCompletion/File/%s/%d/IoOpsCompleted",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqSizeSmaller512, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests with a size smaller than 512 bytes",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqSizeSmaller512",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqSize512To1K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests with a size between 512 bytes and 1KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqSize512To1K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqSize1KTo2K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests with a size between 1KB and 2KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqSize1KTo2K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqSize2KTo4K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests with a size between 2KB and 4KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqSize2KTo4K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqSize4KTo8K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests with a size between 4KB and 8KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqSize4KTo8K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqSize8KTo16K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests with a size between 8KB and 16KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqSize8KTo16K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqSize16KTo32K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests with a size between 16KB and 32KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqSize16KTo32K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqSize32KTo64K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests with a size between 32KB and 64KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqSize32KTo64K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqSize64KTo128K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests with a size between 64KB and 128KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqSize64KTo128K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqSize128KTo256K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests with a size between 128KB and 256KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqSize128KTo256K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqSize256KTo512K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests with a size between 256KB and 512KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqSize256KTo512K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqSizeOver512K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests with a size over 512KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqSizeOver512K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqsUnaligned512, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests which size is not aligned to 512 bytes",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqsUnaligned512",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqsUnaligned4K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests which size is not aligned to 4KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqsUnaligned4K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    if (RT_SUCCESS(rc))
+        rc = STAMR3RegisterF(pVM, &pEndpoint->StatReqsUnaligned8K, STAMTYPE_COUNTER,
+                             STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                             "Number of requests which size is not aligned to 8KB",
+                             "/PDM/AsyncCompletion/File/%s/%d/ReqsUnaligned8K",
+                             RTPathFilename(pEndpoint->pszUri), pEndpoint->iStatId);
+
+    return rc;
+}
+
+
+/**
+ * Deregisters advanced statistics for one endpoint.
+ *
+ * @returns nothing.
+ * @param   pEndpoint    The endpoint to deregister the advanced statistics for.
+ */
+static void pdmR3AsyncCompletionStatisticsDeregister(PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
+{
+    /* I hope this doesn't remove too much... */
+    STAMR3DeregisterF(pEndpoint->pEpClass->pVM->pUVM, "/PDM/AsyncCompletion/File/%s/*", RTPathFilename(pEndpoint->pszUri));
+}
+
+
+/**
+ * Initialize the async completion manager.
+ *
+ * @returns VBox status code
+ * @param   pVM The cross context VM structure.
+ */
+int pdmR3AsyncCompletionInit(PVM pVM)
+{
+    LogFlowFunc((": pVM=%p\n", pVM));
+
+    VM_ASSERT_EMT(pVM);
+
+    PCFGMNODE pCfgRoot            = CFGMR3GetRoot(pVM);
+    PCFGMNODE pCfgAsyncCompletion = CFGMR3GetChild(CFGMR3GetChild(pCfgRoot, "PDM"), "AsyncCompletion");
+
+    int rc = pdmR3AsyncCompletionEpClassInit(pVM, &g_PDMAsyncCompletionEndpointClassFile, pCfgAsyncCompletion);
+    LogFlowFunc((": pVM=%p rc=%Rrc\n", pVM, rc));
+    return rc;
+}
+
+
+/**
+ * Terminates the async completion manager.
+ *
+ * @returns VBox status code
+ * @param   pVM The cross context VM structure.
+ */
+int pdmR3AsyncCompletionTerm(PVM pVM)
+{
+    LogFlowFunc((": pVM=%p\n", pVM));
+    PUVM pUVM = pVM->pUVM;
+
+    for (size_t i = 0; i < RT_ELEMENTS(pUVM->pdm.s.apAsyncCompletionEndpointClass); i++)
+        if (pUVM->pdm.s.apAsyncCompletionEndpointClass[i])
+            pdmR3AsyncCompletionEpClassTerminate(pUVM->pdm.s.apAsyncCompletionEndpointClass[i]);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resume worker for the async completion manager.
+ *
+ * @returns nothing.
+ * @param   pVM The cross context VM structure.
+ */
+void pdmR3AsyncCompletionResume(PVM pVM)
+{
+    LogFlowFunc((": pVM=%p\n", pVM));
+    PUVM pUVM = pVM->pUVM;
+
+    /* Log the bandwidth groups and all assigned endpoints. */
+    for (size_t i = 0; i < RT_ELEMENTS(pUVM->pdm.s.apAsyncCompletionEndpointClass); i++)
+        if (pUVM->pdm.s.apAsyncCompletionEndpointClass[i])
+        {
+            PPDMASYNCCOMPLETIONEPCLASS  pEpClass = pUVM->pdm.s.apAsyncCompletionEndpointClass[i];
+            PPDMACBWMGR                 pBwMgr   = pEpClass->pBwMgrsHead;
+            PPDMASYNCCOMPLETIONENDPOINT pEp;
+
+            if (pBwMgr)
+                LogRel(("AIOMgr: Bandwidth groups for class '%s'\n", i == PDMASYNCCOMPLETIONEPCLASSTYPE_FILE
+                                                                     ? "File" : "<Unknown>"));
+
+            while (pBwMgr)
+            {
+                LogRel(("AIOMgr:     Id:    %s\n", pBwMgr->pszId));
+                LogRel(("AIOMgr:     Max:   %u B/s\n", pBwMgr->cbTransferPerSecMax));
+                LogRel(("AIOMgr:     Start: %u B/s\n", pBwMgr->cbTransferPerSecStart));
+                LogRel(("AIOMgr:     Step:  %u B/s\n", pBwMgr->cbTransferPerSecStep));
+                LogRel(("AIOMgr:     Endpoints:\n"));
+
+                pEp = pEpClass->pEndpointsHead;
+                while (pEp)
+                {
+                    if (pEp->pBwMgr == pBwMgr)
+                        LogRel(("AIOMgr:         %s\n", pEp->pszUri));
+
+                    pEp = pEp->pNext;
+                }
+
+                pBwMgr = pBwMgr->pNext;
+            }
+
+            /* Print all endpoints without assigned bandwidth groups. */
+            pEp = pEpClass->pEndpointsHead;
+            if (pEp)
+                LogRel(("AIOMgr: Endpoints without assigned bandwidth groups:\n"));
+
+            while (pEp)
+            {
+                if (!pEp->pBwMgr)
+                    LogRel(("AIOMgr:     %s\n", pEp->pszUri));
+
+                pEp = pEp->pNext;
+            }
+        }
+}
+
+
+/**
+ * Tries to get a free task from the endpoint or class cache
+ * allocating the task if it fails.
+ *
+ * @returns Pointer to a new and initialized task or NULL
+ * @param   pEndpoint    The endpoint the task is for.
+ * @param   pvUser       Opaque user data for the task.
+ */
+static PPDMASYNCCOMPLETIONTASK pdmR3AsyncCompletionGetTask(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, void *pvUser)
+{
+    PPDMASYNCCOMPLETIONEPCLASS pEndpointClass = pEndpoint->pEpClass;
+    PPDMASYNCCOMPLETIONTASK    pTask = (PPDMASYNCCOMPLETIONTASK)RTMemCacheAlloc(pEndpointClass->hMemCacheTasks);
+    if (RT_LIKELY(pTask))
+    {
+        /* Initialize common parts. */
+        pTask->pvUser    = pvUser;
+        pTask->pEndpoint = pEndpoint;
+        /* Clear list pointers for safety. */
+        pTask->pPrev     = NULL;
+        pTask->pNext     = NULL;
+        pTask->tsNsStart = RTTimeNanoTS();
+        STAM_REL_COUNTER_INC(&pEndpoint->StatIoOpsStarted);
+    }
+
+    return pTask;
+}
+
+
+/**
+ * Puts a task in one of the caches.
+ *
+ * @returns nothing.
+ * @param   pEndpoint    The endpoint the task belongs to.
+ * @param   pTask        The task to cache.
+ */
+static void pdmR3AsyncCompletionPutTask(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, PPDMASYNCCOMPLETIONTASK pTask)
+{
+    PPDMASYNCCOMPLETIONEPCLASS pEndpointClass = pEndpoint->pEpClass;
+    uint64_t cNsRun = RTTimeNanoTS() - pTask->tsNsStart;
+
+    if (RT_UNLIKELY(cNsRun >= RT_NS_10SEC))
+        LogRel(("AsyncCompletion: Task %#p completed after %llu seconds\n", pTask, cNsRun / RT_NS_1SEC));
+
+    if (pEndpointClass->fGatherAdvancedStatistics)
+        pdmR3AsyncCompletionStatisticsRecordCompletionTime(pEndpoint, cNsRun);
+
+    RTMemCacheFree(pEndpointClass->hMemCacheTasks, pTask);
+}
+
+
+static unsigned
+pdmR3AsyncCompletionGetStatId(PPDMASYNCCOMPLETIONEPCLASS pEndpointClass, const char *pszUri)
+{
+    PPDMASYNCCOMPLETIONENDPOINT pEndpoint = pEndpointClass->pEndpointsHead;
+    const char *pszFilename = RTPathFilename(pszUri);
+    unsigned iStatId = 0;
+
+    while (pEndpoint)
+    {
+        if (   !RTStrCmp(RTPathFilename(pEndpoint->pszUri), pszFilename)
+            && pEndpoint->iStatId >= iStatId)
+            iStatId = pEndpoint->iStatId + 1;
+
+        pEndpoint = pEndpoint->pNext;
+    }
+
+    return iStatId;
+}
+
+/**
+ * Opens a file as an async completion endpoint.
+ *
+ * @returns VBox status code.
+ * @param   ppEndpoint      Where to store the opaque endpoint handle on success.
+ * @param   pszFilename     Path to the file which is to be opened. (UTF-8)
+ * @param   fFlags          Open flags, see grp_pdmacep_file_flags.
+ * @param   pTemplate       Handle to the completion callback template to use
+ *                          for this end point.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionEpCreateForFile(PPPDMASYNCCOMPLETIONENDPOINT ppEndpoint,
+                                                   const char *pszFilename, uint32_t fFlags,
+                                                   PPDMASYNCCOMPLETIONTEMPLATE pTemplate)
+{
+    LogFlowFunc((": ppEndpoint=%p pszFilename=%p{%s} fFlags=%u pTemplate=%p\n",
+                 ppEndpoint, pszFilename, pszFilename, fFlags, pTemplate));
+
+    /* Sanity checks. */
+    AssertPtrReturn(ppEndpoint,  VERR_INVALID_POINTER);
+    AssertPtrReturn(pszFilename, VERR_INVALID_POINTER);
+    AssertPtrReturn(pTemplate,   VERR_INVALID_POINTER);
+
+    /* Check that the flags are valid. */
+    AssertReturn(((~(PDMACEP_FILE_FLAGS_READ_ONLY | PDMACEP_FILE_FLAGS_DONT_LOCK | PDMACEP_FILE_FLAGS_HOST_CACHE_ENABLED) & fFlags) == 0),
+                 VERR_INVALID_PARAMETER);
+
+    PVM  pVM  = pTemplate->pVM;
+    PUVM pUVM = pVM->pUVM;
+    PPDMASYNCCOMPLETIONEPCLASS  pEndpointClass = pUVM->pdm.s.apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_FILE];
+    PPDMASYNCCOMPLETIONENDPOINT pEndpoint = NULL;
+
+    AssertMsg(pEndpointClass, ("File endpoint class was not initialized\n"));
+
+    /* Create an endpoint. */
+    int rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_ASYNC_COMPLETION,
+                              pEndpointClass->pEndpointOps->cbEndpoint,
+                              (void **)&pEndpoint);
+    if (RT_SUCCESS(rc))
+    {
+        /* Initialize common parts. */
+        pEndpoint->pNext             = NULL;
+        pEndpoint->pPrev             = NULL;
+        pEndpoint->pEpClass          = pEndpointClass;
+        pEndpoint->pTemplate         = pTemplate;
+        pEndpoint->pszUri            = RTStrDup(pszFilename);
+        pEndpoint->iStatId           = pdmR3AsyncCompletionGetStatId(pEndpointClass, pszFilename);
+        pEndpoint->pBwMgr            = NULL;
+
+        if (   pEndpoint->pszUri
+            && RT_SUCCESS(rc))
+        {
+            /* Call the initializer for the endpoint. */
+            rc = pEndpointClass->pEndpointOps->pfnEpInitialize(pEndpoint, pszFilename, fFlags);
+            if (RT_SUCCESS(rc))
+            {
+                if (pEndpointClass->fGatherAdvancedStatistics)
+                    rc = pdmR3AsyncCompletionStatisticsRegister(pEndpoint);
+
+                if (RT_SUCCESS(rc))
+                {
+                    /* Link it into the list of endpoints. */
+                    rc = RTCritSectEnter(&pEndpointClass->CritSect);
+                    AssertMsg(RT_SUCCESS(rc), ("Failed to enter critical section rc=%Rrc\n", rc));
+
+                    pEndpoint->pNext = pEndpointClass->pEndpointsHead;
+                    if (pEndpointClass->pEndpointsHead)
+                        pEndpointClass->pEndpointsHead->pPrev = pEndpoint;
+
+                    pEndpointClass->pEndpointsHead = pEndpoint;
+                    pEndpointClass->cEndpoints++;
+
+                    rc = RTCritSectLeave(&pEndpointClass->CritSect);
+                    AssertMsg(RT_SUCCESS(rc), ("Failed to enter critical section rc=%Rrc\n", rc));
+
+                    /* Reference the template. */
+                    ASMAtomicIncU32(&pTemplate->cUsed);
+
+                    *ppEndpoint = pEndpoint;
+                    LogFlowFunc((": Created endpoint for %s\n", pszFilename));
+                    return VINF_SUCCESS;
+                }
+                else
+                    pEndpointClass->pEndpointOps->pfnEpClose(pEndpoint);
+
+                if (pEndpointClass->fGatherAdvancedStatistics)
+                    pdmR3AsyncCompletionStatisticsDeregister(pEndpoint);
+            }
+            RTStrFree(pEndpoint->pszUri);
+        }
+        MMR3HeapFree(pEndpoint);
+    }
+
+    LogFlowFunc((": Creation of endpoint for %s failed: rc=%Rrc\n", pszFilename, rc));
+    return rc;
+}
+
+
+/**
+ * Closes a endpoint waiting for any pending tasks to finish.
+ *
+ * @returns nothing.
+ * @param   pEndpoint       Handle of the endpoint.
+ */
+VMMR3DECL(void) PDMR3AsyncCompletionEpClose(PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
+{
+    LogFlowFunc((": pEndpoint=%p\n", pEndpoint));
+
+    /* Sanity checks. */
+    AssertReturnVoid(VALID_PTR(pEndpoint));
+
+    PPDMASYNCCOMPLETIONEPCLASS pEndpointClass = pEndpoint->pEpClass;
+    pEndpointClass->pEndpointOps->pfnEpClose(pEndpoint);
+
+    /* Drop reference from the template. */
+    ASMAtomicDecU32(&pEndpoint->pTemplate->cUsed);
+
+    /* Unlink the endpoint from the list. */
+    int rc = RTCritSectEnter(&pEndpointClass->CritSect);
+    AssertMsg(RT_SUCCESS(rc), ("Failed to enter critical section rc=%Rrc\n", rc));
+
+    PPDMASYNCCOMPLETIONENDPOINT pEndpointNext = pEndpoint->pNext;
+    PPDMASYNCCOMPLETIONENDPOINT pEndpointPrev = pEndpoint->pPrev;
+
+    if (pEndpointPrev)
+        pEndpointPrev->pNext = pEndpointNext;
+    else
+        pEndpointClass->pEndpointsHead = pEndpointNext;
+    if (pEndpointNext)
+        pEndpointNext->pPrev = pEndpointPrev;
+
+    pEndpointClass->cEndpoints--;
+
+    rc = RTCritSectLeave(&pEndpointClass->CritSect);
+    AssertMsg(RT_SUCCESS(rc), ("Failed to enter critical section rc=%Rrc\n", rc));
+
+    if (pEndpointClass->fGatherAdvancedStatistics)
+        pdmR3AsyncCompletionStatisticsDeregister(pEndpoint);
+
+    RTStrFree(pEndpoint->pszUri);
+    MMR3HeapFree(pEndpoint);
+}
+
+
+/**
+ * Creates a read task on the given endpoint.
+ *
+ * @returns VBox status code.
+ * @param   pEndpoint       The file endpoint to read from.
+ * @param   off             Where to start reading from.
+ * @param   paSegments      Scatter gather list to store the data in.
+ * @param   cSegments       Number of segments in the list.
+ * @param   cbRead          The overall number of bytes to read.
+ * @param   pvUser          Opaque user data returned in the completion callback
+ *                          upon completion of the task.
+ * @param   ppTask          Where to store the task handle on success.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionEpRead(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                                          PCRTSGSEG paSegments, unsigned cSegments,
+                                          size_t cbRead, void *pvUser,
+                                          PPPDMASYNCCOMPLETIONTASK ppTask)
+{
+    AssertPtrReturn(pEndpoint, VERR_INVALID_POINTER);
+    AssertPtrReturn(paSegments, VERR_INVALID_POINTER);
+    AssertPtrReturn(ppTask, VERR_INVALID_POINTER);
+    AssertReturn(cSegments > 0, VERR_INVALID_PARAMETER);
+    AssertReturn(cbRead > 0, VERR_INVALID_PARAMETER);
+    AssertReturn(off >= 0, VERR_INVALID_PARAMETER);
+
+    PPDMASYNCCOMPLETIONTASK pTask;
+
+    pTask = pdmR3AsyncCompletionGetTask(pEndpoint, pvUser);
+    if (!pTask)
+        return VERR_NO_MEMORY;
+
+    int rc = pEndpoint->pEpClass->pEndpointOps->pfnEpRead(pTask, pEndpoint, off,
+                                                          paSegments, cSegments, cbRead);
+    if (RT_SUCCESS(rc))
+    {
+        if (pEndpoint->pEpClass->fGatherAdvancedStatistics)
+            pdmR3AsyncCompletionStatisticsRecordSize(pEndpoint, cbRead);
+
+        *ppTask = pTask;
+    }
+    else
+        pdmR3AsyncCompletionPutTask(pEndpoint, pTask);
+
+    return rc;
+}
+
+
+/**
+ * Creates a write task on the given endpoint.
+ *
+ * @returns VBox status code.
+ * @param   pEndpoint       The file endpoint to write to.
+ * @param   off             Where to start writing at.
+ * @param   paSegments      Scatter gather list of the data to write.
+ * @param   cSegments       Number of segments in the list.
+ * @param   cbWrite         The overall number of bytes to write.
+ * @param   pvUser          Opaque user data returned in the completion callback
+ *                          upon completion of the task.
+ * @param   ppTask          Where to store the task handle on success.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionEpWrite(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                                           PCRTSGSEG paSegments, unsigned cSegments,
+                                           size_t cbWrite, void *pvUser,
+                                           PPPDMASYNCCOMPLETIONTASK ppTask)
+{
+    AssertPtrReturn(pEndpoint, VERR_INVALID_POINTER);
+    AssertPtrReturn(paSegments, VERR_INVALID_POINTER);
+    AssertPtrReturn(ppTask, VERR_INVALID_POINTER);
+    AssertReturn(cSegments > 0, VERR_INVALID_PARAMETER);
+    AssertReturn(cbWrite > 0, VERR_INVALID_PARAMETER);
+    AssertReturn(off >= 0, VERR_INVALID_PARAMETER);
+
+    PPDMASYNCCOMPLETIONTASK pTask;
+
+    pTask = pdmR3AsyncCompletionGetTask(pEndpoint, pvUser);
+    if (!pTask)
+        return VERR_NO_MEMORY;
+
+    int rc = pEndpoint->pEpClass->pEndpointOps->pfnEpWrite(pTask, pEndpoint, off,
+                                                           paSegments, cSegments, cbWrite);
+    if (RT_SUCCESS(rc))
+    {
+        if (pEndpoint->pEpClass->fGatherAdvancedStatistics)
+            pdmR3AsyncCompletionStatisticsRecordSize(pEndpoint, cbWrite);
+
+        *ppTask = pTask;
+    }
+    else
+        pdmR3AsyncCompletionPutTask(pEndpoint, pTask);
+
+    return rc;
+}
+
+
+/**
+ * Creates a flush task on the given endpoint.
+ *
+ * Every read and write task initiated before the flush task is
+ * finished upon completion of this task.
+ *
+ * @returns VBox status code.
+ * @param   pEndpoint       The file endpoint to flush.
+ * @param   pvUser          Opaque user data returned in the completion callback
+ *                          upon completion of the task.
+ * @param   ppTask          Where to store the task handle on success.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionEpFlush(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, void *pvUser, PPPDMASYNCCOMPLETIONTASK ppTask)
+{
+    AssertPtrReturn(pEndpoint, VERR_INVALID_POINTER);
+    AssertPtrReturn(ppTask, VERR_INVALID_POINTER);
+
+    PPDMASYNCCOMPLETIONTASK pTask;
+
+    pTask = pdmR3AsyncCompletionGetTask(pEndpoint, pvUser);
+    if (!pTask)
+        return VERR_NO_MEMORY;
+
+    int rc = pEndpoint->pEpClass->pEndpointOps->pfnEpFlush(pTask, pEndpoint);
+    if (RT_SUCCESS(rc))
+        *ppTask = pTask;
+    else
+        pdmR3AsyncCompletionPutTask(pEndpoint, pTask);
+
+    return rc;
+}
+
+
+/**
+ * Queries the size of an endpoint.
+ *
+ * Not that some endpoints may not support this and will return an error
+ * (sockets for example).
+ *
+ * @returns VBox status code.
+ * @retval  VERR_NOT_SUPPORTED if the endpoint does not support this operation.
+ * @param   pEndpoint       The file endpoint.
+ * @param   pcbSize         Where to store the size of the endpoint.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionEpGetSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                             uint64_t *pcbSize)
+{
+    AssertPtrReturn(pEndpoint, VERR_INVALID_POINTER);
+    AssertPtrReturn(pcbSize, VERR_INVALID_POINTER);
+
+    if (pEndpoint->pEpClass->pEndpointOps->pfnEpGetSize)
+        return pEndpoint->pEpClass->pEndpointOps->pfnEpGetSize(pEndpoint, pcbSize);
+    return VERR_NOT_SUPPORTED;
+}
+
+
+/**
+ * Sets the size of an endpoint.
+ *
+ * Not that some endpoints may not support this and will return an error
+ * (sockets for example).
+ *
+ * @returns VBox status code.
+ * @retval  VERR_NOT_SUPPORTED if the endpoint does not support this operation.
+ * @param   pEndpoint       The file endpoint.
+ * @param   cbSize          The size to set.
+ *
+ * @note PDMR3AsyncCompletionEpFlush should be called before this operation is executed.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionEpSetSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint64_t cbSize)
+{
+    AssertPtrReturn(pEndpoint, VERR_INVALID_POINTER);
+
+    if (pEndpoint->pEpClass->pEndpointOps->pfnEpSetSize)
+        return pEndpoint->pEpClass->pEndpointOps->pfnEpSetSize(pEndpoint, cbSize);
+    return VERR_NOT_SUPPORTED;
+}
+
+
+/**
+ * Assigns or removes a bandwidth control manager to/from the endpoint.
+ *
+ * @returns VBox status code.
+ * @param   pEndpoint       The endpoint.
+ * @param   pszBwMgr        The identifer of the new bandwidth manager to assign
+ *                          or NULL to remove the current one.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionEpSetBwMgr(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, const char *pszBwMgr)
+{
+    AssertPtrReturn(pEndpoint, VERR_INVALID_POINTER);
+    PPDMACBWMGR pBwMgrOld = NULL;
+    PPDMACBWMGR pBwMgrNew = NULL;
+
+    int rc = VINF_SUCCESS;
+    if (pszBwMgr)
+    {
+        pBwMgrNew = pdmacBwMgrFindById(pEndpoint->pEpClass, pszBwMgr);
+        if (pBwMgrNew)
+            pdmacBwMgrRetain(pBwMgrNew);
+        else
+            rc = VERR_NOT_FOUND;
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        pBwMgrOld = ASMAtomicXchgPtrT(&pEndpoint->pBwMgr, pBwMgrNew, PPDMACBWMGR);
+        if (pBwMgrOld)
+            pdmacBwMgrRelease(pBwMgrOld);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Cancels an async completion task.
+ *
+ * If you want to use this method, you have to take great create to make sure
+ * you will never attempt cancel a task which has been completed. Since there is
+ * no reference counting or anything on the task it self, you have to serialize
+ * the cancelation and completion paths such that the aren't racing one another.
+ *
+ * @returns VBox status code
+ * @param   pTask           The Task to cancel.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionTaskCancel(PPDMASYNCCOMPLETIONTASK pTask)
+{
+    NOREF(pTask);
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * Changes the limit of a bandwidth manager for file endpoints to the given value.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszBwMgr        The identifer of the bandwidth manager to change.
+ * @param   cbMaxNew        The new maximum for the bandwidth manager in bytes/sec.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionBwMgrSetMaxForFile(PUVM pUVM, const char *pszBwMgr, uint32_t cbMaxNew)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszBwMgr, VERR_INVALID_POINTER);
+
+    int                         rc       = VINF_SUCCESS;
+    PPDMASYNCCOMPLETIONEPCLASS  pEpClass = pVM->pUVM->pdm.s.apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_FILE];
+    PPDMACBWMGR                 pBwMgr   = pdmacBwMgrFindById(pEpClass, pszBwMgr);
+    if (pBwMgr)
+    {
+        /*
+         * Set the new value for the start and max value to let the manager pick up
+         * the new limit immediately.
+         */
+        ASMAtomicWriteU32(&pBwMgr->cbTransferPerSecMax, cbMaxNew);
+        ASMAtomicWriteU32(&pBwMgr->cbTransferPerSecStart, cbMaxNew);
+    }
+    else
+        rc = VERR_NOT_FOUND;
+
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/PDMAsyncCompletionFile.cpp b/src/VBox/VMM/VMMR3/PDMAsyncCompletionFile.cpp
new file mode 100644
index 00000000..616d98ba
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMAsyncCompletionFile.cpp
@@ -0,0 +1,1293 @@
+/* $Id: PDMAsyncCompletionFile.cpp $ */
+/** @file
+ * PDM Async I/O - Transport data asynchronous in R3 using EMT.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/dbg.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/tm.h>
+
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/critsect.h>
+#include <iprt/env.h>
+#include <iprt/file.h>
+#include <iprt/mem.h>
+#include <iprt/semaphore.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/path.h>
+#include <iprt/rand.h>
+
+#include "PDMAsyncCompletionFileInternal.h"
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_DEBUGGER
+static FNDBGCCMD pdmacEpFileErrorInject;
+# ifdef PDM_ASYNC_COMPLETION_FILE_WITH_DELAY
+static FNDBGCCMD pdmacEpFileDelayInject;
+# endif
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_DEBUGGER
+static const DBGCVARDESC g_aInjectErrorArgs[] =
+{
+    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
+    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "direction",    "write/read." },
+    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "filename",     "Filename." },
+    {  1,           1,          DBGCVAR_CAT_NUMBER,     0,                              "errcode",      "VBox status code." },
+};
+
+# ifdef PDM_ASYNC_COMPLETION_FILE_WITH_DELAY
+static const DBGCVARDESC g_aInjectDelayArgs[] =
+{
+    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
+    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "direction",    "write|read|flush|any." },
+    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "filename",     "Filename." },
+    {  1,           1,          DBGCVAR_CAT_NUMBER,     0,                              "delay",        "Delay in milliseconds." },
+    {  1,           1,          DBGCVAR_CAT_NUMBER,     0,                              "jitter",       "Jitter of the delay." },
+    {  1,           1,          DBGCVAR_CAT_NUMBER,     0,                              "reqs",         "Number of requests to delay." }
+
+};
+# endif
+
+/** Command descriptors. */
+static const DBGCCMD g_aCmds[] =
+{
+    /* pszCmd,       cArgsMin, cArgsMax, paArgDesc,                                    cArgDescs, fFlags, pfnHandler              pszSyntax,.pszDescription */
+    { "injecterror",        3, 3,        &g_aInjectErrorArgs[0],                               3,      0, pdmacEpFileErrorInject, "",        "Inject error into I/O subsystem." }
+# ifdef PDM_ASYNC_COMPLETION_FILE_WITH_DELAY
+    ,{ "injectdelay",       3, 5,        &g_aInjectDelayArgs[0], RT_ELEMENTS(g_aInjectDelayArgs),      0, pdmacEpFileDelayInject, "",        "Inject a delay of a request." }
+# endif
+};
+#endif
+
+
+/**
+ * Frees a task.
+ *
+ * @returns nothing.
+ * @param   pEndpoint    Pointer to the endpoint the segment was for.
+ * @param   pTask        The task to free.
+ */
+void pdmacFileTaskFree(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask)
+{
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
+
+    LogFlowFunc((": pEndpoint=%p pTask=%p\n", pEndpoint, pTask));
+
+    /* Try the per endpoint cache first. */
+    if (pEndpoint->cTasksCached < pEpClass->cTasksCacheMax)
+    {
+        /* Add it to the list. */
+        pEndpoint->pTasksFreeTail->pNext = pTask;
+        pEndpoint->pTasksFreeTail        = pTask;
+        ASMAtomicIncU32(&pEndpoint->cTasksCached);
+    }
+    else
+    {
+        Log(("Freeing task %p because all caches are full\n", pTask));
+        MMR3HeapFree(pTask);
+    }
+}
+
+/**
+ * Allocates a task segment
+ *
+ * @returns Pointer to the new task segment or NULL
+ * @param   pEndpoint    Pointer to the endpoint
+ */
+PPDMACTASKFILE pdmacFileTaskAlloc(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    PPDMACTASKFILE pTask = NULL;
+
+    /* Try the small per endpoint cache first. */
+    if (pEndpoint->pTasksFreeHead == pEndpoint->pTasksFreeTail)
+    {
+        /* Try the bigger endpoint class cache. */
+        PPDMASYNCCOMPLETIONEPCLASSFILE pEndpointClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
+
+        /*
+         * Allocate completely new.
+         * If this fails we return NULL.
+         */
+        int rc = MMR3HeapAllocZEx(pEndpointClass->Core.pVM, MM_TAG_PDM_ASYNC_COMPLETION,
+                                  sizeof(PDMACTASKFILE),
+                                  (void **)&pTask);
+        if (RT_FAILURE(rc))
+            pTask = NULL;
+
+        LogFlow(("Allocated task %p\n", pTask));
+    }
+    else
+    {
+        /* Grab a free task from the head. */
+        AssertMsg(pEndpoint->cTasksCached > 0, ("No tasks cached but list contains more than one element\n"));
+
+        pTask = pEndpoint->pTasksFreeHead;
+        pEndpoint->pTasksFreeHead = pTask->pNext;
+        ASMAtomicDecU32(&pEndpoint->cTasksCached);
+    }
+
+    pTask->pNext = NULL;
+
+    return pTask;
+}
+
+PPDMACTASKFILE pdmacFileEpGetNewTasks(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    /*
+     * Get pending tasks.
+     */
+    PPDMACTASKFILE pTasks = ASMAtomicXchgPtrT(&pEndpoint->pTasksNewHead, NULL, PPDMACTASKFILE);
+
+    /* Reverse the list to process in FIFO order. */
+    if (pTasks)
+    {
+        PPDMACTASKFILE pTask = pTasks;
+
+        pTasks = NULL;
+
+        while (pTask)
+        {
+            PPDMACTASKFILE pCur = pTask;
+            pTask = pTask->pNext;
+            pCur->pNext = pTasks;
+            pTasks = pCur;
+        }
+    }
+
+    return pTasks;
+}
+
+static void pdmacFileAioMgrWakeup(PPDMACEPFILEMGR pAioMgr)
+{
+    bool fWokenUp = ASMAtomicXchgBool(&pAioMgr->fWokenUp, true);
+    if (!fWokenUp)
+    {
+        bool fWaitingEventSem = ASMAtomicReadBool(&pAioMgr->fWaitingEventSem);
+        if (fWaitingEventSem)
+        {
+            int rc = RTSemEventSignal(pAioMgr->EventSem);
+            AssertRC(rc);
+        }
+    }
+}
+
+static int pdmacFileAioMgrWaitForBlockingEvent(PPDMACEPFILEMGR pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT enmEvent)
+{
+    ASMAtomicWriteU32((volatile uint32_t *)&pAioMgr->enmBlockingEvent, enmEvent);
+    Assert(!pAioMgr->fBlockingEventPending);
+    ASMAtomicXchgBool(&pAioMgr->fBlockingEventPending, true);
+
+    /* Wakeup the async I/O manager */
+    pdmacFileAioMgrWakeup(pAioMgr);
+
+    /* Wait for completion. */
+    int rc = RTSemEventWait(pAioMgr->EventSemBlock, RT_INDEFINITE_WAIT);
+    AssertRC(rc);
+
+    ASMAtomicXchgBool(&pAioMgr->fBlockingEventPending, false);
+    ASMAtomicWriteU32((volatile uint32_t *)&pAioMgr->enmBlockingEvent, PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID);
+
+    return rc;
+}
+
+int pdmacFileAioMgrAddEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    LogFlowFunc(("pAioMgr=%#p pEndpoint=%#p{%s}\n", pAioMgr, pEndpoint, pEndpoint->Core.pszUri));
+
+    /* Update the assigned I/O manager. */
+    ASMAtomicWritePtr(&pEndpoint->pAioMgr, pAioMgr);
+
+    int rc = RTCritSectEnter(&pAioMgr->CritSectBlockingEvent);
+    AssertRCReturn(rc, rc);
+
+    ASMAtomicWritePtr(&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint, pEndpoint);
+    rc = pdmacFileAioMgrWaitForBlockingEvent(pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT);
+    ASMAtomicWriteNullPtr(&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint);
+
+    RTCritSectLeave(&pAioMgr->CritSectBlockingEvent);
+
+    return rc;
+}
+
+#ifdef SOME_UNUSED_FUNCTION
+static int pdmacFileAioMgrRemoveEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    int rc = RTCritSectEnter(&pAioMgr->CritSectBlockingEvent);
+    AssertRCReturn(rc, rc);
+
+    ASMAtomicWritePtr(&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint, pEndpoint);
+    rc = pdmacFileAioMgrWaitForBlockingEvent(pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT);
+    ASMAtomicWriteNullPtr(&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint);
+
+    RTCritSectLeave(&pAioMgr->CritSectBlockingEvent);
+
+    return rc;
+}
+#endif
+
+static int pdmacFileAioMgrCloseEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    int rc = RTCritSectEnter(&pAioMgr->CritSectBlockingEvent);
+    AssertRCReturn(rc, rc);
+
+    ASMAtomicWritePtr(&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint, pEndpoint);
+    rc = pdmacFileAioMgrWaitForBlockingEvent(pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT);
+    ASMAtomicWriteNullPtr(&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint);
+
+    RTCritSectLeave(&pAioMgr->CritSectBlockingEvent);
+
+    return rc;
+}
+
+static int pdmacFileAioMgrShutdown(PPDMACEPFILEMGR pAioMgr)
+{
+    int rc = RTCritSectEnter(&pAioMgr->CritSectBlockingEvent);
+    AssertRCReturn(rc, rc);
+
+    rc = pdmacFileAioMgrWaitForBlockingEvent(pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN);
+
+    RTCritSectLeave(&pAioMgr->CritSectBlockingEvent);
+
+    return rc;
+}
+
+int pdmacFileEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask)
+{
+    PPDMACTASKFILE pNext;
+    do
+    {
+        pNext = pEndpoint->pTasksNewHead;
+        pTask->pNext = pNext;
+    } while (!ASMAtomicCmpXchgPtr(&pEndpoint->pTasksNewHead, pTask, pNext));
+
+    pdmacFileAioMgrWakeup(ASMAtomicReadPtrT(&pEndpoint->pAioMgr, PPDMACEPFILEMGR));
+
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(void) pdmacFileEpTaskCompleted(PPDMACTASKFILE pTask, void *pvUser, int rc)
+{
+    PPDMASYNCCOMPLETIONTASKFILE pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pvUser;
+
+    LogFlowFunc(("pTask=%#p pvUser=%#p rc=%Rrc\n", pTask, pvUser, rc));
+
+    if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
+        pdmR3AsyncCompletionCompleteTask(&pTaskFile->Core, rc, true);
+    else
+    {
+        Assert((uint32_t)pTask->DataSeg.cbSeg == pTask->DataSeg.cbSeg && (int32_t)pTask->DataSeg.cbSeg >= 0);
+        uint32_t uOld = ASMAtomicSubS32(&pTaskFile->cbTransferLeft, (int32_t)pTask->DataSeg.cbSeg);
+
+        /* The first error will be returned. */
+        if (RT_FAILURE(rc))
+            ASMAtomicCmpXchgS32(&pTaskFile->rc, rc, VINF_SUCCESS);
+#ifdef VBOX_WITH_DEBUGGER
+        else
+        {
+            PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pTaskFile->Core.pEndpoint;
+
+            /* Overwrite with injected error code. */
+            if (pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
+                rc = ASMAtomicXchgS32(&pEpFile->rcReqRead, VINF_SUCCESS);
+            else
+                rc = ASMAtomicXchgS32(&pEpFile->rcReqWrite, VINF_SUCCESS);
+
+            if (RT_FAILURE(rc))
+                ASMAtomicCmpXchgS32(&pTaskFile->rc, rc, VINF_SUCCESS);
+        }
+#endif
+
+        if (!(uOld - pTask->DataSeg.cbSeg)
+            && !ASMAtomicXchgBool(&pTaskFile->fCompleted, true))
+        {
+#ifdef PDM_ASYNC_COMPLETION_FILE_WITH_DELAY
+            PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pTaskFile->Core.pEndpoint;
+            PPDMASYNCCOMPLETIONEPCLASSFILE  pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEpFile->Core.pEpClass;
+
+            /* Check if we should delay completion of the request. */
+            if (   ASMAtomicReadU32(&pEpFile->msDelay) > 0
+                && ASMAtomicReadU32(&pEpFile->cReqsDelay) > 0)
+            {
+                uint64_t tsDelay = pEpFile->msDelay;
+
+                if (pEpFile->msJitter)
+                    tsDelay = (RTRandU32() % 100) > 50 ? pEpFile->msDelay + (RTRandU32() % pEpFile->msJitter)
+                                                       : pEpFile->msDelay - (RTRandU32() % pEpFile->msJitter);
+                ASMAtomicDecU32(&pEpFile->cReqsDelay);
+
+                /* Arm the delay. */
+                pTaskFile->tsDelayEnd = RTTimeProgramMilliTS() + tsDelay;
+
+                /* Append to the list. */
+                PPDMASYNCCOMPLETIONTASKFILE pHead = NULL;
+                do
+                {
+                    pHead = ASMAtomicReadPtrT(&pEpFile->pDelayedHead, PPDMASYNCCOMPLETIONTASKFILE);
+                    pTaskFile->pDelayedNext = pHead;
+                } while (!ASMAtomicCmpXchgPtr(&pEpFile->pDelayedHead, pTaskFile, pHead));
+
+                if (tsDelay < pEpClassFile->cMilliesNext)
+                {
+                    ASMAtomicWriteU64(&pEpClassFile->cMilliesNext, tsDelay);
+                    TMTimerSetMillies(pEpClassFile->pTimer, tsDelay);
+                }
+
+                LogRel(("AIOMgr: Delaying request %#p for %u ms\n", pTaskFile, tsDelay));
+            }
+            else
+#endif
+                pdmR3AsyncCompletionCompleteTask(&pTaskFile->Core, pTaskFile->rc, true);
+        }
+    }
+}
+
+DECLINLINE(void) pdmacFileEpTaskInit(PPDMASYNCCOMPLETIONTASK pTask, size_t cbTransfer)
+{
+    PPDMASYNCCOMPLETIONTASKFILE pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pTask;
+
+    Assert((uint32_t)cbTransfer == cbTransfer && (int32_t)cbTransfer >= 0);
+    ASMAtomicWriteS32(&pTaskFile->cbTransferLeft, (int32_t)cbTransfer);
+    ASMAtomicWriteBool(&pTaskFile->fCompleted, false);
+    ASMAtomicWriteS32(&pTaskFile->rc, VINF_SUCCESS);
+}
+
+int pdmacFileEpTaskInitiate(PPDMASYNCCOMPLETIONTASK pTask,
+                            PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                            PCRTSGSEG paSegments, size_t cSegments,
+                            size_t cbTransfer, PDMACTASKFILETRANSFER enmTransfer)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+    PPDMASYNCCOMPLETIONTASKFILE pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pTask;
+
+    Assert(   (enmTransfer == PDMACTASKFILETRANSFER_READ)
+           || (enmTransfer == PDMACTASKFILETRANSFER_WRITE));
+
+    for (size_t i = 0; i < cSegments; i++)
+    {
+        PPDMACTASKFILE pIoTask = pdmacFileTaskAlloc(pEpFile);
+        AssertPtr(pIoTask);
+
+        pIoTask->pEndpoint       = pEpFile;
+        pIoTask->enmTransferType = enmTransfer;
+        pIoTask->Off             = off;
+        pIoTask->DataSeg.cbSeg   = paSegments[i].cbSeg;
+        pIoTask->DataSeg.pvSeg   = paSegments[i].pvSeg;
+        pIoTask->pvUser          = pTaskFile;
+        pIoTask->pfnCompleted    = pdmacFileEpTaskCompleted;
+
+        /* Send it off to the I/O manager. */
+        pdmacFileEpAddTask(pEpFile, pIoTask);
+        off        += paSegments[i].cbSeg;
+        cbTransfer -= paSegments[i].cbSeg;
+    }
+
+    AssertMsg(!cbTransfer, ("Incomplete transfer %u bytes left\n", cbTransfer));
+
+    return VINF_AIO_TASK_PENDING;
+}
+
+/**
+ * Creates a new async I/O manager.
+ *
+ * @returns VBox status code.
+ * @param   pEpClass    Pointer to the endpoint class data.
+ * @param   ppAioMgr    Where to store the pointer to the new async I/O manager on success.
+ * @param   enmMgrType  Wanted manager type - can be overwritten by the global override.
+ */
+int pdmacFileAioMgrCreate(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass, PPPDMACEPFILEMGR ppAioMgr,
+                          PDMACEPFILEMGRTYPE enmMgrType)
+{
+    LogFlowFunc((": Entered\n"));
+
+    PPDMACEPFILEMGR pAioMgrNew;
+    int rc = MMR3HeapAllocZEx(pEpClass->Core.pVM, MM_TAG_PDM_ASYNC_COMPLETION, sizeof(PDMACEPFILEMGR), (void **)&pAioMgrNew);
+    if (RT_SUCCESS(rc))
+    {
+        if (enmMgrType < pEpClass->enmMgrTypeOverride)
+            pAioMgrNew->enmMgrType = enmMgrType;
+        else
+            pAioMgrNew->enmMgrType = pEpClass->enmMgrTypeOverride;
+
+        pAioMgrNew->msBwLimitExpired = RT_INDEFINITE_WAIT;
+
+        rc = RTSemEventCreate(&pAioMgrNew->EventSem);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTSemEventCreate(&pAioMgrNew->EventSemBlock);
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTCritSectInit(&pAioMgrNew->CritSectBlockingEvent);
+                if (RT_SUCCESS(rc))
+                {
+                    /* Init the rest of the manager. */
+                    if (pAioMgrNew->enmMgrType != PDMACEPFILEMGRTYPE_SIMPLE)
+                        rc = pdmacFileAioMgrNormalInit(pAioMgrNew);
+
+                    if (RT_SUCCESS(rc))
+                    {
+                        pAioMgrNew->enmState = PDMACEPFILEMGRSTATE_RUNNING;
+
+                        rc = RTThreadCreateF(&pAioMgrNew->Thread,
+                                             pAioMgrNew->enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE
+                                             ? pdmacFileAioMgrFailsafe
+                                             : pdmacFileAioMgrNormal,
+                                             pAioMgrNew,
+                                             0,
+                                             RTTHREADTYPE_IO,
+                                             0,
+                                             "AioMgr%d-%s", pEpClass->cAioMgrs,
+                                             pAioMgrNew->enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE
+                                             ? "F"
+                                             : "N");
+                        if (RT_SUCCESS(rc))
+                        {
+                            /* Link it into the list. */
+                            RTCritSectEnter(&pEpClass->CritSect);
+                            pAioMgrNew->pNext = pEpClass->pAioMgrHead;
+                            if (pEpClass->pAioMgrHead)
+                                pEpClass->pAioMgrHead->pPrev = pAioMgrNew;
+                            pEpClass->pAioMgrHead = pAioMgrNew;
+                            pEpClass->cAioMgrs++;
+                            RTCritSectLeave(&pEpClass->CritSect);
+
+                            *ppAioMgr = pAioMgrNew;
+
+                            Log(("PDMAC: Successfully created new file AIO Mgr {%s}\n", RTThreadGetName(pAioMgrNew->Thread)));
+                            return VINF_SUCCESS;
+                        }
+                        pdmacFileAioMgrNormalDestroy(pAioMgrNew);
+                    }
+                    RTCritSectDelete(&pAioMgrNew->CritSectBlockingEvent);
+                }
+                RTSemEventDestroy(pAioMgrNew->EventSem);
+            }
+            RTSemEventDestroy(pAioMgrNew->EventSemBlock);
+        }
+        MMR3HeapFree(pAioMgrNew);
+    }
+
+    LogFlowFunc((": Leave rc=%Rrc\n", rc));
+
+    return rc;
+}
+
+/**
+ * Destroys a async I/O manager.
+ *
+ * @returns nothing.
+ * @param   pEpClassFile    Pointer to globals for the file endpoint class.
+ * @param   pAioMgr         The async I/O manager to destroy.
+ */
+static void pdmacFileAioMgrDestroy(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile, PPDMACEPFILEMGR pAioMgr)
+{
+    int rc = pdmacFileAioMgrShutdown(pAioMgr);
+    AssertRC(rc);
+
+    /* Unlink from the list. */
+    rc = RTCritSectEnter(&pEpClassFile->CritSect);
+    AssertRC(rc);
+
+    PPDMACEPFILEMGR pPrev = pAioMgr->pPrev;
+    PPDMACEPFILEMGR pNext = pAioMgr->pNext;
+
+    if (pPrev)
+        pPrev->pNext = pNext;
+    else
+        pEpClassFile->pAioMgrHead = pNext;
+
+    if (pNext)
+        pNext->pPrev = pPrev;
+
+    pEpClassFile->cAioMgrs--;
+    rc = RTCritSectLeave(&pEpClassFile->CritSect);
+    AssertRC(rc);
+
+    /* Free the resources. */
+    RTCritSectDelete(&pAioMgr->CritSectBlockingEvent);
+    RTSemEventDestroy(pAioMgr->EventSem);
+    RTSemEventDestroy(pAioMgr->EventSemBlock);
+    if (pAioMgr->enmMgrType != PDMACEPFILEMGRTYPE_SIMPLE)
+        pdmacFileAioMgrNormalDestroy(pAioMgr);
+
+    MMR3HeapFree(pAioMgr);
+}
+
+static int pdmacFileMgrTypeFromName(const char *pszVal, PPDMACEPFILEMGRTYPE penmMgrType)
+{
+    int rc = VINF_SUCCESS;
+
+    if (!RTStrCmp(pszVal, "Simple"))
+        *penmMgrType = PDMACEPFILEMGRTYPE_SIMPLE;
+    else if (!RTStrCmp(pszVal, "Async"))
+        *penmMgrType = PDMACEPFILEMGRTYPE_ASYNC;
+    else
+        rc = VERR_CFGM_CONFIG_UNKNOWN_VALUE;
+
+    return rc;
+}
+
+static const char *pdmacFileMgrTypeToName(PDMACEPFILEMGRTYPE enmMgrType)
+{
+    if (enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE)
+        return "Simple";
+    if (enmMgrType == PDMACEPFILEMGRTYPE_ASYNC)
+        return "Async";
+
+    return NULL;
+}
+
+static int pdmacFileBackendTypeFromName(const char *pszVal, PPDMACFILEEPBACKEND penmBackendType)
+{
+    int rc = VINF_SUCCESS;
+
+    if (!RTStrCmp(pszVal, "Buffered"))
+        *penmBackendType = PDMACFILEEPBACKEND_BUFFERED;
+    else if (!RTStrCmp(pszVal, "NonBuffered"))
+        *penmBackendType = PDMACFILEEPBACKEND_NON_BUFFERED;
+    else
+        rc = VERR_CFGM_CONFIG_UNKNOWN_VALUE;
+
+    return rc;
+}
+
+static const char *pdmacFileBackendTypeToName(PDMACFILEEPBACKEND enmBackendType)
+{
+    if (enmBackendType == PDMACFILEEPBACKEND_BUFFERED)
+        return "Buffered";
+    if (enmBackendType == PDMACFILEEPBACKEND_NON_BUFFERED)
+        return "NonBuffered";
+
+    return NULL;
+}
+
+#ifdef VBOX_WITH_DEBUGGER
+
+/**
+ * @callback_method_impl{FNDBGCCMD, The '.injecterror' command.}
+ */
+static DECLCALLBACK(int) pdmacEpFileErrorInject(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR pArgs, unsigned cArgs)
+{
+    /*
+     * Validate input.
+     */
+    DBGC_CMDHLP_REQ_UVM_RET(pCmdHlp, pCmd, pUVM);
+    DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, -1, cArgs == 3);
+    DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, 0, pArgs[0].enmType == DBGCVAR_TYPE_STRING);
+    DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, 1, pArgs[1].enmType == DBGCVAR_TYPE_STRING);
+    DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, 2, pArgs[2].enmType == DBGCVAR_TYPE_NUMBER);
+
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile;
+    pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pUVM->pdm.s.apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_FILE];
+
+    /* Syntax is "read|write <filename> <status code>" */
+    bool fWrite;
+    if (!RTStrCmp(pArgs[0].u.pszString, "read"))
+        fWrite = false;
+    else if (!RTStrCmp(pArgs[0].u.pszString, "write"))
+        fWrite = true;
+    else
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "invalid transfer direction '%s'", pArgs[0].u.pszString);
+
+    int32_t rcToInject = (int32_t)pArgs[2].u.u64Number;
+    if ((uint64_t)rcToInject != pArgs[2].u.u64Number)
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "The status code '%lld' is out of range", pArgs[0].u.u64Number);
+
+    /*
+     * Search for the matching endpoint.
+     */
+    RTCritSectEnter(&pEpClassFile->Core.CritSect);
+
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEpClassFile->Core.pEndpointsHead;
+    while (pEpFile)
+    {
+        if (!RTStrCmp(pArgs[1].u.pszString, RTPathFilename(pEpFile->Core.pszUri)))
+            break;
+        pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEpFile->Core.pNext;
+    }
+
+    if (pEpFile)
+    {
+        /*
+         * Do the job.
+         */
+        if (fWrite)
+            ASMAtomicXchgS32(&pEpFile->rcReqWrite, rcToInject);
+        else
+            ASMAtomicXchgS32(&pEpFile->rcReqRead,  rcToInject);
+
+        DBGCCmdHlpPrintf(pCmdHlp, "Injected %Rrc into '%s' for %s\n",
+                         (int)rcToInject, pArgs[1].u.pszString, pArgs[0].u.pszString);
+    }
+
+    RTCritSectLeave(&pEpClassFile->Core.CritSect);
+
+    if (!pEpFile)
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "No file with name '%s' found", pArgs[1].u.pszString);
+    return VINF_SUCCESS;
+}
+
+# ifdef PDM_ASYNC_COMPLETION_FILE_WITH_DELAY
+/**
+ * @callback_method_impl{FNDBGCCMD, The '.injectdelay' command.}
+ */
+static DECLCALLBACK(int) pdmacEpFileDelayInject(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR pArgs, unsigned cArgs)
+{
+    /*
+     * Validate input.
+     */
+    DBGC_CMDHLP_REQ_UVM_RET(pCmdHlp, pCmd, pUVM);
+    DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, -1, cArgs >= 3);
+    DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, 0, pArgs[0].enmType == DBGCVAR_TYPE_STRING);
+    DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, 1, pArgs[1].enmType == DBGCVAR_TYPE_STRING);
+    DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, 2, pArgs[2].enmType == DBGCVAR_TYPE_NUMBER);
+
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile;
+    pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pUVM->pdm.s.apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_FILE];
+
+    /* Syntax is "read|write|flush|any <filename> <delay> [reqs]" */
+    PDMACFILEREQTYPEDELAY enmDelayType = PDMACFILEREQTYPEDELAY_ANY;
+    if (!RTStrCmp(pArgs[0].u.pszString, "read"))
+        enmDelayType = PDMACFILEREQTYPEDELAY_READ;
+    else if (!RTStrCmp(pArgs[0].u.pszString, "write"))
+        enmDelayType = PDMACFILEREQTYPEDELAY_WRITE;
+    else if (!RTStrCmp(pArgs[0].u.pszString, "flush"))
+        enmDelayType = PDMACFILEREQTYPEDELAY_FLUSH;
+    else if (!RTStrCmp(pArgs[0].u.pszString, "any"))
+        enmDelayType = PDMACFILEREQTYPEDELAY_ANY;
+    else
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "invalid transfer direction '%s'", pArgs[0].u.pszString);
+
+    uint32_t msDelay = (uint32_t)pArgs[2].u.u64Number;
+    if ((uint64_t)msDelay != pArgs[2].u.u64Number)
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "The delay '%lld' is out of range", pArgs[0].u.u64Number);
+
+    uint32_t cReqsDelay = 1;
+    uint32_t msJitter = 0;
+    if (cArgs >= 4)
+        msJitter = (uint32_t)pArgs[3].u.u64Number;
+    if (cArgs == 5)
+        cReqsDelay = (uint32_t)pArgs[4].u.u64Number;
+
+    /*
+     * Search for the matching endpoint.
+     */
+    RTCritSectEnter(&pEpClassFile->Core.CritSect);
+
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEpClassFile->Core.pEndpointsHead;
+    while (pEpFile)
+    {
+        if (!RTStrCmp(pArgs[1].u.pszString, RTPathFilename(pEpFile->Core.pszUri)))
+            break;
+        pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEpFile->Core.pNext;
+    }
+
+    if (pEpFile)
+    {
+        ASMAtomicWriteSize(&pEpFile->enmTypeDelay, enmDelayType);
+        ASMAtomicWriteU32(&pEpFile->msDelay, msDelay);
+        ASMAtomicWriteU32(&pEpFile->msJitter, msJitter);
+        ASMAtomicWriteU32(&pEpFile->cReqsDelay, cReqsDelay);
+
+        DBGCCmdHlpPrintf(pCmdHlp, "Injected delay for the next %u requests of %u ms into '%s' for %s\n",
+                         cReqsDelay, msDelay, pArgs[1].u.pszString, pArgs[0].u.pszString);
+    }
+
+    RTCritSectLeave(&pEpClassFile->Core.CritSect);
+
+    if (!pEpFile)
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "No file with name '%s' found", pArgs[1].u.pszString);
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(void) pdmacR3TimerCallback(PVM pVM, PTMTIMER pTimer, void *pvUser)
+{
+    uint64_t tsCur = RTTimeProgramMilliTS();
+    uint64_t cMilliesNext = UINT64_MAX;
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pvUser;
+
+    ASMAtomicWriteU64(&pEpClassFile->cMilliesNext, UINT64_MAX);
+
+    /* Go through all endpoints and check for expired requests. */
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEpClassFile->Core.pEndpointsHead;
+
+    while (pEpFile)
+    {
+        /* Check for an expired delay. */
+        if (pEpFile->pDelayedHead != NULL)
+        {
+            PPDMASYNCCOMPLETIONTASKFILE pTaskFile = ASMAtomicXchgPtrT(&pEpFile->pDelayedHead, NULL, PPDMASYNCCOMPLETIONTASKFILE);
+
+            while (pTaskFile)
+            {
+                PPDMASYNCCOMPLETIONTASKFILE pTmp = pTaskFile;
+                pTaskFile = pTaskFile->pDelayedNext;
+
+                if (tsCur >= pTmp->tsDelayEnd)
+                {
+                    LogRel(("AIOMgr: Delayed request %#p completed\n", pTmp));
+                    pdmR3AsyncCompletionCompleteTask(&pTmp->Core, pTmp->rc, true);
+                }
+                else
+                {
+                    /* Prepend to the delayed list again. */
+                    PPDMASYNCCOMPLETIONTASKFILE pHead = NULL;
+
+                    if (pTmp->tsDelayEnd - tsCur < cMilliesNext)
+                        cMilliesNext = pTmp->tsDelayEnd - tsCur;
+
+                    do
+                    {
+                        pHead = ASMAtomicReadPtrT(&pEpFile->pDelayedHead, PPDMASYNCCOMPLETIONTASKFILE);
+                        pTmp->pDelayedNext = pHead;
+                    } while (!ASMAtomicCmpXchgPtr(&pEpFile->pDelayedHead, pTmp, pHead));
+                }
+            }
+        }
+
+        pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEpFile->Core.pNext;
+    }
+
+    if (cMilliesNext < pEpClassFile->cMilliesNext)
+    {
+        ASMAtomicWriteU64(&pEpClassFile->cMilliesNext, cMilliesNext);
+        TMTimerSetMillies(pEpClassFile->pTimer, cMilliesNext);
+    }
+}
+
+# endif /* PDM_ASYNC_COMPLETION_FILE_WITH_DELAY */
+
+#endif /* VBOX_WITH_DEBUGGER */
+
+static DECLCALLBACK(int) pdmacFileInitialize(PPDMASYNCCOMPLETIONEPCLASS pClassGlobals, PCFGMNODE pCfgNode)
+{
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pClassGlobals;
+    RTFILEAIOLIMITS                AioLimits; /** < Async I/O limitations. */
+
+    int rc = RTFileAioGetLimits(&AioLimits);
+#ifdef DEBUG
+    if (RT_SUCCESS(rc) && RTEnvExist("VBOX_ASYNC_IO_FAILBACK"))
+        rc = VERR_ENV_VAR_NOT_FOUND;
+#endif
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("AIO: Async I/O manager not supported (rc=%Rrc). Falling back to simple manager\n", rc));
+        pEpClassFile->enmMgrTypeOverride = PDMACEPFILEMGRTYPE_SIMPLE;
+        pEpClassFile->enmEpBackendDefault = PDMACFILEEPBACKEND_BUFFERED;
+    }
+    else
+    {
+        pEpClassFile->uBitmaskAlignment   = AioLimits.cbBufferAlignment ? ~((RTR3UINTPTR)AioLimits.cbBufferAlignment - 1) : RTR3UINTPTR_MAX;
+        pEpClassFile->cReqsOutstandingMax = AioLimits.cReqsOutstandingMax;
+
+        if (pCfgNode)
+        {
+            /* Query the default manager type */
+            char *pszVal = NULL;
+            rc = CFGMR3QueryStringAllocDef(pCfgNode, "IoMgr", &pszVal, "Async");
+            AssertLogRelRCReturn(rc, rc);
+
+            rc = pdmacFileMgrTypeFromName(pszVal, &pEpClassFile->enmMgrTypeOverride);
+            MMR3HeapFree(pszVal);
+            if (RT_FAILURE(rc))
+                return rc;
+
+            LogRel(("AIOMgr: Default manager type is '%s'\n", pdmacFileMgrTypeToName(pEpClassFile->enmMgrTypeOverride)));
+
+            /* Query default backend type */
+            rc = CFGMR3QueryStringAllocDef(pCfgNode, "FileBackend", &pszVal, "NonBuffered");
+            AssertLogRelRCReturn(rc, rc);
+
+            rc = pdmacFileBackendTypeFromName(pszVal, &pEpClassFile->enmEpBackendDefault);
+            MMR3HeapFree(pszVal);
+            if (RT_FAILURE(rc))
+                return rc;
+
+            LogRel(("AIOMgr: Default file backend is '%s'\n", pdmacFileBackendTypeToName(pEpClassFile->enmEpBackendDefault)));
+
+#ifdef RT_OS_LINUX
+            if (   pEpClassFile->enmMgrTypeOverride == PDMACEPFILEMGRTYPE_ASYNC
+                && pEpClassFile->enmEpBackendDefault == PDMACFILEEPBACKEND_BUFFERED)
+            {
+                LogRel(("AIOMgr: Linux does not support buffered async I/O, changing to non buffered\n"));
+                pEpClassFile->enmEpBackendDefault = PDMACFILEEPBACKEND_NON_BUFFERED;
+            }
+#endif
+        }
+        else
+        {
+            /* No configuration supplied, set defaults */
+            pEpClassFile->enmEpBackendDefault = PDMACFILEEPBACKEND_NON_BUFFERED;
+            pEpClassFile->enmMgrTypeOverride  = PDMACEPFILEMGRTYPE_ASYNC;
+        }
+    }
+
+    /* Init critical section. */
+    rc = RTCritSectInit(&pEpClassFile->CritSect);
+
+#ifdef VBOX_WITH_DEBUGGER
+    /* Install the error injection handler. */
+    if (RT_SUCCESS(rc))
+    {
+        rc = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
+        AssertRC(rc);
+    }
+
+#ifdef PDM_ASYNC_COMPLETION_FILE_WITH_DELAY
+    rc = TMR3TimerCreateInternal(pEpClassFile->Core.pVM, TMCLOCK_REAL, pdmacR3TimerCallback, pEpClassFile, "AC Delay", &pEpClassFile->pTimer);
+    AssertRC(rc);
+    pEpClassFile->cMilliesNext = UINT64_MAX;
+#endif
+#endif
+
+    return rc;
+}
+
+static DECLCALLBACK(void) pdmacFileTerminate(PPDMASYNCCOMPLETIONEPCLASS pClassGlobals)
+{
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pClassGlobals;
+
+    /* All endpoints should be closed at this point. */
+    AssertMsg(!pEpClassFile->Core.pEndpointsHead, ("There are still endpoints left\n"));
+
+    /* Destroy all left async I/O managers. */
+    while (pEpClassFile->pAioMgrHead)
+        pdmacFileAioMgrDestroy(pEpClassFile, pEpClassFile->pAioMgrHead);
+
+    RTCritSectDelete(&pEpClassFile->CritSect);
+}
+
+static DECLCALLBACK(int) pdmacFileEpInitialize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                               const char *pszUri, uint32_t fFlags)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->pEpClass;
+    PDMACEPFILEMGRTYPE enmMgrType = pEpClassFile->enmMgrTypeOverride;
+    PDMACFILEEPBACKEND enmEpBackend = pEpClassFile->enmEpBackendDefault;
+
+    AssertMsgReturn((fFlags & ~(PDMACEP_FILE_FLAGS_READ_ONLY | PDMACEP_FILE_FLAGS_DONT_LOCK | PDMACEP_FILE_FLAGS_HOST_CACHE_ENABLED)) == 0,
+                    ("PDMAsyncCompletion: Invalid flag specified\n"), VERR_INVALID_PARAMETER);
+
+    unsigned fFileFlags = RTFILE_O_OPEN;
+
+    /*
+     * Revert to the simple manager and the buffered backend if
+     * the host cache should be enabled.
+     */
+    if (fFlags & PDMACEP_FILE_FLAGS_HOST_CACHE_ENABLED)
+    {
+        enmMgrType   = PDMACEPFILEMGRTYPE_SIMPLE;
+        enmEpBackend = PDMACFILEEPBACKEND_BUFFERED;
+    }
+
+    if (fFlags & PDMACEP_FILE_FLAGS_READ_ONLY)
+        fFileFlags |= RTFILE_O_READ | RTFILE_O_DENY_NONE;
+    else
+    {
+        fFileFlags |= RTFILE_O_READWRITE;
+
+        /*
+         * Opened in read/write mode. Check whether the caller wants to
+         * avoid the lock. Return an error in case caching is enabled
+         * because this can lead to data corruption.
+         */
+        if (fFlags & PDMACEP_FILE_FLAGS_DONT_LOCK)
+            fFileFlags |= RTFILE_O_DENY_NONE;
+        else
+            fFileFlags |= RTFILE_O_DENY_WRITE;
+    }
+
+    if (enmMgrType == PDMACEPFILEMGRTYPE_ASYNC)
+        fFileFlags |= RTFILE_O_ASYNC_IO;
+
+    int rc;
+    if (enmEpBackend == PDMACFILEEPBACKEND_NON_BUFFERED)
+    {
+        /*
+         * We only disable the cache if the size of the file is a multiple of 512.
+         * Certain hosts like Windows, Linux and Solaris require that transfer sizes
+         * are aligned to the volume sector size.
+         * If not we just make sure that the data is written to disk with RTFILE_O_WRITE_THROUGH
+         * which will trash the host cache but ensures that the host cache will not
+         * contain dirty buffers.
+         */
+        RTFILE hFile;
+        rc = RTFileOpen(&hFile, pszUri, RTFILE_O_READ | RTFILE_O_OPEN | RTFILE_O_DENY_NONE);
+        if (RT_SUCCESS(rc))
+        {
+            uint64_t cbSize;
+
+            rc = RTFileQuerySize(hFile, &cbSize);
+
+            if (RT_SUCCESS(rc) && ((cbSize % 512) == 0))
+                fFileFlags |= RTFILE_O_NO_CACHE;
+            else
+            {
+                /* Downgrade to the buffered backend */
+                enmEpBackend = PDMACFILEEPBACKEND_BUFFERED;
+
+#ifdef RT_OS_LINUX
+                fFileFlags &= ~RTFILE_O_ASYNC_IO;
+                enmMgrType   = PDMACEPFILEMGRTYPE_SIMPLE;
+#endif
+            }
+            RTFileClose(hFile);
+        }
+    }
+
+    /* Open with final flags. */
+    rc = RTFileOpen(&pEpFile->hFile, pszUri, fFileFlags);
+    if (   rc == VERR_INVALID_FUNCTION
+        || rc == VERR_INVALID_PARAMETER)
+    {
+        LogRel(("AIOMgr: pdmacFileEpInitialize: RTFileOpen %s / %08x failed with %Rrc\n",
+               pszUri, fFileFlags, rc));
+        /*
+         * Solaris doesn't support directio on ZFS so far. :-\
+         * Trying to enable it returns VERR_INVALID_FUNCTION
+         * (ENOTTY). Remove it and hope for the best.
+         * ZFS supports write throttling in case applications
+         * write more data than can be synced to the disk
+         * without blocking the whole application.
+         *
+         * On Linux we have the same problem with cifs.
+         * Have to disable async I/O here too because it requires O_DIRECT.
+         */
+        fFileFlags &= ~RTFILE_O_NO_CACHE;
+        enmEpBackend = PDMACFILEEPBACKEND_BUFFERED;
+
+#ifdef RT_OS_LINUX
+        fFileFlags &= ~RTFILE_O_ASYNC_IO;
+        enmMgrType   = PDMACEPFILEMGRTYPE_SIMPLE;
+#endif
+
+        /* Open again. */
+        rc = RTFileOpen(&pEpFile->hFile, pszUri, fFileFlags);
+
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("AIOMgr: pdmacFileEpInitialize: RTFileOpen %s / %08x failed AGAIN(!) with %Rrc\n",
+                        pszUri, fFileFlags, rc));
+        }
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        pEpFile->fFlags = fFileFlags;
+
+        rc = RTFileQuerySize(pEpFile->hFile, (uint64_t *)&pEpFile->cbFile);
+        if (RT_SUCCESS(rc))
+        {
+            /* Initialize the segment cache */
+            rc = MMR3HeapAllocZEx(pEpClassFile->Core.pVM, MM_TAG_PDM_ASYNC_COMPLETION,
+                                  sizeof(PDMACTASKFILE),
+                                  (void **)&pEpFile->pTasksFreeHead);
+            if (RT_SUCCESS(rc))
+            {
+                PPDMACEPFILEMGR pAioMgr = NULL;
+
+                pEpFile->pTasksFreeTail = pEpFile->pTasksFreeHead;
+                pEpFile->cTasksCached   = 0;
+                pEpFile->enmBackendType = enmEpBackend;
+                /*
+                 * Disable async flushes on Solaris for now.
+                 * They cause weird hangs which needs more investigations.
+                 */
+#ifndef RT_OS_SOLARIS
+                pEpFile->fAsyncFlushSupported = true;
+#else
+                pEpFile->fAsyncFlushSupported = false;
+#endif
+
+                if (enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE)
+                {
+                    /* Simple mode. Every file has its own async I/O manager. */
+                    rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgr, PDMACEPFILEMGRTYPE_SIMPLE);
+                }
+                else
+                {
+                    pAioMgr = pEpClassFile->pAioMgrHead;
+
+                    /* Check for an idling manager of the same type */
+                    while (pAioMgr)
+                    {
+                        if (pAioMgr->enmMgrType == enmMgrType)
+                            break;
+                        pAioMgr = pAioMgr->pNext;
+                    }
+
+                    if (!pAioMgr)
+                        rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgr, enmMgrType);
+                }
+
+                if (RT_SUCCESS(rc))
+                {
+                    pEpFile->AioMgr.pTreeRangesLocked = (PAVLRFOFFTREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
+                    if (!pEpFile->AioMgr.pTreeRangesLocked)
+                        rc = VERR_NO_MEMORY;
+                    else
+                    {
+                        pEpFile->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
+
+                        /* Assign the endpoint to the thread. */
+                        rc = pdmacFileAioMgrAddEndpoint(pAioMgr, pEpFile);
+                        if (RT_FAILURE(rc))
+                        {
+                            RTMemFree(pEpFile->AioMgr.pTreeRangesLocked);
+                            MMR3HeapFree(pEpFile->pTasksFreeHead);
+                        }
+                    }
+                }
+                else if (rc == VERR_FILE_AIO_INSUFFICIENT_EVENTS)
+                {
+                    PUVM pUVM = VMR3GetUVM(pEpClassFile->Core.pVM);
+#if defined(RT_OS_LINUX)
+                    rc = VMR3SetError(pUVM, rc, RT_SRC_POS,
+                                      N_("Failed to create I/O manager for VM due to insufficient resources on the host. "
+                                         "Either increase the amount of allowed events in /proc/sys/fs/aio-max-nr or enable "
+                                         "the host I/O cache"));
+#else
+                    rc = VMR3SetError(pUVM, rc, RT_SRC_POS,
+                                      N_("Failed to create I/O manager for VM due to insufficient resources on the host. "
+                                         "Enable the host I/O cache"));
+#endif
+                }
+                else
+                {
+                    PUVM pUVM = VMR3GetUVM(pEpClassFile->Core.pVM);
+                    rc = VMR3SetError(pUVM, rc, RT_SRC_POS,
+                                      N_("Failed to create I/O manager for VM due to an unknown error"));
+                }
+            }
+        }
+
+        if (RT_FAILURE(rc))
+            RTFileClose(pEpFile->hFile);
+    }
+
+#ifdef VBOX_WITH_STATISTICS
+    if (RT_SUCCESS(rc))
+    {
+        STAMR3RegisterF(pEpClassFile->Core.pVM, &pEpFile->StatRead,
+                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
+                       STAMUNIT_TICKS_PER_CALL, "Time taken to read from the endpoint",
+                       "/PDM/AsyncCompletion/File/%s/%d/Read", RTPathFilename(pEpFile->Core.pszUri), pEpFile->Core.iStatId);
+
+        STAMR3RegisterF(pEpClassFile->Core.pVM, &pEpFile->StatWrite,
+                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
+                       STAMUNIT_TICKS_PER_CALL, "Time taken to write to the endpoint",
+                       "/PDM/AsyncCompletion/File/%s/%d/Write", RTPathFilename(pEpFile->Core.pszUri), pEpFile->Core.iStatId);
+    }
+#endif
+
+    if (RT_SUCCESS(rc))
+        LogRel(("AIOMgr: Endpoint for file '%s' (flags %08x) created successfully\n", pszUri, pEpFile->fFlags));
+
+    return rc;
+}
+
+static DECLCALLBACK(int) pdmacFileEpRangesLockedDestroy(PAVLRFOFFNODECORE pNode, void *pvUser)
+{
+    NOREF(pNode); NOREF(pvUser);
+    AssertMsgFailed(("The locked ranges tree should be empty at that point\n"));
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(int) pdmacFileEpClose(PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile      = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+    PPDMASYNCCOMPLETIONEPCLASSFILE  pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->pEpClass;
+
+    /* Make sure that all tasks finished for this endpoint. */
+    int rc = pdmacFileAioMgrCloseEndpoint(pEpFile->pAioMgr, pEpFile);
+    AssertRC(rc);
+
+    /*
+     * If the async I/O manager is in failsafe mode this is the only endpoint
+     * he processes and thus can be destroyed now.
+     */
+    if (pEpFile->pAioMgr->enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE)
+        pdmacFileAioMgrDestroy(pEpClassFile, pEpFile->pAioMgr);
+
+    /* Free cached tasks. */
+    PPDMACTASKFILE pTask = pEpFile->pTasksFreeHead;
+
+    while (pTask)
+    {
+        PPDMACTASKFILE pTaskFree = pTask;
+        pTask = pTask->pNext;
+        MMR3HeapFree(pTaskFree);
+    }
+
+    /* Destroy the locked ranges tree now. */
+    RTAvlrFileOffsetDestroy(pEpFile->AioMgr.pTreeRangesLocked, pdmacFileEpRangesLockedDestroy, NULL);
+    RTMemFree(pEpFile->AioMgr.pTreeRangesLocked);
+    pEpFile->AioMgr.pTreeRangesLocked = NULL;
+
+    RTFileClose(pEpFile->hFile);
+
+#ifdef VBOX_WITH_STATISTICS
+    /* Not sure if this might be unnecessary because of similar statement in pdmR3AsyncCompletionStatisticsDeregister? */
+    STAMR3DeregisterF(pEpClassFile->Core.pVM->pUVM, "/PDM/AsyncCompletion/File/%s/*", RTPathFilename(pEpFile->Core.pszUri));
+#endif
+
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(int) pdmacFileEpRead(PPDMASYNCCOMPLETIONTASK pTask,
+                                         PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                                         PCRTSGSEG paSegments, size_t cSegments,
+                                         size_t cbRead)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+
+    LogFlowFunc(("pTask=%#p pEndpoint=%#p off=%RTfoff paSegments=%#p cSegments=%zu cbRead=%zu\n",
+                 pTask, pEndpoint, off, paSegments, cSegments, cbRead));
+
+    if (RT_UNLIKELY((uint64_t)off + cbRead > pEpFile->cbFile))
+        return VERR_EOF;
+
+    STAM_PROFILE_ADV_START(&pEpFile->StatRead, Read);
+    pdmacFileEpTaskInit(pTask, cbRead);
+    int rc = pdmacFileEpTaskInitiate(pTask, pEndpoint, off, paSegments, cSegments, cbRead,
+                                     PDMACTASKFILETRANSFER_READ);
+    STAM_PROFILE_ADV_STOP(&pEpFile->StatRead, Read);
+
+    return rc;
+}
+
+static DECLCALLBACK(int) pdmacFileEpWrite(PPDMASYNCCOMPLETIONTASK pTask,
+                                          PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                                          PCRTSGSEG paSegments, size_t cSegments,
+                                          size_t cbWrite)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+
+    if (RT_UNLIKELY(pEpFile->fReadonly))
+        return VERR_NOT_SUPPORTED;
+
+    STAM_PROFILE_ADV_START(&pEpFile->StatWrite, Write);
+
+    pdmacFileEpTaskInit(pTask, cbWrite);
+
+    int rc = pdmacFileEpTaskInitiate(pTask, pEndpoint, off, paSegments, cSegments, cbWrite,
+                                     PDMACTASKFILETRANSFER_WRITE);
+
+    STAM_PROFILE_ADV_STOP(&pEpFile->StatWrite, Write);
+
+    return rc;
+}
+
+static DECLCALLBACK(int) pdmacFileEpFlush(PPDMASYNCCOMPLETIONTASK pTask,
+                                          PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile   = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+    PPDMASYNCCOMPLETIONTASKFILE     pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pTask;
+
+    if (RT_UNLIKELY(pEpFile->fReadonly))
+        return VERR_NOT_SUPPORTED;
+
+    pdmacFileEpTaskInit(pTask, 0);
+
+    PPDMACTASKFILE pIoTask = pdmacFileTaskAlloc(pEpFile);
+    if (RT_UNLIKELY(!pIoTask))
+        return VERR_NO_MEMORY;
+
+    pIoTask->pEndpoint       = pEpFile;
+    pIoTask->enmTransferType = PDMACTASKFILETRANSFER_FLUSH;
+    pIoTask->pvUser          = pTaskFile;
+    pIoTask->pfnCompleted    = pdmacFileEpTaskCompleted;
+    pdmacFileEpAddTask(pEpFile, pIoTask);
+
+    return VINF_AIO_TASK_PENDING;
+}
+
+static DECLCALLBACK(int) pdmacFileEpGetSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint64_t *pcbSize)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+
+    *pcbSize = ASMAtomicReadU64(&pEpFile->cbFile);
+
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(int) pdmacFileEpSetSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint64_t cbSize)
+{
+    int rc;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+
+    rc = RTFileSetSize(pEpFile->hFile, cbSize);
+    if (RT_SUCCESS(rc))
+        ASMAtomicWriteU64(&pEpFile->cbFile, cbSize);
+
+    return rc;
+}
+
+const PDMASYNCCOMPLETIONEPCLASSOPS g_PDMAsyncCompletionEndpointClassFile =
+{
+    /* u32Version */
+    PDMAC_EPCLASS_OPS_VERSION,
+    /* pcszName */
+    "File",
+    /* enmClassType */
+    PDMASYNCCOMPLETIONEPCLASSTYPE_FILE,
+    /* cbEndpointClassGlobal */
+    sizeof(PDMASYNCCOMPLETIONEPCLASSFILE),
+    /* cbEndpoint */
+    sizeof(PDMASYNCCOMPLETIONENDPOINTFILE),
+    /* cbTask */
+    sizeof(PDMASYNCCOMPLETIONTASKFILE),
+    /* pfnInitialize */
+    pdmacFileInitialize,
+    /* pfnTerminate */
+    pdmacFileTerminate,
+    /* pfnEpInitialize. */
+    pdmacFileEpInitialize,
+    /* pfnEpClose */
+    pdmacFileEpClose,
+    /* pfnEpRead */
+    pdmacFileEpRead,
+    /* pfnEpWrite */
+    pdmacFileEpWrite,
+    /* pfnEpFlush */
+    pdmacFileEpFlush,
+    /* pfnEpGetSize */
+    pdmacFileEpGetSize,
+    /* pfnEpSetSize */
+    pdmacFileEpSetSize,
+    /* u32VersionEnd */
+    PDMAC_EPCLASS_OPS_VERSION
+};
+
diff --git a/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileFailsafe.cpp b/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileFailsafe.cpp
new file mode 100644
index 00000000..c7ebcf6b
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileFailsafe.cpp
@@ -0,0 +1,270 @@
+/* $Id: PDMAsyncCompletionFileFailsafe.cpp $ */
+/** @file
+ * PDM Async I/O - Transport data asynchronous in R3 using EMT.
+ * Simple File I/O manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <VBox/log.h>
+
+#include "PDMAsyncCompletionFileInternal.h"
+
+
+
+/**
+ * Put a list of tasks in the pending request list of an endpoint.
+ */
+DECLINLINE(void) pdmacFileAioMgrEpAddTaskList(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTaskHead)
+{
+    /* Add the rest of the tasks to the pending list */
+    if (!pEndpoint->AioMgr.pReqsPendingHead)
+    {
+        Assert(!pEndpoint->AioMgr.pReqsPendingTail);
+        pEndpoint->AioMgr.pReqsPendingHead = pTaskHead;
+    }
+    else
+    {
+        Assert(pEndpoint->AioMgr.pReqsPendingTail);
+        pEndpoint->AioMgr.pReqsPendingTail->pNext = pTaskHead;
+    }
+
+    /* Update the tail. */
+    while (pTaskHead->pNext)
+        pTaskHead = pTaskHead->pNext;
+
+    pEndpoint->AioMgr.pReqsPendingTail = pTaskHead;
+    pTaskHead->pNext = NULL;
+}
+
+/**
+ * Processes a given task list for assigned to the given endpoint.
+ */
+static int pdmacFileAioMgrFailsafeProcessEndpointTaskList(PPDMACEPFILEMGR pAioMgr,
+                                                          PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                                          PPDMACTASKFILE pTasks)
+{
+    int rc = VINF_SUCCESS;
+
+    while (pTasks)
+    {
+        RTMSINTERVAL msWhenNext;
+        PPDMACTASKFILE pCurr = pTasks;
+
+        if (!pdmacEpIsTransferAllowed(&pEndpoint->Core, (uint32_t)pCurr->DataSeg.cbSeg, &msWhenNext))
+        {
+            pAioMgr->msBwLimitExpired = RT_MIN(pAioMgr->msBwLimitExpired, msWhenNext);
+            break;
+        }
+
+        pTasks = pTasks->pNext;
+
+        switch (pCurr->enmTransferType)
+        {
+            case PDMACTASKFILETRANSFER_FLUSH:
+            {
+                rc = RTFileFlush(pEndpoint->hFile);
+                break;
+            }
+            case PDMACTASKFILETRANSFER_READ:
+            case PDMACTASKFILETRANSFER_WRITE:
+            {
+                if (pCurr->enmTransferType == PDMACTASKFILETRANSFER_READ)
+                {
+                    rc = RTFileReadAt(pEndpoint->hFile, pCurr->Off,
+                                      pCurr->DataSeg.pvSeg,
+                                      pCurr->DataSeg.cbSeg,
+                                      NULL);
+                }
+                else
+                {
+                    if (RT_UNLIKELY((uint64_t)pCurr->Off + pCurr->DataSeg.cbSeg > pEndpoint->cbFile))
+                    {
+                        ASMAtomicWriteU64(&pEndpoint->cbFile, pCurr->Off + pCurr->DataSeg.cbSeg);
+                        RTFileSetSize(pEndpoint->hFile, pCurr->Off + pCurr->DataSeg.cbSeg);
+                    }
+
+                    rc = RTFileWriteAt(pEndpoint->hFile, pCurr->Off,
+                                       pCurr->DataSeg.pvSeg,
+                                       pCurr->DataSeg.cbSeg,
+                                       NULL);
+                }
+
+                break;
+            }
+            default:
+                AssertMsgFailed(("Invalid transfer type %d\n", pTasks->enmTransferType));
+        }
+
+        pCurr->pfnCompleted(pCurr, pCurr->pvUser, rc);
+        pdmacFileTaskFree(pEndpoint, pCurr);
+    }
+
+    if (pTasks)
+    {
+        /* Add the rest of the tasks to the pending list */
+        pdmacFileAioMgrEpAddTaskList(pEndpoint, pTasks);
+    }
+
+    return VINF_SUCCESS;
+}
+
+static int pdmacFileAioMgrFailsafeProcessEndpoint(PPDMACEPFILEMGR pAioMgr,
+                                                  PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    int rc = VINF_SUCCESS;
+    PPDMACTASKFILE pTasks = pEndpoint->AioMgr.pReqsPendingHead;
+
+    pEndpoint->AioMgr.pReqsPendingHead = NULL;
+    pEndpoint->AioMgr.pReqsPendingTail = NULL;
+
+    /* Process the request pending list first in case the endpoint was migrated due to an error. */
+    if (pTasks)
+        rc = pdmacFileAioMgrFailsafeProcessEndpointTaskList(pAioMgr, pEndpoint, pTasks);
+
+    if (RT_SUCCESS(rc))
+    {
+        pTasks = pdmacFileEpGetNewTasks(pEndpoint);
+
+        if (pTasks)
+            rc = pdmacFileAioMgrFailsafeProcessEndpointTaskList(pAioMgr, pEndpoint, pTasks);
+    }
+
+    return rc;
+}
+
+/**
+ * A fallback method in case something goes wrong with the normal
+ * I/O manager.
+ */
+DECLCALLBACK(int) pdmacFileAioMgrFailsafe(RTTHREAD hThreadSelf, void *pvUser)
+{
+    int             rc      = VINF_SUCCESS;
+    PPDMACEPFILEMGR pAioMgr = (PPDMACEPFILEMGR)pvUser;
+    NOREF(hThreadSelf);
+
+    while (   (pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING)
+           || (pAioMgr->enmState == PDMACEPFILEMGRSTATE_SUSPENDING))
+    {
+        ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, true);
+        if (!ASMAtomicReadBool(&pAioMgr->fWokenUp))
+            rc = RTSemEventWait(pAioMgr->EventSem, pAioMgr->msBwLimitExpired);
+        ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, false);
+        Assert(RT_SUCCESS(rc) || rc == VERR_TIMEOUT);
+
+        LogFlow(("Got woken up\n"));
+        ASMAtomicWriteBool(&pAioMgr->fWokenUp, false);
+
+        /* Process endpoint events first. */
+        PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pAioMgr->pEndpointsHead;
+        while (pEndpoint)
+        {
+            pAioMgr->msBwLimitExpired = RT_INDEFINITE_WAIT;
+            rc = pdmacFileAioMgrFailsafeProcessEndpoint(pAioMgr, pEndpoint);
+            AssertRC(rc);
+            pEndpoint = pEndpoint->AioMgr.pEndpointNext;
+        }
+
+        /* Now check for an external blocking event. */
+        if (pAioMgr->fBlockingEventPending)
+        {
+            switch (pAioMgr->enmBlockingEvent)
+            {
+                case PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT:
+                {
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointNew = pAioMgr->BlockingEventData.AddEndpoint.pEndpoint;
+                    AssertMsg(VALID_PTR(pEndpointNew), ("Adding endpoint event without a endpoint to add\n"));
+
+                    pEndpointNew->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
+
+                    pEndpointNew->AioMgr.pEndpointNext = pAioMgr->pEndpointsHead;
+                    pEndpointNew->AioMgr.pEndpointPrev = NULL;
+                    if (pAioMgr->pEndpointsHead)
+                        pAioMgr->pEndpointsHead->AioMgr.pEndpointPrev = pEndpointNew;
+                    pAioMgr->pEndpointsHead = pEndpointNew;
+
+                    pAioMgr->cEndpoints++;
+
+                    /*
+                     * Process the task list the first time. There might be pending requests
+                     * if the endpoint was migrated from another endpoint.
+                     */
+                    rc = pdmacFileAioMgrFailsafeProcessEndpoint(pAioMgr, pEndpointNew);
+                    AssertRC(rc);
+                    break;
+                }
+                case PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT:
+                {
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove = pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint;
+                    AssertMsg(VALID_PTR(pEndpointRemove), ("Removing endpoint event without a endpoint to remove\n"));
+
+                    pEndpointRemove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
+
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEndpointRemove->AioMgr.pEndpointPrev;
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEndpointRemove->AioMgr.pEndpointNext;
+
+                    if (pPrev)
+                        pPrev->AioMgr.pEndpointNext = pNext;
+                    else
+                        pAioMgr->pEndpointsHead = pNext;
+
+                    if (pNext)
+                        pNext->AioMgr.pEndpointPrev = pPrev;
+
+                    pAioMgr->cEndpoints--;
+                    break;
+                }
+                case PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT:
+                {
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointClose = pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint;
+                    AssertMsg(VALID_PTR(pEndpointClose), ("Close endpoint event without a endpoint to Close\n"));
+
+                    pEndpointClose->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING;
+
+                    /* Make sure all tasks finished. */
+                    rc = pdmacFileAioMgrFailsafeProcessEndpoint(pAioMgr, pEndpointClose);
+                    AssertRC(rc);
+                    break;
+                }
+                case PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN:
+                    pAioMgr->enmState = PDMACEPFILEMGRSTATE_SHUTDOWN;
+                    break;
+                case PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND:
+                    pAioMgr->enmState = PDMACEPFILEMGRSTATE_SUSPENDING;
+                    break;
+                case PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME:
+                    pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
+                    break;
+                default:
+                    AssertMsgFailed(("Invalid event type %d\n", pAioMgr->enmBlockingEvent));
+            }
+
+            ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
+            pAioMgr->enmBlockingEvent = PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID;
+
+            /* Release the waiting thread. */
+            rc = RTSemEventSignal(pAioMgr->EventSemBlock);
+            AssertRC(rc);
+        }
+    }
+
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileNormal.cpp b/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileNormal.cpp
new file mode 100644
index 00000000..e3512b08
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileNormal.cpp
@@ -0,0 +1,1736 @@
+/* $Id: PDMAsyncCompletionFileNormal.cpp $ */
+/** @file
+ * PDM Async I/O - Async File I/O manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
+#include <iprt/types.h>
+#include <iprt/asm.h>
+#include <iprt/file.h>
+#include <iprt/mem.h>
+#include <iprt/string.h>
+#include <iprt/assert.h>
+#include <VBox/log.h>
+
+#include "PDMAsyncCompletionFileInternal.h"
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The update period for the I/O load statistics in ms. */
+#define PDMACEPFILEMGR_LOAD_UPDATE_PERIOD   1000
+/** Maximum number of requests a manager will handle. */
+#define PDMACEPFILEMGR_REQS_STEP              64
+
+
+/*********************************************************************************************************************************
+*   Internal functions                                                                                                           *
+*********************************************************************************************************************************/
+static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
+                                                PPDMACEPFILEMGR pAioMgr,
+                                                PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+
+static PPDMACTASKFILE pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
+                                                         PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                                         PPDMACFILERANGELOCK pRangeLock);
+
+static void pdmacFileAioMgrNormalReqCompleteRc(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq,
+                                               int rc, size_t cbTransfered);
+
+
+int pdmacFileAioMgrNormalInit(PPDMACEPFILEMGR pAioMgr)
+{
+    pAioMgr->cRequestsActiveMax = PDMACEPFILEMGR_REQS_STEP;
+
+    int rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, RTFILEAIO_UNLIMITED_REQS, 0 /* fFlags */);
+    if (rc == VERR_OUT_OF_RANGE)
+        rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, pAioMgr->cRequestsActiveMax, 0 /* fFlags */);
+
+    if (RT_SUCCESS(rc))
+    {
+        /* Initialize request handle array. */
+        pAioMgr->iFreeEntry       = 0;
+        pAioMgr->cReqEntries      = pAioMgr->cRequestsActiveMax;
+        pAioMgr->pahReqsFree      = (RTFILEAIOREQ *)RTMemAllocZ(pAioMgr->cReqEntries * sizeof(RTFILEAIOREQ));
+
+        if (pAioMgr->pahReqsFree)
+        {
+            /* Create the range lock memcache. */
+            rc = RTMemCacheCreate(&pAioMgr->hMemCacheRangeLocks, sizeof(PDMACFILERANGELOCK),
+                                  0, UINT32_MAX, NULL, NULL, NULL, 0);
+            if (RT_SUCCESS(rc))
+                return VINF_SUCCESS;
+
+            RTMemFree(pAioMgr->pahReqsFree);
+        }
+        else
+        {
+            RTFileAioCtxDestroy(pAioMgr->hAioCtx);
+            rc = VERR_NO_MEMORY;
+        }
+    }
+
+    return rc;
+}
+
+void pdmacFileAioMgrNormalDestroy(PPDMACEPFILEMGR pAioMgr)
+{
+    RTFileAioCtxDestroy(pAioMgr->hAioCtx);
+
+    while (pAioMgr->iFreeEntry > 0)
+    {
+        pAioMgr->iFreeEntry--;
+        Assert(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] != NIL_RTFILEAIOREQ);
+        RTFileAioReqDestroy(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry]);
+    }
+
+    RTMemFree(pAioMgr->pahReqsFree);
+    RTMemCacheDestroy(pAioMgr->hMemCacheRangeLocks);
+}
+
+#if 0 /* currently unused */
+/**
+ * Sorts the endpoint list with insertion sort.
+ */
+static void pdmacFileAioMgrNormalEndpointsSortByLoad(PPDMACEPFILEMGR pAioMgr)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpPrev, pEpCurr, pEpNextToSort;
+
+    pEpPrev = pAioMgr->pEndpointsHead;
+    pEpCurr = pEpPrev->AioMgr.pEndpointNext;
+
+    while (pEpCurr)
+    {
+        /* Remember the next element to sort because the list might change. */
+        pEpNextToSort = pEpCurr->AioMgr.pEndpointNext;
+
+        /* Unlink the current element from the list. */
+        PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
+        PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
+
+        if (pPrev)
+            pPrev->AioMgr.pEndpointNext = pNext;
+        else
+            pAioMgr->pEndpointsHead = pNext;
+
+        if (pNext)
+            pNext->AioMgr.pEndpointPrev = pPrev;
+
+        /* Go back until we reached the place to insert the current endpoint into. */
+        while (pEpPrev && (pEpPrev->AioMgr.cReqsPerSec < pEpCurr->AioMgr.cReqsPerSec))
+            pEpPrev = pEpPrev->AioMgr.pEndpointPrev;
+
+        /* Link the endpoint into the list. */
+        if (pEpPrev)
+            pNext = pEpPrev->AioMgr.pEndpointNext;
+        else
+            pNext = pAioMgr->pEndpointsHead;
+
+        pEpCurr->AioMgr.pEndpointNext = pNext;
+        pEpCurr->AioMgr.pEndpointPrev = pEpPrev;
+
+        if (pNext)
+            pNext->AioMgr.pEndpointPrev = pEpCurr;
+
+        if (pEpPrev)
+            pEpPrev->AioMgr.pEndpointNext = pEpCurr;
+        else
+            pAioMgr->pEndpointsHead = pEpCurr;
+
+        pEpCurr = pEpNextToSort;
+    }
+
+#ifdef DEBUG
+    /* Validate sorting algorithm */
+    unsigned cEndpoints = 0;
+    pEpCurr = pAioMgr->pEndpointsHead;
+
+    AssertMsg(pEpCurr, ("No endpoint in the list?\n"));
+    AssertMsg(!pEpCurr->AioMgr.pEndpointPrev, ("First element in the list points to previous element\n"));
+
+    while (pEpCurr)
+    {
+        cEndpoints++;
+
+        PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
+        PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
+
+        Assert(!pNext || pNext->AioMgr.cReqsPerSec <= pEpCurr->AioMgr.cReqsPerSec);
+        Assert(!pPrev || pPrev->AioMgr.cReqsPerSec >= pEpCurr->AioMgr.cReqsPerSec);
+
+        pEpCurr = pNext;
+    }
+
+    AssertMsg(cEndpoints == pAioMgr->cEndpoints, ("Endpoints lost during sort!\n"));
+
+#endif
+}
+#endif /* currently unused */
+
+/**
+ * Removes an endpoint from the currently assigned manager.
+ *
+ * @returns TRUE if there are still requests pending on the current manager for this endpoint.
+ *          FALSE otherwise.
+ * @param   pEndpointRemove    The endpoint to remove.
+ */
+static bool pdmacFileAioMgrNormalRemoveEndpoint(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pPrev   = pEndpointRemove->AioMgr.pEndpointPrev;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pNext   = pEndpointRemove->AioMgr.pEndpointNext;
+    PPDMACEPFILEMGR                 pAioMgr = pEndpointRemove->pAioMgr;
+
+    pAioMgr->cEndpoints--;
+
+    if (pPrev)
+        pPrev->AioMgr.pEndpointNext = pNext;
+    else
+        pAioMgr->pEndpointsHead = pNext;
+
+    if (pNext)
+        pNext->AioMgr.pEndpointPrev = pPrev;
+
+    /* Make sure that there is no request pending on this manager for the endpoint. */
+    if (!pEndpointRemove->AioMgr.cRequestsActive)
+    {
+        Assert(!pEndpointRemove->pFlushReq);
+
+        /* Reopen the file so that the new endpoint can re-associate with the file */
+        RTFileClose(pEndpointRemove->hFile);
+        int rc = RTFileOpen(&pEndpointRemove->hFile, pEndpointRemove->Core.pszUri, pEndpointRemove->fFlags);
+        AssertRC(rc);
+        return false;
+    }
+
+    return true;
+}
+
+#if 0 /* currently unused */
+
+static bool pdmacFileAioMgrNormalIsBalancePossible(PPDMACEPFILEMGR pAioMgr)
+{
+    /* Balancing doesn't make sense with only one endpoint. */
+    if (pAioMgr->cEndpoints == 1)
+        return false;
+
+    /* Doesn't make sens to move endpoints if only one produces the whole load */
+    unsigned cEndpointsWithLoad = 0;
+
+    PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
+
+    while (pCurr)
+    {
+        if (pCurr->AioMgr.cReqsPerSec)
+            cEndpointsWithLoad++;
+
+        pCurr = pCurr->AioMgr.pEndpointNext;
+    }
+
+    return (cEndpointsWithLoad > 1);
+}
+
+/**
+ * Creates a new I/O manager and spreads the I/O load of the endpoints
+ * between the given I/O manager and the new one.
+ *
+ * @returns nothing.
+ * @param   pAioMgr    The I/O manager with high I/O load.
+ */
+static void pdmacFileAioMgrNormalBalanceLoad(PPDMACEPFILEMGR pAioMgr)
+{
+    /*
+     * Check if balancing would improve the situation.
+     */
+    if (pdmacFileAioMgrNormalIsBalancePossible(pAioMgr))
+    {
+        PPDMASYNCCOMPLETIONEPCLASSFILE  pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
+        PPDMACEPFILEMGR                 pAioMgrNew = NULL;
+
+        int rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgrNew, PDMACEPFILEMGRTYPE_ASYNC);
+        if (RT_SUCCESS(rc))
+        {
+            /* We will sort the list by request count per second. */
+            pdmacFileAioMgrNormalEndpointsSortByLoad(pAioMgr);
+
+            /* Now move some endpoints to the new manager. */
+            unsigned cReqsHere  = pAioMgr->pEndpointsHead->AioMgr.cReqsPerSec;
+            unsigned cReqsOther = 0;
+            PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead->AioMgr.pEndpointNext;
+
+            while (pCurr)
+            {
+                if (cReqsHere <= cReqsOther)
+                {
+                    /*
+                     * The other manager has more requests to handle now.
+                     * We will keep the current endpoint.
+                     */
+                    Log(("Keeping endpoint %#p{%s} with %u reqs/s\n", pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
+                    cReqsHere += pCurr->AioMgr.cReqsPerSec;
+                    pCurr = pCurr->AioMgr.pEndpointNext;
+                }
+                else
+                {
+                    /* Move to other endpoint. */
+                    Log(("Moving endpoint %#p{%s} with %u reqs/s to other manager\n", pCurr, pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
+                    cReqsOther += pCurr->AioMgr.cReqsPerSec;
+
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pMove = pCurr;
+
+                    pCurr = pCurr->AioMgr.pEndpointNext;
+
+                    bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pMove);
+
+                    if (fReqsPending)
+                    {
+                        pMove->enmState          = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
+                        pMove->AioMgr.fMoving    = true;
+                        pMove->AioMgr.pAioMgrDst = pAioMgrNew;
+                    }
+                    else
+                    {
+                        pMove->AioMgr.fMoving    = false;
+                        pMove->AioMgr.pAioMgrDst = NULL;
+                        pdmacFileAioMgrAddEndpoint(pAioMgrNew, pMove);
+                    }
+                }
+            }
+        }
+        else
+        {
+            /* Don't process further but leave a log entry about reduced performance. */
+            LogRel(("AIOMgr: Could not create new I/O manager (rc=%Rrc). Expect reduced performance\n", rc));
+        }
+    }
+    else
+        Log(("AIOMgr: Load balancing would not improve anything\n"));
+}
+
+#endif /* unused */
+
+/**
+ * Increase the maximum number of active requests for the given I/O manager.
+ *
+ * @returns VBox status code.
+ * @param   pAioMgr    The I/O manager to grow.
+ */
+static int pdmacFileAioMgrNormalGrow(PPDMACEPFILEMGR pAioMgr)
+{
+    LogFlowFunc(("pAioMgr=%#p\n", pAioMgr));
+
+    AssertMsg(    pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING
+              && !pAioMgr->cRequestsActive,
+              ("Invalid state of the I/O manager\n"));
+
+#ifdef RT_OS_WINDOWS
+    /*
+     * Reopen the files of all assigned endpoints first so we can assign them to the new
+     * I/O context.
+     */
+    PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
+
+    while (pCurr)
+    {
+        RTFileClose(pCurr->hFile);
+        int rc2 = RTFileOpen(&pCurr->hFile, pCurr->Core.pszUri, pCurr->fFlags); AssertRC(rc2);
+
+        pCurr = pCurr->AioMgr.pEndpointNext;
+    }
+#endif
+
+    /* Create the new bigger context. */
+    pAioMgr->cRequestsActiveMax += PDMACEPFILEMGR_REQS_STEP;
+
+    RTFILEAIOCTX hAioCtxNew = NIL_RTFILEAIOCTX;
+    int rc = RTFileAioCtxCreate(&hAioCtxNew, RTFILEAIO_UNLIMITED_REQS, 0 /* fFlags */);
+    if (rc == VERR_OUT_OF_RANGE)
+        rc = RTFileAioCtxCreate(&hAioCtxNew, pAioMgr->cRequestsActiveMax, 0 /* fFlags */);
+
+    if (RT_SUCCESS(rc))
+    {
+        /* Close the old context. */
+        rc = RTFileAioCtxDestroy(pAioMgr->hAioCtx);
+        AssertRC(rc); /** @todo r=bird: Ignoring error code, will propagate. */
+
+        pAioMgr->hAioCtx = hAioCtxNew;
+
+        /* Create a new I/O task handle array */
+        uint32_t cReqEntriesNew = pAioMgr->cRequestsActiveMax + 1;
+        RTFILEAIOREQ *pahReqNew = (RTFILEAIOREQ *)RTMemAllocZ(cReqEntriesNew * sizeof(RTFILEAIOREQ));
+
+        if (pahReqNew)
+        {
+            /* Copy the cached request handles. */
+            for (uint32_t iReq = 0; iReq < pAioMgr->cReqEntries; iReq++)
+                pahReqNew[iReq] = pAioMgr->pahReqsFree[iReq];
+
+            RTMemFree(pAioMgr->pahReqsFree);
+            pAioMgr->pahReqsFree = pahReqNew;
+            pAioMgr->cReqEntries = cReqEntriesNew;
+            LogFlowFunc(("I/O manager increased to handle a maximum of %u requests\n",
+                         pAioMgr->cRequestsActiveMax));
+        }
+        else
+            rc = VERR_NO_MEMORY;
+    }
+
+#ifdef RT_OS_WINDOWS
+    /* Assign the file to the new context. */
+    pCurr = pAioMgr->pEndpointsHead;
+    while (pCurr)
+    {
+        rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pCurr->hFile);
+        AssertRC(rc); /** @todo r=bird: Ignoring error code, will propagate. */
+
+        pCurr = pCurr->AioMgr.pEndpointNext;
+    }
+#endif
+
+    if (RT_FAILURE(rc))
+    {
+        LogFlow(("Increasing size of the I/O manager failed with rc=%Rrc\n", rc));
+        pAioMgr->cRequestsActiveMax -= PDMACEPFILEMGR_REQS_STEP;
+    }
+
+    pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
+    LogFlowFunc(("returns rc=%Rrc\n", rc));
+
+    return rc;
+}
+
+/**
+ * Checks if a given status code is fatal.
+ * Non fatal errors can be fixed by migrating the endpoint to a
+ * failsafe manager.
+ *
+ * @returns true If the error is fatal and migrating to a failsafe manager doesn't help
+ *          false If the error can be fixed by a migration. (image on NFS disk for example)
+ * @param   rcReq    The status code to check.
+ */
+DECLINLINE(bool) pdmacFileAioMgrNormalRcIsFatal(int rcReq)
+{
+    return rcReq == VERR_DEV_IO_ERROR
+        || rcReq == VERR_FILE_IO_ERROR
+        || rcReq == VERR_DISK_IO_ERROR
+        || rcReq == VERR_DISK_FULL
+        || rcReq == VERR_FILE_TOO_BIG;
+}
+
+/**
+ * Error handler which will create the failsafe managers and destroy the failed I/O manager.
+ *
+ * @returns VBox status code
+ * @param   pAioMgr     The I/O manager the error occurred on.
+ * @param   rc          The error code.
+ * @param   SRC_POS     The source location of the error (use RT_SRC_POS).
+ */
+static int pdmacFileAioMgrNormalErrorHandler(PPDMACEPFILEMGR pAioMgr, int rc, RT_SRC_POS_DECL)
+{
+    LogRel(("AIOMgr: I/O manager %#p encountered a critical error (rc=%Rrc) during operation. Falling back to failsafe mode. Expect reduced performance\n",
+            pAioMgr, rc));
+    LogRel(("AIOMgr: Error happened in %s:(%u){%s}\n", RT_SRC_POS_ARGS));
+    LogRel(("AIOMgr: Please contact the product vendor\n"));
+
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
+
+    pAioMgr->enmState = PDMACEPFILEMGRSTATE_FAULT;
+    ASMAtomicWriteU32((volatile uint32_t *)&pEpClassFile->enmMgrTypeOverride, PDMACEPFILEMGRTYPE_SIMPLE);
+
+    AssertMsgFailed(("Implement\n"));
+    return VINF_SUCCESS;
+}
+
+/**
+ * Put a list of tasks in the pending request list of an endpoint.
+ */
+DECLINLINE(void) pdmacFileAioMgrEpAddTaskList(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTaskHead)
+{
+    /* Add the rest of the tasks to the pending list */
+    if (!pEndpoint->AioMgr.pReqsPendingHead)
+    {
+        Assert(!pEndpoint->AioMgr.pReqsPendingTail);
+        pEndpoint->AioMgr.pReqsPendingHead = pTaskHead;
+    }
+    else
+    {
+        Assert(pEndpoint->AioMgr.pReqsPendingTail);
+        pEndpoint->AioMgr.pReqsPendingTail->pNext = pTaskHead;
+    }
+
+    /* Update the tail. */
+    while (pTaskHead->pNext)
+        pTaskHead = pTaskHead->pNext;
+
+    pEndpoint->AioMgr.pReqsPendingTail = pTaskHead;
+    pTaskHead->pNext = NULL;
+}
+
+/**
+ * Put one task in the pending request list of an endpoint.
+ */
+DECLINLINE(void) pdmacFileAioMgrEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask)
+{
+    /* Add the rest of the tasks to the pending list */
+    if (!pEndpoint->AioMgr.pReqsPendingHead)
+    {
+        Assert(!pEndpoint->AioMgr.pReqsPendingTail);
+        pEndpoint->AioMgr.pReqsPendingHead = pTask;
+    }
+    else
+    {
+        Assert(pEndpoint->AioMgr.pReqsPendingTail);
+        pEndpoint->AioMgr.pReqsPendingTail->pNext = pTask;
+    }
+
+    pEndpoint->AioMgr.pReqsPendingTail = pTask;
+    pTask->pNext = NULL;
+}
+
+/**
+ * Allocates a async I/O request.
+ *
+ * @returns Handle to the request.
+ * @param   pAioMgr    The I/O manager.
+ */
+static RTFILEAIOREQ pdmacFileAioMgrNormalRequestAlloc(PPDMACEPFILEMGR pAioMgr)
+{
+    /* Get a request handle. */
+    RTFILEAIOREQ hReq;
+    if (pAioMgr->iFreeEntry > 0)
+    {
+        pAioMgr->iFreeEntry--;
+        hReq = pAioMgr->pahReqsFree[pAioMgr->iFreeEntry];
+        pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] = NIL_RTFILEAIOREQ;
+        Assert(hReq != NIL_RTFILEAIOREQ);
+    }
+    else
+    {
+        int rc = RTFileAioReqCreate(&hReq);
+        AssertRCReturn(rc, NIL_RTFILEAIOREQ);
+    }
+
+    return hReq;
+}
+
+/**
+ * Frees a async I/O request handle.
+ *
+ * @returns nothing.
+ * @param   pAioMgr    The I/O manager.
+ * @param   hReq       The I/O request handle to free.
+ */
+static void pdmacFileAioMgrNormalRequestFree(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq)
+{
+    Assert(pAioMgr->iFreeEntry < pAioMgr->cReqEntries);
+    Assert(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] == NIL_RTFILEAIOREQ);
+
+    pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] = hReq;
+    pAioMgr->iFreeEntry++;
+}
+
+/**
+ * Wrapper around RTFIleAioCtxSubmit() which is also doing error handling.
+ */
+static int pdmacFileAioMgrNormalReqsEnqueue(PPDMACEPFILEMGR pAioMgr,
+                                            PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                            PRTFILEAIOREQ pahReqs, unsigned cReqs)
+{
+    pAioMgr->cRequestsActive += cReqs;
+    pEndpoint->AioMgr.cRequestsActive += cReqs;
+
+    LogFlow(("Enqueuing %d requests. I/O manager has a total of %d active requests now\n", cReqs, pAioMgr->cRequestsActive));
+    LogFlow(("Endpoint has a total of %d active requests now\n", pEndpoint->AioMgr.cRequestsActive));
+
+    int rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, pahReqs, cReqs);
+    if (RT_FAILURE(rc))
+    {
+        if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
+        {
+            PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
+
+            /* Append any not submitted task to the waiting list. */
+            for (size_t i = 0; i < cReqs; i++)
+            {
+                int rcReq = RTFileAioReqGetRC(pahReqs[i], NULL);
+
+                if (rcReq != VERR_FILE_AIO_IN_PROGRESS)
+                {
+                    PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(pahReqs[i]);
+
+                    Assert(pTask->hReq == pahReqs[i]);
+                    pdmacFileAioMgrEpAddTask(pEndpoint, pTask);
+                    pAioMgr->cRequestsActive--;
+                    pEndpoint->AioMgr.cRequestsActive--;
+
+                    if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
+                    {
+                        /* Clear the pending flush */
+                        Assert(pEndpoint->pFlushReq == pTask);
+                        pEndpoint->pFlushReq = NULL;
+                    }
+                }
+            }
+
+            pAioMgr->cRequestsActiveMax = pAioMgr->cRequestsActive;
+
+            /* Print an entry in the release log */
+            if (RT_UNLIKELY(!pEpClass->fOutOfResourcesWarningPrinted))
+            {
+                pEpClass->fOutOfResourcesWarningPrinted = true;
+                LogRel(("AIOMgr: Host limits number of active IO requests to %u. Expect a performance impact.\n",
+                        pAioMgr->cRequestsActive));
+            }
+
+            LogFlow(("Removed requests. I/O manager has a total of %u active requests now\n", pAioMgr->cRequestsActive));
+            LogFlow(("Endpoint has a total of %u active requests now\n", pEndpoint->AioMgr.cRequestsActive));
+            rc = VINF_SUCCESS;
+        }
+        else /* Another kind of error happened (full disk, ...) */
+        {
+            /* An error happened. Find out which one caused the error and resubmit all other tasks. */
+            for (size_t i = 0; i < cReqs; i++)
+            {
+                int rcReq = RTFileAioReqGetRC(pahReqs[i], NULL);
+
+                if (rcReq == VERR_FILE_AIO_NOT_SUBMITTED)
+                {
+                    /* We call ourself again to do any error handling which might come up now. */
+                    rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, &pahReqs[i], 1);
+                    AssertRC(rc);
+                }
+                else if (rcReq != VERR_FILE_AIO_IN_PROGRESS)
+                    pdmacFileAioMgrNormalReqCompleteRc(pAioMgr, pahReqs[i], rcReq, 0);
+            }
+
+
+            if (    pEndpoint->pFlushReq
+                && !pAioMgr->cRequestsActive
+                && !pEndpoint->fAsyncFlushSupported)
+            {
+                /*
+                 * Complete a pending flush if we don't have requests enqueued and the host doesn't support
+                 * the async flush API.
+                 * Happens only if this we just noticed that this is not supported
+                 * and the only active request was a flush.
+                 */
+                PPDMACTASKFILE pFlush = pEndpoint->pFlushReq;
+                pEndpoint->pFlushReq = NULL;
+                pFlush->pfnCompleted(pFlush, pFlush->pvUser, VINF_SUCCESS);
+                pdmacFileTaskFree(pEndpoint, pFlush);
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+static bool pdmacFileAioMgrNormalIsRangeLocked(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                               RTFOFF offStart, size_t cbRange,
+                                               PPDMACTASKFILE pTask, bool fAlignedReq)
+{
+    AssertMsg(   pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
+              || pTask->enmTransferType == PDMACTASKFILETRANSFER_READ,
+                 ("Invalid task type %d\n", pTask->enmTransferType));
+
+    /*
+     * If there is no unaligned request active and the current one is aligned
+     * just pass it through.
+     */
+    if (!pEndpoint->AioMgr.cLockedReqsActive && fAlignedReq)
+        return false;
+
+    PPDMACFILERANGELOCK pRangeLock;
+    pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetRangeGet(pEndpoint->AioMgr.pTreeRangesLocked, offStart);
+    if (!pRangeLock)
+    {
+        pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetGetBestFit(pEndpoint->AioMgr.pTreeRangesLocked, offStart, true);
+        /* Check if we intersect with the range. */
+        if (   !pRangeLock
+            || !(   (pRangeLock->Core.Key) <= (offStart + (RTFOFF)cbRange - 1)
+                && (pRangeLock->Core.KeyLast) >= offStart))
+        {
+            pRangeLock = NULL; /* False alarm */
+        }
+    }
+
+    /* Check whether we have one of the situations explained below */
+    if (pRangeLock)
+    {
+        /* Add to the list. */
+        pTask->pNext = NULL;
+
+        if (!pRangeLock->pWaitingTasksHead)
+        {
+            Assert(!pRangeLock->pWaitingTasksTail);
+            pRangeLock->pWaitingTasksHead = pTask;
+            pRangeLock->pWaitingTasksTail = pTask;
+        }
+        else
+        {
+            AssertPtr(pRangeLock->pWaitingTasksTail);
+            pRangeLock->pWaitingTasksTail->pNext = pTask;
+            pRangeLock->pWaitingTasksTail = pTask;
+        }
+        return true;
+    }
+
+    return false;
+}
+
+static int pdmacFileAioMgrNormalRangeLock(PPDMACEPFILEMGR pAioMgr,
+                                          PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                          RTFOFF offStart, size_t cbRange,
+                                          PPDMACTASKFILE pTask, bool fAlignedReq)
+{
+    LogFlowFunc(("pAioMgr=%#p pEndpoint=%#p offStart=%RTfoff cbRange=%zu pTask=%#p\n",
+                 pAioMgr, pEndpoint, offStart, cbRange, pTask));
+
+    AssertMsg(!pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbRange, pTask, fAlignedReq),
+              ("Range is already locked offStart=%RTfoff cbRange=%u\n",
+               offStart, cbRange));
+
+    /*
+     * If there is no unaligned request active and the current one is aligned
+     * just don't use the lock.
+     */
+    if (!pEndpoint->AioMgr.cLockedReqsActive && fAlignedReq)
+    {
+        pTask->pRangeLock = NULL;
+        return VINF_SUCCESS;
+    }
+
+    PPDMACFILERANGELOCK pRangeLock = (PPDMACFILERANGELOCK)RTMemCacheAlloc(pAioMgr->hMemCacheRangeLocks);
+    if (!pRangeLock)
+        return VERR_NO_MEMORY;
+
+    /* Init the lock. */
+    pRangeLock->Core.Key          = offStart;
+    pRangeLock->Core.KeyLast      = offStart + cbRange - 1;
+    pRangeLock->cRefs             = 1;
+    pRangeLock->fReadLock         = pTask->enmTransferType == PDMACTASKFILETRANSFER_READ;
+    pRangeLock->pWaitingTasksHead = NULL;
+    pRangeLock->pWaitingTasksTail = NULL;
+
+    bool fInserted = RTAvlrFileOffsetInsert(pEndpoint->AioMgr.pTreeRangesLocked, &pRangeLock->Core);
+    AssertMsg(fInserted, ("Range lock was not inserted!\n")); NOREF(fInserted);
+
+    /* Let the task point to its lock. */
+    pTask->pRangeLock = pRangeLock;
+    pEndpoint->AioMgr.cLockedReqsActive++;
+
+    return VINF_SUCCESS;
+}
+
+static PPDMACTASKFILE pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
+                                                         PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                                         PPDMACFILERANGELOCK pRangeLock)
+{
+    PPDMACTASKFILE pTasksWaitingHead;
+
+    LogFlowFunc(("pAioMgr=%#p pEndpoint=%#p pRangeLock=%#p\n",
+                 pAioMgr, pEndpoint, pRangeLock));
+
+    /* pRangeLock can be NULL if there was no lock assigned with the task. */
+    if (!pRangeLock)
+        return NULL;
+
+    Assert(pRangeLock->cRefs == 1);
+
+    RTAvlrFileOffsetRemove(pEndpoint->AioMgr.pTreeRangesLocked, pRangeLock->Core.Key);
+    pTasksWaitingHead = pRangeLock->pWaitingTasksHead;
+    pRangeLock->pWaitingTasksHead = NULL;
+    pRangeLock->pWaitingTasksTail = NULL;
+    RTMemCacheFree(pAioMgr->hMemCacheRangeLocks, pRangeLock);
+    pEndpoint->AioMgr.cLockedReqsActive--;
+
+    return pTasksWaitingHead;
+}
+
+static int pdmacFileAioMgrNormalTaskPrepareBuffered(PPDMACEPFILEMGR pAioMgr,
+                                                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                                    PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
+{
+    AssertMsg(   pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
+              || (uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) <= pEndpoint->cbFile,
+              ("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
+               pTask->Off, pTask->DataSeg.cbSeg, pEndpoint->cbFile));
+
+    pTask->fPrefetch = false;
+    pTask->cbBounceBuffer = 0;
+
+    /*
+     * Before we start to setup the request we have to check whether there is a task
+     * already active which range intersects with ours. We have to defer execution
+     * of this task in two cases:
+     *     - The pending task is a write and the current is either read or write
+     *     - The pending task is a read and the current task is a write task.
+     *
+     * To check whether a range is currently "locked" we use the AVL tree where every pending task
+     * is stored by its file offset range. The current task will be added to the active task
+     * and will be executed when the active one completes. (The method below
+     * which checks whether a range is already used will add the task)
+     *
+     * This is necessary because of the requirement to align all requests to a 512 boundary
+     * which is enforced by the host OS (Linux and Windows atm). It is possible that
+     * we have to process unaligned tasks and need to align them using bounce buffers.
+     * While the data is fetched from the file another request might arrive writing to
+     * the same range. This will result in data corruption if both are executed concurrently.
+     */
+    int  rc = VINF_SUCCESS;
+    bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, pTask->Off, pTask->DataSeg.cbSeg, pTask,
+                                                      true /* fAlignedReq */);
+    if (!fLocked)
+    {
+        /* Get a request handle. */
+        RTFILEAIOREQ hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
+        AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
+
+        if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
+        {
+            /* Grow the file if needed. */
+            if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
+            {
+                ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
+                RTFileSetSize(pEndpoint->hFile, pTask->Off + pTask->DataSeg.cbSeg);
+            }
+
+            rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->hFile,
+                                          pTask->Off, pTask->DataSeg.pvSeg,
+                                          pTask->DataSeg.cbSeg, pTask);
+        }
+        else
+            rc = RTFileAioReqPrepareRead(hReq, pEndpoint->hFile,
+                                         pTask->Off, pTask->DataSeg.pvSeg,
+                                         pTask->DataSeg.cbSeg, pTask);
+        AssertRC(rc);
+
+        rc = pdmacFileAioMgrNormalRangeLock(pAioMgr, pEndpoint, pTask->Off,
+                                            pTask->DataSeg.cbSeg,
+                                            pTask, true /* fAlignedReq */);
+
+        if (RT_SUCCESS(rc))
+        {
+            pTask->hReq = hReq;
+            *phReq = hReq;
+        }
+    }
+    else
+        LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
+
+    return rc;
+}
+
+static int pdmacFileAioMgrNormalTaskPrepareNonBuffered(PPDMACEPFILEMGR pAioMgr,
+                                                       PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                                       PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
+{
+    /*
+     * Check if the alignment requirements are met.
+     * Offset, transfer size and buffer address
+     * need to be on a 512 boundary.
+     */
+    RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
+    size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg + (pTask->Off - offStart), 512);
+    PDMACTASKFILETRANSFER enmTransferType = pTask->enmTransferType;
+    bool fAlignedReq =     cbToTransfer == pTask->DataSeg.cbSeg
+                        && offStart == pTask->Off;
+
+    AssertMsg(   pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
+                || (uint64_t)(offStart + cbToTransfer) <= pEndpoint->cbFile,
+                ("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
+                offStart, cbToTransfer, pEndpoint->cbFile));
+
+    pTask->fPrefetch = false;
+
+    /*
+     * Before we start to setup the request we have to check whether there is a task
+     * already active which range intersects with ours. We have to defer execution
+     * of this task in two cases:
+     *     - The pending task is a write and the current is either read or write
+     *     - The pending task is a read and the current task is a write task.
+     *
+     * To check whether a range is currently "locked" we use the AVL tree where every pending task
+     * is stored by its file offset range. The current task will be added to the active task
+     * and will be executed when the active one completes. (The method below
+     * which checks whether a range is already used will add the task)
+     *
+     * This is necessary because of the requirement to align all requests to a 512 boundary
+     * which is enforced by the host OS (Linux and Windows atm). It is possible that
+     * we have to process unaligned tasks and need to align them using bounce buffers.
+     * While the data is fetched from the file another request might arrive writing to
+     * the same range. This will result in data corruption if both are executed concurrently.
+     */
+    int  rc = VINF_SUCCESS;
+    bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbToTransfer, pTask, fAlignedReq);
+    if (!fLocked)
+    {
+        PPDMASYNCCOMPLETIONEPCLASSFILE  pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
+        void                           *pvBuf        = pTask->DataSeg.pvSeg;
+
+        /* Get a request handle. */
+        RTFILEAIOREQ hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
+        AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
+
+        if (   !fAlignedReq
+            || ((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) != (RTR3UINTPTR)pvBuf))
+        {
+            LogFlow(("Using bounce buffer for task %#p cbToTransfer=%zd cbSeg=%zd offStart=%RTfoff off=%RTfoff\n",
+                     pTask, cbToTransfer, pTask->DataSeg.cbSeg, offStart, pTask->Off));
+
+            /* Create bounce buffer. */
+            pTask->cbBounceBuffer = cbToTransfer;
+
+            AssertMsg(pTask->Off >= offStart, ("Overflow in calculation Off=%llu offStart=%llu\n",
+                      pTask->Off, offStart));
+            pTask->offBounceBuffer = pTask->Off - offStart;
+
+            /** @todo I think we need something like a RTMemAllocAligned method here.
+             * Current assumption is that the maximum alignment is 4096byte
+             * (GPT disk on Windows)
+             * so we can use RTMemPageAlloc here.
+             */
+            pTask->pvBounceBuffer = RTMemPageAlloc(cbToTransfer);
+            if (RT_LIKELY(pTask->pvBounceBuffer))
+            {
+                pvBuf = pTask->pvBounceBuffer;
+
+                if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
+                {
+                    if (   RT_UNLIKELY(cbToTransfer != pTask->DataSeg.cbSeg)
+                        || RT_UNLIKELY(offStart != pTask->Off))
+                    {
+                        /* We have to fill the buffer first before we can update the data. */
+                        LogFlow(("Prefetching data for task %#p\n", pTask));
+                        pTask->fPrefetch = true;
+                        enmTransferType = PDMACTASKFILETRANSFER_READ;
+                    }
+                    else
+                        memcpy(pvBuf, pTask->DataSeg.pvSeg, pTask->DataSeg.cbSeg);
+                }
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+        else
+            pTask->cbBounceBuffer = 0;
+
+        if (RT_SUCCESS(rc))
+        {
+            AssertMsg((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) == (RTR3UINTPTR)pvBuf,
+                      ("AIO: Alignment restrictions not met! pvBuf=%p uBitmaskAlignment=%p\n", pvBuf, pEpClassFile->uBitmaskAlignment));
+
+            if (enmTransferType == PDMACTASKFILETRANSFER_WRITE)
+            {
+                /* Grow the file if needed. */
+                if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
+                {
+                    ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
+                    RTFileSetSize(pEndpoint->hFile, pTask->Off + pTask->DataSeg.cbSeg);
+                }
+
+                rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->hFile,
+                                              offStart, pvBuf, cbToTransfer, pTask);
+            }
+            else
+                rc = RTFileAioReqPrepareRead(hReq, pEndpoint->hFile,
+                                             offStart, pvBuf, cbToTransfer, pTask);
+            AssertRC(rc);
+
+            rc = pdmacFileAioMgrNormalRangeLock(pAioMgr, pEndpoint, offStart, cbToTransfer, pTask, fAlignedReq);
+            if (RT_SUCCESS(rc))
+            {
+                pTask->hReq = hReq;
+                *phReq = hReq;
+            }
+            else
+            {
+                /* Cleanup */
+                if (pTask->cbBounceBuffer)
+                    RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
+            }
+        }
+    }
+    else
+        LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
+
+    return rc;
+}
+
+static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
+                                                PPDMACEPFILEMGR pAioMgr,
+                                                PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    RTFILEAIOREQ  apReqs[20];
+    unsigned      cRequests = 0;
+    int           rc        = VINF_SUCCESS;
+
+    AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
+              ("Trying to process request lists of a non active endpoint!\n"));
+
+    /* Go through the list and queue the requests until we get a flush request */
+    while (   pTaskHead
+           && !pEndpoint->pFlushReq
+           && (pAioMgr->cRequestsActive + cRequests < pAioMgr->cRequestsActiveMax)
+           && RT_SUCCESS(rc))
+    {
+        RTMSINTERVAL msWhenNext;
+        PPDMACTASKFILE pCurr = pTaskHead;
+
+        if (!pdmacEpIsTransferAllowed(&pEndpoint->Core, (uint32_t)pCurr->DataSeg.cbSeg, &msWhenNext))
+        {
+            pAioMgr->msBwLimitExpired = RT_MIN(pAioMgr->msBwLimitExpired, msWhenNext);
+            break;
+        }
+
+        pTaskHead = pTaskHead->pNext;
+
+        pCurr->pNext = NULL;
+
+        AssertMsg(VALID_PTR(pCurr->pEndpoint) && (pCurr->pEndpoint == pEndpoint),
+                  ("Endpoints do not match\n"));
+
+        switch (pCurr->enmTransferType)
+        {
+            case PDMACTASKFILETRANSFER_FLUSH:
+            {
+                /* If there is no data transfer request this flush request finished immediately. */
+                if (pEndpoint->fAsyncFlushSupported)
+                {
+                    /* Issue a flush to the host. */
+                    RTFILEAIOREQ hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
+                    AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
+
+                    LogFlow(("Flush request %#p\n", hReq));
+
+                    rc = RTFileAioReqPrepareFlush(hReq, pEndpoint->hFile, pCurr);
+                    if (RT_FAILURE(rc))
+                    {
+                        if (rc == VERR_NOT_SUPPORTED)
+                            LogRel(("AIOMgr: Async flushes not supported\n"));
+                        else
+                            LogRel(("AIOMgr: Preparing flush failed with %Rrc, disabling async flushes\n", rc));
+                        pEndpoint->fAsyncFlushSupported = false;
+                        pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
+                        rc = VINF_SUCCESS; /* Fake success */
+                    }
+                    else
+                    {
+                        pCurr->hReq = hReq;
+                        apReqs[cRequests] = hReq;
+                        pEndpoint->AioMgr.cReqsProcessed++;
+                        cRequests++;
+                    }
+                }
+
+                if (   !pEndpoint->AioMgr.cRequestsActive
+                    && !pEndpoint->fAsyncFlushSupported)
+                {
+                    pCurr->pfnCompleted(pCurr, pCurr->pvUser, VINF_SUCCESS);
+                    pdmacFileTaskFree(pEndpoint, pCurr);
+                }
+                else
+                {
+                    Assert(!pEndpoint->pFlushReq);
+                    pEndpoint->pFlushReq = pCurr;
+                }
+                break;
+            }
+            case PDMACTASKFILETRANSFER_READ:
+            case PDMACTASKFILETRANSFER_WRITE:
+            {
+                RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
+
+                if (pCurr->hReq == NIL_RTFILEAIOREQ)
+                {
+                    if (pEndpoint->enmBackendType == PDMACFILEEPBACKEND_BUFFERED)
+                        rc = pdmacFileAioMgrNormalTaskPrepareBuffered(pAioMgr, pEndpoint, pCurr, &hReq);
+                    else if (pEndpoint->enmBackendType == PDMACFILEEPBACKEND_NON_BUFFERED)
+                        rc = pdmacFileAioMgrNormalTaskPrepareNonBuffered(pAioMgr, pEndpoint, pCurr, &hReq);
+                    else
+                        AssertMsgFailed(("Invalid backend type %d\n", pEndpoint->enmBackendType));
+
+                    AssertRC(rc);
+                }
+                else
+                {
+                    LogFlow(("Task %#p has I/O request %#p already\n", pCurr, pCurr->hReq));
+                    hReq = pCurr->hReq;
+                }
+
+                LogFlow(("Read/Write request %#p\n", hReq));
+
+                if (hReq != NIL_RTFILEAIOREQ)
+                {
+                    apReqs[cRequests] = hReq;
+                    cRequests++;
+                }
+                break;
+            }
+            default:
+                AssertMsgFailed(("Invalid transfer type %d\n", pCurr->enmTransferType));
+        } /* switch transfer type */
+
+        /* Queue the requests if the array is full. */
+        if (cRequests == RT_ELEMENTS(apReqs))
+        {
+            rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
+            cRequests = 0;
+            AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
+                      ("Unexpected return code\n"));
+        }
+    }
+
+    if (cRequests)
+    {
+        rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
+        AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
+                  ("Unexpected return code rc=%Rrc\n", rc));
+    }
+
+    if (pTaskHead)
+    {
+        /* Add the rest of the tasks to the pending list */
+        pdmacFileAioMgrEpAddTaskList(pEndpoint, pTaskHead);
+
+        if (RT_UNLIKELY(   pAioMgr->cRequestsActiveMax == pAioMgr->cRequestsActive
+                        && !pEndpoint->pFlushReq))
+        {
+#if 0
+            /*
+             * The I/O manager has no room left for more requests
+             * but there are still requests to process.
+             * Create a new I/O manager and let it handle some endpoints.
+             */
+            pdmacFileAioMgrNormalBalanceLoad(pAioMgr);
+#else
+            /* Grow the I/O manager */
+            pAioMgr->enmState = PDMACEPFILEMGRSTATE_GROWING;
+#endif
+        }
+    }
+
+    /* Insufficient resources are not fatal. */
+    if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
+        rc = VINF_SUCCESS;
+
+    return rc;
+}
+
+/**
+ * Adds all pending requests for the given endpoint
+ * until a flush request is encountered or there is no
+ * request anymore.
+ *
+ * @returns VBox status code.
+ * @param   pAioMgr    The async I/O manager for the endpoint
+ * @param   pEndpoint  The endpoint to get the requests from.
+ */
+static int pdmacFileAioMgrNormalQueueReqs(PPDMACEPFILEMGR pAioMgr,
+                                          PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    int rc = VINF_SUCCESS;
+    PPDMACTASKFILE pTasksHead = NULL;
+
+    AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
+              ("Trying to process request lists of a non active endpoint!\n"));
+
+    Assert(!pEndpoint->pFlushReq);
+
+    /* Check the pending list first */
+    if (pEndpoint->AioMgr.pReqsPendingHead)
+    {
+        LogFlow(("Queuing pending requests first\n"));
+
+        pTasksHead = pEndpoint->AioMgr.pReqsPendingHead;
+        /*
+         * Clear the list as the processing routine will insert them into the list
+         * again if it gets a flush request.
+         */
+        pEndpoint->AioMgr.pReqsPendingHead = NULL;
+        pEndpoint->AioMgr.pReqsPendingTail = NULL;
+        rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
+        AssertRC(rc);                   /** @todo r=bird: status code potentially overwritten.  */
+    }
+
+    if (!pEndpoint->pFlushReq && !pEndpoint->AioMgr.pReqsPendingHead)
+    {
+        /* Now the request queue. */
+        pTasksHead = pdmacFileEpGetNewTasks(pEndpoint);
+        if (pTasksHead)
+        {
+            rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
+            AssertRC(rc);
+        }
+    }
+
+    return rc;
+}
+
+static int pdmacFileAioMgrNormalProcessBlockingEvent(PPDMACEPFILEMGR pAioMgr)
+{
+    int rc = VINF_SUCCESS;
+    bool fNotifyWaiter = false;
+
+    LogFlowFunc((": Enter\n"));
+
+    Assert(pAioMgr->fBlockingEventPending);
+
+    switch (pAioMgr->enmBlockingEvent)
+    {
+        case PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT:
+        {
+            PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointNew = ASMAtomicReadPtrT(&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint, PPDMASYNCCOMPLETIONENDPOINTFILE);
+            AssertMsg(VALID_PTR(pEndpointNew), ("Adding endpoint event without a endpoint to add\n"));
+
+            pEndpointNew->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
+
+            pEndpointNew->AioMgr.pEndpointNext = pAioMgr->pEndpointsHead;
+            pEndpointNew->AioMgr.pEndpointPrev = NULL;
+            if (pAioMgr->pEndpointsHead)
+                pAioMgr->pEndpointsHead->AioMgr.pEndpointPrev = pEndpointNew;
+            pAioMgr->pEndpointsHead = pEndpointNew;
+
+            /* Assign the completion point to this file. */
+            rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pEndpointNew->hFile);
+            fNotifyWaiter = true;
+            pAioMgr->cEndpoints++;
+            break;
+        }
+        case PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT:
+        {
+            PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove = ASMAtomicReadPtrT(&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint, PPDMASYNCCOMPLETIONENDPOINTFILE);
+            AssertMsg(VALID_PTR(pEndpointRemove), ("Removing endpoint event without a endpoint to remove\n"));
+
+            pEndpointRemove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
+            fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointRemove);
+            break;
+        }
+        case PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT:
+        {
+            PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointClose = ASMAtomicReadPtrT(&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint, PPDMASYNCCOMPLETIONENDPOINTFILE);
+            AssertMsg(VALID_PTR(pEndpointClose), ("Close endpoint event without a endpoint to close\n"));
+
+            if (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
+            {
+                LogFlowFunc((": Closing endpoint %#p{%s}\n", pEndpointClose, pEndpointClose->Core.pszUri));
+
+                /* Make sure all tasks finished. Process the queues a last time first. */
+                rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpointClose);
+                AssertRC(rc);
+
+                pEndpointClose->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING;
+                fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointClose);
+            }
+            else if (   (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING)
+                     && (!pEndpointClose->AioMgr.cRequestsActive))
+                fNotifyWaiter = true;
+            break;
+        }
+        case PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN:
+        {
+            pAioMgr->enmState = PDMACEPFILEMGRSTATE_SHUTDOWN;
+            if (!pAioMgr->cRequestsActive)
+                fNotifyWaiter = true;
+            break;
+        }
+        case PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND:
+        {
+            pAioMgr->enmState = PDMACEPFILEMGRSTATE_SUSPENDING;
+            break;
+        }
+        case PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME:
+        {
+            pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
+            fNotifyWaiter = true;
+            break;
+        }
+        default:
+            AssertReleaseMsgFailed(("Invalid event type %d\n", pAioMgr->enmBlockingEvent));
+    }
+
+    if (fNotifyWaiter)
+    {
+        ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
+        pAioMgr->enmBlockingEvent = PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID;
+
+        /* Release the waiting thread. */
+        LogFlow(("Signalling waiter\n"));
+        rc = RTSemEventSignal(pAioMgr->EventSemBlock);
+        AssertRC(rc);
+    }
+
+    LogFlowFunc((": Leave\n"));
+    return rc;
+}
+
+/**
+ * Checks all endpoints for pending events or new requests.
+ *
+ * @returns VBox status code.
+ * @param   pAioMgr    The I/O manager handle.
+ */
+static int pdmacFileAioMgrNormalCheckEndpoints(PPDMACEPFILEMGR pAioMgr)
+{
+    /* Check the assigned endpoints for new tasks if there isn't a flush request active at the moment. */
+    int rc = VINF_SUCCESS;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pAioMgr->pEndpointsHead;
+
+    pAioMgr->msBwLimitExpired = RT_INDEFINITE_WAIT;
+
+    while (pEndpoint)
+    {
+        if (!pEndpoint->pFlushReq
+            && (pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
+            && !pEndpoint->AioMgr.fMoving)
+        {
+            rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpoint);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+        else if (   !pEndpoint->AioMgr.cRequestsActive
+                 && pEndpoint->enmState != PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
+        {
+            /* Reopen the file so that the new endpoint can re-associate with the file */
+            RTFileClose(pEndpoint->hFile);
+            rc = RTFileOpen(&pEndpoint->hFile, pEndpoint->Core.pszUri, pEndpoint->fFlags);
+            AssertRC(rc);
+
+            if (pEndpoint->AioMgr.fMoving)
+            {
+                pEndpoint->AioMgr.fMoving = false;
+                pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
+            }
+            else
+            {
+                Assert(pAioMgr->fBlockingEventPending);
+                ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
+
+                /* Release the waiting thread. */
+                LogFlow(("Signalling waiter\n"));
+                rc = RTSemEventSignal(pAioMgr->EventSemBlock);
+                AssertRC(rc);
+            }
+        }
+
+        pEndpoint = pEndpoint->AioMgr.pEndpointNext;
+    }
+
+    return rc;
+}
+
+/**
+ * Wrapper around pdmacFileAioMgrNormalReqCompleteRc().
+ */
+static void pdmacFileAioMgrNormalReqComplete(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq)
+{
+    size_t cbTransfered = 0;
+    int rcReq = RTFileAioReqGetRC(hReq, &cbTransfered);
+
+    pdmacFileAioMgrNormalReqCompleteRc(pAioMgr, hReq, rcReq, cbTransfered);
+}
+
+static void pdmacFileAioMgrNormalReqCompleteRc(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq,
+                                               int rcReq, size_t cbTransfered)
+{
+    int rc = VINF_SUCCESS;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
+    PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(hReq);
+    PPDMACTASKFILE pTasksWaiting;
+
+    LogFlowFunc(("pAioMgr=%#p hReq=%#p\n", pAioMgr, hReq));
+
+    pEndpoint = pTask->pEndpoint;
+
+    pTask->hReq = NIL_RTFILEAIOREQ;
+
+    pAioMgr->cRequestsActive--;
+    pEndpoint->AioMgr.cRequestsActive--;
+    pEndpoint->AioMgr.cReqsProcessed++;
+
+    /*
+     * It is possible that the request failed on Linux with kernels < 2.6.23
+     * if the passed buffer was allocated with remap_pfn_range or if the file
+     * is on an NFS endpoint which does not support async and direct I/O at the same time.
+     * The endpoint will be migrated to a failsafe manager in case a request fails.
+     */
+    if (RT_FAILURE(rcReq))
+    {
+        /* Free bounce buffers and the IPRT request. */
+        pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
+
+        if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
+        {
+            LogRel(("AIOMgr: Flush failed with %Rrc, disabling async flushes\n", rcReq));
+            pEndpoint->fAsyncFlushSupported = false;
+            AssertMsg(pEndpoint->pFlushReq == pTask, ("Failed flush request doesn't match active one\n"));
+            /* The other method will take over now. */
+
+            pEndpoint->pFlushReq = NULL;
+            /* Call completion callback */
+            LogFlow(("Flush task=%#p completed with %Rrc\n", pTask, VINF_SUCCESS));
+            pTask->pfnCompleted(pTask, pTask->pvUser, VINF_SUCCESS);
+            pdmacFileTaskFree(pEndpoint, pTask);
+        }
+        else
+        {
+            /* Free the lock and process pending tasks if necessary */
+            pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
+            rc = pdmacFileAioMgrNormalProcessTaskList(pTasksWaiting, pAioMgr, pEndpoint);
+            AssertRC(rc);
+
+            if (pTask->cbBounceBuffer)
+                RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
+
+            /*
+             * Fatal errors are reported to the guest and non-fatal errors
+             * will cause a migration to the failsafe manager in the hope
+             * that the error disappears.
+             */
+            if (!pdmacFileAioMgrNormalRcIsFatal(rcReq))
+            {
+                /* Queue the request on the pending list. */
+                pTask->pNext = pEndpoint->AioMgr.pReqsPendingHead;
+                pEndpoint->AioMgr.pReqsPendingHead = pTask;
+
+                /* Create a new failsafe manager if necessary. */
+                if (!pEndpoint->AioMgr.fMoving)
+                {
+                    PPDMACEPFILEMGR pAioMgrFailsafe;
+
+                    LogRel(("%s: Request %#p failed with rc=%Rrc, migrating endpoint %s to failsafe manager.\n",
+                            RTThreadGetName(pAioMgr->Thread), pTask, rcReq, pEndpoint->Core.pszUri));
+
+                    pEndpoint->AioMgr.fMoving = true;
+
+                    rc = pdmacFileAioMgrCreate((PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass,
+                                                &pAioMgrFailsafe, PDMACEPFILEMGRTYPE_SIMPLE);
+                    AssertRC(rc);
+
+                    pEndpoint->AioMgr.pAioMgrDst = pAioMgrFailsafe;
+
+                    /* Update the flags to open the file with. Disable async I/O and enable the host cache. */
+                    pEndpoint->fFlags &= ~(RTFILE_O_ASYNC_IO | RTFILE_O_NO_CACHE);
+                }
+
+                /* If this was the last request for the endpoint migrate it to the new manager. */
+                if (!pEndpoint->AioMgr.cRequestsActive)
+                {
+                    bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
+                    Assert(!fReqsPending); NOREF(fReqsPending);
+
+                    rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
+                    AssertRC(rc);
+                }
+            }
+            else
+            {
+                pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
+                pdmacFileTaskFree(pEndpoint, pTask);
+            }
+        }
+    }
+    else
+    {
+        if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
+        {
+            /* Clear pending flush */
+            AssertMsg(pEndpoint->pFlushReq == pTask, ("Completed flush request doesn't match active one\n"));
+            pEndpoint->pFlushReq = NULL;
+            pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
+
+            /* Call completion callback */
+            LogFlow(("Flush task=%#p completed with %Rrc\n", pTask, rcReq));
+            pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
+            pdmacFileTaskFree(pEndpoint, pTask);
+        }
+        else
+        {
+            /*
+             * Restart an incomplete transfer.
+             * This usually means that the request will return an error now
+             * but to get the cause of the error (disk full, file too big, I/O error, ...)
+             * the transfer needs to be continued.
+             */
+            pTask->cbTransfered += cbTransfered;
+
+            if (RT_UNLIKELY(   pTask->cbTransfered < pTask->DataSeg.cbSeg
+                            || (   pTask->cbBounceBuffer
+                                && pTask->cbTransfered < pTask->cbBounceBuffer)))
+            {
+                RTFOFF offStart;
+                size_t cbToTransfer;
+                uint8_t *pbBuf = NULL;
+
+                LogFlow(("Restarting incomplete transfer %#p (%zu bytes transferred)\n",
+                         pTask, cbTransfered));
+                Assert(cbTransfered % 512 == 0);
+
+                if (pTask->cbBounceBuffer)
+                {
+                    AssertPtr(pTask->pvBounceBuffer);
+                    offStart     = (pTask->Off & ~((RTFOFF)512-1)) + pTask->cbTransfered;
+                    cbToTransfer = pTask->cbBounceBuffer - pTask->cbTransfered;
+                    pbBuf        = (uint8_t *)pTask->pvBounceBuffer + pTask->cbTransfered;
+                }
+                else
+                {
+                    Assert(!pTask->pvBounceBuffer);
+                    offStart     = pTask->Off + pTask->cbTransfered;
+                    cbToTransfer = pTask->DataSeg.cbSeg - pTask->cbTransfered;
+                    pbBuf        = (uint8_t *)pTask->DataSeg.pvSeg + pTask->cbTransfered;
+                }
+
+                if (pTask->fPrefetch || pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
+                {
+                    rc = RTFileAioReqPrepareRead(hReq, pEndpoint->hFile, offStart,
+                                                 pbBuf, cbToTransfer, pTask);
+                }
+                else
+                {
+                    AssertMsg(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE,
+                                  ("Invalid transfer type\n"));
+                    rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->hFile, offStart,
+                                                  pbBuf, cbToTransfer, pTask);
+                }
+                AssertRC(rc);
+
+                pTask->hReq = hReq;
+                rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, &hReq, 1);
+                AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
+                          ("Unexpected return code rc=%Rrc\n", rc));
+            }
+            else if (pTask->fPrefetch)
+            {
+                Assert(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE);
+                Assert(pTask->cbBounceBuffer);
+
+                memcpy(((uint8_t *)pTask->pvBounceBuffer) + pTask->offBounceBuffer,
+                        pTask->DataSeg.pvSeg,
+                        pTask->DataSeg.cbSeg);
+
+                /* Write it now. */
+                pTask->fPrefetch = false;
+                RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
+                size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg + (pTask->Off - offStart), 512);
+
+                pTask->cbTransfered = 0;
+
+                /* Grow the file if needed. */
+                if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
+                {
+                    ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
+                    RTFileSetSize(pEndpoint->hFile, pTask->Off + pTask->DataSeg.cbSeg);
+                }
+
+                rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->hFile,
+                                              offStart, pTask->pvBounceBuffer, cbToTransfer, pTask);
+                AssertRC(rc);
+                pTask->hReq = hReq;
+                rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, &hReq, 1);
+                AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
+                          ("Unexpected return code rc=%Rrc\n", rc));
+            }
+            else
+            {
+                if (RT_SUCCESS(rc) && pTask->cbBounceBuffer)
+                {
+                    if (pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
+                        memcpy(pTask->DataSeg.pvSeg,
+                               ((uint8_t *)pTask->pvBounceBuffer) + pTask->offBounceBuffer,
+                               pTask->DataSeg.cbSeg);
+
+                    RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
+                }
+
+                pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
+
+                /* Free the lock and process pending tasks if necessary */
+                pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
+                if (pTasksWaiting)
+                {
+                    rc = pdmacFileAioMgrNormalProcessTaskList(pTasksWaiting, pAioMgr, pEndpoint);
+                    AssertRC(rc);
+                }
+
+                /* Call completion callback */
+                LogFlow(("Task=%#p completed with %Rrc\n", pTask, rcReq));
+                pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
+                pdmacFileTaskFree(pEndpoint, pTask);
+
+                /*
+                 * If there is no request left on the endpoint but a flush request is set
+                 * it completed now and we notify the owner.
+                 * Furthermore we look for new requests and continue.
+                 */
+                if (!pEndpoint->AioMgr.cRequestsActive && pEndpoint->pFlushReq)
+                {
+                    /* Call completion callback */
+                    pTask = pEndpoint->pFlushReq;
+                    pEndpoint->pFlushReq = NULL;
+
+                    AssertMsg(pTask->pEndpoint == pEndpoint, ("Endpoint of the flush request does not match assigned one\n"));
+
+                    pTask->pfnCompleted(pTask, pTask->pvUser, VINF_SUCCESS);
+                    pdmacFileTaskFree(pEndpoint, pTask);
+                }
+                else if (RT_UNLIKELY(!pEndpoint->AioMgr.cRequestsActive && pEndpoint->AioMgr.fMoving))
+                {
+                    /* If the endpoint is about to be migrated do it now. */
+                    bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
+                    Assert(!fReqsPending); NOREF(fReqsPending);
+
+                    rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
+                    AssertRC(rc);
+                }
+            }
+        } /* Not a flush request */
+    } /* request completed successfully */
+}
+
+/** Helper macro for checking for error codes. */
+#define CHECK_RC(pAioMgr, rc) \
+    if (RT_FAILURE(rc)) \
+    {\
+        int rc2 = pdmacFileAioMgrNormalErrorHandler(pAioMgr, rc, RT_SRC_POS);\
+        return rc2;\
+    }
+
+/**
+ * The normal I/O manager using the RTFileAio* API
+ *
+ * @returns VBox status code.
+ * @param   hThreadSelf Handle of the thread.
+ * @param   pvUser      Opaque user data.
+ */
+DECLCALLBACK(int) pdmacFileAioMgrNormal(RTTHREAD hThreadSelf, void *pvUser)
+{
+    int             rc          = VINF_SUCCESS;
+    PPDMACEPFILEMGR pAioMgr     = (PPDMACEPFILEMGR)pvUser;
+    uint64_t        uMillisEnd  = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
+    NOREF(hThreadSelf);
+
+    while (   pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING
+           || pAioMgr->enmState == PDMACEPFILEMGRSTATE_SUSPENDING
+           || pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING)
+    {
+        if (!pAioMgr->cRequestsActive)
+        {
+            ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, true);
+            if (!ASMAtomicReadBool(&pAioMgr->fWokenUp))
+                rc = RTSemEventWait(pAioMgr->EventSem, pAioMgr->msBwLimitExpired);
+            ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, false);
+            Assert(RT_SUCCESS(rc) || rc == VERR_TIMEOUT);
+
+            LogFlow(("Got woken up\n"));
+            ASMAtomicWriteBool(&pAioMgr->fWokenUp, false);
+        }
+
+        /* Check for an external blocking event first. */
+        if (pAioMgr->fBlockingEventPending)
+        {
+            rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
+            CHECK_RC(pAioMgr, rc);
+        }
+
+        if (RT_LIKELY(    pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING
+                      ||  pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING))
+        {
+            /* We got woken up because an endpoint issued new requests. Queue them. */
+            rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
+            CHECK_RC(pAioMgr, rc);
+
+            while (pAioMgr->cRequestsActive)
+            {
+                RTFILEAIOREQ apReqs[20];
+                uint32_t     cReqsCompleted = 0;
+                size_t       cReqsWait;
+
+                if (pAioMgr->cRequestsActive > RT_ELEMENTS(apReqs))
+                    cReqsWait = RT_ELEMENTS(apReqs);
+                else
+                    cReqsWait = pAioMgr->cRequestsActive;
+
+                LogFlow(("Waiting for %d of %d tasks to complete\n", 1, cReqsWait));
+
+                rc = RTFileAioCtxWait(pAioMgr->hAioCtx,
+                                      1,
+                                      RT_INDEFINITE_WAIT, apReqs,
+                                      cReqsWait, &cReqsCompleted);
+                if (RT_FAILURE(rc) && (rc != VERR_INTERRUPTED))
+                    CHECK_RC(pAioMgr, rc);
+
+                LogFlow(("%d tasks completed\n", cReqsCompleted));
+
+                for (uint32_t i = 0; i < cReqsCompleted; i++)
+                    pdmacFileAioMgrNormalReqComplete(pAioMgr, apReqs[i]);
+
+                /* Check for an external blocking event before we go to sleep again. */
+                if (pAioMgr->fBlockingEventPending)
+                {
+                    rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
+                    CHECK_RC(pAioMgr, rc);
+                }
+
+                /* Update load statistics. */
+                uint64_t uMillisCurr = RTTimeMilliTS();
+                if (uMillisCurr > uMillisEnd)
+                {
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointCurr = pAioMgr->pEndpointsHead;
+
+                    /* Calculate timespan. */
+                    uMillisCurr -= uMillisEnd;
+
+                    while (pEndpointCurr)
+                    {
+                        pEndpointCurr->AioMgr.cReqsPerSec    = pEndpointCurr->AioMgr.cReqsProcessed / (uMillisCurr + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD);
+                        pEndpointCurr->AioMgr.cReqsProcessed = 0;
+                        pEndpointCurr = pEndpointCurr->AioMgr.pEndpointNext;
+                    }
+
+                    /* Set new update interval */
+                    uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
+                }
+
+                /* Check endpoints for new requests. */
+                if (pAioMgr->enmState != PDMACEPFILEMGRSTATE_GROWING)
+                {
+                    rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
+                    CHECK_RC(pAioMgr, rc);
+                }
+            } /* while requests are active. */
+
+            if (pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING)
+            {
+                rc = pdmacFileAioMgrNormalGrow(pAioMgr);
+                AssertRC(rc);
+                Assert(pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING);
+
+                rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
+                CHECK_RC(pAioMgr, rc);
+            }
+        } /* if still running */
+    } /* while running */
+
+    LogFlowFunc(("rc=%Rrc\n", rc));
+    return rc;
+}
+
+#undef CHECK_RC
+
diff --git a/src/VBox/VMM/VMMR3/PDMBlkCache.cpp b/src/VBox/VMM/VMMR3/PDMBlkCache.cpp
new file mode 100644
index 00000000..868e34e4
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMBlkCache.cpp
@@ -0,0 +1,2809 @@
+/* $Id: PDMBlkCache.cpp $ */
+/** @file
+ * PDM Block Cache.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_pdm_block_cache     PDM Block Cache - The I/O cache
+ * This component implements an I/O cache based on the 2Q cache algorithm.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_BLK_CACHE
+#include "PDMInternal.h"
+#include <iprt/asm.h>
+#include <iprt/mem.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+#include <iprt/trace.h>
+#include <VBox/log.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/vm.h>
+
+#include "PDMBlkCacheInternal.h"
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#ifdef VBOX_STRICT
+# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) \
+    do \
+    { \
+     AssertMsg(RTCritSectIsOwner(&Cache->CritSect), \
+               ("Thread does not own critical section\n"));\
+    } while (0)
+
+# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) \
+    do \
+    { \
+        AssertMsg(RTSemRWIsWriteOwner(pEpCache->SemRWEntries), \
+                  ("Thread is not exclusive owner of the per endpoint RW semaphore\n")); \
+    } while (0)
+
+# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) \
+    do \
+    { \
+        AssertMsg(RTSemRWIsReadOwner(pEpCache->SemRWEntries), \
+                  ("Thread is not read owner of the per endpoint RW semaphore\n")); \
+    } while (0)
+
+#else
+# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) do { } while (0)
+# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) do { } while (0)
+# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) do { } while (0)
+#endif
+
+#define PDM_BLK_CACHE_SAVED_STATE_VERSION 1
+
+/* Enable to enable some tracing in the block cache code for investigating issues. */
+/*#define VBOX_BLKCACHE_TRACING 1*/
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+
+static PPDMBLKCACHEENTRY pdmBlkCacheEntryAlloc(PPDMBLKCACHE pBlkCache,
+                                               uint64_t off, size_t cbData, uint8_t *pbBuffer);
+static bool pdmBlkCacheAddDirtyEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry);
+
+
+/**
+ * Add message to the VM trace buffer.
+ *
+ * @returns nothing.
+ * @param   pBlkCache     The block cache.
+ * @param   pszFmt        The format string.
+ * @param   ...           Additional parameters for the string formatter.
+ */
+DECLINLINE(void) pdmBlkCacheR3TraceMsgF(PPDMBLKCACHE pBlkCache, const char *pszFmt, ...)
+{
+#if defined(VBOX_BLKCACHE_TRACING)
+    va_list va;
+    va_start(va, pszFmt);
+    RTTraceBufAddMsgV(pBlkCache->pCache->pVM->CTX_SUFF(hTraceBuf), pszFmt, va);
+    va_end(va);
+#else
+    RT_NOREF2(pBlkCache, pszFmt);
+#endif
+}
+
+/**
+ * Decrement the reference counter of the given cache entry.
+ *
+ * @returns nothing.
+ * @param   pEntry    The entry to release.
+ */
+DECLINLINE(void) pdmBlkCacheEntryRelease(PPDMBLKCACHEENTRY pEntry)
+{
+    AssertMsg(pEntry->cRefs > 0, ("Trying to release a not referenced entry\n"));
+    ASMAtomicDecU32(&pEntry->cRefs);
+}
+
+/**
+ * Increment the reference counter of the given cache entry.
+ *
+ * @returns nothing.
+ * @param   pEntry    The entry to reference.
+ */
+DECLINLINE(void) pdmBlkCacheEntryRef(PPDMBLKCACHEENTRY pEntry)
+{
+    ASMAtomicIncU32(&pEntry->cRefs);
+}
+
+#ifdef VBOX_STRICT
+static void pdmBlkCacheValidate(PPDMBLKCACHEGLOBAL pCache)
+{
+    /* Amount of cached data should never exceed the maximum amount. */
+    AssertMsg(pCache->cbCached <= pCache->cbMax,
+              ("Current amount of cached data exceeds maximum\n"));
+
+    /* The amount of cached data in the LRU and FRU list should match cbCached */
+    AssertMsg(pCache->LruRecentlyUsedIn.cbCached + pCache->LruFrequentlyUsed.cbCached == pCache->cbCached,
+              ("Amount of cached data doesn't match\n"));
+
+    AssertMsg(pCache->LruRecentlyUsedOut.cbCached <= pCache->cbRecentlyUsedOutMax,
+              ("Paged out list exceeds maximum\n"));
+}
+#endif
+
+DECLINLINE(void) pdmBlkCacheLockEnter(PPDMBLKCACHEGLOBAL pCache)
+{
+    RTCritSectEnter(&pCache->CritSect);
+#ifdef VBOX_STRICT
+    pdmBlkCacheValidate(pCache);
+#endif
+}
+
+DECLINLINE(void) pdmBlkCacheLockLeave(PPDMBLKCACHEGLOBAL pCache)
+{
+#ifdef VBOX_STRICT
+    pdmBlkCacheValidate(pCache);
+#endif
+    RTCritSectLeave(&pCache->CritSect);
+}
+
+DECLINLINE(void) pdmBlkCacheSub(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
+{
+    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
+    pCache->cbCached -= cbAmount;
+}
+
+DECLINLINE(void) pdmBlkCacheAdd(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
+{
+    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
+    pCache->cbCached += cbAmount;
+}
+
+DECLINLINE(void) pdmBlkCacheListAdd(PPDMBLKLRULIST pList, uint32_t cbAmount)
+{
+    pList->cbCached += cbAmount;
+}
+
+DECLINLINE(void) pdmBlkCacheListSub(PPDMBLKLRULIST pList, uint32_t cbAmount)
+{
+    pList->cbCached -= cbAmount;
+}
+
+#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
+/**
+ * Checks consistency of a LRU list.
+ *
+ * @returns nothing
+ * @param    pList         The LRU list to check.
+ * @param    pNotInList    Element which is not allowed to occur in the list.
+ */
+static void pdmBlkCacheCheckList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pNotInList)
+{
+    PPDMBLKCACHEENTRY pCurr = pList->pHead;
+
+    /* Check that there are no double entries and no cycles in the list. */
+    while (pCurr)
+    {
+        PPDMBLKCACHEENTRY pNext = pCurr->pNext;
+
+        while (pNext)
+        {
+            AssertMsg(pCurr != pNext,
+                      ("Entry %#p is at least two times in list %#p or there is a cycle in the list\n",
+                      pCurr, pList));
+            pNext = pNext->pNext;
+        }
+
+        AssertMsg(pCurr != pNotInList, ("Not allowed entry %#p is in list\n", pCurr));
+
+        if (!pCurr->pNext)
+            AssertMsg(pCurr == pList->pTail, ("End of list reached but last element is not list tail\n"));
+
+        pCurr = pCurr->pNext;
+    }
+}
+#endif
+
+/**
+ * Unlinks a cache entry from the LRU list it is assigned to.
+ *
+ * @returns nothing.
+ * @param   pEntry    The entry to unlink.
+ */
+static void pdmBlkCacheEntryRemoveFromList(PPDMBLKCACHEENTRY pEntry)
+{
+    PPDMBLKLRULIST pList = pEntry->pList;
+    PPDMBLKCACHEENTRY pPrev, pNext;
+
+    LogFlowFunc((": Deleting entry %#p from list %#p\n", pEntry, pList));
+
+    AssertPtr(pList);
+
+#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
+    pdmBlkCacheCheckList(pList, NULL);
+#endif
+
+    pPrev = pEntry->pPrev;
+    pNext = pEntry->pNext;
+
+    AssertMsg(pEntry != pPrev, ("Entry links to itself as previous element\n"));
+    AssertMsg(pEntry != pNext, ("Entry links to itself as next element\n"));
+
+    if (pPrev)
+        pPrev->pNext = pNext;
+    else
+    {
+        pList->pHead = pNext;
+
+        if (pNext)
+            pNext->pPrev = NULL;
+    }
+
+    if (pNext)
+        pNext->pPrev = pPrev;
+    else
+    {
+        pList->pTail = pPrev;
+
+        if (pPrev)
+            pPrev->pNext = NULL;
+    }
+
+    pEntry->pList    = NULL;
+    pEntry->pPrev    = NULL;
+    pEntry->pNext    = NULL;
+    pdmBlkCacheListSub(pList, pEntry->cbData);
+#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
+    pdmBlkCacheCheckList(pList, pEntry);
+#endif
+}
+
+/**
+ * Adds a cache entry to the given LRU list unlinking it from the currently
+ * assigned list if needed.
+ *
+ * @returns nothing.
+ * @param    pList    List to the add entry to.
+ * @param    pEntry   Entry to add.
+ */
+static void pdmBlkCacheEntryAddToList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pEntry)
+{
+    LogFlowFunc((": Adding entry %#p to list %#p\n", pEntry, pList));
+#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
+    pdmBlkCacheCheckList(pList, NULL);
+#endif
+
+    /* Remove from old list if needed */
+    if (pEntry->pList)
+        pdmBlkCacheEntryRemoveFromList(pEntry);
+
+    pEntry->pNext = pList->pHead;
+    if (pList->pHead)
+        pList->pHead->pPrev = pEntry;
+    else
+    {
+        Assert(!pList->pTail);
+        pList->pTail = pEntry;
+    }
+
+    pEntry->pPrev    = NULL;
+    pList->pHead     = pEntry;
+    pdmBlkCacheListAdd(pList, pEntry->cbData);
+    pEntry->pList    = pList;
+#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
+    pdmBlkCacheCheckList(pList, NULL);
+#endif
+}
+
+/**
+ * Destroys a LRU list freeing all entries.
+ *
+ * @returns nothing
+ * @param   pList    Pointer to the LRU list to destroy.
+ *
+ * @note The caller must own the critical section of the cache.
+ */
+static void pdmBlkCacheDestroyList(PPDMBLKLRULIST pList)
+{
+    while (pList->pHead)
+    {
+        PPDMBLKCACHEENTRY pEntry = pList->pHead;
+
+        pList->pHead = pEntry->pNext;
+
+        AssertMsg(!(pEntry->fFlags & (PDMBLKCACHE_ENTRY_IO_IN_PROGRESS | PDMBLKCACHE_ENTRY_IS_DIRTY)),
+                  ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
+
+        RTMemPageFree(pEntry->pbData, pEntry->cbData);
+        RTMemFree(pEntry);
+    }
+}
+
+/**
+ * Tries to remove the given amount of bytes from a given list in the cache
+ * moving the entries to one of the given ghosts lists
+ *
+ * @returns Amount of data which could be freed.
+ * @param    pCache           Pointer to the global cache data.
+ * @param    cbData           The amount of the data to free.
+ * @param    pListSrc         The source list to evict data from.
+ * @param    pGhostListDst    Where the ghost list removed entries should be
+ *                            moved to, NULL if the entry should be freed.
+ * @param    fReuseBuffer     Flag whether a buffer should be reused if it has
+ *                            the same size
+ * @param    ppbBuffer        Where to store the address of the buffer if an
+ *                            entry with the same size was found and
+ *                            fReuseBuffer is true.
+ *
+ * @note    This function may return fewer bytes than requested because entries
+ *          may be marked as non evictable if they are used for I/O at the
+ *          moment.
+ */
+static size_t pdmBlkCacheEvictPagesFrom(PPDMBLKCACHEGLOBAL pCache, size_t cbData,
+                                        PPDMBLKLRULIST pListSrc, PPDMBLKLRULIST pGhostListDst,
+                                        bool fReuseBuffer, uint8_t **ppbBuffer)
+{
+    size_t cbEvicted = 0;
+
+    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
+
+    AssertMsg(cbData > 0, ("Evicting 0 bytes not possible\n"));
+    AssertMsg(   !pGhostListDst
+              || (pGhostListDst == &pCache->LruRecentlyUsedOut),
+              ("Destination list must be NULL or the recently used but paged out list\n"));
+
+    if (fReuseBuffer)
+    {
+        AssertPtr(ppbBuffer);
+        *ppbBuffer = NULL;
+    }
+
+    /* Start deleting from the tail. */
+    PPDMBLKCACHEENTRY pEntry = pListSrc->pTail;
+
+    while ((cbEvicted < cbData) && pEntry)
+    {
+        PPDMBLKCACHEENTRY pCurr = pEntry;
+
+        pEntry = pEntry->pPrev;
+
+        /* We can't evict pages which are currently in progress or dirty but not in progress */
+        if (   !(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
+            && (ASMAtomicReadU32(&pCurr->cRefs) == 0))
+        {
+            /* Ok eviction candidate. Grab the endpoint semaphore and check again
+             * because somebody else might have raced us. */
+            PPDMBLKCACHE pBlkCache = pCurr->pBlkCache;
+            RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+
+            if (!(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
+                && (ASMAtomicReadU32(&pCurr->cRefs) == 0))
+            {
+                LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData));
+
+                if (fReuseBuffer && pCurr->cbData == cbData)
+                {
+                    STAM_COUNTER_INC(&pCache->StatBuffersReused);
+                    *ppbBuffer = pCurr->pbData;
+                }
+                else if (pCurr->pbData)
+                    RTMemPageFree(pCurr->pbData, pCurr->cbData);
+
+                pCurr->pbData = NULL;
+                cbEvicted += pCurr->cbData;
+
+                pdmBlkCacheEntryRemoveFromList(pCurr);
+                pdmBlkCacheSub(pCache, pCurr->cbData);
+
+                if (pGhostListDst)
+                {
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+                    PPDMBLKCACHEENTRY pGhostEntFree = pGhostListDst->pTail;
+
+                    /* We have to remove the last entries from the paged out list. */
+                    while (   pGhostListDst->cbCached + pCurr->cbData > pCache->cbRecentlyUsedOutMax
+                           && pGhostEntFree)
+                    {
+                        PPDMBLKCACHEENTRY pFree = pGhostEntFree;
+                        PPDMBLKCACHE pBlkCacheFree = pFree->pBlkCache;
+
+                        pGhostEntFree = pGhostEntFree->pPrev;
+
+                        RTSemRWRequestWrite(pBlkCacheFree->SemRWEntries, RT_INDEFINITE_WAIT);
+
+                        if (ASMAtomicReadU32(&pFree->cRefs) == 0)
+                        {
+                            pdmBlkCacheEntryRemoveFromList(pFree);
+
+                            STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                            RTAvlrU64Remove(pBlkCacheFree->pTree, pFree->Core.Key);
+                            STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+
+                            RTMemFree(pFree);
+                        }
+
+                        RTSemRWReleaseWrite(pBlkCacheFree->SemRWEntries);
+                    }
+
+                    if (pGhostListDst->cbCached + pCurr->cbData > pCache->cbRecentlyUsedOutMax)
+                    {
+                        /* Couldn't remove enough entries. Delete */
+                        STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                        RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
+                        STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+
+                        RTMemFree(pCurr);
+                    }
+                    else
+                        pdmBlkCacheEntryAddToList(pGhostListDst, pCurr);
+                }
+                else
+                {
+                    /* Delete the entry from the AVL tree it is assigned to. */
+                    STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                    RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
+                    STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+                    RTMemFree(pCurr);
+                }
+            }
+
+        }
+        else
+            LogFlow(("Entry %#p (%u bytes) is still in progress and can't be evicted\n", pCurr, pCurr->cbData));
+    }
+
+    return cbEvicted;
+}
+
+static bool pdmBlkCacheReclaim(PPDMBLKCACHEGLOBAL pCache, size_t cbData, bool fReuseBuffer, uint8_t **ppbBuffer)
+{
+    size_t cbRemoved = 0;
+
+    if ((pCache->cbCached + cbData) < pCache->cbMax)
+        return true;
+    else if ((pCache->LruRecentlyUsedIn.cbCached + cbData) > pCache->cbRecentlyUsedInMax)
+    {
+        /* Try to evict as many bytes as possible from A1in */
+        cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruRecentlyUsedIn,
+                                                 &pCache->LruRecentlyUsedOut, fReuseBuffer, ppbBuffer);
+
+        /*
+         * If it was not possible to remove enough entries
+         * try the frequently accessed cache.
+         */
+        if (cbRemoved < cbData)
+        {
+            Assert(!fReuseBuffer || !*ppbBuffer); /* It is not possible that we got a buffer with the correct size but we didn't freed enough data. */
+
+            /*
+             * If we removed something we can't pass the reuse buffer flag anymore because
+             * we don't need to evict that much data
+             */
+            if (!cbRemoved)
+                cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
+                                                          NULL, fReuseBuffer, ppbBuffer);
+            else
+                cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData - cbRemoved, &pCache->LruFrequentlyUsed,
+                                                          NULL, false, NULL);
+        }
+    }
+    else
+    {
+        /* We have to remove entries from frequently access list. */
+        cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
+                                                 NULL, fReuseBuffer, ppbBuffer);
+    }
+
+    LogFlowFunc((": removed %u bytes, requested %u\n", cbRemoved, cbData));
+    return (cbRemoved >= cbData);
+}
+
+DECLINLINE(int) pdmBlkCacheEnqueue(PPDMBLKCACHE pBlkCache, uint64_t off, size_t cbXfer, PPDMBLKCACHEIOXFER pIoXfer)
+{
+    int rc = VINF_SUCCESS;
+
+    LogFlowFunc(("%s: Enqueuing hIoXfer=%#p enmXferDir=%d\n",
+                 __FUNCTION__, pIoXfer, pIoXfer->enmXferDir));
+
+    ASMAtomicIncU32(&pBlkCache->cIoXfersActive);
+    pdmBlkCacheR3TraceMsgF(pBlkCache, "BlkCache: I/O req %#p (%RTbool , %d) queued (%u now active)",
+                           pIoXfer, pIoXfer->fIoCache, pIoXfer->enmXferDir, pBlkCache->cIoXfersActive);
+
+    switch (pBlkCache->enmType)
+    {
+        case PDMBLKCACHETYPE_DEV:
+        {
+            rc = pBlkCache->u.Dev.pfnXferEnqueue(pBlkCache->u.Dev.pDevIns,
+                                                 pIoXfer->enmXferDir,
+                                                 off, cbXfer,
+                                                 &pIoXfer->SgBuf, pIoXfer);
+            break;
+        }
+        case PDMBLKCACHETYPE_DRV:
+        {
+            rc = pBlkCache->u.Drv.pfnXferEnqueue(pBlkCache->u.Drv.pDrvIns,
+                                                 pIoXfer->enmXferDir,
+                                                 off, cbXfer,
+                                                 &pIoXfer->SgBuf, pIoXfer);
+            break;
+        }
+        case PDMBLKCACHETYPE_USB:
+        {
+            rc = pBlkCache->u.Usb.pfnXferEnqueue(pBlkCache->u.Usb.pUsbIns,
+                                                 pIoXfer->enmXferDir,
+                                                 off, cbXfer,
+                                                 &pIoXfer->SgBuf, pIoXfer);
+            break;
+        }
+        case PDMBLKCACHETYPE_INTERNAL:
+        {
+            rc = pBlkCache->u.Int.pfnXferEnqueue(pBlkCache->u.Int.pvUser,
+                                                 pIoXfer->enmXferDir,
+                                                 off, cbXfer,
+                                                 &pIoXfer->SgBuf, pIoXfer);
+            break;
+        }
+        default:
+            AssertMsgFailed(("Unknown block cache type!\n"));
+    }
+
+    if (RT_FAILURE(rc))
+    {
+        pdmBlkCacheR3TraceMsgF(pBlkCache, "BlkCache: Queueing I/O req %#p failed %Rrc", pIoXfer, rc);
+        ASMAtomicDecU32(&pBlkCache->cIoXfersActive);
+    }
+
+    LogFlowFunc(("%s: returns rc=%Rrc\n", __FUNCTION__, rc));
+    return rc;
+}
+
+/**
+ * Initiates a read I/O task for the given entry.
+ *
+ * @returns VBox status code.
+ * @param   pEntry    The entry to fetch the data to.
+ */
+static int pdmBlkCacheEntryReadFromMedium(PPDMBLKCACHEENTRY pEntry)
+{
+    PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
+    LogFlowFunc((": Reading data into cache entry %#p\n", pEntry));
+
+    /* Make sure no one evicts the entry while it is accessed. */
+    pEntry->fFlags |= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
+
+    PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
+    if (RT_UNLIKELY(!pIoXfer))
+        return VERR_NO_MEMORY;
+
+    AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
+
+    pIoXfer->fIoCache = true;
+    pIoXfer->pEntry = pEntry;
+    pIoXfer->SgSeg.pvSeg = pEntry->pbData;
+    pIoXfer->SgSeg.cbSeg = pEntry->cbData;
+    pIoXfer->enmXferDir  = PDMBLKCACHEXFERDIR_READ;
+    RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
+
+    return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pEntry->cbData, pIoXfer);
+}
+
+/**
+ * Initiates a write I/O task for the given entry.
+ *
+ * @returns nothing.
+ * @param    pEntry The entry to read the data from.
+ */
+static int pdmBlkCacheEntryWriteToMedium(PPDMBLKCACHEENTRY pEntry)
+{
+    PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
+    LogFlowFunc((": Writing data from cache entry %#p\n", pEntry));
+
+    /* Make sure no one evicts the entry while it is accessed. */
+    pEntry->fFlags |= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
+
+    PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
+    if (RT_UNLIKELY(!pIoXfer))
+        return VERR_NO_MEMORY;
+
+    AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
+
+    pIoXfer->fIoCache = true;
+    pIoXfer->pEntry = pEntry;
+    pIoXfer->SgSeg.pvSeg = pEntry->pbData;
+    pIoXfer->SgSeg.cbSeg = pEntry->cbData;
+    pIoXfer->enmXferDir  = PDMBLKCACHEXFERDIR_WRITE;
+    RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
+
+    return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pEntry->cbData, pIoXfer);
+}
+
+/**
+ * Passthrough a part of a request directly to the I/O manager handling the
+ * endpoint.
+ *
+ * @returns VBox status code.
+ * @param   pBlkCache       The endpoint cache.
+ * @param   pReq            The request.
+ * @param   pSgBuf          The scatter/gather buffer.
+ * @param   offStart        Offset to start transfer from.
+ * @param   cbData          Amount of data to transfer.
+ * @param   enmXferDir      The transfer type (read/write)
+ */
+static int pdmBlkCacheRequestPassthrough(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
+                                         PRTSGBUF pSgBuf, uint64_t offStart, size_t cbData,
+                                         PDMBLKCACHEXFERDIR enmXferDir)
+{
+
+    PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
+    if (RT_UNLIKELY(!pIoXfer))
+        return VERR_NO_MEMORY;
+
+    ASMAtomicIncU32(&pReq->cXfersPending);
+    pIoXfer->fIoCache    = false;
+    pIoXfer->pReq        = pReq;
+    pIoXfer->enmXferDir  = enmXferDir;
+    if (pSgBuf)
+    {
+        RTSgBufClone(&pIoXfer->SgBuf, pSgBuf);
+        RTSgBufAdvance(pSgBuf, cbData);
+    }
+
+    return pdmBlkCacheEnqueue(pBlkCache, offStart, cbData, pIoXfer);
+}
+
+/**
+ * Commit a single dirty entry to the endpoint
+ *
+ * @returns nothing
+ * @param   pEntry    The entry to commit.
+ */
+static void pdmBlkCacheEntryCommit(PPDMBLKCACHEENTRY pEntry)
+{
+    AssertMsg(   (pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY)
+              && !(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
+              ("Invalid flags set for entry %#p\n", pEntry));
+
+    pdmBlkCacheEntryWriteToMedium(pEntry);
+}
+
+/**
+ * Commit all dirty entries for a single endpoint.
+ *
+ * @returns nothing.
+ * @param   pBlkCache    The endpoint cache to commit.
+ */
+static void pdmBlkCacheCommit(PPDMBLKCACHE pBlkCache)
+{
+    uint32_t cbCommitted = 0;
+
+    /* Return if the cache was suspended. */
+    if (pBlkCache->fSuspended)
+        return;
+
+    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+
+    /* The list is moved to a new header to reduce locking overhead. */
+    RTLISTANCHOR ListDirtyNotCommitted;
+
+    RTSpinlockAcquire(pBlkCache->LockList);
+    RTListMove(&ListDirtyNotCommitted, &pBlkCache->ListDirtyNotCommitted);
+    RTSpinlockRelease(pBlkCache->LockList);
+
+    if (!RTListIsEmpty(&ListDirtyNotCommitted))
+    {
+        PPDMBLKCACHEENTRY pEntry = RTListGetFirst(&ListDirtyNotCommitted, PDMBLKCACHEENTRY, NodeNotCommitted);
+
+        while (!RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted))
+        {
+            PPDMBLKCACHEENTRY pNext = RTListNodeGetNext(&pEntry->NodeNotCommitted, PDMBLKCACHEENTRY,
+                                                        NodeNotCommitted);
+            pdmBlkCacheEntryCommit(pEntry);
+            cbCommitted += pEntry->cbData;
+            RTListNodeRemove(&pEntry->NodeNotCommitted);
+            pEntry = pNext;
+        }
+
+        /* Commit the last endpoint */
+        Assert(RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted));
+        pdmBlkCacheEntryCommit(pEntry);
+        cbCommitted += pEntry->cbData;
+        RTListNodeRemove(&pEntry->NodeNotCommitted);
+        AssertMsg(RTListIsEmpty(&ListDirtyNotCommitted),
+                  ("Committed all entries but list is not empty\n"));
+    }
+
+    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+    AssertMsg(pBlkCache->pCache->cbDirty >= cbCommitted,
+              ("Number of committed bytes exceeds number of dirty bytes\n"));
+    uint32_t cbDirtyOld = ASMAtomicSubU32(&pBlkCache->pCache->cbDirty, cbCommitted);
+
+    /* Reset the commit timer if we don't have any dirty bits. */
+    if (   !(cbDirtyOld - cbCommitted)
+        && pBlkCache->pCache->u32CommitTimeoutMs != 0)
+        TMTimerStop(pBlkCache->pCache->pTimerCommit);
+}
+
+/**
+ * Commit all dirty entries in the cache.
+ *
+ * @returns nothing.
+ * @param   pCache    The global cache instance.
+ */
+static void pdmBlkCacheCommitDirtyEntries(PPDMBLKCACHEGLOBAL pCache)
+{
+    bool fCommitInProgress = ASMAtomicXchgBool(&pCache->fCommitInProgress, true);
+
+    if (!fCommitInProgress)
+    {
+        pdmBlkCacheLockEnter(pCache);
+        Assert(!RTListIsEmpty(&pCache->ListUsers));
+
+        PPDMBLKCACHE pBlkCache = RTListGetFirst(&pCache->ListUsers, PDMBLKCACHE, NodeCacheUser);
+        AssertPtr(pBlkCache);
+
+        while (!RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser))
+        {
+            pdmBlkCacheCommit(pBlkCache);
+
+            pBlkCache = RTListNodeGetNext(&pBlkCache->NodeCacheUser, PDMBLKCACHE,
+                                          NodeCacheUser);
+        }
+
+        /* Commit the last endpoint */
+        Assert(RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser));
+        pdmBlkCacheCommit(pBlkCache);
+
+        pdmBlkCacheLockLeave(pCache);
+        ASMAtomicWriteBool(&pCache->fCommitInProgress, false);
+    }
+}
+
+/**
+ * Adds the given entry as a dirty to the cache.
+ *
+ * @returns Flag whether the amount of dirty bytes in the cache exceeds the threshold
+ * @param   pBlkCache    The endpoint cache the entry belongs to.
+ * @param   pEntry       The entry to add.
+ */
+static bool pdmBlkCacheAddDirtyEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
+{
+    bool fDirtyBytesExceeded = false;
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+
+    /* If the commit timer is disabled we commit right away. */
+    if (pCache->u32CommitTimeoutMs == 0)
+    {
+        pEntry->fFlags |= PDMBLKCACHE_ENTRY_IS_DIRTY;
+        pdmBlkCacheEntryCommit(pEntry);
+    }
+    else if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY))
+    {
+        pEntry->fFlags |= PDMBLKCACHE_ENTRY_IS_DIRTY;
+
+        RTSpinlockAcquire(pBlkCache->LockList);
+        RTListAppend(&pBlkCache->ListDirtyNotCommitted, &pEntry->NodeNotCommitted);
+        RTSpinlockRelease(pBlkCache->LockList);
+
+        uint32_t cbDirty = ASMAtomicAddU32(&pCache->cbDirty, pEntry->cbData);
+
+        /* Prevent committing if the VM was suspended. */
+        if (RT_LIKELY(!ASMAtomicReadBool(&pCache->fIoErrorVmSuspended)))
+            fDirtyBytesExceeded = (cbDirty + pEntry->cbData >= pCache->cbCommitDirtyThreshold);
+        else if (!cbDirty && pCache->u32CommitTimeoutMs > 0)
+        {
+            /* Arm the commit timer. */
+            TMTimerSetMillies(pCache->pTimerCommit, pCache->u32CommitTimeoutMs);
+        }
+    }
+
+    return fDirtyBytesExceeded;
+}
+
+static PPDMBLKCACHE pdmR3BlkCacheFindById(PPDMBLKCACHEGLOBAL pBlkCacheGlobal, const char *pcszId)
+{
+    bool fFound = false;
+
+    PPDMBLKCACHE pBlkCache;
+    RTListForEach(&pBlkCacheGlobal->ListUsers, pBlkCache, PDMBLKCACHE, NodeCacheUser)
+    {
+        if (!RTStrCmp(pBlkCache->pszId, pcszId))
+        {
+            fFound = true;
+            break;
+        }
+    }
+
+    return fFound ?  pBlkCache : NULL;
+}
+
+/**
+ * Commit timer callback.
+ */
+static DECLCALLBACK(void) pdmBlkCacheCommitTimerCallback(PVM pVM, PTMTIMER pTimer, void *pvUser)
+{
+    PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
+    NOREF(pVM); NOREF(pTimer);
+
+    LogFlowFunc(("Commit interval expired, commiting dirty entries\n"));
+
+    if (   ASMAtomicReadU32(&pCache->cbDirty) > 0
+        && !ASMAtomicReadBool(&pCache->fIoErrorVmSuspended))
+        pdmBlkCacheCommitDirtyEntries(pCache);
+
+    LogFlowFunc(("Entries committed, going to sleep\n"));
+}
+
+static DECLCALLBACK(int) pdmR3BlkCacheSaveExec(PVM pVM, PSSMHANDLE pSSM)
+{
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+
+    AssertPtr(pBlkCacheGlobal);
+
+    pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+    SSMR3PutU32(pSSM, pBlkCacheGlobal->cRefs);
+
+    /* Go through the list and save all dirty entries. */
+    PPDMBLKCACHE pBlkCache;
+    RTListForEach(&pBlkCacheGlobal->ListUsers, pBlkCache, PDMBLKCACHE, NodeCacheUser)
+    {
+        uint32_t cEntries = 0;
+        PPDMBLKCACHEENTRY pEntry;
+
+        RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+        SSMR3PutU32(pSSM, (uint32_t)strlen(pBlkCache->pszId));
+        SSMR3PutStrZ(pSSM, pBlkCache->pszId);
+
+        /* Count the number of entries to safe. */
+        RTListForEach(&pBlkCache->ListDirtyNotCommitted, pEntry, PDMBLKCACHEENTRY, NodeNotCommitted)
+        {
+            cEntries++;
+        }
+
+        SSMR3PutU32(pSSM, cEntries);
+
+        /* Walk the list of all dirty entries and save them. */
+        RTListForEach(&pBlkCache->ListDirtyNotCommitted, pEntry, PDMBLKCACHEENTRY, NodeNotCommitted)
+        {
+            /* A few sanity checks. */
+            AssertMsg(!pEntry->cRefs, ("The entry is still referenced\n"));
+            AssertMsg(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY, ("Entry is not dirty\n"));
+            AssertMsg(!(pEntry->fFlags & ~PDMBLKCACHE_ENTRY_IS_DIRTY), ("Invalid flags set\n"));
+            AssertMsg(!pEntry->pWaitingHead && !pEntry->pWaitingTail, ("There are waiting requests\n"));
+            AssertMsg(   pEntry->pList == &pBlkCacheGlobal->LruRecentlyUsedIn
+                      || pEntry->pList == &pBlkCacheGlobal->LruFrequentlyUsed,
+                      ("Invalid list\n"));
+            AssertMsg(pEntry->cbData == pEntry->Core.KeyLast - pEntry->Core.Key + 1,
+                      ("Size and range do not match\n"));
+
+            /* Save */
+            SSMR3PutU64(pSSM, pEntry->Core.Key);
+            SSMR3PutU32(pSSM, pEntry->cbData);
+            SSMR3PutMem(pSSM, pEntry->pbData, pEntry->cbData);
+        }
+
+        RTSemRWReleaseRead(pBlkCache->SemRWEntries);
+    }
+
+    pdmBlkCacheLockLeave(pBlkCacheGlobal);
+
+    /* Terminator */
+    return SSMR3PutU32(pSSM, UINT32_MAX);
+}
+
+static DECLCALLBACK(int) pdmR3BlkCacheLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+    uint32_t           cRefs;
+
+    NOREF(uPass);
+    AssertPtr(pBlkCacheGlobal);
+
+    pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+    if (uVersion != PDM_BLK_CACHE_SAVED_STATE_VERSION)
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+
+    SSMR3GetU32(pSSM, &cRefs);
+
+    /*
+     * Fewer users in the saved state than in the current VM are allowed
+     * because that means that there are only new ones which don't have any saved state
+     * which can get lost.
+     * More saved state entries than registered cache users are only allowed if the
+     * missing users don't have any data saved in the cache.
+     */
+    int rc = VINF_SUCCESS;
+    char *pszId = NULL;
+
+    while (   cRefs > 0
+           && RT_SUCCESS(rc))
+    {
+        PPDMBLKCACHE pBlkCache = NULL;
+        uint32_t cbId = 0;
+
+        SSMR3GetU32(pSSM, &cbId);
+        Assert(cbId > 0);
+
+        cbId++; /* Include terminator */
+        pszId = (char *)RTMemAllocZ(cbId * sizeof(char));
+        if (!pszId)
+        {
+            rc = VERR_NO_MEMORY;
+            break;
+        }
+
+        rc = SSMR3GetStrZ(pSSM, pszId, cbId);
+        AssertRC(rc);
+
+        /* Search for the block cache with the provided id. */
+        pBlkCache = pdmR3BlkCacheFindById(pBlkCacheGlobal, pszId);
+
+        /* Get the entries */
+        uint32_t cEntries;
+        SSMR3GetU32(pSSM, &cEntries);
+
+        if (!pBlkCache && (cEntries > 0))
+        {
+            rc = SSMR3SetCfgError(pSSM, RT_SRC_POS,
+                                  N_("The VM is missing a block device and there is data in the cache. Please make sure the source and target VMs have compatible storage configurations"));
+            break;
+        }
+
+        RTMemFree(pszId);
+        pszId = NULL;
+
+        while (cEntries > 0)
+        {
+            PPDMBLKCACHEENTRY pEntry;
+            uint64_t off;
+            uint32_t cbEntry;
+
+            SSMR3GetU64(pSSM, &off);
+            SSMR3GetU32(pSSM, &cbEntry);
+
+            pEntry = pdmBlkCacheEntryAlloc(pBlkCache, off, cbEntry, NULL);
+            if (!pEntry)
+            {
+                rc = VERR_NO_MEMORY;
+                break;
+            }
+
+            rc = SSMR3GetMem(pSSM, pEntry->pbData, cbEntry);
+            if (RT_FAILURE(rc))
+            {
+                RTMemFree(pEntry->pbData);
+                RTMemFree(pEntry);
+                break;
+            }
+
+            /* Insert into the tree. */
+            bool fInserted = RTAvlrU64Insert(pBlkCache->pTree, &pEntry->Core);
+            Assert(fInserted); NOREF(fInserted);
+
+            /* Add to the dirty list. */
+            pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
+            pdmBlkCacheEntryAddToList(&pBlkCacheGlobal->LruRecentlyUsedIn, pEntry);
+            pdmBlkCacheAdd(pBlkCacheGlobal, cbEntry);
+            pdmBlkCacheEntryRelease(pEntry);
+            cEntries--;
+        }
+
+        cRefs--;
+    }
+
+    if (pszId)
+        RTMemFree(pszId);
+
+    if (cRefs && RT_SUCCESS(rc))
+        rc = SSMR3SetCfgError(pSSM, RT_SRC_POS,
+                              N_("Unexpected error while restoring state. Please make sure the source and target VMs have compatible storage configurations"));
+
+    pdmBlkCacheLockLeave(pBlkCacheGlobal);
+
+    if (RT_SUCCESS(rc))
+    {
+        uint32_t u32 = 0;
+        rc = SSMR3GetU32(pSSM, &u32);
+        if (RT_SUCCESS(rc))
+            AssertMsgReturn(u32 == UINT32_MAX, ("%#x\n", u32), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+    }
+
+    return rc;
+}
+
+int pdmR3BlkCacheInit(PVM pVM)
+{
+    int  rc   = VINF_SUCCESS;
+    PUVM pUVM = pVM->pUVM;
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal;
+
+    LogFlowFunc((": pVM=%p\n", pVM));
+
+    VM_ASSERT_EMT(pVM);
+
+    PCFGMNODE pCfgRoot     = CFGMR3GetRoot(pVM);
+    PCFGMNODE pCfgBlkCache = CFGMR3GetChild(CFGMR3GetChild(pCfgRoot, "PDM"), "BlkCache");
+
+    pBlkCacheGlobal = (PPDMBLKCACHEGLOBAL)RTMemAllocZ(sizeof(PDMBLKCACHEGLOBAL));
+    if (!pBlkCacheGlobal)
+        return VERR_NO_MEMORY;
+
+    RTListInit(&pBlkCacheGlobal->ListUsers);
+    pBlkCacheGlobal->pVM = pVM;
+    pBlkCacheGlobal->cRefs = 0;
+    pBlkCacheGlobal->cbCached  = 0;
+    pBlkCacheGlobal->fCommitInProgress = false;
+
+    /* Initialize members */
+    pBlkCacheGlobal->LruRecentlyUsedIn.pHead    = NULL;
+    pBlkCacheGlobal->LruRecentlyUsedIn.pTail    = NULL;
+    pBlkCacheGlobal->LruRecentlyUsedIn.cbCached = 0;
+
+    pBlkCacheGlobal->LruRecentlyUsedOut.pHead    = NULL;
+    pBlkCacheGlobal->LruRecentlyUsedOut.pTail    = NULL;
+    pBlkCacheGlobal->LruRecentlyUsedOut.cbCached = 0;
+
+    pBlkCacheGlobal->LruFrequentlyUsed.pHead    = NULL;
+    pBlkCacheGlobal->LruFrequentlyUsed.pTail    = NULL;
+    pBlkCacheGlobal->LruFrequentlyUsed.cbCached = 0;
+
+    do
+    {
+        rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheSize", &pBlkCacheGlobal->cbMax, 5 * _1M);
+        AssertLogRelRCBreak(rc);
+        LogFlowFunc(("Maximum number of bytes cached %u\n", pBlkCacheGlobal->cbMax));
+
+        pBlkCacheGlobal->cbRecentlyUsedInMax  = (pBlkCacheGlobal->cbMax / 100) * 25; /* 25% of the buffer size */
+        pBlkCacheGlobal->cbRecentlyUsedOutMax = (pBlkCacheGlobal->cbMax / 100) * 50; /* 50% of the buffer size */
+        LogFlowFunc(("cbRecentlyUsedInMax=%u cbRecentlyUsedOutMax=%u\n",
+                     pBlkCacheGlobal->cbRecentlyUsedInMax, pBlkCacheGlobal->cbRecentlyUsedOutMax));
+
+        /** @todo r=aeichner: Experiment to find optimal default values */
+        rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitIntervalMs", &pBlkCacheGlobal->u32CommitTimeoutMs, 10000 /* 10sec */);
+        AssertLogRelRCBreak(rc);
+        rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitThreshold", &pBlkCacheGlobal->cbCommitDirtyThreshold, pBlkCacheGlobal->cbMax / 2);
+        AssertLogRelRCBreak(rc);
+    } while (0);
+
+    if (RT_SUCCESS(rc))
+    {
+        STAMR3Register(pVM, &pBlkCacheGlobal->cbMax,
+                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/cbMax",
+                       STAMUNIT_BYTES,
+                       "Maximum cache size");
+        STAMR3Register(pVM, &pBlkCacheGlobal->cbCached,
+                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/cbCached",
+                       STAMUNIT_BYTES,
+                       "Currently used cache");
+        STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedIn.cbCached,
+                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/cbCachedMruIn",
+                       STAMUNIT_BYTES,
+                       "Number of bytes cached in MRU list");
+        STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedOut.cbCached,
+                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/cbCachedMruOut",
+                       STAMUNIT_BYTES,
+                       "Number of bytes cached in FRU list");
+        STAMR3Register(pVM, &pBlkCacheGlobal->LruFrequentlyUsed.cbCached,
+                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/cbCachedFru",
+                       STAMUNIT_BYTES,
+                       "Number of bytes cached in FRU ghost list");
+
+#ifdef VBOX_WITH_STATISTICS
+        STAMR3Register(pVM, &pBlkCacheGlobal->cHits,
+                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheHits",
+                       STAMUNIT_COUNT, "Number of hits in the cache");
+        STAMR3Register(pVM, &pBlkCacheGlobal->cPartialHits,
+                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CachePartialHits",
+                       STAMUNIT_COUNT, "Number of partial hits in the cache");
+        STAMR3Register(pVM, &pBlkCacheGlobal->cMisses,
+                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheMisses",
+                       STAMUNIT_COUNT, "Number of misses when accessing the cache");
+        STAMR3Register(pVM, &pBlkCacheGlobal->StatRead,
+                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheRead",
+                       STAMUNIT_BYTES, "Number of bytes read from the cache");
+        STAMR3Register(pVM, &pBlkCacheGlobal->StatWritten,
+                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheWritten",
+                       STAMUNIT_BYTES, "Number of bytes written to the cache");
+        STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeGet,
+                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheTreeGet",
+                       STAMUNIT_TICKS_PER_CALL, "Time taken to access an entry in the tree");
+        STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeInsert,
+                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheTreeInsert",
+                       STAMUNIT_TICKS_PER_CALL, "Time taken to insert an entry in the tree");
+        STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeRemove,
+                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheTreeRemove",
+                       STAMUNIT_TICKS_PER_CALL, "Time taken to remove an entry an the tree");
+        STAMR3Register(pVM, &pBlkCacheGlobal->StatBuffersReused,
+                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheBuffersReused",
+                       STAMUNIT_COUNT, "Number of times a buffer could be reused");
+#endif
+
+        /* Initialize the critical section */
+        rc = RTCritSectInit(&pBlkCacheGlobal->CritSect);
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        /* Create the commit timer */
+        if (pBlkCacheGlobal->u32CommitTimeoutMs > 0)
+            rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL,
+                                         pdmBlkCacheCommitTimerCallback,
+                                         pBlkCacheGlobal,
+                                         "BlkCache-Commit",
+                                         &pBlkCacheGlobal->pTimerCommit);
+
+        if (RT_SUCCESS(rc))
+        {
+            /* Register saved state handler. */
+            rc = SSMR3RegisterInternal(pVM, "pdmblkcache", 0, PDM_BLK_CACHE_SAVED_STATE_VERSION, pBlkCacheGlobal->cbMax,
+                                       NULL, NULL, NULL,
+                                       NULL, pdmR3BlkCacheSaveExec, NULL,
+                                       NULL, pdmR3BlkCacheLoadExec, NULL);
+            if (RT_SUCCESS(rc))
+            {
+                LogRel(("BlkCache: Cache successfully initialized. Cache size is %u bytes\n", pBlkCacheGlobal->cbMax));
+                LogRel(("BlkCache: Cache commit interval is %u ms\n", pBlkCacheGlobal->u32CommitTimeoutMs));
+                LogRel(("BlkCache: Cache commit threshold is %u bytes\n", pBlkCacheGlobal->cbCommitDirtyThreshold));
+                pUVM->pdm.s.pBlkCacheGlobal = pBlkCacheGlobal;
+                return VINF_SUCCESS;
+            }
+        }
+
+        RTCritSectDelete(&pBlkCacheGlobal->CritSect);
+    }
+
+    if (pBlkCacheGlobal)
+        RTMemFree(pBlkCacheGlobal);
+
+    LogFlowFunc((": returns rc=%Rrc\n", rc));
+    return rc;
+}
+
+void pdmR3BlkCacheTerm(PVM pVM)
+{
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+
+    if (pBlkCacheGlobal)
+    {
+        /* Make sure no one else uses the cache now */
+        pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+        /* Cleanup deleting all cache entries waiting for in progress entries to finish. */
+        pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedIn);
+        pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedOut);
+        pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruFrequentlyUsed);
+
+        pdmBlkCacheLockLeave(pBlkCacheGlobal);
+
+        RTCritSectDelete(&pBlkCacheGlobal->CritSect);
+        RTMemFree(pBlkCacheGlobal);
+        pVM->pUVM->pdm.s.pBlkCacheGlobal = NULL;
+    }
+}
+
+int pdmR3BlkCacheResume(PVM pVM)
+{
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+
+    LogFlowFunc(("pVM=%#p\n", pVM));
+
+    if (   pBlkCacheGlobal
+        && ASMAtomicXchgBool(&pBlkCacheGlobal->fIoErrorVmSuspended, false))
+    {
+        /* The VM was suspended because of an I/O error, commit all dirty entries. */
+        pdmBlkCacheCommitDirtyEntries(pBlkCacheGlobal);
+    }
+
+    return VINF_SUCCESS;
+}
+
+static int pdmR3BlkCacheRetain(PVM pVM, PPPDMBLKCACHE ppBlkCache, const char *pcszId)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHE pBlkCache = NULL;
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+
+    if (!pBlkCacheGlobal)
+        return VERR_NOT_SUPPORTED;
+
+    /*
+     * Check that no other user cache has the same id first,
+     * Unique id's are necessary in case the state is saved.
+     */
+    pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+    pBlkCache = pdmR3BlkCacheFindById(pBlkCacheGlobal, pcszId);
+
+    if (!pBlkCache)
+    {
+        pBlkCache = (PPDMBLKCACHE)RTMemAllocZ(sizeof(PDMBLKCACHE));
+
+        if (pBlkCache)
+            pBlkCache->pszId = RTStrDup(pcszId);
+
+        if (   pBlkCache
+            && pBlkCache->pszId)
+        {
+            pBlkCache->fSuspended = false;
+            pBlkCache->cIoXfersActive = 0;
+            pBlkCache->pCache = pBlkCacheGlobal;
+            RTListInit(&pBlkCache->ListDirtyNotCommitted);
+
+            rc = RTSpinlockCreate(&pBlkCache->LockList, RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE, "pdmR3BlkCacheRetain");
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTSemRWCreate(&pBlkCache->SemRWEntries);
+                if (RT_SUCCESS(rc))
+                {
+                    pBlkCache->pTree  = (PAVLRU64TREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
+                    if (pBlkCache->pTree)
+                    {
+#ifdef VBOX_WITH_STATISTICS
+                        STAMR3RegisterF(pBlkCacheGlobal->pVM, &pBlkCache->StatWriteDeferred,
+                                        STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                                        STAMUNIT_COUNT, "Number of deferred writes",
+                                        "/PDM/BlkCache/%s/Cache/DeferredWrites", pBlkCache->pszId);
+#endif
+
+                        /* Add to the list of users. */
+                        pBlkCacheGlobal->cRefs++;
+                        RTListAppend(&pBlkCacheGlobal->ListUsers, &pBlkCache->NodeCacheUser);
+                        pdmBlkCacheLockLeave(pBlkCacheGlobal);
+
+                        *ppBlkCache = pBlkCache;
+                        LogFlowFunc(("returns success\n"));
+                        return VINF_SUCCESS;
+                    }
+
+                    rc = VERR_NO_MEMORY;
+                    RTSemRWDestroy(pBlkCache->SemRWEntries);
+                }
+
+                RTSpinlockDestroy(pBlkCache->LockList);
+            }
+
+            RTStrFree(pBlkCache->pszId);
+        }
+        else
+            rc = VERR_NO_MEMORY;
+
+        if (pBlkCache)
+            RTMemFree(pBlkCache);
+    }
+    else
+        rc = VERR_ALREADY_EXISTS;
+
+    pdmBlkCacheLockLeave(pBlkCacheGlobal);
+
+    LogFlowFunc(("Leave rc=%Rrc\n", rc));
+    return rc;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheRetainDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMBLKCACHE ppBlkCache,
+                                         PFNPDMBLKCACHEXFERCOMPLETEDRV pfnXferComplete,
+                                         PFNPDMBLKCACHEXFERENQUEUEDRV pfnXferEnqueue,
+                                         PFNPDMBLKCACHEXFERENQUEUEDISCARDDRV pfnXferEnqueueDiscard,
+                                         const char *pcszId)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHE pBlkCache;
+
+    rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
+    if (RT_SUCCESS(rc))
+    {
+        pBlkCache->enmType                      = PDMBLKCACHETYPE_DRV;
+        pBlkCache->u.Drv.pfnXferComplete        = pfnXferComplete;
+        pBlkCache->u.Drv.pfnXferEnqueue         = pfnXferEnqueue;
+        pBlkCache->u.Drv.pfnXferEnqueueDiscard  = pfnXferEnqueueDiscard;
+        pBlkCache->u.Drv.pDrvIns                = pDrvIns;
+        *ppBlkCache = pBlkCache;
+    }
+
+    LogFlowFunc(("Leave rc=%Rrc\n", rc));
+    return rc;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheRetainDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMBLKCACHE ppBlkCache,
+                                         PFNPDMBLKCACHEXFERCOMPLETEDEV pfnXferComplete,
+                                         PFNPDMBLKCACHEXFERENQUEUEDEV pfnXferEnqueue,
+                                         PFNPDMBLKCACHEXFERENQUEUEDISCARDDEV pfnXferEnqueueDiscard,
+                                         const char *pcszId)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHE pBlkCache;
+
+    rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
+    if (RT_SUCCESS(rc))
+    {
+        pBlkCache->enmType                      = PDMBLKCACHETYPE_DEV;
+        pBlkCache->u.Dev.pfnXferComplete        = pfnXferComplete;
+        pBlkCache->u.Dev.pfnXferEnqueue         = pfnXferEnqueue;
+        pBlkCache->u.Dev.pfnXferEnqueueDiscard  = pfnXferEnqueueDiscard;
+        pBlkCache->u.Dev.pDevIns                = pDevIns;
+        *ppBlkCache = pBlkCache;
+    }
+
+    LogFlowFunc(("Leave rc=%Rrc\n", rc));
+    return rc;
+
+}
+
+VMMR3DECL(int) PDMR3BlkCacheRetainUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMBLKCACHE ppBlkCache,
+                                      PFNPDMBLKCACHEXFERCOMPLETEUSB pfnXferComplete,
+                                      PFNPDMBLKCACHEXFERENQUEUEUSB pfnXferEnqueue,
+                                      PFNPDMBLKCACHEXFERENQUEUEDISCARDUSB pfnXferEnqueueDiscard,
+                                      const char *pcszId)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHE pBlkCache;
+
+    rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
+    if (RT_SUCCESS(rc))
+    {
+        pBlkCache->enmType                      = PDMBLKCACHETYPE_USB;
+        pBlkCache->u.Usb.pfnXferComplete        = pfnXferComplete;
+        pBlkCache->u.Usb.pfnXferEnqueue         = pfnXferEnqueue;
+        pBlkCache->u.Usb.pfnXferEnqueueDiscard  = pfnXferEnqueueDiscard;
+        pBlkCache->u.Usb.pUsbIns                = pUsbIns;
+        *ppBlkCache = pBlkCache;
+    }
+
+    LogFlowFunc(("Leave rc=%Rrc\n", rc));
+    return rc;
+
+}
+
+VMMR3DECL(int) PDMR3BlkCacheRetainInt(PVM pVM, void *pvUser, PPPDMBLKCACHE ppBlkCache,
+                                      PFNPDMBLKCACHEXFERCOMPLETEINT pfnXferComplete,
+                                      PFNPDMBLKCACHEXFERENQUEUEINT pfnXferEnqueue,
+                                      PFNPDMBLKCACHEXFERENQUEUEDISCARDINT pfnXferEnqueueDiscard,
+                                      const char *pcszId)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHE pBlkCache;
+
+    rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
+    if (RT_SUCCESS(rc))
+    {
+        pBlkCache->enmType                      = PDMBLKCACHETYPE_INTERNAL;
+        pBlkCache->u.Int.pfnXferComplete        = pfnXferComplete;
+        pBlkCache->u.Int.pfnXferEnqueue         = pfnXferEnqueue;
+        pBlkCache->u.Int.pfnXferEnqueueDiscard  = pfnXferEnqueueDiscard;
+        pBlkCache->u.Int.pvUser                 = pvUser;
+        *ppBlkCache = pBlkCache;
+    }
+
+    LogFlowFunc(("Leave rc=%Rrc\n", rc));
+    return rc;
+
+}
+
+/**
+ * Callback for the AVL destroy routine. Frees a cache entry for this endpoint.
+ *
+ * @returns IPRT status code.
+ * @param    pNode     The node to destroy.
+ * @param    pvUser    Opaque user data.
+ */
+static DECLCALLBACK(int) pdmBlkCacheEntryDestroy(PAVLRU64NODECORE pNode, void *pvUser)
+{
+    PPDMBLKCACHEENTRY  pEntry = (PPDMBLKCACHEENTRY)pNode;
+    PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
+    PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
+
+    while (ASMAtomicReadU32(&pEntry->fFlags) & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS)
+    {
+        /* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
+        pdmBlkCacheEntryRef(pEntry);
+        RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+        pdmBlkCacheLockLeave(pCache);
+
+        RTThreadSleep(250);
+
+        /* Re-enter all locks */
+        pdmBlkCacheLockEnter(pCache);
+        RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+        pdmBlkCacheEntryRelease(pEntry);
+    }
+
+    AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
+                ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
+
+    bool fUpdateCache =    pEntry->pList == &pCache->LruFrequentlyUsed
+                        || pEntry->pList == &pCache->LruRecentlyUsedIn;
+
+    pdmBlkCacheEntryRemoveFromList(pEntry);
+
+    if (fUpdateCache)
+        pdmBlkCacheSub(pCache, pEntry->cbData);
+
+    RTMemPageFree(pEntry->pbData, pEntry->cbData);
+    RTMemFree(pEntry);
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Destroys all cache resources used by the given endpoint.
+ *
+ * @returns nothing.
+ * @param   pBlkCache       Block cache handle.
+ */
+VMMR3DECL(void) PDMR3BlkCacheRelease(PPDMBLKCACHE pBlkCache)
+{
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+
+    /*
+     * Commit all dirty entries now (they are waited on for completion during the
+     * destruction of the AVL tree below).
+     * The exception is if the VM was paused because of an I/O error before.
+     */
+    if (!ASMAtomicReadBool(&pCache->fIoErrorVmSuspended))
+        pdmBlkCacheCommit(pBlkCache);
+
+    /* Make sure nobody is accessing the cache while we delete the tree. */
+    pdmBlkCacheLockEnter(pCache);
+    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    RTAvlrU64Destroy(pBlkCache->pTree, pdmBlkCacheEntryDestroy, pCache);
+    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+    RTSpinlockDestroy(pBlkCache->LockList);
+
+    pCache->cRefs--;
+    RTListNodeRemove(&pBlkCache->NodeCacheUser);
+
+    pdmBlkCacheLockLeave(pCache);
+
+    RTMemFree(pBlkCache->pTree);
+    pBlkCache->pTree = NULL;
+    RTSemRWDestroy(pBlkCache->SemRWEntries);
+
+#ifdef VBOX_WITH_STATISTICS
+    STAMR3DeregisterF(pCache->pVM->pUVM, "/PDM/BlkCache/%s/Cache/DeferredWrites", pBlkCache->pszId);
+#endif
+
+    RTStrFree(pBlkCache->pszId);
+    RTMemFree(pBlkCache);
+}
+
+VMMR3DECL(void) PDMR3BlkCacheReleaseDevice(PVM pVM, PPDMDEVINS pDevIns)
+{
+    LogFlow(("%s: pDevIns=%p\n", __FUNCTION__, pDevIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pDevIns)
+        return;
+    VM_ASSERT_EMT(pVM);
+
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+    PPDMBLKCACHE pBlkCache, pBlkCacheNext;
+
+    /* Return silently if not supported. */
+    if (!pBlkCacheGlobal)
+        return;
+
+    pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+    RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
+    {
+        if (    pBlkCache->enmType == PDMBLKCACHETYPE_DEV
+            &&  pBlkCache->u.Dev.pDevIns == pDevIns)
+            PDMR3BlkCacheRelease(pBlkCache);
+    }
+
+    pdmBlkCacheLockLeave(pBlkCacheGlobal);
+}
+
+VMMR3DECL(void) PDMR3BlkCacheReleaseDriver(PVM pVM, PPDMDRVINS pDrvIns)
+{
+    LogFlow(("%s: pDrvIns=%p\n", __FUNCTION__, pDrvIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pDrvIns)
+        return;
+    VM_ASSERT_EMT(pVM);
+
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+    PPDMBLKCACHE pBlkCache, pBlkCacheNext;
+
+    /* Return silently if not supported. */
+    if (!pBlkCacheGlobal)
+        return;
+
+    pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+    RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
+    {
+        if (    pBlkCache->enmType == PDMBLKCACHETYPE_DRV
+            &&  pBlkCache->u.Drv.pDrvIns == pDrvIns)
+            PDMR3BlkCacheRelease(pBlkCache);
+    }
+
+    pdmBlkCacheLockLeave(pBlkCacheGlobal);
+}
+
+VMMR3DECL(void) PDMR3BlkCacheReleaseUsb(PVM pVM, PPDMUSBINS pUsbIns)
+{
+    LogFlow(("%s: pUsbIns=%p\n", __FUNCTION__, pUsbIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pUsbIns)
+        return;
+    VM_ASSERT_EMT(pVM);
+
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+    PPDMBLKCACHE pBlkCache, pBlkCacheNext;
+
+    /* Return silently if not supported. */
+    if (!pBlkCacheGlobal)
+        return;
+
+    pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+    RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
+    {
+        if (    pBlkCache->enmType == PDMBLKCACHETYPE_USB
+            &&  pBlkCache->u.Usb.pUsbIns == pUsbIns)
+            PDMR3BlkCacheRelease(pBlkCache);
+    }
+
+    pdmBlkCacheLockLeave(pBlkCacheGlobal);
+}
+
+static PPDMBLKCACHEENTRY pdmBlkCacheGetCacheEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off)
+{
+    STAM_PROFILE_ADV_START(&pBlkCache->pCache->StatTreeGet, Cache);
+
+    RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    PPDMBLKCACHEENTRY pEntry = (PPDMBLKCACHEENTRY)RTAvlrU64RangeGet(pBlkCache->pTree, off);
+    if (pEntry)
+        pdmBlkCacheEntryRef(pEntry);
+    RTSemRWReleaseRead(pBlkCache->SemRWEntries);
+
+    STAM_PROFILE_ADV_STOP(&pBlkCache->pCache->StatTreeGet, Cache);
+
+    return pEntry;
+}
+
+/**
+ * Return the best fit cache entries for the given offset.
+ *
+ * @returns nothing.
+ * @param   pBlkCache    The endpoint cache.
+ * @param   off          The offset.
+ * @param   ppEntryAbove Where to store the pointer to the best fit entry above
+ *                       the given offset. NULL if not required.
+ */
+static void pdmBlkCacheGetCacheBestFitEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off, PPDMBLKCACHEENTRY *ppEntryAbove)
+{
+    STAM_PROFILE_ADV_START(&pBlkCache->pCache->StatTreeGet, Cache);
+
+    RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    if (ppEntryAbove)
+    {
+        *ppEntryAbove = (PPDMBLKCACHEENTRY)RTAvlrU64GetBestFit(pBlkCache->pTree, off, true /*fAbove*/);
+        if (*ppEntryAbove)
+            pdmBlkCacheEntryRef(*ppEntryAbove);
+    }
+
+    RTSemRWReleaseRead(pBlkCache->SemRWEntries);
+
+    STAM_PROFILE_ADV_STOP(&pBlkCache->pCache->StatTreeGet, Cache);
+}
+
+static void pdmBlkCacheInsertEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
+{
+    STAM_PROFILE_ADV_START(&pBlkCache->pCache->StatTreeInsert, Cache);
+    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    bool fInserted = RTAvlrU64Insert(pBlkCache->pTree, &pEntry->Core);
+    AssertMsg(fInserted, ("Node was not inserted into tree\n")); NOREF(fInserted);
+    STAM_PROFILE_ADV_STOP(&pBlkCache->pCache->StatTreeInsert, Cache);
+    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+}
+
+/**
+ * Allocates and initializes a new entry for the cache.
+ * The entry has a reference count of 1.
+ *
+ * @returns Pointer to the new cache entry or NULL if out of memory.
+ * @param   pBlkCache The cache the entry belongs to.
+ * @param   off       Start offset.
+ * @param   cbData    Size of the cache entry.
+ * @param   pbBuffer  Pointer to the buffer to use.
+ *                    NULL if a new buffer should be allocated.
+ *                    The buffer needs to have the same size of the entry.
+ */
+static PPDMBLKCACHEENTRY pdmBlkCacheEntryAlloc(PPDMBLKCACHE pBlkCache, uint64_t off, size_t cbData, uint8_t *pbBuffer)
+{
+    AssertReturn(cbData <= UINT32_MAX, NULL);
+    PPDMBLKCACHEENTRY pEntryNew = (PPDMBLKCACHEENTRY)RTMemAllocZ(sizeof(PDMBLKCACHEENTRY));
+
+    if (RT_UNLIKELY(!pEntryNew))
+        return NULL;
+
+    pEntryNew->Core.Key      = off;
+    pEntryNew->Core.KeyLast  = off + cbData - 1;
+    pEntryNew->pBlkCache     = pBlkCache;
+    pEntryNew->fFlags        = 0;
+    pEntryNew->cRefs         = 1; /* We are using it now. */
+    pEntryNew->pList         = NULL;
+    pEntryNew->cbData        = (uint32_t)cbData;
+    pEntryNew->pWaitingHead  = NULL;
+    pEntryNew->pWaitingTail  = NULL;
+    if (pbBuffer)
+        pEntryNew->pbData    = pbBuffer;
+    else
+        pEntryNew->pbData    = (uint8_t *)RTMemPageAlloc(cbData);
+
+    if (RT_UNLIKELY(!pEntryNew->pbData))
+    {
+        RTMemFree(pEntryNew);
+        return NULL;
+    }
+
+    return pEntryNew;
+}
+
+/**
+ * Checks that a set of flags is set/clear acquiring the R/W semaphore
+ * in exclusive mode.
+ *
+ * @returns true if the flag in fSet is set and the one in fClear is clear.
+ *          false otherwise.
+ *          The R/W semaphore is only held if true is returned.
+ *
+ * @param   pBlkCache   The endpoint cache instance data.
+ * @param   pEntry           The entry to check the flags for.
+ * @param   fSet             The flag which is tested to be set.
+ * @param   fClear           The flag which is tested to be clear.
+ */
+DECLINLINE(bool) pdmBlkCacheEntryFlagIsSetClearAcquireLock(PPDMBLKCACHE pBlkCache,
+                                                           PPDMBLKCACHEENTRY pEntry,
+                                                           uint32_t fSet, uint32_t fClear)
+{
+    uint32_t fFlags = ASMAtomicReadU32(&pEntry->fFlags);
+    bool fPassed = ((fFlags & fSet) && !(fFlags & fClear));
+
+    if (fPassed)
+    {
+        /* Acquire the lock and check again because the completion callback might have raced us. */
+        RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+
+        fFlags = ASMAtomicReadU32(&pEntry->fFlags);
+        fPassed = ((fFlags & fSet) && !(fFlags & fClear));
+
+        /* Drop the lock if we didn't passed the test. */
+        if (!fPassed)
+            RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+    }
+
+    return fPassed;
+}
+
+/**
+ * Adds a segment to the waiting list for a cache entry
+ * which is currently in progress.
+ *
+ * @returns nothing.
+ * @param   pEntry      The cache entry to add the segment to.
+ * @param   pWaiter     The waiter entry to add.
+ */
+DECLINLINE(void) pdmBlkCacheEntryAddWaiter(PPDMBLKCACHEENTRY pEntry,
+                                           PPDMBLKCACHEWAITER pWaiter)
+{
+    pWaiter->pNext = NULL;
+
+    if (pEntry->pWaitingHead)
+    {
+        AssertPtr(pEntry->pWaitingTail);
+
+        pEntry->pWaitingTail->pNext = pWaiter;
+        pEntry->pWaitingTail = pWaiter;
+    }
+    else
+    {
+        Assert(!pEntry->pWaitingTail);
+
+        pEntry->pWaitingHead = pWaiter;
+        pEntry->pWaitingTail = pWaiter;
+    }
+}
+
+/**
+ * Add a buffer described by the I/O memory context
+ * to the entry waiting for completion.
+ *
+ * @returns VBox status code.
+ * @param   pEntry      The entry to add the buffer to.
+ * @param   pReq        The request.
+ * @param   pSgBuf      The scatter/gather buffer. Will be advanced by cbData.
+ * @param   offDiff     Offset from the start of the buffer in the entry.
+ * @param   cbData      Amount of data to wait for onthis entry.
+ * @param   fWrite      Flag whether the task waits because it wants to write to
+ *                      the cache entry.
+ */
+static int pdmBlkCacheEntryWaitersAdd(PPDMBLKCACHEENTRY pEntry, PPDMBLKCACHEREQ pReq,
+                                      PRTSGBUF pSgBuf, uint64_t offDiff, size_t cbData, bool fWrite)
+{
+    PPDMBLKCACHEWAITER pWaiter  = (PPDMBLKCACHEWAITER)RTMemAllocZ(sizeof(PDMBLKCACHEWAITER));
+    if (!pWaiter)
+        return VERR_NO_MEMORY;
+
+    ASMAtomicIncU32(&pReq->cXfersPending);
+    pWaiter->pReq          = pReq;
+    pWaiter->offCacheEntry = offDiff;
+    pWaiter->cbTransfer    = cbData;
+    pWaiter->fWrite        = fWrite;
+    RTSgBufClone(&pWaiter->SgBuf, pSgBuf);
+    RTSgBufAdvance(pSgBuf, cbData);
+
+    pdmBlkCacheEntryAddWaiter(pEntry, pWaiter);
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Calculate aligned offset and size for a new cache entry which do not
+ * intersect with an already existing entry and the file end.
+ *
+ * @returns The number of bytes the entry can hold of the requested amount
+ *          of bytes.
+ * @param   pBlkCache       The endpoint cache.
+ * @param   off             The start offset.
+ * @param   cb              The number of bytes the entry needs to hold at
+ *                          least.
+ * @param   pcbEntry        Where to store the number of bytes the entry can hold.
+ *                          Can be less than given because of other entries.
+ */
+static uint32_t pdmBlkCacheEntryBoundariesCalc(PPDMBLKCACHE pBlkCache,
+                                               uint64_t off, uint32_t cb,
+                                               uint32_t *pcbEntry)
+{
+    /* Get the best fit entries around the offset */
+    PPDMBLKCACHEENTRY pEntryAbove = NULL;
+    pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off, &pEntryAbove);
+
+    /* Log the info */
+    LogFlow(("%sest fit entry above off=%llu (BestFit=%llu BestFitEnd=%llu BestFitSize=%u)\n",
+             pEntryAbove ? "B" : "No b",
+             off,
+             pEntryAbove ? pEntryAbove->Core.Key : 0,
+             pEntryAbove ? pEntryAbove->Core.KeyLast : 0,
+             pEntryAbove ? pEntryAbove->cbData : 0));
+
+    uint32_t cbNext;
+    uint32_t cbInEntry;
+    if (    pEntryAbove
+        &&  off + cb > pEntryAbove->Core.Key)
+    {
+        cbInEntry = (uint32_t)(pEntryAbove->Core.Key - off);
+        cbNext = (uint32_t)(pEntryAbove->Core.Key - off);
+    }
+    else
+    {
+        cbInEntry = cb;
+        cbNext    = cb;
+    }
+
+    /* A few sanity checks */
+    AssertMsg(!pEntryAbove || off + cbNext <= pEntryAbove->Core.Key,
+              ("Aligned size intersects with another cache entry\n"));
+    Assert(cbInEntry <= cbNext);
+
+    if (pEntryAbove)
+        pdmBlkCacheEntryRelease(pEntryAbove);
+
+    LogFlow(("off=%llu cbNext=%u\n", off, cbNext));
+
+    *pcbEntry  = cbNext;
+
+    return cbInEntry;
+}
+
+/**
+ * Create a new cache entry evicting data from the cache if required.
+ *
+ * @returns Pointer to the new cache entry or NULL
+ *          if not enough bytes could be evicted from the cache.
+ * @param   pBlkCache       The endpoint cache.
+ * @param   off             The offset.
+ * @param   cb              Number of bytes the cache entry should have.
+ * @param   pcbData         Where to store the number of bytes the new
+ *                          entry can hold. May be lower than actually
+ *                          requested due to another entry intersecting the
+ *                          access range.
+ */
+static PPDMBLKCACHEENTRY pdmBlkCacheEntryCreate(PPDMBLKCACHE pBlkCache, uint64_t off, size_t cb, size_t *pcbData)
+{
+    uint32_t cbEntry  = 0;
+
+    *pcbData = pdmBlkCacheEntryBoundariesCalc(pBlkCache, off, (uint32_t)cb, &cbEntry);
+    AssertReturn(cb <= UINT32_MAX, NULL);
+
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+    pdmBlkCacheLockEnter(pCache);
+
+    PPDMBLKCACHEENTRY pEntryNew = NULL;
+    uint8_t          *pbBuffer  = NULL;
+    bool fEnough = pdmBlkCacheReclaim(pCache, cbEntry, true, &pbBuffer);
+    if (fEnough)
+    {
+        LogFlow(("Evicted enough bytes (%u requested). Creating new cache entry\n", cbEntry));
+
+        pEntryNew = pdmBlkCacheEntryAlloc(pBlkCache, off, cbEntry, pbBuffer);
+        if (RT_LIKELY(pEntryNew))
+        {
+            pdmBlkCacheEntryAddToList(&pCache->LruRecentlyUsedIn, pEntryNew);
+            pdmBlkCacheAdd(pCache, cbEntry);
+            pdmBlkCacheLockLeave(pCache);
+
+            pdmBlkCacheInsertEntry(pBlkCache, pEntryNew);
+
+            AssertMsg(   (off >= pEntryNew->Core.Key)
+                      && (off + *pcbData <= pEntryNew->Core.KeyLast + 1),
+                      ("Overflow in calculation off=%llu\n", off));
+        }
+        else
+            pdmBlkCacheLockLeave(pCache);
+    }
+    else
+        pdmBlkCacheLockLeave(pCache);
+
+    return pEntryNew;
+}
+
+static PPDMBLKCACHEREQ pdmBlkCacheReqAlloc(void *pvUser)
+{
+    PPDMBLKCACHEREQ pReq = (PPDMBLKCACHEREQ)RTMemAlloc(sizeof(PDMBLKCACHEREQ));
+
+    if (RT_LIKELY(pReq))
+    {
+        pReq->pvUser = pvUser;
+        pReq->rcReq  = VINF_SUCCESS;
+        pReq->cXfersPending = 0;
+    }
+
+    return pReq;
+}
+
+static void pdmBlkCacheReqComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq)
+{
+    switch (pBlkCache->enmType)
+    {
+        case PDMBLKCACHETYPE_DEV:
+        {
+            pBlkCache->u.Dev.pfnXferComplete(pBlkCache->u.Dev.pDevIns,
+                                             pReq->pvUser, pReq->rcReq);
+            break;
+        }
+        case PDMBLKCACHETYPE_DRV:
+        {
+            pBlkCache->u.Drv.pfnXferComplete(pBlkCache->u.Drv.pDrvIns,
+                                             pReq->pvUser, pReq->rcReq);
+            break;
+        }
+        case PDMBLKCACHETYPE_USB:
+        {
+            pBlkCache->u.Usb.pfnXferComplete(pBlkCache->u.Usb.pUsbIns,
+                                             pReq->pvUser, pReq->rcReq);
+            break;
+        }
+        case PDMBLKCACHETYPE_INTERNAL:
+        {
+            pBlkCache->u.Int.pfnXferComplete(pBlkCache->u.Int.pvUser,
+                                             pReq->pvUser, pReq->rcReq);
+            break;
+        }
+        default:
+            AssertMsgFailed(("Unknown block cache type!\n"));
+    }
+
+    RTMemFree(pReq);
+}
+
+static bool pdmBlkCacheReqUpdate(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
+                                 int rcReq, bool fCallHandler)
+{
+    if (RT_FAILURE(rcReq))
+        ASMAtomicCmpXchgS32(&pReq->rcReq, rcReq, VINF_SUCCESS);
+
+    AssertMsg(pReq->cXfersPending > 0, ("No transfers are pending for this request\n"));
+    uint32_t cXfersPending = ASMAtomicDecU32(&pReq->cXfersPending);
+
+    if (!cXfersPending)
+    {
+        if (fCallHandler)
+            pdmBlkCacheReqComplete(pBlkCache, pReq);
+        return true;
+    }
+
+    LogFlowFunc(("pReq=%#p cXfersPending=%u\n", pReq, cXfersPending));
+    return false;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheRead(PPDMBLKCACHE pBlkCache, uint64_t off,
+                                 PCRTSGBUF pSgBuf, size_t cbRead, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+    PPDMBLKCACHEENTRY  pEntry;
+    PPDMBLKCACHEREQ    pReq;
+
+    LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pSgBuf=%#p cbRead=%u pvUser=%#p\n",
+                 pBlkCache, pBlkCache->pszId, off, pSgBuf, cbRead, pvUser));
+
+    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
+    AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
+
+    RTSGBUF SgBuf;
+    RTSgBufClone(&SgBuf, pSgBuf);
+
+    /* Allocate new request structure. */
+    pReq = pdmBlkCacheReqAlloc(pvUser);
+    if (RT_UNLIKELY(!pReq))
+        return VERR_NO_MEMORY;
+
+    /* Increment data transfer counter to keep the request valid while we access it. */
+    ASMAtomicIncU32(&pReq->cXfersPending);
+
+    while (cbRead)
+    {
+        size_t cbToRead;
+
+        pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
+
+        /*
+         * If there is no entry we try to create a new one eviciting unused pages
+         * if the cache is full. If this is not possible we will pass the request through
+         * and skip the caching (all entries may be still in progress so they can't
+         * be evicted)
+         * If we have an entry it can be in one of the LRU lists where the entry
+         * contains data (recently used or frequently used LRU) so we can just read
+         * the data we need and put the entry at the head of the frequently used LRU list.
+         * In case the entry is in one of the ghost lists it doesn't contain any data.
+         * We have to fetch it again evicting pages from either T1 or T2 to make room.
+         */
+        if (pEntry)
+        {
+            uint64_t offDiff = off - pEntry->Core.Key;
+
+            AssertMsg(off >= pEntry->Core.Key,
+                      ("Overflow in calculation off=%llu OffsetAligned=%llu\n",
+                      off, pEntry->Core.Key));
+
+            AssertPtr(pEntry->pList);
+
+            cbToRead = RT_MIN(pEntry->cbData - offDiff, cbRead);
+
+            AssertMsg(off + cbToRead <= pEntry->Core.Key + pEntry->Core.KeyLast + 1,
+                      ("Buffer of cache entry exceeded off=%llu cbToRead=%d\n",
+                       off, cbToRead));
+
+            cbRead  -= cbToRead;
+
+            if (!cbRead)
+                STAM_COUNTER_INC(&pCache->cHits);
+            else
+                STAM_COUNTER_INC(&pCache->cPartialHits);
+
+            STAM_COUNTER_ADD(&pCache->StatRead, cbToRead);
+
+            /* Ghost lists contain no data. */
+            if (   (pEntry->pList == &pCache->LruRecentlyUsedIn)
+                || (pEntry->pList == &pCache->LruFrequentlyUsed))
+            {
+                if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
+                                                              PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
+                                                              PDMBLKCACHE_ENTRY_IS_DIRTY))
+                {
+                    /* Entry didn't completed yet. Append to the list */
+                    pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
+                                               &SgBuf, offDiff, cbToRead,
+                                               false /* fWrite */);
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+                }
+                else
+                {
+                    /* Read as much as we can from the entry. */
+                    RTSgBufCopyFromBuf(&SgBuf, pEntry->pbData + offDiff, cbToRead);
+                }
+
+                /* Move this entry to the top position */
+                if (pEntry->pList == &pCache->LruFrequentlyUsed)
+                {
+                    pdmBlkCacheLockEnter(pCache);
+                    pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
+                    pdmBlkCacheLockLeave(pCache);
+                }
+                /* Release the entry */
+                pdmBlkCacheEntryRelease(pEntry);
+            }
+            else
+            {
+                uint8_t *pbBuffer = NULL;
+
+                LogFlow(("Fetching data for ghost entry %#p from file\n", pEntry));
+
+                pdmBlkCacheLockEnter(pCache);
+                pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
+                bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
+
+                /* Move the entry to Am and fetch it to the cache. */
+                if (fEnough)
+                {
+                    pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
+                    pdmBlkCacheAdd(pCache, pEntry->cbData);
+                    pdmBlkCacheLockLeave(pCache);
+
+                    if (pbBuffer)
+                        pEntry->pbData = pbBuffer;
+                    else
+                        pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
+                    AssertPtr(pEntry->pbData);
+
+                    pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
+                                               &SgBuf, offDiff, cbToRead,
+                                               false /* fWrite */);
+                    pdmBlkCacheEntryReadFromMedium(pEntry);
+                    /* Release the entry */
+                    pdmBlkCacheEntryRelease(pEntry);
+                }
+                else
+                {
+                    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+                    STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                    RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
+                    STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+                    pdmBlkCacheLockLeave(pCache);
+
+                    RTMemFree(pEntry);
+
+                    pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
+                                                  &SgBuf, off, cbToRead,
+                                                  PDMBLKCACHEXFERDIR_READ);
+                }
+            }
+        }
+        else
+        {
+#ifdef VBOX_WITH_IO_READ_CACHE
+            /* No entry found for this offset. Create a new entry and fetch the data to the cache. */
+            PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
+                                                                 off, cbRead,
+                                                                 &cbToRead);
+
+            cbRead -= cbToRead;
+
+            if (pEntryNew)
+            {
+                if (!cbRead)
+                    STAM_COUNTER_INC(&pCache->cMisses);
+                else
+                    STAM_COUNTER_INC(&pCache->cPartialHits);
+
+                pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
+                                           &SgBuf,
+                                           off - pEntryNew->Core.Key,
+                                           cbToRead,
+                                           false /* fWrite */);
+                pdmBlkCacheEntryReadFromMedium(pEntryNew);
+                pdmBlkCacheEntryRelease(pEntryNew); /* it is protected by the I/O in progress flag now. */
+            }
+            else
+            {
+                /*
+                 * There is not enough free space in the cache.
+                 * Pass the request directly to the I/O manager.
+                 */
+                LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToRead));
+
+                pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
+                                              &SgBuf, off, cbToRead,
+                                              PDMBLKCACHEXFERDIR_READ);
+            }
+#else
+            /* Clip read size if necessary. */
+            PPDMBLKCACHEENTRY pEntryAbove;
+            pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off, &pEntryAbove);
+
+            if (pEntryAbove)
+            {
+                if (off + cbRead > pEntryAbove->Core.Key)
+                    cbToRead = pEntryAbove->Core.Key - off;
+                else
+                    cbToRead = cbRead;
+
+                pdmBlkCacheEntryRelease(pEntryAbove);
+            }
+            else
+                cbToRead = cbRead;
+
+            cbRead -= cbToRead;
+            pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
+                                          &SgBuf, off, cbToRead,
+                                          PDMBLKCACHEXFERDIR_READ);
+#endif
+        }
+        off += cbToRead;
+    }
+
+    if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, rc, false))
+        rc = VINF_AIO_TASK_PENDING;
+    else
+    {
+        rc = pReq->rcReq;
+        RTMemFree(pReq);
+    }
+
+    LogFlowFunc((": Leave rc=%Rrc\n", rc));
+
+   return rc;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheWrite(PPDMBLKCACHE pBlkCache, uint64_t off, PCRTSGBUF pSgBuf, size_t cbWrite, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+    PPDMBLKCACHEENTRY pEntry;
+    PPDMBLKCACHEREQ pReq;
+
+    LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pSgBuf=%#p cbWrite=%u pvUser=%#p\n",
+                 pBlkCache, pBlkCache->pszId, off, pSgBuf, cbWrite, pvUser));
+
+    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
+    AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
+
+    RTSGBUF SgBuf;
+    RTSgBufClone(&SgBuf, pSgBuf);
+
+    /* Allocate new request structure. */
+    pReq = pdmBlkCacheReqAlloc(pvUser);
+    if (RT_UNLIKELY(!pReq))
+        return VERR_NO_MEMORY;
+
+    /* Increment data transfer counter to keep the request valid while we access it. */
+    ASMAtomicIncU32(&pReq->cXfersPending);
+
+    while (cbWrite)
+    {
+        size_t cbToWrite;
+
+        pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
+        if (pEntry)
+        {
+            /* Write the data into the entry and mark it as dirty */
+            AssertPtr(pEntry->pList);
+
+            uint64_t offDiff = off - pEntry->Core.Key;
+            AssertMsg(off >= pEntry->Core.Key, ("Overflow in calculation off=%llu OffsetAligned=%llu\n", off, pEntry->Core.Key));
+
+            cbToWrite = RT_MIN(pEntry->cbData - offDiff, cbWrite);
+            cbWrite  -= cbToWrite;
+
+            if (!cbWrite)
+                STAM_COUNTER_INC(&pCache->cHits);
+            else
+                STAM_COUNTER_INC(&pCache->cPartialHits);
+
+            STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
+
+            /* Ghost lists contain no data. */
+            if (   (pEntry->pList == &pCache->LruRecentlyUsedIn)
+                || (pEntry->pList == &pCache->LruFrequentlyUsed))
+            {
+                /* Check if the entry is dirty. */
+                if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
+                                                              PDMBLKCACHE_ENTRY_IS_DIRTY,
+                                                              0))
+                {
+                    /* If it is already dirty but not in progress just update the data. */
+                    if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS))
+                        RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff, cbToWrite);
+                    else
+                    {
+                        /* The data isn't written to the file yet */
+                        pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
+                                                   &SgBuf, offDiff, cbToWrite,
+                                                   true /* fWrite */);
+                        STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
+                    }
+
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+                }
+                else /* Dirty bit not set */
+                {
+                    /*
+                     * Check if a read is in progress for this entry.
+                     * We have to defer processing in that case.
+                     */
+                    if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
+                                                                  PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
+                                                                  0))
+                    {
+                        pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
+                                                   &SgBuf, offDiff, cbToWrite,
+                                                   true /* fWrite */);
+                        STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
+                        RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+                    }
+                    else /* I/O in progress flag not set */
+                    {
+                        /* Write as much as we can into the entry and update the file. */
+                        RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff, cbToWrite);
+
+                        bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
+                        if (fCommit)
+                            pdmBlkCacheCommitDirtyEntries(pCache);
+                    }
+                } /* Dirty bit not set */
+
+                /* Move this entry to the top position */
+                if (pEntry->pList == &pCache->LruFrequentlyUsed)
+                {
+                    pdmBlkCacheLockEnter(pCache);
+                    pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
+                    pdmBlkCacheLockLeave(pCache);
+                }
+
+                pdmBlkCacheEntryRelease(pEntry);
+            }
+            else /* Entry is on the ghost list */
+            {
+                uint8_t *pbBuffer = NULL;
+
+                pdmBlkCacheLockEnter(pCache);
+                pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
+                bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
+
+                if (fEnough)
+                {
+                    /* Move the entry to Am and fetch it to the cache. */
+                    pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
+                    pdmBlkCacheAdd(pCache, pEntry->cbData);
+                    pdmBlkCacheLockLeave(pCache);
+
+                    if (pbBuffer)
+                        pEntry->pbData = pbBuffer;
+                    else
+                        pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
+                    AssertPtr(pEntry->pbData);
+
+                    pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
+                                               &SgBuf, offDiff, cbToWrite,
+                                               true /* fWrite */);
+                    STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
+                    pdmBlkCacheEntryReadFromMedium(pEntry);
+
+                    /* Release the reference. If it is still needed the I/O in progress flag should protect it now. */
+                    pdmBlkCacheEntryRelease(pEntry);
+                }
+                else
+                {
+                    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+                    STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                    RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
+                    STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+                    pdmBlkCacheLockLeave(pCache);
+
+                    RTMemFree(pEntry);
+                    pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
+                                                  &SgBuf, off, cbToWrite,
+                                                  PDMBLKCACHEXFERDIR_WRITE);
+                }
+            }
+        }
+        else /* No entry found */
+        {
+            /*
+             * No entry found. Try to create a new cache entry to store the data in and if that fails
+             * write directly to the file.
+             */
+            PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
+                                                                 off, cbWrite,
+                                                                 &cbToWrite);
+
+            cbWrite -= cbToWrite;
+
+            if (pEntryNew)
+            {
+                uint64_t offDiff = off - pEntryNew->Core.Key;
+
+                STAM_COUNTER_INC(&pCache->cHits);
+
+                /*
+                 * Check if it is possible to just write the data without waiting
+                 * for it to get fetched first.
+                 */
+                if (!offDiff && pEntryNew->cbData == cbToWrite)
+                {
+                    RTSgBufCopyToBuf(&SgBuf, pEntryNew->pbData, cbToWrite);
+
+                    bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntryNew);
+                    if (fCommit)
+                        pdmBlkCacheCommitDirtyEntries(pCache);
+                    STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
+                }
+                else
+                {
+                    /* Defer the write and fetch the data from the endpoint. */
+                    pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
+                                               &SgBuf, offDiff, cbToWrite,
+                                               true /* fWrite */);
+                    STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
+                    pdmBlkCacheEntryReadFromMedium(pEntryNew);
+                }
+
+                pdmBlkCacheEntryRelease(pEntryNew);
+            }
+            else
+            {
+                /*
+                 * There is not enough free space in the cache.
+                 * Pass the request directly to the I/O manager.
+                 */
+                LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToWrite));
+
+                STAM_COUNTER_INC(&pCache->cMisses);
+
+                pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
+                                              &SgBuf, off, cbToWrite,
+                                              PDMBLKCACHEXFERDIR_WRITE);
+            }
+        }
+
+        off += cbToWrite;
+    }
+
+    if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, rc, false))
+        rc = VINF_AIO_TASK_PENDING;
+    else
+    {
+        rc = pReq->rcReq;
+        RTMemFree(pReq);
+    }
+
+    LogFlowFunc((": Leave rc=%Rrc\n", rc));
+
+    return rc;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheFlush(PPDMBLKCACHE pBlkCache, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHEREQ pReq;
+
+    LogFlowFunc((": pBlkCache=%#p{%s}\n", pBlkCache, pBlkCache->pszId));
+
+    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
+    AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
+
+    /* Commit dirty entries in the cache. */
+    pdmBlkCacheCommit(pBlkCache);
+
+    /* Allocate new request structure. */
+    pReq = pdmBlkCacheReqAlloc(pvUser);
+    if (RT_UNLIKELY(!pReq))
+        return VERR_NO_MEMORY;
+
+    rc = pdmBlkCacheRequestPassthrough(pBlkCache, pReq, NULL, 0, 0,
+                                       PDMBLKCACHEXFERDIR_FLUSH);
+    AssertRC(rc);
+
+    LogFlowFunc((": Leave rc=%Rrc\n", rc));
+    return VINF_AIO_TASK_PENDING;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheDiscard(PPDMBLKCACHE pBlkCache, PCRTRANGE paRanges,
+                                    unsigned cRanges, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+    PPDMBLKCACHEENTRY pEntry;
+    PPDMBLKCACHEREQ pReq;
+
+    LogFlowFunc((": pBlkCache=%#p{%s} paRanges=%#p cRanges=%u pvUser=%#p\n",
+                 pBlkCache, pBlkCache->pszId, paRanges, cRanges, pvUser));
+
+    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
+    AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
+
+    /* Allocate new request structure. */
+    pReq = pdmBlkCacheReqAlloc(pvUser);
+    if (RT_UNLIKELY(!pReq))
+        return VERR_NO_MEMORY;
+
+    /* Increment data transfer counter to keep the request valid while we access it. */
+    ASMAtomicIncU32(&pReq->cXfersPending);
+
+    for (unsigned i = 0; i < cRanges; i++)
+    {
+        uint64_t offCur = paRanges[i].offStart;
+        size_t cbLeft = paRanges[i].cbRange;
+
+        while (cbLeft)
+        {
+            size_t cbThisDiscard = 0;
+
+            pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, offCur);
+
+            if (pEntry)
+            {
+                /* Write the data into the entry and mark it as dirty */
+                AssertPtr(pEntry->pList);
+
+                uint64_t offDiff = offCur - pEntry->Core.Key;
+
+                AssertMsg(offCur >= pEntry->Core.Key,
+                          ("Overflow in calculation offCur=%llu OffsetAligned=%llu\n",
+                          offCur, pEntry->Core.Key));
+
+                cbThisDiscard = RT_MIN(pEntry->cbData - offDiff, cbLeft);
+
+                /* Ghost lists contain no data. */
+                if (   (pEntry->pList == &pCache->LruRecentlyUsedIn)
+                    || (pEntry->pList == &pCache->LruFrequentlyUsed))
+                {
+                    /* Check if the entry is dirty. */
+                    if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
+                                                                  PDMBLKCACHE_ENTRY_IS_DIRTY,
+                                                                  0))
+                    {
+                        /* If it is dirty but not yet in progress remove it. */
+                        if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS))
+                        {
+                            pdmBlkCacheLockEnter(pCache);
+                            pdmBlkCacheEntryRemoveFromList(pEntry);
+
+                            STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                            RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
+                            STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+
+                            pdmBlkCacheLockLeave(pCache);
+
+                            RTMemFree(pEntry);
+                        }
+                        else
+                        {
+#if 0
+                            /* The data isn't written to the file yet */
+                            pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
+                                                       &SgBuf, offDiff, cbToWrite,
+                                                       true /* fWrite */);
+                            STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
+#endif
+                        }
+
+                        RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+                        pdmBlkCacheEntryRelease(pEntry);
+                    }
+                    else /* Dirty bit not set */
+                    {
+                        /*
+                         * Check if a read is in progress for this entry.
+                         * We have to defer processing in that case.
+                         */
+                        if(pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
+                                                                     PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
+                                                                     0))
+                        {
+#if 0
+                            pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
+                                                       &SgBuf, offDiff, cbToWrite,
+                                                       true /* fWrite */);
+#endif
+                            STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
+                            RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+                            pdmBlkCacheEntryRelease(pEntry);
+                        }
+                        else /* I/O in progress flag not set */
+                        {
+                            pdmBlkCacheLockEnter(pCache);
+                            pdmBlkCacheEntryRemoveFromList(pEntry);
+
+                            RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+                            STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                            RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
+                            STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+                            RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+                            pdmBlkCacheLockLeave(pCache);
+
+                            RTMemFree(pEntry);
+                        }
+                    } /* Dirty bit not set */
+                }
+                else /* Entry is on the ghost list just remove cache entry. */
+                {
+                    pdmBlkCacheLockEnter(pCache);
+                    pdmBlkCacheEntryRemoveFromList(pEntry);
+
+                    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+                    STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                    RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
+                    STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+                    pdmBlkCacheLockLeave(pCache);
+
+                    RTMemFree(pEntry);
+                }
+            }
+            /* else: no entry found. */
+
+            offCur += cbThisDiscard;
+            cbLeft -= cbThisDiscard;
+        }
+    }
+
+    if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, rc, false))
+        rc = VINF_AIO_TASK_PENDING;
+    else
+    {
+        rc = pReq->rcReq;
+        RTMemFree(pReq);
+    }
+
+    LogFlowFunc((": Leave rc=%Rrc\n", rc));
+
+    return rc;
+}
+
+/**
+ * Completes a task segment freeing all resources and completes the task handle
+ * if everything was transferred.
+ *
+ * @returns Next task segment handle.
+ * @param   pBlkCache         The endpoint block cache.
+ * @param   pWaiter           Task segment to complete.
+ * @param   rc                Status code to set.
+ */
+static PPDMBLKCACHEWAITER pdmBlkCacheWaiterComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEWAITER pWaiter, int rc)
+{
+    PPDMBLKCACHEWAITER pNext = pWaiter->pNext;
+    PPDMBLKCACHEREQ pReq = pWaiter->pReq;
+
+    pdmBlkCacheReqUpdate(pBlkCache, pReq, rc, true);
+
+    RTMemFree(pWaiter);
+
+    return pNext;
+}
+
+static void pdmBlkCacheIoXferCompleteEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
+{
+    PPDMBLKCACHEENTRY  pEntry    = hIoXfer->pEntry;
+    PPDMBLKCACHEGLOBAL pCache    = pBlkCache->pCache;
+
+    /* Reference the entry now as we are clearing the I/O in progress flag
+     * which protected the entry till now. */
+    pdmBlkCacheEntryRef(pEntry);
+
+    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
+
+    /* Process waiting segment list. The data in entry might have changed in-between. */
+    bool fDirty = false;
+    PPDMBLKCACHEWAITER pComplete = pEntry->pWaitingHead;
+    PPDMBLKCACHEWAITER pCurr     = pComplete;
+
+    AssertMsg((pCurr && pEntry->pWaitingTail) || (!pCurr && !pEntry->pWaitingTail),
+                ("The list tail was not updated correctly\n"));
+    pEntry->pWaitingTail = NULL;
+    pEntry->pWaitingHead = NULL;
+
+    if (hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_WRITE)
+    {
+        /*
+         * An error here is difficult to handle as the original request completed already.
+         * The error is logged for now and the VM is paused.
+         * If the user continues the entry is written again in the hope
+         * the user fixed the problem and the next write succeeds.
+         */
+        if (RT_FAILURE(rcIoXfer))
+        {
+            LogRel(("I/O cache: Error while writing entry at offset %llu (%u bytes) to medium \"%s\" (rc=%Rrc)\n",
+                    pEntry->Core.Key, pEntry->cbData, pBlkCache->pszId, rcIoXfer));
+
+            if (!ASMAtomicXchgBool(&pCache->fIoErrorVmSuspended, true))
+            {
+                int rc = VMSetRuntimeError(pCache->pVM, VMSETRTERR_FLAGS_SUSPEND | VMSETRTERR_FLAGS_NO_WAIT, "BLKCACHE_IOERR",
+                                           N_("The I/O cache encountered an error while updating data in medium \"%s\" (rc=%Rrc). "
+                                              "Make sure there is enough free space on the disk and that the disk is working properly. "
+                                              "Operation can be resumed afterwards"),
+                                           pBlkCache->pszId, rcIoXfer);
+                AssertRC(rc);
+            }
+
+            /* Mark the entry as dirty again to get it added to the list later on. */
+            fDirty = true;
+        }
+
+        pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IS_DIRTY;
+
+        while (pCurr)
+        {
+            AssertMsg(pCurr->fWrite, ("Completed write entries should never have read tasks attached\n"));
+
+            RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
+            fDirty = true;
+            pCurr = pCurr->pNext;
+        }
+    }
+    else
+    {
+        AssertMsg(hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_READ, ("Invalid transfer type\n"));
+        AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY),
+                  ("Invalid flags set\n"));
+
+        while (pCurr)
+        {
+            if (pCurr->fWrite)
+            {
+                RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
+                fDirty = true;
+            }
+            else
+                RTSgBufCopyFromBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
+
+            pCurr = pCurr->pNext;
+        }
+    }
+
+    bool fCommit = false;
+    if (fDirty)
+        fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
+
+    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+    /* Dereference so that it isn't protected anymore except we issued anyother write for it. */
+    pdmBlkCacheEntryRelease(pEntry);
+
+    if (fCommit)
+        pdmBlkCacheCommitDirtyEntries(pCache);
+
+    /* Complete waiters now. */
+    while (pComplete)
+        pComplete = pdmBlkCacheWaiterComplete(pBlkCache, pComplete, rcIoXfer);
+}
+
+VMMR3DECL(void) PDMR3BlkCacheIoXferComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
+{
+    LogFlowFunc(("pBlkCache=%#p hIoXfer=%#p rcIoXfer=%Rrc\n", pBlkCache, hIoXfer, rcIoXfer));
+
+    if (hIoXfer->fIoCache)
+        pdmBlkCacheIoXferCompleteEntry(pBlkCache, hIoXfer, rcIoXfer);
+    else
+        pdmBlkCacheReqUpdate(pBlkCache, hIoXfer->pReq, rcIoXfer, true);
+
+    ASMAtomicDecU32(&pBlkCache->cIoXfersActive);
+    pdmBlkCacheR3TraceMsgF(pBlkCache, "BlkCache: I/O req %#p (%RTbool) completed (%u now active)",
+                           hIoXfer, hIoXfer->fIoCache, pBlkCache->cIoXfersActive);
+    RTMemFree(hIoXfer);
+}
+
+/**
+ * Callback for the AVL do with all routine. Waits for a cachen entry to finish any pending I/O.
+ *
+ * @returns IPRT status code.
+ * @param    pNode     The node to destroy.
+ * @param    pvUser    Opaque user data.
+ */
+static DECLCALLBACK(int) pdmBlkCacheEntryQuiesce(PAVLRU64NODECORE pNode, void *pvUser)
+{
+    PPDMBLKCACHEENTRY   pEntry    = (PPDMBLKCACHEENTRY)pNode;
+    PPDMBLKCACHE        pBlkCache = pEntry->pBlkCache;
+    NOREF(pvUser);
+
+    while (ASMAtomicReadU32(&pEntry->fFlags) & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS)
+    {
+        /* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
+        pdmBlkCacheEntryRef(pEntry);
+        RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+        RTThreadSleep(1);
+
+        /* Re-enter all locks and drop the reference. */
+        RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+        pdmBlkCacheEntryRelease(pEntry);
+    }
+
+    AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
+                ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
+
+    return VINF_SUCCESS;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheSuspend(PPDMBLKCACHE pBlkCache)
+{
+    int rc = VINF_SUCCESS;
+    LogFlowFunc(("pBlkCache=%#p\n", pBlkCache));
+
+    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
+
+    if (!ASMAtomicReadBool(&pBlkCache->pCache->fIoErrorVmSuspended))
+        pdmBlkCacheCommit(pBlkCache); /* Can issue new I/O requests. */
+    ASMAtomicXchgBool(&pBlkCache->fSuspended, true);
+
+    /* Wait for all I/O to complete. */
+    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    rc = RTAvlrU64DoWithAll(pBlkCache->pTree, true, pdmBlkCacheEntryQuiesce, NULL);
+    AssertRC(rc);
+    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+    return rc;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheResume(PPDMBLKCACHE pBlkCache)
+{
+    LogFlowFunc(("pBlkCache=%#p\n", pBlkCache));
+
+    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
+
+    ASMAtomicXchgBool(&pBlkCache->fSuspended, false);
+
+    return VINF_SUCCESS;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheClear(PPDMBLKCACHE pBlkCache)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+
+    /*
+     * Commit all dirty entries now (they are waited on for completion during the
+     * destruction of the AVL tree below).
+     * The exception is if the VM was paused because of an I/O error before.
+     */
+    if (!ASMAtomicReadBool(&pCache->fIoErrorVmSuspended))
+        pdmBlkCacheCommit(pBlkCache);
+
+    /* Make sure nobody is accessing the cache while we delete the tree. */
+    pdmBlkCacheLockEnter(pCache);
+    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    RTAvlrU64Destroy(pBlkCache->pTree, pdmBlkCacheEntryDestroy, pCache);
+    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+    pdmBlkCacheLockLeave(pCache);
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/PDMCritSect.cpp b/src/VBox/VMM/VMMR3/PDMCritSect.cpp
new file mode 100644
index 00000000..102ac439
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMCritSect.cpp
@@ -0,0 +1,1084 @@
+/* $Id: PDMCritSect.cpp $ */
+/** @file
+ * PDM - Critical Sections, Ring-3.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM//_CRITSECT
+#include "PDMInternal.h"
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/pdmcritsectrw.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/sup.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/lockvalidator.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int pdmR3CritSectDeleteOne(PVM pVM, PUVM pUVM, PPDMCRITSECTINT pCritSect, PPDMCRITSECTINT pPrev, bool fFinal);
+static int pdmR3CritSectRwDeleteOne(PVM pVM, PUVM pUVM, PPDMCRITSECTRWINT pCritSect, PPDMCRITSECTRWINT pPrev, bool fFinal);
+
+
+
+/**
+ * Register statistics related to the critical sections.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+int pdmR3CritSectBothInitStats(PVM pVM)
+{
+    RT_NOREF_PV(pVM);
+    STAM_REG(pVM, &pVM->pdm.s.StatQueuedCritSectLeaves, STAMTYPE_COUNTER, "/PDM/QueuedCritSectLeaves", STAMUNIT_OCCURENCES,
+             "Number of times a critical section leave request needed to be queued for ring-3 execution.");
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Relocates all the critical sections.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+void pdmR3CritSectBothRelocate(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+
+    for (PPDMCRITSECTINT pCur = pUVM->pdm.s.pCritSects;
+         pCur;
+         pCur = pCur->pNext)
+        pCur->pVMRC = pVM->pVMRC;
+
+    for (PPDMCRITSECTRWINT pCur = pUVM->pdm.s.pRwCritSects;
+         pCur;
+         pCur = pCur->pNext)
+        pCur->pVMRC = pVM->pVMRC;
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+}
+
+
+/**
+ * Deletes all remaining critical sections.
+ *
+ * This is called at the very end of the termination process.  It is also called
+ * at the end of vmR3CreateU failure cleanup, which may cause it to be called
+ * twice depending on where vmR3CreateU actually failed.  We have to do the
+ * latter call because other components expect the critical sections to be
+ * automatically deleted.
+ *
+ * @returns VBox status code.
+ *          First error code, rest is lost.
+ * @param   pVM             The cross context VM structure.
+ * @remark  Don't confuse this with PDMR3CritSectDelete.
+ */
+VMMR3_INT_DECL(int) PDMR3CritSectBothTerm(PVM pVM)
+{
+    PUVM    pUVM = pVM->pUVM;
+    int     rc   = VINF_SUCCESS;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+
+    while (pUVM->pdm.s.pCritSects)
+    {
+        int rc2 = pdmR3CritSectDeleteOne(pVM, pUVM, pUVM->pdm.s.pCritSects, NULL, true /* final */);
+        AssertRC(rc2);
+        if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+            rc = rc2;
+    }
+
+    while (pUVM->pdm.s.pRwCritSects)
+    {
+        int rc2 = pdmR3CritSectRwDeleteOne(pVM, pUVM, pUVM->pdm.s.pRwCritSects, NULL, true /* final */);
+        AssertRC(rc2);
+        if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+            rc = rc2;
+    }
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Initializes a critical section and inserts it into the list.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pCritSect       The critical section.
+ * @param   pvKey           The owner key.
+ * @param   SRC_POS         The source position.
+ * @param   fUniqueClass    Whether to create a unique lock validator class for
+ *                          it or not.
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   va              Arguments for the format string.
+ */
+static int pdmR3CritSectInitOne(PVM pVM, PPDMCRITSECTINT pCritSect, void *pvKey, RT_SRC_POS_DECL, bool fUniqueClass,
+                                const char *pszNameFmt, va_list va)
+{
+    VM_ASSERT_EMT(pVM);
+    Assert(pCritSect->Core.u32Magic != RTCRITSECT_MAGIC);
+
+    /*
+     * Allocate the semaphore.
+     */
+    AssertCompile(sizeof(SUPSEMEVENT) == sizeof(pCritSect->Core.EventSem));
+    int rc = SUPSemEventCreate(pVM->pSession, (PSUPSEMEVENT)&pCritSect->Core.EventSem);
+    if (RT_SUCCESS(rc))
+    {
+        /* Only format the name once. */
+        char *pszName = RTStrAPrintf2V(pszNameFmt, va); /** @todo plug the "leak"... */
+        if (pszName)
+        {
+            RT_SRC_POS_NOREF(); RT_NOREF(fUniqueClass);
+#ifndef PDMCRITSECT_STRICT
+            pCritSect->Core.pValidatorRec = NULL;
+#else
+            rc = RTLockValidatorRecExclCreate(&pCritSect->Core.pValidatorRec,
+# ifdef RT_LOCK_STRICT_ORDER
+                                              fUniqueClass
+                                              ? RTLockValidatorClassCreateUnique(RT_SRC_POS_ARGS, "%s", pszName)
+                                              : RTLockValidatorClassForSrcPos(RT_SRC_POS_ARGS, "%s", pszName),
+# else
+                                              NIL_RTLOCKVALCLASS,
+# endif
+                                              RTLOCKVAL_SUB_CLASS_NONE,
+                                              pCritSect, true, "%s", pszName);
+#endif
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Initialize the structure (first bit is c&p from RTCritSectInitEx).
+                 */
+                pCritSect->Core.u32Magic             = RTCRITSECT_MAGIC;
+                pCritSect->Core.fFlags               = 0;
+                pCritSect->Core.cNestings            = 0;
+                pCritSect->Core.cLockers             = -1;
+                pCritSect->Core.NativeThreadOwner    = NIL_RTNATIVETHREAD;
+                pCritSect->pVMR3                     = pVM;
+                pCritSect->pVMR0                     = pVM->pVMR0ForCall;
+                pCritSect->pVMRC                     = pVM->pVMRC;
+                pCritSect->pvKey                     = pvKey;
+                pCritSect->fAutomaticDefaultCritsect = false;
+                pCritSect->fUsedByTimerOrSimilar     = false;
+                pCritSect->hEventToSignal            = NIL_SUPSEMEVENT;
+                pCritSect->pszName                   = pszName;
+
+                STAMR3RegisterF(pVM, &pCritSect->StatContentionRZLock,  STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSects/%s/ContentionRZLock", pCritSect->pszName);
+                STAMR3RegisterF(pVM, &pCritSect->StatContentionRZUnlock,STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSects/%s/ContentionRZUnlock", pCritSect->pszName);
+                STAMR3RegisterF(pVM, &pCritSect->StatContentionR3,      STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSects/%s/ContentionR3", pCritSect->pszName);
+#ifdef VBOX_WITH_STATISTICS
+                STAMR3RegisterF(pVM, &pCritSect->StatLocked,        STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_OCCURENCE, NULL, "/PDM/CritSects/%s/Locked", pCritSect->pszName);
+#endif
+
+                PUVM pUVM = pVM->pUVM;
+                RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+                pCritSect->pNext = pUVM->pdm.s.pCritSects;
+                pUVM->pdm.s.pCritSects = pCritSect;
+                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+                return VINF_SUCCESS;
+            }
+
+            RTStrFree(pszName);
+        }
+        else
+            rc = VERR_NO_STR_MEMORY;
+        SUPSemEventClose(pVM->pSession, (SUPSEMEVENT)pCritSect->Core.EventSem);
+    }
+    return rc;
+}
+
+
+/**
+ * Initializes a read/write critical section and inserts it into the list.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pCritSect       The read/write critical section.
+ * @param   pvKey           The owner key.
+ * @param   SRC_POS         The source position.
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   va              Arguments for the format string.
+ */
+static int pdmR3CritSectRwInitOne(PVM pVM, PPDMCRITSECTRWINT pCritSect, void *pvKey, RT_SRC_POS_DECL,
+                                  const char *pszNameFmt, va_list va)
+{
+    VM_ASSERT_EMT(pVM);
+    Assert(pCritSect->Core.u32Magic != RTCRITSECTRW_MAGIC);
+
+    /*
+     * Allocate the semaphores.
+     */
+    AssertCompile(sizeof(SUPSEMEVENT) == sizeof(pCritSect->Core.hEvtWrite));
+    int rc = SUPSemEventCreate(pVM->pSession, (PSUPSEMEVENT)&pCritSect->Core.hEvtWrite);
+    if (RT_SUCCESS(rc))
+    {
+        AssertCompile(sizeof(SUPSEMEVENTMULTI) == sizeof(pCritSect->Core.hEvtRead));
+        rc = SUPSemEventMultiCreate(pVM->pSession, (PSUPSEMEVENT)&pCritSect->Core.hEvtRead);
+        if (RT_SUCCESS(rc))
+        {
+            /* Only format the name once. */
+            char *pszName = RTStrAPrintf2V(pszNameFmt, va); /** @todo plug the "leak"... */
+            if (pszName)
+            {
+                pCritSect->Core.pValidatorRead  = NULL;
+                pCritSect->Core.pValidatorWrite = NULL;
+                RT_SRC_POS_NOREF();
+#ifdef PDMCRITSECTRW_STRICT
+# ifdef RT_LOCK_STRICT_ORDER
+                RTLOCKVALCLASS hClass = RTLockValidatorClassForSrcPos(RT_SRC_POS_ARGS, "%s", pszName);
+# else
+                RTLOCKVALCLASS hClass = NIL_RTLOCKVALCLASS;
+# endif
+                rc = RTLockValidatorRecExclCreate(&pCritSect->Core.pValidatorWrite, hClass, RTLOCKVAL_SUB_CLASS_NONE,
+                                                  pCritSect, true, "%s", pszName);
+                if (RT_SUCCESS(rc))
+                    rc = RTLockValidatorRecSharedCreate(&pCritSect->Core.pValidatorRead, hClass, RTLOCKVAL_SUB_CLASS_NONE,
+                                                        pCritSect, false /*fSignaller*/, true, "%s", pszName);
+#endif
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Initialize the structure (first bit is c&p from RTCritSectRwInitEx).
+                     */
+                    pCritSect->Core.u32Magic             = RTCRITSECTRW_MAGIC;
+                    pCritSect->Core.fNeedReset           = false;
+                    pCritSect->Core.u64State             = 0;
+                    pCritSect->Core.hNativeWriter        = NIL_RTNATIVETHREAD;
+                    pCritSect->Core.cWriterReads         = 0;
+                    pCritSect->Core.cWriteRecursions     = 0;
+#if HC_ARCH_BITS == 32
+                    pCritSect->Core.HCPtrPadding         = NIL_RTHCPTR;
+#endif
+                    pCritSect->pVMR3                     = pVM;
+                    pCritSect->pVMR0                     = pVM->pVMR0ForCall;
+                    pCritSect->pVMRC                     = pVM->pVMRC;
+                    pCritSect->pvKey                     = pvKey;
+                    pCritSect->pszName                   = pszName;
+
+                    STAMR3RegisterF(pVM, &pCritSect->StatContentionRZEnterExcl,   STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSectsRw/%s/ContentionRZEnterExcl", pCritSect->pszName);
+                    STAMR3RegisterF(pVM, &pCritSect->StatContentionRZLeaveExcl,   STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSectsRw/%s/ContentionRZLeaveExcl", pCritSect->pszName);
+                    STAMR3RegisterF(pVM, &pCritSect->StatContentionRZEnterShared, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSectsRw/%s/ContentionRZEnterShared", pCritSect->pszName);
+                    STAMR3RegisterF(pVM, &pCritSect->StatContentionRZLeaveShared, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSectsRw/%s/ContentionRZLeaveShared", pCritSect->pszName);
+                    STAMR3RegisterF(pVM, &pCritSect->StatContentionR3EnterExcl,   STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSectsRw/%s/ContentionR3EnterExcl", pCritSect->pszName);
+                    STAMR3RegisterF(pVM, &pCritSect->StatContentionR3EnterShared, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSectsRw/%s/ContentionR3EnterShared", pCritSect->pszName);
+                    STAMR3RegisterF(pVM, &pCritSect->StatRZEnterExcl,             STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSectsRw/%s/RZEnterExcl", pCritSect->pszName);
+                    STAMR3RegisterF(pVM, &pCritSect->StatRZEnterShared,           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSectsRw/%s/RZEnterShared", pCritSect->pszName);
+                    STAMR3RegisterF(pVM, &pCritSect->StatR3EnterExcl,             STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSectsRw/%s/R3EnterExcl", pCritSect->pszName);
+                    STAMR3RegisterF(pVM, &pCritSect->StatR3EnterShared,           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSectsRw/%s/R3EnterShared", pCritSect->pszName);
+#ifdef VBOX_WITH_STATISTICS
+                    STAMR3RegisterF(pVM, &pCritSect->StatWriteLocked,         STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_OCCURENCE, NULL, "/PDM/CritSectsRw/%s/WriteLocked", pCritSect->pszName);
+#endif
+
+                    PUVM pUVM = pVM->pUVM;
+                    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+                    pCritSect->pNext = pUVM->pdm.s.pRwCritSects;
+                    pUVM->pdm.s.pRwCritSects = pCritSect;
+                    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+                    return VINF_SUCCESS;
+                }
+
+                RTStrFree(pszName);
+            }
+            else
+                rc = VERR_NO_STR_MEMORY;
+            SUPSemEventMultiClose(pVM->pSession, (SUPSEMEVENT)pCritSect->Core.hEvtRead);
+        }
+        SUPSemEventClose(pVM->pSession, (SUPSEMEVENT)pCritSect->Core.hEvtWrite);
+    }
+    return rc;
+}
+
+
+/**
+ * Initializes a PDM critical section for internal use.
+ *
+ * The PDM critical sections are derived from the IPRT critical sections, but
+ * works in ring-0 and raw-mode context as well.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pCritSect       Pointer to the critical section.
+ * @param   SRC_POS         Use RT_SRC_POS.
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   ...             Arguments for the format string.
+ * @thread  EMT
+ */
+VMMR3DECL(int) PDMR3CritSectInit(PVM pVM, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL, const char *pszNameFmt, ...)
+{
+#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
+    AssertCompile(sizeof(pCritSect->padding) >= sizeof(pCritSect->s));
+#endif
+    Assert(RT_ALIGN_P(pCritSect, sizeof(uintptr_t)) == pCritSect);
+    va_list va;
+    va_start(va, pszNameFmt);
+    int rc = pdmR3CritSectInitOne(pVM, &pCritSect->s, pCritSect, RT_SRC_POS_ARGS, false /*fUniqueClass*/, pszNameFmt, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Initializes a PDM read/write critical section for internal use.
+ *
+ * The PDM read/write critical sections are derived from the IPRT read/write
+ * critical sections, but works in ring-0 and raw-mode context as well.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pCritSect       Pointer to the read/write critical section.
+ * @param   SRC_POS         Use RT_SRC_POS.
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   ...             Arguments for the format string.
+ * @thread  EMT
+ */
+VMMR3DECL(int) PDMR3CritSectRwInit(PVM pVM, PPDMCRITSECTRW pCritSect, RT_SRC_POS_DECL, const char *pszNameFmt, ...)
+{
+#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
+    AssertCompile(sizeof(pCritSect->padding) >= sizeof(pCritSect->s));
+#endif
+    Assert(RT_ALIGN_P(pCritSect, sizeof(uintptr_t)) == pCritSect);
+    va_list va;
+    va_start(va, pszNameFmt);
+    int rc = pdmR3CritSectRwInitOne(pVM, &pCritSect->s, pCritSect, RT_SRC_POS_ARGS, pszNameFmt, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Initializes a PDM critical section for a device.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         Device instance.
+ * @param   pCritSect       Pointer to the critical section.
+ * @param   SRC_POS         The source position.  Optional.
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   va              Arguments for the format string.
+ */
+int pdmR3CritSectInitDevice(PVM pVM, PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
+                            const char *pszNameFmt, va_list va)
+{
+    return pdmR3CritSectInitOne(pVM, &pCritSect->s, pDevIns, RT_SRC_POS_ARGS, false /*fUniqueClass*/, pszNameFmt, va);
+}
+
+
+/**
+ * Initializes a PDM read/write critical section for a device.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         Device instance.
+ * @param   pCritSect       Pointer to the read/write critical section.
+ * @param   SRC_POS         The source position.  Optional.
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   va              Arguments for the format string.
+ */
+int pdmR3CritSectRwInitDevice(PVM pVM, PPDMDEVINS pDevIns, PPDMCRITSECTRW pCritSect, RT_SRC_POS_DECL,
+                              const char *pszNameFmt, va_list va)
+{
+    return pdmR3CritSectRwInitOne(pVM, &pCritSect->s, pDevIns, RT_SRC_POS_ARGS, pszNameFmt, va);
+}
+
+
+/**
+ * Initializes the automatic default PDM critical section for a device.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         Device instance.
+ * @param   SRC_POS         The source position.  Optional.
+ * @param   pCritSect       Pointer to the critical section.
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   ...             Arguments for the format string.
+ */
+int pdmR3CritSectInitDeviceAuto(PVM pVM, PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
+                                const char *pszNameFmt, ...)
+{
+    va_list va;
+    va_start(va, pszNameFmt);
+    int rc = pdmR3CritSectInitOne(pVM, &pCritSect->s, pDevIns, RT_SRC_POS_ARGS, true /*fUniqueClass*/, pszNameFmt, va);
+    if (RT_SUCCESS(rc))
+        pCritSect->s.fAutomaticDefaultCritsect = true;
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Initializes a PDM critical section for a driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDrvIns         Driver instance.
+ * @param   pCritSect       Pointer to the critical section.
+ * @param   SRC_POS         The source position.  Optional.
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   ...             Arguments for the format string.
+ */
+int pdmR3CritSectInitDriver(PVM pVM, PPDMDRVINS pDrvIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
+                            const char *pszNameFmt, ...)
+{
+    va_list va;
+    va_start(va, pszNameFmt);
+    int rc = pdmR3CritSectInitOne(pVM, &pCritSect->s, pDrvIns, RT_SRC_POS_ARGS, false /*fUniqueClass*/, pszNameFmt, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Initializes a PDM read/write critical section for a driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDrvIns         Driver instance.
+ * @param   pCritSect       Pointer to the read/write critical section.
+ * @param   SRC_POS         The source position.  Optional.
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   ...             Arguments for the format string.
+ */
+int pdmR3CritSectRwInitDriver(PVM pVM, PPDMDRVINS pDrvIns, PPDMCRITSECTRW pCritSect, RT_SRC_POS_DECL,
+                              const char *pszNameFmt, ...)
+{
+    va_list va;
+    va_start(va, pszNameFmt);
+    int rc = pdmR3CritSectRwInitOne(pVM, &pCritSect->s, pDrvIns, RT_SRC_POS_ARGS, pszNameFmt, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Deletes one critical section.
+ *
+ * @returns Return code from RTCritSectDelete.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pCritSect   The critical section.
+ * @param   pPrev       The previous critical section in the list.
+ * @param   fFinal      Set if this is the final call and statistics shouldn't be deregistered.
+ *
+ * @remarks Caller must have entered the ListCritSect.
+ */
+static int pdmR3CritSectDeleteOne(PVM pVM, PUVM pUVM, PPDMCRITSECTINT pCritSect, PPDMCRITSECTINT pPrev, bool fFinal)
+{
+    /*
+     * Assert free waiters and so on (c&p from RTCritSectDelete).
+     */
+    Assert(pCritSect->Core.u32Magic == RTCRITSECT_MAGIC);
+    //Assert(pCritSect->Core.cNestings == 0); - we no longer reset this when leaving.
+    Assert(pCritSect->Core.cLockers == -1);
+    Assert(pCritSect->Core.NativeThreadOwner == NIL_RTNATIVETHREAD);
+    Assert(RTCritSectIsOwner(&pUVM->pdm.s.ListCritSect));
+
+    /*
+     * Unlink it.
+     */
+    if (pPrev)
+        pPrev->pNext = pCritSect->pNext;
+    else
+        pUVM->pdm.s.pCritSects = pCritSect->pNext;
+
+    /*
+     * Delete it (parts taken from RTCritSectDelete).
+     * In case someone is waiting we'll signal the semaphore cLockers + 1 times.
+     */
+    ASMAtomicWriteU32(&pCritSect->Core.u32Magic, 0);
+    SUPSEMEVENT hEvent = (SUPSEMEVENT)pCritSect->Core.EventSem;
+    pCritSect->Core.EventSem = NIL_RTSEMEVENT;
+    while (pCritSect->Core.cLockers-- >= 0)
+        SUPSemEventSignal(pVM->pSession, hEvent);
+    ASMAtomicWriteS32(&pCritSect->Core.cLockers, -1);
+    int rc = SUPSemEventClose(pVM->pSession, hEvent);
+    AssertRC(rc);
+    RTLockValidatorRecExclDestroy(&pCritSect->Core.pValidatorRec);
+    pCritSect->pNext   = NULL;
+    pCritSect->pvKey   = NULL;
+    pCritSect->pVMR3   = NULL;
+    pCritSect->pVMR0   = NIL_RTR0PTR;
+    pCritSect->pVMRC   = NIL_RTRCPTR;
+    if (!fFinal)
+        STAMR3DeregisterF(pVM->pUVM, "/PDM/CritSects/%s/*", pCritSect->pszName);
+    RTStrFree((char *)pCritSect->pszName);
+    pCritSect->pszName = NULL;
+    return rc;
+}
+
+
+/**
+ * Deletes one read/write critical section.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pCritSect   The read/write critical section.
+ * @param   pPrev       The previous critical section in the list.
+ * @param   fFinal      Set if this is the final call and statistics shouldn't be deregistered.
+ *
+ * @remarks Caller must have entered the ListCritSect.
+ */
+static int pdmR3CritSectRwDeleteOne(PVM pVM, PUVM pUVM, PPDMCRITSECTRWINT pCritSect, PPDMCRITSECTRWINT pPrev, bool fFinal)
+{
+    /*
+     * Assert free waiters and so on (c&p from RTCritSectRwDelete).
+     */
+    Assert(pCritSect->Core.u32Magic == RTCRITSECTRW_MAGIC);
+    //Assert(pCritSect->Core.cNestings == 0);
+    //Assert(pCritSect->Core.cLockers == -1);
+    Assert(pCritSect->Core.hNativeWriter == NIL_RTNATIVETHREAD);
+
+    /*
+     * Invalidate the structure and free the semaphores.
+     */
+    if (!ASMAtomicCmpXchgU32(&pCritSect->Core.u32Magic, RTCRITSECTRW_MAGIC_DEAD, RTCRITSECTRW_MAGIC))
+        AssertFailed();
+
+    /*
+     * Unlink it.
+     */
+    if (pPrev)
+        pPrev->pNext = pCritSect->pNext;
+    else
+        pUVM->pdm.s.pRwCritSects = pCritSect->pNext;
+
+    /*
+     * Delete it (parts taken from RTCritSectRwDelete).
+     * In case someone is waiting we'll signal the semaphore cLockers + 1 times.
+     */
+    pCritSect->Core.fFlags   = 0;
+    pCritSect->Core.u64State = 0;
+
+    SUPSEMEVENT      hEvtWrite = (SUPSEMEVENT)pCritSect->Core.hEvtWrite;
+    pCritSect->Core.hEvtWrite  = NIL_RTSEMEVENT;
+    AssertCompile(sizeof(hEvtWrite) == sizeof(pCritSect->Core.hEvtWrite));
+
+    SUPSEMEVENTMULTI hEvtRead  = (SUPSEMEVENTMULTI)pCritSect->Core.hEvtRead;
+    pCritSect->Core.hEvtRead   = NIL_RTSEMEVENTMULTI;
+    AssertCompile(sizeof(hEvtRead) == sizeof(pCritSect->Core.hEvtRead));
+
+    int rc1 = SUPSemEventClose(pVM->pSession, hEvtWrite);     AssertRC(rc1);
+    int rc2 = SUPSemEventMultiClose(pVM->pSession, hEvtRead); AssertRC(rc2);
+
+    RTLockValidatorRecSharedDestroy(&pCritSect->Core.pValidatorRead);
+    RTLockValidatorRecExclDestroy(&pCritSect->Core.pValidatorWrite);
+
+    pCritSect->pNext   = NULL;
+    pCritSect->pvKey   = NULL;
+    pCritSect->pVMR3   = NULL;
+    pCritSect->pVMR0   = NIL_RTR0PTR;
+    pCritSect->pVMRC   = NIL_RTRCPTR;
+    if (!fFinal)
+        STAMR3DeregisterF(pVM->pUVM, "/PDM/CritSectsRw/%s/*", pCritSect->pszName);
+    RTStrFree((char *)pCritSect->pszName);
+    pCritSect->pszName = NULL;
+
+    return RT_SUCCESS(rc1) ? rc2 : rc1;
+}
+
+
+/**
+ * Deletes all critical sections with a give initializer key.
+ *
+ * @returns VBox status code.
+ *          The entire list is processed on failure, so we'll only
+ *          return the first error code. This shouldn't be a problem
+ *          since errors really shouldn't happen here.
+ * @param   pVM     The cross context VM structure.
+ * @param   pvKey   The initializer key.
+ */
+static int pdmR3CritSectDeleteByKey(PVM pVM, void *pvKey)
+{
+    /*
+     * Iterate the list and match key.
+     */
+    PUVM            pUVM  = pVM->pUVM;
+    int             rc    = VINF_SUCCESS;
+    PPDMCRITSECTINT pPrev = NULL;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMCRITSECTINT pCur  = pUVM->pdm.s.pCritSects;
+    while (pCur)
+    {
+        if (pCur->pvKey == pvKey)
+        {
+            int rc2 = pdmR3CritSectDeleteOne(pVM, pUVM, pCur, pPrev, false /* not final */);
+            AssertRC(rc2);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+        }
+
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Deletes all read/write critical sections with a give initializer key.
+ *
+ * @returns VBox status code.
+ *          The entire list is processed on failure, so we'll only
+ *          return the first error code. This shouldn't be a problem
+ *          since errors really shouldn't happen here.
+ * @param   pVM     The cross context VM structure.
+ * @param   pvKey   The initializer key.
+ */
+static int pdmR3CritSectRwDeleteByKey(PVM pVM, void *pvKey)
+{
+    /*
+     * Iterate the list and match key.
+     */
+    PUVM                pUVM  = pVM->pUVM;
+    int                 rc    = VINF_SUCCESS;
+    PPDMCRITSECTRWINT   pPrev = NULL;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMCRITSECTRWINT   pCur  = pUVM->pdm.s.pRwCritSects;
+    while (pCur)
+    {
+        if (pCur->pvKey == pvKey)
+        {
+            int rc2 = pdmR3CritSectRwDeleteOne(pVM, pUVM, pCur, pPrev, false /* not final */);
+            AssertRC(rc2);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+        }
+
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Deletes all undeleted critical sections (both types) initialized by a given
+ * device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     The device handle.
+ */
+int pdmR3CritSectBothDeleteDevice(PVM pVM, PPDMDEVINS pDevIns)
+{
+    int rc1 = pdmR3CritSectDeleteByKey(pVM, pDevIns);
+    int rc2 = pdmR3CritSectRwDeleteByKey(pVM, pDevIns);
+    return RT_SUCCESS(rc1) ? rc2 : rc1;
+}
+
+
+/**
+ * Deletes all undeleted critical sections (both types) initialized by a given
+ * driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDrvIns     The driver handle.
+ */
+int pdmR3CritSectBothDeleteDriver(PVM pVM, PPDMDRVINS pDrvIns)
+{
+    int rc1 = pdmR3CritSectDeleteByKey(pVM, pDrvIns);
+    int rc2 = pdmR3CritSectRwDeleteByKey(pVM, pDrvIns);
+    return RT_SUCCESS(rc1) ? rc2 : rc1;
+}
+
+
+/**
+ * Deletes the critical section.
+ *
+ * @returns VBox status code.
+ * @param   pCritSect           The PDM critical section to destroy.
+ */
+VMMR3DECL(int) PDMR3CritSectDelete(PPDMCRITSECT pCritSect)
+{
+    if (!RTCritSectIsInitialized(&pCritSect->s.Core))
+        return VINF_SUCCESS;
+
+    /*
+     * Find and unlink it.
+     */
+    PVM             pVM   = pCritSect->s.pVMR3;
+    PUVM            pUVM  = pVM->pUVM;
+    AssertReleaseReturn(pVM, VERR_PDM_CRITSECT_IPE);
+    PPDMCRITSECTINT pPrev = NULL;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMCRITSECTINT pCur  = pUVM->pdm.s.pCritSects;
+    while (pCur)
+    {
+        if (pCur == &pCritSect->s)
+        {
+            int rc = pdmR3CritSectDeleteOne(pVM, pUVM, pCur, pPrev, false /* not final */);
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            return rc;
+        }
+
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    AssertReleaseMsgFailed(("pCritSect=%p wasn't found!\n", pCritSect));
+    return VERR_PDM_CRITSECT_NOT_FOUND;
+}
+
+
+/**
+ * Deletes the read/write critical section.
+ *
+ * @returns VBox status code.
+ * @param   pCritSect           The PDM read/write critical section to destroy.
+ */
+VMMR3DECL(int) PDMR3CritSectRwDelete(PPDMCRITSECTRW pCritSect)
+{
+    if (!PDMCritSectRwIsInitialized(pCritSect))
+        return VINF_SUCCESS;
+
+    /*
+     * Find and unlink it.
+     */
+    PVM                 pVM   = pCritSect->s.pVMR3;
+    PUVM                pUVM  = pVM->pUVM;
+    AssertReleaseReturn(pVM, VERR_PDM_CRITSECT_IPE);
+    PPDMCRITSECTRWINT   pPrev = NULL;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMCRITSECTRWINT   pCur  = pUVM->pdm.s.pRwCritSects;
+    while (pCur)
+    {
+        if (pCur == &pCritSect->s)
+        {
+            int rc = pdmR3CritSectRwDeleteOne(pVM, pUVM, pCur, pPrev, false /* not final */);
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            return rc;
+        }
+
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    AssertReleaseMsgFailed(("pCritSect=%p wasn't found!\n", pCritSect));
+    return VERR_PDM_CRITSECT_NOT_FOUND;
+}
+
+
+/**
+ * Gets the name of the critical section.
+ *
+ *
+ * @returns Pointer to the critical section name (read only) on success,
+ *          NULL on failure (invalid critical section).
+ * @param   pCritSect           The critical section.
+ */
+VMMR3DECL(const char *) PDMR3CritSectName(PCPDMCRITSECT pCritSect)
+{
+    AssertPtrReturn(pCritSect, NULL);
+    AssertReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, NULL);
+    return pCritSect->s.pszName;
+}
+
+
+/**
+ * Gets the name of the read/write critical section.
+ *
+ *
+ * @returns Pointer to the critical section name (read only) on success,
+ *          NULL on failure (invalid critical section).
+ * @param   pCritSect           The read/write critical section.
+ */
+VMMR3DECL(const char *) PDMR3CritSectRwName(PCPDMCRITSECTRW pCritSect)
+{
+    AssertPtrReturn(pCritSect, NULL);
+    AssertReturn(pCritSect->s.Core.u32Magic == RTCRITSECTRW_MAGIC, NULL);
+    return pCritSect->s.pszName;
+}
+
+
+/**
+ * Yield the critical section if someone is waiting on it.
+ *
+ * When yielding, we'll leave the critical section and try to make sure the
+ * other waiting threads get a chance of entering before we reclaim it.
+ *
+ * @retval  true if yielded.
+ * @retval  false if not yielded.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pCritSect           The critical section.
+ */
+VMMR3DECL(bool) PDMR3CritSectYield(PVM pVM, PPDMCRITSECT pCritSect)
+{
+    AssertPtrReturn(pCritSect, false);
+    AssertReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, false);
+    Assert(pCritSect->s.Core.NativeThreadOwner == RTThreadNativeSelf());
+    Assert(!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP));
+    RT_NOREF(pVM);
+
+    /* No recursion allowed here. */
+    int32_t const cNestings = pCritSect->s.Core.cNestings;
+    AssertReturn(cNestings == 1, false);
+
+    int32_t const cLockers  = ASMAtomicReadS32(&pCritSect->s.Core.cLockers);
+    if (cLockers < cNestings)
+        return false;
+
+#ifdef PDMCRITSECT_STRICT
+    RTLOCKVALSRCPOS const SrcPos = pCritSect->s.Core.pValidatorRec->SrcPos;
+#endif
+    PDMCritSectLeave(pCritSect);
+
+    /*
+     * If we're lucky, then one of the waiters has entered the lock already.
+     * We spin a little bit in hope for this to happen so we can avoid the
+     * yield detour.
+     */
+    if (ASMAtomicUoReadS32(&pCritSect->s.Core.cNestings) == 0)
+    {
+        int cLoops = 20;
+        while (   cLoops > 0
+               && ASMAtomicUoReadS32(&pCritSect->s.Core.cNestings) == 0
+               && ASMAtomicUoReadS32(&pCritSect->s.Core.cLockers)  >= 0)
+        {
+            ASMNopPause();
+            cLoops--;
+        }
+        if (cLoops == 0)
+            RTThreadYield();
+    }
+
+#ifdef PDMCRITSECT_STRICT
+    int rc = PDMCritSectEnterDebug(pCritSect, VERR_IGNORED,
+                                   SrcPos.uId, SrcPos.pszFile, SrcPos.uLine, SrcPos.pszFunction);
+#else
+    int rc = PDMCritSectEnter(pCritSect, VERR_IGNORED);
+#endif
+    AssertLogRelRC(rc);
+    return true;
+}
+
+
+/**
+ * PDMR3CritSectBothCountOwned worker.
+ *
+ * @param   pszName         The critical section name.
+ * @param   ppszNames       Pointer to the pszNames variable.
+ * @param   pcchLeft        Pointer to the cchLeft variable.
+ * @param   fFirst          Whether this is the first name or not.
+ */
+static void pdmR3CritSectAppendNameToList(char const *pszName, char **ppszNames, size_t *pcchLeft, bool fFirst)
+{
+    size_t cchLeft = *pcchLeft;
+    if (cchLeft)
+    {
+        char *pszNames = *ppszNames;
+
+        /* try add comma. */
+        if (fFirst)
+        {
+            *pszNames++ = ',';
+            if (--cchLeft)
+            {
+                *pszNames++ = ' ';
+                cchLeft--;
+            }
+        }
+
+        /* try copy the name. */
+        if (cchLeft)
+        {
+            size_t const cchName = strlen(pszName);
+            if (cchName < cchLeft)
+            {
+                memcpy(pszNames, pszName, cchName);
+                pszNames += cchName;
+                cchLeft -= cchName;
+            }
+            else
+            {
+                if (cchLeft > 2)
+                {
+                    memcpy(pszNames, pszName, cchLeft - 2);
+                    pszNames += cchLeft - 2;
+                    cchLeft = 2;
+                }
+                while (cchLeft-- > 0)
+                    *pszNames++ = '+';
+            }
+        }
+        *pszNames = '\0';
+
+        *pcchLeft  = cchLeft;
+        *ppszNames = pszNames;
+    }
+}
+
+
+/**
+ * Counts the critical sections (both type) owned by the calling thread,
+ * optionally returning a comma separated list naming them.
+ *
+ * Read ownerships are not included in non-strict builds.
+ *
+ * This is for diagnostic purposes only.
+ *
+ * @returns Lock count.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pszNames        Where to return the critical section names.
+ * @param   cbNames         The size of the buffer.
+ */
+VMMR3DECL(uint32_t) PDMR3CritSectCountOwned(PVM pVM, char *pszNames, size_t cbNames)
+{
+    /*
+     * Init the name buffer.
+     */
+    size_t cchLeft = cbNames;
+    if (cchLeft)
+    {
+        cchLeft--;
+        pszNames[0] = pszNames[cchLeft] = '\0';
+    }
+
+    /*
+     * Iterate the critical sections.
+     */
+    uint32_t                cCritSects = 0;
+    RTNATIVETHREAD const    hNativeThread = RTThreadNativeSelf();
+    /* This is unsafe, but wtf. */
+    for (PPDMCRITSECTINT pCur = pVM->pUVM->pdm.s.pCritSects;
+         pCur;
+         pCur = pCur->pNext)
+    {
+        /* Same as RTCritSectIsOwner(). */
+        if (pCur->Core.NativeThreadOwner == hNativeThread)
+        {
+            cCritSects++;
+            pdmR3CritSectAppendNameToList(pCur->pszName, &pszNames, &cchLeft, cCritSects == 1);
+        }
+    }
+
+    /* This is unsafe, but wtf. */
+    for (PPDMCRITSECTRWINT pCur = pVM->pUVM->pdm.s.pRwCritSects;
+         pCur;
+         pCur = pCur->pNext)
+    {
+        if (   pCur->Core.hNativeWriter == hNativeThread
+            || PDMCritSectRwIsReadOwner((PPDMCRITSECTRW)pCur, false /*fWannaHear*/) )
+        {
+            cCritSects++;
+            pdmR3CritSectAppendNameToList(pCur->pszName, &pszNames, &cchLeft, cCritSects == 1);
+        }
+    }
+
+    return cCritSects;
+}
+
+
+/**
+ * Leave all critical sections the calling thread owns.
+ *
+ * This is only used when entering guru meditation in order to prevent other
+ * EMTs and I/O threads from deadlocking.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) PDMR3CritSectLeaveAll(PVM pVM)
+{
+    RTNATIVETHREAD const hNativeSelf = RTThreadNativeSelf();
+    PUVM                 pUVM        = pVM->pUVM;
+
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMCRITSECTINT pCur = pUVM->pdm.s.pCritSects;
+         pCur;
+         pCur = pCur->pNext)
+    {
+        while (     pCur->Core.NativeThreadOwner == hNativeSelf
+               &&   pCur->Core.cNestings > 0)
+            PDMCritSectLeave((PPDMCRITSECT)pCur);
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+}
+
+
+/**
+ * Gets the address of the NOP critical section.
+ *
+ * The NOP critical section will not perform any thread serialization but let
+ * all enter immediately and concurrently.
+ *
+ * @returns The address of the NOP critical section.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMR3DECL(PPDMCRITSECT)             PDMR3CritSectGetNop(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, NULL);
+    return &pVM->pdm.s.NopCritSect;
+}
+
+
+/**
+ * Gets the ring-0 address of the NOP critical section.
+ *
+ * @returns The ring-0 address of the NOP critical section.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMR3DECL(R0PTRTYPE(PPDMCRITSECT))  PDMR3CritSectGetNopR0(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, NIL_RTR0PTR);
+    return MMHyperR3ToR0(pVM, &pVM->pdm.s.NopCritSect);
+}
+
+
+/**
+ * Gets the raw-mode context address of the NOP critical section.
+ *
+ * @returns The raw-mode context address of the NOP critical section.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMR3DECL(RCPTRTYPE(PPDMCRITSECT))  PDMR3CritSectGetNopRC(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, NIL_RTRCPTR);
+    return MMHyperR3ToRC(pVM, &pVM->pdm.s.NopCritSect);
+}
+
diff --git a/src/VBox/VMM/VMMR3/PDMDevHlp.cpp b/src/VBox/VMM/VMMR3/PDMDevHlp.cpp
new file mode 100644
index 00000000..02a2d6d9
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMDevHlp.cpp
@@ -0,0 +1,4717 @@
+/* $Id: PDMDevHlp.cpp $ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, Device Helpers.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DEVICE
+#define PDMPCIDEV_INCLUDE_PRIVATE  /* Hack to get pdmpcidevint.h included at the right point. */
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/vmapi.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/version.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+#include "dtrace/VBoxVMM.h"
+#include "PDMInline.h"
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** @def PDM_DEVHLP_DEADLOCK_DETECTION
+ * Define this to enable the deadlock detection when accessing physical memory.
+ */
+#if /*defined(DEBUG_bird) ||*/ defined(DOXYGEN_RUNNING)
+# define PDM_DEVHLP_DEADLOCK_DETECTION /**< @todo enable DevHlp deadlock detection! */
+#endif
+
+
+
+/** @name R3 DevHlp
+ * @{
+ */
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnIoPortCreateEx} */
+static DECLCALLBACK(int) pdmR3DevHlp_IoPortCreateEx(PPDMDEVINS pDevIns, RTIOPORT cPorts, uint32_t fFlags, PPDMPCIDEV pPciDev,
+                                                    uint32_t iPciRegion, PFNIOMIOPORTNEWOUT pfnOut, PFNIOMIOPORTNEWIN pfnIn,
+                                                    PFNIOMIOPORTNEWOUTSTRING pfnOutStr, PFNIOMIOPORTNEWINSTRING pfnInStr, RTR3PTR pvUser,
+                                                    const char *pszDesc, PCIOMIOPORTDESC paExtDescs, PIOMIOPORTHANDLE phIoPorts)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_IoPortCreateEx: caller='%s'/%d: cPorts=%#x fFlags=%#x pPciDev=%p iPciRegion=%#x pfnOut=%p pfnIn=%p pfnOutStr=%p pfnInStr=%p pvUser=%p pszDesc=%p:{%s} paExtDescs=%p phIoPorts=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, cPorts, fFlags, pPciDev, iPciRegion, pfnOut, pfnIn, pfnOutStr, pfnInStr,
+             pvUser, pszDesc, pszDesc, paExtDescs, phIoPorts));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+    int rc = IOMR3IoPortCreate(pVM, pDevIns, cPorts, fFlags, pPciDev, iPciRegion,
+                               pfnOut, pfnIn, pfnOutStr, pfnInStr, pvUser, pszDesc, paExtDescs, phIoPorts);
+
+    LogFlow(("pdmR3DevHlp_IoPortCreateEx: caller='%s'/%d: returns %Rrc (*phIoPorts=%#x)\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc, *phIoPorts));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnIoPortMap} */
+static DECLCALLBACK(int) pdmR3DevHlp_IoPortMap(PPDMDEVINS pDevIns, IOMIOPORTHANDLE hIoPorts, RTIOPORT Port)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_IoPortMap: caller='%s'/%d: hIoPorts=%#x Port=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, hIoPorts, Port));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    int rc = IOMR3IoPortMap(pVM, pDevIns, hIoPorts, Port);
+
+    LogFlow(("pdmR3DevHlp_IoPortMap: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnIoPortUnmap} */
+static DECLCALLBACK(int) pdmR3DevHlp_IoPortUnmap(PPDMDEVINS pDevIns, IOMIOPORTHANDLE hIoPorts)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_IoPortMap: caller='%s'/%d: hIoPorts=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, hIoPorts));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    int rc = IOMR3IoPortUnmap(pVM, pDevIns, hIoPorts);
+
+    LogFlow(("pdmR3DevHlp_IoPortMap: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnIoPortGetMappingAddress} */
+static DECLCALLBACK(uint32_t) pdmR3DevHlp_IoPortGetMappingAddress(PPDMDEVINS pDevIns, IOMIOPORTHANDLE hIoPorts)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_IoPortGetMappingAddress: caller='%s'/%d: hIoPorts=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, hIoPorts));
+
+    uint32_t uAddress = IOMR3IoPortGetMappingAddress(pDevIns->Internal.s.pVMR3, pDevIns, hIoPorts);
+
+    LogFlow(("pdmR3DevHlp_IoPortGetMappingAddress: caller='%s'/%d: returns %#RX32\n", pDevIns->pReg->szName, pDevIns->iInstance, uAddress));
+    return uAddress;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMmioCreateEx} */
+static DECLCALLBACK(int) pdmR3DevHlp_MmioCreateEx(PPDMDEVINS pDevIns, RTGCPHYS cbRegion,
+                                                  uint32_t fFlags, PPDMPCIDEV pPciDev, uint32_t iPciRegion,
+                                                  PFNIOMMMIONEWWRITE pfnWrite, PFNIOMMMIONEWREAD pfnRead, PFNIOMMMIONEWFILL pfnFill,
+                                                  void *pvUser, const char *pszDesc, PIOMMMIOHANDLE phRegion)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_MmioCreateEx: caller='%s'/%d: cbRegion=%#RGp fFlags=%#x pPciDev=%p iPciRegion=%#x pfnWrite=%p pfnRead=%p pfnFill=%p pvUser=%p pszDesc=%p:{%s} phRegion=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, cbRegion, fFlags, pPciDev, iPciRegion, pfnWrite, pfnRead, pfnFill, pvUser, pszDesc, pszDesc, phRegion));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+    if (pDevIns->iInstance > 0)
+    {
+        pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
+        AssertReturn(pszDesc, VERR_NO_STR_MEMORY);
+    }
+
+    /* HACK ALERT! Round the size up to page size.  The PCI bus should do something similar before mapping it. */
+    /** @todo It's possible we need to do dummy MMIO fill-in of the PCI bus or
+     *        guest adds more alignment to an region. */
+    cbRegion = RT_ALIGN_T(cbRegion, PAGE_SIZE, RTGCPHYS);
+
+    int rc = IOMR3MmioCreate(pVM, pDevIns, cbRegion, fFlags, pPciDev, iPciRegion,
+                             pfnWrite, pfnRead, pfnFill, pvUser, pszDesc, phRegion);
+
+    LogFlow(("pdmR3DevHlp_MmioCreateEx: caller='%s'/%d: returns %Rrc (*phRegion=%#x)\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc, *phRegion));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMmioMap} */
+static DECLCALLBACK(int) pdmR3DevHlp_MmioMap(PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS GCPhys)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_MmioMap: caller='%s'/%d: hRegion=%#x GCPhys=%#RGp\n", pDevIns->pReg->szName, pDevIns->iInstance, hRegion, GCPhys));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    int rc = IOMR3MmioMap(pVM, pDevIns, hRegion, GCPhys);
+
+    LogFlow(("pdmR3DevHlp_MmioMap: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMmioUnmap} */
+static DECLCALLBACK(int) pdmR3DevHlp_MmioUnmap(PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_MmioUnmap: caller='%s'/%d: hRegion=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, hRegion));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    int rc = IOMR3MmioUnmap(pVM, pDevIns, hRegion);
+
+    LogFlow(("pdmR3DevHlp_MmioUnmap: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMmioReduce} */
+static DECLCALLBACK(int) pdmR3DevHlp_MmioReduce(PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS cbRegion)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_MmioReduce: caller='%s'/%d: hRegion=%#x cbRegion=%#RGp\n", pDevIns->pReg->szName, pDevIns->iInstance, hRegion, cbRegion));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_LOADING, VERR_VM_INVALID_VM_STATE);
+
+    int rc = IOMR3MmioReduce(pVM, pDevIns, hRegion, cbRegion);
+
+    LogFlow(("pdmR3DevHlp_MmioReduce: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMmioGetMappingAddress} */
+static DECLCALLBACK(RTGCPHYS) pdmR3DevHlp_MmioGetMappingAddress(PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_MmioGetMappingAddress: caller='%s'/%d: hRegion=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, hRegion));
+
+    RTGCPHYS GCPhys = IOMR3MmioGetMappingAddress(pDevIns->Internal.s.pVMR3, pDevIns, hRegion);
+
+    LogFlow(("pdmR3DevHlp_MmioGetMappingAddress: caller='%s'/%d: returns %RGp\n", pDevIns->pReg->szName, pDevIns->iInstance, GCPhys));
+    return GCPhys;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMmio2Create} */
+static DECLCALLBACK(int) pdmR3DevHlp_Mmio2Create(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, uint32_t iPciRegion, RTGCPHYS cbRegion,
+                                                 uint32_t fFlags, const char *pszDesc, void **ppvMapping, PPGMMMIO2HANDLE phRegion)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_Mmio2Create: caller='%s'/%d: pPciDev=%p (%#x) iPciRegion=%#x cbRegion=%#RGp fFlags=%RX32 pszDesc=%p:{%s} ppvMapping=%p phRegion=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPciDev, pPciDev ? pPciDev->uDevFn : UINT32_MAX, iPciRegion, cbRegion,
+             fFlags, pszDesc, pszDesc, ppvMapping, phRegion));
+    *ppvMapping = NULL;
+    *phRegion   = NIL_PGMMMIO2HANDLE;
+    AssertReturn(!pPciDev || pPciDev->Int.s.pDevInsR3 == pDevIns, VERR_INVALID_PARAMETER);
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertMsgReturn(   pVM->enmVMState == VMSTATE_CREATING
+                    || pVM->enmVMState == VMSTATE_LOADING,
+                    ("state %s, expected CREATING or LOADING\n", VMGetStateName(pVM->enmVMState)), VERR_VM_INVALID_VM_STATE);
+
+    AssertReturn(!(iPciRegion & UINT16_MAX), VERR_INVALID_PARAMETER); /* not implemented. */
+
+    /** @todo PGMR3PhysMmio2Register mangles the description, move it here and
+     *        use a real string cache. */
+    int rc = PGMR3PhysMmio2Register(pVM, pDevIns, pPciDev ? pPciDev->Int.s.idxDevCfg : 254, iPciRegion >> 16,
+                                    cbRegion, fFlags, pszDesc, ppvMapping, phRegion);
+
+    LogFlow(("pdmR3DevHlp_Mmio2Create: caller='%s'/%d: returns %Rrc *ppvMapping=%p phRegion=%#RX64\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc, *ppvMapping, *phRegion));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMmio2Destroy} */
+static DECLCALLBACK(int) pdmR3DevHlp_Mmio2Destroy(PPDMDEVINS pDevIns, PGMMMIO2HANDLE hRegion)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_Mmio2Destroy: caller='%s'/%d: hRegion=%#RX64\n", pDevIns->pReg->szName, pDevIns->iInstance, hRegion));
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertMsgReturn(   pVM->enmVMState == VMSTATE_DESTROYING
+                    || pVM->enmVMState == VMSTATE_LOADING,
+                    ("state %s, expected DESTROYING or LOADING\n", VMGetStateName(pVM->enmVMState)), VERR_VM_INVALID_VM_STATE);
+
+    int rc = PGMR3PhysMmio2Deregister(pDevIns->Internal.s.pVMR3, pDevIns, hRegion);
+
+    LogFlow(("pdmR3DevHlp_Mmio2Destroy: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMmio2Map} */
+static DECLCALLBACK(int) pdmR3DevHlp_Mmio2Map(PPDMDEVINS pDevIns, PGMMMIO2HANDLE hRegion, RTGCPHYS GCPhys)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_Mmio2Map: caller='%s'/%d: hRegion=%#RX64 GCPhys=%RGp\n", pDevIns->pReg->szName, pDevIns->iInstance, hRegion, GCPhys));
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    int rc = PGMR3PhysMmio2Map(pDevIns->Internal.s.pVMR3, pDevIns, hRegion, GCPhys);
+
+    LogFlow(("pdmR3DevHlp_Mmio2Map: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMmio2Unmap} */
+static DECLCALLBACK(int) pdmR3DevHlp_Mmio2Unmap(PPDMDEVINS pDevIns, PGMMMIO2HANDLE hRegion)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_Mmio2Unmap: caller='%s'/%d: hRegion=%#RX64\n", pDevIns->pReg->szName, pDevIns->iInstance, hRegion));
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    int rc = PGMR3PhysMmio2Unmap(pDevIns->Internal.s.pVMR3, pDevIns, hRegion, NIL_RTGCPHYS);
+
+    LogFlow(("pdmR3DevHlp_Mmio2Unmap: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMmio2Reduce} */
+static DECLCALLBACK(int) pdmR3DevHlp_Mmio2Reduce(PPDMDEVINS pDevIns, PGMMMIO2HANDLE hRegion, RTGCPHYS cbRegion)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_Mmio2Reduce: caller='%s'/%d: hRegion=%#RX64 cbRegion=%RGp\n", pDevIns->pReg->szName, pDevIns->iInstance, hRegion, cbRegion));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_LOADING, VERR_VM_INVALID_VM_STATE);
+
+    int rc = PGMR3PhysMmio2Reduce(pDevIns->Internal.s.pVMR3, pDevIns, hRegion, cbRegion);
+
+    LogFlow(("pdmR3DevHlp_Mmio2Reduce: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMmio2GetMappingAddress} */
+static DECLCALLBACK(RTGCPHYS) pdmR3DevHlp_Mmio2GetMappingAddress(PPDMDEVINS pDevIns, PGMMMIO2HANDLE hRegion)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_Mmio2GetMappingAddress: caller='%s'/%d: hRegion=%#RX6r\n", pDevIns->pReg->szName, pDevIns->iInstance, hRegion));
+    VM_ASSERT_EMT0_RETURN(pVM, NIL_RTGCPHYS);
+
+    RTGCPHYS GCPhys = PGMR3PhysMmio2GetMappingAddress(pVM, pDevIns, hRegion);
+
+    LogFlow(("pdmR3DevHlp_Mmio2GetMappingAddress: caller='%s'/%d: returns %RGp\n", pDevIns->pReg->szName, pDevIns->iInstance, GCPhys));
+    return GCPhys;
+}
+
+/**
+ * @copydoc PDMDEVHLPR3::pfnMmio2ChangeRegionNo
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_Mmio2ChangeRegionNo(PPDMDEVINS pDevIns, PGMMMIO2HANDLE hRegion, uint32_t iNewRegion)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_Mmio2ChangeRegionNo: caller='%s'/%d: hRegion=%#RX6r iNewRegion=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, hRegion, iNewRegion));
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    int rc = PGMR3PhysMmio2ChangeRegionNo(pVM, pDevIns, hRegion, iNewRegion);
+
+    LogFlow(("pdmR3DevHlp_Mmio2ChangeRegionNo: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnROMRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_ROMRegister(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, uint32_t cbRange,
+                                                 const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_ROMRegister: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x pvBinary=%p cbBinary=%#x fFlags=%#RX32 pszDesc=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange, pvBinary, cbBinary, fFlags, pszDesc, pszDesc));
+
+/** @todo can we mangle pszDesc? */
+    int rc = PGMR3PhysRomRegister(pDevIns->Internal.s.pVMR3, pDevIns, GCPhysStart, cbRange, pvBinary, cbBinary, fFlags, pszDesc);
+
+    LogFlow(("pdmR3DevHlp_ROMRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnROMProtectShadow} */
+static DECLCALLBACK(int) pdmR3DevHlp_ROMProtectShadow(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, uint32_t cbRange, PGMROMPROT enmProt)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_ROMProtectShadow: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x enmProt=%d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange, enmProt));
+
+    int rc = PGMR3PhysRomProtect(pDevIns->Internal.s.pVMR3, GCPhysStart, cbRange, enmProt);
+
+    LogFlow(("pdmR3DevHlp_ROMProtectShadow: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSSMRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_SSMRegister(PPDMDEVINS pDevIns, uint32_t uVersion, size_t cbGuess, const char *pszBefore,
+                                                 PFNSSMDEVLIVEPREP pfnLivePrep, PFNSSMDEVLIVEEXEC pfnLiveExec, PFNSSMDEVLIVEVOTE pfnLiveVote,
+                                                 PFNSSMDEVSAVEPREP pfnSavePrep, PFNSSMDEVSAVEEXEC pfnSaveExec, PFNSSMDEVSAVEDONE pfnSaveDone,
+                                                 PFNSSMDEVLOADPREP pfnLoadPrep, PFNSSMDEVLOADEXEC pfnLoadExec, PFNSSMDEVLOADDONE pfnLoadDone)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_SSMRegister: caller='%s'/%d: uVersion=%#x cbGuess=%#x pszBefore=%p:{%s}\n"
+             "    pfnLivePrep=%p pfnLiveExec=%p pfnLiveVote=%p pfnSavePrep=%p pfnSaveExec=%p pfnSaveDone=%p pszLoadPrep=%p pfnLoadExec=%p pfnLoadDone=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uVersion, cbGuess, pszBefore, pszBefore,
+             pfnLivePrep, pfnLiveExec, pfnLiveVote,
+             pfnSavePrep, pfnSaveExec, pfnSaveDone,
+             pfnLoadPrep, pfnLoadExec, pfnLoadDone));
+
+    int rc = SSMR3RegisterDevice(pDevIns->Internal.s.pVMR3, pDevIns, pDevIns->pReg->szName, pDevIns->iInstance,
+                                 uVersion, cbGuess, pszBefore,
+                                 pfnLivePrep, pfnLiveExec, pfnLiveVote,
+                                 pfnSavePrep, pfnSaveExec, pfnSaveDone,
+                                 pfnLoadPrep, pfnLoadExec, pfnLoadDone);
+
+    LogFlow(("pdmR3DevHlp_SSMRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTMTimerCreate} */
+static DECLCALLBACK(int) pdmR3DevHlp_TMTimerCreate(PPDMDEVINS pDevIns, TMCLOCK enmClock, PFNTMTIMERDEV pfnCallback, void *pvUser, uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_TMTimerCreate: caller='%s'/%d: enmClock=%d pfnCallback=%p pvUser=%p fFlags=%#x pszDesc=%p:{%s} ppTimer=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, enmClock, pfnCallback, pvUser, fFlags, pszDesc, pszDesc, ppTimer));
+
+    if (pDevIns->iInstance > 0) /** @todo use a string cache here later. */
+    {
+         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s[%u]", pszDesc, pDevIns->iInstance);
+         if (pszDesc2)
+             pszDesc = pszDesc2;
+    }
+
+    int rc = TMR3TimerCreateDevice(pVM, pDevIns, enmClock, pfnCallback, pvUser, fFlags, pszDesc, ppTimer);
+
+    LogFlow(("pdmR3DevHlp_TMTimerCreate: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerCreate} */
+static DECLCALLBACK(int) pdmR3DevHlp_TimerCreate(PPDMDEVINS pDevIns, TMCLOCK enmClock, PFNTMTIMERDEV pfnCallback,
+                                                 void *pvUser, uint32_t fFlags, const char *pszDesc, PTMTIMERHANDLE phTimer)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_TimerCreate: caller='%s'/%d: enmClock=%d pfnCallback=%p pvUser=%p fFlags=%#x pszDesc=%p:{%s} phTimer=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, enmClock, pfnCallback, pvUser, fFlags, pszDesc, pszDesc, phTimer));
+
+    if (pDevIns->iInstance > 0) /** @todo use a string cache here later. */
+    {
+         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s[%u]", pszDesc, pDevIns->iInstance);
+         if (pszDesc2)
+             pszDesc = pszDesc2;
+    }
+
+    PTMTIMER pTimer = NULL;
+    int rc = TMR3TimerCreateDevice(pVM, pDevIns, enmClock, pfnCallback, pvUser, fFlags, pszDesc, &pTimer);
+    *phTimer = (uintptr_t)pTimer;
+
+    LogFlow(("pdmR3DevHlp_TimerCreate: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerToPtr} */
+static DECLCALLBACK(PTMTIMERR3) pdmR3DevHlp_TimerToPtr(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return (PTMTIMERR3)hTimer;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerFromMicro} */
+static DECLCALLBACK(uint64_t) pdmR3DevHlp_TimerFromMicro(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cMicroSecs)
+{
+    return TMTimerFromMicro(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), cMicroSecs);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerFromMilli} */
+static DECLCALLBACK(uint64_t) pdmR3DevHlp_TimerFromMilli(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cMilliSecs)
+{
+    return TMTimerFromMilli(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), cMilliSecs);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerFromNano} */
+static DECLCALLBACK(uint64_t) pdmR3DevHlp_TimerFromNano(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cNanoSecs)
+{
+    return TMTimerFromNano(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), cNanoSecs);
+}
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerGet} */
+static DECLCALLBACK(uint64_t) pdmR3DevHlp_TimerGet(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMTimerGet(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerGetFreq} */
+static DECLCALLBACK(uint64_t) pdmR3DevHlp_TimerGetFreq(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMTimerGetFreq(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerGetNano} */
+static DECLCALLBACK(uint64_t) pdmR3DevHlp_TimerGetNano(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMTimerGetNano(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerIsActive} */
+static DECLCALLBACK(bool) pdmR3DevHlp_TimerIsActive(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMTimerIsActive(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerIsLockOwner} */
+static DECLCALLBACK(bool) pdmR3DevHlp_TimerIsLockOwner(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMTimerIsLockOwner(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerLockClock} */
+static DECLCALLBACK(VBOXSTRICTRC) pdmR3DevHlp_TimerLockClock(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, int rcBusy)
+{
+    return TMTimerLock(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), rcBusy);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerLockClock2} */
+static DECLCALLBACK(VBOXSTRICTRC) pdmR3DevHlp_TimerLockClock2(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer,
+                                                              PPDMCRITSECT pCritSect, int rcBusy)
+{
+    VBOXSTRICTRC rc = TMTimerLock(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), rcBusy);
+    if (rc == VINF_SUCCESS)
+    {
+        rc = PDMCritSectEnter(pCritSect, rcBusy);
+        if (rc == VINF_SUCCESS)
+            return rc;
+        AssertRC(VBOXSTRICTRC_VAL(rc));
+        TMTimerUnlock(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer));
+    }
+    else
+        AssertRC(VBOXSTRICTRC_VAL(rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerSet} */
+static DECLCALLBACK(int) pdmR3DevHlp_TimerSet(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t uExpire)
+{
+    return TMTimerSet(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), uExpire);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerSetFrequencyHint} */
+static DECLCALLBACK(int) pdmR3DevHlp_TimerSetFrequencyHint(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint32_t uHz)
+{
+    return TMTimerSetFrequencyHint(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), uHz);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerSetMicro} */
+static DECLCALLBACK(int) pdmR3DevHlp_TimerSetMicro(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cMicrosToNext)
+{
+    return TMTimerSetMicro(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), cMicrosToNext);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerSetMillies} */
+static DECLCALLBACK(int) pdmR3DevHlp_TimerSetMillies(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cMilliesToNext)
+{
+    return TMTimerSetMillies(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), cMilliesToNext);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerSetNano} */
+static DECLCALLBACK(int) pdmR3DevHlp_TimerSetNano(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cNanosToNext)
+{
+    return TMTimerSetNano(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), cNanosToNext);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerSetRelative} */
+static DECLCALLBACK(int) pdmR3DevHlp_TimerSetRelative(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, uint64_t cTicksToNext, uint64_t *pu64Now)
+{
+    return TMTimerSetRelative(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), cTicksToNext, pu64Now);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerStop} */
+static DECLCALLBACK(int) pdmR3DevHlp_TimerStop(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMTimerStop(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerUnlockClock} */
+static DECLCALLBACK(void) pdmR3DevHlp_TimerUnlockClock(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    TMTimerUnlock(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerUnlockClock2} */
+static DECLCALLBACK(void) pdmR3DevHlp_TimerUnlockClock2(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, PPDMCRITSECT pCritSect)
+{
+    TMTimerUnlock(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer));
+    int rc = PDMCritSectLeave(pCritSect);
+    AssertRC(rc);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerSetCritSect} */
+static DECLCALLBACK(int) pdmR3DevHlp_TimerSetCritSect(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, PPDMCRITSECT pCritSect)
+{
+    return TMR3TimerSetCritSect(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerSave} */
+static DECLCALLBACK(int) pdmR3DevHlp_TimerSave(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, PSSMHANDLE pSSM)
+{
+    return TMR3TimerSave(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), pSSM);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerLoad} */
+static DECLCALLBACK(int) pdmR3DevHlp_TimerLoad(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer, PSSMHANDLE pSSM)
+{
+    return TMR3TimerLoad(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer), pSSM);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTimerDestroy} */
+static DECLCALLBACK(int) pdmR3DevHlp_TimerDestroy(PPDMDEVINS pDevIns, TMTIMERHANDLE hTimer)
+{
+    return TMR3TimerDestroy(pdmR3DevHlp_TimerToPtr(pDevIns, hTimer));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTMUtcNow} */
+static DECLCALLBACK(PRTTIMESPEC) pdmR3DevHlp_TMUtcNow(PPDMDEVINS pDevIns, PRTTIMESPEC pTime)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_TMUtcNow: caller='%s'/%d: pTime=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pTime));
+
+    pTime = TMR3UtcNow(pDevIns->Internal.s.pVMR3, pTime);
+
+    LogFlow(("pdmR3DevHlp_TMUtcNow: caller='%s'/%d: returns %RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, RTTimeSpecGetNano(pTime)));
+    return pTime;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTMTimeVirtGet} */
+static DECLCALLBACK(uint64_t) pdmR3DevHlp_TMTimeVirtGet(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_TMTimeVirtGet: caller='%s'/%d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    uint64_t u64Time = TMVirtualSyncGet(pDevIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DevHlp_TMTimeVirtGet: caller='%s'/%d: returns %RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, u64Time));
+    return u64Time;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTMTimeVirtGetFreq} */
+static DECLCALLBACK(uint64_t) pdmR3DevHlp_TMTimeVirtGetFreq(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_TMTimeVirtGetFreq: caller='%s'/%d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    uint64_t u64Freq = TMVirtualGetFreq(pDevIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DevHlp_TMTimeVirtGetFreq: caller='%s'/%d: returns %RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, u64Freq));
+    return u64Freq;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTMTimeVirtGetNano} */
+static DECLCALLBACK(uint64_t) pdmR3DevHlp_TMTimeVirtGetNano(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_TMTimeVirtGetNano: caller='%s'/%d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    uint64_t u64Time = TMVirtualGet(pDevIns->Internal.s.pVMR3);
+    uint64_t u64Nano = TMVirtualToNano(pDevIns->Internal.s.pVMR3, u64Time);
+
+    LogFlow(("pdmR3DevHlp_TMTimeVirtGetNano: caller='%s'/%d: returns %RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, u64Nano));
+    return u64Nano;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetSupDrvSession} */
+static DECLCALLBACK(PSUPDRVSESSION) pdmR3DevHlp_GetSupDrvSession(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_GetSupDrvSession: caller='%s'/%d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    PSUPDRVSESSION pSession = pDevIns->Internal.s.pVMR3->pSession;
+
+    LogFlow(("pdmR3DevHlp_GetSupDrvSession: caller='%s'/%d: returns %#p\n", pDevIns->pReg->szName, pDevIns->iInstance, pSession));
+    return pSession;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnQueryGenericUserObject} */
+static DECLCALLBACK(void *) pdmR3DevHlp_QueryGenericUserObject(PPDMDEVINS pDevIns, PCRTUUID pUuid)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_QueryGenericUserObject: caller='%s'/%d: pUuid=%p:%RTuuid\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pUuid, pUuid));
+
+#if defined(DEBUG_bird) || defined(DEBUG_ramshankar) || defined(DEBUG_sunlover) || defined(DEBUG_michael) || defined(DEBUG_andy)
+    AssertMsgFailed(("'%s' wants %RTuuid - external only interface!\n", pDevIns->pReg->szName, pUuid));
+#endif
+
+    void *pvRet;
+    PUVM  pUVM = pDevIns->Internal.s.pVMR3->pUVM;
+    if (pUVM->pVmm2UserMethods->pfnQueryGenericObject)
+        pvRet = pUVM->pVmm2UserMethods->pfnQueryGenericObject(pUVM->pVmm2UserMethods, pUVM, pUuid);
+    else
+        pvRet = NULL;
+
+    LogRel(("pdmR3DevHlp_QueryGenericUserObject: caller='%s'/%d: returns %#p for %RTuuid\n",
+            pDevIns->pReg->szName, pDevIns->iInstance, pvRet, pUuid));
+    return pvRet;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysRead} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysRead(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysRead: caller='%s'/%d: GCPhys=%RGp pvBuf=%p cbRead=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, pvBuf, cbRead));
+
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    if (!VM_IS_EMT(pVM))
+    {
+        char szNames[128];
+        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
+        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
+    }
+#endif
+
+    VBOXSTRICTRC rcStrict;
+    if (VM_IS_EMT(pVM))
+        rcStrict = PGMPhysRead(pVM, GCPhys, pvBuf, cbRead, PGMACCESSORIGIN_DEVICE);
+    else
+        rcStrict = PGMR3PhysReadExternal(pVM, GCPhys, pvBuf, cbRead, PGMACCESSORIGIN_DEVICE);
+    AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); /** @todo track down the users for this bugger. */
+
+    Log(("pdmR3DevHlp_PhysRead: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VBOXSTRICTRC_VAL(rcStrict) ));
+    return VBOXSTRICTRC_VAL(rcStrict);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysWrite} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysWrite(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysWrite: caller='%s'/%d: GCPhys=%RGp pvBuf=%p cbWrite=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, pvBuf, cbWrite));
+
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    if (!VM_IS_EMT(pVM))
+    {
+        char szNames[128];
+        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
+        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
+    }
+#endif
+
+    VBOXSTRICTRC rcStrict;
+    if (VM_IS_EMT(pVM))
+        rcStrict = PGMPhysWrite(pVM, GCPhys, pvBuf, cbWrite, PGMACCESSORIGIN_DEVICE);
+    else
+        rcStrict = PGMR3PhysWriteExternal(pVM, GCPhys, pvBuf, cbWrite, PGMACCESSORIGIN_DEVICE);
+    AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); /** @todo track down the users for this bugger. */
+
+    Log(("pdmR3DevHlp_PhysWrite: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VBOXSTRICTRC_VAL(rcStrict) ));
+    return VBOXSTRICTRC_VAL(rcStrict);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysGCPhys2CCPtr} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysGCPhys2CCPtr(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint32_t fFlags, void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysGCPhys2CCPtr: caller='%s'/%d: GCPhys=%RGp fFlags=%#x ppv=%p pLock=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, fFlags, ppv, pLock));
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    if (!VM_IS_EMT(pVM))
+    {
+        char szNames[128];
+        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
+        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
+    }
+#endif
+
+    int rc = PGMR3PhysGCPhys2CCPtrExternal(pVM, GCPhys, ppv, pLock);
+
+    Log(("pdmR3DevHlp_PhysGCPhys2CCPtr: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysGCPhys2CCPtrReadOnly} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint32_t fFlags, const void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly: caller='%s'/%d: GCPhys=%RGp fFlags=%#x ppv=%p pLock=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, fFlags, ppv, pLock));
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    if (!VM_IS_EMT(pVM))
+    {
+        char szNames[128];
+        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
+        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
+    }
+#endif
+
+    int rc = PGMR3PhysGCPhys2CCPtrReadOnlyExternal(pVM, GCPhys, ppv, pLock);
+
+    Log(("pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysReleasePageMappingLock} */
+static DECLCALLBACK(void) pdmR3DevHlp_PhysReleasePageMappingLock(PPDMDEVINS pDevIns, PPGMPAGEMAPLOCK pLock)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysReleasePageMappingLock: caller='%s'/%d: pLock=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pLock));
+
+    PGMPhysReleasePageMappingLock(pVM, pLock);
+
+    Log(("pdmR3DevHlp_PhysReleasePageMappingLock: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysBulkGCPhys2CCPtr} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysBulkGCPhys2CCPtr(PPDMDEVINS pDevIns, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
+                                                          uint32_t fFlags, void **papvPages, PPGMPAGEMAPLOCK paLocks)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysBulkGCPhys2CCPtr: caller='%s'/%d: cPages=%#x paGCPhysPages=%p (%RGp,..) fFlags=%#x papvPages=%p paLocks=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, cPages, paGCPhysPages, paGCPhysPages[0], fFlags, papvPages, paLocks));
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(cPages > 0, VERR_INVALID_PARAMETER);
+
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    if (!VM_IS_EMT(pVM))
+    {
+        char szNames[128];
+        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
+        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
+    }
+#endif
+
+    int rc = PGMR3PhysBulkGCPhys2CCPtrExternal(pVM, cPages, paGCPhysPages, papvPages, paLocks);
+
+    Log(("pdmR3DevHlp_PhysBulkGCPhys2CCPtr: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysBulkGCPhys2CCPtrReadOnly} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysBulkGCPhys2CCPtrReadOnly(PPDMDEVINS pDevIns, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
+                                                                  uint32_t fFlags, const void **papvPages, PPGMPAGEMAPLOCK paLocks)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysBulkGCPhys2CCPtrReadOnly: caller='%s'/%d: cPages=%#x paGCPhysPages=%p (%RGp,...) fFlags=%#x papvPages=%p paLocks=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, cPages, paGCPhysPages, paGCPhysPages[0], fFlags, papvPages, paLocks));
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(cPages > 0, VERR_INVALID_PARAMETER);
+
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    if (!VM_IS_EMT(pVM))
+    {
+        char szNames[128];
+        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
+        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
+    }
+#endif
+
+    int rc = PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal(pVM, cPages, paGCPhysPages, papvPages, paLocks);
+
+    Log(("pdmR3DevHlp_PhysBulkGCPhys2CCPtrReadOnly: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysBulkReleasePageMappingLocks} */
+static DECLCALLBACK(void) pdmR3DevHlp_PhysBulkReleasePageMappingLocks(PPDMDEVINS pDevIns, uint32_t cPages, PPGMPAGEMAPLOCK paLocks)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysBulkReleasePageMappingLocks: caller='%s'/%d: cPages=%#x paLocks=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, cPages, paLocks));
+    Assert(cPages > 0);
+
+    PGMPhysBulkReleasePageMappingLocks(pVM, cPages, paLocks);
+
+    Log(("pdmR3DevHlp_PhysBulkReleasePageMappingLocks: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysReadGCVirt} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysReadGCVirt(PPDMDEVINS pDevIns, void *pvDst, RTGCPTR GCVirtSrc, size_t cb)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_PhysReadGCVirt: caller='%s'/%d: pvDst=%p GCVirt=%RGv cb=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pvDst, GCVirtSrc, cb));
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        return VERR_ACCESS_DENIED;
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    /** @todo SMP. */
+#endif
+
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, pvDst, GCVirtSrc, cb);
+
+    LogFlow(("pdmR3DevHlp_PhysReadGCVirt: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysWriteGCVirt} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysWriteGCVirt(PPDMDEVINS pDevIns, RTGCPTR GCVirtDst, const void *pvSrc, size_t cb)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_PhysWriteGCVirt: caller='%s'/%d: GCVirtDst=%RGv pvSrc=%p cb=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCVirtDst, pvSrc, cb));
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        return VERR_ACCESS_DENIED;
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    /** @todo SMP. */
+#endif
+
+    int rc = PGMPhysSimpleWriteGCPtr(pVCpu, GCVirtDst, pvSrc, cb);
+
+    LogFlow(("pdmR3DevHlp_PhysWriteGCVirt: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysGCPtr2GCPhys} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysGCPtr2GCPhys(PPDMDEVINS pDevIns, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_PhysGCPtr2GCPhys: caller='%s'/%d: GCPtr=%RGv pGCPhys=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPtr, pGCPhys));
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        return VERR_ACCESS_DENIED;
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    /** @todo SMP. */
+#endif
+
+    int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, pGCPhys);
+
+    LogFlow(("pdmR3DevHlp_PhysGCPtr2GCPhys: caller='%s'/%d: returns %Rrc *pGCPhys=%RGp\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *pGCPhys));
+
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMMHeapAlloc} */
+static DECLCALLBACK(void *) pdmR3DevHlp_MMHeapAlloc(PPDMDEVINS pDevIns, size_t cb)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_MMHeapAlloc: caller='%s'/%d: cb=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, cb));
+
+    void *pv = MMR3HeapAlloc(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE_USER, cb);
+
+    LogFlow(("pdmR3DevHlp_MMHeapAlloc: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pv));
+    return pv;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMMHeapAllocZ} */
+static DECLCALLBACK(void *) pdmR3DevHlp_MMHeapAllocZ(PPDMDEVINS pDevIns, size_t cb)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_MMHeapAllocZ: caller='%s'/%d: cb=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, cb));
+
+    void *pv = MMR3HeapAllocZ(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE_USER, cb);
+
+    LogFlow(("pdmR3DevHlp_MMHeapAllocZ: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pv));
+    return pv;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMMHeapFree} */
+static DECLCALLBACK(void) pdmR3DevHlp_MMHeapFree(PPDMDEVINS pDevIns, void *pv)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns); RT_NOREF_PV(pDevIns);
+    LogFlow(("pdmR3DevHlp_MMHeapFree: caller='%s'/%d: pv=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pv));
+
+    MMR3HeapFree(pv);
+
+    LogFlow(("pdmR3DevHlp_MMHeapAlloc: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMState} */
+static DECLCALLBACK(VMSTATE) pdmR3DevHlp_VMState(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+
+    VMSTATE enmVMState = VMR3GetState(pDevIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DevHlp_VMState: caller='%s'/%d: returns %d (%s)\n", pDevIns->pReg->szName, pDevIns->iInstance,
+             enmVMState, VMR3GetStateName(enmVMState)));
+    return enmVMState;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMTeleportedAndNotFullyResumedYet} */
+static DECLCALLBACK(bool) pdmR3DevHlp_VMTeleportedAndNotFullyResumedYet(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+
+    bool fRc = VMR3TeleportedAndNotFullyResumedYet(pDevIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DevHlp_VMState: caller='%s'/%d: returns %RTbool\n", pDevIns->pReg->szName, pDevIns->iInstance,
+             fRc));
+    return fRc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSetError} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSetError(PPDMDEVINS pDevIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    va_list args;
+    va_start(args, pszFormat);
+    int rc2 = VMSetErrorV(pDevIns->Internal.s.pVMR3, rc, RT_SRC_POS_ARGS, pszFormat, args); Assert(rc2 == rc); NOREF(rc2);
+    va_end(args);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSetErrorV} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSetErrorV(PPDMDEVINS pDevIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    int rc2 = VMSetErrorV(pDevIns->Internal.s.pVMR3, rc, RT_SRC_POS_ARGS, pszFormat, va); Assert(rc2 == rc); NOREF(rc2);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSetRuntimeError} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSetRuntimeError(PPDMDEVINS pDevIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    va_list args;
+    va_start(args, pszFormat);
+    int rc = VMSetRuntimeErrorV(pDevIns->Internal.s.pVMR3, fFlags, pszErrorId, pszFormat, args);
+    va_end(args);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSetRuntimeErrorV} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSetRuntimeErrorV(PPDMDEVINS pDevIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    int rc = VMSetRuntimeErrorV(pDevIns->Internal.s.pVMR3, fFlags, pszErrorId, pszFormat, va);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDBGFStopV} */
+static DECLCALLBACK(int) pdmR3DevHlp_DBGFStopV(PPDMDEVINS pDevIns, const char *pszFile, unsigned iLine, const char *pszFunction, const char *pszFormat, va_list args)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+#ifdef LOG_ENABLED
+    va_list va2;
+    va_copy(va2, args);
+    LogFlow(("pdmR3DevHlp_DBGFStopV: caller='%s'/%d: pszFile=%p:{%s} iLine=%d pszFunction=%p:{%s} pszFormat=%p:{%s} (%N)\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pszFile, pszFile, iLine, pszFunction, pszFunction, pszFormat, pszFormat, pszFormat, &va2));
+    va_end(va2);
+#endif
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    int rc = DBGFR3EventSrcV(pVM, DBGFEVENT_DEV_STOP, pszFile, iLine, pszFunction, pszFormat, args);
+    if (rc == VERR_DBGF_NOT_ATTACHED)
+        rc = VINF_SUCCESS;
+
+    LogFlow(("pdmR3DevHlp_DBGFStopV: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDBGFInfoRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_DBGFInfoRegister(PPDMDEVINS pDevIns, const char *pszName, const char *pszDesc, PFNDBGFHANDLERDEV pfnHandler)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_DBGFInfoRegister: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pszName, pszName, pszDesc, pszDesc, pfnHandler));
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    int rc = DBGFR3InfoRegisterDevice(pVM, pszName, pszDesc, pfnHandler, pDevIns);
+
+    LogFlow(("pdmR3DevHlp_DBGFInfoRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDBGFInfoRegisterArgv} */
+static DECLCALLBACK(int) pdmR3DevHlp_DBGFInfoRegisterArgv(PPDMDEVINS pDevIns, const char *pszName, const char *pszDesc, PFNDBGFINFOARGVDEV pfnHandler)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_DBGFInfoRegisterArgv: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pszName, pszName, pszDesc, pszDesc, pfnHandler));
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    int rc = DBGFR3InfoRegisterDeviceArgv(pVM, pszName, pszDesc, pfnHandler, pDevIns);
+
+    LogFlow(("pdmR3DevHlp_DBGFInfoRegisterArgv: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDBGFRegRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_DBGFRegRegister(PPDMDEVINS pDevIns, PCDBGFREGDESC paRegisters)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_DBGFRegRegister: caller='%s'/%d: paRegisters=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, paRegisters));
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    int rc = DBGFR3RegRegisterDevice(pVM, paRegisters, pDevIns, pDevIns->pReg->szName, pDevIns->iInstance);
+
+    LogFlow(("pdmR3DevHlp_DBGFRegRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDBGFTraceBuf} */
+static DECLCALLBACK(RTTRACEBUF) pdmR3DevHlp_DBGFTraceBuf(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RTTRACEBUF hTraceBuf = pDevIns->Internal.s.pVMR3->hTraceBufR3;
+    LogFlow(("pdmR3DevHlp_DBGFTraceBuf: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, hTraceBuf));
+    return hTraceBuf;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSTAMRegister} */
+static DECLCALLBACK(void) pdmR3DevHlp_STAMRegister(PPDMDEVINS pDevIns, void *pvSample, STAMTYPE enmType, const char *pszName,
+                                                   STAMUNIT enmUnit, const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    int rc;
+    if (*pszName == '/')
+        rc = STAMR3Register(pVM, pvSample, enmType, STAMVISIBILITY_ALWAYS, pszName, enmUnit, pszDesc);
+    /* Provide default device statistics prefix: */
+    else if (pDevIns->pReg->cMaxInstances == 1)
+        rc = STAMR3RegisterF(pVM, pvSample, enmType, STAMVISIBILITY_ALWAYS, enmUnit, pszDesc,
+                             "/Devices/%s/%s", pDevIns->pReg->szName, pszName);
+    else
+        rc = STAMR3RegisterF(pVM, pvSample, enmType, STAMVISIBILITY_ALWAYS, enmUnit, pszDesc,
+                             "/Devices/%s#%u/%s", pDevIns->pReg->szName, pDevIns->iInstance, pszName);
+    AssertRC(rc);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSTAMRegisterV} */
+static DECLCALLBACK(void) pdmR3DevHlp_STAMRegisterV(PPDMDEVINS pDevIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, va_list args)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    int rc;
+    if (*pszName == '/')
+        rc = STAMR3RegisterV(pVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    else
+    {
+        /* Provide default device statistics prefix: */
+        va_list vaCopy;
+        va_copy(vaCopy, args);
+        if (pDevIns->pReg->cMaxInstances == 1)
+            rc = STAMR3RegisterF(pVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc,
+                                 "/Devices/%s/%N", pDevIns->pReg->szName, pszName, &vaCopy);
+        else
+            rc = STAMR3RegisterF(pVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc,
+                                 "/Devices/%s#%u/%N", pDevIns->pReg->szName, pDevIns->iInstance, pszName, &vaCopy);
+        va_end(vaCopy);
+    }
+    AssertRC(rc);
+}
+
+
+/**
+ * @interface_method_impl{PDMDEVHLPR3,pfnPCIRegister}
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_PCIRegister(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, uint32_t fFlags,
+                                                 uint8_t uPciDevNo, uint8_t uPciFunNo, const char *pszName)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: pPciDev=%p:{.config={%#.256Rhxs} fFlags=%#x uPciDevNo=%#x uPciFunNo=%#x pszName=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPciDev, pPciDev->abConfig, fFlags, uPciDevNo, uPciFunNo, pszName, pszName ? pszName : ""));
+
+    /*
+     * Validate input.
+     */
+    AssertLogRelMsgReturn(pDevIns->pReg->cMaxPciDevices > 0,
+                          ("'%s'/%d: cMaxPciDevices is 0\n", pDevIns->pReg->szName, pDevIns->iInstance),
+                          VERR_WRONG_ORDER);
+    AssertLogRelMsgReturn(RT_VALID_PTR(pPciDev),
+                          ("'%s'/%d: Invalid pPciDev value: %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pPciDev),
+                          VERR_INVALID_POINTER);
+    AssertLogRelMsgReturn(PDMPciDevGetVendorId(pPciDev),
+                          ("'%s'/%d: Vendor ID is not set!\n", pDevIns->pReg->szName, pDevIns->iInstance),
+                          VERR_INVALID_POINTER);
+    AssertLogRelMsgReturn(   uPciDevNo < 32
+                          || uPciDevNo == PDMPCIDEVREG_DEV_NO_FIRST_UNUSED
+                          || uPciDevNo == PDMPCIDEVREG_DEV_NO_SAME_AS_PREV,
+                          ("'%s'/%d: Invalid PCI device number: %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, uPciDevNo),
+                          VERR_INVALID_PARAMETER);
+    AssertLogRelMsgReturn(   uPciFunNo < 8
+                          || uPciFunNo == PDMPCIDEVREG_FUN_NO_FIRST_UNUSED,
+                          ("'%s'/%d: Invalid PCI funcion number: %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, uPciFunNo),
+                          VERR_INVALID_PARAMETER);
+    AssertLogRelMsgReturn(!(fFlags & ~PDMPCIDEVREG_F_VALID_MASK),
+                          ("'%s'/%d: Invalid flags: %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, fFlags),
+                          VERR_INVALID_FLAGS);
+    if (!pszName)
+        pszName = pDevIns->pReg->szName;
+    AssertLogRelReturn(RT_VALID_PTR(pszName), VERR_INVALID_POINTER);
+    AssertLogRelReturn(!pPciDev->Int.s.fRegistered, VERR_PDM_NOT_PCI_DEVICE);
+    AssertLogRelReturn(pPciDev == PDMDEV_GET_PPCIDEV(pDevIns, pPciDev->Int.s.idxSubDev), VERR_PDM_NOT_PCI_DEVICE);
+    AssertLogRelReturn(pPciDev == PDMDEV_CALC_PPCIDEV(pDevIns, pPciDev->Int.s.idxSubDev), VERR_PDM_NOT_PCI_DEVICE);
+    AssertMsgReturn(pPciDev->u32Magic == PDMPCIDEV_MAGIC, ("%#x\n", pPciDev->u32Magic), VERR_PDM_NOT_PCI_DEVICE);
+
+    /*
+     * Check the registration order - must be following PDMDEVINSR3::apPciDevs.
+     */
+    PPDMPCIDEV const pPrevPciDev = pPciDev->Int.s.idxSubDev == 0 ? NULL
+                                 : PDMDEV_GET_PPCIDEV(pDevIns, pPciDev->Int.s.idxSubDev - 1);
+    if (pPrevPciDev)
+    {
+        AssertLogRelReturn(pPrevPciDev->u32Magic == PDMPCIDEV_MAGIC, VERR_INVALID_MAGIC);
+        AssertLogRelReturn(pPrevPciDev->Int.s.fRegistered, VERR_WRONG_ORDER);
+    }
+
+    /*
+     * Resolve the PCI configuration node for the device.  The default (zero'th)
+     * is the same as the PDM device, the rest are "PciCfg1..255" CFGM sub-nodes.
+     */
+    PCFGMNODE pCfg = pDevIns->Internal.s.pCfgHandle;
+    if (pPciDev->Int.s.idxSubDev > 0)
+        pCfg = CFGMR3GetChildF(pDevIns->Internal.s.pCfgHandle, "PciCfg%u", pPciDev->Int.s.idxSubDev);
+
+    /*
+     * We resolve PDMPCIDEVREG_DEV_NO_SAME_AS_PREV, the PCI bus handles
+     * PDMPCIDEVREG_DEV_NO_FIRST_UNUSED and PDMPCIDEVREG_FUN_NO_FIRST_UNUSED.
+     */
+    uint8_t const uPciDevNoRaw = uPciDevNo;
+    uint32_t      uDefPciBusNo = 0;
+    if (uPciDevNo == PDMPCIDEVREG_DEV_NO_SAME_AS_PREV)
+    {
+        if (pPrevPciDev)
+        {
+            uPciDevNo    = pPrevPciDev->uDevFn >> 3;
+            uDefPciBusNo = pPrevPciDev->Int.s.idxPdmBus;
+        }
+        else
+        {
+            /* Look for PCI device registered with an earlier device instance so we can more
+               easily have multiple functions spanning multiple PDM device instances. */
+            PPDMDEVINS pPrevIns = pDevIns->Internal.s.pDevR3->pInstances;
+            for (;;)
+            {
+                AssertLogRelMsgReturn(pPrevIns && pPrevIns != pDevIns,
+                                      ("'%s'/%d: Can't use PDMPCIDEVREG_DEV_NO_SAME_AS_PREV without a previously registered PCI device by the same or earlier PDM device instance!\n",
+                                       pDevIns->pReg->szName, pDevIns->iInstance), VERR_WRONG_ORDER);
+                if (pPrevIns->Internal.s.pNextR3 == pDevIns)
+                    break;
+                pPrevIns = pPrevIns->Internal.s.pNextR3;
+            }
+
+            PPDMPCIDEV pOtherPciDev = PDMDEV_GET_PPCIDEV(pPrevIns, 0);
+            AssertLogRelMsgReturn(pOtherPciDev && pOtherPciDev->Int.s.fRegistered,
+                                  ("'%s'/%d: Can't use PDMPCIDEVREG_DEV_NO_SAME_AS_PREV without a previously registered PCI device by the same or earlier PDM device instance!\n",
+                                   pDevIns->pReg->szName, pDevIns->iInstance),
+                                  VERR_WRONG_ORDER);
+            for (uint32_t iPrevPciDev = 1; iPrevPciDev < pDevIns->cPciDevs; iPrevPciDev++)
+            {
+                PPDMPCIDEV pCur = PDMDEV_GET_PPCIDEV(pPrevIns, iPrevPciDev);
+                AssertBreak(pCur);
+                if (!pCur->Int.s.fRegistered)
+                    break;
+                pOtherPciDev = pCur;
+            }
+
+            uPciDevNo    = pOtherPciDev->uDevFn >> 3;
+            uDefPciBusNo = pOtherPciDev->Int.s.idxPdmBus;
+        }
+    }
+
+    /*
+     * Choose the PCI bus for the device.
+     *
+     * This is simple. If the device was configured for a particular bus, the PCIBusNo
+     * configuration value will be set. If not the default bus is 0.
+     */
+    /** @cfgm{/Devices/NAME/XX/[PciCfgYY/]PCIBusNo, uint8_t, 0, 7, 0}
+     * Selects the PCI bus number of a device.  The default value isn't necessarily
+     * zero if the device is registered using PDMPCIDEVREG_DEV_NO_SAME_AS_PREV, it
+     * will then also inherit the bus number from the previously registered device.
+     */
+    uint8_t u8Bus;
+    int rc = CFGMR3QueryU8Def(pCfg, "PCIBusNo", &u8Bus, (uint8_t)uDefPciBusNo);
+    AssertLogRelMsgRCReturn(rc, ("Configuration error: PCIBusNo query failed with rc=%Rrc (%s/%d)\n",
+                                 rc, pDevIns->pReg->szName, pDevIns->iInstance), rc);
+    AssertLogRelMsgReturn(u8Bus < RT_ELEMENTS(pVM->pdm.s.aPciBuses),
+                          ("Configuration error: PCIBusNo=%d, max is %d. (%s/%d)\n", u8Bus,
+                           RT_ELEMENTS(pVM->pdm.s.aPciBuses), pDevIns->pReg->szName, pDevIns->iInstance),
+                          VERR_PDM_NO_PCI_BUS);
+    pPciDev->Int.s.idxPdmBus = u8Bus;
+    PPDMPCIBUS pBus = &pVM->pdm.s.aPciBuses[u8Bus];
+    if (pBus->pDevInsR3)
+    {
+        /*
+         * Check the configuration for PCI device and function assignment.
+         */
+        /** @cfgm{/Devices/NAME/XX/[PciCfgYY/]PCIDeviceNo, uint8_t, 0, 31}
+         * Overrides the default PCI device number of a device.
+         */
+        uint8_t uCfgDevice;
+        rc = CFGMR3QueryU8(pCfg, "PCIDeviceNo", &uCfgDevice);
+        if (RT_SUCCESS(rc))
+        {
+            AssertMsgReturn(uCfgDevice <= 31,
+                            ("Configuration error: PCIDeviceNo=%d, max is 31. (%s/%d/%d)\n",
+                             uCfgDevice, pDevIns->pReg->szName, pDevIns->iInstance, pPciDev->Int.s.idxSubDev),
+                            VERR_PDM_BAD_PCI_CONFIG);
+            uPciDevNo = uCfgDevice;
+        }
+        else
+            AssertMsgReturn(rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT,
+                            ("Configuration error: PCIDeviceNo query failed with rc=%Rrc (%s/%d/%d)\n",
+                             rc, pDevIns->pReg->szName, pDevIns->iInstance, pPciDev->Int.s.idxSubDev),
+                            rc);
+
+        /** @cfgm{/Devices/NAME/XX/[PciCfgYY/]PCIFunctionNo, uint8_t, 0, 7}
+         * Overrides the default PCI function number of a device.
+         */
+        uint8_t uCfgFunction;
+        rc = CFGMR3QueryU8(pCfg, "PCIFunctionNo", &uCfgFunction);
+        if (RT_SUCCESS(rc))
+        {
+            AssertMsgReturn(uCfgFunction <= 7,
+                            ("Configuration error: PCIFunctionNo=%#x, max is 7. (%s/%d/%d)\n",
+                             uCfgFunction, pDevIns->pReg->szName, pDevIns->iInstance, pPciDev->Int.s.idxSubDev),
+                            VERR_PDM_BAD_PCI_CONFIG);
+            uPciFunNo = uCfgFunction;
+        }
+        else
+            AssertMsgReturn(rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT,
+                            ("Configuration error: PCIFunctionNo query failed with rc=%Rrc (%s/%d/%d)\n",
+                             rc, pDevIns->pReg->szName, pDevIns->iInstance, pPciDev->Int.s.idxSubDev),
+                            rc);
+
+
+        /*
+         * Initialize the internal data.  We only do the wipe and the members
+         * owned by PDM, the PCI bus does the rest in the registration call.
+         */
+        RT_ZERO(pPciDev->Int);
+
+        pPciDev->Int.s.idxDevCfg = pPciDev->Int.s.idxSubDev;
+        pPciDev->Int.s.fReassignableDevNo = uPciDevNoRaw >= VBOX_PCI_MAX_DEVICES;
+        pPciDev->Int.s.fReassignableFunNo = uPciFunNo >= VBOX_PCI_MAX_FUNCTIONS;
+        pPciDev->Int.s.pDevInsR3 = pDevIns;
+        pPciDev->Int.s.idxPdmBus = u8Bus;
+        pPciDev->Int.s.fRegistered = true;
+
+        /* Set some of the public members too. */
+        pPciDev->pszNameR3 = pszName;
+
+        /*
+         * Call the pci bus device to do the actual registration.
+         */
+        pdmLock(pVM);
+        rc = pBus->pfnRegister(pBus->pDevInsR3, pPciDev, fFlags, uPciDevNo, uPciFunNo, pszName);
+        pdmUnlock(pVM);
+        if (RT_SUCCESS(rc))
+            Log(("PDM: Registered device '%s'/%d as PCI device %d on bus %d\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, pPciDev->uDevFn, pBus->iBus));
+        else
+            pPciDev->Int.s.fRegistered = false;
+    }
+    else
+    {
+        AssertLogRelMsgFailed(("Configuration error: No PCI bus available. This could be related to init order too!\n"));
+        rc = VERR_PDM_NO_PCI_BUS;
+    }
+
+    LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCIRegisterMsi} */
+static DECLCALLBACK(int) pdmR3DevHlp_PCIRegisterMsi(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, PPDMMSIREG pMsiReg)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    if (!pPciDev) /* NULL is an alias for the default PCI device. */
+        pPciDev = pDevIns->apPciDevs[0];
+    AssertReturn(pPciDev, VERR_PDM_NOT_PCI_DEVICE);
+    LogFlow(("pdmR3DevHlp_PCIRegisterMsi: caller='%s'/%d: pPciDev=%p:{%#x} pMsgReg=%p:{cMsiVectors=%d, cMsixVectors=%d}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPciDev, pPciDev->uDevFn, pMsiReg, pMsiReg->cMsiVectors, pMsiReg->cMsixVectors));
+    PDMPCIDEV_ASSERT_VALID_RET(pDevIns, pPciDev);
+
+    AssertLogRelMsgReturn(pDevIns->pReg->cMaxPciDevices > 0,
+                          ("'%s'/%d: cMaxPciDevices is 0\n", pDevIns->pReg->szName, pDevIns->iInstance),
+                          VERR_WRONG_ORDER);
+    AssertLogRelMsgReturn(pMsiReg->cMsixVectors <= pDevIns->pReg->cMaxMsixVectors,
+                          ("'%s'/%d: cMsixVectors=%u cMaxMsixVectors=%u\n",
+                           pDevIns->pReg->szName, pDevIns->iInstance, pMsiReg->cMsixVectors, pDevIns->pReg->cMaxMsixVectors),
+                          VERR_INVALID_FLAGS);
+
+    PVM             pVM    = pDevIns->Internal.s.pVMR3;
+    size_t const    idxBus = pPciDev->Int.s.idxPdmBus;
+    AssertReturn(idxBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses), VERR_WRONG_ORDER);
+    PPDMPCIBUS      pBus   = &pVM->pdm.s.aPciBuses[idxBus];
+
+    pdmLock(pVM);
+    int rc;
+    if (pBus->pfnRegisterMsi)
+        rc = pBus->pfnRegisterMsi(pBus->pDevInsR3, pPciDev, pMsiReg);
+    else
+        rc = VERR_NOT_IMPLEMENTED;
+    pdmUnlock(pVM);
+
+    LogFlow(("pdmR3DevHlp_PCIRegisterMsi: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCIIORegionRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_PCIIORegionRegister(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, uint32_t iRegion,
+                                                         RTGCPHYS cbRegion, PCIADDRESSSPACE enmType, uint32_t fFlags,
+                                                         uint64_t hHandle, PFNPCIIOREGIONMAP pfnMapUnmap)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    if (!pPciDev) /* NULL is an alias for the default PCI device. */
+        pPciDev = pDevIns->apPciDevs[0];
+    AssertReturn(pPciDev, VERR_PDM_NOT_PCI_DEVICE);
+    LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: pPciDev=%p:{%#x} iRegion=%d cbRegion=%RGp enmType=%d fFlags=%#x, hHandle=%#RX64 pfnMapUnmap=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPciDev, pPciDev->uDevFn, iRegion, cbRegion, enmType, fFlags, hHandle, pfnMapUnmap));
+    PDMPCIDEV_ASSERT_VALID_RET(pDevIns, pPciDev);
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertLogRelMsgReturn(VMR3GetState(pVM) == VMSTATE_CREATING,
+                          ("caller='%s'/%d: %s\n", pDevIns->pReg->szName, pDevIns->iInstance, VMR3GetStateName(VMR3GetState(pVM))),
+                          VERR_WRONG_ORDER);
+
+    if (iRegion >= VBOX_PCI_NUM_REGIONS)
+    {
+        Assert(iRegion < VBOX_PCI_NUM_REGIONS);
+        LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: returns %Rrc (iRegion)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    switch ((int)enmType)
+    {
+        case PCI_ADDRESS_SPACE_IO:
+            /*
+             * Sanity check: don't allow to register more than 32K of the PCI I/O space.
+             */
+            AssertLogRelMsgReturn(cbRegion <= _32K,
+                                  ("caller='%s'/%d: %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, cbRegion),
+                                  VERR_INVALID_PARAMETER);
+            break;
+
+        case PCI_ADDRESS_SPACE_MEM:
+        case PCI_ADDRESS_SPACE_MEM_PREFETCH:
+            /*
+             * Sanity check: Don't allow to register more than 2GB of the PCI MMIO space.
+             */
+            AssertLogRelMsgReturn(cbRegion <= MM_MMIO_32_MAX,
+                                  ("caller='%s'/%d: %RGp (max %RGp)\n",
+                                   pDevIns->pReg->szName, pDevIns->iInstance, cbRegion, (RTGCPHYS)MM_MMIO_32_MAX),
+                                  VERR_OUT_OF_RANGE);
+            break;
+
+        case PCI_ADDRESS_SPACE_BAR64 | PCI_ADDRESS_SPACE_MEM:
+        case PCI_ADDRESS_SPACE_BAR64 | PCI_ADDRESS_SPACE_MEM_PREFETCH:
+            /*
+             * Sanity check: Don't allow to register more than 64GB of the 64-bit PCI MMIO space.
+             */
+            AssertLogRelMsgReturn(cbRegion <= MM_MMIO_64_MAX,
+                                  ("caller='%s'/%d: %RGp (max %RGp)\n",
+                                   pDevIns->pReg->szName, pDevIns->iInstance, cbRegion, MM_MMIO_64_MAX),
+                                  VERR_OUT_OF_RANGE);
+            break;
+
+        default:
+            AssertMsgFailed(("enmType=%#x is unknown\n", enmType));
+            LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: returns %Rrc (enmType)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+            return VERR_INVALID_PARAMETER;
+    }
+
+    AssertMsgReturn(   pfnMapUnmap
+                    || (   hHandle != UINT64_MAX
+                        && (fFlags & PDMPCIDEV_IORGN_F_HANDLE_MASK) != PDMPCIDEV_IORGN_F_NO_HANDLE),
+                    ("caller='%s'/%d: fFlags=%#x hHandle=%#RX64\n", pDevIns->pReg->szName, pDevIns->iInstance, fFlags, hHandle),
+                    VERR_INVALID_PARAMETER);
+
+    AssertMsgReturn(!(fFlags & ~PDMPCIDEV_IORGN_F_VALID_MASK), ("fFlags=%#x\n", fFlags), VERR_INVALID_FLAGS);
+    int rc;
+    switch (fFlags & PDMPCIDEV_IORGN_F_HANDLE_MASK)
+    {
+        case PDMPCIDEV_IORGN_F_NO_HANDLE:
+            break;
+        case PDMPCIDEV_IORGN_F_IOPORT_HANDLE:
+            AssertReturn(enmType == PCI_ADDRESS_SPACE_IO, VERR_INVALID_FLAGS);
+            rc = IOMR3IoPortValidateHandle(pVM, pDevIns, (IOMIOPORTHANDLE)hHandle);
+            AssertRCReturn(rc, rc);
+            break;
+        case PDMPCIDEV_IORGN_F_MMIO_HANDLE:
+            AssertReturn(   (enmType & ~PCI_ADDRESS_SPACE_BAR64) == PCI_ADDRESS_SPACE_MEM
+                         || (enmType & ~PCI_ADDRESS_SPACE_BAR64) == PCI_ADDRESS_SPACE_MEM_PREFETCH,
+                         VERR_INVALID_FLAGS);
+            rc = IOMR3MmioValidateHandle(pVM, pDevIns, (IOMMMIOHANDLE)hHandle);
+            AssertRCReturn(rc, rc);
+            break;
+        case PDMPCIDEV_IORGN_F_MMIO2_HANDLE:
+            AssertReturn(   (enmType & ~PCI_ADDRESS_SPACE_BAR64) == PCI_ADDRESS_SPACE_MEM
+                         || (enmType & ~PCI_ADDRESS_SPACE_BAR64) == PCI_ADDRESS_SPACE_MEM_PREFETCH,
+                         VERR_INVALID_FLAGS);
+            rc = PGMR3PhysMmio2ValidateHandle(pVM, pDevIns, (PGMMMIO2HANDLE)hHandle);
+            AssertRCReturn(rc, rc);
+            break;
+        default:
+            AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+            break;
+    }
+
+    /* This flag is required now. */
+    AssertLogRelMsgReturn(fFlags & PDMPCIDEV_IORGN_F_NEW_STYLE,
+                          ("'%s'/%d: Invalid flags: %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, fFlags),
+                          VERR_INVALID_FLAGS);
+
+    /*
+     * We're currently restricted to page aligned MMIO regions.
+     */
+    if (   ((enmType & ~(PCI_ADDRESS_SPACE_BAR64 | PCI_ADDRESS_SPACE_MEM_PREFETCH)) == PCI_ADDRESS_SPACE_MEM)
+        && cbRegion != RT_ALIGN_64(cbRegion, PAGE_SIZE))
+    {
+        Log(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: aligning cbRegion %RGp -> %RGp\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, cbRegion, RT_ALIGN_64(cbRegion, PAGE_SIZE)));
+        cbRegion = RT_ALIGN_64(cbRegion, PAGE_SIZE);
+    }
+
+    /*
+     * For registering PCI MMIO memory or PCI I/O memory, the size of the region must be a power of 2!
+     */
+    int iLastSet = ASMBitLastSetU64(cbRegion);
+    Assert(iLastSet > 0);
+    uint64_t cbRegionAligned = RT_BIT_64(iLastSet - 1);
+    if (cbRegion > cbRegionAligned)
+        cbRegion = cbRegionAligned * 2; /* round up */
+
+    size_t const    idxBus = pPciDev->Int.s.idxPdmBus;
+    AssertReturn(idxBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses), VERR_WRONG_ORDER);
+    PPDMPCIBUS      pBus   = &pVM->pdm.s.aPciBuses[idxBus];
+
+    pdmLock(pVM);
+    rc = pBus->pfnIORegionRegister(pBus->pDevInsR3, pPciDev, iRegion, cbRegion, enmType, fFlags, hHandle, pfnMapUnmap);
+    pdmUnlock(pVM);
+
+    LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCIInterceptConfigAccesses} */
+static DECLCALLBACK(int) pdmR3DevHlp_PCIInterceptConfigAccesses(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
+                                                                PFNPCICONFIGREAD pfnRead, PFNPCICONFIGWRITE pfnWrite)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    if (!pPciDev) /* NULL is an alias for the default PCI device. */
+        pPciDev = pDevIns->apPciDevs[0];
+    AssertReturn(pPciDev, VERR_PDM_NOT_PCI_DEVICE);
+    LogFlow(("pdmR3DevHlp_PCIInterceptConfigAccesses: caller='%s'/%d: pPciDev=%p pfnRead=%p pfnWrite=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPciDev, pfnRead, pfnWrite));
+    PDMPCIDEV_ASSERT_VALID_RET(pDevIns, pPciDev);
+
+    /*
+     * Validate input.
+     */
+    AssertPtr(pfnRead);
+    AssertPtr(pfnWrite);
+    AssertPtr(pPciDev);
+
+    size_t const    idxBus = pPciDev->Int.s.idxPdmBus;
+    AssertReturn(idxBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses), VERR_INTERNAL_ERROR_2);
+    PPDMPCIBUS      pBus   = &pVM->pdm.s.aPciBuses[idxBus];
+    AssertRelease(VMR3GetState(pVM) != VMSTATE_RUNNING);
+
+    /*
+     * Do the job.
+     */
+    pdmLock(pVM);
+    pBus->pfnInterceptConfigAccesses(pBus->pDevInsR3, pPciDev, pfnRead, pfnWrite);
+    pdmUnlock(pVM);
+
+    LogFlow(("pdmR3DevHlp_PCIInterceptConfigAccesses: caller='%s'/%d: returns VINF_SUCCESS\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCIConfigWrite} */
+static DECLCALLBACK(VBOXSTRICTRC)
+pdmR3DevHlp_PCIConfigWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, uint32_t uAddress, unsigned cb, uint32_t u32Value)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    AssertPtrReturn(pPciDev, VERR_PDM_NOT_PCI_DEVICE);
+    LogFlow(("pdmR3DevHlp_PCIConfigWrite: caller='%s'/%d: pPciDev=%p uAddress=%#x cd=%d u32Value=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPciDev, uAddress, cb, u32Value));
+
+    /*
+     * Resolve the bus.
+     */
+    size_t const    idxBus = pPciDev->Int.s.idxPdmBus;
+    AssertReturn(idxBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses), VERR_INTERNAL_ERROR_2);
+    PPDMPCIBUS      pBus   = &pVM->pdm.s.aPciBuses[idxBus];
+
+    /*
+     * Do the job.
+     */
+    VBOXSTRICTRC rcStrict = pBus->pfnConfigWrite(pBus->pDevInsR3, pPciDev, uAddress, cb, u32Value);
+
+    LogFlow(("pdmR3DevHlp_PCIConfigWrite: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCIConfigRead} */
+static DECLCALLBACK(VBOXSTRICTRC)
+pdmR3DevHlp_PCIConfigRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, uint32_t uAddress, unsigned cb, uint32_t *pu32Value)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    AssertPtrReturn(pPciDev, VERR_PDM_NOT_PCI_DEVICE);
+    LogFlow(("pdmR3DevHlp_PCIConfigRead: caller='%s'/%d: pPciDev=%p uAddress=%#x cd=%d pu32Value=%p:{%#x}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPciDev, uAddress, cb, pu32Value, *pu32Value));
+
+    /*
+     * Resolve the bus.
+     */
+    size_t const    idxBus = pPciDev->Int.s.idxPdmBus;
+    AssertReturn(idxBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses), VERR_INTERNAL_ERROR_2);
+    PPDMPCIBUS      pBus   = &pVM->pdm.s.aPciBuses[idxBus];
+
+    /*
+     * Do the job.
+     */
+    VBOXSTRICTRC rcStrict = pBus->pfnConfigRead(pBus->pDevInsR3, pPciDev, uAddress, cb, pu32Value);
+
+    LogFlow(("pdmR3DevHlp_PCIConfigRead: caller='%s'/%d: returns %Rrc (*pu32Value=%#x)\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, VBOXSTRICTRC_VAL(rcStrict), *pu32Value));
+    return rcStrict;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCIPhysRead} */
+static DECLCALLBACK(int)
+pdmR3DevHlp_PCIPhysRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    if (!pPciDev) /* NULL is an alias for the default PCI device. */
+        pPciDev = pDevIns->apPciDevs[0];
+    AssertReturn(pPciDev, VERR_PDM_NOT_PCI_DEVICE);
+    PDMPCIDEV_ASSERT_VALID_AND_REGISTERED(pDevIns, pPciDev);
+
+#ifndef PDM_DO_NOT_RESPECT_PCI_BM_BIT
+    /*
+     * Just check the busmaster setting here and forward the request to the generic read helper.
+     */
+    if (PCIDevIsBusmaster(pPciDev))
+    { /* likely */ }
+    else
+    {
+        Log(("pdmR3DevHlp_PCIPhysRead: caller='%s'/%d: returns %Rrc - Not bus master! GCPhys=%RGp cbRead=%#zx\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, VERR_PDM_NOT_PCI_BUS_MASTER, GCPhys, cbRead));
+        memset(pvBuf, 0xff, cbRead);
+        return VERR_PDM_NOT_PCI_BUS_MASTER;
+    }
+#endif
+
+    return pDevIns->pHlpR3->pfnPhysRead(pDevIns, GCPhys, pvBuf, cbRead);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCIPhysWrite} */
+static DECLCALLBACK(int)
+pdmR3DevHlp_PCIPhysWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    if (!pPciDev) /* NULL is an alias for the default PCI device. */
+        pPciDev = pDevIns->apPciDevs[0];
+    AssertReturn(pPciDev, VERR_PDM_NOT_PCI_DEVICE);
+    PDMPCIDEV_ASSERT_VALID_AND_REGISTERED(pDevIns, pPciDev);
+
+#ifndef PDM_DO_NOT_RESPECT_PCI_BM_BIT
+    /*
+     * Just check the busmaster setting here and forward the request to the generic read helper.
+     */
+    if (PCIDevIsBusmaster(pPciDev))
+    { /* likely */ }
+    else
+    {
+        Log(("pdmR3DevHlp_PCIPhysWrite: caller='%s'/%d: returns %Rrc - Not bus master! GCPhys=%RGp cbWrite=%#zx\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, VERR_PDM_NOT_PCI_BUS_MASTER, GCPhys, cbWrite));
+        return VERR_PDM_NOT_PCI_BUS_MASTER;
+    }
+#endif
+
+    return pDevIns->pHlpR3->pfnPhysWrite(pDevIns, GCPhys, pvBuf, cbWrite);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCISetIrq} */
+static DECLCALLBACK(void) pdmR3DevHlp_PCISetIrq(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, int iIrq, int iLevel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    if (!pPciDev) /* NULL is an alias for the default PCI device. */
+        pPciDev = pDevIns->apPciDevs[0];
+    AssertReturnVoid(pPciDev);
+    LogFlow(("pdmR3DevHlp_PCISetIrq: caller='%s'/%d: pPciDev=%p:{%#x} iIrq=%d iLevel=%d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPciDev, pPciDev->uDevFn, iIrq, iLevel));
+    PDMPCIDEV_ASSERT_VALID_AND_REGISTERED(pDevIns, pPciDev);
+
+    /*
+     * Validate input.
+     */
+    Assert(iIrq == 0);
+    Assert((uint32_t)iLevel <= PDM_IRQ_LEVEL_FLIP_FLOP);
+
+    /*
+     * Must have a PCI device registered!
+     */
+    PVM             pVM    = pDevIns->Internal.s.pVMR3;
+    size_t const    idxBus = pPciDev->Int.s.idxPdmBus;
+    AssertReturnVoid(idxBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses));
+    PPDMPCIBUS      pBus   = &pVM->pdm.s.aPciBuses[idxBus];
+
+    pdmLock(pVM);
+    uint32_t uTagSrc;
+    if (iLevel & PDM_IRQ_LEVEL_HIGH)
+    {
+        pDevIns->Internal.s.uLastIrqTag = uTagSrc = pdmCalcIrqTag(pVM, pDevIns->idTracing);
+        if (iLevel == PDM_IRQ_LEVEL_HIGH)
+            VBOXVMM_PDM_IRQ_HIGH(VMMGetCpu(pVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+        else
+            VBOXVMM_PDM_IRQ_HILO(VMMGetCpu(pVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+    }
+    else
+        uTagSrc = pDevIns->Internal.s.uLastIrqTag;
+
+    pBus->pfnSetIrqR3(pBus->pDevInsR3, pPciDev, iIrq, iLevel, uTagSrc);
+
+    if (iLevel == PDM_IRQ_LEVEL_LOW)
+        VBOXVMM_PDM_IRQ_LOW(VMMGetCpu(pVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+    pdmUnlock(pVM);
+
+    LogFlow(("pdmR3DevHlp_PCISetIrq: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCISetIrqNoWait} */
+static DECLCALLBACK(void) pdmR3DevHlp_PCISetIrqNoWait(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, int iIrq, int iLevel)
+{
+    pdmR3DevHlp_PCISetIrq(pDevIns, pPciDev, iIrq, iLevel);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnISASetIrq} */
+static DECLCALLBACK(void) pdmR3DevHlp_ISASetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_ISASetIrq: caller='%s'/%d: iIrq=%d iLevel=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, iIrq, iLevel));
+
+    /*
+     * Validate input.
+     */
+    Assert(iIrq < 16);
+    Assert((uint32_t)iLevel <= PDM_IRQ_LEVEL_FLIP_FLOP);
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+
+    /*
+     * Do the job.
+     */
+    pdmLock(pVM);
+    uint32_t uTagSrc;
+    if (iLevel & PDM_IRQ_LEVEL_HIGH)
+    {
+        pDevIns->Internal.s.uLastIrqTag = uTagSrc = pdmCalcIrqTag(pVM, pDevIns->idTracing);
+        if (iLevel == PDM_IRQ_LEVEL_HIGH)
+            VBOXVMM_PDM_IRQ_HIGH(VMMGetCpu(pVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+        else
+            VBOXVMM_PDM_IRQ_HILO(VMMGetCpu(pVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+    }
+    else
+        uTagSrc = pDevIns->Internal.s.uLastIrqTag;
+
+    PDMIsaSetIrq(pVM, iIrq, iLevel, uTagSrc);  /* (The API takes the lock recursively.) */
+
+    if (iLevel == PDM_IRQ_LEVEL_LOW)
+        VBOXVMM_PDM_IRQ_LOW(VMMGetCpu(pVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+    pdmUnlock(pVM);
+
+    LogFlow(("pdmR3DevHlp_ISASetIrq: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnISASetIrqNoWait} */
+static DECLCALLBACK(void) pdmR3DevHlp_ISASetIrqNoWait(PPDMDEVINS pDevIns, int iIrq, int iLevel)
+{
+    pdmR3DevHlp_ISASetIrq(pDevIns, iIrq, iLevel);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnIoApicSendMsi} */
+static DECLCALLBACK(void) pdmR3DevHlp_IoApicSendMsi(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint32_t uValue)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_IoApicSendMsi: caller='%s'/%d: GCPhys=%RGp uValue=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, uValue));
+
+    /*
+     * Validate input.
+     */
+    Assert(GCPhys != 0);
+    Assert(uValue != 0);
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+
+    /*
+     * Do the job.
+     */
+    pdmLock(pVM);
+    uint32_t uTagSrc;
+    pDevIns->Internal.s.uLastIrqTag = uTagSrc = pdmCalcIrqTag(pVM, pDevIns->idTracing);
+    VBOXVMM_PDM_IRQ_HILO(VMMGetCpu(pVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+
+    PDMIoApicSendMsi(pVM, GCPhys, uValue, uTagSrc);  /* (The API takes the lock recursively.) */
+
+    pdmUnlock(pVM);
+
+    LogFlow(("pdmR3DevHlp_IoApicSendMsi: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDriverAttach} */
+static DECLCALLBACK(int) pdmR3DevHlp_DriverAttach(PPDMDEVINS pDevIns, uint32_t iLun, PPDMIBASE pBaseInterface, PPDMIBASE *ppBaseInterface, const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DriverAttach: caller='%s'/%d: iLun=%d pBaseInterface=%p ppBaseInterface=%p pszDesc=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iLun, pBaseInterface, ppBaseInterface, pszDesc, pszDesc));
+
+    /*
+     * Lookup the LUN, it might already be registered.
+     */
+    PPDMLUN pLunPrev = NULL;
+    PPDMLUN pLun = pDevIns->Internal.s.pLunsR3;
+    for (; pLun; pLunPrev = pLun, pLun = pLun->pNext)
+        if (pLun->iLun == iLun)
+            break;
+
+    /*
+     * Create the LUN if if wasn't found, else check if driver is already attached to it.
+     */
+    if (!pLun)
+    {
+        if (    !pBaseInterface
+            ||  !pszDesc
+            ||  !*pszDesc)
+        {
+            Assert(pBaseInterface);
+            Assert(pszDesc || *pszDesc);
+            return VERR_INVALID_PARAMETER;
+        }
+
+        pLun = (PPDMLUN)MMR3HeapAlloc(pVM, MM_TAG_PDM_LUN, sizeof(*pLun));
+        if (!pLun)
+            return VERR_NO_MEMORY;
+
+        pLun->iLun      = iLun;
+        pLun->pNext     = pLunPrev ? pLunPrev->pNext : NULL;
+        pLun->pTop      = NULL;
+        pLun->pBottom   = NULL;
+        pLun->pDevIns   = pDevIns;
+        pLun->pUsbIns   = NULL;
+        pLun->pszDesc   = pszDesc;
+        pLun->pBase     = pBaseInterface;
+        if (!pLunPrev)
+            pDevIns->Internal.s.pLunsR3 = pLun;
+        else
+            pLunPrev->pNext = pLun;
+        Log(("pdmR3DevHlp_DriverAttach: Registered LUN#%d '%s' with device '%s'/%d.\n",
+             iLun, pszDesc, pDevIns->pReg->szName, pDevIns->iInstance));
+    }
+    else if (pLun->pTop)
+    {
+        AssertMsgFailed(("Already attached! The device should keep track of such things!\n"));
+        LogFlow(("pdmR3DevHlp_DriverAttach: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_PDM_DRIVER_ALREADY_ATTACHED));
+        return VERR_PDM_DRIVER_ALREADY_ATTACHED;
+    }
+    Assert(pLun->pBase == pBaseInterface);
+
+
+    /*
+     * Get the attached driver configuration.
+     */
+    int rc;
+    PCFGMNODE pNode = CFGMR3GetChildF(pDevIns->Internal.s.pCfgHandle, "LUN#%u", iLun);
+    if (pNode)
+        rc = pdmR3DrvInstantiate(pVM, pNode, pBaseInterface, NULL /*pDrvAbove*/, pLun, ppBaseInterface);
+    else
+        rc = VERR_PDM_NO_ATTACHED_DRIVER;
+
+    LogFlow(("pdmR3DevHlp_DriverAttach: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDriverDetach} */
+static DECLCALLBACK(int) pdmR3DevHlp_DriverDetach(PPDMDEVINS pDevIns, PPDMDRVINS pDrvIns, uint32_t fFlags)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns); RT_NOREF_PV(pDevIns);
+    LogFlow(("pdmR3DevHlp_DriverDetach: caller='%s'/%d: pDrvIns=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pDrvIns));
+
+#ifdef VBOX_STRICT
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+#endif
+
+    int rc = pdmR3DrvDetach(pDrvIns, fFlags);
+
+    LogFlow(("pdmR3DevHlp_DriverDetach: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDriverReconfigure} */
+static DECLCALLBACK(int) pdmR3DevHlp_DriverReconfigure(PPDMDEVINS pDevIns, uint32_t iLun, uint32_t cDepth,
+                                                       const char * const *papszDrivers, PCFGMNODE *papConfigs, uint32_t fFlags)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DriverReconfigure: caller='%s'/%d: iLun=%u cDepth=%u fFlags=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iLun, cDepth, fFlags));
+
+    /*
+     * Validate input.
+     */
+    AssertReturn(cDepth <= 8, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(papszDrivers, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(papConfigs, VERR_INVALID_POINTER);
+    for (uint32_t i = 0; i < cDepth; i++)
+    {
+        AssertPtrReturn(papszDrivers[i], VERR_INVALID_POINTER);
+        size_t cchDriver = strlen(papszDrivers[i]);
+        AssertPtrReturn(cchDriver > 0 && cchDriver < RT_SIZEOFMEMB(PDMDRVREG, szName), VERR_OUT_OF_RANGE);
+
+        if (papConfigs)
+            AssertPtrNullReturn(papConfigs[i], VERR_INVALID_POINTER);
+    }
+    AssertReturn(fFlags == 0, VERR_INVALID_FLAGS);
+
+    /*
+     * Do we have to detach an existing driver first?
+     */
+    for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+        if (pLun->iLun == iLun)
+        {
+            if (pLun->pTop)
+            {
+                int rc = pdmR3DrvDetach(pLun->pTop, 0);
+                AssertRCReturn(rc, rc);
+            }
+            break;
+        }
+
+    /*
+     * Remove the old tree.
+     */
+    PCFGMNODE pCfgDev = CFGMR3GetChildF(CFGMR3GetRoot(pVM), "Devices/%s/%u/", pDevIns->pReg->szName, pDevIns->iInstance);
+    AssertReturn(pCfgDev, VERR_INTERNAL_ERROR_2);
+    PCFGMNODE pCfgLun = CFGMR3GetChildF(pCfgDev, "LUN#%u", iLun);
+    if (pCfgLun)
+        CFGMR3RemoveNode(pCfgLun);
+
+    /*
+     * Construct a new tree.
+     */
+    int rc = CFGMR3InsertNodeF(pCfgDev, &pCfgLun, "LUN#%u", iLun);
+    AssertRCReturn(rc, rc);
+    PCFGMNODE pCfgDrv = pCfgLun;
+    for (uint32_t i = 0; i < cDepth; i++)
+    {
+        rc = CFGMR3InsertString(pCfgDrv, "Driver", papszDrivers[i]);
+        AssertRCReturn(rc, rc);
+        if (papConfigs && papConfigs[i])
+        {
+            rc = CFGMR3InsertSubTree(pCfgDrv, "Config", papConfigs[i], NULL);
+            AssertRCReturn(rc, rc);
+            papConfigs[i] = NULL;
+        }
+        else
+        {
+            rc = CFGMR3InsertNode(pCfgDrv, "Config", NULL);
+            AssertRCReturn(rc, rc);
+        }
+
+        if (i + 1 >= cDepth)
+            break;
+        rc = CFGMR3InsertNode(pCfgDrv, "AttachedDriver", &pCfgDrv);
+        AssertRCReturn(rc, rc);
+    }
+
+    LogFlow(("pdmR3DevHlp_DriverReconfigure: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnQueueCreatePtr} */
+static DECLCALLBACK(int) pdmR3DevHlp_QueueCreatePtr(PPDMDEVINS pDevIns, size_t cbItem, uint32_t cItems, uint32_t cMilliesInterval,
+                                                    PFNPDMQUEUEDEV pfnCallback, bool fRZEnabled, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_QueueCreatePtr: caller='%s'/%d: cbItem=%#x cItems=%#x cMilliesInterval=%u pfnCallback=%p fRZEnabled=%RTbool pszName=%p:{%s} ppQueue=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, cbItem, cItems, cMilliesInterval, pfnCallback, fRZEnabled, pszName, pszName, ppQueue));
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    if (pDevIns->iInstance > 0)
+    {
+        pszName = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s_%u", pszName, pDevIns->iInstance);
+        AssertLogRelReturn(pszName, VERR_NO_MEMORY);
+    }
+
+    int rc = PDMR3QueueCreateDevice(pVM, pDevIns, cbItem, cItems, cMilliesInterval, pfnCallback, fRZEnabled, pszName, ppQueue);
+
+    LogFlow(("pdmR3DevHlp_QueueCreatePtr: caller='%s'/%d: returns %Rrc *ppQueue=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *ppQueue));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnQueueCreate} */
+static DECLCALLBACK(int) pdmR3DevHlp_QueueCreate(PPDMDEVINS pDevIns, size_t cbItem, uint32_t cItems, uint32_t cMilliesInterval,
+                                                 PFNPDMQUEUEDEV pfnCallback, bool fRZEnabled, const char *pszName,
+                                                 PDMQUEUEHANDLE *phQueue)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_QueueCreate: caller='%s'/%d: cbItem=%#x cItems=%#x cMilliesInterval=%u pfnCallback=%p fRZEnabled=%RTbool pszName=%p:{%s} phQueue=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, cbItem, cItems, cMilliesInterval, pfnCallback, fRZEnabled, pszName, pszName, phQueue));
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    if (pDevIns->iInstance > 0)
+    {
+        pszName = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s_%u", pszName, pDevIns->iInstance);
+        AssertLogRelReturn(pszName, VERR_NO_MEMORY);
+    }
+
+    PPDMQUEUE pQueue = NULL;
+    int rc = PDMR3QueueCreateDevice(pVM, pDevIns, cbItem, cItems, cMilliesInterval, pfnCallback, fRZEnabled, pszName, &pQueue);
+    *phQueue = (uintptr_t)pQueue;
+
+    LogFlow(("pdmR3DevHlp_QueueCreate: caller='%s'/%d: returns %Rrc *ppQueue=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *phQueue));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnQueueToPtr} */
+static DECLCALLBACK(PPDMQUEUE)  pdmR3DevHlp_QueueToPtr(PPDMDEVINS pDevIns, PDMQUEUEHANDLE hQueue)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return (PPDMQUEUE)hQueue;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnQueueAlloc} */
+static DECLCALLBACK(PPDMQUEUEITEMCORE) pdmR3DevHlp_QueueAlloc(PPDMDEVINS pDevIns, PDMQUEUEHANDLE hQueue)
+{
+    return PDMQueueAlloc(pdmR3DevHlp_QueueToPtr(pDevIns, hQueue));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnQueueInsert} */
+static DECLCALLBACK(void) pdmR3DevHlp_QueueInsert(PPDMDEVINS pDevIns, PDMQUEUEHANDLE hQueue, PPDMQUEUEITEMCORE pItem)
+{
+    return PDMQueueInsert(pdmR3DevHlp_QueueToPtr(pDevIns, hQueue), pItem);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnQueueInsertEx} */
+static DECLCALLBACK(void) pdmR3DevHlp_QueueInsertEx(PPDMDEVINS pDevIns, PDMQUEUEHANDLE hQueue, PPDMQUEUEITEMCORE pItem,
+                                                    uint64_t cNanoMaxDelay)
+{
+    return PDMQueueInsertEx(pdmR3DevHlp_QueueToPtr(pDevIns, hQueue), pItem, cNanoMaxDelay);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnQueueFlushIfNecessary} */
+static DECLCALLBACK(bool) pdmR3DevHlp_QueueFlushIfNecessary(PPDMDEVINS pDevIns, PDMQUEUEHANDLE hQueue)
+{
+    return PDMQueueFlushIfNecessary(pdmR3DevHlp_QueueToPtr(pDevIns, hQueue));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTaskCreate} */
+static DECLCALLBACK(int) pdmR3DevHlp_TaskCreate(PPDMDEVINS pDevIns, uint32_t fFlags, const char *pszName,
+                                                PFNPDMTASKDEV pfnCallback, void *pvUser, PDMTASKHANDLE *phTask)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_TaskTrigger: caller='%s'/%d: pfnCallback=%p fFlags=%#x pszName=%p:{%s} phTask=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pfnCallback, fFlags, pszName, pszName, phTask));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    int rc = PDMR3TaskCreate(pVM, fFlags, pszName, PDMTASKTYPE_DEV, pDevIns, (PFNRT)pfnCallback, pvUser, phTask);
+
+    LogFlow(("pdmR3DevHlp_TaskTrigger: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTaskTrigger} */
+static DECLCALLBACK(int) pdmR3DevHlp_TaskTrigger(PPDMDEVINS pDevIns, PDMTASKHANDLE hTask)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_TaskTrigger: caller='%s'/%d: hTask=%RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, hTask));
+
+    int rc = PDMTaskTrigger(pDevIns->Internal.s.pVMR3, PDMTASKTYPE_DEV, pDevIns, hTask);
+
+    LogFlow(("pdmR3DevHlp_TaskTrigger: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventCreate} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventCreate(PPDMDEVINS pDevIns, PSUPSEMEVENT phEvent)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventCreate: caller='%s'/%d: phEvent=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, phEvent));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    int rc = SUPSemEventCreate(pVM->pSession, phEvent);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventCreate: caller='%s'/%d: returns %Rrc *phEvent=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *phEvent));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventClose} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventClose(PPDMDEVINS pDevIns, SUPSEMEVENT hEvent)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventClose: caller='%s'/%d: hEvent=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, hEvent));
+
+    int rc = SUPSemEventClose(pDevIns->Internal.s.pVMR3->pSession, hEvent);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventClose: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventSignal} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventSignal(PPDMDEVINS pDevIns, SUPSEMEVENT hEvent)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventSignal: caller='%s'/%d: hEvent=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, hEvent));
+
+    int rc = SUPSemEventSignal(pDevIns->Internal.s.pVMR3->pSession, hEvent);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventSignal: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventWaitNoResume} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventWaitNoResume(PPDMDEVINS pDevIns, SUPSEMEVENT hEvent, uint32_t cMillies)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventWaitNoResume: caller='%s'/%d: hEvent=%p cNsTimeout=%RU32\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hEvent, cMillies));
+
+    int rc = SUPSemEventWaitNoResume(pDevIns->Internal.s.pVMR3->pSession, hEvent, cMillies);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventWaitNoResume: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventWaitNsAbsIntr} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventWaitNsAbsIntr(PPDMDEVINS pDevIns, SUPSEMEVENT hEvent, uint64_t uNsTimeout)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventWaitNsAbsIntr: caller='%s'/%d: hEvent=%p uNsTimeout=%RU64\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hEvent, uNsTimeout));
+
+    int rc = SUPSemEventWaitNsAbsIntr(pDevIns->Internal.s.pVMR3->pSession, hEvent, uNsTimeout);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventWaitNsAbsIntr: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventWaitNsRelIntr} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventWaitNsRelIntr(PPDMDEVINS pDevIns, SUPSEMEVENT hEvent, uint64_t cNsTimeout)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventWaitNsRelIntr: caller='%s'/%d: hEvent=%p cNsTimeout=%RU64\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hEvent, cNsTimeout));
+
+    int rc = SUPSemEventWaitNsRelIntr(pDevIns->Internal.s.pVMR3->pSession, hEvent, cNsTimeout);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventWaitNsRelIntr: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventGetResolution} */
+static DECLCALLBACK(uint32_t) pdmR3DevHlp_SUPSemEventGetResolution(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventGetResolution: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+
+    uint32_t cNsResolution = SUPSemEventGetResolution(pDevIns->Internal.s.pVMR3->pSession);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventGetResolution: caller='%s'/%d: returns %u\n", pDevIns->pReg->szName, pDevIns->iInstance, cNsResolution));
+    return cNsResolution;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventMultiCreate} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventMultiCreate(PPDMDEVINS pDevIns, PSUPSEMEVENTMULTI phEventMulti)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiCreate: caller='%s'/%d: phEventMulti=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, phEventMulti));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    int rc = SUPSemEventMultiCreate(pVM->pSession, phEventMulti);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiCreate: caller='%s'/%d: returns %Rrc *phEventMulti=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *phEventMulti));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventMultiClose} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventMultiClose(PPDMDEVINS pDevIns, SUPSEMEVENTMULTI hEventMulti)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiClose: caller='%s'/%d: hEventMulti=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, hEventMulti));
+
+    int rc = SUPSemEventMultiClose(pDevIns->Internal.s.pVMR3->pSession, hEventMulti);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiClose: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventMultiSignal} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventMultiSignal(PPDMDEVINS pDevIns, SUPSEMEVENTMULTI hEventMulti)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiSignal: caller='%s'/%d: hEventMulti=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, hEventMulti));
+
+    int rc = SUPSemEventMultiSignal(pDevIns->Internal.s.pVMR3->pSession, hEventMulti);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiSignal: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventMultiReset} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventMultiReset(PPDMDEVINS pDevIns, SUPSEMEVENTMULTI hEventMulti)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiReset: caller='%s'/%d: hEventMulti=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, hEventMulti));
+
+    int rc = SUPSemEventMultiReset(pDevIns->Internal.s.pVMR3->pSession, hEventMulti);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiReset: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventMultiWaitNoResume} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventMultiWaitNoResume(PPDMDEVINS pDevIns, SUPSEMEVENTMULTI hEventMulti,
+                                                                  uint32_t cMillies)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiWaitNoResume: caller='%s'/%d: hEventMulti=%p cMillies=%RU32\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hEventMulti, cMillies));
+
+    int rc = SUPSemEventMultiWaitNoResume(pDevIns->Internal.s.pVMR3->pSession, hEventMulti, cMillies);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiWaitNoResume: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventMultiWaitNsAbsIntr} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventMultiWaitNsAbsIntr(PPDMDEVINS pDevIns, SUPSEMEVENTMULTI hEventMulti,
+                                                                   uint64_t uNsTimeout)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiWaitNsAbsIntr: caller='%s'/%d: hEventMulti=%p uNsTimeout=%RU64\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hEventMulti, uNsTimeout));
+
+    int rc = SUPSemEventMultiWaitNsAbsIntr(pDevIns->Internal.s.pVMR3->pSession, hEventMulti, uNsTimeout);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiWaitNsAbsIntr: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventMultiWaitNsRelIntr} */
+static DECLCALLBACK(int) pdmR3DevHlp_SUPSemEventMultiWaitNsRelIntr(PPDMDEVINS pDevIns, SUPSEMEVENTMULTI hEventMulti,
+                                                                   uint64_t cNsTimeout)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiWaitNsRelIntr: caller='%s'/%d: hEventMulti=%p cNsTimeout=%RU64\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, hEventMulti, cNsTimeout));
+
+    int rc = SUPSemEventMultiWaitNsRelIntr(pDevIns->Internal.s.pVMR3->pSession, hEventMulti, cNsTimeout);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiWaitNsRelIntr: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSUPSemEventMultiGetResolution} */
+static DECLCALLBACK(uint32_t) pdmR3DevHlp_SUPSemEventMultiGetResolution(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiGetResolution: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+
+    uint32_t cNsResolution = SUPSemEventMultiGetResolution(pDevIns->Internal.s.pVMR3->pSession);
+
+    LogFlow(("pdmR3DevHlp_SUPSemEventMultiGetResolution: caller='%s'/%d: returns %u\n", pDevIns->pReg->szName, pDevIns->iInstance, cNsResolution));
+    return cNsResolution;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectInit} */
+static DECLCALLBACK(int) pdmR3DevHlp_CritSectInit(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
+                                                  const char *pszNameFmt, va_list va)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_CritSectInit: caller='%s'/%d: pCritSect=%p pszNameFmt=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pCritSect, pszNameFmt, pszNameFmt));
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    int rc = pdmR3CritSectInitDevice(pVM, pDevIns, pCritSect, RT_SRC_POS_ARGS, pszNameFmt, va);
+
+    LogFlow(("pdmR3DevHlp_CritSectInit: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectGetNop} */
+static DECLCALLBACK(PPDMCRITSECT) pdmR3DevHlp_CritSectGetNop(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    PPDMCRITSECT pCritSect = PDMR3CritSectGetNop(pVM);
+    LogFlow(("pdmR3DevHlp_CritSectGetNop: caller='%s'/%d: return %p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pCritSect));
+    return pCritSect;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectGetNopR0} */
+static DECLCALLBACK(R0PTRTYPE(PPDMCRITSECT)) pdmR3DevHlp_CritSectGetNopR0(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    R0PTRTYPE(PPDMCRITSECT) pCritSect = PDMR3CritSectGetNopR0(pVM);
+    LogFlow(("pdmR3DevHlp_CritSectGetNopR0: caller='%s'/%d: return %RHv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pCritSect));
+    return pCritSect;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectGetNopRC} */
+static DECLCALLBACK(RCPTRTYPE(PPDMCRITSECT)) pdmR3DevHlp_CritSectGetNopRC(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    RCPTRTYPE(PPDMCRITSECT) pCritSect = PDMR3CritSectGetNopRC(pVM);
+    LogFlow(("pdmR3DevHlp_CritSectGetNopRC: caller='%s'/%d: return %RRv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pCritSect));
+    return pCritSect;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSetDeviceCritSect} */
+static DECLCALLBACK(int) pdmR3DevHlp_SetDeviceCritSect(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect)
+{
+    /*
+     * Validate input.
+     *
+     * Note! We only allow the automatically created default critical section
+     *       to be replaced by this API.
+     */
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertPtrReturn(pCritSect, VERR_INVALID_POINTER);
+    LogFlow(("pdmR3DevHlp_SetDeviceCritSect: caller='%s'/%d: pCritSect=%p (%s)\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pCritSect, pCritSect->s.pszName));
+    AssertReturn(PDMCritSectIsInitialized(pCritSect), VERR_INVALID_PARAMETER);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    AssertReturn(pCritSect->s.pVMR3 == pVM, VERR_INVALID_PARAMETER);
+
+    VM_ASSERT_EMT(pVM);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_WRONG_ORDER);
+
+    AssertReturn(pDevIns->pCritSectRoR3, VERR_PDM_DEV_IPE_1);
+    AssertReturn(pDevIns->pCritSectRoR3->s.fAutomaticDefaultCritsect, VERR_WRONG_ORDER);
+    AssertReturn(!pDevIns->pCritSectRoR3->s.fUsedByTimerOrSimilar, VERR_WRONG_ORDER);
+    AssertReturn(pDevIns->pCritSectRoR3 != pCritSect, VERR_INVALID_PARAMETER);
+
+    /*
+     * Replace the critical section and destroy the automatic default section.
+     */
+    PPDMCRITSECT pOldCritSect = pDevIns->pCritSectRoR3;
+    pDevIns->pCritSectRoR3 = pCritSect;
+    pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_CHANGED_CRITSECT;
+
+    Assert(RT_BOOL(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_R0_ENABLED) == pDevIns->fR0Enabled);
+    if (   (pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_R0_ENABLED)
+        && !(pDevIns->Internal.s.pDevR3->pReg->fFlags & PDM_DEVREG_FLAGS_NEW_STYLE))
+    {
+        PDMDEVICECOMPATSETCRITSECTREQ Req;
+        Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+        Req.Hdr.cbReq    = sizeof(Req);
+        Req.idxR0Device  = pDevIns->Internal.s.idxR0Device;
+        Req.pDevInsR3    = pDevIns;
+        Req.pCritSectR3  = pCritSect;
+        int rc = VMMR3CallR0(pVM, VMMR0_DO_PDM_DEVICE_COMPAT_SET_CRITSECT, 0, &Req.Hdr);
+        AssertLogRelRCReturn(rc, rc);
+    }
+
+    PDMR3CritSectDelete(pOldCritSect);
+    Assert((uintptr_t)pOldCritSect - (uintptr_t)pDevIns < pDevIns->cbRing3);
+
+    LogFlow(("pdmR3DevHlp_SetDeviceCritSect: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectYield} */
+static DECLCALLBACK(bool)     pdmR3DevHlp_CritSectYield(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    return PDMR3CritSectYield(pDevIns->Internal.s.pVMR3, pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectEnter} */
+static DECLCALLBACK(int)      pdmR3DevHlp_CritSectEnter(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, int rcBusy)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns); /** @todo pass pDevIns->Internal.s.pVMR3 to the crit sect code.   */
+    return PDMCritSectEnter(pCritSect, rcBusy);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectEnterDebug} */
+static DECLCALLBACK(int)      pdmR3DevHlp_CritSectEnterDebug(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns); /** @todo pass pDevIns->Internal.s.pVMR3 to the crit sect code.   */
+    return PDMCritSectEnterDebug(pCritSect, rcBusy, uId, RT_SRC_POS_ARGS);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectTryEnter} */
+static DECLCALLBACK(int)      pdmR3DevHlp_CritSectTryEnter(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns); /** @todo pass pDevIns->Internal.s.pVMR3 to the crit sect code.   */
+    return PDMCritSectTryEnter(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectTryEnterDebug} */
+static DECLCALLBACK(int)      pdmR3DevHlp_CritSectTryEnterDebug(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns); /** @todo pass pDevIns->Internal.s.pVMR3 to the crit sect code.   */
+    return PDMCritSectTryEnterDebug(pCritSect, uId, RT_SRC_POS_ARGS);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectLeave} */
+static DECLCALLBACK(int)      pdmR3DevHlp_CritSectLeave(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns); /** @todo pass pDevIns->Internal.s.pVMR3 to the crit sect code.   */
+    return PDMCritSectLeave(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectIsOwner} */
+static DECLCALLBACK(bool)     pdmR3DevHlp_CritSectIsOwner(PPDMDEVINS pDevIns, PCPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns); /** @todo pass pDevIns->Internal.s.pVMR3 to the crit sect code.   */
+    return PDMCritSectIsOwner(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectIsInitialized} */
+static DECLCALLBACK(bool)     pdmR3DevHlp_CritSectIsInitialized(PPDMDEVINS pDevIns, PCPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return PDMCritSectIsInitialized(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectHasWaiters} */
+static DECLCALLBACK(bool)     pdmR3DevHlp_CritSectHasWaiters(PPDMDEVINS pDevIns, PCPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return PDMCritSectHasWaiters(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectGetRecursion} */
+static DECLCALLBACK(uint32_t) pdmR3DevHlp_CritSectGetRecursion(PPDMDEVINS pDevIns, PCPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return PDMCritSectGetRecursion(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectScheduleExitEvent} */
+static DECLCALLBACK(int) pdmR3DevHlp_CritSectScheduleExitEvent(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect,
+                                                               SUPSEMEVENT hEventToSignal)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return PDMHCCritSectScheduleExitEvent(pCritSect, hEventToSignal);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectDelete} */
+static DECLCALLBACK(int) pdmR3DevHlp_CritSectDelete(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    RT_NOREF(pDevIns);
+    return PDMR3CritSectDelete(pCritSect);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnThreadCreate} */
+static DECLCALLBACK(int) pdmR3DevHlp_ThreadCreate(PPDMDEVINS pDevIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDEV pfnThread,
+                                                  PFNPDMTHREADWAKEUPDEV pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_ThreadCreate: caller='%s'/%d: ppThread=%p pvUser=%p pfnThread=%p pfnWakeup=%p cbStack=%#zx enmType=%d pszName=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName, pszName));
+
+    int rc = pdmR3ThreadCreateDevice(pDevIns->Internal.s.pVMR3, pDevIns, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName);
+
+    LogFlow(("pdmR3DevHlp_ThreadCreate: caller='%s'/%d: returns %Rrc *ppThread=%RTthrd\n", pDevIns->pReg->szName, pDevIns->iInstance,
+            rc, *ppThread));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSetAsyncNotification} */
+static DECLCALLBACK(int) pdmR3DevHlp_SetAsyncNotification(PPDMDEVINS pDevIns, PFNPDMDEVASYNCNOTIFY pfnAsyncNotify)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT0(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_SetAsyncNotification: caller='%s'/%d: pfnAsyncNotify=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pfnAsyncNotify));
+
+    int rc = VINF_SUCCESS;
+    AssertStmt(pfnAsyncNotify, rc = VERR_INVALID_PARAMETER);
+    AssertStmt(!pDevIns->Internal.s.pfnAsyncNotify, rc = VERR_WRONG_ORDER);
+    AssertStmt(pDevIns->Internal.s.fIntFlags & (PDMDEVINSINT_FLAGS_SUSPENDED | PDMDEVINSINT_FLAGS_RESET), rc = VERR_WRONG_ORDER);
+    VMSTATE enmVMState = VMR3GetState(pDevIns->Internal.s.pVMR3);
+    AssertStmt(   enmVMState == VMSTATE_SUSPENDING
+               || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+               || enmVMState == VMSTATE_SUSPENDING_LS
+               || enmVMState == VMSTATE_RESETTING
+               || enmVMState == VMSTATE_RESETTING_LS
+               || enmVMState == VMSTATE_POWERING_OFF
+               || enmVMState == VMSTATE_POWERING_OFF_LS,
+               rc = VERR_INVALID_STATE);
+
+    if (RT_SUCCESS(rc))
+        pDevIns->Internal.s.pfnAsyncNotify = pfnAsyncNotify;
+
+    LogFlow(("pdmR3DevHlp_SetAsyncNotification: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnAsyncNotificationCompleted} */
+static DECLCALLBACK(void) pdmR3DevHlp_AsyncNotificationCompleted(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    if (   enmVMState == VMSTATE_SUSPENDING
+        || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+        || enmVMState == VMSTATE_SUSPENDING_LS
+        || enmVMState == VMSTATE_RESETTING
+        || enmVMState == VMSTATE_RESETTING_LS
+        || enmVMState == VMSTATE_POWERING_OFF
+        || enmVMState == VMSTATE_POWERING_OFF_LS)
+    {
+        LogFlow(("pdmR3DevHlp_AsyncNotificationCompleted: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+        VMR3AsyncPdmNotificationWakeupU(pVM->pUVM);
+    }
+    else
+        LogFlow(("pdmR3DevHlp_AsyncNotificationCompleted: caller='%s'/%d: enmVMState=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, enmVMState));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnRTCRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_RTCRegister(PPDMDEVINS pDevIns, PCPDMRTCREG pRtcReg, PCPDMRTCHLP *ppRtcHlp)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: pRtcReg=%p:{.u32Version=%#x, .pfnWrite=%p, .pfnRead=%p} ppRtcHlp=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pRtcReg, pRtcReg->u32Version, pRtcReg->pfnWrite,
+             pRtcReg->pfnWrite, ppRtcHlp));
+
+    /*
+     * Validate input.
+     */
+    if (pRtcReg->u32Version != PDM_RTCREG_VERSION)
+    {
+        AssertMsgFailed(("u32Version=%#x expected %#x\n", pRtcReg->u32Version,
+                         PDM_RTCREG_VERSION));
+        LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc (version)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    !pRtcReg->pfnWrite
+        ||  !pRtcReg->pfnRead)
+    {
+        Assert(pRtcReg->pfnWrite);
+        Assert(pRtcReg->pfnRead);
+        LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc (callbacks)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (!ppRtcHlp)
+    {
+        Assert(ppRtcHlp);
+        LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc (ppRtcHlp)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Only one DMA device.
+     */
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    if (pVM->pdm.s.pRtc)
+    {
+        AssertMsgFailed(("Only one RTC device is supported!\n"));
+        LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Allocate and initialize pci bus structure.
+     */
+    int rc = VINF_SUCCESS;
+    PPDMRTC pRtc = (PPDMRTC)MMR3HeapAlloc(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE, sizeof(*pRtc));
+    if (pRtc)
+    {
+        pRtc->pDevIns   = pDevIns;
+        pRtc->Reg       = *pRtcReg;
+        pVM->pdm.s.pRtc = pRtc;
+
+        /* set the helper pointer. */
+        *ppRtcHlp = &g_pdmR3DevRtcHlp;
+        Log(("PDM: Registered RTC device '%s'/%d pDevIns=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMARegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_DMARegister(PPDMDEVINS pDevIns, unsigned uChannel, PFNDMATRANSFERHANDLER pfnTransferHandler, void *pvUser)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DMARegister: caller='%s'/%d: uChannel=%d pfnTransferHandler=%p pvUser=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uChannel, pfnTransferHandler, pvUser));
+    int rc = VINF_SUCCESS;
+    if (pVM->pdm.s.pDmac)
+        pVM->pdm.s.pDmac->Reg.pfnRegister(pVM->pdm.s.pDmac->pDevIns, uChannel, pDevIns, pfnTransferHandler, pvUser);
+    else
+    {
+        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
+        rc = VERR_PDM_NO_DMAC_INSTANCE;
+    }
+    LogFlow(("pdmR3DevHlp_DMARegister: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMAReadMemory} */
+static DECLCALLBACK(int) pdmR3DevHlp_DMAReadMemory(PPDMDEVINS pDevIns, unsigned uChannel, void *pvBuffer, uint32_t off, uint32_t cbBlock, uint32_t *pcbRead)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DMAReadMemory: caller='%s'/%d: uChannel=%d pvBuffer=%p off=%#x cbBlock=%#x pcbRead=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uChannel, pvBuffer, off, cbBlock, pcbRead));
+    int rc = VINF_SUCCESS;
+    if (pVM->pdm.s.pDmac)
+    {
+        uint32_t cb = pVM->pdm.s.pDmac->Reg.pfnReadMemory(pVM->pdm.s.pDmac->pDevIns, uChannel, pvBuffer, off, cbBlock);
+        if (pcbRead)
+            *pcbRead = cb;
+    }
+    else
+    {
+        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
+        rc = VERR_PDM_NO_DMAC_INSTANCE;
+    }
+    LogFlow(("pdmR3DevHlp_DMAReadMemory: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMAWriteMemory} */
+static DECLCALLBACK(int) pdmR3DevHlp_DMAWriteMemory(PPDMDEVINS pDevIns, unsigned uChannel, const void *pvBuffer, uint32_t off, uint32_t cbBlock, uint32_t *pcbWritten)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DMAWriteMemory: caller='%s'/%d: uChannel=%d pvBuffer=%p off=%#x cbBlock=%#x pcbWritten=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uChannel, pvBuffer, off, cbBlock, pcbWritten));
+    int rc = VINF_SUCCESS;
+    if (pVM->pdm.s.pDmac)
+    {
+        uint32_t cb = pVM->pdm.s.pDmac->Reg.pfnWriteMemory(pVM->pdm.s.pDmac->pDevIns, uChannel, pvBuffer, off, cbBlock);
+        if (pcbWritten)
+            *pcbWritten = cb;
+    }
+    else
+    {
+        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
+        rc = VERR_PDM_NO_DMAC_INSTANCE;
+    }
+    LogFlow(("pdmR3DevHlp_DMAWriteMemory: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMASetDREQ} */
+static DECLCALLBACK(int) pdmR3DevHlp_DMASetDREQ(PPDMDEVINS pDevIns, unsigned uChannel, unsigned uLevel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DMASetDREQ: caller='%s'/%d: uChannel=%d uLevel=%d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uChannel, uLevel));
+    int rc = VINF_SUCCESS;
+    if (pVM->pdm.s.pDmac)
+        pVM->pdm.s.pDmac->Reg.pfnSetDREQ(pVM->pdm.s.pDmac->pDevIns, uChannel, uLevel);
+    else
+    {
+        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
+        rc = VERR_PDM_NO_DMAC_INSTANCE;
+    }
+    LogFlow(("pdmR3DevHlp_DMASetDREQ: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMAGetChannelMode} */
+static DECLCALLBACK(uint8_t) pdmR3DevHlp_DMAGetChannelMode(PPDMDEVINS pDevIns, unsigned uChannel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DMAGetChannelMode: caller='%s'/%d: uChannel=%d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uChannel));
+    uint8_t u8Mode;
+    if (pVM->pdm.s.pDmac)
+        u8Mode = pVM->pdm.s.pDmac->Reg.pfnGetChannelMode(pVM->pdm.s.pDmac->pDevIns, uChannel);
+    else
+    {
+        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
+        u8Mode = 3 << 2 /* illegal mode type */;
+    }
+    LogFlow(("pdmR3DevHlp_DMAGetChannelMode: caller='%s'/%d: returns %#04x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, u8Mode));
+    return u8Mode;
+}
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMASchedule} */
+static DECLCALLBACK(void) pdmR3DevHlp_DMASchedule(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DMASchedule: caller='%s'/%d: VM_FF_PDM_DMA %d -> 1\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, VM_FF_IS_SET(pVM, VM_FF_PDM_DMA)));
+
+    AssertMsg(pVM->pdm.s.pDmac, ("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
+    VM_FF_SET(pVM, VM_FF_PDM_DMA);
+    VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_DONE_REM);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCMOSWrite} */
+static DECLCALLBACK(int) pdmR3DevHlp_CMOSWrite(PPDMDEVINS pDevIns, unsigned iReg, uint8_t u8Value)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    LogFlow(("pdmR3DevHlp_CMOSWrite: caller='%s'/%d: iReg=%#04x u8Value=%#04x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iReg, u8Value));
+    int rc;
+    if (pVM->pdm.s.pRtc)
+    {
+        PPDMDEVINS pDevInsRtc = pVM->pdm.s.pRtc->pDevIns;
+        rc = PDMCritSectEnter(pDevInsRtc->pCritSectRoR3, VERR_IGNORED);
+        if (RT_SUCCESS(rc))
+        {
+            rc = pVM->pdm.s.pRtc->Reg.pfnWrite(pDevInsRtc, iReg, u8Value);
+            PDMCritSectLeave(pDevInsRtc->pCritSectRoR3);
+        }
+    }
+    else
+        rc = VERR_PDM_NO_RTC_INSTANCE;
+
+    LogFlow(("pdmR3DevHlp_CMOSWrite: caller='%s'/%d: return %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCMOSRead} */
+static DECLCALLBACK(int) pdmR3DevHlp_CMOSRead(PPDMDEVINS pDevIns, unsigned iReg, uint8_t *pu8Value)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    LogFlow(("pdmR3DevHlp_CMOSWrite: caller='%s'/%d: iReg=%#04x pu8Value=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iReg, pu8Value));
+    int rc;
+    if (pVM->pdm.s.pRtc)
+    {
+        PPDMDEVINS pDevInsRtc = pVM->pdm.s.pRtc->pDevIns;
+        rc = PDMCritSectEnter(pDevInsRtc->pCritSectRoR3, VERR_IGNORED);
+        if (RT_SUCCESS(rc))
+        {
+            rc = pVM->pdm.s.pRtc->Reg.pfnRead(pDevInsRtc, iReg, pu8Value);
+            PDMCritSectLeave(pDevInsRtc->pCritSectRoR3);
+        }
+    }
+    else
+        rc = VERR_PDM_NO_RTC_INSTANCE;
+
+    LogFlow(("pdmR3DevHlp_CMOSWrite: caller='%s'/%d: return %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnAssertEMT} */
+static DECLCALLBACK(bool) pdmR3DevHlp_AssertEMT(PPDMDEVINS pDevIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    if (VM_IS_EMT(pDevIns->Internal.s.pVMR3))
+        return true;
+
+    char szMsg[100];
+    RTStrPrintf(szMsg, sizeof(szMsg), "AssertEMT '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance);
+    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
+    AssertBreakpoint();
+    return false;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnAssertOther} */
+static DECLCALLBACK(bool) pdmR3DevHlp_AssertOther(PPDMDEVINS pDevIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    if (!VM_IS_EMT(pDevIns->Internal.s.pVMR3))
+        return true;
+
+    char szMsg[100];
+    RTStrPrintf(szMsg, sizeof(szMsg), "AssertOther '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance);
+    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
+    AssertBreakpoint();
+    return false;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnLdrGetRCInterfaceSymbols} */
+static DECLCALLBACK(int) pdmR3DevHlp_LdrGetRCInterfaceSymbols(PPDMDEVINS pDevIns, void *pvInterface, size_t cbInterface,
+                                                              const char *pszSymPrefix, const char *pszSymList)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_PDMLdrGetRCInterfaceSymbols: caller='%s'/%d: pvInterface=%p cbInterface=%zu pszSymPrefix=%p:{%s} pszSymList=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pvInterface, cbInterface, pszSymPrefix, pszSymPrefix, pszSymList, pszSymList));
+
+    int rc;
+    if (   strncmp(pszSymPrefix, "dev", 3) == 0
+        && RTStrIStr(pszSymPrefix + 3, pDevIns->pReg->szName) != NULL)
+    {
+        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC)
+            rc = PDMR3LdrGetInterfaceSymbols(pDevIns->Internal.s.pVMR3,
+                                             pvInterface, cbInterface,
+                                             pDevIns->pReg->pszRCMod, pDevIns->Internal.s.pDevR3->pszRCSearchPath,
+                                             pszSymPrefix, pszSymList,
+                                             false /*fRing0OrRC*/);
+        else
+        {
+            AssertMsgFailed(("Not a raw-mode enabled driver\n"));
+            rc = VERR_PERMISSION_DENIED;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Invalid prefix '%s' for '%s'; must start with 'dev' and contain the driver name!\n",
+                         pszSymPrefix, pDevIns->pReg->szName));
+        rc = VERR_INVALID_NAME;
+    }
+
+    LogFlow(("pdmR3DevHlp_PDMLdrGetRCInterfaceSymbols: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName,
+             pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnLdrGetR0InterfaceSymbols} */
+static DECLCALLBACK(int) pdmR3DevHlp_LdrGetR0InterfaceSymbols(PPDMDEVINS pDevIns, void *pvInterface, size_t cbInterface,
+                                                              const char *pszSymPrefix, const char *pszSymList)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_PDMLdrGetR0InterfaceSymbols: caller='%s'/%d: pvInterface=%p cbInterface=%zu pszSymPrefix=%p:{%s} pszSymList=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pvInterface, cbInterface, pszSymPrefix, pszSymPrefix, pszSymList, pszSymList));
+
+    int rc;
+    if (   strncmp(pszSymPrefix, "dev", 3) == 0
+        && RTStrIStr(pszSymPrefix + 3, pDevIns->pReg->szName) != NULL)
+    {
+        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0)
+            rc = PDMR3LdrGetInterfaceSymbols(pDevIns->Internal.s.pVMR3,
+                                             pvInterface, cbInterface,
+                                             pDevIns->pReg->pszR0Mod, pDevIns->Internal.s.pDevR3->pszR0SearchPath,
+                                             pszSymPrefix, pszSymList,
+                                             true /*fRing0OrRC*/);
+        else
+        {
+            AssertMsgFailed(("Not a ring-0 enabled driver\n"));
+            rc = VERR_PERMISSION_DENIED;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Invalid prefix '%s' for '%s'; must start with 'dev' and contain the driver name!\n",
+                         pszSymPrefix, pDevIns->pReg->szName));
+        rc = VERR_INVALID_NAME;
+    }
+
+    LogFlow(("pdmR3DevHlp_PDMLdrGetR0InterfaceSymbols: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName,
+             pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCallR0} */
+static DECLCALLBACK(int) pdmR3DevHlp_CallR0(PPDMDEVINS pDevIns, uint32_t uOperation, uint64_t u64Arg)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM    pVM = pDevIns->Internal.s.pVMR3;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    AssertReturn(pVCpu, VERR_VM_THREAD_IS_EMT);
+    LogFlow(("pdmR3DevHlp_CallR0: caller='%s'/%d: uOperation=%#x u64Arg=%#RX64\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uOperation, u64Arg));
+
+    /*
+     * Resolve the ring-0 entry point.  There is not need to remember this like
+     * we do for drivers since this is mainly for construction time hacks and
+     * other things that aren't performance critical.
+     */
+    int rc;
+    if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0)
+    {
+        /*
+         * Make the ring-0 call.
+         */
+        PDMDEVICEGENCALLREQ Req;
+        RT_ZERO(Req.Params);
+        Req.Hdr.u32Magic    = SUPVMMR0REQHDR_MAGIC;
+        Req.Hdr.cbReq       = sizeof(Req);
+        Req.pDevInsR3       = pDevIns;
+        Req.idxR0Device     = pDevIns->Internal.s.idxR0Device;
+        Req.enmCall         = PDMDEVICEGENCALL_REQUEST;
+        Req.Params.Req.uReq = uOperation;
+        Req.Params.Req.uArg = u64Arg;
+        rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_PDM_DEVICE_GEN_CALL, 0, &Req.Hdr);
+    }
+    else
+        rc = VERR_ACCESS_DENIED;
+    LogFlow(("pdmR3DevHlp_CallR0: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName,
+             pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMGetSuspendReason} */
+static DECLCALLBACK(VMSUSPENDREASON) pdmR3DevHlp_VMGetSuspendReason(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    VMSUSPENDREASON enmReason = VMR3GetSuspendReason(pVM->pUVM);
+    LogFlow(("pdmR3DevHlp_VMGetSuspendReason: caller='%s'/%d: returns %d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, enmReason));
+    return enmReason;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMGetResumeReason} */
+static DECLCALLBACK(VMRESUMEREASON) pdmR3DevHlp_VMGetResumeReason(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    VMRESUMEREASON enmReason = VMR3GetResumeReason(pVM->pUVM);
+    LogFlow(("pdmR3DevHlp_VMGetResumeReason: caller='%s'/%d: returns %d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, enmReason));
+    return enmReason;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetUVM} */
+static DECLCALLBACK(PUVM) pdmR3DevHlp_GetUVM(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_GetUVM: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns->Internal.s.pVMR3));
+    return pDevIns->Internal.s.pVMR3->pUVM;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetVM} */
+static DECLCALLBACK(PVM) pdmR3DevHlp_GetVM(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_GetVM: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns->Internal.s.pVMR3));
+    return pDevIns->Internal.s.pVMR3;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetVMCPU} */
+static DECLCALLBACK(PVMCPU) pdmR3DevHlp_GetVMCPU(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_GetVMCPU: caller='%s'/%d for CPU %u\n", pDevIns->pReg->szName, pDevIns->iInstance, VMMGetCpuId(pDevIns->Internal.s.pVMR3)));
+    return VMMGetCpu(pDevIns->Internal.s.pVMR3);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetCurrentCpuId} */
+static DECLCALLBACK(VMCPUID) pdmR3DevHlp_GetCurrentCpuId(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VMCPUID idCpu = VMMGetCpuId(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_GetCurrentCpuId: caller='%s'/%d for CPU %u\n", pDevIns->pReg->szName, pDevIns->iInstance, idCpu));
+    return idCpu;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCIBusRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_PCIBusRegister(PPDMDEVINS pDevIns, PPDMPCIBUSREGR3 pPciBusReg,
+                                                    PCPDMPCIHLPR3 *ppPciHlp, uint32_t *piBus)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: pPciBusReg=%p:{.u32Version=%#x, .pfnRegisterR3=%p, .pfnIORegionRegisterR3=%p, "
+             ".pfnInterceptConfigAccesses=%p, pfnConfigRead=%p, pfnConfigWrite=%p, .pfnSetIrqR3=%p, .u32EndVersion=%#x} ppPciHlpR3=%p piBus=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPciBusReg, pPciBusReg->u32Version, pPciBusReg->pfnRegisterR3,
+             pPciBusReg->pfnIORegionRegisterR3, pPciBusReg->pfnInterceptConfigAccesses, pPciBusReg->pfnConfigRead,
+             pPciBusReg->pfnConfigWrite, pPciBusReg->pfnSetIrqR3, pPciBusReg->u32EndVersion, ppPciHlp, piBus));
+
+    /*
+     * Validate the structure and output parameters.
+     */
+    AssertLogRelMsgReturn(pPciBusReg->u32Version == PDM_PCIBUSREGR3_VERSION,
+                          ("u32Version=%#x expected %#x\n", pPciBusReg->u32Version, PDM_PCIBUSREGR3_VERSION),
+                          VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pPciBusReg->pfnRegisterR3, VERR_INVALID_PARAMETER);
+    AssertPtrNullReturn(pPciBusReg->pfnRegisterMsiR3, VERR_INVALID_POINTER);
+    AssertPtrReturn(pPciBusReg->pfnIORegionRegisterR3, VERR_INVALID_POINTER);
+    AssertPtrReturn(pPciBusReg->pfnInterceptConfigAccesses, VERR_INVALID_POINTER);
+    AssertPtrReturn(pPciBusReg->pfnConfigWrite, VERR_INVALID_POINTER);
+    AssertPtrReturn(pPciBusReg->pfnConfigRead, VERR_INVALID_POINTER);
+    AssertPtrReturn(pPciBusReg->pfnSetIrqR3, VERR_INVALID_POINTER);
+    AssertLogRelMsgReturn(pPciBusReg->u32EndVersion == PDM_PCIBUSREGR3_VERSION,
+                          ("u32Version=%#x expected %#x\n", pPciBusReg->u32Version, PDM_PCIBUSREGR3_VERSION),
+                          VERR_INVALID_PARAMETER);
+    AssertPtrReturn(ppPciHlp, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(piBus, VERR_INVALID_POINTER);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_WRONG_ORDER);
+
+    /*
+     * Find free PCI bus entry.
+     */
+    unsigned iBus = 0;
+    for (iBus = 0; iBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses); iBus++)
+        if (!pVM->pdm.s.aPciBuses[iBus].pDevInsR3)
+            break;
+    AssertLogRelMsgReturn(iBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses),
+                          ("Too many PCI buses. Max=%u\n", RT_ELEMENTS(pVM->pdm.s.aPciBuses)),
+                          VERR_OUT_OF_RESOURCES);
+    PPDMPCIBUS pPciBus = &pVM->pdm.s.aPciBuses[iBus];
+
+    /*
+     * Init the R3 bits.
+     */
+    pPciBus->iBus                       = iBus;
+    pPciBus->pDevInsR3                  = pDevIns;
+    pPciBus->pfnRegister                = pPciBusReg->pfnRegisterR3;
+    pPciBus->pfnRegisterMsi             = pPciBusReg->pfnRegisterMsiR3;
+    pPciBus->pfnIORegionRegister        = pPciBusReg->pfnIORegionRegisterR3;
+    pPciBus->pfnInterceptConfigAccesses = pPciBusReg->pfnInterceptConfigAccesses;
+    pPciBus->pfnConfigRead              = pPciBusReg->pfnConfigRead;
+    pPciBus->pfnConfigWrite             = pPciBusReg->pfnConfigWrite;
+    pPciBus->pfnSetIrqR3                = pPciBusReg->pfnSetIrqR3;
+
+    Log(("PDM: Registered PCI bus device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+
+    /* set the helper pointer and return. */
+    *ppPciHlp = &g_pdmR3DevPciHlp;
+    if (piBus)
+        *piBus = iBus;
+    LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc *piBus=%u\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS, iBus));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPICRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_PICRegister(PPDMDEVINS pDevIns, PPDMPICREG pPicReg, PCPDMPICHLP *ppPicHlp)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: pPicReg=%p:{.u32Version=%#x, .pfnSetIrq=%p, .pfnGetInterrupt=%p, .u32TheEnd=%#x } ppPicHlp=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPicReg, pPicReg->u32Version, pPicReg->pfnSetIrq, pPicReg->pfnGetInterrupt, pPicReg->u32TheEnd, ppPicHlp));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(pPicReg->u32Version == PDM_PICREG_VERSION,
+                    ("%s/%d: u32Version=%#x expected %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, pPicReg->u32Version, PDM_PICREG_VERSION),
+                    VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pPicReg->pfnSetIrq, VERR_INVALID_POINTER);
+    AssertPtrReturn(pPicReg->pfnGetInterrupt, VERR_INVALID_POINTER);
+    AssertMsgReturn(pPicReg->u32TheEnd == PDM_PICREG_VERSION,
+                    ("%s/%d: u32TheEnd=%#x expected %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, pPicReg->u32TheEnd, PDM_PICREG_VERSION),
+                    VERR_INVALID_PARAMETER);
+    AssertPtrReturn(ppPicHlp, VERR_INVALID_POINTER);
+
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_WRONG_ORDER);
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    /*
+     * Only one PIC device.
+     */
+    AssertMsgReturn(pVM->pdm.s.Pic.pDevInsR3 == NULL, ("%s/%d: Only one PIC!\n", pDevIns->pReg->szName, pDevIns->iInstance),
+                    VERR_ALREADY_EXISTS);
+
+    /*
+     * Take down the callbacks and instance.
+     */
+    pVM->pdm.s.Pic.pDevInsR3 = pDevIns;
+    pVM->pdm.s.Pic.pfnSetIrqR3 = pPicReg->pfnSetIrq;
+    pVM->pdm.s.Pic.pfnGetInterruptR3 = pPicReg->pfnGetInterrupt;
+    Log(("PDM: Registered PIC device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+
+    /* set the helper pointer and return. */
+    *ppPicHlp = &g_pdmR3DevPicHlp;
+    LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnApicRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_ApicRegister(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+
+    /*
+     * Validate caller context.
+     */
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_WRONG_ORDER);
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    /*
+     * Only one APIC device. On SMP we have single logical device covering all LAPICs,
+     * as they need to communicate and share state easily.
+     */
+    AssertMsgReturn(pVM->pdm.s.Apic.pDevInsR3 == NULL,
+                    ("%s/%u: Only one APIC device is supported!\n", pDevIns->pReg->szName, pDevIns->iInstance),
+                    VERR_ALREADY_EXISTS);
+
+    /*
+     * Set the ring-3 and raw-mode bits, leave the ring-0 to ring-0 setup.
+     */
+    pVM->pdm.s.Apic.pDevInsR3 = pDevIns;
+    pVM->pdm.s.Apic.pDevInsRC = PDMDEVINS_2_RCPTR(pDevIns);
+    Assert(pVM->pdm.s.Apic.pDevInsRC || !VM_IS_RAW_MODE_ENABLED(pVM));
+
+    LogFlow(("pdmR3DevHlp_ApicRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnIoApicRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_IoApicRegister(PPDMDEVINS pDevIns, PPDMIOAPICREG pIoApicReg, PCPDMIOAPICHLP *ppIoApicHlp)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_IoApicRegister: caller='%s'/%d: pIoApicReg=%p:{.u32Version=%#x, .pfnSetIrq=%p, .pfnSendMsi=%p, .pfnSetEoi=%p, .u32TheEnd=%#x } ppIoApicHlp=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pIoApicReg, pIoApicReg->u32Version, pIoApicReg->pfnSetIrq, pIoApicReg->pfnSendMsi, pIoApicReg->pfnSetEoi, pIoApicReg->u32TheEnd, ppIoApicHlp));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(pIoApicReg->u32Version == PDM_IOAPICREG_VERSION,
+                    ("%s/%d: u32Version=%#x expected %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, pIoApicReg->u32Version, PDM_IOAPICREG_VERSION),
+                    VERR_VERSION_MISMATCH);
+    AssertPtrReturn(pIoApicReg->pfnSetIrq, VERR_INVALID_POINTER);
+    AssertPtrReturn(pIoApicReg->pfnSendMsi, VERR_INVALID_POINTER);
+    AssertPtrReturn(pIoApicReg->pfnSetEoi, VERR_INVALID_POINTER);
+    AssertMsgReturn(pIoApicReg->u32TheEnd == PDM_IOAPICREG_VERSION,
+                    ("%s/%d: u32TheEnd=%#x expected %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, pIoApicReg->u32TheEnd, PDM_IOAPICREG_VERSION),
+                    VERR_VERSION_MISMATCH);
+    AssertPtrReturn(ppIoApicHlp, VERR_INVALID_POINTER);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_WRONG_ORDER);
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    /*
+     * The I/O APIC requires the APIC to be present (hacks++).
+     * If the I/O APIC does GC stuff so must the APIC.
+     */
+    AssertMsgReturn(pVM->pdm.s.Apic.pDevInsR3 != NULL, ("Configuration error / Init order error! No APIC!\n"), VERR_WRONG_ORDER);
+
+    /*
+     * Only one I/O APIC device.
+     */
+    AssertMsgReturn(pVM->pdm.s.IoApic.pDevInsR3 == NULL,
+                    ("Only one IOAPIC device is supported! (caller %s/%d)\n", pDevIns->pReg->szName, pDevIns->iInstance),
+                    VERR_ALREADY_EXISTS);
+
+    /*
+     * Initialize the R3 bits.
+     */
+    pVM->pdm.s.IoApic.pDevInsR3    = pDevIns;
+    pVM->pdm.s.IoApic.pfnSetIrqR3  = pIoApicReg->pfnSetIrq;
+    pVM->pdm.s.IoApic.pfnSendMsiR3 = pIoApicReg->pfnSendMsi;
+    pVM->pdm.s.IoApic.pfnSetEoiR3  = pIoApicReg->pfnSetEoi;
+    Log(("PDM: Registered I/O APIC device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+
+    /* set the helper pointer and return. */
+    *ppIoApicHlp = &g_pdmR3DevIoApicHlp;
+    LogFlow(("pdmR3DevHlp_IoApicRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnHpetRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_HpetRegister(PPDMDEVINS pDevIns, PPDMHPETREG pHpetReg, PCPDMHPETHLPR3 *ppHpetHlpR3)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_HpetRegister: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(pHpetReg->u32Version == PDM_HPETREG_VERSION,
+                    ("%s/%u: u32Version=%#x expected %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, pHpetReg->u32Version, PDM_HPETREG_VERSION),
+                    VERR_VERSION_MISMATCH);
+    AssertPtrReturn(ppHpetHlpR3, VERR_INVALID_POINTER);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_WRONG_ORDER);
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    /*
+     * Only one HPET device.
+     */
+    AssertMsgReturn(pVM->pdm.s.pHpet == NULL,
+                    ("Only one HPET device is supported! (caller %s/%d)\n", pDevIns->pReg->szName, pDevIns->iInstance),
+                    VERR_ALREADY_EXISTS);
+
+    /*
+     * Do the job (what there is of it).
+     */
+    pVM->pdm.s.pHpet = pDevIns;
+    *ppHpetHlpR3 = &g_pdmR3DevHpetHlp;
+
+    LogFlow(("pdmR3DevHlp_HpetRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPciRawRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_PciRawRegister(PPDMDEVINS pDevIns, PPDMPCIRAWREG pPciRawReg, PCPDMPCIRAWHLPR3 *ppPciRawHlpR3)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns); RT_NOREF_PV(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_PciRawRegister: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+
+    /*
+     * Validate input.
+     */
+    if (pPciRawReg->u32Version != PDM_PCIRAWREG_VERSION)
+    {
+        AssertMsgFailed(("u32Version=%#x expected %#x\n", pPciRawReg->u32Version, PDM_PCIRAWREG_VERSION));
+        LogFlow(("pdmR3DevHlp_PciRawRegister: caller='%s'/%d: returns %Rrc (version)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (!ppPciRawHlpR3)
+    {
+        Assert(ppPciRawHlpR3);
+        LogFlow(("pdmR3DevHlp_PciRawRegister: caller='%s'/%d: returns %Rrc (ppPciRawHlpR3)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /* set the helper pointer and return. */
+    *ppPciRawHlpR3 = &g_pdmR3DevPciRawHlp;
+    LogFlow(("pdmR3DevHlp_PciRawRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMACRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_DMACRegister(PPDMDEVINS pDevIns, PPDMDMACREG pDmacReg, PCPDMDMACHLP *ppDmacHlp)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: pDmacReg=%p:{.u32Version=%#x, .pfnRun=%p, .pfnRegister=%p, .pfnReadMemory=%p, .pfnWriteMemory=%p, .pfnSetDREQ=%p, .pfnGetChannelMode=%p} ppDmacHlp=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pDmacReg, pDmacReg->u32Version, pDmacReg->pfnRun, pDmacReg->pfnRegister,
+             pDmacReg->pfnReadMemory, pDmacReg->pfnWriteMemory, pDmacReg->pfnSetDREQ, pDmacReg->pfnGetChannelMode, ppDmacHlp));
+
+    /*
+     * Validate input.
+     */
+    if (pDmacReg->u32Version != PDM_DMACREG_VERSION)
+    {
+        AssertMsgFailed(("u32Version=%#x expected %#x\n", pDmacReg->u32Version,
+                         PDM_DMACREG_VERSION));
+        LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc (version)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    !pDmacReg->pfnRun
+        ||  !pDmacReg->pfnRegister
+        ||  !pDmacReg->pfnReadMemory
+        ||  !pDmacReg->pfnWriteMemory
+        ||  !pDmacReg->pfnSetDREQ
+        ||  !pDmacReg->pfnGetChannelMode)
+    {
+        Assert(pDmacReg->pfnRun);
+        Assert(pDmacReg->pfnRegister);
+        Assert(pDmacReg->pfnReadMemory);
+        Assert(pDmacReg->pfnWriteMemory);
+        Assert(pDmacReg->pfnSetDREQ);
+        Assert(pDmacReg->pfnGetChannelMode);
+        LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc (callbacks)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (!ppDmacHlp)
+    {
+        Assert(ppDmacHlp);
+        LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc (ppDmacHlp)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Only one DMA device.
+     */
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    if (pVM->pdm.s.pDmac)
+    {
+        AssertMsgFailed(("Only one DMA device is supported!\n"));
+        LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Allocate and initialize pci bus structure.
+     */
+    int rc = VINF_SUCCESS;
+    PPDMDMAC  pDmac = (PPDMDMAC)MMR3HeapAlloc(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE, sizeof(*pDmac));
+    if (pDmac)
+    {
+        pDmac->pDevIns   = pDevIns;
+        pDmac->Reg       = *pDmacReg;
+        pVM->pdm.s.pDmac = pDmac;
+
+        /* set the helper pointer. */
+        *ppDmacHlp = &g_pdmR3DevDmacHlp;
+        Log(("PDM: Registered DMAC device '%s'/%d pDevIns=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/**
+ * @copydoc PDMDEVHLPR3::pfnRegisterVMMDevHeap
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_RegisterVMMDevHeap(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTR3PTR pvHeap, unsigned cbHeap)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_RegisterVMMDevHeap: caller='%s'/%d: GCPhys=%RGp pvHeap=%p cbHeap=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, pvHeap, cbHeap));
+
+    if (pVM->pdm.s.pvVMMDevHeap == NULL)
+    {
+        pVM->pdm.s.pvVMMDevHeap     = pvHeap;
+        pVM->pdm.s.GCPhysVMMDevHeap = GCPhys;
+        pVM->pdm.s.cbVMMDevHeap     = cbHeap;
+        pVM->pdm.s.cbVMMDevHeapLeft = cbHeap;
+    }
+    else
+    {
+        Assert(pVM->pdm.s.pvVMMDevHeap == pvHeap);
+        Assert(pVM->pdm.s.cbVMMDevHeap == cbHeap);
+        Assert(pVM->pdm.s.GCPhysVMMDevHeap != GCPhys || GCPhys == NIL_RTGCPHYS);
+        if (pVM->pdm.s.GCPhysVMMDevHeap != GCPhys)
+        {
+            pVM->pdm.s.GCPhysVMMDevHeap = GCPhys;
+            if (pVM->pdm.s.pfnVMMDevHeapNotify)
+                pVM->pdm.s.pfnVMMDevHeapNotify(pVM, pvHeap, GCPhys);
+        }
+    }
+
+    LogFlow(("pdmR3DevHlp_RegisterVMMDevHeap: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @interface_method_impl{PDMDEVHLPR3,pfnFirmwareRegister}
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_FirmwareRegister(PPDMDEVINS pDevIns, PCPDMFWREG pFwReg, PCPDMFWHLPR3 *ppFwHlp)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_FirmwareRegister: caller='%s'/%d: pFWReg=%p:{.u32Version=%#x, .pfnIsHardReset=%p, .u32TheEnd=%#x} ppFwHlp=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pFwReg, pFwReg->u32Version, pFwReg->pfnIsHardReset, pFwReg->u32TheEnd, ppFwHlp));
+
+    /*
+     * Validate input.
+     */
+    if (pFwReg->u32Version != PDM_FWREG_VERSION)
+    {
+        AssertMsgFailed(("u32Version=%#x expected %#x\n", pFwReg->u32Version, PDM_FWREG_VERSION));
+        LogFlow(("pdmR3DevHlp_FirmwareRegister: caller='%s'/%d: returns %Rrc (version)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (!pFwReg->pfnIsHardReset)
+    {
+        Assert(pFwReg->pfnIsHardReset);
+        LogFlow(("pdmR3DevHlp_FirmwareRegister: caller='%s'/%d: returns %Rrc (callbacks)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (!ppFwHlp)
+    {
+        Assert(ppFwHlp);
+        LogFlow(("pdmR3DevHlp_FirmwareRegister: caller='%s'/%d: returns %Rrc (ppFwHlp)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Only one DMA device.
+     */
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    if (pVM->pdm.s.pFirmware)
+    {
+        AssertMsgFailed(("Only one firmware device is supported!\n"));
+        LogFlow(("pdmR3DevHlp_FirmwareRegister: caller='%s'/%d: returns %Rrc\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Allocate and initialize pci bus structure.
+     */
+    int rc = VINF_SUCCESS;
+    PPDMFW pFirmware = (PPDMFW)MMR3HeapAlloc(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE, sizeof(*pFirmware));
+    if (pFirmware)
+    {
+        pFirmware->pDevIns   = pDevIns;
+        pFirmware->Reg       = *pFwReg;
+        pVM->pdm.s.pFirmware = pFirmware;
+
+        /* set the helper pointer. */
+        *ppFwHlp = &g_pdmR3DevFirmwareHlp;
+        Log(("PDM: Registered firmware device '%s'/%d pDevIns=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    LogFlow(("pdmR3DevHlp_FirmwareRegister: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMReset} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMReset(PPDMDEVINS pDevIns, uint32_t fFlags)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_VMReset: caller='%s'/%d: fFlags=%#x VM_FF_RESET %d -> 1\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, fFlags, VM_FF_IS_SET(pVM, VM_FF_RESET)));
+
+    /*
+     * We postpone this operation because we're likely to be inside a I/O instruction
+     * and the EIP will be updated when we return.
+     * We still return VINF_EM_RESET to break out of any execution loops and force FF evaluation.
+     */
+    bool fHaltOnReset;
+    int rc = CFGMR3QueryBool(CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM"), "HaltOnReset", &fHaltOnReset);
+    if (RT_SUCCESS(rc) && fHaltOnReset)
+    {
+        Log(("pdmR3DevHlp_VMReset: Halt On Reset!\n"));
+        rc = VINF_EM_HALT;
+    }
+    else
+    {
+        pVM->pdm.s.fResetFlags = fFlags;
+        VM_FF_SET(pVM, VM_FF_RESET);
+        rc = VINF_EM_RESET;
+    }
+
+    LogFlow(("pdmR3DevHlp_VMReset: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSuspend} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSuspend(PPDMDEVINS pDevIns)
+{
+    int rc;
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_VMSuspend: caller='%s'/%d:\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    /** @todo Always take the SMP path - fewer code paths. */
+    if (pVM->cCpus > 1)
+    {
+        /* We own the IOM lock here and could cause a deadlock by waiting for a VCPU that is blocking on the IOM lock. */
+        rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)VMR3Suspend, 2, pVM->pUVM, VMSUSPENDREASON_VM);
+        AssertRC(rc);
+        rc = VINF_EM_SUSPEND;
+    }
+    else
+        rc = VMR3Suspend(pVM->pUVM, VMSUSPENDREASON_VM);
+
+    LogFlow(("pdmR3DevHlp_VMSuspend: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/**
+ * Worker for pdmR3DevHlp_VMSuspendSaveAndPowerOff that is invoked via a queued
+ * EMT request to avoid deadlocks.
+ *
+ * @returns VBox status code fit for scheduling.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pDevIns             The device that triggered this action.
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSuspendSaveAndPowerOffWorker(PVM pVM, PPDMDEVINS pDevIns)
+{
+    /*
+     * Suspend the VM first then do the saving.
+     */
+    int rc = VMR3Suspend(pVM->pUVM, VMSUSPENDREASON_VM);
+    if (RT_SUCCESS(rc))
+    {
+        PUVM pUVM = pVM->pUVM;
+        rc = pUVM->pVmm2UserMethods->pfnSaveState(pVM->pUVM->pVmm2UserMethods, pUVM);
+
+        /*
+         * On success, power off the VM, on failure we'll leave it suspended.
+         */
+        if (RT_SUCCESS(rc))
+        {
+            rc = VMR3PowerOff(pVM->pUVM);
+            if (RT_FAILURE(rc))
+                LogRel(("%s/SSP: VMR3PowerOff failed: %Rrc\n", pDevIns->pReg->szName, rc));
+        }
+        else
+            LogRel(("%s/SSP: pfnSaveState failed: %Rrc\n", pDevIns->pReg->szName, rc));
+    }
+    else
+        LogRel(("%s/SSP: Suspend failed: %Rrc\n", pDevIns->pReg->szName, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSuspendSaveAndPowerOff} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSuspendSaveAndPowerOff(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_VMSuspendSaveAndPowerOff: caller='%s'/%d:\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    int rc;
+    if (   pVM->pUVM->pVmm2UserMethods
+        && pVM->pUVM->pVmm2UserMethods->pfnSaveState)
+    {
+        rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pdmR3DevHlp_VMSuspendSaveAndPowerOffWorker, 2, pVM, pDevIns);
+        if (RT_SUCCESS(rc))
+        {
+            LogRel(("%s: Suspending, Saving and Powering Off the VM\n", pDevIns->pReg->szName));
+            rc = VINF_EM_SUSPEND;
+        }
+    }
+    else
+        rc = VERR_NOT_SUPPORTED;
+
+    LogFlow(("pdmR3DevHlp_VMSuspendSaveAndPowerOff: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMPowerOff} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMPowerOff(PPDMDEVINS pDevIns)
+{
+    int rc;
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_VMPowerOff: caller='%s'/%d:\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    /** @todo Always take the SMP path - fewer code paths. */
+    if (pVM->cCpus > 1)
+    {
+        /* We might be holding locks here and could cause a deadlock since
+           VMR3PowerOff rendezvous with the other CPUs. */
+        rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)VMR3PowerOff, 1, pVM->pUVM);
+        AssertRC(rc);
+        /* Set the VCPU state to stopped here as well to make sure no
+           inconsistency with the EM state occurs. */
+        VMCPU_SET_STATE(VMMGetCpu(pVM), VMCPUSTATE_STOPPED);
+        rc = VINF_EM_OFF;
+    }
+    else
+        rc = VMR3PowerOff(pVM->pUVM);
+
+    LogFlow(("pdmR3DevHlp_VMPowerOff: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnA20IsEnabled} */
+static DECLCALLBACK(bool) pdmR3DevHlp_A20IsEnabled(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+
+    bool fRc = PGMPhysIsA20Enabled(VMMGetCpu(pDevIns->Internal.s.pVMR3));
+
+    LogFlow(("pdmR3DevHlp_A20IsEnabled: caller='%s'/%d: returns %d\n", pDevIns->pReg->szName, pDevIns->iInstance, fRc));
+    return fRc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnA20Set} */
+static DECLCALLBACK(void) pdmR3DevHlp_A20Set(PPDMDEVINS pDevIns, bool fEnable)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_A20Set: caller='%s'/%d: fEnable=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, fEnable));
+    PGMR3PhysSetA20(VMMGetCpu(pDevIns->Internal.s.pVMR3), fEnable);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetCpuId} */
+static DECLCALLBACK(void) pdmR3DevHlp_GetCpuId(PPDMDEVINS pDevIns, uint32_t iLeaf,
+                                               uint32_t *pEax, uint32_t *pEbx, uint32_t *pEcx, uint32_t *pEdx)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DevHlp_GetCpuId: caller='%s'/%d: iLeaf=%d pEax=%p pEbx=%p pEcx=%p pEdx=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iLeaf, pEax, pEbx, pEcx, pEdx));
+    AssertPtr(pEax); AssertPtr(pEbx); AssertPtr(pEcx); AssertPtr(pEdx);
+
+    CPUMGetGuestCpuId(VMMGetCpu(pDevIns->Internal.s.pVMR3), iLeaf, 0 /*iSubLeaf*/, pEax, pEbx, pEcx, pEdx);
+
+    LogFlow(("pdmR3DevHlp_GetCpuId: caller='%s'/%d: returns void - *pEax=%#x *pEbx=%#x *pEcx=%#x *pEdx=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, *pEax, *pEbx, *pEcx, *pEdx));
+}
+
+
+/**
+ * The device helper structure for trusted devices.
+ */
+const PDMDEVHLPR3 g_pdmR3DevHlpTrusted =
+{
+    PDM_DEVHLPR3_VERSION,
+    pdmR3DevHlp_IoPortCreateEx,
+    pdmR3DevHlp_IoPortMap,
+    pdmR3DevHlp_IoPortUnmap,
+    pdmR3DevHlp_IoPortGetMappingAddress,
+    pdmR3DevHlp_MmioCreateEx,
+    pdmR3DevHlp_MmioMap,
+    pdmR3DevHlp_MmioUnmap,
+    pdmR3DevHlp_MmioReduce,
+    pdmR3DevHlp_MmioGetMappingAddress,
+    pdmR3DevHlp_Mmio2Create,
+    pdmR3DevHlp_Mmio2Destroy,
+    pdmR3DevHlp_Mmio2Map,
+    pdmR3DevHlp_Mmio2Unmap,
+    pdmR3DevHlp_Mmio2Reduce,
+    pdmR3DevHlp_Mmio2GetMappingAddress,
+    pdmR3DevHlp_Mmio2ChangeRegionNo,
+    pdmR3DevHlp_ROMRegister,
+    pdmR3DevHlp_ROMProtectShadow,
+    pdmR3DevHlp_SSMRegister,
+    SSMR3PutStruct,
+    SSMR3PutStructEx,
+    SSMR3PutBool,
+    SSMR3PutU8,
+    SSMR3PutS8,
+    SSMR3PutU16,
+    SSMR3PutS16,
+    SSMR3PutU32,
+    SSMR3PutS32,
+    SSMR3PutU64,
+    SSMR3PutS64,
+    SSMR3PutU128,
+    SSMR3PutS128,
+    SSMR3PutUInt,
+    SSMR3PutSInt,
+    SSMR3PutGCUInt,
+    SSMR3PutGCUIntReg,
+    SSMR3PutGCPhys32,
+    SSMR3PutGCPhys64,
+    SSMR3PutGCPhys,
+    SSMR3PutGCPtr,
+    SSMR3PutGCUIntPtr,
+    SSMR3PutRCPtr,
+    SSMR3PutIOPort,
+    SSMR3PutSel,
+    SSMR3PutMem,
+    SSMR3PutStrZ,
+    SSMR3GetStruct,
+    SSMR3GetStructEx,
+    SSMR3GetBool,
+    SSMR3GetBoolV,
+    SSMR3GetU8,
+    SSMR3GetU8V,
+    SSMR3GetS8,
+    SSMR3GetS8V,
+    SSMR3GetU16,
+    SSMR3GetU16V,
+    SSMR3GetS16,
+    SSMR3GetS16V,
+    SSMR3GetU32,
+    SSMR3GetU32V,
+    SSMR3GetS32,
+    SSMR3GetS32V,
+    SSMR3GetU64,
+    SSMR3GetU64V,
+    SSMR3GetS64,
+    SSMR3GetS64V,
+    SSMR3GetU128,
+    SSMR3GetU128V,
+    SSMR3GetS128,
+    SSMR3GetS128V,
+    SSMR3GetGCPhys32,
+    SSMR3GetGCPhys32V,
+    SSMR3GetGCPhys64,
+    SSMR3GetGCPhys64V,
+    SSMR3GetGCPhys,
+    SSMR3GetGCPhysV,
+    SSMR3GetUInt,
+    SSMR3GetSInt,
+    SSMR3GetGCUInt,
+    SSMR3GetGCUIntReg,
+    SSMR3GetGCPtr,
+    SSMR3GetGCUIntPtr,
+    SSMR3GetRCPtr,
+    SSMR3GetIOPort,
+    SSMR3GetSel,
+    SSMR3GetMem,
+    SSMR3GetStrZ,
+    SSMR3GetStrZEx,
+    SSMR3Skip,
+    SSMR3SkipToEndOfUnit,
+    SSMR3SetLoadError,
+    SSMR3SetLoadErrorV,
+    SSMR3SetCfgError,
+    SSMR3SetCfgErrorV,
+    SSMR3HandleGetStatus,
+    SSMR3HandleGetAfter,
+    SSMR3HandleIsLiveSave,
+    SSMR3HandleMaxDowntime,
+    SSMR3HandleHostBits,
+    SSMR3HandleRevision,
+    SSMR3HandleVersion,
+    SSMR3HandleHostOSAndArch,
+    pdmR3DevHlp_TMTimerCreate,
+    pdmR3DevHlp_TimerCreate,
+    pdmR3DevHlp_TimerToPtr,
+    pdmR3DevHlp_TimerFromMicro,
+    pdmR3DevHlp_TimerFromMilli,
+    pdmR3DevHlp_TimerFromNano,
+    pdmR3DevHlp_TimerGet,
+    pdmR3DevHlp_TimerGetFreq,
+    pdmR3DevHlp_TimerGetNano,
+    pdmR3DevHlp_TimerIsActive,
+    pdmR3DevHlp_TimerIsLockOwner,
+    pdmR3DevHlp_TimerLockClock,
+    pdmR3DevHlp_TimerLockClock2,
+    pdmR3DevHlp_TimerSet,
+    pdmR3DevHlp_TimerSetFrequencyHint,
+    pdmR3DevHlp_TimerSetMicro,
+    pdmR3DevHlp_TimerSetMillies,
+    pdmR3DevHlp_TimerSetNano,
+    pdmR3DevHlp_TimerSetRelative,
+    pdmR3DevHlp_TimerStop,
+    pdmR3DevHlp_TimerUnlockClock,
+    pdmR3DevHlp_TimerUnlockClock2,
+    pdmR3DevHlp_TimerSetCritSect,
+    pdmR3DevHlp_TimerSave,
+    pdmR3DevHlp_TimerLoad,
+    pdmR3DevHlp_TimerDestroy,
+    TMR3TimerSkip,
+    pdmR3DevHlp_TMUtcNow,
+    CFGMR3Exists,
+    CFGMR3QueryType,
+    CFGMR3QuerySize,
+    CFGMR3QueryInteger,
+    CFGMR3QueryIntegerDef,
+    CFGMR3QueryString,
+    CFGMR3QueryStringDef,
+    CFGMR3QueryBytes,
+    CFGMR3QueryU64,
+    CFGMR3QueryU64Def,
+    CFGMR3QueryS64,
+    CFGMR3QueryS64Def,
+    CFGMR3QueryU32,
+    CFGMR3QueryU32Def,
+    CFGMR3QueryS32,
+    CFGMR3QueryS32Def,
+    CFGMR3QueryU16,
+    CFGMR3QueryU16Def,
+    CFGMR3QueryS16,
+    CFGMR3QueryS16Def,
+    CFGMR3QueryU8,
+    CFGMR3QueryU8Def,
+    CFGMR3QueryS8,
+    CFGMR3QueryS8Def,
+    CFGMR3QueryBool,
+    CFGMR3QueryBoolDef,
+    CFGMR3QueryPort,
+    CFGMR3QueryPortDef,
+    CFGMR3QueryUInt,
+    CFGMR3QueryUIntDef,
+    CFGMR3QuerySInt,
+    CFGMR3QuerySIntDef,
+    CFGMR3QueryPtr,
+    CFGMR3QueryPtrDef,
+    CFGMR3QueryGCPtr,
+    CFGMR3QueryGCPtrDef,
+    CFGMR3QueryGCPtrU,
+    CFGMR3QueryGCPtrUDef,
+    CFGMR3QueryGCPtrS,
+    CFGMR3QueryGCPtrSDef,
+    CFGMR3QueryStringAlloc,
+    CFGMR3QueryStringAllocDef,
+    CFGMR3GetParent,
+    CFGMR3GetChild,
+    CFGMR3GetChildF,
+    CFGMR3GetChildFV,
+    CFGMR3GetFirstChild,
+    CFGMR3GetNextChild,
+    CFGMR3GetName,
+    CFGMR3GetNameLen,
+    CFGMR3AreChildrenValid,
+    CFGMR3GetFirstValue,
+    CFGMR3GetNextValue,
+    CFGMR3GetValueName,
+    CFGMR3GetValueNameLen,
+    CFGMR3GetValueType,
+    CFGMR3AreValuesValid,
+    CFGMR3ValidateConfig,
+    pdmR3DevHlp_PhysRead,
+    pdmR3DevHlp_PhysWrite,
+    pdmR3DevHlp_PhysGCPhys2CCPtr,
+    pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly,
+    pdmR3DevHlp_PhysReleasePageMappingLock,
+    pdmR3DevHlp_PhysReadGCVirt,
+    pdmR3DevHlp_PhysWriteGCVirt,
+    pdmR3DevHlp_PhysGCPtr2GCPhys,
+    pdmR3DevHlp_MMHeapAlloc,
+    pdmR3DevHlp_MMHeapAllocZ,
+    pdmR3DevHlp_MMHeapFree,
+    pdmR3DevHlp_VMState,
+    pdmR3DevHlp_VMTeleportedAndNotFullyResumedYet,
+    pdmR3DevHlp_VMSetError,
+    pdmR3DevHlp_VMSetErrorV,
+    pdmR3DevHlp_VMSetRuntimeError,
+    pdmR3DevHlp_VMSetRuntimeErrorV,
+    pdmR3DevHlp_DBGFStopV,
+    pdmR3DevHlp_DBGFInfoRegister,
+    pdmR3DevHlp_DBGFInfoRegisterArgv,
+    pdmR3DevHlp_DBGFRegRegister,
+    pdmR3DevHlp_DBGFTraceBuf,
+    pdmR3DevHlp_STAMRegister,
+    pdmR3DevHlp_STAMRegisterV,
+    pdmR3DevHlp_PCIRegister,
+    pdmR3DevHlp_PCIRegisterMsi,
+    pdmR3DevHlp_PCIIORegionRegister,
+    pdmR3DevHlp_PCIInterceptConfigAccesses,
+    pdmR3DevHlp_PCIConfigWrite,
+    pdmR3DevHlp_PCIConfigRead,
+    pdmR3DevHlp_PCIPhysRead,
+    pdmR3DevHlp_PCIPhysWrite,
+    pdmR3DevHlp_PCISetIrq,
+    pdmR3DevHlp_PCISetIrqNoWait,
+    pdmR3DevHlp_ISASetIrq,
+    pdmR3DevHlp_ISASetIrqNoWait,
+    pdmR3DevHlp_IoApicSendMsi,
+    pdmR3DevHlp_DriverAttach,
+    pdmR3DevHlp_DriverDetach,
+    pdmR3DevHlp_DriverReconfigure,
+    pdmR3DevHlp_QueueCreatePtr,
+    pdmR3DevHlp_QueueCreate,
+    pdmR3DevHlp_QueueToPtr,
+    pdmR3DevHlp_QueueAlloc,
+    pdmR3DevHlp_QueueInsert,
+    pdmR3DevHlp_QueueInsertEx,
+    pdmR3DevHlp_QueueFlushIfNecessary,
+    pdmR3DevHlp_TaskCreate,
+    pdmR3DevHlp_TaskTrigger,
+    pdmR3DevHlp_SUPSemEventCreate,
+    pdmR3DevHlp_SUPSemEventClose,
+    pdmR3DevHlp_SUPSemEventSignal,
+    pdmR3DevHlp_SUPSemEventWaitNoResume,
+    pdmR3DevHlp_SUPSemEventWaitNsAbsIntr,
+    pdmR3DevHlp_SUPSemEventWaitNsRelIntr,
+    pdmR3DevHlp_SUPSemEventGetResolution,
+    pdmR3DevHlp_SUPSemEventMultiCreate,
+    pdmR3DevHlp_SUPSemEventMultiClose,
+    pdmR3DevHlp_SUPSemEventMultiSignal,
+    pdmR3DevHlp_SUPSemEventMultiReset,
+    pdmR3DevHlp_SUPSemEventMultiWaitNoResume,
+    pdmR3DevHlp_SUPSemEventMultiWaitNsAbsIntr,
+    pdmR3DevHlp_SUPSemEventMultiWaitNsRelIntr,
+    pdmR3DevHlp_SUPSemEventMultiGetResolution,
+    pdmR3DevHlp_CritSectInit,
+    pdmR3DevHlp_CritSectGetNop,
+    pdmR3DevHlp_CritSectGetNopR0,
+    pdmR3DevHlp_CritSectGetNopRC,
+    pdmR3DevHlp_SetDeviceCritSect,
+    pdmR3DevHlp_CritSectYield,
+    pdmR3DevHlp_CritSectEnter,
+    pdmR3DevHlp_CritSectEnterDebug,
+    pdmR3DevHlp_CritSectTryEnter,
+    pdmR3DevHlp_CritSectTryEnterDebug,
+    pdmR3DevHlp_CritSectLeave,
+    pdmR3DevHlp_CritSectIsOwner,
+    pdmR3DevHlp_CritSectIsInitialized,
+    pdmR3DevHlp_CritSectHasWaiters,
+    pdmR3DevHlp_CritSectGetRecursion,
+    pdmR3DevHlp_CritSectScheduleExitEvent,
+    pdmR3DevHlp_CritSectDelete,
+    pdmR3DevHlp_ThreadCreate,
+    PDMR3ThreadDestroy,
+    PDMR3ThreadIAmSuspending,
+    PDMR3ThreadIAmRunning,
+    PDMR3ThreadSleep,
+    PDMR3ThreadSuspend,
+    PDMR3ThreadResume,
+    pdmR3DevHlp_SetAsyncNotification,
+    pdmR3DevHlp_AsyncNotificationCompleted,
+    pdmR3DevHlp_RTCRegister,
+    pdmR3DevHlp_PCIBusRegister,
+    pdmR3DevHlp_PICRegister,
+    pdmR3DevHlp_ApicRegister,
+    pdmR3DevHlp_IoApicRegister,
+    pdmR3DevHlp_HpetRegister,
+    pdmR3DevHlp_PciRawRegister,
+    pdmR3DevHlp_DMACRegister,
+    pdmR3DevHlp_DMARegister,
+    pdmR3DevHlp_DMAReadMemory,
+    pdmR3DevHlp_DMAWriteMemory,
+    pdmR3DevHlp_DMASetDREQ,
+    pdmR3DevHlp_DMAGetChannelMode,
+    pdmR3DevHlp_DMASchedule,
+    pdmR3DevHlp_CMOSWrite,
+    pdmR3DevHlp_CMOSRead,
+    pdmR3DevHlp_AssertEMT,
+    pdmR3DevHlp_AssertOther,
+    pdmR3DevHlp_LdrGetRCInterfaceSymbols,
+    pdmR3DevHlp_LdrGetR0InterfaceSymbols,
+    pdmR3DevHlp_CallR0,
+    pdmR3DevHlp_VMGetSuspendReason,
+    pdmR3DevHlp_VMGetResumeReason,
+    pdmR3DevHlp_PhysBulkGCPhys2CCPtr,
+    pdmR3DevHlp_PhysBulkGCPhys2CCPtrReadOnly,
+    pdmR3DevHlp_PhysBulkReleasePageMappingLocks,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    pdmR3DevHlp_GetUVM,
+    pdmR3DevHlp_GetVM,
+    pdmR3DevHlp_GetVMCPU,
+    pdmR3DevHlp_GetCurrentCpuId,
+    pdmR3DevHlp_RegisterVMMDevHeap,
+    pdmR3DevHlp_FirmwareRegister,
+    pdmR3DevHlp_VMReset,
+    pdmR3DevHlp_VMSuspend,
+    pdmR3DevHlp_VMSuspendSaveAndPowerOff,
+    pdmR3DevHlp_VMPowerOff,
+    pdmR3DevHlp_A20IsEnabled,
+    pdmR3DevHlp_A20Set,
+    pdmR3DevHlp_GetCpuId,
+    pdmR3DevHlp_TMTimeVirtGet,
+    pdmR3DevHlp_TMTimeVirtGetFreq,
+    pdmR3DevHlp_TMTimeVirtGetNano,
+    pdmR3DevHlp_GetSupDrvSession,
+    pdmR3DevHlp_QueryGenericUserObject,
+    PDM_DEVHLPR3_VERSION /* the end */
+};
+
+
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetUVM} */
+static DECLCALLBACK(PUVM) pdmR3DevHlp_Untrusted_GetUVM(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return NULL;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetVM} */
+static DECLCALLBACK(PVM) pdmR3DevHlp_Untrusted_GetVM(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return NULL;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetVMCPU} */
+static DECLCALLBACK(PVMCPU) pdmR3DevHlp_Untrusted_GetVMCPU(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return NULL;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetCurrentCpuId} */
+static DECLCALLBACK(VMCPUID) pdmR3DevHlp_Untrusted_GetCurrentCpuId(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return NIL_VMCPUID;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnRegisterVMMDevHeap} */
+static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_RegisterVMMDevHeap(PPDMDEVINS pDevIns, RTGCPHYS GCPhys,
+                                                                  RTR3PTR pvHeap, unsigned cbHeap)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    NOREF(GCPhys); NOREF(pvHeap); NOREF(cbHeap);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return VERR_ACCESS_DENIED;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnFirmwareRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_FirmwareRegister(PPDMDEVINS pDevIns, PCPDMFWREG pFwReg, PCPDMFWHLPR3 *ppFwHlp)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    NOREF(pFwReg); NOREF(ppFwHlp);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return VERR_ACCESS_DENIED;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMReset} */
+static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_VMReset(PPDMDEVINS pDevIns, uint32_t fFlags)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns); NOREF(fFlags);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return VERR_ACCESS_DENIED;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSuspend} */
+static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_VMSuspend(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return VERR_ACCESS_DENIED;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSuspendSaveAndPowerOff} */
+static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_VMSuspendSaveAndPowerOff(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return VERR_ACCESS_DENIED;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMPowerOff} */
+static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_VMPowerOff(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return VERR_ACCESS_DENIED;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnA20IsEnabled} */
+static DECLCALLBACK(bool) pdmR3DevHlp_Untrusted_A20IsEnabled(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return false;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnA20Set} */
+static DECLCALLBACK(void) pdmR3DevHlp_Untrusted_A20Set(PPDMDEVINS pDevIns, bool fEnable)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    NOREF(fEnable);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetCpuId} */
+static DECLCALLBACK(void) pdmR3DevHlp_Untrusted_GetCpuId(PPDMDEVINS pDevIns, uint32_t iLeaf,
+                                                         uint32_t *pEax, uint32_t *pEbx, uint32_t *pEcx, uint32_t *pEdx)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    NOREF(iLeaf); NOREF(pEax); NOREF(pEbx); NOREF(pEcx); NOREF(pEdx);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetSupDrvSession} */
+static DECLCALLBACK(PSUPDRVSESSION) pdmR3DevHlp_Untrusted_GetSupDrvSession(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return (PSUPDRVSESSION)0;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnQueryGenericUserObject} */
+static DECLCALLBACK(void *) pdmR3DevHlp_Untrusted_QueryGenericUserObject(PPDMDEVINS pDevIns, PCRTUUID pUuid)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d %RTuuid\n",
+                            pDevIns->pReg->szName, pDevIns->iInstance, pUuid));
+    return NULL;
+}
+
+
+/**
+ * The device helper structure for non-trusted devices.
+ */
+const PDMDEVHLPR3 g_pdmR3DevHlpUnTrusted =
+{
+    PDM_DEVHLPR3_VERSION,
+    pdmR3DevHlp_IoPortCreateEx,
+    pdmR3DevHlp_IoPortMap,
+    pdmR3DevHlp_IoPortUnmap,
+    pdmR3DevHlp_IoPortGetMappingAddress,
+    pdmR3DevHlp_MmioCreateEx,
+    pdmR3DevHlp_MmioMap,
+    pdmR3DevHlp_MmioUnmap,
+    pdmR3DevHlp_MmioReduce,
+    pdmR3DevHlp_MmioGetMappingAddress,
+    pdmR3DevHlp_Mmio2Create,
+    pdmR3DevHlp_Mmio2Destroy,
+    pdmR3DevHlp_Mmio2Map,
+    pdmR3DevHlp_Mmio2Unmap,
+    pdmR3DevHlp_Mmio2Reduce,
+    pdmR3DevHlp_Mmio2GetMappingAddress,
+    pdmR3DevHlp_Mmio2ChangeRegionNo,
+    pdmR3DevHlp_ROMRegister,
+    pdmR3DevHlp_ROMProtectShadow,
+    pdmR3DevHlp_SSMRegister,
+    SSMR3PutStruct,
+    SSMR3PutStructEx,
+    SSMR3PutBool,
+    SSMR3PutU8,
+    SSMR3PutS8,
+    SSMR3PutU16,
+    SSMR3PutS16,
+    SSMR3PutU32,
+    SSMR3PutS32,
+    SSMR3PutU64,
+    SSMR3PutS64,
+    SSMR3PutU128,
+    SSMR3PutS128,
+    SSMR3PutUInt,
+    SSMR3PutSInt,
+    SSMR3PutGCUInt,
+    SSMR3PutGCUIntReg,
+    SSMR3PutGCPhys32,
+    SSMR3PutGCPhys64,
+    SSMR3PutGCPhys,
+    SSMR3PutGCPtr,
+    SSMR3PutGCUIntPtr,
+    SSMR3PutRCPtr,
+    SSMR3PutIOPort,
+    SSMR3PutSel,
+    SSMR3PutMem,
+    SSMR3PutStrZ,
+    SSMR3GetStruct,
+    SSMR3GetStructEx,
+    SSMR3GetBool,
+    SSMR3GetBoolV,
+    SSMR3GetU8,
+    SSMR3GetU8V,
+    SSMR3GetS8,
+    SSMR3GetS8V,
+    SSMR3GetU16,
+    SSMR3GetU16V,
+    SSMR3GetS16,
+    SSMR3GetS16V,
+    SSMR3GetU32,
+    SSMR3GetU32V,
+    SSMR3GetS32,
+    SSMR3GetS32V,
+    SSMR3GetU64,
+    SSMR3GetU64V,
+    SSMR3GetS64,
+    SSMR3GetS64V,
+    SSMR3GetU128,
+    SSMR3GetU128V,
+    SSMR3GetS128,
+    SSMR3GetS128V,
+    SSMR3GetGCPhys32,
+    SSMR3GetGCPhys32V,
+    SSMR3GetGCPhys64,
+    SSMR3GetGCPhys64V,
+    SSMR3GetGCPhys,
+    SSMR3GetGCPhysV,
+    SSMR3GetUInt,
+    SSMR3GetSInt,
+    SSMR3GetGCUInt,
+    SSMR3GetGCUIntReg,
+    SSMR3GetGCPtr,
+    SSMR3GetGCUIntPtr,
+    SSMR3GetRCPtr,
+    SSMR3GetIOPort,
+    SSMR3GetSel,
+    SSMR3GetMem,
+    SSMR3GetStrZ,
+    SSMR3GetStrZEx,
+    SSMR3Skip,
+    SSMR3SkipToEndOfUnit,
+    SSMR3SetLoadError,
+    SSMR3SetLoadErrorV,
+    SSMR3SetCfgError,
+    SSMR3SetCfgErrorV,
+    SSMR3HandleGetStatus,
+    SSMR3HandleGetAfter,
+    SSMR3HandleIsLiveSave,
+    SSMR3HandleMaxDowntime,
+    SSMR3HandleHostBits,
+    SSMR3HandleRevision,
+    SSMR3HandleVersion,
+    SSMR3HandleHostOSAndArch,
+    pdmR3DevHlp_TMTimerCreate,
+    pdmR3DevHlp_TimerCreate,
+    pdmR3DevHlp_TimerToPtr,
+    pdmR3DevHlp_TimerFromMicro,
+    pdmR3DevHlp_TimerFromMilli,
+    pdmR3DevHlp_TimerFromNano,
+    pdmR3DevHlp_TimerGet,
+    pdmR3DevHlp_TimerGetFreq,
+    pdmR3DevHlp_TimerGetNano,
+    pdmR3DevHlp_TimerIsActive,
+    pdmR3DevHlp_TimerIsLockOwner,
+    pdmR3DevHlp_TimerLockClock,
+    pdmR3DevHlp_TimerLockClock2,
+    pdmR3DevHlp_TimerSet,
+    pdmR3DevHlp_TimerSetFrequencyHint,
+    pdmR3DevHlp_TimerSetMicro,
+    pdmR3DevHlp_TimerSetMillies,
+    pdmR3DevHlp_TimerSetNano,
+    pdmR3DevHlp_TimerSetRelative,
+    pdmR3DevHlp_TimerStop,
+    pdmR3DevHlp_TimerUnlockClock,
+    pdmR3DevHlp_TimerUnlockClock2,
+    pdmR3DevHlp_TimerSetCritSect,
+    pdmR3DevHlp_TimerSave,
+    pdmR3DevHlp_TimerLoad,
+    pdmR3DevHlp_TimerDestroy,
+    TMR3TimerSkip,
+    pdmR3DevHlp_TMUtcNow,
+    CFGMR3Exists,
+    CFGMR3QueryType,
+    CFGMR3QuerySize,
+    CFGMR3QueryInteger,
+    CFGMR3QueryIntegerDef,
+    CFGMR3QueryString,
+    CFGMR3QueryStringDef,
+    CFGMR3QueryBytes,
+    CFGMR3QueryU64,
+    CFGMR3QueryU64Def,
+    CFGMR3QueryS64,
+    CFGMR3QueryS64Def,
+    CFGMR3QueryU32,
+    CFGMR3QueryU32Def,
+    CFGMR3QueryS32,
+    CFGMR3QueryS32Def,
+    CFGMR3QueryU16,
+    CFGMR3QueryU16Def,
+    CFGMR3QueryS16,
+    CFGMR3QueryS16Def,
+    CFGMR3QueryU8,
+    CFGMR3QueryU8Def,
+    CFGMR3QueryS8,
+    CFGMR3QueryS8Def,
+    CFGMR3QueryBool,
+    CFGMR3QueryBoolDef,
+    CFGMR3QueryPort,
+    CFGMR3QueryPortDef,
+    CFGMR3QueryUInt,
+    CFGMR3QueryUIntDef,
+    CFGMR3QuerySInt,
+    CFGMR3QuerySIntDef,
+    CFGMR3QueryPtr,
+    CFGMR3QueryPtrDef,
+    CFGMR3QueryGCPtr,
+    CFGMR3QueryGCPtrDef,
+    CFGMR3QueryGCPtrU,
+    CFGMR3QueryGCPtrUDef,
+    CFGMR3QueryGCPtrS,
+    CFGMR3QueryGCPtrSDef,
+    CFGMR3QueryStringAlloc,
+    CFGMR3QueryStringAllocDef,
+    CFGMR3GetParent,
+    CFGMR3GetChild,
+    CFGMR3GetChildF,
+    CFGMR3GetChildFV,
+    CFGMR3GetFirstChild,
+    CFGMR3GetNextChild,
+    CFGMR3GetName,
+    CFGMR3GetNameLen,
+    CFGMR3AreChildrenValid,
+    CFGMR3GetFirstValue,
+    CFGMR3GetNextValue,
+    CFGMR3GetValueName,
+    CFGMR3GetValueNameLen,
+    CFGMR3GetValueType,
+    CFGMR3AreValuesValid,
+    CFGMR3ValidateConfig,
+    pdmR3DevHlp_PhysRead,
+    pdmR3DevHlp_PhysWrite,
+    pdmR3DevHlp_PhysGCPhys2CCPtr,
+    pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly,
+    pdmR3DevHlp_PhysReleasePageMappingLock,
+    pdmR3DevHlp_PhysReadGCVirt,
+    pdmR3DevHlp_PhysWriteGCVirt,
+    pdmR3DevHlp_PhysGCPtr2GCPhys,
+    pdmR3DevHlp_MMHeapAlloc,
+    pdmR3DevHlp_MMHeapAllocZ,
+    pdmR3DevHlp_MMHeapFree,
+    pdmR3DevHlp_VMState,
+    pdmR3DevHlp_VMTeleportedAndNotFullyResumedYet,
+    pdmR3DevHlp_VMSetError,
+    pdmR3DevHlp_VMSetErrorV,
+    pdmR3DevHlp_VMSetRuntimeError,
+    pdmR3DevHlp_VMSetRuntimeErrorV,
+    pdmR3DevHlp_DBGFStopV,
+    pdmR3DevHlp_DBGFInfoRegister,
+    pdmR3DevHlp_DBGFInfoRegisterArgv,
+    pdmR3DevHlp_DBGFRegRegister,
+    pdmR3DevHlp_DBGFTraceBuf,
+    pdmR3DevHlp_STAMRegister,
+    pdmR3DevHlp_STAMRegisterV,
+    pdmR3DevHlp_PCIRegister,
+    pdmR3DevHlp_PCIRegisterMsi,
+    pdmR3DevHlp_PCIIORegionRegister,
+    pdmR3DevHlp_PCIInterceptConfigAccesses,
+    pdmR3DevHlp_PCIConfigWrite,
+    pdmR3DevHlp_PCIConfigRead,
+    pdmR3DevHlp_PCIPhysRead,
+    pdmR3DevHlp_PCIPhysWrite,
+    pdmR3DevHlp_PCISetIrq,
+    pdmR3DevHlp_PCISetIrqNoWait,
+    pdmR3DevHlp_ISASetIrq,
+    pdmR3DevHlp_ISASetIrqNoWait,
+    pdmR3DevHlp_IoApicSendMsi,
+    pdmR3DevHlp_DriverAttach,
+    pdmR3DevHlp_DriverDetach,
+    pdmR3DevHlp_DriverReconfigure,
+    pdmR3DevHlp_QueueCreatePtr,
+    pdmR3DevHlp_QueueCreate,
+    pdmR3DevHlp_QueueToPtr,
+    pdmR3DevHlp_QueueAlloc,
+    pdmR3DevHlp_QueueInsert,
+    pdmR3DevHlp_QueueInsertEx,
+    pdmR3DevHlp_QueueFlushIfNecessary,
+    pdmR3DevHlp_TaskCreate,
+    pdmR3DevHlp_TaskTrigger,
+    pdmR3DevHlp_SUPSemEventCreate,
+    pdmR3DevHlp_SUPSemEventClose,
+    pdmR3DevHlp_SUPSemEventSignal,
+    pdmR3DevHlp_SUPSemEventWaitNoResume,
+    pdmR3DevHlp_SUPSemEventWaitNsAbsIntr,
+    pdmR3DevHlp_SUPSemEventWaitNsRelIntr,
+    pdmR3DevHlp_SUPSemEventGetResolution,
+    pdmR3DevHlp_SUPSemEventMultiCreate,
+    pdmR3DevHlp_SUPSemEventMultiClose,
+    pdmR3DevHlp_SUPSemEventMultiSignal,
+    pdmR3DevHlp_SUPSemEventMultiReset,
+    pdmR3DevHlp_SUPSemEventMultiWaitNoResume,
+    pdmR3DevHlp_SUPSemEventMultiWaitNsAbsIntr,
+    pdmR3DevHlp_SUPSemEventMultiWaitNsRelIntr,
+    pdmR3DevHlp_SUPSemEventMultiGetResolution,
+    pdmR3DevHlp_CritSectInit,
+    pdmR3DevHlp_CritSectGetNop,
+    pdmR3DevHlp_CritSectGetNopR0,
+    pdmR3DevHlp_CritSectGetNopRC,
+    pdmR3DevHlp_SetDeviceCritSect,
+    pdmR3DevHlp_CritSectYield,
+    pdmR3DevHlp_CritSectEnter,
+    pdmR3DevHlp_CritSectEnterDebug,
+    pdmR3DevHlp_CritSectTryEnter,
+    pdmR3DevHlp_CritSectTryEnterDebug,
+    pdmR3DevHlp_CritSectLeave,
+    pdmR3DevHlp_CritSectIsOwner,
+    pdmR3DevHlp_CritSectIsInitialized,
+    pdmR3DevHlp_CritSectHasWaiters,
+    pdmR3DevHlp_CritSectGetRecursion,
+    pdmR3DevHlp_CritSectScheduleExitEvent,
+    pdmR3DevHlp_CritSectDelete,
+    pdmR3DevHlp_ThreadCreate,
+    PDMR3ThreadDestroy,
+    PDMR3ThreadIAmSuspending,
+    PDMR3ThreadIAmRunning,
+    PDMR3ThreadSleep,
+    PDMR3ThreadSuspend,
+    PDMR3ThreadResume,
+    pdmR3DevHlp_SetAsyncNotification,
+    pdmR3DevHlp_AsyncNotificationCompleted,
+    pdmR3DevHlp_RTCRegister,
+    pdmR3DevHlp_PCIBusRegister,
+    pdmR3DevHlp_PICRegister,
+    pdmR3DevHlp_ApicRegister,
+    pdmR3DevHlp_IoApicRegister,
+    pdmR3DevHlp_HpetRegister,
+    pdmR3DevHlp_PciRawRegister,
+    pdmR3DevHlp_DMACRegister,
+    pdmR3DevHlp_DMARegister,
+    pdmR3DevHlp_DMAReadMemory,
+    pdmR3DevHlp_DMAWriteMemory,
+    pdmR3DevHlp_DMASetDREQ,
+    pdmR3DevHlp_DMAGetChannelMode,
+    pdmR3DevHlp_DMASchedule,
+    pdmR3DevHlp_CMOSWrite,
+    pdmR3DevHlp_CMOSRead,
+    pdmR3DevHlp_AssertEMT,
+    pdmR3DevHlp_AssertOther,
+    pdmR3DevHlp_LdrGetRCInterfaceSymbols,
+    pdmR3DevHlp_LdrGetR0InterfaceSymbols,
+    pdmR3DevHlp_CallR0,
+    pdmR3DevHlp_VMGetSuspendReason,
+    pdmR3DevHlp_VMGetResumeReason,
+    pdmR3DevHlp_PhysBulkGCPhys2CCPtr,
+    pdmR3DevHlp_PhysBulkGCPhys2CCPtrReadOnly,
+    pdmR3DevHlp_PhysBulkReleasePageMappingLocks,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    pdmR3DevHlp_Untrusted_GetUVM,
+    pdmR3DevHlp_Untrusted_GetVM,
+    pdmR3DevHlp_Untrusted_GetVMCPU,
+    pdmR3DevHlp_Untrusted_GetCurrentCpuId,
+    pdmR3DevHlp_Untrusted_RegisterVMMDevHeap,
+    pdmR3DevHlp_Untrusted_FirmwareRegister,
+    pdmR3DevHlp_Untrusted_VMReset,
+    pdmR3DevHlp_Untrusted_VMSuspend,
+    pdmR3DevHlp_Untrusted_VMSuspendSaveAndPowerOff,
+    pdmR3DevHlp_Untrusted_VMPowerOff,
+    pdmR3DevHlp_Untrusted_A20IsEnabled,
+    pdmR3DevHlp_Untrusted_A20Set,
+    pdmR3DevHlp_Untrusted_GetCpuId,
+    pdmR3DevHlp_TMTimeVirtGet,
+    pdmR3DevHlp_TMTimeVirtGetFreq,
+    pdmR3DevHlp_TMTimeVirtGetNano,
+    pdmR3DevHlp_Untrusted_GetSupDrvSession,
+    pdmR3DevHlp_Untrusted_QueryGenericUserObject,
+    PDM_DEVHLPR3_VERSION /* the end */
+};
+
+
+
+/**
+ * Queue consumer callback for internal component.
+ *
+ * @returns Success indicator.
+ *          If false the item will not be removed and the flushing will stop.
+ * @param   pVM         The cross context VM structure.
+ * @param   pItem       The item to consume. Upon return this item will be freed.
+ */
+DECLCALLBACK(bool) pdmR3DevHlpQueueConsumer(PVM pVM, PPDMQUEUEITEMCORE pItem)
+{
+    PPDMDEVHLPTASK pTask = (PPDMDEVHLPTASK)pItem;
+    LogFlow(("pdmR3DevHlpQueueConsumer: enmOp=%d pDevIns=%p\n", pTask->enmOp, pTask->pDevInsR3));
+    switch (pTask->enmOp)
+    {
+        case PDMDEVHLPTASKOP_ISA_SET_IRQ:
+            PDMIsaSetIrq(pVM, pTask->u.IsaSetIRQ.iIrq, pTask->u.IsaSetIRQ.iLevel, pTask->u.IsaSetIRQ.uTagSrc);
+            break;
+
+        case PDMDEVHLPTASKOP_PCI_SET_IRQ:
+        {
+            /* Same as pdmR3DevHlp_PCISetIrq, except we've got a tag already. */
+            PPDMPCIDEV pPciDev = pTask->u.PciSetIRQ.pPciDevR3;
+            if (pPciDev)
+            {
+                size_t const    idxBus = pPciDev->Int.s.idxPdmBus;
+                AssertBreak(idxBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses));
+                PPDMPCIBUS      pBus   = &pVM->pdm.s.aPciBuses[idxBus];
+
+                pdmLock(pVM);
+                pBus->pfnSetIrqR3(pBus->pDevInsR3, pPciDev, pTask->u.PciSetIRQ.iIrq,
+                                  pTask->u.PciSetIRQ.iLevel, pTask->u.PciSetIRQ.uTagSrc);
+                pdmUnlock(pVM);
+            }
+            else
+                AssertReleaseMsgFailed(("No PCI device registered!\n"));
+            break;
+        }
+
+        case PDMDEVHLPTASKOP_IOAPIC_SET_IRQ:
+            PDMIoApicSetIrq(pVM, pTask->u.IoApicSetIRQ.iIrq, pTask->u.IoApicSetIRQ.iLevel, pTask->u.IoApicSetIRQ.uTagSrc);
+            break;
+
+        default:
+            AssertReleaseMsgFailed(("Invalid operation %d\n", pTask->enmOp));
+            break;
+    }
+    return true;
+}
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMR3/PDMDevMiscHlp.cpp b/src/VBox/VMM/VMMR3/PDMDevMiscHlp.cpp
new file mode 100644
index 00000000..29e25259
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMDevMiscHlp.cpp
@@ -0,0 +1,410 @@
+/* $Id: PDMDevMiscHlp.cpp $ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, Misc. Device Helpers.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DEVICE
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/vmm.h>
+
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+
+
+#include "PDMInline.h"
+#include "dtrace/VBoxVMM.h"
+
+
+
+/** @name Ring-3 PIC Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMPICHLP,pfnSetInterruptFF} */
+static DECLCALLBACK(void) pdmR3PicHlp_SetInterruptFF(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM    pVM = pDevIns->Internal.s.pVMR3;
+    PVMCPU pVCpu = pVM->apCpusR3[0];  /* for PIC we always deliver to CPU 0, SMP uses APIC */
+
+    /* IRQ state should be loaded as-is by "LoadExec". Changes can be made from LoadDone. */
+    Assert(pVM->enmVMState != VMSTATE_LOADING || pVM->pdm.s.fStateLoaded);
+
+    APICLocalInterrupt(pVCpu, 0 /* u8Pin */, 1 /* u8Level */, VINF_SUCCESS /* rcRZ */);
+}
+
+
+/** @interface_method_impl{PDMPICHLP,pfnClearInterruptFF} */
+static DECLCALLBACK(void) pdmR3PicHlp_ClearInterruptFF(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    PVMCPU pVCpu = pVM->apCpusR3[0];  /* for PIC we always deliver to CPU 0, SMP uses APIC */
+
+    /* IRQ state should be loaded as-is by "LoadExec". Changes can be made from LoadDone. */
+    Assert(pVM->enmVMState != VMSTATE_LOADING || pVM->pdm.s.fStateLoaded);
+
+    APICLocalInterrupt(pVCpu, 0 /* u8Pin */,  0 /* u8Level */, VINF_SUCCESS /* rcRZ */);
+}
+
+
+/** @interface_method_impl{PDMPICHLP,pfnLock} */
+static DECLCALLBACK(int) pdmR3PicHlp_Lock(PPDMDEVINS pDevIns, int rc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    return pdmLockEx(pDevIns->Internal.s.pVMR3, rc);
+}
+
+
+/** @interface_method_impl{PDMPICHLP,pfnUnlock} */
+static DECLCALLBACK(void) pdmR3PicHlp_Unlock(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    pdmUnlock(pDevIns->Internal.s.pVMR3);
+}
+
+
+/**
+ * PIC Device Helpers.
+ */
+const PDMPICHLP g_pdmR3DevPicHlp =
+{
+    PDM_PICHLP_VERSION,
+    pdmR3PicHlp_SetInterruptFF,
+    pdmR3PicHlp_ClearInterruptFF,
+    pdmR3PicHlp_Lock,
+    pdmR3PicHlp_Unlock,
+    PDM_PICHLP_VERSION /* the end */
+};
+
+/** @} */
+
+
+/** @name Ring-3 I/O APIC Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMIOAPICHLP,pfnApicBusDeliver} */
+static DECLCALLBACK(int) pdmR3IoApicHlp_ApicBusDeliver(PPDMDEVINS pDevIns, uint8_t u8Dest, uint8_t u8DestMode,
+                                                       uint8_t u8DeliveryMode, uint8_t uVector, uint8_t u8Polarity,
+                                                       uint8_t u8TriggerMode, uint32_t uTagSrc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3IoApicHlp_ApicBusDeliver: caller='%s'/%d: u8Dest=%RX8 u8DestMode=%RX8 u8DeliveryMode=%RX8 uVector=%RX8 u8Polarity=%RX8 u8TriggerMode=%RX8 uTagSrc=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, u8Dest, u8DestMode, u8DeliveryMode, uVector, u8Polarity, u8TriggerMode, uTagSrc));
+    return APICBusDeliver(pVM, u8Dest, u8DestMode, u8DeliveryMode, uVector, u8Polarity, u8TriggerMode, uTagSrc);
+}
+
+
+/** @interface_method_impl{PDMIOAPICHLP,pfnLock} */
+static DECLCALLBACK(int) pdmR3IoApicHlp_Lock(PPDMDEVINS pDevIns, int rc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3IoApicHlp_Lock: caller='%s'/%d: rc=%Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return pdmLockEx(pDevIns->Internal.s.pVMR3, rc);
+}
+
+
+/** @interface_method_impl{PDMIOAPICHLP,pfnUnlock} */
+static DECLCALLBACK(void) pdmR3IoApicHlp_Unlock(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3IoApicHlp_Unlock: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    pdmUnlock(pDevIns->Internal.s.pVMR3);
+}
+
+
+/**
+ * I/O APIC Device Helpers.
+ */
+const PDMIOAPICHLP g_pdmR3DevIoApicHlp =
+{
+    PDM_IOAPICHLP_VERSION,
+    pdmR3IoApicHlp_ApicBusDeliver,
+    pdmR3IoApicHlp_Lock,
+    pdmR3IoApicHlp_Unlock,
+    PDM_IOAPICHLP_VERSION /* the end */
+};
+
+/** @} */
+
+
+
+
+/** @name Ring-3 PCI Bus Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnIsaSetIrq} */
+static DECLCALLBACK(void) pdmR3PciHlp_IsaSetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel, uint32_t uTagSrc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    Log4(("pdmR3PciHlp_IsaSetIrq: iIrq=%d iLevel=%d uTagSrc=%#x\n", iIrq, iLevel, uTagSrc));
+    PDMIsaSetIrq(pDevIns->Internal.s.pVMR3, iIrq, iLevel, uTagSrc);
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnIoApicSetIrq} */
+static DECLCALLBACK(void) pdmR3PciHlp_IoApicSetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel, uint32_t uTagSrc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    Log4(("pdmR3PciHlp_IoApicSetIrq: iIrq=%d iLevel=%d uTagSrc=%#x\n", iIrq, iLevel, uTagSrc));
+    PDMIoApicSetIrq(pDevIns->Internal.s.pVMR3, iIrq, iLevel, uTagSrc);
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnIoApicSendMsi} */
+static DECLCALLBACK(void) pdmR3PciHlp_IoApicSendMsi(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint32_t uValue, uint32_t uTagSrc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    Log4(("pdmR3PciHlp_IoApicSendMsi: address=%p value=%x uTagSrc=%#x\n", GCPhys, uValue, uTagSrc));
+    PDMIoApicSendMsi(pDevIns->Internal.s.pVMR3, GCPhys, uValue, uTagSrc);
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnLock} */
+static DECLCALLBACK(int) pdmR3PciHlp_Lock(PPDMDEVINS pDevIns, int rc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3PciHlp_Lock: caller='%s'/%d: rc=%Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return pdmLockEx(pDevIns->Internal.s.pVMR3, rc);
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnUnlock} */
+static DECLCALLBACK(void) pdmR3PciHlp_Unlock(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3PciHlp_Unlock: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    pdmUnlock(pDevIns->Internal.s.pVMR3);
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnGetBusByNo} */
+static DECLCALLBACK(PPDMDEVINS) pdmR3PciHlp_GetBusByNo(PPDMDEVINS pDevIns, uint32_t idxPdmBus)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    AssertReturn(idxPdmBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses), NULL);
+    PPDMDEVINS pRetDevIns = pVM->pdm.s.aPciBuses[idxPdmBus].pDevInsR3;
+    LogFlow(("pdmR3PciHlp_GetBusByNo: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pRetDevIns));
+    return pRetDevIns;
+}
+
+
+/**
+ * PCI Bus Device Helpers.
+ */
+const PDMPCIHLPR3 g_pdmR3DevPciHlp =
+{
+    PDM_PCIHLPR3_VERSION,
+    pdmR3PciHlp_IsaSetIrq,
+    pdmR3PciHlp_IoApicSetIrq,
+    pdmR3PciHlp_IoApicSendMsi,
+    pdmR3PciHlp_Lock,
+    pdmR3PciHlp_Unlock,
+    pdmR3PciHlp_GetBusByNo,
+    PDM_PCIHLPR3_VERSION, /* the end */
+};
+
+/** @} */
+
+
+
+
+/** @name Ring-3 HPET Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMHPETHLPR3,pfnSetLegacyMode} */
+static DECLCALLBACK(int) pdmR3HpetHlp_SetLegacyMode(PPDMDEVINS pDevIns, bool fActivated)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3HpetHlp_SetLegacyMode: caller='%s'/%d: fActivated=%RTbool\n", pDevIns->pReg->szName, pDevIns->iInstance, fActivated));
+
+    size_t                      i;
+    int                         rc = VINF_SUCCESS;
+    static const char * const   s_apszDevsToNotify[] =
+    {
+        "i8254",
+        "mc146818"
+    };
+    for (i = 0; i < RT_ELEMENTS(s_apszDevsToNotify); i++)
+    {
+        PPDMIBASE pBase;
+        rc = PDMR3QueryDevice(pDevIns->Internal.s.pVMR3->pUVM, "i8254", 0, &pBase);
+        if (RT_SUCCESS(rc))
+        {
+            PPDMIHPETLEGACYNOTIFY pPort = PDMIBASE_QUERY_INTERFACE(pBase, PDMIHPETLEGACYNOTIFY);
+            AssertLogRelMsgBreakStmt(pPort, ("%s\n", s_apszDevsToNotify[i]), rc = VERR_PDM_HPET_LEGACY_NOTIFY_MISSING);
+            pPort->pfnModeChanged(pPort, fActivated);
+        }
+        else if (   rc == VERR_PDM_DEVICE_NOT_FOUND
+                 || rc == VERR_PDM_DEVICE_INSTANCE_NOT_FOUND)
+            rc = VINF_SUCCESS; /* the device isn't configured, ignore. */
+        else
+            AssertLogRelMsgFailedBreak(("%s -> %Rrc\n", s_apszDevsToNotify[i], rc));
+    }
+
+    /* Don't bother cleaning up, any failure here will cause a guru meditation. */
+
+    LogFlow(("pdmR3HpetHlp_SetLegacyMode: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMHPETHLPR3,pfnSetIrq} */
+static DECLCALLBACK(int) pdmR3HpetHlp_SetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3HpetHlp_SetIrq: caller='%s'/%d: iIrq=%d iLevel=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, iIrq, iLevel));
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+
+    pdmLock(pVM);
+    uint32_t uTagSrc;
+    if (iLevel & PDM_IRQ_LEVEL_HIGH)
+    {
+        pDevIns->Internal.s.uLastIrqTag = uTagSrc = pdmCalcIrqTag(pVM, pDevIns->idTracing);
+        if (iLevel == PDM_IRQ_LEVEL_HIGH)
+            VBOXVMM_PDM_IRQ_HIGH(VMMGetCpu(pVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+        else
+            VBOXVMM_PDM_IRQ_HILO(VMMGetCpu(pVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+    }
+    else
+        uTagSrc = pDevIns->Internal.s.uLastIrqTag;
+
+    PDMIsaSetIrq(pVM, iIrq, iLevel, uTagSrc); /* (The API takes the lock recursively.) */
+
+    if (iLevel == PDM_IRQ_LEVEL_LOW)
+        VBOXVMM_PDM_IRQ_LOW(VMMGetCpu(pVM), RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc));
+    pdmUnlock(pVM);
+    return 0;
+}
+
+
+/**
+ * HPET Device Helpers.
+ */
+const PDMHPETHLPR3 g_pdmR3DevHpetHlp =
+{
+    PDM_HPETHLPR3_VERSION,
+    pdmR3HpetHlp_SetLegacyMode,
+    pdmR3HpetHlp_SetIrq,
+    PDM_HPETHLPR3_VERSION, /* the end */
+};
+
+/** @} */
+
+
+/** @name Ring-3 Raw PCI Device Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMPCIRAWHLPR3,pfnGetRCHelpers} */
+static DECLCALLBACK(PCPDMPCIRAWHLPRC) pdmR3PciRawHlp_GetRCHelpers(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    RTRCPTR pRCHelpers = NIL_RTRCPTR;
+    if (VM_IS_RAW_MODE_ENABLED(pVM))
+    {
+        int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_pdmRCPciRawHlp", &pRCHelpers);
+        AssertReleaseRC(rc);
+        AssertRelease(pRCHelpers);
+    }
+
+    LogFlow(("pdmR3PciRawHlp_GetGCHelpers: caller='%s'/%d: returns %RRv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pRCHelpers));
+    return pRCHelpers;
+}
+
+
+/** @interface_method_impl{PDMPCIRAWHLPR3,pfnGetR0Helpers} */
+static DECLCALLBACK(PCPDMPCIRAWHLPR0) pdmR3PciRawHlp_GetR0Helpers(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    PCPDMHPETHLPR0 pR0Helpers = NIL_RTR0PTR;
+    int rc = PDMR3LdrGetSymbolR0(pVM, NULL, "g_pdmR0PciRawHlp", &pR0Helpers);
+    AssertReleaseRC(rc);
+    AssertRelease(pR0Helpers);
+    LogFlow(("pdmR3PciRawHlp_GetR0Helpers: caller='%s'/%d: returns %RHv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pR0Helpers));
+    return pR0Helpers;
+}
+
+
+/**
+ * Raw PCI Device Helpers.
+ */
+const PDMPCIRAWHLPR3 g_pdmR3DevPciRawHlp =
+{
+    PDM_PCIRAWHLPR3_VERSION,
+    pdmR3PciRawHlp_GetRCHelpers,
+    pdmR3PciRawHlp_GetR0Helpers,
+    PDM_PCIRAWHLPR3_VERSION, /* the end */
+};
+
+/** @} */
+
+
+/* none yet */
+
+/**
+ * Firmware Device Helpers.
+ */
+const PDMFWHLPR3 g_pdmR3DevFirmwareHlp =
+{
+    PDM_FWHLPR3_VERSION,
+    PDM_FWHLPR3_VERSION
+};
+
+/**
+ * DMAC Device Helpers.
+ */
+const PDMDMACHLP g_pdmR3DevDmacHlp =
+{
+    PDM_DMACHLP_VERSION
+};
+
+
+
+
+/* none yet */
+
+/**
+ * RTC Device Helpers.
+ */
+const PDMRTCHLP g_pdmR3DevRtcHlp =
+{
+    PDM_RTCHLP_VERSION
+};
+
diff --git a/src/VBox/VMM/VMMR3/PDMDevice.cpp b/src/VBox/VMM/VMMR3/PDMDevice.cpp
new file mode 100644
index 00000000..7659c283
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMDevice.cpp
@@ -0,0 +1,1211 @@
+/* $Id: PDMDevice.cpp $ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, Device parts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DEVICE
+#define PDMPCIDEV_INCLUDE_PRIVATE  /* Hack to get pdmpcidevint.h included at the right point. */
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/vmm.h>
+
+#include <VBox/version.h>
+#include <VBox/log.h>
+#include <VBox/msi.h>
+#include <VBox/err.h>
+#include <iprt/alloc.h>
+#include <iprt/alloca.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/path.h>
+#include <iprt/semaphore.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Internal callback structure pointer.
+ * The main purpose is to define the extra data we associate
+ * with PDMDEVREGCB so we can find the VM instance and so on.
+ */
+typedef struct PDMDEVREGCBINT
+{
+    /** The callback structure. */
+    PDMDEVREGCB     Core;
+    /** A bit of padding. */
+    uint32_t        u32[4];
+    /** VM Handle. */
+    PVM             pVM;
+    /** Pointer to the configuration node the registrations should be
+     * associated with.  Can be NULL. */
+    PCFGMNODE       pCfgNode;
+} PDMDEVREGCBINT;
+/** Pointer to a PDMDEVREGCBINT structure. */
+typedef PDMDEVREGCBINT *PPDMDEVREGCBINT;
+/** Pointer to a const PDMDEVREGCBINT structure. */
+typedef const PDMDEVREGCBINT *PCPDMDEVREGCBINT;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(int)    pdmR3DevReg_Register(PPDMDEVREGCB pCallbacks, PCPDMDEVREG pReg);
+static int                  pdmR3DevLoadModules(PVM pVM);
+static int                  pdmR3DevLoad(PVM pVM, PPDMDEVREGCBINT pRegCB, const char *pszFilename, const char *pszName);
+
+
+
+
+/**
+ * This function will initialize the devices for this VM instance.
+ *
+ *
+ * First of all this mean loading the builtin device and letting them
+ * register themselves. Beyond that any additional device modules are
+ * loaded and called for registration.
+ *
+ * Then the device configuration is enumerated, the instantiation order
+ * is determined, and finally they are instantiated.
+ *
+ * After all devices have been successfully instantiated the primary
+ * PCI Bus device is called to emulate the PCI BIOS, i.e. making the
+ * resource assignments. If there is no PCI device, this step is of course
+ * skipped.
+ *
+ * Finally the init completion routines of the instantiated devices
+ * are called.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+int pdmR3DevInit(PVM pVM)
+{
+    LogFlow(("pdmR3DevInit:\n"));
+
+    AssertRelease(!(RT_UOFFSETOF(PDMDEVINS, achInstanceData) & 15));
+    AssertRelease(sizeof(pVM->pdm.s.pDevInstances->Internal.s) <= sizeof(pVM->pdm.s.pDevInstances->Internal.padding));
+
+    /*
+     * Load device modules.
+     */
+    int rc = pdmR3DevLoadModules(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+#ifdef VBOX_WITH_USB
+    /* ditto for USB Devices. */
+    rc = pdmR3UsbLoadModules(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+#endif
+
+    /*
+     * Get the RC & R0 devhlps and create the devhlp R3 task queue.
+     */
+    rc = PDMR3QueueCreateInternal(pVM, sizeof(PDMDEVHLPTASK), 8, 0, pdmR3DevHlpQueueConsumer, true, "DevHlp",
+                                  &pVM->pdm.s.pDevHlpQueueR3);
+    AssertRCReturn(rc, rc);
+    pVM->pdm.s.pDevHlpQueueR0 = PDMQueueR0Ptr(pVM->pdm.s.pDevHlpQueueR3);
+    pVM->pdm.s.pDevHlpQueueRC = PDMQueueRCPtr(pVM->pdm.s.pDevHlpQueueR3);
+
+
+    /*
+     *
+     * Enumerate the device instance configurations
+     * and come up with a instantiation order.
+     *
+     */
+    /* Switch to /Devices, which contains the device instantiations. */
+    PCFGMNODE pDevicesNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "Devices");
+
+    /*
+     * Count the device instances.
+     */
+    PCFGMNODE pCur;
+    PCFGMNODE pInstanceNode;
+    unsigned cDevs = 0;
+    for (pCur = CFGMR3GetFirstChild(pDevicesNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+        for (pInstanceNode = CFGMR3GetFirstChild(pCur); pInstanceNode; pInstanceNode = CFGMR3GetNextChild(pInstanceNode))
+            cDevs++;
+    if (!cDevs)
+    {
+        Log(("PDM: No devices were configured!\n"));
+        return VINF_SUCCESS;
+    }
+    Log2(("PDM: cDevs=%u\n", cDevs));
+
+    /*
+     * Collect info on each device instance.
+     */
+    struct DEVORDER
+    {
+        /** Configuration node. */
+        PCFGMNODE   pNode;
+        /** Pointer to device. */
+        PPDMDEV     pDev;
+        /** Init order. */
+        uint32_t    u32Order;
+        /** VBox instance number. */
+        uint32_t    iInstance;
+    } *paDevs = (struct DEVORDER *)alloca(sizeof(paDevs[0]) * (cDevs + 1)); /* (One extra for swapping) */
+    Assert(paDevs);
+    unsigned i = 0;
+    for (pCur = CFGMR3GetFirstChild(pDevicesNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+    {
+        /* Get the device name. */
+        char szName[sizeof(paDevs[0].pDev->pReg->szName)];
+        rc = CFGMR3GetName(pCur, szName, sizeof(szName));
+        AssertMsgRCReturn(rc, ("Configuration error: device name is too long (or something)! rc=%Rrc\n", rc), rc);
+
+        /* Find the device. */
+        PPDMDEV pDev = pdmR3DevLookup(pVM, szName);
+        AssertLogRelMsgReturn(pDev, ("Configuration error: device '%s' not found!\n", szName), VERR_PDM_DEVICE_NOT_FOUND);
+
+        /* Configured priority or use default based on device class? */
+        uint32_t u32Order;
+        rc = CFGMR3QueryU32(pCur, "Priority", &u32Order);
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        {
+            uint32_t u32 = pDev->pReg->fClass;
+            for (u32Order = 1; !(u32 & u32Order); u32Order <<= 1)
+                /* nop */;
+        }
+        else
+            AssertMsgRCReturn(rc, ("Configuration error: reading \"Priority\" for the '%s' device failed rc=%Rrc!\n", szName, rc), rc);
+
+        /* Enumerate the device instances. */
+        uint32_t const iStart = i;
+        for (pInstanceNode = CFGMR3GetFirstChild(pCur); pInstanceNode; pInstanceNode = CFGMR3GetNextChild(pInstanceNode))
+        {
+            paDevs[i].pNode = pInstanceNode;
+            paDevs[i].pDev = pDev;
+            paDevs[i].u32Order = u32Order;
+
+            /* Get the instance number. */
+            char szInstance[32];
+            rc = CFGMR3GetName(pInstanceNode, szInstance, sizeof(szInstance));
+            AssertMsgRCReturn(rc, ("Configuration error: instance name is too long (or something)! rc=%Rrc\n", rc), rc);
+            char *pszNext = NULL;
+            rc = RTStrToUInt32Ex(szInstance, &pszNext, 0, &paDevs[i].iInstance);
+            AssertMsgRCReturn(rc, ("Configuration error: RTStrToInt32Ex failed on the instance name '%s'! rc=%Rrc\n", szInstance, rc), rc);
+            AssertMsgReturn(!*pszNext, ("Configuration error: the instance name '%s' isn't all digits. (%s)\n", szInstance, pszNext), VERR_INVALID_PARAMETER);
+
+            /* next instance */
+            i++;
+        }
+
+        /* check the number of instances */
+        if (i - iStart > pDev->pReg->cMaxInstances)
+            AssertLogRelMsgFailedReturn(("Configuration error: Too many instances of %s was configured: %u, max %u\n",
+                                         szName, i - iStart, pDev->pReg->cMaxInstances),
+                                        VERR_PDM_TOO_MANY_DEVICE_INSTANCES);
+    } /* devices */
+    Assert(i == cDevs);
+
+    /*
+     * Sort (bubble) the device array ascending on u32Order and instance number
+     * for a device.
+     */
+    unsigned c = cDevs - 1;
+    while (c)
+    {
+        unsigned j = 0;
+        for (i = 0; i < c; i++)
+            if (   paDevs[i].u32Order > paDevs[i + 1].u32Order
+                || (   paDevs[i].u32Order  == paDevs[i + 1].u32Order
+                    && paDevs[i].iInstance >  paDevs[i + 1].iInstance
+                    && paDevs[i].pDev      == paDevs[i + 1].pDev) )
+            {
+                paDevs[cDevs] = paDevs[i + 1];
+                paDevs[i + 1] = paDevs[i];
+                paDevs[i] = paDevs[cDevs];
+                j = i;
+            }
+        c = j;
+    }
+
+
+    /*
+     *
+     * Instantiate the devices.
+     *
+     */
+    for (i = 0; i < cDevs; i++)
+    {
+        PDMDEVREGR3 const * const pReg = paDevs[i].pDev->pReg;
+
+        /*
+         * Gather a bit of config.
+         */
+        /* trusted */
+        bool fTrusted;
+        rc = CFGMR3QueryBool(paDevs[i].pNode, "Trusted", &fTrusted);
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+            fTrusted = false;
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("configuration error: failed to query boolean \"Trusted\", rc=%Rrc\n", rc));
+            return rc;
+        }
+
+        /* RZEnabled, R0Enabled, RCEnabled*/
+        bool fR0Enabled = false;
+        bool fRCEnabled = false;
+        if (pReg->fFlags & (PDM_DEVREG_FLAGS_R0 | PDM_DEVREG_FLAGS_RC))
+        {
+            if (pReg->fFlags & PDM_DEVREG_FLAGS_R0)
+            {
+                if (pReg->fFlags & PDM_DEVREG_FLAGS_REQUIRE_R0)
+                    fR0Enabled = true;
+                else
+                {
+                    rc = CFGMR3QueryBoolDef(paDevs[i].pNode, "R0Enabled", &fR0Enabled,
+                                            !(pReg->fFlags & PDM_DEVREG_FLAGS_OPT_IN_R0));
+                    AssertLogRelRCReturn(rc, rc);
+                }
+            }
+
+            if (pReg->fFlags & PDM_DEVREG_FLAGS_RC)
+            {
+                if (pReg->fFlags & PDM_DEVREG_FLAGS_REQUIRE_RC)
+                    fRCEnabled = true;
+                else
+                {
+                    rc = CFGMR3QueryBoolDef(paDevs[i].pNode, "RCEnabled", &fRCEnabled,
+                                            !(pReg->fFlags & PDM_DEVREG_FLAGS_OPT_IN_RC));
+                    AssertLogRelRCReturn(rc, rc);
+                }
+                fRCEnabled = false;
+            }
+        }
+
+        /* config node */
+        PCFGMNODE pConfigNode = CFGMR3GetChild(paDevs[i].pNode, "Config");
+        if (!pConfigNode)
+        {
+            rc = CFGMR3InsertNode(paDevs[i].pNode, "Config", &pConfigNode);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("Failed to create Config node! rc=%Rrc\n", rc));
+                return rc;
+            }
+        }
+        CFGMR3SetRestrictedRoot(pConfigNode);
+
+        /*
+         * Allocate the device instance and critical section.
+         */
+        AssertLogRelReturn(paDevs[i].pDev->cInstances < pReg->cMaxInstances,
+                           VERR_PDM_TOO_MANY_DEVICE_INSTANCES);
+        PPDMDEVINS   pDevIns;
+        PPDMCRITSECT pCritSect;
+        if (fR0Enabled || fRCEnabled)
+        {
+            AssertLogRel(fR0Enabled /* not possible to just enabled raw-mode atm. */);
+
+            rc = PDMR3LdrLoadR0(pVM->pUVM, pReg->pszR0Mod, paDevs[i].pDev->pszR0SearchPath);
+            if (RT_FAILURE(rc))
+                return VMR3SetError(pVM->pUVM, rc, RT_SRC_POS, "Failed to load ring-0 module '%s' for device '%s'",
+                                    pReg->pszR0Mod, pReg->szName);
+
+            PDMDEVICECREATEREQ Req;
+            Req.Hdr.u32Magic     = SUPVMMR0REQHDR_MAGIC;
+            Req.Hdr.cbReq        = sizeof(Req);
+            Req.pDevInsR3        = NULL;
+            Req.fFlags           = pReg->fFlags;
+            Req.fClass           = pReg->fClass;
+            Req.cMaxInstances    = pReg->cMaxInstances;
+            Req.uSharedVersion   = pReg->uSharedVersion;
+            Req.cbInstanceShared = pReg->cbInstanceShared;
+            Req.cbInstanceR3     = pReg->cbInstanceCC;
+            Req.cbInstanceRC     = pReg->cbInstanceRC;
+            Req.cMaxPciDevices   = pReg->cMaxPciDevices;
+            Req.cMaxMsixVectors  = pReg->cMaxMsixVectors;
+            Req.iInstance        = paDevs[i].iInstance;
+            Req.fRCEnabled       = fRCEnabled;
+            Req.afReserved[0]    = false;
+            Req.afReserved[1]    = false;
+            Req.afReserved[2]    = false;
+            rc = RTStrCopy(Req.szDevName, sizeof(Req.szDevName), pReg->szName);
+            AssertLogRelRCReturn(rc, rc);
+            rc = RTStrCopy(Req.szModName, sizeof(Req.szModName), pReg->pszR0Mod);
+            AssertLogRelRCReturn(rc, rc);
+            rc = VMMR3CallR0Emt(pVM, pVM->apCpusR3[0], VMMR0_DO_PDM_DEVICE_CREATE, 0, &Req.Hdr);
+            AssertLogRelMsgRCReturn(rc, ("VMMR0_DO_PDM_DEVICE_CREATE for %s failed: %Rrc\n", pReg->szName, rc), rc);
+            pDevIns = Req.pDevInsR3;
+            pCritSect = pDevIns->pCritSectRoR3;
+            Assert(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_R0_ENABLED);
+        }
+        else
+        {
+            /* The code in this else branch works by the same rules as the PDMR0Device.cpp
+               code, except there is only the ring-3 components of the device instance.
+               Changes here may need to be reflected in PDMR0DEvice.cpp and vice versa! */
+            uint32_t cb = RT_UOFFSETOF_DYN(PDMDEVINS, achInstanceData[pReg->cbInstanceCC]);
+            cb  = RT_ALIGN_32(cb, 64);
+            uint32_t const offShared   = cb;
+            cb += RT_ALIGN_32(pReg->cbInstanceShared, 64);
+            uint32_t const cbCritSect  = RT_ALIGN_32(sizeof(*pCritSect), 64);
+            cb += cbCritSect;
+            uint32_t const cbMsixState = RT_ALIGN_32(pReg->cMaxMsixVectors * 16 + (pReg->cMaxMsixVectors + 7) / 8, _4K);
+            uint32_t const cbPciDev    = RT_ALIGN_32(RT_UOFFSETOF_DYN(PDMPCIDEV, abMsixState[cbMsixState]), 64);
+            uint32_t const cPciDevs    = RT_MIN(pReg->cMaxPciDevices, 1024);
+            uint32_t const cbPciDevs   = cbPciDev * cPciDevs;
+            cb += cbPciDevs;
+            AssertLogRelMsgReturn(cb <= PDM_MAX_DEVICE_INSTANCE_SIZE_R3,
+                                  ("Device %s total instance size is to big: %u, max %u\n",
+                                   pReg->szName, cb, PDM_MAX_DEVICE_INSTANCE_SIZE_R3),
+                                  VERR_ALLOCATION_TOO_BIG);
+
+            rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_DEVICE, cb, (void **)&pDevIns);
+            AssertLogRelMsgRCReturn(rc, ("Failed to allocate %zu bytes of instance data for device '%s'. rc=%Rrc\n",
+                                         cb, pReg->szName, rc), rc);
+
+            /* Initialize it: */
+            pDevIns->u32Version             = PDM_DEVINSR3_VERSION;
+            pDevIns->iInstance              = paDevs[i].iInstance;
+            pDevIns->cbRing3                = cb;
+            //pDevIns->fR0Enabled           = false;
+            //pDevIns->fRCEnabled           = false;
+            pDevIns->pvInstanceDataR3       = (uint8_t *)pDevIns + offShared;
+            pDevIns->pvInstanceDataForR3    = &pDevIns->achInstanceData[0];
+            pCritSect = (PPDMCRITSECT)((uint8_t *)pDevIns + offShared + RT_ALIGN_32(pReg->cbInstanceShared, 64));
+            pDevIns->pCritSectRoR3          = pCritSect;
+            pDevIns->cbPciDev               = cbPciDev;
+            pDevIns->cPciDevs               = cPciDevs;
+            for (uint32_t iPciDev = 0; iPciDev < cPciDevs; iPciDev++)
+            {
+                PPDMPCIDEV pPciDev = (PPDMPCIDEV)((uint8_t *)pDevIns->pCritSectRoR3 + cbCritSect + cbPciDev * iPciDev);
+                if (iPciDev < RT_ELEMENTS(pDevIns->apPciDevs))
+                    pDevIns->apPciDevs[iPciDev] = pPciDev;
+                pPciDev->cbConfig           = _4K;
+                pPciDev->cbMsixState        = cbMsixState;
+                pPciDev->idxSubDev          = (uint16_t)iPciDev;
+                pPciDev->Int.s.idxSubDev    = (uint16_t)iPciDev;
+                pPciDev->u32Magic           = PDMPCIDEV_MAGIC;
+            }
+        }
+
+        pDevIns->pHlpR3                         = fTrusted ? &g_pdmR3DevHlpTrusted : &g_pdmR3DevHlpUnTrusted;
+        pDevIns->pReg                           = pReg;
+        pDevIns->pCfg                           = pConfigNode;
+        //pDevIns->IBase.pfnQueryInterface        = NULL;
+        //pDevIns->fTracing                       = 0;
+        pDevIns->idTracing                      = ++pVM->pdm.s.idTracingDev;
+
+        //pDevIns->Internal.s.pNextR3             = NULL;
+        //pDevIns->Internal.s.pPerDeviceNextR3    = NULL;
+        pDevIns->Internal.s.pDevR3              = paDevs[i].pDev;
+        //pDevIns->Internal.s.pLunsR3             = NULL;
+        //pDevIns->Internal.s.pfnAsyncNotify      = NULL;
+        pDevIns->Internal.s.pCfgHandle          = paDevs[i].pNode;
+        pDevIns->Internal.s.pVMR3               = pVM;
+        //pDevIns->Internal.s.pHeadPciDevR3       = NULL;
+        pDevIns->Internal.s.fIntFlags          |= PDMDEVINSINT_FLAGS_SUSPENDED;
+        //pDevIns->Internal.s.uLastIrqTag         = 0;
+
+        rc = pdmR3CritSectInitDeviceAuto(pVM, pDevIns, pCritSect, RT_SRC_POS,
+                                         "%s#%uAuto", pDevIns->pReg->szName, pDevIns->iInstance);
+        AssertLogRelRCReturn(rc, rc);
+
+        /*
+         * Link it into all the lists.
+         */
+        /* The global instance FIFO. */
+        PPDMDEVINS pPrev1 = pVM->pdm.s.pDevInstances;
+        if (!pPrev1)
+            pVM->pdm.s.pDevInstances = pDevIns;
+        else
+        {
+            while (pPrev1->Internal.s.pNextR3)
+                pPrev1 = pPrev1->Internal.s.pNextR3;
+            pPrev1->Internal.s.pNextR3 = pDevIns;
+        }
+
+        /* The per device instance FIFO. */
+        PPDMDEVINS pPrev2 = paDevs[i].pDev->pInstances;
+        if (!pPrev2)
+            paDevs[i].pDev->pInstances = pDevIns;
+        else
+        {
+            while (pPrev2->Internal.s.pPerDeviceNextR3)
+                pPrev2 = pPrev2->Internal.s.pPerDeviceNextR3;
+            pPrev2->Internal.s.pPerDeviceNextR3 = pDevIns;
+        }
+
+        /*
+         * Call the constructor.
+         */
+        paDevs[i].pDev->cInstances++;
+        Log(("PDM: Constructing device '%s' instance %d...\n", pDevIns->pReg->szName, pDevIns->iInstance));
+        rc = pDevIns->pReg->pfnConstruct(pDevIns, pDevIns->iInstance, pDevIns->pCfg);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDM: Failed to construct '%s'/%d! %Rra\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            if (VMR3GetErrorCount(pVM->pUVM) == 0)
+                VMSetError(pVM, rc, RT_SRC_POS, "Failed to construct device '%s' instance #%u",
+                           pDevIns->pReg->szName, pDevIns->iInstance);
+            /* Because we're damn lazy, the destructor will be called even if
+               the constructor fails.  So, no unlinking. */
+            paDevs[i].pDev->cInstances--;
+            return rc == VERR_VERSION_MISMATCH ? VERR_PDM_DEVICE_VERSION_MISMATCH : rc;
+        }
+
+        /*
+         * Call the ring-0 constructor if applicable.
+         */
+        if (fR0Enabled)
+        {
+            PDMDEVICEGENCALLREQ Req;
+            RT_ZERO(Req.Params);
+            Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+            Req.Hdr.cbReq    = sizeof(Req);
+            Req.enmCall      = PDMDEVICEGENCALL_CONSTRUCT;
+            Req.idxR0Device  = pDevIns->Internal.s.idxR0Device;
+            Req.pDevInsR3    = pDevIns;
+            rc = VMMR3CallR0Emt(pVM, pVM->apCpusR3[0], VMMR0_DO_PDM_DEVICE_GEN_CALL, 0, &Req.Hdr);
+            pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_R0_CONTRUCT;
+            if (RT_FAILURE(rc))
+            {
+                LogRel(("PDM: Failed to construct (ring-0) '%s'/%d! %Rra\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+                if (VMR3GetErrorCount(pVM->pUVM) == 0)
+                    VMSetError(pVM, rc, RT_SRC_POS, "The ring-0 constructor of device '%s' instance #%u failed",
+                               pDevIns->pReg->szName, pDevIns->iInstance);
+                paDevs[i].pDev->cInstances--;
+                return rc == VERR_VERSION_MISMATCH ? VERR_PDM_DEVICE_VERSION_MISMATCH : rc;
+            }
+        }
+
+    } /* for device instances */
+
+#ifdef VBOX_WITH_USB
+    /* ditto for USB Devices. */
+    rc = pdmR3UsbInstantiateDevices(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+#endif
+
+    LogFlow(("pdmR3DevInit: returns %Rrc\n", VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Performs the init complete callback after ring-0 and raw-mode has been
+ * initialized.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+int pdmR3DevInitComplete(PVM pVM)
+{
+    int rc;
+
+    /*
+     * Iterate thru the device instances and work the callback.
+     */
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        if (pDevIns->pReg->pfnInitComplete)
+        {
+            PDMCritSectEnter(pDevIns->pCritSectRoR3, VERR_IGNORED);
+            rc = pDevIns->pReg->pfnInitComplete(pDevIns);
+            PDMCritSectLeave(pDevIns->pCritSectRoR3);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("InitComplete on device '%s'/%d failed with rc=%Rrc\n",
+                                 pDevIns->pReg->szName, pDevIns->iInstance, rc));
+                return rc;
+            }
+        }
+    }
+
+#ifdef VBOX_WITH_USB
+    rc = pdmR3UsbVMInitComplete(pVM);
+    if (RT_FAILURE(rc))
+    {
+        Log(("pdmR3DevInit: returns %Rrc\n", rc));
+        return rc;
+    }
+#endif
+
+    LogFlow(("pdmR3DevInit: returns %Rrc\n", VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Lookups a device structure by name.
+ * @internal
+ */
+PPDMDEV pdmR3DevLookup(PVM pVM, const char *pszName)
+{
+    size_t cchName = strlen(pszName);
+    for (PPDMDEV pDev = pVM->pdm.s.pDevs; pDev; pDev = pDev->pNext)
+        if (    pDev->cchName == cchName
+            &&  !strcmp(pDev->pReg->szName, pszName))
+            return pDev;
+    return NULL;
+}
+
+
+/**
+ * Loads the device modules.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+static int pdmR3DevLoadModules(PVM pVM)
+{
+    /*
+     * Initialize the callback structure.
+     */
+    PDMDEVREGCBINT RegCB;
+    RegCB.Core.u32Version  = PDM_DEVREG_CB_VERSION;
+    RegCB.Core.pfnRegister = pdmR3DevReg_Register;
+    RegCB.pVM              = pVM;
+    RegCB.pCfgNode         = NULL;
+
+    /*
+     * Register the internal VMM APIC device.
+     */
+    int rc = pdmR3DevReg_Register(&RegCB.Core, &g_DeviceAPIC);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Load the builtin module.
+     */
+    PCFGMNODE pDevicesNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM/Devices");
+    bool fLoadBuiltin;
+    rc = CFGMR3QueryBool(pDevicesNode, "LoadBuiltin", &fLoadBuiltin);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+        fLoadBuiltin = true;
+    else if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Configuration error: Querying boolean \"LoadBuiltin\" failed with %Rrc\n", rc));
+        return rc;
+    }
+    if (fLoadBuiltin)
+    {
+        /* make filename */
+        char *pszFilename = pdmR3FileR3("VBoxDD", true /*fShared*/);
+        if (!pszFilename)
+            return VERR_NO_TMP_MEMORY;
+        rc = pdmR3DevLoad(pVM, &RegCB, pszFilename, "VBoxDD");
+        RTMemTmpFree(pszFilename);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* make filename */
+        pszFilename = pdmR3FileR3("VBoxDD2", true /*fShared*/);
+        if (!pszFilename)
+            return VERR_NO_TMP_MEMORY;
+        rc = pdmR3DevLoad(pVM, &RegCB, pszFilename, "VBoxDD2");
+        RTMemTmpFree(pszFilename);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Load additional device modules.
+     */
+    PCFGMNODE pCur;
+    for (pCur = CFGMR3GetFirstChild(pDevicesNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+    {
+        /*
+         * Get the name and path.
+         */
+        char szName[PDMMOD_NAME_LEN];
+        rc = CFGMR3GetName(pCur, &szName[0], sizeof(szName));
+        if (rc == VERR_CFGM_NOT_ENOUGH_SPACE)
+        {
+            AssertMsgFailed(("configuration error: The module name is too long, cchName=%zu.\n", CFGMR3GetNameLen(pCur)));
+            return VERR_PDM_MODULE_NAME_TOO_LONG;
+        }
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("CFGMR3GetName -> %Rrc.\n", rc));
+            return rc;
+        }
+
+        /* the path is optional, if no path the module name + path is used. */
+        char szFilename[RTPATH_MAX];
+        rc = CFGMR3QueryString(pCur, "Path", &szFilename[0], sizeof(szFilename));
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+            strcpy(szFilename, szName);
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("configuration error: Failure to query the module path, rc=%Rrc.\n", rc));
+            return rc;
+        }
+
+        /* prepend path? */
+        if (!RTPathHavePath(szFilename))
+        {
+            char *psz = pdmR3FileR3(szFilename, false /*fShared*/);
+            if (!psz)
+                return VERR_NO_TMP_MEMORY;
+            size_t cch = strlen(psz) + 1;
+            if (cch > sizeof(szFilename))
+            {
+                RTMemTmpFree(psz);
+                AssertMsgFailed(("Filename too long! cch=%d '%s'\n", cch, psz));
+                return VERR_FILENAME_TOO_LONG;
+            }
+            memcpy(szFilename, psz, cch);
+            RTMemTmpFree(psz);
+        }
+
+        /*
+         * Load the module and register it's devices.
+         */
+        RegCB.pCfgNode = pCur;
+        rc = pdmR3DevLoad(pVM, &RegCB, szFilename, szName);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads one device module and call the registration entry point.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pRegCB          The registration callback stuff.
+ * @param   pszFilename     Module filename.
+ * @param   pszName         Module name.
+ */
+static int pdmR3DevLoad(PVM pVM, PPDMDEVREGCBINT pRegCB, const char *pszFilename, const char *pszName)
+{
+    /*
+     * Load it.
+     */
+    int rc = pdmR3LoadR3U(pVM->pUVM, pszFilename, pszName);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Get the registration export and call it.
+         */
+        FNPDMVBOXDEVICESREGISTER *pfnVBoxDevicesRegister;
+        rc = PDMR3LdrGetSymbolR3(pVM, pszName, "VBoxDevicesRegister", (void **)&pfnVBoxDevicesRegister);
+        if (RT_SUCCESS(rc))
+        {
+            Log(("PDM: Calling VBoxDevicesRegister (%p) of %s (%s)\n", pfnVBoxDevicesRegister, pszName, pszFilename));
+            rc = pfnVBoxDevicesRegister(&pRegCB->Core, VBOX_VERSION);
+            if (RT_SUCCESS(rc))
+                Log(("PDM: Successfully loaded device module %s (%s).\n", pszName, pszFilename));
+            else
+            {
+                VMR3SetError(pVM->pUVM, rc, RT_SRC_POS, "VBoxDevicesRegister failed with rc=%Rrc for module %s (%s)",
+                             rc, pszName, pszFilename);
+                AssertMsgFailed(("VBoxDevicesRegister failed with rc=%Rrc for module %s (%s)\n", rc, pszName, pszFilename));
+            }
+        }
+        else
+        {
+            AssertMsgFailed(("Failed to locate 'VBoxDevicesRegister' in %s (%s) rc=%Rrc\n", pszName, pszFilename, rc));
+            if (rc == VERR_SYMBOL_NOT_FOUND)
+                rc = VERR_PDM_NO_REGISTRATION_EXPORT;
+            VMR3SetError(pVM->pUVM, rc, RT_SRC_POS, "Failed to locate 'VBoxDevicesRegister' in %s (%s) rc=%Rrc",
+                         pszName, pszFilename, rc);
+        }
+    }
+    else
+        AssertMsgFailed(("Failed to load %s %s!\n", pszFilename, pszName));
+    return rc;
+}
+
+
+/**
+ * @interface_method_impl{PDMDEVREGCB,pfnRegister}
+ */
+static DECLCALLBACK(int) pdmR3DevReg_Register(PPDMDEVREGCB pCallbacks, PCPDMDEVREG pReg)
+{
+    /*
+     * Validate the registration structure.
+     */
+    Assert(pReg);
+    AssertMsgReturn(pReg->u32Version == PDM_DEVREG_VERSION,
+                    ("Unknown struct version %#x!\n", pReg->u32Version),
+                    VERR_PDM_UNKNOWN_DEVREG_VERSION);
+
+    AssertMsgReturn(    pReg->szName[0]
+                    &&  strlen(pReg->szName) < sizeof(pReg->szName)
+                    &&  pdmR3IsValidName(pReg->szName),
+                    ("Invalid name '%.*s'\n", sizeof(pReg->szName), pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn(   !(pReg->fFlags & PDM_DEVREG_FLAGS_RC)
+                    || (   pReg->pszRCMod[0]
+                        && strlen(pReg->pszRCMod) < RT_SIZEOFMEMB(PDMDEVICECREATEREQ, szModName)),
+                    ("Invalid GC module name '%s' - (Device %s)\n", pReg->pszRCMod, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn(   !(pReg->fFlags & PDM_DEVREG_FLAGS_R0)
+                    || (   pReg->pszR0Mod[0]
+                        && strlen(pReg->pszR0Mod) < RT_SIZEOFMEMB(PDMDEVICECREATEREQ, szModName)),
+                    ("Invalid R0 module name '%s' - (Device %s)\n", pReg->pszR0Mod, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn((pReg->fFlags & PDM_DEVREG_FLAGS_HOST_BITS_MASK) == PDM_DEVREG_FLAGS_HOST_BITS_DEFAULT,
+                    ("Invalid host bits flags! fFlags=%#x (Device %s)\n", pReg->fFlags, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_HOST_BITS);
+    AssertMsgReturn((pReg->fFlags & PDM_DEVREG_FLAGS_GUEST_BITS_MASK),
+                    ("Invalid guest bits flags! fFlags=%#x (Device %s)\n", pReg->fFlags, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn(pReg->fClass,
+                    ("No class! (Device %s)\n", pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn(pReg->cMaxInstances > 0,
+                    ("Max instances %u! (Device %s)\n", pReg->cMaxInstances, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    uint32_t const cbMaxInstance = pReg->fFlags & (PDM_DEVREG_FLAGS_RC | PDM_DEVREG_FLAGS_R0)
+                                 ? PDM_MAX_DEVICE_INSTANCE_SIZE : PDM_MAX_DEVICE_INSTANCE_SIZE_R3;
+    AssertMsgReturn(pReg->cbInstanceShared <= cbMaxInstance,
+                    ("Instance size %u bytes! (Max %u; Device %s)\n", pReg->cbInstanceShared, cbMaxInstance, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn(pReg->cbInstanceCC <= cbMaxInstance,
+                    ("Instance size %d bytes! (Max %u; Device %s)\n", pReg->cbInstanceCC, cbMaxInstance, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn(pReg->pfnConstruct,
+                    ("No constructor! (Device %s)\n", pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertLogRelMsgReturn((pReg->fFlags & PDM_DEVREG_FLAGS_GUEST_BITS_MASK) == PDM_DEVREG_FLAGS_GUEST_BITS_DEFAULT,
+                          ("PDM: Rejected device '%s' because it didn't match the guest bits.\n", pReg->szName),
+                          VERR_PDM_INVALID_DEVICE_GUEST_BITS);
+    AssertLogRelMsg(pReg->u32VersionEnd == PDM_DEVREG_VERSION,
+                    ("u32VersionEnd=%#x, expected %#x. (szName=%s)\n",
+                     pReg->u32VersionEnd, PDM_DEVREG_VERSION, pReg->szName));
+    AssertLogRelMsgReturn(pReg->cMaxPciDevices <= 8, ("%#x (szName=%s)\n", pReg->cMaxPciDevices, pReg->szName),
+                          VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertLogRelMsgReturn(pReg->cMaxMsixVectors <= VBOX_MSIX_MAX_ENTRIES,
+                          ("%#x (szName=%s)\n", pReg->cMaxMsixVectors, pReg->szName),
+                          VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertLogRelMsgReturn(pReg->fFlags & PDM_DEVREG_FLAGS_NEW_STYLE /* the flag is required now */,
+                          ("PDM_DEVREG_FLAGS_NEW_STYLE not set for szName=%s!\n", pReg->szName),
+                          VERR_PDM_INVALID_DEVICE_REGISTRATION);
+
+    /*
+     * Check for duplicate and find FIFO entry at the same time.
+     */
+    PCPDMDEVREGCBINT pRegCB = (PCPDMDEVREGCBINT)pCallbacks;
+    PPDMDEV pDevPrev = NULL;
+    PPDMDEV pDev = pRegCB->pVM->pdm.s.pDevs;
+    for (; pDev; pDevPrev = pDev, pDev = pDev->pNext)
+        AssertMsgReturn(strcmp(pDev->pReg->szName, pReg->szName),
+                        ("Device '%s' already exists\n", pReg->szName),
+                        VERR_PDM_DEVICE_NAME_CLASH);
+
+    /*
+     * Allocate new device structure, initialize and insert it into the list.
+     */
+    int rc;
+    pDev = (PPDMDEV)MMR3HeapAlloc(pRegCB->pVM, MM_TAG_PDM_DEVICE, sizeof(*pDev));
+    if (pDev)
+    {
+        pDev->pNext      = NULL;
+        pDev->cInstances = 0;
+        pDev->pInstances = NULL;
+        pDev->pReg       = pReg;
+        pDev->cchName    = (uint32_t)strlen(pReg->szName);
+        rc = CFGMR3QueryStringAllocDef(    pRegCB->pCfgNode, "RCSearchPath", &pDev->pszRCSearchPath, NULL);
+        if (RT_SUCCESS(rc))
+            rc = CFGMR3QueryStringAllocDef(pRegCB->pCfgNode, "R0SearchPath", &pDev->pszR0SearchPath, NULL);
+        if (RT_SUCCESS(rc))
+        {
+            if (pDevPrev)
+                pDevPrev->pNext = pDev;
+            else
+                pRegCB->pVM->pdm.s.pDevs = pDev;
+            Log(("PDM: Registered device '%s'\n", pReg->szName));
+            return VINF_SUCCESS;
+        }
+
+        MMR3HeapFree(pDev);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+    return rc;
+}
+
+
+/**
+ * Locates a LUN.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   ppLun           Where to store the pointer to the LUN if found.
+ * @thread  Try only do this in EMT...
+ */
+int pdmR3DevFindLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPPDMLUN ppLun)
+{
+    /*
+     * Iterate registered devices looking for the device.
+     */
+    size_t cchDevice = strlen(pszDevice);
+    for (PPDMDEV pDev = pVM->pdm.s.pDevs; pDev; pDev = pDev->pNext)
+    {
+        if (    pDev->cchName == cchDevice
+            &&  !memcmp(pDev->pReg->szName, pszDevice, cchDevice))
+        {
+            /*
+             * Iterate device instances.
+             */
+            for (PPDMDEVINS pDevIns = pDev->pInstances; pDevIns; pDevIns = pDevIns->Internal.s.pPerDeviceNextR3)
+            {
+                if (pDevIns->iInstance == iInstance)
+                {
+                    /*
+                     * Iterate luns.
+                     */
+                    for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+                    {
+                        if (pLun->iLun == iLun)
+                        {
+                            *ppLun = pLun;
+                            return VINF_SUCCESS;
+                        }
+                    }
+                    return VERR_PDM_LUN_NOT_FOUND;
+                }
+            }
+            return VERR_PDM_DEVICE_INSTANCE_NOT_FOUND;
+        }
+    }
+    return VERR_PDM_DEVICE_NOT_FOUND;
+}
+
+
+/**
+ * Attaches a preconfigured driver to an existing device instance.
+ *
+ * This is used to change drivers and suchlike at runtime.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
+ * @param   ppBase          Where to store the base interface pointer. Optional.
+ * @thread  EMT
+ */
+VMMR3DECL(int) PDMR3DeviceAttach(PUVM pUVM, const char *pszDevice, unsigned iInstance, unsigned iLun, uint32_t fFlags, PPPDMIBASE ppBase)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("PDMR3DeviceAttach: pszDevice=%p:{%s} iInstance=%d iLun=%d fFlags=%#x ppBase=%p\n",
+             pszDevice, pszDevice, iInstance, iLun, fFlags, ppBase));
+
+    /*
+     * Find the LUN in question.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Can we attach anything at runtime?
+         */
+        PPDMDEVINS pDevIns = pLun->pDevIns;
+        if (pDevIns->pReg->pfnAttach)
+        {
+            if (!pLun->pTop)
+            {
+                PDMCritSectEnter(pDevIns->pCritSectRoR3, VERR_IGNORED);
+                rc = pDevIns->pReg->pfnAttach(pDevIns, iLun, fFlags);
+                PDMCritSectLeave(pDevIns->pCritSectRoR3);
+            }
+            else
+                rc = VERR_PDM_DRIVER_ALREADY_ATTACHED;
+        }
+        else
+            rc = VERR_PDM_DEVICE_NO_RT_ATTACH;
+
+        if (ppBase)
+            *ppBase = pLun->pTop ? &pLun->pTop->IBase : NULL;
+    }
+    else if (ppBase)
+        *ppBase = NULL;
+
+    if (ppBase)
+        LogFlow(("PDMR3DeviceAttach: returns %Rrc *ppBase=%p\n", rc, *ppBase));
+    else
+        LogFlow(("PDMR3DeviceAttach: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Detaches a driver chain from an existing device instance.
+ *
+ * This is used to change drivers and suchlike at runtime.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
+ * @thread  EMT
+ */
+VMMR3DECL(int) PDMR3DeviceDetach(PUVM pUVM, const char *pszDevice, unsigned iInstance, unsigned iLun, uint32_t fFlags)
+{
+    return PDMR3DriverDetach(pUVM, pszDevice, iInstance, iLun, NULL, 0, fFlags);
+}
+
+
+/**
+ * References the critical section associated with a device for the use by a
+ * timer or similar created by the device.
+ *
+ * @returns Pointer to the critical section.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance in question.
+ *
+ * @internal
+ */
+VMMR3_INT_DECL(PPDMCRITSECT) PDMR3DevGetCritSect(PVM pVM, PPDMDEVINS pDevIns)
+{
+    VM_ASSERT_EMT(pVM); RT_NOREF_PV(pVM);
+    VM_ASSERT_STATE(pVM, VMSTATE_CREATING);
+    AssertPtr(pDevIns);
+
+    PPDMCRITSECT pCritSect = pDevIns->pCritSectRoR3;
+    AssertPtr(pCritSect);
+    pCritSect->s.fUsedByTimerOrSimilar = true;
+
+    return pCritSect;
+}
+
+
+/**
+ * Attaches a preconfigured driver to an existing device or driver instance.
+ *
+ * This is used to change drivers and suchlike at runtime.  The driver or device
+ * at the end of the chain will be told to attach to whatever is configured
+ * below it.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
+ * @param   ppBase          Where to store the base interface pointer. Optional.
+ *
+ * @thread  EMT
+ */
+VMMR3DECL(int) PDMR3DriverAttach(PUVM pUVM, const char *pszDevice, unsigned iInstance, unsigned iLun, uint32_t fFlags, PPPDMIBASE ppBase)
+{
+    LogFlow(("PDMR3DriverAttach: pszDevice=%p:{%s} iInstance=%d iLun=%d fFlags=%#x ppBase=%p\n",
+             pszDevice, pszDevice, iInstance, iLun, fFlags, ppBase));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT(pVM);
+
+    if (ppBase)
+        *ppBase = NULL;
+
+    /*
+     * Find the LUN in question.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Anything attached to the LUN?
+         */
+        PPDMDRVINS pDrvIns = pLun->pTop;
+        if (!pDrvIns)
+        {
+            /* No, ask the device to attach to the new stuff. */
+            PPDMDEVINS pDevIns = pLun->pDevIns;
+            if (pDevIns->pReg->pfnAttach)
+            {
+                PDMCritSectEnter(pDevIns->pCritSectRoR3, VERR_IGNORED);
+                rc = pDevIns->pReg->pfnAttach(pDevIns, iLun, fFlags);
+                if (RT_SUCCESS(rc) && ppBase)
+                    *ppBase = pLun->pTop ? &pLun->pTop->IBase : NULL;
+                PDMCritSectLeave(pDevIns->pCritSectRoR3);
+            }
+            else
+                rc = VERR_PDM_DEVICE_NO_RT_ATTACH;
+        }
+        else
+        {
+            /* Yes, find the bottom most driver and ask it to attach to the new stuff. */
+            while (pDrvIns->Internal.s.pDown)
+                pDrvIns = pDrvIns->Internal.s.pDown;
+            if (pDrvIns->pReg->pfnAttach)
+            {
+                rc = pDrvIns->pReg->pfnAttach(pDrvIns, fFlags);
+                if (RT_SUCCESS(rc) && ppBase)
+                    *ppBase = pDrvIns->Internal.s.pDown
+                            ? &pDrvIns->Internal.s.pDown->IBase
+                            : NULL;
+            }
+            else
+                rc = VERR_PDM_DRIVER_NO_RT_ATTACH;
+        }
+    }
+
+    if (ppBase)
+        LogFlow(("PDMR3DriverAttach: returns %Rrc *ppBase=%p\n", rc, *ppBase));
+    else
+        LogFlow(("PDMR3DriverAttach: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Detaches the specified driver instance.
+ *
+ * This is used to replumb drivers at runtime for simulating hot plugging and
+ * media changes.
+ *
+ * This is a superset of PDMR3DeviceDetach.  It allows detaching drivers from
+ * any driver or device by specifying the driver to start detaching at.  The
+ * only prerequisite is that the driver or device above implements the
+ * pfnDetach callback (PDMDRVREG / PDMDEVREG).
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iDevIns         Device instance.
+ * @param   iLun            The Logical Unit in which to look for the driver.
+ * @param   pszDriver       The name of the driver which to detach.  If NULL
+ *                          then the entire driver chain is detatched.
+ * @param   iOccurrence     The occurrence of that driver in the chain.  This is
+ *                          usually 0.
+ * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
+ * @thread  EMT
+ */
+VMMR3DECL(int) PDMR3DriverDetach(PUVM pUVM, const char *pszDevice, unsigned iDevIns, unsigned iLun,
+                                 const char *pszDriver, unsigned iOccurrence, uint32_t fFlags)
+{
+    LogFlow(("PDMR3DriverDetach: pszDevice=%p:{%s} iDevIns=%u iLun=%u pszDriver=%p:{%s} iOccurrence=%u fFlags=%#x\n",
+             pszDevice, pszDevice, iDevIns, iLun, pszDriver, pszDriver, iOccurrence, fFlags));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT(pVM);
+    AssertPtr(pszDevice);
+    AssertPtrNull(pszDriver);
+    Assert(iOccurrence == 0 || pszDriver);
+    Assert(!(fFlags & ~(PDM_TACH_FLAGS_NOT_HOT_PLUG)));
+
+    /*
+     * Find the LUN in question.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3DevFindLun(pVM, pszDevice, iDevIns, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Locate the driver.
+         */
+        PPDMDRVINS pDrvIns = pLun->pTop;
+        if (pDrvIns)
+        {
+            if (pszDriver)
+            {
+                while (pDrvIns)
+                {
+                    if (!strcmp(pDrvIns->pReg->szName, pszDriver))
+                    {
+                        if (iOccurrence == 0)
+                            break;
+                        iOccurrence--;
+                    }
+                    pDrvIns = pDrvIns->Internal.s.pDown;
+                }
+            }
+            if (pDrvIns)
+                rc = pdmR3DrvDetach(pDrvIns, fFlags);
+            else
+                rc = VERR_PDM_DRIVER_INSTANCE_NOT_FOUND;
+        }
+        else
+            rc = VINF_PDM_NO_DRIVER_ATTACHED_TO_LUN;
+    }
+
+    LogFlow(("PDMR3DriverDetach: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Runtime detach and reattach of a new driver chain or sub chain.
+ *
+ * This is intended to be called on a non-EMT thread, this will instantiate the
+ * new driver (sub-)chain, and then the EMTs will do the actual replumbing.  The
+ * destruction of the old driver chain will be taken care of on the calling
+ * thread.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iDevIns         Device instance.
+ * @param   iLun            The Logical Unit in which to look for the driver.
+ * @param   pszDriver       The name of the driver which to detach and replace.
+ *                          If NULL then the entire driver chain is to be
+ *                          reattached.
+ * @param   iOccurrence     The occurrence of that driver in the chain.  This is
+ *                          usually 0.
+ * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
+ * @param   pCfg            The configuration of the new driver chain that is
+ *                          going to be attached.  The subtree starts with the
+ *                          node containing a Driver key, a Config subtree and
+ *                          optionally an AttachedDriver subtree.
+ *                          If this parameter is NULL, then this call will work
+ *                          like at a non-pause version of PDMR3DriverDetach.
+ * @param   ppBase          Where to store the base interface pointer to the new
+ *                          driver.  Optional.
+ *
+ * @thread  Any thread. The EMTs will be involved at some point though.
+ */
+VMMR3DECL(int)  PDMR3DriverReattach(PUVM pUVM, const char *pszDevice, unsigned iDevIns, unsigned iLun,
+                                    const char *pszDriver, unsigned iOccurrence, uint32_t fFlags,
+                                    PCFGMNODE pCfg, PPPDMIBASE ppBase)
+{
+    NOREF(pUVM); NOREF(pszDevice); NOREF(iDevIns); NOREF(iLun); NOREF(pszDriver); NOREF(iOccurrence);
+    NOREF(fFlags); NOREF(pCfg); NOREF(ppBase);
+    return VERR_NOT_IMPLEMENTED;
+}
+
diff --git a/src/VBox/VMM/VMMR3/PDMDriver.cpp b/src/VBox/VMM/VMMR3/PDMDriver.cpp
new file mode 100644
index 00000000..04945af3
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMDriver.cpp
@@ -0,0 +1,1877 @@
+/* $Id: PDMDriver.cpp $ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, Driver parts.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DRIVER
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/version.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/ctype.h>
+#include <iprt/mem.h>
+#include <iprt/thread.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Internal callback structure pointer.
+ *
+ * The main purpose is to define the extra data we associate
+ * with PDMDRVREGCB so we can find the VM instance and so on.
+ */
+typedef struct PDMDRVREGCBINT
+{
+    /** The callback structure. */
+    PDMDRVREGCB     Core;
+    /** A bit of padding. */
+    uint32_t        u32[4];
+    /** VM Handle. */
+    PVM             pVM;
+    /** Pointer to the configuration node the registrations should be
+     * associated with.  Can be NULL. */
+    PCFGMNODE       pCfgNode;
+} PDMDRVREGCBINT, *PPDMDRVREGCBINT;
+typedef const PDMDRVREGCBINT *PCPDMDRVREGCBINT;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(int) pdmR3DrvRegister(PCPDMDRVREGCB pCallbacks, PCPDMDRVREG pReg);
+static int pdmR3DrvLoad(PVM pVM, PPDMDRVREGCBINT pRegCB, const char *pszFilename, const char *pszName);
+
+
+/**
+ * Register drivers in a statically linked environment.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pfnCallback Driver registration callback
+ */
+VMMR3DECL(int) PDMR3DrvStaticRegistration(PVM pVM, FNPDMVBOXDRIVERSREGISTER pfnCallback)
+{
+    /*
+     * The registration callbacks.
+     */
+    PDMDRVREGCBINT  RegCB;
+    RegCB.Core.u32Version   = PDM_DRVREG_CB_VERSION;
+    RegCB.Core.pfnRegister  = pdmR3DrvRegister;
+    RegCB.pVM               = pVM;
+    RegCB.pCfgNode          = NULL;
+
+    int rc = pfnCallback(&RegCB.Core, VBOX_VERSION);
+    if (RT_FAILURE(rc))
+        AssertMsgFailed(("VBoxDriversRegister failed with rc=%Rrc\n", rc));
+
+    return rc;
+}
+
+
+/**
+ * This function will initialize the drivers for this VM instance.
+ *
+ * First of all this mean loading the builtin drivers and letting them
+ * register themselves. Beyond that any additional driver modules are
+ * loaded and called for registration.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+int pdmR3DrvInit(PVM pVM)
+{
+    LogFlow(("pdmR3DrvInit:\n"));
+
+    AssertRelease(!(RT_UOFFSETOF(PDMDRVINS, achInstanceData) & 15));
+    PPDMDRVINS pDrvInsAssert; NOREF(pDrvInsAssert);
+    AssertCompile(sizeof(pDrvInsAssert->Internal.s) <= sizeof(pDrvInsAssert->Internal.padding));
+    AssertRelease(sizeof(pDrvInsAssert->Internal.s) <= sizeof(pDrvInsAssert->Internal.padding));
+
+    /*
+     * The registration callbacks.
+     */
+    PDMDRVREGCBINT  RegCB;
+    RegCB.Core.u32Version   = PDM_DRVREG_CB_VERSION;
+    RegCB.Core.pfnRegister  = pdmR3DrvRegister;
+    RegCB.pVM               = pVM;
+    RegCB.pCfgNode          = NULL;
+
+    /*
+     * Load the builtin module
+     */
+    PCFGMNODE pDriversNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM/Drivers");
+    bool fLoadBuiltin;
+    int rc = CFGMR3QueryBool(pDriversNode, "LoadBuiltin", &fLoadBuiltin);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+        fLoadBuiltin = true;
+    else if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Configuration error: Querying boolean \"LoadBuiltin\" failed with %Rrc\n", rc));
+        return rc;
+    }
+    if (fLoadBuiltin)
+    {
+        /* make filename */
+        char *pszFilename = pdmR3FileR3("VBoxDD", true /*fShared*/);
+        if (!pszFilename)
+            return VERR_NO_TMP_MEMORY;
+        rc = pdmR3DrvLoad(pVM, &RegCB, pszFilename, "VBoxDD");
+        RTMemTmpFree(pszFilename);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Load additional driver modules.
+     */
+    for (PCFGMNODE pCur = CFGMR3GetFirstChild(pDriversNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+    {
+        /*
+         * Get the name and path.
+         */
+        char szName[PDMMOD_NAME_LEN];
+        rc = CFGMR3GetName(pCur, &szName[0], sizeof(szName));
+        if (rc == VERR_CFGM_NOT_ENOUGH_SPACE)
+        {
+            AssertMsgFailed(("configuration error: The module name is too long, cchName=%zu.\n", CFGMR3GetNameLen(pCur)));
+            return VERR_PDM_MODULE_NAME_TOO_LONG;
+        }
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("CFGMR3GetName -> %Rrc.\n", rc));
+            return rc;
+        }
+
+        /* the path is optional, if no path the module name + path is used. */
+        char szFilename[RTPATH_MAX];
+        rc = CFGMR3QueryString(pCur, "Path", &szFilename[0], sizeof(szFilename));
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+            strcpy(szFilename, szName);
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("configuration error: Failure to query the module path, rc=%Rrc.\n", rc));
+            return rc;
+        }
+
+        /* prepend path? */
+        if (!RTPathHavePath(szFilename))
+        {
+            char *psz = pdmR3FileR3(szFilename, false /*fShared*/);
+            if (!psz)
+                return VERR_NO_TMP_MEMORY;
+            size_t cch = strlen(psz) + 1;
+            if (cch > sizeof(szFilename))
+            {
+                RTMemTmpFree(psz);
+                AssertMsgFailed(("Filename too long! cch=%d '%s'\n", cch, psz));
+                return VERR_FILENAME_TOO_LONG;
+            }
+            memcpy(szFilename, psz, cch);
+            RTMemTmpFree(psz);
+        }
+
+        /*
+         * Load the module and register it's drivers.
+         */
+        RegCB.pCfgNode = pCur;
+        rc = pdmR3DrvLoad(pVM, &RegCB, szFilename, szName);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    LogFlow(("pdmR3DrvInit: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads one driver module and call the registration entry point.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pRegCB          The registration callback stuff.
+ * @param   pszFilename     Module filename.
+ * @param   pszName         Module name.
+ */
+static int pdmR3DrvLoad(PVM pVM, PPDMDRVREGCBINT pRegCB, const char *pszFilename, const char *pszName)
+{
+    /*
+     * Load it.
+     */
+    int rc = pdmR3LoadR3U(pVM->pUVM, pszFilename, pszName);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Get the registration export and call it.
+         */
+        FNPDMVBOXDRIVERSREGISTER *pfnVBoxDriversRegister;
+        rc = PDMR3LdrGetSymbolR3(pVM, pszName, "VBoxDriversRegister", (void **)&pfnVBoxDriversRegister);
+        if (RT_SUCCESS(rc))
+        {
+            Log(("PDM: Calling VBoxDriversRegister (%p) of %s (%s)\n", pfnVBoxDriversRegister, pszName, pszFilename));
+            rc = pfnVBoxDriversRegister(&pRegCB->Core, VBOX_VERSION);
+            if (RT_SUCCESS(rc))
+                Log(("PDM: Successfully loaded driver module %s (%s).\n", pszName, pszFilename));
+            else
+                AssertMsgFailed(("VBoxDriversRegister failed with rc=%Rrc\n", rc));
+        }
+        else
+        {
+            AssertMsgFailed(("Failed to locate 'VBoxDriversRegister' in %s (%s) rc=%Rrc\n", pszName, pszFilename, rc));
+            if (rc == VERR_SYMBOL_NOT_FOUND)
+                rc = VERR_PDM_NO_REGISTRATION_EXPORT;
+        }
+    }
+    else
+        AssertMsgFailed(("Failed to load %s (%s) rc=%Rrc!\n", pszName, pszFilename, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVREGCB,pfnRegister} */
+static DECLCALLBACK(int) pdmR3DrvRegister(PCPDMDRVREGCB pCallbacks, PCPDMDRVREG pReg)
+{
+    /*
+     * Validate the registration structure.
+     */
+    AssertPtrReturn(pReg, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReg->u32Version == PDM_DRVREG_VERSION,
+                    ("%#x\n", pReg->u32Version),
+                    VERR_PDM_UNKNOWN_DRVREG_VERSION);
+    AssertReturn(pReg->szName[0], VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(RTStrEnd(pReg->szName, sizeof(pReg->szName)),
+                    ("%.*s\n", sizeof(pReg->szName), pReg->szName),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(pdmR3IsValidName(pReg->szName), ("%.*s\n", sizeof(pReg->szName), pReg->szName),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(    !(pReg->fFlags & PDM_DRVREG_FLAGS_R0)
+                    ||  (   pReg->szR0Mod[0]
+                         && RTStrEnd(pReg->szR0Mod, sizeof(pReg->szR0Mod))),
+                    ("%s: %.*s\n", pReg->szName, sizeof(pReg->szR0Mod), pReg->szR0Mod),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(    !(pReg->fFlags & PDM_DRVREG_FLAGS_RC)
+                    ||  (   pReg->szRCMod[0]
+                         && RTStrEnd(pReg->szRCMod, sizeof(pReg->szRCMod))),
+                    ("%s: %.*s\n", pReg->szName, sizeof(pReg->szRCMod), pReg->szRCMod),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(VALID_PTR(pReg->pszDescription),
+                    ("%s: %p\n", pReg->szName, pReg->pszDescription),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(!(pReg->fFlags & ~(PDM_DRVREG_FLAGS_HOST_BITS_MASK | PDM_DRVREG_FLAGS_R0 | PDM_DRVREG_FLAGS_RC)),
+                    ("%s: %#x\n", pReg->szName, pReg->fFlags),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn((pReg->fFlags & PDM_DRVREG_FLAGS_HOST_BITS_MASK) == PDM_DRVREG_FLAGS_HOST_BITS_DEFAULT,
+                    ("%s: %#x\n", pReg->szName, pReg->fFlags),
+                    VERR_PDM_INVALID_DRIVER_HOST_BITS);
+    AssertMsgReturn(pReg->cMaxInstances > 0,
+                    ("%s: %#x\n", pReg->szName, pReg->cMaxInstances),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(pReg->cbInstance <= _1M,
+                    ("%s: %#x\n", pReg->szName, pReg->cbInstance),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(VALID_PTR(pReg->pfnConstruct),
+                    ("%s: %p\n", pReg->szName, pReg->pfnConstruct),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(VALID_PTR(pReg->pfnRelocate) || !(pReg->fFlags & PDM_DRVREG_FLAGS_RC),
+                    ("%s: %#x\n", pReg->szName, pReg->cbInstance),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(pReg->pfnSoftReset == NULL,
+                    ("%s: %p\n", pReg->szName, pReg->pfnSoftReset),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(pReg->u32VersionEnd == PDM_DRVREG_VERSION,
+                    ("%s: %#x\n", pReg->szName, pReg->u32VersionEnd),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+
+    /*
+     * Check for duplicate and find FIFO entry at the same time.
+     */
+    PCPDMDRVREGCBINT pRegCB = (PCPDMDRVREGCBINT)pCallbacks;
+    PPDMDRV pDrvPrev = NULL;
+    PPDMDRV pDrv = pRegCB->pVM->pdm.s.pDrvs;
+    for (; pDrv; pDrvPrev = pDrv, pDrv = pDrv->pNext)
+    {
+        if (!strcmp(pDrv->pReg->szName, pReg->szName))
+        {
+            AssertMsgFailed(("Driver '%s' already exists\n", pReg->szName));
+            return VERR_PDM_DRIVER_NAME_CLASH;
+        }
+    }
+
+    /*
+     * Allocate new driver structure and insert it into the list.
+     */
+    int rc;
+    pDrv = (PPDMDRV)MMR3HeapAlloc(pRegCB->pVM, MM_TAG_PDM_DRIVER, sizeof(*pDrv));
+    if (pDrv)
+    {
+        pDrv->pNext         = NULL;
+        pDrv->cInstances    = 0;
+        pDrv->iNextInstance = 0;
+        pDrv->pReg          = pReg;
+        rc = CFGMR3QueryStringAllocDef(    pRegCB->pCfgNode, "RCSearchPath", &pDrv->pszRCSearchPath, NULL);
+        if (RT_SUCCESS(rc))
+            rc = CFGMR3QueryStringAllocDef(pRegCB->pCfgNode, "R0SearchPath", &pDrv->pszR0SearchPath, NULL);
+        if (RT_SUCCESS(rc))
+        {
+            if (pDrvPrev)
+                pDrvPrev->pNext = pDrv;
+            else
+                pRegCB->pVM->pdm.s.pDrvs = pDrv;
+            Log(("PDM: Registered driver '%s'\n", pReg->szName));
+            return VINF_SUCCESS;
+        }
+        MMR3HeapFree(pDrv);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+    return rc;
+}
+
+
+/**
+ * Lookups a driver structure by name.
+ * @internal
+ */
+PPDMDRV pdmR3DrvLookup(PVM pVM, const char *pszName)
+{
+    for (PPDMDRV pDrv = pVM->pdm.s.pDrvs; pDrv; pDrv = pDrv->pNext)
+        if (!strcmp(pDrv->pReg->szName, pszName))
+            return pDrv;
+    return NULL;
+}
+
+
+/**
+ * Transforms the driver chain as it's being instantiated.
+ *
+ * Worker for pdmR3DrvInstantiate.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pDrvAbove           The driver above, NULL if top.
+ * @param   pLun                The LUN.
+ * @param   ppNode              The AttachedDriver node, replaced if any
+ *                              morphing took place.
+ */
+static int pdmR3DrvMaybeTransformChain(PVM pVM, PPDMDRVINS pDrvAbove, PPDMLUN pLun, PCFGMNODE *ppNode)
+{
+    /*
+     * The typical state of affairs is that there are no injections.
+     */
+    PCFGMNODE pCurTrans = CFGMR3GetFirstChild(CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM/DriverTransformations"));
+    if (!pCurTrans)
+        return VINF_SUCCESS;
+
+    /*
+     * Gather the attributes used in the matching process.
+     */
+    const char *pszDevice = pLun->pDevIns
+                          ? pLun->pDevIns->Internal.s.pDevR3->pReg->szName
+                          : pLun->pUsbIns->Internal.s.pUsbDev->pReg->szName;
+    char        szLun[32];
+    RTStrPrintf(szLun, sizeof(szLun), "%u", pLun->iLun);
+    const char *pszAbove  = pDrvAbove ? pDrvAbove->Internal.s.pDrv->pReg->szName : "<top>";
+    char       *pszThisDrv;
+    int rc = CFGMR3QueryStringAlloc(*ppNode, "Driver", &pszThisDrv);
+    AssertMsgRCReturn(rc,  ("Query for string value of \"Driver\" -> %Rrc\n", rc),
+                      rc == VERR_CFGM_VALUE_NOT_FOUND ? VERR_PDM_CFG_MISSING_DRIVER_NAME : rc);
+
+    uint64_t    uInjectTransformationAbove = 0;
+    if (pDrvAbove)
+    {
+        rc = CFGMR3QueryIntegerDef(CFGMR3GetParent(*ppNode), "InjectTransformationPtr", &uInjectTransformationAbove, 0);
+        AssertLogRelRCReturn(rc, rc);
+    }
+
+
+    /*
+     * Enumerate possible driver chain transformations.
+     */
+    unsigned cTransformations = 0;
+    for (; pCurTrans != NULL; pCurTrans = CFGMR3GetNextChild(pCurTrans))
+    {
+        char szCurTransNm[256];
+        rc = CFGMR3GetName(pCurTrans, szCurTransNm, sizeof(szCurTransNm));
+        AssertLogRelRCReturn(rc, rc);
+
+        /** @cfgm{/PDM/DriverTransformations/&lt;name&gt;/Device,string,*}
+         * One or more simple wildcard patters separated by '|' for matching
+         * the devices this transformation rule applies to. */
+        char *pszMultiPat;
+        rc = CFGMR3QueryStringAllocDef(pCurTrans, "Device", &pszMultiPat, "*");
+        AssertLogRelRCReturn(rc, rc);
+        bool fMatch = RTStrSimplePatternMultiMatch(pszMultiPat, RTSTR_MAX, pszDevice, RTSTR_MAX, NULL);
+        MMR3HeapFree(pszMultiPat);
+        if (!fMatch)
+            continue;
+
+        /** @cfgm{/PDM/DriverTransformations/&lt;name&gt;/LUN,string,*}
+         * One or more simple wildcard patters separated by '|' for matching
+         * the LUNs this transformation rule applies to. */
+        rc = CFGMR3QueryStringAllocDef(pCurTrans, "LUN", &pszMultiPat, "*");
+        AssertLogRelRCReturn(rc, rc);
+        fMatch = RTStrSimplePatternMultiMatch(pszMultiPat, RTSTR_MAX, szLun, RTSTR_MAX, NULL);
+        MMR3HeapFree(pszMultiPat);
+        if (!fMatch)
+            continue;
+
+        /** @cfgm{/PDM/DriverTransformations/&lt;name&gt;/BelowDriver,string,*}
+         * One or more simple wildcard patters separated by '|' for matching the
+         * drivers the transformation should be applied below.  This means, that
+         * when the drivers matched here attached another driver below them, the
+         * transformation will be applied.  To represent the device, '&lt;top&gt;'
+         * is used. */
+        rc = CFGMR3QueryStringAllocDef(pCurTrans, "BelowDriver", &pszMultiPat, "*");
+        AssertLogRelRCReturn(rc, rc);
+        fMatch = RTStrSimplePatternMultiMatch(pszMultiPat, RTSTR_MAX, pszAbove, RTSTR_MAX, NULL);
+        MMR3HeapFree(pszMultiPat);
+        if (!fMatch)
+            continue;
+
+        /** @cfgm{/PDM/DriverTransformations/&lt;name&gt;/AboveDriver,string,*}
+         * One or more simple wildcard patters separated by '|' for matching the
+         * drivers the transformation should be applie above or at (depending on
+         * the action).  The value being matched against here is the driver that
+         * is in the process of being attached, so for mergeconfig actions this is
+         * usually what you need to match on. */
+        rc = CFGMR3QueryStringAlloc(pCurTrans, "AboveDriver", &pszMultiPat);
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+            rc = VINF_SUCCESS;
+        else
+        {
+            AssertLogRelRCReturn(rc, rc);
+            fMatch = RTStrSimplePatternMultiMatch(pszMultiPat, RTSTR_MAX, pszThisDrv, RTSTR_MAX, NULL);
+            MMR3HeapFree(pszMultiPat);
+            if (!fMatch)
+                continue;
+            if (uInjectTransformationAbove == (uintptr_t)pCurTrans)
+                continue;
+        }
+
+        /*
+         * We've got a match! Now, what are we supposed to do?
+         */
+        /** @cfgm{/PDM/DriverTransformations/&lt;name&gt;/Action,string,inject}
+         * The action that the transformation takes.  Possible values are:
+         *      - inject
+         *      - mergeconfig: This merges and the content of the 'Config' key under the
+         *        transformation into the driver's own 'Config' key, replacing any
+         *        duplicates.
+         *      - remove
+         *      - removetree
+         *      - replace
+         *      - replacetree
+         */
+        char szAction[16];
+        rc = CFGMR3QueryStringDef(pCurTrans, "Action", szAction, sizeof(szAction), "inject");
+        AssertLogRelRCReturn(rc, rc);
+        AssertLogRelMsgReturn(   !strcmp(szAction, "inject")
+                              || !strcmp(szAction, "mergeconfig")
+                              || !strcmp(szAction, "remove")
+                              || !strcmp(szAction, "removetree")
+                              || !strcmp(szAction, "replace")
+                              || !strcmp(szAction, "replacetree")
+                              ,
+                              ("Action='%s', valid values are 'inject', 'mergeconfig', 'replace', 'replacetree', 'remove', 'removetree'.\n", szAction),
+                              VERR_PDM_MISCONFIGURED_DRV_TRANSFORMATION);
+        LogRel(("PDMDriver: Applying '%s' to '%s'::[%s]...'%s': %s\n", szCurTransNm, pszDevice, szLun, pszThisDrv, szAction));
+        CFGMR3Dump(*ppNode);
+        CFGMR3Dump(pCurTrans);
+
+        /* Get the attached driver to inject. */
+        PCFGMNODE pTransAttDrv = NULL;
+        if (!strcmp(szAction, "inject") || !strcmp(szAction, "replace") || !strcmp(szAction, "replacetree"))
+        {
+            pTransAttDrv = CFGMR3GetChild(pCurTrans, "AttachedDriver");
+            AssertLogRelMsgReturn(pTransAttDrv,
+                                  ("An %s transformation requires an AttachedDriver child node!\n", szAction),
+                                  VERR_PDM_MISCONFIGURED_DRV_TRANSFORMATION);
+        }
+
+
+        /*
+         * Remove the node.
+         */
+        if (!strcmp(szAction, "remove") || !strcmp(szAction, "removetree"))
+        {
+            PCFGMNODE pBelowThis = CFGMR3GetChild(*ppNode, "AttachedDriver");
+            if (!pBelowThis || !strcmp(szAction, "removetree"))
+            {
+                CFGMR3RemoveNode(*ppNode);
+                *ppNode = NULL;
+            }
+            else
+            {
+                PCFGMNODE pBelowThisCopy;
+                rc = CFGMR3DuplicateSubTree(pBelowThis, &pBelowThisCopy);
+                AssertLogRelRCReturn(rc, rc);
+
+                rc = CFGMR3ReplaceSubTree(*ppNode, pBelowThisCopy);
+                AssertLogRelRCReturnStmt(rc, CFGMR3RemoveNode(pBelowThis), rc);
+            }
+        }
+        /*
+         * Replace the driver about to be instantiated.
+         */
+        else if (!strcmp(szAction, "replace") || !strcmp(szAction, "replacetree"))
+        {
+            PCFGMNODE pTransCopy;
+            rc = CFGMR3DuplicateSubTree(pTransAttDrv, &pTransCopy);
+            AssertLogRelRCReturn(rc, rc);
+
+            PCFGMNODE pBelowThis = CFGMR3GetChild(*ppNode, "AttachedDriver");
+            if (!pBelowThis || !strcmp(szAction, "replacetree"))
+                rc = VINF_SUCCESS;
+            else
+            {
+                PCFGMNODE pBelowThisCopy;
+                rc = CFGMR3DuplicateSubTree(pBelowThis, &pBelowThisCopy);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = CFGMR3InsertSubTree(pTransCopy, "AttachedDriver", pBelowThisCopy, NULL);
+                    AssertLogRelRC(rc);
+                    if (RT_FAILURE(rc))
+                        CFGMR3RemoveNode(pBelowThisCopy);
+                }
+            }
+            if (RT_SUCCESS(rc))
+                rc = CFGMR3ReplaceSubTree(*ppNode, pTransCopy);
+            if (RT_FAILURE(rc))
+                CFGMR3RemoveNode(pTransCopy);
+        }
+        /*
+         * Inject a driver before the driver about to be instantiated.
+         */
+        else if (!strcmp(szAction, "inject"))
+        {
+            PCFGMNODE pTransCopy;
+            rc = CFGMR3DuplicateSubTree(pTransAttDrv, &pTransCopy);
+            AssertLogRelRCReturn(rc, rc);
+
+            PCFGMNODE pThisCopy;
+            rc = CFGMR3DuplicateSubTree(*ppNode, &pThisCopy);
+            if (RT_SUCCESS(rc))
+            {
+                rc = CFGMR3InsertSubTree(pTransCopy, "AttachedDriver", pThisCopy, NULL);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = CFGMR3InsertInteger(pTransCopy, "InjectTransformationPtr", (uintptr_t)pCurTrans);
+                    AssertLogRelRC(rc);
+                    rc = CFGMR3InsertString(pTransCopy, "InjectTransformationNm", szCurTransNm);
+                    AssertLogRelRC(rc);
+                    if (RT_SUCCESS(rc))
+                        rc = CFGMR3ReplaceSubTree(*ppNode, pTransCopy);
+                }
+                else
+                {
+                    AssertLogRelRC(rc);
+                    CFGMR3RemoveNode(pThisCopy);
+                }
+            }
+            if (RT_FAILURE(rc))
+                CFGMR3RemoveNode(pTransCopy);
+        }
+        /*
+         * Merge the Config node of the transformation with the one of the
+         * current driver.
+         */
+        else if (!strcmp(szAction, "mergeconfig"))
+        {
+            PCFGMNODE pTransConfig = CFGMR3GetChild(pCurTrans, "Config");
+            AssertLogRelReturn(pTransConfig, VERR_PDM_MISCONFIGURED_DRV_TRANSFORMATION);
+
+            PCFGMNODE pDrvConfig = CFGMR3GetChild(*ppNode, "Config");
+            if (*ppNode)
+                CFGMR3InsertNode(*ppNode, "Config", &pDrvConfig);
+            AssertLogRelReturn(pDrvConfig, VERR_PDM_CANNOT_TRANSFORM_REMOVED_DRIVER);
+
+            rc = CFGMR3CopyTree(pDrvConfig, pTransConfig, CFGM_COPY_FLAGS_REPLACE_VALUES | CFGM_COPY_FLAGS_MERGE_KEYS);
+            AssertLogRelRCReturn(rc, rc);
+        }
+        else
+            AssertFailed();
+
+        cTransformations++;
+        if (*ppNode)
+            CFGMR3Dump(*ppNode);
+        else
+            LogRel(("PDMDriver: The transformation removed the driver.\n"));
+    }
+
+    /*
+     * Note what happened in the release log.
+     */
+    if (cTransformations > 0)
+        LogRel(("PDMDriver: Transformations done. Applied %u driver transformations.\n", cTransformations));
+
+    return rc;
+}
+
+
+/**
+ * Instantiate a driver.
+ *
+ * @returns VBox status code, including informational statuses.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pNode               The CFGM node for the driver.
+ * @param   pBaseInterface      The base interface.
+ * @param   pDrvAbove           The driver above it.  NULL if it's the top-most
+ *                              driver.
+ * @param   pLun                The LUN the driver is being attached to.  NULL
+ *                              if we're instantiating a driver chain before
+ *                              attaching it - untested.
+ * @param   ppBaseInterface     Where to return the pointer to the base
+ *                              interface of the newly created driver.
+ *
+ * @remarks Recursive calls to this function is normal as the drivers will
+ *          attach to anything below them during the pfnContruct call.
+ *
+ * @todo    Need to extend this interface a bit so that the driver
+ *          transformation feature can attach drivers to unconfigured LUNs and
+ *          at the end of chains.
+ */
+int pdmR3DrvInstantiate(PVM pVM, PCFGMNODE pNode, PPDMIBASE pBaseInterface, PPDMDRVINS pDrvAbove,
+                        PPDMLUN pLun, PPDMIBASE *ppBaseInterface)
+{
+    Assert(!pDrvAbove || !pDrvAbove->Internal.s.pDown);
+    Assert(!pDrvAbove || !pDrvAbove->pDownBase);
+
+    Assert(pBaseInterface->pfnQueryInterface(pBaseInterface, PDMIBASE_IID) == pBaseInterface);
+
+    /*
+     * Do driver chain injections
+     */
+    int rc = pdmR3DrvMaybeTransformChain(pVM, pDrvAbove, pLun, &pNode);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (!pNode)
+        return VERR_PDM_NO_ATTACHED_DRIVER;
+
+    /*
+     * Find the driver.
+     */
+    char *pszName;
+    rc = CFGMR3QueryStringAlloc(pNode, "Driver", &pszName);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMDRV pDrv = pdmR3DrvLookup(pVM, pszName);
+        if (    pDrv
+            &&  pDrv->cInstances < pDrv->pReg->cMaxInstances)
+        {
+            /* config node */
+            PCFGMNODE pConfigNode = CFGMR3GetChild(pNode, "Config");
+            if (!pConfigNode)
+                rc = CFGMR3InsertNode(pNode, "Config", &pConfigNode);
+            if (RT_SUCCESS(rc))
+            {
+                CFGMR3SetRestrictedRoot(pConfigNode);
+
+                /*
+                 * Allocate the driver instance.
+                 */
+                size_t cb = RT_UOFFSETOF_DYN(PDMDRVINS, achInstanceData[pDrv->pReg->cbInstance]);
+                cb = RT_ALIGN_Z(cb, 16);
+                bool const fHyperHeap = !!(pDrv->pReg->fFlags & (PDM_DRVREG_FLAGS_R0 | PDM_DRVREG_FLAGS_RC));
+                PPDMDRVINS pNew;
+                if (fHyperHeap)
+                    rc = MMHyperAlloc(pVM, cb, 64, MM_TAG_PDM_DRIVER, (void **)&pNew);
+                else
+                    rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_DRIVER, cb, (void **)&pNew);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Initialize the instance structure (declaration order).
+                     */
+                    pNew->u32Version                = PDM_DRVINS_VERSION;
+                    pNew->iInstance                 = pDrv->iNextInstance;
+                    pNew->Internal.s.pUp            = pDrvAbove ? pDrvAbove : NULL;
+                    //pNew->Internal.s.pDown          = NULL;
+                    pNew->Internal.s.pLun           = pLun;
+                    pNew->Internal.s.pDrv           = pDrv;
+                    pNew->Internal.s.pVMR3          = pVM;
+                    pNew->Internal.s.pVMR0          = pDrv->pReg->fFlags & PDM_DRVREG_FLAGS_R0 ? pVM->pVMR0ForCall : NIL_RTR0PTR;
+                    pNew->Internal.s.pVMRC          = pDrv->pReg->fFlags & PDM_DRVREG_FLAGS_RC ? pVM->pVMRC : NIL_RTRCPTR;
+                    //pNew->Internal.s.fDetaching     = false;
+                    pNew->Internal.s.fVMSuspended   = true; /** @todo should be 'false', if driver is attached at runtime. */
+                    //pNew->Internal.s.fVMReset       = false;
+                    pNew->Internal.s.fHyperHeap     = fHyperHeap;
+                    //pNew->Internal.s.pfnAsyncNotify = NULL;
+                    pNew->Internal.s.pCfgHandle     = pNode;
+                    pNew->pReg                      = pDrv->pReg;
+                    pNew->pCfg                      = pConfigNode;
+                    pNew->pUpBase                   = pBaseInterface;
+                    Assert(!pDrvAbove || pBaseInterface == &pDrvAbove->IBase);
+                    //pNew->pDownBase                 = NULL;
+                    //pNew->IBase.pfnQueryInterface   = NULL;
+                    //pNew->fTracing                  = 0;
+                    pNew->idTracing                 = ++pVM->pdm.s.idTracingOther;
+                    pNew->pHlpR3                    = &g_pdmR3DrvHlp;
+                    pNew->pvInstanceDataR3          = &pNew->achInstanceData[0];
+                    if (pDrv->pReg->fFlags & PDM_DRVREG_FLAGS_R0)
+                    {
+                        pNew->pvInstanceDataR0      = MMHyperR3ToR0(pVM, &pNew->achInstanceData[0]);
+                        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "g_pdmR0DrvHlp", &pNew->pHlpR0);
+                        AssertReleaseRCReturn(rc, rc);
+                    }
+                    if (   (pDrv->pReg->fFlags & PDM_DRVREG_FLAGS_RC)
+                        && VM_IS_RAW_MODE_ENABLED(pVM))
+                    {
+                        pNew->pvInstanceDataR0      = MMHyperR3ToRC(pVM, &pNew->achInstanceData[0]);
+                        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_pdmRCDrvHlp", &pNew->pHlpRC);
+                        AssertReleaseRCReturn(rc, rc);
+                    }
+
+                    pDrv->iNextInstance++;
+                    pDrv->cInstances++;
+
+                    /*
+                     * Link with it with the driver above / LUN.
+                     */
+                    if (pDrvAbove)
+                    {
+                        pDrvAbove->pDownBase        = &pNew->IBase;
+                        pDrvAbove->Internal.s.pDown = pNew;
+                    }
+                    else if (pLun)
+                        pLun->pTop                  = pNew;
+                    if (pLun)
+                        pLun->pBottom               = pNew;
+
+                    /*
+                     * Invoke the constructor.
+                     */
+                    rc = pDrv->pReg->pfnConstruct(pNew, pNew->pCfg, 0 /*fFlags*/);
+                    if (RT_SUCCESS(rc))
+                    {
+                        AssertPtr(pNew->IBase.pfnQueryInterface);
+                        Assert(pNew->IBase.pfnQueryInterface(&pNew->IBase, PDMIBASE_IID) == &pNew->IBase);
+
+                        /* Success! */
+                        *ppBaseInterface = &pNew->IBase;
+                        if (pLun)
+                            Log(("PDM: Attached driver %p:'%s'/%d to LUN#%d on device '%s'/%d, pDrvAbove=%p:'%s'/%d\n",
+                                 pNew, pDrv->pReg->szName, pNew->iInstance,
+                                 pLun->iLun,
+                                 pLun->pDevIns ? pLun->pDevIns->pReg->szName : pLun->pUsbIns->pReg->szName,
+                                 pLun->pDevIns ? pLun->pDevIns->iInstance    : pLun->pUsbIns->iInstance,
+                                 pDrvAbove, pDrvAbove ? pDrvAbove->pReg->szName : "", pDrvAbove ? pDrvAbove->iInstance : UINT32_MAX));
+                        else
+                            Log(("PDM: Attached driver %p:'%s'/%d, pDrvAbove=%p:'%s'/%d\n",
+                                 pNew, pDrv->pReg->szName, pNew->iInstance,
+                                 pDrvAbove, pDrvAbove ? pDrvAbove->pReg->szName : "", pDrvAbove ? pDrvAbove->iInstance : UINT32_MAX));
+                    }
+                    else
+                    {
+                        pdmR3DrvDestroyChain(pNew, PDM_TACH_FLAGS_NO_CALLBACKS);
+                        if (rc == VERR_VERSION_MISMATCH)
+                            rc = VERR_PDM_DRIVER_VERSION_MISMATCH;
+                    }
+                }
+                else
+                    AssertMsgFailed(("Failed to allocate %d bytes for instantiating driver '%s'! rc=%Rrc\n", cb, pszName, rc));
+            }
+            else
+                AssertMsgFailed(("Failed to create Config node! rc=%Rrc\n", rc));
+        }
+        else if (pDrv)
+        {
+            AssertMsgFailed(("Too many instances of driver '%s', max is %u\n", pszName, pDrv->pReg->cMaxInstances));
+            rc = VERR_PDM_TOO_MANY_DRIVER_INSTANCES;
+        }
+        else
+        {
+            AssertMsgFailed(("Driver '%s' wasn't found!\n", pszName));
+            rc = VERR_PDM_DRIVER_NOT_FOUND;
+        }
+        MMR3HeapFree(pszName);
+    }
+    else
+    {
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+            rc = VERR_PDM_CFG_MISSING_DRIVER_NAME;
+        else
+            AssertMsgFailed(("Query for string value of \"Driver\" -> %Rrc\n", rc));
+    }
+    return rc;
+}
+
+
+/**
+ * Detaches a driver from whatever it's attached to.
+ * This will of course lead to the destruction of the driver and all drivers below it in the chain.
+ *
+ * @returns VINF_SUCCESS
+ * @param   pDrvIns     The driver instance to detach.
+ * @param   fFlags      Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
+ */
+int pdmR3DrvDetach(PPDMDRVINS pDrvIns, uint32_t fFlags)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvDetach: pDrvIns=%p '%s'/%d\n", pDrvIns, pDrvIns->pReg->szName, pDrvIns->iInstance));
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    /*
+     * Check that we're not doing this recursively, that could have unwanted sideeffects!
+     */
+    if (pDrvIns->Internal.s.fDetaching)
+    {
+        AssertMsgFailed(("Recursive detach! '%s'/%d\n", pDrvIns->pReg->szName, pDrvIns->iInstance));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Check that we actually can detach this instance.
+     * The requirement is that the driver/device above has a detach method.
+     */
+    if (  pDrvIns->Internal.s.pUp
+        ? !pDrvIns->Internal.s.pUp->pReg->pfnDetach
+        :   pDrvIns->Internal.s.pLun->pDevIns
+          ? !pDrvIns->Internal.s.pLun->pDevIns->pReg->pfnDetach
+          : !pDrvIns->Internal.s.pLun->pUsbIns->pReg->pfnDriverDetach
+       )
+    {
+        AssertMsgFailed(("Cannot detach driver instance because the driver/device above doesn't support it!\n"));
+        return VERR_PDM_DRIVER_DETACH_NOT_POSSIBLE;
+    }
+
+    /*
+     * Join paths with pdmR3DrvDestroyChain.
+     */
+    pdmR3DrvDestroyChain(pDrvIns, fFlags);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroys a driver chain starting with the specified driver.
+ *
+ * This is used when unplugging a device at run time.
+ *
+ * @param   pDrvIns     Pointer to the driver instance to start with.
+ * @param   fFlags      PDM_TACH_FLAGS_NOT_HOT_PLUG, PDM_TACH_FLAGS_NO_CALLBACKS
+ *                      or 0.
+ */
+void pdmR3DrvDestroyChain(PPDMDRVINS pDrvIns, uint32_t fFlags)
+{
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Detach the bottommost driver until we've detached pDrvIns.
+     */
+    pDrvIns->Internal.s.fDetaching = true;
+    PPDMDRVINS pCur;
+    do
+    {
+        /* find the driver to detach. */
+        pCur = pDrvIns;
+        while (pCur->Internal.s.pDown)
+            pCur = pCur->Internal.s.pDown;
+        LogFlow(("pdmR3DrvDestroyChain: pCur=%p '%s'/%d\n", pCur, pCur->pReg->szName, pCur->iInstance));
+
+        /*
+         * Unlink it and notify parent.
+         */
+        pCur->Internal.s.fDetaching = true;
+
+        PPDMLUN pLun = pCur->Internal.s.pLun;
+        Assert(pLun->pBottom == pCur);
+        pLun->pBottom = pCur->Internal.s.pUp;
+
+        if (pCur->Internal.s.pUp)
+        {
+            /* driver parent */
+            PPDMDRVINS pParent = pCur->Internal.s.pUp;
+            pCur->Internal.s.pUp = NULL;
+            pParent->Internal.s.pDown = NULL;
+
+            if (!(fFlags & PDM_TACH_FLAGS_NO_CALLBACKS) && pParent->pReg->pfnDetach)
+                pParent->pReg->pfnDetach(pParent, fFlags);
+
+            pParent->pDownBase = NULL;
+        }
+        else
+        {
+            /* device parent */
+            Assert(pLun->pTop == pCur);
+            pLun->pTop = NULL;
+            if (!(fFlags & PDM_TACH_FLAGS_NO_CALLBACKS))
+            {
+                if (pLun->pDevIns)
+                {
+                    if (pLun->pDevIns->pReg->pfnDetach)
+                    {
+                        PDMCritSectEnter(pLun->pDevIns->pCritSectRoR3, VERR_IGNORED);
+                        pLun->pDevIns->pReg->pfnDetach(pLun->pDevIns, pLun->iLun, fFlags);
+                        PDMCritSectLeave(pLun->pDevIns->pCritSectRoR3);
+                    }
+                }
+                else
+                {
+                    if (pLun->pUsbIns->pReg->pfnDriverDetach)
+                    {
+                        /** @todo USB device locking? */
+                        pLun->pUsbIns->pReg->pfnDriverDetach(pLun->pUsbIns, pLun->iLun, fFlags);
+                    }
+                }
+            }
+        }
+
+        /*
+         * Call destructor.
+         */
+        pCur->pUpBase = NULL;
+        if (pCur->pReg->pfnDestruct)
+            pCur->pReg->pfnDestruct(pCur);
+        pCur->Internal.s.pDrv->cInstances--;
+
+        /*
+         * Free all resources allocated by the driver.
+         */
+        /* Queues. */
+        int rc = PDMR3QueueDestroyDriver(pVM, pCur);
+        AssertRC(rc);
+
+        /* Timers. */
+        rc = TMR3TimerDestroyDriver(pVM, pCur);
+        AssertRC(rc);
+
+        /* SSM data units. */
+        rc = SSMR3DeregisterDriver(pVM, pCur, NULL, 0);
+        AssertRC(rc);
+
+        /* PDM threads. */
+        rc = pdmR3ThreadDestroyDriver(pVM, pCur);
+        AssertRC(rc);
+
+        /* Info handlers. */
+        rc = DBGFR3InfoDeregisterDriver(pVM, pCur, NULL);
+        AssertRC(rc);
+
+        /* PDM critsects. */
+        rc = pdmR3CritSectBothDeleteDriver(pVM, pCur);
+        AssertRC(rc);
+
+        /* Block caches. */
+        PDMR3BlkCacheReleaseDriver(pVM, pCur);
+
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+        /* Completion templates.*/
+        pdmR3AsyncCompletionTemplateDestroyDriver(pVM, pCur);
+#endif
+
+        /* Finally, the driver it self. */
+        bool fHyperHeap = pCur->Internal.s.fHyperHeap;
+        ASMMemFill32(pCur, RT_UOFFSETOF_DYN(PDMDRVINS, achInstanceData[pCur->pReg->cbInstance]), 0xdeadd0d0);
+        if (fHyperHeap)
+            MMHyperFree(pVM, pCur);
+        else
+            MMR3HeapFree(pCur);
+
+    } while (pCur != pDrvIns);
+}
+
+
+
+
+/** @name Driver Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnAttach} */
+static DECLCALLBACK(int) pdmR3DrvHlp_Attach(PPDMDRVINS pDrvIns, uint32_t fFlags, PPDMIBASE *ppBaseInterface)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DrvHlp_Attach: caller='%s'/%d: fFlags=%#x\n", pDrvIns->pReg->szName, pDrvIns->iInstance, fFlags));
+    Assert(!(fFlags & ~(PDM_TACH_FLAGS_NOT_HOT_PLUG)));
+    RT_NOREF_PV(fFlags);
+
+    /*
+     * Check that there isn't anything attached already.
+     */
+    int rc;
+    if (!pDrvIns->Internal.s.pDown)
+    {
+        Assert(pDrvIns->Internal.s.pLun->pBottom == pDrvIns);
+
+        /*
+         * Get the attached driver configuration.
+         */
+        PCFGMNODE pNode = CFGMR3GetChild(pDrvIns->Internal.s.pCfgHandle, "AttachedDriver");
+        if (pNode)
+            rc = pdmR3DrvInstantiate(pVM, pNode, &pDrvIns->IBase, pDrvIns, pDrvIns->Internal.s.pLun, ppBaseInterface);
+        else
+            rc = VERR_PDM_NO_ATTACHED_DRIVER;
+    }
+    else
+    {
+        AssertMsgFailed(("Already got a driver attached. The driver should keep track of such things!\n"));
+        rc = VERR_PDM_DRIVER_ALREADY_ATTACHED;
+    }
+
+    LogFlow(("pdmR3DrvHlp_Attach: caller='%s'/%d: return %Rrc\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnDetach} */
+static DECLCALLBACK(int) pdmR3DrvHlp_Detach(PPDMDRVINS pDrvIns, uint32_t fFlags)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_Detach: caller='%s'/%d: fFlags=%#x\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, fFlags));
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    /*
+     * Anything attached?
+     */
+    int rc;
+    if (pDrvIns->Internal.s.pDown)
+        rc = pdmR3DrvDetach(pDrvIns->Internal.s.pDown, fFlags);
+    else
+    {
+        AssertMsgFailed(("Nothing attached!\n"));
+        rc = VERR_PDM_NO_DRIVER_ATTACHED;
+    }
+
+    LogFlow(("pdmR3DrvHlp_Detach: caller='%s'/%d: returns %Rrc\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnDetachSelf} */
+static DECLCALLBACK(int) pdmR3DrvHlp_DetachSelf(PPDMDRVINS pDrvIns, uint32_t fFlags)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_DetachSelf: caller='%s'/%d: fFlags=%#x\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, fFlags));
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    int rc = pdmR3DrvDetach(pDrvIns, fFlags);
+
+    LogFlow(("pdmR3DrvHlp_Detach: returns %Rrc\n", rc)); /* pDrvIns is freed by now. */
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnMountPrepare} */
+static DECLCALLBACK(int) pdmR3DrvHlp_MountPrepare(PPDMDRVINS pDrvIns, const char *pszFilename, const char *pszCoreDriver)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_MountPrepare: caller='%s'/%d: pszFilename=%p:{%s} pszCoreDriver=%p:{%s}\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pszFilename, pszFilename, pszCoreDriver, pszCoreDriver));
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    /*
+     * Do the caller have anything attached below itself?
+     */
+    if (pDrvIns->Internal.s.pDown)
+    {
+        AssertMsgFailed(("Cannot prepare a mount when something's attached to you!\n"));
+        return VERR_PDM_DRIVER_ALREADY_ATTACHED;
+    }
+
+    /*
+     * We're asked to prepare, so we'll start off by nuking the
+     * attached configuration tree.
+     */
+    PCFGMNODE   pNode = CFGMR3GetChild(pDrvIns->Internal.s.pCfgHandle, "AttachedDriver");
+    if (pNode)
+        CFGMR3RemoveNode(pNode);
+
+    /*
+     * If there is no core driver, we'll have to probe for it.
+     */
+    if (!pszCoreDriver)
+    {
+        /** @todo implement image probing. */
+        AssertReleaseMsgFailed(("Not implemented!\n"));
+        return VERR_NOT_IMPLEMENTED;
+    }
+
+    /*
+     * Construct the basic attached driver configuration.
+     */
+    int rc = CFGMR3InsertNode(pDrvIns->Internal.s.pCfgHandle, "AttachedDriver", &pNode);
+    if (RT_SUCCESS(rc))
+    {
+        rc = CFGMR3InsertString(pNode, "Driver", pszCoreDriver);
+        if (RT_SUCCESS(rc))
+        {
+            PCFGMNODE pCfg;
+            rc = CFGMR3InsertNode(pNode, "Config", &pCfg);
+            if (RT_SUCCESS(rc))
+            {
+                rc = CFGMR3InsertString(pCfg, "Path", pszFilename);
+                if (RT_SUCCESS(rc))
+                {
+                    LogFlow(("pdmR3DrvHlp_MountPrepare: caller='%s'/%d: returns %Rrc (Driver=%s)\n",
+                             pDrvIns->pReg->szName, pDrvIns->iInstance, rc, pszCoreDriver));
+                    return rc;
+                }
+                else
+                    AssertMsgFailed(("Path string insert failed, rc=%Rrc\n", rc));
+            }
+            else
+                AssertMsgFailed(("Config node failed, rc=%Rrc\n", rc));
+        }
+        else
+            AssertMsgFailed(("Driver string insert failed, rc=%Rrc\n", rc));
+        CFGMR3RemoveNode(pNode);
+    }
+    else
+        AssertMsgFailed(("AttachedDriver node insert failed, rc=%Rrc\n", rc));
+
+    LogFlow(("pdmR3DrvHlp_MountPrepare: caller='%s'/%d: returns %Rrc\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnAssertEMT} */
+static DECLCALLBACK(bool) pdmR3DrvHlp_AssertEMT(PPDMDRVINS pDrvIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    if (VM_IS_EMT(pDrvIns->Internal.s.pVMR3))
+        return true;
+
+    char szMsg[100];
+    RTStrPrintf(szMsg, sizeof(szMsg), "AssertEMT '%s'/%d\n", pDrvIns->pReg->szName, pDrvIns->iInstance);
+    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
+    AssertBreakpoint();
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    return false;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnAssertOther} */
+static DECLCALLBACK(bool) pdmR3DrvHlp_AssertOther(PPDMDRVINS pDrvIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    if (!VM_IS_EMT(pDrvIns->Internal.s.pVMR3))
+        return true;
+
+    char szMsg[100];
+    RTStrPrintf(szMsg, sizeof(szMsg), "AssertOther '%s'/%d\n", pDrvIns->pReg->szName, pDrvIns->iInstance);
+    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
+    AssertBreakpoint();
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    return false;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnVMSetError} */
+static DECLCALLBACK(int) pdmR3DrvHlp_VMSetError(PPDMDRVINS pDrvIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    va_list args;
+    va_start(args, pszFormat);
+    int rc2 = VMSetErrorV(pDrvIns->Internal.s.pVMR3, rc, RT_SRC_POS_ARGS, pszFormat, args); Assert(rc2 == rc); NOREF(rc2);
+    va_end(args);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnVMSetErrorV} */
+static DECLCALLBACK(int) pdmR3DrvHlp_VMSetErrorV(PPDMDRVINS pDrvIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    int rc2 = VMSetErrorV(pDrvIns->Internal.s.pVMR3, rc, RT_SRC_POS_ARGS, pszFormat, va); Assert(rc2 == rc); NOREF(rc2);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnVMSetRuntimeError} */
+static DECLCALLBACK(int) pdmR3DrvHlp_VMSetRuntimeError(PPDMDRVINS pDrvIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    va_list args;
+    va_start(args, pszFormat);
+    int rc = VMSetRuntimeErrorV(pDrvIns->Internal.s.pVMR3, fFlags, pszErrorId, pszFormat, args);
+    va_end(args);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnVMSetRuntimeErrorV} */
+static DECLCALLBACK(int) pdmR3DrvHlp_VMSetRuntimeErrorV(PPDMDRVINS pDrvIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    int rc = VMSetRuntimeErrorV(pDrvIns->Internal.s.pVMR3, fFlags, pszErrorId, pszFormat, va);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnVMState} */
+static DECLCALLBACK(VMSTATE) pdmR3DrvHlp_VMState(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+
+    VMSTATE enmVMState = VMR3GetState(pDrvIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DrvHlp_VMState: caller='%s'/%d: returns %d (%s)\n", pDrvIns->pReg->szName, pDrvIns->iInstance,
+             enmVMState, VMR3GetStateName(enmVMState)));
+    return enmVMState;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnVMTeleportedAndNotFullyResumedYet} */
+static DECLCALLBACK(bool) pdmR3DrvHlp_VMTeleportedAndNotFullyResumedYet(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+
+    bool fRc = VMR3TeleportedAndNotFullyResumedYet(pDrvIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DrvHlp_VMState: caller='%s'/%d: returns %RTbool)\n", pDrvIns->pReg->szName, pDrvIns->iInstance,
+             fRc));
+    return fRc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnGetSupDrvSession} */
+static DECLCALLBACK(PSUPDRVSESSION) pdmR3DrvHlp_GetSupDrvSession(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+
+    PSUPDRVSESSION pSession = pDrvIns->Internal.s.pVMR3->pSession;
+    LogFlow(("pdmR3DrvHlp_GetSupDrvSession: caller='%s'/%d: returns %p)\n", pDrvIns->pReg->szName, pDrvIns->iInstance,
+             pSession));
+    return pSession;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnQueueCreate} */
+static DECLCALLBACK(int) pdmR3DrvHlp_QueueCreate(PPDMDRVINS pDrvIns, uint32_t cbItem, uint32_t cItems, uint32_t cMilliesInterval,
+                                                 PFNPDMQUEUEDRV pfnCallback, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_PDMQueueCreate: caller='%s'/%d: cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p pszName=%p:{%s} ppQueue=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, cbItem, cItems, cMilliesInterval, pfnCallback, pszName, pszName, ppQueue));
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    if (pDrvIns->iInstance > 0)
+    {
+        pszName = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DRIVER_DESC, "%s_%u", pszName, pDrvIns->iInstance);
+        AssertLogRelReturn(pszName, VERR_NO_MEMORY);
+    }
+
+    int rc = PDMR3QueueCreateDriver(pVM, pDrvIns, cbItem, cItems, cMilliesInterval, pfnCallback, pszName, ppQueue);
+
+    LogFlow(("pdmR3DrvHlp_PDMQueueCreate: caller='%s'/%d: returns %Rrc *ppQueue=%p\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc, *ppQueue));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnTMGetVirtualFreq} */
+static DECLCALLBACK(uint64_t) pdmR3DrvHlp_TMGetVirtualFreq(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+
+    return TMVirtualGetFreq(pDrvIns->Internal.s.pVMR3);
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnTMGetVirtualTime} */
+static DECLCALLBACK(uint64_t) pdmR3DrvHlp_TMGetVirtualTime(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+
+    return TMVirtualGet(pDrvIns->Internal.s.pVMR3);
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnTMTimerCreate} */
+static DECLCALLBACK(int) pdmR3DrvHlp_TMTimerCreate(PPDMDRVINS pDrvIns, TMCLOCK enmClock, PFNTMTIMERDRV pfnCallback, void *pvUser, uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_TMTimerCreate: caller='%s'/%d: enmClock=%d pfnCallback=%p pvUser=%p fFlags=%#x pszDesc=%p:{%s} ppTimer=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, enmClock, pfnCallback, pvUser, fFlags, pszDesc, pszDesc, ppTimer));
+
+    int rc = TMR3TimerCreateDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, enmClock, pfnCallback, pvUser, fFlags, pszDesc, ppTimer);
+
+    LogFlow(("pdmR3DrvHlp_TMTimerCreate: caller='%s'/%d: returns %Rrc *ppTimer=%p\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc, *ppTimer));
+    return rc;
+}
+
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnSSMRegister} */
+static DECLCALLBACK(int) pdmR3DrvHlp_SSMRegister(PPDMDRVINS pDrvIns, uint32_t uVersion, size_t cbGuess,
+                                                 PFNSSMDRVLIVEPREP pfnLivePrep, PFNSSMDRVLIVEEXEC pfnLiveExec, PFNSSMDRVLIVEVOTE pfnLiveVote,
+                                                 PFNSSMDRVSAVEPREP pfnSavePrep, PFNSSMDRVSAVEEXEC pfnSaveExec, PFNSSMDRVSAVEDONE pfnSaveDone,
+                                                 PFNSSMDRVLOADPREP pfnLoadPrep, PFNSSMDRVLOADEXEC pfnLoadExec, PFNSSMDRVLOADDONE pfnLoadDone)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_SSMRegister: caller='%s'/%d: uVersion=%#x cbGuess=%#x \n"
+             "    pfnLivePrep=%p pfnLiveExec=%p pfnLiveVote=%p  pfnSavePrep=%p pfnSaveExec=%p pfnSaveDone=%p pszLoadPrep=%p pfnLoadExec=%p pfnLoaddone=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, uVersion, cbGuess,
+             pfnLivePrep, pfnLiveExec, pfnLiveVote,
+             pfnSavePrep, pfnSaveExec, pfnSaveDone, pfnLoadPrep, pfnLoadExec, pfnLoadDone));
+
+    int rc = SSMR3RegisterDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, pDrvIns->pReg->szName, pDrvIns->iInstance,
+                                 uVersion, cbGuess,
+                                 pfnLivePrep, pfnLiveExec, pfnLiveVote,
+                                 pfnSavePrep, pfnSaveExec, pfnSaveDone,
+                                 pfnLoadPrep, pfnLoadExec, pfnLoadDone);
+
+    LogFlow(("pdmR3DrvHlp_SSMRegister: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnSSMDeregister} */
+static DECLCALLBACK(int) pdmR3DrvHlp_SSMDeregister(PPDMDRVINS pDrvIns, const char *pszName, uint32_t uInstance)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_SSMDeregister: caller='%s'/%d: pszName=%p:{%s} uInstance=%#x\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pszName, pszName, uInstance));
+
+    int rc = SSMR3DeregisterDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, pszName, uInstance);
+
+    LogFlow(("pdmR3DrvHlp_SSMDeregister: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnDBGFInfoRegister} */
+static DECLCALLBACK(int) pdmR3DrvHlp_DBGFInfoRegister(PPDMDRVINS pDrvIns, const char *pszName, const char *pszDesc, PFNDBGFHANDLERDRV pfnHandler)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_DBGFInfoRegister: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pszName, pszName, pszDesc, pszDesc, pfnHandler));
+
+    int rc = DBGFR3InfoRegisterDriver(pDrvIns->Internal.s.pVMR3, pszName, pszDesc, pfnHandler, pDrvIns);
+
+    LogFlow(("pdmR3DrvHlp_DBGFInfoRegister: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnDBGFInfoRegisterArgv} */
+static DECLCALLBACK(int) pdmR3DrvHlp_DBGFInfoRegisterArgv(PPDMDRVINS pDrvIns, const char *pszName, const char *pszDesc, PFNDBGFINFOARGVDRV pfnHandler)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_DBGFInfoRegisterArgv: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pszName, pszName, pszDesc, pszDesc, pfnHandler));
+
+    int rc = DBGFR3InfoRegisterDriverArgv(pDrvIns->Internal.s.pVMR3, pszName, pszDesc, pfnHandler, pDrvIns);
+
+    LogFlow(("pdmR3DrvHlp_DBGFInfoRegisterArgv: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnDBGFInfoDeregister} */
+static DECLCALLBACK(int) pdmR3DrvHlp_DBGFInfoDeregister(PPDMDRVINS pDrvIns, const char *pszName)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_DBGFInfoDeregister: caller='%s'/%d: pszName=%p:{%s}\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pszName, pszName));
+
+    int rc = DBGFR3InfoDeregisterDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, pszName);
+
+    LogFlow(("pdmR3DrvHlp_DBGFInfoDeregister: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnSTAMRegister} */
+static DECLCALLBACK(void) pdmR3DrvHlp_STAMRegister(PPDMDRVINS pDrvIns, void *pvSample, STAMTYPE enmType, const char *pszName,
+                                                   STAMUNIT enmUnit, const char *pszDesc)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    STAM_REG(pDrvIns->Internal.s.pVMR3, pvSample, enmType, pszName, enmUnit, pszDesc);
+    RT_NOREF6(pDrvIns, pvSample, enmType, pszName, enmUnit, pszDesc);
+    /** @todo track the samples so they can be dumped & deregistered when the driver instance is destroyed.
+     * For now we just have to be careful not to use this call for drivers which can be unloaded. */
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnSTAMRegisterF} */
+static DECLCALLBACK(void) pdmR3DrvHlp_STAMRegisterF(PPDMDRVINS pDrvIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, ...)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    va_list args;
+    va_start(args, pszName);
+    int rc = STAMR3RegisterV(pDrvIns->Internal.s.pVMR3, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    va_end(args);
+    AssertRC(rc);
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnSTAMRegisterV} */
+static DECLCALLBACK(void) pdmR3DrvHlp_STAMRegisterV(PPDMDRVINS pDrvIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, va_list args)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    int rc = STAMR3RegisterV(pDrvIns->Internal.s.pVMR3, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    AssertRC(rc);
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnSTAMDeregister} */
+static DECLCALLBACK(int) pdmR3DrvHlp_STAMDeregister(PPDMDRVINS pDrvIns, void *pvSample)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    return STAMR3DeregisterByAddr(pDrvIns->Internal.s.pVMR3->pUVM, pvSample);
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnSUPCallVMMR0Ex} */
+static DECLCALLBACK(int) pdmR3DrvHlp_SUPCallVMMR0Ex(PPDMDRVINS pDrvIns, unsigned uOperation, void *pvArg, unsigned cbArg)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_SSMCallVMMR0Ex: caller='%s'/%d: uOperation=%u pvArg=%p cbArg=%d\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, uOperation, pvArg, cbArg));
+    RT_NOREF_PV(cbArg);
+
+    int rc;
+    if (    uOperation >= VMMR0_DO_SRV_START
+        &&  uOperation <  VMMR0_DO_SRV_END)
+        rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pDrvIns->Internal.s.pVMR3), NIL_VMCPUID, uOperation, 0, (PSUPVMMR0REQHDR)pvArg);
+    else
+    {
+        AssertMsgFailed(("Invalid uOperation=%u\n", uOperation));
+        rc = VERR_INVALID_PARAMETER;
+    }
+
+    LogFlow(("pdmR3DrvHlp_SUPCallVMMR0Ex: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnUSBRegisterHub} */
+static DECLCALLBACK(int) pdmR3DrvHlp_USBRegisterHub(PPDMDRVINS pDrvIns, uint32_t fVersions, uint32_t cPorts, PCPDMUSBHUBREG pUsbHubReg, PPCPDMUSBHUBHLP ppUsbHubHlp)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_USBRegisterHub: caller='%s'/%d: fVersions=%#x cPorts=%#x pUsbHubReg=%p ppUsbHubHlp=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, fVersions, cPorts, pUsbHubReg, ppUsbHubHlp));
+
+#ifdef VBOX_WITH_USB
+    int rc = pdmR3UsbRegisterHub(pDrvIns->Internal.s.pVMR3, pDrvIns, fVersions, cPorts, pUsbHubReg, ppUsbHubHlp);
+#else
+    int rc = VERR_NOT_SUPPORTED;
+#endif
+
+    LogFlow(("pdmR3DrvHlp_USBRegisterHub: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnSetAsyncNotification} */
+static DECLCALLBACK(int) pdmR3DrvHlp_SetAsyncNotification(PPDMDRVINS pDrvIns, PFNPDMDRVASYNCNOTIFY pfnAsyncNotify)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT0(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_SetAsyncNotification: caller='%s'/%d: pfnAsyncNotify=%p\n", pDrvIns->pReg->szName, pDrvIns->iInstance, pfnAsyncNotify));
+
+    int rc = VINF_SUCCESS;
+    AssertStmt(pfnAsyncNotify, rc = VERR_INVALID_PARAMETER);
+    AssertStmt(!pDrvIns->Internal.s.pfnAsyncNotify, rc = VERR_WRONG_ORDER);
+    AssertStmt(pDrvIns->Internal.s.fVMSuspended || pDrvIns->Internal.s.fVMReset, rc = VERR_WRONG_ORDER);
+    VMSTATE enmVMState = VMR3GetState(pDrvIns->Internal.s.pVMR3);
+    AssertStmt(   enmVMState == VMSTATE_SUSPENDING
+               || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+               || enmVMState == VMSTATE_SUSPENDING_LS
+               || enmVMState == VMSTATE_RESETTING
+               || enmVMState == VMSTATE_RESETTING_LS
+               || enmVMState == VMSTATE_POWERING_OFF
+               || enmVMState == VMSTATE_POWERING_OFF_LS,
+               rc = VERR_INVALID_STATE);
+
+    if (RT_SUCCESS(rc))
+        pDrvIns->Internal.s.pfnAsyncNotify = pfnAsyncNotify;
+
+    LogFlow(("pdmR3DrvHlp_SetAsyncNotification: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnAsyncNotificationCompleted} */
+static DECLCALLBACK(void) pdmR3DrvHlp_AsyncNotificationCompleted(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    if (   enmVMState == VMSTATE_SUSPENDING
+        || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+        || enmVMState == VMSTATE_SUSPENDING_LS
+        || enmVMState == VMSTATE_RESETTING
+        || enmVMState == VMSTATE_RESETTING_LS
+        || enmVMState == VMSTATE_POWERING_OFF
+        || enmVMState == VMSTATE_POWERING_OFF_LS)
+    {
+        LogFlow(("pdmR3DrvHlp_AsyncNotificationCompleted: caller='%s'/%d:\n", pDrvIns->pReg->szName, pDrvIns->iInstance));
+        VMR3AsyncPdmNotificationWakeupU(pVM->pUVM);
+    }
+    else
+        LogFlow(("pdmR3DrvHlp_AsyncNotificationCompleted: caller='%s'/%d: enmVMState=%d\n", pDrvIns->pReg->szName, pDrvIns->iInstance, enmVMState));
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnThreadCreate} */
+static DECLCALLBACK(int) pdmR3DrvHlp_ThreadCreate(PPDMDRVINS pDrvIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDRV pfnThread,
+                                                  PFNPDMTHREADWAKEUPDRV pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_ThreadCreate: caller='%s'/%d: ppThread=%p pvUser=%p pfnThread=%p pfnWakeup=%p cbStack=%#zx enmType=%d pszName=%p:{%s}\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName, pszName));
+
+    int rc = pdmR3ThreadCreateDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName);
+
+    LogFlow(("pdmR3DrvHlp_ThreadCreate: caller='%s'/%d: returns %Rrc *ppThread=%RTthrd\n", pDrvIns->pReg->szName, pDrvIns->iInstance,
+            rc, *ppThread));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnAsyncCompletionTemplateCreate} */
+static DECLCALLBACK(int) pdmR3DrvHlp_AsyncCompletionTemplateCreate(PPDMDRVINS pDrvIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate,
+                                                                   PFNPDMASYNCCOMPLETEDRV pfnCompleted, void *pvTemplateUser,
+                                                                   const char *pszDesc)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_AsyncCompletionTemplateCreate: caller='%s'/%d: ppTemplate=%p pfnCompleted=%p pszDesc=%p:{%s}\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, ppTemplate, pfnCompleted, pszDesc, pszDesc));
+
+    int rc = pdmR3AsyncCompletionTemplateCreateDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, ppTemplate, pfnCompleted, pvTemplateUser, pszDesc);
+
+    LogFlow(("pdmR3DrvHlp_AsyncCompletionTemplateCreate: caller='%s'/%d: returns %Rrc *ppThread=%p\n", pDrvIns->pReg->szName,
+             pDrvIns->iInstance, rc, *ppTemplate));
+    return rc;
+}
+
+
+#ifdef VBOX_WITH_NETSHAPER
+/** @interface_method_impl{PDMDRVHLPR3,pfnNetShaperAttach} */
+static DECLCALLBACK(int) pdmR3DrvHlp_NetShaperAttach(PPDMDRVINS pDrvIns, const char *pszBwGroup, PPDMNSFILTER pFilter)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_NetShaperAttach: caller='%s'/%d: pFilter=%p pszBwGroup=%p:{%s}\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pFilter, pszBwGroup, pszBwGroup));
+
+    int rc = PDMR3NsAttach(pDrvIns->Internal.s.pVMR3->pUVM, pDrvIns, pszBwGroup, pFilter);
+
+    LogFlow(("pdmR3DrvHlp_NetShaperAttach: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
+             pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnNetShaperDetach} */
+static DECLCALLBACK(int) pdmR3DrvHlp_NetShaperDetach(PPDMDRVINS pDrvIns, PPDMNSFILTER pFilter)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_NetShaperDetach: caller='%s'/%d: pFilter=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pFilter));
+
+    int rc = PDMR3NsDetach(pDrvIns->Internal.s.pVMR3->pUVM, pDrvIns, pFilter);
+
+    LogFlow(("pdmR3DrvHlp_NetShaperDetach: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
+             pDrvIns->iInstance, rc));
+    return rc;
+}
+#endif /* VBOX_WITH_NETSHAPER */
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnLdrGetRCInterfaceSymbols} */
+static DECLCALLBACK(int) pdmR3DrvHlp_LdrGetRCInterfaceSymbols(PPDMDRVINS pDrvIns, void *pvInterface, size_t cbInterface,
+                                                              const char *pszSymPrefix, const char *pszSymList)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_LdrGetRCInterfaceSymbols: caller='%s'/%d: pvInterface=%p cbInterface=%zu pszSymPrefix=%p:{%s} pszSymList=%p:{%s}\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pvInterface, cbInterface, pszSymPrefix, pszSymPrefix, pszSymList, pszSymList));
+
+    int rc;
+    if (   strncmp(pszSymPrefix, "drv", 3) == 0
+        && RTStrIStr(pszSymPrefix + 3, pDrvIns->pReg->szName) != NULL)
+    {
+        if (pDrvIns->pReg->fFlags & PDM_DRVREG_FLAGS_RC)
+            rc = PDMR3LdrGetInterfaceSymbols(pDrvIns->Internal.s.pVMR3,
+                                             pvInterface, cbInterface,
+                                             pDrvIns->pReg->szRCMod, pDrvIns->Internal.s.pDrv->pszRCSearchPath,
+                                             pszSymPrefix, pszSymList,
+                                             false /*fRing0OrRC*/);
+        else
+        {
+            AssertMsgFailed(("Not a raw-mode enabled driver\n"));
+            rc = VERR_PERMISSION_DENIED;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Invalid prefix '%s' for '%s'; must start with 'drv' and contain the driver name!\n",
+                         pszSymPrefix, pDrvIns->pReg->szName));
+        rc = VERR_INVALID_NAME;
+    }
+
+    LogFlow(("pdmR3DrvHlp_LdrGetRCInterfaceSymbols: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
+             pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnLdrGetR0InterfaceSymbols} */
+static DECLCALLBACK(int) pdmR3DrvHlp_LdrGetR0InterfaceSymbols(PPDMDRVINS pDrvIns, void *pvInterface, size_t cbInterface,
+                                                              const char *pszSymPrefix, const char *pszSymList)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_LdrGetR0InterfaceSymbols: caller='%s'/%d: pvInterface=%p cbInterface=%zu pszSymPrefix=%p:{%s} pszSymList=%p:{%s}\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pvInterface, cbInterface, pszSymPrefix, pszSymPrefix, pszSymList, pszSymList));
+
+    int rc;
+    if (   strncmp(pszSymPrefix, "drv", 3) == 0
+        && RTStrIStr(pszSymPrefix + 3, pDrvIns->pReg->szName) != NULL)
+    {
+        if (pDrvIns->pReg->fFlags & PDM_DRVREG_FLAGS_R0)
+            rc = PDMR3LdrGetInterfaceSymbols(pDrvIns->Internal.s.pVMR3,
+                                             pvInterface, cbInterface,
+                                             pDrvIns->pReg->szR0Mod, pDrvIns->Internal.s.pDrv->pszR0SearchPath,
+                                             pszSymPrefix, pszSymList,
+                                             true /*fRing0OrRC*/);
+        else
+        {
+            AssertMsgFailed(("Not a ring-0 enabled driver\n"));
+            rc = VERR_PERMISSION_DENIED;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Invalid prefix '%s' for '%s'; must start with 'drv' and contain the driver name!\n",
+                         pszSymPrefix, pDrvIns->pReg->szName));
+        rc = VERR_INVALID_NAME;
+    }
+
+    LogFlow(("pdmR3DrvHlp_LdrGetR0InterfaceSymbols: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
+             pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnCritSectInit} */
+static DECLCALLBACK(int) pdmR3DrvHlp_CritSectInit(PPDMDRVINS pDrvIns, PPDMCRITSECT pCritSect,
+                                                  RT_SRC_POS_DECL, const char *pszName)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DrvHlp_CritSectInit: caller='%s'/%d: pCritSect=%p pszName=%s\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pCritSect, pszName));
+
+    int rc = pdmR3CritSectInitDriver(pVM, pDrvIns, pCritSect, RT_SRC_POS_ARGS, "%s_%u", pszName, pDrvIns->iInstance);
+
+    LogFlow(("pdmR3DrvHlp_CritSectInit: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
+             pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnCallR0} */
+static DECLCALLBACK(int) pdmR3DrvHlp_CallR0(PPDMDRVINS pDrvIns, uint32_t uOperation, uint64_t u64Arg)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DrvHlp_CallR0: caller='%s'/%d: uOperation=%#x u64Arg=%#RX64\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, uOperation, u64Arg));
+
+    /*
+     * Lazy resolve the ring-0 entry point.
+     */
+    int rc = VINF_SUCCESS;
+    PFNPDMDRVREQHANDLERR0 pfnReqHandlerR0 = pDrvIns->Internal.s.pfnReqHandlerR0;
+    if (RT_UNLIKELY(pfnReqHandlerR0 == NIL_RTR0PTR))
+    {
+        if (pDrvIns->pReg->fFlags & PDM_DRVREG_FLAGS_R0)
+        {
+            char szSymbol[          sizeof("drvR0") + sizeof(pDrvIns->pReg->szName) + sizeof("ReqHandler")];
+            strcat(strcat(strcpy(szSymbol, "drvR0"),         pDrvIns->pReg->szName),         "ReqHandler");
+            szSymbol[sizeof("drvR0") - 1] = RT_C_TO_UPPER(szSymbol[sizeof("drvR0") - 1]);
+
+            rc = PDMR3LdrGetSymbolR0Lazy(pVM, pDrvIns->pReg->szR0Mod, pDrvIns->Internal.s.pDrv->pszR0SearchPath, szSymbol,
+                                         &pfnReqHandlerR0);
+            if (RT_SUCCESS(rc))
+                pDrvIns->Internal.s.pfnReqHandlerR0 = pfnReqHandlerR0;
+            else
+                pfnReqHandlerR0 = NIL_RTR0PTR;
+        }
+        else
+            rc = VERR_ACCESS_DENIED;
+    }
+    if (RT_LIKELY(pfnReqHandlerR0 != NIL_RTR0PTR))
+    {
+        /*
+         * Make the ring-0 call.
+         */
+        PDMDRIVERCALLREQHANDLERREQ Req;
+        Req.Hdr.u32Magic    = SUPVMMR0REQHDR_MAGIC;
+        Req.Hdr.cbReq       = sizeof(Req);
+        Req.pDrvInsR0       = PDMDRVINS_2_R0PTR(pDrvIns);
+        Req.uOperation      = uOperation;
+        Req.u32Alignment    = 0;
+        Req.u64Arg          = u64Arg;
+        rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_PDM_DRIVER_CALL_REQ_HANDLER, 0, &Req.Hdr);
+    }
+
+    LogFlow(("pdmR3DrvHlp_CallR0: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
+             pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnBlkCacheRetain} */
+static DECLCALLBACK(int) pdmR3DrvHlp_BlkCacheRetain(PPDMDRVINS pDrvIns, PPPDMBLKCACHE ppBlkCache,
+                                                    PFNPDMBLKCACHEXFERCOMPLETEDRV pfnXferComplete,
+                                                    PFNPDMBLKCACHEXFERENQUEUEDRV pfnXferEnqueue,
+                                                    PFNPDMBLKCACHEXFERENQUEUEDISCARDDRV pfnXferEnqueueDiscard,
+                                                    const char *pcszId)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    return PDMR3BlkCacheRetainDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, ppBlkCache,
+                                     pfnXferComplete, pfnXferEnqueue, pfnXferEnqueueDiscard, pcszId);
+}
+
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnVMGetSuspendReason} */
+static DECLCALLBACK(VMSUSPENDREASON) pdmR3DrvHlp_VMGetSuspendReason(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    VMSUSPENDREASON enmReason = VMR3GetSuspendReason(pVM->pUVM);
+    LogFlow(("pdmR3DrvHlp_VMGetSuspendReason: caller='%s'/%d: returns %d\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, enmReason));
+    return enmReason;
+}
+
+
+/** @interface_method_impl{PDMDRVHLPR3,pfnVMGetResumeReason} */
+static DECLCALLBACK(VMRESUMEREASON) pdmR3DrvHlp_VMGetResumeReason(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    VMRESUMEREASON enmReason = VMR3GetResumeReason(pVM->pUVM);
+    LogFlow(("pdmR3DrvHlp_VMGetResumeReason: caller='%s'/%d: returns %d\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, enmReason));
+    return enmReason;
+}
+
+
+/**
+ * The driver helper structure.
+ */
+const PDMDRVHLPR3 g_pdmR3DrvHlp =
+{
+    PDM_DRVHLPR3_VERSION,
+    pdmR3DrvHlp_Attach,
+    pdmR3DrvHlp_Detach,
+    pdmR3DrvHlp_DetachSelf,
+    pdmR3DrvHlp_MountPrepare,
+    pdmR3DrvHlp_AssertEMT,
+    pdmR3DrvHlp_AssertOther,
+    pdmR3DrvHlp_VMSetError,
+    pdmR3DrvHlp_VMSetErrorV,
+    pdmR3DrvHlp_VMSetRuntimeError,
+    pdmR3DrvHlp_VMSetRuntimeErrorV,
+    pdmR3DrvHlp_VMState,
+    pdmR3DrvHlp_VMTeleportedAndNotFullyResumedYet,
+    pdmR3DrvHlp_GetSupDrvSession,
+    pdmR3DrvHlp_QueueCreate,
+    pdmR3DrvHlp_TMGetVirtualFreq,
+    pdmR3DrvHlp_TMGetVirtualTime,
+    pdmR3DrvHlp_TMTimerCreate,
+    pdmR3DrvHlp_SSMRegister,
+    pdmR3DrvHlp_SSMDeregister,
+    pdmR3DrvHlp_DBGFInfoRegister,
+    pdmR3DrvHlp_DBGFInfoRegisterArgv,
+    pdmR3DrvHlp_DBGFInfoDeregister,
+    pdmR3DrvHlp_STAMRegister,
+    pdmR3DrvHlp_STAMRegisterF,
+    pdmR3DrvHlp_STAMRegisterV,
+    pdmR3DrvHlp_STAMDeregister,
+    pdmR3DrvHlp_SUPCallVMMR0Ex,
+    pdmR3DrvHlp_USBRegisterHub,
+    pdmR3DrvHlp_SetAsyncNotification,
+    pdmR3DrvHlp_AsyncNotificationCompleted,
+    pdmR3DrvHlp_ThreadCreate,
+    pdmR3DrvHlp_AsyncCompletionTemplateCreate,
+#ifdef VBOX_WITH_NETSHAPER
+    pdmR3DrvHlp_NetShaperAttach,
+    pdmR3DrvHlp_NetShaperDetach,
+#endif /* VBOX_WITH_NETSHAPER */
+    pdmR3DrvHlp_LdrGetRCInterfaceSymbols,
+    pdmR3DrvHlp_LdrGetR0InterfaceSymbols,
+    pdmR3DrvHlp_CritSectInit,
+    pdmR3DrvHlp_CallR0,
+    pdmR3DrvHlp_BlkCacheRetain,
+    pdmR3DrvHlp_VMGetSuspendReason,
+    pdmR3DrvHlp_VMGetResumeReason,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    PDM_DRVHLPR3_VERSION /* u32TheEnd */
+};
+
+/** @} */
diff --git a/src/VBox/VMM/VMMR3/PDMLdr.cpp b/src/VBox/VMM/VMMR3/PDMLdr.cpp
new file mode 100644
index 00000000..03936474
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMLdr.cpp
@@ -0,0 +1,1769 @@
+/* $Id: PDMLdr.cpp $ */
+/** @file
+ * PDM - Pluggable Device Manager, module loader.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+//#define PDMLDR_FAKE_MODE
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_LDR
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/sup.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/VBoxTpG.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/file.h>
+#include <iprt/ldr.h>
+#include <iprt/mem.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+
+#include <limits.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Structure which the user argument of the RTLdrGetBits() callback points to.
+ * @internal
+ */
+typedef struct PDMGETIMPORTARGS
+{
+    PVM         pVM;
+    PPDMMOD     pModule;
+} PDMGETIMPORTARGS, *PPDMGETIMPORTARGS;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_RAW_MODE_KEEP
+static DECLCALLBACK(int) pdmR3GetImportRC(RTLDRMOD hLdrMod, const char *pszModule, const char *pszSymbol, unsigned uSymbol, RTUINTPTR *pValue, void *pvUser);
+static char    *pdmR3FileRC(const char *pszFile, const char *pszSearchPath);
+#endif
+static int      pdmR3LoadR0U(PUVM pUVM, const char *pszFilename, const char *pszName, const char *pszSearchPath);
+static char    *pdmR3FileR0(const char *pszFile, const char *pszSearchPath);
+static char    *pdmR3File(const char *pszFile, const char *pszDefaultExt, const char *pszSearchPath, bool fShared);
+
+
+
+/**
+ * Loads the VMMR0.r0 module early in the init process.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+VMMR3_INT_DECL(int) PDMR3LdrLoadVMMR0U(PUVM pUVM)
+{
+    return pdmR3LoadR0U(pUVM, NULL, VMMR0_MAIN_MODULE_NAME, NULL);
+}
+
+
+/**
+ * Init the module loader part of PDM.
+ *
+ * This routine will load the Host Context Ring-0 and Guest
+ * Context VMM modules.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM structure.
+ */
+int pdmR3LdrInitU(PUVM pUVM)
+{
+#if !defined(PDMLDR_FAKE_MODE) && defined(VBOX_WITH_RAW_MODE_KEEP)
+    /*
+     * Load the mandatory RC module, the VMMR0.r0 is loaded before VM creation.
+     */
+    PVM pVM = pUVM->pVM; AssertPtr(pVM);
+    if (VM_IS_RAW_MODE_ENABLED(pVM))
+    {
+        int rc = PDMR3LdrLoadRC(pVM, NULL, VMMRC_MAIN_MODULE_NAME);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+#else
+    RT_NOREF(pUVM);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminate the module loader part of PDM.
+ *
+ * This will unload and free all modules.
+ *
+ * @param   pUVM        The user mode VM structure.
+ *
+ * @remarks This is normally called twice during termination.
+ */
+void pdmR3LdrTermU(PUVM pUVM)
+{
+    /*
+     * Free the modules.
+     */
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMMOD pModule = pUVM->pdm.s.pModules;
+    pUVM->pdm.s.pModules = NULL;
+    while (pModule)
+    {
+        /* free loader item. */
+        if (pModule->hLdrMod != NIL_RTLDRMOD)
+        {
+            int rc2 = RTLdrClose(pModule->hLdrMod);
+            AssertRC(rc2);
+            pModule->hLdrMod = NIL_RTLDRMOD;
+        }
+
+        /* free bits. */
+        switch (pModule->eType)
+        {
+            case PDMMOD_TYPE_R0:
+            {
+                Assert(pModule->ImageBase);
+                int rc2 = SUPR3FreeModule((void *)(uintptr_t)pModule->ImageBase);
+                AssertRC(rc2);
+                pModule->ImageBase = 0;
+                break;
+            }
+
+#ifdef VBOX_WITH_RAW_MODE_KEEP
+            case PDMMOD_TYPE_RC:
+#endif
+            case PDMMOD_TYPE_R3:
+                /* MM will free this memory for us - it's alloc only memory. :-) */
+                break;
+
+            default:
+                AssertMsgFailed(("eType=%d\n", pModule->eType));
+                break;
+        }
+        pModule->pvBits = NULL;
+
+        void    *pvFree = pModule;
+        pModule = pModule->pNext;
+        RTMemFree(pvFree);
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+}
+
+
+/**
+ * Applies relocations to RC modules.
+ *
+ * This must be done very early in the relocation
+ * process so that components can resolve RC symbols during relocation.
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3_INT_DECL(void) PDMR3LdrRelocateU(PUVM pUVM, RTGCINTPTR offDelta)
+{
+#ifdef VBOX_WITH_RAW_MODE_KEEP
+    LogFlow(("PDMR3LdrRelocate: offDelta=%RGv\n", offDelta));
+    RT_NOREF1(offDelta);
+
+    /*
+     * RC Modules.
+     */
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    if (pUVM->pdm.s.pModules)
+    {
+        /*
+         * The relocation have to be done in two passes so imports
+         * can be correctly resolved. The first pass will update
+         * the ImageBase saving the current value in OldImageBase.
+         * The second pass will do the actual relocation.
+         */
+        /* pass 1 */
+        PPDMMOD pCur;
+        for (pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+        {
+            if (pCur->eType == PDMMOD_TYPE_RC)
+            {
+                pCur->OldImageBase = pCur->ImageBase;
+                pCur->ImageBase = MMHyperR3ToRC(pUVM->pVM, pCur->pvBits);
+            }
+        }
+
+        /* pass 2 */
+        for (pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+        {
+            if (pCur->eType == PDMMOD_TYPE_RC)
+            {
+                PDMGETIMPORTARGS Args;
+                Args.pVM = pUVM->pVM;
+                Args.pModule = pCur;
+                int rc = RTLdrRelocate(pCur->hLdrMod, pCur->pvBits, pCur->ImageBase, pCur->OldImageBase,
+                                       pdmR3GetImportRC, &Args);
+                AssertFatalMsgRC(rc, ("RTLdrRelocate failed, rc=%d\n", rc));
+            }
+        }
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+#else
+    RT_NOREF2(pUVM, offDelta);
+#endif
+}
+
+
+/**
+ * Loads a module into the host context ring-3.
+ *
+ * This is used by the driver and device init functions to load modules
+ * containing the drivers and devices. The function can be extended to
+ * load modules which are not native to the environment we're running in,
+ * but at the moment this is not required.
+ *
+ * No reference counting is kept, since we don't implement any facilities
+ * for unloading the module. But the module will naturally be released
+ * when the VM terminates.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszFilename     Filename of the module binary.
+ * @param   pszName         Module name. Case sensitive and the length is limited!
+ */
+int pdmR3LoadR3U(PUVM pUVM, const char *pszFilename, const char *pszName)
+{
+    /*
+     * Validate input.
+     */
+    AssertMsg(RTCritSectIsInitialized(&pUVM->pdm.s.ListCritSect), ("bad init order!\n"));
+    Assert(pszFilename);
+    size_t cchFilename = strlen(pszFilename);
+    Assert(pszName);
+    size_t cchName = strlen(pszName);
+    PPDMMOD  pCur;
+    if (cchName >= sizeof(pCur->szName))
+    {
+        AssertMsgFailed(("Name is too long, cchName=%d pszName='%s'\n", cchName, pszName));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Try lookup the name and see if the module exists.
+     */
+    int rc;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+    {
+        if (!strcmp(pCur->szName, pszName))
+        {
+            if (pCur->eType == PDMMOD_TYPE_R3)
+                rc = VINF_PDM_ALREADY_LOADED;
+            else
+                rc = VERR_PDM_MODULE_NAME_CLASH;
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+            AssertMsgRC(rc, ("We've already got a module '%s' loaded!\n", pszName));
+            return rc;
+        }
+    }
+
+    /*
+     * Allocate the module list node and initialize it.
+     */
+    const char *pszSuff = RTLdrGetSuff();
+    size_t      cchSuff = RTPathHasSuffix(pszFilename) ? 0 : strlen(pszSuff);
+    PPDMMOD     pModule = (PPDMMOD)RTMemAllocZ(RT_UOFFSETOF_DYN(PDMMOD, szFilename[cchFilename + cchSuff + 1]));
+    if (pModule)
+    {
+        pModule->eType = PDMMOD_TYPE_R3;
+        memcpy(pModule->szName, pszName, cchName); /* memory is zero'd, no need to copy terminator :-) */
+        memcpy(pModule->szFilename, pszFilename, cchFilename);
+        memcpy(&pModule->szFilename[cchFilename], pszSuff, cchSuff);
+
+        /*
+         * Load the loader item.
+         */
+        RTERRINFOSTATIC ErrInfo;
+        RTErrInfoInitStatic(&ErrInfo);
+        rc = SUPR3HardenedLdrLoadPlugIn(pModule->szFilename, &pModule->hLdrMod, &ErrInfo.Core);
+        if (RT_SUCCESS(rc))
+        {
+            pModule->pNext = pUVM->pdm.s.pModules;
+            pUVM->pdm.s.pModules = pModule;
+        }
+        else
+        {
+            /* Something went wrong, most likely module not found. Don't consider other unlikely errors */
+            rc = VMSetError(pUVM->pVM, rc, RT_SRC_POS,
+                            N_("Unable to load R3 module %s (%s): %s"), pModule->szFilename, pszName, ErrInfo.Core.pszMsg);
+            RTMemFree(pModule);
+        }
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+#ifdef VBOX_WITH_RAW_MODE_KEEP
+
+/**
+ * Resolve an external symbol during RTLdrGetBits() of a RC module.
+ *
+ * @returns VBox status code.
+ * @param   hLdrMod         The loader module handle.
+ * @param   pszModule       Module name.
+ * @param   pszSymbol       Symbol name, NULL if uSymbol should be used.
+ * @param   uSymbol         Symbol ordinal, ~0 if pszSymbol should be used.
+ * @param   pValue          Where to store the symbol value (address).
+ * @param   pvUser          User argument.
+ */
+static DECLCALLBACK(int) pdmR3GetImportRC(RTLDRMOD hLdrMod, const char *pszModule, const char *pszSymbol, unsigned uSymbol,
+                                          RTUINTPTR *pValue, void *pvUser)
+{
+    PVM         pVM     = ((PPDMGETIMPORTARGS)pvUser)->pVM;
+    PPDMMOD     pModule = ((PPDMGETIMPORTARGS)pvUser)->pModule;
+    NOREF(hLdrMod); NOREF(uSymbol);
+
+    /*
+     * Adjust input.
+     */
+    if (pszModule && !*pszModule)
+        pszModule = NULL;
+
+    /*
+     * Builtin module.
+     */
+    if (!pszModule || !strcmp(pszModule, "VMMRCBuiltin.rc"))
+    {
+        int rc = VINF_SUCCESS;
+        if (!strcmp(pszSymbol, "g_VM"))
+            *pValue = pVM->pVMRC;
+        else if (!strcmp(pszSymbol, "g_VCpu0"))
+            *pValue = pVM->pVMRC + pVM->offVMCPU;
+        else if (!strcmp(pszSymbol, "g_CPUM"))
+            *pValue = VM_RC_ADDR(pVM, &pVM->cpum);
+        else if (   !strncmp(pszSymbol, "g_TRPM", 6)
+                 || !strncmp(pszSymbol, "g_trpm", 6)
+                 || !strncmp(pszSymbol, "TRPM", 4))
+        {
+            RTRCPTR RCPtr = 0;
+            rc = TRPMR3GetImportRC(pVM, pszSymbol, &RCPtr);
+            if (RT_SUCCESS(rc))
+                *pValue = RCPtr;
+        }
+        else if (   !strncmp(pszSymbol, "VMM", 3)
+                 || !strcmp(pszSymbol, "g_Logger")
+                 || !strcmp(pszSymbol, "g_RelLogger"))
+        {
+            RTRCPTR RCPtr = 0;
+            rc = VMMR3GetImportRC(pVM, pszSymbol, &RCPtr);
+            if (RT_SUCCESS(rc))
+                *pValue = RCPtr;
+        }
+        else if (   !strncmp(pszSymbol, "TM", 2)
+                 || !strcmp(pszSymbol, "g_pSUPGlobalInfoPage"))
+        {
+            RTRCPTR RCPtr = 0;
+            rc = TMR3GetImportRC(pVM, pszSymbol, &RCPtr);
+            if (RT_SUCCESS(rc))
+                *pValue = RCPtr;
+        }
+        else
+        {
+            AssertMsg(!pszModule, ("Unknown builtin symbol '%s' for module '%s'!\n", pszSymbol, pModule->szName)); NOREF(pModule);
+            rc = VERR_SYMBOL_NOT_FOUND;
+        }
+        if (RT_SUCCESS(rc) || pszModule)
+        {
+            if (RT_FAILURE(rc))
+                LogRel(("PDMLdr: Couldn't find symbol '%s' in module '%s'!\n", pszSymbol, pszModule));
+            return rc;
+        }
+    }
+
+    /*
+     * Search for module.
+     */
+    PUVM     pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMMOD  pCur = pUVM->pdm.s.pModules;
+    while (pCur)
+    {
+        if (    pCur->eType == PDMMOD_TYPE_RC
+            &&  (   !pszModule
+                 || !strcmp(pCur->szName, pszModule))
+           )
+        {
+            /* Search for the symbol. */
+            int rc = RTLdrGetSymbolEx(pCur->hLdrMod, pCur->pvBits, pCur->ImageBase, UINT32_MAX, pszSymbol, pValue);
+            if (RT_SUCCESS(rc))
+            {
+                AssertMsg(*pValue - pCur->ImageBase < RTLdrSize(pCur->hLdrMod),
+                          ("%RRv-%RRv %s %RRv\n", (RTRCPTR)pCur->ImageBase,
+                           (RTRCPTR)(pCur->ImageBase + RTLdrSize(pCur->hLdrMod) - 1),
+                           pszSymbol, (RTRCPTR)*pValue));
+                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+                return rc;
+            }
+            if (pszModule)
+            {
+                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+                AssertLogRelMsgFailed(("PDMLdr: Couldn't find symbol '%s' in module '%s'!\n", pszSymbol, pszModule));
+                return VERR_SYMBOL_NOT_FOUND;
+            }
+        }
+
+        /* next */
+        pCur = pCur->pNext;
+    }
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    AssertLogRelMsgFailed(("Couldn't find module '%s' for resolving symbol '%s'!\n", pszModule, pszSymbol));
+    return VERR_SYMBOL_NOT_FOUND;
+}
+
+
+/**
+ * Loads a module into the raw-mode context (i.e. into the Hypervisor memory
+ * region).
+ *
+ * @returns VBox status code.
+ * @retval  VINF_PDM_ALREADY_LOADED if the module is already loaded (name +
+ *          filename match).
+ * @retval  VERR_PDM_MODULE_NAME_CLASH if a different file has already been
+ *          loaded with the name module name.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pszFilename     Filename of the module binary.
+ * @param   pszName         Module name. Case sensitive and the length is limited!
+ */
+VMMR3DECL(int) PDMR3LdrLoadRC(PVM pVM, const char *pszFilename, const char *pszName)
+{
+    /*
+     * Validate input.
+     */
+    AssertMsg(MMR3IsInitialized(pVM), ("bad init order!\n"));
+    AssertReturn(VM_IS_RAW_MODE_ENABLED(pVM), VERR_PDM_HM_IPE);
+
+    /*
+     * Find the file if not specified.
+     */
+    char *pszFile = NULL;
+    if (!pszFilename)
+        pszFilename = pszFile = pdmR3FileRC(pszName, NULL);
+
+    /*
+     * Check if a module by that name is already loaded.
+     */
+    int     rc;
+    PUVM    pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMMOD pCur = pUVM->pdm.s.pModules;
+    while (pCur)
+    {
+        if (!strcmp(pCur->szName, pszName))
+        {
+            /* Name clash. Hopefully due to it being the same file. */
+            if (!strcmp(pCur->szFilename, pszFilename))
+                rc = VINF_PDM_ALREADY_LOADED;
+            else
+            {
+                rc = VERR_PDM_MODULE_NAME_CLASH;
+                AssertMsgFailed(("We've already got a module '%s' loaded!\n", pszName));
+            }
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            RTMemTmpFree(pszFile);
+            return rc;
+        }
+        /* next */
+        pCur = pCur->pNext;
+    }
+
+    /*
+     * Allocate the module list node.
+     */
+    PPDMMOD pModule = (PPDMMOD)RTMemAllocZ(sizeof(*pModule) + strlen(pszFilename));
+    if (!pModule)
+    {
+        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+        RTMemTmpFree(pszFile);
+        return VERR_NO_MEMORY;
+    }
+    AssertMsg(strlen(pszName) + 1 < sizeof(pModule->szName),
+              ("pazName is too long (%d chars) max is %d chars.\n", strlen(pszName), sizeof(pModule->szName) - 1));
+    strcpy(pModule->szName, pszName);
+    pModule->eType = PDMMOD_TYPE_RC;
+    strcpy(pModule->szFilename, pszFilename);
+
+
+    /*
+     * Open the loader item.
+     */
+    RTERRINFOSTATIC ErrInfo;
+    RTErrInfoInitStatic(&ErrInfo);
+    rc = SUPR3HardenedVerifyPlugIn(pszFilename, &ErrInfo.Core);
+    if (RT_SUCCESS(rc))
+    {
+        RTErrInfoClear(&ErrInfo.Core);
+        rc = RTLdrOpen(pszFilename, 0, RTLDRARCH_X86_32, &pModule->hLdrMod);
+    }
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Allocate space in the hypervisor.
+         */
+        size_t          cb = RTLdrSize(pModule->hLdrMod);
+        cb = RT_ALIGN_Z(cb, PAGE_SIZE);
+        uint32_t        cPages = (uint32_t)(cb >> PAGE_SHIFT);
+        if (((size_t)cPages << PAGE_SHIFT) == cb)
+        {
+            PSUPPAGE    paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(paPages[0]));
+            if (paPages)
+            {
+                rc = SUPR3PageAllocEx(cPages, 0 /*fFlags*/, &pModule->pvBits, NULL /*pR0Ptr*/, paPages);
+                if (RT_SUCCESS(rc))
+                {
+                    RTGCPTR GCPtr;
+                    rc = MMR3HyperMapPages(pVM, pModule->pvBits, NIL_RTR0PTR,
+                                           cPages, paPages, pModule->szName, &GCPtr);
+                    if (RT_SUCCESS(rc))
+                    {
+                        MMR3HyperReserveFence(pVM);
+
+                        /*
+                         * Get relocated image bits.
+                         */
+                        Assert(MMHyperR3ToRC(pVM, pModule->pvBits) == GCPtr);
+                        pModule->ImageBase = GCPtr;
+                        PDMGETIMPORTARGS Args;
+                        Args.pVM = pVM;
+                        Args.pModule = pModule;
+                        rc = RTLdrGetBits(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, pdmR3GetImportRC, &Args);
+                        if (RT_SUCCESS(rc))
+                        {
+#ifdef VBOX_WITH_DTRACE_RC
+                            /*
+                             * Register the tracer bits if present.
+                             */
+                            RTLDRADDR uValue;
+                            rc = RTLdrGetSymbolEx(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, UINT32_MAX,
+                                                  "g_VTGObjHeader", &uValue);
+                            if (RT_SUCCESS(rc))
+                            {
+                                PVTGOBJHDR pVtgHdr = (PVTGOBJHDR)MMHyperRCToCC(pVM, (RTRCPTR)uValue);
+                                if (   pVtgHdr
+                                    && !memcmp(pVtgHdr->szMagic, VTGOBJHDR_MAGIC, sizeof(pVtgHdr->szMagic)))
+                                    rc = SUPR3TracerRegisterModule(~(uintptr_t)0, pModule->szName, pVtgHdr, uValue,
+                                                                   SUP_TRACER_UMOD_FLAGS_SHARED);
+                                else
+                                    rc = pVtgHdr ? VERR_INVALID_MAGIC : VERR_INVALID_POINTER;
+                                if (RT_FAILURE(rc))
+                                    LogRel(("PDMLdr: Failed to register tracepoints for '%s': %Rrc\n", pModule->szName, rc));
+                            }
+#endif
+
+                            /*
+                             * Insert the module.
+                             */
+                            if (pUVM->pdm.s.pModules)
+                            {
+                                /* we don't expect this list to be very long, so rather save the tail pointer. */
+                                pCur = pUVM->pdm.s.pModules;
+                                while (pCur->pNext)
+                                    pCur = pCur->pNext;
+                                pCur->pNext = pModule;
+                            }
+                            else
+                                pUVM->pdm.s.pModules = pModule; /* (pNext is zeroed by alloc) */
+                            Log(("PDM: RC Module at %RRv %s (%s)\n", (RTRCPTR)pModule->ImageBase, pszName, pszFilename));
+
+                            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+                            RTMemTmpFree(pszFile);
+                            RTMemTmpFree(paPages);
+
+                            return VINF_SUCCESS;
+                        }
+                    }
+                    else
+                    {
+                        AssertRC(rc);
+                        SUPR3PageFreeEx(pModule->pvBits, cPages);
+                    }
+                }
+                else
+                    AssertMsgFailed(("SUPR3PageAlloc(%d,) -> %Rrc\n", cPages, rc));
+                RTMemTmpFree(paPages);
+            }
+            else
+                rc = VERR_NO_TMP_MEMORY;
+        }
+        else
+            rc = VERR_OUT_OF_RANGE;
+        int rc2 = RTLdrClose(pModule->hLdrMod);
+        AssertRC(rc2);
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+    /* Don't consider VERR_PDM_MODULE_NAME_CLASH and VERR_NO_MEMORY above as these are very unlikely. */
+    if (RT_FAILURE(rc) && RTErrInfoIsSet(&ErrInfo.Core))
+        rc = VMSetError(pVM, rc, RT_SRC_POS, N_("Cannot load RC module %s: %s"), pszFilename, ErrInfo.Core.pszMsg);
+    else if (RT_FAILURE(rc))
+        rc = VMSetError(pVM, rc, RT_SRC_POS, N_("Cannot load RC module %s"), pszFilename);
+
+    RTMemFree(pModule);
+    RTMemTmpFree(pszFile);
+    return rc;
+}
+
+#endif /* VBOX_WITH_RAW_MODE_KEEP */
+
+/**
+ * Loads a module into the ring-0 context.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszFilename     Filename of the module binary.
+ * @param   pszName         Module name. Case sensitive and the length is limited!
+ * @param   pszSearchPath   List of directories to search if @a pszFilename is
+ *                          not specified.  Can be NULL, in which case the arch
+ *                          dependent install dir is searched.
+ */
+static int pdmR3LoadR0U(PUVM pUVM, const char *pszFilename, const char *pszName, const char *pszSearchPath)
+{
+    /*
+     * Validate input.
+     */
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMMOD pCur = pUVM->pdm.s.pModules;
+    while (pCur)
+    {
+        if (!strcmp(pCur->szName, pszName))
+        {
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            AssertMsgFailed(("We've already got a module '%s' loaded!\n", pszName));
+            return VERR_PDM_MODULE_NAME_CLASH;
+        }
+        /* next */
+        pCur = pCur->pNext;
+    }
+
+    /*
+     * Find the file if not specified.
+     */
+    char *pszFile = NULL;
+    if (!pszFilename)
+        pszFilename = pszFile = pdmR3FileR0(pszName, pszSearchPath);
+
+    /*
+     * Allocate the module list node.
+     */
+    PPDMMOD     pModule = (PPDMMOD)RTMemAllocZ(sizeof(*pModule) + strlen(pszFilename));
+    if (!pModule)
+    {
+        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+        RTMemTmpFree(pszFile);
+        return VERR_NO_MEMORY;
+    }
+    AssertMsg(strlen(pszName) + 1 < sizeof(pModule->szName),
+              ("pazName is too long (%d chars) max is %d chars.\n", strlen(pszName), sizeof(pModule->szName) - 1));
+    strcpy(pModule->szName, pszName);
+    pModule->eType = PDMMOD_TYPE_R0;
+    strcpy(pModule->szFilename, pszFilename);
+
+    /*
+     * Ask the support library to load it.
+     */
+    void           *pvImageBase;
+    RTERRINFOSTATIC ErrInfo;
+    RTErrInfoInitStatic(&ErrInfo);
+    int rc = SUPR3LoadModule(pszFilename, pszName, &pvImageBase, &ErrInfo.Core);
+    if (RT_SUCCESS(rc))
+    {
+        pModule->hLdrMod = NIL_RTLDRMOD;
+        pModule->ImageBase = (uintptr_t)pvImageBase;
+
+        /*
+         * Insert the module.
+         */
+        if (pUVM->pdm.s.pModules)
+        {
+            /* we don't expect this list to be very long, so rather save the tail pointer. */
+            pCur = pUVM->pdm.s.pModules;
+            while (pCur->pNext)
+                pCur = pCur->pNext;
+            pCur->pNext = pModule;
+        }
+        else
+            pUVM->pdm.s.pModules = pModule; /* (pNext is zeroed by alloc) */
+        Log(("PDM: R0 Module at %RHv %s (%s)\n", (RTR0PTR)pModule->ImageBase, pszName, pszFilename));
+        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+        RTMemTmpFree(pszFile);
+        return VINF_SUCCESS;
+    }
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    RTMemFree(pModule);
+    LogRel(("PDMLdr: pdmR3LoadR0U: pszName=\"%s\" rc=%Rrc szErr=\"%s\"\n", pszName, rc, ErrInfo.Core.pszMsg));
+
+    /* Don't consider VERR_PDM_MODULE_NAME_CLASH and VERR_NO_MEMORY above as these are very unlikely. */
+    if (RT_FAILURE(rc))
+        rc = VMR3SetError(pUVM, rc, RT_SRC_POS, N_("Failed to load R0 module %s: %s"), pszFilename, ErrInfo.Core.pszMsg);
+
+    RTMemTmpFree(pszFile); /* might be reference thru pszFilename in the above VMSetError call. */
+    return rc;
+}
+
+
+/**
+ * Makes sure a ring-0 module is loaded.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszModule       Module name (no path).
+ * @param   pszSearchPath   List of directories to search for the module
+ *                          (assumes @a pszModule is also a filename).
+ */
+VMMR3_INT_DECL(int) PDMR3LdrLoadR0(PUVM pUVM, const char *pszModule, const char *pszSearchPath)
+{
+    /*
+     * Find the module.
+     */
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+    {
+        if (   pModule->eType == PDMMOD_TYPE_R0
+            && !strcmp(pModule->szName, pszModule))
+        {
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            return VINF_SUCCESS;
+        }
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+    /*
+     * Okay, load it.
+     */
+    return pdmR3LoadR0U(pUVM, NULL, pszModule, pszSearchPath);
+}
+
+
+/**
+ * Get the address of a symbol in a given HC ring 3 module.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pszModule       Module name.
+ * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
+ *                          ordinal value rather than a string pointer.
+ * @param   ppvValue        Where to store the symbol value.
+ */
+VMMR3_INT_DECL(int) PDMR3LdrGetSymbolR3(PVM pVM, const char *pszModule, const char *pszSymbol, void **ppvValue)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtr(pVM);
+    AssertPtr(pszModule);
+    AssertPtr(ppvValue);
+    PUVM pUVM = pVM->pUVM;
+    AssertMsg(RTCritSectIsInitialized(&pUVM->pdm.s.ListCritSect), ("bad init order!\n"));
+
+    /*
+     * Find the module.
+     */
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+    {
+        if (    pModule->eType == PDMMOD_TYPE_R3
+            &&  !strcmp(pModule->szName, pszModule))
+        {
+            RTUINTPTR Value = 0;
+            int rc = RTLdrGetSymbolEx(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, UINT32_MAX, pszSymbol, &Value);
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            if (RT_SUCCESS(rc))
+            {
+                *ppvValue = (void *)(uintptr_t)Value;
+                Assert((uintptr_t)*ppvValue == Value);
+            }
+            else
+            {
+                if ((uintptr_t)pszSymbol < 0x10000)
+                    AssertMsg(rc, ("Couldn't symbol '%u' in module '%s'\n", (unsigned)(uintptr_t)pszSymbol, pszModule));
+                else
+                    AssertMsg(rc, ("Couldn't symbol '%s' in module '%s'\n", pszSymbol, pszModule));
+            }
+            return rc;
+        }
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    AssertMsgFailed(("Couldn't locate module '%s'\n", pszModule));
+    return VERR_SYMBOL_NOT_FOUND;
+}
+
+
+/**
+ * Get the address of a symbol in a given HC ring 0 module.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pszModule       Module name. If NULL the main R0 module (VMMR0.r0) is assumes.
+ * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
+ *                          ordinal value rather than a string pointer.
+ * @param   ppvValue        Where to store the symbol value.
+ */
+VMMR3DECL(int) PDMR3LdrGetSymbolR0(PVM pVM, const char *pszModule, const char *pszSymbol, PRTR0PTR ppvValue)
+{
+#ifdef PDMLDR_FAKE_MODE
+    *ppvValue = 0xdeadbeef;
+    return VINF_SUCCESS;
+
+#else
+    /*
+     * Validate input.
+     */
+    AssertPtr(pVM);
+    AssertPtrNull(pszModule);
+    AssertPtr(ppvValue);
+    PUVM pUVM = pVM->pUVM;
+    AssertMsg(RTCritSectIsInitialized(&pUVM->pdm.s.ListCritSect), ("bad init order!\n"));
+
+    if (!pszModule)
+        pszModule = VMMR0_MAIN_MODULE_NAME;
+
+    /*
+     * Find the module.
+     */
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+    {
+        if (    pModule->eType == PDMMOD_TYPE_R0
+            &&  !strcmp(pModule->szName, pszModule))
+        {
+            int rc = SUPR3GetSymbolR0((void *)(uintptr_t)pModule->ImageBase, pszSymbol, (void **)ppvValue);
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgRC(rc, ("Couldn't find symbol '%s' in module '%s'\n", pszSymbol, pszModule));
+                LogRel(("PDMLdr: PDMGetSymbol: Couldn't find symbol '%s' in module '%s'\n", pszSymbol, pszModule));
+            }
+            return rc;
+        }
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    AssertMsgFailed(("Couldn't locate module '%s'\n", pszModule));
+    return VERR_SYMBOL_NOT_FOUND;
+#endif
+}
+
+
+/**
+ * Same as PDMR3LdrGetSymbolR0 except that the module will be attempted loaded if not found.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pszModule       Module name. If NULL the main R0 module (VMMR0.r0) is assumed.
+ * @param   pszSearchPath   List of directories to search if @a pszFile is
+ *                          not qualified with a path.  Can be NULL, in which
+ *                          case the arch dependent install dir is searched.
+ * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
+ *                          ordinal value rather than a string pointer.
+ * @param   ppvValue        Where to store the symbol value.
+ */
+VMMR3DECL(int) PDMR3LdrGetSymbolR0Lazy(PVM pVM, const char *pszModule, const char *pszSearchPath, const char *pszSymbol,
+                                       PRTR0PTR ppvValue)
+{
+#ifdef PDMLDR_FAKE_MODE
+    *ppvValue = 0xdeadbeef;
+    return VINF_SUCCESS;
+
+#else
+    AssertPtr(pVM);
+    AssertPtrNull(pszModule);
+    AssertPtr(ppvValue);
+    PUVM pUVM = pVM->pUVM;
+    AssertMsg(RTCritSectIsInitialized(&pUVM->pdm.s.ListCritSect), ("bad init order!\n"));
+
+    if (pszModule) /* (We don't lazy load the main R0 module.) */
+    {
+        /*
+         * Since we're lazy, we'll only check if the module is present
+         * and hand it over to PDMR3LdrGetSymbolR0 when that's done.
+         */
+        AssertMsgReturn(!strpbrk(pszModule, "/\\:\n\r\t"), ("pszModule=%s\n", pszModule), VERR_INVALID_PARAMETER);
+        PPDMMOD pModule;
+        RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+        for (pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+            if (    pModule->eType == PDMMOD_TYPE_R0
+                &&  !strcmp(pModule->szName, pszModule))
+                break;
+        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+        if (!pModule)
+        {
+            int rc = pdmR3LoadR0U(pUVM, NULL, pszModule, pszSearchPath);
+            AssertMsgRCReturn(rc, ("pszModule=%s rc=%Rrc\n", pszModule, rc), VERR_MODULE_NOT_FOUND);
+        }
+    }
+
+    return PDMR3LdrGetSymbolR0(pVM, pszModule, pszSymbol, ppvValue);
+#endif
+}
+
+
+/**
+ * Get the address of a symbol in a given RC module.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pszModule       Module name.  If NULL the main R0 module (VMMRC.rc)
+ *                          is assumes.
+ * @param   pszSymbol       Symbol name.  If it's value is less than 64k it's
+ *                          treated like a ordinal value rather than a string
+ *                          pointer.
+ * @param   pRCPtrValue     Where to store the symbol value.
+ */
+VMMR3DECL(int) PDMR3LdrGetSymbolRC(PVM pVM, const char *pszModule, const char *pszSymbol, PRTRCPTR pRCPtrValue)
+{
+#if defined(PDMLDR_FAKE_MODE) || !defined(VBOX_WITH_RAW_MODE_KEEP)
+    RT_NOREF(pVM, pszModule, pszSymbol);
+    Assert(VM_IS_RAW_MODE_ENABLED(pVM));
+    *pRCPtrValue = NIL_RTRCPTR;
+    return VINF_SUCCESS;
+
+#else
+    /*
+     * Validate input.
+     */
+    AssertPtr(pVM);
+    AssertPtrNull(pszModule);
+    AssertPtr(pRCPtrValue);
+    AssertMsg(MMR3IsInitialized(pVM), ("bad init order!\n"));
+
+    if (!pszModule)
+        pszModule = VMMRC_MAIN_MODULE_NAME;
+
+    /*
+     * Find the module.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+    {
+        if (    pModule->eType == PDMMOD_TYPE_RC
+            &&  !strcmp(pModule->szName, pszModule))
+        {
+            RTUINTPTR Value;
+            int rc = RTLdrGetSymbolEx(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, UINT32_MAX, pszSymbol, &Value);
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            if (RT_SUCCESS(rc))
+            {
+                *pRCPtrValue = (RTGCPTR)Value;
+                Assert(*pRCPtrValue == Value);
+            }
+            else
+            {
+                if ((uintptr_t)pszSymbol < 0x10000)
+                    AssertMsg(rc, ("Couldn't symbol '%u' in module '%s'\n", (unsigned)(uintptr_t)pszSymbol, pszModule));
+                else
+                    AssertMsg(rc, ("Couldn't symbol '%s' in module '%s'\n", pszSymbol, pszModule));
+            }
+            return rc;
+        }
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    AssertMsgFailed(("Couldn't locate module '%s'\n", pszModule));
+    return VERR_SYMBOL_NOT_FOUND;
+#endif
+}
+
+
+/**
+ * Same as PDMR3LdrGetSymbolRC except that the module will be attempted loaded if not found.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pszModule       Module name.  If NULL the main RC module (VMMRC.rc)
+ *                          is assumed.
+ * @param   pszSearchPath   List of directories to search if @a pszFile is
+ *                          not qualified with a path.  Can be NULL, in which
+ *                          case the arch dependent install dir is searched.
+ * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
+ *                          ordinal value rather than a string pointer.
+ * @param   pRCPtrValue     Where to store the symbol value.
+ */
+VMMR3DECL(int) PDMR3LdrGetSymbolRCLazy(PVM pVM, const char *pszModule, const char *pszSearchPath, const char *pszSymbol,
+                                       PRTRCPTR pRCPtrValue)
+{
+#if defined(PDMLDR_FAKE_MODE) || !defined(VBOX_WITH_RAW_MODE_KEEP)
+    RT_NOREF(pVM, pszModule, pszSearchPath, pszSymbol);
+    Assert(VM_IS_RAW_MODE_ENABLED(pVM));
+    *pRCPtrValue = NIL_RTRCPTR;
+    return VINF_SUCCESS;
+
+#else
+    AssertPtr(pVM);
+    if (!pszModule)
+        pszModule = VMMRC_MAIN_MODULE_NAME;
+    AssertPtr(pszModule);
+    AssertPtr(pRCPtrValue);
+    AssertMsg(MMR3IsInitialized(pVM), ("bad init order!\n"));
+
+    /*
+     * Since we're lazy, we'll only check if the module is present
+     * and hand it over to PDMR3LdrGetSymbolRC when that's done.
+     */
+    AssertMsgReturn(!strpbrk(pszModule, "/\\:\n\r\t"), ("pszModule=%s\n", pszModule), VERR_INVALID_PARAMETER);
+    PUVM    pUVM = pVM->pUVM;
+    PPDMMOD pModule;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+        if (    pModule->eType == PDMMOD_TYPE_RC
+            &&  !strcmp(pModule->szName, pszModule))
+            break;
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    if (!pModule)
+    {
+        char *pszFilename = pdmR3FileRC(pszModule, pszSearchPath);
+        AssertMsgReturn(pszFilename, ("pszModule=%s\n", pszModule), VERR_MODULE_NOT_FOUND);
+        int rc = PDMR3LdrLoadRC(pVM, pszFilename, pszModule);
+        RTMemTmpFree(pszFilename);
+        AssertMsgRCReturn(rc, ("pszModule=%s rc=%Rrc\n", pszModule, rc), VERR_MODULE_NOT_FOUND);
+    }
+
+    return PDMR3LdrGetSymbolRC(pVM, pszModule, pszSymbol, pRCPtrValue);
+#endif
+}
+
+
+/**
+ * Constructs the full filename for a R3 image file.
+ *
+ * @returns Pointer to temporary memory containing the filename.
+ *          Caller must free this using RTMemTmpFree().
+ * @returns NULL on failure.
+ *
+ * @param   pszFile         File name (no path).
+ * @param   fShared         If true, search in the shared directory (/usr/lib on Unix), else
+ *                          search in the private directory (/usr/lib/virtualbox on Unix).
+ *                          Ignored if VBOX_PATH_SHARED_LIBS is not defined.
+ */
+char *pdmR3FileR3(const char *pszFile, bool fShared)
+{
+    return pdmR3File(pszFile, NULL, NULL, fShared);
+}
+
+
+/**
+ * Constructs the full filename for a R0 image file.
+ *
+ * @returns Pointer to temporary memory containing the filename.
+ *          Caller must free this using RTMemTmpFree().
+ * @returns NULL on failure.
+ *
+ * @param   pszFile         File name (no path).
+ * @param   pszSearchPath   List of directories to search if @a pszFile is
+ *                          not qualified with a path.  Can be NULL, in which
+ *                          case the arch dependent install dir is searched.
+ */
+char *pdmR3FileR0(const char *pszFile, const char *pszSearchPath)
+{
+    return pdmR3File(pszFile, NULL, pszSearchPath, /*fShared=*/false);
+}
+
+
+/**
+ * Constructs the full filename for a RC image file.
+ *
+ * @returns Pointer to temporary memory containing the filename.
+ *          Caller must free this using RTMemTmpFree().
+ * @returns NULL on failure.
+ *
+ * @param   pszFile         File name (no path).
+ * @param   pszSearchPath   List of directories to search if @a pszFile is
+ *                          not qualified with a path.  Can be NULL, in which
+ *                          case the arch dependent install dir is searched.
+ */
+char *pdmR3FileRC(const char *pszFile, const char *pszSearchPath)
+{
+    return pdmR3File(pszFile, NULL, pszSearchPath, /*fShared=*/false);
+}
+
+
+/**
+ * Worker for pdmR3File().
+ *
+ * @returns Pointer to temporary memory containing the filename.
+ *          Caller must free this using RTMemTmpFree().
+ * @returns NULL on failure.
+ *
+ * @param   pszDir        Directory part
+ * @param   pszFile       File name part
+ * @param   pszDefaultExt Extension part
+ */
+static char *pdmR3FileConstruct(const char *pszDir, const char *pszFile, const char *pszDefaultExt)
+{
+    /*
+     * Allocate temp memory for return buffer.
+     */
+    size_t cchDir  = strlen(pszDir);
+    size_t cchFile = strlen(pszFile);
+    size_t cchDefaultExt;
+
+    /*
+     * Default extention?
+     */
+    if (!pszDefaultExt || strchr(pszFile, '.'))
+        cchDefaultExt = 0;
+    else
+        cchDefaultExt = strlen(pszDefaultExt);
+
+    size_t cchPath = cchDir + 1 + cchFile + cchDefaultExt + 1;
+    AssertMsgReturn(cchPath <= RTPATH_MAX, ("Path too long!\n"), NULL);
+
+    char *pszRet = (char *)RTMemTmpAlloc(cchDir + 1 + cchFile + cchDefaultExt + 1);
+    AssertMsgReturn(pszRet, ("Out of temporary memory!\n"), NULL);
+
+    /*
+     * Construct the filename.
+     */
+    memcpy(pszRet, pszDir, cchDir);
+    pszRet[cchDir++] = '/';            /* this works everywhere */
+    memcpy(pszRet + cchDir, pszFile, cchFile + 1);
+    if (cchDefaultExt)
+        memcpy(pszRet + cchDir + cchFile, pszDefaultExt, cchDefaultExt + 1);
+
+    return pszRet;
+}
+
+
+/**
+ * Worker for pdmR3FileRC(), pdmR3FileR0() and pdmR3FileR3().
+ *
+ * @returns Pointer to temporary memory containing the filename.
+ *          Caller must free this using RTMemTmpFree().
+ * @returns NULL on failure.
+ * @param   pszFile         File name (no path).
+ * @param   pszDefaultExt   The default extention, NULL if none.
+ * @param   pszSearchPath   List of directories to search if @a pszFile is
+ *                          not qualified with a path.  Can be NULL, in which
+ *                          case the arch dependent install dir is searched.
+ * @param   fShared         If true, search in the shared directory (/usr/lib on Unix), else
+ *                          search in the private directory (/usr/lib/virtualbox on Unix).
+ *                          Ignored if VBOX_PATH_SHARED_LIBS is not defined.
+ * @todo    We'll have this elsewhere than in the root later!
+ * @todo    Remove the fShared hack again once we don't need to link against VBoxDD anymore!
+ */
+static char *pdmR3File(const char *pszFile, const char *pszDefaultExt, const char *pszSearchPath, bool fShared)
+{
+    char szPath[RTPATH_MAX];
+    int  rc;
+
+    AssertLogRelReturn(!fShared || !pszSearchPath, NULL);
+    Assert(!RTPathHavePath(pszFile));
+
+    /*
+     * If there is a path, search it.
+     */
+    if (   pszSearchPath
+        && *pszSearchPath)
+    {
+        /* Check the filename length. */
+        size_t const    cchFile = strlen(pszFile);
+        if (cchFile >= sizeof(szPath))
+            return NULL;
+
+        /*
+         * Walk the search path.
+         */
+        const char *psz = pszSearchPath;
+        while (*psz)
+        {
+            /* Skip leading blanks - no directories with leading spaces, thank you. */
+            while (RT_C_IS_BLANK(*psz))
+                psz++;
+
+            /* Find the end of this element. */
+            const char *pszNext;
+            const char *pszEnd = strchr(psz, ';');
+            if (!pszEnd)
+                pszEnd = pszNext = strchr(psz, '\0');
+            else
+                pszNext = pszEnd + 1;
+            if (pszEnd != psz)
+            {
+                rc = RTPathJoinEx(szPath, sizeof(szPath), psz, pszEnd - psz, pszFile, cchFile);
+                if (RT_SUCCESS(rc))
+                {
+                    if (RTFileExists(szPath))
+                    {
+                        size_t cchPath = strlen(szPath) + 1;
+                        char *pszRet = (char *)RTMemTmpAlloc(cchPath);
+                        if (pszRet)
+                            memcpy(pszRet, szPath, cchPath);
+                        return pszRet;
+                    }
+                }
+            }
+
+            /* advance */
+            psz = pszNext;
+        }
+    }
+
+    /*
+     * Use the default location.
+     */
+    rc = fShared
+       ? RTPathSharedLibs(    szPath, sizeof(szPath))
+       : RTPathAppPrivateArch(szPath, sizeof(szPath));
+    if (!RT_SUCCESS(rc))
+    {
+        AssertMsgFailed(("RTPath[SharedLibs|AppPrivateArch](,%d) failed rc=%d!\n", sizeof(szPath), rc));
+        return NULL;
+    }
+
+    return pdmR3FileConstruct(szPath, pszFile, pszDefaultExt);
+}
+
+
+/** @internal */
+typedef struct QMFEIPARG
+{
+    RTINTPTR    uPC;
+
+    char       *pszNearSym1;
+    size_t      cchNearSym1;
+    RTINTPTR    offNearSym1;
+
+    char       *pszNearSym2;
+    size_t      cchNearSym2;
+    RTINTPTR    offNearSym2;
+} QMFEIPARG, *PQMFEIPARG;
+
+
+/**
+ * Enumeration callback function used by RTLdrEnumSymbols().
+ *
+ * @returns VBox status code. Failure will stop the enumeration.
+ * @param   hLdrMod         The loader module handle.
+ * @param   pszSymbol       Symbol name. NULL if ordinal only.
+ * @param   uSymbol         Symbol ordinal, ~0 if not used.
+ * @param   Value           Symbol value.
+ * @param   pvUser          The user argument specified to RTLdrEnumSymbols().
+ */
+static DECLCALLBACK(int) pdmR3QueryModFromEIPEnumSymbols(RTLDRMOD hLdrMod, const char *pszSymbol, unsigned uSymbol,
+                                                         RTUINTPTR Value, void *pvUser)
+{
+    PQMFEIPARG pArgs = (PQMFEIPARG)pvUser;
+    NOREF(hLdrMod);
+
+    RTINTPTR off = Value - pArgs->uPC;
+    if (off <= 0)   /* near1 is before or at same location. */
+    {
+        if (off > pArgs->offNearSym1)
+        {
+            pArgs->offNearSym1 = off;
+            if (pArgs->pszNearSym1 && pArgs->cchNearSym1)
+            {
+                *pArgs->pszNearSym1 = '\0';
+                if (pszSymbol)
+                    strncat(pArgs->pszNearSym1, pszSymbol, pArgs->cchNearSym1);
+                else
+                {
+                    char szOrd[32];
+                    RTStrPrintf(szOrd, sizeof(szOrd), "#%#x", uSymbol);
+                    strncat(pArgs->pszNearSym1, szOrd, pArgs->cchNearSym1);
+                }
+            }
+        }
+    }
+    else            /* near2 is after */
+    {
+        if (off < pArgs->offNearSym2)
+        {
+            pArgs->offNearSym2 = off;
+            if (pArgs->pszNearSym2 && pArgs->cchNearSym2)
+            {
+                *pArgs->pszNearSym2 = '\0';
+                if (pszSymbol)
+                    strncat(pArgs->pszNearSym2, pszSymbol, pArgs->cchNearSym2);
+                else
+                {
+                    char szOrd[32];
+                    RTStrPrintf(szOrd, sizeof(szOrd), "#%#x", uSymbol);
+                    strncat(pArgs->pszNearSym2, szOrd, pArgs->cchNearSym2);
+                }
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal worker for PDMR3LdrQueryRCModFromPC and PDMR3LdrQueryR0ModFromPC.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   uPC         The program counter (eip/rip) to locate the module for.
+ * @param   enmType     The module type.
+ * @param   pszModName  Where to store the module name.
+ * @param   cchModName  Size of the module name buffer.
+ * @param   pMod        Base address of the module.
+ * @param   pszNearSym1 Name of the closes symbol from below.
+ * @param   cchNearSym1 Size of the buffer pointed to by pszNearSym1.
+ * @param   pNearSym1   The address of pszNearSym1.
+ * @param   pszNearSym2 Name of the closes symbol from below.
+ * @param   cchNearSym2 Size of the buffer pointed to by pszNearSym2.
+ * @param   pNearSym2   The address of pszNearSym2.
+ */
+static int pdmR3LdrQueryModFromPC(PVM pVM, RTUINTPTR uPC, PDMMODTYPE enmType,
+                                  char *pszModName,  size_t cchModName,  PRTUINTPTR pMod,
+                                  char *pszNearSym1, size_t cchNearSym1, PRTUINTPTR pNearSym1,
+                                  char *pszNearSym2, size_t cchNearSym2, PRTUINTPTR pNearSym2)
+{
+    PUVM    pUVM = pVM->pUVM;
+    int     rc   = VERR_MODULE_NOT_FOUND;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pCur= pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+    {
+        if (pCur->eType != enmType)
+            continue;
+
+        /* The following RTLdrOpen call is a dirty hack to get ring-0 module information. */
+        RTLDRMOD hLdrMod = pCur->hLdrMod;
+        if (hLdrMod == NIL_RTLDRMOD && uPC >= pCur->ImageBase)
+        {
+            int rc2 = RTLdrOpen(pCur->szFilename, 0 /*fFlags*/, RTLDRARCH_HOST, &hLdrMod);
+            if (RT_FAILURE(rc2))
+                hLdrMod = NIL_RTLDRMOD;
+        }
+
+        if (   hLdrMod != NIL_RTLDRMOD
+            && uPC - pCur->ImageBase < RTLdrSize(hLdrMod))
+        {
+            if (pMod)
+                *pMod = pCur->ImageBase;
+            if (pszModName && cchModName)
+            {
+                *pszModName = '\0';
+                strncat(pszModName, pCur->szName, cchModName);
+            }
+            if (pNearSym1)   *pNearSym1   = 0;
+            if (pNearSym2)   *pNearSym2   = 0;
+            if (pszNearSym1) *pszNearSym1 = '\0';
+            if (pszNearSym2) *pszNearSym2 = '\0';
+
+            /*
+             * Locate the nearest symbols.
+             */
+            QMFEIPARG   Args;
+            Args.uPC         = uPC;
+            Args.pszNearSym1 = pszNearSym1;
+            Args.cchNearSym1 = cchNearSym1;
+            Args.offNearSym1 = RTINTPTR_MIN;
+            Args.pszNearSym2 = pszNearSym2;
+            Args.cchNearSym2 = cchNearSym2;
+            Args.offNearSym2 = RTINTPTR_MAX;
+
+            rc = RTLdrEnumSymbols(hLdrMod, RTLDR_ENUM_SYMBOL_FLAGS_ALL, pCur->pvBits, pCur->ImageBase,
+                                  pdmR3QueryModFromEIPEnumSymbols, &Args);
+            if (pNearSym1 && Args.offNearSym1 != RTINTPTR_MIN)
+                *pNearSym1 = Args.offNearSym1 + uPC;
+            if (pNearSym2 && Args.offNearSym2 != RTINTPTR_MAX)
+                *pNearSym2 = Args.offNearSym2 + uPC;
+
+            rc = VINF_SUCCESS;
+        }
+
+        if (hLdrMod != pCur->hLdrMod && hLdrMod != NIL_RTLDRMOD)
+            RTLdrClose(hLdrMod);
+
+        if (RT_SUCCESS(rc))
+            break;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Queries raw-mode context module information from an PC (eip/rip).
+ *
+ * This is typically used to locate a crash address.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   uPC         The program counter (eip/rip) to locate the module for.
+ * @param   pszModName  Where to store the module name.
+ * @param   cchModName  Size of the module name buffer.
+ * @param   pMod        Base address of the module.
+ * @param   pszNearSym1 Name of the closes symbol from below.
+ * @param   cchNearSym1 Size of the buffer pointed to by pszNearSym1.
+ * @param   pNearSym1   The address of pszNearSym1.
+ * @param   pszNearSym2 Name of the closes symbol from below.
+ * @param   cchNearSym2 Size of the buffer pointed to by pszNearSym2.
+ * @param   pNearSym2   The address of pszNearSym2.
+ */
+VMMR3_INT_DECL(int) PDMR3LdrQueryRCModFromPC(PVM pVM, RTRCPTR uPC,
+                                             char *pszModName,  size_t cchModName,  PRTRCPTR pMod,
+                                             char *pszNearSym1, size_t cchNearSym1, PRTRCPTR pNearSym1,
+                                             char *pszNearSym2, size_t cchNearSym2, PRTRCPTR pNearSym2)
+{
+    RTUINTPTR AddrMod   = 0;
+    RTUINTPTR AddrNear1 = 0;
+    RTUINTPTR AddrNear2 = 0;
+    int rc = pdmR3LdrQueryModFromPC(pVM, uPC, PDMMOD_TYPE_RC,
+                                    pszModName,  cchModName,  &AddrMod,
+                                    pszNearSym1, cchNearSym1, &AddrNear1,
+                                    pszNearSym2, cchNearSym2, &AddrNear2);
+    if (RT_SUCCESS(rc))
+    {
+        if (pMod)
+            *pMod      = (RTRCPTR)AddrMod;
+        if (pNearSym1)
+            *pNearSym1 = (RTRCPTR)AddrNear1;
+        if (pNearSym2)
+            *pNearSym2 = (RTRCPTR)AddrNear2;
+    }
+    return rc;
+}
+
+
+/**
+ * Queries ring-0 context module information from an PC (eip/rip).
+ *
+ * This is typically used to locate a crash address.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   uPC         The program counter (eip/rip) to locate the module for.
+ * @param   pszModName  Where to store the module name.
+ * @param   cchModName  Size of the module name buffer.
+ * @param   pMod        Base address of the module.
+ * @param   pszNearSym1 Name of the closes symbol from below.
+ * @param   cchNearSym1 Size of the buffer pointed to by pszNearSym1.
+ * @param   pNearSym1   The address of pszNearSym1.
+ * @param   pszNearSym2 Name of the closes symbol from below.
+ * @param   cchNearSym2 Size of the buffer pointed to by pszNearSym2. Optional.
+ * @param   pNearSym2   The address of pszNearSym2. Optional.
+ */
+VMMR3_INT_DECL(int) PDMR3LdrQueryR0ModFromPC(PVM pVM, RTR0PTR uPC,
+                                             char *pszModName,  size_t cchModName,  PRTR0PTR pMod,
+                                             char *pszNearSym1, size_t cchNearSym1, PRTR0PTR pNearSym1,
+                                             char *pszNearSym2, size_t cchNearSym2, PRTR0PTR pNearSym2)
+{
+    RTUINTPTR AddrMod   = 0;
+    RTUINTPTR AddrNear1 = 0;
+    RTUINTPTR AddrNear2 = 0;
+    int rc = pdmR3LdrQueryModFromPC(pVM, uPC, PDMMOD_TYPE_R0,
+                                    pszModName,  cchModName,  &AddrMod,
+                                    pszNearSym1, cchNearSym1, &AddrNear1,
+                                    pszNearSym2, cchNearSym2, &AddrNear2);
+    if (RT_SUCCESS(rc))
+    {
+        if (pMod)
+            *pMod      = (RTR0PTR)AddrMod;
+        if (pNearSym1)
+            *pNearSym1 = (RTR0PTR)AddrNear1;
+        if (pNearSym2)
+            *pNearSym2 = (RTR0PTR)AddrNear2;
+    }
+    return rc;
+}
+
+
+/**
+ * Enumerate all PDM modules.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pfnCallback     Function to call back for each of the modules.
+ * @param   pvArg           User argument.
+ */
+VMMR3DECL(int)  PDMR3LdrEnumModules(PVM pVM, PFNPDMR3ENUM pfnCallback, void *pvArg)
+{
+    PUVM pUVM = pVM->pUVM;
+    int  rc   = VINF_SUCCESS;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+    {
+        rc = pfnCallback(pVM,
+                         pCur->szFilename,
+                         pCur->szName,
+                         pCur->ImageBase,
+                         pCur->eType == PDMMOD_TYPE_RC ? RTLdrSize(pCur->hLdrMod) : 0,
+                           pCur->eType == PDMMOD_TYPE_RC ? PDMLDRCTX_RAW_MODE
+                         : pCur->eType == PDMMOD_TYPE_R0 ? PDMLDRCTX_RING_0
+                         : pCur->eType == PDMMOD_TYPE_R3 ? PDMLDRCTX_RING_3
+                         :                                 PDMLDRCTX_INVALID,
+                         pvArg);
+        if (RT_FAILURE(rc))
+            break;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Locates a module.
+ *
+ * @returns Pointer to the module if found.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszModule       The module name.
+ * @param   enmType         The module type.
+ * @param   fLazy           Lazy loading the module if set.
+ * @param   pszSearchPath   Search path for use when lazy loading.
+ */
+static PPDMMOD pdmR3LdrFindModule(PUVM pUVM, const char *pszModule, PDMMODTYPE enmType,
+                                  bool fLazy, const char *pszSearchPath)
+{
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+        if (    pModule->eType == enmType
+            &&  !strcmp(pModule->szName, pszModule))
+        {
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            return pModule;
+        }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    if (fLazy)
+    {
+        switch (enmType)
+        {
+#ifdef VBOX_WITH_RAW_MODE_KEEP
+            case PDMMOD_TYPE_RC:
+            {
+                char *pszFilename = pdmR3FileRC(pszModule, pszSearchPath);
+                if (pszFilename)
+                {
+                    int rc = PDMR3LdrLoadRC(pUVM->pVM, pszFilename, pszModule);
+                    RTMemTmpFree(pszFilename);
+                    if (RT_SUCCESS(rc))
+                        return pdmR3LdrFindModule(pUVM, pszModule, enmType, false, NULL);
+                }
+                break;
+            }
+#endif
+
+            case PDMMOD_TYPE_R0:
+            {
+                int rc = pdmR3LoadR0U(pUVM, NULL, pszModule, pszSearchPath);
+                if (RT_SUCCESS(rc))
+                    return pdmR3LdrFindModule(pUVM, pszModule, enmType, false, NULL);
+                break;
+            }
+
+            default:
+                AssertFailed();
+        }
+    }
+    return NULL;
+}
+
+
+/**
+ * Resolves a ring-0 or raw-mode context interface.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pvInterface     Pointer to the interface structure.  The symbol list
+ *                          describes the layout.
+ * @param   cbInterface     The size of the structure pvInterface is pointing
+ *                          to.  For bounds checking.
+ * @param   pszModule       The module name.  If NULL we assume it's the default
+ *                          R0 or RC module (@a fRing0OrRC).  We'll attempt to
+ *                          load the module if it isn't found in the module
+ *                          list.
+ * @param   pszSearchPath   The module search path.  If NULL, search the
+ *                          architecture dependent install directory.
+ * @param   pszSymPrefix    What to prefix the symbols in the list with.  The
+ *                          idea is that you define a list that goes with an
+ *                          interface (INTERFACE_SYM_LIST) and reuse it with
+ *                          each implementation.
+ * @param   pszSymList      The symbol list for the interface.  This is a
+ *                          semi-colon separated list of symbol base names.  As
+ *                          mentioned above, each is prefixed with @a
+ *                          pszSymPrefix before resolving.  There are a couple
+ *                          of special symbol names that will cause us to skip
+ *                          ahead a little bit:
+ *                              - U8:whatever,
+ *                              - U16:whatever,
+ *                              - U32:whatever,
+ *                              - U64:whatever,
+ *                              - RCPTR:whatever,
+ *                              - R3PTR:whatever,
+ *                              - R0PTR:whatever,
+ *                              - GCPHYS:whatever,
+ *                              - HCPHYS:whatever.
+ * @param   fRing0          Set if it's a ring-0 context interface, clear if
+ *                          it's raw-mode context interface.
+ */
+VMMR3_INT_DECL(int) PDMR3LdrGetInterfaceSymbols(PVM pVM, void *pvInterface, size_t cbInterface,
+                                                const char *pszModule, const char *pszSearchPath,
+                                                const char *pszSymPrefix, const char *pszSymList,
+                                                bool fRing0)
+{
+    bool const fNullRun = !fRing0 && !VM_IS_RAW_MODE_ENABLED(pVM);
+
+    /*
+     * Find the module.
+     */
+    int     rc      = VINF_SUCCESS;
+    PPDMMOD pModule = NULL;
+    if (!fNullRun)
+        pModule = pdmR3LdrFindModule(pVM->pUVM,
+                                     pszModule ? pszModule : fRing0 ? "VMMR0.r0" : "VMMRC.rc",
+                                     fRing0 ? PDMMOD_TYPE_R0 : PDMMOD_TYPE_RC,
+                                     true /*fLazy*/, pszSearchPath);
+    if (pModule || fNullRun)
+    {
+        /* Prep the symbol name. */
+        char            szSymbol[256];
+        size_t const    cchSymPrefix = strlen(pszSymPrefix);
+        AssertReturn(cchSymPrefix + 5 < sizeof(szSymbol), VERR_SYMBOL_NOT_FOUND);
+        memcpy(szSymbol, pszSymPrefix, cchSymPrefix);
+
+        /*
+         * Iterate the symbol list.
+         */
+        uint32_t        offInterface = 0;
+        const char     *pszCur       = pszSymList;
+        while (pszCur)
+        {
+            /*
+             * Find the end of the current symbol name.
+             */
+            size_t      cchSym;
+            const char *pszNext = strchr(pszCur, ';');
+            if (pszNext)
+            {
+                cchSym = pszNext - pszCur;
+                pszNext++;
+            }
+            else
+                cchSym = strlen(pszCur);
+            AssertBreakStmt(cchSym > 0, rc = VERR_INVALID_PARAMETER);
+
+            /* Is it a skip instruction? */
+            const char *pszColon = (const char *)memchr(pszCur, ':', cchSym);
+            if (pszColon)
+            {
+                /*
+                 * String switch on the instruction and execute it, checking
+                 * that we didn't overshoot the interface structure.
+                 */
+#define IS_SKIP_INSTR(szInstr) \
+                (   cchSkip == sizeof(szInstr) - 1 \
+                 && !memcmp(pszCur, szInstr, sizeof(szInstr) - 1) )
+
+                size_t const cchSkip = pszColon - pszCur;
+                if (IS_SKIP_INSTR("U8"))
+                    offInterface += sizeof(uint8_t);
+                else if (IS_SKIP_INSTR("U16"))
+                    offInterface += sizeof(uint16_t);
+                else if (IS_SKIP_INSTR("U32"))
+                    offInterface += sizeof(uint32_t);
+                else if (IS_SKIP_INSTR("U64"))
+                    offInterface += sizeof(uint64_t);
+                else if (IS_SKIP_INSTR("RCPTR"))
+                    offInterface += sizeof(RTRCPTR);
+                else if (IS_SKIP_INSTR("R3PTR"))
+                    offInterface += sizeof(RTR3PTR);
+                else if (IS_SKIP_INSTR("R0PTR"))
+                    offInterface += sizeof(RTR0PTR);
+                else if (IS_SKIP_INSTR("HCPHYS"))
+                    offInterface += sizeof(RTHCPHYS);
+                else if (IS_SKIP_INSTR("GCPHYS"))
+                    offInterface += sizeof(RTGCPHYS);
+                else
+                    AssertMsgFailedBreakStmt(("Invalid skip instruction %.*s (prefix=%s)\n", cchSym, pszCur, pszSymPrefix),
+                                             rc = VERR_INVALID_PARAMETER);
+                AssertMsgBreakStmt(offInterface <= cbInterface,
+                                   ("off=%#x cb=%#x (sym=%.*s prefix=%s)\n", offInterface, cbInterface, cchSym, pszCur, pszSymPrefix),
+                                   rc = VERR_BUFFER_OVERFLOW);
+#undef IS_SKIP_INSTR
+            }
+            else
+            {
+                /*
+                 * Construct the symbol name, get its value, store it and
+                 * advance the interface cursor.
+                 */
+                AssertReturn(cchSymPrefix + cchSym < sizeof(szSymbol), VERR_SYMBOL_NOT_FOUND);
+                memcpy(&szSymbol[cchSymPrefix], pszCur, cchSym);
+                szSymbol[cchSymPrefix + cchSym] = '\0';
+
+                if (fRing0)
+                {
+                    void *pvValue = NULL;
+                    if (!fNullRun)
+                    {
+                        rc = SUPR3GetSymbolR0((void *)(RTR0PTR)pModule->ImageBase, szSymbol, &pvValue);
+                        AssertMsgRCBreak(rc, ("Couldn't find symbol '%s' in module '%s'\n", szSymbol, pModule->szName));
+                    }
+
+                    PRTR0PTR pValue = (PRTR0PTR)((uintptr_t)pvInterface + offInterface);
+                    AssertMsgBreakStmt(offInterface + sizeof(*pValue) <= cbInterface,
+                                       ("off=%#x cb=%#x sym=%s\n", offInterface, cbInterface, szSymbol),
+                                       rc = VERR_BUFFER_OVERFLOW);
+                    *pValue = (RTR0PTR)pvValue;
+                    Assert((void *)*pValue == pvValue);
+                    offInterface += sizeof(*pValue);
+                }
+                else
+                {
+                    RTUINTPTR Value = 0;
+                    if (!fNullRun)
+                    {
+                        rc = RTLdrGetSymbolEx(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, UINT32_MAX, szSymbol, &Value);
+                        AssertMsgRCBreak(rc, ("Couldn't find symbol '%s' in module '%s'\n", szSymbol, pModule->szName));
+                    }
+
+                    PRTRCPTR pValue = (PRTRCPTR)((uintptr_t)pvInterface + offInterface);
+                    AssertMsgBreakStmt(offInterface + sizeof(*pValue) <= cbInterface,
+                                       ("off=%#x cb=%#x sym=%s\n", offInterface, cbInterface, szSymbol),
+                                       rc = VERR_BUFFER_OVERFLOW);
+                    *pValue = (RTRCPTR)Value;
+                    Assert(*pValue == Value);
+                    offInterface += sizeof(*pValue);
+                }
+            }
+
+            /* advance */
+            pszCur = pszNext;
+        }
+
+    }
+    else
+        rc = VERR_MODULE_NOT_FOUND;
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/PDMNetShaper.cpp b/src/VBox/VMM/VMMR3/PDMNetShaper.cpp
new file mode 100644
index 00000000..6e29eacd
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMNetShaper.cpp
@@ -0,0 +1,551 @@
+/* $Id: PDMNetShaper.cpp $ */
+/** @file
+ * PDM Network Shaper - Limit network traffic according to bandwidth group settings.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_NET_SHAPER
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/mem.h>
+#include <iprt/critsect.h>
+#include <iprt/tcp.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+
+#include <VBox/vmm/pdmnetshaper.h>
+#include "PDMNetShaperInternal.h"
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+
+/**
+ * Network shaper data. One instance per VM.
+ */
+typedef struct PDMNETSHAPER
+{
+    /** Pointer to the VM. */
+    PVM                      pVM;
+    /** Critical section protecting all members below. */
+    RTCRITSECT               Lock;
+    /** Pending TX thread. */
+    PPDMTHREAD               pTxThread;
+    /** Pointer to the first bandwidth group. */
+    PPDMNSBWGROUP            pBwGroupsHead;
+} PDMNETSHAPER;
+
+
+/** Takes the shaper lock (asserts but doesn't return or anything on
+ *  failure). */
+#define LOCK_NETSHAPER(a_pShaper) do { int rcShaper = RTCritSectEnter(&(a_pShaper)->Lock); AssertRC(rcShaper); } while (0)
+
+/** Takes the shaper lock, returns + asserts on failure. */
+#define LOCK_NETSHAPER_RETURN(a_pShaper) \
+    do { int rcShaper = RTCritSectEnter(&(a_pShaper)->Lock); AssertRCReturn(rcShaper, rcShaper); } while (0)
+
+/** Releases the shaper lock (asserts on failure). */
+#define UNLOCK_NETSHAPER(a_pShaper) do { int rcShaper = RTCritSectLeave(&(a_pShaper)->Lock); AssertRC(rcShaper); } while (0)
+
+
+
+
+static PPDMNSBWGROUP pdmNsBwGroupFindById(PPDMNETSHAPER pShaper, const char *pszId)
+{
+    PPDMNSBWGROUP pBwGroup = NULL;
+
+    if (RT_VALID_PTR(pszId))
+    {
+        LOCK_NETSHAPER(pShaper);
+
+        pBwGroup = pShaper->pBwGroupsHead;
+        while (   pBwGroup
+               && RTStrCmp(pBwGroup->pszNameR3, pszId))
+            pBwGroup = pBwGroup->pNextR3;
+
+        UNLOCK_NETSHAPER(pShaper);
+    }
+
+    return pBwGroup;
+}
+
+
+static void pdmNsBwGroupLink(PPDMNSBWGROUP pBwGroup)
+{
+    PPDMNETSHAPER pShaper = pBwGroup->pShaperR3;
+    LOCK_NETSHAPER(pShaper);
+
+    pBwGroup->pNextR3 = pShaper->pBwGroupsHead;
+    pShaper->pBwGroupsHead = pBwGroup;
+
+    UNLOCK_NETSHAPER(pShaper);
+}
+
+
+#if 0
+static void pdmNsBwGroupUnlink(PPDMNSBWGROUP pBwGroup)
+{
+    PPDMNETSHAPER pShaper = pBwGroup->pShaper;
+    LOCK_NETSHAPER(pShaper);
+
+    if (pBwGroup == pShaper->pBwGroupsHead)
+        pShaper->pBwGroupsHead = pBwGroup->pNext;
+    else
+    {
+        PPDMNSBWGROUP pPrev = pShaper->pBwGroupsHead;
+        while (   pPrev
+               && pPrev->pNext != pBwGroup)
+            pPrev = pPrev->pNext;
+
+        AssertPtr(pPrev);
+        pPrev->pNext = pBwGroup->pNext;
+    }
+
+    UNLOCK_NETSHAPER(pShaper);
+}
+#endif
+
+
+static void pdmNsBwGroupSetLimit(PPDMNSBWGROUP pBwGroup, uint64_t cbPerSecMax)
+{
+    pBwGroup->cbPerSecMax = cbPerSecMax;
+    pBwGroup->cbBucket    = RT_MAX(PDM_NETSHAPER_MIN_BUCKET_SIZE, cbPerSecMax * PDM_NETSHAPER_MAX_LATENCY / 1000);
+    LogFlow(("pdmNsBwGroupSetLimit: New rate limit is %llu bytes per second, adjusted bucket size to %u bytes\n",
+             pBwGroup->cbPerSecMax, pBwGroup->cbBucket));
+}
+
+
+static int pdmNsBwGroupCreate(PPDMNETSHAPER pShaper, const char *pszBwGroup, uint64_t cbPerSecMax)
+{
+    LogFlow(("pdmNsBwGroupCreate: pShaper=%#p pszBwGroup=%#p{%s} cbPerSecMax=%llu\n", pShaper, pszBwGroup, pszBwGroup, cbPerSecMax));
+
+    AssertPtrReturn(pShaper, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszBwGroup, VERR_INVALID_POINTER);
+    AssertReturn(*pszBwGroup != '\0', VERR_INVALID_PARAMETER);
+
+    int         rc;
+    PPDMNSBWGROUP pBwGroup = pdmNsBwGroupFindById(pShaper, pszBwGroup);
+    if (!pBwGroup)
+    {
+        rc = MMHyperAlloc(pShaper->pVM, sizeof(PDMNSBWGROUP), 64,
+                          MM_TAG_PDM_NET_SHAPER, (void **)&pBwGroup);
+        if (RT_SUCCESS(rc))
+        {
+            rc = PDMR3CritSectInit(pShaper->pVM, &pBwGroup->Lock, RT_SRC_POS, "BWGRP-%s", pszBwGroup);
+            if (RT_SUCCESS(rc))
+            {
+                pBwGroup->pszNameR3 = MMR3HeapStrDup(pShaper->pVM, MM_TAG_PDM_NET_SHAPER, pszBwGroup);
+                if (pBwGroup->pszNameR3)
+                {
+                    pBwGroup->pShaperR3             = pShaper;
+                    pBwGroup->cRefs                 = 0;
+
+                    pdmNsBwGroupSetLimit(pBwGroup, cbPerSecMax);
+
+                    pBwGroup->cbTokensLast          = pBwGroup->cbBucket;
+                    pBwGroup->tsUpdatedLast         = RTTimeSystemNanoTS();
+
+                    LogFlowFunc(("pszBwGroup={%s} cbBucket=%u\n",
+                                 pszBwGroup, pBwGroup->cbBucket));
+                    pdmNsBwGroupLink(pBwGroup);
+                    return VINF_SUCCESS;
+                }
+                PDMR3CritSectDelete(&pBwGroup->Lock);
+            }
+            MMHyperFree(pShaper->pVM, pBwGroup);
+        }
+        else
+            rc = VERR_NO_MEMORY;
+    }
+    else
+        rc = VERR_ALREADY_EXISTS;
+
+    LogFlowFunc(("returns rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+static void pdmNsBwGroupTerminate(PPDMNSBWGROUP pBwGroup)
+{
+    Assert(pBwGroup->cRefs == 0);
+    if (PDMCritSectIsInitialized(&pBwGroup->Lock))
+        PDMR3CritSectDelete(&pBwGroup->Lock);
+}
+
+
+DECLINLINE(void) pdmNsBwGroupRef(PPDMNSBWGROUP pBwGroup)
+{
+    ASMAtomicIncU32(&pBwGroup->cRefs);
+}
+
+
+DECLINLINE(void) pdmNsBwGroupUnref(PPDMNSBWGROUP pBwGroup)
+{
+    Assert(pBwGroup->cRefs > 0);
+    ASMAtomicDecU32(&pBwGroup->cRefs);
+}
+
+
+static void pdmNsBwGroupXmitPending(PPDMNSBWGROUP pBwGroup)
+{
+    /*
+     * We don't need to hold the bandwidth group lock to iterate over the list
+     * of filters since the filters are removed while the shaper lock is being
+     * held.
+     */
+    AssertPtr(pBwGroup);
+    AssertPtr(pBwGroup->pShaperR3);
+    Assert(RTCritSectIsOwner(&pBwGroup->pShaperR3->Lock));
+    //LOCK_NETSHAPER(pShaper);
+
+    /* Check if the group is disabled. */
+    if (pBwGroup->cbPerSecMax == 0)
+        return;
+
+    PPDMNSFILTER pFilter = pBwGroup->pFiltersHeadR3;
+    while (pFilter)
+    {
+        bool fChoked = ASMAtomicXchgBool(&pFilter->fChoked, false);
+        Log3((LOG_FN_FMT ": pFilter=%#p fChoked=%RTbool\n", __PRETTY_FUNCTION__, pFilter, fChoked));
+        if (fChoked && pFilter->pIDrvNetR3)
+        {
+            LogFlowFunc(("Calling pfnXmitPending for pFilter=%#p\n", pFilter));
+            pFilter->pIDrvNetR3->pfnXmitPending(pFilter->pIDrvNetR3);
+        }
+
+        pFilter = pFilter->pNextR3;
+    }
+
+    //UNLOCK_NETSHAPER(pShaper);
+}
+
+
+static void pdmNsFilterLink(PPDMNSFILTER pFilter)
+{
+    PPDMNSBWGROUP pBwGroup = pFilter->pBwGroupR3;
+    int rc = PDMCritSectEnter(&pBwGroup->Lock, VERR_SEM_BUSY); AssertRC(rc);
+
+    pFilter->pNextR3 = pBwGroup->pFiltersHeadR3;
+    pBwGroup->pFiltersHeadR3 = pFilter;
+
+    rc = PDMCritSectLeave(&pBwGroup->Lock); AssertRC(rc);
+}
+
+
+static void pdmNsFilterUnlink(PPDMNSFILTER pFilter)
+{
+    PPDMNSBWGROUP pBwGroup = pFilter->pBwGroupR3;
+    /*
+     * We need to make sure we hold the shaper lock since pdmNsBwGroupXmitPending()
+     * does not hold the bandwidth group lock while iterating over the list
+     * of group's filters.
+     */
+    AssertPtr(pBwGroup);
+    AssertPtr(pBwGroup->pShaperR3);
+    Assert(RTCritSectIsOwner(&pBwGroup->pShaperR3->Lock));
+    int rc = PDMCritSectEnter(&pBwGroup->Lock, VERR_SEM_BUSY); AssertRC(rc);
+
+    if (pFilter == pBwGroup->pFiltersHeadR3)
+        pBwGroup->pFiltersHeadR3 = pFilter->pNextR3;
+    else
+    {
+        PPDMNSFILTER pPrev = pBwGroup->pFiltersHeadR3;
+        while (   pPrev
+               && pPrev->pNextR3 != pFilter)
+            pPrev = pPrev->pNextR3;
+
+        AssertPtr(pPrev);
+        pPrev->pNextR3 = pFilter->pNextR3;
+    }
+
+    rc = PDMCritSectLeave(&pBwGroup->Lock); AssertRC(rc);
+}
+
+
+/**
+ * Attach network filter driver from bandwidth group.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM structure.
+ * @param   pDrvIns     The driver instance.
+ * @param   pszBwGroup  Name of the bandwidth group to attach to.
+ * @param   pFilter     Pointer to the filter we attach.
+ */
+VMMR3_INT_DECL(int) PDMR3NsAttach(PUVM pUVM, PPDMDRVINS pDrvIns, const char *pszBwGroup, PPDMNSFILTER pFilter)
+{
+    VM_ASSERT_EMT(pUVM->pVM);
+    AssertPtrReturn(pFilter, VERR_INVALID_POINTER);
+    AssertReturn(pFilter->pBwGroupR3 == NULL, VERR_ALREADY_EXISTS);
+    RT_NOREF_PV(pDrvIns);
+
+    PPDMNETSHAPER pShaper = pUVM->pdm.s.pNetShaper;
+    LOCK_NETSHAPER_RETURN(pShaper);
+
+    int             rc          = VINF_SUCCESS;
+    PPDMNSBWGROUP   pBwGroupNew = NULL;
+    if (pszBwGroup)
+    {
+        pBwGroupNew = pdmNsBwGroupFindById(pShaper, pszBwGroup);
+        if (pBwGroupNew)
+            pdmNsBwGroupRef(pBwGroupNew);
+        else
+            rc = VERR_NOT_FOUND;
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        PPDMNSBWGROUP pBwGroupOld = ASMAtomicXchgPtrT(&pFilter->pBwGroupR3, pBwGroupNew, PPDMNSBWGROUP);
+        ASMAtomicWritePtr(&pFilter->pBwGroupR0, MMHyperR3ToR0(pUVM->pVM, pBwGroupNew));
+        if (pBwGroupOld)
+            pdmNsBwGroupUnref(pBwGroupOld);
+        pdmNsFilterLink(pFilter);
+    }
+
+    UNLOCK_NETSHAPER(pShaper);
+    return rc;
+}
+
+
+/**
+ * Detach network filter driver from bandwidth group.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pDrvIns         The driver instance.
+ * @param   pFilter         Pointer to the filter we detach.
+ */
+VMMR3_INT_DECL(int) PDMR3NsDetach(PUVM pUVM, PPDMDRVINS pDrvIns, PPDMNSFILTER pFilter)
+{
+    RT_NOREF_PV(pDrvIns);
+    VM_ASSERT_EMT(pUVM->pVM);
+    AssertPtrReturn(pFilter, VERR_INVALID_POINTER);
+
+    /* Now, return quietly if the filter isn't attached since driver/device
+       destructors are called on constructor failure. */
+    if (!pFilter->pBwGroupR3)
+        return VINF_SUCCESS;
+    AssertPtrReturn(pFilter->pBwGroupR3, VERR_INVALID_POINTER);
+
+    PPDMNETSHAPER pShaper = pUVM->pdm.s.pNetShaper;
+    LOCK_NETSHAPER_RETURN(pShaper);
+
+    pdmNsFilterUnlink(pFilter);
+    PPDMNSBWGROUP pBwGroup = ASMAtomicXchgPtrT(&pFilter->pBwGroupR3, NULL, PPDMNSBWGROUP);
+    if (pBwGroup)
+        pdmNsBwGroupUnref(pBwGroup);
+
+    UNLOCK_NETSHAPER(pShaper);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Adjusts the maximum rate for the bandwidth group.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszBwGroup      Name of the bandwidth group to attach to.
+ * @param   cbPerSecMax     Maximum number of bytes per second to be transmitted.
+ */
+VMMR3DECL(int) PDMR3NsBwGroupSetLimit(PUVM pUVM, const char *pszBwGroup, uint64_t cbPerSecMax)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PPDMNETSHAPER pShaper = pUVM->pdm.s.pNetShaper;
+    LOCK_NETSHAPER_RETURN(pShaper);
+
+    int           rc;
+    PPDMNSBWGROUP pBwGroup = pdmNsBwGroupFindById(pShaper, pszBwGroup);
+    if (pBwGroup)
+    {
+        rc = PDMCritSectEnter(&pBwGroup->Lock, VERR_SEM_BUSY); AssertRC(rc);
+        if (RT_SUCCESS(rc))
+        {
+            pdmNsBwGroupSetLimit(pBwGroup, cbPerSecMax);
+
+            /* Drop extra tokens */
+            if (pBwGroup->cbTokensLast > pBwGroup->cbBucket)
+                pBwGroup->cbTokensLast = pBwGroup->cbBucket;
+
+            int rc2 = PDMCritSectLeave(&pBwGroup->Lock); AssertRC(rc2);
+        }
+    }
+    else
+        rc = VERR_NOT_FOUND;
+
+    UNLOCK_NETSHAPER(pShaper);
+    return rc;
+}
+
+
+/**
+ * I/O thread for pending TX.
+ *
+ * @returns VINF_SUCCESS (ignored).
+ * @param   pVM         The cross context VM structure.
+ * @param   pThread     The PDM thread data.
+ */
+static DECLCALLBACK(int) pdmR3NsTxThread(PVM pVM, PPDMTHREAD pThread)
+{
+    RT_NOREF_PV(pVM);
+
+    PPDMNETSHAPER pShaper = (PPDMNETSHAPER)pThread->pvUser;
+    LogFlow(("pdmR3NsTxThread: pShaper=%p\n", pShaper));
+    while (pThread->enmState == PDMTHREADSTATE_RUNNING)
+    {
+        RTThreadSleep(PDM_NETSHAPER_MAX_LATENCY);
+
+        /* Go over all bandwidth groups/filters calling pfnXmitPending */
+        LOCK_NETSHAPER(pShaper);
+        PPDMNSBWGROUP pBwGroup = pShaper->pBwGroupsHead;
+        while (pBwGroup)
+        {
+            pdmNsBwGroupXmitPending(pBwGroup);
+            pBwGroup = pBwGroup->pNextR3;
+        }
+        UNLOCK_NETSHAPER(pShaper);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @copydoc FNPDMTHREADWAKEUPINT
+ */
+static DECLCALLBACK(int) pdmR3NsTxWakeUp(PVM pVM, PPDMTHREAD pThread)
+{
+    RT_NOREF2(pVM, pThread);
+    LogFlow(("pdmR3NsTxWakeUp: pShaper=%p\n", pThread->pvUser));
+    /* Nothing to do */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminate the network shaper.
+ *
+ * @returns VBox error code.
+ * @param   pVM  The cross context VM structure.
+ *
+ * @remarks This method destroys all bandwidth group objects.
+ */
+int pdmR3NetShaperTerm(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+    AssertPtrReturn(pUVM, VERR_INVALID_POINTER);
+    PPDMNETSHAPER pShaper = pUVM->pdm.s.pNetShaper;
+    AssertPtrReturn(pShaper, VERR_INVALID_POINTER);
+
+    /* Destroy the bandwidth managers. */
+    PPDMNSBWGROUP pBwGroup = pShaper->pBwGroupsHead;
+    while (pBwGroup)
+    {
+        PPDMNSBWGROUP pFree = pBwGroup;
+        pBwGroup = pBwGroup->pNextR3;
+        pdmNsBwGroupTerminate(pFree);
+        MMR3HeapFree(pFree->pszNameR3);
+        MMHyperFree(pVM, pFree);
+    }
+
+    RTCritSectDelete(&pShaper->Lock);
+    MMR3HeapFree(pShaper);
+    pUVM->pdm.s.pNetShaper = NULL;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Initialize the network shaper.
+ *
+ * @returns VBox status code
+ * @param   pVM The cross context VM structure.
+ */
+int pdmR3NetShaperInit(PVM pVM)
+{
+    LogFlow(("pdmR3NetShaperInit: pVM=%p\n", pVM));
+    VM_ASSERT_EMT(pVM);
+    PUVM pUVM = pVM->pUVM;
+    AssertMsgReturn(!pUVM->pdm.s.pNetShaper, ("Network shaper was already initialized\n"), VERR_WRONG_ORDER);
+
+    PPDMNETSHAPER pShaper;
+    int rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_NET_SHAPER, sizeof(PDMNETSHAPER), (void **)&pShaper);
+    if (RT_SUCCESS(rc))
+    {
+        PCFGMNODE pCfgNetShaper = CFGMR3GetChild(CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM"), "NetworkShaper");
+
+        pShaper->pVM = pVM;
+        rc = RTCritSectInit(&pShaper->Lock);
+        if (RT_SUCCESS(rc))
+        {
+            /* Create all bandwidth groups. */
+            PCFGMNODE pCfgBwGrp = CFGMR3GetChild(pCfgNetShaper, "BwGroups");
+            if (pCfgBwGrp)
+            {
+                for (PCFGMNODE pCur = CFGMR3GetFirstChild(pCfgBwGrp); pCur; pCur = CFGMR3GetNextChild(pCur))
+                {
+                    size_t cbName = CFGMR3GetNameLen(pCur) + 1;
+                    char *pszBwGrpId = (char *)RTMemAllocZ(cbName);
+                    if (pszBwGrpId)
+                    {
+                        rc = CFGMR3GetName(pCur, pszBwGrpId, cbName);
+                        if (RT_SUCCESS(rc))
+                        {
+                            uint64_t cbMax;
+                            rc = CFGMR3QueryU64(pCur, "Max", &cbMax);
+                            if (RT_SUCCESS(rc))
+                                rc = pdmNsBwGroupCreate(pShaper, pszBwGrpId, cbMax);
+                        }
+                        RTMemFree(pszBwGrpId);
+                    }
+                    else
+                        rc = VERR_NO_MEMORY;
+                    if (RT_FAILURE(rc))
+                        break;
+                }
+            }
+
+            if (RT_SUCCESS(rc))
+            {
+                rc = PDMR3ThreadCreate(pVM, &pShaper->pTxThread, pShaper, pdmR3NsTxThread, pdmR3NsTxWakeUp,
+                                       0 /*cbStack*/, RTTHREADTYPE_IO, "PDMNsTx");
+                if (RT_SUCCESS(rc))
+                {
+                    pUVM->pdm.s.pNetShaper = pShaper;
+                    return VINF_SUCCESS;
+                }
+            }
+
+            RTCritSectDelete(&pShaper->Lock);
+        }
+
+        MMR3HeapFree(pShaper);
+    }
+
+    LogFlow(("pdmR3NetShaperInit: pVM=%p rc=%Rrc\n", pVM, rc));
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/PDMQueue.cpp b/src/VBox/VMM/VMMR3/PDMQueue.cpp
new file mode 100644
index 00000000..2dbfc70e
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMQueue.cpp
@@ -0,0 +1,877 @@
+/* $Id: PDMQueue.cpp $ */
+/** @file
+ * PDM Queue - Transport data and tasks to EMT and R3.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_QUEUE
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <iprt/errcore.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+DECLINLINE(void)            pdmR3QueueFreeItem(PPDMQUEUE pQueue, PPDMQUEUEITEMCORE pItem);
+static bool                 pdmR3QueueFlush(PPDMQUEUE pQueue);
+static DECLCALLBACK(void)   pdmR3QueueTimer(PVM pVM, PTMTIMER pTimer, void *pvUser);
+
+
+
+/**
+ * Internal worker for the queue creation apis.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   cbItem              Item size.
+ * @param   cItems              Number of items.
+ * @param   cMilliesInterval    Number of milliseconds between polling the queue.
+ *                              If 0 then the emulation thread will be notified whenever an item arrives.
+ * @param   fRZEnabled          Set if the queue will be used from RC/R0 and need to be allocated from the hyper heap.
+ * @param   pszName             The queue name. Unique. Not copied.
+ * @param   ppQueue             Where to store the queue handle.
+ */
+static int pdmR3QueueCreate(PVM pVM, size_t cbItem, uint32_t cItems, uint32_t cMilliesInterval, bool fRZEnabled,
+                            const char *pszName, PPDMQUEUE *ppQueue)
+{
+    PUVM pUVM = pVM->pUVM;
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(cbItem >= sizeof(PDMQUEUEITEMCORE) && cbItem < _1M, ("cbItem=%zu\n", cbItem), VERR_OUT_OF_RANGE);
+    AssertMsgReturn(cItems >= 1 && cItems <= _64K, ("cItems=%u\n", cItems), VERR_OUT_OF_RANGE);
+
+    /*
+     * Align the item size and calculate the structure size.
+     */
+    cbItem = RT_ALIGN(cbItem, sizeof(RTUINTPTR));
+    size_t cb = cbItem * cItems + RT_ALIGN_Z(RT_UOFFSETOF_DYN(PDMQUEUE, aFreeItems[cItems + PDMQUEUE_FREE_SLACK]), 16);
+    PPDMQUEUE pQueue;
+    int rc;
+    if (fRZEnabled)
+        rc = MMHyperAlloc(pVM, cb, 0, MM_TAG_PDM_QUEUE, (void **)&pQueue );
+    else
+        rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_QUEUE, cb, (void **)&pQueue);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Initialize the data fields.
+     */
+    pQueue->pVMR3 = pVM;
+    pQueue->pVMR0 = fRZEnabled ? pVM->pVMR0ForCall : NIL_RTR0PTR;
+    pQueue->pVMRC = fRZEnabled ? pVM->pVMRC : NIL_RTRCPTR;
+    pQueue->pszName = pszName;
+    pQueue->cMilliesInterval = cMilliesInterval;
+    //pQueue->pTimer = NULL;
+    pQueue->cbItem = (uint32_t)cbItem;
+    pQueue->cItems = cItems;
+    //pQueue->pPendingR3 = NULL;
+    //pQueue->pPendingR0 = NULL;
+    //pQueue->pPendingRC = NULL;
+    pQueue->iFreeHead = cItems;
+    //pQueue->iFreeTail = 0;
+    PPDMQUEUEITEMCORE pItem = (PPDMQUEUEITEMCORE)((char *)pQueue + RT_ALIGN_Z(RT_UOFFSETOF_DYN(PDMQUEUE, aFreeItems[cItems + PDMQUEUE_FREE_SLACK]), 16));
+    for (unsigned i = 0; i < cItems; i++, pItem = (PPDMQUEUEITEMCORE)((char *)pItem + cbItem))
+    {
+        pQueue->aFreeItems[i].pItemR3 = pItem;
+        if (fRZEnabled)
+        {
+            pQueue->aFreeItems[i].pItemR0 = MMHyperR3ToR0(pVM, pItem);
+            pQueue->aFreeItems[i].pItemRC = MMHyperR3ToRC(pVM, pItem);
+        }
+    }
+
+    /*
+     * Create timer?
+     */
+    if (cMilliesInterval)
+    {
+        rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, pdmR3QueueTimer, pQueue, "Queue timer", &pQueue->pTimer);
+        if (RT_SUCCESS(rc))
+        {
+            rc = TMTimerSetMillies(pQueue->pTimer, cMilliesInterval);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("TMTimerSetMillies failed rc=%Rrc\n", rc));
+                int rc2 = TMR3TimerDestroy(pQueue->pTimer); AssertRC(rc2);
+            }
+        }
+        else
+            AssertMsgFailed(("TMR3TimerCreateInternal failed rc=%Rrc\n", rc));
+        if (RT_FAILURE(rc))
+        {
+            if (fRZEnabled)
+                MMHyperFree(pVM, pQueue);
+            else
+                MMR3HeapFree(pQueue);
+            return rc;
+        }
+
+        /*
+         * Insert into the queue list for timer driven queues.
+         */
+        pdmLock(pVM);
+        pQueue->pNext = pUVM->pdm.s.pQueuesTimer;
+        pUVM->pdm.s.pQueuesTimer = pQueue;
+        pdmUnlock(pVM);
+    }
+    else
+    {
+        /*
+         * Insert into the queue list for forced action driven queues.
+         * This is a FIFO, so insert at the end.
+         */
+        /** @todo we should add a priority to the queues so we don't have to rely on
+         * the initialization order to deal with problems like @bugref{1605} (pgm/pcnet
+         * deadlock caused by the critsect queue to be last in the chain).
+         * - Update, the critical sections are no longer using queues, so this isn't a real
+         *   problem any longer. The priority might be a nice feature for later though.
+         */
+        pdmLock(pVM);
+        if (!pUVM->pdm.s.pQueuesForced)
+            pUVM->pdm.s.pQueuesForced = pQueue;
+        else
+        {
+            PPDMQUEUE pPrev = pUVM->pdm.s.pQueuesForced;
+            while (pPrev->pNext)
+                pPrev = pPrev->pNext;
+            pPrev->pNext = pQueue;
+        }
+        pdmUnlock(pVM);
+    }
+
+    /*
+     * Register the statistics.
+     */
+    STAMR3RegisterF(pVM, &pQueue->cbItem,               STAMTYPE_U32,     STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,        "Item size.",                       "/PDM/Queue/%s/cbItem",         pQueue->pszName);
+    STAMR3RegisterF(pVM, &pQueue->cItems,               STAMTYPE_U32,     STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,        "Queue size.",                      "/PDM/Queue/%s/cItems",         pQueue->pszName);
+    STAMR3RegisterF(pVM, &pQueue->StatAllocFailures,    STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,   "PDMQueueAlloc failures.",          "/PDM/Queue/%s/AllocFailures",  pQueue->pszName);
+    STAMR3RegisterF(pVM, &pQueue->StatInsert,           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,        "Calls to PDMQueueInsert.",         "/PDM/Queue/%s/Insert",         pQueue->pszName);
+    STAMR3RegisterF(pVM, &pQueue->StatFlush,            STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,        "Calls to pdmR3QueueFlush.",        "/PDM/Queue/%s/Flush",          pQueue->pszName);
+    STAMR3RegisterF(pVM, &pQueue->StatFlushLeftovers,   STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,   "Left over items after flush.",     "/PDM/Queue/%s/FlushLeftovers", pQueue->pszName);
+#ifdef VBOX_WITH_STATISTICS
+    STAMR3RegisterF(pVM, &pQueue->StatFlushPrf,         STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,        "Profiling pdmR3QueueFlush.",       "/PDM/Queue/%s/FlushPrf",       pQueue->pszName);
+    STAMR3RegisterF(pVM, (void *)&pQueue->cStatPending, STAMTYPE_U32,     STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,        "Pending items.",                   "/PDM/Queue/%s/Pending",        pQueue->pszName);
+#endif
+
+    *ppQueue = pQueue;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Create a queue with a device owner.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pDevIns             Device instance.
+ * @param   cbItem              Size a queue item.
+ * @param   cItems              Number of items in the queue.
+ * @param   cMilliesInterval    Number of milliseconds between polling the queue.
+ *                              If 0 then the emulation thread will be notified whenever an item arrives.
+ * @param   pfnCallback         The consumer function.
+ * @param   fRZEnabled          Set if the queue must be usable from RC/R0.
+ * @param   pszName             The queue name. Unique. Not copied.
+ * @param   ppQueue             Where to store the queue handle on success.
+ * @thread  Emulation thread only.
+ */
+VMMR3_INT_DECL(int) PDMR3QueueCreateDevice(PVM pVM, PPDMDEVINS pDevIns, size_t cbItem, uint32_t cItems, uint32_t cMilliesInterval,
+                                           PFNPDMQUEUEDEV pfnCallback, bool fRZEnabled, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    LogFlow(("PDMR3QueueCreateDevice: pDevIns=%p cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p fRZEnabled=%RTbool pszName=%s\n",
+             pDevIns, cbItem, cItems, cMilliesInterval, pfnCallback, fRZEnabled, pszName));
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT0(pVM);
+    if (!pfnCallback)
+    {
+        AssertMsgFailed(("No consumer callback!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Create the queue.
+     */
+    PPDMQUEUE pQueue;
+    int rc = pdmR3QueueCreate(pVM, cbItem, cItems, cMilliesInterval, fRZEnabled, pszName, &pQueue);
+    if (RT_SUCCESS(rc))
+    {
+        pQueue->enmType = PDMQUEUETYPE_DEV;
+        pQueue->u.Dev.pDevIns = pDevIns;
+        pQueue->u.Dev.pfnCallback = pfnCallback;
+
+        *ppQueue = pQueue;
+        Log(("PDM: Created device queue %p; cbItem=%d cItems=%d cMillies=%d pfnCallback=%p pDevIns=%p\n",
+             pQueue, cbItem, cItems, cMilliesInterval, pfnCallback, pDevIns));
+    }
+    return rc;
+}
+
+
+/**
+ * Create a queue with a driver owner.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pDrvIns             Driver instance.
+ * @param   cbItem              Size a queue item.
+ * @param   cItems              Number of items in the queue.
+ * @param   cMilliesInterval    Number of milliseconds between polling the queue.
+ *                              If 0 then the emulation thread will be notified whenever an item arrives.
+ * @param   pfnCallback         The consumer function.
+ * @param   pszName             The queue name. Unique. Not copied.
+ * @param   ppQueue             Where to store the queue handle on success.
+ * @thread  Emulation thread only.
+ */
+VMMR3_INT_DECL(int) PDMR3QueueCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, size_t cbItem, uint32_t cItems, uint32_t cMilliesInterval,
+                                           PFNPDMQUEUEDRV pfnCallback, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    LogFlow(("PDMR3QueueCreateDriver: pDrvIns=%p cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p pszName=%s\n",
+             pDrvIns, cbItem, cItems, cMilliesInterval, pfnCallback, pszName));
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT0(pVM);
+    AssertPtrReturn(pfnCallback, VERR_INVALID_POINTER);
+
+    /*
+     * Create the queue.
+     */
+    PPDMQUEUE pQueue;
+    int rc = pdmR3QueueCreate(pVM, cbItem, cItems, cMilliesInterval, false, pszName, &pQueue);
+    if (RT_SUCCESS(rc))
+    {
+        pQueue->enmType = PDMQUEUETYPE_DRV;
+        pQueue->u.Drv.pDrvIns = pDrvIns;
+        pQueue->u.Drv.pfnCallback = pfnCallback;
+
+        *ppQueue = pQueue;
+        Log(("PDM: Created driver queue %p; cbItem=%d cItems=%d cMillies=%d pfnCallback=%p pDrvIns=%p\n",
+             pQueue, cbItem, cItems, cMilliesInterval, pfnCallback, pDrvIns));
+    }
+    return rc;
+}
+
+
+/**
+ * Create a queue with an internal owner.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   cbItem              Size a queue item.
+ * @param   cItems              Number of items in the queue.
+ * @param   cMilliesInterval    Number of milliseconds between polling the queue.
+ *                              If 0 then the emulation thread will be notified whenever an item arrives.
+ * @param   pfnCallback         The consumer function.
+ * @param   fRZEnabled          Set if the queue must be usable from RC/R0.
+ * @param   pszName             The queue name. Unique. Not copied.
+ * @param   ppQueue             Where to store the queue handle on success.
+ * @thread  Emulation thread only.
+ */
+VMMR3_INT_DECL(int) PDMR3QueueCreateInternal(PVM pVM, size_t cbItem, uint32_t cItems, uint32_t cMilliesInterval,
+                                             PFNPDMQUEUEINT pfnCallback, bool fRZEnabled, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    LogFlow(("PDMR3QueueCreateInternal: cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p fRZEnabled=%RTbool pszName=%s\n",
+             cbItem, cItems, cMilliesInterval, pfnCallback, fRZEnabled, pszName));
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT0(pVM);
+    AssertPtrReturn(pfnCallback, VERR_INVALID_POINTER);
+
+    /*
+     * Create the queue.
+     */
+    PPDMQUEUE pQueue;
+    int rc = pdmR3QueueCreate(pVM, cbItem, cItems, cMilliesInterval, fRZEnabled, pszName, &pQueue);
+    if (RT_SUCCESS(rc))
+    {
+        pQueue->enmType = PDMQUEUETYPE_INTERNAL;
+        pQueue->u.Int.pfnCallback = pfnCallback;
+
+        *ppQueue = pQueue;
+        Log(("PDM: Created internal queue %p; cbItem=%d cItems=%d cMillies=%d pfnCallback=%p\n",
+             pQueue, cbItem, cItems, cMilliesInterval, pfnCallback));
+    }
+    return rc;
+}
+
+
+/**
+ * Create a queue with an external owner.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   cbItem              Size a queue item.
+ * @param   cItems              Number of items in the queue.
+ * @param   cMilliesInterval    Number of milliseconds between polling the queue.
+ *                              If 0 then the emulation thread will be notified whenever an item arrives.
+ * @param   pfnCallback         The consumer function.
+ * @param   pvUser              The user argument to the consumer function.
+ * @param   pszName             The queue name. Unique. Not copied.
+ * @param   ppQueue             Where to store the queue handle on success.
+ * @thread  Emulation thread only.
+ */
+VMMR3_INT_DECL(int) PDMR3QueueCreateExternal(PVM pVM, size_t cbItem, uint32_t cItems, uint32_t cMilliesInterval,
+                                             PFNPDMQUEUEEXT pfnCallback, void *pvUser, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    LogFlow(("PDMR3QueueCreateExternal: cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p pszName=%s\n", cbItem, cItems, cMilliesInterval, pfnCallback, pszName));
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT0(pVM);
+    AssertPtrReturn(pfnCallback, VERR_INVALID_POINTER);
+
+    /*
+     * Create the queue.
+     */
+    PPDMQUEUE pQueue;
+    int rc = pdmR3QueueCreate(pVM, cbItem, cItems, cMilliesInterval, false, pszName, &pQueue);
+    if (RT_SUCCESS(rc))
+    {
+        pQueue->enmType = PDMQUEUETYPE_EXTERNAL;
+        pQueue->u.Ext.pvUser = pvUser;
+        pQueue->u.Ext.pfnCallback = pfnCallback;
+
+        *ppQueue = pQueue;
+        Log(("PDM: Created external queue %p; cbItem=%d cItems=%d cMillies=%d pfnCallback=%p pvUser=%p\n",
+             pQueue, cbItem, cItems, cMilliesInterval, pfnCallback, pvUser));
+    }
+    return rc;
+}
+
+
+/**
+ * Destroy a queue.
+ *
+ * @returns VBox status code.
+ * @param   pQueue      Queue to destroy.
+ * @thread  Emulation thread only.
+ */
+VMMR3_INT_DECL(int) PDMR3QueueDestroy(PPDMQUEUE pQueue)
+{
+    LogFlow(("PDMR3QueueDestroy: pQueue=%p\n", pQueue));
+
+    /*
+     * Validate input.
+     */
+    if (!pQueue)
+        return VERR_INVALID_PARAMETER;
+    Assert(pQueue && pQueue->pVMR3);
+    PVM     pVM  = pQueue->pVMR3;
+    PUVM    pUVM = pVM->pUVM;
+
+    pdmLock(pVM);
+
+    /*
+     * Unlink it.
+     */
+    if (pQueue->pTimer)
+    {
+        if (pUVM->pdm.s.pQueuesTimer != pQueue)
+        {
+            PPDMQUEUE pCur = pUVM->pdm.s.pQueuesTimer;
+            while (pCur)
+            {
+                if (pCur->pNext == pQueue)
+                {
+                    pCur->pNext = pQueue->pNext;
+                    break;
+                }
+                pCur = pCur->pNext;
+            }
+            AssertMsg(pCur, ("Didn't find the queue!\n"));
+        }
+        else
+            pUVM->pdm.s.pQueuesTimer = pQueue->pNext;
+    }
+    else
+    {
+        if (pUVM->pdm.s.pQueuesForced != pQueue)
+        {
+            PPDMQUEUE pCur = pUVM->pdm.s.pQueuesForced;
+            while (pCur)
+            {
+                if (pCur->pNext == pQueue)
+                {
+                    pCur->pNext = pQueue->pNext;
+                    break;
+                }
+                pCur = pCur->pNext;
+            }
+            AssertMsg(pCur, ("Didn't find the queue!\n"));
+        }
+        else
+            pUVM->pdm.s.pQueuesForced = pQueue->pNext;
+    }
+    pQueue->pNext = NULL;
+    pQueue->pVMR3 = NULL;
+    pdmUnlock(pVM);
+
+    /*
+     * Deregister statistics.
+     */
+    STAMR3DeregisterF(pVM->pUVM, "/PDM/Queue/%s/cbItem", pQueue->pszName);
+
+    /*
+     * Destroy the timer and free it.
+     */
+    if (pQueue->pTimer)
+    {
+        TMR3TimerDestroy(pQueue->pTimer);
+        pQueue->pTimer = NULL;
+    }
+    if (pQueue->pVMRC)
+    {
+        pQueue->pVMRC = NIL_RTRCPTR;
+        pQueue->pVMR0 = NIL_RTR0PTR;
+        MMHyperFree(pVM, pQueue);
+    }
+    else
+        MMR3HeapFree(pQueue);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroy a all queues owned by the specified device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     Device instance.
+ * @thread  Emulation thread only.
+ */
+VMMR3_INT_DECL(int) PDMR3QueueDestroyDevice(PVM pVM, PPDMDEVINS pDevIns)
+{
+    LogFlow(("PDMR3QueueDestroyDevice: pDevIns=%p\n", pDevIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pDevIns)
+        return VERR_INVALID_PARAMETER;
+
+    PUVM pUVM = pVM->pUVM;
+    pdmLock(pVM);
+
+    /*
+     * Unlink it.
+     */
+    PPDMQUEUE pQueueNext = pUVM->pdm.s.pQueuesTimer;
+    PPDMQUEUE pQueue     = pUVM->pdm.s.pQueuesForced;
+    do
+    {
+        while (pQueue)
+        {
+            if (    pQueue->enmType == PDMQUEUETYPE_DEV
+                &&  pQueue->u.Dev.pDevIns == pDevIns)
+            {
+                PPDMQUEUE pQueueDestroy = pQueue;
+                pQueue = pQueue->pNext;
+                int rc = PDMR3QueueDestroy(pQueueDestroy);
+                AssertRC(rc);
+            }
+            else
+                pQueue = pQueue->pNext;
+        }
+
+        /* next queue list */
+        pQueue = pQueueNext;
+        pQueueNext = NULL;
+    } while (pQueue);
+
+    pdmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroy a all queues owned by the specified driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDrvIns     Driver instance.
+ * @thread  Emulation thread only.
+ */
+VMMR3_INT_DECL(int) PDMR3QueueDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns)
+{
+    LogFlow(("PDMR3QueueDestroyDriver: pDrvIns=%p\n", pDrvIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pDrvIns)
+        return VERR_INVALID_PARAMETER;
+
+    PUVM pUVM = pVM->pUVM;
+    pdmLock(pVM);
+
+    /*
+     * Unlink it.
+     */
+    PPDMQUEUE pQueueNext = pUVM->pdm.s.pQueuesTimer;
+    PPDMQUEUE pQueue     = pUVM->pdm.s.pQueuesForced;
+    do
+    {
+        while (pQueue)
+        {
+            if (    pQueue->enmType == PDMQUEUETYPE_DRV
+                &&  pQueue->u.Drv.pDrvIns == pDrvIns)
+            {
+                PPDMQUEUE pQueueDestroy = pQueue;
+                pQueue = pQueue->pNext;
+                int rc = PDMR3QueueDestroy(pQueueDestroy);
+                AssertRC(rc);
+            }
+            else
+                pQueue = pQueue->pNext;
+        }
+
+        /* next queue list */
+        pQueue = pQueueNext;
+        pQueueNext = NULL;
+    } while (pQueue);
+
+    pdmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Relocate the queues.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   offDelta        The relocation delta.
+ */
+void pdmR3QueueRelocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    /*
+     * Process the queues.
+     */
+    PUVM      pUVM       = pVM->pUVM;
+    PPDMQUEUE pQueueNext = pUVM->pdm.s.pQueuesTimer;
+    PPDMQUEUE pQueue     = pUVM->pdm.s.pQueuesForced;
+    do
+    {
+        while (pQueue)
+        {
+            if (pQueue->pVMRC)
+            {
+                pQueue->pVMRC = pVM->pVMRC;
+
+                /* Pending RC items. */
+                if (pQueue->pPendingRC)
+                {
+                    pQueue->pPendingRC += offDelta;
+                    PPDMQUEUEITEMCORE pCur = (PPDMQUEUEITEMCORE)MMHyperRCToR3(pVM, pQueue->pPendingRC);
+                    while (pCur->pNextRC)
+                    {
+                        pCur->pNextRC += offDelta;
+                        pCur = (PPDMQUEUEITEMCORE)MMHyperRCToR3(pVM, pCur->pNextRC);
+                    }
+                }
+
+                /* The free items. */
+                uint32_t i = pQueue->iFreeTail;
+                while (i != pQueue->iFreeHead)
+                {
+                    pQueue->aFreeItems[i].pItemRC = MMHyperR3ToRC(pVM, pQueue->aFreeItems[i].pItemR3);
+                    i = (i + 1) % (pQueue->cItems + PDMQUEUE_FREE_SLACK);
+                }
+            }
+
+            /* next queue */
+            pQueue = pQueue->pNext;
+        }
+
+        /* next queue list */
+        pQueue = pQueueNext;
+        pQueueNext = NULL;
+    } while (pQueue);
+}
+
+
+/**
+ * Flush pending queues.
+ * This is a forced action callback.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @thread  Emulation thread only.
+ */
+VMMR3_INT_DECL(void) PDMR3QueueFlushAll(PVM pVM)
+{
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("PDMR3QueuesFlush:\n"));
+
+    /*
+     * Only let one EMT flushing queues at any one time to preserve the order
+     * and to avoid wasting time. The FF is always cleared here, because it's
+     * only used to get someones attention. Queue inserts occurring during the
+     * flush are caught using the pending bit.
+     *
+     * Note! We must check the force action and pending flags after clearing
+     *       the active bit!
+     */
+    VM_FF_CLEAR(pVM, VM_FF_PDM_QUEUES);
+    while (!ASMAtomicBitTestAndSet(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_ACTIVE_BIT))
+    {
+        ASMAtomicBitClear(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_PENDING_BIT);
+
+        for (PPDMQUEUE pCur = pVM->pUVM->pdm.s.pQueuesForced; pCur; pCur = pCur->pNext)
+            if (    pCur->pPendingR3
+                ||  pCur->pPendingR0
+                ||  pCur->pPendingRC)
+                pdmR3QueueFlush(pCur);
+
+        ASMAtomicBitClear(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_ACTIVE_BIT);
+
+        /* We're done if there were no inserts while we were busy. */
+        if (   !ASMBitTest(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_PENDING_BIT)
+            && !VM_FF_IS_SET(pVM, VM_FF_PDM_QUEUES))
+            break;
+        VM_FF_CLEAR(pVM, VM_FF_PDM_QUEUES);
+    }
+}
+
+
+/**
+ * Process pending items in one queue.
+ *
+ * @returns Success indicator.
+ *          If false the item the consumer said "enough!".
+ * @param   pQueue  The queue.
+ */
+static bool pdmR3QueueFlush(PPDMQUEUE pQueue)
+{
+    STAM_PROFILE_START(&pQueue->StatFlushPrf,p);
+
+    /*
+     * Get the lists.
+     */
+    PPDMQUEUEITEMCORE pItems   = ASMAtomicXchgPtrT(&pQueue->pPendingR3, NULL, PPDMQUEUEITEMCORE);
+    RTRCPTR           pItemsRC = ASMAtomicXchgRCPtr(&pQueue->pPendingRC, NIL_RTRCPTR);
+    RTR0PTR           pItemsR0 = ASMAtomicXchgR0Ptr(&pQueue->pPendingR0, NIL_RTR0PTR);
+
+    AssertMsgReturn(   pItemsR0
+                    || pItemsRC
+                    || pItems,
+                    ("Someone is racing us? This shouldn't happen!\n"),
+                    true);
+
+    /*
+     * Reverse the list (it's inserted in LIFO order to avoid semaphores, remember).
+     */
+    PPDMQUEUEITEMCORE pCur = pItems;
+    pItems = NULL;
+    while (pCur)
+    {
+        PPDMQUEUEITEMCORE pInsert = pCur;
+        pCur = pCur->pNextR3;
+        pInsert->pNextR3 = pItems;
+        pItems = pInsert;
+    }
+
+    /*
+     * Do the same for any pending RC items.
+     */
+    while (pItemsRC)
+    {
+        PPDMQUEUEITEMCORE pInsert = (PPDMQUEUEITEMCORE)MMHyperRCToR3(pQueue->pVMR3, pItemsRC);
+        pItemsRC = pInsert->pNextRC;
+        pInsert->pNextRC = NIL_RTRCPTR;
+        pInsert->pNextR3 = pItems;
+        pItems = pInsert;
+    }
+
+    /*
+     * Do the same for any pending R0 items.
+     */
+    while (pItemsR0)
+    {
+        PPDMQUEUEITEMCORE pInsert = (PPDMQUEUEITEMCORE)MMHyperR0ToR3(pQueue->pVMR3, pItemsR0);
+        pItemsR0 = pInsert->pNextR0;
+        pInsert->pNextR0 = NIL_RTR0PTR;
+        pInsert->pNextR3 = pItems;
+        pItems = pInsert;
+    }
+
+    /*
+     * Feed the items to the consumer function.
+     */
+    Log2(("pdmR3QueueFlush: pQueue=%p enmType=%d pItems=%p\n", pQueue, pQueue->enmType, pItems));
+    switch (pQueue->enmType)
+    {
+        case PDMQUEUETYPE_DEV:
+            while (pItems)
+            {
+                if (!pQueue->u.Dev.pfnCallback(pQueue->u.Dev.pDevIns, pItems))
+                    break;
+                pCur = pItems;
+                pItems = pItems->pNextR3;
+                pdmR3QueueFreeItem(pQueue, pCur);
+            }
+            break;
+
+        case PDMQUEUETYPE_DRV:
+            while (pItems)
+            {
+                if (!pQueue->u.Drv.pfnCallback(pQueue->u.Drv.pDrvIns, pItems))
+                    break;
+                pCur = pItems;
+                pItems = pItems->pNextR3;
+                pdmR3QueueFreeItem(pQueue, pCur);
+            }
+            break;
+
+        case PDMQUEUETYPE_INTERNAL:
+            while (pItems)
+            {
+                if (!pQueue->u.Int.pfnCallback(pQueue->pVMR3, pItems))
+                    break;
+                pCur = pItems;
+                pItems = pItems->pNextR3;
+                pdmR3QueueFreeItem(pQueue, pCur);
+            }
+            break;
+
+        case PDMQUEUETYPE_EXTERNAL:
+            while (pItems)
+            {
+                if (!pQueue->u.Ext.pfnCallback(pQueue->u.Ext.pvUser, pItems))
+                    break;
+                pCur = pItems;
+                pItems = pItems->pNextR3;
+                pdmR3QueueFreeItem(pQueue, pCur);
+            }
+            break;
+
+        default:
+            AssertMsgFailed(("Invalid queue type %d\n", pQueue->enmType));
+            break;
+    }
+
+    /*
+     * Success?
+     */
+    if (pItems)
+    {
+        /*
+         * Reverse the list.
+         */
+        pCur = pItems;
+        pItems = NULL;
+        while (pCur)
+        {
+            PPDMQUEUEITEMCORE pInsert = pCur;
+            pCur = pInsert->pNextR3;
+            pInsert->pNextR3 = pItems;
+            pItems = pInsert;
+        }
+
+        /*
+         * Insert the list at the tail of the pending list.
+         */
+        for (;;)
+        {
+            if (ASMAtomicCmpXchgPtr(&pQueue->pPendingR3, pItems, NULL))
+                break;
+            PPDMQUEUEITEMCORE pPending = ASMAtomicXchgPtrT(&pQueue->pPendingR3, NULL, PPDMQUEUEITEMCORE);
+            if (pPending)
+            {
+                pCur = pPending;
+                while (pCur->pNextR3)
+                    pCur = pCur->pNextR3;
+                pCur->pNextR3 = pItems;
+                pItems = pPending;
+            }
+        }
+
+        STAM_REL_COUNTER_INC(&pQueue->StatFlushLeftovers);
+        STAM_PROFILE_STOP(&pQueue->StatFlushPrf,p);
+        return false;
+    }
+
+    STAM_PROFILE_STOP(&pQueue->StatFlushPrf,p);
+    return true;
+}
+
+
+/**
+ * Free an item.
+ *
+ * @param   pQueue  The queue.
+ * @param   pItem   The item.
+ */
+DECLINLINE(void) pdmR3QueueFreeItem(PPDMQUEUE pQueue, PPDMQUEUEITEMCORE pItem)
+{
+    VM_ASSERT_EMT(pQueue->pVMR3);
+
+    int i = pQueue->iFreeHead;
+    int iNext = (i + 1) % (pQueue->cItems + PDMQUEUE_FREE_SLACK);
+
+    pQueue->aFreeItems[i].pItemR3 = pItem;
+    if (pQueue->pVMRC)
+    {
+        pQueue->aFreeItems[i].pItemRC = MMHyperR3ToRC(pQueue->pVMR3, pItem);
+        pQueue->aFreeItems[i].pItemR0 = MMHyperR3ToR0(pQueue->pVMR3, pItem);
+    }
+
+    if (!ASMAtomicCmpXchgU32(&pQueue->iFreeHead, iNext, i))
+        AssertMsgFailed(("huh? i=%d iNext=%d iFreeHead=%d iFreeTail=%d\n", i, iNext, pQueue->iFreeHead, pQueue->iFreeTail));
+    STAM_STATS({ ASMAtomicDecU32(&pQueue->cStatPending); });
+}
+
+
+/**
+ * Timer handler for PDM queues.
+ * This is called by for a single queue.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   pTimer  Pointer to timer.
+ * @param   pvUser  Pointer to the queue.
+ */
+static DECLCALLBACK(void) pdmR3QueueTimer(PVM pVM, PTMTIMER pTimer, void *pvUser)
+{
+    PPDMQUEUE pQueue = (PPDMQUEUE)pvUser;
+    Assert(pTimer == pQueue->pTimer); NOREF(pTimer); NOREF(pVM);
+
+    if (   pQueue->pPendingR3
+        || pQueue->pPendingR0
+        || pQueue->pPendingRC)
+        pdmR3QueueFlush(pQueue);
+    int rc = TMTimerSetMillies(pQueue->pTimer, pQueue->cMilliesInterval);
+    AssertRC(rc);
+}
+
diff --git a/src/VBox/VMM/VMMR3/PDMR3Task.cpp b/src/VBox/VMM/VMMR3/PDMR3Task.cpp
new file mode 100644
index 00000000..a06cf419
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMR3Task.cpp
@@ -0,0 +1,628 @@
+/* $Id: PDMR3Task.cpp $ */
+/** @file
+ * PDM Task - Asynchronous user mode tasks.
+ */
+
+/*
+ * Copyright (C) 2019-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_TASK
+#include "PDMInternal.h"
+#include <VBox/vmm/pdmtask.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <VBox/sup.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/semaphore.h>
+#include <iprt/thread.h>
+
+
+/**
+ * @callback_method_impl{FNDBGFINFOARGVINT}
+ */
+static DECLCALLBACK(void) pdmR3TaskInfo(PVM pVM, PCDBGFINFOHLP pHlp, int cArgs, char **papszArgs)
+{
+    RT_NOREF(cArgs, papszArgs); /* for now. */
+
+    uint32_t cSetsDisplayed = 0;
+    for (size_t i = 0; i < RT_ELEMENTS(pVM->pdm.s.apTaskSets); i++)
+    {
+        PPDMTASKSET pTaskSet = pVM->pdm.s.apTaskSets[i];
+        if (   pTaskSet
+            && (   pTaskSet->cAllocated > 0
+                || ASMAtomicReadU64(&pTaskSet->fTriggered)))
+        {
+            if (cSetsDisplayed > 0)
+                pHlp->pfnPrintf(pHlp, "\n");
+            pHlp->pfnPrintf(pHlp,
+                            "Task set #%u - handle base %u, pending %#RX64%s%s, running %d, %u of %u allocated:\n"
+                          /*  123: triggered internal 0123456789abcdef 0123456789abcdef 0x0000 SomeFunctionName */
+                            " Hnd:   State     Type   pfnCallback      pvUser           Flags  Name\n",
+                            i, pTaskSet->uHandleBase, ASMAtomicReadU64(&pTaskSet->fTriggered),
+                            pTaskSet->fRZEnabled ? " RZ-enabled" : "", pTaskSet->hThread != NIL_RTTHREAD ? "" : " no-thread",
+                            (int)ASMAtomicReadU32(&pTaskSet->idxRunning), pTaskSet->cAllocated, RT_ELEMENTS(pTaskSet->aTasks));
+            for (unsigned j = 0; j < RT_ELEMENTS(pTaskSet->aTasks); j++)
+            {
+                PPDMTASK pTask = &pTaskSet->aTasks[j];
+                if (pTask->pvOwner)
+                {
+                    const char *pszType;
+                    switch (pTask->enmType)
+                    {
+                        case PDMTASKTYPE_DEV:       pszType = " device "; break;
+                        case PDMTASKTYPE_DRV:       pszType = " driver "; break;
+                        case PDMTASKTYPE_USB:       pszType = " usbdev "; break;
+                        case PDMTASKTYPE_INTERNAL:  pszType = "internal"; break;
+                        default:                    pszType = "unknown "; break;
+                    }
+                    pHlp->pfnPrintf(pHlp, " %3u: %s %s %p %p %#06x %s\n", pTaskSet->uHandleBase + j,
+                                    ASMBitTest(&pTaskSet->fTriggered, j) ? "triggered"
+                                    : ASMAtomicReadU32(&pTaskSet->idxRunning) == j  ? " running " : "  idle   ",
+                                    pszType, pTask->pfnCallback, pTask->pvUser, pTask->fFlags, pTask->pszName);
+                }
+            }
+
+            cSetsDisplayed++;
+        }
+    }
+}
+
+
+/**
+ * Initializes the ring-0 capable tasks during VM construction.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+int pdmR3TaskInit(PVM pVM)
+{
+    for (size_t i = 0; i < RT_ELEMENTS(pVM->pdm.s.aTaskSets); i++)
+    {
+        PPDMTASKSET pTaskSet = &pVM->pdm.s.aTaskSets[i];
+
+        pTaskSet->u32Magic      = PDMTASKSET_MAGIC;
+        pTaskSet->fRZEnabled    = true;
+        //pTaskSet->cAllocated  = 0;
+        pTaskSet->uHandleBase   = (uint16_t)(i * RT_ELEMENTS(pTaskSet->aTasks));
+        pTaskSet->hThread       = NIL_RTTHREAD;
+        int rc = SUPSemEventCreate(pVM->pSession, &pTaskSet->hEventR0);
+        AssertRCReturn(rc, rc);
+        pTaskSet->hEventR3      = NIL_RTSEMEVENT;
+        //pTaskSet->fTriggered  = 0;
+        pTaskSet->idxRunning    = UINT8_MAX;
+        //pTaskSet->fShutdown   = false;
+        pTaskSet->pVM           = pVM;
+
+        pVM->pdm.s.apTaskSets[i] = pTaskSet;
+    }
+
+    int rc = DBGFR3InfoRegisterInternalArgv(pVM, "tasks", "PDM tasks", pdmR3TaskInfo, 0 /*fFlags*/);
+    AssertRC(rc);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminates task threads when the VM is destroyed.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+void pdmR3TaskTerm(PVM pVM)
+{
+    /*
+     * Signal all the threads first.
+     */
+    for (size_t i = 0; i < RT_ELEMENTS(pVM->pdm.s.apTaskSets); i++)
+    {
+        PPDMTASKSET pTaskSet = pVM->pdm.s.apTaskSets[i];
+        if (pTaskSet)
+        {
+            /*
+             * Set the shutdown indicator and signal the thread.
+             */
+            ASMAtomicWriteBool(&pTaskSet->fShutdown, true);
+
+            if (pTaskSet->hEventR0 != NIL_SUPSEMEVENT)
+            {
+                int rc = SUPSemEventSignal(pVM->pSession, pTaskSet->hEventR0);
+                AssertRC(rc);
+            }
+
+            if (pTaskSet->hEventR3 != NIL_RTSEMEVENT)
+            {
+                int rc = RTSemEventSignal(pTaskSet->hEventR3);
+                AssertRC(rc);
+            }
+        }
+    }
+
+    /*
+     * Wait for them to terminate and clean up semaphores.
+     */
+    for (size_t i = 0; i < RT_ELEMENTS(pVM->pdm.s.apTaskSets); i++)
+    {
+        PPDMTASKSET pTaskSet = pVM->pdm.s.apTaskSets[i];
+        if (pTaskSet)
+        {
+            /*
+             * Wait for the thread to terminate.
+             */
+            if (pTaskSet->hThread != NIL_RTTHREAD)
+            {
+                int rc = RTThreadWait(pTaskSet->hThread, RT_MS_30SEC, NULL);
+                AssertLogRelMsg(RT_SUCCESS(rc), ("pTaskSet %u: thread wait failed: %Rrc\n", i, rc));
+                if (RT_SUCCESS(rc))
+                    pTaskSet->hThread = NIL_RTTHREAD;
+            }
+
+            /*
+             * Destroy the semaphore.
+             */
+            if (pTaskSet->hEventR0 != NIL_SUPSEMEVENT)
+            {
+                int rc = SUPSemEventClose(pVM->pSession, pTaskSet->hEventR0);
+                AssertRC(rc);
+                pTaskSet->hEventR0 = NIL_SUPSEMEVENT;
+            }
+
+            if (pTaskSet->hEventR3 != NIL_RTSEMEVENT)
+            {
+                int rc = RTSemEventDestroy(pTaskSet->hEventR3);
+                AssertRC(rc);
+                pTaskSet->hEventR3 = NIL_RTSEMEVENT;
+            }
+        }
+    }
+}
+
+
+/**
+ * @callback_method_impl{FNRTTHREAD,
+ *      PDM Asynchronous Task Executor Thread}
+ */
+static DECLCALLBACK(int) pdmR3TaskThread(RTTHREAD ThreadSelf, void *pvUser)
+{
+    PPDMTASKSET const pTaskSet = (PPDMTASKSET)pvUser;
+    AssertPtr(pTaskSet);
+    Assert(pTaskSet->u32Magic == PDMTASKSET_MAGIC);
+    RT_NOREF(ThreadSelf);
+
+    /*
+     * Process stuff until we're told to terminate.
+     */
+    while (!ASMAtomicReadBool(&pTaskSet->fShutdown))
+    {
+        /*
+         * Process pending tasks.
+         *
+         * The outer loop runs till there are no more pending tasks.
+         *
+         * The inner loop takes one snapshot of fTriggered and processes all
+         * pending bits in the snaphot.   This ensure fairness.
+         */
+        for (;;)
+        {
+            uint64_t fTriggered = ASMAtomicReadU64(&pTaskSet->fTriggered);
+            unsigned iTask      = ASMBitFirstSetU64(fTriggered);
+            if (iTask == 0)
+                break;
+            uint32_t cShutdown = 3;
+            do
+            {
+                iTask--;
+                AssertBreak(iTask < RT_ELEMENTS(pTaskSet->aTasks));
+
+                if (ASMAtomicBitTestAndClear(&pTaskSet->fTriggered, iTask))
+                {
+                    PPDMTASK  pTask = &pTaskSet->aTasks[iTask];
+
+                    /* Copy out the data we need here to try avoid destruction race trouble. */
+                    PDMTASKTYPE const enmType     = pTask->enmType;
+                    PFNRT  const      pfnCallback = pTask->pfnCallback;
+                    void * const      pvOwner     = pTask->pvOwner;
+                    void * const      pvTaskUser  = pTask->pvUser;
+
+                    ASMAtomicWriteU32(&pTaskSet->idxRunning, iTask);
+
+                    if (   pvOwner
+                        && pfnCallback
+                        && pvOwner     == pTask->pvOwner
+                        && pfnCallback == pTask->pfnCallback
+                        && pvTaskUser  == pTask->pvUser
+                        && enmType     == pTask->enmType)
+                    {
+                        pTask->cRuns += 1;
+                        switch (pTask->enmType)
+                        {
+                            case PDMTASKTYPE_DEV:
+                                Log2(("pdmR3TaskThread: Runs dev task %s (%#x)\n", pTask->pszName, iTask + pTaskSet->uHandleBase));
+                                ((PFNPDMTASKDEV)(pfnCallback))((PPDMDEVINS)pvOwner, pvTaskUser);
+                                break;
+                            case PDMTASKTYPE_DRV:
+                                Log2(("pdmR3TaskThread: Runs drv task %s (%#x)\n", pTask->pszName, iTask + pTaskSet->uHandleBase));
+                                ((PFNPDMTASKDRV)(pfnCallback))((PPDMDRVINS)pvOwner, pvTaskUser);
+                                break;
+                            case PDMTASKTYPE_USB:
+                                Log2(("pdmR3TaskThread: Runs USB task %s (%#x)\n", pTask->pszName, iTask + pTaskSet->uHandleBase));
+                                ((PFNPDMTASKUSB)(pfnCallback))((PPDMUSBINS)pvOwner, pvTaskUser);
+                                break;
+                            case PDMTASKTYPE_INTERNAL:
+                                Log2(("pdmR3TaskThread: Runs int task %s (%#x)\n", pTask->pszName, iTask + pTaskSet->uHandleBase));
+                                ((PFNPDMTASKINT)(pfnCallback))((PVM)pvOwner, pvTaskUser);
+                                break;
+                            default:
+                                AssertFailed();
+                        }
+                    }
+                    else /* Note! There might be a race here during destruction. */
+                        AssertMsgFailed(("%d %p %p %p\n", enmType, pvOwner, pfnCallback, pvTaskUser));
+
+                    ASMAtomicWriteU32(&pTaskSet->idxRunning, UINT32_MAX);
+                }
+
+                /* Next pending task. */
+                fTriggered &= ~RT_BIT_64(iTask);
+                iTask = ASMBitFirstSetU64(fTriggered);
+            } while (iTask != 0);
+
+            /*
+             * If we're shutting down, we'll try drain the pending tasks by
+             * looping three more times before just quitting.  We don't want
+             * to get stuck here if some stuff is misbehaving.
+             */
+            if (!ASMAtomicReadBool(&pTaskSet->fShutdown))
+            { /* likely */ }
+            else if (--cShutdown == 0)
+                break;
+        }
+
+        /*
+         * Wait unless we're shutting down.
+         */
+        if (!ASMAtomicReadBool(&pTaskSet->fShutdown))
+        {
+            if (pTaskSet->fRZEnabled)
+                SUPSemEventWaitNoResume(pTaskSet->pVM->pSession, pTaskSet->hEventR0, RT_MS_15SEC);
+            else
+                RTSemEventWaitNoResume(pTaskSet->hEventR3, RT_MS_15SEC);
+        }
+    }
+
+    /*
+     * Complain about pending tasks.
+     */
+    uint64_t const fTriggered = ASMAtomicReadU64(&pTaskSet->fTriggered);
+    AssertLogRelMsg(fTriggered == 0, ("fTriggered=%#RX64 - %u %s\n", fTriggered, ASMBitFirstSetU64(fTriggered) - 1,
+                                      pTaskSet->aTasks[ASMBitFirstSetU64(fTriggered) - 1].pszName));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3TaskCreate().
+ */
+DECLINLINE(PPDMTASK) pdmR3TaskAllocInSet(PPDMTASKSET pTaskSet)
+{
+    if (pTaskSet->cAllocated < RT_ELEMENTS(pTaskSet->aTasks))
+    {
+        for (size_t j = 0; j < RT_ELEMENTS(pTaskSet->aTasks); j++)
+            if (pTaskSet->aTasks[j].pvOwner == NULL)
+                return &pTaskSet->aTasks[j];
+        AssertFailed();
+    }
+    return NULL;
+}
+
+/**
+ * Creates a task.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   fFlags      PDMTASK_F_XXX.
+ * @param   pszName     The task name (function name ++).
+ * @param   enmType     The task owner type.
+ * @param   pvOwner     The task owner pointer.
+ * @param   pfnCallback The task callback.
+ * @param   pvUser      The user argument for the callback.
+ * @param   phTask      Where to return the task handle.
+ * @thread  EMT(0)
+ */
+VMMR3_INT_DECL(int) PDMR3TaskCreate(PVM pVM, uint32_t fFlags, const char *pszName, PDMTASKTYPE enmType, void *pvOwner,
+                                    PFNRT pfnCallback, void *pvUser, PDMTASKHANDLE *phTask)
+{
+    /*
+     * Validate input.
+     */
+    AssertReturn(!(fFlags & ~PDMTASK_F_VALID_MASK), VERR_INVALID_FLAGS);
+    AssertPtrReturn(pvOwner, VERR_INVALID_POINTER);
+    AssertPtrReturn(pfnCallback, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszName, VERR_INVALID_POINTER);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT); /* implicit serialization by requiring EMT(0) */
+    switch (enmType)
+    {
+        case PDMTASKTYPE_DEV:
+        case PDMTASKTYPE_DRV:
+        case PDMTASKTYPE_USB:
+            break;
+        case PDMTASKTYPE_INTERNAL:
+            AssertReturn(pvOwner == (void *)pVM, VERR_INVALID_PARAMETER);
+            break;
+        default:
+            AssertFailedReturn(VERR_INVALID_PARAMETER);
+    }
+
+    /*
+     * If the callback must be ring-0 triggerable, we are restricted to the
+     * task sets living the VM structure.  Otherwise, pick from the dynamically
+     * allocated sets living on ring-3 heap.
+     */
+    PPDMTASKSET pTaskSet = NULL;
+    PPDMTASK    pTask    = NULL;
+    if (fFlags & PDMTASK_F_RZ)
+    {
+        for (size_t i = 0; i < RT_ELEMENTS(pVM->pdm.s.aTaskSets); i++)
+        {
+            pTaskSet = &pVM->pdm.s.aTaskSets[i];
+            pTask = pdmR3TaskAllocInSet(pTaskSet);
+            if (pTask)
+                break;
+        }
+    }
+    else
+    {
+        for (size_t i = RT_ELEMENTS(pVM->pdm.s.aTaskSets); i < RT_ELEMENTS(pVM->pdm.s.apTaskSets); i++)
+        {
+            pTaskSet = pVM->pdm.s.apTaskSets[i];
+            if (pTaskSet)
+            {
+                pTask = pdmR3TaskAllocInSet(pTaskSet);
+                if (pTask)
+                    break;
+            }
+            else
+            {
+                /*
+                 * Try allocate a new set.
+                 */
+                LogFlow(("PDMR3TaskCreate: Allocating new task set (%#u)...\n", i));
+                pTaskSet = (PPDMTASKSET)MMR3HeapAllocZ(pVM, MM_TAG_PDM, sizeof(*pTaskSet));
+                AssertReturn(pTaskSet, VERR_NO_MEMORY);
+
+                pTaskSet->u32Magic      = PDMTASKSET_MAGIC;
+                //pTaskSet->fRZEnabled  = false;
+                //pTaskSet->cAllocated  = 0;
+                pTaskSet->uHandleBase   = (uint16_t)(i * RT_ELEMENTS(pTaskSet->aTasks));
+                pTaskSet->hThread       = NIL_RTTHREAD;
+                pTaskSet->hEventR0      = NIL_SUPSEMEVENT;
+                int rc = RTSemEventCreate(&pTaskSet->hEventR3);
+                AssertRCReturnStmt(rc, MMR3HeapFree(pTaskSet), rc);
+                //pTaskSet->fTriggered  = 0;
+                pTaskSet->idxRunning    = UINT8_MAX;
+                //pTaskSet->fShutdown   = false;
+                pTaskSet->pVM           = pVM;
+
+                pVM->pdm.s.apTaskSets[i] = pTaskSet;
+                pTask = &pTaskSet->aTasks[0];
+                break;
+            }
+        }
+    }
+    AssertLogRelReturn(pTask, VERR_OUT_OF_RESOURCES);
+
+    /*
+     * Do we need to start a worker thread?  Do this first as it can fail.
+     */
+    if (pTaskSet->hThread == NIL_RTTHREAD)
+    {
+        int rc = RTThreadCreateF(&pTaskSet->hThread, pdmR3TaskThread, pTaskSet, 0 /*cbStack*/, RTTHREADTYPE_IO,
+                                 RTTHREADFLAGS_WAITABLE, "TaskSet%u", pTaskSet->uHandleBase / RT_ELEMENTS(pTaskSet->aTasks));
+        AssertLogRelRCReturn(rc, rc);
+    }
+
+    /*
+     * Complete the allocation.
+     */
+    pTask->enmType     = enmType;
+    pTask->fFlags      = fFlags;
+    pTask->pvUser      = pvUser;
+    pTask->pfnCallback = pfnCallback;
+    pTask->pszName     = pszName;
+    ASMAtomicWritePtr(&pTask->pvOwner, pvOwner);
+    pTaskSet->cAllocated += 1;
+
+    uint32_t const hTask = pTaskSet->uHandleBase + (uint32_t)(pTask - &pTaskSet->aTasks[0]);
+    *phTask = hTask;
+
+    STAMR3RegisterF(pVM, &pTask->cRuns, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                    "Number of times the task has been executed.", "/PDM/Tasks/%03u-%s-runs", hTask, pszName);
+    STAMR3RegisterF(pVM, (void *)&pTask->cAlreadyTrigged, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                    "Number of times the task was re-triggered.", "/PDM/Tasks/%03u-%s-retriggered", hTask, pszName);
+
+    LogFlow(("PDMR3TaskCreate: Allocated %u for %s\n", hTask, pszName));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates an internal task.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   fFlags      PDMTASK_F_XXX.
+ * @param   pszName     The task name (function name ++).
+ * @param   pfnCallback The task callback.
+ * @param   pvUser      The user argument for the callback.
+ * @param   phTask      Where to return the task handle.
+ * @thread  EMT(0)
+ */
+VMMR3_INT_DECL(int) PDMR3TaskCreateInternal(PVM pVM, uint32_t fFlags, const char *pszName,
+                                            PFNPDMTASKINT pfnCallback, void *pvUser, PDMTASKHANDLE *phTask)
+{
+    return PDMR3TaskCreate(pVM, fFlags, pszName, PDMTASKTYPE_INTERNAL, pVM, (PFNRT)pfnCallback, pvUser, phTask);
+}
+
+
+/**
+ * Worker for PDMR3TaskDestroyAllByOwner() and PDMR3TaskDestroySpecific().
+ */
+static void pdmR3TaskDestroyOne(PVM pVM, PPDMTASKSET pTaskSet, PPDMTASK pTask, size_t iTask)
+{
+    AssertPtr(pTask->pvOwner);
+
+    /*
+     * Delay if busy.
+     */
+    uint32_t cYields = 64;
+    while (   ASMAtomicReadU32(&pTaskSet->idxRunning) == iTask
+           && cYields > 0
+           && pTaskSet->hThread != NIL_RTTHREAD)
+    {
+        ASMNopPause();
+        RTThreadYield();
+    }
+
+    /*
+     * Zap it (very noisy, but whatever).
+     */
+    LogFlow(("pdmR3TaskDestroyOne: Destroying %zu %s\n", iTask + pTaskSet->uHandleBase, pTask->pszName));
+    AssertPtr(pTask->pvOwner);
+
+    char szPrefix[64];
+    RTStrPrintf(szPrefix, sizeof(szPrefix), "/PDM/Tasks/%03zu-", iTask + pTaskSet->uHandleBase);
+    STAMR3DeregisterByPrefix(pVM->pUVM, szPrefix);
+
+    AssertPtr(pTask->pvOwner);
+    ASMAtomicWriteNullPtr(&pTask->pvOwner);
+    pTask->enmType     = (PDMTASKTYPE)0;
+    pTask->fFlags      = 0;
+    ASMAtomicWriteNullPtr(&pTask->pfnCallback);
+    ASMAtomicWriteNullPtr(&pTask->pvUser);
+    ASMAtomicWriteNullPtr(&pTask->pszName);
+
+    AssertReturnVoid(pTaskSet->cAllocated > 0);
+    pTaskSet->cAllocated -= 1;
+}
+
+
+/**
+ * Destroys all tasks belonging to @a pvOwner.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmType     The owner type.
+ * @param   pvOwner     The owner.
+ */
+VMMR3_INT_DECL(int) PDMR3TaskDestroyAllByOwner(PVM pVM, PDMTASKTYPE enmType, void *pvOwner)
+{
+    /*
+     * Validate input.
+     */
+    AssertReturn(enmType >= PDMTASKTYPE_DEV && enmType < PDMTASKTYPE_INTERNAL, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pvOwner, VERR_INVALID_POINTER);
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT); /* implicit serialization by requiring EMT(0) */
+
+    /*
+     * Scan all the task sets.
+     */
+    for (size_t i = 0; i < RT_ELEMENTS(pVM->pdm.s.apTaskSets); i++)
+    {
+        PPDMTASKSET pTaskSet = pVM->pdm.s.apTaskSets[i];
+        if (pTaskSet)
+        {
+            ssize_t cLeft = pTaskSet->cAllocated;
+            for (size_t j = 0; j < RT_ELEMENTS(pTaskSet->aTasks) && cLeft > 0; j++)
+            {
+                PPDMTASK     pTask       = &pTaskSet->aTasks[j];
+                void * const pvTaskOwner = pTask->pvOwner;
+                if (pvTaskOwner)
+                {
+                    if (   pvTaskOwner == pvOwner
+                        && pTask->enmType == enmType)
+                        pdmR3TaskDestroyOne(pVM, pTaskSet, pTask, j);
+                    else
+                        Assert(pvTaskOwner != pvOwner);
+                    cLeft--;
+                }
+            }
+        }
+        else
+            break;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroys the task @a hTask.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmType     The owner type.
+ * @param   pvOwner     The owner.
+ * @param   hTask       Handle to the task to destroy.
+ */
+VMMR3_INT_DECL(int) PDMR3TaskDestroySpecific(PVM pVM, PDMTASKTYPE enmType, void *pvOwner, PDMTASKHANDLE hTask)
+{
+    /*
+     * Validate the input.
+     */
+    AssertReturn(enmType >= PDMTASKTYPE_DEV && enmType <= PDMTASKTYPE_INTERNAL, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pvOwner, VERR_INVALID_POINTER);
+
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    size_t const      iTask    = hTask % RT_ELEMENTS(pVM->pdm.s.apTaskSets[0]->aTasks);
+    size_t const      iTaskSet = hTask / RT_ELEMENTS(pVM->pdm.s.apTaskSets[0]->aTasks);
+    AssertReturn(iTaskSet < RT_ELEMENTS(pVM->pdm.s.apTaskSets), VERR_INVALID_HANDLE);
+    PPDMTASKSET const pTaskSet = pVM->pdm.s.apTaskSets[iTaskSet];
+    AssertPtrReturn(pTaskSet, VERR_INVALID_HANDLE);
+    AssertPtrReturn(pTaskSet->u32Magic == PDMTASKSET_MAGIC, VERR_INVALID_MAGIC);
+    PPDMTASK const    pTask    = &pTaskSet->aTasks[iTask];
+
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT); /* implicit serialization by requiring EMT(0) */
+
+    AssertPtrReturn(pTask->pvOwner == pvOwner, VERR_NOT_OWNER);
+    AssertPtrReturn(pTask->enmType == enmType, VERR_NOT_OWNER);
+
+    /*
+     * Do the job.
+     */
+    pdmR3TaskDestroyOne(pVM, pTaskSet, pTask, iTask);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroys the internal task @a hTask.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTask       Handle to the task to destroy.
+ */
+VMMR3_INT_DECL(int) PDMR3TaskDestroyInternal(PVM pVM, PDMTASKHANDLE hTask)
+{
+    return PDMR3TaskDestroySpecific(pVM, PDMTASKTYPE_INTERNAL, pVM, hTask);
+}
+
diff --git a/src/VBox/VMM/VMMR3/PDMThread.cpp b/src/VBox/VMM/VMMR3/PDMThread.cpp
new file mode 100644
index 00000000..35e5f1f9
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMThread.cpp
@@ -0,0 +1,1093 @@
+/* $Id: PDMThread.cpp $ */
+/** @file
+ * PDM Thread - VM Thread Management.
+ */
+
+/*
+ * Copyright (C) 2007-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+/// @todo \#define LOG_GROUP LOG_GROUP_PDM_THREAD
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/semaphore.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(int) pdmR3ThreadMain(RTTHREAD Thread, void *pvUser);
+
+
+/**
+ * Wrapper around ASMAtomicCmpXchgSize.
+ */
+DECLINLINE(bool) pdmR3AtomicCmpXchgState(PPDMTHREAD pThread, PDMTHREADSTATE enmNewState, PDMTHREADSTATE enmOldState)
+{
+    bool fRc;
+    ASMAtomicCmpXchgSize(&pThread->enmState, enmNewState, enmOldState, fRc);
+    return fRc;
+}
+
+
+/**
+ * Does the wakeup call.
+ *
+ * @returns VBox status code. Already asserted on failure.
+ * @param   pThread     The PDM thread.
+ */
+static DECLCALLBACK(int) pdmR3ThreadWakeUp(PPDMTHREAD pThread)
+{
+    RTSemEventMultiSignal(pThread->Internal.s.SleepEvent);
+
+    int rc;
+    switch (pThread->Internal.s.enmType)
+    {
+        case PDMTHREADTYPE_DEVICE:
+            rc = pThread->u.Dev.pfnWakeUp(pThread->u.Dev.pDevIns, pThread);
+            break;
+
+        case PDMTHREADTYPE_USB:
+            rc = pThread->u.Usb.pfnWakeUp(pThread->u.Usb.pUsbIns, pThread);
+            break;
+
+        case PDMTHREADTYPE_DRIVER:
+            rc = pThread->u.Drv.pfnWakeUp(pThread->u.Drv.pDrvIns, pThread);
+            break;
+
+        case PDMTHREADTYPE_INTERNAL:
+            rc = pThread->u.Int.pfnWakeUp(pThread->Internal.s.pVM, pThread);
+            break;
+
+        case PDMTHREADTYPE_EXTERNAL:
+            rc = pThread->u.Ext.pfnWakeUp(pThread);
+            break;
+
+        default:
+            AssertMsgFailed(("%d\n", pThread->Internal.s.enmType));
+            rc = VERR_PDM_THREAD_IPE_1;
+            break;
+    }
+    AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * Allocates new thread instance.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   ppThread    Where to store the pointer to the instance.
+ */
+static int pdmR3ThreadNew(PVM pVM, PPPDMTHREAD ppThread)
+{
+    PPDMTHREAD pThread;
+    int rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_THREAD, sizeof(*pThread), (void **)&pThread);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    pThread->u32Version     = PDMTHREAD_VERSION;
+    pThread->enmState       = PDMTHREADSTATE_INITIALIZING;
+    pThread->Thread         = NIL_RTTHREAD;
+    pThread->Internal.s.pVM = pVM;
+
+    *ppThread = pThread;
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Initialize a new thread, this actually creates the thread.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   ppThread    Where the thread instance data handle is.
+ * @param   cbStack     The stack size, see RTThreadCreate().
+ * @param   enmType     The thread type, see RTThreadCreate().
+ * @param   pszName     The thread name, see RTThreadCreate().
+ */
+static int pdmR3ThreadInit(PVM pVM, PPPDMTHREAD ppThread, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    PPDMTHREAD  pThread = *ppThread;
+    PUVM        pUVM    = pVM->pUVM;
+
+    /*
+     * Initialize the remainder of the structure.
+     */
+    pThread->Internal.s.pVM = pVM;
+
+    int rc = RTSemEventMultiCreate(&pThread->Internal.s.BlockEvent);
+    if (RT_SUCCESS(rc))
+    {
+        rc = RTSemEventMultiCreate(&pThread->Internal.s.SleepEvent);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Create the thread and wait for it to initialize.
+             * The newly created thread will set the PDMTHREAD::Thread member.
+             */
+            RTTHREAD Thread;
+            rc = RTThreadCreate(&Thread, pdmR3ThreadMain, pThread, cbStack, enmType, RTTHREADFLAGS_WAITABLE, pszName);
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTThreadUserWait(Thread, 60*1000);
+                if (    RT_SUCCESS(rc)
+                    &&  pThread->enmState != PDMTHREADSTATE_SUSPENDED)
+                    rc = VERR_PDM_THREAD_IPE_2;
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Insert it into the thread list.
+                     */
+                    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+                    pThread->Internal.s.pNext = NULL;
+                    if (pUVM->pdm.s.pThreadsTail)
+                        pUVM->pdm.s.pThreadsTail->Internal.s.pNext = pThread;
+                    else
+                        pUVM->pdm.s.pThreads = pThread;
+                    pUVM->pdm.s.pThreadsTail = pThread;
+                    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+                    rc = RTThreadUserReset(Thread);
+                    AssertRC(rc);
+                    return rc;
+                }
+
+                /* bailout */
+                RTThreadWait(Thread, 60*1000, NULL);
+            }
+            RTSemEventMultiDestroy(pThread->Internal.s.SleepEvent);
+            pThread->Internal.s.SleepEvent = NIL_RTSEMEVENTMULTI;
+        }
+        RTSemEventMultiDestroy(pThread->Internal.s.BlockEvent);
+        pThread->Internal.s.BlockEvent = NIL_RTSEMEVENTMULTI;
+    }
+    MMR3HeapFree(pThread);
+    *ppThread = NULL;
+
+    return rc;
+}
+
+
+/**
+ * Device Helper for creating a thread associated with a device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     The device instance.
+ * @param   ppThread    Where to store the thread 'handle'.
+ * @param   pvUser      The user argument to the thread function.
+ * @param   pfnThread   The thread function.
+ * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
+ *                      a state change is pending.
+ * @param   cbStack     See RTThreadCreate.
+ * @param   enmType     See RTThreadCreate.
+ * @param   pszName     See RTThreadCreate.
+ */
+int pdmR3ThreadCreateDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDEV pfnThread,
+                            PFNPDMTHREADWAKEUPDEV pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    int rc = pdmR3ThreadNew(pVM, ppThread);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMTHREAD pThread = *ppThread;
+        pThread->pvUser = pvUser;
+        pThread->Internal.s.enmType = PDMTHREADTYPE_DEVICE;
+        pThread->u.Dev.pDevIns = pDevIns;
+        pThread->u.Dev.pfnThread = pfnThread;
+        pThread->u.Dev.pfnWakeUp = pfnWakeUp;
+        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
+    }
+    return rc;
+}
+
+
+/**
+ * USB Device Helper for creating a thread associated with an USB device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pUsbIns     The USB device instance.
+ * @param   ppThread    Where to store the thread 'handle'.
+ * @param   pvUser      The user argument to the thread function.
+ * @param   pfnThread   The thread function.
+ * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
+ *                      a state change is pending.
+ * @param   cbStack     See RTThreadCreate.
+ * @param   enmType     See RTThreadCreate.
+ * @param   pszName     See RTThreadCreate.
+ */
+int pdmR3ThreadCreateUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADUSB pfnThread,
+                         PFNPDMTHREADWAKEUPUSB pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    int rc = pdmR3ThreadNew(pVM, ppThread);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMTHREAD pThread = *ppThread;
+        pThread->pvUser = pvUser;
+        pThread->Internal.s.enmType = PDMTHREADTYPE_USB;
+        pThread->u.Usb.pUsbIns = pUsbIns;
+        pThread->u.Usb.pfnThread = pfnThread;
+        pThread->u.Usb.pfnWakeUp = pfnWakeUp;
+        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
+    }
+    return rc;
+}
+
+
+/**
+ * Driver Helper for creating a thread associated with a driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDrvIns     The driver instance.
+ * @param   ppThread    Where to store the thread 'handle'.
+ * @param   pvUser      The user argument to the thread function.
+ * @param   pfnThread   The thread function.
+ * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
+ *                      a state change is pending.
+ * @param   cbStack     See RTThreadCreate.
+ * @param   enmType     See RTThreadCreate.
+ * @param   pszName     See RTThreadCreate.
+ */
+int pdmR3ThreadCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDRV pfnThread,
+                            PFNPDMTHREADWAKEUPDRV pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    int rc = pdmR3ThreadNew(pVM, ppThread);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMTHREAD pThread = *ppThread;
+        pThread->pvUser = pvUser;
+        pThread->Internal.s.enmType = PDMTHREADTYPE_DRIVER;
+        pThread->u.Drv.pDrvIns = pDrvIns;
+        pThread->u.Drv.pfnThread = pfnThread;
+        pThread->u.Drv.pfnWakeUp = pfnWakeUp;
+        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
+    }
+    return rc;
+}
+
+
+/**
+ * Creates a PDM thread for internal use in the VM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   ppThread    Where to store the thread 'handle'.
+ * @param   pvUser      The user argument to the thread function.
+ * @param   pfnThread   The thread function.
+ * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
+ *                      a state change is pending.
+ * @param   cbStack     See RTThreadCreate.
+ * @param   enmType     See RTThreadCreate.
+ * @param   pszName     See RTThreadCreate.
+ */
+VMMR3DECL(int) PDMR3ThreadCreate(PVM pVM, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADINT pfnThread,
+                                 PFNPDMTHREADWAKEUPINT pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    int rc = pdmR3ThreadNew(pVM, ppThread);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMTHREAD pThread = *ppThread;
+        pThread->pvUser = pvUser;
+        pThread->Internal.s.enmType = PDMTHREADTYPE_INTERNAL;
+        pThread->u.Int.pfnThread = pfnThread;
+        pThread->u.Int.pfnWakeUp = pfnWakeUp;
+        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
+    }
+    return rc;
+}
+
+
+/**
+ * Creates a PDM thread for VM use by some external party.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   ppThread    Where to store the thread 'handle'.
+ * @param   pvUser      The user argument to the thread function.
+ * @param   pfnThread   The thread function.
+ * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
+ *                      a state change is pending.
+ * @param   cbStack     See RTThreadCreate.
+ * @param   enmType     See RTThreadCreate.
+ * @param   pszName     See RTThreadCreate.
+ */
+VMMR3DECL(int) PDMR3ThreadCreateExternal(PVM pVM, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADEXT pfnThread,
+                                         PFNPDMTHREADWAKEUPEXT pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    int rc = pdmR3ThreadNew(pVM, ppThread);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMTHREAD pThread = *ppThread;
+        pThread->pvUser = pvUser;
+        pThread->Internal.s.enmType = PDMTHREADTYPE_EXTERNAL;
+        pThread->u.Ext.pfnThread = pfnThread;
+        pThread->u.Ext.pfnWakeUp = pfnWakeUp;
+        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
+    }
+    return rc;
+}
+
+
+/**
+ * Destroys a PDM thread.
+ *
+ * This will wakeup the thread, tell it to terminate, and wait for it terminate.
+ *
+ * @returns VBox status code.
+ *          This reflects the success off destroying the thread and not the exit code
+ *          of the thread as this is stored in *pRcThread.
+ * @param   pThread         The thread to destroy.
+ * @param   pRcThread       Where to store the thread exit code. Optional.
+ * @thread  The emulation thread (EMT).
+ */
+VMMR3DECL(int) PDMR3ThreadDestroy(PPDMTHREAD pThread, int *pRcThread)
+{
+    /*
+     * Assert sanity.
+     */
+    AssertPtrReturn(pThread, VERR_INVALID_POINTER);
+    AssertReturn(pThread->u32Version == PDMTHREAD_VERSION, VERR_INVALID_MAGIC);
+    Assert(pThread->Thread != RTThreadSelf());
+    AssertPtrNullReturn(pRcThread, VERR_INVALID_POINTER);
+    PVM pVM = pThread->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+    PUVM pUVM = pVM->pUVM;
+
+    /*
+     * Advance the thread to the terminating state.
+     */
+    int rc = VINF_SUCCESS;
+    if (pThread->enmState <= PDMTHREADSTATE_TERMINATING)
+    {
+        for (;;)
+        {
+            PDMTHREADSTATE enmState = pThread->enmState;
+            switch (enmState)
+            {
+                case PDMTHREADSTATE_RUNNING:
+                    if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+                        continue;
+                    rc = pdmR3ThreadWakeUp(pThread);
+                    break;
+
+                case PDMTHREADSTATE_SUSPENDED:
+                case PDMTHREADSTATE_SUSPENDING:
+                case PDMTHREADSTATE_RESUMING:
+                case PDMTHREADSTATE_INITIALIZING:
+                    if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+                        continue;
+                    break;
+
+                case PDMTHREADSTATE_TERMINATING:
+                case PDMTHREADSTATE_TERMINATED:
+                    break;
+
+                default:
+                    AssertMsgFailed(("enmState=%d\n", enmState));
+                    rc = VERR_PDM_THREAD_IPE_2;
+                    break;
+            }
+            break;
+        }
+    }
+    int rc2 = RTSemEventMultiSignal(pThread->Internal.s.BlockEvent);
+    AssertRC(rc2);
+
+    /*
+     * Wait for it to terminate and the do cleanups.
+     */
+    rc2 = RTThreadWait(pThread->Thread, RT_SUCCESS(rc) ? 60*1000 : 150, pRcThread);
+    if (RT_SUCCESS(rc2))
+    {
+        /* make it invalid. */
+        pThread->u32Version = 0xffffffff;
+        pThread->enmState = PDMTHREADSTATE_INVALID;
+        pThread->Thread = NIL_RTTHREAD;
+
+        /* unlink */
+        RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+        if (pUVM->pdm.s.pThreads == pThread)
+        {
+            pUVM->pdm.s.pThreads = pThread->Internal.s.pNext;
+            if (!pThread->Internal.s.pNext)
+                pUVM->pdm.s.pThreadsTail = NULL;
+        }
+        else
+        {
+            PPDMTHREAD pPrev = pUVM->pdm.s.pThreads;
+            while (pPrev && pPrev->Internal.s.pNext != pThread)
+                pPrev = pPrev->Internal.s.pNext;
+            Assert(pPrev);
+            if (pPrev)
+                pPrev->Internal.s.pNext = pThread->Internal.s.pNext;
+            if (!pThread->Internal.s.pNext)
+                pUVM->pdm.s.pThreadsTail = pPrev;
+        }
+        pThread->Internal.s.pNext = NULL;
+        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+        /* free the resources */
+        RTSemEventMultiDestroy(pThread->Internal.s.BlockEvent);
+        pThread->Internal.s.BlockEvent = NIL_RTSEMEVENTMULTI;
+
+        RTSemEventMultiDestroy(pThread->Internal.s.SleepEvent);
+        pThread->Internal.s.SleepEvent = NIL_RTSEMEVENTMULTI;
+
+        MMR3HeapFree(pThread);
+    }
+    else if (RT_SUCCESS(rc))
+        rc = rc2;
+
+    return rc;
+}
+
+
+/**
+ * Destroys all threads associated with a device.
+ *
+ * This function is called by PDMDevice when a device is
+ * destroyed (not currently implemented).
+ *
+ * @returns VBox status code of the first failure.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     the device instance.
+ */
+int pdmR3ThreadDestroyDevice(PVM pVM, PPDMDEVINS pDevIns)
+{
+    int     rc   = VINF_SUCCESS;
+    PUVM    pUVM = pVM->pUVM;
+
+    AssertPtr(pDevIns);
+
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMTHREAD pThread = pUVM->pdm.s.pThreads;
+    while (pThread)
+    {
+        PPDMTHREAD pNext = pThread->Internal.s.pNext;
+        if (    pThread->Internal.s.enmType == PDMTHREADTYPE_DEVICE
+            &&  pThread->u.Dev.pDevIns == pDevIns)
+        {
+            int rc2 = PDMR3ThreadDestroy(pThread, NULL);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+        }
+        pThread = pNext;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Destroys all threads associated with an USB device.
+ *
+ * This function is called by PDMUsb when a device is destroyed.
+ *
+ * @returns VBox status code of the first failure.
+ * @param   pVM         The cross context VM structure.
+ * @param   pUsbIns     The USB device instance.
+ */
+int pdmR3ThreadDestroyUsb(PVM pVM, PPDMUSBINS pUsbIns)
+{
+    int     rc   = VINF_SUCCESS;
+    PUVM    pUVM = pVM->pUVM;
+
+    AssertPtr(pUsbIns);
+
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMTHREAD pThread = pUVM->pdm.s.pThreads;
+    while (pThread)
+    {
+        PPDMTHREAD pNext = pThread->Internal.s.pNext;
+        if (    pThread->Internal.s.enmType == PDMTHREADTYPE_DEVICE
+            &&  pThread->u.Usb.pUsbIns == pUsbIns)
+        {
+            int rc2 = PDMR3ThreadDestroy(pThread, NULL);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+        }
+        pThread = pNext;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Destroys all threads associated with a driver.
+ *
+ * This function is called by PDMDriver when a driver is destroyed.
+ *
+ * @returns VBox status code of the first failure.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDrvIns     The driver instance.
+ */
+int pdmR3ThreadDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns)
+{
+    int     rc   = VINF_SUCCESS;
+    PUVM    pUVM = pVM->pUVM;
+
+    AssertPtr(pDrvIns);
+
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMTHREAD pThread = pUVM->pdm.s.pThreads;
+    while (pThread)
+    {
+        PPDMTHREAD pNext = pThread->Internal.s.pNext;
+        if (    pThread->Internal.s.enmType == PDMTHREADTYPE_DRIVER
+            &&  pThread->u.Drv.pDrvIns == pDrvIns)
+        {
+            int rc2 = PDMR3ThreadDestroy(pThread, NULL);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+        }
+        pThread = pNext;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Called For VM power off.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+void pdmR3ThreadDestroyAll(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMTHREAD pThread = pUVM->pdm.s.pThreads;
+    while (pThread)
+    {
+        PPDMTHREAD pNext = pThread->Internal.s.pNext;
+        int rc2 = PDMR3ThreadDestroy(pThread, NULL);
+        AssertRC(rc2);
+        pThread = pNext;
+    }
+    Assert(!pUVM->pdm.s.pThreads && !pUVM->pdm.s.pThreadsTail);
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+}
+
+
+/**
+ * Initiate termination of the thread (self) because something failed in a bad way.
+ *
+ * @param   pThread         The PDM thread.
+ */
+static void pdmR3ThreadBailMeOut(PPDMTHREAD pThread)
+{
+    for (;;)
+    {
+        PDMTHREADSTATE enmState = pThread->enmState;
+        switch (enmState)
+        {
+            case PDMTHREADSTATE_SUSPENDING:
+            case PDMTHREADSTATE_SUSPENDED:
+            case PDMTHREADSTATE_RESUMING:
+            case PDMTHREADSTATE_RUNNING:
+                if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+                    continue;
+                break;
+
+            case PDMTHREADSTATE_TERMINATING:
+            case PDMTHREADSTATE_TERMINATED:
+                break;
+
+            case PDMTHREADSTATE_INITIALIZING:
+            default:
+                AssertMsgFailed(("enmState=%d\n", enmState));
+                break;
+        }
+        break;
+    }
+}
+
+
+/**
+ * Called by the PDM thread in response to a wakeup call with
+ * suspending as the new state.
+ *
+ * The thread will block in side this call until the state is changed in
+ * response to a VM state change or to the device/driver/whatever calling the
+ * PDMR3ThreadResume API.
+ *
+ * @returns VBox status code.
+ *          On failure, terminate the thread.
+ * @param   pThread     The PDM thread.
+ */
+VMMR3DECL(int) PDMR3ThreadIAmSuspending(PPDMTHREAD pThread)
+{
+    /*
+     * Assert sanity.
+     */
+    AssertPtr(pThread);
+    AssertReturn(pThread->u32Version == PDMTHREAD_VERSION, VERR_INVALID_MAGIC);
+    Assert(pThread->Thread == RTThreadSelf() || pThread->enmState == PDMTHREADSTATE_INITIALIZING);
+    PDMTHREADSTATE enmState = pThread->enmState;
+    Assert(     enmState == PDMTHREADSTATE_SUSPENDING
+           ||   enmState == PDMTHREADSTATE_INITIALIZING);
+
+    /*
+     * Update the state, notify the control thread (the API caller) and go to sleep.
+     */
+    int rc = VERR_WRONG_ORDER;
+    if (pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_SUSPENDED, enmState))
+    {
+        rc = RTThreadUserSignal(pThread->Thread);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTSemEventMultiWait(pThread->Internal.s.BlockEvent, RT_INDEFINITE_WAIT);
+            if (    RT_SUCCESS(rc)
+                &&  pThread->enmState != PDMTHREADSTATE_SUSPENDED)
+                return rc;
+
+            if (RT_SUCCESS(rc))
+                rc = VERR_PDM_THREAD_IPE_2;
+        }
+    }
+
+    AssertMsgFailed(("rc=%d enmState=%d\n", rc, pThread->enmState));
+    pdmR3ThreadBailMeOut(pThread);
+    return rc;
+}
+
+
+/**
+ * Called by the PDM thread in response to a resuming state.
+ *
+ * The purpose of this API is to tell the PDMR3ThreadResume caller that
+ * the PDM thread has successfully resumed. It will also do the
+ * state transition from the resuming to the running state.
+ *
+ * @returns VBox status code.
+ *          On failure, terminate the thread.
+ * @param   pThread     The PDM thread.
+ */
+VMMR3DECL(int) PDMR3ThreadIAmRunning(PPDMTHREAD pThread)
+{
+    /*
+     * Assert sanity.
+     */
+    Assert(pThread->enmState == PDMTHREADSTATE_RESUMING);
+    Assert(pThread->Thread == RTThreadSelf());
+
+    /*
+     * Update the state and tell the control thread (the guy calling the resume API).
+     */
+    int rc = VERR_WRONG_ORDER;
+    if (pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_RUNNING, PDMTHREADSTATE_RESUMING))
+    {
+        rc = RTThreadUserSignal(pThread->Thread);
+        if (RT_SUCCESS(rc))
+            return rc;
+    }
+
+    AssertMsgFailed(("rc=%d enmState=%d\n", rc, pThread->enmState));
+    pdmR3ThreadBailMeOut(pThread);
+    return rc;
+}
+
+
+/**
+ * Called by the PDM thread instead of RTThreadSleep.
+ *
+ * The difference is that the sleep will be interrupted on state change. The
+ * thread must be in the running state, otherwise it will return immediately.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success or state change.
+ * @retval  VERR_INTERRUPTED on signal or APC.
+ *
+ * @param   pThread     The PDM thread.
+ * @param   cMillies    The number of milliseconds to sleep.
+ */
+VMMR3DECL(int) PDMR3ThreadSleep(PPDMTHREAD pThread, RTMSINTERVAL cMillies)
+{
+    /*
+     * Assert sanity.
+     */
+    AssertReturn(pThread->enmState > PDMTHREADSTATE_INVALID && pThread->enmState < PDMTHREADSTATE_TERMINATED, VERR_PDM_THREAD_IPE_2);
+    AssertReturn(pThread->Thread == RTThreadSelf(), VERR_PDM_THREAD_INVALID_CALLER);
+
+    /*
+     * Reset the event semaphore, check the state and sleep.
+     */
+    RTSemEventMultiReset(pThread->Internal.s.SleepEvent);
+    if (pThread->enmState != PDMTHREADSTATE_RUNNING)
+        return VINF_SUCCESS;
+    return RTSemEventMultiWaitNoResume(pThread->Internal.s.SleepEvent, cMillies);
+}
+
+
+/**
+ * The PDM thread function.
+ *
+ * @returns return from pfnThread.
+ *
+ * @param   Thread  The thread handle.
+ * @param   pvUser  Pointer to the PDMTHREAD structure.
+ */
+static DECLCALLBACK(int) pdmR3ThreadMain(RTTHREAD Thread, void *pvUser)
+{
+    PPDMTHREAD pThread = (PPDMTHREAD)pvUser;
+    Log(("PDMThread: Initializing thread %RTthrd / %p / '%s'...\n", Thread, pThread, RTThreadGetName(Thread)));
+    pThread->Thread = Thread;
+
+    PUVM pUVM = pThread->Internal.s.pVM->pUVM;
+    if (   pUVM->pVmm2UserMethods
+        && pUVM->pVmm2UserMethods->pfnNotifyPdmtInit)
+        pUVM->pVmm2UserMethods->pfnNotifyPdmtInit(pUVM->pVmm2UserMethods, pUVM);
+
+    /*
+     * The run loop.
+     *
+     * It handles simple thread functions which returns when they see a suspending
+     * request and leaves the PDMR3ThreadIAmSuspending and PDMR3ThreadIAmRunning
+     * parts to us.
+     */
+    int rc;
+    for (;;)
+    {
+        switch (pThread->Internal.s.enmType)
+        {
+            case PDMTHREADTYPE_DEVICE:
+                rc = pThread->u.Dev.pfnThread(pThread->u.Dev.pDevIns, pThread);
+                break;
+
+            case PDMTHREADTYPE_USB:
+                rc = pThread->u.Usb.pfnThread(pThread->u.Usb.pUsbIns, pThread);
+                break;
+
+            case PDMTHREADTYPE_DRIVER:
+                rc = pThread->u.Drv.pfnThread(pThread->u.Drv.pDrvIns, pThread);
+                break;
+
+            case PDMTHREADTYPE_INTERNAL:
+                rc = pThread->u.Int.pfnThread(pThread->Internal.s.pVM, pThread);
+                break;
+
+            case PDMTHREADTYPE_EXTERNAL:
+                rc = pThread->u.Ext.pfnThread(pThread);
+                break;
+
+            default:
+                AssertMsgFailed(("%d\n", pThread->Internal.s.enmType));
+                rc = VERR_PDM_THREAD_IPE_1;
+                break;
+        }
+        if (RT_FAILURE(rc))
+            break;
+
+        /*
+         * If this is a simple thread function, the state will be suspending
+         * or initializing now. If it isn't we're supposed to terminate.
+         */
+        if (    pThread->enmState != PDMTHREADSTATE_SUSPENDING
+            &&  pThread->enmState != PDMTHREADSTATE_INITIALIZING)
+        {
+            Assert(pThread->enmState == PDMTHREADSTATE_TERMINATING);
+            break;
+        }
+        rc = PDMR3ThreadIAmSuspending(pThread);
+        if (RT_FAILURE(rc))
+            break;
+        if (pThread->enmState != PDMTHREADSTATE_RESUMING)
+        {
+            Assert(pThread->enmState == PDMTHREADSTATE_TERMINATING);
+            break;
+        }
+
+        rc = PDMR3ThreadIAmRunning(pThread);
+        if (RT_FAILURE(rc))
+            break;
+    }
+
+    if (RT_FAILURE(rc))
+        LogRel(("PDMThread: Thread '%s' (%RTthrd) quit unexpectedly with rc=%Rrc.\n", RTThreadGetName(Thread), Thread, rc));
+
+    /*
+     * Advance the state to terminating and then on to terminated.
+     */
+    for (;;)
+    {
+        PDMTHREADSTATE enmState = pThread->enmState;
+        if (    enmState == PDMTHREADSTATE_TERMINATING
+            ||  pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+            break;
+    }
+
+    ASMAtomicXchgSize(&pThread->enmState, PDMTHREADSTATE_TERMINATED);
+    int rc2 = RTThreadUserSignal(Thread); AssertRC(rc2);
+
+    if (   pUVM->pVmm2UserMethods
+        && pUVM->pVmm2UserMethods->pfnNotifyPdmtTerm)
+        pUVM->pVmm2UserMethods->pfnNotifyPdmtTerm(pUVM->pVmm2UserMethods, pUVM);
+    Log(("PDMThread: Terminating thread %RTthrd / %p / '%s': %Rrc\n", Thread, pThread, RTThreadGetName(Thread), rc));
+    return rc;
+}
+
+
+/**
+ * Initiate termination of the thread because something failed in a bad way.
+ *
+ * @param   pThread         The PDM thread.
+ */
+static void pdmR3ThreadBailOut(PPDMTHREAD pThread)
+{
+    for (;;)
+    {
+        PDMTHREADSTATE enmState = pThread->enmState;
+        switch (enmState)
+        {
+            case PDMTHREADSTATE_SUSPENDING:
+            case PDMTHREADSTATE_SUSPENDED:
+                if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+                    continue;
+                RTSemEventMultiSignal(pThread->Internal.s.BlockEvent);
+                break;
+
+            case PDMTHREADSTATE_RESUMING:
+                if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+                    continue;
+                break;
+
+            case PDMTHREADSTATE_RUNNING:
+                if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+                    continue;
+                pdmR3ThreadWakeUp(pThread);
+                break;
+
+            case PDMTHREADSTATE_TERMINATING:
+            case PDMTHREADSTATE_TERMINATED:
+                break;
+
+            case PDMTHREADSTATE_INITIALIZING:
+            default:
+                AssertMsgFailed(("enmState=%d\n", enmState));
+                break;
+        }
+        break;
+    }
+}
+
+
+/**
+ * Suspends the thread.
+ *
+ * This can be called at the power off / suspend notifications to suspend the
+ * PDM thread a bit early. The thread will be automatically suspend upon
+ * completion of the device/driver notification cycle.
+ *
+ * The caller is responsible for serializing the control operations on the
+ * thread. That basically means, always do these calls from the EMT.
+ *
+ * @returns VBox status code.
+ * @param   pThread     The PDM thread.
+ */
+VMMR3DECL(int) PDMR3ThreadSuspend(PPDMTHREAD pThread)
+{
+    /*
+     * Assert sanity.
+     */
+    AssertPtrReturn(pThread, VERR_INVALID_POINTER);
+    AssertReturn(pThread->u32Version == PDMTHREAD_VERSION, VERR_INVALID_MAGIC);
+    Assert(pThread->Thread != RTThreadSelf());
+
+    /*
+     * This is a noop if the thread is already suspended.
+     */
+    if (pThread->enmState == PDMTHREADSTATE_SUSPENDED)
+        return VINF_SUCCESS;
+
+    /*
+     * Change the state to resuming and kick the thread.
+     */
+    int rc = RTSemEventMultiReset(pThread->Internal.s.BlockEvent);
+    if (RT_SUCCESS(rc))
+    {
+        rc = RTThreadUserReset(pThread->Thread);
+        if (RT_SUCCESS(rc))
+        {
+            rc = VERR_WRONG_ORDER;
+            if (pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_SUSPENDING, PDMTHREADSTATE_RUNNING))
+            {
+                rc = pdmR3ThreadWakeUp(pThread);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Wait for the thread to reach the suspended state.
+                     */
+                    if (pThread->enmState != PDMTHREADSTATE_SUSPENDED)
+                        rc = RTThreadUserWait(pThread->Thread, 60*1000);
+                    if (    RT_SUCCESS(rc)
+                        &&  pThread->enmState != PDMTHREADSTATE_SUSPENDED)
+                        rc = VERR_PDM_THREAD_IPE_2;
+                    if (RT_SUCCESS(rc))
+                        return rc;
+                }
+            }
+        }
+    }
+
+    /*
+     * Something failed, initialize termination.
+     */
+    AssertMsgFailed(("PDMR3ThreadSuspend -> rc=%Rrc enmState=%d suspending '%s'\n",
+                     rc, pThread->enmState, RTThreadGetName(pThread->Thread)));
+    pdmR3ThreadBailOut(pThread);
+    return rc;
+}
+
+
+/**
+ * Suspend all running threads.
+ *
+ * This is called by PDMR3Suspend() and PDMR3PowerOff() after all the devices
+ * and drivers have been notified about the suspend / power off.
+ *
+ * @return VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+int pdmR3ThreadSuspendAll(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect); /* This may cause deadlocks later... */
+    for (PPDMTHREAD pThread = pUVM->pdm.s.pThreads; pThread; pThread = pThread->Internal.s.pNext)
+        switch (pThread->enmState)
+        {
+            case PDMTHREADSTATE_RUNNING:
+            {
+                int rc = PDMR3ThreadSuspend(pThread);
+                AssertLogRelMsgReturnStmt(RT_SUCCESS(rc),
+                                          ("PDMR3ThreadSuspend -> %Rrc for '%s'\n", rc, RTThreadGetName(pThread->Thread)),
+                                          RTCritSectLeave(&pUVM->pdm.s.ListCritSect),
+                                          rc);
+                break;
+            }
+
+            /* suspend -> power off; voluntary suspend. */
+            case PDMTHREADSTATE_SUSPENDED:
+                break;
+
+            default:
+                AssertMsgFailed(("pThread=%p enmState=%d\n", pThread, pThread->enmState));
+                break;
+        }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resumes the thread.
+ *
+ * This can be called the power on / resume notifications to resume the
+ * PDM thread a bit early. The thread will be automatically resumed upon
+ * return from these two notification callbacks (devices/drivers).
+ *
+ * The caller is responsible for serializing the control operations on the
+ * thread. That basically means, always do these calls from the EMT.
+ *
+ * @returns VBox status code.
+ * @param   pThread     The PDM thread.
+ */
+VMMR3DECL(int) PDMR3ThreadResume(PPDMTHREAD pThread)
+{
+    /*
+     * Assert sanity.
+     */
+    AssertPtrReturn(pThread, VERR_INVALID_POINTER);
+    AssertReturn(pThread->u32Version == PDMTHREAD_VERSION, VERR_INVALID_MAGIC);
+    Assert(pThread->Thread != RTThreadSelf());
+
+    /*
+     * Change the state to resuming and kick the thread.
+     */
+    int rc = RTThreadUserReset(pThread->Thread);
+    if (RT_SUCCESS(rc))
+    {
+        rc = VERR_WRONG_ORDER;
+        if (pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_RESUMING, PDMTHREADSTATE_SUSPENDED))
+        {
+            rc = RTSemEventMultiSignal(pThread->Internal.s.BlockEvent);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Wait for the thread to reach the running state.
+                 */
+                rc = RTThreadUserWait(pThread->Thread, 60*1000);
+                if (    RT_SUCCESS(rc)
+                    &&  pThread->enmState != PDMTHREADSTATE_RUNNING)
+                    rc = VERR_PDM_THREAD_IPE_2;
+                if (RT_SUCCESS(rc))
+                    return rc;
+            }
+        }
+    }
+
+    /*
+     * Something failed, initialize termination.
+     */
+    AssertMsgFailed(("PDMR3ThreadResume -> rc=%Rrc enmState=%d\n", rc, pThread->enmState));
+    pdmR3ThreadBailOut(pThread);
+    return rc;
+}
+
+
+/**
+ * Resumes all threads not running.
+ *
+ * This is called by PDMR3Resume() and PDMR3PowerOn() after all the devices
+ * and drivers have been notified about the resume / power on .
+ *
+ * @return VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+int pdmR3ThreadResumeAll(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMTHREAD pThread = pUVM->pdm.s.pThreads; pThread; pThread = pThread->Internal.s.pNext)
+        switch (pThread->enmState)
+        {
+            case PDMTHREADSTATE_SUSPENDED:
+            {
+                int rc = PDMR3ThreadResume(pThread);
+                AssertRCReturn(rc, rc);
+                break;
+            }
+
+            default:
+                AssertMsgFailed(("pThread=%p enmState=%d\n", pThread, pThread->enmState));
+                break;
+        }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMR3/PDMUsb.cpp b/src/VBox/VMM/VMMR3/PDMUsb.cpp
new file mode 100644
index 00000000..274da2dd
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PDMUsb.cpp
@@ -0,0 +1,2004 @@
+/* $Id: PDMUsb.cpp $ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, USB part.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DRIVER
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vusb.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/version.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/string.h>
+#include <iprt/asm.h>
+#include <iprt/alloc.h>
+#include <iprt/alloca.h>
+#include <iprt/path.h>
+#include <iprt/uuid.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Internal callback structure pointer.
+ *
+ * The main purpose is to define the extra data we associate
+ * with PDMUSBREGCB so we can find the VM instance and so on.
+ */
+typedef struct PDMUSBREGCBINT
+{
+    /** The callback structure. */
+    PDMUSBREGCB     Core;
+    /** A bit of padding. */
+    uint32_t        u32[4];
+    /** VM Handle. */
+    PVM             pVM;
+} PDMUSBREGCBINT, *PPDMUSBREGCBINT;
+typedef const PDMUSBREGCBINT *PCPDMUSBREGCBINT;
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** @def PDMUSB_ASSERT_USBINS
+ * Asserts the validity of the USB device instance.
+ */
+#ifdef VBOX_STRICT
+# define PDMUSB_ASSERT_USBINS(pUsbIns) \
+    do { \
+        AssertPtr(pUsbIns); \
+        Assert(pUsbIns->u32Version == PDM_USBINS_VERSION); \
+        Assert(pUsbIns->pvInstanceDataR3 == (void *)&pUsbIns->achInstanceData[0]); \
+    } while (0)
+#else
+# define PDMUSB_ASSERT_USBINS(pUsbIns)   do { } while (0)
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static void pdmR3UsbDestroyDevice(PVM pVM, PPDMUSBINS pUsbIns);
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+extern const PDMUSBHLP g_pdmR3UsbHlp;
+
+
+AssertCompile(sizeof(PDMUSBINSINT) <= RT_SIZEOFMEMB(PDMUSBINS, Internal.padding));
+
+
+/**
+ * Registers a USB hub driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDrvIns         The driver instance of the hub.
+ * @param   fVersions       Indicates the kinds of USB devices that can be attached to this HUB.
+ * @param   cPorts          The number of ports.
+ * @param   pUsbHubReg      The hub callback structure that PDMUsb uses to interact with it.
+ * @param   ppUsbHubHlp     The helper callback structure that the hub uses to talk to PDMUsb.
+ * @thread  EMT
+ */
+int pdmR3UsbRegisterHub(PVM pVM, PPDMDRVINS pDrvIns, uint32_t fVersions, uint32_t cPorts, PCPDMUSBHUBREG pUsbHubReg, PPCPDMUSBHUBHLP ppUsbHubHlp)
+{
+    /*
+     * Validate input.
+     */
+    /* The driver must be in the USB class. */
+    if (!(pDrvIns->pReg->fClass & PDM_DRVREG_CLASS_USB))
+    {
+        LogRel(("PDMUsb: pdmR3UsbRegisterHub: fClass=%#x expected %#x to be set\n", pDrvIns->pReg->fClass, PDM_DRVREG_CLASS_USB));
+        return VERR_INVALID_PARAMETER;
+    }
+    AssertMsgReturn(!(fVersions & ~(VUSB_STDVER_11 | VUSB_STDVER_20 | VUSB_STDVER_30)), ("%#x\n", fVersions), VERR_INVALID_PARAMETER);
+    AssertPtrReturn(ppUsbHubHlp, VERR_INVALID_POINTER);
+    AssertPtrReturn(pUsbHubReg, VERR_INVALID_POINTER);
+    AssertReturn(pUsbHubReg->u32Version == PDM_USBHUBREG_VERSION, VERR_INVALID_MAGIC);
+    AssertReturn(pUsbHubReg->u32TheEnd == PDM_USBHUBREG_VERSION, VERR_INVALID_MAGIC);
+    AssertPtrReturn(pUsbHubReg->pfnAttachDevice, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pUsbHubReg->pfnDetachDevice, VERR_INVALID_PARAMETER);
+
+    /*
+     * Check for duplicate registration and find the last hub for FIFO registration.
+     */
+    PPDMUSBHUB pPrev = NULL;
+    for (PPDMUSBHUB pCur = pVM->pdm.s.pUsbHubs; pCur; pCur = pCur->pNext)
+    {
+        if (pCur->pDrvIns == pDrvIns)
+            return VERR_PDM_USB_HUB_EXISTS;
+        pPrev = pCur;
+    }
+
+    /*
+     * Create an internal USB hub structure.
+     */
+    PPDMUSBHUB pHub = (PPDMUSBHUB)MMR3HeapAlloc(pVM, MM_TAG_PDM_DRIVER, sizeof(*pHub));
+    if (!pHub)
+        return VERR_NO_MEMORY;
+
+    pHub->fVersions = fVersions;
+    pHub->cPorts = cPorts;
+    pHub->cAvailablePorts = cPorts;
+    pHub->pDrvIns = pDrvIns;
+    pHub->Reg = *pUsbHubReg;
+    pHub->pNext = NULL;
+
+    /* link it */
+    if (pPrev)
+        pPrev->pNext = pHub;
+    else
+        pVM->pdm.s.pUsbHubs = pHub;
+
+    Log(("PDM: Registered USB hub %p/%s\n", pDrvIns, pDrvIns->pReg->szName));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads one device module and call the registration entry point.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pRegCB          The registration callback stuff.
+ * @param   pszFilename     Module filename.
+ * @param   pszName         Module name.
+ */
+static int pdmR3UsbLoad(PVM pVM, PCPDMUSBREGCBINT pRegCB, const char *pszFilename, const char *pszName)
+{
+    /*
+     * Load it.
+     */
+    int rc = pdmR3LoadR3U(pVM->pUVM, pszFilename, pszName);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Get the registration export and call it.
+         */
+        FNPDMVBOXUSBREGISTER *pfnVBoxUsbRegister;
+        rc = PDMR3LdrGetSymbolR3(pVM, pszName, "VBoxUsbRegister", (void **)&pfnVBoxUsbRegister);
+        if (RT_SUCCESS(rc))
+        {
+            Log(("PDM: Calling VBoxUsbRegister (%p) of %s (%s)\n", pfnVBoxUsbRegister, pszName, pszFilename));
+            rc = pfnVBoxUsbRegister(&pRegCB->Core, VBOX_VERSION);
+            if (RT_SUCCESS(rc))
+                Log(("PDM: Successfully loaded device module %s (%s).\n", pszName, pszFilename));
+            else
+                AssertMsgFailed(("VBoxDevicesRegister failed with rc=%Rrc for module %s (%s)\n", rc, pszName, pszFilename));
+        }
+        else
+        {
+            AssertMsgFailed(("Failed to locate 'VBoxUsbRegister' in %s (%s) rc=%Rrc\n", pszName, pszFilename, rc));
+            if (rc == VERR_SYMBOL_NOT_FOUND)
+                rc = VERR_PDM_NO_REGISTRATION_EXPORT;
+        }
+    }
+    else
+        AssertMsgFailed(("Failed to load VBoxDD!\n"));
+    return rc;
+}
+
+
+
+/**
+ * @interface_method_impl{PDMUSBREGCB,pfnRegister}
+ */
+static DECLCALLBACK(int) pdmR3UsbReg_Register(PCPDMUSBREGCB pCallbacks, PCPDMUSBREG pReg)
+{
+    /*
+     * Validate the registration structure.
+     */
+    Assert(pReg);
+    AssertMsgReturn(pReg->u32Version == PDM_USBREG_VERSION,
+                    ("Unknown struct version %#x!\n", pReg->u32Version),
+                    VERR_PDM_UNKNOWN_USBREG_VERSION);
+    AssertMsgReturn(    pReg->szName[0]
+                    &&  strlen(pReg->szName) < sizeof(pReg->szName)
+                    &&  pdmR3IsValidName(pReg->szName),
+                    ("Invalid name '%.*s'\n", sizeof(pReg->szName), pReg->szName),
+                    VERR_PDM_INVALID_USB_REGISTRATION);
+    AssertMsgReturn((pReg->fFlags & ~(PDM_USBREG_HIGHSPEED_CAPABLE | PDM_USBREG_SUPERSPEED_CAPABLE | PDM_USBREG_SAVED_STATE_SUPPORTED)) == 0,
+                    ("fFlags=%#x\n", pReg->fFlags), VERR_PDM_INVALID_USB_REGISTRATION);
+    AssertMsgReturn(pReg->cMaxInstances > 0,
+                    ("Max instances %u! (USB Device %s)\n", pReg->cMaxInstances, pReg->szName),
+                    VERR_PDM_INVALID_USB_REGISTRATION);
+    AssertMsgReturn(pReg->cbInstance <= _1M,
+                    ("Instance size %d bytes! (USB Device %s)\n", pReg->cbInstance, pReg->szName),
+                    VERR_PDM_INVALID_USB_REGISTRATION);
+    AssertMsgReturn(pReg->pfnConstruct, ("No constructor! (USB Device %s)\n", pReg->szName),
+                    VERR_PDM_INVALID_USB_REGISTRATION);
+
+    /*
+     * Check for duplicate and find FIFO entry at the same time.
+     */
+    PCPDMUSBREGCBINT pRegCB = (PCPDMUSBREGCBINT)pCallbacks;
+    PPDMUSB pUsbPrev = NULL;
+    PPDMUSB pUsb = pRegCB->pVM->pdm.s.pUsbDevs;
+    for (; pUsb; pUsbPrev = pUsb, pUsb = pUsb->pNext)
+        AssertMsgReturn(strcmp(pUsb->pReg->szName, pReg->szName),
+                        ("USB Device '%s' already exists\n", pReg->szName),
+                        VERR_PDM_USB_NAME_CLASH);
+
+    /*
+     * Allocate new device structure and insert it into the list.
+     */
+    pUsb = (PPDMUSB)MMR3HeapAlloc(pRegCB->pVM, MM_TAG_PDM_DEVICE, sizeof(*pUsb));
+    if (pUsb)
+    {
+        pUsb->pNext = NULL;
+        pUsb->iNextInstance = 0;
+        pUsb->pInstances = NULL;
+        pUsb->pReg = pReg;
+        pUsb->cchName = (RTUINT)strlen(pReg->szName);
+
+        if (pUsbPrev)
+            pUsbPrev->pNext = pUsb;
+        else
+            pRegCB->pVM->pdm.s.pUsbDevs = pUsb;
+        Log(("PDM: Registered USB device '%s'\n", pReg->szName));
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Load USB Device modules.
+ *
+ * This is called by pdmR3DevInit() after it has loaded it's device modules.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+int pdmR3UsbLoadModules(PVM pVM)
+{
+    LogFlow(("pdmR3UsbLoadModules:\n"));
+
+    AssertRelease(!(RT_UOFFSETOF(PDMUSBINS, achInstanceData) & 15));
+    AssertRelease(sizeof(pVM->pdm.s.pUsbInstances->Internal.s) <= sizeof(pVM->pdm.s.pUsbInstances->Internal.padding));
+
+    /*
+     * Initialize the callback structure.
+     */
+    PDMUSBREGCBINT RegCB;
+    RegCB.Core.u32Version = PDM_USBREG_CB_VERSION;
+    RegCB.Core.pfnRegister = pdmR3UsbReg_Register;
+    RegCB.pVM = pVM;
+
+    /*
+     * Load the builtin module
+     */
+    PCFGMNODE pUsbNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM/USB/");
+    bool fLoadBuiltin;
+    int rc = CFGMR3QueryBool(pUsbNode, "LoadBuiltin", &fLoadBuiltin);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+        fLoadBuiltin = true;
+    else if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Configuration error: Querying boolean \"LoadBuiltin\" failed with %Rrc\n", rc));
+        return rc;
+    }
+    if (fLoadBuiltin)
+    {
+        /* make filename */
+        char *pszFilename = pdmR3FileR3("VBoxDD", true /*fShared*/);
+        if (!pszFilename)
+            return VERR_NO_TMP_MEMORY;
+        rc = pdmR3UsbLoad(pVM, &RegCB, pszFilename, "VBoxDD");
+        RTMemTmpFree(pszFilename);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Load additional device modules.
+     */
+    PCFGMNODE pCur;
+    for (pCur = CFGMR3GetFirstChild(pUsbNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+    {
+        /*
+         * Get the name and path.
+         */
+        char szName[PDMMOD_NAME_LEN];
+        rc = CFGMR3GetName(pCur, &szName[0], sizeof(szName));
+        if (rc == VERR_CFGM_NOT_ENOUGH_SPACE)
+        {
+            AssertMsgFailed(("configuration error: The module name is too long, cchName=%zu.\n", CFGMR3GetNameLen(pCur)));
+            return VERR_PDM_MODULE_NAME_TOO_LONG;
+        }
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("CFGMR3GetName -> %Rrc.\n", rc));
+            return rc;
+        }
+
+        /* the path is optional, if no path the module name + path is used. */
+        char szFilename[RTPATH_MAX];
+        rc = CFGMR3QueryString(pCur, "Path", &szFilename[0], sizeof(szFilename));
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+            strcpy(szFilename, szName);
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("configuration error: Failure to query the module path, rc=%Rrc.\n", rc));
+            return rc;
+        }
+
+        /* prepend path? */
+        if (!RTPathHavePath(szFilename))
+        {
+            char *psz = pdmR3FileR3(szFilename, false /*fShared*/);
+            if (!psz)
+                return VERR_NO_TMP_MEMORY;
+            size_t cch = strlen(psz) + 1;
+            if (cch > sizeof(szFilename))
+            {
+                RTMemTmpFree(psz);
+                AssertMsgFailed(("Filename too long! cch=%d '%s'\n", cch, psz));
+                return VERR_FILENAME_TOO_LONG;
+            }
+            memcpy(szFilename, psz, cch);
+            RTMemTmpFree(psz);
+        }
+
+        /*
+         * Load the module and register it's devices.
+         */
+        rc = pdmR3UsbLoad(pVM, &RegCB, szFilename, szName);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Send the init-complete notification to all the USB devices.
+ *
+ * This is called from pdmR3DevInit() after it has do its notification round.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+int pdmR3UsbVMInitComplete(PVM pVM)
+{
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        if (pUsbIns->pReg->pfnVMInitComplete)
+        {
+            int rc = pUsbIns->pReg->pfnVMInitComplete(pUsbIns);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("InitComplete on USB device '%s'/%d failed with rc=%Rrc\n",
+                                 pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+                return rc;
+            }
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Lookups a device structure by name.
+ * @internal
+ */
+PPDMUSB pdmR3UsbLookup(PVM pVM, const char *pszName)
+{
+    size_t cchName = strlen(pszName);
+    for (PPDMUSB pUsb = pVM->pdm.s.pUsbDevs; pUsb; pUsb = pUsb->pNext)
+        if (    pUsb->cchName == cchName
+            &&  !strcmp(pUsb->pReg->szName, pszName))
+            return pUsb;
+    return NULL;
+}
+
+
+/**
+ * Locates a suitable hub for the specified kind of device.
+ *
+ * @returns VINF_SUCCESS and *ppHub on success.
+ *          VERR_PDM_NO_USB_HUBS or VERR_PDM_NO_USB_PORTS on failure.
+ * @param   pVM             The cross context VM structure.
+ * @param   iUsbVersion     The USB device version.
+ * @param   ppHub           Where to store the pointer to the USB hub.
+ */
+static int pdmR3UsbFindHub(PVM pVM, uint32_t iUsbVersion, PPDMUSBHUB *ppHub)
+{
+    *ppHub = NULL;
+    if (!pVM->pdm.s.pUsbHubs)
+        return VERR_PDM_NO_USB_HUBS;
+
+    for (PPDMUSBHUB pCur = pVM->pdm.s.pUsbHubs; pCur; pCur = pCur->pNext)
+        if (pCur->cAvailablePorts > 0)
+        {
+            /* First check for an exact match. */
+            if (pCur->fVersions & iUsbVersion)
+            {
+                *ppHub = pCur;
+                break;
+            }
+            /* For high-speed USB 2.0 devices only, allow USB 1.1 fallback. */
+            if ((iUsbVersion & VUSB_STDVER_20) && (pCur->fVersions == VUSB_STDVER_11))
+                *ppHub = pCur;
+        }
+    if (*ppHub)
+        return VINF_SUCCESS;
+    return VERR_PDM_NO_USB_PORTS;
+}
+
+
+/**
+ * Translates a USB version (a bit-mask) to USB speed (enum). Picks
+ * the highest available version.
+ *
+ * @returns VUSBSPEED enum
+ *
+ * @param   iUsbVersion     The USB version.
+ *
+ */
+static VUSBSPEED pdmR3UsbVer2Spd(uint32_t iUsbVersion)
+{
+    VUSBSPEED   enmSpd = VUSB_SPEED_UNKNOWN;
+    Assert(iUsbVersion);
+
+    if (iUsbVersion & VUSB_STDVER_30)
+        enmSpd = VUSB_SPEED_SUPER;
+    else if (iUsbVersion & VUSB_STDVER_20)
+        enmSpd = VUSB_SPEED_HIGH;
+    else if (iUsbVersion & VUSB_STDVER_11)
+        enmSpd = VUSB_SPEED_FULL;    /* Can't distinguish LS vs. FS. */
+
+    return enmSpd;
+}
+
+
+/**
+ * Translates a USB speed (enum) to USB version.
+ *
+ * @returns USB version mask
+ *
+ * @param   enmSpeed        The USB connection speed.
+ *
+ */
+static uint32_t pdmR3UsbSpd2Ver(VUSBSPEED enmSpeed)
+{
+    uint32_t    iUsbVersion = 0;
+    Assert(enmSpeed != VUSB_SPEED_UNKNOWN);
+
+    switch (enmSpeed)
+    {
+    case VUSB_SPEED_LOW:
+    case VUSB_SPEED_FULL:
+        iUsbVersion = VUSB_STDVER_11;
+        break;
+    case VUSB_SPEED_HIGH:
+        iUsbVersion = VUSB_STDVER_20;
+        break;
+    case VUSB_SPEED_SUPER:
+    case VUSB_SPEED_SUPERPLUS:
+    default:
+        iUsbVersion = VUSB_STDVER_30;
+        break;
+    }
+
+    return iUsbVersion;
+}
+
+
+/**
+ * Creates the device.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pHub                The USB hub it'll be attached to.
+ * @param   pUsbDev             The USB device emulation.
+ * @param   iInstance           -1 if not called by pdmR3UsbInstantiateDevices().
+ * @param   pUuid               The UUID for this device.
+ * @param   ppInstanceNode      Pointer to the device instance pointer. This is set to NULL if inserted
+ *                              into the tree or cleaned up.
+ *
+ *                              In the pdmR3UsbInstantiateDevices() case (iInstance != -1) this is
+ *                              the actual instance node and will not be cleaned up.
+ *
+ * @param   enmSpeed            The speed the USB device is operating at.
+ * @param   pszCaptureFilename  Path to the file for USB traffic capturing, optional.
+ */
+static int pdmR3UsbCreateDevice(PVM pVM, PPDMUSBHUB pHub, PPDMUSB pUsbDev, int iInstance, PCRTUUID pUuid,
+                                PCFGMNODE *ppInstanceNode, VUSBSPEED enmSpeed, const char *pszCaptureFilename)
+{
+    const bool fAtRuntime = iInstance == -1;
+    int rc;
+
+    AssertPtrReturn(ppInstanceNode, VERR_INVALID_POINTER);
+    AssertPtrReturn(*ppInstanceNode, VERR_INVALID_POINTER);
+
+    /*
+     * If not called by pdmR3UsbInstantiateDevices(), we'll have to fix
+     * the configuration now.
+     */
+    /* USB device node. */
+    PCFGMNODE pDevNode = CFGMR3GetChildF(CFGMR3GetRoot(pVM), "USB/%s/", pUsbDev->pReg->szName);
+    if (!pDevNode)
+    {
+        rc = CFGMR3InsertNodeF(CFGMR3GetRoot(pVM), &pDevNode, "USB/%s/", pUsbDev->pReg->szName);
+        AssertRCReturn(rc, rc);
+    }
+
+    /* The instance node and number. */
+    PCFGMNODE pInstanceToDelete = NULL;
+    PCFGMNODE pInstanceNode = NULL;
+    if (fAtRuntime)
+    {
+        /** @todo r=bird: This code is bogus as it ASSUMES that all USB devices are
+         *        capable of infinite number of instances. */
+        rc = VINF_SUCCESS; /* Shut up stupid incorrect uninitialized warning from Visual C++ 2010. */
+        for (unsigned c = 0; c < _2M; c++)
+        {
+            iInstance = pUsbDev->iNextInstance++;
+            rc = CFGMR3InsertNodeF(pDevNode, &pInstanceNode, "%d/", iInstance);
+            if (rc != VERR_CFGM_NODE_EXISTS)
+                break;
+        }
+        AssertRCReturn(rc, rc);
+
+        rc = CFGMR3ReplaceSubTree(pInstanceNode, *ppInstanceNode);
+        AssertRCReturn(rc, rc);
+        *ppInstanceNode = NULL;
+        pInstanceToDelete = pInstanceNode;
+    }
+    else
+    {
+        Assert(iInstance >= 0);
+        if (iInstance >= (int)pUsbDev->iNextInstance)
+            pUsbDev->iNextInstance = iInstance + 1;
+        pInstanceNode = *ppInstanceNode;
+    }
+
+    /* Make sure the instance config node exists. */
+    PCFGMNODE pConfig = CFGMR3GetChild(pInstanceNode, "Config");
+    if (!pConfig)
+    {
+        rc = CFGMR3InsertNode(pInstanceNode, "Config", &pConfig);
+        AssertRCReturn(rc, rc);
+    }
+    Assert(CFGMR3GetChild(pInstanceNode, "Config") == pConfig);
+
+    /* The global device config node. */
+    PCFGMNODE pGlobalConfig = CFGMR3GetChild(pDevNode, "GlobalConfig");
+    if (!pGlobalConfig)
+    {
+        rc = CFGMR3InsertNode(pDevNode, "GlobalConfig", &pGlobalConfig);
+        if (RT_FAILURE(rc))
+        {
+            CFGMR3RemoveNode(pInstanceToDelete);
+            AssertRCReturn(rc, rc);
+        }
+    }
+
+    /*
+     * Allocate the device instance.
+     */
+    size_t cb = RT_UOFFSETOF_DYN(PDMUSBINS, achInstanceData[pUsbDev->pReg->cbInstance]);
+    cb = RT_ALIGN_Z(cb, 16);
+    PPDMUSBINS pUsbIns;
+    rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_USB, cb, (void **)&pUsbIns);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Failed to allocate %d bytes of instance data for USB device '%s'. rc=%Rrc\n",
+                         cb, pUsbDev->pReg->szName, rc));
+        CFGMR3RemoveNode(pInstanceToDelete);
+        return rc;
+    }
+
+    /*
+     * Initialize it.
+     */
+    pUsbIns->u32Version                     = PDM_USBINS_VERSION;
+    //pUsbIns->Internal.s.pNext               = NULL;
+    //pUsbIns->Internal.s.pPerDeviceNext      = NULL;
+    pUsbIns->Internal.s.pUsbDev             = pUsbDev;
+    pUsbIns->Internal.s.pVM                 = pVM;
+    //pUsbIns->Internal.s.pLuns               = NULL;
+    pUsbIns->Internal.s.pCfg                = pInstanceNode;
+    pUsbIns->Internal.s.pCfgDelete          = pInstanceToDelete;
+    pUsbIns->Internal.s.pCfgGlobal          = pGlobalConfig;
+    pUsbIns->Internal.s.Uuid                = *pUuid;
+    //pUsbIns->Internal.s.pHub                = NULL;
+    pUsbIns->Internal.s.iPort               = UINT32_MAX; /* to be determined. */
+    /* Set the flag accordingly.
+     * Otherwise VMPowerOff, VMSuspend will not be called for devices attached at runtime.
+     */
+    pUsbIns->Internal.s.fVMSuspended        = !fAtRuntime;
+    //pUsbIns->Internal.s.pfnAsyncNotify      = NULL;
+    pUsbIns->pHlpR3                         = &g_pdmR3UsbHlp;
+    pUsbIns->pReg                           = pUsbDev->pReg;
+    pUsbIns->pCfg                           = pConfig;
+    pUsbIns->pCfgGlobal                     = pGlobalConfig;
+    pUsbIns->iInstance                      = iInstance;
+    pUsbIns->pvInstanceDataR3               = &pUsbIns->achInstanceData[0];
+    pUsbIns->pszName                        = RTStrDup(pUsbDev->pReg->szName);
+    //pUsbIns->fTracing                       = 0;
+    pUsbIns->idTracing                      = ++pVM->pdm.s.idTracingOther;
+    pUsbIns->enmSpeed                       = enmSpeed;
+
+    /*
+     * Link it into all the lists.
+     */
+    /* The global instance FIFO. */
+    PPDMUSBINS pPrev1 = pVM->pdm.s.pUsbInstances;
+    if (!pPrev1)
+        pVM->pdm.s.pUsbInstances = pUsbIns;
+    else
+    {
+        while (pPrev1->Internal.s.pNext)
+        {
+            Assert(pPrev1->u32Version == PDM_USBINS_VERSION);
+            pPrev1 = pPrev1->Internal.s.pNext;
+        }
+        pPrev1->Internal.s.pNext = pUsbIns;
+    }
+
+    /* The per device instance FIFO. */
+    PPDMUSBINS pPrev2 = pUsbDev->pInstances;
+    if (!pPrev2)
+        pUsbDev->pInstances = pUsbIns;
+    else
+    {
+        while (pPrev2->Internal.s.pPerDeviceNext)
+        {
+            Assert(pPrev2->u32Version == PDM_USBINS_VERSION);
+            pPrev2 = pPrev2->Internal.s.pPerDeviceNext;
+        }
+        pPrev2->Internal.s.pPerDeviceNext = pUsbIns;
+    }
+
+    /*
+     * Call the constructor.
+     */
+    Log(("PDM: Constructing USB device '%s' instance %d...\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+    rc = pUsbIns->pReg->pfnConstruct(pUsbIns, pUsbIns->iInstance, pUsbIns->pCfg, pUsbIns->pCfgGlobal);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Attach it to the hub.
+         */
+        Log(("PDM: Attaching it...\n"));
+        rc = pHub->Reg.pfnAttachDevice(pHub->pDrvIns, pUsbIns, pszCaptureFilename, &pUsbIns->Internal.s.iPort);
+        if (RT_SUCCESS(rc))
+        {
+            pHub->cAvailablePorts--;
+            Assert((int32_t)pHub->cAvailablePorts >= 0 && pHub->cAvailablePorts < pHub->cPorts);
+            pUsbIns->Internal.s.pHub = pHub;
+
+            /* Send the hot-plugged notification if applicable. */
+            if (fAtRuntime && pUsbIns->pReg->pfnHotPlugged)
+                pUsbIns->pReg->pfnHotPlugged(pUsbIns);
+
+            Log(("PDM: Successfully attached USB device '%s' instance %d to hub %p\n",
+                 pUsbIns->pReg->szName, pUsbIns->iInstance, pHub));
+            return VINF_SUCCESS;
+        }
+
+        LogRel(("PDMUsb: Failed to attach USB device '%s' instance %d to hub %p: %Rrc\n",
+                pUsbIns->pReg->szName, pUsbIns->iInstance, pHub, rc));
+    }
+    else
+    {
+        AssertMsgFailed(("Failed to construct '%s'/%d! %Rra\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+        if (rc == VERR_VERSION_MISMATCH)
+            rc = VERR_PDM_USBDEV_VERSION_MISMATCH;
+    }
+    if (fAtRuntime)
+        pdmR3UsbDestroyDevice(pVM, pUsbIns);
+    /* else: destructors are invoked later. */
+    return rc;
+}
+
+
+/**
+ * Instantiate USB devices.
+ *
+ * This is called by pdmR3DevInit() after it has instantiated the
+ * other devices and their drivers. If there aren't any hubs
+ * around, we'll silently skip the USB devices.
+ *
+ * @returns VBox status code.
+ * @param   pVM        The cross context VM structure.
+ */
+int pdmR3UsbInstantiateDevices(PVM pVM)
+{
+    /*
+     * Any hubs?
+     */
+    if (!pVM->pdm.s.pUsbHubs)
+    {
+        Log(("PDM: No USB hubs, skipping USB device instantiation.\n"));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Count the device instances.
+     */
+    PCFGMNODE pCur;
+    PCFGMNODE pUsbNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "USB/");
+    PCFGMNODE pInstanceNode;
+    unsigned cUsbDevs = 0;
+    for (pCur = CFGMR3GetFirstChild(pUsbNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+    {
+        PCFGMNODE pGlobal = CFGMR3GetChild(pCur, "GlobalConfig/");
+        for (pInstanceNode = CFGMR3GetFirstChild(pCur); pInstanceNode; pInstanceNode = CFGMR3GetNextChild(pInstanceNode))
+            if (pInstanceNode != pGlobal)
+                cUsbDevs++;
+    }
+    if (!cUsbDevs)
+    {
+        Log(("PDM: No USB devices were configured!\n"));
+        return VINF_SUCCESS;
+    }
+    Log2(("PDM: cUsbDevs=%d!\n", cUsbDevs));
+
+    /*
+     * Collect info on each USB device instance.
+     */
+    struct USBDEVORDER
+    {
+        /** Configuration node. */
+        PCFGMNODE   pNode;
+        /** Pointer to the USB device. */
+        PPDMUSB     pUsbDev;
+        /** Init order. */
+        uint32_t    u32Order;
+        /** VBox instance number. */
+        uint32_t    iInstance;
+        /** Device UUID. */
+        RTUUID      Uuid;
+    } *paUsbDevs = (struct USBDEVORDER *)alloca(sizeof(paUsbDevs[0]) * (cUsbDevs + 1)); /* (One extra for swapping) */
+    Assert(paUsbDevs);
+    int rc;
+    unsigned i = 0;
+    for (pCur = CFGMR3GetFirstChild(pUsbNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+    {
+        /* Get the device name. */
+        char szName[sizeof(paUsbDevs[0].pUsbDev->pReg->szName)];
+        rc = CFGMR3GetName(pCur, szName, sizeof(szName));
+        AssertMsgRCReturn(rc, ("Configuration error: device name is too long (or something)! rc=%Rrc\n", rc), rc);
+
+        /* Find the device. */
+        PPDMUSB pUsbDev = pdmR3UsbLookup(pVM, szName);
+        AssertMsgReturn(pUsbDev, ("Configuration error: device '%s' not found!\n", szName), VERR_PDM_DEVICE_NOT_FOUND);
+
+        /* Configured priority or use default? */
+        uint32_t u32Order;
+        rc = CFGMR3QueryU32(pCur, "Priority", &u32Order);
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+            u32Order = i << 4;
+        else
+            AssertMsgRCReturn(rc, ("Configuration error: reading \"Priority\" for the '%s' USB device failed rc=%Rrc!\n", szName, rc), rc);
+
+        /* Global config. */
+        PCFGMNODE pGlobal = CFGMR3GetChild(pCur, "GlobalConfig/");
+        if (!pGlobal)
+        {
+            rc = CFGMR3InsertNode(pCur, "GlobalConfig/", &pGlobal);
+            AssertMsgRCReturn(rc, ("Failed to create GlobalConfig node! rc=%Rrc\n", rc), rc);
+            CFGMR3SetRestrictedRoot(pGlobal);
+        }
+
+        /* Enumerate the device instances. */
+        for (pInstanceNode = CFGMR3GetFirstChild(pCur); pInstanceNode; pInstanceNode = CFGMR3GetNextChild(pInstanceNode))
+        {
+            if (pInstanceNode == pGlobal)
+                continue;
+
+            /* Use the configured UUID if present, create our own otherwise. */
+            char *pszUuid = NULL;
+
+            RTUuidClear(&paUsbDevs[i].Uuid);
+            rc = CFGMR3QueryStringAlloc(pInstanceNode, "UUID", &pszUuid);
+            if (RT_SUCCESS(rc))
+            {
+                AssertPtr(pszUuid);
+
+                rc = RTUuidFromStr(&paUsbDevs[i].Uuid, pszUuid);
+                AssertMsgRCReturn(rc, ("Failed to convert UUID from string! rc=%Rrc\n", rc), rc);
+                MMR3HeapFree(pszUuid);
+            }
+            else if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+                rc = RTUuidCreate(&paUsbDevs[i].Uuid);
+
+            AssertRCReturn(rc, rc);
+            paUsbDevs[i].pNode = pInstanceNode;
+            paUsbDevs[i].pUsbDev = pUsbDev;
+            paUsbDevs[i].u32Order = u32Order;
+
+            /* Get the instance number. */
+            char szInstance[32];
+            rc = CFGMR3GetName(pInstanceNode, szInstance, sizeof(szInstance));
+            AssertMsgRCReturn(rc, ("Configuration error: instance name is too long (or something)! rc=%Rrc\n", rc), rc);
+            char *pszNext = NULL;
+            rc = RTStrToUInt32Ex(szInstance, &pszNext, 0, &paUsbDevs[i].iInstance);
+            AssertMsgRCReturn(rc, ("Configuration error: RTStrToInt32Ex failed on the instance name '%s'! rc=%Rrc\n", szInstance, rc), rc);
+            AssertMsgReturn(!*pszNext, ("Configuration error: the instance name '%s' isn't all digits. (%s)\n", szInstance, pszNext), VERR_INVALID_PARAMETER);
+
+            /* next instance */
+            i++;
+        }
+    } /* devices */
+    Assert(i == cUsbDevs);
+
+    /*
+     * Sort the device array ascending on u32Order. (bubble)
+     */
+    unsigned c = cUsbDevs - 1;
+    while (c)
+    {
+        unsigned j = 0;
+        for (i = 0; i < c; i++)
+            if (paUsbDevs[i].u32Order > paUsbDevs[i + 1].u32Order)
+            {
+                paUsbDevs[cUsbDevs] = paUsbDevs[i + 1];
+                paUsbDevs[i + 1] = paUsbDevs[i];
+                paUsbDevs[i] = paUsbDevs[cUsbDevs];
+                j = i;
+            }
+        c = j;
+    }
+
+    /*
+     * Instantiate the devices.
+     */
+    for (i = 0; i < cUsbDevs; i++)
+    {
+        /*
+         * Make sure there is a config node and mark it as restricted.
+         */
+        PCFGMNODE pConfigNode = CFGMR3GetChild(paUsbDevs[i].pNode, "Config/");
+        if (!pConfigNode)
+        {
+            rc = CFGMR3InsertNode(paUsbDevs[i].pNode, "Config", &pConfigNode);
+            AssertMsgRCReturn(rc, ("Failed to create Config node! rc=%Rrc\n", rc), rc);
+        }
+        CFGMR3SetRestrictedRoot(pConfigNode);
+
+        /*
+         * Every emulated device must support USB 1.x hubs; optionally, high-speed USB 2.0 hubs
+         * might be also supported. This determines where to attach the device.
+         */
+        uint32_t iUsbVersion = VUSB_STDVER_11;
+
+        if (paUsbDevs[i].pUsbDev->pReg->fFlags & PDM_USBREG_HIGHSPEED_CAPABLE)
+            iUsbVersion |= VUSB_STDVER_20;
+        if (paUsbDevs[i].pUsbDev->pReg->fFlags & PDM_USBREG_SUPERSPEED_CAPABLE)
+            iUsbVersion |= VUSB_STDVER_30;
+
+        /*
+         * Find a suitable hub with free ports.
+         */
+        PPDMUSBHUB pHub;
+        rc = pdmR3UsbFindHub(pVM, iUsbVersion, &pHub);
+        if (RT_FAILURE(rc))
+        {
+            Log(("pdmR3UsbFindHub failed %Rrc\n", rc));
+            return rc;
+        }
+
+        /*
+         * This is how we inform the device what speed it's communicating at, and hence
+         * which descriptors it should present to the guest.
+         */
+        iUsbVersion &= pHub->fVersions;
+
+        /*
+         * Create and attach the device.
+         */
+        rc = pdmR3UsbCreateDevice(pVM, pHub, paUsbDevs[i].pUsbDev, paUsbDevs[i].iInstance, &paUsbDevs[i].Uuid,
+                                  &paUsbDevs[i].pNode, pdmR3UsbVer2Spd(iUsbVersion), NULL);
+        if (RT_FAILURE(rc))
+            return rc;
+    } /* for device instances */
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates an emulated USB device instance at runtime.
+ *
+ * This will find an appropriate HUB for the USB device
+ * and try instantiate the emulated device.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pszDeviceName       The name of the PDM device to instantiate.
+ * @param   pInstanceNode       The instance CFGM node.
+ * @param   pUuid               The UUID to be associated with the device.
+ * @param   pszCaptureFilename  Path to the file for USB traffic capturing, optional.
+ *
+ * @thread EMT
+ */
+VMMR3DECL(int) PDMR3UsbCreateEmulatedDevice(PUVM pUVM, const char *pszDeviceName, PCFGMNODE pInstanceNode, PCRTUUID pUuid,
+                                            const char *pszCaptureFilename)
+{
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pszDeviceName, VERR_INVALID_POINTER);
+    AssertPtrReturn(pInstanceNode, VERR_INVALID_POINTER);
+
+    /*
+     * Find the device.
+     */
+    PPDMUSB pUsbDev = pdmR3UsbLookup(pVM, pszDeviceName);
+    if (!pUsbDev)
+    {
+        LogRel(("PDMUsb: PDMR3UsbCreateEmulatedDevice: The '%s' device wasn't found\n", pszDeviceName));
+        return VERR_PDM_NO_USBPROXY;
+    }
+
+    /*
+     * Every device must support USB 1.x hubs; optionally, high-speed USB 2.0 hubs
+     * might be also supported. This determines where to attach the device.
+     */
+    uint32_t iUsbVersion = VUSB_STDVER_11;
+    if (pUsbDev->pReg->fFlags & PDM_USBREG_HIGHSPEED_CAPABLE)
+        iUsbVersion |= VUSB_STDVER_20;
+    if (pUsbDev->pReg->fFlags & PDM_USBREG_SUPERSPEED_CAPABLE)
+        iUsbVersion |= VUSB_STDVER_30;
+
+    /*
+     * Find a suitable hub with free ports.
+     */
+    PPDMUSBHUB pHub;
+    int rc = pdmR3UsbFindHub(pVM, iUsbVersion, &pHub);
+    if (RT_FAILURE(rc))
+    {
+        Log(("pdmR3UsbFindHub: failed %Rrc\n", rc));
+        return rc;
+    }
+
+    /*
+     * This is how we inform the device what speed it's communicating at, and hence
+     * which descriptors it should present to the guest.
+     */
+    iUsbVersion &= pHub->fVersions;
+
+    /*
+     * Create and attach the device.
+     */
+    rc = pdmR3UsbCreateDevice(pVM, pHub, pUsbDev, -1, pUuid, &pInstanceNode,
+                              pdmR3UsbVer2Spd(iUsbVersion), pszCaptureFilename);
+    AssertRCReturn(rc, rc);
+
+    return rc;
+}
+
+
+/**
+ * Creates a USB proxy device instance.
+ *
+ * This will find an appropriate HUB for the USB device, create the necessary CFGM stuff
+ * and try instantiate the proxy device.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pUuid               The UUID to be associated with the device.
+ * @param   pszBackend          The proxy backend to use.
+ * @param   pszAddress          The address string.
+ * @param   pvBackend           Pointer to the backend.
+ * @param   enmSpeed            The speed the USB device is operating at.
+ * @param   fMaskedIfs          The interfaces to hide from the guest.
+ * @param   pszCaptureFilename  Path to the file for USB traffic capturing, optional.
+ */
+VMMR3DECL(int) PDMR3UsbCreateProxyDevice(PUVM pUVM, PCRTUUID pUuid, const char *pszBackend, const char *pszAddress, void *pvBackend,
+                                         VUSBSPEED enmSpeed, uint32_t fMaskedIfs, const char *pszCaptureFilename)
+{
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pUuid, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszAddress, VERR_INVALID_POINTER);
+    AssertReturn(    enmSpeed == VUSB_SPEED_LOW
+                 ||  enmSpeed == VUSB_SPEED_FULL
+                 ||  enmSpeed == VUSB_SPEED_HIGH
+                 ||  enmSpeed == VUSB_SPEED_SUPER
+                 ||  enmSpeed == VUSB_SPEED_SUPERPLUS, VERR_INVALID_PARAMETER);
+
+    /*
+     * Find the USBProxy driver.
+     */
+    PPDMUSB pUsbDev = pdmR3UsbLookup(pVM, "USBProxy");
+    if (!pUsbDev)
+    {
+        LogRel(("PDMUsb: PDMR3UsbCreateProxyDevice: The USBProxy device class wasn't found\n"));
+        return VERR_PDM_NO_USBPROXY;
+    }
+
+    /*
+     * Find a suitable hub with free ports.
+     */
+    PPDMUSBHUB pHub;
+    uint32_t iUsbVersion = pdmR3UsbSpd2Ver(enmSpeed);
+    int rc = pdmR3UsbFindHub(pVM, iUsbVersion, &pHub);
+    if (RT_FAILURE(rc))
+    {
+        Log(("pdmR3UsbFindHub: failed %Rrc\n", rc));
+        return rc;
+    }
+
+    /*
+     * Create the CFGM instance node.
+     */
+    PCFGMNODE pInstance = CFGMR3CreateTree(pUVM);
+    AssertReturn(pInstance, VERR_NO_MEMORY);
+    do /* break loop */
+    {
+        PCFGMNODE pConfig;
+        rc = CFGMR3InsertNode(pInstance, "Config", &pConfig);                   AssertRCBreak(rc);
+        rc = CFGMR3InsertString(pConfig,  "Address", pszAddress);               AssertRCBreak(rc);
+        char szUuid[RTUUID_STR_LENGTH];
+        rc = RTUuidToStr(pUuid, &szUuid[0], sizeof(szUuid));                    AssertRCBreak(rc);
+        rc = CFGMR3InsertString(pConfig,  "UUID", szUuid);                      AssertRCBreak(rc);
+        rc = CFGMR3InsertString(pConfig, "Backend", pszBackend);                AssertRCBreak(rc);
+        rc = CFGMR3InsertInteger(pConfig, "pvBackend", (uintptr_t)pvBackend);   AssertRCBreak(rc);
+        rc = CFGMR3InsertInteger(pConfig, "MaskedIfs", fMaskedIfs);             AssertRCBreak(rc);
+        rc = CFGMR3InsertInteger(pConfig, "Force11Device", !(pHub->fVersions & iUsbVersion)); AssertRCBreak(rc);
+    } while (0); /* break loop */
+    if (RT_FAILURE(rc))
+    {
+        CFGMR3RemoveNode(pInstance);
+        LogRel(("PDMUsb: PDMR3UsbCreateProxyDevice: failed to setup CFGM config, rc=%Rrc\n", rc));
+        return rc;
+    }
+
+    if (enmSpeed == VUSB_SPEED_UNKNOWN)
+        enmSpeed = pdmR3UsbVer2Spd(iUsbVersion);
+
+    /*
+     * Finally, try to create it.
+     */
+    rc = pdmR3UsbCreateDevice(pVM, pHub, pUsbDev, -1, pUuid, &pInstance, enmSpeed, pszCaptureFilename);
+    if (RT_FAILURE(rc) && pInstance)
+        CFGMR3RemoveNode(pInstance);
+    return rc;
+}
+
+
+/**
+ * Destroys a hot-plugged USB device.
+ *
+ * The device must be detached from the HUB at this point.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pUsbIns         The USB device instance to destroy.
+ * @thread  EMT
+ */
+static void pdmR3UsbDestroyDevice(PVM pVM, PPDMUSBINS pUsbIns)
+{
+    Assert(!pUsbIns->Internal.s.pHub);
+
+    /*
+     * Do the unplug notification.
+     */
+    /** @todo what about the drivers? */
+    if (pUsbIns->pReg->pfnHotUnplugged)
+        pUsbIns->pReg->pfnHotUnplugged(pUsbIns);
+
+    /*
+     * Destroy the luns with their driver chains and call the device destructor.
+     */
+    while (pUsbIns->Internal.s.pLuns)
+    {
+        PPDMLUN pLun = pUsbIns->Internal.s.pLuns;
+        pUsbIns->Internal.s.pLuns = pLun->pNext;
+        if (pLun->pTop)
+            pdmR3DrvDestroyChain(pLun->pTop, PDM_TACH_FLAGS_NOT_HOT_PLUG); /* Hotplugging is handled differently here atm. */
+        MMR3HeapFree(pLun);
+    }
+
+    /* finally, the device. */
+    if (pUsbIns->pReg->pfnDestruct)
+    {
+        Log(("PDM: Destructing USB device '%s' instance %d...\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+        pUsbIns->pReg->pfnDestruct(pUsbIns);
+    }
+    TMR3TimerDestroyUsb(pVM, pUsbIns);
+    SSMR3DeregisterUsb(pVM, pUsbIns, NULL, 0);
+    pdmR3ThreadDestroyUsb(pVM, pUsbIns);
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+    pdmR3AsyncCompletionTemplateDestroyUsb(pVM, pUsbIns);
+#endif
+
+    /*
+     * Unlink it.
+     */
+    /* The global instance FIFO. */
+    if (pVM->pdm.s.pUsbInstances == pUsbIns)
+        pVM->pdm.s.pUsbInstances = pUsbIns->Internal.s.pNext;
+    else
+    {
+        PPDMUSBINS pPrev = pVM->pdm.s.pUsbInstances;
+        while (pPrev && pPrev->Internal.s.pNext != pUsbIns)
+        {
+            Assert(pPrev->u32Version == PDM_USBINS_VERSION);
+            pPrev = pPrev->Internal.s.pNext;
+        }
+        Assert(pPrev); Assert(pPrev != pUsbIns);
+        if (pPrev)
+            pPrev->Internal.s.pNext = pUsbIns->Internal.s.pNext;
+    }
+
+    /* The per device instance FIFO. */
+    PPDMUSB pUsbDev = pUsbIns->Internal.s.pUsbDev;
+    if (pUsbDev->pInstances == pUsbIns)
+        pUsbDev->pInstances = pUsbIns->Internal.s.pPerDeviceNext;
+    else
+    {
+        PPDMUSBINS pPrev = pUsbDev->pInstances;
+        while (pPrev && pPrev->Internal.s.pPerDeviceNext != pUsbIns)
+        {
+            Assert(pPrev->u32Version == PDM_USBINS_VERSION);
+            pPrev = pPrev->Internal.s.pPerDeviceNext;
+        }
+        Assert(pPrev); Assert(pPrev != pUsbIns);
+        if (pPrev)
+            pPrev->Internal.s.pPerDeviceNext = pUsbIns->Internal.s.pPerDeviceNext;
+    }
+
+    /*
+     * Trash it.
+     */
+    pUsbIns->u32Version = 0;
+    pUsbIns->pReg = NULL;
+    if (pUsbIns->pszName)
+    {
+        RTStrFree(pUsbIns->pszName);
+        pUsbIns->pszName = NULL;
+    }
+    CFGMR3RemoveNode(pUsbIns->Internal.s.pCfgDelete);
+    MMR3HeapFree(pUsbIns);
+}
+
+
+/**
+ * Detaches and destroys a USB device.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pUuid           The UUID associated with the device to detach.
+ * @thread  EMT
+ */
+VMMR3DECL(int) PDMR3UsbDetachDevice(PUVM pUVM, PCRTUUID pUuid)
+{
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT(pVM);
+    AssertPtrReturn(pUuid, VERR_INVALID_POINTER);
+
+    /*
+     * Search the global list for it.
+     */
+    PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances;
+    for ( ; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        if (!RTUuidCompare(&pUsbIns->Internal.s.Uuid, pUuid))
+            break;
+    if (!pUsbIns)
+        return VERR_PDM_DEVICE_INSTANCE_NOT_FOUND; /** @todo VERR_PDM_USB_INSTANCE_NOT_FOUND */
+
+    /*
+     * Detach it from the HUB (if it's actually attached to one).
+     */
+    PPDMUSBHUB pHub = pUsbIns->Internal.s.pHub;
+    if (pHub)
+    {
+        int rc = pHub->Reg.pfnDetachDevice(pHub->pDrvIns, pUsbIns, pUsbIns->Internal.s.iPort);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMUsb: Failed to detach USB device '%s' instance %d from %p: %Rrc\n",
+                    pUsbIns->pReg->szName, pUsbIns->iInstance, pHub, rc));
+            return rc;
+        }
+
+        pHub->cAvailablePorts++;
+        Assert(pHub->cAvailablePorts > 0 && pHub->cAvailablePorts <= pHub->cPorts);
+        pUsbIns->Internal.s.pHub = NULL;
+    }
+
+    /*
+     * Notify about unplugging and destroy the device with it's drivers.
+     */
+    pdmR3UsbDestroyDevice(pVM, pUsbIns);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if there are any USB hubs attached.
+ *
+ * @returns true / false accordingly.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(bool) PDMR3UsbHasHub(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, false);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->pdm.s.pUsbHubs != NULL;
+}
+
+
+/**
+ * Locates a LUN.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   ppLun           Where to store the pointer to the LUN if found.
+ * @thread  Try only do this in EMT...
+ */
+static int pdmR3UsbFindLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPPDMLUN ppLun)
+{
+    /*
+     * Iterate registered devices looking for the device.
+     */
+    size_t cchDevice = strlen(pszDevice);
+    for (PPDMUSB pUsbDev = pVM->pdm.s.pUsbDevs; pUsbDev; pUsbDev = pUsbDev->pNext)
+    {
+        if (    pUsbDev->cchName == cchDevice
+            &&  !memcmp(pUsbDev->pReg->szName, pszDevice, cchDevice))
+        {
+            /*
+             * Iterate device instances.
+             */
+            for (PPDMUSBINS pUsbIns = pUsbDev->pInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pPerDeviceNext)
+            {
+                if (pUsbIns->iInstance == iInstance)
+                {
+                    /*
+                     * Iterate luns.
+                     */
+                    for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+                    {
+                        if (pLun->iLun == iLun)
+                        {
+                            *ppLun = pLun;
+                            return VINF_SUCCESS;
+                        }
+                    }
+                    return VERR_PDM_LUN_NOT_FOUND;
+                }
+            }
+            return VERR_PDM_DEVICE_INSTANCE_NOT_FOUND;
+        }
+    }
+    return VERR_PDM_DEVICE_NOT_FOUND;
+}
+
+
+/**
+ * Attaches a preconfigured driver to an existing device or driver instance.
+ *
+ * This is used to change drivers and suchlike at runtime.  The driver or device
+ * at the end of the chain will be told to attach to whatever is configured
+ * below it.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iDevIns         Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   fFlags          Flags, combination of the PDM_TACH_FLAGS_* \#defines.
+ * @param   ppBase          Where to store the base interface pointer. Optional.
+ *
+ * @thread  EMT
+ */
+VMMR3DECL(int)  PDMR3UsbDriverAttach(PUVM pUVM, const char *pszDevice, unsigned iDevIns, unsigned iLun, uint32_t fFlags,
+                                     PPPDMIBASE ppBase)
+{
+    LogFlow(("PDMR3UsbDriverAttach: pszDevice=%p:{%s} iDevIns=%d iLun=%d fFlags=%#x ppBase=%p\n",
+             pszDevice, pszDevice, iDevIns, iLun, fFlags, ppBase));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT(pVM);
+
+    if (ppBase)
+        *ppBase = NULL;
+
+    /*
+     * Find the LUN in question.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3UsbFindLun(pVM, pszDevice, iDevIns, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Anything attached to the LUN?
+         */
+        PPDMDRVINS pDrvIns = pLun->pTop;
+        if (!pDrvIns)
+        {
+            /* No, ask the device to attach to the new stuff. */
+            PPDMUSBINS pUsbIns = pLun->pUsbIns;
+            if (pUsbIns->pReg->pfnDriverAttach)
+            {
+                rc = pUsbIns->pReg->pfnDriverAttach(pUsbIns, iLun, fFlags);
+                if (RT_SUCCESS(rc) && ppBase)
+                    *ppBase = pLun->pTop ? &pLun->pTop->IBase : NULL;
+            }
+            else
+                rc = VERR_PDM_DEVICE_NO_RT_ATTACH;
+        }
+        else
+        {
+            /* Yes, find the bottom most driver and ask it to attach to the new stuff. */
+            while (pDrvIns->Internal.s.pDown)
+                pDrvIns = pDrvIns->Internal.s.pDown;
+            if (pDrvIns->pReg->pfnAttach)
+            {
+                rc = pDrvIns->pReg->pfnAttach(pDrvIns, fFlags);
+                if (RT_SUCCESS(rc) && ppBase)
+                    *ppBase = pDrvIns->Internal.s.pDown
+                            ? &pDrvIns->Internal.s.pDown->IBase
+                            : NULL;
+            }
+            else
+                rc = VERR_PDM_DRIVER_NO_RT_ATTACH;
+        }
+    }
+
+    if (ppBase)
+        LogFlow(("PDMR3UsbDriverAttach: returns %Rrc *ppBase=%p\n", rc, *ppBase));
+    else
+        LogFlow(("PDMR3UsbDriverAttach: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Detaches the specified driver instance.
+ *
+ * This is used to replumb drivers at runtime for simulating hot plugging and
+ * media changes.
+ *
+ * This method allows detaching drivers from
+ * any driver or device by specifying the driver to start detaching at.  The
+ * only prerequisite is that the driver or device above implements the
+ * pfnDetach callback (PDMDRVREG / PDMUSBREG).
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iDevIns         Device instance.
+ * @param   iLun            The Logical Unit in which to look for the driver.
+ * @param   pszDriver       The name of the driver which to detach.  If NULL
+ *                          then the entire driver chain is detatched.
+ * @param   iOccurrence     The occurrence of that driver in the chain.  This is
+ *                          usually 0.
+ * @param   fFlags          Flags, combination of the PDM_TACH_FLAGS_* \#defines.
+ * @thread  EMT
+ */
+VMMR3DECL(int)  PDMR3UsbDriverDetach(PUVM pUVM, const char *pszDevice, unsigned iDevIns, unsigned iLun,
+                                     const char *pszDriver, unsigned iOccurrence, uint32_t fFlags)
+{
+    LogFlow(("PDMR3UsbDriverDetach: pszDevice=%p:{%s} iDevIns=%u iLun=%u pszDriver=%p:{%s} iOccurrence=%u fFlags=%#x\n",
+             pszDevice, pszDevice, iDevIns, iLun, pszDriver, pszDriver, iOccurrence, fFlags));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT(pVM);
+    AssertPtr(pszDevice);
+    AssertPtrNull(pszDriver);
+    Assert(iOccurrence == 0 || pszDriver);
+    Assert(!(fFlags & ~(PDM_TACH_FLAGS_NOT_HOT_PLUG)));
+
+    /*
+     * Find the LUN in question.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3UsbFindLun(pVM, pszDevice, iDevIns, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Locate the driver.
+         */
+        PPDMDRVINS pDrvIns = pLun->pTop;
+        if (pDrvIns)
+        {
+            if (pszDriver)
+            {
+                while (pDrvIns)
+                {
+                    if (!strcmp(pDrvIns->pReg->szName, pszDriver))
+                    {
+                        if (iOccurrence == 0)
+                            break;
+                        iOccurrence--;
+                    }
+                    pDrvIns = pDrvIns->Internal.s.pDown;
+                }
+            }
+            if (pDrvIns)
+                rc = pdmR3DrvDetach(pDrvIns, fFlags);
+            else
+                rc = VERR_PDM_DRIVER_INSTANCE_NOT_FOUND;
+        }
+        else
+            rc = VINF_PDM_NO_DRIVER_ATTACHED_TO_LUN;
+    }
+
+    LogFlow(("PDMR3UsbDriverDetach: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Query the interface of the top level driver on a LUN.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   ppBase          Where to store the base interface pointer.
+ * @remark  We're not doing any locking ATM, so don't try call this at times when the
+ *          device chain is known to be updated.
+ */
+VMMR3DECL(int)  PDMR3UsbQueryLun(PUVM pUVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPDMIBASE *ppBase)
+{
+    LogFlow(("PDMR3UsbQueryLun: pszDevice=%p:{%s} iInstance=%u iLun=%u ppBase=%p\n",
+             pszDevice, pszDevice, iInstance, iLun, ppBase));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Find the LUN.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3UsbFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLun->pTop)
+        {
+            *ppBase = &pLun->pTop->IBase;
+            LogFlow(("PDMR3UsbQueryLun: return %Rrc and *ppBase=%p\n", VINF_SUCCESS, *ppBase));
+            return VINF_SUCCESS;
+        }
+        rc = VERR_PDM_NO_DRIVER_ATTACHED_TO_LUN;
+    }
+    LogFlow(("PDMR3UsbQueryLun: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Query the interface of a named driver on a LUN.
+ *
+ * If the driver appears more than once in the driver chain, the first instance
+ * is returned.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   pszDriver       The driver name.
+ * @param   ppBase          Where to store the base interface pointer.
+ *
+ * @remark  We're not doing any locking ATM, so don't try call this at times when the
+ *          device chain is known to be updated.
+ */
+VMMR3DECL(int) PDMR3UsbQueryDriverOnLun(PUVM pUVM, const char *pszDevice, unsigned iInstance,
+                                        unsigned iLun, const char *pszDriver, PPPDMIBASE ppBase)
+{
+    LogFlow(("PDMR3QueryDriverOnLun: pszDevice=%p:{%s} iInstance=%u iLun=%u pszDriver=%p:{%s} ppBase=%p\n",
+             pszDevice, pszDevice, iInstance, iLun, pszDriver, pszDriver, ppBase));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Find the LUN.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3UsbFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLun->pTop)
+        {
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                if (!strcmp(pDrvIns->pReg->szName, pszDriver))
+                {
+                    *ppBase = &pDrvIns->IBase;
+                    LogFlow(("PDMR3UsbQueryDriverOnLun: return %Rrc and *ppBase=%p\n", VINF_SUCCESS, *ppBase));
+                    return VINF_SUCCESS;
+
+                }
+            rc = VERR_PDM_DRIVER_NOT_FOUND;
+        }
+        else
+            rc = VERR_PDM_NO_DRIVER_ATTACHED_TO_LUN;
+    }
+    LogFlow(("PDMR3UsbQueryDriverOnLun: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/** @name USB Device Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMUSBHLP,pfnDriverAttach} */
+static DECLCALLBACK(int) pdmR3UsbHlp_DriverAttach(PPDMUSBINS pUsbIns, RTUINT iLun, PPDMIBASE pBaseInterface,
+                                                  PPDMIBASE *ppBaseInterface, const char *pszDesc)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3UsbHlp_DriverAttach: caller='%s'/%d: iLun=%d pBaseInterface=%p ppBaseInterface=%p pszDesc=%p:{%s}\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, iLun, pBaseInterface, ppBaseInterface, pszDesc, pszDesc));
+
+    /*
+     * Lookup the LUN, it might already be registered.
+     */
+    PPDMLUN pLunPrev = NULL;
+    PPDMLUN pLun = pUsbIns->Internal.s.pLuns;
+    for (; pLun; pLunPrev = pLun, pLun = pLun->pNext)
+        if (pLun->iLun == iLun)
+            break;
+
+    /*
+     * Create the LUN if if wasn't found, else check if driver is already attached to it.
+     */
+    if (!pLun)
+    {
+        if (    !pBaseInterface
+            ||  !pszDesc
+            ||  !*pszDesc)
+        {
+            Assert(pBaseInterface);
+            Assert(pszDesc || *pszDesc);
+            return VERR_INVALID_PARAMETER;
+        }
+
+        pLun = (PPDMLUN)MMR3HeapAlloc(pVM, MM_TAG_PDM_LUN, sizeof(*pLun));
+        if (!pLun)
+            return VERR_NO_MEMORY;
+
+        pLun->iLun      = iLun;
+        pLun->pNext     = pLunPrev ? pLunPrev->pNext : NULL;
+        pLun->pTop      = NULL;
+        pLun->pBottom   = NULL;
+        pLun->pDevIns   = NULL;
+        pLun->pUsbIns   = pUsbIns;
+        pLun->pszDesc   = pszDesc;
+        pLun->pBase     = pBaseInterface;
+        if (!pLunPrev)
+            pUsbIns->Internal.s.pLuns = pLun;
+        else
+            pLunPrev->pNext = pLun;
+        Log(("pdmR3UsbHlp_DriverAttach: Registered LUN#%d '%s' with device '%s'/%d.\n",
+             iLun, pszDesc, pUsbIns->pReg->szName, pUsbIns->iInstance));
+    }
+    else if (pLun->pTop)
+    {
+        AssertMsgFailed(("Already attached! The device should keep track of such things!\n"));
+        LogFlow(("pdmR3UsbHlp_DriverAttach: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, VERR_PDM_DRIVER_ALREADY_ATTACHED));
+        return VERR_PDM_DRIVER_ALREADY_ATTACHED;
+    }
+    Assert(pLun->pBase == pBaseInterface);
+
+
+    /*
+     * Get the attached driver configuration.
+     */
+    int rc;
+    PCFGMNODE pNode = CFGMR3GetChildF(pUsbIns->Internal.s.pCfg, "LUN#%u", iLun);
+    if (pNode)
+        rc = pdmR3DrvInstantiate(pVM, pNode, pBaseInterface, NULL /*pDrvAbove*/, pLun, ppBaseInterface);
+    else
+        rc = VERR_PDM_NO_ATTACHED_DRIVER;
+
+
+    LogFlow(("pdmR3UsbHlp_DriverAttach: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnAssertEMT} */
+static DECLCALLBACK(bool) pdmR3UsbHlp_AssertEMT(PPDMUSBINS pUsbIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    if (VM_IS_EMT(pUsbIns->Internal.s.pVM))
+        return true;
+
+    char szMsg[100];
+    RTStrPrintf(szMsg, sizeof(szMsg), "AssertEMT '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance);
+    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
+    AssertBreakpoint();
+    return false;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnAssertOther} */
+static DECLCALLBACK(bool) pdmR3UsbHlp_AssertOther(PPDMUSBINS pUsbIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    if (!VM_IS_EMT(pUsbIns->Internal.s.pVM))
+        return true;
+
+    char szMsg[100];
+    RTStrPrintf(szMsg, sizeof(szMsg), "AssertOther '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance);
+    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
+    AssertBreakpoint();
+    return false;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnDBGFStopV} */
+static DECLCALLBACK(int) pdmR3UsbHlp_DBGFStopV(PPDMUSBINS pUsbIns, const char *pszFile, unsigned iLine, const char *pszFunction,
+                                               const char *pszFormat, va_list va)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+#ifdef LOG_ENABLED
+    va_list va2;
+    va_copy(va2, va);
+    LogFlow(("pdmR3UsbHlp_DBGFStopV: caller='%s'/%d: pszFile=%p:{%s} iLine=%d pszFunction=%p:{%s} pszFormat=%p:{%s} (%N)\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, pszFile, pszFile, iLine, pszFunction, pszFunction, pszFormat, pszFormat, pszFormat, &va2));
+    va_end(va2);
+#endif
+
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+    int rc = DBGFR3EventSrcV(pVM, DBGFEVENT_DEV_STOP, pszFile, iLine, pszFunction, pszFormat, va);
+    if (rc == VERR_DBGF_NOT_ATTACHED)
+        rc = VINF_SUCCESS;
+
+    LogFlow(("pdmR3UsbHlp_DBGFStopV: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnDBGFInfoRegisterArgv} */
+static DECLCALLBACK(int) pdmR3UsbHlp_DBGFInfoRegisterArgv(PPDMUSBINS pUsbIns, const char *pszName, const char *pszDesc,
+                                                          PFNDBGFINFOARGVUSB pfnHandler)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    LogFlow(("pdmR3UsbHlp_DBGFInfoRegister: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, pszName, pszName, pszDesc, pszDesc, pfnHandler));
+
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+    int rc = DBGFR3InfoRegisterUsbArgv(pVM, pszName, pszDesc, pfnHandler, pUsbIns);
+
+    LogFlow(("pdmR3UsbHlp_DBGFInfoRegister: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnMMHeapAlloc} */
+static DECLCALLBACK(void *) pdmR3UsbHlp_MMHeapAlloc(PPDMUSBINS pUsbIns, size_t cb)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    LogFlow(("pdmR3UsbHlp_MMHeapAlloc: caller='%s'/%d: cb=%#x\n", pUsbIns->pReg->szName, pUsbIns->iInstance, cb));
+
+    void *pv = MMR3HeapAlloc(pUsbIns->Internal.s.pVM, MM_TAG_PDM_USB_USER, cb);
+
+    LogFlow(("pdmR3UsbHlp_MMHeapAlloc: caller='%s'/%d: returns %p\n", pUsbIns->pReg->szName, pUsbIns->iInstance, pv));
+    return pv;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnMMHeapAllocZ} */
+static DECLCALLBACK(void *) pdmR3UsbHlp_MMHeapAllocZ(PPDMUSBINS pUsbIns, size_t cb)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    LogFlow(("pdmR3UsbHlp_MMHeapAllocZ: caller='%s'/%d: cb=%#x\n", pUsbIns->pReg->szName, pUsbIns->iInstance, cb));
+
+    void *pv = MMR3HeapAllocZ(pUsbIns->Internal.s.pVM, MM_TAG_PDM_USB_USER, cb);
+
+    LogFlow(("pdmR3UsbHlp_MMHeapAllocZ: caller='%s'/%d: returns %p\n", pUsbIns->pReg->szName, pUsbIns->iInstance, pv));
+    return pv;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnPDMQueueCreate} */
+static DECLCALLBACK(int) pdmR3UsbHlp_PDMQueueCreate(PPDMUSBINS pUsbIns, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval,
+                                                    PFNPDMQUEUEUSB pfnCallback, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    LogFlow(("pdmR3UsbHlp_PDMQueueCreate: caller='%s'/%d: cbItem=%#x cItems=%#x cMilliesInterval=%u pfnCallback=%p pszName=%p:{%s} ppQueue=%p\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, cbItem, cItems, cMilliesInterval, pfnCallback, pszName, pszName, ppQueue));
+
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+
+    if (pUsbIns->iInstance > 0)
+    {
+        pszName = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s_%u", pszName, pUsbIns->iInstance);
+        AssertLogRelReturn(pszName, VERR_NO_MEMORY);
+    }
+
+    RT_NOREF5(cbItem, cItems, cMilliesInterval, pfnCallback, ppQueue);
+    /** @todo int rc = PDMR3QueueCreateUsb(pVM, pUsbIns, cbItem, cItems, cMilliesInterval, pfnCallback, fGCEnabled, pszName, ppQueue); */
+    int rc = VERR_NOT_IMPLEMENTED; AssertFailed();
+
+    LogFlow(("pdmR3UsbHlp_PDMQueueCreate: caller='%s'/%d: returns %Rrc *ppQueue=%p\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc, *ppQueue));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnSSMRegister} */
+static DECLCALLBACK(int) pdmR3UsbHlp_SSMRegister(PPDMUSBINS pUsbIns, uint32_t uVersion, size_t cbGuess,
+                                                 PFNSSMUSBLIVEPREP pfnLivePrep, PFNSSMUSBLIVEEXEC pfnLiveExec, PFNSSMUSBLIVEVOTE pfnLiveVote,
+                                                 PFNSSMUSBSAVEPREP pfnSavePrep, PFNSSMUSBSAVEEXEC pfnSaveExec, PFNSSMUSBSAVEDONE pfnSaveDone,
+                                                 PFNSSMUSBLOADPREP pfnLoadPrep, PFNSSMUSBLOADEXEC pfnLoadExec, PFNSSMUSBLOADDONE pfnLoadDone)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    VM_ASSERT_EMT(pUsbIns->Internal.s.pVM);
+    LogFlow(("pdmR3UsbHlp_SSMRegister: caller='%s'/%d: uVersion=%#x cbGuess=%#x\n"
+             "    pfnLivePrep=%p pfnLiveExec=%p pfnLiveVote=%p pfnSavePrep=%p pfnSaveExec=%p pfnSaveDone=%p pszLoadPrep=%p pfnLoadExec=%p pfnLoadDone=%p\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, uVersion, cbGuess,
+             pfnLivePrep, pfnLiveExec, pfnLiveVote,
+             pfnSavePrep, pfnSaveExec, pfnSaveDone,
+             pfnLoadPrep, pfnLoadExec, pfnLoadDone));
+
+    int rc = SSMR3RegisterUsb(pUsbIns->Internal.s.pVM, pUsbIns, pUsbIns->pReg->szName, pUsbIns->iInstance,
+                              uVersion, cbGuess,
+                              pfnLivePrep, pfnLiveExec, pfnLiveVote,
+                              pfnSavePrep, pfnSaveExec, pfnSaveDone,
+                              pfnLoadPrep, pfnLoadExec, pfnLoadDone);
+
+    LogFlow(("pdmR3UsbHlp_SSMRegister: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnSTAMRegisterV} */
+static DECLCALLBACK(void) pdmR3UsbHlp_STAMRegisterV(PPDMUSBINS pUsbIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, va_list va)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+
+    int rc = STAMR3RegisterV(pVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, va);
+    AssertRC(rc);
+
+    NOREF(pVM);
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnTMTimerCreate} */
+static DECLCALLBACK(int) pdmR3UsbHlp_TMTimerCreate(PPDMUSBINS pUsbIns, TMCLOCK enmClock, PFNTMTIMERUSB pfnCallback, void *pvUser,
+                                                   uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3UsbHlp_TMTimerCreate: caller='%s'/%d: enmClock=%d pfnCallback=%p pvUser=%p fFlags=%#x pszDesc=%p:{%s} ppTimer=%p\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, enmClock, pfnCallback, pvUser, fFlags, pszDesc, pszDesc, ppTimer));
+
+    if (pUsbIns->iInstance > 0) /** @todo use a string cache here later. */
+    {
+         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_USB_DESC, "%s [%u]", pszDesc, pUsbIns->iInstance);
+         if (pszDesc2)
+             pszDesc = pszDesc2;
+    }
+
+    int rc = TMR3TimerCreateUsb(pVM, pUsbIns, enmClock, pfnCallback, pvUser, fFlags, pszDesc, ppTimer);
+
+    LogFlow(("pdmR3UsbHlp_TMTimerCreate: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnVMSetErrorV} */
+static DECLCALLBACK(int) pdmR3UsbHlp_VMSetErrorV(PPDMUSBINS pUsbIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    int rc2 = VMSetErrorV(pUsbIns->Internal.s.pVM, rc, RT_SRC_POS_ARGS, pszFormat, va); Assert(rc2 == rc); NOREF(rc2);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnVMSetRuntimeErrorV} */
+static DECLCALLBACK(int) pdmR3UsbHlp_VMSetRuntimeErrorV(PPDMUSBINS pUsbIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    int rc = VMSetRuntimeErrorV(pUsbIns->Internal.s.pVM, fFlags, pszErrorId, pszFormat, va);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnVMState} */
+static DECLCALLBACK(VMSTATE) pdmR3UsbHlp_VMState(PPDMUSBINS pUsbIns)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+
+    VMSTATE enmVMState = VMR3GetState(pUsbIns->Internal.s.pVM);
+
+    LogFlow(("pdmR3UsbHlp_VMState: caller='%s'/%d: returns %d (%s)\n", pUsbIns->pReg->szName, pUsbIns->iInstance,
+             enmVMState, VMR3GetStateName(enmVMState)));
+    return enmVMState;
+}
+
+/** @interface_method_impl{PDMUSBHLP,pfnThreadCreate} */
+static DECLCALLBACK(int) pdmR3UsbHlp_ThreadCreate(PPDMUSBINS pUsbIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADUSB pfnThread,
+                                                  PFNPDMTHREADWAKEUPUSB pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    VM_ASSERT_EMT(pUsbIns->Internal.s.pVM);
+    LogFlow(("pdmR3UsbHlp_ThreadCreate: caller='%s'/%d: ppThread=%p pvUser=%p pfnThread=%p pfnWakeup=%p cbStack=%#zx enmType=%d pszName=%p:{%s}\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName, pszName));
+
+    int rc = pdmR3ThreadCreateUsb(pUsbIns->Internal.s.pVM, pUsbIns, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName);
+
+    LogFlow(("pdmR3UsbHlp_ThreadCreate: caller='%s'/%d: returns %Rrc *ppThread=%RTthrd\n", pUsbIns->pReg->szName, pUsbIns->iInstance,
+             rc, *ppThread));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnSetAsyncNotification} */
+static DECLCALLBACK(int) pdmR3UsbHlp_SetAsyncNotification(PPDMUSBINS pUsbIns, PFNPDMUSBASYNCNOTIFY pfnAsyncNotify)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    VM_ASSERT_EMT0(pUsbIns->Internal.s.pVM);
+    LogFlow(("pdmR3UsbHlp_SetAsyncNotification: caller='%s'/%d: pfnAsyncNotify=%p\n", pUsbIns->pReg->szName, pUsbIns->iInstance, pfnAsyncNotify));
+
+    int rc = VINF_SUCCESS;
+    AssertStmt(pfnAsyncNotify, rc = VERR_INVALID_PARAMETER);
+    AssertStmt(!pUsbIns->Internal.s.pfnAsyncNotify, rc = VERR_WRONG_ORDER);
+    AssertStmt(pUsbIns->Internal.s.fVMSuspended || pUsbIns->Internal.s.fVMReset, rc = VERR_WRONG_ORDER);
+    VMSTATE enmVMState = VMR3GetState(pUsbIns->Internal.s.pVM);
+    AssertStmt(   enmVMState == VMSTATE_SUSPENDING
+               || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+               || enmVMState == VMSTATE_SUSPENDING_LS
+               || enmVMState == VMSTATE_RESETTING
+               || enmVMState == VMSTATE_RESETTING_LS
+               || enmVMState == VMSTATE_POWERING_OFF
+               || enmVMState == VMSTATE_POWERING_OFF_LS,
+               rc = VERR_INVALID_STATE);
+
+    if (RT_SUCCESS(rc))
+        pUsbIns->Internal.s.pfnAsyncNotify = pfnAsyncNotify;
+
+    LogFlow(("pdmR3UsbHlp_SetAsyncNotification: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnAsyncNotificationCompleted} */
+static DECLCALLBACK(void) pdmR3UsbHlp_AsyncNotificationCompleted(PPDMUSBINS pUsbIns)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    PVM pVM = pUsbIns->Internal.s.pVM;
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    if (   enmVMState == VMSTATE_SUSPENDING
+        || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+        || enmVMState == VMSTATE_SUSPENDING_LS
+        || enmVMState == VMSTATE_RESETTING
+        || enmVMState == VMSTATE_RESETTING_LS
+        || enmVMState == VMSTATE_POWERING_OFF
+        || enmVMState == VMSTATE_POWERING_OFF_LS)
+    {
+        LogFlow(("pdmR3UsbHlp_AsyncNotificationCompleted: caller='%s'/%d:\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+        VMR3AsyncPdmNotificationWakeupU(pVM->pUVM);
+    }
+    else
+        LogFlow(("pdmR3UsbHlp_AsyncNotificationCompleted: caller='%s'/%d: enmVMState=%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance, enmVMState));
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnVMGetSuspendReason} */
+static DECLCALLBACK(VMSUSPENDREASON) pdmR3UsbHlp_VMGetSuspendReason(PPDMUSBINS pUsbIns)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+    VMSUSPENDREASON enmReason = VMR3GetSuspendReason(pVM->pUVM);
+    LogFlow(("pdmR3UsbHlp_VMGetSuspendReason: caller='%s'/%d: returns %d\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, enmReason));
+    return enmReason;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnVMGetResumeReason} */
+static DECLCALLBACK(VMRESUMEREASON) pdmR3UsbHlp_VMGetResumeReason(PPDMUSBINS pUsbIns)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+    VMRESUMEREASON enmReason = VMR3GetResumeReason(pVM->pUVM);
+    LogFlow(("pdmR3UsbHlp_VMGetResumeReason: caller='%s'/%d: returns %d\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, enmReason));
+    return enmReason;
+}
+
+
+/**
+ * The USB device helper structure.
+ */
+const PDMUSBHLP g_pdmR3UsbHlp =
+{
+    PDM_USBHLP_VERSION,
+    pdmR3UsbHlp_DriverAttach,
+    pdmR3UsbHlp_AssertEMT,
+    pdmR3UsbHlp_AssertOther,
+    pdmR3UsbHlp_DBGFStopV,
+    pdmR3UsbHlp_DBGFInfoRegisterArgv,
+    pdmR3UsbHlp_MMHeapAlloc,
+    pdmR3UsbHlp_MMHeapAllocZ,
+    pdmR3UsbHlp_PDMQueueCreate,
+    pdmR3UsbHlp_SSMRegister,
+    pdmR3UsbHlp_STAMRegisterV,
+    pdmR3UsbHlp_TMTimerCreate,
+    pdmR3UsbHlp_VMSetErrorV,
+    pdmR3UsbHlp_VMSetRuntimeErrorV,
+    pdmR3UsbHlp_VMState,
+    pdmR3UsbHlp_ThreadCreate,
+    pdmR3UsbHlp_SetAsyncNotification,
+    pdmR3UsbHlp_AsyncNotificationCompleted,
+    pdmR3UsbHlp_VMGetSuspendReason,
+    pdmR3UsbHlp_VMGetResumeReason,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    PDM_USBHLP_VERSION
+};
+
+/** @}  */
diff --git a/src/VBox/VMM/VMMR3/PGM.cpp b/src/VBox/VMM/VMMR3/PGM.cpp
new file mode 100644
index 00000000..70b68d9a
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PGM.cpp
@@ -0,0 +1,2782 @@
+/* $Id: PGM.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor. (Mixing stuff here, not good?)
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_pgm PGM - The Page Manager and Monitor
+ *
+ * @sa  @ref grp_pgm
+ *      @subpage pg_pgm_pool
+ *      @subpage pg_pgm_phys
+ *
+ *
+ * @section         sec_pgm_modes           Paging Modes
+ *
+ * There are three memory contexts: Host Context (HC), Guest Context (GC)
+ * and intermediate context.  When talking about paging HC can also be referred
+ * to as "host paging", and GC referred to as "shadow paging".
+ *
+ * We define three basic paging modes: 32-bit, PAE and AMD64. The host paging mode
+ * is defined by the host operating system. The mode used in the shadow paging mode
+ * depends on the host paging mode and what the mode the guest is currently in. The
+ * following relation between the two is defined:
+ *
+ * @verbatim
+     Host > 32-bit |  PAE   | AMD64  |
+   Guest  |        |        |        |
+   ==v================================
+   32-bit   32-bit    PAE     PAE
+   -------|--------|--------|--------|
+   PAE       PAE      PAE     PAE
+   -------|--------|--------|--------|
+   AMD64    AMD64    AMD64    AMD64
+   -------|--------|--------|--------| @endverbatim
+ *
+ * All configuration except those in the diagonal (upper left) are expected to
+ * require special effort from the switcher (i.e. a bit slower).
+ *
+ *
+ *
+ *
+ * @section         sec_pgm_shw             The Shadow Memory Context
+ *
+ *
+ *  [..]
+ *
+ * Because of guest context mappings requires PDPT and PML4 entries to allow
+ * writing on AMD64, the two upper levels will have fixed flags whatever the
+ * guest is thinking of using there. So, when shadowing the PD level we will
+ * calculate the effective flags of PD and all the higher levels. In legacy
+ * PAE mode this only applies to the PWT and PCD bits (the rest are
+ * ignored/reserved/MBZ). We will ignore those bits for the present.
+ *
+ *
+ *
+ * @section         sec_pgm_int             The Intermediate Memory Context
+ *
+ * The world switch goes thru an intermediate memory context which purpose it is
+ * to provide different mappings of the switcher code. All guest mappings are also
+ * present in this context.
+ *
+ * The switcher code is mapped at the same location as on the host, at an
+ * identity mapped location (physical equals virtual address), and at the
+ * hypervisor location. The identity mapped location is for when the world
+ * switches that involves disabling paging.
+ *
+ * PGM maintain page tables for 32-bit, PAE and AMD64 paging modes. This
+ * simplifies switching guest CPU mode and consistency at the cost of more
+ * code to do the work. All memory use for those page tables is located below
+ * 4GB (this includes page tables for guest context mappings).
+ *
+ * Note! The intermediate memory context is also used for 64-bit guest
+ *       execution on 32-bit hosts.  Because we need to load 64-bit registers
+ *       prior to switching to guest context, we need to be in 64-bit mode
+ *       first.  So, HM has some 64-bit worker routines in VMMRC.rc that get
+ *       invoked via the special world switcher code in LegacyToAMD64.asm.
+ *
+ *
+ * @subsection      subsec_pgm_int_gc       Guest Context Mappings
+ *
+ * During assignment and relocation of a guest context mapping the intermediate
+ * memory context is used to verify the new location.
+ *
+ * Guest context mappings are currently restricted to below 4GB, for reasons
+ * of simplicity. This may change when we implement AMD64 support.
+ *
+ *
+ *
+ *
+ * @section         sec_pgm_misc            Misc
+ *
+ *
+ * @subsection      sec_pgm_misc_A20        The A20 Gate
+ *
+ * PGM implements the A20 gate masking when translating a virtual guest address
+ * into a physical address for CPU access, i.e. PGMGstGetPage (and friends) and
+ * the code reading the guest page table entries during shadowing.  The masking
+ * is done consistenly for all CPU modes, paged ones included.  Large pages are
+ * also masked correctly.  (On current CPUs, experiments indicates that AMD does
+ * not apply A20M in paged modes and intel only does it for the 2nd MB of
+ * memory.)
+ *
+ * The A20 gate implementation is per CPU core.  It can be configured on a per
+ * core basis via the keyboard device and PC architecture device.  This is
+ * probably not exactly how real CPUs do it, but SMP and A20 isn't a place where
+ * guest OSes try pushing things anyway, so who cares.  (On current real systems
+ * the A20M signal is probably only sent to the boot CPU and it affects all
+ * thread and probably all cores in that package.)
+ *
+ * The keyboard device and the PC architecture device doesn't OR their A20
+ * config bits together, rather they are currently implemented such that they
+ * mirror the CPU state.  So, flipping the bit in either of them will change the
+ * A20 state.  (On real hardware the bits of the two devices should probably be
+ * ORed together to indicate enabled, i.e. both needs to be cleared to disable
+ * A20 masking.)
+ *
+ * The A20 state will change immediately, transmeta fashion.  There is no delays
+ * due to buses, wiring or other physical stuff.  (On real hardware there are
+ * normally delays, the delays differs between the two devices and probably also
+ * between chipsets and CPU generations. Note that it's said that transmeta CPUs
+ * does the change immediately like us, they apparently intercept/handles the
+ * port accesses in microcode. Neat.)
+ *
+ * @sa http://en.wikipedia.org/wiki/A20_line#The_80286_and_the_high_memory_area
+ *
+ *
+ * @subsection      subsec_pgm_misc_diff    Differences Between Legacy PAE and Long Mode PAE
+ *
+ * The differences between legacy PAE and long mode PAE are:
+ *      -# PDPE bits 1, 2, 5 and 6 are defined differently. In leagcy mode they are
+ *         all marked down as must-be-zero, while in long mode 1, 2 and 5 have the
+ *         usual meanings while 6 is ignored (AMD). This means that upon switching to
+ *         legacy PAE mode we'll have to clear these bits and when going to long mode
+ *         they must be set. This applies to both intermediate and shadow contexts,
+ *         however we don't need to do it for the intermediate one since we're
+ *         executing with CR0.WP at that time.
+ *      -# CR3 allows a 32-byte aligned address in legacy mode, while in long mode
+ *         a page aligned one is required.
+ *
+ *
+ * @section         sec_pgm_handlers        Access Handlers
+ *
+ * Placeholder.
+ *
+ *
+ * @subsection      sec_pgm_handlers_phys   Physical Access Handlers
+ *
+ * Placeholder.
+ *
+ *
+ * @subsection      sec_pgm_handlers_virt   Virtual Access Handlers (obsolete)
+ *
+ * We currently implement three types of virtual access handlers:  ALL, WRITE
+ * and HYPERVISOR (WRITE). See PGMVIRTHANDLERKIND for some more details.
+ *
+ * The HYPERVISOR access handlers is kept in a separate tree since it doesn't apply
+ * to physical pages (PGMTREES::HyperVirtHandlers) and only needs to be consulted in
+ * a special \#PF case. The ALL and WRITE are in the PGMTREES::VirtHandlers tree, the
+ * rest of this section is going to be about these handlers.
+ *
+ * We'll go thru the life cycle of a handler and try make sense of it all, don't know
+ * how successful this is gonna be...
+ *
+ * 1. A handler is registered thru the PGMR3HandlerVirtualRegister and
+ * PGMHandlerVirtualRegisterEx APIs. We check for conflicting virtual handlers
+ * and create a new node that is inserted into the AVL tree (range key). Then
+ * a full PGM resync is flagged (clear pool, sync cr3, update virtual bit of PGMPAGE).
+ *
+ * 2. The following PGMSyncCR3/SyncCR3 operation will first make invoke HandlerVirtualUpdate.
+ *
+ * 2a. HandlerVirtualUpdate will will lookup all the pages covered by virtual handlers
+ * via the current guest CR3 and update the physical page -> virtual handler
+ * translation. Needless to say, this doesn't exactly scale very well. If any changes
+ * are detected, it will flag a virtual bit update just like we did on registration.
+ * PGMPHYS pages with changes will have their virtual handler state reset to NONE.
+ *
+ * 2b. The virtual bit update process will iterate all the pages covered by all the
+ * virtual handlers and update the PGMPAGE virtual handler state to the max of all
+ * virtual handlers on that page.
+ *
+ * 2c. Back in SyncCR3 we will now flush the entire shadow page cache to make sure
+ * we don't miss any alias mappings of the monitored pages.
+ *
+ * 2d. SyncCR3 will then proceed with syncing the CR3 table.
+ *
+ * 3. \#PF(np,read) on a page in the range. This will cause it to be synced
+ * read-only and resumed if it's a WRITE handler. If it's an ALL handler we
+ * will call the handlers like in the next step. If the physical mapping has
+ * changed we will - some time in the future - perform a handler callback
+ * (optional) and update the physical -> virtual handler cache.
+ *
+ * 4. \#PF(,write) on a page in the range. This will cause the handler to
+ * be invoked.
+ *
+ * 5. The guest invalidates the page and changes the physical backing or
+ * unmaps it. This should cause the invalidation callback to be invoked
+ * (it might not yet be 100% perfect). Exactly what happens next... is
+ * this where we mess up and end up out of sync for a while?
+ *
+ * 6. The handler is deregistered by the client via PGMHandlerVirtualDeregister.
+ * We will then set all PGMPAGEs in the physical -> virtual handler cache for
+ * this handler to NONE and trigger a full PGM resync (basically the same
+ * as int step 1). Which means 2 is executed again.
+ *
+ *
+ * @subsubsection   sub_sec_pgm_handler_virt_todo   TODOs
+ *
+ * There is a bunch of things that needs to be done to make the virtual handlers
+ * work 100% correctly and work more efficiently.
+ *
+ * The first bit hasn't been implemented yet because it's going to slow the
+ * whole mess down even more, and besides it seems to be working reliably for
+ * our current uses. OTOH, some of the optimizations might end up more or less
+ * implementing the missing bits, so we'll see.
+ *
+ * On the optimization side, the first thing to do is to try avoid unnecessary
+ * cache flushing. Then try team up with the shadowing code to track changes
+ * in mappings by means of access to them (shadow in), updates to shadows pages,
+ * invlpg, and shadow PT discarding (perhaps).
+ *
+ * Some idea that have popped up for optimization for current and new features:
+ *    - bitmap indicating where there are virtual handlers installed.
+ *      (4KB => 2**20 pages, page 2**12 => covers 32-bit address space 1:1!)
+ *    - Further optimize this by min/max (needs min/max avl getters).
+ *    - Shadow page table entry bit (if any left)?
+ *
+ */
+
+
+/** @page pg_pgm_phys   PGM Physical Guest Memory Management
+ *
+ *
+ * Objectives:
+ *      - Guest RAM over-commitment using memory ballooning,
+ *        zero pages and general page sharing.
+ *      - Moving or mirroring a VM onto a different physical machine.
+ *
+ *
+ * @section sec_pgmPhys_Definitions       Definitions
+ *
+ * Allocation chunk - A RTR0MemObjAllocPhysNC object and the tracking
+ * machinery associated with it.
+ *
+ *
+ *
+ *
+ * @section sec_pgmPhys_AllocPage         Allocating a page.
+ *
+ * Initially we map *all* guest memory to the (per VM) zero page, which
+ * means that none of the read functions will cause pages to be allocated.
+ *
+ * Exception, access bit in page tables that have been shared. This must
+ * be handled, but we must also make sure PGMGst*Modify doesn't make
+ * unnecessary modifications.
+ *
+ * Allocation points:
+ *      - PGMPhysSimpleWriteGCPhys and PGMPhysWrite.
+ *      - Replacing a zero page mapping at \#PF.
+ *      - Replacing a shared page mapping at \#PF.
+ *      - ROM registration (currently MMR3RomRegister).
+ *      - VM restore (pgmR3Load).
+ *
+ * For the first three it would make sense to keep a few pages handy
+ * until we've reached the max memory commitment for the VM.
+ *
+ * For the ROM registration, we know exactly how many pages we need
+ * and will request these from ring-0. For restore, we will save
+ * the number of non-zero pages in the saved state and allocate
+ * them up front. This would allow the ring-0 component to refuse
+ * the request if the isn't sufficient memory available for VM use.
+ *
+ * Btw. for both ROM and restore allocations we won't be requiring
+ * zeroed pages as they are going to be filled instantly.
+ *
+ *
+ * @section sec_pgmPhys_FreePage          Freeing a page
+ *
+ * There are a few points where a page can be freed:
+ *      - After being replaced by the zero page.
+ *      - After being replaced by a shared page.
+ *      - After being ballooned by the guest additions.
+ *      - At reset.
+ *      - At restore.
+ *
+ * When freeing one or more pages they will be returned to the ring-0
+ * component and replaced by the zero page.
+ *
+ * The reasoning for clearing out all the pages on reset is that it will
+ * return us to the exact same state as on power on, and may thereby help
+ * us reduce the memory load on the system. Further it might have a
+ * (temporary) positive influence on memory fragmentation (@see subsec_pgmPhys_Fragmentation).
+ *
+ * On restore, as mention under the allocation topic, pages should be
+ * freed / allocated depending on how many is actually required by the
+ * new VM state. The simplest approach is to do like on reset, and free
+ * all non-ROM pages and then allocate what we need.
+ *
+ * A measure to prevent some fragmentation, would be to let each allocation
+ * chunk have some affinity towards the VM having allocated the most pages
+ * from it. Also, try make sure to allocate from allocation chunks that
+ * are almost full. Admittedly, both these measures might work counter to
+ * our intentions and its probably not worth putting a lot of effort,
+ * cpu time or memory into this.
+ *
+ *
+ * @section sec_pgmPhys_SharePage         Sharing a page
+ *
+ * The basic idea is that there there will be a idle priority kernel
+ * thread walking the non-shared VM pages hashing them and looking for
+ * pages with the same checksum. If such pages are found, it will compare
+ * them byte-by-byte to see if they actually are identical. If found to be
+ * identical it will allocate a shared page, copy the content, check that
+ * the page didn't change while doing this, and finally request both the
+ * VMs to use the shared page instead. If the page is all zeros (special
+ * checksum and byte-by-byte check) it will request the VM that owns it
+ * to replace it with the zero page.
+ *
+ * To make this efficient, we will have to make sure not to try share a page
+ * that will change its contents soon. This part requires the most work.
+ * A simple idea would be to request the VM to write monitor the page for
+ * a while to make sure it isn't modified any time soon. Also, it may
+ * make sense to skip pages that are being write monitored since this
+ * information is readily available to the thread if it works on the
+ * per-VM guest memory structures (presently called PGMRAMRANGE).
+ *
+ *
+ * @section sec_pgmPhys_Fragmentation     Fragmentation Concerns and Counter Measures
+ *
+ * The pages are organized in allocation chunks in ring-0, this is a necessity
+ * if we wish to have an OS agnostic approach to this whole thing. (On Linux we
+ * could easily work on a page-by-page basis if we liked. Whether this is possible
+ * or efficient on NT I don't quite know.) Fragmentation within these chunks may
+ * become a problem as part of the idea here is that we wish to return memory to
+ * the host system.
+ *
+ * For instance, starting two VMs at the same time, they will both allocate the
+ * guest memory on-demand and if permitted their page allocations will be
+ * intermixed. Shut down one of the two VMs and it will be difficult to return
+ * any memory to the host system because the page allocation for the two VMs are
+ * mixed up in the same allocation chunks.
+ *
+ * To further complicate matters, when pages are freed because they have been
+ * ballooned or become shared/zero the whole idea is that the page is supposed
+ * to be reused by another VM or returned to the host system. This will cause
+ * allocation chunks to contain pages belonging to different VMs and prevent
+ * returning memory to the host when one of those VM shuts down.
+ *
+ * The only way to really deal with this problem is to move pages. This can
+ * either be done at VM shutdown and or by the idle priority worker thread
+ * that will be responsible for finding sharable/zero pages. The mechanisms
+ * involved for coercing a VM to move a page (or to do it for it) will be
+ * the same as when telling it to share/zero a page.
+ *
+ *
+ * @section sec_pgmPhys_Tracking      Tracking Structures And Their Cost
+ *
+ * There's a difficult balance between keeping the per-page tracking structures
+ * (global and guest page) easy to use and keeping them from eating too much
+ * memory. We have limited virtual memory resources available when operating in
+ * 32-bit kernel space (on 64-bit there'll it's quite a different story). The
+ * tracking structures will be attempted designed such that we can deal with up
+ * to 32GB of memory on a 32-bit system and essentially unlimited on 64-bit ones.
+ *
+ *
+ * @subsection subsec_pgmPhys_Tracking_Kernel     Kernel Space
+ *
+ * @see pg_GMM
+ *
+ * @subsection subsec_pgmPhys_Tracking_PerVM      Per-VM
+ *
+ * Fixed info is the physical address of the page (HCPhys) and the page id
+ * (described above). Theoretically we'll need 48(-12) bits for the HCPhys part.
+ * Today we've restricting ourselves to 40(-12) bits because this is the current
+ * restrictions of all AMD64 implementations (I think Barcelona will up this
+ * to 48(-12) bits, not that it really matters) and I needed the bits for
+ * tracking mappings of a page. 48-12 = 36. That leaves 28 bits, which means a
+ * decent range for the page id: 2^(28+12) = 1024TB.
+ *
+ * In additions to these, we'll have to keep maintaining the page flags as we
+ * currently do. Although it wouldn't harm to optimize these quite a bit, like
+ * for instance the ROM shouldn't depend on having a write handler installed
+ * in order for it to become read-only. A RO/RW bit should be considered so
+ * that the page syncing code doesn't have to mess about checking multiple
+ * flag combinations (ROM || RW handler || write monitored) in order to
+ * figure out how to setup a shadow PTE. But this of course, is second
+ * priority at present. Current this requires 12 bits, but could probably
+ * be optimized to ~8.
+ *
+ * Then there's the 24 bits used to track which shadow page tables are
+ * currently mapping a page for the purpose of speeding up physical
+ * access handlers, and thereby the page pool cache. More bit for this
+ * purpose wouldn't hurt IIRC.
+ *
+ * Then there is a new bit in which we need to record what kind of page
+ * this is, shared, zero, normal or write-monitored-normal. This'll
+ * require 2 bits. One bit might be needed for indicating whether a
+ * write monitored page has been written to. And yet another one or
+ * two for tracking migration status. 3-4 bits total then.
+ *
+ * Whatever is left will can be used to record the sharabilitiy of a
+ * page. The page checksum will not be stored in the per-VM table as
+ * the idle thread will not be permitted to do modifications to it.
+ * It will instead have to keep its own working set of potentially
+ * shareable pages and their check sums and stuff.
+ *
+ * For the present we'll keep the current packing of the
+ * PGMRAMRANGE::aHCPhys to keep the changes simple, only of course,
+ * we'll have to change it to a struct with a total of 128-bits at
+ * our disposal.
+ *
+ * The initial layout will be like this:
+ * @verbatim
+    RTHCPHYS HCPhys;            The current stuff.
+        63:40                   Current shadow PT tracking stuff.
+        39:12                   The physical page frame number.
+        11:0                    The current flags.
+    uint32_t u28PageId : 28;    The page id.
+    uint32_t u2State : 2;       The page state { zero, shared, normal, write monitored }.
+    uint32_t fWrittenTo : 1;    Whether a write monitored page was written to.
+    uint32_t u1Reserved : 1;    Reserved for later.
+    uint32_t u32Reserved;       Reserved for later, mostly sharing stats.
+ @endverbatim
+ *
+ * The final layout will be something like this:
+ * @verbatim
+    RTHCPHYS HCPhys;            The current stuff.
+        63:48                   High page id (12+).
+        47:12                   The physical page frame number.
+        11:0                    Low page id.
+    uint32_t fReadOnly : 1;     Whether it's readonly page (rom or monitored in some way).
+    uint32_t u3Type : 3;        The page type {RESERVED, MMIO, MMIO2, ROM, shadowed ROM, RAM}.
+    uint32_t u2PhysMon : 2;     Physical access handler type {none, read, write, all}.
+    uint32_t u2VirtMon : 2;     Virtual access handler type {none, read, write, all}..
+    uint32_t u2State : 2;       The page state { zero, shared, normal, write monitored }.
+    uint32_t fWrittenTo : 1;    Whether a write monitored page was written to.
+    uint32_t u20Reserved : 20;  Reserved for later, mostly sharing stats.
+    uint32_t u32Tracking;       The shadow PT tracking stuff, roughly.
+ @endverbatim
+ *
+ * Cost wise, this means we'll double the cost for guest memory. There isn't anyway
+ * around that I'm afraid. It means that the cost of dealing out 32GB of memory
+ * to one or more VMs is: (32GB >> PAGE_SHIFT) * 16 bytes, or 128MBs. Or another
+ * example, the VM heap cost when assigning 1GB to a VM will be: 4MB.
+ *
+ * A couple of cost examples for the total cost per-VM + kernel.
+ * 32-bit Windows and 32-bit linux:
+ *      1GB guest ram, 256K pages:  4MB +  2MB(+) =   6MB
+ *      4GB guest ram, 1M pages:   16MB +  8MB(+) =  24MB
+ *     32GB guest ram, 8M pages:  128MB + 64MB(+) = 192MB
+ * 64-bit Windows and 64-bit linux:
+ *      1GB guest ram, 256K pages:  4MB +  3MB(+) =   7MB
+ *      4GB guest ram, 1M pages:   16MB + 12MB(+) =  28MB
+ *     32GB guest ram, 8M pages:  128MB + 96MB(+) = 224MB
+ *
+ * UPDATE - 2007-09-27:
+ * Will need a ballooned flag/state too because we cannot
+ * trust the guest 100% and reporting the same page as ballooned more
+ * than once will put the GMM off balance.
+ *
+ *
+ * @section sec_pgmPhys_Serializing       Serializing Access
+ *
+ * Initially, we'll try a simple scheme:
+ *
+ *      - The per-VM RAM tracking structures (PGMRAMRANGE) is only modified
+ *        by the EMT thread of that VM while in the pgm critsect.
+ *      - Other threads in the VM process that needs to make reliable use of
+ *        the per-VM RAM tracking structures will enter the critsect.
+ *      - No process external thread or kernel thread will ever try enter
+ *        the pgm critical section, as that just won't work.
+ *      - The idle thread (and similar threads) doesn't not need 100% reliable
+ *        data when performing it tasks as the EMT thread will be the one to
+ *        do the actual changes later anyway. So, as long as it only accesses
+ *        the main ram range, it can do so by somehow preventing the VM from
+ *        being destroyed while it works on it...
+ *
+ *      - The over-commitment management, including the allocating/freeing
+ *        chunks, is serialized by a ring-0 mutex lock (a fast one since the
+ *        more mundane mutex implementation is broken on Linux).
+ *      - A separate mutex is protecting the set of allocation chunks so
+ *        that pages can be shared or/and freed up while some other VM is
+ *        allocating more chunks. This mutex can be take from under the other
+ *        one, but not the other way around.
+ *
+ *
+ * @section sec_pgmPhys_Request           VM Request interface
+ *
+ * When in ring-0 it will become necessary to send requests to a VM so it can
+ * for instance move a page while defragmenting during VM destroy. The idle
+ * thread will make use of this interface to request VMs to setup shared
+ * pages and to perform write monitoring of pages.
+ *
+ * I would propose an interface similar to the current VMReq interface, similar
+ * in that it doesn't require locking and that the one sending the request may
+ * wait for completion if it wishes to. This shouldn't be very difficult to
+ * realize.
+ *
+ * The requests themselves are also pretty simple. They are basically:
+ *      -# Check that some precondition is still true.
+ *      -# Do the update.
+ *      -# Update all shadow page tables involved with the page.
+ *
+ * The 3rd step is identical to what we're already doing when updating a
+ * physical handler, see pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs.
+ *
+ *
+ *
+ * @section sec_pgmPhys_MappingCaches   Mapping Caches
+ *
+ * In order to be able to map in and out memory and to be able to support
+ * guest with more RAM than we've got virtual address space, we'll employing
+ * a mapping cache.  Normally ring-0 and ring-3 can share the same cache,
+ * however on 32-bit darwin the ring-0 code is running in a different memory
+ * context and therefore needs a separate cache.  In raw-mode context we also
+ * need a separate cache.  The 32-bit darwin mapping cache and the one for
+ * raw-mode context share a lot of code, see PGMRZDYNMAP.
+ *
+ *
+ * @subsection subsec_pgmPhys_MappingCaches_R3  Ring-3
+ *
+ * We've considered implementing the ring-3 mapping cache page based but found
+ * that this was bother some when one had to take into account TLBs+SMP and
+ * portability (missing the necessary APIs on several platforms). There were
+ * also some performance concerns with this approach which hadn't quite been
+ * worked out.
+ *
+ * Instead, we'll be mapping allocation chunks into the VM process. This simplifies
+ * matters greatly quite a bit since we don't need to invent any new ring-0 stuff,
+ * only some minor RTR0MEMOBJ mapping stuff. The main concern here is that mapping
+ * compared to the previous idea is that mapping or unmapping a 1MB chunk is more
+ * costly than a single page, although how much more costly is uncertain. We'll
+ * try address this by using a very big cache, preferably bigger than the actual
+ * VM RAM size if possible. The current VM RAM sizes should give some idea for
+ * 32-bit boxes, while on 64-bit we can probably get away with employing an
+ * unlimited cache.
+ *
+ * The cache have to parts, as already indicated, the ring-3 side and the
+ * ring-0 side.
+ *
+ * The ring-0 will be tied to the page allocator since it will operate on the
+ * memory objects it contains. It will therefore require the first ring-0 mutex
+ * discussed in @ref sec_pgmPhys_Serializing.  We some double house keeping wrt
+ * to who has mapped what I think, since both VMMR0.r0 and RTR0MemObj will keep
+ * track of mapping relations
+ *
+ * The ring-3 part will be protected by the pgm critsect. For simplicity, we'll
+ * require anyone that desires to do changes to the mapping cache to do that
+ * from within this critsect. Alternatively, we could employ a separate critsect
+ * for serializing changes to the mapping cache as this would reduce potential
+ * contention with other threads accessing mappings unrelated to the changes
+ * that are in process. We can see about this later, contention will show
+ * up in the statistics anyway, so it'll be simple to tell.
+ *
+ * The organization of the ring-3 part will be very much like how the allocation
+ * chunks are organized in ring-0, that is in an AVL tree by chunk id. To avoid
+ * having to walk the tree all the time, we'll have a couple of lookaside entries
+ * like in we do for I/O ports and MMIO in IOM.
+ *
+ * The simplified flow of a PGMPhysRead/Write function:
+ *      -# Enter the PGM critsect.
+ *      -# Lookup GCPhys in the ram ranges and get the Page ID.
+ *      -# Calc the Allocation Chunk ID from the Page ID.
+ *      -# Check the lookaside entries and then the AVL tree for the Chunk ID.
+ *         If not found in cache:
+ *              -# Call ring-0 and request it to be mapped and supply
+ *                 a chunk to be unmapped if the cache is maxed out already.
+ *              -# Insert the new mapping into the AVL tree (id + R3 address).
+ *      -# Update the relevant lookaside entry and return the mapping address.
+ *      -# Do the read/write according to monitoring flags and everything.
+ *      -# Leave the critsect.
+ *
+ *
+ * @section sec_pgmPhys_Fallback            Fallback
+ *
+ * Current all the "second tier" hosts will not support the RTR0MemObjAllocPhysNC
+ * API and thus require a fallback.
+ *
+ * So, when RTR0MemObjAllocPhysNC returns VERR_NOT_SUPPORTED the page allocator
+ * will return to the ring-3 caller (and later ring-0) and asking it to seed
+ * the page allocator with some fresh pages (VERR_GMM_SEED_ME). Ring-3 will
+ * then perform an SUPR3PageAlloc(cbChunk >> PAGE_SHIFT) call and make a
+ * "SeededAllocPages" call to ring-0.
+ *
+ * The first time ring-0 sees the VERR_NOT_SUPPORTED failure it will disable
+ * all page sharing (zero page detection will continue). It will also force
+ * all allocations to come from the VM which seeded the page. Both these
+ * measures are taken to make sure that there will never be any need for
+ * mapping anything into ring-3 - everything will be mapped already.
+ *
+ * Whether we'll continue to use the current MM locked memory management
+ * for this I don't quite know (I'd prefer not to and just ditch that all
+ * together), we'll see what's simplest to do.
+ *
+ *
+ *
+ * @section sec_pgmPhys_Changes             Changes
+ *
+ * Breakdown of the changes involved?
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/hm.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include "PGMInline.h"
+
+#include <VBox/dbg.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/assert.h>
+#include <iprt/env.h>
+#include <iprt/mem.h>
+#include <iprt/file.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#ifdef RT_OS_LINUX
+# include <iprt/linux/sysfs.h>
+#endif
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Argument package for pgmR3RElocatePhysHnadler, pgmR3RelocateVirtHandler and
+ * pgmR3RelocateHyperVirtHandler.
+ */
+typedef struct PGMRELOCHANDLERARGS
+{
+    RTGCINTPTR  offDelta;
+    PVM         pVM;
+} PGMRELOCHANDLERARGS;
+/** Pointer to a page access handlere relocation argument package. */
+typedef PGMRELOCHANDLERARGS const *PCPGMRELOCHANDLERARGS;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int                pgmR3InitPaging(PVM pVM);
+static int                pgmR3InitStats(PVM pVM);
+static DECLCALLBACK(void) pgmR3PhysInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) pgmR3InfoMode(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) pgmR3InfoCr3(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+#ifdef VBOX_STRICT
+static FNVMATSTATE        pgmR3ResetNoMorePhysWritesFlag;
+#endif
+
+#ifdef VBOX_WITH_DEBUGGER
+static FNDBGCCMD          pgmR3CmdError;
+static FNDBGCCMD          pgmR3CmdSync;
+static FNDBGCCMD          pgmR3CmdSyncAlways;
+# ifdef VBOX_STRICT
+static FNDBGCCMD          pgmR3CmdAssertCR3;
+# endif
+static FNDBGCCMD          pgmR3CmdPhysToFile;
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_DEBUGGER
+/** Argument descriptors for '.pgmerror' and '.pgmerroroff'. */
+static const DBGCVARDESC g_aPgmErrorArgs[] =
+{
+    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
+    {  0,           1,          DBGCVAR_CAT_STRING,     0,                              "where",        "Error injection location." },
+};
+
+static const DBGCVARDESC g_aPgmPhysToFileArgs[] =
+{
+    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
+    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "file",         "The file name." },
+    {  0,           1,          DBGCVAR_CAT_STRING,     0,                              "nozero",       "If present, zero pages are skipped." },
+};
+
+# ifdef DEBUG_sandervl
+static const DBGCVARDESC g_aPgmCountPhysWritesArgs[] =
+{
+    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
+    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "enabled",      "on/off." },
+    {  1,           1,          DBGCVAR_CAT_NUMBER_NO_RANGE, 0,                         "interval",     "Interval in ms." },
+};
+# endif
+
+/** Command descriptors. */
+static const DBGCCMD    g_aCmds[] =
+{
+    /* pszCmd,  cArgsMin, cArgsMax, paArgDesc,                cArgDescs, fFlags, pfnHandler          pszSyntax,          ....pszDescription */
+    { "pgmsync",       0, 0,        NULL,                     0,         0,      pgmR3CmdSync,       "",                     "Sync the CR3 page." },
+    { "pgmerror",      0, 1,        &g_aPgmErrorArgs[0],      1,         0,      pgmR3CmdError,      "",                     "Enables inject runtime of errors into parts of PGM." },
+    { "pgmerroroff",   0, 1,        &g_aPgmErrorArgs[0],      1,         0,      pgmR3CmdError,      "",                     "Disables inject runtime errors into parts of PGM." },
+# ifdef VBOX_STRICT
+    { "pgmassertcr3",  0, 0,        NULL,                     0,         0,      pgmR3CmdAssertCR3,  "",                     "Check the shadow CR3 mapping." },
+#  ifdef VBOX_WITH_PAGE_SHARING
+    { "pgmcheckduppages", 0, 0,     NULL,                     0,         0,      pgmR3CmdCheckDuplicatePages,  "",           "Check for duplicate pages in all running VMs." },
+    { "pgmsharedmodules", 0, 0,     NULL,                     0,         0,      pgmR3CmdShowSharedModules,  "",             "Print shared modules info." },
+#  endif
+# endif
+    { "pgmsyncalways", 0, 0,        NULL,                     0,         0,      pgmR3CmdSyncAlways, "",                     "Toggle permanent CR3 syncing." },
+    { "pgmphystofile", 1, 2,        &g_aPgmPhysToFileArgs[0], 2,         0,      pgmR3CmdPhysToFile, "",                     "Save the physical memory to file." },
+};
+#endif
+
+
+
+
+/**
+ * Initiates the paging of VM.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(int) PGMR3Init(PVM pVM)
+{
+    LogFlow(("PGMR3Init:\n"));
+    PCFGMNODE pCfgPGM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/PGM");
+    int rc;
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompile(sizeof(pVM->pgm.s) <= sizeof(pVM->pgm.padding));
+    AssertCompile(sizeof(pVM->apCpusR3[0]->pgm.s) <= sizeof(pVM->apCpusR3[0]->pgm.padding));
+    AssertCompileMemberAlignment(PGM, CritSectX, sizeof(uintptr_t));
+
+    /*
+     * Init the structure.
+     */
+    pVM->pgm.s.offVM       = RT_UOFFSETOF(VM, pgm.s);
+    pVM->pgm.s.offVCpuPGM  = RT_UOFFSETOF(VMCPU, pgm.s);
+    /*pVM->pgm.s.fRestoreRomPagesAtReset = false;*/
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
+    {
+        pVM->pgm.s.aHandyPages[i].HCPhysGCPhys  = NIL_RTHCPHYS;
+        pVM->pgm.s.aHandyPages[i].idPage        = NIL_GMM_PAGEID;
+        pVM->pgm.s.aHandyPages[i].idSharedPage  = NIL_GMM_PAGEID;
+    }
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.aLargeHandyPage); i++)
+    {
+        pVM->pgm.s.aLargeHandyPage[i].HCPhysGCPhys  = NIL_RTHCPHYS;
+        pVM->pgm.s.aLargeHandyPage[i].idPage        = NIL_GMM_PAGEID;
+        pVM->pgm.s.aLargeHandyPage[i].idSharedPage  = NIL_GMM_PAGEID;
+    }
+
+    /* Init the per-CPU part. */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        PPGMCPU pPGM = &pVCpu->pgm.s;
+
+        pPGM->offVM      = (uintptr_t)&pVCpu->pgm.s - (uintptr_t)pVM;
+        pPGM->offVCpu    = RT_UOFFSETOF(VMCPU, pgm.s);
+        pPGM->offPGM     = (uintptr_t)&pVCpu->pgm.s - (uintptr_t)&pVM->pgm.s;
+
+        pPGM->enmShadowMode     = PGMMODE_INVALID;
+        pPGM->enmGuestMode      = PGMMODE_INVALID;
+        pPGM->idxGuestModeData  = UINT8_MAX;
+        pPGM->idxShadowModeData = UINT8_MAX;
+        pPGM->idxBothModeData   = UINT8_MAX;
+
+        pPGM->GCPhysCR3         = NIL_RTGCPHYS;
+
+        pPGM->pGst32BitPdR3     = NULL;
+        pPGM->pGstPaePdptR3     = NULL;
+        pPGM->pGstAmd64Pml4R3   = NULL;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+        pPGM->pGst32BitPdR0     = NIL_RTR0PTR;
+        pPGM->pGstPaePdptR0     = NIL_RTR0PTR;
+        pPGM->pGstAmd64Pml4R0   = NIL_RTR0PTR;
+#endif
+        for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.apGstPaePDsR3); i++)
+        {
+            pPGM->apGstPaePDsR3[i]             = NULL;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+            pPGM->apGstPaePDsR0[i]             = NIL_RTR0PTR;
+#endif
+            pPGM->aGCPhysGstPaePDs[i]          = NIL_RTGCPHYS;
+            pPGM->aGstPaePdpeRegs[i].u         = UINT64_MAX;
+            pPGM->aGCPhysGstPaePDsMonitored[i] = NIL_RTGCPHYS;
+        }
+
+        pPGM->fA20Enabled      = true;
+        pPGM->GCPhysA20Mask    = ~((RTGCPHYS)!pPGM->fA20Enabled << 20);
+    }
+
+    pVM->pgm.s.enmHostMode      = SUPPAGINGMODE_INVALID;
+    pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(32) - 1; /* default; checked later */
+    pVM->pgm.s.GCPtrPrevRamRangeMapping = MM_HYPER_AREA_ADDRESS;
+
+    rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "RamPreAlloc", &pVM->pgm.s.fRamPreAlloc,
+#ifdef VBOX_WITH_PREALLOC_RAM_BY_DEFAULT
+                            true
+#else
+                            false
+#endif
+                           );
+    AssertLogRelRCReturn(rc, rc);
+
+#if HC_ARCH_BITS == 32
+# ifdef RT_OS_DARWIN
+    rc = CFGMR3QueryU32Def(pCfgPGM, "MaxRing3Chunks", &pVM->pgm.s.ChunkR3Map.cMax, _1G / GMM_CHUNK_SIZE * 3);
+# else
+    rc = CFGMR3QueryU32Def(pCfgPGM, "MaxRing3Chunks", &pVM->pgm.s.ChunkR3Map.cMax, _1G / GMM_CHUNK_SIZE);
+# endif
+#else
+    rc = CFGMR3QueryU32Def(pCfgPGM, "MaxRing3Chunks", &pVM->pgm.s.ChunkR3Map.cMax, UINT32_MAX);
+#endif
+    AssertLogRelRCReturn(rc, rc);
+    for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
+        pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
+
+    /*
+     * Get the configured RAM size - to estimate saved state size.
+     */
+    uint64_t    cbRam;
+    rc = CFGMR3QueryU64(CFGMR3GetRoot(pVM), "RamSize", &cbRam);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        cbRam = 0;
+    else if (RT_SUCCESS(rc))
+    {
+        if (cbRam < PAGE_SIZE)
+            cbRam = 0;
+        cbRam = RT_ALIGN_64(cbRam, PAGE_SIZE);
+    }
+    else
+    {
+        AssertMsgFailed(("Configuration error: Failed to query integer \"RamSize\", rc=%Rrc.\n", rc));
+        return rc;
+    }
+
+    /*
+     * Check for PCI pass-through and other configurables.
+     */
+    rc = CFGMR3QueryBoolDef(pCfgPGM, "PciPassThrough", &pVM->pgm.s.fPciPassthrough, false);
+    AssertMsgRCReturn(rc, ("Configuration error: Failed to query integer \"PciPassThrough\", rc=%Rrc.\n", rc), rc);
+    AssertLogRelReturn(!pVM->pgm.s.fPciPassthrough || pVM->pgm.s.fRamPreAlloc, VERR_INVALID_PARAMETER);
+
+    rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "PageFusionAllowed", &pVM->pgm.s.fPageFusionAllowed, false);
+    AssertLogRelRCReturn(rc, rc);
+
+    /** @cfgm{/PGM/ZeroRamPagesOnReset, boolean, true}
+     * Whether to clear RAM pages on (hard) reset. */
+    rc = CFGMR3QueryBoolDef(pCfgPGM, "ZeroRamPagesOnReset", &pVM->pgm.s.fZeroRamPagesOnReset, true);
+    AssertLogRelRCReturn(rc, rc);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Allocate memory for the statistics before someone tries to use them.
+     */
+    size_t cbTotalStats = RT_ALIGN_Z(sizeof(PGMSTATS), 64) + RT_ALIGN_Z(sizeof(PGMCPUSTATS), 64) * pVM->cCpus;
+    void *pv;
+    rc = MMHyperAlloc(pVM, RT_ALIGN_Z(cbTotalStats, PAGE_SIZE), PAGE_SIZE, MM_TAG_PGM, &pv);
+    AssertRCReturn(rc, rc);
+
+    pVM->pgm.s.pStatsR3 = (PGMSTATS *)pv;
+    pVM->pgm.s.pStatsR0 = MMHyperCCToR0(pVM, pv);
+    pv = (uint8_t *)pv + RT_ALIGN_Z(sizeof(PGMSTATS), 64);
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        pVCpu->pgm.s.pStatsR3 = (PGMCPUSTATS *)pv;
+        pVCpu->pgm.s.pStatsR0 = MMHyperCCToR0(pVM, pv);
+
+        pv = (uint8_t *)pv + RT_ALIGN_Z(sizeof(PGMCPUSTATS), 64);
+    }
+#endif /* VBOX_WITH_STATISTICS */
+
+    /*
+     * Register callbacks, string formatters and the saved state data unit.
+     */
+#ifdef VBOX_STRICT
+    VMR3AtStateRegister(pVM->pUVM, pgmR3ResetNoMorePhysWritesFlag, NULL);
+#endif
+    PGMRegisterStringFormatTypes();
+
+    rc = pgmR3InitSavedState(pVM, cbRam);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Initialize the PGM critical section and flush the phys TLBs
+     */
+    rc = PDMR3CritSectInit(pVM, &pVM->pgm.s.CritSectX, RT_SRC_POS, "PGM");
+    AssertRCReturn(rc, rc);
+
+    PGMR3PhysChunkInvalidateTLB(pVM);
+    pgmPhysInvalidatePageMapTLB(pVM);
+
+    /*
+     * For the time being we sport a full set of handy pages in addition to the base
+     * memory to simplify things.
+     */
+    rc = MMR3ReserveHandyPages(pVM, RT_ELEMENTS(pVM->pgm.s.aHandyPages)); /** @todo this should be changed to PGM_HANDY_PAGES_MIN but this needs proper testing... */
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Trees
+     */
+    rc = MMHyperAlloc(pVM, sizeof(PGMTREES), 0, MM_TAG_PGM, (void **)&pVM->pgm.s.pTreesR3);
+    if (RT_SUCCESS(rc))
+        pVM->pgm.s.pTreesR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pTreesR3);
+
+    /*
+     * Allocate the zero page.
+     */
+    if (RT_SUCCESS(rc))
+    {
+        rc = MMHyperAlloc(pVM, PAGE_SIZE, PAGE_SIZE, MM_TAG_PGM, &pVM->pgm.s.pvZeroPgR3);
+        if (RT_SUCCESS(rc))
+        {
+            pVM->pgm.s.pvZeroPgRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pvZeroPgR3);
+            pVM->pgm.s.pvZeroPgR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pvZeroPgR3);
+            pVM->pgm.s.HCPhysZeroPg = MMR3HyperHCVirt2HCPhys(pVM, pVM->pgm.s.pvZeroPgR3);
+            AssertRelease(pVM->pgm.s.HCPhysZeroPg != NIL_RTHCPHYS);
+        }
+    }
+
+    /*
+     * Allocate the invalid MMIO page.
+     * (The invalid bits in HCPhysInvMmioPg are set later on init complete.)
+     */
+    if (RT_SUCCESS(rc))
+    {
+        rc = MMHyperAlloc(pVM, PAGE_SIZE, PAGE_SIZE, MM_TAG_PGM, &pVM->pgm.s.pvMmioPgR3);
+        if (RT_SUCCESS(rc))
+        {
+            ASMMemFill32(pVM->pgm.s.pvMmioPgR3, PAGE_SIZE, 0xfeedface);
+            pVM->pgm.s.HCPhysMmioPg = MMR3HyperHCVirt2HCPhys(pVM, pVM->pgm.s.pvMmioPgR3);
+            AssertRelease(pVM->pgm.s.HCPhysMmioPg != NIL_RTHCPHYS);
+            pVM->pgm.s.HCPhysInvMmioPg = pVM->pgm.s.HCPhysMmioPg;
+        }
+    }
+
+    /*
+     * Register the physical access handler protecting ROMs.
+     */
+    if (RT_SUCCESS(rc))
+        rc = PGMR3HandlerPhysicalTypeRegister(pVM, PGMPHYSHANDLERKIND_WRITE,
+                                              pgmPhysRomWriteHandler,
+                                              NULL, NULL, "pgmPhysRomWritePfHandler",
+                                              NULL, NULL, "pgmPhysRomWritePfHandler",
+                                              "ROM write protection",
+                                              &pVM->pgm.s.hRomPhysHandlerType);
+
+    /*
+     * Init the paging.
+     */
+    if (RT_SUCCESS(rc))
+        rc = pgmR3InitPaging(pVM);
+
+    /*
+     * Init the page pool.
+     */
+    if (RT_SUCCESS(rc))
+        rc = pgmR3PoolInit(pVM);
+
+    if (RT_SUCCESS(rc))
+    {
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            PVMCPU pVCpu = pVM->apCpusR3[i];
+            rc = PGMHCChangeMode(pVM, pVCpu, PGMMODE_REAL);
+            if (RT_FAILURE(rc))
+                break;
+        }
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Info & statistics
+         */
+        DBGFR3InfoRegisterInternalEx(pVM, "mode",
+                                     "Shows the current paging mode. "
+                                     "Recognizes 'all', 'guest', 'shadow' and 'host' as arguments, defaulting to 'all' if nothing is given.",
+                                     pgmR3InfoMode,
+                                     DBGFINFO_FLAGS_ALL_EMTS);
+        DBGFR3InfoRegisterInternal(pVM, "pgmcr3",
+                                   "Dumps all the entries in the top level paging table. No arguments.",
+                                   pgmR3InfoCr3);
+        DBGFR3InfoRegisterInternal(pVM, "phys",
+                                   "Dumps all the physical address ranges. Pass 'verbose' to get more details.",
+                                   pgmR3PhysInfo);
+        DBGFR3InfoRegisterInternal(pVM, "handlers",
+                                   "Dumps physical, virtual and hyper virtual handlers. "
+                                   "Pass 'phys', 'virt', 'hyper' as argument if only one kind is wanted."
+                                   "Add 'nost' if the statistics are unwanted, use together with 'all' or explicit selection.",
+                                   pgmR3InfoHandlers);
+#ifndef PGM_WITHOUT_MAPPINGS
+        DBGFR3InfoRegisterInternal(pVM, "mappings",
+                                   "Dumps guest mappings.",
+                                   pgmR3MapInfo);
+#endif
+
+        pgmR3InitStats(pVM);
+
+#ifdef VBOX_WITH_DEBUGGER
+        /*
+         * Debugger commands.
+         */
+        static bool s_fRegisteredCmds = false;
+        if (!s_fRegisteredCmds)
+        {
+            int rc2 = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
+            if (RT_SUCCESS(rc2))
+                s_fRegisteredCmds = true;
+        }
+#endif
+
+#ifdef RT_OS_LINUX
+        /*
+         * Log the /proc/sys/vm/max_map_count value on linux as that is
+         * frequently giving us grief when too low.
+         */
+        int64_t const cGuessNeeded = MMR3PhysGetRamSize(pVM) / _2M + 16384 /*guesstimate*/;
+        int64_t       cMaxMapCount = 0;
+        int rc2 = RTLinuxSysFsReadIntFile(10, &cMaxMapCount, "/proc/sys/vm/max_map_count");
+        LogRel(("PGM: /proc/sys/vm/max_map_count = %RI64 (rc2=%Rrc); cGuessNeeded=%RI64\n", cMaxMapCount, rc2, cGuessNeeded));
+        if (RT_SUCCESS(rc2) && cMaxMapCount < cGuessNeeded)
+            LogRel(("PGM: WARNING!!\n"
+                    "PGM: WARNING!! Please increase /proc/sys/vm/max_map_count to at least %RI64 (or reduce the amount of RAM assigned to the VM)!\n"
+                    "PGM: WARNING!!\n", cMaxMapCount));
+
+#endif
+
+        return VINF_SUCCESS;
+    }
+
+    /* Almost no cleanup necessary, MM frees all memory. */
+    PDMR3CritSectDelete(&pVM->pgm.s.CritSectX);
+
+    return rc;
+}
+
+
+/**
+ * Init paging.
+ *
+ * Since we need to check what mode the host is operating in before we can choose
+ * the right paging functions for the host we have to delay this until R0 has
+ * been initialized.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ */
+static int pgmR3InitPaging(PVM pVM)
+{
+    /*
+     * Force a recalculation of modes and switcher so everyone gets notified.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+
+        pVCpu->pgm.s.enmShadowMode     = PGMMODE_INVALID;
+        pVCpu->pgm.s.enmGuestMode      = PGMMODE_INVALID;
+        pVCpu->pgm.s.idxGuestModeData  = UINT8_MAX;
+        pVCpu->pgm.s.idxShadowModeData = UINT8_MAX;
+        pVCpu->pgm.s.idxBothModeData   = UINT8_MAX;
+    }
+
+    pVM->pgm.s.enmHostMode   = SUPPAGINGMODE_INVALID;
+
+#ifndef PGM_WITHOUT_MAPPINGS
+    /*
+     * Allocate static mapping space for whatever the cr3 register
+     * points to and in the case of PAE mode to the 4 PDs.
+     */
+    int rc = MMR3HyperReserve(pVM, PAGE_SIZE * 5, "CR3 mapping", &pVM->pgm.s.GCPtrCR3Mapping);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Failed to reserve two pages for cr mapping in HMA, rc=%Rrc\n", rc));
+        return rc;
+    }
+    MMR3HyperReserveFence(pVM);
+#endif
+
+#if 0
+    /*
+     * Allocate pages for the three possible intermediate contexts
+     * (AMD64, PAE and plain 32-Bit). We maintain all three contexts
+     * for the sake of simplicity. The AMD64 uses the PAE for the
+     * lower levels, making the total number of pages 11 (3 + 7 + 1).
+     *
+     * We assume that two page tables will be enought for the core code
+     * mappings (HC virtual and identity).
+     */
+    pVM->pgm.s.pInterPD         = (PX86PD)MMR3PageAllocLow(pVM);    AssertReturn(pVM->pgm.s.pInterPD,         VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPTs[0]    = (PX86PT)MMR3PageAllocLow(pVM);    AssertReturn(pVM->pgm.s.apInterPTs[0],    VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPTs[1]    = (PX86PT)MMR3PageAllocLow(pVM);    AssertReturn(pVM->pgm.s.apInterPTs[1],    VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPaePTs[0] = (PX86PTPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePTs[0], VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPaePTs[1] = (PX86PTPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePTs[1], VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPaePDs[0] = (PX86PDPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePDs[0], VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPaePDs[1] = (PX86PDPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePDs[1], VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPaePDs[2] = (PX86PDPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePDs[2], VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPaePDs[3] = (PX86PDPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePDs[3], VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.pInterPaePDPT    = (PX86PDPT)MMR3PageAllocLow(pVM);  AssertReturn(pVM->pgm.s.pInterPaePDPT,    VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.pInterPaePDPT64  = (PX86PDPT)MMR3PageAllocLow(pVM);  AssertReturn(pVM->pgm.s.pInterPaePDPT64,  VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.pInterPaePML4    = (PX86PML4)MMR3PageAllocLow(pVM);  AssertReturn(pVM->pgm.s.pInterPaePML4,    VERR_NO_PAGE_MEMORY);
+
+    pVM->pgm.s.HCPhysInterPD = MMPage2Phys(pVM, pVM->pgm.s.pInterPD);
+    AssertRelease(pVM->pgm.s.HCPhysInterPD != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPD & PAGE_OFFSET_MASK));
+    pVM->pgm.s.HCPhysInterPaePDPT = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT);
+    AssertRelease(pVM->pgm.s.HCPhysInterPaePDPT != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPaePDPT & PAGE_OFFSET_MASK));
+    pVM->pgm.s.HCPhysInterPaePML4 = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePML4);
+    AssertRelease(pVM->pgm.s.HCPhysInterPaePML4 != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPaePML4 & PAGE_OFFSET_MASK) && pVM->pgm.s.HCPhysInterPaePML4 < 0xffffffff);
+
+    /*
+     * Initialize the pages, setting up the PML4 and PDPT for repetitive 4GB action.
+     */
+    ASMMemZeroPage(pVM->pgm.s.pInterPD);
+    ASMMemZeroPage(pVM->pgm.s.apInterPTs[0]);
+    ASMMemZeroPage(pVM->pgm.s.apInterPTs[1]);
+
+    ASMMemZeroPage(pVM->pgm.s.apInterPaePTs[0]);
+    ASMMemZeroPage(pVM->pgm.s.apInterPaePTs[1]);
+
+    ASMMemZeroPage(pVM->pgm.s.pInterPaePDPT);
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.apInterPaePDs); i++)
+    {
+        ASMMemZeroPage(pVM->pgm.s.apInterPaePDs[i]);
+        pVM->pgm.s.pInterPaePDPT->a[i].u = X86_PDPE_P | PGM_PLXFLAGS_PERMANENT
+                                          | MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[i]);
+    }
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.pInterPaePDPT64->a); i++)
+    {
+        const unsigned iPD = i % RT_ELEMENTS(pVM->pgm.s.apInterPaePDs);
+        pVM->pgm.s.pInterPaePDPT64->a[i].u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A | PGM_PLXFLAGS_PERMANENT
+                                            | MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[iPD]);
+    }
+
+    RTHCPHYS HCPhysInterPaePDPT64 = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT64);
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.pInterPaePML4->a); i++)
+        pVM->pgm.s.pInterPaePML4->a[i].u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A | PGM_PLXFLAGS_PERMANENT
+                                         | HCPhysInterPaePDPT64;
+#endif
+
+    /*
+     * Initialize paging workers and mode from current host mode
+     * and the guest running in real mode.
+     */
+    pVM->pgm.s.enmHostMode = SUPR3GetPagingMode();
+    switch (pVM->pgm.s.enmHostMode)
+    {
+        case SUPPAGINGMODE_32_BIT:
+        case SUPPAGINGMODE_32_BIT_GLOBAL:
+        case SUPPAGINGMODE_PAE:
+        case SUPPAGINGMODE_PAE_GLOBAL:
+        case SUPPAGINGMODE_PAE_NX:
+        case SUPPAGINGMODE_PAE_GLOBAL_NX:
+            break;
+
+        case SUPPAGINGMODE_AMD64:
+        case SUPPAGINGMODE_AMD64_GLOBAL:
+        case SUPPAGINGMODE_AMD64_NX:
+        case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+            if (ARCH_BITS != 64)
+            {
+                AssertMsgFailed(("Host mode %d (64-bit) is not supported by non-64bit builds\n", pVM->pgm.s.enmHostMode));
+                LogRel(("PGM: Host mode %d (64-bit) is not supported by non-64bit builds\n", pVM->pgm.s.enmHostMode));
+                return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
+            }
+            break;
+        default:
+            AssertMsgFailed(("Host mode %d is not supported\n", pVM->pgm.s.enmHostMode));
+            return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
+    }
+
+    LogFlow(("pgmR3InitPaging: returns successfully\n"));
+#if HC_ARCH_BITS == 64 && 0
+    LogRel(("PGM: HCPhysInterPD=%RHp HCPhysInterPaePDPT=%RHp HCPhysInterPaePML4=%RHp\n",
+            pVM->pgm.s.HCPhysInterPD, pVM->pgm.s.HCPhysInterPaePDPT, pVM->pgm.s.HCPhysInterPaePML4));
+    LogRel(("PGM: apInterPTs={%RHp,%RHp} apInterPaePTs={%RHp,%RHp} apInterPaePDs={%RHp,%RHp,%RHp,%RHp} pInterPaePDPT64=%RHp\n",
+            MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[0]),    MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[1]),
+            MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[0]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[1]),
+            MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[0]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[1]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[2]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[3]),
+            MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT64)));
+#endif
+
+    /*
+     * Log the host paging mode. It may come in handy.
+     */
+    const char *pszHostMode;
+    switch (pVM->pgm.s.enmHostMode)
+    {
+        case SUPPAGINGMODE_32_BIT:              pszHostMode = "32-bit"; break;
+        case SUPPAGINGMODE_32_BIT_GLOBAL:       pszHostMode = "32-bit+PGE"; break;
+        case SUPPAGINGMODE_PAE:                 pszHostMode = "PAE"; break;
+        case SUPPAGINGMODE_PAE_GLOBAL:          pszHostMode = "PAE+PGE"; break;
+        case SUPPAGINGMODE_PAE_NX:              pszHostMode = "PAE+NXE"; break;
+        case SUPPAGINGMODE_PAE_GLOBAL_NX:       pszHostMode = "PAE+PGE+NXE"; break;
+        case SUPPAGINGMODE_AMD64:               pszHostMode = "AMD64"; break;
+        case SUPPAGINGMODE_AMD64_GLOBAL:        pszHostMode = "AMD64+PGE"; break;
+        case SUPPAGINGMODE_AMD64_NX:            pszHostMode = "AMD64+NX"; break;
+        case SUPPAGINGMODE_AMD64_GLOBAL_NX:     pszHostMode = "AMD64+PGE+NX"; break;
+        default:                                pszHostMode = "???"; break;
+    }
+    LogRel(("PGM: Host paging mode: %s\n", pszHostMode));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Init statistics
+ * @returns VBox status code.
+ */
+static int pgmR3InitStats(PVM pVM)
+{
+    PPGM pPGM = &pVM->pgm.s;
+    int  rc;
+
+    /*
+     * Release statistics.
+     */
+    /* Common - misc variables */
+    STAM_REL_REG(pVM, &pPGM->cAllPages,                          STAMTYPE_U32,     "/PGM/Page/cAllPages",                STAMUNIT_COUNT,     "The total number of pages.");
+    STAM_REL_REG(pVM, &pPGM->cPrivatePages,                      STAMTYPE_U32,     "/PGM/Page/cPrivatePages",            STAMUNIT_COUNT,     "The number of private pages.");
+    STAM_REL_REG(pVM, &pPGM->cSharedPages,                       STAMTYPE_U32,     "/PGM/Page/cSharedPages",             STAMUNIT_COUNT,     "The number of shared pages.");
+    STAM_REL_REG(pVM, &pPGM->cReusedSharedPages,                 STAMTYPE_U32,     "/PGM/Page/cReusedSharedPages",       STAMUNIT_COUNT,     "The number of reused shared pages.");
+    STAM_REL_REG(pVM, &pPGM->cZeroPages,                         STAMTYPE_U32,     "/PGM/Page/cZeroPages",               STAMUNIT_COUNT,     "The number of zero backed pages.");
+    STAM_REL_REG(pVM, &pPGM->cPureMmioPages,                     STAMTYPE_U32,     "/PGM/Page/cPureMmioPages",           STAMUNIT_COUNT,     "The number of pure MMIO pages.");
+    STAM_REL_REG(pVM, &pPGM->cMonitoredPages,                    STAMTYPE_U32,     "/PGM/Page/cMonitoredPages",          STAMUNIT_COUNT,     "The number of write monitored pages.");
+    STAM_REL_REG(pVM, &pPGM->cWrittenToPages,                    STAMTYPE_U32,     "/PGM/Page/cWrittenToPages",          STAMUNIT_COUNT,     "The number of previously write monitored pages that have been written to.");
+    STAM_REL_REG(pVM, &pPGM->cWriteLockedPages,                  STAMTYPE_U32,     "/PGM/Page/cWriteLockedPages",        STAMUNIT_COUNT,     "The number of write(/read) locked pages.");
+    STAM_REL_REG(pVM, &pPGM->cReadLockedPages,                   STAMTYPE_U32,     "/PGM/Page/cReadLockedPages",         STAMUNIT_COUNT,     "The number of read (only) locked pages.");
+    STAM_REL_REG(pVM, &pPGM->cBalloonedPages,                    STAMTYPE_U32,     "/PGM/Page/cBalloonedPages",          STAMUNIT_COUNT,     "The number of ballooned pages.");
+    STAM_REL_REG(pVM, &pPGM->cHandyPages,                        STAMTYPE_U32,     "/PGM/Page/cHandyPages",              STAMUNIT_COUNT,     "The number of handy pages (not included in cAllPages).");
+    STAM_REL_REG(pVM, &pPGM->cLargePages,                        STAMTYPE_U32,     "/PGM/Page/cLargePages",              STAMUNIT_COUNT,     "The number of large pages allocated (includes disabled).");
+    STAM_REL_REG(pVM, &pPGM->cLargePagesDisabled,                STAMTYPE_U32,     "/PGM/Page/cLargePagesDisabled",      STAMUNIT_COUNT,     "The number of disabled large pages.");
+    STAM_REL_REG(pVM, &pPGM->cRelocations,                       STAMTYPE_COUNTER, "/PGM/cRelocations",                  STAMUNIT_OCCURENCES,"Number of hypervisor relocations.");
+    STAM_REL_REG(pVM, &pPGM->ChunkR3Map.c,                       STAMTYPE_U32,     "/PGM/ChunkR3Map/c",                  STAMUNIT_COUNT,     "Number of mapped chunks.");
+    STAM_REL_REG(pVM, &pPGM->ChunkR3Map.cMax,                    STAMTYPE_U32,     "/PGM/ChunkR3Map/cMax",               STAMUNIT_COUNT,     "Maximum number of mapped chunks.");
+    STAM_REL_REG(pVM, &pPGM->cMappedChunks,                      STAMTYPE_U32,     "/PGM/ChunkR3Map/Mapped",             STAMUNIT_COUNT,     "Number of times we mapped a chunk.");
+    STAM_REL_REG(pVM, &pPGM->cUnmappedChunks,                    STAMTYPE_U32,     "/PGM/ChunkR3Map/Unmapped",           STAMUNIT_COUNT,     "Number of times we unmapped a chunk.");
+
+    STAM_REL_REG(pVM, &pPGM->StatLargePageReused,                STAMTYPE_COUNTER, "/PGM/LargePage/Reused",              STAMUNIT_OCCURENCES, "The number of times we've reused a large page.");
+    STAM_REL_REG(pVM, &pPGM->StatLargePageRefused,               STAMTYPE_COUNTER, "/PGM/LargePage/Refused",             STAMUNIT_OCCURENCES, "The number of times we couldn't use a large page.");
+    STAM_REL_REG(pVM, &pPGM->StatLargePageRecheck,               STAMTYPE_COUNTER, "/PGM/LargePage/Recheck",             STAMUNIT_OCCURENCES, "The number of times we've rechecked a disabled large page.");
+
+    STAM_REL_REG(pVM, &pPGM->StatShModCheck,                     STAMTYPE_PROFILE, "/PGM/ShMod/Check",                   STAMUNIT_TICKS_PER_CALL, "Profiles the shared module checking.");
+
+    /* Live save */
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.fActive,              STAMTYPE_U8,      "/PGM/LiveSave/fActive",              STAMUNIT_COUNT,     "Active or not.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cIgnoredPages,        STAMTYPE_U32,     "/PGM/LiveSave/cIgnoredPages",        STAMUNIT_COUNT,     "The number of ignored pages in the RAM ranges (i.e. MMIO, MMIO2 and ROM).");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cDirtyPagesLong,      STAMTYPE_U32,     "/PGM/LiveSave/cDirtyPagesLong",      STAMUNIT_COUNT,     "Longer term dirty page average.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cDirtyPagesShort,     STAMTYPE_U32,     "/PGM/LiveSave/cDirtyPagesShort",     STAMUNIT_COUNT,     "Short term dirty page average.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cPagesPerSecond,      STAMTYPE_U32,     "/PGM/LiveSave/cPagesPerSecond",      STAMUNIT_COUNT,     "Pages per second.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cSavedPages,          STAMTYPE_U64,     "/PGM/LiveSave/cSavedPages",          STAMUNIT_COUNT,     "The total number of saved pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cReadyPages,      STAMTYPE_U32,     "/PGM/LiveSave/Ram/cReadPages",       STAMUNIT_COUNT,     "RAM: Ready pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cDirtyPages,      STAMTYPE_U32,     "/PGM/LiveSave/Ram/cDirtyPages",      STAMUNIT_COUNT,     "RAM: Dirty pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cZeroPages,       STAMTYPE_U32,     "/PGM/LiveSave/Ram/cZeroPages",       STAMUNIT_COUNT,     "RAM: Ready zero pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cMonitoredPages,  STAMTYPE_U32,     "/PGM/LiveSave/Ram/cMonitoredPages",  STAMUNIT_COUNT,     "RAM: Write monitored pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cReadyPages,      STAMTYPE_U32,     "/PGM/LiveSave/Rom/cReadPages",       STAMUNIT_COUNT,     "ROM: Ready pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cDirtyPages,      STAMTYPE_U32,     "/PGM/LiveSave/Rom/cDirtyPages",      STAMUNIT_COUNT,     "ROM: Dirty pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cZeroPages,       STAMTYPE_U32,     "/PGM/LiveSave/Rom/cZeroPages",       STAMUNIT_COUNT,     "ROM: Ready zero pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cMonitoredPages,  STAMTYPE_U32,     "/PGM/LiveSave/Rom/cMonitoredPages",  STAMUNIT_COUNT,     "ROM: Write monitored pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cReadyPages,    STAMTYPE_U32,     "/PGM/LiveSave/Mmio2/cReadPages",     STAMUNIT_COUNT,     "MMIO2: Ready pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cDirtyPages,    STAMTYPE_U32,     "/PGM/LiveSave/Mmio2/cDirtyPages",    STAMUNIT_COUNT,     "MMIO2: Dirty pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cZeroPages,     STAMTYPE_U32,     "/PGM/LiveSave/Mmio2/cZeroPages",     STAMUNIT_COUNT,     "MMIO2: Ready zero pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cMonitoredPages,STAMTYPE_U32,     "/PGM/LiveSave/Mmio2/cMonitoredPages",STAMUNIT_COUNT,     "MMIO2: Write monitored pages.");
+
+#ifdef VBOX_WITH_STATISTICS
+
+# define PGM_REG_COUNTER(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b); \
+        AssertRC(rc);
+
+# define PGM_REG_COUNTER_BYTES(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, c, b); \
+        AssertRC(rc);
+
+# define PGM_REG_PROFILE(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b); \
+        AssertRC(rc);
+
+    PGMSTATS *pStats = pVM->pgm.s.pStatsR3;
+
+    PGM_REG_PROFILE(&pStats->StatAllocLargePage,                "/PGM/LargePage/Prof/Alloc",          "Time spent by the host OS for large page allocation.");
+    PGM_REG_PROFILE(&pStats->StatClearLargePage,                "/PGM/LargePage/Prof/Clear",          "Time spent clearing the newly allocated large pages.");
+    PGM_REG_COUNTER(&pStats->StatLargePageOverflow,             "/PGM/LargePage/Overflow",            "The number of times allocating a large page took too long.");
+    PGM_REG_PROFILE(&pStats->StatR3IsValidLargePage,            "/PGM/LargePage/Prof/R3/IsValid",     "pgmPhysIsValidLargePage profiling - R3.");
+    PGM_REG_PROFILE(&pStats->StatRZIsValidLargePage,            "/PGM/LargePage/Prof/RZ/IsValid",     "pgmPhysIsValidLargePage profiling - RZ.");
+
+    PGM_REG_COUNTER(&pStats->StatR3DetectedConflicts,           "/PGM/R3/DetectedConflicts",          "The number of times PGMR3CheckMappingConflicts() detected a conflict.");
+    PGM_REG_PROFILE(&pStats->StatR3ResolveConflict,             "/PGM/R3/ResolveConflict",            "pgmR3SyncPTResolveConflict() profiling (includes the entire relocation).");
+    PGM_REG_COUNTER(&pStats->StatR3PhysRead,                    "/PGM/R3/Phys/Read",                  "The number of times PGMPhysRead was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatR3PhysReadBytes,         "/PGM/R3/Phys/Read/Bytes",            "The number of bytes read by PGMPhysRead.");
+    PGM_REG_COUNTER(&pStats->StatR3PhysWrite,                   "/PGM/R3/Phys/Write",                 "The number of times PGMPhysWrite was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatR3PhysWriteBytes,        "/PGM/R3/Phys/Write/Bytes",           "The number of bytes written by PGMPhysWrite.");
+    PGM_REG_COUNTER(&pStats->StatR3PhysSimpleRead,              "/PGM/R3/Phys/Simple/Read",           "The number of times PGMPhysSimpleReadGCPtr was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatR3PhysSimpleReadBytes,   "/PGM/R3/Phys/Simple/Read/Bytes",     "The number of bytes read by PGMPhysSimpleReadGCPtr.");
+    PGM_REG_COUNTER(&pStats->StatR3PhysSimpleWrite,             "/PGM/R3/Phys/Simple/Write",          "The number of times PGMPhysSimpleWriteGCPtr was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatR3PhysSimpleWriteBytes,  "/PGM/R3/Phys/Simple/Write/Bytes",    "The number of bytes written by PGMPhysSimpleWriteGCPtr.");
+
+    PGM_REG_COUNTER(&pStats->StatRZChunkR3MapTlbHits,           "/PGM/ChunkR3Map/TlbHitsRZ",          "TLB hits.");
+    PGM_REG_COUNTER(&pStats->StatRZChunkR3MapTlbMisses,         "/PGM/ChunkR3Map/TlbMissesRZ",        "TLB misses.");
+    PGM_REG_PROFILE(&pStats->StatChunkAging,                    "/PGM/ChunkR3Map/Map/Aging",          "Chunk aging profiling.");
+    PGM_REG_PROFILE(&pStats->StatChunkFindCandidate,            "/PGM/ChunkR3Map/Map/Find",           "Chunk unmap find profiling.");
+    PGM_REG_PROFILE(&pStats->StatChunkUnmap,                    "/PGM/ChunkR3Map/Map/Unmap",          "Chunk unmap of address space profiling.");
+    PGM_REG_PROFILE(&pStats->StatChunkMap,                      "/PGM/ChunkR3Map/Map/Map",            "Chunk map of address space profiling.");
+
+    PGM_REG_COUNTER(&pStats->StatRZPageMapTlbHits,              "/PGM/RZ/Page/MapTlbHits",            "TLB hits.");
+    PGM_REG_COUNTER(&pStats->StatRZPageMapTlbMisses,            "/PGM/RZ/Page/MapTlbMisses",          "TLB misses.");
+    PGM_REG_COUNTER(&pStats->StatR3ChunkR3MapTlbHits,           "/PGM/ChunkR3Map/TlbHitsR3",          "TLB hits.");
+    PGM_REG_COUNTER(&pStats->StatR3ChunkR3MapTlbMisses,         "/PGM/ChunkR3Map/TlbMissesR3",        "TLB misses.");
+    PGM_REG_COUNTER(&pStats->StatR3PageMapTlbHits,              "/PGM/R3/Page/MapTlbHits",            "TLB hits.");
+    PGM_REG_COUNTER(&pStats->StatR3PageMapTlbMisses,            "/PGM/R3/Page/MapTlbMisses",          "TLB misses.");
+    PGM_REG_COUNTER(&pStats->StatPageMapTlbFlushes,             "/PGM/R3/Page/MapTlbFlushes",         "TLB flushes (all contexts).");
+    PGM_REG_COUNTER(&pStats->StatPageMapTlbFlushEntry,          "/PGM/R3/Page/MapTlbFlushEntry",      "TLB entry flushes (all contexts).");
+
+    PGM_REG_COUNTER(&pStats->StatRZRamRangeTlbHits,             "/PGM/RZ/RamRange/TlbHits",           "TLB hits.");
+    PGM_REG_COUNTER(&pStats->StatRZRamRangeTlbMisses,           "/PGM/RZ/RamRange/TlbMisses",         "TLB misses.");
+    PGM_REG_COUNTER(&pStats->StatR3RamRangeTlbHits,             "/PGM/R3/RamRange/TlbHits",           "TLB hits.");
+    PGM_REG_COUNTER(&pStats->StatR3RamRangeTlbMisses,           "/PGM/R3/RamRange/TlbMisses",         "TLB misses.");
+
+    PGM_REG_COUNTER(&pStats->StatRZPhysHandlerReset,            "/PGM/RZ/PhysHandlerReset",           "The number of times PGMHandlerPhysicalReset is called.");
+    PGM_REG_COUNTER(&pStats->StatR3PhysHandlerReset,            "/PGM/R3/PhysHandlerReset",           "The number of times PGMHandlerPhysicalReset is called.");
+    PGM_REG_COUNTER(&pStats->StatRZPhysHandlerLookupHits,       "/PGM/RZ/PhysHandlerLookupHits",      "The number of cache hits when looking up physical handlers.");
+    PGM_REG_COUNTER(&pStats->StatR3PhysHandlerLookupHits,       "/PGM/R3/PhysHandlerLookupHits",      "The number of cache hits when looking up physical handlers.");
+    PGM_REG_COUNTER(&pStats->StatRZPhysHandlerLookupMisses,     "/PGM/RZ/PhysHandlerLookupMisses",    "The number of cache misses when looking up physical handlers.");
+    PGM_REG_COUNTER(&pStats->StatR3PhysHandlerLookupMisses,     "/PGM/R3/PhysHandlerLookupMisses",    "The number of cache misses when looking up physical handlers.");
+
+    PGM_REG_COUNTER(&pStats->StatRZPageReplaceShared,           "/PGM/RZ/Page/ReplacedShared",        "Times a shared page was replaced.");
+    PGM_REG_COUNTER(&pStats->StatRZPageReplaceZero,             "/PGM/RZ/Page/ReplacedZero",          "Times the zero page was replaced.");
+/// @todo    PGM_REG_COUNTER(&pStats->StatRZPageHandyAllocs,             "/PGM/RZ/Page/HandyAllocs",               "Number of times we've allocated more handy pages.");
+    PGM_REG_COUNTER(&pStats->StatR3PageReplaceShared,           "/PGM/R3/Page/ReplacedShared",        "Times a shared page was replaced.");
+    PGM_REG_COUNTER(&pStats->StatR3PageReplaceZero,             "/PGM/R3/Page/ReplacedZero",          "Times the zero page was replaced.");
+/// @todo    PGM_REG_COUNTER(&pStats->StatR3PageHandyAllocs,             "/PGM/R3/Page/HandyAllocs",               "Number of times we've allocated more handy pages.");
+
+    PGM_REG_COUNTER(&pStats->StatRZPhysRead,                    "/PGM/RZ/Phys/Read",                  "The number of times PGMPhysRead was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRZPhysReadBytes,         "/PGM/RZ/Phys/Read/Bytes",            "The number of bytes read by PGMPhysRead.");
+    PGM_REG_COUNTER(&pStats->StatRZPhysWrite,                   "/PGM/RZ/Phys/Write",                 "The number of times PGMPhysWrite was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRZPhysWriteBytes,        "/PGM/RZ/Phys/Write/Bytes",           "The number of bytes written by PGMPhysWrite.");
+    PGM_REG_COUNTER(&pStats->StatRZPhysSimpleRead,              "/PGM/RZ/Phys/Simple/Read",           "The number of times PGMPhysSimpleReadGCPtr was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRZPhysSimpleReadBytes,   "/PGM/RZ/Phys/Simple/Read/Bytes",     "The number of bytes read by PGMPhysSimpleReadGCPtr.");
+    PGM_REG_COUNTER(&pStats->StatRZPhysSimpleWrite,             "/PGM/RZ/Phys/Simple/Write",          "The number of times PGMPhysSimpleWriteGCPtr was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRZPhysSimpleWriteBytes,  "/PGM/RZ/Phys/Simple/Write/Bytes",    "The number of bytes written by PGMPhysSimpleWriteGCPtr.");
+
+    /* GC only: */
+    PGM_REG_COUNTER(&pStats->StatRCInvlPgConflict,              "/PGM/RC/InvlPgConflict",             "Number of times PGMInvalidatePage() detected a mapping conflict.");
+    PGM_REG_COUNTER(&pStats->StatRCInvlPgSyncMonCR3,            "/PGM/RC/InvlPgSyncMonitorCR3",       "Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3.");
+
+    PGM_REG_COUNTER(&pStats->StatRCPhysRead,                    "/PGM/RC/Phys/Read",                  "The number of times PGMPhysRead was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRCPhysReadBytes,         "/PGM/RC/Phys/Read/Bytes",            "The number of bytes read by PGMPhysRead.");
+    PGM_REG_COUNTER(&pStats->StatRCPhysWrite,                   "/PGM/RC/Phys/Write",                 "The number of times PGMPhysWrite was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRCPhysWriteBytes,        "/PGM/RC/Phys/Write/Bytes",           "The number of bytes written by PGMPhysWrite.");
+    PGM_REG_COUNTER(&pStats->StatRCPhysSimpleRead,              "/PGM/RC/Phys/Simple/Read",           "The number of times PGMPhysSimpleReadGCPtr was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRCPhysSimpleReadBytes,   "/PGM/RC/Phys/Simple/Read/Bytes",     "The number of bytes read by PGMPhysSimpleReadGCPtr.");
+    PGM_REG_COUNTER(&pStats->StatRCPhysSimpleWrite,             "/PGM/RC/Phys/Simple/Write",          "The number of times PGMPhysSimpleWriteGCPtr was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRCPhysSimpleWriteBytes,  "/PGM/RC/Phys/Simple/Write/Bytes",    "The number of bytes written by PGMPhysSimpleWriteGCPtr.");
+
+    PGM_REG_COUNTER(&pStats->StatTrackVirgin,                   "/PGM/Track/Virgin",                  "The number of first time shadowings");
+    PGM_REG_COUNTER(&pStats->StatTrackAliased,                  "/PGM/Track/Aliased",                 "The number of times switching to cRef2, i.e. the page is being shadowed by two PTs.");
+    PGM_REG_COUNTER(&pStats->StatTrackAliasedMany,              "/PGM/Track/AliasedMany",             "The number of times we're tracking using cRef2.");
+    PGM_REG_COUNTER(&pStats->StatTrackAliasedLots,              "/PGM/Track/AliasedLots",             "The number of times we're hitting pages which has overflowed cRef2");
+    PGM_REG_COUNTER(&pStats->StatTrackOverflows,                "/PGM/Track/Overflows",               "The number of times the extent list grows too long.");
+    PGM_REG_COUNTER(&pStats->StatTrackNoExtentsLeft,            "/PGM/Track/NoExtentLeft",            "The number of times the extent list was exhausted.");
+    PGM_REG_PROFILE(&pStats->StatTrackDeref,                    "/PGM/Track/Deref",                   "Profiling of SyncPageWorkerTrackDeref (expensive).");
+
+# undef PGM_REG_COUNTER
+# undef PGM_REG_PROFILE
+#endif
+
+    /*
+     * Note! The layout below matches the member layout exactly!
+     */
+
+    /*
+     * Common - stats
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PPGMCPU pPgmCpu = &pVM->apCpusR3[idCpu]->pgm.s;
+
+#define PGM_REG_COUNTER(a, b, c) \
+    rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b, idCpu); \
+    AssertRC(rc);
+#define PGM_REG_PROFILE(a, b, c) \
+    rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, idCpu); \
+    AssertRC(rc);
+
+        PGM_REG_COUNTER(&pPgmCpu->cGuestModeChanges, "/PGM/CPU%u/cGuestModeChanges",  "Number of guest mode changes.");
+        PGM_REG_COUNTER(&pPgmCpu->cA20Changes, "/PGM/CPU%u/cA20Changes",  "Number of A20 gate changes.");
+
+#ifdef VBOX_WITH_STATISTICS
+        PGMCPUSTATS *pCpuStats = pVM->apCpusR3[idCpu]->pgm.s.pStatsR3;
+
+# if 0 /* rarely useful; leave for debugging. */
+        for (unsigned j = 0; j < RT_ELEMENTS(pPgmCpu->StatSyncPtPD); j++)
+            STAMR3RegisterF(pVM, &pCpuStats->StatSyncPtPD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                            "The number of SyncPT per PD n.", "/PGM/CPU%u/PDSyncPT/%04X", i, j);
+        for (unsigned j = 0; j < RT_ELEMENTS(pCpuStats->StatSyncPagePD); j++)
+            STAMR3RegisterF(pVM, &pCpuStats->StatSyncPagePD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                            "The number of SyncPage per PD n.", "/PGM/CPU%u/PDSyncPage/%04X", i, j);
+# endif
+        /* R0 only: */
+        PGM_REG_PROFILE(&pCpuStats->StatR0NpMiscfg,                      "/PGM/CPU%u/R0/NpMiscfg",                     "PGMR0Trap0eHandlerNPMisconfig() profiling.");
+        PGM_REG_COUNTER(&pCpuStats->StatR0NpMiscfgSyncPage,              "/PGM/CPU%u/R0/NpMiscfgSyncPage",             "SyncPage calls from PGMR0Trap0eHandlerNPMisconfig().");
+
+        /* RZ only: */
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0e,                      "/PGM/CPU%u/RZ/Trap0e",                     "Profiling of the PGMTrap0eHandler() body.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2Ballooned,        "/PGM/CPU%u/RZ/Trap0e/Time2/Ballooned",         "Profiling of the Trap0eHandler body when the cause is read access to a ballooned page.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2CSAM,             "/PGM/CPU%u/RZ/Trap0e/Time2/CSAM",              "Profiling of the Trap0eHandler body when the cause is CSAM.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2DirtyAndAccessed, "/PGM/CPU%u/RZ/Trap0e/Time2/DirtyAndAccessedBits", "Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2GuestTrap,        "/PGM/CPU%u/RZ/Trap0e/Time2/GuestTrap",         "Profiling of the Trap0eHandler body when the cause is a guest trap.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2HndPhys,          "/PGM/CPU%u/RZ/Trap0e/Time2/HandlerPhysical",   "Profiling of the Trap0eHandler body when the cause is a physical handler.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2HndUnhandled,     "/PGM/CPU%u/RZ/Trap0e/Time2/HandlerUnhandled",  "Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2InvalidPhys,      "/PGM/CPU%u/RZ/Trap0e/Time2/InvalidPhys",       "Profiling of the Trap0eHandler body when the cause is access to an invalid physical guest address.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2MakeWritable,     "/PGM/CPU%u/RZ/Trap0e/Time2/MakeWritable",      "Profiling of the Trap0eHandler body when the cause is that a page needed to be made writeable.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2Mapping,          "/PGM/CPU%u/RZ/Trap0e/Time2/Mapping",           "Profiling of the Trap0eHandler body when the cause is related to the guest mappings.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2Misc,             "/PGM/CPU%u/RZ/Trap0e/Time2/Misc",              "Profiling of the Trap0eHandler body when the cause is not known.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2OutOfSync,        "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSync",         "Profiling of the Trap0eHandler body when the cause is an out-of-sync page.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2OutOfSyncHndPhys, "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSyncHndPhys",  "Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2OutOfSyncHndObs,  "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSyncObsHnd",   "Profiling of the Trap0eHandler body when the cause is an obsolete handler page.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2SyncPT,           "/PGM/CPU%u/RZ/Trap0e/Time2/SyncPT",            "Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2WPEmulation,      "/PGM/CPU%u/RZ/Trap0e/Time2/WPEmulation",       "Profiling of the Trap0eHandler body when the cause is CR0.WP emulation.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2Wp0RoUsHack,      "/PGM/CPU%u/RZ/Trap0e/Time2/WP0R0USHack",       "Profiling of the Trap0eHandler body when the cause is CR0.WP and netware hack to be enabled.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2Wp0RoUsUnhack,    "/PGM/CPU%u/RZ/Trap0e/Time2/WP0R0USUnhack",     "Profiling of the Trap0eHandler body when the cause is CR0.WP and netware hack to be disabled.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eConflicts,             "/PGM/CPU%u/RZ/Trap0e/Conflicts",               "The number of times #PF was caused by an undetected conflict.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersMapping,       "/PGM/CPU%u/RZ/Trap0e/Handlers/Mapping",        "Number of traps due to access handlers in mappings.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersOutOfSync,     "/PGM/CPU%u/RZ/Trap0e/Handlers/OutOfSync",      "Number of traps due to out-of-sync handled pages.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersPhysAll,       "/PGM/CPU%u/RZ/Trap0e/Handlers/PhysAll",        "Number of traps due to physical all-access handlers.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersPhysAllOpt,    "/PGM/CPU%u/RZ/Trap0e/Handlers/PhysAllOpt",     "Number of the physical all-access handler traps using the optimization.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersPhysWrite,     "/PGM/CPU%u/RZ/Trap0e/Handlers/PhysWrite",      "Number of traps due to physical write-access handlers.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersUnhandled,     "/PGM/CPU%u/RZ/Trap0e/Handlers/Unhandled",      "Number of traps due to access outside range of monitored page(s).");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersInvalid,       "/PGM/CPU%u/RZ/Trap0e/Handlers/Invalid",        "Number of traps due to access to invalid physical memory.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSNotPresentRead,      "/PGM/CPU%u/RZ/Trap0e/Err/User/NPRead",         "Number of user mode not present read page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSNotPresentWrite,     "/PGM/CPU%u/RZ/Trap0e/Err/User/NPWrite",        "Number of user mode not present write page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSWrite,               "/PGM/CPU%u/RZ/Trap0e/Err/User/Write",          "Number of user mode write page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSReserved,            "/PGM/CPU%u/RZ/Trap0e/Err/User/Reserved",       "Number of user mode reserved bit page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSNXE,                 "/PGM/CPU%u/RZ/Trap0e/Err/User/NXE",            "Number of user mode NXE page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSRead,                "/PGM/CPU%u/RZ/Trap0e/Err/User/Read",           "Number of user mode read page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSVNotPresentRead,      "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/NPRead",   "Number of supervisor mode not present read page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSVNotPresentWrite,     "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/NPWrite",  "Number of supervisor mode not present write page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSVWrite,               "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/Write",    "Number of supervisor mode write page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSVReserved,            "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/Reserved", "Number of supervisor mode reserved bit page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSNXE,                  "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/NXE",      "Number of supervisor mode NXE page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eGuestPF,               "/PGM/CPU%u/RZ/Trap0e/GuestPF",                 "Number of real guest page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eGuestPFMapping,        "/PGM/CPU%u/RZ/Trap0e/GuestPF/InMapping",       "Number of real guest page faults in a mapping.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eWPEmulInRZ,            "/PGM/CPU%u/RZ/Trap0e/WP/InRZ",                 "Number of guest page faults due to X86_CR0_WP emulation.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eWPEmulToR3,            "/PGM/CPU%u/RZ/Trap0e/WP/ToR3",                 "Number of guest page faults due to X86_CR0_WP emulation (forward to R3 for emulation).");
+#if 0 /* rarely useful; leave for debugging. */
+        for (unsigned j = 0; j < RT_ELEMENTS(pCpuStats->StatRZTrap0ePD); j++)
+            STAMR3RegisterF(pVM, &pCpuStats->StatRZTrap0ePD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                            "The number of traps in page directory n.", "/PGM/CPU%u/RZ/Trap0e/PD/%04X", i, j);
+#endif
+        PGM_REG_COUNTER(&pCpuStats->StatRZGuestCR3WriteHandled,        "/PGM/CPU%u/RZ/CR3WriteHandled",                "The number of times the Guest CR3 change was successfully handled.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZGuestCR3WriteUnhandled,      "/PGM/CPU%u/RZ/CR3WriteUnhandled",              "The number of times the Guest CR3 change was passed back to the recompiler.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZGuestCR3WriteConflict,       "/PGM/CPU%u/RZ/CR3WriteConflict",               "The number of times the Guest CR3 monitoring detected a conflict.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZGuestROMWriteHandled,        "/PGM/CPU%u/RZ/ROMWriteHandled",                "The number of times the Guest ROM change was successfully handled.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZGuestROMWriteUnhandled,      "/PGM/CPU%u/RZ/ROMWriteUnhandled",              "The number of times the Guest ROM change was passed back to the recompiler.");
+
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapMigrateInvlPg,         "/PGM/CPU%u/RZ/DynMap/MigrateInvlPg",            "invlpg count in PGMR0DynMapMigrateAutoSet.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZDynMapGCPageInl,             "/PGM/CPU%u/RZ/DynMap/PageGCPageInl",            "Calls to pgmR0DynMapGCPageInlined.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapGCPageInlHits,         "/PGM/CPU%u/RZ/DynMap/PageGCPageInl/Hits",       "Hash table lookup hits.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapGCPageInlMisses,       "/PGM/CPU%u/RZ/DynMap/PageGCPageInl/Misses",     "Misses that falls back to the code common.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapGCPageInlRamHits,      "/PGM/CPU%u/RZ/DynMap/PageGCPageInl/RamHits",    "1st ram range hits.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapGCPageInlRamMisses,    "/PGM/CPU%u/RZ/DynMap/PageGCPageInl/RamMisses",  "1st ram range misses, takes slow path.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZDynMapHCPageInl,             "/PGM/CPU%u/RZ/DynMap/PageHCPageInl",            "Calls to pgmRZDynMapHCPageInlined.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapHCPageInlHits,         "/PGM/CPU%u/RZ/DynMap/PageHCPageInl/Hits",       "Hash table lookup hits.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapHCPageInlMisses,       "/PGM/CPU%u/RZ/DynMap/PageHCPageInl/Misses",     "Misses that falls back to the code common.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPage,                  "/PGM/CPU%u/RZ/DynMap/Page",                     "Calls to pgmR0DynMapPage");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSetOptimize,           "/PGM/CPU%u/RZ/DynMap/Page/SetOptimize",         "Calls to pgmRZDynMapOptimizeAutoSet.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSetSearchFlushes,      "/PGM/CPU%u/RZ/DynMap/Page/SetSearchFlushes",    "Set search restoring to subset flushes.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSetSearchHits,         "/PGM/CPU%u/RZ/DynMap/Page/SetSearchHits",       "Set search hits.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSetSearchMisses,       "/PGM/CPU%u/RZ/DynMap/Page/SetSearchMisses",     "Set search misses.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZDynMapHCPage,                "/PGM/CPU%u/RZ/DynMap/Page/HCPage",              "Calls to pgmRZDynMapHCPageCommon (ring-0).");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageHits0,             "/PGM/CPU%u/RZ/DynMap/Page/Hits0",               "Hits at iPage+0");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageHits1,             "/PGM/CPU%u/RZ/DynMap/Page/Hits1",               "Hits at iPage+1");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageHits2,             "/PGM/CPU%u/RZ/DynMap/Page/Hits2",               "Hits at iPage+2");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageInvlPg,            "/PGM/CPU%u/RZ/DynMap/Page/InvlPg",              "invlpg count in pgmR0DynMapPageSlow.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageSlow,              "/PGM/CPU%u/RZ/DynMap/Page/Slow",                "Calls to pgmR0DynMapPageSlow - subtract this from pgmR0DynMapPage to get 1st level hits.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageSlowLoopHits,      "/PGM/CPU%u/RZ/DynMap/Page/SlowLoopHits" ,       "Hits in the loop path.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageSlowLoopMisses,    "/PGM/CPU%u/RZ/DynMap/Page/SlowLoopMisses",      "Misses in the loop path. NonLoopMisses = Slow - SlowLoopHit - SlowLoopMisses");
+        //PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageSlowLostHits,      "/PGM/CPU%u/R0/DynMap/Page/SlowLostHits",        "Lost hits.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSubsets,               "/PGM/CPU%u/RZ/DynMap/Subsets",                  "Times PGMRZDynMapPushAutoSubset was called.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPopFlushes,            "/PGM/CPU%u/RZ/DynMap/SubsetPopFlushes",         "Times PGMRZDynMapPopAutoSubset flushes the subset.");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[0],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct000..09",      "00-09% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[1],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct010..19",      "10-19% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[2],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct020..29",      "20-29% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[3],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct030..39",      "30-39% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[4],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct040..49",      "40-49% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[5],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct050..59",      "50-59% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[6],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct060..69",      "60-69% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[7],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct070..79",      "70-79% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[8],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct080..89",      "80-89% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[9],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct090..99",      "90-99% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[10],     "/PGM/CPU%u/RZ/DynMap/SetFilledPct100",          "100% filled (RC: min(set-size, dynmap-size))");
+
+        /* HC only: */
+
+        /* RZ & R3: */
+        PGM_REG_PROFILE(&pCpuStats->StatRZSyncCR3,                     "/PGM/CPU%u/RZ/SyncCR3",                        "Profiling of the PGMSyncCR3() body.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZSyncCR3Handlers,             "/PGM/CPU%u/RZ/SyncCR3/Handlers",               "Profiling of the PGMSyncCR3() update handler section.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3Global,               "/PGM/CPU%u/RZ/SyncCR3/Global",                 "The number of global CR3 syncs.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3NotGlobal,            "/PGM/CPU%u/RZ/SyncCR3/NotGlobal",              "The number of non-global CR3 syncs.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstCacheHit,          "/PGM/CPU%u/RZ/SyncCR3/DstChacheHit",           "The number of times we got some kind of a cache hit.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstFreed,             "/PGM/CPU%u/RZ/SyncCR3/DstFreed",               "The number of times we've had to free a shadow entry.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstFreedSrcNP,        "/PGM/CPU%u/RZ/SyncCR3/DstFreedSrcNP",          "The number of times we've had to free a shadow entry for which the source entry was not present.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstNotPresent,        "/PGM/CPU%u/RZ/SyncCR3/DstNotPresent",          "The number of times we've encountered a not present shadow entry for a present guest entry.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstSkippedGlobalPD,   "/PGM/CPU%u/RZ/SyncCR3/DstSkippedGlobalPD",     "The number of times a global page directory wasn't flushed.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstSkippedGlobalPT,   "/PGM/CPU%u/RZ/SyncCR3/DstSkippedGlobalPT",     "The number of times a page table with only global entries wasn't flushed.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZSyncPT,                      "/PGM/CPU%u/RZ/SyncPT",                         "Profiling of the pfnSyncPT() body.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPTFailed,                "/PGM/CPU%u/RZ/SyncPT/Failed",                  "The number of times pfnSyncPT() failed.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPT4K,                    "/PGM/CPU%u/RZ/SyncPT/4K",                      "Nr of 4K PT syncs");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPT4M,                    "/PGM/CPU%u/RZ/SyncPT/4M",                      "Nr of 4M PT syncs");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPagePDNAs,               "/PGM/CPU%u/RZ/SyncPagePDNAs",                  "The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPagePDOutOfSync,         "/PGM/CPU%u/RZ/SyncPagePDOutOfSync",            "The number of time we've encountered an out-of-sync PD in SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZAccessedPage,                "/PGM/CPU%u/RZ/AccessedPage",               "The number of pages marked not present for accessed bit emulation.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZDirtyBitTracking,            "/PGM/CPU%u/RZ/DirtyPage",                  "Profiling the dirty bit tracking in CheckPageFault().");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPage,                   "/PGM/CPU%u/RZ/DirtyPage/Mark",             "The number of pages marked read-only for dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPageBig,                "/PGM/CPU%u/RZ/DirtyPage/MarkBig",          "The number of 4MB pages marked read-only for dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPageSkipped,            "/PGM/CPU%u/RZ/DirtyPage/Skipped",          "The number of pages already dirty or readonly.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPageTrap,               "/PGM/CPU%u/RZ/DirtyPage/Trap",             "The number of traps generated for dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPageStale,              "/PGM/CPU%u/RZ/DirtyPage/Stale",            "The number of traps generated for dirty bit tracking (stale tlb entries).");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtiedPage,                 "/PGM/CPU%u/RZ/DirtyPage/SetDirty",         "The number of pages marked dirty because of write accesses.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyTrackRealPF,            "/PGM/CPU%u/RZ/DirtyPage/RealPF",           "The number of real pages faults during dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageAlreadyDirty,            "/PGM/CPU%u/RZ/DirtyPage/AlreadySet",       "The number of pages already marked dirty because of write accesses.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZInvalidatePage,              "/PGM/CPU%u/RZ/InvalidatePage",             "PGMInvalidatePage() profiling.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePage4KBPages,      "/PGM/CPU%u/RZ/InvalidatePage/4KBPages",    "The number of times PGMInvalidatePage() was called for a 4KB page.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePage4MBPages,      "/PGM/CPU%u/RZ/InvalidatePage/4MBPages",    "The number of times PGMInvalidatePage() was called for a 4MB page.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePage4MBPagesSkip,  "/PGM/CPU%u/RZ/InvalidatePage/4MBPagesSkip","The number of times PGMInvalidatePage() skipped a 4MB page.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDMappings,    "/PGM/CPU%u/RZ/InvalidatePage/PDMappings",  "The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict).");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDNAs,         "/PGM/CPU%u/RZ/InvalidatePage/PDNAs",       "The number of times PGMInvalidatePage() was called for a not accessed page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDNPs,         "/PGM/CPU%u/RZ/InvalidatePage/PDNPs",       "The number of times PGMInvalidatePage() was called for a not present page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDOutOfSync,   "/PGM/CPU%u/RZ/InvalidatePage/PDOutOfSync", "The number of times PGMInvalidatePage() was called for an out of sync page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePageSizeChanges,   "/PGM/CPU%u/RZ/InvalidatePage/SizeChanges", "The number of times PGMInvalidatePage() was called on a page size change (4KB <-> 2/4MB).");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePageSkipped,       "/PGM/CPU%u/RZ/InvalidatePage/Skipped",     "The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncSupervisor,     "/PGM/CPU%u/RZ/OutOfSync/SuperVisor",       "Number of traps due to pages out of sync (P) and times VerifyAccessSyncPage calls SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncUser,           "/PGM/CPU%u/RZ/OutOfSync/User",             "Number of traps due to pages out of sync (P) and times VerifyAccessSyncPage calls SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncSupervisorWrite,"/PGM/CPU%u/RZ/OutOfSync/SuperVisorWrite",  "Number of traps due to pages out of sync (RW) and times VerifyAccessSyncPage calls SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncUserWrite,      "/PGM/CPU%u/RZ/OutOfSync/UserWrite",        "Number of traps due to pages out of sync (RW) and times VerifyAccessSyncPage calls SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncBallloon,       "/PGM/CPU%u/RZ/OutOfSync/Balloon",          "The number of times a ballooned page was accessed (read).");
+        PGM_REG_PROFILE(&pCpuStats->StatRZPrefetch,                    "/PGM/CPU%u/RZ/Prefetch",                   "PGMPrefetchPage profiling.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZFlushTLB,                    "/PGM/CPU%u/RZ/FlushTLB",                   "Profiling of the PGMFlushTLB() body.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBNewCR3,              "/PGM/CPU%u/RZ/FlushTLB/NewCR3",            "The number of times PGMFlushTLB was called with a new CR3, non-global. (switch)");
+        PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBNewCR3Global,        "/PGM/CPU%u/RZ/FlushTLB/NewCR3Global",      "The number of times PGMFlushTLB was called with a new CR3, global. (switch)");
+        PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBSameCR3,             "/PGM/CPU%u/RZ/FlushTLB/SameCR3",           "The number of times PGMFlushTLB was called with the same CR3, non-global. (flush)");
+        PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBSameCR3Global,       "/PGM/CPU%u/RZ/FlushTLB/SameCR3Global",     "The number of times PGMFlushTLB was called with the same CR3, global. (flush)");
+        PGM_REG_PROFILE(&pCpuStats->StatRZGstModifyPage,               "/PGM/CPU%u/RZ/GstModifyPage",              "Profiling of the PGMGstModifyPage() body.");
+
+        PGM_REG_PROFILE(&pCpuStats->StatR3SyncCR3,                     "/PGM/CPU%u/R3/SyncCR3",                        "Profiling of the PGMSyncCR3() body.");
+        PGM_REG_PROFILE(&pCpuStats->StatR3SyncCR3Handlers,             "/PGM/CPU%u/R3/SyncCR3/Handlers",               "Profiling of the PGMSyncCR3() update handler section.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3Global,               "/PGM/CPU%u/R3/SyncCR3/Global",                 "The number of global CR3 syncs.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3NotGlobal,            "/PGM/CPU%u/R3/SyncCR3/NotGlobal",              "The number of non-global CR3 syncs.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstCacheHit,          "/PGM/CPU%u/R3/SyncCR3/DstChacheHit",           "The number of times we got some kind of a cache hit.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstFreed,             "/PGM/CPU%u/R3/SyncCR3/DstFreed",               "The number of times we've had to free a shadow entry.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstFreedSrcNP,        "/PGM/CPU%u/R3/SyncCR3/DstFreedSrcNP",          "The number of times we've had to free a shadow entry for which the source entry was not present.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstNotPresent,        "/PGM/CPU%u/R3/SyncCR3/DstNotPresent",          "The number of times we've encountered a not present shadow entry for a present guest entry.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstSkippedGlobalPD,   "/PGM/CPU%u/R3/SyncCR3/DstSkippedGlobalPD",     "The number of times a global page directory wasn't flushed.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstSkippedGlobalPT,   "/PGM/CPU%u/R3/SyncCR3/DstSkippedGlobalPT",     "The number of times a page table with only global entries wasn't flushed.");
+        PGM_REG_PROFILE(&pCpuStats->StatR3SyncPT,                      "/PGM/CPU%u/R3/SyncPT",                         "Profiling of the pfnSyncPT() body.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPTFailed,                "/PGM/CPU%u/R3/SyncPT/Failed",                  "The number of times pfnSyncPT() failed.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPT4K,                    "/PGM/CPU%u/R3/SyncPT/4K",                      "Nr of 4K PT syncs");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPT4M,                    "/PGM/CPU%u/R3/SyncPT/4M",                      "Nr of 4M PT syncs");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPagePDNAs,               "/PGM/CPU%u/R3/SyncPagePDNAs",                  "The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPagePDOutOfSync,         "/PGM/CPU%u/R3/SyncPagePDOutOfSync",            "The number of time we've encountered an out-of-sync PD in SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3AccessedPage,                "/PGM/CPU%u/R3/AccessedPage",               "The number of pages marked not present for accessed bit emulation.");
+        PGM_REG_PROFILE(&pCpuStats->StatR3DirtyBitTracking,            "/PGM/CPU%u/R3/DirtyPage",                  "Profiling the dirty bit tracking in CheckPageFault().");
+        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyPage,                   "/PGM/CPU%u/R3/DirtyPage/Mark",             "The number of pages marked read-only for dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyPageBig,                "/PGM/CPU%u/R3/DirtyPage/MarkBig",          "The number of 4MB pages marked read-only for dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyPageSkipped,            "/PGM/CPU%u/R3/DirtyPage/Skipped",          "The number of pages already dirty or readonly.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyPageTrap,               "/PGM/CPU%u/R3/DirtyPage/Trap",             "The number of traps generated for dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3DirtiedPage,                 "/PGM/CPU%u/R3/DirtyPage/SetDirty",         "The number of pages marked dirty because of write accesses.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyTrackRealPF,            "/PGM/CPU%u/R3/DirtyPage/RealPF",           "The number of real pages faults during dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3PageAlreadyDirty,            "/PGM/CPU%u/R3/DirtyPage/AlreadySet",       "The number of pages already marked dirty because of write accesses.");
+        PGM_REG_PROFILE(&pCpuStats->StatR3InvalidatePage,              "/PGM/CPU%u/R3/InvalidatePage",             "PGMInvalidatePage() profiling.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePage4KBPages,      "/PGM/CPU%u/R3/InvalidatePage/4KBPages",    "The number of times PGMInvalidatePage() was called for a 4KB page.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePage4MBPages,      "/PGM/CPU%u/R3/InvalidatePage/4MBPages",    "The number of times PGMInvalidatePage() was called for a 4MB page.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePage4MBPagesSkip,  "/PGM/CPU%u/R3/InvalidatePage/4MBPagesSkip","The number of times PGMInvalidatePage() skipped a 4MB page.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDMappings,    "/PGM/CPU%u/R3/InvalidatePage/PDMappings",  "The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict).");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDNAs,         "/PGM/CPU%u/R3/InvalidatePage/PDNAs",       "The number of times PGMInvalidatePage() was called for a not accessed page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDNPs,         "/PGM/CPU%u/R3/InvalidatePage/PDNPs",       "The number of times PGMInvalidatePage() was called for a not present page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDOutOfSync,   "/PGM/CPU%u/R3/InvalidatePage/PDOutOfSync", "The number of times PGMInvalidatePage() was called for an out of sync page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePageSizeChanges,   "/PGM/CPU%u/R3/InvalidatePage/SizeChanges", "The number of times PGMInvalidatePage() was called on a page size change (4KB <-> 2/4MB).");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePageSkipped,       "/PGM/CPU%u/R3/InvalidatePage/Skipped",     "The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3PageOutOfSyncSupervisor,     "/PGM/CPU%u/R3/OutOfSync/SuperVisor",       "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3PageOutOfSyncUser,           "/PGM/CPU%u/R3/OutOfSync/User",             "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3PageOutOfSyncBallloon,       "/PGM/CPU%u/R3/OutOfSync/Balloon",          "The number of times a ballooned page was accessed (read).");
+        PGM_REG_PROFILE(&pCpuStats->StatR3Prefetch,                    "/PGM/CPU%u/R3/Prefetch",                   "PGMPrefetchPage profiling.");
+        PGM_REG_PROFILE(&pCpuStats->StatR3FlushTLB,                    "/PGM/CPU%u/R3/FlushTLB",                   "Profiling of the PGMFlushTLB() body.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBNewCR3,              "/PGM/CPU%u/R3/FlushTLB/NewCR3",            "The number of times PGMFlushTLB was called with a new CR3, non-global. (switch)");
+        PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBNewCR3Global,        "/PGM/CPU%u/R3/FlushTLB/NewCR3Global",      "The number of times PGMFlushTLB was called with a new CR3, global. (switch)");
+        PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBSameCR3,             "/PGM/CPU%u/R3/FlushTLB/SameCR3",           "The number of times PGMFlushTLB was called with the same CR3, non-global. (flush)");
+        PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBSameCR3Global,       "/PGM/CPU%u/R3/FlushTLB/SameCR3Global",     "The number of times PGMFlushTLB was called with the same CR3, global. (flush)");
+        PGM_REG_PROFILE(&pCpuStats->StatR3GstModifyPage,               "/PGM/CPU%u/R3/GstModifyPage",              "Profiling of the PGMGstModifyPage() body.");
+#endif /* VBOX_WITH_STATISTICS */
+
+#undef PGM_REG_PROFILE
+#undef PGM_REG_COUNTER
+
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Init the PGM bits that rely on VMMR0 and MM to be fully initialized.
+ *
+ * The dynamic mapping area will also be allocated and initialized at this
+ * time. We could allocate it during PGMR3Init of course, but the mapping
+ * wouldn't be allocated at that time preventing us from setting up the
+ * page table entries with the dummy page.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(int) PGMR3InitDynMap(PVM pVM)
+{
+#ifndef PGM_WITHOUT_MAPPINGS
+    RTGCPTR GCPtr;
+    int     rc;
+
+    /*
+     * Reserve space for the dynamic mappings.
+     */
+    rc = MMR3HyperReserve(pVM, MM_HYPER_DYNAMIC_SIZE, "Dynamic mapping", &GCPtr);
+    if (RT_SUCCESS(rc))
+        pVM->pgm.s.pbDynPageMapBaseGC = GCPtr;
+
+    if (    RT_SUCCESS(rc)
+        &&  (pVM->pgm.s.pbDynPageMapBaseGC >> X86_PD_PAE_SHIFT) != ((pVM->pgm.s.pbDynPageMapBaseGC + MM_HYPER_DYNAMIC_SIZE - 1) >> X86_PD_PAE_SHIFT))
+    {
+        rc = MMR3HyperReserve(pVM, MM_HYPER_DYNAMIC_SIZE, "Dynamic mapping not crossing", &GCPtr);
+        if (RT_SUCCESS(rc))
+            pVM->pgm.s.pbDynPageMapBaseGC = GCPtr;
+    }
+    if (RT_SUCCESS(rc))
+    {
+        AssertRelease((pVM->pgm.s.pbDynPageMapBaseGC >> X86_PD_PAE_SHIFT) == ((pVM->pgm.s.pbDynPageMapBaseGC + MM_HYPER_DYNAMIC_SIZE - 1) >> X86_PD_PAE_SHIFT));
+        MMR3HyperReserveFence(pVM);
+    }
+    return rc;
+#else
+    RT_NOREF(pVM);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/**
+ * Ring-3 init finalizing.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(int) PGMR3InitFinalize(PVM pVM)
+{
+#ifndef PGM_WITHOUT_MAPPINGS
+    int rc = VERR_IPE_UNINITIALIZED_STATUS; /* (MSC incorrectly thinks it can be used uninitialized) */
+
+    /*
+     * Reserve space for the dynamic mappings.
+     * Initialize the dynamic mapping pages with dummy pages to simply the cache.
+     */
+    /* get the pointer to the page table entries. */
+    PPGMMAPPING pMapping = pgmGetMapping(pVM, pVM->pgm.s.pbDynPageMapBaseGC);
+    AssertRelease(pMapping);
+    const uintptr_t off = pVM->pgm.s.pbDynPageMapBaseGC - pMapping->GCPtr;
+    const unsigned iPT =  off >> X86_PD_SHIFT;
+    const unsigned iPG = (off >> X86_PT_SHIFT) & X86_PT_MASK;
+    pVM->pgm.s.paDynPageMap32BitPTEsGC = pMapping->aPTs[iPT].pPTRC      + iPG * sizeof(pMapping->aPTs[0].pPTR3->a[0]);
+    pVM->pgm.s.paDynPageMapPaePTEsGC   = pMapping->aPTs[iPT].paPaePTsRC + iPG * sizeof(pMapping->aPTs[0].paPaePTsR3->a[0]);
+
+    /* init cache area */
+    RTHCPHYS HCPhysDummy = MMR3PageDummyHCPhys(pVM);
+    for (uint32_t offDynMap = 0; offDynMap < MM_HYPER_DYNAMIC_SIZE; offDynMap += PAGE_SIZE)
+    {
+        rc = PGMMap(pVM, pVM->pgm.s.pbDynPageMapBaseGC + offDynMap, HCPhysDummy, PAGE_SIZE, 0);
+        AssertRCReturn(rc, rc);
+    }
+#endif
+
+    /*
+     * Determine the max physical address width (MAXPHYADDR) and apply it to
+     * all the mask members and stuff.
+     */
+    uint32_t cMaxPhysAddrWidth;
+    uint32_t uMaxExtLeaf = ASMCpuId_EAX(0x80000000);
+    if (   uMaxExtLeaf >= 0x80000008
+        && uMaxExtLeaf <= 0x80000fff)
+    {
+        cMaxPhysAddrWidth = ASMCpuId_EAX(0x80000008) & 0xff;
+        LogRel(("PGM: The CPU physical address width is %u bits\n", cMaxPhysAddrWidth));
+        cMaxPhysAddrWidth = RT_MIN(52, cMaxPhysAddrWidth);
+        pVM->pgm.s.fLessThan52PhysicalAddressBits = cMaxPhysAddrWidth < 52;
+        for (uint32_t iBit = cMaxPhysAddrWidth; iBit < 52; iBit++)
+            pVM->pgm.s.HCPhysInvMmioPg |= RT_BIT_64(iBit);
+    }
+    else
+    {
+        LogRel(("PGM: ASSUMING CPU physical address width of 48 bits (uMaxExtLeaf=%#x)\n", uMaxExtLeaf));
+        cMaxPhysAddrWidth = 48;
+        pVM->pgm.s.fLessThan52PhysicalAddressBits = true;
+        pVM->pgm.s.HCPhysInvMmioPg |= UINT64_C(0x000f0000000000);
+    }
+
+    /** @todo query from CPUM. */
+    pVM->pgm.s.GCPhysInvAddrMask = 0;
+    for (uint32_t iBit = cMaxPhysAddrWidth; iBit < 64; iBit++)
+        pVM->pgm.s.GCPhysInvAddrMask |= RT_BIT_64(iBit);
+
+    /*
+     * Initialize the invalid paging entry masks, assuming NX is disabled.
+     */
+    uint64_t fMbzPageFrameMask = pVM->pgm.s.GCPhysInvAddrMask & UINT64_C(0x000ffffffffff000);
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+
+        /** @todo The manuals are not entirely clear whether the physical
+         *        address width is relevant.  See table 5-9 in the intel
+         *        manual vs the PDE4M descriptions.  Write testcase (NP). */
+        pVCpu->pgm.s.fGst32BitMbzBigPdeMask   = ((uint32_t)(fMbzPageFrameMask >> (32 - 13)) & X86_PDE4M_PG_HIGH_MASK)
+                                              | X86_PDE4M_MBZ_MASK;
+
+        pVCpu->pgm.s.fGstPaeMbzPteMask        = fMbzPageFrameMask | X86_PTE_PAE_MBZ_MASK_NO_NX;
+        pVCpu->pgm.s.fGstPaeMbzPdeMask        = fMbzPageFrameMask | X86_PDE_PAE_MBZ_MASK_NO_NX;
+        pVCpu->pgm.s.fGstPaeMbzBigPdeMask     = fMbzPageFrameMask | X86_PDE2M_PAE_MBZ_MASK_NO_NX;
+        pVCpu->pgm.s.fGstPaeMbzPdpeMask       = fMbzPageFrameMask | X86_PDPE_PAE_MBZ_MASK;
+
+        pVCpu->pgm.s.fGstAmd64MbzPteMask      = fMbzPageFrameMask | X86_PTE_LM_MBZ_MASK_NO_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPdeMask      = fMbzPageFrameMask | X86_PDE_LM_MBZ_MASK_NX;
+        pVCpu->pgm.s.fGstAmd64MbzBigPdeMask   = fMbzPageFrameMask | X86_PDE2M_LM_MBZ_MASK_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPdpeMask     = fMbzPageFrameMask | X86_PDPE_LM_MBZ_MASK_NO_NX;
+        pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask  = fMbzPageFrameMask | X86_PDPE1G_LM_MBZ_MASK_NO_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPml4eMask    = fMbzPageFrameMask | X86_PML4E_MBZ_MASK_NO_NX;
+
+        pVCpu->pgm.s.fGst64ShadowedPteMask    = X86_PTE_P   | X86_PTE_RW   | X86_PTE_US   | X86_PTE_G | X86_PTE_A | X86_PTE_D;
+        pVCpu->pgm.s.fGst64ShadowedPdeMask    = X86_PDE_P   | X86_PDE_RW   | X86_PDE_US   | X86_PDE_A;
+        pVCpu->pgm.s.fGst64ShadowedBigPdeMask = X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A;
+        pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask =
+            X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_G | X86_PDE4M_A | X86_PDE4M_D;
+        pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask  = X86_PDPE_P  | X86_PDPE_RW  | X86_PDPE_US  | X86_PDPE_A;
+        pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
+    }
+
+    /*
+     * Note that AMD uses all the 8 reserved bits for the address (so 40 bits in total);
+     * Intel only goes up to 36 bits, so we stick to 36 as well.
+     * Update: More recent intel manuals specifies 40 bits just like AMD.
+     */
+    uint32_t u32Dummy, u32Features;
+    CPUMGetGuestCpuId(VMMGetCpu(pVM), 1, 0, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
+    if (u32Features & X86_CPUID_FEATURE_EDX_PSE36)
+        pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(RT_MAX(36, cMaxPhysAddrWidth)) - 1;
+    else
+        pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(32) - 1;
+
+    /*
+     * Allocate memory if we're supposed to do that.
+     */
+#ifdef PGM_WITHOUT_MAPPINGS
+    int rc = VINF_SUCCESS;
+#endif
+    if (pVM->pgm.s.fRamPreAlloc)
+        rc = pgmR3PhysRamPreAllocate(pVM);
+
+    //pgmLogState(pVM);
+    LogRel(("PGM: PGMR3InitFinalize: 4 MB PSE mask %RGp -> %Rrc\n", pVM->pgm.s.GCPhys4MBPSEMask, rc));
+    return rc;
+}
+
+
+/**
+ * Init phase completed callback.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   enmWhat             What has been completed.
+ * @thread  EMT(0)
+ */
+VMMR3_INT_DECL(int) PGMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    switch (enmWhat)
+    {
+        case VMINITCOMPLETED_HM:
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+            if (pVM->pgm.s.fPciPassthrough)
+            {
+                AssertLogRelReturn(pVM->pgm.s.fRamPreAlloc, VERR_PCI_PASSTHROUGH_NO_RAM_PREALLOC);
+                AssertLogRelReturn(HMIsEnabled(pVM), VERR_PCI_PASSTHROUGH_NO_HM);
+                AssertLogRelReturn(HMIsNestedPagingActive(pVM), VERR_PCI_PASSTHROUGH_NO_NESTED_PAGING);
+
+                /*
+                 * Report assignments to the IOMMU (hope that's good enough for now).
+                 */
+                if (pVM->pgm.s.fPciPassthrough)
+                {
+                    int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_PHYS_SETUP_IOMMU, 0, NULL);
+                    AssertRCReturn(rc, rc);
+                }
+            }
+#else
+            AssertLogRelReturn(!pVM->pgm.s.fPciPassthrough, VERR_PGM_PCI_PASSTHRU_MISCONFIG);
+#endif
+            break;
+
+        default:
+            /* shut up gcc */
+            break;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this component.
+ *
+ * This function will be called at init and whenever the VMM need to relocate it
+ * self inside the GC.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3DECL(void) PGMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    LogFlow(("PGMR3Relocate: offDelta=%RGv\n", offDelta));
+
+    /*
+     * Paging stuff.
+     */
+
+    /* Shadow, guest and both mode switch & relocation for each VCPU. */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU  pVCpu = pVM->apCpusR3[i];
+
+        uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
+        if (   idxShw < RT_ELEMENTS(g_aPgmShadowModeData)
+            && g_aPgmShadowModeData[idxShw].pfnRelocate)
+            g_aPgmShadowModeData[idxShw].pfnRelocate(pVCpu, offDelta);
+        else
+            AssertFailed();
+
+        uintptr_t const idxGst = pVCpu->pgm.s.idxGuestModeData;
+        if (   idxGst < RT_ELEMENTS(g_aPgmGuestModeData)
+            && g_aPgmGuestModeData[idxGst].pfnRelocate)
+            g_aPgmGuestModeData[idxGst].pfnRelocate(pVCpu, offDelta);
+        else
+            AssertFailed();
+    }
+
+    /*
+     * Ram ranges.
+     */
+    if (pVM->pgm.s.pRamRangesXR3)
+        pgmR3PhysRelinkRamRanges(pVM);
+
+#ifndef PGM_WITHOUT_MAPPINGS
+
+    /*
+     * Update the two page directories with all page table mappings.
+     * (One or more of them have changed, that's why we're here.)
+     */
+    pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pMappingsR3);
+    for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur->pNextR3; pCur = pCur->pNextR3)
+        pCur->pNextRC = MMHyperR3ToRC(pVM, pCur->pNextR3);
+
+    /* Relocate GC addresses of Page Tables. */
+    for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur; pCur = pCur->pNextR3)
+    {
+        for (RTHCUINT i = 0; i < pCur->cPTs; i++)
+        {
+            pCur->aPTs[i].pPTRC = MMHyperR3ToRC(pVM, pCur->aPTs[i].pPTR3);
+            pCur->aPTs[i].paPaePTsRC = MMHyperR3ToRC(pVM, pCur->aPTs[i].paPaePTsR3);
+        }
+    }
+
+    /*
+     * Dynamic page mapping area.
+     */
+    pVM->pgm.s.paDynPageMap32BitPTEsGC += offDelta;
+    pVM->pgm.s.paDynPageMapPaePTEsGC   += offDelta;
+    pVM->pgm.s.pbDynPageMapBaseGC      += offDelta;
+
+    if (pVM->pgm.s.pRCDynMap)
+    {
+        pVM->pgm.s.pRCDynMap += offDelta;
+        PPGMRCDYNMAP pDynMap = (PPGMRCDYNMAP)MMHyperRCToCC(pVM, pVM->pgm.s.pRCDynMap);
+
+        pDynMap->paPages     += offDelta;
+        PPGMRCDYNMAPENTRY paPages = (PPGMRCDYNMAPENTRY)MMHyperRCToCC(pVM, pDynMap->paPages);
+
+        for (uint32_t iPage = 0; iPage < pDynMap->cPages; iPage++)
+        {
+            paPages[iPage].pvPage       += offDelta;
+            paPages[iPage].uPte.pLegacy += offDelta;
+            paPages[iPage].uPte.pPae    += offDelta;
+        }
+    }
+
+#endif /* PGM_WITHOUT_MAPPINGS */
+
+    /*
+     * The Zero page.
+     */
+    pVM->pgm.s.pvZeroPgR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pvZeroPgR3);
+    AssertRelease(pVM->pgm.s.pvZeroPgR0 != NIL_RTR0PTR);
+
+    /*
+     * The page pool.
+     */
+    pgmR3PoolRelocate(pVM);
+}
+
+
+/**
+ * Resets a virtual CPU when unplugged.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMR3DECL(void) PGMR3ResetCpu(PVM pVM, PVMCPU pVCpu)
+{
+    uintptr_t const idxGst = pVCpu->pgm.s.idxGuestModeData;
+    if (   idxGst < RT_ELEMENTS(g_aPgmGuestModeData)
+        && g_aPgmGuestModeData[idxGst].pfnExit)
+    {
+        int rc = g_aPgmGuestModeData[idxGst].pfnExit(pVCpu);
+        AssertReleaseRC(rc);
+    }
+    pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
+
+    int rc = PGMHCChangeMode(pVM, pVCpu, PGMMODE_REAL);
+    AssertReleaseRC(rc);
+
+    STAM_REL_COUNTER_RESET(&pVCpu->pgm.s.cGuestModeChanges);
+
+    pgmR3PoolResetUnpluggedCpu(pVM, pVCpu);
+
+    /*
+     * Re-init other members.
+     */
+    pVCpu->pgm.s.fA20Enabled = true;
+    pVCpu->pgm.s.GCPhysA20Mask = ~((RTGCPHYS)!pVCpu->pgm.s.fA20Enabled << 20);
+
+    /*
+     * Clear the FFs PGM owns.
+     */
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * For the PGM component this means that any PD write monitors
+ * needs to be removed.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) PGMR3Reset(PVM pVM)
+{
+    LogFlow(("PGMR3Reset:\n"));
+    VM_ASSERT_EMT(pVM);
+
+    pgmLock(pVM);
+
+    /*
+     * Unfix any fixed mappings and disable CR3 monitoring.
+     */
+    pVM->pgm.s.fMappingsFixed         = false;
+    pVM->pgm.s.fMappingsFixedRestored = false;
+    pVM->pgm.s.GCPtrMappingFixed      = NIL_RTGCPTR;
+    pVM->pgm.s.cbMappingFixed         = 0;
+
+    /*
+     * Exit the guest paging mode before the pgm pool gets reset.
+     * Important to clean up the amd64 case.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU          pVCpu  = pVM->apCpusR3[i];
+        uintptr_t const idxGst = pVCpu->pgm.s.idxGuestModeData;
+        if (   idxGst < RT_ELEMENTS(g_aPgmGuestModeData)
+            && g_aPgmGuestModeData[idxGst].pfnExit)
+        {
+            int rc = g_aPgmGuestModeData[idxGst].pfnExit(pVCpu);
+            AssertReleaseRC(rc);
+        }
+        pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
+    }
+
+#ifdef DEBUG
+    DBGFR3_INFO_LOG_SAFE(pVM, "mappings", NULL);
+    DBGFR3_INFO_LOG_SAFE(pVM, "handlers", "all nostat");
+#endif
+
+    /*
+     * Switch mode back to real mode. (Before resetting the pgm pool!)
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU  pVCpu = pVM->apCpusR3[i];
+
+        int rc = PGMHCChangeMode(pVM, pVCpu, PGMMODE_REAL);
+        AssertReleaseRC(rc);
+
+        STAM_REL_COUNTER_RESET(&pVCpu->pgm.s.cGuestModeChanges);
+        STAM_REL_COUNTER_RESET(&pVCpu->pgm.s.cA20Changes);
+    }
+
+    /*
+     * Reset the shadow page pool.
+     */
+    pgmR3PoolReset(pVM);
+
+    /*
+     * Re-init various other members and clear the FFs that PGM owns.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+
+        pVCpu->pgm.s.fGst32BitPageSizeExtension = false;
+        PGMNotifyNxeChanged(pVCpu, false);
+
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+
+        if (!pVCpu->pgm.s.fA20Enabled)
+        {
+            pVCpu->pgm.s.fA20Enabled = true;
+            pVCpu->pgm.s.GCPhysA20Mask = ~((RTGCPHYS)!pVCpu->pgm.s.fA20Enabled << 20);
+#ifdef PGM_WITH_A20
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+            pgmR3RefreshShadowModeAfterA20Change(pVCpu);
+            HMFlushTlb(pVCpu);
+#endif
+        }
+    }
+
+    //pgmLogState(pVM);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Memory setup after VM construction or reset.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   fAtReset    Indicates the context, after reset if @c true or after
+ *                      construction if @c false.
+ */
+VMMR3_INT_DECL(void) PGMR3MemSetup(PVM pVM, bool fAtReset)
+{
+    if (fAtReset)
+    {
+        pgmLock(pVM);
+
+        int rc = pgmR3PhysRamZeroAll(pVM);
+        AssertReleaseRC(rc);
+
+        rc = pgmR3PhysRomReset(pVM);
+        AssertReleaseRC(rc);
+
+        pgmUnlock(pVM);
+    }
+}
+
+
+#ifdef VBOX_STRICT
+/**
+ * VM state change callback for clearing fNoMorePhysWrites after
+ * a snapshot has been created.
+ */
+static DECLCALLBACK(void) pgmR3ResetNoMorePhysWritesFlag(PUVM pUVM, VMSTATE enmState, VMSTATE enmOldState, void *pvUser)
+{
+    if (   enmState == VMSTATE_RUNNING
+        || enmState == VMSTATE_RESUMING)
+        pUVM->pVM->pgm.s.fNoMorePhysWrites = false;
+    NOREF(enmOldState); NOREF(pvUser);
+}
+#endif
+
+/**
+ * Private API to reset fNoMorePhysWrites.
+ */
+VMMR3_INT_DECL(void) PGMR3ResetNoMorePhysWritesFlag(PVM pVM)
+{
+    pVM->pgm.s.fNoMorePhysWrites = false;
+}
+
+/**
+ * Terminates the PGM.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(int) PGMR3Term(PVM pVM)
+{
+    /* Must free shared pages here. */
+    pgmLock(pVM);
+    pgmR3PhysRamTerm(pVM);
+    pgmR3PhysRomTerm(pVM);
+    pgmUnlock(pVM);
+
+    PGMDeregisterStringFormatTypes();
+    return PDMR3CritSectDelete(&pVM->pgm.s.CritSectX);
+}
+
+
+/**
+ * Show paging mode.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     "all" (default), "guest", "shadow" or "host".
+ */
+static DECLCALLBACK(void) pgmR3InfoMode(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /* digest argument. */
+    bool fGuest, fShadow, fHost;
+    if (pszArgs)
+        pszArgs = RTStrStripL(pszArgs);
+    if (!pszArgs || !*pszArgs || strstr(pszArgs, "all"))
+        fShadow = fHost = fGuest = true;
+    else
+    {
+        fShadow = fHost = fGuest = false;
+        if (strstr(pszArgs, "guest"))
+            fGuest = true;
+        if (strstr(pszArgs, "shadow"))
+            fShadow = true;
+        if (strstr(pszArgs, "host"))
+            fHost = true;
+    }
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+
+    /* print info. */
+    if (fGuest)
+        pHlp->pfnPrintf(pHlp, "Guest paging mode (VCPU #%u):  %s (changed %RU64 times), A20 %s (changed %RU64 times)\n",
+                        pVCpu->idCpu, PGMGetModeName(pVCpu->pgm.s.enmGuestMode), pVCpu->pgm.s.cGuestModeChanges.c,
+                        pVCpu->pgm.s.fA20Enabled ? "enabled" : "disabled", pVCpu->pgm.s.cA20Changes.c);
+    if (fShadow)
+        pHlp->pfnPrintf(pHlp, "Shadow paging mode (VCPU #%u): %s\n", pVCpu->idCpu, PGMGetModeName(pVCpu->pgm.s.enmShadowMode));
+    if (fHost)
+    {
+        const char *psz;
+        switch (pVM->pgm.s.enmHostMode)
+        {
+            case SUPPAGINGMODE_INVALID:             psz = "invalid"; break;
+            case SUPPAGINGMODE_32_BIT:              psz = "32-bit"; break;
+            case SUPPAGINGMODE_32_BIT_GLOBAL:       psz = "32-bit+G"; break;
+            case SUPPAGINGMODE_PAE:                 psz = "PAE"; break;
+            case SUPPAGINGMODE_PAE_GLOBAL:          psz = "PAE+G"; break;
+            case SUPPAGINGMODE_PAE_NX:              psz = "PAE+NX"; break;
+            case SUPPAGINGMODE_PAE_GLOBAL_NX:       psz = "PAE+G+NX"; break;
+            case SUPPAGINGMODE_AMD64:               psz = "AMD64"; break;
+            case SUPPAGINGMODE_AMD64_GLOBAL:        psz = "AMD64+G"; break;
+            case SUPPAGINGMODE_AMD64_NX:            psz = "AMD64+NX"; break;
+            case SUPPAGINGMODE_AMD64_GLOBAL_NX:     psz = "AMD64+G+NX"; break;
+            default:                                psz = "unknown"; break;
+        }
+        pHlp->pfnPrintf(pHlp, "Host paging mode:             %s\n", psz);
+    }
+}
+
+
+/**
+ * Dump registered MMIO ranges to the log.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) pgmR3PhysInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    bool const fVerbose = pszArgs && strstr(pszArgs, "verbose") != NULL;
+
+    pHlp->pfnPrintf(pHlp,
+                    "RAM ranges (pVM=%p)\n"
+                    "%.*s %.*s\n",
+                    pVM,
+                    sizeof(RTGCPHYS) * 4 + 1, "GC Phys Range                    ",
+                    sizeof(RTHCPTR) * 2,      "pvHC            ");
+
+    for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
+    {
+        pHlp->pfnPrintf(pHlp,
+                        "%RGp-%RGp %RHv %s\n",
+                        pCur->GCPhys,
+                        pCur->GCPhysLast,
+                        pCur->pvR3,
+                        pCur->pszDesc);
+        if (fVerbose)
+        {
+            RTGCPHYS const cPages = pCur->cb >> X86_PAGE_SHIFT;
+            RTGCPHYS iPage = 0;
+            while (iPage < cPages)
+            {
+                RTGCPHYS const    iFirstPage = iPage;
+                PGMPAGETYPE const enmType    = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(&pCur->aPages[iPage]);
+                do
+                    iPage++;
+                while (iPage < cPages && (PGMPAGETYPE)PGM_PAGE_GET_TYPE(&pCur->aPages[iPage]) == enmType);
+                const char *pszType;
+                const char *pszMore = NULL;
+                switch (enmType)
+                {
+                    case PGMPAGETYPE_RAM:
+                        pszType = "RAM";
+                        break;
+
+                    case PGMPAGETYPE_MMIO2:
+                        pszType = "MMIO2";
+                        break;
+
+                    case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
+                        pszType = "MMIO2-alias-MMIO";
+                        break;
+
+                    case PGMPAGETYPE_SPECIAL_ALIAS_MMIO:
+                        pszType = "special-alias-MMIO";
+                        break;
+
+                    case PGMPAGETYPE_ROM_SHADOW:
+                    case PGMPAGETYPE_ROM:
+                    {
+                        pszType = enmType == PGMPAGETYPE_ROM_SHADOW ? "ROM-shadowed" : "ROM";
+
+                        RTGCPHYS const  GCPhysFirstPg = iFirstPage * X86_PAGE_SIZE;
+                        PPGMROMRANGE    pRom          = pVM->pgm.s.pRomRangesR3;
+                        while (pRom && GCPhysFirstPg > pRom->GCPhysLast)
+                            pRom = pRom->pNextR3;
+                        if (pRom && GCPhysFirstPg - pRom->GCPhys < pRom->cb)
+                            pszMore = pRom->pszDesc;
+                        break;
+                    }
+
+                    case PGMPAGETYPE_MMIO:
+                    {
+                        pszType = "MMIO";
+                        pgmLock(pVM);
+                        PPGMPHYSHANDLER pHandler = pgmHandlerPhysicalLookup(pVM, iFirstPage * X86_PAGE_SIZE);
+                        if (pHandler)
+                            pszMore = pHandler->pszDesc;
+                        pgmUnlock(pVM);
+                        break;
+                    }
+
+                    case PGMPAGETYPE_INVALID:
+                        pszType = "invalid";
+                        break;
+
+                    default:
+                        pszType = "bad";
+                        break;
+                }
+                if (pszMore)
+                    pHlp->pfnPrintf(pHlp, "    %RGp-%RGp %-20s %s\n",
+                                    pCur->GCPhys + iFirstPage * X86_PAGE_SIZE,
+                                    pCur->GCPhys + iPage      * X86_PAGE_SIZE - 1,
+                                    pszType, pszMore);
+                else
+                    pHlp->pfnPrintf(pHlp, "    %RGp-%RGp %s\n",
+                                    pCur->GCPhys + iFirstPage * X86_PAGE_SIZE,
+                                    pCur->GCPhys + iPage      * X86_PAGE_SIZE - 1,
+                                    pszType);
+
+            }
+        }
+    }
+}
+
+
+/**
+ * Dump the page directory to the log.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) pgmR3InfoCr3(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /** @todo SMP support!! */
+    PVMCPU pVCpu = pVM->apCpusR3[0];
+
+/** @todo fix this! Convert the PGMR3DumpHierarchyHC functions to do guest stuff. */
+    /* Big pages supported? */
+    const bool fPSE  = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
+
+    /* Global pages supported? */
+    const bool fPGE = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PGE);
+
+    NOREF(pszArgs);
+
+    /*
+     * Get page directory addresses.
+     */
+    pgmLock(pVM);
+    PX86PD     pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
+    Assert(pPDSrc);
+
+    /*
+     * Iterate the page directory.
+     */
+    for (unsigned iPD = 0; iPD < RT_ELEMENTS(pPDSrc->a); iPD++)
+    {
+        X86PDE PdeSrc = pPDSrc->a[iPD];
+        if (PdeSrc.n.u1Present)
+        {
+            if (PdeSrc.b.u1Size && fPSE)
+                pHlp->pfnPrintf(pHlp,
+                                "%04X - %RGp P=%d U=%d RW=%d G=%d - BIG\n",
+                                iPD,
+                                pgmGstGet4MBPhysPage(pVM, PdeSrc),
+                                PdeSrc.b.u1Present, PdeSrc.b.u1User, PdeSrc.b.u1Write, PdeSrc.b.u1Global && fPGE);
+            else
+                pHlp->pfnPrintf(pHlp,
+                                "%04X - %RGp P=%d U=%d RW=%d [G=%d]\n",
+                                iPD,
+                                (RTGCPHYS)(PdeSrc.u & X86_PDE_PG_MASK),
+                                PdeSrc.n.u1Present, PdeSrc.n.u1User, PdeSrc.n.u1Write, PdeSrc.b.u1Global && fPGE);
+        }
+    }
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Service a VMMCALLRING3_PGM_LOCK call.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(int) PGMR3LockCall(PVM pVM)
+{
+    int rc = PDMR3CritSectEnterEx(&pVM->pgm.s.CritSectX, true /* fHostCall */);
+    AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * Called by pgmPoolFlushAllInt prior to flushing the pool.
+ *
+ * @returns VBox status code, fully asserted.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+int pgmR3ExitShadowModeBeforePoolFlush(PVMCPU pVCpu)
+{
+    /* Unmap the old CR3 value before flushing everything. */
+    int       rc     = VINF_SUCCESS;
+    uintptr_t idxBth = pVCpu->pgm.s.idxBothModeData;
+    if (   idxBth < RT_ELEMENTS(g_aPgmBothModeData)
+        && g_aPgmBothModeData[idxBth].pfnMapCR3)
+    {
+        rc = g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
+        AssertRC(rc);
+    }
+
+    /* Exit the current shadow paging mode as well; nested paging and EPT use a root CR3 which will get flushed here. */
+    uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
+    if (   idxShw < RT_ELEMENTS(g_aPgmShadowModeData)
+        && g_aPgmShadowModeData[idxShw].pfnExit)
+    {
+        rc = g_aPgmShadowModeData[idxShw].pfnExit(pVCpu);
+        AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
+    }
+
+    Assert(pVCpu->pgm.s.pShwPageCR3R3 == NULL);
+    return rc;
+}
+
+
+/**
+ * Called by pgmPoolFlushAllInt after flushing the pool.
+ *
+ * @returns VBox status code, fully asserted.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu)
+{
+    pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
+    int rc = PGMHCChangeMode(pVM, pVCpu, PGMGetGuestMode(pVCpu));
+    Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+    AssertRCReturn(rc, rc);
+    AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
+
+    Assert(pVCpu->pgm.s.pShwPageCR3R3 != NULL || pVCpu->pgm.s.enmShadowMode == PGMMODE_NONE);
+    AssertMsg(   pVCpu->pgm.s.enmShadowMode >= PGMMODE_NESTED_32BIT
+              || CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu),
+              ("%RHp != %RHp %s\n", (RTHCPHYS)CPUMGetHyperCR3(pVCpu), PGMGetHyperCR3(pVCpu), PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
+    return rc;
+}
+
+
+/**
+ * Called by PGMR3PhysSetA20 after changing the A20 state.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+void pgmR3RefreshShadowModeAfterA20Change(PVMCPU pVCpu)
+{
+    /** @todo Probably doing a bit too much here. */
+    int rc = pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
+    AssertReleaseRC(rc);
+    rc = pgmR3ReEnterShadowModeAfterPoolFlush(pVCpu->CTX_SUFF(pVM), pVCpu);
+    AssertReleaseRC(rc);
+}
+
+
+#ifdef VBOX_WITH_DEBUGGER
+
+/**
+ * @callback_method_impl{FNDBGCCMD, The '.pgmerror' and '.pgmerroroff' commands.}
+ */
+static DECLCALLBACK(int)  pgmR3CmdError(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
+{
+    /*
+     * Validate input.
+     */
+    DBGC_CMDHLP_REQ_UVM_RET(pCmdHlp, pCmd, pUVM);
+    PVM pVM = pUVM->pVM;
+    DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, 0, cArgs == 0 || (cArgs == 1 && paArgs[0].enmType == DBGCVAR_TYPE_STRING));
+
+    if (!cArgs)
+    {
+        /*
+         * Print the list of error injection locations with status.
+         */
+        DBGCCmdHlpPrintf(pCmdHlp, "PGM error inject locations:\n");
+        DBGCCmdHlpPrintf(pCmdHlp, "  handy - %RTbool\n", pVM->pgm.s.fErrInjHandyPages);
+    }
+    else
+    {
+        /*
+         * String switch on where to inject the error.
+         */
+        bool const  fNewState = !strcmp(pCmd->pszCmd, "pgmerror");
+        const char *pszWhere = paArgs[0].u.pszString;
+        if (!strcmp(pszWhere, "handy"))
+            ASMAtomicWriteBool(&pVM->pgm.s.fErrInjHandyPages, fNewState);
+        else
+            return DBGCCmdHlpPrintf(pCmdHlp, "error: Invalid 'where' value: %s.\n", pszWhere);
+        DBGCCmdHlpPrintf(pCmdHlp, "done\n");
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNDBGCCMD, The '.pgmsync' command.}
+ */
+static DECLCALLBACK(int) pgmR3CmdSync(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
+{
+    /*
+     * Validate input.
+     */
+    NOREF(pCmd); NOREF(paArgs); NOREF(cArgs);
+    DBGC_CMDHLP_REQ_UVM_RET(pCmdHlp, pCmd, pUVM);
+    PVMCPU pVCpu = VMMR3GetCpuByIdU(pUVM, DBGCCmdHlpGetCurrentCpu(pCmdHlp));
+    if (!pVCpu)
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "Invalid CPU ID");
+
+    /*
+     * Force page directory sync.
+     */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+
+    int rc = DBGCCmdHlpPrintf(pCmdHlp, "Forcing page directory sync.\n");
+    if (RT_FAILURE(rc))
+        return rc;
+
+    return VINF_SUCCESS;
+}
+
+#ifdef VBOX_STRICT
+
+/**
+ * EMT callback for pgmR3CmdAssertCR3.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   pcErrors    Where to return the error count.
+ */
+static DECLCALLBACK(int) pgmR3CmdAssertCR3EmtWorker(PUVM pUVM, unsigned *pcErrors)
+{
+    PVM     pVM   = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    PVMCPU  pVCpu = VMMGetCpu(pVM);
+
+    *pcErrors = PGMAssertCR3(pVM, pVCpu, CPUMGetGuestCR3(pVCpu), CPUMGetGuestCR4(pVCpu));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNDBGCCMD, The '.pgmassertcr3' command.}
+ */
+static DECLCALLBACK(int) pgmR3CmdAssertCR3(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
+{
+    /*
+     * Validate input.
+     */
+    NOREF(pCmd); NOREF(paArgs); NOREF(cArgs);
+    DBGC_CMDHLP_REQ_UVM_RET(pCmdHlp, pCmd, pUVM);
+
+    int rc = DBGCCmdHlpPrintf(pCmdHlp, "Checking shadow CR3 page tables for consistency.\n");
+    if (RT_FAILURE(rc))
+        return rc;
+
+    unsigned cErrors = 0;
+    rc = VMR3ReqCallWaitU(pUVM, DBGCCmdHlpGetCurrentCpu(pCmdHlp), (PFNRT)pgmR3CmdAssertCR3EmtWorker, 2, pUVM, &cErrors);
+    if (RT_FAILURE(rc))
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "VMR3ReqCallWaitU failed: %Rrc", rc);
+    if (cErrors > 0)
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "PGMAssertCR3: %u error(s)", cErrors);
+    return DBGCCmdHlpPrintf(pCmdHlp, "PGMAssertCR3: OK\n");
+}
+
+#endif /* VBOX_STRICT */
+
+/**
+ * @callback_method_impl{FNDBGCCMD, The '.pgmsyncalways' command.}
+ */
+static DECLCALLBACK(int) pgmR3CmdSyncAlways(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
+{
+    /*
+     * Validate input.
+     */
+    NOREF(pCmd); NOREF(paArgs); NOREF(cArgs);
+    DBGC_CMDHLP_REQ_UVM_RET(pCmdHlp, pCmd, pUVM);
+    PVMCPU pVCpu = VMMR3GetCpuByIdU(pUVM, DBGCCmdHlpGetCurrentCpu(pCmdHlp));
+    if (!pVCpu)
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "Invalid CPU ID");
+
+    /*
+     * Force page directory sync.
+     */
+    int rc;
+    if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS)
+    {
+        ASMAtomicAndU32(&pVCpu->pgm.s.fSyncFlags, ~PGM_SYNC_ALWAYS);
+        rc = DBGCCmdHlpPrintf(pCmdHlp, "Disabled permanent forced page directory syncing.\n");
+    }
+    else
+    {
+        ASMAtomicOrU32(&pVCpu->pgm.s.fSyncFlags, PGM_SYNC_ALWAYS);
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        rc = DBGCCmdHlpPrintf(pCmdHlp, "Enabled permanent forced page directory syncing.\n");
+    }
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNDBGCCMD, The '.pgmphystofile' command.}
+ */
+static DECLCALLBACK(int) pgmR3CmdPhysToFile(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
+{
+    /*
+     * Validate input.
+     */
+    NOREF(pCmd);
+    DBGC_CMDHLP_REQ_UVM_RET(pCmdHlp, pCmd, pUVM);
+    PVM pVM = pUVM->pVM;
+    DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, 0, cArgs == 1 || cArgs == 2);
+    DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, 0, paArgs[0].enmType == DBGCVAR_TYPE_STRING);
+    if (cArgs == 2)
+    {
+        DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, 1, paArgs[1].enmType == DBGCVAR_TYPE_STRING);
+        if (strcmp(paArgs[1].u.pszString, "nozero"))
+            return DBGCCmdHlpFail(pCmdHlp, pCmd, "Invalid 2nd argument '%s', must be 'nozero'.\n", paArgs[1].u.pszString);
+    }
+    bool fIncZeroPgs = cArgs < 2;
+
+    /*
+     * Open the output file and get the ram parameters.
+     */
+    RTFILE hFile;
+    int rc = RTFileOpen(&hFile, paArgs[0].u.pszString, RTFILE_O_WRITE | RTFILE_O_CREATE_REPLACE | RTFILE_O_DENY_WRITE);
+    if (RT_FAILURE(rc))
+        return DBGCCmdHlpPrintf(pCmdHlp, "error: RTFileOpen(,'%s',) -> %Rrc.\n", paArgs[0].u.pszString, rc);
+
+    uint32_t cbRamHole = 0;
+    CFGMR3QueryU32Def(CFGMR3GetRootU(pUVM), "RamHoleSize", &cbRamHole, MM_RAM_HOLE_SIZE_DEFAULT);
+    uint64_t cbRam     = 0;
+    CFGMR3QueryU64Def(CFGMR3GetRootU(pUVM), "RamSize", &cbRam, 0);
+    RTGCPHYS GCPhysEnd = cbRam + cbRamHole;
+
+    /*
+     * Dump the physical memory, page by page.
+     */
+    RTGCPHYS    GCPhys = 0;
+    char        abZeroPg[PAGE_SIZE];
+    RT_ZERO(abZeroPg);
+
+    pgmLock(pVM);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
+         pRam && pRam->GCPhys < GCPhysEnd && RT_SUCCESS(rc);
+         pRam = pRam->pNextR3)
+    {
+        /* fill the gap */
+        if (pRam->GCPhys > GCPhys && fIncZeroPgs)
+        {
+            while (pRam->GCPhys > GCPhys && RT_SUCCESS(rc))
+            {
+                rc = RTFileWrite(hFile, abZeroPg, PAGE_SIZE, NULL);
+                GCPhys += PAGE_SIZE;
+            }
+        }
+
+        PCPGMPAGE pPage = &pRam->aPages[0];
+        while (GCPhys < pRam->GCPhysLast && RT_SUCCESS(rc))
+        {
+            if (    PGM_PAGE_IS_ZERO(pPage)
+                ||  PGM_PAGE_IS_BALLOONED(pPage))
+            {
+                if (fIncZeroPgs)
+                {
+                    rc = RTFileWrite(hFile, abZeroPg, PAGE_SIZE, NULL);
+                    if (RT_FAILURE(rc))
+                        DBGCCmdHlpPrintf(pCmdHlp, "error: RTFileWrite -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
+                }
+            }
+            else
+            {
+                switch (PGM_PAGE_GET_TYPE(pPage))
+                {
+                    case PGMPAGETYPE_RAM:
+                    case PGMPAGETYPE_ROM_SHADOW: /* trouble?? */
+                    case PGMPAGETYPE_ROM:
+                    case PGMPAGETYPE_MMIO2:
+                    {
+                        void const     *pvPage;
+                        PGMPAGEMAPLOCK  Lock;
+                        rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvPage, &Lock);
+                        if (RT_SUCCESS(rc))
+                        {
+                            rc = RTFileWrite(hFile, pvPage, PAGE_SIZE, NULL);
+                            PGMPhysReleasePageMappingLock(pVM, &Lock);
+                            if (RT_FAILURE(rc))
+                                DBGCCmdHlpPrintf(pCmdHlp, "error: RTFileWrite -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
+                        }
+                        else
+                            DBGCCmdHlpPrintf(pCmdHlp, "error: PGMPhysGCPhys2CCPtrReadOnly -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
+                        break;
+                    }
+
+                    default:
+                        AssertFailed();
+                        RT_FALL_THRU();
+                    case PGMPAGETYPE_MMIO:
+                    case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
+                    case PGMPAGETYPE_SPECIAL_ALIAS_MMIO:
+                        if (fIncZeroPgs)
+                        {
+                            rc = RTFileWrite(hFile, abZeroPg, PAGE_SIZE, NULL);
+                            if (RT_FAILURE(rc))
+                                DBGCCmdHlpPrintf(pCmdHlp, "error: RTFileWrite -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
+                        }
+                        break;
+                }
+            }
+
+
+            /* advance */
+            GCPhys += PAGE_SIZE;
+            pPage++;
+        }
+    }
+    pgmUnlock(pVM);
+
+    RTFileClose(hFile);
+    if (RT_SUCCESS(rc))
+        return DBGCCmdHlpPrintf(pCmdHlp, "Successfully saved physical memory to '%s'.\n", paArgs[0].u.pszString);
+    return VINF_SUCCESS;
+}
+
+#endif /* VBOX_WITH_DEBUGGER */
+
+/**
+ * pvUser argument of the pgmR3CheckIntegrity*Node callbacks.
+ */
+typedef struct PGMCHECKINTARGS
+{
+    bool                    fLeftToRight;    /**< true: left-to-right; false: right-to-left. */
+    PPGMPHYSHANDLER         pPrevPhys;
+    PVM                     pVM;
+} PGMCHECKINTARGS, *PPGMCHECKINTARGS;
+
+/**
+ * Validate a node in the physical handler tree.
+ *
+ * @returns 0 on if ok, other wise 1.
+ * @param   pNode       The handler node.
+ * @param   pvUser      pVM.
+ */
+static DECLCALLBACK(int) pgmR3CheckIntegrityPhysHandlerNode(PAVLROGCPHYSNODECORE pNode, void *pvUser)
+{
+    PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)pNode;
+    AssertReleaseReturn(!((uintptr_t)pCur & 7), 1);
+    AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,
+                     ("pCur=%p %RGp-%RGp %s\n", pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
+    AssertReleaseMsg(   !pArgs->pPrevPhys
+                     || (  pArgs->fLeftToRight
+                         ? pArgs->pPrevPhys->Core.KeyLast < pCur->Core.Key
+                         : pArgs->pPrevPhys->Core.KeyLast > pCur->Core.Key),
+                     ("pPrevPhys=%p %RGp-%RGp %s\n"
+                      "     pCur=%p %RGp-%RGp %s\n",
+                      pArgs->pPrevPhys, pArgs->pPrevPhys->Core.Key, pArgs->pPrevPhys->Core.KeyLast, pArgs->pPrevPhys->pszDesc,
+                      pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
+    pArgs->pPrevPhys = pCur;
+    return 0;
+}
+
+
+/**
+ * Perform an integrity check on the PGM component.
+ *
+ * @returns VINF_SUCCESS if everything is fine.
+ * @returns VBox error status after asserting on integrity breach.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(int) PGMR3CheckIntegrity(PVM pVM)
+{
+    AssertReleaseReturn(pVM->pgm.s.offVM, VERR_INTERNAL_ERROR);
+
+    /*
+     * Check the trees.
+     */
+    int cErrors = 0;
+    const PGMCHECKINTARGS LeftToRight = {  true, NULL, pVM };
+    const PGMCHECKINTARGS RightToLeft = { false, NULL, pVM };
+    PGMCHECKINTARGS Args = LeftToRight;
+    cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers,       true,  pgmR3CheckIntegrityPhysHandlerNode, &Args);
+    Args = RightToLeft;
+    cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers,       false, pgmR3CheckIntegrityPhysHandlerNode, &Args);
+
+    return !cErrors ? VINF_SUCCESS : VERR_INTERNAL_ERROR;
+}
+
diff --git a/src/VBox/VMM/VMMR3/PGMDbg.cpp b/src/VBox/VMM/VMMR3/PGMDbg.cpp
new file mode 100644
index 00000000..520017d2
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PGMDbg.cpp
@@ -0,0 +1,2863 @@
+/* $Id: PGMDbg.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor - Debugger & Debugging APIs.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/stam.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include "PGMInline.h"
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <VBox/log.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The max needle size that we will bother searching for
+ * This must not be more than half a page! */
+#define MAX_NEEDLE_SIZE     256
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * State structure for the paging hierarchy dumpers.
+ */
+typedef struct PGMR3DUMPHIERARCHYSTATE
+{
+    /** Pointer to the VM. */
+    PVM             pVM;
+    /** Output helpers. */
+    PCDBGFINFOHLP   pHlp;
+    /** Set if PSE, PAE or long mode is enabled. */
+    bool            fPse;
+    /** Set if PAE or long mode is enabled. */
+    bool            fPae;
+    /** Set if long mode is enabled. */
+    bool            fLme;
+    /** Set if nested paging. */
+    bool            fNp;
+    /** Set if EPT. */
+    bool            fEpt;
+    /** Set if NXE is enabled. */
+    bool            fNxe;
+    /** The number or chars the address needs. */
+    uint8_t         cchAddress;
+    /** The last reserved bit. */
+    uint8_t         uLastRsvdBit;
+    /** Dump the page info as well (shadow page summary / guest physical
+     *  page summary). */
+    bool            fDumpPageInfo;
+    /** Whether or not to print the header. */
+    bool            fPrintHeader;
+    /** Whether to print the CR3 value */
+    bool            fPrintCr3;
+    /** Padding*/
+    bool            afReserved[5];
+    /** The current address.  */
+    uint64_t        u64Address;
+    /** The last address to dump structures for. */
+    uint64_t        u64FirstAddress;
+    /** The last address to dump structures for. */
+    uint64_t        u64LastAddress;
+    /** Mask with the high reserved bits set. */
+    uint64_t        u64HighReservedBits;
+    /** The number of leaf entries that we've printed. */
+    uint64_t        cLeaves;
+} PGMR3DUMPHIERARCHYSTATE;
+/** Pointer to the paging hierarchy dumper state. */
+typedef PGMR3DUMPHIERARCHYSTATE *PPGMR3DUMPHIERARCHYSTATE;
+
+
+/**
+ * Assembly scanning function.
+ *
+ * @returns Pointer to possible match or NULL.
+ * @param   pvHaystack      Pointer to what we search in.
+ * @param   cbHaystack      Number of bytes to search.
+ * @param   pvNeedle        Pointer to what we search for.
+ * @param   cbNeedle        Size of what we're searching for.
+ */
+
+typedef DECLCALLBACK(uint8_t const *) FNPGMR3DBGFIXEDMEMSCAN(void const *pvHaystack, uint32_t cbHaystack,
+                                                             void const *pvNeedle, size_t cbNeedle);
+/** Pointer to an fixed size and step assembly scanner function. */
+typedef FNPGMR3DBGFIXEDMEMSCAN *PFNPGMR3DBGFIXEDMEMSCAN;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+DECLASM(uint8_t const *) pgmR3DbgFixedMemScan8Wide8Step(void const *, uint32_t, void const *, size_t cbNeedle);
+DECLASM(uint8_t const *) pgmR3DbgFixedMemScan4Wide4Step(void const *, uint32_t, void const *, size_t cbNeedle);
+DECLASM(uint8_t const *) pgmR3DbgFixedMemScan2Wide2Step(void const *, uint32_t, void const *, size_t cbNeedle);
+DECLASM(uint8_t const *) pgmR3DbgFixedMemScan1Wide1Step(void const *, uint32_t, void const *, size_t cbNeedle);
+DECLASM(uint8_t const *) pgmR3DbgFixedMemScan4Wide1Step(void const *, uint32_t, void const *, size_t cbNeedle);
+DECLASM(uint8_t const *) pgmR3DbgFixedMemScan8Wide1Step(void const *, uint32_t, void const *, size_t cbNeedle);
+
+
+/**
+ * Converts a R3 pointer to a GC physical address.
+ *
+ * Only for the debugger.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success, *pGCPhys is set.
+ * @retval  VERR_INVALID_POINTER if the pointer is not within the GC physical memory.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   R3Ptr       The R3 pointer to convert.
+ * @param   pGCPhys     Where to store the GC physical address on success.
+ */
+VMMR3DECL(int) PGMR3DbgR3Ptr2GCPhys(PUVM pUVM, RTR3PTR R3Ptr, PRTGCPHYS pGCPhys)
+{
+    NOREF(pUVM); NOREF(R3Ptr);
+    *pGCPhys = NIL_RTGCPHYS;
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * Converts a R3 pointer to a HC physical address.
+ *
+ * Only for the debugger.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success, *pHCPhys is set.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical page but has no physical backing.
+ * @retval  VERR_INVALID_POINTER if the pointer is not within the GC physical memory.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   R3Ptr       The R3 pointer to convert.
+ * @param   pHCPhys     Where to store the HC physical address on success.
+ */
+VMMR3DECL(int) PGMR3DbgR3Ptr2HCPhys(PUVM pUVM, RTR3PTR R3Ptr, PRTHCPHYS pHCPhys)
+{
+    NOREF(pUVM); NOREF(R3Ptr);
+    *pHCPhys = NIL_RTHCPHYS;
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * Converts a HC physical address to a GC physical address.
+ *
+ * Only for the debugger.
+ *
+ * @returns VBox status code
+ * @retval  VINF_SUCCESS on success, *pGCPhys is set.
+ * @retval  VERR_INVALID_POINTER if the HC physical address is not within the GC physical memory.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   HCPhys  The HC physical address to convert.
+ * @param   pGCPhys Where to store the GC physical address on success.
+ */
+VMMR3DECL(int) PGMR3DbgHCPhys2GCPhys(PUVM pUVM, RTHCPHYS HCPhys, PRTGCPHYS pGCPhys)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Validate and adjust the input a bit.
+     */
+    if (HCPhys == NIL_RTHCPHYS)
+        return VERR_INVALID_POINTER;
+    unsigned off = HCPhys & PAGE_OFFSET_MASK;
+    HCPhys &= X86_PTE_PAE_PG_MASK;
+    if (HCPhys == 0)
+        return VERR_INVALID_POINTER;
+
+    for (PPGMRAMRANGE pRam = pUVM->pVM->pgm.s.CTX_SUFF(pRamRangesX);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        uint32_t iPage = pRam->cb >> PAGE_SHIFT;
+        while (iPage-- > 0)
+            if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
+            {
+                *pGCPhys = pRam->GCPhys + (iPage << PAGE_SHIFT) + off;
+                return VINF_SUCCESS;
+            }
+    }
+    return VERR_INVALID_POINTER;
+}
+
+
+/**
+ * Read physical memory API for the debugger, similar to
+ * PGMPhysSimpleReadGCPhys.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pvDst       Where to store what's read.
+ * @param   GCPhysSrc   Where to start reading from.
+ * @param   cb          The number of bytes to attempt reading.
+ * @param   fFlags      Flags, MBZ.
+ * @param   pcbRead     For store the actual number of bytes read, pass NULL if
+ *                      partial reads are unwanted.
+ * @todo    Unused?
+ */
+VMMR3_INT_DECL(int) PGMR3DbgReadGCPhys(PVM pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb, uint32_t fFlags, size_t *pcbRead)
+{
+    /* validate */
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+
+    /* try simple first. */
+    int rc = PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc, cb);
+    if (RT_SUCCESS(rc) || !pcbRead)
+        return rc;
+
+    /* partial read that failed, chop it up in pages. */
+    *pcbRead = 0;
+    rc = VINF_SUCCESS;
+    while (cb > 0)
+    {
+        size_t cbChunk = PAGE_SIZE;
+        cbChunk -= GCPhysSrc & PAGE_OFFSET_MASK;
+        if (cbChunk > cb)
+            cbChunk = cb;
+
+        rc = PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc, cbChunk);
+
+        /* advance */
+        if (RT_FAILURE(rc))
+            break;
+        *pcbRead  += cbChunk;
+        cb        -= cbChunk;
+        GCPhysSrc += cbChunk;
+        pvDst = (uint8_t *)pvDst + cbChunk;
+    }
+
+    return *pcbRead && RT_FAILURE(rc) ? -rc : rc;
+}
+
+
+/**
+ * Write physical memory API for the debugger, similar to
+ * PGMPhysSimpleWriteGCPhys.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhysDst   Where to start writing.
+ * @param   pvSrc       What to write.
+ * @param   cb          The number of bytes to attempt writing.
+ * @param   fFlags      Flags, MBZ.
+ * @param   pcbWritten  For store the actual number of bytes written, pass NULL
+ *                      if partial writes are unwanted.
+ * @todo    Unused?
+ */
+VMMR3_INT_DECL(int) PGMR3DbgWriteGCPhys(PVM pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb, uint32_t fFlags, size_t *pcbWritten)
+{
+    /* validate */
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+
+    /* try simple first. */
+    int rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysDst, pvSrc, cb);
+    if (RT_SUCCESS(rc) || !pcbWritten)
+        return rc;
+
+    /* partial write that failed, chop it up in pages. */
+    *pcbWritten = 0;
+    rc = VINF_SUCCESS;
+    while (cb > 0)
+    {
+        size_t cbChunk = PAGE_SIZE;
+        cbChunk -= GCPhysDst & PAGE_OFFSET_MASK;
+        if (cbChunk > cb)
+            cbChunk = cb;
+
+        rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysDst, pvSrc, cbChunk);
+
+        /* advance */
+        if (RT_FAILURE(rc))
+            break;
+        *pcbWritten += cbChunk;
+        cb          -= cbChunk;
+        GCPhysDst   += cbChunk;
+        pvSrc = (uint8_t const *)pvSrc + cbChunk;
+    }
+
+    return *pcbWritten && RT_FAILURE(rc) ? -rc : rc;
+
+}
+
+
+/**
+ * Read virtual memory API for the debugger, similar to PGMPhysSimpleReadGCPtr.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pvDst       Where to store what's read.
+ * @param   GCPtrSrc    Where to start reading from.
+ * @param   cb          The number of bytes to attempt reading.
+ * @param   fFlags      Flags, MBZ.
+ * @param   pcbRead     For store the actual number of bytes read, pass NULL if
+ *                      partial reads are unwanted.
+ * @todo    Unused?
+ */
+VMMR3_INT_DECL(int) PGMR3DbgReadGCPtr(PVM pVM, void *pvDst, RTGCPTR GCPtrSrc, size_t cb, uint32_t fFlags, size_t *pcbRead)
+{
+    /* validate */
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+
+    /** @todo SMP support! */
+    PVMCPU pVCpu = pVM->apCpusR3[0];
+
+/** @todo deal with HMA */
+    /* try simple first. */
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, pvDst, GCPtrSrc, cb);
+    if (RT_SUCCESS(rc) || !pcbRead)
+        return rc;
+
+    /* partial read that failed, chop it up in pages. */
+    *pcbRead = 0;
+    rc = VINF_SUCCESS;
+    while (cb > 0)
+    {
+        size_t cbChunk = PAGE_SIZE;
+        cbChunk -= GCPtrSrc & PAGE_OFFSET_MASK;
+        if (cbChunk > cb)
+            cbChunk = cb;
+
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, pvDst, GCPtrSrc, cbChunk);
+
+        /* advance */
+        if (RT_FAILURE(rc))
+            break;
+        *pcbRead  += cbChunk;
+        cb        -= cbChunk;
+        GCPtrSrc  += cbChunk;
+        pvDst = (uint8_t *)pvDst + cbChunk;
+    }
+
+    return *pcbRead && RT_FAILURE(rc) ? -rc : rc;
+
+}
+
+
+/**
+ * Write virtual memory API for the debugger, similar to
+ * PGMPhysSimpleWriteGCPtr.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPtrDst    Where to start writing.
+ * @param   pvSrc       What to write.
+ * @param   cb          The number of bytes to attempt writing.
+ * @param   fFlags      Flags, MBZ.
+ * @param   pcbWritten  For store the actual number of bytes written, pass NULL
+ *                      if partial writes are unwanted.
+ * @todo    Unused?
+ */
+VMMR3_INT_DECL(int) PGMR3DbgWriteGCPtr(PVM pVM, RTGCPTR GCPtrDst, void const *pvSrc, size_t cb, uint32_t fFlags, size_t *pcbWritten)
+{
+    /* validate */
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+
+    /** @todo SMP support! */
+    PVMCPU pVCpu = pVM->apCpusR3[0];
+
+/** @todo deal with HMA */
+    /* try simple first. */
+    int rc = PGMPhysSimpleWriteGCPtr(pVCpu, GCPtrDst, pvSrc, cb);
+    if (RT_SUCCESS(rc) || !pcbWritten)
+        return rc;
+
+    /* partial write that failed, chop it up in pages. */
+    *pcbWritten = 0;
+    rc = VINF_SUCCESS;
+    while (cb > 0)
+    {
+        size_t cbChunk = PAGE_SIZE;
+        cbChunk -= GCPtrDst & PAGE_OFFSET_MASK;
+        if (cbChunk > cb)
+            cbChunk = cb;
+
+        rc = PGMPhysSimpleWriteGCPtr(pVCpu, GCPtrDst, pvSrc, cbChunk);
+
+        /* advance */
+        if (RT_FAILURE(rc))
+            break;
+        *pcbWritten += cbChunk;
+        cb          -= cbChunk;
+        GCPtrDst    += cbChunk;
+        pvSrc = (uint8_t const *)pvSrc + cbChunk;
+    }
+
+    return *pcbWritten && RT_FAILURE(rc) ? -rc : rc;
+
+}
+
+
+/**
+ * memchr() with alignment considerations.
+ *
+ * @returns Pointer to matching byte, NULL if none found.
+ * @param   pb                  Where to search. Aligned.
+ * @param   b                   What to search for.
+ * @param   cb                  How much to search .
+ * @param   uAlign              The alignment restriction of the result.
+ */
+static const uint8_t *pgmR3DbgAlignedMemChr(const uint8_t *pb, uint8_t b, size_t cb, uint32_t uAlign)
+{
+    const uint8_t *pbRet;
+    if (uAlign <= 32)
+    {
+        pbRet = (const uint8_t *)memchr(pb, b, cb);
+        if ((uintptr_t)pbRet & (uAlign - 1))
+        {
+            do
+            {
+                pbRet++;
+                size_t cbLeft = cb - (pbRet - pb);
+                if (!cbLeft)
+                {
+                    pbRet = NULL;
+                    break;
+                }
+                pbRet = (const uint8_t *)memchr(pbRet, b, cbLeft);
+            } while ((uintptr_t)pbRet & (uAlign - 1));
+        }
+    }
+    else
+    {
+        pbRet = NULL;
+        if (cb)
+        {
+            for (;;)
+            {
+                if (*pb == b)
+                {
+                    pbRet = pb;
+                    break;
+                }
+                if (cb <= uAlign)
+                    break;
+                cb -= uAlign;
+                pb += uAlign;
+            }
+        }
+    }
+    return pbRet;
+}
+
+
+/**
+ * Scans a page for a byte string, keeping track of potential
+ * cross page matches.
+ *
+ * @returns true and *poff on match.
+ *          false on mismatch.
+ * @param   pbPage          Pointer to the current page.
+ * @param   poff            Input: The offset into the page (aligned).
+ *                          Output: The page offset of the match on success.
+ * @param   cb              The number of bytes to search, starting of *poff.
+ * @param   uAlign          The needle alignment. This is of course less than a page.
+ * @param   pabNeedle       The byte string to search for.
+ * @param   cbNeedle        The length of the byte string.
+ * @param   pfnFixedMemScan Pointer to assembly scan function, if available for
+ *                          the given needle and alignment combination.
+ * @param   pabPrev         The buffer that keeps track of a partial match that we
+ *                          bring over from the previous page. This buffer must be
+ *                          at least cbNeedle - 1 big.
+ * @param   pcbPrev         Input: The number of partial matching bytes from the previous page.
+ *                          Output: The number of partial matching bytes from this page.
+ *                          Initialize to 0 before the first call to this function.
+ */
+static bool pgmR3DbgScanPage(const uint8_t *pbPage, int32_t *poff, uint32_t cb, uint32_t uAlign,
+                             const uint8_t *pabNeedle, size_t cbNeedle, PFNPGMR3DBGFIXEDMEMSCAN pfnFixedMemScan,
+                             uint8_t *pabPrev, size_t *pcbPrev)
+{
+    /*
+     * Try complete any partial match from the previous page.
+     */
+    if (*pcbPrev > 0)
+    {
+        size_t cbPrev = *pcbPrev;
+        Assert(!*poff);
+        Assert(cbPrev < cbNeedle);
+        if (!memcmp(pbPage, pabNeedle + cbPrev, cbNeedle - cbPrev))
+        {
+            if (cbNeedle - cbPrev > cb)
+                return false;
+            *poff = -(int32_t)cbPrev;
+            return true;
+        }
+
+        /* check out the remainder of the previous page. */
+        const uint8_t *pb = pabPrev;
+        for (;;)
+        {
+            if (cbPrev <= uAlign)
+                break;
+            cbPrev -= uAlign;
+            pb = pgmR3DbgAlignedMemChr(pb + uAlign, *pabNeedle, cbPrev, uAlign);
+            if (!pb)
+                break;
+            cbPrev = *pcbPrev - (pb - pabPrev);
+            if (    !memcmp(pb + 1, &pabNeedle[1], cbPrev - 1)
+                &&  !memcmp(pbPage, pabNeedle + cbPrev, cbNeedle - cbPrev))
+            {
+                if (cbNeedle - cbPrev > cb)
+                    return false;
+                *poff = -(int32_t)cbPrev;
+                return true;
+            }
+        }
+
+        *pcbPrev = 0;
+    }
+
+    /*
+     * Match the body of the page.
+     */
+    const uint8_t *pb = pbPage + *poff;
+    const uint8_t * const pbEnd = pb + cb;
+    for (;;)
+    {
+        AssertMsg(((uintptr_t)pb & (uAlign - 1)) == 0, ("%#p %#x\n", pb, uAlign));
+        if (pfnFixedMemScan)
+            pb = pfnFixedMemScan(pb, cb, pabNeedle, cbNeedle);
+        else
+            pb = pgmR3DbgAlignedMemChr(pb, *pabNeedle, cb, uAlign);
+        if (!pb)
+            break;
+        cb = pbEnd - pb;
+        if (cb >= cbNeedle)
+        {
+            /* match? */
+            if (!memcmp(pb + 1, &pabNeedle[1], cbNeedle - 1))
+            {
+                *poff = pb - pbPage;
+                return true;
+            }
+        }
+        else
+        {
+            /* partial match at the end of the page? */
+            if (!memcmp(pb + 1, &pabNeedle[1], cb - 1))
+            {
+                /* We're copying one byte more that we really need here, but wtf. */
+                memcpy(pabPrev, pb, cb);
+                *pcbPrev = cb;
+                return false;
+            }
+        }
+
+        /* no match, skip ahead. */
+        if (cb <= uAlign)
+            break;
+        pb += uAlign;
+        cb -= uAlign;
+    }
+
+    return false;
+}
+
+
+static void pgmR3DbgSelectMemScanFunction(PFNPGMR3DBGFIXEDMEMSCAN *ppfnMemScan, uint32_t GCPhysAlign, size_t cbNeedle)
+{
+    *ppfnMemScan = NULL;
+    switch (GCPhysAlign)
+    {
+        case 1:
+            if (cbNeedle >= 8)
+                *ppfnMemScan = pgmR3DbgFixedMemScan8Wide1Step;
+            else if (cbNeedle >= 4)
+                *ppfnMemScan = pgmR3DbgFixedMemScan4Wide1Step;
+            else
+                *ppfnMemScan = pgmR3DbgFixedMemScan1Wide1Step;
+            break;
+        case 2:
+            if (cbNeedle >= 2)
+                *ppfnMemScan = pgmR3DbgFixedMemScan2Wide2Step;
+            break;
+        case 4:
+            if (cbNeedle >= 4)
+                *ppfnMemScan = pgmR3DbgFixedMemScan4Wide4Step;
+            break;
+        case 8:
+            if (cbNeedle >= 8)
+                *ppfnMemScan = pgmR3DbgFixedMemScan8Wide8Step;
+            break;
+    }
+}
+
+
+
+/**
+ * Scans guest physical memory for a byte string.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS and *pGCPtrHit on success.
+ * @retval  VERR_DBGF_MEM_NOT_FOUND if not found.
+ * @retval  VERR_INVALID_POINTER if any of the pointer arguments are invalid.
+ * @retval  VERR_INVALID_ARGUMENT if any other arguments are invalid.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Where to start searching.
+ * @param   cbRange         The number of bytes to search.
+ * @param   GCPhysAlign     The alignment of the needle. Must be a power of two
+ *                          and less or equal to 4GB.
+ * @param   pabNeedle       The byte string to search for.
+ * @param   cbNeedle        The length of the byte string. Max 256 bytes.
+ * @param   pGCPhysHit      Where to store the address of the first occurrence on success.
+ */
+VMMR3_INT_DECL(int) PGMR3DbgScanPhysical(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cbRange, RTGCPHYS GCPhysAlign,
+                                         const uint8_t *pabNeedle, size_t cbNeedle, PRTGCPHYS pGCPhysHit)
+{
+    /*
+     * Validate and adjust the input a bit.
+     */
+    if (!VALID_PTR(pGCPhysHit))
+        return VERR_INVALID_POINTER;
+    *pGCPhysHit = NIL_RTGCPHYS;
+
+    if (    !VALID_PTR(pabNeedle)
+        ||  GCPhys == NIL_RTGCPHYS)
+        return VERR_INVALID_POINTER;
+    if (!cbNeedle)
+        return VERR_INVALID_PARAMETER;
+    if (cbNeedle > MAX_NEEDLE_SIZE)
+        return VERR_INVALID_PARAMETER;
+
+    if (!cbRange)
+        return VERR_DBGF_MEM_NOT_FOUND;
+    if (GCPhys + cbNeedle - 1 < GCPhys)
+        return VERR_DBGF_MEM_NOT_FOUND;
+
+    if (!GCPhysAlign)
+        return VERR_INVALID_PARAMETER;
+    if (GCPhysAlign > UINT32_MAX)
+        return VERR_NOT_POWER_OF_TWO;
+    if (GCPhysAlign & (GCPhysAlign - 1))
+        return VERR_INVALID_PARAMETER;
+
+    if (GCPhys & (GCPhysAlign - 1))
+    {
+        RTGCPHYS Adj = GCPhysAlign - (GCPhys & (GCPhysAlign - 1));
+        if (    cbRange <= Adj
+            ||  GCPhys + Adj < GCPhys)
+            return VERR_DBGF_MEM_NOT_FOUND;
+        GCPhys  += Adj;
+        cbRange -= Adj;
+    }
+
+    const bool      fAllZero   = ASMMemIsZero(pabNeedle, cbNeedle);
+    const uint32_t  cIncPages  = GCPhysAlign <= PAGE_SIZE
+                               ? 1
+                               : GCPhysAlign >> PAGE_SHIFT;
+    const RTGCPHYS  GCPhysLast = GCPhys + cbRange - 1 >= GCPhys
+                               ? GCPhys + cbRange - 1
+                               : ~(RTGCPHYS)0;
+
+    PFNPGMR3DBGFIXEDMEMSCAN pfnMemScan;
+    pgmR3DbgSelectMemScanFunction(&pfnMemScan, (uint32_t)GCPhysAlign, cbNeedle);
+
+    /*
+     * Search the memory - ignore MMIO and zero pages, also don't
+     * bother to match across ranges.
+     */
+    pgmLock(pVM);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        /*
+         * If the search range starts prior to the current ram range record,
+         * adjust the search range and possibly conclude the search.
+         */
+        RTGCPHYS off;
+        if (GCPhys < pRam->GCPhys)
+        {
+            if (GCPhysLast < pRam->GCPhys)
+                break;
+            GCPhys = pRam->GCPhys;
+            off = 0;
+        }
+        else
+            off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            /*
+             * Iterate the relevant pages.
+             */
+            uint8_t         abPrev[MAX_NEEDLE_SIZE];
+            size_t          cbPrev   = 0;
+            const uint32_t  cPages   = pRam->cb >> PAGE_SHIFT;
+            uint32_t        iPage    = off >> PAGE_SHIFT;
+            uint32_t        offPage  = GCPhys & PAGE_OFFSET_MASK;
+            GCPhys &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
+            for (;; offPage = 0)
+            {
+                PPGMPAGE pPage = &pRam->aPages[iPage];
+                if (   (   !PGM_PAGE_IS_ZERO(pPage)
+                        || fAllZero)
+                    && !PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)
+                    && !PGM_PAGE_IS_BALLOONED(pPage))
+                {
+                    void const     *pvPage;
+                    PGMPAGEMAPLOCK  Lock;
+                    int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvPage, &Lock);
+                    if (RT_SUCCESS(rc))
+                    {
+                        int32_t     offHit = offPage;
+                        bool        fRc;
+                        if (GCPhysAlign < PAGE_SIZE)
+                        {
+                            uint32_t cbSearch = (GCPhys ^ GCPhysLast) & ~(RTGCPHYS)PAGE_OFFSET_MASK
+                                              ? PAGE_SIZE                           - (uint32_t)offPage
+                                              : (GCPhysLast & PAGE_OFFSET_MASK) + 1 - (uint32_t)offPage;
+                            fRc = pgmR3DbgScanPage((uint8_t const *)pvPage, &offHit, cbSearch, (uint32_t)GCPhysAlign,
+                                                   pabNeedle, cbNeedle, pfnMemScan, &abPrev[0], &cbPrev);
+                        }
+                        else
+                            fRc = memcmp(pvPage, pabNeedle, cbNeedle) == 0
+                               && (GCPhysLast - GCPhys) >= cbNeedle;
+                        PGMPhysReleasePageMappingLock(pVM, &Lock);
+                        if (fRc)
+                        {
+                            *pGCPhysHit = GCPhys + offHit;
+                            pgmUnlock(pVM);
+                            return VINF_SUCCESS;
+                        }
+                    }
+                    else
+                        cbPrev = 0; /* ignore error. */
+                }
+                else
+                    cbPrev = 0;
+
+                /* advance to the next page. */
+                GCPhys += (RTGCPHYS)cIncPages << PAGE_SHIFT;
+                if (GCPhys >= GCPhysLast) /* (may not always hit, but we're run out of ranges.) */
+                {
+                    pgmUnlock(pVM);
+                    return VERR_DBGF_MEM_NOT_FOUND;
+                }
+                iPage += cIncPages;
+                if (    iPage < cIncPages
+                    ||  iPage >= cPages)
+                    break;
+            }
+        }
+    }
+    pgmUnlock(pVM);
+    return VERR_DBGF_MEM_NOT_FOUND;
+}
+
+
+/**
+ * Scans (guest) virtual memory for a byte string.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS and *pGCPtrHit on success.
+ * @retval  VERR_DBGF_MEM_NOT_FOUND if not found.
+ * @retval  VERR_INVALID_POINTER if any of the pointer arguments are invalid.
+ * @retval  VERR_INVALID_ARGUMENT if any other arguments are invalid.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the CPU
+ *                          context to search from.
+ * @param   GCPtr           Where to start searching.
+ * @param   GCPtrAlign      The alignment of the needle. Must be a power of two
+ *                          and less or equal to 4GB.
+ * @param   cbRange         The number of bytes to search. Max 256 bytes.
+ * @param   pabNeedle       The byte string to search for.
+ * @param   cbNeedle        The length of the byte string.
+ * @param   pGCPtrHit       Where to store the address of the first occurrence on success.
+ */
+VMMR3_INT_DECL(int) PGMR3DbgScanVirtual(PVM pVM, PVMCPU pVCpu, RTGCPTR GCPtr, RTGCPTR cbRange, RTGCPTR GCPtrAlign,
+                                        const uint8_t *pabNeedle, size_t cbNeedle, PRTGCUINTPTR pGCPtrHit)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Validate and adjust the input a bit.
+     */
+    if (!VALID_PTR(pGCPtrHit))
+        return VERR_INVALID_POINTER;
+    *pGCPtrHit = 0;
+
+    if (!VALID_PTR(pabNeedle))
+        return VERR_INVALID_POINTER;
+    if (!cbNeedle)
+        return VERR_INVALID_PARAMETER;
+    if (cbNeedle > MAX_NEEDLE_SIZE)
+        return VERR_INVALID_PARAMETER;
+
+    if (!cbRange)
+        return VERR_DBGF_MEM_NOT_FOUND;
+    if (GCPtr + cbNeedle - 1 < GCPtr)
+        return VERR_DBGF_MEM_NOT_FOUND;
+
+    if (!GCPtrAlign)
+        return VERR_INVALID_PARAMETER;
+    if (GCPtrAlign > UINT32_MAX)
+        return VERR_NOT_POWER_OF_TWO;
+    if (GCPtrAlign & (GCPtrAlign - 1))
+        return VERR_INVALID_PARAMETER;
+
+    if (GCPtr & (GCPtrAlign - 1))
+    {
+        RTGCPTR Adj = GCPtrAlign - (GCPtr & (GCPtrAlign - 1));
+        if (    cbRange <= Adj
+            ||  GCPtr + Adj < GCPtr)
+            return VERR_DBGF_MEM_NOT_FOUND;
+        GCPtr   += Adj;
+        cbRange -= Adj;
+    }
+
+    /* Only paged protected mode or long mode here, use the physical scan for
+       the other modes. */
+    PGMMODE enmMode   = PGMGetGuestMode(pVCpu);
+    AssertReturn(PGMMODE_WITH_PAGING(enmMode), VERR_PGM_NOT_USED_IN_MODE);
+
+    /*
+     * Search the memory - ignore MMIO, zero and not-present pages.
+     */
+    const bool      fAllZero  = ASMMemIsZero(pabNeedle, cbNeedle);
+    RTGCPTR         GCPtrMask = PGMMODE_IS_LONG_MODE(enmMode) ? UINT64_MAX : UINT32_MAX;
+    uint8_t         abPrev[MAX_NEEDLE_SIZE];
+    size_t          cbPrev    = 0;
+    const uint32_t  cIncPages = GCPtrAlign <= PAGE_SIZE
+                              ? 1
+                              : GCPtrAlign >> PAGE_SHIFT;
+    const RTGCPTR   GCPtrLast = GCPtr + cbRange - 1 >= GCPtr
+                              ? (GCPtr + cbRange - 1) & GCPtrMask
+                              : GCPtrMask;
+    RTGCPTR         cPages    = (((GCPtrLast - GCPtr) + (GCPtr & PAGE_OFFSET_MASK)) >> PAGE_SHIFT) + 1;
+    uint32_t        offPage   = GCPtr & PAGE_OFFSET_MASK;
+    GCPtr &= ~(RTGCPTR)PAGE_OFFSET_MASK;
+
+    PFNPGMR3DBGFIXEDMEMSCAN pfnMemScan;
+    pgmR3DbgSelectMemScanFunction(&pfnMemScan, (uint32_t)GCPtrAlign, cbNeedle);
+
+    VMSTATE         enmVMState              = pVM->enmVMState;
+    uint32_t const  cYieldCountDownReload   = VMSTATE_IS_RUNNING(enmVMState) ? 4096 : 65536;
+    uint32_t        cYieldCountDown         = cYieldCountDownReload;
+    RTGCPHYS        GCPhysPrev              = NIL_RTGCPHYS;
+    bool            fFullWalk               = true;
+    PGMPTWALKGST    Walk;
+    RT_ZERO(Walk);
+
+    pgmLock(pVM);
+    for (;; offPage = 0)
+    {
+        int rc;
+        if (fFullWalk)
+            rc = pgmGstPtWalk(pVCpu, GCPtr, &Walk);
+        else
+            rc = pgmGstPtWalkNext(pVCpu, GCPtr, &Walk);
+        if (RT_SUCCESS(rc) && Walk.u.Core.fSucceeded)
+        {
+            fFullWalk = false;
+
+            /* Skip if same page as previous one (W10 optimization). */
+            if (   Walk.u.Core.GCPhys != GCPhysPrev
+                || cbPrev != 0)
+            {
+                PPGMPAGE pPage = pgmPhysGetPage(pVM, Walk.u.Core.GCPhys);
+                if (   pPage
+                    && (   !PGM_PAGE_IS_ZERO(pPage)
+                        || fAllZero)
+                    && !PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)
+                    && !PGM_PAGE_IS_BALLOONED(pPage))
+                {
+                    GCPhysPrev = Walk.u.Core.GCPhys;
+                    void const *pvPage;
+                    PGMPAGEMAPLOCK Lock;
+                    rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, Walk.u.Core.GCPhys, &pvPage, &Lock);
+                    if (RT_SUCCESS(rc))
+                    {
+                        int32_t offHit = offPage;
+                        bool    fRc;
+                        if (GCPtrAlign < PAGE_SIZE)
+                        {
+                            uint32_t cbSearch = cPages > 0
+                                              ? PAGE_SIZE                          - (uint32_t)offPage
+                                              : (GCPtrLast & PAGE_OFFSET_MASK) + 1 - (uint32_t)offPage;
+                            fRc = pgmR3DbgScanPage((uint8_t const *)pvPage, &offHit, cbSearch, (uint32_t)GCPtrAlign,
+                                                   pabNeedle, cbNeedle, pfnMemScan, &abPrev[0], &cbPrev);
+                        }
+                        else
+                            fRc = memcmp(pvPage, pabNeedle, cbNeedle) == 0
+                               && (GCPtrLast - GCPtr) >= cbNeedle;
+                        PGMPhysReleasePageMappingLock(pVM, &Lock);
+                        if (fRc)
+                        {
+                            *pGCPtrHit = GCPtr + offHit;
+                            pgmUnlock(pVM);
+                            return VINF_SUCCESS;
+                        }
+                    }
+                    else
+                        cbPrev = 0; /* ignore error. */
+                }
+                else
+                    cbPrev = 0;
+            }
+            else
+                cbPrev = 0;
+        }
+        else
+        {
+            Assert(Walk.enmType != PGMPTWALKGSTTYPE_INVALID);
+            Assert(!Walk.u.Core.fSucceeded);
+            cbPrev = 0; /* ignore error. */
+
+            /*
+             * Try skip as much as possible. No need to figure out that a PDE
+             * is not present 512 times!
+             */
+            uint64_t cPagesCanSkip;
+            switch (Walk.u.Core.uLevel)
+            {
+                case 1:
+                    /* page level, use cIncPages */
+                    cPagesCanSkip = 1;
+                    break;
+                case 2:
+                    if (Walk.enmType == PGMPTWALKGSTTYPE_32BIT)
+                    {
+                        cPagesCanSkip = X86_PG_ENTRIES     - ((GCPtr >> X86_PT_SHIFT)     & X86_PT_MASK);
+                        Assert(!((GCPtr + ((RTGCPTR)cPagesCanSkip << X86_PT_PAE_SHIFT)) & (RT_BIT_64(X86_PD_SHIFT) - 1)));
+                    }
+                    else
+                    {
+                        cPagesCanSkip = X86_PG_PAE_ENTRIES - ((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK);
+                        Assert(!((GCPtr + ((RTGCPTR)cPagesCanSkip << X86_PT_PAE_SHIFT)) & (RT_BIT_64(X86_PD_PAE_SHIFT) - 1)));
+                    }
+                    break;
+                case 3:
+                    cPagesCanSkip = (X86_PG_PAE_ENTRIES - ((GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK)) * X86_PG_PAE_ENTRIES
+                                  - ((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK);
+                    Assert(!((GCPtr + ((RTGCPTR)cPagesCanSkip << X86_PT_PAE_SHIFT)) & (RT_BIT_64(X86_PDPT_SHIFT) - 1)));
+                    break;
+                case 4:
+                    cPagesCanSkip =   (X86_PG_PAE_ENTRIES  - ((GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64))
+                                    * X86_PG_PAE_ENTRIES * X86_PG_PAE_ENTRIES
+                                  - ((((GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK)) * X86_PG_PAE_ENTRIES)
+                                  - ((  GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK);
+                    Assert(!((GCPtr + ((RTGCPTR)cPagesCanSkip << X86_PT_PAE_SHIFT)) & (RT_BIT_64(X86_PML4_SHIFT) - 1)));
+                    break;
+                case 8:
+                    /* The CR3 value is bad, forget the whole search. */
+                    cPagesCanSkip = cPages;
+                    break;
+                default:
+                    AssertMsgFailed(("%d\n", Walk.u.Core.uLevel));
+                    cPagesCanSkip = 0;
+                    break;
+            }
+            if (cPages <= cPagesCanSkip)
+                break;
+            fFullWalk = true;
+            if (cPagesCanSkip >= cIncPages)
+            {
+                cPages -= cPagesCanSkip;
+                GCPtr += (RTGCPTR)cPagesCanSkip << X86_PT_PAE_SHIFT;
+                continue;
+            }
+        }
+
+        /* advance to the next page. */
+        if (cPages <= cIncPages)
+            break;
+        cPages -= cIncPages;
+        GCPtr += (RTGCPTR)cIncPages << X86_PT_PAE_SHIFT;
+
+        /* Yield the PGM lock every now and then. */
+        if (!--cYieldCountDown)
+        {
+            fFullWalk = PDMR3CritSectYield(pVM, &pVM->pgm.s.CritSectX);
+            cYieldCountDown = cYieldCountDownReload;
+        }
+    }
+    pgmUnlock(pVM);
+    return VERR_DBGF_MEM_NOT_FOUND;
+}
+
+
+/**
+ * Initializes the dumper state.
+ *
+ * @param   pState          The state to initialize.
+ * @param   pVM             The cross context VM structure.
+ * @param   fFlags          The flags.
+ * @param   u64FirstAddr    The first address.
+ * @param   u64LastAddr     The last address.
+ * @param   pHlp            The output helpers.
+ */
+static void pgmR3DumpHierarchyInitState(PPGMR3DUMPHIERARCHYSTATE pState, PVM pVM, uint32_t fFlags,
+                                        uint64_t u64FirstAddr, uint64_t u64LastAddr, PCDBGFINFOHLP pHlp)
+{
+    pState->pVM                 = pVM;
+    pState->pHlp                = pHlp ? pHlp : DBGFR3InfoLogHlp();
+    pState->fPse                = !!(fFlags & (DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME));
+    pState->fPae                = !!(fFlags & (DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME));
+    pState->fLme                = !!(fFlags & DBGFPGDMP_FLAGS_LME);
+    pState->fNp                 = !!(fFlags & DBGFPGDMP_FLAGS_NP);
+    pState->fEpt                = !!(fFlags & DBGFPGDMP_FLAGS_EPT);
+    pState->fNxe                = !!(fFlags & DBGFPGDMP_FLAGS_NXE);
+    pState->cchAddress          = pState->fLme ? 16 : 8;
+    pState->uLastRsvdBit        = pState->fNxe ? 62 : 63;
+    pState->fDumpPageInfo       = !!(fFlags & DBGFPGDMP_FLAGS_PAGE_INFO);
+    pState->fPrintHeader        = !!(fFlags & DBGFPGDMP_FLAGS_HEADER);
+    pState->fPrintCr3           = !!(fFlags & DBGFPGDMP_FLAGS_PRINT_CR3);
+    pState->afReserved[0]       = false;
+    pState->afReserved[1]       = false;
+    pState->afReserved[2]       = false;
+    pState->afReserved[3]       = false;
+    pState->afReserved[4]       = false;
+    pState->u64Address          = u64FirstAddr;
+    pState->u64FirstAddress     = u64FirstAddr;
+    pState->u64LastAddress      = u64LastAddr;
+    pState->u64HighReservedBits = pState->uLastRsvdBit == 62 ? UINT64_C(0x7ff) << 52 : UINT64_C(0xfff) << 52;
+    pState->cLeaves             = 0;
+}
+
+
+/**
+ * The simple way out, too tired to think of a more elegant solution.
+ *
+ * @returns The base address of this page table/directory/whatever.
+ * @param   pState              The state where we get the current address.
+ * @param   cShift              The shift count for the table entries.
+ * @param   cEntries            The number of table entries.
+ * @param   piFirst             Where to return the table index of the first
+ *                              entry to dump.
+ * @param   piLast              Where to return the table index of the last
+ *                              entry.
+ */
+static uint64_t pgmR3DumpHierarchyCalcRange(PPGMR3DUMPHIERARCHYSTATE pState, uint32_t cShift, uint32_t cEntries,
+                                            uint32_t *piFirst, uint32_t *piLast)
+{
+    const uint64_t iBase  = (pState->u64Address     >> cShift) & ~(uint64_t)(cEntries - 1);
+    const uint64_t iFirst = pState->u64FirstAddress >> cShift;
+    const uint64_t iLast  = pState->u64LastAddress  >> cShift;
+
+    if (   iBase                >= iFirst
+        && iBase + cEntries - 1 <= iLast)
+    {
+        /* full range. */
+        *piFirst = 0;
+        *piLast  = cEntries - 1;
+    }
+    else if (   iBase + cEntries - 1 < iFirst
+             || iBase                > iLast)
+    {
+        /* no match */
+        *piFirst = cEntries;
+        *piLast  = 0;
+    }
+    else
+    {
+        /* partial overlap */
+        *piFirst = iBase <= iFirst
+                 ? iFirst - iBase
+                 : 0;
+        *piLast  = iBase + cEntries - 1 <= iLast
+                 ? cEntries - 1
+                 : iLast - iBase;
+    }
+
+    return iBase << cShift;
+}
+
+
+/**
+ * Maps/finds the shadow page.
+ *
+ * @returns VBox status code.
+ * @param   pState              The dumper state.
+ * @param   HCPhys              The physical address of the shadow page.
+ * @param   pszDesc             The description.
+ * @param   fIsMapping          Set if it's a mapping.
+ * @param   ppv                 Where to return the pointer.
+ */
+static int pgmR3DumpHierarchyShwMapPage(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, const char *pszDesc,
+                                        bool fIsMapping, void const **ppv)
+{
+    void *pvPage;
+    if (!fIsMapping)
+    {
+        PPGMPOOLPAGE pPoolPage = pgmPoolQueryPageForDbg(pState->pVM->pgm.s.pPoolR3, HCPhys);
+        if (pPoolPage)
+        {
+            pState->pHlp->pfnPrintf(pState->pHlp, "%0*llx error! %s at HCPhys=%RHp was not found in the page pool!\n",
+                                    pState->cchAddress, pState->u64Address, pszDesc, HCPhys);
+            return VERR_PGM_POOL_GET_PAGE_FAILED;
+        }
+        pvPage = (uint8_t *)pPoolPage->pvPageR3 + (HCPhys & PAGE_OFFSET_MASK);
+    }
+    else
+    {
+        pvPage = NULL;
+#ifndef PGM_WITHOUT_MAPPINGS
+        for (PPGMMAPPING pMap = pState->pVM->pgm.s.pMappingsR3; pMap; pMap = pMap->pNextR3)
+        {
+            uint64_t off = pState->u64Address - pMap->GCPtr;
+            if (off < pMap->cb)
+            {
+                const int iPDE = (uint32_t)(off >> X86_PD_SHIFT);
+                const int iSub = (int)((off >> X86_PD_PAE_SHIFT) & 1); /* MSC is a pain sometimes */
+                if ((iSub ? pMap->aPTs[iPDE].HCPhysPaePT1 : pMap->aPTs[iPDE].HCPhysPaePT0) != HCPhys)
+                    pState->pHlp->pfnPrintf(pState->pHlp,
+                                            "%0*llx error! Mapping error! PT %d has HCPhysPT=%RHp not %RHp is in the PD.\n",
+                                            pState->cchAddress, pState->u64Address, iPDE,
+                                            iSub ? pMap->aPTs[iPDE].HCPhysPaePT1 : pMap->aPTs[iPDE].HCPhysPaePT0, HCPhys);
+                pvPage = &pMap->aPTs[iPDE].paPaePTsR3[iSub];
+                break;
+            }
+        }
+#endif /* !PGM_WITHOUT_MAPPINGS */
+        if (!pvPage)
+        {
+            pState->pHlp->pfnPrintf(pState->pHlp, "%0*llx error! PT mapping %s at HCPhys=%RHp was not found in the page pool!\n",
+                                    pState->cchAddress, pState->u64Address, pszDesc, HCPhys);
+            return VERR_INVALID_PARAMETER;
+        }
+    }
+    *ppv = pvPage;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps the a shadow page summary or smth.
+ *
+ * @param   pState              The dumper state.
+ * @param   HCPhys              The page address.
+ */
+static void pgmR3DumpHierarchyShwTablePageInfo(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys)
+{
+    pgmLock(pState->pVM);
+    char            szPage[80];
+    PPGMPOOLPAGE    pPage = pgmPoolQueryPageForDbg(pState->pVM->pgm.s.CTX_SUFF(pPool), HCPhys);
+    if (pPage)
+        RTStrPrintf(szPage, sizeof(szPage), " idx=0i%u", pPage->idx);
+    else
+    {
+        /* probably a mapping */
+        strcpy(szPage, " not found");
+#ifndef PGM_WITHOUT_MAPPINGS
+        for (PPGMMAPPING pMap = pState->pVM->pgm.s.pMappingsR3; pMap; pMap = pMap->pNextR3)
+        {
+            uint64_t off = pState->u64Address - pMap->GCPtr;
+            if (off < pMap->cb)
+            {
+                const int iPDE = (uint32_t)(off >> X86_PD_SHIFT);
+                if (pMap->aPTs[iPDE].HCPhysPT == HCPhys)
+                    RTStrPrintf(szPage, sizeof(szPage), " #%u: %s", iPDE, pMap->pszDesc);
+                else if (pMap->aPTs[iPDE].HCPhysPaePT0 == HCPhys)
+                    RTStrPrintf(szPage, sizeof(szPage), " #%u/0: %s", iPDE, pMap->pszDesc);
+                else if (pMap->aPTs[iPDE].HCPhysPaePT1 == HCPhys)
+                    RTStrPrintf(szPage, sizeof(szPage), " #%u/1: %s", iPDE, pMap->pszDesc);
+                else
+                    continue;
+                break;
+            }
+        }
+#endif /* !PGM_WITHOUT_MAPPINGS */
+    }
+    pgmUnlock(pState->pVM);
+    pState->pHlp->pfnPrintf(pState->pHlp, "%s", szPage);
+}
+
+
+/**
+ * Figures out which guest page this is and dumps a summary.
+ *
+ * @param   pState              The dumper state.
+ * @param   HCPhys              The page address.
+ * @param   cbPage              The page size.
+ */
+static void pgmR3DumpHierarchyShwGuestPageInfo(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, uint32_t cbPage)
+{
+    char        szPage[80];
+    RTGCPHYS    GCPhys;
+    int rc = PGMR3DbgHCPhys2GCPhys(pState->pVM->pUVM, HCPhys, &GCPhys);
+    if (RT_SUCCESS(rc))
+    {
+        pgmLock(pState->pVM);
+        PCPGMPAGE pPage = pgmPhysGetPage(pState->pVM, GCPhys);
+        if (pPage)
+            RTStrPrintf(szPage, sizeof(szPage), "%R[pgmpage]", pPage);
+        else
+            strcpy(szPage, "not found");
+        pgmUnlock(pState->pVM);
+        pState->pHlp->pfnPrintf(pState->pHlp, " -> %RGp %s", GCPhys, szPage);
+    }
+    else
+    {
+#ifndef PGM_WITHOUT_MAPPINGS
+        /* check the heap */
+        uint32_t cbAlloc;
+        rc = MMR3HyperQueryInfoFromHCPhys(pState->pVM, HCPhys, szPage, sizeof(szPage), &cbAlloc);
+        if (RT_SUCCESS(rc))
+            pState->pHlp->pfnPrintf(pState->pHlp, " %s %#x bytes", szPage, cbAlloc);
+        else
+#endif
+            pState->pHlp->pfnPrintf(pState->pHlp, " not found");
+    }
+    NOREF(cbPage);
+}
+
+
+/**
+ * Dumps a PAE shadow page table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState              The dumper state.
+ * @param   HCPhys              The page table address.
+ * @param   fIsMapping          Whether it is a mapping.
+ */
+static int pgmR3DumpHierarchyShwPaePT(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, bool fIsMapping)
+{
+    PCPGMSHWPTPAE pPT;
+    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page table", fIsMapping, (void const **)&pPT);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PT_PAE_SHIFT, X86_PG_PAE_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+        if (PGMSHWPTEPAE_GET_U(pPT->a[i]) & X86_PTE_P)
+        {
+            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PT_PAE_SHIFT);
+            if (PGMSHWPTEPAE_IS_P(pPT->a[i]))
+            {
+                X86PTEPAE Pte;
+                Pte.u = PGMSHWPTEPAE_GET_U(pPT->a[i]);
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme  /*P R  S  A  D  G  WT CD AT NX 4M a p ?  */
+                                        ? "%016llx 3    | P %c %c %c %c %c %s %s %s %s 4K %c%c%c  %016llx"
+                                        :  "%08llx 2   |  P %c %c %c %c %c %s %s %s %s 4K %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pte.n.u1Write       ? 'W'  : 'R',
+                                        Pte.n.u1User        ? 'U'  : 'S',
+                                        Pte.n.u1Accessed    ? 'A'  : '-',
+                                        Pte.n.u1Dirty       ? 'D'  : '-',
+                                        Pte.n.u1Global      ? 'G'  : '-',
+                                        Pte.n.u1WriteThru   ? "WT" : "--",
+                                        Pte.n.u1CacheDisable? "CD" : "--",
+                                        Pte.n.u1PAT         ? "AT" : "--",
+                                        Pte.n.u1NoExecute   ? "NX" : "--",
+                                        Pte.u & PGM_PTFLAGS_TRACK_DIRTY   ? 'd' : '-',
+                                        Pte.u & RT_BIT(10)                ? '1' : '0',
+                                        Pte.u & PGM_PTFLAGS_CSAM_VALIDATED? 'v' : '-',
+                                        Pte.u & X86_PTE_PAE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwGuestPageInfo(pState, Pte.u & X86_PTE_PAE_PG_MASK, _4K);
+                if ((Pte.u >> 52) & 0x7ff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx%s", (Pte.u >> 52) & 0x7ff, pState->fLme ? "" : "!");
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+            }
+            else if (   (PGMSHWPTEPAE_GET_U(pPT->a[i]) & (pState->pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX))
+                     ==                                  (pState->pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX))
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme
+                                        ? "%016llx 3    | invalid / MMIO optimization\n"
+                                        :  "%08llx 2   |  invalid / MMIO optimization\n",
+                                        pState->u64Address);
+            else
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme
+                                        ? "%016llx 3    | invalid: %RX64\n"
+                                        :  "%08llx 2   |  invalid: %RX64\n",
+                                        pState->u64Address, PGMSHWPTEPAE_GET_U(pPT->a[i]));
+            pState->cLeaves++;
+        }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps a PAE shadow page directory table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   HCPhys      The physical address of the page directory table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int  pgmR3DumpHierarchyShwPaePD(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, unsigned cMaxDepth)
+{
+    PCX86PDPAE pPD;
+    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page directory", false, (void const **)&pPD);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth > 0);
+    cMaxDepth--;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PD_PAE_SHIFT, X86_PG_PAE_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PDEPAE Pde = pPD->a[i];
+        if (Pde.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PD_PAE_SHIFT);
+            if (Pde.b.u1Size)
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme    /*P R  S  A  D  G  WT CD AT NX 2M a  p ?  phys*/
+                                        ? "%016llx 2   |  P %c %c %c %c %c %s %s %s %s 2M %c%c%c  %016llx"
+                                        :  "%08llx 1  |   P %c %c %c %c %c %s %s %s %s 2M %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pde.b.u1Write       ? 'W'  : 'R',
+                                        Pde.b.u1User        ? 'U'  : 'S',
+                                        Pde.b.u1Accessed    ? 'A'  : '-',
+                                        Pde.b.u1Dirty       ? 'D'  : '-',
+                                        Pde.b.u1Global      ? 'G'  : '-',
+                                        Pde.b.u1WriteThru   ? "WT" : "--",
+                                        Pde.b.u1CacheDisable? "CD" : "--",
+                                        Pde.b.u1PAT         ? "AT" : "--",
+                                        Pde.b.u1NoExecute   ? "NX" : "--",
+                                        Pde.u & PGM_PDFLAGS_BIG_PAGE    ? 'b' : '-',
+                                        Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
+                                        Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
+                                        Pde.u & X86_PDE2M_PAE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwGuestPageInfo(pState, Pde.u & X86_PDE2M_PAE_PG_MASK, _2M);
+                if ((Pde.u >> 52) & 0x7ff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx%s", (Pde.u >> 52) & 0x7ff, pState->fLme ? "" : "!");
+                if ((Pde.u >> 13) & 0xff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 20:13=%02llx%s", (Pde.u >> 13) & 0x0ff, pState->fLme ? "" : "!");
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+                pState->cLeaves++;
+            }
+            else
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme    /*P R  S  A  D  G  WT CD AT NX 4M a  p ?  phys */
+                                        ? "%016llx 2   |  P %c %c %c %c %c %s %s .. %s .. %c%c%c  %016llx"
+                                        :  "%08llx 1  |   P %c %c %c %c %c %s %s .. %s .. %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pde.n.u1Write       ? 'W'  : 'R',
+                                        Pde.n.u1User        ? 'U'  : 'S',
+                                        Pde.n.u1Accessed    ? 'A'  : '-',
+                                        Pde.n.u1Reserved0   ? '?'  : '.', /* ignored */
+                                        Pde.n.u1Reserved1   ? '?'  : '.', /* ignored */
+                                        Pde.n.u1WriteThru   ? "WT" : "--",
+                                        Pde.n.u1CacheDisable? "CD" : "--",
+                                        Pde.n.u1NoExecute   ? "NX" : "--",
+                                        Pde.u & PGM_PDFLAGS_BIG_PAGE    ? 'b' : '-',
+                                        Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
+                                        Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
+                                        Pde.u & X86_PDE_PAE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwTablePageInfo(pState, Pde.u & X86_PDE_PAE_PG_MASK);
+                if ((Pde.u >> 52) & 0x7ff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx!", (Pde.u >> 52) & 0x7ff);
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+                if (cMaxDepth)
+                {
+                    int rc2 = pgmR3DumpHierarchyShwPaePT(pState, Pde.u & X86_PDE_PAE_PG_MASK, !!(Pde.u & PGM_PDFLAGS_MAPPING));
+                    if (rc2 < rc && RT_SUCCESS(rc))
+                        rc = rc2;
+                }
+                else
+                    pState->cLeaves++;
+            }
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Dumps a PAE shadow page directory pointer table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   HCPhys      The physical address of the page directory pointer table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int  pgmR3DumpHierarchyShwPaePDPT(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, unsigned cMaxDepth)
+{
+    /* Fend of addresses that are out of range in PAE mode - simplifies the code below. */
+    if (!pState->fLme && pState->u64Address >= _4G)
+        return VINF_SUCCESS;
+
+    PCX86PDPT pPDPT;
+    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page directory pointer table", false, (void const **)&pPDPT);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth > 0);
+    cMaxDepth--;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PDPT_SHIFT,
+                                                          pState->fLme ? X86_PG_AMD64_PDPE_ENTRIES : X86_PG_PAE_PDPE_ENTRIES,
+                                                          &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PDPE Pdpe = pPDPT->a[i];
+        if (Pdpe.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PDPT_SHIFT);
+            if (pState->fLme)
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp, /*P R  S  A  D  G  WT CD AT NX .. a p ?  */
+                                        "%016llx 1  |   P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pdpe.lm.u1Write       ? 'W'  : 'R',
+                                        Pdpe.lm.u1User        ? 'U'  : 'S',
+                                        Pdpe.lm.u1Accessed    ? 'A'  : '-',
+                                        Pdpe.lm.u3Reserved & 1? '?'  : '.', /* ignored */
+                                        Pdpe.lm.u3Reserved & 4? '!'  : '.', /* mbz */
+                                        Pdpe.lm.u1WriteThru   ? "WT" : "--",
+                                        Pdpe.lm.u1CacheDisable? "CD" : "--",
+                                        Pdpe.lm.u3Reserved & 2? "!"  : "..",/* mbz */
+                                        Pdpe.lm.u1NoExecute   ? "NX" : "--",
+                                        Pdpe.u & RT_BIT(9)                ? '1' : '0',
+                                        Pdpe.u & PGM_PLXFLAGS_PERMANENT   ? 'p' : '-',
+                                        Pdpe.u & RT_BIT(11)               ? '1' : '0',
+                                        Pdpe.u & X86_PDPE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwTablePageInfo(pState, Pdpe.u & X86_PDPE_PG_MASK);
+                if ((Pdpe.u >> 52) & 0x7ff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx", (Pdpe.u >> 52) & 0x7ff);
+            }
+            else
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX .. a p ?  */
+                                        "%08llx 0 |    P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pdpe.n.u2Reserved & 1? '!'  : '.', /* mbz */
+                                        Pdpe.n.u2Reserved & 2? '!'  : '.', /* mbz */
+                                        Pdpe.n.u4Reserved & 1? '!'  : '.', /* mbz */
+                                        Pdpe.n.u4Reserved & 2? '!'  : '.', /* mbz */
+                                        Pdpe.n.u4Reserved & 8? '!'  : '.', /* mbz */
+                                        Pdpe.n.u1WriteThru   ? "WT" : "--",
+                                        Pdpe.n.u1CacheDisable? "CD" : "--",
+                                        Pdpe.n.u4Reserved & 2? "!"  : "..",/* mbz */
+                                        Pdpe.lm.u1NoExecute  ? "!!"  : "..",/* mbz */
+                                        Pdpe.u & RT_BIT(9)                ? '1' : '0',
+                                        Pdpe.u & PGM_PLXFLAGS_PERMANENT   ? 'p' : '-',
+                                        Pdpe.u & RT_BIT(11)               ? '1' : '0',
+                                        Pdpe.u & X86_PDPE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwTablePageInfo(pState, Pdpe.u & X86_PDPE_PG_MASK);
+                if ((Pdpe.u >> 52) & 0xfff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 63:52=%03llx!", (Pdpe.u >> 52) & 0xfff);
+            }
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+            if (cMaxDepth)
+            {
+                int rc2 = pgmR3DumpHierarchyShwPaePD(pState, Pdpe.u & X86_PDPE_PG_MASK, cMaxDepth);
+                if (rc2 < rc && RT_SUCCESS(rc))
+                    rc = rc2;
+            }
+            else
+                pState->cLeaves++;
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Dumps a 32-bit shadow page table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   HCPhys      The physical address of the table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int pgmR3DumpHierarchyShwPaePML4(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, unsigned cMaxDepth)
+{
+    PCX86PML4 pPML4;
+    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page map level 4", false, (void const **)&pPML4);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth);
+    cMaxDepth--;
+
+    /*
+     * This is a bit tricky as we're working on unsigned addresses while the
+     * AMD64 spec uses signed tricks.
+     */
+    uint32_t iFirst = (pState->u64FirstAddress >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    uint32_t iLast  = (pState->u64LastAddress  >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    if (   pState->u64LastAddress  <= UINT64_C(0x00007fffffffffff)
+        || pState->u64FirstAddress >= UINT64_C(0xffff800000000000))
+    { /* Simple, nothing to adjust */ }
+    else if (pState->u64FirstAddress <= UINT64_C(0x00007fffffffffff))
+        iLast = X86_PG_AMD64_ENTRIES / 2 - 1;
+    else if (pState->u64LastAddress  >= UINT64_C(0xffff800000000000))
+        iFirst = X86_PG_AMD64_ENTRIES / 2;
+    else
+        iFirst = X86_PG_AMD64_ENTRIES; /* neither address is canonical */
+
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PML4E Pml4e = pPML4->a[i];
+        if (Pml4e.n.u1Present)
+        {
+            pState->u64Address = ((uint64_t)i << X86_PML4_SHIFT)
+                               | (i >= RT_ELEMENTS(pPML4->a) / 2 ? UINT64_C(0xffff000000000000) : 0);
+            pState->pHlp->pfnPrintf(pState->pHlp, /*P R  S  A  D  G  WT CD AT NX 4M a p ?  */
+                                    "%016llx 0 |    P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
+                                    pState->u64Address,
+                                    Pml4e.n.u1Write       ? 'W'  : 'R',
+                                    Pml4e.n.u1User        ? 'U'  : 'S',
+                                    Pml4e.n.u1Accessed    ? 'A'  : '-',
+                                    Pml4e.n.u3Reserved & 1? '?'  : '.', /* ignored */
+                                    Pml4e.n.u3Reserved & 4? '!'  : '.', /* mbz */
+                                    Pml4e.n.u1WriteThru   ? "WT" : "--",
+                                    Pml4e.n.u1CacheDisable? "CD" : "--",
+                                    Pml4e.n.u3Reserved & 2? "!"  : "..",/* mbz */
+                                    Pml4e.n.u1NoExecute   ? "NX" : "--",
+                                    Pml4e.u & RT_BIT(9)                ? '1' : '0',
+                                    Pml4e.u & PGM_PLXFLAGS_PERMANENT   ? 'p' : '-',
+                                    Pml4e.u & RT_BIT(11)               ? '1' : '0',
+                                    Pml4e.u & X86_PML4E_PG_MASK);
+            if (pState->fDumpPageInfo)
+                pgmR3DumpHierarchyShwTablePageInfo(pState, Pml4e.u & X86_PML4E_PG_MASK);
+            if ((Pml4e.u >> 52) & 0x7ff)
+                pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx!", (Pml4e.u >> 52) & 0x7ff);
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+            if (cMaxDepth)
+            {
+                int rc2 = pgmR3DumpHierarchyShwPaePDPT(pState, Pml4e.u & X86_PML4E_PG_MASK, cMaxDepth);
+                if (rc2 < rc && RT_SUCCESS(rc))
+                    rc = rc2;
+            }
+            else
+                pState->cLeaves++;
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Dumps a 32-bit shadow page table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   HCPhys      The physical address of the table.
+ * @param   fMapping    Set if it's a guest mapping.
+ */
+static int pgmR3DumpHierarchyShw32BitPT(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, bool fMapping)
+{
+    PCX86PT pPT;
+    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page table", fMapping, (void const **)&pPT);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PT_SHIFT, X86_PG_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PTE Pte = pPT->a[i];
+        if (Pte.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + (i << X86_PT_SHIFT);
+            pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d  */
+                                    "%08llx 1  |   P %c %c %c %c %c %s %s %s .. 4K %c%c%c  %08x",
+                                    pState->u64Address,
+                                    Pte.n.u1Write       ? 'W'  : 'R',
+                                    Pte.n.u1User        ? 'U'  : 'S',
+                                    Pte.n.u1Accessed    ? 'A'  : '-',
+                                    Pte.n.u1Dirty       ? 'D'  : '-',
+                                    Pte.n.u1Global      ? 'G'  : '-',
+                                    Pte.n.u1WriteThru   ? "WT" : "--",
+                                    Pte.n.u1CacheDisable? "CD" : "--",
+                                    Pte.n.u1PAT         ? "AT" : "--",
+                                    Pte.u & PGM_PTFLAGS_TRACK_DIRTY     ? 'd' : '-',
+                                    Pte.u & RT_BIT(10)                  ? '1' : '0',
+                                    Pte.u & PGM_PTFLAGS_CSAM_VALIDATED  ? 'v' : '-',
+                                    Pte.u & X86_PDE_PG_MASK);
+            if (pState->fDumpPageInfo)
+                pgmR3DumpHierarchyShwGuestPageInfo(pState, Pte.u & X86_PDE_PG_MASK, _4K);
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps a 32-bit shadow page directory and page tables.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   HCPhys      The physical address of the table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int pgmR3DumpHierarchyShw32BitPD(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, unsigned cMaxDepth)
+{
+    if (pState->u64Address >= _4G)
+        return VINF_SUCCESS;
+
+    PCX86PD pPD;
+    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page directory", false, (void const **)&pPD);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth > 0);
+    cMaxDepth--;
+
+    uint32_t iFirst, iLast;
+    pgmR3DumpHierarchyCalcRange(pState, X86_PD_SHIFT, X86_PG_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PDE Pde = pPD->a[i];
+        if (Pde.n.u1Present)
+        {
+            pState->u64Address = (uint32_t)i << X86_PD_SHIFT;
+            if (Pde.b.u1Size && pState->fPse)
+            {
+                uint64_t u64Phys = ((uint64_t)(Pde.u & X86_PDE4M_PG_HIGH_MASK) << X86_PDE4M_PG_HIGH_SHIFT)
+                                 | (Pde.u & X86_PDE4M_PG_MASK);
+                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d   phys */
+                                        "%08llx 0 |    P %c %c %c %c %c %s %s %s .. 4M %c%c%c  %08llx",
+                                        pState->u64Address,
+                                        Pde.b.u1Write       ? 'W'  : 'R',
+                                        Pde.b.u1User        ? 'U'  : 'S',
+                                        Pde.b.u1Accessed    ? 'A'  : '-',
+                                        Pde.b.u1Dirty       ? 'D'  : '-',
+                                        Pde.b.u1Global      ? 'G'  : '-',
+                                        Pde.b.u1WriteThru   ? "WT" : "--",
+                                        Pde.b.u1CacheDisable? "CD" : "--",
+                                        Pde.b.u1PAT         ? "AT" : "--",
+                                        Pde.u & PGM_PDFLAGS_BIG_PAGE    ? 'b' : '-',
+                                        Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
+                                        Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
+                                        u64Phys);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwGuestPageInfo(pState, u64Phys, _4M);
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+                pState->cLeaves++;
+            }
+            else
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d   phys */
+                                        "%08llx 0 |    P %c %c %c %c %c %s %s .. .. 4K %c%c%c  %08x",
+                                        pState->u64Address,
+                                        Pde.n.u1Write       ? 'W'  : 'R',
+                                        Pde.n.u1User        ? 'U'  : 'S',
+                                        Pde.n.u1Accessed    ? 'A'  : '-',
+                                        Pde.n.u1Reserved0   ? '?'  : '.', /* ignored */
+                                        Pde.n.u1Reserved1   ? '?'  : '.', /* ignored */
+                                        Pde.n.u1WriteThru   ? "WT" : "--",
+                                        Pde.n.u1CacheDisable? "CD" : "--",
+                                        Pde.u & PGM_PDFLAGS_BIG_PAGE    ? 'b' : '-',
+                                        Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
+                                        Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
+                                        Pde.u & X86_PDE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwTablePageInfo(pState, Pde.u & X86_PDE_PG_MASK);
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+                if (cMaxDepth)
+                {
+                    int rc2 = pgmR3DumpHierarchyShw32BitPT(pState, Pde.u & X86_PDE_PG_MASK, !!(Pde.u & PGM_PDFLAGS_MAPPING));
+                    if (rc2 < rc && RT_SUCCESS(rc))
+                        rc = rc2;
+                }
+                else
+                    pState->cLeaves++;
+            }
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Internal worker that initiates the actual dump.
+ *
+ * @returns VBox status code.
+ * @param   pState              The dumper state.
+ * @param   cr3                 The CR3 value.
+ * @param   cMaxDepth           The max depth.
+ */
+static int pgmR3DumpHierarchyShwDoIt(PPGMR3DUMPHIERARCHYSTATE pState, uint64_t cr3, unsigned cMaxDepth)
+{
+    int             rc;
+    unsigned const  cch     = pState->cchAddress;
+    uint64_t const  cr3Mask = pState->fEpt ? X86_CR3_AMD64_PAGE_MASK
+                            : pState->fLme ? X86_CR3_AMD64_PAGE_MASK
+                            : pState->fPae ? X86_CR3_PAE_PAGE_MASK
+                            :                X86_CR3_PAGE_MASK;
+    if (pState->fPrintCr3)
+    {
+        const char * const  pszMode = pState->fEpt ? "Extended Page Tables"
+                                    : pState->fLme ? "Long Mode"
+                                    : pState->fPae ? "PAE Mode"
+                                    : pState->fPse ? "32-bit w/ PSE"
+                                    :                "32-bit";
+        pState->pHlp->pfnPrintf(pState->pHlp, "cr3=%0*llx", cch, cr3);
+        if (pState->fDumpPageInfo)
+            pgmR3DumpHierarchyShwTablePageInfo(pState, cr3 & X86_CR3_AMD64_PAGE_MASK);
+        pState->pHlp->pfnPrintf(pState->pHlp, " %s%s%s\n",
+                                pszMode,
+                                pState->fNp  ? " + Nested Paging" : "",
+                                pState->fNxe ? " + NX" : "");
+    }
+
+
+    if (pState->fEpt)
+    {
+        if (pState->fPrintHeader)
+            pState->pHlp->pfnPrintf(pState->pHlp,
+                                    "%-*s        R - Readable\n"
+                                    "%-*s        | W - Writeable\n"
+                                    "%-*s        | | X - Executable\n"
+                                    "%-*s        | | | EMT - EPT memory type\n"
+                                    "%-*s        | | | |   PAT - Ignored PAT?\n"
+                                    "%-*s        | | | |   |    AVL1 - 4 available bits\n"
+                                    "%-*s        | | | |   |    |   AVL2 - 12 available bits\n"
+                                    "%-*s Level  | | | |   |    |    |     page  \n"
+                                  /* xxxx n **** R W X EMT PAT AVL1 AVL2   xxxxxxxxxxxxx
+                                                 R W X  7   0   f   fff    0123456701234567 */
+                                    ,
+                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
+
+        pState->pHlp->pfnPrintf(pState->pHlp, "EPT dumping is not yet implemented, sorry.\n");
+        /** @todo implemented EPT dumping. */
+        rc = VERR_NOT_IMPLEMENTED;
+    }
+    else
+    {
+        if (pState->fPrintHeader)
+            pState->pHlp->pfnPrintf(pState->pHlp,
+                                    "%-*s        P - Present\n"
+                                    "%-*s        | R/W - Read (0) / Write (1)\n"
+                                    "%-*s        | | U/S - User (1) / Supervisor (0)\n"
+                                    "%-*s        | | | A - Accessed\n"
+                                    "%-*s        | | | | D - Dirty\n"
+                                    "%-*s        | | | | | G - Global\n"
+                                    "%-*s        | | | | | | WT - Write thru\n"
+                                    "%-*s        | | | | | | |  CD - Cache disable\n"
+                                    "%-*s        | | | | | | |  |  AT - Attribute table (PAT)\n"
+                                    "%-*s        | | | | | | |  |  |  NX - No execute (K8)\n"
+                                    "%-*s        | | | | | | |  |  |  |  4K/4M/2M - Page size.\n"
+                                    "%-*s        | | | | | | |  |  |  |  |  AVL - a=allocated; m=mapping; d=track dirty;\n"
+                                    "%-*s        | | | | | | |  |  |  |  |  |     p=permanent; v=validated;\n"
+                                    "%-*s Level  | | | | | | |  |  |  |  |  |    Page\n"
+                                  /* xxxx n **** P R S A D G WT CD AT NX 4M AVL xxxxxxxxxxxxx
+                                                 - W U - - - -- -- -- -- -- 010 */
+                                    ,
+                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "",
+                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
+        if (pState->fLme)
+            rc = pgmR3DumpHierarchyShwPaePML4(pState, cr3 & cr3Mask, cMaxDepth);
+        else if (pState->fPae)
+            rc = pgmR3DumpHierarchyShwPaePDPT(pState, cr3 & cr3Mask, cMaxDepth);
+        else
+            rc = pgmR3DumpHierarchyShw32BitPD(pState, cr3 & cr3Mask, cMaxDepth);
+    }
+
+    if (!pState->cLeaves)
+        pState->pHlp->pfnPrintf(pState->pHlp, "not present\n");
+    return rc;
+}
+
+
+/**
+ * dbgfR3PagingDumpEx worker.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   cr3             The CR3 register value.
+ * @param   fFlags          The flags, DBGFPGDMP_FLAGS_XXX.
+ * @param   u64FirstAddr    The start address.
+ * @param   u64LastAddr     The address to stop after.
+ * @param   cMaxDepth       The max depth.
+ * @param   pHlp            The output callbacks.  Defaults to log if NULL.
+ *
+ * @internal
+ */
+VMMR3_INT_DECL(int) PGMR3DumpHierarchyShw(PVM pVM, uint64_t cr3, uint32_t fFlags, uint64_t u64FirstAddr, uint64_t u64LastAddr,
+                                          uint32_t cMaxDepth, PCDBGFINFOHLP pHlp)
+{
+    /* Minimal validation as we're only supposed to service DBGF. */
+    AssertReturn(~(fFlags & ~DBGFPGDMP_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(!(fFlags & (DBGFPGDMP_FLAGS_CURRENT_MODE | DBGFPGDMP_FLAGS_CURRENT_CR3)), VERR_INVALID_PARAMETER);
+    AssertReturn(fFlags & DBGFPGDMP_FLAGS_SHADOW, VERR_INVALID_PARAMETER);
+
+    PGMR3DUMPHIERARCHYSTATE State;
+    pgmR3DumpHierarchyInitState(&State, pVM, fFlags, u64FirstAddr, u64LastAddr, pHlp);
+    return pgmR3DumpHierarchyShwDoIt(&State, cr3, cMaxDepth);
+}
+
+
+/**
+ * Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pVM         The cross context VM structure.
+ * @param   cr3         The root of the hierarchy.
+ * @param   cr4         The cr4, only PAE and PSE is currently used.
+ * @param   fLongMode   Set if long mode, false if not long mode.
+ * @param   cMaxDepth   Number of levels to dump.
+ * @param   pHlp        Pointer to the output functions.
+ *
+ * @deprecated Use DBGFR3PagingDumpEx.
+ */
+VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint64_t cr3, uint64_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
+{
+    if (!cMaxDepth)
+        return VINF_SUCCESS;
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    uint32_t fFlags = DBGFPGDMP_FLAGS_HEADER | DBGFPGDMP_FLAGS_PRINT_CR3 | DBGFPGDMP_FLAGS_PAGE_INFO | DBGFPGDMP_FLAGS_SHADOW;
+    fFlags |= cr4 & (X86_CR4_PAE | X86_CR4_PSE);
+    if (fLongMode)
+        fFlags |= DBGFPGDMP_FLAGS_LME;
+
+    return DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, fFlags, cr3, 0, fLongMode ? UINT64_MAX : UINT32_MAX, cMaxDepth, pHlp);
+}
+
+
+/**
+ * Maps the guest page.
+ *
+ * @returns VBox status code.
+ * @param   pState              The dumper state.
+ * @param   GCPhys              The physical address of the guest page.
+ * @param   pszDesc             The description.
+ * @param   ppv                 Where to return the pointer.
+ * @param   pLock               Where to return the mapping lock.  Hand this to
+ *                              PGMPhysReleasePageMappingLock when done.
+ */
+static int pgmR3DumpHierarchyGstMapPage(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys, const char *pszDesc,
+                                        void const **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    int rc = PGMPhysGCPhys2CCPtrReadOnly(pState->pVM, GCPhys, ppv, pLock);
+    if (RT_FAILURE(rc))
+    {
+        pState->pHlp->pfnPrintf(pState->pHlp, "%0*llx error! Failed to map %s at GCPhys=%RGp: %Rrc!\n",
+                                pState->cchAddress, pState->u64Address, pszDesc, GCPhys, rc);
+        return rc;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Figures out which guest page this is and dumps a summary.
+ *
+ * @param   pState              The dumper state.
+ * @param   GCPhys              The page address.
+ * @param   cbPage              The page size.
+ */
+static void pgmR3DumpHierarchyGstPageInfo(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys, uint32_t cbPage)
+{
+    char szPage[80];
+    pgmLock(pState->pVM);
+    PCPGMPAGE pPage = pgmPhysGetPage(pState->pVM, GCPhys);
+    if (pPage)
+        RTStrPrintf(szPage, sizeof(szPage), " %R[pgmpage]", pPage);
+    else
+        strcpy(szPage, " not found");
+    pgmUnlock(pState->pVM);
+    pState->pHlp->pfnPrintf(pState->pHlp, "%s", szPage);
+    NOREF(cbPage);
+}
+
+
+/**
+ * Checks the entry for reserved bits.
+ *
+ * @param   pState              The dumper state.
+ * @param   u64Entry            The entry to check.
+ */
+static void pgmR3DumpHierarchyGstCheckReservedHighBits(PPGMR3DUMPHIERARCHYSTATE pState, uint64_t u64Entry)
+{
+    uint32_t uRsvd = (u64Entry & pState->u64HighReservedBits) >> 52;
+    if (uRsvd)
+        pState->pHlp->pfnPrintf(pState->pHlp, " %u:52=%03x%s",
+                                pState->uLastRsvdBit, uRsvd, pState->fLme ? "" : "!");
+    /** @todo check the valid physical bits as well. */
+}
+
+
+/**
+ * Dumps a PAE shadow page table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState              The dumper state.
+ * @param   GCPhys              The page table address.
+ */
+static int pgmR3DumpHierarchyGstPaePT(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys)
+{
+    PCX86PTPAE      pPT;
+    PGMPAGEMAPLOCK  Lock;
+    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page table", (void const **)&pPT, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PT_PAE_SHIFT, X86_PG_PAE_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PTEPAE Pte = pPT->a[i];
+        if (Pte.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PT_PAE_SHIFT);
+            pState->pHlp->pfnPrintf(pState->pHlp,
+                                    pState->fLme  /*P R  S  A  D  G  WT CD AT NX 4M a p ?  */
+                                    ? "%016llx 3    | P %c %c %c %c %c %s %s %s %s 4K %c%c%c  %016llx"
+                                    :  "%08llx 2   |  P %c %c %c %c %c %s %s %s %s 4K %c%c%c  %016llx",
+                                    pState->u64Address,
+                                    Pte.n.u1Write       ? 'W'  : 'R',
+                                    Pte.n.u1User        ? 'U'  : 'S',
+                                    Pte.n.u1Accessed    ? 'A'  : '-',
+                                    Pte.n.u1Dirty       ? 'D'  : '-',
+                                    Pte.n.u1Global      ? 'G'  : '-',
+                                    Pte.n.u1WriteThru   ? "WT" : "--",
+                                    Pte.n.u1CacheDisable? "CD" : "--",
+                                    Pte.n.u1PAT         ? "AT" : "--",
+                                    Pte.n.u1NoExecute   ? "NX" : "--",
+                                    Pte.u & RT_BIT(9)   ? '1' : '0',
+                                    Pte.u & RT_BIT(10)  ? '1' : '0',
+                                    Pte.u & RT_BIT(11)  ? '1' : '0',
+                                    Pte.u & X86_PTE_PAE_PG_MASK);
+            if (pState->fDumpPageInfo)
+                pgmR3DumpHierarchyGstPageInfo(pState, Pte.u & X86_PTE_PAE_PG_MASK, _4K);
+            pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pte.u);
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+            pState->cLeaves++;
+        }
+    }
+
+    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps a PAE shadow page directory table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   GCPhys      The physical address of the table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int  pgmR3DumpHierarchyGstPaePD(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys, unsigned cMaxDepth)
+{
+    PCX86PDPAE      pPD;
+    PGMPAGEMAPLOCK  Lock;
+    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page directory", (void const **)&pPD, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth > 0);
+    cMaxDepth--;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PD_PAE_SHIFT, X86_PG_PAE_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PDEPAE Pde = pPD->a[i];
+        if (Pde.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PD_PAE_SHIFT);
+            if (Pde.b.u1Size)
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme    /*P R  S  A  D  G  WT CD AT NX 2M a  p ?  phys*/
+                                        ? "%016llx 2   |  P %c %c %c %c %c %s %s %s %s 2M %c%c%c  %016llx"
+                                        :  "%08llx 1  |   P %c %c %c %c %c %s %s %s %s 2M %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pde.b.u1Write         ? 'W'  : 'R',
+                                        Pde.b.u1User          ? 'U'  : 'S',
+                                        Pde.b.u1Accessed      ? 'A'  : '-',
+                                        Pde.b.u1Dirty         ? 'D'  : '-',
+                                        Pde.b.u1Global        ? 'G'  : '-',
+                                        Pde.b.u1WriteThru     ? "WT" : "--",
+                                        Pde.b.u1CacheDisable  ? "CD" : "--",
+                                        Pde.b.u1PAT           ? "AT" : "--",
+                                        Pde.b.u1NoExecute     ? "NX" : "--",
+                                        Pde.u & RT_BIT_64(9)  ? '1' : '0',
+                                        Pde.u & RT_BIT_64(10) ? '1' : '0',
+                                        Pde.u & RT_BIT_64(11) ? '1' : '0',
+                                        Pde.u & X86_PDE2M_PAE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyGstPageInfo(pState, Pde.u & X86_PDE2M_PAE_PG_MASK, _2M);
+                pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pde.u);
+                if ((Pde.u >> 13) & 0xff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 20:13=%02llx%s", (Pde.u >> 13) & 0x0ff, pState->fLme ? "" : "!");
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+                pState->cLeaves++;
+            }
+            else
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme    /*P R  S  A  D  G  WT CD AT NX 4M a  p ?  phys */
+                                        ? "%016llx 2   |  P %c %c %c %c %c %s %s .. %s .. %c%c%c  %016llx"
+                                        :  "%08llx 1  |   P %c %c %c %c %c %s %s .. %s .. %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pde.n.u1Write         ? 'W'  : 'R',
+                                        Pde.n.u1User          ? 'U'  : 'S',
+                                        Pde.n.u1Accessed      ? 'A'  : '-',
+                                        Pde.n.u1Reserved0     ? '?'  : '.', /* ignored */
+                                        Pde.n.u1Reserved1     ? '?'  : '.', /* ignored */
+                                        Pde.n.u1WriteThru     ? "WT" : "--",
+                                        Pde.n.u1CacheDisable  ? "CD" : "--",
+                                        Pde.n.u1NoExecute     ? "NX" : "--",
+                                        Pde.u & RT_BIT_64(9)  ? '1' : '0',
+                                        Pde.u & RT_BIT_64(10) ? '1' : '0',
+                                        Pde.u & RT_BIT_64(11) ? '1' : '0',
+                                        Pde.u & X86_PDE_PAE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyGstPageInfo(pState, Pde.u & X86_PDE_PAE_PG_MASK, _4K);
+                pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pde.u);
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+                if (cMaxDepth)
+                {
+                    int rc2 = pgmR3DumpHierarchyGstPaePT(pState, Pde.u & X86_PDE_PAE_PG_MASK);
+                    if (rc2 < rc && RT_SUCCESS(rc))
+                        rc = rc2;
+                }
+                else
+                    pState->cLeaves++;
+            }
+        }
+    }
+
+    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
+    return rc;
+}
+
+
+/**
+ * Dumps a PAE shadow page directory pointer table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   GCPhys      The physical address of the table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int  pgmR3DumpHierarchyGstPaePDPT(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys, unsigned cMaxDepth)
+{
+    /* Fend of addresses that are out of range in PAE mode - simplifies the code below. */
+    if (!pState->fLme && pState->u64Address >= _4G)
+        return VINF_SUCCESS;
+
+    PCX86PDPT       pPDPT;
+    PGMPAGEMAPLOCK  Lock;
+    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page directory pointer table", (void const **)&pPDPT, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth > 0);
+    cMaxDepth--;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PDPT_SHIFT,
+                                                          pState->fLme ? X86_PG_AMD64_PDPE_ENTRIES : X86_PG_PAE_PDPE_ENTRIES,
+                                                          &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PDPE Pdpe = pPDPT->a[i];
+        if (Pdpe.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PDPT_SHIFT);
+            if (pState->fLme)
+            {
+                /** @todo Do 1G pages.  */
+                pState->pHlp->pfnPrintf(pState->pHlp, /*P R  S  A  D  G  WT CD AT NX .. a p ?  */
+                                        "%016llx 1  |   P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pdpe.lm.u1Write        ? 'W'  : 'R',
+                                        Pdpe.lm.u1User         ? 'U'  : 'S',
+                                        Pdpe.lm.u1Accessed     ? 'A'  : '-',
+                                        Pdpe.lm.u3Reserved & 1 ? '?'  : '.', /* ignored */
+                                        Pdpe.lm.u3Reserved & 4 ? '!'  : '.', /* mbz */
+                                        Pdpe.lm.u1WriteThru    ? "WT" : "--",
+                                        Pdpe.lm.u1CacheDisable ? "CD" : "--",
+                                        Pdpe.lm.u3Reserved & 2 ? "!"  : "..",/* mbz */
+                                        Pdpe.lm.u1NoExecute    ? "NX" : "--",
+                                        Pdpe.u & RT_BIT_64(9)  ? '1' : '0',
+                                        Pdpe.u & RT_BIT_64(10) ? '1' : '0',
+                                        Pdpe.u & RT_BIT_64(11) ? '1' : '0',
+                                        Pdpe.u & X86_PDPE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyGstPageInfo(pState, Pdpe.u & X86_PDPE_PG_MASK, _4K);
+                pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pdpe.u);
+            }
+            else
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX .. a p ?  */
+                                        "%08llx 0 |    P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pdpe.n.u2Reserved & 1  ? '!'  : '.',  /* mbz */
+                                        Pdpe.n.u2Reserved & 2  ? '!'  : '.',  /* mbz */
+                                        Pdpe.n.u4Reserved & 1  ? '!'  : '.',  /* mbz */
+                                        Pdpe.n.u4Reserved & 2  ? '!'  : '.',  /* mbz */
+                                        Pdpe.n.u4Reserved & 8  ? '!'  : '.',  /* mbz */
+                                        Pdpe.n.u1WriteThru     ? "WT" : "--",
+                                        Pdpe.n.u1CacheDisable  ? "CD" : "--",
+                                        Pdpe.n.u4Reserved & 2  ? "!"  : "..", /* mbz */
+                                        Pdpe.lm.u1NoExecute    ? "!!"  : "..",/* mbz */
+                                        Pdpe.u & RT_BIT_64(9)  ? '1' : '0',
+                                        Pdpe.u & RT_BIT_64(10) ? '1' : '0',
+                                        Pdpe.u & RT_BIT_64(11) ? '1' : '0',
+                                        Pdpe.u & X86_PDPE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyGstPageInfo(pState, Pdpe.u & X86_PDPE_PG_MASK, _4K);
+                pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pdpe.u);
+            }
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+            if (cMaxDepth)
+            {
+                int rc2 = pgmR3DumpHierarchyGstPaePD(pState, Pdpe.u & X86_PDPE_PG_MASK, cMaxDepth);
+                if (rc2 < rc && RT_SUCCESS(rc))
+                    rc = rc2;
+            }
+            else
+                pState->cLeaves++;
+        }
+    }
+
+    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
+    return rc;
+}
+
+
+/**
+ * Dumps a 32-bit shadow page table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   GCPhys      The physical address of the table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int pgmR3DumpHierarchyGstPaePML4(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS GCPhys, unsigned cMaxDepth)
+{
+    PCX86PML4       pPML4;
+    PGMPAGEMAPLOCK  Lock;
+    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page map level 4", (void const **)&pPML4, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth);
+    cMaxDepth--;
+
+    /*
+     * This is a bit tricky as we're working on unsigned addresses while the
+     * AMD64 spec uses signed tricks.
+     */
+    uint32_t iFirst = (pState->u64FirstAddress >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    uint32_t iLast  = (pState->u64LastAddress  >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    if (   pState->u64LastAddress  <= UINT64_C(0x00007fffffffffff)
+        || pState->u64FirstAddress >= UINT64_C(0xffff800000000000))
+    { /* Simple, nothing to adjust */ }
+    else if (pState->u64FirstAddress <= UINT64_C(0x00007fffffffffff))
+        iLast = X86_PG_AMD64_ENTRIES / 2 - 1;
+    else if (pState->u64LastAddress  >= UINT64_C(0xffff800000000000))
+        iFirst = X86_PG_AMD64_ENTRIES / 2;
+    else
+        iFirst = X86_PG_AMD64_ENTRIES; /* neither address is canonical */
+
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PML4E Pml4e = pPML4->a[i];
+        if (Pml4e.n.u1Present)
+        {
+            pState->u64Address = ((uint64_t)i << X86_PML4_SHIFT)
+                               | (i >= RT_ELEMENTS(pPML4->a) / 2 ? UINT64_C(0xffff000000000000) : 0);
+            pState->pHlp->pfnPrintf(pState->pHlp, /*P R  S  A  D  G  WT CD AT NX 4M a p ?  */
+                                    "%016llx 0 |    P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
+                                    pState->u64Address,
+                                    Pml4e.n.u1Write         ? 'W'  : 'R',
+                                    Pml4e.n.u1User          ? 'U'  : 'S',
+                                    Pml4e.n.u1Accessed      ? 'A'  : '-',
+                                    Pml4e.n.u3Reserved & 1  ? '?'  : '.', /* ignored */
+                                    Pml4e.n.u3Reserved & 4  ? '!'  : '.', /* mbz */
+                                    Pml4e.n.u1WriteThru     ? "WT" : "--",
+                                    Pml4e.n.u1CacheDisable  ? "CD" : "--",
+                                    Pml4e.n.u3Reserved & 2  ? "!"  : "..",/* mbz */
+                                    Pml4e.n.u1NoExecute     ? "NX" : "--",
+                                    Pml4e.u & RT_BIT_64(9)  ? '1' : '0',
+                                    Pml4e.u & RT_BIT_64(10) ? '1' : '0',
+                                    Pml4e.u & RT_BIT_64(11) ? '1' : '0',
+                                    Pml4e.u & X86_PML4E_PG_MASK);
+            if (pState->fDumpPageInfo)
+                pgmR3DumpHierarchyGstPageInfo(pState, Pml4e.u & X86_PML4E_PG_MASK, _4K);
+            pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pml4e.u);
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+            if (cMaxDepth)
+            {
+                int rc2 = pgmR3DumpHierarchyGstPaePDPT(pState, Pml4e.u & X86_PML4E_PG_MASK, cMaxDepth);
+                if (rc2 < rc && RT_SUCCESS(rc))
+                    rc = rc2;
+            }
+            else
+                pState->cLeaves++;
+        }
+    }
+
+    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
+    return rc;
+}
+
+
+/**
+ * Dumps a 32-bit shadow page table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   GCPhys      The physical address of the table.
+ */
+static int pgmR3DumpHierarchyGst32BitPT(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS GCPhys)
+{
+    PCX86PT         pPT;
+    PGMPAGEMAPLOCK  Lock;
+    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page table", (void const **)&pPT, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PT_SHIFT, X86_PG_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PTE Pte = pPT->a[i];
+        if (Pte.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + (i << X86_PT_SHIFT);
+            pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d  */
+                                    "%08llx 1  |   P %c %c %c %c %c %s %s %s .. 4K %c%c%c  %08x",
+                                    pState->u64Address,
+                                    Pte.n.u1Write         ? 'W'  : 'R',
+                                    Pte.n.u1User          ? 'U'  : 'S',
+                                    Pte.n.u1Accessed      ? 'A'  : '-',
+                                    Pte.n.u1Dirty         ? 'D'  : '-',
+                                    Pte.n.u1Global        ? 'G'  : '-',
+                                    Pte.n.u1WriteThru     ? "WT" : "--",
+                                    Pte.n.u1CacheDisable  ? "CD" : "--",
+                                    Pte.n.u1PAT           ? "AT" : "--",
+                                    Pte.u & RT_BIT_32(9)  ? '1' : '0',
+                                    Pte.u & RT_BIT_32(10) ? '1' : '0',
+                                    Pte.u & RT_BIT_32(11) ? '1' : '0',
+                                    Pte.u & X86_PDE_PG_MASK);
+            if (pState->fDumpPageInfo)
+                pgmR3DumpHierarchyGstPageInfo(pState, Pte.u & X86_PDE_PG_MASK, _4K);
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+        }
+    }
+
+    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps a 32-bit shadow page directory and page tables.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   GCPhys      The physical address of the table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int pgmR3DumpHierarchyGst32BitPD(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS GCPhys, unsigned cMaxDepth)
+{
+    if (pState->u64Address >= _4G)
+        return VINF_SUCCESS;
+
+    PCX86PD         pPD;
+    PGMPAGEMAPLOCK  Lock;
+    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page directory", (void const **)&pPD, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth > 0);
+    cMaxDepth--;
+
+    uint32_t iFirst, iLast;
+    pgmR3DumpHierarchyCalcRange(pState, X86_PD_SHIFT, X86_PG_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PDE Pde = pPD->a[i];
+        if (Pde.n.u1Present)
+        {
+            pState->u64Address = (uint32_t)i << X86_PD_SHIFT;
+            if (Pde.b.u1Size && pState->fPse)
+            {
+                uint64_t u64Phys = ((uint64_t)(Pde.u & X86_PDE4M_PG_HIGH_MASK) << X86_PDE4M_PG_HIGH_SHIFT)
+                                 | (Pde.u & X86_PDE4M_PG_MASK);
+                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d   phys */
+                                        "%08llx 0 |    P %c %c %c %c %c %s %s %s .. 4M %c%c%c  %08llx",
+                                        pState->u64Address,
+                                        Pde.b.u1Write         ? 'W'  : 'R',
+                                        Pde.b.u1User          ? 'U'  : 'S',
+                                        Pde.b.u1Accessed      ? 'A'  : '-',
+                                        Pde.b.u1Dirty         ? 'D'  : '-',
+                                        Pde.b.u1Global        ? 'G'  : '-',
+                                        Pde.b.u1WriteThru     ? "WT" : "--",
+                                        Pde.b.u1CacheDisable  ? "CD" : "--",
+                                        Pde.b.u1PAT           ? "AT" : "--",
+                                        Pde.u & RT_BIT_32(9)  ? '1' : '0',
+                                        Pde.u & RT_BIT_32(10) ? '1' : '0',
+                                        Pde.u & RT_BIT_32(11) ? '1' : '0',
+                                        u64Phys);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyGstPageInfo(pState, u64Phys, _4M);
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+                pState->cLeaves++;
+            }
+            else
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d   phys */
+                                        "%08llx 0 |    P %c %c %c %c %c %s %s .. .. .. %c%c%c  %08x",
+                                        pState->u64Address,
+                                        Pde.n.u1Write         ? 'W'  : 'R',
+                                        Pde.n.u1User          ? 'U'  : 'S',
+                                        Pde.n.u1Accessed      ? 'A'  : '-',
+                                        Pde.n.u1Reserved0     ? '?'  : '.', /* ignored */
+                                        Pde.n.u1Reserved1     ? '?'  : '.', /* ignored */
+                                        Pde.n.u1WriteThru     ? "WT" : "--",
+                                        Pde.n.u1CacheDisable  ? "CD" : "--",
+                                        Pde.u & RT_BIT_32(9)  ? '1' : '0',
+                                        Pde.u & RT_BIT_32(10) ? '1' : '0',
+                                        Pde.u & RT_BIT_32(11) ? '1' : '0',
+                                        Pde.u & X86_PDE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyGstPageInfo(pState, Pde.u & X86_PDE_PG_MASK, _4K);
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+                if (cMaxDepth)
+                {
+                    int rc2 = pgmR3DumpHierarchyGst32BitPT(pState, Pde.u & X86_PDE_PG_MASK);
+                    if (rc2 < rc && RT_SUCCESS(rc))
+                        rc = rc2;
+                }
+                else
+                    pState->cLeaves++;
+            }
+        }
+    }
+
+    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
+    return rc;
+}
+
+
+/**
+ * Internal worker that initiates the actual dump.
+ *
+ * @returns VBox status code.
+ * @param   pState              The dumper state.
+ * @param   cr3                 The CR3 value.
+ * @param   cMaxDepth           The max depth.
+ */
+static int pgmR3DumpHierarchyGstDoIt(PPGMR3DUMPHIERARCHYSTATE pState, uint64_t cr3, unsigned cMaxDepth)
+{
+    int             rc;
+    unsigned const  cch     = pState->cchAddress;
+    uint64_t const  cr3Mask = pState->fEpt ? X86_CR3_AMD64_PAGE_MASK
+                            : pState->fLme ? X86_CR3_AMD64_PAGE_MASK
+                            : pState->fPae ? X86_CR3_PAE_PAGE_MASK
+                            :                X86_CR3_PAGE_MASK;
+    if (pState->fPrintCr3)
+    {
+        const char * const  pszMode = pState->fEpt ? "Extended Page Tables"
+                                    : pState->fLme ? "Long Mode"
+                                    : pState->fPae ? "PAE Mode"
+                                    : pState->fPse ? "32-bit w/ PSE"
+                                    :                "32-bit";
+        pState->pHlp->pfnPrintf(pState->pHlp, "cr3=%0*llx", cch, cr3);
+        if (pState->fDumpPageInfo)
+            pgmR3DumpHierarchyGstPageInfo(pState, cr3 & X86_CR3_AMD64_PAGE_MASK, _4K);
+        pState->pHlp->pfnPrintf(pState->pHlp, " %s%s%s\n",
+                                pszMode,
+                                pState->fNp  ? " + Nested Paging" : "",
+                                pState->fNxe ? " + NX" : "");
+    }
+
+
+    if (pState->fEpt)
+    {
+        if (pState->fPrintHeader)
+            pState->pHlp->pfnPrintf(pState->pHlp,
+                                    "%-*s        R - Readable\n"
+                                    "%-*s        | W - Writeable\n"
+                                    "%-*s        | | X - Executable\n"
+                                    "%-*s        | | | EMT - EPT memory type\n"
+                                    "%-*s        | | | |   PAT - Ignored PAT?\n"
+                                    "%-*s        | | | |   |    AVL1 - 4 available bits\n"
+                                    "%-*s        | | | |   |    |   AVL2 - 12 available bits\n"
+                                    "%-*s Level  | | | |   |    |    |     page  \n"
+                                  /* xxxx n **** R W X EMT PAT AVL1 AVL2   xxxxxxxxxxxxx
+                                                 R W X  7   0   f   fff    0123456701234567 */
+                                    ,
+                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
+
+        pState->pHlp->pfnPrintf(pState->pHlp, "EPT dumping is not yet implemented, sorry.\n");
+        /** @todo implemented EPT dumping. */
+        rc = VERR_NOT_IMPLEMENTED;
+    }
+    else
+    {
+        if (pState->fPrintHeader)
+            pState->pHlp->pfnPrintf(pState->pHlp,
+                                    "%-*s        P - Present\n"
+                                    "%-*s        | R/W - Read (0) / Write (1)\n"
+                                    "%-*s        | | U/S - User (1) / Supervisor (0)\n"
+                                    "%-*s        | | | A - Accessed\n"
+                                    "%-*s        | | | | D - Dirty\n"
+                                    "%-*s        | | | | | G - Global\n"
+                                    "%-*s        | | | | | | WT - Write thru\n"
+                                    "%-*s        | | | | | | |  CD - Cache disable\n"
+                                    "%-*s        | | | | | | |  |  AT - Attribute table (PAT)\n"
+                                    "%-*s        | | | | | | |  |  |  NX - No execute (K8)\n"
+                                    "%-*s        | | | | | | |  |  |  |  4K/4M/2M - Page size.\n"
+                                    "%-*s        | | | | | | |  |  |  |  |  AVL - 3 available bits.\n"
+                                    "%-*s Level  | | | | | | |  |  |  |  |  |    Page\n"
+                                  /* xxxx n **** P R S A D G WT CD AT NX 4M AVL xxxxxxxxxxxxx
+                                                 - W U - - - -- -- -- -- -- 010 */
+                                    ,
+                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "",
+                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
+        if (pState->fLme)
+            rc = pgmR3DumpHierarchyGstPaePML4(pState, cr3 & cr3Mask, cMaxDepth);
+        else if (pState->fPae)
+            rc = pgmR3DumpHierarchyGstPaePDPT(pState, cr3 & cr3Mask, cMaxDepth);
+        else
+            rc = pgmR3DumpHierarchyGst32BitPD(pState, cr3 & cr3Mask, cMaxDepth);
+    }
+
+    if (!pState->cLeaves)
+        pState->pHlp->pfnPrintf(pState->pHlp, "not present\n");
+    return rc;
+}
+
+
+/**
+ * dbgfR3PagingDumpEx worker.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   cr3             The CR3 register value.
+ * @param   fFlags          The flags, DBGFPGDMP_FLAGS_XXX.
+ * @param   FirstAddr       The start address.
+ * @param   LastAddr        The address to stop after.
+ * @param   cMaxDepth       The max depth.
+ * @param   pHlp            The output callbacks.  Defaults to log if NULL.
+ *
+ * @internal
+ */
+VMMR3_INT_DECL(int) PGMR3DumpHierarchyGst(PVM pVM, uint64_t cr3, uint32_t fFlags, RTGCPTR FirstAddr, RTGCPTR LastAddr,
+                                          uint32_t cMaxDepth, PCDBGFINFOHLP pHlp)
+{
+    /* Minimal validation as we're only supposed to service DBGF. */
+    AssertReturn(~(fFlags & ~DBGFPGDMP_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(!(fFlags & (DBGFPGDMP_FLAGS_CURRENT_MODE | DBGFPGDMP_FLAGS_CURRENT_CR3)), VERR_INVALID_PARAMETER);
+    AssertReturn(fFlags & DBGFPGDMP_FLAGS_GUEST, VERR_INVALID_PARAMETER);
+
+    PGMR3DUMPHIERARCHYSTATE State;
+    pgmR3DumpHierarchyInitState(&State, pVM, fFlags, FirstAddr, LastAddr, pHlp);
+    return pgmR3DumpHierarchyGstDoIt(&State, cr3, cMaxDepth);
+}
+
+
+/**
+ * For aiding with reset problems and similar.
+ *
+ * @param   pVM                 The cross context VM handle.
+ */
+void pgmLogState(PVM pVM)
+{
+#if 0
+    RTLogRelPrintf("\npgmLogState pgmLogState pgmLogState pgmLogState pgmLogState\n");
+
+    /*
+     * Per CPU stuff.
+     */
+    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
+    {
+        PPGMCPU pPgmCpu = &pVM->aCpus[iCpu].pgm.s;
+        RTLogRelPrintf("pgmLogState: CPU #%u\n", iCpu);
+# define LOG_PGMCPU_MEMBER(aFmt, aMember) RTLogRelPrintf(" %32s: %" aFmt "\n", #aMember, pPgmCpu->aMember)
+        LOG_PGMCPU_MEMBER("#RX32",  offVM);
+        LOG_PGMCPU_MEMBER("#RX32",  offVCpu);
+        LOG_PGMCPU_MEMBER("#RX32",  offPGM);
+        LOG_PGMCPU_MEMBER("RGp",    GCPhysA20Mask);
+        LOG_PGMCPU_MEMBER("RTbool", fA20Enabled);
+        LOG_PGMCPU_MEMBER("RTbool", fNoExecuteEnabled);
+        LOG_PGMCPU_MEMBER("#RX32",  fSyncFlags);
+        LOG_PGMCPU_MEMBER("d",      enmShadowMode);
+        LOG_PGMCPU_MEMBER("d",      enmGuestMode);
+        LOG_PGMCPU_MEMBER("RGp",    GCPhysCR3);
+
+        LOG_PGMCPU_MEMBER("p",      pGst32BitPdR3);
+# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+        LOG_PGMCPU_MEMBER("p",      pGst32BitPdR0);
+# endif
+        LOG_PGMCPU_MEMBER("RRv",    pGst32BitPdRC);
+        LOG_PGMCPU_MEMBER("#RX32",  fGst32BitMbzBigPdeMask);
+        LOG_PGMCPU_MEMBER("RTbool", fGst32BitPageSizeExtension);
+
+        LOG_PGMCPU_MEMBER("p",      pGstPaePdptR3);
+# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+        LOG_PGMCPU_MEMBER("p",      pGstPaePdptR0);
+# endif
+        LOG_PGMCPU_MEMBER("RRv",    pGstPaePdptRC);
+        LOG_PGMCPU_MEMBER("p",      apGstPaePDsR3[0]);
+        LOG_PGMCPU_MEMBER("p",      apGstPaePDsR3[1]);
+        LOG_PGMCPU_MEMBER("p",      apGstPaePDsR3[2]);
+        LOG_PGMCPU_MEMBER("p",      apGstPaePDsR3[3]);
+# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+        LOG_PGMCPU_MEMBER("p",      apGstPaePDsR0[0]);
+        LOG_PGMCPU_MEMBER("p",      apGstPaePDsR0[1]);
+        LOG_PGMCPU_MEMBER("p",      apGstPaePDsR0[2]);
+        LOG_PGMCPU_MEMBER("p",      apGstPaePDsR0[3]);
+# endif
+        LOG_PGMCPU_MEMBER("RRv",    apGstPaePDsR0[0]);
+        LOG_PGMCPU_MEMBER("RRv",    apGstPaePDsR0[1]);
+        LOG_PGMCPU_MEMBER("RRv",    apGstPaePDsR0[2]);
+        LOG_PGMCPU_MEMBER("RRv",    apGstPaePDsR0[3]);
+        LOG_PGMCPU_MEMBER("RGp",    aGCPhysGstPaePDs[0]);
+        LOG_PGMCPU_MEMBER("RGp",    aGCPhysGstPaePDs[1]);
+        LOG_PGMCPU_MEMBER("RGp",    aGCPhysGstPaePDs[2]);
+        LOG_PGMCPU_MEMBER("RGp",    aGCPhysGstPaePDs[3]);
+        LOG_PGMCPU_MEMBER("#RX64",  aGstPaePdpeRegs[0].u);
+        LOG_PGMCPU_MEMBER("#RX64",  aGstPaePdpeRegs[1].u);
+        LOG_PGMCPU_MEMBER("#RX64",  aGstPaePdpeRegs[2].u);
+        LOG_PGMCPU_MEMBER("#RX64",  aGstPaePdpeRegs[3].u);
+        LOG_PGMCPU_MEMBER("RGp",    aGCPhysGstPaePDsMonitored[0]);
+        LOG_PGMCPU_MEMBER("RGp",    aGCPhysGstPaePDsMonitored[1]);
+        LOG_PGMCPU_MEMBER("RGp",    aGCPhysGstPaePDsMonitored[2]);
+        LOG_PGMCPU_MEMBER("RGp",    aGCPhysGstPaePDsMonitored[3]);
+        LOG_PGMCPU_MEMBER("#RX64",  fGstPaeMbzPteMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGstPaeMbzPdeMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGstPaeMbzBigPdeMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGstPaeMbzBigPdeMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGstPaeMbzPdpeMask);
+
+        LOG_PGMCPU_MEMBER("p",      pGstAmd64Pml4R3);
+# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+        LOG_PGMCPU_MEMBER("p",      pGstAmd64Pml4R0);
+# endif
+        LOG_PGMCPU_MEMBER("#RX64",  fGstAmd64MbzPteMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGstAmd64MbzPdeMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGstAmd64MbzBigPdeMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGstAmd64MbzPdpeMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGstAmd64MbzBigPdpeMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGstAmd64MbzPml4eMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGstAmd64ShadowedPdpeMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGstAmd64ShadowedPml4eMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGst64ShadowedPteMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGst64ShadowedPdeMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGst64ShadowedBigPdeMask);
+        LOG_PGMCPU_MEMBER("#RX64",  fGst64ShadowedBigPde4PteMask);
+
+        LOG_PGMCPU_MEMBER("p",      pShwPageCR3R3);
+        LOG_PGMCPU_MEMBER("p",      pShwPageCR3R0);
+        LOG_PGMCPU_MEMBER("RRv",    pShwPageCR3RC);
+
+        LOG_PGMCPU_MEMBER("p",      pfnR3ShwRelocate);
+        LOG_PGMCPU_MEMBER("p",      pfnR3ShwExit);
+        LOG_PGMCPU_MEMBER("p",      pfnR3ShwGetPage);
+        LOG_PGMCPU_MEMBER("p",      pfnR3ShwModifyPage);
+        LOG_PGMCPU_MEMBER("p",      pfnR0ShwGetPage);
+        LOG_PGMCPU_MEMBER("p",      pfnR0ShwModifyPage);
+        LOG_PGMCPU_MEMBER("p",      pfnR3GstRelocate);
+        LOG_PGMCPU_MEMBER("p",      pfnR3GstExit);
+        LOG_PGMCPU_MEMBER("p",      pfnR3GstGetPage);
+        LOG_PGMCPU_MEMBER("p",      pfnR3GstModifyPage);
+        LOG_PGMCPU_MEMBER("p",      pfnR0GstGetPage);
+        LOG_PGMCPU_MEMBER("p",      pfnR0GstModifyPage);
+        LOG_PGMCPU_MEMBER("p",      pfnR3BthRelocate);
+        LOG_PGMCPU_MEMBER("p",      pfnR3BthInvalidatePage);
+        LOG_PGMCPU_MEMBER("p",      pfnR3BthSyncCR3);
+        LOG_PGMCPU_MEMBER("p",      pfnR3BthPrefetchPage);
+        LOG_PGMCPU_MEMBER("p",      pfnR3BthMapCR3);
+        LOG_PGMCPU_MEMBER("p",      pfnR3BthUnmapCR3);
+        LOG_PGMCPU_MEMBER("p",      pfnR0BthMapCR3);
+        LOG_PGMCPU_MEMBER("p",      pfnR0BthUnmapCR3);
+        LOG_PGMCPU_MEMBER("#RX64",  cNetwareWp0Hacks);
+        LOG_PGMCPU_MEMBER("#RX64",  cPoolAccessHandler);
+
+    }
+
+    /*
+     * PGM globals.
+     */
+    RTLogRelPrintf("PGM globals\n");
+    PPGM pPgm = &pVM->pgm.s;
+# define LOG_PGM_MEMBER(aFmt, aMember) RTLogRelPrintf(" %32s: %" aFmt "\n", #aMember, pPgm->aMember)
+    LOG_PGM_MEMBER("#RX32",         offVM);
+    LOG_PGM_MEMBER("#RX32",         offVCpuPGM);
+    LOG_PGM_MEMBER("RTbool",        fRamPreAlloc);
+    LOG_PGM_MEMBER("RTbool",        fPhysWriteMonitoringEngaged);
+    LOG_PGM_MEMBER("RTbool",        fLessThan52PhysicalAddressBits);
+    LOG_PGM_MEMBER("RTbool",        fNestedPaging);
+    LOG_PGM_MEMBER("d",             enmHostMode);
+    LOG_PGM_MEMBER("RTbool",        fNoMorePhysWrites);
+    LOG_PGM_MEMBER("RTbool",        fPageFusionAllowed);
+    LOG_PGM_MEMBER("RTbool",        fPciPassthrough);
+    LOG_PGM_MEMBER("#x",            cMmio2Regions);
+    LOG_PGM_MEMBER("RTbool",        fRestoreRomPagesOnReset);
+    LOG_PGM_MEMBER("RTbool",        fZeroRamPagesOnReset);
+    LOG_PGM_MEMBER("RTbool",        fFinalizedMappings);
+    LOG_PGM_MEMBER("RTbool",        fMappingsFixed);
+    LOG_PGM_MEMBER("RTbool",        fMappingsFixedRestored);
+    LOG_PGM_MEMBER("%#x",           cbMappingFixed);
+    LOG_PGM_MEMBER("%#x",           idRamRangesGen);
+    LOG_PGM_MEMBER("#RGv",          GCPtrMappingFixed);
+    LOG_PGM_MEMBER("#RGv",          GCPtrPrevRamRangeMapping);
+    LOG_PGM_MEMBER("%#x",           hRomPhysHandlerType);
+    LOG_PGM_MEMBER("#RGp",          GCPhys4MBPSEMask);
+    LOG_PGM_MEMBER("#RGp",          GCPhysInvAddrMask);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR3[0]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR3[1]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR3[2]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR3[3]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR3[4]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR3[5]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR3[6]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR3[7]);
+    LOG_PGM_MEMBER("p",             pRamRangesXR3);
+    LOG_PGM_MEMBER("p",             pRamRangeTreeR3);
+    LOG_PGM_MEMBER("p",             pTreesR3);
+    LOG_PGM_MEMBER("p",             pLastPhysHandlerR3);
+    LOG_PGM_MEMBER("p",             pPoolR3);
+    LOG_PGM_MEMBER("p",             pMappingsR3);
+    LOG_PGM_MEMBER("p",             pRomRangesR3);
+    LOG_PGM_MEMBER("p",             pRegMmioRangesR3);
+    LOG_PGM_MEMBER("p",             paModeData);
+    LOG_PGM_MEMBER("p",             apMmio2RangesR3[0]);
+    LOG_PGM_MEMBER("p",             apMmio2RangesR3[1]);
+    LOG_PGM_MEMBER("p",             apMmio2RangesR3[2]);
+    LOG_PGM_MEMBER("p",             apMmio2RangesR3[3]);
+    LOG_PGM_MEMBER("p",             apMmio2RangesR3[4]);
+    LOG_PGM_MEMBER("p",             apMmio2RangesR3[5]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR0[0]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR0[1]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR0[2]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR0[3]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR0[4]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR0[5]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR0[6]);
+    LOG_PGM_MEMBER("p",             apRamRangesTlbR0[7]);
+    LOG_PGM_MEMBER("p",             pRamRangesXR0);
+    LOG_PGM_MEMBER("p",             pRamRangeTreeR0);
+    LOG_PGM_MEMBER("p",             pTreesR0);
+    LOG_PGM_MEMBER("p",             pLastPhysHandlerR0);
+    LOG_PGM_MEMBER("p",             pPoolR0);
+    LOG_PGM_MEMBER("p",             pMappingsR0);
+    LOG_PGM_MEMBER("p",             pRomRangesR0);
+    LOG_PGM_MEMBER("p",             apMmio2RangesR0[0]);
+    LOG_PGM_MEMBER("p",             apMmio2RangesR0[1]);
+    LOG_PGM_MEMBER("p",             apMmio2RangesR0[2]);
+    LOG_PGM_MEMBER("p",             apMmio2RangesR0[3]);
+    LOG_PGM_MEMBER("p",             apMmio2RangesR0[4]);
+    LOG_PGM_MEMBER("p",             apMmio2RangesR0[5]);
+    LOG_PGM_MEMBER("RRv",           apRamRangesTlbRC[0]);
+    LOG_PGM_MEMBER("RRv",           apRamRangesTlbRC[1]);
+    LOG_PGM_MEMBER("RRv",           apRamRangesTlbRC[2]);
+    LOG_PGM_MEMBER("RRv",           apRamRangesTlbRC[3]);
+    LOG_PGM_MEMBER("RRv",           apRamRangesTlbRC[4]);
+    LOG_PGM_MEMBER("RRv",           apRamRangesTlbRC[5]);
+    LOG_PGM_MEMBER("RRv",           apRamRangesTlbRC[6]);
+    LOG_PGM_MEMBER("RRv",           apRamRangesTlbRC[7]);
+    LOG_PGM_MEMBER("RRv",           pRamRangesXRC);
+    LOG_PGM_MEMBER("RRv",           pRamRangeTreeRC);
+    LOG_PGM_MEMBER("RRv",           pTreesRC);
+    LOG_PGM_MEMBER("RRv",           pLastPhysHandlerRC);
+    LOG_PGM_MEMBER("RRv",           pPoolRC);
+    LOG_PGM_MEMBER("RRv",           pMappingsRC);
+    LOG_PGM_MEMBER("RRv",           pRomRangesRC);
+    LOG_PGM_MEMBER("RRv",           paDynPageMap32BitPTEsGC);
+    LOG_PGM_MEMBER("RRv",           paDynPageMapPaePTEsGC);
+
+    LOG_PGM_MEMBER("#RGv",          GCPtrCR3Mapping);
+    LOG_PGM_MEMBER("p",             pInterPD);
+    LOG_PGM_MEMBER("p",             apInterPTs[0]);
+    LOG_PGM_MEMBER("p",             apInterPTs[1]);
+    LOG_PGM_MEMBER("p",             apInterPaePTs[0]);
+    LOG_PGM_MEMBER("p",             apInterPaePTs[1]);
+    LOG_PGM_MEMBER("p",             apInterPaePDs[0]);
+    LOG_PGM_MEMBER("p",             apInterPaePDs[1]);
+    LOG_PGM_MEMBER("p",             apInterPaePDs[2]);
+    LOG_PGM_MEMBER("p",             apInterPaePDs[3]);
+    LOG_PGM_MEMBER("p",             pInterPaePDPT);
+    LOG_PGM_MEMBER("p",             pInterPaePML4);
+    LOG_PGM_MEMBER("p",             pInterPaePDPT64);
+    LOG_PGM_MEMBER("#RHp",          HCPhysInterPD);
+    LOG_PGM_MEMBER("#RHp",          HCPhysInterPaePDPT);
+    LOG_PGM_MEMBER("#RHp",          HCPhysInterPaePML4);
+    LOG_PGM_MEMBER("RRv",           pbDynPageMapBaseGC);
+    LOG_PGM_MEMBER("RRv",           pRCDynMap);
+    LOG_PGM_MEMBER("p",             pvR0DynMapUsed);
+    LOG_PGM_MEMBER("%#x",           cDeprecatedPageLocks);
+
+    /**
+     * Data associated with managing the ring-3 mappings of the allocation chunks.
+     */
+    LOG_PGM_MEMBER("p",             ChunkR3Map.pTree);
+    //LOG_PGM_MEMBER(PGMCHUNKR3MAPTLB ChunkR3Map.Tlb);
+    LOG_PGM_MEMBER("%#x",           ChunkR3Map.c);
+    LOG_PGM_MEMBER("%#x",           ChunkR3Map.cMax);
+    LOG_PGM_MEMBER("%#x",           ChunkR3Map.iNow);
+    //LOG_PGM_MEMBER(PGMPAGER3MAPTLB  PhysTlbHC);
+
+    LOG_PGM_MEMBER("#RHp",          HCPhysZeroPg);
+    LOG_PGM_MEMBER("p",             pvZeroPgR3);
+    LOG_PGM_MEMBER("p",             pvZeroPgR0);
+    LOG_PGM_MEMBER("RRv",           pvZeroPgRC);
+    LOG_PGM_MEMBER("#RHp",          HCPhysMmioPg);
+    LOG_PGM_MEMBER("#RHp",          HCPhysInvMmioPg);
+    LOG_PGM_MEMBER("p",             pvMmioPgR3);
+    LOG_PGM_MEMBER("RTbool",        fErrInjHandyPages);
+
+    /*
+     * PGM page pool.
+     */
+    PPGMPOOL pPool = pVM->pgm.s.pPoolR3;
+    RTLogRelPrintf("PGM Page Pool\n");
+# define LOG_PGMPOOL_MEMBER(aFmt, aMember) RTLogRelPrintf(" %32s: %" aFmt "\n", #aMember, pPool->aMember)
+    LOG_PGMPOOL_MEMBER("p",         pVMR3);
+    LOG_PGMPOOL_MEMBER("p",         pVMR0);
+    LOG_PGMPOOL_MEMBER("RRv",       pVMRC);
+    LOG_PGMPOOL_MEMBER("#x",        cMaxPages);
+    LOG_PGMPOOL_MEMBER("#x",        cCurPages);
+    LOG_PGMPOOL_MEMBER("#x",        iFreeHead);
+    LOG_PGMPOOL_MEMBER("#x",        u16Padding);
+    LOG_PGMPOOL_MEMBER("#x",        iUserFreeHead);
+    LOG_PGMPOOL_MEMBER("#x",        cMaxUsers);
+    LOG_PGMPOOL_MEMBER("#x",        cPresent);
+    LOG_PGMPOOL_MEMBER("RRv",       paUsersRC);
+    LOG_PGMPOOL_MEMBER("p",         paUsersR3);
+    LOG_PGMPOOL_MEMBER("p",         paUsersR0);
+    LOG_PGMPOOL_MEMBER("#x",        iPhysExtFreeHead);
+    LOG_PGMPOOL_MEMBER("#x",        cMaxPhysExts);
+    LOG_PGMPOOL_MEMBER("RRv",       paPhysExtsRC);
+    LOG_PGMPOOL_MEMBER("p",         paPhysExtsR3);
+    LOG_PGMPOOL_MEMBER("p",         paPhysExtsR0);
+    for (uint32_t i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
+        RTLogRelPrintf(" aiHash[%u]: %#x\n", i, pPool->aiHash[i]);
+    LOG_PGMPOOL_MEMBER("#x",        iAgeHead);
+    LOG_PGMPOOL_MEMBER("#x",        iAgeTail);
+    LOG_PGMPOOL_MEMBER("RTbool",    fCacheEnabled);
+    LOG_PGMPOOL_MEMBER("RTbool",    afPadding1[0]);
+    LOG_PGMPOOL_MEMBER("RTbool",    afPadding1[1]);
+    LOG_PGMPOOL_MEMBER("RTbool",    afPadding1[2]);
+    LOG_PGMPOOL_MEMBER("#x",        iModifiedHead);
+    LOG_PGMPOOL_MEMBER("#x",        cModifiedPages);
+    LOG_PGMPOOL_MEMBER("#x",        hAccessHandlerType);
+    LOG_PGMPOOL_MEMBER("#x",        idxFreeDirtyPage);
+    LOG_PGMPOOL_MEMBER("#x",        cDirtyPages);
+    for (uint32_t i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+        RTLogRelPrintf(" aDirtyPages[%u].uIdx: %#x\n", i, pPool->aDirtyPages[i].uIdx);
+    LOG_PGMPOOL_MEMBER("#x",        cUsedPages);
+    LOG_PGMPOOL_MEMBER("#x",        HCPhysTree);
+    for (uint32_t i = 0; i < pPool->cCurPages; i++)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[i];
+# define LOG_PAGE_MEMBER(aFmt, aMember) RTLogRelPrintf(" %3u:%-32s: %" aFmt "\n", i, #aMember, pPage->aMember)
+        RTLogRelPrintf("%3u:%-32s: %p\n", i, "", pPage);
+        LOG_PAGE_MEMBER("RHp",      Core.Key);
+        LOG_PAGE_MEMBER("p",        pvPageR3);
+        LOG_PAGE_MEMBER("RGp",      GCPhys);
+        LOG_PAGE_MEMBER("d",        enmKind);
+        LOG_PAGE_MEMBER("d",        enmAccess);
+        LOG_PAGE_MEMBER("RTbool",   fA20Enabled);
+        LOG_PAGE_MEMBER("RTbool",   fZeroed);
+        LOG_PAGE_MEMBER("RTbool",   fSeenNonGlobal);
+        LOG_PAGE_MEMBER("RTbool",   fMonitored);
+        LOG_PAGE_MEMBER("RTbool",   fCached);
+        LOG_PAGE_MEMBER("RTbool",   fReusedFlushPending);
+        LOG_PAGE_MEMBER("RTbool",   fDirty);
+        LOG_PAGE_MEMBER("RTbool",   fPadding1);
+        LOG_PAGE_MEMBER("RTbool",   fPadding2);
+        LOG_PAGE_MEMBER("#x",       idx);
+        LOG_PAGE_MEMBER("#x",       iNext);
+        LOG_PAGE_MEMBER("#x",       iUserHead);
+        LOG_PAGE_MEMBER("#x",       cPresent);
+        LOG_PAGE_MEMBER("#x",       iFirstPresent);
+        LOG_PAGE_MEMBER("#x",       cModifications);
+        LOG_PAGE_MEMBER("#x",       iModifiedNext);
+        LOG_PAGE_MEMBER("#x",       iModifiedPrev);
+        LOG_PAGE_MEMBER("#x",       iMonitoredNext);
+        LOG_PAGE_MEMBER("#x",       iMonitoredPrev);
+        LOG_PAGE_MEMBER("#x",       iAgeNext);
+        LOG_PAGE_MEMBER("#x",       iAgePrev);
+        LOG_PAGE_MEMBER("#x",       idxDirtyEntry);
+        LOG_PAGE_MEMBER("RGv",      GCPtrLastAccessHandlerRip);
+        LOG_PAGE_MEMBER("RGv",      GCPtrLastAccessHandlerFault);
+        LOG_PAGE_MEMBER("#RX64",    cLastAccessHandler);
+        LOG_PAGE_MEMBER("#RX32",    cLocked);
+# ifdef VBOX_STRICT
+        LOG_PAGE_MEMBER("RGv",      GCPtrDirtyFault);
+# endif
+        if (   pPage->enmKind == PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
+            || pPage->enmKind == PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
+            || pPage->enmKind == PGMPOOLKIND_32BIT_PD
+            || pPage->enmKind == PGMPOOLKIND_32BIT_PD_PHYS)
+        {
+            uint32_t const *pu32Page = (uint32_t const *)pPage->pvPageR3;
+            for (uint32_t i = 0; i < 1024/2; i += 4)
+                RTLogRelPrintf(" %#05x: %RX32 %RX32 %RX32 %RX32\n", i, pu32Page[i], pu32Page[i+1], pu32Page[i+2], pu32Page[i+3]);
+        }
+        else if (   pPage->enmKind != PGMPOOLKIND_FREE
+                 && pPage->enmKind != PGMPOOLKIND_INVALID)
+        {
+            uint64_t const *pu64Page = (uint64_t const *)pPage->pvPageR3;
+            for (uint32_t i = 0; i < 512/2; i += 2)
+                RTLogRelPrintf(" %#05x: %RX64 %RX64\n", i, pu64Page[i], pu64Page[i+1]);
+        }
+    }
+
+    RTLogRelPrintf("pgmLogState pgmLogState pgmLogState pgmLogState pgmLogState\n\n");
+#else
+    RT_NOREF(pVM);
+#endif
+}
+
diff --git a/src/VBox/VMM/VMMR3/PGMHandler.cpp b/src/VBox/VMM/VMMR3/PGMHandler.cpp
new file mode 100644
index 00000000..1be93d91
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PGMHandler.cpp
@@ -0,0 +1,395 @@
+/* $Id: PGMHandler.cpp $ */
+/** @file
+ * PGM - Page Manager / Monitor, Access Handlers.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/ssm.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vm.h>
+#include "PGMInline.h"
+#include <VBox/dbg.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/asm.h>
+#include <iprt/errcore.h>
+#include <iprt/thread.h>
+#include <iprt/string.h>
+#include <VBox/param.h>
+#include <VBox/vmm/hm.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(int) pgmR3HandlerPhysicalOneClear(PAVLROGCPHYSNODECORE pNode, void *pvUser);
+static DECLCALLBACK(int) pgmR3HandlerPhysicalOneSet(PAVLROGCPHYSNODECORE pNode, void *pvUser);
+static DECLCALLBACK(int) pgmR3InfoHandlersPhysicalOne(PAVLROGCPHYSNODECORE pNode, void *pvUser);
+
+
+
+
+/**
+ * Register a physical page access handler type, extended version.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   enmKind         The kind of access handler.
+ * @param   pfnHandlerR3    Pointer to the ring-3 handler callback.
+ * @param   pfnHandlerR0    Pointer to the ring-0 handler callback.
+ * @param   pfnPfHandlerR0  Pointer to the ring-0 \#PF handler callback.
+ *                          callback.
+ * @param   pszDesc         The type description.
+ * @param   phType          Where to return the type handle (cross context
+ *                          safe).
+ */
+VMMR3_INT_DECL(int) PGMR3HandlerPhysicalTypeRegisterEx(PVM pVM, PGMPHYSHANDLERKIND enmKind,
+                                                       PFNPGMPHYSHANDLER pfnHandlerR3,
+                                                       R0PTRTYPE(PFNPGMPHYSHANDLER) pfnHandlerR0,
+                                                       R0PTRTYPE(PFNPGMRZPHYSPFHANDLER) pfnPfHandlerR0,
+                                                       const char *pszDesc, PPGMPHYSHANDLERTYPE phType)
+{
+    AssertPtrReturn(pfnHandlerR3, VERR_INVALID_POINTER);
+    AssertReturn(pfnHandlerR0   != NIL_RTR0PTR, VERR_INVALID_POINTER);
+    AssertReturn(pfnPfHandlerR0 != NIL_RTR0PTR, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertReturn(   enmKind == PGMPHYSHANDLERKIND_WRITE
+                 || enmKind == PGMPHYSHANDLERKIND_ALL
+                 || enmKind == PGMPHYSHANDLERKIND_MMIO,
+                 VERR_INVALID_PARAMETER);
+
+    PPGMPHYSHANDLERTYPEINT pType;
+    int rc = MMHyperAlloc(pVM, sizeof(*pType), 0, MM_TAG_PGM_HANDLER_TYPES, (void **)&pType);
+    if (RT_SUCCESS(rc))
+    {
+        pType->u32Magic         = PGMPHYSHANDLERTYPEINT_MAGIC;
+        pType->cRefs            = 1;
+        pType->enmKind          = enmKind;
+        pType->uState           = enmKind == PGMPHYSHANDLERKIND_WRITE
+                                ? PGM_PAGE_HNDL_PHYS_STATE_WRITE : PGM_PAGE_HNDL_PHYS_STATE_ALL;
+        pType->pfnHandlerR3     = pfnHandlerR3;
+        pType->pfnHandlerR0     = pfnHandlerR0;
+        pType->pfnPfHandlerR0   = pfnPfHandlerR0;
+        pType->pszDesc          = pszDesc;
+
+        pgmLock(pVM);
+        RTListOff32Append(&pVM->pgm.s.CTX_SUFF(pTrees)->HeadPhysHandlerTypes, &pType->ListNode);
+        pgmUnlock(pVM);
+
+        *phType = MMHyperHeapPtrToOffset(pVM, pType);
+        LogFlow(("PGMR3HandlerPhysicalTypeRegisterEx: %p/%#x: enmKind=%d pfnHandlerR3=%RHv pfnHandlerR0=%RHv pszDesc=%s\n",
+                 pType, *phType, enmKind, pfnHandlerR3, pfnPfHandlerR0, pszDesc));
+        return VINF_SUCCESS;
+    }
+    *phType = NIL_PGMPHYSHANDLERTYPE;
+    return rc;
+}
+
+
+/**
+ * Register a physical page access handler type.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   enmKind         The kind of access handler.
+ * @param   pfnHandlerR3    Pointer to the ring-3 handler callback.
+ * @param   pszModR0        The name of the ring-0 module, NULL is an alias for
+ *                          the main ring-0 module.
+ * @param   pszHandlerR0    The name of the ring-0 handler, NULL if the ring-3
+ *                          handler should be called.
+ * @param   pszPfHandlerR0  The name of the ring-0 \#PF handler, NULL if the
+ *                          ring-3 handler should be called.
+ * @param   pszModRC        The name of the raw-mode context module, NULL is an
+ *                          alias for the main RC module.
+ * @param   pszHandlerRC    The name of the raw-mode context handler, NULL if
+ *                          the ring-3 handler should be called.
+ * @param   pszPfHandlerRC  The name of the raw-mode context \#PF handler, NULL
+ *                          if the ring-3 handler should be called.
+ * @param   pszDesc         The type description.
+ * @param   phType          Where to return the type handle (cross context
+ *                          safe).
+ */
+VMMR3DECL(int) PGMR3HandlerPhysicalTypeRegister(PVM pVM, PGMPHYSHANDLERKIND enmKind,
+                                                R3PTRTYPE(PFNPGMPHYSHANDLER) pfnHandlerR3,
+                                                const char *pszModR0, const char *pszHandlerR0, const char *pszPfHandlerR0,
+                                                const char *pszModRC, const char *pszHandlerRC, const char *pszPfHandlerRC,
+                                                const char *pszDesc, PPGMPHYSHANDLERTYPE phType)
+{
+    LogFlow(("PGMR3HandlerPhysicalTypeRegister: enmKind=%d pfnHandlerR3=%RHv pszModR0=%s pszHandlerR0=%s pszPfHandlerR0=%s pszModRC=%s pszHandlerRC=%s pszPfHandlerRC=%s pszDesc=%s\n",
+             enmKind, pfnHandlerR3, pszModR0, pszHandlerR0, pszPfHandlerR0, pszModRC, pszHandlerRC, pszPfHandlerRC, pszDesc));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnHandlerR3, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pszModR0, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pszHandlerR0, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pszPfHandlerR0, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pszModRC, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pszHandlerRC, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pszPfHandlerRC, VERR_INVALID_POINTER);
+
+    /*
+     * Resolve the R0 handlers.
+     */
+    R0PTRTYPE(PFNPGMPHYSHANDLER) pfnHandlerR0 = NIL_RTR0PTR;
+    int rc = PDMR3LdrGetSymbolR0Lazy(pVM, pszHandlerR0 ? pszModR0 : NULL, NULL /*pszSearchPath*/,
+                                     pszHandlerR0 ? pszHandlerR0 : "pgmPhysHandlerRedirectToHC", &pfnHandlerR0);
+    if (RT_SUCCESS(rc))
+    {
+        R0PTRTYPE(PFNPGMR0PHYSPFHANDLER) pfnPfHandlerR0 = NIL_RTR0PTR;
+        rc = PDMR3LdrGetSymbolR0Lazy(pVM, pszPfHandlerR0 ? pszModR0 : NULL, NULL /*pszSearchPath*/,
+                                     pszPfHandlerR0 ? pszPfHandlerR0 : "pgmPhysPfHandlerRedirectToHC", &pfnPfHandlerR0);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Resolve the GC handler.
+             */
+            RTRCPTR pfnHandlerRC   = NIL_RTRCPTR;
+            RTRCPTR pfnPfHandlerRC = NIL_RTRCPTR;
+            if (VM_IS_RAW_MODE_ENABLED(pVM))
+            {
+                rc = PDMR3LdrGetSymbolRCLazy(pVM, pszHandlerRC ? pszModRC : NULL, NULL /*pszSearchPath*/,
+                                             pszHandlerRC ? pszHandlerRC : "pgmPhysHandlerRedirectToHC", &pfnHandlerRC);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = PDMR3LdrGetSymbolRCLazy(pVM, pszPfHandlerRC ? pszModRC : NULL, NULL /*pszSearchPath*/,
+                                                 pszPfHandlerRC ? pszPfHandlerRC : "pgmPhysPfHandlerRedirectToHC", &pfnPfHandlerRC);
+                    AssertMsgRC(rc, ("Failed to resolve %s.%s, rc=%Rrc.\n", pszPfHandlerRC ? pszModRC : VMMRC_MAIN_MODULE_NAME,
+                                     pszPfHandlerRC ? pszPfHandlerRC : "pgmPhysPfHandlerRedirectToHC", rc));
+                }
+                else
+                    AssertMsgFailed(("Failed to resolve %s.%s, rc=%Rrc.\n", pszHandlerRC ? pszModRC : VMMRC_MAIN_MODULE_NAME,
+                                     pszHandlerRC ? pszHandlerRC : "pgmPhysHandlerRedirectToHC", rc));
+
+            }
+            if (RT_SUCCESS(rc))
+                return PGMR3HandlerPhysicalTypeRegisterEx(pVM, enmKind, pfnHandlerR3,
+                                                          pfnHandlerR0, pfnPfHandlerR0, pszDesc, phType);
+        }
+        else
+            AssertMsgFailed(("Failed to resolve %s.%s, rc=%Rrc.\n", pszPfHandlerR0 ? pszModR0 : VMMR0_MAIN_MODULE_NAME,
+                             pszPfHandlerR0 ? pszPfHandlerR0 : "pgmPhysHandlerRedirectToHC", rc));
+    }
+    else
+        AssertMsgFailed(("Failed to resolve %s.%s, rc=%Rrc.\n", pszHandlerR0 ? pszModR0 : VMMR0_MAIN_MODULE_NAME,
+                         pszHandlerR0 ? pszHandlerR0 : "pgmPhysHandlerRedirectToHC", rc));
+
+    return rc;
+}
+
+
+/**
+ * Updates the physical page access handlers.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @remark  Only used when restoring a saved state.
+ */
+void pgmR3HandlerPhysicalUpdateAll(PVM pVM)
+{
+    LogFlow(("pgmHandlerPhysicalUpdateAll:\n"));
+
+    /*
+     * Clear and set.
+     * (the right -> left on the setting pass is just bird speculating on cache hits)
+     */
+    pgmLock(pVM);
+    RTAvlroGCPhysDoWithAll(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,  true, pgmR3HandlerPhysicalOneClear, pVM);
+    RTAvlroGCPhysDoWithAll(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, false, pgmR3HandlerPhysicalOneSet, pVM);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Clears all the page level flags for one physical handler range.
+ *
+ * @returns 0
+ * @param   pNode   Pointer to a PGMPHYSHANDLER.
+ * @param   pvUser  Pointer to the VM.
+ */
+static DECLCALLBACK(int) pgmR3HandlerPhysicalOneClear(PAVLROGCPHYSNODECORE pNode, void *pvUser)
+{
+    PPGMPHYSHANDLER pCur     = (PPGMPHYSHANDLER)pNode;
+    PPGMRAMRANGE    pRamHint = NULL;
+    RTGCPHYS        GCPhys   = pCur->Core.Key;
+    RTUINT          cPages   = pCur->cPages;
+    PVM             pVM      = (PVM)pvUser;
+    for (;;)
+    {
+        PPGMPAGE pPage;
+        int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
+        if (RT_SUCCESS(rc))
+        {
+            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, PGM_PAGE_HNDL_PHYS_STATE_NONE);
+
+            /* Tell NEM about the protection change. */
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+                NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
+                                               pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+        }
+        else
+            AssertRC(rc);
+
+        if (--cPages == 0)
+            return 0;
+        GCPhys += PAGE_SIZE;
+    }
+}
+
+
+/**
+ * Sets all the page level flags for one physical handler range.
+ *
+ * @returns 0
+ * @param   pNode   Pointer to a PGMPHYSHANDLER.
+ * @param   pvUser  Pointer to the VM.
+ */
+static DECLCALLBACK(int) pgmR3HandlerPhysicalOneSet(PAVLROGCPHYSNODECORE pNode, void *pvUser)
+{
+    PVM                     pVM       = (PVM)pvUser;
+    PPGMPHYSHANDLER         pCur      = (PPGMPHYSHANDLER)pNode;
+    PPGMPHYSHANDLERTYPEINT  pCurType  = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+    unsigned                uState    = pCurType->uState;
+    PPGMRAMRANGE            pRamHint  = NULL;
+    RTGCPHYS                GCPhys    = pCur->Core.Key;
+    RTUINT                  cPages    = pCur->cPages;
+    for (;;)
+    {
+        PPGMPAGE pPage;
+        int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
+        if (RT_SUCCESS(rc))
+        {
+            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, uState);
+
+            /* Tell NEM about the protection change. */
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+                NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
+                                               pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+        }
+        else
+            AssertRC(rc);
+
+        if (--cPages == 0)
+            return 0;
+        GCPhys += PAGE_SIZE;
+    }
+}
+
+
+/**
+ * Arguments for pgmR3InfoHandlersPhysicalOne and pgmR3InfoHandlersVirtualOne.
+ */
+typedef struct PGMHANDLERINFOARG
+{
+    /** The output helpers.*/
+    PCDBGFINFOHLP   pHlp;
+    /** Pointer to the cross context VM handle. */
+    PVM             pVM;
+    /** Set if statistics should be dumped. */
+    bool            fStats;
+} PGMHANDLERINFOARG, *PPGMHANDLERINFOARG;
+
+
+/**
+ * Info callback for 'pgmhandlers'.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The output helpers.
+ * @param   pszArgs     The arguments. phys or virt.
+ */
+DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /*
+     * Parse options.
+     */
+    PGMHANDLERINFOARG Args = { pHlp, pVM, /* .fStats = */ true };
+    if (pszArgs)
+        Args.fStats = strstr(pszArgs, "nost") == NULL;
+
+    /*
+     * Dump the handlers.
+     */
+    pHlp->pfnPrintf(pHlp,
+                    "Physical handlers: (PhysHandlers=%d (%#x))\n"
+                    "%*s %*s %*s %*s HandlerGC UserGC    Type     Description\n",
+                    pVM->pgm.s.pTreesR3->PhysHandlers, pVM->pgm.s.pTreesR3->PhysHandlers,
+                    - (int)sizeof(RTGCPHYS) * 2,     "From",
+                    - (int)sizeof(RTGCPHYS) * 2 - 3, "- To (incl)",
+                    - (int)sizeof(RTHCPTR)  * 2 - 1, "HandlerHC",
+                    - (int)sizeof(RTHCPTR)  * 2 - 1, "UserHC");
+    RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers, true, pgmR3InfoHandlersPhysicalOne, &Args);
+}
+
+
+/**
+ * Displays one physical handler range.
+ *
+ * @returns 0
+ * @param   pNode   Pointer to a PGMPHYSHANDLER.
+ * @param   pvUser  Pointer to command helper functions.
+ */
+static DECLCALLBACK(int) pgmR3InfoHandlersPhysicalOne(PAVLROGCPHYSNODECORE pNode, void *pvUser)
+{
+    PPGMPHYSHANDLER        pCur     = (PPGMPHYSHANDLER)pNode;
+    PPGMHANDLERINFOARG     pArgs    = (PPGMHANDLERINFOARG)pvUser;
+    PCDBGFINFOHLP          pHlp     = pArgs->pHlp;
+    PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pArgs->pVM, pCur);
+    const char *pszType;
+    switch (pCurType->enmKind)
+    {
+        case PGMPHYSHANDLERKIND_MMIO:   pszType = "MMIO   "; break;
+        case PGMPHYSHANDLERKIND_WRITE:  pszType = "Write  "; break;
+        case PGMPHYSHANDLERKIND_ALL:    pszType = "All    "; break;
+        default:                        pszType = "????"; break;
+    }
+    pHlp->pfnPrintf(pHlp,
+                    "%RGp - %RGp  %RHv  %RHv  %RHv  %RHv  %s  %s\n",
+                    pCur->Core.Key, pCur->Core.KeyLast, pCurType->pfnHandlerR3, pCur->pvUserR3,
+                    pCurType->pfnPfHandlerR0, pCur->pvUserR0, pszType, pCur->pszDesc);
+#ifdef VBOX_WITH_STATISTICS
+    if (pArgs->fStats)
+        pHlp->pfnPrintf(pHlp, "   cPeriods: %9RU64  cTicks: %11RU64  Min: %11RU64  Avg: %11RU64 Max: %11RU64\n",
+                        pCur->Stat.cPeriods, pCur->Stat.cTicks, pCur->Stat.cTicksMin,
+                        pCur->Stat.cPeriods ? pCur->Stat.cTicks / pCur->Stat.cPeriods : 0, pCur->Stat.cTicksMax);
+#endif
+    return 0;
+}
+
diff --git a/src/VBox/VMM/VMMR3/PGMMap.cpp b/src/VBox/VMM/VMMR3/PGMMap.cpp
new file mode 100644
index 00000000..d843b2b0
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PGMMap.cpp
@@ -0,0 +1,1472 @@
+/* $Id: PGMMap.cpp $ */
+/** @file
+ * PGM - Page Manager, Guest Context Mappings.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pgm.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vm.h>
+#include "PGMInline.h"
+
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+#ifndef PGM_WITHOUT_MAPPINGS
+static void pgmR3MapClearPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iOldPDE);
+static void pgmR3MapSetPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
+static int  pgmR3MapIntermediateCheckOne(PVM pVM, uintptr_t uAddress, unsigned cPages, PX86PT pPTDefault, PX86PTPAE pPTPaeDefault);
+static void pgmR3MapIntermediateDoOne(PVM pVM, uintptr_t uAddress, RTHCPHYS HCPhys, unsigned cPages, PX86PT pPTDefault, PX86PTPAE pPTPaeDefault);
+#endif
+
+
+#ifndef PGM_WITHOUT_MAPPINGS
+
+/**
+ * Creates a page table based mapping in GC.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPtr           Virtual Address. (Page table aligned!)
+ * @param   cb              Size of the range. Must be a 4MB aligned!
+ * @param   fFlags          PGMR3MAPPT_FLAGS_UNMAPPABLE or 0.
+ * @param   pfnRelocate     Relocation callback function.
+ * @param   pvUser          User argument to the callback.
+ * @param   pszDesc         Pointer to description string. This must not be freed.
+ */
+VMMR3DECL(int) PGMR3MapPT(PVM pVM, RTGCPTR GCPtr, uint32_t cb, uint32_t fFlags, PFNPGMRELOCATE pfnRelocate, void *pvUser, const char *pszDesc)
+{
+    LogFlow(("PGMR3MapPT: GCPtr=%#x cb=%d fFlags=%#x pfnRelocate=%p pvUser=%p pszDesc=%s\n", GCPtr, cb, fFlags, pfnRelocate, pvUser, pszDesc));
+    AssertMsg(pVM->pgm.s.pInterPD, ("Paging isn't initialized, init order problems!\n"));
+
+    /*
+     * Validate input.
+     * Note! The lower limit (1 MB) matches how pgmR3PhysMMIOExCreate works.
+     */
+    Assert(!fFlags || fFlags == PGMR3MAPPT_FLAGS_UNMAPPABLE);
+    AssertMsgReturn(cb >= _1M && cb <= _64M, ("Seriously? cb=%d (%#x)\n", cb, cb), VERR_OUT_OF_RANGE);
+
+    cb = RT_ALIGN_32(cb, _4M);
+    RTGCPTR GCPtrLast = GCPtr + cb - 1;
+
+    AssertMsgReturn(GCPtrLast >= GCPtr, ("Range wraps! GCPtr=%x GCPtrLast=%x\n", GCPtr, GCPtrLast),
+                    VERR_INVALID_PARAMETER);
+    AssertMsgReturn(!pVM->pgm.s.fMappingsFixed, ("Mappings are fixed! It's not possible to add new mappings at this time!\n"),
+                    VERR_PGM_MAPPINGS_FIXED);
+    AssertPtrReturn(pfnRelocate, VERR_INVALID_PARAMETER);
+
+    /*
+     * Find list location.
+     */
+    PPGMMAPPING pPrev = NULL;
+    PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
+    while (pCur)
+    {
+        if (pCur->GCPtrLast >= GCPtr && pCur->GCPtr <= GCPtrLast)
+        {
+            AssertMsgFailed(("Address is already in use by %s. req %#x-%#x take %#x-%#x\n",
+                             pCur->pszDesc, GCPtr, GCPtrLast, pCur->GCPtr, pCur->GCPtrLast));
+            LogRel(("VERR_PGM_MAPPING_CONFLICT: Address is already in use by %s. req %#x-%#x take %#x-%#x\n",
+                    pCur->pszDesc, GCPtr, GCPtrLast, pCur->GCPtr, pCur->GCPtrLast));
+            return VERR_PGM_MAPPING_CONFLICT;
+        }
+        if (pCur->GCPtr > GCPtr)
+            break;
+        pPrev = pCur;
+        pCur = pCur->pNextR3;
+    }
+
+    /*
+     * Check for conflicts with intermediate mappings.
+     */
+    const unsigned iPageDir = GCPtr >> X86_PD_SHIFT;
+    const unsigned cPTs     = cb >> X86_PD_SHIFT;
+    if (pVM->pgm.s.fFinalizedMappings)
+    {
+        for (unsigned i = 0; i < cPTs; i++)
+            if (pVM->pgm.s.pInterPD->a[iPageDir + i].n.u1Present)
+            {
+                AssertMsgFailed(("Address %#x is already in use by an intermediate mapping.\n", GCPtr + (i << PAGE_SHIFT)));
+                LogRel(("VERR_PGM_MAPPING_CONFLICT: Address %#x is already in use by an intermediate mapping.\n", GCPtr + (i << PAGE_SHIFT)));
+                return VERR_PGM_MAPPING_CONFLICT;
+            }
+        /** @todo AMD64: add check in PAE structures too, so we can remove all the 32-Bit paging stuff there. */
+    }
+
+    /*
+     * Allocate and initialize the new list node.
+     */
+    PPGMMAPPING pNew;
+    int rc;
+    if (fFlags & PGMR3MAPPT_FLAGS_UNMAPPABLE)
+        rc = MMHyperAlloc(           pVM, RT_UOFFSETOF_DYN(PGMMAPPING, aPTs[cPTs]), 0, MM_TAG_PGM_MAPPINGS, (void **)&pNew);
+    else
+        rc = MMR3HyperAllocOnceNoRel(pVM, RT_UOFFSETOF_DYN(PGMMAPPING, aPTs[cPTs]), 0, MM_TAG_PGM_MAPPINGS, (void **)&pNew);
+    if (RT_FAILURE(rc))
+        return rc;
+    pNew->GCPtr         = GCPtr;
+    pNew->GCPtrLast     = GCPtrLast;
+    pNew->cb            = cb;
+    pNew->pfnRelocate   = pfnRelocate;
+    pNew->pvUser        = pvUser;
+    pNew->pszDesc       = pszDesc;
+    pNew->cPTs          = cPTs;
+
+    /*
+     * Allocate page tables and insert them into the page directories.
+     * (One 32-bit PT and two PAE PTs.)
+     */
+    uint8_t *pbPTs;
+    if (fFlags & PGMR3MAPPT_FLAGS_UNMAPPABLE)
+        rc = MMHyperAlloc(           pVM, PAGE_SIZE * 3 * cPTs, PAGE_SIZE, MM_TAG_PGM_MAPPINGS, (void **)&pbPTs);
+    else
+        rc = MMR3HyperAllocOnceNoRel(pVM, PAGE_SIZE * 3 * cPTs, PAGE_SIZE, MM_TAG_PGM_MAPPINGS, (void **)&pbPTs);
+    if (RT_FAILURE(rc))
+    {
+        MMHyperFree(pVM, pNew);
+        return VERR_NO_MEMORY;
+    }
+
+    /*
+     * Init the page tables and insert them into the page directories.
+     */
+    Log4(("PGMR3MapPT: GCPtr=%RGv cPTs=%u pbPTs=%p\n", GCPtr, cPTs, pbPTs));
+    for (unsigned i = 0; i < cPTs; i++)
+    {
+        /*
+         * 32-bit.
+         */
+        pNew->aPTs[i].pPTR3    = (PX86PT)pbPTs;
+        pNew->aPTs[i].pPTRC    = MMHyperR3ToRC(pVM, pNew->aPTs[i].pPTR3);
+        pNew->aPTs[i].pPTR0    = MMHyperR3ToR0(pVM, pNew->aPTs[i].pPTR3);
+        pNew->aPTs[i].HCPhysPT = MMR3HyperHCVirt2HCPhys(pVM, pNew->aPTs[i].pPTR3);
+        pbPTs += PAGE_SIZE;
+        Log4(("PGMR3MapPT: i=%d: pPTR3=%RHv pPTRC=%RRv pPRTR0=%RHv HCPhysPT=%RHp\n",
+              i, pNew->aPTs[i].pPTR3, pNew->aPTs[i].pPTRC, pNew->aPTs[i].pPTR0, pNew->aPTs[i].HCPhysPT));
+
+        /*
+         * PAE.
+         */
+        pNew->aPTs[i].HCPhysPaePT0 = MMR3HyperHCVirt2HCPhys(pVM, pbPTs);
+        pNew->aPTs[i].HCPhysPaePT1 = MMR3HyperHCVirt2HCPhys(pVM, pbPTs + PAGE_SIZE);
+        pNew->aPTs[i].paPaePTsR3 = (PPGMSHWPTPAE)pbPTs;
+        pNew->aPTs[i].paPaePTsRC = MMHyperR3ToRC(pVM, pbPTs);
+        pNew->aPTs[i].paPaePTsR0 = MMHyperR3ToR0(pVM, pbPTs);
+        pbPTs += PAGE_SIZE * 2;
+        Log4(("PGMR3MapPT: i=%d: paPaePTsR#=%RHv paPaePTsRC=%RRv paPaePTsR#=%RHv HCPhysPaePT0=%RHp HCPhysPaePT1=%RHp\n",
+              i, pNew->aPTs[i].paPaePTsR3, pNew->aPTs[i].paPaePTsRC, pNew->aPTs[i].paPaePTsR0, pNew->aPTs[i].HCPhysPaePT0, pNew->aPTs[i].HCPhysPaePT1));
+    }
+    if (pVM->pgm.s.fFinalizedMappings)
+        pgmR3MapSetPDEs(pVM, pNew, iPageDir);
+    /* else PGMR3FinalizeMappings() */
+
+    /*
+     * Insert the new mapping.
+     */
+    pNew->pNextR3 = pCur;
+    pNew->pNextRC = pCur ? MMHyperR3ToRC(pVM, pCur) : NIL_RTRCPTR;
+    pNew->pNextR0 = pCur ? MMHyperR3ToR0(pVM, pCur) : NIL_RTR0PTR;
+    if (pPrev)
+    {
+        pPrev->pNextR3 = pNew;
+        pPrev->pNextRC = MMHyperR3ToRC(pVM, pNew);
+        pPrev->pNextR0 = MMHyperR3ToR0(pVM, pNew);
+    }
+    else
+    {
+        pVM->pgm.s.pMappingsR3 = pNew;
+        pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pNew);
+        pVM->pgm.s.pMappingsR0 = MMHyperR3ToR0(pVM, pNew);
+    }
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    }
+    return VINF_SUCCESS;
+}
+
+#ifdef VBOX_WITH_UNUSED_CODE
+
+/**
+ * Removes a page table based mapping.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPtr   Virtual Address. (Page table aligned!)
+ *
+ * @remarks Don't call this without passing PGMR3MAPPT_FLAGS_UNMAPPABLE to
+ *          PGMR3MapPT or you'll burn in the heap.
+ */
+VMMR3DECL(int)  PGMR3UnmapPT(PVM pVM, RTGCPTR GCPtr)
+{
+    LogFlow(("PGMR3UnmapPT: GCPtr=%#x\n", GCPtr));
+    AssertReturn(pVM->pgm.s.fFinalizedMappings, VERR_WRONG_ORDER);
+
+    /*
+     * Find it.
+     */
+    PPGMMAPPING pPrev = NULL;
+    PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
+    while (pCur)
+    {
+        if (pCur->GCPtr == GCPtr)
+        {
+            /*
+             * Unlink it.
+             */
+            if (pPrev)
+            {
+                pPrev->pNextR3 = pCur->pNextR3;
+                pPrev->pNextRC = pCur->pNextRC;
+                pPrev->pNextR0 = pCur->pNextR0;
+            }
+            else
+            {
+                pVM->pgm.s.pMappingsR3 = pCur->pNextR3;
+                pVM->pgm.s.pMappingsRC = pCur->pNextRC;
+                pVM->pgm.s.pMappingsR0 = pCur->pNextR0;
+            }
+
+            /*
+             * Free the page table memory, clear page directory entries
+             * and free the page tables and node memory.
+             */
+            MMHyperFree(pVM, pCur->aPTs[0].pPTR3);
+            if (pCur->GCPtr != NIL_RTGCPTR)
+                pgmR3MapClearPDEs(pVM, pCur, pCur->GCPtr >> X86_PD_SHIFT);
+            MMHyperFree(pVM, pCur);
+
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+            {
+                PVMCPU pVCpu = pVM->apCpusR3[i];
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+            }
+            return VINF_SUCCESS;
+        }
+
+        /* done? */
+        if (pCur->GCPtr > GCPtr)
+            break;
+
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNextR3;
+    }
+
+    AssertMsgFailed(("No mapping for %#x found!\n", GCPtr));
+    return VERR_INVALID_PARAMETER;
+}
+
+#endif /* unused */
+
+
+/**
+ * Checks whether a range of PDEs in the intermediate
+ * memory context are unused.
+ *
+ * We're talking 32-bit PDEs here.
+ *
+ * @returns true/false.
+ * @param   pVM         The cross context VM structure.
+ * @param   iPD         The first PDE in the range.
+ * @param   cPTs        The number of PDEs in the range.
+ */
+DECLINLINE(bool) pgmR3AreIntermediatePDEsUnused(PVM pVM, unsigned iPD, unsigned cPTs)
+{
+    if (pVM->pgm.s.pInterPD->a[iPD].n.u1Present)
+        return false;
+    while (cPTs > 1)
+    {
+        iPD++;
+        if (pVM->pgm.s.pInterPD->a[iPD].n.u1Present)
+            return false;
+        cPTs--;
+    }
+    return true;
+}
+
+
+/**
+ * Unlinks the mapping.
+ *
+ * The mapping *must* be in the list.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pMapping        The mapping to unlink.
+ */
+static void pgmR3MapUnlink(PVM pVM, PPGMMAPPING pMapping)
+{
+    PPGMMAPPING pAfterThis = pVM->pgm.s.pMappingsR3;
+    if (pAfterThis == pMapping)
+    {
+        /* head */
+        pVM->pgm.s.pMappingsR3 = pMapping->pNextR3;
+        pVM->pgm.s.pMappingsRC = pMapping->pNextRC;
+        pVM->pgm.s.pMappingsR0 = pMapping->pNextR0;
+    }
+    else
+    {
+        /* in the list */
+        while (pAfterThis->pNextR3 != pMapping)
+        {
+            pAfterThis = pAfterThis->pNextR3;
+            AssertReleaseReturnVoid(pAfterThis);
+        }
+
+        pAfterThis->pNextR3 = pMapping->pNextR3;
+        pAfterThis->pNextRC = pMapping->pNextRC;
+        pAfterThis->pNextR0 = pMapping->pNextR0;
+    }
+}
+
+
+/**
+ * Links the mapping.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pMapping        The mapping to linked.
+ */
+static void pgmR3MapLink(PVM pVM, PPGMMAPPING pMapping)
+{
+    /*
+     * Find the list location (it's sorted by GCPhys) and link it in.
+     */
+    if (    !pVM->pgm.s.pMappingsR3
+        ||  pVM->pgm.s.pMappingsR3->GCPtr > pMapping->GCPtr)
+    {
+        /* head */
+        pMapping->pNextR3 = pVM->pgm.s.pMappingsR3;
+        pMapping->pNextRC = pVM->pgm.s.pMappingsRC;
+        pMapping->pNextR0 = pVM->pgm.s.pMappingsR0;
+        pVM->pgm.s.pMappingsR3 = pMapping;
+        pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pMapping);
+        pVM->pgm.s.pMappingsR0 = MMHyperR3ToR0(pVM, pMapping);
+    }
+    else
+    {
+        /* in the list */
+        PPGMMAPPING pAfterThis  = pVM->pgm.s.pMappingsR3;
+        PPGMMAPPING pBeforeThis = pAfterThis->pNextR3;
+        while (pBeforeThis && pBeforeThis->GCPtr <= pMapping->GCPtr)
+        {
+            pAfterThis = pBeforeThis;
+            pBeforeThis = pBeforeThis->pNextR3;
+        }
+
+        pMapping->pNextR3 = pAfterThis->pNextR3;
+        pMapping->pNextRC = pAfterThis->pNextRC;
+        pMapping->pNextR0 = pAfterThis->pNextR0;
+        pAfterThis->pNextR3 = pMapping;
+        pAfterThis->pNextRC = MMHyperR3ToRC(pVM, pMapping);
+        pAfterThis->pNextR0 = MMHyperR3ToR0(pVM, pMapping);
+    }
+}
+
+
+/**
+ * Finalizes the intermediate context.
+ *
+ * This is called at the end of the ring-3 init and will construct the
+ * intermediate paging structures, relocating all the mappings in the process.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @thread  EMT(0)
+ */
+VMMR3DECL(int) PGMR3FinalizeMappings(PVM pVM)
+{
+    AssertReturn(!pVM->pgm.s.fFinalizedMappings, VERR_WRONG_ORDER);
+    pVM->pgm.s.fFinalizedMappings = true;
+
+    /*
+     * Loop until all mappings have been finalized.
+     */
+#if 0
+    unsigned    iPDNext = UINT32_C(0xc0000000) >> X86_PD_SHIFT; /* makes CSAM/PATM freak out booting linux. :-/ */
+#elif 0
+    unsigned    iPDNext = MM_HYPER_AREA_ADDRESS >> X86_PD_SHIFT;
+#else
+    unsigned    iPDNext = 1 << X86_PD_SHIFT; /* no hint, map them from the top. */
+#endif
+
+    PPGMMAPPING pCur;
+    do
+    {
+        pCur = pVM->pgm.s.pMappingsR3;
+        while (pCur)
+        {
+            if (!pCur->fFinalized)
+            {
+                /*
+                 * Find a suitable location.
+                 */
+                RTGCPTR const   GCPtrOld = pCur->GCPtr;
+                const unsigned  cPTs     = pCur->cPTs;
+                unsigned        iPDNew   = iPDNext;
+                if (    iPDNew + cPTs >= X86_PG_ENTRIES /* exclude the last PD */
+                    ||  !pgmR3AreIntermediatePDEsUnused(pVM, iPDNew, cPTs)
+                    ||  !pCur->pfnRelocate(pVM, GCPtrOld, (RTGCPTR)iPDNew << X86_PD_SHIFT, PGMRELOCATECALL_SUGGEST, pCur->pvUser))
+                {
+                    /* No luck, just scan down from 4GB-4MB, giving up at 4MB. */
+                    iPDNew = X86_PG_ENTRIES - cPTs - 1;
+                    while (     iPDNew > 0
+                           &&   (   !pgmR3AreIntermediatePDEsUnused(pVM, iPDNew, cPTs)
+                                 || !pCur->pfnRelocate(pVM, GCPtrOld, (RTGCPTR)iPDNew << X86_PD_SHIFT, PGMRELOCATECALL_SUGGEST, pCur->pvUser))
+                           )
+                        iPDNew--;
+                    AssertLogRelReturn(iPDNew != 0, VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
+                }
+
+                /*
+                 * Relocate it (something akin to pgmR3MapRelocate).
+                 */
+                pgmR3MapSetPDEs(pVM, pCur, iPDNew);
+
+                /* unlink the mapping, update the entry and relink it. */
+                pgmR3MapUnlink(pVM, pCur);
+
+                RTGCPTR const GCPtrNew = (RTGCPTR)iPDNew << X86_PD_SHIFT;
+                pCur->GCPtr      = GCPtrNew;
+                pCur->GCPtrLast  = GCPtrNew + pCur->cb - 1;
+                pCur->fFinalized = true;
+
+                pgmR3MapLink(pVM, pCur);
+
+                /* Finally work the callback. */
+                pCur->pfnRelocate(pVM, GCPtrOld, GCPtrNew, PGMRELOCATECALL_RELOCATE, pCur->pvUser);
+
+                /*
+                 * The list order might have changed, start from the beginning again.
+                 */
+                iPDNext = iPDNew + cPTs;
+                break;
+            }
+
+            /* next */
+            pCur = pCur->pNextR3;
+        }
+    } while (pCur);
+
+    return VINF_SUCCESS;
+}
+
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
+
+/**
+ * Gets the size of the current guest mappings if they were to be
+ * put next to one another.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   pcb     Where to store the size.
+ */
+VMMR3DECL(int) PGMR3MappingsSize(PVM pVM, uint32_t *pcb)
+{
+    RTGCPTR cb = 0;
+#ifndef PGM_WITHOUT_MAPPINGS
+    for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur; pCur = pCur->pNextR3)
+        cb += pCur->cb;
+#else
+    RT_NOREF(pVM);
+#endif
+
+    *pcb = cb;
+    AssertReturn(*pcb == cb, VERR_NUMBER_TOO_BIG);
+    Log(("PGMR3MappingsSize: return %d (%#x) bytes\n", cb, cb));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Fixates the guest context mappings in a range reserved from the Guest OS.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPtrBase   The address of the reserved range of guest memory.
+ * @param   cb          The size of the range starting at GCPtrBase.
+ */
+VMMR3DECL(int) PGMR3MappingsFix(PVM pVM, RTGCPTR GCPtrBase, uint32_t cb)
+{
+    Log(("PGMR3MappingsFix: GCPtrBase=%RGv cb=%#x (fMappingsFixed=%RTbool MappingEnabled=%RTbool)\n",
+         GCPtrBase, cb, pVM->pgm.s.fMappingsFixed, pgmMapAreMappingsEnabled(pVM)));
+
+#ifndef PGM_WITHOUT_MAPPINGS
+    if (pgmMapAreMappingsEnabled(pVM))
+    {
+        /*
+         * Only applies to VCPU 0 as we don't support SMP guests with raw mode.
+         */
+        Assert(pVM->cCpus == 1);
+        PVMCPU pVCpu = pVM->apCpusR3[0];
+
+        /*
+         * Before we do anything we'll do a forced PD sync to try make sure any
+         * pending relocations because of these mappings have been resolved.
+         */
+        PGMSyncCR3(pVCpu, CPUMGetGuestCR0(pVCpu), CPUMGetGuestCR3(pVCpu), CPUMGetGuestCR4(pVCpu), true);
+
+        return pgmR3MappingsFixInternal(pVM, GCPtrBase, cb);
+    }
+
+#else  /* PGM_WITHOUT_MAPPINGS */
+    RT_NOREF(pVM, GCPtrBase, cb);
+#endif /* PGM_WITHOUT_MAPPINGS */
+
+    Assert(!VM_IS_RAW_MODE_ENABLED(pVM));
+    return VINF_SUCCESS;
+}
+
+
+#ifndef PGM_WITHOUT_MAPPINGS
+/**
+ * Internal worker for PGMR3MappingsFix and pgmR3Load.
+ *
+ * (This does not perform a SyncCR3 before the fixation like PGMR3MappingsFix.)
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPtrBase   The address of the reserved range of guest memory.
+ * @param   cb          The size of the range starting at GCPtrBase.
+ */
+int pgmR3MappingsFixInternal(PVM pVM, RTGCPTR GCPtrBase, uint32_t cb)
+{
+    /*
+     * Check input arguments and pre-conditions.
+     */
+    AssertMsgReturn(!(GCPtrBase & X86_PAGE_4M_OFFSET_MASK), ("GCPtrBase (%#x) has to be aligned on a 4MB address!\n", GCPtrBase),
+                    VERR_INVALID_PARAMETER);
+    AssertMsgReturn(cb && !(cb & X86_PAGE_4M_OFFSET_MASK), ("cb (%#x) is 0 or not aligned on a 4MB address!\n", cb),
+                    VERR_INVALID_PARAMETER);
+    AssertReturn(pgmMapAreMappingsEnabled(pVM), VERR_PGM_MAPPINGS_DISABLED);
+    AssertReturn(pVM->cCpus == 1, VERR_PGM_MAPPINGS_SMP);
+
+    /*
+     * Check that it's not conflicting with a core code mapping in the intermediate page table.
+     */
+    unsigned    iPDNew = GCPtrBase >> X86_PD_SHIFT;
+    unsigned    i = cb >> X86_PD_SHIFT;
+    while (i-- > 0)
+    {
+        if (pVM->pgm.s.pInterPD->a[iPDNew + i].n.u1Present)
+        {
+            /* Check that it's not one or our mappings. */
+            PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
+            while (pCur)
+            {
+                if (iPDNew + i - (pCur->GCPtr >> X86_PD_SHIFT) < (pCur->cb >> X86_PD_SHIFT))
+                    break;
+                pCur = pCur->pNextR3;
+            }
+            if (!pCur)
+            {
+                LogRel(("PGMR3MappingsFix: Conflicts with intermediate PDE %#x (GCPtrBase=%RGv cb=%#zx). The guest should retry.\n",
+                        iPDNew + i, GCPtrBase, cb));
+                return VERR_PGM_MAPPINGS_FIX_CONFLICT;
+            }
+        }
+    }
+
+    /*
+     * In PAE / PAE mode, make sure we don't cross page directories.
+     */
+    PVMCPU pVCpu = pVM->apCpusR3[0];
+    if (    (   pVCpu->pgm.s.enmGuestMode  == PGMMODE_PAE
+             || pVCpu->pgm.s.enmGuestMode  == PGMMODE_PAE_NX)
+        &&  (   pVCpu->pgm.s.enmShadowMode == PGMMODE_PAE
+             || pVCpu->pgm.s.enmShadowMode == PGMMODE_PAE_NX))
+    {
+        unsigned iPdptBase = GCPtrBase >> X86_PDPT_SHIFT;
+        unsigned iPdptLast = (GCPtrBase + cb - 1) >> X86_PDPT_SHIFT;
+        if (iPdptBase != iPdptLast)
+        {
+            LogRel(("PGMR3MappingsFix: Crosses PD boundary; iPdptBase=%#x iPdptLast=%#x (GCPtrBase=%RGv cb=%#zx). The guest should retry.\n",
+                    iPdptBase, iPdptLast, GCPtrBase, cb));
+            return VERR_PGM_MAPPINGS_FIX_CONFLICT;
+        }
+    }
+
+    /*
+     * Loop the mappings and check that they all agree on their new locations.
+     */
+    RTGCPTR     GCPtrCur = GCPtrBase;
+    PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
+    while (pCur)
+    {
+        if (!pCur->pfnRelocate(pVM, pCur->GCPtr, GCPtrCur, PGMRELOCATECALL_SUGGEST, pCur->pvUser))
+        {
+            AssertMsgFailed(("The suggested fixed address %#x was rejected by '%s'!\n", GCPtrCur, pCur->pszDesc));
+            return VERR_PGM_MAPPINGS_FIX_REJECTED;
+        }
+        /* next */
+        GCPtrCur += pCur->cb;
+        pCur = pCur->pNextR3;
+    }
+    if (GCPtrCur > GCPtrBase + cb)
+    {
+        AssertMsgFailed(("cb (%#x) is less than the required range %#x!\n", cb, GCPtrCur - GCPtrBase));
+        return VERR_PGM_MAPPINGS_FIX_TOO_SMALL;
+    }
+
+    /*
+     * Loop the table assigning the mappings to the passed in memory
+     * and call their relocator callback.
+     */
+    GCPtrCur = GCPtrBase;
+    pCur = pVM->pgm.s.pMappingsR3;
+    while (pCur)
+    {
+        RTGCPTR const GCPtrOld = pCur->GCPtr;
+
+        /*
+         * Relocate the page table(s).
+         */
+        if (pCur->GCPtr != NIL_RTGCPTR)
+            pgmR3MapClearPDEs(pVM, pCur, GCPtrOld >> X86_PD_SHIFT);
+        pgmR3MapSetPDEs(pVM, pCur, GCPtrCur >> X86_PD_SHIFT);
+
+        /*
+         * Update the entry.
+         */
+        pCur->GCPtr = GCPtrCur;
+        pCur->GCPtrLast = GCPtrCur + pCur->cb - 1;
+
+        /*
+         * Callback to execute the relocation.
+         */
+        pCur->pfnRelocate(pVM, GCPtrOld, GCPtrCur, PGMRELOCATECALL_RELOCATE, pCur->pvUser);
+
+        /*
+         * Advance.
+         */
+        GCPtrCur += pCur->cb;
+        pCur = pCur->pNextR3;
+    }
+
+    /*
+     * Mark the mappings as fixed at this new location and return.
+     */
+    pVM->pgm.s.fMappingsFixed           = true;
+    pVM->pgm.s.fMappingsFixedRestored   = false;
+    pVM->pgm.s.GCPtrMappingFixed        = GCPtrBase;
+    pVM->pgm.s.cbMappingFixed           = cb;
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        pVM->aCpus[idCpu].pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
+        VMCPU_FF_SET(pVM->apCpusR3[idCpu], VMCPU_FF_PGM_SYNC_CR3);
+    }
+    return VINF_SUCCESS;
+}
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
+
+/**
+ * Unfixes the mappings.
+ *
+ * Unless PGMR3MappingsDisable is in effect, mapping conflict detection will be
+ * enabled after this call.  If the mappings are fixed, a full CR3 resync will
+ * take place afterwards.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(int) PGMR3MappingsUnfix(PVM pVM)
+{
+    Log(("PGMR3MappingsUnfix: fMappingsFixed=%RTbool MappingsEnabled=%RTbool\n", pVM->pgm.s.fMappingsFixed, pgmMapAreMappingsEnabled(pVM)));
+    if (   pgmMapAreMappingsEnabled(pVM)
+        && (    pVM->pgm.s.fMappingsFixed
+            ||  pVM->pgm.s.fMappingsFixedRestored)
+       )
+    {
+        bool const fResyncCR3 = pVM->pgm.s.fMappingsFixed;
+
+        pVM->pgm.s.fMappingsFixed           = false;
+        pVM->pgm.s.fMappingsFixedRestored   = false;
+        pVM->pgm.s.GCPtrMappingFixed        = 0;
+        pVM->pgm.s.cbMappingFixed           = 0;
+
+        if (fResyncCR3)
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+                VMCPU_FF_SET(pVM->apCpusR3[i], VMCPU_FF_PGM_SYNC_CR3);
+    }
+    return VINF_SUCCESS;
+}
+
+#ifndef PGM_WITHOUT_MAPPINGS
+
+/**
+ * Checks if the mappings needs re-fixing after a restore.
+ *
+ * @returns true if they need, false if not.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMR3DECL(bool) PGMR3MappingsNeedReFixing(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->pgm.s.fMappingsFixedRestored;
+}
+
+
+/**
+ * Map pages into the intermediate context (switcher code).
+ *
+ * These pages are mapped at both the give virtual address and at the physical
+ * address (for identity mapping).
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   Addr        Intermediate context address of the mapping.
+ * @param   HCPhys      Start of the range of physical pages. This must be entriely below 4GB!
+ * @param   cbPages     Number of bytes to map.
+ *
+ * @remark  This API shall not be used to anything but mapping the switcher code.
+ */
+VMMR3DECL(int) PGMR3MapIntermediate(PVM pVM, RTUINTPTR Addr, RTHCPHYS HCPhys, unsigned cbPages)
+{
+    LogFlow(("PGMR3MapIntermediate: Addr=%RTptr HCPhys=%RHp cbPages=%#x\n", Addr, HCPhys, cbPages));
+
+    /*
+     * Adjust input.
+     */
+    cbPages += (uint32_t)HCPhys & PAGE_OFFSET_MASK;
+    cbPages  = RT_ALIGN(cbPages, PAGE_SIZE);
+    HCPhys  &= X86_PTE_PAE_PG_MASK;
+    Addr    &= PAGE_BASE_MASK;
+    /* We only care about the first 4GB, because on AMD64 we'll be repeating them all over the address space. */
+    uint32_t uAddress = (uint32_t)Addr;
+
+    /*
+     * Assert input and state.
+     */
+    AssertMsg(pVM->pgm.s.offVM, ("Bad init order\n"));
+    AssertMsg(pVM->pgm.s.pInterPD, ("Bad init order, paging.\n"));
+    AssertMsg(cbPages <= (512 << PAGE_SHIFT), ("The mapping is too big %d bytes\n", cbPages));
+    AssertMsg(HCPhys < _4G && HCPhys + cbPages < _4G, ("Addr=%RTptr HCPhys=%RHp cbPages=%d\n", Addr, HCPhys, cbPages));
+    AssertReturn(!pVM->pgm.s.fFinalizedMappings, VERR_WRONG_ORDER);
+
+    /*
+     * Check for internal conflicts between the virtual address and the physical address.
+     * A 1:1 mapping is fine, but partial overlapping is a no-no.
+     */
+    if (    uAddress != HCPhys
+        &&  (   uAddress < HCPhys
+                ? HCPhys - uAddress < cbPages
+                : uAddress - HCPhys < cbPages
+            )
+       )
+        AssertLogRelMsgFailedReturn(("Addr=%RTptr HCPhys=%RHp cbPages=%d\n", Addr, HCPhys, cbPages),
+                                    VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
+
+    const unsigned cPages = cbPages >> PAGE_SHIFT;
+    int rc = pgmR3MapIntermediateCheckOne(pVM, uAddress, cPages, pVM->pgm.s.apInterPTs[0], pVM->pgm.s.apInterPaePTs[0]);
+    if (RT_FAILURE(rc))
+        return rc;
+    rc = pgmR3MapIntermediateCheckOne(pVM, (uintptr_t)HCPhys, cPages, pVM->pgm.s.apInterPTs[1], pVM->pgm.s.apInterPaePTs[1]);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Everythings fine, do the mapping.
+     */
+    pgmR3MapIntermediateDoOne(pVM, uAddress, HCPhys, cPages, pVM->pgm.s.apInterPTs[0], pVM->pgm.s.apInterPaePTs[0]);
+    pgmR3MapIntermediateDoOne(pVM, (uintptr_t)HCPhys, HCPhys, cPages, pVM->pgm.s.apInterPTs[1], pVM->pgm.s.apInterPaePTs[1]);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Validates that there are no conflicts for this mapping into the intermediate context.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   uAddress        Address of the mapping.
+ * @param   cPages          Number of pages.
+ * @param   pPTDefault      Pointer to the default page table for this mapping.
+ * @param   pPTPaeDefault   Pointer to the default page table for this mapping.
+ */
+static int pgmR3MapIntermediateCheckOne(PVM pVM, uintptr_t uAddress, unsigned cPages, PX86PT pPTDefault, PX86PTPAE pPTPaeDefault)
+{
+    AssertMsg((uAddress >> X86_PD_SHIFT) + cPages <= 1024, ("64-bit fixme uAddress=%RGv cPages=%u\n", uAddress, cPages));
+
+    /*
+     * Check that the ranges are available.
+     * (This code doesn't have to be fast.)
+     */
+    while (cPages > 0)
+    {
+        /*
+         * 32-Bit.
+         */
+        unsigned iPDE = (uAddress >> X86_PD_SHIFT) & X86_PD_MASK;
+        unsigned iPTE = (uAddress >> X86_PT_SHIFT) & X86_PT_MASK;
+        PX86PT pPT = pPTDefault;
+        if (pVM->pgm.s.pInterPD->a[iPDE].u)
+        {
+            RTHCPHYS HCPhysPT = pVM->pgm.s.pInterPD->a[iPDE].u & X86_PDE_PG_MASK;
+            if (HCPhysPT == MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[0]))
+                pPT = pVM->pgm.s.apInterPTs[0];
+            else if (HCPhysPT == MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[1]))
+                pPT = pVM->pgm.s.apInterPTs[1];
+            else
+            {
+                /** @todo this must be handled with a relocation of the conflicting mapping!
+                 * Which of course cannot be done because we're in the middle of the initialization. bad design! */
+                AssertLogRelMsgFailedReturn(("Conflict between core code and PGMR3Mapping(). uAddress=%RHv\n", uAddress),
+                                            VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
+            }
+        }
+        if (pPT->a[iPTE].u)
+            AssertLogRelMsgFailedReturn(("Conflict iPTE=%#x iPDE=%#x uAddress=%RHv pPT->a[iPTE].u=%RX32\n", iPTE, iPDE, uAddress, pPT->a[iPTE].u),
+                                        VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
+
+        /*
+         * PAE.
+         */
+        const unsigned iPDPE= (uAddress >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+        iPDE = (uAddress >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+        iPTE = (uAddress >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK;
+        Assert(iPDPE < 4);
+        Assert(pVM->pgm.s.apInterPaePDs[iPDPE]);
+        PX86PTPAE pPTPae = pPTPaeDefault;
+        if (pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u)
+        {
+            RTHCPHYS HCPhysPT = pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u & X86_PDE_PAE_PG_MASK;
+            if (HCPhysPT == MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[0]))
+                pPTPae = pVM->pgm.s.apInterPaePTs[0];
+            else if (HCPhysPT == MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[0]))
+                pPTPae = pVM->pgm.s.apInterPaePTs[1];
+            else
+            {
+                /** @todo this must be handled with a relocation of the conflicting mapping!
+                 * Which of course cannot be done because we're in the middle of the initialization. bad design! */
+                AssertLogRelMsgFailedReturn(("Conflict between core code and PGMR3Mapping(). uAddress=%RHv\n", uAddress),
+                                            VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
+            }
+        }
+        if (pPTPae->a[iPTE].u)
+            AssertLogRelMsgFailedReturn(("Conflict iPTE=%#x iPDE=%#x uAddress=%RHv pPTPae->a[iPTE].u=%#RX64\n", iPTE, iPDE, uAddress, pPTPae->a[iPTE].u),
+                                        VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
+
+        /* next */
+        uAddress += PAGE_SIZE;
+        cPages--;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Sets up the intermediate page tables for a verified mapping.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   uAddress        Address of the mapping.
+ * @param   HCPhys          The physical address of the page range.
+ * @param   cPages          Number of pages.
+ * @param   pPTDefault      Pointer to the default page table for this mapping.
+ * @param   pPTPaeDefault   Pointer to the default page table for this mapping.
+ */
+static void pgmR3MapIntermediateDoOne(PVM pVM, uintptr_t uAddress, RTHCPHYS HCPhys, unsigned cPages, PX86PT pPTDefault, PX86PTPAE pPTPaeDefault)
+{
+    while (cPages > 0)
+    {
+        /*
+         * 32-Bit.
+         */
+        unsigned iPDE = (uAddress >> X86_PD_SHIFT) & X86_PD_MASK;
+        unsigned iPTE = (uAddress >> X86_PT_SHIFT) & X86_PT_MASK;
+        PX86PT pPT;
+        if (pVM->pgm.s.pInterPD->a[iPDE].u)
+            pPT = (PX86PT)MMPagePhys2Page(pVM, pVM->pgm.s.pInterPD->a[iPDE].u & X86_PDE_PG_MASK);
+        else
+        {
+            pVM->pgm.s.pInterPD->a[iPDE].u = X86_PDE_P | X86_PDE_A | X86_PDE_RW
+                                           | (uint32_t)MMPage2Phys(pVM, pPTDefault);
+            pPT = pPTDefault;
+        }
+        pPT->a[iPTE].u = X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D | (uint32_t)HCPhys;
+
+        /*
+         * PAE
+         */
+        const unsigned iPDPE= (uAddress >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+        iPDE = (uAddress >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+        iPTE = (uAddress >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK;
+        Assert(iPDPE < 4);
+        Assert(pVM->pgm.s.apInterPaePDs[iPDPE]);
+        PX86PTPAE pPTPae;
+        if (pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u)
+            pPTPae = (PX86PTPAE)MMPagePhys2Page(pVM, pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u & X86_PDE_PAE_PG_MASK);
+        else
+        {
+            pPTPae = pPTPaeDefault;
+            pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u = X86_PDE_P | X86_PDE_A | X86_PDE_RW
+                                                       | MMPage2Phys(pVM, pPTPaeDefault);
+        }
+        pPTPae->a[iPTE].u = X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D | HCPhys;
+
+        /* next */
+        cPages--;
+        HCPhys += PAGE_SIZE;
+        uAddress += PAGE_SIZE;
+    }
+}
+
+
+/**
+ * Clears all PDEs involved with the mapping in the shadow and intermediate page tables.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pMap        Pointer to the mapping in question.
+ * @param   iOldPDE     The index of the 32-bit PDE corresponding to the base of the mapping.
+ */
+static void pgmR3MapClearPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iOldPDE)
+{
+    unsigned i     = pMap->cPTs;
+    PVMCPU   pVCpu = VMMGetCpu(pVM);
+    pgmLock(pVM);                           /* to avoid assertions */
+
+    pgmMapClearShadowPDEs(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pMap, iOldPDE, false /*fDeactivateCR3*/);
+
+    iOldPDE += i;
+    while (i-- > 0)
+    {
+        iOldPDE--;
+
+        /*
+         * 32-bit.
+         */
+        pVM->pgm.s.pInterPD->a[iOldPDE].u        = 0;
+
+        /*
+         * PAE.
+         */
+        const unsigned iPD = iOldPDE / 256;         /* iOldPDE * 2 / 512; iOldPDE is in 4 MB pages */
+        unsigned iPDE = iOldPDE * 2 % 512;
+        pVM->pgm.s.apInterPaePDs[iPD]->a[iPDE].u = 0;
+        iPDE++;
+        AssertFatal(iPDE < 512);
+        pVM->pgm.s.apInterPaePDs[iPD]->a[iPDE].u = 0;
+    }
+
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Sets all PDEs involved with the mapping in the shadow and intermediate page tables.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pMap        Pointer to the mapping in question.
+ * @param   iNewPDE     The index of the 32-bit PDE corresponding to the base of the mapping.
+ */
+static void pgmR3MapSetPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE)
+{
+    PPGM   pPGM  = &pVM->pgm.s;
+# ifdef VBOX_STRICT
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+# endif
+    pgmLock(pVM);                           /* to avoid assertions */
+
+    Assert(!pgmMapAreMappingsEnabled(pVM) || PGMGetGuestMode(pVCpu) <= PGMMODE_PAE_NX);
+
+    pgmMapSetShadowPDEs(pVM, pMap, iNewPDE);
+
+    /*
+     * Init the page tables and insert them into the page directories.
+     */
+    unsigned i = pMap->cPTs;
+    iNewPDE += i;
+    while (i-- > 0)
+    {
+        iNewPDE--;
+
+        /*
+         * 32-bit.
+         */
+        X86PDE Pde;
+        /* Default mapping page directory flags are read/write and supervisor; individual page attributes determine the final flags */
+        Pde.u = PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | (uint32_t)pMap->aPTs[i].HCPhysPT;
+        pPGM->pInterPD->a[iNewPDE]        = Pde;
+
+        /*
+         * PAE.
+         */
+        const unsigned iPD = iNewPDE / 256;
+        unsigned iPDE = iNewPDE * 2 % 512;
+        X86PDEPAE PdePae0;
+        PdePae0.u = PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | pMap->aPTs[i].HCPhysPaePT0;
+        pPGM->apInterPaePDs[iPD]->a[iPDE] = PdePae0;
+        iPDE++;
+        AssertFatal(iPDE < 512);
+        X86PDEPAE PdePae1;
+        PdePae1.u = PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | pMap->aPTs[i].HCPhysPaePT1;
+        pPGM->apInterPaePDs[iPD]->a[iPDE] = PdePae1;
+    }
+
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Relocates a mapping to a new address.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pMapping            The mapping to relocate.
+ * @param   GCPtrOldMapping     The address of the start of the old mapping.
+ *                              NIL_RTGCPTR if not currently mapped.
+ * @param   GCPtrNewMapping     The address of the start of the new mapping.
+ */
+static void pgmR3MapRelocate(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping, RTGCPTR GCPtrNewMapping)
+{
+    Log(("PGM: Relocating %s from %RGv to %RGv\n", pMapping->pszDesc, GCPtrOldMapping, GCPtrNewMapping));
+    AssertMsg(GCPtrOldMapping == pMapping->GCPtr, ("%RGv vs %RGv\n", GCPtrOldMapping, pMapping->GCPtr));
+    AssertMsg((GCPtrOldMapping >> X86_PD_SHIFT) < X86_PG_ENTRIES, ("%RGv\n", GCPtrOldMapping));
+    AssertMsg((GCPtrNewMapping >> X86_PD_SHIFT) < X86_PG_ENTRIES, ("%RGv\n", GCPtrOldMapping));
+
+    /*
+     * Relocate the page table(s).
+     */
+    if (GCPtrOldMapping != NIL_RTGCPTR)
+        pgmR3MapClearPDEs(pVM, pMapping, GCPtrOldMapping >> X86_PD_SHIFT);
+    pgmR3MapSetPDEs(pVM, pMapping, GCPtrNewMapping >> X86_PD_SHIFT);
+
+    /*
+     * Update and resort the mapping list.
+     */
+
+    /* Find previous mapping for pMapping, put result into pPrevMap. */
+    PPGMMAPPING pPrevMap = NULL;
+    PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
+    while (pCur && pCur != pMapping)
+    {
+        /* next */
+        pPrevMap = pCur;
+        pCur = pCur->pNextR3;
+    }
+    Assert(pCur);
+
+    /* Find mapping which >= than pMapping. */
+    RTGCPTR     GCPtrNew = GCPtrNewMapping;
+    PPGMMAPPING pPrev = NULL;
+    pCur = pVM->pgm.s.pMappingsR3;
+    while (pCur && pCur->GCPtr < GCPtrNew)
+    {
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNextR3;
+    }
+
+    if (pCur != pMapping && pPrev != pMapping)
+    {
+        /*
+         * Unlink.
+         */
+        if (pPrevMap)
+        {
+            pPrevMap->pNextR3 = pMapping->pNextR3;
+            pPrevMap->pNextRC = pMapping->pNextRC;
+            pPrevMap->pNextR0 = pMapping->pNextR0;
+        }
+        else
+        {
+            pVM->pgm.s.pMappingsR3 = pMapping->pNextR3;
+            pVM->pgm.s.pMappingsRC = pMapping->pNextRC;
+            pVM->pgm.s.pMappingsR0 = pMapping->pNextR0;
+        }
+
+        /*
+         * Link
+         */
+        pMapping->pNextR3 = pCur;
+        if (pPrev)
+        {
+            pMapping->pNextRC = pPrev->pNextRC;
+            pMapping->pNextR0 = pPrev->pNextR0;
+            pPrev->pNextR3 = pMapping;
+            pPrev->pNextRC = MMHyperR3ToRC(pVM, pMapping);
+            pPrev->pNextR0 = MMHyperR3ToR0(pVM, pMapping);
+        }
+        else
+        {
+            pMapping->pNextRC = pVM->pgm.s.pMappingsRC;
+            pMapping->pNextR0 = pVM->pgm.s.pMappingsR0;
+            pVM->pgm.s.pMappingsR3 = pMapping;
+            pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pMapping);
+            pVM->pgm.s.pMappingsR0 = MMHyperR3ToR0(pVM, pMapping);
+        }
+    }
+
+    /*
+     * Update the entry.
+     */
+    pMapping->GCPtr = GCPtrNew;
+    pMapping->GCPtrLast = GCPtrNew + pMapping->cb - 1;
+
+    /*
+     * Callback to execute the relocation.
+     */
+    pMapping->pfnRelocate(pVM, GCPtrOldMapping, GCPtrNewMapping, PGMRELOCATECALL_RELOCATE, pMapping->pvUser);
+}
+
+
+/**
+ * Checks if a new mapping address wasn't previously used and caused a clash with guest mappings.
+ *
+ * @returns VBox status code.
+ * @param   pMapping            The mapping which conflicts.
+ * @param   GCPtr               New mapping address to try
+ */
+bool pgmR3MapIsKnownConflictAddress(PPGMMAPPING pMapping, RTGCPTR GCPtr)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pMapping->aGCPtrConflicts); i++)
+    {
+        if (GCPtr == pMapping->aGCPtrConflicts[i])
+            return true;
+    }
+    return false;
+}
+
+
+/**
+ * Resolves a conflict between a page table based GC mapping and
+ * the Guest OS page tables. (32 bits version)
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pMapping            The mapping which conflicts.
+ * @param   pPDSrc              The page directory of the guest OS.
+ * @param   GCPtrOldMapping     The address of the start of the current mapping.
+ */
+int pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping)
+{
+    STAM_REL_COUNTER_INC(&pVM->pgm.s.cRelocations);
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
+
+    /* Raw mode only which implies one VCPU. */
+    Assert(pVM->cCpus == 1);
+
+    pMapping->aGCPtrConflicts[pMapping->cConflicts & (PGMMAPPING_CONFLICT_MAX-1)] = GCPtrOldMapping;
+    pMapping->cConflicts++;
+
+    /*
+     * Scan for free page directory entries.
+     *
+     * Note that we do not support mappings at the very end of the
+     * address space since that will break our GCPtrEnd assumptions.
+     */
+    const unsigned  cPTs = pMapping->cPTs;
+    unsigned        iPDNew = RT_ELEMENTS(pPDSrc->a) - cPTs; /* (+ 1 - 1) */
+    while (iPDNew-- > 0)
+    {
+        if (pPDSrc->a[iPDNew].n.u1Present)
+            continue;
+
+        if (pgmR3MapIsKnownConflictAddress(pMapping, iPDNew << X86_PD_SHIFT))
+            continue;
+
+        if (cPTs > 1)
+        {
+            bool fOk = true;
+            for (unsigned i = 1; fOk && i < cPTs; i++)
+                if (pPDSrc->a[iPDNew + i].n.u1Present)
+                    fOk = false;
+            if (!fOk)
+                continue;
+        }
+
+        /*
+         * Check that it's not conflicting with an intermediate page table mapping.
+         */
+        bool        fOk = true;
+        unsigned    i   = cPTs;
+        while (fOk && i-- > 0)
+            fOk = !pVM->pgm.s.pInterPD->a[iPDNew + i].n.u1Present;
+        if (!fOk)
+            continue;
+        /** @todo AMD64 should check the PAE directories and skip the 32bit stuff. */
+
+        /*
+         * Ask for the mapping.
+         */
+        RTGCPTR GCPtrNewMapping = (RTGCPTR32)iPDNew << X86_PD_SHIFT;
+
+        if (pMapping->pfnRelocate(pVM, GCPtrOldMapping, GCPtrNewMapping, PGMRELOCATECALL_SUGGEST, pMapping->pvUser))
+        {
+            pgmR3MapRelocate(pVM, pMapping, GCPtrOldMapping, GCPtrNewMapping);
+            STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
+            return VINF_SUCCESS;
+        }
+    }
+
+    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
+# ifdef DEBUG_bird
+    /*
+     * Ended up here frequently recently with an NT4.0 VM (using SMP kernel).
+     *
+     * The problem is when enabling large pages (i.e. updating CR4) using the
+     * _Ki386EnableCurrentLargePage@8 assembly routine (address 0x801c97ad-9).
+     * The routine loads a sparsely popuplated page tables with identiy mappings
+     * of its own code, most entries are whatever ExAllocatePool returned, which
+     * is documented as undefined but all 0xffffffff in this case.  Once loaded,
+     * it jumps to the physical code address, disables paging, set CR4.PSE=1,
+     * re-enables paging, restore the original page table and returns successfully.
+     *
+     * Theory: if CSAM/PATM patches the pushf;cli;mov eax,cr3; sequence, at the
+     * start of that function we're apparently in trouble, if CSAM/PATM doesn't
+     * we're switching back to REM and doing disabling of paging there instead.
+     *
+     * Normal PD: CR3=00030000; Problematic identity mapped PD: CR3=0x5fa000.
+     */
+    DBGFSTOP(pVM);
+# endif
+    AssertMsgFailed(("Failed to relocate page table mapping '%s' from %#x! (cPTs=%d)\n", pMapping->pszDesc, GCPtrOldMapping, cPTs));
+    return VERR_PGM_NO_HYPERVISOR_ADDRESS;
+}
+
+
+/**
+ * Resolves a conflict between a page table based GC mapping and
+ * the Guest OS page tables. (PAE bits version)
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pMapping            The mapping which conflicts.
+ * @param   GCPtrOldMapping     The address of the start of the current mapping.
+ */
+int pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping)
+{
+    STAM_REL_COUNTER_INC(&pVM->pgm.s.cRelocations);
+    STAM_PROFILE_START(&pVM->pgm.s.StatR3ResolveConflict, a);
+
+    /* Raw mode only which implies one VCPU. */
+    Assert(pVM->cCpus == 1);
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    pMapping->aGCPtrConflicts[pMapping->cConflicts & (PGMMAPPING_CONFLICT_MAX-1)] = GCPtrOldMapping;
+    pMapping->cConflicts++;
+
+    for (int iPDPTE = X86_PG_PAE_PDPE_ENTRIES - 1; iPDPTE >= 0; iPDPTE--)
+    {
+        unsigned  iPDSrc;
+        PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, (RTGCPTR32)iPDPTE << X86_PDPT_SHIFT, &iPDSrc, NULL);
+
+        /*
+         * Scan for free page directory entries.
+         *
+         * Note that we do not support mappings at the very end of the
+         * address space since that will break our GCPtrEnd assumptions.
+         * Nor do we support mappings crossing page directories.
+         */
+        const unsigned  cPTs = pMapping->cb >> X86_PD_PAE_SHIFT;
+        unsigned        iPDNew = RT_ELEMENTS(pPDSrc->a) - cPTs; /* (+ 1 - 1) */
+
+        while (iPDNew-- > 0)
+        {
+            /* Ugly assumption that mappings start on a 4 MB boundary. */
+            if (iPDNew & 1)
+                continue;
+
+            if (pgmR3MapIsKnownConflictAddress(pMapping, ((RTGCPTR32)iPDPTE << X86_PDPT_SHIFT) + (iPDNew << X86_PD_PAE_SHIFT)))
+                continue;
+
+            if (pPDSrc)
+            {
+                if (pPDSrc->a[iPDNew].n.u1Present)
+                    continue;
+                if (cPTs > 1)
+                {
+                    bool fOk = true;
+                    for (unsigned i = 1; fOk && i < cPTs; i++)
+                        if (pPDSrc->a[iPDNew + i].n.u1Present)
+                            fOk = false;
+                    if (!fOk)
+                        continue;
+                }
+            }
+            /*
+             * Check that it's not conflicting with an intermediate page table mapping.
+             */
+            bool        fOk = true;
+            unsigned    i   = cPTs;
+            while (fOk && i-- > 0)
+                fOk = !pVM->pgm.s.apInterPaePDs[iPDPTE]->a[iPDNew + i].n.u1Present;
+            if (!fOk)
+                continue;
+
+            /*
+             * Ask for the mapping.
+             */
+            RTGCPTR GCPtrNewMapping = ((RTGCPTR32)iPDPTE << X86_PDPT_SHIFT) + ((RTGCPTR32)iPDNew << X86_PD_PAE_SHIFT);
+
+            if (pMapping->pfnRelocate(pVM, GCPtrOldMapping, GCPtrNewMapping, PGMRELOCATECALL_SUGGEST, pMapping->pvUser))
+            {
+                pgmR3MapRelocate(pVM, pMapping, GCPtrOldMapping, GCPtrNewMapping);
+                STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
+                return VINF_SUCCESS;
+            }
+        }
+    }
+    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
+    AssertMsgFailed(("Failed to relocate page table mapping '%s' from %#x! (cPTs=%d)\n", pMapping->pszDesc, GCPtrOldMapping, pMapping->cb >> X86_PD_PAE_SHIFT));
+    return VERR_PGM_NO_HYPERVISOR_ADDRESS;
+}
+
+
+/**
+ * Read memory from the guest mappings.
+ *
+ * This will use the page tables associated with the mappings to
+ * read the memory. This means that not all kind of memory is readable
+ * since we don't necessarily know how to convert that physical address
+ * to a HC virtual one.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvDst       The destination address (HC of course).
+ * @param   GCPtrSrc    The source address (GC virtual address).
+ * @param   cb          Number of bytes to read.
+ *
+ * @remarks The is indirectly for DBGF only.
+ * @todo    Consider renaming it to indicate it's special usage, or just
+ *          reimplement it in MMR3HyperReadGCVirt.
+ */
+VMMR3DECL(int) PGMR3MapRead(PVM pVM, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
+{
+    /*
+     * Simplicity over speed... Chop the request up into chunks
+     * which don't cross pages.
+     */
+    if (cb + (GCPtrSrc & PAGE_OFFSET_MASK) > PAGE_SIZE)
+    {
+        for (;;)
+        {
+            size_t cbRead = RT_MIN(cb, PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK));
+            int rc = PGMR3MapRead(pVM, pvDst, GCPtrSrc, cbRead);
+            if (RT_FAILURE(rc))
+                return rc;
+            cb -= cbRead;
+            if (!cb)
+                break;
+            pvDst = (char *)pvDst + cbRead;
+            GCPtrSrc += cbRead;
+        }
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Find the mapping.
+     */
+    PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings);
+    while (pCur)
+    {
+        RTGCPTR off = GCPtrSrc - pCur->GCPtr;
+        if (off < pCur->cb)
+        {
+            if (off + cb > pCur->cb)
+            {
+                AssertMsgFailed(("Invalid page range %RGv LB%#x. mapping '%s' %RGv to %RGv\n",
+                                 GCPtrSrc, cb, pCur->pszDesc, pCur->GCPtr, pCur->GCPtrLast));
+                return VERR_INVALID_PARAMETER;
+            }
+
+            unsigned iPT  = off >> X86_PD_SHIFT;
+            unsigned iPTE = (off >> PAGE_SHIFT) & X86_PT_MASK;
+            while (cb > 0 && iPTE < RT_ELEMENTS(CTXALLSUFF(pCur->aPTs[iPT].pPT)->a))
+            {
+                PCPGMSHWPTEPAE pPte = &pCur->aPTs[iPT].CTXALLSUFF(paPaePTs)[iPTE / 512].a[iPTE % 512];
+                if (!PGMSHWPTEPAE_IS_P(*pPte))
+                    return VERR_PAGE_NOT_PRESENT;
+                RTHCPHYS HCPhys = PGMSHWPTEPAE_GET_HCPHYS(*pPte);
+
+                /*
+                 * Get the virtual page from the physical one.
+                 */
+                void *pvPage;
+                int rc = MMR3HCPhys2HCVirt(pVM, HCPhys, &pvPage);
+                if (RT_FAILURE(rc))
+                    return rc;
+
+                memcpy(pvDst, (char *)pvPage + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
+                return VINF_SUCCESS;
+            }
+        }
+
+        /* next */
+        pCur = CTXALLSUFF(pCur->pNext);
+    }
+
+    return VERR_INVALID_POINTER;
+}
+
+
+/**
+ * Info callback for 'pgmhandlers'.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The output helpers.
+ * @param   pszArgs     The arguments. phys or virt.
+ */
+DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    if (!pgmMapAreMappingsEnabled(pVM))
+        pHlp->pfnPrintf(pHlp, "\nThe mappings are DISABLED.\n");
+    else if (pVM->pgm.s.fMappingsFixed)
+        pHlp->pfnPrintf(pHlp, "\nThe mappings are FIXED: %RGv-%RGv\n",
+                        pVM->pgm.s.GCPtrMappingFixed, pVM->pgm.s.GCPtrMappingFixed + pVM->pgm.s.cbMappingFixed - 1);
+    else if (pVM->pgm.s.fMappingsFixedRestored)
+        pHlp->pfnPrintf(pHlp, "\nThe mappings are FLOATING-RESTORED-FIXED: %RGv-%RGv\n",
+                        pVM->pgm.s.GCPtrMappingFixed, pVM->pgm.s.GCPtrMappingFixed + pVM->pgm.s.cbMappingFixed - 1);
+    else
+        pHlp->pfnPrintf(pHlp, "\nThe mappings are FLOATING.\n");
+
+    PPGMMAPPING pCur;
+    for (pCur = pVM->pgm.s.pMappingsR3; pCur; pCur = pCur->pNextR3)
+    {
+        pHlp->pfnPrintf(pHlp, "%RGv - %RGv  %s\n", pCur->GCPtr, pCur->GCPtrLast, pCur->pszDesc);
+        if (pCur->cConflicts > 0)
+        {
+            pHlp->pfnPrintf(pHlp, "  %u conflict%s: ", pCur->cConflicts, pCur->cConflicts == 1 ? "" : "s");
+            uint32_t cLeft = RT_MIN(pCur->cConflicts, RT_ELEMENTS(pCur->aGCPtrConflicts));
+            uint32_t i     = pCur->cConflicts;
+            while (cLeft-- > 0)
+            {
+                i = (i - 1) & (PGMMAPPING_CONFLICT_MAX - 1);
+                pHlp->pfnPrintf(pHlp, cLeft ? "%RGv, " : "%RGv\n", pCur->aGCPtrConflicts[i]);
+            }
+        }
+    }
+}
+
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
diff --git a/src/VBox/VMM/VMMR3/PGMPhys.cpp b/src/VBox/VMM/VMMR3/PGMPhys.cpp
new file mode 100644
index 00000000..29d653b6
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PGMPhys.cpp
@@ -0,0 +1,5486 @@
+/* $Id: PGMPhys.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor, Physical Memory Addressing.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_PHYS
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/pdmdev.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include "PGMInline.h"
+
+#include <VBox/sup.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/asm.h>
+#ifdef VBOX_STRICT
+# include <iprt/crc.h>
+#endif
+#include <iprt/thread.h>
+#include <iprt/string.h>
+#include <iprt/system.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The number of pages to free in one batch. */
+#define PGMPHYS_FREE_PAGE_BATCH_SIZE    128
+
+
+/*
+ * PGMR3PhysReadU8-64
+ * PGMR3PhysWriteU8-64
+ */
+#define PGMPHYSFN_READNAME  PGMR3PhysReadU8
+#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
+#define PGMPHYS_DATASIZE    1
+#define PGMPHYS_DATATYPE    uint8_t
+#include "PGMPhysRWTmpl.h"
+
+#define PGMPHYSFN_READNAME  PGMR3PhysReadU16
+#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
+#define PGMPHYS_DATASIZE    2
+#define PGMPHYS_DATATYPE    uint16_t
+#include "PGMPhysRWTmpl.h"
+
+#define PGMPHYSFN_READNAME  PGMR3PhysReadU32
+#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
+#define PGMPHYS_DATASIZE    4
+#define PGMPHYS_DATATYPE    uint32_t
+#include "PGMPhysRWTmpl.h"
+
+#define PGMPHYSFN_READNAME  PGMR3PhysReadU64
+#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
+#define PGMPHYS_DATASIZE    8
+#define PGMPHYS_DATATYPE    uint64_t
+#include "PGMPhysRWTmpl.h"
+
+
+/**
+ * EMT worker for PGMR3PhysReadExternal.
+ */
+static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead,
+                                                  PGMACCESSORIGIN enmOrigin)
+{
+    VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead, enmOrigin);
+    AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); NOREF(rcStrict);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Read from physical memory, external users.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Physical address to read from.
+ * @param   pvBuf           Where to read into.
+ * @param   cbRead          How many bytes to read.
+ * @param   enmOrigin       Who is calling.
+ *
+ * @thread  Any but EMTs.
+ */
+VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin)
+{
+    VM_ASSERT_OTHER_THREAD(pVM);
+
+    AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
+    LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
+
+    pgmLock(pVM);
+
+    /*
+     * Copy loop on ram ranges.
+     */
+    PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
+    for (;;)
+    {
+        /* Inside range or not? */
+        if (pRam && GCPhys >= pRam->GCPhys)
+        {
+            /*
+             * Must work our way thru this page by page.
+             */
+            RTGCPHYS off = GCPhys - pRam->GCPhys;
+            while (off < pRam->cb)
+            {
+                unsigned iPage = off >> PAGE_SHIFT;
+                PPGMPAGE pPage = &pRam->aPages[iPage];
+
+                /*
+                 * If the page has an ALL access handler, we'll have to
+                 * delegate the job to EMT.
+                 */
+                if (   PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
+                    || PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+                {
+                    pgmUnlock(pVM);
+
+                    return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 5,
+                                                   pVM, &GCPhys, pvBuf, cbRead, enmOrigin);
+                }
+                Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
+
+                /*
+                 * Simple stuff, go ahead.
+                 */
+                size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
+                if (cb > cbRead)
+                    cb = cbRead;
+                PGMPAGEMAPLOCK PgMpLck;
+                const void    *pvSrc;
+                int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
+                if (RT_SUCCESS(rc))
+                {
+                    memcpy(pvBuf, pvSrc, cb);
+                    pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+                }
+                else
+                {
+                    AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
+                                           pRam->GCPhys + off, pPage, rc));
+                    memset(pvBuf, 0xff, cb);
+                }
+
+                /* next page */
+                if (cb >= cbRead)
+                {
+                    pgmUnlock(pVM);
+                    return VINF_SUCCESS;
+                }
+                cbRead -= cb;
+                off    += cb;
+                GCPhys += cb;
+                pvBuf   = (char *)pvBuf + cb;
+            } /* walk pages in ram range. */
+        }
+        else
+        {
+            LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
+
+            /*
+             * Unassigned address space.
+             */
+            size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
+            if (cb >= cbRead)
+            {
+                memset(pvBuf, 0xff, cbRead);
+                break;
+            }
+            memset(pvBuf, 0xff, cb);
+
+            cbRead -= cb;
+            pvBuf   = (char *)pvBuf + cb;
+            GCPhys += cb;
+        }
+
+        /* Advance range if necessary. */
+        while (pRam && GCPhys > pRam->GCPhysLast)
+            pRam = pRam->CTX_SUFF(pNext);
+    } /* Ram range walk */
+
+    pgmUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * EMT worker for PGMR3PhysWriteExternal.
+ */
+static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite,
+                                                   PGMACCESSORIGIN enmOrigin)
+{
+    /** @todo VERR_EM_NO_MEMORY */
+    VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite, enmOrigin);
+    AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); NOREF(rcStrict);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Write to physical memory, external users.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS.
+ * @retval  VERR_EM_NO_MEMORY.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Physical address to write to.
+ * @param   pvBuf           What to write.
+ * @param   cbWrite         How many bytes to write.
+ * @param   enmOrigin       Who is calling.
+ *
+ * @thread  Any but EMTs.
+ */
+VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin)
+{
+    VM_ASSERT_OTHER_THREAD(pVM);
+
+    AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
+              ("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x enmOrigin=%d\n",
+               GCPhys, cbWrite, enmOrigin));
+    AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
+    LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
+
+    pgmLock(pVM);
+
+    /*
+     * Copy loop on ram ranges, stop when we hit something difficult.
+     */
+    PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
+    for (;;)
+    {
+        /* Inside range or not? */
+        if (pRam && GCPhys >= pRam->GCPhys)
+        {
+            /*
+             * Must work our way thru this page by page.
+             */
+            RTGCPTR off = GCPhys - pRam->GCPhys;
+            while (off < pRam->cb)
+            {
+                RTGCPTR     iPage = off >> PAGE_SHIFT;
+                PPGMPAGE    pPage = &pRam->aPages[iPage];
+
+                /*
+                 * Is the page problematic, we have to do the work on the EMT.
+                 *
+                 * Allocating writable pages and access handlers are
+                 * problematic, write monitored pages are simple and can be
+                 * dealt with here.
+                 */
+                if (    PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
+                    ||  PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+                    ||  PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+                {
+                    if (    PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
+                        && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+                        pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
+                    else
+                    {
+                        pgmUnlock(pVM);
+
+                        return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 5,
+                                                       pVM, &GCPhys, pvBuf, cbWrite, enmOrigin);
+                    }
+                }
+                Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
+
+                /*
+                 * Simple stuff, go ahead.
+                 */
+                size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
+                if (cb > cbWrite)
+                    cb = cbWrite;
+                PGMPAGEMAPLOCK PgMpLck;
+                void          *pvDst;
+                int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
+                if (RT_SUCCESS(rc))
+                {
+                    memcpy(pvDst, pvBuf, cb);
+                    pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+                }
+                else
+                    AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
+                                           pRam->GCPhys + off, pPage, rc));
+
+                /* next page */
+                if (cb >= cbWrite)
+                {
+                    pgmUnlock(pVM);
+                    return VINF_SUCCESS;
+                }
+
+                cbWrite -= cb;
+                off     += cb;
+                GCPhys  += cb;
+                pvBuf    = (const char *)pvBuf + cb;
+            } /* walk pages in ram range */
+        }
+        else
+        {
+            /*
+             * Unassigned address space, skip it.
+             */
+            if (!pRam)
+                break;
+            size_t cb = pRam->GCPhys - GCPhys;
+            if (cb >= cbWrite)
+                break;
+            cbWrite -= cb;
+            pvBuf   = (const char *)pvBuf + cb;
+            GCPhys += cb;
+        }
+
+        /* Advance range if necessary. */
+        while (pRam && GCPhys > pRam->GCPhysLast)
+            pRam = pRam->CTX_SUFF(pNext);
+    } /* Ram range walk */
+
+    pgmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
+ *
+ * @returns see PGMR3PhysGCPhys2CCPtrExternal
+ * @param   pVM         The cross context VM structure.
+ * @param   pGCPhys     Pointer to the guest physical address.
+ * @param   ppv         Where to store the mapping address.
+ * @param   pLock       Where to store the lock.
+ */
+static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    /*
+     * Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
+     * an access handler after it succeeds.
+     */
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
+    if (RT_SUCCESS(rc))
+    {
+        PPGMPAGEMAPTLBE pTlbe;
+        int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
+        AssertFatalRC(rc2);
+        PPGMPAGE pPage = pTlbe->pPage;
+        if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
+        {
+            PGMPhysReleasePageMappingLock(pVM, pLock);
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+        }
+        else if (    PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+                 ||  pgmPoolIsDirtyPage(pVM, *pGCPhys)
+#endif
+                )
+        {
+            /* We *must* flush any corresponding pgm pool page here, otherwise we'll
+             * not be informed about writes and keep bogus gst->shw mappings around.
+             */
+            pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
+            Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
+            /** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
+             *        active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
+        }
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Requests the mapping of a guest page into ring-3, external threads.
+ *
+ * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
+ * release it.
+ *
+ * This API will assume your intention is to write to the page, and will
+ * therefore replace shared and zero pages. If you do not intend to modify the
+ * page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
+ *          backing or if the page has any active access handlers. The caller
+ *          must fall back on using PGMR3PhysWriteExternal.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address of the page that should be mapped.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that PGMPhysReleasePageMappingLock needs.
+ *
+ * @remark  Avoid calling this API from within critical sections (other than the
+ *          PGM one) because of the deadlock risk when we have to delegating the
+ *          task to an EMT.
+ * @thread  Any.
+ */
+VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    AssertPtr(ppv);
+    AssertPtr(pLock);
+
+    Assert(VM_IS_EMT(pVM) || !PGMIsLockOwner(pVM));
+
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Query the Physical TLB entry for the page (may fail).
+     */
+    PPGMPAGEMAPTLBE pTlbe;
+    rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
+    if (RT_SUCCESS(rc))
+    {
+        PPGMPAGE pPage = pTlbe->pPage;
+        if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+        else
+        {
+            /*
+             * If the page is shared, the zero page, or being write monitored
+             * it must be converted to an page that's writable if possible.
+             * We can only deal with write monitored pages here, the rest have
+             * to be on an EMT.
+             */
+            if (    PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
+                ||  PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+                ||  pgmPoolIsDirtyPage(pVM, GCPhys)
+#endif
+               )
+            {
+                if (    PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
+                    &&  !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+                    &&  !pgmPoolIsDirtyPage(pVM, GCPhys) /** @todo we're very likely doing this twice. */
+#endif
+                   )
+                    pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
+                else
+                {
+                    pgmUnlock(pVM);
+
+                    return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
+                                                   pVM, &GCPhys, ppv, pLock);
+                }
+            }
+
+            /*
+             * Now, just perform the locking and calculate the return address.
+             */
+            PPGMPAGEMAP pMap = pTlbe->pMap;
+            if (pMap)
+                pMap->cRefs++;
+
+            unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
+            if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
+            {
+                if (cLocks == 0)
+                    pVM->pgm.s.cWriteLockedPages++;
+                PGM_PAGE_INC_WRITE_LOCKS(pPage);
+            }
+            else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
+            {
+                PGM_PAGE_INC_WRITE_LOCKS(pPage);
+                AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
+                if (pMap)
+                    pMap->cRefs++; /* Extra ref to prevent it from going away. */
+            }
+
+            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+            pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE;
+            pLock->pvMap = pMap;
+        }
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Requests the mapping of a guest page into ring-3, external threads.
+ *
+ * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
+ * release it.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
+ *          backing or if the page as an active ALL access handler. The caller
+ *          must fall back on using PGMPhysRead.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address of the page that should be mapped.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that PGMPhysReleasePageMappingLock needs.
+ *
+ * @remark  Avoid calling this API from within critical sections (other than
+ *          the PGM one) because of the deadlock risk.
+ * @thread  Any.
+ */
+VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Query the Physical TLB entry for the page (may fail).
+     */
+    PPGMPAGEMAPTLBE pTlbe;
+    rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
+    if (RT_SUCCESS(rc))
+    {
+        PPGMPAGE pPage = pTlbe->pPage;
+#if 1
+        /* MMIO pages doesn't have any readable backing. */
+        if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+#else
+        if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+#endif
+        else
+        {
+            /*
+             * Now, just perform the locking and calculate the return address.
+             */
+            PPGMPAGEMAP pMap = pTlbe->pMap;
+            if (pMap)
+                pMap->cRefs++;
+
+            unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
+            if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
+            {
+                if (cLocks == 0)
+                    pVM->pgm.s.cReadLockedPages++;
+                PGM_PAGE_INC_READ_LOCKS(pPage);
+            }
+            else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
+            {
+                PGM_PAGE_INC_READ_LOCKS(pPage);
+                AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
+                if (pMap)
+                    pMap->cRefs++; /* Extra ref to prevent it from going away. */
+            }
+
+            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+            pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ;
+            pLock->pvMap = pMap;
+        }
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Requests the mapping of multiple guest page into ring-3, external threads.
+ *
+ * When you're done with the pages, call PGMPhysBulkReleasePageMappingLock()
+ * ASAP to release them.
+ *
+ * This API will assume your intention is to write to the pages, and will
+ * therefore replace shared and zero pages. If you do not intend to modify the
+ * pages, use the PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal() API.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED if any of the pages has no physical
+ *          backing or if any of the pages the page has any active access
+ *          handlers. The caller must fall back on using PGMR3PhysWriteExternal.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if @a paGCPhysPages contains
+ *          an invalid physical address.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   cPages          Number of pages to lock.
+ * @param   paGCPhysPages   The guest physical address of the pages that
+ *                          should be mapped (@a cPages entries).
+ * @param   papvPages       Where to store the ring-3 mapping addresses
+ *                          corresponding to @a paGCPhysPages.
+ * @param   paLocks         Where to store the locking information that
+ *                          pfnPhysBulkReleasePageMappingLock needs (@a cPages
+ *                          in length).
+ *
+ * @remark  Avoid calling this API from within critical sections (other than the
+ *          PGM one) because of the deadlock risk when we have to delegating the
+ *          task to an EMT.
+ * @thread  Any.
+ */
+VMMR3DECL(int) PGMR3PhysBulkGCPhys2CCPtrExternal(PVM pVM, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
+                                                 void **papvPages, PPGMPAGEMAPLOCK paLocks)
+{
+    Assert(cPages > 0);
+    AssertPtr(papvPages);
+    AssertPtr(paLocks);
+
+    Assert(VM_IS_EMT(pVM) || !PGMIsLockOwner(pVM));
+
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Lock the pages one by one.
+     * The loop body is similar to PGMR3PhysGCPhys2CCPtrExternal.
+     */
+    int32_t  cNextYield = 128;
+    uint32_t iPage;
+    for (iPage = 0; iPage < cPages; iPage++)
+    {
+        if (--cNextYield > 0)
+        { /* likely */ }
+        else
+        {
+            pgmUnlock(pVM);
+            ASMNopPause();
+            pgmLock(pVM);
+            cNextYield = 128;
+        }
+
+        /*
+         * Query the Physical TLB entry for the page (may fail).
+         */
+        PPGMPAGEMAPTLBE pTlbe;
+        rc = pgmPhysPageQueryTlbe(pVM, paGCPhysPages[iPage], &pTlbe);
+        if (RT_SUCCESS(rc))
+        { }
+        else
+            break;
+        PPGMPAGE pPage = pTlbe->pPage;
+
+        /*
+         * No MMIO or active access handlers.
+         */
+        if (   !PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)
+            && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+        { }
+        else
+        {
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+            break;
+        }
+
+        /*
+         * The page must be in the allocated state and not be a dirty pool page.
+         * We can handle converting a write monitored page to an allocated one, but
+         * anything more complicated must be delegated to an EMT.
+         */
+        bool fDelegateToEmt = false;
+        if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+            fDelegateToEmt = pgmPoolIsDirtyPage(pVM, paGCPhysPages[iPage]);
+#else
+            fDelegateToEmt = false;
+#endif
+        else if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
+        {
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+            if (!pgmPoolIsDirtyPage(pVM, paGCPhysPages[iPage]))
+                pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, paGCPhysPages[iPage]);
+            else
+                fDelegateToEmt = true;
+#endif
+        }
+        else
+            fDelegateToEmt = true;
+        if (!fDelegateToEmt)
+        { }
+        else
+        {
+            /* We could do this delegation in bulk, but considered too much work vs gain. */
+            pgmUnlock(pVM);
+            rc = VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
+                                         pVM, &paGCPhysPages[iPage], &papvPages[iPage], &paLocks[iPage]);
+            pgmLock(pVM);
+            if (RT_FAILURE(rc))
+                break;
+            cNextYield = 128;
+        }
+
+        /*
+         * Now, just perform the locking and address calculation.
+         */
+        PPGMPAGEMAP pMap = pTlbe->pMap;
+        if (pMap)
+            pMap->cRefs++;
+
+        unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
+        if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
+        {
+            if (cLocks == 0)
+                pVM->pgm.s.cWriteLockedPages++;
+            PGM_PAGE_INC_WRITE_LOCKS(pPage);
+        }
+        else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
+        {
+            PGM_PAGE_INC_WRITE_LOCKS(pPage);
+            AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", paGCPhysPages[iPage], pPage));
+            if (pMap)
+                pMap->cRefs++; /* Extra ref to prevent it from going away. */
+        }
+
+        papvPages[iPage] = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(paGCPhysPages[iPage] & PAGE_OFFSET_MASK));
+        paLocks[iPage].uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE;
+        paLocks[iPage].pvMap        = pMap;
+    }
+
+    pgmUnlock(pVM);
+
+    /*
+     * On failure we must unlock any pages we managed to get already.
+     */
+    if (RT_FAILURE(rc) && iPage > 0)
+        PGMPhysBulkReleasePageMappingLocks(pVM, iPage, paLocks);
+
+    return rc;
+}
+
+
+/**
+ * Requests the mapping of multiple guest page into ring-3, for reading only,
+ * external threads.
+ *
+ * When you're done with the pages, call PGMPhysReleasePageMappingLock() ASAP
+ * to release them.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED if any of the pages has no physical
+ *          backing or if any of the pages the page has an active ALL access
+ *          handler. The caller must fall back on using PGMR3PhysWriteExternal.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if @a paGCPhysPages contains
+ *          an invalid physical address.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   cPages          Number of pages to lock.
+ * @param   paGCPhysPages   The guest physical address of the pages that
+ *                          should be mapped (@a cPages entries).
+ * @param   papvPages       Where to store the ring-3 mapping addresses
+ *                          corresponding to @a paGCPhysPages.
+ * @param   paLocks         Where to store the lock information that
+ *                          pfnPhysReleasePageMappingLock needs (@a cPages
+ *                          in length).
+ *
+ * @remark  Avoid calling this API from within critical sections (other than
+ *          the PGM one) because of the deadlock risk.
+ * @thread  Any.
+ */
+VMMR3DECL(int) PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal(PVM pVM, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
+                                                         void const **papvPages, PPGMPAGEMAPLOCK paLocks)
+{
+    Assert(cPages > 0);
+    AssertPtr(papvPages);
+    AssertPtr(paLocks);
+
+    Assert(VM_IS_EMT(pVM) || !PGMIsLockOwner(pVM));
+
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Lock the pages one by one.
+     * The loop body is similar to PGMR3PhysGCPhys2CCPtrReadOnlyExternal.
+     */
+    int32_t  cNextYield = 256;
+    uint32_t iPage;
+    for (iPage = 0; iPage < cPages; iPage++)
+    {
+        if (--cNextYield > 0)
+        { /* likely */ }
+        else
+        {
+            pgmUnlock(pVM);
+            ASMNopPause();
+            pgmLock(pVM);
+            cNextYield = 256;
+        }
+
+        /*
+         * Query the Physical TLB entry for the page (may fail).
+         */
+        PPGMPAGEMAPTLBE pTlbe;
+        rc = pgmPhysPageQueryTlbe(pVM, paGCPhysPages[iPage], &pTlbe);
+        if (RT_SUCCESS(rc))
+        { }
+        else
+            break;
+        PPGMPAGE pPage = pTlbe->pPage;
+
+        /*
+         * No MMIO or active all access handlers, everything else can be accessed.
+         */
+        if (   !PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)
+            && !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+        { }
+        else
+        {
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+            break;
+        }
+
+        /*
+         * Now, just perform the locking and address calculation.
+         */
+        PPGMPAGEMAP pMap = pTlbe->pMap;
+        if (pMap)
+            pMap->cRefs++;
+
+        unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
+        if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
+        {
+            if (cLocks == 0)
+                pVM->pgm.s.cReadLockedPages++;
+            PGM_PAGE_INC_READ_LOCKS(pPage);
+        }
+        else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
+        {
+            PGM_PAGE_INC_READ_LOCKS(pPage);
+            AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", paGCPhysPages[iPage], pPage));
+            if (pMap)
+                pMap->cRefs++; /* Extra ref to prevent it from going away. */
+        }
+
+        papvPages[iPage] = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(paGCPhysPages[iPage] & PAGE_OFFSET_MASK));
+        paLocks[iPage].uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ;
+        paLocks[iPage].pvMap        = pMap;
+    }
+
+    pgmUnlock(pVM);
+
+    /*
+     * On failure we must unlock any pages we managed to get already.
+     */
+    if (RT_FAILURE(rc) && iPage > 0)
+        PGMPhysBulkReleasePageMappingLocks(pVM, iPage, paLocks);
+
+    return rc;
+}
+
+
+#define MAKE_LEAF(a_pNode) \
+    do { \
+        (a_pNode)->pLeftR3  = NIL_RTR3PTR; \
+        (a_pNode)->pRightR3 = NIL_RTR3PTR; \
+        (a_pNode)->pLeftR0  = NIL_RTR0PTR; \
+        (a_pNode)->pRightR0 = NIL_RTR0PTR; \
+    } while (0)
+
+#define INSERT_LEFT(a_pParent, a_pNode) \
+    do { \
+        (a_pParent)->pLeftR3 = (a_pNode); \
+        (a_pParent)->pLeftR0 = (a_pNode)->pSelfR0; \
+    } while (0)
+#define INSERT_RIGHT(a_pParent, a_pNode) \
+    do { \
+        (a_pParent)->pRightR3 = (a_pNode); \
+        (a_pParent)->pRightR0 = (a_pNode)->pSelfR0; \
+    } while (0)
+
+
+/**
+ * Recursive tree builder.
+ *
+ * @param   ppRam           Pointer to the iterator variable.
+ * @param   iDepth          The current depth.  Inserts a leaf node if 0.
+ */
+static PPGMRAMRANGE pgmR3PhysRebuildRamRangeSearchTreesRecursively(PPGMRAMRANGE *ppRam, int iDepth)
+{
+    PPGMRAMRANGE pRam;
+    if (iDepth <= 0)
+    {
+        /*
+         * Leaf node.
+         */
+        pRam = *ppRam;
+        if (pRam)
+        {
+            *ppRam = pRam->pNextR3;
+            MAKE_LEAF(pRam);
+        }
+    }
+    else
+    {
+
+        /*
+         * Intermediate node.
+         */
+        PPGMRAMRANGE pLeft = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
+
+        pRam = *ppRam;
+        if (!pRam)
+            return pLeft;
+        *ppRam = pRam->pNextR3;
+        MAKE_LEAF(pRam);
+        INSERT_LEFT(pRam, pLeft);
+
+        PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
+        if (pRight)
+            INSERT_RIGHT(pRam, pRight);
+    }
+    return pRam;
+}
+
+
+/**
+ * Rebuilds the RAM range search trees.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+static void pgmR3PhysRebuildRamRangeSearchTrees(PVM pVM)
+{
+
+    /*
+     * Create the reasonably balanced tree in a sequential fashion.
+     * For simplicity (laziness) we use standard recursion here.
+     */
+    int             iDepth = 0;
+    PPGMRAMRANGE    pRam   = pVM->pgm.s.pRamRangesXR3;
+    PPGMRAMRANGE    pRoot  = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, 0);
+    while (pRam)
+    {
+        PPGMRAMRANGE pLeft = pRoot;
+
+        pRoot = pRam;
+        pRam = pRam->pNextR3;
+        MAKE_LEAF(pRoot);
+        INSERT_LEFT(pRoot, pLeft);
+
+        PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, iDepth);
+        if (pRight)
+            INSERT_RIGHT(pRoot, pRight);
+        /** @todo else: rotate the tree. */
+
+        iDepth++;
+    }
+
+    pVM->pgm.s.pRamRangeTreeR3 = pRoot;
+    pVM->pgm.s.pRamRangeTreeR0 = pRoot ? pRoot->pSelfR0 : NIL_RTR0PTR;
+
+#ifdef VBOX_STRICT
+    /*
+     * Verify that the above code works.
+     */
+    unsigned cRanges = 0;
+    for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
+        cRanges++;
+    Assert(cRanges > 0);
+
+    unsigned cMaxDepth = ASMBitLastSetU32(cRanges);
+    if ((1U << cMaxDepth) < cRanges)
+        cMaxDepth++;
+
+    for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
+    {
+        unsigned     cDepth = 0;
+        PPGMRAMRANGE pRam2 = pVM->pgm.s.pRamRangeTreeR3;
+        for (;;)
+        {
+            if (pRam == pRam2)
+                break;
+            Assert(pRam2);
+            if (pRam->GCPhys < pRam2->GCPhys)
+                pRam2 = pRam2->pLeftR3;
+            else
+                pRam2 = pRam2->pRightR3;
+        }
+        AssertMsg(cDepth <= cMaxDepth, ("cDepth=%d cMaxDepth=%d\n", cDepth, cMaxDepth));
+    }
+#endif /* VBOX_STRICT */
+}
+
+#undef MAKE_LEAF
+#undef INSERT_LEFT
+#undef INSERT_RIGHT
+
+/**
+ * Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
+ *
+ * Called when anything was relocated.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+void pgmR3PhysRelinkRamRanges(PVM pVM)
+{
+    PPGMRAMRANGE pCur;
+
+#ifdef VBOX_STRICT
+    for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
+    {
+        Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
+        Assert((pCur->GCPhys     & PAGE_OFFSET_MASK) == 0);
+        Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
+        Assert((pCur->cb         & PAGE_OFFSET_MASK) == 0);
+        Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
+        for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesXR3; pCur2; pCur2 = pCur2->pNextR3)
+            Assert(   pCur2 == pCur
+                   || strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
+    }
+#endif
+
+    pCur = pVM->pgm.s.pRamRangesXR3;
+    if (pCur)
+    {
+        pVM->pgm.s.pRamRangesXR0 = pCur->pSelfR0;
+
+        for (; pCur->pNextR3; pCur = pCur->pNextR3)
+            pCur->pNextR0 = pCur->pNextR3->pSelfR0;
+
+        Assert(pCur->pNextR0 == NIL_RTR0PTR);
+    }
+    else
+    {
+        Assert(pVM->pgm.s.pRamRangesXR0 == NIL_RTR0PTR);
+    }
+    ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
+
+    pgmR3PhysRebuildRamRangeSearchTrees(pVM);
+}
+
+
+/**
+ * Links a new RAM range into the list.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pNew        Pointer to the new list entry.
+ * @param   pPrev       Pointer to the previous list entry. If NULL, insert as head.
+ */
+static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
+{
+    AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
+    Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
+
+    pgmLock(pVM);
+
+    PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesXR3;
+    pNew->pNextR3 = pRam;
+    pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
+
+    if (pPrev)
+    {
+        pPrev->pNextR3 = pNew;
+        pPrev->pNextR0 = pNew->pSelfR0;
+    }
+    else
+    {
+        pVM->pgm.s.pRamRangesXR3 = pNew;
+        pVM->pgm.s.pRamRangesXR0 = pNew->pSelfR0;
+    }
+    ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
+
+    pgmR3PhysRebuildRamRangeSearchTrees(pVM);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Unlink an existing RAM range from the list.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pRam        Pointer to the new list entry.
+ * @param   pPrev       Pointer to the previous list entry. If NULL, insert as head.
+ */
+static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
+{
+    Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesXR3 == pRam);
+    Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
+
+    pgmLock(pVM);
+
+    PPGMRAMRANGE pNext = pRam->pNextR3;
+    if (pPrev)
+    {
+        pPrev->pNextR3 = pNext;
+        pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
+    }
+    else
+    {
+        Assert(pVM->pgm.s.pRamRangesXR3 == pRam);
+        pVM->pgm.s.pRamRangesXR3 = pNext;
+        pVM->pgm.s.pRamRangesXR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
+    }
+    ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
+
+    pgmR3PhysRebuildRamRangeSearchTrees(pVM);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Unlink an existing RAM range from the list.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pRam        Pointer to the new list entry.
+ */
+static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
+{
+    pgmLock(pVM);
+
+    /* find prev. */
+    PPGMRAMRANGE pPrev = NULL;
+    PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3;
+    while (pCur != pRam)
+    {
+        pPrev = pCur;
+        pCur = pCur->pNextR3;
+    }
+    AssertFatal(pCur);
+
+    pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Frees a range of pages, replacing them with ZERO pages of the specified type.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pRam        The RAM range in which the pages resides.
+ * @param   GCPhys      The address of the first page.
+ * @param   GCPhysLast  The address of the last page.
+ * @param   enmType     The page type to replace then with.
+ */
+static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, PGMPAGETYPE enmType)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    uint32_t            cPendingPages = 0;
+    PGMMFREEPAGESREQ    pReq;
+    int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+    AssertLogRelRCReturn(rc, rc);
+
+    /* Iterate the pages. */
+    PPGMPAGE pPageDst   = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+    uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
+    while (cPagesLeft-- > 0)
+    {
+        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys, enmType);
+        AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
+
+        PGM_PAGE_SET_TYPE(pVM, pPageDst, enmType);
+
+        GCPhys += PAGE_SIZE;
+        pPageDst++;
+    }
+
+    if (cPendingPages)
+    {
+        rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+        AssertLogRelRCReturn(rc, rc);
+    }
+    GMMR3FreePagesCleanup(pReq);
+
+    return rc;
+}
+
+#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
+
+/**
+ * Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
+ *
+ * This is only called on one of the EMTs while the other ones are waiting for
+ * it to complete this function.
+ *
+ * @returns VINF_SUCCESS (VBox strict status code).
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT. Unused.
+ * @param   pvUser      User parameter
+ */
+static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    uintptr_t          *paUser          = (uintptr_t *)pvUser;
+    bool                fInflate        = !!paUser[0];
+    unsigned            cPages          = paUser[1];
+    RTGCPHYS           *paPhysPage      = (RTGCPHYS *)paUser[2];
+    uint32_t            cPendingPages   = 0;
+    PGMMFREEPAGESREQ    pReq;
+    int                 rc;
+
+    Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
+    pgmLock(pVM);
+
+    if (fInflate)
+    {
+        /* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
+        pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
+
+        /* Replace pages with ZERO pages. */
+        rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+        if (RT_FAILURE(rc))
+        {
+            pgmUnlock(pVM);
+            AssertLogRelRC(rc);
+            return rc;
+        }
+
+        /* Iterate the pages. */
+        for (unsigned i = 0; i < cPages; i++)
+        {
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
+            if (    pPage == NULL
+                ||  PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
+            {
+                Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
+                break;
+            }
+
+            LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
+
+            /* Flush the shadow PT if this page was previously used as a guest page table. */
+            pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
+
+            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i], (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage));
+            if (RT_FAILURE(rc))
+            {
+                pgmUnlock(pVM);
+                AssertLogRelRC(rc);
+                return rc;
+            }
+            Assert(PGM_PAGE_IS_ZERO(pPage));
+            PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
+        }
+
+        if (cPendingPages)
+        {
+            rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+            if (RT_FAILURE(rc))
+            {
+                pgmUnlock(pVM);
+                AssertLogRelRC(rc);
+                return rc;
+            }
+        }
+        GMMR3FreePagesCleanup(pReq);
+    }
+    else
+    {
+        /* Iterate the pages. */
+        for (unsigned i = 0; i < cPages; i++)
+        {
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
+            AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
+
+            LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
+
+            Assert(PGM_PAGE_IS_BALLOONED(pPage));
+
+            /* Change back to zero page.  (NEM does not need to be informed.) */
+            PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
+        }
+
+        /* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
+    }
+
+    /* Notify GMM about the balloon change. */
+    rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
+    if (RT_SUCCESS(rc))
+    {
+        if (!fInflate)
+        {
+            Assert(pVM->pgm.s.cBalloonedPages >= cPages);
+            pVM->pgm.s.cBalloonedPages -= cPages;
+        }
+        else
+            pVM->pgm.s.cBalloonedPages += cPages;
+    }
+
+    pgmUnlock(pVM);
+
+    /* Flush the recompiler's TLB as well. */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        CPUMSetChangedFlags(pVM->apCpusR3[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
+
+    AssertLogRelRC(rc);
+    return rc;
+}
+
+
+/**
+ * Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   fInflate    Inflate or deflate memory balloon
+ * @param   cPages      Number of pages to free
+ * @param   paPhysPage  Array of guest physical addresses
+ */
+static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
+{
+    uintptr_t paUser[3];
+
+    paUser[0] = fInflate;
+    paUser[1] = cPages;
+    paUser[2] = (uintptr_t)paPhysPage;
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
+    AssertRC(rc);
+
+    /* Made a copy in PGMR3PhysFreeRamPages; free it here. */
+    RTMemFree(paPhysPage);
+}
+
+#endif /* 64-bit host && (Windows || Solaris || Linux || FreeBSD) */
+
+/**
+ * Inflate or deflate a memory balloon
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   fInflate    Inflate or deflate memory balloon
+ * @param   cPages      Number of pages to free
+ * @param   paPhysPage  Array of guest physical addresses
+ */
+VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
+{
+    /* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
+#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
+    int rc;
+
+    /* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
+    AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
+
+    /* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
+     * In the SMP case we post a request packet to postpone the job.
+     */
+    if (pVM->cCpus > 1)
+    {
+        unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
+        RTGCPHYS *paPhysPageCopy = (RTGCPHYS *)RTMemAlloc(cbPhysPage);
+        AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
+
+        memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
+
+        rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
+        AssertRC(rc);
+    }
+    else
+    {
+        uintptr_t paUser[3];
+
+        paUser[0] = fInflate;
+        paUser[1] = cPages;
+        paUser[2] = (uintptr_t)paPhysPage;
+        rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
+        AssertRC(rc);
+    }
+    return rc;
+
+#else
+    NOREF(pVM); NOREF(fInflate); NOREF(cPages); NOREF(paPhysPage);
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/**
+ * Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
+ * physical RAM.
+ *
+ * This is only called on one of the EMTs while the other ones are waiting for
+ * it to complete this function.
+ *
+ * @returns VINF_SUCCESS (VBox strict status code).
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT. Unused.
+ * @param   pvUser      User parameter, unused.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    NOREF(pvUser); NOREF(pVCpu);
+
+    pgmLock(pVM);
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    pgmPoolResetDirtyPages(pVM);
+#endif
+
+    /** @todo pointless to write protect the physical page pointed to by RSP. */
+
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        uint32_t cPages = pRam->cb >> PAGE_SHIFT;
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            PPGMPAGE    pPage = &pRam->aPages[iPage];
+            PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+
+            if (    RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
+                ||  enmPageType == PGMPAGETYPE_MMIO2)
+            {
+                /*
+                 * A RAM page.
+                 */
+                switch (PGM_PAGE_GET_STATE(pPage))
+                {
+                    case PGM_PAGE_STATE_ALLOCATED:
+                        /** @todo Optimize this: Don't always re-enable write
+                         * monitoring if the page is known to be very busy. */
+                        if (PGM_PAGE_IS_WRITTEN_TO(pPage))
+                            PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
+
+                        pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
+                        break;
+
+                    case PGM_PAGE_STATE_SHARED:
+                        AssertFailed();
+                        break;
+
+                    case PGM_PAGE_STATE_WRITE_MONITORED:    /* nothing to change. */
+                    default:
+                        break;
+                }
+            }
+        }
+    }
+    pgmR3PoolWriteProtectPages(pVM);
+    PGM_INVL_ALL_VCPU_TLBS(pVM);
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        CPUMSetChangedFlags(pVM->apCpusR3[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+/**
+ * Protect all physical RAM to monitor writes
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
+{
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
+    AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * Gets the number of ram ranges.
+ *
+ * @returns Number of ram ranges.  Returns UINT32_MAX if @a pVM is invalid.
+ * @param   pVM             The cross context VM structure.
+ */
+VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
+
+    pgmLock(pVM);
+    uint32_t cRamRanges = 0;
+    for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext))
+        cRamRanges++;
+    pgmUnlock(pVM);
+    return cRamRanges;
+}
+
+
+/**
+ * Get information about a range.
+ *
+ * @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
+ * @param   pVM             The cross context VM structure.
+ * @param   iRange          The ordinal of the range.
+ * @param   pGCPhysStart    Where to return the start of the range. Optional.
+ * @param   pGCPhysLast     Where to return the address of the last byte in the
+ *                          range. Optional.
+ * @param   ppszDesc        Where to return the range description. Optional.
+ * @param   pfIsMmio        Where to indicate that this is a pure MMIO range.
+ *                          Optional.
+ */
+VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
+                                 const char **ppszDesc, bool *pfIsMmio)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    pgmLock(pVM);
+    uint32_t iCurRange = 0;
+    for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
+        if (iCurRange == iRange)
+        {
+            if (pGCPhysStart)
+                *pGCPhysStart = pCur->GCPhys;
+            if (pGCPhysLast)
+                *pGCPhysLast  = pCur->GCPhysLast;
+            if (ppszDesc)
+                *ppszDesc     = pCur->pszDesc;
+            if (pfIsMmio)
+                *pfIsMmio     = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
+
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+    pgmUnlock(pVM);
+    return VERR_OUT_OF_RANGE;
+}
+
+
+/**
+ * Query the amount of free memory inside VMMR0
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pcbAllocMem         Where to return the amount of memory allocated
+ *                              by VMs.
+ * @param   pcbFreeMem          Where to return the amount of memory that is
+ *                              allocated from the host but not currently used
+ *                              by any VMs.
+ * @param   pcbBallonedMem      Where to return the sum of memory that is
+ *                              currently ballooned by the VMs.
+ * @param   pcbSharedMem        Where to return the amount of memory that is
+ *                              currently shared.
+ */
+VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PUVM pUVM, uint64_t *pcbAllocMem, uint64_t *pcbFreeMem,
+                                           uint64_t *pcbBallonedMem, uint64_t *pcbSharedMem)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+
+    uint64_t cAllocPages   = 0;
+    uint64_t cFreePages    = 0;
+    uint64_t cBalloonPages = 0;
+    uint64_t cSharedPages  = 0;
+    int rc = GMMR3QueryHypervisorMemoryStats(pUVM->pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
+    AssertRCReturn(rc, rc);
+
+    if (pcbAllocMem)
+        *pcbAllocMem    = cAllocPages * _4K;
+
+    if (pcbFreeMem)
+        *pcbFreeMem     = cFreePages * _4K;
+
+    if (pcbBallonedMem)
+        *pcbBallonedMem = cBalloonPages * _4K;
+
+    if (pcbSharedMem)
+        *pcbSharedMem   = cSharedPages * _4K;
+
+    Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
+         cAllocPages, cFreePages, cBalloonPages, cSharedPages));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Query memory stats for the VM.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pcbTotalMem         Where to return total amount memory the VM may
+ *                              possibly use.
+ * @param   pcbPrivateMem       Where to return the amount of private memory
+ *                              currently allocated.
+ * @param   pcbSharedMem        Where to return the amount of actually shared
+ *                              memory currently used by the VM.
+ * @param   pcbZeroMem          Where to return the amount of memory backed by
+ *                              zero pages.
+ *
+ * @remarks The total mem is normally larger than the sum of the three
+ *          components.  There are two reasons for this, first the amount of
+ *          shared memory is what we're sure is shared instead of what could
+ *          possibly be shared with someone.  Secondly, because the total may
+ *          include some pure MMIO pages that doesn't go into any of the three
+ *          sub-counts.
+ *
+ * @todo Why do we return reused shared pages instead of anything that could
+ *       potentially be shared?  Doesn't this mean the first VM gets a much
+ *       lower number of shared pages?
+ */
+VMMR3DECL(int) PGMR3QueryMemoryStats(PUVM pUVM, uint64_t *pcbTotalMem, uint64_t *pcbPrivateMem,
+                                     uint64_t *pcbSharedMem, uint64_t *pcbZeroMem)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    if (pcbTotalMem)
+        *pcbTotalMem    = (uint64_t)pVM->pgm.s.cAllPages            * PAGE_SIZE;
+
+    if (pcbPrivateMem)
+        *pcbPrivateMem  = (uint64_t)pVM->pgm.s.cPrivatePages        * PAGE_SIZE;
+
+    if (pcbSharedMem)
+        *pcbSharedMem   = (uint64_t)pVM->pgm.s.cReusedSharedPages   * PAGE_SIZE;
+
+    if (pcbZeroMem)
+        *pcbZeroMem     = (uint64_t)pVM->pgm.s.cZeroPages           * PAGE_SIZE;
+
+    Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * PGMR3PhysRegisterRam worker that initializes and links a RAM range.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pNew            The new RAM range.
+ * @param   GCPhys          The address of the RAM range.
+ * @param   GCPhysLast      The last address of the RAM range.
+ * @param   RCPtrNew        The RC address if the range is floating. NIL_RTRCPTR
+ *                          if in HMA.
+ * @param   R0PtrNew        Ditto for R0.
+ * @param   pszDesc         The description.
+ * @param   pPrev           The previous RAM range (for linking).
+ */
+static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
+                                         RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
+{
+    /*
+     * Initialize the range.
+     */
+    pNew->pSelfR0       = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
+    pNew->GCPhys        = GCPhys;
+    pNew->GCPhysLast    = GCPhysLast;
+    pNew->cb            = GCPhysLast - GCPhys + 1;
+    pNew->pszDesc       = pszDesc;
+    pNew->fFlags        = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
+    pNew->pvR3          = NULL;
+    pNew->paLSPages     = NULL;
+
+    uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
+    RTGCPHYS iPage = cPages;
+    while (iPage-- > 0)
+        PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
+
+    /* Update the page count stats. */
+    pVM->pgm.s.cZeroPages += cPages;
+    pVM->pgm.s.cAllPages  += cPages;
+
+    /*
+     * Link it.
+     */
+    pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
+}
+
+
+#ifndef PGM_WITHOUT_MAPPINGS
+/**
+ * @callback_method_impl{FNPGMRELOCATE, Relocate a floating RAM range.}
+ * @sa pgmR3PhysMMIO2ExRangeRelocate
+ */
+static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew,
+                                                    PGMRELOCATECALL enmMode, void *pvUser)
+{
+    PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
+    Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
+    Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE); RT_NOREF_PV(GCPtrOld);
+
+    switch (enmMode)
+    {
+        case PGMRELOCATECALL_SUGGEST:
+            return true;
+
+        case PGMRELOCATECALL_RELOCATE:
+        {
+            /*
+             * Update myself, then relink all the ranges and flush the RC TLB.
+             */
+            pgmLock(pVM);
+
+            pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
+
+            pgmR3PhysRelinkRamRanges(pVM);
+            for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
+                pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
+
+            pgmUnlock(pVM);
+            return true;
+        }
+
+        default:
+            AssertFailedReturn(false);
+    }
+}
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
+
+/**
+ * PGMR3PhysRegisterRam worker that registers a high chunk.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The address of the RAM.
+ * @param   cRamPages       The number of RAM pages to register.
+ * @param   cbChunk         The size of the PGMRAMRANGE guest mapping.
+ * @param   iChunk          The chunk number.
+ * @param   pszDesc         The RAM range description.
+ * @param   ppPrev          Previous RAM range pointer. In/Out.
+ */
+static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
+                                         uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
+                                         PPGMRAMRANGE *ppPrev)
+{
+    const char *pszDescChunk = iChunk == 0
+                             ? pszDesc
+                             : MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
+    AssertReturn(pszDescChunk, VERR_NO_MEMORY);
+
+    /*
+     * Allocate memory for the new chunk.
+     */
+    size_t const cChunkPages  = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
+    PSUPPAGE     paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
+    AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
+    RTR0PTR      R0PtrChunk   = NIL_RTR0PTR;
+    void        *pvChunk      = NULL;
+    int rc = SUPR3PageAllocEx(cChunkPages, 0 /*fFlags*/, &pvChunk, &R0PtrChunk, paChunkPages);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(R0PtrChunk != NIL_RTR0PTR);
+        memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
+
+        PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
+
+        /*
+         * Create a mapping and map the pages into it.
+         * We push these in below the HMA.
+         */
+        RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
+#ifndef PGM_WITHOUT_MAPPINGS
+        rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /*fFlags*/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
+        if (RT_SUCCESS(rc))
+#endif /* !PGM_WITHOUT_MAPPINGS */
+        {
+            pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
+
+            RTGCPTR const   GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
+#ifndef PGM_WITHOUT_MAPPINGS
+            RTGCPTR         GCPtrPage  = GCPtrChunk;
+            for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
+                rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
+            if (RT_SUCCESS(rc))
+#endif /* !PGM_WITHOUT_MAPPINGS */
+            {
+                /*
+                 * Ok, init and link the range.
+                 */
+                pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
+                                             (RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
+                *ppPrev = pNew;
+            }
+        }
+
+        if (RT_FAILURE(rc))
+            SUPR3PageFreeEx(pvChunk, cChunkPages);
+    }
+
+    RTMemTmpFree(paChunkPages);
+    return rc;
+}
+
+
+/**
+ * Sets up a range RAM.
+ *
+ * This will check for conflicting registrations, make a resource
+ * reservation for the memory (with GMM), and setup the per-page
+ * tracking structures (PGMPAGE).
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The physical address of the RAM.
+ * @param   cb              The size of the RAM.
+ * @param   pszDesc         The description - not copied, so, don't free or change it.
+ */
+VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
+{
+   /*
+     * Validate input.
+     */
+    Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
+    AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
+    AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
+    AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
+    RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
+    AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    pgmLock(pVM);
+
+    /*
+     * Find range location and check for conflicts.
+     * (We don't lock here because the locking by EMT is only required on update.)
+     */
+    PPGMRAMRANGE    pPrev = NULL;
+    PPGMRAMRANGE    pRam = pVM->pgm.s.pRamRangesXR3;
+    while (pRam && GCPhysLast >= pRam->GCPhys)
+    {
+        if (    GCPhysLast >= pRam->GCPhys
+            &&  GCPhys     <= pRam->GCPhysLast)
+            AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
+                                         GCPhys, GCPhysLast, pszDesc,
+                                         pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
+                                        VERR_PGM_RAM_CONFLICT);
+
+        /* next */
+        pPrev = pRam;
+        pRam = pRam->pNextR3;
+    }
+
+    /*
+     * Register it with GMM (the API bitches).
+     */
+    const RTGCPHYS cPages = cb >> PAGE_SHIFT;
+    int rc = MMR3IncreaseBaseReservation(pVM, cPages);
+    if (RT_FAILURE(rc))
+    {
+        pgmUnlock(pVM);
+        return rc;
+    }
+
+    if (    GCPhys >= _4G
+        &&  cPages > 256)
+    {
+        /*
+         * The PGMRAMRANGE structures for the high memory can get very big.
+         * In order to avoid SUPR3PageAllocEx allocation failures due to the
+         * allocation size limit there and also to avoid being unable to find
+         * guest mapping space for them, we split this memory up into 4MB in
+         * (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
+         * mode.
+         *
+         * The first and last page of each mapping are guard pages and marked
+         * not-present. So, we've got 4186112 and 16769024 bytes available for
+         * the PGMRAMRANGE structure.
+         *
+         * Note! The sizes used here will influence the saved state.
+         */
+        uint32_t cbChunk = 16U*_1M;
+        uint32_t cPagesPerChunk = 1048048; /* max ~1048059 */
+        AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U*_1M - PAGE_SIZE * 2);
+        AssertRelease(RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
+
+        RTGCPHYS cPagesLeft  = cPages;
+        RTGCPHYS GCPhysChunk = GCPhys;
+        uint32_t iChunk      = 0;
+        while (cPagesLeft > 0)
+        {
+            uint32_t cPagesInChunk = cPagesLeft;
+            if (cPagesInChunk > cPagesPerChunk)
+                cPagesInChunk = cPagesPerChunk;
+
+            rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
+            AssertRCReturn(rc, rc);
+
+            /* advance */
+            GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
+            cPagesLeft  -= cPagesInChunk;
+            iChunk++;
+        }
+    }
+    else
+    {
+        /*
+         * Allocate, initialize and link the new RAM range.
+         */
+        const size_t cbRamRange = RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]);
+        PPGMRAMRANGE pNew;
+        rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
+        AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
+
+        pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
+    }
+    pgmPhysInvalidatePageMapTLB(pVM);
+
+    /*
+     * Notify NEM while holding the lock (experimental) and REM without (like always).
+     */
+    rc = NEMR3NotifyPhysRamRegister(pVM, GCPhys, cb);
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
+ *
+ * We do this late in the init process so that all the ROM and MMIO ranges have
+ * been registered already and we don't go wasting memory on them.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+int pgmR3PhysRamPreAllocate(PVM pVM)
+{
+    Assert(pVM->pgm.s.fRamPreAlloc);
+    Log(("pgmR3PhysRamPreAllocate: enter\n"));
+
+    /*
+     * Walk the RAM ranges and allocate all RAM pages, halt at
+     * the first allocation error.
+     */
+    uint64_t cPages = 0;
+    uint64_t NanoTS = RTTimeNanoTS();
+    pgmLock(pVM);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
+    {
+        PPGMPAGE    pPage  = &pRam->aPages[0];
+        RTGCPHYS    GCPhys = pRam->GCPhys;
+        uint32_t    cLeft  = pRam->cb >> PAGE_SHIFT;
+        while (cLeft-- > 0)
+        {
+            if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
+            {
+                switch (PGM_PAGE_GET_STATE(pPage))
+                {
+                    case PGM_PAGE_STATE_ZERO:
+                    {
+                        int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
+                        if (RT_FAILURE(rc))
+                        {
+                            LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
+                            pgmUnlock(pVM);
+                            return rc;
+                        }
+                        cPages++;
+                        break;
+                    }
+
+                    case PGM_PAGE_STATE_BALLOONED:
+                    case PGM_PAGE_STATE_ALLOCATED:
+                    case PGM_PAGE_STATE_WRITE_MONITORED:
+                    case PGM_PAGE_STATE_SHARED:
+                        /* nothing to do here. */
+                        break;
+                }
+            }
+
+            /* next */
+            pPage++;
+            GCPhys += PAGE_SIZE;
+        }
+    }
+    pgmUnlock(pVM);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+
+    LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
+    Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks shared page checksums.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+void pgmR3PhysAssertSharedPageChecksums(PVM pVM)
+{
+#ifdef VBOX_STRICT
+    pgmLock(pVM);
+
+    if (pVM->pgm.s.cSharedPages > 0)
+    {
+        /*
+         * Walk the ram ranges.
+         */
+        for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
+        {
+            uint32_t iPage = pRam->cb >> PAGE_SHIFT;
+            AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
+
+            while (iPage-- > 0)
+            {
+                PPGMPAGE pPage = &pRam->aPages[iPage];
+                if (PGM_PAGE_IS_SHARED(pPage))
+                {
+                    uint32_t u32Checksum = pPage->s.u2Unused0/* | ((uint32_t)pPage->s.u2Unused1 << 8)*/;
+                    if (!u32Checksum)
+                    {
+                        RTGCPHYS    GCPhysPage  = pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+                        void const *pvPage;
+                        int rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhysPage, &pvPage);
+                        if (RT_SUCCESS(rc))
+                        {
+                            uint32_t u32Checksum2 = RTCrc32(pvPage, PAGE_SIZE);
+# if 0
+                            AssertMsg((u32Checksum2 & /*UINT32_C(0x00000303)*/ 0x3) == u32Checksum, ("GCPhysPage=%RGp\n", GCPhysPage));
+# else
+                            if ((u32Checksum2 & /*UINT32_C(0x00000303)*/ 0x3) == u32Checksum)
+                                LogFlow(("shpg %#x @ %RGp %#x [OK]\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
+                            else
+                                AssertMsgFailed(("shpg %#x @ %RGp %#x\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
+# endif
+                        }
+                        else
+                            AssertRC(rc);
+                    }
+                }
+
+            } /* for each page */
+
+        } /* for each ram range */
+    }
+
+    pgmUnlock(pVM);
+#endif /* VBOX_STRICT */
+    NOREF(pVM);
+}
+
+
+/**
+ * Resets the physical memory state.
+ *
+ * ASSUMES that the caller owns the PGM lock.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+int pgmR3PhysRamReset(PVM pVM)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /* Reset the memory balloon. */
+    int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
+    AssertRC(rc);
+
+#ifdef VBOX_WITH_PAGE_SHARING
+    /* Clear all registered shared modules. */
+    pgmR3PhysAssertSharedPageChecksums(pVM);
+    rc = GMMR3ResetSharedModules(pVM);
+    AssertRC(rc);
+#endif
+    /* Reset counters. */
+    pVM->pgm.s.cReusedSharedPages = 0;
+    pVM->pgm.s.cBalloonedPages    = 0;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resets (zeros) the RAM after all devices and components have been reset.
+ *
+ * ASSUMES that the caller owns the PGM lock.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+int pgmR3PhysRamZeroAll(PVM pVM)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * We batch up pages that should be freed instead of calling GMM for
+     * each and every one of them.
+     */
+    uint32_t            cPendingPages = 0;
+    PGMMFREEPAGESREQ    pReq;
+    int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Walk the ram ranges.
+     */
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
+    {
+        uint32_t iPage = pRam->cb >> PAGE_SHIFT;
+        AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
+
+        if (   !pVM->pgm.s.fRamPreAlloc
+            && pVM->pgm.s.fZeroRamPagesOnReset)
+        {
+            /* Replace all RAM pages by ZERO pages. */
+            while (iPage-- > 0)
+            {
+                PPGMPAGE pPage = &pRam->aPages[iPage];
+                switch (PGM_PAGE_GET_TYPE(pPage))
+                {
+                    case PGMPAGETYPE_RAM:
+                        /* Do not replace pages part of a 2 MB continuous range
+                           with zero pages, but zero them instead. */
+                        if (   PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
+                            || PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
+                        {
+                            void *pvPage;
+                            rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
+                            AssertLogRelRCReturn(rc, rc);
+                            ASMMemZeroPage(pvPage);
+                        }
+                        else if (PGM_PAGE_IS_BALLOONED(pPage))
+                        {
+                            /* Turn into a zero page; the balloon status is lost when the VM reboots. */
+                            PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
+                        }
+                        else if (!PGM_PAGE_IS_ZERO(pPage))
+                        {
+                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
+                                                 PGMPAGETYPE_RAM);
+                            AssertLogRelRCReturn(rc, rc);
+                        }
+                        break;
+
+                    case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
+                    case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone?  I don't think VT-x copes with this code. */
+                        pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
+                                                           true /*fDoAccounting*/);
+                        break;
+
+                    case PGMPAGETYPE_MMIO2:
+                    case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
+                    case PGMPAGETYPE_ROM:
+                    case PGMPAGETYPE_MMIO:
+                        break;
+                    default:
+                        AssertFailed();
+                }
+            } /* for each page */
+        }
+        else
+        {
+            /* Zero the memory. */
+            while (iPage-- > 0)
+            {
+                PPGMPAGE pPage = &pRam->aPages[iPage];
+                switch (PGM_PAGE_GET_TYPE(pPage))
+                {
+                    case PGMPAGETYPE_RAM:
+                        switch (PGM_PAGE_GET_STATE(pPage))
+                        {
+                            case PGM_PAGE_STATE_ZERO:
+                                break;
+
+                            case PGM_PAGE_STATE_BALLOONED:
+                                /* Turn into a zero page; the balloon status is lost when the VM reboots. */
+                                PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
+                                break;
+
+                            case PGM_PAGE_STATE_SHARED:
+                            case PGM_PAGE_STATE_WRITE_MONITORED:
+                                rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
+                                AssertLogRelRCReturn(rc, rc);
+                                RT_FALL_THRU();
+
+                            case PGM_PAGE_STATE_ALLOCATED:
+                                if (pVM->pgm.s.fZeroRamPagesOnReset)
+                                {
+                                    void *pvPage;
+                                    rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
+                                    AssertLogRelRCReturn(rc, rc);
+                                    ASMMemZeroPage(pvPage);
+                                }
+                                break;
+                        }
+                        break;
+
+                    case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
+                    case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone?  I don't think VT-x copes with this code. */
+                        pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
+                                                           true /*fDoAccounting*/);
+                        break;
+
+                    case PGMPAGETYPE_MMIO2:
+                    case PGMPAGETYPE_ROM_SHADOW:
+                    case PGMPAGETYPE_ROM:
+                    case PGMPAGETYPE_MMIO:
+                        break;
+                    default:
+                        AssertFailed();
+
+                }
+            } /* for each page */
+        }
+
+    }
+
+    /*
+     * Finish off any pages pending freeing.
+     */
+    if (cPendingPages)
+    {
+        rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+        AssertLogRelRCReturn(rc, rc);
+    }
+    GMMR3FreePagesCleanup(pReq);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Frees all RAM during VM termination
+ *
+ * ASSUMES that the caller owns the PGM lock.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+int pgmR3PhysRamTerm(PVM pVM)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /* Reset the memory balloon. */
+    int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
+    AssertRC(rc);
+
+#ifdef VBOX_WITH_PAGE_SHARING
+    /*
+     * Clear all registered shared modules.
+     */
+    pgmR3PhysAssertSharedPageChecksums(pVM);
+    rc = GMMR3ResetSharedModules(pVM);
+    AssertRC(rc);
+
+    /*
+     * Flush the handy pages updates to make sure no shared pages are hiding
+     * in there.  (No unlikely if the VM shuts down, apparently.)
+     */
+    rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_FLUSH_HANDY_PAGES, 0, NULL);
+#endif
+
+    /*
+     * We batch up pages that should be freed instead of calling GMM for
+     * each and every one of them.
+     */
+    uint32_t            cPendingPages = 0;
+    PGMMFREEPAGESREQ    pReq;
+    rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Walk the ram ranges.
+     */
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
+    {
+        uint32_t iPage = pRam->cb >> PAGE_SHIFT;
+        AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
+
+        while (iPage-- > 0)
+        {
+            PPGMPAGE pPage = &pRam->aPages[iPage];
+            switch (PGM_PAGE_GET_TYPE(pPage))
+            {
+                case PGMPAGETYPE_RAM:
+                    /* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
+                    /** @todo change this to explicitly free private pages here. */
+                    if (PGM_PAGE_IS_SHARED(pPage))
+                    {
+                        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
+                                             PGMPAGETYPE_RAM);
+                        AssertLogRelRCReturn(rc, rc);
+                    }
+                    break;
+
+                case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
+                case PGMPAGETYPE_SPECIAL_ALIAS_MMIO:
+                case PGMPAGETYPE_MMIO2:
+                case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
+                case PGMPAGETYPE_ROM:
+                case PGMPAGETYPE_MMIO:
+                    break;
+                default:
+                    AssertFailed();
+            }
+        } /* for each page */
+    }
+
+    /*
+     * Finish off any pages pending freeing.
+     */
+    if (cPendingPages)
+    {
+        rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+        AssertLogRelRCReturn(rc, rc);
+    }
+    GMMR3FreePagesCleanup(pReq);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * This is the interface IOM is using to register an MMIO region.
+ *
+ * It will check for conflicts and ensure that a RAM range structure
+ * is present before calling the PGMR3HandlerPhysicalRegister API to
+ * register the callbacks.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The start of the MMIO region.
+ * @param   cb              The size of the MMIO region.
+ * @param   hType           The physical access handler type registration.
+ * @param   pvUserR3        The user argument for R3.
+ * @param   pvUserR0        The user argument for R0.
+ * @param   pvUserRC        The user argument for RC.
+ * @param   pszDesc         The description of the MMIO region.
+ */
+VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMPHYSHANDLERTYPE hType,
+                                     RTR3PTR pvUserR3, RTR0PTR pvUserR0, RTRCPTR pvUserRC, const char *pszDesc)
+{
+    /*
+     * Assert on some assumption.
+     */
+    VM_ASSERT_EMT(pVM);
+    AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
+    Assert(((PPGMPHYSHANDLERTYPEINT)MMHyperHeapOffsetToPtr(pVM, hType))->enmKind == PGMPHYSHANDLERKIND_MMIO);
+
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Make sure there's a RAM range structure for the region.
+     */
+    RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
+    bool fRamExists = false;
+    PPGMRAMRANGE pRamPrev = NULL;
+    PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
+    while (pRam && GCPhysLast >= pRam->GCPhys)
+    {
+        if (    GCPhysLast >= pRam->GCPhys
+            &&  GCPhys     <= pRam->GCPhysLast)
+        {
+            /* Simplification: all within the same range. */
+            AssertLogRelMsgReturnStmt(   GCPhys     >= pRam->GCPhys
+                                      && GCPhysLast <= pRam->GCPhysLast,
+                                      ("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
+                                       GCPhys, GCPhysLast, pszDesc,
+                                       pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
+                                      pgmUnlock(pVM),
+                                      VERR_PGM_RAM_CONFLICT);
+
+            /* Check that it's all RAM or MMIO pages. */
+            PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+            uint32_t cLeft = cb >> PAGE_SHIFT;
+            while (cLeft-- > 0)
+            {
+                AssertLogRelMsgReturnStmt(   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
+                                          || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
+                                          ("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
+                                           GCPhys, GCPhysLast, pszDesc, pRam->GCPhys, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
+                                          pgmUnlock(pVM),
+                                          VERR_PGM_RAM_CONFLICT);
+                pPage++;
+            }
+
+            /* Looks good. */
+            fRamExists = true;
+            break;
+        }
+
+        /* next */
+        pRamPrev = pRam;
+        pRam = pRam->pNextR3;
+    }
+    PPGMRAMRANGE pNew;
+    if (fRamExists)
+    {
+        pNew = NULL;
+
+        /*
+         * Make all the pages in the range MMIO/ZERO pages, freeing any
+         * RAM pages currently mapped here. This might not be 100% correct
+         * for PCI memory, but we're doing the same thing for MMIO2 pages.
+         */
+        rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
+        AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
+
+        /* Force a PGM pool flush as guest ram references have been changed. */
+        /** @todo not entirely SMP safe; assuming for now the guest takes
+         *   care of this internally (not touch mapped mmio while changing the
+         *   mapping). */
+        PVMCPU pVCpu = VMMGetCpu(pVM);
+        pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    }
+    else
+    {
+
+        /*
+         * No RAM range, insert an ad hoc one.
+         *
+         * Note that we don't have to tell REM about this range because
+         * PGMHandlerPhysicalRegisterEx will do that for us.
+         */
+        Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
+
+        const uint32_t cPages = cb >> PAGE_SHIFT;
+        const size_t cbRamRange = RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]);
+        rc = MMHyperAlloc(pVM, RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
+        AssertLogRelMsgRCReturnStmt(rc, ("cbRamRange=%zu\n", cbRamRange), pgmUnlock(pVM), rc);
+
+        /* Initialize the range. */
+        pNew->pSelfR0       = MMHyperCCToR0(pVM, pNew);
+        pNew->GCPhys        = GCPhys;
+        pNew->GCPhysLast    = GCPhysLast;
+        pNew->cb            = cb;
+        pNew->pszDesc       = pszDesc;
+        pNew->fFlags        = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
+        pNew->pvR3          = NULL;
+        pNew->paLSPages     = NULL;
+
+        uint32_t iPage = cPages;
+        while (iPage-- > 0)
+            PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
+        Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
+
+        /* update the page count stats. */
+        pVM->pgm.s.cPureMmioPages += cPages;
+        pVM->pgm.s.cAllPages      += cPages;
+
+        /* link it */
+        pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
+    }
+
+    /*
+     * Register the access handler.
+     */
+    rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, hType, pvUserR3, pvUserR0, pvUserRC, pszDesc);
+    if (    RT_FAILURE(rc)
+        &&  !fRamExists)
+    {
+        pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
+        pVM->pgm.s.cAllPages      -= cb >> PAGE_SHIFT;
+
+        /* remove the ad hoc range. */
+        pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
+        pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
+        MMHyperFree(pVM, pRam);
+    }
+    pgmPhysInvalidatePageMapTLB(pVM);
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * This is the interface IOM is using to register an MMIO region.
+ *
+ * It will take care of calling PGMHandlerPhysicalDeregister and clean up
+ * any ad hoc PGMRAMRANGE left behind.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The start of the MMIO region.
+ * @param   cb              The size of the MMIO region.
+ */
+VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
+{
+    VM_ASSERT_EMT(pVM);
+
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * First deregister the handler, then check if we should remove the ram range.
+     */
+    rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCPHYS        GCPhysLast  = GCPhys + (cb - 1);
+        PPGMRAMRANGE    pRamPrev    = NULL;
+        PPGMRAMRANGE    pRam        = pVM->pgm.s.pRamRangesXR3;
+        while (pRam && GCPhysLast >= pRam->GCPhys)
+        {
+            /** @todo We're being a bit too careful here. rewrite. */
+            if (    GCPhysLast == pRam->GCPhysLast
+                &&  GCPhys     == pRam->GCPhys)
+            {
+                Assert(pRam->cb == cb);
+
+                /*
+                 * See if all the pages are dead MMIO pages.
+                 */
+                uint32_t const  cPages   = cb >> PAGE_SHIFT;
+                bool            fAllMMIO = true;
+                uint32_t        iPage    = 0;
+                uint32_t        cLeft    = cPages;
+                while (cLeft-- > 0)
+                {
+                    PPGMPAGE    pPage    = &pRam->aPages[iPage];
+                    if (   !PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)
+                        /*|| not-out-of-action later */)
+                    {
+                        fAllMMIO = false;
+                        AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
+                        break;
+                    }
+                    Assert(   PGM_PAGE_IS_ZERO(pPage)
+                           || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
+                           || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO);
+                    pPage++;
+                }
+                if (fAllMMIO)
+                {
+                    /*
+                     * Ad-hoc range, unlink and free it.
+                     */
+                    Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
+                         GCPhys, GCPhysLast, pRam->pszDesc));
+
+                    pVM->pgm.s.cAllPages      -= cPages;
+                    pVM->pgm.s.cPureMmioPages -= cPages;
+
+                    pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
+                    pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
+                    MMHyperFree(pVM, pRam);
+                    break;
+                }
+            }
+
+            /*
+             * Range match? It will all be within one range (see PGMAllHandler.cpp).
+             */
+            if (    GCPhysLast >= pRam->GCPhys
+                &&  GCPhys     <= pRam->GCPhysLast)
+            {
+                Assert(GCPhys     >= pRam->GCPhys);
+                Assert(GCPhysLast <= pRam->GCPhysLast);
+
+                /*
+                 * Turn the pages back into RAM pages.
+                 */
+                uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
+                uint32_t cLeft = cb >> PAGE_SHIFT;
+                while (cLeft--)
+                {
+                    PPGMPAGE pPage = &pRam->aPages[iPage];
+                    AssertMsg(   (PGM_PAGE_IS_MMIO(pPage) && PGM_PAGE_IS_ZERO(pPage))
+                              || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
+                              || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO,
+                              ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
+                    if (PGM_PAGE_IS_MMIO_OR_ALIAS(pPage))
+                        PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
+                }
+                break;
+            }
+
+            /* next */
+            pRamPrev = pRam;
+            pRam = pRam->pNextR3;
+        }
+    }
+
+    /* Force a PGM pool flush as guest ram references have been changed. */
+    /** @todo Not entirely SMP safe; assuming for now the guest takes care of
+     *       this internally (not touch mapped mmio while changing the mapping). */
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+
+    pgmPhysInvalidatePageMapTLB(pVM);
+    pgmPhysInvalidRamRangeTlbs(pVM);
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Locate a MMIO2 range.
+ *
+ * @returns Pointer to the MMIO2 range.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance owning the region.
+ * @param   iSubDev         The sub-device number.
+ * @param   iRegion         The region.
+ * @param   hMmio2          Handle to look up.  If NIL, use the @a iSubDev and
+ *                          @a iRegion.
+ */
+DECLINLINE(PPGMREGMMIO2RANGE) pgmR3PhysMmio2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev,
+                                                 uint32_t iRegion, PGMMMIO2HANDLE hMmio2)
+{
+    if (hMmio2 != NIL_PGMMMIO2HANDLE)
+    {
+        if (hMmio2 <= RT_ELEMENTS(pVM->pgm.s.apMmio2RangesR3) && hMmio2 != 0)
+        {
+            PPGMREGMMIO2RANGE pCur = pVM->pgm.s.apMmio2RangesR3[hMmio2 - 1];
+            if (pCur && pCur->pDevInsR3 == pDevIns)
+            {
+                Assert(pCur->idMmio2 == hMmio2);
+                AssertReturn(pCur->fFlags & PGMREGMMIO2RANGE_F_MMIO2, NULL);
+                AssertReturn(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK, NULL);
+                return pCur;
+            }
+            Assert(!pCur);
+        }
+        for (PPGMREGMMIO2RANGE pCur = pVM->pgm.s.pRegMmioRangesR3; pCur; pCur = pCur->pNextR3)
+            if (pCur->idMmio2 == hMmio2)
+            {
+                AssertBreak(pCur->pDevInsR3 == pDevIns);
+                AssertReturn(pCur->fFlags & PGMREGMMIO2RANGE_F_MMIO2, NULL);
+                AssertReturn(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK, NULL);
+                return pCur;
+            }
+    }
+    else
+    {
+        /*
+         * Search the list.  There shouldn't be many entries.
+         */
+        /** @todo Optimize this lookup! There may now be many entries and it'll
+         *        become really slow when doing MMR3HyperMapMMIO2 and similar. */
+        for (PPGMREGMMIO2RANGE pCur = pVM->pgm.s.pRegMmioRangesR3; pCur; pCur = pCur->pNextR3)
+            if (   pCur->pDevInsR3 == pDevIns
+                && pCur->iRegion == iRegion
+                && pCur->iSubDev == iSubDev)
+                return pCur;
+    }
+    return NULL;
+}
+
+
+#ifndef PGM_WITHOUT_MAPPINGS
+/**
+ * @callback_method_impl{FNPGMRELOCATE, Relocate a floating MMIO/MMIO2 range.}
+ * @sa pgmR3PhysRamRangeRelocate
+ */
+static DECLCALLBACK(bool) pgmR3PhysMmio2RangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew,
+                                                      PGMRELOCATECALL enmMode, void *pvUser)
+{
+    PPGMREGMMIO2RANGE pMmio = (PPGMREGMMIO2RANGE)pvUser;
+    Assert(pMmio->RamRange.fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
+    Assert(pMmio->RamRange.pSelfRC == GCPtrOld + PAGE_SIZE + RT_UOFFSETOF(PGMREGMMIO2RANGE, RamRange)); RT_NOREF_PV(GCPtrOld);
+
+    switch (enmMode)
+    {
+        case PGMRELOCATECALL_SUGGEST:
+            return true;
+
+        case PGMRELOCATECALL_RELOCATE:
+        {
+            /*
+             * Update myself, then relink all the ranges and flush the RC TLB.
+             */
+            pgmLock(pVM);
+
+            pMmio->RamRange.pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE + RT_UOFFSETOF(PGMREGMMIO2RANGE, RamRange));
+
+            pgmR3PhysRelinkRamRanges(pVM);
+            for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
+                pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
+
+            pgmUnlock(pVM);
+            return true;
+        }
+
+        default:
+            AssertFailedReturn(false);
+    }
+}
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
+
+/**
+ * Calculates the number of chunks
+ *
+ * @returns Number of registration chunk needed.
+ * @param   pVM             The cross context VM structure.
+ * @param   cb              The size of the MMIO/MMIO2 range.
+ * @param   pcPagesPerChunk Where to return the number of pages tracked by each
+ *                          chunk.  Optional.
+ * @param   pcbChunk        Where to return the guest mapping size for a chunk.
+ */
+static uint16_t pgmR3PhysMmio2CalcChunkCount(PVM pVM, RTGCPHYS cb, uint32_t *pcPagesPerChunk, uint32_t *pcbChunk)
+{
+    RT_NOREF_PV(pVM); /* without raw mode */
+
+    /*
+     * This is the same calculation as PGMR3PhysRegisterRam does, except we'll be
+     * needing a few bytes extra the PGMREGMMIO2RANGE structure.
+     *
+     * Note! In additions, we've got a 24 bit sub-page range for MMIO2 ranges, leaving
+     *       us with an absolute maximum of 16777215 pages per chunk (close to 64 GB).
+     */
+    uint32_t cbChunk = 16U*_1M;
+    uint32_t cPagesPerChunk = 1048048; /* max ~1048059 */
+    AssertCompile(sizeof(PGMREGMMIO2RANGE) + sizeof(PGMPAGE) * 1048048 < 16U*_1M - PAGE_SIZE * 2);
+    AssertRelease(cPagesPerChunk <= PGM_MMIO2_MAX_PAGE_COUNT); /* See above note. */
+    AssertRelease(RT_UOFFSETOF_DYN(PGMREGMMIO2RANGE, RamRange.aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
+    if (pcbChunk)
+        *pcbChunk = cbChunk;
+    if (pcPagesPerChunk)
+        *pcPagesPerChunk = cPagesPerChunk;
+
+    /* Calc the number of chunks we need. */
+    RTGCPHYS const cPages = cb >> X86_PAGE_SHIFT;
+    uint16_t cChunks = (uint16_t)((cPages + cPagesPerChunk - 1) / cPagesPerChunk);
+    AssertRelease((RTGCPHYS)cChunks * cPagesPerChunk >= cPages);
+    return cChunks;
+}
+
+
+/**
+ * Worker for PGMR3PhysMMIO2Register that allocates and the PGMREGMMIO2RANGE
+ * structures and does basic initialization.
+ *
+ * Caller must set type specfic members and initialize the PGMPAGE structures.
+ *
+ * This was previously also used by PGMR3PhysMmio2PreRegister, a function for
+ * pre-registering MMIO that was later (6.1) replaced by a new handle based IOM
+ * interface.  The reference to caller and type above is purely historical.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance owning the region.
+ * @param   iSubDev         The sub-device number (internal PCI config number).
+ * @param   iRegion         The region number.  If the MMIO2 memory is a PCI
+ *                          I/O region this number has to be the number of that
+ *                          region. Otherwise it can be any number safe
+ *                          UINT8_MAX.
+ * @param   cb              The size of the region.  Must be page aligned.
+ * @param   pszDesc         The description.
+ * @param   ppHeadRet       Where to return the pointer to the first
+ *                          registration chunk.
+ *
+ * @thread  EMT
+ */
+static int pgmR3PhysMmio2Create(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS cb,
+                                const char *pszDesc, PPGMREGMMIO2RANGE *ppHeadRet)
+{
+    /*
+     * Figure out how many chunks we need and of which size.
+     */
+    uint32_t cPagesPerChunk;
+    uint16_t cChunks = pgmR3PhysMmio2CalcChunkCount(pVM, cb, &cPagesPerChunk, NULL);
+    AssertReturn(cChunks, VERR_PGM_PHYS_MMIO_EX_IPE);
+
+    /*
+     * Allocate the chunks.
+     */
+    PPGMREGMMIO2RANGE *ppNext = ppHeadRet;
+    *ppNext = NULL;
+
+    int rc = VINF_SUCCESS;
+    uint32_t cPagesLeft = cb >> X86_PAGE_SHIFT;
+    for (uint16_t iChunk = 0; iChunk < cChunks && RT_SUCCESS(rc); iChunk++)
+    {
+        /*
+         * We currently do a single RAM range for the whole thing.  This will
+         * probably have to change once someone needs really large MMIO regions,
+         * as we will be running into SUPR3PageAllocEx limitations and such.
+         */
+        const uint32_t   cPagesTrackedByChunk = RT_MIN(cPagesLeft, cPagesPerChunk);
+        const size_t     cbRange = RT_UOFFSETOF_DYN(PGMREGMMIO2RANGE, RamRange.aPages[cPagesTrackedByChunk]);
+        PPGMREGMMIO2RANGE pNew    = NULL;
+        if (   iChunk + 1 < cChunks
+            || cbRange >= _1M)
+        {
+            /*
+             * Allocate memory for the registration structure.
+             */
+            size_t const cChunkPages  = RT_ALIGN_Z(cbRange, PAGE_SIZE) >> PAGE_SHIFT;
+            size_t const cbChunk      = (1 + cChunkPages + 1) << PAGE_SHIFT;
+            AssertLogRelBreakStmt(cbChunk == (uint32_t)cbChunk, rc = VERR_OUT_OF_RANGE);
+            PSUPPAGE     paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
+            AssertBreakStmt(paChunkPages, rc = VERR_NO_TMP_MEMORY);
+            RTR0PTR      R0PtrChunk   = NIL_RTR0PTR;
+            void        *pvChunk      = NULL;
+            rc = SUPR3PageAllocEx(cChunkPages, 0 /*fFlags*/, &pvChunk, &R0PtrChunk, paChunkPages);
+            AssertLogRelMsgRCBreakStmt(rc, ("rc=%Rrc, cChunkPages=%#zx\n", rc, cChunkPages), RTMemTmpFree(paChunkPages));
+
+            Assert(R0PtrChunk != NIL_RTR0PTR);
+            memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
+
+            pNew = (PPGMREGMMIO2RANGE)pvChunk;
+            pNew->RamRange.fFlags   = PGM_RAM_RANGE_FLAGS_FLOATING;
+            pNew->RamRange.pSelfR0  = R0PtrChunk + RT_UOFFSETOF(PGMREGMMIO2RANGE, RamRange);
+
+            RTMemTmpFree(paChunkPages);
+        }
+        /*
+         * Not so big, do a one time hyper allocation.
+         */
+        else
+        {
+            rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
+            AssertLogRelMsgRCBreak(rc, ("cbRange=%zu\n", cbRange));
+
+            /*
+             * Initialize allocation specific items.
+             */
+            //pNew->RamRange.fFlags = 0;
+            pNew->RamRange.pSelfR0  = MMHyperCCToR0(pVM, &pNew->RamRange);
+        }
+
+        /*
+         * Initialize the registration structure (caller does specific bits).
+         */
+        pNew->pDevInsR3             = pDevIns;
+        //pNew->pvR3                = NULL;
+        //pNew->pNext               = NULL;
+        //pNew->fFlags              = 0;
+        if (iChunk == 0)
+            pNew->fFlags |= PGMREGMMIO2RANGE_F_FIRST_CHUNK;
+        if (iChunk + 1 == cChunks)
+            pNew->fFlags |= PGMREGMMIO2RANGE_F_LAST_CHUNK;
+        pNew->iSubDev               = iSubDev;
+        pNew->iRegion               = iRegion;
+        pNew->idSavedState          = UINT8_MAX;
+        pNew->idMmio2               = UINT8_MAX;
+        //pNew->pPhysHandlerR3      = NULL;
+        //pNew->paLSPages           = NULL;
+        pNew->RamRange.GCPhys       = NIL_RTGCPHYS;
+        pNew->RamRange.GCPhysLast   = NIL_RTGCPHYS;
+        pNew->RamRange.pszDesc      = pszDesc;
+        pNew->RamRange.cb           = pNew->cbReal = (RTGCPHYS)cPagesTrackedByChunk << X86_PAGE_SHIFT;
+        pNew->RamRange.fFlags      |= PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX;
+        //pNew->RamRange.pvR3       = NULL;
+        //pNew->RamRange.paLSPages  = NULL;
+
+        *ppNext = pNew;
+        ASMCompilerBarrier();
+        cPagesLeft -= cPagesTrackedByChunk;
+        ppNext = &pNew->pNextR3;
+    }
+    Assert(cPagesLeft == 0);
+
+    if (RT_SUCCESS(rc))
+    {
+        Assert((*ppHeadRet)->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Free floating ranges.
+     */
+    while (*ppHeadRet)
+    {
+        PPGMREGMMIO2RANGE pFree = *ppHeadRet;
+        *ppHeadRet = pFree->pNextR3;
+
+        if (pFree->RamRange.fFlags & PGM_RAM_RANGE_FLAGS_FLOATING)
+        {
+            const size_t    cbRange     = RT_UOFFSETOF_DYN(PGMREGMMIO2RANGE, RamRange.aPages[pFree->RamRange.cb >> X86_PAGE_SHIFT]);
+            size_t const    cChunkPages = RT_ALIGN_Z(cbRange, PAGE_SIZE) >> PAGE_SHIFT;
+            SUPR3PageFreeEx(pFree, cChunkPages);
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Common worker PGMR3PhysMmio2PreRegister & PGMR3PhysMMIO2Register that links a
+ * complete registration entry into the lists and lookup tables.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pNew            The new MMIO / MMIO2 registration to link.
+ */
+static void pgmR3PhysMmio2Link(PVM pVM, PPGMREGMMIO2RANGE pNew)
+{
+    /*
+     * Link it into the list (order doesn't matter, so insert it at the head).
+     *
+     * Note! The range we're linking may consist of multiple chunks, so we
+     *       have to find the last one.
+     */
+    PPGMREGMMIO2RANGE pLast = pNew;
+    for (pLast = pNew; ; pLast = pLast->pNextR3)
+    {
+        if (pLast->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
+            break;
+        Assert(pLast->pNextR3);
+        Assert(pLast->pNextR3->pDevInsR3 == pNew->pDevInsR3);
+        Assert(pLast->pNextR3->iSubDev   == pNew->iSubDev);
+        Assert(pLast->pNextR3->iRegion   == pNew->iRegion);
+        Assert((pLast->pNextR3->fFlags & PGMREGMMIO2RANGE_F_MMIO2) == (pNew->fFlags & PGMREGMMIO2RANGE_F_MMIO2));
+        Assert(pLast->pNextR3->idMmio2   == (pLast->fFlags & PGMREGMMIO2RANGE_F_MMIO2 ? pLast->idMmio2 + 1 : UINT8_MAX));
+    }
+
+    pgmLock(pVM);
+
+    /* Link in the chain of ranges at the head of the list. */
+    pLast->pNextR3 = pVM->pgm.s.pRegMmioRangesR3;
+    pVM->pgm.s.pRegMmioRangesR3 = pNew;
+
+    /* If MMIO, insert the MMIO2 range/page IDs. */
+    uint8_t idMmio2 = pNew->idMmio2;
+    if (idMmio2 != UINT8_MAX)
+    {
+        for (;;)
+        {
+            Assert(pNew->fFlags & PGMREGMMIO2RANGE_F_MMIO2);
+            Assert(pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] == NULL);
+            Assert(pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] == NIL_RTR0PTR);
+            pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] = pNew;
+            pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] = pNew->RamRange.pSelfR0 - RT_UOFFSETOF(PGMREGMMIO2RANGE, RamRange);
+            if (pNew->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
+                break;
+            pNew = pNew->pNextR3;
+            idMmio2++;
+        }
+    }
+    else
+        Assert(!(pNew->fFlags & PGMREGMMIO2RANGE_F_MMIO2));
+
+    pgmPhysInvalidatePageMapTLB(pVM);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Allocate and register an MMIO2 region.
+ *
+ * As mentioned elsewhere, MMIO2 is just RAM spelled differently.  It's RAM
+ * associated with a device. It is also non-shared memory with a permanent
+ * ring-3 mapping and page backing (presently).
+ *
+ * A MMIO2 range may overlap with base memory if a lot of RAM is configured for
+ * the VM, in which case we'll drop the base memory pages.  Presently we will
+ * make no attempt to preserve anything that happens to be present in the base
+ * memory that is replaced, this is of course incorrect but it's too much
+ * effort.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
+ *          memory.
+ * @retval  VERR_ALREADY_EXISTS if the region already exists.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance owning the region.
+ * @param   iSubDev         The sub-device number.
+ * @param   iRegion         The region number.  If the MMIO2 memory is a PCI
+ *                          I/O region this number has to be the number of that
+ *                          region.  Otherwise it can be any number save
+ *                          UINT8_MAX.
+ * @param   cb              The size of the region.  Must be page aligned.
+ * @param   fFlags          Reserved for future use, must be zero.
+ * @param   pszDesc         The description.
+ * @param   ppv             Where to store the pointer to the ring-3 mapping of
+ *                          the memory.
+ * @param   phRegion        Where to return the MMIO2 region handle.  Optional.
+ * @thread  EMT
+ */
+VMMR3_INT_DECL(int) PGMR3PhysMmio2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS cb,
+                                           uint32_t fFlags, const char *pszDesc, void **ppv, PGMMMIO2HANDLE *phRegion)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(ppv, VERR_INVALID_POINTER);
+    *ppv = NULL;
+    if (phRegion)
+    {
+        AssertPtrReturn(phRegion, VERR_INVALID_POINTER);
+        *phRegion = NIL_PGMMMIO2HANDLE;
+    }
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
+    AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
+    AssertReturn(pgmR3PhysMmio2Find(pVM, pDevIns, iSubDev, iRegion, NIL_PGMMMIO2HANDLE) == NULL, VERR_ALREADY_EXISTS);
+    AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(cb, VERR_INVALID_PARAMETER);
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+
+    const uint32_t cPages = cb >> PAGE_SHIFT;
+    AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
+    AssertLogRelReturn(cPages <= (MM_MMIO_64_MAX >> X86_PAGE_SHIFT), VERR_OUT_OF_RANGE);
+    AssertLogRelReturn(cPages <= PGM_MMIO2_MAX_PAGE_COUNT, VERR_OUT_OF_RANGE);
+
+    /*
+     * For the 2nd+ instance, mangle the description string so it's unique.
+     */
+    if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
+    {
+        pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
+        if (!pszDesc)
+            return VERR_NO_MEMORY;
+    }
+
+    /*
+     * Allocate an MMIO2 range ID (not freed on failure).
+     *
+     * The zero ID is not used as it could be confused with NIL_GMM_PAGEID, so
+     * the IDs goes from 1 thru PGM_MMIO2_MAX_RANGES.
+     */
+    unsigned cChunks = pgmR3PhysMmio2CalcChunkCount(pVM, cb, NULL, NULL);
+    pgmLock(pVM);
+    uint8_t  idMmio2 = pVM->pgm.s.cMmio2Regions + 1;
+    unsigned cNewMmio2Regions = pVM->pgm.s.cMmio2Regions + cChunks;
+    if (cNewMmio2Regions > PGM_MMIO2_MAX_RANGES)
+    {
+        pgmUnlock(pVM);
+        AssertLogRelFailedReturn(VERR_PGM_TOO_MANY_MMIO2_RANGES);
+    }
+    pVM->pgm.s.cMmio2Regions = cNewMmio2Regions;
+    pgmUnlock(pVM);
+
+    /*
+     * Try reserve and allocate the backing memory first as this is what is
+     * most likely to fail.
+     */
+    int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
+    if (RT_SUCCESS(rc))
+    {
+        PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
+        if (RT_SUCCESS(rc))
+        {
+            void *pvPages;
+#if defined(VBOX_WITH_RAM_IN_KERNEL) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+            RTR0PTR pvPagesR0;
+            rc = SUPR3PageAllocEx(cPages, 0 /*fFlags*/, &pvPages, &pvPagesR0, paPages);
+#else
+            rc = SUPR3PageAllocEx(cPages, 0 /*fFlags*/, &pvPages, NULL /*pR0Ptr*/, paPages);
+#endif
+            if (RT_SUCCESS(rc))
+            {
+                memset(pvPages, 0, cPages * PAGE_SIZE);
+
+                /*
+                 * Create the registered MMIO range record for it.
+                 */
+                PPGMREGMMIO2RANGE pNew;
+                rc = pgmR3PhysMmio2Create(pVM, pDevIns, iSubDev, iRegion, cb, pszDesc, &pNew);
+                if (RT_SUCCESS(rc))
+                {
+                    if (phRegion)
+                        *phRegion = idMmio2;    /* The ID of the first chunk. */
+
+                    uint32_t iSrcPage   = 0;
+                    uint8_t *pbCurPages = (uint8_t *)pvPages;
+                    for (PPGMREGMMIO2RANGE pCur = pNew; pCur; pCur = pCur->pNextR3)
+                    {
+                        pCur->pvR3          = pbCurPages;
+#if defined(VBOX_WITH_RAM_IN_KERNEL) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+                        pCur->pvR0          = pvPagesR0 + (iSrcPage << PAGE_SHIFT);
+#endif
+                        pCur->RamRange.pvR3 = pbCurPages;
+                        pCur->idMmio2       = idMmio2;
+                        pCur->fFlags       |= PGMREGMMIO2RANGE_F_MMIO2;
+
+                        uint32_t iDstPage = pCur->RamRange.cb >> X86_PAGE_SHIFT;
+                        while (iDstPage-- > 0)
+                        {
+                            PGM_PAGE_INIT(&pNew->RamRange.aPages[iDstPage],
+                                          paPages[iDstPage + iSrcPage].Phys,
+                                          PGM_MMIO2_PAGEID_MAKE(idMmio2, iDstPage),
+                                          PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
+                        }
+
+                        /* advance. */
+                        iSrcPage   += pCur->RamRange.cb >> X86_PAGE_SHIFT;
+                        pbCurPages += pCur->RamRange.cb;
+                        idMmio2++;
+                    }
+
+                    RTMemTmpFree(paPages);
+
+                    /*
+                     * Update the page count stats, link the registration and we're done.
+                     */
+                    pVM->pgm.s.cAllPages += cPages;
+                    pVM->pgm.s.cPrivatePages += cPages;
+
+                    pgmR3PhysMmio2Link(pVM, pNew);
+
+                    *ppv = pvPages;
+                    return VINF_SUCCESS;
+                }
+
+                SUPR3PageFreeEx(pvPages, cPages);
+            }
+        }
+        RTMemTmpFree(paPages);
+        MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
+    }
+    if (pDevIns->iInstance > 0)
+        MMR3HeapFree((void *)pszDesc);
+    return rc;
+}
+
+
+/**
+ * Deregisters and frees an MMIO2 region.
+ *
+ * Any physical access handlers registered for the region must be deregistered
+ * before calling this function.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance owning the region.
+ * @param   hMmio2          The MMIO2 handle to deregister, or NIL if all
+ *                          regions for the given device is to be deregistered.
+ */
+VMMR3_INT_DECL(int) PGMR3PhysMmio2Deregister(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
+{
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+
+    /*
+     * The loop here scanning all registrations will make sure that multi-chunk ranges
+     * get properly deregistered, though it's original purpose was the wildcard iRegion.
+     */
+    pgmLock(pVM);
+    int rc = VINF_SUCCESS;
+    unsigned cFound = 0;
+    PPGMREGMMIO2RANGE pPrev = NULL;
+    PPGMREGMMIO2RANGE pCur = pVM->pgm.s.pRegMmioRangesR3;
+    while (pCur)
+    {
+        uint32_t const fFlags = pCur->fFlags;
+        if (    pCur->pDevInsR3 == pDevIns
+            &&  (   hMmio2 == NIL_PGMMMIO2HANDLE
+                 || pCur->idMmio2 == hMmio2))
+        {
+            Assert(fFlags & PGMREGMMIO2RANGE_F_MMIO2);
+            cFound++;
+
+            /*
+             * Unmap it if it's mapped.
+             */
+            if (fFlags & PGMREGMMIO2RANGE_F_MAPPED)
+            {
+                int rc2 = PGMR3PhysMmio2Unmap(pVM, pCur->pDevInsR3, pCur->idMmio2, pCur->RamRange.GCPhys);
+                AssertRC(rc2);
+                if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                    rc = rc2;
+            }
+
+            /*
+             * Unlink it
+             */
+            PPGMREGMMIO2RANGE pNext = pCur->pNextR3;
+            if (pPrev)
+                pPrev->pNextR3 = pNext;
+            else
+                pVM->pgm.s.pRegMmioRangesR3 = pNext;
+            pCur->pNextR3 = NULL;
+
+            uint8_t idMmio2 = pCur->idMmio2;
+            if (idMmio2 != UINT8_MAX)
+            {
+                Assert(pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] == pCur);
+                pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] = NULL;
+                pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] = NIL_RTR0PTR;
+            }
+
+            /*
+             * Free the memory.
+             */
+            const bool     fIsMmio2 = RT_BOOL(fFlags & PGMREGMMIO2RANGE_F_MMIO2);
+            uint32_t const cPages   = pCur->cbReal >> PAGE_SHIFT;
+            if (fIsMmio2)
+            {
+                int rc2 = SUPR3PageFreeEx(pCur->pvR3, cPages);
+                AssertRC(rc2);
+                if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                    rc = rc2;
+
+                rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
+                AssertRC(rc2);
+                if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                    rc = rc2;
+            }
+
+            /* we're leaking hyper memory here if done at runtime. */
+#ifdef VBOX_STRICT
+            VMSTATE const enmState = VMR3GetState(pVM);
+            AssertMsg(   enmState == VMSTATE_POWERING_OFF
+                      || enmState == VMSTATE_POWERING_OFF_LS
+                      || enmState == VMSTATE_OFF
+                      || enmState == VMSTATE_OFF_LS
+                      || enmState == VMSTATE_DESTROYING
+                      || enmState == VMSTATE_TERMINATED
+                      || enmState == VMSTATE_CREATING
+                      , ("%s\n", VMR3GetStateName(enmState)));
+#endif
+
+            if (pCur->RamRange.fFlags & PGM_RAM_RANGE_FLAGS_FLOATING)
+            {
+                const size_t    cbRange     = RT_UOFFSETOF_DYN(PGMREGMMIO2RANGE, RamRange.aPages[cPages]);
+                size_t const    cChunkPages = RT_ALIGN_Z(cbRange, PAGE_SIZE) >> PAGE_SHIFT;
+                SUPR3PageFreeEx(pCur, cChunkPages);
+            }
+            /*else
+            {
+                rc = MMHyperFree(pVM, pCur); - does not work, see the alloc call.
+                AssertRCReturn(rc, rc);
+            } */
+
+
+            /* update page count stats */
+            pVM->pgm.s.cAllPages -= cPages;
+            if (fIsMmio2)
+                pVM->pgm.s.cPrivatePages -= cPages;
+            else
+                pVM->pgm.s.cPureMmioPages -= cPages;
+
+            /* next */
+            pCur = pNext;
+            if (hMmio2 != NIL_PGMMMIO2HANDLE)
+            {
+                if (fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
+                    break;
+                hMmio2++;
+                Assert(pCur->idMmio2 == hMmio2);
+                Assert(pCur->pDevInsR3 == pDevIns);
+                Assert(!(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK));
+            }
+        }
+        else
+        {
+            pPrev = pCur;
+            pCur = pCur->pNextR3;
+        }
+    }
+    pgmPhysInvalidatePageMapTLB(pVM);
+    pgmUnlock(pVM);
+    return !cFound && hMmio2 != NIL_PGMMMIO2HANDLE  ? VERR_NOT_FOUND : rc;
+}
+
+
+/**
+ * Maps a MMIO2 region.
+ *
+ * This is typically done when a guest / the bios / state loading changes the
+ * PCI config.  The replacing of base memory has the same restrictions as during
+ * registration, of course.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance owning the region.
+ * @param   hMmio2          The handle of the region to map.
+ * @param   GCPhys          The guest-physical address to be remapped.
+ */
+VMMR3_INT_DECL(int) PGMR3PhysMmio2Map(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS GCPhys)
+{
+    /*
+     * Validate input.
+     *
+     * Note! It's safe to walk the MMIO/MMIO2 list since registrations only
+     *       happens during VM construction.
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
+    AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
+    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE, VERR_INVALID_HANDLE);
+
+    PPGMREGMMIO2RANGE pFirstMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
+    AssertReturn(pFirstMmio, VERR_NOT_FOUND);
+    Assert(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK);
+
+    PPGMREGMMIO2RANGE pLastMmio = pFirstMmio;
+    RTGCPHYS          cbRange   = 0;
+    for (;;)
+    {
+        AssertReturn(!(pLastMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED), VERR_WRONG_ORDER);
+        Assert(pLastMmio->RamRange.GCPhys == NIL_RTGCPHYS);
+        Assert(pLastMmio->RamRange.GCPhysLast == NIL_RTGCPHYS);
+        Assert(pLastMmio->pDevInsR3 == pFirstMmio->pDevInsR3);
+        Assert(pLastMmio->iSubDev   == pFirstMmio->iSubDev);
+        Assert(pLastMmio->iRegion   == pFirstMmio->iRegion);
+        cbRange += pLastMmio->RamRange.cb;
+        if (pLastMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
+            break;
+        pLastMmio = pLastMmio->pNextR3;
+    }
+
+    RTGCPHYS GCPhysLast = GCPhys + cbRange - 1;
+    AssertLogRelReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
+
+    /*
+     * Find our location in the ram range list, checking for restriction
+     * we don't bother implementing yet (partially overlapping, multiple
+     * ram ranges).
+     */
+    pgmLock(pVM);
+
+    AssertReturnStmt(!(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED), pgmUnlock(pVM), VERR_WRONG_ORDER);
+
+    bool fRamExists = false;
+    PPGMRAMRANGE pRamPrev = NULL;
+    PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
+    while (pRam && GCPhysLast >= pRam->GCPhys)
+    {
+        if (    GCPhys     <= pRam->GCPhysLast
+            &&  GCPhysLast >= pRam->GCPhys)
+        {
+            /* Completely within? */
+            AssertLogRelMsgReturnStmt(   GCPhys     >= pRam->GCPhys
+                                      && GCPhysLast <= pRam->GCPhysLast,
+                                      ("%RGp-%RGp (MMIOEx/%s) falls partly outside %RGp-%RGp (%s)\n",
+                                       GCPhys, GCPhysLast, pFirstMmio->RamRange.pszDesc,
+                                       pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
+                                      pgmUnlock(pVM),
+                                      VERR_PGM_RAM_CONFLICT);
+
+            /* Check that all the pages are RAM pages. */
+            PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+            uint32_t cPagesLeft = cbRange >> PAGE_SHIFT;
+            while (cPagesLeft-- > 0)
+            {
+                AssertLogRelMsgReturnStmt(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
+                                          ("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
+                                           GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pFirstMmio->RamRange.pszDesc),
+                                          pgmUnlock(pVM),
+                                          VERR_PGM_RAM_CONFLICT);
+                pPage++;
+            }
+
+            /* There can only be one MMIO/MMIO2 chunk matching here! */
+            AssertLogRelMsgReturnStmt(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK,
+                                      ("%RGp-%RGp (MMIOEx/%s, flags %#X) consists of multiple chunks whereas the RAM somehow doesn't!\n",
+                                       GCPhys, GCPhysLast, pFirstMmio->RamRange.pszDesc, pFirstMmio->fFlags),
+                                      pgmUnlock(pVM),
+                                      VERR_PGM_PHYS_MMIO_EX_IPE);
+
+            fRamExists = true;
+            break;
+        }
+
+        /* next */
+        pRamPrev = pRam;
+        pRam = pRam->pNextR3;
+    }
+    Log(("PGMR3PhysMmio2Map: %RGp-%RGp fRamExists=%RTbool %s\n", GCPhys, GCPhysLast, fRamExists, pFirstMmio->RamRange.pszDesc));
+
+
+    /*
+     * Make the changes.
+     */
+    RTGCPHYS GCPhysCur = GCPhys;
+    for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
+    {
+        pCurMmio->RamRange.GCPhys = GCPhysCur;
+        pCurMmio->RamRange.GCPhysLast = GCPhysCur + pCurMmio->RamRange.cb - 1;
+        if (pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
+        {
+            Assert(pCurMmio->RamRange.GCPhysLast == GCPhysLast);
+            break;
+        }
+        GCPhysCur += pCurMmio->RamRange.cb;
+    }
+
+    if (fRamExists)
+    {
+        /*
+         * Make all the pages in the range MMIO/ZERO pages, freeing any
+         * RAM pages currently mapped here. This might not be 100% correct
+         * for PCI memory, but we're doing the same thing for MMIO2 pages.
+         *
+         * We replace this MMIO/ZERO pages with real pages in the MMIO2 case.
+         */
+        Assert(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK); /* Only one chunk */
+
+        int rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
+        AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
+
+        if (pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2)
+        {
+            /* replace the pages, freeing all present RAM pages. */
+            PPGMPAGE pPageSrc = &pFirstMmio->RamRange.aPages[0];
+            PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+            uint32_t cPagesLeft = pFirstMmio->RamRange.cb >> PAGE_SHIFT;
+            while (cPagesLeft-- > 0)
+            {
+                Assert(PGM_PAGE_IS_MMIO(pPageDst));
+
+                RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
+                uint32_t const idPage = PGM_PAGE_GET_PAGEID(pPageSrc);
+                PGM_PAGE_SET_PAGEID(pVM, pPageDst, idPage);
+                PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
+                PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
+                PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
+                PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
+                PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
+                PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
+                /* (We tell NEM at the end of the function.) */
+
+                pVM->pgm.s.cZeroPages--;
+                GCPhys += PAGE_SIZE;
+                pPageSrc++;
+                pPageDst++;
+            }
+        }
+
+        /* Flush physical page map TLB. */
+        pgmPhysInvalidatePageMapTLB(pVM);
+
+        /* Force a PGM pool flush as guest ram references have been changed. */
+        /** @todo not entirely SMP safe; assuming for now the guest takes care of
+         *  this internally (not touch mapped mmio while changing the mapping). */
+        PVMCPU pVCpu = VMMGetCpu(pVM);
+        pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    }
+    else
+    {
+        /*
+         * No RAM range, insert the ones prepared during registration.
+         */
+        for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
+        {
+            /* Clear the tracking data of pages we're going to reactivate. */
+            PPGMPAGE pPageSrc = &pCurMmio->RamRange.aPages[0];
+            uint32_t cPagesLeft = pCurMmio->RamRange.cb >> PAGE_SHIFT;
+            while (cPagesLeft-- > 0)
+            {
+                PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
+                PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
+                pPageSrc++;
+            }
+
+            /* link in the ram range */
+            pgmR3PhysLinkRamRange(pVM, &pCurMmio->RamRange, pRamPrev);
+
+            if (pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
+            {
+                Assert(pCurMmio->RamRange.GCPhysLast == GCPhysLast);
+                break;
+            }
+            pRamPrev = &pCurMmio->RamRange;
+        }
+    }
+
+    /*
+     * Register the access handler if plain MMIO.
+     *
+     * We must register access handlers for each range since the access handler
+     * code refuses to deal with multiple ranges (and we can).
+     */
+    if (!(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2))
+    {
+        AssertFailed();
+        int rc = VINF_SUCCESS;
+        for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
+        {
+            Assert(!(pCurMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED));
+            rc = pgmHandlerPhysicalExRegister(pVM, pCurMmio->pPhysHandlerR3, pCurMmio->RamRange.GCPhys,
+                                              pCurMmio->RamRange.GCPhysLast);
+            if (RT_FAILURE(rc))
+                break;
+            pCurMmio->fFlags |= PGMREGMMIO2RANGE_F_MAPPED; /* Use this to mark that the handler is registered. */
+            if (pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
+                break;
+        }
+        if (RT_FAILURE(rc))
+        {
+            /* Almost impossible, but try clean up properly and get out of here. */
+            for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
+            {
+                if (pCurMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED)
+                {
+                    pCurMmio->fFlags &= ~PGMREGMMIO2RANGE_F_MAPPED;
+                    pgmHandlerPhysicalExDeregister(pVM, pCurMmio->pPhysHandlerR3, fRamExists);
+                }
+
+                if (!fRamExists)
+                    pgmR3PhysUnlinkRamRange(pVM, &pCurMmio->RamRange);
+                else
+                {
+                    Assert(pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK); /* Only one chunk */
+
+                    uint32_t cPagesLeft = pCurMmio->RamRange.cb >> PAGE_SHIFT;
+                    PPGMPAGE pPageDst = &pRam->aPages[(pCurMmio->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+                    while (cPagesLeft-- > 0)
+                    {
+                        PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
+                        pPageDst++;
+                    }
+                }
+
+                pCurMmio->RamRange.GCPhys     = NIL_RTGCPHYS;
+                pCurMmio->RamRange.GCPhysLast = NIL_RTGCPHYS;
+                if (pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
+                    break;
+            }
+
+            pgmUnlock(pVM);
+            return rc;
+        }
+    }
+
+    /*
+     * We're good, set the flags and invalid the mapping TLB.
+     */
+    for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
+    {
+        pCurMmio->fFlags |= PGMREGMMIO2RANGE_F_MAPPED;
+        if (fRamExists)
+            pCurMmio->fFlags |= PGMREGMMIO2RANGE_F_OVERLAPPING;
+        else
+            pCurMmio->fFlags &= ~PGMREGMMIO2RANGE_F_OVERLAPPING;
+        if (pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
+            break;
+    }
+    pgmPhysInvalidatePageMapTLB(pVM);
+
+    /*
+     * Notify NEM while holding the lock (experimental) and REM without (like always).
+     */
+    uint32_t const fNemNotify = (pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2       ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2   : 0)
+                              | (pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_OVERLAPPING ? NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE : 0);
+    int rc = NEMR3NotifyPhysMmioExMap(pVM, GCPhys, cbRange, fNemNotify, pFirstMmio->pvR3);
+
+    pgmUnlock(pVM);
+
+    return rc;
+}
+
+
+/**
+ * Unmaps an MMIO2 region.
+ *
+ * This is typically done when a guest / the bios / state loading changes the
+ * PCI config. The replacing of base memory has the same restrictions as during
+ * registration, of course.
+ */
+VMMR3_INT_DECL(int) PGMR3PhysMmio2Unmap(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS GCPhys)
+{
+    /*
+     * Validate input
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE, VERR_INVALID_HANDLE);
+    if (GCPhys != NIL_RTGCPHYS)
+    {
+        AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
+        AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    }
+
+    PPGMREGMMIO2RANGE pFirstMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
+    AssertReturn(pFirstMmio, VERR_NOT_FOUND);
+    Assert(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK);
+
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    PPGMREGMMIO2RANGE pLastMmio = pFirstMmio;
+    RTGCPHYS          cbRange   = 0;
+    for (;;)
+    {
+        AssertReturnStmt(pLastMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED, pgmUnlock(pVM), VERR_WRONG_ORDER);
+        AssertReturnStmt(pLastMmio->RamRange.GCPhys == GCPhys + cbRange || GCPhys == NIL_RTGCPHYS, pgmUnlock(pVM), VERR_INVALID_PARAMETER);
+        Assert(pLastMmio->pDevInsR3 == pFirstMmio->pDevInsR3);
+        Assert(pLastMmio->iSubDev   == pFirstMmio->iSubDev);
+        Assert(pLastMmio->iRegion   == pFirstMmio->iRegion);
+        cbRange += pLastMmio->RamRange.cb;
+        if (pLastMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
+            break;
+        pLastMmio = pLastMmio->pNextR3;
+    }
+
+    Log(("PGMR3PhysMmio2Unmap: %RGp-%RGp %s\n",
+         pFirstMmio->RamRange.GCPhys, pLastMmio->RamRange.GCPhysLast, pFirstMmio->RamRange.pszDesc));
+
+    uint16_t const fOldFlags = pFirstMmio->fFlags;
+    AssertReturnStmt(fOldFlags & PGMREGMMIO2RANGE_F_MAPPED, pgmUnlock(pVM), VERR_WRONG_ORDER);
+
+    /*
+     * If plain MMIO, we must deregister the handlers first.
+     */
+    if (!(fOldFlags & PGMREGMMIO2RANGE_F_MMIO2))
+    {
+        AssertFailed();
+
+        PPGMREGMMIO2RANGE pCurMmio = pFirstMmio;
+        rc = pgmHandlerPhysicalExDeregister(pVM, pFirstMmio->pPhysHandlerR3, RT_BOOL(fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING));
+        AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
+        while (!(pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK))
+        {
+            pCurMmio = pCurMmio->pNextR3;
+            rc = pgmHandlerPhysicalExDeregister(pVM, pCurMmio->pPhysHandlerR3, RT_BOOL(fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING));
+            AssertRCReturnStmt(rc, pgmUnlock(pVM), VERR_PGM_PHYS_MMIO_EX_IPE);
+        }
+    }
+
+    /*
+     * Unmap it.
+     */
+    RTGCPHYS const GCPhysRangeNotify = pFirstMmio->RamRange.GCPhys;
+    if (fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING)
+    {
+        /*
+         * We've replaced RAM, replace with zero pages.
+         *
+         * Note! This is where we might differ a little from a real system, because
+         *       it's likely to just show the RAM pages as they were before the
+         *       MMIO/MMIO2 region was mapped here.
+         */
+        /* Only one chunk allowed when overlapping! */
+        Assert(fOldFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK);
+
+        /* Restore the RAM pages we've replaced. */
+        PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
+        while (pRam->GCPhys > pFirstMmio->RamRange.GCPhysLast)
+            pRam = pRam->pNextR3;
+
+        uint32_t cPagesLeft = pFirstMmio->RamRange.cb >> PAGE_SHIFT;
+        if (fOldFlags & PGMREGMMIO2RANGE_F_MMIO2)
+            pVM->pgm.s.cZeroPages += cPagesLeft;
+
+        PPGMPAGE pPageDst = &pRam->aPages[(pFirstMmio->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+        while (cPagesLeft-- > 0)
+        {
+            PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
+            pPageDst++;
+        }
+
+        /* Flush physical page map TLB. */
+        pgmPhysInvalidatePageMapTLB(pVM);
+
+        /* Update range state. */
+        pFirstMmio->RamRange.GCPhys = NIL_RTGCPHYS;
+        pFirstMmio->RamRange.GCPhysLast = NIL_RTGCPHYS;
+        pFirstMmio->fFlags &= ~(PGMREGMMIO2RANGE_F_OVERLAPPING | PGMREGMMIO2RANGE_F_MAPPED);
+    }
+    else
+    {
+        /*
+         * Unlink the chunks related to the MMIO/MMIO2 region.
+         */
+        for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
+        {
+            pgmR3PhysUnlinkRamRange(pVM, &pCurMmio->RamRange);
+            pCurMmio->RamRange.GCPhys = NIL_RTGCPHYS;
+            pCurMmio->RamRange.GCPhysLast = NIL_RTGCPHYS;
+            pCurMmio->fFlags &= ~(PGMREGMMIO2RANGE_F_OVERLAPPING | PGMREGMMIO2RANGE_F_MAPPED);
+            if (pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
+                break;
+        }
+    }
+
+    /* Force a PGM pool flush as guest ram references have been changed. */
+    /** @todo not entirely SMP safe; assuming for now the guest takes care
+     *  of this internally (not touch mapped mmio while changing the
+     *  mapping). */
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+
+    pgmPhysInvalidatePageMapTLB(pVM);
+    pgmPhysInvalidRamRangeTlbs(pVM);
+
+    /*
+     * Notify NEM while holding the lock (experimental) and REM without (like always).
+     */
+    uint32_t const fNemFlags = (fOldFlags & PGMREGMMIO2RANGE_F_MMIO2       ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2   : 0)
+                             | (fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING ? NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE : 0);
+    rc = NEMR3NotifyPhysMmioExUnmap(pVM, GCPhysRangeNotify, cbRange, fNemFlags);
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Reduces the mapping size of a MMIO2 region.
+ *
+ * This is mainly for dealing with old saved states after changing the default
+ * size of a mapping region.  See PGMDevHlpMMIOExReduce and
+ * PDMPCIDEV::pfnRegionLoadChangeHookR3.
+ *
+ * The region must not currently be mapped when making this call.  The VM state
+ * must be state restore or VM construction.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance owning the region.
+ * @param   hMmio2          The handle of the region to reduce.
+ * @param   cbRegion        The new mapping size.
+ */
+VMMR3_INT_DECL(int) PGMR3PhysMmio2Reduce(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS cbRegion)
+{
+    /*
+     * Validate input
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE, VERR_INVALID_HANDLE);
+    AssertReturn(cbRegion >= X86_PAGE_SIZE, VERR_INVALID_PARAMETER);
+    AssertReturn(!(cbRegion & X86_PAGE_OFFSET_MASK), VERR_UNSUPPORTED_ALIGNMENT);
+    VMSTATE enmVmState = VMR3GetState(pVM);
+    AssertLogRelMsgReturn(   enmVmState == VMSTATE_CREATING
+                          || enmVmState == VMSTATE_LOADING,
+                          ("enmVmState=%d (%s)\n", enmVmState, VMR3GetStateName(enmVmState)),
+                          VERR_VM_INVALID_VM_STATE);
+
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    PPGMREGMMIO2RANGE pFirstMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
+    if (pFirstMmio)
+    {
+        Assert(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK);
+        if (!(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED))
+        {
+            /*
+             * NOTE! Current implementation does not support multiple ranges.
+             *       Implement when there is a real world need and thus a testcase.
+             */
+            AssertLogRelMsgStmt(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK,
+                                ("%s: %#x\n", pFirstMmio->RamRange.pszDesc, pFirstMmio->fFlags),
+                                rc = VERR_NOT_SUPPORTED);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Make the change.
+                 */
+                Log(("PGMR3PhysMmio2Reduce: %s changes from %RGp bytes (%RGp) to %RGp bytes.\n",
+                     pFirstMmio->RamRange.pszDesc, pFirstMmio->RamRange.cb, pFirstMmio->cbReal, cbRegion));
+
+                AssertLogRelMsgStmt(cbRegion <= pFirstMmio->cbReal,
+                                    ("%s: cbRegion=%#RGp cbReal=%#RGp\n", pFirstMmio->RamRange.pszDesc, cbRegion, pFirstMmio->cbReal),
+                                    rc = VERR_OUT_OF_RANGE);
+                if (RT_SUCCESS(rc))
+                {
+                    pFirstMmio->RamRange.cb = cbRegion;
+                }
+            }
+        }
+        else
+            rc = VERR_WRONG_ORDER;
+    }
+    else
+        rc = VERR_NOT_FOUND;
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Validates @a hMmio2, making sure it belongs to @a pDevIns.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     The device which allegedly owns @a hMmio2.
+ * @param   hMmio2      The handle to validate.
+ */
+VMMR3_INT_DECL(int) PGMR3PhysMmio2ValidateHandle(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
+{
+    /*
+     * Validate input
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
+
+    /*
+     * Just do this the simple way.  No need for locking as this is only taken at
+     */
+    pgmLock(pVM);
+    PPGMREGMMIO2RANGE pFirstMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
+    pgmUnlock(pVM);
+    AssertReturn(pFirstMmio, VERR_INVALID_HANDLE);
+    AssertReturn(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2, VERR_INVALID_HANDLE);
+    AssertReturn(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK, VERR_INVALID_HANDLE);
+    return VINF_SUCCESS;
+}
+
+
+#ifndef PGM_WITHOUT_MAPPINGS
+/**
+ * Gets the HC physical address of a page in the MMIO2 region.
+ *
+ * This is API is intended for MMHyper and shouldn't be called
+ * by anyone else...
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The owner of the memory, optional.
+ * @param   iSubDev         Sub-device number.
+ * @param   iRegion         The region.
+ * @param   off             The page expressed an offset into the MMIO2 region.
+ * @param   pHCPhys         Where to store the result.
+ */
+VMMR3_INT_DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion,
+                                            RTGCPHYS off, PRTHCPHYS pHCPhys)
+{
+    /*
+     * Validate input
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
+    AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
+
+    pgmLock(pVM);
+    PPGMREGMMIO2RANGE pCurMmio = pgmR3PhysMmio2Find(pVM, pDevIns, iSubDev, iRegion, NIL_PGMMMIO2HANDLE);
+    AssertReturn(pCurMmio, VERR_NOT_FOUND);
+    AssertReturn(pCurMmio->fFlags & (PGMREGMMIO2RANGE_F_MMIO2 | PGMREGMMIO2RANGE_F_FIRST_CHUNK), VERR_WRONG_TYPE);
+
+    while (   off >= pCurMmio->RamRange.cb
+           && !(pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK))
+    {
+        off -= pCurMmio->RamRange.cb;
+        pCurMmio = pCurMmio->pNextR3;
+    }
+    AssertReturn(off < pCurMmio->RamRange.cb, VERR_INVALID_PARAMETER);
+
+    PCPGMPAGE pPage = &pCurMmio->RamRange.aPages[off >> PAGE_SHIFT];
+    *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+    pgmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
+
+/**
+ * Gets the mapping address of an MMIO2 region.
+ *
+ * @returns Mapping address, NIL_RTGCPHYS if not mapped or invalid handle.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     The device owning the MMIO2 handle.
+ * @param   hMmio2      The region handle.
+ */
+VMMR3_INT_DECL(RTGCPHYS) PGMR3PhysMmio2GetMappingAddress(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
+{
+    AssertPtrReturn(pDevIns, NIL_RTGCPHYS);
+
+    PPGMREGMMIO2RANGE pFirstRegMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
+    AssertReturn(pFirstRegMmio, NIL_RTGCPHYS);
+
+    if (pFirstRegMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED)
+        return pFirstRegMmio->RamRange.GCPhys;
+    return NIL_RTGCPHYS;
+}
+
+/**
+ * Changes the region number of an MMIO2 region.
+ *
+ * This is only for dealing with save state issues, nothing else.
+ *
+ * @return VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pDevIns     The device owning the MMIO2 memory.
+ * @param   hMmio2      The handle of the region.
+ * @param   iNewRegion  The new region index.
+ *
+ * @thread  EMT(0)
+ * @sa      @bugref{9359}
+ */
+VMMR3_INT_DECL(int) PGMR3PhysMmio2ChangeRegionNo(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, uint32_t iNewRegion)
+{
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_LOADING, VERR_VM_INVALID_VM_STATE);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE, VERR_INVALID_HANDLE);
+    AssertReturn(iNewRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
+
+    AssertReturn(pVM->enmVMState == VMSTATE_LOADING, VERR_INVALID_STATE);
+
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    PPGMREGMMIO2RANGE pFirstRegMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
+    AssertReturnStmt(pFirstRegMmio, pgmUnlock(pVM), VERR_NOT_FOUND);
+    AssertReturnStmt(pgmR3PhysMmio2Find(pVM, pDevIns, pFirstRegMmio->iSubDev, iNewRegion, NIL_PGMMMIO2HANDLE) == NULL,
+                     pgmUnlock(pVM), VERR_RESOURCE_IN_USE);
+
+    /*
+     * Make the change.
+     */
+    pFirstRegMmio->iRegion = (uint8_t)iNewRegion;
+
+    pgmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PGMR3PhysRomRegister.
+ *
+ * This is here to simplify lock management, i.e. the caller does all the
+ * locking and we can simply return without needing to remember to unlock
+ * anything first.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pDevIns             The device instance owning the ROM.
+ * @param   GCPhys              First physical address in the range.
+ *                              Must be page aligned!
+ * @param   cb                  The size of the range (in bytes).
+ *                              Must be page aligned!
+ * @param   pvBinary            Pointer to the binary data backing the ROM image.
+ * @param   cbBinary            The size of the binary data pvBinary points to.
+ *                              This must be less or equal to @a cb.
+ * @param   fFlags              Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
+ *                              and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
+ * @param   pszDesc             Pointer to description string. This must not be freed.
+ */
+static int pgmR3PhysRomRegisterLocked(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
+                                      const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
+    AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
+    RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
+    AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertReturn(!(fFlags & ~PGMPHYS_ROM_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+    const uint32_t cPages = cb >> PAGE_SHIFT;
+
+    /*
+     * Find the ROM location in the ROM list first.
+     */
+    PPGMROMRANGE    pRomPrev = NULL;
+    PPGMROMRANGE    pRom = pVM->pgm.s.pRomRangesR3;
+    while (pRom && GCPhysLast >= pRom->GCPhys)
+    {
+        if (    GCPhys     <= pRom->GCPhysLast
+            &&  GCPhysLast >= pRom->GCPhys)
+            AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
+                                         GCPhys, GCPhysLast, pszDesc,
+                                         pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
+                                        VERR_PGM_RAM_CONFLICT);
+        /* next */
+        pRomPrev = pRom;
+        pRom = pRom->pNextR3;
+    }
+
+    /*
+     * Find the RAM location and check for conflicts.
+     *
+     * Conflict detection is a bit different than for RAM
+     * registration since a ROM can be located within a RAM
+     * range. So, what we have to check for is other memory
+     * types (other than RAM that is) and that we don't span
+     * more than one RAM range (layz).
+     */
+    bool            fRamExists = false;
+    PPGMRAMRANGE    pRamPrev = NULL;
+    PPGMRAMRANGE    pRam = pVM->pgm.s.pRamRangesXR3;
+    while (pRam && GCPhysLast >= pRam->GCPhys)
+    {
+        if (    GCPhys     <= pRam->GCPhysLast
+            &&  GCPhysLast >= pRam->GCPhys)
+        {
+            /* completely within? */
+            AssertLogRelMsgReturn(   GCPhys     >= pRam->GCPhys
+                                  && GCPhysLast <= pRam->GCPhysLast,
+                                  ("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
+                                   GCPhys, GCPhysLast, pszDesc,
+                                   pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
+                                  VERR_PGM_RAM_CONFLICT);
+            fRamExists = true;
+            break;
+        }
+
+        /* next */
+        pRamPrev = pRam;
+        pRam = pRam->pNextR3;
+    }
+    if (fRamExists)
+    {
+        PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+        uint32_t cPagesLeft = cPages;
+        while (cPagesLeft-- > 0)
+        {
+            AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
+                                  ("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
+                                   pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
+                                   pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
+            Assert(PGM_PAGE_IS_ZERO(pPage));
+            pPage++;
+        }
+    }
+
+    /*
+     * Update the base memory reservation if necessary.
+     */
+    uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
+    if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+        cExtraBaseCost += cPages;
+    if (cExtraBaseCost)
+    {
+        int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Allocate memory for the virgin copy of the RAM.
+     */
+    PGMMALLOCATEPAGESREQ pReq;
+    int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
+    AssertRCReturn(rc, rc);
+
+    for (uint32_t iPage = 0; iPage < cPages; iPage++)
+    {
+        pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
+        pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
+        pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
+    }
+
+    rc = GMMR3AllocatePagesPerform(pVM, pReq);
+    if (RT_FAILURE(rc))
+    {
+        GMMR3AllocatePagesCleanup(pReq);
+        return rc;
+    }
+
+    /*
+     * Allocate the new ROM range and RAM range (if necessary).
+     */
+    PPGMROMRANGE pRomNew;
+    rc = MMHyperAlloc(pVM, RT_UOFFSETOF_DYN(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
+    if (RT_SUCCESS(rc))
+    {
+        PPGMRAMRANGE pRamNew = NULL;
+        if (!fRamExists)
+            rc = MMHyperAlloc(pVM, RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Initialize and insert the RAM range (if required).
+             */
+            PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
+            if (!fRamExists)
+            {
+                pRamNew->pSelfR0       = MMHyperCCToR0(pVM, pRamNew);
+                pRamNew->GCPhys        = GCPhys;
+                pRamNew->GCPhysLast    = GCPhysLast;
+                pRamNew->cb            = cb;
+                pRamNew->pszDesc       = pszDesc;
+                pRamNew->fFlags        = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
+                pRamNew->pvR3          = NULL;
+                pRamNew->paLSPages     = NULL;
+
+                PPGMPAGE pPage = &pRamNew->aPages[0];
+                for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
+                {
+                    PGM_PAGE_INIT(pPage,
+                                  pReq->aPages[iPage].HCPhysGCPhys,
+                                  pReq->aPages[iPage].idPage,
+                                  PGMPAGETYPE_ROM,
+                                  PGM_PAGE_STATE_ALLOCATED);
+
+                    pRomPage->Virgin = *pPage;
+                }
+
+                pVM->pgm.s.cAllPages += cPages;
+                pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
+            }
+            else
+            {
+                PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+                for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
+                {
+                    PGM_PAGE_SET_TYPE(pVM, pPage,   PGMPAGETYPE_ROM);
+                    PGM_PAGE_SET_HCPHYS(pVM, pPage, pReq->aPages[iPage].HCPhysGCPhys);
+                    PGM_PAGE_SET_STATE(pVM, pPage,  PGM_PAGE_STATE_ALLOCATED);
+                    PGM_PAGE_SET_PAGEID(pVM, pPage, pReq->aPages[iPage].idPage);
+                    PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
+                    PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
+                    PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
+
+                    pRomPage->Virgin = *pPage;
+                }
+
+                pRamNew = pRam;
+
+                pVM->pgm.s.cZeroPages -= cPages;
+            }
+            pVM->pgm.s.cPrivatePages += cPages;
+
+            /* Flush physical page map TLB. */
+            pgmPhysInvalidatePageMapTLB(pVM);
+
+
+            /* Notify NEM before we register handlers. */
+            uint32_t const fNemNotify = (fRamExists ? NEM_NOTIFY_PHYS_ROM_F_REPLACE : 0)
+                                      | (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED ? NEM_NOTIFY_PHYS_ROM_F_SHADOW : 0);
+            rc = NEMR3NotifyPhysRomRegisterEarly(pVM, GCPhys, cb, fNemNotify);
+
+            /* Register the ROM access handler. */
+            if (RT_SUCCESS(rc))
+                rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, pVM->pgm.s.hRomPhysHandlerType,
+                                                pRomNew, MMHyperCCToR0(pVM, pRomNew), MMHyperCCToRC(pVM, pRomNew),
+                                                pszDesc);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Copy the image over to the virgin pages.
+                 * This must be done after linking in the RAM range.
+                 */
+                size_t          cbBinaryLeft = cbBinary;
+                PPGMPAGE        pRamPage     = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
+                for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
+                {
+                    void *pvDstPage;
+                    rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
+                    if (RT_FAILURE(rc))
+                    {
+                        VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
+                        break;
+                    }
+                    if (cbBinaryLeft >= PAGE_SIZE)
+                    {
+                        memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), PAGE_SIZE);
+                        cbBinaryLeft -= PAGE_SIZE;
+                    }
+                    else
+                    {
+                        ASMMemZeroPage(pvDstPage); /* (shouldn't be necessary, but can't hurt either) */
+                        if (cbBinaryLeft > 0)
+                        {
+                            memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), cbBinaryLeft);
+                            cbBinaryLeft = 0;
+                        }
+                    }
+                }
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Initialize the ROM range.
+                     * Note that the Virgin member of the pages has already been initialized above.
+                     */
+                    pRomNew->GCPhys     = GCPhys;
+                    pRomNew->GCPhysLast = GCPhysLast;
+                    pRomNew->cb         = cb;
+                    pRomNew->fFlags     = fFlags;
+                    pRomNew->idSavedState = UINT8_MAX;
+                    pRomNew->cbOriginal = cbBinary;
+                    pRomNew->pszDesc    = pszDesc;
+                    pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
+                                        ? pvBinary : RTMemDup(pvBinary, cbBinary);
+                    if (pRomNew->pvOriginal)
+                    {
+                        for (unsigned iPage = 0; iPage < cPages; iPage++)
+                        {
+                            PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
+                            pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
+                            PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
+                        }
+
+                        /* update the page count stats for the shadow pages. */
+                        if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+                        {
+                            pVM->pgm.s.cZeroPages += cPages;
+                            pVM->pgm.s.cAllPages  += cPages;
+                        }
+
+                        /*
+                         * Insert the ROM range, tell REM and return successfully.
+                         */
+                        pRomNew->pNextR3 = pRom;
+                        pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
+
+                        if (pRomPrev)
+                        {
+                            pRomPrev->pNextR3 = pRomNew;
+                            pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
+                        }
+                        else
+                        {
+                            pVM->pgm.s.pRomRangesR3 = pRomNew;
+                            pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
+                        }
+
+                        pgmPhysInvalidatePageMapTLB(pVM);
+                        GMMR3AllocatePagesCleanup(pReq);
+
+                        /* Notify NEM again. */
+                        return NEMR3NotifyPhysRomRegisterLate(pVM, GCPhys, cb, fNemNotify);
+                    }
+
+                    /* bail out */
+                    rc = VERR_NO_MEMORY;
+                }
+
+                int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
+                AssertRC(rc2);
+            }
+
+            if (!fRamExists)
+            {
+                pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
+                MMHyperFree(pVM, pRamNew);
+            }
+        }
+        MMHyperFree(pVM, pRomNew);
+    }
+
+    /** @todo Purge the mapping cache or something... */
+    GMMR3FreeAllocatedPages(pVM, pReq);
+    GMMR3AllocatePagesCleanup(pReq);
+    return rc;
+}
+
+
+/**
+ * Registers a ROM image.
+ *
+ * Shadowed ROM images requires double the amount of backing memory, so,
+ * don't use that unless you have to. Shadowing of ROM images is process
+ * where we can select where the reads go and where the writes go. On real
+ * hardware the chipset provides means to configure this. We provide
+ * PGMR3PhysProtectROM() for this purpose.
+ *
+ * A read-only copy of the ROM image will always be kept around while we
+ * will allocate RAM pages for the changes on demand (unless all memory
+ * is configured to be preallocated).
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pDevIns             The device instance owning the ROM.
+ * @param   GCPhys              First physical address in the range.
+ *                              Must be page aligned!
+ * @param   cb                  The size of the range (in bytes).
+ *                              Must be page aligned!
+ * @param   pvBinary            Pointer to the binary data backing the ROM image.
+ * @param   cbBinary            The size of the binary data pvBinary points to.
+ *                              This must be less or equal to @a cb.
+ * @param   fFlags              Mask of flags, PGMPHYS_ROM_FLAGS_XXX.
+ * @param   pszDesc             Pointer to description string. This must not be freed.
+ *
+ * @remark  There is no way to remove the rom, automatically on device cleanup or
+ *          manually from the device yet. This isn't difficult in any way, it's
+ *          just not something we expect to be necessary for a while.
+ */
+VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
+                                    const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
+{
+    Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
+         pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
+    pgmLock(pVM);
+    int rc = pgmR3PhysRomRegisterLocked(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Called by PGMR3MemSetup to reset the shadow, switch to the virgin, and verify
+ * that the virgin part is untouched.
+ *
+ * This is done after the normal memory has been cleared.
+ *
+ * ASSUMES that the caller owns the PGM lock.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+int pgmR3PhysRomReset(PVM pVM)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
+
+        if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+        {
+            /*
+             * Reset the physical handler.
+             */
+            int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
+            AssertRCReturn(rc, rc);
+
+            /*
+             * What we do with the shadow pages depends on the memory
+             * preallocation option. If not enabled, we'll just throw
+             * out all the dirty pages and replace them by the zero page.
+             */
+            if (!pVM->pgm.s.fRamPreAlloc)
+            {
+                /* Free the dirty pages. */
+                uint32_t            cPendingPages = 0;
+                PGMMFREEPAGESREQ    pReq;
+                rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+                AssertRCReturn(rc, rc);
+
+                for (uint32_t iPage = 0; iPage < cPages; iPage++)
+                    if (   !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
+                        && !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
+                    {
+                        Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
+                        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
+                                             pRom->GCPhys + (iPage << PAGE_SHIFT),
+                                             (PGMPAGETYPE)PGM_PAGE_GET_TYPE(&pRom->aPages[iPage].Shadow));
+                        AssertLogRelRCReturn(rc, rc);
+                    }
+
+                if (cPendingPages)
+                {
+                    rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+                    AssertLogRelRCReturn(rc, rc);
+                }
+                GMMR3FreePagesCleanup(pReq);
+            }
+            else
+            {
+                /* clear all the shadow pages. */
+                for (uint32_t iPage = 0; iPage < cPages; iPage++)
+                {
+                    if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
+                        continue;
+                    Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
+                    void *pvDstPage;
+                    const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
+                    rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
+                    if (RT_FAILURE(rc))
+                        break;
+                    ASMMemZeroPage(pvDstPage);
+                }
+                AssertRCReturn(rc, rc);
+            }
+        }
+
+        /*
+         * Restore the original ROM pages after a saved state load.
+         * Also, in strict builds check that ROM pages remain unmodified.
+         */
+#ifndef VBOX_STRICT
+        if (pVM->pgm.s.fRestoreRomPagesOnReset)
+#endif
+        {
+            size_t         cbSrcLeft = pRom->cbOriginal;
+            uint8_t const *pbSrcPage = (uint8_t const *)pRom->pvOriginal;
+            uint32_t       cRestored = 0;
+            for (uint32_t iPage = 0; iPage < cPages && cbSrcLeft > 0; iPage++, pbSrcPage += PAGE_SIZE)
+            {
+                const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
+                void const *pvDstPage;
+                int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
+                if (RT_FAILURE(rc))
+                    break;
+
+                if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE)))
+                {
+                    if (pVM->pgm.s.fRestoreRomPagesOnReset)
+                    {
+                        void *pvDstPageW;
+                        rc = pgmPhysPageMap(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPageW);
+                        AssertLogRelRCReturn(rc, rc);
+                        memcpy(pvDstPageW, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE));
+                        cRestored++;
+                    }
+                    else
+                        LogRel(("pgmR3PhysRomReset: %RGp: ROM page changed (%s)\n", GCPhys, pRom->pszDesc));
+                }
+                cbSrcLeft -= RT_MIN(cbSrcLeft, PAGE_SIZE);
+            }
+            if (cRestored > 0)
+                LogRel(("PGM: ROM \"%s\": Reloaded %u of %u pages.\n", pRom->pszDesc, cRestored, cPages));
+        }
+    }
+
+    /* Clear the ROM restore flag now as we only need to do this once after
+       loading saved state. */
+    pVM->pgm.s.fRestoreRomPagesOnReset = false;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Called by PGMR3Term to free resources.
+ *
+ * ASSUMES that the caller owns the PGM lock.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+void pgmR3PhysRomTerm(PVM pVM)
+{
+    /*
+     * Free the heap copy of the original bits.
+     */
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        if (   pRom->pvOriginal
+            && !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
+        {
+            RTMemFree((void *)pRom->pvOriginal);
+            pRom->pvOriginal = NULL;
+        }
+    }
+}
+
+
+/**
+ * Change the shadowing of a range of ROM pages.
+ *
+ * This is intended for implementing chipset specific memory registers
+ * and will not be very strict about the input. It will silently ignore
+ * any pages that are not the part of a shadowed ROM.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_PGM_SYNC_CR3
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      Where to start. Page aligned.
+ * @param   cb          How much to change. Page aligned.
+ * @param   enmProt     The new ROM protection.
+ */
+VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
+{
+    /*
+     * Check input
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
+    AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
+    AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
+
+    /*
+     * Process the request.
+     */
+    pgmLock(pVM);
+    int  rc = VINF_SUCCESS;
+    bool fFlushTLB = false;
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        if (   GCPhys     <= pRom->GCPhysLast
+            && GCPhysLast >= pRom->GCPhys
+            && (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
+        {
+            /*
+             * Iterate the relevant pages and make necessary the changes.
+             */
+            bool fChanges = false;
+            uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
+                                  ? pRom->cb >> PAGE_SHIFT
+                                  : (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
+            for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
+                 iPage < cPages;
+                 iPage++)
+            {
+                PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
+                if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
+                {
+                    fChanges = true;
+
+                    /* flush references to the page. */
+                    PPGMPAGE pRamPage = pgmPhysGetPage(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT));
+                    int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
+                                                       true /*fFlushPTEs*/, &fFlushTLB);
+                    if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS || RT_FAILURE(rc2)))
+                        rc = rc2;
+                    uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pRamPage);
+
+                    PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
+                    PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
+
+                    *pOld = *pRamPage;
+                    *pRamPage = *pNew;
+                    /** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
+
+                    /* Tell NEM about the backing and protection change. */
+                    if (VM_IS_NEM_ENABLED(pVM))
+                    {
+                        PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pNew);
+                        NEMHCNotifyPhysPageChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pOld), PGM_PAGE_GET_HCPHYS(pNew),
+                                                   pgmPhysPageCalcNemProtection(pRamPage, enmType), enmType, &u2State);
+                        PGM_PAGE_SET_NEM_STATE(pRamPage, u2State);
+                    }
+                }
+                pRomPage->enmProt = enmProt;
+            }
+
+            /*
+             * Reset the access handler if we made changes, no need
+             * to optimize this.
+             */
+            if (fChanges)
+            {
+                int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
+                if (RT_FAILURE(rc2))
+                {
+                    pgmUnlock(pVM);
+                    AssertRC(rc);
+                    return rc2;
+                }
+            }
+
+            /* Advance - cb isn't updated. */
+            GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
+        }
+    }
+    pgmUnlock(pVM);
+    if (fFlushTLB)
+        PGM_INVL_ALL_VCPU_TLBS(pVM);
+
+    return rc;
+}
+
+
+/**
+ * Sets the Address Gate 20 state.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   fEnable     True if the gate should be enabled.
+ *                      False if the gate should be disabled.
+ */
+VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
+{
+    LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
+    if (pVCpu->pgm.s.fA20Enabled != fEnable)
+    {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        PCCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
+        if (   CPUMIsGuestInVmxRootMode(pCtx)
+            && !fEnable)
+        {
+            Log(("Cannot enter A20M mode while in VMX root mode\n"));
+            return;
+        }
+#endif
+        pVCpu->pgm.s.fA20Enabled = fEnable;
+        pVCpu->pgm.s.GCPhysA20Mask = ~((RTGCPHYS)!fEnable << 20);
+        NEMR3NotifySetA20(pVCpu, fEnable);
+#ifdef PGM_WITH_A20
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        pgmR3RefreshShadowModeAfterA20Change(pVCpu);
+        HMFlushTlb(pVCpu);
+#endif
+        IEMTlbInvalidateAllPhysical(pVCpu);
+        STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cA20Changes);
+    }
+}
+
+
+/**
+ * Tree enumeration callback for dealing with age rollover.
+ * It will perform a simple compression of the current age.
+ */
+static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
+{
+    /* Age compression - ASSUMES iNow == 4. */
+    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
+    if (pChunk->iLastUsed >= UINT32_C(0xffffff00))
+        pChunk->iLastUsed = 3;
+    else if (pChunk->iLastUsed >= UINT32_C(0xfffff000))
+        pChunk->iLastUsed = 2;
+    else if (pChunk->iLastUsed)
+        pChunk->iLastUsed = 1;
+    else /* iLastUsed = 0 */
+        pChunk->iLastUsed = 4;
+
+    NOREF(pvUser);
+    return 0;
+}
+
+
+/**
+ * The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
+ */
+typedef struct PGMR3PHYSCHUNKUNMAPCB
+{
+    PVM                 pVM;            /**< Pointer to the VM. */
+    PPGMCHUNKR3MAP      pChunk;         /**< The chunk to unmap. */
+} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
+
+
+/**
+ * Callback used to find the mapping that's been unused for
+ * the longest time.
+ */
+static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
+{
+    PPGMCHUNKR3MAP          pChunk = (PPGMCHUNKR3MAP)pNode;
+    PPGMR3PHYSCHUNKUNMAPCB  pArg   = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
+
+    /*
+     * Check for locks and compare when last used.
+     */
+    if (pChunk->cRefs)
+        return 0;
+    if (pChunk->cPermRefs)
+        return 0;
+    if (   pArg->pChunk
+        && pChunk->iLastUsed >= pArg->pChunk->iLastUsed)
+        return 0;
+
+    /*
+     * Check that it's not in any of the TLBs.
+     */
+    PVM pVM = pArg->pVM;
+    if (   pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(pChunk->Core.Key)].idChunk
+        == pChunk->Core.Key)
+    {
+        pChunk = NULL;
+        return 0;
+    }
+#ifdef VBOX_STRICT
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
+    {
+        Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk != pChunk);
+        Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk != pChunk->Core.Key);
+    }
+#endif
+
+#ifndef VBOX_WITH_RAM_IN_KERNEL
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR0.aEntries); i++)
+        if (pVM->pgm.s.PhysTlbR0.aEntries[i].pMap == pChunk)
+            return 0;
+#endif
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR3.aEntries); i++)
+        if (pVM->pgm.s.PhysTlbR3.aEntries[i].pMap == pChunk)
+            return 0;
+
+    pArg->pChunk = pChunk;
+    return 0;
+}
+
+
+/**
+ * Finds a good candidate for unmapping when the ring-3 mapping cache is full.
+ *
+ * The candidate will not be part of any TLBs, so no need to flush
+ * anything afterwards.
+ *
+ * @returns Chunk id.
+ * @param   pVM         The cross context VM structure.
+ */
+static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Enumerate the age tree starting with the left most node.
+     */
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
+    PGMR3PHYSCHUNKUNMAPCB Args;
+    Args.pVM    = pVM;
+    Args.pChunk = NULL;
+    RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
+    Assert(Args.pChunk);
+    if (Args.pChunk)
+    {
+        Assert(Args.pChunk->cRefs == 0);
+        Assert(Args.pChunk->cPermRefs == 0);
+        STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
+        return Args.pChunk->Core.Key;
+    }
+
+    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
+    return INT32_MAX;
+}
+
+
+/**
+ * Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
+ *
+ * This is only called on one of the EMTs while the other ones are waiting for
+ * it to complete this function.
+ *
+ * @returns VINF_SUCCESS (VBox strict status code).
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT. Unused.
+ * @param   pvUser      User pointer. Unused
+ *
+ */
+static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    pgmLock(pVM);
+    NOREF(pVCpu); NOREF(pvUser);
+
+    if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
+    {
+        /* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
+        /** @todo also not really efficient to unmap a chunk that contains PD
+         *  or PT pages. */
+        pgmR3PoolClearAllRendezvous(pVM, pVM->apCpusR3[0], NULL /* no need to flush the REM TLB as we already did that above */);
+
+        /*
+         * Request the ring-0 part to unmap a chunk to make space in the mapping cache.
+         */
+        GMMMAPUNMAPCHUNKREQ Req;
+        Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+        Req.Hdr.cbReq    = sizeof(Req);
+        Req.pvR3         = NULL;
+        Req.idChunkMap   = NIL_GMM_CHUNKID;
+        Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
+        if (Req.idChunkUnmap != INT32_MAX)
+        {
+            STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
+            rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
+            STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Remove the unmapped one.
+                 */
+                PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
+                AssertRelease(pUnmappedChunk);
+                AssertRelease(!pUnmappedChunk->cRefs);
+                AssertRelease(!pUnmappedChunk->cPermRefs);
+                pUnmappedChunk->pv       = NULL;
+                pUnmappedChunk->Core.Key = UINT32_MAX;
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+                MMR3HeapFree(pUnmappedChunk);
+#else
+                MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
+#endif
+                pVM->pgm.s.ChunkR3Map.c--;
+                pVM->pgm.s.cUnmappedChunks++;
+
+                /*
+                 * Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses).
+                 */
+                /** @todo We should not flush chunks which include cr3 mappings. */
+                for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                {
+                    PPGMCPU pPGM = &pVM->apCpusR3[idCpu]->pgm.s;
+
+                    pPGM->pGst32BitPdR3    = NULL;
+                    pPGM->pGstPaePdptR3    = NULL;
+                    pPGM->pGstAmd64Pml4R3  = NULL;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+                    pPGM->pGst32BitPdR0    = NIL_RTR0PTR;
+                    pPGM->pGstPaePdptR0    = NIL_RTR0PTR;
+                    pPGM->pGstAmd64Pml4R0  = NIL_RTR0PTR;
+#endif
+                    for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
+                    {
+                        pPGM->apGstPaePDsR3[i]             = NULL;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+                        pPGM->apGstPaePDsR0[i]             = NIL_RTR0PTR;
+#endif
+                    }
+
+                    /* Flush REM TLBs. */
+                    CPUMSetChangedFlags(pVM->apCpusR3[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
+                }
+            }
+        }
+    }
+    pgmUnlock(pVM);
+    return rc;
+}
+
+/**
+ * Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+void pgmR3PhysUnmapChunk(PVM pVM)
+{
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
+    AssertRC(rc);
+}
+
+
+/**
+ * Maps the given chunk into the ring-3 mapping cache.
+ *
+ * This will call ring-0.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   idChunk     The chunk in question.
+ * @param   ppChunk     Where to store the chunk tracking structure.
+ *
+ * @remarks Called from within the PGM critical section.
+ * @remarks Can be called from any thread!
+ */
+int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
+{
+    int rc;
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Move the chunk time forward.
+     */
+    pVM->pgm.s.ChunkR3Map.iNow++;
+    if (pVM->pgm.s.ChunkR3Map.iNow == 0)
+    {
+        pVM->pgm.s.ChunkR3Map.iNow = 4;
+        RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkAgeingRolloverCallback, NULL);
+    }
+
+    /*
+     * Allocate a new tracking structure first.
+     */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
+#else
+    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
+#endif
+    AssertReturn(pChunk, VERR_NO_MEMORY);
+    pChunk->Core.Key  = idChunk;
+    pChunk->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
+
+    /*
+     * Request the ring-0 part to map the chunk in question.
+     */
+    GMMMAPUNMAPCHUNKREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq    = sizeof(Req);
+    Req.pvR3         = NULL;
+    Req.idChunkMap   = idChunk;
+    Req.idChunkUnmap = NIL_GMM_CHUNKID;
+
+    /* Must be callable from any thread, so can't use VMMR3CallR0. */
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
+    rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
+    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
+    if (RT_SUCCESS(rc))
+    {
+        pChunk->pv = Req.pvR3;
+
+        /*
+         * If we're running out of virtual address space, then we should
+         * unmap another chunk.
+         *
+         * Currently, an unmap operation requires that all other virtual CPUs
+         * are idling and not by chance making use of the memory we're
+         * unmapping.  So, we create an async unmap operation here.
+         *
+         * Now, when creating or restoring a saved state this wont work very
+         * well since we may want to restore all guest RAM + a little something.
+         * So, we have to do the unmap synchronously.  Fortunately for us
+         * though, during these operations the other virtual CPUs are inactive
+         * and it should be safe to do this.
+         */
+        /** @todo Eventually we should lock all memory when used and do
+         *        map+unmap as one kernel call without any rendezvous or
+         *        other precautions. */
+        if (pVM->pgm.s.ChunkR3Map.c + 1 >= pVM->pgm.s.ChunkR3Map.cMax)
+        {
+            switch (VMR3GetState(pVM))
+            {
+                case VMSTATE_LOADING:
+                case VMSTATE_SAVING:
+                {
+                    PVMCPU pVCpu = VMMGetCpu(pVM);
+                    if (   pVCpu
+                        && pVM->pgm.s.cDeprecatedPageLocks == 0)
+                    {
+                        pgmR3PhysUnmapChunkRendezvous(pVM, pVCpu, NULL);
+                        break;
+                    }
+                }
+                RT_FALL_THRU();
+                default:
+                    rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
+                    AssertRC(rc);
+                    break;
+            }
+        }
+
+        /*
+         * Update the tree.  We must do this after any unmapping to make sure
+         * the chunk we're going to return isn't unmapped by accident.
+         */
+        AssertPtr(Req.pvR3);
+        bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
+        AssertRelease(fRc);
+        pVM->pgm.s.ChunkR3Map.c++;
+        pVM->pgm.s.cMappedChunks++;
+    }
+    else
+    {
+        /** @todo this may fail because of /proc/sys/vm/max_map_count, so we
+         *        should probably restrict ourselves on linux. */
+        AssertRC(rc);
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+        MMR3HeapFree(pChunk);
+#else
+        MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
+#endif
+        pChunk = NULL;
+    }
+
+    *ppChunk = pChunk;
+    return rc;
+}
+
+
+/**
+ * For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
+ *
+ * @returns see pgmR3PhysChunkMap.
+ * @param   pVM         The cross context VM structure.
+ * @param   idChunk     The chunk to map.
+ */
+VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
+{
+    PPGMCHUNKR3MAP pChunk;
+    int rc;
+
+    pgmLock(pVM);
+    rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Invalidates the TLB for the ring-3 mapping cache.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
+{
+    pgmLock(pVM);
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
+    {
+        pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
+        pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
+    }
+    /* The page map TLB references chunks, so invalidate that one too. */
+    pgmPhysInvalidatePageMapTLB(pVM);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE to allocate a large
+ * (2MB) page for use with a nested paging PDE.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_EM_NO_MEMORY if we're out of memory.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      GC physical start address of the 2 MB range
+ */
+VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
+{
+#ifdef PGM_WITH_LARGE_PAGES
+    uint64_t u64TimeStamp1, u64TimeStamp2;
+
+    pgmLock(pVM);
+
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
+    u64TimeStamp1 = RTTimeMilliTS();
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
+    u64TimeStamp2 = RTTimeMilliTS();
+    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pVM->pgm.s.cLargeHandyPages == 1);
+
+        uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
+        RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
+
+        void *pv;
+
+        /* Map the large page into our address space.
+         *
+         * Note: assuming that within the 2 MB range:
+         * - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
+         * - user space mapping is continuous as well
+         * - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
+         */
+        rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
+        AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
+
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Clear the pages.
+             */
+            STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
+            for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
+            {
+                ASMMemZeroPage(pv);
+
+                PPGMPAGE pPage;
+                rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+                AssertRC(rc);
+
+                Assert(PGM_PAGE_IS_ZERO(pPage));
+                STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
+                pVM->pgm.s.cZeroPages--;
+
+                /*
+                 * Do the PGMPAGE modifications.
+                 */
+                pVM->pgm.s.cPrivatePages++;
+                PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
+                PGM_PAGE_SET_PAGEID(pVM, pPage, idPage);
+                PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
+                PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE);
+                PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
+                PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
+
+                /* Somewhat dirty assumption that page ids are increasing. */
+                idPage++;
+
+                HCPhys += PAGE_SIZE;
+                GCPhys += PAGE_SIZE;
+
+                pv = (void *)((uintptr_t)pv + PAGE_SIZE);
+
+                Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
+            }
+            STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
+
+            /* Flush all TLBs. */
+            PGM_INVL_ALL_VCPU_TLBS(pVM);
+            pgmPhysInvalidatePageMapTLB(pVM);
+        }
+        pVM->pgm.s.cLargeHandyPages = 0;
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        static uint32_t cTimeOut = 0;
+        uint64_t u64TimeStampDelta = u64TimeStamp2 - u64TimeStamp1;
+
+        if (u64TimeStampDelta > 100)
+        {
+            STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatLargePageOverflow);
+            if (    ++cTimeOut > 10
+                ||  u64TimeStampDelta > 1000 /* more than one second forces an early retirement from allocating large pages. */)
+            {
+                /* If repeated attempts to allocate a large page takes more than 100 ms, then we fall back to normal 4k pages.
+                 * E.g. Vista 64 tries to move memory around, which takes a huge amount of time.
+                 */
+                LogRel(("PGMR3PhysAllocateLargePage: allocating large pages takes too long (last attempt %d ms; nr of timeouts %d); DISABLE\n", u64TimeStampDelta, cTimeOut));
+                PGMSetLargePageUsage(pVM, false);
+            }
+        }
+        else
+        if (cTimeOut > 0)
+            cTimeOut--;
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+#else
+    RT_NOREF(pVM, GCPhys);
+    return VERR_NOT_IMPLEMENTED;
+#endif /* PGM_WITH_LARGE_PAGES */
+}
+
+
+/**
+ * Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
+ *
+ * This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
+ * signal and clear the out of memory condition. When contracted, this API is
+ * used to try clear the condition when the user wants to resume.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success. FFs cleared.
+ * @retval  VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
+ *          this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
+ *
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
+ *          in EM.cpp and shouldn't be propagated outside TRPM, HM, EM and
+ *          pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
+ *          handler.
+ */
+VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
+{
+    pgmLock(pVM);
+
+    /*
+     * Allocate more pages, noting down the index of the first new page.
+     */
+    uint32_t iClear = pVM->pgm.s.cHandyPages;
+    AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_PGM_HANDY_PAGE_IPE);
+    Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
+    int rcAlloc = VINF_SUCCESS;
+    int rcSeed  = VINF_SUCCESS;
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
+    while (rc == VERR_GMM_SEED_ME)
+    {
+        void *pvChunk;
+        rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
+        if (RT_SUCCESS(rc))
+        {
+            rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
+            if (RT_FAILURE(rc))
+                SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
+        }
+        if (RT_SUCCESS(rc))
+            rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
+    }
+
+    /** @todo we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
+    if (    rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
+        &&  pVM->pgm.s.cHandyPages > 0)
+    {
+        /* Still handy pages left, so don't panic. */
+        rc = VINF_SUCCESS;
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        Assert(pVM->pgm.s.cHandyPages > 0);
+        VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
+        VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
+
+#ifdef VBOX_STRICT
+        uint32_t i;
+        for (i = iClear; i < pVM->pgm.s.cHandyPages; i++)
+            if (   pVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID
+                || pVM->pgm.s.aHandyPages[i].idSharedPage != NIL_GMM_PAGEID
+                || (pVM->pgm.s.aHandyPages[i].HCPhysGCPhys & PAGE_OFFSET_MASK))
+                break;
+        if (i != pVM->pgm.s.cHandyPages)
+        {
+            RTAssertMsg1Weak(NULL, __LINE__, __FILE__, __FUNCTION__);
+            RTAssertMsg2Weak("i=%d iClear=%d cHandyPages=%d\n", i, iClear, pVM->pgm.s.cHandyPages);
+            for (uint32_t j = iClear; j < pVM->pgm.s.cHandyPages; j++)
+                RTAssertMsg2Add("%03d: idPage=%d HCPhysGCPhys=%RHp idSharedPage=%d%\n", j,
+                                pVM->pgm.s.aHandyPages[j].idPage,
+                                pVM->pgm.s.aHandyPages[j].HCPhysGCPhys,
+                                pVM->pgm.s.aHandyPages[j].idSharedPage,
+                                j == i ? " <---" : "");
+            RTAssertPanic();
+        }
+#endif
+        /*
+         * Clear the pages.
+         */
+        while (iClear < pVM->pgm.s.cHandyPages)
+        {
+            PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
+            void *pv;
+            rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
+            AssertLogRelMsgBreak(RT_SUCCESS(rc),
+                                 ("%u/%u: idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n",
+                                  iClear, pVM->pgm.s.cHandyPages, pPage->idPage, pPage->HCPhysGCPhys, rc));
+            ASMMemZeroPage(pv);
+            iClear++;
+            Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
+        }
+    }
+    else
+    {
+        uint64_t cAllocPages, cMaxPages, cBalloonPages;
+
+        /*
+         * We should never get here unless there is a genuine shortage of
+         * memory (or some internal error). Flag the error so the VM can be
+         * suspended ASAP and the user informed. If we're totally out of
+         * handy pages we will return failure.
+         */
+        /* Report the failure. */
+        LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
+                "     cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
+                rc, rcAlloc, rcSeed,
+                pVM->pgm.s.cHandyPages,
+                pVM->pgm.s.cAllPages,
+                pVM->pgm.s.cPrivatePages,
+                pVM->pgm.s.cSharedPages,
+                pVM->pgm.s.cZeroPages));
+
+        if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
+        {
+            LogRel(("GMM: Statistics:\n"
+                    "     Allocated pages: %RX64\n"
+                    "     Maximum   pages: %RX64\n"
+                    "     Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
+        }
+
+        if (   rc != VERR_NO_MEMORY
+            && rc != VERR_NO_PHYS_MEMORY
+            && rc != VERR_LOCK_FAILED)
+        {
+            for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
+            {
+                LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
+                        i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
+                        pVM->pgm.s.aHandyPages[i].idSharedPage));
+                uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
+                if (idPage != NIL_GMM_PAGEID)
+                {
+                    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
+                         pRam;
+                         pRam = pRam->pNextR3)
+                    {
+                        uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
+                        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+                            if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
+                                LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
+                                        pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
+                    }
+                }
+            }
+        }
+
+        if (rc == VERR_NO_MEMORY)
+        {
+            uint64_t cbHostRamAvail = 0;
+            int rc2 = RTSystemQueryAvailableRam(&cbHostRamAvail);
+            if (RT_SUCCESS(rc2))
+                LogRel(("Host RAM: %RU64MB available\n", cbHostRamAvail / _1M));
+            else
+                LogRel(("Cannot determine the amount of available host memory\n"));
+        }
+
+        /* Set the FFs and adjust rc. */
+        VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
+        VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
+        if (    rc == VERR_NO_MEMORY
+            ||  rc == VERR_NO_PHYS_MEMORY
+            ||  rc == VERR_LOCK_FAILED)
+            rc = VINF_EM_NO_MEMORY;
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Frees the specified RAM page and replaces it with the ZERO page.
+ *
+ * This is used by ballooning, remapping MMIO2, RAM reset and state loading.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pReq            Pointer to the request.
+ * @param   pcPendingPages  Where the number of pages waiting to be freed are
+ *                          kept.  This will normally be incremented.
+ * @param   pPage           Pointer to the page structure.
+ * @param   GCPhys          The guest physical address of the page, if applicable.
+ * @param   enmNewType      New page type for NEM notification, since several
+ *                          callers will change the type upon successful return.
+ *
+ * @remarks The caller must own the PGM lock.
+ */
+int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys,
+                    PGMPAGETYPE enmNewType)
+{
+    /*
+     * Assert sanity.
+     */
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    if (RT_UNLIKELY(    PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
+                    &&  PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
+    {
+        AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
+        return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
+    }
+
+    /** @todo What about ballooning of large pages??! */
+    Assert(   PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
+           && PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
+
+    if (    PGM_PAGE_IS_ZERO(pPage)
+        ||  PGM_PAGE_IS_BALLOONED(pPage))
+        return VINF_SUCCESS;
+
+    const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
+    Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
+    if (RT_UNLIKELY(    idPage == NIL_GMM_PAGEID
+                    ||  idPage > GMM_PAGEID_LAST
+                    ||  PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
+    {
+        AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
+        return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
+    }
+    const RTHCPHYS HCPhysPrev = PGM_PAGE_GET_HCPHYS(pPage);
+
+    /* update page count stats. */
+    if (PGM_PAGE_IS_SHARED(pPage))
+        pVM->pgm.s.cSharedPages--;
+    else
+        pVM->pgm.s.cPrivatePages--;
+    pVM->pgm.s.cZeroPages++;
+
+    /* Deal with write monitored pages. */
+    if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
+    {
+        PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
+        pVM->pgm.s.cWrittenToPages++;
+    }
+
+    /*
+     * pPage = ZERO page.
+     */
+    PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
+    PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
+    PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
+    PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
+    PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
+    PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
+
+    /* Flush physical page map TLB entry. */
+    pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
+
+    /* Notify NEM. */
+    /** @todo consider doing batch NEM notifications.  */
+    if (VM_IS_NEM_ENABLED(pVM))
+    {
+        uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+        NEMHCNotifyPhysPageChanged(pVM, GCPhys, HCPhysPrev, pVM->pgm.s.HCPhysZeroPg,
+                                   pgmPhysPageCalcNemProtection(pPage, enmNewType), enmNewType, &u2State);
+        PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+    }
+
+    /*
+     * Make sure it's not in the handy page array.
+     */
+    for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
+    {
+        if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
+        {
+            pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
+            break;
+        }
+        if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
+        {
+            pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
+            break;
+        }
+    }
+
+    /*
+     * Push it onto the page array.
+     */
+    uint32_t iPage = *pcPendingPages;
+    Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
+    *pcPendingPages += 1;
+
+    pReq->aPages[iPage].idPage = idPage;
+
+    if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
+        return VINF_SUCCESS;
+
+    /*
+     * Flush the pages.
+     */
+    int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
+    if (RT_SUCCESS(rc))
+    {
+        GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+        *pcPendingPages = 0;
+    }
+    return rc;
+}
+
+
+/**
+ * Converts a GC physical address to a HC ring-3 pointer, with some
+ * additional checks.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
+ *          access handler of some kind.
+ * @retval  VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
+ *          accesses or is odd in any way.
+ * @retval  VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address to convert.  Since this is only
+ *                      used for filling the REM TLB, the A20 mask must be
+ *                      applied before calling this API.
+ * @param   fWritable   Whether write access is required.
+ * @param   ppv         Where to store the pointer corresponding to GCPhys on
+ *                      success.
+ */
+VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
+{
+    pgmLock(pVM);
+    PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
+
+    PPGMRAMRANGE pRam;
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
+    if (RT_SUCCESS(rc))
+    {
+        if (PGM_PAGE_IS_BALLOONED(pPage))
+            rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
+        else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
+            rc = VINF_SUCCESS;
+        else
+        {
+            if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
+                rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
+            else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+            {
+                /** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
+                 *        in -norawr0 mode. */
+                if (fWritable)
+                    rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
+            }
+            else
+            {
+                /* Temporarily disabled physical handler(s), since the recompiler
+                   doesn't get notified when it's reset we'll have to pretend it's
+                   operating normally. */
+                if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
+                    rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
+                else
+                    rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
+            }
+        }
+        if (RT_SUCCESS(rc))
+        {
+            int rc2;
+
+            /* Make sure what we return is writable. */
+            if (fWritable)
+                switch (PGM_PAGE_GET_STATE(pPage))
+                {
+                    case PGM_PAGE_STATE_ALLOCATED:
+                        break;
+                    case PGM_PAGE_STATE_BALLOONED:
+                        AssertFailed();
+                        break;
+                    case PGM_PAGE_STATE_ZERO:
+                    case PGM_PAGE_STATE_SHARED:
+                        if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
+                            break;
+                        RT_FALL_THRU();
+                    case PGM_PAGE_STATE_WRITE_MONITORED:
+                        rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
+                        AssertLogRelRCReturn(rc2, rc2);
+                        break;
+                }
+
+            /* Get a ring-3 mapping of the address. */
+            PPGMPAGER3MAPTLBE pTlbe;
+            rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
+            AssertLogRelRCReturn(rc2, rc2);
+            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+            /** @todo mapping/locking hell; this isn't horribly efficient since
+             *        pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
+
+            Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] *ppv=%p\n", GCPhys, rc, pPage, *ppv));
+        }
+        else
+            Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
+
+        /* else: handler catching all access, no pointer returned. */
+    }
+    else
+        rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMR3/PGMPhysRWTmpl.h b/src/VBox/VMM/VMMR3/PGMPhysRWTmpl.h
new file mode 100644
index 00000000..8afbfcd8
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PGMPhysRWTmpl.h
@@ -0,0 +1,61 @@
+/* $Id: PGMPhysRWTmpl.h $ */
+/** @file
+ * PGM - Page Manager and Monitor, Physical Memory Access Template.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/**
+ * Read physical memory. (one byte/word/dword)
+ *
+ * This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
+ * want to ignore those.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Physical address start reading from.
+ * @param   enmOrigin       Who is calling.
+ */
+VMMDECL(PGMPHYS_DATATYPE) PGMPHYSFN_READNAME(PVM pVM, RTGCPHYS GCPhys, PGMACCESSORIGIN enmOrigin)
+{
+    Assert(VM_IS_EMT(pVM));
+    PGMPHYS_DATATYPE val;
+    VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, GCPhys, &val, sizeof(val), enmOrigin);
+    AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); NOREF(rcStrict);
+    return val;
+}
+
+
+/**
+ * Write to physical memory. (one byte/word/dword)
+ *
+ * This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
+ * want to ignore those.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Physical address to write to.
+ * @param   val             What to write.
+ * @param   enmOrigin       Who is calling.
+ */
+VMMDECL(void) PGMPHYSFN_WRITENAME(PVM pVM, RTGCPHYS GCPhys, PGMPHYS_DATATYPE val, PGMACCESSORIGIN enmOrigin)
+{
+    Assert(VM_IS_EMT(pVM));
+    VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, GCPhys, &val, sizeof(val), enmOrigin);
+    AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); NOREF(rcStrict);
+}
+
+#undef PGMPHYSFN_READNAME
+#undef PGMPHYSFN_WRITENAME
+#undef PGMPHYS_DATATYPE
+#undef PGMPHYS_DATASIZE
+
diff --git a/src/VBox/VMM/VMMR3/PGMPool.cpp b/src/VBox/VMM/VMMR3/PGMPool.cpp
new file mode 100644
index 00000000..c4130849
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PGMPool.cpp
@@ -0,0 +1,929 @@
+/* $Id: PGMPool.cpp $ */
+/** @file
+ * PGM Shadow Page Pool.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_pgm_pool       PGM Shadow Page Pool
+ *
+ * Motivations:
+ *      -# Relationship between shadow page tables and physical guest pages. This
+ *         should allow us to skip most of the global flushes now following access
+ *         handler changes. The main expense is flushing shadow pages.
+ *      -# Limit the pool size if necessary (default is kind of limitless).
+ *      -# Allocate shadow pages from RC. We use to only do this in SyncCR3.
+ *      -# Required for 64-bit guests.
+ *      -# Combining the PD cache and page pool in order to simplify caching.
+ *
+ *
+ * @section sec_pgm_pool_outline    Design Outline
+ *
+ * The shadow page pool tracks pages used for shadowing paging structures (i.e.
+ * page tables, page directory, page directory pointer table and page map
+ * level-4). Each page in the pool has an unique identifier. This identifier is
+ * used to link a guest physical page to a shadow PT. The identifier is a
+ * non-zero value and has a relativly low max value - say 14 bits. This makes it
+ * possible to fit it into the upper bits of the of the aHCPhys entries in the
+ * ram range.
+ *
+ * By restricting host physical memory to the first 48 bits (which is the
+ * announced physical memory range of the K8L chip (scheduled for 2008)), we
+ * can safely use the upper 16 bits for shadow page ID and reference counting.
+ *
+ * Update: The 48 bit assumption will be lifted with the new physical memory
+ * management (PGMPAGE), so we won't have any trouble when someone stuffs 2TB
+ * into a box in some years.
+ *
+ * Now, it's possible for a page to be aliased, i.e. mapped by more than one PT
+ * or PD. This is solved by creating a list of physical cross reference extents
+ * when ever this happens. Each node in the list (extent) is can contain 3 page
+ * pool indexes. The list it self is chained using indexes into the paPhysExt
+ * array.
+ *
+ *
+ * @section sec_pgm_pool_life       Life Cycle of a Shadow Page
+ *
+ * -# The SyncPT function requests a page from the pool.
+ *    The request includes the kind of page it is (PT/PD, PAE/legacy), the
+ *    address of the page it's shadowing, and more.
+ * -# The pool responds to the request by allocating a new page.
+ *    When the cache is enabled, it will first check if it's in the cache.
+ *    Should the pool be exhausted, one of two things can be done:
+ *      -# Flush the whole pool and current CR3.
+ *      -# Use the cache to find a page which can be flushed (~age).
+ * -# The SyncPT function will sync one or more pages and insert it into the
+ *    shadow PD.
+ * -# The SyncPage function may sync more pages on a later \#PFs.
+ * -# The page is freed / flushed in SyncCR3 (perhaps) and some other cases.
+ *    When caching is enabled, the page isn't flush but remains in the cache.
+ *
+ *
+ * @section sec_pgm_pool_monitoring Monitoring
+ *
+ * We always monitor PAGE_SIZE chunks of memory. When we've got multiple shadow
+ * pages for the same PAGE_SIZE of guest memory (PAE and mixed PD/PT) the pages
+ * sharing the monitor get linked using the iMonitoredNext/Prev. The head page
+ * is the pvUser to the access handlers.
+ *
+ *
+ * @section sec_pgm_pool_impl       Implementation
+ *
+ * The pool will take pages from the MM page pool. The tracking data
+ * (attributes, bitmaps and so on) are allocated from the hypervisor heap. The
+ * pool content can be accessed both by using the page id and the physical
+ * address (HC). The former is managed by means of an array, the latter by an
+ * offset based AVL tree.
+ *
+ * Flushing of a pool page means that we iterate the content (we know what kind
+ * it is) and updates the link information in the ram range.
+ *
+ * ...
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_POOL
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/mm.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include "PGMInline.h"
+
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <VBox/dbg.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_DEBUGGER
+static FNDBGCCMD pgmR3PoolCmdCheck;
+#endif
+
+#ifdef VBOX_WITH_DEBUGGER
+/** Command descriptors. */
+static const DBGCCMD    g_aCmds[] =
+{
+    /* pszCmd,  cArgsMin, cArgsMax, paArgDesc, cArgDescs, fFlags, pfnHandler          pszSyntax,  ....pszDescription */
+    { "pgmpoolcheck",  0, 0,        NULL,      0,         0,      pgmR3PoolCmdCheck,  "",         "Check the pgm pool pages." },
+};
+#endif
+
+/**
+ * Initializes the pool
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+int pgmR3PoolInit(PVM pVM)
+{
+    int rc;
+
+    AssertCompile(NIL_PGMPOOL_IDX == 0);
+    /* pPage->cLocked is an unsigned byte. */
+    AssertCompile(VMM_MAX_CPU_COUNT <= 255);
+
+    /*
+     * Query Pool config.
+     */
+    PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/PGM/Pool");
+
+    /* Default pgm pool size is 1024 pages (4MB). */
+    uint16_t cMaxPages = 1024;
+
+    /* Adjust it up relative to the RAM size, using the nested paging formula. */
+    uint64_t cbRam;
+    rc = CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRam, 0); AssertRCReturn(rc, rc);
+    uint64_t u64MaxPages = (cbRam >> 9)
+                         + (cbRam >> 18)
+                         + (cbRam >> 27)
+                         + 32 * PAGE_SIZE;
+    u64MaxPages >>= PAGE_SHIFT;
+    if (u64MaxPages > PGMPOOL_IDX_LAST)
+        cMaxPages = PGMPOOL_IDX_LAST;
+    else
+        cMaxPages = (uint16_t)u64MaxPages;
+
+    /** @cfgm{/PGM/Pool/MaxPages, uint16_t, \#pages, 16, 0x3fff, F(ram-size)}
+     * The max size of the shadow page pool in pages. The pool will grow dynamically
+     * up to this limit.
+     */
+    rc = CFGMR3QueryU16Def(pCfg, "MaxPages", &cMaxPages, cMaxPages);
+    AssertLogRelRCReturn(rc, rc);
+    AssertLogRelMsgReturn(cMaxPages <= PGMPOOL_IDX_LAST && cMaxPages >= RT_ALIGN(PGMPOOL_IDX_FIRST, 16),
+                          ("cMaxPages=%u (%#x)\n", cMaxPages, cMaxPages), VERR_INVALID_PARAMETER);
+    AssertCompile(RT_IS_POWER_OF_TWO(PGMPOOL_CFG_MAX_GROW));
+    if (cMaxPages < PGMPOOL_IDX_LAST)
+        cMaxPages = RT_ALIGN(cMaxPages, PGMPOOL_CFG_MAX_GROW / 2);
+    if (cMaxPages > PGMPOOL_IDX_LAST)
+        cMaxPages = PGMPOOL_IDX_LAST;
+    LogRel(("PGM: PGMPool: cMaxPages=%u (u64MaxPages=%llu)\n", cMaxPages, u64MaxPages));
+
+    /** @todo
+     * We need to be much more careful with our allocation strategy here.
+     * For nested paging we don't need pool user info nor extents at all, but
+     * we can't check for nested paging here (too early during init to get a
+     * confirmation it can be used).  The default for large memory configs is a
+     * bit large for shadow paging, so I've restricted the extent maximum to 8k
+     * (8k * 16 = 128k of hyper heap).
+     *
+     * Also when large page support is enabled, we typically don't need so much,
+     * although that depends on the availability of 2 MB chunks on the host.
+     */
+
+    /** @cfgm{/PGM/Pool/MaxUsers, uint16_t, \#users, MaxUsers, 32K, MaxPages*2}
+     * The max number of shadow page user tracking records. Each shadow page has
+     * zero of other shadow pages (or CR3s) that references it, or uses it if you
+     * like. The structures describing these relationships are allocated from a
+     * fixed sized pool. This configuration variable defines the pool size.
+     */
+    uint16_t cMaxUsers;
+    rc = CFGMR3QueryU16Def(pCfg, "MaxUsers", &cMaxUsers, cMaxPages * 2);
+    AssertLogRelRCReturn(rc, rc);
+    AssertLogRelMsgReturn(cMaxUsers >= cMaxPages && cMaxPages <= _32K,
+                          ("cMaxUsers=%u (%#x)\n", cMaxUsers, cMaxUsers), VERR_INVALID_PARAMETER);
+
+    /** @cfgm{/PGM/Pool/MaxPhysExts, uint16_t, \#extents, 16, MaxPages * 2, MIN(MaxPages*2\,8192)}
+     * The max number of extents for tracking aliased guest pages.
+     */
+    uint16_t cMaxPhysExts;
+    rc = CFGMR3QueryU16Def(pCfg, "MaxPhysExts", &cMaxPhysExts,
+                           RT_MIN(cMaxPages * 2, 8192 /* 8Ki max as this eat too much hyper heap */));
+    AssertLogRelRCReturn(rc, rc);
+    AssertLogRelMsgReturn(cMaxPhysExts >= 16 && cMaxPhysExts <= PGMPOOL_IDX_LAST,
+                          ("cMaxPhysExts=%u (%#x)\n", cMaxPhysExts, cMaxPhysExts), VERR_INVALID_PARAMETER);
+
+    /** @cfgm{/PGM/Pool/ChacheEnabled, bool, true}
+     * Enables or disabling caching of shadow pages. Caching means that we will try
+     * reuse shadow pages instead of recreating them everything SyncCR3, SyncPT or
+     * SyncPage requests one. When reusing a shadow page, we can save time
+     * reconstructing it and it's children.
+     */
+    bool fCacheEnabled;
+    rc = CFGMR3QueryBoolDef(pCfg, "CacheEnabled", &fCacheEnabled, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    LogRel(("PGM: pgmR3PoolInit: cMaxPages=%#RX16 cMaxUsers=%#RX16 cMaxPhysExts=%#RX16 fCacheEnable=%RTbool\n",
+             cMaxPages, cMaxUsers, cMaxPhysExts, fCacheEnabled));
+
+    /*
+     * Allocate the data structures.
+     */
+    uint32_t cb = RT_UOFFSETOF_DYN(PGMPOOL, aPages[cMaxPages]);
+    cb += cMaxUsers * sizeof(PGMPOOLUSER);
+    cb += cMaxPhysExts * sizeof(PGMPOOLPHYSEXT);
+    PPGMPOOL pPool;
+    rc = MMR3HyperAllocOnceNoRel(pVM, cb, 0, MM_TAG_PGM_POOL, (void **)&pPool);
+    if (RT_FAILURE(rc))
+        return rc;
+    pVM->pgm.s.pPoolR3 = pPool;
+    pVM->pgm.s.pPoolR0 = MMHyperR3ToR0(pVM, pPool);
+
+    /*
+     * Initialize it.
+     */
+    pPool->pVMR3     = pVM;
+    pPool->pVMR0     = pVM->pVMR0ForCall;
+    pPool->cMaxPages = cMaxPages;
+    pPool->cCurPages = PGMPOOL_IDX_FIRST;
+    pPool->iUserFreeHead = 0;
+    pPool->cMaxUsers = cMaxUsers;
+    PPGMPOOLUSER paUsers = (PPGMPOOLUSER)&pPool->aPages[pPool->cMaxPages];
+    pPool->paUsersR3 = paUsers;
+    pPool->paUsersR0 = MMHyperR3ToR0(pVM, paUsers);
+    for (unsigned i = 0; i < cMaxUsers; i++)
+    {
+        paUsers[i].iNext = i + 1;
+        paUsers[i].iUser = NIL_PGMPOOL_IDX;
+        paUsers[i].iUserTable = 0xfffffffe;
+    }
+    paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
+    pPool->iPhysExtFreeHead = 0;
+    pPool->cMaxPhysExts = cMaxPhysExts;
+    PPGMPOOLPHYSEXT paPhysExts = (PPGMPOOLPHYSEXT)&paUsers[cMaxUsers];
+    pPool->paPhysExtsR3 = paPhysExts;
+    pPool->paPhysExtsR0 = MMHyperR3ToR0(pVM, paPhysExts);
+    for (unsigned i = 0; i < cMaxPhysExts; i++)
+    {
+        paPhysExts[i].iNext = i + 1;
+        paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+    }
+    paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
+        pPool->aiHash[i] = NIL_PGMPOOL_IDX;
+    pPool->iAgeHead = NIL_PGMPOOL_IDX;
+    pPool->iAgeTail = NIL_PGMPOOL_IDX;
+    pPool->fCacheEnabled = fCacheEnabled;
+
+    pPool->hAccessHandlerType = NIL_PGMPHYSHANDLERTYPE;
+    rc = PGMR3HandlerPhysicalTypeRegister(pVM, PGMPHYSHANDLERKIND_WRITE,
+                                          pgmPoolAccessHandler,
+                                          NULL, "pgmPoolAccessHandler", "pgmRZPoolAccessPfHandler",
+                                          NULL, "pgmPoolAccessHandler", "pgmRZPoolAccessPfHandler",
+                                          "Guest Paging Access Handler",
+                                          &pPool->hAccessHandlerType);
+    AssertLogRelRCReturn(rc, rc);
+
+    pPool->HCPhysTree = 0;
+
+    /*
+     * The NIL entry.
+     */
+    Assert(NIL_PGMPOOL_IDX == 0);
+    pPool->aPages[NIL_PGMPOOL_IDX].enmKind          = PGMPOOLKIND_INVALID;
+    pPool->aPages[NIL_PGMPOOL_IDX].idx              = NIL_PGMPOOL_IDX;
+    pPool->aPages[NIL_PGMPOOL_IDX].Core.Key         = NIL_RTHCPHYS;
+    pPool->aPages[NIL_PGMPOOL_IDX].GCPhys           = NIL_RTGCPHYS;
+    pPool->aPages[NIL_PGMPOOL_IDX].iNext            = NIL_PGMPOOL_IDX;
+    /* pPool->aPages[NIL_PGMPOOL_IDX].cLocked          = INT32_MAX; - test this out... */
+    pPool->aPages[NIL_PGMPOOL_IDX].pvPageR3         = 0;
+    pPool->aPages[NIL_PGMPOOL_IDX].iUserHead        = NIL_PGMPOOL_USER_INDEX;
+    pPool->aPages[NIL_PGMPOOL_IDX].iModifiedNext    = NIL_PGMPOOL_IDX;
+    pPool->aPages[NIL_PGMPOOL_IDX].iModifiedPrev    = NIL_PGMPOOL_IDX;
+    pPool->aPages[NIL_PGMPOOL_IDX].iMonitoredNext   = NIL_PGMPOOL_IDX;
+    pPool->aPages[NIL_PGMPOOL_IDX].iMonitoredPrev   = NIL_PGMPOOL_IDX;
+    pPool->aPages[NIL_PGMPOOL_IDX].iAgeNext         = NIL_PGMPOOL_IDX;
+    pPool->aPages[NIL_PGMPOOL_IDX].iAgePrev         = NIL_PGMPOOL_IDX;
+
+    Assert(pPool->aPages[NIL_PGMPOOL_IDX].idx == NIL_PGMPOOL_IDX);
+    Assert(pPool->aPages[NIL_PGMPOOL_IDX].GCPhys == NIL_RTGCPHYS);
+    Assert(!pPool->aPages[NIL_PGMPOOL_IDX].fSeenNonGlobal);
+    Assert(!pPool->aPages[NIL_PGMPOOL_IDX].fMonitored);
+    Assert(!pPool->aPages[NIL_PGMPOOL_IDX].fCached);
+    Assert(!pPool->aPages[NIL_PGMPOOL_IDX].fZeroed);
+    Assert(!pPool->aPages[NIL_PGMPOOL_IDX].fReusedFlushPending);
+
+    /*
+     * Register statistics.
+     */
+    STAM_REL_REG(pVM, &pPool->StatGrow,                 STAMTYPE_PROFILE,   "/PGM/Pool/Grow",           STAMUNIT_TICKS, "Profiling PGMR0PoolGrow");
+#ifdef VBOX_WITH_STATISTICS
+    STAM_REG(pVM, &pPool->cCurPages,                    STAMTYPE_U16,       "/PGM/Pool/cCurPages",      STAMUNIT_PAGES,             "Current pool size.");
+    STAM_REG(pVM, &pPool->cMaxPages,                    STAMTYPE_U16,       "/PGM/Pool/cMaxPages",      STAMUNIT_PAGES,             "Max pool size.");
+    STAM_REG(pVM, &pPool->cUsedPages,                   STAMTYPE_U16,       "/PGM/Pool/cUsedPages",     STAMUNIT_PAGES,             "The number of pages currently in use.");
+    STAM_REG(pVM, &pPool->cUsedPagesHigh,               STAMTYPE_U16_RESET, "/PGM/Pool/cUsedPagesHigh", STAMUNIT_PAGES,             "The high watermark for cUsedPages.");
+    STAM_REG(pVM, &pPool->StatAlloc,                  STAMTYPE_PROFILE_ADV, "/PGM/Pool/Alloc",          STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmPoolAlloc.");
+    STAM_REG(pVM, &pPool->StatClearAll,                 STAMTYPE_PROFILE,   "/PGM/Pool/ClearAll",       STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmR3PoolClearAll.");
+    STAM_REG(pVM, &pPool->StatR3Reset,                  STAMTYPE_PROFILE,   "/PGM/Pool/R3Reset",        STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmR3PoolReset.");
+    STAM_REG(pVM, &pPool->StatFlushPage,                STAMTYPE_PROFILE,   "/PGM/Pool/FlushPage",      STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmPoolFlushPage.");
+    STAM_REG(pVM, &pPool->StatFree,                     STAMTYPE_PROFILE,   "/PGM/Pool/Free",           STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmPoolFree.");
+    STAM_REG(pVM, &pPool->StatForceFlushPage,           STAMTYPE_COUNTER,   "/PGM/Pool/FlushForce",     STAMUNIT_OCCURENCES,        "Counting explicit flushes by PGMPoolFlushPage().");
+    STAM_REG(pVM, &pPool->StatForceFlushDirtyPage,      STAMTYPE_COUNTER,   "/PGM/Pool/FlushForceDirty",     STAMUNIT_OCCURENCES,   "Counting explicit flushes of dirty pages by PGMPoolFlushPage().");
+    STAM_REG(pVM, &pPool->StatForceFlushReused,         STAMTYPE_COUNTER,   "/PGM/Pool/FlushReused",    STAMUNIT_OCCURENCES,        "Counting flushes for reused pages.");
+    STAM_REG(pVM, &pPool->StatZeroPage,                 STAMTYPE_PROFILE,   "/PGM/Pool/ZeroPage",       STAMUNIT_TICKS_PER_CALL,    "Profiling time spent zeroing pages. Overlaps with Alloc.");
+    STAM_REG(pVM, &pPool->cMaxUsers,                    STAMTYPE_U16,       "/PGM/Pool/Track/cMaxUsers",            STAMUNIT_COUNT,      "Max user tracking records.");
+    STAM_REG(pVM, &pPool->cPresent,                     STAMTYPE_U32,       "/PGM/Pool/Track/cPresent",             STAMUNIT_COUNT,      "Number of present page table entries.");
+    STAM_REG(pVM, &pPool->StatTrackDeref,               STAMTYPE_PROFILE,   "/PGM/Pool/Track/Deref",                STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackDeref.");
+    STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPT,       STAMTYPE_PROFILE,   "/PGM/Pool/Track/FlushGCPhysPT",        STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPT.");
+    STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPTs,      STAMTYPE_PROFILE,   "/PGM/Pool/Track/FlushGCPhysPTs",       STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPTs.");
+    STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPTsSlow,  STAMTYPE_PROFILE,   "/PGM/Pool/Track/FlushGCPhysPTsSlow",   STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPTsSlow.");
+    STAM_REG(pVM, &pPool->StatTrackFlushEntry,          STAMTYPE_COUNTER,   "/PGM/Pool/Track/Entry/Flush",          STAMUNIT_COUNT,          "Nr of flushed entries.");
+    STAM_REG(pVM, &pPool->StatTrackFlushEntryKeep,      STAMTYPE_COUNTER,   "/PGM/Pool/Track/Entry/Update",         STAMUNIT_COUNT,          "Nr of updated entries.");
+    STAM_REG(pVM, &pPool->StatTrackFreeUpOneUser,       STAMTYPE_COUNTER,   "/PGM/Pool/Track/FreeUpOneUser",        STAMUNIT_TICKS_PER_CALL, "The number of times we were out of user tracking records.");
+    STAM_REG(pVM, &pPool->StatTrackDerefGCPhys,         STAMTYPE_PROFILE,   "/PGM/Pool/Track/DrefGCPhys",           STAMUNIT_TICKS_PER_CALL, "Profiling deref activity related tracking GC physical pages.");
+    STAM_REG(pVM, &pPool->StatTrackLinearRamSearches,   STAMTYPE_COUNTER,   "/PGM/Pool/Track/LinearRamSearches",    STAMUNIT_OCCURENCES, "The number of times we had to do linear ram searches.");
+    STAM_REG(pVM, &pPool->StamTrackPhysExtAllocFailures,STAMTYPE_COUNTER,   "/PGM/Pool/Track/PhysExtAllocFailures", STAMUNIT_OCCURENCES, "The number of failing pgmPoolTrackPhysExtAlloc calls.");
+
+    STAM_REG(pVM, &pPool->StatMonitorPfRZ,                STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/#PF",                 STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 #PF access handler.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZEmulateInstr,    STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/EmulateInstr",    STAMUNIT_OCCURENCES,     "Times we've failed interpreting the instruction.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZFlushPage,       STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/#PF/FlushPage",       STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZFlushReinit,     STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/FlushReinit",     STAMUNIT_OCCURENCES,     "Times we've detected a page table reinit.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZFlushModOverflow,STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/FlushOverflow",   STAMUNIT_OCCURENCES,     "Counting flushes for pages that are modified too often.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZFork,            STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/Fork",            STAMUNIT_OCCURENCES,     "Times we've detected fork().");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZHandled,         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/#PF/Handled",         STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 #PF access we've handled (except REP STOSD).");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZIntrFailPatch1,  STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/IntrFailPatch1",  STAMUNIT_OCCURENCES,     "Times we've failed interpreting a patch code instruction.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZIntrFailPatch2,  STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/IntrFailPatch2",  STAMUNIT_OCCURENCES,     "Times we've failed interpreting a patch code instruction during flushing.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZRepPrefix,       STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/RepPrefix",       STAMUNIT_OCCURENCES,     "The number of times we've seen rep prefixes we can't handle.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZRepStosd,        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/#PF/RepStosd",        STAMUNIT_TICKS_PER_CALL, "Profiling the REP STOSD cases we've handled.");
+
+    STAM_REG(pVM, &pPool->StatMonitorRZ,                  STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM",                 STAMUNIT_TICKS_PER_CALL, "Profiling the regular access handler.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFlushPage,         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/FlushPage",       STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the regular access handler.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[0],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size01",          STAMUNIT_OCCURENCES,     "Number of 1 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[1],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size02",          STAMUNIT_OCCURENCES,     "Number of 2 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[2],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size03",          STAMUNIT_OCCURENCES,     "Number of 3 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[3],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size04",          STAMUNIT_OCCURENCES,     "Number of 4 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[4],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size05",          STAMUNIT_OCCURENCES,     "Number of 5 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[5],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size06",          STAMUNIT_OCCURENCES,     "Number of 6 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[6],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size07",          STAMUNIT_OCCURENCES,     "Number of 7 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[7],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size08",          STAMUNIT_OCCURENCES,     "Number of 8 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[8],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size09",          STAMUNIT_OCCURENCES,     "Number of 9 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[9],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size0a",          STAMUNIT_OCCURENCES,     "Number of 10 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[10],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size0b",          STAMUNIT_OCCURENCES,     "Number of 11 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[11],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size0c",          STAMUNIT_OCCURENCES,     "Number of 12 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[12],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size0d",          STAMUNIT_OCCURENCES,     "Number of 13 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[13],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size0e",          STAMUNIT_OCCURENCES,     "Number of 14 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[14],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size0f",          STAMUNIT_OCCURENCES,     "Number of 15 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[15],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size10",          STAMUNIT_OCCURENCES,     "Number of 16 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[16],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size11-2f",       STAMUNIT_OCCURENCES,     "Number of 17-31 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[17],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size20-3f",       STAMUNIT_OCCURENCES,     "Number of 32-63 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[18],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size40+",         STAMUNIT_OCCURENCES,     "Number of 64+ byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[0],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned1",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 1.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[1],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned2",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 2.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[2],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned3",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 3.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[3],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned4",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 4.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[4],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned5",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 5.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[5],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned6",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 6.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[6],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned7",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 7.");
+
+    STAM_REG(pVM, &pPool->StatMonitorRZFaultPT,           STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PT",            STAMUNIT_OCCURENCES,     "Nr of handled PT faults.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFaultPD,           STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PD",            STAMUNIT_OCCURENCES,     "Nr of handled PD faults.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFaultPDPT,         STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PDPT",          STAMUNIT_OCCURENCES,     "Nr of handled PDPT faults.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFaultPML4,         STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PML4",          STAMUNIT_OCCURENCES,     "Nr of handled PML4 faults.");
+
+    STAM_REG(pVM, &pPool->StatMonitorR3,                  STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3",                     STAMUNIT_TICKS_PER_CALL, "Profiling the R3 access handler.");
+    STAM_REG(pVM, &pPool->StatMonitorR3FlushPage,         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/FlushPage",           STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the R3 access handler.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[0],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size01",              STAMUNIT_OCCURENCES,     "Number of 1 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[1],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size02",              STAMUNIT_OCCURENCES,     "Number of 2 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[2],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size03",              STAMUNIT_OCCURENCES,     "Number of 3 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[3],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size04",              STAMUNIT_OCCURENCES,     "Number of 4 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[4],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size05",              STAMUNIT_OCCURENCES,     "Number of 5 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[5],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size06",              STAMUNIT_OCCURENCES,     "Number of 6 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[6],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size07",              STAMUNIT_OCCURENCES,     "Number of 7 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[7],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size08",              STAMUNIT_OCCURENCES,     "Number of 8 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[8],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size09",              STAMUNIT_OCCURENCES,     "Number of 9 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[9],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size0a",              STAMUNIT_OCCURENCES,     "Number of 10 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[10],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size0b",              STAMUNIT_OCCURENCES,     "Number of 11 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[11],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size0c",              STAMUNIT_OCCURENCES,     "Number of 12 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[12],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size0d",              STAMUNIT_OCCURENCES,     "Number of 13 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[13],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size0e",              STAMUNIT_OCCURENCES,     "Number of 14 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[14],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size0f",              STAMUNIT_OCCURENCES,     "Number of 15 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[15],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size10",              STAMUNIT_OCCURENCES,     "Number of 16 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[16],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size11-2f",           STAMUNIT_OCCURENCES,     "Number of 17-31 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[17],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size20-3f",           STAMUNIT_OCCURENCES,     "Number of 32-63 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[18],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size40+",             STAMUNIT_OCCURENCES,     "Number of 64+ byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[0],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned1",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 1 in R3.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[1],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned2",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 2 in R3.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[2],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned3",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 3 in R3.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[3],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned4",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 4 in R3.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[4],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned5",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 5 in R3.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[5],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned6",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 6 in R3.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[6],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned7",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 7 in R3.");
+
+    STAM_REG(pVM, &pPool->StatMonitorR3FaultPT,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PT",        STAMUNIT_OCCURENCES,     "Nr of handled PT faults.");
+    STAM_REG(pVM, &pPool->StatMonitorR3FaultPD,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PD",        STAMUNIT_OCCURENCES,     "Nr of handled PD faults.");
+    STAM_REG(pVM, &pPool->StatMonitorR3FaultPDPT,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PDPT",      STAMUNIT_OCCURENCES,     "Nr of handled PDPT faults.");
+    STAM_REG(pVM, &pPool->StatMonitorR3FaultPML4,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PML4",      STAMUNIT_OCCURENCES,     "Nr of handled PML4 faults.");
+
+    STAM_REG(pVM, &pPool->cModifiedPages,               STAMTYPE_U16,       "/PGM/Pool/Monitor/cModifiedPages",     STAMUNIT_PAGES,          "The current cModifiedPages value.");
+    STAM_REG(pVM, &pPool->cModifiedPagesHigh,           STAMTYPE_U16_RESET, "/PGM/Pool/Monitor/cModifiedPagesHigh", STAMUNIT_PAGES,          "The high watermark for cModifiedPages.");
+    STAM_REG(pVM, &pPool->StatResetDirtyPages,          STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/Dirty/Resets",       STAMUNIT_OCCURENCES,     "Times we've called pgmPoolResetDirtyPages (and there were dirty page).");
+    STAM_REG(pVM, &pPool->StatDirtyPage,                STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/Dirty/Pages",        STAMUNIT_OCCURENCES,     "Times we've called pgmPoolAddDirtyPage.");
+    STAM_REG(pVM, &pPool->StatDirtyPageDupFlush,        STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/Dirty/FlushDup",     STAMUNIT_OCCURENCES,     "Times we've had to flush duplicates for dirty page management.");
+    STAM_REG(pVM, &pPool->StatDirtyPageOverFlowFlush,   STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/Dirty/FlushOverflow",STAMUNIT_OCCURENCES,     "Times we've had to flush because of overflow.");
+    STAM_REG(pVM, &pPool->StatCacheHits,                STAMTYPE_COUNTER,   "/PGM/Pool/Cache/Hits",                 STAMUNIT_OCCURENCES, "The number of pgmPoolAlloc calls satisfied by the cache.");
+    STAM_REG(pVM, &pPool->StatCacheMisses,              STAMTYPE_COUNTER,   "/PGM/Pool/Cache/Misses",               STAMUNIT_OCCURENCES, "The number of pgmPoolAlloc calls not statisfied by the cache.");
+    STAM_REG(pVM, &pPool->StatCacheKindMismatches,      STAMTYPE_COUNTER,   "/PGM/Pool/Cache/KindMismatches",       STAMUNIT_OCCURENCES, "The number of shadow page kind mismatches. (Better be low, preferably 0!)");
+    STAM_REG(pVM, &pPool->StatCacheFreeUpOne,           STAMTYPE_COUNTER,   "/PGM/Pool/Cache/FreeUpOne",            STAMUNIT_OCCURENCES, "The number of times the cache was asked to free up a page.");
+    STAM_REG(pVM, &pPool->StatCacheCacheable,           STAMTYPE_COUNTER,   "/PGM/Pool/Cache/Cacheable",            STAMUNIT_OCCURENCES, "The number of cacheable allocations.");
+    STAM_REG(pVM, &pPool->StatCacheUncacheable,         STAMTYPE_COUNTER,   "/PGM/Pool/Cache/Uncacheable",          STAMUNIT_OCCURENCES, "The number of uncacheable allocations.");
+#endif /* VBOX_WITH_STATISTICS */
+
+#ifdef VBOX_WITH_DEBUGGER
+    /*
+     * Debugger commands.
+     */
+    static bool s_fRegisteredCmds = false;
+    if (!s_fRegisteredCmds)
+    {
+        rc = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
+        if (RT_SUCCESS(rc))
+            s_fRegisteredCmds = true;
+    }
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Relocate the page pool data.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+void pgmR3PoolRelocate(PVM pVM)
+{
+    RT_NOREF(pVM);
+}
+
+
+/**
+ * Grows the shadow page pool.
+ *
+ * I.e. adds more pages to it, assuming that hasn't reached cMaxPages yet.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ */
+VMMR3_INT_DECL(int) PGMR3PoolGrow(PVM pVM, PVMCPU pVCpu)
+{
+    /* This used to do a lot of stuff, but it has moved to ring-0 (PGMR0PoolGrow). */
+    AssertReturn(pVM->pgm.s.pPoolR3->cCurPages < pVM->pgm.s.pPoolR3->cMaxPages, VERR_PGM_POOL_MAXED_OUT_ALREADY);
+    int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_PGM_POOL_GROW, 0, NULL);
+    if (rc == VINF_SUCCESS)
+        return rc;
+    LogRel(("PGMR3PoolGrow: rc=%Rrc cCurPages=%#x cMaxPages=%#x\n",
+            rc, pVM->pgm.s.pPoolR3->cCurPages, pVM->pgm.s.pPoolR3->cMaxPages));
+    if (pVM->pgm.s.pPoolR3->cCurPages > 128 && RT_FAILURE_NP(rc))
+        return -rc;
+    return rc;
+}
+
+
+/**
+ * Rendezvous callback used by pgmR3PoolClearAll that clears all shadow pages
+ * and all modification counters.
+ *
+ * This is only called on one of the EMTs while the other ones are waiting for
+ * it to complete this function.
+ *
+ * @returns VINF_SUCCESS (VBox strict status code).
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT. Unused.
+ * @param   fpvFlushRemTlb  When not NULL, we'll flush the REM TLB as well.
+ *                          (This is the pvUser, so it has to be void *.)
+ *
+ */
+DECLCALLBACK(VBOXSTRICTRC) pgmR3PoolClearAllRendezvous(PVM pVM, PVMCPU pVCpu, void *fpvFlushRemTlb)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    STAM_PROFILE_START(&pPool->StatClearAll, c);
+    NOREF(pVCpu);
+
+    pgmLock(pVM);
+    Log(("pgmR3PoolClearAllRendezvous: cUsedPages=%d fpvFlushRemTlb=%RTbool\n", pPool->cUsedPages, !!fpvFlushRemTlb));
+
+    /*
+     * Iterate all the pages until we've encountered all that are in use.
+     * This is a simple but not quite optimal solution.
+     */
+    unsigned cModifiedPages = 0; NOREF(cModifiedPages);
+    unsigned cLeft = pPool->cUsedPages;
+    uint32_t iPage = pPool->cCurPages;
+    while (--iPage >= PGMPOOL_IDX_FIRST)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
+        if (pPage->GCPhys != NIL_RTGCPHYS)
+        {
+            switch (pPage->enmKind)
+            {
+                /*
+                 * We only care about shadow page tables that reference physical memory
+                 */
+#ifdef PGM_WITH_LARGE_PAGES
+                case PGMPOOLKIND_EPT_PD_FOR_PHYS: /* Large pages reference 2 MB of physical memory, so we must clear them. */
+                    if (pPage->cPresent)
+                    {
+                        PX86PDPAE pShwPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pPool->CTX_SUFF(pVM), pVCpu, pPage);
+                        for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
+                        {
+                            if (    pShwPD->a[i].n.u1Present
+                                &&  pShwPD->a[i].b.u1Size)
+                            {
+                                Assert(!(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING));
+                                pShwPD->a[i].u = 0;
+                                Assert(pPage->cPresent);
+                                pPage->cPresent--;
+                            }
+                        }
+                        if (pPage->cPresent == 0)
+                            pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
+                    }
+                    goto default_case;
+
+                case PGMPOOLKIND_PAE_PD_PHYS:   /* Large pages reference 2 MB of physical memory, so we must clear them. */
+                    if (pPage->cPresent)
+                    {
+                        PEPTPD pShwPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pPool->CTX_SUFF(pVM), pVCpu, pPage);
+                        for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
+                        {
+                            Assert((pShwPD->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
+                            if (    pShwPD->a[i].n.u1Present
+                                &&  pShwPD->a[i].b.u1Size)
+                            {
+                                Assert(!(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING));
+                                pShwPD->a[i].u = 0;
+                                Assert(pPage->cPresent);
+                                pPage->cPresent--;
+                            }
+                        }
+                        if (pPage->cPresent == 0)
+                            pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
+                    }
+                    goto default_case;
+#endif /* PGM_WITH_LARGE_PAGES */
+
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                {
+                    if (pPage->cPresent)
+                    {
+                        void *pvShw = PGMPOOL_PAGE_2_PTR_V2(pPool->CTX_SUFF(pVM), pVCpu, pPage);
+                        STAM_PROFILE_START(&pPool->StatZeroPage, z);
+#if 0
+                        /* Useful check for leaking references; *very* expensive though. */
+                        switch (pPage->enmKind)
+                        {
+                            case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                            case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                            case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                            case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                            case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                            {
+                                bool fFoundFirst = false;
+                                PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)pvShw;
+                                for (unsigned ptIndex = 0; ptIndex < RT_ELEMENTS(pPT->a); ptIndex++)
+                                {
+                                    if (pPT->a[ptIndex].u)
+                                    {
+                                        if (!fFoundFirst)
+                                        {
+                                            AssertFatalMsg(pPage->iFirstPresent <= ptIndex, ("ptIndex = %d first present = %d\n", ptIndex, pPage->iFirstPresent));
+                                            if (pPage->iFirstPresent != ptIndex)
+                                                Log(("ptIndex = %d first present = %d\n", ptIndex, pPage->iFirstPresent));
+                                            fFoundFirst = true;
+                                        }
+                                        if (PGMSHWPTEPAE_IS_P(pPT->a[ptIndex]))
+                                        {
+                                            pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pPT->a[ptIndex]), NIL_RTGCPHYS);
+                                            if (pPage->iFirstPresent == ptIndex)
+                                                pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
+                                        }
+                                    }
+                                }
+                                AssertFatalMsg(pPage->cPresent == 0, ("cPresent = %d pPage = %RGv\n", pPage->cPresent, pPage->GCPhys));
+                                break;
+                            }
+                            default:
+                                break;
+                        }
+#endif
+                        ASMMemZeroPage(pvShw);
+                        STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
+                        pPage->cPresent = 0;
+                        pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
+                    }
+                }
+                RT_FALL_THRU();
+#ifdef PGM_WITH_LARGE_PAGES
+                default_case:
+#endif
+                default:
+                    Assert(!pPage->cModifications || ++cModifiedPages);
+                    Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
+                    Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
+                    pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+                    pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
+                    pPage->cModifications = 0;
+                    break;
+
+            }
+            if (!--cLeft)
+                break;
+        }
+    }
+
+#ifndef DEBUG_michael
+    AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
+#endif
+    pPool->iModifiedHead = NIL_PGMPOOL_IDX;
+    pPool->cModifiedPages = 0;
+
+    /*
+     * Clear all the GCPhys links and rebuild the phys ext free list.
+     */
+    for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRangesX);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        iPage = pRam->cb >> PAGE_SHIFT;
+        while (iPage-- > 0)
+            PGM_PAGE_SET_TRACKING(pVM, &pRam->aPages[iPage], 0);
+    }
+
+    pPool->iPhysExtFreeHead = 0;
+    PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
+    const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
+    for (unsigned i = 0; i < cMaxPhysExts; i++)
+    {
+        paPhysExts[i].iNext = i + 1;
+        paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+    }
+    paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
+
+
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    /* Reset all dirty pages to reactivate the page monitoring. */
+    /* Note: we must do this *after* clearing all page references and shadow page tables as there might be stale references to
+     *       recently removed MMIO ranges around that might otherwise end up asserting in pgmPoolTracDerefGCPhysHint
+     */
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+    {
+        unsigned idxPage = pPool->aidxDirtyPages[i];
+        if (idxPage == NIL_PGMPOOL_IDX)
+            continue;
+
+        PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
+        Assert(pPage->idx == idxPage);
+        Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
+
+        AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, i));
+
+        Log(("Reactivate dirty page %RGp\n", pPage->GCPhys));
+
+        /* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */
+        int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK);
+        AssertRCSuccess(rc);
+        pPage->fDirty = false;
+
+        pPool->aidxDirtyPages[i] = NIL_PGMPOOL_IDX;
+    }
+
+    /* Clear all dirty pages. */
+    pPool->idxFreeDirtyPage = 0;
+    pPool->cDirtyPages      = 0;
+#endif
+
+    /* Clear the PGM_SYNC_CLEAR_PGM_POOL flag on all VCPUs to prevent redundant flushes. */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        pVM->apCpusR3[idCpu]->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
+
+    /* Flush job finished. */
+    VM_FF_CLEAR(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
+    pPool->cPresent = 0;
+    pgmUnlock(pVM);
+
+    PGM_INVL_ALL_VCPU_TLBS(pVM);
+
+    if (fpvFlushRemTlb)
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+            CPUMSetChangedFlags(pVM->apCpusR3[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
+
+    STAM_PROFILE_STOP(&pPool->StatClearAll, c);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Clears the shadow page pool.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fFlushRemTlb    When set, the REM TLB is scheduled for flushing as
+ *                          well.
+ */
+void pgmR3PoolClearAll(PVM pVM, bool fFlushRemTlb)
+{
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PoolClearAllRendezvous, &fFlushRemTlb);
+    AssertRC(rc);
+}
+
+
+/**
+ * Protect all pgm pool page table entries to monitor writes
+ *
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks ASSUMES the caller will flush all TLBs!!
+ */
+void pgmR3PoolWriteProtectPages(PVM pVM)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    unsigned cLeft = pPool->cUsedPages;
+    unsigned iPage = pPool->cCurPages;
+    while (--iPage >= PGMPOOL_IDX_FIRST)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
+        if (    pPage->GCPhys != NIL_RTGCPHYS
+            &&  pPage->cPresent)
+        {
+            union
+            {
+                void           *pv;
+                PX86PT          pPT;
+                PPGMSHWPTPAE    pPTPae;
+                PEPTPT          pPTEpt;
+            } uShw;
+            uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+
+            switch (pPage->enmKind)
+            {
+                /*
+                 * We only care about shadow page tables.
+                 */
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                    for (unsigned iShw = 0; iShw < RT_ELEMENTS(uShw.pPT->a); iShw++)
+                    {
+                        if (uShw.pPT->a[iShw].n.u1Present)
+                            uShw.pPT->a[iShw].n.u1Write = 0;
+                    }
+                    break;
+
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                    for (unsigned iShw = 0; iShw < RT_ELEMENTS(uShw.pPTPae->a); iShw++)
+                    {
+                        if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
+                            PGMSHWPTEPAE_SET_RO(uShw.pPTPae->a[iShw]);
+                    }
+                    break;
+
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                    for (unsigned iShw = 0; iShw < RT_ELEMENTS(uShw.pPTEpt->a); iShw++)
+                    {
+                        if (uShw.pPTEpt->a[iShw].n.u1Present)
+                            uShw.pPTEpt->a[iShw].n.u1Write = 0;
+                    }
+                    break;
+
+                default:
+                    break;
+            }
+            if (!--cLeft)
+                break;
+        }
+    }
+}
+
+#ifdef VBOX_WITH_DEBUGGER
+/**
+ * @callback_method_impl{FNDBGCCMD, The '.pgmpoolcheck' command.}
+ */
+static DECLCALLBACK(int) pgmR3PoolCmdCheck(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
+{
+    DBGC_CMDHLP_REQ_UVM_RET(pCmdHlp, pCmd, pUVM);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    DBGC_CMDHLP_ASSERT_PARSER_RET(pCmdHlp, pCmd, -1, cArgs == 0);
+    uint32_t cErrors = 0;
+    NOREF(paArgs);
+
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    for (unsigned i = 0; i < pPool->cCurPages; i++)
+    {
+        PPGMPOOLPAGE    pPage     = &pPool->aPages[i];
+        bool            fFirstMsg = true;
+
+        /** @todo cover other paging modes too. */
+        if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
+        {
+            PPGMSHWPTPAE pShwPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
+            {
+                PX86PTPAE       pGstPT;
+                PGMPAGEMAPLOCK  LockPage;
+                int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, pPage->GCPhys, (const void **)&pGstPT, &LockPage);     AssertReleaseRC(rc);
+
+                /* Check if any PTEs are out of sync. */
+                for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
+                {
+                    if (PGMSHWPTEPAE_IS_P(pShwPT->a[j]))
+                    {
+                        RTHCPHYS HCPhys = NIL_RTHCPHYS;
+                        rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[j].u & X86_PTE_PAE_PG_MASK, &HCPhys);
+                        if (   rc != VINF_SUCCESS
+                            || PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[j]) != HCPhys)
+                        {
+                            if (fFirstMsg)
+                            {
+                                DBGCCmdHlpPrintf(pCmdHlp, "Check pool page %RGp\n", pPage->GCPhys);
+                                fFirstMsg = false;
+                            }
+                            DBGCCmdHlpPrintf(pCmdHlp, "Mismatch HCPhys: rc=%Rrc idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, j, pGstPT->a[j].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), HCPhys);
+                            cErrors++;
+                        }
+                        else if (   PGMSHWPTEPAE_IS_RW(pShwPT->a[j])
+                                 && !pGstPT->a[j].n.u1Write)
+                        {
+                            if (fFirstMsg)
+                            {
+                                DBGCCmdHlpPrintf(pCmdHlp, "Check pool page %RGp\n", pPage->GCPhys);
+                                fFirstMsg = false;
+                            }
+                            DBGCCmdHlpPrintf(pCmdHlp, "Mismatch r/w gst/shw: idx=%d guest %RX64 shw=%RX64 vs %RHp\n", j, pGstPT->a[j].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), HCPhys);
+                            cErrors++;
+                        }
+                    }
+                }
+                PGMPhysReleasePageMappingLock(pVM, &LockPage);
+            }
+
+            /* Make sure this page table can't be written to from any shadow mapping. */
+            RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
+            int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pPage->GCPhys, &HCPhysPT);
+            AssertMsgRC(rc, ("PGMPhysGCPhys2HCPhys failed with rc=%d for %RGp\n", rc, pPage->GCPhys));
+            if (rc == VINF_SUCCESS)
+            {
+                for (unsigned j = 0; j < pPool->cCurPages; j++)
+                {
+                    PPGMPOOLPAGE pTempPage = &pPool->aPages[j];
+
+                    if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
+                    {
+                        PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pTempPage);
+
+                        for (unsigned k = 0; k < RT_ELEMENTS(pShwPT->a); k++)
+                        {
+                            if (    PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[k])
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+                                &&  !pPage->fDirty
+# endif
+                                &&  PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[k]) == HCPhysPT)
+                            {
+                                if (fFirstMsg)
+                                {
+                                    DBGCCmdHlpPrintf(pCmdHlp, "Check pool page %RGp\n", pPage->GCPhys);
+                                    fFirstMsg = false;
+                                }
+                                DBGCCmdHlpPrintf(pCmdHlp, "Mismatch: r/w: GCPhys=%RGp idx=%d shw %RX64 %RX64\n", pTempPage->GCPhys, k, PGMSHWPTEPAE_GET_LOG(pShwPT->a[k]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[k]));
+                                cErrors++;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    if (cErrors > 0)
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "Found %#x errors", cErrors);
+    return VINF_SUCCESS;
+}
+#endif /* VBOX_WITH_DEBUGGER */
diff --git a/src/VBox/VMM/VMMR3/PGMR3DbgA.asm b/src/VBox/VMM/VMMR3/PGMR3DbgA.asm
new file mode 100644
index 00000000..e3db726e
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PGMR3DbgA.asm
@@ -0,0 +1,475 @@
+; $Id: PGMR3DbgA.asm $
+;; @file
+; PGM - Page Manager and Monitor - Debugger & Debugging API Optimizations.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+
+;*******************************************************************************
+;* Header Files                                                                *
+;*******************************************************************************
+%define RT_ASM_WITH_SEH64
+%include "VBox/asmdefs.mac"
+
+BEGINCODE ;; Doesn't end up in code seg on 64-bit darwin. weird.
+
+
+;
+; Common to all code below.
+;
+%ifdef ASM_CALL64_MSC
+ %define pvNeedle       r8
+ %define cbNeedle       r9d
+ %define bTmp           dl
+%elifdef ASM_CALL64_GCC
+ %define pvNeedle       rdx
+ %define cbNeedle       esi
+ %define bTmp           r9b
+%elifdef RT_ARCH_X86
+ %define pvNeedle       dword [esp + 8h]
+ %define cbNeedle       dword [esp + 10h]
+%else
+ %error "Unsupported arch!"
+%endif
+
+;;
+; Searches for a 8 byte needle in steps of 8.
+;
+; In 32-bit mode, this will only actually search for a 8 byte needle.
+;
+; @param    pbHaystack  [msc:rcx,     gcc:rdi, x86:ebp+08h]  What to search thru.
+; @param    cbHaystack  [msc:edx,     gcc:rsi, x86:ebp+0ch]  The amount of hay to search.
+; @param    pvNeedle    [msc:r8,      gcc:rdx, x86:ebp+10h]  What we're searching for
+; @param    cbNeedle    [msc:r9,      gcc:rcx, x86:esp+10h]  Size of what we're searcing for. Currently ignored.
+;
+; @remarks  ASSUMES pbHaystack is aligned at uAlign.
+;
+BEGINPROC pgmR3DbgFixedMemScan8Wide8Step
+%ifdef ASM_CALL64_MSC
+        mov     r10, rdi                ; save it
+        mov     rdi, rcx                ; rdi=pbHaystack
+        mov     ecx, edx                ; rcx=cbHaystack
+        mov     rax, [r8]               ; *(uint64_t *)pvNeedle
+%elifdef ASM_CALL64_GCC
+        xchg    rcx, rsi                ; rcx=cbHaystack, rsi=cbNeedle
+        mov     rax, [rdx]              ; *(uint64_t *)pvNeedle
+%elifdef RT_ARCH_X86
+        push    ebp
+        mov     ebp, esp
+        push    edi                     ; save it
+        mov     edi, [ebp + 08h]        ; pbHaystack
+        mov     ecx, [ebp + 0ch]        ; cbHaystack
+        mov     eax, [ebp + 10h]        ; pvNeedle
+        mov     edx, [eax + 4]          ; ((uint32_t *)pvNeedle)[1]
+        mov     eax, [eax]              ; ((uint32_t *)pvNeedle)[0]
+%else
+ %error "Unsupported arch!"
+%endif
+SEH64_END_PROLOGUE
+
+%ifdef RT_ARCH_X86
+        ;
+        ; No string instruction to help us here.  Do a simple tight loop instead.
+        ;
+        shr     ecx, 3
+        jz      .return_null
+.again:
+        cmp     [edi], eax
+        je      .needle_check
+.continue:
+        add     edi, 8
+        dec     ecx
+        jnz     .again
+        jmp     .return_null
+
+        ; Check the needle 2nd dword, caller can do the rest.
+.needle_check:
+        cmp     edx, [edi + 4]
+        jne     .continue
+
+.return_edi:
+        mov     eax, edi
+
+%else  ; RT_ARCH_AMD64
+        cmp     ecx, 8
+        jb      .return_null
+.continue:
+        shr     ecx, 3
+        repne   scasq
+        jne     .return_null
+        ; check more of the needle if we can.
+        mov     r11d, 8
+        shl     ecx, 3
+.needle_check:
+        cmp     cbNeedle, r11d
+        je      .return_edi
+        cmp     ecx, r11d
+        jb      .return_edi             ; returns success here as we've might've lost stuff while shifting ecx around.
+        mov     bTmp, [pvNeedle + r11]
+        cmp     bTmp, [xDI + r11 - 8]
+        jne     .continue
+        inc     r11d
+        jmp     .needle_check
+
+.return_edi:
+        lea     xAX, [xDI - 8]
+%endif ; RT_ARCH_AMD64
+
+.return:
+%ifdef ASM_CALL64_MSC
+        mov     rdi, r10
+%elifdef RT_ARCH_X86
+        pop     edi
+        leave
+%endif
+        ret
+
+.return_null:
+        xor     eax, eax
+        jmp     .return
+ENDPROC   pgmR3DbgFixedMemScan8Wide8Step
+
+
+;;
+; Searches for a 4 byte needle in steps of 4.
+;
+; @param    pbHaystack  [msc:rcx,     gcc:rdi, x86:esp+04h]  What to search thru.
+; @param    cbHaystack  [msc:edx,     gcc:rsi, x86:esp+08h]  The amount of hay to search.
+; @param    pvNeedle    [msc:r8,      gcc:rdx, x86:esp+0ch]  What we're searching for
+; @param    cbNeedle    [msc:r9,      gcc:rcx, x86:esp+10h]  Size of what we're searcing for. Currently ignored.
+;
+; @remarks  ASSUMES pbHaystack is aligned at uAlign.
+;
+BEGINPROC pgmR3DbgFixedMemScan4Wide4Step
+%ifdef ASM_CALL64_MSC
+        mov     r10, rdi                ; save it
+        mov     rdi, rcx                ; rdi=pbHaystack
+        mov     ecx, edx                ; rcx=cbHaystack
+        mov     eax, [r8]               ; *(uint32_t *)pvNeedle
+%elifdef ASM_CALL64_GCC
+        xchg    rcx, rsi                ; rcx=cbHaystack, rsi=cbNeedle
+        mov     eax, [rdx]              ; *(uint32_t *)pvNeedle
+%elifdef RT_ARCH_X86
+        mov     edx, edi                ; save it
+        mov     edi, [esp + 04h]        ; pbHaystack
+        mov     ecx, [esp + 08h]        ; cbHaystack
+        mov     eax, [esp + 0ch]        ; pvNeedle
+        mov     eax, [eax]              ; *(uint32_t *)pvNeedle
+%else
+ %error "Unsupported arch!"
+%endif
+SEH64_END_PROLOGUE
+
+.continue:
+        cmp     ecx, 4
+        jb      .return_null
+        shr     ecx, 2
+        repne   scasd
+        jne     .return_null
+
+%ifdef RT_ARCH_AMD64
+        ; check more of the needle if we can.
+        mov     r11d, 4
+.needle_check:
+        cmp     cbNeedle, r11d
+        je      .return_edi
+        cmp     ecx, r11d               ; don't bother converting ecx to bytes.
+        jb      .return_edi
+        mov     bTmp, [pvNeedle + r11]
+        cmp     bTmp, [xDI + r11 - 4]
+        jne     .continue
+        inc     r11d
+        jmp     .needle_check
+%endif
+
+.return_edi:
+        lea     xAX, [xDI - 4]
+.return:
+%ifdef ASM_CALL64_MSC
+        mov     rdi, r10
+%elifdef RT_ARCH_X86
+        mov     edi, edx
+%endif
+        ret
+
+.return_null:
+        xor     eax, eax
+        jmp     .return
+ENDPROC   pgmR3DbgFixedMemScan4Wide4Step
+
+
+;;
+; Searches for a 2 byte needle in steps of 2.
+;
+; @param    pbHaystack  [msc:rcx,     gcc:rdi, x86:esp+04h]  What to search thru.
+; @param    cbHaystack  [msc:edx,     gcc:rsi, x86:esp+08h]  The amount of hay to search.
+; @param    pvNeedle    [msc:r8,      gcc:rdx, x86:esp+0ch]  What we're searching for
+; @param    cbNeedle    [msc:r9,      gcc:rcx, x86:esp+10h]  Size of what we're searcing for. Currently ignored.
+;
+; @remarks  ASSUMES pbHaystack is aligned at uAlign.
+;
+BEGINPROC pgmR3DbgFixedMemScan2Wide2Step
+%ifdef ASM_CALL64_MSC
+        mov     r10, rdi                ; save it
+        mov     rdi, rcx                ; rdi=pbHaystack
+        mov     ecx, edx                ; rcx=cbHaystack
+        mov     ax, [r8]                ; *(uint16_t *)pvNeedle
+%elifdef ASM_CALL64_GCC
+        xchg    rcx, rsi                ; rcx=cbHaystack, rsi=cbNeedle
+        mov     ax, [rdx]               ; *(uint16_t *)pvNeedle
+%elifdef RT_ARCH_X86
+        mov     edx, edi                ; save it
+        mov     edi, [esp + 04h]        ; pbHaystack
+        mov     ecx, [esp + 08h]        ; cbHaystack
+        mov     eax, [esp + 0ch]        ; pvNeedle
+        mov     ax, [eax]               ; *(uint16_t *)pvNeedle
+%else
+ %error "Unsupported arch!"
+%endif
+SEH64_END_PROLOGUE
+
+.continue:
+        cmp     ecx, 2
+        jb      .return_null
+        shr     ecx, 1
+        repne   scasw
+        jne     .return_null
+
+%ifdef RT_ARCH_AMD64
+        ; check more of the needle if we can.
+        mov     r11d, 2
+.needle_check:
+        cmp     cbNeedle, r11d
+        je      .return_edi
+        cmp     ecx, r11d               ; don't bother converting ecx to bytes.
+        jb      .return_edi
+        mov     bTmp, [pvNeedle + r11]
+        cmp     bTmp, [xDI + r11 - 2]
+        jne     .continue
+        inc     r11d
+        jmp     .needle_check
+%endif
+
+.return_edi:
+        lea     xAX, [xDI - 2]
+.return:
+%ifdef ASM_CALL64_MSC
+        mov     rdi, r10
+%elifdef RT_ARCH_X86
+        mov     edi, edx
+%endif
+        ret
+
+.return_null:
+        xor     eax, eax
+        jmp     .return
+ENDPROC   pgmR3DbgFixedMemScan2Wide2Step
+
+
+;;
+; Searches for a 1 byte needle in steps of 1.
+;
+; @param    pbHaystack  [msc:rcx,     gcc:rdi, x86:esp+04h]  What to search thru.
+; @param    cbHaystack  [msc:edx,     gcc:rsi, x86:esp+08h]  The amount of hay to search.
+; @param    pvNeedle    [msc:r8,      gcc:rdx, x86:esp+0ch]  What we're searching for
+; @param    cbNeedle    [msc:r9,      gcc:rcx, x86:esp+10h]  Size of what we're searcing for. Currently ignored.
+;
+BEGINPROC pgmR3DbgFixedMemScan1Wide1Step
+%ifdef ASM_CALL64_MSC
+        mov     r10, rdi                ; save it
+        mov     rdi, rcx                ; rdi=pbHaystack
+        mov     ecx, edx                ; rcx=cbHaystack
+        mov     al, [r8]                ; *(uint8_t *)pvNeedle
+%elifdef ASM_CALL64_GCC
+        xchg    rcx, rsi                ; rcx=cbHaystack, rsi=cbNeedle
+        mov     al, [rdx]               ; *(uint8_t *)pvNeedle
+%elifdef RT_ARCH_X86
+        mov     edx, edi                ; save it
+        mov     edi, [esp + 04h]        ; pbHaystack
+        mov     ecx, [esp + 08h]        ; cbHaystack
+        mov     eax, [esp + 0ch]        ; pvNeedle
+        mov     al, [eax]               ; *(uint8_t *)pvNeedle
+%else
+ %error "Unsupported arch!"
+%endif
+SEH64_END_PROLOGUE
+
+        cmp     ecx, 1
+        jb      .return_null
+.continue:
+        repne   scasb
+        jne     .return_null
+
+%ifdef RT_ARCH_AMD64
+        ; check more of the needle if we can.
+        mov     r11d, 1
+.needle_check:
+        cmp     cbNeedle, r11d
+        je      .return_edi
+        cmp     ecx, r11d
+        jb      .return_edi
+        mov     bTmp, [pvNeedle + r11]
+        cmp     bTmp, [xDI + r11 - 1]
+        jne     .continue
+        inc     r11d
+        jmp     .needle_check
+%endif
+
+.return_edi:
+        lea     xAX, [xDI - 1]
+.return:
+%ifdef ASM_CALL64_MSC
+        mov     rdi, r10
+%elifdef RT_ARCH_X86
+        mov     edi, edx
+%endif
+        ret
+
+.return_null:
+        xor     eax, eax
+%ifdef ASM_CALL64_MSC
+        mov     rdi, r10
+%elifdef RT_ARCH_X86
+        mov     edi, edx
+%endif
+        ret
+ENDPROC   pgmR3DbgFixedMemScan1Wide1Step
+
+
+;;
+; Searches for a 4 byte needle in steps of 1.
+;
+; @param    pbHaystack  [msc:rcx,     gcc:rdi, x86:esp+04h]  What to search thru.
+; @param    cbHaystack  [msc:edx,     gcc:rsi, x86:esp+08h]  The amount of hay to search.
+; @param    pvNeedle    [msc:r8,      gcc:rdx, x86:esp+0ch]  What we're searching for
+; @param    cbNeedle    [msc:r9,      gcc:rcx, x86:esp+10h]  Size of what we're searcing for. Currently ignored.
+;
+BEGINPROC pgmR3DbgFixedMemScan4Wide1Step
+%ifdef ASM_CALL64_MSC
+        mov     r10, rdi                ; save it
+        mov     rdi, rcx                ; rdi=pbHaystack
+        mov     ecx, edx                ; rcx=cbHaystack
+        mov     eax, [r8]               ; *(uint32_t *)pvNeedle
+%elifdef ASM_CALL64_GCC
+        xchg    rcx, rsi                ; rcx=cbHaystack, rsi=cbNeedle
+        mov     eax, [rdx]              ; *(uint32_t *)pvNeedle
+%elifdef RT_ARCH_X86
+        mov     edx, edi                ; save it
+        mov     edi, [esp + 04h]        ; pbHaystack
+        mov     ecx, [esp + 08h]        ; cbHaystack
+        mov     eax, [esp + 0ch]        ; pvNeedle
+        mov     eax, [eax]              ; *(uint32_t *)pvNeedle
+%else
+ %error "Unsupported arch!"
+%endif
+SEH64_END_PROLOGUE
+
+        cmp     ecx, 1
+        jb      .return_null
+.continue:
+        repne   scasb
+        jne     .return_null
+        cmp     ecx, 3
+        jb      .return_null
+        cmp     eax, [xDI - 1]
+        jne     .continue
+
+.return_edi:
+        lea     xAX, [xDI - 1]
+.return:
+%ifdef ASM_CALL64_MSC
+        mov     rdi, r10
+%elifdef RT_ARCH_X86
+        mov     edi, edx
+%endif
+        ret
+
+.return_null:
+        xor     eax, eax
+%ifdef ASM_CALL64_MSC
+        mov     rdi, r10
+%elifdef RT_ARCH_X86
+        mov     edi, edx
+%endif
+        ret
+ENDPROC   pgmR3DbgFixedMemScan4Wide1Step
+
+;;
+; Searches for a 8 byte needle in steps of 1.
+;
+; @param    pbHaystack  [msc:rcx,     gcc:rdi, x86:esp+04h]  What to search thru.
+; @param    cbHaystack  [msc:edx,     gcc:rsi, x86:esp+08h]  The amount of hay to search.
+; @param    pvNeedle    [msc:r8,      gcc:rdx, x86:esp+0ch]  What we're searching for
+; @param    cbNeedle    [msc:r9,      gcc:rcx, x86:esp+10h]  Size of what we're searcing for. Currently ignored.
+;
+; @remarks  The 32-bit version is currently identical to pgmR3DbgFixedMemScan4Wide1Step.
+;
+BEGINPROC pgmR3DbgFixedMemScan8Wide1Step
+%ifdef ASM_CALL64_MSC
+        mov     r10, rdi                ; save it
+        mov     rdi, rcx                ; rdi=pbHaystack
+        mov     ecx, edx                ; rcx=cbHaystack
+        mov     rax, [r8]               ; *(uint64_t *)pvNeedle
+%elifdef ASM_CALL64_GCC
+        xchg    rcx, rsi                ; rcx=cbHaystack, rsi=cbNeedle
+        mov     rax, [rdx]              ; *(uint64_t *)pvNeedle
+%elifdef RT_ARCH_X86
+        mov     edx, edi                ; save it
+        mov     edi, [esp + 04h]        ; pbHaystack
+        mov     ecx, [esp + 08h]        ; cbHaystack
+        mov     eax, [esp + 0ch]        ; pvNeedle
+        mov     eax, [eax]              ; *(uint32_t *)pvNeedle
+%else
+ %error "Unsupported arch!"
+%endif
+SEH64_END_PROLOGUE
+
+        cmp     ecx, 1
+        jb      .return_null
+.continue:
+        repne   scasb
+        jne     .return_null
+%ifdef RT_ARCH_AMD64
+        cmp     ecx, 7
+        jb      .check_smaller
+        cmp     rax, [xDI - 1]
+        jne     .continue
+        jmp     .return_edi
+.check_smaller:
+%endif
+        cmp     ecx, 3
+        jb      .return_null
+        cmp     eax, [xDI - 1]
+        jne     .continue
+
+.return_edi:
+        lea     xAX, [xDI - 1]
+.return:
+%ifdef ASM_CALL64_MSC
+        mov     rdi, r10
+%elifdef RT_ARCH_X86
+        mov     edi, edx
+%endif
+        ret
+
+.return_null:
+        xor     eax, eax
+%ifdef ASM_CALL64_MSC
+        mov     rdi, r10
+%elifdef RT_ARCH_X86
+        mov     edi, edx
+%endif
+        ret
+ENDPROC   pgmR3DbgFixedMemScan8Wide1Step
+
diff --git a/src/VBox/VMM/VMMR3/PGMSavedState.cpp b/src/VBox/VMM/VMMR3/PGMSavedState.cpp
new file mode 100644
index 00000000..0b44024b
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PGMSavedState.cpp
@@ -0,0 +1,3303 @@
+/* $Id: PGMSavedState.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor, The Saved State Part.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/pdmdrv.h>
+#include <VBox/vmm/pdmdev.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vm.h>
+#include "PGMInline.h"
+
+#include <VBox/param.h>
+#include <VBox/err.h>
+
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/crc.h>
+#include <iprt/mem.h>
+#include <iprt/sha.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** Saved state data unit version.  */
+#define PGM_SAVED_STATE_VERSION                 14
+/** Saved state data unit version before the PAE PDPE registers. */
+#define PGM_SAVED_STATE_VERSION_PRE_PAE         13
+/** Saved state data unit version after this includes ballooned page flags in
+ *  the state (see @bugref{5515}). */
+#define PGM_SAVED_STATE_VERSION_BALLOON_BROKEN  12
+/** Saved state before the balloon change. */
+#define PGM_SAVED_STATE_VERSION_PRE_BALLOON     11
+/** Saved state data unit version used during 3.1 development, misses the RAM
+ *  config. */
+#define PGM_SAVED_STATE_VERSION_NO_RAM_CFG      10
+/** Saved state data unit version for 3.0 (pre teleportation). */
+#define PGM_SAVED_STATE_VERSION_3_0_0           9
+/** Saved state data unit version for 2.2.2 and later. */
+#define PGM_SAVED_STATE_VERSION_2_2_2           8
+/** Saved state data unit version for 2.2.0. */
+#define PGM_SAVED_STATE_VERSION_RR_DESC         7
+/** Saved state data unit version. */
+#define PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE   6
+
+
+/** @name Sparse state record types
+ * @{  */
+/** Zero page. No data. */
+#define PGM_STATE_REC_RAM_ZERO          UINT8_C(0x00)
+/** Raw page. */
+#define PGM_STATE_REC_RAM_RAW           UINT8_C(0x01)
+/** Raw MMIO2 page. */
+#define PGM_STATE_REC_MMIO2_RAW         UINT8_C(0x02)
+/** Zero MMIO2 page. */
+#define PGM_STATE_REC_MMIO2_ZERO        UINT8_C(0x03)
+/** Virgin ROM page. Followed by protection (8-bit) and the raw bits. */
+#define PGM_STATE_REC_ROM_VIRGIN        UINT8_C(0x04)
+/** Raw shadowed ROM page. The protection (8-bit) precedes the raw bits. */
+#define PGM_STATE_REC_ROM_SHW_RAW       UINT8_C(0x05)
+/** Zero shadowed ROM page. The protection (8-bit) is the only payload. */
+#define PGM_STATE_REC_ROM_SHW_ZERO      UINT8_C(0x06)
+/** ROM protection (8-bit). */
+#define PGM_STATE_REC_ROM_PROT          UINT8_C(0x07)
+/** Ballooned page. No data. */
+#define PGM_STATE_REC_RAM_BALLOONED     UINT8_C(0x08)
+/** The last record type. */
+#define PGM_STATE_REC_LAST              PGM_STATE_REC_RAM_BALLOONED
+/** End marker. */
+#define PGM_STATE_REC_END               UINT8_C(0xff)
+/** Flag indicating that the data is preceded by the page address.
+ *  For RAW pages this is a RTGCPHYS.  For MMIO2 and ROM pages this is a 8-bit
+ *  range ID and a 32-bit page index.
+ */
+#define PGM_STATE_REC_FLAG_ADDR         UINT8_C(0x80)
+/** @} */
+
+/** The CRC-32 for a zero page. */
+#define PGM_STATE_CRC32_ZERO_PAGE       UINT32_C(0xc71c0011)
+/** The CRC-32 for a zero half page. */
+#define PGM_STATE_CRC32_ZERO_HALF_PAGE  UINT32_C(0xf1e8ba9e)
+
+
+
+/** @name Old Page types used in older saved states.
+ * @{  */
+/** Old saved state: The usual invalid zero entry. */
+#define PGMPAGETYPE_OLD_INVALID             0
+/** Old saved state: RAM page. (RWX) */
+#define PGMPAGETYPE_OLD_RAM                 1
+/** Old saved state: MMIO2 page. (RWX) */
+#define PGMPAGETYPE_OLD_MMIO2               1
+/** Old saved state: MMIO2 page aliased over an MMIO page. (RWX)
+ * See PGMHandlerPhysicalPageAlias(). */
+#define PGMPAGETYPE_OLD_MMIO2_ALIAS_MMIO    2
+/** Old saved state: Shadowed ROM. (RWX) */
+#define PGMPAGETYPE_OLD_ROM_SHADOW          3
+/** Old saved state: ROM page. (R-X) */
+#define PGMPAGETYPE_OLD_ROM                 4
+/** Old saved state: MMIO page. (---) */
+#define PGMPAGETYPE_OLD_MMIO                5
+/** @}  */
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/** For loading old saved states. (pre-smp) */
+typedef struct
+{
+    /** If set no conflict checks are required.  (boolean) */
+    bool                            fMappingsFixed;
+    /** Size of fixed mapping */
+    uint32_t                        cbMappingFixed;
+    /** Base address (GC) of fixed mapping */
+    RTGCPTR                         GCPtrMappingFixed;
+    /** A20 gate mask.
+     * Our current approach to A20 emulation is to let REM do it and don't bother
+     * anywhere else. The interesting guests will be operating with it enabled anyway.
+     * But should the need arise, we'll subject physical addresses to this mask. */
+    RTGCPHYS                        GCPhysA20Mask;
+    /** A20 gate state - boolean! */
+    bool                            fA20Enabled;
+    /** The guest paging mode. */
+    PGMMODE                         enmGuestMode;
+} PGMOLD;
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** PGM fields to save/load. */
+
+static const SSMFIELD s_aPGMFields[] =
+{
+    SSMFIELD_ENTRY(         PGM, fMappingsFixed),
+    SSMFIELD_ENTRY_GCPTR(   PGM, GCPtrMappingFixed),
+    SSMFIELD_ENTRY(         PGM, cbMappingFixed),
+    SSMFIELD_ENTRY(         PGM, cBalloonedPages),
+    SSMFIELD_ENTRY_TERM()
+};
+
+static const SSMFIELD s_aPGMFieldsPreBalloon[] =
+{
+    SSMFIELD_ENTRY(         PGM, fMappingsFixed),
+    SSMFIELD_ENTRY_GCPTR(   PGM, GCPtrMappingFixed),
+    SSMFIELD_ENTRY(         PGM, cbMappingFixed),
+    SSMFIELD_ENTRY_TERM()
+};
+
+static const SSMFIELD s_aPGMCpuFields[] =
+{
+    SSMFIELD_ENTRY(         PGMCPU, fA20Enabled),
+    SSMFIELD_ENTRY_GCPHYS(  PGMCPU, GCPhysA20Mask),
+    SSMFIELD_ENTRY(         PGMCPU, enmGuestMode),
+    SSMFIELD_ENTRY(         PGMCPU, aGCPhysGstPaePDs[0]),
+    SSMFIELD_ENTRY(         PGMCPU, aGCPhysGstPaePDs[1]),
+    SSMFIELD_ENTRY(         PGMCPU, aGCPhysGstPaePDs[2]),
+    SSMFIELD_ENTRY(         PGMCPU, aGCPhysGstPaePDs[3]),
+    SSMFIELD_ENTRY_TERM()
+};
+
+static const SSMFIELD s_aPGMCpuFieldsPrePae[] =
+{
+    SSMFIELD_ENTRY(         PGMCPU, fA20Enabled),
+    SSMFIELD_ENTRY_GCPHYS(  PGMCPU, GCPhysA20Mask),
+    SSMFIELD_ENTRY(         PGMCPU, enmGuestMode),
+    SSMFIELD_ENTRY_TERM()
+};
+
+static const SSMFIELD s_aPGMFields_Old[] =
+{
+    SSMFIELD_ENTRY(         PGMOLD, fMappingsFixed),
+    SSMFIELD_ENTRY_GCPTR(   PGMOLD, GCPtrMappingFixed),
+    SSMFIELD_ENTRY(         PGMOLD, cbMappingFixed),
+    SSMFIELD_ENTRY(         PGMOLD, fA20Enabled),
+    SSMFIELD_ENTRY_GCPHYS(  PGMOLD, GCPhysA20Mask),
+    SSMFIELD_ENTRY(         PGMOLD, enmGuestMode),
+    SSMFIELD_ENTRY_TERM()
+};
+
+
+/**
+ * Find the ROM tracking structure for the given page.
+ *
+ * @returns Pointer to the ROM page structure. NULL if the caller didn't check
+ *          that it's a ROM page.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The address of the ROM page.
+ */
+static PPGMROMPAGE pgmR3GetRomPage(PVM pVM, RTGCPHYS GCPhys) /** @todo change this to take a hint. */
+{
+    for (PPGMROMRANGE pRomRange = pVM->pgm.s.CTX_SUFF(pRomRanges);
+         pRomRange;
+         pRomRange = pRomRange->CTX_SUFF(pNext))
+    {
+        RTGCPHYS off = GCPhys - pRomRange->GCPhys;
+        if (GCPhys - pRomRange->GCPhys < pRomRange->cb)
+            return &pRomRange->aPages[off >> PAGE_SHIFT];
+    }
+    return NULL;
+}
+
+
+/**
+ * Prepares the ROM pages for a live save.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ */
+static int pgmR3PrepRomPages(PVM pVM)
+{
+    /*
+     * Initialize the live save tracking in the ROM page descriptors.
+     */
+    pgmLock(pVM);
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        PPGMRAMRANGE    pRamHint = NULL;;
+        uint32_t const  cPages   = pRom->cb >> PAGE_SHIFT;
+
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            pRom->aPages[iPage].LiveSave.u8Prot           = (uint8_t)PGMROMPROT_INVALID;
+            pRom->aPages[iPage].LiveSave.fWrittenTo       = false;
+            pRom->aPages[iPage].LiveSave.fDirty           = true;
+            pRom->aPages[iPage].LiveSave.fDirtiedRecently = true;
+            if (!(pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
+            {
+                if (PGMROMPROT_IS_ROM(pRom->aPages[iPage].enmProt))
+                    pRom->aPages[iPage].LiveSave.fWrittenTo = !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow) && !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow);
+                else
+                {
+                    RTGCPHYS GCPhys = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+                    PPGMPAGE pPage;
+                    int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
+                    AssertLogRelMsgRC(rc, ("%Rrc GCPhys=%RGp\n", rc, GCPhys));
+                    if (RT_SUCCESS(rc))
+                        pRom->aPages[iPage].LiveSave.fWrittenTo = !PGM_PAGE_IS_ZERO(pPage) && !PGM_PAGE_IS_BALLOONED(pPage);
+                    else
+                        pRom->aPages[iPage].LiveSave.fWrittenTo = !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow) && !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow);
+                }
+            }
+        }
+
+        pVM->pgm.s.LiveSave.Rom.cDirtyPages += cPages;
+        if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+            pVM->pgm.s.LiveSave.Rom.cDirtyPages += cPages;
+    }
+    pgmUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Assigns IDs to the ROM ranges and saves them.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                Saved state handle.
+ */
+static int pgmR3SaveRomRanges(PVM pVM, PSSMHANDLE pSSM)
+{
+    pgmLock(pVM);
+    uint8_t id = 1;
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3, id++)
+    {
+        pRom->idSavedState = id;
+        SSMR3PutU8(pSSM, id);
+        SSMR3PutStrZ(pSSM, "");         /* device name */
+        SSMR3PutU32(pSSM, 0);           /* device instance */
+        SSMR3PutU8(pSSM, 0);            /* region */
+        SSMR3PutStrZ(pSSM, pRom->pszDesc);
+        SSMR3PutGCPhys(pSSM, pRom->GCPhys);
+        int rc = SSMR3PutGCPhys(pSSM, pRom->cb);
+        if (RT_FAILURE(rc))
+            break;
+    }
+    pgmUnlock(pVM);
+    return SSMR3PutU8(pSSM, UINT8_MAX);
+}
+
+
+/**
+ * Loads the ROM range ID assignments.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int pgmR3LoadRomRanges(PVM pVM, PSSMHANDLE pSSM)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+        pRom->idSavedState = UINT8_MAX;
+
+    for (;;)
+    {
+        /*
+         * Read the data.
+         */
+        uint8_t id;
+        int rc = SSMR3GetU8(pSSM, &id);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (id == UINT8_MAX)
+        {
+            for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+                if (pRom->idSavedState != UINT8_MAX)
+                { /* likely */ }
+                else if (pRom->fFlags & PGMPHYS_ROM_FLAGS_MAYBE_MISSING_FROM_STATE)
+                    LogRel(("PGM: The '%s' ROM was not found in the saved state, but it is marked as maybe-missing, so that's probably okay.\n",
+                            pRom->pszDesc));
+                else
+                    AssertLogRelMsg(pRom->idSavedState != UINT8_MAX,
+                                    ("The '%s' ROM was not found in the saved state. Probably due to some misconfiguration\n",
+                                     pRom->pszDesc));
+            return VINF_SUCCESS;        /* the end */
+        }
+        AssertLogRelReturn(id != 0, VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        char szDevName[RT_SIZEOFMEMB(PDMDEVREG, szName)];
+        rc = SSMR3GetStrZ(pSSM, szDevName, sizeof(szDevName));
+        AssertLogRelRCReturn(rc, rc);
+
+        uint32_t    uInstance;
+        SSMR3GetU32(pSSM, &uInstance);
+        uint8_t     iRegion;
+        SSMR3GetU8(pSSM, &iRegion);
+
+        char szDesc[64];
+        rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
+        AssertLogRelRCReturn(rc, rc);
+
+        RTGCPHYS GCPhys;
+        SSMR3GetGCPhys(pSSM, &GCPhys);
+        RTGCPHYS cb;
+        rc = SSMR3GetGCPhys(pSSM, &cb);
+        if (RT_FAILURE(rc))
+            return rc;
+        AssertLogRelMsgReturn(!(GCPhys & PAGE_OFFSET_MASK), ("GCPhys=%RGp %s\n", GCPhys, szDesc), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+        AssertLogRelMsgReturn(!(cb & PAGE_OFFSET_MASK),     ("cb=%RGp %s\n", cb, szDesc),         VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        /*
+         * Locate a matching ROM range.
+         */
+        AssertLogRelMsgReturn(   uInstance == 0
+                              && iRegion == 0
+                              && szDevName[0] == '\0',
+                              ("GCPhys=%RGp %s\n", GCPhys, szDesc),
+                              VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+        PPGMROMRANGE pRom;
+        for (pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+        {
+            if (    pRom->idSavedState == UINT8_MAX
+                &&  !strcmp(pRom->pszDesc, szDesc))
+            {
+                pRom->idSavedState = id;
+                break;
+            }
+        }
+        if (!pRom)
+            return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("ROM at %RGp by the name '%s' was not found"), GCPhys, szDesc);
+    } /* forever */
+}
+
+
+/**
+ * Scan ROM pages.
+ *
+ * @param   pVM                 The cross context VM structure.
+ */
+static void pgmR3ScanRomPages(PVM pVM)
+{
+    /*
+     * The shadow ROMs.
+     */
+    pgmLock(pVM);
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+        {
+            uint32_t const cPages = pRom->cb >> PAGE_SHIFT;
+            for (uint32_t iPage = 0; iPage < cPages; iPage++)
+            {
+                PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
+                if (pRomPage->LiveSave.fWrittenTo)
+                {
+                    pRomPage->LiveSave.fWrittenTo = false;
+                    if (!pRomPage->LiveSave.fDirty)
+                    {
+                        pRomPage->LiveSave.fDirty = true;
+                        pVM->pgm.s.LiveSave.Rom.cReadyPages--;
+                        pVM->pgm.s.LiveSave.Rom.cDirtyPages++;
+                    }
+                    pRomPage->LiveSave.fDirtiedRecently = true;
+                }
+                else
+                    pRomPage->LiveSave.fDirtiedRecently = false;
+            }
+        }
+    }
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Takes care of the virgin ROM pages in the first pass.
+ *
+ * This is an attempt at simplifying the handling of ROM pages a little bit.
+ * This ASSUMES that no new ROM ranges will be added and that they won't be
+ * relinked in any way.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The SSM handle.
+ * @param   fLiveSave           Whether we're in a live save or not.
+ */
+static int pgmR3SaveRomVirginPages(PVM pVM, PSSMHANDLE pSSM, bool fLiveSave)
+{
+    pgmLock(pVM);
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        uint32_t const cPages = pRom->cb >> PAGE_SHIFT;
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            RTGCPHYS   GCPhys  = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+            PGMROMPROT enmProt = pRom->aPages[iPage].enmProt;
+
+            /* Get the virgin page descriptor. */
+            PPGMPAGE pPage;
+            if (PGMROMPROT_IS_ROM(enmProt))
+                pPage = pgmPhysGetPage(pVM, GCPhys);
+            else
+                pPage = &pRom->aPages[iPage].Virgin;
+
+            /* Get the page bits. (Cannot use pgmPhysGCPhys2CCPtrInternalReadOnly here!) */
+            int rc = VINF_SUCCESS;
+            char abPage[PAGE_SIZE];
+            if (    !PGM_PAGE_IS_ZERO(pPage)
+                &&  !PGM_PAGE_IS_BALLOONED(pPage))
+            {
+                void const *pvPage;
+                rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvPage);
+                if (RT_SUCCESS(rc))
+                    memcpy(abPage, pvPage, PAGE_SIZE);
+            }
+            else
+                ASMMemZeroPage(abPage);
+            pgmUnlock(pVM);
+            AssertLogRelMsgRCReturn(rc, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys), rc);
+
+            /* Save it. */
+            if (iPage > 0)
+                SSMR3PutU8(pSSM, PGM_STATE_REC_ROM_VIRGIN);
+            else
+            {
+                SSMR3PutU8(pSSM, PGM_STATE_REC_ROM_VIRGIN | PGM_STATE_REC_FLAG_ADDR);
+                SSMR3PutU8(pSSM, pRom->idSavedState);
+                SSMR3PutU32(pSSM, iPage);
+            }
+            SSMR3PutU8(pSSM, (uint8_t)enmProt);
+            rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
+            if (RT_FAILURE(rc))
+                return rc;
+
+            /* Update state. */
+            pgmLock(pVM);
+            pRom->aPages[iPage].LiveSave.u8Prot = (uint8_t)enmProt;
+            if (fLiveSave)
+            {
+                pVM->pgm.s.LiveSave.Rom.cDirtyPages--;
+                pVM->pgm.s.LiveSave.Rom.cReadyPages++;
+                pVM->pgm.s.LiveSave.cSavedPages++;
+            }
+        }
+    }
+    pgmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Saves dirty pages in the shadowed ROM ranges.
+ *
+ * Used by pgmR3LiveExecPart2 and pgmR3SaveExecMemory.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The SSM handle.
+ * @param   fLiveSave           Whether it's a live save or not.
+ * @param   fFinalPass          Whether this is the final pass or not.
+ */
+static int pgmR3SaveShadowedRomPages(PVM pVM, PSSMHANDLE pSSM, bool fLiveSave, bool fFinalPass)
+{
+    /*
+     * The Shadowed ROMs.
+     *
+     * ASSUMES that the ROM ranges are fixed.
+     * ASSUMES that all the ROM ranges are mapped.
+     */
+    pgmLock(pVM);
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+        {
+            uint32_t const cPages    = pRom->cb >> PAGE_SHIFT;
+            uint32_t       iPrevPage = cPages;
+            for (uint32_t iPage = 0; iPage < cPages; iPage++)
+            {
+                PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
+                if (    !fLiveSave
+                    ||  (   pRomPage->LiveSave.fDirty
+                         && (   (   !pRomPage->LiveSave.fDirtiedRecently
+                                 && !pRomPage->LiveSave.fWrittenTo)
+                             || fFinalPass
+                             )
+                         )
+                    )
+                {
+                    uint8_t     abPage[PAGE_SIZE];
+                    PGMROMPROT  enmProt = pRomPage->enmProt;
+                    RTGCPHYS    GCPhys  = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+                    PPGMPAGE    pPage   = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Shadow : pgmPhysGetPage(pVM, GCPhys);
+                    bool        fZero   = PGM_PAGE_IS_ZERO(pPage) || PGM_PAGE_IS_BALLOONED(pPage); Assert(!PGM_PAGE_IS_BALLOONED(pPage)); /* Shouldn't be ballooned. */
+                    int         rc      = VINF_SUCCESS;
+                    if (!fZero)
+                    {
+                        void const *pvPage;
+                        rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvPage);
+                        if (RT_SUCCESS(rc))
+                            memcpy(abPage, pvPage, PAGE_SIZE);
+                    }
+                    if (fLiveSave && RT_SUCCESS(rc))
+                    {
+                        pRomPage->LiveSave.u8Prot = (uint8_t)enmProt;
+                        pRomPage->LiveSave.fDirty = false;
+                        pVM->pgm.s.LiveSave.Rom.cReadyPages++;
+                        pVM->pgm.s.LiveSave.Rom.cDirtyPages--;
+                        pVM->pgm.s.LiveSave.cSavedPages++;
+                    }
+                    pgmUnlock(pVM);
+                    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys), rc);
+
+                    if (iPage - 1U == iPrevPage && iPage > 0)
+                        SSMR3PutU8(pSSM, (fZero ? PGM_STATE_REC_ROM_SHW_ZERO : PGM_STATE_REC_ROM_SHW_RAW));
+                    else
+                    {
+                        SSMR3PutU8(pSSM, (fZero ? PGM_STATE_REC_ROM_SHW_ZERO : PGM_STATE_REC_ROM_SHW_RAW) | PGM_STATE_REC_FLAG_ADDR);
+                        SSMR3PutU8(pSSM, pRom->idSavedState);
+                        SSMR3PutU32(pSSM, iPage);
+                    }
+                    rc = SSMR3PutU8(pSSM, (uint8_t)enmProt);
+                    if (!fZero)
+                        rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
+                    if (RT_FAILURE(rc))
+                        return rc;
+
+                    pgmLock(pVM);
+                    iPrevPage = iPage;
+                }
+                /*
+                 * In the final pass, make sure the protection is in sync.
+                 */
+                else if (   fFinalPass
+                         && pRomPage->LiveSave.u8Prot != pRomPage->enmProt)
+                {
+                    PGMROMPROT enmProt = pRomPage->enmProt;
+                    pRomPage->LiveSave.u8Prot = (uint8_t)enmProt;
+                    pgmUnlock(pVM);
+
+                    if (iPage - 1U == iPrevPage && iPage > 0)
+                        SSMR3PutU8(pSSM, PGM_STATE_REC_ROM_PROT);
+                    else
+                    {
+                        SSMR3PutU8(pSSM, PGM_STATE_REC_ROM_PROT | PGM_STATE_REC_FLAG_ADDR);
+                        SSMR3PutU8(pSSM, pRom->idSavedState);
+                        SSMR3PutU32(pSSM, iPage);
+                    }
+                    int rc = SSMR3PutU8(pSSM, (uint8_t)enmProt);
+                    if (RT_FAILURE(rc))
+                        return rc;
+
+                    pgmLock(pVM);
+                    iPrevPage = iPage;
+                }
+            }
+        }
+    }
+    pgmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Cleans up ROM pages after a live save.
+ *
+ * @param   pVM                 The cross context VM structure.
+ */
+static void pgmR3DoneRomPages(PVM pVM)
+{
+    NOREF(pVM);
+}
+
+
+/**
+ * Prepares the MMIO2 pages for a live save.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ */
+static int pgmR3PrepMmio2Pages(PVM pVM)
+{
+    /*
+     * Initialize the live save tracking in the MMIO2 ranges.
+     * ASSUME nothing changes here.
+     */
+    pgmLock(pVM);
+    for (PPGMREGMMIO2RANGE pRegMmio = pVM->pgm.s.pRegMmioRangesR3; pRegMmio; pRegMmio = pRegMmio->pNextR3)
+    {
+        if (pRegMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2)
+        {
+            uint32_t const  cPages = pRegMmio->RamRange.cb >> PAGE_SHIFT;
+            pgmUnlock(pVM);
+
+            PPGMLIVESAVEMMIO2PAGE paLSPages = (PPGMLIVESAVEMMIO2PAGE)MMR3HeapAllocZ(pVM, MM_TAG_PGM, sizeof(PGMLIVESAVEMMIO2PAGE) * cPages);
+            if (!paLSPages)
+                return VERR_NO_MEMORY;
+            for (uint32_t iPage = 0; iPage < cPages; iPage++)
+            {
+                /* Initialize it as a dirty zero page. */
+                paLSPages[iPage].fDirty          = true;
+                paLSPages[iPage].cUnchangedScans = 0;
+                paLSPages[iPage].fZero           = true;
+                paLSPages[iPage].u32CrcH1        = PGM_STATE_CRC32_ZERO_HALF_PAGE;
+                paLSPages[iPage].u32CrcH2        = PGM_STATE_CRC32_ZERO_HALF_PAGE;
+            }
+
+            pgmLock(pVM);
+            pRegMmio->paLSPages = paLSPages;
+            pVM->pgm.s.LiveSave.Mmio2.cDirtyPages += cPages;
+        }
+    }
+    pgmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Assigns IDs to the MMIO2 ranges and saves them.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                Saved state handle.
+ */
+static int pgmR3SaveMmio2Ranges(PVM pVM, PSSMHANDLE pSSM)
+{
+    pgmLock(pVM);
+    uint8_t id = 1;
+    for (PPGMREGMMIO2RANGE pRegMmio = pVM->pgm.s.pRegMmioRangesR3; pRegMmio; pRegMmio = pRegMmio->pNextR3)
+    {
+        if (pRegMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2)
+        {
+            pRegMmio->idSavedState = id;
+            SSMR3PutU8(pSSM, id);
+            SSMR3PutStrZ(pSSM, pRegMmio->pDevInsR3->pReg->szName);
+            SSMR3PutU32(pSSM, pRegMmio->pDevInsR3->iInstance);
+            SSMR3PutU8(pSSM, pRegMmio->iRegion);
+            SSMR3PutStrZ(pSSM, pRegMmio->RamRange.pszDesc);
+            int rc = SSMR3PutGCPhys(pSSM, pRegMmio->RamRange.cb);
+            if (RT_FAILURE(rc))
+                break;
+            id++;
+        }
+    }
+    pgmUnlock(pVM);
+    return SSMR3PutU8(pSSM, UINT8_MAX);
+}
+
+
+/**
+ * Loads the MMIO2 range ID assignments.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int pgmR3LoadMmio2Ranges(PVM pVM, PSSMHANDLE pSSM)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    for (PPGMREGMMIO2RANGE pRegMmio = pVM->pgm.s.pRegMmioRangesR3; pRegMmio; pRegMmio = pRegMmio->pNextR3)
+        if (pRegMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2)
+            pRegMmio->idSavedState = UINT8_MAX;
+
+    for (;;)
+    {
+        /*
+         * Read the data.
+         */
+        uint8_t id;
+        int rc = SSMR3GetU8(pSSM, &id);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (id == UINT8_MAX)
+        {
+            for (PPGMREGMMIO2RANGE pRegMmio = pVM->pgm.s.pRegMmioRangesR3; pRegMmio; pRegMmio = pRegMmio->pNextR3)
+                AssertLogRelMsg(   pRegMmio->idSavedState != UINT8_MAX
+                                || !(pRegMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2),
+                                ("%s\n", pRegMmio->RamRange.pszDesc));
+            return VINF_SUCCESS;        /* the end */
+        }
+        AssertLogRelReturn(id != 0, VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        char szDevName[RT_SIZEOFMEMB(PDMDEVREG, szName)];
+        rc = SSMR3GetStrZ(pSSM, szDevName, sizeof(szDevName));
+        AssertLogRelRCReturn(rc, rc);
+
+        uint32_t    uInstance;
+        SSMR3GetU32(pSSM, &uInstance);
+        uint8_t     iRegion;
+        SSMR3GetU8(pSSM, &iRegion);
+
+        char szDesc[64];
+        rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
+        AssertLogRelRCReturn(rc, rc);
+
+        RTGCPHYS cb;
+        rc = SSMR3GetGCPhys(pSSM, &cb);
+        AssertLogRelMsgReturn(!(cb & PAGE_OFFSET_MASK), ("cb=%RGp %s\n", cb, szDesc), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        /*
+         * Locate a matching MMIO2 range.
+         */
+        PPGMREGMMIO2RANGE pRegMmio;
+        for (pRegMmio = pVM->pgm.s.pRegMmioRangesR3; pRegMmio; pRegMmio = pRegMmio->pNextR3)
+        {
+            if (    pRegMmio->idSavedState == UINT8_MAX
+                &&  pRegMmio->iRegion == iRegion
+                &&  pRegMmio->pDevInsR3->iInstance == uInstance
+                &&  (pRegMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2)
+                &&  !strcmp(pRegMmio->pDevInsR3->pReg->szName, szDevName))
+            {
+                pRegMmio->idSavedState = id;
+                break;
+            }
+        }
+        if (!pRegMmio)
+            return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Failed to locate a MMIO2 range called '%s' owned by %s/%u, region %d"),
+                                    szDesc, szDevName, uInstance, iRegion);
+
+        /*
+         * Validate the configuration, the size of the MMIO2 region should be
+         * the same.
+         */
+        if (cb != pRegMmio->RamRange.cb)
+        {
+            LogRel(("PGM: MMIO2 region \"%s\" size mismatch: saved=%RGp config=%RGp\n",
+                    pRegMmio->RamRange.pszDesc, cb, pRegMmio->RamRange.cb));
+            if (cb > pRegMmio->RamRange.cb) /* bad idea? */
+                return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("MMIO2 region \"%s\" size mismatch: saved=%RGp config=%RGp"),
+                                        pRegMmio->RamRange.pszDesc, cb, pRegMmio->RamRange.cb);
+        }
+    } /* forever */
+}
+
+
+/**
+ * Scans one MMIO2 page.
+ *
+ * @returns True if changed, false if unchanged.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pbPage              The page bits.
+ * @param   pLSPage             The live save tracking structure for the page.
+ *
+ */
+DECLINLINE(bool) pgmR3ScanMmio2Page(PVM pVM, uint8_t const *pbPage, PPGMLIVESAVEMMIO2PAGE pLSPage)
+{
+    /*
+     * Special handling of zero pages.
+     */
+    bool const fZero = pLSPage->fZero;
+    if (fZero)
+    {
+        if (ASMMemIsZeroPage(pbPage))
+        {
+            /* Not modified. */
+            if (pLSPage->fDirty)
+                pLSPage->cUnchangedScans++;
+            return false;
+        }
+
+        pLSPage->fZero    = false;
+        pLSPage->u32CrcH1 = RTCrc32(pbPage, PAGE_SIZE / 2);
+    }
+    else
+    {
+        /*
+         * CRC the first half, if it doesn't match the page is dirty and
+         * we won't check the 2nd half (we'll do that next time).
+         */
+        uint32_t u32CrcH1 = RTCrc32(pbPage, PAGE_SIZE / 2);
+        if (u32CrcH1 == pLSPage->u32CrcH1)
+        {
+            uint32_t u32CrcH2 = RTCrc32(pbPage + PAGE_SIZE / 2, PAGE_SIZE / 2);
+            if (u32CrcH2 == pLSPage->u32CrcH2)
+            {
+                /* Probably not modified. */
+                if (pLSPage->fDirty)
+                    pLSPage->cUnchangedScans++;
+                return false;
+            }
+
+            pLSPage->u32CrcH2 = u32CrcH2;
+        }
+        else
+        {
+            pLSPage->u32CrcH1 = u32CrcH1;
+            if (    u32CrcH1 == PGM_STATE_CRC32_ZERO_HALF_PAGE
+                &&  ASMMemIsZeroPage(pbPage))
+            {
+                pLSPage->u32CrcH2 = PGM_STATE_CRC32_ZERO_HALF_PAGE;
+                pLSPage->fZero    = true;
+            }
+        }
+    }
+
+    /* dirty page path */
+    pLSPage->cUnchangedScans = 0;
+    if (!pLSPage->fDirty)
+    {
+        pLSPage->fDirty = true;
+        pVM->pgm.s.LiveSave.Mmio2.cReadyPages--;
+        pVM->pgm.s.LiveSave.Mmio2.cDirtyPages++;
+        if (fZero)
+            pVM->pgm.s.LiveSave.Mmio2.cZeroPages--;
+    }
+    return true;
+}
+
+
+/**
+ * Scan for MMIO2 page modifications.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   uPass               The pass number.
+ */
+static void pgmR3ScanMmio2Pages(PVM pVM, uint32_t uPass)
+{
+    /*
+     * Since this is a bit expensive we lower the scan rate after a little while.
+     */
+    if (    (    (uPass & 3) != 0
+             &&  uPass > 10)
+        ||  uPass == SSM_PASS_FINAL)
+        return;
+
+    pgmLock(pVM);                       /* paranoia */
+    for (PPGMREGMMIO2RANGE pRegMmio = pVM->pgm.s.pRegMmioRangesR3; pRegMmio; pRegMmio = pRegMmio->pNextR3)
+        if (pRegMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2)
+        {
+            PPGMLIVESAVEMMIO2PAGE paLSPages = pRegMmio->paLSPages;
+            uint32_t              cPages    = pRegMmio->RamRange.cb >> PAGE_SHIFT;
+            pgmUnlock(pVM);
+
+            for (uint32_t iPage = 0; iPage < cPages; iPage++)
+            {
+                uint8_t const *pbPage = (uint8_t const *)pRegMmio->pvR3 + iPage * PAGE_SIZE;
+                pgmR3ScanMmio2Page(pVM, pbPage, &paLSPages[iPage]);
+            }
+
+            pgmLock(pVM);
+        }
+    pgmUnlock(pVM);
+
+}
+
+
+/**
+ * Save quiescent MMIO2 pages.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The SSM handle.
+ * @param   fLiveSave           Whether it's a live save or not.
+ * @param   uPass               The pass number.
+ */
+static int pgmR3SaveMmio2Pages(PVM pVM, PSSMHANDLE pSSM, bool fLiveSave, uint32_t uPass)
+{
+    /** @todo implement live saving of MMIO2 pages. (Need some way of telling the
+     *        device that we wish to know about changes.) */
+
+    int rc = VINF_SUCCESS;
+    if (uPass == SSM_PASS_FINAL)
+    {
+        /*
+         * The mop up round.
+         */
+        pgmLock(pVM);
+        for (PPGMREGMMIO2RANGE pRegMmio = pVM->pgm.s.pRegMmioRangesR3;
+             pRegMmio && RT_SUCCESS(rc);
+             pRegMmio = pRegMmio->pNextR3)
+            if (pRegMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2)
+            {
+                PPGMLIVESAVEMMIO2PAGE paLSPages = pRegMmio->paLSPages;
+                uint8_t const        *pbPage    = (uint8_t const *)pRegMmio->RamRange.pvR3;
+                uint32_t              cPages    = pRegMmio->RamRange.cb >> PAGE_SHIFT;
+                uint32_t              iPageLast = cPages;
+                for (uint32_t iPage = 0; iPage < cPages; iPage++, pbPage += PAGE_SIZE)
+                {
+                    uint8_t u8Type;
+                    if (!fLiveSave)
+                        u8Type = ASMMemIsZeroPage(pbPage) ? PGM_STATE_REC_MMIO2_ZERO : PGM_STATE_REC_MMIO2_RAW;
+                    else
+                    {
+                        /* Try figure if it's a clean page, compare the SHA-1 to be really sure. */
+                        if (   !paLSPages[iPage].fDirty
+                            && !pgmR3ScanMmio2Page(pVM, pbPage, &paLSPages[iPage]))
+                        {
+                            if (paLSPages[iPage].fZero)
+                                continue;
+
+                            uint8_t abSha1Hash[RTSHA1_HASH_SIZE];
+                            RTSha1(pbPage, PAGE_SIZE, abSha1Hash);
+                            if (!memcmp(abSha1Hash, paLSPages[iPage].abSha1Saved, sizeof(abSha1Hash)))
+                                continue;
+                        }
+                        u8Type = paLSPages[iPage].fZero ? PGM_STATE_REC_MMIO2_ZERO : PGM_STATE_REC_MMIO2_RAW;
+                        pVM->pgm.s.LiveSave.cSavedPages++;
+                    }
+
+                    if (iPage != 0 && iPage == iPageLast + 1)
+                        rc = SSMR3PutU8(pSSM, u8Type);
+                    else
+                    {
+                        SSMR3PutU8(pSSM, u8Type | PGM_STATE_REC_FLAG_ADDR);
+                        SSMR3PutU8(pSSM, pRegMmio->idSavedState);
+                        rc = SSMR3PutU32(pSSM, iPage);
+                    }
+                    if (u8Type == PGM_STATE_REC_MMIO2_RAW)
+                        rc = SSMR3PutMem(pSSM, pbPage, PAGE_SIZE);
+                    if (RT_FAILURE(rc))
+                        break;
+                    iPageLast = iPage;
+                }
+            }
+        pgmUnlock(pVM);
+    }
+    /*
+     * Reduce the rate after a little while since the current MMIO2 approach is
+     * a bit expensive.
+     * We position it two passes after the scan pass to avoid saving busy pages.
+     */
+    else if (   uPass <= 10
+             || (uPass & 3) == 2)
+    {
+        pgmLock(pVM);
+        for (PPGMREGMMIO2RANGE pRegMmio = pVM->pgm.s.pRegMmioRangesR3;
+             pRegMmio && RT_SUCCESS(rc);
+             pRegMmio = pRegMmio->pNextR3)
+            if (pRegMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2)
+            {
+                PPGMLIVESAVEMMIO2PAGE paLSPages = pRegMmio->paLSPages;
+                uint8_t const        *pbPage    = (uint8_t const *)pRegMmio->RamRange.pvR3;
+                uint32_t              cPages    = pRegMmio->RamRange.cb >> PAGE_SHIFT;
+                uint32_t              iPageLast = cPages;
+                pgmUnlock(pVM);
+
+                for (uint32_t iPage = 0; iPage < cPages; iPage++, pbPage += PAGE_SIZE)
+                {
+                    /* Skip clean pages and pages which hasn't quiesced. */
+                    if (!paLSPages[iPage].fDirty)
+                        continue;
+                    if (paLSPages[iPage].cUnchangedScans < 3)
+                        continue;
+                    if (pgmR3ScanMmio2Page(pVM, pbPage, &paLSPages[iPage]))
+                        continue;
+
+                    /* Save it. */
+                    bool const fZero = paLSPages[iPage].fZero;
+                    uint8_t abPage[PAGE_SIZE];
+                    if (!fZero)
+                    {
+                        memcpy(abPage, pbPage, PAGE_SIZE);
+                        RTSha1(abPage, PAGE_SIZE, paLSPages[iPage].abSha1Saved);
+                    }
+
+                    uint8_t u8Type = paLSPages[iPage].fZero ? PGM_STATE_REC_MMIO2_ZERO : PGM_STATE_REC_MMIO2_RAW;
+                    if (iPage != 0 && iPage == iPageLast + 1)
+                        rc = SSMR3PutU8(pSSM, u8Type);
+                    else
+                    {
+                        SSMR3PutU8(pSSM, u8Type | PGM_STATE_REC_FLAG_ADDR);
+                        SSMR3PutU8(pSSM, pRegMmio->idSavedState);
+                        rc = SSMR3PutU32(pSSM, iPage);
+                    }
+                    if (u8Type == PGM_STATE_REC_MMIO2_RAW)
+                        rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
+                    if (RT_FAILURE(rc))
+                        break;
+
+                    /* Housekeeping. */
+                    paLSPages[iPage].fDirty = false;
+                    pVM->pgm.s.LiveSave.Mmio2.cDirtyPages--;
+                    pVM->pgm.s.LiveSave.Mmio2.cReadyPages++;
+                    if (u8Type == PGM_STATE_REC_MMIO2_ZERO)
+                        pVM->pgm.s.LiveSave.Mmio2.cZeroPages++;
+                    pVM->pgm.s.LiveSave.cSavedPages++;
+                    iPageLast = iPage;
+                }
+
+                pgmLock(pVM);
+            }
+        pgmUnlock(pVM);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Cleans up MMIO2 pages after a live save.
+ *
+ * @param   pVM                 The cross context VM structure.
+ */
+static void pgmR3DoneMmio2Pages(PVM pVM)
+{
+    /*
+     * Free the tracking structures for the MMIO2 pages.
+     * We do the freeing outside the lock in case the VM is running.
+     */
+    pgmLock(pVM);
+    for (PPGMREGMMIO2RANGE pRegMmio = pVM->pgm.s.pRegMmioRangesR3; pRegMmio; pRegMmio = pRegMmio->pNextR3)
+        if (pRegMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2)
+        {
+            void *pvMmio2ToFree = pRegMmio->paLSPages;
+            if (pvMmio2ToFree)
+            {
+                pRegMmio->paLSPages = NULL;
+                pgmUnlock(pVM);
+                MMR3HeapFree(pvMmio2ToFree);
+                pgmLock(pVM);
+            }
+        }
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Prepares the RAM pages for a live save.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ */
+static int pgmR3PrepRamPages(PVM pVM)
+{
+
+    /*
+     * Try allocating tracking structures for the ram ranges.
+     *
+     * To avoid lock contention, we leave the lock every time we're allocating
+     * a new array.  This means we'll have to ditch the allocation and start
+     * all over again if the RAM range list changes in-between.
+     *
+     * Note! pgmR3SaveDone will always be called and it is therefore responsible
+     *       for cleaning up.
+     */
+    PPGMRAMRANGE pCur;
+    pgmLock(pVM);
+    do
+    {
+        for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
+        {
+            if (   !pCur->paLSPages
+                && !PGM_RAM_RANGE_IS_AD_HOC(pCur))
+            {
+                uint32_t const  idRamRangesGen = pVM->pgm.s.idRamRangesGen;
+                uint32_t const  cPages = pCur->cb >> PAGE_SHIFT;
+                pgmUnlock(pVM);
+                PPGMLIVESAVERAMPAGE paLSPages = (PPGMLIVESAVERAMPAGE)MMR3HeapAllocZ(pVM, MM_TAG_PGM, cPages * sizeof(PGMLIVESAVERAMPAGE));
+                if (!paLSPages)
+                    return VERR_NO_MEMORY;
+                pgmLock(pVM);
+                if (pVM->pgm.s.idRamRangesGen != idRamRangesGen)
+                {
+                    pgmUnlock(pVM);
+                    MMR3HeapFree(paLSPages);
+                    pgmLock(pVM);
+                    break;              /* try again */
+                }
+                pCur->paLSPages = paLSPages;
+
+                /*
+                 * Initialize the array.
+                 */
+                uint32_t iPage = cPages;
+                while (iPage-- > 0)
+                {
+                    /** @todo yield critsect! (after moving this away from EMT0) */
+                    PCPGMPAGE pPage = &pCur->aPages[iPage];
+                    paLSPages[iPage].cDirtied               = 0;
+                    paLSPages[iPage].fDirty                 = 1; /* everything is dirty at this time */
+                    paLSPages[iPage].fWriteMonitored        = 0;
+                    paLSPages[iPage].fWriteMonitoredJustNow = 0;
+                    paLSPages[iPage].u2Reserved             = 0;
+                    switch (PGM_PAGE_GET_TYPE(pPage))
+                    {
+                        case PGMPAGETYPE_RAM:
+                            if (    PGM_PAGE_IS_ZERO(pPage)
+                                ||  PGM_PAGE_IS_BALLOONED(pPage))
+                            {
+                                paLSPages[iPage].fZero   = 1;
+                                paLSPages[iPage].fShared = 0;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                paLSPages[iPage].u32Crc  = PGM_STATE_CRC32_ZERO_PAGE;
+#endif
+                            }
+                            else if (PGM_PAGE_IS_SHARED(pPage))
+                            {
+                                paLSPages[iPage].fZero   = 0;
+                                paLSPages[iPage].fShared = 1;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                paLSPages[iPage].u32Crc  = UINT32_MAX;
+#endif
+                            }
+                            else
+                            {
+                                paLSPages[iPage].fZero   = 0;
+                                paLSPages[iPage].fShared = 0;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                paLSPages[iPage].u32Crc  = UINT32_MAX;
+#endif
+                            }
+                            paLSPages[iPage].fIgnore     = 0;
+                            pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
+                            break;
+
+                        case PGMPAGETYPE_ROM_SHADOW:
+                        case PGMPAGETYPE_ROM:
+                        {
+                            paLSPages[iPage].fZero   = 0;
+                            paLSPages[iPage].fShared = 0;
+                            paLSPages[iPage].fDirty  = 0;
+                            paLSPages[iPage].fIgnore = 1;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                            paLSPages[iPage].u32Crc  = UINT32_MAX;
+#endif
+                            pVM->pgm.s.LiveSave.cIgnoredPages++;
+                            break;
+                        }
+
+                        default:
+                            AssertMsgFailed(("%R[pgmpage]", pPage));
+                            RT_FALL_THRU();
+                        case PGMPAGETYPE_MMIO2:
+                        case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
+                            paLSPages[iPage].fZero   = 0;
+                            paLSPages[iPage].fShared = 0;
+                            paLSPages[iPage].fDirty  = 0;
+                            paLSPages[iPage].fIgnore = 1;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                            paLSPages[iPage].u32Crc  = UINT32_MAX;
+#endif
+                            pVM->pgm.s.LiveSave.cIgnoredPages++;
+                            break;
+
+                        case PGMPAGETYPE_MMIO:
+                        case PGMPAGETYPE_SPECIAL_ALIAS_MMIO:
+                            paLSPages[iPage].fZero   = 0;
+                            paLSPages[iPage].fShared = 0;
+                            paLSPages[iPage].fDirty  = 0;
+                            paLSPages[iPage].fIgnore = 1;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                            paLSPages[iPage].u32Crc  = UINT32_MAX;
+#endif
+                            pVM->pgm.s.LiveSave.cIgnoredPages++;
+                            break;
+                    }
+                }
+            }
+        }
+    } while (pCur);
+    pgmUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Saves the RAM configuration.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int pgmR3SaveRamConfig(PVM pVM, PSSMHANDLE pSSM)
+{
+    uint32_t cbRamHole = 0;
+    int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "RamHoleSize", &cbRamHole, MM_RAM_HOLE_SIZE_DEFAULT);
+    AssertRCReturn(rc, rc);
+
+    uint64_t cbRam     = 0;
+    rc = CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRam, 0);
+    AssertRCReturn(rc, rc);
+
+    SSMR3PutU32(pSSM, cbRamHole);
+    return SSMR3PutU64(pSSM, cbRam);
+}
+
+
+/**
+ * Loads and verifies the RAM configuration.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int pgmR3LoadRamConfig(PVM pVM, PSSMHANDLE pSSM)
+{
+    uint32_t cbRamHoleCfg = 0;
+    int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "RamHoleSize", &cbRamHoleCfg, MM_RAM_HOLE_SIZE_DEFAULT);
+    AssertRCReturn(rc, rc);
+
+    uint64_t cbRamCfg     = 0;
+    rc = CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRamCfg, 0);
+    AssertRCReturn(rc, rc);
+
+    uint32_t cbRamHoleSaved;
+    SSMR3GetU32(pSSM, &cbRamHoleSaved);
+
+    uint64_t cbRamSaved;
+    rc = SSMR3GetU64(pSSM, &cbRamSaved);
+    AssertRCReturn(rc, rc);
+
+    if (   cbRamHoleCfg != cbRamHoleSaved
+        || cbRamCfg     != cbRamSaved)
+        return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Ram config mismatch: saved=%RX64/%RX32 config=%RX64/%RX32 (RAM/Hole)"),
+                                cbRamSaved, cbRamHoleSaved, cbRamCfg, cbRamHoleCfg);
+    return VINF_SUCCESS;
+}
+
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+
+/**
+ * Calculates the CRC-32 for a RAM page and updates the live save page tracking
+ * info with it.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pCur                The current RAM range.
+ * @param   paLSPages           The current array of live save page tracking
+ *                              structures.
+ * @param   iPage               The page index.
+ */
+static void pgmR3StateCalcCrc32ForRamPage(PVM pVM, PPGMRAMRANGE pCur, PPGMLIVESAVERAMPAGE paLSPages, uint32_t iPage)
+{
+    RTGCPHYS        GCPhys = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+    PGMPAGEMAPLOCK  PgMpLck;
+    void const     *pvPage;
+    int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, &pCur->aPages[iPage], GCPhys, &pvPage, &PgMpLck);
+    if (RT_SUCCESS(rc))
+    {
+        paLSPages[iPage].u32Crc = RTCrc32(pvPage, PAGE_SIZE);
+        pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+    }
+    else
+        paLSPages[iPage].u32Crc = UINT32_MAX; /* Invalid */
+}
+
+
+/**
+ * Verifies the CRC-32 for a page given it's raw bits.
+ *
+ * @param   pvPage              The page bits.
+ * @param   pCur                The current RAM range.
+ * @param   paLSPages           The current array of live save page tracking
+ *                              structures.
+ * @param   iPage               The page index.
+ */
+static void pgmR3StateVerifyCrc32ForPage(void const *pvPage, PPGMRAMRANGE pCur, PPGMLIVESAVERAMPAGE paLSPages, uint32_t iPage, const  char *pszWhere)
+{
+    if (paLSPages[iPage].u32Crc != UINT32_MAX)
+    {
+        uint32_t u32Crc = RTCrc32(pvPage, PAGE_SIZE);
+        Assert(   (   !PGM_PAGE_IS_ZERO(&pCur->aPages[iPage])
+                   && !PGM_PAGE_IS_BALLOONED(&pCur->aPages[iPage]))
+               || u32Crc == PGM_STATE_CRC32_ZERO_PAGE);
+        AssertMsg(paLSPages[iPage].u32Crc == u32Crc,
+                  ("%08x != %08x for %RGp %R[pgmpage] %s\n", paLSPages[iPage].u32Crc, u32Crc,
+                   pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pCur->aPages[iPage], pszWhere));
+    }
+}
+
+
+/**
+ * Verifies the CRC-32 for a RAM page.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pCur                The current RAM range.
+ * @param   paLSPages           The current array of live save page tracking
+ *                              structures.
+ * @param   iPage               The page index.
+ */
+static void pgmR3StateVerifyCrc32ForRamPage(PVM pVM, PPGMRAMRANGE pCur, PPGMLIVESAVERAMPAGE paLSPages, uint32_t iPage, const char *pszWhere)
+{
+    if (paLSPages[iPage].u32Crc != UINT32_MAX)
+    {
+        RTGCPHYS        GCPhys = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+        PGMPAGEMAPLOCK  PgMpLck;
+        void const     *pvPage;
+        int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, &pCur->aPages[iPage], GCPhys, &pvPage, &PgMpLck);
+        if (RT_SUCCESS(rc))
+        {
+            pgmR3StateVerifyCrc32ForPage(pvPage, pCur, paLSPages, iPage, pszWhere);
+            pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+        }
+    }
+}
+
+#endif /* PGMLIVESAVERAMPAGE_WITH_CRC32 */
+
+/**
+ * Scan for RAM page modifications and reprotect them.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   fFinalPass          Whether this is the final pass or not.
+ */
+static void pgmR3ScanRamPages(PVM pVM, bool fFinalPass)
+{
+    /*
+     * The RAM.
+     */
+    RTGCPHYS GCPhysCur = 0;
+    PPGMRAMRANGE pCur;
+    pgmLock(pVM);
+    do
+    {
+        uint32_t const  idRamRangesGen = pVM->pgm.s.idRamRangesGen;
+        for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
+        {
+            if (    pCur->GCPhysLast > GCPhysCur
+                && !PGM_RAM_RANGE_IS_AD_HOC(pCur))
+            {
+                PPGMLIVESAVERAMPAGE paLSPages = pCur->paLSPages;
+                uint32_t         cPages    = pCur->cb >> PAGE_SHIFT;
+                uint32_t         iPage     = GCPhysCur <= pCur->GCPhys ? 0 : (GCPhysCur - pCur->GCPhys) >> PAGE_SHIFT;
+                GCPhysCur = 0;
+                for (; iPage < cPages; iPage++)
+                {
+                    /* Do yield first. */
+                    if (   !fFinalPass
+#ifndef PGMLIVESAVERAMPAGE_WITH_CRC32
+                        && (iPage & 0x7ff) == 0x100
+#endif
+                        && PDMR3CritSectYield(pVM, &pVM->pgm.s.CritSectX)
+                        && pVM->pgm.s.idRamRangesGen != idRamRangesGen)
+                    {
+                        GCPhysCur = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+                        break; /* restart */
+                    }
+
+                    /* Skip already ignored pages. */
+                    if (paLSPages[iPage].fIgnore)
+                        continue;
+
+                    if (RT_LIKELY(PGM_PAGE_GET_TYPE(&pCur->aPages[iPage]) == PGMPAGETYPE_RAM))
+                    {
+                        /*
+                         * A RAM page.
+                         */
+                        switch (PGM_PAGE_GET_STATE(&pCur->aPages[iPage]))
+                        {
+                            case PGM_PAGE_STATE_ALLOCATED:
+                                /** @todo Optimize this: Don't always re-enable write
+                                 * monitoring if the page is known to be very busy. */
+                                if (PGM_PAGE_IS_WRITTEN_TO(&pCur->aPages[iPage]))
+                                {
+                                    AssertMsg(paLSPages[iPage].fWriteMonitored,
+                                              ("%RGp %R[pgmpage]\n", pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pCur->aPages[iPage]));
+                                    PGM_PAGE_CLEAR_WRITTEN_TO(pVM, &pCur->aPages[iPage]);
+                                    Assert(pVM->pgm.s.cWrittenToPages > 0);
+                                    pVM->pgm.s.cWrittenToPages--;
+                                }
+                                else
+                                {
+                                    AssertMsg(!paLSPages[iPage].fWriteMonitored,
+                                              ("%RGp %R[pgmpage]\n", pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pCur->aPages[iPage]));
+                                    pVM->pgm.s.LiveSave.Ram.cMonitoredPages++;
+                                }
+
+                                if (!paLSPages[iPage].fDirty)
+                                {
+                                    pVM->pgm.s.LiveSave.Ram.cReadyPages--;
+                                    if (paLSPages[iPage].fZero)
+                                        pVM->pgm.s.LiveSave.Ram.cZeroPages--;
+                                    pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
+                                    if (++paLSPages[iPage].cDirtied > PGMLIVSAVEPAGE_MAX_DIRTIED)
+                                        paLSPages[iPage].cDirtied = PGMLIVSAVEPAGE_MAX_DIRTIED;
+                                }
+
+                                pgmPhysPageWriteMonitor(pVM, &pCur->aPages[iPage],
+                                                        pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
+                                paLSPages[iPage].fWriteMonitored        = 1;
+                                paLSPages[iPage].fWriteMonitoredJustNow = 1;
+                                paLSPages[iPage].fDirty                 = 1;
+                                paLSPages[iPage].fZero                  = 0;
+                                paLSPages[iPage].fShared                = 0;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                paLSPages[iPage].u32Crc                 = UINT32_MAX; /* invalid */
+#endif
+                                break;
+
+                            case PGM_PAGE_STATE_WRITE_MONITORED:
+                                Assert(paLSPages[iPage].fWriteMonitored);
+                                if (PGM_PAGE_GET_WRITE_LOCKS(&pCur->aPages[iPage]) == 0)
+                                {
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                    if (paLSPages[iPage].fWriteMonitoredJustNow)
+                                        pgmR3StateCalcCrc32ForRamPage(pVM, pCur, paLSPages, iPage);
+                                    else
+                                        pgmR3StateVerifyCrc32ForRamPage(pVM, pCur, paLSPages, iPage, "scan");
+#endif
+                                    paLSPages[iPage].fWriteMonitoredJustNow = 0;
+                                }
+                                else
+                                {
+                                    paLSPages[iPage].fWriteMonitoredJustNow = 1;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                    paLSPages[iPage].u32Crc                 = UINT32_MAX; /* invalid */
+#endif
+                                    if (!paLSPages[iPage].fDirty)
+                                    {
+                                        pVM->pgm.s.LiveSave.Ram.cReadyPages--;
+                                        pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
+                                        if (++paLSPages[iPage].cDirtied > PGMLIVSAVEPAGE_MAX_DIRTIED)
+                                            paLSPages[iPage].cDirtied = PGMLIVSAVEPAGE_MAX_DIRTIED;
+                                    }
+                                }
+                                break;
+
+                            case PGM_PAGE_STATE_ZERO:
+                            case PGM_PAGE_STATE_BALLOONED:
+                                if (!paLSPages[iPage].fZero)
+                                {
+                                    if (!paLSPages[iPage].fDirty)
+                                    {
+                                        paLSPages[iPage].fDirty = 1;
+                                        pVM->pgm.s.LiveSave.Ram.cReadyPages--;
+                                        pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
+                                    }
+                                    paLSPages[iPage].fZero = 1;
+                                    paLSPages[iPage].fShared = 0;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                    paLSPages[iPage].u32Crc = PGM_STATE_CRC32_ZERO_PAGE;
+#endif
+                                }
+                                break;
+
+                            case PGM_PAGE_STATE_SHARED:
+                                if (!paLSPages[iPage].fShared)
+                                {
+                                    if (!paLSPages[iPage].fDirty)
+                                    {
+                                        paLSPages[iPage].fDirty = 1;
+                                        pVM->pgm.s.LiveSave.Ram.cReadyPages--;
+                                        if (paLSPages[iPage].fZero)
+                                            pVM->pgm.s.LiveSave.Ram.cZeroPages--;
+                                        pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
+                                    }
+                                    paLSPages[iPage].fZero = 0;
+                                    paLSPages[iPage].fShared = 1;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                    pgmR3StateCalcCrc32ForRamPage(pVM, pCur, paLSPages, iPage);
+#endif
+                                }
+                                break;
+                        }
+                    }
+                    else
+                    {
+                        /*
+                         * All other types => Ignore the page.
+                         */
+                        Assert(!paLSPages[iPage].fIgnore); /* skipped before switch */
+                        paLSPages[iPage].fIgnore = 1;
+                        if (paLSPages[iPage].fWriteMonitored)
+                        {
+                            /** @todo this doesn't hold water when we start monitoring MMIO2 and ROM shadow
+                             *        pages! */
+                            if (RT_UNLIKELY(PGM_PAGE_GET_STATE(&pCur->aPages[iPage]) == PGM_PAGE_STATE_WRITE_MONITORED))
+                            {
+                                AssertMsgFailed(("%R[pgmpage]", &pCur->aPages[iPage])); /* shouldn't happen. */
+                                PGM_PAGE_SET_STATE(pVM, &pCur->aPages[iPage], PGM_PAGE_STATE_ALLOCATED);
+                                Assert(pVM->pgm.s.cMonitoredPages > 0);
+                                pVM->pgm.s.cMonitoredPages--;
+                            }
+                            if (PGM_PAGE_IS_WRITTEN_TO(&pCur->aPages[iPage]))
+                            {
+                                PGM_PAGE_CLEAR_WRITTEN_TO(pVM, &pCur->aPages[iPage]);
+                                Assert(pVM->pgm.s.cWrittenToPages > 0);
+                                pVM->pgm.s.cWrittenToPages--;
+                            }
+                            pVM->pgm.s.LiveSave.Ram.cMonitoredPages--;
+                        }
+
+                        /** @todo the counting doesn't quite work out here. fix later? */
+                        if (paLSPages[iPage].fDirty)
+                            pVM->pgm.s.LiveSave.Ram.cDirtyPages--;
+                        else
+                        {
+                            pVM->pgm.s.LiveSave.Ram.cReadyPages--;
+                            if (paLSPages[iPage].fZero)
+                                pVM->pgm.s.LiveSave.Ram.cZeroPages--;
+                        }
+                        pVM->pgm.s.LiveSave.cIgnoredPages++;
+                    }
+                } /* for each page in range */
+
+                if (GCPhysCur != 0)
+                    break; /* Yield + ramrange change */
+                GCPhysCur = pCur->GCPhysLast;
+            }
+        } /* for each range */
+    } while (pCur);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Save quiescent RAM pages.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The SSM handle.
+ * @param   fLiveSave           Whether it's a live save or not.
+ * @param   uPass               The pass number.
+ */
+static int pgmR3SaveRamPages(PVM pVM, PSSMHANDLE pSSM, bool fLiveSave, uint32_t uPass)
+{
+    NOREF(fLiveSave);
+
+    /*
+     * The RAM.
+     */
+    RTGCPHYS GCPhysLast = NIL_RTGCPHYS;
+    RTGCPHYS GCPhysCur = 0;
+    PPGMRAMRANGE pCur;
+
+    pgmLock(pVM);
+    do
+    {
+        uint32_t const  idRamRangesGen = pVM->pgm.s.idRamRangesGen;
+        for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
+        {
+            if (   pCur->GCPhysLast > GCPhysCur
+                && !PGM_RAM_RANGE_IS_AD_HOC(pCur))
+            {
+                PPGMLIVESAVERAMPAGE paLSPages = pCur->paLSPages;
+                uint32_t         cPages    = pCur->cb >> PAGE_SHIFT;
+                uint32_t         iPage     = GCPhysCur <= pCur->GCPhys ? 0 : (GCPhysCur - pCur->GCPhys) >> PAGE_SHIFT;
+                GCPhysCur = 0;
+                for (; iPage < cPages; iPage++)
+                {
+                    /* Do yield first. */
+                    if (   uPass != SSM_PASS_FINAL
+                        && (iPage & 0x7ff) == 0x100
+                        && PDMR3CritSectYield(pVM, &pVM->pgm.s.CritSectX)
+                        && pVM->pgm.s.idRamRangesGen != idRamRangesGen)
+                    {
+                        GCPhysCur = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+                        break; /* restart */
+                    }
+
+                    PPGMPAGE pCurPage = &pCur->aPages[iPage];
+
+                    /*
+                     * Only save pages that haven't changed since last scan and are dirty.
+                     */
+                    if (    uPass != SSM_PASS_FINAL
+                        &&  paLSPages)
+                    {
+                        if (!paLSPages[iPage].fDirty)
+                            continue;
+                        if (paLSPages[iPage].fWriteMonitoredJustNow)
+                            continue;
+                        if (paLSPages[iPage].fIgnore)
+                            continue;
+                        if (PGM_PAGE_GET_TYPE(pCurPage) != PGMPAGETYPE_RAM) /* in case of recent remappings */
+                            continue;
+                        if (    PGM_PAGE_GET_STATE(pCurPage)
+                            !=  (  paLSPages[iPage].fZero
+                                 ? PGM_PAGE_STATE_ZERO
+                                 : paLSPages[iPage].fShared
+                                 ? PGM_PAGE_STATE_SHARED
+                                 : PGM_PAGE_STATE_WRITE_MONITORED))
+                            continue;
+                        if (PGM_PAGE_GET_WRITE_LOCKS(&pCur->aPages[iPage]) > 0)
+                            continue;
+                    }
+                    else
+                    {
+                        if (   paLSPages
+                            && !paLSPages[iPage].fDirty
+                            && !paLSPages[iPage].fIgnore)
+                        {
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                            if (PGM_PAGE_GET_TYPE(pCurPage) != PGMPAGETYPE_RAM)
+                                pgmR3StateVerifyCrc32ForRamPage(pVM, pCur, paLSPages, iPage, "save#1");
+#endif
+                            continue;
+                        }
+                        if (PGM_PAGE_GET_TYPE(pCurPage) != PGMPAGETYPE_RAM)
+                            continue;
+                    }
+
+                    /*
+                     * Do the saving outside the PGM critsect since SSM may block on I/O.
+                     */
+                    int         rc;
+                    RTGCPHYS    GCPhys = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+                    bool        fZero  = PGM_PAGE_IS_ZERO(pCurPage);
+                    bool        fBallooned = PGM_PAGE_IS_BALLOONED(pCurPage);
+                    bool        fSkipped = false;
+
+                    if (!fZero && !fBallooned)
+                    {
+                        /*
+                         * Copy the page and then save it outside the lock (since any
+                         * SSM call may block).
+                         */
+                        uint8_t         abPage[PAGE_SIZE];
+                        PGMPAGEMAPLOCK  PgMpLck;
+                        void const     *pvPage;
+                        rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pCurPage, GCPhys, &pvPage, &PgMpLck);
+                        if (RT_SUCCESS(rc))
+                        {
+                            memcpy(abPage, pvPage, PAGE_SIZE);
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                            if (paLSPages)
+                                pgmR3StateVerifyCrc32ForPage(abPage, pCur, paLSPages, iPage, "save#3");
+#endif
+                            pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+                        }
+                        pgmUnlock(pVM);
+                        AssertLogRelMsgRCReturn(rc, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys), rc);
+
+                        /* Try save some memory when restoring. */
+                        if (!ASMMemIsZeroPage(pvPage))
+                        {
+                            if (GCPhys == GCPhysLast + PAGE_SIZE)
+                                SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_RAW);
+                            else
+                            {
+                                SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_RAW | PGM_STATE_REC_FLAG_ADDR);
+                                SSMR3PutGCPhys(pSSM, GCPhys);
+                            }
+                            rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
+                        }
+                        else
+                        {
+                            if (GCPhys == GCPhysLast + PAGE_SIZE)
+                                rc = SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_ZERO);
+                            else
+                            {
+                                SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_ZERO | PGM_STATE_REC_FLAG_ADDR);
+                                rc = SSMR3PutGCPhys(pSSM, GCPhys);
+                            }
+                        }
+                    }
+                    else
+                    {
+                        /*
+                         * Dirty zero or ballooned page.
+                         */
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                        if (paLSPages)
+                            pgmR3StateVerifyCrc32ForRamPage(pVM, pCur, paLSPages, iPage, "save#2");
+#endif
+                        pgmUnlock(pVM);
+
+                        uint8_t u8RecType = fBallooned ? PGM_STATE_REC_RAM_BALLOONED : PGM_STATE_REC_RAM_ZERO;
+                        if (GCPhys == GCPhysLast + PAGE_SIZE)
+                            rc = SSMR3PutU8(pSSM, u8RecType);
+                        else
+                        {
+                            SSMR3PutU8(pSSM, u8RecType | PGM_STATE_REC_FLAG_ADDR);
+                            rc = SSMR3PutGCPhys(pSSM, GCPhys);
+                        }
+                    }
+                    if (RT_FAILURE(rc))
+                        return rc;
+
+                    pgmLock(pVM);
+                    if (!fSkipped)
+                        GCPhysLast = GCPhys;
+                    if (paLSPages)
+                    {
+                        paLSPages[iPage].fDirty = 0;
+                        pVM->pgm.s.LiveSave.Ram.cReadyPages++;
+                        if (fZero)
+                            pVM->pgm.s.LiveSave.Ram.cZeroPages++;
+                        pVM->pgm.s.LiveSave.Ram.cDirtyPages--;
+                        pVM->pgm.s.LiveSave.cSavedPages++;
+                    }
+                    if (idRamRangesGen != pVM->pgm.s.idRamRangesGen)
+                    {
+                        GCPhysCur = GCPhys | PAGE_OFFSET_MASK;
+                        break; /* restart */
+                    }
+
+                } /* for each page in range */
+
+                if (GCPhysCur != 0)
+                    break; /* Yield + ramrange change */
+                GCPhysCur = pCur->GCPhysLast;
+            }
+        } /* for each range */
+    } while (pCur);
+
+    pgmUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Cleans up RAM pages after a live save.
+ *
+ * @param   pVM                 The cross context VM structure.
+ */
+static void pgmR3DoneRamPages(PVM pVM)
+{
+    /*
+     * Free the tracking arrays and disable write monitoring.
+     *
+     * Play nice with the PGM lock in case we're called while the VM is still
+     * running.  This means we have to delay the freeing since we wish to use
+     * paLSPages as an indicator of which RAM ranges which we need to scan for
+     * write monitored pages.
+     */
+    void *pvToFree = NULL;
+    PPGMRAMRANGE pCur;
+    uint32_t cMonitoredPages = 0;
+    pgmLock(pVM);
+    do
+    {
+        for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
+        {
+            if (pCur->paLSPages)
+            {
+                if (pvToFree)
+                {
+                    uint32_t idRamRangesGen = pVM->pgm.s.idRamRangesGen;
+                    pgmUnlock(pVM);
+                    MMR3HeapFree(pvToFree);
+                    pvToFree = NULL;
+                    pgmLock(pVM);
+                    if (idRamRangesGen != pVM->pgm.s.idRamRangesGen)
+                        break;          /* start over again. */
+                }
+
+                pvToFree = pCur->paLSPages;
+                pCur->paLSPages = NULL;
+
+                uint32_t iPage = pCur->cb >> PAGE_SHIFT;
+                while (iPage--)
+                {
+                    PPGMPAGE pPage = &pCur->aPages[iPage];
+                    PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
+                    if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
+                    {
+                        PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
+                        cMonitoredPages++;
+                    }
+                }
+            }
+        }
+    } while (pCur);
+
+    Assert(pVM->pgm.s.cMonitoredPages >= cMonitoredPages);
+    if (pVM->pgm.s.cMonitoredPages < cMonitoredPages)
+        pVM->pgm.s.cMonitoredPages = 0;
+    else
+        pVM->pgm.s.cMonitoredPages -= cMonitoredPages;
+
+    pgmUnlock(pVM);
+
+    MMR3HeapFree(pvToFree);
+    pvToFree = NULL;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTLIVEEXEC}
+ */
+static DECLCALLBACK(int) pgmR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    int rc;
+
+    /*
+     * Save the MMIO2 and ROM range IDs in pass 0.
+     */
+    if (uPass == 0)
+    {
+        rc = pgmR3SaveRamConfig(pVM, pSSM);
+        if (RT_FAILURE(rc))
+            return rc;
+        rc = pgmR3SaveRomRanges(pVM, pSSM);
+        if (RT_FAILURE(rc))
+            return rc;
+        rc = pgmR3SaveMmio2Ranges(pVM, pSSM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+    /*
+     * Reset the page-per-second estimate to avoid inflation by the initial
+     * load of zero pages.  pgmR3LiveVote ASSUMES this is done at pass 7.
+     */
+    else if (uPass == 7)
+    {
+        pVM->pgm.s.LiveSave.cSavedPages  = 0;
+        pVM->pgm.s.LiveSave.uSaveStartNS = RTTimeNanoTS();
+    }
+
+    /*
+     * Do the scanning.
+     */
+    pgmR3ScanRomPages(pVM);
+    pgmR3ScanMmio2Pages(pVM, uPass);
+    pgmR3ScanRamPages(pVM, false /*fFinalPass*/);
+    pgmR3PoolClearAll(pVM, true /*fFlushRemTlb*/); /** @todo this could perhaps be optimized a bit. */
+
+    /*
+     * Save the pages.
+     */
+    if (uPass == 0)
+        rc = pgmR3SaveRomVirginPages(  pVM, pSSM, true /*fLiveSave*/);
+    else
+        rc = VINF_SUCCESS;
+    if (RT_SUCCESS(rc))
+        rc = pgmR3SaveShadowedRomPages(pVM, pSSM, true /*fLiveSave*/, false /*fFinalPass*/);
+    if (RT_SUCCESS(rc))
+        rc = pgmR3SaveMmio2Pages(      pVM, pSSM, true /*fLiveSave*/, uPass);
+    if (RT_SUCCESS(rc))
+        rc = pgmR3SaveRamPages(        pVM, pSSM, true /*fLiveSave*/, uPass);
+    SSMR3PutU8(pSSM, PGM_STATE_REC_END);    /* (Ignore the rc, SSM takes care of it.) */
+
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTLIVEVOTE}
+ */
+static DECLCALLBACK(int)  pgmR3LiveVote(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    /*
+     * Update and calculate parameters used in the decision making.
+     */
+    const uint32_t cHistoryEntries = RT_ELEMENTS(pVM->pgm.s.LiveSave.acDirtyPagesHistory);
+
+    /* update history. */
+    pgmLock(pVM);
+    uint32_t const cWrittenToPages = pVM->pgm.s.cWrittenToPages;
+    pgmUnlock(pVM);
+    uint32_t const cDirtyNow = pVM->pgm.s.LiveSave.Rom.cDirtyPages
+                             + pVM->pgm.s.LiveSave.Mmio2.cDirtyPages
+                             + pVM->pgm.s.LiveSave.Ram.cDirtyPages
+                             + cWrittenToPages;
+    uint32_t i = pVM->pgm.s.LiveSave.iDirtyPagesHistory;
+    pVM->pgm.s.LiveSave.acDirtyPagesHistory[i] = cDirtyNow;
+    pVM->pgm.s.LiveSave.iDirtyPagesHistory = (i + 1) % cHistoryEntries;
+
+    /* calc shortterm average (4 passes). */
+    AssertCompile(RT_ELEMENTS(pVM->pgm.s.LiveSave.acDirtyPagesHistory) > 4);
+    uint64_t cTotal = pVM->pgm.s.LiveSave.acDirtyPagesHistory[i];
+    cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[(i + cHistoryEntries - 1) % cHistoryEntries];
+    cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[(i + cHistoryEntries - 2) % cHistoryEntries];
+    cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[(i + cHistoryEntries - 3) % cHistoryEntries];
+    uint32_t const cDirtyPagesShort = cTotal / 4;
+    pVM->pgm.s.LiveSave.cDirtyPagesShort = cDirtyPagesShort;
+
+    /* calc longterm average. */
+    cTotal = 0;
+    if (uPass < cHistoryEntries)
+        for (i = 0; i < cHistoryEntries && i <= uPass; i++)
+              cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[i];
+    else
+        for (i = 0; i < cHistoryEntries; i++)
+            cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[i];
+    uint32_t const cDirtyPagesLong = cTotal / cHistoryEntries;
+    pVM->pgm.s.LiveSave.cDirtyPagesLong = cDirtyPagesLong;
+
+    /* estimate the speed */
+    uint64_t cNsElapsed = RTTimeNanoTS() - pVM->pgm.s.LiveSave.uSaveStartNS;
+    uint32_t cPagesPerSecond = (uint32_t)(   pVM->pgm.s.LiveSave.cSavedPages
+                                          / ((long double)cNsElapsed / 1000000000.0) );
+    pVM->pgm.s.LiveSave.cPagesPerSecond = cPagesPerSecond;
+
+    /*
+     * Try make a decision.
+     */
+    if (    cDirtyPagesShort <= cDirtyPagesLong
+        &&  (   cDirtyNow    <= cDirtyPagesShort
+             || cDirtyNow - cDirtyPagesShort < RT_MIN(cDirtyPagesShort / 8, 16)
+            )
+       )
+    {
+        if (uPass > 10)
+        {
+            uint32_t cMsLeftShort   = (uint32_t)(cDirtyPagesShort / (long double)cPagesPerSecond * 1000.0);
+            uint32_t cMsLeftLong    = (uint32_t)(cDirtyPagesLong  / (long double)cPagesPerSecond * 1000.0);
+            uint32_t cMsMaxDowntime = SSMR3HandleMaxDowntime(pSSM);
+            if (cMsMaxDowntime < 32)
+                cMsMaxDowntime = 32;
+            if (   (   cMsLeftLong  <= cMsMaxDowntime
+                    && cMsLeftShort <  cMsMaxDowntime)
+                || cMsLeftShort < cMsMaxDowntime / 2
+               )
+            {
+                Log(("pgmR3LiveVote: VINF_SUCCESS - pass=%d cDirtyPagesShort=%u|%ums cDirtyPagesLong=%u|%ums cMsMaxDowntime=%u\n",
+                     uPass, cDirtyPagesShort, cMsLeftShort, cDirtyPagesLong, cMsLeftLong, cMsMaxDowntime));
+                return VINF_SUCCESS;
+            }
+        }
+        else
+        {
+            if (   (   cDirtyPagesShort <= 128
+                    && cDirtyPagesLong  <= 1024)
+                || cDirtyPagesLong <= 256
+               )
+            {
+                Log(("pgmR3LiveVote: VINF_SUCCESS - pass=%d cDirtyPagesShort=%u cDirtyPagesLong=%u\n", uPass, cDirtyPagesShort, cDirtyPagesLong));
+                return VINF_SUCCESS;
+            }
+        }
+    }
+
+    /*
+     * Come up with a completion percentage.  Currently this is a simple
+     * dirty page (long term) vs. total pages ratio + some pass trickery.
+     */
+    unsigned uPctDirty = (unsigned)(  (long double)cDirtyPagesLong
+                                    / (pVM->pgm.s.cAllPages - pVM->pgm.s.LiveSave.cIgnoredPages - pVM->pgm.s.cZeroPages) );
+    if (uPctDirty <= 100)
+        SSMR3HandleReportLivePercent(pSSM, RT_MIN(100 - uPctDirty, uPass * 2));
+    else
+        AssertMsgFailed(("uPctDirty=%u cDirtyPagesLong=%#x cAllPages=%#x cIgnoredPages=%#x cZeroPages=%#x\n",
+                         uPctDirty, cDirtyPagesLong, pVM->pgm.s.cAllPages, pVM->pgm.s.LiveSave.cIgnoredPages, pVM->pgm.s.cZeroPages));
+
+    return VINF_SSM_VOTE_FOR_ANOTHER_PASS;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTLIVEPREP}
+ *
+ * This will attempt to allocate and initialize the tracking structures.  It
+ * will also prepare for write monitoring of pages and initialize PGM::LiveSave.
+ * pgmR3SaveDone will do the cleanups.
+ */
+static DECLCALLBACK(int) pgmR3LivePrep(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Indicate that we will be using the write monitoring.
+     */
+    pgmLock(pVM);
+    /** @todo find a way of mediating this when more users are added. */
+    if (pVM->pgm.s.fPhysWriteMonitoringEngaged)
+    {
+        pgmUnlock(pVM);
+        AssertLogRelFailedReturn(VERR_PGM_WRITE_MONITOR_ENGAGED);
+    }
+    pVM->pgm.s.fPhysWriteMonitoringEngaged = true;
+    pgmUnlock(pVM);
+
+    /*
+     * Initialize the statistics.
+     */
+    pVM->pgm.s.LiveSave.Rom.cReadyPages   = 0;
+    pVM->pgm.s.LiveSave.Rom.cDirtyPages   = 0;
+    pVM->pgm.s.LiveSave.Mmio2.cReadyPages = 0;
+    pVM->pgm.s.LiveSave.Mmio2.cDirtyPages = 0;
+    pVM->pgm.s.LiveSave.Ram.cReadyPages   = 0;
+    pVM->pgm.s.LiveSave.Ram.cDirtyPages   = 0;
+    pVM->pgm.s.LiveSave.cIgnoredPages     = 0;
+    pVM->pgm.s.LiveSave.fActive           = true;
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.LiveSave.acDirtyPagesHistory); i++)
+        pVM->pgm.s.LiveSave.acDirtyPagesHistory[i] = UINT32_MAX / 2;
+    pVM->pgm.s.LiveSave.iDirtyPagesHistory = 0;
+    pVM->pgm.s.LiveSave.cSavedPages       = 0;
+    pVM->pgm.s.LiveSave.uSaveStartNS      = RTTimeNanoTS();
+    pVM->pgm.s.LiveSave.cPagesPerSecond   = 8192;
+
+    /*
+     * Per page type.
+     */
+    int rc = pgmR3PrepRomPages(pVM);
+    if (RT_SUCCESS(rc))
+        rc = pgmR3PrepMmio2Pages(pVM);
+    if (RT_SUCCESS(rc))
+        rc = pgmR3PrepRamPages(pVM);
+
+    NOREF(pSSM);
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTSAVEEXEC}
+ */
+static DECLCALLBACK(int) pgmR3SaveExec(PVM pVM, PSSMHANDLE pSSM)
+{
+    int     rc   = VINF_SUCCESS;
+    PPGM    pPGM = &pVM->pgm.s;
+
+    /*
+     * Lock PGM and set the no-more-writes indicator.
+     */
+    pgmLock(pVM);
+    pVM->pgm.s.fNoMorePhysWrites = true;
+
+    /*
+     * Save basic data (required / unaffected by relocation).
+     */
+    bool const fMappingsFixed  = pVM->pgm.s.fMappingsFixed;
+    pVM->pgm.s.fMappingsFixed |= pVM->pgm.s.fMappingsFixedRestored;
+    SSMR3PutStruct(pSSM, pPGM, &s_aPGMFields[0]);
+    pVM->pgm.s.fMappingsFixed  = fMappingsFixed;
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        rc = SSMR3PutStruct(pSSM, &pVM->apCpusR3[idCpu]->pgm.s, &s_aPGMCpuFields[0]);
+
+    /*
+     * Save the (remainder of the) memory.
+     */
+    if (RT_SUCCESS(rc))
+    {
+        if (pVM->pgm.s.LiveSave.fActive)
+        {
+            pgmR3ScanRomPages(pVM);
+            pgmR3ScanMmio2Pages(pVM, SSM_PASS_FINAL);
+            pgmR3ScanRamPages(pVM, true /*fFinalPass*/);
+
+            rc = pgmR3SaveShadowedRomPages(    pVM, pSSM, true /*fLiveSave*/, true /*fFinalPass*/);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveMmio2Pages(      pVM, pSSM, true /*fLiveSave*/, SSM_PASS_FINAL);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveRamPages(        pVM, pSSM, true /*fLiveSave*/, SSM_PASS_FINAL);
+        }
+        else
+        {
+            rc = pgmR3SaveRamConfig(pVM, pSSM);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveRomRanges(pVM, pSSM);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveMmio2Ranges(pVM, pSSM);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveRomVirginPages(  pVM, pSSM, false /*fLiveSave*/);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveShadowedRomPages(pVM, pSSM, false /*fLiveSave*/, true /*fFinalPass*/);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveMmio2Pages(      pVM, pSSM, false /*fLiveSave*/, SSM_PASS_FINAL);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveRamPages(        pVM, pSSM, false /*fLiveSave*/, SSM_PASS_FINAL);
+        }
+        SSMR3PutU8(pSSM, PGM_STATE_REC_END);    /* (Ignore the rc, SSM takes of it.) */
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTSAVEDONE}
+ */
+static DECLCALLBACK(int) pgmR3SaveDone(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Do per page type cleanups first.
+     */
+    if (pVM->pgm.s.LiveSave.fActive)
+    {
+        pgmR3DoneRomPages(pVM);
+        pgmR3DoneMmio2Pages(pVM);
+        pgmR3DoneRamPages(pVM);
+    }
+
+    /*
+     * Clear the live save indicator and disengage write monitoring.
+     */
+    pgmLock(pVM);
+    pVM->pgm.s.LiveSave.fActive = false;
+    /** @todo this is blindly assuming that we're the only user of write
+     *        monitoring. Fix this when more users are added. */
+    pVM->pgm.s.fPhysWriteMonitoringEngaged = false;
+    pgmUnlock(pVM);
+
+    NOREF(pSSM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTLOADPREP}
+ */
+static DECLCALLBACK(int) pgmR3LoadPrep(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Call the reset function to make sure all the memory is cleared.
+     */
+    PGMR3Reset(pVM);
+    pVM->pgm.s.LiveSave.fActive = false;
+    NOREF(pSSM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Load an ignored page.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ */
+static int pgmR3LoadPageToDevNullOld(PSSMHANDLE pSSM)
+{
+    uint8_t abPage[PAGE_SIZE];
+    return SSMR3GetMem(pSSM, &abPage[0], sizeof(abPage));
+}
+
+
+/**
+ * Compares a page with an old save type value.
+ *
+ * @returns true if equal, false if not.
+ * @param   pPage           The page to compare.
+ * @param   uOldType        The old type value from the saved state.
+ */
+DECLINLINE(bool) pgmR3CompareNewAndOldPageTypes(PPGMPAGE pPage, uint8_t uOldType)
+{
+    uint8_t uOldPageType;
+    switch (PGM_PAGE_GET_TYPE(pPage))
+    {
+        case PGMPAGETYPE_INVALID:               uOldPageType = PGMPAGETYPE_OLD_INVALID; break;
+        case PGMPAGETYPE_RAM:                   uOldPageType = PGMPAGETYPE_OLD_RAM; break;
+        case PGMPAGETYPE_MMIO2:                 uOldPageType = PGMPAGETYPE_OLD_MMIO2; break;
+        case PGMPAGETYPE_MMIO2_ALIAS_MMIO:      uOldPageType = PGMPAGETYPE_OLD_MMIO2_ALIAS_MMIO; break;
+        case PGMPAGETYPE_ROM_SHADOW:            uOldPageType = PGMPAGETYPE_OLD_ROM_SHADOW; break;
+        case PGMPAGETYPE_ROM:                   uOldPageType = PGMPAGETYPE_OLD_ROM; break;
+        case PGMPAGETYPE_SPECIAL_ALIAS_MMIO:    RT_FALL_THRU();
+        case PGMPAGETYPE_MMIO:                  uOldPageType = PGMPAGETYPE_OLD_MMIO; break;
+        default:
+            AssertFailed();
+            uOldPageType = PGMPAGETYPE_OLD_INVALID;
+            break;
+    }
+    return uOldPageType == uOldType;
+}
+
+
+/**
+ * Loads a page without any bits in the saved state, i.e. making sure it's
+ * really zero.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   uOldType        The page type or PGMPAGETYPE_OLD_INVALID (old saved
+ *                          state).
+ * @param   pPage           The guest page tracking structure.
+ * @param   GCPhys          The page address.
+ * @param   pRam            The ram range (logging).
+ */
+static int pgmR3LoadPageZeroOld(PVM pVM, uint8_t uOldType, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
+{
+    if (   uOldType != PGMPAGETYPE_OLD_INVALID
+        && !pgmR3CompareNewAndOldPageTypes(pPage, uOldType))
+        return VERR_SSM_UNEXPECTED_DATA;
+
+    /* I think this should be sufficient. */
+    if (    !PGM_PAGE_IS_ZERO(pPage)
+        &&  !PGM_PAGE_IS_BALLOONED(pPage))
+        return VERR_SSM_UNEXPECTED_DATA;
+
+    NOREF(pVM);
+    NOREF(GCPhys);
+    NOREF(pRam);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads a page from the saved state.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The SSM handle.
+ * @param   uOldType        The page type or PGMPAGETYPE_OLD_INVALID (old saved
+ *                          state).
+ * @param   pPage           The guest page tracking structure.
+ * @param   GCPhys          The page address.
+ * @param   pRam            The ram range (logging).
+ */
+static int pgmR3LoadPageBitsOld(PVM pVM, PSSMHANDLE pSSM, uint8_t uOldType, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
+{
+    /*
+     * Match up the type, dealing with MMIO2 aliases (dropped).
+     */
+    AssertLogRelMsgReturn(   uOldType == PGMPAGETYPE_INVALID
+                          || pgmR3CompareNewAndOldPageTypes(pPage, uOldType)
+                          /* kudge for the expanded PXE bios (r67885) - @bugref{5687}: */
+                          || (   uOldType == PGMPAGETYPE_OLD_RAM
+                              && GCPhys >= 0xed000
+                              && GCPhys <= 0xeffff
+                              && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_ROM)
+                          ,
+                          ("pPage=%R[pgmpage] GCPhys=%#x %s\n", pPage, GCPhys, pRam->pszDesc),
+                          VERR_SSM_UNEXPECTED_DATA);
+
+    /*
+     * Load the page.
+     */
+    PGMPAGEMAPLOCK PgMpLck;
+    void          *pvPage;
+    int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvPage, &PgMpLck);
+    if (RT_SUCCESS(rc))
+    {
+        rc = SSMR3GetMem(pSSM, pvPage, PAGE_SIZE);
+        pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Loads a page (counter part to pgmR3SavePage).
+ *
+ * @returns VBox status code, fully bitched errors.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The SSM handle.
+ * @param   uOldType        The page type.
+ * @param   pPage           The page.
+ * @param   GCPhys          The page address.
+ * @param   pRam            The RAM range (for error messages).
+ */
+static int pgmR3LoadPageOld(PVM pVM, PSSMHANDLE pSSM, uint8_t uOldType, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
+{
+    uint8_t uState;
+    int rc = SSMR3GetU8(pSSM, &uState);
+    AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] GCPhys=%#x %s rc=%Rrc\n", pPage, GCPhys, pRam->pszDesc, rc), rc);
+    if (uState == 0 /* zero */)
+        rc = pgmR3LoadPageZeroOld(pVM, uOldType, pPage, GCPhys, pRam);
+    else if (uState == 1)
+        rc = pgmR3LoadPageBitsOld(pVM, pSSM, uOldType, pPage, GCPhys, pRam);
+    else
+        rc = VERR_PGM_INVALID_SAVED_PAGE_STATE;
+    AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] uState=%d uOldType=%d GCPhys=%RGp %s rc=%Rrc\n",
+                                 pPage, uState, uOldType, GCPhys, pRam->pszDesc, rc),
+                            rc);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads a shadowed ROM page.
+ *
+ * @returns VBox status code, errors are fully bitched.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The saved state handle.
+ * @param   pPage           The page.
+ * @param   GCPhys          The page address.
+ * @param   pRam            The RAM range (for error messages).
+ */
+static int pgmR3LoadShadowedRomPageOld(PVM pVM, PSSMHANDLE pSSM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
+{
+    /*
+     * Load and set the protection first, then load the two pages, the first
+     * one is the active the other is the passive.
+     */
+    PPGMROMPAGE pRomPage = pgmR3GetRomPage(pVM, GCPhys);
+    AssertLogRelMsgReturn(pRomPage, ("GCPhys=%RGp %s\n", GCPhys, pRam->pszDesc), VERR_PGM_SAVED_ROM_PAGE_NOT_FOUND);
+
+    uint8_t uProt;
+    int rc = SSMR3GetU8(pSSM, &uProt);
+    AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] GCPhys=%#x %s\n", pPage, GCPhys, pRam->pszDesc), rc);
+    PGMROMPROT  enmProt = (PGMROMPROT)uProt;
+    AssertLogRelMsgReturn(    enmProt >= PGMROMPROT_INVALID
+                          &&  enmProt <  PGMROMPROT_END,
+                          ("enmProt=%d pPage=%R[pgmpage] GCPhys=%#x %s\n", enmProt, pPage, GCPhys, pRam->pszDesc),
+                          VERR_SSM_UNEXPECTED_DATA);
+
+    if (pRomPage->enmProt != enmProt)
+    {
+        rc = PGMR3PhysRomProtect(pVM, GCPhys, PAGE_SIZE, enmProt);
+        AssertLogRelRCReturn(rc, rc);
+        AssertLogRelReturn(pRomPage->enmProt == enmProt, VERR_PGM_SAVED_ROM_PAGE_PROT);
+    }
+
+    PPGMPAGE pPageActive  = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Virgin      : &pRomPage->Shadow;
+    PPGMPAGE pPagePassive = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Shadow      : &pRomPage->Virgin;
+    uint8_t  u8ActiveType = PGMROMPROT_IS_ROM(enmProt) ? PGMPAGETYPE_ROM        : PGMPAGETYPE_ROM_SHADOW;
+    uint8_t  u8PassiveType= PGMROMPROT_IS_ROM(enmProt) ? PGMPAGETYPE_ROM_SHADOW : PGMPAGETYPE_ROM;
+
+    /** @todo this isn't entirely correct as long as pgmPhysGCPhys2CCPtrInternal is
+     *        used down the line (will the 2nd page will be written to the first
+     *        one because of a false TLB hit since the TLB is using GCPhys and
+     *        doesn't check the HCPhys of the desired page). */
+    rc = pgmR3LoadPageOld(pVM, pSSM, u8ActiveType, pPage, GCPhys, pRam);
+    if (RT_SUCCESS(rc))
+    {
+        *pPageActive = *pPage;
+        rc = pgmR3LoadPageOld(pVM, pSSM, u8PassiveType, pPagePassive, GCPhys, pRam);
+    }
+    return rc;
+}
+
+/**
+ * Ram range flags and bits for older versions of the saved state.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pSSM        The SSM handle.
+ * @param   uVersion    The saved state version.
+ */
+static int pgmR3LoadMemoryOld(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion)
+{
+    PPGM pPGM = &pVM->pgm.s;
+
+    /*
+     * Ram range flags and bits.
+     */
+    uint32_t i = 0;
+    for (PPGMRAMRANGE pRam = pPGM->pRamRangesXR3; ; pRam = pRam->pNextR3, i++)
+    {
+        /* Check the sequence number / separator. */
+        uint32_t u32Sep;
+        int rc = SSMR3GetU32(pSSM, &u32Sep);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u32Sep == ~0U)
+            break;
+        if (u32Sep != i)
+        {
+            AssertMsgFailed(("u32Sep=%#x (last)\n", u32Sep));
+            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+        }
+        AssertLogRelReturn(pRam, VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        /* Get the range details. */
+        RTGCPHYS GCPhys;
+        SSMR3GetGCPhys(pSSM, &GCPhys);
+        RTGCPHYS GCPhysLast;
+        SSMR3GetGCPhys(pSSM, &GCPhysLast);
+        RTGCPHYS cb;
+        SSMR3GetGCPhys(pSSM, &cb);
+        uint8_t     fHaveBits;
+        rc = SSMR3GetU8(pSSM, &fHaveBits);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (fHaveBits & ~1)
+        {
+            AssertMsgFailed(("u32Sep=%#x (last)\n", u32Sep));
+            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+        }
+        size_t  cchDesc = 0;
+        char    szDesc[256];
+        szDesc[0] = '\0';
+        if (uVersion >= PGM_SAVED_STATE_VERSION_RR_DESC)
+        {
+            rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
+            if (RT_FAILURE(rc))
+                return rc;
+            /* Since we've modified the description strings in r45878, only compare
+               them if the saved state is more recent. */
+            if (uVersion != PGM_SAVED_STATE_VERSION_RR_DESC)
+                cchDesc = strlen(szDesc);
+        }
+
+        /*
+         * Match it up with the current range.
+         *
+         * Note there is a hack for dealing with the high BIOS mapping
+         * in the old saved state format, this means we might not have
+         * a 1:1 match on success.
+         */
+        if (    (   GCPhys     != pRam->GCPhys
+                 || GCPhysLast != pRam->GCPhysLast
+                 || cb         != pRam->cb
+                 ||  (   cchDesc
+                      && strcmp(szDesc, pRam->pszDesc)) )
+                /* Hack for PDMDevHlpPhysReserve(pDevIns, 0xfff80000, 0x80000, "High ROM Region"); */
+            &&  (   uVersion != PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE
+                 || GCPhys     != UINT32_C(0xfff80000)
+                 || GCPhysLast != UINT32_C(0xffffffff)
+                 || pRam->GCPhysLast != GCPhysLast
+                 || pRam->GCPhys     <  GCPhys
+                 || !fHaveBits)
+           )
+        {
+            LogRel(("Ram range: %RGp-%RGp %RGp bytes %s %s\n"
+                    "State    : %RGp-%RGp %RGp bytes %s %s\n",
+                    pRam->GCPhys, pRam->GCPhysLast, pRam->cb, pRam->pvR3 ? "bits" : "nobits", pRam->pszDesc,
+                    GCPhys, GCPhysLast, cb, fHaveBits ? "bits" : "nobits", szDesc));
+            /*
+             * If we're loading a state for debugging purpose, don't make a fuss if
+             * the MMIO and ROM stuff isn't 100% right, just skip the mismatches.
+             */
+            if (    SSMR3HandleGetAfter(pSSM) != SSMAFTER_DEBUG_IT
+                ||  GCPhys < 8 * _1M)
+                return SSMR3SetCfgError(pSSM, RT_SRC_POS,
+                                        N_("RAM range mismatch; saved={%RGp-%RGp %RGp bytes %s %s} config={%RGp-%RGp %RGp bytes %s %s}"),
+                                        GCPhys, GCPhysLast, cb, fHaveBits ? "bits" : "nobits", szDesc,
+                                        pRam->GCPhys, pRam->GCPhysLast, pRam->cb, pRam->pvR3 ? "bits" : "nobits", pRam->pszDesc);
+
+            AssertMsgFailed(("debug skipping not implemented, sorry\n"));
+            continue;
+        }
+
+        uint32_t cPages = (GCPhysLast - GCPhys + 1) >> PAGE_SHIFT;
+        if (uVersion >= PGM_SAVED_STATE_VERSION_RR_DESC)
+        {
+            /*
+             * Load the pages one by one.
+             */
+            for (uint32_t iPage = 0; iPage < cPages; iPage++)
+            {
+                RTGCPHYS const  GCPhysPage = ((RTGCPHYS)iPage << PAGE_SHIFT) + pRam->GCPhys;
+                PPGMPAGE        pPage      = &pRam->aPages[iPage];
+                uint8_t         uOldType;
+                rc = SSMR3GetU8(pSSM, &uOldType);
+                AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] iPage=%#x GCPhysPage=%#x %s\n", pPage, iPage, GCPhysPage, pRam->pszDesc), rc);
+                if (uOldType == PGMPAGETYPE_OLD_ROM_SHADOW)
+                    rc = pgmR3LoadShadowedRomPageOld(pVM, pSSM, pPage, GCPhysPage, pRam);
+                else
+                    rc = pgmR3LoadPageOld(pVM, pSSM, uOldType, pPage, GCPhysPage, pRam);
+                AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhysPage=%#x %s\n", rc, iPage, GCPhysPage, pRam->pszDesc), rc);
+            }
+        }
+        else
+        {
+            /*
+             * Old format.
+             */
+
+            /* Of the page flags, pick up MMIO2 and ROM/RESERVED for the !fHaveBits case.
+               The rest is generally irrelevant and wrong since the stuff have to match registrations. */
+            uint32_t fFlags = 0;
+            for (uint32_t iPage = 0; iPage < cPages; iPage++)
+            {
+                uint16_t u16Flags;
+                rc = SSMR3GetU16(pSSM, &u16Flags);
+                AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhys=%#x %s\n", rc, iPage, pRam->GCPhys, pRam->pszDesc), rc);
+                fFlags |= u16Flags;
+            }
+
+            /* Load the bits */
+            if (    !fHaveBits
+                &&  GCPhysLast < UINT32_C(0xe0000000))
+            {
+                /*
+                 * Dynamic chunks.
+                 */
+                const uint32_t cPagesInChunk = (1*1024*1024) >> PAGE_SHIFT;
+                AssertLogRelMsgReturn(cPages % cPagesInChunk == 0,
+                                      ("cPages=%#x cPagesInChunk=%#x GCPhys=%RGp %s\n", cPages, cPagesInChunk, pRam->GCPhys, pRam->pszDesc),
+                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+                for (uint32_t iPage = 0; iPage < cPages; /* incremented by inner loop */ )
+                {
+                    uint8_t fPresent;
+                    rc = SSMR3GetU8(pSSM, &fPresent);
+                    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhys=%#x %s\n", rc, iPage, pRam->GCPhys, pRam->pszDesc), rc);
+                    AssertLogRelMsgReturn(fPresent == (uint8_t)true || fPresent == (uint8_t)false,
+                                          ("fPresent=%#x iPage=%#x GCPhys=%#x %s\n", fPresent, iPage, pRam->GCPhys, pRam->pszDesc),
+                                          VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+                    for (uint32_t iChunkPage = 0; iChunkPage < cPagesInChunk; iChunkPage++, iPage++)
+                    {
+                        RTGCPHYS const  GCPhysPage = ((RTGCPHYS)iPage << PAGE_SHIFT) + pRam->GCPhys;
+                        PPGMPAGE        pPage      = &pRam->aPages[iPage];
+                        if (fPresent)
+                        {
+                            if (   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO
+                                || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO)
+                                rc = pgmR3LoadPageToDevNullOld(pSSM);
+                            else
+                                rc = pgmR3LoadPageBitsOld(pVM, pSSM, PGMPAGETYPE_INVALID, pPage, GCPhysPage, pRam);
+                        }
+                        else
+                            rc = pgmR3LoadPageZeroOld(pVM, PGMPAGETYPE_INVALID, pPage, GCPhysPage, pRam);
+                        AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhysPage=%#x %s\n", rc, iPage, GCPhysPage, pRam->pszDesc), rc);
+                    }
+                }
+            }
+            else if (pRam->pvR3)
+            {
+                /*
+                 * MMIO2.
+                 */
+                AssertLogRelMsgReturn((fFlags & 0x0f) == RT_BIT(3) /*MM_RAM_FLAGS_MMIO2*/,
+                                      ("fFlags=%#x GCPhys=%#x %s\n", fFlags, pRam->GCPhys, pRam->pszDesc),
+                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+                AssertLogRelMsgReturn(pRam->pvR3,
+                                      ("GCPhys=%#x %s\n", pRam->GCPhys, pRam->pszDesc),
+                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+                rc = SSMR3GetMem(pSSM, pRam->pvR3, pRam->cb);
+                AssertLogRelMsgRCReturn(rc, ("GCPhys=%#x %s\n", pRam->GCPhys, pRam->pszDesc), rc);
+            }
+            else if (GCPhysLast < UINT32_C(0xfff80000))
+            {
+                /*
+                 * PCI MMIO, no pages saved.
+                 */
+            }
+            else
+            {
+                /*
+                 * Load the 0xfff80000..0xffffffff BIOS range.
+                 * It starts with X reserved pages that we have to skip over since
+                 * the RAMRANGE create by the new code won't include those.
+                 */
+                AssertLogRelMsgReturn(   !(fFlags & RT_BIT(3) /*MM_RAM_FLAGS_MMIO2*/)
+                                      && (fFlags  & RT_BIT(0) /*MM_RAM_FLAGS_RESERVED*/),
+                                      ("fFlags=%#x GCPhys=%#x %s\n", fFlags, pRam->GCPhys, pRam->pszDesc),
+                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+                AssertLogRelMsgReturn(GCPhys == UINT32_C(0xfff80000),
+                                      ("GCPhys=%RGp pRamRange{GCPhys=%#x %s}\n", GCPhys, pRam->GCPhys, pRam->pszDesc),
+                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+                /* Skip wasted reserved pages before the ROM. */
+                while (GCPhys < pRam->GCPhys)
+                {
+                    rc = pgmR3LoadPageToDevNullOld(pSSM);
+                    GCPhys += PAGE_SIZE;
+                }
+
+                /* Load the bios pages. */
+                cPages = pRam->cb >> PAGE_SHIFT;
+                for (uint32_t iPage = 0; iPage < cPages; iPage++)
+                {
+                    RTGCPHYS const  GCPhysPage = ((RTGCPHYS)iPage << PAGE_SHIFT) + pRam->GCPhys;
+                    PPGMPAGE        pPage      = &pRam->aPages[iPage];
+
+                    AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_ROM,
+                                          ("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, GCPhys),
+                                          VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+                    rc = pgmR3LoadPageBitsOld(pVM, pSSM, PGMPAGETYPE_ROM, pPage, GCPhysPage, pRam);
+                    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhys=%#x %s\n", rc, iPage, pRam->GCPhys, pRam->pszDesc), rc);
+                }
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for pgmR3Load and pgmR3LoadLocked.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The SSM handle.
+ * @param   uVersion            The PGM saved state unit version.
+ * @param   uPass               The pass number.
+ *
+ * @todo    This needs splitting up if more record types or code twists are
+ *          added...
+ */
+static int pgmR3LoadMemory(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    NOREF(uPass);
+
+    /*
+     * Process page records until we hit the terminator.
+     */
+    RTGCPHYS            GCPhys   = NIL_RTGCPHYS;
+    PPGMRAMRANGE        pRamHint = NULL;
+    uint8_t             id       = UINT8_MAX;
+    uint32_t            iPage    = UINT32_MAX - 10;
+    PPGMROMRANGE        pRom     = NULL;
+    PPGMREGMMIO2RANGE    pRegMmio = NULL;
+
+    /*
+     * We batch up pages that should be freed instead of calling GMM for
+     * each and every one of them.  Note that we'll lose the pages in most
+     * failure paths - this should probably be addressed one day.
+     */
+    uint32_t            cPendingPages = 0;
+    PGMMFREEPAGESREQ    pReq;
+    int rc = GMMR3FreePagesPrepare(pVM, &pReq, 128 /* batch size */, GMMACCOUNT_BASE);
+    AssertLogRelRCReturn(rc, rc);
+
+    for (;;)
+    {
+        /*
+         * Get the record type and flags.
+         */
+        uint8_t u8;
+        rc = SSMR3GetU8(pSSM, &u8);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u8 == PGM_STATE_REC_END)
+        {
+            /*
+             * Finish off any pages pending freeing.
+             */
+            if (cPendingPages)
+            {
+                Log(("pgmR3LoadMemory: GMMR3FreePagesPerform pVM=%p cPendingPages=%u\n", pVM, cPendingPages));
+                rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+                AssertLogRelRCReturn(rc, rc);
+            }
+            GMMR3FreePagesCleanup(pReq);
+            return VINF_SUCCESS;
+        }
+        AssertLogRelMsgReturn((u8 & ~PGM_STATE_REC_FLAG_ADDR) <= PGM_STATE_REC_LAST, ("%#x\n", u8), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+        switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
+        {
+            /*
+             * RAM page.
+             */
+            case PGM_STATE_REC_RAM_ZERO:
+            case PGM_STATE_REC_RAM_RAW:
+            case PGM_STATE_REC_RAM_BALLOONED:
+            {
+                /*
+                 * Get the address and resolve it into a page descriptor.
+                 */
+                if (!(u8 & PGM_STATE_REC_FLAG_ADDR))
+                    GCPhys += PAGE_SIZE;
+                else
+                {
+                    rc = SSMR3GetGCPhys(pSSM, &GCPhys);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+                AssertLogRelMsgReturn(!(GCPhys & PAGE_OFFSET_MASK), ("%RGp\n", GCPhys), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+                PPGMPAGE pPage;
+                rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
+                AssertLogRelMsgRCReturn(rc, ("rc=%Rrc %RGp\n", rc, GCPhys), rc);
+
+                /*
+                 * Take action according to the record type.
+                 */
+                switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
+                {
+                    case PGM_STATE_REC_RAM_ZERO:
+                    {
+                        if (PGM_PAGE_IS_ZERO(pPage))
+                            break;
+
+                        /* Ballooned pages must be unmarked (live snapshot and
+                           teleportation scenarios). */
+                        if (PGM_PAGE_IS_BALLOONED(pPage))
+                        {
+                            Assert(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
+                            if (uVersion == PGM_SAVED_STATE_VERSION_BALLOON_BROKEN)
+                                break;
+                            PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
+                            break;
+                        }
+
+                        AssertLogRelMsgReturn(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED, ("GCPhys=%RGp %R[pgmpage]\n", GCPhys, pPage), VERR_PGM_UNEXPECTED_PAGE_STATE);
+
+                        /* If this is a ROM page, we must clear it and not try to
+                         * free it.  Ditto if the VM is using RamPreAlloc (see
+                         * @bugref{6318}). */
+                        if (   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_ROM
+                            || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_ROM_SHADOW
+                            || pVM->pgm.s.fRamPreAlloc)
+                        {
+                            PGMPAGEMAPLOCK PgMpLck;
+                            void          *pvDstPage;
+                            rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDstPage, &PgMpLck);
+                            AssertLogRelMsgRCReturn(rc, ("GCPhys=%RGp %R[pgmpage] rc=%Rrc\n", GCPhys, pPage, rc), rc);
+
+                            ASMMemZeroPage(pvDstPage);
+                            pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+                        }
+                        /* Free it only if it's not part of a previously
+                           allocated large page (no need to clear the page). */
+                        else if (   PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
+                                 && PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED)
+                        {
+                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, GCPhys, (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage));
+                            AssertRCReturn(rc, rc);
+                        }
+                        /** @todo handle large pages (see @bugref{5545}) */
+                        break;
+                    }
+
+                    case PGM_STATE_REC_RAM_BALLOONED:
+                    {
+                        Assert(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
+                        if (PGM_PAGE_IS_BALLOONED(pPage))
+                            break;
+
+                        /* We don't map ballooned pages in our shadow page tables, let's
+                           just free it if allocated and mark as ballooned.  See @bugref{5515}. */
+                        if (PGM_PAGE_IS_ALLOCATED(pPage))
+                        {
+                            /** @todo handle large pages + ballooning when it works. (see @bugref{5515},
+                             *        @bugref{5545}). */
+                            AssertLogRelMsgReturn(   PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
+                                                  && PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED,
+                                                     ("GCPhys=%RGp %R[pgmpage]\n", GCPhys, pPage), VERR_PGM_LOAD_UNEXPECTED_PAGE_TYPE);
+
+                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, GCPhys, (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage));
+                            AssertRCReturn(rc, rc);
+                        }
+                        Assert(PGM_PAGE_IS_ZERO(pPage));
+                        PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
+                        break;
+                    }
+
+                    case PGM_STATE_REC_RAM_RAW:
+                    {
+                        PGMPAGEMAPLOCK PgMpLck;
+                        void          *pvDstPage;
+                        rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDstPage, &PgMpLck);
+                        AssertLogRelMsgRCReturn(rc, ("GCPhys=%RGp %R[pgmpage] rc=%Rrc\n", GCPhys, pPage, rc), rc);
+                        rc = SSMR3GetMem(pSSM, pvDstPage, PAGE_SIZE);
+                        pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+                        if (RT_FAILURE(rc))
+                            return rc;
+                        break;
+                    }
+
+                    default:
+                        AssertMsgFailedReturn(("%#x\n", u8), VERR_PGM_SAVED_REC_TYPE);
+                }
+                id = UINT8_MAX;
+                break;
+            }
+
+            /*
+             * MMIO2 page.
+             */
+            case PGM_STATE_REC_MMIO2_RAW:
+            case PGM_STATE_REC_MMIO2_ZERO:
+            {
+                /*
+                 * Get the ID + page number and resolved that into a MMIO2 page.
+                 */
+                if (!(u8 & PGM_STATE_REC_FLAG_ADDR))
+                    iPage++;
+                else
+                {
+                    SSMR3GetU8(pSSM, &id);
+                    rc = SSMR3GetU32(pSSM, &iPage);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+                if (   !pRegMmio
+                    || pRegMmio->idSavedState != id)
+                {
+                    for (pRegMmio = pVM->pgm.s.pRegMmioRangesR3; pRegMmio; pRegMmio = pRegMmio->pNextR3)
+                        if (   pRegMmio->idSavedState == id
+                            && (pRegMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2))
+                            break;
+                    AssertLogRelMsgReturn(pRegMmio, ("id=%#u iPage=%#x\n", id, iPage), VERR_PGM_SAVED_MMIO2_RANGE_NOT_FOUND);
+                }
+                AssertLogRelMsgReturn(iPage < (pRegMmio->RamRange.cb >> PAGE_SHIFT), ("iPage=%#x cb=%RGp %s\n", iPage, pRegMmio->RamRange.cb, pRegMmio->RamRange.pszDesc), VERR_PGM_SAVED_MMIO2_PAGE_NOT_FOUND);
+                void *pvDstPage = (uint8_t *)pRegMmio->RamRange.pvR3 + ((size_t)iPage << PAGE_SHIFT);
+
+                /*
+                 * Load the page bits.
+                 */
+                if ((u8 & ~PGM_STATE_REC_FLAG_ADDR) == PGM_STATE_REC_MMIO2_ZERO)
+                    ASMMemZeroPage(pvDstPage);
+                else
+                {
+                    rc = SSMR3GetMem(pSSM, pvDstPage, PAGE_SIZE);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+                GCPhys = NIL_RTGCPHYS;
+                break;
+            }
+
+            /*
+             * ROM pages.
+             */
+            case PGM_STATE_REC_ROM_VIRGIN:
+            case PGM_STATE_REC_ROM_SHW_RAW:
+            case PGM_STATE_REC_ROM_SHW_ZERO:
+            case PGM_STATE_REC_ROM_PROT:
+            {
+                /*
+                 * Get the ID + page number and resolved that into a ROM page descriptor.
+                 */
+                if (!(u8 & PGM_STATE_REC_FLAG_ADDR))
+                    iPage++;
+                else
+                {
+                    SSMR3GetU8(pSSM, &id);
+                    rc = SSMR3GetU32(pSSM, &iPage);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+                if (    !pRom
+                    ||  pRom->idSavedState != id)
+                {
+                    for (pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+                        if (pRom->idSavedState == id)
+                            break;
+                    AssertLogRelMsgReturn(pRom, ("id=%#u iPage=%#x\n", id, iPage), VERR_PGM_SAVED_ROM_RANGE_NOT_FOUND);
+                }
+                AssertLogRelMsgReturn(iPage < (pRom->cb >> PAGE_SHIFT), ("iPage=%#x cb=%RGp %s\n", iPage, pRom->cb, pRom->pszDesc), VERR_PGM_SAVED_ROM_PAGE_NOT_FOUND);
+                PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
+                GCPhys = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+
+                /*
+                 * Get and set the protection.
+                 */
+                uint8_t u8Prot;
+                rc = SSMR3GetU8(pSSM, &u8Prot);
+                if (RT_FAILURE(rc))
+                    return rc;
+                PGMROMPROT enmProt = (PGMROMPROT)u8Prot;
+                AssertLogRelMsgReturn(enmProt > PGMROMPROT_INVALID && enmProt < PGMROMPROT_END, ("GCPhys=%RGp enmProt=%d\n", GCPhys, enmProt), VERR_PGM_SAVED_ROM_PAGE_PROT);
+
+                if (enmProt != pRomPage->enmProt)
+                {
+                    if (RT_UNLIKELY(!(pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)))
+                        return SSMR3SetCfgError(pSSM, RT_SRC_POS,
+                                                N_("Protection change of unshadowed ROM page: GCPhys=%RGp enmProt=%d %s"),
+                                                GCPhys, enmProt, pRom->pszDesc);
+                    rc = PGMR3PhysRomProtect(pVM, GCPhys, PAGE_SIZE, enmProt);
+                    AssertLogRelMsgRCReturn(rc, ("GCPhys=%RGp rc=%Rrc\n", GCPhys, rc), rc);
+                    AssertLogRelReturn(pRomPage->enmProt == enmProt, VERR_PGM_SAVED_ROM_PAGE_PROT);
+                }
+                if ((u8 & ~PGM_STATE_REC_FLAG_ADDR) == PGM_STATE_REC_ROM_PROT)
+                    break; /* done */
+
+                /*
+                 * Get the right page descriptor.
+                 */
+                PPGMPAGE pRealPage;
+                switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
+                {
+                    case PGM_STATE_REC_ROM_VIRGIN:
+                        if (!PGMROMPROT_IS_ROM(enmProt))
+                            pRealPage = &pRomPage->Virgin;
+                        else
+                            pRealPage = NULL;
+                        break;
+
+                    case PGM_STATE_REC_ROM_SHW_RAW:
+                    case PGM_STATE_REC_ROM_SHW_ZERO:
+                        if (RT_UNLIKELY(!(pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)))
+                            return SSMR3SetCfgError(pSSM, RT_SRC_POS,
+                                                    N_("Shadowed / non-shadowed page type mismatch: GCPhys=%RGp enmProt=%d %s"),
+                                                    GCPhys, enmProt, pRom->pszDesc);
+                        if (PGMROMPROT_IS_ROM(enmProt))
+                            pRealPage = &pRomPage->Shadow;
+                        else
+                            pRealPage = NULL;
+                        break;
+
+                    default: AssertLogRelFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE); /* shut up gcc */
+                }
+                if (!pRealPage)
+                {
+                    rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pRealPage, &pRamHint);
+                    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc %RGp\n", rc, GCPhys), rc);
+                }
+
+                /*
+                 * Make it writable and map it (if necessary).
+                 */
+                void *pvDstPage = NULL;
+                switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
+                {
+                    case PGM_STATE_REC_ROM_SHW_ZERO:
+                        if (    PGM_PAGE_IS_ZERO(pRealPage)
+                            ||  PGM_PAGE_IS_BALLOONED(pRealPage))
+                            break;
+                        /** @todo implement zero page replacing. */
+                        RT_FALL_THRU();
+                    case PGM_STATE_REC_ROM_VIRGIN:
+                    case PGM_STATE_REC_ROM_SHW_RAW:
+                    {
+                        rc = pgmPhysPageMakeWritableAndMap(pVM, pRealPage, GCPhys, &pvDstPage);
+                        AssertLogRelMsgRCReturn(rc, ("GCPhys=%RGp rc=%Rrc\n", GCPhys, rc), rc);
+                        break;
+                    }
+                }
+
+                /*
+                 * Load the bits.
+                 */
+                switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
+                {
+                    case PGM_STATE_REC_ROM_SHW_ZERO:
+                        if (pvDstPage)
+                            ASMMemZeroPage(pvDstPage);
+                        break;
+
+                    case PGM_STATE_REC_ROM_VIRGIN:
+                    case PGM_STATE_REC_ROM_SHW_RAW:
+                        rc = SSMR3GetMem(pSSM, pvDstPage, PAGE_SIZE);
+                        if (RT_FAILURE(rc))
+                            return rc;
+                        break;
+                }
+                GCPhys = NIL_RTGCPHYS;
+                break;
+            }
+
+            /*
+             * Unknown type.
+             */
+            default:
+                AssertLogRelMsgFailedReturn(("%#x\n", u8), VERR_PGM_SAVED_REC_TYPE);
+        }
+    } /* forever */
+}
+
+
+/**
+ * Worker for pgmR3Load.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The SSM handle.
+ * @param   uVersion            The saved state version.
+ */
+static int pgmR3LoadFinalLocked(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion)
+{
+    PPGM        pPGM = &pVM->pgm.s;
+    int         rc;
+    uint32_t    u32Sep;
+
+    /*
+     * Load basic data (required / unaffected by relocation).
+     */
+    if (uVersion >= PGM_SAVED_STATE_VERSION_3_0_0)
+    {
+        if (uVersion > PGM_SAVED_STATE_VERSION_PRE_BALLOON)
+            rc = SSMR3GetStruct(pSSM, pPGM, &s_aPGMFields[0]);
+        else
+            rc = SSMR3GetStruct(pSSM, pPGM, &s_aPGMFieldsPreBalloon[0]);
+
+        AssertLogRelRCReturn(rc, rc);
+
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            if (uVersion <= PGM_SAVED_STATE_VERSION_PRE_PAE)
+                rc = SSMR3GetStruct(pSSM, &pVM->apCpusR3[i]->pgm.s, &s_aPGMCpuFieldsPrePae[0]);
+            else
+                rc = SSMR3GetStruct(pSSM, &pVM->apCpusR3[i]->pgm.s, &s_aPGMCpuFields[0]);
+            AssertLogRelRCReturn(rc, rc);
+        }
+    }
+    else if (uVersion >= PGM_SAVED_STATE_VERSION_RR_DESC)
+    {
+        AssertRelease(pVM->cCpus == 1);
+
+        PGMOLD pgmOld;
+        rc = SSMR3GetStruct(pSSM, &pgmOld, &s_aPGMFields_Old[0]);
+        AssertLogRelRCReturn(rc, rc);
+
+        pPGM->fMappingsFixed    = pgmOld.fMappingsFixed;
+        pPGM->GCPtrMappingFixed = pgmOld.GCPtrMappingFixed;
+        pPGM->cbMappingFixed    = pgmOld.cbMappingFixed;
+
+        PVMCPU pVCpu0 = pVM->apCpusR3[0];
+        pVCpu0->pgm.s.fA20Enabled   = pgmOld.fA20Enabled;
+        pVCpu0->pgm.s.GCPhysA20Mask = pgmOld.GCPhysA20Mask;
+        pVCpu0->pgm.s.enmGuestMode  = pgmOld.enmGuestMode;
+    }
+    else
+    {
+        AssertRelease(pVM->cCpus == 1);
+
+        SSMR3GetBool(pSSM,      &pPGM->fMappingsFixed);
+        SSMR3GetGCPtr(pSSM,     &pPGM->GCPtrMappingFixed);
+        SSMR3GetU32(pSSM,       &pPGM->cbMappingFixed);
+
+        uint32_t cbRamSizeIgnored;
+        rc = SSMR3GetU32(pSSM,  &cbRamSizeIgnored);
+        if (RT_FAILURE(rc))
+            return rc;
+        PVMCPU pVCpu0 = pVM->apCpusR3[0];
+        SSMR3GetGCPhys(pSSM,    &pVCpu0->pgm.s.GCPhysA20Mask);
+
+        uint32_t u32 = 0;
+        SSMR3GetUInt(pSSM,      &u32);
+        pVCpu0->pgm.s.fA20Enabled = !!u32;
+        SSMR3GetUInt(pSSM,      &pVCpu0->pgm.s.fSyncFlags);
+        RTUINT uGuestMode;
+        SSMR3GetUInt(pSSM,      &uGuestMode);
+        pVCpu0->pgm.s.enmGuestMode = (PGMMODE)uGuestMode;
+
+        /* check separator. */
+        SSMR3GetU32(pSSM, &u32Sep);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u32Sep != (uint32_t)~0)
+        {
+            AssertMsgFailed(("u32Sep=%#x (first)\n", u32Sep));
+            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+        }
+    }
+
+    /*
+     * Fix the A20 mask.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+        pVCpu->pgm.s.GCPhysA20Mask = ~((RTGCPHYS)!pVCpu->pgm.s.fA20Enabled << 20);
+        pgmR3RefreshShadowModeAfterA20Change(pVCpu);
+    }
+
+    /*
+     * The guest mappings - skipped now, see re-fixation in the caller.
+     */
+    if (uVersion <= PGM_SAVED_STATE_VERSION_PRE_PAE)
+    {
+        for (uint32_t i = 0; ; i++)
+        {
+            rc = SSMR3GetU32(pSSM, &u32Sep);        /* sequence number */
+            if (RT_FAILURE(rc))
+                return rc;
+            if (u32Sep == ~0U)
+                break;
+            AssertMsgReturn(u32Sep == i, ("u32Sep=%#x i=%#x\n", u32Sep, i), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+            char szDesc[256];
+            rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
+            if (RT_FAILURE(rc))
+                return rc;
+            RTGCPTR GCPtrIgnore;
+            SSMR3GetGCPtr(pSSM, &GCPtrIgnore);      /* GCPtr */
+            rc = SSMR3GetGCPtr(pSSM, &GCPtrIgnore); /* cPTs  */
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+    }
+
+    /*
+     * Load the RAM contents.
+     */
+    if (uVersion > PGM_SAVED_STATE_VERSION_3_0_0)
+    {
+        if (!pVM->pgm.s.LiveSave.fActive)
+        {
+            if (uVersion > PGM_SAVED_STATE_VERSION_NO_RAM_CFG)
+            {
+                rc = pgmR3LoadRamConfig(pVM, pSSM);
+                if (RT_FAILURE(rc))
+                    return rc;
+            }
+            rc = pgmR3LoadRomRanges(pVM, pSSM);
+            if (RT_FAILURE(rc))
+                return rc;
+            rc = pgmR3LoadMmio2Ranges(pVM, pSSM);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+
+        rc = pgmR3LoadMemory(pVM, pSSM, uVersion, SSM_PASS_FINAL);
+    }
+    else
+        rc = pgmR3LoadMemoryOld(pVM, pSSM, uVersion);
+
+    /* Refresh balloon accounting. */
+    if (pVM->pgm.s.cBalloonedPages)
+    {
+        Log(("pgmR3LoadFinalLocked: pVM=%p cBalloonedPages=%#x\n", pVM, pVM->pgm.s.cBalloonedPages));
+        rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_INFLATE, pVM->pgm.s.cBalloonedPages);
+        AssertRCReturn(rc, rc);
+    }
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTLOADEXEC}
+ */
+static DECLCALLBACK(int) pgmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    int rc;
+
+    /*
+     * Validate version.
+     */
+    if (   (   uPass != SSM_PASS_FINAL
+            && uVersion != PGM_SAVED_STATE_VERSION
+            && uVersion != PGM_SAVED_STATE_VERSION_PRE_PAE
+            && uVersion != PGM_SAVED_STATE_VERSION_BALLOON_BROKEN
+            && uVersion != PGM_SAVED_STATE_VERSION_PRE_BALLOON
+            && uVersion != PGM_SAVED_STATE_VERSION_NO_RAM_CFG)
+        || (   uVersion != PGM_SAVED_STATE_VERSION
+            && uVersion != PGM_SAVED_STATE_VERSION_PRE_PAE
+            && uVersion != PGM_SAVED_STATE_VERSION_BALLOON_BROKEN
+            && uVersion != PGM_SAVED_STATE_VERSION_PRE_BALLOON
+            && uVersion != PGM_SAVED_STATE_VERSION_NO_RAM_CFG
+            && uVersion != PGM_SAVED_STATE_VERSION_3_0_0
+            && uVersion != PGM_SAVED_STATE_VERSION_2_2_2
+            && uVersion != PGM_SAVED_STATE_VERSION_RR_DESC
+            && uVersion != PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE)
+       )
+    {
+        AssertMsgFailed(("pgmR3Load: Invalid version uVersion=%d (current %d)!\n", uVersion, PGM_SAVED_STATE_VERSION));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    /*
+     * Do the loading while owning the lock because a bunch of the functions
+     * we're using requires this.
+     */
+    if (uPass != SSM_PASS_FINAL)
+    {
+        pgmLock(pVM);
+        if (uPass != 0)
+            rc = pgmR3LoadMemory(pVM, pSSM, uVersion, uPass);
+        else
+        {
+            pVM->pgm.s.LiveSave.fActive = true;
+            if (uVersion > PGM_SAVED_STATE_VERSION_NO_RAM_CFG)
+                rc = pgmR3LoadRamConfig(pVM, pSSM);
+            else
+                rc = VINF_SUCCESS;
+            if (RT_SUCCESS(rc))
+                rc = pgmR3LoadRomRanges(pVM, pSSM);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3LoadMmio2Ranges(pVM, pSSM);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3LoadMemory(pVM, pSSM, uVersion, uPass);
+        }
+        pgmUnlock(pVM);
+    }
+    else
+    {
+        pgmLock(pVM);
+        rc = pgmR3LoadFinalLocked(pVM, pSSM, uVersion);
+        pVM->pgm.s.LiveSave.fActive = false;
+        pgmUnlock(pVM);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * We require a full resync now.
+             */
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+            {
+                PVMCPU pVCpu = pVM->apCpusR3[i];
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+                /** @todo For guest PAE, we might get the wrong
+                 *        aGCPhysGstPaePDs values now. We should used the
+                 *        saved ones... Postponing this since it nothing new
+                 *        and PAE/PDPTR needs some general readjusting, see
+                 *        @bugref{5880}. */
+            }
+
+            pgmR3HandlerPhysicalUpdateAll(pVM);
+
+            /*
+             * Change the paging mode (indirectly restores PGMCPU::GCPhysCR3).
+             * (Requires the CPUM state to be restored already!)
+             */
+            if (CPUMR3IsStateRestorePending(pVM))
+                return SSMR3SetLoadError(pSSM, VERR_WRONG_ORDER, RT_SRC_POS,
+                                         N_("PGM was unexpectedly restored before CPUM"));
+
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+            {
+                PVMCPU pVCpu = pVM->apCpusR3[i];
+
+                rc = PGMHCChangeMode(pVM, pVCpu, pVCpu->pgm.s.enmGuestMode);
+                AssertLogRelRCReturn(rc, rc);
+
+                /* Update the PSE, NX flags and validity masks. */
+                pVCpu->pgm.s.fGst32BitPageSizeExtension = CPUMIsGuestPageSizeExtEnabled(pVCpu);
+                PGMNotifyNxeChanged(pVCpu, CPUMIsGuestNXEnabled(pVCpu));
+            }
+
+            /*
+             * Try re-fixate the guest mappings.
+             */
+            pVM->pgm.s.fMappingsFixedRestored = false;
+            if (   pVM->pgm.s.fMappingsFixed
+                && pgmMapAreMappingsEnabled(pVM))
+            {
+#ifndef PGM_WITHOUT_MAPPINGS
+                RTGCPTR     GCPtrFixed    = pVM->pgm.s.GCPtrMappingFixed;
+                uint32_t    cbFixed       = pVM->pgm.s.cbMappingFixed;
+                pVM->pgm.s.fMappingsFixed = false;
+
+                uint32_t    cbRequired;
+                int rc2 = PGMR3MappingsSize(pVM, &cbRequired); AssertRC(rc2);
+                if (   RT_SUCCESS(rc2)
+                    && cbRequired > cbFixed)
+                    rc2 = VERR_OUT_OF_RANGE;
+                if (RT_SUCCESS(rc2))
+                    rc2 = pgmR3MappingsFixInternal(pVM, GCPtrFixed, cbFixed);
+                if (RT_FAILURE(rc2))
+                {
+                    LogRel(("PGM: Unable to re-fixate the guest mappings at %RGv-%RGv: rc=%Rrc (cbRequired=%#x)\n",
+                            GCPtrFixed, GCPtrFixed + cbFixed, rc2, cbRequired));
+                    pVM->pgm.s.fMappingsFixed         = false;
+                    pVM->pgm.s.fMappingsFixedRestored = true;
+                    pVM->pgm.s.GCPtrMappingFixed      = GCPtrFixed;
+                    pVM->pgm.s.cbMappingFixed         = cbFixed;
+                }
+#else
+                AssertFailed();
+#endif
+            }
+            else
+            {
+                /* We used to set fixed + disabled while we only use disabled now,
+                   so wipe the state to avoid any confusion. */
+                pVM->pgm.s.fMappingsFixed    = false;
+                pVM->pgm.s.GCPtrMappingFixed = NIL_RTGCPTR;
+                pVM->pgm.s.cbMappingFixed    = 0;
+            }
+
+            /*
+             * If we have floating mappings, do a CR3 sync now to make sure the HMA
+             * doesn't conflict with guest code / data and thereby cause trouble
+             * when restoring other components like PATM.
+             */
+            if (pgmMapAreMappingsFloating(pVM))
+            {
+                PVMCPU pVCpu = pVM->apCpusR3[0];
+                rc = PGMSyncCR3(pVCpu, CPUMGetGuestCR0(pVCpu), CPUMGetGuestCR3(pVCpu),  CPUMGetGuestCR4(pVCpu), true);
+                if (RT_FAILURE(rc))
+                    return SSMR3SetLoadError(pSSM, VERR_WRONG_ORDER, RT_SRC_POS,
+                                             N_("PGMSyncCR3 failed unexpectedly with rc=%Rrc"), rc);
+
+                /* Make sure to re-sync before executing code. */
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+            }
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTLOADDONE}
+ */
+static DECLCALLBACK(int) pgmR3LoadDone(PVM pVM, PSSMHANDLE pSSM)
+{
+    pVM->pgm.s.fRestoreRomPagesOnReset = true;
+    NOREF(pSSM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Registers the saved state callbacks with SSM.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   cbRam   The RAM size.
+ */
+int pgmR3InitSavedState(PVM pVM, uint64_t cbRam)
+{
+    return SSMR3RegisterInternal(pVM, "pgm", 1, PGM_SAVED_STATE_VERSION, (size_t)cbRam + sizeof(PGM),
+                                 pgmR3LivePrep, pgmR3LiveExec, pgmR3LiveVote,
+                                 NULL,          pgmR3SaveExec, pgmR3SaveDone,
+                                 pgmR3LoadPrep, pgmR3Load,     pgmR3LoadDone);
+}
+
diff --git a/src/VBox/VMM/VMMR3/PGMSharedPage.cpp b/src/VBox/VMM/VMMR3/PGMSharedPage.cpp
new file mode 100644
index 00000000..dec43d5d
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/PGMSharedPage.cpp
@@ -0,0 +1,442 @@
+/* $Id: PGMSharedPage.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor, Shared page handling
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_SHARED
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/uvm.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/sup.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/VMMDev.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/mem.h>
+#include <iprt/string.h>
+
+#include "PGMInline.h"
+
+
+#ifdef VBOX_WITH_PAGE_SHARING
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+# ifdef VBOX_STRICT
+/** Keep a copy of all registered shared modules for the .pgmcheckduppages debugger command. */
+static PGMMREGISTERSHAREDMODULEREQ  g_apSharedModules[512] = {0};
+static unsigned                     g_cSharedModules = 0;
+# endif /* VBOX_STRICT */
+
+
+/**
+ * Registers a new shared module for the VM
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   enmGuestOS          Guest OS type.
+ * @param   pszModuleName       Module name.
+ * @param   pszVersion          Module version.
+ * @param   GCBaseAddr          Module base address.
+ * @param   cbModule            Module size.
+ * @param   cRegions            Number of shared region descriptors.
+ * @param   paRegions           Shared region(s).
+ *
+ * @todo    This should be a GMMR3 call. No need to involve GMM here.
+ */
+VMMR3DECL(int) PGMR3SharedModuleRegister(PVM pVM, VBOXOSFAMILY enmGuestOS, char *pszModuleName, char *pszVersion,
+                                         RTGCPTR GCBaseAddr, uint32_t cbModule, uint32_t cRegions,
+                                         VMMDEVSHAREDREGIONDESC const *paRegions)
+{
+    Log(("PGMR3SharedModuleRegister family=%d name=%s version=%s base=%RGv size=%x cRegions=%d\n",
+         enmGuestOS, pszModuleName, pszVersion, GCBaseAddr, cbModule, cRegions));
+
+    /*
+     * Sanity check.
+     */
+    AssertReturn(cRegions <= VMMDEVSHAREDREGIONDESC_MAX, VERR_INVALID_PARAMETER);
+    if (!pVM->pgm.s.fPageFusionAllowed)
+        return VERR_NOT_SUPPORTED;
+
+    /*
+     * Allocate and initialize a GMM request.
+     */
+    PGMMREGISTERSHAREDMODULEREQ pReq;
+    pReq = (PGMMREGISTERSHAREDMODULEREQ)RTMemAllocZ(RT_UOFFSETOF_DYN(GMMREGISTERSHAREDMODULEREQ, aRegions[cRegions]));
+    AssertReturn(pReq, VERR_NO_MEMORY);
+
+    pReq->enmGuestOS    = enmGuestOS;
+    pReq->GCBaseAddr    = GCBaseAddr;
+    pReq->cbModule      = cbModule;
+    pReq->cRegions      = cRegions;
+    for (uint32_t i = 0; i < cRegions; i++)
+        pReq->aRegions[i] = paRegions[i];
+
+    int rc = RTStrCopy(pReq->szName, sizeof(pReq->szName), pszModuleName);
+    if (RT_SUCCESS(rc))
+    {
+        rc = RTStrCopy(pReq->szVersion, sizeof(pReq->szVersion), pszVersion);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Issue the request.  In strict builds, do some local tracking.
+             */
+            pgmR3PhysAssertSharedPageChecksums(pVM);
+            rc = GMMR3RegisterSharedModule(pVM, pReq);
+            if (RT_SUCCESS(rc))
+                rc = pReq->rc;
+            AssertMsg(rc == VINF_SUCCESS || rc == VINF_GMM_SHARED_MODULE_ALREADY_REGISTERED, ("%Rrc\n", rc));
+
+# ifdef VBOX_STRICT
+            if (   rc == VINF_SUCCESS
+                && g_cSharedModules < RT_ELEMENTS(g_apSharedModules))
+            {
+                unsigned i;
+                for (i = 0; i < RT_ELEMENTS(g_apSharedModules); i++)
+                    if (g_apSharedModules[i] == NULL)
+                    {
+
+                        size_t const cbSharedModule = RT_UOFFSETOF_DYN(GMMREGISTERSHAREDMODULEREQ, aRegions[cRegions]);
+                        g_apSharedModules[i] = (PGMMREGISTERSHAREDMODULEREQ)RTMemDup(pReq, cbSharedModule);
+                        g_cSharedModules++;
+                        break;
+                    }
+                Assert(i < RT_ELEMENTS(g_apSharedModules));
+            }
+# endif /* VBOX_STRICT */
+            if (RT_SUCCESS(rc))
+                rc = VINF_SUCCESS;
+        }
+    }
+
+    RTMemFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Unregisters a shared module for the VM
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pszModuleName       Module name.
+ * @param   pszVersion          Module version.
+ * @param   GCBaseAddr          Module base address.
+ * @param   cbModule            Module size.
+ *
+ * @todo    This should be a GMMR3 call. No need to involve GMM here.
+ */
+VMMR3DECL(int) PGMR3SharedModuleUnregister(PVM pVM, char *pszModuleName, char *pszVersion, RTGCPTR GCBaseAddr, uint32_t cbModule)
+{
+    Log(("PGMR3SharedModuleUnregister name=%s version=%s base=%RGv size=%x\n", pszModuleName, pszVersion, GCBaseAddr, cbModule));
+
+    AssertMsgReturn(cbModule > 0 && cbModule < _1G, ("%u\n", cbModule), VERR_OUT_OF_RANGE);
+    if (!pVM->pgm.s.fPageFusionAllowed)
+        return VERR_NOT_SUPPORTED;
+
+    /*
+     * Forward the request to GMM (ring-0).
+     */
+    PGMMUNREGISTERSHAREDMODULEREQ pReq = (PGMMUNREGISTERSHAREDMODULEREQ)RTMemAlloc(sizeof(*pReq));
+    AssertReturn(pReq, VERR_NO_MEMORY);
+
+    pReq->GCBaseAddr    = GCBaseAddr;
+    pReq->u32Alignment  = 0;
+    pReq->cbModule      = cbModule;
+
+    int rc = RTStrCopy(pReq->szName, sizeof(pReq->szName), pszModuleName);
+    if (RT_SUCCESS(rc))
+    {
+        rc = RTStrCopy(pReq->szVersion, sizeof(pReq->szVersion), pszVersion);
+        if (RT_SUCCESS(rc))
+        {
+            pgmR3PhysAssertSharedPageChecksums(pVM);
+            rc = GMMR3UnregisterSharedModule(pVM, pReq);
+            pgmR3PhysAssertSharedPageChecksums(pVM);
+
+# ifdef VBOX_STRICT
+            /*
+             * Update our local tracking.
+             */
+            for (unsigned i = 0; i < g_cSharedModules; i++)
+            {
+                if (    g_apSharedModules[i]
+                    &&  !strcmp(g_apSharedModules[i]->szName, pszModuleName)
+                    &&  !strcmp(g_apSharedModules[i]->szVersion, pszVersion))
+                {
+                    RTMemFree(g_apSharedModules[i]);
+                    g_apSharedModules[i] = NULL;
+                    g_cSharedModules--;
+                    break;
+                }
+            }
+# endif /* VBOX_STRICT */
+        }
+    }
+
+    RTMemFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Rendezvous callback that will be called once.
+ *
+ * @returns VBox strict status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser              Pointer to a VMCPUID with the requester's ID.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) pgmR3SharedModuleRegRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    VMCPUID idCpu = *(VMCPUID *)pvUser;
+
+    /* Execute on the VCPU that issued the original request to make sure we're in the right cr3 context. */
+    if (pVCpu->idCpu != idCpu)
+    {
+        Assert(pVM->cCpus > 1);
+        return VINF_SUCCESS;
+    }
+
+
+    /* Flush all pending handy page operations before changing any shared page assignments. */
+    int rc = PGMR3PhysAllocateHandyPages(pVM);
+    AssertRC(rc);
+
+    /*
+     * Lock it here as we can't deal with busy locks in this ring-0 path.
+     */
+    LogFlow(("pgmR3SharedModuleRegRendezvous: start (%d)\n", pVM->pgm.s.cSharedPages));
+
+    pgmLock(pVM);
+    pgmR3PhysAssertSharedPageChecksums(pVM);
+    rc = GMMR3CheckSharedModules(pVM);
+    pgmR3PhysAssertSharedPageChecksums(pVM);
+    pgmUnlock(pVM);
+    AssertLogRelRC(rc);
+
+    LogFlow(("pgmR3SharedModuleRegRendezvous: done (%d)\n", pVM->pgm.s.cSharedPages));
+    return rc;
+}
+
+/**
+ * Shared module check helper (called on the way out).
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   idCpu       VCPU id.
+ */
+static DECLCALLBACK(void) pgmR3CheckSharedModulesHelper(PVM pVM, VMCPUID idCpu)
+{
+    /* We must stall other VCPUs as we'd otherwise have to send IPI flush commands for every single change we make. */
+    STAM_REL_PROFILE_START(&pVM->pgm.s.StatShModCheck, a);
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, pgmR3SharedModuleRegRendezvous, &idCpu);
+    AssertRCSuccess(rc);
+    STAM_REL_PROFILE_STOP(&pVM->pgm.s.StatShModCheck, a);
+}
+
+
+/**
+ * Check all registered modules for changes.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMR3DECL(int) PGMR3SharedModuleCheckAll(PVM pVM)
+{
+    if (!pVM->pgm.s.fPageFusionAllowed)
+        return VERR_NOT_SUPPORTED;
+
+    /* Queue the actual registration as we are under the IOM lock right now. Perform this operation on the way out. */
+    return VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3CheckSharedModulesHelper, 2, pVM, VMMGetCpuId(pVM));
+}
+
+
+# ifdef DEBUG
+/**
+ * Query the state of a page in a shared module
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   GCPtrPage           Page address.
+ * @param   pfShared            Shared status (out).
+ * @param   pfPageFlags         Page flags (out).
+ */
+VMMR3DECL(int) PGMR3SharedModuleGetPageState(PVM pVM, RTGCPTR GCPtrPage, bool *pfShared, uint64_t *pfPageFlags)
+{
+    /* Debug only API for the page fusion testcase. */
+    RTGCPHYS GCPhys;
+    uint64_t fFlags;
+
+    pgmLock(pVM);
+
+    int rc = PGMGstGetPage(VMMGetCpu(pVM), GCPtrPage, &fFlags, &GCPhys);
+    switch (rc)
+    {
+        case VINF_SUCCESS:
+        {
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+            if (pPage)
+            {
+                *pfShared    = PGM_PAGE_IS_SHARED(pPage);
+                *pfPageFlags = fFlags;
+            }
+            else
+                rc = VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+            break;
+        }
+
+        case VERR_PAGE_NOT_PRESENT:
+        case VERR_PAGE_TABLE_NOT_PRESENT:
+        case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
+        case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
+            *pfShared    = false;
+            *pfPageFlags = 0;
+            rc = VINF_SUCCESS;
+            break;
+
+        default:
+            break;
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+# endif /* DEBUG */
+
+# ifdef VBOX_STRICT
+
+/**
+ * @callback_method_impl{FNDBGCCMD, The '.pgmcheckduppages' command.}
+ */
+DECLCALLBACK(int) pgmR3CmdCheckDuplicatePages(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
+{
+    unsigned cBallooned = 0;
+    unsigned cShared    = 0;
+    unsigned cZero      = 0;
+    unsigned cUnique    = 0;
+    unsigned cDuplicate = 0;
+    unsigned cAllocZero = 0;
+    unsigned cPages     = 0;
+    NOREF(pCmd); NOREF(paArgs); NOREF(cArgs);
+    PVM      pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    pgmLock(pVM);
+
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
+    {
+        PPGMPAGE    pPage  = &pRam->aPages[0];
+        RTGCPHYS    GCPhys = pRam->GCPhys;
+        uint32_t    cLeft  = pRam->cb >> PAGE_SHIFT;
+        while (cLeft-- > 0)
+        {
+            if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
+            {
+                switch (PGM_PAGE_GET_STATE(pPage))
+                {
+                    case PGM_PAGE_STATE_ZERO:
+                        cZero++;
+                        break;
+
+                    case PGM_PAGE_STATE_BALLOONED:
+                        cBallooned++;
+                        break;
+
+                    case PGM_PAGE_STATE_SHARED:
+                        cShared++;
+                        break;
+
+                    case PGM_PAGE_STATE_ALLOCATED:
+                    case PGM_PAGE_STATE_WRITE_MONITORED:
+                    {
+                        /* Check if the page was allocated, but completely zero. */
+                        PGMPAGEMAPLOCK PgMpLck;
+                        const void    *pvPage;
+                        int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvPage, &PgMpLck);
+                        if (    RT_SUCCESS(rc)
+                            &&  ASMMemIsZeroPage(pvPage))
+                            cAllocZero++;
+                        else if (GMMR3IsDuplicatePage(pVM, PGM_PAGE_GET_PAGEID(pPage)))
+                            cDuplicate++;
+                        else
+                            cUnique++;
+                        if (RT_SUCCESS(rc))
+                            pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+                        break;
+                    }
+
+                    default:
+                        AssertFailed();
+                        break;
+                }
+            }
+
+            /* next */
+            pPage++;
+            GCPhys += PAGE_SIZE;
+            cPages++;
+            /* Give some feedback for every processed megabyte. */
+            if ((cPages & 0x7f) == 0)
+                pCmdHlp->pfnPrintf(pCmdHlp, NULL, ".");
+        }
+    }
+    pgmUnlock(pVM);
+
+    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "\nNumber of zero pages      %08x (%d MB)\n", cZero, cZero / 256);
+    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of alloczero pages %08x (%d MB)\n", cAllocZero, cAllocZero / 256);
+    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of ballooned pages %08x (%d MB)\n", cBallooned, cBallooned / 256);
+    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of shared pages    %08x (%d MB)\n", cShared, cShared / 256);
+    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of unique pages    %08x (%d MB)\n", cUnique, cUnique / 256);
+    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of duplicate pages %08x (%d MB)\n", cDuplicate, cDuplicate / 256);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @callback_method_impl{FNDBGCCMD, The '.pgmsharedmodules' command.}
+ */
+DECLCALLBACK(int) pgmR3CmdShowSharedModules(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
+{
+    NOREF(pCmd); NOREF(paArgs); NOREF(cArgs);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    pgmLock(pVM);
+    for (unsigned i = 0; i < RT_ELEMENTS(g_apSharedModules); i++)
+    {
+        if (g_apSharedModules[i])
+        {
+            pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Shared module %s (%s):\n", g_apSharedModules[i]->szName, g_apSharedModules[i]->szVersion);
+            for (unsigned j = 0; j < g_apSharedModules[i]->cRegions; j++)
+                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "--- Region %d: base %RGv size %x\n", j, g_apSharedModules[i]->aRegions[j].GCRegionAddr, g_apSharedModules[i]->aRegions[j].cbRegion);
+        }
+    }
+    pgmUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+# endif /* VBOX_STRICT*/
+#endif /* VBOX_WITH_PAGE_SHARING */
diff --git a/src/VBox/VMM/VMMR3/SELM.cpp b/src/VBox/VMM/VMMR3/SELM.cpp
new file mode 100644
index 00000000..341f8868
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/SELM.cpp
@@ -0,0 +1,671 @@
+/* $Id: SELM.cpp $ */
+/** @file
+ * SELM - The Selector Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_selm   SELM - The Selector Manager
+ *
+ * SELM takes care of GDT, LDT and TSS shadowing in raw-mode, and the injection
+ * of a few hyper selector for the raw-mode context.  In the hardware assisted
+ * virtualization mode its only task is to decode entries in the guest GDT or
+ * LDT once in a while.
+ *
+ * @see grp_selm
+ *
+ *
+ * @section seg_selm_shadowing   Shadowing
+ *
+ * SELMR3UpdateFromCPUM() and SELMR3SyncTSS() does the bulk synchronization
+ * work.  The three structures (GDT, LDT, TSS) are all shadowed wholesale atm.
+ * The idea is to do it in a more on-demand fashion when we get time.  There
+ * also a whole bunch of issues with the current synchronization of all three
+ * tables, see notes and todos in the code.
+ *
+ * When the guest makes changes to the GDT we will try update the shadow copy
+ * without involving SELMR3UpdateFromCPUM(), see selmGCSyncGDTEntry().
+ *
+ * When the guest make LDT changes we'll trigger a full resync of the LDT
+ * (SELMR3UpdateFromCPUM()), which, needless to say, isn't optimal.
+ *
+ * The TSS shadowing is limited to the fields we need to care about, namely SS0
+ * and ESP0.  The Patch Manager makes use of these.  We monitor updates to the
+ * guest TSS and will try keep our SS0 and ESP0 copies up to date this way
+ * rather than go the SELMR3SyncTSS() route.
+ *
+ * When in raw-mode SELM also injects a few extra GDT selectors which are used
+ * by the raw-mode (hyper) context.  These start their life at the high end of
+ * the table and will be relocated when the guest tries to make use of them...
+ * Well, that was that idea at least, only the code isn't quite there yet which
+ * is why we have trouble with guests which actually have a full sized GDT.
+ *
+ * So, the summary of the current GDT, LDT and TSS shadowing is that there is a
+ * lot of relatively simple and enjoyable work to be done, see @bugref{3267}.
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_SELM
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/dbgf.h>
+#include "SELMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+
+#include <iprt/assert.h>
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(void) selmR3InfoGdtGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) selmR3InfoLdtGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+//static DECLCALLBACK(void) selmR3InfoTssGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+
+/**
+ * Initializes the SELM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(int) SELMR3Init(PVM pVM)
+{
+    int rc;
+    LogFlow(("SELMR3Init\n"));
+
+    /*
+     * Assert alignment and sizes.
+     * (The TSS block requires contiguous back.)
+     */
+    AssertCompile(sizeof(pVM->selm.s) <= sizeof(pVM->selm.padding));    AssertRelease(sizeof(pVM->selm.s) <= sizeof(pVM->selm.padding));
+    AssertCompileMemberAlignment(VM, selm.s, 32);                       AssertRelease(!(RT_UOFFSETOF(VM, selm.s) & 31));
+
+    /*
+     * Register the saved state data unit.
+     */
+    rc = SSMR3RegisterStub(pVM, "selm", 1);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Statistics.
+     */
+    STAM_REG(    pVM, &pVM->selm.s.StatLoadHidSelGst,              STAMTYPE_COUNTER, "/SELM/LoadHidSel/LoadedGuest",   STAMUNIT_OCCURENCES, "SELMLoadHiddenSelectorReg: Loaded from guest tables.");
+    STAM_REG(    pVM, &pVM->selm.s.StatLoadHidSelShw,              STAMTYPE_COUNTER, "/SELM/LoadHidSel/LoadedShadow",  STAMUNIT_OCCURENCES, "SELMLoadHiddenSelectorReg: Loaded from shadow tables.");
+    STAM_REL_REG(pVM, &pVM->selm.s.StatLoadHidSelReadErrors,       STAMTYPE_COUNTER, "/SELM/LoadHidSel/GstReadErrors", STAMUNIT_OCCURENCES, "SELMLoadHiddenSelectorReg: Guest table read errors.");
+    STAM_REL_REG(pVM, &pVM->selm.s.StatLoadHidSelGstNoGood,        STAMTYPE_COUNTER, "/SELM/LoadHidSel/NoGoodGuest",   STAMUNIT_OCCURENCES, "SELMLoadHiddenSelectorReg: No good guest table entry.");
+
+    /*
+     * Register info handlers.
+     */
+    DBGFR3InfoRegisterInternalEx(pVM, "gdt", "Displays the guest GDT. No arguments.", &selmR3InfoGdtGuest, DBGFINFO_FLAGS_RUN_ON_EMT);
+    DBGFR3InfoRegisterInternalEx(pVM, "ldt", "Displays the guest LDT. No arguments.", &selmR3InfoLdtGuest, DBGFINFO_FLAGS_RUN_ON_EMT);
+    //DBGFR3InfoRegisterInternal(pVM, "tss", "Displays the guest TSS. No arguments.", &selmR3InfoTssGuest, DBGFINFO_FLAGS_RUN_ON_EMT);
+
+    return rc;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(void) SELMR3Relocate(PVM pVM)
+{
+    LogFlow(("SELMR3Relocate\n"));
+    RT_NOREF(pVM);
+}
+
+
+/**
+ * Terminates the SELM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(int) SELMR3Term(PVM pVM)
+{
+    NOREF(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * For the SELM component this means that any GDT/LDT/TSS monitors
+ * needs to be removed.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(void) SELMR3Reset(PVM pVM)
+{
+    LogFlow(("SELMR3Reset:\n"));
+    VM_ASSERT_EMT(pVM);
+    RT_NOREF(pVM);
+}
+
+
+/**
+ * Gets information about a 64-bit selector, SELMR3GetSelectorInfo helper.
+ *
+ * See SELMR3GetSelectorInfo for details.
+ *
+ * @returns VBox status code, see SELMR3GetSelectorInfo for details.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   Sel         The selector to get info about.
+ * @param   pSelInfo    Where to store the information.
+ */
+static int selmR3GetSelectorInfo64(PVMCPU pVCpu, RTSEL Sel, PDBGFSELINFO pSelInfo)
+{
+    /*
+     * Read it from the guest descriptor table.
+     */
+/** @todo this is bogus wrt the LDT/GDT limit on long selectors. */
+    X86DESC64   Desc;
+    RTGCPTR     GCPtrDesc;
+    if (!(Sel & X86_SEL_LDT))
+    {
+        /* GDT */
+        VBOXGDTR Gdtr;
+        CPUMGetGuestGDTR(pVCpu, &Gdtr);
+        if ((Sel | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
+            return VERR_INVALID_SELECTOR;
+        GCPtrDesc = Gdtr.pGdt + (Sel & X86_SEL_MASK);
+    }
+    else
+    {
+        /* LDT */
+        uint64_t GCPtrBase;
+        uint32_t cbLimit;
+        CPUMGetGuestLdtrEx(pVCpu, &GCPtrBase, &cbLimit);
+        if ((Sel | X86_SEL_RPL_LDT) > cbLimit)
+            return VERR_INVALID_SELECTOR;
+
+        /* calc the descriptor location. */
+        GCPtrDesc = GCPtrBase + (Sel & X86_SEL_MASK);
+    }
+
+    /* read the descriptor. */
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(Desc));
+    if (RT_FAILURE(rc))
+    {
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(X86DESC));
+        if (RT_FAILURE(rc))
+            return rc;
+        Desc.au64[1] = 0;
+    }
+
+    /*
+     * Extract the base and limit
+     * (We ignore the present bit here, which is probably a bit silly...)
+     */
+    pSelInfo->Sel     = Sel;
+    pSelInfo->fFlags  = DBGFSELINFO_FLAGS_LONG_MODE;
+    pSelInfo->u.Raw64 = Desc;
+    if (Desc.Gen.u1DescType)
+    {
+        /*
+         * 64-bit code selectors are wide open, it's not possible to detect
+         * 64-bit data or stack selectors without also dragging in assumptions
+         * about current CS (i.e. that's we're executing in 64-bit mode).  So,
+         * the selinfo user needs to deal with this in the context the info is
+         * used unfortunately.
+         */
+        if (    Desc.Gen.u1Long
+            &&  !Desc.Gen.u1DefBig
+            &&  (Desc.Gen.u4Type & X86_SEL_TYPE_CODE))
+        {
+            /* Note! We ignore the segment limit hacks that was added by AMD. */
+            pSelInfo->GCPtrBase = 0;
+            pSelInfo->cbLimit   = ~(RTGCUINTPTR)0;
+        }
+        else
+        {
+            pSelInfo->cbLimit   = X86DESC_LIMIT_G(&Desc);
+            pSelInfo->GCPtrBase = X86DESC_BASE(&Desc);
+        }
+        pSelInfo->SelGate = 0;
+    }
+    else if (   Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_LDT
+             || Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_TSS_AVAIL
+             || Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY)
+    {
+        /* Note. LDT descriptors are weird in long mode, we ignore the footnote
+           in the AMD manual here as a simplification. */
+        pSelInfo->GCPtrBase = X86DESC64_BASE(&Desc);
+        pSelInfo->cbLimit   = X86DESC_LIMIT_G(&Desc);
+        pSelInfo->SelGate   = 0;
+    }
+    else if (   Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE
+             || Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_TRAP_GATE
+             || Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_INT_GATE)
+    {
+        pSelInfo->cbLimit   = X86DESC64_BASE(&Desc);
+        pSelInfo->GCPtrBase = Desc.Gate.u16OffsetLow
+                            | ((uint32_t)Desc.Gate.u16OffsetHigh << 16)
+                            | ((uint64_t)Desc.Gate.u32OffsetTop << 32);
+        pSelInfo->SelGate   = Desc.Gate.u16Sel;
+        pSelInfo->fFlags   |= DBGFSELINFO_FLAGS_GATE;
+    }
+    else
+    {
+        pSelInfo->cbLimit   = 0;
+        pSelInfo->GCPtrBase = 0;
+        pSelInfo->SelGate   = 0;
+        pSelInfo->fFlags   |= DBGFSELINFO_FLAGS_INVALID;
+    }
+    if (!Desc.Gen.u1Present)
+        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_NOT_PRESENT;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for selmR3GetSelectorInfo32 and SELMR3GetShadowSelectorInfo that
+ * interprets a legacy descriptor table entry and fills in the selector info
+ * structure from it.
+ *
+ * @param  pSelInfo     Where to store the selector info. Only the fFlags and
+ *                      Sel members have been initialized.
+ * @param  pDesc        The legacy descriptor to parse.
+ */
+DECLINLINE(void) selmR3SelInfoFromDesc32(PDBGFSELINFO pSelInfo, PCX86DESC pDesc)
+{
+    pSelInfo->u.Raw64.au64[1] = 0;
+    pSelInfo->u.Raw = *pDesc;
+    if (    pDesc->Gen.u1DescType
+        ||  !(pDesc->Gen.u4Type & 4))
+    {
+        pSelInfo->cbLimit   = X86DESC_LIMIT_G(pDesc);
+        pSelInfo->GCPtrBase = X86DESC_BASE(pDesc);
+        pSelInfo->SelGate   = 0;
+    }
+    else if (pDesc->Gen.u4Type != X86_SEL_TYPE_SYS_UNDEFINED4)
+    {
+        pSelInfo->cbLimit = 0;
+        if (pDesc->Gen.u4Type == X86_SEL_TYPE_SYS_TASK_GATE)
+            pSelInfo->GCPtrBase = 0;
+        else
+            pSelInfo->GCPtrBase = pDesc->Gate.u16OffsetLow
+                                | (uint32_t)pDesc->Gate.u16OffsetHigh << 16;
+        pSelInfo->SelGate = pDesc->Gate.u16Sel;
+        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_GATE;
+    }
+    else
+    {
+        pSelInfo->cbLimit = 0;
+        pSelInfo->GCPtrBase = 0;
+        pSelInfo->SelGate = 0;
+        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_INVALID;
+    }
+    if (!pDesc->Gen.u1Present)
+        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_NOT_PRESENT;
+}
+
+
+/**
+ * Gets information about a 64-bit selector, SELMR3GetSelectorInfo helper.
+ *
+ * See SELMR3GetSelectorInfo for details.
+ *
+ * @returns VBox status code, see SELMR3GetSelectorInfo for details.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   Sel         The selector to get info about.
+ * @param   pSelInfo    Where to store the information.
+ */
+static int selmR3GetSelectorInfo32(PVMCPU pVCpu, RTSEL Sel, PDBGFSELINFO pSelInfo)
+{
+    /*
+     * Read the descriptor entry
+     */
+    pSelInfo->fFlags = 0;
+    if (CPUMIsGuestInProtectedMode(pVCpu))
+    {
+        /*
+         * Read it from the guest descriptor table.
+         */
+        pSelInfo->fFlags = DBGFSELINFO_FLAGS_PROT_MODE;
+
+        RTGCPTR GCPtrDesc;
+        if (!(Sel & X86_SEL_LDT))
+        {
+            /* GDT */
+            VBOXGDTR Gdtr;
+            CPUMGetGuestGDTR(pVCpu, &Gdtr);
+            if ((Sel | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
+                return VERR_INVALID_SELECTOR;
+            GCPtrDesc = Gdtr.pGdt + (Sel & X86_SEL_MASK);
+        }
+        else
+        {
+            /* LDT */
+            uint64_t GCPtrBase;
+            uint32_t cbLimit;
+            CPUMGetGuestLdtrEx(pVCpu, &GCPtrBase, &cbLimit);
+            if ((Sel | X86_SEL_RPL_LDT) > cbLimit)
+                return VERR_INVALID_SELECTOR;
+
+            /* calc the descriptor location. */
+            GCPtrDesc = GCPtrBase + (Sel & X86_SEL_MASK);
+        }
+
+        /* read the descriptor. */
+        X86DESC Desc;
+        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(Desc));
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Extract the base and limit or sel:offset for gates.
+             */
+            pSelInfo->Sel = Sel;
+            selmR3SelInfoFromDesc32(pSelInfo, &Desc);
+
+            return VINF_SUCCESS;
+        }
+        return rc;
+    }
+
+    /*
+     * We're in real mode.
+     */
+    pSelInfo->Sel       = Sel;
+    pSelInfo->GCPtrBase = Sel << 4;
+    pSelInfo->cbLimit   = 0xffff;
+    pSelInfo->fFlags    = DBGFSELINFO_FLAGS_REAL_MODE;
+    pSelInfo->u.Raw64.au64[0] = 0;
+    pSelInfo->u.Raw64.au64[1] = 0;
+    pSelInfo->SelGate   = 0;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets information about a selector.
+ *
+ * Intended for the debugger mostly and will prefer the guest descriptor tables
+ * over the shadow ones.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_INVALID_SELECTOR if the selector isn't fully inside the
+ *          descriptor table.
+ * @retval  VERR_SELECTOR_NOT_PRESENT if the LDT is invalid or not present. This
+ *          is not returned if the selector itself isn't present, you have to
+ *          check that for yourself (see DBGFSELINFO::fFlags).
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT or VERR_PAGE_NOT_PRESENT if the
+ *          pagetable or page backing the selector table wasn't present.
+ * @returns Other VBox status code on other errors.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   Sel         The selector to get info about.
+ * @param   pSelInfo    Where to store the information.
+ */
+VMMR3DECL(int) SELMR3GetSelectorInfo(PVMCPU pVCpu, RTSEL Sel, PDBGFSELINFO pSelInfo)
+{
+    AssertPtr(pSelInfo);
+    if (CPUMIsGuestInLongMode(pVCpu))
+        return selmR3GetSelectorInfo64(pVCpu, Sel, pSelInfo);
+    return selmR3GetSelectorInfo32(pVCpu, Sel, pSelInfo);
+}
+
+
+/**
+ * Formats a descriptor.
+ *
+ * @param   Desc        Descriptor to format.
+ * @param   Sel         Selector number.
+ * @param   pszOutput   Output buffer.
+ * @param   cchOutput   Size of output buffer.
+ */
+static void selmR3FormatDescriptor(X86DESC Desc, RTSEL Sel, char *pszOutput, size_t cchOutput)
+{
+    /*
+     * Make variable description string.
+     */
+    static struct
+    {
+        unsigned    cch;
+        const char *psz;
+    } const aTypes[32] =
+    {
+#define STRENTRY(str) { sizeof(str) - 1, str }
+        /* system */
+        STRENTRY("Reserved0 "),                  /* 0x00 */
+        STRENTRY("TSS16Avail "),                 /* 0x01 */
+        STRENTRY("LDT "),                        /* 0x02 */
+        STRENTRY("TSS16Busy "),                  /* 0x03 */
+        STRENTRY("Call16 "),                     /* 0x04 */
+        STRENTRY("Task "),                       /* 0x05 */
+        STRENTRY("Int16 "),                      /* 0x06 */
+        STRENTRY("Trap16 "),                     /* 0x07 */
+        STRENTRY("Reserved8 "),                  /* 0x08 */
+        STRENTRY("TSS32Avail "),                 /* 0x09 */
+        STRENTRY("ReservedA "),                  /* 0x0a */
+        STRENTRY("TSS32Busy "),                  /* 0x0b */
+        STRENTRY("Call32 "),                     /* 0x0c */
+        STRENTRY("ReservedD "),                  /* 0x0d */
+        STRENTRY("Int32 "),                      /* 0x0e */
+        STRENTRY("Trap32 "),                     /* 0x0f */
+        /* non system */
+        STRENTRY("DataRO "),                     /* 0x10 */
+        STRENTRY("DataRO Accessed "),            /* 0x11 */
+        STRENTRY("DataRW "),                     /* 0x12 */
+        STRENTRY("DataRW Accessed "),            /* 0x13 */
+        STRENTRY("DataDownRO "),                 /* 0x14 */
+        STRENTRY("DataDownRO Accessed "),        /* 0x15 */
+        STRENTRY("DataDownRW "),                 /* 0x16 */
+        STRENTRY("DataDownRW Accessed "),        /* 0x17 */
+        STRENTRY("CodeEO "),                     /* 0x18 */
+        STRENTRY("CodeEO Accessed "),            /* 0x19 */
+        STRENTRY("CodeER "),                     /* 0x1a */
+        STRENTRY("CodeER Accessed "),            /* 0x1b */
+        STRENTRY("CodeConfEO "),                 /* 0x1c */
+        STRENTRY("CodeConfEO Accessed "),        /* 0x1d */
+        STRENTRY("CodeConfER "),                 /* 0x1e */
+        STRENTRY("CodeConfER Accessed ")         /* 0x1f */
+#undef SYSENTRY
+    };
+#define ADD_STR(psz, pszAdd) do { strcpy(psz, pszAdd); psz += strlen(pszAdd); } while (0)
+    char        szMsg[128];
+    char       *psz = &szMsg[0];
+    unsigned    i = Desc.Gen.u1DescType << 4 | Desc.Gen.u4Type;
+    memcpy(psz, aTypes[i].psz, aTypes[i].cch);
+    psz += aTypes[i].cch;
+
+    if (Desc.Gen.u1Present)
+        ADD_STR(psz, "Present ");
+    else
+        ADD_STR(psz, "Not-Present ");
+    if (Desc.Gen.u1Granularity)
+        ADD_STR(psz, "Page ");
+    if (Desc.Gen.u1DefBig)
+        ADD_STR(psz, "32-bit ");
+    else
+        ADD_STR(psz, "16-bit ");
+#undef ADD_STR
+    *psz = '\0';
+
+    /*
+     * Limit and Base and format the output.
+     */
+    uint32_t    u32Limit = X86DESC_LIMIT_G(&Desc);
+    uint32_t    u32Base  = X86DESC_BASE(&Desc);
+
+    RTStrPrintf(pszOutput, cchOutput, "%04x - %08x %08x - base=%08x limit=%08x dpl=%d %s",
+                Sel, Desc.au32[0], Desc.au32[1], u32Base, u32Limit, Desc.Gen.u2Dpl, szMsg);
+}
+
+
+/**
+ * Dumps a descriptor.
+ *
+ * @param   Desc    Descriptor to dump.
+ * @param   Sel     Selector number.
+ * @param   pszMsg  Message to prepend the log entry with.
+ */
+VMMR3DECL(void) SELMR3DumpDescriptor(X86DESC Desc, RTSEL Sel, const char *pszMsg)
+{
+#ifdef LOG_ENABLED
+    if (LogIsEnabled())
+    {
+        char szOutput[128];
+        selmR3FormatDescriptor(Desc, Sel, &szOutput[0], sizeof(szOutput));
+        Log(("%s: %s\n", pszMsg, szOutput));
+    }
+#else
+    RT_NOREF3(Desc, Sel, pszMsg);
+#endif
+}
+
+
+/**
+ * Display the guest gdt.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) selmR3InfoGdtGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /** @todo SMP support! */
+    PVMCPU      pVCpu = pVM->apCpusR3[0];
+
+    VBOXGDTR    GDTR;
+    CPUMGetGuestGDTR(pVCpu, &GDTR);
+    RTGCPTR     GCPtrGDT = GDTR.pGdt;
+    unsigned    cGDTs = ((unsigned)GDTR.cbGdt + 1) / sizeof(X86DESC);
+
+    pHlp->pfnPrintf(pHlp, "Guest GDT (GCAddr=%RGv limit=%x):\n", GCPtrGDT, GDTR.cbGdt);
+    for (unsigned iGDT = 0; iGDT < cGDTs; iGDT++, GCPtrGDT += sizeof(X86DESC))
+    {
+        X86DESC GDTE;
+        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &GDTE, GCPtrGDT, sizeof(GDTE));
+        if (RT_SUCCESS(rc))
+        {
+            if (GDTE.Gen.u1Present)
+            {
+                char szOutput[128];
+                selmR3FormatDescriptor(GDTE, iGDT << X86_SEL_SHIFT, &szOutput[0], sizeof(szOutput));
+                pHlp->pfnPrintf(pHlp, "%s\n", szOutput);
+            }
+        }
+        else if (rc == VERR_PAGE_NOT_PRESENT)
+        {
+            if ((GCPtrGDT & PAGE_OFFSET_MASK) + sizeof(X86DESC) - 1 < sizeof(X86DESC))
+                pHlp->pfnPrintf(pHlp, "%04x - page not present (GCAddr=%RGv)\n", iGDT << X86_SEL_SHIFT, GCPtrGDT);
+        }
+        else
+            pHlp->pfnPrintf(pHlp, "%04x - read error rc=%Rrc GCAddr=%RGv\n", iGDT << X86_SEL_SHIFT, rc, GCPtrGDT);
+    }
+    NOREF(pszArgs);
+}
+
+
+/**
+ * Display the guest ldt.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) selmR3InfoLdtGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /** @todo SMP support! */
+    PVMCPU   pVCpu = pVM->apCpusR3[0];
+
+    uint64_t GCPtrLdt;
+    uint32_t cbLdt;
+    RTSEL    SelLdt = CPUMGetGuestLdtrEx(pVCpu, &GCPtrLdt, &cbLdt);
+    if (!(SelLdt & X86_SEL_MASK_OFF_RPL))
+    {
+        pHlp->pfnPrintf(pHlp, "Guest LDT (Sel=%x): Null-Selector\n", SelLdt);
+        return;
+    }
+
+    pHlp->pfnPrintf(pHlp, "Guest LDT (Sel=%x GCAddr=%RX64 limit=%x):\n", SelLdt, GCPtrLdt, cbLdt);
+    unsigned cLdts  = (cbLdt + 1) >> X86_SEL_SHIFT;
+    for (unsigned iLdt = 0; iLdt < cLdts; iLdt++, GCPtrLdt += sizeof(X86DESC))
+    {
+        X86DESC LdtE;
+        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &LdtE, GCPtrLdt, sizeof(LdtE));
+        if (RT_SUCCESS(rc))
+        {
+            if (LdtE.Gen.u1Present)
+            {
+                char szOutput[128];
+                selmR3FormatDescriptor(LdtE, (iLdt << X86_SEL_SHIFT) | X86_SEL_LDT, &szOutput[0], sizeof(szOutput));
+                pHlp->pfnPrintf(pHlp, "%s\n", szOutput);
+            }
+        }
+        else if (rc == VERR_PAGE_NOT_PRESENT)
+        {
+            if ((GCPtrLdt & PAGE_OFFSET_MASK) + sizeof(X86DESC) - 1 < sizeof(X86DESC))
+                pHlp->pfnPrintf(pHlp, "%04x - page not present (GCAddr=%RGv)\n", (iLdt << X86_SEL_SHIFT) | X86_SEL_LDT, GCPtrLdt);
+        }
+        else
+            pHlp->pfnPrintf(pHlp, "%04x - read error rc=%Rrc GCAddr=%RGv\n", (iLdt << X86_SEL_SHIFT) | X86_SEL_LDT, rc, GCPtrLdt);
+    }
+    NOREF(pszArgs);
+}
+
+
+/**
+ * Dumps the guest GDT
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(void) SELMR3DumpGuestGDT(PVM pVM)
+{
+    DBGFR3Info(pVM->pUVM, "gdt", NULL, NULL);
+}
+
+
+/**
+ * Dumps the guest LDT
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(void) SELMR3DumpGuestLDT(PVM pVM)
+{
+    DBGFR3Info(pVM->pUVM, "ldt", NULL, NULL);
+}
+
diff --git a/src/VBox/VMM/VMMR3/SSM.cpp b/src/VBox/VMM/VMMR3/SSM.cpp
new file mode 100644
index 00000000..4d270d80
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/SSM.cpp
@@ -0,0 +1,9923 @@
+/* $Id: SSM.cpp $ */
+/** @file
+ * SSM - Saved State Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_ssm        SSM - The Saved State Manager
+ *
+ * The Saved State Manager (SSM) implements facilities for saving and loading a
+ * VM state in a structural manner using callbacks for named data units.
+ *
+ * At init time each of the VMM components, Devices, Drivers and one or two
+ * other things will register data units which they need to save and restore.
+ * Each unit have a unique name (ascii), instance number, and a set of callbacks
+ * associated with it.  The name will be used to identify the unit during
+ * restore.  The callbacks are for the two operations, save and restore.  There
+ * are three callbacks for each of the two - a prepare, a execute and a complete
+ * - giving each component ample opportunity to perform actions both before and
+ * afterwards.
+ *
+ * The SSM provides a number of APIs for encoding and decoding the data: @see
+ * grp_ssm
+ *
+ *
+ *
+ * @section sec_ssm_live_snapshots  Live Snapshots
+ *
+ * The live snapshots feature (LS) is similar to teleportation (TP) and was a
+ * natural first step when implementing TP.  The main differences between LS and
+ * TP are that after a live snapshot we will have a saved state file, disk image
+ * snapshots, and the VM will still be running.
+ *
+ * Compared to normal saved stated and snapshots, the difference is in that the
+ * VM is running while we do most of the saving.  Prior to LS, there was only
+ * one round of callbacks during saving and the VM was paused during it.  With
+ * LS there are 1 or more passes while the VM is still running and a final one
+ * after it has been paused.  The runtime passes are executed on a dedicated
+ * thread running at at the same priority as the EMTs so that the saving doesn't
+ * starve or lose in scheduling questions (note: not implemented yet). The final
+ * pass is done on EMT(0).
+ *
+ * There are a couple of common reasons why LS and TP will fail:
+ *   - Memory configuration changed (PCI memory mappings).
+ *   - Takes too long (TP) / Too much output (LS).
+ *
+ *
+ * The live saving sequence is something like this:
+ *
+ *      -# SSMR3LiveSave is called on EMT0.  It returns a saved state
+ *         handle.
+ *      -# SSMR3LiveDoStep1 is called on a non-EMT.  This will save the major
+ *         parts of the state while the VM may still be running.
+ *      -# The VM is suspended.
+ *      -# SSMR3LiveDoStep2 is called on EMT0 to save the remainder of the state
+ *         in the normal way.
+ *      -# The client does any necessary reconfiguration of harddisks and
+ *         similar.
+ *      -# SSMR3LiveDone is called on EMT0 to close the handle.
+ *      -# The VM is resumed or powered off and destroyed.
+ *
+ *
+ * @section sec_ssm_teleportation   Teleportation
+ *
+ * As mentioned in the previous section, the main differences between this and
+ * live snapshots are in where the saved state is written and what state the
+ * local VM is in afterwards - at least from the VMM point of view.  The
+ * necessary administrative work - establishing the connection to the remote
+ * machine, cloning the VM config on it and doing lowlevel saved state data
+ * transfer - is taken care of by layer above the VMM (i.e.  Main).
+ *
+ * The SSM data format was made streamable for the purpose of teleportation
+ * (v1.2 was the last non-streamable version).
+ *
+ *
+ * @section sec_ssm_format          Saved State Format
+ *
+ * The stream format starts with a header (SSMFILEHDR) that indicates the
+ * version and such things, it is followed by zero or more saved state units
+ * (name + instance + pass), and the stream concludes with a footer
+ * (SSMFILEFTR) that contains unit counts and optionally a checksum for the
+ * entire file.  (In version 1.2 and earlier, the checksum was in the header and
+ * there was no footer.  This meant that the header was updated after the entire
+ * file was written.)
+ *
+ * The saved state units each starts with a variable sized header
+ * (SSMFILEUNITHDRV2) that contains the name, instance and pass.  The data
+ * follows the header and is encoded as records with a 2-8 byte record header
+ * indicating the type, flags and size.  The first byte in the record header
+ * indicates the type and flags:
+ *
+ *   - bits 0..3: Record type:
+ *       - type 0: Invalid.
+ *       - type 1: Terminator with CRC-32 and unit size.
+ *       - type 2: Raw data record.
+ *       - type 3: Raw data compressed by LZF. The data is prefixed by a 8-bit
+ *                 field containing the length of the uncompressed data given in
+ *                 1KB units.
+ *       - type 4: Zero data. The record header is followed by a 8-bit field
+ *                 counting the length of the zero data given in 1KB units.
+ *       - type 5: Named data - length prefixed name followed by the data. This
+ *                 type is not implemented yet as we're missing the API part, so
+ *                 the type assignment is tentative.
+ *       - types 6 thru 15 are current undefined.
+ *   - bit 4: Important (set), can be skipped (clear).
+ *   - bit 5: Undefined flag, must be zero.
+ *   - bit 6: Undefined flag, must be zero.
+ *   - bit 7: "magic" bit, always set.
+ *
+ * Record header byte 2 (optionally thru 7) is the size of the following data
+ * encoded in UTF-8 style.  To make buffering simpler and more efficient during
+ * the save operation, the strict checks enforcing optimal encoding has been
+ * relaxed for the 2 and 3 byte encodings.
+ *
+ * (In version 1.2 and earlier the unit data was compressed and not record
+ * based. The unit header contained the compressed size of the data, i.e. it
+ * needed updating after the data was written.)
+ *
+ *
+ * @section sec_ssm_future          Future Changes
+ *
+ * There are plans to extend SSM to make it easier to be both backwards and
+ * (somewhat) forwards compatible.  One of the new features will be being able
+ * to classify units and data items as unimportant (added to the format in
+ * v2.0).  Another suggested feature is naming data items (also added to the
+ * format in v2.0), perhaps by extending the SSMR3PutStruct API.  Both features
+ * will require API changes, the naming may possibly require both buffering of
+ * the stream as well as some helper managing them.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_SSM
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/mm.h>
+#include "SSMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/version.h>
+
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/crc.h>
+#include <iprt/file.h>
+#include <iprt/mem.h>
+#include <iprt/param.h>
+#include <iprt/thread.h>
+#include <iprt/semaphore.h>
+#include <iprt/string.h>
+#include <iprt/uuid.h>
+#include <iprt/zip.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The max length of a unit name. */
+#define SSM_MAX_NAME_SIZE                       48
+
+/** Saved state file magic base string. */
+#define SSMFILEHDR_MAGIC_BASE                   "\177VirtualBox SavedState "
+/** Saved state file magic indicating version 1.x. */
+#define SSMFILEHDR_MAGIC_V1_X                   "\177VirtualBox SavedState V1."
+/** Saved state file v1.1 magic. */
+#define SSMFILEHDR_MAGIC_V1_1                   "\177VirtualBox SavedState V1.1\n"
+/** Saved state file v1.2 magic. */
+#define SSMFILEHDR_MAGIC_V1_2                   "\177VirtualBox SavedState V1.2\n\0\0\0"
+/** Saved state file v2.0 magic. */
+#define SSMFILEHDR_MAGIC_V2_0                   "\177VirtualBox SavedState V2.0\n\0\0\0"
+
+/** @name SSMFILEHDR::fFlags
+ * @{ */
+/** The stream is checksummed up to the footer using CRC-32. */
+#define SSMFILEHDR_FLAGS_STREAM_CRC32           RT_BIT_32(0)
+/** Indicates that the file was produced by a live save. */
+#define SSMFILEHDR_FLAGS_STREAM_LIVE_SAVE       RT_BIT_32(1)
+/** @} */
+
+/** The directory magic. */
+#define SSMFILEDIR_MAGIC                        "\nDir\n\0\0"
+
+/** Saved state file v2.0 magic. */
+#define SSMFILEFTR_MAGIC                        "\nFooter"
+
+/** Data unit magic. */
+#define SSMFILEUNITHDR_MAGIC                    "\nUnit\n\0"
+/** Data end marker magic. */
+#define SSMFILEUNITHDR_END                      "\nTheEnd"
+
+
+/** @name Record Types (data unit)
+ * @{ */
+/** The record type mask. */
+#define SSM_REC_TYPE_MASK                       UINT8_C(0x0f)
+/** Invalid record. */
+#define SSM_REC_TYPE_INVALID                    0
+/** Normal termination record, see SSMRECTERM. */
+#define SSM_REC_TYPE_TERM                       1
+/** Raw data. The data follows the size field without further ado. */
+#define SSM_REC_TYPE_RAW                        2
+/** Raw data compressed by LZF.
+ * The record header is followed by a 8-bit field containing the size of the
+ * uncompressed data in 1KB units.  The compressed data is after it. */
+#define SSM_REC_TYPE_RAW_LZF                    3
+/** Raw zero data.
+ * The record header is followed by a 8-bit field containing the size of the
+ * zero data in 1KB units. */
+#define SSM_REC_TYPE_RAW_ZERO                   4
+/** Named data items.
+ * A length prefix zero terminated string (i.e. max 255) followed by the data.  */
+#define SSM_REC_TYPE_NAMED                      5
+/** Macro for validating the record type.
+ * This can be used with the flags+type byte, no need to mask out the type first. */
+#define SSM_REC_TYPE_IS_VALID(u8Type)           (   ((u8Type) & SSM_REC_TYPE_MASK) >  SSM_REC_TYPE_INVALID \
+                                                 && ((u8Type) & SSM_REC_TYPE_MASK) <= SSM_REC_TYPE_NAMED )
+/** @} */
+
+/** The flag mask. */
+#define SSM_REC_FLAGS_MASK                      UINT8_C(0xf0)
+/** The record is important if this flag is set, if clear it can be omitted. */
+#define SSM_REC_FLAGS_IMPORTANT                 UINT8_C(0x10)
+/** This flag is always set. */
+#define SSM_REC_FLAGS_FIXED                     UINT8_C(0x80)
+/** Macro for validating the flags.
+ * No need to mask the flags out of the flags+type byte before invoking this macro.  */
+#define SSM_REC_FLAGS_ARE_VALID(fFlags)         ( ((fFlags) & UINT8_C(0xe0)) == UINT8_C(0x80) )
+
+/** Macro for validating the type and flags byte in a data record. */
+#define SSM_REC_ARE_TYPE_AND_FLAGS_VALID(u8)    ( SSM_REC_FLAGS_ARE_VALID(u8) && SSM_REC_TYPE_IS_VALID(u8) )
+
+/** @name SSMRECTERM::fFlags
+ * @{ */
+/** There is a CRC-32 value for the stream. */
+#define SSMRECTERM_FLAGS_CRC32                  UINT16_C(0x0001)
+/** @} */
+
+/** Start structure magic. (Isaac Asimov) */
+#define SSMR3STRUCT_BEGIN                       UINT32_C(0x19200102)
+/** End structure magic. (Isaac Asimov) */
+#define SSMR3STRUCT_END                         UINT32_C(0x19920406)
+
+
+/** Number of bytes to log in Log2 and Log4 statements. */
+#define SSM_LOG_BYTES                           16
+
+/** SSMHANDLE::fCancelled value indicating that the operation has been
+ *  cancelled. */
+#define SSMHANDLE_CANCELLED                     UINT32_C(0xdeadbeef)
+/** SSMHANDLE::fCancelled value indicating no cancellation. */
+#define SSMHANDLE_OK                            UINT32_C(0x77777777)
+
+
+/** Macro for checking the u32CRC field of a structure.
+ * The Msg can assume there are  u32ActualCRC and u32CRC in the context. */
+#define SSM_CHECK_CRC32_RET(p, cb, Msg) \
+    do \
+    { \
+        uint32_t u32CRC = (p)->u32CRC; \
+        (p)->u32CRC = 0; \
+        uint32_t u32ActualCRC = RTCrc32((p), (cb)); \
+        (p)->u32CRC = u32CRC; \
+        AssertLogRelMsgReturn(u32ActualCRC == u32CRC, Msg, VERR_SSM_INTEGRITY_CRC); \
+    } while (0)
+
+/** The number of bytes to compress is one block.
+ * Must be a multiple of 1KB.  */
+#define SSM_ZIP_BLOCK_SIZE                      _4K
+AssertCompile(SSM_ZIP_BLOCK_SIZE / _1K * _1K == SSM_ZIP_BLOCK_SIZE);
+
+
+/**
+ * Asserts that the handle is writable and returns with VERR_SSM_INVALID_STATE
+ * if it isn't.
+ */
+#define SSM_ASSERT_WRITEABLE_RET(pSSM) \
+    AssertMsgReturn(   pSSM->enmOp == SSMSTATE_SAVE_EXEC \
+                    || pSSM->enmOp == SSMSTATE_LIVE_EXEC,\
+                    ("Invalid state %d\n", pSSM->enmOp), VERR_SSM_INVALID_STATE);
+
+/**
+ * Asserts that the handle is readable and returns with VERR_SSM_INVALID_STATE
+ * if it isn't.
+ */
+#define SSM_ASSERT_READABLE_RET(pSSM) \
+    AssertMsgReturn(   pSSM->enmOp == SSMSTATE_LOAD_EXEC \
+                    || pSSM->enmOp == SSMSTATE_OPEN_READ,\
+                    ("Invalid state %d\n", pSSM->enmOp), VERR_SSM_INVALID_STATE);
+
+/** Checks for cancellation and returns if pending.
+ * Sets SSMHANDLE::rc to VERR_SSM_CANCELLED (if it still indicates success) and
+ * then returns SSMHANDLE::rc. (Debug logging only.) */
+#define SSM_CHECK_CANCELLED_RET(pSSM) \
+    do \
+    { \
+        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED)) \
+        { \
+            LogFlow(("%Rfn: Cancelled -> VERR_SSM_CANCELLED\n", __PRETTY_FUNCTION__)); \
+            if (RT_SUCCESS((pSSM)->rc)) \
+                (pSSM)->rc = VERR_SSM_CANCELLED; \
+            return (pSSM)->rc; \
+        } \
+    } while (0)
+
+/**
+ * Asserts that the handle is somewhat valid.  No returns as this is just a
+ * simple safeguard for catching bad API calls. */
+#define SSM_ASSERT_VALID_HANDLE(pSSM) \
+    do \
+    { \
+        AssertPtr(pSSM); \
+        Assert(pSSM->enmOp > SSMSTATE_INVALID && pSSM->enmOp < SSMSTATE_END); \
+    } while (0)
+
+
+/** @def SSM_HOST_IS_MSC_32
+ * Set to 1 if the host is 32-bit MSC, otherwise set to 0.
+ *   */
+#if defined(_MSC_VER) && HC_ARCH_BITS == 32
+# define SSM_HOST_IS_MSC_32     1
+#else
+# define SSM_HOST_IS_MSC_32     0
+#endif
+
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/** SSM state. */
+typedef enum SSMSTATE
+{
+    SSMSTATE_INVALID = 0,
+    SSMSTATE_LIVE_PREP,
+    SSMSTATE_LIVE_STEP1,
+    SSMSTATE_LIVE_EXEC,
+    SSMSTATE_LIVE_VOTE,
+    SSMSTATE_LIVE_STEP2,
+    SSMSTATE_SAVE_PREP,
+    SSMSTATE_SAVE_EXEC,
+    SSMSTATE_SAVE_DONE,
+    SSMSTATE_LOAD_PREP,
+    SSMSTATE_LOAD_EXEC,
+    SSMSTATE_LOAD_DONE,
+    SSMSTATE_OPEN_READ,
+    SSMSTATE_END
+} SSMSTATE;
+
+
+/** Pointer to a SSM stream buffer. */
+typedef struct SSMSTRMBUF *PSSMSTRMBUF;
+/**
+ * A SSM stream buffer.
+ */
+typedef struct SSMSTRMBUF
+{
+    /** The buffer data. */
+    uint8_t                 abData[_64K];
+
+    /** The stream position of this buffer. */
+    uint64_t                offStream;
+    /** The amount of buffered data. */
+    uint32_t                cb;
+    /** End of stream indicator (for read streams only). */
+    bool                    fEndOfStream;
+    /** The nano timestamp set by ssmR3StrmGetFreeBuf. */
+    uint64_t                NanoTS;
+    /** Pointer to the next buffer in the chain. */
+    PSSMSTRMBUF volatile    pNext;
+} SSMSTRMBUF;
+
+/**
+ * SSM stream.
+ *
+ * This is a typical producer / consumer setup with a dedicated I/O thread and
+ * fixed number of buffers for read ahead and write back.
+ */
+typedef struct SSMSTRM
+{
+    /** The stream method table. */
+    PCSSMSTRMOPS            pOps;
+    /** The user argument for the stream methods.
+     * For file based streams, this is the file handle and not a pointer. */
+    void                   *pvUser;
+
+    /** Write (set) or read (clear) stream. */
+    bool                    fWrite;
+    /** Termination indicator. */
+    bool volatile           fTerminating;
+    /** Indicates whether it is necessary to seek before the next buffer is
+     *  read from the stream.  This is used to avoid a seek in ssmR3StrmPeekAt. */
+    bool                    fNeedSeek;
+    /** Stream error status. */
+    int32_t volatile        rc;
+    /** The handle of the I/O thread. This is set to nil when not active. */
+    RTTHREAD                hIoThread;
+    /** Where to seek to. */
+    uint64_t                offNeedSeekTo;
+
+    /** The head of the consumer queue.
+     * For save the consumer is the I/O thread.  For load the I/O thread is the
+     * producer. */
+    PSSMSTRMBUF volatile    pHead;
+    /** Chain of free buffers.
+     * The consumer/producer roles are the inverse of pHead.  */
+    PSSMSTRMBUF volatile    pFree;
+    /** Event that's signalled when pHead is updated. */
+    RTSEMEVENT              hEvtHead;
+    /** Event that's signalled when pFree is updated. */
+    RTSEMEVENT              hEvtFree;
+
+    /** List of pending buffers that has been dequeued from pHead and reversed. */
+    PSSMSTRMBUF             pPending;
+    /** Pointer to the current buffer. */
+    PSSMSTRMBUF             pCur;
+    /** The stream offset of the current buffer. */
+    uint64_t                offCurStream;
+    /** The current buffer offset. */
+    uint32_t                off;
+    /** Whether we're checksumming reads/writes. */
+    bool                    fChecksummed;
+    /** The stream CRC if fChecksummed is set. */
+    uint32_t                u32StreamCRC;
+    /** How far into the buffer u32StreamCRC is up-to-date.
+     * This may lag behind off as it's desirable to checksum as large blocks as
+     * possible.  */
+    uint32_t                offStreamCRC;
+} SSMSTRM;
+/** Pointer to a SSM stream. */
+typedef SSMSTRM *PSSMSTRM;
+
+
+/**
+ * Handle structure.
+ */
+typedef struct SSMHANDLE
+{
+    /** Stream/buffer manager. */
+    SSMSTRM                 Strm;
+
+    /** Pointer to the VM. */
+    PVM                     pVM;
+    /** The current operation. */
+    SSMSTATE                enmOp;
+    /** What to do after save completes. (move the enum) */
+    SSMAFTER                enmAfter;
+    /** Flag indicating that the operation has been cancelled. */
+    uint32_t volatile       fCancelled;
+    /** The current rc of the save operation. */
+    int32_t                 rc;
+    /** Number of compressed bytes left in the current data unit (V1). */
+    uint64_t                cbUnitLeftV1;
+    /** The current compressed? offset into the data unit. */
+    uint64_t                offUnit;
+    /** The current user data offset into the unit (debug purposes). */
+    uint64_t                offUnitUser;
+    /** Indicates that this is a live save or restore operation. */
+    bool                    fLiveSave;
+
+    /** Pointer to the progress callback function. */
+    PFNVMPROGRESS           pfnProgress;
+    /** User specified argument to the callback function. */
+    void                   *pvUser;
+    /** Next completion percentage. (corresponds to offEstProgress) */
+    unsigned                uPercent;
+    /** The position of the next progress callback in the estimated file. */
+    uint64_t                offEstProgress;
+    /** The estimated total byte count.
+     * (Only valid after the prep.) */
+    uint64_t                cbEstTotal;
+    /** Current position in the estimated file. */
+    uint64_t                offEst;
+    /** End of current unit in the estimated file. */
+    uint64_t                offEstUnitEnd;
+    /** The amount of % we reserve for the 'live' stage */
+    unsigned                uPercentLive;
+    /** The amount of % we reserve for the 'prepare' phase */
+    unsigned                uPercentPrepare;
+    /** The amount of % we reserve for the 'done' stage */
+    unsigned                uPercentDone;
+    /** The lowest value reported via SSMR3HandleReportLivePercent during one
+     * vote run. */
+    unsigned                uReportedLivePercent;
+    /** The filename, NULL if remote stream. */
+    const char             *pszFilename;
+
+    union
+    {
+        /** Write data. */
+        struct
+        {
+            /** Offset into the databuffer. */
+            uint32_t        offDataBuffer;
+            /** Space for the record header.  */
+            uint8_t         abRecHdr[1+7];
+            /** Data buffer. */
+            uint8_t         abDataBuffer[4096];
+            /** The maximum downtime given as milliseconds. */
+            uint32_t        cMsMaxDowntime;
+        } Write;
+
+        /** Read data. */
+        struct
+        {
+            /** V1: The decompressor of the current data unit. */
+            PRTZIPDECOMP    pZipDecompV1;
+            /** The major format version number. */
+            uint32_t        uFmtVerMajor;
+            /** The minor format version number. */
+            uint32_t        uFmtVerMinor;
+
+            /** V2: Unread bytes in the current record. */
+            uint32_t        cbRecLeft;
+            /** V2: Bytes in the data buffer. */
+            uint32_t        cbDataBuffer;
+            /** V2: Current buffer position. */
+            uint32_t        offDataBuffer;
+            /** V2: End of data indicator. */
+            bool            fEndOfData;
+            /** V2: The type and flags byte fo the current record. */
+            uint8_t         u8TypeAndFlags;
+
+            /** @name Context info for SSMR3SetLoadError.
+             * @{  */
+            /** Pointer to the header for the current unit. */
+            PSSMUNIT        pCurUnit;
+            /** The version of the current unit if in the load exec stage. */
+            uint32_t        uCurUnitVer;
+            /** The pass number of the current unit if in the load exec stage. */
+            uint32_t        uCurUnitPass;
+            /** Whether SSMR3SetLoadError[V] has been called.
+             * @note Using ASMAtomicXchgBool because I'm very lazy. */
+            bool volatile   fHaveSetError;
+            /** @} */
+
+            /** RTGCPHYS size in bytes. (Only applicable when loading/reading.) */
+            unsigned        cbGCPhys;
+            /** RTGCPTR size in bytes. (Only applicable when loading/reading.) */
+            unsigned        cbGCPtr;
+            /** Whether cbGCPtr is fixed or settable. */
+            bool            fFixedGCPtrSize;
+
+            /** 32-bit MSC saved this? */
+            bool            fIsHostMsc32;
+            /** "Host OS" dot "architecture", picked up from recent SSM data units. */
+            char            szHostOSAndArch[32];
+
+            /** @name Header info (set by ssmR3ValidateFile)
+             * @{ */
+            /** The size of the file header. */
+            uint32_t        cbFileHdr;
+            /** The major version number. */
+            uint16_t        u16VerMajor;
+            /** The minor version number. */
+            uint16_t        u16VerMinor;
+            /** The build number. */
+            uint32_t        u32VerBuild;
+            /** The SVN revision. */
+            uint32_t        u32SvnRev;
+            /** 32 or 64 depending on the host. */
+            uint8_t         cHostBits;
+            /** Whether the stream is checksummed (SSMFILEHDR_FLAGS_STREAM_CRC32). */
+            bool            fStreamCrc32;
+            /** The CRC of the loaded file. */
+            uint32_t        u32LoadCRC;
+            /** The size of the load file. */
+            uint64_t        cbLoadFile;
+            /** @} */
+
+            /** V2: Data buffer.
+             * @remarks Be extremely careful when changing the size of this buffer! */
+            uint8_t         abDataBuffer[4096];
+
+            /** V2: Decompression buffer for when we cannot use the stream buffer. */
+            uint8_t         abComprBuffer[4096];
+        } Read;
+    } u;
+} SSMHANDLE;
+
+
+/**
+ * Header of the saved state file.
+ *
+ * Added in r5xxxx on 2009-07-2?, VirtualBox v3.0.51.
+ */
+typedef struct SSMFILEHDR
+{
+    /** Magic string which identifies this file as a version of VBox saved state
+     *  file format (SSMFILEHDR_MAGIC_V2_0). */
+    char            szMagic[32];
+    /** The major version number. */
+    uint16_t        u16VerMajor;
+    /** The minor version number. */
+    uint16_t        u16VerMinor;
+    /** The build number. */
+    uint32_t        u32VerBuild;
+    /** The SVN revision. */
+    uint32_t        u32SvnRev;
+    /** 32 or 64 depending on the host. */
+    uint8_t         cHostBits;
+    /** The size of RTGCPHYS. */
+    uint8_t         cbGCPhys;
+    /** The size of RTGCPTR. */
+    uint8_t         cbGCPtr;
+    /** Reserved header space - must be zero. */
+    uint8_t         u8Reserved;
+    /** The number of units that (may) have stored data in the file. */
+    uint32_t        cUnits;
+    /** Flags, see SSMFILEHDR_FLAGS_XXX.  */
+    uint32_t        fFlags;
+    /** The maximum size of decompressed data. */
+    uint32_t        cbMaxDecompr;
+    /** The checksum of this header.
+     * This field is set to zero when calculating the checksum. */
+    uint32_t        u32CRC;
+} SSMFILEHDR;
+AssertCompileSize(SSMFILEHDR, 64);
+AssertCompileMemberOffset(SSMFILEHDR, u32CRC, 60);
+AssertCompileMemberSize(SSMFILEHDR, szMagic, sizeof(SSMFILEHDR_MAGIC_V2_0));
+/** Pointer to a saved state file header. */
+typedef SSMFILEHDR *PSSMFILEHDR;
+/** Pointer to a const saved state file header. */
+typedef SSMFILEHDR const *PCSSMFILEHDR;
+
+
+/**
+ * Header of the saved state file.
+ *
+ * Added in r40980 on 2008-12-15, VirtualBox v2.0.51.
+ *
+ * @remarks This is a superset of SSMFILEHDRV11.
+ */
+typedef struct SSMFILEHDRV12
+{
+    /** Magic string which identifies this file as a version of VBox saved state
+     *  file format (SSMFILEHDR_MAGIC_V1_2). */
+    char            achMagic[32];
+    /** The size of this file. Used to check
+     * whether the save completed and that things are fine otherwise. */
+    uint64_t        cbFile;
+    /** File checksum. The actual calculation skips past the u32CRC field. */
+    uint32_t        u32CRC;
+    /** Padding. */
+    uint32_t        u32Reserved;
+    /** The machine UUID. (Ignored if NIL.) */
+    RTUUID          MachineUuid;
+
+    /** The major version number. */
+    uint16_t        u16VerMajor;
+    /** The minor version number. */
+    uint16_t        u16VerMinor;
+    /** The build number. */
+    uint32_t        u32VerBuild;
+    /** The SVN revision. */
+    uint32_t        u32SvnRev;
+
+    /** 32 or 64 depending on the host. */
+    uint8_t         cHostBits;
+    /** The size of RTGCPHYS. */
+    uint8_t         cbGCPhys;
+    /** The size of RTGCPTR. */
+    uint8_t         cbGCPtr;
+    /** Padding. */
+    uint8_t         au8Reserved;
+} SSMFILEHDRV12;
+AssertCompileSize(SSMFILEHDRV12, 64+16);
+AssertCompileMemberOffset(SSMFILEHDRV12, u32CRC, 40);
+AssertCompileMemberSize(SSMFILEHDRV12, achMagic, sizeof(SSMFILEHDR_MAGIC_V1_2));
+/** Pointer to a saved state file header. */
+typedef SSMFILEHDRV12 *PSSMFILEHDRV12;
+
+
+/**
+ * Header of the saved state file, version 1.1.
+ *
+ * Added in r23677 on 2007-08-17, VirtualBox v1.4.1.
+ */
+typedef struct SSMFILEHDRV11
+{
+    /** Magic string which identifies this file as a version of VBox saved state
+     *  file format (SSMFILEHDR_MAGIC_V1_1). */
+    char            achMagic[32];
+    /** The size of this file. Used to check
+     * whether the save completed and that things are fine otherwise. */
+    uint64_t        cbFile;
+    /** File checksum. The actual calculation skips past the u32CRC field. */
+    uint32_t        u32CRC;
+    /** Padding. */
+    uint32_t        u32Reserved;
+    /** The machine UUID. (Ignored if NIL.) */
+    RTUUID          MachineUuid;
+} SSMFILEHDRV11;
+AssertCompileSize(SSMFILEHDRV11, 64);
+AssertCompileMemberOffset(SSMFILEHDRV11, u32CRC, 40);
+/** Pointer to a saved state file header. */
+typedef SSMFILEHDRV11 *PSSMFILEHDRV11;
+
+
+/**
+ * Data unit header.
+ */
+typedef struct SSMFILEUNITHDRV2
+{
+    /** Magic (SSMFILEUNITHDR_MAGIC or SSMFILEUNITHDR_END). */
+    char            szMagic[8];
+    /** The offset in the saved state stream of the start of this unit.
+     * This is mainly intended for sanity checking. */
+    uint64_t        offStream;
+    /** The CRC-in-progress value this unit starts at. */
+    uint32_t        u32CurStreamCRC;
+    /** The checksum of this structure, including the whole name.
+     * Calculated with this field set to zero.  */
+    uint32_t        u32CRC;
+    /** Data version. */
+    uint32_t        u32Version;
+    /** Instance number. */
+    uint32_t        u32Instance;
+    /** Data pass number. */
+    uint32_t        u32Pass;
+    /** Flags reserved for future extensions. Must be zero. */
+    uint32_t        fFlags;
+    /** Size of the data unit name including the terminator. (bytes) */
+    uint32_t        cbName;
+    /** Data unit name, variable size. */
+    char            szName[SSM_MAX_NAME_SIZE];
+} SSMFILEUNITHDRV2;
+AssertCompileMemberOffset(SSMFILEUNITHDRV2, szName, 44);
+AssertCompileMemberSize(SSMFILEUNITHDRV2, szMagic, sizeof(SSMFILEUNITHDR_MAGIC));
+AssertCompileMemberSize(SSMFILEUNITHDRV2, szMagic, sizeof(SSMFILEUNITHDR_END));
+/** Pointer to SSMFILEUNITHDRV2.  */
+typedef SSMFILEUNITHDRV2 *PSSMFILEUNITHDRV2;
+
+
+/**
+ * Data unit header.
+ *
+ * This is used by v1.0, v1.1 and v1.2 of the format.
+ */
+typedef struct SSMFILEUNITHDRV1
+{
+    /** Magic (SSMFILEUNITHDR_MAGIC or SSMFILEUNITHDR_END). */
+    char            achMagic[8];
+    /** Number of bytes in this data unit including the header. */
+    uint64_t        cbUnit;
+    /** Data version. */
+    uint32_t        u32Version;
+    /** Instance number. */
+    uint32_t        u32Instance;
+    /** Size of the data unit name including the terminator. (bytes) */
+    uint32_t        cchName;
+    /** Data unit name. */
+    char            szName[1];
+} SSMFILEUNITHDRV1;
+/** Pointer to SSMFILEUNITHDR.  */
+typedef SSMFILEUNITHDRV1 *PSSMFILEUNITHDRV1;
+
+
+/**
+ * Termination data record.
+ */
+typedef struct SSMRECTERM
+{
+    uint8_t         u8TypeAndFlags;
+    /** The record size (sizeof(SSMRECTERM) - 2). */
+    uint8_t         cbRec;
+    /** Flags, see SSMRECTERM_FLAGS_CRC32. */
+    uint16_t        fFlags;
+    /** The checksum of the stream up to fFlags (exclusive). */
+    uint32_t        u32StreamCRC;
+    /** The length of this data unit in bytes (including this record). */
+    uint64_t        cbUnit;
+} SSMRECTERM;
+AssertCompileSize(SSMRECTERM, 16);
+AssertCompileMemberAlignment(SSMRECTERM, cbUnit, 8);
+/** Pointer to a termination record. */
+typedef SSMRECTERM *PSSMRECTERM;
+/** Pointer to a const termination record. */
+typedef SSMRECTERM const *PCSSMRECTERM;
+
+
+/**
+ * Directory entry.
+ */
+typedef struct SSMFILEDIRENTRY
+{
+    /** The offset of the data unit. */
+    uint64_t        off;
+    /** The instance number. */
+    uint32_t        u32Instance;
+    /** The CRC-32 of the name excluding the terminator. (lazy bird) */
+    uint32_t        u32NameCRC;
+} SSMFILEDIRENTRY;
+AssertCompileSize(SSMFILEDIRENTRY, 16);
+/** Pointer to a directory entry. */
+typedef SSMFILEDIRENTRY *PSSMFILEDIRENTRY;
+/** Pointer to a const directory entry. */
+typedef SSMFILEDIRENTRY const *PCSSMFILEDIRENTRY;
+
+/**
+ * Directory for the data units from the final pass.
+ *
+ * This is used to speed up SSMR3Seek (it would have to decompress and parse the
+ * whole stream otherwise).
+ */
+typedef struct SSMFILEDIR
+{
+    /** Magic string (SSMFILEDIR_MAGIC). */
+    char            szMagic[8];
+    /** The CRC-32 for the whole directory.
+     * Calculated with this field set to zero. */
+    uint32_t        u32CRC;
+    /** The number of directory entries. */
+    uint32_t        cEntries;
+    /** The directory entries (variable size). */
+    SSMFILEDIRENTRY aEntries[1];
+} SSMFILEDIR;
+AssertCompileSize(SSMFILEDIR, 32);
+/** Pointer to a directory. */
+typedef SSMFILEDIR *PSSMFILEDIR;
+/** Pointer to a const directory. */
+typedef SSMFILEDIR *PSSMFILEDIR;
+
+
+/**
+ * Footer structure
+ */
+typedef struct SSMFILEFTR
+{
+    /** Magic string (SSMFILEFTR_MAGIC). */
+    char            szMagic[8];
+    /** The offset of this record in the stream. */
+    uint64_t        offStream;
+    /** The CRC for the stream.
+     * This is set to zero if SSMFILEHDR_FLAGS_STREAM_CRC32 is clear. */
+    uint32_t        u32StreamCRC;
+    /** Number directory entries. */
+    uint32_t        cDirEntries;
+    /** Reserved footer space - must be zero. */
+    uint32_t        u32Reserved;
+    /** The CRC-32 for this structure.
+     * Calculated with this field set to zero. */
+    uint32_t        u32CRC;
+} SSMFILEFTR;
+AssertCompileSize(SSMFILEFTR, 32);
+/** Pointer to a footer. */
+typedef SSMFILEFTR *PSSMFILEFTR;
+/** Pointer to a const footer. */
+typedef SSMFILEFTR const *PCSSMFILEFTR;
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#ifndef SSM_STANDALONE
+/** Zeros used by the struct putter.
+ * This must be at least 8 bytes or the code breaks. */
+static uint8_t const    g_abZero[_1K] = {0};
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+#ifndef SSM_STANDALONE
+static int                  ssmR3LazyInit(PVM pVM);
+static DECLCALLBACK(int)    ssmR3SelfLiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass);
+static DECLCALLBACK(int)    ssmR3SelfSaveExec(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)    ssmR3SelfLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(int)    ssmR3LiveControlLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static int                  ssmR3Register(PVM pVM, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess, const char *pszBefore, PSSMUNIT *ppUnit);
+static int                  ssmR3LiveControlEmit(PSSMHANDLE pSSM, long double lrdPct, uint32_t uPass);
+#endif
+
+static int                  ssmR3StrmWriteBuffers(PSSMSTRM pStrm);
+static int                  ssmR3StrmReadMore(PSSMSTRM pStrm);
+
+#ifndef SSM_STANDALONE
+static int                  ssmR3DataFlushBuffer(PSSMHANDLE pSSM);
+#endif
+static int                  ssmR3DataReadRecHdrV2(PSSMHANDLE pSSM);
+
+
+#ifndef SSM_STANDALONE
+
+/**
+ * Cleans up resources allocated by SSM on VM termination.
+ *
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) SSMR3Term(PVM pVM)
+{
+    if (pVM->ssm.s.fInitialized)
+    {
+        pVM->ssm.s.fInitialized = false;
+        RTCritSectDelete(&pVM->ssm.s.CancelCritSect);
+    }
+}
+
+
+/**
+ * Performs lazy initialization of the SSM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+static int ssmR3LazyInit(PVM pVM)
+{
+    /*
+     * Register a saved state unit which we use to put the VirtualBox version,
+     * revision and similar stuff in.
+     */
+    pVM->ssm.s.fInitialized = true;
+    int rc = SSMR3RegisterInternal(pVM, "SSM", 0 /*uInstance*/, 1 /*uVersion*/, 64 /*cbGuess*/,
+                                   NULL /*pfnLivePrep*/, ssmR3SelfLiveExec, NULL /*pfnLiveVote*/,
+                                   NULL /*pfnSavePrep*/, ssmR3SelfSaveExec, NULL /*pfnSaveDone*/,
+                                   NULL /*pfnSavePrep*/, ssmR3SelfLoadExec, NULL /*pfnSaveDone*/);
+    if (RT_SUCCESS(rc))
+        rc = SSMR3RegisterInternal(pVM, "SSMLiveControl", 0 /*uInstance*/, 1 /*uVersion*/, 1 /*cbGuess*/,
+                                   NULL /*pfnLivePrep*/, NULL /*pfnLiveExec*/,     NULL /*pfnLiveVote*/,
+                                   NULL /*pfnSavePrep*/, NULL /*pfnSaveExec*/,     NULL /*pfnSaveDone*/,
+                                   NULL /*pfnSavePrep*/, ssmR3LiveControlLoadExec, NULL /*pfnSaveDone*/);
+
+    /*
+     * Initialize the cancellation critsect now.
+     */
+    if (RT_SUCCESS(rc))
+        rc = RTCritSectInit(&pVM->ssm.s.CancelCritSect);
+    if (RT_SUCCESS(rc))
+    {
+        STAM_REL_REG_USED(pVM, &pVM->ssm.s.uPass, STAMTYPE_U32, "/SSM/uPass", STAMUNIT_COUNT, "Current pass");
+    }
+
+    pVM->ssm.s.fInitialized = RT_SUCCESS(rc);
+    return rc;
+}
+
+
+/**
+ * Do ssmR3SelfSaveExec in pass 0.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The SSM handle.
+ * @param   uPass           The data pass number.
+ */
+static DECLCALLBACK(int) ssmR3SelfLiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    if (uPass == 0)
+    {
+        int rc = ssmR3SelfSaveExec(pVM, pSSM);
+        if (RT_SUCCESS(rc))
+            rc = VINF_SSM_DONT_CALL_AGAIN;
+        return rc;
+    }
+    AssertFailed();
+    return VERR_SSM_UNEXPECTED_PASS;
+}
+
+
+/**
+ * For saving usful things without having to go thru the tedious process of
+ * adding it to the header.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The SSM handle.
+ */
+static DECLCALLBACK(int) ssmR3SelfSaveExec(PVM pVM, PSSMHANDLE pSSM)
+{
+    NOREF(pVM);
+
+    /*
+     * String table containing pairs of variable and value string.
+     * Terminated by two empty strings.
+     */
+    SSMR3PutStrZ(pSSM, "Build Type");
+    SSMR3PutStrZ(pSSM, KBUILD_TYPE);
+    SSMR3PutStrZ(pSSM, "Host OS");
+    SSMR3PutStrZ(pSSM, KBUILD_TARGET "." KBUILD_TARGET_ARCH);
+#ifdef VBOX_OSE
+    SSMR3PutStrZ(pSSM, "OSE");
+    SSMR3PutStrZ(pSSM, "true");
+#endif
+
+    /* terminator */
+    SSMR3PutStrZ(pSSM, "");
+    return SSMR3PutStrZ(pSSM, "");
+}
+
+
+/**
+ * For load the version + revision and stuff.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The SSM handle.
+ * @param   uVersion        The version (1).
+ * @param   uPass           The pass.
+ */
+static DECLCALLBACK(int) ssmR3SelfLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    AssertLogRelMsgReturn(uVersion == 1, ("%d\n", uVersion), VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION);
+    NOREF(pVM); NOREF(uPass);
+
+    /*
+     * The first and last passes contains a {name, value} string table that is
+     * terminated by two emptry strings.  It contains useful informal build
+     * info and can be very handy when something goes wrong after restore.
+     */
+    if (    uPass == 0
+        ||  uPass == SSM_PASS_FINAL)
+    {
+        for (unsigned i = 0; ; i++)
+        {
+            char szVar[128];
+            char szValue[1024];
+            int rc = SSMR3GetStrZ(pSSM, szVar, sizeof(szVar));
+            AssertRCReturn(rc, rc);
+            rc = SSMR3GetStrZ(pSSM, szValue, sizeof(szValue));
+            AssertRCReturn(rc, rc);
+            if (!szVar[0] && !szValue[0])
+                break;
+            if (i == 0)
+                LogRel(("SSM: Saved state info:\n"));
+            LogRel(("SSM:   %s: %s\n", szVar, szValue));
+
+            /*
+             * Detect 32-bit MSC for handling SSMFIELD_ENTRY_PAD_MSC32_AUTO.
+             * Save the Host OS for SSMR3HandleHostOSAndArch
+             */
+            if (!strcmp(szVar, "Host OS"))
+            {
+                bool fIsHostMsc32 = !strcmp(szValue, "win.x86");
+                if (fIsHostMsc32 != pSSM->u.Read.fIsHostMsc32)
+                {
+                    LogRel(("SSM: (fIsHostMsc32 %RTbool => %RTbool)\n", pSSM->u.Read.fIsHostMsc32, fIsHostMsc32));
+                    pSSM->u.Read.fIsHostMsc32 = fIsHostMsc32;
+                }
+
+                size_t cchValue = strlen(szValue);
+                size_t cchCopy = RT_MIN(cchValue, sizeof(pSSM->u.Read.szHostOSAndArch) - 1);
+                Assert(cchValue == cchCopy);
+                memcpy(pSSM->u.Read.szHostOSAndArch, szValue, cchCopy);
+                pSSM->u.Read.szHostOSAndArch[cchCopy] = '\0';
+            }
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Load exec callback for the special live save state unit that tracks the
+ * progress of a live save.
+ *
+ * This is saved by ssmR3LiveControlEmit().
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The SSM handle.
+ * @param   uVersion        The version (1).
+ * @param   uPass           The pass.
+ */
+static DECLCALLBACK(int) ssmR3LiveControlLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    AssertLogRelMsgReturn(uVersion == 1, ("%d\n", uVersion), VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION);
+    NOREF(uPass);
+
+    uint16_t uPartsPerTenThousand;
+    int rc = SSMR3GetU16(pSSM, &uPartsPerTenThousand);
+    if (RT_SUCCESS(rc))
+    {
+        /* Scale it down to fit in our exec range. */
+        unsigned uPct = (unsigned)(  (long double)uPartsPerTenThousand / 100
+                                   * (100 - pSSM->uPercentPrepare - pSSM->uPercentDone) / 100)
+                      + pSSM->uPercentPrepare;
+        if (uPct != pSSM->uPercent)
+        {
+            AssertMsg(uPct < 100, ("uPct=%d uPartsPerTenThousand=%d uPercentPrepare=%d uPercentDone=%d\n", uPct, uPartsPerTenThousand, pSSM->uPercentPrepare, pSSM->uPercentDone));
+            pSSM->uPercent = uPct;
+            if (pSSM->pfnProgress)
+                pSSM->pfnProgress(pVM->pUVM, RT_MIN(uPct, 100 - pSSM->uPercentDone), pSSM->pvUser);
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Internal registration worker.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The instance id.
+ * @param   uVersion        The data unit version.
+ * @param   cbGuess         The guessed data unit size.
+ * @param   pszBefore       Name of data unit to be placed in front of.
+ *                          Optional.
+ * @param   ppUnit          Where to store the inserted unit node.
+ *                          Caller must fill in the missing details.
+ */
+static int ssmR3Register(PVM pVM, const char *pszName, uint32_t uInstance,
+                         uint32_t uVersion, size_t cbGuess, const char *pszBefore, PSSMUNIT *ppUnit)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtr(pszName);
+    AssertReturn(*pszName, VERR_INVALID_PARAMETER);
+    size_t cchName = strlen(pszName);
+    AssertMsgReturn(cchName < SSM_MAX_NAME_SIZE, ("%zu >= %u: %s\n", cchName, SSM_MAX_NAME_SIZE, pszName), VERR_OUT_OF_RANGE);
+
+    AssertReturn(!pszBefore || *pszBefore, VERR_INVALID_PARAMETER);
+    size_t cchBefore = pszBefore ? strlen(pszBefore) : 0;
+    AssertMsgReturn(cchBefore < SSM_MAX_NAME_SIZE, ("%zu >= %u: %s\n", cchBefore, SSM_MAX_NAME_SIZE, pszBefore), VERR_OUT_OF_RANGE);
+
+    /*
+     * Lazy init.
+     */
+    if (!pVM->ssm.s.fInitialized)
+    {
+        int rc = ssmR3LazyInit(pVM);
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Walk to the end of the list checking for duplicates as we go.
+     */
+    PSSMUNIT    pUnitBeforePrev = NULL;
+    PSSMUNIT    pUnitBefore     = NULL;
+    PSSMUNIT    pUnitPrev       = NULL;
+    PSSMUNIT    pUnit           = pVM->ssm.s.pHead;
+    while (pUnit)
+    {
+        if (    pUnit->u32Instance == uInstance
+            &&  pUnit->cchName == cchName
+            &&  !memcmp(pUnit->szName, pszName, cchName))
+        {
+            AssertMsgFailed(("Duplicate registration %s\n", pszName));
+            return VERR_SSM_UNIT_EXISTS;
+        }
+        if (    pUnit->cchName == cchBefore
+            &&  !pUnitBefore
+            &&  !memcmp(pUnit->szName, pszBefore, cchBefore))
+        {
+            pUnitBeforePrev = pUnitPrev;
+            pUnitBefore     = pUnit;
+        }
+
+        /* next */
+        pUnitPrev = pUnit;
+        pUnit = pUnit->pNext;
+    }
+
+    /*
+     * Allocate new node.
+     */
+    pUnit = (PSSMUNIT)MMR3HeapAllocZ(pVM, MM_TAG_SSM, RT_UOFFSETOF_DYN(SSMUNIT, szName[cchName + 1]));
+    if (!pUnit)
+        return VERR_NO_MEMORY;
+
+    /*
+     * Fill in (some) data. (Stuff is zero'd.)
+     */
+    pUnit->u32Version   = uVersion;
+    pUnit->u32Instance  = uInstance;
+    pUnit->cbGuess      = cbGuess;
+    pUnit->cchName      = cchName;
+    memcpy(pUnit->szName, pszName, cchName);
+
+    /*
+     * Insert
+     */
+    if (pUnitBefore)
+    {
+        pUnit->pNext = pUnitBefore;
+        if (pUnitBeforePrev)
+            pUnitBeforePrev->pNext = pUnit;
+        else
+            pVM->ssm.s.pHead       = pUnit;
+    }
+    else if (pUnitPrev)
+        pUnitPrev->pNext = pUnit;
+    else
+        pVM->ssm.s.pHead = pUnit;
+    pVM->ssm.s.cUnits++;
+
+    *ppUnit = pUnit;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Register a PDM Devices data unit.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         Device instance.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique.
+ * @param   uVersion        Data layout version number.
+ * @param   cbGuess         The approximate amount of data in the unit.
+ *                          Only for progress indicators.
+ * @param   pszBefore       Name of data unit which we should be put in front
+ *                          of. Optional (NULL).
+ *
+ * @param   pfnLivePrep     Prepare live save callback, optional.
+ * @param   pfnLiveExec     Execute live save callback, optional.
+ * @param   pfnLiveVote     Vote live save callback, optional.
+ *
+ * @param   pfnSavePrep     Prepare save callback, optional.
+ * @param   pfnSaveExec     Execute save callback, optional.
+ * @param   pfnSaveDone     Done save callback, optional.
+ *
+ * @param   pfnLoadPrep     Prepare load callback, optional.
+ * @param   pfnLoadExec     Execute load callback, optional.
+ * @param   pfnLoadDone     Done load callback, optional.
+ */
+VMMR3_INT_DECL(int)
+SSMR3RegisterDevice(PVM pVM, PPDMDEVINS pDevIns, const char *pszName,
+                    uint32_t uInstance, uint32_t uVersion, size_t cbGuess, const char *pszBefore,
+                    PFNSSMDEVLIVEPREP pfnLivePrep, PFNSSMDEVLIVEEXEC pfnLiveExec, PFNSSMDEVLIVEVOTE pfnLiveVote,
+                    PFNSSMDEVSAVEPREP pfnSavePrep, PFNSSMDEVSAVEEXEC pfnSaveExec, PFNSSMDEVSAVEDONE pfnSaveDone,
+                    PFNSSMDEVLOADPREP pfnLoadPrep, PFNSSMDEVLOADEXEC pfnLoadExec, PFNSSMDEVLOADDONE pfnLoadDone)
+{
+    PSSMUNIT pUnit;
+    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, pszBefore, &pUnit);
+    if (RT_SUCCESS(rc))
+    {
+        pUnit->enmType = SSMUNITTYPE_DEV;
+        pUnit->u.Dev.pfnLivePrep = pfnLivePrep;
+        pUnit->u.Dev.pfnLiveExec = pfnLiveExec;
+        pUnit->u.Dev.pfnLiveVote = pfnLiveVote;
+        pUnit->u.Dev.pfnSavePrep = pfnSavePrep;
+        pUnit->u.Dev.pfnSaveExec = pfnSaveExec;
+        pUnit->u.Dev.pfnSaveDone = pfnSaveDone;
+        pUnit->u.Dev.pfnLoadPrep = pfnLoadPrep;
+        pUnit->u.Dev.pfnLoadExec = pfnLoadExec;
+        pUnit->u.Dev.pfnLoadDone = pfnLoadDone;
+        pUnit->u.Dev.pDevIns = pDevIns;
+        pUnit->pCritSect = PDMR3DevGetCritSect(pVM, pDevIns);
+    }
+    return rc;
+}
+
+
+/**
+ * Register a PDM driver data unit.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDrvIns         Driver instance.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique.
+ * @param   uVersion        Data layout version number.
+ * @param   cbGuess         The approximate amount of data in the unit.
+ *                          Only for progress indicators.
+ *
+ * @param   pfnLivePrep     Prepare live save callback, optional.
+ * @param   pfnLiveExec     Execute live save callback, optional.
+ * @param   pfnLiveVote     Vote live save callback, optional.
+ *
+ * @param   pfnSavePrep     Prepare save callback, optional.
+ * @param   pfnSaveExec     Execute save callback, optional.
+ * @param   pfnSaveDone     Done save callback, optional.
+ *
+ * @param   pfnLoadPrep     Prepare load callback, optional.
+ * @param   pfnLoadExec     Execute load callback, optional.
+ * @param   pfnLoadDone     Done load callback, optional.
+ */
+VMMR3_INT_DECL(int)
+SSMR3RegisterDriver(PVM pVM, PPDMDRVINS pDrvIns, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess,
+                    PFNSSMDRVLIVEPREP pfnLivePrep, PFNSSMDRVLIVEEXEC pfnLiveExec, PFNSSMDRVLIVEVOTE pfnLiveVote,
+                    PFNSSMDRVSAVEPREP pfnSavePrep, PFNSSMDRVSAVEEXEC pfnSaveExec, PFNSSMDRVSAVEDONE pfnSaveDone,
+                    PFNSSMDRVLOADPREP pfnLoadPrep, PFNSSMDRVLOADEXEC pfnLoadExec, PFNSSMDRVLOADDONE pfnLoadDone)
+{
+    PSSMUNIT pUnit;
+    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, NULL, &pUnit);
+    if (RT_SUCCESS(rc))
+    {
+        pUnit->enmType = SSMUNITTYPE_DRV;
+        pUnit->u.Drv.pfnLivePrep = pfnLivePrep;
+        pUnit->u.Drv.pfnLiveExec = pfnLiveExec;
+        pUnit->u.Drv.pfnLiveVote = pfnLiveVote;
+        pUnit->u.Drv.pfnSavePrep = pfnSavePrep;
+        pUnit->u.Drv.pfnSaveExec = pfnSaveExec;
+        pUnit->u.Drv.pfnSaveDone = pfnSaveDone;
+        pUnit->u.Drv.pfnLoadPrep = pfnLoadPrep;
+        pUnit->u.Drv.pfnLoadExec = pfnLoadExec;
+        pUnit->u.Drv.pfnLoadDone = pfnLoadDone;
+        pUnit->u.Drv.pDrvIns = pDrvIns;
+    }
+    return rc;
+}
+
+
+/**
+ * Register a PDM USB device data unit.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pUsbIns         USB instance.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique.
+ * @param   uVersion        Data layout version number.
+ * @param   cbGuess         The approximate amount of data in the unit.
+ *                          Only for progress indicators.
+ *
+ * @param   pfnLivePrep     Prepare live save callback, optional.
+ * @param   pfnLiveExec     Execute live save callback, optional.
+ * @param   pfnLiveVote     Vote live save callback, optional.
+ *
+ * @param   pfnSavePrep     Prepare save callback, optional.
+ * @param   pfnSaveExec     Execute save callback, optional.
+ * @param   pfnSaveDone     Done save callback, optional.
+ *
+ * @param   pfnLoadPrep     Prepare load callback, optional.
+ * @param   pfnLoadExec     Execute load callback, optional.
+ * @param   pfnLoadDone     Done load callback, optional.
+ */
+VMMR3_INT_DECL(int)
+SSMR3RegisterUsb(PVM pVM, PPDMUSBINS pUsbIns, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess,
+                 PFNSSMUSBLIVEPREP pfnLivePrep, PFNSSMUSBLIVEEXEC pfnLiveExec, PFNSSMUSBLIVEVOTE pfnLiveVote,
+                 PFNSSMUSBSAVEPREP pfnSavePrep, PFNSSMUSBSAVEEXEC pfnSaveExec, PFNSSMUSBSAVEDONE pfnSaveDone,
+                 PFNSSMUSBLOADPREP pfnLoadPrep, PFNSSMUSBLOADEXEC pfnLoadExec, PFNSSMUSBLOADDONE pfnLoadDone)
+{
+    PSSMUNIT pUnit;
+    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, NULL, &pUnit);
+    if (RT_SUCCESS(rc))
+    {
+        pUnit->enmType = SSMUNITTYPE_USB;
+        pUnit->u.Usb.pfnLivePrep = pfnLivePrep;
+        pUnit->u.Usb.pfnLiveExec = pfnLiveExec;
+        pUnit->u.Usb.pfnLiveVote = pfnLiveVote;
+        pUnit->u.Usb.pfnSavePrep = pfnSavePrep;
+        pUnit->u.Usb.pfnSaveExec = pfnSaveExec;
+        pUnit->u.Usb.pfnSaveDone = pfnSaveDone;
+        pUnit->u.Usb.pfnLoadPrep = pfnLoadPrep;
+        pUnit->u.Usb.pfnLoadExec = pfnLoadExec;
+        pUnit->u.Usb.pfnLoadDone = pfnLoadDone;
+        pUnit->u.Usb.pUsbIns = pUsbIns;
+    }
+    return rc;
+}
+
+
+/**
+ * Register a internal data unit.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique.
+ * @param   uVersion        Data layout version number.
+ * @param   cbGuess         The approximate amount of data in the unit.
+ *                          Only for progress indicators.
+ *
+ * @param   pfnLivePrep     Prepare live save callback, optional.
+ * @param   pfnLiveExec     Execute live save callback, optional.
+ * @param   pfnLiveVote     Vote live save callback, optional.
+ *
+ * @param   pfnSavePrep     Prepare save callback, optional.
+ * @param   pfnSaveExec     Execute save callback, optional.
+ * @param   pfnSaveDone     Done save callback, optional.
+ *
+ * @param   pfnLoadPrep     Prepare load callback, optional.
+ * @param   pfnLoadExec     Execute load callback, optional.
+ * @param   pfnLoadDone     Done load callback, optional.
+ */
+VMMR3DECL(int) SSMR3RegisterInternal(PVM pVM, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess,
+    PFNSSMINTLIVEPREP pfnLivePrep, PFNSSMINTLIVEEXEC pfnLiveExec, PFNSSMINTLIVEVOTE pfnLiveVote,
+    PFNSSMINTSAVEPREP pfnSavePrep, PFNSSMINTSAVEEXEC pfnSaveExec, PFNSSMINTSAVEDONE pfnSaveDone,
+    PFNSSMINTLOADPREP pfnLoadPrep, PFNSSMINTLOADEXEC pfnLoadExec, PFNSSMINTLOADDONE pfnLoadDone)
+{
+    PSSMUNIT pUnit;
+    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, NULL /* pszBefore */, &pUnit);
+    if (RT_SUCCESS(rc))
+    {
+        pUnit->enmType = SSMUNITTYPE_INTERNAL;
+        pUnit->u.Internal.pfnLivePrep = pfnLivePrep;
+        pUnit->u.Internal.pfnLiveExec = pfnLiveExec;
+        pUnit->u.Internal.pfnLiveVote = pfnLiveVote;
+        pUnit->u.Internal.pfnSavePrep = pfnSavePrep;
+        pUnit->u.Internal.pfnSaveExec = pfnSaveExec;
+        pUnit->u.Internal.pfnSaveDone = pfnSaveDone;
+        pUnit->u.Internal.pfnLoadPrep = pfnLoadPrep;
+        pUnit->u.Internal.pfnLoadExec = pfnLoadExec;
+        pUnit->u.Internal.pfnLoadDone = pfnLoadDone;
+    }
+    return rc;
+}
+
+
+/**
+ * Register an external data unit.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique.
+ * @param   uVersion        Data layout version number.
+ * @param   cbGuess         The approximate amount of data in the unit.
+ *                          Only for progress indicators.
+ *
+ * @param   pfnLivePrep     Prepare live save callback, optional.
+ * @param   pfnLiveExec     Execute live save callback, optional.
+ * @param   pfnLiveVote     Vote live save callback, optional.
+ *
+ * @param   pfnSavePrep     Prepare save callback, optional.
+ * @param   pfnSaveExec     Execute save callback, optional.
+ * @param   pfnSaveDone     Done save callback, optional.
+ *
+ * @param   pfnLoadPrep     Prepare load callback, optional.
+ * @param   pfnLoadExec     Execute load callback, optional.
+ * @param   pfnLoadDone     Done load callback, optional.
+ * @param   pvUser          User argument.
+ */
+VMMR3DECL(int) SSMR3RegisterExternal(PUVM pUVM, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess,
+    PFNSSMEXTLIVEPREP pfnLivePrep, PFNSSMEXTLIVEEXEC pfnLiveExec, PFNSSMEXTLIVEVOTE pfnLiveVote,
+    PFNSSMEXTSAVEPREP pfnSavePrep, PFNSSMEXTSAVEEXEC pfnSaveExec, PFNSSMEXTSAVEDONE pfnSaveDone,
+    PFNSSMEXTLOADPREP pfnLoadPrep, PFNSSMEXTLOADEXEC pfnLoadExec, PFNSSMEXTLOADDONE pfnLoadDone, void *pvUser)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    PSSMUNIT pUnit;
+    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, NULL /* pszBefore */, &pUnit);
+    if (RT_SUCCESS(rc))
+    {
+        pUnit->enmType = SSMUNITTYPE_EXTERNAL;
+        pUnit->u.External.pfnLivePrep = pfnLivePrep;
+        pUnit->u.External.pfnLiveExec = pfnLiveExec;
+        pUnit->u.External.pfnLiveVote = pfnLiveVote;
+        pUnit->u.External.pfnSavePrep = pfnSavePrep;
+        pUnit->u.External.pfnSaveExec = pfnSaveExec;
+        pUnit->u.External.pfnSaveDone = pfnSaveDone;
+        pUnit->u.External.pfnLoadPrep = pfnLoadPrep;
+        pUnit->u.External.pfnLoadExec = pfnLoadExec;
+        pUnit->u.External.pfnLoadDone = pfnLoadDone;
+        pUnit->u.External.pvUser      = pvUser;
+    }
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNSSMINTLOADEXEC,
+ * Stub that skips the whole unit (see SSMR3RegisterStub).}
+ */
+static DECLCALLBACK(int) ssmR3LoadExecStub(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    NOREF(pVM); NOREF(uVersion); NOREF(uPass);
+    return SSMR3SkipToEndOfUnit(pSSM);
+}
+
+
+/**
+ * Registers a stub state loader for working around legacy.
+ *
+ * This is used to deal with irelevant PATM and CSAM saved state units in HM
+ * mode and when built without raw-mode.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pszName             Data unit name.
+ * @param   uInstance           Instance number.
+ */
+VMMR3DECL(int) SSMR3RegisterStub(PVM pVM, const char *pszName, uint32_t uInstance)
+{
+    return SSMR3RegisterInternal(pVM, pszName, uInstance, UINT32_MAX, 0,
+                                 NULL, NULL, NULL,
+                                 NULL, NULL, NULL,
+                                 NULL, ssmR3LoadExecStub, NULL);
+}
+
+
+/**
+ * Deregister one or more PDM Device data units.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         Device instance.
+ * @param   pszName         Data unit name.
+ *                          Use NULL to deregister all data units for that device instance.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique.
+ * @remark  Only for dynamic data units and dynamic unloaded modules.
+ */
+VMMR3_INT_DECL(int) SSMR3DeregisterDevice(PVM pVM, PPDMDEVINS pDevIns, const char *pszName, uint32_t uInstance)
+{
+    /*
+     * Validate input.
+     */
+    if (!pDevIns)
+    {
+        AssertMsgFailed(("pDevIns is NULL!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Search the list.
+     */
+    size_t      cchName = pszName ? strlen(pszName) : 0;
+    int         rc = pszName ? VERR_SSM_UNIT_NOT_FOUND : VINF_SUCCESS;
+    PSSMUNIT    pUnitPrev = NULL;
+    PSSMUNIT    pUnit = pVM->ssm.s.pHead;
+    while (pUnit)
+    {
+        if (    pUnit->enmType == SSMUNITTYPE_DEV
+            &&  (   !pszName
+                 || (   pUnit->cchName == cchName
+                     && !memcmp(pUnit->szName, pszName, cchName)))
+            &&  pUnit->u32Instance == uInstance
+            )
+        {
+            if (pUnit->u.Dev.pDevIns == pDevIns)
+            {
+                /*
+                 * Unlink it, advance pointer, and free the node.
+                 */
+                PSSMUNIT pFree = pUnit;
+                pUnit = pUnit->pNext;
+                if (pUnitPrev)
+                    pUnitPrev->pNext = pUnit;
+                else
+                    pVM->ssm.s.pHead = pUnit;
+                pVM->ssm.s.cUnits--;
+                Log(("SSM: Removed data unit '%s' (pdm dev).\n", pFree->szName));
+                MMR3HeapFree(pFree);
+
+                if (pszName)
+                    return VINF_SUCCESS;
+                rc = VINF_SUCCESS;
+                continue;
+            }
+            else if (pszName)
+            {
+                AssertMsgFailed(("Caller is not owner! Owner=%p Caller=%p %s\n",
+                                 pUnit->u.Dev.pDevIns, pDevIns, pszName));
+                return VERR_SSM_UNIT_NOT_OWNER;
+            }
+        }
+
+        /* next */
+        pUnitPrev = pUnit;
+        pUnit = pUnit->pNext;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Deregister one ore more PDM Driver data units.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDrvIns         Driver instance.
+ * @param   pszName         Data unit name.
+ *                          Use NULL to deregister all data units for that driver instance.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique. Ignored if pszName is NULL.
+ * @remark  Only for dynamic data units and dynamic unloaded modules.
+ */
+VMMR3_INT_DECL(int) SSMR3DeregisterDriver(PVM pVM, PPDMDRVINS pDrvIns, const char *pszName, uint32_t uInstance)
+{
+    /*
+     * Validate input.
+     */
+    if (!pDrvIns)
+    {
+        AssertMsgFailed(("pDrvIns is NULL!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Search the list.
+     */
+    size_t      cchName = pszName ? strlen(pszName) : 0;
+    int         rc = pszName ? VERR_SSM_UNIT_NOT_FOUND : VINF_SUCCESS;
+    PSSMUNIT    pUnitPrev = NULL;
+    PSSMUNIT    pUnit = pVM->ssm.s.pHead;
+    while (pUnit)
+    {
+        if (    pUnit->enmType == SSMUNITTYPE_DRV
+            &&  (   !pszName
+                 || (   pUnit->cchName == cchName
+                     && !memcmp(pUnit->szName, pszName, cchName)
+                     && pUnit->u32Instance == uInstance))
+            )
+        {
+            if (pUnit->u.Drv.pDrvIns == pDrvIns)
+            {
+                /*
+                 * Unlink it, advance pointer, and free the node.
+                 */
+                PSSMUNIT pFree = pUnit;
+                pUnit = pUnit->pNext;
+                if (pUnitPrev)
+                    pUnitPrev->pNext = pUnit;
+                else
+                    pVM->ssm.s.pHead = pUnit;
+                pVM->ssm.s.cUnits--;
+                Log(("SSM: Removed data unit '%s' (pdm drv).\n", pFree->szName));
+                MMR3HeapFree(pFree);
+
+                if (pszName)
+                    return VINF_SUCCESS;
+                rc = VINF_SUCCESS;
+                continue;
+            }
+
+            AssertMsgReturn(!pszName,
+                            ("Caller is not owner! Owner=%p Caller=%p %s\n", pUnit->u.Drv.pDrvIns, pDrvIns, pszName),
+                            VERR_SSM_UNIT_NOT_OWNER);
+        }
+
+        /* next */
+        pUnitPrev = pUnit;
+        pUnit = pUnit->pNext;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Deregister one or more PDM USB device data units.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pUsbIns         USB device instance.
+ * @param   pszName         Data unit name.
+ *                          Use NULL to deregister all data units for that driver instance.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique. Ignored if pszName is NULL.
+ * @remark  Only for dynamic data units and dynamic unloaded modules.
+ */
+VMMR3_INT_DECL(int) SSMR3DeregisterUsb(PVM pVM, PPDMUSBINS pUsbIns, const char *pszName, uint32_t uInstance)
+{
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(VALID_PTR(pUsbIns), ("pUsbIns is NULL!\n"), VERR_INVALID_PARAMETER);
+
+    /*
+     * Search the list.
+     */
+    size_t      cchName = pszName ? strlen(pszName) : 0;
+    int         rc = pszName ? VERR_SSM_UNIT_NOT_FOUND : VINF_SUCCESS;
+    PSSMUNIT    pUnitPrev = NULL;
+    PSSMUNIT    pUnit = pVM->ssm.s.pHead;
+    while (pUnit)
+    {
+        if (    pUnit->enmType == SSMUNITTYPE_USB
+            &&  (   !pszName
+                 || (   pUnit->cchName == cchName
+                     && !memcmp(pUnit->szName, pszName, cchName)
+                     && pUnit->u32Instance == uInstance))
+            )
+        {
+            if (pUnit->u.Usb.pUsbIns == pUsbIns)
+            {
+                /*
+                 * Unlink it, advance pointer, and free the node.
+                 */
+                PSSMUNIT pFree = pUnit;
+                pUnit = pUnit->pNext;
+                if (pUnitPrev)
+                    pUnitPrev->pNext = pUnit;
+                else
+                    pVM->ssm.s.pHead = pUnit;
+                pVM->ssm.s.cUnits--;
+                Log(("SSM: Removed data unit '%s' (pdm drv).\n", pFree->szName));
+                MMR3HeapFree(pFree);
+
+                if (pszName)
+                    return VINF_SUCCESS;
+                rc = VINF_SUCCESS;
+                continue;
+            }
+
+            AssertMsgReturn(!pszName,
+                            ("Caller is not owner! Owner=%p Caller=%p %s\n", pUnit->u.Usb.pUsbIns, pUsbIns, pszName),
+                            VERR_SSM_UNIT_NOT_OWNER);
+        }
+
+        /* next */
+        pUnitPrev = pUnit;
+        pUnit = pUnit->pNext;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Deregister a data unit.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   enmType         Unit type
+ * @param   pszName         Data unit name.
+ * @remark  Only for dynamic data units.
+ */
+static int ssmR3DeregisterByNameAndType(PVM pVM, const char *pszName, SSMUNITTYPE enmType)
+{
+    /*
+     * Validate input.
+     */
+    if (!pszName)
+    {
+        AssertMsgFailed(("pszName is NULL!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Search the list.
+     */
+    size_t      cchName = strlen(pszName);
+    int         rc = VERR_SSM_UNIT_NOT_FOUND;
+    PSSMUNIT    pUnitPrev = NULL;
+    PSSMUNIT    pUnit = pVM->ssm.s.pHead;
+    while (pUnit)
+    {
+        if (    pUnit->enmType == enmType
+            &&  pUnit->cchName == cchName
+            &&  !memcmp(pUnit->szName, pszName, cchName))
+        {
+            /*
+             * Unlink it, advance pointer, and free the node.
+             */
+            PSSMUNIT pFree = pUnit;
+            pUnit = pUnit->pNext;
+            if (pUnitPrev)
+                pUnitPrev->pNext = pUnit;
+            else
+                pVM->ssm.s.pHead = pUnit;
+            pVM->ssm.s.cUnits--;
+            Log(("SSM: Removed data unit '%s' (type=%d).\n", pFree->szName, enmType));
+            MMR3HeapFree(pFree);
+            return VINF_SUCCESS;
+        }
+
+        /* next */
+        pUnitPrev = pUnit;
+        pUnit = pUnit->pNext;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Deregister an internal data unit.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pszName         Data unit name.
+ * @remark  Only for dynamic data units.
+ */
+VMMR3DECL(int) SSMR3DeregisterInternal(PVM pVM, const char *pszName)
+{
+    return ssmR3DeregisterByNameAndType(pVM, pszName, SSMUNITTYPE_INTERNAL);
+}
+
+
+/**
+ * Deregister an external data unit.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM structure.
+ * @param   pszName         Data unit name.
+ * @remark  Only for dynamic data units.
+ */
+VMMR3DECL(int) SSMR3DeregisterExternal(PUVM pUVM, const char *pszName)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    return ssmR3DeregisterByNameAndType(pVM, pszName, SSMUNITTYPE_EXTERNAL);
+}
+
+#endif /* !SSM_STANDALONE */
+
+
+/**
+ * Initializes the stream after/before opening the file/whatever.
+ *
+ * @returns VINF_SUCCESS or VERR_NO_MEMORY.
+ * @param   pStrm           The stream handle.
+ * @param   fChecksummed    Whether the stream is to be checksummed while
+ *                          written/read.
+ * @param   cBuffers        The number of buffers.
+ */
+static int ssmR3StrmInitInternal(PSSMSTRM pStrm, bool fChecksummed, uint32_t cBuffers)
+{
+    Assert(cBuffers > 0);
+
+    /*
+     * Init the common data members.
+     */
+    pStrm->fTerminating = false;
+    pStrm->fNeedSeek    = false;
+    pStrm->rc           = VINF_SUCCESS;
+    pStrm->hIoThread    = NIL_RTTHREAD;
+    pStrm->offNeedSeekTo= UINT64_MAX;
+
+    pStrm->pHead        = NULL;
+    pStrm->pFree        = NULL;
+    pStrm->hEvtHead     = NIL_RTSEMEVENT;
+    pStrm->hEvtFree     = NIL_RTSEMEVENT;
+
+    pStrm->pPending     = NULL;
+    pStrm->pCur         = NULL;
+    pStrm->offCurStream = 0;
+    pStrm->off          = 0;
+    pStrm->fChecksummed = fChecksummed;
+    pStrm->u32StreamCRC = fChecksummed ? RTCrc32Start() : 0;
+    pStrm->offStreamCRC = 0;
+
+    /*
+     * Allocate the buffers.  Page align them in case that makes the kernel
+     * and/or cpu happier in some way.
+     */
+    int rc = VINF_SUCCESS;
+    for (uint32_t i = 0; i < cBuffers; i++)
+    {
+        PSSMSTRMBUF pBuf = (PSSMSTRMBUF)RTMemPageAllocZ(sizeof(*pBuf));
+        if (!pBuf)
+        {
+            if (i > 2)
+            {
+                LogRel(("ssmR3StrmAllocBuffer: WARNING: Could only get %d stream buffers.\n", i));
+                break;
+            }
+            LogRel(("ssmR3StrmAllocBuffer: Failed to allocate stream buffers. (i=%d)\n", i));
+            return VERR_NO_MEMORY;
+        }
+
+        /* link it */
+        pBuf->pNext = pStrm->pFree;
+        pStrm->pFree = pBuf;
+    }
+
+    /*
+     * Create the event semaphores.
+     */
+    rc = RTSemEventCreate(&pStrm->hEvtHead);
+    if (RT_FAILURE(rc))
+        return rc;
+    rc = RTSemEventCreate(&pStrm->hEvtFree);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroys a list of buffers.
+ *
+ * @param   pHead            Pointer to the head.
+ */
+static void ssmR3StrmDestroyBufList(PSSMSTRMBUF pHead)
+{
+    while (pHead)
+    {
+        PSSMSTRMBUF pCur = pHead;
+        pHead = pCur->pNext;
+        pCur->pNext = NULL;
+        RTMemPageFree(pCur, sizeof(*pCur));
+    }
+}
+
+
+/**
+ * Cleans up a stream after ssmR3StrmInitInternal has been called (regardless of
+ * it succeeded or not).
+ *
+ * @param   pStrm           The stream handle.
+ */
+static void ssmR3StrmDelete(PSSMSTRM pStrm)
+{
+    RTMemPageFree(pStrm->pCur, sizeof(*pStrm->pCur));
+    pStrm->pCur = NULL;
+    ssmR3StrmDestroyBufList(pStrm->pHead);
+    pStrm->pHead = NULL;
+    ssmR3StrmDestroyBufList(pStrm->pPending);
+    pStrm->pPending = NULL;
+    ssmR3StrmDestroyBufList(pStrm->pFree);
+    pStrm->pFree = NULL;
+
+    RTSemEventDestroy(pStrm->hEvtHead);
+    pStrm->hEvtHead = NIL_RTSEMEVENT;
+
+    RTSemEventDestroy(pStrm->hEvtFree);
+    pStrm->hEvtFree = NIL_RTSEMEVENT;
+}
+
+
+/**
+ * Initializes a stream that uses a method table.
+ *
+ * @returns VBox status code.
+ * @param   pStrm           The stream manager structure.
+ * @param   pStreamOps      The stream method table.
+ * @param   pvUser          The user argument for the stream methods.
+ * @param   fWrite          Whether to open for writing or reading.
+ * @param   fChecksummed    Whether the stream is to be checksummed while
+ *                          written/read.
+ * @param   cBuffers        The number of buffers.
+ */
+static int ssmR3StrmInit(PSSMSTRM pStrm, PCSSMSTRMOPS pStreamOps, void *pvUser, bool fWrite, bool fChecksummed, uint32_t cBuffers)
+{
+    int rc = ssmR3StrmInitInternal(pStrm, fChecksummed, cBuffers);
+    if (RT_SUCCESS(rc))
+    {
+        pStrm->pOps   = pStreamOps;
+        pStrm->pvUser = pvUser;
+        pStrm->fWrite = fWrite;
+        return VINF_SUCCESS;
+    }
+
+    ssmR3StrmDelete(pStrm);
+    pStrm->rc = rc;
+    return rc;
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnWrite
+ */
+static DECLCALLBACK(int) ssmR3FileWrite(void *pvUser, uint64_t offStream, const void *pvBuf, size_t cbToWrite)
+{
+    NOREF(offStream);
+    return RTFileWriteAt((RTFILE)(uintptr_t)pvUser, offStream, pvBuf, cbToWrite, NULL); /** @todo use RTFileWrite */
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnRead
+ */
+static DECLCALLBACK(int) ssmR3FileRead(void *pvUser, uint64_t offStream, void *pvBuf, size_t cbToRead, size_t *pcbRead)
+{
+    Assert(RTFileTell((RTFILE)(uintptr_t)pvUser) == offStream); NOREF(offStream);
+    return RTFileRead((RTFILE)(uintptr_t)pvUser, pvBuf, cbToRead, pcbRead);
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnSeek
+ */
+static DECLCALLBACK(int) ssmR3FileSeek(void *pvUser, int64_t offSeek, unsigned uMethod, uint64_t *poffActual)
+{
+    return RTFileSeek((RTFILE)(uintptr_t)pvUser, offSeek, uMethod, poffActual);
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnTell
+ */
+static DECLCALLBACK(uint64_t) ssmR3FileTell(void *pvUser)
+{
+    return RTFileTell((RTFILE)(uintptr_t)pvUser);
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnSize
+ */
+static DECLCALLBACK(int) ssmR3FileSize(void *pvUser, uint64_t *pcb)
+{
+    return RTFileQuerySize((RTFILE)(uintptr_t)pvUser, pcb);
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnIsOk
+ */
+static DECLCALLBACK(int) ssmR3FileIsOk(void *pvUser)
+{
+    /*
+     * Check that there is still some space left on the disk.
+     */
+    RTFOFF cbFree;
+    int rc = RTFileQueryFsSizes((RTFILE)(uintptr_t)pvUser, NULL, &cbFree, NULL, NULL);
+#define SSM_MIN_DISK_FREE    ((RTFOFF)( 10 * _1M ))
+    if (RT_SUCCESS(rc))
+    {
+        if (cbFree < SSM_MIN_DISK_FREE)
+        {
+            LogRel(("SSM: Giving up: Low on disk space. (cbFree=%RTfoff, SSM_MIN_DISK_FREE=%RTfoff).\n",
+                    cbFree, SSM_MIN_DISK_FREE));
+            rc = VERR_SSM_LOW_ON_DISK_SPACE;
+        }
+    }
+    else if (rc == VERR_NOT_SUPPORTED)
+        rc = VINF_SUCCESS;
+    else
+        AssertLogRelRC(rc);
+    return rc;
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnClose
+ */
+static DECLCALLBACK(int) ssmR3FileClose(void *pvUser, bool fCancelled)
+{
+    NOREF(fCancelled);
+    return RTFileClose((RTFILE)(uintptr_t)pvUser);
+}
+
+
+/**
+ * Method table for a file based stream.
+ */
+static SSMSTRMOPS const g_ssmR3FileOps =
+{
+    SSMSTRMOPS_VERSION,
+    ssmR3FileWrite,
+    ssmR3FileRead,
+    ssmR3FileSeek,
+    ssmR3FileTell,
+    ssmR3FileSize,
+    ssmR3FileIsOk,
+    ssmR3FileClose,
+    SSMSTRMOPS_VERSION
+};
+
+
+/**
+ * Opens a file stream.
+ *
+ * @returns VBox status code.
+ * @param   pStrm           The stream manager structure.
+ * @param   pszFilename     The file to open or create.
+ * @param   fWrite          Whether to open for writing or reading.
+ * @param   fChecksummed    Whether the stream is to be checksummed while
+ *                          written/read.
+ * @param   cBuffers        The number of buffers.
+ */
+static int ssmR3StrmOpenFile(PSSMSTRM pStrm, const char *pszFilename, bool fWrite, bool fChecksummed, uint32_t cBuffers)
+{
+    int rc = ssmR3StrmInitInternal(pStrm, fChecksummed, cBuffers);
+    if (RT_SUCCESS(rc))
+    {
+        uint32_t fFlags = fWrite
+                        ? RTFILE_O_READWRITE | RTFILE_O_CREATE_REPLACE | RTFILE_O_DENY_WRITE
+                        : RTFILE_O_READ      | RTFILE_O_OPEN           | RTFILE_O_DENY_WRITE;
+        RTFILE hFile;
+        rc = RTFileOpen(&hFile, pszFilename, fFlags);
+        if (RT_SUCCESS(rc))
+        {
+            pStrm->pOps   = &g_ssmR3FileOps;
+            pStrm->pvUser = (void *)(uintptr_t)hFile;
+            pStrm->fWrite = fWrite;
+            return VINF_SUCCESS;
+        }
+    }
+
+    ssmR3StrmDelete(pStrm);
+    pStrm->rc = rc;
+    return rc;
+}
+
+
+/**
+ * Raise an error condition on the stream.
+ *
+ * @returns true if we raised the error condition, false if the stream already
+ *          had an error condition set.
+ *
+ * @param   pStrm           The stream handle.
+ * @param   rc              The VBox error status code.
+ *
+ * @thread  Any.
+ */
+DECLINLINE(bool) ssmR3StrmSetError(PSSMSTRM pStrm, int rc)
+{
+    Assert(RT_FAILURE_NP(rc));
+    return ASMAtomicCmpXchgS32(&pStrm->rc, rc, VINF_SUCCESS);
+}
+
+
+/**
+ * Puts a buffer into the free list.
+ *
+ * @param   pStrm           The stream handle.
+ * @param   pBuf            The buffer.
+ *
+ * @thread  The consumer.
+ */
+static void ssmR3StrmPutFreeBuf(PSSMSTRM pStrm, PSSMSTRMBUF pBuf)
+{
+    for (;;)
+    {
+        PSSMSTRMBUF pCurFreeHead = ASMAtomicUoReadPtrT(&pStrm->pFree, PSSMSTRMBUF);
+        ASMAtomicUoWritePtr(&pBuf->pNext, pCurFreeHead);
+        if (ASMAtomicCmpXchgPtr(&pStrm->pFree, pBuf, pCurFreeHead))
+        {
+            int rc = RTSemEventSignal(pStrm->hEvtFree);
+            AssertRC(rc);
+            return;
+        }
+    }
+}
+
+
+/**
+ * Gets a free buffer, waits for one if necessary.
+ *
+ * @returns Pointer to the buffer on success. NULL if we're terminating.
+ * @param   pStrm           The stream handle.
+ *
+ * @thread  The producer.
+ */
+static PSSMSTRMBUF ssmR3StrmGetFreeBuf(PSSMSTRM pStrm)
+{
+    for (;;)
+    {
+        PSSMSTRMBUF pMine = ASMAtomicUoReadPtrT(&pStrm->pFree, PSSMSTRMBUF);
+        if (!pMine)
+        {
+            if (pStrm->fTerminating)
+                return NULL;
+            if (RT_FAILURE(pStrm->rc))
+                return NULL;
+            if (    pStrm->fWrite
+                &&  pStrm->hIoThread == NIL_RTTHREAD)
+            {
+                int rc = ssmR3StrmWriteBuffers(pStrm);
+                if (RT_FAILURE(rc))
+                    return NULL;
+            }
+            int rc = RTSemEventWaitNoResume(pStrm->hEvtFree, 30000);
+            if (    rc == VERR_SEM_DESTROYED
+                ||  pStrm->fTerminating)
+                return NULL;
+            continue;
+        }
+
+        if (ASMAtomicCmpXchgPtr(&pStrm->pFree, pMine->pNext, pMine))
+        {
+            pMine->offStream    = UINT64_MAX;
+            pMine->cb           = 0;
+            pMine->pNext        = NULL;
+            pMine->fEndOfStream = false;
+            pMine->NanoTS       = RTTimeNanoTS();
+            return pMine;
+        }
+    }
+}
+
+
+/**
+ * Puts a buffer onto the queue.
+ *
+ * @param   pStrm           The stream handle.
+ * @param   pBuf            The stream buffer to put.
+ *
+ * @thread  The producer.
+ */
+static void ssmR3StrmPutBuf(PSSMSTRM pStrm, PSSMSTRMBUF pBuf)
+{
+    for (;;)
+    {
+        PSSMSTRMBUF pCurHead = ASMAtomicUoReadPtrT(&pStrm->pHead, PSSMSTRMBUF);
+        ASMAtomicUoWritePtr(&pBuf->pNext, pCurHead);
+        if (ASMAtomicCmpXchgPtr(&pStrm->pHead, pBuf, pCurHead))
+        {
+            int rc = RTSemEventSignal(pStrm->hEvtHead);
+            AssertRC(rc);
+            return;
+        }
+    }
+}
+
+
+/**
+ * Reverses the list.
+ *
+ * @returns The head of the reversed list.
+ * @param   pHead           The head of the list to reverse.
+ */
+static PSSMSTRMBUF ssmR3StrmReverseList(PSSMSTRMBUF pHead)
+{
+    PSSMSTRMBUF pRevHead = NULL;
+    while (pHead)
+    {
+        PSSMSTRMBUF pCur = pHead;
+        pHead = pCur->pNext;
+        pCur->pNext = pRevHead;
+        pRevHead = pCur;
+    }
+    return pRevHead;
+}
+
+
+/**
+ * Gets one buffer from the queue, will wait for one to become ready if
+ * necessary.
+ *
+ * @returns Pointer to the buffer on success. NULL if we're terminating.
+ * @param   pStrm           The stream handle.
+ *
+ * @thread  The consumer.
+ */
+static PSSMSTRMBUF ssmR3StrmGetBuf(PSSMSTRM pStrm)
+{
+    for (;;)
+    {
+        PSSMSTRMBUF pMine = pStrm->pPending;
+        if (pMine)
+        {
+            pStrm->pPending = pMine->pNext;
+            pMine->pNext = NULL;
+            return pMine;
+        }
+
+        pMine = ASMAtomicXchgPtrT(&pStrm->pHead, NULL, PSSMSTRMBUF);
+        if (pMine)
+            pStrm->pPending = ssmR3StrmReverseList(pMine);
+        else
+        {
+            if (pStrm->fTerminating)
+                return NULL;
+            if (RT_FAILURE(pStrm->rc))
+                return NULL;
+            if (    !pStrm->fWrite
+                &&  pStrm->hIoThread == NIL_RTTHREAD)
+            {
+                int rc = ssmR3StrmReadMore(pStrm);
+                if (RT_FAILURE(rc))
+                    return NULL;
+                continue;
+            }
+
+            int rc = RTSemEventWaitNoResume(pStrm->hEvtHead, 30000);
+            if (    rc == VERR_SEM_DESTROYED
+                ||  pStrm->fTerminating)
+                return NULL;
+        }
+    }
+}
+
+
+/**
+ * Flushes the current buffer (both write and read streams).
+ *
+ * @param   pStrm           The stream handle.
+ */
+static void ssmR3StrmFlushCurBuf(PSSMSTRM pStrm)
+{
+    if (pStrm->pCur)
+    {
+        PSSMSTRMBUF pBuf = pStrm->pCur;
+        pStrm->pCur = NULL;
+
+        if (pStrm->fWrite)
+        {
+            uint32_t cb     = pStrm->off;
+            pBuf->cb        = cb;
+            pBuf->offStream = pStrm->offCurStream;
+            if (    pStrm->fChecksummed
+                &&  pStrm->offStreamCRC < cb)
+                pStrm->u32StreamCRC = RTCrc32Process(pStrm->u32StreamCRC,
+                                                     &pBuf->abData[pStrm->offStreamCRC],
+                                                     cb - pStrm->offStreamCRC);
+            pStrm->offCurStream += cb;
+            pStrm->off           = 0;
+            pStrm->offStreamCRC  = 0;
+
+            ssmR3StrmPutBuf(pStrm, pBuf);
+        }
+        else
+        {
+            uint32_t cb = pBuf->cb;
+            if (    pStrm->fChecksummed
+                &&  pStrm->offStreamCRC < cb)
+                pStrm->u32StreamCRC = RTCrc32Process(pStrm->u32StreamCRC,
+                                                     &pBuf->abData[pStrm->offStreamCRC],
+                                                     cb - pStrm->offStreamCRC);
+            pStrm->offCurStream += cb;
+            pStrm->off           = 0;
+            pStrm->offStreamCRC  = 0;
+
+            ssmR3StrmPutFreeBuf(pStrm, pBuf);
+        }
+    }
+}
+
+
+/**
+ * Flush buffered data.
+ *
+ * @returns VBox status code. Returns VINF_EOF if we encounter a buffer with the
+ *          fEndOfStream indicator set.
+ * @param   pStrm           The stream handle.
+ *
+ * @thread  The producer thread.
+ */
+static int ssmR3StrmWriteBuffers(PSSMSTRM pStrm)
+{
+    Assert(pStrm->fWrite);
+
+    /*
+     * Just return if the stream has a pending error condition.
+     */
+    int rc = pStrm->rc;
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Grab the pending list and write it out.
+     */
+    PSSMSTRMBUF pHead = ASMAtomicXchgPtrT(&pStrm->pHead, NULL, PSSMSTRMBUF);
+    if (!pHead)
+        return VINF_SUCCESS;
+    pHead = ssmR3StrmReverseList(pHead);
+
+    while (pHead)
+    {
+        /* pop */
+        PSSMSTRMBUF pCur = pHead;
+        pHead = pCur->pNext;
+
+        /* flush */
+        rc = pStrm->pOps->pfnIsOk(pStrm->pvUser);
+        if (RT_SUCCESS(rc))
+            rc = pStrm->pOps->pfnWrite(pStrm->pvUser, pCur->offStream, &pCur->abData[0], pCur->cb);
+        if (    RT_FAILURE(rc)
+            &&  ssmR3StrmSetError(pStrm, rc))
+            LogRel(("ssmR3StrmWriteBuffers: Write failed with rc=%Rrc at offStream=%#llx\n", rc, pCur->offStream));
+
+        /* free */
+        bool fEndOfStream = pCur->fEndOfStream;
+        ssmR3StrmPutFreeBuf(pStrm, pCur);
+        if (fEndOfStream)
+        {
+            Assert(!pHead);
+            return VINF_EOF;
+        }
+    }
+
+    return pStrm->rc;
+}
+
+
+/**
+ * Closes the stream after first flushing any pending write.
+ *
+ * @returns VBox status code.
+ * @param   pStrm           The stream handle.
+ * @param   fCancelled      Indicates whether the operation was cancelled or
+ *                          not.
+ */
+static int ssmR3StrmClose(PSSMSTRM pStrm, bool fCancelled)
+{
+    /*
+     * Flush, terminate the I/O thread, and close the stream.
+     */
+    if (pStrm->fWrite)
+    {
+        ssmR3StrmFlushCurBuf(pStrm);
+        if (pStrm->hIoThread == NIL_RTTHREAD)
+            ssmR3StrmWriteBuffers(pStrm);
+    }
+
+    if (pStrm->hIoThread != NIL_RTTHREAD)
+        ASMAtomicWriteBool(&pStrm->fTerminating, true);
+
+    int rc;
+    if (pStrm->fWrite)
+    {
+        if (pStrm->hIoThread != NIL_RTTHREAD)
+        {
+            int rc2 = RTSemEventSignal(pStrm->hEvtHead);
+            AssertLogRelRC(rc2);
+            int rc3 = RTThreadWait(pStrm->hIoThread, RT_INDEFINITE_WAIT, NULL);
+            AssertLogRelRC(rc3);
+            pStrm->hIoThread = NIL_RTTHREAD;
+        }
+
+        rc = pStrm->pOps->pfnClose(pStrm->pvUser, fCancelled);
+        if (RT_FAILURE(rc))
+            ssmR3StrmSetError(pStrm, rc);
+    }
+    else
+    {
+        rc = pStrm->pOps->pfnClose(pStrm->pvUser, fCancelled);
+        if (RT_FAILURE(rc))
+            ssmR3StrmSetError(pStrm, rc);
+
+        if (pStrm->hIoThread != NIL_RTTHREAD)
+        {
+            int rc2 = RTSemEventSignal(pStrm->hEvtFree);
+            AssertLogRelRC(rc2);
+            int rc3 = RTThreadWait(pStrm->hIoThread, RT_INDEFINITE_WAIT, NULL);
+            AssertLogRelRC(rc3);
+            pStrm->hIoThread = NIL_RTTHREAD;
+        }
+    }
+
+    pStrm->pOps   = NULL;
+    pStrm->pvUser = NULL;
+
+    rc = pStrm->rc;
+    ssmR3StrmDelete(pStrm);
+
+    return rc;
+}
+
+#ifndef SSM_STANDALONE
+
+/**
+ * Stream output routine.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle.
+ * @param   pvBuf       What to write.
+ * @param   cbToWrite   How much to write.
+ *
+ * @thread  The producer in a write stream (never the I/O thread).
+ */
+static int ssmR3StrmWrite(PSSMSTRM pStrm, const void *pvBuf, size_t cbToWrite)
+{
+    AssertReturn(cbToWrite > 0, VINF_SUCCESS);
+    Assert(pStrm->fWrite);
+
+    /*
+     * Squeeze as much as possible into the current buffer.
+     */
+    PSSMSTRMBUF pBuf = pStrm->pCur;
+    if (RT_LIKELY(pBuf))
+    {
+        uint32_t cbLeft = RT_SIZEOFMEMB(SSMSTRMBUF, abData) - pStrm->off;
+        if (RT_LIKELY(cbLeft >= cbToWrite))
+        {
+            memcpy(&pBuf->abData[pStrm->off], pvBuf, cbToWrite);
+            pStrm->off += (uint32_t)cbToWrite;
+            return VINF_SUCCESS;
+        }
+
+        if (cbLeft > 0)
+        {
+            memcpy(&pBuf->abData[pStrm->off], pvBuf, cbLeft);
+            pStrm->off += cbLeft;
+            cbToWrite  -= cbLeft;
+            pvBuf       = (uint8_t const *)pvBuf + cbLeft;
+        }
+        Assert(pStrm->off == RT_SIZEOFMEMB(SSMSTRMBUF, abData));
+    }
+
+    /*
+     * Need one or more new buffers.
+     */
+    do
+    {
+        /*
+         * Flush the current buffer and replace it with a new one.
+         */
+        ssmR3StrmFlushCurBuf(pStrm);
+        pBuf = ssmR3StrmGetFreeBuf(pStrm);
+        if (!pBuf)
+            break;
+        pStrm->pCur = pBuf;
+        Assert(pStrm->off == 0);
+
+        /*
+         * Copy data to the buffer.
+         */
+        uint32_t cbCopy = RT_SIZEOFMEMB(SSMSTRMBUF, abData);
+        if (cbCopy > cbToWrite)
+            cbCopy = (uint32_t)cbToWrite;
+        memcpy(&pBuf->abData[0], pvBuf, cbCopy);
+        pStrm->off  = cbCopy;
+        cbToWrite  -= cbCopy;
+        pvBuf       = (uint8_t const *)pvBuf + cbCopy;
+    } while (cbToWrite > 0);
+
+    return pStrm->rc;
+}
+
+
+/**
+ * Reserves space in the current buffer so the caller can write directly to the
+ * buffer instead of doing double buffering.
+ *
+ * @returns VBox status code
+ * @param   pStrm       The stream handle.
+ * @param   cb          The amount of buffer space to reserve.
+ * @param   ppb         Where to return the pointer.
+ */
+static int ssmR3StrmReserveWriteBufferSpace(PSSMSTRM pStrm, size_t cb, uint8_t **ppb)
+{
+    Assert(pStrm->fWrite);
+    Assert(RT_SIZEOFMEMB(SSMSTRMBUF, abData) / 4 >= cb);
+
+    /*
+     * Check if there is room in the current buffer, it not flush it.
+     */
+    PSSMSTRMBUF pBuf = pStrm->pCur;
+    if (pBuf)
+    {
+        uint32_t cbLeft = RT_SIZEOFMEMB(SSMSTRMBUF, abData) - pStrm->off;
+        if (cbLeft >= cb)
+        {
+            *ppb = &pBuf->abData[pStrm->off];
+            return VINF_SUCCESS;
+        }
+
+        ssmR3StrmFlushCurBuf(pStrm);
+    }
+
+    /*
+     * Get a fresh buffer and return a pointer into it.
+     */
+    pBuf = ssmR3StrmGetFreeBuf(pStrm);
+    if (pBuf)
+    {
+        pStrm->pCur = pBuf;
+        Assert(pStrm->off == 0);
+        *ppb = &pBuf->abData[0];
+    }
+    else
+        *ppb = NULL; /* make gcc happy. */
+    return pStrm->rc;
+}
+
+
+/**
+ * Commits buffer space reserved by ssmR3StrmReserveWriteBufferSpace.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle.
+ * @param   cb          The amount of buffer space to commit.  This can be less
+ *                      that what was reserved initially.
+ */
+static int ssmR3StrmCommitWriteBufferSpace(PSSMSTRM pStrm, size_t cb)
+{
+    Assert(pStrm->pCur);
+    Assert(pStrm->off + cb <= RT_SIZEOFMEMB(SSMSTRMBUF, abData));
+    pStrm->off += (uint32_t)cb;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Marks the end of the stream.
+ *
+ * This will cause the I/O thread to quit waiting for more buffers.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle.
+ */
+static int ssmR3StrmSetEnd(PSSMSTRM pStrm)
+{
+    Assert(pStrm->fWrite);
+    PSSMSTRMBUF pBuf = pStrm->pCur;
+    if (RT_UNLIKELY(!pStrm->pCur))
+    {
+        pBuf = ssmR3StrmGetFreeBuf(pStrm);
+        if (!pBuf)
+            return pStrm->rc;
+        pStrm->pCur = pBuf;
+        Assert(pStrm->off == 0);
+    }
+    pBuf->fEndOfStream = true;
+    ssmR3StrmFlushCurBuf(pStrm);
+    return VINF_SUCCESS;
+}
+
+#endif /* !SSM_STANDALONE */
+
+/**
+ * Read more from the stream.
+ *
+ * @returns VBox status code. VERR_EOF gets translated into VINF_EOF.
+ * @param   pStrm       The stream handle.
+ *
+ * @thread  The I/O thread when we got one, otherwise the stream user.
+ */
+static int ssmR3StrmReadMore(PSSMSTRM pStrm)
+{
+    int rc;
+    Log6(("ssmR3StrmReadMore:\n"));
+
+    /*
+     * Undo seek done by ssmR3StrmPeekAt.
+     */
+    if (pStrm->fNeedSeek)
+    {
+        rc = pStrm->pOps->pfnSeek(pStrm->pvUser, pStrm->offNeedSeekTo, RTFILE_SEEK_BEGIN, NULL);
+        if (RT_FAILURE(rc))
+        {
+            if (ssmR3StrmSetError(pStrm, rc))
+                LogRel(("ssmR3StrmReadMore: RTFileSeek(,%#llx,) failed with rc=%Rrc\n", pStrm->offNeedSeekTo, rc));
+            return rc;
+        }
+        pStrm->fNeedSeek     = false;
+        pStrm->offNeedSeekTo = UINT64_MAX;
+    }
+
+    /*
+     * Get a free buffer and try fill it up.
+     */
+    PSSMSTRMBUF pBuf = ssmR3StrmGetFreeBuf(pStrm);
+    if (!pBuf)
+        return pStrm->rc;
+
+    pBuf->offStream = pStrm->pOps->pfnTell(pStrm->pvUser);
+    size_t cbRead   = sizeof(pBuf->abData);
+    rc = pStrm->pOps->pfnRead(pStrm->pvUser, pBuf->offStream, &pBuf->abData[0], cbRead, &cbRead);
+    if (    RT_SUCCESS(rc)
+        &&  cbRead > 0)
+    {
+        pBuf->cb           = (uint32_t)cbRead;
+        pBuf->fEndOfStream = false;
+        Log6(("ssmR3StrmReadMore: %#010llx %#x\n", pBuf->offStream, pBuf->cb));
+        ssmR3StrmPutBuf(pStrm, pBuf);
+    }
+    else if (   (   RT_SUCCESS_NP(rc)
+                 && cbRead == 0)
+             || rc == VERR_EOF)
+    {
+        pBuf->cb           = 0;
+        pBuf->fEndOfStream = true;
+        Log6(("ssmR3StrmReadMore: %#010llx 0 EOF!\n", pBuf->offStream));
+        ssmR3StrmPutBuf(pStrm, pBuf);
+        rc = VINF_EOF;
+    }
+    else
+    {
+        Log6(("ssmR3StrmReadMore: %#010llx rc=%Rrc!\n", pBuf->offStream, rc));
+        if (ssmR3StrmSetError(pStrm, rc))
+            LogRel(("ssmR3StrmReadMore: RTFileRead(,,%#x,) -> %Rrc at offset %#llx\n",
+                    sizeof(pBuf->abData), rc, pBuf->offStream));
+        ssmR3StrmPutFreeBuf(pStrm, pBuf);
+    }
+    return rc;
+}
+
+
+/**
+ * Stream input routine.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle.
+ * @param   pvBuf       Where to put what we read.
+ * @param   cbToRead    How much to read.
+ */
+static int ssmR3StrmRead(PSSMSTRM pStrm, void *pvBuf, size_t cbToRead)
+{
+    AssertReturn(cbToRead > 0, VINF_SUCCESS);
+    Assert(!pStrm->fWrite);
+
+    /*
+     * Read from the current buffer if we got one.
+     */
+    PSSMSTRMBUF pBuf = pStrm->pCur;
+    if (RT_LIKELY(pBuf))
+    {
+        Assert(pStrm->off <= pBuf->cb);
+        uint32_t cbLeft = pBuf->cb - pStrm->off;
+        if (cbLeft >= cbToRead)
+        {
+            memcpy(pvBuf, &pBuf->abData[pStrm->off], cbToRead);
+            pStrm->off += (uint32_t)cbToRead;
+            Assert(pStrm->off <= pBuf->cb);
+            return VINF_SUCCESS;
+        }
+        if (cbLeft)
+        {
+            memcpy(pvBuf, &pBuf->abData[pStrm->off], cbLeft);
+            pStrm->off += cbLeft;
+            cbToRead   -= cbLeft;
+            pvBuf       = (uint8_t *)pvBuf + cbLeft;
+        }
+        else if (pBuf->fEndOfStream)
+            return VERR_EOF;
+        Assert(pStrm->off == pBuf->cb);
+    }
+
+    /*
+     * Get more buffers from the stream.
+     */
+    int rc = VINF_SUCCESS;
+    do
+    {
+        /*
+         * Check for EOF first - never flush the EOF buffer.
+         */
+        if (    pBuf
+            &&  pBuf->fEndOfStream)
+            return VERR_EOF;
+
+        /*
+         * Flush the current buffer and get the next one.
+         */
+        ssmR3StrmFlushCurBuf(pStrm);
+        pBuf = ssmR3StrmGetBuf(pStrm);
+        if (!pBuf)
+        {
+            rc = pStrm->rc;
+            break;
+        }
+        pStrm->pCur = pBuf;
+        Assert(pStrm->off == 0);
+        Assert(pStrm->offCurStream == pBuf->offStream);
+        if (!pBuf->cb)
+        {
+            Assert(pBuf->fEndOfStream);
+            return VERR_EOF;
+        }
+
+        /*
+         * Read data from the buffer.
+         */
+        uint32_t cbCopy = pBuf->cb;
+        if (cbCopy > cbToRead)
+            cbCopy = (uint32_t)cbToRead;
+        memcpy(pvBuf, &pBuf->abData[0], cbCopy);
+        pStrm->off  = cbCopy;
+        cbToRead   -= cbCopy;
+        pvBuf       = (uint8_t *)pvBuf + cbCopy;
+        Assert(!pStrm->pCur || pStrm->off <= pStrm->pCur->cb);
+    } while (cbToRead > 0);
+
+    return rc;
+}
+
+
+/**
+ * Reads data from the stream but instead of copying it to some output buffer
+ * the caller gets a pointer to into the current stream buffer.
+ *
+ * The returned pointer becomes invalid after the next stream operation!
+ *
+ * @returns Pointer to the read data residing in the stream buffer.  NULL is
+ *          returned if the request amount of data isn't available in the
+ *          buffer.  The caller must fall back on ssmR3StrmRead when this
+ *          happens.
+ *
+ * @param   pStrm       The stream handle.
+ * @param   cbToRead    The number of bytes to tread.
+ */
+static uint8_t const *ssmR3StrmReadDirect(PSSMSTRM pStrm, size_t cbToRead)
+{
+    AssertReturn(cbToRead > 0, VINF_SUCCESS);
+    Assert(!pStrm->fWrite);
+
+    /*
+     * Too lazy to fetch more data for the odd case that we're
+     * exactly at the boundary between two buffers.
+     */
+    PSSMSTRMBUF pBuf = pStrm->pCur;
+    if (RT_LIKELY(pBuf))
+    {
+        Assert(pStrm->off <= pBuf->cb);
+        uint32_t cbLeft = pBuf->cb - pStrm->off;
+        if (cbLeft >= cbToRead)
+        {
+            uint8_t const *pb = &pBuf->abData[pStrm->off];
+            pStrm->off += (uint32_t)cbToRead;
+            Assert(pStrm->off <= pBuf->cb);
+            return pb;
+        }
+    }
+    return NULL;
+}
+
+
+#ifndef SSM_STANDALONE
+/**
+ * Check that the stream is OK and flush data that is getting old
+ *
+ * The checking is mainly for testing for cancellation and out of space
+ * conditions.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle.
+ */
+static int ssmR3StrmCheckAndFlush(PSSMSTRM pStrm)
+{
+    int rc = pStrm->pOps->pfnIsOk(pStrm->pvUser);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    if (    pStrm->fWrite
+        &&  pStrm->hIoThread != NIL_RTTHREAD
+        &&  !pStrm->pHead /* the worker is probably idle */
+        &&  pStrm->pCur
+        &&  RTTimeNanoTS() - pStrm->pCur->NanoTS > 500*1000*1000 /* 0.5s */
+       )
+        ssmR3StrmFlushCurBuf(pStrm);
+    return VINF_SUCCESS;
+}
+#endif /* !SSM_STANDALONE */
+
+
+#if !defined(SSM_STANDALONE) || defined(LOG_ENABLED)
+/**
+ * Tell current stream position.
+ *
+ * @returns stream position.
+ * @param   pStrm       The stream handle.
+ */
+static uint64_t ssmR3StrmTell(PSSMSTRM pStrm)
+{
+    return pStrm->offCurStream + pStrm->off;
+}
+#endif
+
+
+/**
+ * Gets the intermediate stream CRC up to the current position.
+ *
+ * @returns CRC.
+ * @param   pStrm       The stream handle.
+ */
+static uint32_t ssmR3StrmCurCRC(PSSMSTRM pStrm)
+{
+    if (!pStrm->fChecksummed)
+        return 0;
+    if (pStrm->offStreamCRC < pStrm->off)
+    {
+        PSSMSTRMBUF pBuf = pStrm->pCur; Assert(pBuf);
+        pStrm->u32StreamCRC = RTCrc32Process(pStrm->u32StreamCRC, &pBuf->abData[pStrm->offStreamCRC], pStrm->off - pStrm->offStreamCRC);
+        pStrm->offStreamCRC = pStrm->off;
+    }
+    else
+        Assert(pStrm->offStreamCRC == pStrm->off);
+    return pStrm->u32StreamCRC;
+}
+
+
+/**
+ * Gets the final stream CRC up to the current position.
+ *
+ * @returns CRC.
+ * @param   pStrm       The stream handle.
+ */
+static uint32_t ssmR3StrmFinalCRC(PSSMSTRM pStrm)
+{
+    if (!pStrm->fChecksummed)
+        return 0;
+    return RTCrc32Finish(ssmR3StrmCurCRC(pStrm));
+}
+
+
+/**
+ * Disables checksumming of the stream.
+ *
+ * @param   pStrm       The stream handle.
+ */
+static void ssmR3StrmDisableChecksumming(PSSMSTRM pStrm)
+{
+    pStrm->fChecksummed = false;
+}
+
+
+/**
+ * Used by SSMR3Seek to position the stream at the new unit.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The strem handle.
+ * @param   off         The seek offset.
+ * @param   uMethod     The seek method.
+ * @param   u32CurCRC   The current CRC at the seek position.
+ */
+static int ssmR3StrmSeek(PSSMSTRM pStrm, int64_t off, uint32_t uMethod, uint32_t u32CurCRC)
+{
+    AssertReturn(!pStrm->fWrite, VERR_NOT_SUPPORTED);
+    AssertReturn(pStrm->hIoThread == NIL_RTTHREAD, VERR_WRONG_ORDER);
+
+    uint64_t offStream;
+    int rc = pStrm->pOps->pfnSeek(pStrm->pvUser, off, uMethod, &offStream);
+    if (RT_SUCCESS(rc))
+    {
+        pStrm->fNeedSeek    = false;
+        pStrm->offNeedSeekTo= UINT64_MAX;
+        pStrm->offCurStream = offStream;
+        pStrm->off          = 0;
+        pStrm->offStreamCRC = 0;
+        if (pStrm->fChecksummed)
+            pStrm->u32StreamCRC = u32CurCRC;
+        if (pStrm->pCur)
+        {
+            ssmR3StrmPutFreeBuf(pStrm, pStrm->pCur);
+            pStrm->pCur = NULL;
+        }
+        if (pStrm->pPending)
+        {
+            ssmR3StrmDestroyBufList(pStrm->pPending);
+            pStrm->pPending = NULL;
+        }
+        if (pStrm->pHead)
+        {
+            ssmR3StrmDestroyBufList(pStrm->pHead);
+            pStrm->pHead = NULL;
+        }
+    }
+    return rc;
+}
+
+
+#ifndef SSM_STANDALONE
+/**
+ * Skip some bytes in the stream.
+ *
+ * This is only used if someone didn't read all of their data in the V1 format,
+ * so don't bother making this very efficient yet.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle.
+ * @param   offDst      The destination offset.
+ */
+static int ssmR3StrmSkipTo(PSSMSTRM pStrm, uint64_t offDst)
+{
+    /* dead simple - lazy bird! */
+    for (;;)
+    {
+        uint64_t offCur = ssmR3StrmTell(pStrm);
+        AssertReturn(offCur <= offDst, VERR_SSM_SKIP_BACKWARDS);
+        if (offCur == offDst)
+            return VINF_SUCCESS;
+
+        uint8_t abBuf[4096];
+        size_t cbToRead = RT_MIN(sizeof(abBuf), offDst - offCur);
+        int rc = ssmR3StrmRead(pStrm, abBuf, cbToRead);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+}
+#endif /* !SSM_STANDALONE */
+
+
+/**
+ * Get the size of the file.
+ *
+ * This does not work for non-file streams!
+ *
+ * @returns The file size, or UINT64_MAX if not a file stream.
+ * @param   pStrm       The stream handle.
+ */
+static uint64_t ssmR3StrmGetSize(PSSMSTRM pStrm)
+{
+    uint64_t cbFile;
+    int rc = pStrm->pOps->pfnSize(pStrm->pvUser, &cbFile);
+    AssertLogRelRCReturn(rc, UINT64_MAX);
+    return cbFile;
+}
+
+
+/***
+ * Tests if the stream is a file stream or not.
+ *
+ * @returns true / false.
+ * @param   pStrm       The stream handle.
+ */
+static bool ssmR3StrmIsFile(PSSMSTRM pStrm)
+{
+    return pStrm->pOps == &g_ssmR3FileOps;
+}
+
+
+/**
+ * Peeks at data in a file stream without buffering anything (or upsetting
+ * the buffering for that matter).
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle
+ * @param   off         The offset to start peeking at. Use a negative offset to
+ *                      peek at something relative to the end of the file.
+ * @param   pvBuf       Output buffer.
+ * @param   cbToRead    How much to read.
+ * @param   poff        Where to optionally store the position.  Useful when
+ *                      using a negative off.
+ *
+ * @remarks Failures occurring while peeking will not be raised on the stream.
+ */
+static int ssmR3StrmPeekAt(PSSMSTRM pStrm, RTFOFF off, void *pvBuf, size_t cbToRead, uint64_t *poff)
+{
+    AssertReturn(!pStrm->fWrite, VERR_NOT_SUPPORTED);
+    AssertReturn(pStrm->hIoThread == NIL_RTTHREAD, VERR_WRONG_ORDER);
+
+    if (!pStrm->fNeedSeek)
+    {
+        pStrm->fNeedSeek     = true;
+        pStrm->offNeedSeekTo = pStrm->offCurStream + (pStrm->pCur ? pStrm->pCur->cb : 0);
+    }
+    uint64_t offActual;
+    int rc = pStrm->pOps->pfnSeek(pStrm->pvUser, off, off >= 0 ? RTFILE_SEEK_BEGIN : RTFILE_SEEK_END, &offActual);
+    if (RT_SUCCESS(rc))
+    {
+        if (poff)
+            *poff = offActual;
+        rc = pStrm->pOps->pfnRead(pStrm->pvUser, offActual, pvBuf, cbToRead, NULL);
+    }
+
+    return rc;
+}
+
+#ifndef SSM_STANDALONE
+
+/**
+ * The I/O thread.
+ *
+ * @returns VINF_SUCCESS (ignored).
+ * @param   hSelf       The thread handle.
+ * @param   pvStrm      The stream handle.
+ */
+static DECLCALLBACK(int) ssmR3StrmIoThread(RTTHREAD hSelf, void *pvStrm)
+{
+    PSSMSTRM pStrm = (PSSMSTRM)pvStrm;
+    ASMAtomicWriteHandle(&pStrm->hIoThread, hSelf); /* paranoia */
+
+    Log(("ssmR3StrmIoThread: starts working\n"));
+    if (pStrm->fWrite)
+    {
+        /*
+         * Write until error or terminated.
+         */
+        for (;;)
+        {
+            int rc = ssmR3StrmWriteBuffers(pStrm);
+            if (    RT_FAILURE(rc)
+                ||  rc == VINF_EOF)
+            {
+                Log(("ssmR3StrmIoThread: quitting writing with rc=%Rrc.\n", rc));
+                break;
+            }
+            if (RT_FAILURE(pStrm->rc))
+            {
+                Log(("ssmR3StrmIoThread: quitting writing with stream rc=%Rrc\n", pStrm->rc));
+                break;
+            }
+
+            if (ASMAtomicReadBool(&pStrm->fTerminating))
+            {
+                if (!ASMAtomicReadPtrT(&pStrm->pHead, PSSMSTRMBUF))
+                {
+                    Log(("ssmR3StrmIoThread: quitting writing because of pending termination.\n"));
+                    break;
+                }
+                Log(("ssmR3StrmIoThread: postponing termination because of pending buffers.\n"));
+            }
+            else if (!ASMAtomicReadPtrT(&pStrm->pHead, PSSMSTRMBUF))
+            {
+                rc = RTSemEventWait(pStrm->hEvtHead, RT_INDEFINITE_WAIT);
+                AssertLogRelRC(rc);
+            }
+        }
+
+        if (!ASMAtomicReadBool(&pStrm->fTerminating))
+            RTSemEventSignal(pStrm->hEvtFree);
+    }
+    else
+    {
+        /*
+         * Read until end of file, error or termination.
+         */
+        for (;;)
+        {
+            if (ASMAtomicReadBool(&pStrm->fTerminating))
+            {
+                Log(("ssmR3StrmIoThread: quitting reading because of pending termination.\n"));
+                break;
+            }
+
+            int rc = ssmR3StrmReadMore(pStrm);
+            if (    RT_FAILURE(rc)
+                ||  rc == VINF_EOF)
+            {
+                Log(("ssmR3StrmIoThread: quitting reading with rc=%Rrc\n", rc));
+                break;
+            }
+            if (RT_FAILURE(pStrm->rc))
+            {
+                Log(("ssmR3StrmIoThread: quitting reading with stream rc=%Rrc\n", pStrm->rc));
+                break;
+            }
+        }
+
+        if (!ASMAtomicReadBool(&pStrm->fTerminating))
+            RTSemEventSignal(pStrm->hEvtHead);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Starts the I/O thread for the specified stream.
+ *
+ * @param   pStrm       The stream handle.
+ */
+static void ssmR3StrmStartIoThread(PSSMSTRM pStrm)
+{
+    Assert(pStrm->hIoThread == NIL_RTTHREAD);
+
+    RTTHREAD hThread;
+    int rc = RTThreadCreate(&hThread, ssmR3StrmIoThread, pStrm, 0, RTTHREADTYPE_IO, RTTHREADFLAGS_WAITABLE, "SSM-IO");
+    AssertRCReturnVoid(rc);
+    ASMAtomicWriteHandle(&pStrm->hIoThread, hThread); /* paranoia */
+}
+
+
+/**
+ * Stops the I/O thread.
+ *
+ * @param   pStrm       The stream handle.
+ */
+static void ssmR3StrmStopIoThread(PSSMSTRM pStrm)
+{
+    LogFlow(("ssmR3StrmStopIoThread: %p\n", pStrm->hIoThread));
+    if (pStrm->hIoThread != NIL_RTTHREAD)
+    {
+        /*
+         * Signal the I/O thread and wait for it to complete.
+         */
+        ASMAtomicWriteBool(&pStrm->fTerminating, true);
+        if (pStrm->fWrite)
+        {
+            int rc1 = RTSemEventSignal(pStrm->hEvtHead);
+            AssertLogRelRC(rc1);
+        }
+        else
+        {
+            int rc2 = RTSemEventSignal(pStrm->hEvtFree);
+            AssertLogRelRC(rc2);
+        }
+        int rc3 = RTThreadWait(pStrm->hIoThread, RT_INDEFINITE_WAIT, NULL);
+        AssertLogRelRC(rc3);
+        pStrm->hIoThread = NIL_RTTHREAD;
+        pStrm->fTerminating = false; /* Can't read stuff otherwise. */
+    }
+}
+
+#endif /* !SSM_STANDALONE */
+
+/**
+ * Works the progress calculation for non-live saves and restores.
+ *
+ * @param   pSSM        The SSM handle.
+ * @param   cbAdvance   Number of bytes to advance (with in the current unit).
+ */
+static void ssmR3ProgressByByte(PSSMHANDLE pSSM, uint64_t cbAdvance)
+{
+    if (!pSSM->fLiveSave)
+    {
+        /* Can't advance it beyond the estimated end of the unit. */
+        uint64_t cbLeft = pSSM->offEstUnitEnd - pSSM->offEst;
+        if (cbAdvance > cbLeft)
+            cbAdvance = cbLeft;
+        pSSM->offEst += cbAdvance;
+
+        /* uPercentPrepare% prepare, xx% exec, uPercentDone% done+crc. This is not
+           quite right for live save, but the non-live stage there is very short. */
+        while (   pSSM->offEst >= pSSM->offEstProgress
+               && pSSM->uPercent <= 100 - pSSM->uPercentDone)
+        {
+            if (pSSM->pfnProgress)
+                pSSM->pfnProgress(pSSM->pVM->pUVM, pSSM->uPercent, pSSM->pvUser);
+            pSSM->uPercent++;
+            pSSM->offEstProgress = (pSSM->uPercent - pSSM->uPercentPrepare - pSSM->uPercentLive) * pSSM->cbEstTotal
+                                 / (100 - pSSM->uPercentDone - pSSM->uPercentPrepare - pSSM->uPercentLive);
+        }
+    }
+}
+
+
+#ifndef SSM_STANDALONE
+/**
+ * Makes the SSM operation cancellable or not (via SSMR3Cancel).
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The saved state handle. (SSMHANDLE::rc may be set.)
+ * @param   fCancellable    The new state.
+ */
+static void ssmR3SetCancellable(PVM pVM, PSSMHANDLE pSSM, bool fCancellable)
+{
+    RTCritSectEnter(&pVM->ssm.s.CancelCritSect);
+    if (fCancellable)
+    {
+        Assert(!pVM->ssm.s.pSSM);
+        pVM->ssm.s.pSSM = pSSM;
+    }
+    else
+    {
+        if (pVM->ssm.s.pSSM == pSSM)
+            pVM->ssm.s.pSSM = NULL;
+
+        uint32_t fCancelled = ASMAtomicUoReadU32(&pSSM->fCancelled);
+        if (    fCancelled == SSMHANDLE_CANCELLED
+            &&  RT_SUCCESS(pSSM->rc))
+            pSSM->rc = VERR_SSM_CANCELLED;
+    }
+
+    RTCritSectLeave(&pVM->ssm.s.CancelCritSect);
+}
+#endif /* !SSM_STANDALONE */
+
+
+/**
+ * Gets the host bit count of the saved state.
+ *
+ * Works for on both save and load handles.
+ *
+ * @returns 32 or 64.
+ * @param   pSSM                The saved state handle.
+ */
+DECLINLINE(uint32_t) ssmR3GetHostBits(PSSMHANDLE pSSM)
+{
+    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
+    {
+        uint32_t cBits = pSSM->u.Read.cHostBits;
+        if (cBits)
+            return cBits;
+    }
+    return HC_ARCH_BITS;
+}
+
+
+/**
+ * Saved state origins on a host using 32-bit MSC?
+ *
+ * Works for on both save and load handles.
+ *
+ * @returns true/false.
+ * @param   pSSM                The saved state handle.
+ */
+DECLINLINE(bool) ssmR3IsHostMsc32(PSSMHANDLE pSSM)
+{
+    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
+        return pSSM->u.Read.fIsHostMsc32;
+    return SSM_HOST_IS_MSC_32;
+}
+
+#ifndef SSM_STANDALONE
+
+/**
+ * Finishes a data unit.
+ * All buffers and compressor instances are flushed and destroyed.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ */
+static int ssmR3DataWriteFinish(PSSMHANDLE pSSM)
+{
+    //Log2(("ssmR3DataWriteFinish: %#010llx start\n", ssmR3StrmTell(&pSSM->Strm)));
+    int rc = ssmR3DataFlushBuffer(pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        pSSM->offUnit     = UINT64_MAX;
+        pSSM->offUnitUser = UINT64_MAX;
+        return VINF_SUCCESS;
+    }
+
+    if (RT_SUCCESS(pSSM->rc))
+        pSSM->rc = rc;
+    Log2(("ssmR3DataWriteFinish: failure rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Begins writing the data of a data unit.
+ *
+ * Errors are signalled via pSSM->rc.
+ *
+ * @param   pSSM            The saved state handle.
+ */
+static void ssmR3DataWriteBegin(PSSMHANDLE pSSM)
+{
+    pSSM->offUnit     = 0;
+    pSSM->offUnitUser = 0;
+}
+
+
+/**
+ * Writes a record to the current data item in the saved state file.
+ *
+ * @returns VBox status code. Sets pSSM->rc on failure.
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           The bits to write.
+ * @param   cbBuf           The number of bytes to write.
+ */
+static int ssmR3DataWriteRaw(PSSMHANDLE pSSM, const void *pvBuf, size_t cbBuf)
+{
+    Log2(("ssmR3DataWriteRaw: %08llx|%08llx: pvBuf=%p cbBuf=%#x %.*Rhxs%s\n",
+          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pvBuf, cbBuf, RT_MIN(cbBuf, SSM_LOG_BYTES), pvBuf, cbBuf > SSM_LOG_BYTES ? "..." : ""));
+
+    /*
+     * Check that everything is fine.
+     */
+    if (RT_FAILURE(pSSM->rc))
+        return pSSM->rc;
+
+    /*
+     * Write the data item in 1MB chunks for progress indicator reasons.
+     */
+    while (cbBuf > 0)
+    {
+        size_t cbChunk = RT_MIN(cbBuf, _1M);
+        int rc = ssmR3StrmWrite(&pSSM->Strm, pvBuf, cbChunk);
+        if (RT_FAILURE(rc))
+            return rc;
+        pSSM->offUnit += cbChunk;
+        cbBuf -= cbChunk;
+        pvBuf = (char *)pvBuf + cbChunk;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Writes a record header for the specified amount of data.
+ *
+ * @returns VBox status code. Sets pSSM->rc on failure.
+ * @param   pSSM            The saved state handle
+ * @param   cb              The amount of data.
+ * @param   u8TypeAndFlags  The record type and flags.
+ */
+static int ssmR3DataWriteRecHdr(PSSMHANDLE pSSM, size_t cb, uint8_t u8TypeAndFlags)
+{
+    size_t  cbHdr;
+    uint8_t abHdr[8];
+    abHdr[0] = u8TypeAndFlags;
+    if (cb < 0x80)
+    {
+        cbHdr = 2;
+        abHdr[1] = (uint8_t)cb;
+    }
+    else if (cb < 0x00000800)
+    {
+        cbHdr = 3;
+        abHdr[1] = (uint8_t)(0xc0 | (cb >> 6));
+        abHdr[2] = (uint8_t)(0x80 | (cb & 0x3f));
+    }
+    else if (cb < 0x00010000)
+    {
+        cbHdr = 4;
+        abHdr[1] = (uint8_t)(0xe0 | (cb >> 12));
+        abHdr[2] = (uint8_t)(0x80 | ((cb >> 6) & 0x3f));
+        abHdr[3] = (uint8_t)(0x80 | (cb & 0x3f));
+    }
+    else if (cb < 0x00200000)
+    {
+        cbHdr = 5;
+        abHdr[1] = (uint8_t)(0xf0 |  (cb >> 18));
+        abHdr[2] = (uint8_t)(0x80 | ((cb >> 12) & 0x3f));
+        abHdr[3] = (uint8_t)(0x80 | ((cb >>  6) & 0x3f));
+        abHdr[4] = (uint8_t)(0x80 |  (cb        & 0x3f));
+    }
+    else if (cb < 0x04000000)
+    {
+        cbHdr = 6;
+        abHdr[1] = (uint8_t)(0xf8 |  (cb >> 24));
+        abHdr[2] = (uint8_t)(0x80 | ((cb >> 18) & 0x3f));
+        abHdr[3] = (uint8_t)(0x80 | ((cb >> 12) & 0x3f));
+        abHdr[4] = (uint8_t)(0x80 | ((cb >>  6) & 0x3f));
+        abHdr[5] = (uint8_t)(0x80 |  (cb        & 0x3f));
+    }
+    else if (cb <= 0x7fffffff)
+    {
+        cbHdr = 7;
+        abHdr[1] = (uint8_t)(0xfc |  (cb >> 30));
+        abHdr[2] = (uint8_t)(0x80 | ((cb >> 24) & 0x3f));
+        abHdr[3] = (uint8_t)(0x80 | ((cb >> 18) & 0x3f));
+        abHdr[4] = (uint8_t)(0x80 | ((cb >> 12) & 0x3f));
+        abHdr[5] = (uint8_t)(0x80 | ((cb >>  6) & 0x3f));
+        abHdr[6] = (uint8_t)(0x80 | (cb & 0x3f));
+    }
+    else
+        AssertLogRelMsgFailedReturn(("cb=%#x\n", cb), pSSM->rc = VERR_SSM_MEM_TOO_BIG);
+
+    Log3(("ssmR3DataWriteRecHdr: %08llx|%08llx/%08x: Type=%02x fImportant=%RTbool cbHdr=%u\n",
+          ssmR3StrmTell(&pSSM->Strm) + cbHdr, pSSM->offUnit + cbHdr, cb, u8TypeAndFlags & SSM_REC_TYPE_MASK, !!(u8TypeAndFlags & SSM_REC_FLAGS_IMPORTANT), cbHdr));
+
+    return ssmR3DataWriteRaw(pSSM, &abHdr[0], cbHdr);
+}
+
+
+/**
+ * Worker that flushes the buffered data.
+ *
+ * @returns VBox status code. Will set pSSM->rc on error.
+ * @param   pSSM            The saved state handle.
+ */
+static int ssmR3DataFlushBuffer(PSSMHANDLE pSSM)
+{
+    /*
+     * Check how much there current is in the buffer.
+     */
+    uint32_t cb = pSSM->u.Write.offDataBuffer;
+    if (!cb)
+        return pSSM->rc;
+    pSSM->u.Write.offDataBuffer = 0;
+
+    /*
+     * Write a record header and then the data.
+     * (No need for fancy optimizations here any longer since the stream is
+     * fully buffered.)
+     */
+    int rc = ssmR3DataWriteRecHdr(pSSM, cb, SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW);
+    if (RT_SUCCESS(rc))
+        rc = ssmR3DataWriteRaw(pSSM, pSSM->u.Write.abDataBuffer, cb);
+    ssmR3ProgressByByte(pSSM, cb);
+    return rc;
+}
+
+
+/**
+ * ssmR3DataWrite worker that writes big stuff.
+ *
+ * @returns VBox status code
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           The bits to write.
+ * @param   cbBuf           The number of bytes to write.
+ */
+static int ssmR3DataWriteBig(PSSMHANDLE pSSM, const void *pvBuf, size_t cbBuf)
+{
+    int rc = ssmR3DataFlushBuffer(pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        pSSM->offUnitUser += cbBuf;
+
+        /*
+         * Split it up into compression blocks.
+         */
+        for (;;)
+        {
+            AssertCompile(SSM_ZIP_BLOCK_SIZE == PAGE_SIZE);
+            if (    cbBuf >= SSM_ZIP_BLOCK_SIZE
+                && (    ((uintptr_t)pvBuf & 0xf)
+                    ||  !ASMMemIsZeroPage(pvBuf))
+               )
+            {
+                /*
+                 * Compress it.
+                 */
+                AssertCompile(1 + 3 + 1 + SSM_ZIP_BLOCK_SIZE < 0x00010000);
+                uint8_t *pb;
+                rc = ssmR3StrmReserveWriteBufferSpace(&pSSM->Strm, 1 + 3 + 1 + SSM_ZIP_BLOCK_SIZE, &pb);
+                if (RT_FAILURE(rc))
+                    break;
+                size_t cbRec = SSM_ZIP_BLOCK_SIZE - (SSM_ZIP_BLOCK_SIZE / 16);
+                rc = RTZipBlockCompress(RTZIPTYPE_LZF, RTZIPLEVEL_FAST, 0 /*fFlags*/,
+                                        pvBuf, SSM_ZIP_BLOCK_SIZE,
+                                        pb + 1 + 3 + 1, cbRec, &cbRec);
+                if (RT_SUCCESS(rc))
+                {
+                    pb[0] = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW_LZF;
+                    pb[4] = SSM_ZIP_BLOCK_SIZE / _1K;
+                    cbRec += 1;
+                }
+                else
+                {
+                    pb[0] = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW;
+                    memcpy(&pb[4], pvBuf, SSM_ZIP_BLOCK_SIZE);
+                    cbRec = SSM_ZIP_BLOCK_SIZE;
+                }
+                pb[1] = (uint8_t)(0xe0 | ( cbRec >> 12));
+                pb[2] = (uint8_t)(0x80 | ((cbRec >>  6) & 0x3f));
+                pb[3] = (uint8_t)(0x80 | ( cbRec        & 0x3f));
+                cbRec += 1 + 3;
+                rc = ssmR3StrmCommitWriteBufferSpace(&pSSM->Strm, cbRec);
+                if (RT_FAILURE(rc))
+                    break;
+
+                pSSM->offUnit += cbRec;
+                ssmR3ProgressByByte(pSSM, SSM_ZIP_BLOCK_SIZE);
+
+                /* advance */
+                if (cbBuf == SSM_ZIP_BLOCK_SIZE)
+                    return VINF_SUCCESS;
+                cbBuf -= SSM_ZIP_BLOCK_SIZE;
+                pvBuf = (uint8_t const*)pvBuf + SSM_ZIP_BLOCK_SIZE;
+            }
+            else if (cbBuf >= SSM_ZIP_BLOCK_SIZE)
+            {
+                /*
+                 * Zero block.
+                 */
+                uint8_t abRec[3];
+                abRec[0] = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW_ZERO;
+                abRec[1] = 1;
+                abRec[2] = SSM_ZIP_BLOCK_SIZE / _1K;
+                Log3(("ssmR3DataWriteBig: %08llx|%08llx/%08x: ZERO\n", ssmR3StrmTell(&pSSM->Strm) + 2, pSSM->offUnit + 2, 1));
+                rc = ssmR3DataWriteRaw(pSSM, &abRec[0], sizeof(abRec));
+                if (RT_FAILURE(rc))
+                    break;
+
+                /* advance */
+                ssmR3ProgressByByte(pSSM, SSM_ZIP_BLOCK_SIZE);
+                if (cbBuf == SSM_ZIP_BLOCK_SIZE)
+                    return VINF_SUCCESS;
+                cbBuf -= SSM_ZIP_BLOCK_SIZE;
+                pvBuf = (uint8_t const*)pvBuf + SSM_ZIP_BLOCK_SIZE;
+            }
+            else
+            {
+                /*
+                 * Less than one block left, store it the simple way.
+                 */
+                rc = ssmR3DataWriteRecHdr(pSSM, cbBuf, SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW);
+                if (RT_SUCCESS(rc))
+                    rc = ssmR3DataWriteRaw(pSSM, pvBuf, cbBuf);
+                ssmR3ProgressByByte(pSSM, cbBuf);
+                break;
+            }
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * ssmR3DataWrite worker that is called when there isn't enough room in the
+ * buffer for the current chunk of data.
+ *
+ * This will first flush the buffer and then add the new bits to it.
+ *
+ * @returns VBox status code
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           The bits to write.
+ * @param   cbBuf           The number of bytes to write.
+ */
+static int ssmR3DataWriteFlushAndBuffer(PSSMHANDLE pSSM, const void *pvBuf, size_t cbBuf)
+{
+    int rc = ssmR3DataFlushBuffer(pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        memcpy(&pSSM->u.Write.abDataBuffer[0], pvBuf, cbBuf);
+        pSSM->u.Write.offDataBuffer = (uint32_t)cbBuf;
+        pSSM->offUnitUser += cbBuf;
+    }
+    return rc;
+}
+
+
+/**
+ * Writes data to the current data unit.
+ *
+ * This is an inlined wrapper that optimizes the small writes that so many of
+ * the APIs make.
+ *
+ * @returns VBox status code
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           The bits to write.
+ * @param   cbBuf           The number of bytes to write.
+ */
+DECLINLINE(int) ssmR3DataWrite(PSSMHANDLE pSSM, const void *pvBuf, size_t cbBuf)
+{
+    if (cbBuf > sizeof(pSSM->u.Write.abDataBuffer) / 8)
+        return ssmR3DataWriteBig(pSSM, pvBuf, cbBuf);
+    if (!cbBuf)
+        return VINF_SUCCESS;
+
+    uint32_t off = pSSM->u.Write.offDataBuffer;
+    if (RT_UNLIKELY(cbBuf + off > sizeof(pSSM->u.Write.abDataBuffer)))
+        return ssmR3DataWriteFlushAndBuffer(pSSM, pvBuf, cbBuf);
+
+    memcpy(&pSSM->u.Write.abDataBuffer[off], pvBuf, cbBuf);
+    pSSM->u.Write.offDataBuffer = off + (uint32_t)cbBuf;
+    pSSM->offUnitUser += cbBuf;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Puts a structure.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pvStruct        The structure address.
+ * @param   paFields        The array of structure fields descriptions.
+ *                          The array must be terminated by a SSMFIELD_ENTRY_TERM().
+ */
+VMMR3DECL(int) SSMR3PutStruct(PSSMHANDLE pSSM, const void *pvStruct, PCSSMFIELD paFields)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    AssertPtr(pvStruct);
+    AssertPtr(paFields);
+
+    /* begin marker. */
+    int rc = SSMR3PutU32(pSSM, SSMR3STRUCT_BEGIN);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* put the fields */
+    for (PCSSMFIELD pCur = paFields;
+         pCur->cb != UINT32_MAX && pCur->off != UINT32_MAX;
+         pCur++)
+    {
+        uint8_t const *pbField = (uint8_t const *)pvStruct + pCur->off;
+        switch ((uintptr_t)pCur->pfnGetPutOrTransformer)
+        {
+            case SSMFIELDTRANS_NO_TRANSFORMATION:
+                rc = ssmR3DataWrite(pSSM, pbField, pCur->cb);
+                break;
+
+            case SSMFIELDTRANS_GCPTR:
+                AssertMsgBreakStmt(pCur->cb == sizeof(RTGCPTR), ("%#x (%s)\n", pCur->cb, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3PutGCPtr(pSSM, *(PRTGCPTR)pbField);
+                break;
+
+            case SSMFIELDTRANS_GCPHYS:
+                AssertMsgBreakStmt(pCur->cb == sizeof(RTGCPHYS), ("%#x (%s)\n", pCur->cb, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3PutGCPhys(pSSM, *(PRTGCPHYS)pbField);
+                break;
+
+            case SSMFIELDTRANS_RCPTR:
+                AssertMsgBreakStmt(pCur->cb == sizeof(RTRCPTR), ("%#x (%s)\n", pCur->cb, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3PutRCPtr(pSSM, *(PRTRCPTR)pbField);
+                break;
+
+            case SSMFIELDTRANS_RCPTR_ARRAY:
+            {
+                uint32_t const cEntries = pCur->cb / sizeof(RTRCPTR);
+                AssertMsgBreakStmt(pCur->cb == cEntries * sizeof(RTRCPTR) && cEntries, ("%#x (%s)\n", pCur->cb, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = VINF_SUCCESS;
+                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+                    rc = SSMR3PutRCPtr(pSSM, ((PRTRCPTR)pbField)[i]);
+                break;
+            }
+
+            default:
+                AssertMsgFailedBreakStmt(("%#x\n", pCur->pfnGetPutOrTransformer), rc = VERR_SSM_FIELD_COMPLEX);
+        }
+        if (RT_FAILURE(rc))
+        {
+            if (RT_SUCCESS(pSSM->rc))
+                pSSM->rc = rc;
+            return rc;
+        }
+    }
+
+    /* end marker */
+    return SSMR3PutU32(pSSM, SSMR3STRUCT_END);
+}
+
+
+/**
+ * SSMR3PutStructEx helper that puts a HCPTR that is used as a NULL indicator.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSSM            The saved state handle.
+ * @param   pv              The value to put.
+ * @param   fFlags          SSMSTRUCT_FLAGS_XXX.
+ */
+DECLINLINE(int) ssmR3PutHCPtrNI(PSSMHANDLE pSSM, void *pv, uint32_t fFlags)
+{
+    int rc;
+    if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+        rc = ssmR3DataWrite(pSSM, &pv, sizeof(void *));
+    else
+        rc = SSMR3PutBool(pSSM, pv != NULL);
+    return rc;
+}
+
+
+/**
+ * SSMR3PutStructEx helper that puts an arbitrary number of zeros.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   cbToFill        The number of zeros to stuff into the state.
+ */
+static int ssmR3PutZeros(PSSMHANDLE pSSM, uint32_t cbToFill)
+{
+    while (cbToFill > 0)
+    {
+        uint32_t cb = RT_MIN(sizeof(g_abZero), cbToFill);
+        int rc = ssmR3DataWrite(pSSM, g_abZero, cb);
+        if (RT_FAILURE(rc))
+            return rc;
+        cbToFill -= cb;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Puts a structure, extended API.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pvStruct        The structure address.
+ * @param   cbStruct        The size of the struct (use for validation only).
+ * @param   fFlags          Combination of SSMSTRUCT_FLAGS_XXX defines.
+ * @param   paFields        The array of structure fields descriptions. The
+ *                          array must be terminated by a SSMFIELD_ENTRY_TERM().
+ * @param   pvUser          User argument for any callbacks that paFields might
+ *                          contain.
+ */
+VMMR3DECL(int) SSMR3PutStructEx(PSSMHANDLE pSSM, const void *pvStruct, size_t cbStruct,
+                                uint32_t fFlags, PCSSMFIELD paFields, void *pvUser)
+{
+    int rc;
+
+    /*
+     * Validation.
+     */
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    AssertMsgReturn(!(fFlags & ~SSMSTRUCT_FLAGS_VALID_MASK), ("%#x\n", fFlags), pSSM->rc = VERR_INVALID_PARAMETER);
+    AssertPtr(pvStruct);
+    AssertPtr(paFields);
+
+
+    /*
+     * Begin marker.
+     */
+    if (!(fFlags & (SSMSTRUCT_FLAGS_NO_MARKERS | SSMSTRUCT_FLAGS_NO_LEAD_MARKER)))
+    {
+        rc = SSMR3PutU32(pSSM, SSMR3STRUCT_BEGIN);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Put the fields
+     */
+    rc = VINF_SUCCESS;
+    uint32_t off = 0;
+    for (PCSSMFIELD pCur = paFields;
+         pCur->cb != UINT32_MAX && pCur->off != UINT32_MAX;
+         pCur++)
+    {
+        uint32_t const offField =    (!SSMFIELDTRANS_IS_PADDING(pCur->pfnGetPutOrTransformer) || pCur->off != UINT32_MAX / 2)
+                                  &&  !SSMFIELDTRANS_IS_OLD(pCur->pfnGetPutOrTransformer)
+                                ? pCur->off
+                                : off;
+        uint32_t const cbField  = SSMFIELDTRANS_IS_OLD(pCur->pfnGetPutOrTransformer)
+                                ? 0
+                                : SSMFIELDTRANS_IS_PADDING(pCur->pfnGetPutOrTransformer)
+                                ? RT_HIWORD(pCur->cb)
+                                : pCur->cb;
+        AssertMsgBreakStmt(   cbField            <= cbStruct
+                           && offField + cbField <= cbStruct
+                           && offField + cbField >= offField,
+                           ("offField=%#x cbField=%#x cbStruct=%#x (%s)\n", offField, cbField, cbStruct, pCur->pszName),
+                           rc = VERR_SSM_FIELD_OUT_OF_BOUNDS);
+        AssertMsgBreakStmt(   !(fFlags & SSMSTRUCT_FLAGS_FULL_STRUCT)
+                           || off == offField,
+                           ("off=%#x offField=%#x (%s)\n", off, offField, pCur->pszName),
+                           rc = VERR_SSM_FIELD_NOT_CONSECUTIVE);
+
+        rc = VINF_SUCCESS;
+        uint8_t const *pbField = (uint8_t const *)pvStruct + offField;
+        switch ((uintptr_t)pCur->pfnGetPutOrTransformer)
+        {
+            case SSMFIELDTRANS_NO_TRANSFORMATION:
+                rc = ssmR3DataWrite(pSSM, pbField, cbField);
+                break;
+
+            case SSMFIELDTRANS_GCPHYS:
+                AssertMsgBreakStmt(cbField == sizeof(RTGCPHYS), ("%#x (%s)\n", cbField, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3PutGCPhys(pSSM, *(PRTGCPHYS)pbField);
+                break;
+
+            case SSMFIELDTRANS_GCPTR:
+                AssertMsgBreakStmt(cbField == sizeof(RTGCPTR), ("%#x (%s)\n", cbField, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3PutGCPtr(pSSM, *(PRTGCPTR)pbField);
+                break;
+
+            case SSMFIELDTRANS_RCPTR:
+                AssertMsgBreakStmt(cbField == sizeof(RTRCPTR), ("%#x (%s)\n", cbField, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3PutRCPtr(pSSM, *(PRTRCPTR)pbField);
+                break;
+
+            case SSMFIELDTRANS_RCPTR_ARRAY:
+            {
+                uint32_t const cEntries = cbField / sizeof(RTRCPTR);
+                AssertMsgBreakStmt(cbField == cEntries * sizeof(RTRCPTR) && cEntries, ("%#x (%s)\n", cbField, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+                    rc = SSMR3PutRCPtr(pSSM, ((PRTRCPTR)pbField)[i]);
+                break;
+            }
+
+            case SSMFIELDTRANS_HCPTR_NI:
+                AssertMsgBreakStmt(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3PutHCPtrNI(pSSM, *(void * const *)pbField, fFlags);
+                break;
+
+            case SSMFIELDTRANS_HCPTR_NI_ARRAY:
+            {
+                uint32_t const cEntries = cbField / sizeof(void *);
+                AssertMsgBreakStmt(cbField == cEntries * sizeof(void *) && cEntries, ("%#x (%s)\n", cbField, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+                    rc = ssmR3PutHCPtrNI(pSSM, ((void * const *)pbField)[i], fFlags);
+                break;
+            }
+
+            case SSMFIELDTRANS_HCPTR_HACK_U32:
+                AssertMsgBreakStmt(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                AssertMsgBreakStmt(*(uintptr_t *)pbField <= UINT32_MAX, ("%p (%s)\n", *(uintptr_t *)pbField, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_VALUE);
+                rc = ssmR3DataWrite(pSSM, pbField, sizeof(uint32_t));
+                if ((fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE) && sizeof(void *) != sizeof(uint32_t) && RT_SUCCESS(rc))
+                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(uint32_t));
+                break;
+
+            case SSMFIELDTRANS_U32_ZX_U64:
+                AssertFailedBreakStmt(rc = VERR_SSM_FIELD_LOAD_ONLY_TRANSFORMATION);
+                break;
+
+            case SSMFIELDTRANS_IGNORE:
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = ssmR3PutZeros(pSSM, cbField);
+                break;
+
+            case SSMFIELDTRANS_IGN_GCPHYS:
+                AssertMsgBreakStmt(cbField == sizeof(RTGCPHYS), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTGCPHYS));
+                break;
+
+            case SSMFIELDTRANS_IGN_GCPTR:
+                AssertMsgBreakStmt(cbField == sizeof(RTGCPTR), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTGCPTR));
+                break;
+
+            case SSMFIELDTRANS_IGN_RCPTR:
+                AssertMsgBreakStmt(cbField == sizeof(RTRCPTR), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTRCPTR));
+                break;
+
+            case SSMFIELDTRANS_IGN_HCPTR:
+                AssertMsgBreakStmt(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(void *));
+                break;
+
+
+            case SSMFIELDTRANS_OLD:
+                AssertMsgBreakStmt(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3PutZeros(pSSM, pCur->cb);
+                break;
+
+            case SSMFIELDTRANS_OLD_GCPHYS:
+                AssertMsgBreakStmt(pCur->cb == sizeof(RTGCPHYS) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTGCPHYS));
+                break;
+
+            case SSMFIELDTRANS_OLD_GCPTR:
+                AssertMsgBreakStmt(pCur->cb == sizeof(RTGCPTR) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTGCPTR));
+                break;
+
+            case SSMFIELDTRANS_OLD_RCPTR:
+                AssertMsgBreakStmt(pCur->cb == sizeof(RTRCPTR) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTRCPTR));
+                break;
+
+            case SSMFIELDTRANS_OLD_HCPTR:
+                AssertMsgBreakStmt(pCur->cb == sizeof(void *) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(void *));
+                break;
+
+            case SSMFIELDTRANS_OLD_PAD_HC:
+                AssertMsgBreakStmt(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3PutZeros(pSSM, HC_ARCH_BITS == 64 ? RT_HIWORD(pCur->cb) : RT_LOWORD(pCur->cb));
+                break;
+
+            case SSMFIELDTRANS_OLD_PAD_MSC32:
+                AssertMsgBreakStmt(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName),
+                                   rc = VERR_SSM_FIELD_INVALID_SIZE);
+                if (SSM_HOST_IS_MSC_32)
+                    rc = ssmR3PutZeros(pSSM, pCur->cb);
+                break;
+
+
+            case SSMFIELDTRANS_PAD_HC:
+            case SSMFIELDTRANS_PAD_HC32:
+            case SSMFIELDTRANS_PAD_HC64:
+            case SSMFIELDTRANS_PAD_HC_AUTO:
+            case SSMFIELDTRANS_PAD_MSC32_AUTO:
+            {
+                uint32_t cb32    = RT_BYTE1(pCur->cb);
+                uint32_t cb64    = RT_BYTE2(pCur->cb);
+                uint32_t cbCtx   =    HC_ARCH_BITS == 64
+                                   || (   (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
+                                       && !SSM_HOST_IS_MSC_32)
+                                 ? cb64 : cb32;
+                uint32_t cbSaved =    ssmR3GetHostBits(pSSM) == 64
+                                   || (   (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
+                                       && !ssmR3IsHostMsc32(pSSM))
+                                 ? cb64 : cb32;
+                AssertMsgBreakStmt(    cbField == cbCtx
+                                   &&  (   (   pCur->off == UINT32_MAX / 2
+                                            && (   cbField == 0
+                                                || (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_HC_AUTO
+                                                || (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
+                                               )
+                                           )
+                                        || (pCur->off != UINT32_MAX / 2 && cbField != 0)
+                                        )
+                                   , ("cbField=%#x cb32=%#x cb64=%#x HC_ARCH_BITS=%u cbCtx=%#x cbSaved=%#x off=%#x\n",
+                                      cbField, cb32, cb64, HC_ARCH_BITS, cbCtx, cbSaved, pCur->off),
+                                   rc = VERR_SSM_FIELD_INVALID_PADDING_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = ssmR3PutZeros(pSSM, cbSaved);
+                break;
+            }
+
+            default:
+                AssertPtrBreakStmt(pCur->pfnGetPutOrTransformer, rc = VERR_SSM_FIELD_INVALID_CALLBACK);
+                rc = pCur->pfnGetPutOrTransformer(pSSM, pCur, (void *)pvStruct, fFlags, false /*fGetOrPut*/, pvUser);
+                break;
+        }
+        if (RT_FAILURE(rc))
+            break; /* Deal with failures in one place (see below). */
+
+        off = offField + cbField;
+    }
+
+    if (RT_SUCCESS(rc))
+        AssertMsgStmt(   !(fFlags & SSMSTRUCT_FLAGS_FULL_STRUCT)
+                      || off == cbStruct,
+                      ("off=%#x cbStruct=%#x\n", off, cbStruct),
+                      rc = VERR_SSM_FIELD_NOT_CONSECUTIVE);
+
+    if (RT_FAILURE(rc))
+    {
+        if (RT_SUCCESS(pSSM->rc))
+            pSSM->rc = rc;
+        return rc;
+    }
+
+    /*
+     * End marker
+     */
+    if (!(fFlags & (SSMSTRUCT_FLAGS_NO_MARKERS | SSMSTRUCT_FLAGS_NO_TAIL_MARKER)))
+    {
+        rc = SSMR3PutU32(pSSM, SSMR3STRUCT_END);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Saves a boolean item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   fBool           Item to save.
+ */
+VMMR3DECL(int) SSMR3PutBool(PSSMHANDLE pSSM, bool fBool)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    uint8_t u8 = fBool; /* enforce 1 byte size */
+    return ssmR3DataWrite(pSSM, &u8, sizeof(u8));
+}
+
+
+/**
+ * Saves a 8-bit unsigned integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   u8              Item to save.
+ */
+VMMR3DECL(int) SSMR3PutU8(PSSMHANDLE pSSM, uint8_t u8)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u8, sizeof(u8));
+}
+
+
+/**
+ * Saves a 8-bit signed integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   i8              Item to save.
+ */
+VMMR3DECL(int) SSMR3PutS8(PSSMHANDLE pSSM, int8_t i8)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &i8, sizeof(i8));
+}
+
+
+/**
+ * Saves a 16-bit unsigned integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   u16             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutU16(PSSMHANDLE pSSM, uint16_t u16)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u16, sizeof(u16));
+}
+
+
+/**
+ * Saves a 16-bit signed integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   i16             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutS16(PSSMHANDLE pSSM, int16_t i16)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &i16, sizeof(i16));
+}
+
+
+/**
+ * Saves a 32-bit unsigned integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   u32             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutU32(PSSMHANDLE pSSM, uint32_t u32)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u32, sizeof(u32));
+}
+
+
+/**
+ * Saves a 32-bit signed integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   i32             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutS32(PSSMHANDLE pSSM, int32_t i32)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &i32, sizeof(i32));
+}
+
+
+/**
+ * Saves a 64-bit unsigned integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   u64             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutU64(PSSMHANDLE pSSM, uint64_t u64)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u64, sizeof(u64));
+}
+
+
+/**
+ * Saves a 64-bit signed integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   i64             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutS64(PSSMHANDLE pSSM, int64_t i64)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &i64, sizeof(i64));
+}
+
+
+/**
+ * Saves a 128-bit unsigned integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   u128            Item to save.
+ */
+VMMR3DECL(int) SSMR3PutU128(PSSMHANDLE pSSM, uint128_t u128)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u128, sizeof(u128));
+}
+
+
+/**
+ * Saves a 128-bit signed integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   i128            Item to save.
+ */
+VMMR3DECL(int) SSMR3PutS128(PSSMHANDLE pSSM, int128_t i128)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &i128, sizeof(i128));
+}
+
+
+/**
+ * Saves a VBox unsigned integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   u               Item to save.
+ */
+VMMR3DECL(int) SSMR3PutUInt(PSSMHANDLE pSSM, RTUINT u)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u, sizeof(u));
+}
+
+
+/**
+ * Saves a VBox signed integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   i               Item to save.
+ */
+VMMR3DECL(int) SSMR3PutSInt(PSSMHANDLE pSSM, RTINT i)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &i, sizeof(i));
+}
+
+
+/**
+ * Saves a GC natural unsigned integer item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   u               Item to save.
+ *
+ * @deprecated Silly type, don't use it.
+ */
+VMMR3DECL(int) SSMR3PutGCUInt(PSSMHANDLE pSSM, RTGCUINT u)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u, sizeof(u));
+}
+
+
+/**
+ * Saves a GC unsigned integer register item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   u               Item to save.
+ */
+VMMR3DECL(int) SSMR3PutGCUIntReg(PSSMHANDLE pSSM, RTGCUINTREG u)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u, sizeof(u));
+}
+
+
+/**
+ * Saves a 32 bits GC physical address item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   GCPhys          The item to save
+ */
+VMMR3DECL(int) SSMR3PutGCPhys32(PSSMHANDLE pSSM, RTGCPHYS32 GCPhys)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &GCPhys, sizeof(GCPhys));
+}
+
+
+/**
+ * Saves a 64 bits GC physical address item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   GCPhys          The item to save
+ */
+VMMR3DECL(int) SSMR3PutGCPhys64(PSSMHANDLE pSSM, RTGCPHYS64 GCPhys)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &GCPhys, sizeof(GCPhys));
+}
+
+
+/**
+ * Saves a GC physical address item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   GCPhys          The item to save
+ */
+VMMR3DECL(int) SSMR3PutGCPhys(PSSMHANDLE pSSM, RTGCPHYS GCPhys)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &GCPhys, sizeof(GCPhys));
+}
+
+
+/**
+ * Saves a GC virtual address item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   GCPtr           The item to save.
+ */
+VMMR3DECL(int) SSMR3PutGCPtr(PSSMHANDLE pSSM, RTGCPTR GCPtr)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &GCPtr, sizeof(GCPtr));
+}
+
+
+/**
+ * Saves an RC virtual address item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   RCPtr           The item to save.
+ */
+VMMR3DECL(int) SSMR3PutRCPtr(PSSMHANDLE pSSM, RTRCPTR RCPtr)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &RCPtr, sizeof(RCPtr));
+}
+
+
+/**
+ * Saves a GC virtual address (represented as an unsigned integer) item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   GCPtr           The item to save.
+ */
+VMMR3DECL(int) SSMR3PutGCUIntPtr(PSSMHANDLE pSSM, RTGCUINTPTR GCPtr)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &GCPtr, sizeof(GCPtr));
+}
+
+
+/**
+ * Saves a I/O port address item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   IOPort          The item to save.
+ */
+VMMR3DECL(int) SSMR3PutIOPort(PSSMHANDLE pSSM, RTIOPORT IOPort)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &IOPort, sizeof(IOPort));
+}
+
+
+/**
+ * Saves a selector item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   Sel             The item to save.
+ */
+VMMR3DECL(int) SSMR3PutSel(PSSMHANDLE pSSM, RTSEL Sel)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &Sel, sizeof(Sel));
+}
+
+
+/**
+ * Saves a memory item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pv              Item to save.
+ * @param   cb              Size of the item.
+ */
+VMMR3DECL(int) SSMR3PutMem(PSSMHANDLE pSSM, const void *pv, size_t cb)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, pv, cb);
+}
+
+
+/**
+ * Saves a zero terminated string item to the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   psz             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutStrZ(PSSMHANDLE pSSM, const char *psz)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+
+    size_t cch = strlen(psz);
+    if (cch > _1M)
+    {
+        AssertMsgFailed(("a %zu byte long string, what's this!?!\n", cch));
+        return VERR_TOO_MUCH_DATA;
+    }
+    uint32_t u32 = (uint32_t)cch;
+    int rc = ssmR3DataWrite(pSSM, &u32, sizeof(u32));
+    if (rc)
+        return rc;
+    return ssmR3DataWrite(pSSM, psz, cch);
+}
+
+
+/**
+ * Emits a SSMLiveControl unit with a new progress report.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                    The saved state handle.
+ * @param   lrdPct                  The progress of the live save.
+ * @param   uPass                   The current pass.
+ */
+static int ssmR3LiveControlEmit(PSSMHANDLE pSSM, long double lrdPct, uint32_t uPass)
+{
+    AssertMsg(lrdPct <= 100.0, ("%u\n", lrdPct * 100));
+
+    /*
+     * Make sure we're in one of the two EXEC states or we may fail.
+     */
+    SSMSTATE enmSavedState = pSSM->enmOp;
+    if (enmSavedState == SSMSTATE_LIVE_VOTE)
+        pSSM->enmOp = SSMSTATE_LIVE_EXEC;
+    else if (enmSavedState == SSMSTATE_SAVE_DONE)
+        pSSM->enmOp = SSMSTATE_SAVE_EXEC;
+
+    /*
+     * Write the unit header.
+     */
+    SSMFILEUNITHDRV2 UnitHdr;
+    memcpy(&UnitHdr.szMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic));
+    UnitHdr.offStream       = ssmR3StrmTell(&pSSM->Strm);
+    UnitHdr.u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
+    UnitHdr.u32CRC          = 0;
+    UnitHdr.u32Version      = 1;
+    UnitHdr.u32Instance     = 0;
+    UnitHdr.u32Pass         = uPass;
+    UnitHdr.fFlags          = 0;
+    UnitHdr.cbName          = sizeof("SSMLiveControl");
+    memcpy(&UnitHdr.szName[0], "SSMLiveControl", UnitHdr.cbName);
+    UnitHdr.u32CRC          = RTCrc32(&UnitHdr, RT_UOFFSETOF_DYN(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
+    Log(("SSM: Unit at %#9llx: '%s', instance %u, pass %#x, version %u\n",
+         UnitHdr.offStream, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass, UnitHdr.u32Version));
+    int rc = ssmR3StrmWrite(&pSSM->Strm, &UnitHdr, RT_UOFFSETOF_DYN(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Write the payload.
+         */
+        ssmR3DataWriteBegin(pSSM);
+
+        uint16_t u16PartsPerTenThousand = (uint16_t)(lrdPct * (100 - pSSM->uPercentDone));
+        AssertMsg(u16PartsPerTenThousand <= 10000, ("%u\n", u16PartsPerTenThousand));
+        ssmR3DataWrite(pSSM, &u16PartsPerTenThousand, sizeof(u16PartsPerTenThousand));
+
+        rc = ssmR3DataFlushBuffer(pSSM); /* will return SSMHANDLE::rc if it is set */
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Write the termination record and flush the compression stream.
+             */
+            SSMRECTERM TermRec;
+            TermRec.u8TypeAndFlags   = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_TERM;
+            TermRec.cbRec            = sizeof(TermRec) - 2;
+            if (pSSM->Strm.fChecksummed)
+            {
+                TermRec.fFlags       = SSMRECTERM_FLAGS_CRC32;
+                TermRec.u32StreamCRC = RTCrc32Finish(RTCrc32Process(ssmR3StrmCurCRC(&pSSM->Strm), &TermRec, 2));
+            }
+            else
+            {
+                TermRec.fFlags       = 0;
+                TermRec.u32StreamCRC = 0;
+            }
+            TermRec.cbUnit           = pSSM->offUnit + sizeof(TermRec);
+            rc = ssmR3DataWriteRaw(pSSM, &TermRec, sizeof(TermRec));
+            if (RT_SUCCESS(rc))
+                rc = ssmR3DataWriteFinish(pSSM);
+            if (RT_SUCCESS(rc))
+            {
+                pSSM->enmOp = enmSavedState;
+                return rc;
+            }
+        }
+    }
+
+    LogRel(("SSM: Failed to write live control unit. rc=%Rrc\n", rc));
+    if (RT_SUCCESS_NP(pSSM->rc))
+        pSSM->rc = rc;
+    pSSM->enmOp = enmSavedState;
+    return rc;
+}
+
+
+
+/**
+ * Enters the critical session (optionally) associated with the unit.
+ *
+ * @param   pUnit               The unit.
+ */
+DECLINLINE(void) ssmR3UnitCritSectEnter(PSSMUNIT pUnit)
+{
+    PPDMCRITSECT pCritSect = pUnit->pCritSect;
+    if (pCritSect)
+    {
+        int rc = PDMCritSectEnter(pCritSect, VERR_IGNORED);
+        AssertRC(rc);
+    }
+}
+
+
+/**
+ * Leaves the critical session (optionally) associated with the unit.
+ *
+ * @param   pUnit               The unit.
+ */
+DECLINLINE(void) ssmR3UnitCritSectLeave(PSSMUNIT pUnit)
+{
+    PPDMCRITSECT pCritSect = pUnit->pCritSect;
+    if (pCritSect)
+    {
+        int rc = PDMCritSectLeave(pCritSect);
+        AssertRC(rc);
+    }
+}
+
+
+/**
+ * Do the pfnSaveDone run.
+ *
+ * @returns VBox status code (pSSM->rc).
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3SaveDoDoneRun(PVM pVM, PSSMHANDLE pSSM)
+{
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Do the done run.
+     */
+    pSSM->enmOp = SSMSTATE_SAVE_DONE;
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+    {
+        if (    pUnit->u.Common.pfnSaveDone
+            &&  (   pUnit->fCalled
+                 || (!pUnit->u.Common.pfnSavePrep && !pUnit->u.Common.pfnSaveExec)))
+        {
+            int rcOld = pSSM->rc;
+            int rc;
+            ssmR3UnitCritSectEnter(pUnit);
+            switch (pUnit->enmType)
+            {
+                case SSMUNITTYPE_DEV:
+                    rc = pUnit->u.Dev.pfnSaveDone(pUnit->u.Dev.pDevIns, pSSM);
+                    break;
+                case SSMUNITTYPE_DRV:
+                    rc = pUnit->u.Drv.pfnSaveDone(pUnit->u.Drv.pDrvIns, pSSM);
+                    break;
+                case SSMUNITTYPE_USB:
+                    rc = pUnit->u.Usb.pfnSaveDone(pUnit->u.Usb.pUsbIns, pSSM);
+                    break;
+                case SSMUNITTYPE_INTERNAL:
+                    rc = pUnit->u.Internal.pfnSaveDone(pVM, pSSM);
+                    break;
+                case SSMUNITTYPE_EXTERNAL:
+                    rc = pUnit->u.External.pfnSaveDone(pSSM, pUnit->u.External.pvUser);
+                    break;
+                default:
+                    rc = VERR_SSM_IPE_1;
+                    break;
+            }
+            ssmR3UnitCritSectLeave(pUnit);
+            if (RT_SUCCESS(rc) && pSSM->rc != rcOld)
+                rc = pSSM->rc;
+            if (RT_FAILURE(rc))
+            {
+                LogRel(("SSM: Done save failed with rc=%Rrc for data unit '%s.\n", rc, pUnit->szName));
+                if (RT_SUCCESS_NP(pSSM->rc))
+                    pSSM->rc = rc;
+            }
+        }
+    }
+    return pSSM->rc;
+}
+
+
+/**
+ * Worker for SSMR3LiveDone and SSMR3Save that closes the handle and deletes the
+ * saved state file on failure.
+ *
+ * @returns VBox status code (pSSM->rc).
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3SaveDoClose(PVM pVM, PSSMHANDLE pSSM)
+{
+    VM_ASSERT_EMT0(pVM);
+    pVM->ssm.s.uPass = 0;
+
+    /*
+     * Make it non-cancellable, close the stream and delete the file on failure.
+     */
+    ssmR3SetCancellable(pVM, pSSM, false);
+    int rc = ssmR3StrmClose(&pSSM->Strm, pSSM->rc == VERR_SSM_CANCELLED);
+    if (RT_SUCCESS(rc))
+        rc = pSSM->rc;
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pSSM->enmOp == SSMSTATE_SAVE_DONE);
+        if (pSSM->pfnProgress)
+            pSSM->pfnProgress(pVM->pUVM, 100, pSSM->pvUser);
+        LogRel(("SSM: Successfully saved the VM state to '%s'\n",
+                pSSM->pszFilename ? pSSM->pszFilename : "<remote-machine>"));
+    }
+    else
+    {
+        if (pSSM->pszFilename)
+        {
+            int rc2 = RTFileDelete(pSSM->pszFilename);
+            AssertRC(rc2);
+            if (RT_SUCCESS(rc2))
+                LogRel(("SSM: Failed to save the VM state to '%s' (file deleted): %Rrc\n",
+                        pSSM->pszFilename, rc));
+            else
+                LogRel(("SSM: Failed to save the VM state to '%s' (file deletion failed, rc2=%Rrc): %Rrc\n",
+                        pSSM->pszFilename, rc2, rc));
+        }
+        else
+            LogRel(("SSM: Failed to save the VM state.\n"));
+
+        Assert(pSSM->enmOp <= SSMSTATE_SAVE_DONE);
+        if (pSSM->enmOp != SSMSTATE_SAVE_DONE)
+            ssmR3SaveDoDoneRun(pVM, pSSM);
+    }
+
+    /*
+     * Trash the handle before freeing it.
+     */
+    ASMAtomicWriteU32(&pSSM->fCancelled, 0);
+    pSSM->pVM = NULL;
+    pSSM->enmAfter = SSMAFTER_INVALID;
+    pSSM->enmOp = SSMSTATE_INVALID;
+    RTMemFree(pSSM);
+
+    return rc;
+}
+
+
+/**
+ * Closes the SSM handle.
+ *
+ * This must always be called on a handled returned by SSMR3LiveSave.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSSM            The SSM handle returned by SSMR3LiveSave.
+ *
+ * @thread  EMT(0).
+ */
+VMMR3_INT_DECL(int) SSMR3LiveDone(PSSMHANDLE pSSM)
+{
+    LogFlow(("SSMR3LiveDone: pSSM=%p\n", pSSM));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pSSM, VERR_INVALID_POINTER);
+    PVM pVM = pSSM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT0(pVM);
+    AssertMsgReturn(   pSSM->enmAfter == SSMAFTER_DESTROY
+                    || pSSM->enmAfter == SSMAFTER_CONTINUE
+                    || pSSM->enmAfter == SSMAFTER_TELEPORT,
+                    ("%d\n", pSSM->enmAfter),
+                    VERR_INVALID_PARAMETER);
+    AssertMsgReturn(    pSSM->enmOp >= SSMSTATE_LIVE_PREP
+                    &&  pSSM->enmOp <= SSMSTATE_SAVE_DONE,
+                    ("%d\n", pSSM->enmOp), VERR_INVALID_STATE);
+
+    /*
+     * Join paths with SSMR3Save again.
+     */
+    return ssmR3SaveDoClose(pVM, pSSM);
+}
+
+
+/**
+ * Writes the directory.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The SSM handle.
+ * @param   pcEntries           Where to return the number of directory entries.
+ */
+static int ssmR3WriteDirectory(PVM pVM, PSSMHANDLE pSSM, uint32_t *pcEntries)
+{
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Grab some temporary memory for the dictionary.
+     */
+    size_t      cbDir = RT_UOFFSETOF_DYN(SSMFILEDIR, aEntries[pVM->ssm.s.cUnits]);
+    PSSMFILEDIR pDir  = (PSSMFILEDIR)RTMemTmpAlloc(cbDir);
+    if (!pDir)
+    {
+        LogRel(("ssmR3WriteDirectory: failed to allocate %zu bytes!\n", cbDir));
+        return VERR_NO_TMP_MEMORY;
+    }
+
+    /*
+     * Initialize it.
+     */
+    memcpy(pDir->szMagic, SSMFILEDIR_MAGIC, sizeof(pDir->szMagic));
+    pDir->u32CRC   = 0;
+    pDir->cEntries = 0;
+
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+        if (pUnit->offStream != RTFOFF_MIN)
+        {
+            PSSMFILEDIRENTRY pEntry = &pDir->aEntries[pDir->cEntries++];
+            Assert(pDir->cEntries <= pVM->ssm.s.cUnits);
+            Assert(pUnit->offStream >= (RTFOFF)sizeof(SSMFILEHDR));
+            pEntry->off         = pUnit->offStream;
+            pEntry->u32Instance = pUnit->u32Instance;
+            pEntry->u32NameCRC  = RTCrc32(pUnit->szName, pUnit->cchName);
+        }
+
+    /*
+     * Calculate the actual size and CRC-32, then write the directory
+     * out to the stream.
+     */
+    *pcEntries = pDir->cEntries;
+    cbDir = RT_UOFFSETOF_DYN(SSMFILEDIR, aEntries[pDir->cEntries]);
+    pDir->u32CRC = RTCrc32(pDir, cbDir);
+    int rc = ssmR3StrmWrite(&pSSM->Strm, pDir, cbDir);
+    RTMemTmpFree(pDir);
+    return rc;
+}
+
+
+/**
+ * Finalize the saved state stream, i.e. add the end unit, directory
+ * and footer.
+ *
+ * @returns VBox status code (pSSM->rc).
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3SaveDoFinalization(PVM pVM, PSSMHANDLE pSSM)
+{
+    VM_ASSERT_EMT0(pVM);
+    Assert(RT_SUCCESS(pSSM->rc));
+
+    /*
+     * Write the end unit.
+     */
+    SSMFILEUNITHDRV2 UnitHdr;
+    memcpy(&UnitHdr.szMagic[0], SSMFILEUNITHDR_END, sizeof(UnitHdr.szMagic));
+    UnitHdr.offStream       = ssmR3StrmTell(&pSSM->Strm);
+    UnitHdr.u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
+    UnitHdr.u32CRC          = 0;
+    UnitHdr.u32Version      = 0;
+    UnitHdr.u32Instance     = 0;
+    UnitHdr.u32Pass         = SSM_PASS_FINAL;
+    UnitHdr.fFlags          = 0;
+    UnitHdr.cbName          = 0;
+    UnitHdr.u32CRC          = RTCrc32(&UnitHdr, RT_UOFFSETOF(SSMFILEUNITHDRV2, szName[0]));
+    Log(("SSM: Unit at %#9llx: END UNIT\n", UnitHdr.offStream));
+    int rc = ssmR3StrmWrite(&pSSM->Strm, &UnitHdr, RT_UOFFSETOF(SSMFILEUNITHDRV2, szName[0]));
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("SSM: Failed writing the end unit: %Rrc\n", rc));
+        return pSSM->rc = rc;
+    }
+
+    /*
+     * Write the directory for the final units and then the footer.
+     */
+    SSMFILEFTR Footer;
+    rc = ssmR3WriteDirectory(pVM, pSSM, &Footer.cDirEntries);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("SSM: Failed writing the directory: %Rrc\n", rc));
+        return pSSM->rc = rc;
+    }
+
+    memcpy(Footer.szMagic, SSMFILEFTR_MAGIC, sizeof(Footer.szMagic));
+    Footer.offStream    = ssmR3StrmTell(&pSSM->Strm);
+    Footer.u32StreamCRC = ssmR3StrmFinalCRC(&pSSM->Strm);
+    Footer.u32Reserved  = 0;
+    Footer.u32CRC       = 0;
+    Footer.u32CRC       = RTCrc32(&Footer, sizeof(Footer));
+    Log(("SSM: Footer at %#9llx: \n", Footer.offStream));
+    rc = ssmR3StrmWrite(&pSSM->Strm, &Footer, sizeof(Footer));
+    if (RT_SUCCESS(rc))
+        rc = ssmR3StrmSetEnd(&pSSM->Strm);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("SSM: Failed writing the footer: %Rrc\n", rc));
+        return pSSM->rc = rc;
+    }
+
+    LogRel(("SSM: Footer at %#llx (%lld), %u directory entries.\n",
+            Footer.offStream, Footer.offStream, Footer.cDirEntries));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Works the progress calculation during the exec part of a live save.
+ *
+ * @param   pSSM        The SSM handle.
+ * @param   iUnit       The current unit number.
+ */
+static void ssmR3ProgressByUnit(PSSMHANDLE pSSM, uint32_t iUnit)
+{
+    if (pSSM->fLiveSave)
+    {
+        unsigned    uPctExec = iUnit * 100 / pSSM->pVM->ssm.s.cUnits;
+        unsigned    cPctExec = 100 - pSSM->uPercentDone - pSSM->uPercentPrepare - pSSM->uPercentLive;
+        long double lrdPct   = (long double)uPctExec * cPctExec / 100 + pSSM->uPercentPrepare + pSSM->uPercentLive;
+        unsigned    uPct     = (unsigned)lrdPct;
+        if (uPct != pSSM->uPercent)
+        {
+            ssmR3LiveControlEmit(pSSM, lrdPct, SSM_PASS_FINAL);
+            pSSM->uPercent =  uPct;
+            pSSM->pfnProgress(pSSM->pVM->pUVM, uPct, pSSM->pvUser);
+        }
+    }
+}
+
+
+/**
+ * Do the pfnSaveExec run.
+ *
+ * @returns VBox status code (pSSM->rc).
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3SaveDoExecRun(PVM pVM, PSSMHANDLE pSSM)
+{
+    VM_ASSERT_EMT0(pVM);
+    AssertRC(pSSM->rc);
+    pSSM->rc = VINF_SUCCESS;
+    pSSM->enmOp = SSMSTATE_SAVE_EXEC;
+    unsigned iUnit = 0;
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext, iUnit++)
+    {
+        /*
+         * Not all unit have a callback. Skip those which don't and
+         * make sure to keep the progress indicator up to date.
+         */
+        ssmR3ProgressByUnit(pSSM, iUnit);
+        pSSM->offEstUnitEnd += pUnit->cbGuess;
+        if (!pUnit->u.Common.pfnSaveExec)
+        {
+            pUnit->fCalled = true;
+            if (pUnit->cbGuess)
+                ssmR3ProgressByByte(pSSM, pSSM->offEstUnitEnd - pSSM->offEst);
+            continue;
+        }
+        pUnit->offStream = ssmR3StrmTell(&pSSM->Strm);
+
+        /*
+         * Check for cancellation.
+         */
+        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED))
+        {
+            LogRel(("SSM: Cancelled!\n"));
+            AssertRC(pSSM->rc);
+            return pSSM->rc = VERR_SSM_CANCELLED;
+        }
+
+        /*
+         * Write data unit header
+         */
+        SSMFILEUNITHDRV2 UnitHdr;
+        memcpy(&UnitHdr.szMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic));
+        UnitHdr.offStream       = pUnit->offStream;
+        UnitHdr.u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
+        UnitHdr.u32CRC          = 0;
+        UnitHdr.u32Version      = pUnit->u32Version;
+        UnitHdr.u32Instance     = pUnit->u32Instance;
+        UnitHdr.u32Pass         = SSM_PASS_FINAL;
+        UnitHdr.fFlags          = 0;
+        UnitHdr.cbName          = (uint32_t)pUnit->cchName + 1;
+        memcpy(&UnitHdr.szName[0], &pUnit->szName[0], UnitHdr.cbName);
+        UnitHdr.u32CRC          = RTCrc32(&UnitHdr, RT_UOFFSETOF_DYN(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
+        Log(("SSM: Unit at %#9llx: '%s', instance %u, pass %#x, version %u\n",
+             UnitHdr.offStream, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass, UnitHdr.u32Version));
+        int rc = ssmR3StrmWrite(&pSSM->Strm, &UnitHdr, RT_UOFFSETOF_DYN(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: Failed to write unit header. rc=%Rrc\n", rc));
+            return pSSM->rc = rc;
+        }
+
+        /*
+         * Call the execute handler.
+         */
+        ssmR3DataWriteBegin(pSSM);
+        ssmR3UnitCritSectEnter(pUnit);
+        switch (pUnit->enmType)
+        {
+            case SSMUNITTYPE_DEV:
+                rc = pUnit->u.Dev.pfnSaveExec(pUnit->u.Dev.pDevIns, pSSM);
+                break;
+            case SSMUNITTYPE_DRV:
+                rc = pUnit->u.Drv.pfnSaveExec(pUnit->u.Drv.pDrvIns, pSSM);
+                break;
+            case SSMUNITTYPE_USB:
+                rc = pUnit->u.Usb.pfnSaveExec(pUnit->u.Usb.pUsbIns, pSSM);
+                break;
+            case SSMUNITTYPE_INTERNAL:
+                rc = pUnit->u.Internal.pfnSaveExec(pVM, pSSM);
+                break;
+            case SSMUNITTYPE_EXTERNAL:
+                pUnit->u.External.pfnSaveExec(pSSM, pUnit->u.External.pvUser);
+                rc = pSSM->rc;
+                break;
+            default:
+                rc = VERR_SSM_IPE_1;
+                break;
+        }
+        ssmR3UnitCritSectLeave(pUnit);
+        pUnit->fCalled = true;
+        if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
+            pSSM->rc = rc;
+        else
+            rc = ssmR3DataFlushBuffer(pSSM); /* will return SSMHANDLE::rc if it is set */
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: Execute save failed with rc=%Rrc for data unit '%s'/#%u.\n", rc, pUnit->szName, pUnit->u32Instance));
+            return rc;
+        }
+
+        /*
+         * Write the termination record and flush the compression stream.
+         */
+        SSMRECTERM TermRec;
+        TermRec.u8TypeAndFlags   = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_TERM;
+        TermRec.cbRec            = sizeof(TermRec) - 2;
+        if (pSSM->Strm.fChecksummed)
+        {
+            TermRec.fFlags       = SSMRECTERM_FLAGS_CRC32;
+            TermRec.u32StreamCRC = RTCrc32Finish(RTCrc32Process(ssmR3StrmCurCRC(&pSSM->Strm), &TermRec, 2));
+        }
+        else
+        {
+            TermRec.fFlags       = 0;
+            TermRec.u32StreamCRC = 0;
+        }
+        TermRec.cbUnit           = pSSM->offUnit + sizeof(TermRec);
+        rc = ssmR3DataWriteRaw(pSSM, &TermRec, sizeof(TermRec));
+        if (RT_SUCCESS(rc))
+            rc = ssmR3DataWriteFinish(pSSM);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: Failed terminating unit: %Rrc\n", rc));
+            return pSSM->rc = rc;
+        }
+
+        /*
+         * Advance the progress indicator to the end of the current unit.
+         */
+        ssmR3ProgressByByte(pSSM, pSSM->offEstUnitEnd - pSSM->offEst);
+    } /* for each unit */
+    ssmR3ProgressByUnit(pSSM, pVM->ssm.s.cUnits);
+
+    /* (progress should be pending 99% now) */
+    AssertMsg(   pSSM->uPercent == 101 - pSSM->uPercentDone
+              || pSSM->uPercent == 100 - pSSM->uPercentDone,
+              ("%d\n", pSSM->uPercent));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Do the pfnSavePrep run.
+ *
+ * @returns VBox status code (pSSM->rc).
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3SaveDoPrepRun(PVM pVM, PSSMHANDLE pSSM)
+{
+    VM_ASSERT_EMT0(pVM);
+    Assert(RT_SUCCESS(pSSM->rc));
+    pSSM->enmOp = SSMSTATE_SAVE_PREP;
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+    {
+        if (pUnit->u.Common.pfnSavePrep)
+        {
+            int rc;
+            ssmR3UnitCritSectEnter(pUnit);
+            switch (pUnit->enmType)
+            {
+                case SSMUNITTYPE_DEV:
+                    rc = pUnit->u.Dev.pfnSavePrep(pUnit->u.Dev.pDevIns, pSSM);
+                    break;
+                case SSMUNITTYPE_DRV:
+                    rc = pUnit->u.Drv.pfnSavePrep(pUnit->u.Drv.pDrvIns, pSSM);
+                    break;
+                case SSMUNITTYPE_USB:
+                    rc = pUnit->u.Usb.pfnSavePrep(pUnit->u.Usb.pUsbIns, pSSM);
+                    break;
+                case SSMUNITTYPE_INTERNAL:
+                    rc = pUnit->u.Internal.pfnSavePrep(pVM, pSSM);
+                    break;
+                case SSMUNITTYPE_EXTERNAL:
+                    rc = pUnit->u.External.pfnSavePrep(pSSM, pUnit->u.External.pvUser);
+                    break;
+                default:
+                    rc = VERR_SSM_IPE_1;
+                    break;
+            }
+            ssmR3UnitCritSectLeave(pUnit);
+            pUnit->fCalled = true;
+            if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
+                pSSM->rc = rc;
+            else
+                rc = pSSM->rc;
+            if (RT_FAILURE(rc))
+            {
+                LogRel(("SSM: Prepare save failed with rc=%Rrc for data unit '%s.\n", rc, pUnit->szName));
+                return rc;
+            }
+        }
+
+        pSSM->cbEstTotal += pUnit->cbGuess;
+    }
+
+    /*
+     * Work the progress indicator if we got one.
+     */
+    if (pSSM->pfnProgress)
+        pSSM->pfnProgress(pVM->pUVM, pSSM->uPercentPrepare + pSSM->uPercentLive - 1, pSSM->pvUser);
+    pSSM->uPercent = pSSM->uPercentPrepare + pSSM->uPercentLive;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for SSMR3Save and SSMR3LiveSave.
+ *
+ * @returns VBox status code (no need to check pSSM->rc).
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The state handle.
+ *
+ * @thread  EMT(0)
+ */
+static int ssmR3SaveDoCommon(PVM pVM, PSSMHANDLE pSSM)
+{
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Do the work.
+     */
+    int rc = ssmR3SaveDoPrepRun(pVM, pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        rc = ssmR3SaveDoExecRun(pVM, pSSM);
+        if (RT_SUCCESS(rc))
+            rc = ssmR3SaveDoFinalization(pVM, pSSM);
+    }
+    Assert(pSSM->rc == rc);
+    int rc2 = ssmR3SaveDoDoneRun(pVM, pSSM);
+    if (RT_SUCCESS(rc))
+        rc = rc2;
+
+    return rc;
+}
+
+
+/**
+ * Saves the rest of the state on EMT0.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSSM            The SSM handle returned by SSMR3LiveSave.
+ *
+ * @thread  Non-EMT thread. Will involve the EMT at the end of the operation.
+ */
+VMMR3_INT_DECL(int) SSMR3LiveDoStep2(PSSMHANDLE pSSM)
+{
+    LogFlow(("SSMR3LiveDoStep2: pSSM=%p\n", pSSM));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pSSM, VERR_INVALID_POINTER);
+    PVM pVM = pSSM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT0(pVM);
+    AssertMsgReturn(   pSSM->enmAfter == SSMAFTER_DESTROY
+                    || pSSM->enmAfter == SSMAFTER_CONTINUE
+                    || pSSM->enmAfter == SSMAFTER_TELEPORT,
+                    ("%d\n", pSSM->enmAfter),
+                    VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pSSM->enmOp == SSMSTATE_LIVE_STEP2, ("%d\n", pSSM->enmOp), VERR_INVALID_STATE);
+    AssertRCReturn(pSSM->rc, pSSM->rc);
+
+    /*
+     * Join paths with VMMR3Save.
+     */
+    return ssmR3SaveDoCommon(pVM, pSSM);
+}
+
+
+/**
+ * Writes the file header and clear the per-unit data.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The SSM handle.
+ */
+static int ssmR3WriteHeaderAndClearPerUnitData(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Write the header.
+     */
+    SSMFILEHDR FileHdr;
+    memcpy(&FileHdr.szMagic, SSMFILEHDR_MAGIC_V2_0, sizeof(FileHdr.szMagic));
+    FileHdr.u16VerMajor  = VBOX_VERSION_MAJOR;
+    FileHdr.u16VerMinor  = VBOX_VERSION_MINOR;
+    FileHdr.u32VerBuild  = VBOX_VERSION_BUILD;
+    FileHdr.u32SvnRev    = VMMGetSvnRev();
+    FileHdr.cHostBits    = HC_ARCH_BITS;
+    FileHdr.cbGCPhys     = sizeof(RTGCPHYS);
+    FileHdr.cbGCPtr      = sizeof(RTGCPTR);
+    FileHdr.u8Reserved   = 0;
+    FileHdr.cUnits       = pVM->ssm.s.cUnits;
+    FileHdr.fFlags       = SSMFILEHDR_FLAGS_STREAM_CRC32;
+    if (pSSM->fLiveSave)
+        FileHdr.fFlags  |= SSMFILEHDR_FLAGS_STREAM_LIVE_SAVE;
+    FileHdr.cbMaxDecompr = RT_SIZEOFMEMB(SSMHANDLE, u.Read.abDataBuffer);
+    FileHdr.u32CRC       = 0;
+    FileHdr.u32CRC       = RTCrc32(&FileHdr, sizeof(FileHdr));
+    int rc = ssmR3StrmWrite(&pSSM->Strm, &FileHdr, sizeof(FileHdr));
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Clear the per unit flags and offsets.
+     */
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+    {
+        pUnit->fCalled   = false;
+        pUnit->offStream = RTFOFF_MIN;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates a new saved state file.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pszFilename         The name of the file.  NULL if pStreamOps is
+ *                              used.
+ * @param   pStreamOps          The stream methods.  NULL if pszFilename is
+ *                              used.
+ * @param   pvStreamOpsUser     The user argument to the stream methods.
+ * @param   enmAfter            What to do afterwards.
+ * @param   pfnProgress         The progress callback.
+ * @param   pvProgressUser      The progress callback user argument.
+ * @param   ppSSM               Where to return the pointer to the saved state
+ *                              handle upon successful return.  Free it using
+ *                              RTMemFree after closing the stream.
+ */
+static int ssmR3SaveDoCreateFile(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                                 SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser, PSSMHANDLE *ppSSM)
+{
+    PSSMHANDLE pSSM = (PSSMHANDLE)RTMemAllocZ(sizeof(*pSSM));
+    if (!pSSM)
+        return VERR_NO_MEMORY;
+
+    pSSM->pVM                       = pVM;
+    pSSM->enmOp                     = SSMSTATE_INVALID;
+    pSSM->enmAfter                  = enmAfter;
+    pSSM->fCancelled                = SSMHANDLE_OK;
+    pSSM->rc                        = VINF_SUCCESS;
+    pSSM->cbUnitLeftV1              = 0;
+    pSSM->offUnit                   = UINT64_MAX;
+    pSSM->offUnitUser               = UINT64_MAX;
+    pSSM->fLiveSave                 = false;
+    pSSM->pfnProgress               = pfnProgress;
+    pSSM->pvUser                    = pvProgressUser;
+    pSSM->uPercent                  = 0;
+    pSSM->offEstProgress            = 0;
+    pSSM->cbEstTotal                = 0;
+    pSSM->offEst                    = 0;
+    pSSM->offEstUnitEnd             = 0;
+    pSSM->uPercentLive              = 0;
+    pSSM->uPercentPrepare           = 0;
+    pSSM->uPercentDone              = 0;
+    pSSM->uReportedLivePercent      = 0;
+    pSSM->pszFilename               = pszFilename;
+    pSSM->u.Write.offDataBuffer     = 0;
+    pSSM->u.Write.cMsMaxDowntime    = UINT32_MAX;
+
+    int rc;
+    if (pStreamOps)
+        rc = ssmR3StrmInit(&pSSM->Strm, pStreamOps, pvStreamOpsUser, true /*fWrite*/, true /*fChecksummed*/, 8 /*cBuffers*/);
+    else
+        rc = ssmR3StrmOpenFile(&pSSM->Strm, pszFilename,             true /*fWrite*/, true /*fChecksummed*/, 8 /*cBuffers*/);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("SSM: Failed to create save state file '%s', rc=%Rrc.\n",  pszFilename, rc));
+        RTMemFree(pSSM);
+        return rc;
+    }
+
+    *ppSSM = pSSM;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Start VM save operation.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pszFilename     Name of the file to save the state in. NULL if pStreamOps is used.
+ * @param   pStreamOps      The stream method table. NULL if pszFilename is
+ *                          used.
+ * @param   pvStreamOpsUser The user argument to the stream methods.
+ * @param   enmAfter        What is planned after a successful save operation.
+ * @param   pfnProgress     Progress callback. Optional.
+ * @param   pvUser          User argument for the progress callback.
+ *
+ * @thread  EMT
+ */
+VMMR3DECL(int) SSMR3Save(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                         SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvUser)
+{
+    LogFlow(("SSMR3Save: pszFilename=%p:{%s} enmAfter=%d pfnProgress=%p pvUser=%p\n", pszFilename, pszFilename, enmAfter, pfnProgress, pvUser));
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(   enmAfter == SSMAFTER_DESTROY
+                    || enmAfter == SSMAFTER_CONTINUE,
+                    ("%d\n", enmAfter),
+                    VERR_INVALID_PARAMETER);
+
+    AssertReturn(!pszFilename != !pStreamOps, VERR_INVALID_PARAMETER);
+    if (pStreamOps)
+    {
+        AssertReturn(pStreamOps->u32Version == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
+        AssertReturn(pStreamOps->u32EndVersion == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
+        AssertReturn(pStreamOps->pfnWrite, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnRead, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnSeek, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnTell, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnSize, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnClose, VERR_INVALID_PARAMETER);
+    }
+
+    /*
+     * Create the saved state file and handle.
+     *
+     * Note that there might be quite some work to do after executing the saving,
+     * so we reserve 20% for the 'Done' period.
+     */
+    PSSMHANDLE pSSM;
+    int rc = ssmR3SaveDoCreateFile(pVM, pszFilename, pStreamOps, pvStreamOpsUser,
+                                   enmAfter, pfnProgress, pvUser, &pSSM);
+    if (RT_FAILURE(rc))
+        return rc;
+    pSSM->uPercentLive    = 0;
+    pSSM->uPercentPrepare = 20;
+    pSSM->uPercentDone    = 2;
+    pSSM->fLiveSave       = false;
+
+    /*
+     * Write the saved state stream header and join paths with
+     * the other save methods for the rest of the job.
+     */
+    Log(("SSM: Starting state save to file '%s'...\n", pszFilename));
+    ssmR3StrmStartIoThread(&pSSM->Strm);
+    rc = ssmR3WriteHeaderAndClearPerUnitData(pVM, pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        ssmR3SetCancellable(pVM, pSSM, true);
+        ssmR3SaveDoCommon(pVM, pSSM);
+    }
+
+    return ssmR3SaveDoClose(pVM, pSSM);
+}
+
+
+/**
+ * Used by PGM to report the completion percentage of the live stage during the
+ * vote run.
+ *
+ * @param   pSSM                The saved state handle.
+ * @param   uPercent            The completion percentage.
+ */
+VMMR3DECL(void) SSMR3HandleReportLivePercent(PSSMHANDLE pSSM, unsigned uPercent)
+{
+    AssertMsgReturnVoid(pSSM->enmOp == SSMSTATE_LIVE_VOTE, ("%d\n", pSSM->enmOp));
+    AssertReturnVoid(uPercent <= 100);
+    if (uPercent < pSSM->uReportedLivePercent)
+        pSSM->uReportedLivePercent = uPercent;
+}
+
+
+/**
+ * Calls pfnLiveVote for all units.
+ *
+ * @returns VBox status code (no need to check pSSM->rc).
+ * @retval  VINF_SUCCESS if we can pass on to step 2.
+ * @retval  VINF_SSM_VOTE_FOR_ANOTHER_PASS if we need another pass.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ * @param   uPass               The current pass.
+ */
+static int ssmR3LiveDoVoteRun(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    int rcRet = VINF_SUCCESS;
+    AssertRC(pSSM->rc);
+    pSSM->rc = VINF_SUCCESS;
+    pSSM->enmOp = SSMSTATE_LIVE_VOTE;
+
+    unsigned uPrevPrecent = pSSM->uReportedLivePercent;
+    pSSM->uReportedLivePercent = 101;
+
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+    {
+        if (    pUnit->u.Common.pfnLiveVote
+            &&  !pUnit->fDoneLive)
+        {
+            int rc;
+            ssmR3UnitCritSectEnter(pUnit);
+            switch (pUnit->enmType)
+            {
+                case SSMUNITTYPE_DEV:
+                    rc = pUnit->u.Dev.pfnLiveVote(pUnit->u.Dev.pDevIns, pSSM, uPass);
+                    break;
+                case SSMUNITTYPE_DRV:
+                    rc = pUnit->u.Drv.pfnLiveVote(pUnit->u.Drv.pDrvIns, pSSM, uPass);
+                    break;
+                case SSMUNITTYPE_USB:
+                    rc = pUnit->u.Usb.pfnLiveVote(pUnit->u.Usb.pUsbIns, pSSM, uPass);
+                    break;
+                case SSMUNITTYPE_INTERNAL:
+                    rc = pUnit->u.Internal.pfnLiveVote(pVM, pSSM, uPass);
+                    break;
+                case SSMUNITTYPE_EXTERNAL:
+                    rc = pUnit->u.External.pfnLiveVote(pSSM, pUnit->u.External.pvUser, uPass);
+                    break;
+                default:
+                    rc = VERR_SSM_IPE_1;
+                    break;
+            }
+            ssmR3UnitCritSectLeave(pUnit);
+            pUnit->fCalled = true;
+            Assert(pSSM->rc == VINF_SUCCESS);
+            if (rc != VINF_SUCCESS)
+            {
+                if (rc == VINF_SSM_VOTE_FOR_ANOTHER_PASS)
+                {
+                    Log(("ssmR3DoLiveVoteRun: '%s'/#%u -> VINF_SSM_VOTE_FOR_ANOTHER_PASS (pass=%u)\n", pUnit->szName, pUnit->u32Instance, uPass));
+                    rcRet = VINF_SSM_VOTE_FOR_ANOTHER_PASS;
+                }
+                else if (rc == VINF_SSM_VOTE_DONE_DONT_CALL_AGAIN)
+                {
+                    pUnit->fDoneLive = true;
+                    Log(("ssmR3DoLiveVoteRun: '%s'/#%u -> VINF_SSM_VOTE_DONE_DONT_CALL_AGAIN (pass=%u)\n", pUnit->szName, pUnit->u32Instance, uPass));
+                }
+                else
+                {
+                    /*
+                     * rc is usually VERR_SSM_VOTE_FOR_GIVING_UP here, but we allow
+                     * other status codes for better user feed back.  However, no
+                     * other non-error status is allowed.
+                     */
+                    LogRel(("SSM: Error - '%s'/#%u voted %Rrc! (pass=%u)\n", pUnit->szName, pUnit->u32Instance, rc, uPass));
+                    AssertMsgReturn(RT_FAILURE(rc), ("%Rrc; '%s'\n", rc, pUnit->szName), pSSM->rc = VERR_IPE_UNEXPECTED_INFO_STATUS);
+                    return pSSM->rc = rc;
+                }
+            }
+        }
+    }
+    if (rcRet == VINF_SUCCESS)
+    {
+        LogRel(("SSM: Step 1 completed after pass %u.\n", uPass));
+        pSSM->uReportedLivePercent = 100;
+    }
+    else
+    {
+        /*
+         * Work the progress callback.
+         */
+        if (pSSM->uReportedLivePercent > 100)
+            pSSM->uReportedLivePercent = 0;
+        if (   pSSM->uReportedLivePercent != uPrevPrecent
+            && pSSM->pfnProgress
+            && pSSM->uPercentLive)
+        {
+            long double lrdPct = (long double)pSSM->uReportedLivePercent * pSSM->uPercentLive / 100;
+            unsigned    uPct   = (unsigned)lrdPct;
+            if (uPct != pSSM->uPercent)
+            {
+                ssmR3LiveControlEmit(pSSM, lrdPct, uPass);
+                pSSM->uPercent = uPct;
+                pSSM->pfnProgress(pVM->pUVM, uPct, pSSM->pvUser);
+            }
+        }
+    }
+    return rcRet;
+}
+
+
+/**
+ * Calls pfnLiveExec for all units.
+ *
+ * @returns VBox status code (no need to check pSSM->rc).
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ * @param   uPass               The current pass.
+ */
+static int ssmR3LiveDoExecRun(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    AssertRC(pSSM->rc);
+    pSSM->rc = VINF_SUCCESS;
+    pSSM->enmOp = SSMSTATE_LIVE_EXEC;
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+    {
+        /*
+         * Skip units without a callback (this is most).
+         */
+        if (   !pUnit->u.Common.pfnLiveExec
+            || pUnit->fDoneLive)
+            continue;
+        pUnit->offStream = ssmR3StrmTell(&pSSM->Strm);
+
+        /*
+         * Check for cancellation.
+         */
+        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED))
+        {
+            LogRel(("SSM: Cancelled!\n"));
+            AssertRC(pSSM->rc);
+            return pSSM->rc = VERR_SSM_CANCELLED;
+        }
+
+        /*
+         * Write data unit header.
+         */
+        SSMFILEUNITHDRV2 UnitHdr;
+        memcpy(&UnitHdr.szMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic));
+        UnitHdr.offStream       = pUnit->offStream;
+        UnitHdr.u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
+        UnitHdr.u32CRC          = 0;
+        UnitHdr.u32Version      = pUnit->u32Version;
+        UnitHdr.u32Instance     = pUnit->u32Instance;
+        UnitHdr.u32Pass         = uPass;
+        UnitHdr.fFlags          = 0;
+        UnitHdr.cbName          = (uint32_t)pUnit->cchName + 1;
+        memcpy(&UnitHdr.szName[0], &pUnit->szName[0], UnitHdr.cbName);
+        UnitHdr.u32CRC          = RTCrc32(&UnitHdr, RT_UOFFSETOF_DYN(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
+        Log(("SSM: Unit at %#9llx: '%s', instance %u, pass %#x, version %u\n",
+             UnitHdr.offStream, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass, UnitHdr.u32Version));
+        int rc = ssmR3StrmWrite(&pSSM->Strm, &UnitHdr, RT_UOFFSETOF_DYN(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: Failed to write unit header. rc=%Rrc\n", rc));
+            return pSSM->rc = rc;
+        }
+
+        /*
+         * Call the execute handler.
+         */
+        ssmR3DataWriteBegin(pSSM);
+        ssmR3UnitCritSectEnter(pUnit);
+        switch (pUnit->enmType)
+        {
+            case SSMUNITTYPE_DEV:
+                rc = pUnit->u.Dev.pfnLiveExec(pUnit->u.Dev.pDevIns, pSSM, uPass);
+                break;
+            case SSMUNITTYPE_DRV:
+                rc = pUnit->u.Drv.pfnLiveExec(pUnit->u.Drv.pDrvIns, pSSM, uPass);
+                break;
+            case SSMUNITTYPE_USB:
+                rc = pUnit->u.Usb.pfnLiveExec(pUnit->u.Usb.pUsbIns, pSSM, uPass);
+                break;
+            case SSMUNITTYPE_INTERNAL:
+                rc = pUnit->u.Internal.pfnLiveExec(pVM, pSSM, uPass);
+                break;
+            case SSMUNITTYPE_EXTERNAL:
+                rc = pUnit->u.External.pfnLiveExec(pSSM, pUnit->u.External.pvUser, uPass);
+                break;
+            default:
+                rc = VERR_SSM_IPE_1;
+                break;
+        }
+        ssmR3UnitCritSectLeave(pUnit);
+        pUnit->fCalled = true;
+        if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
+            pSSM->rc = rc;
+        else
+        {
+            if (rc == VINF_SSM_DONT_CALL_AGAIN)
+                pUnit->fDoneLive = true;
+            rc = ssmR3DataFlushBuffer(pSSM); /* will return SSMHANDLE::rc if it is set */
+        }
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: Execute save failed with rc=%Rrc for data unit '%s'/#%u.\n", rc, pUnit->szName, pUnit->u32Instance));
+            if (RT_SUCCESS(pSSM->rc))
+                pSSM->rc = rc;
+            return rc;
+        }
+
+        /*
+         * Write the termination record and flush the compression stream.
+         */
+        SSMRECTERM TermRec;
+        TermRec.u8TypeAndFlags   = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_TERM;
+        TermRec.cbRec            = sizeof(TermRec) - 2;
+        if (pSSM->Strm.fChecksummed)
+        {
+            TermRec.fFlags       = SSMRECTERM_FLAGS_CRC32;
+            TermRec.u32StreamCRC = RTCrc32Finish(RTCrc32Process(ssmR3StrmCurCRC(&pSSM->Strm), &TermRec, 2));
+        }
+        else
+        {
+            TermRec.fFlags       = 0;
+            TermRec.u32StreamCRC = 0;
+        }
+        TermRec.cbUnit           = pSSM->offUnit + sizeof(TermRec);
+        rc = ssmR3DataWriteRaw(pSSM, &TermRec, sizeof(TermRec));
+        if (RT_SUCCESS(rc))
+            rc = ssmR3DataWriteFinish(pSSM);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: Failed terminating unit: %Rrc (pass=%u)\n", rc, uPass));
+            return pSSM->rc = rc;
+        }
+    } /* for each unit */
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements the live exec+vote loop.
+ *
+ * @returns VBox status code (no need to check pSSM->rc).
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3DoLiveExecVoteLoop(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Calc the max saved state size before we should give up because of insane
+     * amounts of data.
+     */
+#define SSM_MAX_GROWTH_FILE      10000
+#define SSM_MAX_GROWTH_REMOTE   100000
+    uint64_t cbSum = 0;
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+        cbSum += pUnit->cbGuess;
+    uint64_t cbMax = cbSum * (pSSM->pszFilename ? SSM_MAX_GROWTH_FILE : SSM_MAX_GROWTH_REMOTE);
+    AssertLogRelMsgReturn(cbMax > cbSum, ("cbMax=%#RX64, cbSum=%#RX64\n", cbMax, cbSum), pSSM->rc = VERR_OUT_OF_RANGE);
+    if (cbMax < _1G)
+        cbMax = _1G;
+
+    /*
+     * The pass loop.
+     *
+     * The number of iterations is restricted for two reasons, first
+     * to make sure
+     */
+#define SSM_MAX_PASSES  _1M
+    for (uint32_t uPass = 0; uPass < SSM_MAX_PASSES; uPass++)
+    {
+        pVM->ssm.s.uPass = uPass;
+
+        /*
+         * Save state and vote on whether we need more passes or not.
+         */
+        int rc = ssmR3LiveDoExecRun(pVM, pSSM, uPass);
+        if (RT_FAILURE(rc))
+            return rc;
+        rc = ssmR3LiveDoVoteRun(pVM, pSSM, uPass);
+        if (rc == VINF_SUCCESS)
+        {
+            pSSM->enmOp = SSMSTATE_LIVE_STEP2;
+            return VINF_SUCCESS;
+        }
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /*
+         * Check that we're still within sane data amounts.
+         */
+        uint64_t cbSaved = ssmR3StrmTell(&pSSM->Strm);
+        if (cbSaved > cbMax)
+        {
+            LogRel(("SSM: Giving up: Exceeded max state size. (cbSaved=%#RX64, cbMax=%#RX64)\n", cbSaved, cbMax));
+            return pSSM->rc = VERR_SSM_STATE_GREW_TOO_BIG;
+        }
+
+        /*
+         * Check that the stream is still OK.
+         */
+        rc = ssmR3StrmCheckAndFlush(&pSSM->Strm);
+        if (RT_FAILURE(rc))
+            return pSSM->rc = rc;
+    }
+
+    LogRel(("SSM: Giving up: Too many passes! (%u)\n", SSM_MAX_PASSES));
+    return pSSM->rc = VERR_SSM_TOO_MANY_PASSES;
+}
+
+
+/**
+ * Calls pfnLivePrep for all units.
+ *
+ * @returns VBox status code (no need to check pSSM->rc).
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3DoLivePrepRun(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Do the prepare run.
+     */
+    pSSM->rc = VINF_SUCCESS;
+    pSSM->enmOp = SSMSTATE_SAVE_PREP;
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+    {
+        if (pUnit->u.Common.pfnLivePrep)
+        {
+            int rc;
+            ssmR3UnitCritSectEnter(pUnit);
+            switch (pUnit->enmType)
+            {
+                case SSMUNITTYPE_DEV:
+                    rc = pUnit->u.Dev.pfnLivePrep(pUnit->u.Dev.pDevIns, pSSM);
+                    break;
+                case SSMUNITTYPE_DRV:
+                    rc = pUnit->u.Drv.pfnLivePrep(pUnit->u.Drv.pDrvIns, pSSM);
+                    break;
+                case SSMUNITTYPE_USB:
+                    rc = pUnit->u.Usb.pfnLivePrep(pUnit->u.Usb.pUsbIns, pSSM);
+                    break;
+                case SSMUNITTYPE_INTERNAL:
+                    rc = pUnit->u.Internal.pfnLivePrep(pVM, pSSM);
+                    break;
+                case SSMUNITTYPE_EXTERNAL:
+                    rc = pUnit->u.External.pfnLivePrep(pSSM, pUnit->u.External.pvUser);
+                    break;
+                default:
+                    rc = VERR_SSM_IPE_1;
+                    break;
+            }
+            ssmR3UnitCritSectLeave(pUnit);
+            pUnit->fCalled = true;
+            if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
+                pSSM->rc = rc;
+            else
+                rc = pSSM->rc;
+            if (RT_FAILURE(rc))
+            {
+                LogRel(("SSM: Prepare save failed with rc=%Rrc for data unit '%s.\n", rc, pUnit->szName));
+                return rc;
+            }
+        }
+
+        pSSM->cbEstTotal += pUnit->cbGuess;
+    }
+
+    /*
+     * Work the progress indicator if we got one.
+     */
+    if (pSSM->pfnProgress)
+        pSSM->pfnProgress(pVM->pUVM, 2, pSSM->pvUser);
+    pSSM->uPercent = 2;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Continue a live state saving operation on the worker thread.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSSM            The SSM handle returned by SSMR3LiveSave.
+ *
+ * @thread  Non-EMT thread. Will involve the EMT at the end of the operation.
+ */
+VMMR3_INT_DECL(int) SSMR3LiveDoStep1(PSSMHANDLE pSSM)
+{
+    LogFlow(("SSMR3LiveDoStep1: pSSM=%p\n", pSSM));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pSSM, VERR_INVALID_POINTER);
+    PVM pVM = pSSM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_OTHER_THREAD(pVM);
+    AssertMsgReturn(   pSSM->enmAfter == SSMAFTER_DESTROY
+                    || pSSM->enmAfter == SSMAFTER_CONTINUE
+                    || pSSM->enmAfter == SSMAFTER_TELEPORT,
+                    ("%d\n", pSSM->enmAfter),
+                    VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pSSM->enmOp == SSMSTATE_LIVE_STEP1, ("%d\n", pSSM->enmOp), VERR_INVALID_STATE);
+    AssertRCReturn(pSSM->rc, pSSM->rc);
+
+    /*
+     * Do the prep run, then the exec+vote cycle.
+     */
+    int rc = ssmR3DoLivePrepRun(pVM, pSSM);
+    if (RT_SUCCESS(rc))
+        rc = ssmR3DoLiveExecVoteLoop(pVM, pSSM);
+    return rc;
+}
+
+
+/**
+ * Start saving the live state.
+ *
+ * Call SSMR3LiveDoStep1, SSMR3LiveDoStep2 and finally SSMR3LiveDone on success.
+ * SSMR3LiveDone should be called even if SSMR3LiveDoStep1 or SSMR3LiveDoStep2
+ * fails.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   cMsMaxDowntime  The maximum downtime given as milliseconds.
+ * @param   pszFilename     Name of the file to save the state in. This string
+ *                          must remain valid until SSMR3LiveDone is called.
+ *                          Must be NULL if pStreamOps is used.
+ * @param   pStreamOps      The stream method table. NULL if pszFilename is
+ *                          used.
+ * @param   pvStreamOpsUser The user argument to the stream methods.
+ * @param   enmAfter        What is planned after a successful save operation.
+ * @param   pfnProgress     Progress callback. Optional.
+ * @param   pvProgressUser  User argument for the progress callback.
+ * @param   ppSSM           Where to return the saved state handle on success.
+ *
+ * @thread  EMT0
+ */
+VMMR3_INT_DECL(int) SSMR3LiveSave(PVM pVM, uint32_t cMsMaxDowntime,
+                                  const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                                  SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser,
+                                  PSSMHANDLE *ppSSM)
+{
+    LogFlow(("SSMR3LiveSave: cMsMaxDowntime=%u pszFilename=%p:{%s} pStreamOps=%p pvStreamOpsUser=%p enmAfter=%d pfnProgress=%p pvProgressUser=%p\n",
+             cMsMaxDowntime, pszFilename, pszFilename, pStreamOps, pvStreamOpsUser, enmAfter, pfnProgress, pvProgressUser));
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(   enmAfter == SSMAFTER_DESTROY
+                    || enmAfter == SSMAFTER_CONTINUE
+                    || enmAfter == SSMAFTER_TELEPORT,
+                    ("%d\n", enmAfter),
+                    VERR_INVALID_PARAMETER);
+    AssertReturn(!pszFilename != !pStreamOps, VERR_INVALID_PARAMETER);
+    if (pStreamOps)
+    {
+        AssertReturn(pStreamOps->u32Version == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
+        AssertReturn(pStreamOps->u32EndVersion == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
+        AssertReturn(pStreamOps->pfnWrite, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnRead, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnSeek, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnTell, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnSize, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnClose, VERR_INVALID_PARAMETER);
+    }
+
+    /*
+     * Create the saved state file and handle.
+     *
+     * Note that there might be quite some work to do after executing the saving,
+     * so we reserve 20% for the 'Done' period.
+     */
+    PSSMHANDLE pSSM;
+    int rc = ssmR3SaveDoCreateFile(pVM, pszFilename, pStreamOps, pvStreamOpsUser,
+                                   enmAfter, pfnProgress, pvProgressUser, &pSSM);
+    if (RT_FAILURE(rc))
+        return rc;
+    pSSM->uPercentLive           = 93;
+    pSSM->uPercentPrepare        = 2;
+    pSSM->uPercentDone           = 2;
+    pSSM->fLiveSave              = true;
+    pSSM->u.Write.cMsMaxDowntime = cMsMaxDowntime;
+
+    /*
+     * Write the saved state stream header and do the prep run for live saving.
+     */
+    Log(("SSM: Starting state save to file '%s'...\n", pszFilename));
+    ssmR3StrmStartIoThread(&pSSM->Strm);
+    rc = ssmR3WriteHeaderAndClearPerUnitData(pVM, pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Return and let the requestor thread do the pfnLiveExec/Vote part
+         * via SSMR3SaveFinishLive
+         */
+        pSSM->enmOp = SSMSTATE_LIVE_STEP1;
+        ssmR3SetCancellable(pVM, pSSM, true);
+        *ppSSM = pSSM;
+        return VINF_SUCCESS;
+    }
+    /* bail out. */
+    int rc2 = ssmR3StrmClose(&pSSM->Strm, pSSM->rc == VERR_SSM_CANCELLED);
+    RTMemFree(pSSM);
+    rc2 = RTFileDelete(pszFilename);
+    AssertRC(rc2);
+    return rc;
+}
+
+#endif /* !SSM_STANDALONE */
+
+
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+
+
+#ifndef SSM_STANDALONE
+/**
+ * Closes the decompressor of a data unit.
+ *
+ * @returns pSSM->rc.
+ * @param   pSSM            The saved state handle.
+ */
+static int ssmR3DataReadFinishV1(PSSMHANDLE pSSM)
+{
+    if (pSSM->u.Read.pZipDecompV1)
+    {
+        int rc = RTZipDecompDestroy(pSSM->u.Read.pZipDecompV1);
+        AssertRC(rc);
+        pSSM->u.Read.pZipDecompV1 = NULL;
+    }
+    return pSSM->rc;
+}
+#endif /* !SSM_STANDALONE */
+
+
+/**
+ * Callback for reading compressed data into the input buffer of the
+ * decompressor, for saved file format version 1.
+ *
+ * @returns VBox status code. Set pSSM->rc on error.
+ * @param   pvSSM       The SSM handle.
+ * @param   pvBuf       Where to store the compressed data.
+ * @param   cbBuf       Size of the buffer.
+ * @param   pcbRead     Number of bytes actually stored in the buffer.
+ */
+static DECLCALLBACK(int) ssmR3ReadInV1(void *pvSSM, void *pvBuf, size_t cbBuf, size_t *pcbRead)
+{
+    PSSMHANDLE  pSSM   = (PSSMHANDLE)pvSSM;
+    size_t      cbRead = cbBuf;
+    if (pSSM->cbUnitLeftV1 < cbBuf)
+        cbRead = (size_t)pSSM->cbUnitLeftV1;
+    if (cbRead)
+    {
+        //Log2(("ssmR3ReadInV1: %#010llx cbBug=%#x cbRead=%#x\n", ssmR3StrmTell(&pSSM->Strm), cbBuf, cbRead));
+        int rc = ssmR3StrmRead(&pSSM->Strm, pvBuf, cbRead);
+        if (RT_SUCCESS(rc))
+        {
+            pSSM->cbUnitLeftV1 -= cbRead;
+            if (pcbRead)
+                *pcbRead = cbRead;
+            ssmR3ProgressByByte(pSSM, cbRead);
+            return VINF_SUCCESS;
+        }
+        return pSSM->rc = rc;
+    }
+
+    if (pSSM->enmAfter != SSMAFTER_DEBUG_IT)
+        AssertMsgFailed(("SSM: attempted reading more than the unit!\n"));
+    return pSSM->rc = VERR_SSM_LOADED_TOO_MUCH;
+}
+
+
+/**
+ * Internal read worker for reading data from a version 1 unit.
+ *
+ * @returns VBox status code, pSSM->rc is set on error.
+ *
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           Where to store the read data.
+ * @param   cbBuf           Number of bytes to read.
+ */
+static int ssmR3DataReadV1(PSSMHANDLE pSSM, void *pvBuf, size_t cbBuf)
+{
+    /*
+     * Open the decompressor on the first read.
+     */
+    if (!pSSM->u.Read.pZipDecompV1)
+    {
+        pSSM->rc = RTZipDecompCreate(&pSSM->u.Read.pZipDecompV1, pSSM, ssmR3ReadInV1);
+        if (RT_FAILURE(pSSM->rc))
+            return pSSM->rc;
+    }
+
+    /*
+     * Do the requested read.
+     */
+    int rc = pSSM->rc = RTZipDecompress(pSSM->u.Read.pZipDecompV1, pvBuf, cbBuf, NULL);
+    if (RT_SUCCESS(rc))
+    {
+        Log2(("ssmR3DataRead: pvBuf=%p cbBuf=%#x offUnit=%#llx %.*Rhxs%s\n", pvBuf, cbBuf, pSSM->offUnit, RT_MIN(cbBuf, SSM_LOG_BYTES), pvBuf, cbBuf > SSM_LOG_BYTES ? "..." : ""));
+        pSSM->offUnit     += cbBuf;
+        pSSM->offUnitUser += cbBuf;
+        return VINF_SUCCESS;
+    }
+    AssertMsgFailed(("rc=%Rrc cbBuf=%#x\n", rc, cbBuf));
+    return rc;
+}
+
+
+/**
+ * Creates the decompressor for the data unit.
+ *
+ * pSSM->rc will be set on error.
+ *
+ * @param   pSSM            The saved state handle.
+ */
+static void ssmR3DataReadBeginV2(PSSMHANDLE pSSM)
+{
+    Assert(!pSSM->u.Read.cbDataBuffer || pSSM->u.Read.cbDataBuffer == pSSM->u.Read.offDataBuffer);
+    Assert(!pSSM->u.Read.cbRecLeft);
+
+    pSSM->offUnit     = 0;
+    pSSM->offUnitUser = 0;
+    pSSM->u.Read.cbRecLeft      = 0;
+    pSSM->u.Read.cbDataBuffer   = 0;
+    pSSM->u.Read.offDataBuffer  = 0;
+    pSSM->u.Read.fEndOfData     = false;
+    pSSM->u.Read.u8TypeAndFlags = 0;
+}
+
+
+#ifndef SSM_STANDALONE
+/**
+ * Checks for the termination record and closes the decompressor.
+ *
+ * pSSM->rc will be set on error.
+ *
+ * @returns pSSM->rc.
+ * @param   pSSM            The saved state handle.
+ */
+static int ssmR3DataReadFinishV2(PSSMHANDLE pSSM)
+{
+    /*
+     * If we haven't encountered the end of the record, it must be the next one.
+     */
+    int rc = pSSM->rc;
+    if (    !pSSM->u.Read.fEndOfData
+        &&  RT_SUCCESS(rc))
+    {
+        if (   pSSM->u.Read.cbDataBuffer != pSSM->u.Read.offDataBuffer
+            && pSSM->u.Read.cbDataBuffer > 0)
+        {
+            LogRel(("SSM: At least %#x bytes left to read\n", pSSM->u.Read.cbDataBuffer - pSSM->u.Read.offDataBuffer));
+            rc = VERR_SSM_LOADED_TOO_LITTLE;
+        }
+        else
+        {
+            rc = ssmR3DataReadRecHdrV2(pSSM);
+            if (    RT_SUCCESS(rc)
+                &&  !pSSM->u.Read.fEndOfData)
+            {
+                LogRel(("SSM: At least %#x bytes left to read\n", pSSM->u.Read.cbDataBuffer));
+                rc = VERR_SSM_LOADED_TOO_LITTLE;
+                AssertFailed();
+            }
+        }
+        pSSM->rc = rc;
+    }
+    return rc;
+}
+#endif /* !SSM_STANDALONE */
+
+
+/**
+ * Read raw record bytes, work the progress indicator and unit offset.
+ *
+ * @returns VBox status code. Does NOT set pSSM->rc.
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           Where to put the bits
+ * @param   cbToRead        How many bytes to read.
+ */
+DECLINLINE(int) ssmR3DataReadV2Raw(PSSMHANDLE pSSM, void *pvBuf, size_t cbToRead)
+{
+    int rc = ssmR3StrmRead(&pSSM->Strm, pvBuf, cbToRead);
+    if (RT_SUCCESS(rc))
+    {
+        pSSM->offUnit += cbToRead;
+        ssmR3ProgressByByte(pSSM, cbToRead);
+        return VINF_SUCCESS;
+    }
+
+    if (rc == VERR_SSM_CANCELLED)
+        return rc;
+
+    if (pSSM->enmAfter != SSMAFTER_DEBUG_IT && rc == VERR_EOF)
+        AssertMsgFailedReturn(("SSM: attempted reading more than the unit! rc=%Rrc\n", rc), VERR_SSM_LOADED_TOO_MUCH);
+    return VERR_SSM_STREAM_ERROR;
+}
+
+
+/**
+ * Reads and checks the LZF "header".
+ *
+ * @returns VBox status code. Sets pSSM->rc on error.
+ * @param   pSSM            The saved state handle..
+ * @param   pcbDecompr      Where to store the size of the decompressed data.
+ */
+DECLINLINE(int) ssmR3DataReadV2RawLzfHdr(PSSMHANDLE pSSM, uint32_t *pcbDecompr)
+{
+    *pcbDecompr = 0; /* shuts up gcc. */
+    AssertLogRelMsgReturn(   pSSM->u.Read.cbRecLeft > 1
+                          && pSSM->u.Read.cbRecLeft <= RT_SIZEOFMEMB(SSMHANDLE, u.Read.abComprBuffer) + 2,
+                          ("%#x\n", pSSM->u.Read.cbRecLeft),
+                          pSSM->rc = VERR_SSM_INTEGRITY_DECOMPRESSION);
+
+    uint8_t cKB;
+    int rc = ssmR3DataReadV2Raw(pSSM, &cKB, 1);
+    if (RT_FAILURE(rc))
+        return pSSM->rc = rc;
+    pSSM->u.Read.cbRecLeft -= sizeof(cKB);
+
+    uint32_t cbDecompr = (uint32_t)cKB * _1K;
+    AssertLogRelMsgReturn(   cbDecompr >= pSSM->u.Read.cbRecLeft
+                          && cbDecompr <= RT_SIZEOFMEMB(SSMHANDLE, u.Read.abDataBuffer),
+                          ("%#x\n", cbDecompr),
+                          pSSM->rc = VERR_SSM_INTEGRITY_DECOMPRESSION);
+
+    *pcbDecompr = cbDecompr;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Reads an LZF block from the stream and decompresses into the specified
+ * buffer.
+ *
+ * @returns VBox status code. Sets pSSM->rc on error.
+ * @param   pSSM            The saved state handle.
+ * @param   pvDst           Pointer to the output buffer.
+ * @param   cbDecompr       The size of the decompressed data.
+ */
+static int ssmR3DataReadV2RawLzf(PSSMHANDLE pSSM, void *pvDst, size_t cbDecompr)
+{
+    int         rc;
+    uint32_t    cbCompr    = pSSM->u.Read.cbRecLeft;
+    pSSM->u.Read.cbRecLeft = 0;
+
+    /*
+     * Try use the stream buffer directly to avoid copying things around.
+     */
+    uint8_t const *pb = ssmR3StrmReadDirect(&pSSM->Strm, cbCompr);
+    if (pb)
+    {
+        pSSM->offUnit += cbCompr;
+        ssmR3ProgressByByte(pSSM, cbCompr);
+    }
+    else
+    {
+        rc = ssmR3DataReadV2Raw(pSSM, &pSSM->u.Read.abComprBuffer[0], cbCompr);
+        if (RT_FAILURE(rc))
+            return pSSM->rc = rc;
+        pb = &pSSM->u.Read.abComprBuffer[0];
+    }
+
+    /*
+     * Decompress it.
+     */
+    size_t cbDstActual;
+    rc = RTZipBlockDecompress(RTZIPTYPE_LZF, 0 /*fFlags*/,
+                              pb, cbCompr, NULL /*pcbSrcActual*/,
+                              pvDst, cbDecompr, &cbDstActual);
+    if (RT_SUCCESS(rc))
+    {
+        AssertLogRelMsgReturn(cbDstActual == cbDecompr, ("%#x %#x\n", cbDstActual, cbDecompr), pSSM->rc = VERR_SSM_INTEGRITY_DECOMPRESSION);
+        return VINF_SUCCESS;
+    }
+
+    AssertLogRelMsgFailed(("cbCompr=%#x cbDecompr=%#x rc=%Rrc\n", cbCompr, cbDecompr, rc));
+    return pSSM->rc = VERR_SSM_INTEGRITY_DECOMPRESSION;
+}
+
+
+/**
+ * Reads and checks the raw zero "header".
+ *
+ * @returns VBox status code. Sets pSSM->rc on error.
+ * @param   pSSM            The saved state handle..
+ * @param   pcbZero         Where to store the size of the zero data.
+ */
+DECLINLINE(int) ssmR3DataReadV2RawZeroHdr(PSSMHANDLE pSSM, uint32_t *pcbZero)
+{
+    *pcbZero = 0; /* shuts up gcc. */
+    AssertLogRelMsgReturn(pSSM->u.Read.cbRecLeft == 1, ("%#x\n", pSSM->u.Read.cbRecLeft), pSSM->rc = VERR_SSM_INTEGRITY_DECOMPRESSION);
+
+    uint8_t cKB;
+    int rc = ssmR3DataReadV2Raw(pSSM, &cKB, 1);
+    if (RT_FAILURE(rc))
+        return pSSM->rc = rc;
+    pSSM->u.Read.cbRecLeft = 0;
+
+    uint32_t cbZero = (uint32_t)cKB * _1K;
+    AssertLogRelMsgReturn(cbZero <= RT_SIZEOFMEMB(SSMHANDLE, u.Read.abDataBuffer),
+                          ("%#x\n", cbZero), pSSM->rc = VERR_SSM_INTEGRITY_DECOMPRESSION);
+
+    *pcbZero = cbZero;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for reading the record header.
+ *
+ * It sets pSSM->u.Read.cbRecLeft, pSSM->u.Read.u8TypeAndFlags and
+ * pSSM->u.Read.fEndOfData.  When a termination record is encounter, it will be
+ * read in full and validated, the fEndOfData indicator is set, and VINF_SUCCESS
+ * is returned.
+ *
+ * @returns VBox status code. Does not set pSSM->rc.
+ * @param   pSSM            The saved state handle.
+ */
+static int ssmR3DataReadRecHdrV2(PSSMHANDLE pSSM)
+{
+    AssertLogRelReturn(!pSSM->u.Read.fEndOfData, VERR_SSM_LOADED_TOO_MUCH);
+
+    /*
+     * Read the two mandatory bytes.
+     */
+    uint8_t  abHdr[8];
+    int rc = ssmR3DataReadV2Raw(pSSM, abHdr, 2);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Validate the first byte and check for the termination records.
+     */
+    pSSM->u.Read.u8TypeAndFlags = abHdr[0];
+    AssertLogRelMsgReturn(SSM_REC_ARE_TYPE_AND_FLAGS_VALID(abHdr[0]), ("%#x %#x\n", abHdr[0], abHdr[1]), VERR_SSM_INTEGRITY_REC_HDR);
+    if ((abHdr[0] & SSM_REC_TYPE_MASK) == SSM_REC_TYPE_TERM)
+    {
+        pSSM->u.Read.cbRecLeft = 0;
+        pSSM->u.Read.fEndOfData = true;
+        AssertLogRelMsgReturn(abHdr[1] == sizeof(SSMRECTERM) - 2, ("%#x\n", abHdr[1]), VERR_SSM_INTEGRITY_REC_TERM);
+        AssertLogRelMsgReturn(abHdr[0] & SSM_REC_FLAGS_IMPORTANT, ("%#x\n", abHdr[0]), VERR_SSM_INTEGRITY_REC_TERM);
+
+        /* get the rest */
+        uint32_t    u32StreamCRC = ssmR3StrmFinalCRC(&pSSM->Strm);
+        SSMRECTERM  TermRec;
+        rc = ssmR3DataReadV2Raw(pSSM, (uint8_t *)&TermRec + 2, sizeof(SSMRECTERM) - 2);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* validate integrity */
+        AssertLogRelMsgReturn(TermRec.cbUnit == pSSM->offUnit,
+                              ("cbUnit=%#llx offUnit=%#llx\n", TermRec.cbUnit, pSSM->offUnit),
+                              VERR_SSM_INTEGRITY_REC_TERM);
+        AssertLogRelMsgReturn(!(TermRec.fFlags & ~SSMRECTERM_FLAGS_CRC32), ("%#x\n", TermRec.fFlags), VERR_SSM_INTEGRITY_REC_TERM);
+        if (!(TermRec.fFlags & SSMRECTERM_FLAGS_CRC32))
+            AssertLogRelMsgReturn(TermRec.u32StreamCRC == 0, ("%#x\n", TermRec.u32StreamCRC), VERR_SSM_INTEGRITY_REC_TERM);
+        else if (pSSM->Strm.fChecksummed)
+            AssertLogRelMsgReturn(TermRec.u32StreamCRC == u32StreamCRC, ("%#x, %#x\n", TermRec.u32StreamCRC, u32StreamCRC),
+                                  VERR_SSM_INTEGRITY_REC_TERM_CRC);
+
+        Log3(("ssmR3DataReadRecHdrV2: %08llx|%08llx: TERM\n", ssmR3StrmTell(&pSSM->Strm) - sizeof(SSMRECTERM), pSSM->offUnit));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Figure the size. The 2nd byte is encoded in UTF-8 fashion, so this
+     * is can be highly enjoyable.
+     */
+    uint32_t cbHdr = 2;
+    uint32_t cb = abHdr[1];
+    if (!(cb & 0x80))
+        pSSM->u.Read.cbRecLeft = cb;
+    else
+    {
+        /*
+         * Need more data. Figure how much and read it.
+         */
+        if (!(cb & RT_BIT(5)))
+            cb = 2;
+        else if (!(cb & RT_BIT(4)))
+            cb = 3;
+        else if (!(cb & RT_BIT(3)))
+            cb = 4;
+        else if (!(cb & RT_BIT(2)))
+            cb = 5;
+        else if (!(cb & RT_BIT(1)))
+            cb = 6;
+        else
+            AssertLogRelMsgFailedReturn(("Invalid record size byte: %#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
+        cbHdr = cb + 1;
+
+        rc = ssmR3DataReadV2Raw(pSSM, &abHdr[2], cb - 1);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /*
+         * Validate what we've read.
+         */
+        switch (cb)
+        {
+            case 6:
+                AssertLogRelMsgReturn((abHdr[6] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
+                RT_FALL_THRU();
+            case 5:
+                AssertLogRelMsgReturn((abHdr[5] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
+                RT_FALL_THRU();
+            case 4:
+                AssertLogRelMsgReturn((abHdr[4] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
+                RT_FALL_THRU();
+            case 3:
+                AssertLogRelMsgReturn((abHdr[3] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
+                RT_FALL_THRU();
+            case 2:
+                AssertLogRelMsgReturn((abHdr[2] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
+                break;
+            default:
+                return VERR_IPE_NOT_REACHED_DEFAULT_CASE;
+        }
+
+        /*
+         * Decode it and validate the range.
+         */
+        switch (cb)
+        {
+            case 6:
+                cb =             (abHdr[6] & 0x3f)
+                    | ((uint32_t)(abHdr[5] & 0x3f) << 6)
+                    | ((uint32_t)(abHdr[4] & 0x3f) << 12)
+                    | ((uint32_t)(abHdr[3] & 0x3f) << 18)
+                    | ((uint32_t)(abHdr[2] & 0x3f) << 24)
+                    | ((uint32_t)(abHdr[1] & 0x01) << 30);
+                AssertLogRelMsgReturn(cb >= 0x04000000 && cb <= 0x7fffffff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
+                break;
+            case 5:
+                cb =             (abHdr[5] & 0x3f)
+                    | ((uint32_t)(abHdr[4] & 0x3f) << 6)
+                    | ((uint32_t)(abHdr[3] & 0x3f) << 12)
+                    | ((uint32_t)(abHdr[2] & 0x3f) << 18)
+                    | ((uint32_t)(abHdr[1] & 0x03) << 24);
+                AssertLogRelMsgReturn(cb >= 0x00200000 && cb <= 0x03ffffff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
+                break;
+            case 4:
+                cb =             (abHdr[4] & 0x3f)
+                    | ((uint32_t)(abHdr[3] & 0x3f) << 6)
+                    | ((uint32_t)(abHdr[2] & 0x3f) << 12)
+                    | ((uint32_t)(abHdr[1] & 0x07) << 18);
+                AssertLogRelMsgReturn(cb >= 0x00010000 && cb <= 0x001fffff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
+                break;
+            case 3:
+                cb =             (abHdr[3] & 0x3f)
+                    | ((uint32_t)(abHdr[2] & 0x3f) << 6)
+                    | ((uint32_t)(abHdr[1] & 0x0f) << 12);
+#if 0  /* disabled to optimize buffering */
+                AssertLogRelMsgReturn(cb >= 0x00000800 && cb <= 0x0000ffff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
+#endif
+                break;
+            case 2:
+                cb =             (abHdr[2] & 0x3f)
+                    | ((uint32_t)(abHdr[1] & 0x1f) << 6);
+#if 0  /* disabled to optimize buffering */
+                AssertLogRelMsgReturn(cb >= 0x00000080 && cb <= 0x000007ff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
+#endif
+                break;
+            default:
+                return VERR_IPE_NOT_REACHED_DEFAULT_CASE;
+        }
+
+        pSSM->u.Read.cbRecLeft = cb;
+    }
+
+    Log3(("ssmR3DataReadRecHdrV2: %08llx|%08llx/%08x: Type=%02x fImportant=%RTbool cbHdr=%u\n",
+          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft,
+          pSSM->u.Read.u8TypeAndFlags & SSM_REC_TYPE_MASK,
+          !!(pSSM->u.Read.u8TypeAndFlags & SSM_REC_FLAGS_IMPORTANT),
+          cbHdr
+        )); NOREF(cbHdr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Buffer miss, do an unbuffered read.
+ *
+ * @returns VBox status code. Sets pSSM->rc on error.
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           Where to store the read data.
+ * @param   cbBuf           Number of bytes to read.
+ */
+static int ssmR3DataReadUnbufferedV2(PSSMHANDLE pSSM, void *pvBuf, size_t cbBuf)
+{
+    void   const *pvBufOrg = pvBuf; NOREF(pvBufOrg);
+    size_t const  cbBufOrg = cbBuf; NOREF(cbBufOrg);
+
+    /*
+     * Copy out what we've got in the buffer.
+     */
+    uint32_t off        = pSSM->u.Read.offDataBuffer;
+    int32_t  cbInBuffer = pSSM->u.Read.cbDataBuffer - off;
+    Log4(("ssmR3DataReadUnbufferedV2: %08llx|%08llx/%08x/%08x: cbBuf=%#x\n", ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, cbInBuffer, cbBufOrg));
+    if (cbInBuffer > 0)
+    {
+        uint32_t const cbToCopy = (uint32_t)cbInBuffer;
+        Assert(cbBuf > cbToCopy);
+        memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[off], cbToCopy);
+        pvBuf  = (uint8_t *)pvBuf + cbToCopy;
+        cbBuf -= cbToCopy;
+        pSSM->u.Read.cbDataBuffer  = 0;
+        pSSM->u.Read.offDataBuffer = 0;
+    }
+
+    /*
+     * Read data.
+     */
+    do
+    {
+        /*
+         * Read the next record header if no more data.
+         */
+        if (!pSSM->u.Read.cbRecLeft)
+        {
+            int rc = ssmR3DataReadRecHdrV2(pSSM);
+            if (RT_FAILURE(rc))
+                return pSSM->rc = rc;
+        }
+        AssertLogRelMsgReturn(!pSSM->u.Read.fEndOfData, ("cbBuf=%zu\n", cbBuf), pSSM->rc = VERR_SSM_LOADED_TOO_MUCH);
+
+        /*
+         * Read data from the current record.
+         */
+        uint32_t cbToRead;
+        switch (pSSM->u.Read.u8TypeAndFlags & SSM_REC_TYPE_MASK)
+        {
+            case SSM_REC_TYPE_RAW:
+            {
+                cbToRead = (uint32_t)RT_MIN(cbBuf, pSSM->u.Read.cbRecLeft);
+                int rc = ssmR3DataReadV2Raw(pSSM, pvBuf, cbToRead);
+                if (RT_FAILURE(rc))
+                    return pSSM->rc = rc;
+                pSSM->u.Read.cbRecLeft -= cbToRead;
+                break;
+            }
+
+            case SSM_REC_TYPE_RAW_LZF:
+            {
+                int rc = ssmR3DataReadV2RawLzfHdr(pSSM, &cbToRead);
+                if (RT_FAILURE(rc))
+                    return rc;
+                if (cbToRead <= cbBuf)
+                {
+                    rc = ssmR3DataReadV2RawLzf(pSSM, pvBuf, cbToRead);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+                else
+                {
+                    /* The output buffer is too small, use the data buffer. */
+                    rc = ssmR3DataReadV2RawLzf(pSSM, &pSSM->u.Read.abDataBuffer[0], cbToRead);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                    pSSM->u.Read.cbDataBuffer  = cbToRead;
+                    cbToRead = (uint32_t)cbBuf;
+                    pSSM->u.Read.offDataBuffer = cbToRead;
+                    memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[0], cbToRead);
+                }
+                break;
+            }
+
+            case SSM_REC_TYPE_RAW_ZERO:
+            {
+                int rc = ssmR3DataReadV2RawZeroHdr(pSSM, &cbToRead);
+                if (RT_FAILURE(rc))
+                    return rc;
+                if (cbToRead > cbBuf)
+                {
+                    /* Spill the remainder into the data buffer. */
+                    memset(&pSSM->u.Read.abDataBuffer[0], 0, cbToRead - cbBuf);
+                    pSSM->u.Read.cbDataBuffer  = cbToRead - (uint32_t)cbBuf;
+                    pSSM->u.Read.offDataBuffer = 0;
+                    cbToRead = (uint32_t)cbBuf;
+                }
+                memset(pvBuf, 0, cbToRead);
+                break;
+            }
+
+            default:
+                AssertMsgFailedReturn(("%x\n", pSSM->u.Read.u8TypeAndFlags), pSSM->rc = VERR_SSM_BAD_REC_TYPE);
+        }
+
+        pSSM->offUnitUser += cbToRead;
+        cbBuf -= cbToRead;
+        pvBuf = (uint8_t *)pvBuf + cbToRead;
+    } while (cbBuf > 0);
+
+    Log4(("ssmR3DataReadUnBufferedV2: %08llx|%08llx/%08x/%08x: cbBuf=%#x %.*Rhxs%s\n",
+          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, 0, cbBufOrg, RT_MIN(SSM_LOG_BYTES, cbBufOrg), pvBufOrg, cbBufOrg > SSM_LOG_BYTES ? "..." : ""));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Buffer miss, do a buffered read.
+ *
+ * @returns VBox status code. Sets pSSM->rc on error.
+ *
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           Where to store the read data.
+ * @param   cbBuf           Number of bytes to read.
+ */
+static int ssmR3DataReadBufferedV2(PSSMHANDLE pSSM, void *pvBuf, size_t cbBuf)
+{
+    void   const *pvBufOrg = pvBuf; NOREF(pvBufOrg);
+    size_t const  cbBufOrg = cbBuf; NOREF(cbBufOrg);
+
+    /*
+     * Copy out what we've got in the buffer.
+     */
+    uint32_t off        = pSSM->u.Read.offDataBuffer;
+    int32_t  cbInBuffer = pSSM->u.Read.cbDataBuffer - off;
+    Log4(("ssmR3DataReadBufferedV2: %08llx|%08llx/%08x/%08x: cbBuf=%#x\n", ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, cbInBuffer, cbBufOrg));
+    if (cbInBuffer > 0)
+    {
+        uint32_t const cbToCopy = (uint32_t)cbInBuffer;
+        Assert(cbBuf > cbToCopy);
+        memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[off], cbToCopy);
+        pvBuf  = (uint8_t *)pvBuf + cbToCopy;
+        cbBuf -= cbToCopy;
+        pSSM->offUnitUser += cbToCopy;
+        pSSM->u.Read.cbDataBuffer  = 0;
+        pSSM->u.Read.offDataBuffer = 0;
+    }
+
+    /*
+     * Buffer more data.
+     */
+    do
+    {
+        /*
+         * Read the next record header if no more data.
+         */
+        if (!pSSM->u.Read.cbRecLeft)
+        {
+            int rc = ssmR3DataReadRecHdrV2(pSSM);
+            if (RT_FAILURE(rc))
+                return pSSM->rc = rc;
+        }
+        AssertLogRelMsgReturn(!pSSM->u.Read.fEndOfData, ("cbBuf=%zu\n", cbBuf), pSSM->rc = VERR_SSM_LOADED_TOO_MUCH);
+
+        /*
+         * Read data from the current record.
+         * LATER: optimize by reading directly into the output buffer for some cases.
+         */
+        uint32_t cbToRead;
+        switch (pSSM->u.Read.u8TypeAndFlags & SSM_REC_TYPE_MASK)
+        {
+            case SSM_REC_TYPE_RAW:
+            {
+                cbToRead = RT_MIN(sizeof(pSSM->u.Read.abDataBuffer), pSSM->u.Read.cbRecLeft);
+                int rc = ssmR3DataReadV2Raw(pSSM, &pSSM->u.Read.abDataBuffer[0], cbToRead);
+                if (RT_FAILURE(rc))
+                    return pSSM->rc = rc;
+                pSSM->u.Read.cbRecLeft   -= cbToRead;
+                pSSM->u.Read.cbDataBuffer = cbToRead;
+                break;
+            }
+
+            case SSM_REC_TYPE_RAW_LZF:
+            {
+                int rc = ssmR3DataReadV2RawLzfHdr(pSSM, &cbToRead);
+                if (RT_FAILURE(rc))
+                    return rc;
+                rc = ssmR3DataReadV2RawLzf(pSSM, &pSSM->u.Read.abDataBuffer[0], cbToRead);
+                if (RT_FAILURE(rc))
+                    return rc;
+                pSSM->u.Read.cbDataBuffer = cbToRead;
+                break;
+            }
+
+            case SSM_REC_TYPE_RAW_ZERO:
+            {
+                int rc = ssmR3DataReadV2RawZeroHdr(pSSM, &cbToRead);
+                if (RT_FAILURE(rc))
+                    return rc;
+                memset(&pSSM->u.Read.abDataBuffer[0], 0, cbToRead);
+                pSSM->u.Read.cbDataBuffer = cbToRead;
+                break;
+            }
+
+            default:
+                AssertMsgFailedReturn(("%x\n", pSSM->u.Read.u8TypeAndFlags), pSSM->rc = VERR_SSM_BAD_REC_TYPE);
+        }
+        /*pSSM->u.Read.offDataBuffer = 0;*/
+
+        /*
+         * Copy data from the buffer.
+         */
+        uint32_t cbToCopy = (uint32_t)RT_MIN(cbBuf, cbToRead);
+        memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[0], cbToCopy);
+        cbBuf -= cbToCopy;
+        pvBuf = (uint8_t *)pvBuf + cbToCopy;
+        pSSM->offUnitUser       += cbToCopy;
+        pSSM->u.Read.offDataBuffer = cbToCopy;
+    } while (cbBuf > 0);
+
+    Log4(("ssmR3DataReadBufferedV2: %08llx|%08llx/%08x/%08x: cbBuf=%#x %.*Rhxs%s\n",
+          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, pSSM->u.Read.cbDataBuffer - pSSM->u.Read.offDataBuffer,
+          cbBufOrg, RT_MIN(SSM_LOG_BYTES, cbBufOrg), pvBufOrg, cbBufOrg > SSM_LOG_BYTES ? "..." : ""));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inlined worker that handles format checks and buffered reads.
+ *
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           Where to store the read data.
+ * @param   cbBuf           Number of bytes to read.
+ */
+DECLINLINE(int) ssmR3DataRead(PSSMHANDLE pSSM, void *pvBuf, size_t cbBuf)
+{
+    /*
+     * Fend off previous errors and V1 data units.
+     */
+    if (RT_SUCCESS(pSSM->rc))
+    {
+        if (RT_LIKELY(pSSM->u.Read.uFmtVerMajor != 1))
+        {
+            /*
+             * Check if the requested data is buffered.
+             */
+            uint32_t off = pSSM->u.Read.offDataBuffer;
+            if (   off + cbBuf > pSSM->u.Read.cbDataBuffer
+                || cbBuf > sizeof(pSSM->u.Read.abDataBuffer))
+            {
+                if (cbBuf <= sizeof(pSSM->u.Read.abDataBuffer) / 8)
+                    return ssmR3DataReadBufferedV2(pSSM, pvBuf, cbBuf);
+                return ssmR3DataReadUnbufferedV2(pSSM, pvBuf, cbBuf);
+            }
+
+            memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[off], cbBuf);
+            pSSM->u.Read.offDataBuffer = off + (uint32_t)cbBuf;
+            pSSM->offUnitUser += cbBuf;
+            Log4((cbBuf
+                  ? "ssmR3DataRead: %08llx|%08llx/%08x/%08x: cbBuf=%#x %.*Rhxs%s\n"
+                  : "ssmR3DataRead: %08llx|%08llx/%08x/%08x: cbBuf=%#x\n",
+                  ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, pSSM->u.Read.cbDataBuffer - pSSM->u.Read.offDataBuffer,
+                  cbBuf, RT_MIN(SSM_LOG_BYTES, cbBuf), pvBuf, cbBuf > SSM_LOG_BYTES ? "..." : ""));
+
+            return VINF_SUCCESS;
+        }
+        return ssmR3DataReadV1(pSSM, pvBuf, cbBuf);
+    }
+    return pSSM->rc;
+}
+
+
+/**
+ * Gets a structure.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pvStruct        The structure address.
+ * @param   paFields        The array of structure fields descriptions.
+ *                          The array must be terminated by a SSMFIELD_ENTRY_TERM().
+ */
+VMMR3DECL(int) SSMR3GetStruct(PSSMHANDLE pSSM, void *pvStruct, PCSSMFIELD paFields)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    AssertPtr(pvStruct);
+    AssertPtr(paFields);
+
+    /* begin marker. */
+    uint32_t u32Magic;
+    int rc = SSMR3GetU32(pSSM, &u32Magic);
+    if (RT_FAILURE(rc))
+        return rc;
+    AssertMsgReturn(u32Magic == SSMR3STRUCT_BEGIN, ("u32Magic=%#RX32\n", u32Magic), pSSM->rc = VERR_SSM_STRUCTURE_MAGIC);
+
+    /* get the fields */
+    for (PCSSMFIELD pCur = paFields;
+         pCur->cb != UINT32_MAX && pCur->off != UINT32_MAX;
+         pCur++)
+    {
+        if (pCur->uFirstVer <= pSSM->u.Read.uCurUnitVer)
+        {
+            uint8_t *pbField = (uint8_t *)pvStruct + pCur->off;
+            switch ((uintptr_t)pCur->pfnGetPutOrTransformer)
+            {
+                case SSMFIELDTRANS_NO_TRANSFORMATION:
+                    rc = ssmR3DataRead(pSSM, pbField, pCur->cb);
+                    break;
+
+                case SSMFIELDTRANS_GCPTR:
+                    AssertMsgBreakStmt(pCur->cb == sizeof(RTGCPTR), ("%#x (%s)\n", pCur->cb, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = SSMR3GetGCPtr(pSSM, (PRTGCPTR)pbField);
+                    break;
+
+                case SSMFIELDTRANS_GCPHYS:
+                    AssertMsgBreakStmt(pCur->cb == sizeof(RTGCPHYS), ("%#x (%s)\n", pCur->cb, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = SSMR3GetGCPhys(pSSM, (PRTGCPHYS)pbField);
+                    break;
+
+                case SSMFIELDTRANS_RCPTR:
+                    AssertMsgBreakStmt(pCur->cb == sizeof(RTRCPTR), ("%#x (%s)\n", pCur->cb, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = SSMR3GetRCPtr(pSSM, (PRTRCPTR)pbField);
+                    break;
+
+                case SSMFIELDTRANS_RCPTR_ARRAY:
+                {
+                    uint32_t const cEntries = pCur->cb / sizeof(RTRCPTR);
+                    AssertMsgBreakStmt(pCur->cb == cEntries * sizeof(RTRCPTR) && cEntries, ("%#x (%s)\n", pCur->cb, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = VINF_SUCCESS;
+                    for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+                        rc = SSMR3GetRCPtr(pSSM, &((PRTRCPTR)pbField)[i]);
+                    break;
+                }
+
+                default:
+                    AssertMsgFailedBreakStmt(("%#x\n", pCur->pfnGetPutOrTransformer), rc = VERR_SSM_FIELD_COMPLEX);
+            }
+            if (RT_FAILURE(rc))
+            {
+                if (RT_SUCCESS(pSSM->rc))
+                    pSSM->rc = rc;
+                return rc;
+            }
+        }
+    }
+
+    /* end marker */
+    rc = SSMR3GetU32(pSSM, &u32Magic);
+    if (RT_FAILURE(rc))
+        return rc;
+    AssertMsgReturn(u32Magic == SSMR3STRUCT_END, ("u32Magic=%#RX32\n", u32Magic), pSSM->rc = VERR_SSM_STRUCTURE_MAGIC);
+    return rc;
+}
+
+
+/**
+ * SSMR3GetStructEx helper that gets a HCPTR that is used as a NULL indicator.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSSM            The saved state handle.
+ * @param   ppv             Where to return the value (0/1).
+ * @param   fFlags          SSMSTRUCT_FLAGS_XXX.
+ */
+DECLINLINE(int) ssmR3GetHCPtrNI(PSSMHANDLE pSSM, void **ppv, uint32_t fFlags)
+{
+    uintptr_t uPtrNI;
+    if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+    {
+        if (ssmR3GetHostBits(pSSM) == 64)
+        {
+            uint64_t u;
+            int rc = ssmR3DataRead(pSSM, &u, sizeof(u));
+            if (RT_FAILURE(rc))
+                return rc;
+            uPtrNI = u ? 1 : 0;
+        }
+        else
+        {
+            uint32_t u;
+            int rc = ssmR3DataRead(pSSM, &u, sizeof(u));
+            if (RT_FAILURE(rc))
+                return rc;
+            uPtrNI = u ? 1 : 0;
+        }
+    }
+    else
+    {
+        bool f;
+        int rc = SSMR3GetBool(pSSM, &f);
+        if (RT_FAILURE(rc))
+            return rc;
+        uPtrNI = f ? 1 : 0;
+    }
+    *ppv = (void *)uPtrNI;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets a structure, extended API.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pvStruct        The structure address.
+ * @param   cbStruct        The size of the struct (use for validation only).
+ * @param   fFlags          Combination of SSMSTRUCT_FLAGS_XXX defines.
+ * @param   paFields        The array of structure fields descriptions. The
+ *                          array must be terminated by a SSMFIELD_ENTRY_TERM().
+ * @param   pvUser          User argument for any callbacks that paFields might
+ *                          contain.
+ */
+VMMR3DECL(int) SSMR3GetStructEx(PSSMHANDLE pSSM, void *pvStruct, size_t cbStruct,
+                                uint32_t fFlags, PCSSMFIELD paFields, void *pvUser)
+{
+    int         rc;
+    uint32_t    u32Magic;
+
+    /*
+     * Validation.
+     */
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    AssertMsgReturn(!(fFlags & ~SSMSTRUCT_FLAGS_VALID_MASK), ("%#x\n", fFlags), pSSM->rc = VERR_INVALID_PARAMETER);
+    AssertPtr(pvStruct);
+    AssertPtr(paFields);
+
+    /*
+     * Begin marker.
+     */
+    if (!(fFlags & (SSMSTRUCT_FLAGS_NO_MARKERS | SSMSTRUCT_FLAGS_NO_LEAD_MARKER)))
+    {
+        rc = SSMR3GetU32(pSSM, &u32Magic);
+        if (RT_FAILURE(rc))
+            return rc;
+        AssertMsgReturn(u32Magic == SSMR3STRUCT_BEGIN, ("u32Magic=%#RX32\n", u32Magic), pSSM->rc = VERR_SSM_STRUCTURE_MAGIC);
+    }
+
+    /*
+     * Put the fields
+     */
+    rc = VINF_SUCCESS;
+    uint32_t off = 0;
+    for (PCSSMFIELD pCur = paFields;
+         pCur->cb != UINT32_MAX && pCur->off != UINT32_MAX;
+         pCur++)
+    {
+        uint32_t const offField =    (!SSMFIELDTRANS_IS_PADDING(pCur->pfnGetPutOrTransformer) || pCur->off != UINT32_MAX / 2)
+                                  &&  !SSMFIELDTRANS_IS_OLD(pCur->pfnGetPutOrTransformer)
+                                ? pCur->off
+                                : off;
+        uint32_t const cbField  = SSMFIELDTRANS_IS_OLD(pCur->pfnGetPutOrTransformer)
+                                ? 0
+                                : SSMFIELDTRANS_IS_PADDING(pCur->pfnGetPutOrTransformer)
+                                ? RT_HIWORD(pCur->cb)
+                                : pCur->cb;
+        AssertMsgReturn(   cbField            <= cbStruct
+                        && offField + cbField <= cbStruct
+                        && offField + cbField >= offField,
+                        ("off=%#x cb=%#x cbStruct=%#x (%s)\n", cbField, offField, cbStruct, pCur->pszName),
+                        pSSM->rc = VERR_SSM_FIELD_OUT_OF_BOUNDS);
+        AssertMsgReturn(    !(fFlags & SSMSTRUCT_FLAGS_FULL_STRUCT)
+                        || off == offField,
+                        ("off=%#x offField=%#x (%s)\n", off, offField, pCur->pszName),
+                        pSSM->rc = VERR_SSM_FIELD_NOT_CONSECUTIVE);
+
+        if (pCur->uFirstVer <= pSSM->u.Read.uCurUnitVer)
+        {
+            rc = VINF_SUCCESS;
+            uint8_t       *pbField  = (uint8_t *)pvStruct + offField;
+            switch ((uintptr_t)pCur->pfnGetPutOrTransformer)
+            {
+                case SSMFIELDTRANS_NO_TRANSFORMATION:
+                    rc = ssmR3DataRead(pSSM, pbField, cbField);
+                    break;
+
+                case SSMFIELDTRANS_GCPHYS:
+                    AssertMsgBreakStmt(cbField == sizeof(RTGCPHYS), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = SSMR3GetGCPhys(pSSM, (PRTGCPHYS)pbField);
+                    break;
+
+                case SSMFIELDTRANS_GCPTR:
+                    AssertMsgBreakStmt(cbField == sizeof(RTGCPTR), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = SSMR3GetGCPtr(pSSM, (PRTGCPTR)pbField);
+                    break;
+
+                case SSMFIELDTRANS_RCPTR:
+                    AssertMsgBreakStmt(cbField == sizeof(RTRCPTR), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = SSMR3GetRCPtr(pSSM, (PRTRCPTR)pbField);
+                    break;
+
+                case SSMFIELDTRANS_RCPTR_ARRAY:
+                {
+                    uint32_t const cEntries = cbField / sizeof(RTRCPTR);
+                    AssertMsgBreakStmt(cbField == cEntries * sizeof(RTRCPTR) && cEntries, ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = VINF_SUCCESS;
+                    for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+                        rc = SSMR3GetRCPtr(pSSM, &((PRTRCPTR)pbField)[i]);
+                    break;
+                }
+
+                case SSMFIELDTRANS_HCPTR_NI:
+                    AssertMsgBreakStmt(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = ssmR3GetHCPtrNI(pSSM, (void **)pbField, fFlags);
+                    break;
+
+                case SSMFIELDTRANS_HCPTR_NI_ARRAY:
+                {
+                    uint32_t const cEntries = cbField / sizeof(void *);
+                    AssertMsgBreakStmt(cbField == cEntries * sizeof(void *) && cEntries, ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = VINF_SUCCESS;
+                    for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+                        rc = ssmR3GetHCPtrNI(pSSM, &((void **)pbField)[i], fFlags);
+                    break;
+                }
+
+                case SSMFIELDTRANS_HCPTR_HACK_U32:
+                    AssertMsgBreakStmt(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    *(uintptr_t *)pbField = 0;
+                    rc = ssmR3DataRead(pSSM, pbField, sizeof(uint32_t));
+                    if ((fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE) && ssmR3GetHostBits(pSSM) == 64)
+                    {
+                        uint32_t u32;
+                        rc = ssmR3DataRead(pSSM, &u32, sizeof(uint32_t));
+                        AssertMsgBreakStmt(RT_FAILURE(rc) || u32 == 0 || (fFlags & SSMSTRUCT_FLAGS_SAVED_AS_MEM),
+                                           ("high=%#x low=%#x (%s)\n", u32, *(uint32_t *)pbField, pCur->pszName),
+                                           rc = VERR_SSM_FIELD_INVALID_VALUE);
+                    }
+                    break;
+
+                case SSMFIELDTRANS_U32_ZX_U64:
+                    AssertMsgBreakStmt(cbField == sizeof(uint64_t), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    ((uint32_t *)pbField)[1] = 0;
+                    rc = SSMR3GetU32(pSSM, (uint32_t *)pbField);
+                    break;
+
+
+                case SSMFIELDTRANS_IGNORE:
+                    if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                        rc = SSMR3Skip(pSSM, cbField);
+                    break;
+
+                case SSMFIELDTRANS_IGN_GCPHYS:
+                    AssertMsgBreakStmt(cbField == sizeof(RTGCPHYS), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                        rc = SSMR3Skip(pSSM, pSSM->u.Read.cbGCPhys);
+                    break;
+
+                case SSMFIELDTRANS_IGN_GCPTR:
+                    AssertMsgBreakStmt(cbField == sizeof(RTGCPTR), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                        rc = SSMR3Skip(pSSM, pSSM->u.Read.cbGCPtr);
+                    break;
+
+                case SSMFIELDTRANS_IGN_RCPTR:
+                    AssertMsgBreakStmt(cbField == sizeof(RTRCPTR), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                        rc = SSMR3Skip(pSSM, sizeof(RTRCPTR));
+                    break;
+
+                case SSMFIELDTRANS_IGN_HCPTR:
+                    AssertMsgBreakStmt(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                        rc = SSMR3Skip(pSSM, ssmR3GetHostBits(pSSM) / 8);
+                    break;
+
+
+                case SSMFIELDTRANS_OLD:
+                    AssertMsgBreakStmt(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = SSMR3Skip(pSSM, pCur->cb);
+                    break;
+
+                case SSMFIELDTRANS_OLD_GCPHYS:
+                    AssertMsgBreakStmt(pCur->cb == sizeof(RTGCPHYS) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = SSMR3Skip(pSSM, pSSM->u.Read.cbGCPhys);
+                    break;
+
+                case SSMFIELDTRANS_OLD_GCPTR:
+                    AssertMsgBreakStmt(pCur->cb == sizeof(RTGCPTR) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = SSMR3Skip(pSSM, pSSM->u.Read.cbGCPtr);
+                    break;
+
+                case SSMFIELDTRANS_OLD_RCPTR:
+                    AssertMsgBreakStmt(pCur->cb == sizeof(RTRCPTR) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = SSMR3Skip(pSSM, sizeof(RTRCPTR));
+                    break;
+
+                case SSMFIELDTRANS_OLD_HCPTR:
+                    AssertMsgBreakStmt(pCur->cb == sizeof(void *) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = SSMR3Skip(pSSM, ssmR3GetHostBits(pSSM) / 8);
+                    break;
+
+                case SSMFIELDTRANS_OLD_PAD_HC:
+                    AssertMsgBreakStmt(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    rc = SSMR3Skip(pSSM, ssmR3GetHostBits(pSSM) == 64 ? RT_HIWORD(pCur->cb) : RT_LOWORD(pCur->cb));
+                    break;
+
+                case SSMFIELDTRANS_OLD_PAD_MSC32:
+                    AssertMsgBreakStmt(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), rc = VERR_SSM_FIELD_INVALID_SIZE);
+                    if (ssmR3IsHostMsc32(pSSM))
+                        rc = SSMR3Skip(pSSM, pCur->cb);
+                    break;
+
+
+                case SSMFIELDTRANS_PAD_HC:
+                case SSMFIELDTRANS_PAD_HC32:
+                case SSMFIELDTRANS_PAD_HC64:
+                case SSMFIELDTRANS_PAD_HC_AUTO:
+                case SSMFIELDTRANS_PAD_MSC32_AUTO:
+                {
+                    uint32_t cb32    = RT_BYTE1(pCur->cb);
+                    uint32_t cb64    = RT_BYTE2(pCur->cb);
+                    uint32_t cbCtx   =    HC_ARCH_BITS == 64
+                                       || (   (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
+                                           && !SSM_HOST_IS_MSC_32)
+                                     ? cb64 : cb32;
+                    uint32_t cbSaved =    ssmR3GetHostBits(pSSM) == 64
+                                       || (   (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
+                                           && !ssmR3IsHostMsc32(pSSM))
+                                     ? cb64 : cb32;
+                    AssertMsgBreakStmt(    cbField == cbCtx
+                                       &&  (   (   pCur->off == UINT32_MAX / 2
+                                                && (   cbField == 0
+                                                    || (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_HC_AUTO
+                                                    || (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
+                                                   )
+                                               )
+                                            || (pCur->off != UINT32_MAX / 2 && cbField != 0)
+                                            )
+                                       , ("cbField=%#x cb32=%#x cb64=%#x HC_ARCH_BITS=%u cbCtx=%#x cbSaved=%#x off=%#x\n",
+                                          cbField, cb32, cb64, HC_ARCH_BITS, cbCtx, cbSaved, pCur->off),
+                                       rc = VERR_SSM_FIELD_INVALID_PADDING_SIZE);
+                    if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                        rc = SSMR3Skip(pSSM, cbSaved);
+                    break;
+                }
+
+                default:
+                    AssertBreakStmt(pCur->pfnGetPutOrTransformer, rc = VERR_SSM_FIELD_INVALID_CALLBACK);
+                    rc = pCur->pfnGetPutOrTransformer(pSSM, pCur, pvStruct, fFlags, true /*fGetOrPut*/, pvUser);
+                    break;
+            }
+            if (RT_FAILURE(rc))
+                break;
+        }
+
+        off = offField + cbField;
+    }
+
+    if (RT_SUCCESS(rc))
+        AssertMsgStmt(   !(fFlags & SSMSTRUCT_FLAGS_FULL_STRUCT)
+                      || off == cbStruct,
+                      ("off=%#x cbStruct=%#x\n", off, cbStruct),
+                      rc = VERR_SSM_FIELD_NOT_CONSECUTIVE);
+
+    if (RT_FAILURE(rc))
+    {
+        if (RT_SUCCESS(pSSM->rc))
+            pSSM->rc = rc;
+        return rc;
+    }
+
+    /*
+     * End marker
+     */
+    if (!(fFlags & (SSMSTRUCT_FLAGS_NO_MARKERS | SSMSTRUCT_FLAGS_NO_TAIL_MARKER)))
+    {
+        rc = SSMR3GetU32(pSSM, &u32Magic);
+        if (RT_FAILURE(rc))
+            return rc;
+        AssertMsgReturn(u32Magic == SSMR3STRUCT_END, ("u32Magic=%#RX32\n", u32Magic), pSSM->rc = VERR_SSM_STRUCTURE_MAGIC);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads a boolean item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pfBool          Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetBool(PSSMHANDLE pSSM, bool *pfBool)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    uint8_t u8; /* see SSMR3PutBool */
+    int rc = ssmR3DataRead(pSSM, &u8, sizeof(u8));
+    if (RT_SUCCESS(rc))
+    {
+        Assert(u8 <= 1);
+        *pfBool = RT_BOOL(u8);
+    }
+    return rc;
+}
+
+
+/**
+ * Loads a volatile boolean item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pfBool          Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetBoolV(PSSMHANDLE pSSM, bool volatile *pfBool)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    uint8_t u8; /* see SSMR3PutBool */
+    int rc = ssmR3DataRead(pSSM, &u8, sizeof(u8));
+    if (RT_SUCCESS(rc))
+    {
+        Assert(u8 <= 1);
+        *pfBool = RT_BOOL(u8);
+    }
+    return rc;
+}
+
+
+/**
+ * Loads a 8-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu8             Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU8(PSSMHANDLE pSSM, uint8_t *pu8)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pu8, sizeof(*pu8));
+}
+
+
+/**
+ * Loads a volatile 8-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu8             Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU8V(PSSMHANDLE pSSM, uint8_t volatile *pu8)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, (void *)pu8, sizeof(*pu8));
+}
+
+
+/**
+ * Loads a 8-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pi8             Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS8(PSSMHANDLE pSSM, int8_t *pi8)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pi8, sizeof(*pi8));
+}
+
+
+/**
+ * Loads a volatile 8-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pi8             Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS8V(PSSMHANDLE pSSM, int8_t volatile *pi8)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, (void *)pi8, sizeof(*pi8));
+}
+
+
+/**
+ * Loads a 16-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu16            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU16(PSSMHANDLE pSSM, uint16_t *pu16)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pu16, sizeof(*pu16));
+}
+
+
+/**
+ * Loads a volatile 16-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu16            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU16V(PSSMHANDLE pSSM, uint16_t volatile *pu16)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, (void *)pu16, sizeof(*pu16));
+}
+
+
+/**
+ * Loads a 16-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pi16            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS16(PSSMHANDLE pSSM, int16_t *pi16)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pi16, sizeof(*pi16));
+}
+
+
+/**
+ * Loads a volatile 16-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pi16            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS16V(PSSMHANDLE pSSM, int16_t volatile *pi16)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, (void *)pi16, sizeof(*pi16));
+}
+
+
+/**
+ * Loads a 32-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu32            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU32(PSSMHANDLE pSSM, uint32_t *pu32)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pu32, sizeof(*pu32));
+}
+
+
+/**
+ * Loads a volatile 32-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu32            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU32V(PSSMHANDLE pSSM, uint32_t volatile *pu32)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, (void *)pu32, sizeof(*pu32));
+}
+
+
+/**
+ * Loads a 32-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pi32            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS32(PSSMHANDLE pSSM, int32_t *pi32)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pi32, sizeof(*pi32));
+}
+
+
+/**
+ * Loads a volatile 32-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pi32            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS32V(PSSMHANDLE pSSM, int32_t volatile *pi32)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, (void *)pi32, sizeof(*pi32));
+}
+
+
+/**
+ * Loads a 64-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu64            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU64(PSSMHANDLE pSSM, uint64_t *pu64)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pu64, sizeof(*pu64));
+}
+
+
+/**
+ * Loads a volatile 64-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu64            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU64V(PSSMHANDLE pSSM, uint64_t volatile *pu64)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, (void *)pu64, sizeof(*pu64));
+}
+
+
+/**
+ * Loads a 64-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pi64            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS64(PSSMHANDLE pSSM, int64_t *pi64)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pi64, sizeof(*pi64));
+}
+
+
+/**
+ * Loads a volatile 64-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pi64            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS64V(PSSMHANDLE pSSM, int64_t volatile *pi64)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, (void *)pi64, sizeof(*pi64));
+}
+
+
+/**
+ * Loads a 128-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu128           Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU128(PSSMHANDLE pSSM, uint128_t *pu128)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pu128, sizeof(*pu128));
+}
+
+
+/**
+ * Loads a volatile 128-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu128           Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU128V(PSSMHANDLE pSSM, uint128_t volatile *pu128)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, (void *)pu128, sizeof(*pu128));
+}
+
+
+/**
+ * Loads a 128-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pi128           Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS128(PSSMHANDLE pSSM, int128_t *pi128)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pi128, sizeof(*pi128));
+}
+
+
+/**
+ * Loads a volatile 128-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pi128           Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS128V(PSSMHANDLE pSSM, int128_t volatile *pi128)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, (void *)pi128, sizeof(*pi128));
+}
+
+
+/**
+ * Loads a VBox unsigned integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu              Where to store the integer.
+ */
+VMMR3DECL(int) SSMR3GetUInt(PSSMHANDLE pSSM, PRTUINT pu)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pu, sizeof(*pu));
+}
+
+
+/**
+ * Loads a VBox signed integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pi              Where to store the integer.
+ */
+VMMR3DECL(int) SSMR3GetSInt(PSSMHANDLE pSSM, PRTINT pi)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pi, sizeof(*pi));
+}
+
+
+/**
+ * Loads a GC natural unsigned integer item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu              Where to store the integer.
+ *
+ * @deprecated Silly type with an incorrect size, don't use it.
+ */
+VMMR3DECL(int) SSMR3GetGCUInt(PSSMHANDLE pSSM, PRTGCUINT pu)
+{
+    AssertCompile(sizeof(RTGCPTR) == sizeof(*pu));
+    return SSMR3GetGCPtr(pSSM, (PRTGCPTR)pu);
+}
+
+
+/**
+ * Loads a GC unsigned integer register item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pu              Where to store the integer.
+ */
+VMMR3DECL(int) SSMR3GetGCUIntReg(PSSMHANDLE pSSM, PRTGCUINTREG pu)
+{
+    AssertCompile(sizeof(RTGCPTR) == sizeof(*pu));
+    return SSMR3GetGCPtr(pSSM, (PRTGCPTR)pu);
+}
+
+
+/**
+ * Loads a 32 bits GC physical address item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPhys         Where to store the GC physical address.
+ */
+VMMR3DECL(int) SSMR3GetGCPhys32(PSSMHANDLE pSSM, PRTGCPHYS32 pGCPhys)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pGCPhys, sizeof(*pGCPhys));
+}
+
+
+/**
+ * Loads a 32 bits GC physical address item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPhys         Where to store the GC physical address.
+ */
+VMMR3DECL(int) SSMR3GetGCPhys32V(PSSMHANDLE pSSM, RTGCPHYS32 volatile *pGCPhys)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, (void *)pGCPhys, sizeof(*pGCPhys));
+}
+
+
+/**
+ * Loads a 64 bits GC physical address item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPhys         Where to store the GC physical address.
+ */
+VMMR3DECL(int) SSMR3GetGCPhys64(PSSMHANDLE pSSM, PRTGCPHYS64 pGCPhys)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pGCPhys, sizeof(*pGCPhys));
+}
+
+
+/**
+ * Loads a volatile 64 bits GC physical address item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPhys         Where to store the GC physical address.
+ */
+VMMR3DECL(int) SSMR3GetGCPhys64V(PSSMHANDLE pSSM, RTGCPHYS64 volatile *pGCPhys)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, (void *)pGCPhys, sizeof(*pGCPhys));
+}
+
+
+/**
+ * Loads a GC physical address item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPhys         Where to store the GC physical address.
+ */
+VMMR3DECL(int) SSMR3GetGCPhys(PSSMHANDLE pSSM, PRTGCPHYS pGCPhys)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+
+    /*
+     * Default size?
+     */
+    if (RT_LIKELY(sizeof(*pGCPhys) == pSSM->u.Read.cbGCPhys))
+        return ssmR3DataRead(pSSM, pGCPhys, sizeof(*pGCPhys));
+
+    /*
+     * Fiddly.
+     */
+    Assert(sizeof(*pGCPhys)      == sizeof(uint64_t) || sizeof(*pGCPhys)      == sizeof(uint32_t));
+    Assert(pSSM->u.Read.cbGCPhys == sizeof(uint64_t) || pSSM->u.Read.cbGCPhys == sizeof(uint32_t));
+    if (pSSM->u.Read.cbGCPhys == sizeof(uint64_t))
+    {
+        /* 64-bit saved, 32-bit load: try truncate it. */
+        uint64_t u64;
+        int rc = ssmR3DataRead(pSSM, &u64, sizeof(uint64_t));
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u64 >= _4G)
+            return VERR_SSM_GCPHYS_OVERFLOW;
+        *pGCPhys = (RTGCPHYS)u64;
+        return rc;
+    }
+
+    /* 32-bit saved, 64-bit load: clear the high part. */
+    *pGCPhys = 0;
+    return ssmR3DataRead(pSSM, pGCPhys, sizeof(uint32_t));
+}
+
+/**
+ * Loads a volatile GC physical address item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPhys         Where to store the GC physical address.
+ */
+VMMR3DECL(int) SSMR3GetGCPhysV(PSSMHANDLE pSSM, RTGCPHYS volatile *pGCPhys)
+{
+    return SSMR3GetGCPhys(pSSM, (PRTGCPHYS)pGCPhys);
+}
+
+
+/**
+ * Loads a GC virtual address item from the current data unit.
+ *
+ * Only applies to in the 1.1 format:
+ *  - SSMR3GetGCPtr
+ *  - SSMR3GetGCUIntPtr
+ *  - SSMR3GetGCUInt
+ *  - SSMR3GetGCUIntReg
+ *
+ * Put functions are not affected.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   cbGCPtr         Size of RTGCPTR
+ *
+ * @remarks This interface only works with saved state version 1.1, if the
+ *          format isn't 1.1 the call will be ignored.
+ */
+VMMR3_INT_DECL(int) SSMR3HandleSetGCPtrSize(PSSMHANDLE pSSM, unsigned cbGCPtr)
+{
+    Assert(cbGCPtr == sizeof(RTGCPTR32) || cbGCPtr == sizeof(RTGCPTR64));
+    if (!pSSM->u.Read.fFixedGCPtrSize)
+    {
+        Log(("SSMR3SetGCPtrSize: %u -> %u bytes\n", pSSM->u.Read.cbGCPtr, cbGCPtr));
+        pSSM->u.Read.cbGCPtr = cbGCPtr;
+        pSSM->u.Read.fFixedGCPtrSize = true;
+    }
+    else if (   pSSM->u.Read.cbGCPtr != cbGCPtr
+             && pSSM->u.Read.uFmtVerMajor == 1
+             && pSSM->u.Read.uFmtVerMinor == 1)
+        AssertMsgFailed(("SSMR3SetGCPtrSize: already fixed at %u bytes; requested %u bytes\n", pSSM->u.Read.cbGCPtr, cbGCPtr));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads a GC virtual address item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPtr          Where to store the GC virtual address.
+ */
+VMMR3DECL(int) SSMR3GetGCPtr(PSSMHANDLE pSSM, PRTGCPTR pGCPtr)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+
+    /*
+     * Default size?
+     */
+    if (RT_LIKELY(sizeof(*pGCPtr) == pSSM->u.Read.cbGCPtr))
+        return ssmR3DataRead(pSSM, pGCPtr, sizeof(*pGCPtr));
+
+    /*
+     * Fiddly.
+     */
+    Assert(sizeof(*pGCPtr)      == sizeof(uint64_t) || sizeof(*pGCPtr)      == sizeof(uint32_t));
+    Assert(pSSM->u.Read.cbGCPtr == sizeof(uint64_t) || pSSM->u.Read.cbGCPtr == sizeof(uint32_t));
+    if (pSSM->u.Read.cbGCPtr == sizeof(uint64_t))
+    {
+        /* 64-bit saved, 32-bit load: try truncate it. */
+        uint64_t u64;
+        int rc = ssmR3DataRead(pSSM, &u64, sizeof(uint64_t));
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u64 >= _4G)
+            return VERR_SSM_GCPTR_OVERFLOW;
+        *pGCPtr = (RTGCPTR)u64;
+        return rc;
+    }
+
+    /* 32-bit saved, 64-bit load: clear the high part. */
+    *pGCPtr = 0;
+    return ssmR3DataRead(pSSM, pGCPtr, sizeof(uint32_t));
+}
+
+
+/**
+ * Loads a GC virtual address (represented as unsigned integer) item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPtr          Where to store the GC virtual address.
+ */
+VMMR3DECL(int) SSMR3GetGCUIntPtr(PSSMHANDLE pSSM, PRTGCUINTPTR pGCPtr)
+{
+    AssertCompile(sizeof(RTGCPTR) == sizeof(*pGCPtr));
+    return SSMR3GetGCPtr(pSSM, (PRTGCPTR)pGCPtr);
+}
+
+
+/**
+ * Loads an RC virtual address item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pRCPtr          Where to store the RC virtual address.
+ */
+VMMR3DECL(int) SSMR3GetRCPtr(PSSMHANDLE pSSM, PRTRCPTR pRCPtr)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pRCPtr, sizeof(*pRCPtr));
+}
+
+
+/**
+ * Loads a I/O port address item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pIOPort         Where to store the I/O port address.
+ */
+VMMR3DECL(int) SSMR3GetIOPort(PSSMHANDLE pSSM, PRTIOPORT pIOPort)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pIOPort, sizeof(*pIOPort));
+}
+
+
+/**
+ * Loads a selector item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pSel            Where to store the selector.
+ */
+VMMR3DECL(int) SSMR3GetSel(PSSMHANDLE pSSM, PRTSEL pSel)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pSel, sizeof(*pSel));
+}
+
+
+/**
+ * Loads a memory item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pv              Where to store the item.
+ * @param   cb              Size of the item.
+ */
+VMMR3DECL(int) SSMR3GetMem(PSSMHANDLE pSSM, void *pv, size_t cb)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pv, cb);
+}
+
+
+/**
+ * Loads a string item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   psz             Where to store the item.
+ * @param   cbMax           Max size of the item (including '\\0').
+ */
+VMMR3DECL(int) SSMR3GetStrZ(PSSMHANDLE pSSM, char *psz, size_t cbMax)
+{
+    return SSMR3GetStrZEx(pSSM, psz, cbMax, NULL);
+}
+
+
+/**
+ * Loads a string item from the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   psz             Where to store the item.
+ * @param   cbMax           Max size of the item (including '\\0').
+ * @param   pcbStr          The length of the loaded string excluding the '\\0'. (optional)
+ */
+VMMR3DECL(int) SSMR3GetStrZEx(PSSMHANDLE pSSM, char *psz, size_t cbMax, size_t *pcbStr)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+
+    /* read size prefix. */
+    uint32_t u32;
+    int rc = SSMR3GetU32(pSSM, &u32);
+    if (RT_SUCCESS(rc))
+    {
+        if (pcbStr)
+            *pcbStr = u32;
+        if (u32 < cbMax)
+        {
+            /* terminate and read string content. */
+            psz[u32] = '\0';
+            return ssmR3DataRead(pSSM, psz, u32);
+        }
+        return VERR_TOO_MUCH_DATA;
+    }
+    return rc;
+}
+
+
+/**
+ * Skips a number of bytes in the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The SSM handle.
+ * @param   cb                  The number of bytes to skip.
+ */
+VMMR3DECL(int) SSMR3Skip(PSSMHANDLE pSSM, size_t cb)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    while (cb > 0)
+    {
+        uint8_t abBuf[8192];
+        size_t  cbCur = RT_MIN(sizeof(abBuf), cb);
+        cb -= cbCur;
+        int rc = ssmR3DataRead(pSSM, abBuf, cbCur);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Skips to the end of the current data unit.
+ *
+ * Since version 2 of the format, the load exec callback have to explicitly call
+ * this API if it wish to be lazy for some reason.  This is because there seldom
+ * is a good reason to not read your entire data unit and it was hiding bugs.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The saved state handle.
+ */
+VMMR3DECL(int) SSMR3SkipToEndOfUnit(PSSMHANDLE pSSM)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    if (pSSM->u.Read.uFmtVerMajor >= 2)
+    {
+        /*
+         * Read until we the end of data condition is raised.
+         */
+        pSSM->u.Read.cbDataBuffer  = 0;
+        pSSM->u.Read.offDataBuffer = 0;
+        if (!pSSM->u.Read.fEndOfData)
+        {
+            do
+            {
+                /* read the rest of the current record */
+                while (pSSM->u.Read.cbRecLeft)
+                {
+                    uint8_t  abBuf[8192];
+                    uint32_t cbToRead = RT_MIN(pSSM->u.Read.cbRecLeft, sizeof(abBuf));
+                    int rc = ssmR3DataReadV2Raw(pSSM, abBuf, cbToRead);
+                    if (RT_FAILURE(rc))
+                        return pSSM->rc = rc;
+                    pSSM->u.Read.cbRecLeft -= cbToRead;
+                }
+
+                /* read the next header. */
+                int rc = ssmR3DataReadRecHdrV2(pSSM);
+                if (RT_FAILURE(rc))
+                    return pSSM->rc = rc;
+            } while (!pSSM->u.Read.fEndOfData);
+        }
+    }
+    /* else: Doesn't matter for the version 1 loading. */
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Calculate the checksum of a file portion.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle
+ * @param   off         Where to start checksumming.
+ * @param   cb          How much to checksum.
+ * @param   pu32CRC     Where to store the calculated checksum.
+ */
+static int ssmR3CalcChecksum(PSSMSTRM pStrm, uint64_t off, uint64_t cb, uint32_t *pu32CRC)
+{
+    /*
+     * Allocate a buffer.
+     */
+    const size_t cbBuf = _32K;
+    void *pvBuf = RTMemTmpAlloc(cbBuf);
+    if (!pvBuf)
+        return VERR_NO_TMP_MEMORY;
+
+    /*
+     * Loop reading and calculating CRC32.
+     */
+    int         rc     = VINF_SUCCESS;
+    uint32_t    u32CRC = RTCrc32Start();
+    while (cb > 0)
+    {
+        /* read chunk */
+        size_t cbToRead = cbBuf;
+        if (cb < cbBuf)
+            cbToRead = cb;
+        rc = ssmR3StrmPeekAt(pStrm, off, pvBuf, cbToRead, NULL);
+        if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("Failed with rc=%Rrc while calculating crc.\n", rc));
+            RTMemTmpFree(pvBuf);
+            return rc;
+        }
+
+        /* advance */
+        cb  -= cbToRead;
+        off += cbToRead;
+
+        /* calc crc32. */
+        u32CRC = RTCrc32Process(u32CRC, pvBuf, cbToRead);
+    }
+    RTMemTmpFree(pvBuf);
+
+    /* store the calculated crc */
+    u32CRC = RTCrc32Finish(u32CRC);
+    Log(("SSM: u32CRC=0x%08x\n", u32CRC));
+    *pu32CRC = u32CRC;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Validates a version 2 footer.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pFooter             The footer.
+ * @param   offFooter           The stream offset of the footer.
+ * @param   cDirEntries         The number of directory entries. UINT32_MAX if
+ *                              unknown.
+ * @param   fStreamCrc32        Whether the stream is checksummed using CRC-32.
+ * @param   u32StreamCRC        The stream checksum.
+ */
+static int ssmR3ValidateFooter(PSSMFILEFTR pFooter, uint64_t offFooter, uint32_t cDirEntries, bool fStreamCrc32, uint32_t u32StreamCRC)
+{
+    if (memcmp(pFooter->szMagic, SSMFILEFTR_MAGIC, sizeof(pFooter->szMagic)))
+    {
+        LogRel(("SSM: Bad footer magic: %.*Rhxs\n", sizeof(pFooter->szMagic), &pFooter->szMagic[0]));
+        return VERR_SSM_INTEGRITY_FOOTER;
+    }
+    SSM_CHECK_CRC32_RET(pFooter, sizeof(*pFooter), ("Footer CRC mismatch: %08x, correct is %08x\n", u32CRC, u32ActualCRC));
+    if (pFooter->offStream != offFooter)
+    {
+        LogRel(("SSM: SSMFILEFTR::offStream is wrong: %llx, expected %llx\n", pFooter->offStream, offFooter));
+        return VERR_SSM_INTEGRITY_FOOTER;
+    }
+    if (pFooter->u32Reserved)
+    {
+        LogRel(("SSM: Reserved footer field isn't zero: %08x\n", pFooter->u32Reserved));
+        return VERR_SSM_INTEGRITY_FOOTER;
+    }
+    if (cDirEntries != UINT32_MAX)
+        AssertLogRelMsgReturn(pFooter->cDirEntries == cDirEntries,
+                              ("Footer: cDirEntries=%#x, expected %#x\n", pFooter->cDirEntries, cDirEntries),
+                              VERR_SSM_INTEGRITY_FOOTER);
+    else
+        AssertLogRelMsgReturn(pFooter->cDirEntries < _64K,
+                              ("Footer: cDirEntries=%#x\n", pFooter->cDirEntries),
+                              VERR_SSM_INTEGRITY_FOOTER);
+    if (    !fStreamCrc32
+        &&  pFooter->u32StreamCRC)
+    {
+        LogRel(("SSM: u32StreamCRC field isn't zero, but header says stream checksumming is disabled.\n"));
+        return VERR_SSM_INTEGRITY_FOOTER;
+    }
+    if (    fStreamCrc32
+        &&  pFooter->u32StreamCRC != u32StreamCRC)
+    {
+        LogRel(("SSM: Bad stream CRC: %#x, expected %#x.\n", pFooter->u32StreamCRC, u32StreamCRC));
+        return VERR_SSM_INTEGRITY_CRC;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Validates the header information stored in the handle.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSSM            The handle.
+ * @param   fHaveHostBits   Set if the host bits field is valid.
+ * @param   fHaveVersion    Set if we have a version.
+ */
+static int ssmR3ValidateHeaderInfo(PSSMHANDLE pSSM, bool fHaveHostBits, bool fHaveVersion)
+{
+    Assert(pSSM->u.Read.cbFileHdr < 256 && pSSM->u.Read.cbFileHdr > 32);
+    Assert(pSSM->u.Read.uFmtVerMajor == 1 || pSSM->u.Read.uFmtVerMajor == 2);
+    Assert(pSSM->u.Read.uFmtVerMinor <= 2);
+
+    if (fHaveVersion)
+    {
+        if (    pSSM->u.Read.u16VerMajor == 0
+            ||  pSSM->u.Read.u16VerMajor > 1000
+            ||  pSSM->u.Read.u16VerMinor > 1000
+            ||  pSSM->u.Read.u32VerBuild > _1M
+            ||  pSSM->u.Read.u32SvnRev == 0
+            ||  pSSM->u.Read.u32SvnRev > 10000000 /*100M*/)
+        {
+            LogRel(("SSM: Incorrect version values: %u.%u.%u.r%u\n",
+                    pSSM->u.Read.u16VerMajor, pSSM->u.Read.u16VerMinor, pSSM->u.Read.u32VerBuild, pSSM->u.Read.u32SvnRev));
+            return VERR_SSM_INTEGRITY_VBOX_VERSION;
+        }
+    }
+    else
+        AssertLogRelReturn(   pSSM->u.Read.u16VerMajor == 0
+                           && pSSM->u.Read.u16VerMinor == 0
+                           && pSSM->u.Read.u32VerBuild == 0
+                           && pSSM->u.Read.u32SvnRev   == 0,
+                           VERR_SSM_INTEGRITY_VBOX_VERSION);
+
+    if (fHaveHostBits)
+    {
+        if (    pSSM->u.Read.cHostBits != 32
+            &&  pSSM->u.Read.cHostBits != 64)
+        {
+            LogRel(("SSM: Incorrect cHostBits value: %u\n", pSSM->u.Read.cHostBits));
+            return VERR_SSM_INTEGRITY_HEADER;
+        }
+    }
+    else
+        AssertLogRelReturn(pSSM->u.Read.cHostBits == 0, VERR_SSM_INTEGRITY_HEADER);
+
+    if (    pSSM->u.Read.cbGCPhys != sizeof(uint32_t)
+        &&  pSSM->u.Read.cbGCPhys != sizeof(uint64_t))
+    {
+        LogRel(("SSM: Incorrect cbGCPhys value: %d\n", pSSM->u.Read.cbGCPhys));
+        return VERR_SSM_INTEGRITY_HEADER;
+    }
+    if (    pSSM->u.Read.cbGCPtr != sizeof(uint32_t)
+        &&  pSSM->u.Read.cbGCPtr != sizeof(uint64_t))
+    {
+        LogRel(("SSM: Incorrect cbGCPtr value: %d\n", pSSM->u.Read.cbGCPtr));
+        return VERR_SSM_INTEGRITY_HEADER;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Reads the header, detects the format version and performs integrity
+ * validations.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The saved state handle.  A number of field will
+ *                              be updated, mostly header related information.
+ *                              fLiveSave is also set if appropriate.
+ * @param   fChecksumIt         Whether to checksum the file or not.  This will
+ *                              be ignored if it the stream isn't a file.
+ * @param   fChecksumOnRead     Whether to validate the checksum while reading
+ *                              the stream instead of up front. If not possible,
+ *                              verify the checksum up front.
+ */
+static int ssmR3HeaderAndValidate(PSSMHANDLE pSSM, bool fChecksumIt, bool fChecksumOnRead)
+{
+    /*
+     * Read and check the header magic.
+     */
+    union
+    {
+        SSMFILEHDR          v2_0;
+        SSMFILEHDRV12       v1_2;
+        SSMFILEHDRV11       v1_1;
+    } uHdr;
+    int rc = ssmR3StrmRead(&pSSM->Strm, &uHdr, sizeof(uHdr.v2_0.szMagic));
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("SSM: Failed to read file magic header. rc=%Rrc\n", rc));
+        return rc;
+    }
+    if (memcmp(uHdr.v2_0.szMagic, SSMFILEHDR_MAGIC_BASE, sizeof(SSMFILEHDR_MAGIC_BASE) - 1))
+    {
+        Log(("SSM: Not a saved state file. magic=%.*s\n", sizeof(uHdr.v2_0.szMagic) - 1, uHdr.v2_0.szMagic));
+        return VERR_SSM_INTEGRITY_MAGIC;
+    }
+
+    /*
+     * Find the header size and read the rest.
+     */
+    static const struct
+    {
+        char        szMagic[sizeof(SSMFILEHDR_MAGIC_V2_0)];
+        uint32_t    cbHdr;
+        unsigned    uFmtVerMajor;
+        unsigned    uFmtVerMinor;
+    }   s_aVers[] =
+    {
+        { SSMFILEHDR_MAGIC_V2_0, sizeof(SSMFILEHDR),    2, 0 },
+        { SSMFILEHDR_MAGIC_V1_2, sizeof(SSMFILEHDRV12), 1, 2 },
+        { SSMFILEHDR_MAGIC_V1_1, sizeof(SSMFILEHDRV11), 1, 1 },
+    };
+    int iVer = RT_ELEMENTS(s_aVers);
+    while (iVer-- > 0)
+        if (!memcmp(uHdr.v2_0.szMagic, s_aVers[iVer].szMagic, sizeof(uHdr.v2_0.szMagic)))
+            break;
+    if (iVer < 0)
+    {
+        Log(("SSM: Unknown file format version. magic=%.*s\n", sizeof(uHdr.v2_0.szMagic) - 1, uHdr.v2_0.szMagic));
+        return VERR_SSM_INTEGRITY_VERSION;
+    }
+    pSSM->u.Read.uFmtVerMajor   = s_aVers[iVer].uFmtVerMajor;
+    pSSM->u.Read.uFmtVerMinor   = s_aVers[iVer].uFmtVerMinor;
+    pSSM->u.Read.cbFileHdr      = s_aVers[iVer].cbHdr;
+
+    rc = ssmR3StrmRead(&pSSM->Strm, (uint8_t *)&uHdr + sizeof(uHdr.v2_0.szMagic), pSSM->u.Read.cbFileHdr - sizeof(uHdr.v2_0.szMagic));
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("SSM: Failed to read the file header. rc=%Rrc\n", rc));
+        return rc;
+    }
+
+    /*
+     * Make version specific adjustments.
+     */
+    if (pSSM->u.Read.uFmtVerMajor >= 2)
+    {
+        /*
+         * Version 2.0 and later.
+         */
+        if (pSSM->u.Read.uFmtVerMinor == 0)
+        {
+            /* validate the header. */
+            SSM_CHECK_CRC32_RET(&uHdr.v2_0, sizeof(uHdr.v2_0), ("Header CRC mismatch: %08x, correct is %08x\n", u32CRC, u32ActualCRC));
+            if (uHdr.v2_0.u8Reserved)
+            {
+                LogRel(("SSM: Reserved header field isn't zero: %02x\n", uHdr.v2_0.u8Reserved));
+                return VERR_SSM_INTEGRITY;
+            }
+            if (uHdr.v2_0.fFlags & ~(SSMFILEHDR_FLAGS_STREAM_CRC32 | SSMFILEHDR_FLAGS_STREAM_LIVE_SAVE))
+            {
+                LogRel(("SSM: Unknown header flags: %08x\n", uHdr.v2_0.fFlags));
+                return VERR_SSM_INTEGRITY;
+            }
+            if (    uHdr.v2_0.cbMaxDecompr > sizeof(pSSM->u.Read.abDataBuffer)
+                ||  uHdr.v2_0.cbMaxDecompr < _1K
+                ||  (uHdr.v2_0.cbMaxDecompr & 0xff) != 0)
+            {
+                LogRel(("SSM: The cbMaxDecompr header field is out of range: %#x\n", uHdr.v2_0.cbMaxDecompr));
+                return VERR_SSM_INTEGRITY;
+            }
+
+            /* set the header info. */
+            pSSM->u.Read.cHostBits      = uHdr.v2_0.cHostBits;
+            pSSM->u.Read.u16VerMajor    = uHdr.v2_0.u16VerMajor;
+            pSSM->u.Read.u16VerMinor    = uHdr.v2_0.u16VerMinor;
+            pSSM->u.Read.u32VerBuild    = uHdr.v2_0.u32VerBuild;
+            pSSM->u.Read.u32SvnRev      = uHdr.v2_0.u32SvnRev;
+            pSSM->u.Read.cbGCPhys       = uHdr.v2_0.cbGCPhys;
+            pSSM->u.Read.cbGCPtr        = uHdr.v2_0.cbGCPtr;
+            pSSM->u.Read.fFixedGCPtrSize= true;
+            pSSM->u.Read.fStreamCrc32   = !!(uHdr.v2_0.fFlags & SSMFILEHDR_FLAGS_STREAM_CRC32);
+            pSSM->fLiveSave             = !!(uHdr.v2_0.fFlags & SSMFILEHDR_FLAGS_STREAM_LIVE_SAVE);
+        }
+        else
+            AssertFailedReturn(VERR_SSM_IPE_2);
+        if (!pSSM->u.Read.fStreamCrc32)
+            ssmR3StrmDisableChecksumming(&pSSM->Strm);
+
+        /*
+         * Read and validate the footer if it's a file.
+         */
+        if (ssmR3StrmIsFile(&pSSM->Strm))
+        {
+            SSMFILEFTR  Footer;
+            uint64_t    offFooter;
+            rc = ssmR3StrmPeekAt(&pSSM->Strm, -(RTFOFF)sizeof(SSMFILEFTR), &Footer, sizeof(Footer), &offFooter);
+            AssertLogRelRCReturn(rc, rc);
+
+            rc = ssmR3ValidateFooter(&Footer, offFooter, UINT32_MAX, pSSM->u.Read.fStreamCrc32, Footer.u32StreamCRC);
+            if (RT_FAILURE(rc))
+                return rc;
+
+            pSSM->u.Read.cbLoadFile = offFooter + sizeof(Footer);
+            pSSM->u.Read.u32LoadCRC = Footer.u32StreamCRC;
+        }
+        else
+        {
+            pSSM->u.Read.cbLoadFile = UINT64_MAX;
+            pSSM->u.Read.u32LoadCRC = 0;
+        }
+
+        /*
+         * Validate the header info we've set in the handle.
+         */
+        rc = ssmR3ValidateHeaderInfo(pSSM, true /*fHaveHostBits*/, true /*fHaveVersion*/);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /*
+         * Check the checksum if that's called for and possible.
+         */
+        if (    pSSM->u.Read.fStreamCrc32
+            &&  fChecksumIt
+            &&  !fChecksumOnRead
+            &&  ssmR3StrmIsFile(&pSSM->Strm))
+        {
+            uint32_t u32CRC;
+            rc = ssmR3CalcChecksum(&pSSM->Strm, 0, pSSM->u.Read.cbLoadFile - sizeof(SSMFILEFTR), &u32CRC);
+            if (RT_FAILURE(rc))
+                return rc;
+            if (u32CRC != pSSM->u.Read.u32LoadCRC)
+            {
+                LogRel(("SSM: Invalid CRC! Calculated %#010x, in footer %#010x\n", u32CRC, pSSM->u.Read.u32LoadCRC));
+                return VERR_SSM_INTEGRITY_CRC;
+            }
+        }
+    }
+    else
+    {
+        /*
+         * Version 1.x of the format.
+         */
+        bool        fHaveHostBits = true;
+        bool        fHaveVersion  = false;
+        RTUUID      MachineUuidFromHdr;
+
+        ssmR3StrmDisableChecksumming(&pSSM->Strm);
+        if (pSSM->u.Read.uFmtVerMinor == 1)
+        {
+            pSSM->u.Read.cHostBits      = 0; /* unknown */
+            pSSM->u.Read.u16VerMajor    = 0;
+            pSSM->u.Read.u16VerMinor    = 0;
+            pSSM->u.Read.u32VerBuild    = 0;
+            pSSM->u.Read.u32SvnRev      = 0;
+            pSSM->u.Read.cbLoadFile     = uHdr.v1_1.cbFile;
+            pSSM->u.Read.u32LoadCRC     = uHdr.v1_1.u32CRC;
+            pSSM->u.Read.cbGCPhys       = sizeof(RTGCPHYS);
+            pSSM->u.Read.cbGCPtr        = sizeof(RTGCPTR);
+            pSSM->u.Read.fFixedGCPtrSize = false; /* settable */
+            pSSM->u.Read.fStreamCrc32   = false;
+
+            MachineUuidFromHdr  = uHdr.v1_1.MachineUuid;
+            fHaveHostBits       = false;
+        }
+        else if (pSSM->u.Read.uFmtVerMinor == 2)
+        {
+            pSSM->u.Read.cHostBits      = uHdr.v1_2.cHostBits;
+            pSSM->u.Read.u16VerMajor    = uHdr.v1_2.u16VerMajor;
+            pSSM->u.Read.u16VerMinor    = uHdr.v1_2.u16VerMinor;
+            pSSM->u.Read.u32VerBuild    = uHdr.v1_2.u32VerBuild;
+            pSSM->u.Read.u32SvnRev      = uHdr.v1_2.u32SvnRev;
+            pSSM->u.Read.cbLoadFile     = uHdr.v1_2.cbFile;
+            pSSM->u.Read.u32LoadCRC     = uHdr.v1_2.u32CRC;
+            pSSM->u.Read.cbGCPhys       = uHdr.v1_2.cbGCPhys;
+            pSSM->u.Read.cbGCPtr        = uHdr.v1_2.cbGCPtr;
+            pSSM->u.Read.fFixedGCPtrSize = true;
+            pSSM->u.Read.fStreamCrc32   = false;
+
+            MachineUuidFromHdr  = uHdr.v1_2.MachineUuid;
+            fHaveVersion        = true;
+        }
+        else
+            AssertFailedReturn(VERR_SSM_IPE_1);
+
+        /*
+         * The MachineUuid must be NULL (was never used).
+         */
+        if (!RTUuidIsNull(&MachineUuidFromHdr))
+        {
+            LogRel(("SSM: The UUID of the saved state doesn't match the running VM.\n"));
+            return VERR_SMM_INTEGRITY_MACHINE;
+        }
+
+        /*
+         * Verify the file size.
+         */
+        uint64_t cbFile = ssmR3StrmGetSize(&pSSM->Strm);
+        if (cbFile != pSSM->u.Read.cbLoadFile)
+        {
+            LogRel(("SSM: File size mismatch. hdr.cbFile=%lld actual %lld\n", pSSM->u.Read.cbLoadFile, cbFile));
+            return VERR_SSM_INTEGRITY_SIZE;
+        }
+
+        /*
+         * Validate the header info we've set in the handle.
+         */
+        rc = ssmR3ValidateHeaderInfo(pSSM, fHaveHostBits, fHaveVersion);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /*
+         * Verify the checksum if requested.
+         *
+         * Note! The checksum is not actually generated for the whole file,
+         *       this is of course a bug in the v1.x code that we cannot do
+         *       anything about.
+         */
+        if (    fChecksumIt
+            ||  fChecksumOnRead)
+        {
+            uint32_t u32CRC;
+            rc = ssmR3CalcChecksum(&pSSM->Strm,
+                                   RT_UOFFSETOF(SSMFILEHDRV11, u32CRC) + sizeof(uHdr.v1_1.u32CRC),
+                                   cbFile - pSSM->u.Read.cbFileHdr,
+                                   &u32CRC);
+            if (RT_FAILURE(rc))
+                return rc;
+            if (u32CRC != pSSM->u.Read.u32LoadCRC)
+            {
+                LogRel(("SSM: Invalid CRC! Calculated %#010x, in header %#010x\n", u32CRC, pSSM->u.Read.u32LoadCRC));
+                return VERR_SSM_INTEGRITY_CRC;
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Open a saved state for reading.
+ *
+ * The file will be positioned at the first data unit upon successful return.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pszFilename         The filename. NULL if pStreamOps is used.
+ * @param   pStreamOps          The stream method table. NULL if pszFilename is
+ *                              used.
+ * @param   pvUser              The user argument to the stream methods.
+ * @param   fChecksumIt         Check the checksum for the entire file.
+ * @param   fChecksumOnRead     Whether to validate the checksum while reading
+ *                              the stream instead of up front. If not possible,
+ *                              verify the checksum up front.
+ * @param   pSSM                Pointer to the handle structure. This will be
+ *                              completely initialized on success.
+ * @param   cBuffers            The number of stream buffers.
+ */
+static int ssmR3OpenFile(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvUser,
+                         bool fChecksumIt, bool fChecksumOnRead, uint32_t cBuffers, PSSMHANDLE pSSM)
+{
+    /*
+     * Initialize the handle.
+     */
+    pSSM->pVM                   = pVM;
+    pSSM->enmOp                 = SSMSTATE_INVALID;
+    pSSM->enmAfter              = SSMAFTER_INVALID;
+    pSSM->fCancelled            = SSMHANDLE_OK;
+    pSSM->rc                    = VINF_SUCCESS;
+    pSSM->cbUnitLeftV1          = 0;
+    pSSM->offUnit               = UINT64_MAX;
+    pSSM->offUnitUser           = UINT64_MAX;
+    pSSM->fLiveSave             = false;
+    pSSM->pfnProgress           = NULL;
+    pSSM->pvUser                = NULL;
+    pSSM->uPercent              = 0;
+    pSSM->offEstProgress        = 0;
+    pSSM->cbEstTotal            = 0;
+    pSSM->offEst                = 0;
+    pSSM->offEstUnitEnd         = 0;
+    pSSM->uPercentLive          = 0;
+    pSSM->uPercentPrepare       = 5;
+    pSSM->uPercentDone          = 2;
+    pSSM->uReportedLivePercent  = 0;
+    pSSM->pszFilename           = pszFilename;
+
+    pSSM->u.Read.pZipDecompV1   = NULL;
+    pSSM->u.Read.uFmtVerMajor   = UINT32_MAX;
+    pSSM->u.Read.uFmtVerMinor   = UINT32_MAX;
+    pSSM->u.Read.cbFileHdr      = UINT32_MAX;
+    pSSM->u.Read.cbGCPhys       = UINT8_MAX;
+    pSSM->u.Read.cbGCPtr        = UINT8_MAX;
+    pSSM->u.Read.fFixedGCPtrSize= false;
+    pSSM->u.Read.fIsHostMsc32   = SSM_HOST_IS_MSC_32;
+    RT_ZERO(pSSM->u.Read.szHostOSAndArch);
+    pSSM->u.Read.u16VerMajor    = UINT16_MAX;
+    pSSM->u.Read.u16VerMinor    = UINT16_MAX;
+    pSSM->u.Read.u32VerBuild    = UINT32_MAX;
+    pSSM->u.Read.u32SvnRev      = UINT32_MAX;
+    pSSM->u.Read.cHostBits      = UINT8_MAX;
+    pSSM->u.Read.cbLoadFile     = UINT64_MAX;
+
+    pSSM->u.Read.cbRecLeft      = 0;
+    pSSM->u.Read.cbDataBuffer   = 0;
+    pSSM->u.Read.offDataBuffer  = 0;
+    pSSM->u.Read.fEndOfData     = 0;
+    pSSM->u.Read.u8TypeAndFlags = 0;
+
+    pSSM->u.Read.pCurUnit       = NULL;
+    pSSM->u.Read.uCurUnitVer    = UINT32_MAX;
+    pSSM->u.Read.uCurUnitPass   = 0;
+    pSSM->u.Read.fHaveSetError  = false;
+
+    /*
+     * Try open and validate the file.
+     */
+    int rc;
+    if (pStreamOps)
+        rc = ssmR3StrmInit(&pSSM->Strm, pStreamOps, pvUser, false /*fWrite*/, fChecksumOnRead, cBuffers);
+    else
+        rc = ssmR3StrmOpenFile(&pSSM->Strm, pszFilename, false /*fWrite*/, fChecksumOnRead, cBuffers);
+    if (RT_SUCCESS(rc))
+    {
+        rc = ssmR3HeaderAndValidate(pSSM, fChecksumIt, fChecksumOnRead);
+        if (RT_SUCCESS(rc))
+            return rc;
+
+        /* failure path */
+        ssmR3StrmClose(&pSSM->Strm, pSSM->rc == VERR_SSM_CANCELLED);
+    }
+    else
+        Log(("SSM: Failed to open save state file '%s', rc=%Rrc.\n",  pszFilename, rc));
+    return rc;
+}
+
+
+/**
+ * Verifies the directory.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pDir        The full directory.
+ * @param   cbDir       The size of the directory.
+ * @param   offDir      The directory stream offset.
+ * @param   cDirEntries The directory entry count from the footer.
+ * @param   cbHdr       The header size.
+ * @param   uSvnRev     The SVN revision that saved the state. Bug detection.
+ */
+static int ssmR3ValidateDirectory(PSSMFILEDIR pDir, size_t cbDir, uint64_t offDir, uint32_t cDirEntries,
+                                  uint32_t cbHdr, uint32_t uSvnRev)
+{
+    AssertLogRelReturn(!memcmp(pDir->szMagic, SSMFILEDIR_MAGIC, sizeof(pDir->szMagic)), VERR_SSM_INTEGRITY_DIR_MAGIC);
+    SSM_CHECK_CRC32_RET(pDir, cbDir, ("Bad directory CRC: %08x, actual %08x\n", u32CRC, u32ActualCRC));
+    AssertLogRelMsgReturn(pDir->cEntries == cDirEntries,
+                          ("Bad directory entry count: %#x, expected %#x (from the footer)\n", pDir->cEntries, cDirEntries),
+                           VERR_SSM_INTEGRITY_DIR);
+    AssertLogRelReturn(RT_UOFFSETOF_DYN(SSMFILEDIR, aEntries[pDir->cEntries]) == cbDir, VERR_SSM_INTEGRITY_DIR);
+
+    for (uint32_t i = 0; i < pDir->cEntries; i++)
+    {
+        AssertLogRelMsgReturn(  (   pDir->aEntries[i].off >= cbHdr
+                                 && pDir->aEntries[i].off <  offDir)
+                              || (   pDir->aEntries[i].off == 0 /* bug in unreleased code */
+                                  && uSvnRev < 53365),
+                              ("off=%#llx cbHdr=%#x offDir=%#llx\n", pDir->aEntries[i].off, cbHdr, offDir),
+                              VERR_SSM_INTEGRITY_DIR);
+    }
+    return VINF_SUCCESS;
+}
+
+#ifndef SSM_STANDALONE
+
+/**
+ * LogRel the unit content.
+ *
+ * @param   pSSM                The save state handle.
+ * @param   pUnitHdr            The unit head (for cbName).
+ * @param   offUnit             The offset of the unit header.
+ * @param   offStart            Where to start.
+ * @param   offEnd              Where to end.
+ */
+static void ssmR3StrmLogUnitContent(PSSMHANDLE pSSM, SSMFILEUNITHDRV2 const *pUnitHdr, uint64_t offUnit,
+                                    uint64_t offStart, uint64_t offEnd)
+{
+    /*
+     * Stop the I/O thread (if present).
+     */
+    ssmR3StrmStopIoThread(&pSSM->Strm);
+
+    /*
+     * Save the current status, resetting it so we can read + log the unit bytes.
+     */
+    int rcSaved = pSSM->rc;
+    pSSM->rc = VINF_SUCCESS;
+
+    /*
+     * Reverse back to the start of the unit if we can.
+     */
+    uint32_t cbUnitHdr = RT_UOFFSETOF_DYN(SSMFILEUNITHDRV2, szName[pUnitHdr->cbName]);
+    int rc = ssmR3StrmSeek(&pSSM->Strm, offUnit/* + cbUnitHdr*/, RTFILE_SEEK_BEGIN, pUnitHdr->u32CurStreamCRC);
+    if (RT_SUCCESS(rc))
+    {
+        SSMFILEUNITHDRV2 UnitHdr2;
+        rc = ssmR3StrmRead(&pSSM->Strm, &UnitHdr2, cbUnitHdr);
+        if (   RT_SUCCESS(rc)
+            && memcmp(&UnitHdr2, pUnitHdr, cbUnitHdr) == 0)
+        {
+            pSSM->u.Read.cbDataBuffer = 0; /* avoid assertions */
+            pSSM->u.Read.cbRecLeft    = 0;
+            ssmR3DataReadBeginV2(pSSM);
+
+            /*
+             * Read the unit, dumping the requested bits.
+             */
+            uint8_t     cbLine = 0;
+            uint8_t     abLine[16];
+            uint64_t    offCur = 0;
+            offStart &= ~(uint64_t)(sizeof(abLine) - 1);
+            Assert(offStart < offEnd);
+            LogRel(("SSM: Unit '%s' contents:\n", pUnitHdr->szName));
+
+            do
+            {
+                /*
+                 * Read the next 16 bytes into abLine.  We have to take some care to
+                 * get all the bytes in the unit, since we don't really know its size.
+                 */
+                while (   cbLine < sizeof(abLine)
+                       && !pSSM->u.Read.fEndOfData
+                       && RT_SUCCESS(pSSM->rc))
+                {
+                    uint32_t cbToRead = sizeof(abLine) - cbLine;
+                    if (cbToRead > 1)
+                    {
+                        int32_t cbInBuffer = pSSM->u.Read.cbDataBuffer - pSSM->u.Read.offDataBuffer;
+                        if ((int32_t)cbToRead > cbInBuffer)
+                        {
+                            if (cbInBuffer > 0)
+                                cbToRead = cbInBuffer;
+                            else if (pSSM->u.Read.cbRecLeft)
+                                cbToRead = 1;
+                            else
+                            {
+                                rc = ssmR3DataReadRecHdrV2(pSSM);
+                                if (RT_FAILURE(rc))
+                                {
+                                    pSSM->rc = rc;
+                                    break;
+                                }
+                                if (pSSM->u.Read.fEndOfData)
+                                    break;
+                            }
+                        }
+                    }
+                    rc = ssmR3DataRead(pSSM, &abLine[cbLine], cbToRead);
+                    if (RT_SUCCESS(rc))
+                        cbLine += cbToRead;
+                    else
+                        break;
+                }
+
+                /*
+                 * Display the bytes if in the requested range.
+                 */
+                if (   offCur >= offStart
+                    && offCur <= offEnd)
+                {
+                    char    szLine[132];
+                    char   *pchDst = szLine;
+                    uint8_t offSrc = 0;
+                    while (offSrc < cbLine)
+                    {
+                        static char const s_szHex[17] = "0123456789abcdef";
+                        uint8_t const b = abLine[offSrc++];
+                        *pchDst++ = s_szHex[b >> 4];
+                        *pchDst++ = s_szHex[b & 0xf];
+                        *pchDst++ = offSrc != 8 ? ' ' : '-';
+                    }
+                    while (offSrc < sizeof(abLine))
+                    {
+                        *pchDst++ = ' ';
+                        *pchDst++ = ' ';
+                        *pchDst++ = offSrc != 7 ? ' ' : '-';
+                        offSrc++;
+                    }
+                    *pchDst++ = ' ';
+
+                    offSrc = 0;
+                    while (offSrc < cbLine)
+                    {
+                        char const ch = (int8_t)abLine[offSrc++];
+                        if (ch < 0x20 || ch >= 0x7f)
+                            *pchDst++ = '.';
+                        else
+                            *pchDst++ = ch;
+                    }
+                    *pchDst = '\0';
+                    Assert((uintptr_t)(pchDst - &szLine[0]) < sizeof(szLine));
+                    Assert(strchr(szLine, '\0') == pchDst);
+
+                    LogRel(("%#010llx: %s\n", offCur, szLine));
+                }
+                offCur += cbLine;
+                cbLine = 0;
+            } while (   !pSSM->u.Read.fEndOfData
+                     && RT_SUCCESS(pSSM->rc));
+            LogRel(("SSM: offCur=%#llx fEndOfData=%d (rc=%Rrc)\n", offCur, pSSM->u.Read.fEndOfData, rc));
+        }
+        else if (RT_SUCCESS(rc))
+            LogRel(("SSM: Cannot dump unit - mismatching unit head\n"));
+        else
+            LogRel(("SSM: Cannot dump unit - unit header read error: %Rrc\n", rc));
+    }
+    else
+        LogRel(("SSM: Cannot dump unit - ssmR3StrmSeek error: %Rrc\n", rc));
+
+    pSSM->rc = rcSaved;
+}
+
+
+/**
+ * Find a data unit by name.
+ *
+ * @returns Pointer to the unit.
+ * @returns NULL if not found.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The data unit instance id.
+ */
+static PSSMUNIT ssmR3Find(PVM pVM, const char *pszName, uint32_t uInstance)
+{
+    size_t   cchName = strlen(pszName);
+    PSSMUNIT pUnit = pVM->ssm.s.pHead;
+    while (     pUnit
+           &&   (   pUnit->u32Instance != uInstance
+                 || pUnit->cchName != cchName
+                 || memcmp(pUnit->szName, pszName, cchName)))
+        pUnit = pUnit->pNext;
+    return pUnit;
+}
+
+
+/**
+ * Executes the loading of a V1.X file.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3LoadExecV1(PVM pVM, PSSMHANDLE pSSM)
+{
+    int     rc;
+    char   *pszName = NULL;
+    size_t  cchName = 0;
+    pSSM->enmOp = SSMSTATE_LOAD_EXEC;
+    for (;;)
+    {
+        /*
+         * Save the current file position and read the data unit header.
+         */
+        uint64_t         offUnit = ssmR3StrmTell(&pSSM->Strm);
+        SSMFILEUNITHDRV1 UnitHdr;
+        rc = ssmR3StrmRead(&pSSM->Strm, &UnitHdr, RT_UOFFSETOF(SSMFILEUNITHDRV1, szName));
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Check the magic and see if it's valid and whether it is a end header or not.
+             */
+            if (memcmp(&UnitHdr.achMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(SSMFILEUNITHDR_MAGIC)))
+            {
+                if (!memcmp(&UnitHdr.achMagic[0], SSMFILEUNITHDR_END, sizeof(SSMFILEUNITHDR_END)))
+                {
+                    Log(("SSM: EndOfFile: offset %#9llx size %9d\n", offUnit, UnitHdr.cbUnit));
+                    /* Complete the progress bar (pending 99% afterwards). */
+                    ssmR3ProgressByByte(pSSM, pSSM->cbEstTotal - pSSM->offEst);
+                    break;
+                }
+                LogRel(("SSM: Invalid unit magic at offset %#llx (%lld), '%.*s'!\n",
+                        offUnit, offUnit, sizeof(UnitHdr.achMagic) - 1, &UnitHdr.achMagic[0]));
+                rc = VERR_SSM_INTEGRITY_UNIT_MAGIC;
+                break;
+            }
+
+            /*
+             * Read the name.
+             * Adjust the name buffer first.
+             */
+            if (cchName < UnitHdr.cchName)
+            {
+                if (pszName)
+                    RTMemTmpFree(pszName);
+                cchName = RT_ALIGN_Z(UnitHdr.cchName, 64);
+                pszName = (char *)RTMemTmpAlloc(cchName);
+            }
+            if (pszName)
+            {
+                rc = ssmR3StrmRead(&pSSM->Strm, pszName, UnitHdr.cchName);
+                if (RT_SUCCESS(rc))
+                {
+                    if (pszName[UnitHdr.cchName - 1])
+                    {
+                        LogRel(("SSM: Unit name '%.*s' was not properly terminated.\n", UnitHdr.cchName, pszName));
+                        rc = VERR_SSM_INTEGRITY_UNIT;
+                        break;
+                    }
+                    Log(("SSM: Data unit: offset %#9llx size %9lld '%s'\n", offUnit, UnitHdr.cbUnit, pszName));
+
+                    /*
+                     * Find the data unit in our internal table.
+                     */
+                    PSSMUNIT pUnit = ssmR3Find(pVM, pszName, UnitHdr.u32Instance);
+                    if (pUnit)
+                    {
+                        /*
+                         * Call the execute handler.
+                         */
+                        pSSM->cbUnitLeftV1 = UnitHdr.cbUnit - RT_UOFFSETOF_DYN(SSMFILEUNITHDRV1, szName[UnitHdr.cchName]);
+                        pSSM->offUnit      = 0;
+                        pSSM->offUnitUser  = 0;
+                        pSSM->u.Read.uCurUnitVer  = UnitHdr.u32Version;
+                        pSSM->u.Read.uCurUnitPass = SSM_PASS_FINAL;
+                        pSSM->u.Read.pCurUnit     = pUnit;
+                        if (!pUnit->u.Common.pfnLoadExec)
+                        {
+                            LogRel(("SSM: No load exec callback for unit '%s'!\n", pszName));
+                            pSSM->rc = rc = VERR_SSM_NO_LOAD_EXEC;
+                            break;
+                        }
+                        ssmR3UnitCritSectEnter(pUnit);
+                        switch (pUnit->enmType)
+                        {
+                            case SSMUNITTYPE_DEV:
+                                rc = pUnit->u.Dev.pfnLoadExec(pUnit->u.Dev.pDevIns, pSSM, UnitHdr.u32Version, SSM_PASS_FINAL);
+                                break;
+                            case SSMUNITTYPE_DRV:
+                                rc = pUnit->u.Drv.pfnLoadExec(pUnit->u.Drv.pDrvIns, pSSM, UnitHdr.u32Version, SSM_PASS_FINAL);
+                                break;
+                            case SSMUNITTYPE_USB:
+                                rc = pUnit->u.Usb.pfnLoadExec(pUnit->u.Usb.pUsbIns, pSSM, UnitHdr.u32Version, SSM_PASS_FINAL);
+                                break;
+                            case SSMUNITTYPE_INTERNAL:
+                                rc = pUnit->u.Internal.pfnLoadExec(pVM, pSSM, UnitHdr.u32Version, SSM_PASS_FINAL);
+                                break;
+                            case SSMUNITTYPE_EXTERNAL:
+                                rc = pUnit->u.External.pfnLoadExec(pSSM, pUnit->u.External.pvUser, UnitHdr.u32Version, SSM_PASS_FINAL);
+                                break;
+                            default:
+                                rc = VERR_SSM_IPE_1;
+                                break;
+                        }
+                        ssmR3UnitCritSectLeave(pUnit);
+                        pUnit->fCalled = true;
+                        if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
+                            pSSM->rc = rc;
+
+                        /*
+                         * Close the reader stream.
+                         */
+                        rc = ssmR3DataReadFinishV1(pSSM);
+                        if (RT_SUCCESS(rc))
+                        {
+                            /*
+                             * Now, we'll check the current position to see if all, or
+                             * more than all, the data was read.
+                             *
+                             * Note! Because of buffering / compression we'll only see the
+                             * really bad ones here.
+                             */
+                            uint64_t off = ssmR3StrmTell(&pSSM->Strm);
+                            int64_t i64Diff = off - (offUnit + UnitHdr.cbUnit);
+                            if (i64Diff < 0)
+                            {
+                                Log(("SSM: Unit '%s' left %lld bytes unread!\n", pszName, -i64Diff));
+                                rc = ssmR3StrmSkipTo(&pSSM->Strm, offUnit + UnitHdr.cbUnit);
+                                ssmR3ProgressByByte(pSSM, offUnit + UnitHdr.cbUnit - pSSM->offEst);
+                            }
+                            else if (i64Diff > 0)
+                            {
+                                LogRel(("SSM: Unit '%s' read %lld bytes too much!\n", pszName, i64Diff));
+                                if (!ASMAtomicXchgBool(&pSSM->u.Read.fHaveSetError, true))
+                                    rc = VMSetError(pVM, VERR_SSM_LOADED_TOO_MUCH, RT_SRC_POS,
+                                                    N_("Unit '%s' read %lld bytes too much"), pszName, i64Diff);
+                                break;
+                            }
+
+                            pSSM->offUnit     = UINT64_MAX;
+                            pSSM->offUnitUser = UINT64_MAX;
+                        }
+                        else
+                        {
+                            LogRel(("SSM: Load exec failed for '%s' instance #%u ! (version %u)\n",
+                                    pszName, UnitHdr.u32Instance, UnitHdr.u32Version));
+                            if (!ASMAtomicXchgBool(&pSSM->u.Read.fHaveSetError, true))
+                            {
+                                if (rc == VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION)
+                                    VMSetError(pVM, rc, RT_SRC_POS, N_("Unsupported version %u of data unit '%s' (instance #%u)"),
+                                               UnitHdr.u32Version, UnitHdr.szName, UnitHdr.u32Instance);
+                                else
+                                    VMSetError(pVM, rc, RT_SRC_POS, N_("Load exec failed for '%s' instance #%u (version %u)"),
+                                               pszName, UnitHdr.u32Instance, UnitHdr.u32Version);
+                            }
+                            break;
+                        }
+
+                        pSSM->u.Read.pCurUnit       = NULL;
+                        pSSM->u.Read.uCurUnitVer    = UINT32_MAX;
+                        pSSM->u.Read.uCurUnitPass   = 0;
+                    }
+                    else
+                    {
+                        /*
+                         * SSM unit wasn't found - ignore this when loading for the debugger.
+                         */
+                        LogRel(("SSM: Found no handler for unit '%s'!\n", pszName));
+                        rc = VERR_SSM_INTEGRITY_UNIT_NOT_FOUND;
+                        if (pSSM->enmAfter != SSMAFTER_DEBUG_IT)
+                            break;
+                        rc = ssmR3StrmSkipTo(&pSSM->Strm, offUnit + UnitHdr.cbUnit);
+                    }
+                }
+            }
+            else
+                rc = VERR_NO_TMP_MEMORY;
+        }
+
+        /*
+         * I/O errors ends up here (yea, I know, very nice programming).
+         */
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: I/O error. rc=%Rrc\n", rc));
+            break;
+        }
+
+        /*
+         * Check for cancellation.
+         */
+        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED))
+        {
+            LogRel(("SSM: Cancelled!n"));
+            rc = pSSM->rc;
+            if (RT_SUCCESS(pSSM->rc))
+                pSSM->rc = rc = VERR_SSM_CANCELLED;
+            break;
+        }
+    }
+
+    RTMemTmpFree(pszName);
+    return rc;
+}
+
+
+/**
+ * Reads and verifies the directory and footer.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3LoadDirectoryAndFooter(PSSMHANDLE pSSM)
+{
+    /*
+     * The directory.
+     *
+     * Get the header containing the number of entries first.  Then read the
+     * entries and pass the combined block to the validation function.
+     */
+    uint64_t        off      = ssmR3StrmTell(&pSSM->Strm);
+    size_t const    cbDirHdr = RT_UOFFSETOF(SSMFILEDIR, aEntries);
+    SSMFILEDIR      DirHdr;
+    int rc = ssmR3StrmRead(&pSSM->Strm, &DirHdr, cbDirHdr);
+    if (RT_FAILURE(rc))
+        return rc;
+    AssertLogRelMsgReturn(!memcmp(DirHdr.szMagic, SSMFILEDIR_MAGIC, sizeof(DirHdr.szMagic)),
+                          ("Invalid directory magic at %#llx (%lld): %.*Rhxs\n", off, off, sizeof(DirHdr.szMagic), DirHdr.szMagic),
+                          VERR_SSM_INTEGRITY_DIR_MAGIC);
+    AssertLogRelMsgReturn(DirHdr.cEntries < _64K,
+                          ("Too many directory entries at %#llx (%lld): %#x\n", off, off, DirHdr.cEntries),
+                          VERR_SSM_INTEGRITY_DIR);
+
+    size_t      cbDir = RT_UOFFSETOF_DYN(SSMFILEDIR, aEntries[DirHdr.cEntries]);
+    PSSMFILEDIR pDir  = (PSSMFILEDIR)RTMemTmpAlloc(cbDir);
+    if (!pDir)
+        return VERR_NO_TMP_MEMORY;
+    memcpy(pDir, &DirHdr, cbDirHdr);
+    rc = ssmR3StrmRead(&pSSM->Strm, (uint8_t *)pDir + cbDirHdr, cbDir - cbDirHdr);
+    if (RT_SUCCESS(rc))
+        rc = ssmR3ValidateDirectory(pDir, cbDir, off, DirHdr.cEntries, pSSM->u.Read.cbFileHdr, pSSM->u.Read.u32SvnRev);
+    RTMemTmpFree(pDir);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Read and validate the footer.
+     */
+    off = ssmR3StrmTell(&pSSM->Strm);
+    uint32_t    u32StreamCRC = ssmR3StrmFinalCRC(&pSSM->Strm);
+    SSMFILEFTR  Footer;
+    rc = ssmR3StrmRead(&pSSM->Strm, &Footer, sizeof(Footer));
+    if (RT_FAILURE(rc))
+        return rc;
+    return ssmR3ValidateFooter(&Footer, off, DirHdr.cEntries, pSSM->u.Read.fStreamCrc32, u32StreamCRC);
+}
+
+
+/**
+ * Executes the loading of a V2.X file.
+ *
+ * @returns VBox status code.  May or may not set pSSM->rc, the returned
+ *          status code is ALWAYS the more accurate of the two.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3LoadExecV2(PVM pVM, PSSMHANDLE pSSM)
+{
+    pSSM->enmOp = SSMSTATE_LOAD_EXEC;
+    for (;;)
+    {
+        /*
+         * Read the unit header and check its integrity.
+         */
+        uint64_t            offUnit         = ssmR3StrmTell(&pSSM->Strm);
+        uint32_t            u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
+        SSMFILEUNITHDRV2    UnitHdr;
+        int rc = ssmR3StrmRead(&pSSM->Strm, &UnitHdr, RT_UOFFSETOF(SSMFILEUNITHDRV2, szName));
+        if (RT_FAILURE(rc))
+            return rc;
+        if (RT_UNLIKELY(    memcmp(&UnitHdr.szMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic))
+                        &&  memcmp(&UnitHdr.szMagic[0], SSMFILEUNITHDR_END,   sizeof(UnitHdr.szMagic))))
+        {
+            LogRel(("SSM: Unit at %#llx (%lld): Invalid unit magic: %.*Rhxs!\n",
+                    offUnit, offUnit, sizeof(UnitHdr.szMagic) - 1, &UnitHdr.szMagic[0]));
+            pSSM->u.Read.fHaveSetError = true;
+            return VMSetError(pVM, VERR_SSM_INTEGRITY_UNIT_MAGIC, RT_SRC_POS,
+                              N_("Unit at %#llx (%lld): Invalid unit magic"), offUnit, offUnit);
+        }
+        if (UnitHdr.cbName)
+        {
+            AssertLogRelMsgReturn(UnitHdr.cbName <= sizeof(UnitHdr.szName),
+                                  ("Unit at %#llx (%lld): UnitHdr.cbName=%u > %u\n",
+                                   offUnit, offUnit, UnitHdr.cbName, sizeof(UnitHdr.szName)),
+                                  VERR_SSM_INTEGRITY_UNIT);
+            rc = ssmR3StrmRead(&pSSM->Strm, &UnitHdr.szName[0], UnitHdr.cbName);
+            if (RT_FAILURE(rc))
+                return rc;
+            AssertLogRelMsgReturn(!UnitHdr.szName[UnitHdr.cbName - 1],
+                                  ("Unit at %#llx (%lld): Name %.*Rhxs was not properly terminated.\n",
+                                   offUnit, offUnit, UnitHdr.cbName, UnitHdr.szName),
+                                  VERR_SSM_INTEGRITY_UNIT);
+        }
+        SSM_CHECK_CRC32_RET(&UnitHdr, RT_UOFFSETOF_DYN(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]),
+                            ("Unit at %#llx (%lld): CRC mismatch: %08x, correct is %08x\n", offUnit, offUnit, u32CRC, u32ActualCRC));
+        AssertLogRelMsgReturn(UnitHdr.offStream == offUnit,
+                              ("Unit at %#llx (%lld): offStream=%#llx, expected %#llx\n", offUnit, offUnit, UnitHdr.offStream, offUnit),
+                              VERR_SSM_INTEGRITY_UNIT);
+        AssertLogRelMsgReturn(UnitHdr.u32CurStreamCRC == u32CurStreamCRC || !pSSM->Strm.fChecksummed,
+                              ("Unit at %#llx (%lld): Stream CRC mismatch: %08x, correct is %08x\n", offUnit, offUnit, UnitHdr.u32CurStreamCRC, u32CurStreamCRC),
+                              VERR_SSM_INTEGRITY_UNIT);
+        AssertLogRelMsgReturn(!UnitHdr.fFlags, ("Unit at %#llx (%lld): fFlags=%08x\n", offUnit, offUnit, UnitHdr.fFlags),
+                              VERR_SSM_INTEGRITY_UNIT);
+        if (!memcmp(&UnitHdr.szMagic[0], SSMFILEUNITHDR_END,   sizeof(UnitHdr.szMagic)))
+        {
+            AssertLogRelMsgReturn(   UnitHdr.cbName       == 0
+                                  && UnitHdr.u32Instance  == 0
+                                  && UnitHdr.u32Version   == 0
+                                  && UnitHdr.u32Pass      == SSM_PASS_FINAL,
+                                  ("Unit at %#llx (%lld): Malformed END unit\n", offUnit, offUnit),
+                                  VERR_SSM_INTEGRITY_UNIT);
+
+            /*
+             * Complete the progress bar (pending 99% afterwards) and RETURN.
+             */
+            Log(("SSM: Unit at %#9llx: END UNIT\n", offUnit));
+            ssmR3ProgressByByte(pSSM, pSSM->cbEstTotal - pSSM->offEst);
+            return ssmR3LoadDirectoryAndFooter(pSSM);
+        }
+        AssertLogRelMsgReturn(UnitHdr.cbName > 1, ("Unit at %#llx (%lld): No name\n", offUnit, offUnit), VERR_SSM_INTEGRITY);
+
+        Log(("SSM: Unit at %#9llx: '%s', instance %u, pass %#x, version %u\n",
+             offUnit, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass, UnitHdr.u32Version));
+
+        /*
+         * Find the data unit in our internal table.
+         */
+        PSSMUNIT pUnit = ssmR3Find(pVM, UnitHdr.szName, UnitHdr.u32Instance);
+        if (pUnit)
+        {
+            /*
+             * Call the execute handler.
+             */
+            AssertLogRelMsgReturn(pUnit->u.Common.pfnLoadExec,
+                                  ("SSM: No load exec callback for unit '%s'!\n", UnitHdr.szName),
+                                  VERR_SSM_NO_LOAD_EXEC);
+            pSSM->u.Read.uCurUnitVer  = UnitHdr.u32Version;
+            pSSM->u.Read.uCurUnitPass = UnitHdr.u32Pass;
+            pSSM->u.Read.pCurUnit     = pUnit;
+            ssmR3DataReadBeginV2(pSSM);
+            ssmR3UnitCritSectEnter(pUnit);
+            switch (pUnit->enmType)
+            {
+                case SSMUNITTYPE_DEV:
+                    rc = pUnit->u.Dev.pfnLoadExec(pUnit->u.Dev.pDevIns, pSSM, UnitHdr.u32Version, UnitHdr.u32Pass);
+                    break;
+                case SSMUNITTYPE_DRV:
+                    rc = pUnit->u.Drv.pfnLoadExec(pUnit->u.Drv.pDrvIns, pSSM, UnitHdr.u32Version, UnitHdr.u32Pass);
+                    break;
+                case SSMUNITTYPE_USB:
+                    rc = pUnit->u.Usb.pfnLoadExec(pUnit->u.Usb.pUsbIns, pSSM, UnitHdr.u32Version, UnitHdr.u32Pass);
+                    break;
+                case SSMUNITTYPE_INTERNAL:
+                    rc = pUnit->u.Internal.pfnLoadExec(pVM, pSSM, UnitHdr.u32Version, UnitHdr.u32Pass);
+                    break;
+                case SSMUNITTYPE_EXTERNAL:
+                    rc = pUnit->u.External.pfnLoadExec(pSSM, pUnit->u.External.pvUser, UnitHdr.u32Version, UnitHdr.u32Pass);
+                    break;
+                default:
+                    rc = VERR_SSM_IPE_1;
+                    break;
+            }
+            ssmR3UnitCritSectLeave(pUnit);
+            pUnit->fCalled = true;
+            if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
+                pSSM->rc = rc;
+            rc = ssmR3DataReadFinishV2(pSSM);
+            if (RT_SUCCESS(rc))
+            {
+                pSSM->offUnit     = UINT64_MAX;
+                pSSM->offUnitUser = UINT64_MAX;
+            }
+            else
+            {
+                LogRel(("SSM: LoadExec failed for '%s' instance #%u (version %u, pass %#x): %Rrc\n",
+                        UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Version, UnitHdr.u32Pass, rc));
+                LogRel(("SSM: Unit at %#llx, current position: offUnit=%#llx offUnitUser=%#llx\n",
+                        offUnit, pSSM->offUnit, pSSM->offUnitUser));
+
+                if (!ASMAtomicXchgBool(&pSSM->u.Read.fHaveSetError, true))
+                {
+                    if (rc == VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION)
+                        rc = VMSetError(pVM, rc, RT_SRC_POS, N_("Unsupported version %u of data unit '%s' (instance #%u, pass %#x)"),
+                                        UnitHdr.u32Version, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass);
+                    else
+                        rc = VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to load unit '%s'"), UnitHdr.szName);
+                }
+
+                /* Try log the unit content, unless it's too big. */
+                if (pSSM->offUnitUser < _512K)
+                    ssmR3StrmLogUnitContent(pSSM, &UnitHdr, offUnit, 0, pSSM->offUnitUser + _16K);
+                else
+                    ssmR3StrmLogUnitContent(pSSM, &UnitHdr, offUnit, pSSM->offUnitUser - _256K, pSSM->offUnitUser + _16K);
+                return rc;
+            }
+        }
+        else
+        {
+            /*
+             * SSM unit wasn't found - ignore this when loading for the debugger.
+             */
+            LogRel(("SSM: Found no handler for unit '%s' instance #%u!\n", UnitHdr.szName, UnitHdr.u32Instance));
+            if (pSSM->enmAfter != SSMAFTER_DEBUG_IT)
+            {
+                pSSM->u.Read.fHaveSetError = true;
+                return VMSetError(pVM, VERR_SSM_INTEGRITY_UNIT_NOT_FOUND, RT_SRC_POS,
+                                  N_("Found no handler for unit '%s' instance #%u"), UnitHdr.szName, UnitHdr.u32Instance);
+            }
+            SSMR3SkipToEndOfUnit(pSSM);
+            ssmR3DataReadFinishV2(pSSM);
+        }
+
+        /*
+         * Check for cancellation.
+         */
+        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED))
+        {
+            LogRel(("SSM: Cancelled!\n"));
+            if (RT_SUCCESS(pSSM->rc))
+                pSSM->rc = VERR_SSM_CANCELLED;
+            return pSSM->rc;
+        }
+    }
+    /* won't get here */
+}
+
+
+
+
+/**
+ * Load VM save operation.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pszFilename     The name of the saved state file. NULL if pStreamOps
+ *                          is used.
+ * @param   pStreamOps      The stream method table. NULL if pszFilename is
+ *                          used.
+ * @param   pvStreamOpsUser The user argument for the stream methods.
+ * @param   enmAfter        What is planned after a successful load operation.
+ *                          Only acceptable values are SSMAFTER_RESUME and SSMAFTER_DEBUG_IT.
+ * @param   pfnProgress     Progress callback. Optional.
+ * @param   pvProgressUser  User argument for the progress callback.
+ *
+ * @thread  EMT
+ */
+VMMR3DECL(int) SSMR3Load(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                         SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser)
+{
+    LogFlow(("SSMR3Load: pszFilename=%p:{%s} pStreamOps=%p pvStreamOpsUser=%p enmAfter=%d pfnProgress=%p pvProgressUser=%p\n",
+             pszFilename, pszFilename, pStreamOps, pvStreamOpsUser, enmAfter, pfnProgress, pvProgressUser));
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(   enmAfter == SSMAFTER_RESUME
+                    || enmAfter == SSMAFTER_TELEPORT
+                    || enmAfter == SSMAFTER_DEBUG_IT,
+                    ("%d\n", enmAfter),
+                    VERR_INVALID_PARAMETER);
+    AssertReturn(!pszFilename != !pStreamOps, VERR_INVALID_PARAMETER);
+    if (pStreamOps)
+    {
+        AssertReturn(pStreamOps->u32Version == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
+        AssertReturn(pStreamOps->u32EndVersion == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
+        AssertReturn(pStreamOps->pfnWrite, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnRead, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnSeek, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnTell, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnSize, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnClose, VERR_INVALID_PARAMETER);
+    }
+
+    /*
+     * Create the handle and open the file.
+     */
+    SSMHANDLE Handle;
+    int rc = ssmR3OpenFile(pVM, pszFilename, pStreamOps, pvStreamOpsUser, false /* fChecksumIt */,
+                           true /* fChecksumOnRead */, 8 /*cBuffers*/, &Handle);
+    if (RT_SUCCESS(rc))
+    {
+        ssmR3StrmStartIoThread(&Handle.Strm);
+        ssmR3SetCancellable(pVM, &Handle, true);
+
+        Handle.enmAfter         = enmAfter;
+        Handle.pfnProgress      = pfnProgress;
+        Handle.pvUser           = pvProgressUser;
+        Handle.uPercentLive     = 0;
+        Handle.uPercentPrepare  = 2;
+        Handle.uPercentDone     = 2;
+
+        if (Handle.u.Read.u16VerMajor)
+            LogRel(("SSM: File header: Format %u.%u, VirtualBox Version %u.%u.%u r%u, %u-bit host, cbGCPhys=%u, cbGCPtr=%u\n",
+                    Handle.u.Read.uFmtVerMajor, Handle.u.Read.uFmtVerMinor,
+                    Handle.u.Read.u16VerMajor, Handle.u.Read.u16VerMinor, Handle.u.Read.u32VerBuild, Handle.u.Read.u32SvnRev,
+                    Handle.u.Read.cHostBits, Handle.u.Read.cbGCPhys, Handle.u.Read.cbGCPtr));
+        else
+            LogRel(("SSM: File header: Format %u.%u, %u-bit host, cbGCPhys=%u, cbGCPtr=%u\n" ,
+                    Handle.u.Read.uFmtVerMajor, Handle.u.Read.uFmtVerMinor,
+                    Handle.u.Read.cHostBits, Handle.u.Read.cbGCPhys, Handle.u.Read.cbGCPtr));
+
+        if (pfnProgress)
+            pfnProgress(pVM->pUVM, Handle.uPercent, pvProgressUser);
+
+        /*
+         * Clear the per unit flags.
+         */
+        PSSMUNIT pUnit;
+        for (pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+            pUnit->fCalled = false;
+
+        /*
+         * Do the prepare run.
+         */
+        Handle.rc = VINF_SUCCESS;
+        Handle.enmOp = SSMSTATE_LOAD_PREP;
+        for (pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+        {
+            if (pUnit->u.Common.pfnLoadPrep)
+            {
+                Handle.u.Read.pCurUnit = pUnit;
+                pUnit->fCalled = true;
+                ssmR3UnitCritSectEnter(pUnit);
+                switch (pUnit->enmType)
+                {
+                    case SSMUNITTYPE_DEV:
+                        rc = pUnit->u.Dev.pfnLoadPrep(pUnit->u.Dev.pDevIns, &Handle);
+                        break;
+                    case SSMUNITTYPE_DRV:
+                        rc = pUnit->u.Drv.pfnLoadPrep(pUnit->u.Drv.pDrvIns, &Handle);
+                        break;
+                    case SSMUNITTYPE_USB:
+                        rc = pUnit->u.Usb.pfnLoadPrep(pUnit->u.Usb.pUsbIns, &Handle);
+                        break;
+                    case SSMUNITTYPE_INTERNAL:
+                        rc = pUnit->u.Internal.pfnLoadPrep(pVM, &Handle);
+                        break;
+                    case SSMUNITTYPE_EXTERNAL:
+                        rc = pUnit->u.External.pfnLoadPrep(&Handle, pUnit->u.External.pvUser);
+                        break;
+                    default:
+                        rc = VERR_SSM_IPE_1;
+                        break;
+                }
+                ssmR3UnitCritSectLeave(pUnit);
+                Handle.u.Read.pCurUnit = NULL;
+                if (RT_FAILURE(rc) && RT_SUCCESS_NP(Handle.rc))
+                    Handle.rc = rc;
+                else
+                    rc = Handle.rc;
+                if (RT_FAILURE(rc))
+                {
+                    LogRel(("SSM: Prepare load failed with rc=%Rrc for data unit '%s.\n", rc, pUnit->szName));
+                    break;
+                }
+            }
+        }
+
+        /* end of prepare % */
+        if (pfnProgress)
+            pfnProgress(pVM->pUVM, Handle.uPercentPrepare - 1, pvProgressUser);
+        Handle.uPercent      = Handle.uPercentPrepare;
+        Handle.cbEstTotal    = Handle.u.Read.cbLoadFile;
+        Handle.offEstUnitEnd = Handle.u.Read.cbLoadFile;
+
+        /*
+         * Do the execute run.
+         */
+        if (RT_SUCCESS(rc))
+        {
+            if (Handle.u.Read.uFmtVerMajor >= 2)
+                rc = ssmR3LoadExecV2(pVM, &Handle);
+            else
+                rc = ssmR3LoadExecV1(pVM, &Handle);
+            Handle.u.Read.pCurUnit       = NULL;
+            Handle.u.Read.uCurUnitVer    = UINT32_MAX;
+            Handle.u.Read.uCurUnitPass   = 0;
+
+            /* (progress should be pending 99% now) */
+            AssertMsg(   Handle.fLiveSave
+                      || RT_FAILURE(rc)
+                      || Handle.uPercent == 101 - Handle.uPercentDone, ("%d\n", Handle.uPercent));
+        }
+
+        /*
+         * Do the done run.
+         */
+        Handle.rc = rc;
+        Handle.enmOp = SSMSTATE_LOAD_DONE;
+        for (pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+        {
+            if (    pUnit->u.Common.pfnLoadDone
+                && (   pUnit->fCalled
+                    || (!pUnit->u.Common.pfnLoadPrep && !pUnit->u.Common.pfnLoadExec)))
+            {
+                Handle.u.Read.pCurUnit = pUnit;
+                int const rcOld = Handle.rc;
+                rc = VINF_SUCCESS;
+                ssmR3UnitCritSectEnter(pUnit);
+                switch (pUnit->enmType)
+                {
+                    case SSMUNITTYPE_DEV:
+                        rc = pUnit->u.Dev.pfnLoadDone(pUnit->u.Dev.pDevIns, &Handle);
+                        break;
+                    case SSMUNITTYPE_DRV:
+                        rc = pUnit->u.Drv.pfnLoadDone(pUnit->u.Drv.pDrvIns, &Handle);
+                        break;
+                    case SSMUNITTYPE_USB:
+                        rc = pUnit->u.Usb.pfnLoadDone(pUnit->u.Usb.pUsbIns, &Handle);
+                        break;
+                    case SSMUNITTYPE_INTERNAL:
+                        rc = pUnit->u.Internal.pfnLoadDone(pVM, &Handle);
+                        break;
+                    case SSMUNITTYPE_EXTERNAL:
+                        rc = pUnit->u.External.pfnLoadDone(&Handle, pUnit->u.External.pvUser);
+                        break;
+                    default:
+                        rc = VERR_SSM_IPE_1;
+                        break;
+                }
+                ssmR3UnitCritSectLeave(pUnit);
+                Handle.u.Read.pCurUnit = NULL;
+                if (RT_SUCCESS(rc) && Handle.rc != rcOld)
+                    rc = Handle.rc;
+                if (RT_FAILURE(rc))
+                {
+                    LogRel(("SSM: LoadDone failed with rc=%Rrc for data unit '%s' instance #%u.\n",
+                            rc, pUnit->szName, pUnit->u32Instance));
+                    if (!ASMAtomicXchgBool(&Handle.u.Read.fHaveSetError, true))
+                        VMSetError(pVM, rc, RT_SRC_POS, N_("LoadDone failed with rc=%Rrc for data unit '%s' instance #%u."),
+                                   rc, pUnit->szName, pUnit->u32Instance);
+                    if (RT_SUCCESS_NP(Handle.rc))
+                        Handle.rc = rc;
+                }
+            }
+        }
+
+        /* progress */
+        if (pfnProgress)
+            pfnProgress(pVM->pUVM, 99, pvProgressUser);
+
+        ssmR3SetCancellable(pVM, &Handle, false);
+        ssmR3StrmClose(&Handle.Strm, Handle.rc == VERR_SSM_CANCELLED);
+        rc = Handle.rc;
+    }
+
+    /*
+     * Done
+     */
+    if (RT_SUCCESS(rc))
+    {
+        /* progress */
+        if (pfnProgress)
+            pfnProgress(pVM->pUVM, 100, pvProgressUser);
+        Log(("SSM: Load of '%s' completed!\n", pszFilename));
+    }
+    return rc;
+}
+
+
+/**
+ * VMSetError wrapper for load errors that inserts the saved state details.
+ *
+ * @returns rc.
+ * @param   pSSM                The saved state handle.
+ * @param   rc                  The status code of the error. Use RT_SRC_POS.
+ * @param   SRC_POS             The source location.
+ * @param   pszFormat           The message format string.
+ * @param   ...                 Variable argument list.
+ */
+VMMR3DECL(int) SSMR3SetLoadError(PSSMHANDLE pSSM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    rc = SSMR3SetLoadErrorV(pSSM, rc, RT_SRC_POS_ARGS, pszFormat, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * VMSetError wrapper for load errors that inserts the saved state details.
+ *
+ * @returns rc.
+ * @param   pSSM                The saved state handle.
+ * @param   rc                  The status code of the error.
+ * @param   SRC_POS             The error location, use RT_SRC_POS.
+ * @param   pszFormat           The message format string.
+ * @param   va                  Variable argument list.
+ */
+VMMR3DECL(int) SSMR3SetLoadErrorV(PSSMHANDLE pSSM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
+{
+    /*
+     * Input validations.
+     */
+    SSM_ASSERT_READABLE_RET(pSSM);
+    AssertPtr(pszFormat);
+    Assert(RT_FAILURE_NP(rc));
+
+    /*
+     * Format the incoming error.
+     */
+    char *pszMsg;
+    RTStrAPrintfV(&pszMsg, pszFormat, va);
+    if (!pszMsg)
+    {
+        VMSetError(pSSM->pVM, VERR_NO_MEMORY, RT_SRC_POS,
+                   N_("SSMR3SetLoadErrorV ran out of memory formatting: %s\n"), pszFormat);
+        return rc;
+    }
+
+    /*
+     * Forward to VMSetError with the additional info.
+     */
+    PSSMUNIT    pUnit       = pSSM->u.Read.pCurUnit;
+    const char *pszName     = pUnit ? pUnit->szName      : "unknown";
+    uint32_t    uInstance   = pUnit ? pUnit->u32Instance : 0;
+    if (   pSSM->enmOp == SSMSTATE_LOAD_EXEC
+        && pSSM->u.Read.uCurUnitPass == SSM_PASS_FINAL)
+        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [ver=%u pass=final]"),
+                        pszName, uInstance, pszMsg, pSSM->u.Read.uCurUnitVer);
+    else if (pSSM->enmOp == SSMSTATE_LOAD_EXEC)
+        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [ver=%u pass=#%u]"),
+                        pszName, uInstance, pszMsg, pSSM->u.Read.uCurUnitVer, pSSM->u.Read.uCurUnitPass);
+    else if (pSSM->enmOp == SSMSTATE_LOAD_PREP)
+        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [prep]"),
+                        pszName, uInstance, pszMsg);
+    else if (pSSM->enmOp == SSMSTATE_LOAD_DONE)
+        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [done]"),
+                        pszName, uInstance, pszMsg);
+    else if (pSSM->enmOp == SSMSTATE_OPEN_READ)
+        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [read]"),
+                        pszName, uInstance, pszMsg);
+    else
+        AssertFailed();
+    pSSM->u.Read.fHaveSetError = true;
+    RTStrFree(pszMsg);
+    return rc;
+}
+
+
+/**
+ * SSMR3SetLoadError wrapper that returns VERR_SSM_LOAD_CONFIG_MISMATCH.
+ *
+ * @returns VERR_SSM_LOAD_CONFIG_MISMATCH.
+ * @param   pSSM                The saved state handle.
+ * @param   SRC_POS             The error location, use RT_SRC_POS.
+ * @param   pszFormat           The message format string.
+ * @param   ...                 Variable argument list.
+ */
+VMMR3DECL(int) SSMR3SetCfgError(PSSMHANDLE pSSM, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    int rc = SSMR3SetLoadErrorV(pSSM, VERR_SSM_LOAD_CONFIG_MISMATCH, RT_SRC_POS_ARGS, pszFormat, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * SSMR3SetLoadError wrapper that returns VERR_SSM_LOAD_CONFIG_MISMATCH.
+ *
+ * @returns VERR_SSM_LOAD_CONFIG_MISMATCH.
+ * @param   pSSM                The saved state handle.
+ * @param   SRC_POS             The error location, use RT_SRC_POS.
+ * @param   pszFormat           The message format string.
+ * @param   va                  Variable argument list.
+ */
+VMMR3DECL(int) SSMR3SetCfgErrorV(PSSMHANDLE pSSM, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
+{
+    return SSMR3SetLoadErrorV(pSSM, VERR_SSM_LOAD_CONFIG_MISMATCH, RT_SRC_POS_ARGS, pszFormat, va);
+}
+
+#endif /* !SSM_STANDALONE */
+
+/**
+ * Validates a file as a validate SSM saved state.
+ *
+ * This will only verify the file format, the format and content of individual
+ * data units are not inspected.
+ *
+ * @returns VINF_SUCCESS if valid.
+ * @returns VBox status code on other failures.
+ *
+ * @param   pszFilename     The path to the file to validate.
+ * @param   fChecksumIt     Whether to checksum the file or not.
+ *
+ * @thread  Any.
+ */
+VMMR3DECL(int) SSMR3ValidateFile(const char *pszFilename, bool fChecksumIt)
+{
+    LogFlow(("SSMR3ValidateFile: pszFilename=%p:{%s} fChecksumIt=%RTbool\n", pszFilename, pszFilename, fChecksumIt));
+
+    /*
+     * Try open the file and validate it.
+     */
+    SSMHANDLE Handle;
+    int rc = ssmR3OpenFile(NULL, pszFilename, NULL /*pStreamOps*/, NULL /*pvUser*/, fChecksumIt,
+                           false /*fChecksumOnRead*/, 1 /*cBuffers*/, &Handle);
+    if (RT_SUCCESS(rc))
+        ssmR3StrmClose(&Handle.Strm, false /*fCancelled*/);
+    else
+        Log(("SSM: Failed to open saved state file '%s', rc=%Rrc.\n",  pszFilename, rc));
+    return rc;
+}
+
+
+/**
+ * Opens a saved state file for reading.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pszFilename     The path to the saved state file.
+ * @param   fFlags          Open flags. Reserved, must be 0.
+ * @param   ppSSM           Where to store the SSM handle.
+ *
+ * @thread  Any.
+ */
+VMMR3DECL(int) SSMR3Open(const char *pszFilename, unsigned fFlags, PSSMHANDLE *ppSSM)
+{
+    LogFlow(("SSMR3Open: pszFilename=%p:{%s} fFlags=%#x ppSSM=%p\n", pszFilename, pszFilename, fFlags, ppSSM));
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(VALID_PTR(pszFilename), ("%p\n", pszFilename), VERR_INVALID_PARAMETER);
+    AssertMsgReturn(!fFlags, ("%#x\n", fFlags), VERR_INVALID_PARAMETER);
+    AssertMsgReturn(VALID_PTR(ppSSM), ("%p\n", ppSSM), VERR_INVALID_PARAMETER);
+
+    /*
+     * Allocate a handle.
+     */
+    PSSMHANDLE pSSM = (PSSMHANDLE)RTMemAllocZ(sizeof(*pSSM));
+    AssertReturn(pSSM, VERR_NO_MEMORY);
+
+    /*
+     * Try open the file and validate it.
+     */
+    int rc = ssmR3OpenFile(NULL, pszFilename, NULL /*pStreamOps*/, NULL /*pvUser*/, false /*fChecksumIt*/,
+                           true /*fChecksumOnRead*/, 1 /*cBuffers*/, pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        pSSM->enmAfter = SSMAFTER_OPENED;
+        pSSM->enmOp    = SSMSTATE_OPEN_READ;
+        *ppSSM = pSSM;
+        LogFlow(("SSMR3Open: returns VINF_SUCCESS *ppSSM=%p\n", *ppSSM));
+        return VINF_SUCCESS;
+    }
+
+    Log(("SSMR3Open: Failed to open saved state file '%s', rc=%Rrc.\n",  pszFilename, rc));
+    RTMemFree(pSSM);
+    return rc;
+
+}
+
+
+/**
+ * Closes a saved state file opened by SSMR3Open().
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSSM            The SSM handle returned by SSMR3Open().
+ *
+ * @thread  Any, but the caller is responsible for serializing calls per handle.
+ */
+VMMR3DECL(int) SSMR3Close(PSSMHANDLE pSSM)
+{
+    LogFlow(("SSMR3Close: pSSM=%p\n", pSSM));
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(VALID_PTR(pSSM), ("%p\n", pSSM), VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pSSM->enmAfter == SSMAFTER_OPENED, ("%d\n", pSSM->enmAfter),VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pSSM->enmOp == SSMSTATE_OPEN_READ, ("%d\n", pSSM->enmOp), VERR_INVALID_PARAMETER);
+    Assert(pSSM->fCancelled == SSMHANDLE_OK);
+
+    /*
+     * Close the stream and free the handle.
+     */
+    int rc = ssmR3StrmClose(&pSSM->Strm, pSSM->rc == VERR_SSM_CANCELLED);
+    if (pSSM->u.Read.pZipDecompV1)
+    {
+        RTZipDecompDestroy(pSSM->u.Read.pZipDecompV1);
+        pSSM->u.Read.pZipDecompV1 = NULL;
+    }
+    RTMemFree(pSSM);
+    return rc;
+}
+
+
+/**
+ * Worker for SSMR3Seek that seeks version 1 saved state files.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The SSM handle.
+ * @param   pszUnit             The unit to seek to.
+ * @param   iInstance           The particular instance we seek.
+ * @param   piVersion           Where to store the unit version number.
+ */
+static int ssmR3FileSeekV1(PSSMHANDLE pSSM, const char *pszUnit, uint32_t iInstance, uint32_t *piVersion)
+{
+    /*
+     * Walk the data units until we find EOF or a match.
+     */
+    size_t              cbUnitNm = strlen(pszUnit) + 1;
+    AssertLogRelReturn(cbUnitNm <= SSM_MAX_NAME_SIZE, VERR_SSM_UNIT_NOT_FOUND);
+    char                szName[SSM_MAX_NAME_SIZE];
+    SSMFILEUNITHDRV1    UnitHdr;
+    for (RTFOFF off = pSSM->u.Read.cbFileHdr; ; off += UnitHdr.cbUnit)
+    {
+        /*
+         * Read the unit header and verify it.
+         */
+        int rc = ssmR3StrmPeekAt(&pSSM->Strm, off, &UnitHdr, RT_UOFFSETOF(SSMFILEUNITHDRV1, szName), NULL);
+        AssertRCReturn(rc, rc);
+        if (!memcmp(&UnitHdr.achMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(SSMFILEUNITHDR_MAGIC)))
+        {
+            /*
+             * Does what we've got match, if so read the name.
+             */
+            if (    UnitHdr.u32Instance == iInstance
+                &&  UnitHdr.cchName     == cbUnitNm)
+            {
+                rc = ssmR3StrmPeekAt(&pSSM->Strm, off + RT_UOFFSETOF(SSMFILEUNITHDRV1, szName), szName, cbUnitNm, NULL);
+                AssertRCReturn(rc, rc);
+                AssertLogRelMsgReturn(!szName[UnitHdr.cchName - 1],
+                                      (" Unit name '%.*s' was not properly terminated.\n", cbUnitNm, szName),
+                                      VERR_SSM_INTEGRITY_UNIT);
+
+                /*
+                 * Does the name match?
+                 */
+                if (!memcmp(szName, pszUnit, cbUnitNm))
+                {
+                    rc = ssmR3StrmSeek(&pSSM->Strm, off + RT_UOFFSETOF(SSMFILEUNITHDRV1, szName) + cbUnitNm, RTFILE_SEEK_BEGIN, 0);
+                    pSSM->cbUnitLeftV1 = UnitHdr.cbUnit - RT_UOFFSETOF_DYN(SSMFILEUNITHDRV1, szName[cbUnitNm]);
+                    pSSM->offUnit      = 0;
+                    pSSM->offUnitUser  = 0;
+                    if (piVersion)
+                        *piVersion = UnitHdr.u32Version;
+                    return VINF_SUCCESS;
+                }
+            }
+        }
+        else if (!memcmp(&UnitHdr.achMagic[0], SSMFILEUNITHDR_END, sizeof(SSMFILEUNITHDR_END)))
+            return VERR_SSM_UNIT_NOT_FOUND;
+        else
+            AssertLogRelMsgFailedReturn(("Invalid unit magic at offset %RTfoff, '%.*s'!\n",
+                                         off, sizeof(UnitHdr.achMagic) - 1, &UnitHdr.achMagic[0]),
+                                        VERR_SSM_INTEGRITY_UNIT_MAGIC);
+    }
+    /* won't get here. */
+}
+
+
+/**
+ * Worker for ssmR3FileSeekV2 for simplifying memory cleanup.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The SSM handle.
+ * @param   pDir                The directory buffer.
+ * @param   cbDir               The size of the directory.
+ * @param   cDirEntries         The number of directory entries.
+ * @param   offDir              The directory offset in the file.
+ * @param   pszUnit             The unit to seek to.
+ * @param   iInstance           The particular instance we seek.
+ * @param   piVersion           Where to store the unit version number.
+ */
+static int ssmR3FileSeekSubV2(PSSMHANDLE pSSM, PSSMFILEDIR pDir, size_t cbDir, uint32_t cDirEntries, uint64_t offDir,
+                              const char *pszUnit, uint32_t iInstance, uint32_t *piVersion)
+{
+    /*
+     * Read it.
+     */
+    int rc = ssmR3StrmPeekAt(&pSSM->Strm, offDir, pDir, cbDir, NULL);
+    AssertLogRelRCReturn(rc, rc);
+    rc = ssmR3ValidateDirectory(pDir, (uint32_t)cbDir, offDir, cDirEntries, pSSM->u.Read.cbFileHdr, pSSM->u.Read.u32SvnRev);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Search the directory.
+     */
+    size_t          cbUnitNm   = strlen(pszUnit) + 1;
+    uint32_t const  u32NameCRC = RTCrc32(pszUnit, cbUnitNm - 1);
+    for (uint32_t i = 0; i < cDirEntries; i++)
+    {
+        if (    pDir->aEntries[i].u32NameCRC  == u32NameCRC
+            &&  pDir->aEntries[i].u32Instance == iInstance
+            &&  pDir->aEntries[i].off != 0 /* bug in unreleased code */
+           )
+        {
+            /*
+             * Read and validate the unit header.
+             */
+            SSMFILEUNITHDRV2    UnitHdr;
+            size_t              cbToRead = sizeof(UnitHdr);
+            if (pDir->aEntries[i].off + cbToRead > offDir)
+            {
+                cbToRead = offDir - pDir->aEntries[i].off;
+                RT_ZERO(UnitHdr);
+            }
+            rc = ssmR3StrmPeekAt(&pSSM->Strm, pDir->aEntries[i].off, &UnitHdr, cbToRead, NULL);
+            AssertLogRelRCReturn(rc, rc);
+
+            AssertLogRelMsgReturn(!memcmp(UnitHdr.szMagic, SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic)),
+                                  ("Bad unit header or dictionary offset: i=%u off=%lld\n", i, pDir->aEntries[i].off),
+                                  VERR_SSM_INTEGRITY_UNIT);
+            AssertLogRelMsgReturn(UnitHdr.offStream == pDir->aEntries[i].off,
+                                  ("Bad unit header: i=%d off=%lld offStream=%lld\n", i, pDir->aEntries[i].off, UnitHdr.offStream),
+                                  VERR_SSM_INTEGRITY_UNIT);
+            AssertLogRelMsgReturn(UnitHdr.u32Instance == pDir->aEntries[i].u32Instance,
+                                  ("Bad unit header: i=%d off=%lld u32Instance=%u Dir.u32Instance=%u\n",
+                                   i, pDir->aEntries[i].off, UnitHdr.u32Instance, pDir->aEntries[i].u32Instance),
+                                  VERR_SSM_INTEGRITY_UNIT);
+            uint32_t cbUnitHdr = RT_UOFFSETOF_DYN(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]);
+            AssertLogRelMsgReturn(   UnitHdr.cbName > 0
+                                  && UnitHdr.cbName < sizeof(UnitHdr)
+                                  && cbUnitHdr <= cbToRead,
+                                  ("Bad unit header: i=%u off=%lld cbName=%#x cbToRead=%#x\n", i, pDir->aEntries[i].off, UnitHdr.cbName, cbToRead),
+                                  VERR_SSM_INTEGRITY_UNIT);
+            SSM_CHECK_CRC32_RET(&UnitHdr, RT_UOFFSETOF_DYN(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]),
+                                ("Bad unit header CRC: i=%u off=%lld u32CRC=%#x u32ActualCRC=%#x\n",
+                                 i, pDir->aEntries[i].off, u32CRC, u32ActualCRC));
+
+            /*
+             * Ok, it is valid, get on with the comparing now.
+             */
+            if (    UnitHdr.cbName == cbUnitNm
+                &&  !memcmp(UnitHdr.szName, pszUnit, cbUnitNm))
+            {
+                if (piVersion)
+                    *piVersion = UnitHdr.u32Version;
+                rc = ssmR3StrmSeek(&pSSM->Strm, pDir->aEntries[i].off + cbUnitHdr, RTFILE_SEEK_BEGIN,
+                                   RTCrc32Process(UnitHdr.u32CurStreamCRC, &UnitHdr, cbUnitHdr));
+                AssertLogRelRCReturn(rc, rc);
+                ssmR3DataReadBeginV2(pSSM);
+                return VINF_SUCCESS;
+            }
+        }
+    }
+
+    return VERR_SSM_UNIT_NOT_FOUND;
+}
+
+
+/**
+ * Worker for SSMR3Seek that seeks version 2 saved state files.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The SSM handle.
+ * @param   pszUnit             The unit to seek to.
+ * @param   iInstance           The particular instance we seek.
+ * @param   piVersion           Where to store the unit version number.
+ */
+static int ssmR3FileSeekV2(PSSMHANDLE pSSM, const char *pszUnit, uint32_t iInstance, uint32_t *piVersion)
+{
+    /*
+     * Read the footer, allocate a temporary buffer for the dictionary and
+     * pass it down to a worker to simplify cleanup.
+     */
+    uint64_t        offFooter;
+    SSMFILEFTR      Footer;
+    int rc = ssmR3StrmPeekAt(&pSSM->Strm, -(RTFOFF)sizeof(Footer), &Footer, sizeof(Footer), &offFooter);
+    AssertLogRelRCReturn(rc, rc);
+    AssertLogRelReturn(!memcmp(Footer.szMagic, SSMFILEFTR_MAGIC, sizeof(Footer.szMagic)), VERR_SSM_INTEGRITY);
+    SSM_CHECK_CRC32_RET(&Footer, sizeof(Footer), ("Bad footer CRC: %08x, actual %08x\n", u32CRC, u32ActualCRC));
+
+    size_t const    cbDir = RT_UOFFSETOF_DYN(SSMFILEDIR, aEntries[Footer.cDirEntries]);
+    PSSMFILEDIR     pDir  = (PSSMFILEDIR)RTMemTmpAlloc(cbDir);
+    if (RT_UNLIKELY(!pDir))
+        return VERR_NO_TMP_MEMORY;
+    rc = ssmR3FileSeekSubV2(pSSM, pDir, cbDir, Footer.cDirEntries, offFooter - cbDir,
+                            pszUnit, iInstance, piVersion);
+    RTMemTmpFree(pDir);
+
+    return rc;
+}
+
+
+/**
+ * Seeks to a specific data unit.
+ *
+ * After seeking it's possible to use the getters to on
+ * that data unit.
+ *
+ * @returns VBox status code.
+ * @returns VERR_SSM_UNIT_NOT_FOUND if the unit+instance wasn't found.
+ *
+ * @param   pSSM            The SSM handle returned by SSMR3Open().
+ * @param   pszUnit         The name of the data unit.
+ * @param   iInstance       The instance number.
+ * @param   piVersion       Where to store the version number. (Optional)
+ *
+ * @thread  Any, but the caller is responsible for serializing calls per handle.
+ */
+VMMR3DECL(int) SSMR3Seek(PSSMHANDLE pSSM, const char *pszUnit, uint32_t iInstance, uint32_t *piVersion)
+{
+    LogFlow(("SSMR3Seek: pSSM=%p pszUnit=%p:{%s} iInstance=%RU32 piVersion=%p\n",
+             pSSM, pszUnit, pszUnit, iInstance, piVersion));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pSSM, VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pSSM->enmAfter == SSMAFTER_OPENED, ("%d\n", pSSM->enmAfter),VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pSSM->enmOp == SSMSTATE_OPEN_READ, ("%d\n", pSSM->enmOp), VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pszUnit, VERR_INVALID_POINTER);
+    AssertMsgReturn(!piVersion || VALID_PTR(piVersion), ("%p\n", piVersion), VERR_INVALID_POINTER);
+
+    /*
+     * Reset the state.
+     */
+    if (pSSM->u.Read.pZipDecompV1)
+    {
+        RTZipDecompDestroy(pSSM->u.Read.pZipDecompV1);
+        pSSM->u.Read.pZipDecompV1 = NULL;
+    }
+    pSSM->cbUnitLeftV1  = 0;
+    pSSM->offUnit       = UINT64_MAX;
+    pSSM->offUnitUser   = UINT64_MAX;
+
+    /*
+     * Call the version specific workers.
+     */
+    if (pSSM->u.Read.uFmtVerMajor >= 2)
+        pSSM->rc = ssmR3FileSeekV2(pSSM, pszUnit, iInstance, piVersion);
+    else
+        pSSM->rc = ssmR3FileSeekV1(pSSM, pszUnit, iInstance, piVersion);
+    return pSSM->rc;
+}
+
+
+
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+
+
+
+/**
+ * Query what the VBox status code of the operation is.
+ *
+ * This can be used for putting and getting a batch of values
+ * without bother checking the result till all the calls have
+ * been made.
+ *
+ * @returns SSMAFTER enum value.
+ * @param   pSSM            The saved state handle.
+ */
+VMMR3DECL(int) SSMR3HandleGetStatus(PSSMHANDLE pSSM)
+{
+    SSM_ASSERT_VALID_HANDLE(pSSM);
+    return pSSM->rc;
+}
+
+
+/**
+ * Fail the load operation.
+ *
+ * This is mainly intended for sub item loaders (like timers) which
+ * return code isn't necessarily heeded by the caller but is important
+ * to SSM.
+ *
+ * @returns VBox status code of the handle, or VERR_INVALID_PARAMETER.
+ * @param   pSSM            The saved state handle.
+ * @param   iStatus         Failure status code. This MUST be a VERR_*.
+ */
+VMMR3DECL(int) SSMR3HandleSetStatus(PSSMHANDLE pSSM, int iStatus)
+{
+    SSM_ASSERT_VALID_HANDLE(pSSM);
+    Assert(pSSM->enmOp != SSMSTATE_LIVE_VOTE);
+    if (RT_FAILURE(iStatus))
+    {
+        int rc = pSSM->rc;
+        if (RT_SUCCESS(rc))
+            pSSM->rc = rc = iStatus;
+        return rc;
+    }
+    AssertMsgFailed(("iStatus=%d %Rrc\n", iStatus, iStatus));
+    return VERR_INVALID_PARAMETER;
+}
+
+
+/**
+ * Get what to do after this operation.
+ *
+ * @returns SSMAFTER enum value.
+ * @param   pSSM            The saved state handle.
+ */
+VMMR3DECL(SSMAFTER) SSMR3HandleGetAfter(PSSMHANDLE pSSM)
+{
+    SSM_ASSERT_VALID_HANDLE(pSSM);
+    return pSSM->enmAfter;
+}
+
+
+/**
+ * Checks if it is a live save operation or not.
+ *
+ * @returns True if it is, false if it isn't.
+ * @param   pSSM            The saved state handle.
+ */
+VMMR3DECL(bool) SSMR3HandleIsLiveSave(PSSMHANDLE pSSM)
+{
+    SSM_ASSERT_VALID_HANDLE(pSSM);
+    return pSSM->fLiveSave;
+}
+
+
+/**
+ * Gets the maximum downtime for a live operation.
+ *
+ * @returns The max downtime in milliseconds.  Can be anything from 0 thru
+ *          UINT32_MAX.
+ *
+ * @param   pSSM            The saved state handle.
+ */
+VMMR3DECL(uint32_t) SSMR3HandleMaxDowntime(PSSMHANDLE pSSM)
+{
+    SSM_ASSERT_VALID_HANDLE(pSSM);
+    if (pSSM->enmOp <= SSMSTATE_SAVE_DONE)
+        return pSSM->u.Write.cMsMaxDowntime;
+    return UINT32_MAX;
+}
+
+
+/**
+ * Gets the host bit count of a saved state.
+ *
+ * @returns 32 or 64. If pSSM is invalid, 0 is returned.
+ * @param   pSSM            The saved state handle.
+ *
+ * @remarks This method should ONLY be used for hacks when loading OLDER saved
+ *          state that have data layout or semantic changes without the
+ *          compulsory version number change.
+ */
+VMMR3DECL(uint32_t) SSMR3HandleHostBits(PSSMHANDLE pSSM)
+{
+    SSM_ASSERT_VALID_HANDLE(pSSM);
+    return ssmR3GetHostBits(pSSM);
+}
+
+
+/**
+ * Get the VirtualBox SVN revision that created the saved state.
+ *
+ * @returns The revision number on success.
+ *          form.  If we don't know, it's 0.
+ * @param   pSSM            The saved state handle.
+ *
+ * @remarks This method should ONLY be used for hacks when loading OLDER saved
+ *          state that have data layout or semantic changes without the
+ *          compulsory version number change.  Be VERY careful with this
+ *          function since it will return different values for OSE builds!
+ */
+VMMR3DECL(uint32_t)     SSMR3HandleRevision(PSSMHANDLE pSSM)
+{
+    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
+        return pSSM->u.Read.u32SvnRev;
+#ifdef SSM_STANDALONE
+    return 0;
+#else
+    return VMMGetSvnRev();
+#endif
+}
+
+
+/**
+ * Gets the VirtualBox version that created the saved state.
+ *
+ * @returns VBOX_FULL_VERSION style version number.
+ *          Returns UINT32_MAX if unknown or somehow out of range.
+ *
+ * @param   pSSM            The saved state handle.
+ *
+ * @remarks This method should ONLY be used for hacks when loading OLDER saved
+ *          state that have data layout or semantic changes without the
+ *          compulsory version number change.
+ */
+VMMR3DECL(uint32_t)     SSMR3HandleVersion(PSSMHANDLE pSSM)
+{
+    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
+    {
+        if (    !pSSM->u.Read.u16VerMajor
+            &&  !pSSM->u.Read.u16VerMinor
+            &&  !pSSM->u.Read.u32VerBuild)
+            return UINT32_MAX;
+        AssertReturn(pSSM->u.Read.u16VerMajor <=   0xff, UINT32_MAX);
+        AssertReturn(pSSM->u.Read.u16VerMinor <=   0xff, UINT32_MAX);
+        AssertReturn(pSSM->u.Read.u32VerBuild <= 0xffff, UINT32_MAX);
+        return VBOX_FULL_VERSION_MAKE(pSSM->u.Read.u16VerMajor, pSSM->u.Read.u16VerMinor, pSSM->u.Read.u32VerBuild);
+    }
+    return VBOX_FULL_VERSION;
+}
+
+
+/**
+ * Get the host OS and architecture where the saved state was created.
+ *
+ * @returns Pointer to a read only string.  When known, this is on the os.arch
+ *          form.  If we don't know, it's an empty string.
+ * @param   pSSM            The saved state handle.
+ *
+ * @remarks This method should ONLY be used for hacks when loading OLDER saved
+ *          state that have data layout or semantic changes without the
+ *          compulsory version number change.
+ */
+VMMR3DECL(const char *) SSMR3HandleHostOSAndArch(PSSMHANDLE pSSM)
+{
+    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
+        return pSSM->u.Read.szHostOSAndArch;
+    return KBUILD_TARGET "." KBUILD_TARGET_ARCH;
+}
+
+
+#ifdef DEBUG
+/**
+ * Gets current data offset, relative to the start of the unit - only for debugging
+ */
+VMMR3DECL(uint64_t) SSMR3HandleTellInUnit(PSSMHANDLE pSSM)
+{
+    return ssmR3StrmTell(&pSSM->Strm) - pSSM->offUnitUser;
+}
+#endif
+
+
+#ifndef SSM_STANDALONE
+/**
+ * Asynchronously cancels the current SSM operation ASAP.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SSM_NO_PENDING_OPERATION if nothing around that can be
+ *          cancelled.
+ * @retval  VERR_SSM_ALREADY_CANCELLED if the operation as already been
+ *          cancelled.
+ *
+ * @param   pUVM            The VM handle.
+ *
+ * @thread  Any.
+ */
+VMMR3DECL(int) SSMR3Cancel(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    int rc = RTCritSectEnter(&pVM->ssm.s.CancelCritSect);
+    AssertRCReturn(rc, rc);
+
+    PSSMHANDLE pSSM = pVM->ssm.s.pSSM;
+    if (pSSM)
+    {
+        uint32_t u32Old;
+        if (ASMAtomicCmpXchgExU32(&pSSM->fCancelled, SSMHANDLE_CANCELLED, SSMHANDLE_OK, &u32Old))
+        {
+            LogRel(("SSM: Cancelled pending operation\n"));
+            rc = VINF_SUCCESS;
+        }
+        else if (u32Old == SSMHANDLE_CANCELLED)
+            rc = VERR_SSM_ALREADY_CANCELLED;
+        else
+        {
+            AssertLogRelMsgFailed(("fCancelled=%RX32 enmOp=%d\n", u32Old, pSSM->enmOp));
+            rc = VERR_SSM_IPE_3;
+        }
+    }
+    else
+        rc = VERR_SSM_NO_PENDING_OPERATION;
+
+    RTCritSectLeave(&pVM->ssm.s.CancelCritSect);
+    return rc;
+}
+#endif /* !SSM_STANDALONE */
+
diff --git a/src/VBox/VMM/VMMR3/STAM.cpp b/src/VBox/VMM/VMMR3/STAM.cpp
new file mode 100644
index 00000000..70d7b1c6
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/STAM.cpp
@@ -0,0 +1,3090 @@
+/* $Id: STAM.cpp $ */
+/** @file
+ * STAM - The Statistics Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_stam       STAM - The Statistics Manager
+ *
+ * The purpose for the statistics manager is to present the rest of the system
+ * with a somewhat uniform way of accessing VMM statistics.  STAM sports a
+ * couple of different APIs for accessing them: STAMR3EnumU, STAMR3SnapshotU,
+ * STAMR3DumpU, STAMR3DumpToReleaseLogU and the debugger.  Main is exposing the
+ * XML based one, STAMR3SnapshotU.
+ *
+ * The rest of the VMM together with the devices and drivers registers their
+ * statistics with STAM giving them a name.  The name is hierarchical, the
+ * components separated by slashes ('/') and must start with a slash.
+ *
+ * Each item registered with STAM - also, half incorrectly, called a sample -
+ * has a type, unit, visibility, data pointer and description associated with it
+ * in addition to the name (described above).  The type tells STAM what kind of
+ * structure the pointer is pointing to.  The visibility allows unused
+ * statistics from cluttering the output or showing up in the GUI.  All the bits
+ * together makes STAM able to present the items in a sensible way to the user.
+ * Some types also allows STAM to reset the data, which is very convenient when
+ * digging into specific operations and such.
+ *
+ * PS. The VirtualBox Debugger GUI has a viewer for inspecting the statistics
+ * STAM provides.  You will also find statistics in the release and debug logs.
+ * And as mentioned in the introduction, the debugger console features a couple
+ * of command: .stats and .statsreset.
+ *
+ * @see grp_stam
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_STAM
+#include <VBox/vmm/stam.h>
+#include "STAMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/err.h>
+#include <VBox/dbg.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/mem.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The maximum name length excluding the terminator. */
+#define STAM_MAX_NAME_LEN   239
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Argument structure for stamR3PrintOne().
+ */
+typedef struct STAMR3PRINTONEARGS
+{
+    PUVM        pUVM;
+    void       *pvArg;
+    DECLCALLBACKMEMBER(void, pfnPrintf)(struct STAMR3PRINTONEARGS *pvArg, const char *pszFormat, ...);
+} STAMR3PRINTONEARGS, *PSTAMR3PRINTONEARGS;
+
+
+/**
+ * Argument structure to stamR3EnumOne().
+ */
+typedef struct STAMR3ENUMONEARGS
+{
+    PVM             pVM;
+    PFNSTAMR3ENUM   pfnEnum;
+    void           *pvUser;
+} STAMR3ENUMONEARGS, *PSTAMR3ENUMONEARGS;
+
+
+/**
+ * The snapshot status structure.
+ * Argument package passed to stamR3SnapshotOne, stamR3SnapshotPrintf and stamR3SnapshotOutput.
+ */
+typedef struct STAMR3SNAPSHOTONE
+{
+    /** Pointer to the buffer start. */
+    char           *pszStart;
+    /** Pointer to the buffer end. */
+    char           *pszEnd;
+    /** Pointer to the current buffer position. */
+    char           *psz;
+    /** Pointer to the VM. */
+    PVM             pVM;
+    /** The number of bytes allocated. */
+    size_t          cbAllocated;
+    /** The status code. */
+    int             rc;
+    /** Whether to include the description strings. */
+    bool            fWithDesc;
+} STAMR3SNAPSHOTONE, *PSTAMR3SNAPSHOTONE;
+
+
+/**
+ * Init record for a ring-0 statistic sample.
+ */
+typedef struct STAMR0SAMPLE
+{
+    /** The GVMMSTATS structure offset of the variable. */
+    unsigned        offVar;
+    /** The type. */
+    STAMTYPE        enmType;
+    /** The unit. */
+    STAMUNIT        enmUnit;
+    /** The name. */
+    const char     *pszName;
+    /** The description. */
+    const char     *pszDesc;
+} STAMR0SAMPLE;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static void                 stamR3LookupDestroyTree(PSTAMLOOKUP pRoot);
+static int                  stamR3RegisterU(PUVM pUVM, void *pvSample, PFNSTAMR3CALLBACKRESET pfnReset,
+                                            PFNSTAMR3CALLBACKPRINT pfnPrint, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+                                            const char *pszName, STAMUNIT enmUnit, const char *pszDesc, uint8_t iRefreshGrp);
+static int                  stamR3ResetOne(PSTAMDESC pDesc, void *pvArg);
+static DECLCALLBACK(void)   stamR3EnumLogPrintf(PSTAMR3PRINTONEARGS pvArg, const char *pszFormat, ...);
+static DECLCALLBACK(void)   stamR3EnumRelLogPrintf(PSTAMR3PRINTONEARGS pvArg, const char *pszFormat, ...);
+static DECLCALLBACK(void)   stamR3EnumPrintf(PSTAMR3PRINTONEARGS pvArg, const char *pszFormat, ...);
+static int                  stamR3SnapshotOne(PSTAMDESC pDesc, void *pvArg);
+static int                  stamR3SnapshotPrintf(PSTAMR3SNAPSHOTONE pThis, const char *pszFormat, ...);
+static int                  stamR3PrintOne(PSTAMDESC pDesc, void *pvArg);
+static int                  stamR3EnumOne(PSTAMDESC pDesc, void *pvArg);
+static bool                 stamR3MultiMatch(const char * const *papszExpressions, unsigned cExpressions, unsigned *piExpression, const char *pszName);
+static char **              stamR3SplitPattern(const char *pszPat, unsigned *pcExpressions, char **ppszCopy);
+static int                  stamR3EnumU(PUVM pUVM, const char *pszPat, bool fUpdateRing0, int (pfnCallback)(PSTAMDESC pDesc, void *pvArg), void *pvArg);
+static void                 stamR3Ring0StatsRegisterU(PUVM pUVM);
+
+#ifdef VBOX_WITH_DEBUGGER
+static FNDBGCCMD            stamR3CmdStats;
+static DECLCALLBACK(void)   stamR3EnumDbgfPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...);
+static FNDBGCCMD            stamR3CmdStatsReset;
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_DEBUGGER
+/** Pattern argument. */
+static const DBGCVARDESC    g_aArgPat[] =
+{
+    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
+    {  0,           1,          DBGCVAR_CAT_STRING,     0,                              "pattern",      "Which samples the command shall be applied to. Use '*' as wildcard. Use ';' to separate expression." }
+};
+
+/** Command descriptors. */
+static const DBGCCMD    g_aCmds[] =
+{
+    /* pszCmd,      cArgsMin, cArgsMax, paArgDesc,          cArgDescs,                  fFlags,     pfnHandler          pszSyntax,          ....pszDescription */
+    { "stats",      0,        1,        &g_aArgPat[0],      RT_ELEMENTS(g_aArgPat),     0,          stamR3CmdStats,     "[pattern]",        "Display statistics." },
+    { "statsreset", 0,        1,        &g_aArgPat[0],      RT_ELEMENTS(g_aArgPat),     0,          stamR3CmdStatsReset,"[pattern]",        "Resets statistics." }
+};
+#endif
+
+
+/**
+ * The GVMM mapping records - sans the host cpus.
+ */
+static const STAMR0SAMPLE g_aGVMMStats[] =
+{
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltCalls),        STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltCalls", "The number of calls to GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltBlocking),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltBlocking", "The number of times we did go to sleep in GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltTimeouts),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltTimeouts", "The number of times we timed out in GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltNotBlocking),  STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltNotBlocking", "The number of times we didn't go to sleep in GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltWakeUps),      STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltWakeUps", "The number of wake ups done during GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cWakeUpCalls),      STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/WakeUpCalls", "The number of calls to GVMMR0WakeUp." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cWakeUpNotHalted),  STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/WakeUpNotHalted", "The number of times the EMT thread wasn't actually halted when GVMMR0WakeUp was called." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cWakeUpWakeUps),    STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/WakeUpWakeUps", "The number of wake ups done during GVMMR0WakeUp (not counting the explicit one)." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPokeCalls),        STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PokeCalls", "The number of calls to GVMMR0Poke." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPokeNotBusy),      STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PokeNotBusy", "The number of times the EMT thread wasn't actually busy when GVMMR0Poke was called." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPollCalls),        STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PollCalls", "The number of calls to GVMMR0SchedPoll." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPollHalts),        STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PollHalts", "The number of times the EMT has halted in a GVMMR0SchedPoll call." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPollWakeUps),      STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PollWakeUps", "The number of wake ups done during GVMMR0SchedPoll." },
+
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltCalls),       STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltCalls", "The number of calls to GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltBlocking),    STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltBlocking", "The number of times we did go to sleep in GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltTimeouts),    STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltTimeouts", "The number of times we timed out in GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltNotBlocking), STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltNotBlocking", "The number of times we didn't go to sleep in GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltWakeUps),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltWakeUps", "The number of wake ups done during GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cWakeUpCalls),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/WakeUpCalls", "The number of calls to GVMMR0WakeUp." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cWakeUpNotHalted), STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/WakeUpNotHalted", "The number of times the EMT thread wasn't actually halted when GVMMR0WakeUp was called." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cWakeUpWakeUps),   STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/WakeUpWakeUps", "The number of wake ups done during GVMMR0WakeUp (not counting the explicit one)." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPokeCalls),       STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PokeCalls", "The number of calls to GVMMR0Poke." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPokeNotBusy),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PokeNotBusy", "The number of times the EMT thread wasn't actually busy when GVMMR0Poke was called." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPollCalls),       STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PollCalls", "The number of calls to GVMMR0SchedPoll." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPollHalts),       STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PollHalts", "The number of times the EMT has halted in a GVMMR0SchedPoll call." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPollWakeUps),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PollWakeUps", "The number of wake ups done during GVMMR0SchedPoll." },
+
+    { RT_UOFFSETOF(GVMMSTATS, cVMs),                      STAMTYPE_U32,       STAMUNIT_CALLS, "/GVMM/VMs", "The number of VMs accessible to the caller." },
+    { RT_UOFFSETOF(GVMMSTATS, cEMTs),                     STAMTYPE_U32,       STAMUNIT_CALLS, "/GVMM/EMTs", "The number of emulation threads." },
+    { RT_UOFFSETOF(GVMMSTATS, cHostCpus),                 STAMTYPE_U32,       STAMUNIT_CALLS, "/GVMM/HostCPUs", "The number of host CPUs." },
+};
+
+
+/**
+ * The GMM mapping records.
+ */
+static const STAMR0SAMPLE g_aGMMStats[] =
+{
+    { RT_UOFFSETOF(GMMSTATS, cMaxPages),                        STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/cMaxPages",                   "The maximum number of pages GMM is allowed to allocate." },
+    { RT_UOFFSETOF(GMMSTATS, cReservedPages),                   STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/cReservedPages",              "The number of pages that has been reserved." },
+    { RT_UOFFSETOF(GMMSTATS, cOverCommittedPages),              STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/cOverCommittedPages",         "The number of pages that we have over-committed in reservations." },
+    { RT_UOFFSETOF(GMMSTATS, cAllocatedPages),                  STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/cAllocatedPages",             "The number of actually allocated (committed if you like) pages." },
+    { RT_UOFFSETOF(GMMSTATS, cSharedPages),                     STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/cSharedPages",                "The number of pages that are shared. A subset of cAllocatedPages." },
+    { RT_UOFFSETOF(GMMSTATS, cDuplicatePages),                  STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/cDuplicatePages",             "The number of pages that are actually shared between VMs." },
+    { RT_UOFFSETOF(GMMSTATS, cLeftBehindSharedPages),           STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/cLeftBehindSharedPages",      "The number of pages that are shared that has been left behind by VMs not doing proper cleanups." },
+    { RT_UOFFSETOF(GMMSTATS, cBalloonedPages),                  STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/cBalloonedPages",             "The number of current ballooned pages." },
+    { RT_UOFFSETOF(GMMSTATS, cChunks),                          STAMTYPE_U32,   STAMUNIT_COUNT, "/GMM/cChunks",                     "The number of allocation chunks." },
+    { RT_UOFFSETOF(GMMSTATS, cFreedChunks),                     STAMTYPE_U32,   STAMUNIT_COUNT, "/GMM/cFreedChunks",                "The number of freed chunks ever." },
+    { RT_UOFFSETOF(GMMSTATS, cShareableModules),                STAMTYPE_U32,   STAMUNIT_COUNT, "/GMM/cShareableModules",           "The number of shareable modules." },
+    { RT_UOFFSETOF(GMMSTATS, idFreeGeneration),                 STAMTYPE_U64,   STAMUNIT_NONE,  "/GMM/idFreeGeneration",            "The current chunk freeing generation number (for per-VM chunk lookup TLB versioning)." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.Reserved.cBasePages),      STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/VM/Reserved/cBasePages",      "The amount of base memory (RAM, ROM, ++) reserved by the VM." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.Reserved.cShadowPages),    STAMTYPE_U32,   STAMUNIT_PAGES, "/GMM/VM/Reserved/cShadowPages",    "The amount of memory reserved for shadow/nested page tables." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.Reserved.cFixedPages),     STAMTYPE_U32,   STAMUNIT_PAGES, "/GMM/VM/Reserved/cFixedPages",     "The amount of memory reserved for fixed allocations like MMIO2 and the hyper heap." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.Allocated.cBasePages),     STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/VM/Allocated/cBasePages",     "The amount of base memory (RAM, ROM, ++) allocated by the VM." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.Allocated.cShadowPages),   STAMTYPE_U32,   STAMUNIT_PAGES, "/GMM/VM/Allocated/cShadowPages",   "The amount of memory allocated for shadow/nested page tables." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.Allocated.cFixedPages),    STAMTYPE_U32,   STAMUNIT_PAGES, "/GMM/VM/Allocated/cFixedPages",    "The amount of memory allocated for fixed allocations like MMIO2 and the hyper heap." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.cPrivatePages),            STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/VM/cPrivatePages",            "The current number of private pages." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.cSharedPages),             STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/VM/cSharedPages",             "The current number of shared pages." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.cBalloonedPages),          STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/VM/cBalloonedPages",          "The current number of ballooned pages." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.cMaxBalloonedPages),       STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/VM/cMaxBalloonedPages",       "The max number of pages that can be ballooned." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.cReqBalloonedPages),       STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/VM/cReqBalloonedPages",       "The number of pages we've currently requested the guest to give us." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.cReqActuallyBalloonedPages),STAMTYPE_U64,  STAMUNIT_PAGES, "/GMM/VM/cReqActuallyBalloonedPages","The number of pages the guest has given us in response to the request." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.cReqDeflatePages),         STAMTYPE_U64,   STAMUNIT_PAGES, "/GMM/VM/cReqDeflatePages",         "The number of pages we've currently requested the guest to take back." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.cShareableModules),        STAMTYPE_U32,   STAMUNIT_COUNT, "/GMM/VM/cShareableModules",        "The number of shareable modules traced by the VM." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.enmPolicy),                STAMTYPE_U32,   STAMUNIT_NONE,  "/GMM/VM/enmPolicy",                "The current over-commit policy." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.enmPriority),              STAMTYPE_U32,   STAMUNIT_NONE,  "/GMM/VM/enmPriority",              "The VM priority for arbitrating VMs in low and out of memory situation." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.fBallooningEnabled),       STAMTYPE_BOOL,  STAMUNIT_NONE,  "/GMM/VM/fBallooningEnabled",       "Whether ballooning is enabled or not." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.fSharedPagingEnabled),     STAMTYPE_BOOL,  STAMUNIT_NONE,  "/GMM/VM/fSharedPagingEnabled",     "Whether shared paging is enabled or not." },
+    { RT_UOFFSETOF(GMMSTATS, VMStats.fMayAllocate),             STAMTYPE_BOOL,  STAMUNIT_NONE,  "/GMM/VM/fMayAllocate",             "Whether the VM is allowed to allocate memory or not." },
+};
+
+
+/**
+ * Initializes the STAM.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM structure.
+ */
+VMMR3DECL(int) STAMR3InitUVM(PUVM pUVM)
+{
+    LogFlow(("STAMR3Init\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompile(sizeof(pUVM->stam.s) <= sizeof(pUVM->stam.padding));
+    AssertRelease(sizeof(pUVM->stam.s) <= sizeof(pUVM->stam.padding));
+
+    /*
+     * Initialize the read/write lock and list.
+     */
+    int rc = RTSemRWCreate(&pUVM->stam.s.RWSem);
+    AssertRCReturn(rc, rc);
+
+    RTListInit(&pUVM->stam.s.List);
+
+    /*
+     * Initialize the root node.
+     */
+    PSTAMLOOKUP pRoot = (PSTAMLOOKUP)RTMemAlloc(sizeof(STAMLOOKUP));
+    if (!pRoot)
+    {
+        RTSemRWDestroy(pUVM->stam.s.RWSem);
+        pUVM->stam.s.RWSem = NIL_RTSEMRW;
+        return VERR_NO_MEMORY;
+    }
+    pRoot->pParent      = NULL;
+    pRoot->papChildren   = NULL;
+    pRoot->pDesc        = NULL;
+    pRoot->cDescsInTree = 0;
+    pRoot->cChildren    = 0;
+    pRoot->iParent      = UINT16_MAX;
+    pRoot->off          = 0;
+    pRoot->cch          = 0;
+    pRoot->szName[0]    = '\0';
+
+    pUVM->stam.s.pRoot = pRoot;
+
+    /*
+     * Register the ring-0 statistics (GVMM/GMM).
+     */
+    stamR3Ring0StatsRegisterU(pUVM);
+
+#ifdef VBOX_WITH_DEBUGGER
+    /*
+     * Register debugger commands.
+     */
+    static bool fRegisteredCmds = false;
+    if (!fRegisteredCmds)
+    {
+        rc = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
+        if (RT_SUCCESS(rc))
+            fRegisteredCmds = true;
+    }
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminates the STAM.
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+VMMR3DECL(void) STAMR3TermUVM(PUVM pUVM)
+{
+    /*
+     * Free used memory and the RWLock.
+     */
+    PSTAMDESC pCur, pNext;
+    RTListForEachSafe(&pUVM->stam.s.List, pCur, pNext, STAMDESC, ListEntry)
+    {
+        pCur->pLookup->pDesc = NULL;
+        RTMemFree(pCur);
+    }
+
+    stamR3LookupDestroyTree(pUVM->stam.s.pRoot);
+    pUVM->stam.s.pRoot = NULL;
+
+    Assert(pUVM->stam.s.RWSem != NIL_RTSEMRW);
+    RTSemRWDestroy(pUVM->stam.s.RWSem);
+    pUVM->stam.s.RWSem = NIL_RTSEMRW;
+}
+
+
+/**
+ * Registers a sample with the statistics manager.
+ *
+ * Statistics are maintained on a per VM basis and is normally registered
+ * during the VM init stage, but there is nothing preventing you from
+ * register them at runtime.
+ *
+ * Use STAMR3Deregister() to deregister statistics at runtime, however do
+ * not bother calling at termination time.
+ *
+ * It is not possible to register the same sample twice.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   pszName     Sample name. The name is on this form "/<component>/<sample>".
+ *                      Further nesting is possible.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ */
+VMMR3DECL(int)  STAMR3RegisterU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName,
+                                STAMUNIT enmUnit, const char *pszDesc)
+{
+    AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    return stamR3RegisterU(pUVM, pvSample, NULL, NULL, enmType, enmVisibility, pszName, enmUnit, pszDesc, STAM_REFRESH_GRP_NONE);
+}
+
+
+/**
+ * Registers a sample with the statistics manager.
+ *
+ * Statistics are maintained on a per VM basis and is normally registered
+ * during the VM init stage, but there is nothing preventing you from
+ * register them at runtime.
+ *
+ * Use STAMR3Deregister() to deregister statistics at runtime, however do
+ * not bother calling at termination time.
+ *
+ * It is not possible to register the same sample twice.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   pszName     Sample name. The name is on this form "/<component>/<sample>".
+ *                      Further nesting is possible.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ */
+VMMR3DECL(int)  STAMR3Register(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName,
+                               STAMUNIT enmUnit, const char *pszDesc)
+{
+    AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
+    return stamR3RegisterU(pVM->pUVM, pvSample, NULL, NULL, enmType, enmVisibility, pszName, enmUnit, pszDesc,
+                           STAM_REFRESH_GRP_NONE);
+}
+
+
+/**
+ * Same as STAMR3RegisterU except that the name is specified in a
+ * RTStrPrintf like fashion.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   ...         Arguments to the format string.
+ */
+VMMR3DECL(int)  STAMR3RegisterFU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                 const char *pszDesc, const char *pszName, ...)
+{
+    va_list args;
+    va_start(args, pszName);
+    int rc = STAMR3RegisterVU(pUVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    va_end(args);
+    return rc;
+}
+
+
+/**
+ * Same as STAMR3Register except that the name is specified in a
+ * RTStrPrintf like fashion.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   ...         Arguments to the format string.
+ */
+VMMR3DECL(int)  STAMR3RegisterF(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                const char *pszDesc, const char *pszName, ...)
+{
+    va_list args;
+    va_start(args, pszName);
+    int rc = STAMR3RegisterVU(pVM->pUVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    va_end(args);
+    return rc;
+}
+
+
+/**
+ * Same as STAMR3Register except that the name is specified in a
+ * RTStrPrintfV like fashion.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   args        Arguments to the format string.
+ */
+VMMR3DECL(int)  STAMR3RegisterVU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                 const char *pszDesc, const char *pszName, va_list args)
+{
+    AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
+
+    char   szFormattedName[STAM_MAX_NAME_LEN + 8];
+    size_t cch = RTStrPrintfV(szFormattedName, sizeof(szFormattedName), pszName, args);
+    AssertReturn(cch <= STAM_MAX_NAME_LEN, VERR_OUT_OF_RANGE);
+
+    return STAMR3RegisterU(pUVM, pvSample, enmType, enmVisibility, szFormattedName, enmUnit, pszDesc);
+}
+
+
+/**
+ * Same as STAMR3Register except that the name is specified in a
+ * RTStrPrintfV like fashion.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   args        Arguments to the format string.
+ */
+VMMR3DECL(int)  STAMR3RegisterV(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                const char *pszDesc, const char *pszName, va_list args)
+{
+    return STAMR3RegisterVU(pVM->pUVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+}
+
+
+/**
+ * Similar to STAMR3Register except for the two callbacks, the implied type (STAMTYPE_CALLBACK),
+ * and name given in an RTStrPrintf like fashion.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pfnReset    Callback for resetting the sample. NULL should be used if the sample can't be reset.
+ * @param   pfnPrint    Print the sample.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   ...         Arguments to the format string.
+ * @remark  There is currently no device or driver variant of this API. Add one if it should become necessary!
+ */
+VMMR3DECL(int)  STAMR3RegisterCallback(PVM pVM, void *pvSample, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                       PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
+                                       const char *pszDesc, const char *pszName, ...)
+{
+    va_list args;
+    va_start(args, pszName);
+    int rc = STAMR3RegisterCallbackV(pVM, pvSample, enmVisibility, enmUnit, pfnReset, pfnPrint, pszDesc, pszName, args);
+    va_end(args);
+    return rc;
+}
+
+
+/**
+ * Same as STAMR3RegisterCallback() except for the ellipsis which is a va_list here.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pfnReset    Callback for resetting the sample. NULL should be used if the sample can't be reset.
+ * @param   pfnPrint    Print the sample.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   args        Arguments to the format string.
+ * @remark  There is currently no device or driver variant of this API. Add one if it should become necessary!
+ */
+VMMR3DECL(int)  STAMR3RegisterCallbackV(PVM pVM, void *pvSample, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                        PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
+                                        const char *pszDesc, const char *pszName, va_list args)
+{
+    char *pszFormattedName;
+    RTStrAPrintfV(&pszFormattedName, pszName, args);
+    if (!pszFormattedName)
+        return VERR_NO_MEMORY;
+
+    int rc = stamR3RegisterU(pVM->pUVM, pvSample, pfnReset, pfnPrint, STAMTYPE_CALLBACK, enmVisibility, pszFormattedName,
+                             enmUnit, pszDesc, STAM_REFRESH_GRP_NONE);
+    RTStrFree(pszFormattedName);
+    return rc;
+}
+
+
+/**
+ * Same as STAMR3RegisterFU, except there is an extra refresh group parameter.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   iRefreshGrp The refresh group, STAM_REFRESH_GRP_XXX.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   ...         Arguments to the format string.
+ */
+VMMR3DECL(int) STAMR3RegisterRefresh(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                     uint8_t iRefreshGrp, const char *pszDesc, const char *pszName, ...)
+{
+    va_list args;
+    va_start(args, pszName);
+    int rc = STAMR3RegisterRefreshV(pUVM, pvSample, enmType, enmVisibility, enmUnit, iRefreshGrp, pszDesc, pszName, args);
+    va_end(args);
+    return rc;
+}
+
+
+/**
+ * Same as STAMR3RegisterVU, except there is an extra refresh group parameter.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   iRefreshGrp The refresh group, STAM_REFRESH_GRP_XXX.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   va          Arguments to the format string.
+ */
+VMMR3DECL(int) STAMR3RegisterRefreshV(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                      uint8_t iRefreshGrp, const char *pszDesc, const char *pszName, va_list va)
+{
+    AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
+
+    char   szFormattedName[STAM_MAX_NAME_LEN + 8];
+    size_t cch = RTStrPrintfV(szFormattedName, sizeof(szFormattedName), pszName, va);
+    AssertReturn(cch <= STAM_MAX_NAME_LEN, VERR_OUT_OF_RANGE);
+
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    return stamR3RegisterU(pUVM, pvSample, NULL, NULL, enmType, enmVisibility, pszName, enmUnit, pszDesc, iRefreshGrp);
+}
+
+
+#ifdef VBOX_STRICT
+/**
+ * Divide the strings into sub-strings using '/' as delimiter
+ * and then compare them in strcmp fashion.
+ *
+ * @returns Difference.
+ * @retval  0 if equal.
+ * @retval  < 0 if psz1 is less than psz2.
+ * @retval  > 0 if psz1 greater than psz2.
+ *
+ * @param   psz1        The first string.
+ * @param   psz2        The second string.
+ */
+static int stamR3SlashCompare(const char *psz1, const char *psz2)
+{
+    for (;;)
+    {
+        unsigned int ch1 = *psz1++;
+        unsigned int ch2 = *psz2++;
+        if (ch1 != ch2)
+        {
+            /* slash is end-of-sub-string, so it trumps everything but '\0'. */
+            if (ch1 == '/')
+                return ch2 ? -1 : 1;
+            if (ch2 == '/')
+                return ch1 ? 1 : -1;
+            return ch1 - ch2;
+        }
+
+        /* done? */
+        if (ch1 == '\0')
+            return 0;
+    }
+}
+#endif /* VBOX_STRICT */
+
+
+/**
+ * Compares a lookup node with a name.
+ *
+ * @returns like strcmp and memcmp.
+ * @param   pNode               The lookup node.
+ * @param   pchName             The name, not necessarily terminated.
+ * @param   cchName             The length of the name.
+ */
+DECL_FORCE_INLINE(int) stamR3LookupCmp(PSTAMLOOKUP pNode, const char *pchName, uint32_t cchName)
+{
+    uint32_t cchComp = RT_MIN(pNode->cch, cchName);
+    int iDiff = memcmp(pNode->szName, pchName, cchComp);
+    if (!iDiff && pNode->cch != cchName)
+        iDiff = pNode->cch > cchName ? 2 : -2;
+    return iDiff;
+}
+
+
+/**
+ * Creates a new lookup child node.
+ *
+ * @returns Pointer to the newly created lookup node.
+ * @param   pParent             The parent node.
+ * @param   pchName             The name (not necessarily terminated).
+ * @param   cchName             The length of the name.
+ * @param   offName             The offset of the node in a path.
+ * @param   iChild              Child index of a node that's before the one
+ *                              we're inserting (returned by
+ *                              stamR3LookupFindChild).
+ */
+static PSTAMLOOKUP stamR3LookupNewChild(PSTAMLOOKUP pParent, const char *pchName, uint32_t cchName, uint32_t offName,
+                                        uint32_t iChild)
+{
+    Assert(cchName <= UINT8_MAX);
+    Assert(offName <= UINT8_MAX);
+    Assert(iChild  <  UINT16_MAX);
+
+    /*
+     * Allocate a new entry.
+     */
+    PSTAMLOOKUP pNew = (PSTAMLOOKUP)RTMemAlloc(RT_UOFFSETOF_DYN(STAMLOOKUP, szName[cchName + 1]));
+    if (!pNew)
+        return NULL;
+    pNew->pParent       = pParent;
+    pNew->papChildren    = NULL;
+    pNew->pDesc         = NULL;
+    pNew->cDescsInTree  = 0;
+    pNew->cChildren     = 0;
+    pNew->cch           = (uint16_t)cchName;
+    pNew->off           = (uint16_t)offName;
+    memcpy(pNew->szName, pchName, cchName);
+    pNew->szName[cchName] = '\0';
+
+    /*
+     * Reallocate the array?
+     */
+    if (RT_IS_POWER_OF_TWO(pParent->cChildren))
+    {
+        uint32_t cNew = pParent->cChildren ? (uint32_t)pParent->cChildren * 2 : 8;
+        AssertReturnStmt(cNew <= 0x8000, RTMemFree(pNew), NULL);
+        void *pvNew = RTMemRealloc(pParent->papChildren, cNew * sizeof(pParent->papChildren[0]));
+        if (!pvNew)
+        {
+            RTMemFree(pNew);
+            return NULL;
+        }
+        pParent->papChildren = (PSTAMLOOKUP *)pvNew;
+    }
+
+    /*
+     * Find the exact insertion point using iChild as a very good clue from
+     * the find function.
+     */
+    if (!pParent->cChildren)
+        iChild = 0;
+    else
+    {
+        if (iChild >= pParent->cChildren)
+            iChild = pParent->cChildren - 1;
+        while (   iChild < pParent->cChildren
+               && stamR3LookupCmp(pParent->papChildren[iChild], pchName, cchName) < 0)
+            iChild++;
+    }
+
+    /*
+     * Insert it.
+     */
+    if (iChild < pParent->cChildren)
+    {
+        /* Do shift. */
+        uint32_t i = pParent->cChildren;
+        while (i > iChild)
+        {
+            PSTAMLOOKUP pNode = pParent->papChildren[i - 1];
+            pParent->papChildren[i] = pNode;
+            pNode->iParent = i;
+            i--;
+        }
+    }
+
+    pNew->iParent = iChild;
+    pParent->papChildren[iChild] = pNew;
+    pParent->cChildren++;
+
+    return pNew;
+}
+
+
+/**
+ * Looks up a child.
+ *
+ * @returns Pointer to child node if found, NULL if not.
+ * @param   pParent             The parent node.
+ * @param   pchName             The name (not necessarily terminated).
+ * @param   cchName             The length of the name.
+ * @param   piChild             Where to store a child index suitable for
+ *                              passing to stamR3LookupNewChild when NULL is
+ *                              returned.
+ */
+static PSTAMLOOKUP stamR3LookupFindChild(PSTAMLOOKUP pParent, const char *pchName, uint32_t cchName, uint32_t *piChild)
+{
+    uint32_t iChild = pParent->cChildren;
+    if (iChild > 4)
+    {
+        uint32_t iFirst = 0;
+        uint32_t iEnd   = iChild;
+        iChild /= 2;
+        for (;;)
+        {
+            int iDiff = stamR3LookupCmp(pParent->papChildren[iChild], pchName, cchName);
+            if (!iDiff)
+            {
+                if (piChild)
+                    *piChild = iChild;
+                return pParent->papChildren[iChild];
+            }
+
+            /* Split. */
+            if (iDiff < 0)
+            {
+                iFirst = iChild + 1;
+                if (iFirst >= iEnd)
+                {
+                    if (piChild)
+                        *piChild = iChild;
+                    break;
+                }
+            }
+            else
+            {
+                if (iChild == iFirst)
+                {
+                    if (piChild)
+                        *piChild = iChild ? iChild - 1 : 0;
+                    break;
+                }
+                iEnd = iChild;
+            }
+
+            /* Calc next child. */
+            iChild = (iEnd - iFirst) / 2 + iFirst;
+        }
+        return NULL;
+    }
+
+    /*
+     * Linear search.
+     */
+    while (iChild-- > 0)
+    {
+        int iDiff = stamR3LookupCmp(pParent->papChildren[iChild], pchName, cchName);
+        if (iDiff <= 0)
+        {
+            if (piChild)
+                *piChild = iChild;
+            return !iDiff ? pParent->papChildren[iChild] : NULL;
+        }
+    }
+    if (piChild)
+        *piChild = 0;
+    return NULL;
+}
+
+
+/**
+ * Find the next sample descriptor node.
+ *
+ * This is for use with insertion in the big list and pattern range lookups.
+ *
+ * @returns Pointer to the next sample descriptor. NULL if not found (i.e.
+ *          we're at the end of the list).
+ * @param   pLookup             The current node.
+ */
+static PSTAMDESC stamR3LookupFindNextWithDesc(PSTAMLOOKUP pLookup)
+{
+    Assert(!pLookup->pDesc);
+    PSTAMLOOKUP pCur = pLookup;
+    uint32_t    iCur = 0;
+    for (;;)
+    {
+        /*
+         * Check all children.
+         */
+        uint32_t cChildren = pCur->cChildren;
+        if (iCur < cChildren)
+        {
+            PSTAMLOOKUP *papChildren = pCur->papChildren;
+            do
+            {
+                PSTAMLOOKUP pChild = papChildren[iCur];
+                if (pChild->pDesc)
+                    return pChild->pDesc;
+
+                if (pChild->cChildren > 0)
+                {
+                    /* One level down. */
+                    iCur = 0;
+                    pCur = pChild;
+                    break;
+                }
+            } while (++iCur < cChildren);
+        }
+        else
+        {
+            /*
+             * One level up, resuming after the current.
+             */
+            iCur = pCur->iParent + 1;
+            pCur = pCur->pParent;
+            if (!pCur)
+                return NULL;
+        }
+    }
+}
+
+
+/**
+ * Look up a sample descriptor by name.
+ *
+ * @returns Pointer to a sample descriptor.
+ * @param   pRoot               The root node.
+ * @param   pszName             The name to lookup.
+ */
+static PSTAMDESC stamR3LookupFindDesc(PSTAMLOOKUP pRoot, const char *pszName)
+{
+    Assert(!pRoot->pParent);
+    while (*pszName++ == '/')
+    {
+        const char *pszEnd = strchr(pszName, '/');
+        uint32_t    cch    = pszEnd ? pszEnd - pszName : (uint32_t)strlen(pszName);
+        PSTAMLOOKUP pChild = stamR3LookupFindChild(pRoot, pszName, cch, NULL);
+        if (!pChild)
+            break;
+        if (!pszEnd)
+            return pChild->pDesc;
+        pszName = pszEnd;
+        pRoot   = pChild;
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Finds the first sample descriptor for a given lookup range.
+ *
+ * This is for pattern range lookups.
+ *
+ * @returns Pointer to the first descriptor.
+ * @param   pFirst              The first node in the range.
+ * @param   pLast               The last node in the range.
+ */
+static PSTAMDESC stamR3LookupFindFirstDescForRange(PSTAMLOOKUP pFirst, PSTAMLOOKUP pLast)
+{
+    if (pFirst->pDesc)
+        return pFirst->pDesc;
+
+    PSTAMLOOKUP pCur = pFirst;
+    uint32_t    iCur = 0;
+    for (;;)
+    {
+        uint32_t cChildren = pCur->cChildren;
+        if (iCur < pCur->cChildren)
+        {
+            /*
+             * Check all children.
+             */
+            PSTAMLOOKUP * const papChildren = pCur->papChildren;
+            do
+            {
+                PSTAMLOOKUP pChild = papChildren[iCur];
+                if (pChild->pDesc)
+                    return pChild->pDesc;
+                if (pChild->cChildren > 0)
+                {
+                    /* One level down. */
+                    iCur = 0;
+                    pCur = pChild;
+                    break;
+                }
+                if (pChild == pLast)
+                    return NULL;
+            } while (++iCur < cChildren);
+        }
+        else
+        {
+            /*
+             * One level up, checking current and its 'older' sibilings.
+             */
+            if (pCur == pLast)
+                return NULL;
+            iCur = pCur->iParent + 1;
+            pCur = pCur->pParent;
+            if (!pCur)
+                break;
+        }
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Finds the last sample descriptor for a given lookup range.
+ *
+ * This is for pattern range lookups.
+ *
+ * @returns Pointer to the first descriptor.
+ * @param   pFirst              The first node in the range.
+ * @param   pLast               The last node in the range.
+ */
+static PSTAMDESC stamR3LookupFindLastDescForRange(PSTAMLOOKUP pFirst, PSTAMLOOKUP pLast)
+{
+    PSTAMLOOKUP pCur = pLast;
+    uint32_t    iCur = pCur->cChildren - 1;
+    for (;;)
+    {
+        if (iCur < pCur->cChildren)
+        {
+            /*
+             * Check children backwards, depth first.
+             */
+            PSTAMLOOKUP * const papChildren = pCur->papChildren;
+            do
+            {
+                PSTAMLOOKUP pChild = papChildren[iCur];
+                if (pChild->cChildren > 0)
+                {
+                    /* One level down. */
+                    iCur = pChild->cChildren - 1;
+                    pCur = pChild;
+                    break;
+                }
+
+                if (pChild->pDesc)
+                    return pChild->pDesc;
+                if (pChild == pFirst)
+                    return NULL;
+            } while (iCur-- > 0); /* (underflow handled above) */
+        }
+        else
+        {
+            /*
+             * One level up, checking current and its 'older' sibilings.
+             */
+            if (pCur->pDesc)
+                return pCur->pDesc;
+            if (pCur == pFirst)
+                return NULL;
+            iCur = pCur->iParent - 1; /* (underflow handled above) */
+            pCur = pCur->pParent;
+            if (!pCur)
+                break;
+        }
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Look up the first and last descriptors for a (single) pattern expression.
+ *
+ * This is used to optimize pattern enumerations and doesn't have to return 100%
+ * accurate results if that costs too much.
+ *
+ * @returns Pointer to the first descriptor in the range.
+ * @param   pRoot               The root node.
+ * @param   pList               The descriptor list anchor.
+ * @param   pszPat              The name patter to lookup.
+ * @param   ppLastDesc          Where to store the address of the last
+ *                              descriptor (approximate).
+ */
+static PSTAMDESC stamR3LookupFindPatternDescRange(PSTAMLOOKUP pRoot, PRTLISTANCHOR pList, const char *pszPat,
+                                                  PSTAMDESC *ppLastDesc)
+{
+    Assert(!pRoot->pParent);
+
+    /*
+     * If there is an early enough wildcard, the whole list needs to be searched.
+     */
+    if (   pszPat[0] == '*' || pszPat[0] == '?'
+        || pszPat[1] == '*' || pszPat[1] == '?')
+    {
+        *ppLastDesc = RTListGetLast(pList, STAMDESC, ListEntry);
+        return RTListGetFirst(pList, STAMDESC, ListEntry);
+    }
+
+    /*
+     * All statistics starts with a slash.
+     */
+    while (   *pszPat++ == '/'
+           && pRoot->cDescsInTree > 0
+           && pRoot->cChildren    > 0)
+    {
+        const char *pszEnd = strchr(pszPat, '/');
+        uint32_t    cch    = pszEnd ? pszEnd - pszPat : (uint32_t)strlen(pszPat);
+        if (!cch)
+            break;
+
+        const char *pszPat1 = (const char *)memchr(pszPat, '*', cch);
+        const char *pszPat2 = (const char *)memchr(pszPat, '?', cch);
+        if (pszPat1 || pszPat2)
+        {
+            /* We've narrowed it down to a sub-tree now. */
+            PSTAMLOOKUP pFirst = pRoot->papChildren[0];
+            PSTAMLOOKUP pLast  = pRoot->papChildren[pRoot->cChildren - 1];
+            /** @todo narrow the range further if both pszPat1/2 != pszPat. */
+
+            *ppLastDesc = stamR3LookupFindLastDescForRange(pFirst, pLast);
+            return stamR3LookupFindFirstDescForRange(pFirst, pLast);
+        }
+
+        PSTAMLOOKUP pChild = stamR3LookupFindChild(pRoot, pszPat, cch, NULL);
+        if (!pChild)
+            break;
+
+        /* Advance */
+        if (!pszEnd)
+            return *ppLastDesc = pChild->pDesc;
+        pszPat = pszEnd;
+        pRoot  = pChild;
+    }
+
+    /* No match. */
+    *ppLastDesc = NULL;
+    return NULL;
+}
+
+
+/**
+ * Look up the first descriptors for starts-with name string.
+ *
+ * This is used to optimize deletion.
+ *
+ * @returns Pointer to the first descriptor in the range.
+ * @param   pRoot       The root node.
+ * @param   pchPrefix   The name prefix.
+ * @param   cchPrefix   The name prefix length (can be shorter than the
+ *                      actual string).
+ * @param   ppLastDesc  Where to store the address of the last descriptor.
+ * @sa      stamR3LookupFindPatternDescRange
+ */
+static PSTAMDESC stamR3LookupFindByPrefixRange(PSTAMLOOKUP pRoot, const char *pchPrefix, uint32_t cchPrefix,
+                                               PSTAMDESC *ppLastDesc)
+
+{
+    *ppLastDesc = NULL;
+    Assert(!pRoot->pParent);
+    AssertReturn(cchPrefix > 0, NULL);
+
+    /*
+     * We start with a root slash.
+     */
+    if (!cchPrefix || *pchPrefix != '/')
+        return NULL;
+
+    /*
+     * Walk thru the prefix component by component, since that's how
+     * the lookup tree is organized.
+     */
+    while (   cchPrefix
+           && *pchPrefix == '/'
+           && pRoot->cDescsInTree > 0
+           && pRoot->cChildren    > 0)
+    {
+        cchPrefix -= 1;
+        pchPrefix += 1;
+
+        const char *pszEnd = (const char *)memchr(pchPrefix, '/', cchPrefix);
+        if (!pszEnd)
+        {
+            /*
+             * We've narrowed it down to a sub-tree now.  If we've no more prefix to work
+             * with now (e.g. '/Devices/'), the prefix matches all the children.  Otherwise,
+             * traverse the children to find the ones matching the prefix.
+             */
+            if (!cchPrefix)
+            {
+                *ppLastDesc = stamR3LookupFindLastDescForRange(pRoot->papChildren[0], pRoot->papChildren[pRoot->cChildren - 1]);
+                return stamR3LookupFindFirstDescForRange(pRoot->papChildren[0], pRoot->papChildren[pRoot->cChildren - 1]);
+            }
+
+            size_t iEnd = pRoot->cChildren;
+            if (iEnd < 16)
+            {
+                /* Linear scan of the children: */
+                for (size_t i = 0; i < pRoot->cChildren; i++)
+                {
+                    PSTAMLOOKUP pCur = pRoot->papChildren[i];
+                    if (pCur->cch >= cchPrefix)
+                    {
+                        int iDiff = memcmp(pCur->szName, pchPrefix, cchPrefix);
+                        if (iDiff == 0)
+                        {
+                            size_t iLast = i;
+                            while (++iLast < pRoot->cChildren)
+                            {
+                                PSTAMLOOKUP pCur2 = pRoot->papChildren[iLast];
+                                if (   pCur2->cch < cchPrefix
+                                    || memcmp(pCur2->szName, pchPrefix, cchPrefix) != 0)
+                                    break;
+                            }
+                            iLast--;
+
+                            *ppLastDesc = stamR3LookupFindLastDescForRange(pCur, pRoot->papChildren[iLast]);
+                            return stamR3LookupFindFirstDescForRange(pCur, pRoot->papChildren[iLast]);
+                        }
+                        if (iDiff > 0)
+                            break;
+                    }
+                }
+            }
+            else
+            {
+                /* Binary search to find something matching the prefix, followed
+                   by a reverse scan to locate the first child: */
+                size_t iFirst = 0;
+                size_t i      = iEnd / 2;
+                for (;;)
+                {
+                    PSTAMLOOKUP pCur = pRoot->papChildren[i];
+                    int iDiff;
+                    if (pCur->cch >= cchPrefix)
+                        iDiff = memcmp(pCur->szName, pchPrefix, cchPrefix);
+                    else
+                    {
+                        iDiff = memcmp(pCur->szName, pchPrefix, pCur->cch);
+                        if (!iDiff)
+                            iDiff = 1;
+                    }
+                    if (iDiff > 0)
+                    {
+                        if (iFirst < i)
+                            iEnd = i;
+                        else
+                            return NULL;
+                    }
+                    else if (iDiff < 0)
+                    {
+                        i += 1;
+                        if (i < iEnd)
+                            iFirst = i;
+                        else
+                            return NULL;
+                    }
+                    else
+                    {
+                        /* Match.  Reverse scan to find the first. */
+                        iFirst = i;
+                        while (   iFirst > 0
+                               && (pCur = pRoot->papChildren[iFirst - 1])->cch >= cchPrefix
+                               && memcmp(pCur->szName, pchPrefix, cchPrefix) == 0)
+                            iFirst--;
+
+                        /* Forward scan to find the last.*/
+                        size_t iLast = i;
+                        while (++iLast < pRoot->cChildren)
+                        {
+                            pCur = pRoot->papChildren[iLast];
+                            if (   pCur->cch < cchPrefix
+                                || memcmp(pCur->szName, pchPrefix, cchPrefix) != 0)
+                                break;
+                        }
+                        iLast--;
+
+                        *ppLastDesc = stamR3LookupFindLastDescForRange(pRoot->papChildren[iFirst], pRoot->papChildren[iLast]);
+                        return stamR3LookupFindFirstDescForRange(pRoot->papChildren[iFirst], pRoot->papChildren[iLast]);
+                    }
+
+                    i = iFirst + (iEnd - iFirst) / 2;
+                }
+            }
+            break;
+        }
+
+        /* Find child matching the path component:  */
+        uint32_t    cchChild = pszEnd - pchPrefix;
+        PSTAMLOOKUP pChild   = stamR3LookupFindChild(pRoot, pchPrefix, cchChild, NULL);
+        if (!pChild)
+            break;
+
+        /* Advance: */
+        cchPrefix -= cchChild;
+        pchPrefix  = pszEnd;
+        pRoot      = pChild;
+    }
+    return NULL;
+}
+
+
+/**
+ * Increments the cDescInTree member of the given node an all ancestors.
+ *
+ * @param   pLookup             The lookup node.
+ */
+static void stamR3LookupIncUsage(PSTAMLOOKUP pLookup)
+{
+    Assert(pLookup->pDesc);
+
+    PSTAMLOOKUP pCur = pLookup;
+    while (pCur != NULL)
+    {
+        pCur->cDescsInTree++;
+        pCur = pCur->pParent;
+    }
+}
+
+
+/**
+ * Descrements the cDescInTree member of the given node an all ancestors.
+ *
+ * @param   pLookup             The lookup node.
+ */
+static void stamR3LookupDecUsage(PSTAMLOOKUP pLookup)
+{
+    Assert(!pLookup->pDesc);
+
+    PSTAMLOOKUP pCur = pLookup;
+    while (pCur != NULL)
+    {
+        Assert(pCur->cDescsInTree > 0);
+        pCur->cDescsInTree--;
+        pCur = pCur->pParent;
+    }
+}
+
+
+/**
+ * Frees empty lookup nodes if it's worth it.
+ *
+ * @param   pLookup             The lookup node.
+ */
+static void stamR3LookupMaybeFree(PSTAMLOOKUP pLookup)
+{
+    Assert(!pLookup->pDesc);
+
+    /*
+     * Free between two and three levels of nodes.  Freeing too much most
+     * likely wasted effort since we're either going to repopluate the tree
+     * or quit the whole thing.
+     */
+    if (pLookup->cDescsInTree > 0)
+        return;
+
+    PSTAMLOOKUP pCur = pLookup->pParent;
+    if (!pCur)
+        return;
+    if (pCur->cDescsInTree > 0)
+        return;
+    PSTAMLOOKUP pParent = pCur->pParent;
+    if (!pParent)
+        return;
+
+    if (pParent->cDescsInTree == 0 && pParent->pParent)
+    {
+        pCur = pParent;
+        pParent = pCur->pParent;
+    }
+
+    /*
+     * Remove pCur from pParent.
+     */
+    PSTAMLOOKUP *papChildren = pParent->papChildren;
+    uint32_t     cChildren   = --pParent->cChildren;
+    for (uint32_t i = pCur->iParent; i < cChildren; i++)
+    {
+        PSTAMLOOKUP pChild = papChildren[i + 1];
+        pChild->iParent = i;
+        papChildren[i] = pChild;
+    }
+    pCur->pParent = NULL;
+    pCur->iParent = UINT16_MAX;
+
+    /*
+     * Destroy pCur.
+     */
+    stamR3LookupDestroyTree(pCur);
+}
+
+
+/**
+ * Destroys a lookup tree.
+ *
+ * This is used by STAMR3Term as well as stamR3LookupMaybeFree.
+ *
+ * @param   pRoot               The root of the tree (must have no parent).
+ */
+static void stamR3LookupDestroyTree(PSTAMLOOKUP pRoot)
+{
+    Assert(pRoot); Assert(!pRoot->pParent);
+    PSTAMLOOKUP pCur = pRoot;
+    for (;;)
+    {
+        uint32_t i = pCur->cChildren;
+        if (i > 0)
+        {
+            /*
+             * Push child (with leaf optimization).
+             */
+            PSTAMLOOKUP pChild = pCur->papChildren[--i];
+            if (pChild->cChildren != 0)
+                pCur = pChild;
+            else
+            {
+                /* free leaves. */
+                for (;;)
+                {
+                    if (pChild->papChildren)
+                    {
+                        RTMemFree(pChild->papChildren);
+                        pChild->papChildren = NULL;
+                    }
+                    RTMemFree(pChild);
+                    pCur->papChildren[i] = NULL;
+
+                    /* next */
+                    if (i == 0)
+                    {
+                        pCur->cChildren = 0;
+                        break;
+                    }
+                    pChild = pCur->papChildren[--i];
+                    if (pChild->cChildren != 0)
+                    {
+                        pCur->cChildren = i + 1;
+                        pCur = pChild;
+                        break;
+                    }
+                }
+            }
+        }
+        else
+        {
+            /*
+             * Pop and free current.
+             */
+            Assert(!pCur->pDesc);
+
+            PSTAMLOOKUP pParent = pCur->pParent;
+            Assert(pCur->iParent == (pParent ? pParent->cChildren - 1 : UINT16_MAX));
+
+            RTMemFree(pCur->papChildren);
+            pCur->papChildren = NULL;
+            RTMemFree(pCur);
+
+            pCur = pParent;
+            if (!pCur)
+                break;
+            pCur->papChildren[--pCur->cChildren] = NULL;
+        }
+    }
+}
+
+
+/**
+ * Internal worker for the different register calls.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   pfnReset    Callback for resetting the sample. NULL should be used if the sample can't be reset.
+ * @param   pfnPrint    Print the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   pszName     The sample name format string.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ * @param   iRefreshGrp The refresh group, STAM_REFRESH_GRP_XXX.
+ * @remark  There is currently no device or driver variant of this API. Add one if it should become necessary!
+ */
+static int stamR3RegisterU(PUVM pUVM, void *pvSample, PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
+                           STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+                           const char *pszName, STAMUNIT enmUnit, const char *pszDesc, uint8_t iRefreshGrp)
+{
+    AssertReturn(pszName[0] == '/', VERR_INVALID_NAME);
+    AssertReturn(pszName[1] != '/' && pszName[1], VERR_INVALID_NAME);
+    uint32_t const cchName = (uint32_t)strlen(pszName);
+    AssertReturn(cchName <= STAM_MAX_NAME_LEN, VERR_OUT_OF_RANGE);
+    AssertReturn(pszName[cchName - 1] != '/', VERR_INVALID_NAME);
+    AssertReturn(memchr(pszName, '\\', cchName) == NULL, VERR_INVALID_NAME);
+    AssertReturn(iRefreshGrp == STAM_REFRESH_GRP_NONE || iRefreshGrp < 64, VERR_INVALID_PARAMETER);
+
+    STAM_LOCK_WR(pUVM);
+
+    /*
+     * Look up the tree location, populating the lookup tree as we walk it.
+     */
+    PSTAMLOOKUP pLookup = pUVM->stam.s.pRoot; Assert(pLookup);
+    uint32_t    offName = 1;
+    for (;;)
+    {
+        /* Get the next part of the path. */
+        const char *pszStart = &pszName[offName];
+        const char *pszEnd   = strchr(pszStart, '/');
+        uint32_t    cch      = pszEnd ? (uint32_t)(pszEnd - pszStart) : cchName - offName;
+        if (cch == 0)
+        {
+            STAM_UNLOCK_WR(pUVM);
+            AssertMsgFailed(("No double or trailing slashes are allowed: '%s'\n", pszName));
+            return VERR_INVALID_NAME;
+        }
+
+        /* Do the looking up. */
+        uint32_t    iChild = 0;
+        PSTAMLOOKUP pChild = stamR3LookupFindChild(pLookup, pszStart, cch, &iChild);
+        if (!pChild)
+        {
+            pChild = stamR3LookupNewChild(pLookup, pszStart, cch, offName, iChild);
+            if (!pChild)
+            {
+                STAM_UNLOCK_WR(pUVM);
+                return VERR_NO_MEMORY;
+            }
+        }
+
+        /* Advance. */
+        pLookup = pChild;
+        if (!pszEnd)
+            break;
+        offName += cch + 1;
+    }
+    if (pLookup->pDesc)
+    {
+        STAM_UNLOCK_WR(pUVM);
+        AssertMsgFailed(("Duplicate sample name: %s\n", pszName));
+        return VERR_ALREADY_EXISTS;
+    }
+
+    PSTAMDESC pCur = stamR3LookupFindNextWithDesc(pLookup);
+
+    /*
+     * Check that the name doesn't screw up sorting order when taking
+     * slashes into account. The QT GUI makes some assumptions.
+     * Problematic chars are: !"#$%&'()*+,-.
+     */
+#ifdef VBOX_STRICT
+    Assert(pszName[0] == '/');
+    PSTAMDESC pPrev = pCur
+                    ? RTListGetPrev(&pUVM->stam.s.List, pCur, STAMDESC, ListEntry)
+                    : RTListGetLast(&pUVM->stam.s.List, STAMDESC, ListEntry);
+    Assert(!pPrev || strcmp(pszName, pPrev->pszName) > 0);
+    Assert(!pCur  || strcmp(pszName, pCur->pszName)  < 0);
+    Assert(!pPrev || stamR3SlashCompare(pPrev->pszName, pszName) < 0);
+    Assert(!pCur  || stamR3SlashCompare(pCur->pszName, pszName) > 0);
+
+    /*
+     * Check alignment requirements.
+     */
+    switch (enmType)
+    {
+            /* 8 byte / 64-bit */
+        case STAMTYPE_U64:
+        case STAMTYPE_U64_RESET:
+        case STAMTYPE_X64:
+        case STAMTYPE_X64_RESET:
+        case STAMTYPE_COUNTER:
+        case STAMTYPE_PROFILE:
+        case STAMTYPE_PROFILE_ADV:
+            AssertMsg(!((uintptr_t)pvSample & 7), ("%p - %s\n", pvSample, pszName));
+            break;
+
+            /* 4 byte / 32-bit */
+        case STAMTYPE_RATIO_U32:
+        case STAMTYPE_RATIO_U32_RESET:
+        case STAMTYPE_U32:
+        case STAMTYPE_U32_RESET:
+        case STAMTYPE_X32:
+        case STAMTYPE_X32_RESET:
+            AssertMsg(!((uintptr_t)pvSample & 3), ("%p - %s\n", pvSample, pszName));
+            break;
+
+            /* 2 byte / 32-bit */
+        case STAMTYPE_U16:
+        case STAMTYPE_U16_RESET:
+        case STAMTYPE_X16:
+        case STAMTYPE_X16_RESET:
+            AssertMsg(!((uintptr_t)pvSample & 1), ("%p - %s\n", pvSample, pszName));
+            break;
+
+            /* 1 byte / 8-bit / unaligned */
+        case STAMTYPE_U8:
+        case STAMTYPE_U8_RESET:
+        case STAMTYPE_X8:
+        case STAMTYPE_X8_RESET:
+        case STAMTYPE_BOOL:
+        case STAMTYPE_BOOL_RESET:
+        case STAMTYPE_CALLBACK:
+            break;
+
+        default:
+            AssertMsgFailed(("%d\n", enmType));
+            break;
+    }
+#endif /* VBOX_STRICT */
+
+    /*
+     * Create a new node and insert it at the current location.
+     */
+    int rc;
+    size_t cbDesc = pszDesc ? strlen(pszDesc) + 1 : 0;
+    PSTAMDESC pNew = (PSTAMDESC)RTMemAlloc(sizeof(*pNew) + cchName + 1 + cbDesc);
+    if (pNew)
+    {
+        pNew->pszName       = (char *)memcpy((char *)(pNew + 1), pszName, cchName + 1);
+        pNew->enmType       = enmType;
+        pNew->enmVisibility = enmVisibility;
+        if (enmType != STAMTYPE_CALLBACK)
+            pNew->u.pv      = pvSample;
+        else
+        {
+            pNew->u.Callback.pvSample = pvSample;
+            pNew->u.Callback.pfnReset = pfnReset;
+            pNew->u.Callback.pfnPrint = pfnPrint;
+        }
+        pNew->enmUnit       = enmUnit;
+        pNew->iRefreshGroup = iRefreshGrp;
+        pNew->pszDesc       = NULL;
+        if (pszDesc)
+            pNew->pszDesc   = (char *)memcpy((char *)(pNew + 1) + cchName + 1, pszDesc, cbDesc);
+
+        if (pCur)
+            RTListNodeInsertBefore(&pCur->ListEntry, &pNew->ListEntry);
+        else
+            RTListAppend(&pUVM->stam.s.List, &pNew->ListEntry);
+
+        pNew->pLookup       = pLookup;
+        pLookup->pDesc      = pNew;
+        stamR3LookupIncUsage(pLookup);
+
+        stamR3ResetOne(pNew, pUVM->pVM);
+        rc = VINF_SUCCESS;
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    STAM_UNLOCK_WR(pUVM);
+    return rc;
+}
+
+
+/**
+ * Destroys the statistics descriptor, unlinking it and freeing all resources.
+ *
+ * @returns VINF_SUCCESS
+ * @param   pCur        The descriptor to destroy.
+ */
+static int stamR3DestroyDesc(PSTAMDESC pCur)
+{
+    RTListNodeRemove(&pCur->ListEntry);
+    pCur->pLookup->pDesc = NULL; /** @todo free lookup nodes once it's working. */
+    stamR3LookupDecUsage(pCur->pLookup);
+    stamR3LookupMaybeFree(pCur->pLookup);
+    RTMemFree(pCur);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters a sample previously registered by STAR3Register() given its
+ * address.
+ *
+ * This is intended used for devices which can be unplugged and for
+ * temporary samples.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pvSample    Pointer to the sample registered with STAMR3Register().
+ */
+VMMR3DECL(int)  STAMR3DeregisterByAddr(PUVM pUVM, void *pvSample)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /* This is a complete waste of time when shutting down. */
+    VMSTATE enmState = VMR3GetStateU(pUVM);
+    if (enmState >= VMSTATE_DESTROYING)
+        return VINF_SUCCESS;
+
+    STAM_LOCK_WR(pUVM);
+
+    /*
+     * Search for it.
+     */
+    int         rc = VERR_INVALID_HANDLE;
+    PSTAMDESC   pCur, pNext;
+    RTListForEachSafe(&pUVM->stam.s.List, pCur, pNext, STAMDESC, ListEntry)
+    {
+        if (pCur->u.pv == pvSample)
+            rc = stamR3DestroyDesc(pCur);
+    }
+
+    STAM_UNLOCK_WR(pUVM);
+    return rc;
+}
+
+
+/**
+ * Worker for STAMR3Deregister, STAMR3DeregisterV and STAMR3DeregisterF.
+ *
+ * @returns VBox status code.
+ * @retval  VWRN_NOT_FOUND if no matching names found.
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pszPat      The name pattern.
+ */
+static int stamR3DeregisterByPattern(PUVM pUVM, const char *pszPat)
+{
+    Assert(!strchr(pszPat, '|')); /* single pattern! */
+
+    int rc = VWRN_NOT_FOUND;
+    STAM_LOCK_WR(pUVM);
+
+    PSTAMDESC pLast;
+    PSTAMDESC pCur = stamR3LookupFindPatternDescRange(pUVM->stam.s.pRoot, &pUVM->stam.s.List, pszPat, &pLast);
+    if (pCur)
+    {
+        for (;;)
+        {
+            PSTAMDESC pNext = RTListNodeGetNext(&pCur->ListEntry, STAMDESC, ListEntry);
+
+            if (RTStrSimplePatternMatch(pszPat, pCur->pszName))
+                rc = stamR3DestroyDesc(pCur);
+
+            /* advance. */
+            if (pCur == pLast)
+                break;
+            pCur = pNext;
+        }
+        Assert(pLast);
+    }
+    else
+        Assert(!pLast);
+
+    STAM_UNLOCK_WR(pUVM);
+    return rc;
+}
+
+
+/**
+ * Deregister zero or more samples given a (single) pattern matching their
+ * names.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pszPat      The name pattern.
+ * @sa      STAMR3DeregisterF, STAMR3DeregisterV
+ */
+VMMR3DECL(int)  STAMR3Deregister(PUVM pUVM, const char *pszPat)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /* This is a complete waste of time when shutting down. */
+    VMSTATE enmState = VMR3GetStateU(pUVM);
+    if (enmState >= VMSTATE_DESTROYING)
+        return VINF_SUCCESS;
+
+    return stamR3DeregisterByPattern(pUVM, pszPat);
+}
+
+
+/**
+ * Deregister zero or more samples given a (single) pattern matching their
+ * names.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pszPatFmt   The name pattern format string.
+ * @param   ...         Format string arguments.
+ * @sa      STAMR3Deregister, STAMR3DeregisterV
+ */
+VMMR3DECL(int)  STAMR3DeregisterF(PUVM pUVM, const char *pszPatFmt, ...)
+{
+    va_list va;
+    va_start(va, pszPatFmt);
+    int rc = STAMR3DeregisterV(pUVM, pszPatFmt, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Deregister zero or more samples given a (single) pattern matching their
+ * names.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pszPatFmt   The name pattern format string.
+ * @param   va          Format string arguments.
+ * @sa      STAMR3Deregister, STAMR3DeregisterF
+ */
+VMMR3DECL(int)  STAMR3DeregisterV(PUVM pUVM, const char *pszPatFmt, va_list va)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /* This is a complete waste of time when shutting down. */
+    VMSTATE enmState = VMR3GetStateU(pUVM);
+    if (enmState >= VMSTATE_DESTROYING)
+        return VINF_SUCCESS;
+
+    char   szPat[STAM_MAX_NAME_LEN + 8];
+    size_t cchPat = RTStrPrintfV(szPat, sizeof(szPat), pszPatFmt, va);
+    AssertReturn(cchPat <= STAM_MAX_NAME_LEN, VERR_OUT_OF_RANGE);
+
+    return stamR3DeregisterByPattern(pUVM, szPat);
+}
+
+
+/**
+ * Deregister zero or more samples given their name prefix.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pszPrefix   The name prefix of the samples to remove.
+ * @sa      STAMR3Deregister, STAMR3DeregisterF, STAMR3DeregisterV
+ */
+VMMR3DECL(int)  STAMR3DeregisterByPrefix(PUVM pUVM, const char *pszPrefix)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /* This is a complete waste of time when shutting down. */
+    VMSTATE enmState = VMR3GetStateU(pUVM);
+    if (enmState >= VMSTATE_DESTROYING)
+        return VINF_SUCCESS;
+
+    size_t const cchPrefix = strlen(pszPrefix);
+    int          rc        = VWRN_NOT_FOUND;
+    STAM_LOCK_WR(pUVM);
+
+    PSTAMDESC pLast;
+    PSTAMDESC pCur = stamR3LookupFindByPrefixRange(pUVM->stam.s.pRoot, pszPrefix, (uint32_t)cchPrefix, &pLast);
+    if (pCur)
+        for (;;)
+        {
+            PSTAMDESC const pNext = RTListNodeGetNext(&pCur->ListEntry, STAMDESC, ListEntry);
+            Assert(strncmp(pCur->pszName, pszPrefix, cchPrefix) == 0);
+
+            rc = stamR3DestroyDesc(pCur);
+
+            /* advance. */
+            if (pCur == pLast)
+                break;
+            pCur = pNext;
+        }
+
+    STAM_UNLOCK_WR(pUVM);
+    return rc;
+}
+
+
+/**
+ * Resets statistics for the specified VM.
+ * It's possible to select a subset of the samples.
+ *
+ * @returns VBox status code. (Basically, it cannot fail.)
+ * @param   pUVM        The user mode VM handle.
+ * @param   pszPat      The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                      If NULL all samples are reset.
+ * @remarks Don't confuse this with the other 'XYZR3Reset' methods, it's not called at VM reset.
+ */
+VMMR3DECL(int)  STAMR3Reset(PUVM pUVM, const char *pszPat)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+
+    int rc = VINF_SUCCESS;
+
+    /* ring-0 */
+    GVMMRESETSTATISTICSSREQ GVMMReq;
+    GMMRESETSTATISTICSSREQ  GMMReq;
+    bool fGVMMMatched = !pszPat || !*pszPat;
+    bool fGMMMatched  = fGVMMMatched;
+    if (fGVMMMatched)
+    {
+        memset(&GVMMReq.Stats, 0xff, sizeof(GVMMReq.Stats));
+        memset(&GMMReq.Stats,  0xff, sizeof(GMMReq.Stats));
+    }
+    else
+    {
+        char *pszCopy;
+        unsigned cExpressions;
+        char **papszExpressions = stamR3SplitPattern(pszPat, &cExpressions, &pszCopy);
+        if (!papszExpressions)
+            return VERR_NO_MEMORY;
+
+        /* GVMM */
+        RT_ZERO(GVMMReq.Stats);
+        for (unsigned i = 0; i < RT_ELEMENTS(g_aGVMMStats); i++)
+            if (stamR3MultiMatch(papszExpressions, cExpressions, NULL, g_aGVMMStats[i].pszName))
+            {
+                *((uint8_t *)&GVMMReq.Stats + g_aGVMMStats[i].offVar) = 0xff;
+                fGVMMMatched = true;
+            }
+        if (!fGVMMMatched)
+        {
+            /** @todo match cpu leaves some rainy day. */
+        }
+
+        /* GMM */
+        RT_ZERO(GMMReq.Stats);
+        for (unsigned i = 0; i < RT_ELEMENTS(g_aGMMStats); i++)
+            if (stamR3MultiMatch(papszExpressions, cExpressions, NULL, g_aGMMStats[i].pszName))
+            {
+                 *((uint8_t *)&GMMReq.Stats + g_aGMMStats[i].offVar) = 0xff;
+                 fGMMMatched = true;
+            }
+
+        RTMemTmpFree(papszExpressions);
+        RTStrFree(pszCopy);
+    }
+
+    STAM_LOCK_WR(pUVM);
+
+    if (fGVMMMatched)
+    {
+        PVM pVM = pUVM->pVM;
+        GVMMReq.Hdr.cbReq    = sizeof(GVMMReq);
+        GVMMReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+        GVMMReq.pSession     = pVM->pSession;
+        rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_GVMM_RESET_STATISTICS, 0, &GVMMReq.Hdr);
+    }
+
+    if (fGMMMatched)
+    {
+        PVM pVM = pUVM->pVM;
+        GMMReq.Hdr.cbReq    = sizeof(GMMReq);
+        GMMReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+        GMMReq.pSession     = pVM->pSession;
+        rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_GMM_RESET_STATISTICS, 0, &GMMReq.Hdr);
+    }
+
+    /* and the reset */
+    stamR3EnumU(pUVM, pszPat, false /* fUpdateRing0 */, stamR3ResetOne, pUVM->pVM);
+
+    STAM_UNLOCK_WR(pUVM);
+    return rc;
+}
+
+
+/**
+ * Resets one statistics sample.
+ * Callback for stamR3EnumU().
+ *
+ * @returns VINF_SUCCESS
+ * @param   pDesc   Pointer to the current descriptor.
+ * @param   pvArg   User argument - Pointer to the VM.
+ */
+static int stamR3ResetOne(PSTAMDESC pDesc, void *pvArg)
+{
+    switch (pDesc->enmType)
+    {
+        case STAMTYPE_COUNTER:
+            ASMAtomicXchgU64(&pDesc->u.pCounter->c, 0);
+            break;
+
+        case STAMTYPE_PROFILE:
+        case STAMTYPE_PROFILE_ADV:
+            ASMAtomicXchgU64(&pDesc->u.pProfile->cPeriods, 0);
+            ASMAtomicXchgU64(&pDesc->u.pProfile->cTicks, 0);
+            ASMAtomicXchgU64(&pDesc->u.pProfile->cTicksMax, 0);
+            ASMAtomicXchgU64(&pDesc->u.pProfile->cTicksMin, UINT64_MAX);
+            break;
+
+        case STAMTYPE_RATIO_U32_RESET:
+            ASMAtomicXchgU32(&pDesc->u.pRatioU32->u32A, 0);
+            ASMAtomicXchgU32(&pDesc->u.pRatioU32->u32B, 0);
+            break;
+
+        case STAMTYPE_CALLBACK:
+            if (pDesc->u.Callback.pfnReset)
+                pDesc->u.Callback.pfnReset((PVM)pvArg, pDesc->u.Callback.pvSample);
+            break;
+
+        case STAMTYPE_U8_RESET:
+        case STAMTYPE_X8_RESET:
+            ASMAtomicXchgU8(pDesc->u.pu8, 0);
+            break;
+
+        case STAMTYPE_U16_RESET:
+        case STAMTYPE_X16_RESET:
+            ASMAtomicXchgU16(pDesc->u.pu16, 0);
+            break;
+
+        case STAMTYPE_U32_RESET:
+        case STAMTYPE_X32_RESET:
+            ASMAtomicXchgU32(pDesc->u.pu32, 0);
+            break;
+
+        case STAMTYPE_U64_RESET:
+        case STAMTYPE_X64_RESET:
+            ASMAtomicXchgU64(pDesc->u.pu64, 0);
+            break;
+
+        case STAMTYPE_BOOL_RESET:
+            ASMAtomicXchgBool(pDesc->u.pf, false);
+            break;
+
+        /* These are custom and will not be touched. */
+        case STAMTYPE_U8:
+        case STAMTYPE_X8:
+        case STAMTYPE_U16:
+        case STAMTYPE_X16:
+        case STAMTYPE_U32:
+        case STAMTYPE_X32:
+        case STAMTYPE_U64:
+        case STAMTYPE_X64:
+        case STAMTYPE_RATIO_U32:
+        case STAMTYPE_BOOL:
+            break;
+
+        default:
+            AssertMsgFailed(("enmType=%d\n", pDesc->enmType));
+            break;
+    }
+    NOREF(pvArg);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get a snapshot of the statistics.
+ * It's possible to select a subset of the samples.
+ *
+ * @returns VBox status code. (Basically, it cannot fail.)
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are reset.
+ * @param   fWithDesc       Whether to include the descriptions.
+ * @param   ppszSnapshot    Where to store the pointer to the snapshot data.
+ *                          The format of the snapshot should be XML, but that will have to be discussed
+ *                          when this function is implemented.
+ *                          The returned pointer must be freed by calling STAMR3SnapshotFree().
+ * @param   pcchSnapshot    Where to store the size of the snapshot data. (Excluding the trailing '\0')
+ */
+VMMR3DECL(int) STAMR3Snapshot(PUVM pUVM, const char *pszPat, char **ppszSnapshot, size_t *pcchSnapshot, bool fWithDesc)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+
+    STAMR3SNAPSHOTONE State = { NULL, NULL, NULL, pUVM->pVM, 0, VINF_SUCCESS, fWithDesc };
+
+    /*
+     * Write the XML header.
+     */
+    /** @todo Make this proper & valid XML. */
+    stamR3SnapshotPrintf(&State, "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n");
+
+    /*
+     * Write the content.
+     */
+    stamR3SnapshotPrintf(&State, "<Statistics>\n");
+    int rc = stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3SnapshotOne, &State);
+    stamR3SnapshotPrintf(&State, "</Statistics>\n");
+
+    if (RT_SUCCESS(rc))
+        rc = State.rc;
+    else
+    {
+        RTMemFree(State.pszStart);
+        State.pszStart = State.pszEnd = State.psz = NULL;
+        State.cbAllocated = 0;
+    }
+
+    /*
+     * Done.
+     */
+    *ppszSnapshot = State.pszStart;
+    if (pcchSnapshot)
+        *pcchSnapshot = State.psz - State.pszStart;
+    return rc;
+}
+
+
+/**
+ * stamR3EnumU callback employed by STAMR3Snapshot.
+ *
+ * @returns VBox status code, but it's interpreted as 0 == success / !0 == failure by enmR3Enum.
+ * @param   pDesc       The sample.
+ * @param   pvArg       The snapshot status structure.
+ */
+static int stamR3SnapshotOne(PSTAMDESC pDesc, void *pvArg)
+{
+    PSTAMR3SNAPSHOTONE pThis = (PSTAMR3SNAPSHOTONE)pvArg;
+
+    switch (pDesc->enmType)
+    {
+        case STAMTYPE_COUNTER:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && pDesc->u.pCounter->c == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<Counter c=\"%lld\"", pDesc->u.pCounter->c);
+            break;
+
+        case STAMTYPE_PROFILE:
+        case STAMTYPE_PROFILE_ADV:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && pDesc->u.pProfile->cPeriods == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<Profile cPeriods=\"%lld\" cTicks=\"%lld\" cTicksMin=\"%lld\" cTicksMax=\"%lld\"",
+                                 pDesc->u.pProfile->cPeriods, pDesc->u.pProfile->cTicks, pDesc->u.pProfile->cTicksMin,
+                                 pDesc->u.pProfile->cTicksMax);
+            break;
+
+        case STAMTYPE_RATIO_U32:
+        case STAMTYPE_RATIO_U32_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && !pDesc->u.pRatioU32->u32A && !pDesc->u.pRatioU32->u32B)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<Ratio32 u32A=\"%lld\" u32B=\"%lld\"",
+                                 pDesc->u.pRatioU32->u32A, pDesc->u.pRatioU32->u32B);
+            break;
+
+        case STAMTYPE_CALLBACK:
+        {
+            char szBuf[512];
+            pDesc->u.Callback.pfnPrint(pThis->pVM, pDesc->u.Callback.pvSample, szBuf, sizeof(szBuf));
+            stamR3SnapshotPrintf(pThis, "<Callback val=\"%s\"", szBuf);
+            break;
+        }
+
+        case STAMTYPE_U8:
+        case STAMTYPE_U8_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu8 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<U8 val=\"%u\"", *pDesc->u.pu8);
+            break;
+
+        case STAMTYPE_X8:
+        case STAMTYPE_X8_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu8 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<X8 val=\"%#x\"", *pDesc->u.pu8);
+            break;
+
+        case STAMTYPE_U16:
+        case STAMTYPE_U16_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu16 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<U16 val=\"%u\"", *pDesc->u.pu16);
+            break;
+
+        case STAMTYPE_X16:
+        case STAMTYPE_X16_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu16 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<X16 val=\"%#x\"", *pDesc->u.pu16);
+            break;
+
+        case STAMTYPE_U32:
+        case STAMTYPE_U32_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu32 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<U32 val=\"%u\"", *pDesc->u.pu32);
+            break;
+
+        case STAMTYPE_X32:
+        case STAMTYPE_X32_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu32 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<X32 val=\"%#x\"", *pDesc->u.pu32);
+            break;
+
+        case STAMTYPE_U64:
+        case STAMTYPE_U64_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu64 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<U64 val=\"%llu\"", *pDesc->u.pu64);
+            break;
+
+        case STAMTYPE_X64:
+        case STAMTYPE_X64_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu64 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<X64 val=\"%#llx\"", *pDesc->u.pu64);
+            break;
+
+        case STAMTYPE_BOOL:
+        case STAMTYPE_BOOL_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pf == false)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<BOOL val=\"%RTbool\"", *pDesc->u.pf);
+            break;
+
+        default:
+            AssertMsgFailed(("%d\n", pDesc->enmType));
+            return 0;
+    }
+
+    stamR3SnapshotPrintf(pThis, " unit=\"%s\"", STAMR3GetUnit(pDesc->enmUnit));
+
+    switch (pDesc->enmVisibility)
+    {
+        default:
+        case STAMVISIBILITY_ALWAYS:
+            break;
+        case STAMVISIBILITY_USED:
+            stamR3SnapshotPrintf(pThis, " vis=\"used\"");
+            break;
+        case STAMVISIBILITY_NOT_GUI:
+            stamR3SnapshotPrintf(pThis, " vis=\"not-gui\"");
+            break;
+    }
+
+    stamR3SnapshotPrintf(pThis, " name=\"%s\"", pDesc->pszName);
+
+    if (pThis->fWithDesc && pDesc->pszDesc)
+    {
+        /*
+         * The description is a bit tricky as it may include chars that
+         * xml requires to be escaped.
+         */
+        const char *pszBadChar = strpbrk(pDesc->pszDesc, "&<>\"'");
+        if (!pszBadChar)
+            return stamR3SnapshotPrintf(pThis, " desc=\"%s\"/>\n", pDesc->pszDesc);
+
+        stamR3SnapshotPrintf(pThis, " desc=\"");
+        const char *pszCur = pDesc->pszDesc;
+        do
+        {
+            stamR3SnapshotPrintf(pThis, "%.*s", pszBadChar - pszCur, pszCur);
+            switch (*pszBadChar)
+            {
+                case '&':   stamR3SnapshotPrintf(pThis, "&amp;");   break;
+                case '<':   stamR3SnapshotPrintf(pThis, "&lt;");    break;
+                case '>':   stamR3SnapshotPrintf(pThis, "&gt;");    break;
+                case '"':   stamR3SnapshotPrintf(pThis, "&quot;");  break;
+                case '\'':  stamR3SnapshotPrintf(pThis, "&apos;");  break;
+                default:    AssertMsgFailed(("%c", *pszBadChar));    break;
+            }
+            pszCur = pszBadChar + 1;
+            pszBadChar = strpbrk(pszCur, "&<>\"'");
+        } while (pszBadChar);
+        return stamR3SnapshotPrintf(pThis, "%s\"/>\n", pszCur);
+    }
+    return stamR3SnapshotPrintf(pThis, "/>\n");
+}
+
+
+/**
+ * Output callback for stamR3SnapshotPrintf.
+ *
+ * @returns number of bytes written.
+ * @param   pvArg       The snapshot status structure.
+ * @param   pach        Pointer to an array of characters (bytes).
+ * @param   cch         The number or chars (bytes) to write from the array.
+ */
+static DECLCALLBACK(size_t) stamR3SnapshotOutput(void *pvArg, const char *pach, size_t cch)
+{
+    PSTAMR3SNAPSHOTONE pThis = (PSTAMR3SNAPSHOTONE)pvArg;
+
+    /*
+     * Make sure we've got space for it.
+     */
+    if (RT_UNLIKELY((uintptr_t)pThis->pszEnd - (uintptr_t)pThis->psz < cch + 1))
+    {
+        if (RT_FAILURE(pThis->rc))
+            return 0;
+
+        size_t cbNewSize = pThis->cbAllocated;
+        if (cbNewSize > cch)
+            cbNewSize *= 2;
+        else
+            cbNewSize += RT_ALIGN(cch + 1, 0x1000);
+        char *pszNew = (char *)RTMemRealloc(pThis->pszStart, cbNewSize);
+        if (!pszNew)
+        {
+            /*
+             * Free up immediately, out-of-memory is bad news and this
+             * isn't an important allocations / API.
+             */
+            pThis->rc = VERR_NO_MEMORY;
+            RTMemFree(pThis->pszStart);
+            pThis->pszStart = pThis->pszEnd = pThis->psz = NULL;
+            pThis->cbAllocated = 0;
+            return 0;
+        }
+
+        pThis->psz = pszNew + (pThis->psz - pThis->pszStart);
+        pThis->pszStart = pszNew;
+        pThis->pszEnd = pszNew + cbNewSize;
+        pThis->cbAllocated = cbNewSize;
+    }
+
+    /*
+     * Copy the chars to the buffer and terminate it.
+     */
+    if (cch)
+    {
+        memcpy(pThis->psz, pach, cch);
+        pThis->psz += cch;
+    }
+    *pThis->psz = '\0';
+    return cch;
+}
+
+
+/**
+ * Wrapper around RTStrFormatV for use by the snapshot API.
+ *
+ * @returns VBox status code.
+ * @param   pThis       The snapshot status structure.
+ * @param   pszFormat   The format string.
+ * @param   ...         Optional arguments.
+ */
+static int stamR3SnapshotPrintf(PSTAMR3SNAPSHOTONE pThis, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    RTStrFormatV(stamR3SnapshotOutput, pThis, NULL, NULL, pszFormat, va);
+    va_end(va);
+    return pThis->rc;
+}
+
+
+/**
+ * Releases a statistics snapshot returned by STAMR3Snapshot().
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszSnapshot     The snapshot data pointer returned by STAMR3Snapshot().
+ *                          NULL is allowed.
+ */
+VMMR3DECL(int)  STAMR3SnapshotFree(PUVM pUVM, char *pszSnapshot)
+{
+    if (pszSnapshot)
+        RTMemFree(pszSnapshot);
+    NOREF(pUVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps the selected statistics to the log.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are written to the log.
+ */
+VMMR3DECL(int)  STAMR3Dump(PUVM pUVM, const char *pszPat)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+
+    STAMR3PRINTONEARGS Args;
+    Args.pUVM = pUVM;
+    Args.pvArg = NULL;
+    Args.pfnPrintf = stamR3EnumLogPrintf;
+
+    stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Prints to the log.
+ *
+ * @param   pArgs       Pointer to the print one argument structure.
+ * @param   pszFormat   Format string.
+ * @param   ...         Format arguments.
+ */
+static DECLCALLBACK(void) stamR3EnumLogPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    RTLogPrintfV(pszFormat, va);
+    va_end(va);
+    NOREF(pArgs);
+}
+
+
+/**
+ * Dumps the selected statistics to the release log.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are written to the log.
+ */
+VMMR3DECL(int)  STAMR3DumpToReleaseLog(PUVM pUVM, const char *pszPat)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+
+    STAMR3PRINTONEARGS Args;
+    Args.pUVM = pUVM;
+    Args.pvArg = NULL;
+    Args.pfnPrintf = stamR3EnumRelLogPrintf;
+
+    stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
+    return VINF_SUCCESS;
+}
+
+/**
+ * Prints to the release log.
+ *
+ * @param   pArgs       Pointer to the print one argument structure.
+ * @param   pszFormat   Format string.
+ * @param   ...         Format arguments.
+ */
+static DECLCALLBACK(void) stamR3EnumRelLogPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    RTLogRelPrintfV(pszFormat, va);
+    va_end(va);
+    NOREF(pArgs);
+}
+
+
+/**
+ * Prints the selected statistics to standard out.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The user mode VM handle.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are reset.
+ */
+VMMR3DECL(int)  STAMR3Print(PUVM pUVM, const char *pszPat)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+
+    STAMR3PRINTONEARGS Args;
+    Args.pUVM = pUVM;
+    Args.pvArg = NULL;
+    Args.pfnPrintf = stamR3EnumPrintf;
+
+    stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Prints to stdout.
+ *
+ * @param   pArgs       Pointer to the print one argument structure.
+ * @param   pszFormat   Format string.
+ * @param   ...         Format arguments.
+ */
+static DECLCALLBACK(void) stamR3EnumPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    RTPrintfV(pszFormat, va);
+    va_end(va);
+    NOREF(pArgs);
+}
+
+
+/**
+ * Prints one sample.
+ * Callback for stamR3EnumU().
+ *
+ * @returns VINF_SUCCESS
+ * @param   pDesc   Pointer to the current descriptor.
+ * @param   pvArg   User argument - STAMR3PRINTONEARGS.
+ */
+static int stamR3PrintOne(PSTAMDESC pDesc, void *pvArg)
+{
+    PSTAMR3PRINTONEARGS pArgs = (PSTAMR3PRINTONEARGS)pvArg;
+
+    switch (pDesc->enmType)
+    {
+        case STAMTYPE_COUNTER:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && pDesc->u.pCounter->c == 0)
+                return VINF_SUCCESS;
+
+            pArgs->pfnPrintf(pArgs, "%-32s %8llu %s\n", pDesc->pszName, pDesc->u.pCounter->c, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_PROFILE:
+        case STAMTYPE_PROFILE_ADV:
+        {
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && pDesc->u.pProfile->cPeriods == 0)
+                return VINF_SUCCESS;
+
+            uint64_t u64 = pDesc->u.pProfile->cPeriods ? pDesc->u.pProfile->cPeriods : 1;
+            pArgs->pfnPrintf(pArgs, "%-32s %8llu %s (%12llu ticks, %7llu times, max %9llu, min %7lld)\n", pDesc->pszName,
+                             pDesc->u.pProfile->cTicks / u64, STAMR3GetUnit(pDesc->enmUnit),
+                             pDesc->u.pProfile->cTicks, pDesc->u.pProfile->cPeriods, pDesc->u.pProfile->cTicksMax, pDesc->u.pProfile->cTicksMin);
+            break;
+        }
+
+        case STAMTYPE_RATIO_U32:
+        case STAMTYPE_RATIO_U32_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && !pDesc->u.pRatioU32->u32A && !pDesc->u.pRatioU32->u32B)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8u:%-8u %s\n", pDesc->pszName,
+                             pDesc->u.pRatioU32->u32A, pDesc->u.pRatioU32->u32B, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_CALLBACK:
+        {
+            char szBuf[512];
+            pDesc->u.Callback.pfnPrint(pArgs->pUVM->pVM, pDesc->u.Callback.pvSample, szBuf, sizeof(szBuf));
+            pArgs->pfnPrintf(pArgs, "%-32s %s %s\n", pDesc->pszName, szBuf, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+        }
+
+        case STAMTYPE_U8:
+        case STAMTYPE_U8_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu8 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8u %s\n", pDesc->pszName, *pDesc->u.pu8, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_X8:
+        case STAMTYPE_X8_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu8 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8x %s\n", pDesc->pszName, *pDesc->u.pu8, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_U16:
+        case STAMTYPE_U16_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu16 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8u %s\n", pDesc->pszName, *pDesc->u.pu16, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_X16:
+        case STAMTYPE_X16_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu16 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8x %s\n", pDesc->pszName, *pDesc->u.pu16, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_U32:
+        case STAMTYPE_U32_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu32 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8u %s\n", pDesc->pszName, *pDesc->u.pu32, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_X32:
+        case STAMTYPE_X32_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu32 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8x %s\n", pDesc->pszName, *pDesc->u.pu32, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_U64:
+        case STAMTYPE_U64_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu64 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8llu %s\n", pDesc->pszName, *pDesc->u.pu64, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_X64:
+        case STAMTYPE_X64_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu64 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8llx %s\n", pDesc->pszName, *pDesc->u.pu64, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_BOOL:
+        case STAMTYPE_BOOL_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pf == false)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %s %s\n", pDesc->pszName, *pDesc->u.pf ? "true    " : "false   ", STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        default:
+            AssertMsgFailed(("enmType=%d\n", pDesc->enmType));
+            break;
+    }
+    NOREF(pvArg);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Enumerate the statistics by the means of a callback function.
+ *
+ * @returns Whatever the callback returns.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   pszPat      The pattern to match samples.
+ * @param   pfnEnum     The callback function.
+ * @param   pvUser      The pvUser argument of the callback function.
+ */
+VMMR3DECL(int) STAMR3Enum(PUVM pUVM, const char *pszPat, PFNSTAMR3ENUM pfnEnum, void *pvUser)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+
+    STAMR3ENUMONEARGS Args;
+    Args.pVM     = pUVM->pVM;
+    Args.pfnEnum = pfnEnum;
+    Args.pvUser  = pvUser;
+
+    return stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3EnumOne, &Args);
+}
+
+
+/**
+ * Callback function for STARTR3Enum().
+ *
+ * @returns whatever the callback returns.
+ * @param   pDesc       Pointer to the current descriptor.
+ * @param   pvArg       Points to a STAMR3ENUMONEARGS structure.
+ */
+static int stamR3EnumOne(PSTAMDESC pDesc, void *pvArg)
+{
+    PSTAMR3ENUMONEARGS pArgs = (PSTAMR3ENUMONEARGS)pvArg;
+    int rc;
+    if (pDesc->enmType == STAMTYPE_CALLBACK)
+    {
+        /* Give the enumerator something useful. */
+        char szBuf[512];
+        pDesc->u.Callback.pfnPrint(pArgs->pVM, pDesc->u.Callback.pvSample, szBuf, sizeof(szBuf));
+        rc = pArgs->pfnEnum(pDesc->pszName, pDesc->enmType, szBuf, pDesc->enmUnit,
+                            pDesc->enmVisibility, pDesc->pszDesc, pArgs->pvUser);
+    }
+    else
+        rc = pArgs->pfnEnum(pDesc->pszName, pDesc->enmType, pDesc->u.pv, pDesc->enmUnit,
+                            pDesc->enmVisibility, pDesc->pszDesc, pArgs->pvUser);
+    return rc;
+}
+
+static void stamR3RefreshGroup(PUVM pUVM, uint8_t iRefreshGroup, uint64_t *pbmRefreshedGroups)
+{
+    *pbmRefreshedGroups |= RT_BIT_64(iRefreshGroup);
+
+    PVM pVM = pUVM->pVM;
+    if (pVM && pVM->pSession)
+    {
+        switch (iRefreshGroup)
+        {
+            /*
+             * GVMM
+             */
+            case STAM_REFRESH_GRP_GVMM:
+            {
+                GVMMQUERYSTATISTICSSREQ Req;
+                Req.Hdr.cbReq    = sizeof(Req);
+                Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+                Req.pSession     = pVM->pSession;
+                int rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_GVMM_QUERY_STATISTICS, 0, &Req.Hdr);
+                if (RT_SUCCESS(rc))
+                {
+                    pUVM->stam.s.GVMMStats = Req.Stats;
+
+                    /*
+                     * Check if the number of host CPUs has changed (it will the first
+                     * time around and normally never again).
+                     */
+                    if (RT_UNLIKELY(pUVM->stam.s.GVMMStats.cHostCpus > pUVM->stam.s.cRegisteredHostCpus))
+                    {
+                        if (RT_UNLIKELY(pUVM->stam.s.GVMMStats.cHostCpus > pUVM->stam.s.cRegisteredHostCpus))
+                        {
+                            STAM_UNLOCK_RD(pUVM);
+                            STAM_LOCK_WR(pUVM);
+                            uint32_t cCpus = pUVM->stam.s.GVMMStats.cHostCpus;
+                            for (uint32_t iCpu = pUVM->stam.s.cRegisteredHostCpus; iCpu < cCpus; iCpu++)
+                            {
+                                char   szName[120];
+                                size_t cchBase = RTStrPrintf(szName, sizeof(szName), "/GVMM/HostCpus/%u", iCpu);
+                                stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].idCpu, NULL, NULL,
+                                                STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_NONE,
+                                                "Host CPU ID", STAM_REFRESH_GRP_GVMM);
+                                strcpy(&szName[cchBase], "/idxCpuSet");
+                                stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].idxCpuSet, NULL, NULL,
+                                                STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_NONE,
+                                                "CPU Set index", STAM_REFRESH_GRP_GVMM);
+                                strcpy(&szName[cchBase], "/DesiredHz");
+                                stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].uDesiredHz, NULL, NULL,
+                                                STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_HZ,
+                                                "The desired frequency", STAM_REFRESH_GRP_GVMM);
+                                strcpy(&szName[cchBase], "/CurTimerHz");
+                                stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].uTimerHz, NULL, NULL,
+                                                STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_HZ,
+                                                "The current timer frequency", STAM_REFRESH_GRP_GVMM);
+                                strcpy(&szName[cchBase], "/PPTChanges");
+                                stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].cChanges, NULL, NULL,
+                                                STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_OCCURENCES,
+                                                "RTTimerChangeInterval calls", STAM_REFRESH_GRP_GVMM);
+                                strcpy(&szName[cchBase], "/PPTStarts");
+                                stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].cStarts, NULL, NULL,
+                                                STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_OCCURENCES,
+                                                "RTTimerStart calls", STAM_REFRESH_GRP_GVMM);
+                            }
+                            pUVM->stam.s.cRegisteredHostCpus = cCpus;
+                            STAM_UNLOCK_WR(pUVM);
+                            STAM_LOCK_RD(pUVM);
+                        }
+                    }
+                }
+                break;
+            }
+
+            /*
+             * GMM
+             */
+            case STAM_REFRESH_GRP_GMM:
+            {
+                GMMQUERYSTATISTICSSREQ Req;
+                Req.Hdr.cbReq    = sizeof(Req);
+                Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+                Req.pSession     = pVM->pSession;
+                int rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_GMM_QUERY_STATISTICS, 0, &Req.Hdr);
+                if (RT_SUCCESS(rc))
+                    pUVM->stam.s.GMMStats = Req.Stats;
+                break;
+            }
+
+            /*
+             * NEM.
+             */
+            case STAM_REFRESH_GRP_NEM:
+                SUPR3CallVMMR0(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_NEM_UPDATE_STATISTICS, NULL);
+                break;
+
+            default:
+                AssertMsgFailed(("iRefreshGroup=%d\n", iRefreshGroup));
+        }
+    }
+}
+
+
+/**
+ * Refreshes the statistics behind the given entry, if necessary.
+ *
+ * This helps implement fetching global ring-0 stats into ring-3 accessible
+ * storage.  GVMM, GMM and NEM makes use of this.
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @param   pCur                The statistics descriptor which group to check
+ *                              and maybe update.
+ * @param   pbmRefreshedGroups  Bitmap tracking what has already been updated.
+ */
+DECLINLINE(void) stamR3Refresh(PUVM pUVM, PSTAMDESC pCur, uint64_t *pbmRefreshedGroups)
+{
+    uint8_t const iRefreshGroup = pCur->iRefreshGroup;
+    if (RT_LIKELY(iRefreshGroup == STAM_REFRESH_GRP_NONE))
+    { /* likely */ }
+    else if (!(*pbmRefreshedGroups & RT_BIT_64(iRefreshGroup)))
+        stamR3RefreshGroup(pUVM, iRefreshGroup, pbmRefreshedGroups);
+}
+
+
+/**
+ * Match a name against an array of patterns.
+ *
+ * @returns true if it matches, false if it doesn't match.
+ * @param   papszExpressions    The array of pattern expressions.
+ * @param   cExpressions        The number of array entries.
+ * @param   piExpression        Where to read/store the current skip index. Optional.
+ * @param   pszName             The name to match.
+ */
+static bool stamR3MultiMatch(const char * const *papszExpressions, unsigned cExpressions,
+                             unsigned *piExpression, const char *pszName)
+{
+    for (unsigned i = piExpression ? *piExpression : 0; i < cExpressions; i++)
+    {
+        const char *pszPat = papszExpressions[i];
+        if (RTStrSimplePatternMatch(pszPat, pszName))
+        {
+            /* later:
+            if (piExpression && i > *piExpression)
+            {
+                Check if we can skip some expressions.
+                Requires the expressions to be sorted.
+            }*/
+            return true;
+        }
+    }
+    return false;
+}
+
+
+/**
+ * Splits a multi pattern into single ones.
+ *
+ * @returns Pointer to an array of single patterns. Free it with RTMemTmpFree.
+ * @param   pszPat          The pattern to split.
+ * @param   pcExpressions   The number of array elements.
+ * @param   ppszCopy        The pattern copy to free using RTStrFree.
+ */
+static char **stamR3SplitPattern(const char *pszPat, unsigned *pcExpressions, char **ppszCopy)
+{
+    Assert(pszPat && *pszPat);
+
+    char *pszCopy = RTStrDup(pszPat);
+    if (!pszCopy)
+        return NULL;
+
+    /* count them & allocate array. */
+    char *psz = pszCopy;
+    unsigned cExpressions = 1;
+    while ((psz = strchr(psz, '|')) != NULL)
+        cExpressions++, psz++;
+
+    char **papszExpressions = (char **)RTMemTmpAllocZ((cExpressions + 1) * sizeof(char *));
+    if (!papszExpressions)
+    {
+        RTStrFree(pszCopy);
+        return NULL;
+    }
+
+    /* split */
+    psz = pszCopy;
+    for (unsigned i = 0;;)
+    {
+        papszExpressions[i] = psz;
+        if (++i >= cExpressions)
+            break;
+        psz = strchr(psz, '|');
+        *psz++ = '\0';
+    }
+
+    /* sort the array, putting '*' last. */
+    /** @todo sort it... */
+
+    *pcExpressions = cExpressions;
+    *ppszCopy = pszCopy;
+    return papszExpressions;
+}
+
+
+/**
+ * Enumerates the nodes selected by a pattern or all nodes if no pattern
+ * is specified.
+ *
+ * The call may lock STAM for writing before calling this function, however do
+ * not lock it for reading as this function may need to write lock STAM.
+ *
+ * @returns The rc from the callback.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszPat          Pattern.
+ * @param   fUpdateRing0    Update the stats residing in ring-0.
+ * @param   pfnCallback     Callback function which shall be called for matching nodes.
+ *                          If it returns anything but VINF_SUCCESS the enumeration is
+ *                          terminated and the status code returned to the caller.
+ * @param   pvArg           User parameter for the callback.
+ */
+static int stamR3EnumU(PUVM pUVM, const char *pszPat, bool fUpdateRing0,
+                       int (*pfnCallback)(PSTAMDESC pDesc, void *pvArg), void *pvArg)
+{
+    size_t const cchPat            = pszPat ? strlen(pszPat) : 0;
+    int          rc                = VINF_SUCCESS;
+    uint64_t     bmRefreshedGroups = 0;
+    PSTAMDESC    pCur;
+
+    /*
+     * All.
+     */
+    if (   cchPat < 1
+        || (   cchPat == 1
+            && *pszPat == '*'))
+    {
+        STAM_LOCK_RD(pUVM);
+        RTListForEach(&pUVM->stam.s.List, pCur, STAMDESC, ListEntry)
+        {
+            if (fUpdateRing0)
+                stamR3Refresh(pUVM, pCur, &bmRefreshedGroups);
+            rc = pfnCallback(pCur, pvArg);
+            if (rc)
+                break;
+        }
+        STAM_UNLOCK_RD(pUVM);
+    }
+
+    /*
+     * Single expression pattern.
+     */
+    else if (memchr(pszPat, '|', cchPat) == NULL)
+    {
+        const char  *pszAsterisk = (const char *)memchr(pszPat, '*',  cchPat);
+        const char  *pszQuestion = (const char *)memchr(pszPat, '?',  cchPat);
+
+        STAM_LOCK_RD(pUVM);
+        if (!pszAsterisk && !pszQuestion)
+        {
+            pCur = stamR3LookupFindDesc(pUVM->stam.s.pRoot, pszPat);
+            if (pCur)
+            {
+                if (fUpdateRing0)
+                    stamR3Refresh(pUVM, pCur, &bmRefreshedGroups);
+                rc = pfnCallback(pCur, pvArg);
+            }
+        }
+        /* Is this a prefix expression where we can use the lookup tree to
+           efficiently figure out the exact range? */
+        else if (   pszAsterisk == &pszPat[cchPat - 1]
+                 && pszPat[0] == '/'
+                 && !pszQuestion)
+        {
+            PSTAMDESC pLast;
+            pCur = stamR3LookupFindByPrefixRange(pUVM->stam.s.pRoot, pszPat, (uint32_t)(cchPat - 1), &pLast);
+            if (pCur)
+            {
+                for (;;)
+                {
+                    Assert(strncmp(pCur->pszName, pszPat, cchPat - 1) == 0);
+                    if (fUpdateRing0)
+                        stamR3Refresh(pUVM, pCur, &bmRefreshedGroups);
+                    rc = pfnCallback(pCur, pvArg);
+                    if (rc)
+                        break;
+                    if (pCur == pLast)
+                        break;
+                    pCur = RTListNodeGetNext(&pCur->ListEntry, STAMDESC, ListEntry);
+                }
+                Assert(pLast);
+            }
+            else
+                Assert(!pLast);
+        }
+        else
+        {
+            /* It's a more complicated pattern.  Find the approximate range
+               and scan it for matches. */
+            PSTAMDESC pLast;
+            pCur = stamR3LookupFindPatternDescRange(pUVM->stam.s.pRoot, &pUVM->stam.s.List, pszPat, &pLast);
+            if (pCur)
+            {
+                for (;;)
+                {
+                    if (RTStrSimplePatternMatch(pszPat, pCur->pszName))
+                    {
+                        if (fUpdateRing0)
+                            stamR3Refresh(pUVM, pCur, &bmRefreshedGroups);
+                        rc = pfnCallback(pCur, pvArg);
+                        if (rc)
+                            break;
+                    }
+                    if (pCur == pLast)
+                        break;
+                    pCur = RTListNodeGetNext(&pCur->ListEntry, STAMDESC, ListEntry);
+                }
+                Assert(pLast);
+            }
+            else
+                Assert(!pLast);
+        }
+        STAM_UNLOCK_RD(pUVM);
+    }
+
+    /*
+     * Multi expression pattern.
+     */
+    else
+    {
+        /*
+         * Split up the pattern first.
+         */
+        char *pszCopy;
+        unsigned cExpressions;
+        char **papszExpressions = stamR3SplitPattern(pszPat, &cExpressions, &pszCopy);
+        if (!papszExpressions)
+            return VERR_NO_MEMORY;
+
+        /*
+         * Perform the enumeration.
+         */
+        STAM_LOCK_RD(pUVM);
+        unsigned iExpression = 0;
+        RTListForEach(&pUVM->stam.s.List, pCur, STAMDESC, ListEntry)
+        {
+            if (stamR3MultiMatch(papszExpressions, cExpressions, &iExpression, pCur->pszName))
+            {
+                if (fUpdateRing0)
+                    stamR3Refresh(pUVM, pCur, &bmRefreshedGroups);
+                rc = pfnCallback(pCur, pvArg);
+                if (rc)
+                    break;
+            }
+        }
+        STAM_UNLOCK_RD(pUVM);
+
+        RTMemTmpFree(papszExpressions);
+        RTStrFree(pszCopy);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Registers the ring-0 statistics.
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+static void stamR3Ring0StatsRegisterU(PUVM pUVM)
+{
+    /* GVMM */
+    for (unsigned i = 0; i < RT_ELEMENTS(g_aGVMMStats); i++)
+        stamR3RegisterU(pUVM, (uint8_t *)&pUVM->stam.s.GVMMStats + g_aGVMMStats[i].offVar, NULL, NULL,
+                        g_aGVMMStats[i].enmType, STAMVISIBILITY_ALWAYS, g_aGVMMStats[i].pszName,
+                        g_aGVMMStats[i].enmUnit, g_aGVMMStats[i].pszDesc, STAM_REFRESH_GRP_GVMM);
+    pUVM->stam.s.cRegisteredHostCpus = 0;
+
+    /* GMM */
+    for (unsigned i = 0; i < RT_ELEMENTS(g_aGMMStats); i++)
+        stamR3RegisterU(pUVM, (uint8_t *)&pUVM->stam.s.GMMStats + g_aGMMStats[i].offVar, NULL, NULL,
+                        g_aGMMStats[i].enmType, STAMVISIBILITY_ALWAYS, g_aGMMStats[i].pszName,
+                        g_aGMMStats[i].enmUnit, g_aGMMStats[i].pszDesc, STAM_REFRESH_GRP_GMM);
+}
+
+
+/**
+ * Get the unit string.
+ *
+ * @returns Pointer to read only unit string.
+ * @param   enmUnit     The unit.
+ */
+VMMR3DECL(const char *) STAMR3GetUnit(STAMUNIT enmUnit)
+{
+    switch (enmUnit)
+    {
+        case STAMUNIT_NONE:                 return "";
+        case STAMUNIT_CALLS:                return "calls";
+        case STAMUNIT_COUNT:                return "count";
+        case STAMUNIT_BYTES:                return "bytes";
+        case STAMUNIT_PAGES:                return "pages";
+        case STAMUNIT_ERRORS:               return "errors";
+        case STAMUNIT_OCCURENCES:           return "times";
+        case STAMUNIT_TICKS:                return "ticks";
+        case STAMUNIT_TICKS_PER_CALL:       return "ticks/call";
+        case STAMUNIT_TICKS_PER_OCCURENCE:  return "ticks/time";
+        case STAMUNIT_GOOD_BAD:             return "good:bad";
+        case STAMUNIT_MEGABYTES:            return "megabytes";
+        case STAMUNIT_KILOBYTES:            return "kilobytes";
+        case STAMUNIT_NS:                   return "ns";
+        case STAMUNIT_NS_PER_CALL:          return "ns/call";
+        case STAMUNIT_NS_PER_OCCURENCE:     return "ns/time";
+        case STAMUNIT_PCT:                  return "%";
+        case STAMUNIT_HZ:                   return "Hz";
+
+        default:
+            AssertMsgFailed(("Unknown unit %d\n", enmUnit));
+            return "(?unit?)";
+    }
+}
+
+#ifdef VBOX_WITH_DEBUGGER
+
+/**
+ * @callback_method_impl{FNDBGCCMD, The '.stats' command.}
+ */
+static DECLCALLBACK(int) stamR3CmdStats(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
+{
+    /*
+     * Validate input.
+     */
+    DBGC_CMDHLP_REQ_UVM_RET(pCmdHlp, pCmd, pUVM);
+    if (RTListIsEmpty(&pUVM->stam.s.List))
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "No statistics present");
+
+    /*
+     * Do the printing.
+     */
+    STAMR3PRINTONEARGS Args;
+    Args.pUVM       = pUVM;
+    Args.pvArg      = pCmdHlp;
+    Args.pfnPrintf  = stamR3EnumDbgfPrintf;
+
+    return stamR3EnumU(pUVM, cArgs ? paArgs[0].u.pszString : NULL, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
+}
+
+
+/**
+ * Display one sample in the debugger.
+ *
+ * @param   pArgs       Pointer to the print one argument structure.
+ * @param   pszFormat   Format string.
+ * @param   ...         Format arguments.
+ */
+static DECLCALLBACK(void) stamR3EnumDbgfPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...)
+{
+    PDBGCCMDHLP pCmdHlp = (PDBGCCMDHLP)pArgs->pvArg;
+
+    va_list va;
+    va_start(va, pszFormat);
+    pCmdHlp->pfnPrintfV(pCmdHlp, NULL, pszFormat, va);
+    va_end(va);
+    NOREF(pArgs);
+}
+
+
+/**
+ * @callback_method_impl{FNDBGCCMD, The '.statsreset' command.}
+ */
+static DECLCALLBACK(int) stamR3CmdStatsReset(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
+{
+    /*
+     * Validate input.
+     */
+    DBGC_CMDHLP_REQ_UVM_RET(pCmdHlp, pCmd, pUVM);
+    if (RTListIsEmpty(&pUVM->stam.s.List))
+        return DBGCCmdHlpFail(pCmdHlp, pCmd, "No statistics present");
+
+    /*
+     * Execute reset.
+     */
+    int rc = STAMR3Reset(pUVM, cArgs ? paArgs[0].u.pszString : NULL);
+    if (RT_SUCCESS(rc))
+        return DBGCCmdHlpFailRc(pCmdHlp, pCmd, rc, "STAMR3ResetU");
+    return DBGCCmdHlpPrintf(pCmdHlp, "Statistics have been reset.\n");
+}
+
+#endif /* VBOX_WITH_DEBUGGER */
+
diff --git a/src/VBox/VMM/VMMR3/TM.cpp b/src/VBox/VMM/VMMR3/TM.cpp
new file mode 100644
index 00000000..1519010a
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/TM.cpp
@@ -0,0 +1,4071 @@
+/* $Id: TM.cpp $ */
+/** @file
+ * TM - Time Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_tm        TM - The Time Manager
+ *
+ * The Time Manager abstracts the CPU clocks and manages timers used by the VMM,
+ * device and drivers.
+ *
+ * @see grp_tm
+ *
+ *
+ * @section sec_tm_clocks   Clocks
+ *
+ * There are currently 4 clocks:
+ *   - Virtual (guest).
+ *   - Synchronous virtual (guest).
+ *   - CPU Tick (TSC) (guest).  Only current use is rdtsc emulation. Usually a
+ *     function of the virtual clock.
+ *   - Real (host).  This is only used for display updates atm.
+ *
+ * The most important clocks are the three first ones and of these the second is
+ * the most interesting.
+ *
+ *
+ * The synchronous virtual clock is tied to the virtual clock except that it
+ * will take into account timer delivery lag caused by host scheduling.  It will
+ * normally never advance beyond the head timer, and when lagging too far behind
+ * it will gradually speed up to catch up with the virtual clock.  All devices
+ * implementing time sources accessible to and used by the guest is using this
+ * clock (for timers and other things).  This ensures consistency between the
+ * time sources.
+ *
+ * The virtual clock is implemented as an offset to a monotonic, high
+ * resolution, wall clock.  The current time source is using the RTTimeNanoTS()
+ * machinery based upon the Global Info Pages (GIP), that is, we're using TSC
+ * deltas (usually 10 ms) to fill the gaps between GIP updates.  The result is
+ * a fairly high res clock that works in all contexts and on all hosts.  The
+ * virtual clock is paused when the VM isn't in the running state.
+ *
+ * The CPU tick (TSC) is normally virtualized as a function of the synchronous
+ * virtual clock, where the frequency defaults to the host cpu frequency (as we
+ * measure it). In this mode it is possible to configure the frequency. Another
+ * (non-default) option is to use the raw unmodified host TSC values. And yet
+ * another, to tie it to time spent executing guest code.  All these things are
+ * configurable should non-default behavior be desirable.
+ *
+ * The real clock is a monotonic clock (when available) with relatively low
+ * resolution, though this a bit host specific.  Note that we're currently not
+ * servicing timers using the real clock when the VM is not running, this is
+ * simply because it has not been needed yet therefore not implemented.
+ *
+ *
+ * @subsection subsec_tm_timesync Guest Time Sync / UTC time
+ *
+ * Guest time syncing is primarily taken care of by the VMM device.  The
+ * principle is very simple, the guest additions periodically asks the VMM
+ * device what the current UTC time is and makes adjustments accordingly.
+ *
+ * A complicating factor is that the synchronous virtual clock might be doing
+ * catchups and the guest perception is currently a little bit behind the world
+ * but it will (hopefully) be catching up soon as we're feeding timer interrupts
+ * at a slightly higher rate.  Adjusting the guest clock to the current wall
+ * time in the real world would be a bad idea then because the guest will be
+ * advancing too fast and run ahead of world time (if the catchup works out).
+ * To solve this problem TM provides the VMM device with an UTC time source that
+ * gets adjusted with the current lag, so that when the guest eventually catches
+ * up the lag it will be showing correct real world time.
+ *
+ *
+ * @section sec_tm_timers   Timers
+ *
+ * The timers can use any of the TM clocks described in the previous section.
+ * Each clock has its own scheduling facility, or timer queue if you like.
+ * There are a few factors which makes it a bit complex.  First, there is the
+ * usual R0 vs R3 vs. RC thing.  Then there are multiple threads, and then there
+ * is the timer thread that periodically checks whether any timers has expired
+ * without EMT noticing.  On the API level, all but the create and save APIs
+ * must be multithreaded.  EMT will always run the timers.
+ *
+ * The design is using a doubly linked list of active timers which is ordered
+ * by expire date.  This list is only modified by the EMT thread.  Updates to
+ * the list are batched in a singly linked list, which is then processed by the
+ * EMT thread at the first opportunity (immediately, next time EMT modifies a
+ * timer on that clock, or next timer timeout).  Both lists are offset based and
+ * all the elements are therefore allocated from the hyper heap.
+ *
+ * For figuring out when there is need to schedule and run timers TM will:
+ *    - Poll whenever somebody queries the virtual clock.
+ *    - Poll the virtual clocks from the EM and REM loops.
+ *    - Poll the virtual clocks from trap exit path.
+ *    - Poll the virtual clocks and calculate first timeout from the halt loop.
+ *    - Employ a thread which periodically (100Hz) polls all the timer queues.
+ *
+ *
+ * @image html TMTIMER-Statechart-Diagram.gif
+ *
+ * @section sec_tm_timer    Logging
+ *
+ * Level 2: Logs a most of the timer state transitions and queue servicing.
+ * Level 3: Logs a few oddments.
+ * Level 4: Logs TMCLOCK_VIRTUAL_SYNC catch-up events.
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_TM
+#ifdef DEBUG_bird
+# define DBGFTRACE_DISABLED /* annoying */
+#endif
+#include <VBox/vmm/tm.h>
+#include <iprt/asm-amd64-x86.h> /* for SUPGetCpuHzFromGip from sup.h  */
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/dbgftrace.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/iom.h>
+#include "TMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/log.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+
+#include <iprt/asm.h>
+#include <iprt/asm-math.h>
+#include <iprt/assert.h>
+#include <iprt/env.h>
+#include <iprt/file.h>
+#include <iprt/getopt.h>
+#include <iprt/semaphore.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/time.h>
+#include <iprt/timer.h>
+
+#include "TMInline.h"
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The current saved state version.*/
+#define TM_SAVED_STATE_VERSION  3
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static bool                 tmR3HasFixedTSC(PVM pVM);
+static uint64_t             tmR3CalibrateTSC(void);
+static DECLCALLBACK(int)    tmR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)    tmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(void)   tmR3TimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
+static void                 tmR3TimerQueueRun(PVM pVM, PTMTIMERQUEUE pQueue);
+static void                 tmR3TimerQueueRunVirtualSync(PVM pVM);
+static DECLCALLBACK(int)    tmR3SetWarpDrive(PUVM pUVM, uint32_t u32Percent);
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+static DECLCALLBACK(void)   tmR3CpuLoadTimer(PVM pVM, PTMTIMER pTimer, void *pvUser);
+#endif
+static DECLCALLBACK(void)   tmR3TimerInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void)   tmR3TimerInfoActive(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void)   tmR3InfoClocks(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void)   tmR3InfoCpuLoad(PVM pVM, PCDBGFINFOHLP pHlp, int cArgs, char **papszArgs);
+static DECLCALLBACK(VBOXSTRICTRC) tmR3CpuTickParavirtDisable(PVM pVM, PVMCPU pVCpu, void *pvData);
+static const char *         tmR3GetTSCModeName(PVM pVM);
+static const char *         tmR3GetTSCModeNameEx(TMTSCMODE enmMode);
+
+
+/**
+ * Initializes the TM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(int) TMR3Init(PVM pVM)
+{
+    LogFlow(("TMR3Init:\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompileMemberAlignment(VM, tm.s, 32);
+    AssertCompile(sizeof(pVM->tm.s) <= sizeof(pVM->tm.padding));
+    AssertCompileMemberAlignment(TM, TimerCritSect, 8);
+    AssertCompileMemberAlignment(TM, VirtualSyncLock, 8);
+
+    /*
+     * Init the structure.
+     */
+    void *pv;
+    int rc = MMHyperAlloc(pVM, sizeof(pVM->tm.s.paTimerQueuesR3[0]) * TMCLOCK_MAX, 0, MM_TAG_TM, &pv);
+    AssertRCReturn(rc, rc);
+    pVM->tm.s.paTimerQueuesR3 = (PTMTIMERQUEUE)pv;
+    pVM->tm.s.paTimerQueuesR0 = MMHyperR3ToR0(pVM, pv);
+    pVM->tm.s.paTimerQueuesRC = MMHyperR3ToRC(pVM, pv);
+
+    pVM->tm.s.offVM = RT_UOFFSETOF(VM, tm.s);
+    pVM->tm.s.idTimerCpu = pVM->cCpus - 1; /* The last CPU. */
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].enmClock        = TMCLOCK_VIRTUAL;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].u64Expire       = INT64_MAX;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].enmClock   = TMCLOCK_VIRTUAL_SYNC;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].u64Expire  = INT64_MAX;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].enmClock           = TMCLOCK_REAL;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].u64Expire          = INT64_MAX;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].enmClock            = TMCLOCK_TSC;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].u64Expire           = INT64_MAX;
+
+
+    /*
+     * We directly use the GIP to calculate the virtual time. We map the
+     * the GIP into the guest context so we can do this calculation there
+     * as well and save costly world switches.
+     */
+    PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
+    pVM->tm.s.pvGIPR3 = (void *)pGip;
+    AssertMsgReturn(pVM->tm.s.pvGIPR3, ("GIP support is now required!\n"), VERR_TM_GIP_REQUIRED);
+    AssertMsgReturn((pGip->u32Version >> 16) == (SUPGLOBALINFOPAGE_VERSION >> 16),
+                    ("Unsupported GIP version %#x! (expected=%#x)\n", pGip->u32Version, SUPGLOBALINFOPAGE_VERSION),
+                    VERR_TM_GIP_VERSION);
+
+    RTHCPHYS HCPhysGIP;
+    rc = SUPR3GipGetPhys(&HCPhysGIP);
+    AssertMsgRCReturn(rc, ("Failed to get GIP physical address!\n"), rc);
+
+#ifndef PGM_WITHOUT_MAPPINGS
+    RTGCPTR GCPtr;
+# ifdef SUP_WITH_LOTS_OF_CPUS
+    rc = MMR3HyperMapHCPhys(pVM, pVM->tm.s.pvGIPR3, NIL_RTR0PTR, HCPhysGIP, (size_t)pGip->cPages * PAGE_SIZE,
+                            "GIP", &GCPtr);
+# else
+    rc = MMR3HyperMapHCPhys(pVM, pVM->tm.s.pvGIPR3, NIL_RTR0PTR, HCPhysGIP, PAGE_SIZE, "GIP", &GCPtr);
+# endif
+    if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Failed to map GIP into GC, rc=%Rrc!\n", rc));
+        return rc;
+    }
+    pVM->tm.s.pvGIPRC = GCPtr;
+    LogFlow(("TMR3Init: HCPhysGIP=%RHp at %RRv\n", HCPhysGIP, pVM->tm.s.pvGIPRC));
+    MMR3HyperReserveFence(pVM);
+#endif
+
+
+    /* Check assumptions made in TMAllVirtual.cpp about the GIP update interval. */
+    if (    pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC
+        &&  pGip->u32UpdateIntervalNS >= 250000000 /* 0.25s */)
+        return VMSetError(pVM, VERR_TM_GIP_UPDATE_INTERVAL_TOO_BIG, RT_SRC_POS,
+                          N_("The GIP update interval is too big. u32UpdateIntervalNS=%RU32 (u32UpdateHz=%RU32)"),
+                          pGip->u32UpdateIntervalNS, pGip->u32UpdateHz);
+
+    /* Log GIP info that may come in handy. */
+    LogRel(("TM: GIP - u32Mode=%d (%s) u32UpdateHz=%u u32UpdateIntervalNS=%u enmUseTscDelta=%d (%s) fGetGipCpu=%#x cCpus=%d\n",
+            pGip->u32Mode, SUPGetGIPModeName(pGip), pGip->u32UpdateHz, pGip->u32UpdateIntervalNS,
+            pGip->enmUseTscDelta, SUPGetGIPTscDeltaModeName(pGip), pGip->fGetGipCpu, pGip->cCpus));
+    LogRel(("TM: GIP - u64CpuHz=%'RU64 (%#RX64)  SUPGetCpuHzFromGip => %'RU64\n",
+            pGip->u64CpuHz, pGip->u64CpuHz, SUPGetCpuHzFromGip(pGip)));
+    for (uint32_t iCpuSet = 0; iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx); iCpuSet++)
+    {
+        uint16_t iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
+        if (iGipCpu != UINT16_MAX)
+            LogRel(("TM: GIP - CPU: iCpuSet=%#x idCpu=%#x idApic=%#x iGipCpu=%#x i64TSCDelta=%RI64 enmState=%d u64CpuHz=%RU64(*) cErrors=%u\n",
+                    iCpuSet, pGip->aCPUs[iGipCpu].idCpu, pGip->aCPUs[iGipCpu].idApic, iGipCpu, pGip->aCPUs[iGipCpu].i64TSCDelta,
+                    pGip->aCPUs[iGipCpu].enmState, pGip->aCPUs[iGipCpu].u64CpuHz, pGip->aCPUs[iGipCpu].cErrors));
+    }
+
+    /*
+     * Setup the VirtualGetRaw backend.
+     */
+    pVM->tm.s.pfnVirtualGetRawR3                 = tmVirtualNanoTSRediscover;
+    pVM->tm.s.VirtualGetRawDataR3.pfnRediscover  = tmVirtualNanoTSRediscover;
+    pVM->tm.s.VirtualGetRawDataR3.pfnBad         = tmVirtualNanoTSBad;
+    pVM->tm.s.VirtualGetRawDataR3.pfnBadCpuIndex = tmVirtualNanoTSBadCpuIndex;
+    pVM->tm.s.VirtualGetRawDataR3.pu64Prev       = &pVM->tm.s.u64VirtualRawPrev;
+    pVM->tm.s.VirtualGetRawDataRC.pu64Prev       = MMHyperR3ToRC(pVM, (void *)&pVM->tm.s.u64VirtualRawPrev);
+    pVM->tm.s.VirtualGetRawDataR0.pu64Prev       = MMHyperR3ToR0(pVM, (void *)&pVM->tm.s.u64VirtualRawPrev);
+    AssertRelease(pVM->tm.s.VirtualGetRawDataR0.pu64Prev);
+    /* The rest is done in TMR3InitFinalize() since it's too early to call PDM. */
+
+    /*
+     * Init the locks.
+     */
+    rc = PDMR3CritSectInit(pVM, &pVM->tm.s.TimerCritSect, RT_SRC_POS, "TM Timer Lock");
+    if (RT_FAILURE(rc))
+        return rc;
+    rc = PDMR3CritSectInit(pVM, &pVM->tm.s.VirtualSyncLock, RT_SRC_POS, "TM VirtualSync Lock");
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Get our CFGM node, create it if necessary.
+     */
+    PCFGMNODE pCfgHandle = CFGMR3GetChild(CFGMR3GetRoot(pVM), "TM");
+    if (!pCfgHandle)
+    {
+        rc = CFGMR3InsertNode(CFGMR3GetRoot(pVM), "TM", &pCfgHandle);
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Specific errors about some obsolete TM settings (remove after 2015-12-03).
+     */
+    if (CFGMR3Exists(pCfgHandle, "TSCVirtualized"))
+        return VMSetError(pVM, VERR_CFGM_CONFIG_UNKNOWN_VALUE, RT_SRC_POS,
+                          N_("Configuration error: TM setting \"TSCVirtualized\" is no longer supported. Use the \"TSCMode\" setting instead."));
+    if (CFGMR3Exists(pCfgHandle, "UseRealTSC"))
+        return VMSetError(pVM, VERR_CFGM_CONFIG_UNKNOWN_VALUE, RT_SRC_POS,
+                          N_("Configuration error: TM setting \"UseRealTSC\" is no longer supported. Use the \"TSCMode\" setting instead."));
+
+    if (CFGMR3Exists(pCfgHandle, "MaybeUseOffsettedHostTSC"))
+        return VMSetError(pVM, VERR_CFGM_CONFIG_UNKNOWN_VALUE, RT_SRC_POS,
+                          N_("Configuration error: TM setting \"MaybeUseOffsettedHostTSC\" is no longer supported. Use the \"TSCMode\" setting instead."));
+
+    /*
+     * Validate the rest of the TM settings.
+     */
+    rc = CFGMR3ValidateConfig(pCfgHandle, "/TM/",
+                              "TSCMode|"
+                              "TSCModeSwitchAllowed|"
+                              "TSCTicksPerSecond|"
+                              "TSCTiedToExecution|"
+                              "TSCNotTiedToHalt|"
+                              "ScheduleSlack|"
+                              "CatchUpStopThreshold|"
+                              "CatchUpGiveUpThreshold|"
+                              "CatchUpStartThreshold0|CatchUpStartThreshold1|CatchUpStartThreshold2|CatchUpStartThreshold3|"
+                              "CatchUpStartThreshold4|CatchUpStartThreshold5|CatchUpStartThreshold6|CatchUpStartThreshold7|"
+                              "CatchUpStartThreshold8|CatchUpStartThreshold9|"
+                              "CatchUpPrecentage0|CatchUpPrecentage1|CatchUpPrecentage2|CatchUpPrecentage3|"
+                              "CatchUpPrecentage4|CatchUpPrecentage5|CatchUpPrecentage6|CatchUpPrecentage7|"
+                              "CatchUpPrecentage8|CatchUpPrecentage9|"
+                              "UTCOffset|"
+                              "UTCTouchFileOnJump|"
+                              "WarpDrivePercentage|"
+                              "HostHzMax|"
+                              "HostHzFudgeFactorTimerCpu|"
+                              "HostHzFudgeFactorOtherCpu|"
+                              "HostHzFudgeFactorCatchUp100|"
+                              "HostHzFudgeFactorCatchUp200|"
+                              "HostHzFudgeFactorCatchUp400|"
+                              "TimerMillies"
+                              ,
+                              "",
+                              "TM", 0);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Determine the TSC configuration and frequency.
+     */
+    /** @cfgm{/TM/TSCMode, string, Depends on the CPU and VM config}
+     * The name of the TSC mode to use: VirtTSCEmulated, RealTSCOffset or Dynamic.
+     * The default depends on the VM configuration and the capabilities of the
+     * host CPU.  Other config options or runtime changes may override the TSC
+     * mode specified here.
+     */
+    char szTSCMode[32];
+    rc = CFGMR3QueryString(pCfgHandle, "TSCMode", szTSCMode, sizeof(szTSCMode));
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+    {
+        /** @todo Rainy-day/never: Dynamic mode isn't currently suitable for SMP VMs, so
+         * fall back on the more expensive emulated mode. With the current TSC handling
+         * (frequent switching between offsetted mode and taking VM exits, on all VCPUs
+         * without any kind of coordination) will lead to inconsistent TSC behavior with
+         * guest SMP, including TSC going backwards. */
+        pVM->tm.s.enmTSCMode = NEMR3NeedSpecialTscMode(pVM) ? TMTSCMODE_NATIVE_API
+                             : pVM->cCpus == 1 && tmR3HasFixedTSC(pVM) ? TMTSCMODE_DYNAMIC : TMTSCMODE_VIRT_TSC_EMULATED;
+    }
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS, N_("Configuration error: Failed to querying string value \"TSCMode\""));
+    else
+    {
+        if (!RTStrCmp(szTSCMode, "VirtTSCEmulated"))
+            pVM->tm.s.enmTSCMode = TMTSCMODE_VIRT_TSC_EMULATED;
+        else if (!RTStrCmp(szTSCMode, "RealTSCOffset"))
+            pVM->tm.s.enmTSCMode = TMTSCMODE_REAL_TSC_OFFSET;
+        else if (!RTStrCmp(szTSCMode, "Dynamic"))
+            pVM->tm.s.enmTSCMode = TMTSCMODE_DYNAMIC;
+        else
+            return VMSetError(pVM, rc, RT_SRC_POS, N_("Configuration error: Unrecognized TM TSC mode value \"%s\""), szTSCMode);
+        if (NEMR3NeedSpecialTscMode(pVM))
+        {
+            LogRel(("TM: NEM overrides the /TM/TSCMode=%s settings.\n", szTSCMode));
+            pVM->tm.s.enmTSCMode = TMTSCMODE_NATIVE_API;
+        }
+    }
+
+    /**
+     * @cfgm{/TM/TSCModeSwitchAllowed, bool, Whether TM TSC mode switch is allowed
+     *      at runtime}
+     * When using paravirtualized guests, we dynamically switch TSC modes to a more
+     * optimal one for performance. This setting allows overriding this behaviour.
+     */
+    rc = CFGMR3QueryBool(pCfgHandle, "TSCModeSwitchAllowed", &pVM->tm.s.fTSCModeSwitchAllowed);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+    {
+        /* This is finally determined in TMR3InitFinalize() as GIM isn't initialized yet. */
+        pVM->tm.s.fTSCModeSwitchAllowed = true;
+    }
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS, N_("Configuration error: Failed to querying bool value \"TSCModeSwitchAllowed\""));
+    if (pVM->tm.s.fTSCModeSwitchAllowed && pVM->tm.s.enmTSCMode == TMTSCMODE_NATIVE_API)
+    {
+        LogRel(("TM: NEM overrides the /TM/TSCModeSwitchAllowed setting.\n"));
+        pVM->tm.s.fTSCModeSwitchAllowed = false;
+    }
+
+    /** @cfgm{/TM/TSCTicksPerSecond, uint32_t, Current TSC frequency from GIP}
+     * The number of TSC ticks per second (i.e. the TSC frequency). This will
+     * override enmTSCMode.
+     */
+    rc = CFGMR3QueryU64(pCfgHandle, "TSCTicksPerSecond", &pVM->tm.s.cTSCTicksPerSecond);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+    {
+        pVM->tm.s.cTSCTicksPerSecond = tmR3CalibrateTSC();
+        if (   (   pVM->tm.s.enmTSCMode == TMTSCMODE_DYNAMIC
+                || pVM->tm.s.enmTSCMode == TMTSCMODE_VIRT_TSC_EMULATED)
+            && pVM->tm.s.cTSCTicksPerSecond >= _4G)
+        {
+            pVM->tm.s.cTSCTicksPerSecond = _4G - 1; /* (A limitation of our math code) */
+            pVM->tm.s.enmTSCMode = TMTSCMODE_VIRT_TSC_EMULATED;
+        }
+    }
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint64_t value \"TSCTicksPerSecond\""));
+    else if (   pVM->tm.s.cTSCTicksPerSecond < _1M
+             || pVM->tm.s.cTSCTicksPerSecond >= _4G)
+        return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS,
+                          N_("Configuration error: \"TSCTicksPerSecond\" = %RI64 is not in the range 1MHz..4GHz-1"),
+                          pVM->tm.s.cTSCTicksPerSecond);
+    else if (pVM->tm.s.enmTSCMode != TMTSCMODE_NATIVE_API)
+        pVM->tm.s.enmTSCMode = TMTSCMODE_VIRT_TSC_EMULATED;
+    else
+    {
+        LogRel(("TM: NEM overrides the /TM/TSCTicksPerSecond=%RU64 setting.\n", pVM->tm.s.cTSCTicksPerSecond));
+        pVM->tm.s.cTSCTicksPerSecond = tmR3CalibrateTSC();
+    }
+
+    /** @cfgm{/TM/TSCTiedToExecution, bool, false}
+     * Whether the TSC should be tied to execution. This will exclude most of the
+     * virtualization overhead, but will by default include the time spent in the
+     * halt state (see TM/TSCNotTiedToHalt). This setting will override all other
+     * TSC settings except for TSCTicksPerSecond and TSCNotTiedToHalt, which should
+     * be used avoided or used with great care. Note that this will only work right
+     * together with VT-x or AMD-V, and with a single virtual CPU. */
+    rc = CFGMR3QueryBoolDef(pCfgHandle, "TSCTiedToExecution", &pVM->tm.s.fTSCTiedToExecution, false);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying bool value \"TSCTiedToExecution\""));
+    if (pVM->tm.s.fTSCTiedToExecution && pVM->tm.s.enmTSCMode == TMTSCMODE_NATIVE_API)
+        return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, N_("/TM/TSCTiedToExecution is not supported in NEM mode!"));
+    if (pVM->tm.s.fTSCTiedToExecution)
+        pVM->tm.s.enmTSCMode = TMTSCMODE_VIRT_TSC_EMULATED;
+
+
+    /** @cfgm{/TM/TSCNotTiedToHalt, bool, false}
+     * This is used with /TM/TSCTiedToExecution to control how TSC operates
+     * accross HLT instructions.  When true HLT is considered execution time and
+     * TSC continues to run, while when false (default) TSC stops during halt. */
+    rc = CFGMR3QueryBoolDef(pCfgHandle, "TSCNotTiedToHalt", &pVM->tm.s.fTSCNotTiedToHalt, false);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying bool value \"TSCNotTiedToHalt\""));
+
+    /*
+     * Configure the timer synchronous virtual time.
+     */
+    /** @cfgm{/TM/ScheduleSlack, uint32_t, ns, 0, UINT32_MAX, 100000}
+     * Scheduling slack when processing timers. */
+    rc = CFGMR3QueryU32(pCfgHandle, "ScheduleSlack", &pVM->tm.s.u32VirtualSyncScheduleSlack);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.u32VirtualSyncScheduleSlack           =   100000; /* 0.100ms (ASSUMES virtual time is nanoseconds) */
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying 32-bit integer value \"ScheduleSlack\""));
+
+    /** @cfgm{/TM/CatchUpStopThreshold, uint64_t, ns, 0, UINT64_MAX, 500000}
+     * When to stop a catch-up, considering it successful. */
+    rc = CFGMR3QueryU64(pCfgHandle, "CatchUpStopThreshold", &pVM->tm.s.u64VirtualSyncCatchUpStopThreshold);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.u64VirtualSyncCatchUpStopThreshold    =   500000; /* 0.5ms */
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying 64-bit integer value \"CatchUpStopThreshold\""));
+
+    /** @cfgm{/TM/CatchUpGiveUpThreshold, uint64_t, ns, 0, UINT64_MAX, 60000000000}
+     * When to give up a catch-up attempt. */
+    rc = CFGMR3QueryU64(pCfgHandle, "CatchUpGiveUpThreshold", &pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold  = UINT64_C(60000000000); /* 60 sec */
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying 64-bit integer value \"CatchUpGiveUpThreshold\""));
+
+
+    /** @cfgm{/TM/CatchUpPrecentage[0..9], uint32_t, %, 1, 2000, various}
+     * The catch-up percent for a given period.  */
+    /** @cfgm{/TM/CatchUpStartThreshold[0..9], uint64_t, ns, 0, UINT64_MAX}
+     * The catch-up period threshold, or if you like, when a period starts.  */
+#define TM_CFG_PERIOD(iPeriod, DefStart, DefPct) \
+    do \
+    { \
+        uint64_t u64; \
+        rc = CFGMR3QueryU64(pCfgHandle, "CatchUpStartThreshold" #iPeriod, &u64); \
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND) \
+            u64 = UINT64_C(DefStart); \
+        else if (RT_FAILURE(rc)) \
+            return VMSetError(pVM, rc, RT_SRC_POS, N_("Configuration error: Failed to querying 64-bit integer value \"CatchUpThreshold" #iPeriod "\"")); \
+        if (    (iPeriod > 0 && u64 <= pVM->tm.s.aVirtualSyncCatchUpPeriods[iPeriod - 1].u64Start) \
+            ||  u64 >= pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold) \
+            return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, N_("Configuration error: Invalid start of period #" #iPeriod ": %'RU64"), u64); \
+        pVM->tm.s.aVirtualSyncCatchUpPeriods[iPeriod].u64Start = u64; \
+        rc = CFGMR3QueryU32(pCfgHandle, "CatchUpPrecentage" #iPeriod, &pVM->tm.s.aVirtualSyncCatchUpPeriods[iPeriod].u32Percentage); \
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND) \
+            pVM->tm.s.aVirtualSyncCatchUpPeriods[iPeriod].u32Percentage = (DefPct); \
+        else if (RT_FAILURE(rc)) \
+            return VMSetError(pVM, rc, RT_SRC_POS, N_("Configuration error: Failed to querying 32-bit integer value \"CatchUpPrecentage" #iPeriod "\"")); \
+    } while (0)
+    /* This needs more tuning. Not sure if we really need so many period and be so gentle. */
+    TM_CFG_PERIOD(0,     750000,   5); /* 0.75ms at 1.05x */
+    TM_CFG_PERIOD(1,    1500000,  10); /* 1.50ms at 1.10x */
+    TM_CFG_PERIOD(2,    8000000,  25); /*    8ms at 1.25x */
+    TM_CFG_PERIOD(3,   30000000,  50); /*   30ms at 1.50x */
+    TM_CFG_PERIOD(4,   75000000,  75); /*   75ms at 1.75x */
+    TM_CFG_PERIOD(5,  175000000, 100); /*  175ms at 2x */
+    TM_CFG_PERIOD(6,  500000000, 200); /*  500ms at 3x */
+    TM_CFG_PERIOD(7, 3000000000, 300); /*    3s  at 4x */
+    TM_CFG_PERIOD(8,30000000000, 400); /*   30s  at 5x */
+    TM_CFG_PERIOD(9,55000000000, 500); /*   55s  at 6x */
+    AssertCompile(RT_ELEMENTS(pVM->tm.s.aVirtualSyncCatchUpPeriods) == 10);
+#undef TM_CFG_PERIOD
+
+    /*
+     * Configure real world time (UTC).
+     */
+    /** @cfgm{/TM/UTCOffset, int64_t, ns, INT64_MIN, INT64_MAX, 0}
+     * The UTC offset. This is used to put the guest back or forwards in time.  */
+    rc = CFGMR3QueryS64(pCfgHandle, "UTCOffset", &pVM->tm.s.offUTC);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.offUTC = 0; /* ns */
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying 64-bit integer value \"UTCOffset\""));
+
+    /** @cfgm{/TM/UTCTouchFileOnJump, string, none}
+     * File to be written to everytime the host time jumps.  */
+    rc = CFGMR3QueryStringAlloc(pCfgHandle, "UTCTouchFileOnJump", &pVM->tm.s.pszUtcTouchFileOnJump);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.pszUtcTouchFileOnJump = NULL;
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying string value \"UTCTouchFileOnJump\""));
+
+    /*
+     * Setup the warp drive.
+     */
+    /** @cfgm{/TM/WarpDrivePercentage, uint32_t, %, 0, 20000, 100}
+     * The warp drive percentage, 100% is normal speed.  This is used to speed up
+     * or slow down the virtual clock, which can be useful for fast forwarding
+     * borring periods during tests. */
+    rc = CFGMR3QueryU32(pCfgHandle, "WarpDrivePercentage", &pVM->tm.s.u32VirtualWarpDrivePercentage);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        rc = CFGMR3QueryU32(CFGMR3GetRoot(pVM), "WarpDrivePercentage", &pVM->tm.s.u32VirtualWarpDrivePercentage); /* legacy */
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.u32VirtualWarpDrivePercentage = 100;
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"WarpDrivePercent\""));
+    else if (   pVM->tm.s.u32VirtualWarpDrivePercentage < 2
+             || pVM->tm.s.u32VirtualWarpDrivePercentage > 20000)
+        return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS,
+                          N_("Configuration error: \"WarpDrivePercent\" = %RI32 is not in the range 2..20000"),
+                          pVM->tm.s.u32VirtualWarpDrivePercentage);
+    pVM->tm.s.fVirtualWarpDrive = pVM->tm.s.u32VirtualWarpDrivePercentage != 100;
+    if (pVM->tm.s.fVirtualWarpDrive)
+    {
+        if (pVM->tm.s.enmTSCMode == TMTSCMODE_NATIVE_API)
+            LogRel(("TM: Warp-drive active, escept for TSC which is in NEM mode. u32VirtualWarpDrivePercentage=%RI32\n",
+                    pVM->tm.s.u32VirtualWarpDrivePercentage));
+        else
+        {
+            pVM->tm.s.enmTSCMode = TMTSCMODE_VIRT_TSC_EMULATED;
+            LogRel(("TM: Warp-drive active. u32VirtualWarpDrivePercentage=%RI32\n", pVM->tm.s.u32VirtualWarpDrivePercentage));
+        }
+    }
+
+    /*
+     * Gather the Host Hz configuration values.
+     */
+    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzMax", &pVM->tm.s.cHostHzMax, 20000);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"HostHzMax\""));
+
+    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorTimerCpu", &pVM->tm.s.cPctHostHzFudgeFactorTimerCpu, 111);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorTimerCpu\""));
+
+    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorOtherCpu", &pVM->tm.s.cPctHostHzFudgeFactorOtherCpu, 110);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorOtherCpu\""));
+
+    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorCatchUp100", &pVM->tm.s.cPctHostHzFudgeFactorCatchUp100, 300);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorCatchUp100\""));
+
+    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorCatchUp200", &pVM->tm.s.cPctHostHzFudgeFactorCatchUp200, 250);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorCatchUp200\""));
+
+    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorCatchUp400", &pVM->tm.s.cPctHostHzFudgeFactorCatchUp400, 200);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorCatchUp400\""));
+
+    /*
+     * Finally, setup and report.
+     */
+    pVM->tm.s.enmOriginalTSCMode = pVM->tm.s.enmTSCMode;
+    CPUMR3SetCR4Feature(pVM, X86_CR4_TSD, ~X86_CR4_TSD);
+    LogRel(("TM: cTSCTicksPerSecond=%'RU64 (%#RX64) enmTSCMode=%d (%s)\n"
+            "TM: TSCTiedToExecution=%RTbool TSCNotTiedToHalt=%RTbool\n",
+            pVM->tm.s.cTSCTicksPerSecond, pVM->tm.s.cTSCTicksPerSecond, pVM->tm.s.enmTSCMode, tmR3GetTSCModeName(pVM),
+            pVM->tm.s.fTSCTiedToExecution, pVM->tm.s.fTSCNotTiedToHalt));
+
+    /*
+     * Start the timer (guard against REM not yielding).
+     */
+    /** @cfgm{/TM/TimerMillies, uint32_t, ms, 1, 1000, 10}
+     * The watchdog timer interval.  */
+    uint32_t u32Millies;
+    rc = CFGMR3QueryU32(pCfgHandle, "TimerMillies", &u32Millies);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        u32Millies = VM_IS_HM_ENABLED(pVM) ? 1000 : 10;
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to query uint32_t value \"TimerMillies\""));
+    rc = RTTimerCreate(&pVM->tm.s.pTimer, u32Millies, tmR3TimerCallback, pVM);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Failed to create timer, u32Millies=%d rc=%Rrc.\n", u32Millies, rc));
+        return rc;
+    }
+    Log(("TM: Created timer %p firing every %d milliseconds\n", pVM->tm.s.pTimer, u32Millies));
+    pVM->tm.s.u32TimerMillies = u32Millies;
+
+    /*
+     * Register saved state.
+     */
+    rc = SSMR3RegisterInternal(pVM, "tm", 1, TM_SAVED_STATE_VERSION, sizeof(uint64_t) * 8,
+                               NULL, NULL, NULL,
+                               NULL, tmR3Save, NULL,
+                               NULL, tmR3Load, NULL);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Register statistics.
+     */
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataR3.c1nsSteps,STAMTYPE_U32, "/TM/R3/1nsSteps",                     STAMUNIT_OCCURENCES, "Virtual time 1ns steps (due to TSC / GIP variations).");
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataR3.cBadPrev, STAMTYPE_U32, "/TM/R3/cBadPrev",                     STAMUNIT_OCCURENCES, "Times the previous virtual time was considered erratic (shouldn't ever happen).");
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataR0.c1nsSteps,STAMTYPE_U32, "/TM/R0/1nsSteps",                     STAMUNIT_OCCURENCES, "Virtual time 1ns steps (due to TSC / GIP variations).");
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataR0.cBadPrev, STAMTYPE_U32, "/TM/R0/cBadPrev",                     STAMUNIT_OCCURENCES, "Times the previous virtual time was considered erratic (shouldn't ever happen).");
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataRC.c1nsSteps,STAMTYPE_U32, "/TM/RC/1nsSteps",                     STAMUNIT_OCCURENCES, "Virtual time 1ns steps (due to TSC / GIP variations).");
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataRC.cBadPrev, STAMTYPE_U32, "/TM/RC/cBadPrev",                     STAMUNIT_OCCURENCES, "Times the previous virtual time was considered erratic (shouldn't ever happen).");
+    STAM_REL_REG(     pVM,(void*)&pVM->tm.s.offVirtualSync,               STAMTYPE_U64, "/TM/VirtualSync/CurrentOffset",               STAMUNIT_NS, "The current offset. (subtract GivenUp to get the lag)");
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.offVirtualSyncGivenUp,        STAMTYPE_U64, "/TM/VirtualSync/GivenUp",                     STAMUNIT_NS, "Nanoseconds of the 'CurrentOffset' that's been given up and won't ever be attempted caught up with.");
+    STAM_REL_REG(     pVM,(void*)&pVM->tm.s.uMaxHzHint,                   STAMTYPE_U32, "/TM/MaxHzHint",                               STAMUNIT_HZ, "Max guest timer frequency hint.");
+
+#ifdef VBOX_WITH_STATISTICS
+    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataR3.cExpired,    STAMTYPE_U32, "/TM/R3/cExpired",                     STAMUNIT_OCCURENCES, "Times the TSC interval expired (overlaps 1ns steps).");
+    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataR3.cUpdateRaces,STAMTYPE_U32, "/TM/R3/cUpdateRaces",                 STAMUNIT_OCCURENCES, "Thread races when updating the previous timestamp.");
+    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataR0.cExpired,    STAMTYPE_U32, "/TM/R0/cExpired",                     STAMUNIT_OCCURENCES, "Times the TSC interval expired (overlaps 1ns steps).");
+    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataR0.cUpdateRaces,STAMTYPE_U32, "/TM/R0/cUpdateRaces",                 STAMUNIT_OCCURENCES, "Thread races when updating the previous timestamp.");
+    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataRC.cExpired,    STAMTYPE_U32, "/TM/RC/cExpired",                     STAMUNIT_OCCURENCES, "Times the TSC interval expired (overlaps 1ns steps).");
+    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataRC.cUpdateRaces,STAMTYPE_U32, "/TM/RC/cUpdateRaces",                 STAMUNIT_OCCURENCES, "Thread races when updating the previous timestamp.");
+    STAM_REG(pVM, &pVM->tm.s.StatDoQueues,                            STAMTYPE_PROFILE, "/TM/DoQueues",                    STAMUNIT_TICKS_PER_CALL, "Profiling timer TMR3TimerQueuesDo.");
+    STAM_REG(pVM, &pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL],      STAMTYPE_PROFILE_ADV, "/TM/DoQueues/Virtual",            STAMUNIT_TICKS_PER_CALL, "Time spent on the virtual clock queue.");
+    STAM_REG(pVM, &pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL_SYNC], STAMTYPE_PROFILE_ADV, "/TM/DoQueues/VirtualSync",        STAMUNIT_TICKS_PER_CALL, "Time spent on the virtual sync clock queue.");
+    STAM_REG(pVM, &pVM->tm.s.aStatDoQueues[TMCLOCK_REAL],         STAMTYPE_PROFILE_ADV, "/TM/DoQueues/Real",               STAMUNIT_TICKS_PER_CALL, "Time spent on the real clock queue.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatPoll,                                STAMTYPE_COUNTER, "/TM/Poll",                            STAMUNIT_OCCURENCES, "TMTimerPoll calls.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollAlreadySet,                      STAMTYPE_COUNTER, "/TM/Poll/AlreadySet",                 STAMUNIT_OCCURENCES, "TMTimerPoll calls where the FF was already set.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollELoop,                           STAMTYPE_COUNTER, "/TM/Poll/ELoop",                      STAMUNIT_OCCURENCES, "Times TMTimerPoll has given up getting a consistent virtual sync data set.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollMiss,                            STAMTYPE_COUNTER, "/TM/Poll/Miss",                       STAMUNIT_OCCURENCES, "TMTimerPoll calls where nothing had expired.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollRunning,                         STAMTYPE_COUNTER, "/TM/Poll/Running",                    STAMUNIT_OCCURENCES, "TMTimerPoll calls where the queues were being run.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollSimple,                          STAMTYPE_COUNTER, "/TM/Poll/Simple",                     STAMUNIT_OCCURENCES, "TMTimerPoll calls where we could take the simple path.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollVirtual,                         STAMTYPE_COUNTER, "/TM/Poll/HitsVirtual",                STAMUNIT_OCCURENCES, "The number of times TMTimerPoll found an expired TMCLOCK_VIRTUAL queue.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollVirtualSync,                     STAMTYPE_COUNTER, "/TM/Poll/HitsVirtualSync",            STAMUNIT_OCCURENCES, "The number of times TMTimerPoll found an expired TMCLOCK_VIRTUAL_SYNC queue.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatPostponedR3,                         STAMTYPE_COUNTER, "/TM/PostponedR3",                     STAMUNIT_OCCURENCES, "Postponed due to unschedulable state, in ring-3.");
+    STAM_REG(pVM, &pVM->tm.s.StatPostponedRZ,                         STAMTYPE_COUNTER, "/TM/PostponedRZ",                     STAMUNIT_OCCURENCES, "Postponed due to unschedulable state, in ring-0 / RC.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatScheduleOneR3,                       STAMTYPE_PROFILE, "/TM/ScheduleOneR3",               STAMUNIT_TICKS_PER_CALL, "Profiling the scheduling of one queue during a TMTimer* call in EMT.");
+    STAM_REG(pVM, &pVM->tm.s.StatScheduleOneRZ,                       STAMTYPE_PROFILE, "/TM/ScheduleOneRZ",               STAMUNIT_TICKS_PER_CALL, "Profiling the scheduling of one queue during a TMTimer* call in EMT.");
+    STAM_REG(pVM, &pVM->tm.s.StatScheduleSetFF,                       STAMTYPE_COUNTER, "/TM/ScheduleSetFF",                   STAMUNIT_OCCURENCES, "The number of times the timer FF was set instead of doing scheduling.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSet,                            STAMTYPE_COUNTER, "/TM/TimerSet",                        STAMUNIT_OCCURENCES, "Calls, except virtual sync timers");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetOpt,                         STAMTYPE_COUNTER, "/TM/TimerSet/Opt",                    STAMUNIT_OCCURENCES, "Optimized path taken.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetR3,                          STAMTYPE_PROFILE, "/TM/TimerSet/R3",                 STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSet calls made in ring-3.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRZ,                          STAMTYPE_PROFILE, "/TM/TimerSet/RZ",                 STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSet calls made in ring-0 / RC.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStActive,                    STAMTYPE_COUNTER, "/TM/TimerSet/StActive",               STAMUNIT_OCCURENCES, "ACTIVE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStExpDeliver,                STAMTYPE_COUNTER, "/TM/TimerSet/StExpDeliver",           STAMUNIT_OCCURENCES, "EXPIRED_DELIVER");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStOther,                     STAMTYPE_COUNTER, "/TM/TimerSet/StOther",                STAMUNIT_OCCURENCES, "Other states");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStPendStop,                  STAMTYPE_COUNTER, "/TM/TimerSet/StPendStop",             STAMUNIT_OCCURENCES, "PENDING_STOP");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStPendStopSched,             STAMTYPE_COUNTER, "/TM/TimerSet/StPendStopSched",        STAMUNIT_OCCURENCES, "PENDING_STOP_SCHEDULE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStPendSched,                 STAMTYPE_COUNTER, "/TM/TimerSet/StPendSched",            STAMUNIT_OCCURENCES, "PENDING_SCHEDULE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStPendResched,               STAMTYPE_COUNTER, "/TM/TimerSet/StPendResched",          STAMUNIT_OCCURENCES, "PENDING_RESCHEDULE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStStopped,                   STAMTYPE_COUNTER, "/TM/TimerSet/StStopped",              STAMUNIT_OCCURENCES, "STOPPED");
+
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetVs,                          STAMTYPE_COUNTER, "/TM/TimerSetVs",                      STAMUNIT_OCCURENCES, "TMTimerSet calls on virtual sync timers");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetVsR3,                        STAMTYPE_PROFILE, "/TM/TimerSetVs/R3",               STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSet calls made in ring-3 on virtual sync timers.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetVsRZ,                        STAMTYPE_PROFILE, "/TM/TimerSetVs/RZ",               STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSet calls made in ring-0 / RC on virtual sync timers.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetVsStActive,                  STAMTYPE_COUNTER, "/TM/TimerSetVs/StActive",             STAMUNIT_OCCURENCES, "ACTIVE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetVsStExpDeliver,              STAMTYPE_COUNTER, "/TM/TimerSetVs/StExpDeliver",         STAMUNIT_OCCURENCES, "EXPIRED_DELIVER");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetVsStStopped,                 STAMTYPE_COUNTER, "/TM/TimerSetVs/StStopped",            STAMUNIT_OCCURENCES, "STOPPED");
+
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelative,                    STAMTYPE_COUNTER, "/TM/TimerSetRelative",                STAMUNIT_OCCURENCES, "Calls, except virtual sync timers");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeOpt,                 STAMTYPE_COUNTER, "/TM/TimerSetRelative/Opt",            STAMUNIT_OCCURENCES, "Optimized path taken.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeR3,                  STAMTYPE_PROFILE, "/TM/TimerSetRelative/R3",         STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSetRelative calls made in ring-3 (sans virtual sync).");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeRZ,                  STAMTYPE_PROFILE, "/TM/TimerSetRelative/RZ",         STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSetReltaive calls made in ring-0 / RC (sans virtual sync).");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStActive,            STAMTYPE_COUNTER, "/TM/TimerSetRelative/StActive",       STAMUNIT_OCCURENCES, "ACTIVE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStExpDeliver,        STAMTYPE_COUNTER, "/TM/TimerSetRelative/StExpDeliver",   STAMUNIT_OCCURENCES, "EXPIRED_DELIVER");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStOther,             STAMTYPE_COUNTER, "/TM/TimerSetRelative/StOther",        STAMUNIT_OCCURENCES, "Other states");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStPendStop,          STAMTYPE_COUNTER, "/TM/TimerSetRelative/StPendStop",     STAMUNIT_OCCURENCES, "PENDING_STOP");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStPendStopSched,     STAMTYPE_COUNTER, "/TM/TimerSetRelative/StPendStopSched",STAMUNIT_OCCURENCES, "PENDING_STOP_SCHEDULE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStPendSched,         STAMTYPE_COUNTER, "/TM/TimerSetRelative/StPendSched",    STAMUNIT_OCCURENCES, "PENDING_SCHEDULE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStPendResched,       STAMTYPE_COUNTER, "/TM/TimerSetRelative/StPendResched",  STAMUNIT_OCCURENCES, "PENDING_RESCHEDULE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStStopped,           STAMTYPE_COUNTER, "/TM/TimerSetRelative/StStopped",      STAMUNIT_OCCURENCES, "STOPPED");
+
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeVs,                  STAMTYPE_COUNTER, "/TM/TimerSetRelativeVs",              STAMUNIT_OCCURENCES, "TMTimerSetRelative calls on virtual sync timers");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeVsR3,                STAMTYPE_PROFILE, "/TM/TimerSetRelativeVs/R3",       STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSetRelative calls made in ring-3 on virtual sync timers.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeVsRZ,                STAMTYPE_PROFILE, "/TM/TimerSetRelativeVs/RZ",       STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSetReltaive calls made in ring-0 / RC on virtual sync timers.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeVsStActive,          STAMTYPE_COUNTER, "/TM/TimerSetRelativeVs/StActive",     STAMUNIT_OCCURENCES, "ACTIVE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeVsStExpDeliver,      STAMTYPE_COUNTER, "/TM/TimerSetRelativeVs/StExpDeliver", STAMUNIT_OCCURENCES, "EXPIRED_DELIVER");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeVsStStopped,         STAMTYPE_COUNTER, "/TM/TimerSetRelativeVs/StStopped",    STAMUNIT_OCCURENCES, "STOPPED");
+
+    STAM_REG(pVM, &pVM->tm.s.StatTimerStopR3,                         STAMTYPE_PROFILE, "/TM/TimerStopR3",                 STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerStop calls made in ring-3.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerStopRZ,                         STAMTYPE_PROFILE, "/TM/TimerStopRZ",                 STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerStop calls made in ring-0 / RC.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualGet,                          STAMTYPE_COUNTER, "/TM/VirtualGet",                      STAMUNIT_OCCURENCES, "The number of times TMTimerGet was called when the clock was running.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualGetSetFF,                     STAMTYPE_COUNTER, "/TM/VirtualGetSetFF",                 STAMUNIT_OCCURENCES, "Times we set the FF when calling TMTimerGet.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGet,                      STAMTYPE_COUNTER, "/TM/VirtualSyncGet",                  STAMUNIT_OCCURENCES, "The number of times tmVirtualSyncGetEx was called.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetAdjLast,               STAMTYPE_COUNTER, "/TM/VirtualSyncGet/AdjLast",          STAMUNIT_OCCURENCES, "Times we've adjusted against the last returned time stamp .");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetELoop,                 STAMTYPE_COUNTER, "/TM/VirtualSyncGet/ELoop",            STAMUNIT_OCCURENCES, "Times tmVirtualSyncGetEx has given up getting a consistent virtual sync data set.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetExpired,               STAMTYPE_COUNTER, "/TM/VirtualSyncGet/Expired",          STAMUNIT_OCCURENCES, "Times tmVirtualSyncGetEx encountered an expired timer stopping the clock.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetLocked,                STAMTYPE_COUNTER, "/TM/VirtualSyncGet/Locked",           STAMUNIT_OCCURENCES, "Times we successfully acquired the lock in tmVirtualSyncGetEx.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetLockless,              STAMTYPE_COUNTER, "/TM/VirtualSyncGet/Lockless",         STAMUNIT_OCCURENCES, "Times tmVirtualSyncGetEx returned without needing to take the lock.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetSetFF,                 STAMTYPE_COUNTER, "/TM/VirtualSyncGet/SetFF",            STAMUNIT_OCCURENCES, "Times we set the FF when calling tmVirtualSyncGetEx.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualPause,                        STAMTYPE_COUNTER, "/TM/VirtualPause",                    STAMUNIT_OCCURENCES, "The number of times TMR3TimerPause was called.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualResume,                       STAMTYPE_COUNTER, "/TM/VirtualResume",                   STAMUNIT_OCCURENCES, "The number of times TMR3TimerResume was called.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatTimerCallbackSetFF,                  STAMTYPE_COUNTER, "/TM/CallbackSetFF",                   STAMUNIT_OCCURENCES, "The number of times the timer callback set FF.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerCallback,                       STAMTYPE_COUNTER, "/TM/Callback",                        STAMUNIT_OCCURENCES, "The number of times the timer callback is invoked.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatTSCCatchupLE010,                     STAMTYPE_COUNTER, "/TM/TSC/Intercept/CatchupLE010",      STAMUNIT_OCCURENCES, "In catch-up mode, 10% or lower.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCCatchupLE025,                     STAMTYPE_COUNTER, "/TM/TSC/Intercept/CatchupLE025",      STAMUNIT_OCCURENCES, "In catch-up mode, 25%-11%.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCCatchupLE100,                     STAMTYPE_COUNTER, "/TM/TSC/Intercept/CatchupLE100",      STAMUNIT_OCCURENCES, "In catch-up mode, 100%-26%.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCCatchupOther,                     STAMTYPE_COUNTER, "/TM/TSC/Intercept/CatchupOther",      STAMUNIT_OCCURENCES, "In catch-up mode, > 100%.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCNotFixed,                         STAMTYPE_COUNTER, "/TM/TSC/Intercept/NotFixed",          STAMUNIT_OCCURENCES, "TSC is not fixed, it may run at variable speed.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCNotTicking,                       STAMTYPE_COUNTER, "/TM/TSC/Intercept/NotTicking",        STAMUNIT_OCCURENCES, "TSC is not ticking.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCSyncNotTicking,                   STAMTYPE_COUNTER, "/TM/TSC/Intercept/SyncNotTicking",    STAMUNIT_OCCURENCES, "VirtualSync isn't ticking.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCWarp,                             STAMTYPE_COUNTER, "/TM/TSC/Intercept/Warp",              STAMUNIT_OCCURENCES, "Warpdrive is active.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCSet,                              STAMTYPE_COUNTER, "/TM/TSC/Sets",                        STAMUNIT_OCCURENCES, "Calls to TMCpuTickSet.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCUnderflow,                        STAMTYPE_COUNTER, "/TM/TSC/Underflow",                   STAMUNIT_OCCURENCES, "TSC underflow; corrected with last seen value .");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualPause,                        STAMTYPE_COUNTER, "/TM/TSC/Pause",                       STAMUNIT_OCCURENCES, "The number of times the TSC was paused.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualResume,                       STAMTYPE_COUNTER, "/TM/TSC/Resume",                      STAMUNIT_OCCURENCES, "The number of times the TSC was resumed.");
+#endif /* VBOX_WITH_STATISTICS */
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.offTSCRawSrc,          STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS, "TSC offset relative the raw source",           "/TM/TSC/offCPU%u", i);
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS)
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.StatNsTotal,       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,               "Resettable: Total CPU run time.",   "/TM/CPU/%02u", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.StatNsExecuting,   STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent executing guest code.",    "/TM/CPU/%02u/PrfExecuting", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.StatNsExecLong,    STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent executing guest code - long hauls.",    "/TM/CPU/%02u/PrfExecLong", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.StatNsExecShort,   STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent executing guest code - short stretches.",    "/TM/CPU/%02u/PrfExecShort", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.StatNsExecTiny,    STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent executing guest code - tiny bits.",    "/TM/CPU/%02u/PrfExecTiny", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.StatNsHalted,      STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent halted.",                  "/TM/CPU/%02u/PrfHalted", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.StatNsOther,       STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent in the VMM or preempted.", "/TM/CPU/%02u/PrfOther", i);
+# endif
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.cNsTotal,              STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,    "Total CPU run time.",                          "/TM/CPU/%02u/cNsTotal", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.cNsExecuting,          STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,    "Time spent executing guest code.",             "/TM/CPU/%02u/cNsExecuting", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.cNsHalted,             STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,    "Time spent halted.",                           "/TM/CPU/%02u/cNsHalted", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.cNsOther,              STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,    "Time spent in the VMM or preempted.",          "/TM/CPU/%02u/cNsOther", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.cPeriodsExecuting,     STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Times executed guest code.",                   "/TM/CPU/%02u/cPeriodsExecuting", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.cPeriodsHalted,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Times halted.",                                "/TM/CPU/%02u/cPeriodsHalted", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.CpuLoad.cPctExecuting, STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent executing guest code recently.",    "/TM/CPU/%02u/pctExecuting", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.CpuLoad.cPctHalted,    STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent halted recently.",                  "/TM/CPU/%02u/pctHalted", i);
+        STAMR3RegisterF(pVM, &pVCpu->tm.s.CpuLoad.cPctOther,     STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent in the VMM or preempted recently.", "/TM/CPU/%02u/pctOther", i);
+#endif
+    }
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    STAMR3RegisterF(pVM, &pVM->tm.s.CpuLoad.cPctExecuting, STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent executing guest code recently.",    "/TM/CPU/pctExecuting");
+    STAMR3RegisterF(pVM, &pVM->tm.s.CpuLoad.cPctHalted,    STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent halted recently.",                  "/TM/CPU/pctHalted");
+    STAMR3RegisterF(pVM, &pVM->tm.s.CpuLoad.cPctOther,     STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent in the VMM or preempted recently.", "/TM/CPU/pctOther");
+#endif
+
+#ifdef VBOX_WITH_STATISTICS
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncCatchup,              STAMTYPE_PROFILE_ADV, "/TM/VirtualSync/CatchUp",    STAMUNIT_TICKS_PER_OCCURENCE, "Counting and measuring the times spent catching up.");
+    STAM_REG(pVM, (void *)&pVM->tm.s.fVirtualSyncCatchUp,                  STAMTYPE_U8, "/TM/VirtualSync/CatchUpActive",             STAMUNIT_NONE, "Catch-Up active indicator.");
+    STAM_REG(pVM, (void *)&pVM->tm.s.u32VirtualSyncCatchUpPercentage,     STAMTYPE_U32, "/TM/VirtualSync/CatchUpPercentage",          STAMUNIT_PCT, "The catch-up percentage. (+100/100 to get clock multiplier)");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncFF,                       STAMTYPE_PROFILE, "/TM/VirtualSync/FF",         STAMUNIT_TICKS_PER_OCCURENCE, "Time spent in TMR3VirtualSyncFF by all but the dedicate timer EMT.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGiveUp,                   STAMTYPE_COUNTER, "/TM/VirtualSync/GiveUp",              STAMUNIT_OCCURENCES, "Times the catch-up was abandoned.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGiveUpBeforeStarting,     STAMTYPE_COUNTER, "/TM/VirtualSync/GiveUpBeforeStarting",STAMUNIT_OCCURENCES, "Times the catch-up was abandoned before even starting. (Typically debugging++.)");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRun,                      STAMTYPE_COUNTER, "/TM/VirtualSync/Run",                 STAMUNIT_OCCURENCES, "Times the virtual sync timer queue was considered.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRunRestart,               STAMTYPE_COUNTER, "/TM/VirtualSync/Run/Restarts",        STAMUNIT_OCCURENCES, "Times the clock was restarted after a run.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRunStop,                  STAMTYPE_COUNTER, "/TM/VirtualSync/Run/Stop",            STAMUNIT_OCCURENCES, "Times the clock was stopped when calculating the current time before examining the timers.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRunStoppedAlready,        STAMTYPE_COUNTER, "/TM/VirtualSync/Run/StoppedAlready",  STAMUNIT_OCCURENCES, "Times the clock was already stopped elsewhere (TMVirtualSyncGet).");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRunSlack,                 STAMTYPE_PROFILE, "/TM/VirtualSync/Run/Slack",     STAMUNIT_NS_PER_OCCURENCE, "The scheduling slack. (Catch-up handed out when running timers.)");
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->tm.s.aVirtualSyncCatchUpPeriods); i++)
+    {
+        STAMR3RegisterF(pVM, &pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u32Percentage,    STAMTYPE_U32, STAMVISIBILITY_ALWAYS,          STAMUNIT_PCT, "The catch-up percentage.",         "/TM/VirtualSync/Periods/%u", i);
+        STAMR3RegisterF(pVM, &pVM->tm.s.aStatVirtualSyncCatchupAdjust[i],           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,   STAMUNIT_OCCURENCES, "Times adjusted to this period.",   "/TM/VirtualSync/Periods/%u/Adjust", i);
+        STAMR3RegisterF(pVM, &pVM->tm.s.aStatVirtualSyncCatchupInitial[i],          STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,   STAMUNIT_OCCURENCES, "Times started in this period.",    "/TM/VirtualSync/Periods/%u/Initial", i);
+        STAMR3RegisterF(pVM, &pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u64Start,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS,           STAMUNIT_NS, "Start of this period (lag).",      "/TM/VirtualSync/Periods/%u/Start", i);
+    }
+#endif /* VBOX_WITH_STATISTICS */
+
+    /*
+     * Register info handlers.
+     */
+    DBGFR3InfoRegisterInternalEx(pVM, "timers",       "Dumps all timers. No arguments.",          tmR3TimerInfo,        DBGFINFO_FLAGS_RUN_ON_EMT);
+    DBGFR3InfoRegisterInternalEx(pVM, "activetimers", "Dumps active all timers. No arguments.",   tmR3TimerInfoActive,  DBGFINFO_FLAGS_RUN_ON_EMT);
+    DBGFR3InfoRegisterInternalEx(pVM, "clocks",       "Display the time of the various clocks.",  tmR3InfoClocks,       DBGFINFO_FLAGS_RUN_ON_EMT);
+    DBGFR3InfoRegisterInternalArgv(pVM, "cpuload",    "Display the CPU load stats (--help for details).", tmR3InfoCpuLoad, 0);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the host CPU has a fixed TSC frequency.
+ *
+ * @returns true if it has, false if it hasn't.
+ *
+ * @remarks This test doesn't bother with very old CPUs that don't do power
+ *          management or any other stuff that might influence the TSC rate.
+ *          This isn't currently relevant.
+ */
+static bool tmR3HasFixedTSC(PVM pVM)
+{
+    /*
+     * ASSUME that if the GIP is in invariant TSC mode, it's because the CPU
+     * actually has invariant TSC.
+     */
+    PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
+    if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
+        return true;
+
+    /*
+     * Go by features and model info from the CPUID instruction.
+     */
+    if (ASMHasCpuId())
+    {
+        uint32_t uEAX, uEBX, uECX, uEDX;
+
+        /*
+         * By feature. (Used to be AMD specific, intel seems to have picked it up.)
+         */
+        ASMCpuId(0x80000000, &uEAX, &uEBX, &uECX, &uEDX);
+        if (uEAX >= 0x80000007 && ASMIsValidExtRange(uEAX))
+        {
+            ASMCpuId(0x80000007, &uEAX, &uEBX, &uECX, &uEDX);
+            if (   (uEDX & X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR) /* TscInvariant */
+                && pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)       /* No fixed tsc if the gip timer is in async mode. */
+                return true;
+        }
+
+        /*
+         * By model.
+         */
+        if (CPUMGetHostCpuVendor(pVM) == CPUMCPUVENDOR_AMD)
+        {
+            /*
+             * AuthenticAMD - Check for APM support and that TscInvariant is set.
+             *
+             * This test isn't correct with respect to fixed/non-fixed TSC and
+             * older models, but this isn't relevant since the result is currently
+             * only used for making a decision on AMD-V models.
+             */
+#if 0 /* Promoted to generic */
+            ASMCpuId(0x80000000, &uEAX, &uEBX, &uECX, &uEDX);
+            if (uEAX >= 0x80000007)
+            {
+                ASMCpuId(0x80000007, &uEAX, &uEBX, &uECX, &uEDX);
+                if (   (uEDX & X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR) /* TscInvariant */
+                    && (   pGip->u32Mode == SUPGIPMODE_SYNC_TSC     /* No fixed tsc if the gip timer is in async mode. */
+                        || pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC))
+                    return true;
+            }
+#endif
+        }
+        else if (CPUMGetHostCpuVendor(pVM) == CPUMCPUVENDOR_INTEL)
+        {
+            /*
+             * GenuineIntel - Check the model number.
+             *
+             * This test is lacking in the same way and for the same reasons
+             * as the AMD test above.
+             */
+            /** @todo use ASMGetCpuFamily() and ASMGetCpuModel() here. */
+            ASMCpuId(1, &uEAX, &uEBX, &uECX, &uEDX);
+            unsigned uModel  = (uEAX >> 4) & 0x0f;
+            unsigned uFamily = (uEAX >> 8) & 0x0f;
+            if (uFamily == 0x0f)
+                uFamily += (uEAX >> 20) & 0xff;
+            if (uFamily >= 0x06)
+                uModel += ((uEAX >> 16) & 0x0f) << 4;
+            if (    (uFamily == 0x0f /*P4*/     && uModel >= 0x03)
+                ||  (uFamily == 0x06 /*P2/P3*/  && uModel >= 0x0e))
+                return true;
+        }
+        else if (CPUMGetHostCpuVendor(pVM) == CPUMCPUVENDOR_VIA)
+        {
+            /*
+             * CentaurHauls - Check the model, family and stepping.
+             *
+             * This only checks for VIA CPU models Nano X2, Nano X3,
+             * Eden X2 and QuadCore.
+             */
+            /** @todo use ASMGetCpuFamily() and ASMGetCpuModel() here. */
+            ASMCpuId(1, &uEAX, &uEBX, &uECX, &uEDX);
+            unsigned uStepping = (uEAX & 0x0f);
+            unsigned uModel    = (uEAX >> 4) & 0x0f;
+            unsigned uFamily   = (uEAX >> 8) & 0x0f;
+            if (   uFamily   == 0x06
+                && uModel    == 0x0f
+                && uStepping >= 0x0c
+                && uStepping <= 0x0f)
+                return true;
+        }
+        else if (CPUMGetHostCpuVendor(pVM) == CPUMCPUVENDOR_SHANGHAI)
+        {
+            /*
+             * Shanghai - Check the model, family and stepping.
+             */
+            /** @todo use ASMGetCpuFamily() and ASMGetCpuModel() here. */
+            ASMCpuId(1, &uEAX, &uEBX, &uECX, &uEDX);
+            unsigned uFamily   = (uEAX >> 8) & 0x0f;
+            if (   uFamily == 0x06
+                || uFamily == 0x07)
+            {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+
+/**
+ * Calibrate the CPU tick.
+ *
+ * @returns Number of ticks per second.
+ */
+static uint64_t tmR3CalibrateTSC(void)
+{
+    uint64_t u64Hz;
+
+    /*
+     * Use GIP when available. Prefere the nominal one, no need to wait for it.
+     */
+    PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
+    if (pGip)
+    {
+        u64Hz = pGip->u64CpuHz;
+        if (u64Hz < _1T && u64Hz > _1M)
+            return u64Hz;
+        AssertFailed(); /* This shouldn't happen. */
+
+        u64Hz = SUPGetCpuHzFromGip(pGip);
+        if (u64Hz < _1T && u64Hz > _1M)
+            return u64Hz;
+
+        AssertFailed(); /* This shouldn't happen. */
+    }
+    /* else: This should only happen in fake SUPLib mode, which we don't really support any more... */
+
+    /* Call this once first to make sure it's initialized. */
+    RTTimeNanoTS();
+
+    /*
+     * Yield the CPU to increase our chances of getting
+     * a correct value.
+     */
+    RTThreadYield();                    /* Try avoid interruptions between TSC and NanoTS samplings. */
+    static const unsigned   s_auSleep[5] = { 50, 30, 30, 40, 40 };
+    uint64_t                au64Samples[5];
+    unsigned                i;
+    for (i = 0; i < RT_ELEMENTS(au64Samples); i++)
+    {
+        RTMSINTERVAL    cMillies;
+        int             cTries   = 5;
+        uint64_t        u64Start = ASMReadTSC();
+        uint64_t        u64End;
+        uint64_t        StartTS  = RTTimeNanoTS();
+        uint64_t        EndTS;
+        do
+        {
+            RTThreadSleep(s_auSleep[i]);
+            u64End = ASMReadTSC();
+            EndTS  = RTTimeNanoTS();
+            cMillies = (RTMSINTERVAL)((EndTS - StartTS + 500000) / 1000000);
+        } while (   cMillies == 0       /* the sleep may be interrupted... */
+                 || (cMillies < 20 && --cTries > 0));
+        uint64_t    u64Diff = u64End - u64Start;
+
+        au64Samples[i] = (u64Diff * 1000) / cMillies;
+        AssertMsg(cTries > 0, ("cMillies=%d i=%d\n", cMillies, i));
+    }
+
+    /*
+     * Discard the highest and lowest results and calculate the average.
+     */
+    unsigned iHigh = 0;
+    unsigned iLow = 0;
+    for (i = 1; i < RT_ELEMENTS(au64Samples); i++)
+    {
+        if (au64Samples[i] < au64Samples[iLow])
+            iLow = i;
+        if (au64Samples[i] > au64Samples[iHigh])
+            iHigh = i;
+    }
+    au64Samples[iLow] = 0;
+    au64Samples[iHigh] = 0;
+
+    u64Hz = au64Samples[0];
+    for (i = 1; i < RT_ELEMENTS(au64Samples); i++)
+        u64Hz += au64Samples[i];
+    u64Hz /= RT_ELEMENTS(au64Samples) - 2;
+
+    return u64Hz;
+}
+
+
+/**
+ * Finalizes the TM initialization.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(int) TMR3InitFinalize(PVM pVM)
+{
+    int rc;
+
+    /*
+     * Resolve symbols.
+     */
+    if (VM_IS_RAW_MODE_ENABLED(pVM))
+    {
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "tmVirtualNanoTSBad",           &pVM->tm.s.VirtualGetRawDataRC.pfnBad);
+        AssertRCReturn(rc, rc);
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "tmVirtualNanoTSBadCpuIndex",   &pVM->tm.s.VirtualGetRawDataRC.pfnBadCpuIndex);
+        AssertRCReturn(rc, rc);
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "tmVirtualNanoTSRediscover",    &pVM->tm.s.VirtualGetRawDataRC.pfnRediscover);
+        AssertRCReturn(rc, rc);
+        pVM->tm.s.pfnVirtualGetRawRC = pVM->tm.s.VirtualGetRawDataRC.pfnRediscover;
+    }
+
+    rc = PDMR3LdrGetSymbolR0(pVM, NULL, "tmVirtualNanoTSBad",           &pVM->tm.s.VirtualGetRawDataR0.pfnBad);
+    AssertRCReturn(rc, rc);
+    rc = PDMR3LdrGetSymbolR0(pVM, NULL, "tmVirtualNanoTSBadCpuIndex",   &pVM->tm.s.VirtualGetRawDataR0.pfnBadCpuIndex);
+    AssertRCReturn(rc, rc);
+    rc = PDMR3LdrGetSymbolR0(pVM, NULL, "tmVirtualNanoTSRediscover",    &pVM->tm.s.VirtualGetRawDataR0.pfnRediscover);
+    AssertRCReturn(rc, rc);
+    pVM->tm.s.pfnVirtualGetRawR0 = pVM->tm.s.VirtualGetRawDataR0.pfnRediscover;
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    /*
+     * Create a timer for refreshing the CPU load stats.
+     */
+    PTMTIMER pTimer;
+    rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, tmR3CpuLoadTimer, NULL, "CPU Load Timer", &pTimer);
+    if (RT_SUCCESS(rc))
+        rc = TMTimerSetMillies(pTimer, 1000);
+#endif
+
+    /*
+     * GIM is now initialized. Determine if TSC mode switching is allowed (respecting CFGM override).
+     */
+    pVM->tm.s.fTSCModeSwitchAllowed &= tmR3HasFixedTSC(pVM) && GIMIsEnabled(pVM) && !VM_IS_RAW_MODE_ENABLED(pVM);
+    LogRel(("TM: TMR3InitFinalize: fTSCModeSwitchAllowed=%RTbool\n", pVM->tm.s.fTSCModeSwitchAllowed));
+    return rc;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMM_INT_DECL(void) TMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    LogFlow(("TMR3Relocate\n"));
+
+    pVM->tm.s.paTimerQueuesR0 = MMHyperR3ToR0(pVM, pVM->tm.s.paTimerQueuesR3);
+
+    if (VM_IS_RAW_MODE_ENABLED(pVM))
+    {
+        pVM->tm.s.pvGIPRC           = MMHyperR3ToRC(pVM, pVM->tm.s.pvGIPR3);
+        pVM->tm.s.paTimerQueuesRC   = MMHyperR3ToRC(pVM, pVM->tm.s.paTimerQueuesR3);
+        pVM->tm.s.VirtualGetRawDataRC.pu64Prev       += offDelta;
+        pVM->tm.s.VirtualGetRawDataRC.pfnBad         += offDelta;
+        pVM->tm.s.VirtualGetRawDataRC.pfnBadCpuIndex += offDelta;
+        pVM->tm.s.VirtualGetRawDataRC.pfnRediscover  += offDelta;
+        pVM->tm.s.pfnVirtualGetRawRC                 += offDelta;
+    }
+
+    /*
+     * Iterate the timers updating the pVMRC pointers.
+     */
+    for (PTMTIMER pTimer = pVM->tm.s.pCreated; pTimer; pTimer = pTimer->pBigNext)
+    {
+        pTimer->pVMRC = pVM->pVMRC;
+        pTimer->pVMR0 = pVM->pVMR0ForCall; /** @todo fix properly */
+    }
+}
+
+
+/**
+ * Terminates the TM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(int) TMR3Term(PVM pVM)
+{
+    AssertMsg(pVM->tm.s.offVM, ("bad init order!\n"));
+    if (pVM->tm.s.pTimer)
+    {
+        int rc = RTTimerDestroy(pVM->tm.s.pTimer);
+        AssertRC(rc);
+        pVM->tm.s.pTimer = NULL;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * For the TM component this means that a rescheduling is preformed,
+ * the FF is cleared and but without running the queues. We'll have to
+ * check if this makes sense or not, but it seems like a good idea now....
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(void) TMR3Reset(PVM pVM)
+{
+    LogFlow(("TMR3Reset:\n"));
+    VM_ASSERT_EMT(pVM);
+    TM_LOCK_TIMERS(pVM);
+
+    /*
+     * Abort any pending catch up.
+     * This isn't perfect...
+     */
+    if (pVM->tm.s.fVirtualSyncCatchUp)
+    {
+        const uint64_t offVirtualNow = TMVirtualGetNoCheck(pVM);
+        const uint64_t offVirtualSyncNow = TMVirtualSyncGetNoCheck(pVM);
+        if (pVM->tm.s.fVirtualSyncCatchUp)
+        {
+            STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
+
+            const uint64_t offOld = pVM->tm.s.offVirtualSyncGivenUp;
+            const uint64_t offNew = offVirtualNow - offVirtualSyncNow;
+            Assert(offOld <= offNew);
+            ASMAtomicWriteU64((uint64_t volatile *)&pVM->tm.s.offVirtualSyncGivenUp, offNew);
+            ASMAtomicWriteU64((uint64_t volatile *)&pVM->tm.s.offVirtualSync, offNew);
+            ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
+            LogRel(("TM: Aborting catch-up attempt on reset with a %'RU64 ns lag on reset; new total: %'RU64 ns\n", offNew - offOld, offNew));
+        }
+    }
+
+    /*
+     * Process the queues.
+     */
+    for (int i = 0; i < TMCLOCK_MAX; i++)
+        tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[i]);
+#ifdef VBOX_STRICT
+    tmTimerQueuesSanityChecks(pVM, "TMR3Reset");
+#endif
+
+    PVMCPU pVCpuDst = pVM->apCpusR3[pVM->tm.s.idTimerCpu];
+    VMCPU_FF_CLEAR(pVCpuDst, VMCPU_FF_TIMER); /** @todo FIXME: this isn't right. */
+
+    /*
+     * Switch TM TSC mode back to the original mode after a reset for
+     * paravirtualized guests that alter the TM TSC mode during operation.
+     */
+    if (   pVM->tm.s.fTSCModeSwitchAllowed
+        && pVM->tm.s.enmTSCMode != pVM->tm.s.enmOriginalTSCMode)
+    {
+        VM_ASSERT_EMT0(pVM);
+        tmR3CpuTickParavirtDisable(pVM, pVM->apCpusR3[0], NULL /* pvData */);
+    }
+    Assert(!GIMIsParavirtTscEnabled(pVM));
+    pVM->tm.s.fParavirtTscEnabled = false;
+
+    /*
+     * Reset TSC to avoid a Windows 8+ bug (see @bugref{8926}). If Windows
+     * sees TSC value beyond 0x40000000000 at startup, it will reset the
+     * TSC on boot-up CPU only, causing confusion and mayhem with SMP.
+     */
+    VM_ASSERT_EMT0(pVM);
+    uint64_t offTscRawSrc;
+    switch (pVM->tm.s.enmTSCMode)
+    {
+        case TMTSCMODE_REAL_TSC_OFFSET:
+            offTscRawSrc = SUPReadTsc();
+            break;
+        case TMTSCMODE_DYNAMIC:
+        case TMTSCMODE_VIRT_TSC_EMULATED:
+            offTscRawSrc = TMVirtualSyncGetNoCheck(pVM);
+            offTscRawSrc = ASMMultU64ByU32DivByU32(offTscRawSrc, pVM->tm.s.cTSCTicksPerSecond, TMCLOCK_FREQ_VIRTUAL);
+            break;
+        case TMTSCMODE_NATIVE_API:
+            /** @todo NEM TSC reset on reset for Windows8+ bug workaround. */
+            offTscRawSrc = 0;
+            break;
+        default:
+            AssertFailedBreakStmt(offTscRawSrc = 0);
+    }
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+        pVCpu->tm.s.offTSCRawSrc   = offTscRawSrc;
+        pVCpu->tm.s.u64TSC         = 0;
+        pVCpu->tm.s.u64TSCLastSeen = 0;
+    }
+
+    TM_UNLOCK_TIMERS(pVM);
+}
+
+
+/**
+ * Resolve a builtin RC symbol.
+ * Called by PDM when loading or relocating GC modules.
+ *
+ * @returns VBox status
+ * @param   pVM             The cross context VM structure.
+ * @param   pszSymbol       Symbol to resolve.
+ * @param   pRCPtrValue     Where to store the symbol value.
+ * @remark  This has to     work before TMR3Relocate() is called.
+ */
+VMM_INT_DECL(int) TMR3GetImportRC(PVM pVM, const char *pszSymbol, PRTRCPTR pRCPtrValue)
+{
+    if (!strcmp(pszSymbol, "g_pSUPGlobalInfoPage"))
+        *pRCPtrValue = MMHyperR3ToRC(pVM, &pVM->tm.s.pvGIPRC);
+    //else if (..)
+    else
+        return VERR_SYMBOL_NOT_FOUND;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) tmR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("tmR3Save:\n"));
+#ifdef VBOX_STRICT
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+        Assert(!pVCpu->tm.s.fTSCTicking);
+    }
+    Assert(!pVM->tm.s.cVirtualTicking);
+    Assert(!pVM->tm.s.fVirtualSyncTicking);
+    Assert(!pVM->tm.s.cTSCsTicking);
+#endif
+
+    /*
+     * Save the virtual clocks.
+     */
+    /* the virtual clock. */
+    SSMR3PutU64(pSSM, TMCLOCK_FREQ_VIRTUAL);
+    SSMR3PutU64(pSSM, pVM->tm.s.u64Virtual);
+
+    /* the virtual timer synchronous clock. */
+    SSMR3PutU64(pSSM, pVM->tm.s.u64VirtualSync);
+    SSMR3PutU64(pSSM, pVM->tm.s.offVirtualSync);
+    SSMR3PutU64(pSSM, pVM->tm.s.offVirtualSyncGivenUp);
+    SSMR3PutU64(pSSM, pVM->tm.s.u64VirtualSyncCatchUpPrev);
+    SSMR3PutBool(pSSM, pVM->tm.s.fVirtualSyncCatchUp);
+
+    /* real time clock */
+    SSMR3PutU64(pSSM, TMCLOCK_FREQ_REAL);
+
+    /* the cpu tick clock. */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+        SSMR3PutU64(pSSM, TMCpuTickGet(pVCpu));
+    }
+    return SSMR3PutU64(pSSM, pVM->tm.s.cTSCTicksPerSecond);
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) tmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    LogFlow(("tmR3Load:\n"));
+
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+#ifdef VBOX_STRICT
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+        Assert(!pVCpu->tm.s.fTSCTicking);
+    }
+    Assert(!pVM->tm.s.cVirtualTicking);
+    Assert(!pVM->tm.s.fVirtualSyncTicking);
+    Assert(!pVM->tm.s.cTSCsTicking);
+#endif
+
+    /*
+     * Validate version.
+     */
+    if (uVersion != TM_SAVED_STATE_VERSION)
+    {
+        AssertMsgFailed(("tmR3Load: Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    /*
+     * Load the virtual clock.
+     */
+    pVM->tm.s.cVirtualTicking = 0;
+    /* the virtual clock. */
+    uint64_t u64Hz;
+    int rc = SSMR3GetU64(pSSM, &u64Hz);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (u64Hz != TMCLOCK_FREQ_VIRTUAL)
+    {
+        AssertMsgFailed(("The virtual clock frequency differs! Saved: %'RU64 Binary: %'RU64\n",
+                         u64Hz, TMCLOCK_FREQ_VIRTUAL));
+        return VERR_SSM_VIRTUAL_CLOCK_HZ;
+    }
+    SSMR3GetU64(pSSM, &pVM->tm.s.u64Virtual);
+    pVM->tm.s.u64VirtualOffset = 0;
+
+    /* the virtual timer synchronous clock. */
+    pVM->tm.s.fVirtualSyncTicking = false;
+    uint64_t u64;
+    SSMR3GetU64(pSSM, &u64);
+    pVM->tm.s.u64VirtualSync = u64;
+    SSMR3GetU64(pSSM, &u64);
+    pVM->tm.s.offVirtualSync = u64;
+    SSMR3GetU64(pSSM, &u64);
+    pVM->tm.s.offVirtualSyncGivenUp = u64;
+    SSMR3GetU64(pSSM, &u64);
+    pVM->tm.s.u64VirtualSyncCatchUpPrev = u64;
+    bool f;
+    SSMR3GetBool(pSSM, &f);
+    pVM->tm.s.fVirtualSyncCatchUp = f;
+
+    /* the real clock  */
+    rc = SSMR3GetU64(pSSM, &u64Hz);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (u64Hz != TMCLOCK_FREQ_REAL)
+    {
+        AssertMsgFailed(("The real clock frequency differs! Saved: %'RU64 Binary: %'RU64\n",
+                         u64Hz, TMCLOCK_FREQ_REAL));
+        return VERR_SSM_VIRTUAL_CLOCK_HZ; /* misleading... */
+    }
+
+    /* the cpu tick clock. */
+    pVM->tm.s.cTSCsTicking = 0;
+    pVM->tm.s.offTSCPause = 0;
+    pVM->tm.s.u64LastPausedTSC = 0;
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+
+        pVCpu->tm.s.fTSCTicking = false;
+        SSMR3GetU64(pSSM, &pVCpu->tm.s.u64TSC);
+        if (pVM->tm.s.u64LastPausedTSC < pVCpu->tm.s.u64TSC)
+            pVM->tm.s.u64LastPausedTSC = pVCpu->tm.s.u64TSC;
+
+        if (pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET)
+            pVCpu->tm.s.offTSCRawSrc = 0; /** @todo TSC restore stuff and HWACC. */
+    }
+
+    rc = SSMR3GetU64(pSSM, &u64Hz);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (pVM->tm.s.enmTSCMode != TMTSCMODE_REAL_TSC_OFFSET)
+        pVM->tm.s.cTSCTicksPerSecond = u64Hz;
+
+    LogRel(("TM: cTSCTicksPerSecond=%#RX64 (%'RU64) enmTSCMode=%d (%s) (state load)\n",
+            pVM->tm.s.cTSCTicksPerSecond, pVM->tm.s.cTSCTicksPerSecond, pVM->tm.s.enmTSCMode, tmR3GetTSCModeName(pVM)));
+
+    /* Disabled as this isn't tested, also should this apply only if GIM is enabled etc. */
+#if 0
+    /*
+     * If the current host TSC frequency is incompatible with what is in the
+     * saved state of the VM, fall back to emulating TSC and disallow TSC mode
+     * switches during VM runtime (e.g. by GIM).
+     */
+    if (   GIMIsEnabled(pVM)
+        || pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET)
+    {
+        uint64_t uGipCpuHz;
+        bool fRelax  = RTSystemIsInsideVM();
+        bool fCompat = SUPIsTscFreqCompatible(pVM->tm.s.cTSCTicksPerSecond, &uGipCpuHz, fRelax);
+        if (!fCompat)
+        {
+            pVM->tm.s.enmTSCMode = TMTSCMODE_VIRT_TSC_EMULATED;
+            pVM->tm.s.fTSCModeSwitchAllowed = false;
+            if (g_pSUPGlobalInfoPage->u32Mode != SUPGIPMODE_ASYNC_TSC)
+            {
+                LogRel(("TM: TSC frequency incompatible! uGipCpuHz=%#RX64 (%'RU64) enmTSCMode=%d (%s) fTSCModeSwitchAllowed=%RTbool (state load)\n",
+                        uGipCpuHz, uGipCpuHz, pVM->tm.s.enmTSCMode, tmR3GetTSCModeName(pVM), pVM->tm.s.fTSCModeSwitchAllowed));
+            }
+            else
+            {
+                LogRel(("TM: GIP is async, enmTSCMode=%d (%s) fTSCModeSwitchAllowed=%RTbool (state load)\n",
+                        uGipCpuHz, uGipCpuHz, pVM->tm.s.enmTSCMode, tmR3GetTSCModeName(pVM), pVM->tm.s.fTSCModeSwitchAllowed));
+            }
+        }
+    }
+#endif
+
+    /*
+     * Make sure timers get rescheduled immediately.
+     */
+    PVMCPU pVCpuDst = pVM->apCpusR3[pVM->tm.s.idTimerCpu];
+    VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+
+    return VINF_SUCCESS;
+}
+
+#ifdef VBOX_WITH_STATISTICS
+/** Names the clock of the timer.   */
+static const char *tmR3TimerClockName(PTMTIMERR3 pTimer)
+{
+    switch (pTimer->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:       return "virtual";
+        case TMCLOCK_VIRTUAL_SYNC:  return "virtual-sync";
+        case TMCLOCK_REAL:          return "real";
+        case TMCLOCK_TSC:           return "tsc";
+        case TMCLOCK_MAX:           break;
+    }
+    return "corrupt clock value";
+}
+#endif
+
+
+/**
+ * Internal TMR3TimerCreate worker.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmClock    The timer clock.
+ * @param   pszDesc     The timer description.
+ * @param   ppTimer     Where to store the timer pointer on success.
+ */
+static int tmr3TimerCreate(PVM pVM, TMCLOCK enmClock, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Allocate the timer.
+     */
+    PTMTIMERR3 pTimer = NULL;
+    if (pVM->tm.s.pFree && VM_IS_EMT(pVM))
+    {
+        pTimer = pVM->tm.s.pFree;
+        pVM->tm.s.pFree = pTimer->pBigNext;
+        Log3(("TM: Recycling timer %p, new free head %p.\n", pTimer, pTimer->pBigNext));
+    }
+
+    if (!pTimer)
+    {
+        int rc = MMHyperAlloc(pVM, sizeof(*pTimer), 0, MM_TAG_TM, (void **)&pTimer);
+        if (RT_FAILURE(rc))
+            return rc;
+        Log3(("TM: Allocated new timer %p\n", pTimer));
+    }
+
+    /*
+     * Initialize it.
+     */
+    pTimer->u64Expire       = 0;
+    pTimer->enmClock        = enmClock;
+    pTimer->pVMR3           = pVM;
+    pTimer->pVMR0           = pVM->pVMR0ForCall; /** @todo fix properly */
+    pTimer->pVMRC           = pVM->pVMRC;
+    pTimer->enmState        = TMTIMERSTATE_STOPPED;
+    pTimer->offScheduleNext = 0;
+    pTimer->offNext         = 0;
+    pTimer->offPrev         = 0;
+    pTimer->pvUser          = NULL;
+    pTimer->pCritSect       = NULL;
+    pTimer->pszDesc         = pszDesc;
+
+    /* insert into the list of created timers. */
+    TM_LOCK_TIMERS(pVM);
+    pTimer->pBigPrev        = NULL;
+    pTimer->pBigNext        = pVM->tm.s.pCreated;
+    pVM->tm.s.pCreated      = pTimer;
+    if (pTimer->pBigNext)
+        pTimer->pBigNext->pBigPrev = pTimer;
+#ifdef VBOX_STRICT
+    tmTimerQueuesSanityChecks(pVM, "tmR3TimerCreate");
+#endif
+    TM_UNLOCK_TIMERS(pVM);
+
+    /*
+     * Register statistics.
+     */
+#ifdef VBOX_WITH_STATISTICS
+
+    STAMR3RegisterF(pVM, &pTimer->StatTimer,        STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL,
+                    tmR3TimerClockName(pTimer), "/TM/Timers/%s", pszDesc);
+    STAMR3RegisterF(pVM, &pTimer->StatCritSectEnter, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL,
+                    "", "/TM/Timers/%s/CritSectEnter", pszDesc);
+    STAMR3RegisterF(pVM, &pTimer->StatGet,          STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,
+                    "", "/TM/Timers/%s/Get", pszDesc);
+    STAMR3RegisterF(pVM, &pTimer->StatSetAbsolute,  STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,
+                    "", "/TM/Timers/%s/SetAbsolute", pszDesc);
+    STAMR3RegisterF(pVM, &pTimer->StatSetRelative,  STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,
+                    "", "/TM/Timers/%s/SetRelative", pszDesc);
+    STAMR3RegisterF(pVM, &pTimer->StatStop,         STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,
+                    "", "/TM/Timers/%s/Stop", pszDesc);
+#endif
+
+    *ppTimer = pTimer;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates a device timer.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         Device instance.
+ * @param   enmClock        The clock to use on this timer.
+ * @param   pfnCallback     Callback function.
+ * @param   pvUser          The user argument to the callback.
+ * @param   fFlags          Timer creation flags, see grp_tm_timer_flags.
+ * @param   pszDesc         Pointer to description string which must stay around
+ *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
+ * @param   ppTimer         Where to store the timer on success.
+ */
+VMM_INT_DECL(int) TMR3TimerCreateDevice(PVM pVM, PPDMDEVINS pDevIns, TMCLOCK enmClock,
+                                        PFNTMTIMERDEV pfnCallback, void *pvUser,
+                                        uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    AssertReturn(!(fFlags & ~(TMTIMER_FLAGS_NO_CRIT_SECT)), VERR_INVALID_PARAMETER);
+
+    /*
+     * Allocate and init stuff.
+     */
+    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, ppTimer);
+    if (RT_SUCCESS(rc))
+    {
+        (*ppTimer)->enmType         = TMTIMERTYPE_DEV;
+        (*ppTimer)->u.Dev.pfnTimer  = pfnCallback;
+        (*ppTimer)->u.Dev.pDevIns   = pDevIns;
+        (*ppTimer)->pvUser          = pvUser;
+        if (!(fFlags & TMTIMER_FLAGS_NO_CRIT_SECT))
+            (*ppTimer)->pCritSect = PDMR3DevGetCritSect(pVM, pDevIns);
+        Log(("TM: Created device timer %p clock %d callback %p '%s'\n", (*ppTimer), enmClock, pfnCallback, pszDesc));
+    }
+
+    return rc;
+}
+
+
+
+
+/**
+ * Creates a USB device timer.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pUsbIns         The USB device instance.
+ * @param   enmClock        The clock to use on this timer.
+ * @param   pfnCallback     Callback function.
+ * @param   pvUser          The user argument to the callback.
+ * @param   fFlags          Timer creation flags, see grp_tm_timer_flags.
+ * @param   pszDesc         Pointer to description string which must stay around
+ *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
+ * @param   ppTimer         Where to store the timer on success.
+ */
+VMM_INT_DECL(int) TMR3TimerCreateUsb(PVM pVM, PPDMUSBINS pUsbIns, TMCLOCK enmClock,
+                                     PFNTMTIMERUSB pfnCallback, void *pvUser,
+                                     uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    AssertReturn(!(fFlags & ~(TMTIMER_FLAGS_NO_CRIT_SECT)), VERR_INVALID_PARAMETER);
+
+    /*
+     * Allocate and init stuff.
+     */
+    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, ppTimer);
+    if (RT_SUCCESS(rc))
+    {
+        (*ppTimer)->enmType         = TMTIMERTYPE_USB;
+        (*ppTimer)->u.Usb.pfnTimer  = pfnCallback;
+        (*ppTimer)->u.Usb.pUsbIns   = pUsbIns;
+        (*ppTimer)->pvUser          = pvUser;
+        //if (!(fFlags & TMTIMER_FLAGS_NO_CRIT_SECT))
+        //{
+        //    if (pDevIns->pCritSectR3)
+        //        (*ppTimer)->pCritSect = pUsbIns->pCritSectR3;
+        //    else
+        //        (*ppTimer)->pCritSect = IOMR3GetCritSect(pVM);
+        //}
+        Log(("TM: Created USB device timer %p clock %d callback %p '%s'\n", (*ppTimer), enmClock, pfnCallback, pszDesc));
+    }
+
+    return rc;
+}
+
+
+/**
+ * Creates a driver timer.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDrvIns         Driver instance.
+ * @param   enmClock        The clock to use on this timer.
+ * @param   pfnCallback     Callback function.
+ * @param   pvUser          The user argument to the callback.
+ * @param   fFlags          Timer creation flags, see grp_tm_timer_flags.
+ * @param   pszDesc         Pointer to description string which must stay around
+ *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
+ * @param   ppTimer         Where to store the timer on success.
+ */
+VMM_INT_DECL(int) TMR3TimerCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, TMCLOCK enmClock, PFNTMTIMERDRV pfnCallback, void *pvUser,
+                                        uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    AssertReturn(!(fFlags & ~(TMTIMER_FLAGS_NO_CRIT_SECT)), VERR_INVALID_PARAMETER);
+
+    /*
+     * Allocate and init stuff.
+     */
+    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, ppTimer);
+    if (RT_SUCCESS(rc))
+    {
+        (*ppTimer)->enmType         = TMTIMERTYPE_DRV;
+        (*ppTimer)->u.Drv.pfnTimer  = pfnCallback;
+        (*ppTimer)->u.Drv.pDrvIns   = pDrvIns;
+        (*ppTimer)->pvUser          = pvUser;
+        Log(("TM: Created device timer %p clock %d callback %p '%s'\n", (*ppTimer), enmClock, pfnCallback, pszDesc));
+    }
+
+    return rc;
+}
+
+
+/**
+ * Creates an internal timer.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   enmClock        The clock to use on this timer.
+ * @param   pfnCallback     Callback function.
+ * @param   pvUser          User argument to be passed to the callback.
+ * @param   pszDesc         Pointer to description string which must stay around
+ *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
+ * @param   ppTimer         Where to store the timer on success.
+ */
+VMMR3DECL(int) TMR3TimerCreateInternal(PVM pVM, TMCLOCK enmClock, PFNTMTIMERINT pfnCallback, void *pvUser, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    /*
+     * Allocate and init  stuff.
+     */
+    PTMTIMER pTimer;
+    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, &pTimer);
+    if (RT_SUCCESS(rc))
+    {
+        pTimer->enmType             = TMTIMERTYPE_INTERNAL;
+        pTimer->u.Internal.pfnTimer = pfnCallback;
+        pTimer->pvUser              = pvUser;
+        *ppTimer = pTimer;
+        Log(("TM: Created internal timer %p clock %d callback %p '%s'\n", pTimer, enmClock, pfnCallback, pszDesc));
+    }
+
+    return rc;
+}
+
+/**
+ * Creates an external timer.
+ *
+ * @returns Timer handle on success.
+ * @returns NULL on failure.
+ * @param   pVM             The cross context VM structure.
+ * @param   enmClock        The clock to use on this timer.
+ * @param   pfnCallback     Callback function.
+ * @param   pvUser          User argument.
+ * @param   pszDesc         Pointer to description string which must stay around
+ *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
+ */
+VMMR3DECL(PTMTIMERR3) TMR3TimerCreateExternal(PVM pVM, TMCLOCK enmClock, PFNTMTIMEREXT pfnCallback, void *pvUser, const char *pszDesc)
+{
+    /*
+     * Allocate and init stuff.
+     */
+    PTMTIMERR3 pTimer;
+    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, &pTimer);
+    if (RT_SUCCESS(rc))
+    {
+        pTimer->enmType             = TMTIMERTYPE_EXTERNAL;
+        pTimer->u.External.pfnTimer = pfnCallback;
+        pTimer->pvUser              = pvUser;
+        Log(("TM: Created external timer %p clock %d callback %p '%s'\n", pTimer, enmClock, pfnCallback, pszDesc));
+        return pTimer;
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Destroy a timer
+ *
+ * @returns VBox status code.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ */
+VMMR3DECL(int) TMR3TimerDestroy(PTMTIMER pTimer)
+{
+    /*
+     * Be extra careful here.
+     */
+    if (!pTimer)
+        return VINF_SUCCESS;
+    AssertPtr(pTimer);
+    Assert((unsigned)pTimer->enmClock < (unsigned)TMCLOCK_MAX);
+
+    PVM             pVM      = pTimer->CTX_SUFF(pVM);
+    PTMTIMERQUEUE   pQueue   = &pVM->tm.s.CTX_SUFF(paTimerQueues)[pTimer->enmClock];
+    bool            fActive  = false;
+    bool            fPending = false;
+
+    AssertMsg(   !pTimer->pCritSect
+              || VMR3GetState(pVM) != VMSTATE_RUNNING
+              || PDMCritSectIsOwner(pTimer->pCritSect), ("%s\n", pTimer->pszDesc));
+
+    /*
+     * The rest of the game happens behind the lock, just
+     * like create does. All the work is done here.
+     */
+    TM_LOCK_TIMERS(pVM);
+    for (int cRetries = 1000;; cRetries--)
+    {
+        /*
+         * Change to the DESTROY state.
+         */
+        TMTIMERSTATE const enmState = pTimer->enmState;
+        Log2(("TMTimerDestroy: %p:{.enmState=%s, .pszDesc='%s'} cRetries=%d\n",
+              pTimer, tmTimerState(enmState), R3STRING(pTimer->pszDesc), cRetries));
+        switch (enmState)
+        {
+            case TMTIMERSTATE_STOPPED:
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+                break;
+
+            case TMTIMERSTATE_ACTIVE:
+                fActive     = true;
+                break;
+
+            case TMTIMERSTATE_PENDING_STOP:
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+                fActive     = true;
+                fPending    = true;
+                break;
+
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+                fPending    = true;
+                break;
+
+            /*
+             * This shouldn't happen as the caller should make sure there are no races.
+             */
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+                AssertMsgFailed(("%p:.enmState=%s %s\n", pTimer, tmTimerState(enmState), pTimer->pszDesc));
+                TM_UNLOCK_TIMERS(pVM);
+                if (!RTThreadYield())
+                    RTThreadSleep(1);
+                AssertMsgReturn(cRetries > 0, ("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->pszDesc),
+                                VERR_TM_UNSTABLE_STATE);
+                TM_LOCK_TIMERS(pVM);
+                continue;
+
+            /*
+             * Invalid states.
+             */
+            case TMTIMERSTATE_FREE:
+            case TMTIMERSTATE_DESTROY:
+                TM_UNLOCK_TIMERS(pVM);
+                AssertLogRelMsgFailedReturn(("pTimer=%p %s\n", pTimer, tmTimerState(enmState)), VERR_TM_INVALID_STATE);
+
+            default:
+                AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, R3STRING(pTimer->pszDesc)));
+                TM_UNLOCK_TIMERS(pVM);
+                return VERR_TM_UNKNOWN_STATE;
+        }
+
+        /*
+         * Try switch to the destroy state.
+         * This should always succeed as the caller should make sure there are no race.
+         */
+        bool fRc;
+        TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_DESTROY, enmState, fRc);
+        if (fRc)
+            break;
+        AssertMsgFailed(("%p:.enmState=%s %s\n", pTimer, tmTimerState(enmState), pTimer->pszDesc));
+        TM_UNLOCK_TIMERS(pVM);
+        AssertMsgReturn(cRetries > 0, ("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->pszDesc),
+                        VERR_TM_UNSTABLE_STATE);
+        TM_LOCK_TIMERS(pVM);
+    }
+
+    /*
+     * Unlink from the active list.
+     */
+    if (fActive)
+    {
+        const PTMTIMER pPrev = TMTIMER_GET_PREV(pTimer);
+        const PTMTIMER pNext = TMTIMER_GET_NEXT(pTimer);
+        if (pPrev)
+            TMTIMER_SET_NEXT(pPrev, pNext);
+        else
+        {
+            TMTIMER_SET_HEAD(pQueue, pNext);
+            pQueue->u64Expire = pNext ? pNext->u64Expire : INT64_MAX;
+        }
+        if (pNext)
+            TMTIMER_SET_PREV(pNext, pPrev);
+        pTimer->offNext = 0;
+        pTimer->offPrev = 0;
+    }
+
+    /*
+     * Unlink from the schedule list by running it.
+     */
+    if (fPending)
+    {
+        Log3(("TMR3TimerDestroy: tmTimerQueueSchedule\n"));
+        STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a);
+        Assert(pQueue->offSchedule);
+        tmTimerQueueSchedule(pVM, pQueue);
+        STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a);
+    }
+
+    /*
+     * Deregister statistics.
+     */
+#ifdef VBOX_WITH_STATISTICS
+    char szPrefix[128];
+    RTStrPrintf(szPrefix, sizeof(szPrefix), "/TM/Timers/%s", pTimer->pszDesc);
+    STAMR3DeregisterByPrefix(pVM->pUVM, szPrefix);
+#endif
+
+    /*
+     * Ready to move the timer from the created list and onto the free list.
+     */
+    Assert(!pTimer->offNext); Assert(!pTimer->offPrev); Assert(!pTimer->offScheduleNext);
+
+    /* unlink from created list */
+    if (pTimer->pBigPrev)
+        pTimer->pBigPrev->pBigNext = pTimer->pBigNext;
+    else
+        pVM->tm.s.pCreated         = pTimer->pBigNext;
+    if (pTimer->pBigNext)
+        pTimer->pBigNext->pBigPrev = pTimer->pBigPrev;
+    pTimer->pBigNext = 0;
+    pTimer->pBigPrev = 0;
+
+    /* free */
+    Log2(("TM: Inserting %p into the free list ahead of %p!\n", pTimer, pVM->tm.s.pFree));
+    TM_SET_STATE(pTimer, TMTIMERSTATE_FREE);
+    pTimer->pBigNext = pVM->tm.s.pFree;
+    pVM->tm.s.pFree = pTimer;
+
+#ifdef VBOX_STRICT
+    tmTimerQueuesSanityChecks(pVM, "TMR3TimerDestroy");
+#endif
+    TM_UNLOCK_TIMERS(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroy all timers owned by a device.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         Device which timers should be destroyed.
+ */
+VMM_INT_DECL(int) TMR3TimerDestroyDevice(PVM pVM, PPDMDEVINS pDevIns)
+{
+    LogFlow(("TMR3TimerDestroyDevice: pDevIns=%p\n", pDevIns));
+    if (!pDevIns)
+        return VERR_INVALID_PARAMETER;
+
+    TM_LOCK_TIMERS(pVM);
+    PTMTIMER    pCur = pVM->tm.s.pCreated;
+    while (pCur)
+    {
+        PTMTIMER pDestroy = pCur;
+        pCur = pDestroy->pBigNext;
+        if (    pDestroy->enmType == TMTIMERTYPE_DEV
+            &&  pDestroy->u.Dev.pDevIns == pDevIns)
+        {
+            int rc = TMR3TimerDestroy(pDestroy);
+            AssertRC(rc);
+        }
+    }
+    TM_UNLOCK_TIMERS(pVM);
+
+    LogFlow(("TMR3TimerDestroyDevice: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroy all timers owned by a USB device.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pUsbIns         USB device which timers should be destroyed.
+ */
+VMM_INT_DECL(int) TMR3TimerDestroyUsb(PVM pVM, PPDMUSBINS pUsbIns)
+{
+    LogFlow(("TMR3TimerDestroyUsb: pUsbIns=%p\n", pUsbIns));
+    if (!pUsbIns)
+        return VERR_INVALID_PARAMETER;
+
+    TM_LOCK_TIMERS(pVM);
+    PTMTIMER    pCur = pVM->tm.s.pCreated;
+    while (pCur)
+    {
+        PTMTIMER pDestroy = pCur;
+        pCur = pDestroy->pBigNext;
+        if (    pDestroy->enmType == TMTIMERTYPE_USB
+            &&  pDestroy->u.Usb.pUsbIns == pUsbIns)
+        {
+            int rc = TMR3TimerDestroy(pDestroy);
+            AssertRC(rc);
+        }
+    }
+    TM_UNLOCK_TIMERS(pVM);
+
+    LogFlow(("TMR3TimerDestroyUsb: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroy all timers owned by a driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pDrvIns         Driver which timers should be destroyed.
+ */
+VMM_INT_DECL(int) TMR3TimerDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns)
+{
+    LogFlow(("TMR3TimerDestroyDriver: pDrvIns=%p\n", pDrvIns));
+    if (!pDrvIns)
+        return VERR_INVALID_PARAMETER;
+
+    TM_LOCK_TIMERS(pVM);
+    PTMTIMER    pCur = pVM->tm.s.pCreated;
+    while (pCur)
+    {
+        PTMTIMER pDestroy = pCur;
+        pCur = pDestroy->pBigNext;
+        if (    pDestroy->enmType == TMTIMERTYPE_DRV
+            &&  pDestroy->u.Drv.pDrvIns == pDrvIns)
+        {
+            int rc = TMR3TimerDestroy(pDestroy);
+            AssertRC(rc);
+        }
+    }
+    TM_UNLOCK_TIMERS(pVM);
+
+    LogFlow(("TMR3TimerDestroyDriver: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal function for getting the clock time.
+ *
+ * @returns clock time.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmClock    The clock.
+ */
+DECLINLINE(uint64_t) tmClock(PVM pVM, TMCLOCK enmClock)
+{
+    switch (enmClock)
+    {
+        case TMCLOCK_VIRTUAL:       return TMVirtualGet(pVM);
+        case TMCLOCK_VIRTUAL_SYNC:  return TMVirtualSyncGet(pVM);
+        case TMCLOCK_REAL:          return TMRealGet(pVM);
+        case TMCLOCK_TSC:           return TMCpuTickGet(pVM->apCpusR3[0] /* just take VCPU 0 */);
+        default:
+            AssertMsgFailed(("enmClock=%d\n", enmClock));
+            return ~(uint64_t)0;
+    }
+}
+
+
+/**
+ * Checks if the sync queue has one or more expired timers.
+ *
+ * @returns true / false.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   enmClock    The queue.
+ */
+DECLINLINE(bool) tmR3HasExpiredTimer(PVM pVM, TMCLOCK enmClock)
+{
+    const uint64_t u64Expire = pVM->tm.s.CTX_SUFF(paTimerQueues)[enmClock].u64Expire;
+    return u64Expire != INT64_MAX && u64Expire <= tmClock(pVM, enmClock);
+}
+
+
+/**
+ * Checks for expired timers in all the queues.
+ *
+ * @returns true / false.
+ * @param   pVM         The cross context VM structure.
+ */
+DECLINLINE(bool) tmR3AnyExpiredTimers(PVM pVM)
+{
+    /*
+     * Combine the time calculation for the first two since we're not on EMT
+     * TMVirtualSyncGet only permits EMT.
+     */
+    uint64_t u64Now = TMVirtualGetNoCheck(pVM);
+    if (pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL].u64Expire <= u64Now)
+        return true;
+    u64Now = pVM->tm.s.fVirtualSyncTicking
+           ? u64Now - pVM->tm.s.offVirtualSync
+           : pVM->tm.s.u64VirtualSync;
+    if (pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire <= u64Now)
+        return true;
+
+    /*
+     * The remaining timers.
+     */
+    if (tmR3HasExpiredTimer(pVM, TMCLOCK_REAL))
+        return true;
+    if (tmR3HasExpiredTimer(pVM, TMCLOCK_TSC))
+        return true;
+    return false;
+}
+
+
+/**
+ * Schedule timer callback.
+ *
+ * @param   pTimer      Timer handle.
+ * @param   pvUser      Pointer to the VM.
+ * @thread  Timer thread.
+ *
+ * @remark  We cannot do the scheduling and queues running from a timer handler
+ *          since it's not executing in EMT, and even if it was it would be async
+ *          and we wouldn't know the state of the affairs.
+ *          So, we'll just raise the timer FF and force any REM execution to exit.
+ */
+static DECLCALLBACK(void) tmR3TimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t /*iTick*/)
+{
+    PVM     pVM      = (PVM)pvUser;
+    PVMCPU  pVCpuDst = pVM->apCpusR3[pVM->tm.s.idTimerCpu];
+    NOREF(pTimer);
+
+    AssertCompile(TMCLOCK_MAX == 4);
+    STAM_COUNTER_INC(&pVM->tm.s.StatTimerCallback);
+
+#ifdef DEBUG_Sander /* very annoying, keep it private. */
+    if (VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+        Log(("tmR3TimerCallback: timer event still pending!!\n"));
+#endif
+    if (    !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)
+        &&  (   pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule /** @todo FIXME - reconsider offSchedule as a reason for running the timer queues. */
+            ||  pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].offSchedule
+            ||  pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].offSchedule
+            ||  pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].offSchedule
+            ||  tmR3AnyExpiredTimers(pVM)
+            )
+        && !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)
+        && !pVM->tm.s.fRunningQueues
+       )
+    {
+        Log5(("TM(%u): FF: 0 -> 1\n", __LINE__));
+        VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+        VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM | VMNOTIFYFF_FLAGS_POKE);
+        STAM_COUNTER_INC(&pVM->tm.s.StatTimerCallbackSetFF);
+    }
+}
+
+
+/**
+ * Schedules and runs any pending timers.
+ *
+ * This is normally called from a forced action handler in EMT.
+ *
+ * @param   pVM             The cross context VM structure.
+ *
+ * @thread  EMT (actually EMT0, but we fend off the others)
+ */
+VMMR3DECL(void) TMR3TimerQueuesDo(PVM pVM)
+{
+    /*
+     * Only the dedicated timer EMT should do stuff here.
+     * (fRunningQueues is only used as an indicator.)
+     */
+    Assert(pVM->tm.s.idTimerCpu < pVM->cCpus);
+    PVMCPU pVCpuDst = pVM->apCpusR3[pVM->tm.s.idTimerCpu];
+    if (VMMGetCpu(pVM) != pVCpuDst)
+    {
+        Assert(pVM->cCpus > 1);
+        return;
+    }
+    STAM_PROFILE_START(&pVM->tm.s.StatDoQueues, a);
+    Log2(("TMR3TimerQueuesDo:\n"));
+    Assert(!pVM->tm.s.fRunningQueues);
+    ASMAtomicWriteBool(&pVM->tm.s.fRunningQueues, true);
+    TM_LOCK_TIMERS(pVM);
+
+    /*
+     * Process the queues.
+     */
+    AssertCompile(TMCLOCK_MAX == 4);
+
+    /* TMCLOCK_VIRTUAL_SYNC (see also TMR3VirtualSyncFF) */
+    STAM_PROFILE_ADV_START(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL_SYNC], s1);
+    PDMCritSectEnter(&pVM->tm.s.VirtualSyncLock, VERR_IGNORED);
+    ASMAtomicWriteBool(&pVM->tm.s.fRunningVirtualSyncQueue, true);
+    VMCPU_FF_CLEAR(pVCpuDst, VMCPU_FF_TIMER);   /* Clear the FF once we started working for real. */
+
+    Assert(!pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule);
+    tmR3TimerQueueRunVirtualSync(pVM);
+    if (pVM->tm.s.fVirtualSyncTicking) /** @todo move into tmR3TimerQueueRunVirtualSync - FIXME */
+        VM_FF_CLEAR(pVM, VM_FF_TM_VIRTUAL_SYNC);
+
+    ASMAtomicWriteBool(&pVM->tm.s.fRunningVirtualSyncQueue, false);
+    PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+    STAM_PROFILE_ADV_STOP(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL_SYNC], s1);
+
+    /* TMCLOCK_VIRTUAL */
+    STAM_PROFILE_ADV_START(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL], s2);
+    if (pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].offSchedule)
+        tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL]);
+    tmR3TimerQueueRun(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL]);
+    STAM_PROFILE_ADV_STOP(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL], s2);
+
+    /* TMCLOCK_TSC */
+    Assert(!pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].offActive); /* not used */
+
+    /* TMCLOCK_REAL */
+    STAM_PROFILE_ADV_START(&pVM->tm.s.aStatDoQueues[TMCLOCK_REAL], s3);
+    if (pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].offSchedule)
+        tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL]);
+    tmR3TimerQueueRun(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL]);
+    STAM_PROFILE_ADV_STOP(&pVM->tm.s.aStatDoQueues[TMCLOCK_REAL], s3);
+
+#ifdef VBOX_STRICT
+    /* check that we didn't screw up. */
+    tmTimerQueuesSanityChecks(pVM, "TMR3TimerQueuesDo");
+#endif
+
+    /* done */
+    Log2(("TMR3TimerQueuesDo: returns void\n"));
+    ASMAtomicWriteBool(&pVM->tm.s.fRunningQueues, false);
+    TM_UNLOCK_TIMERS(pVM);
+    STAM_PROFILE_STOP(&pVM->tm.s.StatDoQueues, a);
+}
+
+//RT_C_DECLS_BEGIN
+//int     iomLock(PVM pVM);
+//void    iomUnlock(PVM pVM);
+//RT_C_DECLS_END
+
+
+/**
+ * Schedules and runs any pending times in the specified queue.
+ *
+ * This is normally called from a forced action handler in EMT.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pQueue          The queue to run.
+ */
+static void tmR3TimerQueueRun(PVM pVM, PTMTIMERQUEUE pQueue)
+{
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Run timers.
+     *
+     * We check the clock once and run all timers which are ACTIVE
+     * and have an expire time less or equal to the time we read.
+     *
+     * N.B. A generic unlink must be applied since other threads
+     *      are allowed to mess with any active timer at any time.
+     *      However, we only allow EMT to handle EXPIRED_PENDING
+     *      timers, thus enabling the timer handler function to
+     *      arm the timer again.
+     */
+    PTMTIMER pNext = TMTIMER_GET_HEAD(pQueue);
+    if (!pNext)
+        return;
+    const uint64_t u64Now = tmClock(pVM, pQueue->enmClock);
+    while (pNext && pNext->u64Expire <= u64Now)
+    {
+        PTMTIMER        pTimer    = pNext;
+        pNext = TMTIMER_GET_NEXT(pTimer);
+        PPDMCRITSECT    pCritSect = pTimer->pCritSect;
+        if (pCritSect)
+        {
+            STAM_PROFILE_START(&pTimer->StatCritSectEnter, Locking);
+            PDMCritSectEnter(pCritSect, VERR_IGNORED);
+            STAM_PROFILE_STOP(&pTimer->StatCritSectEnter, Locking);
+        }
+        Log2(("tmR3TimerQueueRun: %p:{.enmState=%s, .enmClock=%d, .enmType=%d, u64Expire=%llx (now=%llx) .pszDesc=%s}\n",
+              pTimer, tmTimerState(pTimer->enmState), pTimer->enmClock, pTimer->enmType, pTimer->u64Expire, u64Now, pTimer->pszDesc));
+        bool fRc;
+        TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_EXPIRED_GET_UNLINK, TMTIMERSTATE_ACTIVE, fRc);
+        if (fRc)
+        {
+            Assert(!pTimer->offScheduleNext); /* this can trigger falsely */
+
+            /* unlink */
+            const PTMTIMER pPrev = TMTIMER_GET_PREV(pTimer);
+            if (pPrev)
+                TMTIMER_SET_NEXT(pPrev, pNext);
+            else
+            {
+                TMTIMER_SET_HEAD(pQueue, pNext);
+                pQueue->u64Expire = pNext ? pNext->u64Expire : INT64_MAX;
+            }
+            if (pNext)
+                TMTIMER_SET_PREV(pNext, pPrev);
+            pTimer->offNext = 0;
+            pTimer->offPrev = 0;
+
+            /* fire */
+            TM_SET_STATE(pTimer, TMTIMERSTATE_EXPIRED_DELIVER);
+            STAM_PROFILE_START(&pTimer->StatTimer, PrfTimer);
+            switch (pTimer->enmType)
+            {
+                case TMTIMERTYPE_DEV:       pTimer->u.Dev.pfnTimer(pTimer->u.Dev.pDevIns, pTimer, pTimer->pvUser); break;
+                case TMTIMERTYPE_USB:       pTimer->u.Usb.pfnTimer(pTimer->u.Usb.pUsbIns, pTimer, pTimer->pvUser); break;
+                case TMTIMERTYPE_DRV:       pTimer->u.Drv.pfnTimer(pTimer->u.Drv.pDrvIns, pTimer, pTimer->pvUser); break;
+                case TMTIMERTYPE_INTERNAL:  pTimer->u.Internal.pfnTimer(pVM, pTimer, pTimer->pvUser); break;
+                case TMTIMERTYPE_EXTERNAL:  pTimer->u.External.pfnTimer(pTimer->pvUser); break;
+                default:
+                    AssertMsgFailed(("Invalid timer type %d (%s)\n", pTimer->enmType, pTimer->pszDesc));
+                    break;
+            }
+            STAM_PROFILE_STOP(&pTimer->StatTimer, PrfTimer);
+
+            /* change the state if it wasn't changed already in the handler. */
+            TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_STOPPED, TMTIMERSTATE_EXPIRED_DELIVER, fRc);
+            Log2(("tmR3TimerQueueRun: new state %s\n", tmTimerState(pTimer->enmState)));
+        }
+        if (pCritSect)
+            PDMCritSectLeave(pCritSect);
+    } /* run loop */
+}
+
+
+/**
+ * Schedules and runs any pending times in the timer queue for the
+ * synchronous virtual clock.
+ *
+ * This scheduling is a bit different from the other queues as it need
+ * to implement the special requirements of the timer synchronous virtual
+ * clock, thus this 2nd queue run function.
+ *
+ * @param   pVM             The cross context VM structure.
+ *
+ * @remarks The caller must the Virtual Sync lock.  Owning the TM lock is no
+ *          longer important.
+ */
+static void tmR3TimerQueueRunVirtualSync(PVM pVM)
+{
+    PTMTIMERQUEUE const pQueue = &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC];
+    VM_ASSERT_EMT(pVM);
+    Assert(PDMCritSectIsOwner(&pVM->tm.s.VirtualSyncLock));
+
+    /*
+     * Any timers?
+     */
+    PTMTIMER pNext = TMTIMER_GET_HEAD(pQueue);
+    if (RT_UNLIKELY(!pNext))
+    {
+        Assert(pVM->tm.s.fVirtualSyncTicking || !pVM->tm.s.cVirtualTicking);
+        return;
+    }
+    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncRun);
+
+    /*
+     * Calculate the time frame for which we will dispatch timers.
+     *
+     * We use a time frame ranging from the current sync time (which is most likely the
+     * same as the head timer) and some configurable period (100000ns) up towards the
+     * current virtual time. This period might also need to be restricted by the catch-up
+     * rate so frequent calls to this function won't accelerate the time too much, however
+     * this will be implemented at a later point if necessary.
+     *
+     * Without this frame we would 1) having to run timers much more frequently
+     * and 2) lag behind at a steady rate.
+     */
+    const uint64_t  u64VirtualNow  = TMVirtualGetNoCheck(pVM);
+    uint64_t const  offSyncGivenUp = pVM->tm.s.offVirtualSyncGivenUp;
+    uint64_t        u64Now;
+    if (!pVM->tm.s.fVirtualSyncTicking)
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncRunStoppedAlready);
+        u64Now = pVM->tm.s.u64VirtualSync;
+        Assert(u64Now <= pNext->u64Expire);
+    }
+    else
+    {
+        /* Calc 'now'. */
+        bool        fStopCatchup   = false;
+        bool        fUpdateStuff   = false;
+        uint64_t    off            = pVM->tm.s.offVirtualSync;
+        if (pVM->tm.s.fVirtualSyncCatchUp)
+        {
+            uint64_t u64Delta = u64VirtualNow - pVM->tm.s.u64VirtualSyncCatchUpPrev;
+            if (RT_LIKELY(!(u64Delta >> 32)))
+            {
+                uint64_t u64Sub = ASMMultU64ByU32DivByU32(u64Delta, pVM->tm.s.u32VirtualSyncCatchUpPercentage, 100);
+                if (off > u64Sub + offSyncGivenUp)
+                {
+                    off -= u64Sub;
+                    Log4(("TM: %'RU64/-%'8RU64: sub %'RU64 [tmR3TimerQueueRunVirtualSync]\n", u64VirtualNow - off, off - offSyncGivenUp, u64Sub));
+                }
+                else
+                {
+                    STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
+                    fStopCatchup = true;
+                    off = offSyncGivenUp;
+                }
+                fUpdateStuff = true;
+            }
+        }
+        u64Now = u64VirtualNow - off;
+
+        /* Adjust against last returned time. */
+        uint64_t u64Last = ASMAtomicUoReadU64(&pVM->tm.s.u64VirtualSync);
+        if (u64Last > u64Now)
+        {
+            u64Now = u64Last + 1;
+            STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetAdjLast);
+        }
+
+        /* Check if stopped by expired timer. */
+        uint64_t const u64Expire = pNext->u64Expire;
+        if (u64Now >= u64Expire)
+        {
+            STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncRunStop);
+            u64Now = u64Expire;
+            ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64Now);
+            ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false);
+            Log4(("TM: %'RU64/-%'8RU64: exp tmr [tmR3TimerQueueRunVirtualSync]\n", u64Now, u64VirtualNow - u64Now - offSyncGivenUp));
+        }
+        else
+        {
+            ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64Now);
+            if (fUpdateStuff)
+            {
+                ASMAtomicWriteU64(&pVM->tm.s.offVirtualSync, off);
+                ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev, u64VirtualNow);
+                ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64Now);
+                if (fStopCatchup)
+                {
+                    ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
+                    Log4(("TM: %'RU64/0: caught up [tmR3TimerQueueRunVirtualSync]\n", u64VirtualNow));
+                }
+            }
+        }
+    }
+
+    /* calc end of frame. */
+    uint64_t u64Max = u64Now + pVM->tm.s.u32VirtualSyncScheduleSlack;
+    if (u64Max > u64VirtualNow - offSyncGivenUp)
+        u64Max = u64VirtualNow - offSyncGivenUp;
+
+    /* assert sanity */
+    Assert(u64Now <= u64VirtualNow - offSyncGivenUp);
+    Assert(u64Max <= u64VirtualNow - offSyncGivenUp);
+    Assert(u64Now <= u64Max);
+    Assert(offSyncGivenUp == pVM->tm.s.offVirtualSyncGivenUp);
+
+    /*
+     * Process the expired timers moving the clock along as we progress.
+     */
+#ifdef VBOX_STRICT
+    uint64_t u64Prev = u64Now; NOREF(u64Prev);
+#endif
+    while (pNext && pNext->u64Expire <= u64Max)
+    {
+        /* Advance */
+        PTMTIMER pTimer = pNext;
+        pNext = TMTIMER_GET_NEXT(pTimer);
+
+        /* Take the associated lock. */
+        PPDMCRITSECT pCritSect = pTimer->pCritSect;
+        if (pCritSect)
+        {
+            STAM_PROFILE_START(&pTimer->StatCritSectEnter, Locking);
+            PDMCritSectEnter(pCritSect, VERR_IGNORED);
+            STAM_PROFILE_STOP(&pTimer->StatCritSectEnter, Locking);
+        }
+
+        Log2(("tmR3TimerQueueRun: %p:{.enmState=%s, .enmClock=%d, .enmType=%d, u64Expire=%llx (now=%llx) .pszDesc=%s}\n",
+              pTimer, tmTimerState(pTimer->enmState), pTimer->enmClock, pTimer->enmType, pTimer->u64Expire, u64Now, pTimer->pszDesc));
+
+        /* Advance the clock - don't permit timers to be out of order or armed
+           in the 'past'. */
+#ifdef VBOX_STRICT
+        AssertMsg(pTimer->u64Expire >= u64Prev, ("%'RU64 < %'RU64 %s\n", pTimer->u64Expire, u64Prev, pTimer->pszDesc));
+        u64Prev = pTimer->u64Expire;
+#endif
+        ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, pTimer->u64Expire);
+        ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false);
+
+        /* Unlink it, change the state and do the callout. */
+        tmTimerQueueUnlinkActive(pQueue, pTimer);
+        TM_SET_STATE(pTimer, TMTIMERSTATE_EXPIRED_DELIVER);
+        STAM_PROFILE_START(&pTimer->StatTimer, PrfTimer);
+        switch (pTimer->enmType)
+        {
+            case TMTIMERTYPE_DEV:       pTimer->u.Dev.pfnTimer(pTimer->u.Dev.pDevIns, pTimer, pTimer->pvUser); break;
+            case TMTIMERTYPE_USB:       pTimer->u.Usb.pfnTimer(pTimer->u.Usb.pUsbIns, pTimer, pTimer->pvUser); break;
+            case TMTIMERTYPE_DRV:       pTimer->u.Drv.pfnTimer(pTimer->u.Drv.pDrvIns, pTimer, pTimer->pvUser); break;
+            case TMTIMERTYPE_INTERNAL:  pTimer->u.Internal.pfnTimer(pVM, pTimer, pTimer->pvUser); break;
+            case TMTIMERTYPE_EXTERNAL:  pTimer->u.External.pfnTimer(pTimer->pvUser); break;
+            default:
+                AssertMsgFailed(("Invalid timer type %d (%s)\n", pTimer->enmType, pTimer->pszDesc));
+                break;
+        }
+        STAM_PROFILE_STOP(&pTimer->StatTimer, PrfTimer);
+
+        /* Change the state if it wasn't changed already in the handler.
+           Reset the Hz hint too since this is the same as TMTimerStop. */
+        bool fRc;
+        TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_STOPPED, TMTIMERSTATE_EXPIRED_DELIVER, fRc);
+        if (fRc && pTimer->uHzHint)
+        {
+            if (pTimer->uHzHint >= pVM->tm.s.uMaxHzHint)
+                ASMAtomicWriteBool(&pVM->tm.s.fHzHintNeedsUpdating, true);
+            pTimer->uHzHint = 0;
+        }
+        Log2(("tmR3TimerQueueRun: new state %s\n", tmTimerState(pTimer->enmState)));
+
+        /* Leave the associated lock. */
+        if (pCritSect)
+            PDMCritSectLeave(pCritSect);
+    } /* run loop */
+
+
+    /*
+     * Restart the clock if it was stopped to serve any timers,
+     * and start/adjust catch-up if necessary.
+     */
+    if (    !pVM->tm.s.fVirtualSyncTicking
+        &&  pVM->tm.s.cVirtualTicking)
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncRunRestart);
+
+        /* calc the slack we've handed out. */
+        const uint64_t u64VirtualNow2 = TMVirtualGetNoCheck(pVM);
+        Assert(u64VirtualNow2 >= u64VirtualNow);
+        AssertMsg(pVM->tm.s.u64VirtualSync >= u64Now, ("%'RU64 < %'RU64\n", pVM->tm.s.u64VirtualSync, u64Now));
+        const uint64_t offSlack = pVM->tm.s.u64VirtualSync - u64Now;
+        STAM_STATS({
+            if (offSlack)
+            {
+                PSTAMPROFILE p = &pVM->tm.s.StatVirtualSyncRunSlack;
+                p->cPeriods++;
+                p->cTicks += offSlack;
+                if (p->cTicksMax < offSlack) p->cTicksMax = offSlack;
+                if (p->cTicksMin > offSlack) p->cTicksMin = offSlack;
+            }
+        });
+
+        /* Let the time run a little bit while we were busy running timers(?). */
+        uint64_t u64Elapsed;
+#define MAX_ELAPSED 30000U /* ns */
+        if (offSlack > MAX_ELAPSED)
+            u64Elapsed = 0;
+        else
+        {
+            u64Elapsed = u64VirtualNow2 - u64VirtualNow;
+            if (u64Elapsed > MAX_ELAPSED)
+                u64Elapsed = MAX_ELAPSED;
+            u64Elapsed = u64Elapsed > offSlack ? u64Elapsed - offSlack : 0;
+        }
+#undef MAX_ELAPSED
+
+        /* Calc the current offset. */
+        uint64_t offNew = u64VirtualNow2 - pVM->tm.s.u64VirtualSync - u64Elapsed;
+        Assert(!(offNew & RT_BIT_64(63)));
+        uint64_t offLag = offNew - pVM->tm.s.offVirtualSyncGivenUp;
+        Assert(!(offLag & RT_BIT_64(63)));
+
+        /*
+         * Deal with starting, adjusting and stopping catchup.
+         */
+        if (pVM->tm.s.fVirtualSyncCatchUp)
+        {
+            if (offLag <= pVM->tm.s.u64VirtualSyncCatchUpStopThreshold)
+            {
+                /* stop */
+                STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
+                ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
+                Log4(("TM: %'RU64/-%'8RU64: caught up [pt]\n", u64VirtualNow2 - offNew, offLag));
+            }
+            else if (offLag <= pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold)
+            {
+                /* adjust */
+                unsigned i = 0;
+                while (     i + 1 < RT_ELEMENTS(pVM->tm.s.aVirtualSyncCatchUpPeriods)
+                       &&   offLag >= pVM->tm.s.aVirtualSyncCatchUpPeriods[i + 1].u64Start)
+                    i++;
+                if (pVM->tm.s.u32VirtualSyncCatchUpPercentage < pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u32Percentage)
+                {
+                    STAM_COUNTER_INC(&pVM->tm.s.aStatVirtualSyncCatchupAdjust[i]);
+                    ASMAtomicWriteU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage, pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u32Percentage);
+                    Log4(("TM: %'RU64/%'8RU64: adj %u%%\n", u64VirtualNow2 - offNew, offLag, pVM->tm.s.u32VirtualSyncCatchUpPercentage));
+                }
+                pVM->tm.s.u64VirtualSyncCatchUpPrev = u64VirtualNow2;
+            }
+            else
+            {
+                /* give up */
+                STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGiveUp);
+                STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
+                ASMAtomicWriteU64((uint64_t volatile *)&pVM->tm.s.offVirtualSyncGivenUp, offNew);
+                ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
+                Log4(("TM: %'RU64/%'8RU64: give up %u%%\n", u64VirtualNow2 - offNew, offLag, pVM->tm.s.u32VirtualSyncCatchUpPercentage));
+                LogRel(("TM: Giving up catch-up attempt at a %'RU64 ns lag; new total: %'RU64 ns\n", offLag, offNew));
+            }
+        }
+        else if (offLag >= pVM->tm.s.aVirtualSyncCatchUpPeriods[0].u64Start)
+        {
+            if (offLag <= pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold)
+            {
+                /* start */
+                STAM_PROFILE_ADV_START(&pVM->tm.s.StatVirtualSyncCatchup, c);
+                unsigned i = 0;
+                while (     i + 1 < RT_ELEMENTS(pVM->tm.s.aVirtualSyncCatchUpPeriods)
+                       &&   offLag >= pVM->tm.s.aVirtualSyncCatchUpPeriods[i + 1].u64Start)
+                    i++;
+                STAM_COUNTER_INC(&pVM->tm.s.aStatVirtualSyncCatchupInitial[i]);
+                ASMAtomicWriteU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage, pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u32Percentage);
+                ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, true);
+                Log4(("TM: %'RU64/%'8RU64: catch-up %u%%\n", u64VirtualNow2 - offNew, offLag, pVM->tm.s.u32VirtualSyncCatchUpPercentage));
+            }
+            else
+            {
+                /* don't bother */
+                STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGiveUpBeforeStarting);
+                ASMAtomicWriteU64((uint64_t volatile *)&pVM->tm.s.offVirtualSyncGivenUp, offNew);
+                Log4(("TM: %'RU64/%'8RU64: give up\n", u64VirtualNow2 - offNew, offLag));
+                LogRel(("TM: Not bothering to attempt catching up a %'RU64 ns lag; new total: %'RU64\n", offLag, offNew));
+            }
+        }
+
+        /*
+         * Update the offset and restart the clock.
+         */
+        Assert(!(offNew & RT_BIT_64(63)));
+        ASMAtomicWriteU64(&pVM->tm.s.offVirtualSync, offNew);
+        ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, true);
+    }
+}
+
+
+/**
+ * Deals with stopped Virtual Sync clock.
+ *
+ * This is called by the forced action flag handling code in EM when it
+ * encounters the VM_FF_TM_VIRTUAL_SYNC flag. It is called by all VCPUs and they
+ * will block on the VirtualSyncLock until the pending timers has been executed
+ * and the clock restarted.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ *
+ * @thread  EMTs
+ */
+VMMR3_INT_DECL(void) TMR3VirtualSyncFF(PVM pVM, PVMCPU pVCpu)
+{
+    Log2(("TMR3VirtualSyncFF:\n"));
+
+    /*
+     * The EMT doing the timers is diverted to them.
+     */
+    if (pVCpu->idCpu == pVM->tm.s.idTimerCpu)
+        TMR3TimerQueuesDo(pVM);
+    /*
+     * The other EMTs will block on the virtual sync lock and the first owner
+     * will run the queue and thus restarting the clock.
+     *
+     * Note! This is very suboptimal code wrt to resuming execution when there
+     *       are more than two Virtual CPUs, since they will all have to enter
+     *       the critical section one by one. But it's a very simple solution
+     *       which will have to do the job for now.
+     */
+    else
+    {
+        STAM_PROFILE_START(&pVM->tm.s.StatVirtualSyncFF, a);
+        PDMCritSectEnter(&pVM->tm.s.VirtualSyncLock, VERR_IGNORED);
+        if (pVM->tm.s.fVirtualSyncTicking)
+        {
+            STAM_PROFILE_STOP(&pVM->tm.s.StatVirtualSyncFF, a); /* before the unlock! */
+            PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+            Log2(("TMR3VirtualSyncFF: ticking\n"));
+        }
+        else
+        {
+            PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+
+            /* try run it. */
+            TM_LOCK_TIMERS(pVM);
+            PDMCritSectEnter(&pVM->tm.s.VirtualSyncLock, VERR_IGNORED);
+            if (pVM->tm.s.fVirtualSyncTicking)
+                Log2(("TMR3VirtualSyncFF: ticking (2)\n"));
+            else
+            {
+                ASMAtomicWriteBool(&pVM->tm.s.fRunningVirtualSyncQueue, true);
+                Log2(("TMR3VirtualSyncFF: running queue\n"));
+
+                Assert(!pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule);
+                tmR3TimerQueueRunVirtualSync(pVM);
+                if (pVM->tm.s.fVirtualSyncTicking) /** @todo move into tmR3TimerQueueRunVirtualSync - FIXME */
+                    VM_FF_CLEAR(pVM, VM_FF_TM_VIRTUAL_SYNC);
+
+                ASMAtomicWriteBool(&pVM->tm.s.fRunningVirtualSyncQueue, false);
+            }
+            STAM_PROFILE_STOP(&pVM->tm.s.StatVirtualSyncFF, a); /* before the unlock! */
+            PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock);
+            TM_UNLOCK_TIMERS(pVM);
+        }
+    }
+}
+
+
+/** @name Saved state values
+ * @{ */
+#define TMTIMERSTATE_SAVED_PENDING_STOP         4
+#define TMTIMERSTATE_SAVED_PENDING_SCHEDULE     7
+/** @}  */
+
+
+/**
+ * Saves the state of a timer to a saved state.
+ *
+ * @returns VBox status code.
+ * @param   pTimer          Timer to save.
+ * @param   pSSM            Save State Manager handle.
+ */
+VMMR3DECL(int) TMR3TimerSave(PTMTIMERR3 pTimer, PSSMHANDLE pSSM)
+{
+    LogFlow(("TMR3TimerSave: %p:{enmState=%s, .pszDesc={%s}} pSSM=%p\n", pTimer, tmTimerState(pTimer->enmState), pTimer->pszDesc, pSSM));
+    switch (pTimer->enmState)
+    {
+        case TMTIMERSTATE_STOPPED:
+        case TMTIMERSTATE_PENDING_STOP:
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+            return SSMR3PutU8(pSSM, TMTIMERSTATE_SAVED_PENDING_STOP);
+
+        case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+            AssertMsgFailed(("u64Expire is being updated! (%s)\n", pTimer->pszDesc));
+            if (!RTThreadYield())
+                RTThreadSleep(1);
+            RT_FALL_THRU();
+        case TMTIMERSTATE_ACTIVE:
+        case TMTIMERSTATE_PENDING_SCHEDULE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE:
+            SSMR3PutU8(pSSM, TMTIMERSTATE_SAVED_PENDING_SCHEDULE);
+            return SSMR3PutU64(pSSM, pTimer->u64Expire);
+
+        case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+        case TMTIMERSTATE_EXPIRED_DELIVER:
+        case TMTIMERSTATE_DESTROY:
+        case TMTIMERSTATE_FREE:
+            AssertMsgFailed(("Invalid timer state %d %s (%s)\n", pTimer->enmState, tmTimerState(pTimer->enmState), pTimer->pszDesc));
+            return SSMR3HandleSetStatus(pSSM, VERR_TM_INVALID_STATE);
+    }
+
+    AssertMsgFailed(("Unknown timer state %d (%s)\n", pTimer->enmState, pTimer->pszDesc));
+    return SSMR3HandleSetStatus(pSSM, VERR_TM_UNKNOWN_STATE);
+}
+
+
+/**
+ * Loads the state of a timer from a saved state.
+ *
+ * @returns VBox status code.
+ * @param   pTimer          Timer to restore.
+ * @param   pSSM            Save State Manager handle.
+ */
+VMMR3DECL(int) TMR3TimerLoad(PTMTIMERR3 pTimer, PSSMHANDLE pSSM)
+{
+    Assert(pTimer); Assert(pSSM); VM_ASSERT_EMT(pTimer->pVMR3);
+    LogFlow(("TMR3TimerLoad: %p:{enmState=%s, .pszDesc={%s}} pSSM=%p\n", pTimer, tmTimerState(pTimer->enmState), pTimer->pszDesc, pSSM));
+
+    /*
+     * Load the state and validate it.
+     */
+    uint8_t u8State;
+    int rc = SSMR3GetU8(pSSM, &u8State);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* TMTIMERSTATE_SAVED_XXX: Workaround for accidental state shift in r47786 (2009-05-26 19:12:12). */
+    if (    u8State == TMTIMERSTATE_SAVED_PENDING_STOP + 1
+        ||  u8State == TMTIMERSTATE_SAVED_PENDING_SCHEDULE + 1)
+        u8State--;
+
+    if (    u8State != TMTIMERSTATE_SAVED_PENDING_STOP
+        &&  u8State != TMTIMERSTATE_SAVED_PENDING_SCHEDULE)
+    {
+        AssertLogRelMsgFailed(("u8State=%d\n", u8State));
+        return SSMR3HandleSetStatus(pSSM, VERR_TM_LOAD_STATE);
+    }
+
+    /* Enter the critical sections to make TMTimerSet/Stop happy. */
+    if (pTimer->enmClock == TMCLOCK_VIRTUAL_SYNC)
+        PDMCritSectEnter(&pTimer->pVMR3->tm.s.VirtualSyncLock, VERR_IGNORED);
+    PPDMCRITSECT pCritSect = pTimer->pCritSect;
+    if (pCritSect)
+        PDMCritSectEnter(pCritSect, VERR_IGNORED);
+
+    if (u8State == TMTIMERSTATE_SAVED_PENDING_SCHEDULE)
+    {
+        /*
+         * Load the expire time.
+         */
+        uint64_t u64Expire;
+        rc = SSMR3GetU64(pSSM, &u64Expire);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /*
+         * Set it.
+         */
+        Log(("u8State=%d u64Expire=%llu\n", u8State, u64Expire));
+        rc = TMTimerSet(pTimer, u64Expire);
+    }
+    else
+    {
+        /*
+         * Stop it.
+         */
+        Log(("u8State=%d\n", u8State));
+        rc = TMTimerStop(pTimer);
+    }
+
+    if (pCritSect)
+        PDMCritSectLeave(pCritSect);
+    if (pTimer->enmClock == TMCLOCK_VIRTUAL_SYNC)
+        PDMCritSectLeave(&pTimer->pVMR3->tm.s.VirtualSyncLock);
+
+    /*
+     * On failure set SSM status.
+     */
+    if (RT_FAILURE(rc))
+        rc = SSMR3HandleSetStatus(pSSM, rc);
+    return rc;
+}
+
+
+/**
+ * Skips the state of a timer in a given saved state.
+ *
+ * @returns VBox status.
+ * @param   pSSM            Save State Manager handle.
+ * @param   pfActive        Where to store whether the timer was active
+ *                          when the state was saved.
+ */
+VMMR3DECL(int) TMR3TimerSkip(PSSMHANDLE pSSM, bool *pfActive)
+{
+    Assert(pSSM); AssertPtr(pfActive);
+    LogFlow(("TMR3TimerSkip: pSSM=%p pfActive=%p\n", pSSM, pfActive));
+
+    /*
+     * Load the state and validate it.
+     */
+    uint8_t u8State;
+    int rc = SSMR3GetU8(pSSM, &u8State);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* TMTIMERSTATE_SAVED_XXX: Workaround for accidental state shift in r47786 (2009-05-26 19:12:12). */
+    if (    u8State == TMTIMERSTATE_SAVED_PENDING_STOP + 1
+        ||  u8State == TMTIMERSTATE_SAVED_PENDING_SCHEDULE + 1)
+        u8State--;
+
+    if (    u8State != TMTIMERSTATE_SAVED_PENDING_STOP
+        &&  u8State != TMTIMERSTATE_SAVED_PENDING_SCHEDULE)
+    {
+        AssertLogRelMsgFailed(("u8State=%d\n", u8State));
+        return SSMR3HandleSetStatus(pSSM, VERR_TM_LOAD_STATE);
+    }
+
+    *pfActive = (u8State == TMTIMERSTATE_SAVED_PENDING_SCHEDULE);
+    if (*pfActive)
+    {
+        /*
+         * Load the expire time.
+         */
+        uint64_t u64Expire;
+        rc = SSMR3GetU64(pSSM, &u64Expire);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Associates a critical section with a timer.
+ *
+ * The critical section will be entered prior to doing the timer call back, thus
+ * avoiding potential races between the timer thread and other threads trying to
+ * stop or adjust the timer expiration while it's being delivered. The timer
+ * thread will leave the critical section when the timer callback returns.
+ *
+ * In strict builds, ownership of the critical section will be asserted by
+ * TMTimerSet, TMTimerStop, TMTimerGetExpire and TMTimerDestroy (when called at
+ * runtime).
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_INVALID_HANDLE if the timer handle is NULL or invalid
+ *          (asserted).
+ * @retval  VERR_INVALID_PARAMETER if pCritSect is NULL or has an invalid magic
+ *          (asserted).
+ * @retval  VERR_ALREADY_EXISTS if a critical section was already associated
+ *          with the timer (asserted).
+ * @retval  VERR_INVALID_STATE if the timer isn't stopped.
+ *
+ * @param   pTimer          The timer handle.
+ * @param   pCritSect       The critical section. The caller must make sure this
+ *                          is around for the life time of the timer.
+ *
+ * @thread  Any, but the caller is responsible for making sure the timer is not
+ *          active.
+ */
+VMMR3DECL(int) TMR3TimerSetCritSect(PTMTIMERR3 pTimer, PPDMCRITSECT pCritSect)
+{
+    AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
+    AssertPtrReturn(pCritSect, VERR_INVALID_PARAMETER);
+    const char *pszName = PDMR3CritSectName(pCritSect); /* exploited for validation */
+    AssertReturn(pszName, VERR_INVALID_PARAMETER);
+    AssertReturn(!pTimer->pCritSect, VERR_ALREADY_EXISTS);
+    AssertReturn(pTimer->enmState == TMTIMERSTATE_STOPPED, VERR_INVALID_STATE);
+    LogFlow(("pTimer=%p (%s) pCritSect=%p (%s)\n", pTimer, pTimer->pszDesc, pCritSect, pszName));
+
+    pTimer->pCritSect = pCritSect;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get the real world UTC time adjusted for VM lag.
+ *
+ * @returns pTime.
+ * @param   pVM             The cross context VM structure.
+ * @param   pTime           Where to store the time.
+ */
+VMMR3_INT_DECL(PRTTIMESPEC) TMR3UtcNow(PVM pVM, PRTTIMESPEC pTime)
+{
+    /*
+     * Get a stable set of VirtualSync parameters and calc the lag.
+     */
+    uint64_t offVirtualSync;
+    uint64_t offVirtualSyncGivenUp;
+    do
+    {
+        offVirtualSync        = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync);
+        offVirtualSyncGivenUp = ASMAtomicReadU64((uint64_t volatile *)&pVM->tm.s.offVirtualSyncGivenUp);
+    } while (ASMAtomicReadU64(&pVM->tm.s.offVirtualSync) != offVirtualSync);
+
+    Assert(offVirtualSync >= offVirtualSyncGivenUp);
+    uint64_t const offLag = offVirtualSync - offVirtualSyncGivenUp;
+
+    /*
+     * Get current time and adjust for virtual sync lag and do time displacement.
+     */
+    RTTimeNow(pTime);
+    RTTimeSpecSubNano(pTime, offLag);
+    RTTimeSpecAddNano(pTime, pVM->tm.s.offUTC);
+
+    /*
+     * Log details if the time changed radically (also triggers on first call).
+     */
+    int64_t nsPrev = ASMAtomicXchgS64(&pVM->tm.s.nsLastUtcNow, RTTimeSpecGetNano(pTime));
+    int64_t cNsDelta = RTTimeSpecGetNano(pTime) - nsPrev;
+    if ((uint64_t)RT_ABS(cNsDelta) > RT_NS_1HOUR / 2)
+    {
+        RTTIMESPEC NowAgain;
+        RTTimeNow(&NowAgain);
+        LogRel(("TMR3UtcNow: nsNow=%'RI64 nsPrev=%'RI64 -> cNsDelta=%'RI64 (offLag=%'RI64 offVirtualSync=%'RU64 offVirtualSyncGivenUp=%'RU64, NowAgain=%'RI64)\n",
+                RTTimeSpecGetNano(pTime), nsPrev, cNsDelta, offLag, offVirtualSync, offVirtualSyncGivenUp, RTTimeSpecGetNano(&NowAgain)));
+        if (pVM->tm.s.pszUtcTouchFileOnJump && nsPrev != 0)
+        {
+            RTFILE hFile;
+            int rc = RTFileOpen(&hFile, pVM->tm.s.pszUtcTouchFileOnJump,
+                                RTFILE_O_WRITE | RTFILE_O_APPEND | RTFILE_O_OPEN_CREATE | RTFILE_O_DENY_NONE);
+            if (RT_SUCCESS(rc))
+            {
+                char   szMsg[256];
+                size_t cch;
+                cch = RTStrPrintf(szMsg, sizeof(szMsg),
+                                  "TMR3UtcNow: nsNow=%'RI64 nsPrev=%'RI64 -> cNsDelta=%'RI64 (offLag=%'RI64 offVirtualSync=%'RU64 offVirtualSyncGivenUp=%'RU64, NowAgain=%'RI64)\n",
+                                  RTTimeSpecGetNano(pTime), nsPrev, cNsDelta, offLag, offVirtualSync, offVirtualSyncGivenUp, RTTimeSpecGetNano(&NowAgain));
+                RTFileWrite(hFile, szMsg, cch, NULL);
+                RTFileClose(hFile);
+            }
+        }
+    }
+
+    return pTime;
+}
+
+
+/**
+ * Pauses all clocks except TMCLOCK_REAL.
+ *
+ * @returns VBox status code, all errors are asserted.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @thread  EMT corresponding to Pointer to the VMCPU.
+ */
+VMMR3DECL(int) TMR3NotifySuspend(PVM pVM, PVMCPU pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * The shared virtual clock (includes virtual sync which is tied to it).
+     */
+    TM_LOCK_TIMERS(pVM);                        /* Paranoia: Exploiting the timer lock here. */
+    int rc = tmVirtualPauseLocked(pVM);
+    TM_UNLOCK_TIMERS(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Pause the TSC last since it is normally linked to the virtual
+     * sync clock, so the above code may actually stop both clocks.
+     */
+    if (!pVM->tm.s.fTSCTiedToExecution)
+    {
+        TM_LOCK_TIMERS(pVM);    /* Exploit the timer lock for synchronization. */
+        rc = tmCpuTickPauseLocked(pVM, pVCpu);
+        TM_UNLOCK_TIMERS(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    /*
+     * Update cNsTotal.
+     */
+    uint32_t uGen = ASMAtomicIncU32(&pVCpu->tm.s.uTimesGen); Assert(uGen & 1);
+    pVCpu->tm.s.cNsTotal = RTTimeNanoTS() - pVCpu->tm.s.u64NsTsStartTotal;
+    pVCpu->tm.s.cNsOther = pVCpu->tm.s.cNsTotal - pVCpu->tm.s.cNsExecuting - pVCpu->tm.s.cNsHalted;
+    ASMAtomicWriteU32(&pVCpu->tm.s.uTimesGen, (uGen | 1) + 1);
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resumes all clocks except TMCLOCK_REAL.
+ *
+ * @returns VBox status code, all errors are asserted.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @thread  EMT corresponding to Pointer to the VMCPU.
+ */
+VMMR3DECL(int) TMR3NotifyResume(PVM pVM, PVMCPU pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    int rc;
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    /*
+     * Set u64NsTsStartTotal.  There is no need to back this out if either of
+     * the two calls below fail.
+     */
+    pVCpu->tm.s.u64NsTsStartTotal = RTTimeNanoTS() - pVCpu->tm.s.cNsTotal;
+#endif
+
+    /*
+     * Resume the TSC first since it is normally linked to the virtual sync
+     * clock, so it may actually not be resumed until we've executed the code
+     * below.
+     */
+    if (!pVM->tm.s.fTSCTiedToExecution)
+    {
+        TM_LOCK_TIMERS(pVM);    /* Exploit the timer lock for synchronization. */
+        rc = tmCpuTickResumeLocked(pVM, pVCpu);
+        TM_UNLOCK_TIMERS(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * The shared virtual clock (includes virtual sync which is tied to it).
+     */
+    TM_LOCK_TIMERS(pVM);                        /* Paranoia: Exploiting the timer lock here. */
+    rc = tmVirtualResumeLocked(pVM);
+    TM_UNLOCK_TIMERS(pVM);
+
+    return rc;
+}
+
+
+/**
+ * Sets the warp drive percent of the virtual time.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM structure.
+ * @param   u32Percent  The new percentage. 100 means normal operation.
+ */
+VMMDECL(int) TMR3SetWarpDrive(PUVM pUVM, uint32_t u32Percent)
+{
+    return VMR3ReqPriorityCallWaitU(pUVM, VMCPUID_ANY, (PFNRT)tmR3SetWarpDrive, 2, pUVM, u32Percent);
+}
+
+
+/**
+ * EMT worker for TMR3SetWarpDrive.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        The user mode VM handle.
+ * @param   u32Percent  See TMR3SetWarpDrive().
+ * @internal
+ */
+static DECLCALLBACK(int) tmR3SetWarpDrive(PUVM pUVM, uint32_t u32Percent)
+{
+    PVM    pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    /*
+     * Validate it.
+     */
+    AssertMsgReturn(u32Percent >= 2 && u32Percent <= 20000,
+                    ("%RX32 is not between 2 and 20000 (inclusive).\n", u32Percent),
+                    VERR_INVALID_PARAMETER);
+
+/** @todo This isn't a feature specific to virtual time, move the variables to
+ * TM level and make it affect TMR3UTCNow as well! */
+
+    /*
+     * If the time is running we'll have to pause it before we can change
+     * the warp drive settings.
+     */
+    TM_LOCK_TIMERS(pVM);                        /* Paranoia: Exploiting the timer lock here. */
+    bool fPaused = !!pVM->tm.s.cVirtualTicking;
+    if (fPaused) /** @todo this isn't really working, but wtf. */
+        TMR3NotifySuspend(pVM, pVCpu);
+
+    /** @todo Should switch TM mode to virt-tsc-emulated if it isn't already! */
+    pVM->tm.s.u32VirtualWarpDrivePercentage = u32Percent;
+    pVM->tm.s.fVirtualWarpDrive = u32Percent != 100;
+    LogRel(("TM: u32VirtualWarpDrivePercentage=%RI32 fVirtualWarpDrive=%RTbool\n",
+            pVM->tm.s.u32VirtualWarpDrivePercentage, pVM->tm.s.fVirtualWarpDrive));
+
+    if (fPaused)
+        TMR3NotifyResume(pVM, pVCpu);
+    TM_UNLOCK_TIMERS(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL time without checking
+ * timers or anything.
+ *
+ * @returns The timestamp.
+ * @param   pUVM        The user mode VM structure.
+ *
+ * @remarks See TMVirtualGetNoCheck.
+ */
+VMMR3DECL(uint64_t) TMR3TimeVirtGet(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, UINT64_MAX);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT64_MAX);
+    return TMVirtualGetNoCheck(pVM);
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL time in milliseconds without checking
+ * timers or anything.
+ *
+ * @returns The timestamp in milliseconds.
+ * @param   pUVM        The user mode VM structure.
+ *
+ * @remarks See TMVirtualGetNoCheck.
+ */
+VMMR3DECL(uint64_t) TMR3TimeVirtGetMilli(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, UINT64_MAX);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT64_MAX);
+    return TMVirtualToMilli(pVM, TMVirtualGetNoCheck(pVM));
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL time in microseconds without checking
+ * timers or anything.
+ *
+ * @returns The timestamp in microseconds.
+ * @param   pUVM        The user mode VM structure.
+ *
+ * @remarks See TMVirtualGetNoCheck.
+ */
+VMMR3DECL(uint64_t) TMR3TimeVirtGetMicro(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, UINT64_MAX);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT64_MAX);
+    return TMVirtualToMicro(pVM, TMVirtualGetNoCheck(pVM));
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL time in nanoseconds without checking
+ * timers or anything.
+ *
+ * @returns The timestamp in nanoseconds.
+ * @param   pUVM        The user mode VM structure.
+ *
+ * @remarks See TMVirtualGetNoCheck.
+ */
+VMMR3DECL(uint64_t) TMR3TimeVirtGetNano(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, UINT64_MAX);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT64_MAX);
+    return TMVirtualToNano(pVM, TMVirtualGetNoCheck(pVM));
+}
+
+
+/**
+ * Gets the current warp drive percent.
+ *
+ * @returns The warp drive percent.
+ * @param   pUVM        The user mode VM structure.
+ */
+VMMR3DECL(uint32_t) TMR3GetWarpDrive(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, UINT32_MAX);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
+    return pVM->tm.s.u32VirtualWarpDrivePercentage;
+}
+
+
+/**
+ * Gets the performance information for one virtual CPU as seen by the VMM.
+ *
+ * The returned times covers the period where the VM is running and will be
+ * reset when restoring a previous VM state (at least for the time being).
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_NOT_IMPLEMENTED if not compiled in.
+ * @retval  VERR_INVALID_STATE if the VM handle is bad.
+ * @retval  VERR_INVALID_CPU_ID if idCpu is out of range.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   idCpu           The ID of the virtual CPU which times to get.
+ * @param   pcNsTotal       Where to store the total run time (nano seconds) of
+ *                          the CPU, i.e. the sum of the three other returns.
+ *                          Optional.
+ * @param   pcNsExecuting   Where to store the time (nano seconds) spent
+ *                          executing guest code.  Optional.
+ * @param   pcNsHalted      Where to store the time (nano seconds) spent
+ *                          halted.  Optional
+ * @param   pcNsOther       Where to store the time (nano seconds) spent
+ *                          preempted by the host scheduler, on virtualization
+ *                          overhead and on other tasks.
+ */
+VMMR3DECL(int) TMR3GetCpuLoadTimes(PVM pVM, VMCPUID idCpu, uint64_t *pcNsTotal, uint64_t *pcNsExecuting,
+                                   uint64_t *pcNsHalted, uint64_t *pcNsOther)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_STATE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    /*
+     * Get a stable result set.
+     * This should be way quicker than an EMT request.
+     */
+    PVMCPU      pVCpu        = pVM->apCpusR3[idCpu];
+    uint32_t    uTimesGen    = ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen);
+    uint64_t    cNsTotal     = pVCpu->tm.s.cNsTotal;
+    uint64_t    cNsExecuting = pVCpu->tm.s.cNsExecuting;
+    uint64_t    cNsHalted    = pVCpu->tm.s.cNsHalted;
+    uint64_t    cNsOther     = pVCpu->tm.s.cNsOther;
+    while (   (uTimesGen & 1) /* update in progress */
+           || uTimesGen != ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen))
+    {
+        RTThreadYield();
+        uTimesGen    = ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen);
+        cNsTotal     = pVCpu->tm.s.cNsTotal;
+        cNsExecuting = pVCpu->tm.s.cNsExecuting;
+        cNsHalted    = pVCpu->tm.s.cNsHalted;
+        cNsOther     = pVCpu->tm.s.cNsOther;
+    }
+
+    /*
+     * Fill in the return values.
+     */
+    if (pcNsTotal)
+        *pcNsTotal = cNsTotal;
+    if (pcNsExecuting)
+        *pcNsExecuting = cNsExecuting;
+    if (pcNsHalted)
+        *pcNsHalted = cNsHalted;
+    if (pcNsOther)
+        *pcNsOther = cNsOther;
+
+    return VINF_SUCCESS;
+
+#else
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/**
+ * Gets the performance information for one virtual CPU as seen by the VMM in
+ * percents.
+ *
+ * The returned times covers the period where the VM is running and will be
+ * reset when restoring a previous VM state (at least for the time being).
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_NOT_IMPLEMENTED if not compiled in.
+ * @retval  VERR_INVALID_VM_HANDLE if the VM handle is bad.
+ * @retval  VERR_INVALID_CPU_ID if idCpu is out of range.
+ *
+ * @param   pUVM            The usermode VM structure.
+ * @param   idCpu           The ID of the virtual CPU which times to get.
+ * @param   pcMsInterval    Where to store the interval of the percentages in
+ *                          milliseconds. Optional.
+ * @param   pcPctExecuting  Where to return the percentage of time spent
+ *                          executing guest code.  Optional.
+ * @param   pcPctHalted     Where to return the percentage of time spent halted.
+ *                          Optional
+ * @param   pcPctOther      Where to return the percentage of time spent
+ *                          preempted by the host scheduler, on virtualization
+ *                          overhead and on other tasks.
+ */
+VMMR3DECL(int) TMR3GetCpuLoadPercents(PUVM pUVM, VMCPUID idCpu, uint64_t *pcMsInterval, uint8_t *pcPctExecuting,
+                                      uint8_t *pcPctHalted, uint8_t *pcPctOther)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu == VMCPUID_ALL || idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    TMCPULOADSTATE volatile *pState;
+    if (idCpu == VMCPUID_ALL)
+        pState = &pVM->tm.s.CpuLoad;
+    else
+        pState = &pVM->apCpusR3[idCpu]->tm.s.CpuLoad;
+
+    if (pcMsInterval)
+        *pcMsInterval   = RT_MS_1SEC;
+    if (pcPctExecuting)
+        *pcPctExecuting = pState->cPctExecuting;
+    if (pcPctHalted)
+        *pcPctHalted    = pState->cPctHalted;
+    if (pcPctOther)
+        *pcPctOther     = pState->cPctOther;
+
+    return VINF_SUCCESS;
+
+#else
+    RT_NOREF(pcMsInterval, pcPctExecuting, pcPctHalted, pcPctOther);
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+
+/**
+ * Helper for tmR3CpuLoadTimer.
+ * @returns
+ * @param   pState          The state to update.
+ * @param   cNsTotal        Total time.
+ * @param   cNsExecuting    Time executing.
+ * @param   cNsHalted       Time halted.
+ */
+DECLINLINE(void) tmR3CpuLoadTimerMakeUpdate(PTMCPULOADSTATE pState, uint64_t cNsTotal, uint64_t cNsExecuting, uint64_t cNsHalted)
+{
+    /* Calc & update deltas */
+    uint64_t cNsTotalDelta      = cNsTotal     - pState->cNsPrevTotal;
+    pState->cNsPrevTotal        = cNsTotal;
+
+    uint64_t cNsExecutingDelta  = cNsExecuting - pState->cNsPrevExecuting;
+    pState->cNsPrevExecuting    = cNsExecuting;
+
+    uint64_t cNsHaltedDelta     = cNsHalted    - pState->cNsPrevHalted;
+    pState->cNsPrevHalted       = cNsHalted;
+
+    /* Calc pcts. */
+    uint8_t cPctExecuting, cPctHalted, cPctOther;
+    if (!cNsTotalDelta)
+    {
+        cPctExecuting = 0;
+        cPctHalted    = 100;
+        cPctOther     = 0;
+    }
+    else if (cNsTotalDelta < UINT64_MAX / 4)
+    {
+        cPctExecuting = (uint8_t)(cNsExecutingDelta * 100 / cNsTotalDelta);
+        cPctHalted    = (uint8_t)(cNsHaltedDelta    * 100 / cNsTotalDelta);
+        cPctOther     = (uint8_t)((cNsTotalDelta - cNsExecutingDelta - cNsHaltedDelta) * 100 / cNsTotalDelta);
+    }
+    else
+    {
+        cPctExecuting = 0;
+        cPctHalted    = 100;
+        cPctOther     = 0;
+    }
+
+    /* Update percentages: */
+    size_t idxHistory = pState->idxHistory + 1;
+    if (idxHistory >= RT_ELEMENTS(pState->aHistory))
+        idxHistory = 0;
+
+    pState->cPctExecuting                      = cPctExecuting;
+    pState->cPctHalted                         = cPctHalted;
+    pState->cPctOther                          = cPctOther;
+
+    pState->aHistory[idxHistory].cPctExecuting = cPctExecuting;
+    pState->aHistory[idxHistory].cPctHalted    = cPctHalted;
+    pState->aHistory[idxHistory].cPctOther     = cPctOther;
+
+    pState->idxHistory = (uint16_t)idxHistory;
+    if (pState->cHistoryEntries < RT_ELEMENTS(pState->aHistory))
+        pState->cHistoryEntries++;
+}
+
+
+/**
+ * Timer callback that calculates the CPU load since the last time it was
+ * called.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pTimer              The timer.
+ * @param   pvUser              NULL, unused.
+ */
+static DECLCALLBACK(void) tmR3CpuLoadTimer(PVM pVM, PTMTIMER pTimer, void *pvUser)
+{
+    /*
+     * Re-arm the timer first.
+     */
+    int rc = TMTimerSetMillies(pTimer, 1000);
+    AssertLogRelRC(rc);
+    NOREF(pvUser);
+
+    /*
+     * Update the values for each CPU.
+     */
+    uint64_t cNsTotalAll       = 0;
+    uint64_t cNsExecutingAll   = 0;
+    uint64_t cNsHaltedAll      = 0;
+    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
+    {
+        PVMCPU      pVCpu = pVM->apCpusR3[iCpu];
+
+        /* Try get a stable data set. */
+        uint32_t    cTries       = 3;
+        uint32_t    uTimesGen    = ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen);
+        uint64_t    cNsTotal     = pVCpu->tm.s.cNsTotal;
+        uint64_t    cNsExecuting = pVCpu->tm.s.cNsExecuting;
+        uint64_t    cNsHalted    = pVCpu->tm.s.cNsHalted;
+        while (RT_UNLIKELY(   (uTimesGen & 1) /* update in progress */
+                           || uTimesGen != ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen)))
+        {
+            if (!--cTries)
+                break;
+            ASMNopPause();
+            uTimesGen    = ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen);
+            cNsTotal     = pVCpu->tm.s.cNsTotal;
+            cNsExecuting = pVCpu->tm.s.cNsExecuting;
+            cNsHalted    = pVCpu->tm.s.cNsHalted;
+        }
+
+        /* Totals */
+        cNsTotalAll     += cNsTotal;
+        cNsExecutingAll += cNsExecuting;
+        cNsHaltedAll    += cNsHalted;
+
+        /* Calc the PCTs and update the state. */
+        tmR3CpuLoadTimerMakeUpdate(&pVCpu->tm.s.CpuLoad, cNsTotal, cNsExecuting, cNsHalted);
+    }
+
+    /*
+     * Update the value for all the CPUs.
+     */
+    tmR3CpuLoadTimerMakeUpdate(&pVM->tm.s.CpuLoad, cNsTotalAll, cNsExecutingAll, cNsHaltedAll);
+
+}
+
+#endif /* !VBOX_WITHOUT_NS_ACCOUNTING */
+
+
+/**
+ * @callback_method_impl{PFNVMMEMTRENDEZVOUS,
+ *      Worker for TMR3CpuTickParavirtEnable}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) tmR3CpuTickParavirtEnable(PVM pVM, PVMCPU pVCpuEmt, void *pvData)
+{
+    AssertPtr(pVM); Assert(pVM->tm.s.fTSCModeSwitchAllowed); NOREF(pVCpuEmt); NOREF(pvData);
+    Assert(pVM->tm.s.enmTSCMode != TMTSCMODE_REAL_TSC_OFFSET);
+    Assert(pVM->tm.s.enmTSCMode != TMTSCMODE_NATIVE_API); /** @todo figure out NEM/win and paravirt */
+    Assert(tmR3HasFixedTSC(pVM));
+
+    /*
+     * The return value of TMCpuTickGet() and the guest's TSC value for each
+     * CPU must remain constant across the TM TSC mode-switch.  Thus we have
+     * the following equation (new/old signifies the new/old tsc modes):
+     *      uNewTsc = uOldTsc
+     *
+     * Where (see tmCpuTickGetInternal):
+     *      uOldTsc = uRawOldTsc - offTscRawSrcOld
+     *      uNewTsc = uRawNewTsc - offTscRawSrcNew
+     *
+     * Solve it for offTscRawSrcNew without replacing uOldTsc:
+     *     uRawNewTsc - offTscRawSrcNew = uOldTsc
+     *  => -offTscRawSrcNew = uOldTsc - uRawNewTsc
+     *  => offTscRawSrcNew  = uRawNewTsc - uOldTsc
+     */
+    uint64_t uRawOldTsc = tmR3CpuTickGetRawVirtualNoCheck(pVM);
+    uint64_t uRawNewTsc = SUPReadTsc();
+    uint32_t cCpus = pVM->cCpus;
+    for (uint32_t i = 0; i < cCpus; i++)
+    {
+        PVMCPU   pVCpu   = pVM->apCpusR3[i];
+        uint64_t uOldTsc = uRawOldTsc - pVCpu->tm.s.offTSCRawSrc;
+        pVCpu->tm.s.offTSCRawSrc = uRawNewTsc - uOldTsc;
+        Assert(uRawNewTsc - pVCpu->tm.s.offTSCRawSrc >= uOldTsc); /* paranoia^256 */
+    }
+
+    LogRel(("TM: Switching TSC mode from '%s' to '%s'\n", tmR3GetTSCModeNameEx(pVM->tm.s.enmTSCMode),
+            tmR3GetTSCModeNameEx(TMTSCMODE_REAL_TSC_OFFSET)));
+    pVM->tm.s.enmTSCMode = TMTSCMODE_REAL_TSC_OFFSET;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Notify TM that the guest has enabled usage of a paravirtualized TSC.
+ *
+ * This may perform a EMT rendezvous and change the TSC virtualization mode.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) TMR3CpuTickParavirtEnable(PVM pVM)
+{
+    int rc = VINF_SUCCESS;
+    if (pVM->tm.s.fTSCModeSwitchAllowed)
+    {
+        if (pVM->tm.s.enmTSCMode != TMTSCMODE_REAL_TSC_OFFSET)
+            rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, tmR3CpuTickParavirtEnable, NULL);
+    }
+    else
+        LogRel(("TM: Host/VM is not suitable for using TSC mode '%s', request to change TSC mode ignored\n",
+                tmR3GetTSCModeNameEx(TMTSCMODE_REAL_TSC_OFFSET)));
+    pVM->tm.s.fParavirtTscEnabled = true;
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{PFNVMMEMTRENDEZVOUS,
+ *      Worker for TMR3CpuTickParavirtDisable}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) tmR3CpuTickParavirtDisable(PVM pVM, PVMCPU pVCpuEmt, void *pvData)
+{
+    AssertPtr(pVM); Assert(pVM->tm.s.fTSCModeSwitchAllowed); NOREF(pVCpuEmt);
+    Assert(   pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET
+           && pVM->tm.s.enmTSCMode != pVM->tm.s.enmOriginalTSCMode);
+    RT_NOREF1(pvData);
+
+    /*
+     * See tmR3CpuTickParavirtEnable for an explanation of the conversion math.
+     */
+    uint64_t uRawOldTsc = SUPReadTsc();
+    uint64_t uRawNewTsc = tmR3CpuTickGetRawVirtualNoCheck(pVM);
+    uint32_t cCpus = pVM->cCpus;
+    for (uint32_t i = 0; i < cCpus; i++)
+    {
+        PVMCPU   pVCpu   = pVM->apCpusR3[i];
+        uint64_t uOldTsc = uRawOldTsc - pVCpu->tm.s.offTSCRawSrc;
+        pVCpu->tm.s.offTSCRawSrc = uRawNewTsc - uOldTsc;
+        Assert(uRawNewTsc - pVCpu->tm.s.offTSCRawSrc >= uOldTsc); /* paranoia^256 */
+
+        /* Update the last-seen tick here as we havent't been updating it (as we don't
+           need it) while in pure TSC-offsetting mode. */
+        pVCpu->tm.s.u64TSCLastSeen = uOldTsc;
+    }
+
+    LogRel(("TM: Switching TSC mode from '%s' to '%s'\n", tmR3GetTSCModeNameEx(pVM->tm.s.enmTSCMode),
+            tmR3GetTSCModeNameEx(pVM->tm.s.enmOriginalTSCMode)));
+    pVM->tm.s.enmTSCMode = pVM->tm.s.enmOriginalTSCMode;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Notify TM that the guest has disabled usage of a paravirtualized TSC.
+ *
+ * If TMR3CpuTickParavirtEnable() changed the TSC virtualization mode, this will
+ * perform an EMT  rendezvous to revert those changes.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) TMR3CpuTickParavirtDisable(PVM pVM)
+{
+    int rc = VINF_SUCCESS;
+    if (   pVM->tm.s.fTSCModeSwitchAllowed
+        && pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET
+        && pVM->tm.s.enmTSCMode != pVM->tm.s.enmOriginalTSCMode)
+        rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, tmR3CpuTickParavirtDisable, NULL);
+    pVM->tm.s.fParavirtTscEnabled = false;
+    return rc;
+}
+
+
+/**
+ * Check whether the guest can be presented a fixed rate & monotonic TSC.
+ *
+ * @returns true if TSC is stable, false otherwise.
+ * @param   pVM                     The cross context VM structure.
+ * @param   fWithParavirtEnabled    Whether it's fixed & monotonic when
+ *                                  paravirt. TSC is enabled or not.
+ *
+ * @remarks Must be called only after TMR3InitFinalize().
+ */
+VMMR3_INT_DECL(bool) TMR3CpuTickIsFixedRateMonotonic(PVM pVM, bool fWithParavirtEnabled)
+{
+    /** @todo figure out what exactly we want here later. */
+    NOREF(fWithParavirtEnabled);
+    return (   tmR3HasFixedTSC(pVM)                                        /* Host has fixed-rate TSC. */
+            && g_pSUPGlobalInfoPage->u32Mode != SUPGIPMODE_ASYNC_TSC);     /* GIP thinks it's monotonic. */
+}
+
+
+/**
+ * Gets the 5 char clock name for the info tables.
+ *
+ * @returns The name.
+ * @param   enmClock        The clock.
+ */
+DECLINLINE(const char *) tmR3Get5CharClockName(TMCLOCK enmClock)
+{
+    switch (enmClock)
+    {
+        case TMCLOCK_REAL:          return "Real ";
+        case TMCLOCK_VIRTUAL:       return "Virt ";
+        case TMCLOCK_VIRTUAL_SYNC:  return "VrSy ";
+        case TMCLOCK_TSC:           return "TSC  ";
+        default:                    return "Bad  ";
+    }
+}
+
+
+/**
+ * Display all timers.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) tmR3TimerInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    pHlp->pfnPrintf(pHlp,
+                    "Timers (pVM=%p)\n"
+                    "%.*s %.*s %.*s %.*s Clock %18s %18s %6s %-25s Description\n",
+                    pVM,
+                    sizeof(RTR3PTR) * 2,        "pTimerR3        ",
+                    sizeof(int32_t) * 2,        "offNext         ",
+                    sizeof(int32_t) * 2,        "offPrev         ",
+                    sizeof(int32_t) * 2,        "offSched        ",
+                                                "Time",
+                                                "Expire",
+                                                "HzHint",
+                                                "State");
+    TM_LOCK_TIMERS(pVM);
+    for (PTMTIMERR3 pTimer = pVM->tm.s.pCreated; pTimer; pTimer = pTimer->pBigNext)
+    {
+        pHlp->pfnPrintf(pHlp,
+                        "%p %08RX32 %08RX32 %08RX32 %s %18RU64 %18RU64 %6RU32 %-25s %s\n",
+                        pTimer,
+                        pTimer->offNext,
+                        pTimer->offPrev,
+                        pTimer->offScheduleNext,
+                        tmR3Get5CharClockName(pTimer->enmClock),
+                        TMTimerGet(pTimer),
+                        pTimer->u64Expire,
+                        pTimer->uHzHint,
+                        tmTimerState(pTimer->enmState),
+                        pTimer->pszDesc);
+    }
+    TM_UNLOCK_TIMERS(pVM);
+}
+
+
+/**
+ * Display all active timers.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) tmR3TimerInfoActive(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    pHlp->pfnPrintf(pHlp,
+                    "Active Timers (pVM=%p)\n"
+                    "%.*s %.*s %.*s %.*s Clock %18s %18s %6s %-25s Description\n",
+                    pVM,
+                    sizeof(RTR3PTR) * 2,        "pTimerR3        ",
+                    sizeof(int32_t) * 2,        "offNext         ",
+                    sizeof(int32_t) * 2,        "offPrev         ",
+                    sizeof(int32_t) * 2,        "offSched        ",
+                                                "Time",
+                                                "Expire",
+                                                "HzHint",
+                                                "State");
+    for (unsigned iQueue = 0; iQueue < TMCLOCK_MAX; iQueue++)
+    {
+        TM_LOCK_TIMERS(pVM);
+        for (PTMTIMERR3 pTimer = TMTIMER_GET_HEAD(&pVM->tm.s.paTimerQueuesR3[iQueue]);
+             pTimer;
+             pTimer = TMTIMER_GET_NEXT(pTimer))
+        {
+            pHlp->pfnPrintf(pHlp,
+                            "%p %08RX32 %08RX32 %08RX32 %s %18RU64 %18RU64 %6RU32 %-25s %s\n",
+                            pTimer,
+                            pTimer->offNext,
+                            pTimer->offPrev,
+                            pTimer->offScheduleNext,
+                            tmR3Get5CharClockName(pTimer->enmClock),
+                            TMTimerGet(pTimer),
+                            pTimer->u64Expire,
+                            pTimer->uHzHint,
+                            tmTimerState(pTimer->enmState),
+                            pTimer->pszDesc);
+        }
+        TM_UNLOCK_TIMERS(pVM);
+    }
+}
+
+
+/**
+ * Display all clocks.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) tmR3InfoClocks(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+
+    /*
+     * Read the times first to avoid more than necessary time variation.
+     */
+    const uint64_t u64Virtual     = TMVirtualGet(pVM);
+    const uint64_t u64VirtualSync = TMVirtualSyncGet(pVM);
+    const uint64_t u64Real        = TMRealGet(pVM);
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU   pVCpu  = pVM->apCpusR3[i];
+        uint64_t u64TSC = TMCpuTickGet(pVCpu);
+
+        /*
+         * TSC
+         */
+        pHlp->pfnPrintf(pHlp,
+                        "Cpu Tick: %18RU64 (%#016RX64) %RU64Hz %s - virtualized",
+                        u64TSC, u64TSC, TMCpuTicksPerSecond(pVM),
+                        pVCpu->tm.s.fTSCTicking ? "ticking" : "paused");
+        if (pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET)
+        {
+            pHlp->pfnPrintf(pHlp, " - real tsc offset");
+            if (pVCpu->tm.s.offTSCRawSrc)
+                pHlp->pfnPrintf(pHlp, "\n          offset %RU64", pVCpu->tm.s.offTSCRawSrc);
+        }
+        else if (pVM->tm.s.enmTSCMode == TMTSCMODE_NATIVE_API)
+            pHlp->pfnPrintf(pHlp, " - native api");
+        else
+            pHlp->pfnPrintf(pHlp, " - virtual clock");
+        pHlp->pfnPrintf(pHlp, "\n");
+    }
+
+    /*
+     * virtual
+     */
+    pHlp->pfnPrintf(pHlp,
+                    " Virtual: %18RU64 (%#016RX64) %RU64Hz %s",
+                    u64Virtual, u64Virtual, TMVirtualGetFreq(pVM),
+                    pVM->tm.s.cVirtualTicking ? "ticking" : "paused");
+    if (pVM->tm.s.fVirtualWarpDrive)
+        pHlp->pfnPrintf(pHlp, " WarpDrive %RU32 %%", pVM->tm.s.u32VirtualWarpDrivePercentage);
+    pHlp->pfnPrintf(pHlp, "\n");
+
+    /*
+     * virtual sync
+     */
+    pHlp->pfnPrintf(pHlp,
+                    "VirtSync: %18RU64 (%#016RX64) %s%s",
+                    u64VirtualSync, u64VirtualSync,
+                    pVM->tm.s.fVirtualSyncTicking ? "ticking" : "paused",
+                    pVM->tm.s.fVirtualSyncCatchUp ? " - catchup" : "");
+    if (pVM->tm.s.offVirtualSync)
+    {
+        pHlp->pfnPrintf(pHlp, "\n          offset %RU64", pVM->tm.s.offVirtualSync);
+        if (pVM->tm.s.u32VirtualSyncCatchUpPercentage)
+            pHlp->pfnPrintf(pHlp, "  catch-up rate %u %%", pVM->tm.s.u32VirtualSyncCatchUpPercentage);
+    }
+    pHlp->pfnPrintf(pHlp, "\n");
+
+    /*
+     * real
+     */
+    pHlp->pfnPrintf(pHlp,
+                    "    Real: %18RU64 (%#016RX64) %RU64Hz\n",
+                    u64Real, u64Real, TMRealGetFreq(pVM));
+}
+
+
+/**
+ * Helper for tmR3InfoCpuLoad that adjust @a uPct to the given graph width.
+ */
+DECLINLINE(size_t) tmR3InfoCpuLoadAdjustWidth(size_t uPct, size_t cchWidth)
+{
+    if (cchWidth != 100)
+        uPct = (uPct + 0.5) * (cchWidth / 100.0);
+    return uPct;
+}
+
+
+/**
+ * @callback_method_impl{FNDBGFINFOARGVINT}
+ */
+static DECLCALLBACK(void) tmR3InfoCpuLoad(PVM pVM, PCDBGFINFOHLP pHlp, int cArgs, char **papszArgs)
+{
+    char szTmp[1024];
+
+    /*
+     * Parse arguments.
+     */
+    PTMCPULOADSTATE pState      = &pVM->tm.s.CpuLoad;
+    VMCPUID         idCpu       = 0;
+    bool            fAllCpus    = true;
+    bool            fExpGraph   = true;
+    uint32_t        cchWidth    = 80;
+    uint32_t        cPeriods    = RT_ELEMENTS(pState->aHistory);
+    uint32_t        cRows       = 60;
+
+    static const RTGETOPTDEF s_aOptions[] =
+    {
+        { "all",            'a',    RTGETOPT_REQ_NOTHING },
+        { "cpu",            'c',    RTGETOPT_REQ_UINT32 },
+        { "periods",        'p',    RTGETOPT_REQ_UINT32 },
+        { "rows",           'r',    RTGETOPT_REQ_UINT32 },
+        { "uni",            'u',    RTGETOPT_REQ_NOTHING },
+        { "uniform",        'u',    RTGETOPT_REQ_NOTHING },
+        { "width",          'w',    RTGETOPT_REQ_UINT32 },
+        { "exp",            'x',    RTGETOPT_REQ_NOTHING },
+        { "exponential",    'x',    RTGETOPT_REQ_NOTHING },
+    };
+
+    RTGETOPTSTATE State;
+    int rc = RTGetOptInit(&State, cArgs, papszArgs, s_aOptions, RT_ELEMENTS(s_aOptions), 0, 0 /*fFlags*/);
+    AssertRC(rc);
+
+    RTGETOPTUNION ValueUnion;
+    while ((rc = RTGetOpt(&State, &ValueUnion)) != 0)
+    {
+        switch (rc)
+        {
+            case 'a':
+                pState   = &pVM->apCpusR3[0]->tm.s.CpuLoad;
+                idCpu    = 0;
+                fAllCpus = true;
+                break;
+            case 'c':
+                if (ValueUnion.u32 < pVM->cCpus)
+                {
+                    pState = &pVM->apCpusR3[ValueUnion.u32]->tm.s.CpuLoad;
+                    idCpu  = ValueUnion.u32;
+                }
+                else
+                {
+                    pState = &pVM->tm.s.CpuLoad;
+                    idCpu  = VMCPUID_ALL;
+                }
+                fAllCpus = false;
+                break;
+            case 'p':
+                cPeriods = RT_MIN(RT_MAX(ValueUnion.u32, 1), RT_ELEMENTS(pState->aHistory));
+                break;
+            case 'r':
+                cRows = RT_MIN(RT_MAX(ValueUnion.u32, 5), RT_ELEMENTS(pState->aHistory));
+                break;
+            case 'w':
+                cchWidth = RT_MIN(RT_MAX(ValueUnion.u32, 10), sizeof(szTmp) - 32);
+                break;
+            case 'x':
+                fExpGraph = true;
+                break;
+            case 'u':
+                fExpGraph = false;
+                break;
+            case 'h':
+                pHlp->pfnPrintf(pHlp,
+                                "Usage: cpuload [parameters]\n"
+                                "  all, -a\n"
+                                "    Show statistics for all CPUs. (default)\n"
+                                "  cpu=id, -c id\n"
+                                "    Show statistics for the specified CPU ID.  Show combined stats if out of range.\n"
+                                "  periods=count, -p count\n"
+                                "    Number of periods to show.  Default: all\n"
+                                "  rows=count, -r count\n"
+                                "    Number of rows in the graphs.  Default: 60\n"
+                                "  width=count, -w count\n"
+                                "    Core graph width in characters. Default: 80\n"
+                                "  exp, exponential, -e\n"
+                                "    Do 1:1 for more recent half / 30 seconds of the graph, combine the\n"
+                                "    rest into increasinly larger chunks.  Default.\n"
+                                "  uniform, uni, -u\n"
+                                "    Combine periods into rows in a uniform manner for the whole graph.\n");
+                return;
+            default:
+                pHlp->pfnGetOptError(pHlp, rc, &ValueUnion, &State);
+                return;
+        }
+    }
+
+    /*
+     * Do the job.
+     */
+    for (;;)
+    {
+        uint32_t const cMaxPeriods = pState->cHistoryEntries;
+        if (cPeriods > cMaxPeriods)
+            cPeriods = cMaxPeriods;
+        if (cPeriods > 0)
+        {
+            if (fAllCpus)
+            {
+                if (idCpu > 0)
+                    pHlp->pfnPrintf(pHlp, "\n");
+                pHlp->pfnPrintf(pHlp, "    CPU load for virtual CPU %#04x\n"
+                                      "    -------------------------------\n", idCpu);
+            }
+
+            /*
+             * Figure number of periods per chunk.  We can either do this in a linear
+             * fashion or a exponential fashion that compresses old history more.
+             */
+            size_t cPerRowDecrement = 0;
+            size_t cPeriodsPerRow   = 1;
+            if (cRows < cPeriods)
+            {
+                if (!fExpGraph)
+                    cPeriodsPerRow   = (cPeriods + cRows / 2) / cRows;
+                else
+                {
+                    /* The last 30 seconds or half of the rows are 1:1, the other part
+                       is in increasing period counts.  Code is a little simple but seems
+                       to do the job most of the time, which is all I have time now. */
+                    size_t cPeriodsOneToOne = RT_MIN(30, cRows / 2);
+                    size_t cRestRows        = cRows    - cPeriodsOneToOne;
+                    size_t cRestPeriods     = cPeriods - cPeriodsOneToOne;
+
+                    size_t cPeriodsInWindow = 0;
+                    for (cPeriodsPerRow = 0; cPeriodsPerRow <= cRestRows && cPeriodsInWindow < cRestPeriods; cPeriodsPerRow++)
+                        cPeriodsInWindow += cPeriodsPerRow + 1;
+
+                    size_t iLower = 1;
+                    while (cPeriodsInWindow < cRestPeriods)
+                    {
+                        cPeriodsPerRow++;
+                        cPeriodsInWindow += cPeriodsPerRow;
+                        cPeriodsInWindow -= iLower;
+                        iLower++;
+                    }
+
+                    cPerRowDecrement = 1;
+                }
+            }
+
+            /*
+             * Do the work.
+             */
+            size_t cPctExecuting       = 0;
+            size_t cPctOther           = 0;
+            size_t cPeriodsAccumulated = 0;
+
+            size_t cRowsLeft = cRows;
+            size_t iHistory  = (pState->idxHistory - cPeriods) % RT_ELEMENTS(pState->aHistory);
+            while (cPeriods-- > 0)
+            {
+                iHistory++;
+                if (iHistory >= RT_ELEMENTS(pState->aHistory))
+                    iHistory = 0;
+
+                cPctExecuting        += pState->aHistory[iHistory].cPctExecuting;
+                cPctOther            += pState->aHistory[iHistory].cPctOther;
+                cPeriodsAccumulated  += 1;
+                if (   cPeriodsAccumulated >= cPeriodsPerRow
+                    || cPeriods < cRowsLeft)
+                {
+                    /*
+                     * Format and output the line.
+                     */
+                    size_t offTmp = 0;
+                    size_t i      = tmR3InfoCpuLoadAdjustWidth(cPctExecuting / cPeriodsAccumulated, cchWidth);
+                    while (i-- > 0)
+                        szTmp[offTmp++] = '#';
+                    i = tmR3InfoCpuLoadAdjustWidth(cPctOther / cPeriodsAccumulated, cchWidth);
+                    while (i-- > 0)
+                        szTmp[offTmp++] = 'O';
+                    szTmp[offTmp] = '\0';
+
+                    cRowsLeft--;
+                    pHlp->pfnPrintf(pHlp, "%3zus: %s\n", cPeriods + cPeriodsAccumulated / 2, szTmp);
+
+                    /* Reset the state: */
+                    cPctExecuting       = 0;
+                    cPctOther           = 0;
+                    cPeriodsAccumulated = 0;
+                    if (cPeriodsPerRow > cPerRowDecrement)
+                        cPeriodsPerRow -= cPerRowDecrement;
+                }
+            }
+            pHlp->pfnPrintf(pHlp, "    (#=guest, O=VMM overhead)  idCpu=%#x\n", idCpu);
+
+        }
+        else
+            pHlp->pfnPrintf(pHlp, "No load data.\n");
+
+        /*
+         * Next CPU if we're display all.
+         */
+        if (!fAllCpus)
+            break;
+        idCpu++;
+        if (idCpu >= pVM->cCpus)
+            break;
+        pState   = &pVM->apCpusR3[idCpu]->tm.s.CpuLoad;
+    }
+
+}
+
+
+/**
+ * Gets the descriptive TM TSC mode name given the enum value.
+ *
+ * @returns The name.
+ * @param   enmMode             The mode to name.
+ */
+static const char *tmR3GetTSCModeNameEx(TMTSCMODE enmMode)
+{
+    switch (enmMode)
+    {
+        case TMTSCMODE_REAL_TSC_OFFSET:    return "RealTscOffset";
+        case TMTSCMODE_VIRT_TSC_EMULATED:  return "VirtTscEmulated";
+        case TMTSCMODE_DYNAMIC:            return "Dynamic";
+        case TMTSCMODE_NATIVE_API:         return "NativeApi";
+        default:                           return "???";
+    }
+}
+
+
+/**
+ * Gets the descriptive TM TSC mode name.
+ *
+ * @returns The name.
+ * @param   pVM      The cross context VM structure.
+ */
+static const char *tmR3GetTSCModeName(PVM pVM)
+{
+    Assert(pVM);
+    return tmR3GetTSCModeNameEx(pVM->tm.s.enmTSCMode);
+}
+
diff --git a/src/VBox/VMM/VMMR3/TRPM.cpp b/src/VBox/VMM/VMMR3/TRPM.cpp
new file mode 100644
index 00000000..3ede9f16
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/TRPM.cpp
@@ -0,0 +1,461 @@
+/* $Id: TRPM.cpp $ */
+/** @file
+ * TRPM - The Trap Monitor.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_trpm   TRPM - The Trap Monitor
+ *
+ * The Trap Monitor (TRPM) is responsible for all trap and interrupt handling in
+ * the VMM.  It plays a major role in raw-mode execution and a lesser one in the
+ * hardware assisted mode.
+ *
+ * Note first, the following will use trap as a collective term for faults,
+ * aborts and traps.
+ *
+ * @see grp_trpm
+ *
+ *
+ * @section sec_trpm_rc     Raw-Mode Context
+ *
+ * When executing in the raw-mode context, TRPM will be managing the IDT and
+ * processing all traps and interrupts.  It will also monitor the guest IDT
+ * because CSAM wishes to know about changes to it (trap/interrupt/syscall
+ * handler patching) and TRPM needs to keep the \#BP gate in sync (ring-3
+ * considerations).  See TRPMR3SyncIDT and CSAMR3CheckGates.
+ *
+ * External interrupts will be forwarded to the host context by the quickest
+ * possible route where they will be reasserted.  The other events will be
+ * categorized into virtualization traps, genuine guest traps and hypervisor
+ * traps.  The latter group may be recoverable depending on when they happen and
+ * whether there is a handler for it, otherwise it will cause a guru meditation.
+ *
+ * TRPM distinguishes the between the first two (virt and guest traps) and the
+ * latter (hyper) by checking the CPL of the trapping code, if CPL == 0 then
+ * it's a hyper trap otherwise it's a virt/guest trap.  There are three trap
+ * dispatcher tables, one ad-hoc for one time traps registered via
+ * TRPMGCSetTempHandler(), one for hyper traps and one for virt/guest traps.
+ * The latter two live in TRPMGCHandlersA.asm, the former in the VM structure.
+ *
+ * The raw-mode context trap handlers found in TRPMGCHandlers.cpp (for the most
+ * part), will call up the other VMM sub-systems depending on what it things
+ * happens.  The two most busy traps are page faults (\#PF) and general
+ * protection fault/trap (\#GP).
+ *
+ * Before resuming guest code after having taken a virtualization trap or
+ * injected a guest trap, TRPM will check for pending forced action and
+ * every now and again let TM check for timed out timers.  This allows code that
+ * is being executed as part of virtualization traps to signal ring-3 exits,
+ * page table resyncs and similar without necessarily using the status code.  It
+ * also make sure we're more responsive to timers and requests from other
+ * threads (necessarily running on some different core/cpu in most cases).
+ *
+ *
+ * @section sec_trpm_all    All Contexts
+ *
+ * TRPM will also dispatch / inject interrupts and traps to the guest, both when
+ * in raw-mode and when in hardware assisted mode.  See TRPMInject().
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_TRPM
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/iem.h>
+#include "TRPMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <iprt/alloc.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** TRPM saved state version. */
+#define TRPM_SAVED_STATE_VERSION                10
+#define TRPM_SAVED_STATE_VERSION_PRE_ICEBP      9   /* INT1/ICEBP support bumped the version */
+#define TRPM_SAVED_STATE_VERSION_UNI            8   /* SMP support bumped the version */
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static DECLCALLBACK(int) trpmR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int) trpmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(void) trpmR3InfoEvent(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+/**
+ * Initializes the Trap Manager
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(int) TRPMR3Init(PVM pVM)
+{
+    LogFlow(("TRPMR3Init\n"));
+    int rc;
+
+    /*
+     * Assert sizes and alignments.
+     */
+    AssertRelease(sizeof(pVM->trpm.s) <= sizeof(pVM->trpm.padding));
+
+    /*
+     * Initialize members.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+        pVCpu->trpm.s.uActiveVector = ~0U;
+    }
+
+    /*
+     * Register the saved state data unit.
+     */
+    rc = SSMR3RegisterInternal(pVM, "trpm", 1, TRPM_SAVED_STATE_VERSION, sizeof(TRPM),
+                               NULL, NULL, NULL,
+                               NULL, trpmR3Save, NULL,
+                               NULL, trpmR3Load, NULL);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Register info handlers.
+     */
+    rc = DBGFR3InfoRegisterInternalEx(pVM, "trpmevent", "Dumps TRPM pending event.", trpmR3InfoEvent,
+                                      DBGFINFO_FLAGS_ALL_EMTS);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Statistics.
+     */
+#ifdef VBOX_WITH_STATISTICS
+    rc = MMHyperAlloc(pVM, sizeof(STAMCOUNTER) * 256, sizeof(STAMCOUNTER), MM_TAG_TRPM, (void **)&pVM->trpm.s.paStatForwardedIRQR3);
+    AssertRCReturn(rc, rc);
+    for (unsigned i = 0; i < 256; i++)
+        STAMR3RegisterF(pVM, &pVM->trpm.s.paStatForwardedIRQR3[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "Forwarded interrupts.",
+                        i < 0x20 ? "/TRPM/ForwardRaw/TRAP/%02X" : "/TRPM/ForwardRaw/IRQ/%02X", i);
+#endif
+
+    return 0;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this component.
+ *
+ * This function will be called at init and whenever the VMM need
+ * to relocate itself inside the GC.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3DECL(void) TRPMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    RT_NOREF(pVM, offDelta);
+}
+
+
+/**
+ * Terminates the Trap Manager
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(int) TRPMR3Term(PVM pVM)
+{
+    NOREF(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resets a virtual CPU.
+ *
+ * Used by TRPMR3Reset and CPU hot plugging.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMR3DECL(void) TRPMR3ResetCpu(PVMCPU pVCpu)
+{
+    pVCpu->trpm.s.uActiveVector = ~0U;
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * For the TRPM component this means that any IDT write monitors
+ * needs to be removed, any pending trap cleared, and the IDT reset.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3DECL(void) TRPMR3Reset(PVM pVM)
+{
+    /*
+     * Reinitialize other members calling the relocator to get things right.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        TRPMR3ResetCpu(pVM->apCpusR3[i]);
+    TRPMR3Relocate(pVM, 0);
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) trpmR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("trpmR3Save:\n"));
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PCTRPMCPU pTrpmCpu = &pVM->apCpusR3[i]->trpm.s;
+        SSMR3PutUInt(pSSM,       pTrpmCpu->uActiveVector);
+        SSMR3PutUInt(pSSM,       pTrpmCpu->enmActiveType);
+        SSMR3PutU32(pSSM,        pTrpmCpu->uActiveErrorCode);
+        SSMR3PutGCUIntPtr(pSSM,  pTrpmCpu->uActiveCR2);
+        SSMR3PutU8(pSSM,         pTrpmCpu->cbInstr);
+        SSMR3PutBool(pSSM,       pTrpmCpu->fIcebp);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) trpmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    LogFlow(("trpmR3Load:\n"));
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+
+    /*
+     * Validate version.
+     */
+    if (    uVersion != TRPM_SAVED_STATE_VERSION
+        &&  uVersion != TRPM_SAVED_STATE_VERSION_PRE_ICEBP
+        &&  uVersion != TRPM_SAVED_STATE_VERSION_UNI)
+    {
+        AssertMsgFailed(("trpmR3Load: Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    if (uVersion == TRPM_SAVED_STATE_VERSION)
+    {
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            PTRPMCPU pTrpmCpu = &pVM->apCpusR3[i]->trpm.s;
+            SSMR3GetU32(pSSM,      &pTrpmCpu->uActiveVector);
+            SSM_GET_ENUM32_RET(pSSM, pTrpmCpu->enmActiveType, TRPMEVENT);
+            SSMR3GetU32(pSSM,       &pTrpmCpu->uActiveErrorCode);
+            SSMR3GetGCUIntPtr(pSSM, &pTrpmCpu->uActiveCR2);
+            SSMR3GetU8(pSSM,        &pTrpmCpu->cbInstr);
+            SSMR3GetBool(pSSM,      &pTrpmCpu->fIcebp);
+        }
+    }
+    else
+    {
+        /*
+         * Active and saved traps.
+         */
+        if (uVersion == TRPM_SAVED_STATE_VERSION_PRE_ICEBP)
+        {
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+            {
+                RTGCUINT GCUIntErrCode;
+                PTRPMCPU pTrpmCpu = &pVM->apCpusR3[i]->trpm.s;
+                SSMR3GetU32(pSSM,      &pTrpmCpu->uActiveVector);
+                SSM_GET_ENUM32_RET(pSSM,  pTrpmCpu->enmActiveType, TRPMEVENT);
+                SSMR3GetGCUInt(pSSM,    &GCUIntErrCode);
+                SSMR3GetGCUIntPtr(pSSM, &pTrpmCpu->uActiveCR2);
+                SSMR3Skip(pSSM,          sizeof(RTGCUINT));      /* uSavedVector    - No longer used. */
+                SSMR3Skip(pSSM,          sizeof(RTUINT));        /* enmSavedType    - No longer used. */
+                SSMR3Skip(pSSM,          sizeof(RTGCUINT));      /* uSavedErrorCode - No longer used. */
+                SSMR3Skip(pSSM,          sizeof(RTGCUINTPTR));   /* uSavedCR2       - No longer used. */
+                SSMR3Skip(pSSM,          sizeof(RTGCUINT));      /* uPrevVector     - No longer used. */
+
+                /*
+                 * We lose the high 64-bits here (if RTGCUINT is 64-bit) after making the
+                 * active error code as 32-bits. However, for error codes even 16-bit should
+                 * be sufficient. Despite this, we decided to use and keep it at 32-bits
+                 * since VMX/SVM defines these as 32-bit in their event fields and converting
+                 * to/from these events are safer.
+                 */
+                pTrpmCpu->uActiveErrorCode = GCUIntErrCode;
+            }
+        }
+        else
+        {
+            RTGCUINT GCUIntErrCode;
+            PTRPMCPU pTrpmCpu = &pVM->apCpusR3[0]->trpm.s;
+            SSMR3GetU32(pSSM,      &pTrpmCpu->uActiveVector);
+            SSM_GET_ENUM32_RET(pSSM, pTrpmCpu->enmActiveType, TRPMEVENT);
+            SSMR3GetGCUInt(pSSM,    &GCUIntErrCode);
+            SSMR3GetGCUIntPtr(pSSM, &pTrpmCpu->uActiveCR2);
+            pTrpmCpu->uActiveErrorCode = GCUIntErrCode;
+        }
+
+        /*
+         * Skip rest of TRPM saved-state unit involving IDT and trampoline gates.
+         * With the removal of raw-mode support, we no longer need these.
+         */
+        SSMR3SkipToEndOfUnit(pSSM);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inject event (such as external irq or trap).
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   enmEvent    Trpm event type
+ * @param   pfInjected  Where to store whether the event was injected or not.
+ */
+VMMR3DECL(int) TRPMR3InjectEvent(PVM pVM, PVMCPU pVCpu, TRPMEVENT enmEvent, bool *pfInjected)
+{
+    PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
+    Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
+    Assert(pfInjected);
+    *pfInjected = false;
+
+    /* Currently only useful for external hardware interrupts. */
+    Assert(enmEvent == TRPM_HARDWARE_INT);
+
+    RT_NOREF3(pVM, enmEvent, pCtx);
+    uint8_t u8Interrupt = 0;
+    int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
+    Log(("TRPMR3InjectEvent: u8Interrupt=%d (%#x) rc=%Rrc\n", u8Interrupt, u8Interrupt, rc));
+    if (RT_SUCCESS(rc))
+    {
+        *pfInjected = true;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        if (   CPUMIsGuestInVmxNonRootMode(pCtx)
+            && CPUMIsGuestVmxInterceptEvents(pCtx)
+            && CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
+        {
+            VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
+            Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
+            return VBOXSTRICTRC_VAL(rcStrict);
+        }
+#endif
+        if (!VM_IS_NEM_ENABLED(pVM))
+        {
+            rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
+            AssertRC(rc);
+        }
+        else
+        {
+            VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8Interrupt, enmEvent, 0, 0, 0);
+            /** @todo NSTVMX: NSTSVM: We don't support nested VMX or nested SVM with NEM yet.
+             *        If so we should handle VINF_SVM_VMEXIT and VINF_VMX_VMEXIT codes here. */
+            if (rcStrict != VINF_SUCCESS)
+                return VBOXSTRICTRC_TODO(rcStrict);
+        }
+        STAM_COUNTER_INC(&pVM->trpm.s.paStatForwardedIRQR3[u8Interrupt]);
+    }
+    else
+    {
+        /* Can happen if the interrupt is masked by TPR or APIC is disabled. */
+        AssertMsg(rc == VERR_APIC_INTR_MASKED_BY_TPR || rc == VERR_NO_DATA, ("PDMGetInterrupt failed. rc=%Rrc\n", rc));
+    }
+    return HMR3IsActive(pVCpu)    ? VINF_EM_RESCHEDULE_HM
+         : VM_IS_NEM_ENABLED(pVM) ? VINF_EM_RESCHEDULE
+         :                          VINF_EM_RESCHEDULE_REM; /* (Heed the halted state if this is changed!) */
+}
+
+
+/**
+ * Displays the pending TRPM event.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) trpmR3InfoEvent(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = pVM->apCpusR3[0];
+
+    uint8_t     uVector;
+    uint8_t     cbInstr;
+    TRPMEVENT   enmTrapEvent;
+    uint32_t    uErrorCode;
+    RTGCUINTPTR uCR2;
+    bool        fIcebp;
+    int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrapEvent, &uErrorCode, &uCR2, &cbInstr, &fIcebp);
+    if (RT_SUCCESS(rc))
+    {
+        pHlp->pfnPrintf(pHlp, "CPU[%u]: TRPM event\n", pVCpu->idCpu);
+        static const char * const s_apszTrpmEventType[] =
+        {
+            "Trap",
+            "Hardware Int",
+            "Software Int"
+        };
+        if (RT_LIKELY((size_t)enmTrapEvent < RT_ELEMENTS(s_apszTrpmEventType)))
+        {
+            pHlp->pfnPrintf(pHlp, " Type       = %s\n", s_apszTrpmEventType[enmTrapEvent]);
+            pHlp->pfnPrintf(pHlp, " uVector    = %#x\n", uVector);
+            pHlp->pfnPrintf(pHlp, " uErrorCode = %#x\n", uErrorCode);
+            pHlp->pfnPrintf(pHlp, " uCR2       = %#RGp\n", uCR2);
+            pHlp->pfnPrintf(pHlp, " cbInstr    = %u bytes\n", cbInstr);
+            pHlp->pfnPrintf(pHlp, " fIcebp     = %RTbool\n", fIcebp);
+        }
+        else
+            pHlp->pfnPrintf(pHlp, " Type       = %#x (Invalid!)\n", enmTrapEvent);
+    }
+    else if (rc == VERR_TRPM_NO_ACTIVE_TRAP)
+        pHlp->pfnPrintf(pHlp, "CPU[%u]: TRPM event (None)\n", pVCpu->idCpu);
+    else
+        pHlp->pfnPrintf(pHlp, "CPU[%u]: TRPM event - Query failed! rc=%Rrc\n", pVCpu->idCpu, rc);
+}
+
diff --git a/src/VBox/VMM/VMMR3/VM.cpp b/src/VBox/VMM/VMMR3/VM.cpp
new file mode 100644
index 00000000..2ef84439
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/VM.cpp
@@ -0,0 +1,4428 @@
+/* $Id: VM.cpp $ */
+/** @file
+ * VM - Virtual Machine
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_vm     VM API
+ *
+ * This is the encapsulating bit.  It provides the APIs that Main and VBoxBFE
+ * use to create a VMM instance for running a guest in.  It also provides
+ * facilities for queuing request for execution in EMT (serialization purposes
+ * mostly) and for reporting error back to the VMM user (Main/VBoxBFE).
+ *
+ *
+ * @section sec_vm_design   Design Critique / Things To Do
+ *
+ * In hindsight this component is a big design mistake, all this stuff really
+ * belongs in the VMM component.  It just seemed like a kind of ok idea at a
+ * time when the VMM bit was a kind of vague.  'VM' also happened to be the name
+ * of the per-VM instance structure (see vm.h), so it kind of made sense.
+ * However as it turned out, VMM(.cpp) is almost empty all it provides in ring-3
+ * is some minor functionally and some "routing" services.
+ *
+ * Fixing this is just a matter of some more or less straight forward
+ * refactoring, the question is just when someone will get to it. Moving the EMT
+ * would be a good start.
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VM
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/gvmm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/gim.h>
+#include "VMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/sup.h>
+#if defined(VBOX_WITH_DTRACE_R3) && !defined(VBOX_WITH_NATIVE_DTRACE)
+# include <VBox/VBoxTpG.h>
+#endif
+#include <VBox/dbg.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/asm.h>
+#include <iprt/env.h>
+#include <iprt/string.h>
+#include <iprt/time.h>
+#include <iprt/semaphore.h>
+#include <iprt/thread.h>
+#include <iprt/uuid.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int                  vmR3CreateUVM(uint32_t cCpus, PCVMM2USERMETHODS pVmm2UserMethods, PUVM *ppUVM);
+static int                  vmR3CreateU(PUVM pUVM, uint32_t cCpus, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM);
+static int                  vmR3ReadBaseConfig(PVM pVM, PUVM pUVM, uint32_t cCpus);
+static int                  vmR3InitRing3(PVM pVM, PUVM pUVM);
+static int                  vmR3InitRing0(PVM pVM);
+static int                  vmR3InitDoCompleted(PVM pVM, VMINITCOMPLETED enmWhat);
+static void                 vmR3DestroyUVM(PUVM pUVM, uint32_t cMilliesEMTWait);
+static bool                 vmR3ValidateStateTransition(VMSTATE enmStateOld, VMSTATE enmStateNew);
+static void                 vmR3DoAtState(PVM pVM, PUVM pUVM, VMSTATE enmStateNew, VMSTATE enmStateOld);
+static int                  vmR3TrySetState(PVM pVM, const char *pszWho, unsigned cTransitions, ...);
+static void                 vmR3SetStateLocked(PVM pVM, PUVM pUVM, VMSTATE enmStateNew, VMSTATE enmStateOld, bool fSetRatherThanClearFF);
+static void                 vmR3SetState(PVM pVM, VMSTATE enmStateNew, VMSTATE enmStateOld);
+static int                  vmR3SetErrorU(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...) RT_IPRT_FORMAT_ATTR(6, 7);
+
+
+/**
+ * Creates a virtual machine by calling the supplied configuration constructor.
+ *
+ * On successful returned the VM is powered, i.e. VMR3PowerOn() should be
+ * called to start the execution.
+ *
+ * @returns 0 on success.
+ * @returns VBox error code on failure.
+ * @param   cCpus               Number of virtual CPUs for the new VM.
+ * @param   pVmm2UserMethods    An optional method table that the VMM can use
+ *                              to make the user perform various action, like
+ *                              for instance state saving.
+ * @param   pfnVMAtError        Pointer to callback function for setting VM
+ *                              errors.  This was added as an implicit call to
+ *                              VMR3AtErrorRegister() since there is no way the
+ *                              caller can get to the VM handle early enough to
+ *                              do this on its own.
+ *                              This is called in the context of an EMT.
+ * @param   pvUserVM            The user argument passed to pfnVMAtError.
+ * @param   pfnCFGMConstructor  Pointer to callback function for constructing the VM configuration tree.
+ *                              This is called in the context of an EMT0.
+ * @param   pvUserCFGM          The user argument passed to pfnCFGMConstructor.
+ * @param   ppVM                Where to optionally store the 'handle' of the
+ *                              created VM.
+ * @param   ppUVM               Where to optionally store the user 'handle' of
+ *                              the created VM, this includes one reference as
+ *                              if VMR3RetainUVM() was called.  The caller
+ *                              *MUST* remember to pass the returned value to
+ *                              VMR3ReleaseUVM() once done with the handle.
+ */
+VMMR3DECL(int)   VMR3Create(uint32_t cCpus, PCVMM2USERMETHODS pVmm2UserMethods,
+                            PFNVMATERROR pfnVMAtError, void *pvUserVM,
+                            PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM,
+                            PVM *ppVM, PUVM *ppUVM)
+{
+    LogFlow(("VMR3Create: cCpus=%RU32 pVmm2UserMethods=%p pfnVMAtError=%p pvUserVM=%p  pfnCFGMConstructor=%p pvUserCFGM=%p ppVM=%p ppUVM=%p\n",
+             cCpus, pVmm2UserMethods, pfnVMAtError, pvUserVM, pfnCFGMConstructor, pvUserCFGM, ppVM, ppUVM));
+
+    if (pVmm2UserMethods)
+    {
+        AssertPtrReturn(pVmm2UserMethods, VERR_INVALID_POINTER);
+        AssertReturn(pVmm2UserMethods->u32Magic    == VMM2USERMETHODS_MAGIC,   VERR_INVALID_PARAMETER);
+        AssertReturn(pVmm2UserMethods->u32Version  == VMM2USERMETHODS_VERSION, VERR_INVALID_PARAMETER);
+        AssertPtrNullReturn(pVmm2UserMethods->pfnSaveState, VERR_INVALID_POINTER);
+        AssertPtrNullReturn(pVmm2UserMethods->pfnNotifyEmtInit, VERR_INVALID_POINTER);
+        AssertPtrNullReturn(pVmm2UserMethods->pfnNotifyEmtTerm, VERR_INVALID_POINTER);
+        AssertPtrNullReturn(pVmm2UserMethods->pfnNotifyPdmtInit, VERR_INVALID_POINTER);
+        AssertPtrNullReturn(pVmm2UserMethods->pfnNotifyPdmtTerm, VERR_INVALID_POINTER);
+        AssertPtrNullReturn(pVmm2UserMethods->pfnNotifyResetTurnedIntoPowerOff, VERR_INVALID_POINTER);
+        AssertReturn(pVmm2UserMethods->u32EndMagic == VMM2USERMETHODS_MAGIC,   VERR_INVALID_PARAMETER);
+    }
+    AssertPtrNullReturn(pfnVMAtError, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnCFGMConstructor, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(ppVM, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(ppUVM, VERR_INVALID_POINTER);
+    AssertReturn(ppVM || ppUVM, VERR_INVALID_PARAMETER);
+
+    /*
+     * Validate input.
+     */
+    AssertLogRelMsgReturn(cCpus > 0 && cCpus <= VMM_MAX_CPU_COUNT, ("%RU32\n", cCpus), VERR_TOO_MANY_CPUS);
+
+    /*
+     * Create the UVM so we can register the at-error callback
+     * and consolidate a bit of cleanup code.
+     */
+    PUVM pUVM = NULL;                   /* shuts up gcc */
+    int rc = vmR3CreateUVM(cCpus, pVmm2UserMethods, &pUVM);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (pfnVMAtError)
+        rc = VMR3AtErrorRegister(pUVM, pfnVMAtError, pvUserVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Initialize the support library creating the session for this VM.
+         */
+        rc = SUPR3Init(&pUVM->vm.s.pSession);
+        if (RT_SUCCESS(rc))
+        {
+#if defined(VBOX_WITH_DTRACE_R3) && !defined(VBOX_WITH_NATIVE_DTRACE)
+            /* Now that we've opened the device, we can register trace probes. */
+            static bool s_fRegisteredProbes = false;
+            if (ASMAtomicCmpXchgBool(&s_fRegisteredProbes, true, false))
+                SUPR3TracerRegisterModule(~(uintptr_t)0, "VBoxVMM", &g_VTGObjHeader, (uintptr_t)&g_VTGObjHeader,
+                                          SUP_TRACER_UMOD_FLAGS_SHARED);
+#endif
+
+            /*
+             * Call vmR3CreateU in the EMT thread and wait for it to finish.
+             *
+             * Note! VMCPUID_ANY is used here because VMR3ReqQueueU would have trouble
+             *       submitting a request to a specific VCPU without a pVM. So, to make
+             *       sure init is running on EMT(0), vmR3EmulationThreadWithId makes sure
+             *       that only EMT(0) is servicing VMCPUID_ANY requests when pVM is NULL.
+             */
+            PVMREQ pReq;
+            rc = VMR3ReqCallU(pUVM, VMCPUID_ANY, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VBOX_STATUS,
+                              (PFNRT)vmR3CreateU, 4, pUVM, cCpus, pfnCFGMConstructor, pvUserCFGM);
+            if (RT_SUCCESS(rc))
+            {
+                rc = pReq->iStatus;
+                VMR3ReqFree(pReq);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Success!
+                     */
+                    if (ppVM)
+                        *ppVM = pUVM->pVM;
+                    if (ppUVM)
+                    {
+                        VMR3RetainUVM(pUVM);
+                        *ppUVM = pUVM;
+                    }
+                    LogFlow(("VMR3Create: returns VINF_SUCCESS (pVM=%p, pUVM=%p\n", pUVM->pVM, pUVM));
+                    return VINF_SUCCESS;
+                }
+            }
+            else
+                AssertMsgFailed(("VMR3ReqCallU failed rc=%Rrc\n", rc));
+
+            /*
+             * An error occurred during VM creation.  Set the error message directly
+             * using the initial callback, as the callback list might not exist yet.
+             */
+            const char *pszError;
+            switch (rc)
+            {
+                case VERR_VMX_IN_VMX_ROOT_MODE:
+#ifdef RT_OS_LINUX
+                    pszError = N_("VirtualBox can't operate in VMX root mode. "
+                                  "Please disable the KVM kernel extension, recompile your kernel and reboot");
+#else
+                    pszError = N_("VirtualBox can't operate in VMX root mode. Please close all other virtualization programs.");
+#endif
+                    break;
+
+#ifndef RT_OS_DARWIN
+                case VERR_HM_CONFIG_MISMATCH:
+                    pszError = N_("VT-x/AMD-V is either not available on your host or disabled. "
+                                  "This hardware extension is required by the VM configuration");
+                    break;
+#endif
+
+                case VERR_SVM_IN_USE:
+#ifdef RT_OS_LINUX
+                    pszError = N_("VirtualBox can't enable the AMD-V extension. "
+                                  "Please disable the KVM kernel extension, recompile your kernel and reboot");
+#else
+                    pszError = N_("VirtualBox can't enable the AMD-V extension. Please close all other virtualization programs.");
+#endif
+                    break;
+
+#ifdef RT_OS_LINUX
+                case VERR_SUPDRV_COMPONENT_NOT_FOUND:
+                    pszError = N_("One of the kernel modules was not successfully loaded. Make sure "
+                                  "that VirtualBox is correctly installed, and if you are using EFI "
+                                  "Secure Boot that the modules are signed if necessary in the right "
+                                  "way for your host system.  Then try to recompile and reload the "
+                                  "kernel modules by executing "
+                                  "'/sbin/vboxconfig' as root");
+                    break;
+#endif
+
+                case VERR_RAW_MODE_INVALID_SMP:
+                    pszError = N_("VT-x/AMD-V is either not available on your host or disabled. "
+                                  "VirtualBox requires this hardware extension to emulate more than one "
+                                  "guest CPU");
+                    break;
+
+                case VERR_SUPDRV_KERNEL_TOO_OLD_FOR_VTX:
+#ifdef RT_OS_LINUX
+                    pszError = N_("Because the host kernel is too old, VirtualBox cannot enable the VT-x "
+                                  "extension. Either upgrade your kernel to Linux 2.6.13 or later or disable "
+                                  "the VT-x extension in the VM settings. Note that without VT-x you have "
+                                  "to reduce the number of guest CPUs to one");
+#else
+                    pszError = N_("Because the host kernel is too old, VirtualBox cannot enable the VT-x "
+                                  "extension. Either upgrade your kernel or disable the VT-x extension in the "
+                                  "VM settings. Note that without VT-x you have to reduce the number of guest "
+                                  "CPUs to one");
+#endif
+                    break;
+
+                case VERR_PDM_DEVICE_NOT_FOUND:
+                    pszError = N_("A virtual device is configured in the VM settings but the device "
+                                  "implementation is missing.\n"
+                                  "A possible reason for this error is a missing extension pack. Note "
+                                  "that as of VirtualBox 4.0, certain features (for example USB 2.0 "
+                                  "support and remote desktop) are only available from an 'extension "
+                                  "pack' which must be downloaded and installed separately");
+                    break;
+
+                case VERR_PCI_PASSTHROUGH_NO_HM:
+                    pszError = N_("PCI passthrough requires VT-x/AMD-V");
+                    break;
+
+                case VERR_PCI_PASSTHROUGH_NO_NESTED_PAGING:
+                    pszError = N_("PCI passthrough requires nested paging");
+                    break;
+
+                default:
+                    if (VMR3GetErrorCount(pUVM) == 0)
+                        pszError = RTErrGetFull(rc);
+                    else
+                        pszError = NULL; /* already set. */
+                    break;
+            }
+            if (pszError)
+                vmR3SetErrorU(pUVM, rc, RT_SRC_POS, pszError, rc);
+        }
+        else
+        {
+            /*
+             * An error occurred at support library initialization time (before the
+             * VM could be created). Set the error message directly using the
+             * initial callback, as the callback list doesn't exist yet.
+             */
+            const char *pszError;
+            switch (rc)
+            {
+                case VERR_VM_DRIVER_LOAD_ERROR:
+#ifdef RT_OS_LINUX
+                    pszError = N_("VirtualBox kernel driver not loaded. The vboxdrv kernel module "
+                                  "was either not loaded, /dev/vboxdrv is not set up properly, "
+                                  "or you are using EFI Secure Boot and the module is not signed "
+                                  "in the right way for your system.  If necessary, try setting up "
+                                  "the kernel module again by executing "
+                                  "'/sbin/vboxconfig' as root");
+#else
+                    pszError = N_("VirtualBox kernel driver not loaded");
+#endif
+                    break;
+                case VERR_VM_DRIVER_OPEN_ERROR:
+                    pszError = N_("VirtualBox kernel driver cannot be opened");
+                    break;
+                case VERR_VM_DRIVER_NOT_ACCESSIBLE:
+#ifdef VBOX_WITH_HARDENING
+                    /* This should only happen if the executable wasn't hardened - bad code/build. */
+                    pszError = N_("VirtualBox kernel driver not accessible, permission problem. "
+                                  "Re-install VirtualBox. If you are building it yourself, you "
+                                  "should make sure it installed correctly and that the setuid "
+                                  "bit is set on the executables calling VMR3Create.");
+#else
+                    /* This should only happen when mixing builds or with the usual /dev/vboxdrv access issues. */
+# if defined(RT_OS_DARWIN)
+                    pszError = N_("VirtualBox KEXT is not accessible, permission problem. "
+                                  "If you have built VirtualBox yourself, make sure that you do not "
+                                  "have the vboxdrv KEXT from a different build or installation loaded.");
+# elif defined(RT_OS_LINUX)
+                    pszError = N_("VirtualBox kernel driver is not accessible, permission problem. "
+                                  "If you have built VirtualBox yourself, make sure that you do "
+                                  "not have the vboxdrv kernel module from a different build or "
+                                  "installation loaded. Also, make sure the vboxdrv udev rule gives "
+                                  "you the permission you need to access the device.");
+# elif defined(RT_OS_WINDOWS)
+                    pszError = N_("VirtualBox kernel driver is not accessible, permission problem.");
+# else /* solaris, freebsd, ++. */
+                    pszError = N_("VirtualBox kernel module is not accessible, permission problem. "
+                                  "If you have built VirtualBox yourself, make sure that you do "
+                                  "not have the vboxdrv kernel module from a different install loaded.");
+# endif
+#endif
+                    break;
+                case VERR_INVALID_HANDLE: /** @todo track down and fix this error. */
+                case VERR_VM_DRIVER_NOT_INSTALLED:
+#ifdef RT_OS_LINUX
+                    pszError = N_("VirtualBox kernel driver not Installed. The vboxdrv kernel module "
+                                  "was either not loaded, /dev/vboxdrv is not set up properly, "
+                                  "or you are using EFI Secure Boot and the module is not signed "
+                                  "in the right way for your system.  If necessary, try setting up "
+                                  "the kernel module again by executing "
+                                  "'/sbin/vboxconfig' as root");
+#else
+                    pszError = N_("VirtualBox kernel driver not installed");
+#endif
+                    break;
+                case VERR_NO_MEMORY:
+                    pszError = N_("VirtualBox support library out of memory");
+                    break;
+                case VERR_VERSION_MISMATCH:
+                case VERR_VM_DRIVER_VERSION_MISMATCH:
+                    pszError = N_("The VirtualBox support driver which is running is from a different "
+                                  "version of VirtualBox.  You can correct this by stopping all "
+                                  "running instances of VirtualBox and reinstalling the software.");
+                    break;
+                default:
+                    pszError = N_("Unknown error initializing kernel driver");
+                    AssertMsgFailed(("Add error message for rc=%d (%Rrc)\n", rc, rc));
+            }
+            vmR3SetErrorU(pUVM, rc, RT_SRC_POS, pszError, rc);
+        }
+    }
+
+    /* cleanup */
+    vmR3DestroyUVM(pUVM, 2000);
+    LogFlow(("VMR3Create: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Creates the UVM.
+ *
+ * This will not initialize the support library even if vmR3DestroyUVM
+ * will terminate that.
+ *
+ * @returns VBox status code.
+ * @param   cCpus               Number of virtual CPUs
+ * @param   pVmm2UserMethods    Pointer to the optional VMM -> User method
+ *                              table.
+ * @param   ppUVM               Where to store the UVM pointer.
+ */
+static int vmR3CreateUVM(uint32_t cCpus, PCVMM2USERMETHODS pVmm2UserMethods, PUVM *ppUVM)
+{
+    uint32_t i;
+
+    /*
+     * Create and initialize the UVM.
+     */
+    PUVM pUVM = (PUVM)RTMemPageAllocZ(RT_UOFFSETOF_DYN(UVM, aCpus[cCpus]));
+    AssertReturn(pUVM, VERR_NO_MEMORY);
+    pUVM->u32Magic          = UVM_MAGIC;
+    pUVM->cCpus             = cCpus;
+    pUVM->pVmm2UserMethods  = pVmm2UserMethods;
+
+    AssertCompile(sizeof(pUVM->vm.s) <= sizeof(pUVM->vm.padding));
+
+    pUVM->vm.s.cUvmRefs      = 1;
+    pUVM->vm.s.ppAtStateNext = &pUVM->vm.s.pAtState;
+    pUVM->vm.s.ppAtErrorNext = &pUVM->vm.s.pAtError;
+    pUVM->vm.s.ppAtRuntimeErrorNext = &pUVM->vm.s.pAtRuntimeError;
+
+    pUVM->vm.s.enmHaltMethod = VMHALTMETHOD_BOOTSTRAP;
+    RTUuidClear(&pUVM->vm.s.Uuid);
+
+    /* Initialize the VMCPU array in the UVM. */
+    for (i = 0; i < cCpus; i++)
+    {
+        pUVM->aCpus[i].pUVM   = pUVM;
+        pUVM->aCpus[i].idCpu  = i;
+    }
+
+    /* Allocate a TLS entry to store the VMINTUSERPERVMCPU pointer. */
+    int rc = RTTlsAllocEx(&pUVM->vm.s.idxTLS, NULL);
+    AssertRC(rc);
+    if (RT_SUCCESS(rc))
+    {
+        /* Allocate a halt method event semaphore for each VCPU. */
+        for (i = 0; i < cCpus; i++)
+            pUVM->aCpus[i].vm.s.EventSemWait = NIL_RTSEMEVENT;
+        for (i = 0; i < cCpus; i++)
+        {
+            rc = RTSemEventCreate(&pUVM->aCpus[i].vm.s.EventSemWait);
+            if (RT_FAILURE(rc))
+                break;
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTCritSectInit(&pUVM->vm.s.AtStateCritSect);
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTCritSectInit(&pUVM->vm.s.AtErrorCritSect);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Init fundamental (sub-)components - STAM, MMR3Heap and PDMLdr.
+                     */
+                    rc = PDMR3InitUVM(pUVM);
+                    if (RT_SUCCESS(rc))
+                    {
+                        rc = STAMR3InitUVM(pUVM);
+                        if (RT_SUCCESS(rc))
+                        {
+                            rc = MMR3InitUVM(pUVM);
+                            if (RT_SUCCESS(rc))
+                            {
+                                /*
+                                 * Start the emulation threads for all VMCPUs.
+                                 */
+                                for (i = 0; i < cCpus; i++)
+                                {
+                                    rc = RTThreadCreateF(&pUVM->aCpus[i].vm.s.ThreadEMT, vmR3EmulationThread, &pUVM->aCpus[i],
+                                                         _1M, RTTHREADTYPE_EMULATION, RTTHREADFLAGS_WAITABLE,
+                                                         cCpus > 1 ? "EMT-%u" : "EMT", i);
+                                    if (RT_FAILURE(rc))
+                                        break;
+
+                                    pUVM->aCpus[i].vm.s.NativeThreadEMT = RTThreadGetNative(pUVM->aCpus[i].vm.s.ThreadEMT);
+                                }
+
+                                if (RT_SUCCESS(rc))
+                                {
+                                    *ppUVM = pUVM;
+                                    return VINF_SUCCESS;
+                                }
+
+                                /* bail out. */
+                                while (i-- > 0)
+                                {
+                                    /** @todo rainy day: terminate the EMTs. */
+                                }
+                                MMR3TermUVM(pUVM);
+                            }
+                            STAMR3TermUVM(pUVM);
+                        }
+                        PDMR3TermUVM(pUVM);
+                    }
+                    RTCritSectDelete(&pUVM->vm.s.AtErrorCritSect);
+                }
+                RTCritSectDelete(&pUVM->vm.s.AtStateCritSect);
+            }
+        }
+        for (i = 0; i < cCpus; i++)
+        {
+            RTSemEventDestroy(pUVM->aCpus[i].vm.s.EventSemWait);
+            pUVM->aCpus[i].vm.s.EventSemWait = NIL_RTSEMEVENT;
+        }
+        RTTlsFree(pUVM->vm.s.idxTLS);
+    }
+    RTMemPageFree(pUVM, RT_UOFFSETOF_DYN(UVM, aCpus[pUVM->cCpus]));
+    return rc;
+}
+
+
+/**
+ * Creates and initializes the VM.
+ *
+ * @thread EMT
+ */
+static int vmR3CreateU(PUVM pUVM, uint32_t cCpus, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM)
+{
+#if (defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)) && !defined(VBOX_WITH_OLD_CPU_SUPPORT)
+    /*
+     * Require SSE2 to be present (already checked for in supdrv, so we
+     * shouldn't ever really get here).
+     */
+    if (!(ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_SSE2))
+    {
+        LogRel(("vboxdrv: Requires SSE2 (cpuid(0).EDX=%#x)\n", ASMCpuId_EDX(1)));
+        return VERR_UNSUPPORTED_CPU;
+    }
+#endif
+
+    /*
+     * Load the VMMR0.r0 module so that we can call GVMMR0CreateVM.
+     */
+    int rc = PDMR3LdrLoadVMMR0U(pUVM);
+    if (RT_FAILURE(rc))
+    {
+        /** @todo we need a cleaner solution for this (VERR_VMX_IN_VMX_ROOT_MODE).
+          * bird: what about moving the message down here? Main picks the first message, right? */
+        if (rc == VERR_VMX_IN_VMX_ROOT_MODE)
+            return rc;  /* proper error message set later on */
+        return vmR3SetErrorU(pUVM, rc, RT_SRC_POS, N_("Failed to load VMMR0.r0"));
+    }
+
+    /*
+     * Request GVMM to create a new VM for us.
+     */
+    GVMMCREATEVMREQ CreateVMReq;
+    CreateVMReq.Hdr.u32Magic    = SUPVMMR0REQHDR_MAGIC;
+    CreateVMReq.Hdr.cbReq       = sizeof(CreateVMReq);
+    CreateVMReq.pSession        = pUVM->vm.s.pSession;
+    CreateVMReq.pVMR0           = NIL_RTR0PTR;
+    CreateVMReq.pVMR3           = NULL;
+    CreateVMReq.cCpus           = cCpus;
+    rc = SUPR3CallVMMR0Ex(NIL_RTR0PTR, NIL_VMCPUID, VMMR0_DO_GVMM_CREATE_VM, 0, &CreateVMReq.Hdr);
+    if (RT_SUCCESS(rc))
+    {
+        PVM pVM = pUVM->pVM = CreateVMReq.pVMR3;
+        AssertRelease(VALID_PTR(pVM));
+        AssertRelease(pVM->pVMR0ForCall == CreateVMReq.pVMR0);
+        AssertRelease(pVM->pSession == pUVM->vm.s.pSession);
+        AssertRelease(pVM->cCpus == cCpus);
+        AssertRelease(pVM->uCpuExecutionCap == 100);
+        AssertCompileMemberAlignment(VM, cpum, 64);
+        AssertCompileMemberAlignment(VM, tm, 64);
+
+        Log(("VMR3Create: Created pUVM=%p pVM=%p pVMR0=%p hSelf=%#x cCpus=%RU32\n",
+             pUVM, pVM, CreateVMReq.pVMR0, pVM->hSelf, pVM->cCpus));
+
+        /*
+         * Initialize the VM structure and our internal data (VMINT).
+         */
+        pVM->pUVM = pUVM;
+
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            PVMCPU pVCpu = pVM->apCpusR3[i];
+            pVCpu->pUVCpu            = &pUVM->aCpus[i];
+            pVCpu->idCpu             = i;
+            pVCpu->hNativeThread     = pUVM->aCpus[i].vm.s.NativeThreadEMT;
+            Assert(pVCpu->hNativeThread != NIL_RTNATIVETHREAD);
+            /* hNativeThreadR0 is initialized on EMT registration. */
+            pUVM->aCpus[i].pVCpu     = pVCpu;
+            pUVM->aCpus[i].pVM       = pVM;
+        }
+
+
+        /*
+         * Init the configuration.
+         */
+        rc = CFGMR3Init(pVM, pfnCFGMConstructor, pvUserCFGM);
+        if (RT_SUCCESS(rc))
+        {
+            rc = vmR3ReadBaseConfig(pVM, pUVM, cCpus);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Init the ring-3 components and ring-3 per cpu data, finishing it off
+                 * by a relocation round (intermediate context finalization will do this).
+                 */
+                rc = vmR3InitRing3(pVM, pUVM);
+                if (RT_SUCCESS(rc))
+                {
+#ifndef PGM_WITHOUT_MAPPINGS
+                    rc = PGMR3FinalizeMappings(pVM);
+                    if (RT_SUCCESS(rc))
+#endif
+                    {
+
+                        LogFlow(("Ring-3 init succeeded\n"));
+
+                        /*
+                         * Init the Ring-0 components.
+                         */
+                        rc = vmR3InitRing0(pVM);
+                        if (RT_SUCCESS(rc))
+                        {
+                            /* Relocate again, because some switcher fixups depends on R0 init results. */
+                            VMR3Relocate(pVM, 0 /* offDelta */);
+
+#ifdef VBOX_WITH_DEBUGGER
+                            /*
+                             * Init the tcp debugger console if we're building
+                             * with debugger support.
+                             */
+                            void *pvUser = NULL;
+                            rc = DBGCTcpCreate(pUVM, &pvUser);
+                            if (    RT_SUCCESS(rc)
+                                ||  rc == VERR_NET_ADDRESS_IN_USE)
+                            {
+                                pUVM->vm.s.pvDBGC = pvUser;
+#endif
+                                /*
+                                 * Now we can safely set the VM halt method to default.
+                                 */
+                                rc = vmR3SetHaltMethodU(pUVM, VMHALTMETHOD_DEFAULT);
+                                if (RT_SUCCESS(rc))
+                                {
+                                    /*
+                                     * Set the state and we're done.
+                                     */
+                                    vmR3SetState(pVM, VMSTATE_CREATED, VMSTATE_CREATING);
+                                    return VINF_SUCCESS;
+                                }
+#ifdef VBOX_WITH_DEBUGGER
+                                DBGCTcpTerminate(pUVM, pUVM->vm.s.pvDBGC);
+                                pUVM->vm.s.pvDBGC = NULL;
+                            }
+#endif
+                            //..
+                        }
+                    }
+                    vmR3Destroy(pVM);
+                }
+            }
+            //..
+
+            /* Clean CFGM. */
+            int rc2 = CFGMR3Term(pVM);
+            AssertRC(rc2);
+        }
+
+        /*
+         * Do automatic cleanups while the VM structure is still alive and all
+         * references to it are still working.
+         */
+        PDMR3CritSectBothTerm(pVM);
+
+        /*
+         * Drop all references to VM and the VMCPU structures, then
+         * tell GVMM to destroy the VM.
+         */
+        pUVM->pVM = NULL;
+        for (VMCPUID i = 0; i < pUVM->cCpus; i++)
+        {
+            pUVM->aCpus[i].pVM = NULL;
+            pUVM->aCpus[i].pVCpu = NULL;
+        }
+        Assert(pUVM->vm.s.enmHaltMethod == VMHALTMETHOD_BOOTSTRAP);
+
+        if (pUVM->cCpus > 1)
+        {
+            /* Poke the other EMTs since they may have stale pVM and pVCpu references
+               on the stack (see VMR3WaitU for instance) if they've been awakened after
+               VM creation. */
+            for (VMCPUID i = 1; i < pUVM->cCpus; i++)
+                VMR3NotifyCpuFFU(&pUVM->aCpus[i], 0);
+            RTThreadSleep(RT_MIN(100 + 25 *(pUVM->cCpus - 1), 500)); /* very sophisticated */
+        }
+
+        int rc2 = SUPR3CallVMMR0Ex(CreateVMReq.pVMR0, 0 /*idCpu*/, VMMR0_DO_GVMM_DESTROY_VM, 0, NULL);
+        AssertRC(rc2);
+    }
+    else
+        vmR3SetErrorU(pUVM, rc, RT_SRC_POS, N_("VM creation failed (GVMM)"));
+
+    LogFlow(("vmR3CreateU: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Reads the base configuation from CFGM.
+ *
+ * @returns VBox status code.
+ * @param   pVM                The cross context VM structure.
+ * @param   pUVM                The user mode VM structure.
+ * @param   cCpus               The CPU count given to VMR3Create.
+ */
+static int vmR3ReadBaseConfig(PVM pVM, PUVM pUVM, uint32_t cCpus)
+{
+    int         rc;
+    PCFGMNODE   pRoot = CFGMR3GetRoot(pVM);
+
+    /*
+     * Base EM and HM config properties.
+     */
+    pVM->fHMEnabled = true;
+
+    /*
+     * Make sure the CPU count in the config data matches.
+     */
+    uint32_t cCPUsCfg;
+    rc = CFGMR3QueryU32Def(pRoot, "NumCPUs", &cCPUsCfg, 1);
+    AssertLogRelMsgRCReturn(rc, ("Configuration error: Querying \"NumCPUs\" as integer failed, rc=%Rrc\n", rc), rc);
+    AssertLogRelMsgReturn(cCPUsCfg == cCpus,
+                          ("Configuration error: \"NumCPUs\"=%RU32 and VMR3Create::cCpus=%RU32 does not match!\n",
+                           cCPUsCfg, cCpus),
+                          VERR_INVALID_PARAMETER);
+
+    /*
+     * Get the CPU execution cap.
+     */
+    rc = CFGMR3QueryU32Def(pRoot, "CpuExecutionCap", &pVM->uCpuExecutionCap, 100);
+    AssertLogRelMsgRCReturn(rc, ("Configuration error: Querying \"CpuExecutionCap\" as integer failed, rc=%Rrc\n", rc), rc);
+
+    /*
+     * Get the VM name and UUID.
+     */
+    rc = CFGMR3QueryStringAllocDef(pRoot, "Name", &pUVM->vm.s.pszName, "<unknown>");
+    AssertLogRelMsgRCReturn(rc, ("Configuration error: Querying \"Name\" failed, rc=%Rrc\n", rc), rc);
+
+    rc = CFGMR3QueryBytes(pRoot, "UUID", &pUVM->vm.s.Uuid, sizeof(pUVM->vm.s.Uuid));
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        rc = VINF_SUCCESS;
+    AssertLogRelMsgRCReturn(rc, ("Configuration error: Querying \"UUID\" failed, rc=%Rrc\n", rc), rc);
+
+    rc = CFGMR3QueryBoolDef(pRoot, "PowerOffInsteadOfReset", &pVM->vm.s.fPowerOffInsteadOfReset, false);
+    AssertLogRelMsgRCReturn(rc, ("Configuration error: Querying \"PowerOffInsteadOfReset\" failed, rc=%Rrc\n", rc), rc);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Register the calling EMT with GVM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   idCpu       The Virtual CPU ID.
+ */
+static DECLCALLBACK(int) vmR3RegisterEMT(PVM pVM, VMCPUID idCpu)
+{
+    Assert(VMMGetCpuId(pVM) == idCpu);
+    int rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), idCpu, VMMR0_DO_GVMM_REGISTER_VMCPU, 0, NULL);
+    if (RT_FAILURE(rc))
+        LogRel(("idCpu=%u rc=%Rrc\n", idCpu, rc));
+    return rc;
+}
+
+
+/**
+ * Initializes all R3 components of the VM
+ */
+static int vmR3InitRing3(PVM pVM, PUVM pUVM)
+{
+    int rc;
+
+    /*
+     * Register the other EMTs with GVM.
+     */
+    for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
+    {
+        rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)vmR3RegisterEMT, 2, pVM, idCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Register statistics.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltYield,           STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Profiling halted state yielding.",  "/PROF/CPU%d/VM/Halt/Yield", idCpu);
+        AssertRC(rc);
+        rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltBlock,           STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Profiling halted state blocking.",  "/PROF/CPU%d/VM/Halt/Block", idCpu);
+        AssertRC(rc);
+        rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltBlockOverslept,  STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Time wasted by blocking too long.", "/PROF/CPU%d/VM/Halt/BlockOverslept", idCpu);
+        AssertRC(rc);
+        rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltBlockInsomnia,   STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Time slept when returning to early.","/PROF/CPU%d/VM/Halt/BlockInsomnia", idCpu);
+        AssertRC(rc);
+        rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltBlockOnTime,     STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Time slept on time.",                "/PROF/CPU%d/VM/Halt/BlockOnTime", idCpu);
+        AssertRC(rc);
+        rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltTimers,          STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Profiling halted state timer tasks.", "/PROF/CPU%d/VM/Halt/Timers", idCpu);
+        AssertRC(rc);
+    }
+
+    STAM_REG(pVM, &pUVM->vm.s.StatReqAllocNew,   STAMTYPE_COUNTER,     "/VM/Req/AllocNew",       STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc returning a new packet.");
+    STAM_REG(pVM, &pUVM->vm.s.StatReqAllocRaces, STAMTYPE_COUNTER,     "/VM/Req/AllocRaces",     STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc causing races.");
+    STAM_REG(pVM, &pUVM->vm.s.StatReqAllocRecycled, STAMTYPE_COUNTER,  "/VM/Req/AllocRecycled",  STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc returning a recycled packet.");
+    STAM_REG(pVM, &pUVM->vm.s.StatReqFree,       STAMTYPE_COUNTER,     "/VM/Req/Free",           STAMUNIT_OCCURENCES,        "Number of VMR3ReqFree calls.");
+    STAM_REG(pVM, &pUVM->vm.s.StatReqFreeOverflow, STAMTYPE_COUNTER,   "/VM/Req/FreeOverflow",   STAMUNIT_OCCURENCES,        "Number of times the request was actually freed.");
+    STAM_REG(pVM, &pUVM->vm.s.StatReqProcessed,  STAMTYPE_COUNTER,     "/VM/Req/Processed",      STAMUNIT_OCCURENCES,        "Number of processed requests (any queue).");
+    STAM_REG(pVM, &pUVM->vm.s.StatReqMoreThan1,  STAMTYPE_COUNTER,     "/VM/Req/MoreThan1",      STAMUNIT_OCCURENCES,        "Number of times there are more than one request on the queue when processing it.");
+    STAM_REG(pVM, &pUVM->vm.s.StatReqPushBackRaces, STAMTYPE_COUNTER,  "/VM/Req/PushBackRaces",  STAMUNIT_OCCURENCES,        "Number of push back races.");
+
+    /* Statistics for ring-0 components: */
+    STAM_REL_REG(pVM, &pVM->R0Stats.gmm.cChunkTlbHits,   STAMTYPE_COUNTER, "/GMM/ChunkTlbHits",   STAMUNIT_OCCURENCES, "GMMR0PageIdToVirt chunk TBL hits");
+    STAM_REL_REG(pVM, &pVM->R0Stats.gmm.cChunkTlbMisses, STAMTYPE_COUNTER, "/GMM/ChunkTlbMisses", STAMUNIT_OCCURENCES, "GMMR0PageIdToVirt chunk TBL misses");
+
+    /*
+     * Init all R3 components, the order here might be important.
+     * NEM and HM shall be initialized first!
+     */
+    Assert(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NOT_SET);
+    rc = NEMR3InitConfig(pVM);
+    if (RT_SUCCESS(rc))
+        rc = HMR3Init(pVM);
+    if (RT_SUCCESS(rc))
+    {
+        ASMCompilerBarrier(); /* HMR3Init will have modified bMainExecutionEngine */
+        Assert(   pVM->bMainExecutionEngine == VM_EXEC_ENGINE_HW_VIRT
+               || pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API);
+        rc = MMR3Init(pVM);
+        if (RT_SUCCESS(rc))
+        {
+            rc = CPUMR3Init(pVM);
+            if (RT_SUCCESS(rc))
+            {
+                rc = NEMR3InitAfterCPUM(pVM);
+                if (RT_SUCCESS(rc))
+                    rc = PGMR3Init(pVM);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = MMR3InitPaging(pVM);
+                    if (RT_SUCCESS(rc))
+                        rc = TMR3Init(pVM);
+                    if (RT_SUCCESS(rc))
+                    {
+                        rc = VMMR3Init(pVM);
+                        if (RT_SUCCESS(rc))
+                        {
+                            rc = SELMR3Init(pVM);
+                            if (RT_SUCCESS(rc))
+                            {
+                                rc = TRPMR3Init(pVM);
+                                if (RT_SUCCESS(rc))
+                                {
+                                    rc = SSMR3RegisterStub(pVM, "CSAM", 0);
+                                    if (RT_SUCCESS(rc))
+                                    {
+                                        rc = SSMR3RegisterStub(pVM, "PATM", 0);
+                                        if (RT_SUCCESS(rc))
+                                        {
+                                            rc = IOMR3Init(pVM);
+                                            if (RT_SUCCESS(rc))
+                                            {
+                                                rc = EMR3Init(pVM);
+                                                if (RT_SUCCESS(rc))
+                                                {
+                                                    rc = IEMR3Init(pVM);
+                                                    if (RT_SUCCESS(rc))
+                                                    {
+                                                        rc = DBGFR3Init(pVM);
+                                                        if (RT_SUCCESS(rc))
+                                                        {
+                                                            /* GIM must be init'd before PDM, gimdevR3Construct()
+                                                               requires GIM provider to be setup. */
+                                                            rc = GIMR3Init(pVM);
+                                                            if (RT_SUCCESS(rc))
+                                                            {
+                                                                rc = PDMR3Init(pVM);
+                                                                if (RT_SUCCESS(rc))
+                                                                {
+                                                                    rc = PGMR3InitDynMap(pVM);
+                                                                    if (RT_SUCCESS(rc))
+                                                                        rc = MMR3HyperInitFinalize(pVM);
+                                                                    if (RT_SUCCESS(rc))
+                                                                        rc = PGMR3InitFinalize(pVM);
+                                                                    if (RT_SUCCESS(rc))
+                                                                        rc = TMR3InitFinalize(pVM);
+                                                                    if (RT_SUCCESS(rc))
+                                                                    {
+                                                                        PGMR3MemSetup(pVM, false /*fAtReset*/);
+                                                                        PDMR3MemSetup(pVM, false /*fAtReset*/);
+                                                                    }
+                                                                    if (RT_SUCCESS(rc))
+                                                                        rc = vmR3InitDoCompleted(pVM, VMINITCOMPLETED_RING3);
+                                                                    if (RT_SUCCESS(rc))
+                                                                    {
+                                                                        LogFlow(("vmR3InitRing3: returns %Rrc\n", VINF_SUCCESS));
+                                                                        return VINF_SUCCESS;
+                                                                    }
+
+                                                                    int rc2 = PDMR3Term(pVM);
+                                                                    AssertRC(rc2);
+                                                                }
+                                                                int rc2 = GIMR3Term(pVM);
+                                                                AssertRC(rc2);
+                                                            }
+                                                            int rc2 = DBGFR3Term(pVM);
+                                                            AssertRC(rc2);
+                                                        }
+                                                        int rc2 = IEMR3Term(pVM);
+                                                        AssertRC(rc2);
+                                                    }
+                                                    int rc2 = EMR3Term(pVM);
+                                                    AssertRC(rc2);
+                                                }
+                                                int rc2 = IOMR3Term(pVM);
+                                                AssertRC(rc2);
+                                            }
+                                        }
+                                    }
+                                    int rc2 = TRPMR3Term(pVM);
+                                    AssertRC(rc2);
+                                }
+                                int rc2 = SELMR3Term(pVM);
+                                AssertRC(rc2);
+                            }
+                            int rc2 = VMMR3Term(pVM);
+                            AssertRC(rc2);
+                        }
+                        int rc2 = TMR3Term(pVM);
+                        AssertRC(rc2);
+                    }
+                    int rc2 = PGMR3Term(pVM);
+                    AssertRC(rc2);
+                }
+                //int rc2 = CPUMR3Term(pVM);
+                //AssertRC(rc2);
+            }
+            /* MMR3Term is not called here because it'll kill the heap. */
+        }
+        int rc2 = HMR3Term(pVM);
+        AssertRC(rc2);
+    }
+    NEMR3Term(pVM);
+
+    LogFlow(("vmR3InitRing3: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Initializes all R0 components of the VM.
+ */
+static int vmR3InitRing0(PVM pVM)
+{
+    LogFlow(("vmR3InitRing0:\n"));
+
+    /*
+     * Check for FAKE suplib mode.
+     */
+    int rc = VINF_SUCCESS;
+    const char *psz = RTEnvGet("VBOX_SUPLIB_FAKE");
+    if (!psz || strcmp(psz, "fake"))
+    {
+        /*
+         * Call the VMMR0 component and let it do the init.
+         */
+        rc = VMMR3InitR0(pVM);
+    }
+    else
+        Log(("vmR3InitRing0: skipping because of VBOX_SUPLIB_FAKE=fake\n"));
+
+    /*
+     * Do notifications and return.
+     */
+    if (RT_SUCCESS(rc))
+        rc = vmR3InitDoCompleted(pVM, VMINITCOMPLETED_RING0);
+    if (RT_SUCCESS(rc))
+        rc = vmR3InitDoCompleted(pVM, VMINITCOMPLETED_HM);
+
+    LogFlow(("vmR3InitRing0: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Do init completed notifications.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   enmWhat     What's completed.
+ */
+static int vmR3InitDoCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    int rc = VMMR3InitCompleted(pVM, enmWhat);
+    if (RT_SUCCESS(rc))
+        rc = HMR3InitCompleted(pVM, enmWhat);
+    if (RT_SUCCESS(rc))
+        rc = NEMR3InitCompleted(pVM, enmWhat);
+    if (RT_SUCCESS(rc))
+        rc = PGMR3InitCompleted(pVM, enmWhat);
+    if (RT_SUCCESS(rc))
+        rc = CPUMR3InitCompleted(pVM, enmWhat);
+    if (RT_SUCCESS(rc))
+        rc = EMR3InitCompleted(pVM, enmWhat);
+    if (enmWhat == VMINITCOMPLETED_RING3)
+    {
+        if (RT_SUCCESS(rc))
+            rc = SSMR3RegisterStub(pVM, "rem", 1);
+    }
+    if (RT_SUCCESS(rc))
+        rc = PDMR3InitCompleted(pVM, enmWhat);
+
+    /* IOM *must* come after PDM, as device (DevPcArch) may register some final
+       handlers in their init completion method. */
+    if (RT_SUCCESS(rc))
+        rc = IOMR3InitCompleted(pVM, enmWhat);
+    return rc;
+}
+
+
+/**
+ * Calls the relocation functions for all VMM components so they can update
+ * any GC pointers. When this function is called all the basic VM members
+ * have been updated  and the actual memory relocation have been done
+ * by the PGM/MM.
+ *
+ * This is used both on init and on runtime relocations.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3_INT_DECL(void) VMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    LogFlow(("VMR3Relocate: offDelta=%RGv\n", offDelta));
+
+    /*
+     * The order here is very important!
+     */
+    PGMR3Relocate(pVM, offDelta);
+    PDMR3LdrRelocateU(pVM->pUVM, offDelta);
+    PGMR3Relocate(pVM, 0);              /* Repeat after PDM relocation. */
+    CPUMR3Relocate(pVM);
+    HMR3Relocate(pVM);
+    SELMR3Relocate(pVM);
+    VMMR3Relocate(pVM, offDelta);
+    SELMR3Relocate(pVM);                /* !hack! fix stack! */
+    TRPMR3Relocate(pVM, offDelta);
+    IOMR3Relocate(pVM, offDelta);
+    EMR3Relocate(pVM);
+    TMR3Relocate(pVM, offDelta);
+    IEMR3Relocate(pVM);
+    DBGFR3Relocate(pVM, offDelta);
+    PDMR3Relocate(pVM, offDelta);
+    GIMR3Relocate(pVM, offDelta);
+}
+
+
+/**
+ * EMT rendezvous worker for VMR3PowerOn.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE or VINF_SUCCESS. (This is a strict return
+ *          code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser          Ignored.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3PowerOn(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    LogFlow(("vmR3PowerOn: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
+    Assert(!pvUser); NOREF(pvUser);
+
+    /*
+     * The first thread thru here tries to change the state.  We shouldn't be
+     * called again if this fails.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "VMR3PowerOn", 1, VMSTATE_POWERING_ON, VMSTATE_CREATED);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertMsgReturn(enmVMState == VMSTATE_POWERING_ON,
+                    ("%s\n", VMR3GetStateName(enmVMState)),
+                    VERR_VM_UNEXPECTED_UNSTABLE_STATE);
+
+    /*
+     * All EMTs changes their state to started.
+     */
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
+
+    /*
+     * EMT(0) is last thru here and it will make the notification calls
+     * and advance the state.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        PDMR3PowerOn(pVM);
+        vmR3SetState(pVM, VMSTATE_RUNNING, VMSTATE_POWERING_ON);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Powers on the virtual machine.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM        The VM to power on.
+ *
+ * @thread      Any thread.
+ * @vmstate     Created
+ * @vmstateto   PoweringOn+Running
+ */
+VMMR3DECL(int) VMR3PowerOn(PUVM pUVM)
+{
+    LogFlow(("VMR3PowerOn: pUVM=%p\n", pUVM));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Gather all the EMTs to reduce the init TSC drift and keep
+     * the state changing APIs a bit uniform.
+     */
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3PowerOn, NULL);
+    LogFlow(("VMR3PowerOn: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Does the suspend notifications.
+ *
+ * @param  pVM      The cross context VM structure.
+ * @thread  EMT(0)
+ */
+static void vmR3SuspendDoWork(PVM pVM)
+{
+    PDMR3Suspend(pVM);
+}
+
+
+/**
+ * EMT rendezvous worker for VMR3Suspend.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_SUSPEND. (This is a strict
+ *          return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser          Ignored.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3Suspend(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    VMSUSPENDREASON enmReason = (VMSUSPENDREASON)(uintptr_t)pvUser;
+    LogFlow(("vmR3Suspend: pVM=%p pVCpu=%p/#%u enmReason=%d\n", pVM, pVCpu, pVCpu->idCpu, enmReason));
+
+    /*
+     * The first EMT switches the state to suspending.  If this fails because
+     * something was racing us in one way or the other, there will be no more
+     * calls and thus the state assertion below is not going to annoy anyone.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "VMR3Suspend", 2,
+                                 VMSTATE_SUSPENDING,        VMSTATE_RUNNING,
+                                 VMSTATE_SUSPENDING_EXT_LS, VMSTATE_RUNNING_LS);
+        if (RT_FAILURE(rc))
+            return rc;
+        pVM->pUVM->vm.s.enmSuspendReason = enmReason;
+    }
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertMsgReturn(    enmVMState == VMSTATE_SUSPENDING
+                    ||  enmVMState == VMSTATE_SUSPENDING_EXT_LS,
+                    ("%s\n", VMR3GetStateName(enmVMState)),
+                    VERR_VM_UNEXPECTED_UNSTABLE_STATE);
+
+    /*
+     * EMT(0) does the actually suspending *after* all the other CPUs have
+     * been thru here.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        vmR3SuspendDoWork(pVM);
+
+        int rc = vmR3TrySetState(pVM, "VMR3Suspend", 2,
+                                 VMSTATE_SUSPENDED,        VMSTATE_SUSPENDING,
+                                 VMSTATE_SUSPENDED_EXT_LS, VMSTATE_SUSPENDING_EXT_LS);
+        if (RT_FAILURE(rc))
+            return VERR_VM_UNEXPECTED_UNSTABLE_STATE;
+    }
+
+    return VINF_EM_SUSPEND;
+}
+
+
+/**
+ * Suspends a running VM.
+ *
+ * @returns VBox status code. When called on EMT, this will be a strict status
+ *          code that has to be propagated up the call stack.
+ *
+ * @param   pUVM        The VM to suspend.
+ * @param   enmReason   The reason for suspending.
+ *
+ * @thread      Any thread.
+ * @vmstate     Running or RunningLS
+ * @vmstateto   Suspending + Suspended or SuspendingExtLS + SuspendedExtLS
+ */
+VMMR3DECL(int) VMR3Suspend(PUVM pUVM, VMSUSPENDREASON enmReason)
+{
+    LogFlow(("VMR3Suspend: pUVM=%p\n", pUVM));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(enmReason > VMSUSPENDREASON_INVALID && enmReason < VMSUSPENDREASON_END, VERR_INVALID_PARAMETER);
+
+    /*
+     * Gather all the EMTs to make sure there are no races before
+     * changing the VM state.
+     */
+    int rc = VMMR3EmtRendezvous(pUVM->pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3Suspend, (void *)(uintptr_t)enmReason);
+    LogFlow(("VMR3Suspend: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Retrieves the reason for the most recent suspend.
+ *
+ * @returns Suspend reason. VMSUSPENDREASON_INVALID if no suspend has been done
+ *          or the handle is invalid.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(VMSUSPENDREASON) VMR3GetSuspendReason(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VMSUSPENDREASON_INVALID);
+    return pUVM->vm.s.enmSuspendReason;
+}
+
+
+/**
+ * EMT rendezvous worker for VMR3Resume.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_RESUME. (This is a strict
+ *          return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser          Reason.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3Resume(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    VMRESUMEREASON enmReason = (VMRESUMEREASON)(uintptr_t)pvUser;
+    LogFlow(("vmR3Resume: pVM=%p pVCpu=%p/#%u enmReason=%d\n", pVM, pVCpu, pVCpu->idCpu, enmReason));
+
+    /*
+     * The first thread thru here tries to change the state.  We shouldn't be
+     * called again if this fails.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "VMR3Resume", 1, VMSTATE_RESUMING, VMSTATE_SUSPENDED);
+        if (RT_FAILURE(rc))
+            return rc;
+        pVM->pUVM->vm.s.enmResumeReason = enmReason;
+    }
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertMsgReturn(enmVMState == VMSTATE_RESUMING,
+                    ("%s\n", VMR3GetStateName(enmVMState)),
+                    VERR_VM_UNEXPECTED_UNSTABLE_STATE);
+
+#if 0
+    /*
+     * All EMTs changes their state to started.
+     */
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
+#endif
+
+    /*
+     * EMT(0) is last thru here and it will make the notification calls
+     * and advance the state.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        PDMR3Resume(pVM);
+        vmR3SetState(pVM, VMSTATE_RUNNING, VMSTATE_RESUMING);
+        pVM->vm.s.fTeleportedAndNotFullyResumedYet = false;
+    }
+
+    return VINF_EM_RESUME;
+}
+
+
+/**
+ * Resume VM execution.
+ *
+ * @returns VBox status code. When called on EMT, this will be a strict status
+ *          code that has to be propagated up the call stack.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   enmReason   The reason we're resuming.
+ *
+ * @thread      Any thread.
+ * @vmstate     Suspended
+ * @vmstateto   Running
+ */
+VMMR3DECL(int) VMR3Resume(PUVM pUVM, VMRESUMEREASON enmReason)
+{
+    LogFlow(("VMR3Resume: pUVM=%p\n", pUVM));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(enmReason > VMRESUMEREASON_INVALID && enmReason < VMRESUMEREASON_END, VERR_INVALID_PARAMETER);
+
+    /*
+     * Gather all the EMTs to make sure there are no races before
+     * changing the VM state.
+     */
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3Resume, (void *)(uintptr_t)enmReason);
+    LogFlow(("VMR3Resume: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Retrieves the reason for the most recent resume.
+ *
+ * @returns Resume reason. VMRESUMEREASON_INVALID if no suspend has been
+ *          done or the handle is invalid.
+ * @param   pUVM        The user mode VM handle.
+ */
+VMMR3DECL(VMRESUMEREASON) VMR3GetResumeReason(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VMRESUMEREASON_INVALID);
+    return pUVM->vm.s.enmResumeReason;
+}
+
+
+/**
+ * EMT rendezvous worker for VMR3Save and VMR3Teleport that suspends the VM
+ * after the live step has been completed.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_RESUME. (This is a strict
+ *          return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser          The pfSuspended argument of vmR3SaveTeleport.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3LiveDoSuspend(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    LogFlow(("vmR3LiveDoSuspend: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
+    bool *pfSuspended = (bool *)pvUser;
+
+    /*
+     * The first thread thru here tries to change the state.  We shouldn't be
+     * called again if this fails.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1U)
+    {
+        PUVM     pUVM = pVM->pUVM;
+        int      rc;
+
+        RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+        VMSTATE enmVMState = pVM->enmVMState;
+        switch (enmVMState)
+        {
+            case VMSTATE_RUNNING_LS:
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_SUSPENDING_LS, VMSTATE_RUNNING_LS, false /*fSetRatherThanClearFF*/);
+                rc = VINF_SUCCESS;
+                break;
+
+            case VMSTATE_SUSPENDED_EXT_LS:
+            case VMSTATE_SUSPENDED_LS:          /* (via reset) */
+                rc = VINF_SUCCESS;
+                break;
+
+            case VMSTATE_DEBUGGING_LS:
+                rc = VERR_TRY_AGAIN;
+                break;
+
+            case VMSTATE_OFF_LS:
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_OFF, VMSTATE_OFF_LS, false /*fSetRatherThanClearFF*/);
+                rc = VERR_SSM_LIVE_POWERED_OFF;
+                break;
+
+            case VMSTATE_FATAL_ERROR_LS:
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_FATAL_ERROR, VMSTATE_FATAL_ERROR_LS, false /*fSetRatherThanClearFF*/);
+                rc = VERR_SSM_LIVE_FATAL_ERROR;
+                break;
+
+            case VMSTATE_GURU_MEDITATION_LS:
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_GURU_MEDITATION, VMSTATE_GURU_MEDITATION_LS, false /*fSetRatherThanClearFF*/);
+                rc = VERR_SSM_LIVE_GURU_MEDITATION;
+                break;
+
+            case VMSTATE_POWERING_OFF_LS:
+            case VMSTATE_SUSPENDING_EXT_LS:
+            case VMSTATE_RESETTING_LS:
+            default:
+                AssertMsgFailed(("%s\n", VMR3GetStateName(enmVMState)));
+                rc = VERR_VM_UNEXPECTED_VM_STATE;
+                break;
+        }
+        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+        if (RT_FAILURE(rc))
+        {
+            LogFlow(("vmR3LiveDoSuspend: returns %Rrc (state was %s)\n", rc, VMR3GetStateName(enmVMState)));
+            return rc;
+        }
+    }
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertMsgReturn(enmVMState == VMSTATE_SUSPENDING_LS,
+                    ("%s\n", VMR3GetStateName(enmVMState)),
+                    VERR_VM_UNEXPECTED_UNSTABLE_STATE);
+
+    /*
+     * Only EMT(0) have work to do since it's last thru here.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        vmR3SuspendDoWork(pVM);
+        int rc = vmR3TrySetState(pVM, "VMR3Suspend", 1,
+                                 VMSTATE_SUSPENDED_LS, VMSTATE_SUSPENDING_LS);
+        if (RT_FAILURE(rc))
+            return VERR_VM_UNEXPECTED_UNSTABLE_STATE;
+
+        *pfSuspended = true;
+    }
+
+    return VINF_EM_SUSPEND;
+}
+
+
+/**
+ * EMT rendezvous worker that VMR3Save and VMR3Teleport uses to clean up a
+ * SSMR3LiveDoStep1 failure.
+ *
+ * Doing this as a rendezvous operation avoids all annoying transition
+ * states.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE, VINF_SUCCESS or some specific VERR_SSM_*
+ *          status code. (This is a strict return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser          The pfSuspended argument of vmR3SaveTeleport.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3LiveDoStep1Cleanup(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    LogFlow(("vmR3LiveDoStep1Cleanup: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
+    bool *pfSuspended = (bool *)pvUser;
+    NOREF(pVCpu);
+
+    int rc = vmR3TrySetState(pVM, "vmR3LiveDoStep1Cleanup", 8,
+                             VMSTATE_OFF,               VMSTATE_OFF_LS,                     /* 1 */
+                             VMSTATE_FATAL_ERROR,       VMSTATE_FATAL_ERROR_LS,             /* 2 */
+                             VMSTATE_GURU_MEDITATION,   VMSTATE_GURU_MEDITATION_LS,         /* 3 */
+                             VMSTATE_SUSPENDED,         VMSTATE_SUSPENDED_LS,               /* 4 */
+                             VMSTATE_SUSPENDED,         VMSTATE_SAVING,
+                             VMSTATE_SUSPENDED,         VMSTATE_SUSPENDED_EXT_LS,
+                             VMSTATE_RUNNING,           VMSTATE_RUNNING_LS,
+                             VMSTATE_DEBUGGING,         VMSTATE_DEBUGGING_LS);
+    if (rc == 1)
+        rc = VERR_SSM_LIVE_POWERED_OFF;
+    else if (rc == 2)
+        rc = VERR_SSM_LIVE_FATAL_ERROR;
+    else if (rc == 3)
+        rc = VERR_SSM_LIVE_GURU_MEDITATION;
+    else if (rc == 4)
+    {
+        *pfSuspended = true;
+        rc = VINF_SUCCESS;
+    }
+    else if (rc > 0)
+        rc = VINF_SUCCESS;
+    return rc;
+}
+
+
+/**
+ * EMT(0) worker for VMR3Save and VMR3Teleport that completes the live save.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SSM_LIVE_SUSPENDED if VMR3Suspend was called.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            The handle of saved state operation.
+ *
+ * @thread  EMT(0)
+ */
+static DECLCALLBACK(int) vmR3LiveDoStep2(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("vmR3LiveDoStep2: pVM=%p pSSM=%p\n", pVM, pSSM));
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Advance the state and mark if VMR3Suspend was called.
+     */
+    int rc = VINF_SUCCESS;
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    if (enmVMState == VMSTATE_SUSPENDED_LS)
+        vmR3SetState(pVM, VMSTATE_SAVING, VMSTATE_SUSPENDED_LS);
+    else
+    {
+        if (enmVMState != VMSTATE_SAVING)
+            vmR3SetState(pVM, VMSTATE_SAVING, VMSTATE_SUSPENDED_EXT_LS);
+        rc = VINF_SSM_LIVE_SUSPENDED;
+    }
+
+    /*
+     * Finish up and release the handle. Careful with the status codes.
+     */
+    int rc2 = SSMR3LiveDoStep2(pSSM);
+    if (rc == VINF_SUCCESS || (RT_FAILURE(rc2) && RT_SUCCESS(rc)))
+        rc = rc2;
+
+    rc2 = SSMR3LiveDone(pSSM);
+    if (rc == VINF_SUCCESS || (RT_FAILURE(rc2) && RT_SUCCESS(rc)))
+        rc = rc2;
+
+    /*
+     * Advance to the final state and return.
+     */
+    vmR3SetState(pVM, VMSTATE_SUSPENDED, VMSTATE_SAVING);
+    Assert(rc > VINF_EM_LAST || rc < VINF_EM_FIRST);
+    return rc;
+}
+
+
+/**
+ * Worker for vmR3SaveTeleport that validates the state and calls SSMR3Save or
+ * SSMR3LiveSave.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   cMsMaxDowntime      The maximum downtime given as milliseconds.
+ * @param   pszFilename         The name of the file.  NULL if pStreamOps is used.
+ * @param   pStreamOps          The stream methods.  NULL if pszFilename is used.
+ * @param   pvStreamOpsUser     The user argument to the stream methods.
+ * @param   enmAfter            What to do afterwards.
+ * @param   pfnProgress         Progress callback. Optional.
+ * @param   pvProgressUser      User argument for the progress callback.
+ * @param   ppSSM               Where to return the saved state handle in case of a
+ *                              live snapshot scenario.
+ *
+ * @thread  EMT
+ */
+static DECLCALLBACK(int) vmR3Save(PVM pVM, uint32_t cMsMaxDowntime, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                                  SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser, PSSMHANDLE *ppSSM)
+{
+    int rc = VINF_SUCCESS;
+
+    LogFlow(("vmR3Save: pVM=%p cMsMaxDowntime=%u pszFilename=%p:{%s} pStreamOps=%p pvStreamOpsUser=%p enmAfter=%d pfnProgress=%p pvProgressUser=%p ppSSM=%p\n",
+             pVM, cMsMaxDowntime, pszFilename, pszFilename, pStreamOps, pvStreamOpsUser, enmAfter, pfnProgress, pvProgressUser, ppSSM));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrNull(pszFilename);
+    AssertPtrNull(pStreamOps);
+    AssertPtr(pVM);
+    Assert(   enmAfter == SSMAFTER_DESTROY
+           || enmAfter == SSMAFTER_CONTINUE
+           || enmAfter == SSMAFTER_TELEPORT);
+    AssertPtr(ppSSM);
+    *ppSSM = NULL;
+
+    /*
+     * Change the state and perform/start the saving.
+     */
+    rc = vmR3TrySetState(pVM, "VMR3Save", 2,
+                         VMSTATE_SAVING,     VMSTATE_SUSPENDED,
+                         VMSTATE_RUNNING_LS, VMSTATE_RUNNING);
+    if (rc == 1 && enmAfter != SSMAFTER_TELEPORT)
+    {
+        rc = SSMR3Save(pVM, pszFilename, pStreamOps, pvStreamOpsUser, enmAfter, pfnProgress, pvProgressUser);
+        vmR3SetState(pVM, VMSTATE_SUSPENDED, VMSTATE_SAVING);
+    }
+    else if (rc == 2 || enmAfter == SSMAFTER_TELEPORT)
+    {
+        if (enmAfter == SSMAFTER_TELEPORT)
+            pVM->vm.s.fTeleportedAndNotFullyResumedYet = true;
+        rc = SSMR3LiveSave(pVM, cMsMaxDowntime, pszFilename, pStreamOps, pvStreamOpsUser,
+                           enmAfter, pfnProgress, pvProgressUser, ppSSM);
+        /* (We're not subject to cancellation just yet.) */
+    }
+    else
+        Assert(RT_FAILURE(rc));
+    return rc;
+}
+
+
+/**
+ * Common worker for VMR3Save and VMR3Teleport.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   cMsMaxDowntime      The maximum downtime given as milliseconds.
+ * @param   pszFilename         The name of the file.  NULL if pStreamOps is used.
+ * @param   pStreamOps          The stream methods.  NULL if pszFilename is used.
+ * @param   pvStreamOpsUser     The user argument to the stream methods.
+ * @param   enmAfter            What to do afterwards.
+ * @param   pfnProgress         Progress callback. Optional.
+ * @param   pvProgressUser      User argument for the progress callback.
+ * @param   pfSuspended         Set if we suspended the VM.
+ *
+ * @thread  Non-EMT
+ */
+static int vmR3SaveTeleport(PVM pVM, uint32_t cMsMaxDowntime,
+                            const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                            SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser, bool *pfSuspended)
+{
+    /*
+     * Request the operation in EMT(0).
+     */
+    PSSMHANDLE pSSM;
+    int rc = VMR3ReqCallWait(pVM, 0 /*idDstCpu*/,
+                             (PFNRT)vmR3Save, 9, pVM, cMsMaxDowntime, pszFilename, pStreamOps, pvStreamOpsUser,
+                             enmAfter, pfnProgress, pvProgressUser, &pSSM);
+    if (   RT_SUCCESS(rc)
+        && pSSM)
+    {
+        /*
+         * Live snapshot.
+         *
+         * The state handling here is kind of tricky, doing it on EMT(0) helps
+         * a bit. See the VMSTATE diagram for details.
+         */
+        rc = SSMR3LiveDoStep1(pSSM);
+        if (RT_SUCCESS(rc))
+        {
+            if (VMR3GetState(pVM) != VMSTATE_SAVING)
+                for (;;)
+                {
+                    /* Try suspend the VM. */
+                    rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                            vmR3LiveDoSuspend, pfSuspended);
+                    if (rc != VERR_TRY_AGAIN)
+                        break;
+
+                    /* Wait for the state to change. */
+                    RTThreadSleep(250); /** @todo Live Migration: fix this polling wait by some smart use of multiple release event  semaphores.. */
+                }
+            if (RT_SUCCESS(rc))
+                rc = VMR3ReqCallWait(pVM, 0 /*idDstCpu*/, (PFNRT)vmR3LiveDoStep2, 2, pVM, pSSM);
+            else
+            {
+                int rc2 = VMR3ReqCallWait(pVM, 0 /*idDstCpu*/, (PFNRT)SSMR3LiveDone, 1, pSSM);
+                AssertMsg(rc2 == rc, ("%Rrc != %Rrc\n", rc2, rc)); NOREF(rc2);
+            }
+        }
+        else
+        {
+            int rc2 = VMR3ReqCallWait(pVM, 0 /*idDstCpu*/, (PFNRT)SSMR3LiveDone, 1, pSSM);
+            AssertMsg(rc2 == rc, ("%Rrc != %Rrc\n", rc2, rc));
+
+            rc2 = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, vmR3LiveDoStep1Cleanup, pfSuspended);
+            if (RT_FAILURE(rc2) && rc == VERR_SSM_CANCELLED)
+                rc = rc2;
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Save current VM state.
+ *
+ * Can be used for both saving the state and creating snapshots.
+ *
+ * When called for a VM in the Running state, the saved state is created live
+ * and the VM is only suspended when the final part of the saving is preformed.
+ * The VM state will not be restored to Running in this case and it's up to the
+ * caller to call VMR3Resume if this is desirable.  (The rational is that the
+ * caller probably wish to reconfigure the disks before resuming the VM.)
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM                The VM which state should be saved.
+ * @param   pszFilename         The name of the save state file.
+ * @param   fContinueAfterwards Whether continue execution afterwards or not.
+ *                              When in doubt, set this to true.
+ * @param   pfnProgress         Progress callback. Optional.
+ * @param   pvUser              User argument for the progress callback.
+ * @param   pfSuspended         Set if we suspended the VM.
+ *
+ * @thread      Non-EMT.
+ * @vmstate     Suspended or Running
+ * @vmstateto   Saving+Suspended or
+ *              RunningLS+SuspendingLS+SuspendedLS+Saving+Suspended.
+ */
+VMMR3DECL(int) VMR3Save(PUVM pUVM, const char *pszFilename, bool fContinueAfterwards, PFNVMPROGRESS pfnProgress, void *pvUser,
+                        bool *pfSuspended)
+{
+    LogFlow(("VMR3Save: pUVM=%p pszFilename=%p:{%s} fContinueAfterwards=%RTbool pfnProgress=%p pvUser=%p pfSuspended=%p\n",
+             pUVM, pszFilename, pszFilename, fContinueAfterwards, pfnProgress, pvUser, pfSuspended));
+
+    /*
+     * Validate input.
+     */
+    AssertPtr(pfSuspended);
+    *pfSuspended = false;
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_OTHER_THREAD(pVM);
+    AssertReturn(VALID_PTR(pszFilename), VERR_INVALID_POINTER);
+    AssertReturn(*pszFilename, VERR_INVALID_PARAMETER);
+    AssertPtrNullReturn(pfnProgress, VERR_INVALID_POINTER);
+
+    /*
+     * Join paths with VMR3Teleport.
+     */
+    SSMAFTER enmAfter = fContinueAfterwards ? SSMAFTER_CONTINUE : SSMAFTER_DESTROY;
+    int rc = vmR3SaveTeleport(pVM, 250 /*cMsMaxDowntime*/,
+                              pszFilename, NULL /* pStreamOps */, NULL /* pvStreamOpsUser */,
+                              enmAfter, pfnProgress, pvUser, pfSuspended);
+    LogFlow(("VMR3Save: returns %Rrc (*pfSuspended=%RTbool)\n", rc, *pfSuspended));
+    return rc;
+}
+
+
+/**
+ * Teleport the VM (aka live migration).
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM                The VM which state should be saved.
+ * @param   cMsMaxDowntime      The maximum downtime given as milliseconds.
+ * @param   pStreamOps          The stream methods.
+ * @param   pvStreamOpsUser     The user argument to the stream methods.
+ * @param   pfnProgress         Progress callback. Optional.
+ * @param   pvProgressUser      User argument for the progress callback.
+ * @param   pfSuspended         Set if we suspended the VM.
+ *
+ * @thread      Non-EMT.
+ * @vmstate     Suspended or Running
+ * @vmstateto   Saving+Suspended or
+ *              RunningLS+SuspendingLS+SuspendedLS+Saving+Suspended.
+ */
+VMMR3DECL(int) VMR3Teleport(PUVM pUVM, uint32_t cMsMaxDowntime, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                            PFNVMPROGRESS pfnProgress, void *pvProgressUser, bool *pfSuspended)
+{
+    LogFlow(("VMR3Teleport: pUVM=%p cMsMaxDowntime=%u pStreamOps=%p pvStreamOps=%p pfnProgress=%p pvProgressUser=%p\n",
+             pUVM, cMsMaxDowntime, pStreamOps, pvStreamOpsUser, pfnProgress, pvProgressUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtr(pfSuspended);
+    *pfSuspended = false;
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_OTHER_THREAD(pVM);
+    AssertPtrReturn(pStreamOps, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnProgress, VERR_INVALID_POINTER);
+
+    /*
+     * Join paths with VMR3Save.
+     */
+    int rc = vmR3SaveTeleport(pVM, cMsMaxDowntime, NULL /*pszFilename*/, pStreamOps, pvStreamOpsUser,
+                              SSMAFTER_TELEPORT, pfnProgress, pvProgressUser, pfSuspended);
+    LogFlow(("VMR3Teleport: returns %Rrc (*pfSuspended=%RTbool)\n", rc, *pfSuspended));
+    return rc;
+}
+
+
+
+/**
+ * EMT(0) worker for VMR3LoadFromFile and VMR3LoadFromStream.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM                Pointer to the VM.
+ * @param   pszFilename         The name of the file.  NULL if pStreamOps is used.
+ * @param   pStreamOps          The stream methods.  NULL if pszFilename is used.
+ * @param   pvStreamOpsUser     The user argument to the stream methods.
+ * @param   pfnProgress         Progress callback. Optional.
+ * @param   pvProgressUser      User argument for the progress callback.
+ * @param   fTeleporting        Indicates whether we're teleporting or not.
+ *
+ * @thread  EMT.
+ */
+static DECLCALLBACK(int) vmR3Load(PUVM pUVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                                  PFNVMPROGRESS pfnProgress, void *pvProgressUser, bool fTeleporting)
+{
+    LogFlow(("vmR3Load: pUVM=%p pszFilename=%p:{%s} pStreamOps=%p pvStreamOpsUser=%p pfnProgress=%p pvProgressUser=%p fTeleporting=%RTbool\n",
+             pUVM, pszFilename, pszFilename, pStreamOps, pvStreamOpsUser, pfnProgress, pvProgressUser, fTeleporting));
+
+    /*
+     * Validate input (paranoia).
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrNull(pszFilename);
+    AssertPtrNull(pStreamOps);
+    AssertPtrNull(pfnProgress);
+
+    /*
+     * Change the state and perform the load.
+     *
+     * Always perform a relocation round afterwards to make sure hypervisor
+     * selectors and such are correct.
+     */
+    int rc = vmR3TrySetState(pVM, "VMR3Load", 2,
+                             VMSTATE_LOADING, VMSTATE_CREATED,
+                             VMSTATE_LOADING, VMSTATE_SUSPENDED);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    pVM->vm.s.fTeleportedAndNotFullyResumedYet = fTeleporting;
+
+    uint32_t cErrorsPriorToSave = VMR3GetErrorCount(pUVM);
+    rc = SSMR3Load(pVM, pszFilename, pStreamOps, pvStreamOpsUser, SSMAFTER_RESUME, pfnProgress, pvProgressUser);
+    if (RT_SUCCESS(rc))
+    {
+        VMR3Relocate(pVM, 0 /*offDelta*/);
+        vmR3SetState(pVM, VMSTATE_SUSPENDED, VMSTATE_LOADING);
+    }
+    else
+    {
+        pVM->vm.s.fTeleportedAndNotFullyResumedYet = false;
+        vmR3SetState(pVM, VMSTATE_LOAD_FAILURE, VMSTATE_LOADING);
+
+        if (cErrorsPriorToSave == VMR3GetErrorCount(pUVM))
+            rc = VMSetError(pVM, rc, RT_SRC_POS,
+                            N_("Unable to restore the virtual machine's saved state from '%s'. "
+                               "It may be damaged or from an older version of VirtualBox.  "
+                               "Please discard the saved state before starting the virtual machine"),
+                            pszFilename);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Loads a VM state into a newly created VM or a one that is suspended.
+ *
+ * To restore a saved state on VM startup, call this function and then resume
+ * the VM instead of powering it on.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM            The user mode VM structure.
+ * @param   pszFilename     The name of the save state file.
+ * @param   pfnProgress     Progress callback. Optional.
+ * @param   pvUser          User argument for the progress callback.
+ *
+ * @thread      Any thread.
+ * @vmstate     Created, Suspended
+ * @vmstateto   Loading+Suspended
+ */
+VMMR3DECL(int) VMR3LoadFromFile(PUVM pUVM, const char *pszFilename, PFNVMPROGRESS pfnProgress, void *pvUser)
+{
+    LogFlow(("VMR3LoadFromFile: pUVM=%p pszFilename=%p:{%s} pfnProgress=%p pvUser=%p\n",
+             pUVM, pszFilename, pszFilename, pfnProgress, pvUser));
+
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszFilename, VERR_INVALID_POINTER);
+
+    /*
+     * Forward the request to EMT(0).  No need to setup a rendezvous here
+     * since there is no execution taking place when this call is allowed.
+     */
+    int rc = VMR3ReqCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)vmR3Load, 7,
+                              pUVM, pszFilename, (uintptr_t)NULL /*pStreamOps*/, (uintptr_t)NULL /*pvStreamOpsUser*/,
+                              pfnProgress, pvUser, false /*fTeleporting*/);
+    LogFlow(("VMR3LoadFromFile: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VMR3LoadFromFile for arbitrary file streams.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM            Pointer to the VM.
+ * @param   pStreamOps      The stream methods.
+ * @param   pvStreamOpsUser The user argument to the stream methods.
+ * @param   pfnProgress     Progress callback. Optional.
+ * @param   pvProgressUser  User argument for the progress callback.
+ *
+ * @thread      Any thread.
+ * @vmstate     Created, Suspended
+ * @vmstateto   Loading+Suspended
+ */
+VMMR3DECL(int) VMR3LoadFromStream(PUVM pUVM, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                                  PFNVMPROGRESS pfnProgress, void *pvProgressUser)
+{
+    LogFlow(("VMR3LoadFromStream: pUVM=%p pStreamOps=%p pvStreamOpsUser=%p pfnProgress=%p pvProgressUser=%p\n",
+             pUVM, pStreamOps, pvStreamOpsUser, pfnProgress, pvProgressUser));
+
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pStreamOps, VERR_INVALID_POINTER);
+
+    /*
+     * Forward the request to EMT(0).  No need to setup a rendezvous here
+     * since there is no execution taking place when this call is allowed.
+     */
+    int rc = VMR3ReqCallWaitU(pUVM, 0 /*idDstCpu*/, (PFNRT)vmR3Load, 7,
+                              pUVM, (uintptr_t)NULL /*pszFilename*/, pStreamOps, pvStreamOpsUser, pfnProgress,
+                              pvProgressUser, true /*fTeleporting*/);
+    LogFlow(("VMR3LoadFromStream: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * EMT rendezvous worker for VMR3PowerOff.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_OFF. (This is a strict
+ *          return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser          Ignored.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3PowerOff(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    LogFlow(("vmR3PowerOff: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
+    Assert(!pvUser); NOREF(pvUser);
+
+    /*
+     * The first EMT thru here will change the state to PoweringOff.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "VMR3PowerOff", 11,
+                                 VMSTATE_POWERING_OFF,    VMSTATE_RUNNING,           /* 1 */
+                                 VMSTATE_POWERING_OFF,    VMSTATE_SUSPENDED,         /* 2 */
+                                 VMSTATE_POWERING_OFF,    VMSTATE_DEBUGGING,         /* 3 */
+                                 VMSTATE_POWERING_OFF,    VMSTATE_LOAD_FAILURE,      /* 4 */
+                                 VMSTATE_POWERING_OFF,    VMSTATE_GURU_MEDITATION,   /* 5 */
+                                 VMSTATE_POWERING_OFF,    VMSTATE_FATAL_ERROR,       /* 6 */
+                                 VMSTATE_POWERING_OFF,    VMSTATE_CREATED,           /* 7 */   /** @todo update the diagram! */
+                                 VMSTATE_POWERING_OFF_LS, VMSTATE_RUNNING_LS,        /* 8 */
+                                 VMSTATE_POWERING_OFF_LS, VMSTATE_DEBUGGING_LS,      /* 9 */
+                                 VMSTATE_POWERING_OFF_LS, VMSTATE_GURU_MEDITATION_LS,/* 10 */
+                                 VMSTATE_POWERING_OFF_LS, VMSTATE_FATAL_ERROR_LS);   /* 11 */
+        if (RT_FAILURE(rc))
+            return rc;
+        if (rc >= 7)
+            SSMR3Cancel(pVM->pUVM);
+    }
+
+    /*
+     * Check the state.
+     */
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertMsgReturn(   enmVMState == VMSTATE_POWERING_OFF
+                    || enmVMState == VMSTATE_POWERING_OFF_LS,
+                    ("%s\n", VMR3GetStateName(enmVMState)),
+                    VERR_VM_INVALID_VM_STATE);
+
+    /*
+     * EMT(0) does the actual power off work here *after* all the other EMTs
+     * have been thru and entered the STOPPED state.
+     */
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STOPPED);
+    if (pVCpu->idCpu == 0)
+    {
+        /*
+         * For debugging purposes, we will log a summary of the guest state at this point.
+         */
+        if (enmVMState != VMSTATE_GURU_MEDITATION)
+        {
+            /** @todo make the state dumping at VMR3PowerOff optional. */
+            bool fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
+            RTLogRelPrintf("****************** Guest state at power off for VCpu %u ******************\n", pVCpu->idCpu);
+            DBGFR3InfoEx(pVM->pUVM, pVCpu->idCpu, "cpumguest", "verbose", DBGFR3InfoLogRelHlp());
+            RTLogRelPrintf("***\n");
+            DBGFR3InfoEx(pVM->pUVM, pVCpu->idCpu, "cpumguesthwvirt", "verbose", DBGFR3InfoLogRelHlp());
+            RTLogRelPrintf("***\n");
+            DBGFR3InfoEx(pVM->pUVM, pVCpu->idCpu, "mode", NULL, DBGFR3InfoLogRelHlp());
+            RTLogRelPrintf("***\n");
+            DBGFR3Info(pVM->pUVM, "activetimers", NULL, DBGFR3InfoLogRelHlp());
+            RTLogRelPrintf("***\n");
+            DBGFR3Info(pVM->pUVM, "gdt", NULL, DBGFR3InfoLogRelHlp());
+            /** @todo dump guest call stack. */
+            RTLogRelSetBuffering(fOldBuffered);
+            RTLogRelPrintf("************** End of Guest state at power off ***************\n");
+        }
+
+        /*
+         * Perform the power off notifications and advance the state to
+         * Off or OffLS.
+         */
+        PDMR3PowerOff(pVM);
+        DBGFR3PowerOff(pVM);
+
+        PUVM pUVM = pVM->pUVM;
+        RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+        enmVMState = pVM->enmVMState;
+        if (enmVMState == VMSTATE_POWERING_OFF_LS)
+            vmR3SetStateLocked(pVM, pUVM, VMSTATE_OFF_LS, VMSTATE_POWERING_OFF_LS, false /*fSetRatherThanClearFF*/);
+        else
+            vmR3SetStateLocked(pVM, pUVM, VMSTATE_OFF,    VMSTATE_POWERING_OFF, false /*fSetRatherThanClearFF*/);
+        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+    }
+    else if (enmVMState != VMSTATE_GURU_MEDITATION)
+    {
+        /** @todo make the state dumping at VMR3PowerOff optional. */
+        bool fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
+        RTLogRelPrintf("****************** Guest state at power off for VCpu %u ******************\n", pVCpu->idCpu);
+        DBGFR3InfoEx(pVM->pUVM, pVCpu->idCpu, "cpumguest", "verbose", DBGFR3InfoLogRelHlp());
+        RTLogRelPrintf("***\n");
+        DBGFR3InfoEx(pVM->pUVM, pVCpu->idCpu, "cpumguesthwvirt", "verbose", DBGFR3InfoLogRelHlp());
+        RTLogRelPrintf("***\n");
+        DBGFR3InfoEx(pVM->pUVM, pVCpu->idCpu, "mode", NULL, DBGFR3InfoLogRelHlp());
+        RTLogRelPrintf("***\n");
+        RTLogRelSetBuffering(fOldBuffered);
+        RTLogRelPrintf("************** End of Guest state at power off for VCpu %u ***************\n", pVCpu->idCpu);
+    }
+
+    return VINF_EM_OFF;
+}
+
+
+/**
+ * Power off the VM.
+ *
+ * @returns VBox status code. When called on EMT, this will be a strict status
+ *          code that has to be propagated up the call stack.
+ *
+ * @param   pUVM    The handle of the VM to be powered off.
+ *
+ * @thread      Any thread.
+ * @vmstate     Suspended, Running, Guru Meditation, Load Failure
+ * @vmstateto   Off or OffLS
+ */
+VMMR3DECL(int)   VMR3PowerOff(PUVM pUVM)
+{
+    LogFlow(("VMR3PowerOff: pUVM=%p\n", pUVM));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Gather all the EMTs to make sure there are no races before
+     * changing the VM state.
+     */
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3PowerOff, NULL);
+    LogFlow(("VMR3PowerOff: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Destroys the VM.
+ *
+ * The VM must be powered off (or never really powered on) to call this
+ * function. The VM handle is destroyed and can no longer be used up successful
+ * return.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM    The user mode VM handle.
+ *
+ * @thread      Any none emulation thread.
+ * @vmstate     Off, Created
+ * @vmstateto   N/A
+ */
+VMMR3DECL(int) VMR3Destroy(PUVM pUVM)
+{
+    LogFlow(("VMR3Destroy: pUVM=%p\n", pUVM));
+
+    /*
+     * Validate input.
+     */
+    if (!pUVM)
+        return VERR_INVALID_VM_HANDLE;
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertLogRelReturn(!VM_IS_EMT(pVM), VERR_VM_THREAD_IS_EMT);
+
+    /*
+     * Change VM state to destroying and aall vmR3Destroy on each of the EMTs
+     * ending with EMT(0) doing the bulk of the cleanup.
+     */
+    int rc = vmR3TrySetState(pVM, "VMR3Destroy", 1, VMSTATE_DESTROYING, VMSTATE_OFF);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    rc = VMR3ReqCallWait(pVM, VMCPUID_ALL_REVERSE, (PFNRT)vmR3Destroy, 1, pVM);
+    AssertLogRelRC(rc);
+
+    /*
+     * Wait for EMTs to quit and destroy the UVM.
+     */
+    vmR3DestroyUVM(pUVM, 30000);
+
+    LogFlow(("VMR3Destroy: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal destruction worker.
+ *
+ * This is either called from VMR3Destroy via VMR3ReqCallU or from
+ * vmR3EmulationThreadWithId when EMT(0) terminates after having called
+ * VMR3Destroy().
+ *
+ * When called on EMT(0), it will performed the great bulk of the destruction.
+ * When called on the other EMTs, they will do nothing and the whole purpose is
+ * to return VINF_EM_TERMINATE so they break out of their run loops.
+ *
+ * @returns VINF_EM_TERMINATE.
+ * @param   pVM     The cross context VM structure.
+ */
+DECLCALLBACK(int) vmR3Destroy(PVM pVM)
+{
+    PUVM   pUVM  = pVM->pUVM;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    Assert(pVCpu);
+    LogFlow(("vmR3Destroy: pVM=%p pUVM=%p pVCpu=%p idCpu=%u\n", pVM, pUVM, pVCpu, pVCpu->idCpu));
+
+    /*
+     * Only VCPU 0 does the full cleanup (last).
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        /*
+         * Dump statistics to the log.
+         */
+#if defined(VBOX_WITH_STATISTICS) || defined(LOG_ENABLED)
+        RTLogFlags(NULL, "nodisabled nobuffered");
+#endif
+//#ifdef VBOX_WITH_STATISTICS
+//        STAMR3Dump(pUVM, "*");
+//#else
+        LogRel(("************************* Statistics *************************\n"));
+        STAMR3DumpToReleaseLog(pUVM, "*");
+        LogRel(("********************* End of statistics **********************\n"));
+//#endif
+
+        /*
+         * Destroy the VM components.
+         */
+        int rc = TMR3Term(pVM);
+        AssertRC(rc);
+#ifdef VBOX_WITH_DEBUGGER
+        rc = DBGCTcpTerminate(pUVM, pUVM->vm.s.pvDBGC);
+        pUVM->vm.s.pvDBGC = NULL;
+#endif
+        AssertRC(rc);
+        rc = PDMR3Term(pVM);
+        AssertRC(rc);
+        rc = GIMR3Term(pVM);
+        AssertRC(rc);
+        rc = DBGFR3Term(pVM);
+        AssertRC(rc);
+        rc = IEMR3Term(pVM);
+        AssertRC(rc);
+        rc = EMR3Term(pVM);
+        AssertRC(rc);
+        rc = IOMR3Term(pVM);
+        AssertRC(rc);
+        rc = TRPMR3Term(pVM);
+        AssertRC(rc);
+        rc = SELMR3Term(pVM);
+        AssertRC(rc);
+        rc = HMR3Term(pVM);
+        AssertRC(rc);
+        rc = NEMR3Term(pVM);
+        AssertRC(rc);
+        rc = PGMR3Term(pVM);
+        AssertRC(rc);
+        rc = VMMR3Term(pVM); /* Terminates the ring-0 code! */
+        AssertRC(rc);
+        rc = CPUMR3Term(pVM);
+        AssertRC(rc);
+        SSMR3Term(pVM);
+        rc = PDMR3CritSectBothTerm(pVM);
+        AssertRC(rc);
+        rc = MMR3Term(pVM);
+        AssertRC(rc);
+
+        /*
+         * We're done, tell the other EMTs to quit.
+         */
+        ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
+        ASMAtomicWriteU32(&pVM->fGlobalForcedActions, VM_FF_CHECK_VM_STATE); /* Can't hurt... */
+        LogFlow(("vmR3Destroy: returning %Rrc\n", VINF_EM_TERMINATE));
+    }
+
+    /*
+     * Decrement the active EMT count here.
+     */
+    PUVMCPU pUVCpu = &pUVM->aCpus[pVCpu->idCpu];
+    if (!pUVCpu->vm.s.fBeenThruVmDestroy)
+    {
+        pUVCpu->vm.s.fBeenThruVmDestroy = true;
+        ASMAtomicDecU32(&pUVM->vm.s.cActiveEmts);
+    }
+    else
+        AssertFailed();
+
+    return VINF_EM_TERMINATE;
+}
+
+
+/**
+ * Destroys the UVM portion.
+ *
+ * This is called as the final step in the VM destruction or as the cleanup
+ * in case of a creation failure.
+ *
+ * @param   pUVM            The user mode VM structure.
+ * @param   cMilliesEMTWait The number of milliseconds to wait for the emulation
+ *                          threads.
+ */
+static void vmR3DestroyUVM(PUVM pUVM, uint32_t cMilliesEMTWait)
+{
+    /*
+     * Signal termination of each the emulation threads and
+     * wait for them to complete.
+     */
+    /* Signal them - in reverse order since EMT(0) waits for the others. */
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
+    if (pUVM->pVM)
+        VM_FF_SET(pUVM->pVM, VM_FF_CHECK_VM_STATE); /* Can't hurt... */
+    VMCPUID iCpu = pUVM->cCpus;
+    while (iCpu-- > 0)
+    {
+        VMR3NotifyGlobalFFU(pUVM, VMNOTIFYFF_FLAGS_DONE_REM);
+        RTSemEventSignal(pUVM->aCpus[iCpu].vm.s.EventSemWait);
+    }
+
+    /* Wait for EMT(0), it in turn waits for the rest. */
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
+
+    RTTHREAD const hSelf = RTThreadSelf();
+    RTTHREAD hThread = pUVM->aCpus[0].vm.s.ThreadEMT;
+    if (   hThread != NIL_RTTHREAD
+        && hThread != hSelf)
+    {
+        int rc2 = RTThreadWait(hThread, RT_MAX(cMilliesEMTWait, 2000), NULL);
+        if (rc2 == VERR_TIMEOUT) /* avoid the assertion when debugging. */
+            rc2 = RTThreadWait(hThread, 1000, NULL);
+        AssertLogRelMsgRC(rc2, ("iCpu=0 rc=%Rrc\n", rc2));
+        if (RT_SUCCESS(rc2))
+            pUVM->aCpus[0].vm.s.ThreadEMT = NIL_RTTHREAD;
+    }
+
+    /* Just in case we're in a weird failure situation w/o EMT(0) to do the
+       waiting, wait the other EMTs too. */
+    for (iCpu = 1; iCpu < pUVM->cCpus; iCpu++)
+    {
+        ASMAtomicXchgHandle(&pUVM->aCpus[iCpu].vm.s.ThreadEMT, NIL_RTTHREAD, &hThread);
+        if (hThread != NIL_RTTHREAD)
+        {
+            if (hThread != hSelf)
+            {
+                int rc2 = RTThreadWait(hThread, 250 /*ms*/, NULL);
+                AssertLogRelMsgRC(rc2, ("iCpu=%u rc=%Rrc\n", iCpu, rc2));
+                if (RT_SUCCESS(rc2))
+                    continue;
+            }
+            pUVM->aCpus[iCpu].vm.s.ThreadEMT = hThread;
+        }
+    }
+
+    /* Cleanup the semaphores. */
+    iCpu = pUVM->cCpus;
+    while (iCpu-- > 0)
+    {
+        RTSemEventDestroy(pUVM->aCpus[iCpu].vm.s.EventSemWait);
+        pUVM->aCpus[iCpu].vm.s.EventSemWait = NIL_RTSEMEVENT;
+    }
+
+    /*
+     * Free the event semaphores associated with the request packets.
+     */
+    unsigned cReqs = 0;
+    for (unsigned i = 0; i < RT_ELEMENTS(pUVM->vm.s.apReqFree); i++)
+    {
+        PVMREQ pReq = pUVM->vm.s.apReqFree[i];
+        pUVM->vm.s.apReqFree[i] = NULL;
+        for (; pReq; pReq = pReq->pNext, cReqs++)
+        {
+            pReq->enmState = VMREQSTATE_INVALID;
+            RTSemEventDestroy(pReq->EventSem);
+        }
+    }
+    Assert(cReqs == pUVM->vm.s.cReqFree); NOREF(cReqs);
+
+    /*
+     * Kill all queued requests. (There really shouldn't be any!)
+     */
+    for (unsigned i = 0; i < 10; i++)
+    {
+        PVMREQ pReqHead = ASMAtomicXchgPtrT(&pUVM->vm.s.pPriorityReqs, NULL, PVMREQ);
+        if (!pReqHead)
+        {
+            pReqHead = ASMAtomicXchgPtrT(&pUVM->vm.s.pNormalReqs, NULL, PVMREQ);
+            if (!pReqHead)
+                break;
+        }
+        AssertLogRelMsgFailed(("Requests pending! VMR3Destroy caller has to serialize this.\n"));
+
+        for (PVMREQ pReq = pReqHead; pReq; pReq = pReq->pNext)
+        {
+            ASMAtomicUoWriteS32(&pReq->iStatus, VERR_VM_REQUEST_KILLED);
+            ASMAtomicWriteSize(&pReq->enmState, VMREQSTATE_INVALID);
+            RTSemEventSignal(pReq->EventSem);
+            RTThreadSleep(2);
+            RTSemEventDestroy(pReq->EventSem);
+        }
+        /* give them a chance to respond before we free the request memory. */
+        RTThreadSleep(32);
+    }
+
+    /*
+     * Now all queued VCPU requests (again, there shouldn't be any).
+     */
+    for (VMCPUID idCpu = 0; idCpu < pUVM->cCpus; idCpu++)
+    {
+        PUVMCPU pUVCpu = &pUVM->aCpus[idCpu];
+
+        for (unsigned i = 0; i < 10; i++)
+        {
+            PVMREQ pReqHead = ASMAtomicXchgPtrT(&pUVCpu->vm.s.pPriorityReqs, NULL, PVMREQ);
+            if (!pReqHead)
+            {
+                pReqHead = ASMAtomicXchgPtrT(&pUVCpu->vm.s.pNormalReqs, NULL, PVMREQ);
+                if (!pReqHead)
+                    break;
+            }
+            AssertLogRelMsgFailed(("Requests pending! VMR3Destroy caller has to serialize this.\n"));
+
+            for (PVMREQ pReq = pReqHead; pReq; pReq = pReq->pNext)
+            {
+                ASMAtomicUoWriteS32(&pReq->iStatus, VERR_VM_REQUEST_KILLED);
+                ASMAtomicWriteSize(&pReq->enmState, VMREQSTATE_INVALID);
+                RTSemEventSignal(pReq->EventSem);
+                RTThreadSleep(2);
+                RTSemEventDestroy(pReq->EventSem);
+            }
+            /* give them a chance to respond before we free the request memory. */
+            RTThreadSleep(32);
+        }
+    }
+
+    /*
+     * Make sure the VMMR0.r0 module and whatever else is unloaded.
+     */
+    PDMR3TermUVM(pUVM);
+
+    RTCritSectDelete(&pUVM->vm.s.AtErrorCritSect);
+    RTCritSectDelete(&pUVM->vm.s.AtStateCritSect);
+
+    /*
+     * Terminate the support library if initialized.
+     */
+    if (pUVM->vm.s.pSession)
+    {
+        int rc = SUPR3Term(false /*fForced*/);
+        AssertRC(rc);
+        pUVM->vm.s.pSession = NIL_RTR0PTR;
+    }
+
+    /*
+     * Release the UVM structure reference.
+     */
+    VMR3ReleaseUVM(pUVM);
+
+    /*
+     * Clean up and flush logs.
+     */
+    RTLogFlush(NULL);
+}
+
+
+/**
+ * Worker which checks integrity of some internal structures.
+ * This is yet another attempt to track down that AVL tree crash.
+ */
+static void vmR3CheckIntegrity(PVM pVM)
+{
+#ifdef VBOX_STRICT
+    int rc = PGMR3CheckIntegrity(pVM);
+    AssertReleaseRC(rc);
+#else
+    RT_NOREF_PV(pVM);
+#endif
+}
+
+
+/**
+ * EMT rendezvous worker for VMR3ResetFF for doing soft/warm reset.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE, VINF_EM_RESCHEDULE.
+ *          (This is a strict return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser          The reset flags.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3SoftReset(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    uint32_t fResetFlags = *(uint32_t *)pvUser;
+
+
+    /*
+     * The first EMT will try change the state to resetting.  If this fails,
+     * we won't get called for the other EMTs.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "vmR3ResetSoft", 3,
+                                 VMSTATE_SOFT_RESETTING,     VMSTATE_RUNNING,
+                                 VMSTATE_SOFT_RESETTING,     VMSTATE_SUSPENDED,
+                                 VMSTATE_SOFT_RESETTING_LS,  VMSTATE_RUNNING_LS);
+        if (RT_FAILURE(rc))
+            return rc;
+        pVM->vm.s.cResets++;
+        pVM->vm.s.cSoftResets++;
+    }
+
+    /*
+     * Check the state.
+     */
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertLogRelMsgReturn(   enmVMState == VMSTATE_SOFT_RESETTING
+                          || enmVMState == VMSTATE_SOFT_RESETTING_LS,
+                          ("%s\n", VMR3GetStateName(enmVMState)),
+                          VERR_VM_UNEXPECTED_UNSTABLE_STATE);
+
+    /*
+     * EMT(0) does the full cleanup *after* all the other EMTs has been
+     * thru here and been told to enter the EMSTATE_WAIT_SIPI state.
+     *
+     * Because there are per-cpu reset routines and order may/is important,
+     * the following sequence looks a bit ugly...
+     */
+
+    /* Reset the VCpu state. */
+    VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED);
+
+    /*
+     * Soft reset the VM components.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        PDMR3SoftReset(pVM, fResetFlags);
+        TRPMR3Reset(pVM);
+        CPUMR3Reset(pVM);               /* This must come *after* PDM (due to APIC base MSR caching). */
+        EMR3Reset(pVM);
+        HMR3Reset(pVM);                 /* This must come *after* PATM, CSAM, CPUM, SELM and TRPM. */
+        NEMR3Reset(pVM);
+
+        /*
+         * Since EMT(0) is the last to go thru here, it will advance the state.
+         * (Unlike vmR3HardReset we won't be doing any suspending of live
+         * migration VMs here since memory is unchanged.)
+         */
+        PUVM pUVM = pVM->pUVM;
+        RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+        enmVMState = pVM->enmVMState;
+        if (enmVMState == VMSTATE_SOFT_RESETTING)
+        {
+            if (pUVM->vm.s.enmPrevVMState == VMSTATE_SUSPENDED)
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_SUSPENDED, VMSTATE_SOFT_RESETTING, false /*fSetRatherThanClearFF*/);
+            else
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_RUNNING,   VMSTATE_SOFT_RESETTING, false /*fSetRatherThanClearFF*/);
+        }
+        else
+            vmR3SetStateLocked(pVM, pUVM, VMSTATE_RUNNING_LS, VMSTATE_SOFT_RESETTING_LS, false /*fSetRatherThanClearFF*/);
+        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+    }
+
+    return VINF_EM_RESCHEDULE;
+}
+
+
+/**
+ * EMT rendezvous worker for VMR3Reset and VMR3ResetFF.
+ *
+ * This is called by the emulation threads as a response to the reset request
+ * issued by VMR3Reset().
+ *
+ * @returns VERR_VM_INVALID_VM_STATE, VINF_EM_RESET or VINF_EM_SUSPEND. (This
+ *          is a strict return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser          Ignored.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3HardReset(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    Assert(!pvUser); NOREF(pvUser);
+
+    /*
+     * The first EMT will try change the state to resetting.  If this fails,
+     * we won't get called for the other EMTs.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "vmR3HardReset", 3,
+                                 VMSTATE_RESETTING,     VMSTATE_RUNNING,
+                                 VMSTATE_RESETTING,     VMSTATE_SUSPENDED,
+                                 VMSTATE_RESETTING_LS,  VMSTATE_RUNNING_LS);
+        if (RT_FAILURE(rc))
+            return rc;
+        pVM->vm.s.cResets++;
+        pVM->vm.s.cHardResets++;
+    }
+
+    /*
+     * Check the state.
+     */
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertLogRelMsgReturn(   enmVMState == VMSTATE_RESETTING
+                          || enmVMState == VMSTATE_RESETTING_LS,
+                          ("%s\n", VMR3GetStateName(enmVMState)),
+                          VERR_VM_UNEXPECTED_UNSTABLE_STATE);
+
+    /*
+     * EMT(0) does the full cleanup *after* all the other EMTs has been
+     * thru here and been told to enter the EMSTATE_WAIT_SIPI state.
+     *
+     * Because there are per-cpu reset routines and order may/is important,
+     * the following sequence looks a bit ugly...
+     */
+    if (pVCpu->idCpu == 0)
+        vmR3CheckIntegrity(pVM);
+
+    /* Reset the VCpu state. */
+    VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED);
+
+    /* Clear all pending forced actions. */
+    VMCPU_FF_CLEAR_MASK(pVCpu, VMCPU_FF_ALL_MASK & ~VMCPU_FF_REQUEST);
+
+    /*
+     * Reset the VM components.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        GIMR3Reset(pVM);                /* This must come *before* PDM and TM. */
+        PDMR3Reset(pVM);
+        PGMR3Reset(pVM);
+        SELMR3Reset(pVM);
+        TRPMR3Reset(pVM);
+        IOMR3Reset(pVM);
+        CPUMR3Reset(pVM);               /* This must come *after* PDM (due to APIC base MSR caching). */
+        TMR3Reset(pVM);
+        EMR3Reset(pVM);
+        HMR3Reset(pVM);                 /* This must come *after* PATM, CSAM, CPUM, SELM and TRPM. */
+        NEMR3Reset(pVM);
+
+        /*
+         * Do memory setup.
+         */
+        PGMR3MemSetup(pVM, true /*fAtReset*/);
+        PDMR3MemSetup(pVM, true /*fAtReset*/);
+
+        /*
+         * Since EMT(0) is the last to go thru here, it will advance the state.
+         * When a live save is active, we will move on to SuspendingLS but
+         * leave it for VMR3Reset to do the actual suspending due to deadlock risks.
+         */
+        PUVM pUVM = pVM->pUVM;
+        RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+        enmVMState = pVM->enmVMState;
+        if (enmVMState == VMSTATE_RESETTING)
+        {
+            if (pUVM->vm.s.enmPrevVMState == VMSTATE_SUSPENDED)
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_SUSPENDED, VMSTATE_RESETTING, false /*fSetRatherThanClearFF*/);
+            else
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_RUNNING,   VMSTATE_RESETTING, false /*fSetRatherThanClearFF*/);
+        }
+        else
+            vmR3SetStateLocked(pVM, pUVM, VMSTATE_SUSPENDING_LS, VMSTATE_RESETTING_LS, false /*fSetRatherThanClearFF*/);
+        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+
+        vmR3CheckIntegrity(pVM);
+
+        /*
+         * Do the suspend bit as well.
+         * It only requires some EMT(0) work at present.
+         */
+        if (enmVMState != VMSTATE_RESETTING)
+        {
+            vmR3SuspendDoWork(pVM);
+            vmR3SetState(pVM, VMSTATE_SUSPENDED_LS, VMSTATE_SUSPENDING_LS);
+        }
+    }
+
+    return enmVMState == VMSTATE_RESETTING
+         ? VINF_EM_RESET
+         : VINF_EM_SUSPEND; /** @todo VINF_EM_SUSPEND has lower priority than VINF_EM_RESET, so fix races. Perhaps add a new code for this combined case. */
+}
+
+
+/**
+ * Internal worker for VMR3Reset, VMR3ResetFF, VMR3TripleFault.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   fHardReset      Whether it's a hard reset or not.
+ * @param   fResetFlags     The reset flags (PDMVMRESET_F_XXX).
+ */
+static VBOXSTRICTRC vmR3ResetCommon(PVM pVM, bool fHardReset, uint32_t fResetFlags)
+{
+    LogFlow(("vmR3ResetCommon: fHardReset=%RTbool fResetFlags=%#x\n", fHardReset, fResetFlags));
+    int rc;
+    if (fHardReset)
+    {
+        /*
+         * Hard reset.
+         */
+        /* Check whether we're supposed to power off instead of resetting. */
+        if (pVM->vm.s.fPowerOffInsteadOfReset)
+        {
+            PUVM pUVM = pVM->pUVM;
+            if (   pUVM->pVmm2UserMethods
+                && pUVM->pVmm2UserMethods->pfnNotifyResetTurnedIntoPowerOff)
+                pUVM->pVmm2UserMethods->pfnNotifyResetTurnedIntoPowerOff(pUVM->pVmm2UserMethods, pUVM);
+            return VMR3PowerOff(pUVM);
+        }
+
+        /* Gather all the EMTs to make sure there are no races before changing
+           the VM state. */
+        rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3HardReset, NULL);
+    }
+    else
+    {
+        /*
+         * Soft reset. Since we only support this with a single CPU active,
+         * we must be on EMT #0 here.
+         */
+        VM_ASSERT_EMT0(pVM);
+        rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3SoftReset, &fResetFlags);
+    }
+
+    LogFlow(("vmR3ResetCommon: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+
+/**
+ * Reset the current VM.
+ *
+ * @returns VBox status code.
+ * @param   pUVM    The VM to reset.
+ */
+VMMR3DECL(int) VMR3Reset(PUVM pUVM)
+{
+    LogFlow(("VMR3Reset:\n"));
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    return VBOXSTRICTRC_VAL(vmR3ResetCommon(pVM, true, 0));
+}
+
+
+/**
+ * Handle the reset force flag or triple fault.
+ *
+ * This handles both soft and hard resets (see PDMVMRESET_F_XXX).
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @thread  EMT
+ *
+ * @remarks Caller is expected to clear the VM_FF_RESET force flag.
+ */
+VMMR3_INT_DECL(VBOXSTRICTRC) VMR3ResetFF(PVM pVM)
+{
+    LogFlow(("VMR3ResetFF:\n"));
+
+    /*
+     * First consult the firmware on whether this is a hard or soft reset.
+     */
+    uint32_t fResetFlags;
+    bool fHardReset = PDMR3GetResetInfo(pVM, 0 /*fOverride*/, &fResetFlags);
+    return vmR3ResetCommon(pVM, fHardReset, fResetFlags);
+}
+
+
+/**
+ * For handling a CPU reset on triple fault.
+ *
+ * According to one mainboard manual, a CPU triple fault causes the 286 CPU to
+ * send a SHUTDOWN signal to the chipset.  The chipset responds by sending a
+ * RESET signal to the CPU.  So, it should be very similar to a soft/warm reset.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @thread  EMT
+ */
+VMMR3_INT_DECL(VBOXSTRICTRC) VMR3ResetTripleFault(PVM pVM)
+{
+    LogFlow(("VMR3ResetTripleFault:\n"));
+
+    /*
+     * First consult the firmware on whether this is a hard or soft reset.
+     */
+    uint32_t fResetFlags;
+    bool fHardReset = PDMR3GetResetInfo(pVM, PDMVMRESET_F_TRIPLE_FAULT, &fResetFlags);
+    return vmR3ResetCommon(pVM, fHardReset, fResetFlags);
+}
+
+
+/**
+ * Gets the user mode VM structure pointer given Pointer to the VM.
+ *
+ * @returns Pointer to the user mode VM structure on success. NULL if @a pVM is
+ *          invalid (asserted).
+ * @param   pVM                 The cross context VM structure.
+ * @sa      VMR3GetVM, VMR3RetainUVM
+ */
+VMMR3DECL(PUVM) VMR3GetUVM(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, NULL);
+    return pVM->pUVM;
+}
+
+
+/**
+ * Gets the shared VM structure pointer given the pointer to the user mode VM
+ * structure.
+ *
+ * @returns Pointer to the VM.
+ *          NULL if @a pUVM is invalid (asserted) or if no shared VM structure
+ *          is currently associated with it.
+ * @param   pUVM                The user mode VM handle.
+ * @sa      VMR3GetUVM
+ */
+VMMR3DECL(PVM) VMR3GetVM(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+    return pUVM->pVM;
+}
+
+
+/**
+ * Retain the user mode VM handle.
+ *
+ * @returns Reference count.
+ *          UINT32_MAX if @a pUVM is invalid.
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @sa      VMR3ReleaseUVM
+ */
+VMMR3DECL(uint32_t) VMR3RetainUVM(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, UINT32_MAX);
+    uint32_t cRefs = ASMAtomicIncU32(&pUVM->vm.s.cUvmRefs);
+    AssertMsg(cRefs > 0 && cRefs < _64K, ("%u\n", cRefs));
+    return cRefs;
+}
+
+
+/**
+ * Does the final release of the UVM structure.
+ *
+ * @param   pUVM                The user mode VM handle.
+ */
+static void vmR3DoReleaseUVM(PUVM pUVM)
+{
+    /*
+     * Free the UVM.
+     */
+    Assert(!pUVM->pVM);
+
+    MMR3HeapFree(pUVM->vm.s.pszName);
+    pUVM->vm.s.pszName = NULL;
+
+    MMR3TermUVM(pUVM);
+    STAMR3TermUVM(pUVM);
+
+    ASMAtomicUoWriteU32(&pUVM->u32Magic, UINT32_MAX);
+    RTTlsFree(pUVM->vm.s.idxTLS);
+    RTMemPageFree(pUVM, RT_UOFFSETOF_DYN(UVM, aCpus[pUVM->cCpus]));
+}
+
+
+/**
+ * Releases a refernece to the mode VM handle.
+ *
+ * @returns The new reference count, 0 if destroyed.
+ *          UINT32_MAX if @a pUVM is invalid.
+ *
+ * @param   pUVM                The user mode VM handle.
+ * @sa      VMR3RetainUVM
+ */
+VMMR3DECL(uint32_t) VMR3ReleaseUVM(PUVM pUVM)
+{
+    if (!pUVM)
+        return 0;
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, UINT32_MAX);
+    uint32_t cRefs = ASMAtomicDecU32(&pUVM->vm.s.cUvmRefs);
+    if (!cRefs)
+        vmR3DoReleaseUVM(pUVM);
+    else
+        AssertMsg(cRefs < _64K, ("%u\n", cRefs));
+    return cRefs;
+}
+
+
+/**
+ * Gets the VM name.
+ *
+ * @returns Pointer to a read-only string containing the name. NULL if called
+ *          too early.
+ * @param   pUVM                The user mode VM handle.
+ */
+VMMR3DECL(const char *) VMR3GetName(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+    return pUVM->vm.s.pszName;
+}
+
+
+/**
+ * Gets the VM UUID.
+ *
+ * @returns pUuid on success, NULL on failure.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pUuid               Where to store the UUID.
+ */
+VMMR3DECL(PRTUUID) VMR3GetUuid(PUVM pUVM, PRTUUID pUuid)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+    AssertPtrReturn(pUuid, NULL);
+
+    *pUuid = pUVM->vm.s.Uuid;
+    return pUuid;
+}
+
+
+/**
+ * Gets the current VM state.
+ *
+ * @returns The current VM state.
+ * @param   pVM             The cross context VM structure.
+ * @thread  Any
+ */
+VMMR3DECL(VMSTATE) VMR3GetState(PVM pVM)
+{
+    AssertMsgReturn(RT_VALID_ALIGNED_PTR(pVM, PAGE_SIZE), ("%p\n", pVM), VMSTATE_TERMINATED);
+    VMSTATE enmVMState = pVM->enmVMState;
+    return enmVMState >= VMSTATE_CREATING && enmVMState <= VMSTATE_TERMINATED ? enmVMState : VMSTATE_TERMINATED;
+}
+
+
+/**
+ * Gets the current VM state.
+ *
+ * @returns The current VM state.
+ * @param   pUVM            The user-mode VM handle.
+ * @thread  Any
+ */
+VMMR3DECL(VMSTATE) VMR3GetStateU(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VMSTATE_TERMINATED);
+    if (RT_UNLIKELY(!pUVM->pVM))
+        return VMSTATE_TERMINATED;
+    return pUVM->pVM->enmVMState;
+}
+
+
+/**
+ * Gets the state name string for a VM state.
+ *
+ * @returns Pointer to the state name. (readonly)
+ * @param   enmState        The state.
+ */
+VMMR3DECL(const char *) VMR3GetStateName(VMSTATE enmState)
+{
+    switch (enmState)
+    {
+        case VMSTATE_CREATING:          return "CREATING";
+        case VMSTATE_CREATED:           return "CREATED";
+        case VMSTATE_LOADING:           return "LOADING";
+        case VMSTATE_POWERING_ON:       return "POWERING_ON";
+        case VMSTATE_RESUMING:          return "RESUMING";
+        case VMSTATE_RUNNING:           return "RUNNING";
+        case VMSTATE_RUNNING_LS:        return "RUNNING_LS";
+        case VMSTATE_RESETTING:         return "RESETTING";
+        case VMSTATE_RESETTING_LS:      return "RESETTING_LS";
+        case VMSTATE_SOFT_RESETTING:    return "SOFT_RESETTING";
+        case VMSTATE_SOFT_RESETTING_LS: return "SOFT_RESETTING_LS";
+        case VMSTATE_SUSPENDED:         return "SUSPENDED";
+        case VMSTATE_SUSPENDED_LS:      return "SUSPENDED_LS";
+        case VMSTATE_SUSPENDED_EXT_LS:  return "SUSPENDED_EXT_LS";
+        case VMSTATE_SUSPENDING:        return "SUSPENDING";
+        case VMSTATE_SUSPENDING_LS:     return "SUSPENDING_LS";
+        case VMSTATE_SUSPENDING_EXT_LS: return "SUSPENDING_EXT_LS";
+        case VMSTATE_SAVING:            return "SAVING";
+        case VMSTATE_DEBUGGING:         return "DEBUGGING";
+        case VMSTATE_DEBUGGING_LS:      return "DEBUGGING_LS";
+        case VMSTATE_POWERING_OFF:      return "POWERING_OFF";
+        case VMSTATE_POWERING_OFF_LS:   return "POWERING_OFF_LS";
+        case VMSTATE_FATAL_ERROR:       return "FATAL_ERROR";
+        case VMSTATE_FATAL_ERROR_LS:    return "FATAL_ERROR_LS";
+        case VMSTATE_GURU_MEDITATION:   return "GURU_MEDITATION";
+        case VMSTATE_GURU_MEDITATION_LS:return "GURU_MEDITATION_LS";
+        case VMSTATE_LOAD_FAILURE:      return "LOAD_FAILURE";
+        case VMSTATE_OFF:               return "OFF";
+        case VMSTATE_OFF_LS:            return "OFF_LS";
+        case VMSTATE_DESTROYING:        return "DESTROYING";
+        case VMSTATE_TERMINATED:        return "TERMINATED";
+
+        default:
+            AssertMsgFailed(("Unknown state %d\n", enmState));
+            return "Unknown!\n";
+    }
+}
+
+
+/**
+ * Validates the state transition in strict builds.
+ *
+ * @returns true if valid, false if not.
+ *
+ * @param   enmStateOld         The old (current) state.
+ * @param   enmStateNew         The proposed new state.
+ *
+ * @remarks The reference for this is found in doc/vp/VMM.vpp, the VMSTATE
+ *          diagram (under State Machine Diagram).
+ */
+static bool vmR3ValidateStateTransition(VMSTATE enmStateOld, VMSTATE enmStateNew)
+{
+#ifndef VBOX_STRICT
+    RT_NOREF2(enmStateOld, enmStateNew);
+#else
+    switch (enmStateOld)
+    {
+        case VMSTATE_CREATING:
+            AssertMsgReturn(enmStateNew == VMSTATE_CREATED, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_CREATED:
+            AssertMsgReturn(   enmStateNew == VMSTATE_LOADING
+                            || enmStateNew == VMSTATE_POWERING_ON
+                            || enmStateNew == VMSTATE_POWERING_OFF
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_LOADING:
+            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDED
+                            || enmStateNew == VMSTATE_LOAD_FAILURE
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_POWERING_ON:
+            AssertMsgReturn(   enmStateNew == VMSTATE_RUNNING
+                            /*|| enmStateNew == VMSTATE_FATAL_ERROR ?*/
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_RESUMING:
+            AssertMsgReturn(   enmStateNew == VMSTATE_RUNNING
+                            /*|| enmStateNew == VMSTATE_FATAL_ERROR ?*/
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_RUNNING:
+            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF
+                            || enmStateNew == VMSTATE_SUSPENDING
+                            || enmStateNew == VMSTATE_RESETTING
+                            || enmStateNew == VMSTATE_SOFT_RESETTING
+                            || enmStateNew == VMSTATE_RUNNING_LS
+                            || enmStateNew == VMSTATE_DEBUGGING
+                            || enmStateNew == VMSTATE_FATAL_ERROR
+                            || enmStateNew == VMSTATE_GURU_MEDITATION
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_RUNNING_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF_LS
+                            || enmStateNew == VMSTATE_SUSPENDING_LS
+                            || enmStateNew == VMSTATE_SUSPENDING_EXT_LS
+                            || enmStateNew == VMSTATE_RESETTING_LS
+                            || enmStateNew == VMSTATE_SOFT_RESETTING_LS
+                            || enmStateNew == VMSTATE_RUNNING
+                            || enmStateNew == VMSTATE_DEBUGGING_LS
+                            || enmStateNew == VMSTATE_FATAL_ERROR_LS
+                            || enmStateNew == VMSTATE_GURU_MEDITATION_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_RESETTING:
+            AssertMsgReturn(enmStateNew == VMSTATE_RUNNING, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SOFT_RESETTING:
+            AssertMsgReturn(enmStateNew == VMSTATE_RUNNING, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_RESETTING_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDING_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SOFT_RESETTING_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_RUNNING_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SUSPENDING:
+            AssertMsgReturn(enmStateNew == VMSTATE_SUSPENDED, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SUSPENDING_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDING
+                            || enmStateNew == VMSTATE_SUSPENDED_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SUSPENDING_EXT_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDING
+                            || enmStateNew == VMSTATE_SUSPENDED_EXT_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SUSPENDED:
+            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF
+                            || enmStateNew == VMSTATE_SAVING
+                            || enmStateNew == VMSTATE_RESETTING
+                            || enmStateNew == VMSTATE_SOFT_RESETTING
+                            || enmStateNew == VMSTATE_RESUMING
+                            || enmStateNew == VMSTATE_LOADING
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SUSPENDED_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDED
+                            || enmStateNew == VMSTATE_SAVING
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SUSPENDED_EXT_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDED
+                            || enmStateNew == VMSTATE_SAVING
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SAVING:
+            AssertMsgReturn(enmStateNew == VMSTATE_SUSPENDED, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_DEBUGGING:
+            AssertMsgReturn(   enmStateNew == VMSTATE_RUNNING
+                            || enmStateNew == VMSTATE_POWERING_OFF
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_DEBUGGING_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_DEBUGGING
+                            || enmStateNew == VMSTATE_RUNNING_LS
+                            || enmStateNew == VMSTATE_POWERING_OFF_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_POWERING_OFF:
+            AssertMsgReturn(enmStateNew == VMSTATE_OFF, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_POWERING_OFF_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF
+                            || enmStateNew == VMSTATE_OFF_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_OFF:
+            AssertMsgReturn(enmStateNew == VMSTATE_DESTROYING, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_OFF_LS:
+            AssertMsgReturn(enmStateNew == VMSTATE_OFF, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_FATAL_ERROR:
+            AssertMsgReturn(enmStateNew == VMSTATE_POWERING_OFF, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_FATAL_ERROR_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_FATAL_ERROR
+                            || enmStateNew == VMSTATE_POWERING_OFF_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_GURU_MEDITATION:
+            AssertMsgReturn(   enmStateNew == VMSTATE_DEBUGGING
+                            || enmStateNew == VMSTATE_POWERING_OFF
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_GURU_MEDITATION_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_GURU_MEDITATION
+                            || enmStateNew == VMSTATE_DEBUGGING_LS
+                            || enmStateNew == VMSTATE_POWERING_OFF_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_LOAD_FAILURE:
+            AssertMsgReturn(enmStateNew == VMSTATE_POWERING_OFF, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_DESTROYING:
+            AssertMsgReturn(enmStateNew == VMSTATE_TERMINATED, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_TERMINATED:
+        default:
+            AssertMsgFailedReturn(("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+    }
+#endif /* VBOX_STRICT */
+    return true;
+}
+
+
+/**
+ * Does the state change callouts.
+ *
+ * The caller owns the AtStateCritSect.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pUVM                The UVM handle.
+ * @param   enmStateNew         The New state.
+ * @param   enmStateOld         The old state.
+ */
+static void vmR3DoAtState(PVM pVM, PUVM pUVM, VMSTATE enmStateNew, VMSTATE enmStateOld)
+{
+    LogRel(("Changing the VM state from '%s' to '%s'\n", VMR3GetStateName(enmStateOld),  VMR3GetStateName(enmStateNew)));
+
+    for (PVMATSTATE pCur = pUVM->vm.s.pAtState; pCur; pCur = pCur->pNext)
+    {
+        pCur->pfnAtState(pUVM, enmStateNew, enmStateOld, pCur->pvUser);
+        if (    enmStateNew     != VMSTATE_DESTROYING
+            &&  pVM->enmVMState == VMSTATE_DESTROYING)
+            break;
+        AssertMsg(pVM->enmVMState == enmStateNew,
+                  ("You are not allowed to change the state while in the change callback, except "
+                   "from destroying the VM. There are restrictions in the way the state changes "
+                   "are propagated up to the EM execution loop and it makes the program flow very "
+                   "difficult to follow. (%s, expected %s, old %s)\n",
+                   VMR3GetStateName(pVM->enmVMState), VMR3GetStateName(enmStateNew),
+                   VMR3GetStateName(enmStateOld)));
+    }
+}
+
+
+/**
+ * Sets the current VM state, with the AtStatCritSect already entered.
+ *
+ * @param   pVM                     The cross context VM structure.
+ * @param   pUVM                    The UVM handle.
+ * @param   enmStateNew             The new state.
+ * @param   enmStateOld             The old state.
+ * @param   fSetRatherThanClearFF   The usual behavior is to clear the
+ *                                  VM_FF_CHECK_VM_STATE force flag, but for
+ *                                  some transitions (-> guru) we need to kick
+ *                                  the other EMTs to stop what they're doing.
+ */
+static void vmR3SetStateLocked(PVM pVM, PUVM pUVM, VMSTATE enmStateNew, VMSTATE enmStateOld, bool fSetRatherThanClearFF)
+{
+    vmR3ValidateStateTransition(enmStateOld, enmStateNew);
+
+    AssertMsg(pVM->enmVMState == enmStateOld,
+              ("%s != %s\n", VMR3GetStateName(pVM->enmVMState), VMR3GetStateName(enmStateOld)));
+
+    pUVM->vm.s.enmPrevVMState = enmStateOld;
+    pVM->enmVMState           = enmStateNew;
+
+    if (!fSetRatherThanClearFF)
+        VM_FF_CLEAR(pVM, VM_FF_CHECK_VM_STATE);
+    else if (pVM->cCpus > 0)
+        VM_FF_SET(pVM, VM_FF_CHECK_VM_STATE);
+
+    vmR3DoAtState(pVM, pUVM, enmStateNew, enmStateOld);
+}
+
+
+/**
+ * Sets the current VM state.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   enmStateNew     The new state.
+ * @param   enmStateOld     The old state (for asserting only).
+ */
+static void vmR3SetState(PVM pVM, VMSTATE enmStateNew, VMSTATE enmStateOld)
+{
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+
+    RT_NOREF_PV(enmStateOld);
+    AssertMsg(pVM->enmVMState == enmStateOld,
+              ("%s != %s\n", VMR3GetStateName(pVM->enmVMState), VMR3GetStateName(enmStateOld)));
+    vmR3SetStateLocked(pVM, pUVM, enmStateNew, pVM->enmVMState, false /*fSetRatherThanClearFF*/);
+
+    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+}
+
+
+/**
+ * Tries to perform a state transition.
+ *
+ * @returns The 1-based ordinal of the succeeding transition.
+ *          VERR_VM_INVALID_VM_STATE and Assert+LogRel on failure.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pszWho              Who is trying to change it.
+ * @param   cTransitions        The number of transitions in the ellipsis.
+ * @param   ...                 Transition pairs; new, old.
+ */
+static int vmR3TrySetState(PVM pVM, const char *pszWho, unsigned cTransitions, ...)
+{
+    va_list va;
+    VMSTATE enmStateNew = VMSTATE_CREATED;
+    VMSTATE enmStateOld = VMSTATE_CREATED;
+
+#ifdef VBOX_STRICT
+    /*
+     * Validate the input first.
+     */
+    va_start(va, cTransitions);
+    for (unsigned i = 0; i < cTransitions; i++)
+    {
+        enmStateNew = (VMSTATE)va_arg(va, /*VMSTATE*/int);
+        enmStateOld = (VMSTATE)va_arg(va, /*VMSTATE*/int);
+        vmR3ValidateStateTransition(enmStateOld, enmStateNew);
+    }
+    va_end(va);
+#endif
+
+    /*
+     * Grab the lock and see if any of the proposed transitions works out.
+     */
+    va_start(va, cTransitions);
+    int     rc          = VERR_VM_INVALID_VM_STATE;
+    PUVM    pUVM        = pVM->pUVM;
+    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+
+    VMSTATE enmStateCur = pVM->enmVMState;
+
+    for (unsigned i = 0; i < cTransitions; i++)
+    {
+        enmStateNew = (VMSTATE)va_arg(va, /*VMSTATE*/int);
+        enmStateOld = (VMSTATE)va_arg(va, /*VMSTATE*/int);
+        if (enmStateCur == enmStateOld)
+        {
+            vmR3SetStateLocked(pVM, pUVM, enmStateNew, enmStateOld, false /*fSetRatherThanClearFF*/);
+            rc = i + 1;
+            break;
+        }
+    }
+
+    if (RT_FAILURE(rc))
+    {
+        /*
+         * Complain about it.
+         */
+        if (cTransitions == 1)
+        {
+            LogRel(("%s: %s -> %s failed, because the VM state is actually %s\n",
+                    pszWho, VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew), VMR3GetStateName(enmStateCur)));
+            VMSetError(pVM, VERR_VM_INVALID_VM_STATE, RT_SRC_POS,
+                       N_("%s failed because the VM state is %s instead of %s"),
+                       pszWho, VMR3GetStateName(enmStateCur), VMR3GetStateName(enmStateOld));
+            AssertMsgFailed(("%s: %s -> %s failed, because the VM state is actually %s\n",
+                             pszWho, VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew), VMR3GetStateName(enmStateCur)));
+        }
+        else
+        {
+            va_end(va);
+            va_start(va, cTransitions);
+            LogRel(("%s:\n", pszWho));
+            for (unsigned i = 0; i < cTransitions; i++)
+            {
+                enmStateNew = (VMSTATE)va_arg(va, /*VMSTATE*/int);
+                enmStateOld = (VMSTATE)va_arg(va, /*VMSTATE*/int);
+                LogRel(("%s%s -> %s",
+                        i ? ", " : " ", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)));
+            }
+            LogRel((" failed, because the VM state is actually %s\n", VMR3GetStateName(enmStateCur)));
+            VMSetError(pVM, VERR_VM_INVALID_VM_STATE, RT_SRC_POS,
+                       N_("%s failed because the current VM state, %s, was not found in the state transition table (old state %s)"),
+                       pszWho, VMR3GetStateName(enmStateCur), VMR3GetStateName(enmStateOld));
+            AssertMsgFailed(("%s - state=%s, see release log for full details. Check the cTransitions passed us.\n",
+                             pszWho, VMR3GetStateName(enmStateCur)));
+        }
+    }
+
+    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+    va_end(va);
+    Assert(rc > 0 || rc < 0);
+    return rc;
+}
+
+
+/**
+ * Interface used by EM to signal that it's entering the guru meditation state.
+ *
+ * This will notifying other threads.
+ *
+ * @returns true if the state changed to Guru, false if no state change.
+ * @param   pVM             The cross context VM structure.
+ */
+VMMR3_INT_DECL(bool) VMR3SetGuruMeditation(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+
+    VMSTATE enmStateCur = pVM->enmVMState;
+    bool fRc = true;
+    if (enmStateCur == VMSTATE_RUNNING)
+        vmR3SetStateLocked(pVM, pUVM, VMSTATE_GURU_MEDITATION, VMSTATE_RUNNING, true /*fSetRatherThanClearFF*/);
+    else if (enmStateCur == VMSTATE_RUNNING_LS)
+    {
+        vmR3SetStateLocked(pVM, pUVM, VMSTATE_GURU_MEDITATION_LS, VMSTATE_RUNNING_LS, true /*fSetRatherThanClearFF*/);
+        SSMR3Cancel(pUVM);
+    }
+    else
+        fRc = false;
+
+    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+    return fRc;
+}
+
+
+/**
+ * Called by vmR3EmulationThreadWithId just before the VM structure is freed.
+ *
+ * @param   pVM             The cross context VM structure.
+ */
+void vmR3SetTerminated(PVM pVM)
+{
+    vmR3SetState(pVM, VMSTATE_TERMINATED, VMSTATE_DESTROYING);
+}
+
+
+/**
+ * Checks if the VM was teleported and hasn't been fully resumed yet.
+ *
+ * This applies to both sides of the teleportation since we may leave a working
+ * clone behind and the user is allowed to resume this...
+ *
+ * @returns true / false.
+ * @param   pVM                 The cross context VM structure.
+ * @thread  Any thread.
+ */
+VMMR3_INT_DECL(bool) VMR3TeleportedAndNotFullyResumedYet(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->vm.s.fTeleportedAndNotFullyResumedYet;
+}
+
+
+/**
+ * Registers a VM state change callback.
+ *
+ * You are not allowed to call any function which changes the VM state from a
+ * state callback.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The VM handle.
+ * @param   pfnAtState      Pointer to callback.
+ * @param   pvUser          User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int) VMR3AtStateRegister(PUVM pUVM, PFNVMATSTATE pfnAtState, void *pvUser)
+{
+    LogFlow(("VMR3AtStateRegister: pfnAtState=%p pvUser=%p\n", pfnAtState, pvUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnAtState, VERR_INVALID_PARAMETER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Allocate a new record.
+     */
+    PVMATSTATE pNew = (PVMATSTATE)MMR3HeapAllocU(pUVM, MM_TAG_VM, sizeof(*pNew));
+    if (!pNew)
+        return VERR_NO_MEMORY;
+
+    /* fill */
+    pNew->pfnAtState = pfnAtState;
+    pNew->pvUser     = pvUser;
+
+    /* insert */
+    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+    pNew->pNext      = *pUVM->vm.s.ppAtStateNext;
+    *pUVM->vm.s.ppAtStateNext = pNew;
+    pUVM->vm.s.ppAtStateNext = &pNew->pNext;
+    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters a VM state change callback.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The VM handle.
+ * @param   pfnAtState      Pointer to callback.
+ * @param   pvUser          User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int) VMR3AtStateDeregister(PUVM pUVM, PFNVMATSTATE pfnAtState, void *pvUser)
+{
+    LogFlow(("VMR3AtStateDeregister: pfnAtState=%p pvUser=%p\n", pfnAtState, pvUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnAtState, VERR_INVALID_PARAMETER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+
+    /*
+     * Search the list for the entry.
+     */
+    PVMATSTATE pPrev = NULL;
+    PVMATSTATE pCur = pUVM->vm.s.pAtState;
+    while (     pCur
+           &&   (   pCur->pfnAtState != pfnAtState
+                 || pCur->pvUser != pvUser))
+    {
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    if (!pCur)
+    {
+        AssertMsgFailed(("pfnAtState=%p was not found\n", pfnAtState));
+        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+        return VERR_FILE_NOT_FOUND;
+    }
+
+    /*
+     * Unlink it.
+     */
+    if (pPrev)
+    {
+        pPrev->pNext = pCur->pNext;
+        if (!pCur->pNext)
+            pUVM->vm.s.ppAtStateNext = &pPrev->pNext;
+    }
+    else
+    {
+        pUVM->vm.s.pAtState = pCur->pNext;
+        if (!pCur->pNext)
+            pUVM->vm.s.ppAtStateNext = &pUVM->vm.s.pAtState;
+    }
+
+    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+
+    /*
+     * Free it.
+     */
+    pCur->pfnAtState = NULL;
+    pCur->pNext = NULL;
+    MMR3HeapFree(pCur);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Registers a VM error callback.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The VM handle.
+ * @param   pfnAtError      Pointer to callback.
+ * @param   pvUser          User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int)  VMR3AtErrorRegister(PUVM pUVM, PFNVMATERROR pfnAtError, void *pvUser)
+{
+    LogFlow(("VMR3AtErrorRegister: pfnAtError=%p pvUser=%p\n", pfnAtError, pvUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnAtError, VERR_INVALID_PARAMETER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Allocate a new record.
+     */
+    PVMATERROR pNew = (PVMATERROR)MMR3HeapAllocU(pUVM, MM_TAG_VM, sizeof(*pNew));
+    if (!pNew)
+        return VERR_NO_MEMORY;
+
+    /* fill */
+    pNew->pfnAtError = pfnAtError;
+    pNew->pvUser     = pvUser;
+
+    /* insert */
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+    pNew->pNext      = *pUVM->vm.s.ppAtErrorNext;
+    *pUVM->vm.s.ppAtErrorNext = pNew;
+    pUVM->vm.s.ppAtErrorNext = &pNew->pNext;
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters a VM error callback.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The VM handle.
+ * @param   pfnAtError      Pointer to callback.
+ * @param   pvUser          User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int) VMR3AtErrorDeregister(PUVM pUVM, PFNVMATERROR pfnAtError, void *pvUser)
+{
+    LogFlow(("VMR3AtErrorDeregister: pfnAtError=%p pvUser=%p\n", pfnAtError, pvUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnAtError, VERR_INVALID_PARAMETER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+
+    /*
+     * Search the list for the entry.
+     */
+    PVMATERROR pPrev = NULL;
+    PVMATERROR pCur = pUVM->vm.s.pAtError;
+    while (     pCur
+           &&   (   pCur->pfnAtError != pfnAtError
+                 || pCur->pvUser != pvUser))
+    {
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    if (!pCur)
+    {
+        AssertMsgFailed(("pfnAtError=%p was not found\n", pfnAtError));
+        RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+        return VERR_FILE_NOT_FOUND;
+    }
+
+    /*
+     * Unlink it.
+     */
+    if (pPrev)
+    {
+        pPrev->pNext = pCur->pNext;
+        if (!pCur->pNext)
+            pUVM->vm.s.ppAtErrorNext = &pPrev->pNext;
+    }
+    else
+    {
+        pUVM->vm.s.pAtError = pCur->pNext;
+        if (!pCur->pNext)
+            pUVM->vm.s.ppAtErrorNext = &pUVM->vm.s.pAtError;
+    }
+
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+
+    /*
+     * Free it.
+     */
+    pCur->pfnAtError = NULL;
+    pCur->pNext = NULL;
+    MMR3HeapFree(pCur);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Ellipsis to va_list wrapper for calling pfnAtError.
+ */
+static void vmR3SetErrorWorkerDoCall(PVM pVM, PVMATERROR pCur, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    pCur->pfnAtError(pVM->pUVM, pCur->pvUser, rc, RT_SRC_POS_ARGS, pszFormat, va);
+    va_end(va);
+}
+
+
+/**
+ * This is a worker function for GC and Ring-0 calls to VMSetError and VMSetErrorV.
+ * The message is found in VMINT.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @thread  EMT.
+ */
+VMMR3_INT_DECL(void) VMR3SetErrorWorker(PVM pVM)
+{
+    VM_ASSERT_EMT(pVM);
+    AssertReleaseMsgFailed(("And we have a winner! You get to implement Ring-0 and GC VMSetErrorV! Congrats!\n"));
+
+    /*
+     * Unpack the error (if we managed to format one).
+     */
+    PVMERROR pErr = pVM->vm.s.pErrorR3;
+    const char *pszFile = NULL;
+    const char *pszFunction = NULL;
+    uint32_t    iLine = 0;
+    const char *pszMessage;
+    int32_t     rc = VERR_MM_HYPER_NO_MEMORY;
+    if (pErr)
+    {
+        AssertCompile(sizeof(const char) == sizeof(uint8_t));
+        if (pErr->offFile)
+            pszFile = (const char *)pErr + pErr->offFile;
+        iLine = pErr->iLine;
+        if (pErr->offFunction)
+            pszFunction = (const char *)pErr + pErr->offFunction;
+        if (pErr->offMessage)
+            pszMessage = (const char *)pErr + pErr->offMessage;
+        else
+            pszMessage = "No message!";
+    }
+    else
+        pszMessage = "No message! (Failed to allocate memory to put the error message in!)";
+
+    /*
+     * Call the at error callbacks.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+    ASMAtomicIncU32(&pUVM->vm.s.cRuntimeErrors);
+    for (PVMATERROR pCur = pUVM->vm.s.pAtError; pCur; pCur = pCur->pNext)
+        vmR3SetErrorWorkerDoCall(pVM, pCur, rc, RT_SRC_POS_ARGS, "%s", pszMessage);
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+}
+
+
+/**
+ * Gets the number of errors raised via VMSetError.
+ *
+ * This can be used avoid double error messages.
+ *
+ * @returns The error count.
+ * @param   pUVM            The VM handle.
+ */
+VMMR3_INT_DECL(uint32_t) VMR3GetErrorCount(PUVM pUVM)
+{
+    AssertPtrReturn(pUVM, 0);
+    AssertReturn(pUVM->u32Magic == UVM_MAGIC, 0);
+    return pUVM->vm.s.cErrors;
+}
+
+
+/**
+ * Creation time wrapper for vmR3SetErrorUV.
+ *
+ * @returns rc.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   rc              The VBox status code.
+ * @param   SRC_POS         The source position of this error.
+ * @param   pszFormat       Format string.
+ * @param   ...             The arguments.
+ * @thread  Any thread.
+ */
+static int vmR3SetErrorU(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    vmR3SetErrorUV(pUVM, rc, pszFile, iLine, pszFunction, pszFormat, &va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Worker which calls everyone listening to the VM error messages.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   rc              The VBox status code.
+ * @param   SRC_POS         The source position of this error.
+ * @param   pszFormat       Format string.
+ * @param   pArgs           Pointer to the format arguments.
+ * @thread  EMT
+ */
+DECLCALLBACK(void) vmR3SetErrorUV(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list *pArgs)
+{
+    /*
+     * Log the error.
+     */
+    va_list va3;
+    va_copy(va3, *pArgs);
+    RTLogRelPrintf("VMSetError: %s(%d) %s; rc=%Rrc\n"
+                   "VMSetError: %N\n",
+                   pszFile, iLine, pszFunction, rc,
+                   pszFormat, &va3);
+    va_end(va3);
+
+#ifdef LOG_ENABLED
+    va_copy(va3, *pArgs);
+    RTLogPrintf("VMSetError: %s(%d) %s; rc=%Rrc\n"
+                "%N\n",
+                pszFile, iLine, pszFunction, rc,
+                pszFormat, &va3);
+    va_end(va3);
+#endif
+
+    /*
+     * Make a copy of the message.
+     */
+    if (pUVM->pVM)
+        vmSetErrorCopy(pUVM->pVM, rc, RT_SRC_POS_ARGS, pszFormat, *pArgs);
+
+    /*
+     * Call the at error callbacks.
+     */
+    bool fCalledSomeone = false;
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+    ASMAtomicIncU32(&pUVM->vm.s.cErrors);
+    for (PVMATERROR pCur = pUVM->vm.s.pAtError; pCur; pCur = pCur->pNext)
+    {
+        va_list va2;
+        va_copy(va2, *pArgs);
+        pCur->pfnAtError(pUVM, pCur->pvUser, rc, RT_SRC_POS_ARGS, pszFormat, va2);
+        va_end(va2);
+        fCalledSomeone = true;
+    }
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+}
+
+
+/**
+ * Sets the error message.
+ *
+ * @returns rc. Meaning you can do:
+ *    @code
+ *    return VM_SET_ERROR_U(pUVM, VERR_OF_YOUR_CHOICE, "descriptive message");
+ *    @endcode
+ * @param   pUVM            The user mode VM handle.
+ * @param   rc              VBox status code.
+ * @param   SRC_POS         Use RT_SRC_POS.
+ * @param   pszFormat       Error message format string.
+ * @param   ...             Error message arguments.
+ * @thread  Any
+ */
+VMMR3DECL(int) VMR3SetError(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    int rcRet = VMR3SetErrorV(pUVM, rc, pszFile, iLine, pszFunction, pszFormat, va);
+    va_end(va);
+    return rcRet;
+}
+
+
+/**
+ * Sets the error message.
+ *
+ * @returns rc. Meaning you can do:
+ *    @code
+ *    return VM_SET_ERROR_U(pUVM, VERR_OF_YOUR_CHOICE, "descriptive message");
+ *    @endcode
+ * @param   pUVM            The user mode VM handle.
+ * @param   rc              VBox status code.
+ * @param   SRC_POS         Use RT_SRC_POS.
+ * @param   pszFormat       Error message format string.
+ * @param   va              Error message arguments.
+ * @thread  Any
+ */
+VMMR3DECL(int) VMR3SetErrorV(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /* Take shortcut when called on EMT, skipping VM handle requirement + validation. */
+    if (VMR3GetVMCPUThread(pUVM) != NIL_RTTHREAD)
+    {
+        va_list vaCopy;
+        va_copy(vaCopy, va);
+        vmR3SetErrorUV(pUVM, rc, RT_SRC_POS_ARGS, pszFormat, &vaCopy);
+        va_end(vaCopy);
+        return rc;
+    }
+
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
+    return VMSetErrorV(pUVM->pVM, rc, pszFile, iLine, pszFunction, pszFormat, va);
+}
+
+
+
+/**
+ * Registers a VM runtime error callback.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM structure.
+ * @param   pfnAtRuntimeError   Pointer to callback.
+ * @param   pvUser              User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int)   VMR3AtRuntimeErrorRegister(PUVM pUVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser)
+{
+    LogFlow(("VMR3AtRuntimeErrorRegister: pfnAtRuntimeError=%p pvUser=%p\n", pfnAtRuntimeError, pvUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnAtRuntimeError, VERR_INVALID_PARAMETER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Allocate a new record.
+     */
+    PVMATRUNTIMEERROR pNew = (PVMATRUNTIMEERROR)MMR3HeapAllocU(pUVM, MM_TAG_VM, sizeof(*pNew));
+    if (!pNew)
+        return VERR_NO_MEMORY;
+
+    /* fill */
+    pNew->pfnAtRuntimeError = pfnAtRuntimeError;
+    pNew->pvUser            = pvUser;
+
+    /* insert */
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+    pNew->pNext             = *pUVM->vm.s.ppAtRuntimeErrorNext;
+    *pUVM->vm.s.ppAtRuntimeErrorNext = pNew;
+    pUVM->vm.s.ppAtRuntimeErrorNext = &pNew->pNext;
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters a VM runtime error callback.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   pfnAtRuntimeError   Pointer to callback.
+ * @param   pvUser              User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int) VMR3AtRuntimeErrorDeregister(PUVM pUVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser)
+{
+    LogFlow(("VMR3AtRuntimeErrorDeregister: pfnAtRuntimeError=%p pvUser=%p\n", pfnAtRuntimeError, pvUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnAtRuntimeError, VERR_INVALID_PARAMETER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+
+    /*
+     * Search the list for the entry.
+     */
+    PVMATRUNTIMEERROR pPrev = NULL;
+    PVMATRUNTIMEERROR pCur = pUVM->vm.s.pAtRuntimeError;
+    while (     pCur
+           &&   (   pCur->pfnAtRuntimeError != pfnAtRuntimeError
+                 || pCur->pvUser != pvUser))
+    {
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    if (!pCur)
+    {
+        AssertMsgFailed(("pfnAtRuntimeError=%p was not found\n", pfnAtRuntimeError));
+        RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+        return VERR_FILE_NOT_FOUND;
+    }
+
+    /*
+     * Unlink it.
+     */
+    if (pPrev)
+    {
+        pPrev->pNext = pCur->pNext;
+        if (!pCur->pNext)
+            pUVM->vm.s.ppAtRuntimeErrorNext = &pPrev->pNext;
+    }
+    else
+    {
+        pUVM->vm.s.pAtRuntimeError = pCur->pNext;
+        if (!pCur->pNext)
+            pUVM->vm.s.ppAtRuntimeErrorNext = &pUVM->vm.s.pAtRuntimeError;
+    }
+
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+
+    /*
+     * Free it.
+     */
+    pCur->pfnAtRuntimeError = NULL;
+    pCur->pNext = NULL;
+    MMR3HeapFree(pCur);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * EMT rendezvous worker that vmR3SetRuntimeErrorCommon uses to safely change
+ * the state to FatalError(LS).
+ *
+ * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_SUSPEND.  (This is a strict
+ *          return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser          Ignored.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3SetRuntimeErrorChangeState(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    NOREF(pVCpu);
+    Assert(!pvUser); NOREF(pvUser);
+
+    /*
+     * The first EMT thru here changes the state.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "VMSetRuntimeError", 2,
+                                 VMSTATE_FATAL_ERROR,    VMSTATE_RUNNING,
+                                 VMSTATE_FATAL_ERROR_LS, VMSTATE_RUNNING_LS);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (rc == 2)
+            SSMR3Cancel(pVM->pUVM);
+
+        VM_FF_SET(pVM, VM_FF_CHECK_VM_STATE);
+    }
+
+    /* This'll make sure we get out of whereever we are (e.g. REM). */
+    return VINF_EM_SUSPEND;
+}
+
+
+/**
+ * Worker for VMR3SetRuntimeErrorWorker and vmR3SetRuntimeErrorV.
+ *
+ * This does the common parts after the error has been saved / retrieved.
+ *
+ * @returns VBox status code with modifications, see VMSetRuntimeErrorV.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fFlags          The error flags.
+ * @param   pszErrorId      Error ID string.
+ * @param   pszFormat       Format string.
+ * @param   pVa             Pointer to the format arguments.
+ */
+static int vmR3SetRuntimeErrorCommon(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list *pVa)
+{
+    LogRel(("VM: Raising runtime error '%s' (fFlags=%#x)\n", pszErrorId, fFlags));
+    PUVM pUVM = pVM->pUVM;
+
+    /*
+     * Take actions before the call.
+     */
+    int rc;
+    if (fFlags & VMSETRTERR_FLAGS_FATAL)
+        rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3SetRuntimeErrorChangeState, NULL);
+    else if (fFlags & VMSETRTERR_FLAGS_SUSPEND)
+        rc = VMR3Suspend(pUVM, VMSUSPENDREASON_RUNTIME_ERROR);
+    else
+        rc = VINF_SUCCESS;
+
+    /*
+     * Do the callback round.
+     */
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+    ASMAtomicIncU32(&pUVM->vm.s.cRuntimeErrors);
+    for (PVMATRUNTIMEERROR pCur = pUVM->vm.s.pAtRuntimeError; pCur; pCur = pCur->pNext)
+    {
+        va_list va;
+        va_copy(va, *pVa);
+        pCur->pfnAtRuntimeError(pUVM, pCur->pvUser, fFlags, pszErrorId, pszFormat, va);
+        va_end(va);
+    }
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+
+    return rc;
+}
+
+
+/**
+ * Ellipsis to va_list wrapper for calling vmR3SetRuntimeErrorCommon.
+ */
+static int vmR3SetRuntimeErrorCommonF(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    int rc = vmR3SetRuntimeErrorCommon(pVM, fFlags, pszErrorId, pszFormat, &va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * This is a worker function for RC and Ring-0 calls to VMSetError and
+ * VMSetErrorV.
+ *
+ * The message is found in VMINT.
+ *
+ * @returns VBox status code, see VMSetRuntimeError.
+ * @param   pVM             The cross context VM structure.
+ * @thread  EMT.
+ */
+VMMR3_INT_DECL(int) VMR3SetRuntimeErrorWorker(PVM pVM)
+{
+    VM_ASSERT_EMT(pVM);
+    AssertReleaseMsgFailed(("And we have a winner! You get to implement Ring-0 and GC VMSetRuntimeErrorV! Congrats!\n"));
+
+    /*
+     * Unpack the error (if we managed to format one).
+     */
+    const char     *pszErrorId = "SetRuntimeError";
+    const char     *pszMessage = "No message!";
+    uint32_t        fFlags     = VMSETRTERR_FLAGS_FATAL;
+    PVMRUNTIMEERROR pErr       = pVM->vm.s.pRuntimeErrorR3;
+    if (pErr)
+    {
+        AssertCompile(sizeof(const char) == sizeof(uint8_t));
+        if (pErr->offErrorId)
+            pszErrorId = (const char *)pErr + pErr->offErrorId;
+        if (pErr->offMessage)
+            pszMessage = (const char *)pErr + pErr->offMessage;
+        fFlags = pErr->fFlags;
+    }
+
+    /*
+     * Join cause with vmR3SetRuntimeErrorV.
+     */
+    return vmR3SetRuntimeErrorCommonF(pVM, fFlags, pszErrorId, "%s", pszMessage);
+}
+
+
+/**
+ * Worker for VMSetRuntimeErrorV for doing the job on EMT in ring-3.
+ *
+ * @returns VBox status code with modifications, see VMSetRuntimeErrorV.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fFlags          The error flags.
+ * @param   pszErrorId      Error ID string.
+ * @param   pszMessage      The error message residing the MM heap.
+ *
+ * @thread  EMT
+ */
+DECLCALLBACK(int) vmR3SetRuntimeError(PVM pVM, uint32_t fFlags, const char *pszErrorId, char *pszMessage)
+{
+#if 0 /** @todo make copy of the error msg. */
+    /*
+     * Make a copy of the message.
+     */
+    va_list va2;
+    va_copy(va2, *pVa);
+    vmSetRuntimeErrorCopy(pVM, fFlags, pszErrorId, pszFormat, va2);
+    va_end(va2);
+#endif
+
+    /*
+     * Join paths with VMR3SetRuntimeErrorWorker.
+     */
+    int rc = vmR3SetRuntimeErrorCommonF(pVM, fFlags, pszErrorId, "%s", pszMessage);
+    MMR3HeapFree(pszMessage);
+    return rc;
+}
+
+
+/**
+ * Worker for VMSetRuntimeErrorV for doing the job on EMT in ring-3.
+ *
+ * @returns VBox status code with modifications, see VMSetRuntimeErrorV.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fFlags          The error flags.
+ * @param   pszErrorId      Error ID string.
+ * @param   pszFormat       Format string.
+ * @param   pVa             Pointer to the format arguments.
+ *
+ * @thread  EMT
+ */
+DECLCALLBACK(int) vmR3SetRuntimeErrorV(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list *pVa)
+{
+    /*
+     * Make a copy of the message.
+     */
+    va_list va2;
+    va_copy(va2, *pVa);
+    vmSetRuntimeErrorCopy(pVM, fFlags, pszErrorId, pszFormat, va2);
+    va_end(va2);
+
+    /*
+     * Join paths with VMR3SetRuntimeErrorWorker.
+     */
+    return vmR3SetRuntimeErrorCommon(pVM, fFlags, pszErrorId, pszFormat, pVa);
+}
+
+
+/**
+ * Gets the number of runtime errors raised via VMR3SetRuntimeError.
+ *
+ * This can be used avoid double error messages.
+ *
+ * @returns The runtime error count.
+ * @param   pUVM            The user mode VM handle.
+ */
+VMMR3_INT_DECL(uint32_t) VMR3GetRuntimeErrorCount(PUVM pUVM)
+{
+    return pUVM->vm.s.cRuntimeErrors;
+}
+
+
+/**
+ * Gets the ID virtual of the virtual CPU associated with the calling thread.
+ *
+ * @returns The CPU ID. NIL_VMCPUID if the thread isn't an EMT.
+ *
+ * @param   pVM             The cross context VM structure.
+ */
+VMMR3_INT_DECL(RTCPUID) VMR3GetVMCPUId(PVM pVM)
+{
+    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pVM->pUVM->vm.s.idxTLS);
+    return pUVCpu
+         ? pUVCpu->idCpu
+         : NIL_VMCPUID;
+}
+
+
+/**
+ * Checks if the VM is long-mode (64-bit) capable or not.
+ *
+ * @returns true if VM can operate in long-mode, false otherwise.
+ * @param   pVM             The cross context VM structure.
+ */
+VMMR3_INT_DECL(bool) VMR3IsLongModeAllowed(PVM pVM)
+{
+    switch (pVM->bMainExecutionEngine)
+    {
+        case VM_EXEC_ENGINE_HW_VIRT:
+            return HMIsLongModeAllowed(pVM);
+
+        case VM_EXEC_ENGINE_NATIVE_API:
+            return NEMHCIsLongModeAllowed(pVM);
+
+        case VM_EXEC_ENGINE_NOT_SET:
+            AssertFailed();
+            RT_FALL_THRU();
+        default:
+            return false;
+    }
+}
+
+
+/**
+ * Returns the native ID of the current EMT VMCPU thread.
+ *
+ * @returns Handle if this is an EMT thread; NIL_RTNATIVETHREAD otherwise
+ * @param   pVM             The cross context VM structure.
+ * @thread  EMT
+ */
+VMMR3DECL(RTNATIVETHREAD) VMR3GetVMCPUNativeThread(PVM pVM)
+{
+    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pVM->pUVM->vm.s.idxTLS);
+
+    if (!pUVCpu)
+        return NIL_RTNATIVETHREAD;
+
+    return pUVCpu->vm.s.NativeThreadEMT;
+}
+
+
+/**
+ * Returns the native ID of the current EMT VMCPU thread.
+ *
+ * @returns Handle if this is an EMT thread; NIL_RTNATIVETHREAD otherwise
+ * @param   pUVM        The user mode VM structure.
+ * @thread  EMT
+ */
+VMMR3DECL(RTNATIVETHREAD) VMR3GetVMCPUNativeThreadU(PUVM pUVM)
+{
+    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pUVM->vm.s.idxTLS);
+
+    if (!pUVCpu)
+        return NIL_RTNATIVETHREAD;
+
+    return pUVCpu->vm.s.NativeThreadEMT;
+}
+
+
+/**
+ * Returns the handle of the current EMT VMCPU thread.
+ *
+ * @returns Handle if this is an EMT thread; NIL_RTNATIVETHREAD otherwise
+ * @param   pUVM            The user mode VM handle.
+ * @thread  EMT
+ */
+VMMR3DECL(RTTHREAD) VMR3GetVMCPUThread(PUVM pUVM)
+{
+    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pUVM->vm.s.idxTLS);
+
+    if (!pUVCpu)
+        return NIL_RTTHREAD;
+
+    return pUVCpu->vm.s.ThreadEMT;
+}
+
+
+/**
+ * Returns the handle of the current EMT VMCPU thread.
+ *
+ * @returns The IPRT thread handle.
+ * @param   pUVCpu          The user mode CPU handle.
+ * @thread  EMT
+ */
+VMMR3_INT_DECL(RTTHREAD) VMR3GetThreadHandle(PUVMCPU pUVCpu)
+{
+    return pUVCpu->vm.s.ThreadEMT;
+}
+
+
+/**
+ * Return the package and core ID of a CPU.
+ *
+ * @returns VBOX status code.
+ * @param   pUVM             The user mode VM handle.
+ * @param   idCpu            Virtual CPU to get the ID from.
+ * @param   pidCpuCore       Where to store the core ID of the virtual CPU.
+ * @param   pidCpuPackage    Where to store the package ID of the virtual CPU.
+ *
+ */
+VMMR3DECL(int) VMR3GetCpuCoreAndPackageIdFromCpuId(PUVM pUVM, VMCPUID idCpu, uint32_t *pidCpuCore, uint32_t *pidCpuPackage)
+{
+    /*
+     * Validate input.
+     */
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pidCpuCore, VERR_INVALID_POINTER);
+    AssertPtrReturn(pidCpuPackage, VERR_INVALID_POINTER);
+    if (idCpu >= pVM->cCpus)
+        return VERR_INVALID_CPU_ID;
+
+    /*
+     * Set return values.
+     */
+#ifdef VBOX_WITH_MULTI_CORE
+    *pidCpuCore    = idCpu;
+    *pidCpuPackage = 0;
+#else
+    *pidCpuCore    = 0;
+    *pidCpuPackage = idCpu;
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for VMR3HotUnplugCpu.
+ *
+ * @returns VINF_EM_WAIT_SPIP (strict status code).
+ * @param   pVM                 The cross context VM structure.
+ * @param   idCpu               The current CPU.
+ */
+static DECLCALLBACK(int) vmR3HotUnplugCpu(PVM pVM, VMCPUID idCpu)
+{
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Reset per CPU resources.
+     *
+     * Actually only needed for VT-x because the CPU seems to be still in some
+     * paged mode and startup fails after a new hot plug event. SVM works fine
+     * even without this.
+     */
+    Log(("vmR3HotUnplugCpu for VCPU %u\n", idCpu));
+    PGMR3ResetCpu(pVM, pVCpu);
+    PDMR3ResetCpu(pVCpu);
+    TRPMR3ResetCpu(pVCpu);
+    CPUMR3ResetCpu(pVM, pVCpu);
+    EMR3ResetCpu(pVCpu);
+    HMR3ResetCpu(pVCpu);
+    NEMR3ResetCpu(pVCpu, false /*fInitIpi*/);
+    return VINF_EM_WAIT_SIPI;
+}
+
+
+/**
+ * Hot-unplugs a CPU from the guest.
+ *
+ * @returns VBox status code.
+ * @param   pUVM    The user mode VM handle.
+ * @param   idCpu   Virtual CPU to perform the hot unplugging operation on.
+ */
+VMMR3DECL(int) VMR3HotUnplugCpu(PUVM pUVM, VMCPUID idCpu)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+
+    /** @todo r=bird: Don't destroy the EMT, it'll break VMMR3EmtRendezvous and
+     *        broadcast requests.  Just note down somewhere that the CPU is
+     *        offline and send it to SPIP wait.  Maybe modify VMCPUSTATE and push
+     *        it out of the EM loops when offline. */
+    return VMR3ReqCallNoWaitU(pUVM, idCpu, (PFNRT)vmR3HotUnplugCpu, 2, pVM, idCpu);
+}
+
+
+/**
+ * Hot-plugs a CPU on the guest.
+ *
+ * @returns VBox status code.
+ * @param   pUVM    The user mode VM handle.
+ * @param   idCpu   Virtual CPU to perform the hot plugging operation on.
+ */
+VMMR3DECL(int) VMR3HotPlugCpu(PUVM pUVM, VMCPUID idCpu)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+
+    /** @todo r-bird: Just mark it online and make sure it waits on SPIP. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Changes the VMM execution cap.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM structure.
+ * @param   uCpuExecutionCap    New CPU execution cap in precent, 1-100. Where
+ *                              100 is max performance (default).
+ */
+VMMR3DECL(int) VMR3SetCpuExecutionCap(PUVM pUVM, uint32_t uCpuExecutionCap)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(uCpuExecutionCap > 0 && uCpuExecutionCap <= 100, VERR_INVALID_PARAMETER);
+
+    Log(("VMR3SetCpuExecutionCap: new priority = %d\n", uCpuExecutionCap));
+    /* Note: not called from EMT. */
+    pVM->uCpuExecutionCap = uCpuExecutionCap;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Control whether the VM should power off when resetting.
+ *
+ * @returns VBox status code.
+ * @param   pUVM                The user mode VM handle.
+ * @param   fPowerOffInsteadOfReset Flag whether the VM should power off when
+ *                                  resetting.
+ */
+VMMR3DECL(int) VMR3SetPowerOffInsteadOfReset(PUVM pUVM, bool fPowerOffInsteadOfReset)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    PVM pVM = pUVM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /* Note: not called from EMT. */
+    pVM->vm.s.fPowerOffInsteadOfReset = fPowerOffInsteadOfReset;
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMR3/VMEmt.cpp b/src/VBox/VMM/VMMR3/VMEmt.cpp
new file mode 100644
index 00000000..59304d24
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/VMEmt.cpp
@@ -0,0 +1,1437 @@
+/* $Id: VMEmt.cpp $ */
+/** @file
+ * VM - Virtual Machine, The Emulation Thread.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VM
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/tm.h>
+#include "VMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/asm-math.h>
+#include <iprt/semaphore.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/time.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+int vmR3EmulationThreadWithId(RTTHREAD hThreadSelf, PUVMCPU pUVCpu, VMCPUID idCpu);
+
+
+/**
+ * The emulation thread main function.
+ *
+ * @returns Thread exit code.
+ * @param   hThreadSelf The handle to the executing thread.
+ * @param   pvArgs      Pointer to the user mode per-VCpu structure (UVMPCU).
+ */
+DECLCALLBACK(int) vmR3EmulationThread(RTTHREAD hThreadSelf, void *pvArgs)
+{
+    PUVMCPU pUVCpu = (PUVMCPU)pvArgs;
+    return vmR3EmulationThreadWithId(hThreadSelf, pUVCpu, pUVCpu->idCpu);
+}
+
+
+/**
+ * The emulation thread main function, with Virtual CPU ID for debugging.
+ *
+ * @returns Thread exit code.
+ * @param   hThreadSelf The handle to the executing thread.
+ * @param   pUVCpu      Pointer to the user mode per-VCpu structure.
+ * @param   idCpu       The virtual CPU ID, for backtrace purposes.
+ */
+int vmR3EmulationThreadWithId(RTTHREAD hThreadSelf, PUVMCPU pUVCpu, VMCPUID idCpu)
+{
+    PUVM    pUVM = pUVCpu->pUVM;
+    int     rc;
+    RT_NOREF_PV(hThreadSelf);
+
+    AssertReleaseMsg(VALID_PTR(pUVM) && pUVM->u32Magic == UVM_MAGIC,
+                     ("Invalid arguments to the emulation thread!\n"));
+
+    rc = RTTlsSet(pUVM->vm.s.idxTLS, pUVCpu);
+    AssertReleaseMsgRCReturn(rc, ("RTTlsSet %x failed with %Rrc\n", pUVM->vm.s.idxTLS, rc), rc);
+
+    if (   pUVM->pVmm2UserMethods
+        && pUVM->pVmm2UserMethods->pfnNotifyEmtInit)
+        pUVM->pVmm2UserMethods->pfnNotifyEmtInit(pUVM->pVmm2UserMethods, pUVM, pUVCpu);
+
+    /*
+     * The request loop.
+     */
+    rc = VINF_SUCCESS;
+    Log(("vmR3EmulationThread: Emulation thread starting the days work... Thread=%#x pUVM=%p\n", hThreadSelf, pUVM));
+    VMSTATE enmBefore = VMSTATE_CREATED; /* (only used for logging atm.) */
+    ASMAtomicIncU32(&pUVM->vm.s.cActiveEmts);
+    for (;;)
+    {
+        /*
+         * During early init there is no pVM and/or pVCpu, so make a special path
+         * for that to keep things clearly separate.
+         */
+        PVM    pVM   = pUVM->pVM;
+        PVMCPU pVCpu = pUVCpu->pVCpu;
+        if (!pVCpu || !pVM)
+        {
+            /*
+             * Check for termination first.
+             */
+            if (pUVM->vm.s.fTerminateEMT)
+            {
+                rc = VINF_EM_TERMINATE;
+                break;
+            }
+
+            /*
+             * Only the first VCPU may initialize the VM during early init
+             * and must therefore service all VMCPUID_ANY requests.
+             * See also VMR3Create
+             */
+            if (    (pUVM->vm.s.pNormalReqs || pUVM->vm.s.pPriorityReqs)
+                &&  pUVCpu->idCpu == 0)
+            {
+                /*
+                 * Service execute in any EMT request.
+                 */
+                rc = VMR3ReqProcessU(pUVM, VMCPUID_ANY, false /*fPriorityOnly*/);
+                Log(("vmR3EmulationThread: Req rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), pUVM->pVM ? VMR3GetStateName(pUVM->pVM->enmVMState) : "CREATING"));
+            }
+            else if (pUVCpu->vm.s.pNormalReqs || pUVCpu->vm.s.pPriorityReqs)
+            {
+                /*
+                 * Service execute in specific EMT request.
+                 */
+                rc = VMR3ReqProcessU(pUVM, pUVCpu->idCpu, false /*fPriorityOnly*/);
+                Log(("vmR3EmulationThread: Req (cpu=%u) rc=%Rrc, VM state %s -> %s\n", pUVCpu->idCpu, rc, VMR3GetStateName(enmBefore), pUVM->pVM ? VMR3GetStateName(pUVM->pVM->enmVMState) : "CREATING"));
+            }
+            else
+            {
+                /*
+                 * Nothing important is pending, so wait for something.
+                 */
+                rc = VMR3WaitU(pUVCpu);
+                if (RT_FAILURE(rc))
+                {
+                    AssertLogRelMsgFailed(("VMR3WaitU failed with %Rrc\n", rc));
+                    break;
+                }
+            }
+        }
+        else
+        {
+            /*
+             * Pending requests which needs servicing?
+             *
+             * We check for state changes in addition to status codes when
+             * servicing requests. (Look after the ifs.)
+             */
+            enmBefore = pVM->enmVMState;
+            if (pUVM->vm.s.fTerminateEMT)
+            {
+                rc = VINF_EM_TERMINATE;
+                break;
+            }
+
+            if (VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
+            {
+                rc = VMMR3EmtRendezvousFF(pVM, pVM->apCpusR3[idCpu]);
+                Log(("vmR3EmulationThread: Rendezvous rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
+            }
+            else if (pUVM->vm.s.pNormalReqs || pUVM->vm.s.pPriorityReqs)
+            {
+                /*
+                 * Service execute in any EMT request.
+                 */
+                rc = VMR3ReqProcessU(pUVM, VMCPUID_ANY, false /*fPriorityOnly*/);
+                Log(("vmR3EmulationThread: Req rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
+            }
+            else if (pUVCpu->vm.s.pNormalReqs || pUVCpu->vm.s.pPriorityReqs)
+            {
+                /*
+                 * Service execute in specific EMT request.
+                 */
+                rc = VMR3ReqProcessU(pUVM, pUVCpu->idCpu, false /*fPriorityOnly*/);
+                Log(("vmR3EmulationThread: Req (cpu=%u) rc=%Rrc, VM state %s -> %s\n", pUVCpu->idCpu, rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
+            }
+            else if (   VM_FF_IS_SET(pVM, VM_FF_DBGF)
+                     || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_DBGF))
+            {
+                /*
+                 * Service the debugger request.
+                 */
+                rc = DBGFR3VMMForcedAction(pVM, pVCpu);
+                Log(("vmR3EmulationThread: Dbg rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
+            }
+            else if (VM_FF_TEST_AND_CLEAR(pVM, VM_FF_RESET))
+            {
+                /*
+                 * Service a delayed reset request.
+                 */
+                rc = VBOXSTRICTRC_VAL(VMR3ResetFF(pVM));
+                VM_FF_CLEAR(pVM, VM_FF_RESET);
+                Log(("vmR3EmulationThread: Reset rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
+            }
+            else
+            {
+                /*
+                 * Nothing important is pending, so wait for something.
+                 */
+                rc = VMR3WaitU(pUVCpu);
+                if (RT_FAILURE(rc))
+                {
+                    AssertLogRelMsgFailed(("VMR3WaitU failed with %Rrc\n", rc));
+                    break;
+                }
+            }
+
+            /*
+             * Check for termination requests, these have extremely high priority.
+             */
+            if (    rc == VINF_EM_TERMINATE
+                ||  pUVM->vm.s.fTerminateEMT)
+                break;
+        }
+
+        /*
+         * Some requests (both VMR3Req* and the DBGF) can potentially resume
+         * or start the VM, in that case we'll get a change in VM status
+         * indicating that we're now running.
+         */
+        if (RT_SUCCESS(rc))
+        {
+            pVM = pUVM->pVM;
+            if (pVM)
+            {
+                pVCpu = pVM->apCpusR3[idCpu];
+                if (   pVM->enmVMState == VMSTATE_RUNNING
+                    && VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(pVCpu)))
+                {
+                    rc = EMR3ExecuteVM(pVM, pVCpu);
+                    Log(("vmR3EmulationThread: EMR3ExecuteVM() -> rc=%Rrc, enmVMState=%d\n", rc, pVM->enmVMState));
+                }
+            }
+        }
+
+    } /* forever */
+
+
+    /*
+     * Decrement the active EMT count if we haven't done it yet in vmR3Destroy.
+     */
+    if (!pUVCpu->vm.s.fBeenThruVmDestroy)
+        ASMAtomicDecU32(&pUVM->vm.s.cActiveEmts);
+
+
+    /*
+     * Cleanup and exit.
+     * EMT0 does the VM destruction after all other EMTs have deregistered and terminated.
+     */
+    Log(("vmR3EmulationThread: Terminating emulation thread! Thread=%#x pUVM=%p rc=%Rrc enmBefore=%d enmVMState=%d\n",
+         hThreadSelf, pUVM, rc, enmBefore, pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_TERMINATED));
+    PVM pVM;
+    if (   idCpu == 0
+        && (pVM = pUVM->pVM) != NULL)
+    {
+        /* Wait for any other EMTs to terminate before we destroy the VM (see vmR3DestroyVM). */
+        for (VMCPUID iCpu = 1; iCpu < pUVM->cCpus; iCpu++)
+        {
+            RTTHREAD hThread;
+            ASMAtomicXchgHandle(&pUVM->aCpus[iCpu].vm.s.ThreadEMT, NIL_RTTHREAD, &hThread);
+            if (hThread != NIL_RTTHREAD)
+            {
+                int rc2 = RTThreadWait(hThread, 5 * RT_MS_1SEC, NULL);
+                AssertLogRelMsgRC(rc2, ("iCpu=%u rc=%Rrc\n", iCpu, rc2));
+                if (RT_FAILURE(rc2))
+                    pUVM->aCpus[iCpu].vm.s.ThreadEMT = hThread;
+            }
+        }
+
+        /* Switch to the terminated state, clearing the VM pointer and finally destroy the VM. */
+        vmR3SetTerminated(pVM);
+
+        pUVM->pVM = NULL;
+        for (VMCPUID iCpu = 0; iCpu < pUVM->cCpus; iCpu++)
+        {
+            pUVM->aCpus[iCpu].pVM   = NULL;
+            pUVM->aCpus[iCpu].pVCpu = NULL;
+        }
+
+        int rc2 = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), 0 /*idCpu*/, VMMR0_DO_GVMM_DESTROY_VM, 0, NULL);
+        AssertLogRelRC(rc2);
+    }
+    /* Deregister the EMT with VMMR0. */
+    else if (   idCpu != 0
+             && (pVM = pUVM->pVM) != NULL)
+    {
+        int rc2 = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), idCpu, VMMR0_DO_GVMM_DEREGISTER_VMCPU, 0, NULL);
+        AssertLogRelRC(rc2);
+    }
+
+    if (   pUVM->pVmm2UserMethods
+        && pUVM->pVmm2UserMethods->pfnNotifyEmtTerm)
+        pUVM->pVmm2UserMethods->pfnNotifyEmtTerm(pUVM->pVmm2UserMethods, pUVM, pUVCpu);
+
+    pUVCpu->vm.s.NativeThreadEMT = NIL_RTNATIVETHREAD;
+    Log(("vmR3EmulationThread: EMT is terminated.\n"));
+    return rc;
+}
+
+
+/**
+ * Gets the name of a halt method.
+ *
+ * @returns Pointer to a read only string.
+ * @param   enmMethod   The method.
+ */
+static const char *vmR3GetHaltMethodName(VMHALTMETHOD enmMethod)
+{
+    switch (enmMethod)
+    {
+        case VMHALTMETHOD_BOOTSTRAP:    return "bootstrap";
+        case VMHALTMETHOD_DEFAULT:      return "default";
+        case VMHALTMETHOD_OLD:          return "old";
+        case VMHALTMETHOD_1:            return "method1";
+        //case VMHALTMETHOD_2:            return "method2";
+        case VMHALTMETHOD_GLOBAL_1:     return "global1";
+        default:                        return "unknown";
+    }
+}
+
+
+/**
+ * Signal a fatal wait error.
+ *
+ * @returns Fatal error code to be propagated up the call stack.
+ * @param   pUVCpu              The user mode per CPU structure of the calling
+ *                              EMT.
+ * @param   pszFmt              The error format with a single %Rrc in it.
+ * @param   rcFmt               The status code to format.
+ */
+static int vmR3FatalWaitError(PUVMCPU pUVCpu, const char *pszFmt, int rcFmt)
+{
+    /** @todo This is wrong ... raise a fatal error / guru meditation
+     *        instead. */
+    AssertLogRelMsgFailed((pszFmt, rcFmt));
+    ASMAtomicUoWriteBool(&pUVCpu->pUVM->vm.s.fTerminateEMT, true);
+    if (pUVCpu->pVM)
+        VM_FF_SET(pUVCpu->pVM, VM_FF_CHECK_VM_STATE);
+    return VERR_VM_FATAL_WAIT_ERROR;
+}
+
+
+/**
+ * The old halt loop.
+ */
+static DECLCALLBACK(int) vmR3HaltOldDoHalt(PUVMCPU pUVCpu, const uint32_t fMask, uint64_t /* u64Now*/)
+{
+    /*
+     * Halt loop.
+     */
+    PVM    pVM   = pUVCpu->pVM;
+    PVMCPU pVCpu = pUVCpu->pVCpu;
+
+    int rc = VINF_SUCCESS;
+    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
+    //unsigned cLoops = 0;
+    for (;;)
+    {
+        /*
+         * Work the timers and check if we can exit.
+         * The poll call gives us the ticks left to the next event in
+         * addition to perhaps set an FF.
+         */
+        uint64_t const u64StartTimers   = RTTimeNanoTS();
+        TMR3TimerQueuesDo(pVM);
+        uint64_t const cNsElapsedTimers = RTTimeNanoTS() - u64StartTimers;
+        STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltTimers, cNsElapsedTimers);
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, fMask))
+            break;
+        uint64_t u64NanoTS;
+        TMTimerPollGIP(pVM, pVCpu, &u64NanoTS);
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, fMask))
+            break;
+
+        /*
+         * Wait for a while. Someone will wake us up or interrupt the call if
+         * anything needs our attention.
+         */
+        if (u64NanoTS < 50000)
+        {
+            //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d spin\n", u64NanoTS, cLoops++);
+            /* spin */;
+        }
+        else
+        {
+            VMMR3YieldStop(pVM);
+            //uint64_t u64Start = RTTimeNanoTS();
+            if (u64NanoTS <  870000) /* this is a bit speculative... works fine on linux. */
+            {
+                //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d yield", u64NanoTS, cLoops++);
+                uint64_t const u64StartSchedYield   = RTTimeNanoTS();
+                RTThreadYield(); /* this is the best we can do here */
+                uint64_t const cNsElapsedSchedYield = RTTimeNanoTS() - u64StartSchedYield;
+                STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltYield, cNsElapsedSchedYield);
+            }
+            else if (u64NanoTS < 2000000)
+            {
+                //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d sleep 1ms", u64NanoTS, cLoops++);
+                uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
+                rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, 1);
+                uint64_t const cNsElapsedSchedHalt = RTTimeNanoTS() - u64StartSchedHalt;
+                STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlock, cNsElapsedSchedHalt);
+            }
+            else
+            {
+                //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d sleep %dms", u64NanoTS, cLoops++, (uint32_t)RT_MIN((u64NanoTS - 500000) / 1000000, 15));
+                uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
+                rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, RT_MIN((u64NanoTS - 1000000) / 1000000, 15));
+                uint64_t const cNsElapsedSchedHalt = RTTimeNanoTS() - u64StartSchedHalt;
+                STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlock, cNsElapsedSchedHalt);
+            }
+            //uint64_t u64Slept = RTTimeNanoTS() - u64Start;
+            //RTLogPrintf(" -> rc=%Rrc in %RU64 ns / %RI64 ns delta\n", rc, u64Slept, u64NanoTS - u64Slept);
+        }
+        if (rc == VERR_TIMEOUT)
+            rc = VINF_SUCCESS;
+        else if (RT_FAILURE(rc))
+        {
+            rc = vmR3FatalWaitError(pUVCpu, "RTSemEventWait->%Rrc\n", rc);
+            break;
+        }
+    }
+
+    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * Initialize the configuration of halt method 1 & 2.
+ *
+ * @return VBox status code. Failure on invalid CFGM data.
+ * @param   pUVM        The user mode VM structure.
+ */
+static int vmR3HaltMethod12ReadConfigU(PUVM pUVM)
+{
+    /*
+     * The defaults.
+     */
+#if 1 /* DEBUGGING STUFF - REMOVE LATER */
+    pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = 4;
+    pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg =   2*1000000;
+    pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg =  75*1000000;
+    pUVM->vm.s.Halt.Method12.u32StartSpinningCfg    =  30*1000000;
+    pUVM->vm.s.Halt.Method12.u32StopSpinningCfg     =  20*1000000;
+#else
+    pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = 4;
+    pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg =   5*1000000;
+    pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg = 200*1000000;
+    pUVM->vm.s.Halt.Method12.u32StartSpinningCfg    =  20*1000000;
+    pUVM->vm.s.Halt.Method12.u32StopSpinningCfg     =   2*1000000;
+#endif
+
+    /*
+     * Query overrides.
+     *
+     * I don't have time to bother with niceties such as invalid value checks
+     * here right now. sorry.
+     */
+    PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pUVM->pVM), "/VMM/HaltedMethod1");
+    if (pCfg)
+    {
+        uint32_t u32;
+        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "LagBlockIntervalDivisor", &u32)))
+            pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = u32;
+        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "MinBlockInterval", &u32)))
+            pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg = u32;
+        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "MaxBlockInterval", &u32)))
+            pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg = u32;
+        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "StartSpinning", &u32)))
+            pUVM->vm.s.Halt.Method12.u32StartSpinningCfg = u32;
+        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "StopSpinning", &u32)))
+            pUVM->vm.s.Halt.Method12.u32StopSpinningCfg = u32;
+        LogRel(("VMEmt: HaltedMethod1 config: %d/%d/%d/%d/%d\n",
+                pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg,
+                pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg,
+                pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg,
+                pUVM->vm.s.Halt.Method12.u32StartSpinningCfg,
+                pUVM->vm.s.Halt.Method12.u32StopSpinningCfg));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Initialize halt method 1.
+ *
+ * @return VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+static DECLCALLBACK(int) vmR3HaltMethod1Init(PUVM pUVM)
+{
+    return vmR3HaltMethod12ReadConfigU(pUVM);
+}
+
+
+/**
+ * Method 1 - Block whenever possible, and when lagging behind
+ * switch to spinning for 10-30ms with occasional blocking until
+ * the lag has been eliminated.
+ */
+static DECLCALLBACK(int) vmR3HaltMethod1Halt(PUVMCPU pUVCpu, const uint32_t fMask, uint64_t u64Now)
+{
+    PUVM    pUVM    = pUVCpu->pUVM;
+    PVMCPU  pVCpu   = pUVCpu->pVCpu;
+    PVM     pVM     = pUVCpu->pVM;
+
+    /*
+     * To simplify things, we decide up-front whether we should switch to spinning or
+     * not. This makes some ASSUMPTIONS about the cause of the spinning (PIT/RTC/PCNet)
+     * and that it will generate interrupts or other events that will cause us to exit
+     * the halt loop.
+     */
+    bool fBlockOnce = false;
+    bool fSpinning = false;
+    uint32_t u32CatchUpPct = TMVirtualSyncGetCatchUpPct(pVM);
+    if (u32CatchUpPct /* non-zero if catching up */)
+    {
+        if (pUVCpu->vm.s.Halt.Method12.u64StartSpinTS)
+        {
+            fSpinning = TMVirtualSyncGetLag(pVM) >= pUVM->vm.s.Halt.Method12.u32StopSpinningCfg;
+            if (fSpinning)
+            {
+                uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
+                fBlockOnce = u64Now - pUVCpu->vm.s.Halt.Method12.u64LastBlockTS
+                           > RT_MAX(pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg,
+                                    RT_MIN(u64Lag / pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg,
+                                           pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg));
+            }
+            else
+            {
+                //RTLogRelPrintf("Stopped spinning (%u ms)\n", (u64Now - pUVCpu->vm.s.Halt.Method12.u64StartSpinTS) / 1000000);
+                pUVCpu->vm.s.Halt.Method12.u64StartSpinTS = 0;
+            }
+        }
+        else
+        {
+            fSpinning = TMVirtualSyncGetLag(pVM) >= pUVM->vm.s.Halt.Method12.u32StartSpinningCfg;
+            if (fSpinning)
+                pUVCpu->vm.s.Halt.Method12.u64StartSpinTS = u64Now;
+        }
+    }
+    else if (pUVCpu->vm.s.Halt.Method12.u64StartSpinTS)
+    {
+        //RTLogRelPrintf("Stopped spinning (%u ms)\n", (u64Now - pUVCpu->vm.s.Halt.Method12.u64StartSpinTS) / 1000000);
+        pUVCpu->vm.s.Halt.Method12.u64StartSpinTS = 0;
+    }
+
+    /*
+     * Halt loop.
+     */
+    int rc = VINF_SUCCESS;
+    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
+    unsigned cLoops = 0;
+    for (;; cLoops++)
+    {
+        /*
+         * Work the timers and check if we can exit.
+         */
+        uint64_t const u64StartTimers   = RTTimeNanoTS();
+        TMR3TimerQueuesDo(pVM);
+        uint64_t const cNsElapsedTimers = RTTimeNanoTS() - u64StartTimers;
+        STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltTimers, cNsElapsedTimers);
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, fMask))
+            break;
+
+        /*
+         * Estimate time left to the next event.
+         */
+        uint64_t u64NanoTS;
+        TMTimerPollGIP(pVM, pVCpu, &u64NanoTS);
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, fMask))
+            break;
+
+        /*
+         * Block if we're not spinning and the interval isn't all that small.
+         */
+        if (    (   !fSpinning
+                 || fBlockOnce)
+#if 1 /* DEBUGGING STUFF - REMOVE LATER */
+            &&  u64NanoTS >= 100000) /* 0.100 ms */
+#else
+            &&  u64NanoTS >= 250000) /* 0.250 ms */
+#endif
+        {
+            const uint64_t Start = pUVCpu->vm.s.Halt.Method12.u64LastBlockTS = RTTimeNanoTS();
+            VMMR3YieldStop(pVM);
+
+            uint32_t cMilliSecs = RT_MIN(u64NanoTS / 1000000, 15);
+            if (cMilliSecs <= pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLongAvg)
+                cMilliSecs = 1;
+            else
+                cMilliSecs -= pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLongAvg;
+
+            //RTLogRelPrintf("u64NanoTS=%RI64 cLoops=%3d sleep %02dms (%7RU64) ", u64NanoTS, cLoops, cMilliSecs, u64NanoTS);
+            uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
+            rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, cMilliSecs);
+            uint64_t const cNsElapsedSchedHalt = RTTimeNanoTS() - u64StartSchedHalt;
+            STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlock, cNsElapsedSchedHalt);
+
+            if (rc == VERR_TIMEOUT)
+                rc = VINF_SUCCESS;
+            else if (RT_FAILURE(rc))
+            {
+                rc = vmR3FatalWaitError(pUVCpu, "RTSemEventWait->%Rrc\n", rc);
+                break;
+            }
+
+            /*
+             * Calc the statistics.
+             * Update averages every 16th time, and flush parts of the history every 64th time.
+             */
+            const uint64_t Elapsed = RTTimeNanoTS() - Start;
+            pUVCpu->vm.s.Halt.Method12.cNSBlocked += Elapsed;
+            if (Elapsed > u64NanoTS)
+                pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLong += Elapsed - u64NanoTS;
+            pUVCpu->vm.s.Halt.Method12.cBlocks++;
+            if (!(pUVCpu->vm.s.Halt.Method12.cBlocks & 0xf))
+            {
+                pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLongAvg = pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLong / pUVCpu->vm.s.Halt.Method12.cBlocks;
+                if (!(pUVCpu->vm.s.Halt.Method12.cBlocks & 0x3f))
+                {
+                    pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLong = pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLongAvg * 0x40;
+                    pUVCpu->vm.s.Halt.Method12.cBlocks = 0x40;
+                }
+            }
+            //RTLogRelPrintf(" -> %7RU64 ns / %7RI64 ns delta%s\n", Elapsed, Elapsed - u64NanoTS, fBlockOnce ? " (block once)" : "");
+
+            /*
+             * Clear the block once flag if we actually blocked.
+             */
+            if (    fBlockOnce
+                &&  Elapsed > 100000 /* 0.1 ms */)
+                fBlockOnce = false;
+        }
+    }
+    //if (fSpinning) RTLogRelPrintf("spun for %RU64 ns %u loops; lag=%RU64 pct=%d\n", RTTimeNanoTS() - u64Now, cLoops, TMVirtualSyncGetLag(pVM), u32CatchUpPct);
+
+    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * Initialize the global 1 halt method.
+ *
+ * @return VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+static DECLCALLBACK(int) vmR3HaltGlobal1Init(PUVM pUVM)
+{
+    /*
+     * The defaults.
+     */
+    uint32_t cNsResolution = SUPSemEventMultiGetResolution(pUVM->vm.s.pSession);
+    if (cNsResolution > 5*RT_NS_100US)
+        pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg = 50000;
+    else if (cNsResolution > RT_NS_100US)
+        pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg = cNsResolution / 4;
+    else
+        pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg = 2000;
+
+    /*
+     * Query overrides.
+     *
+     * I don't have time to bother with niceties such as invalid value checks
+     * here right now. sorry.
+     */
+    PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pUVM->pVM), "/VMM/HaltedGlobal1");
+    if (pCfg)
+    {
+        uint32_t u32;
+        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "SpinBlockThreshold", &u32)))
+            pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg = u32;
+    }
+    LogRel(("VMEmt: HaltedGlobal1 config: cNsSpinBlockThresholdCfg=%u\n",
+            pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * The global 1 halt method - Block in GMM (ring-0) and let it
+ * try take care of the global scheduling of EMT threads.
+ */
+static DECLCALLBACK(int) vmR3HaltGlobal1Halt(PUVMCPU pUVCpu, const uint32_t fMask, uint64_t u64Now)
+{
+    PUVM    pUVM  = pUVCpu->pUVM;
+    PVMCPU  pVCpu = pUVCpu->pVCpu;
+    PVM     pVM   = pUVCpu->pVM;
+    Assert(VMMGetCpu(pVM) == pVCpu);
+    NOREF(u64Now);
+
+    /*
+     * Halt loop.
+     */
+    //uint64_t u64NowLog, u64Start;
+    //u64Start = u64NowLog = RTTimeNanoTS();
+    int rc = VINF_SUCCESS;
+    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
+    unsigned cLoops = 0;
+    for (;; cLoops++)
+    {
+        /*
+         * Work the timers and check if we can exit.
+         */
+        uint64_t const u64StartTimers   = RTTimeNanoTS();
+        TMR3TimerQueuesDo(pVM);
+        uint64_t const cNsElapsedTimers = RTTimeNanoTS() - u64StartTimers;
+        STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltTimers, cNsElapsedTimers);
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, fMask))
+            break;
+
+        /*
+         * Estimate time left to the next event.
+         */
+        //u64NowLog = RTTimeNanoTS();
+        uint64_t u64Delta;
+        uint64_t u64GipTime = TMTimerPollGIP(pVM, pVCpu, &u64Delta);
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, fMask))
+            break;
+
+        /*
+         * Block if we're not spinning and the interval isn't all that small.
+         */
+        if (u64Delta >= pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg)
+        {
+            VMMR3YieldStop(pVM);
+            if (    VM_FF_IS_ANY_SET(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+                ||  VMCPU_FF_IS_ANY_SET(pVCpu, fMask))
+                break;
+
+            //RTLogPrintf("loop=%-3d  u64GipTime=%'llu / %'llu   now=%'llu / %'llu\n", cLoops, u64GipTime, u64Delta, u64NowLog, u64GipTime - u64NowLog);
+            uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
+            rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), pVCpu->idCpu, VMMR0_DO_GVMM_SCHED_HALT, u64GipTime, NULL);
+            uint64_t const u64EndSchedHalt     = RTTimeNanoTS();
+            uint64_t const cNsElapsedSchedHalt = u64EndSchedHalt - u64StartSchedHalt;
+            STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlock, cNsElapsedSchedHalt);
+
+            if (rc == VERR_INTERRUPTED)
+                rc = VINF_SUCCESS;
+            else if (RT_FAILURE(rc))
+            {
+                rc = vmR3FatalWaitError(pUVCpu, "vmR3HaltGlobal1Halt: VMMR0_DO_GVMM_SCHED_HALT->%Rrc\n", rc);
+                break;
+            }
+            else
+            {
+                int64_t const cNsOverslept = u64EndSchedHalt - u64GipTime;
+                if (cNsOverslept > 50000)
+                    STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlockOverslept, cNsOverslept);
+                else if (cNsOverslept < -50000)
+                    STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlockInsomnia,  cNsElapsedSchedHalt);
+                else
+                    STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlockOnTime,    cNsElapsedSchedHalt);
+            }
+        }
+        /*
+         * When spinning call upon the GVMM and do some wakups once
+         * in a while, it's not like we're actually busy or anything.
+         */
+        else if (!(cLoops & 0x1fff))
+        {
+            uint64_t const u64StartSchedYield   = RTTimeNanoTS();
+            rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), pVCpu->idCpu, VMMR0_DO_GVMM_SCHED_POLL, false /* don't yield */, NULL);
+            uint64_t const cNsElapsedSchedYield = RTTimeNanoTS() - u64StartSchedYield;
+            STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltYield, cNsElapsedSchedYield);
+        }
+    }
+    //RTLogPrintf("*** %u loops %'llu;  lag=%RU64\n", cLoops, u64NowLog - u64Start, TMVirtualSyncGetLag(pVM));
+
+    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * The global 1 halt method - VMR3Wait() worker.
+ *
+ * @returns VBox status code.
+ * @param   pUVCpu            Pointer to the user mode VMCPU structure.
+ */
+static DECLCALLBACK(int) vmR3HaltGlobal1Wait(PUVMCPU pUVCpu)
+{
+    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
+
+    PVM    pVM   = pUVCpu->pUVM->pVM;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    Assert(pVCpu->idCpu == pUVCpu->idCpu);
+
+    int rc = VINF_SUCCESS;
+    for (;;)
+    {
+        /*
+         * Check Relevant FFs.
+         */
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_EXTERNAL_SUSPENDED_MASK))
+            break;
+
+        /*
+         * Wait for a while. Someone will wake us up or interrupt the call if
+         * anything needs our attention.
+         */
+        rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), pVCpu->idCpu, VMMR0_DO_GVMM_SCHED_HALT, RTTimeNanoTS() + 1000000000 /* +1s */, NULL);
+        if (rc == VERR_INTERRUPTED)
+            rc = VINF_SUCCESS;
+        else if (RT_FAILURE(rc))
+        {
+            rc = vmR3FatalWaitError(pUVCpu, "vmR3HaltGlobal1Wait: VMMR0_DO_GVMM_SCHED_HALT->%Rrc\n", rc);
+            break;
+        }
+    }
+
+    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * The global 1 halt method - VMR3NotifyFF() worker.
+ *
+ * @param   pUVCpu          Pointer to the user mode VMCPU structure.
+ * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
+ */
+static DECLCALLBACK(void) vmR3HaltGlobal1NotifyCpuFF(PUVMCPU pUVCpu, uint32_t fFlags)
+{
+    /*
+     * With ring-0 halting, the fWait flag isn't set, so we have to check the
+     * CPU state to figure out whether to do a wakeup call.
+     */
+    PVMCPU pVCpu = pUVCpu->pVCpu;
+    if (pVCpu)
+    {
+        VMCPUSTATE enmState = VMCPU_GET_STATE(pVCpu);
+        if (enmState == VMCPUSTATE_STARTED_HALTED || pUVCpu->vm.s.fWait)
+        {
+            int rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pUVCpu->pVM), pUVCpu->idCpu, VMMR0_DO_GVMM_SCHED_WAKE_UP, 0, NULL);
+            AssertRC(rc);
+
+        }
+        else if (   (fFlags & VMNOTIFYFF_FLAGS_POKE)
+                 || !(fFlags & VMNOTIFYFF_FLAGS_DONE_REM))
+        {
+            if (enmState == VMCPUSTATE_STARTED_EXEC)
+            {
+                if (fFlags & VMNOTIFYFF_FLAGS_POKE)
+                {
+                    int rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pUVCpu->pVM), pUVCpu->idCpu, VMMR0_DO_GVMM_SCHED_POKE, 0, NULL);
+                    AssertRC(rc);
+                }
+            }
+            else if (   enmState == VMCPUSTATE_STARTED_EXEC_NEM
+                     || enmState == VMCPUSTATE_STARTED_EXEC_NEM_WAIT)
+                NEMR3NotifyFF(pUVCpu->pVM, pVCpu, fFlags);
+        }
+    }
+    /* This probably makes little sense: */
+    else if (pUVCpu->vm.s.fWait)
+    {
+        int rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pUVCpu->pVM), pUVCpu->idCpu, VMMR0_DO_GVMM_SCHED_WAKE_UP, 0, NULL);
+        AssertRC(rc);
+    }
+}
+
+
+/**
+ * Bootstrap VMR3Wait() worker.
+ *
+ * @returns VBox status code.
+ * @param   pUVCpu      Pointer to the user mode VMCPU structure.
+ */
+static DECLCALLBACK(int) vmR3BootstrapWait(PUVMCPU pUVCpu)
+{
+    PUVM pUVM = pUVCpu->pUVM;
+
+    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
+
+    int rc = VINF_SUCCESS;
+    for (;;)
+    {
+        /*
+         * Check Relevant FFs.
+         */
+        if (pUVM->vm.s.pNormalReqs   || pUVM->vm.s.pPriorityReqs)   /* global requests pending? */
+            break;
+        if (pUVCpu->vm.s.pNormalReqs || pUVCpu->vm.s.pPriorityReqs) /* local requests pending? */
+            break;
+
+        if (    pUVCpu->pVM
+            &&  (   VM_FF_IS_ANY_SET(pUVCpu->pVM, VM_FF_EXTERNAL_SUSPENDED_MASK)
+                 || VMCPU_FF_IS_ANY_SET(VMMGetCpu(pUVCpu->pVM), VMCPU_FF_EXTERNAL_SUSPENDED_MASK)
+                )
+            )
+            break;
+        if (pUVM->vm.s.fTerminateEMT)
+            break;
+
+        /*
+         * Wait for a while. Someone will wake us up or interrupt the call if
+         * anything needs our attention.
+         */
+        rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, 1000);
+        if (rc == VERR_TIMEOUT)
+            rc = VINF_SUCCESS;
+        else if (RT_FAILURE(rc))
+        {
+            rc = vmR3FatalWaitError(pUVCpu, "RTSemEventWait->%Rrc\n", rc);
+            break;
+        }
+    }
+
+    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * Bootstrap VMR3NotifyFF() worker.
+ *
+ * @param   pUVCpu          Pointer to the user mode VMCPU structure.
+ * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
+ */
+static DECLCALLBACK(void) vmR3BootstrapNotifyCpuFF(PUVMCPU pUVCpu, uint32_t fFlags)
+{
+    if (pUVCpu->vm.s.fWait)
+    {
+        int rc = RTSemEventSignal(pUVCpu->vm.s.EventSemWait);
+        AssertRC(rc);
+    }
+    NOREF(fFlags);
+}
+
+
+/**
+ * Default VMR3Wait() worker.
+ *
+ * @returns VBox status code.
+ * @param   pUVCpu          Pointer to the user mode VMCPU structure.
+ */
+static DECLCALLBACK(int) vmR3DefaultWait(PUVMCPU pUVCpu)
+{
+    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
+
+    PVM    pVM   = pUVCpu->pVM;
+    PVMCPU pVCpu = pUVCpu->pVCpu;
+    int    rc    = VINF_SUCCESS;
+    for (;;)
+    {
+        /*
+         * Check Relevant FFs.
+         */
+        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK)
+            ||  VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_EXTERNAL_SUSPENDED_MASK))
+            break;
+
+        /*
+         * Wait for a while. Someone will wake us up or interrupt the call if
+         * anything needs our attention.
+         */
+        rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, 1000);
+        if (rc == VERR_TIMEOUT)
+            rc = VINF_SUCCESS;
+        else if (RT_FAILURE(rc))
+        {
+            rc = vmR3FatalWaitError(pUVCpu, "RTSemEventWait->%Rrc", rc);
+            break;
+        }
+    }
+
+    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * Default VMR3NotifyFF() worker.
+ *
+ * @param   pUVCpu          Pointer to the user mode VMCPU structure.
+ * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
+ */
+static DECLCALLBACK(void) vmR3DefaultNotifyCpuFF(PUVMCPU pUVCpu, uint32_t fFlags)
+{
+    if (pUVCpu->vm.s.fWait)
+    {
+        int rc = RTSemEventSignal(pUVCpu->vm.s.EventSemWait);
+        AssertRC(rc);
+    }
+    else
+    {
+        PVMCPU pVCpu = pUVCpu->pVCpu;
+        if (pVCpu)
+        {
+            VMCPUSTATE enmState = pVCpu->enmState;
+            if (   enmState == VMCPUSTATE_STARTED_EXEC_NEM
+                || enmState == VMCPUSTATE_STARTED_EXEC_NEM_WAIT)
+                NEMR3NotifyFF(pUVCpu->pVM, pVCpu, fFlags);
+        }
+    }
+}
+
+
+/**
+ * Array with halt method descriptors.
+ * VMINT::iHaltMethod contains an index into this array.
+ */
+static const struct VMHALTMETHODDESC
+{
+    /** The halt method ID. */
+    VMHALTMETHOD                enmHaltMethod;
+    /** Set if the method support halting directly in ring-0. */
+    bool                        fMayHaltInRing0;
+    /** The init function for loading config and initialize variables. */
+    DECLR3CALLBACKMEMBER(int,   pfnInit,(PUVM pUVM));
+    /** The term function. */
+    DECLR3CALLBACKMEMBER(void,  pfnTerm,(PUVM pUVM));
+    /** The VMR3WaitHaltedU function. */
+    DECLR3CALLBACKMEMBER(int,   pfnHalt,(PUVMCPU pUVCpu, const uint32_t fMask, uint64_t u64Now));
+    /** The VMR3WaitU function. */
+    DECLR3CALLBACKMEMBER(int,   pfnWait,(PUVMCPU pUVCpu));
+    /** The VMR3NotifyCpuFFU function. */
+    DECLR3CALLBACKMEMBER(void,  pfnNotifyCpuFF,(PUVMCPU pUVCpu, uint32_t fFlags));
+    /** The VMR3NotifyGlobalFFU function. */
+    DECLR3CALLBACKMEMBER(void,  pfnNotifyGlobalFF,(PUVM pUVM, uint32_t fFlags));
+} g_aHaltMethods[] =
+{
+    { VMHALTMETHOD_BOOTSTRAP, false, NULL,                NULL,   NULL,                vmR3BootstrapWait,   vmR3BootstrapNotifyCpuFF,   NULL },
+    { VMHALTMETHOD_OLD,       false, NULL,                NULL,   vmR3HaltOldDoHalt,   vmR3DefaultWait,     vmR3DefaultNotifyCpuFF,     NULL },
+    { VMHALTMETHOD_1,         false, vmR3HaltMethod1Init, NULL,   vmR3HaltMethod1Halt, vmR3DefaultWait,     vmR3DefaultNotifyCpuFF,     NULL },
+    { VMHALTMETHOD_GLOBAL_1,   true, vmR3HaltGlobal1Init, NULL,   vmR3HaltGlobal1Halt, vmR3HaltGlobal1Wait, vmR3HaltGlobal1NotifyCpuFF, NULL },
+};
+
+
+/**
+ * Notify the emulation thread (EMT) about pending Forced Action (FF).
+ *
+ * This function is called by thread other than EMT to make
+ * sure EMT wakes up and promptly service an FF request.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
+ * @internal
+ */
+VMMR3_INT_DECL(void) VMR3NotifyGlobalFFU(PUVM pUVM, uint32_t fFlags)
+{
+    LogFlow(("VMR3NotifyGlobalFFU:\n"));
+    uint32_t iHaltMethod = pUVM->vm.s.iHaltMethod;
+
+    if (g_aHaltMethods[iHaltMethod].pfnNotifyGlobalFF) /** @todo make mandatory. */
+        g_aHaltMethods[iHaltMethod].pfnNotifyGlobalFF(pUVM, fFlags);
+    else
+        for (VMCPUID iCpu = 0; iCpu < pUVM->cCpus; iCpu++)
+            g_aHaltMethods[iHaltMethod].pfnNotifyCpuFF(&pUVM->aCpus[iCpu], fFlags);
+}
+
+
+/**
+ * Notify the emulation thread (EMT) about pending Forced Action (FF).
+ *
+ * This function is called by thread other than EMT to make
+ * sure EMT wakes up and promptly service an FF request.
+ *
+ * @param   pUVCpu          Pointer to the user mode per CPU VM structure.
+ * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
+ * @internal
+ */
+VMMR3_INT_DECL(void) VMR3NotifyCpuFFU(PUVMCPU pUVCpu, uint32_t fFlags)
+{
+    PUVM pUVM = pUVCpu->pUVM;
+
+    LogFlow(("VMR3NotifyCpuFFU:\n"));
+    g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnNotifyCpuFF(pUVCpu, fFlags);
+}
+
+
+/**
+ * Halted VM Wait.
+ * Any external event will unblock the thread.
+ *
+ * @returns VINF_SUCCESS unless a fatal error occurred. In the latter
+ *          case an appropriate status code is returned.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   fIgnoreInterrupts   If set the VM_FF_INTERRUPT flags is ignored.
+ * @thread  The emulation thread.
+ * @remarks Made visible for implementing vmsvga sync register.
+ * @internal
+ */
+VMMR3_INT_DECL(int) VMR3WaitHalted(PVM pVM, PVMCPU pVCpu, bool fIgnoreInterrupts)
+{
+    LogFlow(("VMR3WaitHalted: fIgnoreInterrupts=%d\n", fIgnoreInterrupts));
+
+    /*
+     * Check Relevant FFs.
+     */
+    const uint32_t fMask = !fIgnoreInterrupts
+        ? VMCPU_FF_EXTERNAL_HALTED_MASK
+        : VMCPU_FF_EXTERNAL_HALTED_MASK & ~(VMCPU_FF_UPDATE_APIC | VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC);
+    if (    VM_FF_IS_ANY_SET(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+        ||  VMCPU_FF_IS_ANY_SET(pVCpu, fMask))
+    {
+        LogFlow(("VMR3WaitHalted: returns VINF_SUCCESS (FF %#x FFCPU %#RX64)\n", pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * The yielder is suspended while we're halting, while TM might have clock(s) running
+     * only at certain times and need to be notified..
+     */
+    if (pVCpu->idCpu == 0)
+        VMMR3YieldSuspend(pVM);
+    TMNotifyStartOfHalt(pVCpu);
+
+    /*
+     * Record halt averages for the last second.
+     */
+    PUVMCPU pUVCpu = pVCpu->pUVCpu;
+    uint64_t u64Now = RTTimeNanoTS();
+    int64_t off = u64Now - pUVCpu->vm.s.u64HaltsStartTS;
+    if (off > 1000000000)
+    {
+        if (off > _4G || !pUVCpu->vm.s.cHalts)
+        {
+            pUVCpu->vm.s.HaltInterval = 1000000000 /* 1 sec */;
+            pUVCpu->vm.s.HaltFrequency = 1;
+        }
+        else
+        {
+            pUVCpu->vm.s.HaltInterval = (uint32_t)off / pUVCpu->vm.s.cHalts;
+            pUVCpu->vm.s.HaltFrequency = ASMMultU64ByU32DivByU32(pUVCpu->vm.s.cHalts, 1000000000, (uint32_t)off);
+        }
+        pUVCpu->vm.s.u64HaltsStartTS = u64Now;
+        pUVCpu->vm.s.cHalts = 0;
+    }
+    pUVCpu->vm.s.cHalts++;
+
+    /*
+     * Do the halt.
+     */
+    VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED);
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HALTED);
+    PUVM pUVM = pUVCpu->pUVM;
+    int rc = g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnHalt(pUVCpu, fMask, u64Now);
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
+
+    /*
+     * Notify TM and resume the yielder
+     */
+    TMNotifyEndOfHalt(pVCpu);
+    if (pVCpu->idCpu == 0)
+        VMMR3YieldResume(pVM);
+
+    LogFlow(("VMR3WaitHalted: returns %Rrc (FF %#x)\n", rc, pVM->fGlobalForcedActions));
+    return rc;
+}
+
+
+/**
+ * Suspended VM Wait.
+ * Only a handful of forced actions will cause the function to
+ * return to the caller.
+ *
+ * @returns VINF_SUCCESS unless a fatal error occurred. In the latter
+ *          case an appropriate status code is returned.
+ * @param   pUVCpu          Pointer to the user mode VMCPU structure.
+ * @thread  The emulation thread.
+ * @internal
+ */
+VMMR3_INT_DECL(int) VMR3WaitU(PUVMCPU pUVCpu)
+{
+    LogFlow(("VMR3WaitU:\n"));
+
+    /*
+     * Check Relevant FFs.
+     */
+    PVM    pVM   = pUVCpu->pVM;
+    PVMCPU pVCpu = pUVCpu->pVCpu;
+
+    if (    pVM
+        &&  (   VM_FF_IS_ANY_SET(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK)
+             || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_EXTERNAL_SUSPENDED_MASK)
+            )
+        )
+    {
+        LogFlow(("VMR3Wait: returns VINF_SUCCESS (FF %#x)\n", pVM->fGlobalForcedActions));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Do waiting according to the halt method (so VMR3NotifyFF
+     * doesn't have to special case anything).
+     */
+    PUVM pUVM = pUVCpu->pUVM;
+    int rc = g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnWait(pUVCpu);
+    LogFlow(("VMR3WaitU: returns %Rrc (FF %#x)\n", rc, pUVM->pVM ? pUVM->pVM->fGlobalForcedActions : 0));
+    return rc;
+}
+
+
+/**
+ * Interface that PDMR3Suspend, PDMR3PowerOff and PDMR3Reset uses when they wait
+ * for the handling of asynchronous notifications to complete.
+ *
+ * @returns VINF_SUCCESS unless a fatal error occurred. In the latter
+ *          case an appropriate status code is returned.
+ * @param   pUVCpu              Pointer to the user mode VMCPU structure.
+ * @thread  The emulation thread.
+ */
+VMMR3_INT_DECL(int) VMR3AsyncPdmNotificationWaitU(PUVMCPU pUVCpu)
+{
+    LogFlow(("VMR3AsyncPdmNotificationWaitU:\n"));
+    return VMR3WaitU(pUVCpu);
+}
+
+
+/**
+ * Interface that PDM the helper asynchronous notification completed methods
+ * uses for EMT0 when it is waiting inside VMR3AsyncPdmNotificationWaitU().
+ *
+ * @param   pUVM                Pointer to the user mode VM structure.
+ */
+VMMR3_INT_DECL(void) VMR3AsyncPdmNotificationWakeupU(PUVM pUVM)
+{
+    LogFlow(("VMR3AsyncPdmNotificationWakeupU:\n"));
+    VM_FF_SET(pUVM->pVM, VM_FF_REQUEST); /* this will have to do for now. */
+    g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnNotifyCpuFF(&pUVM->aCpus[0], 0 /*fFlags*/);
+}
+
+
+/**
+ * Rendezvous callback that will be called once.
+ *
+ * @returns VBox strict status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   pvUser              The new g_aHaltMethods index.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3SetHaltMethodCallback(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    PUVM      pUVM = pVM->pUVM;
+    int       rc   = VINF_SUCCESS;
+    uintptr_t i    = (uintptr_t)pvUser;
+    Assert(i < RT_ELEMENTS(g_aHaltMethods));
+
+    /*
+     * Main job is done once on EMT0 (it goes thru here first).
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        /*
+         * Terminate the old one.
+         */
+        if (    pUVM->vm.s.enmHaltMethod != VMHALTMETHOD_INVALID
+            &&  g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnTerm)
+        {
+            g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnTerm(pUVM);
+            pUVM->vm.s.enmHaltMethod = VMHALTMETHOD_INVALID;
+        }
+
+        /* Assert that the failure fallback is where we expect. */
+        Assert(g_aHaltMethods[0].enmHaltMethod == VMHALTMETHOD_BOOTSTRAP);
+        Assert(!g_aHaltMethods[0].pfnTerm && !g_aHaltMethods[0].pfnInit);
+
+        /*
+         * Init the new one.
+         */
+        memset(&pUVM->vm.s.Halt, 0, sizeof(pUVM->vm.s.Halt));
+        if (g_aHaltMethods[i].pfnInit)
+        {
+            rc = g_aHaltMethods[i].pfnInit(pUVM);
+            if (RT_FAILURE(rc))
+            {
+                /* Fall back on the bootstrap method. This requires no
+                   init/term (see assertion above), and will always work. */
+                AssertLogRelRC(rc);
+                i = 0;
+            }
+        }
+
+        /*
+         * Commit it.
+         */
+        pUVM->vm.s.enmHaltMethod = g_aHaltMethods[i].enmHaltMethod;
+        ASMAtomicWriteU32(&pUVM->vm.s.iHaltMethod, i);
+    }
+    else
+        i = pUVM->vm.s.iHaltMethod;
+
+    /*
+     * All EMTs must update their ring-0 halt configuration.
+     */
+    VMMR3SetMayHaltInRing0(pVCpu, g_aHaltMethods[i].fMayHaltInRing0,
+                           g_aHaltMethods[i].enmHaltMethod == VMHALTMETHOD_GLOBAL_1
+                           ? pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg : 0);
+
+    return rc;
+}
+
+
+/**
+ * Changes the halt method.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   enmHaltMethod   The new halt method.
+ * @thread  EMT.
+ */
+int vmR3SetHaltMethodU(PUVM pUVM, VMHALTMETHOD enmHaltMethod)
+{
+    PVM pVM = pUVM->pVM; Assert(pVM);
+    VM_ASSERT_EMT(pVM);
+    AssertReturn(enmHaltMethod > VMHALTMETHOD_INVALID && enmHaltMethod < VMHALTMETHOD_END, VERR_INVALID_PARAMETER);
+
+    /*
+     * Resolve default (can be overridden in the configuration).
+     */
+    if (enmHaltMethod == VMHALTMETHOD_DEFAULT)
+    {
+        uint32_t u32;
+        int rc = CFGMR3QueryU32(CFGMR3GetChild(CFGMR3GetRoot(pVM), "VM"), "HaltMethod", &u32);
+        if (RT_SUCCESS(rc))
+        {
+            enmHaltMethod = (VMHALTMETHOD)u32;
+            if (enmHaltMethod <= VMHALTMETHOD_INVALID || enmHaltMethod >= VMHALTMETHOD_END)
+                return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, N_("Invalid VM/HaltMethod value %d"), enmHaltMethod);
+        }
+        else if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_CHILD_NOT_FOUND)
+            return VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to Query VM/HaltMethod as uint32_t"));
+        else
+            enmHaltMethod = VMHALTMETHOD_GLOBAL_1;
+            //enmHaltMethod = VMHALTMETHOD_1;
+            //enmHaltMethod = VMHALTMETHOD_OLD;
+    }
+    LogRel(("VMEmt: Halt method %s (%d)\n", vmR3GetHaltMethodName(enmHaltMethod), enmHaltMethod));
+
+    /*
+     * Find the descriptor.
+     */
+    unsigned i = 0;
+    while (     i < RT_ELEMENTS(g_aHaltMethods)
+           &&   g_aHaltMethods[i].enmHaltMethod != enmHaltMethod)
+        i++;
+    AssertReturn(i < RT_ELEMENTS(g_aHaltMethods), VERR_INVALID_PARAMETER);
+
+    /*
+     * This needs to be done while the other EMTs are not sleeping or otherwise messing around.
+     */
+    return VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING, vmR3SetHaltMethodCallback, (void *)(uintptr_t)i);
+}
+
+
+/**
+ * Special interface for implementing a HLT-like port on a device.
+ *
+ * This can be called directly from device code, provide the device is trusted
+ * to access the VMM directly.  Since we may not have an accurate register set
+ * and the caller certainly shouldn't (device code does not access CPU
+ * registers), this function will return when interrupts are pending regardless
+ * of the actual EFLAGS.IF state.
+ *
+ * @returns VBox error status (never informational statuses).
+ * @param   pVM                 The cross context VM structure.
+ * @param   idCpu               The id of the calling EMT.
+ */
+VMMR3DECL(int) VMR3WaitForDeviceReady(PVM pVM, VMCPUID idCpu)
+{
+    /*
+     * Validate caller and resolve the CPU ID.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+    PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+    VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
+
+    /*
+     * Tag along with the HLT mechanics for now.
+     */
+    int rc = VMR3WaitHalted(pVM, pVCpu, false /*fIgnoreInterrupts*/);
+    if (RT_SUCCESS(rc))
+        return VINF_SUCCESS;
+    return rc;
+}
+
+
+/**
+ * Wakes up a CPU that has called VMR3WaitForDeviceReady.
+ *
+ * @returns VBox error status (never informational statuses).
+ * @param   pVM                 The cross context VM structure.
+ * @param   idCpu               The id of the calling EMT.
+ */
+VMMR3DECL(int) VMR3NotifyCpuDeviceReady(PVM pVM, VMCPUID idCpu)
+{
+    /*
+     * Validate caller and resolve the CPU ID.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+    PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+
+    /*
+     * Pretend it was an FF that got set since we've got logic for that already.
+     */
+    VMR3NotifyCpuFFU(pVCpu->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Returns the number of active EMTs.
+ *
+ * This is used by the rendezvous code during VM destruction to avoid waiting
+ * for EMTs that aren't around any more.
+ *
+ * @returns Number of active EMTs.  0 if invalid parameter.
+ * @param   pUVM                The user mode VM structure.
+ */
+VMMR3_INT_DECL(uint32_t) VMR3GetActiveEmts(PUVM pUVM)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, 0);
+    return pUVM->vm.s.cActiveEmts;
+}
+
diff --git a/src/VBox/VMM/VMMR3/VMM.cpp b/src/VBox/VMM/VMMR3/VMM.cpp
new file mode 100644
index 00000000..c26ee07f
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/VMM.cpp
@@ -0,0 +1,2650 @@
+/* $Id: VMM.cpp $ */
+/** @file
+ * VMM - The Virtual Machine Monitor Core.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+//#define NO_SUPCALLR0VMM
+
+/** @page pg_vmm        VMM - The Virtual Machine Monitor
+ *
+ * The VMM component is two things at the moment, it's a component doing a few
+ * management and routing tasks, and it's the whole virtual machine monitor
+ * thing.  For hysterical reasons, it is not doing all the management that one
+ * would expect, this is instead done by @ref pg_vm.  We'll address this
+ * misdesign eventually, maybe.
+ *
+ * VMM is made up of these components:
+ *  - @subpage pg_cfgm
+ *  - @subpage pg_cpum
+ *  - @subpage pg_dbgf
+ *  - @subpage pg_em
+ *  - @subpage pg_gim
+ *  - @subpage pg_gmm
+ *  - @subpage pg_gvmm
+ *  - @subpage pg_hm
+ *  - @subpage pg_iem
+ *  - @subpage pg_iom
+ *  - @subpage pg_mm
+ *  - @subpage pg_nem
+ *  - @subpage pg_pdm
+ *  - @subpage pg_pgm
+ *  - @subpage pg_selm
+ *  - @subpage pg_ssm
+ *  - @subpage pg_stam
+ *  - @subpage pg_tm
+ *  - @subpage pg_trpm
+ *  - @subpage pg_vm
+ *
+ *
+ * @see @ref grp_vmm @ref grp_vm @subpage pg_vmm_guideline @subpage pg_raw
+ *
+ *
+ * @section sec_vmmstate        VMM State
+ *
+ * @image html VM_Statechart_Diagram.gif
+ *
+ * To be written.
+ *
+ *
+ * @subsection  subsec_vmm_init     VMM Initialization
+ *
+ * To be written.
+ *
+ *
+ * @subsection  subsec_vmm_term     VMM Termination
+ *
+ * To be written.
+ *
+ *
+ * @section sec_vmm_limits     VMM Limits
+ *
+ * There are various resource limits imposed by the VMM and it's
+ * sub-components.  We'll list some of them here.
+ *
+ * On 64-bit hosts:
+ *      - Max 8191 VMs.  Imposed by GVMM's handle allocation (GVMM_MAX_HANDLES),
+ *        can be increased up to 64K - 1.
+ *      - Max 16TB - 64KB of the host memory can be used for backing VM RAM and
+ *        ROM pages.  The limit is imposed by the 32-bit page ID used by GMM.
+ *      - A VM can be assigned all the memory we can use (16TB), however, the
+ *        Main API will restrict this to 2TB (MM_RAM_MAX_IN_MB).
+ *      - Max 32 virtual CPUs (VMM_MAX_CPU_COUNT).
+ *
+ * On 32-bit hosts:
+ *      - Max 127 VMs.  Imposed by GMM's per page structure.
+ *      - Max 64GB - 64KB of the host memory can be used for backing VM RAM and
+ *        ROM pages.  The limit is imposed by the 28-bit page ID used
+ *        internally in GMM.  It is also limited by PAE.
+ *      - A VM can be assigned all the memory GMM can allocate, however, the
+ *        Main API will restrict this to 3584MB (MM_RAM_MAX_IN_MB).
+ *      - Max 32 virtual CPUs (VMM_MAX_CPU_COUNT).
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/vmapi.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/pdmqueue.h>
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/pdmcritsectrw.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/nem.h>
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+# include <VBox/vmm/iem.h>
+#endif
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/tm.h>
+#include "VMMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/version.h>
+#include <VBox/vmm/hm.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/asm.h>
+#include <iprt/time.h>
+#include <iprt/semaphore.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+#include <iprt/stdarg.h>
+#include <iprt/ctype.h>
+#include <iprt/x86.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The saved state version. */
+#define VMM_SAVED_STATE_VERSION     4
+/** The saved state version used by v3.0 and earlier. (Teleportation) */
+#define VMM_SAVED_STATE_VERSION_3_0 3
+
+/** Macro for flushing the ring-0 logging. */
+#define VMM_FLUSH_R0_LOG(a_pR0Logger, a_pR3Logger) \
+    do { \
+        PVMMR0LOGGER pVmmLogger = (a_pR0Logger); \
+        if (!pVmmLogger || pVmmLogger->Logger.offScratch == 0) \
+        { /* likely? */ } \
+        else \
+            RTLogFlushR0(a_pR3Logger, &pVmmLogger->Logger); \
+    } while (0)
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int                  vmmR3InitStacks(PVM pVM);
+static int                  vmmR3InitLoggers(PVM pVM);
+static void                 vmmR3InitRegisterStats(PVM pVM);
+static DECLCALLBACK(int)    vmmR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)    vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(void)   vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser);
+static VBOXSTRICTRC         vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
+                                                     uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser);
+static int                  vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu);
+static DECLCALLBACK(void)   vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+/**
+ * Initializes the VMM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) VMMR3Init(PVM pVM)
+{
+    LogFlow(("VMMR3Init\n"));
+
+    /*
+     * Assert alignment, sizes and order.
+     */
+    AssertCompile(sizeof(pVM->vmm.s) <= sizeof(pVM->vmm.padding));
+    AssertCompile(RT_SIZEOFMEMB(VMCPU, vmm.s) <= RT_SIZEOFMEMB(VMCPU, vmm.padding));
+
+    /*
+     * Init basic VM VMM members.
+     */
+    pVM->vmm.s.pahEvtRendezvousEnterOrdered     = NULL;
+    pVM->vmm.s.hEvtRendezvousEnterOneByOne      = NIL_RTSEMEVENT;
+    pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce  = NIL_RTSEMEVENTMULTI;
+    pVM->vmm.s.hEvtMulRendezvousDone            = NIL_RTSEMEVENTMULTI;
+    pVM->vmm.s.hEvtRendezvousDoneCaller         = NIL_RTSEMEVENT;
+    pVM->vmm.s.hEvtMulRendezvousRecursionPush   = NIL_RTSEMEVENTMULTI;
+    pVM->vmm.s.hEvtMulRendezvousRecursionPop    = NIL_RTSEMEVENTMULTI;
+    pVM->vmm.s.hEvtRendezvousRecursionPushCaller = NIL_RTSEMEVENT;
+    pVM->vmm.s.hEvtRendezvousRecursionPopCaller = NIL_RTSEMEVENT;
+
+    /** @cfgm{/YieldEMTInterval, uint32_t, 1, UINT32_MAX, 23, ms}
+     * The EMT yield interval.  The EMT yielding is a hack we employ to play a
+     * bit nicer with the rest of the system (like for instance the GUI).
+     */
+    int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "YieldEMTInterval", &pVM->vmm.s.cYieldEveryMillies,
+                               23 /* Value arrived at after experimenting with the grub boot prompt. */);
+    AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"YieldEMTInterval\", rc=%Rrc\n", rc), rc);
+
+
+    /** @cfgm{/VMM/UsePeriodicPreemptionTimers, boolean, true}
+     * Controls whether we employ per-cpu preemption timers to limit the time
+     * spent executing guest code.  This option is not available on all
+     * platforms and we will silently ignore this setting then.  If we are
+     * running in VT-x mode, we will use the VMX-preemption timer instead of
+     * this one when possible.
+     */
+    PCFGMNODE pCfgVMM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "VMM");
+    rc = CFGMR3QueryBoolDef(pCfgVMM, "UsePeriodicPreemptionTimers", &pVM->vmm.s.fUsePeriodicPreemptionTimers, true);
+    AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"VMM/UsePeriodicPreemptionTimers\", rc=%Rrc\n", rc), rc);
+
+    /*
+     * Initialize the VMM rendezvous semaphores.
+     */
+    pVM->vmm.s.pahEvtRendezvousEnterOrdered = (PRTSEMEVENT)MMR3HeapAlloc(pVM, MM_TAG_VMM, sizeof(RTSEMEVENT) * pVM->cCpus);
+    if (!pVM->vmm.s.pahEvtRendezvousEnterOrdered)
+        return VERR_NO_MEMORY;
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        rc = RTSemEventCreate(&pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
+        AssertRCReturn(rc, rc);
+    }
+    rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousEnterOneByOne);
+    AssertRCReturn(rc, rc);
+    rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
+    AssertRCReturn(rc, rc);
+    rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousDone);
+    AssertRCReturn(rc, rc);
+    rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousDoneCaller);
+    AssertRCReturn(rc, rc);
+    rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousRecursionPush);
+    AssertRCReturn(rc, rc);
+    rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousRecursionPop);
+    AssertRCReturn(rc, rc);
+    rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
+    AssertRCReturn(rc, rc);
+    rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Register the saved state data unit.
+     */
+    rc = SSMR3RegisterInternal(pVM, "vmm", 1, VMM_SAVED_STATE_VERSION, VMM_STACK_SIZE + sizeof(RTGCPTR),
+                               NULL, NULL, NULL,
+                               NULL, vmmR3Save, NULL,
+                               NULL, vmmR3Load, NULL);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Register the Ring-0 VM handle with the session for fast ioctl calls.
+     */
+    rc = SUPR3SetVMForFastIOCtl(VMCC_GET_VMR0_FOR_CALL(pVM));
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Init various sub-components.
+     */
+    rc = vmmR3InitStacks(pVM);
+    if (RT_SUCCESS(rc))
+    {
+        rc = vmmR3InitLoggers(pVM);
+
+#ifdef VBOX_WITH_NMI
+        /*
+         * Allocate mapping for the host APIC.
+         */
+        if (RT_SUCCESS(rc))
+        {
+            rc = MMR3HyperReserve(pVM, PAGE_SIZE, "Host APIC", &pVM->vmm.s.GCPtrApicBase);
+            AssertRC(rc);
+        }
+#endif
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Debug info and statistics.
+             */
+            DBGFR3InfoRegisterInternal(pVM, "fflags", "Displays the current Forced actions Flags.", vmmR3InfoFF);
+            vmmR3InitRegisterStats(pVM);
+            vmmInitFormatTypes();
+
+            return VINF_SUCCESS;
+        }
+    }
+    /** @todo Need failure cleanup? */
+
+    return rc;
+}
+
+
+/**
+ * Allocate & setup the VMM RC stack(s) (for EMTs).
+ *
+ * The stacks are also used for long jumps in Ring-0.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ *
+ * @remarks The optional guard page gets it protection setup up during R3 init
+ *          completion because of init order issues.
+ */
+static int vmmR3InitStacks(PVM pVM)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VMM_R0_SWITCH_STACK
+    uint32_t fFlags = MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
+#else
+    uint32_t fFlags = 0;
+#endif
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+
+#ifdef VBOX_STRICT_VMM_STACK
+        rc = MMR3HyperAllocOnceNoRelEx(pVM, PAGE_SIZE + VMM_STACK_SIZE + PAGE_SIZE,
+#else
+        rc = MMR3HyperAllocOnceNoRelEx(pVM, VMM_STACK_SIZE,
+#endif
+                                       PAGE_SIZE, MM_TAG_VMM, fFlags, (void **)&pVCpu->vmm.s.pbEMTStackR3);
+        if (RT_SUCCESS(rc))
+        {
+#ifdef VBOX_STRICT_VMM_STACK
+            pVCpu->vmm.s.pbEMTStackR3 += PAGE_SIZE;
+#endif
+            pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
+
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Initialize the loggers.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+static int vmmR3InitLoggers(PVM pVM)
+{
+    int rc;
+#define RTLogCalcSizeForR0(cGroups, fFlags) (RT_UOFFSETOF_DYN(VMMR0LOGGER, Logger.afGroups[cGroups]) + PAGE_SIZE)
+
+    /*
+     * Allocate R0 Logger instance (finalized in the relocator).
+     */
+#if defined(LOG_ENABLED) && defined(VBOX_WITH_R0_LOGGING)
+    PRTLOGGER pLogger = RTLogDefaultInstance();
+    if (pLogger)
+    {
+        size_t const cbLogger = RTLogCalcSizeForR0(pLogger->cGroups, 0);
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+            rc = MMR3HyperAllocOnceNoRelEx(pVM, cbLogger, PAGE_SIZE, MM_TAG_VMM, MMHYPER_AONR_FLAGS_KERNEL_MAPPING,
+                                           (void **)&pVCpu->vmm.s.pR0LoggerR3);
+            if (RT_FAILURE(rc))
+                return rc;
+            pVCpu->vmm.s.pR0LoggerR3->pVM        = VMCC_GET_VMR0_FOR_CALL(pVM);
+            //pVCpu->vmm.s.pR0LoggerR3->fCreated = false;
+            pVCpu->vmm.s.pR0LoggerR3->cbLogger   = (uint32_t)cbLogger;
+            pVCpu->vmm.s.pR0LoggerR0 = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pR0LoggerR3);
+        }
+    }
+#endif /* LOG_ENABLED && VBOX_WITH_R0_LOGGING */
+
+    /*
+     * Release logging.
+     */
+    PRTLOGGER pRelLogger = RTLogRelGetDefaultInstance();
+    if (pRelLogger)
+    {
+        /*
+         * Ring-0 release logger.
+         */
+        RTR0PTR pfnLoggerWrapper = NIL_RTR0PTR;
+        rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerWrapper", &pfnLoggerWrapper);
+        AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerWrapper not found! rc=%Rra\n", rc), rc);
+
+        RTR0PTR pfnLoggerFlush = NIL_RTR0PTR;
+        rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerFlush", &pfnLoggerFlush);
+        AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerFlush not found! rc=%Rra\n", rc), rc);
+
+        size_t const cbLogger = RTLogCalcSizeForR0(pRelLogger->cGroups, 0);
+
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+            rc = MMR3HyperAllocOnceNoRelEx(pVM, cbLogger, PAGE_SIZE, MM_TAG_VMM, MMHYPER_AONR_FLAGS_KERNEL_MAPPING,
+                                           (void **)&pVCpu->vmm.s.pR0RelLoggerR3);
+            if (RT_FAILURE(rc))
+                return rc;
+            PVMMR0LOGGER pVmmLogger = pVCpu->vmm.s.pR0RelLoggerR3;
+            RTR0PTR      R0PtrVmmLogger = MMHyperR3ToR0(pVM, pVmmLogger);
+            pVCpu->vmm.s.pR0RelLoggerR0     = R0PtrVmmLogger;
+            pVmmLogger->pVM                 = VMCC_GET_VMR0_FOR_CALL(pVM);
+            pVmmLogger->cbLogger            = (uint32_t)cbLogger;
+            pVmmLogger->fCreated            = false;
+            pVmmLogger->fFlushingDisabled   = false;
+            pVmmLogger->fRegistered         = false;
+            pVmmLogger->idCpu               = idCpu;
+
+            char szR0ThreadName[16];
+            RTStrPrintf(szR0ThreadName, sizeof(szR0ThreadName), "EMT-%u-R0", idCpu);
+            rc = RTLogCreateForR0(&pVmmLogger->Logger, pVmmLogger->cbLogger, R0PtrVmmLogger + RT_UOFFSETOF(VMMR0LOGGER, Logger),
+                                  pfnLoggerWrapper, pfnLoggerFlush,
+                                  RTLOGFLAGS_BUFFERED, RTLOGDEST_DUMMY, szR0ThreadName);
+            AssertReleaseMsgRCReturn(rc, ("RTLogCreateForR0 failed! rc=%Rra\n", rc), rc);
+
+            /* We only update the release log instance here. */
+            rc = RTLogCopyGroupsAndFlagsForR0(&pVmmLogger->Logger, R0PtrVmmLogger + RT_UOFFSETOF(VMMR0LOGGER, Logger),
+                                              pRelLogger, RTLOGFLAGS_BUFFERED, UINT32_MAX);
+            AssertReleaseMsgRCReturn(rc, ("RTLogCopyGroupsAndFlagsForR0 failed! rc=%Rra\n", rc), rc);
+
+            pVmmLogger->fCreated = true;
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VMMR3Init worker that register the statistics with STAM.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+static void vmmR3InitRegisterStats(PVM pVM)
+{
+    RT_NOREF_PV(pVM);
+
+    /*
+     * Statistics.
+     */
+    STAM_REG(pVM, &pVM->vmm.s.StatRunGC,                    STAMTYPE_COUNTER, "/VMM/RunGC",                     STAMUNIT_OCCURENCES, "Number of context switches.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetNormal,              STAMTYPE_COUNTER, "/VMM/RZRet/Normal",              STAMUNIT_OCCURENCES, "Number of VINF_SUCCESS returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterrupt,           STAMTYPE_COUNTER, "/VMM/RZRet/Interrupt",           STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptHyper,      STAMTYPE_COUNTER, "/VMM/RZRet/InterruptHyper",      STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_HYPER returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetGuestTrap,           STAMTYPE_COUNTER, "/VMM/RZRet/GuestTrap",           STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_GUEST_TRAP returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitch,          STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitch",          STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitchInt,       STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitchInt",       STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH_INT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetStaleSelector,       STAMTYPE_COUNTER, "/VMM/RZRet/StaleSelector",       STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_STALE_SELECTOR returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIRETTrap,            STAMTYPE_COUNTER, "/VMM/RZRet/IRETTrap",            STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_IRET_TRAP returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetEmulate,             STAMTYPE_COUNTER, "/VMM/RZRet/Emulate",             STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchEmulate,        STAMTYPE_COUNTER, "/VMM/RZRet/PatchEmulate",        STAMUNIT_OCCURENCES, "Number of VINF_PATCH_EMULATE_INSTR returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIORead,              STAMTYPE_COUNTER, "/VMM/RZRet/IORead",              STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_READ returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOWrite,             STAMTYPE_COUNTER, "/VMM/RZRet/IOWrite",             STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_WRITE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOCommitWrite,       STAMTYPE_COUNTER, "/VMM/RZRet/IOCommitWrite",       STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_COMMIT_WRITE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIORead,            STAMTYPE_COUNTER, "/VMM/RZRet/MMIORead",            STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_READ returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOWrite,           STAMTYPE_COUNTER, "/VMM/RZRet/MMIOWrite",           STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_WRITE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOCommitWrite,     STAMTYPE_COUNTER, "/VMM/RZRet/MMIOCommitWrite",     STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_COMMIT_WRITE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOReadWrite,       STAMTYPE_COUNTER, "/VMM/RZRet/MMIOReadWrite",       STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_READ_WRITE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchRead,       STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchRead",       STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_READ returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchWrite,      STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchWrite",      STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_WRITE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMSRRead,             STAMTYPE_COUNTER, "/VMM/RZRet/MSRRead",             STAMUNIT_OCCURENCES, "Number of VINF_CPUM_R3_MSR_READ returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMSRWrite,            STAMTYPE_COUNTER, "/VMM/RZRet/MSRWrite",            STAMUNIT_OCCURENCES, "Number of VINF_CPUM_R3_MSR_WRITE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetLDTFault,            STAMTYPE_COUNTER, "/VMM/RZRet/LDTFault",            STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_GDT_FAULT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetGDTFault,            STAMTYPE_COUNTER, "/VMM/RZRet/GDTFault",            STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_LDT_FAULT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIDTFault,            STAMTYPE_COUNTER, "/VMM/RZRet/IDTFault",            STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_IDT_FAULT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetTSSFault,            STAMTYPE_COUNTER, "/VMM/RZRet/TSSFault",            STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_TSS_FAULT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetCSAMTask,            STAMTYPE_COUNTER, "/VMM/RZRet/CSAMTask",            STAMUNIT_OCCURENCES, "Number of VINF_CSAM_PENDING_ACTION returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetSyncCR3,             STAMTYPE_COUNTER, "/VMM/RZRet/SyncCR",              STAMUNIT_OCCURENCES, "Number of VINF_PGM_SYNC_CR3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMisc,                STAMTYPE_COUNTER, "/VMM/RZRet/Misc",                STAMUNIT_OCCURENCES, "Number of misc returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchInt3,           STAMTYPE_COUNTER, "/VMM/RZRet/PatchInt3",           STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_INT3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchPF,             STAMTYPE_COUNTER, "/VMM/RZRet/PatchPF",             STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_PF returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchGP,             STAMTYPE_COUNTER, "/VMM/RZRet/PatchGP",             STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_GP returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchIretIRQ,        STAMTYPE_COUNTER, "/VMM/RZRet/PatchIret",           STAMUNIT_OCCURENCES, "Number of VINF_PATM_PENDING_IRQ_AFTER_IRET returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetRescheduleREM,       STAMTYPE_COUNTER, "/VMM/RZRet/ScheduleREM",         STAMUNIT_OCCURENCES, "Number of VINF_EM_RESCHEDULE_REM returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Total,           STAMTYPE_COUNTER, "/VMM/RZRet/ToR3",                STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Unknown,         STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Unknown",        STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns without responsible force flag.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3FF,              STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/ToR3",           STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_TO_R3.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3TMVirt,          STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/TMVirt",         STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_TM_VIRTUAL_SYNC.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3HandyPages,      STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Handy",          STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_PGM_NEED_HANDY_PAGES.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3PDMQueues,       STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/PDMQueue",       STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_PDM_QUEUES.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Rendezvous,      STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Rendezvous",     STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_EMT_RENDEZVOUS.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Timer,           STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Timer",          STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_TIMER.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3DMA,             STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/DMA",            STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_PDM_DMA.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3CritSect,        STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/CritSect",       STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_PDM_CRITSECT.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Iem,             STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/IEM",            STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_IEM.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Iom,             STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/IOM",            STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_IOM.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetTimerPending,        STAMTYPE_COUNTER, "/VMM/RZRet/TimerPending",        STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TIMER_PENDING returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptPending,    STAMTYPE_COUNTER, "/VMM/RZRet/InterruptPending",    STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_PENDING returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPATMDuplicateFn,     STAMTYPE_COUNTER, "/VMM/RZRet/PATMDuplicateFn",     STAMUNIT_OCCURENCES, "Number of VINF_PATM_DUPLICATE_FUNCTION returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMChangeMode,       STAMTYPE_COUNTER, "/VMM/RZRet/PGMChangeMode",       STAMUNIT_OCCURENCES, "Number of VINF_PGM_CHANGE_MODE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMFlushPending,     STAMTYPE_COUNTER, "/VMM/RZRet/PGMFlushPending",     STAMUNIT_OCCURENCES, "Number of VINF_PGM_POOL_FLUSH_PENDING returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPendingRequest,      STAMTYPE_COUNTER, "/VMM/RZRet/PendingRequest",      STAMUNIT_OCCURENCES, "Number of VINF_EM_PENDING_REQUEST returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchTPR,            STAMTYPE_COUNTER, "/VMM/RZRet/PatchTPR",            STAMUNIT_OCCURENCES, "Number of VINF_EM_HM_PATCH_TPR_INSTR returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetCallRing3,           STAMTYPE_COUNTER, "/VMM/RZCallR3/Misc",             STAMUNIT_OCCURENCES, "Number of Other ring-3 calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMLock,            STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMLock",          STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_LOCK calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMCritSectEnter,   STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMCritSectEnter", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_CRITSECT_ENTER calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMLock,            STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMLock",          STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_LOCK calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMPoolGrow,        STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMPoolGrow",      STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_POOL_GROW calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMMapChunk,        STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMMapChunk",      STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_MAP_CHUNK calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMAllocHandy,      STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMAllocHandy",    STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallLogFlush,           STAMTYPE_COUNTER, "/VMM/RZCallR3/VMMLogFlush",      STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VMM_LOGGER_FLUSH calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetError,         STAMTYPE_COUNTER, "/VMM/RZCallR3/VMSetError",       STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_ERROR calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetRuntimeError,  STAMTYPE_COUNTER, "/VMM/RZCallR3/VMRuntimeError",   STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_RUNTIME_ERROR calls.");
+
+#ifdef VBOX_WITH_STATISTICS
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedMax,  STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,      "Max amount of stack used.", "/VMM/Stack/CPU%u/Max", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedAvg,  STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,      "Average stack usage.",      "/VMM/Stack/CPU%u/Avg", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.CallRing3JmpBufR0.cUsedTotal, STAMTYPE_U64,       STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of stack usages.",   "/VMM/Stack/CPU%u/Uses", i);
+    }
+#endif
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = pVM->apCpusR3[i];
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltBlock,          STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "", "/PROF/CPU%u/VM/Halt/R0HaltBlock", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltBlockOnTime,    STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "", "/PROF/CPU%u/VM/Halt/R0HaltBlockOnTime", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltBlockOverslept, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "", "/PROF/CPU%u/VM/Halt/R0HaltBlockOverslept", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltBlockInsomnia,  STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "", "/PROF/CPU%u/VM/Halt/R0HaltBlockInsomnia", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltExec,           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltExec", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltExecFromSpin,   STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltExec/FromSpin", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltExecFromBlock,  STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltExec/FromBlock", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3,           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltToR3", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3FromSpin,   STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltToR3/FromSpin", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3Other,      STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltToR3/Other", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3PendingFF,  STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltToR3/PendingFF", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3SmallDelta, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltToR3/SmallDelta", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3PostNoInt,  STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltToR3/PostWaitNoInt", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3PostPendingFF,STAMTYPE_COUNTER,STAMVISIBILITY_ALWAYS,STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltToR3/PostWaitPendingFF", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.cR0Halts,                 STAMTYPE_U32,     STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltHistoryCounter", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.cR0HaltsSucceeded,        STAMTYPE_U32,     STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltHistorySucceeded", i);
+        STAMR3RegisterF(pVM, &pVCpu->vmm.s.cR0HaltsToRing3,          STAMTYPE_U32,     STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,  "", "/PROF/CPU%u/VM/Halt/R0HaltHistoryToRing3", i);
+    }
+}
+
+
+/**
+ * Worker for VMMR3InitR0 that calls ring-0 to do EMT specific initialization.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context per CPU structure.
+ * @thread  EMT(pVCpu)
+ */
+static DECLCALLBACK(int) vmmR3InitR0Emt(PVM pVM, PVMCPU pVCpu)
+{
+    return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_VMMR0_INIT_EMT, 0, NULL);
+}
+
+
+/**
+ * Initializes the R0 VMM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) VMMR3InitR0(PVM pVM)
+{
+    int    rc;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    Assert(pVCpu && pVCpu->idCpu == 0);
+
+#ifdef LOG_ENABLED
+    /*
+     * Initialize the ring-0 logger if we haven't done so yet.
+     */
+    if (    pVCpu->vmm.s.pR0LoggerR3
+        &&  !pVCpu->vmm.s.pR0LoggerR3->fCreated)
+    {
+        rc = VMMR3UpdateLoggers(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+#endif
+
+    /*
+     * Call Ring-0 entry with init code.
+     */
+    for (;;)
+    {
+#ifdef NO_SUPCALLR0VMM
+        //rc = VERR_GENERAL_FAILURE;
+        rc = VINF_SUCCESS;
+#else
+        rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), 0 /*idCpu*/, VMMR0_DO_VMMR0_INIT, RT_MAKE_U64(VMMGetSvnRev(), vmmGetBuildType()), NULL);
+#endif
+        /*
+         * Flush the logs.
+         */
+#ifdef LOG_ENABLED
+        VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0LoggerR3, NULL);
+#endif
+        VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0RelLoggerR3, RTLogRelGetDefaultInstance());
+        if (rc != VINF_VMM_CALL_HOST)
+            break;
+        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
+            break;
+        /* Resume R0 */
+    }
+
+    if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
+    {
+        LogRel(("VMM: R0 init failed, rc=%Rra\n", rc));
+        if (RT_SUCCESS(rc))
+            rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
+    }
+
+    /* Log whether thread-context hooks are used (on Linux this can depend on how the kernel is configured). */
+    if (pVM->apCpusR3[0]->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
+        LogRel(("VMM: Enabled thread-context hooks\n"));
+    else
+        LogRel(("VMM: Thread-context hooks unavailable\n"));
+
+    /* Log RTThreadPreemptIsPendingTrusty() and RTThreadPreemptIsPossible() results. */
+    if (pVM->vmm.s.fIsPreemptPendingApiTrusty)
+        LogRel(("VMM: RTThreadPreemptIsPending() can be trusted\n"));
+    else
+        LogRel(("VMM: Warning! RTThreadPreemptIsPending() cannot be trusted!  Need to update kernel info?\n"));
+    if (pVM->vmm.s.fIsPreemptPossible)
+        LogRel(("VMM: Kernel preemption is possible\n"));
+    else
+        LogRel(("VMM: Kernel preemption is not possible it seems\n"));
+
+    /*
+     * Send all EMTs to ring-0 to get their logger initialized.
+     */
+    for (VMCPUID idCpu = 0; RT_SUCCESS(rc) && idCpu < pVM->cCpus; idCpu++)
+        rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)vmmR3InitR0Emt, 2, pVM, pVM->apCpusR3[idCpu]);
+
+    return rc;
+}
+
+
+/**
+ * Called when an init phase completes.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   enmWhat             Which init phase.
+ */
+VMMR3_INT_DECL(int) VMMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    int rc = VINF_SUCCESS;
+
+    switch (enmWhat)
+    {
+        case VMINITCOMPLETED_RING3:
+        {
+            /*
+             * Create the EMT yield timer.
+             */
+            rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, vmmR3YieldEMT, NULL, "EMT Yielder", &pVM->vmm.s.pYieldTimer);
+            AssertRCReturn(rc, rc);
+
+            rc = TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldEveryMillies);
+            AssertRCReturn(rc, rc);
+            break;
+        }
+
+        case VMINITCOMPLETED_HM:
+        {
+            /*
+             * Disable the periodic preemption timers if we can use the
+             * VMX-preemption timer instead.
+             */
+            if (   pVM->vmm.s.fUsePeriodicPreemptionTimers
+                && HMR3IsVmxPreemptionTimerUsed(pVM))
+                pVM->vmm.s.fUsePeriodicPreemptionTimers = false;
+            LogRel(("VMM: fUsePeriodicPreemptionTimers=%RTbool\n", pVM->vmm.s.fUsePeriodicPreemptionTimers));
+
+            /*
+             * Last chance for GIM to update its CPUID leaves if it requires
+             * knowledge/information from HM initialization.
+             */
+            rc = GIMR3InitCompleted(pVM);
+            AssertRCReturn(rc, rc);
+
+            /*
+             * CPUM's post-initialization (print CPUIDs).
+             */
+            CPUMR3LogCpuIdAndMsrFeatures(pVM);
+            break;
+        }
+
+        default: /* shuts up gcc */
+            break;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Terminate the VMM bits.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) VMMR3Term(PVM pVM)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    Assert(pVCpu && pVCpu->idCpu == 0);
+
+    /*
+     * Call Ring-0 entry with termination code.
+     */
+    int rc;
+    for (;;)
+    {
+#ifdef NO_SUPCALLR0VMM
+        //rc = VERR_GENERAL_FAILURE;
+        rc = VINF_SUCCESS;
+#else
+        rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), 0 /*idCpu*/, VMMR0_DO_VMMR0_TERM, 0, NULL);
+#endif
+        /*
+         * Flush the logs.
+         */
+#ifdef LOG_ENABLED
+        VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0LoggerR3, NULL);
+#endif
+        VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0RelLoggerR3, RTLogRelGetDefaultInstance());
+        if (rc != VINF_VMM_CALL_HOST)
+            break;
+        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
+            break;
+        /* Resume R0 */
+    }
+    if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
+    {
+        LogRel(("VMM: VMMR3Term: R0 term failed, rc=%Rra. (warning)\n", rc));
+        if (RT_SUCCESS(rc))
+            rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
+    }
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        RTSemEventDestroy(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
+        pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
+    }
+    RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
+    pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
+    RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
+    pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
+    RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousDone);
+    pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
+    RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousDoneCaller);
+    pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
+    RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousRecursionPush);
+    pVM->vmm.s.hEvtMulRendezvousRecursionPush = NIL_RTSEMEVENTMULTI;
+    RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousRecursionPop);
+    pVM->vmm.s.hEvtMulRendezvousRecursionPop = NIL_RTSEMEVENTMULTI;
+    RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
+    pVM->vmm.s.hEvtRendezvousRecursionPushCaller = NIL_RTSEMEVENT;
+    RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
+    pVM->vmm.s.hEvtRendezvousRecursionPopCaller = NIL_RTSEMEVENT;
+
+    vmmTermFormatTypes();
+    return rc;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * The VMM will need to apply relocations to the core code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   offDelta    The relocation delta.
+ */
+VMMR3_INT_DECL(void) VMMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    LogFlow(("VMMR3Relocate: offDelta=%RGv\n", offDelta));
+    RT_NOREF(offDelta);
+
+    /*
+     * Update the logger.
+     */
+    VMMR3UpdateLoggers(pVM);
+}
+
+
+/**
+ * Updates the settings for the RC and R0 loggers.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(int) VMMR3UpdateLoggers(PVM pVM)
+{
+    int rc = VINF_SUCCESS;
+
+#ifdef LOG_ENABLED
+    /*
+     * For the ring-0 EMT logger, we use a per-thread logger instance
+     * in ring-0. Only initialize it once.
+     */
+    PRTLOGGER const pDefault = RTLogDefaultInstance();
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU       pVCpu = pVM->apCpusR3[i];
+        PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
+        if (pR0LoggerR3)
+        {
+            if (!pR0LoggerR3->fCreated)
+            {
+                RTR0PTR pfnLoggerWrapper = NIL_RTR0PTR;
+                rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerWrapper", &pfnLoggerWrapper);
+                AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerWrapper not found! rc=%Rra\n", rc), rc);
+
+                RTR0PTR pfnLoggerFlush = NIL_RTR0PTR;
+                rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerFlush", &pfnLoggerFlush);
+                AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerFlush not found! rc=%Rra\n", rc), rc);
+
+                char szR0ThreadName[16];
+                RTStrPrintf(szR0ThreadName, sizeof(szR0ThreadName), "EMT-%u-R0", i);
+                rc = RTLogCreateForR0(&pR0LoggerR3->Logger, pR0LoggerR3->cbLogger,
+                                      pVCpu->vmm.s.pR0LoggerR0 + RT_UOFFSETOF(VMMR0LOGGER, Logger),
+                                      pfnLoggerWrapper, pfnLoggerFlush,
+                                      RTLOGFLAGS_BUFFERED, RTLOGDEST_DUMMY, szR0ThreadName);
+                AssertReleaseMsgRCReturn(rc, ("RTLogCreateForR0 failed! rc=%Rra\n", rc), rc);
+
+                pR0LoggerR3->idCpu = i;
+                pR0LoggerR3->fCreated = true;
+                pR0LoggerR3->fFlushingDisabled = false;
+            }
+
+            rc = RTLogCopyGroupsAndFlagsForR0(&pR0LoggerR3->Logger, pVCpu->vmm.s.pR0LoggerR0 + RT_UOFFSETOF(VMMR0LOGGER, Logger),
+                                              pDefault, RTLOGFLAGS_BUFFERED, UINT32_MAX);
+            AssertRC(rc);
+        }
+    }
+#else
+    RT_NOREF(pVM);
+#endif
+
+    return rc;
+}
+
+
+/**
+ * Gets the pointer to a buffer containing the R0/RC RTAssertMsg1Weak output.
+ *
+ * @returns Pointer to the buffer.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(const char *) VMMR3GetRZAssertMsg1(PVM pVM)
+{
+    return pVM->vmm.s.szRing0AssertMsg1;
+}
+
+
+/**
+ * Returns the VMCPU of the specified virtual CPU.
+ *
+ * @returns The VMCPU pointer. NULL if @a idCpu or @a pUVM is invalid.
+ *
+ * @param   pUVM        The user mode VM handle.
+ * @param   idCpu       The ID of the virtual CPU.
+ */
+VMMR3DECL(PVMCPU) VMMR3GetCpuByIdU(PUVM pUVM, RTCPUID idCpu)
+{
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
+    AssertReturn(idCpu < pUVM->cCpus, NULL);
+    VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, NULL);
+    return pUVM->pVM->apCpusR3[idCpu];
+}
+
+
+/**
+ * Gets the pointer to a buffer containing the R0/RC RTAssertMsg2Weak output.
+ *
+ * @returns Pointer to the buffer.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR3DECL(const char *) VMMR3GetRZAssertMsg2(PVM pVM)
+{
+    return pVM->vmm.s.szRing0AssertMsg2;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("vmmR3Save:\n"));
+
+    /*
+     * Save the started/stopped state of all CPUs except 0 as it will always
+     * be running. This avoids breaking the saved state version. :-)
+     */
+    for (VMCPUID i = 1; i < pVM->cCpus; i++)
+        SSMR3PutBool(pSSM, VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(pVM->apCpusR3[i])));
+
+    return SSMR3PutU32(pSSM, UINT32_MAX); /* terminator */
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    LogFlow(("vmmR3Load:\n"));
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+
+    /*
+     * Validate version.
+     */
+    if (    uVersion != VMM_SAVED_STATE_VERSION
+        &&  uVersion != VMM_SAVED_STATE_VERSION_3_0)
+    {
+        AssertMsgFailed(("vmmR3Load: Invalid version uVersion=%u!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    if (uVersion <= VMM_SAVED_STATE_VERSION_3_0)
+    {
+        /* Ignore the stack bottom, stack pointer and stack bits. */
+        RTRCPTR RCPtrIgnored;
+        SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
+        SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
+#ifdef RT_OS_DARWIN
+        if (   SSMR3HandleVersion(pSSM)  >= VBOX_FULL_VERSION_MAKE(3,0,0)
+            && SSMR3HandleVersion(pSSM)  <  VBOX_FULL_VERSION_MAKE(3,1,0)
+            && SSMR3HandleRevision(pSSM) >= 48858
+            && (   !strcmp(SSMR3HandleHostOSAndArch(pSSM), "darwin.x86")
+                || !strcmp(SSMR3HandleHostOSAndArch(pSSM), "") )
+           )
+            SSMR3Skip(pSSM, 16384);
+        else
+            SSMR3Skip(pSSM, 8192);
+#else
+        SSMR3Skip(pSSM, 8192);
+#endif
+    }
+
+    /*
+     * Restore the VMCPU states. VCPU 0 is always started.
+     */
+    VMCPU_SET_STATE(pVM->apCpusR3[0], VMCPUSTATE_STARTED);
+    for (VMCPUID i = 1; i < pVM->cCpus; i++)
+    {
+        bool fStarted;
+        int rc = SSMR3GetBool(pSSM, &fStarted);
+        if (RT_FAILURE(rc))
+            return rc;
+        VMCPU_SET_STATE(pVM->apCpusR3[i], fStarted ? VMCPUSTATE_STARTED : VMCPUSTATE_STOPPED);
+    }
+
+    /* terminator */
+    uint32_t u32;
+    int rc = SSMR3GetU32(pSSM, &u32);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (u32 != UINT32_MAX)
+    {
+        AssertMsgFailed(("u32=%#x\n", u32));
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Suspends the CPU yielder.
+ *
+ * @param   pVM             The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) VMMR3YieldSuspend(PVM pVM)
+{
+    VMCPU_ASSERT_EMT(pVM->apCpusR3[0]);
+    if (!pVM->vmm.s.cYieldResumeMillies)
+    {
+        uint64_t u64Now = TMTimerGet(pVM->vmm.s.pYieldTimer);
+        uint64_t u64Expire = TMTimerGetExpire(pVM->vmm.s.pYieldTimer);
+        if (u64Now >= u64Expire || u64Expire == ~(uint64_t)0)
+            pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
+        else
+            pVM->vmm.s.cYieldResumeMillies = TMTimerToMilli(pVM->vmm.s.pYieldTimer, u64Expire - u64Now);
+        TMTimerStop(pVM->vmm.s.pYieldTimer);
+    }
+    pVM->vmm.s.u64LastYield = RTTimeNanoTS();
+}
+
+
+/**
+ * Stops the CPU yielder.
+ *
+ * @param   pVM             The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) VMMR3YieldStop(PVM pVM)
+{
+    if (!pVM->vmm.s.cYieldResumeMillies)
+        TMTimerStop(pVM->vmm.s.pYieldTimer);
+    pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
+    pVM->vmm.s.u64LastYield = RTTimeNanoTS();
+}
+
+
+/**
+ * Resumes the CPU yielder when it has been a suspended or stopped.
+ *
+ * @param   pVM             The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) VMMR3YieldResume(PVM pVM)
+{
+    if (pVM->vmm.s.cYieldResumeMillies)
+    {
+        TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldResumeMillies);
+        pVM->vmm.s.cYieldResumeMillies = 0;
+    }
+}
+
+
+/**
+ * Internal timer callback function.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pTimer          The timer handle.
+ * @param   pvUser          User argument specified upon timer creation.
+ */
+static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser)
+{
+    NOREF(pvUser);
+
+    /*
+     * This really needs some careful tuning. While we shouldn't be too greedy since
+     * that'll cause the rest of the system to stop up, we shouldn't be too nice either
+     * because that'll cause us to stop up.
+     *
+     * The current logic is to use the default interval when there is no lag worth
+     * mentioning, but when we start accumulating lag we don't bother yielding at all.
+     *
+     * (This depends on the TMCLOCK_VIRTUAL_SYNC to be scheduled before TMCLOCK_REAL
+     * so the lag is up to date.)
+     */
+    const uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
+    if (    u64Lag     <   50000000 /* 50ms */
+        ||  (   u64Lag < 1000000000 /*  1s */
+             && RTTimeNanoTS() - pVM->vmm.s.u64LastYield < 500000000 /* 500 ms */)
+       )
+    {
+        uint64_t u64Elapsed = RTTimeNanoTS();
+        pVM->vmm.s.u64LastYield = u64Elapsed;
+
+        RTThreadYield();
+
+#ifdef LOG_ENABLED
+        u64Elapsed = RTTimeNanoTS() - u64Elapsed;
+        Log(("vmmR3YieldEMT: %RI64 ns\n", u64Elapsed));
+#endif
+    }
+    TMTimerSetMillies(pTimer, pVM->vmm.s.cYieldEveryMillies);
+}
+
+
+/**
+ * Executes guest code (Intel VT-x and AMD-V).
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMR3_INT_DECL(int) VMMR3HmRunGC(PVM pVM, PVMCPU pVCpu)
+{
+    Log2(("VMMR3HmRunGC: (cs:rip=%04x:%RX64)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestRIP(pVCpu)));
+
+    for (;;)
+    {
+        int rc;
+        do
+        {
+#ifdef NO_SUPCALLR0VMM
+            rc = VERR_GENERAL_FAILURE;
+#else
+            rc = SUPR3CallVMMR0Fast(VMCC_GET_VMR0_FOR_CALL(pVM), VMMR0_DO_HM_RUN, pVCpu->idCpu);
+            if (RT_LIKELY(rc == VINF_SUCCESS))
+                rc = pVCpu->vmm.s.iLastGZRc;
+#endif
+        } while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
+
+#if 0 /** @todo triggers too often */
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TO_R3));
+#endif
+
+        /*
+         * Flush the logs
+         */
+#ifdef LOG_ENABLED
+        VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0LoggerR3, NULL);
+#endif
+        VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0RelLoggerR3, RTLogRelGetDefaultInstance());
+        if (rc != VINF_VMM_CALL_HOST)
+        {
+            Log2(("VMMR3HmRunGC: returns %Rrc (cs:rip=%04x:%RX64)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestRIP(pVCpu)));
+            return rc;
+        }
+        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+        /* Resume R0 */
+    }
+}
+
+
+/**
+ * Perform one of the fast I/O control VMMR0 operation.
+ *
+ * @returns VBox strict status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmOperation    The operation to perform.
+ */
+VMMR3_INT_DECL(VBOXSTRICTRC) VMMR3CallR0EmtFast(PVM pVM, PVMCPU pVCpu, VMMR0OPERATION enmOperation)
+{
+    for (;;)
+    {
+        VBOXSTRICTRC rcStrict;
+        do
+        {
+#ifdef NO_SUPCALLR0VMM
+            rcStrict = VERR_GENERAL_FAILURE;
+#else
+            rcStrict = SUPR3CallVMMR0Fast(VMCC_GET_VMR0_FOR_CALL(pVM), enmOperation, pVCpu->idCpu);
+            if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+                rcStrict = pVCpu->vmm.s.iLastGZRc;
+#endif
+        } while (rcStrict == VINF_EM_RAW_INTERRUPT_HYPER);
+
+        /*
+         * Flush the logs
+         */
+#ifdef LOG_ENABLED
+        VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0LoggerR3, NULL);
+#endif
+        VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0RelLoggerR3, RTLogRelGetDefaultInstance());
+        if (rcStrict != VINF_VMM_CALL_HOST)
+            return rcStrict;
+        int rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+        /* Resume R0 */
+    }
+}
+
+
+/**
+ * VCPU worker for VMMR3SendStartupIpi.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   idCpu       Virtual CPU to perform SIPI on.
+ * @param   uVector     The SIPI vector.
+ */
+static DECLCALLBACK(int) vmmR3SendStarupIpi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
+{
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * In the INIT state, the target CPU is only responsive to an SIPI.
+     * This is also true for when when the CPU is in VMX non-root mode.
+     *
+     * See AMD spec. 16.5 "Interprocessor Interrupts (IPI)".
+     * See Intel spec. 26.6.2 "Activity State".
+     */
+    if (EMGetState(pVCpu) != EMSTATE_WAIT_SIPI)
+        return VINF_SUCCESS;
+
+    PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (CPUMIsGuestInVmxRootMode(pCtx))
+    {
+        /* If the CPU is in VMX non-root mode we must cause a VM-exit. */
+        if (CPUMIsGuestInVmxNonRootMode(pCtx))
+            return VBOXSTRICTRC_TODO(IEMExecVmxVmexitStartupIpi(pVCpu, uVector));
+
+        /* If the CPU is in VMX root mode (and not in VMX non-root mode) SIPIs are blocked. */
+        return VINF_SUCCESS;
+    }
+#endif
+
+    pCtx->cs.Sel        = uVector << 8;
+    pCtx->cs.ValidSel   = uVector << 8;
+    pCtx->cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+    pCtx->cs.u64Base    = uVector << 12;
+    pCtx->cs.u32Limit   = UINT32_C(0x0000ffff);
+    pCtx->rip           = 0;
+
+    Log(("vmmR3SendSipi for VCPU %d with vector %x\n", idCpu, uVector));
+
+# if 1 /* If we keep the EMSTATE_WAIT_SIPI method, then move this to EM.cpp. */
+    EMSetState(pVCpu, EMSTATE_HALTED);
+    return VINF_EM_RESCHEDULE;
+# else /* And if we go the VMCPU::enmState way it can stay here. */
+    VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STOPPED);
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
+    return VINF_SUCCESS;
+# endif
+}
+
+
+/**
+ * VCPU worker for VMMR3SendInitIpi.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   idCpu       Virtual CPU to perform SIPI on.
+ */
+static DECLCALLBACK(int) vmmR3SendInitIpi(PVM pVM, VMCPUID idCpu)
+{
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    Log(("vmmR3SendInitIpi for VCPU %d\n", idCpu));
+
+    /** @todo r=ramshankar: We should probably block INIT signal when the CPU is in
+     *        wait-for-SIPI state. Verify. */
+
+    /* If the CPU is in VMX non-root mode, INIT signals cause VM-exits. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    PCCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
+    if (CPUMIsGuestInVmxNonRootMode(pCtx))
+        return VBOXSTRICTRC_TODO(IEMExecVmxVmexit(pVCpu, VMX_EXIT_INIT_SIGNAL, 0 /* uExitQual */));
+#endif
+
+    /** @todo Figure out how to handle a SVM nested-guest intercepts here for INIT
+     *  IPI (e.g. SVM_EXIT_INIT). */
+
+    PGMR3ResetCpu(pVM, pVCpu);
+    PDMR3ResetCpu(pVCpu);   /* Only clears pending interrupts force flags */
+    APICR3InitIpi(pVCpu);
+    TRPMR3ResetCpu(pVCpu);
+    CPUMR3ResetCpu(pVM, pVCpu);
+    EMR3ResetCpu(pVCpu);
+    HMR3ResetCpu(pVCpu);
+    NEMR3ResetCpu(pVCpu, true /*fInitIpi*/);
+
+    /* This will trickle up on the target EMT. */
+    return VINF_EM_WAIT_SIPI;
+}
+
+
+/**
+ * Sends a Startup IPI to the virtual CPU by setting CS:EIP into
+ * vector-dependent state and unhalting processor.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   idCpu       Virtual CPU to perform SIPI on.
+ * @param   uVector     SIPI vector.
+ */
+VMMR3_INT_DECL(void) VMMR3SendStartupIpi(PVM pVM, VMCPUID idCpu,  uint32_t uVector)
+{
+    AssertReturnVoid(idCpu < pVM->cCpus);
+
+    int rc = VMR3ReqCallNoWait(pVM, idCpu, (PFNRT)vmmR3SendStarupIpi, 3, pVM, idCpu, uVector);
+    AssertRC(rc);
+}
+
+
+/**
+ * Sends init IPI to the virtual CPU.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   idCpu       Virtual CPU to perform int IPI on.
+ */
+VMMR3_INT_DECL(void) VMMR3SendInitIpi(PVM pVM, VMCPUID idCpu)
+{
+    AssertReturnVoid(idCpu < pVM->cCpus);
+
+    int rc = VMR3ReqCallNoWait(pVM, idCpu, (PFNRT)vmmR3SendInitIpi, 2, pVM, idCpu);
+    AssertRC(rc);
+}
+
+
+/**
+ * Registers the guest memory range that can be used for patching.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pPatchMem   Patch memory range.
+ * @param   cbPatchMem  Size of the memory range.
+ */
+VMMR3DECL(int) VMMR3RegisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
+{
+    VM_ASSERT_EMT(pVM);
+    if (HMIsEnabled(pVM))
+        return HMR3EnablePatching(pVM, pPatchMem, cbPatchMem);
+
+    return VERR_NOT_SUPPORTED;
+}
+
+
+/**
+ * Deregisters the guest memory range that can be used for patching.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pPatchMem   Patch memory range.
+ * @param   cbPatchMem  Size of the memory range.
+ */
+VMMR3DECL(int) VMMR3DeregisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
+{
+    if (HMIsEnabled(pVM))
+        return HMR3DisablePatching(pVM, pPatchMem, cbPatchMem);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common recursion handler for the other EMTs.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   rcStrict            Current status code to be combined with the one
+ *                              from this recursion and returned.
+ */
+static VBOXSTRICTRC vmmR3EmtRendezvousCommonRecursion(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rcStrict)
+{
+    int rc2;
+
+    /*
+     * We wait here while the initiator of this recursion reconfigures
+     * everything.  The last EMT to get in signals the initiator.
+     */
+    if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPush) == pVM->cCpus)
+    {
+        rc2 = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
+        AssertLogRelRC(rc2);
+    }
+
+    rc2 = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousRecursionPush, RT_INDEFINITE_WAIT);
+    AssertLogRelRC(rc2);
+
+    /*
+     * Do the normal rendezvous processing.
+     */
+    VBOXSTRICTRC rcStrict2 = vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
+                                                      pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
+
+    /*
+     * Wait for the initiator to restore everything.
+     */
+    rc2 = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousRecursionPop, RT_INDEFINITE_WAIT);
+    AssertLogRelRC(rc2);
+
+    /*
+     * Last thread out of here signals the initiator.
+     */
+    if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPop) == pVM->cCpus)
+    {
+        rc2 = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
+        AssertLogRelRC(rc2);
+    }
+
+    /*
+     * Merge status codes and return.
+     */
+    AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
+    if (    rcStrict2 != VINF_SUCCESS
+        &&  (   rcStrict == VINF_SUCCESS
+             || rcStrict > rcStrict2))
+        rcStrict = rcStrict2;
+    return rcStrict;
+}
+
+
+/**
+ * Count returns and have the last non-caller EMT wake up the caller.
+ *
+ * @returns VBox strict informational status code for EM scheduling. No failures
+ *          will be returned here, those are for the caller only.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   rcStrict            The current accumulated recursive status code,
+ *                              to be merged with i32RendezvousStatus and
+ *                              returned.
+ */
+DECL_FORCE_INLINE(VBOXSTRICTRC) vmmR3EmtRendezvousNonCallerReturn(PVM pVM, VBOXSTRICTRC rcStrict)
+{
+    VBOXSTRICTRC rcStrict2 = ASMAtomicReadS32(&pVM->vmm.s.i32RendezvousStatus);
+
+    uint32_t cReturned = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsReturned);
+    if (cReturned == pVM->cCpus - 1U)
+    {
+        int rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
+        AssertLogRelRC(rc);
+    }
+
+    /*
+     * Merge the status codes, ignoring error statuses in this code path.
+     */
+    AssertLogRelMsgReturn(   rcStrict2 <= VINF_SUCCESS
+                          || (rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST),
+                          ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
+                          VERR_IPE_UNEXPECTED_INFO_STATUS);
+
+    if (RT_SUCCESS(rcStrict2))
+    {
+        if (    rcStrict2 != VINF_SUCCESS
+            &&  (   rcStrict == VINF_SUCCESS
+                 || rcStrict > rcStrict2))
+            rcStrict = rcStrict2;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Common worker for VMMR3EmtRendezvous and VMMR3EmtRendezvousFF.
+ *
+ * @returns VBox strict informational status code for EM scheduling. No failures
+ *          will be returned here, those are for the caller only.  When
+ *          fIsCaller is set, VINF_SUCCESS is always returned.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   fIsCaller           Whether we're the VMMR3EmtRendezvous caller or
+ *                              not.
+ * @param   fFlags              The flags.
+ * @param   pfnRendezvous       The callback.
+ * @param   pvUser              The user argument for the callback.
+ */
+static VBOXSTRICTRC vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
+                                             uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
+{
+    int rc;
+    VBOXSTRICTRC rcStrictRecursion = VINF_SUCCESS;
+
+    /*
+     * Enter, the last EMT triggers the next callback phase.
+     */
+    uint32_t cEntered = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsEntered);
+    if (cEntered != pVM->cCpus)
+    {
+        if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
+        {
+            /* Wait for our turn. */
+            for (;;)
+            {
+                rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, RT_INDEFINITE_WAIT);
+                AssertLogRelRC(rc);
+                if (!pVM->vmm.s.fRendezvousRecursion)
+                    break;
+                rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
+            }
+        }
+        else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
+        {
+            /* Wait for the last EMT to arrive and wake everyone up. */
+            rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce, RT_INDEFINITE_WAIT);
+            AssertLogRelRC(rc);
+            Assert(!pVM->vmm.s.fRendezvousRecursion);
+        }
+        else if (   (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
+                 || (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
+        {
+            /* Wait for our turn. */
+            for (;;)
+            {
+                rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
+                AssertLogRelRC(rc);
+                if (!pVM->vmm.s.fRendezvousRecursion)
+                    break;
+                rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
+            }
+        }
+        else
+        {
+            Assert((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE);
+
+            /*
+             * The execute once is handled specially to optimize the code flow.
+             *
+             * The last EMT to arrive will perform the callback and the other
+             * EMTs will wait on the Done/DoneCaller semaphores (instead of
+             * the EnterOneByOne/AllAtOnce) in the meanwhile. When the callback
+             * returns, that EMT will initiate the normal return sequence.
+             */
+            if (!fIsCaller)
+            {
+                for (;;)
+                {
+                    rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
+                    AssertLogRelRC(rc);
+                    if (!pVM->vmm.s.fRendezvousRecursion)
+                        break;
+                    rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
+                }
+
+                return vmmR3EmtRendezvousNonCallerReturn(pVM, rcStrictRecursion);
+            }
+            return VINF_SUCCESS;
+        }
+    }
+    else
+    {
+        /*
+         * All EMTs are waiting, clear the FF and take action according to the
+         * execution method.
+         */
+        VM_FF_CLEAR(pVM, VM_FF_EMT_RENDEZVOUS);
+
+        if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
+        {
+            /* Wake up everyone. */
+            rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
+            AssertLogRelRC(rc);
+        }
+        else if (   (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
+                 || (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
+        {
+            /* Figure out who to wake up and wake it up. If it's ourself, then
+               it's easy otherwise wait for our turn. */
+            VMCPUID iFirst = (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
+                           ? 0
+                           : pVM->cCpus - 1U;
+            if (pVCpu->idCpu != iFirst)
+            {
+                rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iFirst]);
+                AssertLogRelRC(rc);
+                for (;;)
+                {
+                    rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
+                    AssertLogRelRC(rc);
+                    if (!pVM->vmm.s.fRendezvousRecursion)
+                        break;
+                    rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
+                }
+            }
+        }
+        /* else: execute the handler on the current EMT and wake up one or more threads afterwards. */
+    }
+
+
+    /*
+     * Do the callback and update the status if necessary.
+     */
+    if (    !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
+        ||  RT_SUCCESS(ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus)) )
+    {
+        VBOXSTRICTRC rcStrict2 = pfnRendezvous(pVM, pVCpu, pvUser);
+        if (rcStrict2 != VINF_SUCCESS)
+        {
+            AssertLogRelMsg(   rcStrict2 <= VINF_SUCCESS
+                            || (rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST),
+                            ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
+            int32_t i32RendezvousStatus;
+            do
+            {
+                i32RendezvousStatus = ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus);
+                if (    rcStrict2 == i32RendezvousStatus
+                    ||  RT_FAILURE(i32RendezvousStatus)
+                    ||  (   i32RendezvousStatus != VINF_SUCCESS
+                         && rcStrict2 > i32RendezvousStatus))
+                    break;
+            } while (!ASMAtomicCmpXchgS32(&pVM->vmm.s.i32RendezvousStatus, VBOXSTRICTRC_VAL(rcStrict2), i32RendezvousStatus));
+        }
+    }
+
+    /*
+     * Increment the done counter and take action depending on whether we're
+     * the last to finish callback execution.
+     */
+    uint32_t cDone = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsDone);
+    if (    cDone != pVM->cCpus
+        &&  (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE)
+    {
+        /* Signal the next EMT? */
+        if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
+        {
+            rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
+            AssertLogRelRC(rc);
+        }
+        else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
+        {
+            Assert(cDone == pVCpu->idCpu + 1U);
+            rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu + 1U]);
+            AssertLogRelRC(rc);
+        }
+        else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
+        {
+            Assert(pVM->cCpus - cDone == pVCpu->idCpu);
+            rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVM->cCpus - cDone - 1U]);
+            AssertLogRelRC(rc);
+        }
+
+        /* Wait for the rest to finish (the caller waits on hEvtRendezvousDoneCaller). */
+        if (!fIsCaller)
+        {
+            for (;;)
+            {
+                rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
+                AssertLogRelRC(rc);
+                if (!pVM->vmm.s.fRendezvousRecursion)
+                    break;
+                rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
+            }
+        }
+    }
+    else
+    {
+        /* Callback execution is all done, tell the rest to return. */
+        rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
+        AssertLogRelRC(rc);
+    }
+
+    if (!fIsCaller)
+        return vmmR3EmtRendezvousNonCallerReturn(pVM, rcStrictRecursion);
+    return rcStrictRecursion;
+}
+
+
+/**
+ * Called in response to VM_FF_EMT_RENDEZVOUS.
+ *
+ * @returns VBox strict status code - EM scheduling.  No errors will be returned
+ *          here, nor will any non-EM scheduling status codes be returned.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ *
+ * @thread  EMT
+ */
+VMMR3_INT_DECL(int) VMMR3EmtRendezvousFF(PVM pVM, PVMCPU pVCpu)
+{
+    Assert(!pVCpu->vmm.s.fInRendezvous);
+    Log(("VMMR3EmtRendezvousFF: EMT%#u\n", pVCpu->idCpu));
+    pVCpu->vmm.s.fInRendezvous = true;
+    VBOXSTRICTRC rcStrict = vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
+                                                     pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
+    pVCpu->vmm.s.fInRendezvous = false;
+    Log(("VMMR3EmtRendezvousFF: EMT%#u returns %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict)));
+    return VBOXSTRICTRC_TODO(rcStrict);
+}
+
+
+/**
+ * Helper for resetting an single wakeup event sempahore.
+ *
+ * @returns VERR_TIMEOUT on success, RTSemEventWait status otherwise.
+ * @param   hEvt        The event semaphore to reset.
+ */
+static int vmmR3HlpResetEvent(RTSEMEVENT hEvt)
+{
+    for (uint32_t cLoops = 0; ; cLoops++)
+    {
+        int rc = RTSemEventWait(hEvt, 0 /*cMsTimeout*/);
+        if (rc != VINF_SUCCESS || cLoops > _4K)
+            return rc;
+    }
+}
+
+
+/**
+ * Worker for VMMR3EmtRendezvous that handles recursion.
+ *
+ * @returns VBox strict status code.  This will be the first error,
+ *          VINF_SUCCESS, or an EM scheduling status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.
+ * @param   fFlags          Flags indicating execution methods. See
+ *                          grp_VMMR3EmtRendezvous_fFlags.
+ * @param   pfnRendezvous   The callback.
+ * @param   pvUser          User argument for the callback.
+ *
+ * @thread  EMT(pVCpu)
+ */
+static VBOXSTRICTRC vmmR3EmtRendezvousRecursive(PVM pVM, PVMCPU pVCpu, uint32_t fFlags,
+                                                PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
+{
+    Log(("vmmR3EmtRendezvousRecursive: %#x EMT#%u depth=%d\n", fFlags, pVCpu->idCpu, pVM->vmm.s.cRendezvousRecursions));
+    AssertLogRelReturn(pVM->vmm.s.cRendezvousRecursions < 3, VERR_DEADLOCK);
+    Assert(pVCpu->vmm.s.fInRendezvous);
+
+    /*
+     * Save the current state.
+     */
+    uint32_t const              fParentFlags    = pVM->vmm.s.fRendezvousFlags;
+    uint32_t const              cParentDone     = pVM->vmm.s.cRendezvousEmtsDone;
+    int32_t const               iParentStatus   = pVM->vmm.s.i32RendezvousStatus;
+    PFNVMMEMTRENDEZVOUS const   pfnParent       = pVM->vmm.s.pfnRendezvous;
+    void * const                pvParentUser    = pVM->vmm.s.pvRendezvousUser;
+
+    /*
+     * Check preconditions and save the current state.
+     */
+    AssertReturn(   (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
+                 || (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
+                 || (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE
+                 || (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE,
+                 VERR_INTERNAL_ERROR);
+    AssertReturn(pVM->vmm.s.cRendezvousEmtsEntered == pVM->cCpus, VERR_INTERNAL_ERROR_2);
+    AssertReturn(pVM->vmm.s.cRendezvousEmtsReturned == 0, VERR_INTERNAL_ERROR_3);
+
+    /*
+     * Reset the recursion prep and pop semaphores.
+     */
+    int rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousRecursionPush);
+    AssertLogRelRCReturn(rc, rc);
+    rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousRecursionPop);
+    AssertLogRelRCReturn(rc, rc);
+    rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
+    AssertLogRelMsgReturn(rc == VERR_TIMEOUT, ("%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+    rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
+    AssertLogRelMsgReturn(rc == VERR_TIMEOUT, ("%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+
+    /*
+     * Usher the other thread into the recursion routine.
+     */
+    ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsRecursingPush, 0);
+    ASMAtomicWriteBool(&pVM->vmm.s.fRendezvousRecursion, true);
+
+    uint32_t cLeft = pVM->cCpus - (cParentDone + 1U);
+    if ((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
+        while (cLeft-- > 0)
+        {
+            rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
+            AssertLogRelRC(rc);
+        }
+    else if ((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
+    {
+        Assert(cLeft == pVM->cCpus - (pVCpu->idCpu + 1U));
+        for (VMCPUID iCpu = pVCpu->idCpu + 1U; iCpu < pVM->cCpus; iCpu++)
+        {
+            rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iCpu]);
+            AssertLogRelRC(rc);
+        }
+    }
+    else if ((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
+    {
+        Assert(cLeft == pVCpu->idCpu);
+        for (VMCPUID iCpu = pVCpu->idCpu; iCpu > 0; iCpu--)
+        {
+            rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iCpu - 1U]);
+            AssertLogRelRC(rc);
+        }
+    }
+    else
+        AssertLogRelReturn((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE,
+                           VERR_INTERNAL_ERROR_4);
+
+    rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
+    AssertLogRelRC(rc);
+    rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
+    AssertLogRelRC(rc);
+
+
+    /*
+     * Wait for the EMTs to wake up and get out of the parent rendezvous code.
+     */
+    if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPush) != pVM->cCpus)
+    {
+        rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousRecursionPushCaller, RT_INDEFINITE_WAIT);
+        AssertLogRelRC(rc);
+    }
+
+    ASMAtomicWriteBool(&pVM->vmm.s.fRendezvousRecursion, false);
+
+    /*
+     * Clear the slate and setup the new rendezvous.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        rc = vmmR3HlpResetEvent(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
+        AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
+    }
+    rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousEnterOneByOne);        AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
+    rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);  AssertLogRelRC(rc);
+    rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone);            AssertLogRelRC(rc);
+    rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousDoneCaller);           AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
+
+    ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
+    ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
+    ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
+    ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
+    ASMAtomicWritePtr((void * volatile *)&pVM->vmm.s.pfnRendezvous, (void *)(uintptr_t)pfnRendezvous);
+    ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
+    ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
+    ASMAtomicIncU32(&pVM->vmm.s.cRendezvousRecursions);
+
+    /*
+     * We're ready to go now, do normal rendezvous processing.
+     */
+    rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousRecursionPush);
+    AssertLogRelRC(rc);
+
+    VBOXSTRICTRC rcStrict = vmmR3EmtRendezvousCommon(pVM, pVCpu, true /*fIsCaller*/, fFlags, pfnRendezvous, pvUser);
+
+    /*
+     * The caller waits for the other EMTs to be done, return and waiting on the
+     * pop semaphore.
+     */
+    for (;;)
+    {
+        rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
+        AssertLogRelRC(rc);
+        if (!pVM->vmm.s.fRendezvousRecursion)
+            break;
+        rcStrict = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrict);
+    }
+
+    /*
+     * Get the return code and merge it with the above recursion status.
+     */
+    VBOXSTRICTRC rcStrict2 = pVM->vmm.s.i32RendezvousStatus;
+    if (    rcStrict2 != VINF_SUCCESS
+        &&  (   rcStrict == VINF_SUCCESS
+             || rcStrict > rcStrict2))
+        rcStrict = rcStrict2;
+
+    /*
+     * Restore the parent rendezvous state.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        rc = vmmR3HlpResetEvent(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
+        AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
+    }
+    rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousEnterOneByOne);        AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
+    rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);  AssertLogRelRC(rc);
+    rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone);            AssertLogRelRC(rc);
+    rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousDoneCaller);           AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
+
+    ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered,   pVM->cCpus);
+    ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned,  0);
+    ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone,      cParentDone);
+    ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus,      iParentStatus);
+    ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags,         fParentFlags);
+    ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser,         pvParentUser);
+    ASMAtomicWritePtr((void * volatile *)&pVM->vmm.s.pfnRendezvous, (void *)(uintptr_t)pfnParent);
+
+    /*
+     * Usher the other EMTs back to their parent recursion routine, waiting
+     * for them to all get there before we return (makes sure they've been
+     * scheduled and are past the pop event sem, see below).
+     */
+    ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsRecursingPop, 0);
+    rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousRecursionPop);
+    AssertLogRelRC(rc);
+
+    if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPop) != pVM->cCpus)
+    {
+        rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousRecursionPopCaller, RT_INDEFINITE_WAIT);
+        AssertLogRelRC(rc);
+    }
+
+    /*
+     * We must reset the pop semaphore on the way out (doing the pop caller too,
+     * just in case).  The parent may be another recursion.
+     */
+    rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousRecursionPop);    AssertLogRelRC(rc);
+    rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);   AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
+
+    ASMAtomicDecU32(&pVM->vmm.s.cRendezvousRecursions);
+
+    Log(("vmmR3EmtRendezvousRecursive: %#x EMT#%u depth=%d returns %Rrc\n",
+         fFlags, pVCpu->idCpu, pVM->vmm.s.cRendezvousRecursions, VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * EMT rendezvous.
+ *
+ * Gathers all the EMTs and execute some code on each of them, either in a one
+ * by one fashion or all at once.
+ *
+ * @returns VBox strict status code.  This will be the first error,
+ *          VINF_SUCCESS, or an EM scheduling status code.
+ *
+ * @retval  VERR_DEADLOCK if recursion is attempted using a rendezvous type that
+ *          doesn't support it or if the recursion is too deep.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fFlags          Flags indicating execution methods. See
+ *                          grp_VMMR3EmtRendezvous_fFlags.  The one-by-one,
+ *                          descending and ascending rendezvous types support
+ *                          recursion from inside @a pfnRendezvous.
+ * @param   pfnRendezvous   The callback.
+ * @param   pvUser          User argument for the callback.
+ *
+ * @thread  Any.
+ */
+VMMR3DECL(int) VMMR3EmtRendezvous(PVM pVM, uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
+{
+    /*
+     * Validate input.
+     */
+    AssertReturn(pVM, VERR_INVALID_VM_HANDLE);
+    AssertMsg(   (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_INVALID
+              && (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) <= VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
+              && !(fFlags & ~VMMEMTRENDEZVOUS_FLAGS_VALID_MASK), ("%#x\n", fFlags));
+    AssertMsg(   !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
+              || (   (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE
+                  && (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE),
+              ("type %u\n", fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK));
+
+    VBOXSTRICTRC rcStrict;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+    {
+        /*
+         * Forward the request to an EMT thread.
+         */
+        Log(("VMMR3EmtRendezvous: %#x non-EMT\n", fFlags));
+        if (!(fFlags & VMMEMTRENDEZVOUS_FLAGS_PRIORITY))
+            rcStrict = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
+        else
+            rcStrict = VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
+        Log(("VMMR3EmtRendezvous: %#x non-EMT returns %Rrc\n", fFlags, VBOXSTRICTRC_VAL(rcStrict)));
+    }
+    else if (   pVM->cCpus == 1
+             || (   pVM->enmVMState == VMSTATE_DESTROYING
+                 && VMR3GetActiveEmts(pVM->pUVM) < pVM->cCpus ) )
+    {
+        /*
+         * Shortcut for the single EMT case.
+         *
+         * We also ends up here if EMT(0) (or others) tries to issue a rendezvous
+         * during vmR3Destroy after other emulation threads have started terminating.
+         */
+        if (!pVCpu->vmm.s.fInRendezvous)
+        {
+            Log(("VMMR3EmtRendezvous: %#x EMT (uni)\n", fFlags));
+            pVCpu->vmm.s.fInRendezvous  = true;
+            pVM->vmm.s.fRendezvousFlags = fFlags;
+            rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
+            pVCpu->vmm.s.fInRendezvous  = false;
+        }
+        else
+        {
+            /* Recursion. Do the same checks as in the SMP case. */
+            Log(("VMMR3EmtRendezvous: %#x EMT (uni), recursion depth=%d\n", fFlags, pVM->vmm.s.cRendezvousRecursions));
+            uint32_t fType = pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK;
+            AssertLogRelReturn(   !pVCpu->vmm.s.fInRendezvous
+                               || fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
+                               || fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
+                               || fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE
+                               || fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE
+                               , VERR_DEADLOCK);
+
+            AssertLogRelReturn(pVM->vmm.s.cRendezvousRecursions < 3, VERR_DEADLOCK);
+            pVM->vmm.s.cRendezvousRecursions++;
+            uint32_t const fParentFlags = pVM->vmm.s.fRendezvousFlags;
+            pVM->vmm.s.fRendezvousFlags = fFlags;
+
+            rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
+
+            pVM->vmm.s.fRendezvousFlags = fParentFlags;
+            pVM->vmm.s.cRendezvousRecursions--;
+        }
+        Log(("VMMR3EmtRendezvous: %#x EMT (uni) returns %Rrc\n", fFlags, VBOXSTRICTRC_VAL(rcStrict)));
+    }
+    else
+    {
+        /*
+         * Spin lock. If busy, check for recursion, if not recursing wait for
+         * the other EMT to finish while keeping a lookout for the RENDEZVOUS FF.
+         */
+        int rc;
+        rcStrict = VINF_SUCCESS;
+        if (RT_UNLIKELY(!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0)))
+        {
+            /* Allow recursion in some cases. */
+            if (   pVCpu->vmm.s.fInRendezvous
+                && (   (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
+                    || (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
+                    || (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE
+                    || (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE
+                       ))
+                return VBOXSTRICTRC_TODO(vmmR3EmtRendezvousRecursive(pVM, pVCpu, fFlags, pfnRendezvous, pvUser));
+
+            AssertLogRelMsgReturn(!pVCpu->vmm.s.fInRendezvous, ("fRendezvousFlags=%#x\n", pVM->vmm.s.fRendezvousFlags),
+                                  VERR_DEADLOCK);
+
+            Log(("VMMR3EmtRendezvous: %#x EMT#%u, waiting for lock...\n", fFlags, pVCpu->idCpu));
+            while (!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0))
+            {
+                if (VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
+                {
+                    rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
+                    if (    rc != VINF_SUCCESS
+                        &&  (   rcStrict == VINF_SUCCESS
+                             || rcStrict > rc))
+                        rcStrict = rc;
+                    /** @todo Perhaps deal with termination here?  */
+                }
+                ASMNopPause();
+            }
+        }
+
+        Log(("VMMR3EmtRendezvous: %#x EMT#%u\n", fFlags, pVCpu->idCpu));
+        Assert(!VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS));
+        Assert(!pVCpu->vmm.s.fInRendezvous);
+        pVCpu->vmm.s.fInRendezvous = true;
+
+        /*
+         * Clear the slate and setup the rendezvous. This is a semaphore ping-pong orgy. :-)
+         */
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i], 0);
+            AssertLogRelMsg(rc == VERR_TIMEOUT || rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        }
+        rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, 0);         AssertLogRelMsg(rc == VERR_TIMEOUT || rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);  AssertLogRelRC(rc);
+        rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone);            AssertLogRelRC(rc);
+        rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, 0);            AssertLogRelMsg(rc == VERR_TIMEOUT || rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
+        ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
+        ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
+        ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
+        ASMAtomicWritePtr((void * volatile *)&pVM->vmm.s.pfnRendezvous, (void *)(uintptr_t)pfnRendezvous);
+        ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
+        ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
+
+        /*
+         * Set the FF and poke the other EMTs.
+         */
+        VM_FF_SET(pVM, VM_FF_EMT_RENDEZVOUS);
+        VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_POKE);
+
+        /*
+         * Do the same ourselves.
+         */
+        VBOXSTRICTRC rcStrict2 = vmmR3EmtRendezvousCommon(pVM, pVCpu, true /* fIsCaller */, fFlags, pfnRendezvous, pvUser);
+
+        /*
+         * The caller waits for the other EMTs to be done and return before doing
+         * the cleanup. This makes away with wakeup / reset races we would otherwise
+         * risk in the multiple release event semaphore code (hEvtRendezvousDoneCaller).
+         */
+        for (;;)
+        {
+            rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
+            AssertLogRelRC(rc);
+            if (!pVM->vmm.s.fRendezvousRecursion)
+                break;
+            rcStrict2 = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrict2);
+        }
+
+        /*
+         * Get the return code and clean up a little bit.
+         */
+        VBOXSTRICTRC rcStrict3 = pVM->vmm.s.i32RendezvousStatus;
+        ASMAtomicWriteNullPtr((void * volatile *)&pVM->vmm.s.pfnRendezvous);
+
+        ASMAtomicWriteU32(&pVM->vmm.s.u32RendezvousLock, 0);
+        pVCpu->vmm.s.fInRendezvous = false;
+
+        /*
+         * Merge rcStrict, rcStrict2 and rcStrict3.
+         */
+        AssertRC(VBOXSTRICTRC_VAL(rcStrict));
+        AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
+        if (    rcStrict2 != VINF_SUCCESS
+            &&  (   rcStrict == VINF_SUCCESS
+                 || rcStrict > rcStrict2))
+            rcStrict = rcStrict2;
+        if (    rcStrict3 != VINF_SUCCESS
+            &&  (   rcStrict == VINF_SUCCESS
+                 || rcStrict > rcStrict3))
+            rcStrict = rcStrict3;
+        Log(("VMMR3EmtRendezvous: %#x EMT#%u returns %Rrc\n", fFlags, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict)));
+    }
+
+    AssertLogRelMsgReturn(   rcStrict <= VINF_SUCCESS
+                          || (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
+                          ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)),
+                          VERR_IPE_UNEXPECTED_INFO_STATUS);
+    return VBOXSTRICTRC_VAL(rcStrict);
+}
+
+
+/**
+ * Interface for vmR3SetHaltMethodU.
+ *
+ * @param   pVCpu                   The cross context virtual CPU structure of the
+ *                                  calling EMT.
+ * @param   fMayHaltInRing0         The new state.
+ * @param   cNsSpinBlockThreshold   The spin-vs-blocking threashold.
+ * @thread  EMT(pVCpu)
+ *
+ * @todo    Move the EMT handling to VMM (or EM).  I soooooo regret that VM
+ *          component.
+ */
+VMMR3_INT_DECL(void) VMMR3SetMayHaltInRing0(PVMCPU pVCpu, bool fMayHaltInRing0, uint32_t cNsSpinBlockThreshold)
+{
+    LogFlow(("VMMR3SetMayHaltInRing0(#%u, %d, %u)\n", pVCpu->idCpu, fMayHaltInRing0, cNsSpinBlockThreshold));
+    pVCpu->vmm.s.fMayHaltInRing0       = fMayHaltInRing0;
+    pVCpu->vmm.s.cNsSpinBlockThreshold = cNsSpinBlockThreshold;
+}
+
+
+/**
+ * Read from the ring 0 jump buffer stack.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   idCpu           The ID of the source CPU context (for the address).
+ * @param   R0Addr          Where to start reading.
+ * @param   pvBuf           Where to store the data we've read.
+ * @param   cbRead          The number of bytes to read.
+ */
+VMMR3_INT_DECL(int) VMMR3ReadR0Stack(PVM pVM, VMCPUID idCpu, RTHCUINTPTR R0Addr, void *pvBuf, size_t cbRead)
+{
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    AssertReturn(pVCpu, VERR_INVALID_PARAMETER);
+    AssertReturn(cbRead < ~(size_t)0 / 2, VERR_INVALID_PARAMETER);
+
+    int rc;
+#ifdef VMM_R0_SWITCH_STACK
+    RTHCUINTPTR off = R0Addr - MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
+#else
+    RTHCUINTPTR off = pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack - (pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck - R0Addr);
+#endif
+    if (   off < VMM_STACK_SIZE
+        && off + cbRead <= VMM_STACK_SIZE)
+    {
+        memcpy(pvBuf, &pVCpu->vmm.s.pbEMTStackR3[off], cbRead);
+        rc = VINF_SUCCESS;
+    }
+    else
+        rc = VERR_INVALID_POINTER;
+
+    /* Supply the setjmp return RIP/EIP.  */
+    if (   pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation + sizeof(RTR0UINTPTR) > R0Addr
+        && pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation < R0Addr + cbRead)
+    {
+        uint8_t const  *pbSrc  = (uint8_t const *)&pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcValue;
+        size_t          cbSrc  = sizeof(pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcValue);
+        size_t          offDst = 0;
+        if (R0Addr < pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation)
+            offDst = pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation - R0Addr;
+        else if (R0Addr > pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation)
+        {
+            size_t offSrc = R0Addr - pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation;
+            Assert(offSrc < cbSrc);
+            pbSrc -= offSrc;
+            cbSrc -= offSrc;
+        }
+        if (cbSrc > cbRead - offDst)
+            cbSrc = cbRead - offDst;
+        memcpy((uint8_t *)pvBuf + offDst, pbSrc, cbSrc);
+
+        if (cbSrc == cbRead)
+            rc = VINF_SUCCESS;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Used by the DBGF stack unwinder to initialize the register state.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   idCpu           The ID of the CPU being unwound.
+ * @param   pState          The unwind state to initialize.
+ */
+VMMR3_INT_DECL(void) VMMR3InitR0StackUnwindState(PUVM pUVM, VMCPUID idCpu, struct RTDBGUNWINDSTATE *pState)
+{
+    PVMCPU pVCpu = VMMR3GetCpuByIdU(pUVM, idCpu);
+    AssertReturnVoid(pVCpu);
+
+    /*
+     * Locate the resume point on the stack.
+     */
+#ifdef VMM_R0_SWITCH_STACK
+    uintptr_t off = pVCpu->vmm.s.CallRing3JmpBufR0.SpResume - MMHyperCCToR0(pVCpu->pVMR3, pVCpu->vmm.s.pbEMTStackR3);
+    AssertReturnVoid(off < VMM_STACK_SIZE);
+#else
+    uintptr_t off = 0;
+#endif
+
+#ifdef RT_ARCH_AMD64
+    /*
+     * This code must match the .resume stuff in VMMR0JmpA-amd64.asm exactly.
+     */
+# ifdef VBOX_STRICT
+    Assert(*(uint64_t const *)&pVCpu->vmm.s.pbEMTStackR3[off] == UINT32_C(0x7eadf00d));
+    off += 8; /* RESUME_MAGIC */
+# endif
+# ifdef RT_OS_WINDOWS
+    off += 0xa0; /* XMM6 thru XMM15 */
+# endif
+    pState->u.x86.uRFlags              = *(uint64_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 8;
+    pState->u.x86.auRegs[X86_GREG_xBX] = *(uint64_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 8;
+# ifdef RT_OS_WINDOWS
+    pState->u.x86.auRegs[X86_GREG_xSI] = *(uint64_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 8;
+    pState->u.x86.auRegs[X86_GREG_xDI] = *(uint64_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 8;
+# endif
+    pState->u.x86.auRegs[X86_GREG_x12] = *(uint64_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 8;
+    pState->u.x86.auRegs[X86_GREG_x13] = *(uint64_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 8;
+    pState->u.x86.auRegs[X86_GREG_x14] = *(uint64_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 8;
+    pState->u.x86.auRegs[X86_GREG_x15] = *(uint64_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 8;
+    pState->u.x86.auRegs[X86_GREG_xBP] = *(uint64_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 8;
+    pState->uPc                        = *(uint64_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 8;
+
+#elif defined(RT_ARCH_X86)
+    /*
+     * This code must match the .resume stuff in VMMR0JmpA-x86.asm exactly.
+     */
+# ifdef VBOX_STRICT
+    Assert(*(uint32_t const *)&pVCpu->vmm.s.pbEMTStackR3[off] == UINT32_C(0x7eadf00d));
+    off += 4; /* RESUME_MAGIC */
+# endif
+    pState->u.x86.uRFlags              = *(uint32_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 4;
+    pState->u.x86.auRegs[X86_GREG_xBX] = *(uint32_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 4;
+    pState->u.x86.auRegs[X86_GREG_xSI] = *(uint32_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 4;
+    pState->u.x86.auRegs[X86_GREG_xDI] = *(uint32_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 4;
+    pState->u.x86.auRegs[X86_GREG_xBP] = *(uint32_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 4;
+    pState->uPc                        = *(uint32_t const *)&pVCpu->vmm.s.pbEMTStackR3[off];
+    off += 4;
+#else
+# error "Port me"
+#endif
+
+    /*
+     * This is all we really need here, though the above helps if the assembly
+     * doesn't contain unwind info (currently only on win/64, so that is useful).
+     */
+    pState->u.x86.auRegs[X86_GREG_xBP] = pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp;
+    pState->u.x86.auRegs[X86_GREG_xSP] = pVCpu->vmm.s.CallRing3JmpBufR0.SpResume;
+}
+
+
+/**
+ * Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   uOperation  Operation to execute.
+ * @param   u64Arg      Constant argument.
+ * @param   pReqHdr     Pointer to a request header. See SUPR3CallVMMR0Ex for
+ *                      details.
+ */
+VMMR3DECL(int) VMMR3CallR0(PVM pVM, uint32_t uOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
+    return VMMR3CallR0Emt(pVM, pVCpu, (VMMR0OPERATION)uOperation, u64Arg, pReqHdr);
+}
+
+
+/**
+ * Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context VM structure.
+ * @param   enmOperation    Operation to execute.
+ * @param   u64Arg          Constant argument.
+ * @param   pReqHdr         Pointer to a request header. See SUPR3CallVMMR0Ex for
+ *                          details.
+ */
+VMMR3_INT_DECL(int) VMMR3CallR0Emt(PVM pVM, PVMCPU pVCpu, VMMR0OPERATION enmOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
+{
+    int rc;
+    for (;;)
+    {
+#ifdef NO_SUPCALLR0VMM
+        rc = VERR_GENERAL_FAILURE;
+#else
+        rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), pVCpu->idCpu, enmOperation, u64Arg, pReqHdr);
+#endif
+        /*
+         * Flush the logs.
+         */
+#ifdef LOG_ENABLED
+        VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0LoggerR3, NULL);
+#endif
+        VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0RelLoggerR3, RTLogRelGetDefaultInstance());
+        if (rc != VINF_VMM_CALL_HOST)
+            break;
+        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
+            break;
+        /* Resume R0 */
+    }
+
+    AssertLogRelMsgReturn(rc == VINF_SUCCESS || RT_FAILURE(rc),
+                          ("enmOperation=%u rc=%Rrc\n", enmOperation, rc),
+                          VERR_IPE_UNEXPECTED_INFO_STATUS);
+    return rc;
+}
+
+
+/**
+ * Service a call to the ring-3 host code.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @remarks Careful with critsects.
+ */
+static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu)
+{
+    /*
+     * We must also check for pending critsect exits or else we can deadlock
+     * when entering other critsects here.
+     */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PDM_CRITSECT))
+        PDMCritSectBothFF(pVCpu);
+
+    switch (pVCpu->vmm.s.enmCallRing3Operation)
+    {
+        /*
+         * Acquire a critical section.
+         */
+        case VMMCALLRING3_PDM_CRIT_SECT_ENTER:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PDMR3CritSectEnterEx((PPDMCRITSECT)(uintptr_t)pVCpu->vmm.s.u64CallRing3Arg,
+                                                            true /*fCallRing3*/);
+            break;
+        }
+
+        /*
+         * Enter a r/w critical section exclusively.
+         */
+        case VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_EXCL:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PDMR3CritSectRwEnterExclEx((PPDMCRITSECTRW)(uintptr_t)pVCpu->vmm.s.u64CallRing3Arg,
+                                                                    true /*fCallRing3*/);
+            break;
+        }
+
+        /*
+         * Enter a r/w critical section shared.
+         */
+        case VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_SHARED:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PDMR3CritSectRwEnterSharedEx((PPDMCRITSECTRW)(uintptr_t)pVCpu->vmm.s.u64CallRing3Arg,
+                                                                    true /*fCallRing3*/);
+            break;
+        }
+
+        /*
+         * Acquire the PDM lock.
+         */
+        case VMMCALLRING3_PDM_LOCK:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PDMR3LockCall(pVM);
+            break;
+        }
+
+        /*
+         * Grow the PGM pool.
+         */
+        case VMMCALLRING3_PGM_POOL_GROW:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PGMR3PoolGrow(pVM, pVCpu);
+            break;
+        }
+
+        /*
+         * Maps an page allocation chunk into ring-3 so ring-0 can use it.
+         */
+        case VMMCALLRING3_PGM_MAP_CHUNK:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PGMR3PhysChunkMap(pVM, pVCpu->vmm.s.u64CallRing3Arg);
+            break;
+        }
+
+        /*
+         * Allocates more handy pages.
+         */
+        case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateHandyPages(pVM);
+            break;
+        }
+
+        /*
+         * Allocates a large page.
+         */
+        case VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateLargeHandyPage(pVM, pVCpu->vmm.s.u64CallRing3Arg);
+            break;
+        }
+
+        /*
+         * Acquire the PGM lock.
+         */
+        case VMMCALLRING3_PGM_LOCK:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PGMR3LockCall(pVM);
+            break;
+        }
+
+        /*
+         * Acquire the MM hypervisor heap lock.
+         */
+        case VMMCALLRING3_MMHYPER_LOCK:
+        {
+            pVCpu->vmm.s.rcCallRing3 = MMR3LockCall(pVM);
+            break;
+        }
+
+        /*
+         * This is a noop. We just take this route to avoid unnecessary
+         * tests in the loops.
+         */
+        case VMMCALLRING3_VMM_LOGGER_FLUSH:
+            pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
+            LogAlways(("*FLUSH*\n"));
+            break;
+
+        /*
+         * Set the VM error message.
+         */
+        case VMMCALLRING3_VM_SET_ERROR:
+            VMR3SetErrorWorker(pVM);
+            pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
+            break;
+
+        /*
+         * Set the VM runtime error message.
+         */
+        case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
+            pVCpu->vmm.s.rcCallRing3 = VMR3SetRuntimeErrorWorker(pVM);
+            break;
+
+        /*
+         * Signal a ring 0 hypervisor assertion.
+         * Cancel the longjmp operation that's in progress.
+         */
+        case VMMCALLRING3_VM_R0_ASSERTION:
+            pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
+            pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call = false;
+#ifdef RT_ARCH_X86
+            pVCpu->vmm.s.CallRing3JmpBufR0.eip = 0;
+#else
+            pVCpu->vmm.s.CallRing3JmpBufR0.rip = 0;
+#endif
+#ifdef VMM_R0_SWITCH_STACK
+            *(uint64_t *)pVCpu->vmm.s.pbEMTStackR3 = 0; /* clear marker  */
+#endif
+            LogRel(("%s", pVM->vmm.s.szRing0AssertMsg1));
+            LogRel(("%s", pVM->vmm.s.szRing0AssertMsg2));
+            return VERR_VMM_RING0_ASSERTION;
+
+        /*
+         * A forced switch to ring 0 for preemption purposes.
+         */
+        case VMMCALLRING3_VM_R0_PREEMPT:
+            pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
+            break;
+
+        default:
+            AssertMsgFailed(("enmCallRing3Operation=%d\n", pVCpu->vmm.s.enmCallRing3Operation));
+            return VERR_VMM_UNKNOWN_RING3_CALL;
+    }
+
+    pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Displays the Force action Flags.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pHlp        The output helpers.
+ * @param   pszArgs     The additional arguments (ignored).
+ */
+static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    int         c;
+    uint32_t    f;
+    NOREF(pszArgs);
+
+#define PRINT_FLAG(prf,flag) do { \
+        if (f & (prf##flag)) \
+        { \
+            static const char *s_psz = #flag; \
+            if (!(c % 6)) \
+                pHlp->pfnPrintf(pHlp, "%s\n    %s", c ? "," : "", s_psz); \
+            else \
+                pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
+            c++; \
+            f &= ~(prf##flag); \
+        } \
+    } while (0)
+
+#define PRINT_GROUP(prf,grp,sfx) do { \
+        if (f & (prf##grp##sfx)) \
+        { \
+            static const char *s_psz = #grp; \
+            if (!(c % 5)) \
+                pHlp->pfnPrintf(pHlp, "%s    %s", c ? ",\n" : "  Groups:\n", s_psz); \
+            else \
+                pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
+            c++; \
+        } \
+    } while (0)
+
+    /*
+     * The global flags.
+     */
+    const uint32_t fGlobalForcedActions = pVM->fGlobalForcedActions;
+    pHlp->pfnPrintf(pHlp, "Global FFs: %#RX32", fGlobalForcedActions);
+
+    /* show the flag mnemonics  */
+    c = 0;
+    f = fGlobalForcedActions;
+    PRINT_FLAG(VM_FF_,TM_VIRTUAL_SYNC);
+    PRINT_FLAG(VM_FF_,PDM_QUEUES);
+    PRINT_FLAG(VM_FF_,PDM_DMA);
+    PRINT_FLAG(VM_FF_,DBGF);
+    PRINT_FLAG(VM_FF_,REQUEST);
+    PRINT_FLAG(VM_FF_,CHECK_VM_STATE);
+    PRINT_FLAG(VM_FF_,RESET);
+    PRINT_FLAG(VM_FF_,EMT_RENDEZVOUS);
+    PRINT_FLAG(VM_FF_,PGM_NEED_HANDY_PAGES);
+    PRINT_FLAG(VM_FF_,PGM_NO_MEMORY);
+    PRINT_FLAG(VM_FF_,PGM_POOL_FLUSH_PENDING);
+    PRINT_FLAG(VM_FF_,DEBUG_SUSPEND);
+    if (f)
+        pHlp->pfnPrintf(pHlp, "%s\n    Unknown bits: %#RX32\n", c ? "," : "", f);
+    else
+        pHlp->pfnPrintf(pHlp, "\n");
+
+    /* the groups */
+    c = 0;
+    f = fGlobalForcedActions;
+    PRINT_GROUP(VM_FF_,EXTERNAL_SUSPENDED,_MASK);
+    PRINT_GROUP(VM_FF_,EXTERNAL_HALTED,_MASK);
+    PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE,_MASK);
+    PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
+    PRINT_GROUP(VM_FF_,HIGH_PRIORITY_POST,_MASK);
+    PRINT_GROUP(VM_FF_,NORMAL_PRIORITY_POST,_MASK);
+    PRINT_GROUP(VM_FF_,NORMAL_PRIORITY,_MASK);
+    PRINT_GROUP(VM_FF_,ALL_REM,_MASK);
+    if (c)
+        pHlp->pfnPrintf(pHlp, "\n");
+
+    /*
+     * Per CPU flags.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU         pVCpu               = pVM->apCpusR3[i];
+        const uint64_t fLocalForcedActions = pVCpu->fLocalForcedActions;
+        pHlp->pfnPrintf(pHlp, "CPU %u FFs: %#RX64", i, fLocalForcedActions);
+
+        /* show the flag mnemonics */
+        c = 0;
+        f = fLocalForcedActions;
+        PRINT_FLAG(VMCPU_FF_,INTERRUPT_APIC);
+        PRINT_FLAG(VMCPU_FF_,INTERRUPT_PIC);
+        PRINT_FLAG(VMCPU_FF_,TIMER);
+        PRINT_FLAG(VMCPU_FF_,INTERRUPT_NMI);
+        PRINT_FLAG(VMCPU_FF_,INTERRUPT_SMI);
+        PRINT_FLAG(VMCPU_FF_,PDM_CRITSECT);
+        PRINT_FLAG(VMCPU_FF_,UNHALT);
+        PRINT_FLAG(VMCPU_FF_,IEM);
+        PRINT_FLAG(VMCPU_FF_,UPDATE_APIC);
+        PRINT_FLAG(VMCPU_FF_,DBGF);
+        PRINT_FLAG(VMCPU_FF_,REQUEST);
+        PRINT_FLAG(VMCPU_FF_,HM_UPDATE_CR3);
+        PRINT_FLAG(VMCPU_FF_,HM_UPDATE_PAE_PDPES);
+        PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3);
+        PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3_NON_GLOBAL);
+        PRINT_FLAG(VMCPU_FF_,TLB_FLUSH);
+        PRINT_FLAG(VMCPU_FF_,INHIBIT_INTERRUPTS);
+        PRINT_FLAG(VMCPU_FF_,BLOCK_NMIS);
+        PRINT_FLAG(VMCPU_FF_,TO_R3);
+        PRINT_FLAG(VMCPU_FF_,IOM);
+        if (f)
+            pHlp->pfnPrintf(pHlp, "%s\n    Unknown bits: %#RX64\n", c ? "," : "", f);
+        else
+            pHlp->pfnPrintf(pHlp, "\n");
+
+        if (fLocalForcedActions & VMCPU_FF_INHIBIT_INTERRUPTS)
+            pHlp->pfnPrintf(pHlp, "    intr inhibit RIP: %RGp\n", EMGetInhibitInterruptsPC(pVCpu));
+
+        /* the groups */
+        c = 0;
+        f = fLocalForcedActions;
+        PRINT_GROUP(VMCPU_FF_,EXTERNAL_SUSPENDED,_MASK);
+        PRINT_GROUP(VMCPU_FF_,EXTERNAL_HALTED,_MASK);
+        PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE,_MASK);
+        PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
+        PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_POST,_MASK);
+        PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY_POST,_MASK);
+        PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY,_MASK);
+        PRINT_GROUP(VMCPU_FF_,RESUME_GUEST,_MASK);
+        PRINT_GROUP(VMCPU_FF_,HM_TO_R3,_MASK);
+        PRINT_GROUP(VMCPU_FF_,ALL_REM,_MASK);
+        if (c)
+            pHlp->pfnPrintf(pHlp, "\n");
+    }
+
+#undef PRINT_FLAG
+#undef PRINT_GROUP
+}
+
diff --git a/src/VBox/VMM/VMMR3/VMMGuruMeditation.cpp b/src/VBox/VMM/VMMR3/VMMGuruMeditation.cpp
new file mode 100644
index 00000000..9f6ec49e
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/VMMGuruMeditation.cpp
@@ -0,0 +1,685 @@
+/* $Id: VMMGuruMeditation.cpp $ */
+/** @file
+ * VMM - The Virtual Machine Monitor, Guru Meditation Code.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/dbgf.h>
+#include "VMMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/em.h>
+
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/version.h>
+#include <VBox/vmm/hm.h>
+#include <iprt/assert.h>
+#include <iprt/dbg.h>
+#include <iprt/time.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+#include <iprt/stdarg.h>
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Structure to pass to DBGFR3Info() and for doing all other
+ * output during fatal dump.
+ */
+typedef struct VMMR3FATALDUMPINFOHLP
+{
+    /** The helper core. */
+    DBGFINFOHLP Core;
+    /** The release logger instance. */
+    PRTLOGGER   pRelLogger;
+    /** The saved release logger flags. */
+    uint32_t    fRelLoggerFlags;
+    /** The logger instance. */
+    PRTLOGGER   pLogger;
+    /** The saved logger flags. */
+    uint32_t    fLoggerFlags;
+    /** The saved logger destination flags. */
+    uint32_t    fLoggerDestFlags;
+    /** Whether to output to stderr or not. */
+    bool        fStdErr;
+    /** Whether we're still recording the summary or not. */
+    bool        fRecSummary;
+    /** Buffer for the summary. */
+    char        szSummary[4096 - 2];
+    /** The current summary offset. */
+    size_t      offSummary;
+    /** Standard error buffer.   */
+    char        achStdErrBuf[4096 - 8];
+    /** Standard error buffer offset. */
+    size_t      offStdErrBuf;
+} VMMR3FATALDUMPINFOHLP, *PVMMR3FATALDUMPINFOHLP;
+/** Pointer to a VMMR3FATALDUMPINFOHLP structure. */
+typedef const VMMR3FATALDUMPINFOHLP *PCVMMR3FATALDUMPINFOHLP;
+
+
+/**
+ * Flushes the content of achStdErrBuf, setting offStdErrBuf to zero.
+ *
+ * @param   pHlp        The instance to flush.
+ */
+static void vmmR3FatalDumpInfoHlpFlushStdErr(PVMMR3FATALDUMPINFOHLP pHlp)
+{
+    size_t cch = pHlp->offStdErrBuf;
+    if (cch)
+    {
+        RTStrmWrite(g_pStdErr, pHlp->achStdErrBuf, cch);
+        pHlp->offStdErrBuf = 0;
+    }
+}
+
+/**
+ * @callback_method_impl{FNRTSTROUTPUT, For buffering stderr output.}
+ */
+static DECLCALLBACK(size_t) vmmR3FatalDumpInfoHlp_BufferedStdErrOutput(void *pvArg, const char *pachChars, size_t cbChars)
+{
+    PVMMR3FATALDUMPINFOHLP pHlp = (PVMMR3FATALDUMPINFOHLP)pvArg;
+    if (cbChars)
+    {
+        size_t offBuf = pHlp->offStdErrBuf;
+        if (cbChars < sizeof(pHlp->achStdErrBuf) - offBuf)
+        { /* likely */ }
+        else
+        {
+            vmmR3FatalDumpInfoHlpFlushStdErr(pHlp);
+            if (cbChars < sizeof(pHlp->achStdErrBuf))
+                offBuf = 0;
+            else
+            {
+                RTStrmWrite(g_pStdErr, pachChars, cbChars);
+                return cbChars;
+            }
+        }
+        memcpy(&pHlp->achStdErrBuf[offBuf], pachChars, cbChars);
+        pHlp->offStdErrBuf = offBuf + cbChars;
+    }
+    return cbChars;
+}
+
+
+/**
+ * Print formatted string.
+ *
+ * @param   pHlp        Pointer to this structure.
+ * @param   pszFormat   The format string.
+ * @param   ...         Arguments.
+ */
+static DECLCALLBACK(void) vmmR3FatalDumpInfoHlp_pfnPrintf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...)
+{
+    va_list args;
+    va_start(args, pszFormat);
+    pHlp->pfnPrintfV(pHlp, pszFormat, args);
+    va_end(args);
+}
+
+/**
+ * Print formatted string.
+ *
+ * @param   pHlp        Pointer to this structure.
+ * @param   pszFormat   The format string.
+ * @param   args        Argument list.
+ */
+static DECLCALLBACK(void) vmmR3FatalDumpInfoHlp_pfnPrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args)
+{
+    PVMMR3FATALDUMPINFOHLP pMyHlp = (PVMMR3FATALDUMPINFOHLP)pHlp;
+
+    if (pMyHlp->pRelLogger)
+    {
+        va_list args2;
+        va_copy(args2, args);
+        RTLogLoggerV(pMyHlp->pRelLogger, pszFormat, args2);
+        va_end(args2);
+    }
+    if (pMyHlp->pLogger)
+    {
+        va_list args2;
+        va_copy(args2, args);
+        RTLogLoggerV(pMyHlp->pLogger, pszFormat, args);
+        va_end(args2);
+    }
+    if (pMyHlp->fStdErr)
+    {
+        va_list args2;
+        va_copy(args2, args);
+        RTStrFormatV(vmmR3FatalDumpInfoHlp_BufferedStdErrOutput, pMyHlp, NULL, NULL, pszFormat, args2);
+        //RTStrmPrintfV(g_pStdErr, pszFormat, args2);
+        va_end(args2);
+    }
+    if (pMyHlp->fRecSummary)
+    {
+        size_t cchLeft = sizeof(pMyHlp->szSummary) - pMyHlp->offSummary;
+        if (cchLeft > 1)
+        {
+            va_list args2;
+            va_copy(args2, args);
+            size_t cch = RTStrPrintfV(&pMyHlp->szSummary[pMyHlp->offSummary], cchLeft, pszFormat, args);
+            va_end(args2);
+            Assert(cch <= cchLeft);
+            pMyHlp->offSummary += cch;
+        }
+    }
+}
+
+
+/**
+ * Initializes the fatal dump output helper.
+ *
+ * @param   pHlp        The structure to initialize.
+ */
+static void vmmR3FatalDumpInfoHlpInit(PVMMR3FATALDUMPINFOHLP pHlp)
+{
+    RT_BZERO(pHlp, sizeof(*pHlp));
+
+    pHlp->Core.pfnPrintf      = vmmR3FatalDumpInfoHlp_pfnPrintf;
+    pHlp->Core.pfnPrintfV     = vmmR3FatalDumpInfoHlp_pfnPrintfV;
+    pHlp->Core.pfnGetOptError = DBGFR3InfoGenricGetOptError;
+
+    /*
+     * The loggers.
+     */
+    pHlp->pRelLogger  = RTLogRelGetDefaultInstance();
+#ifdef LOG_ENABLED
+    pHlp->pLogger     = RTLogDefaultInstance();
+#else
+    if (pHlp->pRelLogger)
+        pHlp->pLogger = RTLogGetDefaultInstance();
+    else
+        pHlp->pLogger = RTLogDefaultInstance();
+#endif
+
+    if (pHlp->pRelLogger)
+    {
+        pHlp->fRelLoggerFlags = pHlp->pRelLogger->fFlags;
+        pHlp->pRelLogger->fFlags &= ~RTLOGFLAGS_DISABLED;
+        pHlp->pRelLogger->fFlags |= RTLOGFLAGS_BUFFERED;
+    }
+
+    if (pHlp->pLogger)
+    {
+        pHlp->fLoggerFlags     = pHlp->pLogger->fFlags;
+        pHlp->fLoggerDestFlags = pHlp->pLogger->fDestFlags;
+        pHlp->pLogger->fFlags     &= ~RTLOGFLAGS_DISABLED;
+        pHlp->pLogger->fFlags     |= RTLOGFLAGS_BUFFERED;
+#ifndef DEBUG_sandervl
+        pHlp->pLogger->fDestFlags |= RTLOGDEST_DEBUGGER;
+#endif
+    }
+
+    /*
+     * Check if we need write to stderr.
+     */
+    pHlp->fStdErr = (!pHlp->pRelLogger || !(pHlp->pRelLogger->fDestFlags & (RTLOGDEST_STDOUT | RTLOGDEST_STDERR)))
+                 && (!pHlp->pLogger    || !(pHlp->pLogger->fDestFlags    & (RTLOGDEST_STDOUT | RTLOGDEST_STDERR)));
+#ifdef DEBUG_sandervl
+    pHlp->fStdErr = false; /* takes too long to display here */
+#endif
+    pHlp->offStdErrBuf = 0;
+
+    /*
+     * Init the summary recording.
+     */
+    pHlp->fRecSummary  = true;
+    pHlp->offSummary   = 0;
+    pHlp->szSummary[0] = '\0';
+}
+
+
+/**
+ * Deletes the fatal dump output helper.
+ *
+ * @param   pHlp        The structure to delete.
+ */
+static void vmmR3FatalDumpInfoHlpDelete(PVMMR3FATALDUMPINFOHLP pHlp)
+{
+    if (pHlp->pRelLogger)
+    {
+        RTLogFlush(pHlp->pRelLogger);
+        pHlp->pRelLogger->fFlags = pHlp->fRelLoggerFlags;
+    }
+
+    if (pHlp->pLogger)
+    {
+        RTLogFlush(pHlp->pLogger);
+        pHlp->pLogger->fFlags     = pHlp->fLoggerFlags;
+        pHlp->pLogger->fDestFlags = pHlp->fLoggerDestFlags;
+    }
+
+    if (pHlp->fStdErr)
+        vmmR3FatalDumpInfoHlpFlushStdErr(pHlp);
+}
+
+
+/**
+ * Dumps the VM state on a fatal error.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   rcErr       VBox status code.
+ */
+VMMR3DECL(void) VMMR3FatalDump(PVM pVM, PVMCPU pVCpu, int rcErr)
+{
+    /*
+     * Create our output helper and sync it with the log settings.
+     * This helper will be used for all the output.
+     */
+    VMMR3FATALDUMPINFOHLP   Hlp;
+    PCDBGFINFOHLP           pHlp = &Hlp.Core;
+    vmmR3FatalDumpInfoHlpInit(&Hlp);
+
+    /* Release owned locks to make sure other VCPUs can continue in case they were waiting for one. */
+    PDMR3CritSectLeaveAll(pVM);
+
+    /*
+     * Header.
+     */
+    pHlp->pfnPrintf(pHlp,
+                    "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+                    "!!\n"
+                    "!!         VCPU%u: Guru Meditation %d (%Rrc)\n"
+                    "!!\n",
+                    pVCpu->idCpu, rcErr, rcErr);
+
+    /*
+     * Continue according to context.
+     */
+    bool fDoneHyper = false;
+    bool fDoneImport = false;
+    switch (rcErr)
+    {
+        /*
+         * Hypervisor errors.
+         */
+        case VERR_VMM_RING0_ASSERTION:
+        case VINF_EM_DBG_HYPER_ASSERTION:
+        case VERR_VMM_RING3_CALL_DISABLED:
+        {
+            const char *pszMsg1 = VMMR3GetRZAssertMsg1(pVM);
+            while (pszMsg1 && *pszMsg1 == '\n')
+                pszMsg1++;
+            const char *pszMsg2 = VMMR3GetRZAssertMsg2(pVM);
+            while (pszMsg2 && *pszMsg2 == '\n')
+                pszMsg2++;
+            pHlp->pfnPrintf(pHlp,
+                            "%s"
+                            "%s",
+                            pszMsg1,
+                            pszMsg2);
+            if (    !pszMsg2
+                ||  !*pszMsg2
+                ||  strchr(pszMsg2, '\0')[-1] != '\n')
+                pHlp->pfnPrintf(pHlp, "\n");
+        }
+        RT_FALL_THRU();
+        case VERR_TRPM_DONT_PANIC:
+        case VERR_TRPM_PANIC:
+        case VINF_EM_RAW_STALE_SELECTOR:
+        case VINF_EM_RAW_IRET_TRAP:
+        case VINF_EM_DBG_HYPER_BREAKPOINT:
+        case VINF_EM_DBG_HYPER_STEPPED:
+        case VINF_EM_TRIPLE_FAULT:
+        case VERR_VMM_HYPER_CR3_MISMATCH:
+        {
+            /*
+             * Active trap? This is only of partial interest when in hardware
+             * assisted virtualization mode, thus the different messages.
+             */
+            uint32_t        uEIP       = 0; //CPUMGetHyperEIP(pVCpu);
+            TRPMEVENT       enmType;
+            uint8_t         u8TrapNo   =       0xce;
+            uint32_t        uErrorCode = 0xdeadface;
+            RTGCUINTPTR     uCR2       = 0xdeadface;
+            uint8_t         cbInstr    = UINT8_MAX;
+            bool            fIcebp     = false;
+            int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrorCode, &uCR2, &cbInstr, &fIcebp);
+            if (VM_IS_RAW_MODE_ENABLED(pVM))
+            {
+                if (RT_SUCCESS(rc2))
+                    pHlp->pfnPrintf(pHlp,
+                                    "!! TRAP=%02x ERRCD=%RX32 CR2=%RGv EIP=%RX32 Type=%d cbInstr=%02x fIcebp=%RTbool\n",
+                                    u8TrapNo, uErrorCode, uCR2, uEIP, enmType, cbInstr, fIcebp);
+                else
+                    pHlp->pfnPrintf(pHlp,
+                                    "!! EIP=%RX32 NOTRAP\n",
+                                    uEIP);
+            }
+            else if (RT_SUCCESS(rc2))
+                pHlp->pfnPrintf(pHlp,
+                                "!! ACTIVE TRAP=%02x ERRCD=%RX32 CR2=%RGv PC=%RGr Type=%d cbInstr=%02x fIcebp=%RTbool (Guest!)\n",
+                                u8TrapNo, uErrorCode, uCR2, CPUMGetGuestRIP(pVCpu), enmType, cbInstr, fIcebp);
+
+            /*
+             * Dump the relevant hypervisor registers and stack.
+             */
+            if (   rcErr == VERR_VMM_RING0_ASSERTION /* fInRing3Call has already been cleared here. */
+                || pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
+            {
+                /* Dump the jmpbuf.  */
+                pHlp->pfnPrintf(pHlp,
+                                "!!\n"
+                                "!! CallRing3JmpBuf:\n"
+                                "!!\n");
+                pHlp->pfnPrintf(pHlp,
+                                "SavedEsp=%RHv SavedEbp=%RHv SpResume=%RHv SpCheck=%RHv\n",
+                                pVCpu->vmm.s.CallRing3JmpBufR0.SavedEsp,
+                                pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp,
+                                pVCpu->vmm.s.CallRing3JmpBufR0.SpResume,
+                                pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck);
+                pHlp->pfnPrintf(pHlp,
+                                "pvSavedStack=%RHv cbSavedStack=%#x  fInRing3Call=%RTbool\n",
+                                pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack,
+                                pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack,
+                                pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call);
+                pHlp->pfnPrintf(pHlp,
+                                "cbUsedMax=%#x cbUsedAvg=%#x cbUsedTotal=%#llx cUsedTotal=%#llx\n",
+                                pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedMax,
+                                pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedAvg,
+                                pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedTotal,
+                                pVCpu->vmm.s.CallRing3JmpBufR0.cUsedTotal);
+
+                /* Dump the resume register frame on the stack. */
+                PRTHCUINTPTR pBP;
+#ifdef VMM_R0_SWITCH_STACK
+                pBP = (PRTHCUINTPTR)&pVCpu->vmm.s.pbEMTStackR3[  pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp
+                                                               - MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3)];
+#else
+                pBP = (PRTHCUINTPTR)&pVCpu->vmm.s.pbEMTStackR3[  pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack
+                                                               - pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck
+                                                               + pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp];
+#endif
+#if HC_ARCH_BITS == 32
+                pHlp->pfnPrintf(pHlp,
+                                "eax=volatile ebx=%08x ecx=volatile edx=volatile esi=%08x edi=%08x\n"
+                                "eip=%08x esp=%08x ebp=%08x efl=%08x\n"
+                                ,
+                                pBP[-3], pBP[-2], pBP[-1],
+                                pBP[1], pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp - 8, pBP[0], pBP[-4]);
+#else
+# ifdef RT_OS_WINDOWS
+                pHlp->pfnPrintf(pHlp,
+                                "rax=volatile         rbx=%016RX64 rcx=volatile         rdx=volatile\n"
+                                "rsi=%016RX64 rdi=%016RX64  r8=volatile          r9=volatile        \n"
+                                "r10=volatile         r11=volatile         r12=%016RX64 r13=%016RX64\n"
+                                "r14=%016RX64 r15=%016RX64\n"
+                                "rip=%016RX64 rsp=%016RX64 rbp=%016RX64 rfl=%08RX64\n"
+                                ,
+                                pBP[-7],
+                                pBP[-6], pBP[-5],
+                                pBP[-4], pBP[-3],
+                                pBP[-2], pBP[-1],
+                                pBP[1], pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp - 16, pBP[0], pBP[-8]);
+# else
+                pHlp->pfnPrintf(pHlp,
+                                "rax=volatile         rbx=%016RX64 rcx=volatile         rdx=volatile\n"
+                                "rsi=volatile         rdi=volatile          r8=volatile          r9=volatile        \n"
+                                "r10=volatile         r11=volatile         r12=%016RX64 r13=%016RX64\n"
+                                "r14=%016RX64 r15=%016RX64\n"
+                                "rip=%016RX64 rsp=%016RX64 rbp=%016RX64 rflags=%08RX64\n"
+                                ,
+                                pBP[-5],
+                                pBP[-4], pBP[-3],
+                                pBP[-2], pBP[-1],
+                                pBP[1], pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp - 16, pBP[0], pBP[-6]);
+# endif
+#endif
+
+                /* Callstack. */
+                DBGFADDRESS AddrPc, AddrBp, AddrSp;
+                PCDBGFSTACKFRAME pFirstFrame;
+                rc2 = DBGFR3StackWalkBeginEx(pVM->pUVM, pVCpu->idCpu, DBGFCODETYPE_RING0,
+                                             DBGFR3AddrFromHostR0(&AddrBp, pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp),
+                                             DBGFR3AddrFromHostR0(&AddrSp, pVCpu->vmm.s.CallRing3JmpBufR0.SpResume),
+                                             DBGFR3AddrFromHostR0(&AddrPc, pVCpu->vmm.s.CallRing3JmpBufR0.SavedEipForUnwind),
+                                             RTDBGRETURNTYPE_INVALID, &pFirstFrame);
+                if (RT_SUCCESS(rc2))
+                {
+                    pHlp->pfnPrintf(pHlp,
+                                    "!!\n"
+                                    "!! Call Stack:\n"
+                                    "!!\n");
+#if HC_ARCH_BITS == 32
+                    pHlp->pfnPrintf(pHlp, "EBP      Ret EBP  Ret CS:EIP    Arg0     Arg1     Arg2     Arg3     CS:EIP        Symbol [line]\n");
+#else
+                    pHlp->pfnPrintf(pHlp, "RBP              Ret RBP          Ret RIP          RIP              Symbol [line]\n");
+#endif
+                    for (PCDBGFSTACKFRAME pFrame = pFirstFrame;
+                         pFrame;
+                         pFrame = DBGFR3StackWalkNext(pFrame))
+                    {
+#if HC_ARCH_BITS == 32
+                        pHlp->pfnPrintf(pHlp,
+                                        "%RHv %RHv %04RX32:%RHv %RHv %RHv %RHv %RHv",
+                                        (RTHCUINTPTR)pFrame->AddrFrame.off,
+                                        (RTHCUINTPTR)pFrame->AddrReturnFrame.off,
+                                        (RTHCUINTPTR)pFrame->AddrReturnPC.Sel,
+                                        (RTHCUINTPTR)pFrame->AddrReturnPC.off,
+                                        pFrame->Args.au32[0],
+                                        pFrame->Args.au32[1],
+                                        pFrame->Args.au32[2],
+                                        pFrame->Args.au32[3]);
+                        pHlp->pfnPrintf(pHlp, " %RTsel:%08RHv", pFrame->AddrPC.Sel, pFrame->AddrPC.off);
+#else
+                        pHlp->pfnPrintf(pHlp,
+                                        "%RHv %RHv %RHv %RHv",
+                                        (RTHCUINTPTR)pFrame->AddrFrame.off,
+                                        (RTHCUINTPTR)pFrame->AddrReturnFrame.off,
+                                        (RTHCUINTPTR)pFrame->AddrReturnPC.off,
+                                        (RTHCUINTPTR)pFrame->AddrPC.off);
+#endif
+                        if (pFrame->pSymPC)
+                        {
+                            RTGCINTPTR offDisp = pFrame->AddrPC.FlatPtr - pFrame->pSymPC->Value;
+                            if (offDisp > 0)
+                                pHlp->pfnPrintf(pHlp, " %s+%llx", pFrame->pSymPC->szName, (int64_t)offDisp);
+                            else if (offDisp < 0)
+                                pHlp->pfnPrintf(pHlp, " %s-%llx", pFrame->pSymPC->szName, -(int64_t)offDisp);
+                            else
+                                pHlp->pfnPrintf(pHlp, " %s", pFrame->pSymPC->szName);
+                        }
+                        if (pFrame->pLinePC)
+                            pHlp->pfnPrintf(pHlp, " [%s @ 0i%d]", pFrame->pLinePC->szFilename, pFrame->pLinePC->uLineNo);
+                        pHlp->pfnPrintf(pHlp, "\n");
+                        for (uint32_t iReg = 0; iReg < pFrame->cSureRegs; iReg++)
+                        {
+                            const char *pszName = pFrame->paSureRegs[iReg].pszName;
+                            if (!pszName)
+                                pszName = DBGFR3RegCpuName(pVM->pUVM, pFrame->paSureRegs[iReg].enmReg,
+                                                           pFrame->paSureRegs[iReg].enmType);
+                            char szValue[1024];
+                            szValue[0] = '\0';
+                            DBGFR3RegFormatValue(szValue, sizeof(szValue), &pFrame->paSureRegs[iReg].Value,
+                                                 pFrame->paSureRegs[iReg].enmType, false);
+                            pHlp->pfnPrintf(pHlp, "     %-3s=%s\n", pszName, szValue);
+                        }
+                    }
+                    DBGFR3StackWalkEnd(pFirstFrame);
+                }
+
+                /* Symbols on the stack. */
+#ifdef VMM_R0_SWITCH_STACK
+                uint32_t const   iLast   = VMM_STACK_SIZE / sizeof(uintptr_t);
+                uint32_t         iAddr   = (uint32_t)(  pVCpu->vmm.s.CallRing3JmpBufR0.SavedEsp
+                                                      - MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3)) / sizeof(uintptr_t);
+                if (iAddr > iLast)
+                    iAddr = 0;
+#else
+                uint32_t const   iLast   = RT_MIN(pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack, VMM_STACK_SIZE)
+                                         / sizeof(uintptr_t);
+                uint32_t         iAddr   = 0;
+#endif
+                pHlp->pfnPrintf(pHlp,
+                                "!!\n"
+                                "!! Addresses on the stack (iAddr=%#x, iLast=%#x)\n"
+                                "!!\n",
+                                iAddr, iLast);
+                uintptr_t const *paAddr  = (uintptr_t const *)pVCpu->vmm.s.pbEMTStackR3;
+                while (iAddr < iLast)
+                {
+                    uintptr_t const uAddr = paAddr[iAddr];
+                    if (uAddr > X86_PAGE_SIZE)
+                    {
+                        DBGFADDRESS  Addr;
+                        DBGFR3AddrFromFlat(pVM->pUVM, &Addr, uAddr);
+                        RTGCINTPTR   offDisp = 0;
+                        PRTDBGSYMBOL pSym  = DBGFR3AsSymbolByAddrA(pVM->pUVM, DBGF_AS_R0, &Addr,
+                                                                   RTDBGSYMADDR_FLAGS_LESS_OR_EQUAL | RTDBGSYMADDR_FLAGS_SKIP_ABS_IN_DEFERRED,
+                                                                   &offDisp, NULL);
+                        RTGCINTPTR   offLineDisp;
+                        PRTDBGLINE   pLine = DBGFR3AsLineByAddrA(pVM->pUVM, DBGF_AS_R0, &Addr, &offLineDisp, NULL);
+                        if (pLine || pSym)
+                        {
+                            pHlp->pfnPrintf(pHlp, "%#06x: %p =>", iAddr * sizeof(uintptr_t), uAddr);
+                            if (pSym)
+                                pHlp->pfnPrintf(pHlp, " %s + %#x", pSym->szName, (intptr_t)offDisp);
+                            if (pLine)
+                                pHlp->pfnPrintf(pHlp, " [%s:%u + %#x]\n", pLine->szFilename, pLine->uLineNo, offLineDisp);
+                            else
+                                pHlp->pfnPrintf(pHlp, "\n");
+                            RTDbgSymbolFree(pSym);
+                            RTDbgLineFree(pLine);
+                        }
+                    }
+                    iAddr++;
+                }
+
+                /* raw stack */
+                Hlp.fRecSummary = false;
+                pHlp->pfnPrintf(pHlp,
+                                "!!\n"
+                                "!! Raw stack (mind the direction).\n"
+                                "!! pbEMTStackR0=%RHv pbEMTStackBottomR0=%RHv VMM_STACK_SIZE=%#x\n"
+                                "!! pbEmtStackR3=%p\n"
+                                "!!\n"
+                                "%.*Rhxd\n",
+                                MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3),
+                                MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3) + VMM_STACK_SIZE,
+                                VMM_STACK_SIZE,
+                                pVCpu->vmm.s.pbEMTStackR3,
+                                VMM_STACK_SIZE, pVCpu->vmm.s.pbEMTStackR3);
+            }
+            else
+            {
+                pHlp->pfnPrintf(pHlp,
+                                "!! Skipping ring-0 registers and stack, rcErr=%Rrc\n", rcErr);
+            }
+            break;
+        }
+
+        case VERR_IEM_INSTR_NOT_IMPLEMENTED:
+        case VERR_IEM_ASPECT_NOT_IMPLEMENTED:
+        case VERR_PATM_IPE_TRAP_IN_PATCH_CODE:
+        case VERR_EM_GUEST_CPU_HANG:
+        {
+            CPUMImportGuestStateOnDemand(pVCpu, CPUMCTX_EXTRN_ABSOLUTELY_ALL);
+            fDoneImport = true;
+
+            DBGFR3Info(pVM->pUVM, "cpumguest", NULL, pHlp);
+            DBGFR3Info(pVM->pUVM, "cpumguestinstr", NULL, pHlp);
+            DBGFR3Info(pVM->pUVM, "cpumguesthwvirt", NULL, pHlp);
+            break;
+        }
+
+        default:
+        {
+            break;
+        }
+
+    } /* switch (rcErr) */
+    Hlp.fRecSummary = false;
+
+
+    /*
+     * Generic info dumper loop.
+     */
+    if (!fDoneImport)
+        CPUMImportGuestStateOnDemand(pVCpu, CPUMCTX_EXTRN_ABSOLUTELY_ALL);
+    static struct
+    {
+        const char *pszInfo;
+        const char *pszArgs;
+    } const     aInfo[] =
+    {
+        { "mappings",        NULL },
+        { "hma",             NULL },
+        { "cpumguest",       "verbose" },
+        { "cpumguesthwvirt", "verbose" },
+        { "cpumguestinstr",  "verbose" },
+        { "cpumhyper",       "verbose" },
+        { "cpumhost",        "verbose" },
+        { "mode",            "all" },
+        { "cpuid",           "verbose" },
+        { "handlers",        "phys virt hyper stats" },
+        { "timers",          NULL },
+        { "activetimers",    NULL },
+    };
+    for (unsigned i = 0; i < RT_ELEMENTS(aInfo); i++)
+    {
+        if (fDoneHyper && !strcmp(aInfo[i].pszInfo, "cpumhyper"))
+            continue;
+        pHlp->pfnPrintf(pHlp,
+                        "!!\n"
+                        "!! {%s, %s}\n"
+                        "!!\n",
+                        aInfo[i].pszInfo, aInfo[i].pszArgs);
+        DBGFR3Info(pVM->pUVM, aInfo[i].pszInfo, aInfo[i].pszArgs, pHlp);
+    }
+
+    /* All other info items */
+    DBGFR3InfoMulti(pVM,
+                    "*",
+                    "mappings|hma|cpum|cpumguest|cpumguesthwvirt|cpumguestinstr|cpumhyper|cpumhost|mode|cpuid"
+                    "|pgmpd|pgmcr3|timers|activetimers|handlers|help|exithistory",
+                    "!!\n"
+                    "!! {%s}\n"
+                    "!!\n",
+                    pHlp);
+
+
+    /* done */
+    pHlp->pfnPrintf(pHlp,
+                    "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n");
+
+
+    /*
+     * Repeat the summary to stderr so we don't have to scroll half a mile up.
+     */
+    vmmR3FatalDumpInfoHlpFlushStdErr(&Hlp);
+    if (Hlp.szSummary[0])
+        RTStrmPrintf(g_pStdErr,
+                     "%s"
+                     "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
+                     Hlp.szSummary);
+
+    /*
+     * Delete the output instance (flushing and restoring of flags).
+     */
+    vmmR3FatalDumpInfoHlpDelete(&Hlp);
+}
+
diff --git a/src/VBox/VMM/VMMR3/VMMR3.def b/src/VBox/VMM/VMMR3/VMMR3.def
new file mode 100644
index 00000000..cb88318b
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/VMMR3.def
@@ -0,0 +1,457 @@
+; $Id: VMMR3.def $
+;; @file
+; VMM Ring-3 Context DLL - Definition file.
+
+;
+; Copyright (C) 2010-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+LIBRARY VBoxVMM.dll
+EXPORTS
+    ; data
+
+    ; code
+    CFGMR3GetRoot
+    CFGMR3GetFirstChild
+    CFGMR3GetNextChild
+    CFGMR3GetNameLen
+    CFGMR3GetFirstValue
+    CFGMR3GetNextValue
+    CFGMR3GetValueNameLen
+    CFGMR3GetValueType
+    CFGMR3Dump
+    CFGMR3CreateTree
+    CFGMR3DestroyTree
+    CFGMR3GetValueName
+    CFGMR3GetName
+    CFGMR3RemoveNode
+    CFGMR3InsertBytes
+    CFGMR3InsertStringFV
+    CFGMR3InsertStringF
+    CFGMR3InsertStringN
+    CFGMR3InsertString
+    CFGMR3InsertStringW
+    CFGMR3InsertInteger
+    CFGMR3QueryStringAllocDef
+    CFGMR3RemoveValue
+    CFGMR3QueryIntegerDef
+    CFGMR3QueryGCPtrSDef
+    CFGMR3QueryGCPtrUDef
+    CFGMR3QueryGCPtrDef
+    CFGMR3QueryPtrDef
+    CFGMR3QueryBoolDef
+    CFGMR3QueryS8Def
+    CFGMR3QueryU8Def
+    CFGMR3QueryS16Def
+    CFGMR3QueryU16Def
+    CFGMR3QueryPortDef
+    CFGMR3QueryS32Def
+    CFGMR3QuerySIntDef
+    CFGMR3QueryU32Def
+    CFGMR3QueryUIntDef
+    CFGMR3QueryS64Def
+    CFGMR3QueryU64Def
+    CFGMR3QueryInteger
+    CFGMR3QueryGCPtrS
+    CFGMR3QueryGCPtrU
+    CFGMR3QueryGCPtr
+    CFGMR3QueryPtr
+    CFGMR3QueryBool
+    CFGMR3QueryS8
+    CFGMR3QueryU8
+    CFGMR3QueryS16
+    CFGMR3QueryU16
+    CFGMR3QueryPort
+    CFGMR3QueryS32
+    CFGMR3QuerySInt
+    CFGMR3QueryU32
+    CFGMR3QueryUInt
+    CFGMR3QueryS64
+    CFGMR3QueryU64
+    CFGMR3QuerySize
+    CFGMR3QueryType
+    CFGMR3AreValuesValid
+    CFGMR3AreChildrenValid
+    CFGMR3GetChildFV
+    CFGMR3GetChildF
+    CFGMR3GetChild
+    CFGMR3InsertNode
+    CFGMR3InsertNodeFV
+    CFGMR3InsertNodeF
+    CFGMR3InsertSubTree
+    CFGMR3ValidateConfig
+    CFGMR3QueryBytes
+    CFGMR3QueryStringDef
+    CFGMR3QueryString
+    CFGMR3QueryStringAlloc
+    CFGMR3GetParent
+    CFGMR3GetRootU
+
+    CPUMGetHostMicroarch
+    CPUMGetGuestMicroarch
+
+    DBGCCreate
+
+    DBGFR3CoreWrite
+    DBGFR3Info
+    DBGFR3InfoRegisterExternal
+    DBGFR3InfoDeregisterExternal
+    DBGFR3InfoGenricGetOptError
+    DBGFR3InjectNMI
+    DBGFR3LogModifyDestinations
+    DBGFR3LogModifyFlags
+    DBGFR3LogModifyGroups
+    DBGFR3OSDetect
+    DBGFR3OSQueryNameAndVersion
+    DBGFR3RegCpuQueryU8
+    DBGFR3RegCpuQueryU16
+    DBGFR3RegCpuQueryU32
+    DBGFR3RegCpuQueryU64
+    DBGFR3RegCpuQueryXdtr
+    DBGFR3RegCpuQueryLrd
+    DBGFR3RegFormatValue
+    DBGFR3RegNmQuery
+    DBGFR3RegNmQueryAll
+    DBGFR3RegNmQueryAllCount
+    DBGFR3OSDeregister
+    DBGFR3OSRegister
+    DBGFR3OSQueryInterface
+    DBGFR3MemReadString
+    DBGFR3MemRead
+    DBGFR3MemScan
+    DBGFR3ModInMem
+    DBGFR3AddrFromFlat
+    DBGFR3AsSymbolByName
+    DBGFR3AsResolveAndRetain
+    DBGFR3AsSetAlias
+    DBGFR3AddrAdd
+    DBGFR3AddrSub
+    DBGFR3AsGetConfig
+    DBGFR3CpuGetMode
+    DBGFR3AddrFromSelOff
+    DBGFR3FlowCreate
+    DBGFR3FlowRetain
+    DBGFR3FlowRelease
+    DBGFR3FlowQueryStartBb
+    DBGFR3FlowQueryBbByAddress
+    DBGFR3FlowQueryBranchTblByAddress
+    DBGFR3FlowGetBbCount
+    DBGFR3FlowGetBranchTblCount
+    DBGFR3FlowBbRetain
+    DBGFR3FlowBbRelease
+    DBGFR3FlowBbGetStartAddress
+    DBGFR3FlowBbGetEndAddress
+    DBGFR3FlowBbGetBranchAddress
+    DBGFR3FlowBbGetFollowingAddress
+    DBGFR3FlowBbGetType
+    DBGFR3FlowBbGetInstrCount
+    DBGFR3FlowBbGetFlags
+    DBGFR3FlowBbQueryBranchTbl
+    DBGFR3FlowBbQueryError
+    DBGFR3FlowBbQueryInstr
+    DBGFR3FlowBbQuerySuccessors
+    DBGFR3FlowBbGetRefBbCount
+    DBGFR3FlowBbGetRefBb
+    DBGFR3FlowBranchTblRetain
+    DBGFR3FlowBranchTblRelease
+    DBGFR3FlowBranchTblGetSlots
+    DBGFR3FlowBranchTblGetStartAddress
+    DBGFR3FlowBranchTblGetAddrAtSlot
+    DBGFR3FlowBranchTblQueryAddresses
+    DBGFR3FlowItCreate
+    DBGFR3FlowItDestroy
+    DBGFR3FlowItNext
+    DBGFR3FlowItReset
+    DBGFR3FlowBranchTblItCreate
+    DBGFR3FlowBranchTblItDestroy
+    DBGFR3FlowBranchTblItNext
+    DBGFR3FlowBranchTblItReset
+    DBGFR3PlugInLoad
+    DBGFR3PlugInUnload
+    DBGFR3PlugInLoadAll
+    DBGFR3PlugInUnloadAll
+    DBGFR3SelQueryInfo
+    DBGFR3StackWalkBegin
+    DBGFR3StackWalkNext
+    DBGFR3StackWalkEnd
+    DBGFR3TypeDeregister
+    DBGFR3TypeDumpEx
+    DBGFR3TypeQueryReg
+    DBGFR3TypeQuerySize
+    DBGFR3TypeQueryValByType
+    DBGFR3TypeRegister
+    DBGFR3TypeSetSize
+    DBGFR3TypeValFree
+    DBGFR3TypeValDumpEx
+
+    EMR3QueryExecutionPolicy
+    EMR3QueryMainExecutionEngine
+    EMR3SetExecutionPolicy
+
+    MMHyperR3ToR0
+    MMHyperR3ToRC
+
+    HMR3IsEnabled
+    HMR3IsNestedPagingActive
+    HMR3IsUXActive
+    HMR3IsVpidActive
+
+    MMR3HeapFree
+    MMR3HeapRealloc
+    MMR3HeapAllocU
+
+    MMR3HyperAllocOnceNoRel
+
+    PDMR3AsyncCompletionBwMgrSetMaxForFile
+    PDMR3DeviceAttach
+    PDMR3DeviceDetach
+    PDMR3DriverAttach
+    PDMR3DriverDetach
+    PDMR3NsBwGroupSetLimit
+    PDMR3QueryDeviceLun
+    PDMR3QueryDriverOnLun
+    PDMR3QueryLun
+
+    PDMCritSectEnter
+    PDMCritSectEnterDebug
+    PDMCritSectTryEnter
+    PDMCritSectTryEnterDebug
+    PDMR3CritSectEnterEx
+    PDMCritSectLeave
+    PDMCritSectIsOwner
+    PDMCritSectIsOwnerEx
+    PDMCritSectIsOwned
+    PDMCritSectIsInitialized
+    PDMCritSectHasWaiters
+    PDMCritSectGetRecursion
+    PDMR3CritSectYield
+    PDMR3CritSectName
+    PDMR3CritSectScheduleExitEvent
+    PDMR3CritSectDelete
+
+    PDMR3QueueDestroy
+    PDMQueueAlloc
+    PDMQueueInsert
+    PDMQueueInsertEx
+    PDMQueueR0Ptr
+    PDMQueueRCPtr
+
+    PDMR3ThreadDestroy
+    PDMR3ThreadIAmRunning
+    PDMR3ThreadIAmSuspending
+    PDMR3ThreadResume
+    PDMR3ThreadSleep
+    PDMR3ThreadSuspend
+
+    PDMR3UsbCreateEmulatedDevice
+    PDMR3UsbCreateProxyDevice
+    PDMR3UsbDetachDevice
+    PDMR3UsbHasHub
+    PDMR3UsbDriverAttach
+    PDMR3UsbDriverDetach
+    PDMR3UsbQueryLun
+    PDMR3UsbQueryDriverOnLun
+
+    PGMHandlerPhysicalPageTempOff
+    PGMPhysReadGCPtr
+    PGMPhysSimpleDirtyWriteGCPtr
+    PGMPhysSimpleReadGCPtr
+    PGMPhysSimpleWriteGCPhys
+    PGMPhysSimpleWriteGCPtr
+    PGMPhysWriteGCPtr
+    PGMShwMakePageWritable
+    PGMR3QueryGlobalMemoryStats
+    PGMR3QueryMemoryStats
+
+    SSMR3Close
+    SSMR3DeregisterExternal
+    SSMR3DeregisterInternal
+    SSMR3GetBool
+    SSMR3GetBoolV
+    SSMR3GetGCPhys
+    SSMR3GetGCPhysV
+    SSMR3GetGCPhys32
+    SSMR3GetGCPhys32V
+    SSMR3GetGCPhys64
+    SSMR3GetGCPhys64V
+    SSMR3GetGCPtr
+    SSMR3GetGCUInt
+    SSMR3GetGCUIntPtr
+    SSMR3GetGCUIntReg
+    SSMR3GetIOPort
+    SSMR3GetMem
+    SSMR3GetRCPtr
+    SSMR3GetS128
+    SSMR3GetS128V
+    SSMR3GetS16
+    SSMR3GetS16V
+    SSMR3GetS32
+    SSMR3GetS32V
+    SSMR3GetS64
+    SSMR3GetS64V
+    SSMR3GetS8
+    SSMR3GetS8V
+    SSMR3GetSInt
+    SSMR3GetSel
+    SSMR3GetStrZ
+    SSMR3GetStrZEx
+    SSMR3GetStruct
+    SSMR3GetStructEx
+    SSMR3GetU128
+    SSMR3GetU128V
+    SSMR3GetU16
+    SSMR3GetU16V
+    SSMR3GetU32
+    SSMR3GetU32V
+    SSMR3GetU64
+    SSMR3GetU64V
+    SSMR3GetU8
+    SSMR3GetU8V
+    SSMR3GetUInt
+    SSMR3HandleGetAfter
+    SSMR3HandleGetStatus
+    SSMR3HandleHostBits
+    SSMR3HandleHostOSAndArch
+    SSMR3HandleIsLiveSave
+    SSMR3HandleMaxDowntime
+    SSMR3HandleReportLivePercent
+    SSMR3HandleRevision
+    SSMR3HandleSetStatus
+    SSMR3HandleVersion
+    SSMR3Open
+    SSMR3PutBool
+    SSMR3PutGCPhys
+    SSMR3PutGCPhys32
+    SSMR3PutGCPhys64
+    SSMR3PutGCPtr
+    SSMR3PutGCUInt
+    SSMR3PutGCUIntPtr
+    SSMR3PutGCUIntReg
+    SSMR3PutIOPort
+    SSMR3PutMem
+    SSMR3PutRCPtr
+    SSMR3PutS128
+    SSMR3PutS16
+    SSMR3PutS32
+    SSMR3PutS64
+    SSMR3PutS8
+    SSMR3PutSInt
+    SSMR3PutSel
+    SSMR3PutStrZ
+    SSMR3PutStruct
+    SSMR3PutStructEx
+    SSMR3PutU128
+    SSMR3PutU16
+    SSMR3PutU32
+    SSMR3PutU64
+    SSMR3PutU8
+    SSMR3PutUInt
+    SSMR3Seek
+    SSMR3SetCfgError
+    SSMR3SetLoadError
+    SSMR3SetLoadErrorV
+    SSMR3Skip
+    SSMR3SkipToEndOfUnit
+    SSMR3ValidateFile
+    SSMR3Cancel
+    SSMR3RegisterExternal
+
+    STAMR3Dump
+    STAMR3Enum
+    STAMR3Reset
+    STAMR3Snapshot
+    STAMR3SnapshotFree
+    STAMR3GetUnit
+    STAMR3RegisterFU
+    STAMR3RegisterVU
+    STAMR3DeregisterF
+    STAMR3DeregisterV
+
+    TMR3GetCpuLoadPercents
+    TMR3TimerSetCritSect
+    TMR3TimerLoad
+    TMR3TimerSave
+    TMR3TimerSkip
+    TMR3TimerDestroy
+    TMTimerFromMicro
+    TMTimerFromMilli
+    TMTimerFromNano
+    TMTimerGet
+    TMTimerGetFreq
+    TMTimerGetMicro
+    TMTimerGetMilli
+    TMTimerGetNano
+    TMTimerIsActive
+    TMTimerIsLockOwner
+    TMTimerLock
+    TMTimerR0Ptr
+    TMTimerR3Ptr
+    TMTimerRCPtr
+    TMTimerSet
+    TMTimerSetFrequencyHint
+    TMTimerSetMicro
+    TMTimerSetMillies
+    TMTimerSetNano
+    TMTimerSetRelative
+    TMTimerStop
+    TMTimerToMicro
+    TMTimerToMilli
+    TMTimerToNano
+    TMTimerUnlock
+    TMR3GetWarpDrive
+    TMR3SetWarpDrive
+    TMR3TimeVirtGet
+    TMR3TimeVirtGetMicro
+    TMR3TimeVirtGetMilli
+    TMR3TimeVirtGetNano
+
+    VMMGetCpu
+
+    VMMGetSvnRev
+    VMSetError
+    VMSetErrorV
+    VMR3AtErrorDeregister
+    VMR3AtErrorRegister
+    VMR3AtRuntimeErrorRegister
+    VMR3AtStateRegister
+    VMR3Create
+    VMR3Destroy
+    VMR3GetCpuCoreAndPackageIdFromCpuId
+    VMR3GetStateName
+    VMR3GetStateU
+    VMR3GetSuspendReason
+    VMR3GetVM
+    VMR3HotPlugCpu
+    VMR3HotUnplugCpu
+    VMR3LoadFromFile
+    VMR3LoadFromStream
+    VMR3PowerOff
+    VMR3PowerOn
+    VMR3ReleaseUVM
+    VMR3ReqCallNoWaitU
+    VMR3ReqCallU
+    VMR3ReqCallVoidWaitU
+    VMR3ReqCallWaitU
+    VMR3ReqFree
+    VMR3ReqPriorityCallWaitU
+    VMR3ReqWait
+    VMR3Reset
+    VMR3Resume
+    VMR3RetainUVM
+    VMR3Save
+    VMR3SetCpuExecutionCap
+    VMR3SetError
+    VMR3SetPowerOffInsteadOfReset
+    VMR3Suspend
+    VMR3Teleport
+    VMR3AtStateDeregister
+    VMR3GetUVM
+
diff --git a/src/VBox/VMM/VMMR3/VMMTests.cpp b/src/VBox/VMM/VMMR3/VMMTests.cpp
new file mode 100644
index 00000000..edb88d01
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/VMMTests.cpp
@@ -0,0 +1,188 @@
+/* $Id: VMMTests.cpp $ */
+/** @file
+ * VMM - The Virtual Machine Monitor Core, Tests.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+//#define NO_SUPCALLR0VMM
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <iprt/asm-amd64-x86.h> /* for SUPGetCpuHzFromGIP */
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/dbg.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/selm.h>
+#include "VMMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <iprt/errcore.h>
+#include <VBox/param.h>
+
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/time.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+#include <iprt/x86.h>
+
+
+#define SYNC_SEL(pHyperCtx, reg)                                                        \
+        if (pHyperCtx->reg.Sel)                                                         \
+        {                                                                               \
+            DBGFSELINFO selInfo;                                                        \
+            int rc2 = SELMR3GetShadowSelectorInfo(pVM, pHyperCtx->reg.Sel, &selInfo);   \
+            AssertRC(rc2);                                                              \
+                                                                                        \
+            pHyperCtx->reg.u64Base              = selInfo.GCPtrBase;                    \
+            pHyperCtx->reg.u32Limit             = selInfo.cbLimit;                      \
+            pHyperCtx->reg.Attr.n.u1Present     = selInfo.u.Raw.Gen.u1Present;          \
+            pHyperCtx->reg.Attr.n.u1DefBig      = selInfo.u.Raw.Gen.u1DefBig;           \
+            pHyperCtx->reg.Attr.n.u1Granularity = selInfo.u.Raw.Gen.u1Granularity;      \
+            pHyperCtx->reg.Attr.n.u4Type        = selInfo.u.Raw.Gen.u4Type;             \
+            pHyperCtx->reg.Attr.n.u2Dpl         = selInfo.u.Raw.Gen.u2Dpl;              \
+            pHyperCtx->reg.Attr.n.u1DescType    = selInfo.u.Raw.Gen.u1DescType;         \
+            pHyperCtx->reg.Attr.n.u1Long        = selInfo.u.Raw.Gen.u1Long;             \
+        }
+
+/* execute the switch. */
+VMMR3DECL(int) VMMDoHmTest(PVM pVM)
+{
+#if 1
+    RTPrintf("FIXME!\n");
+    RT_NOREF(pVM);
+    return 0;
+#else
+
+    uint32_t i;
+    int      rc;
+    PCPUMCTX pHyperCtx, pGuestCtx;
+    RTGCPHYS CR3Phys = 0x0; /* fake address */
+    PVMCPU   pVCpu = &pVM->aCpus[0];
+
+    if (!HMIsEnabled(pVM))
+    {
+        RTPrintf("VMM: Hardware accelerated test not available!\n");
+        return VERR_ACCESS_DENIED;
+    }
+
+    /* Enable mapping of the hypervisor into the shadow page table. */
+    uint32_t cb;
+    rc = PGMR3MappingsSize(pVM, &cb);
+    AssertRCReturn(rc, rc);
+
+    /* Pretend the mappings are now fixed; to force a refresh of the reserved PDEs. */
+    rc = PGMR3MappingsFix(pVM, MM_HYPER_AREA_ADDRESS, cb);
+    AssertRCReturn(rc, rc);
+
+    pHyperCtx = CPUMGetHyperCtxPtr(pVCpu);
+
+    pHyperCtx->cr0 = X86_CR0_PE | X86_CR0_WP | X86_CR0_PG | X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
+    pHyperCtx->cr4 = X86_CR4_PGE | X86_CR4_OSFXSR | X86_CR4_OSXMMEEXCPT;
+    PGMChangeMode(pVCpu, pHyperCtx->cr0, pHyperCtx->cr4, pHyperCtx->msrEFER);
+    PGMSyncCR3(pVCpu, pHyperCtx->cr0, CR3Phys, pHyperCtx->cr4, true);
+
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TIMER);
+    VM_FF_CLEAR(pVM, VM_FF_TM_VIRTUAL_SYNC);
+    VM_FF_CLEAR(pVM, VM_FF_REQUEST);
+
+    /*
+     * Setup stack for calling VMMRCEntry().
+     */
+    RTRCPTR RCPtrEP;
+    rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "VMMRCEntry", &RCPtrEP);
+    if (RT_SUCCESS(rc))
+    {
+        RTPrintf("VMM: VMMRCEntry=%RRv\n", RCPtrEP);
+
+        pHyperCtx = CPUMGetHyperCtxPtr(pVCpu);
+
+        /* Fill in hidden selector registers for the hypervisor state. */
+        SYNC_SEL(pHyperCtx, cs);
+        SYNC_SEL(pHyperCtx, ds);
+        SYNC_SEL(pHyperCtx, es);
+        SYNC_SEL(pHyperCtx, fs);
+        SYNC_SEL(pHyperCtx, gs);
+        SYNC_SEL(pHyperCtx, ss);
+        SYNC_SEL(pHyperCtx, tr);
+
+        /*
+         * Profile switching.
+         */
+        RTPrintf("VMM: profiling switcher...\n");
+        Log(("VMM: profiling switcher...\n"));
+        uint64_t TickMin = UINT64_MAX;
+        uint64_t tsBegin = RTTimeNanoTS();
+        uint64_t TickStart = ASMReadTSC();
+        for (i = 0; i < 1000000; i++)
+        {
+            CPUMSetHyperState(pVCpu, pVM->vmm.s.pfnCallTrampolineRC, pVCpu->vmm.s.pbEMTStackBottomRC, 0, 0);
+            CPUMPushHyper(pVCpu, 0);
+            CPUMPushHyper(pVCpu, VMMRC_DO_TESTCASE_HM_NOP);
+            CPUMPushHyper(pVCpu, pVM->pVMRC);
+            CPUMPushHyper(pVCpu, 3 * sizeof(RTRCPTR));    /* stack frame size */
+            CPUMPushHyper(pVCpu, RCPtrEP);                /* what to call */
+
+            pHyperCtx = CPUMGetHyperCtxPtr(pVCpu);
+            pGuestCtx = CPUMQueryGuestCtxPtr(pVCpu);
+
+            /* Copy the hypervisor context to make sure we have a valid guest context. */
+            *pGuestCtx = *pHyperCtx;
+            pGuestCtx->cr3 = CR3Phys;
+
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TIMER);
+            VM_FF_CLEAR(pVM, VM_FF_TM_VIRTUAL_SYNC);
+
+            uint64_t TickThisStart = ASMReadTSC();
+            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_HM_RUN, 0);
+            uint64_t TickThisElapsed = ASMReadTSC() - TickThisStart;
+            if (RT_FAILURE(rc))
+            {
+                Log(("VMM: R0 returned fatal %Rrc in iteration %d\n", rc, i));
+                VMMR3FatalDump(pVM, pVCpu, rc);
+                return rc;
+            }
+            if (TickThisElapsed < TickMin)
+                TickMin = TickThisElapsed;
+        }
+        uint64_t TickEnd = ASMReadTSC();
+        uint64_t tsEnd = RTTimeNanoTS();
+
+        uint64_t Elapsed = tsEnd - tsBegin;
+        uint64_t PerIteration = Elapsed / (uint64_t)i;
+        uint64_t cTicksElapsed = TickEnd - TickStart;
+        uint64_t cTicksPerIteration = cTicksElapsed / (uint64_t)i;
+
+        RTPrintf("VMM: %8d cycles     in %11llu ns (%11lld ticks),  %10llu ns/iteration (%11lld ticks)  Min %11lld ticks\n",
+                 i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration, TickMin);
+        Log(("VMM: %8d cycles     in %11llu ns (%11lld ticks),  %10llu ns/iteration (%11lld ticks)  Min %11lld ticks\n",
+             i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration, TickMin));
+
+        rc = VINF_SUCCESS;
+    }
+    else
+        AssertMsgFailed(("Failed to resolved VMMRC.rc::VMMRCEntry(), rc=%Rrc\n", rc));
+
+    return rc;
+#endif
+}
+
diff --git a/src/VBox/VMM/VMMR3/VMReq.cpp b/src/VBox/VMM/VMMR3/VMReq.cpp
new file mode 100644
index 00000000..ed3a4726
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/VMReq.cpp
@@ -0,0 +1,1333 @@
+/* $Id: VMReq.cpp $ */
+/** @file
+ * VM - Virtual Machine
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VM
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vmm.h>
+#include "VMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <iprt/time.h>
+#include <iprt/semaphore.h>
+#include <iprt/thread.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int  vmR3ReqProcessOne(PVMREQ pReq);
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns an VBox status code,
+ * (2) that you want it's return code on success, and (3) that you wish to wait
+ * for ever for it to return.
+ *
+ * @returns VBox status code.  In the unlikely event that VMR3ReqCallVU fails,
+ *          its status code is return.  Otherwise, the status of pfnFunction is
+ *          returned.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ * @internal
+ */
+VMMR3_INT_DECL(int) VMR3ReqCallWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VBOX_STATUS,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    if (RT_SUCCESS(rc))
+        rc = pReq->iStatus;
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns an VBox status code,
+ * (2) that you want it's return code on success, and (3) that you wish to wait
+ * for ever for it to return.
+ *
+ * @returns VBox status code.  In the unlikely event that VMR3ReqCallVU fails,
+ *          its status code is return.  Otherwise, the status of pfnFunction is
+ *          returned.
+ *
+ * @param   pUVM            The user mode VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ * @internal
+ */
+VMMR3DECL(int) VMR3ReqCallWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VBOX_STATUS,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    if (RT_SUCCESS(rc))
+        rc = pReq->iStatus;
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns an VBox status code
+ * and that you do not wish to wait for it to complete.
+ *
+ * @returns VBox status code returned by VMR3ReqCallVU.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ * @internal
+ */
+VMMR3DECL(int) VMR3ReqCallNoWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, NULL, 0, VMREQFLAGS_VBOX_STATUS | VMREQFLAGS_NO_WAIT,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns an VBox status code
+ * and that you do not wish to wait for it to complete.
+ *
+ * @returns VBox status code returned by VMR3ReqCallVU.
+ *
+ * @param   pUVM            Pointer to the VM.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCallNoWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, idDstCpu, NULL, 0, VMREQFLAGS_VBOX_STATUS | VMREQFLAGS_NO_WAIT,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns void, and (2) that
+ * you wish to wait for ever for it to return.
+ *
+ * @returns VBox status code of VMR3ReqCallVU.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ * @internal
+ */
+VMMR3_INT_DECL(int) VMR3ReqCallVoidWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VOID,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns void, and (2) that
+ * you wish to wait for ever for it to return.
+ *
+ * @returns VBox status code of VMR3ReqCallVU.
+ *
+ * @param   pUVM            Pointer to the VM.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCallVoidWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VOID,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns void, and (2) that
+ * you do not wish to wait for it to complete.
+ *
+ * @returns VBox status code of VMR3ReqCallVU.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ * @internal
+ */
+VMMR3DECL(int) VMR3ReqCallVoidNoWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VOID | VMREQFLAGS_NO_WAIT,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns an VBox status code,
+ * (2) that you want it's return code on success, (3) that you wish to wait for
+ * ever for it to return, and (4) that it's priority request that can be safely
+ * be handled during async suspend and power off.
+ *
+ * @returns VBox status code.  In the unlikely event that VMR3ReqCallVU fails,
+ *          its status code is return.  Otherwise, the status of pfnFunction is
+ *          returned.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ * @internal
+ */
+VMMR3DECL(int) VMR3ReqPriorityCallWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VBOX_STATUS | VMREQFLAGS_PRIORITY,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    if (RT_SUCCESS(rc))
+        rc = pReq->iStatus;
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns an VBox status code,
+ * (2) that you want it's return code on success, (3) that you wish to wait for
+ * ever for it to return, and (4) that it's priority request that can be safely
+ * be handled during async suspend and power off.
+ *
+ * @returns VBox status code.  In the unlikely event that VMR3ReqCallVU fails,
+ *          its status code is return.  Otherwise, the status of pfnFunction is
+ *          returned.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqPriorityCallWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VBOX_STATUS | VMREQFLAGS_PRIORITY,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    if (RT_SUCCESS(rc))
+        rc = pReq->iStatus;
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns void, (2) that you
+ * wish to wait for ever for it to return, and (3) that it's priority request
+ * that can be safely be handled during async suspend and power off.
+ *
+ * @returns VBox status code of VMR3ReqCallVU.
+ *
+ * @param   pUVM            The user mode VM handle.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqPriorityCallVoidWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VOID | VMREQFLAGS_PRIORITY,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Allocate and queue a call request to a void function.
+ *
+ * If it's desired to poll on the completion of the request set cMillies
+ * to 0 and use VMR3ReqWait() to check for completion. In the other case
+ * use RT_INDEFINITE_WAIT.
+ * The returned request packet must be freed using VMR3ReqFree().
+ *
+ * @returns VBox status code.
+ *          Will not return VERR_INTERRUPTED.
+ * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   ppReq           Where to store the pointer to the request.
+ *                          This will be NULL or a valid request pointer not matter what happens, unless fFlags
+ *                          contains VMREQFLAGS_NO_WAIT when it will be optional and always NULL.
+ * @param   cMillies        Number of milliseconds to wait for the request to
+ *                          be completed. Use RT_INDEFINITE_WAIT to only
+ *                          wait till it's completed.
+ * @param   fFlags          A combination of the VMREQFLAGS values.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCallU(PUVM pUVM, VMCPUID idDstCpu, PVMREQ *ppReq, RTMSINTERVAL cMillies, uint32_t fFlags,
+                            PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, idDstCpu, ppReq, cMillies, fFlags, pfnFunction, cArgs, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Allocate and queue a call request.
+ *
+ * If it's desired to poll on the completion of the request set cMillies
+ * to 0 and use VMR3ReqWait() to check for completion. In the other case
+ * use RT_INDEFINITE_WAIT.
+ * The returned request packet must be freed using VMR3ReqFree().
+ *
+ * @returns VBox status code.
+ *          Will not return VERR_INTERRUPTED.
+ * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   ppReq           Where to store the pointer to the request.
+ *                          This will be NULL or a valid request pointer not matter what happens, unless fFlags
+ *                          contains VMREQFLAGS_NO_WAIT when it will be optional and always NULL.
+ * @param   cMillies        Number of milliseconds to wait for the request to
+ *                          be completed. Use RT_INDEFINITE_WAIT to only
+ *                          wait till it's completed.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   fFlags          A combination of the VMREQFLAGS values.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ *                          Stuff which differs in size from uintptr_t is gonna make trouble, so don't try!
+ * @param   Args            Argument vector.
+ *
+ * @remarks Caveats:
+ *              - Do not pass anything which is larger than an uintptr_t.
+ *              - 64-bit integers are larger than uintptr_t on 32-bit hosts.
+ *                Pass integers > 32-bit by reference (pointers).
+ *              - Don't use NULL since it should be the integer 0 in C++ and may
+ *                therefore end up with garbage in the bits 63:32 on 64-bit
+ *                hosts because 'int' is 32-bit.
+ *                Use (void *)NULL or (uintptr_t)0 instead of NULL.
+ */
+VMMR3DECL(int) VMR3ReqCallVU(PUVM pUVM, VMCPUID idDstCpu, PVMREQ *ppReq, RTMSINTERVAL cMillies, uint32_t fFlags,
+                             PFNRT pfnFunction, unsigned cArgs, va_list Args)
+{
+    LogFlow(("VMR3ReqCallV: idDstCpu=%u cMillies=%d fFlags=%#x pfnFunction=%p cArgs=%d\n", idDstCpu, cMillies, fFlags, pfnFunction, cArgs));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnFunction, VERR_INVALID_POINTER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(!(fFlags & ~(VMREQFLAGS_RETURN_MASK | VMREQFLAGS_NO_WAIT | VMREQFLAGS_POKE | VMREQFLAGS_PRIORITY)), VERR_INVALID_PARAMETER);
+    if (!(fFlags & VMREQFLAGS_NO_WAIT) || ppReq)
+    {
+        AssertPtrReturn(ppReq, VERR_INVALID_POINTER);
+        *ppReq = NULL;
+    }
+    PVMREQ pReq = NULL;
+    AssertMsgReturn(cArgs * sizeof(uintptr_t) <= sizeof(pReq->u.Internal.aArgs),
+                    ("cArg=%d\n", cArgs),
+                    VERR_TOO_MUCH_DATA);
+
+    /*
+     * Allocate request
+     */
+    int rc = VMR3ReqAlloc(pUVM, &pReq, VMREQTYPE_INTERNAL, idDstCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Initialize the request data.
+     */
+    pReq->fFlags         = fFlags;
+    pReq->u.Internal.pfn = pfnFunction;
+    pReq->u.Internal.cArgs = cArgs;
+    for (unsigned iArg = 0; iArg < cArgs; iArg++)
+        pReq->u.Internal.aArgs[iArg] = va_arg(Args, uintptr_t);
+
+    /*
+     * Queue the request and return.
+     */
+    rc = VMR3ReqQueue(pReq, cMillies);
+    if (    RT_FAILURE(rc)
+        && rc != VERR_TIMEOUT)
+    {
+        VMR3ReqFree(pReq);
+        pReq = NULL;
+    }
+    if (!(fFlags & VMREQFLAGS_NO_WAIT))
+    {
+        *ppReq = pReq;
+        LogFlow(("VMR3ReqCallV: returns %Rrc *ppReq=%p\n", rc, pReq));
+    }
+    else
+        LogFlow(("VMR3ReqCallV: returns %Rrc\n", rc));
+    Assert(rc != VERR_INTERRUPTED);
+    return rc;
+}
+
+
+/**
+ * Joins the list pList with whatever is linked up at *pHead.
+ */
+static void vmr3ReqJoinFreeSub(volatile PVMREQ *ppHead, PVMREQ pList)
+{
+    for (unsigned cIterations = 0;; cIterations++)
+    {
+        PVMREQ pHead = ASMAtomicXchgPtrT(ppHead, pList, PVMREQ);
+        if (!pHead)
+            return;
+        PVMREQ pTail = pHead;
+        while (pTail->pNext)
+            pTail = pTail->pNext;
+        ASMAtomicWritePtr(&pTail->pNext, pList);
+        ASMCompilerBarrier();
+        if (ASMAtomicCmpXchgPtr(ppHead, pHead, pList))
+            return;
+        ASMAtomicWriteNullPtr(&pTail->pNext);
+        ASMCompilerBarrier();
+        if (ASMAtomicCmpXchgPtr(ppHead, pHead, NULL))
+            return;
+        pList = pHead;
+        Assert(cIterations != 32);
+        Assert(cIterations != 64);
+    }
+}
+
+
+/**
+ * Joins the list pList with whatever is linked up at *pHead.
+ */
+static void vmr3ReqJoinFree(PVMINTUSERPERVM pVMInt, PVMREQ pList)
+{
+    /*
+     * Split the list if it's too long.
+     */
+    unsigned cReqs = 1;
+    PVMREQ pTail = pList;
+    while (pTail->pNext)
+    {
+        if (cReqs++ > 25)
+        {
+            const uint32_t i = pVMInt->iReqFree;
+            vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(i + 2) % RT_ELEMENTS(pVMInt->apReqFree)], pTail->pNext);
+
+            pTail->pNext = NULL;
+            vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(i + 2 + (i == pVMInt->iReqFree)) % RT_ELEMENTS(pVMInt->apReqFree)], pTail->pNext);
+            return;
+        }
+        pTail = pTail->pNext;
+    }
+    vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(pVMInt->iReqFree + 2) % RT_ELEMENTS(pVMInt->apReqFree)], pList);
+}
+
+
+/**
+ * Allocates a request packet.
+ *
+ * The caller allocates a request packet, fills in the request data
+ * union and queues the request.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   ppReq           Where to store the pointer to the allocated packet.
+ * @param   enmType         Package type.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ */
+VMMR3DECL(int) VMR3ReqAlloc(PUVM pUVM, PVMREQ *ppReq, VMREQTYPE enmType, VMCPUID idDstCpu)
+{
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(enmType > VMREQTYPE_INVALID && enmType < VMREQTYPE_MAX,
+                    ("Invalid package type %d valid range %d-%d inclusively.\n",
+                     enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
+                    VERR_VM_REQUEST_INVALID_TYPE);
+    AssertPtrReturn(ppReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(    idDstCpu == VMCPUID_ANY
+                    ||  idDstCpu == VMCPUID_ANY_QUEUE
+                    ||  idDstCpu < pUVM->cCpus
+                    ||  idDstCpu == VMCPUID_ALL
+                    ||  idDstCpu == VMCPUID_ALL_REVERSE,
+                    ("Invalid destination %u (max=%u)\n", idDstCpu, pUVM->cCpus), VERR_INVALID_PARAMETER);
+
+    /*
+     * Try get a recycled packet.
+     * While this could all be solved with a single list with a lock, it's a sport
+     * of mine to avoid locks.
+     */
+    int cTries = RT_ELEMENTS(pUVM->vm.s.apReqFree) * 2;
+    while (--cTries >= 0)
+    {
+        PVMREQ volatile *ppHead = &pUVM->vm.s.apReqFree[ASMAtomicIncU32(&pUVM->vm.s.iReqFree) % RT_ELEMENTS(pUVM->vm.s.apReqFree)];
+#if 0 /* sad, but this won't work safely because the reading of pReq->pNext. */
+        PVMREQ pNext = NULL;
+        PVMREQ pReq = *ppHead;
+        if (    pReq
+            &&  !ASMAtomicCmpXchgPtr(ppHead, (pNext = pReq->pNext), pReq)
+            &&  (pReq = *ppHead)
+            &&  !ASMAtomicCmpXchgPtr(ppHead, (pNext = pReq->pNext), pReq))
+            pReq = NULL;
+        if (pReq)
+        {
+            Assert(pReq->pNext == pNext); NOREF(pReq);
+#else
+        PVMREQ pReq = ASMAtomicXchgPtrT(ppHead, NULL, PVMREQ);
+        if (pReq)
+        {
+            PVMREQ pNext = pReq->pNext;
+            if (    pNext
+                &&  !ASMAtomicCmpXchgPtr(ppHead, pNext, NULL))
+            {
+                STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocRaces);
+                vmr3ReqJoinFree(&pUVM->vm.s, pReq->pNext);
+            }
+#endif
+            ASMAtomicDecU32(&pUVM->vm.s.cReqFree);
+
+            /*
+             * Make sure the event sem is not signaled.
+             */
+            if (!pReq->fEventSemClear)
+            {
+                int rc = RTSemEventWait(pReq->EventSem, 0);
+                if (rc != VINF_SUCCESS && rc != VERR_TIMEOUT)
+                {
+                    /*
+                     * This shall not happen, but if it does we'll just destroy
+                     * the semaphore and create a new one.
+                     */
+                    AssertMsgFailed(("rc=%Rrc from RTSemEventWait(%#x).\n", rc, pReq->EventSem));
+                    RTSemEventDestroy(pReq->EventSem);
+                    rc = RTSemEventCreate(&pReq->EventSem);
+                    AssertRC(rc);
+                    if (RT_FAILURE(rc))
+                        return rc;
+#if 0 /// @todo @bugref{4725} - def RT_LOCK_STRICT
+                    for (VMCPUID idCpu = 0; idCpu < pUVM->cCpus; idCpu++)
+                        RTSemEventAddSignaller(pReq->EventSem, pUVM->aCpus[idCpu].vm.s.ThreadEMT);
+#endif
+                }
+                pReq->fEventSemClear = true;
+            }
+            else
+                Assert(RTSemEventWait(pReq->EventSem, 0) == VERR_TIMEOUT);
+
+            /*
+             * Initialize the packet and return it.
+             */
+            Assert(pReq->enmType == VMREQTYPE_INVALID);
+            Assert(pReq->enmState == VMREQSTATE_FREE);
+            Assert(pReq->pUVM == pUVM);
+            ASMAtomicWriteNullPtr(&pReq->pNext);
+            pReq->enmState = VMREQSTATE_ALLOCATED;
+            pReq->iStatus  = VERR_VM_REQUEST_STATUS_STILL_PENDING;
+            pReq->fFlags   = VMREQFLAGS_VBOX_STATUS;
+            pReq->enmType  = enmType;
+            pReq->idDstCpu = idDstCpu;
+
+            *ppReq = pReq;
+            STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocRecycled);
+            LogFlow(("VMR3ReqAlloc: returns VINF_SUCCESS *ppReq=%p recycled\n", pReq));
+            return VINF_SUCCESS;
+        }
+    }
+
+    /*
+     * Ok allocate one.
+     */
+    PVMREQ pReq = (PVMREQ)MMR3HeapAllocU(pUVM, MM_TAG_VM_REQ, sizeof(*pReq));
+    if (!pReq)
+        return VERR_NO_MEMORY;
+
+    /*
+     * Create the semaphore.
+     */
+    int rc = RTSemEventCreate(&pReq->EventSem);
+    AssertRC(rc);
+    if (RT_FAILURE(rc))
+    {
+        MMR3HeapFree(pReq);
+        return rc;
+    }
+#if 0 /// @todo @bugref{4725} - def RT_LOCK_STRICT
+    for (VMCPUID idCpu = 0; idCpu < pUVM->cCpus; idCpu++)
+        RTSemEventAddSignaller(pReq->EventSem, pUVM->aCpus[idCpu].vm.s.ThreadEMT);
+#endif
+
+    /*
+     * Initialize the packet and return it.
+     */
+    pReq->pNext    = NULL;
+    pReq->pUVM     = pUVM;
+    pReq->enmState = VMREQSTATE_ALLOCATED;
+    pReq->iStatus  = VERR_VM_REQUEST_STATUS_STILL_PENDING;
+    pReq->fEventSemClear = true;
+    pReq->fFlags   = VMREQFLAGS_VBOX_STATUS;
+    pReq->enmType  = enmType;
+    pReq->idDstCpu = idDstCpu;
+
+    *ppReq = pReq;
+    STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocNew);
+    LogFlow(("VMR3ReqAlloc: returns VINF_SUCCESS *ppReq=%p new\n", pReq));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Free a request packet.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pReq            Package to free.
+ * @remark  The request packet must be in allocated or completed state!
+ */
+VMMR3DECL(int) VMR3ReqFree(PVMREQ pReq)
+{
+    /*
+     * Ignore NULL (all free functions should do this imho).
+     */
+    if (!pReq)
+        return VINF_SUCCESS;
+
+    /*
+     * Check packet state.
+     */
+    switch (pReq->enmState)
+    {
+        case VMREQSTATE_ALLOCATED:
+        case VMREQSTATE_COMPLETED:
+            break;
+        default:
+            AssertMsgFailed(("Invalid state %d!\n", pReq->enmState));
+            return VERR_VM_REQUEST_STATE;
+    }
+
+    /*
+     * Make it a free packet and put it into one of the free packet lists.
+     */
+    pReq->enmState = VMREQSTATE_FREE;
+    pReq->iStatus  = VERR_VM_REQUEST_STATUS_FREED;
+    pReq->enmType  = VMREQTYPE_INVALID;
+
+    PUVM pUVM = pReq->pUVM;
+    STAM_COUNTER_INC(&pUVM->vm.s.StatReqFree);
+
+    if (pUVM->vm.s.cReqFree < 128)
+    {
+        ASMAtomicIncU32(&pUVM->vm.s.cReqFree);
+        PVMREQ volatile *ppHead = &pUVM->vm.s.apReqFree[ASMAtomicIncU32(&pUVM->vm.s.iReqFree) % RT_ELEMENTS(pUVM->vm.s.apReqFree)];
+        PVMREQ pNext;
+        do
+        {
+            pNext = ASMAtomicUoReadPtrT(ppHead, PVMREQ);
+            ASMAtomicWritePtr(&pReq->pNext, pNext);
+            ASMCompilerBarrier();
+        } while (!ASMAtomicCmpXchgPtr(ppHead, pReq, pNext));
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pReq->pUVM->vm.s.StatReqFreeOverflow);
+        RTSemEventDestroy(pReq->EventSem);
+        MMR3HeapFree(pReq);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Queue a request.
+ *
+ * The quest must be allocated using VMR3ReqAlloc() and contain
+ * all the required data.
+ * If it's desired to poll on the completion of the request set cMillies
+ * to 0 and use VMR3ReqWait() to check for completion. In the other case
+ * use RT_INDEFINITE_WAIT.
+ *
+ * @returns VBox status code.
+ *          Will not return VERR_INTERRUPTED.
+ * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
+ *
+ * @param   pReq            The request to queue.
+ * @param   cMillies        Number of milliseconds to wait for the request to
+ *                          be completed. Use RT_INDEFINITE_WAIT to only
+ *                          wait till it's completed.
+ */
+VMMR3DECL(int) VMR3ReqQueue(PVMREQ pReq, RTMSINTERVAL cMillies)
+{
+    LogFlow(("VMR3ReqQueue: pReq=%p cMillies=%d\n", pReq, cMillies));
+    /*
+     * Verify the supplied package.
+     */
+    AssertMsgReturn(pReq->enmState == VMREQSTATE_ALLOCATED, ("%d\n", pReq->enmState), VERR_VM_REQUEST_STATE);
+    AssertMsgReturn(    VALID_PTR(pReq->pUVM)
+                    &&  !pReq->pNext
+                    &&  pReq->EventSem != NIL_RTSEMEVENT,
+                    ("Invalid request package! Anyone cooking their own packages???\n"),
+                    VERR_VM_REQUEST_INVALID_PACKAGE);
+    AssertMsgReturn(    pReq->enmType > VMREQTYPE_INVALID
+                    &&  pReq->enmType < VMREQTYPE_MAX,
+                    ("Invalid package type %d valid range %d-%d inclusively. This was verified on alloc too...\n",
+                     pReq->enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
+                    VERR_VM_REQUEST_INVALID_TYPE);
+    Assert(!(pReq->fFlags & ~(VMREQFLAGS_RETURN_MASK | VMREQFLAGS_NO_WAIT | VMREQFLAGS_POKE | VMREQFLAGS_PRIORITY)));
+
+    /*
+     * Are we the EMT or not?
+     * Also, store pVM (and fFlags) locally since pReq may be invalid after queuing it.
+     */
+    int     rc      = VINF_SUCCESS;
+    PUVM    pUVM    = ((VMREQ volatile *)pReq)->pUVM;                 /* volatile paranoia */
+    PUVMCPU pUVCpu  = (PUVMCPU)RTTlsGet(pUVM->vm.s.idxTLS);
+
+    if (pReq->idDstCpu == VMCPUID_ALL)
+    {
+        /* One-by-one. */
+        Assert(!(pReq->fFlags & VMREQFLAGS_NO_WAIT));
+        for (unsigned i = 0; i < pUVM->cCpus; i++)
+        {
+            /* Reinit some members. */
+            pReq->enmState = VMREQSTATE_ALLOCATED;
+            pReq->idDstCpu = i;
+            rc = VMR3ReqQueue(pReq, cMillies);
+            if (RT_FAILURE(rc))
+                break;
+        }
+    }
+    else if (pReq->idDstCpu == VMCPUID_ALL_REVERSE)
+    {
+        /* One-by-one. */
+        Assert(!(pReq->fFlags & VMREQFLAGS_NO_WAIT));
+        for (int i = pUVM->cCpus-1; i >= 0; i--)
+        {
+            /* Reinit some members. */
+            pReq->enmState = VMREQSTATE_ALLOCATED;
+            pReq->idDstCpu = i;
+            rc = VMR3ReqQueue(pReq, cMillies);
+            if (RT_FAILURE(rc))
+                break;
+        }
+    }
+    else if (   pReq->idDstCpu != VMCPUID_ANY   /* for a specific VMCPU? */
+             && pReq->idDstCpu != VMCPUID_ANY_QUEUE
+             && (   !pUVCpu                     /* and it's not the current thread. */
+                 || pUVCpu->idCpu != pReq->idDstCpu))
+    {
+        VMCPUID  idTarget = pReq->idDstCpu;     Assert(idTarget < pUVM->cCpus);
+        PVMCPU   pVCpu = pUVM->pVM->apCpusR3[idTarget];
+        unsigned fFlags = ((VMREQ volatile *)pReq)->fFlags;     /* volatile paranoia */
+
+        /* Fetch the right UVMCPU */
+        pUVCpu = &pUVM->aCpus[idTarget];
+
+        /*
+         * Insert it.
+         */
+        volatile PVMREQ *ppQueueHead = pReq->fFlags & VMREQFLAGS_PRIORITY ? &pUVCpu->vm.s.pPriorityReqs : &pUVCpu->vm.s.pNormalReqs;
+        pReq->enmState = VMREQSTATE_QUEUED;
+        PVMREQ pNext;
+        do
+        {
+            pNext = ASMAtomicUoReadPtrT(ppQueueHead, PVMREQ);
+            ASMAtomicWritePtr(&pReq->pNext, pNext);
+            ASMCompilerBarrier();
+        } while (!ASMAtomicCmpXchgPtr(ppQueueHead, pReq, pNext));
+
+        /*
+         * Notify EMT.
+         */
+        if (pUVM->pVM)
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_REQUEST);
+        VMR3NotifyCpuFFU(pUVCpu, fFlags & VMREQFLAGS_POKE ? VMNOTIFYFF_FLAGS_POKE : 0);
+
+        /*
+         * Wait and return.
+         */
+        if (!(fFlags & VMREQFLAGS_NO_WAIT))
+            rc = VMR3ReqWait(pReq, cMillies);
+        LogFlow(("VMR3ReqQueue: returns %Rrc\n", rc));
+    }
+    else if (   (    pReq->idDstCpu == VMCPUID_ANY
+                 && !pUVCpu /* only EMT threads have a valid pointer stored in the TLS slot. */)
+             || pReq->idDstCpu == VMCPUID_ANY_QUEUE)
+    {
+        unsigned fFlags = ((VMREQ volatile *)pReq)->fFlags;     /* volatile paranoia */
+
+        /* Note: pUVCpu may or may not be NULL in the VMCPUID_ANY_QUEUE case; we don't care. */
+
+        /*
+         * Insert it.
+         */
+        volatile PVMREQ *ppQueueHead = pReq->fFlags & VMREQFLAGS_PRIORITY ? &pUVM->vm.s.pPriorityReqs : &pUVM->vm.s.pNormalReqs;
+        pReq->enmState = VMREQSTATE_QUEUED;
+        PVMREQ pNext;
+        do
+        {
+            pNext = ASMAtomicUoReadPtrT(ppQueueHead, PVMREQ);
+            ASMAtomicWritePtr(&pReq->pNext, pNext);
+            ASMCompilerBarrier();
+        } while (!ASMAtomicCmpXchgPtr(ppQueueHead, pReq, pNext));
+
+        /*
+         * Notify EMT.
+         */
+        if (pUVM->pVM)
+            VM_FF_SET(pUVM->pVM, VM_FF_REQUEST);
+        VMR3NotifyGlobalFFU(pUVM, fFlags & VMREQFLAGS_POKE ? VMNOTIFYFF_FLAGS_POKE : 0);
+
+        /*
+         * Wait and return.
+         */
+        if (!(fFlags & VMREQFLAGS_NO_WAIT))
+            rc = VMR3ReqWait(pReq, cMillies);
+        LogFlow(("VMR3ReqQueue: returns %Rrc\n", rc));
+    }
+    else
+    {
+        Assert(pUVCpu);
+
+        /*
+         * The requester was an EMT, just execute it.
+         */
+        pReq->enmState = VMREQSTATE_QUEUED;
+        rc = vmR3ReqProcessOne(pReq);
+        LogFlow(("VMR3ReqQueue: returns %Rrc (processed)\n", rc));
+    }
+    return rc;
+}
+
+
+/**
+ * Wait for a request to be completed.
+ *
+ * @returns VBox status code.
+ * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
+ *
+ * @param   pReq            The request to wait for.
+ * @param   cMillies        Number of milliseconds to wait.
+ *                          Use RT_INDEFINITE_WAIT to only wait till it's completed.
+ */
+VMMR3DECL(int) VMR3ReqWait(PVMREQ pReq, RTMSINTERVAL cMillies)
+{
+    LogFlow(("VMR3ReqWait: pReq=%p cMillies=%d\n", pReq, cMillies));
+
+    /*
+     * Verify the supplied package.
+     */
+    AssertMsgReturn(    pReq->enmState == VMREQSTATE_QUEUED
+                    ||  pReq->enmState == VMREQSTATE_PROCESSING
+                    ||  pReq->enmState == VMREQSTATE_COMPLETED,
+                    ("Invalid state %d\n", pReq->enmState),
+                    VERR_VM_REQUEST_STATE);
+    AssertMsgReturn(    VALID_PTR(pReq->pUVM)
+                    &&  pReq->EventSem != NIL_RTSEMEVENT,
+                    ("Invalid request package! Anyone cooking their own packages???\n"),
+                    VERR_VM_REQUEST_INVALID_PACKAGE);
+    AssertMsgReturn(    pReq->enmType > VMREQTYPE_INVALID
+                    &&  pReq->enmType < VMREQTYPE_MAX,
+                    ("Invalid package type %d valid range %d-%d inclusively. This was verified on alloc too...\n",
+                     pReq->enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
+                    VERR_VM_REQUEST_INVALID_TYPE);
+
+    /*
+     * Check for deadlock condition
+     */
+    PUVM pUVM = pReq->pUVM;
+    NOREF(pUVM);
+
+    /*
+     * Wait on the package.
+     */
+    int rc;
+    if (cMillies != RT_INDEFINITE_WAIT)
+        rc = RTSemEventWait(pReq->EventSem, cMillies);
+    else
+    {
+        do
+        {
+            rc = RTSemEventWait(pReq->EventSem, RT_INDEFINITE_WAIT);
+            Assert(rc != VERR_TIMEOUT);
+        } while (   pReq->enmState != VMREQSTATE_COMPLETED
+                 && pReq->enmState != VMREQSTATE_INVALID);
+    }
+    if (RT_SUCCESS(rc))
+        ASMAtomicXchgSize(&pReq->fEventSemClear, true);
+    if (pReq->enmState == VMREQSTATE_COMPLETED)
+        rc = VINF_SUCCESS;
+    LogFlow(("VMR3ReqWait: returns %Rrc\n", rc));
+    Assert(rc != VERR_INTERRUPTED);
+    return rc;
+}
+
+
+/**
+ * Sets the relevant FF.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   idDstCpu        VMCPUID_ANY or the ID of the current CPU.
+ */
+DECLINLINE(void) vmR3ReqSetFF(PUVM pUVM, VMCPUID idDstCpu)
+{
+    if (RT_LIKELY(pUVM->pVM))
+    {
+        if (idDstCpu == VMCPUID_ANY)
+            VM_FF_SET(pUVM->pVM, VM_FF_REQUEST);
+        else
+            VMCPU_FF_SET(pUVM->pVM->apCpusR3[idDstCpu], VMCPU_FF_REQUEST);
+    }
+}
+
+
+/**
+ * VMR3ReqProcessU helper that handles cases where there are more than one
+ * pending request.
+ *
+ * @returns The oldest request.
+ * @param   pUVM                Pointer to the user mode VM structure
+ * @param   idDstCpu            VMCPUID_ANY or virtual CPU ID.
+ * @param   pReqList            The list of requests.
+ * @param   ppReqs              Pointer to the list head.
+ */
+static PVMREQ vmR3ReqProcessUTooManyHelper(PUVM pUVM, VMCPUID idDstCpu, PVMREQ pReqList, PVMREQ volatile *ppReqs)
+{
+    STAM_COUNTER_INC(&pUVM->vm.s.StatReqMoreThan1);
+
+    /*
+     * Chop off the last one (pReq).
+     */
+    PVMREQ pPrev;
+    PVMREQ pReqRet = pReqList;
+    do
+    {
+        pPrev = pReqRet;
+        pReqRet = pReqRet->pNext;
+    } while (pReqRet->pNext);
+    ASMAtomicWriteNullPtr(&pPrev->pNext);
+
+    /*
+     * Push the others back onto the list (end of it).
+     */
+    Log2(("VMR3ReqProcess: Pushing back %p %p...\n", pReqList, pReqList->pNext));
+    if (RT_UNLIKELY(!ASMAtomicCmpXchgPtr(ppReqs, pReqList, NULL)))
+    {
+        STAM_COUNTER_INC(&pUVM->vm.s.StatReqPushBackRaces);
+        do
+        {
+            ASMNopPause();
+            PVMREQ pReqList2 = ASMAtomicXchgPtrT(ppReqs, NULL, PVMREQ);
+            if (pReqList2)
+            {
+                PVMREQ pLast = pReqList2;
+                while (pLast->pNext)
+                    pLast = pLast->pNext;
+                ASMAtomicWritePtr(&pLast->pNext, pReqList);
+                pReqList = pReqList2;
+            }
+        } while (!ASMAtomicCmpXchgPtr(ppReqs, pReqList, NULL));
+    }
+
+    vmR3ReqSetFF(pUVM, idDstCpu);
+    return pReqRet;
+}
+
+
+/**
+ * Process pending request(s).
+ *
+ * This function is called from a forced action handler in the EMT
+ * or from one of the EMT loops.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   idDstCpu        Pass VMCPUID_ANY to process the common request queue
+ *                          and the CPU ID for a CPU specific one. In the latter
+ *                          case the calling thread must be the EMT of that CPU.
+ * @param   fPriorityOnly   When set, only process the priority request queue.
+ *
+ * @note    SMP safe (multiple EMTs trying to satisfy VM_FF_REQUESTs).
+ *
+ * @remarks This was made reentrant for async PDM handling, the debugger and
+ *          others.
+ * @internal
+ */
+VMMR3_INT_DECL(int) VMR3ReqProcessU(PUVM pUVM, VMCPUID idDstCpu, bool fPriorityOnly)
+{
+    LogFlow(("VMR3ReqProcessU: (enmVMState=%d) idDstCpu=%d\n", pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_CREATING, idDstCpu));
+
+    /*
+     * Determine which queues to process.
+     */
+    PVMREQ volatile *ppNormalReqs;
+    PVMREQ volatile *ppPriorityReqs;
+    if (idDstCpu == VMCPUID_ANY)
+    {
+        ppPriorityReqs = &pUVM->vm.s.pPriorityReqs;
+        ppNormalReqs   = !fPriorityOnly ? &pUVM->vm.s.pNormalReqs                 : ppPriorityReqs;
+    }
+    else
+    {
+        Assert(idDstCpu < pUVM->cCpus);
+        Assert(pUVM->aCpus[idDstCpu].vm.s.NativeThreadEMT == RTThreadNativeSelf());
+        ppPriorityReqs = &pUVM->aCpus[idDstCpu].vm.s.pPriorityReqs;
+        ppNormalReqs   = !fPriorityOnly ? &pUVM->aCpus[idDstCpu].vm.s.pNormalReqs : ppPriorityReqs;
+    }
+
+    /*
+     * Process loop.
+     *
+     * We do not repeat the outer loop if we've got an informational status code
+     * since that code needs processing by our caller (usually EM).
+     */
+    int rc = VINF_SUCCESS;
+    for (;;)
+    {
+        /*
+         * Get the pending requests.
+         *
+         * If there are more than one request, unlink the oldest and put the
+         * rest back so that we're reentrant.
+         */
+        if (RT_LIKELY(pUVM->pVM))
+        {
+            if (idDstCpu == VMCPUID_ANY)
+                VM_FF_CLEAR(pUVM->pVM, VM_FF_REQUEST);
+            else
+                VMCPU_FF_CLEAR(pUVM->pVM->apCpusR3[idDstCpu], VMCPU_FF_REQUEST);
+        }
+
+        PVMREQ pReq = ASMAtomicXchgPtrT(ppPriorityReqs, NULL, PVMREQ);
+        if (pReq)
+        {
+            if (RT_UNLIKELY(pReq->pNext))
+                pReq = vmR3ReqProcessUTooManyHelper(pUVM, idDstCpu, pReq, ppPriorityReqs);
+            else if (ASMAtomicReadPtrT(ppNormalReqs, PVMREQ))
+                vmR3ReqSetFF(pUVM, idDstCpu);
+        }
+        else
+        {
+            pReq = ASMAtomicXchgPtrT(ppNormalReqs, NULL, PVMREQ);
+            if (!pReq)
+                break;
+            if (RT_UNLIKELY(pReq->pNext))
+                pReq = vmR3ReqProcessUTooManyHelper(pUVM, idDstCpu, pReq, ppNormalReqs);
+        }
+
+        /*
+         * Process the request
+         */
+        STAM_COUNTER_INC(&pUVM->vm.s.StatReqProcessed);
+        int rc2 = vmR3ReqProcessOne(pReq);
+        if (    rc2 >= VINF_EM_FIRST
+            &&  rc2 <= VINF_EM_LAST)
+        {
+            rc = rc2;
+            break;
+        }
+    }
+
+    LogFlow(("VMR3ReqProcess: returns %Rrc (enmVMState=%d)\n", rc, pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_CREATING));
+    return rc;
+}
+
+
+/**
+ * Process one request.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pReq        Request packet to process.
+ */
+static int  vmR3ReqProcessOne(PVMREQ pReq)
+{
+    LogFlow(("vmR3ReqProcessOne: pReq=%p type=%d fFlags=%#x\n", pReq, pReq->enmType, pReq->fFlags));
+
+    /*
+     * Process the request.
+     */
+    Assert(pReq->enmState == VMREQSTATE_QUEUED);
+    pReq->enmState = VMREQSTATE_PROCESSING;
+    int     rcRet = VINF_SUCCESS;           /* the return code of this function. */
+    int     rcReq = VERR_NOT_IMPLEMENTED;   /* the request status. */
+    switch (pReq->enmType)
+    {
+        /*
+         * A packed down call frame.
+         */
+        case VMREQTYPE_INTERNAL:
+        {
+            uintptr_t *pauArgs = &pReq->u.Internal.aArgs[0];
+            union
+            {
+                PFNRT pfn;
+                DECLCALLBACKMEMBER(int, pfn00)(void);
+                DECLCALLBACKMEMBER(int, pfn01)(uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn02)(uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn03)(uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn04)(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn05)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn06)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn07)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn08)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn09)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn10)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn11)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn12)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn13)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn14)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn15)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+            } u;
+            u.pfn = pReq->u.Internal.pfn;
+#ifdef RT_ARCH_AMD64
+            switch (pReq->u.Internal.cArgs)
+            {
+                case 0:  rcRet = u.pfn00(); break;
+                case 1:  rcRet = u.pfn01(pauArgs[0]); break;
+                case 2:  rcRet = u.pfn02(pauArgs[0], pauArgs[1]); break;
+                case 3:  rcRet = u.pfn03(pauArgs[0], pauArgs[1], pauArgs[2]); break;
+                case 4:  rcRet = u.pfn04(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3]); break;
+                case 5:  rcRet = u.pfn05(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4]); break;
+                case 6:  rcRet = u.pfn06(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5]); break;
+                case 7:  rcRet = u.pfn07(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6]); break;
+                case 8:  rcRet = u.pfn08(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7]); break;
+                case 9:  rcRet = u.pfn09(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8]); break;
+                case 10: rcRet = u.pfn10(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9]); break;
+                case 11: rcRet = u.pfn11(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10]); break;
+                case 12: rcRet = u.pfn12(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11]); break;
+                case 13: rcRet = u.pfn13(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11], pauArgs[12]); break;
+                case 14: rcRet = u.pfn14(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11], pauArgs[12], pauArgs[13]); break;
+                case 15: rcRet = u.pfn15(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11], pauArgs[12], pauArgs[13], pauArgs[14]); break;
+                default:
+                    AssertReleaseMsgFailed(("cArgs=%d\n", pReq->u.Internal.cArgs));
+                    rcRet = rcReq = VERR_VM_REQUEST_TOO_MANY_ARGS_IPE;
+                    break;
+            }
+#else /* x86: */
+            size_t cbArgs = pReq->u.Internal.cArgs * sizeof(uintptr_t);
+# ifdef __GNUC__
+            __asm__ __volatile__("movl  %%esp, %%edx\n\t"
+                                 "subl  %2, %%esp\n\t"
+                                 "andl  $0xfffffff0, %%esp\n\t"
+                                 "shrl  $2, %2\n\t"
+                                 "movl  %%esp, %%edi\n\t"
+                                 "rep movsl\n\t"
+                                 "movl  %%edx, %%edi\n\t"
+                                 "call  *%%eax\n\t"
+                                 "mov   %%edi, %%esp\n\t"
+                                 : "=a" (rcRet),
+                                   "=S" (pauArgs),
+                                   "=c" (cbArgs)
+                                 : "0" (u.pfn),
+                                   "1" (pauArgs),
+                                   "2" (cbArgs)
+                                 : "edi", "edx");
+# else
+            __asm
+            {
+                xor     edx, edx        /* just mess it up. */
+                mov     eax, u.pfn
+                mov     ecx, cbArgs
+                shr     ecx, 2
+                mov     esi, pauArgs
+                mov     ebx, esp
+                sub     esp, cbArgs
+                and     esp, 0xfffffff0
+                mov     edi, esp
+                rep movsd
+                call    eax
+                mov     esp, ebx
+                mov     rcRet, eax
+            }
+# endif
+#endif /* x86 */
+            if ((pReq->fFlags & (VMREQFLAGS_RETURN_MASK)) == VMREQFLAGS_VOID)
+                rcRet = VINF_SUCCESS;
+            rcReq = rcRet;
+            break;
+        }
+
+        default:
+            AssertMsgFailed(("pReq->enmType=%d\n", pReq->enmType));
+            rcReq = VERR_NOT_IMPLEMENTED;
+            break;
+    }
+
+    /*
+     * Complete the request.
+     */
+    pReq->iStatus  = rcReq;
+    pReq->enmState = VMREQSTATE_COMPLETED;
+    if (pReq->fFlags & VMREQFLAGS_NO_WAIT)
+    {
+        /* Free the packet, nobody is waiting. */
+        LogFlow(("vmR3ReqProcessOne: Completed request %p: rcReq=%Rrc rcRet=%Rrc - freeing it\n",
+                 pReq, rcReq, rcRet));
+        VMR3ReqFree(pReq);
+    }
+    else
+    {
+        /* Notify the waiter and him free up the packet. */
+        LogFlow(("vmR3ReqProcessOne: Completed request %p: rcReq=%Rrc rcRet=%Rrc - notifying waiting thread\n",
+                 pReq, rcReq, rcRet));
+        ASMAtomicXchgSize(&pReq->fEventSemClear, false);
+        int rc2 = RTSemEventSignal(pReq->EventSem);
+        if (RT_FAILURE(rc2))
+        {
+            AssertRC(rc2);
+            rcRet = rc2;
+        }
+    }
+
+    return rcRet;
+}
+
diff --git a/src/VBox/VMM/VMMR3/cpus/AMD_Athlon_64_3200.h b/src/VBox/VMM/VMMR3/cpus/AMD_Athlon_64_3200.h
new file mode 100644
index 00000000..a99e1c4e
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/AMD_Athlon_64_3200.h
@@ -0,0 +1,224 @@
+/* $Id: AMD_Athlon_64_3200.h $ */
+/** @file
+ * CPU database entry "AMD Athlon 64 3200+".
+ * Generated at 2013-07-12T02:09:05Z by VBoxCpuReport v4.3.53r91376 on win.x86.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_AMD_Athlon_64_3200_h
+#define VBOX_CPUDB_AMD_Athlon_64_3200_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for AMD Athlon(tm) 64 Processor 3200+.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_AMD_Athlon_64_3200[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x68747541, 0x444d4163, 0x69746e65, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00000f48, 0x00000800, 0x00000000, 0x078bfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000018, 0x68747541, 0x444d4163, 0x69746e65, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000f48, 0x0000010a, 0x00000000, 0xe1d3fbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20444d41, 0x6c687441, 0x74286e6f, 0x3620296d, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x72502034, 0x7365636f, 0x20726f73, 0x30303233, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x0000002b, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0xff08ff08, 0xff20ff20, 0x40020140, 0x40020140, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x42004200, 0x04008140, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x0000000f, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003028, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000a, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000b, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000d, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000e, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000f, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000010, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000011, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000012, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000013, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000014, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000015, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000016, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000017, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000018, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8fffffff, 0x00000000, 0x00000000, 0x53275449, 0x4d414820, 0x2052454d, 0x454d4954, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for AMD Athlon(tm) 64 Processor 3200+.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_AMD_Athlon_64_3200[] =
+{
+    MAL(0x00000000, "IA32_P5_MC_ADDR", 0x00000402),
+    MAL(0x00000001, "IA32_P5_MC_TYPE", 0x00000401),
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x28`4505cb65 */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00900), 0, UINT64_C(0xffffff00000006ff)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFO(0x0000008b, "AMD_K8_PATCH_LEVEL", AmdK8PatchLevel), /* value=0x39 */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x508, 0, 0), /* value=0x508 */
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x8 */
+    MFX(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x8059e000 */
+    MFX(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x81872950 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x105, 0, 0), /* value=0x105 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, UINT64_C(0xfffffffffffffff8), 0), /* value=0x0 */
+    MFX(0x0000017b, "IA32_MCG_CTL", Ia32McgCtl, Ia32McgCtl, 0, UINT64_C(0xffffffffffffffe0), 0), /* value=0x1f */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, UINT64_C(0xffffffffffffff80), 0x40), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0xffffffed`bf1be178 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xffff7f49`bf1bedec */
+    MFO(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp), /* value=0x0 */
+    MFO(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp), /* value=0x0 */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xffffff00000007ff)), /* value=0xff`c0000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xffffff0000000ff8)), /* value=0xf8000001 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xffffff00000007ff)), /* value=0xff`fc000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    RFN(0x00000400, 0x00000413, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0x800, 0xfe, UINT64_C(0xfffffffffffff200)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x0 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0x0 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0x0 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x81913800 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0x0 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x0 */
+    RSN(0xc0010000, 0xc0010003, "AMD_K8_PERF_CTL_n", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x0, UINT64_C(0xffffffff00200000), 0),
+    RSN(0xc0010004, 0xc0010007, "AMD_K8_PERF_CTR_n", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x0, UINT64_C(0xffff000000000000), 0),
+    MFX(0xc0010010, "AMD_K8_SYS_CFG", AmdK8SysCfg, AmdK8SysCfg, 0x160601, UINT64_C(0xffffffffffc0f800), 0), /* value=0x160601 */
+    MFX(0xc0010015, "AMD_K8_HW_CFG", AmdK8HwCr, AmdK8HwCr, 0xc000000, UINT64_C(0xffffffff3ff00000), 0), /* value=0xc000000 */
+    MFW(0xc0010016, "AMD_K8_IORR_BASE_0", AmdK8IorrBaseN, AmdK8IorrBaseN, UINT64_C(0xffffff0000000fe7)), /* value=0x0 */
+    MFW(0xc0010017, "AMD_K8_IORR_MASK_0", AmdK8IorrMaskN, AmdK8IorrMaskN, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0xc0010018, "AMD_K8_IORR_BASE_1", AmdK8IorrBaseN, AmdK8IorrBaseN, 0x1, UINT64_C(0xffffff0000000fe7), 0), /* value=0xf8000018 */
+    MFX(0xc0010019, "AMD_K8_IORR_MASK_1", AmdK8IorrMaskN, AmdK8IorrMaskN, 0x1, UINT64_C(0xffffff00000007ff), 0), /* value=0xff`fc000800 */
+    MFW(0xc001001a, "AMD_K8_TOP_MEM", AmdK8TopOfMemN, AmdK8TopOfMemN, UINT64_C(0xffffff00007fffff)), /* value=0x40000000 */
+    MFX(0xc001001d, "AMD_K8_TOP_MEM2", AmdK8TopOfMemN, AmdK8TopOfMemN, 0x1, UINT64_C(0xffffff00007fffff), 0), /* value=0x0 */
+    MVI(0xc001001e, "AMD_K8_MANID", 0x20),
+    MFX(0xc001001f, "AMD_K8_NB_CFG1", AmdK8NbCfg1, AmdK8NbCfg1, 0, UINT64_C(0xffffff0000000000), 0), /* value=0x11`00000008 */
+    MFN(0xc0010020, "AMD_K8_PATCH_LOADER", WriteOnly, AmdK8PatchLoader),
+    MVX(0xc0010021, "AMD_K8_UNK_c001_0021", 0, UINT64_C(0xfffffffe00000000), 0),
+    MFX(0xc0010022, "AMD_K8_MC_XCPT_REDIR", AmdK8McXcptRedir, AmdK8McXcptRedir, 0, UINT64_C(0xfffffffeffffffff), 0), /* value=0x0 */
+    RFN(0xc0010030, 0xc0010035, "AMD_K8_CPU_NAME_n", AmdK8CpuNameN, AmdK8CpuNameN),
+    MFX(0xc001003e, "AMD_K8_HTC", AmdK8HwThermalCtrl, AmdK8HwThermalCtrl, 0, UINT64_MAX, 0), /* value=0x0 */
+    MFI(0xc001003f, "AMD_K8_STC", AmdK8SwThermalCtrl), /* value=0x0 */
+    MFX(0xc0010041, "AMD_K8_FIDVID_CTL", AmdK8FidVidControl, AmdK8FidVidControl, UINT64_C(0x4e200000000c), 0x33, UINT64_C(0xfff00000fffee0c0)), /* value=0x4e20`0000000c */
+    MFX(0xc0010042, "AMD_K8_FIDVID_STATUS", AmdK8FidVidStatus, ReadOnly, UINT64_C(0x200000c0c0c), 0, 0), /* value=0x200`000c0c0c */
+    MVO(0xc0010043, "AMD_K8_THERMTRIP_STATUS", 0x521020),
+    RFN(0xc0010044, 0xc0010048, "AMD_K8_MC_CTL_MASK_n", AmdK8McCtlMaskN, AmdK8McCtlMaskN),
+    RSN(0xc0010050, 0xc0010053, "AMD_K8_SMI_ON_IO_TRAP_n", AmdK8SmiOnIoTrapN, AmdK8SmiOnIoTrapN, 0x0, 0, UINT64_C(0x1f00000000000000)),
+    MFX(0xc0010054, "AMD_K8_SMI_ON_IO_TRAP_CTL_STS", AmdK8SmiOnIoTrapCtlSts, AmdK8SmiOnIoTrapCtlSts, 0, 0, UINT64_C(0xffffffffffff1f00)), /* value=0x0 */
+    MFX(0xc0010111, "AMD_K8_SMM_BASE", AmdK8SmmBase, AmdK8SmmBase, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x98000 */
+    MFX(0xc0010112, "AMD_K8_SMM_ADDR", AmdK8SmmAddr, AmdK8SmmAddr, 0, UINT64_C(0xffffff000001ffff), 0), /* value=0x0 */
+    MFX(0xc0010113, "AMD_K8_SMM_MASK", AmdK8SmmMask, AmdK8SmmMask, 0, UINT64_C(0xffffff00000188c0), 0), /* value=0x1 */
+    MVX(0xc0010114, "AMD_K8_UNK_c001_0114", 0, 0, UINT64_C(0xffffffffffffffe4)),
+    MVX(0xc0010115, "AMD_K8_UNK_c001_0115", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0xc0010116, "AMD_K8_UNK_c001_0116", 0, 0, UINT64_C(0xffff0000ffff0000)),
+    MVX(0xc0010117, "AMD_K8_UNK_c001_0117", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0xc0010118, "AMD_K8_UNK_c001_0118",0,0,0),
+    MVX(0xc0010119, "AMD_K8_UNK_c001_0119",0,0,0),
+    MVX(0xc001011a, "AMD_K8_UNK_c001_011a", 0, 0, UINT64_C(0xffffffff00000fff)),
+    MVX(0xc001011b, "AMD_K8_UNK_c001_011b", 0, 0, ~(uint64_t)UINT32_MAX),
+    MVX(0xc001011c, "AMD_K8_UNK_c001_011c", UINT32_C(0xdb1f5000), 0, UINT64_C(0xffffffff00000fff)),
+    MFX(0xc0011000, "AMD_K7_MCODE_CTL", AmdK7MicrocodeCtl, AmdK7MicrocodeCtl, 0, ~(uint64_t)UINT32_MAX, 0x204), /* value=0x0 */
+    MFX(0xc0011001, "AMD_K7_APIC_CLUSTER_ID", AmdK7ClusterIdMaybe, AmdK7ClusterIdMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0011004, "AMD_K8_CPUID_CTL_STD01", AmdK8CpuIdCtlStd01hEdcx, AmdK8CpuIdCtlStd01hEdcx, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x78bfbff */
+    MFX(0xc0011005, "AMD_K8_CPUID_CTL_EXT01", AmdK8CpuIdCtlExt01hEdcx, AmdK8CpuIdCtlExt01hEdcx, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0xf1f3fbff */
+    MFX(0xc0011006, "AMD_K7_DEBUG_STS?", AmdK7DebugStatusMaybe, AmdK7DebugStatusMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFN(0xc0011007, "AMD_K7_BH_TRACE_BASE?", AmdK7BHTraceBaseMaybe, AmdK7BHTraceBaseMaybe), /* value=0x0 */
+    MFN(0xc0011008, "AMD_K7_BH_TRACE_PTR?", AmdK7BHTracePtrMaybe, AmdK7BHTracePtrMaybe), /* value=0x0 */
+    MFN(0xc0011009, "AMD_K7_BH_TRACE_LIM?", AmdK7BHTraceLimitMaybe, AmdK7BHTraceLimitMaybe), /* value=0x0 */
+    MFX(0xc001100a, "AMD_K7_HDT_CFG?", AmdK7HardwareDebugToolCfgMaybe, AmdK7HardwareDebugToolCfgMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc001100b, "AMD_K7_FAST_FLUSH_COUNT?", AmdK7FastFlushCountMaybe, AmdK7FastFlushCountMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x7c0 */
+    MFX(0xc001100c, "AMD_K7_NODE_ID", AmdK7NodeId, AmdK7NodeId, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x20906 */
+    MVX(0xc001100d, "AMD_K8_LOGICAL_CPUS_NUM?", 0x10a, 0, 0),
+    MVX(0xc001100e, "AMD_K8_WRMSR_BP?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001100f, "AMD_K8_WRMSR_BP_MASK?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011010, "AMD_K8_BH_TRACE_CTL?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011011, "AMD_K8_BH_TRACE_USRD?", 0, 0, 0), /* value=0xc0011011`00000283 */
+    MVX(0xc0011014, "AMD_K8_XCPT_BP_RIP?", 0, 0, 0),
+    MVX(0xc0011015, "AMD_K8_XCPT_BP_RIP_MASK?", 0, 0, 0),
+    MVX(0xc0011016, "AMD_K8_COND_HDT_VAL?", 0, 0, 0),
+    MVX(0xc0011017, "AMD_K8_COND_HDT_VAL_MASK?", 0, 0, 0),
+    MVX(0xc0011018, "AMD_K8_XCPT_BP_CTL?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001101d, "AMD_K8_NB_BIST?", 0, UINT64_C(0xfffffffffc000000), 0),
+    MVI(0xc001101e, "AMD_K8_THERMTRIP_2?", 0x521020), /* Villain? */
+    MVX(0xc001101f, "AMD_K8_NB_CFG?", UINT64_C(0x1100000008), UINT64_C(0xffffff0000000000), 0),
+    MFX(0xc0011020, "AMD_K7_LS_CFG", AmdK7LoadStoreCfg, AmdK7LoadStoreCfg, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x1000 */
+    MFX(0xc0011021, "AMD_K7_IC_CFG", AmdK7InstrCacheCfg, AmdK7InstrCacheCfg, 0x800, ~(uint64_t)UINT32_MAX, 0), /* value=0x800 */
+    MFX(0xc0011022, "AMD_K7_DC_CFG", AmdK7DataCacheCfg, AmdK7DataCacheCfg, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x24000008 */
+    MFN(0xc0011023, "AMD_K7_BU_CFG", AmdK7BusUnitCfg, AmdK7BusUnitCfg), /* Villain? value=0x2020 */
+    MFX(0xc0011024, "AMD_K7_DEBUG_CTL_2?", AmdK7DebugCtl2Maybe, AmdK7DebugCtl2Maybe, 0, UINT64_C(0xffffffffffffff00), 0), /* value=0x0 */
+    MFN(0xc0011025, "AMD_K7_DR0_DATA_MATCH?", AmdK7Dr0DataMatchMaybe, AmdK7Dr0DataMatchMaybe), /* value=0x0 */
+    MFN(0xc0011026, "AMD_K7_DR0_DATA_MATCH?", AmdK7Dr0DataMaskMaybe, AmdK7Dr0DataMaskMaybe), /* value=0x0 */
+    MFX(0xc0011027, "AMD_K7_DR0_ADDR_MASK", AmdK7DrXAddrMaskN, AmdK7DrXAddrMaskN, 0x0, UINT64_C(0xfffffffffffff000), 0), /* value=0x0 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for AMD Athlon(tm) 64 Processor 3200+.
+ */
+static CPUMDBENTRY const g_Entry_AMD_Athlon_64_3200 =
+{
+    /*.pszName          = */ "AMD Athlon 64 3200+",
+    /*.pszFullName      = */ "AMD Athlon(tm) 64 Processor 3200+",
+    /*.enmVendor        = */ CPUMCPUVENDOR_AMD,
+    /*.uFamily          = */ 15,
+    /*.uModel           = */ 4,
+    /*.uStepping        = */ 8,
+    /*.enmMicroarch     = */ kCpumMicroarch_AMD_K8_130nm,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 40,
+    /*.fMxCsrMask       = */ 0xffff, ///< @todo check.
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_AMD_Athlon_64_3200),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_AMD_Athlon_64_3200)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_AMD_Athlon_64_3200)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_AMD_Athlon_64_3200),
+};
+
+#endif /* !VBOX_CPUDB_AMD_Athlon_64_3200_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/AMD_Athlon_64_X2_Dual_Core_4200.h b/src/VBox/VMM/VMMR3/cpus/AMD_Athlon_64_X2_Dual_Core_4200.h
new file mode 100644
index 00000000..4099bb39
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/AMD_Athlon_64_X2_Dual_Core_4200.h
@@ -0,0 +1,232 @@
+/* $Id: AMD_Athlon_64_X2_Dual_Core_4200.h $ */
+/** @file
+ * CPU database entry "AMD Athlon 64 X2 Dual Core 4200+".
+ * Generated at 2014-02-28T15:19:16Z by VBoxCpuReport v4.3.53r92578 on linux.amd64                                                        .
+ *                                                                                                                                        .
+ * @remarks Possible that we're missing a few special MSRs due to no                                                                      .
+ *          magic register value capabilities in the linux hosted                                                                         .
+ *          MSR probing code.
+ * @todo Regenerate this using windows/whatever where we can get to the secret AMD MSRs.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_AMD_Athlon_64_X2_Dual_Core_4200_h
+#define VBOX_CPUDB_AMD_Athlon_64_X2_Dual_Core_4200_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for AMD Athlon(tm) 64 X2 Dual Core Processor 4200+.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_AMD_Athlon_64_X2_Dual_Core_4200[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x68747541, 0x444d4163, 0x69746e65, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00040fb2, 0x01020800, 0x00002001, 0x178bfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000018, 0x68747541, 0x444d4163, 0x69746e65, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00040fb2, 0x000008d1, 0x0000001f, 0xebd3fbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20444d41, 0x6c687441, 0x74286e6f, 0x3620296d, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x32582034, 0x61754420, 0x6f43206c, 0x50206572, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x65636f72, 0x726f7373, 0x30323420, 0x00002b30, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0xff08ff08, 0xff20ff20, 0x40020140, 0x40020140, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x42004200, 0x02008140, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x0000003f, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003028, 0x00000000, 0x00000001, 0x00000000, 0 },
+    { 0x80000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000a, 0x00000000, 0x00000000, 0x00000001, 0x00000040, 0x00000000, 0x00000000, 0 },
+    { 0x8000000b, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000d, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000e, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000f, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000010, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000011, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000012, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000013, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000014, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000015, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000016, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000017, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000018, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for AMD Athlon(tm) 64 X2 Dual Core Processor 4200+.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_AMD_Athlon_64_X2_Dual_Core_4200[] =
+{
+    MAL(0x00000000, "IA32_P5_MC_ADDR", 0x00000402),
+    MAL(0x00000001, "IA32_P5_MC_TYPE", 0x00000401),
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x7e`171166b8 */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0, UINT64_C(0xffffff00000006ff)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFO(0x0000008b, "AMD_K8_PATCH_LEVEL", AmdK8PatchLevel), /* value=0x0 */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x508, 0, 0), /* value=0x508 */
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x10 */
+    MFX(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x8103ca80 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x105, 0, 0), /* value=0x105 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, UINT64_C(0xfffffffffffffff8), 0), /* value=0x0 */
+    MFX(0x0000017b, "IA32_MCG_CTL", Ia32McgCtl, Ia32McgCtl, 0, UINT64_C(0xffffffffffffffe0), 0), /* value=0x1f */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, UINT64_C(0xffffffffffffff80), 0x40), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0xffffffff`a0425995 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xffffffff`8103124a */
+    MFO(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp), /* value=0x0 */
+    MFO(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp), /* value=0x0 */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xffffff00000007ff)), /* value=0xff`80000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x80000006 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xffffff00000007ff)), /* value=0xff`c0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xffffff0000000ff8)), /* value=0xf8000001 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xffffff00000007ff)), /* value=0xff`ff000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    RFN(0x00000400, 0x00000413, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0xfe, UINT64_C(0xffffffffffff8200)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xffffffff`81011d20 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xffffffff`8103ccb0 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x3700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x1da4880 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffff8800`28300000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x0 */
+    MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x1 */
+    RSN(0xc0010000, 0xc0010003, "AMD_K8_PERF_CTL_n", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x0, UINT64_C(0xffffffff00200000), 0),
+    RSN(0xc0010004, 0xc0010007, "AMD_K8_PERF_CTR_n", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x0, UINT64_C(0xffff000000000000), 0),
+    MFX(0xc0010010, "AMD_K8_SYS_CFG", AmdK8SysCfg, AmdK8SysCfg, 0x760601, UINT64_C(0xffffffffff80f800), 0), /* value=0x760601 */
+    MFX(0xc0010015, "AMD_K8_HW_CFG", AmdK8HwCr, AmdK8HwCr, 0x2000060, UINT64_C(0xffffffff3ff00020), 0), /* value=0x2000060 */
+    MFW(0xc0010016, "AMD_K8_IORR_BASE_0", AmdK8IorrBaseN, AmdK8IorrBaseN, UINT64_C(0xffffff0000000fe7)), /* value=0xa30000 */
+    MFW(0xc0010017, "AMD_K8_IORR_MASK_0", AmdK8IorrMaskN, AmdK8IorrMaskN, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0xc0010018, "AMD_K8_IORR_BASE_1", AmdK8IorrBaseN, AmdK8IorrBaseN, 0x1, UINT64_C(0xffffff0000000fe7), 0), /* value=0x0 */
+    MFX(0xc0010019, "AMD_K8_IORR_MASK_1", AmdK8IorrMaskN, AmdK8IorrMaskN, 0x1, UINT64_C(0xffffff00000007ff), 0), /* value=0x0 */
+    MFW(0xc001001a, "AMD_K8_TOP_MEM", AmdK8TopOfMemN, AmdK8TopOfMemN, UINT64_C(0xffffff00007fffff)), /* value=0xc0000000 */
+    MFX(0xc001001d, "AMD_K8_TOP_MEM2", AmdK8TopOfMemN, AmdK8TopOfMemN, 0x1, UINT64_C(0xffffff00007fffff), 0), /* value=0x1`40000000 */
+    MVI(0xc001001e, "AMD_K8_MANID", 0x52),
+    MFX(0xc001001f, "AMD_K8_NB_CFG1", AmdK8NbCfg1, AmdK8NbCfg1, 0, UINT64_C(0x3fbf000000000000), 0), /* value=0x400001`00100008 */
+    MFN(0xc0010020, "AMD_K8_PATCH_LOADER", WriteOnly, AmdK8PatchLoader),
+    MFN(0xc0010021, "AMD_K8_UNK_c001_0021", WriteOnly, IgnoreWrite),
+    RFN(0xc0010030, 0xc0010035, "AMD_K8_CPU_NAME_n", AmdK8CpuNameN, AmdK8CpuNameN),
+    MFX(0xc001003e, "AMD_K8_HTC", AmdK8HwThermalCtrl, AmdK8HwThermalCtrl, 0, UINT64_C(0xfffffffff0e088fc), 0), /* value=0x0 */
+    MFX(0xc001003f, "AMD_K8_STC", AmdK8SwThermalCtrl, AmdK8SwThermalCtrl, 0, UINT64_C(0xfffffffff0e088e0), 0), /* value=0x0 */
+    MFX(0xc0010041, "AMD_K8_FIDVID_CTL", AmdK8FidVidControl, AmdK8FidVidControl, UINT64_C(0x100001202), 0xc31, UINT64_C(0xfff00000fffec0c0)), /* value=0x1`00001202 */
+    MFX(0xc0010042, "AMD_K8_FIDVID_STATUS", AmdK8FidVidStatus, ReadOnly, UINT64_C(0x310c12120c0e0202), 0, 0), /* value=0x310c1212`0c0e0202 */
+    MVO(0xc0010043, "AMD_K8_THERMTRIP_STATUS", 0x4e1a24),
+    RFN(0xc0010044, 0xc0010048, "AMD_K8_MC_CTL_MASK_n", AmdK8McCtlMaskN, AmdK8McCtlMaskN),
+    RSN(0xc0010050, 0xc0010053, "AMD_K8_SMI_ON_IO_TRAP_n", AmdK8SmiOnIoTrapN, AmdK8SmiOnIoTrapN, 0x0, 0, UINT64_C(0x1f00000000000000)),
+    MFX(0xc0010054, "AMD_K8_SMI_ON_IO_TRAP_CTL_STS", AmdK8SmiOnIoTrapCtlSts, AmdK8SmiOnIoTrapCtlSts, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xffff1f00)), /* value=0x0 */
+    MFX(0xc0010055, "AMD_K8_INT_PENDING_MSG", AmdK8IntPendingMessage, AmdK8IntPendingMessage, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xe0000000)), /* value=0x3000000 */
+    MVO(0xc0010060, "AMD_K8_BIST_RESULT", 0),
+    MFX(0xc0010111, "AMD_K8_SMM_BASE", AmdK8SmmBase, AmdK8SmmBase, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x98200 */
+    MFX(0xc0010112, "AMD_K8_SMM_ADDR", AmdK8SmmAddr, AmdK8SmmAddr, 0, UINT64_C(0xffffff000001ffff), 0), /* value=0x0 */
+    MFX(0xc0010113, "AMD_K8_SMM_MASK", AmdK8SmmMask, AmdK8SmmMask, 0, UINT64_C(0xffffff00000188c0), 0), /* value=0x1 */
+    MFX(0xc0010114, "AMD_K8_VM_CR", AmdK8VmCr, AmdK8VmCr, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xffffffe0)), /* value=0x0 */
+    MFX(0xc0010115, "AMD_K8_IGNNE", AmdK8IgnNe, AmdK8IgnNe, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xfffffffe)), /* value=0x0 */
+    MFN(0xc0010116, "AMD_K8_SMM_CTL", WriteOnly, AmdK8SmmCtl),
+    MFX(0xc0010117, "AMD_K8_VM_HSAVE_PA", AmdK8VmHSavePa, AmdK8VmHSavePa, 0, 0, UINT64_C(0xffffff0000000fff)), /* value=0x0 */
+
+    /* Copy & paste from the AMD_Athlon_64_3200 (130nm) profile: */
+    MVX(0xc0010118, "AMD_K8_UNK_c001_0118",0,0,0),
+    MVX(0xc0010119, "AMD_K8_UNK_c001_0119",0,0,0),
+    MVX(0xc001011a, "AMD_K8_UNK_c001_011a", 0, 0, UINT64_C(0xffffffff00000fff)),
+    MVX(0xc001011b, "AMD_K8_UNK_c001_011b", 0, 0, ~(uint64_t)UINT32_MAX),
+    MVX(0xc001011c, "AMD_K8_UNK_c001_011c", UINT32_C(0xdb1f5000), 0, UINT64_C(0xffffffff00000fff)),
+    MFX(0xc0011000, "AMD_K7_MCODE_CTL", AmdK7MicrocodeCtl, AmdK7MicrocodeCtl, 0, ~(uint64_t)UINT32_MAX, 0x204), /* value=0x0 */
+    MFX(0xc0011001, "AMD_K7_APIC_CLUSTER_ID", AmdK7ClusterIdMaybe, AmdK7ClusterIdMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0011004, "AMD_K8_CPUID_CTL_STD01", AmdK8CpuIdCtlStd01hEdcx, AmdK8CpuIdCtlStd01hEdcx, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x78bfbff */
+    MFX(0xc0011005, "AMD_K8_CPUID_CTL_EXT01", AmdK8CpuIdCtlExt01hEdcx, AmdK8CpuIdCtlExt01hEdcx, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0xf1f3fbff */
+    MFX(0xc0011006, "AMD_K7_DEBUG_STS?", AmdK7DebugStatusMaybe, AmdK7DebugStatusMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFN(0xc0011007, "AMD_K7_BH_TRACE_BASE?", AmdK7BHTraceBaseMaybe, AmdK7BHTraceBaseMaybe), /* value=0x0 */
+    MFN(0xc0011008, "AMD_K7_BH_TRACE_PTR?", AmdK7BHTracePtrMaybe, AmdK7BHTracePtrMaybe), /* value=0x0 */
+    MFN(0xc0011009, "AMD_K7_BH_TRACE_LIM?", AmdK7BHTraceLimitMaybe, AmdK7BHTraceLimitMaybe), /* value=0x0 */
+    MFX(0xc001100a, "AMD_K7_HDT_CFG?", AmdK7HardwareDebugToolCfgMaybe, AmdK7HardwareDebugToolCfgMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc001100b, "AMD_K7_FAST_FLUSH_COUNT?", AmdK7FastFlushCountMaybe, AmdK7FastFlushCountMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x7c0 */
+    MFX(0xc001100c, "AMD_K7_NODE_ID", AmdK7NodeId, AmdK7NodeId, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x20906 */
+    MVX(0xc001100d, "AMD_K8_LOGICAL_CPUS_NUM?", 0x10a, 0, 0),
+    MVX(0xc001100e, "AMD_K8_WRMSR_BP?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001100f, "AMD_K8_WRMSR_BP_MASK?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011010, "AMD_K8_BH_TRACE_CTL?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011011, "AMD_K8_BH_TRACE_USRD?", 0, 0, 0), /* value=0xc0011011`00000283 */
+    MVX(0xc0011014, "AMD_K8_XCPT_BP_RIP?", 0, 0, 0),
+    MVX(0xc0011015, "AMD_K8_XCPT_BP_RIP_MASK?", 0, 0, 0),
+    MVX(0xc0011016, "AMD_K8_COND_HDT_VAL?", 0, 0, 0),
+    MVX(0xc0011017, "AMD_K8_COND_HDT_VAL_MASK?", 0, 0, 0),
+    MVX(0xc0011018, "AMD_K8_XCPT_BP_CTL?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001101d, "AMD_K8_NB_BIST?", 0, UINT64_C(0xfffffffffc000000), 0),
+    MVI(0xc001101e, "AMD_K8_THERMTRIP_2?", 0x521020), /* Villain? */
+    MVX(0xc001101f, "AMD_K8_NB_CFG?", UINT64_C(0x1100000008), UINT64_C(0xffffff0000000000), 0),
+    MFX(0xc0011020, "AMD_K7_LS_CFG", AmdK7LoadStoreCfg, AmdK7LoadStoreCfg, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x1000 */
+    MFX(0xc0011021, "AMD_K7_IC_CFG", AmdK7InstrCacheCfg, AmdK7InstrCacheCfg, 0x800, ~(uint64_t)UINT32_MAX, 0), /* value=0x800 */
+    MFX(0xc0011022, "AMD_K7_DC_CFG", AmdK7DataCacheCfg, AmdK7DataCacheCfg, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x24000008 */
+    MFN(0xc0011023, "AMD_K7_BU_CFG", AmdK7BusUnitCfg, AmdK7BusUnitCfg), /* Villain? value=0x2020 */
+    MFX(0xc0011024, "AMD_K7_DEBUG_CTL_2?", AmdK7DebugCtl2Maybe, AmdK7DebugCtl2Maybe, 0, UINT64_C(0xffffffffffffff00), 0), /* value=0x0 */
+    MFN(0xc0011025, "AMD_K7_DR0_DATA_MATCH?", AmdK7Dr0DataMatchMaybe, AmdK7Dr0DataMatchMaybe), /* value=0x0 */
+    MFN(0xc0011026, "AMD_K7_DR0_DATA_MATCH?", AmdK7Dr0DataMaskMaybe, AmdK7Dr0DataMaskMaybe), /* value=0x0 */
+    MFX(0xc0011027, "AMD_K7_DR0_ADDR_MASK", AmdK7DrXAddrMaskN, AmdK7DrXAddrMaskN, 0x0, UINT64_C(0xfffffffffffff000), 0), /* value=0x0 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for AMD Athlon(tm) 64 X2 Dual Core Processor 4200+.
+ */
+static CPUMDBENTRY const g_Entry_AMD_Athlon_64_X2_Dual_Core_4200 =
+{
+    /*.pszName          = */ "AMD Athlon 64 X2 Dual Core 4200+",
+    /*.pszFullName      = */ "AMD Athlon(tm) 64 X2 Dual Core Processor 4200+",
+    /*.enmVendor        = */ CPUMCPUVENDOR_AMD,
+    /*.uFamily          = */ 15,
+    /*.uModel           = */ 75,
+    /*.uStepping        = */ 2,
+    /*.enmMicroarch     = */ kCpumMicroarch_AMD_K8_90nm_AMDV,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 40,
+    /*.fMxCsrMask       = */ 0xffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_AMD_Athlon_64_X2_Dual_Core_4200),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_AMD_Athlon_64_X2_Dual_Core_4200)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_AMD_Athlon_64_X2_Dual_Core_4200)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_AMD_Athlon_64_X2_Dual_Core_4200),
+};
+
+#endif /* !VBOX_CPUDB_AMD_Athlon_64_X2_Dual_Core_4200_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/AMD_FX_8150_Eight_Core.h b/src/VBox/VMM/VMMR3/cpus/AMD_FX_8150_Eight_Core.h
new file mode 100644
index 00000000..d01cb3f4
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/AMD_FX_8150_Eight_Core.h
@@ -0,0 +1,383 @@
+/* $Id: AMD_FX_8150_Eight_Core.h $ */
+/** @file
+ * CPU database entry "AMD FX-8150 Eight-Core".
+ * Generated at 2013-12-09T11:27:04Z by VBoxCpuReport v4.3.51r91084 on win.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_AMD_FX_8150_Eight_Core_h
+#define VBOX_CPUDB_AMD_FX_8150_Eight_Core_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for AMD FX(tm)-8150 Eight-Core Processor.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_AMD_FX_8150_Eight_Core[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000d, 0x68747541, 0x444d4163, 0x69746e65, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00600f12, 0x02080800, 0x1e98220b, 0x178bfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00000000, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000b, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000000, UINT32_MAX, 0x00000007, 0x00000340, 0x000003c0, 0x40000000, 0 },
+    { 0x0000000d, 0x00000001, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x8000001e, 0x68747541, 0x444d4163, 0x69746e65, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00600f12, 0x10000000, 0x01c9bfff, 0x2fd3fbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20444d41, 0x74285846, 0x382d296d, 0x20303531, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x68676945, 0x6f432d74, 0x50206572, 0x65636f72, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x726f7373, 0x20202020, 0x20202020, 0x00202020, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0xff20ff18, 0xff20ff30, 0x10040140, 0x40020140, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x64000000, 0x64004200, 0x08008140, 0x0040c140, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x000003d9, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003030, 0x00000000, 0x00004007, 0x00000000, 0 },
+    { 0x80000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000a, 0x00000000, 0x00000000, 0x00000001, 0x00010000, 0x00000000, 0x000014ff, 0 },
+    { 0x8000000b, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000d, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000e, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000f, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000010, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000011, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000012, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000013, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000014, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000015, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000016, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000017, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000018, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000019, 0x00000000, 0x00000000, 0xf020f018, 0x64000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001a, 0x00000000, 0x00000000, 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001b, 0x00000000, 0x00000000, 0x000000ff, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001c, 0x00000000, 0x00000000, 0x00000000, 0x80032013, 0x00010200, 0x8000000f, 0 },
+    { 0x8000001d, 0x00000000, UINT32_MAX, 0x00000121, 0x00c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x8000001d, 0x00000001, UINT32_MAX, 0x00004122, 0x0040003f, 0x000001ff, 0x00000000, 0 },
+    { 0x8000001d, 0x00000002, UINT32_MAX, 0x00004143, 0x03c0003f, 0x000007ff, 0x00000001, 0 },
+    { 0x8000001d, 0x00000003, UINT32_MAX, 0x0001c163, 0x0fc0003f, 0x000007ff, 0x00000001, 0 },
+    { 0x8000001d, 0x00000004, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001e, 0x00000000, 0x00000000, 0x00000012, 0x00000101, 0x00000000, 0x00000000, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for AMD FX(tm)-8150 Eight-Core Processor.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_AMD_FX_8150_Eight_Core[] =
+{
+    MAL(0x00000000, "IA32_P5_MC_ADDR", 0x00000402),
+    MAL(0x00000001, "IA32_P5_MC_TYPE", 0x00000401),
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter),
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0, UINT64_C(0xffff0000000006ff)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MVO(0x0000008b, "BBL_CR_D3|BIOS_SIGN", 0x6000626),
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf),
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf),
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x508, 0, 0), /* value=0x508 */
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x107, 0, 0), /* value=0x107 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, UINT64_C(0xfffffffffffffff8), 0), /* value=0x0 */
+    MFX(0x0000017b, "IA32_MCG_CTL", Ia32McgCtl, Ia32McgCtl, 0, UINT64_C(0xffffffffffffff88), 0), /* value=0x77 */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, UINT64_C(0xffffffffffffff80), 0x40), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0x0 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0x0 */
+    MFO(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp), /* value=0x0 */
+    MFO(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp), /* value=0x0 */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xffff000000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`80000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xffff000000000ff8)), /* value=0x80000006 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`c0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xffff000000000ff8)), /* value=0xc0000006 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`f0000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xffff000000000ff8)), /* value=0xcdf00000 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`fff00800 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xffff000000000ff8)), /* value=0xce000000 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`fe000800 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    RFN(0x00000400, 0x0000041b, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0x4d01, 0xfe, UINT64_C(0xffffffffffff8200)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xfffff800`02ed0bc0 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xfffff800`02ed0900 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x4700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0xfffe0000 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xfffff880`02f65000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x7ff`fffde000 */
+    MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0000104, "AMD_15H_TSC_RATE", AmdFam15hTscRate, AmdFam15hTscRate, 0, 0, UINT64_C(0xffffff0000000000)), /* value=0x1`00000000 */
+    MFX(0xc0000105, "AMD_15H_LWP_CFG", AmdFam15hLwpCfg, AmdFam15hLwpCfg, 0, UINT64_C(0xffff000000000001), 0x7ffffff0), /* value=0x0 */
+    MFX(0xc0000106, "AMD_15H_LWP_CBADDR", AmdFam15hLwpCbAddr, AmdFam15hLwpCbAddr, 0, 0, UINT64_MAX), /* value=0x0 */
+    RSN(0xc0000408, 0xc0000409, "AMD_10H_MC4_MISCn", AmdFam10hMc4MiscN, AmdFam10hMc4MiscN, 0, UINT64_C(0xff00f000ffffffff), 0),
+    RVI(0xc000040a, 0xc000040f, "AMD_10H_MC4_MISCn", 0),
+    MAL(0xc0010000, "AMD_K8_PERF_CTL_0", 0xc0010200),
+    MAL(0xc0010001, "AMD_K8_PERF_CTL_1", 0xc0010202),
+    MAL(0xc0010002, "AMD_K8_PERF_CTL_2", 0xc0010204),
+    MAL(0xc0010003, "AMD_K8_PERF_CTL_3", 0xc0010206),
+    MAL(0xc0010004, "AMD_K8_PERF_CTR_0", 0xc0010201),
+    MAL(0xc0010005, "AMD_K8_PERF_CTR_1", 0xc0010203),
+    MAL(0xc0010006, "AMD_K8_PERF_CTR_2", 0xc0010205),
+    MAL(0xc0010007, "AMD_K8_PERF_CTR_3", 0xc0010207),
+    MFX(0xc0010010, "AMD_K8_SYS_CFG", AmdK8SysCfg, AmdK8SysCfg, 0x740000, UINT64_C(0xffffffffff82ffff), 0), /* value=0x740000 */
+    MFX(0xc0010015, "AMD_K8_HW_CFG", AmdK8HwCr, AmdK8HwCr, 0, UINT64_C(0xffffffff01006020), 0), /* value=0x1001031 */
+    MFW(0xc0010016, "AMD_K8_IORR_BASE_0", AmdK8IorrBaseN, AmdK8IorrBaseN, UINT64_C(0xffff000000000fe7)), /* value=0x0 */
+    MFW(0xc0010017, "AMD_K8_IORR_MASK_0", AmdK8IorrMaskN, AmdK8IorrMaskN, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0xc0010018, "AMD_K8_IORR_BASE_1", AmdK8IorrBaseN, AmdK8IorrBaseN, 0x1, UINT64_C(0xffff000000000fe7), 0), /* value=0x0 */
+    MFX(0xc0010019, "AMD_K8_IORR_MASK_1", AmdK8IorrMaskN, AmdK8IorrMaskN, 0x1, UINT64_C(0xffff0000000007ff), 0), /* value=0x0 */
+    MFW(0xc001001a, "AMD_K8_TOP_MEM", AmdK8TopOfMemN, AmdK8TopOfMemN, UINT64_C(0xffff0000007fffff)), /* value=0xd0000000 */
+    MFX(0xc001001d, "AMD_K8_TOP_MEM2", AmdK8TopOfMemN, AmdK8TopOfMemN, 0x1, UINT64_C(0xffff0000007fffff), 0), /* value=0x4`2f000000 */
+    MFN(0xc001001f, "AMD_K8_NB_CFG1", AmdK8NbCfg1, AmdK8NbCfg1), /* value=0x400000`00810008 */
+    MFN(0xc0010020, "AMD_K8_PATCH_LOADER", WriteOnly, AmdK8PatchLoader),
+    MFX(0xc0010022, "AMD_K8_MC_XCPT_REDIR", AmdK8McXcptRedir, AmdK8McXcptRedir, 0, UINT64_C(0xffffffffffff0000), 0), /* value=0x0 */
+    MVO(0xc0010028, "AMD_K8_UNK_c001_0028", 0),
+    MVO(0xc0010029, "AMD_K8_UNK_c001_0029", 0),
+    MVO(0xc001002a, "AMD_K8_UNK_c001_002a", 0),
+    MVO(0xc001002b, "AMD_K8_UNK_c001_002b", 0),
+    MVO(0xc001002c, "AMD_K8_UNK_c001_002c", 0),
+    MVO(0xc001002d, "AMD_K8_UNK_c001_002d", 0),
+    RFN(0xc0010030, 0xc0010035, "AMD_K8_CPU_NAME_n", AmdK8CpuNameN, AmdK8CpuNameN),
+    MFX(0xc001003e, "AMD_K8_HTC", AmdK8HwThermalCtrl, AmdK8HwThermalCtrl, 0x664c0005, UINT64_C(0xffffffff90008838), 0), /* value=0x664c0005 */
+    MFX(0xc001003f, "AMD_K8_STC", AmdK8SwThermalCtrl, AmdK8SwThermalCtrl, 0, UINT64_C(0xffffffff9fffffdf), 0), /* value=0x60000000 */
+    MVO(0xc0010043, "AMD_K8_THERMTRIP_STATUS", 0x20),
+    MFX(0xc0010044, "AMD_K8_MC_CTL_MASK_0", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x0, UINT64_C(0xfffffffffffffc00), 0), /* value=0x0 */
+    MFX(0xc0010045, "AMD_K8_MC_CTL_MASK_1", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x1, UINT64_C(0xffffffffff004d01), 0), /* value=0x48080 */
+    MFX(0xc0010046, "AMD_K8_MC_CTL_MASK_2", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x2, UINT64_C(0xffffffffffff8000), 0), /* value=0x0 */
+    MFX(0xc0010047, "AMD_K8_MC_CTL_MASK_3", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x3, UINT64_MAX, 0), /* value=0x0 */
+    MFX(0xc0010048, "AMD_K8_MC_CTL_MASK_4", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x4, ~(uint64_t)UINT32_MAX, 0), /* value=0x780400 */
+    MFX(0xc0010049, "AMD_K8_MC_CTL_MASK_5", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x5, UINT64_C(0xffffffffffffe000), 0), /* value=0x0 */
+    MFX(0xc001004a, "AMD_K8_MC_CTL_MASK_6", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x6, UINT64_C(0xffffffffffffffc0), 0), /* value=0x0 */
+    RFN(0xc0010050, 0xc0010053, "AMD_K8_SMI_ON_IO_TRAP_n", AmdK8SmiOnIoTrapN, AmdK8SmiOnIoTrapN),
+    MFX(0xc0010054, "AMD_K8_SMI_ON_IO_TRAP_CTL_STS", AmdK8SmiOnIoTrapCtlSts, AmdK8SmiOnIoTrapCtlSts, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0010055, "AMD_K8_INT_PENDING_MSG", AmdK8IntPendingMessage, AmdK8IntPendingMessage, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x20000800 */
+    MFX(0xc0010056, "AMD_K8_SMI_TRIGGER_IO_CYCLE", AmdK8SmiTriggerIoCycle, AmdK8SmiTriggerIoCycle, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x2000000 */
+    MFX(0xc0010058, "AMD_10H_MMIO_CFG_BASE_ADDR", AmdFam10hMmioCfgBaseAddr, AmdFam10hMmioCfgBaseAddr, 0, UINT64_C(0xffff0000000fffc0), 0), /* value=0xe0000021 */
+    MFX(0xc0010059, "AMD_10H_TRAP_CTL?", AmdFam10hTrapCtlMaybe, AmdFam10hTrapCtlMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MVX(0xc001005a, "AMD_10H_UNK_c001_005a", 0, 0, 0),
+    MVX(0xc001005b, "AMD_10H_UNK_c001_005b", 0, 0, 0),
+    MVX(0xc001005c, "AMD_10H_UNK_c001_005c", 0, 0, 0),
+    MVX(0xc001005d, "AMD_10H_UNK_c001_005d", 0, 0, 0),
+    MVO(0xc0010060, "AMD_K8_BIST_RESULT", 0),
+    MFX(0xc0010061, "AMD_10H_P_ST_CUR_LIM", AmdFam10hPStateCurLimit, ReadOnly, 0x40, 0, 0), /* value=0x40 */
+    MFX(0xc0010062, "AMD_10H_P_ST_CTL", AmdFam10hPStateControl, AmdFam10hPStateControl, 0, 0, UINT64_C(0xfffffffffffffff8)), /* value=0x0 */
+    MFX(0xc0010063, "AMD_10H_P_ST_STS", AmdFam10hPStateStatus, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0xc0010064, "AMD_10H_P_ST_0", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x800001b10000161a), UINT64_C(0x7ffffc00ffbf0000), 0), /* value=0x800001b1`0000161a */
+    MFX(0xc0010065, "AMD_10H_P_ST_1", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x800001b100001a17), UINT64_C(0x7ffffc00ffbf0000), 0), /* value=0x800001b1`00001a17 */
+    MFX(0xc0010066, "AMD_10H_P_ST_2", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x8000017300003014), UINT64_C(0x7ffffc00ffbf0000), 0), /* value=0x80000173`00003014 */
+    MFX(0xc0010067, "AMD_10H_P_ST_3", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x8000016300003a11), UINT64_C(0x7ffffc00ffbf0000), 0), /* value=0x80000163`00003a11 */
+    MFX(0xc0010068, "AMD_10H_P_ST_4", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x8000014900004c0b), UINT64_C(0x7ffffc00ffbf0000), 0), /* value=0x80000149`00004c0b */
+    MFX(0xc0010069, "AMD_10H_P_ST_5", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x8000013100006205), UINT64_C(0x7ffffc00ffbf0000), 0), /* value=0x80000131`00006205 */
+    MFX(0xc001006a, "AMD_10H_P_ST_6", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x800001200000724c), UINT64_C(0x7ffffc00ffbf0000), 0), /* value=0x80000120`0000724c */
+    MFX(0xc001006b, "AMD_10H_P_ST_7", AmdFam10hPStateN, AmdFam10hPStateN, 0, UINT64_C(0x7ffffc00ffbf0000), 0), /* value=0x0 */
+    MFX(0xc0010070, "AMD_10H_COFVID_CTL", AmdFam10hCofVidControl, AmdFam10hCofVidControl, 0x40011a17, UINT64_C(0xffffffff00b80000), 0), /* value=0x40011a17 */
+    MFX(0xc0010071, "AMD_10H_COFVID_STS", AmdFam10hCofVidStatus, AmdFam10hCofVidStatus, UINT64_C(0x18000064006724c), UINT64_MAX, 0), /* value=0x1800006`4006724c */
+    MFX(0xc0010073, "AMD_10H_C_ST_IO_BASE_ADDR", AmdFam10hCStateIoBaseAddr, AmdFam10hCStateIoBaseAddr, 0, UINT64_C(0xffffffffffff0000), 0), /* value=0x814 */
+    MFX(0xc0010074, "AMD_10H_CPU_WD_TMR_CFG", AmdFam10hCpuWatchdogTimer, AmdFam10hCpuWatchdogTimer, 0, UINT64_C(0xffffffffffffff80), 0), /* value=0x0 */
+    MFX(0xc0010111, "AMD_K8_SMM_BASE", AmdK8SmmBase, AmdK8SmmBase, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0xcdef8800 */
+    MFX(0xc0010112, "AMD_K8_SMM_ADDR", AmdK8SmmAddr, AmdK8SmmAddr, 0, UINT64_C(0xffff00000001ffff), 0), /* value=0xcdf00000 */
+    MFX(0xc0010113, "AMD_K8_SMM_MASK", AmdK8SmmMask, AmdK8SmmMask, 0, UINT64_C(0xffff0000000188c0), 0), /* value=0xffff`fff00003 */
+    MFX(0xc0010114, "AMD_K8_VM_CR", AmdK8VmCr, AmdK8VmCr, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xffffffe0)), /* value=0x8 */
+    MFX(0xc0010115, "AMD_K8_IGNNE", AmdK8IgnNe, AmdK8IgnNe, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xfffffffe)), /* value=0x0 */
+    MFX(0xc0010117, "AMD_K8_VM_HSAVE_PA", AmdK8VmHSavePa, AmdK8VmHSavePa, 0, 0, UINT64_C(0xffff000000000fff)), /* value=0x0 */
+    MFN(0xc0010118, "AMD_10H_VM_LOCK_KEY", AmdFam10hVmLockKey, AmdFam10hVmLockKey), /* value=0x0 */
+    MFN(0xc0010119, "AMD_10H_SSM_LOCK_KEY", AmdFam10hSmmLockKey, AmdFam10hSmmLockKey), /* value=0x0 */
+    MFX(0xc001011a, "AMD_10H_LOCAL_SMI_STS", AmdFam10hLocalSmiStatus, AmdFam10hLocalSmiStatus, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0010140, "AMD_10H_OSVW_ID_LEN", AmdFam10hOsVisWrkIdLength, AmdFam10hOsVisWrkIdLength, 0x4, 0, 0), /* value=0x4 */
+    MFN(0xc0010141, "AMD_10H_OSVW_STS", AmdFam10hOsVisWrkStatus, AmdFam10hOsVisWrkStatus), /* value=0x0 */
+    MFX(0xc0010200, "AMD_K8_PERF_CTL_0", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x0, UINT64_C(0xfffffcf000200000), 0), /* value=0x0 */
+    MFX(0xc0010201, "AMD_K8_PERF_CTR_0", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x0, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010202, "AMD_K8_PERF_CTL_1", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x1, UINT64_C(0xfffffcf000200000), 0), /* value=0x0 */
+    MFX(0xc0010203, "AMD_K8_PERF_CTR_1", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x1, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010204, "AMD_K8_PERF_CTL_2", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x2, UINT64_C(0xfffffcf000200000), 0), /* value=0x0 */
+    MFX(0xc0010205, "AMD_K8_PERF_CTR_2", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x2, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010206, "AMD_K8_PERF_CTL_3", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x3, UINT64_C(0xfffffcf000200000), 0), /* value=0x0 */
+    MFX(0xc0010207, "AMD_K8_PERF_CTR_3", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x3, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010208, "AMD_K8_PERF_CTL_4", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x4, UINT64_C(0xfffffcf000200000), 0), /* value=0x0 */
+    MFX(0xc0010209, "AMD_K8_PERF_CTR_4", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x4, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc001020a, "AMD_K8_PERF_CTL_5", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x5, UINT64_C(0xfffffcf000200000), 0), /* value=0x0 */
+    MFX(0xc001020b, "AMD_K8_PERF_CTR_5", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x5, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010240, "AMD_15H_NB_PERF_CTL_0", AmdFam15hNorthbridgePerfCtlN, AmdFam15hNorthbridgePerfCtlN, 0x0, UINT64_C(0xfffffe00ffa70000), 0), /* value=0x0 */
+    MFX(0xc0010241, "AMD_15H_NB_PERF_CTR_0", AmdFam15hNorthbridgePerfCtrN, AmdFam15hNorthbridgePerfCtrN, 0x0, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010242, "AMD_15H_NB_PERF_CTL_1", AmdFam15hNorthbridgePerfCtlN, AmdFam15hNorthbridgePerfCtlN, 0x1, UINT64_C(0xfffffe00ffa70000), 0), /* value=0x0 */
+    MFX(0xc0010243, "AMD_15H_NB_PERF_CTR_1", AmdFam15hNorthbridgePerfCtrN, AmdFam15hNorthbridgePerfCtrN, 0x1, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010244, "AMD_15H_NB_PERF_CTL_2", AmdFam15hNorthbridgePerfCtlN, AmdFam15hNorthbridgePerfCtlN, 0x2, UINT64_C(0xfffffe00ffa70000), 0), /* value=0x0 */
+    MFX(0xc0010245, "AMD_15H_NB_PERF_CTR_2", AmdFam15hNorthbridgePerfCtrN, AmdFam15hNorthbridgePerfCtrN, 0x2, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010246, "AMD_15H_NB_PERF_CTL_3", AmdFam15hNorthbridgePerfCtlN, AmdFam15hNorthbridgePerfCtlN, 0x3, UINT64_C(0xfffffe00ffa70000), 0), /* value=0x0 */
+    MFX(0xc0010247, "AMD_15H_NB_PERF_CTR_3", AmdFam15hNorthbridgePerfCtrN, AmdFam15hNorthbridgePerfCtrN, 0x3, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0011000, "AMD_K7_MCODE_CTL", AmdK7MicrocodeCtl, AmdK7MicrocodeCtl, 0x30000, ~(uint64_t)UINT32_MAX, 0x204), /* value=0x30000 */
+    MFX(0xc0011001, "AMD_K7_APIC_CLUSTER_ID", AmdK7ClusterIdMaybe, AmdK7ClusterIdMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0011003, "AMD_K8_CPUID_CTL_STD06", AmdK8CpuIdCtlStd06hEcx, AmdK8CpuIdCtlStd06hEcx, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x1 */
+    MFN(0xc0011004, "AMD_K8_CPUID_CTL_STD01", AmdK8CpuIdCtlStd01hEdcx, AmdK8CpuIdCtlStd01hEdcx), /* value=0x1e98220b`178bfbff */
+    MFN(0xc0011005, "AMD_K8_CPUID_CTL_EXT01", AmdK8CpuIdCtlExt01hEdcx, AmdK8CpuIdCtlExt01hEdcx), /* value=0x1c9ffff`2fd3fbff */
+    MFX(0xc0011006, "AMD_K7_DEBUG_STS?", AmdK7DebugStatusMaybe, AmdK7DebugStatusMaybe, 0, UINT64_C(0xffffffff00000080), 0), /* value=0x10 */
+    MFN(0xc0011007, "AMD_K7_BH_TRACE_BASE?", AmdK7BHTraceBaseMaybe, AmdK7BHTraceBaseMaybe), /* value=0x0 */
+    MFN(0xc0011008, "AMD_K7_BH_TRACE_PTR?", AmdK7BHTracePtrMaybe, AmdK7BHTracePtrMaybe), /* value=0x0 */
+    MFN(0xc0011009, "AMD_K7_BH_TRACE_LIM?", AmdK7BHTraceLimitMaybe, AmdK7BHTraceLimitMaybe), /* value=0x0 */
+    MFX(0xc001100a, "AMD_K7_HDT_CFG?", AmdK7HardwareDebugToolCfgMaybe, AmdK7HardwareDebugToolCfgMaybe, 0, UINT64_C(0xffffffff00800000), 0), /* value=0x0 */
+    MFX(0xc001100b, "AMD_K7_FAST_FLUSH_COUNT?", AmdK7FastFlushCountMaybe, AmdK7FastFlushCountMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x7c0 */
+    MFX(0xc001100c, "AMD_K7_NODE_ID", AmdK7NodeId, AmdK7NodeId, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x80 */
+    MVX(0xc001100e, "AMD_K8_WRMSR_BP?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001100f, "AMD_K8_WRMSR_BP_MASK?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011010, "AMD_K8_BH_TRACE_CTL?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011011, "AMD_K8_BH_TRACE_USRD?", 0, 0, 0), /* value=0xfffffcf0`093634f0 */
+    MVI(0xc0011012, "AMD_K7_UNK_c001_1012", UINT32_MAX),
+    MVI(0xc0011013, "AMD_K7_UNK_c001_1013", UINT64_MAX),
+    MVX(0xc0011014, "AMD_K8_XCPT_BP_RIP?", 0, 0, 0),
+    MVX(0xc0011015, "AMD_K8_XCPT_BP_RIP_MASK?", 0, 0, 0),
+    MVX(0xc0011016, "AMD_K8_COND_HDT_VAL?", 0, 0, 0),
+    MVX(0xc0011017, "AMD_K8_COND_HDT_VAL_MASK?", 0, 0, 0),
+    MVX(0xc0011018, "AMD_K8_XCPT_BP_CTL?", 0, 0, 0),
+    MVX(0xc001101d, "AMD_K8_NB_BIST?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0xc001101e, "AMD_K8_THERMTRIP_2?", 0x20), /* Villain? */
+    MVX(0xc001101f, "AMD_K8_NB_CFG?", UINT64_C(0x40000000810008), 0, 0),
+    MFX(0xc0011020, "AMD_K7_LS_CFG", AmdK7LoadStoreCfg, AmdK7LoadStoreCfg, 0, UINT64_C(0x3fffedafbffe2a), 0), /* value=0x0 */
+    MFW(0xc0011021, "AMD_K7_IC_CFG", AmdK7InstrCacheCfg, AmdK7InstrCacheCfg, UINT64_C(0xffffff0000000000)), /* value=0x0 */
+    MFX(0xc0011022, "AMD_K7_DC_CFG", AmdK7DataCacheCfg, AmdK7DataCacheCfg, 0, UINT64_C(0x1ffffbfffff13e0), 0), /* value=0x0 */
+    MFX(0xc0011023, "AMD_15H_CU_CFG", AmdFam15hCombUnitCfg, AmdFam15hCombUnitCfg, 0x220, UINT64_C(0x3ff03c760042000), 0), /* value=0x80004000`00000220 */
+    MFX(0xc0011024, "AMD_K7_DEBUG_CTL_2?", AmdK7DebugCtl2Maybe, AmdK7DebugCtl2Maybe, 0, UINT64_C(0xfffffffffffffe04), 0), /* value=0x0 */
+    MFN(0xc0011025, "AMD_K7_DR0_DATA_MATCH?", AmdK7Dr0DataMatchMaybe, AmdK7Dr0DataMatchMaybe), /* value=0x0 */
+    MFN(0xc0011026, "AMD_K7_DR0_DATA_MATCH?", AmdK7Dr0DataMaskMaybe, AmdK7Dr0DataMaskMaybe), /* value=0x0 */
+    MFX(0xc0011027, "AMD_K7_DR0_ADDR_MASK", AmdK7DrXAddrMaskN, AmdK7DrXAddrMaskN, 0x0, UINT64_C(0xfffffffffffff000), 0), /* value=0x0 */
+    MFX(0xc0011028, "AMD_15H_FP_CFG", AmdFam15hFpuCfg, AmdFam15hFpuCfg, 0, UINT64_C(0xffffe000000000ff), 0), /* value=0x40`e91d0000 */
+    MFX(0xc0011029, "AMD_15H_DC_CFG", AmdFam15hDecoderCfg, AmdFam15hDecoderCfg, 0, UINT64_C(0xffffffffc0188001), 0), /* value=0x488400 */
+    MFX(0xc001102a, "AMD_15H_CU_CFG2", AmdFam15hCombUnitCfg2, AmdFam15hCombUnitCfg2, 0, UINT64_C(0xfffbfb8ff2fc623f), 0), /* value=0x40040`00000cc0 */
+    MFX(0xc001102b, "AMD_15H_CU_CFG3", AmdFam15hCombUnitCfg3, AmdFam15hCombUnitCfg3, 0, UINT64_C(0xffe0027afff00000), 0), /* value=0x33400`00002b93 */
+    MFX(0xc001102c, "AMD_15H_EX_CFG", AmdFam15hExecUnitCfg, AmdFam15hExecUnitCfg, 0x7aac0, UINT64_C(0xffb0c003fbe00024), 0), /* value=0x400`0007aac0 */
+    MFX(0xc0011030, "AMD_10H_IBS_FETCH_CTL", AmdFam10hIbsFetchCtl, AmdFam10hIbsFetchCtl, 0, UINT64_C(0xfdfeffffffff0000), 0), /* value=0x0 */
+    MFI(0xc0011031, "AMD_10H_IBS_FETCH_LIN_ADDR", AmdFam10hIbsFetchLinAddr), /* value=0x0 */
+    MFI(0xc0011032, "AMD_10H_IBS_FETCH_PHYS_ADDR", AmdFam10hIbsFetchPhysAddr), /* value=0x0 */
+    MFX(0xc0011033, "AMD_10H_IBS_OP_EXEC_CTL", AmdFam10hIbsOpExecCtl, AmdFam10hIbsOpExecCtl, 0, UINT64_C(0xf8000000f8010000), 0), /* value=0x0 */
+    MFN(0xc0011034, "AMD_10H_IBS_OP_RIP", AmdFam10hIbsOpRip, AmdFam10hIbsOpRip), /* value=0x0 */
+    MFX(0xc0011035, "AMD_10H_IBS_OP_DATA", AmdFam10hIbsOpData, AmdFam10hIbsOpData, 0, UINT64_C(0xffffffc000000000), 0), /* value=0x0 */
+    MFX(0xc0011036, "AMD_10H_IBS_OP_DATA2", AmdFam10hIbsOpData2, AmdFam10hIbsOpData2, 0, UINT64_C(0xffffffffffffffc8), 0), /* value=0x0 */
+    MFX(0xc0011037, "AMD_10H_IBS_OP_DATA3", AmdFam10hIbsOpData3, AmdFam10hIbsOpData3, 0, UINT64_C(0xffff0000fff00400), 0), /* value=0x0 */
+    MFN(0xc0011038, "AMD_10H_IBS_DC_LIN_ADDR", AmdFam10hIbsDcLinAddr, AmdFam10hIbsDcLinAddr), /* value=0x0 */
+    MFX(0xc0011039, "AMD_10H_IBS_DC_PHYS_ADDR", AmdFam10hIbsDcPhysAddr, AmdFam10hIbsDcPhysAddr, 0, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFO(0xc001103a, "AMD_10H_IBS_CTL", AmdFam10hIbsCtl), /* value=0x100 */
+    MFN(0xc001103b, "AMD_14H_IBS_BR_TARGET", AmdFam14hIbsBrTarget, AmdFam14hIbsBrTarget), /* value=0x0 */
+    MVX(0xc0011040, "AMD_15H_UNK_c001_1040", 0, UINT64_C(0xffe0000000000003), 0),
+    MVX(0xc0011041, "AMD_15H_UNK_c001_1041", UINT64_C(0x99dd57b219), 0xa0c820, 0),
+    MVX(0xc0011042, "AMD_15H_UNK_c001_1042", 0, 0, 0),
+    MVX(0xc0011043, "AMD_15H_UNK_c001_1043", UINT64_C(0x300000438), 0, 0),
+    MVX(0xc0011044, "AMD_15H_UNK_c001_1044", UINT64_C(0x300000438), 0, 0),
+    MVX(0xc0011045, "AMD_15H_UNK_c001_1045", UINT64_C(0x300000420), 0, 0),
+    MVX(0xc0011046, "AMD_15H_UNK_c001_1046", UINT64_C(0x300000420), 0, 0),
+    MVX(0xc0011047, "AMD_15H_UNK_c001_1047", 0, UINT64_C(0xffff000000000000), 0),
+    MVX(0xc0011048, "AMD_15H_UNK_c001_1048", 0xc000001, UINT64_C(0xffff000000000000), 0),
+    MVX(0xc0011049, "AMD_15H_UNK_c001_1049", 0, UINT64_C(0xffff000000000000), 0),
+    MVX(0xc001104a, "AMD_15H_UNK_c001_104a", 0, UINT64_C(0xffff000000000000), 0),
+    MVX(0xc001104b, "AMD_15H_UNK_c001_104b", 0, 0, 0),
+    MVX(0xc001104c, "AMD_15H_UNK_c001_104c", 0, 0, 0),
+    MVX(0xc001104d, "AMD_15H_UNK_c001_104d", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001104e, "AMD_15H_UNK_c001_104e", 0, UINT64_C(0xfffffc0000000000), 0),
+    MVX(0xc001104f, "AMD_15H_UNK_c001_104f", 0, UINT64_C(0xfffffc0000000000), 0),
+    MVX(0xc0011050, "AMD_15H_UNK_c001_1050", 0, UINT64_C(0xfffffc0000000000), 0),
+    MVX(0xc0011051, "AMD_15H_UNK_c001_1051", 0, UINT64_C(0xfffffc0000000000), 0),
+    MVX(0xc0011052, "AMD_15H_UNK_c001_1052", 0, UINT64_C(0xfffffc0000000000), 0),
+    MVX(0xc0011053, "AMD_15H_UNK_c001_1053", 0, UINT64_C(0xfffffc0000000000), 0),
+    MVX(0xc0011054, "AMD_15H_UNK_c001_1054", 0, UINT64_C(0xfffffc0000000000), 0),
+    MVX(0xc0011055, "AMD_15H_UNK_c001_1055", 0, UINT64_C(0xfffffc0000000000), 0),
+    MVX(0xc0011056, "AMD_15H_UNK_c001_1056", 0, UINT64_C(0xfffffc0000000000), 0),
+    MVX(0xc0011057, "AMD_15H_UNK_c001_1057", 0, UINT64_C(0xfffffc0000000000), 0),
+    MVX(0xc0011058, "AMD_15H_UNK_c001_1058", 0, UINT64_C(0xfffffc0000000000), 0),
+    MVX(0xc0011059, "AMD_15H_UNK_c001_1059", 0, UINT64_C(0xfffffc0000000000), 0),
+    MVX(0xc001105a, "AMD_15H_UNK_c001_105a", UINT64_C(0x3060c183060c183), UINT64_C(0x8000000000000000), 0),
+    MVX(0xc001105b, "AMD_15H_UNK_c001_105b", UINT64_C(0x318c6318c60c183), UINT64_C(0xe000000000000000), 0),
+    MVX(0xc001105c, "AMD_15H_UNK_c001_105c", 0, UINT64_C(0xff00000000000000), 0),
+    MVX(0xc001105d, "AMD_15H_UNK_c001_105d", 0, UINT64_C(0xff00000000000000), 0),
+    MVX(0xc001105e, "AMD_15H_UNK_c001_105e", 0, UINT64_C(0xfffffffffffffc00), 0),
+    MVX(0xc001105f, "AMD_15H_UNK_c001_105f", 0, UINT64_C(0xffff000000000000), 0),
+    MVX(0xc0011060, "AMD_15H_UNK_c001_1060", 0, UINT64_C(0xffff000000000000), 0),
+    MVX(0xc0011061, "AMD_15H_UNK_c001_1061", 0, 0, 0),
+    MVX(0xc0011062, "AMD_15H_UNK_c001_1062", 0, UINT64_C(0xffffffffffffe000), 0),
+    MVX(0xc0011063, "AMD_15H_UNK_c001_1063", 0, UINT64_C(0xfffffffffffe4000), 0),
+    MVX(0xc0011064, "AMD_15H_UNK_c001_1064", 0x1, UINT64_C(0xfffffffffffff000), 0),
+    MVX(0xc0011065, "AMD_15H_UNK_c001_1065", 0x1, UINT64_C(0xfffffffff0000000), 0),
+    MVX(0xc0011066, "AMD_15H_UNK_c001_1066", 0, 0, 0),
+    MVX(0xc0011067, "AMD_15H_UNK_c001_1067", 0x1, UINT64_C(0xffffffffffffff80), 0),
+    MVX(0xc0011068, "AMD_15H_UNK_c001_1068", 0, 0, 0),
+    MVX(0xc0011069, "AMD_15H_UNK_c001_1069", 0, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0xc001106a, "AMD_15H_UNK_c001_106a", 0x1, 0, 0),
+    MVX(0xc001106b, "AMD_15H_UNK_c001_106b", 0, UINT64_C(0xfffffffffffffff0), 0),
+    MVX(0xc001106c, "AMD_15H_UNK_c001_106c", 0x1, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0xc001106d, "AMD_15H_UNK_c001_106d", 0x1, UINT64_C(0xf000000000000080), 0),
+    MVX(0xc001106e, "AMD_15H_UNK_c001_106e", 0x1, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0xc001106f, "AMD_15H_UNK_c001_106f", 0x1, UINT64_C(0xfffffffffffff800), 0),
+    MVI(0xc0011070, "AMD_15H_UNK_c001_1070", UINT64_C(0x20000000000)),
+    MVX(0xc0011071, "AMD_15H_UNK_c001_1071", 0x400000, UINT64_C(0xffffffff01ffffff), 0),
+    MVI(0xc0011072, "AMD_15H_UNK_c001_1072", UINT64_C(0x101592c00000021)),
+    MVI(0xc0011073, "AMD_15H_UNK_c001_1073", UINT64_C(0xec541c0050000000)),
+    MVX(0xc0011080, "AMD_15H_UNK_c001_1080", 0, 0, 0),
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for AMD FX(tm)-8150 Eight-Core Processor.
+ */
+static CPUMDBENTRY const g_Entry_AMD_FX_8150_Eight_Core =
+{
+    /*.pszName          = */ "AMD FX-8150 Eight-Core",
+    /*.pszFullName      = */ "AMD FX(tm)-8150 Eight-Core Processor",
+    /*.enmVendor        = */ CPUMCPUVENDOR_AMD,
+    /*.uFamily          = */ 21,
+    /*.uModel           = */ 1,
+    /*.uStepping        = */ 2,
+    /*.enmMicroarch     = */ kCpumMicroarch_AMD_15h_Bulldozer,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 48,
+    /*.fMxCsrMask       = */ 0x2ffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_AMD_FX_8150_Eight_Core),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_AMD_FX_8150_Eight_Core)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_AMD_FX_8150_Eight_Core)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_AMD_FX_8150_Eight_Core),
+};
+
+#endif /* !VBOX_CPUDB_AMD_FX_8150_Eight_Core_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/AMD_Phenom_II_X6_1100T.h b/src/VBox/VMM/VMMR3/cpus/AMD_Phenom_II_X6_1100T.h
new file mode 100644
index 00000000..879864bf
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/AMD_Phenom_II_X6_1100T.h
@@ -0,0 +1,272 @@
+/* $Id: AMD_Phenom_II_X6_1100T.h $ */
+/** @file
+ * CPU database entry "AMD Phenom II X6 1100T".
+ * Generated at 2013-12-17T13:39:08Z by VBoxCpuReport v4.3.53r91360 on linux.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_AMD_Phenom_II_X6_1100T_h
+#define VBOX_CPUDB_AMD_Phenom_II_X6_1100T_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for AMD Phenom(tm) II X6 1100T Processor.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_AMD_Phenom_II_X6_1100T[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000006, 0x68747541, 0x444d4163, 0x69746e65, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00100fa0, 0x01060800, 0x00802009, 0x178bfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00000000, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x8000001b, 0x68747541, 0x444d4163, 0x69746e65, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00100fa0, 0x100000a1, 0x000837ff, 0xefd3fbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20444d41, 0x6e656850, 0x74286d6f, 0x4920296d, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x36582049, 0x30313120, 0x50205430, 0x65636f72, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x726f7373, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0xff30ff10, 0xff30ff20, 0x40020140, 0x40020140, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x20800000, 0x42004200, 0x02008140, 0x0030b140, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x000003f9, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003030, 0x00000000, 0x00003005, 0x00000000, 0 },
+    { 0x80000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000a, 0x00000000, 0x00000000, 0x00000001, 0x00000040, 0x00000000, 0x0000040f, 0 },
+    { 0x8000000b, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000d, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000e, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000f, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000010, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000011, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000012, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000013, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000014, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000015, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000016, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000017, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000018, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000019, 0x00000000, 0x00000000, 0xf0300000, 0x60100000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001a, 0x00000000, 0x00000000, 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001b, 0x00000000, 0x00000000, 0x0000001f, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for AMD Phenom(tm) II X6 1100T Processor.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_AMD_Phenom_II_X6_1100T[] =
+{
+    MAL(0x00000000, "IA32_P5_MC_ADDR", 0x00000402),
+    MAL(0x00000001, "IA32_P5_MC_TYPE", 0x00000401),
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x6db`c482d0b9 */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0, UINT64_C(0xffff0000000006ff)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MVO(0x0000008b, "BBL_CR_D3|BIOS_SIGN", 0x10000bf),
+    MFX(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf, 0, UINT64_C(0x8644930520000000), 0), /* value=0xa66664d9`32c329b1 */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x25`092f34be */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x508, 0, 0), /* value=0x508 */
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x10 */
+    MFX(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x8174c700 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x106, 0, 0), /* value=0x106 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, UINT64_C(0xfffffffffffffff8), 0), /* value=0x0 */
+    MFX(0x0000017b, "IA32_MCG_CTL", Ia32McgCtl, Ia32McgCtl, 0, UINT64_C(0xffffffffffffffc0), 0), /* value=0x3f */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, UINT64_C(0xffffffffffffff80), 0x40), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0xffffefdf`00890004 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xffffeed0`c7b3ffbc */
+    MFO(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp), /* value=0x0 */
+    MFO(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp), /* value=0x0 */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xffff000000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`00000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xffff000000000ff8)), /* value=0xbdf00000 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`fff00800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xffff000000000ff8)), /* value=0xbe000000 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`fe000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xffff000000000ff8)), /* value=0xc0000000 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`c0000800 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    RFN(0x00000400, 0x00000417, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0xfe, UINT64_C(0xffffffffffff8200)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xffffffff`8174b4f0 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xffffffff`8174c860 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x3700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x7f01`3f916740 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffff8804`3fc00000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0xf2c95840 */
+    MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    RSN(0xc0000408, 0xc000040a, "AMD_10H_MC4_MISCn", AmdFam10hMc4MiscN, AmdFam10hMc4MiscN, 0, UINT64_C(0xff00f000ffffffff), 0),
+    RVI(0xc000040b, 0xc000040f, "AMD_10H_MC4_MISCn", 0),
+    RSN(0xc0010000, 0xc0010003, "AMD_K8_PERF_CTL_n", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x0, UINT64_C(0xfffffcf000200000), 0),
+    RSN(0xc0010004, 0xc0010007, "AMD_K8_PERF_CTR_n", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x0, UINT64_C(0xffff000000000000), 0),
+    MFX(0xc0010010, "AMD_K8_SYS_CFG", AmdK8SysCfg, AmdK8SysCfg, 0x760600, UINT64_C(0xffffffffff80f8ff), 0), /* value=0x760600 */
+    MFX(0xc0010015, "AMD_K8_HW_CFG", AmdK8HwCr, AmdK8HwCr, 0x1000031, UINT64_C(0xffffffff00006020), 0), /* value=0x1000031 */
+    MFW(0xc0010016, "AMD_K8_IORR_BASE_0", AmdK8IorrBaseN, AmdK8IorrBaseN, UINT64_C(0xffff000000000fe7)), /* value=0x3`40200000 */
+    MFW(0xc0010017, "AMD_K8_IORR_MASK_0", AmdK8IorrMaskN, AmdK8IorrMaskN, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0xc0010018, "AMD_K8_IORR_BASE_1", AmdK8IorrBaseN, AmdK8IorrBaseN, 0x1, UINT64_C(0xffff000000000fe7), 0), /* value=0x0 */
+    MFX(0xc0010019, "AMD_K8_IORR_MASK_1", AmdK8IorrMaskN, AmdK8IorrMaskN, 0x1, UINT64_C(0xffff0000000007ff), 0), /* value=0x0 */
+    MFW(0xc001001a, "AMD_K8_TOP_MEM", AmdK8TopOfMemN, AmdK8TopOfMemN, UINT64_C(0xffff0000007fffff)), /* value=0xc0000000 */
+    MFX(0xc001001d, "AMD_K8_TOP_MEM2", AmdK8TopOfMemN, AmdK8TopOfMemN, 0x1, UINT64_C(0xffff0000007fffff), 0), /* value=0x4`40000000 */
+    MFN(0xc001001f, "AMD_K8_NB_CFG1", AmdK8NbCfg1, AmdK8NbCfg1), /* value=0x584000`00000008 */
+    MFN(0xc0010020, "AMD_K8_PATCH_LOADER", WriteOnly, AmdK8PatchLoader),
+    MFN(0xc0010021, "AMD_10H_UNK_c001_0021", WriteOnly, IgnoreWrite),
+    MFX(0xc0010022, "AMD_K8_MC_XCPT_REDIR", AmdK8McXcptRedir, AmdK8McXcptRedir, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    RFN(0xc0010030, 0xc0010035, "AMD_K8_CPU_NAME_n", AmdK8CpuNameN, AmdK8CpuNameN),
+    MFX(0xc001003e, "AMD_K8_HTC", AmdK8HwThermalCtrl, AmdK8HwThermalCtrl, 0x4a4c0005, UINT64_C(0xffffffffb0008838), 0), /* value=0x4a4c0005 */
+    MFX(0xc001003f, "AMD_K8_STC", AmdK8SwThermalCtrl, AmdK8SwThermalCtrl, 0, UINT64_C(0xffffffffc00088c0), 0), /* value=0x10000000 */
+    MVO(0xc0010043, "AMD_K8_THERMTRIP_STATUS", 0x1dc01430),
+    MFX(0xc0010044, "AMD_K8_MC_CTL_MASK_0", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x0, UINT64_C(0xffffffffffffff00), 0), /* value=0x80 */
+    MFX(0xc0010045, "AMD_K8_MC_CTL_MASK_1", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x1, ~(uint64_t)UINT32_MAX, 0), /* value=0x80 */
+    MFX(0xc0010046, "AMD_K8_MC_CTL_MASK_2", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x2, UINT64_C(0xfffffffffffff000), 0), /* value=0x200 */
+    MFX(0xc0010047, "AMD_K8_MC_CTL_MASK_3", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x3, UINT64_C(0xfffffffffffffffc), 0), /* value=0x0 */
+    MFX(0xc0010048, "AMD_K8_MC_CTL_MASK_4", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x4, UINT64_C(0xffffffffc0000000), 0), /* value=0x780400 */
+    MFX(0xc0010049, "AMD_K8_MC_CTL_MASK_5", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x5, UINT64_C(0xfffffffffffffffe), 0), /* value=0x0 */
+    RFN(0xc0010050, 0xc0010053, "AMD_K8_SMI_ON_IO_TRAP_n", AmdK8SmiOnIoTrapN, AmdK8SmiOnIoTrapN),
+    MFX(0xc0010054, "AMD_K8_SMI_ON_IO_TRAP_CTL_STS", AmdK8SmiOnIoTrapCtlSts, AmdK8SmiOnIoTrapCtlSts, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0010055, "AMD_K8_INT_PENDING_MSG", AmdK8IntPendingMessage, AmdK8IntPendingMessage, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x14000815 */
+    MFX(0xc0010056, "AMD_K8_SMI_TRIGGER_IO_CYCLE", AmdK8SmiTriggerIoCycle, AmdK8SmiTriggerIoCycle, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x2000000 */
+    MFX(0xc0010058, "AMD_10H_MMIO_CFG_BASE_ADDR", AmdFam10hMmioCfgBaseAddr, AmdFam10hMmioCfgBaseAddr, 0, UINT64_C(0xffff0000000fffc0), 0), /* value=0xe0000021 */
+    MFX(0xc0010059, "AMD_10H_TRAP_CTL?", AmdFam10hTrapCtlMaybe, AmdFam10hTrapCtlMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MVX(0xc001005a, "AMD_10H_UNK_c001_005a", 0, 0, 0),
+    MVX(0xc001005b, "AMD_10H_UNK_c001_005b", 0, 0, 0),
+    MVX(0xc001005c, "AMD_10H_UNK_c001_005c", 0, 0, 0),
+    MVX(0xc001005d, "AMD_10H_UNK_c001_005d", 0, 0, 0),
+    MVO(0xc0010060, "AMD_K8_BIST_RESULT", 0),
+    MFX(0xc0010061, "AMD_10H_P_ST_CUR_LIM", AmdFam10hPStateCurLimit, ReadOnly, 0x30, 0, 0), /* value=0x30 */
+    MFX(0xc0010062, "AMD_10H_P_ST_CTL", AmdFam10hPStateControl, AmdFam10hPStateControl, 0x3, 0, UINT64_C(0xfffffffffffffff8)), /* value=0x3 */
+    MFX(0xc0010063, "AMD_10H_P_ST_STS", AmdFam10hPStateStatus, ReadOnly, 0x3, 0, 0), /* value=0x3 */
+    MFX(0xc0010064, "AMD_10H_P_ST_0", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x8000019e40001015), 0, 0), /* value=0x8000019e`40001015 */
+    MFX(0xc0010065, "AMD_10H_P_ST_1", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x8000019f40002411), 0, 0), /* value=0x8000019f`40002411 */
+    MFX(0xc0010066, "AMD_10H_P_ST_2", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x8000017540002809), 0, 0), /* value=0x80000175`40002809 */
+    MFX(0xc0010067, "AMD_10H_P_ST_3", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x8000015540002c01), 0, 0), /* value=0x80000155`40002c01 */
+    MFX(0xc0010068, "AMD_10H_P_ST_4", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x8000013340003840), 0, 0), /* value=0x80000133`40003840 */
+    MFX(0xc0010070, "AMD_10H_COFVID_CTL", AmdFam10hCofVidControl, AmdFam10hCofVidControl, 0x40043840, UINT64_C(0xffffffff01b80000), 0), /* value=0x40043840 */
+    MFX(0xc0010071, "AMD_10H_COFVID_STS", AmdFam10hCofVidStatus, AmdFam10hCofVidStatus, UINT64_C(0x140043840), UINT64_MAX, 0), /* value=0x1`40043840 */
+    MFO(0xc0010073, "AMD_10H_C_ST_IO_BASE_ADDR", AmdFam10hCStateIoBaseAddr), /* value=0x814 */
+    MFX(0xc0010074, "AMD_10H_CPU_WD_TMR_CFG", AmdFam10hCpuWatchdogTimer, AmdFam10hCpuWatchdogTimer, 0, UINT64_C(0xffffffffffffff80), 0), /* value=0x0 */
+    MFX(0xc0010111, "AMD_K8_SMM_BASE", AmdK8SmmBase, AmdK8SmmBase, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0xbdef8000 */
+    MFX(0xc0010112, "AMD_K8_SMM_ADDR", AmdK8SmmAddr, AmdK8SmmAddr, 0, UINT64_C(0xffff00000001ffff), 0), /* value=0xbdf00000 */
+    MFX(0xc0010113, "AMD_K8_SMM_MASK", AmdK8SmmMask, AmdK8SmmMask, 0, UINT64_C(0xffff0000000188c0), 0), /* value=0xffff`fff00003 */
+    MFX(0xc0010114, "AMD_K8_VM_CR", AmdK8VmCr, AmdK8VmCr, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xffffffe0)), /* value=0x8 */
+    MFX(0xc0010115, "AMD_K8_IGNNE", AmdK8IgnNe, AmdK8IgnNe, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xfffffffe)), /* value=0x0 */
+    MFX(0xc0010117, "AMD_K8_VM_HSAVE_PA", AmdK8VmHSavePa, AmdK8VmHSavePa, 0, 0, UINT64_C(0xffff000000000fff)), /* value=0x0 */
+    MFN(0xc0010118, "AMD_10H_VM_LOCK_KEY", AmdFam10hVmLockKey, AmdFam10hVmLockKey), /* value=0x0 */
+    MFN(0xc0010119, "AMD_10H_SSM_LOCK_KEY", AmdFam10hSmmLockKey, AmdFam10hSmmLockKey), /* value=0x0 */
+    MFX(0xc001011a, "AMD_10H_LOCAL_SMI_STS", AmdFam10hLocalSmiStatus, AmdFam10hLocalSmiStatus, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0010140, "AMD_10H_OSVW_ID_LEN", AmdFam10hOsVisWrkIdLength, AmdFam10hOsVisWrkIdLength, 0x4, 0, 0), /* value=0x4 */
+    MFX(0xc0010141, "AMD_10H_OSVW_STS", AmdFam10hOsVisWrkStatus, AmdFam10hOsVisWrkStatus, 0xe, 0, 0), /* value=0xe */
+    MFX(0xc0011000, "AMD_K7_MCODE_CTL", AmdK7MicrocodeCtl, AmdK7MicrocodeCtl, 0, ~(uint64_t)UINT32_MAX, 0x4), /* value=0x0 */
+    MFX(0xc0011001, "AMD_K7_APIC_CLUSTER_ID", AmdK7ClusterIdMaybe, AmdK7ClusterIdMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFN(0xc0011004, "AMD_K8_CPUID_CTL_STD01", AmdK8CpuIdCtlStd01hEdcx, AmdK8CpuIdCtlStd01hEdcx), /* value=0x802009`178bfbff */
+    MFN(0xc0011005, "AMD_K8_CPUID_CTL_EXT01", AmdK8CpuIdCtlExt01hEdcx, AmdK8CpuIdCtlExt01hEdcx), /* value=0x837ff`efd3fbff */
+    MFX(0xc0011006, "AMD_K7_DEBUG_STS?", AmdK7DebugStatusMaybe, AmdK7DebugStatusMaybe, 0, UINT64_C(0xffffffff00000080), 0), /* value=0x10 */
+    MFN(0xc0011007, "AMD_K7_BH_TRACE_BASE?", AmdK7BHTraceBaseMaybe, AmdK7BHTraceBaseMaybe), /* value=0x0 */
+    MFN(0xc0011008, "AMD_K7_BH_TRACE_PTR?", AmdK7BHTracePtrMaybe, AmdK7BHTracePtrMaybe), /* value=0x0 */
+    MFN(0xc0011009, "AMD_K7_BH_TRACE_LIM?", AmdK7BHTraceLimitMaybe, AmdK7BHTraceLimitMaybe), /* value=0x0 */
+    MFX(0xc001100a, "AMD_K7_HDT_CFG?", AmdK7HardwareDebugToolCfgMaybe, AmdK7HardwareDebugToolCfgMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc001100b, "AMD_K7_FAST_FLUSH_COUNT?", AmdK7FastFlushCountMaybe, AmdK7FastFlushCountMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x7c0 */
+    MFX(0xc001100c, "AMD_K7_NODE_ID", AmdK7NodeId, AmdK7NodeId, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MVX(0xc001100d, "AMD_K8_LOGICAL_CPUS_NUM?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001100e, "AMD_K8_WRMSR_BP?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001100f, "AMD_K8_WRMSR_BP_MASK?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011010, "AMD_K8_BH_TRACE_CTL?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011011, "AMD_K8_BH_TRACE_USRD?", 0, 0, 0), /* value=0x259a5de0`ffffffff */
+    MVX(0xc0011014, "AMD_K8_XCPT_BP_RIP?", 0, 0, 0),
+    MVX(0xc0011015, "AMD_K8_XCPT_BP_RIP_MASK?", 0, 0, 0),
+    MVX(0xc0011016, "AMD_K8_COND_HDT_VAL?", 0, 0, 0),
+    MVX(0xc0011017, "AMD_K8_COND_HDT_VAL_MASK?", 0, 0, 0),
+    MVX(0xc0011018, "AMD_K8_XCPT_BP_CTL?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001101d, "AMD_K8_NB_BIST?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0xc001101e, "AMD_K8_THERMTRIP_2?", 0x1dc01430), /* Villain? */
+    MVX(0xc001101f, "AMD_K8_NB_CFG?", UINT64_C(0x58400000000008), 0, 0),
+    MFX(0xc0011020, "AMD_K7_LS_CFG", AmdK7LoadStoreCfg, AmdK7LoadStoreCfg, 0, UINT64_C(0xfffc012000000000), 0), /* value=0x20010`00001000 */
+    MFW(0xc0011021, "AMD_K7_IC_CFG", AmdK7InstrCacheCfg, AmdK7InstrCacheCfg, ~(uint64_t)UINT32_MAX), /* value=0x0 */
+    MFX(0xc0011022, "AMD_K7_DC_CFG", AmdK7DataCacheCfg, AmdK7DataCacheCfg, 0, UINT64_C(0xffc0000000000000), 0), /* value=0x9c`49000000 */
+    MFN(0xc0011023, "AMD_K7_BU_CFG", AmdK7BusUnitCfg, AmdK7BusUnitCfg), /* Villain? value=0x10200020 */
+    MFX(0xc0011024, "AMD_K7_DEBUG_CTL_2?", AmdK7DebugCtl2Maybe, AmdK7DebugCtl2Maybe, 0, UINT64_C(0xffffffffffffff00), 0), /* value=0x0 */
+    MFN(0xc0011025, "AMD_K7_DR0_DATA_MATCH?", AmdK7Dr0DataMatchMaybe, AmdK7Dr0DataMatchMaybe), /* value=0x0 */
+    MFN(0xc0011026, "AMD_K7_DR0_DATA_MATCH?", AmdK7Dr0DataMaskMaybe, AmdK7Dr0DataMaskMaybe), /* value=0x0 */
+    MFX(0xc0011027, "AMD_K7_DR0_ADDR_MASK", AmdK7DrXAddrMaskN, AmdK7DrXAddrMaskN, 0x0, UINT64_C(0xfffffffffffff000), 0), /* value=0x0 */
+    MVX(0xc0011028, "AMD_10H_UNK_c001_1028", 0, UINT64_C(0xfffffffffffffff8), 0),
+    MVX(0xc0011029, "AMD_10H_UNK_c001_1029", 0, ~(uint64_t)UINT32_MAX, 0),
+    MFX(0xc001102a, "AMD_10H_BU_CFG2", AmdFam10hBusUnitCfg2, AmdFam10hBusUnitCfg2, 0, UINT64_C(0xfff00000c0000000), 0), /* value=0x40050`01000040 */
+    MFX(0xc0011030, "AMD_10H_IBS_FETCH_CTL", AmdFam10hIbsFetchCtl, AmdFam10hIbsFetchCtl, 0, UINT64_C(0xfdfcffff00000000), 0), /* value=0x140003`00000000 */
+    MFI(0xc0011031, "AMD_10H_IBS_FETCH_LIN_ADDR", AmdFam10hIbsFetchLinAddr), /* value=0xffffffff`a08cf13e */
+    MFI(0xc0011032, "AMD_10H_IBS_FETCH_PHYS_ADDR", AmdFam10hIbsFetchPhysAddr), /* value=0x4`24ce313e */
+    MFX(0xc0011033, "AMD_10H_IBS_OP_EXEC_CTL", AmdFam10hIbsOpExecCtl, AmdFam10hIbsOpExecCtl, 0, UINT64_C(0xfffffffffff00000), 0), /* value=0x0 */
+    MFN(0xc0011034, "AMD_10H_IBS_OP_RIP", AmdFam10hIbsOpRip, AmdFam10hIbsOpRip), /* value=0x4d231923 */
+    MFI(0xc0011035, "AMD_10H_IBS_OP_DATA", AmdFam10hIbsOpData), /* value=0x12`7fc7bc0e */
+    MFX(0xc0011036, "AMD_10H_IBS_OP_DATA2", AmdFam10hIbsOpData2, AmdFam10hIbsOpData2, 0, UINT64_C(0xffffffffffffffc8), 0), /* value=0x0 */
+    MFI(0xc0011037, "AMD_10H_IBS_OP_DATA3", AmdFam10hIbsOpData3), /* value=0x0 */
+    MFX(0xc0011038, "AMD_10H_IBS_DC_LIN_ADDR", AmdFam10hIbsDcLinAddr, AmdFam10hIbsDcLinAddr, 0, UINT64_C(0x7fffffffffff), 0), /* value=0x0 */
+    MFI(0xc0011039, "AMD_10H_IBS_DC_PHYS_ADDR", AmdFam10hIbsDcPhysAddr), /* value=0x0 */
+    MFO(0xc001103a, "AMD_10H_IBS_CTL", AmdFam10hIbsCtl), /* value=0x101 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for AMD Phenom(tm) II X6 1100T Processor.
+ */
+static CPUMDBENTRY const g_Entry_AMD_Phenom_II_X6_1100T =
+{
+    /*.pszName          = */ "AMD Phenom II X6 1100T",
+    /*.pszFullName      = */ "AMD Phenom(tm) II X6 1100T Processor",
+    /*.enmVendor        = */ CPUMCPUVENDOR_AMD,
+    /*.uFamily          = */ 16,
+    /*.uModel           = */ 10,
+    /*.uStepping        = */ 0,
+    /*.enmMicroarch     = */ kCpumMicroarch_AMD_K10,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 48,
+    /*.fMxCsrMask       = */ 0x2ffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_AMD_Phenom_II_X6_1100T),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_AMD_Phenom_II_X6_1100T)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_AMD_Phenom_II_X6_1100T)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_AMD_Phenom_II_X6_1100T),
+};
+
+#endif /* !VBOX_CPUDB_AMD_Phenom_II_X6_1100T_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Hygon_C86_7185_32_core.h b/src/VBox/VMM/VMMR3/cpus/Hygon_C86_7185_32_core.h
new file mode 100644
index 00000000..95939fc6
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Hygon_C86_7185_32_core.h
@@ -0,0 +1,5222 @@
+/* $Id: Hygon_C86_7185_32_core.h $ */
+/** @file
+ * CPU database entry "Hygon C86 7185 32-core".
+ * Generated at 2019-09-25T11:07:33Z by VBoxCpuReport v6.1.0_BETA1r80830 on linux.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Hygon_C86_7185_32_core_h
+#define VBOX_CPUDB_Hygon_C86_7185_32_core_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Hygon C86 7185 32-core Processor.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Hygon_C86_7185_32_core[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000d, 0x6f677948, 0x656e6975, 0x6e65476e, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00900f01, 0x3b400800, 0x7cd83209, 0x178bfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00000011, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000004, 0x00000000, 0x00000001, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, UINT32_MAX, 0x00000000, 0x009c01a9, 0x00000000, 0x00000000, 0 },
+    { 0x00000007, 0x00000001, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000b, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000000, UINT32_MAX, 0x00000007, 0x00000340, 0x00000340, 0x00000000, 0 },
+    { 0x0000000d, 0x00000001, UINT32_MAX, 0x0000000f, 0x00000340, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000002, UINT32_MAX, 0x00000100, 0x00000240, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000003, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x8000001f, 0x6f677948, 0x656e6975, 0x6e65476e, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00900f01, 0x40000000, 0x35c233ff, 0x2fd3fbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x80000002, 0x00000000, 0x00000000, 0x6f677948, 0x3843206e, 0x31372036, 0x33203538, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x6f632d32, 0x50206572, 0x65636f72, 0x726f7373, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x00202020, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0xff40ff40, 0xff40ff40, 0x20080140, 0x40040140, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x36006400, 0x56006400, 0x02006140, 0x0200c140, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x0000001b, 0x00000000, 0x00006599, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003030, 0x00001007, 0x0000603f, 0x00000000, 0 },
+    { 0x80000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000a, 0x00000000, 0x00000000, 0x00000001, 0x00008000, 0x00000000, 0x0001bcff, 0 },
+    { 0x8000000b, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000d, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000e, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000f, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000010, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000011, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000012, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000013, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000014, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000015, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000016, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000017, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000018, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000019, 0x00000000, 0x00000000, 0xf040f040, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001a, 0x00000000, 0x00000000, 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001b, 0x00000000, 0x00000000, 0x000003ff, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001d, 0x00000000, UINT32_MAX, 0x00004121, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x8000001d, 0x00000001, UINT32_MAX, 0x00004122, 0x00c0003f, 0x000000ff, 0x00000000, 0 },
+    { 0x8000001d, 0x00000002, UINT32_MAX, 0x00004143, 0x01c0003f, 0x000003ff, 0x00000002, 0 },
+    { 0x8000001d, 0x00000003, UINT32_MAX, 0x0001c163, 0x03c0003f, 0x00001fff, 0x00000001, 0 },
+    { 0x8000001d, 0x00000004, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001e, 0x00000000, 0x00000000, 0x0000003b, 0x0000011d, 0x00000303, 0x00000000, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x8000001f, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Hygon C86 7185 32-core Processor.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Hygon_C86_7185_32_core[] =
+{
+    MAL(0x00000000, "IA32_P5_MC_ADDR", 0x00000402),
+    MAL(0x00000001, "IA32_P5_MC_TYPE", 0x00000401),
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* Villain? value=0x1284`714a5328 */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0, UINT64_C(0xffff0000000006ff)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFN(0x00000049, "MSR_LASTBRANCH_9", WriteOnly, IgnoreWrite),
+    MFO(0x0000008b, "AMD_K8_PATCH_LEVEL", AmdK8PatchLevel), /* value=0x900006 */
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf), /* value=0x11c5`3efacda4 */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x2e`1e9ecd0c */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x508, 0, 0), /* value=0x508 */
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x10 */
+    MFX(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x9e578a50 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x117, 0, 0), /* value=0x117 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, UINT64_C(0xfffffffffffffff8), 0), /* value=0x0 */
+    MFX(0x0000017b, "IA32_MCG_CTL", Ia32McgCtl, Ia32McgCtl, 0, 0x10, 0), /* value=0xffffffff`ffffffef */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, UINT64_C(0xffffffffffffff80), 0x40), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0x0 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0x0 */
+    MFO(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp), /* value=0x0 */
+    MFO(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp), /* value=0x0 */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xffff000000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`80000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xffff000000000ff8)), /* value=0x7c000000 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`fc000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    RFN(0x00000400, 0x0000041b, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MVX(0x00000da0, "TODO_0000_0da0", 0, 0, UINT64_MAX),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0xfe, UINT64_C(0xffffffffffff0200)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* Might bite. value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* Might bite. value=0xffffffff`9e574c70 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* Might bite. value=0xffffffff`9e578de0 */
+    MFN(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask), /* Might bite. value=0x43700 */
+    MVO(0xc00000e7, "TODO_c000_00e7", UINT64_C(0x11c7b9402a58)),
+    MVO(0xc00000e8, "TODO_c000_00e8", UINT64_C(0x2f70233a93)),
+    MVO(0xc00000e9, "TODO_c000_00e9", 0),
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* Might bite. value=0x7f4f`f293d700 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* Might bite. value=0xffff8a48`dd400000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* Might bite. value=0x0 */
+    MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0000104, "AMD_15H_TSC_RATE", AmdFam15hTscRate, AmdFam15hTscRate, 0, 0, UINT64_C(0xffffff0000000000)), /* value=0x1`00000000 */
+    RVI(0xc0000408, 0xc000040f, "AMD_10H_MC4_MISCn", 0),
+    MVX(0xc0000410, "TODO_c000_0410", 0x100102a, UINT64_C(0xfffffffffe000f01), 0),
+    MVX(0xc0002000, "TODO_c000_2000", 0x1fffff, UINT64_C(0xffffffffffe00000), 0),
+    MVX(0xc0002001, "TODO_c000_2001", 0, 0, UINT64_MAX),
+    MVX(0xc0002002, "TODO_c000_2002", 0, 0, 0),
+    MVX(0xc0002003, "TODO_c000_2003", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002004, "TODO_c000_2004", UINT64_C(0x2300000035), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002005, "TODO_c000_2005", UINT64_C(0xb000000000), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002006, "TODO_c000_2006", 0, UINT64_C(0xffffff8000000000), 0),
+    MVI(0xc0002007, "TODO_c000_2007", 0),
+    MVX(0xc0002008, "TODO_c000_2008", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc0002009, "TODO_c000_2009", 0, 0, 0),
+    MVI(0xc000200a, "TODO_c000_200a", 0),
+    MVI(0xc000200b, "TODO_c000_200b", 0),
+    MVI(0xc000200c, "TODO_c000_200c", 0),
+    MVI(0xc000200d, "TODO_c000_200d", 0),
+    MVI(0xc000200e, "TODO_c000_200e", 0),
+    MVI(0xc000200f, "TODO_c000_200f", 0),
+    MVX(0xc0002010, "TODO_c000_2010", 0x3fff, UINT64_C(0xffffffffffffc000), 0),
+    MVX(0xc0002011, "TODO_c000_2011", 0, 0, UINT64_MAX),
+    MVX(0xc0002012, "TODO_c000_2012", 0, 0, 0),
+    MVX(0xc0002013, "TODO_c000_2013", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002014, "TODO_c000_2014", UINT64_C(0x2300000031), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002015, "TODO_c000_2015", UINT64_C(0x100b000000000), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002016, "TODO_c000_2016", 0, UINT64_C(0xfffffffe00000000), 0),
+    MVI(0xc0002017, "TODO_c000_2017", 0),
+    MVI(0xc0002018, "TODO_c000_2018", 0),
+    MVI(0xc0002019, "TODO_c000_2019", 0),
+    MVI(0xc000201a, "TODO_c000_201a", 0),
+    MVI(0xc000201b, "TODO_c000_201b", 0),
+    MVI(0xc000201c, "TODO_c000_201c", 0),
+    MVI(0xc000201d, "TODO_c000_201d", 0),
+    MVI(0xc000201e, "TODO_c000_201e", 0),
+    MVI(0xc000201f, "TODO_c000_201f", 0),
+    MVX(0xc0002020, "TODO_c000_2020", 0xf, UINT64_C(0xfffffffffffffff0), 0),
+    MVX(0xc0002021, "TODO_c000_2021", 0, 0, UINT64_MAX),
+    MVX(0xc0002022, "TODO_c000_2022", 0, 0, 0),
+    MVX(0xc0002023, "TODO_c000_2023", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002024, "TODO_c000_2024", UINT64_C(0x2100000037), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002025, "TODO_c000_2025", UINT64_C(0x200b000000000), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002026, "TODO_c000_2026", 0, UINT64_C(0xfffe000000000000), 0),
+    MVI(0xc0002027, "TODO_c000_2027", 0),
+    MVX(0xc0002028, "TODO_c000_2028", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc0002029, "TODO_c000_2029", 0, 0, 0),
+    MVI(0xc000202a, "TODO_c000_202a", 0),
+    MVI(0xc000202b, "TODO_c000_202b", 0),
+    MVI(0xc000202c, "TODO_c000_202c", 0),
+    MVI(0xc000202d, "TODO_c000_202d", 0),
+    MVI(0xc000202e, "TODO_c000_202e", 0),
+    MVI(0xc000202f, "TODO_c000_202f", 0),
+    MVX(0xc0002030, "TODO_c000_2030", 0x1ff, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0xc0002031, "TODO_c000_2031", 0, 0, UINT64_MAX),
+    MVX(0xc0002032, "TODO_c000_2032", 0, 0, 0),
+    MVX(0xc0002033, "TODO_c000_2033", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002034, "TODO_c000_2034", UINT64_C(0x2300000031), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002035, "TODO_c000_2035", UINT64_C(0x300b000000000), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002036, "TODO_c000_2036", 0, UINT64_C(0xfffffffe00000000), 0),
+    MVI(0xc0002037, "TODO_c000_2037", 0),
+    MVI(0xc0002038, "TODO_c000_2038", 0),
+    MVI(0xc0002039, "TODO_c000_2039", 0),
+    MVI(0xc000203a, "TODO_c000_203a", 0),
+    MVI(0xc000203b, "TODO_c000_203b", 0),
+    MVI(0xc000203c, "TODO_c000_203c", 0),
+    MVI(0xc000203d, "TODO_c000_203d", 0),
+    MVI(0xc000203e, "TODO_c000_203e", 0),
+    MVI(0xc000203f, "TODO_c000_203f", 0),
+    MVI(0xc0002040, "TODO_c000_2040", 0),
+    MVX(0xc0002041, "TODO_c000_2041", 0, 0, UINT64_MAX),
+    MVI(0xc0002042, "TODO_c000_2042", 0),
+    MVI(0xc0002043, "TODO_c000_2043", 0),
+    MVI(0xc0002044, "TODO_c000_2044", 0),
+    MVI(0xc0002045, "TODO_c000_2045", 0),
+    MVI(0xc0002046, "TODO_c000_2046", 0),
+    MVI(0xc0002047, "TODO_c000_2047", 0),
+    MVI(0xc0002048, "TODO_c000_2048", 0),
+    MVI(0xc0002049, "TODO_c000_2049", 0),
+    MVI(0xc000204a, "TODO_c000_204a", 0),
+    MVI(0xc000204b, "TODO_c000_204b", 0),
+    MVI(0xc000204c, "TODO_c000_204c", 0),
+    MVI(0xc000204d, "TODO_c000_204d", 0),
+    MVI(0xc000204e, "TODO_c000_204e", 0),
+    MVI(0xc000204f, "TODO_c000_204f", 0),
+    MVX(0xc0002050, "TODO_c000_2050", 0x7ff, UINT64_C(0xfffffffffffff800), 0),
+    MVX(0xc0002051, "TODO_c000_2051", 0, 0, UINT64_MAX),
+    MVX(0xc0002052, "TODO_c000_2052", 0, 0, 0),
+    MVX(0xc0002053, "TODO_c000_2053", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002054, "TODO_c000_2054", UINT64_C(0x2300000031), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002055, "TODO_c000_2055", UINT64_C(0x500b000000000), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002056, "TODO_c000_2056", 0, UINT64_C(0xfffffffe00000000), 0),
+    MVI(0xc0002057, "TODO_c000_2057", 0),
+    MVI(0xc0002058, "TODO_c000_2058", 0),
+    MVI(0xc0002059, "TODO_c000_2059", 0),
+    MVI(0xc000205a, "TODO_c000_205a", 0),
+    MVI(0xc000205b, "TODO_c000_205b", 0),
+    MVI(0xc000205c, "TODO_c000_205c", 0),
+    MVI(0xc000205d, "TODO_c000_205d", 0),
+    MVI(0xc000205e, "TODO_c000_205e", 0),
+    MVI(0xc000205f, "TODO_c000_205f", 0),
+    MVX(0xc0002060, "TODO_c000_2060", 0x7f, UINT64_C(0xffffffffffffff80), 0),
+    MVX(0xc0002061, "TODO_c000_2061", 0, 0, UINT64_MAX),
+    MVI(0xc0002062, "TODO_c000_2062", 0),
+    MVX(0xc0002063, "TODO_c000_2063", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002064, "TODO_c000_2064", UINT64_C(0x2300000031), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002065, "TODO_c000_2065", UINT64_C(0x600b000000000), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002066, "TODO_c000_2066", 0, UINT64_C(0xfffffffe00000000), 0),
+    MVI(0xc0002067, "TODO_c000_2067", 0),
+    MVI(0xc0002068, "TODO_c000_2068", 0),
+    MVI(0xc0002069, "TODO_c000_2069", 0),
+    MVI(0xc000206a, "TODO_c000_206a", 0),
+    MVI(0xc000206b, "TODO_c000_206b", 0),
+    MVI(0xc000206c, "TODO_c000_206c", 0),
+    MVI(0xc000206d, "TODO_c000_206d", 0),
+    MVI(0xc000206e, "TODO_c000_206e", 0),
+    MVI(0xc000206f, "TODO_c000_206f", 0),
+    MVX(0xc0002070, "TODO_c000_2070", 0xff, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc0002071, "TODO_c000_2071", 0, 0, UINT64_MAX),
+    MVX(0xc0002072, "TODO_c000_2072", 0, 0, 0),
+    MVX(0xc0002073, "TODO_c000_2073", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002074, "TODO_c000_2074", UINT64_C(0x2100000037), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002075, "TODO_c000_2075", UINT64_C(0x700b018090100), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002076, "TODO_c000_2076", 0, UINT64_C(0xfffe000000000000), 0),
+    MVI(0xc0002077, "TODO_c000_2077", 0),
+    MVX(0xc0002078, "TODO_c000_2078", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc0002079, "TODO_c000_2079", 0, 0, 0),
+    MVI(0xc000207a, "TODO_c000_207a", 0),
+    MVI(0xc000207b, "TODO_c000_207b", 0),
+    MVI(0xc000207c, "TODO_c000_207c", 0),
+    MVI(0xc000207d, "TODO_c000_207d", 0),
+    MVI(0xc000207e, "TODO_c000_207e", 0),
+    MVI(0xc000207f, "TODO_c000_207f", 0),
+    MVX(0xc0002080, "TODO_c000_2080", 0xff, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc0002081, "TODO_c000_2081", 0, 0, UINT64_MAX),
+    MVX(0xc0002082, "TODO_c000_2082", 0, 0, 0),
+    MVX(0xc0002083, "TODO_c000_2083", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002084, "TODO_c000_2084", UINT64_C(0x2100000037), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002085, "TODO_c000_2085", UINT64_C(0x700b018090200), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002086, "TODO_c000_2086", 0, UINT64_C(0xfffe000000000000), 0),
+    MVI(0xc0002087, "TODO_c000_2087", 0),
+    MVX(0xc0002088, "TODO_c000_2088", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc0002089, "TODO_c000_2089", 0, 0, 0),
+    MVI(0xc000208a, "TODO_c000_208a", 0),
+    MVI(0xc000208b, "TODO_c000_208b", 0),
+    MVI(0xc000208c, "TODO_c000_208c", 0),
+    MVI(0xc000208d, "TODO_c000_208d", 0),
+    MVI(0xc000208e, "TODO_c000_208e", 0),
+    MVI(0xc000208f, "TODO_c000_208f", 0),
+    MVX(0xc0002090, "TODO_c000_2090", 0xff, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc0002091, "TODO_c000_2091", 0, 0, UINT64_MAX),
+    MVX(0xc0002092, "TODO_c000_2092", 0, 0, 0),
+    MVX(0xc0002093, "TODO_c000_2093", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002094, "TODO_c000_2094", UINT64_C(0x2100000037), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002095, "TODO_c000_2095", UINT64_C(0x700b018090300), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002096, "TODO_c000_2096", 0, UINT64_C(0xfffe000000000000), 0),
+    MVI(0xc0002097, "TODO_c000_2097", 0),
+    MVX(0xc0002098, "TODO_c000_2098", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc0002099, "TODO_c000_2099", 0, 0, 0),
+    MVI(0xc000209a, "TODO_c000_209a", 0),
+    MVI(0xc000209b, "TODO_c000_209b", 0),
+    MVI(0xc000209c, "TODO_c000_209c", 0),
+    MVI(0xc000209d, "TODO_c000_209d", 0),
+    MVI(0xc000209e, "TODO_c000_209e", 0),
+    MVI(0xc000209f, "TODO_c000_209f", 0),
+    MVX(0xc00020a0, "TODO_c000_20a0", 0xff, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc00020a1, "TODO_c000_20a1", 0, 0, UINT64_MAX),
+    MVX(0xc00020a2, "TODO_c000_20a2", 0, 0, 0),
+    MVX(0xc00020a3, "TODO_c000_20a3", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc00020a4, "TODO_c000_20a4", UINT64_C(0x2100000037), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc00020a5, "TODO_c000_20a5", UINT64_C(0x700b018090400), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc00020a6, "TODO_c000_20a6", 0, UINT64_C(0xfffe000000000000), 0),
+    MVI(0xc00020a7, "TODO_c000_20a7", 0),
+    MVX(0xc00020a8, "TODO_c000_20a8", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc00020a9, "TODO_c000_20a9", 0, 0, 0),
+    MVI(0xc00020aa, "TODO_c000_20aa", 0),
+    MVI(0xc00020ab, "TODO_c000_20ab", 0),
+    MVI(0xc00020ac, "TODO_c000_20ac", 0),
+    MVI(0xc00020ad, "TODO_c000_20ad", 0),
+    MVI(0xc00020ae, "TODO_c000_20ae", 0),
+    MVI(0xc00020af, "TODO_c000_20af", 0),
+    MVX(0xc00020b0, "TODO_c000_20b0", 0xff, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc00020b1, "TODO_c000_20b1", 0, 0, UINT64_MAX),
+    MVX(0xc00020b2, "TODO_c000_20b2", 0, 0, 0),
+    MVX(0xc00020b3, "TODO_c000_20b3", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc00020b4, "TODO_c000_20b4", UINT64_C(0x2100000037), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc00020b5, "TODO_c000_20b5", UINT64_C(0x700b018490100), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc00020b6, "TODO_c000_20b6", 0, UINT64_C(0xfffe000000000000), 0),
+    MVI(0xc00020b7, "TODO_c000_20b7", 0),
+    MVX(0xc00020b8, "TODO_c000_20b8", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc00020b9, "TODO_c000_20b9", 0, 0, 0),
+    MVI(0xc00020ba, "TODO_c000_20ba", 0),
+    MVI(0xc00020bb, "TODO_c000_20bb", 0),
+    MVI(0xc00020bc, "TODO_c000_20bc", 0),
+    MVI(0xc00020bd, "TODO_c000_20bd", 0),
+    MVI(0xc00020be, "TODO_c000_20be", 0),
+    MVI(0xc00020bf, "TODO_c000_20bf", 0),
+    MVX(0xc00020c0, "TODO_c000_20c0", 0xff, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc00020c1, "TODO_c000_20c1", 0, 0, UINT64_MAX),
+    MVX(0xc00020c2, "TODO_c000_20c2", 0, 0, 0),
+    MVX(0xc00020c3, "TODO_c000_20c3", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc00020c4, "TODO_c000_20c4", UINT64_C(0x2100000037), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc00020c5, "TODO_c000_20c5", UINT64_C(0x700b018490200), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc00020c6, "TODO_c000_20c6", 0, UINT64_C(0xfffe000000000000), 0),
+    MVI(0xc00020c7, "TODO_c000_20c7", 0),
+    MVX(0xc00020c8, "TODO_c000_20c8", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc00020c9, "TODO_c000_20c9", 0, 0, 0),
+    MVI(0xc00020ca, "TODO_c000_20ca", 0),
+    MVI(0xc00020cb, "TODO_c000_20cb", 0),
+    MVI(0xc00020cc, "TODO_c000_20cc", 0),
+    MVI(0xc00020cd, "TODO_c000_20cd", 0),
+    MVI(0xc00020ce, "TODO_c000_20ce", 0),
+    MVI(0xc00020cf, "TODO_c000_20cf", 0),
+    MVX(0xc00020d0, "TODO_c000_20d0", 0xff, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc00020d1, "TODO_c000_20d1", 0, 0, UINT64_MAX),
+    MVX(0xc00020d2, "TODO_c000_20d2", 0, 0, 0),
+    MVX(0xc00020d3, "TODO_c000_20d3", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc00020d4, "TODO_c000_20d4", UINT64_C(0x2100000037), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc00020d5, "TODO_c000_20d5", UINT64_C(0x700b018490300), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc00020d6, "TODO_c000_20d6", 0, UINT64_C(0xfffe000000000000), 0),
+    MVI(0xc00020d7, "TODO_c000_20d7", 0),
+    MVX(0xc00020d8, "TODO_c000_20d8", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc00020d9, "TODO_c000_20d9", 0, 0, 0),
+    MVI(0xc00020da, "TODO_c000_20da", 0),
+    MVI(0xc00020db, "TODO_c000_20db", 0),
+    MVI(0xc00020dc, "TODO_c000_20dc", 0),
+    MVI(0xc00020dd, "TODO_c000_20dd", 0),
+    MVI(0xc00020de, "TODO_c000_20de", 0),
+    MVI(0xc00020df, "TODO_c000_20df", 0),
+    MVX(0xc00020e0, "TODO_c000_20e0", 0xff, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc00020e1, "TODO_c000_20e1", 0, 0, UINT64_MAX),
+    MVX(0xc00020e2, "TODO_c000_20e2", 0, 0, 0),
+    MVX(0xc00020e3, "TODO_c000_20e3", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc00020e4, "TODO_c000_20e4", UINT64_C(0x2100000037), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc00020e5, "TODO_c000_20e5", UINT64_C(0x700b018490400), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc00020e6, "TODO_c000_20e6", 0, UINT64_C(0xfffe000000000000), 0),
+    MVI(0xc00020e7, "TODO_c000_20e7", 0),
+    MVX(0xc00020e8, "TODO_c000_20e8", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc00020e9, "TODO_c000_20e9", 0, 0, 0),
+    MVI(0xc00020ea, "TODO_c000_20ea", 0),
+    MVI(0xc00020eb, "TODO_c000_20eb", 0),
+    MVI(0xc00020ec, "TODO_c000_20ec", 0),
+    MVI(0xc00020ed, "TODO_c000_20ed", 0),
+    MVI(0xc00020ee, "TODO_c000_20ee", 0),
+    MVI(0xc00020ef, "TODO_c000_20ef", 0),
+    MVX(0xc00020f0, "TODO_c000_20f0", 0x3f, UINT64_C(0xffffffffffffffc0), 0),
+    MVX(0xc00020f1, "TODO_c000_20f1", 0, 0, UINT64_MAX),
+    MVX(0xc00020f2, "TODO_c000_20f2", 0, 0, 0),
+    MVX(0xc00020f3, "TODO_c000_20f3", UINT64_C(0xd01a000001000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc00020f4, "TODO_c000_20f4", UINT64_C(0x2300000035), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc00020f5, "TODO_c000_20f5", UINT64_C(0x9600050f00), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc00020f6, "TODO_c000_20f6", 0, UINT64_C(0xffff000000000000), 0),
+    MVI(0xc00020f7, "TODO_c000_20f7", 0),
+    MVX(0xc00020f8, "TODO_c000_20f8", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc00020f9, "TODO_c000_20f9", 0, 0, 0),
+    MVX(0xc00020fa, "TODO_c000_20fa", UINT64_C(0xd00a000000000000), UINT64_C(0xff0f00000ffffff), 0),
+    MVI(0xc00020fb, "TODO_c000_20fb", 0),
+    MVI(0xc00020fc, "TODO_c000_20fc", 0),
+    MVI(0xc00020fd, "TODO_c000_20fd", 0),
+    MVI(0xc00020fe, "TODO_c000_20fe", 0),
+    MVI(0xc00020ff, "TODO_c000_20ff", 0),
+    MVX(0xc0002100, "TODO_c000_2100", 0x3f, UINT64_C(0xffffffffffffffc0), 0),
+    MVX(0xc0002101, "TODO_c000_2101", 0, 0, UINT64_MAX),
+    MVX(0xc0002102, "TODO_c000_2102", 0, 0, 0),
+    MVX(0xc0002103, "TODO_c000_2103", UINT64_C(0xd01a000001000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002104, "TODO_c000_2104", UINT64_C(0x2300000035), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002105, "TODO_c000_2105", UINT64_C(0x9600150f00), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002106, "TODO_c000_2106", 0, UINT64_C(0xffff000000000000), 0),
+    MVI(0xc0002107, "TODO_c000_2107", 0),
+    MVX(0xc0002108, "TODO_c000_2108", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc0002109, "TODO_c000_2109", 0, 0, 0),
+    MVX(0xc000210a, "TODO_c000_210a", UINT64_C(0xd00a000000000000), UINT64_C(0xff0f00000ffffff), 0),
+    MVI(0xc000210b, "TODO_c000_210b", 0),
+    MVI(0xc000210c, "TODO_c000_210c", 0),
+    MVI(0xc000210d, "TODO_c000_210d", 0),
+    MVI(0xc000210e, "TODO_c000_210e", 0),
+    MVI(0xc000210f, "TODO_c000_210f", 0),
+    MVX(0xc0002110, "TODO_c000_2110", 0x1, UINT64_C(0xfffffffffffffffe), 0),
+    MVX(0xc0002111, "TODO_c000_2111", 0, 0, UINT64_MAX),
+    MVI(0xc0002112, "TODO_c000_2112", 0),
+    MVX(0xc0002113, "TODO_c000_2113", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002114, "TODO_c000_2114", UINT64_C(0x2300000031), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002115, "TODO_c000_2115", UINT64_C(0x103b30400), ~(uint64_t)UINT32_MAX, 0),
+    MVI(0xc0002116, "TODO_c000_2116", 0),
+    MVI(0xc0002117, "TODO_c000_2117", 0),
+    MVI(0xc0002118, "TODO_c000_2118", 0),
+    MVI(0xc0002119, "TODO_c000_2119", 0),
+    MVI(0xc000211a, "TODO_c000_211a", 0),
+    MVI(0xc000211b, "TODO_c000_211b", 0),
+    MVI(0xc000211c, "TODO_c000_211c", 0),
+    MVI(0xc000211d, "TODO_c000_211d", 0),
+    MVI(0xc000211e, "TODO_c000_211e", 0),
+    MVI(0xc000211f, "TODO_c000_211f", 0),
+    MVX(0xc0002120, "TODO_c000_2120", 0x1, UINT64_C(0xfffffffffffffffe), 0),
+    MVX(0xc0002121, "TODO_c000_2121", 0, 0, UINT64_MAX),
+    MVI(0xc0002122, "TODO_c000_2122", 0),
+    MVX(0xc0002123, "TODO_c000_2123", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002124, "TODO_c000_2124", UINT64_C(0x2300000031), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002125, "TODO_c000_2125", UINT64_C(0xff03830400), ~(uint64_t)UINT32_MAX, 0),
+    MVI(0xc0002126, "TODO_c000_2126", 0),
+    MVI(0xc0002127, "TODO_c000_2127", 0),
+    MVI(0xc0002128, "TODO_c000_2128", 0),
+    MVI(0xc0002129, "TODO_c000_2129", 0),
+    MVI(0xc000212a, "TODO_c000_212a", 0),
+    MVI(0xc000212b, "TODO_c000_212b", 0),
+    MVI(0xc000212c, "TODO_c000_212c", 0),
+    MVI(0xc000212d, "TODO_c000_212d", 0),
+    MVI(0xc000212e, "TODO_c000_212e", 0),
+    MVI(0xc000212f, "TODO_c000_212f", 0),
+    MVX(0xc0002130, "TODO_c000_2130", 0x1, UINT64_C(0xfffffffffffffffe), 0),
+    MVX(0xc0002131, "TODO_c000_2131", 0, 0, UINT64_MAX),
+    MVI(0xc0002132, "TODO_c000_2132", 0),
+    MVX(0xc0002133, "TODO_c000_2133", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002134, "TODO_c000_2134", UINT64_C(0x2300000031), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002135, "TODO_c000_2135", UINT64_C(0x500000000), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002136, "TODO_c000_2136", 0, UINT64_C(0xfffffffe00000000), 0),
+    MVI(0xc0002137, "TODO_c000_2137", 0),
+    MVI(0xc0002138, "TODO_c000_2138", 0),
+    MVI(0xc0002139, "TODO_c000_2139", 0),
+    MVI(0xc000213a, "TODO_c000_213a", 0),
+    MVI(0xc000213b, "TODO_c000_213b", 0),
+    MVI(0xc000213c, "TODO_c000_213c", 0),
+    MVI(0xc000213d, "TODO_c000_213d", 0),
+    MVI(0xc000213e, "TODO_c000_213e", 0),
+    MVI(0xc000213f, "TODO_c000_213f", 0),
+    MVX(0xc0002140, "TODO_c000_2140", 0x1ff, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0xc0002141, "TODO_c000_2141", 0, 0, UINT64_MAX),
+    MVX(0xc0002142, "TODO_c000_2142", 0, 0, 0),
+    MVX(0xc0002143, "TODO_c000_2143", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002144, "TODO_c000_2144", UINT64_C(0x2100000037), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002145, "TODO_c000_2145", UINT64_C(0x2e00000000), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002146, "TODO_c000_2146", 0, UINT64_C(0xfffffe0000000000), 0),
+    MVI(0xc0002147, "TODO_c000_2147", 0),
+    MVX(0xc0002148, "TODO_c000_2148", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc0002149, "TODO_c000_2149", 0, 0, 0),
+    MVI(0xc000214a, "TODO_c000_214a", 0),
+    MVI(0xc000214b, "TODO_c000_214b", 0),
+    MVI(0xc000214c, "TODO_c000_214c", 0),
+    MVI(0xc000214d, "TODO_c000_214d", 0),
+    MVI(0xc000214e, "TODO_c000_214e", 0),
+    MVI(0xc000214f, "TODO_c000_214f", 0),
+    MVX(0xc0002150, "TODO_c000_2150", 0x1ff, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0xc0002151, "TODO_c000_2151", 0, 0, UINT64_MAX),
+    MVX(0xc0002152, "TODO_c000_2152", 0, 0, 0),
+    MVX(0xc0002153, "TODO_c000_2153", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002154, "TODO_c000_2154", UINT64_C(0x2100000037), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002155, "TODO_c000_2155", UINT64_C(0x2e00000001), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002156, "TODO_c000_2156", 0, UINT64_C(0xfffffe0000000000), 0),
+    MVI(0xc0002157, "TODO_c000_2157", 0),
+    MVX(0xc0002158, "TODO_c000_2158", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVX(0xc0002159, "TODO_c000_2159", 0, 0, 0),
+    MVI(0xc000215a, "TODO_c000_215a", 0),
+    MVI(0xc000215b, "TODO_c000_215b", 0),
+    MVI(0xc000215c, "TODO_c000_215c", 0),
+    MVI(0xc000215d, "TODO_c000_215d", 0),
+    MVI(0xc000215e, "TODO_c000_215e", 0),
+    MVI(0xc000215f, "TODO_c000_215f", 0),
+    MVX(0xc0002160, "TODO_c000_2160", 0xf, UINT64_C(0xfffffffffffffff0), 0),
+    MVX(0xc0002161, "TODO_c000_2161", 0, 0, UINT64_MAX),
+    MVI(0xc0002162, "TODO_c000_2162", 0),
+    MVX(0xc0002163, "TODO_c000_2163", UINT64_C(0xd01a000000000000), UINT64_C(0xef00f00000ffffff), 0),
+    MVX(0xc0002164, "TODO_c000_2164", UINT64_C(0x2300000035), UINT64_C(0xffffff80ffffffff), 0),
+    MVX(0xc0002165, "TODO_c000_2165", UINT64_C(0x1002e00000002), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0002166, "TODO_c000_2166", 0, UINT64_C(0xfffffffe00000000), 0),
+    MVI(0xc0002167, "TODO_c000_2167", 0),
+    MVX(0xc0002168, "TODO_c000_2168", 0, UINT64_C(0x3bdfefffffffffff), 0),
+    MVI(0xc0002169, "TODO_c000_2169", 0),
+    MVI(0xc000216a, "TODO_c000_216a", 0),
+    MVI(0xc000216b, "TODO_c000_216b", 0),
+    MVI(0xc000216c, "TODO_c000_216c", 0),
+    MVI(0xc000216d, "TODO_c000_216d", 0),
+    MVI(0xc000216e, "TODO_c000_216e", 0),
+    MVI(0xc000216f, "TODO_c000_216f", 0),
+    MVI(0xc0002170, "TODO_c000_2170", 0),
+    MVI(0xc0002171, "TODO_c000_2171", 0),
+    MVI(0xc0002172, "TODO_c000_2172", 0),
+    MVI(0xc0002173, "TODO_c000_2173", 0),
+    MVI(0xc0002174, "TODO_c000_2174", 0),
+    MVI(0xc0002175, "TODO_c000_2175", 0),
+    MVI(0xc0002176, "TODO_c000_2176", 0),
+    MVI(0xc0002177, "TODO_c000_2177", 0),
+    MVI(0xc0002178, "TODO_c000_2178", 0),
+    MVI(0xc0002179, "TODO_c000_2179", 0),
+    MVI(0xc000217a, "TODO_c000_217a", 0),
+    MVI(0xc000217b, "TODO_c000_217b", 0),
+    MVI(0xc000217c, "TODO_c000_217c", 0),
+    MVI(0xc000217d, "TODO_c000_217d", 0),
+    MVI(0xc000217e, "TODO_c000_217e", 0),
+    MVI(0xc000217f, "TODO_c000_217f", 0),
+    MVI(0xc0002180, "TODO_c000_2180", 0),
+    MVI(0xc0002181, "TODO_c000_2181", 0),
+    MVI(0xc0002182, "TODO_c000_2182", 0),
+    MVI(0xc0002183, "TODO_c000_2183", 0),
+    MVI(0xc0002184, "TODO_c000_2184", 0),
+    MVI(0xc0002185, "TODO_c000_2185", 0),
+    MVI(0xc0002186, "TODO_c000_2186", 0),
+    MVI(0xc0002187, "TODO_c000_2187", 0),
+    MVI(0xc0002188, "TODO_c000_2188", 0),
+    MVI(0xc0002189, "TODO_c000_2189", 0),
+    MVI(0xc000218a, "TODO_c000_218a", 0),
+    MVI(0xc000218b, "TODO_c000_218b", 0),
+    MVI(0xc000218c, "TODO_c000_218c", 0),
+    MVI(0xc000218d, "TODO_c000_218d", 0),
+    MVI(0xc000218e, "TODO_c000_218e", 0),
+    MVI(0xc000218f, "TODO_c000_218f", 0),
+    MVI(0xc0002190, "TODO_c000_2190", 0),
+    MVI(0xc0002191, "TODO_c000_2191", 0),
+    MVI(0xc0002192, "TODO_c000_2192", 0),
+    MVI(0xc0002193, "TODO_c000_2193", 0),
+    MVI(0xc0002194, "TODO_c000_2194", 0),
+    MVI(0xc0002195, "TODO_c000_2195", 0),
+    MVI(0xc0002196, "TODO_c000_2196", 0),
+    MVI(0xc0002197, "TODO_c000_2197", 0),
+    MVI(0xc0002198, "TODO_c000_2198", 0),
+    MVI(0xc0002199, "TODO_c000_2199", 0),
+    MVI(0xc000219a, "TODO_c000_219a", 0),
+    MVI(0xc000219b, "TODO_c000_219b", 0),
+    MVI(0xc000219c, "TODO_c000_219c", 0),
+    MVI(0xc000219d, "TODO_c000_219d", 0),
+    MVI(0xc000219e, "TODO_c000_219e", 0),
+    MVI(0xc000219f, "TODO_c000_219f", 0),
+    MVI(0xc00021a0, "TODO_c000_21a0", 0),
+    MVI(0xc00021a1, "TODO_c000_21a1", 0),
+    MVI(0xc00021a2, "TODO_c000_21a2", 0),
+    MVI(0xc00021a3, "TODO_c000_21a3", 0),
+    MVI(0xc00021a4, "TODO_c000_21a4", 0),
+    MVI(0xc00021a5, "TODO_c000_21a5", 0),
+    MVI(0xc00021a6, "TODO_c000_21a6", 0),
+    MVI(0xc00021a7, "TODO_c000_21a7", 0),
+    MVI(0xc00021a8, "TODO_c000_21a8", 0),
+    MVI(0xc00021a9, "TODO_c000_21a9", 0),
+    MVI(0xc00021aa, "TODO_c000_21aa", 0),
+    MVI(0xc00021ab, "TODO_c000_21ab", 0),
+    MVI(0xc00021ac, "TODO_c000_21ac", 0),
+    MVI(0xc00021ad, "TODO_c000_21ad", 0),
+    MVI(0xc00021ae, "TODO_c000_21ae", 0),
+    MVI(0xc00021af, "TODO_c000_21af", 0),
+    MVI(0xc00021b0, "TODO_c000_21b0", 0),
+    MVI(0xc00021b1, "TODO_c000_21b1", 0),
+    MVI(0xc00021b2, "TODO_c000_21b2", 0),
+    MVI(0xc00021b3, "TODO_c000_21b3", 0),
+    MVI(0xc00021b4, "TODO_c000_21b4", 0),
+    MVI(0xc00021b5, "TODO_c000_21b5", 0),
+    MVI(0xc00021b6, "TODO_c000_21b6", 0),
+    MVI(0xc00021b7, "TODO_c000_21b7", 0),
+    MVI(0xc00021b8, "TODO_c000_21b8", 0),
+    MVI(0xc00021b9, "TODO_c000_21b9", 0),
+    MVI(0xc00021ba, "TODO_c000_21ba", 0),
+    MVI(0xc00021bb, "TODO_c000_21bb", 0),
+    MVI(0xc00021bc, "TODO_c000_21bc", 0),
+    MVI(0xc00021bd, "TODO_c000_21bd", 0),
+    MVI(0xc00021be, "TODO_c000_21be", 0),
+    MVI(0xc00021bf, "TODO_c000_21bf", 0),
+    MVI(0xc00021c0, "TODO_c000_21c0", 0),
+    MVI(0xc00021c1, "TODO_c000_21c1", 0),
+    MVI(0xc00021c2, "TODO_c000_21c2", 0),
+    MVI(0xc00021c3, "TODO_c000_21c3", 0),
+    MVI(0xc00021c4, "TODO_c000_21c4", 0),
+    MVI(0xc00021c5, "TODO_c000_21c5", 0),
+    MVI(0xc00021c6, "TODO_c000_21c6", 0),
+    MVI(0xc00021c7, "TODO_c000_21c7", 0),
+    MVI(0xc00021c8, "TODO_c000_21c8", 0),
+    MVI(0xc00021c9, "TODO_c000_21c9", 0),
+    MVI(0xc00021ca, "TODO_c000_21ca", 0),
+    MVI(0xc00021cb, "TODO_c000_21cb", 0),
+    MVI(0xc00021cc, "TODO_c000_21cc", 0),
+    MVI(0xc00021cd, "TODO_c000_21cd", 0),
+    MVI(0xc00021ce, "TODO_c000_21ce", 0),
+    MVI(0xc00021cf, "TODO_c000_21cf", 0),
+    MVI(0xc00021d0, "TODO_c000_21d0", 0),
+    MVI(0xc00021d1, "TODO_c000_21d1", 0),
+    MVI(0xc00021d2, "TODO_c000_21d2", 0),
+    MVI(0xc00021d3, "TODO_c000_21d3", 0),
+    MVI(0xc00021d4, "TODO_c000_21d4", 0),
+    MVI(0xc00021d5, "TODO_c000_21d5", 0),
+    MVI(0xc00021d6, "TODO_c000_21d6", 0),
+    MVI(0xc00021d7, "TODO_c000_21d7", 0),
+    MVI(0xc00021d8, "TODO_c000_21d8", 0),
+    MVI(0xc00021d9, "TODO_c000_21d9", 0),
+    MVI(0xc00021da, "TODO_c000_21da", 0),
+    MVI(0xc00021db, "TODO_c000_21db", 0),
+    MVI(0xc00021dc, "TODO_c000_21dc", 0),
+    MVI(0xc00021dd, "TODO_c000_21dd", 0),
+    MVI(0xc00021de, "TODO_c000_21de", 0),
+    MVI(0xc00021df, "TODO_c000_21df", 0),
+    MVI(0xc00021e0, "TODO_c000_21e0", 0),
+    MVI(0xc00021e1, "TODO_c000_21e1", 0),
+    MVI(0xc00021e2, "TODO_c000_21e2", 0),
+    MVI(0xc00021e3, "TODO_c000_21e3", 0),
+    MVI(0xc00021e4, "TODO_c000_21e4", 0),
+    MVI(0xc00021e5, "TODO_c000_21e5", 0),
+    MVI(0xc00021e6, "TODO_c000_21e6", 0),
+    MVI(0xc00021e7, "TODO_c000_21e7", 0),
+    MVI(0xc00021e8, "TODO_c000_21e8", 0),
+    MVI(0xc00021e9, "TODO_c000_21e9", 0),
+    MVI(0xc00021ea, "TODO_c000_21ea", 0),
+    MVI(0xc00021eb, "TODO_c000_21eb", 0),
+    MVI(0xc00021ec, "TODO_c000_21ec", 0),
+    MVI(0xc00021ed, "TODO_c000_21ed", 0),
+    MVI(0xc00021ee, "TODO_c000_21ee", 0),
+    MVI(0xc00021ef, "TODO_c000_21ef", 0),
+    MVI(0xc00021f0, "TODO_c000_21f0", 0),
+    MVI(0xc00021f1, "TODO_c000_21f1", 0),
+    MVI(0xc00021f2, "TODO_c000_21f2", 0),
+    MVI(0xc00021f3, "TODO_c000_21f3", 0),
+    MVI(0xc00021f4, "TODO_c000_21f4", 0),
+    MVI(0xc00021f5, "TODO_c000_21f5", 0),
+    MVI(0xc00021f6, "TODO_c000_21f6", 0),
+    MVI(0xc00021f7, "TODO_c000_21f7", 0),
+    MVI(0xc00021f8, "TODO_c000_21f8", 0),
+    MVI(0xc00021f9, "TODO_c000_21f9", 0),
+    MVI(0xc00021fa, "TODO_c000_21fa", 0),
+    MVI(0xc00021fb, "TODO_c000_21fb", 0),
+    MVI(0xc00021fc, "TODO_c000_21fc", 0),
+    MVI(0xc00021fd, "TODO_c000_21fd", 0),
+    MVI(0xc00021fe, "TODO_c000_21fe", 0),
+    MVI(0xc00021ff, "TODO_c000_21ff", 0),
+    MVI(0xc0002200, "TODO_c000_2200", 0),
+    MVI(0xc0002201, "TODO_c000_2201", 0),
+    MVI(0xc0002202, "TODO_c000_2202", 0),
+    MVI(0xc0002203, "TODO_c000_2203", 0),
+    MVI(0xc0002204, "TODO_c000_2204", 0),
+    MVI(0xc0002205, "TODO_c000_2205", 0),
+    MVI(0xc0002206, "TODO_c000_2206", 0),
+    MVI(0xc0002207, "TODO_c000_2207", 0),
+    MVI(0xc0002208, "TODO_c000_2208", 0),
+    MVI(0xc0002209, "TODO_c000_2209", 0),
+    MVI(0xc000220a, "TODO_c000_220a", 0),
+    MVI(0xc000220b, "TODO_c000_220b", 0),
+    MVI(0xc000220c, "TODO_c000_220c", 0),
+    MVI(0xc000220d, "TODO_c000_220d", 0),
+    MVI(0xc000220e, "TODO_c000_220e", 0),
+    MVI(0xc000220f, "TODO_c000_220f", 0),
+    MVI(0xc0002210, "TODO_c000_2210", 0),
+    MVI(0xc0002211, "TODO_c000_2211", 0),
+    MVI(0xc0002212, "TODO_c000_2212", 0),
+    MVI(0xc0002213, "TODO_c000_2213", 0),
+    MVI(0xc0002214, "TODO_c000_2214", 0),
+    MVI(0xc0002215, "TODO_c000_2215", 0),
+    MVI(0xc0002216, "TODO_c000_2216", 0),
+    MVI(0xc0002217, "TODO_c000_2217", 0),
+    MVI(0xc0002218, "TODO_c000_2218", 0),
+    MVI(0xc0002219, "TODO_c000_2219", 0),
+    MVI(0xc000221a, "TODO_c000_221a", 0),
+    MVI(0xc000221b, "TODO_c000_221b", 0),
+    MVI(0xc000221c, "TODO_c000_221c", 0),
+    MVI(0xc000221d, "TODO_c000_221d", 0),
+    MVI(0xc000221e, "TODO_c000_221e", 0),
+    MVI(0xc000221f, "TODO_c000_221f", 0),
+    MVI(0xc0002220, "TODO_c000_2220", 0),
+    MVI(0xc0002221, "TODO_c000_2221", 0),
+    MVI(0xc0002222, "TODO_c000_2222", 0),
+    MVI(0xc0002223, "TODO_c000_2223", 0),
+    MVI(0xc0002224, "TODO_c000_2224", 0),
+    MVI(0xc0002225, "TODO_c000_2225", 0),
+    MVI(0xc0002226, "TODO_c000_2226", 0),
+    MVI(0xc0002227, "TODO_c000_2227", 0),
+    MVI(0xc0002228, "TODO_c000_2228", 0),
+    MVI(0xc0002229, "TODO_c000_2229", 0),
+    MVI(0xc000222a, "TODO_c000_222a", 0),
+    MVI(0xc000222b, "TODO_c000_222b", 0),
+    MVI(0xc000222c, "TODO_c000_222c", 0),
+    MVI(0xc000222d, "TODO_c000_222d", 0),
+    MVI(0xc000222e, "TODO_c000_222e", 0),
+    MVI(0xc000222f, "TODO_c000_222f", 0),
+    MVI(0xc0002230, "TODO_c000_2230", 0),
+    MVI(0xc0002231, "TODO_c000_2231", 0),
+    MVI(0xc0002232, "TODO_c000_2232", 0),
+    MVI(0xc0002233, "TODO_c000_2233", 0),
+    MVI(0xc0002234, "TODO_c000_2234", 0),
+    MVI(0xc0002235, "TODO_c000_2235", 0),
+    MVI(0xc0002236, "TODO_c000_2236", 0),
+    MVI(0xc0002237, "TODO_c000_2237", 0),
+    MVI(0xc0002238, "TODO_c000_2238", 0),
+    MVI(0xc0002239, "TODO_c000_2239", 0),
+    MVI(0xc000223a, "TODO_c000_223a", 0),
+    MVI(0xc000223b, "TODO_c000_223b", 0),
+    MVI(0xc000223c, "TODO_c000_223c", 0),
+    MVI(0xc000223d, "TODO_c000_223d", 0),
+    MVI(0xc000223e, "TODO_c000_223e", 0),
+    MVI(0xc000223f, "TODO_c000_223f", 0),
+    MVI(0xc0002240, "TODO_c000_2240", 0),
+    MVI(0xc0002241, "TODO_c000_2241", 0),
+    MVI(0xc0002242, "TODO_c000_2242", 0),
+    MVI(0xc0002243, "TODO_c000_2243", 0),
+    MVI(0xc0002244, "TODO_c000_2244", 0),
+    MVI(0xc0002245, "TODO_c000_2245", 0),
+    MVI(0xc0002246, "TODO_c000_2246", 0),
+    MVI(0xc0002247, "TODO_c000_2247", 0),
+    MVI(0xc0002248, "TODO_c000_2248", 0),
+    MVI(0xc0002249, "TODO_c000_2249", 0),
+    MVI(0xc000224a, "TODO_c000_224a", 0),
+    MVI(0xc000224b, "TODO_c000_224b", 0),
+    MVI(0xc000224c, "TODO_c000_224c", 0),
+    MVI(0xc000224d, "TODO_c000_224d", 0),
+    MVI(0xc000224e, "TODO_c000_224e", 0),
+    MVI(0xc000224f, "TODO_c000_224f", 0),
+    MVI(0xc0002250, "TODO_c000_2250", 0),
+    MVI(0xc0002251, "TODO_c000_2251", 0),
+    MVI(0xc0002252, "TODO_c000_2252", 0),
+    MVI(0xc0002253, "TODO_c000_2253", 0),
+    MVI(0xc0002254, "TODO_c000_2254", 0),
+    MVI(0xc0002255, "TODO_c000_2255", 0),
+    MVI(0xc0002256, "TODO_c000_2256", 0),
+    MVI(0xc0002257, "TODO_c000_2257", 0),
+    MVI(0xc0002258, "TODO_c000_2258", 0),
+    MVI(0xc0002259, "TODO_c000_2259", 0),
+    MVI(0xc000225a, "TODO_c000_225a", 0),
+    MVI(0xc000225b, "TODO_c000_225b", 0),
+    MVI(0xc000225c, "TODO_c000_225c", 0),
+    MVI(0xc000225d, "TODO_c000_225d", 0),
+    MVI(0xc000225e, "TODO_c000_225e", 0),
+    MVI(0xc000225f, "TODO_c000_225f", 0),
+    MVI(0xc0002260, "TODO_c000_2260", 0),
+    MVI(0xc0002261, "TODO_c000_2261", 0),
+    MVI(0xc0002262, "TODO_c000_2262", 0),
+    MVI(0xc0002263, "TODO_c000_2263", 0),
+    MVI(0xc0002264, "TODO_c000_2264", 0),
+    MVI(0xc0002265, "TODO_c000_2265", 0),
+    MVI(0xc0002266, "TODO_c000_2266", 0),
+    MVI(0xc0002267, "TODO_c000_2267", 0),
+    MVI(0xc0002268, "TODO_c000_2268", 0),
+    MVI(0xc0002269, "TODO_c000_2269", 0),
+    MVI(0xc000226a, "TODO_c000_226a", 0),
+    MVI(0xc000226b, "TODO_c000_226b", 0),
+    MVI(0xc000226c, "TODO_c000_226c", 0),
+    MVI(0xc000226d, "TODO_c000_226d", 0),
+    MVI(0xc000226e, "TODO_c000_226e", 0),
+    MVI(0xc000226f, "TODO_c000_226f", 0),
+    MVI(0xc0002270, "TODO_c000_2270", 0),
+    MVI(0xc0002271, "TODO_c000_2271", 0),
+    MVI(0xc0002272, "TODO_c000_2272", 0),
+    MVI(0xc0002273, "TODO_c000_2273", 0),
+    MVI(0xc0002274, "TODO_c000_2274", 0),
+    MVI(0xc0002275, "TODO_c000_2275", 0),
+    MVI(0xc0002276, "TODO_c000_2276", 0),
+    MVI(0xc0002277, "TODO_c000_2277", 0),
+    MVI(0xc0002278, "TODO_c000_2278", 0),
+    MVI(0xc0002279, "TODO_c000_2279", 0),
+    MVI(0xc000227a, "TODO_c000_227a", 0),
+    MVI(0xc000227b, "TODO_c000_227b", 0),
+    MVI(0xc000227c, "TODO_c000_227c", 0),
+    MVI(0xc000227d, "TODO_c000_227d", 0),
+    MVI(0xc000227e, "TODO_c000_227e", 0),
+    MVI(0xc000227f, "TODO_c000_227f", 0),
+    MVI(0xc0002280, "TODO_c000_2280", 0),
+    MVI(0xc0002281, "TODO_c000_2281", 0),
+    MVI(0xc0002282, "TODO_c000_2282", 0),
+    MVI(0xc0002283, "TODO_c000_2283", 0),
+    MVI(0xc0002284, "TODO_c000_2284", 0),
+    MVI(0xc0002285, "TODO_c000_2285", 0),
+    MVI(0xc0002286, "TODO_c000_2286", 0),
+    MVI(0xc0002287, "TODO_c000_2287", 0),
+    MVI(0xc0002288, "TODO_c000_2288", 0),
+    MVI(0xc0002289, "TODO_c000_2289", 0),
+    MVI(0xc000228a, "TODO_c000_228a", 0),
+    MVI(0xc000228b, "TODO_c000_228b", 0),
+    MVI(0xc000228c, "TODO_c000_228c", 0),
+    MVI(0xc000228d, "TODO_c000_228d", 0),
+    MVI(0xc000228e, "TODO_c000_228e", 0),
+    MVI(0xc000228f, "TODO_c000_228f", 0),
+    MVI(0xc0002290, "TODO_c000_2290", 0),
+    MVI(0xc0002291, "TODO_c000_2291", 0),
+    MVI(0xc0002292, "TODO_c000_2292", 0),
+    MVI(0xc0002293, "TODO_c000_2293", 0),
+    MVI(0xc0002294, "TODO_c000_2294", 0),
+    MVI(0xc0002295, "TODO_c000_2295", 0),
+    MVI(0xc0002296, "TODO_c000_2296", 0),
+    MVI(0xc0002297, "TODO_c000_2297", 0),
+    MVI(0xc0002298, "TODO_c000_2298", 0),
+    MVI(0xc0002299, "TODO_c000_2299", 0),
+    MVI(0xc000229a, "TODO_c000_229a", 0),
+    MVI(0xc000229b, "TODO_c000_229b", 0),
+    MVI(0xc000229c, "TODO_c000_229c", 0),
+    MVI(0xc000229d, "TODO_c000_229d", 0),
+    MVI(0xc000229e, "TODO_c000_229e", 0),
+    MVI(0xc000229f, "TODO_c000_229f", 0),
+    MVI(0xc00022a0, "TODO_c000_22a0", 0),
+    MVI(0xc00022a1, "TODO_c000_22a1", 0),
+    MVI(0xc00022a2, "TODO_c000_22a2", 0),
+    MVI(0xc00022a3, "TODO_c000_22a3", 0),
+    MVI(0xc00022a4, "TODO_c000_22a4", 0),
+    MVI(0xc00022a5, "TODO_c000_22a5", 0),
+    MVI(0xc00022a6, "TODO_c000_22a6", 0),
+    MVI(0xc00022a7, "TODO_c000_22a7", 0),
+    MVI(0xc00022a8, "TODO_c000_22a8", 0),
+    MVI(0xc00022a9, "TODO_c000_22a9", 0),
+    MVI(0xc00022aa, "TODO_c000_22aa", 0),
+    MVI(0xc00022ab, "TODO_c000_22ab", 0),
+    MVI(0xc00022ac, "TODO_c000_22ac", 0),
+    MVI(0xc00022ad, "TODO_c000_22ad", 0),
+    MVI(0xc00022ae, "TODO_c000_22ae", 0),
+    MVI(0xc00022af, "TODO_c000_22af", 0),
+    MVI(0xc00022b0, "TODO_c000_22b0", 0),
+    MVI(0xc00022b1, "TODO_c000_22b1", 0),
+    MVI(0xc00022b2, "TODO_c000_22b2", 0),
+    MVI(0xc00022b3, "TODO_c000_22b3", 0),
+    MVI(0xc00022b4, "TODO_c000_22b4", 0),
+    MVI(0xc00022b5, "TODO_c000_22b5", 0),
+    MVI(0xc00022b6, "TODO_c000_22b6", 0),
+    MVI(0xc00022b7, "TODO_c000_22b7", 0),
+    MVI(0xc00022b8, "TODO_c000_22b8", 0),
+    MVI(0xc00022b9, "TODO_c000_22b9", 0),
+    MVI(0xc00022ba, "TODO_c000_22ba", 0),
+    MVI(0xc00022bb, "TODO_c000_22bb", 0),
+    MVI(0xc00022bc, "TODO_c000_22bc", 0),
+    MVI(0xc00022bd, "TODO_c000_22bd", 0),
+    MVI(0xc00022be, "TODO_c000_22be", 0),
+    MVI(0xc00022bf, "TODO_c000_22bf", 0),
+    MVI(0xc00022c0, "TODO_c000_22c0", 0),
+    MVI(0xc00022c1, "TODO_c000_22c1", 0),
+    MVI(0xc00022c2, "TODO_c000_22c2", 0),
+    MVI(0xc00022c3, "TODO_c000_22c3", 0),
+    MVI(0xc00022c4, "TODO_c000_22c4", 0),
+    MVI(0xc00022c5, "TODO_c000_22c5", 0),
+    MVI(0xc00022c6, "TODO_c000_22c6", 0),
+    MVI(0xc00022c7, "TODO_c000_22c7", 0),
+    MVI(0xc00022c8, "TODO_c000_22c8", 0),
+    MVI(0xc00022c9, "TODO_c000_22c9", 0),
+    MVI(0xc00022ca, "TODO_c000_22ca", 0),
+    MVI(0xc00022cb, "TODO_c000_22cb", 0),
+    MVI(0xc00022cc, "TODO_c000_22cc", 0),
+    MVI(0xc00022cd, "TODO_c000_22cd", 0),
+    MVI(0xc00022ce, "TODO_c000_22ce", 0),
+    MVI(0xc00022cf, "TODO_c000_22cf", 0),
+    MVI(0xc00022d0, "TODO_c000_22d0", 0),
+    MVI(0xc00022d1, "TODO_c000_22d1", 0),
+    MVI(0xc00022d2, "TODO_c000_22d2", 0),
+    MVI(0xc00022d3, "TODO_c000_22d3", 0),
+    MVI(0xc00022d4, "TODO_c000_22d4", 0),
+    MVI(0xc00022d5, "TODO_c000_22d5", 0),
+    MVI(0xc00022d6, "TODO_c000_22d6", 0),
+    MVI(0xc00022d7, "TODO_c000_22d7", 0),
+    MVI(0xc00022d8, "TODO_c000_22d8", 0),
+    MVI(0xc00022d9, "TODO_c000_22d9", 0),
+    MVI(0xc00022da, "TODO_c000_22da", 0),
+    MVI(0xc00022db, "TODO_c000_22db", 0),
+    MVI(0xc00022dc, "TODO_c000_22dc", 0),
+    MVI(0xc00022dd, "TODO_c000_22dd", 0),
+    MVI(0xc00022de, "TODO_c000_22de", 0),
+    MVI(0xc00022df, "TODO_c000_22df", 0),
+    MVI(0xc00022e0, "TODO_c000_22e0", 0),
+    MVI(0xc00022e1, "TODO_c000_22e1", 0),
+    MVI(0xc00022e2, "TODO_c000_22e2", 0),
+    MVI(0xc00022e3, "TODO_c000_22e3", 0),
+    MVI(0xc00022e4, "TODO_c000_22e4", 0),
+    MVI(0xc00022e5, "TODO_c000_22e5", 0),
+    MVI(0xc00022e6, "TODO_c000_22e6", 0),
+    MVI(0xc00022e7, "TODO_c000_22e7", 0),
+    MVI(0xc00022e8, "TODO_c000_22e8", 0),
+    MVI(0xc00022e9, "TODO_c000_22e9", 0),
+    MVI(0xc00022ea, "TODO_c000_22ea", 0),
+    MVI(0xc00022eb, "TODO_c000_22eb", 0),
+    MVI(0xc00022ec, "TODO_c000_22ec", 0),
+    MVI(0xc00022ed, "TODO_c000_22ed", 0),
+    MVI(0xc00022ee, "TODO_c000_22ee", 0),
+    MVI(0xc00022ef, "TODO_c000_22ef", 0),
+    MVI(0xc00022f0, "TODO_c000_22f0", 0),
+    MVI(0xc00022f1, "TODO_c000_22f1", 0),
+    MVI(0xc00022f2, "TODO_c000_22f2", 0),
+    MVI(0xc00022f3, "TODO_c000_22f3", 0),
+    MVI(0xc00022f4, "TODO_c000_22f4", 0),
+    MVI(0xc00022f5, "TODO_c000_22f5", 0),
+    MVI(0xc00022f6, "TODO_c000_22f6", 0),
+    MVI(0xc00022f7, "TODO_c000_22f7", 0),
+    MVI(0xc00022f8, "TODO_c000_22f8", 0),
+    MVI(0xc00022f9, "TODO_c000_22f9", 0),
+    MVI(0xc00022fa, "TODO_c000_22fa", 0),
+    MVI(0xc00022fb, "TODO_c000_22fb", 0),
+    MVI(0xc00022fc, "TODO_c000_22fc", 0),
+    MVI(0xc00022fd, "TODO_c000_22fd", 0),
+    MVI(0xc00022fe, "TODO_c000_22fe", 0),
+    MVI(0xc00022ff, "TODO_c000_22ff", 0),
+    MVI(0xc0002300, "TODO_c000_2300", 0),
+    MVI(0xc0002301, "TODO_c000_2301", 0),
+    MVI(0xc0002302, "TODO_c000_2302", 0),
+    MVI(0xc0002303, "TODO_c000_2303", 0),
+    MVI(0xc0002304, "TODO_c000_2304", 0),
+    MVI(0xc0002305, "TODO_c000_2305", 0),
+    MVI(0xc0002306, "TODO_c000_2306", 0),
+    MVI(0xc0002307, "TODO_c000_2307", 0),
+    MVI(0xc0002308, "TODO_c000_2308", 0),
+    MVI(0xc0002309, "TODO_c000_2309", 0),
+    MVI(0xc000230a, "TODO_c000_230a", 0),
+    MVI(0xc000230b, "TODO_c000_230b", 0),
+    MVI(0xc000230c, "TODO_c000_230c", 0),
+    MVI(0xc000230d, "TODO_c000_230d", 0),
+    MVI(0xc000230e, "TODO_c000_230e", 0),
+    MVI(0xc000230f, "TODO_c000_230f", 0),
+    MVI(0xc0002310, "TODO_c000_2310", 0),
+    MVI(0xc0002311, "TODO_c000_2311", 0),
+    MVI(0xc0002312, "TODO_c000_2312", 0),
+    MVI(0xc0002313, "TODO_c000_2313", 0),
+    MVI(0xc0002314, "TODO_c000_2314", 0),
+    MVI(0xc0002315, "TODO_c000_2315", 0),
+    MVI(0xc0002316, "TODO_c000_2316", 0),
+    MVI(0xc0002317, "TODO_c000_2317", 0),
+    MVI(0xc0002318, "TODO_c000_2318", 0),
+    MVI(0xc0002319, "TODO_c000_2319", 0),
+    MVI(0xc000231a, "TODO_c000_231a", 0),
+    MVI(0xc000231b, "TODO_c000_231b", 0),
+    MVI(0xc000231c, "TODO_c000_231c", 0),
+    MVI(0xc000231d, "TODO_c000_231d", 0),
+    MVI(0xc000231e, "TODO_c000_231e", 0),
+    MVI(0xc000231f, "TODO_c000_231f", 0),
+    MVI(0xc0002320, "TODO_c000_2320", 0),
+    MVI(0xc0002321, "TODO_c000_2321", 0),
+    MVI(0xc0002322, "TODO_c000_2322", 0),
+    MVI(0xc0002323, "TODO_c000_2323", 0),
+    MVI(0xc0002324, "TODO_c000_2324", 0),
+    MVI(0xc0002325, "TODO_c000_2325", 0),
+    MVI(0xc0002326, "TODO_c000_2326", 0),
+    MVI(0xc0002327, "TODO_c000_2327", 0),
+    MVI(0xc0002328, "TODO_c000_2328", 0),
+    MVI(0xc0002329, "TODO_c000_2329", 0),
+    MVI(0xc000232a, "TODO_c000_232a", 0),
+    MVI(0xc000232b, "TODO_c000_232b", 0),
+    MVI(0xc000232c, "TODO_c000_232c", 0),
+    MVI(0xc000232d, "TODO_c000_232d", 0),
+    MVI(0xc000232e, "TODO_c000_232e", 0),
+    MVI(0xc000232f, "TODO_c000_232f", 0),
+    MVI(0xc0002330, "TODO_c000_2330", 0),
+    MVI(0xc0002331, "TODO_c000_2331", 0),
+    MVI(0xc0002332, "TODO_c000_2332", 0),
+    MVI(0xc0002333, "TODO_c000_2333", 0),
+    MVI(0xc0002334, "TODO_c000_2334", 0),
+    MVI(0xc0002335, "TODO_c000_2335", 0),
+    MVI(0xc0002336, "TODO_c000_2336", 0),
+    MVI(0xc0002337, "TODO_c000_2337", 0),
+    MVI(0xc0002338, "TODO_c000_2338", 0),
+    MVI(0xc0002339, "TODO_c000_2339", 0),
+    MVI(0xc000233a, "TODO_c000_233a", 0),
+    MVI(0xc000233b, "TODO_c000_233b", 0),
+    MVI(0xc000233c, "TODO_c000_233c", 0),
+    MVI(0xc000233d, "TODO_c000_233d", 0),
+    MVI(0xc000233e, "TODO_c000_233e", 0),
+    MVI(0xc000233f, "TODO_c000_233f", 0),
+    MVI(0xc0002340, "TODO_c000_2340", 0),
+    MVI(0xc0002341, "TODO_c000_2341", 0),
+    MVI(0xc0002342, "TODO_c000_2342", 0),
+    MVI(0xc0002343, "TODO_c000_2343", 0),
+    MVI(0xc0002344, "TODO_c000_2344", 0),
+    MVI(0xc0002345, "TODO_c000_2345", 0),
+    MVI(0xc0002346, "TODO_c000_2346", 0),
+    MVI(0xc0002347, "TODO_c000_2347", 0),
+    MVI(0xc0002348, "TODO_c000_2348", 0),
+    MVI(0xc0002349, "TODO_c000_2349", 0),
+    MVI(0xc000234a, "TODO_c000_234a", 0),
+    MVI(0xc000234b, "TODO_c000_234b", 0),
+    MVI(0xc000234c, "TODO_c000_234c", 0),
+    MVI(0xc000234d, "TODO_c000_234d", 0),
+    MVI(0xc000234e, "TODO_c000_234e", 0),
+    MVI(0xc000234f, "TODO_c000_234f", 0),
+    MVI(0xc0002350, "TODO_c000_2350", 0),
+    MVI(0xc0002351, "TODO_c000_2351", 0),
+    MVI(0xc0002352, "TODO_c000_2352", 0),
+    MVI(0xc0002353, "TODO_c000_2353", 0),
+    MVI(0xc0002354, "TODO_c000_2354", 0),
+    MVI(0xc0002355, "TODO_c000_2355", 0),
+    MVI(0xc0002356, "TODO_c000_2356", 0),
+    MVI(0xc0002357, "TODO_c000_2357", 0),
+    MVI(0xc0002358, "TODO_c000_2358", 0),
+    MVI(0xc0002359, "TODO_c000_2359", 0),
+    MVI(0xc000235a, "TODO_c000_235a", 0),
+    MVI(0xc000235b, "TODO_c000_235b", 0),
+    MVI(0xc000235c, "TODO_c000_235c", 0),
+    MVI(0xc000235d, "TODO_c000_235d", 0),
+    MVI(0xc000235e, "TODO_c000_235e", 0),
+    MVI(0xc000235f, "TODO_c000_235f", 0),
+    MVI(0xc0002360, "TODO_c000_2360", 0),
+    MVI(0xc0002361, "TODO_c000_2361", 0),
+    MVI(0xc0002362, "TODO_c000_2362", 0),
+    MVI(0xc0002363, "TODO_c000_2363", 0),
+    MVI(0xc0002364, "TODO_c000_2364", 0),
+    MVI(0xc0002365, "TODO_c000_2365", 0),
+    MVI(0xc0002366, "TODO_c000_2366", 0),
+    MVI(0xc0002367, "TODO_c000_2367", 0),
+    MVI(0xc0002368, "TODO_c000_2368", 0),
+    MVI(0xc0002369, "TODO_c000_2369", 0),
+    MVI(0xc000236a, "TODO_c000_236a", 0),
+    MVI(0xc000236b, "TODO_c000_236b", 0),
+    MVI(0xc000236c, "TODO_c000_236c", 0),
+    MVI(0xc000236d, "TODO_c000_236d", 0),
+    MVI(0xc000236e, "TODO_c000_236e", 0),
+    MVI(0xc000236f, "TODO_c000_236f", 0),
+    MVI(0xc0002370, "TODO_c000_2370", 0),
+    MVI(0xc0002371, "TODO_c000_2371", 0),
+    MVI(0xc0002372, "TODO_c000_2372", 0),
+    MVI(0xc0002373, "TODO_c000_2373", 0),
+    MVI(0xc0002374, "TODO_c000_2374", 0),
+    MVI(0xc0002375, "TODO_c000_2375", 0),
+    MVI(0xc0002376, "TODO_c000_2376", 0),
+    MVI(0xc0002377, "TODO_c000_2377", 0),
+    MVI(0xc0002378, "TODO_c000_2378", 0),
+    MVI(0xc0002379, "TODO_c000_2379", 0),
+    MVI(0xc000237a, "TODO_c000_237a", 0),
+    MVI(0xc000237b, "TODO_c000_237b", 0),
+    MVI(0xc000237c, "TODO_c000_237c", 0),
+    MVI(0xc000237d, "TODO_c000_237d", 0),
+    MVI(0xc000237e, "TODO_c000_237e", 0),
+    MVI(0xc000237f, "TODO_c000_237f", 0),
+    MVI(0xc0002380, "TODO_c000_2380", 0),
+    MVI(0xc0002381, "TODO_c000_2381", 0),
+    MVI(0xc0002382, "TODO_c000_2382", 0),
+    MVI(0xc0002383, "TODO_c000_2383", 0),
+    MVI(0xc0002384, "TODO_c000_2384", 0),
+    MVI(0xc0002385, "TODO_c000_2385", 0),
+    MVI(0xc0002386, "TODO_c000_2386", 0),
+    MVI(0xc0002387, "TODO_c000_2387", 0),
+    MVI(0xc0002388, "TODO_c000_2388", 0),
+    MVI(0xc0002389, "TODO_c000_2389", 0),
+    MVI(0xc000238a, "TODO_c000_238a", 0),
+    MVI(0xc000238b, "TODO_c000_238b", 0),
+    MVI(0xc000238c, "TODO_c000_238c", 0),
+    MVI(0xc000238d, "TODO_c000_238d", 0),
+    MVI(0xc000238e, "TODO_c000_238e", 0),
+    MVI(0xc000238f, "TODO_c000_238f", 0),
+    MVI(0xc0002390, "TODO_c000_2390", 0),
+    MVI(0xc0002391, "TODO_c000_2391", 0),
+    MVI(0xc0002392, "TODO_c000_2392", 0),
+    MVI(0xc0002393, "TODO_c000_2393", 0),
+    MVI(0xc0002394, "TODO_c000_2394", 0),
+    MVI(0xc0002395, "TODO_c000_2395", 0),
+    MVI(0xc0002396, "TODO_c000_2396", 0),
+    MVI(0xc0002397, "TODO_c000_2397", 0),
+    MVI(0xc0002398, "TODO_c000_2398", 0),
+    MVI(0xc0002399, "TODO_c000_2399", 0),
+    MVI(0xc000239a, "TODO_c000_239a", 0),
+    MVI(0xc000239b, "TODO_c000_239b", 0),
+    MVI(0xc000239c, "TODO_c000_239c", 0),
+    MVI(0xc000239d, "TODO_c000_239d", 0),
+    MVI(0xc000239e, "TODO_c000_239e", 0),
+    MVI(0xc000239f, "TODO_c000_239f", 0),
+    MVI(0xc00023a0, "TODO_c000_23a0", 0),
+    MVI(0xc00023a1, "TODO_c000_23a1", 0),
+    MVI(0xc00023a2, "TODO_c000_23a2", 0),
+    MVI(0xc00023a3, "TODO_c000_23a3", 0),
+    MVI(0xc00023a4, "TODO_c000_23a4", 0),
+    MVI(0xc00023a5, "TODO_c000_23a5", 0),
+    MVI(0xc00023a6, "TODO_c000_23a6", 0),
+    MVI(0xc00023a7, "TODO_c000_23a7", 0),
+    MVI(0xc00023a8, "TODO_c000_23a8", 0),
+    MVI(0xc00023a9, "TODO_c000_23a9", 0),
+    MVI(0xc00023aa, "TODO_c000_23aa", 0),
+    MVI(0xc00023ab, "TODO_c000_23ab", 0),
+    MVI(0xc00023ac, "TODO_c000_23ac", 0),
+    MVI(0xc00023ad, "TODO_c000_23ad", 0),
+    MVI(0xc00023ae, "TODO_c000_23ae", 0),
+    MVI(0xc00023af, "TODO_c000_23af", 0),
+    MVI(0xc00023b0, "TODO_c000_23b0", 0),
+    MVI(0xc00023b1, "TODO_c000_23b1", 0),
+    MVI(0xc00023b2, "TODO_c000_23b2", 0),
+    MVI(0xc00023b3, "TODO_c000_23b3", 0),
+    MVI(0xc00023b4, "TODO_c000_23b4", 0),
+    MVI(0xc00023b5, "TODO_c000_23b5", 0),
+    MVI(0xc00023b6, "TODO_c000_23b6", 0),
+    MVI(0xc00023b7, "TODO_c000_23b7", 0),
+    MVI(0xc00023b8, "TODO_c000_23b8", 0),
+    MVI(0xc00023b9, "TODO_c000_23b9", 0),
+    MVI(0xc00023ba, "TODO_c000_23ba", 0),
+    MVI(0xc00023bb, "TODO_c000_23bb", 0),
+    MVI(0xc00023bc, "TODO_c000_23bc", 0),
+    MVI(0xc00023bd, "TODO_c000_23bd", 0),
+    MVI(0xc00023be, "TODO_c000_23be", 0),
+    MVI(0xc00023bf, "TODO_c000_23bf", 0),
+    MVI(0xc00023c0, "TODO_c000_23c0", 0),
+    MVI(0xc00023c1, "TODO_c000_23c1", 0),
+    MVI(0xc00023c2, "TODO_c000_23c2", 0),
+    MVI(0xc00023c3, "TODO_c000_23c3", 0),
+    MVI(0xc00023c4, "TODO_c000_23c4", 0),
+    MVI(0xc00023c5, "TODO_c000_23c5", 0),
+    MVI(0xc00023c6, "TODO_c000_23c6", 0),
+    MVI(0xc00023c7, "TODO_c000_23c7", 0),
+    MVI(0xc00023c8, "TODO_c000_23c8", 0),
+    MVI(0xc00023c9, "TODO_c000_23c9", 0),
+    MVI(0xc00023ca, "TODO_c000_23ca", 0),
+    MVI(0xc00023cb, "TODO_c000_23cb", 0),
+    MVI(0xc00023cc, "TODO_c000_23cc", 0),
+    MVI(0xc00023cd, "TODO_c000_23cd", 0),
+    MVI(0xc00023ce, "TODO_c000_23ce", 0),
+    MVI(0xc00023cf, "TODO_c000_23cf", 0),
+    MVI(0xc00023d0, "TODO_c000_23d0", 0),
+    MVI(0xc00023d1, "TODO_c000_23d1", 0),
+    MVI(0xc00023d2, "TODO_c000_23d2", 0),
+    MVI(0xc00023d3, "TODO_c000_23d3", 0),
+    MVI(0xc00023d4, "TODO_c000_23d4", 0),
+    MVI(0xc00023d5, "TODO_c000_23d5", 0),
+    MVI(0xc00023d6, "TODO_c000_23d6", 0),
+    MVI(0xc00023d7, "TODO_c000_23d7", 0),
+    MVI(0xc00023d8, "TODO_c000_23d8", 0),
+    MVI(0xc00023d9, "TODO_c000_23d9", 0),
+    MVI(0xc00023da, "TODO_c000_23da", 0),
+    MVI(0xc00023db, "TODO_c000_23db", 0),
+    MVI(0xc00023dc, "TODO_c000_23dc", 0),
+    MVI(0xc00023dd, "TODO_c000_23dd", 0),
+    MVI(0xc00023de, "TODO_c000_23de", 0),
+    MVI(0xc00023df, "TODO_c000_23df", 0),
+    MVI(0xc00023e0, "TODO_c000_23e0", 0),
+    MVI(0xc00023e1, "TODO_c000_23e1", 0),
+    MVI(0xc00023e2, "TODO_c000_23e2", 0),
+    MVI(0xc00023e3, "TODO_c000_23e3", 0),
+    MVI(0xc00023e4, "TODO_c000_23e4", 0),
+    MVI(0xc00023e5, "TODO_c000_23e5", 0),
+    MVI(0xc00023e6, "TODO_c000_23e6", 0),
+    MVI(0xc00023e7, "TODO_c000_23e7", 0),
+    MVI(0xc00023e8, "TODO_c000_23e8", 0),
+    MVI(0xc00023e9, "TODO_c000_23e9", 0),
+    MVI(0xc00023ea, "TODO_c000_23ea", 0),
+    MVI(0xc00023eb, "TODO_c000_23eb", 0),
+    MVI(0xc00023ec, "TODO_c000_23ec", 0),
+    MVI(0xc00023ed, "TODO_c000_23ed", 0),
+    MVI(0xc00023ee, "TODO_c000_23ee", 0),
+    MVI(0xc00023ef, "TODO_c000_23ef", 0),
+    MVI(0xc00023f0, "TODO_c000_23f0", 0),
+    MVI(0xc00023f1, "TODO_c000_23f1", 0),
+    MVI(0xc00023f2, "TODO_c000_23f2", 0),
+    MVI(0xc00023f3, "TODO_c000_23f3", 0),
+    MVI(0xc00023f4, "TODO_c000_23f4", 0),
+    MVI(0xc00023f5, "TODO_c000_23f5", 0),
+    MVI(0xc00023f6, "TODO_c000_23f6", 0),
+    MVI(0xc00023f7, "TODO_c000_23f7", 0),
+    MVI(0xc00023f8, "TODO_c000_23f8", 0),
+    MVI(0xc00023f9, "TODO_c000_23f9", 0),
+    MVI(0xc00023fa, "TODO_c000_23fa", 0),
+    MVI(0xc00023fb, "TODO_c000_23fb", 0),
+    MVI(0xc00023fc, "TODO_c000_23fc", 0),
+    MVI(0xc00023fd, "TODO_c000_23fd", 0),
+    MVI(0xc00023fe, "TODO_c000_23fe", 0),
+    MVI(0xc00023ff, "TODO_c000_23ff", 0),
+    MVI(0xc0002400, "TODO_c000_2400", 0),
+    MVI(0xc0002401, "TODO_c000_2401", 0),
+    MVI(0xc0002402, "TODO_c000_2402", 0),
+    MVI(0xc0002403, "TODO_c000_2403", 0),
+    MVI(0xc0002404, "TODO_c000_2404", 0),
+    MVI(0xc0002405, "TODO_c000_2405", 0),
+    MVI(0xc0002406, "TODO_c000_2406", 0),
+    MVI(0xc0002407, "TODO_c000_2407", 0),
+    MVI(0xc0002408, "TODO_c000_2408", 0),
+    MVI(0xc0002409, "TODO_c000_2409", 0),
+    MVI(0xc000240a, "TODO_c000_240a", 0),
+    MVI(0xc000240b, "TODO_c000_240b", 0),
+    MVI(0xc000240c, "TODO_c000_240c", 0),
+    MVI(0xc000240d, "TODO_c000_240d", 0),
+    MVI(0xc000240e, "TODO_c000_240e", 0),
+    MVI(0xc000240f, "TODO_c000_240f", 0),
+    MVI(0xc0002410, "TODO_c000_2410", 0),
+    MVI(0xc0002411, "TODO_c000_2411", 0),
+    MVI(0xc0002412, "TODO_c000_2412", 0),
+    MVI(0xc0002413, "TODO_c000_2413", 0),
+    MVI(0xc0002414, "TODO_c000_2414", 0),
+    MVI(0xc0002415, "TODO_c000_2415", 0),
+    MVI(0xc0002416, "TODO_c000_2416", 0),
+    MVI(0xc0002417, "TODO_c000_2417", 0),
+    MVI(0xc0002418, "TODO_c000_2418", 0),
+    MVI(0xc0002419, "TODO_c000_2419", 0),
+    MVI(0xc000241a, "TODO_c000_241a", 0),
+    MVI(0xc000241b, "TODO_c000_241b", 0),
+    MVI(0xc000241c, "TODO_c000_241c", 0),
+    MVI(0xc000241d, "TODO_c000_241d", 0),
+    MVI(0xc000241e, "TODO_c000_241e", 0),
+    MVI(0xc000241f, "TODO_c000_241f", 0),
+    MVI(0xc0002420, "TODO_c000_2420", 0),
+    MVI(0xc0002421, "TODO_c000_2421", 0),
+    MVI(0xc0002422, "TODO_c000_2422", 0),
+    MVI(0xc0002423, "TODO_c000_2423", 0),
+    MVI(0xc0002424, "TODO_c000_2424", 0),
+    MVI(0xc0002425, "TODO_c000_2425", 0),
+    MVI(0xc0002426, "TODO_c000_2426", 0),
+    MVI(0xc0002427, "TODO_c000_2427", 0),
+    MVI(0xc0002428, "TODO_c000_2428", 0),
+    MVI(0xc0002429, "TODO_c000_2429", 0),
+    MVI(0xc000242a, "TODO_c000_242a", 0),
+    MVI(0xc000242b, "TODO_c000_242b", 0),
+    MVI(0xc000242c, "TODO_c000_242c", 0),
+    MVI(0xc000242d, "TODO_c000_242d", 0),
+    MVI(0xc000242e, "TODO_c000_242e", 0),
+    MVI(0xc000242f, "TODO_c000_242f", 0),
+    MVI(0xc0002430, "TODO_c000_2430", 0),
+    MVI(0xc0002431, "TODO_c000_2431", 0),
+    MVI(0xc0002432, "TODO_c000_2432", 0),
+    MVI(0xc0002433, "TODO_c000_2433", 0),
+    MVI(0xc0002434, "TODO_c000_2434", 0),
+    MVI(0xc0002435, "TODO_c000_2435", 0),
+    MVI(0xc0002436, "TODO_c000_2436", 0),
+    MVI(0xc0002437, "TODO_c000_2437", 0),
+    MVI(0xc0002438, "TODO_c000_2438", 0),
+    MVI(0xc0002439, "TODO_c000_2439", 0),
+    MVI(0xc000243a, "TODO_c000_243a", 0),
+    MVI(0xc000243b, "TODO_c000_243b", 0),
+    MVI(0xc000243c, "TODO_c000_243c", 0),
+    MVI(0xc000243d, "TODO_c000_243d", 0),
+    MVI(0xc000243e, "TODO_c000_243e", 0),
+    MVI(0xc000243f, "TODO_c000_243f", 0),
+    MVI(0xc0002440, "TODO_c000_2440", 0),
+    MVI(0xc0002441, "TODO_c000_2441", 0),
+    MVI(0xc0002442, "TODO_c000_2442", 0),
+    MVI(0xc0002443, "TODO_c000_2443", 0),
+    MVI(0xc0002444, "TODO_c000_2444", 0),
+    MVI(0xc0002445, "TODO_c000_2445", 0),
+    MVI(0xc0002446, "TODO_c000_2446", 0),
+    MVI(0xc0002447, "TODO_c000_2447", 0),
+    MVI(0xc0002448, "TODO_c000_2448", 0),
+    MVI(0xc0002449, "TODO_c000_2449", 0),
+    MVI(0xc000244a, "TODO_c000_244a", 0),
+    MVI(0xc000244b, "TODO_c000_244b", 0),
+    MVI(0xc000244c, "TODO_c000_244c", 0),
+    MVI(0xc000244d, "TODO_c000_244d", 0),
+    MVI(0xc000244e, "TODO_c000_244e", 0),
+    MVI(0xc000244f, "TODO_c000_244f", 0),
+    MVI(0xc0002450, "TODO_c000_2450", 0),
+    MVI(0xc0002451, "TODO_c000_2451", 0),
+    MVI(0xc0002452, "TODO_c000_2452", 0),
+    MVI(0xc0002453, "TODO_c000_2453", 0),
+    MVI(0xc0002454, "TODO_c000_2454", 0),
+    MVI(0xc0002455, "TODO_c000_2455", 0),
+    MVI(0xc0002456, "TODO_c000_2456", 0),
+    MVI(0xc0002457, "TODO_c000_2457", 0),
+    MVI(0xc0002458, "TODO_c000_2458", 0),
+    MVI(0xc0002459, "TODO_c000_2459", 0),
+    MVI(0xc000245a, "TODO_c000_245a", 0),
+    MVI(0xc000245b, "TODO_c000_245b", 0),
+    MVI(0xc000245c, "TODO_c000_245c", 0),
+    MVI(0xc000245d, "TODO_c000_245d", 0),
+    MVI(0xc000245e, "TODO_c000_245e", 0),
+    MVI(0xc000245f, "TODO_c000_245f", 0),
+    MVI(0xc0002460, "TODO_c000_2460", 0),
+    MVI(0xc0002461, "TODO_c000_2461", 0),
+    MVI(0xc0002462, "TODO_c000_2462", 0),
+    MVI(0xc0002463, "TODO_c000_2463", 0),
+    MVI(0xc0002464, "TODO_c000_2464", 0),
+    MVI(0xc0002465, "TODO_c000_2465", 0),
+    MVI(0xc0002466, "TODO_c000_2466", 0),
+    MVI(0xc0002467, "TODO_c000_2467", 0),
+    MVI(0xc0002468, "TODO_c000_2468", 0),
+    MVI(0xc0002469, "TODO_c000_2469", 0),
+    MVI(0xc000246a, "TODO_c000_246a", 0),
+    MVI(0xc000246b, "TODO_c000_246b", 0),
+    MVI(0xc000246c, "TODO_c000_246c", 0),
+    MVI(0xc000246d, "TODO_c000_246d", 0),
+    MVI(0xc000246e, "TODO_c000_246e", 0),
+    MVI(0xc000246f, "TODO_c000_246f", 0),
+    MVI(0xc0002470, "TODO_c000_2470", 0),
+    MVI(0xc0002471, "TODO_c000_2471", 0),
+    MVI(0xc0002472, "TODO_c000_2472", 0),
+    MVI(0xc0002473, "TODO_c000_2473", 0),
+    MVI(0xc0002474, "TODO_c000_2474", 0),
+    MVI(0xc0002475, "TODO_c000_2475", 0),
+    MVI(0xc0002476, "TODO_c000_2476", 0),
+    MVI(0xc0002477, "TODO_c000_2477", 0),
+    MVI(0xc0002478, "TODO_c000_2478", 0),
+    MVI(0xc0002479, "TODO_c000_2479", 0),
+    MVI(0xc000247a, "TODO_c000_247a", 0),
+    MVI(0xc000247b, "TODO_c000_247b", 0),
+    MVI(0xc000247c, "TODO_c000_247c", 0),
+    MVI(0xc000247d, "TODO_c000_247d", 0),
+    MVI(0xc000247e, "TODO_c000_247e", 0),
+    MVI(0xc000247f, "TODO_c000_247f", 0),
+    MVI(0xc0002480, "TODO_c000_2480", 0),
+    MVI(0xc0002481, "TODO_c000_2481", 0),
+    MVI(0xc0002482, "TODO_c000_2482", 0),
+    MVI(0xc0002483, "TODO_c000_2483", 0),
+    MVI(0xc0002484, "TODO_c000_2484", 0),
+    MVI(0xc0002485, "TODO_c000_2485", 0),
+    MVI(0xc0002486, "TODO_c000_2486", 0),
+    MVI(0xc0002487, "TODO_c000_2487", 0),
+    MVI(0xc0002488, "TODO_c000_2488", 0),
+    MVI(0xc0002489, "TODO_c000_2489", 0),
+    MVI(0xc000248a, "TODO_c000_248a", 0),
+    MVI(0xc000248b, "TODO_c000_248b", 0),
+    MVI(0xc000248c, "TODO_c000_248c", 0),
+    MVI(0xc000248d, "TODO_c000_248d", 0),
+    MVI(0xc000248e, "TODO_c000_248e", 0),
+    MVI(0xc000248f, "TODO_c000_248f", 0),
+    MVI(0xc0002490, "TODO_c000_2490", 0),
+    MVI(0xc0002491, "TODO_c000_2491", 0),
+    MVI(0xc0002492, "TODO_c000_2492", 0),
+    MVI(0xc0002493, "TODO_c000_2493", 0),
+    MVI(0xc0002494, "TODO_c000_2494", 0),
+    MVI(0xc0002495, "TODO_c000_2495", 0),
+    MVI(0xc0002496, "TODO_c000_2496", 0),
+    MVI(0xc0002497, "TODO_c000_2497", 0),
+    MVI(0xc0002498, "TODO_c000_2498", 0),
+    MVI(0xc0002499, "TODO_c000_2499", 0),
+    MVI(0xc000249a, "TODO_c000_249a", 0),
+    MVI(0xc000249b, "TODO_c000_249b", 0),
+    MVI(0xc000249c, "TODO_c000_249c", 0),
+    MVI(0xc000249d, "TODO_c000_249d", 0),
+    MVI(0xc000249e, "TODO_c000_249e", 0),
+    MVI(0xc000249f, "TODO_c000_249f", 0),
+    MVI(0xc00024a0, "TODO_c000_24a0", 0),
+    MVI(0xc00024a1, "TODO_c000_24a1", 0),
+    MVI(0xc00024a2, "TODO_c000_24a2", 0),
+    MVI(0xc00024a3, "TODO_c000_24a3", 0),
+    MVI(0xc00024a4, "TODO_c000_24a4", 0),
+    MVI(0xc00024a5, "TODO_c000_24a5", 0),
+    MVI(0xc00024a6, "TODO_c000_24a6", 0),
+    MVI(0xc00024a7, "TODO_c000_24a7", 0),
+    MVI(0xc00024a8, "TODO_c000_24a8", 0),
+    MVI(0xc00024a9, "TODO_c000_24a9", 0),
+    MVI(0xc00024aa, "TODO_c000_24aa", 0),
+    MVI(0xc00024ab, "TODO_c000_24ab", 0),
+    MVI(0xc00024ac, "TODO_c000_24ac", 0),
+    MVI(0xc00024ad, "TODO_c000_24ad", 0),
+    MVI(0xc00024ae, "TODO_c000_24ae", 0),
+    MVI(0xc00024af, "TODO_c000_24af", 0),
+    MVI(0xc00024b0, "TODO_c000_24b0", 0),
+    MVI(0xc00024b1, "TODO_c000_24b1", 0),
+    MVI(0xc00024b2, "TODO_c000_24b2", 0),
+    MVI(0xc00024b3, "TODO_c000_24b3", 0),
+    MVI(0xc00024b4, "TODO_c000_24b4", 0),
+    MVI(0xc00024b5, "TODO_c000_24b5", 0),
+    MVI(0xc00024b6, "TODO_c000_24b6", 0),
+    MVI(0xc00024b7, "TODO_c000_24b7", 0),
+    MVI(0xc00024b8, "TODO_c000_24b8", 0),
+    MVI(0xc00024b9, "TODO_c000_24b9", 0),
+    MVI(0xc00024ba, "TODO_c000_24ba", 0),
+    MVI(0xc00024bb, "TODO_c000_24bb", 0),
+    MVI(0xc00024bc, "TODO_c000_24bc", 0),
+    MVI(0xc00024bd, "TODO_c000_24bd", 0),
+    MVI(0xc00024be, "TODO_c000_24be", 0),
+    MVI(0xc00024bf, "TODO_c000_24bf", 0),
+    MVI(0xc00024c0, "TODO_c000_24c0", 0),
+    MVI(0xc00024c1, "TODO_c000_24c1", 0),
+    MVI(0xc00024c2, "TODO_c000_24c2", 0),
+    MVI(0xc00024c3, "TODO_c000_24c3", 0),
+    MVI(0xc00024c4, "TODO_c000_24c4", 0),
+    MVI(0xc00024c5, "TODO_c000_24c5", 0),
+    MVI(0xc00024c6, "TODO_c000_24c6", 0),
+    MVI(0xc00024c7, "TODO_c000_24c7", 0),
+    MVI(0xc00024c8, "TODO_c000_24c8", 0),
+    MVI(0xc00024c9, "TODO_c000_24c9", 0),
+    MVI(0xc00024ca, "TODO_c000_24ca", 0),
+    MVI(0xc00024cb, "TODO_c000_24cb", 0),
+    MVI(0xc00024cc, "TODO_c000_24cc", 0),
+    MVI(0xc00024cd, "TODO_c000_24cd", 0),
+    MVI(0xc00024ce, "TODO_c000_24ce", 0),
+    MVI(0xc00024cf, "TODO_c000_24cf", 0),
+    MVI(0xc00024d0, "TODO_c000_24d0", 0),
+    MVI(0xc00024d1, "TODO_c000_24d1", 0),
+    MVI(0xc00024d2, "TODO_c000_24d2", 0),
+    MVI(0xc00024d3, "TODO_c000_24d3", 0),
+    MVI(0xc00024d4, "TODO_c000_24d4", 0),
+    MVI(0xc00024d5, "TODO_c000_24d5", 0),
+    MVI(0xc00024d6, "TODO_c000_24d6", 0),
+    MVI(0xc00024d7, "TODO_c000_24d7", 0),
+    MVI(0xc00024d8, "TODO_c000_24d8", 0),
+    MVI(0xc00024d9, "TODO_c000_24d9", 0),
+    MVI(0xc00024da, "TODO_c000_24da", 0),
+    MVI(0xc00024db, "TODO_c000_24db", 0),
+    MVI(0xc00024dc, "TODO_c000_24dc", 0),
+    MVI(0xc00024dd, "TODO_c000_24dd", 0),
+    MVI(0xc00024de, "TODO_c000_24de", 0),
+    MVI(0xc00024df, "TODO_c000_24df", 0),
+    MVI(0xc00024e0, "TODO_c000_24e0", 0),
+    MVI(0xc00024e1, "TODO_c000_24e1", 0),
+    MVI(0xc00024e2, "TODO_c000_24e2", 0),
+    MVI(0xc00024e3, "TODO_c000_24e3", 0),
+    MVI(0xc00024e4, "TODO_c000_24e4", 0),
+    MVI(0xc00024e5, "TODO_c000_24e5", 0),
+    MVI(0xc00024e6, "TODO_c000_24e6", 0),
+    MVI(0xc00024e7, "TODO_c000_24e7", 0),
+    MVI(0xc00024e8, "TODO_c000_24e8", 0),
+    MVI(0xc00024e9, "TODO_c000_24e9", 0),
+    MVI(0xc00024ea, "TODO_c000_24ea", 0),
+    MVI(0xc00024eb, "TODO_c000_24eb", 0),
+    MVI(0xc00024ec, "TODO_c000_24ec", 0),
+    MVI(0xc00024ed, "TODO_c000_24ed", 0),
+    MVI(0xc00024ee, "TODO_c000_24ee", 0),
+    MVI(0xc00024ef, "TODO_c000_24ef", 0),
+    MVI(0xc00024f0, "TODO_c000_24f0", 0),
+    MVI(0xc00024f1, "TODO_c000_24f1", 0),
+    MVI(0xc00024f2, "TODO_c000_24f2", 0),
+    MVI(0xc00024f3, "TODO_c000_24f3", 0),
+    MVI(0xc00024f4, "TODO_c000_24f4", 0),
+    MVI(0xc00024f5, "TODO_c000_24f5", 0),
+    MVI(0xc00024f6, "TODO_c000_24f6", 0),
+    MVI(0xc00024f7, "TODO_c000_24f7", 0),
+    MVI(0xc00024f8, "TODO_c000_24f8", 0),
+    MVI(0xc00024f9, "TODO_c000_24f9", 0),
+    MVI(0xc00024fa, "TODO_c000_24fa", 0),
+    MVI(0xc00024fb, "TODO_c000_24fb", 0),
+    MVI(0xc00024fc, "TODO_c000_24fc", 0),
+    MVI(0xc00024fd, "TODO_c000_24fd", 0),
+    MVI(0xc00024fe, "TODO_c000_24fe", 0),
+    MVI(0xc00024ff, "TODO_c000_24ff", 0),
+    MVI(0xc0002500, "TODO_c000_2500", 0),
+    MVI(0xc0002501, "TODO_c000_2501", 0),
+    MVI(0xc0002502, "TODO_c000_2502", 0),
+    MVI(0xc0002503, "TODO_c000_2503", 0),
+    MVI(0xc0002504, "TODO_c000_2504", 0),
+    MVI(0xc0002505, "TODO_c000_2505", 0),
+    MVI(0xc0002506, "TODO_c000_2506", 0),
+    MVI(0xc0002507, "TODO_c000_2507", 0),
+    MVI(0xc0002508, "TODO_c000_2508", 0),
+    MVI(0xc0002509, "TODO_c000_2509", 0),
+    MVI(0xc000250a, "TODO_c000_250a", 0),
+    MVI(0xc000250b, "TODO_c000_250b", 0),
+    MVI(0xc000250c, "TODO_c000_250c", 0),
+    MVI(0xc000250d, "TODO_c000_250d", 0),
+    MVI(0xc000250e, "TODO_c000_250e", 0),
+    MVI(0xc000250f, "TODO_c000_250f", 0),
+    MVI(0xc0002510, "TODO_c000_2510", 0),
+    MVI(0xc0002511, "TODO_c000_2511", 0),
+    MVI(0xc0002512, "TODO_c000_2512", 0),
+    MVI(0xc0002513, "TODO_c000_2513", 0),
+    MVI(0xc0002514, "TODO_c000_2514", 0),
+    MVI(0xc0002515, "TODO_c000_2515", 0),
+    MVI(0xc0002516, "TODO_c000_2516", 0),
+    MVI(0xc0002517, "TODO_c000_2517", 0),
+    MVI(0xc0002518, "TODO_c000_2518", 0),
+    MVI(0xc0002519, "TODO_c000_2519", 0),
+    MVI(0xc000251a, "TODO_c000_251a", 0),
+    MVI(0xc000251b, "TODO_c000_251b", 0),
+    MVI(0xc000251c, "TODO_c000_251c", 0),
+    MVI(0xc000251d, "TODO_c000_251d", 0),
+    MVI(0xc000251e, "TODO_c000_251e", 0),
+    MVI(0xc000251f, "TODO_c000_251f", 0),
+    MVI(0xc0002520, "TODO_c000_2520", 0),
+    MVI(0xc0002521, "TODO_c000_2521", 0),
+    MVI(0xc0002522, "TODO_c000_2522", 0),
+    MVI(0xc0002523, "TODO_c000_2523", 0),
+    MVI(0xc0002524, "TODO_c000_2524", 0),
+    MVI(0xc0002525, "TODO_c000_2525", 0),
+    MVI(0xc0002526, "TODO_c000_2526", 0),
+    MVI(0xc0002527, "TODO_c000_2527", 0),
+    MVI(0xc0002528, "TODO_c000_2528", 0),
+    MVI(0xc0002529, "TODO_c000_2529", 0),
+    MVI(0xc000252a, "TODO_c000_252a", 0),
+    MVI(0xc000252b, "TODO_c000_252b", 0),
+    MVI(0xc000252c, "TODO_c000_252c", 0),
+    MVI(0xc000252d, "TODO_c000_252d", 0),
+    MVI(0xc000252e, "TODO_c000_252e", 0),
+    MVI(0xc000252f, "TODO_c000_252f", 0),
+    MVI(0xc0002530, "TODO_c000_2530", 0),
+    MVI(0xc0002531, "TODO_c000_2531", 0),
+    MVI(0xc0002532, "TODO_c000_2532", 0),
+    MVI(0xc0002533, "TODO_c000_2533", 0),
+    MVI(0xc0002534, "TODO_c000_2534", 0),
+    MVI(0xc0002535, "TODO_c000_2535", 0),
+    MVI(0xc0002536, "TODO_c000_2536", 0),
+    MVI(0xc0002537, "TODO_c000_2537", 0),
+    MVI(0xc0002538, "TODO_c000_2538", 0),
+    MVI(0xc0002539, "TODO_c000_2539", 0),
+    MVI(0xc000253a, "TODO_c000_253a", 0),
+    MVI(0xc000253b, "TODO_c000_253b", 0),
+    MVI(0xc000253c, "TODO_c000_253c", 0),
+    MVI(0xc000253d, "TODO_c000_253d", 0),
+    MVI(0xc000253e, "TODO_c000_253e", 0),
+    MVI(0xc000253f, "TODO_c000_253f", 0),
+    MVI(0xc0002540, "TODO_c000_2540", 0),
+    MVI(0xc0002541, "TODO_c000_2541", 0),
+    MVI(0xc0002542, "TODO_c000_2542", 0),
+    MVI(0xc0002543, "TODO_c000_2543", 0),
+    MVI(0xc0002544, "TODO_c000_2544", 0),
+    MVI(0xc0002545, "TODO_c000_2545", 0),
+    MVI(0xc0002546, "TODO_c000_2546", 0),
+    MVI(0xc0002547, "TODO_c000_2547", 0),
+    MVI(0xc0002548, "TODO_c000_2548", 0),
+    MVI(0xc0002549, "TODO_c000_2549", 0),
+    MVI(0xc000254a, "TODO_c000_254a", 0),
+    MVI(0xc000254b, "TODO_c000_254b", 0),
+    MVI(0xc000254c, "TODO_c000_254c", 0),
+    MVI(0xc000254d, "TODO_c000_254d", 0),
+    MVI(0xc000254e, "TODO_c000_254e", 0),
+    MVI(0xc000254f, "TODO_c000_254f", 0),
+    MVI(0xc0002550, "TODO_c000_2550", 0),
+    MVI(0xc0002551, "TODO_c000_2551", 0),
+    MVI(0xc0002552, "TODO_c000_2552", 0),
+    MVI(0xc0002553, "TODO_c000_2553", 0),
+    MVI(0xc0002554, "TODO_c000_2554", 0),
+    MVI(0xc0002555, "TODO_c000_2555", 0),
+    MVI(0xc0002556, "TODO_c000_2556", 0),
+    MVI(0xc0002557, "TODO_c000_2557", 0),
+    MVI(0xc0002558, "TODO_c000_2558", 0),
+    MVI(0xc0002559, "TODO_c000_2559", 0),
+    MVI(0xc000255a, "TODO_c000_255a", 0),
+    MVI(0xc000255b, "TODO_c000_255b", 0),
+    MVI(0xc000255c, "TODO_c000_255c", 0),
+    MVI(0xc000255d, "TODO_c000_255d", 0),
+    MVI(0xc000255e, "TODO_c000_255e", 0),
+    MVI(0xc000255f, "TODO_c000_255f", 0),
+    MVI(0xc0002560, "TODO_c000_2560", 0),
+    MVI(0xc0002561, "TODO_c000_2561", 0),
+    MVI(0xc0002562, "TODO_c000_2562", 0),
+    MVI(0xc0002563, "TODO_c000_2563", 0),
+    MVI(0xc0002564, "TODO_c000_2564", 0),
+    MVI(0xc0002565, "TODO_c000_2565", 0),
+    MVI(0xc0002566, "TODO_c000_2566", 0),
+    MVI(0xc0002567, "TODO_c000_2567", 0),
+    MVI(0xc0002568, "TODO_c000_2568", 0),
+    MVI(0xc0002569, "TODO_c000_2569", 0),
+    MVI(0xc000256a, "TODO_c000_256a", 0),
+    MVI(0xc000256b, "TODO_c000_256b", 0),
+    MVI(0xc000256c, "TODO_c000_256c", 0),
+    MVI(0xc000256d, "TODO_c000_256d", 0),
+    MVI(0xc000256e, "TODO_c000_256e", 0),
+    MVI(0xc000256f, "TODO_c000_256f", 0),
+    MVI(0xc0002570, "TODO_c000_2570", 0),
+    MVI(0xc0002571, "TODO_c000_2571", 0),
+    MVI(0xc0002572, "TODO_c000_2572", 0),
+    MVI(0xc0002573, "TODO_c000_2573", 0),
+    MVI(0xc0002574, "TODO_c000_2574", 0),
+    MVI(0xc0002575, "TODO_c000_2575", 0),
+    MVI(0xc0002576, "TODO_c000_2576", 0),
+    MVI(0xc0002577, "TODO_c000_2577", 0),
+    MVI(0xc0002578, "TODO_c000_2578", 0),
+    MVI(0xc0002579, "TODO_c000_2579", 0),
+    MVI(0xc000257a, "TODO_c000_257a", 0),
+    MVI(0xc000257b, "TODO_c000_257b", 0),
+    MVI(0xc000257c, "TODO_c000_257c", 0),
+    MVI(0xc000257d, "TODO_c000_257d", 0),
+    MVI(0xc000257e, "TODO_c000_257e", 0),
+    MVI(0xc000257f, "TODO_c000_257f", 0),
+    MVI(0xc0002580, "TODO_c000_2580", 0),
+    MVI(0xc0002581, "TODO_c000_2581", 0),
+    MVI(0xc0002582, "TODO_c000_2582", 0),
+    MVI(0xc0002583, "TODO_c000_2583", 0),
+    MVI(0xc0002584, "TODO_c000_2584", 0),
+    MVI(0xc0002585, "TODO_c000_2585", 0),
+    MVI(0xc0002586, "TODO_c000_2586", 0),
+    MVI(0xc0002587, "TODO_c000_2587", 0),
+    MVI(0xc0002588, "TODO_c000_2588", 0),
+    MVI(0xc0002589, "TODO_c000_2589", 0),
+    MVI(0xc000258a, "TODO_c000_258a", 0),
+    MVI(0xc000258b, "TODO_c000_258b", 0),
+    MVI(0xc000258c, "TODO_c000_258c", 0),
+    MVI(0xc000258d, "TODO_c000_258d", 0),
+    MVI(0xc000258e, "TODO_c000_258e", 0),
+    MVI(0xc000258f, "TODO_c000_258f", 0),
+    MVI(0xc0002590, "TODO_c000_2590", 0),
+    MVI(0xc0002591, "TODO_c000_2591", 0),
+    MVI(0xc0002592, "TODO_c000_2592", 0),
+    MVI(0xc0002593, "TODO_c000_2593", 0),
+    MVI(0xc0002594, "TODO_c000_2594", 0),
+    MVI(0xc0002595, "TODO_c000_2595", 0),
+    MVI(0xc0002596, "TODO_c000_2596", 0),
+    MVI(0xc0002597, "TODO_c000_2597", 0),
+    MVI(0xc0002598, "TODO_c000_2598", 0),
+    MVI(0xc0002599, "TODO_c000_2599", 0),
+    MVI(0xc000259a, "TODO_c000_259a", 0),
+    MVI(0xc000259b, "TODO_c000_259b", 0),
+    MVI(0xc000259c, "TODO_c000_259c", 0),
+    MVI(0xc000259d, "TODO_c000_259d", 0),
+    MVI(0xc000259e, "TODO_c000_259e", 0),
+    MVI(0xc000259f, "TODO_c000_259f", 0),
+    MVI(0xc00025a0, "TODO_c000_25a0", 0),
+    MVI(0xc00025a1, "TODO_c000_25a1", 0),
+    MVI(0xc00025a2, "TODO_c000_25a2", 0),
+    MVI(0xc00025a3, "TODO_c000_25a3", 0),
+    MVI(0xc00025a4, "TODO_c000_25a4", 0),
+    MVI(0xc00025a5, "TODO_c000_25a5", 0),
+    MVI(0xc00025a6, "TODO_c000_25a6", 0),
+    MVI(0xc00025a7, "TODO_c000_25a7", 0),
+    MVI(0xc00025a8, "TODO_c000_25a8", 0),
+    MVI(0xc00025a9, "TODO_c000_25a9", 0),
+    MVI(0xc00025aa, "TODO_c000_25aa", 0),
+    MVI(0xc00025ab, "TODO_c000_25ab", 0),
+    MVI(0xc00025ac, "TODO_c000_25ac", 0),
+    MVI(0xc00025ad, "TODO_c000_25ad", 0),
+    MVI(0xc00025ae, "TODO_c000_25ae", 0),
+    MVI(0xc00025af, "TODO_c000_25af", 0),
+    MVI(0xc00025b0, "TODO_c000_25b0", 0),
+    MVI(0xc00025b1, "TODO_c000_25b1", 0),
+    MVI(0xc00025b2, "TODO_c000_25b2", 0),
+    MVI(0xc00025b3, "TODO_c000_25b3", 0),
+    MVI(0xc00025b4, "TODO_c000_25b4", 0),
+    MVI(0xc00025b5, "TODO_c000_25b5", 0),
+    MVI(0xc00025b6, "TODO_c000_25b6", 0),
+    MVI(0xc00025b7, "TODO_c000_25b7", 0),
+    MVI(0xc00025b8, "TODO_c000_25b8", 0),
+    MVI(0xc00025b9, "TODO_c000_25b9", 0),
+    MVI(0xc00025ba, "TODO_c000_25ba", 0),
+    MVI(0xc00025bb, "TODO_c000_25bb", 0),
+    MVI(0xc00025bc, "TODO_c000_25bc", 0),
+    MVI(0xc00025bd, "TODO_c000_25bd", 0),
+    MVI(0xc00025be, "TODO_c000_25be", 0),
+    MVI(0xc00025bf, "TODO_c000_25bf", 0),
+    MVI(0xc00025c0, "TODO_c000_25c0", 0),
+    MVI(0xc00025c1, "TODO_c000_25c1", 0),
+    MVI(0xc00025c2, "TODO_c000_25c2", 0),
+    MVI(0xc00025c3, "TODO_c000_25c3", 0),
+    MVI(0xc00025c4, "TODO_c000_25c4", 0),
+    MVI(0xc00025c5, "TODO_c000_25c5", 0),
+    MVI(0xc00025c6, "TODO_c000_25c6", 0),
+    MVI(0xc00025c7, "TODO_c000_25c7", 0),
+    MVI(0xc00025c8, "TODO_c000_25c8", 0),
+    MVI(0xc00025c9, "TODO_c000_25c9", 0),
+    MVI(0xc00025ca, "TODO_c000_25ca", 0),
+    MVI(0xc00025cb, "TODO_c000_25cb", 0),
+    MVI(0xc00025cc, "TODO_c000_25cc", 0),
+    MVI(0xc00025cd, "TODO_c000_25cd", 0),
+    MVI(0xc00025ce, "TODO_c000_25ce", 0),
+    MVI(0xc00025cf, "TODO_c000_25cf", 0),
+    MVI(0xc00025d0, "TODO_c000_25d0", 0),
+    MVI(0xc00025d1, "TODO_c000_25d1", 0),
+    MVI(0xc00025d2, "TODO_c000_25d2", 0),
+    MVI(0xc00025d3, "TODO_c000_25d3", 0),
+    MVI(0xc00025d4, "TODO_c000_25d4", 0),
+    MVI(0xc00025d5, "TODO_c000_25d5", 0),
+    MVI(0xc00025d6, "TODO_c000_25d6", 0),
+    MVI(0xc00025d7, "TODO_c000_25d7", 0),
+    MVI(0xc00025d8, "TODO_c000_25d8", 0),
+    MVI(0xc00025d9, "TODO_c000_25d9", 0),
+    MVI(0xc00025da, "TODO_c000_25da", 0),
+    MVI(0xc00025db, "TODO_c000_25db", 0),
+    MVI(0xc00025dc, "TODO_c000_25dc", 0),
+    MVI(0xc00025dd, "TODO_c000_25dd", 0),
+    MVI(0xc00025de, "TODO_c000_25de", 0),
+    MVI(0xc00025df, "TODO_c000_25df", 0),
+    MVI(0xc00025e0, "TODO_c000_25e0", 0),
+    MVI(0xc00025e1, "TODO_c000_25e1", 0),
+    MVI(0xc00025e2, "TODO_c000_25e2", 0),
+    MVI(0xc00025e3, "TODO_c000_25e3", 0),
+    MVI(0xc00025e4, "TODO_c000_25e4", 0),
+    MVI(0xc00025e5, "TODO_c000_25e5", 0),
+    MVI(0xc00025e6, "TODO_c000_25e6", 0),
+    MVI(0xc00025e7, "TODO_c000_25e7", 0),
+    MVI(0xc00025e8, "TODO_c000_25e8", 0),
+    MVI(0xc00025e9, "TODO_c000_25e9", 0),
+    MVI(0xc00025ea, "TODO_c000_25ea", 0),
+    MVI(0xc00025eb, "TODO_c000_25eb", 0),
+    MVI(0xc00025ec, "TODO_c000_25ec", 0),
+    MVI(0xc00025ed, "TODO_c000_25ed", 0),
+    MVI(0xc00025ee, "TODO_c000_25ee", 0),
+    MVI(0xc00025ef, "TODO_c000_25ef", 0),
+    MVI(0xc00025f0, "TODO_c000_25f0", 0),
+    MVI(0xc00025f1, "TODO_c000_25f1", 0),
+    MVI(0xc00025f2, "TODO_c000_25f2", 0),
+    MVI(0xc00025f3, "TODO_c000_25f3", 0),
+    MVI(0xc00025f4, "TODO_c000_25f4", 0),
+    MVI(0xc00025f5, "TODO_c000_25f5", 0),
+    MVI(0xc00025f6, "TODO_c000_25f6", 0),
+    MVI(0xc00025f7, "TODO_c000_25f7", 0),
+    MVI(0xc00025f8, "TODO_c000_25f8", 0),
+    MVI(0xc00025f9, "TODO_c000_25f9", 0),
+    MVI(0xc00025fa, "TODO_c000_25fa", 0),
+    MVI(0xc00025fb, "TODO_c000_25fb", 0),
+    MVI(0xc00025fc, "TODO_c000_25fc", 0),
+    MVI(0xc00025fd, "TODO_c000_25fd", 0),
+    MVI(0xc00025fe, "TODO_c000_25fe", 0),
+    MVI(0xc00025ff, "TODO_c000_25ff", 0),
+    MVI(0xc0002600, "TODO_c000_2600", 0),
+    MVI(0xc0002601, "TODO_c000_2601", 0),
+    MVI(0xc0002602, "TODO_c000_2602", 0),
+    MVI(0xc0002603, "TODO_c000_2603", 0),
+    MVI(0xc0002604, "TODO_c000_2604", 0),
+    MVI(0xc0002605, "TODO_c000_2605", 0),
+    MVI(0xc0002606, "TODO_c000_2606", 0),
+    MVI(0xc0002607, "TODO_c000_2607", 0),
+    MVI(0xc0002608, "TODO_c000_2608", 0),
+    MVI(0xc0002609, "TODO_c000_2609", 0),
+    MVI(0xc000260a, "TODO_c000_260a", 0),
+    MVI(0xc000260b, "TODO_c000_260b", 0),
+    MVI(0xc000260c, "TODO_c000_260c", 0),
+    MVI(0xc000260d, "TODO_c000_260d", 0),
+    MVI(0xc000260e, "TODO_c000_260e", 0),
+    MVI(0xc000260f, "TODO_c000_260f", 0),
+    MVI(0xc0002610, "TODO_c000_2610", 0),
+    MVI(0xc0002611, "TODO_c000_2611", 0),
+    MVI(0xc0002612, "TODO_c000_2612", 0),
+    MVI(0xc0002613, "TODO_c000_2613", 0),
+    MVI(0xc0002614, "TODO_c000_2614", 0),
+    MVI(0xc0002615, "TODO_c000_2615", 0),
+    MVI(0xc0002616, "TODO_c000_2616", 0),
+    MVI(0xc0002617, "TODO_c000_2617", 0),
+    MVI(0xc0002618, "TODO_c000_2618", 0),
+    MVI(0xc0002619, "TODO_c000_2619", 0),
+    MVI(0xc000261a, "TODO_c000_261a", 0),
+    MVI(0xc000261b, "TODO_c000_261b", 0),
+    MVI(0xc000261c, "TODO_c000_261c", 0),
+    MVI(0xc000261d, "TODO_c000_261d", 0),
+    MVI(0xc000261e, "TODO_c000_261e", 0),
+    MVI(0xc000261f, "TODO_c000_261f", 0),
+    MVI(0xc0002620, "TODO_c000_2620", 0),
+    MVI(0xc0002621, "TODO_c000_2621", 0),
+    MVI(0xc0002622, "TODO_c000_2622", 0),
+    MVI(0xc0002623, "TODO_c000_2623", 0),
+    MVI(0xc0002624, "TODO_c000_2624", 0),
+    MVI(0xc0002625, "TODO_c000_2625", 0),
+    MVI(0xc0002626, "TODO_c000_2626", 0),
+    MVI(0xc0002627, "TODO_c000_2627", 0),
+    MVI(0xc0002628, "TODO_c000_2628", 0),
+    MVI(0xc0002629, "TODO_c000_2629", 0),
+    MVI(0xc000262a, "TODO_c000_262a", 0),
+    MVI(0xc000262b, "TODO_c000_262b", 0),
+    MVI(0xc000262c, "TODO_c000_262c", 0),
+    MVI(0xc000262d, "TODO_c000_262d", 0),
+    MVI(0xc000262e, "TODO_c000_262e", 0),
+    MVI(0xc000262f, "TODO_c000_262f", 0),
+    MVI(0xc0002630, "TODO_c000_2630", 0),
+    MVI(0xc0002631, "TODO_c000_2631", 0),
+    MVI(0xc0002632, "TODO_c000_2632", 0),
+    MVI(0xc0002633, "TODO_c000_2633", 0),
+    MVI(0xc0002634, "TODO_c000_2634", 0),
+    MVI(0xc0002635, "TODO_c000_2635", 0),
+    MVI(0xc0002636, "TODO_c000_2636", 0),
+    MVI(0xc0002637, "TODO_c000_2637", 0),
+    MVI(0xc0002638, "TODO_c000_2638", 0),
+    MVI(0xc0002639, "TODO_c000_2639", 0),
+    MVI(0xc000263a, "TODO_c000_263a", 0),
+    MVI(0xc000263b, "TODO_c000_263b", 0),
+    MVI(0xc000263c, "TODO_c000_263c", 0),
+    MVI(0xc000263d, "TODO_c000_263d", 0),
+    MVI(0xc000263e, "TODO_c000_263e", 0),
+    MVI(0xc000263f, "TODO_c000_263f", 0),
+    MVI(0xc0002640, "TODO_c000_2640", 0),
+    MVI(0xc0002641, "TODO_c000_2641", 0),
+    MVI(0xc0002642, "TODO_c000_2642", 0),
+    MVI(0xc0002643, "TODO_c000_2643", 0),
+    MVI(0xc0002644, "TODO_c000_2644", 0),
+    MVI(0xc0002645, "TODO_c000_2645", 0),
+    MVI(0xc0002646, "TODO_c000_2646", 0),
+    MVI(0xc0002647, "TODO_c000_2647", 0),
+    MVI(0xc0002648, "TODO_c000_2648", 0),
+    MVI(0xc0002649, "TODO_c000_2649", 0),
+    MVI(0xc000264a, "TODO_c000_264a", 0),
+    MVI(0xc000264b, "TODO_c000_264b", 0),
+    MVI(0xc000264c, "TODO_c000_264c", 0),
+    MVI(0xc000264d, "TODO_c000_264d", 0),
+    MVI(0xc000264e, "TODO_c000_264e", 0),
+    MVI(0xc000264f, "TODO_c000_264f", 0),
+    MVI(0xc0002650, "TODO_c000_2650", 0),
+    MVI(0xc0002651, "TODO_c000_2651", 0),
+    MVI(0xc0002652, "TODO_c000_2652", 0),
+    MVI(0xc0002653, "TODO_c000_2653", 0),
+    MVI(0xc0002654, "TODO_c000_2654", 0),
+    MVI(0xc0002655, "TODO_c000_2655", 0),
+    MVI(0xc0002656, "TODO_c000_2656", 0),
+    MVI(0xc0002657, "TODO_c000_2657", 0),
+    MVI(0xc0002658, "TODO_c000_2658", 0),
+    MVI(0xc0002659, "TODO_c000_2659", 0),
+    MVI(0xc000265a, "TODO_c000_265a", 0),
+    MVI(0xc000265b, "TODO_c000_265b", 0),
+    MVI(0xc000265c, "TODO_c000_265c", 0),
+    MVI(0xc000265d, "TODO_c000_265d", 0),
+    MVI(0xc000265e, "TODO_c000_265e", 0),
+    MVI(0xc000265f, "TODO_c000_265f", 0),
+    MVI(0xc0002660, "TODO_c000_2660", 0),
+    MVI(0xc0002661, "TODO_c000_2661", 0),
+    MVI(0xc0002662, "TODO_c000_2662", 0),
+    MVI(0xc0002663, "TODO_c000_2663", 0),
+    MVI(0xc0002664, "TODO_c000_2664", 0),
+    MVI(0xc0002665, "TODO_c000_2665", 0),
+    MVI(0xc0002666, "TODO_c000_2666", 0),
+    MVI(0xc0002667, "TODO_c000_2667", 0),
+    MVI(0xc0002668, "TODO_c000_2668", 0),
+    MVI(0xc0002669, "TODO_c000_2669", 0),
+    MVI(0xc000266a, "TODO_c000_266a", 0),
+    MVI(0xc000266b, "TODO_c000_266b", 0),
+    MVI(0xc000266c, "TODO_c000_266c", 0),
+    MVI(0xc000266d, "TODO_c000_266d", 0),
+    MVI(0xc000266e, "TODO_c000_266e", 0),
+    MVI(0xc000266f, "TODO_c000_266f", 0),
+    MVI(0xc0002670, "TODO_c000_2670", 0),
+    MVI(0xc0002671, "TODO_c000_2671", 0),
+    MVI(0xc0002672, "TODO_c000_2672", 0),
+    MVI(0xc0002673, "TODO_c000_2673", 0),
+    MVI(0xc0002674, "TODO_c000_2674", 0),
+    MVI(0xc0002675, "TODO_c000_2675", 0),
+    MVI(0xc0002676, "TODO_c000_2676", 0),
+    MVI(0xc0002677, "TODO_c000_2677", 0),
+    MVI(0xc0002678, "TODO_c000_2678", 0),
+    MVI(0xc0002679, "TODO_c000_2679", 0),
+    MVI(0xc000267a, "TODO_c000_267a", 0),
+    MVI(0xc000267b, "TODO_c000_267b", 0),
+    MVI(0xc000267c, "TODO_c000_267c", 0),
+    MVI(0xc000267d, "TODO_c000_267d", 0),
+    MVI(0xc000267e, "TODO_c000_267e", 0),
+    MVI(0xc000267f, "TODO_c000_267f", 0),
+    MVI(0xc0002680, "TODO_c000_2680", 0),
+    MVI(0xc0002681, "TODO_c000_2681", 0),
+    MVI(0xc0002682, "TODO_c000_2682", 0),
+    MVI(0xc0002683, "TODO_c000_2683", 0),
+    MVI(0xc0002684, "TODO_c000_2684", 0),
+    MVI(0xc0002685, "TODO_c000_2685", 0),
+    MVI(0xc0002686, "TODO_c000_2686", 0),
+    MVI(0xc0002687, "TODO_c000_2687", 0),
+    MVI(0xc0002688, "TODO_c000_2688", 0),
+    MVI(0xc0002689, "TODO_c000_2689", 0),
+    MVI(0xc000268a, "TODO_c000_268a", 0),
+    MVI(0xc000268b, "TODO_c000_268b", 0),
+    MVI(0xc000268c, "TODO_c000_268c", 0),
+    MVI(0xc000268d, "TODO_c000_268d", 0),
+    MVI(0xc000268e, "TODO_c000_268e", 0),
+    MVI(0xc000268f, "TODO_c000_268f", 0),
+    MVI(0xc0002690, "TODO_c000_2690", 0),
+    MVI(0xc0002691, "TODO_c000_2691", 0),
+    MVI(0xc0002692, "TODO_c000_2692", 0),
+    MVI(0xc0002693, "TODO_c000_2693", 0),
+    MVI(0xc0002694, "TODO_c000_2694", 0),
+    MVI(0xc0002695, "TODO_c000_2695", 0),
+    MVI(0xc0002696, "TODO_c000_2696", 0),
+    MVI(0xc0002697, "TODO_c000_2697", 0),
+    MVI(0xc0002698, "TODO_c000_2698", 0),
+    MVI(0xc0002699, "TODO_c000_2699", 0),
+    MVI(0xc000269a, "TODO_c000_269a", 0),
+    MVI(0xc000269b, "TODO_c000_269b", 0),
+    MVI(0xc000269c, "TODO_c000_269c", 0),
+    MVI(0xc000269d, "TODO_c000_269d", 0),
+    MVI(0xc000269e, "TODO_c000_269e", 0),
+    MVI(0xc000269f, "TODO_c000_269f", 0),
+    MVI(0xc00026a0, "TODO_c000_26a0", 0),
+    MVI(0xc00026a1, "TODO_c000_26a1", 0),
+    MVI(0xc00026a2, "TODO_c000_26a2", 0),
+    MVI(0xc00026a3, "TODO_c000_26a3", 0),
+    MVI(0xc00026a4, "TODO_c000_26a4", 0),
+    MVI(0xc00026a5, "TODO_c000_26a5", 0),
+    MVI(0xc00026a6, "TODO_c000_26a6", 0),
+    MVI(0xc00026a7, "TODO_c000_26a7", 0),
+    MVI(0xc00026a8, "TODO_c000_26a8", 0),
+    MVI(0xc00026a9, "TODO_c000_26a9", 0),
+    MVI(0xc00026aa, "TODO_c000_26aa", 0),
+    MVI(0xc00026ab, "TODO_c000_26ab", 0),
+    MVI(0xc00026ac, "TODO_c000_26ac", 0),
+    MVI(0xc00026ad, "TODO_c000_26ad", 0),
+    MVI(0xc00026ae, "TODO_c000_26ae", 0),
+    MVI(0xc00026af, "TODO_c000_26af", 0),
+    MVI(0xc00026b0, "TODO_c000_26b0", 0),
+    MVI(0xc00026b1, "TODO_c000_26b1", 0),
+    MVI(0xc00026b2, "TODO_c000_26b2", 0),
+    MVI(0xc00026b3, "TODO_c000_26b3", 0),
+    MVI(0xc00026b4, "TODO_c000_26b4", 0),
+    MVI(0xc00026b5, "TODO_c000_26b5", 0),
+    MVI(0xc00026b6, "TODO_c000_26b6", 0),
+    MVI(0xc00026b7, "TODO_c000_26b7", 0),
+    MVI(0xc00026b8, "TODO_c000_26b8", 0),
+    MVI(0xc00026b9, "TODO_c000_26b9", 0),
+    MVI(0xc00026ba, "TODO_c000_26ba", 0),
+    MVI(0xc00026bb, "TODO_c000_26bb", 0),
+    MVI(0xc00026bc, "TODO_c000_26bc", 0),
+    MVI(0xc00026bd, "TODO_c000_26bd", 0),
+    MVI(0xc00026be, "TODO_c000_26be", 0),
+    MVI(0xc00026bf, "TODO_c000_26bf", 0),
+    MVI(0xc00026c0, "TODO_c000_26c0", 0),
+    MVI(0xc00026c1, "TODO_c000_26c1", 0),
+    MVI(0xc00026c2, "TODO_c000_26c2", 0),
+    MVI(0xc00026c3, "TODO_c000_26c3", 0),
+    MVI(0xc00026c4, "TODO_c000_26c4", 0),
+    MVI(0xc00026c5, "TODO_c000_26c5", 0),
+    MVI(0xc00026c6, "TODO_c000_26c6", 0),
+    MVI(0xc00026c7, "TODO_c000_26c7", 0),
+    MVI(0xc00026c8, "TODO_c000_26c8", 0),
+    MVI(0xc00026c9, "TODO_c000_26c9", 0),
+    MVI(0xc00026ca, "TODO_c000_26ca", 0),
+    MVI(0xc00026cb, "TODO_c000_26cb", 0),
+    MVI(0xc00026cc, "TODO_c000_26cc", 0),
+    MVI(0xc00026cd, "TODO_c000_26cd", 0),
+    MVI(0xc00026ce, "TODO_c000_26ce", 0),
+    MVI(0xc00026cf, "TODO_c000_26cf", 0),
+    MVI(0xc00026d0, "TODO_c000_26d0", 0),
+    MVI(0xc00026d1, "TODO_c000_26d1", 0),
+    MVI(0xc00026d2, "TODO_c000_26d2", 0),
+    MVI(0xc00026d3, "TODO_c000_26d3", 0),
+    MVI(0xc00026d4, "TODO_c000_26d4", 0),
+    MVI(0xc00026d5, "TODO_c000_26d5", 0),
+    MVI(0xc00026d6, "TODO_c000_26d6", 0),
+    MVI(0xc00026d7, "TODO_c000_26d7", 0),
+    MVI(0xc00026d8, "TODO_c000_26d8", 0),
+    MVI(0xc00026d9, "TODO_c000_26d9", 0),
+    MVI(0xc00026da, "TODO_c000_26da", 0),
+    MVI(0xc00026db, "TODO_c000_26db", 0),
+    MVI(0xc00026dc, "TODO_c000_26dc", 0),
+    MVI(0xc00026dd, "TODO_c000_26dd", 0),
+    MVI(0xc00026de, "TODO_c000_26de", 0),
+    MVI(0xc00026df, "TODO_c000_26df", 0),
+    MVI(0xc00026e0, "TODO_c000_26e0", 0),
+    MVI(0xc00026e1, "TODO_c000_26e1", 0),
+    MVI(0xc00026e2, "TODO_c000_26e2", 0),
+    MVI(0xc00026e3, "TODO_c000_26e3", 0),
+    MVI(0xc00026e4, "TODO_c000_26e4", 0),
+    MVI(0xc00026e5, "TODO_c000_26e5", 0),
+    MVI(0xc00026e6, "TODO_c000_26e6", 0),
+    MVI(0xc00026e7, "TODO_c000_26e7", 0),
+    MVI(0xc00026e8, "TODO_c000_26e8", 0),
+    MVI(0xc00026e9, "TODO_c000_26e9", 0),
+    MVI(0xc00026ea, "TODO_c000_26ea", 0),
+    MVI(0xc00026eb, "TODO_c000_26eb", 0),
+    MVI(0xc00026ec, "TODO_c000_26ec", 0),
+    MVI(0xc00026ed, "TODO_c000_26ed", 0),
+    MVI(0xc00026ee, "TODO_c000_26ee", 0),
+    MVI(0xc00026ef, "TODO_c000_26ef", 0),
+    MVI(0xc00026f0, "TODO_c000_26f0", 0),
+    MVI(0xc00026f1, "TODO_c000_26f1", 0),
+    MVI(0xc00026f2, "TODO_c000_26f2", 0),
+    MVI(0xc00026f3, "TODO_c000_26f3", 0),
+    MVI(0xc00026f4, "TODO_c000_26f4", 0),
+    MVI(0xc00026f5, "TODO_c000_26f5", 0),
+    MVI(0xc00026f6, "TODO_c000_26f6", 0),
+    MVI(0xc00026f7, "TODO_c000_26f7", 0),
+    MVI(0xc00026f8, "TODO_c000_26f8", 0),
+    MVI(0xc00026f9, "TODO_c000_26f9", 0),
+    MVI(0xc00026fa, "TODO_c000_26fa", 0),
+    MVI(0xc00026fb, "TODO_c000_26fb", 0),
+    MVI(0xc00026fc, "TODO_c000_26fc", 0),
+    MVI(0xc00026fd, "TODO_c000_26fd", 0),
+    MVI(0xc00026fe, "TODO_c000_26fe", 0),
+    MVI(0xc00026ff, "TODO_c000_26ff", 0),
+    MVI(0xc0002700, "TODO_c000_2700", 0),
+    MVI(0xc0002701, "TODO_c000_2701", 0),
+    MVI(0xc0002702, "TODO_c000_2702", 0),
+    MVI(0xc0002703, "TODO_c000_2703", 0),
+    MVI(0xc0002704, "TODO_c000_2704", 0),
+    MVI(0xc0002705, "TODO_c000_2705", 0),
+    MVI(0xc0002706, "TODO_c000_2706", 0),
+    MVI(0xc0002707, "TODO_c000_2707", 0),
+    MVI(0xc0002708, "TODO_c000_2708", 0),
+    MVI(0xc0002709, "TODO_c000_2709", 0),
+    MVI(0xc000270a, "TODO_c000_270a", 0),
+    MVI(0xc000270b, "TODO_c000_270b", 0),
+    MVI(0xc000270c, "TODO_c000_270c", 0),
+    MVI(0xc000270d, "TODO_c000_270d", 0),
+    MVI(0xc000270e, "TODO_c000_270e", 0),
+    MVI(0xc000270f, "TODO_c000_270f", 0),
+    MVI(0xc0002710, "TODO_c000_2710", 0),
+    MVI(0xc0002711, "TODO_c000_2711", 0),
+    MVI(0xc0002712, "TODO_c000_2712", 0),
+    MVI(0xc0002713, "TODO_c000_2713", 0),
+    MVI(0xc0002714, "TODO_c000_2714", 0),
+    MVI(0xc0002715, "TODO_c000_2715", 0),
+    MVI(0xc0002716, "TODO_c000_2716", 0),
+    MVI(0xc0002717, "TODO_c000_2717", 0),
+    MVI(0xc0002718, "TODO_c000_2718", 0),
+    MVI(0xc0002719, "TODO_c000_2719", 0),
+    MVI(0xc000271a, "TODO_c000_271a", 0),
+    MVI(0xc000271b, "TODO_c000_271b", 0),
+    MVI(0xc000271c, "TODO_c000_271c", 0),
+    MVI(0xc000271d, "TODO_c000_271d", 0),
+    MVI(0xc000271e, "TODO_c000_271e", 0),
+    MVI(0xc000271f, "TODO_c000_271f", 0),
+    MVI(0xc0002720, "TODO_c000_2720", 0),
+    MVI(0xc0002721, "TODO_c000_2721", 0),
+    MVI(0xc0002722, "TODO_c000_2722", 0),
+    MVI(0xc0002723, "TODO_c000_2723", 0),
+    MVI(0xc0002724, "TODO_c000_2724", 0),
+    MVI(0xc0002725, "TODO_c000_2725", 0),
+    MVI(0xc0002726, "TODO_c000_2726", 0),
+    MVI(0xc0002727, "TODO_c000_2727", 0),
+    MVI(0xc0002728, "TODO_c000_2728", 0),
+    MVI(0xc0002729, "TODO_c000_2729", 0),
+    MVI(0xc000272a, "TODO_c000_272a", 0),
+    MVI(0xc000272b, "TODO_c000_272b", 0),
+    MVI(0xc000272c, "TODO_c000_272c", 0),
+    MVI(0xc000272d, "TODO_c000_272d", 0),
+    MVI(0xc000272e, "TODO_c000_272e", 0),
+    MVI(0xc000272f, "TODO_c000_272f", 0),
+    MVI(0xc0002730, "TODO_c000_2730", 0),
+    MVI(0xc0002731, "TODO_c000_2731", 0),
+    MVI(0xc0002732, "TODO_c000_2732", 0),
+    MVI(0xc0002733, "TODO_c000_2733", 0),
+    MVI(0xc0002734, "TODO_c000_2734", 0),
+    MVI(0xc0002735, "TODO_c000_2735", 0),
+    MVI(0xc0002736, "TODO_c000_2736", 0),
+    MVI(0xc0002737, "TODO_c000_2737", 0),
+    MVI(0xc0002738, "TODO_c000_2738", 0),
+    MVI(0xc0002739, "TODO_c000_2739", 0),
+    MVI(0xc000273a, "TODO_c000_273a", 0),
+    MVI(0xc000273b, "TODO_c000_273b", 0),
+    MVI(0xc000273c, "TODO_c000_273c", 0),
+    MVI(0xc000273d, "TODO_c000_273d", 0),
+    MVI(0xc000273e, "TODO_c000_273e", 0),
+    MVI(0xc000273f, "TODO_c000_273f", 0),
+    MVI(0xc0002740, "TODO_c000_2740", 0),
+    MVI(0xc0002741, "TODO_c000_2741", 0),
+    MVI(0xc0002742, "TODO_c000_2742", 0),
+    MVI(0xc0002743, "TODO_c000_2743", 0),
+    MVI(0xc0002744, "TODO_c000_2744", 0),
+    MVI(0xc0002745, "TODO_c000_2745", 0),
+    MVI(0xc0002746, "TODO_c000_2746", 0),
+    MVI(0xc0002747, "TODO_c000_2747", 0),
+    MVI(0xc0002748, "TODO_c000_2748", 0),
+    MVI(0xc0002749, "TODO_c000_2749", 0),
+    MVI(0xc000274a, "TODO_c000_274a", 0),
+    MVI(0xc000274b, "TODO_c000_274b", 0),
+    MVI(0xc000274c, "TODO_c000_274c", 0),
+    MVI(0xc000274d, "TODO_c000_274d", 0),
+    MVI(0xc000274e, "TODO_c000_274e", 0),
+    MVI(0xc000274f, "TODO_c000_274f", 0),
+    MVI(0xc0002750, "TODO_c000_2750", 0),
+    MVI(0xc0002751, "TODO_c000_2751", 0),
+    MVI(0xc0002752, "TODO_c000_2752", 0),
+    MVI(0xc0002753, "TODO_c000_2753", 0),
+    MVI(0xc0002754, "TODO_c000_2754", 0),
+    MVI(0xc0002755, "TODO_c000_2755", 0),
+    MVI(0xc0002756, "TODO_c000_2756", 0),
+    MVI(0xc0002757, "TODO_c000_2757", 0),
+    MVI(0xc0002758, "TODO_c000_2758", 0),
+    MVI(0xc0002759, "TODO_c000_2759", 0),
+    MVI(0xc000275a, "TODO_c000_275a", 0),
+    MVI(0xc000275b, "TODO_c000_275b", 0),
+    MVI(0xc000275c, "TODO_c000_275c", 0),
+    MVI(0xc000275d, "TODO_c000_275d", 0),
+    MVI(0xc000275e, "TODO_c000_275e", 0),
+    MVI(0xc000275f, "TODO_c000_275f", 0),
+    MVI(0xc0002760, "TODO_c000_2760", 0),
+    MVI(0xc0002761, "TODO_c000_2761", 0),
+    MVI(0xc0002762, "TODO_c000_2762", 0),
+    MVI(0xc0002763, "TODO_c000_2763", 0),
+    MVI(0xc0002764, "TODO_c000_2764", 0),
+    MVI(0xc0002765, "TODO_c000_2765", 0),
+    MVI(0xc0002766, "TODO_c000_2766", 0),
+    MVI(0xc0002767, "TODO_c000_2767", 0),
+    MVI(0xc0002768, "TODO_c000_2768", 0),
+    MVI(0xc0002769, "TODO_c000_2769", 0),
+    MVI(0xc000276a, "TODO_c000_276a", 0),
+    MVI(0xc000276b, "TODO_c000_276b", 0),
+    MVI(0xc000276c, "TODO_c000_276c", 0),
+    MVI(0xc000276d, "TODO_c000_276d", 0),
+    MVI(0xc000276e, "TODO_c000_276e", 0),
+    MVI(0xc000276f, "TODO_c000_276f", 0),
+    MVI(0xc0002770, "TODO_c000_2770", 0),
+    MVI(0xc0002771, "TODO_c000_2771", 0),
+    MVI(0xc0002772, "TODO_c000_2772", 0),
+    MVI(0xc0002773, "TODO_c000_2773", 0),
+    MVI(0xc0002774, "TODO_c000_2774", 0),
+    MVI(0xc0002775, "TODO_c000_2775", 0),
+    MVI(0xc0002776, "TODO_c000_2776", 0),
+    MVI(0xc0002777, "TODO_c000_2777", 0),
+    MVI(0xc0002778, "TODO_c000_2778", 0),
+    MVI(0xc0002779, "TODO_c000_2779", 0),
+    MVI(0xc000277a, "TODO_c000_277a", 0),
+    MVI(0xc000277b, "TODO_c000_277b", 0),
+    MVI(0xc000277c, "TODO_c000_277c", 0),
+    MVI(0xc000277d, "TODO_c000_277d", 0),
+    MVI(0xc000277e, "TODO_c000_277e", 0),
+    MVI(0xc000277f, "TODO_c000_277f", 0),
+    MVI(0xc0002780, "TODO_c000_2780", 0),
+    MVI(0xc0002781, "TODO_c000_2781", 0),
+    MVI(0xc0002782, "TODO_c000_2782", 0),
+    MVI(0xc0002783, "TODO_c000_2783", 0),
+    MVI(0xc0002784, "TODO_c000_2784", 0),
+    MVI(0xc0002785, "TODO_c000_2785", 0),
+    MVI(0xc0002786, "TODO_c000_2786", 0),
+    MVI(0xc0002787, "TODO_c000_2787", 0),
+    MVI(0xc0002788, "TODO_c000_2788", 0),
+    MVI(0xc0002789, "TODO_c000_2789", 0),
+    MVI(0xc000278a, "TODO_c000_278a", 0),
+    MVI(0xc000278b, "TODO_c000_278b", 0),
+    MVI(0xc000278c, "TODO_c000_278c", 0),
+    MVI(0xc000278d, "TODO_c000_278d", 0),
+    MVI(0xc000278e, "TODO_c000_278e", 0),
+    MVI(0xc000278f, "TODO_c000_278f", 0),
+    MVI(0xc0002790, "TODO_c000_2790", 0),
+    MVI(0xc0002791, "TODO_c000_2791", 0),
+    MVI(0xc0002792, "TODO_c000_2792", 0),
+    MVI(0xc0002793, "TODO_c000_2793", 0),
+    MVI(0xc0002794, "TODO_c000_2794", 0),
+    MVI(0xc0002795, "TODO_c000_2795", 0),
+    MVI(0xc0002796, "TODO_c000_2796", 0),
+    MVI(0xc0002797, "TODO_c000_2797", 0),
+    MVI(0xc0002798, "TODO_c000_2798", 0),
+    MVI(0xc0002799, "TODO_c000_2799", 0),
+    MVI(0xc000279a, "TODO_c000_279a", 0),
+    MVI(0xc000279b, "TODO_c000_279b", 0),
+    MVI(0xc000279c, "TODO_c000_279c", 0),
+    MVI(0xc000279d, "TODO_c000_279d", 0),
+    MVI(0xc000279e, "TODO_c000_279e", 0),
+    MVI(0xc000279f, "TODO_c000_279f", 0),
+    MVI(0xc00027a0, "TODO_c000_27a0", 0),
+    MVI(0xc00027a1, "TODO_c000_27a1", 0),
+    MVI(0xc00027a2, "TODO_c000_27a2", 0),
+    MVI(0xc00027a3, "TODO_c000_27a3", 0),
+    MVI(0xc00027a4, "TODO_c000_27a4", 0),
+    MVI(0xc00027a5, "TODO_c000_27a5", 0),
+    MVI(0xc00027a6, "TODO_c000_27a6", 0),
+    MVI(0xc00027a7, "TODO_c000_27a7", 0),
+    MVI(0xc00027a8, "TODO_c000_27a8", 0),
+    MVI(0xc00027a9, "TODO_c000_27a9", 0),
+    MVI(0xc00027aa, "TODO_c000_27aa", 0),
+    MVI(0xc00027ab, "TODO_c000_27ab", 0),
+    MVI(0xc00027ac, "TODO_c000_27ac", 0),
+    MVI(0xc00027ad, "TODO_c000_27ad", 0),
+    MVI(0xc00027ae, "TODO_c000_27ae", 0),
+    MVI(0xc00027af, "TODO_c000_27af", 0),
+    MVI(0xc00027b0, "TODO_c000_27b0", 0),
+    MVI(0xc00027b1, "TODO_c000_27b1", 0),
+    MVI(0xc00027b2, "TODO_c000_27b2", 0),
+    MVI(0xc00027b3, "TODO_c000_27b3", 0),
+    MVI(0xc00027b4, "TODO_c000_27b4", 0),
+    MVI(0xc00027b5, "TODO_c000_27b5", 0),
+    MVI(0xc00027b6, "TODO_c000_27b6", 0),
+    MVI(0xc00027b7, "TODO_c000_27b7", 0),
+    MVI(0xc00027b8, "TODO_c000_27b8", 0),
+    MVI(0xc00027b9, "TODO_c000_27b9", 0),
+    MVI(0xc00027ba, "TODO_c000_27ba", 0),
+    MVI(0xc00027bb, "TODO_c000_27bb", 0),
+    MVI(0xc00027bc, "TODO_c000_27bc", 0),
+    MVI(0xc00027bd, "TODO_c000_27bd", 0),
+    MVI(0xc00027be, "TODO_c000_27be", 0),
+    MVI(0xc00027bf, "TODO_c000_27bf", 0),
+    MVI(0xc00027c0, "TODO_c000_27c0", 0),
+    MVI(0xc00027c1, "TODO_c000_27c1", 0),
+    MVI(0xc00027c2, "TODO_c000_27c2", 0),
+    MVI(0xc00027c3, "TODO_c000_27c3", 0),
+    MVI(0xc00027c4, "TODO_c000_27c4", 0),
+    MVI(0xc00027c5, "TODO_c000_27c5", 0),
+    MVI(0xc00027c6, "TODO_c000_27c6", 0),
+    MVI(0xc00027c7, "TODO_c000_27c7", 0),
+    MVI(0xc00027c8, "TODO_c000_27c8", 0),
+    MVI(0xc00027c9, "TODO_c000_27c9", 0),
+    MVI(0xc00027ca, "TODO_c000_27ca", 0),
+    MVI(0xc00027cb, "TODO_c000_27cb", 0),
+    MVI(0xc00027cc, "TODO_c000_27cc", 0),
+    MVI(0xc00027cd, "TODO_c000_27cd", 0),
+    MVI(0xc00027ce, "TODO_c000_27ce", 0),
+    MVI(0xc00027cf, "TODO_c000_27cf", 0),
+    MVI(0xc00027d0, "TODO_c000_27d0", 0),
+    MVI(0xc00027d1, "TODO_c000_27d1", 0),
+    MVI(0xc00027d2, "TODO_c000_27d2", 0),
+    MVI(0xc00027d3, "TODO_c000_27d3", 0),
+    MVI(0xc00027d4, "TODO_c000_27d4", 0),
+    MVI(0xc00027d5, "TODO_c000_27d5", 0),
+    MVI(0xc00027d6, "TODO_c000_27d6", 0),
+    MVI(0xc00027d7, "TODO_c000_27d7", 0),
+    MVI(0xc00027d8, "TODO_c000_27d8", 0),
+    MVI(0xc00027d9, "TODO_c000_27d9", 0),
+    MVI(0xc00027da, "TODO_c000_27da", 0),
+    MVI(0xc00027db, "TODO_c000_27db", 0),
+    MVI(0xc00027dc, "TODO_c000_27dc", 0),
+    MVI(0xc00027dd, "TODO_c000_27dd", 0),
+    MVI(0xc00027de, "TODO_c000_27de", 0),
+    MVI(0xc00027df, "TODO_c000_27df", 0),
+    MVI(0xc00027e0, "TODO_c000_27e0", 0),
+    MVI(0xc00027e1, "TODO_c000_27e1", 0),
+    MVI(0xc00027e2, "TODO_c000_27e2", 0),
+    MVI(0xc00027e3, "TODO_c000_27e3", 0),
+    MVI(0xc00027e4, "TODO_c000_27e4", 0),
+    MVI(0xc00027e5, "TODO_c000_27e5", 0),
+    MVI(0xc00027e6, "TODO_c000_27e6", 0),
+    MVI(0xc00027e7, "TODO_c000_27e7", 0),
+    MVI(0xc00027e8, "TODO_c000_27e8", 0),
+    MVI(0xc00027e9, "TODO_c000_27e9", 0),
+    MVI(0xc00027ea, "TODO_c000_27ea", 0),
+    MVI(0xc00027eb, "TODO_c000_27eb", 0),
+    MVI(0xc00027ec, "TODO_c000_27ec", 0),
+    MVI(0xc00027ed, "TODO_c000_27ed", 0),
+    MVI(0xc00027ee, "TODO_c000_27ee", 0),
+    MVI(0xc00027ef, "TODO_c000_27ef", 0),
+    MVI(0xc00027f0, "TODO_c000_27f0", 0),
+    MVI(0xc00027f1, "TODO_c000_27f1", 0),
+    MVI(0xc00027f2, "TODO_c000_27f2", 0),
+    MVI(0xc00027f3, "TODO_c000_27f3", 0),
+    MVI(0xc00027f4, "TODO_c000_27f4", 0),
+    MVI(0xc00027f5, "TODO_c000_27f5", 0),
+    MVI(0xc00027f6, "TODO_c000_27f6", 0),
+    MVI(0xc00027f7, "TODO_c000_27f7", 0),
+    MVI(0xc00027f8, "TODO_c000_27f8", 0),
+    MVI(0xc00027f9, "TODO_c000_27f9", 0),
+    MVI(0xc00027fa, "TODO_c000_27fa", 0),
+    MVI(0xc00027fb, "TODO_c000_27fb", 0),
+    MVI(0xc00027fc, "TODO_c000_27fc", 0),
+    MVI(0xc00027fd, "TODO_c000_27fd", 0),
+    MVI(0xc00027fe, "TODO_c000_27fe", 0),
+    MVI(0xc00027ff, "TODO_c000_27ff", 0),
+    MVI(0xc0002800, "TODO_c000_2800", 0),
+    MVI(0xc0002801, "TODO_c000_2801", 0),
+    MVI(0xc0002802, "TODO_c000_2802", 0),
+    MVI(0xc0002803, "TODO_c000_2803", 0),
+    MVI(0xc0002804, "TODO_c000_2804", 0),
+    MVI(0xc0002805, "TODO_c000_2805", 0),
+    MVI(0xc0002806, "TODO_c000_2806", 0),
+    MVI(0xc0002807, "TODO_c000_2807", 0),
+    MVI(0xc0002808, "TODO_c000_2808", 0),
+    MVI(0xc0002809, "TODO_c000_2809", 0),
+    MVI(0xc000280a, "TODO_c000_280a", 0),
+    MVI(0xc000280b, "TODO_c000_280b", 0),
+    MVI(0xc000280c, "TODO_c000_280c", 0),
+    MVI(0xc000280d, "TODO_c000_280d", 0),
+    MVI(0xc000280e, "TODO_c000_280e", 0),
+    MVI(0xc000280f, "TODO_c000_280f", 0),
+    MVI(0xc0002810, "TODO_c000_2810", 0),
+    MVI(0xc0002811, "TODO_c000_2811", 0),
+    MVI(0xc0002812, "TODO_c000_2812", 0),
+    MVI(0xc0002813, "TODO_c000_2813", 0),
+    MVI(0xc0002814, "TODO_c000_2814", 0),
+    MVI(0xc0002815, "TODO_c000_2815", 0),
+    MVI(0xc0002816, "TODO_c000_2816", 0),
+    MVI(0xc0002817, "TODO_c000_2817", 0),
+    MVI(0xc0002818, "TODO_c000_2818", 0),
+    MVI(0xc0002819, "TODO_c000_2819", 0),
+    MVI(0xc000281a, "TODO_c000_281a", 0),
+    MVI(0xc000281b, "TODO_c000_281b", 0),
+    MVI(0xc000281c, "TODO_c000_281c", 0),
+    MVI(0xc000281d, "TODO_c000_281d", 0),
+    MVI(0xc000281e, "TODO_c000_281e", 0),
+    MVI(0xc000281f, "TODO_c000_281f", 0),
+    MVI(0xc0002820, "TODO_c000_2820", 0),
+    MVI(0xc0002821, "TODO_c000_2821", 0),
+    MVI(0xc0002822, "TODO_c000_2822", 0),
+    MVI(0xc0002823, "TODO_c000_2823", 0),
+    MVI(0xc0002824, "TODO_c000_2824", 0),
+    MVI(0xc0002825, "TODO_c000_2825", 0),
+    MVI(0xc0002826, "TODO_c000_2826", 0),
+    MVI(0xc0002827, "TODO_c000_2827", 0),
+    MVI(0xc0002828, "TODO_c000_2828", 0),
+    MVI(0xc0002829, "TODO_c000_2829", 0),
+    MVI(0xc000282a, "TODO_c000_282a", 0),
+    MVI(0xc000282b, "TODO_c000_282b", 0),
+    MVI(0xc000282c, "TODO_c000_282c", 0),
+    MVI(0xc000282d, "TODO_c000_282d", 0),
+    MVI(0xc000282e, "TODO_c000_282e", 0),
+    MVI(0xc000282f, "TODO_c000_282f", 0),
+    MVI(0xc0002830, "TODO_c000_2830", 0),
+    MVI(0xc0002831, "TODO_c000_2831", 0),
+    MVI(0xc0002832, "TODO_c000_2832", 0),
+    MVI(0xc0002833, "TODO_c000_2833", 0),
+    MVI(0xc0002834, "TODO_c000_2834", 0),
+    MVI(0xc0002835, "TODO_c000_2835", 0),
+    MVI(0xc0002836, "TODO_c000_2836", 0),
+    MVI(0xc0002837, "TODO_c000_2837", 0),
+    MVI(0xc0002838, "TODO_c000_2838", 0),
+    MVI(0xc0002839, "TODO_c000_2839", 0),
+    MVI(0xc000283a, "TODO_c000_283a", 0),
+    MVI(0xc000283b, "TODO_c000_283b", 0),
+    MVI(0xc000283c, "TODO_c000_283c", 0),
+    MVI(0xc000283d, "TODO_c000_283d", 0),
+    MVI(0xc000283e, "TODO_c000_283e", 0),
+    MVI(0xc000283f, "TODO_c000_283f", 0),
+    MVI(0xc0002840, "TODO_c000_2840", 0),
+    MVI(0xc0002841, "TODO_c000_2841", 0),
+    MVI(0xc0002842, "TODO_c000_2842", 0),
+    MVI(0xc0002843, "TODO_c000_2843", 0),
+    MVI(0xc0002844, "TODO_c000_2844", 0),
+    MVI(0xc0002845, "TODO_c000_2845", 0),
+    MVI(0xc0002846, "TODO_c000_2846", 0),
+    MVI(0xc0002847, "TODO_c000_2847", 0),
+    MVI(0xc0002848, "TODO_c000_2848", 0),
+    MVI(0xc0002849, "TODO_c000_2849", 0),
+    MVI(0xc000284a, "TODO_c000_284a", 0),
+    MVI(0xc000284b, "TODO_c000_284b", 0),
+    MVI(0xc000284c, "TODO_c000_284c", 0),
+    MVI(0xc000284d, "TODO_c000_284d", 0),
+    MVI(0xc000284e, "TODO_c000_284e", 0),
+    MVI(0xc000284f, "TODO_c000_284f", 0),
+    MVI(0xc0002850, "TODO_c000_2850", 0),
+    MVI(0xc0002851, "TODO_c000_2851", 0),
+    MVI(0xc0002852, "TODO_c000_2852", 0),
+    MVI(0xc0002853, "TODO_c000_2853", 0),
+    MVI(0xc0002854, "TODO_c000_2854", 0),
+    MVI(0xc0002855, "TODO_c000_2855", 0),
+    MVI(0xc0002856, "TODO_c000_2856", 0),
+    MVI(0xc0002857, "TODO_c000_2857", 0),
+    MVI(0xc0002858, "TODO_c000_2858", 0),
+    MVI(0xc0002859, "TODO_c000_2859", 0),
+    MVI(0xc000285a, "TODO_c000_285a", 0),
+    MVI(0xc000285b, "TODO_c000_285b", 0),
+    MVI(0xc000285c, "TODO_c000_285c", 0),
+    MVI(0xc000285d, "TODO_c000_285d", 0),
+    MVI(0xc000285e, "TODO_c000_285e", 0),
+    MVI(0xc000285f, "TODO_c000_285f", 0),
+    MVI(0xc0002860, "TODO_c000_2860", 0),
+    MVI(0xc0002861, "TODO_c000_2861", 0),
+    MVI(0xc0002862, "TODO_c000_2862", 0),
+    MVI(0xc0002863, "TODO_c000_2863", 0),
+    MVI(0xc0002864, "TODO_c000_2864", 0),
+    MVI(0xc0002865, "TODO_c000_2865", 0),
+    MVI(0xc0002866, "TODO_c000_2866", 0),
+    MVI(0xc0002867, "TODO_c000_2867", 0),
+    MVI(0xc0002868, "TODO_c000_2868", 0),
+    MVI(0xc0002869, "TODO_c000_2869", 0),
+    MVI(0xc000286a, "TODO_c000_286a", 0),
+    MVI(0xc000286b, "TODO_c000_286b", 0),
+    MVI(0xc000286c, "TODO_c000_286c", 0),
+    MVI(0xc000286d, "TODO_c000_286d", 0),
+    MVI(0xc000286e, "TODO_c000_286e", 0),
+    MVI(0xc000286f, "TODO_c000_286f", 0),
+    MVI(0xc0002870, "TODO_c000_2870", 0),
+    MVI(0xc0002871, "TODO_c000_2871", 0),
+    MVI(0xc0002872, "TODO_c000_2872", 0),
+    MVI(0xc0002873, "TODO_c000_2873", 0),
+    MVI(0xc0002874, "TODO_c000_2874", 0),
+    MVI(0xc0002875, "TODO_c000_2875", 0),
+    MVI(0xc0002876, "TODO_c000_2876", 0),
+    MVI(0xc0002877, "TODO_c000_2877", 0),
+    MVI(0xc0002878, "TODO_c000_2878", 0),
+    MVI(0xc0002879, "TODO_c000_2879", 0),
+    MVI(0xc000287a, "TODO_c000_287a", 0),
+    MVI(0xc000287b, "TODO_c000_287b", 0),
+    MVI(0xc000287c, "TODO_c000_287c", 0),
+    MVI(0xc000287d, "TODO_c000_287d", 0),
+    MVI(0xc000287e, "TODO_c000_287e", 0),
+    MVI(0xc000287f, "TODO_c000_287f", 0),
+    MVI(0xc0002880, "TODO_c000_2880", 0),
+    MVI(0xc0002881, "TODO_c000_2881", 0),
+    MVI(0xc0002882, "TODO_c000_2882", 0),
+    MVI(0xc0002883, "TODO_c000_2883", 0),
+    MVI(0xc0002884, "TODO_c000_2884", 0),
+    MVI(0xc0002885, "TODO_c000_2885", 0),
+    MVI(0xc0002886, "TODO_c000_2886", 0),
+    MVI(0xc0002887, "TODO_c000_2887", 0),
+    MVI(0xc0002888, "TODO_c000_2888", 0),
+    MVI(0xc0002889, "TODO_c000_2889", 0),
+    MVI(0xc000288a, "TODO_c000_288a", 0),
+    MVI(0xc000288b, "TODO_c000_288b", 0),
+    MVI(0xc000288c, "TODO_c000_288c", 0),
+    MVI(0xc000288d, "TODO_c000_288d", 0),
+    MVI(0xc000288e, "TODO_c000_288e", 0),
+    MVI(0xc000288f, "TODO_c000_288f", 0),
+    MVI(0xc0002890, "TODO_c000_2890", 0),
+    MVI(0xc0002891, "TODO_c000_2891", 0),
+    MVI(0xc0002892, "TODO_c000_2892", 0),
+    MVI(0xc0002893, "TODO_c000_2893", 0),
+    MVI(0xc0002894, "TODO_c000_2894", 0),
+    MVI(0xc0002895, "TODO_c000_2895", 0),
+    MVI(0xc0002896, "TODO_c000_2896", 0),
+    MVI(0xc0002897, "TODO_c000_2897", 0),
+    MVI(0xc0002898, "TODO_c000_2898", 0),
+    MVI(0xc0002899, "TODO_c000_2899", 0),
+    MVI(0xc000289a, "TODO_c000_289a", 0),
+    MVI(0xc000289b, "TODO_c000_289b", 0),
+    MVI(0xc000289c, "TODO_c000_289c", 0),
+    MVI(0xc000289d, "TODO_c000_289d", 0),
+    MVI(0xc000289e, "TODO_c000_289e", 0),
+    MVI(0xc000289f, "TODO_c000_289f", 0),
+    MVI(0xc00028a0, "TODO_c000_28a0", 0),
+    MVI(0xc00028a1, "TODO_c000_28a1", 0),
+    MVI(0xc00028a2, "TODO_c000_28a2", 0),
+    MVI(0xc00028a3, "TODO_c000_28a3", 0),
+    MVI(0xc00028a4, "TODO_c000_28a4", 0),
+    MVI(0xc00028a5, "TODO_c000_28a5", 0),
+    MVI(0xc00028a6, "TODO_c000_28a6", 0),
+    MVI(0xc00028a7, "TODO_c000_28a7", 0),
+    MVI(0xc00028a8, "TODO_c000_28a8", 0),
+    MVI(0xc00028a9, "TODO_c000_28a9", 0),
+    MVI(0xc00028aa, "TODO_c000_28aa", 0),
+    MVI(0xc00028ab, "TODO_c000_28ab", 0),
+    MVI(0xc00028ac, "TODO_c000_28ac", 0),
+    MVI(0xc00028ad, "TODO_c000_28ad", 0),
+    MVI(0xc00028ae, "TODO_c000_28ae", 0),
+    MVI(0xc00028af, "TODO_c000_28af", 0),
+    MVI(0xc00028b0, "TODO_c000_28b0", 0),
+    MVI(0xc00028b1, "TODO_c000_28b1", 0),
+    MVI(0xc00028b2, "TODO_c000_28b2", 0),
+    MVI(0xc00028b3, "TODO_c000_28b3", 0),
+    MVI(0xc00028b4, "TODO_c000_28b4", 0),
+    MVI(0xc00028b5, "TODO_c000_28b5", 0),
+    MVI(0xc00028b6, "TODO_c000_28b6", 0),
+    MVI(0xc00028b7, "TODO_c000_28b7", 0),
+    MVI(0xc00028b8, "TODO_c000_28b8", 0),
+    MVI(0xc00028b9, "TODO_c000_28b9", 0),
+    MVI(0xc00028ba, "TODO_c000_28ba", 0),
+    MVI(0xc00028bb, "TODO_c000_28bb", 0),
+    MVI(0xc00028bc, "TODO_c000_28bc", 0),
+    MVI(0xc00028bd, "TODO_c000_28bd", 0),
+    MVI(0xc00028be, "TODO_c000_28be", 0),
+    MVI(0xc00028bf, "TODO_c000_28bf", 0),
+    MVI(0xc00028c0, "TODO_c000_28c0", 0),
+    MVI(0xc00028c1, "TODO_c000_28c1", 0),
+    MVI(0xc00028c2, "TODO_c000_28c2", 0),
+    MVI(0xc00028c3, "TODO_c000_28c3", 0),
+    MVI(0xc00028c4, "TODO_c000_28c4", 0),
+    MVI(0xc00028c5, "TODO_c000_28c5", 0),
+    MVI(0xc00028c6, "TODO_c000_28c6", 0),
+    MVI(0xc00028c7, "TODO_c000_28c7", 0),
+    MVI(0xc00028c8, "TODO_c000_28c8", 0),
+    MVI(0xc00028c9, "TODO_c000_28c9", 0),
+    MVI(0xc00028ca, "TODO_c000_28ca", 0),
+    MVI(0xc00028cb, "TODO_c000_28cb", 0),
+    MVI(0xc00028cc, "TODO_c000_28cc", 0),
+    MVI(0xc00028cd, "TODO_c000_28cd", 0),
+    MVI(0xc00028ce, "TODO_c000_28ce", 0),
+    MVI(0xc00028cf, "TODO_c000_28cf", 0),
+    MVI(0xc00028d0, "TODO_c000_28d0", 0),
+    MVI(0xc00028d1, "TODO_c000_28d1", 0),
+    MVI(0xc00028d2, "TODO_c000_28d2", 0),
+    MVI(0xc00028d3, "TODO_c000_28d3", 0),
+    MVI(0xc00028d4, "TODO_c000_28d4", 0),
+    MVI(0xc00028d5, "TODO_c000_28d5", 0),
+    MVI(0xc00028d6, "TODO_c000_28d6", 0),
+    MVI(0xc00028d7, "TODO_c000_28d7", 0),
+    MVI(0xc00028d8, "TODO_c000_28d8", 0),
+    MVI(0xc00028d9, "TODO_c000_28d9", 0),
+    MVI(0xc00028da, "TODO_c000_28da", 0),
+    MVI(0xc00028db, "TODO_c000_28db", 0),
+    MVI(0xc00028dc, "TODO_c000_28dc", 0),
+    MVI(0xc00028dd, "TODO_c000_28dd", 0),
+    MVI(0xc00028de, "TODO_c000_28de", 0),
+    MVI(0xc00028df, "TODO_c000_28df", 0),
+    MVI(0xc00028e0, "TODO_c000_28e0", 0),
+    MVI(0xc00028e1, "TODO_c000_28e1", 0),
+    MVI(0xc00028e2, "TODO_c000_28e2", 0),
+    MVI(0xc00028e3, "TODO_c000_28e3", 0),
+    MVI(0xc00028e4, "TODO_c000_28e4", 0),
+    MVI(0xc00028e5, "TODO_c000_28e5", 0),
+    MVI(0xc00028e6, "TODO_c000_28e6", 0),
+    MVI(0xc00028e7, "TODO_c000_28e7", 0),
+    MVI(0xc00028e8, "TODO_c000_28e8", 0),
+    MVI(0xc00028e9, "TODO_c000_28e9", 0),
+    MVI(0xc00028ea, "TODO_c000_28ea", 0),
+    MVI(0xc00028eb, "TODO_c000_28eb", 0),
+    MVI(0xc00028ec, "TODO_c000_28ec", 0),
+    MVI(0xc00028ed, "TODO_c000_28ed", 0),
+    MVI(0xc00028ee, "TODO_c000_28ee", 0),
+    MVI(0xc00028ef, "TODO_c000_28ef", 0),
+    MVI(0xc00028f0, "TODO_c000_28f0", 0),
+    MVI(0xc00028f1, "TODO_c000_28f1", 0),
+    MVI(0xc00028f2, "TODO_c000_28f2", 0),
+    MVI(0xc00028f3, "TODO_c000_28f3", 0),
+    MVI(0xc00028f4, "TODO_c000_28f4", 0),
+    MVI(0xc00028f5, "TODO_c000_28f5", 0),
+    MVI(0xc00028f6, "TODO_c000_28f6", 0),
+    MVI(0xc00028f7, "TODO_c000_28f7", 0),
+    MVI(0xc00028f8, "TODO_c000_28f8", 0),
+    MVI(0xc00028f9, "TODO_c000_28f9", 0),
+    MVI(0xc00028fa, "TODO_c000_28fa", 0),
+    MVI(0xc00028fb, "TODO_c000_28fb", 0),
+    MVI(0xc00028fc, "TODO_c000_28fc", 0),
+    MVI(0xc00028fd, "TODO_c000_28fd", 0),
+    MVI(0xc00028fe, "TODO_c000_28fe", 0),
+    MVI(0xc00028ff, "TODO_c000_28ff", 0),
+    MVI(0xc0002900, "TODO_c000_2900", 0),
+    MVI(0xc0002901, "TODO_c000_2901", 0),
+    MVI(0xc0002902, "TODO_c000_2902", 0),
+    MVI(0xc0002903, "TODO_c000_2903", 0),
+    MVI(0xc0002904, "TODO_c000_2904", 0),
+    MVI(0xc0002905, "TODO_c000_2905", 0),
+    MVI(0xc0002906, "TODO_c000_2906", 0),
+    MVI(0xc0002907, "TODO_c000_2907", 0),
+    MVI(0xc0002908, "TODO_c000_2908", 0),
+    MVI(0xc0002909, "TODO_c000_2909", 0),
+    MVI(0xc000290a, "TODO_c000_290a", 0),
+    MVI(0xc000290b, "TODO_c000_290b", 0),
+    MVI(0xc000290c, "TODO_c000_290c", 0),
+    MVI(0xc000290d, "TODO_c000_290d", 0),
+    MVI(0xc000290e, "TODO_c000_290e", 0),
+    MVI(0xc000290f, "TODO_c000_290f", 0),
+    MVI(0xc0002910, "TODO_c000_2910", 0),
+    MVI(0xc0002911, "TODO_c000_2911", 0),
+    MVI(0xc0002912, "TODO_c000_2912", 0),
+    MVI(0xc0002913, "TODO_c000_2913", 0),
+    MVI(0xc0002914, "TODO_c000_2914", 0),
+    MVI(0xc0002915, "TODO_c000_2915", 0),
+    MVI(0xc0002916, "TODO_c000_2916", 0),
+    MVI(0xc0002917, "TODO_c000_2917", 0),
+    MVI(0xc0002918, "TODO_c000_2918", 0),
+    MVI(0xc0002919, "TODO_c000_2919", 0),
+    MVI(0xc000291a, "TODO_c000_291a", 0),
+    MVI(0xc000291b, "TODO_c000_291b", 0),
+    MVI(0xc000291c, "TODO_c000_291c", 0),
+    MVI(0xc000291d, "TODO_c000_291d", 0),
+    MVI(0xc000291e, "TODO_c000_291e", 0),
+    MVI(0xc000291f, "TODO_c000_291f", 0),
+    MVI(0xc0002920, "TODO_c000_2920", 0),
+    MVI(0xc0002921, "TODO_c000_2921", 0),
+    MVI(0xc0002922, "TODO_c000_2922", 0),
+    MVI(0xc0002923, "TODO_c000_2923", 0),
+    MVI(0xc0002924, "TODO_c000_2924", 0),
+    MVI(0xc0002925, "TODO_c000_2925", 0),
+    MVI(0xc0002926, "TODO_c000_2926", 0),
+    MVI(0xc0002927, "TODO_c000_2927", 0),
+    MVI(0xc0002928, "TODO_c000_2928", 0),
+    MVI(0xc0002929, "TODO_c000_2929", 0),
+    MVI(0xc000292a, "TODO_c000_292a", 0),
+    MVI(0xc000292b, "TODO_c000_292b", 0),
+    MVI(0xc000292c, "TODO_c000_292c", 0),
+    MVI(0xc000292d, "TODO_c000_292d", 0),
+    MVI(0xc000292e, "TODO_c000_292e", 0),
+    MVI(0xc000292f, "TODO_c000_292f", 0),
+    MVI(0xc0002930, "TODO_c000_2930", 0),
+    MVI(0xc0002931, "TODO_c000_2931", 0),
+    MVI(0xc0002932, "TODO_c000_2932", 0),
+    MVI(0xc0002933, "TODO_c000_2933", 0),
+    MVI(0xc0002934, "TODO_c000_2934", 0),
+    MVI(0xc0002935, "TODO_c000_2935", 0),
+    MVI(0xc0002936, "TODO_c000_2936", 0),
+    MVI(0xc0002937, "TODO_c000_2937", 0),
+    MVI(0xc0002938, "TODO_c000_2938", 0),
+    MVI(0xc0002939, "TODO_c000_2939", 0),
+    MVI(0xc000293a, "TODO_c000_293a", 0),
+    MVI(0xc000293b, "TODO_c000_293b", 0),
+    MVI(0xc000293c, "TODO_c000_293c", 0),
+    MVI(0xc000293d, "TODO_c000_293d", 0),
+    MVI(0xc000293e, "TODO_c000_293e", 0),
+    MVI(0xc000293f, "TODO_c000_293f", 0),
+    MVI(0xc0002940, "TODO_c000_2940", 0),
+    MVI(0xc0002941, "TODO_c000_2941", 0),
+    MVI(0xc0002942, "TODO_c000_2942", 0),
+    MVI(0xc0002943, "TODO_c000_2943", 0),
+    MVI(0xc0002944, "TODO_c000_2944", 0),
+    MVI(0xc0002945, "TODO_c000_2945", 0),
+    MVI(0xc0002946, "TODO_c000_2946", 0),
+    MVI(0xc0002947, "TODO_c000_2947", 0),
+    MVI(0xc0002948, "TODO_c000_2948", 0),
+    MVI(0xc0002949, "TODO_c000_2949", 0),
+    MVI(0xc000294a, "TODO_c000_294a", 0),
+    MVI(0xc000294b, "TODO_c000_294b", 0),
+    MVI(0xc000294c, "TODO_c000_294c", 0),
+    MVI(0xc000294d, "TODO_c000_294d", 0),
+    MVI(0xc000294e, "TODO_c000_294e", 0),
+    MVI(0xc000294f, "TODO_c000_294f", 0),
+    MVI(0xc0002950, "TODO_c000_2950", 0),
+    MVI(0xc0002951, "TODO_c000_2951", 0),
+    MVI(0xc0002952, "TODO_c000_2952", 0),
+    MVI(0xc0002953, "TODO_c000_2953", 0),
+    MVI(0xc0002954, "TODO_c000_2954", 0),
+    MVI(0xc0002955, "TODO_c000_2955", 0),
+    MVI(0xc0002956, "TODO_c000_2956", 0),
+    MVI(0xc0002957, "TODO_c000_2957", 0),
+    MVI(0xc0002958, "TODO_c000_2958", 0),
+    MVI(0xc0002959, "TODO_c000_2959", 0),
+    MVI(0xc000295a, "TODO_c000_295a", 0),
+    MVI(0xc000295b, "TODO_c000_295b", 0),
+    MVI(0xc000295c, "TODO_c000_295c", 0),
+    MVI(0xc000295d, "TODO_c000_295d", 0),
+    MVI(0xc000295e, "TODO_c000_295e", 0),
+    MVI(0xc000295f, "TODO_c000_295f", 0),
+    MVI(0xc0002960, "TODO_c000_2960", 0),
+    MVI(0xc0002961, "TODO_c000_2961", 0),
+    MVI(0xc0002962, "TODO_c000_2962", 0),
+    MVI(0xc0002963, "TODO_c000_2963", 0),
+    MVI(0xc0002964, "TODO_c000_2964", 0),
+    MVI(0xc0002965, "TODO_c000_2965", 0),
+    MVI(0xc0002966, "TODO_c000_2966", 0),
+    MVI(0xc0002967, "TODO_c000_2967", 0),
+    MVI(0xc0002968, "TODO_c000_2968", 0),
+    MVI(0xc0002969, "TODO_c000_2969", 0),
+    MVI(0xc000296a, "TODO_c000_296a", 0),
+    MVI(0xc000296b, "TODO_c000_296b", 0),
+    MVI(0xc000296c, "TODO_c000_296c", 0),
+    MVI(0xc000296d, "TODO_c000_296d", 0),
+    MVI(0xc000296e, "TODO_c000_296e", 0),
+    MVI(0xc000296f, "TODO_c000_296f", 0),
+    MVI(0xc0002970, "TODO_c000_2970", 0),
+    MVI(0xc0002971, "TODO_c000_2971", 0),
+    MVI(0xc0002972, "TODO_c000_2972", 0),
+    MVI(0xc0002973, "TODO_c000_2973", 0),
+    MVI(0xc0002974, "TODO_c000_2974", 0),
+    MVI(0xc0002975, "TODO_c000_2975", 0),
+    MVI(0xc0002976, "TODO_c000_2976", 0),
+    MVI(0xc0002977, "TODO_c000_2977", 0),
+    MVI(0xc0002978, "TODO_c000_2978", 0),
+    MVI(0xc0002979, "TODO_c000_2979", 0),
+    MVI(0xc000297a, "TODO_c000_297a", 0),
+    MVI(0xc000297b, "TODO_c000_297b", 0),
+    MVI(0xc000297c, "TODO_c000_297c", 0),
+    MVI(0xc000297d, "TODO_c000_297d", 0),
+    MVI(0xc000297e, "TODO_c000_297e", 0),
+    MVI(0xc000297f, "TODO_c000_297f", 0),
+    MVI(0xc0002980, "TODO_c000_2980", 0),
+    MVI(0xc0002981, "TODO_c000_2981", 0),
+    MVI(0xc0002982, "TODO_c000_2982", 0),
+    MVI(0xc0002983, "TODO_c000_2983", 0),
+    MVI(0xc0002984, "TODO_c000_2984", 0),
+    MVI(0xc0002985, "TODO_c000_2985", 0),
+    MVI(0xc0002986, "TODO_c000_2986", 0),
+    MVI(0xc0002987, "TODO_c000_2987", 0),
+    MVI(0xc0002988, "TODO_c000_2988", 0),
+    MVI(0xc0002989, "TODO_c000_2989", 0),
+    MVI(0xc000298a, "TODO_c000_298a", 0),
+    MVI(0xc000298b, "TODO_c000_298b", 0),
+    MVI(0xc000298c, "TODO_c000_298c", 0),
+    MVI(0xc000298d, "TODO_c000_298d", 0),
+    MVI(0xc000298e, "TODO_c000_298e", 0),
+    MVI(0xc000298f, "TODO_c000_298f", 0),
+    MVI(0xc0002990, "TODO_c000_2990", 0),
+    MVI(0xc0002991, "TODO_c000_2991", 0),
+    MVI(0xc0002992, "TODO_c000_2992", 0),
+    MVI(0xc0002993, "TODO_c000_2993", 0),
+    MVI(0xc0002994, "TODO_c000_2994", 0),
+    MVI(0xc0002995, "TODO_c000_2995", 0),
+    MVI(0xc0002996, "TODO_c000_2996", 0),
+    MVI(0xc0002997, "TODO_c000_2997", 0),
+    MVI(0xc0002998, "TODO_c000_2998", 0),
+    MVI(0xc0002999, "TODO_c000_2999", 0),
+    MVI(0xc000299a, "TODO_c000_299a", 0),
+    MVI(0xc000299b, "TODO_c000_299b", 0),
+    MVI(0xc000299c, "TODO_c000_299c", 0),
+    MVI(0xc000299d, "TODO_c000_299d", 0),
+    MVI(0xc000299e, "TODO_c000_299e", 0),
+    MVI(0xc000299f, "TODO_c000_299f", 0),
+    MVI(0xc00029a0, "TODO_c000_29a0", 0),
+    MVI(0xc00029a1, "TODO_c000_29a1", 0),
+    MVI(0xc00029a2, "TODO_c000_29a2", 0),
+    MVI(0xc00029a3, "TODO_c000_29a3", 0),
+    MVI(0xc00029a4, "TODO_c000_29a4", 0),
+    MVI(0xc00029a5, "TODO_c000_29a5", 0),
+    MVI(0xc00029a6, "TODO_c000_29a6", 0),
+    MVI(0xc00029a7, "TODO_c000_29a7", 0),
+    MVI(0xc00029a8, "TODO_c000_29a8", 0),
+    MVI(0xc00029a9, "TODO_c000_29a9", 0),
+    MVI(0xc00029aa, "TODO_c000_29aa", 0),
+    MVI(0xc00029ab, "TODO_c000_29ab", 0),
+    MVI(0xc00029ac, "TODO_c000_29ac", 0),
+    MVI(0xc00029ad, "TODO_c000_29ad", 0),
+    MVI(0xc00029ae, "TODO_c000_29ae", 0),
+    MVI(0xc00029af, "TODO_c000_29af", 0),
+    MVI(0xc00029b0, "TODO_c000_29b0", 0),
+    MVI(0xc00029b1, "TODO_c000_29b1", 0),
+    MVI(0xc00029b2, "TODO_c000_29b2", 0),
+    MVI(0xc00029b3, "TODO_c000_29b3", 0),
+    MVI(0xc00029b4, "TODO_c000_29b4", 0),
+    MVI(0xc00029b5, "TODO_c000_29b5", 0),
+    MVI(0xc00029b6, "TODO_c000_29b6", 0),
+    MVI(0xc00029b7, "TODO_c000_29b7", 0),
+    MVI(0xc00029b8, "TODO_c000_29b8", 0),
+    MVI(0xc00029b9, "TODO_c000_29b9", 0),
+    MVI(0xc00029ba, "TODO_c000_29ba", 0),
+    MVI(0xc00029bb, "TODO_c000_29bb", 0),
+    MVI(0xc00029bc, "TODO_c000_29bc", 0),
+    MVI(0xc00029bd, "TODO_c000_29bd", 0),
+    MVI(0xc00029be, "TODO_c000_29be", 0),
+    MVI(0xc00029bf, "TODO_c000_29bf", 0),
+    MVI(0xc00029c0, "TODO_c000_29c0", 0),
+    MVI(0xc00029c1, "TODO_c000_29c1", 0),
+    MVI(0xc00029c2, "TODO_c000_29c2", 0),
+    MVI(0xc00029c3, "TODO_c000_29c3", 0),
+    MVI(0xc00029c4, "TODO_c000_29c4", 0),
+    MVI(0xc00029c5, "TODO_c000_29c5", 0),
+    MVI(0xc00029c6, "TODO_c000_29c6", 0),
+    MVI(0xc00029c7, "TODO_c000_29c7", 0),
+    MVI(0xc00029c8, "TODO_c000_29c8", 0),
+    MVI(0xc00029c9, "TODO_c000_29c9", 0),
+    MVI(0xc00029ca, "TODO_c000_29ca", 0),
+    MVI(0xc00029cb, "TODO_c000_29cb", 0),
+    MVI(0xc00029cc, "TODO_c000_29cc", 0),
+    MVI(0xc00029cd, "TODO_c000_29cd", 0),
+    MVI(0xc00029ce, "TODO_c000_29ce", 0),
+    MVI(0xc00029cf, "TODO_c000_29cf", 0),
+    MVI(0xc00029d0, "TODO_c000_29d0", 0),
+    MVI(0xc00029d1, "TODO_c000_29d1", 0),
+    MVI(0xc00029d2, "TODO_c000_29d2", 0),
+    MVI(0xc00029d3, "TODO_c000_29d3", 0),
+    MVI(0xc00029d4, "TODO_c000_29d4", 0),
+    MVI(0xc00029d5, "TODO_c000_29d5", 0),
+    MVI(0xc00029d6, "TODO_c000_29d6", 0),
+    MVI(0xc00029d7, "TODO_c000_29d7", 0),
+    MVI(0xc00029d8, "TODO_c000_29d8", 0),
+    MVI(0xc00029d9, "TODO_c000_29d9", 0),
+    MVI(0xc00029da, "TODO_c000_29da", 0),
+    MVI(0xc00029db, "TODO_c000_29db", 0),
+    MVI(0xc00029dc, "TODO_c000_29dc", 0),
+    MVI(0xc00029dd, "TODO_c000_29dd", 0),
+    MVI(0xc00029de, "TODO_c000_29de", 0),
+    MVI(0xc00029df, "TODO_c000_29df", 0),
+    MVI(0xc00029e0, "TODO_c000_29e0", 0),
+    MVI(0xc00029e1, "TODO_c000_29e1", 0),
+    MVI(0xc00029e2, "TODO_c000_29e2", 0),
+    MVI(0xc00029e3, "TODO_c000_29e3", 0),
+    MVI(0xc00029e4, "TODO_c000_29e4", 0),
+    MVI(0xc00029e5, "TODO_c000_29e5", 0),
+    MVI(0xc00029e6, "TODO_c000_29e6", 0),
+    MVI(0xc00029e7, "TODO_c000_29e7", 0),
+    MVI(0xc00029e8, "TODO_c000_29e8", 0),
+    MVI(0xc00029e9, "TODO_c000_29e9", 0),
+    MVI(0xc00029ea, "TODO_c000_29ea", 0),
+    MVI(0xc00029eb, "TODO_c000_29eb", 0),
+    MVI(0xc00029ec, "TODO_c000_29ec", 0),
+    MVI(0xc00029ed, "TODO_c000_29ed", 0),
+    MVI(0xc00029ee, "TODO_c000_29ee", 0),
+    MVI(0xc00029ef, "TODO_c000_29ef", 0),
+    MVI(0xc00029f0, "TODO_c000_29f0", 0),
+    MVI(0xc00029f1, "TODO_c000_29f1", 0),
+    MVI(0xc00029f2, "TODO_c000_29f2", 0),
+    MVI(0xc00029f3, "TODO_c000_29f3", 0),
+    MVI(0xc00029f4, "TODO_c000_29f4", 0),
+    MVI(0xc00029f5, "TODO_c000_29f5", 0),
+    MVI(0xc00029f6, "TODO_c000_29f6", 0),
+    MVI(0xc00029f7, "TODO_c000_29f7", 0),
+    MVI(0xc00029f8, "TODO_c000_29f8", 0),
+    MVI(0xc00029f9, "TODO_c000_29f9", 0),
+    MVI(0xc00029fa, "TODO_c000_29fa", 0),
+    MVI(0xc00029fb, "TODO_c000_29fb", 0),
+    MVI(0xc00029fc, "TODO_c000_29fc", 0),
+    MVI(0xc00029fd, "TODO_c000_29fd", 0),
+    MVI(0xc00029fe, "TODO_c000_29fe", 0),
+    MVI(0xc00029ff, "TODO_c000_29ff", 0),
+    MVI(0xc0002a00, "TODO_c000_2a00", 0),
+    MVI(0xc0002a01, "TODO_c000_2a01", 0),
+    MVI(0xc0002a02, "TODO_c000_2a02", 0),
+    MVI(0xc0002a03, "TODO_c000_2a03", 0),
+    MVI(0xc0002a04, "TODO_c000_2a04", 0),
+    MVI(0xc0002a05, "TODO_c000_2a05", 0),
+    MVI(0xc0002a06, "TODO_c000_2a06", 0),
+    MVI(0xc0002a07, "TODO_c000_2a07", 0),
+    MVI(0xc0002a08, "TODO_c000_2a08", 0),
+    MVI(0xc0002a09, "TODO_c000_2a09", 0),
+    MVI(0xc0002a0a, "TODO_c000_2a0a", 0),
+    MVI(0xc0002a0b, "TODO_c000_2a0b", 0),
+    MVI(0xc0002a0c, "TODO_c000_2a0c", 0),
+    MVI(0xc0002a0d, "TODO_c000_2a0d", 0),
+    MVI(0xc0002a0e, "TODO_c000_2a0e", 0),
+    MVI(0xc0002a0f, "TODO_c000_2a0f", 0),
+    MVI(0xc0002a10, "TODO_c000_2a10", 0),
+    MVI(0xc0002a11, "TODO_c000_2a11", 0),
+    MVI(0xc0002a12, "TODO_c000_2a12", 0),
+    MVI(0xc0002a13, "TODO_c000_2a13", 0),
+    MVI(0xc0002a14, "TODO_c000_2a14", 0),
+    MVI(0xc0002a15, "TODO_c000_2a15", 0),
+    MVI(0xc0002a16, "TODO_c000_2a16", 0),
+    MVI(0xc0002a17, "TODO_c000_2a17", 0),
+    MVI(0xc0002a18, "TODO_c000_2a18", 0),
+    MVI(0xc0002a19, "TODO_c000_2a19", 0),
+    MVI(0xc0002a1a, "TODO_c000_2a1a", 0),
+    MVI(0xc0002a1b, "TODO_c000_2a1b", 0),
+    MVI(0xc0002a1c, "TODO_c000_2a1c", 0),
+    MVI(0xc0002a1d, "TODO_c000_2a1d", 0),
+    MVI(0xc0002a1e, "TODO_c000_2a1e", 0),
+    MVI(0xc0002a1f, "TODO_c000_2a1f", 0),
+    MVI(0xc0002a20, "TODO_c000_2a20", 0),
+    MVI(0xc0002a21, "TODO_c000_2a21", 0),
+    MVI(0xc0002a22, "TODO_c000_2a22", 0),
+    MVI(0xc0002a23, "TODO_c000_2a23", 0),
+    MVI(0xc0002a24, "TODO_c000_2a24", 0),
+    MVI(0xc0002a25, "TODO_c000_2a25", 0),
+    MVI(0xc0002a26, "TODO_c000_2a26", 0),
+    MVI(0xc0002a27, "TODO_c000_2a27", 0),
+    MVI(0xc0002a28, "TODO_c000_2a28", 0),
+    MVI(0xc0002a29, "TODO_c000_2a29", 0),
+    MVI(0xc0002a2a, "TODO_c000_2a2a", 0),
+    MVI(0xc0002a2b, "TODO_c000_2a2b", 0),
+    MVI(0xc0002a2c, "TODO_c000_2a2c", 0),
+    MVI(0xc0002a2d, "TODO_c000_2a2d", 0),
+    MVI(0xc0002a2e, "TODO_c000_2a2e", 0),
+    MVI(0xc0002a2f, "TODO_c000_2a2f", 0),
+    MVI(0xc0002a30, "TODO_c000_2a30", 0),
+    MVI(0xc0002a31, "TODO_c000_2a31", 0),
+    MVI(0xc0002a32, "TODO_c000_2a32", 0),
+    MVI(0xc0002a33, "TODO_c000_2a33", 0),
+    MVI(0xc0002a34, "TODO_c000_2a34", 0),
+    MVI(0xc0002a35, "TODO_c000_2a35", 0),
+    MVI(0xc0002a36, "TODO_c000_2a36", 0),
+    MVI(0xc0002a37, "TODO_c000_2a37", 0),
+    MVI(0xc0002a38, "TODO_c000_2a38", 0),
+    MVI(0xc0002a39, "TODO_c000_2a39", 0),
+    MVI(0xc0002a3a, "TODO_c000_2a3a", 0),
+    MVI(0xc0002a3b, "TODO_c000_2a3b", 0),
+    MVI(0xc0002a3c, "TODO_c000_2a3c", 0),
+    MVI(0xc0002a3d, "TODO_c000_2a3d", 0),
+    MVI(0xc0002a3e, "TODO_c000_2a3e", 0),
+    MVI(0xc0002a3f, "TODO_c000_2a3f", 0),
+    MVI(0xc0002a40, "TODO_c000_2a40", 0),
+    MVI(0xc0002a41, "TODO_c000_2a41", 0),
+    MVI(0xc0002a42, "TODO_c000_2a42", 0),
+    MVI(0xc0002a43, "TODO_c000_2a43", 0),
+    MVI(0xc0002a44, "TODO_c000_2a44", 0),
+    MVI(0xc0002a45, "TODO_c000_2a45", 0),
+    MVI(0xc0002a46, "TODO_c000_2a46", 0),
+    MVI(0xc0002a47, "TODO_c000_2a47", 0),
+    MVI(0xc0002a48, "TODO_c000_2a48", 0),
+    MVI(0xc0002a49, "TODO_c000_2a49", 0),
+    MVI(0xc0002a4a, "TODO_c000_2a4a", 0),
+    MVI(0xc0002a4b, "TODO_c000_2a4b", 0),
+    MVI(0xc0002a4c, "TODO_c000_2a4c", 0),
+    MVI(0xc0002a4d, "TODO_c000_2a4d", 0),
+    MVI(0xc0002a4e, "TODO_c000_2a4e", 0),
+    MVI(0xc0002a4f, "TODO_c000_2a4f", 0),
+    MVI(0xc0002a50, "TODO_c000_2a50", 0),
+    MVI(0xc0002a51, "TODO_c000_2a51", 0),
+    MVI(0xc0002a52, "TODO_c000_2a52", 0),
+    MVI(0xc0002a53, "TODO_c000_2a53", 0),
+    MVI(0xc0002a54, "TODO_c000_2a54", 0),
+    MVI(0xc0002a55, "TODO_c000_2a55", 0),
+    MVI(0xc0002a56, "TODO_c000_2a56", 0),
+    MVI(0xc0002a57, "TODO_c000_2a57", 0),
+    MVI(0xc0002a58, "TODO_c000_2a58", 0),
+    MVI(0xc0002a59, "TODO_c000_2a59", 0),
+    MVI(0xc0002a5a, "TODO_c000_2a5a", 0),
+    MVI(0xc0002a5b, "TODO_c000_2a5b", 0),
+    MVI(0xc0002a5c, "TODO_c000_2a5c", 0),
+    MVI(0xc0002a5d, "TODO_c000_2a5d", 0),
+    MVI(0xc0002a5e, "TODO_c000_2a5e", 0),
+    MVI(0xc0002a5f, "TODO_c000_2a5f", 0),
+    MVI(0xc0002a60, "TODO_c000_2a60", 0),
+    MVI(0xc0002a61, "TODO_c000_2a61", 0),
+    MVI(0xc0002a62, "TODO_c000_2a62", 0),
+    MVI(0xc0002a63, "TODO_c000_2a63", 0),
+    MVI(0xc0002a64, "TODO_c000_2a64", 0),
+    MVI(0xc0002a65, "TODO_c000_2a65", 0),
+    MVI(0xc0002a66, "TODO_c000_2a66", 0),
+    MVI(0xc0002a67, "TODO_c000_2a67", 0),
+    MVI(0xc0002a68, "TODO_c000_2a68", 0),
+    MVI(0xc0002a69, "TODO_c000_2a69", 0),
+    MVI(0xc0002a6a, "TODO_c000_2a6a", 0),
+    MVI(0xc0002a6b, "TODO_c000_2a6b", 0),
+    MVI(0xc0002a6c, "TODO_c000_2a6c", 0),
+    MVI(0xc0002a6d, "TODO_c000_2a6d", 0),
+    MVI(0xc0002a6e, "TODO_c000_2a6e", 0),
+    MVI(0xc0002a6f, "TODO_c000_2a6f", 0),
+    MVI(0xc0002a70, "TODO_c000_2a70", 0),
+    MVI(0xc0002a71, "TODO_c000_2a71", 0),
+    MVI(0xc0002a72, "TODO_c000_2a72", 0),
+    MVI(0xc0002a73, "TODO_c000_2a73", 0),
+    MVI(0xc0002a74, "TODO_c000_2a74", 0),
+    MVI(0xc0002a75, "TODO_c000_2a75", 0),
+    MVI(0xc0002a76, "TODO_c000_2a76", 0),
+    MVI(0xc0002a77, "TODO_c000_2a77", 0),
+    MVI(0xc0002a78, "TODO_c000_2a78", 0),
+    MVI(0xc0002a79, "TODO_c000_2a79", 0),
+    MVI(0xc0002a7a, "TODO_c000_2a7a", 0),
+    MVI(0xc0002a7b, "TODO_c000_2a7b", 0),
+    MVI(0xc0002a7c, "TODO_c000_2a7c", 0),
+    MVI(0xc0002a7d, "TODO_c000_2a7d", 0),
+    MVI(0xc0002a7e, "TODO_c000_2a7e", 0),
+    MVI(0xc0002a7f, "TODO_c000_2a7f", 0),
+    MVI(0xc0002a80, "TODO_c000_2a80", 0),
+    MVI(0xc0002a81, "TODO_c000_2a81", 0),
+    MVI(0xc0002a82, "TODO_c000_2a82", 0),
+    MVI(0xc0002a83, "TODO_c000_2a83", 0),
+    MVI(0xc0002a84, "TODO_c000_2a84", 0),
+    MVI(0xc0002a85, "TODO_c000_2a85", 0),
+    MVI(0xc0002a86, "TODO_c000_2a86", 0),
+    MVI(0xc0002a87, "TODO_c000_2a87", 0),
+    MVI(0xc0002a88, "TODO_c000_2a88", 0),
+    MVI(0xc0002a89, "TODO_c000_2a89", 0),
+    MVI(0xc0002a8a, "TODO_c000_2a8a", 0),
+    MVI(0xc0002a8b, "TODO_c000_2a8b", 0),
+    MVI(0xc0002a8c, "TODO_c000_2a8c", 0),
+    MVI(0xc0002a8d, "TODO_c000_2a8d", 0),
+    MVI(0xc0002a8e, "TODO_c000_2a8e", 0),
+    MVI(0xc0002a8f, "TODO_c000_2a8f", 0),
+    MVI(0xc0002a90, "TODO_c000_2a90", 0),
+    MVI(0xc0002a91, "TODO_c000_2a91", 0),
+    MVI(0xc0002a92, "TODO_c000_2a92", 0),
+    MVI(0xc0002a93, "TODO_c000_2a93", 0),
+    MVI(0xc0002a94, "TODO_c000_2a94", 0),
+    MVI(0xc0002a95, "TODO_c000_2a95", 0),
+    MVI(0xc0002a96, "TODO_c000_2a96", 0),
+    MVI(0xc0002a97, "TODO_c000_2a97", 0),
+    MVI(0xc0002a98, "TODO_c000_2a98", 0),
+    MVI(0xc0002a99, "TODO_c000_2a99", 0),
+    MVI(0xc0002a9a, "TODO_c000_2a9a", 0),
+    MVI(0xc0002a9b, "TODO_c000_2a9b", 0),
+    MVI(0xc0002a9c, "TODO_c000_2a9c", 0),
+    MVI(0xc0002a9d, "TODO_c000_2a9d", 0),
+    MVI(0xc0002a9e, "TODO_c000_2a9e", 0),
+    MVI(0xc0002a9f, "TODO_c000_2a9f", 0),
+    MVI(0xc0002aa0, "TODO_c000_2aa0", 0),
+    MVI(0xc0002aa1, "TODO_c000_2aa1", 0),
+    MVI(0xc0002aa2, "TODO_c000_2aa2", 0),
+    MVI(0xc0002aa3, "TODO_c000_2aa3", 0),
+    MVI(0xc0002aa4, "TODO_c000_2aa4", 0),
+    MVI(0xc0002aa5, "TODO_c000_2aa5", 0),
+    MVI(0xc0002aa6, "TODO_c000_2aa6", 0),
+    MVI(0xc0002aa7, "TODO_c000_2aa7", 0),
+    MVI(0xc0002aa8, "TODO_c000_2aa8", 0),
+    MVI(0xc0002aa9, "TODO_c000_2aa9", 0),
+    MVI(0xc0002aaa, "TODO_c000_2aaa", 0),
+    MVI(0xc0002aab, "TODO_c000_2aab", 0),
+    MVI(0xc0002aac, "TODO_c000_2aac", 0),
+    MVI(0xc0002aad, "TODO_c000_2aad", 0),
+    MVI(0xc0002aae, "TODO_c000_2aae", 0),
+    MVI(0xc0002aaf, "TODO_c000_2aaf", 0),
+    MVI(0xc0002ab0, "TODO_c000_2ab0", 0),
+    MVI(0xc0002ab1, "TODO_c000_2ab1", 0),
+    MVI(0xc0002ab2, "TODO_c000_2ab2", 0),
+    MVI(0xc0002ab3, "TODO_c000_2ab3", 0),
+    MVI(0xc0002ab4, "TODO_c000_2ab4", 0),
+    MVI(0xc0002ab5, "TODO_c000_2ab5", 0),
+    MVI(0xc0002ab6, "TODO_c000_2ab6", 0),
+    MVI(0xc0002ab7, "TODO_c000_2ab7", 0),
+    MVI(0xc0002ab8, "TODO_c000_2ab8", 0),
+    MVI(0xc0002ab9, "TODO_c000_2ab9", 0),
+    MVI(0xc0002aba, "TODO_c000_2aba", 0),
+    MVI(0xc0002abb, "TODO_c000_2abb", 0),
+    MVI(0xc0002abc, "TODO_c000_2abc", 0),
+    MVI(0xc0002abd, "TODO_c000_2abd", 0),
+    MVI(0xc0002abe, "TODO_c000_2abe", 0),
+    MVI(0xc0002abf, "TODO_c000_2abf", 0),
+    MVI(0xc0002ac0, "TODO_c000_2ac0", 0),
+    MVI(0xc0002ac1, "TODO_c000_2ac1", 0),
+    MVI(0xc0002ac2, "TODO_c000_2ac2", 0),
+    MVI(0xc0002ac3, "TODO_c000_2ac3", 0),
+    MVI(0xc0002ac4, "TODO_c000_2ac4", 0),
+    MVI(0xc0002ac5, "TODO_c000_2ac5", 0),
+    MVI(0xc0002ac6, "TODO_c000_2ac6", 0),
+    MVI(0xc0002ac7, "TODO_c000_2ac7", 0),
+    MVI(0xc0002ac8, "TODO_c000_2ac8", 0),
+    MVI(0xc0002ac9, "TODO_c000_2ac9", 0),
+    MVI(0xc0002aca, "TODO_c000_2aca", 0),
+    MVI(0xc0002acb, "TODO_c000_2acb", 0),
+    MVI(0xc0002acc, "TODO_c000_2acc", 0),
+    MVI(0xc0002acd, "TODO_c000_2acd", 0),
+    MVI(0xc0002ace, "TODO_c000_2ace", 0),
+    MVI(0xc0002acf, "TODO_c000_2acf", 0),
+    MVI(0xc0002ad0, "TODO_c000_2ad0", 0),
+    MVI(0xc0002ad1, "TODO_c000_2ad1", 0),
+    MVI(0xc0002ad2, "TODO_c000_2ad2", 0),
+    MVI(0xc0002ad3, "TODO_c000_2ad3", 0),
+    MVI(0xc0002ad4, "TODO_c000_2ad4", 0),
+    MVI(0xc0002ad5, "TODO_c000_2ad5", 0),
+    MVI(0xc0002ad6, "TODO_c000_2ad6", 0),
+    MVI(0xc0002ad7, "TODO_c000_2ad7", 0),
+    MVI(0xc0002ad8, "TODO_c000_2ad8", 0),
+    MVI(0xc0002ad9, "TODO_c000_2ad9", 0),
+    MVI(0xc0002ada, "TODO_c000_2ada", 0),
+    MVI(0xc0002adb, "TODO_c000_2adb", 0),
+    MVI(0xc0002adc, "TODO_c000_2adc", 0),
+    MVI(0xc0002add, "TODO_c000_2add", 0),
+    MVI(0xc0002ade, "TODO_c000_2ade", 0),
+    MVI(0xc0002adf, "TODO_c000_2adf", 0),
+    MVI(0xc0002ae0, "TODO_c000_2ae0", 0),
+    MVI(0xc0002ae1, "TODO_c000_2ae1", 0),
+    MVI(0xc0002ae2, "TODO_c000_2ae2", 0),
+    MVI(0xc0002ae3, "TODO_c000_2ae3", 0),
+    MVI(0xc0002ae4, "TODO_c000_2ae4", 0),
+    MVI(0xc0002ae5, "TODO_c000_2ae5", 0),
+    MVI(0xc0002ae6, "TODO_c000_2ae6", 0),
+    MVI(0xc0002ae7, "TODO_c000_2ae7", 0),
+    MVI(0xc0002ae8, "TODO_c000_2ae8", 0),
+    MVI(0xc0002ae9, "TODO_c000_2ae9", 0),
+    MVI(0xc0002aea, "TODO_c000_2aea", 0),
+    MVI(0xc0002aeb, "TODO_c000_2aeb", 0),
+    MVI(0xc0002aec, "TODO_c000_2aec", 0),
+    MVI(0xc0002aed, "TODO_c000_2aed", 0),
+    MVI(0xc0002aee, "TODO_c000_2aee", 0),
+    MVI(0xc0002aef, "TODO_c000_2aef", 0),
+    MVI(0xc0002af0, "TODO_c000_2af0", 0),
+    MVI(0xc0002af1, "TODO_c000_2af1", 0),
+    MVI(0xc0002af2, "TODO_c000_2af2", 0),
+    MVI(0xc0002af3, "TODO_c000_2af3", 0),
+    MVI(0xc0002af4, "TODO_c000_2af4", 0),
+    MVI(0xc0002af5, "TODO_c000_2af5", 0),
+    MVI(0xc0002af6, "TODO_c000_2af6", 0),
+    MVI(0xc0002af7, "TODO_c000_2af7", 0),
+    MVI(0xc0002af8, "TODO_c000_2af8", 0),
+    MVI(0xc0002af9, "TODO_c000_2af9", 0),
+    MVI(0xc0002afa, "TODO_c000_2afa", 0),
+    MVI(0xc0002afb, "TODO_c000_2afb", 0),
+    MVI(0xc0002afc, "TODO_c000_2afc", 0),
+    MVI(0xc0002afd, "TODO_c000_2afd", 0),
+    MVI(0xc0002afe, "TODO_c000_2afe", 0),
+    MVI(0xc0002aff, "TODO_c000_2aff", 0),
+    MVI(0xc0002b00, "TODO_c000_2b00", 0),
+    MVI(0xc0002b01, "TODO_c000_2b01", 0),
+    MVI(0xc0002b02, "TODO_c000_2b02", 0),
+    MVI(0xc0002b03, "TODO_c000_2b03", 0),
+    MVI(0xc0002b04, "TODO_c000_2b04", 0),
+    MVI(0xc0002b05, "TODO_c000_2b05", 0),
+    MVI(0xc0002b06, "TODO_c000_2b06", 0),
+    MVI(0xc0002b07, "TODO_c000_2b07", 0),
+    MVI(0xc0002b08, "TODO_c000_2b08", 0),
+    MVI(0xc0002b09, "TODO_c000_2b09", 0),
+    MVI(0xc0002b0a, "TODO_c000_2b0a", 0),
+    MVI(0xc0002b0b, "TODO_c000_2b0b", 0),
+    MVI(0xc0002b0c, "TODO_c000_2b0c", 0),
+    MVI(0xc0002b0d, "TODO_c000_2b0d", 0),
+    MVI(0xc0002b0e, "TODO_c000_2b0e", 0),
+    MVI(0xc0002b0f, "TODO_c000_2b0f", 0),
+    MVI(0xc0002b10, "TODO_c000_2b10", 0),
+    MVI(0xc0002b11, "TODO_c000_2b11", 0),
+    MVI(0xc0002b12, "TODO_c000_2b12", 0),
+    MVI(0xc0002b13, "TODO_c000_2b13", 0),
+    MVI(0xc0002b14, "TODO_c000_2b14", 0),
+    MVI(0xc0002b15, "TODO_c000_2b15", 0),
+    MVI(0xc0002b16, "TODO_c000_2b16", 0),
+    MVI(0xc0002b17, "TODO_c000_2b17", 0),
+    MVI(0xc0002b18, "TODO_c000_2b18", 0),
+    MVI(0xc0002b19, "TODO_c000_2b19", 0),
+    MVI(0xc0002b1a, "TODO_c000_2b1a", 0),
+    MVI(0xc0002b1b, "TODO_c000_2b1b", 0),
+    MVI(0xc0002b1c, "TODO_c000_2b1c", 0),
+    MVI(0xc0002b1d, "TODO_c000_2b1d", 0),
+    MVI(0xc0002b1e, "TODO_c000_2b1e", 0),
+    MVI(0xc0002b1f, "TODO_c000_2b1f", 0),
+    MVI(0xc0002b20, "TODO_c000_2b20", 0),
+    MVI(0xc0002b21, "TODO_c000_2b21", 0),
+    MVI(0xc0002b22, "TODO_c000_2b22", 0),
+    MVI(0xc0002b23, "TODO_c000_2b23", 0),
+    MVI(0xc0002b24, "TODO_c000_2b24", 0),
+    MVI(0xc0002b25, "TODO_c000_2b25", 0),
+    MVI(0xc0002b26, "TODO_c000_2b26", 0),
+    MVI(0xc0002b27, "TODO_c000_2b27", 0),
+    MVI(0xc0002b28, "TODO_c000_2b28", 0),
+    MVI(0xc0002b29, "TODO_c000_2b29", 0),
+    MVI(0xc0002b2a, "TODO_c000_2b2a", 0),
+    MVI(0xc0002b2b, "TODO_c000_2b2b", 0),
+    MVI(0xc0002b2c, "TODO_c000_2b2c", 0),
+    MVI(0xc0002b2d, "TODO_c000_2b2d", 0),
+    MVI(0xc0002b2e, "TODO_c000_2b2e", 0),
+    MVI(0xc0002b2f, "TODO_c000_2b2f", 0),
+    MVI(0xc0002b30, "TODO_c000_2b30", 0),
+    MVI(0xc0002b31, "TODO_c000_2b31", 0),
+    MVI(0xc0002b32, "TODO_c000_2b32", 0),
+    MVI(0xc0002b33, "TODO_c000_2b33", 0),
+    MVI(0xc0002b34, "TODO_c000_2b34", 0),
+    MVI(0xc0002b35, "TODO_c000_2b35", 0),
+    MVI(0xc0002b36, "TODO_c000_2b36", 0),
+    MVI(0xc0002b37, "TODO_c000_2b37", 0),
+    MVI(0xc0002b38, "TODO_c000_2b38", 0),
+    MVI(0xc0002b39, "TODO_c000_2b39", 0),
+    MVI(0xc0002b3a, "TODO_c000_2b3a", 0),
+    MVI(0xc0002b3b, "TODO_c000_2b3b", 0),
+    MVI(0xc0002b3c, "TODO_c000_2b3c", 0),
+    MVI(0xc0002b3d, "TODO_c000_2b3d", 0),
+    MVI(0xc0002b3e, "TODO_c000_2b3e", 0),
+    MVI(0xc0002b3f, "TODO_c000_2b3f", 0),
+    MVI(0xc0002b40, "TODO_c000_2b40", 0),
+    MVI(0xc0002b41, "TODO_c000_2b41", 0),
+    MVI(0xc0002b42, "TODO_c000_2b42", 0),
+    MVI(0xc0002b43, "TODO_c000_2b43", 0),
+    MVI(0xc0002b44, "TODO_c000_2b44", 0),
+    MVI(0xc0002b45, "TODO_c000_2b45", 0),
+    MVI(0xc0002b46, "TODO_c000_2b46", 0),
+    MVI(0xc0002b47, "TODO_c000_2b47", 0),
+    MVI(0xc0002b48, "TODO_c000_2b48", 0),
+    MVI(0xc0002b49, "TODO_c000_2b49", 0),
+    MVI(0xc0002b4a, "TODO_c000_2b4a", 0),
+    MVI(0xc0002b4b, "TODO_c000_2b4b", 0),
+    MVI(0xc0002b4c, "TODO_c000_2b4c", 0),
+    MVI(0xc0002b4d, "TODO_c000_2b4d", 0),
+    MVI(0xc0002b4e, "TODO_c000_2b4e", 0),
+    MVI(0xc0002b4f, "TODO_c000_2b4f", 0),
+    MVI(0xc0002b50, "TODO_c000_2b50", 0),
+    MVI(0xc0002b51, "TODO_c000_2b51", 0),
+    MVI(0xc0002b52, "TODO_c000_2b52", 0),
+    MVI(0xc0002b53, "TODO_c000_2b53", 0),
+    MVI(0xc0002b54, "TODO_c000_2b54", 0),
+    MVI(0xc0002b55, "TODO_c000_2b55", 0),
+    MVI(0xc0002b56, "TODO_c000_2b56", 0),
+    MVI(0xc0002b57, "TODO_c000_2b57", 0),
+    MVI(0xc0002b58, "TODO_c000_2b58", 0),
+    MVI(0xc0002b59, "TODO_c000_2b59", 0),
+    MVI(0xc0002b5a, "TODO_c000_2b5a", 0),
+    MVI(0xc0002b5b, "TODO_c000_2b5b", 0),
+    MVI(0xc0002b5c, "TODO_c000_2b5c", 0),
+    MVI(0xc0002b5d, "TODO_c000_2b5d", 0),
+    MVI(0xc0002b5e, "TODO_c000_2b5e", 0),
+    MVI(0xc0002b5f, "TODO_c000_2b5f", 0),
+    MVI(0xc0002b60, "TODO_c000_2b60", 0),
+    MVI(0xc0002b61, "TODO_c000_2b61", 0),
+    MVI(0xc0002b62, "TODO_c000_2b62", 0),
+    MVI(0xc0002b63, "TODO_c000_2b63", 0),
+    MVI(0xc0002b64, "TODO_c000_2b64", 0),
+    MVI(0xc0002b65, "TODO_c000_2b65", 0),
+    MVI(0xc0002b66, "TODO_c000_2b66", 0),
+    MVI(0xc0002b67, "TODO_c000_2b67", 0),
+    MVI(0xc0002b68, "TODO_c000_2b68", 0),
+    MVI(0xc0002b69, "TODO_c000_2b69", 0),
+    MVI(0xc0002b6a, "TODO_c000_2b6a", 0),
+    MVI(0xc0002b6b, "TODO_c000_2b6b", 0),
+    MVI(0xc0002b6c, "TODO_c000_2b6c", 0),
+    MVI(0xc0002b6d, "TODO_c000_2b6d", 0),
+    MVI(0xc0002b6e, "TODO_c000_2b6e", 0),
+    MVI(0xc0002b6f, "TODO_c000_2b6f", 0),
+    MVI(0xc0002b70, "TODO_c000_2b70", 0),
+    MVI(0xc0002b71, "TODO_c000_2b71", 0),
+    MVI(0xc0002b72, "TODO_c000_2b72", 0),
+    MVI(0xc0002b73, "TODO_c000_2b73", 0),
+    MVI(0xc0002b74, "TODO_c000_2b74", 0),
+    MVI(0xc0002b75, "TODO_c000_2b75", 0),
+    MVI(0xc0002b76, "TODO_c000_2b76", 0),
+    MVI(0xc0002b77, "TODO_c000_2b77", 0),
+    MVI(0xc0002b78, "TODO_c000_2b78", 0),
+    MVI(0xc0002b79, "TODO_c000_2b79", 0),
+    MVI(0xc0002b7a, "TODO_c000_2b7a", 0),
+    MVI(0xc0002b7b, "TODO_c000_2b7b", 0),
+    MVI(0xc0002b7c, "TODO_c000_2b7c", 0),
+    MVI(0xc0002b7d, "TODO_c000_2b7d", 0),
+    MVI(0xc0002b7e, "TODO_c000_2b7e", 0),
+    MVI(0xc0002b7f, "TODO_c000_2b7f", 0),
+    MVI(0xc0002b80, "TODO_c000_2b80", 0),
+    MVI(0xc0002b81, "TODO_c000_2b81", 0),
+    MVI(0xc0002b82, "TODO_c000_2b82", 0),
+    MVI(0xc0002b83, "TODO_c000_2b83", 0),
+    MVI(0xc0002b84, "TODO_c000_2b84", 0),
+    MVI(0xc0002b85, "TODO_c000_2b85", 0),
+    MVI(0xc0002b86, "TODO_c000_2b86", 0),
+    MVI(0xc0002b87, "TODO_c000_2b87", 0),
+    MVI(0xc0002b88, "TODO_c000_2b88", 0),
+    MVI(0xc0002b89, "TODO_c000_2b89", 0),
+    MVI(0xc0002b8a, "TODO_c000_2b8a", 0),
+    MVI(0xc0002b8b, "TODO_c000_2b8b", 0),
+    MVI(0xc0002b8c, "TODO_c000_2b8c", 0),
+    MVI(0xc0002b8d, "TODO_c000_2b8d", 0),
+    MVI(0xc0002b8e, "TODO_c000_2b8e", 0),
+    MVI(0xc0002b8f, "TODO_c000_2b8f", 0),
+    MVI(0xc0002b90, "TODO_c000_2b90", 0),
+    MVI(0xc0002b91, "TODO_c000_2b91", 0),
+    MVI(0xc0002b92, "TODO_c000_2b92", 0),
+    MVI(0xc0002b93, "TODO_c000_2b93", 0),
+    MVI(0xc0002b94, "TODO_c000_2b94", 0),
+    MVI(0xc0002b95, "TODO_c000_2b95", 0),
+    MVI(0xc0002b96, "TODO_c000_2b96", 0),
+    MVI(0xc0002b97, "TODO_c000_2b97", 0),
+    MVI(0xc0002b98, "TODO_c000_2b98", 0),
+    MVI(0xc0002b99, "TODO_c000_2b99", 0),
+    MVI(0xc0002b9a, "TODO_c000_2b9a", 0),
+    MVI(0xc0002b9b, "TODO_c000_2b9b", 0),
+    MVI(0xc0002b9c, "TODO_c000_2b9c", 0),
+    MVI(0xc0002b9d, "TODO_c000_2b9d", 0),
+    MVI(0xc0002b9e, "TODO_c000_2b9e", 0),
+    MVI(0xc0002b9f, "TODO_c000_2b9f", 0),
+    MVI(0xc0002ba0, "TODO_c000_2ba0", 0),
+    MVI(0xc0002ba1, "TODO_c000_2ba1", 0),
+    MVI(0xc0002ba2, "TODO_c000_2ba2", 0),
+    MVI(0xc0002ba3, "TODO_c000_2ba3", 0),
+    MVI(0xc0002ba4, "TODO_c000_2ba4", 0),
+    MVI(0xc0002ba5, "TODO_c000_2ba5", 0),
+    MVI(0xc0002ba6, "TODO_c000_2ba6", 0),
+    MVI(0xc0002ba7, "TODO_c000_2ba7", 0),
+    MVI(0xc0002ba8, "TODO_c000_2ba8", 0),
+    MVI(0xc0002ba9, "TODO_c000_2ba9", 0),
+    MVI(0xc0002baa, "TODO_c000_2baa", 0),
+    MVI(0xc0002bab, "TODO_c000_2bab", 0),
+    MVI(0xc0002bac, "TODO_c000_2bac", 0),
+    MVI(0xc0002bad, "TODO_c000_2bad", 0),
+    MVI(0xc0002bae, "TODO_c000_2bae", 0),
+    MVI(0xc0002baf, "TODO_c000_2baf", 0),
+    MVI(0xc0002bb0, "TODO_c000_2bb0", 0),
+    MVI(0xc0002bb1, "TODO_c000_2bb1", 0),
+    MVI(0xc0002bb2, "TODO_c000_2bb2", 0),
+    MVI(0xc0002bb3, "TODO_c000_2bb3", 0),
+    MVI(0xc0002bb4, "TODO_c000_2bb4", 0),
+    MVI(0xc0002bb5, "TODO_c000_2bb5", 0),
+    MVI(0xc0002bb6, "TODO_c000_2bb6", 0),
+    MVI(0xc0002bb7, "TODO_c000_2bb7", 0),
+    MVI(0xc0002bb8, "TODO_c000_2bb8", 0),
+    MVI(0xc0002bb9, "TODO_c000_2bb9", 0),
+    MVI(0xc0002bba, "TODO_c000_2bba", 0),
+    MVI(0xc0002bbb, "TODO_c000_2bbb", 0),
+    MVI(0xc0002bbc, "TODO_c000_2bbc", 0),
+    MVI(0xc0002bbd, "TODO_c000_2bbd", 0),
+    MVI(0xc0002bbe, "TODO_c000_2bbe", 0),
+    MVI(0xc0002bbf, "TODO_c000_2bbf", 0),
+    MVI(0xc0002bc0, "TODO_c000_2bc0", 0),
+    MVI(0xc0002bc1, "TODO_c000_2bc1", 0),
+    MVI(0xc0002bc2, "TODO_c000_2bc2", 0),
+    MVI(0xc0002bc3, "TODO_c000_2bc3", 0),
+    MVI(0xc0002bc4, "TODO_c000_2bc4", 0),
+    MVI(0xc0002bc5, "TODO_c000_2bc5", 0),
+    MVI(0xc0002bc6, "TODO_c000_2bc6", 0),
+    MVI(0xc0002bc7, "TODO_c000_2bc7", 0),
+    MVI(0xc0002bc8, "TODO_c000_2bc8", 0),
+    MVI(0xc0002bc9, "TODO_c000_2bc9", 0),
+    MVI(0xc0002bca, "TODO_c000_2bca", 0),
+    MVI(0xc0002bcb, "TODO_c000_2bcb", 0),
+    MVI(0xc0002bcc, "TODO_c000_2bcc", 0),
+    MVI(0xc0002bcd, "TODO_c000_2bcd", 0),
+    MVI(0xc0002bce, "TODO_c000_2bce", 0),
+    MVI(0xc0002bcf, "TODO_c000_2bcf", 0),
+    MVI(0xc0002bd0, "TODO_c000_2bd0", 0),
+    MVI(0xc0002bd1, "TODO_c000_2bd1", 0),
+    MVI(0xc0002bd2, "TODO_c000_2bd2", 0),
+    MVI(0xc0002bd3, "TODO_c000_2bd3", 0),
+    MVI(0xc0002bd4, "TODO_c000_2bd4", 0),
+    MVI(0xc0002bd5, "TODO_c000_2bd5", 0),
+    MVI(0xc0002bd6, "TODO_c000_2bd6", 0),
+    MVI(0xc0002bd7, "TODO_c000_2bd7", 0),
+    MVI(0xc0002bd8, "TODO_c000_2bd8", 0),
+    MVI(0xc0002bd9, "TODO_c000_2bd9", 0),
+    MVI(0xc0002bda, "TODO_c000_2bda", 0),
+    MVI(0xc0002bdb, "TODO_c000_2bdb", 0),
+    MVI(0xc0002bdc, "TODO_c000_2bdc", 0),
+    MVI(0xc0002bdd, "TODO_c000_2bdd", 0),
+    MVI(0xc0002bde, "TODO_c000_2bde", 0),
+    MVI(0xc0002bdf, "TODO_c000_2bdf", 0),
+    MVI(0xc0002be0, "TODO_c000_2be0", 0),
+    MVI(0xc0002be1, "TODO_c000_2be1", 0),
+    MVI(0xc0002be2, "TODO_c000_2be2", 0),
+    MVI(0xc0002be3, "TODO_c000_2be3", 0),
+    MVI(0xc0002be4, "TODO_c000_2be4", 0),
+    MVI(0xc0002be5, "TODO_c000_2be5", 0),
+    MVI(0xc0002be6, "TODO_c000_2be6", 0),
+    MVI(0xc0002be7, "TODO_c000_2be7", 0),
+    MVI(0xc0002be8, "TODO_c000_2be8", 0),
+    MVI(0xc0002be9, "TODO_c000_2be9", 0),
+    MVI(0xc0002bea, "TODO_c000_2bea", 0),
+    MVI(0xc0002beb, "TODO_c000_2beb", 0),
+    MVI(0xc0002bec, "TODO_c000_2bec", 0),
+    MVI(0xc0002bed, "TODO_c000_2bed", 0),
+    MVI(0xc0002bee, "TODO_c000_2bee", 0),
+    MVI(0xc0002bef, "TODO_c000_2bef", 0),
+    MVI(0xc0002bf0, "TODO_c000_2bf0", 0),
+    MVI(0xc0002bf1, "TODO_c000_2bf1", 0),
+    MVI(0xc0002bf2, "TODO_c000_2bf2", 0),
+    MVI(0xc0002bf3, "TODO_c000_2bf3", 0),
+    MVI(0xc0002bf4, "TODO_c000_2bf4", 0),
+    MVI(0xc0002bf5, "TODO_c000_2bf5", 0),
+    MVI(0xc0002bf6, "TODO_c000_2bf6", 0),
+    MVI(0xc0002bf7, "TODO_c000_2bf7", 0),
+    MVI(0xc0002bf8, "TODO_c000_2bf8", 0),
+    MVI(0xc0002bf9, "TODO_c000_2bf9", 0),
+    MVI(0xc0002bfa, "TODO_c000_2bfa", 0),
+    MVI(0xc0002bfb, "TODO_c000_2bfb", 0),
+    MVI(0xc0002bfc, "TODO_c000_2bfc", 0),
+    MVI(0xc0002bfd, "TODO_c000_2bfd", 0),
+    MVI(0xc0002bfe, "TODO_c000_2bfe", 0),
+    MVI(0xc0002bff, "TODO_c000_2bff", 0),
+    MVI(0xc0002c00, "TODO_c000_2c00", 0),
+    MVI(0xc0002c01, "TODO_c000_2c01", 0),
+    MVI(0xc0002c02, "TODO_c000_2c02", 0),
+    MVI(0xc0002c03, "TODO_c000_2c03", 0),
+    MVI(0xc0002c04, "TODO_c000_2c04", 0),
+    MVI(0xc0002c05, "TODO_c000_2c05", 0),
+    MVI(0xc0002c06, "TODO_c000_2c06", 0),
+    MVI(0xc0002c07, "TODO_c000_2c07", 0),
+    MVI(0xc0002c08, "TODO_c000_2c08", 0),
+    MVI(0xc0002c09, "TODO_c000_2c09", 0),
+    MVI(0xc0002c0a, "TODO_c000_2c0a", 0),
+    MVI(0xc0002c0b, "TODO_c000_2c0b", 0),
+    MVI(0xc0002c0c, "TODO_c000_2c0c", 0),
+    MVI(0xc0002c0d, "TODO_c000_2c0d", 0),
+    MVI(0xc0002c0e, "TODO_c000_2c0e", 0),
+    MVI(0xc0002c0f, "TODO_c000_2c0f", 0),
+    MVI(0xc0002c10, "TODO_c000_2c10", 0),
+    MVI(0xc0002c11, "TODO_c000_2c11", 0),
+    MVI(0xc0002c12, "TODO_c000_2c12", 0),
+    MVI(0xc0002c13, "TODO_c000_2c13", 0),
+    MVI(0xc0002c14, "TODO_c000_2c14", 0),
+    MVI(0xc0002c15, "TODO_c000_2c15", 0),
+    MVI(0xc0002c16, "TODO_c000_2c16", 0),
+    MVI(0xc0002c17, "TODO_c000_2c17", 0),
+    MVI(0xc0002c18, "TODO_c000_2c18", 0),
+    MVI(0xc0002c19, "TODO_c000_2c19", 0),
+    MVI(0xc0002c1a, "TODO_c000_2c1a", 0),
+    MVI(0xc0002c1b, "TODO_c000_2c1b", 0),
+    MVI(0xc0002c1c, "TODO_c000_2c1c", 0),
+    MVI(0xc0002c1d, "TODO_c000_2c1d", 0),
+    MVI(0xc0002c1e, "TODO_c000_2c1e", 0),
+    MVI(0xc0002c1f, "TODO_c000_2c1f", 0),
+    MVI(0xc0002c20, "TODO_c000_2c20", 0),
+    MVI(0xc0002c21, "TODO_c000_2c21", 0),
+    MVI(0xc0002c22, "TODO_c000_2c22", 0),
+    MVI(0xc0002c23, "TODO_c000_2c23", 0),
+    MVI(0xc0002c24, "TODO_c000_2c24", 0),
+    MVI(0xc0002c25, "TODO_c000_2c25", 0),
+    MVI(0xc0002c26, "TODO_c000_2c26", 0),
+    MVI(0xc0002c27, "TODO_c000_2c27", 0),
+    MVI(0xc0002c28, "TODO_c000_2c28", 0),
+    MVI(0xc0002c29, "TODO_c000_2c29", 0),
+    MVI(0xc0002c2a, "TODO_c000_2c2a", 0),
+    MVI(0xc0002c2b, "TODO_c000_2c2b", 0),
+    MVI(0xc0002c2c, "TODO_c000_2c2c", 0),
+    MVI(0xc0002c2d, "TODO_c000_2c2d", 0),
+    MVI(0xc0002c2e, "TODO_c000_2c2e", 0),
+    MVI(0xc0002c2f, "TODO_c000_2c2f", 0),
+    MVI(0xc0002c30, "TODO_c000_2c30", 0),
+    MVI(0xc0002c31, "TODO_c000_2c31", 0),
+    MVI(0xc0002c32, "TODO_c000_2c32", 0),
+    MVI(0xc0002c33, "TODO_c000_2c33", 0),
+    MVI(0xc0002c34, "TODO_c000_2c34", 0),
+    MVI(0xc0002c35, "TODO_c000_2c35", 0),
+    MVI(0xc0002c36, "TODO_c000_2c36", 0),
+    MVI(0xc0002c37, "TODO_c000_2c37", 0),
+    MVI(0xc0002c38, "TODO_c000_2c38", 0),
+    MVI(0xc0002c39, "TODO_c000_2c39", 0),
+    MVI(0xc0002c3a, "TODO_c000_2c3a", 0),
+    MVI(0xc0002c3b, "TODO_c000_2c3b", 0),
+    MVI(0xc0002c3c, "TODO_c000_2c3c", 0),
+    MVI(0xc0002c3d, "TODO_c000_2c3d", 0),
+    MVI(0xc0002c3e, "TODO_c000_2c3e", 0),
+    MVI(0xc0002c3f, "TODO_c000_2c3f", 0),
+    MVI(0xc0002c40, "TODO_c000_2c40", 0),
+    MVI(0xc0002c41, "TODO_c000_2c41", 0),
+    MVI(0xc0002c42, "TODO_c000_2c42", 0),
+    MVI(0xc0002c43, "TODO_c000_2c43", 0),
+    MVI(0xc0002c44, "TODO_c000_2c44", 0),
+    MVI(0xc0002c45, "TODO_c000_2c45", 0),
+    MVI(0xc0002c46, "TODO_c000_2c46", 0),
+    MVI(0xc0002c47, "TODO_c000_2c47", 0),
+    MVI(0xc0002c48, "TODO_c000_2c48", 0),
+    MVI(0xc0002c49, "TODO_c000_2c49", 0),
+    MVI(0xc0002c4a, "TODO_c000_2c4a", 0),
+    MVI(0xc0002c4b, "TODO_c000_2c4b", 0),
+    MVI(0xc0002c4c, "TODO_c000_2c4c", 0),
+    MVI(0xc0002c4d, "TODO_c000_2c4d", 0),
+    MVI(0xc0002c4e, "TODO_c000_2c4e", 0),
+    MVI(0xc0002c4f, "TODO_c000_2c4f", 0),
+    MVI(0xc0002c50, "TODO_c000_2c50", 0),
+    MVI(0xc0002c51, "TODO_c000_2c51", 0),
+    MVI(0xc0002c52, "TODO_c000_2c52", 0),
+    MVI(0xc0002c53, "TODO_c000_2c53", 0),
+    MVI(0xc0002c54, "TODO_c000_2c54", 0),
+    MVI(0xc0002c55, "TODO_c000_2c55", 0),
+    MVI(0xc0002c56, "TODO_c000_2c56", 0),
+    MVI(0xc0002c57, "TODO_c000_2c57", 0),
+    MVI(0xc0002c58, "TODO_c000_2c58", 0),
+    MVI(0xc0002c59, "TODO_c000_2c59", 0),
+    MVI(0xc0002c5a, "TODO_c000_2c5a", 0),
+    MVI(0xc0002c5b, "TODO_c000_2c5b", 0),
+    MVI(0xc0002c5c, "TODO_c000_2c5c", 0),
+    MVI(0xc0002c5d, "TODO_c000_2c5d", 0),
+    MVI(0xc0002c5e, "TODO_c000_2c5e", 0),
+    MVI(0xc0002c5f, "TODO_c000_2c5f", 0),
+    MVI(0xc0002c60, "TODO_c000_2c60", 0),
+    MVI(0xc0002c61, "TODO_c000_2c61", 0),
+    MVI(0xc0002c62, "TODO_c000_2c62", 0),
+    MVI(0xc0002c63, "TODO_c000_2c63", 0),
+    MVI(0xc0002c64, "TODO_c000_2c64", 0),
+    MVI(0xc0002c65, "TODO_c000_2c65", 0),
+    MVI(0xc0002c66, "TODO_c000_2c66", 0),
+    MVI(0xc0002c67, "TODO_c000_2c67", 0),
+    MVI(0xc0002c68, "TODO_c000_2c68", 0),
+    MVI(0xc0002c69, "TODO_c000_2c69", 0),
+    MVI(0xc0002c6a, "TODO_c000_2c6a", 0),
+    MVI(0xc0002c6b, "TODO_c000_2c6b", 0),
+    MVI(0xc0002c6c, "TODO_c000_2c6c", 0),
+    MVI(0xc0002c6d, "TODO_c000_2c6d", 0),
+    MVI(0xc0002c6e, "TODO_c000_2c6e", 0),
+    MVI(0xc0002c6f, "TODO_c000_2c6f", 0),
+    MVI(0xc0002c70, "TODO_c000_2c70", 0),
+    MVI(0xc0002c71, "TODO_c000_2c71", 0),
+    MVI(0xc0002c72, "TODO_c000_2c72", 0),
+    MVI(0xc0002c73, "TODO_c000_2c73", 0),
+    MVI(0xc0002c74, "TODO_c000_2c74", 0),
+    MVI(0xc0002c75, "TODO_c000_2c75", 0),
+    MVI(0xc0002c76, "TODO_c000_2c76", 0),
+    MVI(0xc0002c77, "TODO_c000_2c77", 0),
+    MVI(0xc0002c78, "TODO_c000_2c78", 0),
+    MVI(0xc0002c79, "TODO_c000_2c79", 0),
+    MVI(0xc0002c7a, "TODO_c000_2c7a", 0),
+    MVI(0xc0002c7b, "TODO_c000_2c7b", 0),
+    MVI(0xc0002c7c, "TODO_c000_2c7c", 0),
+    MVI(0xc0002c7d, "TODO_c000_2c7d", 0),
+    MVI(0xc0002c7e, "TODO_c000_2c7e", 0),
+    MVI(0xc0002c7f, "TODO_c000_2c7f", 0),
+    MVI(0xc0002c80, "TODO_c000_2c80", 0),
+    MVI(0xc0002c81, "TODO_c000_2c81", 0),
+    MVI(0xc0002c82, "TODO_c000_2c82", 0),
+    MVI(0xc0002c83, "TODO_c000_2c83", 0),
+    MVI(0xc0002c84, "TODO_c000_2c84", 0),
+    MVI(0xc0002c85, "TODO_c000_2c85", 0),
+    MVI(0xc0002c86, "TODO_c000_2c86", 0),
+    MVI(0xc0002c87, "TODO_c000_2c87", 0),
+    MVI(0xc0002c88, "TODO_c000_2c88", 0),
+    MVI(0xc0002c89, "TODO_c000_2c89", 0),
+    MVI(0xc0002c8a, "TODO_c000_2c8a", 0),
+    MVI(0xc0002c8b, "TODO_c000_2c8b", 0),
+    MVI(0xc0002c8c, "TODO_c000_2c8c", 0),
+    MVI(0xc0002c8d, "TODO_c000_2c8d", 0),
+    MVI(0xc0002c8e, "TODO_c000_2c8e", 0),
+    MVI(0xc0002c8f, "TODO_c000_2c8f", 0),
+    MVI(0xc0002c90, "TODO_c000_2c90", 0),
+    MVI(0xc0002c91, "TODO_c000_2c91", 0),
+    MVI(0xc0002c92, "TODO_c000_2c92", 0),
+    MVI(0xc0002c93, "TODO_c000_2c93", 0),
+    MVI(0xc0002c94, "TODO_c000_2c94", 0),
+    MVI(0xc0002c95, "TODO_c000_2c95", 0),
+    MVI(0xc0002c96, "TODO_c000_2c96", 0),
+    MVI(0xc0002c97, "TODO_c000_2c97", 0),
+    MVI(0xc0002c98, "TODO_c000_2c98", 0),
+    MVI(0xc0002c99, "TODO_c000_2c99", 0),
+    MVI(0xc0002c9a, "TODO_c000_2c9a", 0),
+    MVI(0xc0002c9b, "TODO_c000_2c9b", 0),
+    MVI(0xc0002c9c, "TODO_c000_2c9c", 0),
+    MVI(0xc0002c9d, "TODO_c000_2c9d", 0),
+    MVI(0xc0002c9e, "TODO_c000_2c9e", 0),
+    MVI(0xc0002c9f, "TODO_c000_2c9f", 0),
+    MVI(0xc0002ca0, "TODO_c000_2ca0", 0),
+    MVI(0xc0002ca1, "TODO_c000_2ca1", 0),
+    MVI(0xc0002ca2, "TODO_c000_2ca2", 0),
+    MVI(0xc0002ca3, "TODO_c000_2ca3", 0),
+    MVI(0xc0002ca4, "TODO_c000_2ca4", 0),
+    MVI(0xc0002ca5, "TODO_c000_2ca5", 0),
+    MVI(0xc0002ca6, "TODO_c000_2ca6", 0),
+    MVI(0xc0002ca7, "TODO_c000_2ca7", 0),
+    MVI(0xc0002ca8, "TODO_c000_2ca8", 0),
+    MVI(0xc0002ca9, "TODO_c000_2ca9", 0),
+    MVI(0xc0002caa, "TODO_c000_2caa", 0),
+    MVI(0xc0002cab, "TODO_c000_2cab", 0),
+    MVI(0xc0002cac, "TODO_c000_2cac", 0),
+    MVI(0xc0002cad, "TODO_c000_2cad", 0),
+    MVI(0xc0002cae, "TODO_c000_2cae", 0),
+    MVI(0xc0002caf, "TODO_c000_2caf", 0),
+    MVI(0xc0002cb0, "TODO_c000_2cb0", 0),
+    MVI(0xc0002cb1, "TODO_c000_2cb1", 0),
+    MVI(0xc0002cb2, "TODO_c000_2cb2", 0),
+    MVI(0xc0002cb3, "TODO_c000_2cb3", 0),
+    MVI(0xc0002cb4, "TODO_c000_2cb4", 0),
+    MVI(0xc0002cb5, "TODO_c000_2cb5", 0),
+    MVI(0xc0002cb6, "TODO_c000_2cb6", 0),
+    MVI(0xc0002cb7, "TODO_c000_2cb7", 0),
+    MVI(0xc0002cb8, "TODO_c000_2cb8", 0),
+    MVI(0xc0002cb9, "TODO_c000_2cb9", 0),
+    MVI(0xc0002cba, "TODO_c000_2cba", 0),
+    MVI(0xc0002cbb, "TODO_c000_2cbb", 0),
+    MVI(0xc0002cbc, "TODO_c000_2cbc", 0),
+    MVI(0xc0002cbd, "TODO_c000_2cbd", 0),
+    MVI(0xc0002cbe, "TODO_c000_2cbe", 0),
+    MVI(0xc0002cbf, "TODO_c000_2cbf", 0),
+    MVI(0xc0002cc0, "TODO_c000_2cc0", 0),
+    MVI(0xc0002cc1, "TODO_c000_2cc1", 0),
+    MVI(0xc0002cc2, "TODO_c000_2cc2", 0),
+    MVI(0xc0002cc3, "TODO_c000_2cc3", 0),
+    MVI(0xc0002cc4, "TODO_c000_2cc4", 0),
+    MVI(0xc0002cc5, "TODO_c000_2cc5", 0),
+    MVI(0xc0002cc6, "TODO_c000_2cc6", 0),
+    MVI(0xc0002cc7, "TODO_c000_2cc7", 0),
+    MVI(0xc0002cc8, "TODO_c000_2cc8", 0),
+    MVI(0xc0002cc9, "TODO_c000_2cc9", 0),
+    MVI(0xc0002cca, "TODO_c000_2cca", 0),
+    MVI(0xc0002ccb, "TODO_c000_2ccb", 0),
+    MVI(0xc0002ccc, "TODO_c000_2ccc", 0),
+    MVI(0xc0002ccd, "TODO_c000_2ccd", 0),
+    MVI(0xc0002cce, "TODO_c000_2cce", 0),
+    MVI(0xc0002ccf, "TODO_c000_2ccf", 0),
+    MVI(0xc0002cd0, "TODO_c000_2cd0", 0),
+    MVI(0xc0002cd1, "TODO_c000_2cd1", 0),
+    MVI(0xc0002cd2, "TODO_c000_2cd2", 0),
+    MVI(0xc0002cd3, "TODO_c000_2cd3", 0),
+    MVI(0xc0002cd4, "TODO_c000_2cd4", 0),
+    MVI(0xc0002cd5, "TODO_c000_2cd5", 0),
+    MVI(0xc0002cd6, "TODO_c000_2cd6", 0),
+    MVI(0xc0002cd7, "TODO_c000_2cd7", 0),
+    MVI(0xc0002cd8, "TODO_c000_2cd8", 0),
+    MVI(0xc0002cd9, "TODO_c000_2cd9", 0),
+    MVI(0xc0002cda, "TODO_c000_2cda", 0),
+    MVI(0xc0002cdb, "TODO_c000_2cdb", 0),
+    MVI(0xc0002cdc, "TODO_c000_2cdc", 0),
+    MVI(0xc0002cdd, "TODO_c000_2cdd", 0),
+    MVI(0xc0002cde, "TODO_c000_2cde", 0),
+    MVI(0xc0002cdf, "TODO_c000_2cdf", 0),
+    MVI(0xc0002ce0, "TODO_c000_2ce0", 0),
+    MVI(0xc0002ce1, "TODO_c000_2ce1", 0),
+    MVI(0xc0002ce2, "TODO_c000_2ce2", 0),
+    MVI(0xc0002ce3, "TODO_c000_2ce3", 0),
+    MVI(0xc0002ce4, "TODO_c000_2ce4", 0),
+    MVI(0xc0002ce5, "TODO_c000_2ce5", 0),
+    MVI(0xc0002ce6, "TODO_c000_2ce6", 0),
+    MVI(0xc0002ce7, "TODO_c000_2ce7", 0),
+    MVI(0xc0002ce8, "TODO_c000_2ce8", 0),
+    MVI(0xc0002ce9, "TODO_c000_2ce9", 0),
+    MVI(0xc0002cea, "TODO_c000_2cea", 0),
+    MVI(0xc0002ceb, "TODO_c000_2ceb", 0),
+    MVI(0xc0002cec, "TODO_c000_2cec", 0),
+    MVI(0xc0002ced, "TODO_c000_2ced", 0),
+    MVI(0xc0002cee, "TODO_c000_2cee", 0),
+    MVI(0xc0002cef, "TODO_c000_2cef", 0),
+    MVI(0xc0002cf0, "TODO_c000_2cf0", 0),
+    MVI(0xc0002cf1, "TODO_c000_2cf1", 0),
+    MVI(0xc0002cf2, "TODO_c000_2cf2", 0),
+    MVI(0xc0002cf3, "TODO_c000_2cf3", 0),
+    MVI(0xc0002cf4, "TODO_c000_2cf4", 0),
+    MVI(0xc0002cf5, "TODO_c000_2cf5", 0),
+    MVI(0xc0002cf6, "TODO_c000_2cf6", 0),
+    MVI(0xc0002cf7, "TODO_c000_2cf7", 0),
+    MVI(0xc0002cf8, "TODO_c000_2cf8", 0),
+    MVI(0xc0002cf9, "TODO_c000_2cf9", 0),
+    MVI(0xc0002cfa, "TODO_c000_2cfa", 0),
+    MVI(0xc0002cfb, "TODO_c000_2cfb", 0),
+    MVI(0xc0002cfc, "TODO_c000_2cfc", 0),
+    MVI(0xc0002cfd, "TODO_c000_2cfd", 0),
+    MVI(0xc0002cfe, "TODO_c000_2cfe", 0),
+    MVI(0xc0002cff, "TODO_c000_2cff", 0),
+    MVI(0xc0002d00, "TODO_c000_2d00", 0),
+    MVI(0xc0002d01, "TODO_c000_2d01", 0),
+    MVI(0xc0002d02, "TODO_c000_2d02", 0),
+    MVI(0xc0002d03, "TODO_c000_2d03", 0),
+    MVI(0xc0002d04, "TODO_c000_2d04", 0),
+    MVI(0xc0002d05, "TODO_c000_2d05", 0),
+    MVI(0xc0002d06, "TODO_c000_2d06", 0),
+    MVI(0xc0002d07, "TODO_c000_2d07", 0),
+    MVI(0xc0002d08, "TODO_c000_2d08", 0),
+    MVI(0xc0002d09, "TODO_c000_2d09", 0),
+    MVI(0xc0002d0a, "TODO_c000_2d0a", 0),
+    MVI(0xc0002d0b, "TODO_c000_2d0b", 0),
+    MVI(0xc0002d0c, "TODO_c000_2d0c", 0),
+    MVI(0xc0002d0d, "TODO_c000_2d0d", 0),
+    MVI(0xc0002d0e, "TODO_c000_2d0e", 0),
+    MVI(0xc0002d0f, "TODO_c000_2d0f", 0),
+    MVI(0xc0002d10, "TODO_c000_2d10", 0),
+    MVI(0xc0002d11, "TODO_c000_2d11", 0),
+    MVI(0xc0002d12, "TODO_c000_2d12", 0),
+    MVI(0xc0002d13, "TODO_c000_2d13", 0),
+    MVI(0xc0002d14, "TODO_c000_2d14", 0),
+    MVI(0xc0002d15, "TODO_c000_2d15", 0),
+    MVI(0xc0002d16, "TODO_c000_2d16", 0),
+    MVI(0xc0002d17, "TODO_c000_2d17", 0),
+    MVI(0xc0002d18, "TODO_c000_2d18", 0),
+    MVI(0xc0002d19, "TODO_c000_2d19", 0),
+    MVI(0xc0002d1a, "TODO_c000_2d1a", 0),
+    MVI(0xc0002d1b, "TODO_c000_2d1b", 0),
+    MVI(0xc0002d1c, "TODO_c000_2d1c", 0),
+    MVI(0xc0002d1d, "TODO_c000_2d1d", 0),
+    MVI(0xc0002d1e, "TODO_c000_2d1e", 0),
+    MVI(0xc0002d1f, "TODO_c000_2d1f", 0),
+    MVI(0xc0002d20, "TODO_c000_2d20", 0),
+    MVI(0xc0002d21, "TODO_c000_2d21", 0),
+    MVI(0xc0002d22, "TODO_c000_2d22", 0),
+    MVI(0xc0002d23, "TODO_c000_2d23", 0),
+    MVI(0xc0002d24, "TODO_c000_2d24", 0),
+    MVI(0xc0002d25, "TODO_c000_2d25", 0),
+    MVI(0xc0002d26, "TODO_c000_2d26", 0),
+    MVI(0xc0002d27, "TODO_c000_2d27", 0),
+    MVI(0xc0002d28, "TODO_c000_2d28", 0),
+    MVI(0xc0002d29, "TODO_c000_2d29", 0),
+    MVI(0xc0002d2a, "TODO_c000_2d2a", 0),
+    MVI(0xc0002d2b, "TODO_c000_2d2b", 0),
+    MVI(0xc0002d2c, "TODO_c000_2d2c", 0),
+    MVI(0xc0002d2d, "TODO_c000_2d2d", 0),
+    MVI(0xc0002d2e, "TODO_c000_2d2e", 0),
+    MVI(0xc0002d2f, "TODO_c000_2d2f", 0),
+    MVI(0xc0002d30, "TODO_c000_2d30", 0),
+    MVI(0xc0002d31, "TODO_c000_2d31", 0),
+    MVI(0xc0002d32, "TODO_c000_2d32", 0),
+    MVI(0xc0002d33, "TODO_c000_2d33", 0),
+    MVI(0xc0002d34, "TODO_c000_2d34", 0),
+    MVI(0xc0002d35, "TODO_c000_2d35", 0),
+    MVI(0xc0002d36, "TODO_c000_2d36", 0),
+    MVI(0xc0002d37, "TODO_c000_2d37", 0),
+    MVI(0xc0002d38, "TODO_c000_2d38", 0),
+    MVI(0xc0002d39, "TODO_c000_2d39", 0),
+    MVI(0xc0002d3a, "TODO_c000_2d3a", 0),
+    MVI(0xc0002d3b, "TODO_c000_2d3b", 0),
+    MVI(0xc0002d3c, "TODO_c000_2d3c", 0),
+    MVI(0xc0002d3d, "TODO_c000_2d3d", 0),
+    MVI(0xc0002d3e, "TODO_c000_2d3e", 0),
+    MVI(0xc0002d3f, "TODO_c000_2d3f", 0),
+    MVI(0xc0002d40, "TODO_c000_2d40", 0),
+    MVI(0xc0002d41, "TODO_c000_2d41", 0),
+    MVI(0xc0002d42, "TODO_c000_2d42", 0),
+    MVI(0xc0002d43, "TODO_c000_2d43", 0),
+    MVI(0xc0002d44, "TODO_c000_2d44", 0),
+    MVI(0xc0002d45, "TODO_c000_2d45", 0),
+    MVI(0xc0002d46, "TODO_c000_2d46", 0),
+    MVI(0xc0002d47, "TODO_c000_2d47", 0),
+    MVI(0xc0002d48, "TODO_c000_2d48", 0),
+    MVI(0xc0002d49, "TODO_c000_2d49", 0),
+    MVI(0xc0002d4a, "TODO_c000_2d4a", 0),
+    MVI(0xc0002d4b, "TODO_c000_2d4b", 0),
+    MVI(0xc0002d4c, "TODO_c000_2d4c", 0),
+    MVI(0xc0002d4d, "TODO_c000_2d4d", 0),
+    MVI(0xc0002d4e, "TODO_c000_2d4e", 0),
+    MVI(0xc0002d4f, "TODO_c000_2d4f", 0),
+    MVI(0xc0002d50, "TODO_c000_2d50", 0),
+    MVI(0xc0002d51, "TODO_c000_2d51", 0),
+    MVI(0xc0002d52, "TODO_c000_2d52", 0),
+    MVI(0xc0002d53, "TODO_c000_2d53", 0),
+    MVI(0xc0002d54, "TODO_c000_2d54", 0),
+    MVI(0xc0002d55, "TODO_c000_2d55", 0),
+    MVI(0xc0002d56, "TODO_c000_2d56", 0),
+    MVI(0xc0002d57, "TODO_c000_2d57", 0),
+    MVI(0xc0002d58, "TODO_c000_2d58", 0),
+    MVI(0xc0002d59, "TODO_c000_2d59", 0),
+    MVI(0xc0002d5a, "TODO_c000_2d5a", 0),
+    MVI(0xc0002d5b, "TODO_c000_2d5b", 0),
+    MVI(0xc0002d5c, "TODO_c000_2d5c", 0),
+    MVI(0xc0002d5d, "TODO_c000_2d5d", 0),
+    MVI(0xc0002d5e, "TODO_c000_2d5e", 0),
+    MVI(0xc0002d5f, "TODO_c000_2d5f", 0),
+    MVI(0xc0002d60, "TODO_c000_2d60", 0),
+    MVI(0xc0002d61, "TODO_c000_2d61", 0),
+    MVI(0xc0002d62, "TODO_c000_2d62", 0),
+    MVI(0xc0002d63, "TODO_c000_2d63", 0),
+    MVI(0xc0002d64, "TODO_c000_2d64", 0),
+    MVI(0xc0002d65, "TODO_c000_2d65", 0),
+    MVI(0xc0002d66, "TODO_c000_2d66", 0),
+    MVI(0xc0002d67, "TODO_c000_2d67", 0),
+    MVI(0xc0002d68, "TODO_c000_2d68", 0),
+    MVI(0xc0002d69, "TODO_c000_2d69", 0),
+    MVI(0xc0002d6a, "TODO_c000_2d6a", 0),
+    MVI(0xc0002d6b, "TODO_c000_2d6b", 0),
+    MVI(0xc0002d6c, "TODO_c000_2d6c", 0),
+    MVI(0xc0002d6d, "TODO_c000_2d6d", 0),
+    MVI(0xc0002d6e, "TODO_c000_2d6e", 0),
+    MVI(0xc0002d6f, "TODO_c000_2d6f", 0),
+    MVI(0xc0002d70, "TODO_c000_2d70", 0),
+    MVI(0xc0002d71, "TODO_c000_2d71", 0),
+    MVI(0xc0002d72, "TODO_c000_2d72", 0),
+    MVI(0xc0002d73, "TODO_c000_2d73", 0),
+    MVI(0xc0002d74, "TODO_c000_2d74", 0),
+    MVI(0xc0002d75, "TODO_c000_2d75", 0),
+    MVI(0xc0002d76, "TODO_c000_2d76", 0),
+    MVI(0xc0002d77, "TODO_c000_2d77", 0),
+    MVI(0xc0002d78, "TODO_c000_2d78", 0),
+    MVI(0xc0002d79, "TODO_c000_2d79", 0),
+    MVI(0xc0002d7a, "TODO_c000_2d7a", 0),
+    MVI(0xc0002d7b, "TODO_c000_2d7b", 0),
+    MVI(0xc0002d7c, "TODO_c000_2d7c", 0),
+    MVI(0xc0002d7d, "TODO_c000_2d7d", 0),
+    MVI(0xc0002d7e, "TODO_c000_2d7e", 0),
+    MVI(0xc0002d7f, "TODO_c000_2d7f", 0),
+    MVI(0xc0002d80, "TODO_c000_2d80", 0),
+    MVI(0xc0002d81, "TODO_c000_2d81", 0),
+    MVI(0xc0002d82, "TODO_c000_2d82", 0),
+    MVI(0xc0002d83, "TODO_c000_2d83", 0),
+    MVI(0xc0002d84, "TODO_c000_2d84", 0),
+    MVI(0xc0002d85, "TODO_c000_2d85", 0),
+    MVI(0xc0002d86, "TODO_c000_2d86", 0),
+    MVI(0xc0002d87, "TODO_c000_2d87", 0),
+    MVI(0xc0002d88, "TODO_c000_2d88", 0),
+    MVI(0xc0002d89, "TODO_c000_2d89", 0),
+    MVI(0xc0002d8a, "TODO_c000_2d8a", 0),
+    MVI(0xc0002d8b, "TODO_c000_2d8b", 0),
+    MVI(0xc0002d8c, "TODO_c000_2d8c", 0),
+    MVI(0xc0002d8d, "TODO_c000_2d8d", 0),
+    MVI(0xc0002d8e, "TODO_c000_2d8e", 0),
+    MVI(0xc0002d8f, "TODO_c000_2d8f", 0),
+    MVI(0xc0002d90, "TODO_c000_2d90", 0),
+    MVI(0xc0002d91, "TODO_c000_2d91", 0),
+    MVI(0xc0002d92, "TODO_c000_2d92", 0),
+    MVI(0xc0002d93, "TODO_c000_2d93", 0),
+    MVI(0xc0002d94, "TODO_c000_2d94", 0),
+    MVI(0xc0002d95, "TODO_c000_2d95", 0),
+    MVI(0xc0002d96, "TODO_c000_2d96", 0),
+    MVI(0xc0002d97, "TODO_c000_2d97", 0),
+    MVI(0xc0002d98, "TODO_c000_2d98", 0),
+    MVI(0xc0002d99, "TODO_c000_2d99", 0),
+    MVI(0xc0002d9a, "TODO_c000_2d9a", 0),
+    MVI(0xc0002d9b, "TODO_c000_2d9b", 0),
+    MVI(0xc0002d9c, "TODO_c000_2d9c", 0),
+    MVI(0xc0002d9d, "TODO_c000_2d9d", 0),
+    MVI(0xc0002d9e, "TODO_c000_2d9e", 0),
+    MVI(0xc0002d9f, "TODO_c000_2d9f", 0),
+    MVI(0xc0002da0, "TODO_c000_2da0", 0),
+    MVI(0xc0002da1, "TODO_c000_2da1", 0),
+    MVI(0xc0002da2, "TODO_c000_2da2", 0),
+    MVI(0xc0002da3, "TODO_c000_2da3", 0),
+    MVI(0xc0002da4, "TODO_c000_2da4", 0),
+    MVI(0xc0002da5, "TODO_c000_2da5", 0),
+    MVI(0xc0002da6, "TODO_c000_2da6", 0),
+    MVI(0xc0002da7, "TODO_c000_2da7", 0),
+    MVI(0xc0002da8, "TODO_c000_2da8", 0),
+    MVI(0xc0002da9, "TODO_c000_2da9", 0),
+    MVI(0xc0002daa, "TODO_c000_2daa", 0),
+    MVI(0xc0002dab, "TODO_c000_2dab", 0),
+    MVI(0xc0002dac, "TODO_c000_2dac", 0),
+    MVI(0xc0002dad, "TODO_c000_2dad", 0),
+    MVI(0xc0002dae, "TODO_c000_2dae", 0),
+    MVI(0xc0002daf, "TODO_c000_2daf", 0),
+    MVI(0xc0002db0, "TODO_c000_2db0", 0),
+    MVI(0xc0002db1, "TODO_c000_2db1", 0),
+    MVI(0xc0002db2, "TODO_c000_2db2", 0),
+    MVI(0xc0002db3, "TODO_c000_2db3", 0),
+    MVI(0xc0002db4, "TODO_c000_2db4", 0),
+    MVI(0xc0002db5, "TODO_c000_2db5", 0),
+    MVI(0xc0002db6, "TODO_c000_2db6", 0),
+    MVI(0xc0002db7, "TODO_c000_2db7", 0),
+    MVI(0xc0002db8, "TODO_c000_2db8", 0),
+    MVI(0xc0002db9, "TODO_c000_2db9", 0),
+    MVI(0xc0002dba, "TODO_c000_2dba", 0),
+    MVI(0xc0002dbb, "TODO_c000_2dbb", 0),
+    MVI(0xc0002dbc, "TODO_c000_2dbc", 0),
+    MVI(0xc0002dbd, "TODO_c000_2dbd", 0),
+    MVI(0xc0002dbe, "TODO_c000_2dbe", 0),
+    MVI(0xc0002dbf, "TODO_c000_2dbf", 0),
+    MVI(0xc0002dc0, "TODO_c000_2dc0", 0),
+    MVI(0xc0002dc1, "TODO_c000_2dc1", 0),
+    MVI(0xc0002dc2, "TODO_c000_2dc2", 0),
+    MVI(0xc0002dc3, "TODO_c000_2dc3", 0),
+    MVI(0xc0002dc4, "TODO_c000_2dc4", 0),
+    MVI(0xc0002dc5, "TODO_c000_2dc5", 0),
+    MVI(0xc0002dc6, "TODO_c000_2dc6", 0),
+    MVI(0xc0002dc7, "TODO_c000_2dc7", 0),
+    MVI(0xc0002dc8, "TODO_c000_2dc8", 0),
+    MVI(0xc0002dc9, "TODO_c000_2dc9", 0),
+    MVI(0xc0002dca, "TODO_c000_2dca", 0),
+    MVI(0xc0002dcb, "TODO_c000_2dcb", 0),
+    MVI(0xc0002dcc, "TODO_c000_2dcc", 0),
+    MVI(0xc0002dcd, "TODO_c000_2dcd", 0),
+    MVI(0xc0002dce, "TODO_c000_2dce", 0),
+    MVI(0xc0002dcf, "TODO_c000_2dcf", 0),
+    MVI(0xc0002dd0, "TODO_c000_2dd0", 0),
+    MVI(0xc0002dd1, "TODO_c000_2dd1", 0),
+    MVI(0xc0002dd2, "TODO_c000_2dd2", 0),
+    MVI(0xc0002dd3, "TODO_c000_2dd3", 0),
+    MVI(0xc0002dd4, "TODO_c000_2dd4", 0),
+    MVI(0xc0002dd5, "TODO_c000_2dd5", 0),
+    MVI(0xc0002dd6, "TODO_c000_2dd6", 0),
+    MVI(0xc0002dd7, "TODO_c000_2dd7", 0),
+    MVI(0xc0002dd8, "TODO_c000_2dd8", 0),
+    MVI(0xc0002dd9, "TODO_c000_2dd9", 0),
+    MVI(0xc0002dda, "TODO_c000_2dda", 0),
+    MVI(0xc0002ddb, "TODO_c000_2ddb", 0),
+    MVI(0xc0002ddc, "TODO_c000_2ddc", 0),
+    MVI(0xc0002ddd, "TODO_c000_2ddd", 0),
+    MVI(0xc0002dde, "TODO_c000_2dde", 0),
+    MVI(0xc0002ddf, "TODO_c000_2ddf", 0),
+    MVI(0xc0002de0, "TODO_c000_2de0", 0),
+    MVI(0xc0002de1, "TODO_c000_2de1", 0),
+    MVI(0xc0002de2, "TODO_c000_2de2", 0),
+    MVI(0xc0002de3, "TODO_c000_2de3", 0),
+    MVI(0xc0002de4, "TODO_c000_2de4", 0),
+    MVI(0xc0002de5, "TODO_c000_2de5", 0),
+    MVI(0xc0002de6, "TODO_c000_2de6", 0),
+    MVI(0xc0002de7, "TODO_c000_2de7", 0),
+    MVI(0xc0002de8, "TODO_c000_2de8", 0),
+    MVI(0xc0002de9, "TODO_c000_2de9", 0),
+    MVI(0xc0002dea, "TODO_c000_2dea", 0),
+    MVI(0xc0002deb, "TODO_c000_2deb", 0),
+    MVI(0xc0002dec, "TODO_c000_2dec", 0),
+    MVI(0xc0002ded, "TODO_c000_2ded", 0),
+    MVI(0xc0002dee, "TODO_c000_2dee", 0),
+    MVI(0xc0002def, "TODO_c000_2def", 0),
+    MVI(0xc0002df0, "TODO_c000_2df0", 0),
+    MVI(0xc0002df1, "TODO_c000_2df1", 0),
+    MVI(0xc0002df2, "TODO_c000_2df2", 0),
+    MVI(0xc0002df3, "TODO_c000_2df3", 0),
+    MVI(0xc0002df4, "TODO_c000_2df4", 0),
+    MVI(0xc0002df5, "TODO_c000_2df5", 0),
+    MVI(0xc0002df6, "TODO_c000_2df6", 0),
+    MVI(0xc0002df7, "TODO_c000_2df7", 0),
+    MVI(0xc0002df8, "TODO_c000_2df8", 0),
+    MVI(0xc0002df9, "TODO_c000_2df9", 0),
+    MVI(0xc0002dfa, "TODO_c000_2dfa", 0),
+    MVI(0xc0002dfb, "TODO_c000_2dfb", 0),
+    MVI(0xc0002dfc, "TODO_c000_2dfc", 0),
+    MVI(0xc0002dfd, "TODO_c000_2dfd", 0),
+    MVI(0xc0002dfe, "TODO_c000_2dfe", 0),
+    MVI(0xc0002dff, "TODO_c000_2dff", 0),
+    MVI(0xc0002e00, "TODO_c000_2e00", 0),
+    MVI(0xc0002e01, "TODO_c000_2e01", 0),
+    MVI(0xc0002e02, "TODO_c000_2e02", 0),
+    MVI(0xc0002e03, "TODO_c000_2e03", 0),
+    MVI(0xc0002e04, "TODO_c000_2e04", 0),
+    MVI(0xc0002e05, "TODO_c000_2e05", 0),
+    MVI(0xc0002e06, "TODO_c000_2e06", 0),
+    MVI(0xc0002e07, "TODO_c000_2e07", 0),
+    MVI(0xc0002e08, "TODO_c000_2e08", 0),
+    MVI(0xc0002e09, "TODO_c000_2e09", 0),
+    MVI(0xc0002e0a, "TODO_c000_2e0a", 0),
+    MVI(0xc0002e0b, "TODO_c000_2e0b", 0),
+    MVI(0xc0002e0c, "TODO_c000_2e0c", 0),
+    MVI(0xc0002e0d, "TODO_c000_2e0d", 0),
+    MVI(0xc0002e0e, "TODO_c000_2e0e", 0),
+    MVI(0xc0002e0f, "TODO_c000_2e0f", 0),
+    MVI(0xc0002e10, "TODO_c000_2e10", 0),
+    MVI(0xc0002e11, "TODO_c000_2e11", 0),
+    MVI(0xc0002e12, "TODO_c000_2e12", 0),
+    MVI(0xc0002e13, "TODO_c000_2e13", 0),
+    MVI(0xc0002e14, "TODO_c000_2e14", 0),
+    MVI(0xc0002e15, "TODO_c000_2e15", 0),
+    MVI(0xc0002e16, "TODO_c000_2e16", 0),
+    MVI(0xc0002e17, "TODO_c000_2e17", 0),
+    MVI(0xc0002e18, "TODO_c000_2e18", 0),
+    MVI(0xc0002e19, "TODO_c000_2e19", 0),
+    MVI(0xc0002e1a, "TODO_c000_2e1a", 0),
+    MVI(0xc0002e1b, "TODO_c000_2e1b", 0),
+    MVI(0xc0002e1c, "TODO_c000_2e1c", 0),
+    MVI(0xc0002e1d, "TODO_c000_2e1d", 0),
+    MVI(0xc0002e1e, "TODO_c000_2e1e", 0),
+    MVI(0xc0002e1f, "TODO_c000_2e1f", 0),
+    MVI(0xc0002e20, "TODO_c000_2e20", 0),
+    MVI(0xc0002e21, "TODO_c000_2e21", 0),
+    MVI(0xc0002e22, "TODO_c000_2e22", 0),
+    MVI(0xc0002e23, "TODO_c000_2e23", 0),
+    MVI(0xc0002e24, "TODO_c000_2e24", 0),
+    MVI(0xc0002e25, "TODO_c000_2e25", 0),
+    MVI(0xc0002e26, "TODO_c000_2e26", 0),
+    MVI(0xc0002e27, "TODO_c000_2e27", 0),
+    MVI(0xc0002e28, "TODO_c000_2e28", 0),
+    MVI(0xc0002e29, "TODO_c000_2e29", 0),
+    MVI(0xc0002e2a, "TODO_c000_2e2a", 0),
+    MVI(0xc0002e2b, "TODO_c000_2e2b", 0),
+    MVI(0xc0002e2c, "TODO_c000_2e2c", 0),
+    MVI(0xc0002e2d, "TODO_c000_2e2d", 0),
+    MVI(0xc0002e2e, "TODO_c000_2e2e", 0),
+    MVI(0xc0002e2f, "TODO_c000_2e2f", 0),
+    MVI(0xc0002e30, "TODO_c000_2e30", 0),
+    MVI(0xc0002e31, "TODO_c000_2e31", 0),
+    MVI(0xc0002e32, "TODO_c000_2e32", 0),
+    MVI(0xc0002e33, "TODO_c000_2e33", 0),
+    MVI(0xc0002e34, "TODO_c000_2e34", 0),
+    MVI(0xc0002e35, "TODO_c000_2e35", 0),
+    MVI(0xc0002e36, "TODO_c000_2e36", 0),
+    MVI(0xc0002e37, "TODO_c000_2e37", 0),
+    MVI(0xc0002e38, "TODO_c000_2e38", 0),
+    MVI(0xc0002e39, "TODO_c000_2e39", 0),
+    MVI(0xc0002e3a, "TODO_c000_2e3a", 0),
+    MVI(0xc0002e3b, "TODO_c000_2e3b", 0),
+    MVI(0xc0002e3c, "TODO_c000_2e3c", 0),
+    MVI(0xc0002e3d, "TODO_c000_2e3d", 0),
+    MVI(0xc0002e3e, "TODO_c000_2e3e", 0),
+    MVI(0xc0002e3f, "TODO_c000_2e3f", 0),
+    MVI(0xc0002e40, "TODO_c000_2e40", 0),
+    MVI(0xc0002e41, "TODO_c000_2e41", 0),
+    MVI(0xc0002e42, "TODO_c000_2e42", 0),
+    MVI(0xc0002e43, "TODO_c000_2e43", 0),
+    MVI(0xc0002e44, "TODO_c000_2e44", 0),
+    MVI(0xc0002e45, "TODO_c000_2e45", 0),
+    MVI(0xc0002e46, "TODO_c000_2e46", 0),
+    MVI(0xc0002e47, "TODO_c000_2e47", 0),
+    MVI(0xc0002e48, "TODO_c000_2e48", 0),
+    MVI(0xc0002e49, "TODO_c000_2e49", 0),
+    MVI(0xc0002e4a, "TODO_c000_2e4a", 0),
+    MVI(0xc0002e4b, "TODO_c000_2e4b", 0),
+    MVI(0xc0002e4c, "TODO_c000_2e4c", 0),
+    MVI(0xc0002e4d, "TODO_c000_2e4d", 0),
+    MVI(0xc0002e4e, "TODO_c000_2e4e", 0),
+    MVI(0xc0002e4f, "TODO_c000_2e4f", 0),
+    MVI(0xc0002e50, "TODO_c000_2e50", 0),
+    MVI(0xc0002e51, "TODO_c000_2e51", 0),
+    MVI(0xc0002e52, "TODO_c000_2e52", 0),
+    MVI(0xc0002e53, "TODO_c000_2e53", 0),
+    MVI(0xc0002e54, "TODO_c000_2e54", 0),
+    MVI(0xc0002e55, "TODO_c000_2e55", 0),
+    MVI(0xc0002e56, "TODO_c000_2e56", 0),
+    MVI(0xc0002e57, "TODO_c000_2e57", 0),
+    MVI(0xc0002e58, "TODO_c000_2e58", 0),
+    MVI(0xc0002e59, "TODO_c000_2e59", 0),
+    MVI(0xc0002e5a, "TODO_c000_2e5a", 0),
+    MVI(0xc0002e5b, "TODO_c000_2e5b", 0),
+    MVI(0xc0002e5c, "TODO_c000_2e5c", 0),
+    MVI(0xc0002e5d, "TODO_c000_2e5d", 0),
+    MVI(0xc0002e5e, "TODO_c000_2e5e", 0),
+    MVI(0xc0002e5f, "TODO_c000_2e5f", 0),
+    MVI(0xc0002e60, "TODO_c000_2e60", 0),
+    MVI(0xc0002e61, "TODO_c000_2e61", 0),
+    MVI(0xc0002e62, "TODO_c000_2e62", 0),
+    MVI(0xc0002e63, "TODO_c000_2e63", 0),
+    MVI(0xc0002e64, "TODO_c000_2e64", 0),
+    MVI(0xc0002e65, "TODO_c000_2e65", 0),
+    MVI(0xc0002e66, "TODO_c000_2e66", 0),
+    MVI(0xc0002e67, "TODO_c000_2e67", 0),
+    MVI(0xc0002e68, "TODO_c000_2e68", 0),
+    MVI(0xc0002e69, "TODO_c000_2e69", 0),
+    MVI(0xc0002e6a, "TODO_c000_2e6a", 0),
+    MVI(0xc0002e6b, "TODO_c000_2e6b", 0),
+    MVI(0xc0002e6c, "TODO_c000_2e6c", 0),
+    MVI(0xc0002e6d, "TODO_c000_2e6d", 0),
+    MVI(0xc0002e6e, "TODO_c000_2e6e", 0),
+    MVI(0xc0002e6f, "TODO_c000_2e6f", 0),
+    MVI(0xc0002e70, "TODO_c000_2e70", 0),
+    MVI(0xc0002e71, "TODO_c000_2e71", 0),
+    MVI(0xc0002e72, "TODO_c000_2e72", 0),
+    MVI(0xc0002e73, "TODO_c000_2e73", 0),
+    MVI(0xc0002e74, "TODO_c000_2e74", 0),
+    MVI(0xc0002e75, "TODO_c000_2e75", 0),
+    MVI(0xc0002e76, "TODO_c000_2e76", 0),
+    MVI(0xc0002e77, "TODO_c000_2e77", 0),
+    MVI(0xc0002e78, "TODO_c000_2e78", 0),
+    MVI(0xc0002e79, "TODO_c000_2e79", 0),
+    MVI(0xc0002e7a, "TODO_c000_2e7a", 0),
+    MVI(0xc0002e7b, "TODO_c000_2e7b", 0),
+    MVI(0xc0002e7c, "TODO_c000_2e7c", 0),
+    MVI(0xc0002e7d, "TODO_c000_2e7d", 0),
+    MVI(0xc0002e7e, "TODO_c000_2e7e", 0),
+    MVI(0xc0002e7f, "TODO_c000_2e7f", 0),
+    MVI(0xc0002e80, "TODO_c000_2e80", 0),
+    MVI(0xc0002e81, "TODO_c000_2e81", 0),
+    MVI(0xc0002e82, "TODO_c000_2e82", 0),
+    MVI(0xc0002e83, "TODO_c000_2e83", 0),
+    MVI(0xc0002e84, "TODO_c000_2e84", 0),
+    MVI(0xc0002e85, "TODO_c000_2e85", 0),
+    MVI(0xc0002e86, "TODO_c000_2e86", 0),
+    MVI(0xc0002e87, "TODO_c000_2e87", 0),
+    MVI(0xc0002e88, "TODO_c000_2e88", 0),
+    MVI(0xc0002e89, "TODO_c000_2e89", 0),
+    MVI(0xc0002e8a, "TODO_c000_2e8a", 0),
+    MVI(0xc0002e8b, "TODO_c000_2e8b", 0),
+    MVI(0xc0002e8c, "TODO_c000_2e8c", 0),
+    MVI(0xc0002e8d, "TODO_c000_2e8d", 0),
+    MVI(0xc0002e8e, "TODO_c000_2e8e", 0),
+    MVI(0xc0002e8f, "TODO_c000_2e8f", 0),
+    MVI(0xc0002e90, "TODO_c000_2e90", 0),
+    MVI(0xc0002e91, "TODO_c000_2e91", 0),
+    MVI(0xc0002e92, "TODO_c000_2e92", 0),
+    MVI(0xc0002e93, "TODO_c000_2e93", 0),
+    MVI(0xc0002e94, "TODO_c000_2e94", 0),
+    MVI(0xc0002e95, "TODO_c000_2e95", 0),
+    MVI(0xc0002e96, "TODO_c000_2e96", 0),
+    MVI(0xc0002e97, "TODO_c000_2e97", 0),
+    MVI(0xc0002e98, "TODO_c000_2e98", 0),
+    MVI(0xc0002e99, "TODO_c000_2e99", 0),
+    MVI(0xc0002e9a, "TODO_c000_2e9a", 0),
+    MVI(0xc0002e9b, "TODO_c000_2e9b", 0),
+    MVI(0xc0002e9c, "TODO_c000_2e9c", 0),
+    MVI(0xc0002e9d, "TODO_c000_2e9d", 0),
+    MVI(0xc0002e9e, "TODO_c000_2e9e", 0),
+    MVI(0xc0002e9f, "TODO_c000_2e9f", 0),
+    MVI(0xc0002ea0, "TODO_c000_2ea0", 0),
+    MVI(0xc0002ea1, "TODO_c000_2ea1", 0),
+    MVI(0xc0002ea2, "TODO_c000_2ea2", 0),
+    MVI(0xc0002ea3, "TODO_c000_2ea3", 0),
+    MVI(0xc0002ea4, "TODO_c000_2ea4", 0),
+    MVI(0xc0002ea5, "TODO_c000_2ea5", 0),
+    MVI(0xc0002ea6, "TODO_c000_2ea6", 0),
+    MVI(0xc0002ea7, "TODO_c000_2ea7", 0),
+    MVI(0xc0002ea8, "TODO_c000_2ea8", 0),
+    MVI(0xc0002ea9, "TODO_c000_2ea9", 0),
+    MVI(0xc0002eaa, "TODO_c000_2eaa", 0),
+    MVI(0xc0002eab, "TODO_c000_2eab", 0),
+    MVI(0xc0002eac, "TODO_c000_2eac", 0),
+    MVI(0xc0002ead, "TODO_c000_2ead", 0),
+    MVI(0xc0002eae, "TODO_c000_2eae", 0),
+    MVI(0xc0002eaf, "TODO_c000_2eaf", 0),
+    MVI(0xc0002eb0, "TODO_c000_2eb0", 0),
+    MVI(0xc0002eb1, "TODO_c000_2eb1", 0),
+    MVI(0xc0002eb2, "TODO_c000_2eb2", 0),
+    MVI(0xc0002eb3, "TODO_c000_2eb3", 0),
+    MVI(0xc0002eb4, "TODO_c000_2eb4", 0),
+    MVI(0xc0002eb5, "TODO_c000_2eb5", 0),
+    MVI(0xc0002eb6, "TODO_c000_2eb6", 0),
+    MVI(0xc0002eb7, "TODO_c000_2eb7", 0),
+    MVI(0xc0002eb8, "TODO_c000_2eb8", 0),
+    MVI(0xc0002eb9, "TODO_c000_2eb9", 0),
+    MVI(0xc0002eba, "TODO_c000_2eba", 0),
+    MVI(0xc0002ebb, "TODO_c000_2ebb", 0),
+    MVI(0xc0002ebc, "TODO_c000_2ebc", 0),
+    MVI(0xc0002ebd, "TODO_c000_2ebd", 0),
+    MVI(0xc0002ebe, "TODO_c000_2ebe", 0),
+    MVI(0xc0002ebf, "TODO_c000_2ebf", 0),
+    MVI(0xc0002ec0, "TODO_c000_2ec0", 0),
+    MVI(0xc0002ec1, "TODO_c000_2ec1", 0),
+    MVI(0xc0002ec2, "TODO_c000_2ec2", 0),
+    MVI(0xc0002ec3, "TODO_c000_2ec3", 0),
+    MVI(0xc0002ec4, "TODO_c000_2ec4", 0),
+    MVI(0xc0002ec5, "TODO_c000_2ec5", 0),
+    MVI(0xc0002ec6, "TODO_c000_2ec6", 0),
+    MVI(0xc0002ec7, "TODO_c000_2ec7", 0),
+    MVI(0xc0002ec8, "TODO_c000_2ec8", 0),
+    MVI(0xc0002ec9, "TODO_c000_2ec9", 0),
+    MVI(0xc0002eca, "TODO_c000_2eca", 0),
+    MVI(0xc0002ecb, "TODO_c000_2ecb", 0),
+    MVI(0xc0002ecc, "TODO_c000_2ecc", 0),
+    MVI(0xc0002ecd, "TODO_c000_2ecd", 0),
+    MVI(0xc0002ece, "TODO_c000_2ece", 0),
+    MVI(0xc0002ecf, "TODO_c000_2ecf", 0),
+    MVI(0xc0002ed0, "TODO_c000_2ed0", 0),
+    MVI(0xc0002ed1, "TODO_c000_2ed1", 0),
+    MVI(0xc0002ed2, "TODO_c000_2ed2", 0),
+    MVI(0xc0002ed3, "TODO_c000_2ed3", 0),
+    MVI(0xc0002ed4, "TODO_c000_2ed4", 0),
+    MVI(0xc0002ed5, "TODO_c000_2ed5", 0),
+    MVI(0xc0002ed6, "TODO_c000_2ed6", 0),
+    MVI(0xc0002ed7, "TODO_c000_2ed7", 0),
+    MVI(0xc0002ed8, "TODO_c000_2ed8", 0),
+    MVI(0xc0002ed9, "TODO_c000_2ed9", 0),
+    MVI(0xc0002eda, "TODO_c000_2eda", 0),
+    MVI(0xc0002edb, "TODO_c000_2edb", 0),
+    MVI(0xc0002edc, "TODO_c000_2edc", 0),
+    MVI(0xc0002edd, "TODO_c000_2edd", 0),
+    MVI(0xc0002ede, "TODO_c000_2ede", 0),
+    MVI(0xc0002edf, "TODO_c000_2edf", 0),
+    MVI(0xc0002ee0, "TODO_c000_2ee0", 0),
+    MVI(0xc0002ee1, "TODO_c000_2ee1", 0),
+    MVI(0xc0002ee2, "TODO_c000_2ee2", 0),
+    MVI(0xc0002ee3, "TODO_c000_2ee3", 0),
+    MVI(0xc0002ee4, "TODO_c000_2ee4", 0),
+    MVI(0xc0002ee5, "TODO_c000_2ee5", 0),
+    MVI(0xc0002ee6, "TODO_c000_2ee6", 0),
+    MVI(0xc0002ee7, "TODO_c000_2ee7", 0),
+    MVI(0xc0002ee8, "TODO_c000_2ee8", 0),
+    MVI(0xc0002ee9, "TODO_c000_2ee9", 0),
+    MVI(0xc0002eea, "TODO_c000_2eea", 0),
+    MVI(0xc0002eeb, "TODO_c000_2eeb", 0),
+    MVI(0xc0002eec, "TODO_c000_2eec", 0),
+    MVI(0xc0002eed, "TODO_c000_2eed", 0),
+    MVI(0xc0002eee, "TODO_c000_2eee", 0),
+    MVI(0xc0002eef, "TODO_c000_2eef", 0),
+    MVI(0xc0002ef0, "TODO_c000_2ef0", 0),
+    MVI(0xc0002ef1, "TODO_c000_2ef1", 0),
+    MVI(0xc0002ef2, "TODO_c000_2ef2", 0),
+    MVI(0xc0002ef3, "TODO_c000_2ef3", 0),
+    MVI(0xc0002ef4, "TODO_c000_2ef4", 0),
+    MVI(0xc0002ef5, "TODO_c000_2ef5", 0),
+    MVI(0xc0002ef6, "TODO_c000_2ef6", 0),
+    MVI(0xc0002ef7, "TODO_c000_2ef7", 0),
+    MVI(0xc0002ef8, "TODO_c000_2ef8", 0),
+    MVI(0xc0002ef9, "TODO_c000_2ef9", 0),
+    MVI(0xc0002efa, "TODO_c000_2efa", 0),
+    MVI(0xc0002efb, "TODO_c000_2efb", 0),
+    MVI(0xc0002efc, "TODO_c000_2efc", 0),
+    MVI(0xc0002efd, "TODO_c000_2efd", 0),
+    MVI(0xc0002efe, "TODO_c000_2efe", 0),
+    MVI(0xc0002eff, "TODO_c000_2eff", 0),
+    MVI(0xc0002f00, "TODO_c000_2f00", 0),
+    MVI(0xc0002f01, "TODO_c000_2f01", 0),
+    MVI(0xc0002f02, "TODO_c000_2f02", 0),
+    MVI(0xc0002f03, "TODO_c000_2f03", 0),
+    MVI(0xc0002f04, "TODO_c000_2f04", 0),
+    MVI(0xc0002f05, "TODO_c000_2f05", 0),
+    MVI(0xc0002f06, "TODO_c000_2f06", 0),
+    MVI(0xc0002f07, "TODO_c000_2f07", 0),
+    MVI(0xc0002f08, "TODO_c000_2f08", 0),
+    MVI(0xc0002f09, "TODO_c000_2f09", 0),
+    MVI(0xc0002f0a, "TODO_c000_2f0a", 0),
+    MVI(0xc0002f0b, "TODO_c000_2f0b", 0),
+    MVI(0xc0002f0c, "TODO_c000_2f0c", 0),
+    MVI(0xc0002f0d, "TODO_c000_2f0d", 0),
+    MVI(0xc0002f0e, "TODO_c000_2f0e", 0),
+    MVI(0xc0002f0f, "TODO_c000_2f0f", 0),
+    MVI(0xc0002f10, "TODO_c000_2f10", 0),
+    MVI(0xc0002f11, "TODO_c000_2f11", 0),
+    MVI(0xc0002f12, "TODO_c000_2f12", 0),
+    MVI(0xc0002f13, "TODO_c000_2f13", 0),
+    MVI(0xc0002f14, "TODO_c000_2f14", 0),
+    MVI(0xc0002f15, "TODO_c000_2f15", 0),
+    MVI(0xc0002f16, "TODO_c000_2f16", 0),
+    MVI(0xc0002f17, "TODO_c000_2f17", 0),
+    MVI(0xc0002f18, "TODO_c000_2f18", 0),
+    MVI(0xc0002f19, "TODO_c000_2f19", 0),
+    MVI(0xc0002f1a, "TODO_c000_2f1a", 0),
+    MVI(0xc0002f1b, "TODO_c000_2f1b", 0),
+    MVI(0xc0002f1c, "TODO_c000_2f1c", 0),
+    MVI(0xc0002f1d, "TODO_c000_2f1d", 0),
+    MVI(0xc0002f1e, "TODO_c000_2f1e", 0),
+    MVI(0xc0002f1f, "TODO_c000_2f1f", 0),
+    MVI(0xc0002f20, "TODO_c000_2f20", 0),
+    MVI(0xc0002f21, "TODO_c000_2f21", 0),
+    MVI(0xc0002f22, "TODO_c000_2f22", 0),
+    MVI(0xc0002f23, "TODO_c000_2f23", 0),
+    MVI(0xc0002f24, "TODO_c000_2f24", 0),
+    MVI(0xc0002f25, "TODO_c000_2f25", 0),
+    MVI(0xc0002f26, "TODO_c000_2f26", 0),
+    MVI(0xc0002f27, "TODO_c000_2f27", 0),
+    MVI(0xc0002f28, "TODO_c000_2f28", 0),
+    MVI(0xc0002f29, "TODO_c000_2f29", 0),
+    MVI(0xc0002f2a, "TODO_c000_2f2a", 0),
+    MVI(0xc0002f2b, "TODO_c000_2f2b", 0),
+    MVI(0xc0002f2c, "TODO_c000_2f2c", 0),
+    MVI(0xc0002f2d, "TODO_c000_2f2d", 0),
+    MVI(0xc0002f2e, "TODO_c000_2f2e", 0),
+    MVI(0xc0002f2f, "TODO_c000_2f2f", 0),
+    MVI(0xc0002f30, "TODO_c000_2f30", 0),
+    MVI(0xc0002f31, "TODO_c000_2f31", 0),
+    MVI(0xc0002f32, "TODO_c000_2f32", 0),
+    MVI(0xc0002f33, "TODO_c000_2f33", 0),
+    MVI(0xc0002f34, "TODO_c000_2f34", 0),
+    MVI(0xc0002f35, "TODO_c000_2f35", 0),
+    MVI(0xc0002f36, "TODO_c000_2f36", 0),
+    MVI(0xc0002f37, "TODO_c000_2f37", 0),
+    MVI(0xc0002f38, "TODO_c000_2f38", 0),
+    MVI(0xc0002f39, "TODO_c000_2f39", 0),
+    MVI(0xc0002f3a, "TODO_c000_2f3a", 0),
+    MVI(0xc0002f3b, "TODO_c000_2f3b", 0),
+    MVI(0xc0002f3c, "TODO_c000_2f3c", 0),
+    MVI(0xc0002f3d, "TODO_c000_2f3d", 0),
+    MVI(0xc0002f3e, "TODO_c000_2f3e", 0),
+    MVI(0xc0002f3f, "TODO_c000_2f3f", 0),
+    MVI(0xc0002f40, "TODO_c000_2f40", 0),
+    MVI(0xc0002f41, "TODO_c000_2f41", 0),
+    MVI(0xc0002f42, "TODO_c000_2f42", 0),
+    MVI(0xc0002f43, "TODO_c000_2f43", 0),
+    MVI(0xc0002f44, "TODO_c000_2f44", 0),
+    MVI(0xc0002f45, "TODO_c000_2f45", 0),
+    MVI(0xc0002f46, "TODO_c000_2f46", 0),
+    MVI(0xc0002f47, "TODO_c000_2f47", 0),
+    MVI(0xc0002f48, "TODO_c000_2f48", 0),
+    MVI(0xc0002f49, "TODO_c000_2f49", 0),
+    MVI(0xc0002f4a, "TODO_c000_2f4a", 0),
+    MVI(0xc0002f4b, "TODO_c000_2f4b", 0),
+    MVI(0xc0002f4c, "TODO_c000_2f4c", 0),
+    MVI(0xc0002f4d, "TODO_c000_2f4d", 0),
+    MVI(0xc0002f4e, "TODO_c000_2f4e", 0),
+    MVI(0xc0002f4f, "TODO_c000_2f4f", 0),
+    MVI(0xc0002f50, "TODO_c000_2f50", 0),
+    MVI(0xc0002f51, "TODO_c000_2f51", 0),
+    MVI(0xc0002f52, "TODO_c000_2f52", 0),
+    MVI(0xc0002f53, "TODO_c000_2f53", 0),
+    MVI(0xc0002f54, "TODO_c000_2f54", 0),
+    MVI(0xc0002f55, "TODO_c000_2f55", 0),
+    MVI(0xc0002f56, "TODO_c000_2f56", 0),
+    MVI(0xc0002f57, "TODO_c000_2f57", 0),
+    MVI(0xc0002f58, "TODO_c000_2f58", 0),
+    MVI(0xc0002f59, "TODO_c000_2f59", 0),
+    MVI(0xc0002f5a, "TODO_c000_2f5a", 0),
+    MVI(0xc0002f5b, "TODO_c000_2f5b", 0),
+    MVI(0xc0002f5c, "TODO_c000_2f5c", 0),
+    MVI(0xc0002f5d, "TODO_c000_2f5d", 0),
+    MVI(0xc0002f5e, "TODO_c000_2f5e", 0),
+    MVI(0xc0002f5f, "TODO_c000_2f5f", 0),
+    MVI(0xc0002f60, "TODO_c000_2f60", 0),
+    MVI(0xc0002f61, "TODO_c000_2f61", 0),
+    MVI(0xc0002f62, "TODO_c000_2f62", 0),
+    MVI(0xc0002f63, "TODO_c000_2f63", 0),
+    MVI(0xc0002f64, "TODO_c000_2f64", 0),
+    MVI(0xc0002f65, "TODO_c000_2f65", 0),
+    MVI(0xc0002f66, "TODO_c000_2f66", 0),
+    MVI(0xc0002f67, "TODO_c000_2f67", 0),
+    MVI(0xc0002f68, "TODO_c000_2f68", 0),
+    MVI(0xc0002f69, "TODO_c000_2f69", 0),
+    MVI(0xc0002f6a, "TODO_c000_2f6a", 0),
+    MVI(0xc0002f6b, "TODO_c000_2f6b", 0),
+    MVI(0xc0002f6c, "TODO_c000_2f6c", 0),
+    MVI(0xc0002f6d, "TODO_c000_2f6d", 0),
+    MVI(0xc0002f6e, "TODO_c000_2f6e", 0),
+    MVI(0xc0002f6f, "TODO_c000_2f6f", 0),
+    MVI(0xc0002f70, "TODO_c000_2f70", 0),
+    MVI(0xc0002f71, "TODO_c000_2f71", 0),
+    MVI(0xc0002f72, "TODO_c000_2f72", 0),
+    MVI(0xc0002f73, "TODO_c000_2f73", 0),
+    MVI(0xc0002f74, "TODO_c000_2f74", 0),
+    MVI(0xc0002f75, "TODO_c000_2f75", 0),
+    MVI(0xc0002f76, "TODO_c000_2f76", 0),
+    MVI(0xc0002f77, "TODO_c000_2f77", 0),
+    MVI(0xc0002f78, "TODO_c000_2f78", 0),
+    MVI(0xc0002f79, "TODO_c000_2f79", 0),
+    MVI(0xc0002f7a, "TODO_c000_2f7a", 0),
+    MVI(0xc0002f7b, "TODO_c000_2f7b", 0),
+    MVI(0xc0002f7c, "TODO_c000_2f7c", 0),
+    MVI(0xc0002f7d, "TODO_c000_2f7d", 0),
+    MVI(0xc0002f7e, "TODO_c000_2f7e", 0),
+    MVI(0xc0002f7f, "TODO_c000_2f7f", 0),
+    MVI(0xc0002f80, "TODO_c000_2f80", 0),
+    MVI(0xc0002f81, "TODO_c000_2f81", 0),
+    MVI(0xc0002f82, "TODO_c000_2f82", 0),
+    MVI(0xc0002f83, "TODO_c000_2f83", 0),
+    MVI(0xc0002f84, "TODO_c000_2f84", 0),
+    MVI(0xc0002f85, "TODO_c000_2f85", 0),
+    MVI(0xc0002f86, "TODO_c000_2f86", 0),
+    MVI(0xc0002f87, "TODO_c000_2f87", 0),
+    MVI(0xc0002f88, "TODO_c000_2f88", 0),
+    MVI(0xc0002f89, "TODO_c000_2f89", 0),
+    MVI(0xc0002f8a, "TODO_c000_2f8a", 0),
+    MVI(0xc0002f8b, "TODO_c000_2f8b", 0),
+    MVI(0xc0002f8c, "TODO_c000_2f8c", 0),
+    MVI(0xc0002f8d, "TODO_c000_2f8d", 0),
+    MVI(0xc0002f8e, "TODO_c000_2f8e", 0),
+    MVI(0xc0002f8f, "TODO_c000_2f8f", 0),
+    MVI(0xc0002f90, "TODO_c000_2f90", 0),
+    MVI(0xc0002f91, "TODO_c000_2f91", 0),
+    MVI(0xc0002f92, "TODO_c000_2f92", 0),
+    MVI(0xc0002f93, "TODO_c000_2f93", 0),
+    MVI(0xc0002f94, "TODO_c000_2f94", 0),
+    MVI(0xc0002f95, "TODO_c000_2f95", 0),
+    MVI(0xc0002f96, "TODO_c000_2f96", 0),
+    MVI(0xc0002f97, "TODO_c000_2f97", 0),
+    MVI(0xc0002f98, "TODO_c000_2f98", 0),
+    MVI(0xc0002f99, "TODO_c000_2f99", 0),
+    MVI(0xc0002f9a, "TODO_c000_2f9a", 0),
+    MVI(0xc0002f9b, "TODO_c000_2f9b", 0),
+    MVI(0xc0002f9c, "TODO_c000_2f9c", 0),
+    MVI(0xc0002f9d, "TODO_c000_2f9d", 0),
+    MVI(0xc0002f9e, "TODO_c000_2f9e", 0),
+    MVI(0xc0002f9f, "TODO_c000_2f9f", 0),
+    MVI(0xc0002fa0, "TODO_c000_2fa0", 0),
+    MVI(0xc0002fa1, "TODO_c000_2fa1", 0),
+    MVI(0xc0002fa2, "TODO_c000_2fa2", 0),
+    MVI(0xc0002fa3, "TODO_c000_2fa3", 0),
+    MVI(0xc0002fa4, "TODO_c000_2fa4", 0),
+    MVI(0xc0002fa5, "TODO_c000_2fa5", 0),
+    MVI(0xc0002fa6, "TODO_c000_2fa6", 0),
+    MVI(0xc0002fa7, "TODO_c000_2fa7", 0),
+    MVI(0xc0002fa8, "TODO_c000_2fa8", 0),
+    MVI(0xc0002fa9, "TODO_c000_2fa9", 0),
+    MVI(0xc0002faa, "TODO_c000_2faa", 0),
+    MVI(0xc0002fab, "TODO_c000_2fab", 0),
+    MVI(0xc0002fac, "TODO_c000_2fac", 0),
+    MVI(0xc0002fad, "TODO_c000_2fad", 0),
+    MVI(0xc0002fae, "TODO_c000_2fae", 0),
+    MVI(0xc0002faf, "TODO_c000_2faf", 0),
+    MVI(0xc0002fb0, "TODO_c000_2fb0", 0),
+    MVI(0xc0002fb1, "TODO_c000_2fb1", 0),
+    MVI(0xc0002fb2, "TODO_c000_2fb2", 0),
+    MVI(0xc0002fb3, "TODO_c000_2fb3", 0),
+    MVI(0xc0002fb4, "TODO_c000_2fb4", 0),
+    MVI(0xc0002fb5, "TODO_c000_2fb5", 0),
+    MVI(0xc0002fb6, "TODO_c000_2fb6", 0),
+    MVI(0xc0002fb7, "TODO_c000_2fb7", 0),
+    MVI(0xc0002fb8, "TODO_c000_2fb8", 0),
+    MVI(0xc0002fb9, "TODO_c000_2fb9", 0),
+    MVI(0xc0002fba, "TODO_c000_2fba", 0),
+    MVI(0xc0002fbb, "TODO_c000_2fbb", 0),
+    MVI(0xc0002fbc, "TODO_c000_2fbc", 0),
+    MVI(0xc0002fbd, "TODO_c000_2fbd", 0),
+    MVI(0xc0002fbe, "TODO_c000_2fbe", 0),
+    MVI(0xc0002fbf, "TODO_c000_2fbf", 0),
+    MVI(0xc0002fc0, "TODO_c000_2fc0", 0),
+    MVI(0xc0002fc1, "TODO_c000_2fc1", 0),
+    MVI(0xc0002fc2, "TODO_c000_2fc2", 0),
+    MVI(0xc0002fc3, "TODO_c000_2fc3", 0),
+    MVI(0xc0002fc4, "TODO_c000_2fc4", 0),
+    MVI(0xc0002fc5, "TODO_c000_2fc5", 0),
+    MVI(0xc0002fc6, "TODO_c000_2fc6", 0),
+    MVI(0xc0002fc7, "TODO_c000_2fc7", 0),
+    MVI(0xc0002fc8, "TODO_c000_2fc8", 0),
+    MVI(0xc0002fc9, "TODO_c000_2fc9", 0),
+    MVI(0xc0002fca, "TODO_c000_2fca", 0),
+    MVI(0xc0002fcb, "TODO_c000_2fcb", 0),
+    MVI(0xc0002fcc, "TODO_c000_2fcc", 0),
+    MVI(0xc0002fcd, "TODO_c000_2fcd", 0),
+    MVI(0xc0002fce, "TODO_c000_2fce", 0),
+    MVI(0xc0002fcf, "TODO_c000_2fcf", 0),
+    MVI(0xc0002fd0, "TODO_c000_2fd0", 0),
+    MVI(0xc0002fd1, "TODO_c000_2fd1", 0),
+    MVI(0xc0002fd2, "TODO_c000_2fd2", 0),
+    MVI(0xc0002fd3, "TODO_c000_2fd3", 0),
+    MVI(0xc0002fd4, "TODO_c000_2fd4", 0),
+    MVI(0xc0002fd5, "TODO_c000_2fd5", 0),
+    MVI(0xc0002fd6, "TODO_c000_2fd6", 0),
+    MVI(0xc0002fd7, "TODO_c000_2fd7", 0),
+    MVI(0xc0002fd8, "TODO_c000_2fd8", 0),
+    MVI(0xc0002fd9, "TODO_c000_2fd9", 0),
+    MVI(0xc0002fda, "TODO_c000_2fda", 0),
+    MVI(0xc0002fdb, "TODO_c000_2fdb", 0),
+    MVI(0xc0002fdc, "TODO_c000_2fdc", 0),
+    MVI(0xc0002fdd, "TODO_c000_2fdd", 0),
+    MVI(0xc0002fde, "TODO_c000_2fde", 0),
+    MVI(0xc0002fdf, "TODO_c000_2fdf", 0),
+    MVI(0xc0002fe0, "TODO_c000_2fe0", 0),
+    MVI(0xc0002fe1, "TODO_c000_2fe1", 0),
+    MVI(0xc0002fe2, "TODO_c000_2fe2", 0),
+    MVI(0xc0002fe3, "TODO_c000_2fe3", 0),
+    MVI(0xc0002fe4, "TODO_c000_2fe4", 0),
+    MVI(0xc0002fe5, "TODO_c000_2fe5", 0),
+    MVI(0xc0002fe6, "TODO_c000_2fe6", 0),
+    MVI(0xc0002fe7, "TODO_c000_2fe7", 0),
+    MVI(0xc0002fe8, "TODO_c000_2fe8", 0),
+    MVI(0xc0002fe9, "TODO_c000_2fe9", 0),
+    MVI(0xc0002fea, "TODO_c000_2fea", 0),
+    MVI(0xc0002feb, "TODO_c000_2feb", 0),
+    MVI(0xc0002fec, "TODO_c000_2fec", 0),
+    MVI(0xc0002fed, "TODO_c000_2fed", 0),
+    MVI(0xc0002fee, "TODO_c000_2fee", 0),
+    MVI(0xc0002fef, "TODO_c000_2fef", 0),
+    MVI(0xc0002ff0, "TODO_c000_2ff0", 0),
+    MVI(0xc0002ff1, "TODO_c000_2ff1", 0),
+    MVI(0xc0002ff2, "TODO_c000_2ff2", 0),
+    MVI(0xc0002ff3, "TODO_c000_2ff3", 0),
+    MVI(0xc0002ff4, "TODO_c000_2ff4", 0),
+    MVI(0xc0002ff5, "TODO_c000_2ff5", 0),
+    MVI(0xc0002ff6, "TODO_c000_2ff6", 0),
+    MVI(0xc0002ff7, "TODO_c000_2ff7", 0),
+    MVI(0xc0002ff8, "TODO_c000_2ff8", 0),
+    MVI(0xc0002ff9, "TODO_c000_2ff9", 0),
+    MVI(0xc0002ffa, "TODO_c000_2ffa", 0),
+    MVI(0xc0002ffb, "TODO_c000_2ffb", 0),
+    MVI(0xc0002ffc, "TODO_c000_2ffc", 0),
+    MVI(0xc0002ffd, "TODO_c000_2ffd", 0),
+    MVI(0xc0002ffe, "TODO_c000_2ffe", 0),
+    MVI(0xc0002fff, "TODO_c000_2fff", 0),
+    RSN(0xc0010000, 0xc0010003, "AMD_K8_PERF_CTL_n", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x0, UINT64_C(0xfffffcf000200000), 0),
+    MFX(0xc0010004, "AMD_K8_PERF_CTR_0", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x0, UINT64_C(0xffff0000000022a0), 0), /* XXX: The range ended earlier than expected! */
+    MFX(0xc0010005, "AMD_K8_PERF_CTR_1", AmdK8PerfCtrN, AmdK8PerfCtrN, 0, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010006, "AMD_K8_PERF_CTR_2", AmdK8PerfCtrN, AmdK8PerfCtrN, 0, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010007, "AMD_K8_PERF_CTR_3", AmdK8PerfCtrN, AmdK8PerfCtrN, 0, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010010, "AMD_K8_SYS_CFG", AmdK8SysCfg, AmdK8SysCfg, 0x740000, UINT64_C(0xffffffffff00ffff), 0), /* value=0x740000 */
+    MFX(0xc0010015, "AMD_K8_HW_CFG", AmdK8HwCr, AmdK8HwCr, 0x9000011, UINT64_C(0xffffffff89006000), 0), /* value=0x9000011 */
+    MFW(0xc0010016, "AMD_K8_IORR_BASE_0", AmdK8IorrBaseN, AmdK8IorrBaseN, UINT64_C(0xffff000000000fe7)), /* value=0x0 */
+    MFW(0xc0010017, "AMD_K8_IORR_MASK_0", AmdK8IorrMaskN, AmdK8IorrMaskN, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0xc0010018, "AMD_K8_IORR_BASE_1", AmdK8IorrBaseN, AmdK8IorrBaseN, 0x1, UINT64_C(0xffff000000000fe7), 0), /* value=0x0 */
+    MFX(0xc0010019, "AMD_K8_IORR_MASK_1", AmdK8IorrMaskN, AmdK8IorrMaskN, 0x1, UINT64_C(0xffff0000000007ff), 0), /* value=0x0 */
+    MFW(0xc001001a, "AMD_K8_TOP_MEM", AmdK8TopOfMemN, AmdK8TopOfMemN, UINT64_C(0xffff0000007fffff)), /* value=0x80000000 */
+    MFX(0xc001001d, "AMD_K8_TOP_MEM2", AmdK8TopOfMemN, AmdK8TopOfMemN, 0x1, UINT64_C(0xffff0000007fffff), 0), /* value=0x8`80000000 */
+    MFI(0xc001001f, "AMD_K8_NB_CFG1", AmdK8NbCfg1), /* value=0x0 */
+    MFN(0xc0010021, "AMD_K8_UNK_c001_0021", WriteOnly, IgnoreWrite),
+    MFX(0xc0010022, "AMD_K8_MC_XCPT_REDIR", AmdK8McXcptRedir, AmdK8McXcptRedir, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    RFN(0xc0010030, 0xc0010035, "AMD_K8_CPU_NAME_n", AmdK8CpuNameN, AmdK8CpuNameN),
+    MFX(0xc001003e, "AMD_K8_HTC", AmdK8HwThermalCtrl, AmdK8HwThermalCtrl, 0, UINT64_MAX, 0), /* value=0x0 */
+    RFN(0xc0010050, 0xc0010053, "AMD_K8_SMI_ON_IO_TRAP_n", AmdK8SmiOnIoTrapN, AmdK8SmiOnIoTrapN),
+    MFX(0xc0010054, "AMD_K8_SMI_ON_IO_TRAP_CTL_STS", AmdK8SmiOnIoTrapCtlSts, AmdK8SmiOnIoTrapCtlSts, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFI(0xc0010055, "AMD_K8_INT_PENDING_MSG", AmdK8IntPendingMessage), /* value=0x0 */
+    MFX(0xc0010056, "AMD_K8_SMI_TRIGGER_IO_CYCLE", AmdK8SmiTriggerIoCycle, AmdK8SmiTriggerIoCycle, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0010058, "AMD_10H_MMIO_CFG_BASE_ADDR", AmdFam10hMmioCfgBaseAddr, AmdFam10hMmioCfgBaseAddr, 0, UINT64_C(0xffff0000000fffc0), 0), /* value=0xe0000021 */
+    MVO(0xc0010060, "AMD_K8_BIST_RESULT", 0),
+    MFX(0xc0010061, "AMD_10H_P_ST_CUR_LIM", AmdFam10hPStateCurLimit, ReadOnly, 0x20, 0, 0), /* value=0x20 */
+    MFX(0xc0010062, "AMD_10H_P_ST_CTL", AmdFam10hPStateControl, AmdFam10hPStateControl, 0x2, 0, UINT64_C(0xfffffffffffffff8)), /* value=0x2 */
+    MFX(0xc0010063, "AMD_10H_P_ST_STS", AmdFam10hPStateStatus, ReadOnly, 0x2, 0, 0), /* value=0x2 */
+    MFX(0xc0010064, "AMD_10H_P_ST_0", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x8000000000178a64), 0, 0), /* value=0x80000000`00178a64 */
+    MFX(0xc0010065, "AMD_10H_P_ST_1", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x800000000018cc60), 0, 0), /* value=0x80000000`0018cc60 */
+    MFX(0xc0010066, "AMD_10H_P_ST_2", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x80000000001a5060), 0, 0), /* value=0x80000000`001a5060 */
+    MFX(0xc0010067, "AMD_10H_P_ST_3", AmdFam10hPStateN, AmdFam10hPStateN, 0, 0, 0), /* value=0x0 */
+    MFX(0xc0010068, "AMD_10H_P_ST_4", AmdFam10hPStateN, AmdFam10hPStateN, 0, 0, 0), /* value=0x0 */
+    MFX(0xc0010069, "AMD_10H_P_ST_5", AmdFam10hPStateN, AmdFam10hPStateN, 0, 0, 0), /* value=0x0 */
+    MFX(0xc001006a, "AMD_10H_P_ST_6", AmdFam10hPStateN, AmdFam10hPStateN, 0, 0, 0), /* value=0x0 */
+    MFX(0xc001006b, "AMD_10H_P_ST_7", AmdFam10hPStateN, AmdFam10hPStateN, 0, 0, 0), /* value=0x0 */
+    MFX(0xc0010073, "AMD_10H_C_ST_IO_BASE_ADDR", AmdFam10hCStateIoBaseAddr, AmdFam10hCStateIoBaseAddr, 0, UINT64_C(0xffffffffffff0000), 0), /* value=0x813 */
+    MFX(0xc0010074, "AMD_10H_CPU_WD_TMR_CFG", AmdFam10hCpuWatchdogTimer, AmdFam10hCpuWatchdogTimer, 0, UINT64_C(0xffffffffffffff80), 0), /* value=0x1 */
+    MFX(0xc0010111, "AMD_K8_SMM_BASE", AmdK8SmmBase, AmdK8SmmBase, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x7fbcd000 */
+    MFX(0xc0010112, "AMD_K8_SMM_ADDR", AmdK8SmmAddr, AmdK8SmmAddr, 0, UINT64_C(0xffff00000001ffff), 0), /* value=0x7c000000 */
+    MFX(0xc0010113, "AMD_K8_SMM_MASK", AmdK8SmmMask, AmdK8SmmMask, 0, UINT64_C(0xffff0000000188c0), 0), /* value=0xffff`fc006003 */
+    MFX(0xc0010114, "AMD_K8_VM_CR", AmdK8VmCr, AmdK8VmCr, 0, UINT64_C(0xffffffff00000005), UINT32_C(0xffffffe0)), /* value=0x0 */
+    MFX(0xc0010115, "AMD_K8_IGNNE", AmdK8IgnNe, AmdK8IgnNe, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xfffffffe)), /* value=0x0 */
+    MFX(0xc0010117, "AMD_K8_VM_HSAVE_PA", AmdK8VmHSavePa, AmdK8VmHSavePa, 0, 0, UINT64_C(0xffff000000000fff)), /* value=0x0 */
+    MFN(0xc0010118, "AMD_10H_VM_LOCK_KEY", AmdFam10hVmLockKey, AmdFam10hVmLockKey), /* value=0x0 */
+    MFN(0xc0010119, "AMD_10H_SSM_LOCK_KEY", AmdFam10hSmmLockKey, AmdFam10hSmmLockKey), /* value=0x0 */
+    MFX(0xc001011a, "AMD_10H_LOCAL_SMI_STS", AmdFam10hLocalSmiStatus, AmdFam10hLocalSmiStatus, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFN(0xc001011b, "AMD_K8_UNK_c001_011b", WriteOnly, IgnoreWrite),
+    MVI(0xc0010120, "TODO_c001_0120", 0),
+    MVI(0xc0010121, "TODO_c001_0121", 0),
+    MVI(0xc0010122, "TODO_c001_0122", 0),
+    MFN(0xc0010140, "AMD_10H_OSVW_ID_LEN", AmdFam10hOsVisWrkIdLength, AmdFam10hOsVisWrkIdLength), /* value=0x0 */
+    MFN(0xc0010141, "AMD_10H_OSVW_STS", AmdFam10hOsVisWrkStatus, AmdFam10hOsVisWrkStatus), /* value=0x0 */
+    MVX(0xc0010188, "TODO_c001_0188", 0, UINT64_C(0xfffffcf000200000), 0),
+    MVX(0xc0010189, "TODO_c001_0189", 0, UINT64_C(0xffff000000000000), 0),
+    MVX(0xc001018a, "TODO_c001_018a", 0, UINT64_C(0xfffffcf000200000), 0),
+    MVX(0xc001018b, "TODO_c001_018b", 0, UINT64_C(0xffff000000000000), 0),
+    MFX(0xc0010200, "AMD_K8_PERF_CTL_0", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x0, UINT64_C(0xfffffcf000200000), 0), /* value=0x530076 */
+    MFX(0xc0010201, "AMD_K8_PERF_CTR_0", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x0, UINT64_C(0xffff000000002191), 0), /* value=0xffff`9124f0a3 */
+    MFX(0xc0010202, "AMD_K8_PERF_CTL_1", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x1, UINT64_C(0xfffffcf000200000), 0), /* value=0x0 */
+    MFX(0xc0010203, "AMD_K8_PERF_CTR_1", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x1, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010204, "AMD_K8_PERF_CTL_2", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x2, UINT64_C(0xfffffcf000200000), 0), /* value=0x0 */
+    MFX(0xc0010205, "AMD_K8_PERF_CTR_2", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x2, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010206, "AMD_K8_PERF_CTL_3", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x3, UINT64_C(0xfffffcf000200000), 0), /* value=0x0 */
+    MFX(0xc0010207, "AMD_K8_PERF_CTR_3", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x3, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010208, "AMD_K8_PERF_CTL_4", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x4, UINT64_C(0xfffffcf000200000), 0), /* value=0x0 */
+    MFX(0xc0010209, "AMD_K8_PERF_CTR_4", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x4, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc001020a, "AMD_K8_PERF_CTL_5", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x5, UINT64_C(0xfffffcf000200000), 0), /* value=0x0 */
+    MFX(0xc001020b, "AMD_K8_PERF_CTR_5", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x5, UINT64_C(0xffff000000000000), 0), /* value=0xffff */
+    MFX(0xc0010230, "AMD_16H_L2I_PERF_CTL_0", AmdFam16hL2IPerfCtlN, AmdFam16hL2IPerfCtlN, 0x0, UINT64_C(0xf0ffffffbf0000), 0), /* value=0x0 */
+    MFX(0xc0010231, "AMD_16H_L2I_PERF_CTR_0", AmdFam16hL2IPerfCtrN, AmdFam16hL2IPerfCtrN, 0x0, UINT64_C(0xfffe000000000000), 0), /* value=0x0 */
+    MFX(0xc0010232, "AMD_16H_L2I_PERF_CTL_1", AmdFam16hL2IPerfCtlN, AmdFam16hL2IPerfCtlN, 0x1, UINT64_C(0xf0ffffffbf0000), 0), /* value=0x0 */
+    MFX(0xc0010233, "AMD_16H_L2I_PERF_CTR_1", AmdFam16hL2IPerfCtrN, AmdFam16hL2IPerfCtrN, 0x1, UINT64_C(0xfffe000000000000), 0), /* value=0x0 */
+    MFX(0xc0010234, "AMD_16H_L2I_PERF_CTL_2", AmdFam16hL2IPerfCtlN, AmdFam16hL2IPerfCtlN, 0x2, UINT64_C(0xf0ffffffbf0000), 0), /* value=0x0 */
+    MFX(0xc0010235, "AMD_16H_L2I_PERF_CTR_2", AmdFam16hL2IPerfCtrN, AmdFam16hL2IPerfCtrN, 0x2, UINT64_C(0xfffe000000000000), 0), /* value=0x0 */
+    MFX(0xc0010236, "AMD_16H_L2I_PERF_CTL_3", AmdFam16hL2IPerfCtlN, AmdFam16hL2IPerfCtlN, 0x3, UINT64_C(0xf0ffffffbf0000), 0), /* value=0x0 */
+    MFX(0xc0010237, "AMD_16H_L2I_PERF_CTR_3", AmdFam16hL2IPerfCtrN, AmdFam16hL2IPerfCtrN, 0x3, UINT64_C(0xfffe000000000000), 0), /* value=0x0 */
+    MVX(0xc0010238, "TODO_c001_0238", 0, UINT64_C(0xf0ffffffbf0000), 0),
+    MVX(0xc0010239, "TODO_c001_0239", 0, UINT64_C(0xfffe000000000000), 0),
+    MVX(0xc001023a, "TODO_c001_023a", 0, UINT64_C(0xf0ffffffbf0000), 0),
+    MVX(0xc001023b, "TODO_c001_023b", 0, UINT64_C(0xfffe000000000000), 0),
+    MFX(0xc0010240, "AMD_15H_NB_PERF_CTL_0", AmdFam15hNorthbridgePerfCtlN, AmdFam15hNorthbridgePerfCtlN, 0x0, UINT64_C(0x1ffffff0ff970000), 0), /* value=0x0 */
+    MFX(0xc0010241, "AMD_15H_NB_PERF_CTR_0", AmdFam15hNorthbridgePerfCtrN, AmdFam15hNorthbridgePerfCtrN, 0x0, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010242, "AMD_15H_NB_PERF_CTL_1", AmdFam15hNorthbridgePerfCtlN, AmdFam15hNorthbridgePerfCtlN, 0x1, UINT64_C(0x1ffffff0ff970000), 0), /* value=0x0 */
+    MFX(0xc0010243, "AMD_15H_NB_PERF_CTR_1", AmdFam15hNorthbridgePerfCtrN, AmdFam15hNorthbridgePerfCtrN, 0x1, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010244, "AMD_15H_NB_PERF_CTL_2", AmdFam15hNorthbridgePerfCtlN, AmdFam15hNorthbridgePerfCtlN, 0x2, UINT64_C(0x1ffffff0ff970000), 0), /* value=0x0 */
+    MFX(0xc0010245, "AMD_15H_NB_PERF_CTR_2", AmdFam15hNorthbridgePerfCtrN, AmdFam15hNorthbridgePerfCtrN, 0x2, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0010246, "AMD_15H_NB_PERF_CTL_3", AmdFam15hNorthbridgePerfCtlN, AmdFam15hNorthbridgePerfCtlN, 0x3, UINT64_C(0x1ffffff0ff970000), 0), /* value=0x0 */
+    MFX(0xc0010247, "AMD_15H_NB_PERF_CTR_3", AmdFam15hNorthbridgePerfCtrN, AmdFam15hNorthbridgePerfCtrN, 0x3, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MVX(0xc0010290, "TODO_c001_0290", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0010292, "TODO_c001_0292", 0xf0052, UINT32_MAX, 0),
+    MVI(0xc0010293, "TODO_c001_0293", 0xa01060),
+    MVX(0xc0010294, "TODO_c001_0294", UINT64_C(0x1001f47f000f0912), UINT64_C(0xc0000000ffe00000), 0),
+    MVX(0xc0010296, "TODO_c001_0296", 0x484848, UINT64_C(0xffffffffff808080), 0),
+    MVI(0xc0010297, "TODO_c001_0297", UINT64_C(0x380000fc000)),
+    MVI(0xc0010299, "TODO_c001_0299", 0xa1003),
+    MVI(0xc001029a, "TODO_c001_029a", 0x5382a5),
+    MVI(0xc001029b, "TODO_c001_029b", 0x7a66d6c3),
+    MVX(0xc0010400, "TODO_c001_0400", 0x600, UINT64_C(0xffffffffffe00000), 0),
+    MVX(0xc0010401, "TODO_c001_0401", 0x2000, UINT64_C(0xffffffffffffc000), 0),
+    MVX(0xc0010402, "TODO_c001_0402", 0x8, UINT64_C(0xfffffffffffffff0), 0),
+    MVX(0xc0010403, "TODO_c001_0403", 0, UINT64_C(0xfffffffffffffe00), 0),
+    MVI(0xc0010404, "TODO_c001_0404", 0),
+    MVX(0xc0010405, "TODO_c001_0405", 0, UINT64_C(0xfffffffffffff800), 0),
+    MVX(0xc0010406, "TODO_c001_0406", 0x40, UINT64_C(0xffffffffffffff80), 0),
+    MVX(0xc0010407, "TODO_c001_0407", 0x80, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc0010408, "TODO_c001_0408", 0x80, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc0010409, "TODO_c001_0409", 0x80, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc001040a, "TODO_c001_040a", 0x80, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc001040b, "TODO_c001_040b", 0x80, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc001040c, "TODO_c001_040c", 0x80, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc001040d, "TODO_c001_040d", 0x80, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc001040e, "TODO_c001_040e", 0x80, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0xc001040f, "TODO_c001_040f", 0, UINT64_C(0xffffffffffffffc0), 0),
+    MVX(0xc0010410, "TODO_c001_0410", 0, UINT64_C(0xffffffffffffffc0), 0),
+    MVX(0xc0010411, "TODO_c001_0411", 0, UINT64_C(0xfffffffffffffffe), 0),
+    MVX(0xc0010412, "TODO_c001_0412", 0, UINT64_C(0xfffffffffffffffe), 0),
+    MVX(0xc0010413, "TODO_c001_0413", 0, UINT64_C(0xfffffffffffffffe), 0),
+    MVX(0xc0010414, "TODO_c001_0414", 0, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0xc0010415, "TODO_c001_0415", 0, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0xc0010416, "TODO_c001_0416", 0, UINT64_C(0xfffffffffffffff0), 0),
+    MVI(0xc0010417, "TODO_c001_0417", 0),
+    MVI(0xc0010418, "TODO_c001_0418", 0),
+    MVI(0xc0010419, "TODO_c001_0419", 0),
+    MVI(0xc001041a, "TODO_c001_041a", 0),
+    MVI(0xc001041b, "TODO_c001_041b", 0),
+    MVI(0xc001041c, "TODO_c001_041c", 0),
+    MVI(0xc001041d, "TODO_c001_041d", 0),
+    MVI(0xc001041e, "TODO_c001_041e", 0),
+    MVI(0xc001041f, "TODO_c001_041f", 0),
+    MVI(0xc0010420, "TODO_c001_0420", 0),
+    MVI(0xc0010421, "TODO_c001_0421", 0),
+    MVI(0xc0010422, "TODO_c001_0422", 0),
+    MVI(0xc0010423, "TODO_c001_0423", 0),
+    MVI(0xc0010424, "TODO_c001_0424", 0),
+    MVI(0xc0010425, "TODO_c001_0425", 0),
+    MVI(0xc0010426, "TODO_c001_0426", 0),
+    MVI(0xc0010427, "TODO_c001_0427", 0),
+    MVI(0xc0010428, "TODO_c001_0428", 0),
+    MVI(0xc0010429, "TODO_c001_0429", 0),
+    MVI(0xc001042a, "TODO_c001_042a", 0),
+    MVI(0xc001042b, "TODO_c001_042b", 0),
+    MVI(0xc001042c, "TODO_c001_042c", 0),
+    MVI(0xc001042d, "TODO_c001_042d", 0),
+    MVI(0xc001042e, "TODO_c001_042e", 0),
+    MVI(0xc001042f, "TODO_c001_042f", 0),
+    MVI(0xc0010430, "TODO_c001_0430", 0),
+    MVI(0xc0010431, "TODO_c001_0431", 0),
+    MVI(0xc0010432, "TODO_c001_0432", 0),
+    MVI(0xc0010433, "TODO_c001_0433", 0),
+    MVI(0xc0010434, "TODO_c001_0434", 0),
+    MVI(0xc0010435, "TODO_c001_0435", 0),
+    MVI(0xc0010436, "TODO_c001_0436", 0),
+    MVI(0xc0010437, "TODO_c001_0437", 0),
+    MVI(0xc0010438, "TODO_c001_0438", 0),
+    MVI(0xc0010439, "TODO_c001_0439", 0),
+    MVI(0xc001043a, "TODO_c001_043a", 0),
+    MVI(0xc001043b, "TODO_c001_043b", 0),
+    MVI(0xc001043c, "TODO_c001_043c", 0),
+    MVI(0xc001043d, "TODO_c001_043d", 0),
+    MVI(0xc001043e, "TODO_c001_043e", 0),
+    MVI(0xc001043f, "TODO_c001_043f", 0),
+    MVI(0xc0010440, "TODO_c001_0440", 0),
+    MVI(0xc0010441, "TODO_c001_0441", 0),
+    MVI(0xc0010442, "TODO_c001_0442", 0),
+    MVI(0xc0010443, "TODO_c001_0443", 0),
+    MVI(0xc0010444, "TODO_c001_0444", 0),
+    MVI(0xc0010445, "TODO_c001_0445", 0),
+    MVI(0xc0010446, "TODO_c001_0446", 0),
+    MVI(0xc0010447, "TODO_c001_0447", 0),
+    MVI(0xc0010448, "TODO_c001_0448", 0),
+    MVI(0xc0010449, "TODO_c001_0449", 0),
+    MVI(0xc001044a, "TODO_c001_044a", 0),
+    MVI(0xc001044b, "TODO_c001_044b", 0),
+    MVI(0xc001044c, "TODO_c001_044c", 0),
+    MVI(0xc001044d, "TODO_c001_044d", 0),
+    MVI(0xc001044e, "TODO_c001_044e", 0),
+    MVI(0xc001044f, "TODO_c001_044f", 0),
+    MVI(0xc0010450, "TODO_c001_0450", 0),
+    MVI(0xc0010451, "TODO_c001_0451", 0),
+    MVI(0xc0010452, "TODO_c001_0452", 0),
+    MVI(0xc0010453, "TODO_c001_0453", 0),
+    MVI(0xc0010454, "TODO_c001_0454", 0),
+    MVI(0xc0010455, "TODO_c001_0455", 0),
+    MVI(0xc0010456, "TODO_c001_0456", 0),
+    MVI(0xc0010457, "TODO_c001_0457", 0),
+    MVI(0xc0010458, "TODO_c001_0458", 0),
+    MVI(0xc0010459, "TODO_c001_0459", 0),
+    MVI(0xc001045a, "TODO_c001_045a", 0),
+    MVI(0xc001045b, "TODO_c001_045b", 0),
+    MVI(0xc001045c, "TODO_c001_045c", 0),
+    MVI(0xc001045d, "TODO_c001_045d", 0),
+    MVI(0xc001045e, "TODO_c001_045e", 0),
+    MVI(0xc001045f, "TODO_c001_045f", 0),
+    MVI(0xc0010460, "TODO_c001_0460", 0),
+    MVI(0xc0010461, "TODO_c001_0461", 0),
+    MVI(0xc0010462, "TODO_c001_0462", 0),
+    MVI(0xc0010463, "TODO_c001_0463", 0),
+    MVI(0xc0010464, "TODO_c001_0464", 0),
+    MVI(0xc0010465, "TODO_c001_0465", 0),
+    MVI(0xc0010466, "TODO_c001_0466", 0),
+    MVI(0xc0010467, "TODO_c001_0467", 0),
+    MVI(0xc0010468, "TODO_c001_0468", 0),
+    MVI(0xc0010469, "TODO_c001_0469", 0),
+    MVI(0xc001046a, "TODO_c001_046a", 0),
+    MVI(0xc001046b, "TODO_c001_046b", 0),
+    MVI(0xc001046c, "TODO_c001_046c", 0),
+    MVI(0xc001046d, "TODO_c001_046d", 0),
+    MVI(0xc001046e, "TODO_c001_046e", 0),
+    MVI(0xc001046f, "TODO_c001_046f", 0),
+    MVI(0xc0010470, "TODO_c001_0470", 0),
+    MVI(0xc0010471, "TODO_c001_0471", 0),
+    MVI(0xc0010472, "TODO_c001_0472", 0),
+    MVI(0xc0010473, "TODO_c001_0473", 0),
+    MVI(0xc0010474, "TODO_c001_0474", 0),
+    MVI(0xc0010475, "TODO_c001_0475", 0),
+    MVI(0xc0010476, "TODO_c001_0476", 0),
+    MVI(0xc0010477, "TODO_c001_0477", 0),
+    MVI(0xc0010478, "TODO_c001_0478", 0),
+    MVI(0xc0010479, "TODO_c001_0479", 0),
+    MVI(0xc001047a, "TODO_c001_047a", 0),
+    MVI(0xc001047b, "TODO_c001_047b", 0),
+    MVI(0xc001047c, "TODO_c001_047c", 0),
+    MVI(0xc001047d, "TODO_c001_047d", 0),
+    MVI(0xc001047e, "TODO_c001_047e", 0),
+    MVI(0xc001047f, "TODO_c001_047f", 0),
+    MVI(0xc0010480, "TODO_c001_0480", 0),
+    MVI(0xc0010481, "TODO_c001_0481", 0),
+    MVI(0xc0010482, "TODO_c001_0482", 0),
+    MVI(0xc0010483, "TODO_c001_0483", 0),
+    MVI(0xc0010484, "TODO_c001_0484", 0),
+    MVI(0xc0010485, "TODO_c001_0485", 0),
+    MVI(0xc0010486, "TODO_c001_0486", 0),
+    MVI(0xc0010487, "TODO_c001_0487", 0),
+    MVI(0xc0010488, "TODO_c001_0488", 0),
+    MVI(0xc0010489, "TODO_c001_0489", 0),
+    MVI(0xc001048a, "TODO_c001_048a", 0),
+    MVI(0xc001048b, "TODO_c001_048b", 0),
+    MVI(0xc001048c, "TODO_c001_048c", 0),
+    MVI(0xc001048d, "TODO_c001_048d", 0),
+    MVI(0xc001048e, "TODO_c001_048e", 0),
+    MVI(0xc001048f, "TODO_c001_048f", 0),
+    MVI(0xc0010490, "TODO_c001_0490", 0),
+    MVI(0xc0010491, "TODO_c001_0491", 0),
+    MVI(0xc0010492, "TODO_c001_0492", 0),
+    MVI(0xc0010493, "TODO_c001_0493", 0),
+    MVI(0xc0010494, "TODO_c001_0494", 0),
+    MVI(0xc0010495, "TODO_c001_0495", 0),
+    MVI(0xc0010496, "TODO_c001_0496", 0),
+    MVI(0xc0010497, "TODO_c001_0497", 0),
+    MVI(0xc0010498, "TODO_c001_0498", 0),
+    MVI(0xc0010499, "TODO_c001_0499", 0),
+    MVI(0xc001049a, "TODO_c001_049a", 0),
+    MVI(0xc001049b, "TODO_c001_049b", 0),
+    MVI(0xc001049c, "TODO_c001_049c", 0),
+    MVI(0xc001049d, "TODO_c001_049d", 0),
+    MVI(0xc001049e, "TODO_c001_049e", 0),
+    MVI(0xc001049f, "TODO_c001_049f", 0),
+    MVI(0xc00104a0, "TODO_c001_04a0", 0),
+    MVI(0xc00104a1, "TODO_c001_04a1", 0),
+    MVI(0xc00104a2, "TODO_c001_04a2", 0),
+    MVI(0xc00104a3, "TODO_c001_04a3", 0),
+    MVI(0xc00104a4, "TODO_c001_04a4", 0),
+    MVI(0xc00104a5, "TODO_c001_04a5", 0),
+    MVI(0xc00104a6, "TODO_c001_04a6", 0),
+    MVI(0xc00104a7, "TODO_c001_04a7", 0),
+    MVI(0xc00104a8, "TODO_c001_04a8", 0),
+    MVI(0xc00104a9, "TODO_c001_04a9", 0),
+    MVI(0xc00104aa, "TODO_c001_04aa", 0),
+    MVI(0xc00104ab, "TODO_c001_04ab", 0),
+    MVI(0xc00104ac, "TODO_c001_04ac", 0),
+    MVI(0xc00104ad, "TODO_c001_04ad", 0),
+    MVI(0xc00104ae, "TODO_c001_04ae", 0),
+    MVI(0xc00104af, "TODO_c001_04af", 0),
+    MVI(0xc00104b0, "TODO_c001_04b0", 0),
+    MVI(0xc00104b1, "TODO_c001_04b1", 0),
+    MVI(0xc00104b2, "TODO_c001_04b2", 0),
+    MVI(0xc00104b3, "TODO_c001_04b3", 0),
+    MVI(0xc00104b4, "TODO_c001_04b4", 0),
+    MVI(0xc00104b5, "TODO_c001_04b5", 0),
+    MVI(0xc00104b6, "TODO_c001_04b6", 0),
+    MVI(0xc00104b7, "TODO_c001_04b7", 0),
+    MVI(0xc00104b8, "TODO_c001_04b8", 0),
+    MVI(0xc00104b9, "TODO_c001_04b9", 0),
+    MVI(0xc00104ba, "TODO_c001_04ba", 0),
+    MVI(0xc00104bb, "TODO_c001_04bb", 0),
+    MVI(0xc00104bc, "TODO_c001_04bc", 0),
+    MVI(0xc00104bd, "TODO_c001_04bd", 0),
+    MVI(0xc00104be, "TODO_c001_04be", 0),
+    MVI(0xc00104bf, "TODO_c001_04bf", 0),
+    MVI(0xc00104c0, "TODO_c001_04c0", 0),
+    MVI(0xc00104c1, "TODO_c001_04c1", 0),
+    MVI(0xc00104c2, "TODO_c001_04c2", 0),
+    MVI(0xc00104c3, "TODO_c001_04c3", 0),
+    MVI(0xc00104c4, "TODO_c001_04c4", 0),
+    MVI(0xc00104c5, "TODO_c001_04c5", 0),
+    MVI(0xc00104c6, "TODO_c001_04c6", 0),
+    MVI(0xc00104c7, "TODO_c001_04c7", 0),
+    MVI(0xc00104c8, "TODO_c001_04c8", 0),
+    MVI(0xc00104c9, "TODO_c001_04c9", 0),
+    MVI(0xc00104ca, "TODO_c001_04ca", 0),
+    MVI(0xc00104cb, "TODO_c001_04cb", 0),
+    MVI(0xc00104cc, "TODO_c001_04cc", 0),
+    MVI(0xc00104cd, "TODO_c001_04cd", 0),
+    MVI(0xc00104ce, "TODO_c001_04ce", 0),
+    MVI(0xc00104cf, "TODO_c001_04cf", 0),
+    MVI(0xc00104d0, "TODO_c001_04d0", 0),
+    MVI(0xc00104d1, "TODO_c001_04d1", 0),
+    MVI(0xc00104d2, "TODO_c001_04d2", 0),
+    MVI(0xc00104d3, "TODO_c001_04d3", 0),
+    MVI(0xc00104d4, "TODO_c001_04d4", 0),
+    MVI(0xc00104d5, "TODO_c001_04d5", 0),
+    MVI(0xc00104d6, "TODO_c001_04d6", 0),
+    MVI(0xc00104d7, "TODO_c001_04d7", 0),
+    MVI(0xc00104d8, "TODO_c001_04d8", 0),
+    MVI(0xc00104d9, "TODO_c001_04d9", 0),
+    MVI(0xc00104da, "TODO_c001_04da", 0),
+    MVI(0xc00104db, "TODO_c001_04db", 0),
+    MVI(0xc00104dc, "TODO_c001_04dc", 0),
+    MVI(0xc00104dd, "TODO_c001_04dd", 0),
+    MVI(0xc00104de, "TODO_c001_04de", 0),
+    MVI(0xc00104df, "TODO_c001_04df", 0),
+    MVI(0xc00104e0, "TODO_c001_04e0", 0),
+    MVI(0xc00104e1, "TODO_c001_04e1", 0),
+    MVI(0xc00104e2, "TODO_c001_04e2", 0),
+    MVI(0xc00104e3, "TODO_c001_04e3", 0),
+    MVI(0xc00104e4, "TODO_c001_04e4", 0),
+    MVI(0xc00104e5, "TODO_c001_04e5", 0),
+    MVI(0xc00104e6, "TODO_c001_04e6", 0),
+    MVI(0xc00104e7, "TODO_c001_04e7", 0),
+    MVI(0xc00104e8, "TODO_c001_04e8", 0),
+    MVI(0xc00104e9, "TODO_c001_04e9", 0),
+    MVI(0xc00104ea, "TODO_c001_04ea", 0),
+    MVI(0xc00104eb, "TODO_c001_04eb", 0),
+    MVI(0xc00104ec, "TODO_c001_04ec", 0),
+    MVI(0xc00104ed, "TODO_c001_04ed", 0),
+    MVI(0xc00104ee, "TODO_c001_04ee", 0),
+    MVI(0xc00104ef, "TODO_c001_04ef", 0),
+    MVI(0xc00104f0, "TODO_c001_04f0", 0),
+    MVI(0xc00104f1, "TODO_c001_04f1", 0),
+    MVI(0xc00104f2, "TODO_c001_04f2", 0),
+    MVI(0xc00104f3, "TODO_c001_04f3", 0),
+    MVI(0xc00104f4, "TODO_c001_04f4", 0),
+    MVI(0xc00104f5, "TODO_c001_04f5", 0),
+    MVI(0xc00104f6, "TODO_c001_04f6", 0),
+    MVI(0xc00104f7, "TODO_c001_04f7", 0),
+    MVI(0xc00104f8, "TODO_c001_04f8", 0),
+    MVI(0xc00104f9, "TODO_c001_04f9", 0),
+    MVI(0xc00104fa, "TODO_c001_04fa", 0),
+    MVI(0xc00104fb, "TODO_c001_04fb", 0),
+    MVI(0xc00104fc, "TODO_c001_04fc", 0),
+    MVI(0xc00104fd, "TODO_c001_04fd", 0),
+    MVI(0xc00104fe, "TODO_c001_04fe", 0),
+    MVI(0xc00104ff, "TODO_c001_04ff", 0),
+    MVI(0xc0010500, "TODO_c001_0500", 0),
+    MVI(0xc0010501, "TODO_c001_0501", 0),
+    MVX(0xc0010502, "TODO_c001_0502", 0, UINT64_C(0xfffe000000000000), 0),
+    MVI(0xc0010503, "TODO_c001_0503", 0),
+    MVI(0xc0010504, "TODO_c001_0504", 0),
+    MVI(0xc0010505, "TODO_c001_0505", 0),
+    MVI(0xc0010506, "TODO_c001_0506", 0),
+    MVX(0xc0010507, "TODO_c001_0507", 0, UINT64_C(0xfffe000000000000), 0),
+    MVX(0xc0010508, "TODO_c001_0508", 0, UINT64_C(0xfffe000000000000), 0),
+    MVX(0xc0010509, "TODO_c001_0509", 0, UINT64_C(0xfffe000000000000), 0),
+    MVX(0xc001050a, "TODO_c001_050a", 0, UINT64_C(0xfffe000000000000), 0),
+    MVX(0xc001050b, "TODO_c001_050b", 0, UINT64_C(0xfffe000000000000), 0),
+    MVX(0xc001050c, "TODO_c001_050c", 0, UINT64_C(0xfffe000000000000), 0),
+    MVX(0xc001050d, "TODO_c001_050d", 0, UINT64_C(0xfffe000000000000), 0),
+    MVX(0xc001050e, "TODO_c001_050e", 0, UINT64_C(0xfffe000000000000), 0),
+    MVI(0xc001050f, "TODO_c001_050f", 0),
+    MVI(0xc0010510, "TODO_c001_0510", 0),
+    MVI(0xc0010511, "TODO_c001_0511", 0),
+    MVI(0xc0010512, "TODO_c001_0512", 0),
+    MVI(0xc0010513, "TODO_c001_0513", 0),
+    MVX(0xc0010514, "TODO_c001_0514", 0, UINT64_C(0xfffffe0000000000), 0),
+    MVX(0xc0010515, "TODO_c001_0515", 0, UINT64_C(0xfffffe0000000000), 0),
+    MVI(0xc0010516, "TODO_c001_0516", 0),
+    MVI(0xc0010517, "TODO_c001_0517", 0),
+    MVI(0xc0010518, "TODO_c001_0518", 0),
+    MVI(0xc0010519, "TODO_c001_0519", 0),
+    MVI(0xc001051a, "TODO_c001_051a", 0),
+    MVI(0xc001051b, "TODO_c001_051b", 0),
+    MVI(0xc001051c, "TODO_c001_051c", 0),
+    MVI(0xc001051d, "TODO_c001_051d", 0),
+    MVI(0xc001051e, "TODO_c001_051e", 0),
+    MVI(0xc001051f, "TODO_c001_051f", 0),
+    MVI(0xc0010520, "TODO_c001_0520", 0),
+    MVI(0xc0010521, "TODO_c001_0521", 0),
+    MVI(0xc0010522, "TODO_c001_0522", 0),
+    MVI(0xc0010523, "TODO_c001_0523", 0),
+    MVI(0xc0010524, "TODO_c001_0524", 0),
+    MVI(0xc0010525, "TODO_c001_0525", 0),
+    MVI(0xc0010526, "TODO_c001_0526", 0),
+    MVI(0xc0010527, "TODO_c001_0527", 0),
+    MVI(0xc0010528, "TODO_c001_0528", 0),
+    MVI(0xc0010529, "TODO_c001_0529", 0),
+    MVI(0xc001052a, "TODO_c001_052a", 0),
+    MVI(0xc001052b, "TODO_c001_052b", 0),
+    MVI(0xc001052c, "TODO_c001_052c", 0),
+    MVI(0xc001052d, "TODO_c001_052d", 0),
+    MVI(0xc001052e, "TODO_c001_052e", 0),
+    MVI(0xc001052f, "TODO_c001_052f", 0),
+    MVI(0xc0010530, "TODO_c001_0530", 0),
+    MVI(0xc0010531, "TODO_c001_0531", 0),
+    MVI(0xc0010532, "TODO_c001_0532", 0),
+    MVI(0xc0010533, "TODO_c001_0533", 0),
+    MVI(0xc0010534, "TODO_c001_0534", 0),
+    MVI(0xc0010535, "TODO_c001_0535", 0),
+    MVI(0xc0010536, "TODO_c001_0536", 0),
+    MVI(0xc0010537, "TODO_c001_0537", 0),
+    MVI(0xc0010538, "TODO_c001_0538", 0),
+    MVI(0xc0010539, "TODO_c001_0539", 0),
+    MVI(0xc001053a, "TODO_c001_053a", 0),
+    MVI(0xc001053b, "TODO_c001_053b", 0),
+    MVI(0xc001053c, "TODO_c001_053c", 0),
+    MVI(0xc001053d, "TODO_c001_053d", 0),
+    MVI(0xc001053e, "TODO_c001_053e", 0),
+    MVI(0xc001053f, "TODO_c001_053f", 0),
+    MVI(0xc0010540, "TODO_c001_0540", 0),
+    MVI(0xc0010541, "TODO_c001_0541", 0),
+    MVI(0xc0010542, "TODO_c001_0542", 0),
+    MVI(0xc0010543, "TODO_c001_0543", 0),
+    MVI(0xc0010544, "TODO_c001_0544", 0),
+    MVI(0xc0010545, "TODO_c001_0545", 0),
+    MVI(0xc0010546, "TODO_c001_0546", 0),
+    MVI(0xc0010547, "TODO_c001_0547", 0),
+    MVI(0xc0010548, "TODO_c001_0548", 0),
+    MVI(0xc0010549, "TODO_c001_0549", 0),
+    MVI(0xc001054a, "TODO_c001_054a", 0),
+    MVI(0xc001054b, "TODO_c001_054b", 0),
+    MVI(0xc001054c, "TODO_c001_054c", 0),
+    MVI(0xc001054d, "TODO_c001_054d", 0),
+    MVI(0xc001054e, "TODO_c001_054e", 0),
+    MVI(0xc001054f, "TODO_c001_054f", 0),
+    MVI(0xc0010550, "TODO_c001_0550", 0),
+    MVI(0xc0010551, "TODO_c001_0551", 0),
+    MVI(0xc0010552, "TODO_c001_0552", 0),
+    MVI(0xc0010553, "TODO_c001_0553", 0),
+    MVI(0xc0010554, "TODO_c001_0554", 0),
+    MVI(0xc0010555, "TODO_c001_0555", 0),
+    MVI(0xc0010556, "TODO_c001_0556", 0),
+    MVI(0xc0010557, "TODO_c001_0557", 0),
+    MVI(0xc0010558, "TODO_c001_0558", 0),
+    MVI(0xc0010559, "TODO_c001_0559", 0),
+    MVI(0xc001055a, "TODO_c001_055a", 0),
+    MVI(0xc001055b, "TODO_c001_055b", 0),
+    MVI(0xc001055c, "TODO_c001_055c", 0),
+    MVI(0xc001055d, "TODO_c001_055d", 0),
+    MVI(0xc001055e, "TODO_c001_055e", 0),
+    MVI(0xc001055f, "TODO_c001_055f", 0),
+    MVI(0xc0010560, "TODO_c001_0560", 0),
+    MVI(0xc0010561, "TODO_c001_0561", 0),
+    MVI(0xc0010562, "TODO_c001_0562", 0),
+    MVI(0xc0010563, "TODO_c001_0563", 0),
+    MVI(0xc0010564, "TODO_c001_0564", 0),
+    MVI(0xc0010565, "TODO_c001_0565", 0),
+    MVI(0xc0010566, "TODO_c001_0566", 0),
+    MVI(0xc0010567, "TODO_c001_0567", 0),
+    MVI(0xc0010568, "TODO_c001_0568", 0),
+    MVI(0xc0010569, "TODO_c001_0569", 0),
+    MVI(0xc001056a, "TODO_c001_056a", 0),
+    MVI(0xc001056b, "TODO_c001_056b", 0),
+    MVI(0xc001056c, "TODO_c001_056c", 0),
+    MVI(0xc001056d, "TODO_c001_056d", 0),
+    MVI(0xc001056e, "TODO_c001_056e", 0),
+    MVI(0xc001056f, "TODO_c001_056f", 0),
+    MVI(0xc0010570, "TODO_c001_0570", 0),
+    MVI(0xc0010571, "TODO_c001_0571", 0),
+    MVI(0xc0010572, "TODO_c001_0572", 0),
+    MVI(0xc0010573, "TODO_c001_0573", 0),
+    MVI(0xc0010574, "TODO_c001_0574", 0),
+    MVI(0xc0010575, "TODO_c001_0575", 0),
+    MVI(0xc0010576, "TODO_c001_0576", 0),
+    MVI(0xc0010577, "TODO_c001_0577", 0),
+    MVI(0xc0010578, "TODO_c001_0578", 0),
+    MVI(0xc0010579, "TODO_c001_0579", 0),
+    MVI(0xc001057a, "TODO_c001_057a", 0),
+    MVI(0xc001057b, "TODO_c001_057b", 0),
+    MVI(0xc001057c, "TODO_c001_057c", 0),
+    MVI(0xc001057d, "TODO_c001_057d", 0),
+    MVI(0xc001057e, "TODO_c001_057e", 0),
+    MVI(0xc001057f, "TODO_c001_057f", 0),
+    MVI(0xc0010580, "TODO_c001_0580", 0),
+    MVI(0xc0010581, "TODO_c001_0581", 0),
+    MVI(0xc0010582, "TODO_c001_0582", 0),
+    MVI(0xc0010583, "TODO_c001_0583", 0),
+    MVI(0xc0010584, "TODO_c001_0584", 0),
+    MVI(0xc0010585, "TODO_c001_0585", 0),
+    MVI(0xc0010586, "TODO_c001_0586", 0),
+    MVI(0xc0010587, "TODO_c001_0587", 0),
+    MVI(0xc0010588, "TODO_c001_0588", 0),
+    MVI(0xc0010589, "TODO_c001_0589", 0),
+    MVI(0xc001058a, "TODO_c001_058a", 0),
+    MVI(0xc001058b, "TODO_c001_058b", 0),
+    MVI(0xc001058c, "TODO_c001_058c", 0),
+    MVI(0xc001058d, "TODO_c001_058d", 0),
+    MVI(0xc001058e, "TODO_c001_058e", 0),
+    MVI(0xc001058f, "TODO_c001_058f", 0),
+    MVI(0xc0010590, "TODO_c001_0590", 0),
+    MVI(0xc0010591, "TODO_c001_0591", 0),
+    MVI(0xc0010592, "TODO_c001_0592", 0),
+    MVI(0xc0010593, "TODO_c001_0593", 0),
+    MVI(0xc0010594, "TODO_c001_0594", 0),
+    MVI(0xc0010595, "TODO_c001_0595", 0),
+    MVI(0xc0010596, "TODO_c001_0596", 0),
+    MVI(0xc0010597, "TODO_c001_0597", 0),
+    MVI(0xc0010598, "TODO_c001_0598", 0),
+    MVI(0xc0010599, "TODO_c001_0599", 0),
+    MVI(0xc001059a, "TODO_c001_059a", 0),
+    MVI(0xc001059b, "TODO_c001_059b", 0),
+    MVI(0xc001059c, "TODO_c001_059c", 0),
+    MVI(0xc001059d, "TODO_c001_059d", 0),
+    MVI(0xc001059e, "TODO_c001_059e", 0),
+    MVI(0xc001059f, "TODO_c001_059f", 0),
+    MVI(0xc00105a0, "TODO_c001_05a0", 0),
+    MVI(0xc00105a1, "TODO_c001_05a1", 0),
+    MVI(0xc00105a2, "TODO_c001_05a2", 0),
+    MVI(0xc00105a3, "TODO_c001_05a3", 0),
+    MVI(0xc00105a4, "TODO_c001_05a4", 0),
+    MVI(0xc00105a5, "TODO_c001_05a5", 0),
+    MVI(0xc00105a6, "TODO_c001_05a6", 0),
+    MVI(0xc00105a7, "TODO_c001_05a7", 0),
+    MVI(0xc00105a8, "TODO_c001_05a8", 0),
+    MVI(0xc00105a9, "TODO_c001_05a9", 0),
+    MVI(0xc00105aa, "TODO_c001_05aa", 0),
+    MVI(0xc00105ab, "TODO_c001_05ab", 0),
+    MVI(0xc00105ac, "TODO_c001_05ac", 0),
+    MVI(0xc00105ad, "TODO_c001_05ad", 0),
+    MVI(0xc00105ae, "TODO_c001_05ae", 0),
+    MVI(0xc00105af, "TODO_c001_05af", 0),
+    MVI(0xc00105b0, "TODO_c001_05b0", 0),
+    MVI(0xc00105b1, "TODO_c001_05b1", 0),
+    MVI(0xc00105b2, "TODO_c001_05b2", 0),
+    MVI(0xc00105b3, "TODO_c001_05b3", 0),
+    MVI(0xc00105b4, "TODO_c001_05b4", 0),
+    MVI(0xc00105b5, "TODO_c001_05b5", 0),
+    MVI(0xc00105b6, "TODO_c001_05b6", 0),
+    MVI(0xc00105b7, "TODO_c001_05b7", 0),
+    MVI(0xc00105b8, "TODO_c001_05b8", 0),
+    MVI(0xc00105b9, "TODO_c001_05b9", 0),
+    MVI(0xc00105ba, "TODO_c001_05ba", 0),
+    MVI(0xc00105bb, "TODO_c001_05bb", 0),
+    MVI(0xc00105bc, "TODO_c001_05bc", 0),
+    MVI(0xc00105bd, "TODO_c001_05bd", 0),
+    MVI(0xc00105be, "TODO_c001_05be", 0),
+    MVI(0xc00105bf, "TODO_c001_05bf", 0),
+    MVI(0xc00105c0, "TODO_c001_05c0", 0),
+    MVI(0xc00105c1, "TODO_c001_05c1", 0),
+    MVI(0xc00105c2, "TODO_c001_05c2", 0),
+    MVI(0xc00105c3, "TODO_c001_05c3", 0),
+    MVI(0xc00105c4, "TODO_c001_05c4", 0),
+    MVI(0xc00105c5, "TODO_c001_05c5", 0),
+    MVI(0xc00105c6, "TODO_c001_05c6", 0),
+    MVI(0xc00105c7, "TODO_c001_05c7", 0),
+    MVI(0xc00105c8, "TODO_c001_05c8", 0),
+    MVI(0xc00105c9, "TODO_c001_05c9", 0),
+    MVI(0xc00105ca, "TODO_c001_05ca", 0),
+    MVI(0xc00105cb, "TODO_c001_05cb", 0),
+    MVI(0xc00105cc, "TODO_c001_05cc", 0),
+    MVI(0xc00105cd, "TODO_c001_05cd", 0),
+    MVI(0xc00105ce, "TODO_c001_05ce", 0),
+    MVI(0xc00105cf, "TODO_c001_05cf", 0),
+    MVI(0xc00105d0, "TODO_c001_05d0", 0),
+    MVI(0xc00105d1, "TODO_c001_05d1", 0),
+    MVI(0xc00105d2, "TODO_c001_05d2", 0),
+    MVI(0xc00105d3, "TODO_c001_05d3", 0),
+    MVI(0xc00105d4, "TODO_c001_05d4", 0),
+    MVI(0xc00105d5, "TODO_c001_05d5", 0),
+    MVI(0xc00105d6, "TODO_c001_05d6", 0),
+    MVI(0xc00105d7, "TODO_c001_05d7", 0),
+    MVI(0xc00105d8, "TODO_c001_05d8", 0),
+    MVI(0xc00105d9, "TODO_c001_05d9", 0),
+    MVI(0xc00105da, "TODO_c001_05da", 0),
+    MVI(0xc00105db, "TODO_c001_05db", 0),
+    MVI(0xc00105dc, "TODO_c001_05dc", 0),
+    MVI(0xc00105dd, "TODO_c001_05dd", 0),
+    MVI(0xc00105de, "TODO_c001_05de", 0),
+    MVI(0xc00105df, "TODO_c001_05df", 0),
+    MVI(0xc00105e0, "TODO_c001_05e0", 0),
+    MVI(0xc00105e1, "TODO_c001_05e1", 0),
+    MVI(0xc00105e2, "TODO_c001_05e2", 0),
+    MVI(0xc00105e3, "TODO_c001_05e3", 0),
+    MVI(0xc00105e4, "TODO_c001_05e4", 0),
+    MVI(0xc00105e5, "TODO_c001_05e5", 0),
+    MVI(0xc00105e6, "TODO_c001_05e6", 0),
+    MVI(0xc00105e7, "TODO_c001_05e7", 0),
+    MVI(0xc00105e8, "TODO_c001_05e8", 0),
+    MVI(0xc00105e9, "TODO_c001_05e9", 0),
+    MVI(0xc00105ea, "TODO_c001_05ea", 0),
+    MVI(0xc00105eb, "TODO_c001_05eb", 0),
+    MVI(0xc00105ec, "TODO_c001_05ec", 0),
+    MVI(0xc00105ed, "TODO_c001_05ed", 0),
+    MVI(0xc00105ee, "TODO_c001_05ee", 0),
+    MVI(0xc00105ef, "TODO_c001_05ef", 0),
+    MVI(0xc00105f0, "TODO_c001_05f0", 0),
+    MVI(0xc00105f1, "TODO_c001_05f1", 0),
+    MVI(0xc00105f2, "TODO_c001_05f2", 0),
+    MVI(0xc00105f3, "TODO_c001_05f3", 0),
+    MVI(0xc00105f4, "TODO_c001_05f4", 0),
+    MVI(0xc00105f5, "TODO_c001_05f5", 0),
+    MVI(0xc00105f6, "TODO_c001_05f6", 0),
+    MVI(0xc00105f7, "TODO_c001_05f7", 0),
+    MVI(0xc00105f8, "TODO_c001_05f8", 0),
+    MVI(0xc00105f9, "TODO_c001_05f9", 0),
+    MVI(0xc00105fa, "TODO_c001_05fa", 0),
+    MVI(0xc00105fb, "TODO_c001_05fb", 0),
+    MVI(0xc00105fc, "TODO_c001_05fc", 0),
+    MVI(0xc00105fd, "TODO_c001_05fd", 0),
+    MVI(0xc00105fe, "TODO_c001_05fe", 0),
+    MVI(0xc00105ff, "TODO_c001_05ff", 0),
+    MVI(0xc0010600, "TODO_c001_0600", 0),
+    MVI(0xc0010601, "TODO_c001_0601", 0),
+    MVX(0xc0010602, "TODO_c001_0602", 0, 0, 0),
+    MVI(0xc0010603, "TODO_c001_0603", 0),
+    MVI(0xc0010604, "TODO_c001_0604", 0),
+    MVI(0xc0010605, "TODO_c001_0605", 0),
+    MVI(0xc0010606, "TODO_c001_0606", 0),
+    MVX(0xc0010607, "TODO_c001_0607", 0, 0, 0),
+    MVX(0xc0010608, "TODO_c001_0608", 0, 0, 0),
+    MVX(0xc0010609, "TODO_c001_0609", 0, 0, 0),
+    MVX(0xc001060a, "TODO_c001_060a", 0, 0, 0),
+    MVX(0xc001060b, "TODO_c001_060b", 0, 0, 0),
+    MVX(0xc001060c, "TODO_c001_060c", 0, 0, 0),
+    MVX(0xc001060d, "TODO_c001_060d", 0, 0, 0),
+    MVX(0xc001060e, "TODO_c001_060e", 0, 0, 0),
+    MVI(0xc001060f, "TODO_c001_060f", 0),
+    MVI(0xc0010610, "TODO_c001_0610", 0),
+    MVI(0xc0010611, "TODO_c001_0611", 0),
+    MVI(0xc0010612, "TODO_c001_0612", 0),
+    MVI(0xc0010613, "TODO_c001_0613", 0),
+    MVX(0xc0010614, "TODO_c001_0614", 0, 0, 0),
+    MVX(0xc0010615, "TODO_c001_0615", 0, 0, 0),
+    MVI(0xc0010616, "TODO_c001_0616", 0),
+    MVI(0xc0010617, "TODO_c001_0617", 0),
+    MVI(0xc0010618, "TODO_c001_0618", 0),
+    MVI(0xc0010619, "TODO_c001_0619", 0),
+    MVI(0xc001061a, "TODO_c001_061a", 0),
+    MVI(0xc001061b, "TODO_c001_061b", 0),
+    MVI(0xc001061c, "TODO_c001_061c", 0),
+    MVI(0xc001061d, "TODO_c001_061d", 0),
+    MVI(0xc001061e, "TODO_c001_061e", 0),
+    MVI(0xc001061f, "TODO_c001_061f", 0),
+    MVI(0xc0010620, "TODO_c001_0620", 0),
+    MVI(0xc0010621, "TODO_c001_0621", 0),
+    MVI(0xc0010622, "TODO_c001_0622", 0),
+    MVI(0xc0010623, "TODO_c001_0623", 0),
+    MVI(0xc0010624, "TODO_c001_0624", 0),
+    MVI(0xc0010625, "TODO_c001_0625", 0),
+    MVI(0xc0010626, "TODO_c001_0626", 0),
+    MVI(0xc0010627, "TODO_c001_0627", 0),
+    MVI(0xc0010628, "TODO_c001_0628", 0),
+    MVI(0xc0010629, "TODO_c001_0629", 0),
+    MVI(0xc001062a, "TODO_c001_062a", 0),
+    MVI(0xc001062b, "TODO_c001_062b", 0),
+    MVI(0xc001062c, "TODO_c001_062c", 0),
+    MVI(0xc001062d, "TODO_c001_062d", 0),
+    MVI(0xc001062e, "TODO_c001_062e", 0),
+    MVI(0xc001062f, "TODO_c001_062f", 0),
+    MVI(0xc0010630, "TODO_c001_0630", 0),
+    MVI(0xc0010631, "TODO_c001_0631", 0),
+    MVI(0xc0010632, "TODO_c001_0632", 0),
+    MVI(0xc0010633, "TODO_c001_0633", 0),
+    MVI(0xc0010634, "TODO_c001_0634", 0),
+    MVI(0xc0010635, "TODO_c001_0635", 0),
+    MVI(0xc0010636, "TODO_c001_0636", 0),
+    MVI(0xc0010637, "TODO_c001_0637", 0),
+    MVI(0xc0010638, "TODO_c001_0638", 0),
+    MVI(0xc0010639, "TODO_c001_0639", 0),
+    MVI(0xc001063a, "TODO_c001_063a", 0),
+    MVI(0xc001063b, "TODO_c001_063b", 0),
+    MVI(0xc001063c, "TODO_c001_063c", 0),
+    MVI(0xc001063d, "TODO_c001_063d", 0),
+    MVI(0xc001063e, "TODO_c001_063e", 0),
+    MVI(0xc001063f, "TODO_c001_063f", 0),
+    MVI(0xc0010640, "TODO_c001_0640", 0),
+    MVI(0xc0010641, "TODO_c001_0641", 0),
+    MVI(0xc0010642, "TODO_c001_0642", 0),
+    MVI(0xc0010643, "TODO_c001_0643", 0),
+    MVI(0xc0010644, "TODO_c001_0644", 0),
+    MVI(0xc0010645, "TODO_c001_0645", 0),
+    MVI(0xc0010646, "TODO_c001_0646", 0),
+    MVI(0xc0010647, "TODO_c001_0647", 0),
+    MVI(0xc0010648, "TODO_c001_0648", 0),
+    MVI(0xc0010649, "TODO_c001_0649", 0),
+    MVI(0xc001064a, "TODO_c001_064a", 0),
+    MVI(0xc001064b, "TODO_c001_064b", 0),
+    MVI(0xc001064c, "TODO_c001_064c", 0),
+    MVI(0xc001064d, "TODO_c001_064d", 0),
+    MVI(0xc001064e, "TODO_c001_064e", 0),
+    MVI(0xc001064f, "TODO_c001_064f", 0),
+    MVI(0xc0010650, "TODO_c001_0650", 0),
+    MVI(0xc0010651, "TODO_c001_0651", 0),
+    MVI(0xc0010652, "TODO_c001_0652", 0),
+    MVI(0xc0010653, "TODO_c001_0653", 0),
+    MVI(0xc0010654, "TODO_c001_0654", 0),
+    MVI(0xc0010655, "TODO_c001_0655", 0),
+    MVI(0xc0010656, "TODO_c001_0656", 0),
+    MVI(0xc0010657, "TODO_c001_0657", 0),
+    MVI(0xc0010658, "TODO_c001_0658", 0),
+    MVI(0xc0010659, "TODO_c001_0659", 0),
+    MVI(0xc001065a, "TODO_c001_065a", 0),
+    MVI(0xc001065b, "TODO_c001_065b", 0),
+    MVI(0xc001065c, "TODO_c001_065c", 0),
+    MVI(0xc001065d, "TODO_c001_065d", 0),
+    MVI(0xc001065e, "TODO_c001_065e", 0),
+    MVI(0xc001065f, "TODO_c001_065f", 0),
+    MVI(0xc0010660, "TODO_c001_0660", 0),
+    MVI(0xc0010661, "TODO_c001_0661", 0),
+    MVI(0xc0010662, "TODO_c001_0662", 0),
+    MVI(0xc0010663, "TODO_c001_0663", 0),
+    MVI(0xc0010664, "TODO_c001_0664", 0),
+    MVI(0xc0010665, "TODO_c001_0665", 0),
+    MVI(0xc0010666, "TODO_c001_0666", 0),
+    MVI(0xc0010667, "TODO_c001_0667", 0),
+    MVI(0xc0010668, "TODO_c001_0668", 0),
+    MVI(0xc0010669, "TODO_c001_0669", 0),
+    MVI(0xc001066a, "TODO_c001_066a", 0),
+    MVI(0xc001066b, "TODO_c001_066b", 0),
+    MVI(0xc001066c, "TODO_c001_066c", 0),
+    MVI(0xc001066d, "TODO_c001_066d", 0),
+    MVI(0xc001066e, "TODO_c001_066e", 0),
+    MVI(0xc001066f, "TODO_c001_066f", 0),
+    MVI(0xc0010670, "TODO_c001_0670", 0),
+    MVI(0xc0010671, "TODO_c001_0671", 0),
+    MVI(0xc0010672, "TODO_c001_0672", 0),
+    MVI(0xc0010673, "TODO_c001_0673", 0),
+    MVI(0xc0010674, "TODO_c001_0674", 0),
+    MVI(0xc0010675, "TODO_c001_0675", 0),
+    MVI(0xc0010676, "TODO_c001_0676", 0),
+    MVI(0xc0010677, "TODO_c001_0677", 0),
+    MVI(0xc0010678, "TODO_c001_0678", 0),
+    MVI(0xc0010679, "TODO_c001_0679", 0),
+    MVI(0xc001067a, "TODO_c001_067a", 0),
+    MVI(0xc001067b, "TODO_c001_067b", 0),
+    MVI(0xc001067c, "TODO_c001_067c", 0),
+    MVI(0xc001067d, "TODO_c001_067d", 0),
+    MVI(0xc001067e, "TODO_c001_067e", 0),
+    MVI(0xc001067f, "TODO_c001_067f", 0),
+    MVI(0xc0010680, "TODO_c001_0680", 0),
+    MVI(0xc0010681, "TODO_c001_0681", 0),
+    MVI(0xc0010682, "TODO_c001_0682", 0),
+    MVI(0xc0010683, "TODO_c001_0683", 0),
+    MVI(0xc0010684, "TODO_c001_0684", 0),
+    MVI(0xc0010685, "TODO_c001_0685", 0),
+    MVI(0xc0010686, "TODO_c001_0686", 0),
+    MVI(0xc0010687, "TODO_c001_0687", 0),
+    MVI(0xc0010688, "TODO_c001_0688", 0),
+    MVI(0xc0010689, "TODO_c001_0689", 0),
+    MVI(0xc001068a, "TODO_c001_068a", 0),
+    MVI(0xc001068b, "TODO_c001_068b", 0),
+    MVI(0xc001068c, "TODO_c001_068c", 0),
+    MVI(0xc001068d, "TODO_c001_068d", 0),
+    MVI(0xc001068e, "TODO_c001_068e", 0),
+    MVI(0xc001068f, "TODO_c001_068f", 0),
+    MVI(0xc0010690, "TODO_c001_0690", 0),
+    MVI(0xc0010691, "TODO_c001_0691", 0),
+    MVI(0xc0010692, "TODO_c001_0692", 0),
+    MVI(0xc0010693, "TODO_c001_0693", 0),
+    MVI(0xc0010694, "TODO_c001_0694", 0),
+    MVI(0xc0010695, "TODO_c001_0695", 0),
+    MVI(0xc0010696, "TODO_c001_0696", 0),
+    MVI(0xc0010697, "TODO_c001_0697", 0),
+    MVI(0xc0010698, "TODO_c001_0698", 0),
+    MVI(0xc0010699, "TODO_c001_0699", 0),
+    MVI(0xc001069a, "TODO_c001_069a", 0),
+    MVI(0xc001069b, "TODO_c001_069b", 0),
+    MVI(0xc001069c, "TODO_c001_069c", 0),
+    MVI(0xc001069d, "TODO_c001_069d", 0),
+    MVI(0xc001069e, "TODO_c001_069e", 0),
+    MVI(0xc001069f, "TODO_c001_069f", 0),
+    MVI(0xc00106a0, "TODO_c001_06a0", 0),
+    MVI(0xc00106a1, "TODO_c001_06a1", 0),
+    MVI(0xc00106a2, "TODO_c001_06a2", 0),
+    MVI(0xc00106a3, "TODO_c001_06a3", 0),
+    MVI(0xc00106a4, "TODO_c001_06a4", 0),
+    MVI(0xc00106a5, "TODO_c001_06a5", 0),
+    MVI(0xc00106a6, "TODO_c001_06a6", 0),
+    MVI(0xc00106a7, "TODO_c001_06a7", 0),
+    MVI(0xc00106a8, "TODO_c001_06a8", 0),
+    MVI(0xc00106a9, "TODO_c001_06a9", 0),
+    MVI(0xc00106aa, "TODO_c001_06aa", 0),
+    MVI(0xc00106ab, "TODO_c001_06ab", 0),
+    MVI(0xc00106ac, "TODO_c001_06ac", 0),
+    MVI(0xc00106ad, "TODO_c001_06ad", 0),
+    MVI(0xc00106ae, "TODO_c001_06ae", 0),
+    MVI(0xc00106af, "TODO_c001_06af", 0),
+    MVI(0xc00106b0, "TODO_c001_06b0", 0),
+    MVI(0xc00106b1, "TODO_c001_06b1", 0),
+    MVI(0xc00106b2, "TODO_c001_06b2", 0),
+    MVI(0xc00106b3, "TODO_c001_06b3", 0),
+    MVI(0xc00106b4, "TODO_c001_06b4", 0),
+    MVI(0xc00106b5, "TODO_c001_06b5", 0),
+    MVI(0xc00106b6, "TODO_c001_06b6", 0),
+    MVI(0xc00106b7, "TODO_c001_06b7", 0),
+    MVI(0xc00106b8, "TODO_c001_06b8", 0),
+    MVI(0xc00106b9, "TODO_c001_06b9", 0),
+    MVI(0xc00106ba, "TODO_c001_06ba", 0),
+    MVI(0xc00106bb, "TODO_c001_06bb", 0),
+    MVI(0xc00106bc, "TODO_c001_06bc", 0),
+    MVI(0xc00106bd, "TODO_c001_06bd", 0),
+    MVI(0xc00106be, "TODO_c001_06be", 0),
+    MVI(0xc00106bf, "TODO_c001_06bf", 0),
+    MVI(0xc00106c0, "TODO_c001_06c0", 0),
+    MVI(0xc00106c1, "TODO_c001_06c1", 0),
+    MVI(0xc00106c2, "TODO_c001_06c2", 0),
+    MVI(0xc00106c3, "TODO_c001_06c3", 0),
+    MVI(0xc00106c4, "TODO_c001_06c4", 0),
+    MVI(0xc00106c5, "TODO_c001_06c5", 0),
+    MVI(0xc00106c6, "TODO_c001_06c6", 0),
+    MVI(0xc00106c7, "TODO_c001_06c7", 0),
+    MVI(0xc00106c8, "TODO_c001_06c8", 0),
+    MVI(0xc00106c9, "TODO_c001_06c9", 0),
+    MVI(0xc00106ca, "TODO_c001_06ca", 0),
+    MVI(0xc00106cb, "TODO_c001_06cb", 0),
+    MVI(0xc00106cc, "TODO_c001_06cc", 0),
+    MVI(0xc00106cd, "TODO_c001_06cd", 0),
+    MVI(0xc00106ce, "TODO_c001_06ce", 0),
+    MVI(0xc00106cf, "TODO_c001_06cf", 0),
+    MVI(0xc00106d0, "TODO_c001_06d0", 0),
+    MVI(0xc00106d1, "TODO_c001_06d1", 0),
+    MVI(0xc00106d2, "TODO_c001_06d2", 0),
+    MVI(0xc00106d3, "TODO_c001_06d3", 0),
+    MVI(0xc00106d4, "TODO_c001_06d4", 0),
+    MVI(0xc00106d5, "TODO_c001_06d5", 0),
+    MVI(0xc00106d6, "TODO_c001_06d6", 0),
+    MVI(0xc00106d7, "TODO_c001_06d7", 0),
+    MVI(0xc00106d8, "TODO_c001_06d8", 0),
+    MVI(0xc00106d9, "TODO_c001_06d9", 0),
+    MVI(0xc00106da, "TODO_c001_06da", 0),
+    MVI(0xc00106db, "TODO_c001_06db", 0),
+    MVI(0xc00106dc, "TODO_c001_06dc", 0),
+    MVI(0xc00106dd, "TODO_c001_06dd", 0),
+    MVI(0xc00106de, "TODO_c001_06de", 0),
+    MVI(0xc00106df, "TODO_c001_06df", 0),
+    MVI(0xc00106e0, "TODO_c001_06e0", 0),
+    MVI(0xc00106e1, "TODO_c001_06e1", 0),
+    MVI(0xc00106e2, "TODO_c001_06e2", 0),
+    MVI(0xc00106e3, "TODO_c001_06e3", 0),
+    MVI(0xc00106e4, "TODO_c001_06e4", 0),
+    MVI(0xc00106e5, "TODO_c001_06e5", 0),
+    MVI(0xc00106e6, "TODO_c001_06e6", 0),
+    MVI(0xc00106e7, "TODO_c001_06e7", 0),
+    MVI(0xc00106e8, "TODO_c001_06e8", 0),
+    MVI(0xc00106e9, "TODO_c001_06e9", 0),
+    MVI(0xc00106ea, "TODO_c001_06ea", 0),
+    MVI(0xc00106eb, "TODO_c001_06eb", 0),
+    MVI(0xc00106ec, "TODO_c001_06ec", 0),
+    MVI(0xc00106ed, "TODO_c001_06ed", 0),
+    MVI(0xc00106ee, "TODO_c001_06ee", 0),
+    MVI(0xc00106ef, "TODO_c001_06ef", 0),
+    MVI(0xc00106f0, "TODO_c001_06f0", 0),
+    MVI(0xc00106f1, "TODO_c001_06f1", 0),
+    MVI(0xc00106f2, "TODO_c001_06f2", 0),
+    MVI(0xc00106f3, "TODO_c001_06f3", 0),
+    MVI(0xc00106f4, "TODO_c001_06f4", 0),
+    MVI(0xc00106f5, "TODO_c001_06f5", 0),
+    MVI(0xc00106f6, "TODO_c001_06f6", 0),
+    MVI(0xc00106f7, "TODO_c001_06f7", 0),
+    MVI(0xc00106f8, "TODO_c001_06f8", 0),
+    MVI(0xc00106f9, "TODO_c001_06f9", 0),
+    MVI(0xc00106fa, "TODO_c001_06fa", 0),
+    MVI(0xc00106fb, "TODO_c001_06fb", 0),
+    MVI(0xc00106fc, "TODO_c001_06fc", 0),
+    MVI(0xc00106fd, "TODO_c001_06fd", 0),
+    MVI(0xc00106fe, "TODO_c001_06fe", 0),
+    MVI(0xc00106ff, "TODO_c001_06ff", 0),
+    MFX(0xc0011000, "AMD_K7_MCODE_CTL", AmdK7MicrocodeCtl, AmdK7MicrocodeCtl, 0x18000000, ~(uint64_t)UINT32_MAX, 0x4), /* value=0x18000000 */
+    MFX(0xc0011001, "AMD_K7_APIC_CLUSTER_ID", AmdK7ClusterIdMaybe, AmdK7ClusterIdMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFN(0xc0011002, "AMD_K8_CPUID_CTL_STD07", AmdK8CpuIdCtlStd07hEbax, AmdK8CpuIdCtlStd07hEbax), /* value=0x9c01a9 */
+    MFX(0xc0011003, "AMD_K8_CPUID_CTL_STD06", AmdK8CpuIdCtlStd06hEcx, AmdK8CpuIdCtlStd06hEcx, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x1 */
+    MFN(0xc0011004, "AMD_K8_CPUID_CTL_STD01", AmdK8CpuIdCtlStd01hEdcx, AmdK8CpuIdCtlStd01hEdcx), /* value=0x7cd83209`178bfbff */
+    MFN(0xc0011005, "AMD_K8_CPUID_CTL_EXT01", AmdK8CpuIdCtlExt01hEdcx, AmdK8CpuIdCtlExt01hEdcx), /* value=0x35c233ff`2fd3fbff */
+    MFX(0xc0011006, "AMD_K7_DEBUG_STS?", AmdK7DebugStatusMaybe, AmdK7DebugStatusMaybe, 0, UINT64_C(0xffffffff00000080), 0), /* value=0x0 */
+    MFN(0xc0011007, "AMD_K7_BH_TRACE_BASE?", AmdK7BHTraceBaseMaybe, AmdK7BHTraceBaseMaybe), /* value=0x0 */
+    MFN(0xc0011008, "AMD_K7_BH_TRACE_PTR?", AmdK7BHTracePtrMaybe, AmdK7BHTracePtrMaybe), /* value=0x0 */
+    MFN(0xc0011009, "AMD_K7_BH_TRACE_LIM?", AmdK7BHTraceLimitMaybe, AmdK7BHTraceLimitMaybe), /* value=0x0 */
+    MFI(0xc001100a, "AMD_K7_HDT_CFG?", AmdK7HardwareDebugToolCfgMaybe), /* value=0x0 */
+    MFX(0xc001100b, "AMD_K7_FAST_FLUSH_COUNT?", AmdK7FastFlushCountMaybe, AmdK7FastFlushCountMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc001100c, "AMD_K7_NODE_ID", AmdK7NodeId, AmdK7NodeId, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MVX(0xc001100e, "AMD_K8_WRMSR_BP?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001100f, "AMD_K8_WRMSR_BP_MASK?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011010, "AMD_K8_BH_TRACE_CTL?", 0, UINT64_C(0xffffffff00010000), 0),
+    MVI(0xc0011011, "AMD_K8_BH_TRACE_USRD?", 0), /* value=0x0 */
+    MVX(0xc0011014, "AMD_K8_XCPT_BP_RIP?", 0, 0, 0),
+    MVX(0xc0011015, "AMD_K8_XCPT_BP_RIP_MASK?", 0, 0, 0),
+    MVX(0xc0011016, "AMD_K8_COND_HDT_VAL?", 0, 0, 0),
+    MVX(0xc0011017, "AMD_K8_COND_HDT_VAL_MASK?", 0, 0, 0),
+    MVX(0xc0011018, "AMD_K8_XCPT_BP_CTL?", 0, 0, 0),
+    RSN(0xc0011019, 0xc001101a, "AMD_16H_DR1_ADDR_MASn", AmdK7DrXAddrMaskN, AmdK7DrXAddrMaskN, 0x1, ~(uint64_t)UINT32_MAX, 0),
+    MFX(0xc001101b, "AMD_16H_DR3_ADDR_MASK", AmdK7DrXAddrMaskN, AmdK7DrXAddrMaskN, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0011020, "AMD_K7_LS_CFG", AmdK7LoadStoreCfg, AmdK7LoadStoreCfg, 0, UINT64_C(0x7c000fffc000), 0), /* value=0x2068000`00000000 */
+    MFX(0xc0011021, "AMD_K7_IC_CFG", AmdK7InstrCacheCfg, AmdK7InstrCacheCfg, 0x201000, 0, 0), /* value=0x201000 */
+    MFX(0xc0011022, "AMD_K7_DC_CFG", AmdK7DataCacheCfg, AmdK7DataCacheCfg, 0, UINT64_C(0xfffffffff000000), 0), /* value=0x500000 */
+    MFN(0xc0011023, "AMD_K7_BU_CFG", AmdK7BusUnitCfg, AmdK7BusUnitCfg), /* Villain? value=0x20000`00000000 */
+    MFX(0xc0011024, "AMD_K7_DEBUG_CTL_2?", AmdK7DebugCtl2Maybe, AmdK7DebugCtl2Maybe, 0, UINT64_C(0xfffffffffffffffc), 0), /* value=0x0 */
+    MFN(0xc0011025, "AMD_K7_DR0_DATA_MATCH?", AmdK7Dr0DataMatchMaybe, AmdK7Dr0DataMatchMaybe), /* value=0x0 */
+    MFN(0xc0011026, "AMD_K7_DR0_DATA_MATCH?", AmdK7Dr0DataMaskMaybe, AmdK7Dr0DataMaskMaybe), /* value=0x0 */
+    MFX(0xc0011027, "AMD_K7_DR0_ADDR_MASK", AmdK7DrXAddrMaskN, AmdK7DrXAddrMaskN, 0x0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0011028, "AMD_15H_FP_CFG", AmdFam15hFpuCfg, AmdFam15hFpuCfg, 0, UINT64_C(0xffa0f0fc0004fc00), 0), /* value=0x140a00`248000d4 */
+    MFX(0xc0011029, "AMD_15H_DC_CFG", AmdFam15hDecoderCfg, AmdFam15hDecoderCfg, 0, 0x18001, 0), /* value=0x10e26002 */
+    MFN(0xc001102a, "AMD_10H_BU_CFG2", AmdFam10hBusUnitCfg2, AmdFam10hBusUnitCfg2), /* value=0x6800000`00028080 */
+    MVX(0xc001102b, "TODO_c001_102b", 0x1808cc17, 0, 0),
+    MVI(0xc001102c, "TODO_c001_102c", UINT64_C(0x68000000000000)), /* Villain? */
+    MVX(0xc001102d, "TODO_c001_102d", UINT64_C(0x1000000500000020), UINT64_C(0x1ffc0ffe00000), 0),
+    MVX(0xc001102e, "TODO_c001_102e", 0x1, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001102f, "TODO_c001_102f", 0, UINT64_C(0xffff000000000000), 0),
+    MFX(0xc0011030, "AMD_10H_IBS_FETCH_CTL", AmdFam10hIbsFetchCtl, AmdFam10hIbsFetchCtl, 0, UINT64_C(0xfdfeffffffff0000), 0), /* value=0x0 */
+    MFX(0xc0011031, "AMD_10H_IBS_FETCH_LIN_ADDR", AmdFam10hIbsFetchLinAddr, AmdFam10hIbsFetchLinAddr, 0, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0011032, "AMD_10H_IBS_FETCH_PHYS_ADDR", AmdFam10hIbsFetchPhysAddr, AmdFam10hIbsFetchPhysAddr, 0, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFX(0xc0011033, "AMD_10H_IBS_OP_EXEC_CTL", AmdFam10hIbsOpExecCtl, AmdFam10hIbsOpExecCtl, 0, UINT64_C(0xf8000000f8010000), 0), /* value=0x0 */
+    MFN(0xc0011034, "AMD_10H_IBS_OP_RIP", AmdFam10hIbsOpRip, AmdFam10hIbsOpRip), /* value=0x0 */
+    MFX(0xc0011035, "AMD_10H_IBS_OP_DATA", AmdFam10hIbsOpData, AmdFam10hIbsOpData, 0, UINT64_C(0xfffffe0300000000), 0), /* value=0x0 */
+    MFX(0xc0011036, "AMD_10H_IBS_OP_DATA2", AmdFam10hIbsOpData2, AmdFam10hIbsOpData2, 0, UINT64_C(0xffffffffffffffc8), 0), /* value=0x0 */
+    MFX(0xc0011037, "AMD_10H_IBS_OP_DATA3", AmdFam10hIbsOpData3, AmdFam10hIbsOpData3, 0, 0x1e00, 0), /* value=0x0 */
+    MFN(0xc0011038, "AMD_10H_IBS_DC_LIN_ADDR", AmdFam10hIbsDcLinAddr, AmdFam10hIbsDcLinAddr), /* value=0x0 */
+    MFX(0xc0011039, "AMD_10H_IBS_DC_PHYS_ADDR", AmdFam10hIbsDcPhysAddr, AmdFam10hIbsDcPhysAddr, 0, UINT64_C(0xffff000000000000), 0), /* value=0x0 */
+    MFO(0xc001103a, "AMD_10H_IBS_CTL", AmdFam10hIbsCtl), /* value=0x100 */
+    MFN(0xc001103b, "AMD_14H_IBS_BR_TARGET", AmdFam14hIbsBrTarget, AmdFam14hIbsBrTarget), /* value=0x0 */
+    MVI(0xc001103c, "TODO_c001_103c", 0),
+    MVX(0xc0011041, "AMD_15H_UNK_c001_1041", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0xc0011042, "AMD_15H_UNK_c001_1042", 0),
+    MVX(0xc0011074, "TODO_c001_1074", UINT64_C(0x8000000000000000), UINT64_C(0x8fffffffffffffff), 0),
+    MVX(0xc0011075, "TODO_c001_1075", 0, UINT64_C(0xfffffffff0000000), 0),
+    MVX(0xc0011076, "TODO_c001_1076", 0x14, UINT64_C(0xffffffffffffffe0), 0),
+    MVI(0xc0011077, "TODO_c001_1077", UINT64_C(0xd400005d5150595b)),
+    MVI(0xc0011078, "TODO_c001_1078", 0),
+    MVI(0xc0011083, "TODO_c001_1083", UINT64_C(0x1bc6f1bc1bc6f1bc)),
+    MVX(0xc0011093, "TODO_c001_1093", 0xe860e0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011094, "TODO_c001_1094", 0x11fd, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011095, "TODO_c001_1095", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011096, "TODO_c001_1096", 0, UINT64_C(0xffffffff87ff0000), 0),
+    MVX(0xc0011097, "TODO_c001_1097", 0xff6, UINT64_C(0xffffffffffffc000), 0),
+    MVO(0xc00110a2, "TODO_c001_10a2", UINT32_C(0xfeb00000)),
+    MFN(0xc00110e0, "TODO_c001_10e0", WriteOnly, IgnoreWrite),
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Hygon C86 7185 32-core Processor.
+ */
+static CPUMDBENTRY const g_Entry_Hygon_C86_7185_32_core =
+{
+    /*.pszName          = */ "Hygon C86 7185 32-core",
+    /*.pszFullName      = */ "Hygon C86 7185 32-core Processor",
+    /*.enmVendor        = */ CPUMCPUVENDOR_HYGON,
+    /*.uFamily          = */ 24,
+    /*.uModel           = */ 0,
+    /*.uStepping        = */ 1,
+    /*.enmMicroarch     = */ kCpumMicroarch_Hygon_Dhyana,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 48,
+    /*.fMxCsrMask       = */ 0x0002ffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Hygon_C86_7185_32_core),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Hygon_C86_7185_32_core)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Hygon_C86_7185_32_core)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Hygon_C86_7185_32_core),
+};
+
+#endif /* !VBOX_CPUDB_Hygon_C86_7185_32_core_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_80186.h b/src/VBox/VMM/VMMR3/cpus/Intel_80186.h
new file mode 100644
index 00000000..2390b885
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_80186.h
@@ -0,0 +1,75 @@
+/* $Id: Intel_80186.h $ */
+/** @file
+ * CPU database entry "Intel 80186".
+ * Handcrafted.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_80186_h
+#define VBOX_CPUDB_Intel_80186_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * Fake CPUID leaves for Intel(R) 80186.
+ *
+ * We fake these to keep the CPUM ignorant of CPUs wihtout CPUID leaves
+ * and avoid having to seed CPUM::GuestFeatures filling with bits from the
+ * CPUMDBENTRY.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_80186[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00000100, 0x00000100, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x65746e49, 0x2952286c, 0x31303820, 0x20203638, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x20202020, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x20202020, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00001414, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+/**
+ * Database entry for Intel(R) 80186.
+ */
+static CPUMDBENTRY const g_Entry_Intel_80186 =
+{
+    /*.pszName          = */ "Intel 80186",
+    /*.pszFullName      = */ "Intel(R) 80186",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 2,
+    /*.uModel           = */ 0,
+    /*.uStepping        = */ 0,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_80186,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ CPUDB_F_EXECUTE_ALL_IN_IEM,
+    /*.cMaxPhysAddrWidth= */ 20,
+    /*.fMxCsrMask       = */ 0,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_80186),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_80186)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ 0,
+    /*.cMsrRanges       = */ 0,
+    /*.paMsrRanges      = */ NULL,
+};
+
+#endif /* !VBOX_CPUDB_Intel_80186_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_80286.h b/src/VBox/VMM/VMMR3/cpus/Intel_80286.h
new file mode 100644
index 00000000..d39a33cc
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_80286.h
@@ -0,0 +1,75 @@
+/* $Id: Intel_80286.h $ */
+/** @file
+ * CPU database entry "Intel 80286".
+ * Handcrafted.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_80286_h
+#define VBOX_CPUDB_Intel_80286_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * Fake CPUID leaves for Intel(R) 80286.
+ *
+ * We fake these to keep the CPUM ignorant of CPUs wihtout CPUID leaves
+ * and avoid having to seed CPUM::GuestFeatures filling with bits from the
+ * CPUMDBENTRY.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_80286[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00000200, 0x00000100, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x65746e49, 0x2952286c, 0x32303820, 0x20203638, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x20202020, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x20202020, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00001818, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+/**
+ * Database entry for Intel(R) 80286.
+ */
+static CPUMDBENTRY const g_Entry_Intel_80286 =
+{
+    /*.pszName          = */ "Intel 80286",
+    /*.pszFullName      = */ "Intel(R) 80286",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 2,
+    /*.uModel           = */ 0,
+    /*.uStepping        = */ 0,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_80286,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ CPUDB_F_EXECUTE_ALL_IN_IEM,
+    /*.cMaxPhysAddrWidth= */ 24,
+    /*.fMxCsrMask       = */ 0,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_80286),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_80286)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ 0,
+    /*.cMsrRanges       = */ 0,
+    /*.paMsrRanges      = */ NULL,
+};
+
+#endif /* !VBOX_CPUDB_Intel_80286_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_80386.h b/src/VBox/VMM/VMMR3/cpus/Intel_80386.h
new file mode 100644
index 00000000..e2cf52cc
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_80386.h
@@ -0,0 +1,75 @@
+/* $Id: Intel_80386.h $ */
+/** @file
+ * CPU database entry "Intel 80386".
+ * Handcrafted.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_80386_h
+#define VBOX_CPUDB_Intel_80386_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * Fake CPUID leaves for Intel(R) 80386.
+ *
+ * We fake these to keep the CPUM ignorant of CPUs withou CPUID leaves
+ * and avoid having to seed CPUM::GuestFeatures filling with bits from the
+ * CPUMDBENTRY.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_80386[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00000300, 0x00000100, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x65746e49, 0x2952286c, 0x33303820, 0x20203638, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x20202020, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x20202020, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00001818, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+/**
+ * Database entry for Intel(R) 80386.
+ */
+static CPUMDBENTRY const g_Entry_Intel_80386 =
+{
+    /*.pszName          = */ "Intel 80386",
+    /*.pszFullName      = */ "Intel(R) 80386",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 3,
+    /*.uModel           = */ 0,
+    /*.uStepping        = */ 0,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_80386,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ CPUDB_F_EXECUTE_ALL_IN_IEM,
+    /*.cMaxPhysAddrWidth= */ 24,
+    /*.fMxCsrMask       = */ 0,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_80386),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_80386)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ 0,
+    /*.cMsrRanges       = */ 0,
+    /*.paMsrRanges      = */ NULL,
+};
+
+#endif /* !VBOX_CPUDB_Intel_80386_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_80486.h b/src/VBox/VMM/VMMR3/cpus/Intel_80486.h
new file mode 100644
index 00000000..0266e548
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_80486.h
@@ -0,0 +1,73 @@
+/* $Id: Intel_80486.h $ */
+/** @file
+ * CPU database entry "Intel 80486".
+ * Handcrafted.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_80486_h
+#define VBOX_CPUDB_Intel_80486_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * Fake CPUID leaves for Intel(R) 80486(DX2).
+ *
+ * The extended leaves are fake to make CPUM happy.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_80486[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00000430, 0x00000100, 0x00000000, 0x00000111, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x65746e49, 0x2952286c, 0x34303820, 0x58443638, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x20202032, 0x20202020, 0x20202020, 0x20202020, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x20202020, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00002020, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+/**
+ * Database entry for Intel(R) 80486.
+ */
+static CPUMDBENTRY const g_Entry_Intel_80486 =
+{
+    /*.pszName          = */ "Intel 80486",
+    /*.pszFullName      = */ "Intel(R) 80486DX2",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 4,
+    /*.uModel           = */ 3,
+    /*.uStepping        = */ 0,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_80486,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 32,
+    /*.fMxCsrMask       = */ 0,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_80486),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_80486)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ 0,
+    /*.cMsrRanges       = */ 0,
+    /*.paMsrRanges      = */ NULL,
+};
+
+#endif /* !VBOX_CPUDB_Intel_80486_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_8086.h b/src/VBox/VMM/VMMR3/cpus/Intel_8086.h
new file mode 100644
index 00000000..30f69032
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_8086.h
@@ -0,0 +1,75 @@
+/* $Id: Intel_8086.h $ */
+/** @file
+ * CPU database entry "Intel 8086".
+ * Handcrafted.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_8086_h
+#define VBOX_CPUDB_Intel_8086_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * Fake CPUID leaves for Intel(R) 8086.
+ *
+ * We fake these to keep the CPUM ignorant of CPUs wihtout CPUID leaves
+ * and avoid having to seed CPUM::GuestFeatures filling with bits from the
+ * CPUMDBENTRY.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_8086[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00000000, 0x00000100, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x65746e49, 0x2952286c, 0x38303820, 0x20202036, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x20202020, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x20202020, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00001414, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+/**
+ * Database entry for Intel(R) 8086.
+ */
+static CPUMDBENTRY const g_Entry_Intel_8086 =
+{
+    /*.pszName          = */ "Intel 8086",
+    /*.pszFullName      = */ "Intel(R) 8086",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 2,
+    /*.uModel           = */ 0,
+    /*.uStepping        = */ 0,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_8086,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ CPUDB_F_EXECUTE_ALL_IN_IEM,
+    /*.cMaxPhysAddrWidth= */ 20,
+    /*.fMxCsrMask       = */ 0,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_8086),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_8086)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ 0,
+    /*.cMsrRanges       = */ 0,
+    /*.paMsrRanges      = */ NULL,
+};
+
+#endif /* !VBOX_CPUDB_Intel_8086_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Atom_330_1_60GHz.h b/src/VBox/VMM/VMMR3/cpus/Intel_Atom_330_1_60GHz.h
new file mode 100644
index 00000000..dc6eab62
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Atom_330_1_60GHz.h
@@ -0,0 +1,210 @@
+/* $Id: Intel_Atom_330_1_60GHz.h $ */
+/** @file
+ * CPU database entry "Intel Atom 330 1.60GHz".
+ * Generated at 2015-11-04T12:58:59Z by VBoxCpuReport v5.0.51r103818 on linux.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Atom_330_1_60GHz_h
+#define VBOX_CPUDB_Intel_Atom_330_1_60GHz_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Atom(TM) CPU  330   @ 1.60GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Atom_330_1_60GHz[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000a, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000106c2, 0x01040800, 0x0040e31d, 0xbfe9fbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x4fba5901, 0x0e3080c0, 0x00000000, 0x00000000, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, UINT32_MAX, 0x04004121, 0x0140003f, 0x0000003f, 0x00000001, 0 },
+    { 0x00000004, 0x00000001, UINT32_MAX, 0x04004122, 0x01c0003f, 0x0000003f, 0x00000001, 0 },
+    { 0x00000004, 0x00000002, UINT32_MAX, 0x04004143, 0x01c0003f, 0x000003ff, 0x00000001, 0 },
+    { 0x00000004, 0x00000003, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00000010, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000001, 0x00000002, 0x00000001, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07280203, 0x00000000, 0x00000000, 0x00002501, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x20100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x746e4920, 0x52286c65, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x74412029, 0x54286d6f, 0x4320294d, 0x20205550, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x20303333, 0x20402020, 0x30362e31, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x02008040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003020, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Atom(TM) CPU  330   @ 1.60GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Atom_330_1_60GHz[] =
+{
+    MFI(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr), /* value=0x0 */
+    MFX(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType, Ia32P5McType, 0, 0, UINT64_MAX), /* value=0x0 */
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, 0, UINT64_C(0xffffffffffff0000)), /* value=0x40 */
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x5a7`e94bd2c0 */
+    MFX(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0xc00008836ac1b), 0, 0), /* value=0xc0000`8836ac1b */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0, UINT64_C(0xffffffff000006ff)),
+    MVX(0x00000033, "TEST_CTL", 0, 0, UINT64_C(0xffffffff7fffffff)),
+    MVO(0x00000039, "C2_UNK_0000_0039", 0x1),
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x1 */
+    MVO(0x0000003f, "P6_UNK_0000_003f", 0),
+    RFN(0x00000040, 0x00000047, "MSR_LASTBRANCH_n_FROM_IP", IntelLastBranchToN, IntelLastBranchToN),
+    RFN(0x00000060, 0x00000067, "MSR_LASTBRANCH_n_TO_IP", IntelLastBranchFromN, IntelLastBranchFromN),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, IgnoreWrite),
+    MFX(0x0000008b, "BBL_CR_D3|BIOS_SIGN", Ia32BiosSignId, Ia32BiosSignId, 0, 0, UINT32_MAX), /* value=0x20d`00000000 */
+    RSN(0x000000c1, 0x000000c2, "IA32_PMCn", Ia32PmcN, Ia32PmcN, 0x0, ~(uint64_t)UINT32_MAX, 0),
+    MFX(0x000000c7, "IA32_PMC6", Ia32PmcN, Ia32PmcN, 0, UINT64_C(0xfff7bdefff7df7df), 0), /* value=0x16101c00`00000000 */
+    MFX(0x000000cd, "MSR_FSB_FREQ", IntelP6FsbFrequency, ReadOnly, 0x101, 0, 0), /* value=0x101 */
+    MVO(0x000000ce, "IA32_PLATFORM_INFO", UINT64_C(0x1b1b0c004e4e0000)),
+    MVO(0x000000cf, "C2_UNK_0000_00cf", 0x1f),
+    MVO(0x000000e0, "C2_UNK_0000_00e0", 0x6800f0),
+    MVO(0x000000e1, "C2_UNK_0000_00e1", UINT32_C(0xf0f00000)),
+    MFX(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl, IntelPkgCStConfigControl, 0, 0xbfff, UINT64_C(0xfffffffffc804000)), /* value=0x26b001 */
+    MFX(0x000000e3, "C2_SMM_CST_MISC_INFO", IntelCore2SmmCStMiscInfo, IntelCore2SmmCStMiscInfo, 0, 0, ~(uint64_t)UINT32_MAX), /* value=0x0 */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, 0, UINT64_C(0xffffffffff800000)), /* value=0x0 */
+    MVO(0x000000e5, "C2_UNK_0000_00e5", UINT32_C(0xd00a00f8)),
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf), /* value=0x63`19743600 */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x63`199424b8 */
+    MFX(0x000000ee, "C1_EXT_CONFIG", IntelCore1ExtConfig, IntelCore1ExtConfig, 0, UINT64_C(0xff7bdeffffc5ffff), 0), /* value=0x3384103 */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x508, 0, 0), /* value=0x508 */
+    MVX(0x00000116, "BBL_CR_ADDR", 0x3fc0, UINT64_C(0xfffffff00000001f), 0),
+    MVX(0x00000118, "BBL_CR_DECC", 0, UINT64_C(0xfffc0000fffc0000), 0),
+    MFX(0x00000119, "BBL_CR_CTL", IntelBblCrCtl, IntelBblCrCtl, 0x938008, 0x4080017f, ~(uint64_t)UINT32_MAX), /* value=0x938008 */
+    MFN(0x0000011a, "BBL_CR_TRIG", WriteOnly, IgnoreWrite),
+    MVX(0x0000011b, "P6_UNK_0000_011b", 0, 0x1, UINT64_C(0xfffffffffffffffe)),
+    MVX(0x0000011c, "C2_UNK_0000_011c", 0xd96000, 0, UINT64_C(0xfffffffff0000000)),
+    MFX(0x0000011e, "BBL_CR_CTL3", IntelBblCrCtl3, IntelBblCrCtl3, 0x7f00011f, UINT32_C(0xff83f81f), UINT64_C(0xffffffff007c06e0)), /* value=0x7f00011f */
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x10 */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0x0 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0xffffffff`81573970 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x805, 0, 0), /* value=0x805 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, 0, UINT64_MAX), /* value=0x0 */
+    RSN(0x00000186, 0x00000187, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, 0, ~(uint64_t)UINT32_MAX),
+    MFX(0x00000194, "CLOCK_FLEX_MAX", IntelFlexRatio, IntelFlexRatio, 0, UINT32_C(0xfffee0c0), ~(uint64_t)UINT32_MAX), /* value=0x0 */
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0xc1b0c1b06000c1b), 0, 0), /* value=0xc1b0c1b`06000c1b */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0xc1b, 0, 0), /* Might bite. value=0xc1b */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0x2, 0, UINT64_C(0xffffffffffffffe1)), /* value=0x2 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0x3, 0, UINT64_C(0xffffffffff0000e0)), /* value=0x3 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, UINT32_C(0x884c0000), UINT32_C(0xf87f03ff), UINT64_C(0xffffffff0780fc00)), /* value=0x884c0000 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0x61b, 0, 0), /* value=0x61b */
+    MVX(0x0000019e, "P6_UNK_0000_019e", 0, UINT32_C(0xffff0000), ~(uint64_t)UINT32_MAX),
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, 0x60940488, UINT64_C(0x366131884), UINT64_C(0xfffffff89908c372)), /* value=0x60940488 */
+    MVX(0x000001aa, "P6_PIC_SENS_CFG", UINT32_C(0x800f0421), UINT64_C(0xffffffffff80000e), 0),
+    MFX(0x000001c9, "MSR_LASTBRANCH_TOS", IntelLastBranchTos, IntelLastBranchTos, 0, 0, UINT64_C(0xfffffffffffffff8)), /* value=0x0 */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffffe03c)), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0xffffffff`a07ac16e */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xffffffff`8105c4f0 */
+    MFN(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp, P6LastIntFromIp), /* value=0x0 */
+    MFN(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp, P6LastIntToIp), /* value=0x0 */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xffffffff00000ff8)), /* value=0xe0000000 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xe0000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xffffffff00000ff8)), /* value=0x6 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xffffffff00000ff8)), /* value=0x0 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xffffffff00000ff8)), /* value=0x0 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xffffffff00000ff8)), /* value=0x0 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xffffffff00000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xffffffff00000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xffffffff00000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MVX(0x000002e0, "I7_SB_NO_EVICT_MODE", 0, 0, UINT64_C(0xffffffff7ffffffc)),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    MFX(0x00000309, "IA32_FIXED_CTR0", Ia32FixedCtrN, Ia32FixedCtrN, 0x0, 0, UINT64_C(0xffffff0000000000)), /* value=0x8c */
+    MFX(0x0000030a, "IA32_FIXED_CTR1", Ia32FixedCtrN, Ia32FixedCtrN, 0x1, 0x81201, UINT64_C(0xffffff0000000000)), /* value=0xff`ad893763 */
+    MFX(0x0000030b, "IA32_FIXED_CTR2", Ia32FixedCtrN, Ia32FixedCtrN, 0x2, 0, UINT64_C(0xffffff0000000000)), /* value=0x8f4 */
+    MFX(0x00000345, "IA32_PERF_CAPABILITIES", Ia32PerfCapabilities, ReadOnly, 0xc1, 0, 0), /* value=0xc1 */
+    MFX(0x0000038d, "IA32_FIXED_CTR_CTRL", Ia32FixedCtrCtrl, Ia32FixedCtrCtrl, 0, 0, UINT64_C(0xfffffffffffff000)), /* value=0xb0 */
+    MFX(0x0000038e, "IA32_PERF_GLOBAL_STATUS", Ia32PerfGlobalStatus, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x0000038f, "IA32_PERF_GLOBAL_CTRL", Ia32PerfGlobalCtrl, Ia32PerfGlobalCtrl, 0, 0, UINT64_C(0xfffffff8fffffffc)), /* value=0x7`00000003 */
+    MFX(0x00000390, "IA32_PERF_GLOBAL_OVF_CTRL", Ia32PerfGlobalOvfCtrl, Ia32PerfGlobalOvfCtrl, 0, UINT64_C(0xc000000700000003), UINT64_C(0x3ffffff8fffffffc)), /* value=0x0 */
+    MVX(0x000003ca, "TODO_0000_03ca", 0x10510, 0, UINT64_C(0xffffffffffe00000)),
+    MFX(0x000003f1, "IA32_PEBS_ENABLE", Ia32PebsEnable, Ia32PebsEnable, 0, 0, UINT64_C(0xfffffffffffffffe)), /* value=0x0 */
+    RFN(0x00000400, 0x00000417, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MVX(0x000004f8, "C2_UNK_0000_04f8", 0, 0, 0),
+    MVX(0x000004f9, "C2_UNK_0000_04f9", 0, 0, 0),
+    MVX(0x000004fa, "C2_UNK_0000_04fa", 0, 0, 0),
+    MVX(0x000004fb, "C2_UNK_0000_04fb", 0, 0, 0),
+    MVX(0x000004fc, "C2_UNK_0000_04fc", 0, 0, 0),
+    MVX(0x000004fd, "C2_UNK_0000_04fd", 0, 0, 0),
+    MVX(0x000004fe, "C2_UNK_0000_04fe", 0, 0, 0),
+    MVX(0x000004ff, "C2_UNK_0000_04ff", 0, 0, 0),
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0xffff8800`d6ee1c00 */
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xfffffffffffff2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xffffffff`815715d0 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xffffffff`81573ad0 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x47700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x7fe4`93136740 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffff8800`db500000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x0 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Atom(TM) CPU  330   @ 1.60GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Atom_330_1_60GHz =
+{
+    /*.pszName          = */ "Intel Atom 330 1.60GHz",
+    /*.pszFullName      = */ "Intel(R) Atom(TM) CPU  330   @ 1.60GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 28,
+    /*.uStepping        = */ 2,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_Atom_Bonnell,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_133MHZ,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 32,
+    /*.fMxCsrMask       = */ 0xffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Atom_330_1_60GHz),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Atom_330_1_60GHz)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF,
+    /*.DefUnknownCpuId  = */ { 0x07280203, 0x00000000, 0x00000000, 0x00002501 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Atom_330_1_60GHz)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Atom_330_1_60GHz),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Atom_330_1_60GHz_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Core2_T7600_2_33GHz.h b/src/VBox/VMM/VMMR3/cpus/Intel_Core2_T7600_2_33GHz.h
new file mode 100644
index 00000000..3a85dc7e
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Core2_T7600_2_33GHz.h
@@ -0,0 +1,195 @@
+/* $Id: Intel_Core2_T7600_2_33GHz.h $ */
+/** @file
+ * CPU database entry "Intel Core2 T7600 2.33GHz".
+ * Generated at 2017-10-12T18:17:56Z by VBoxCpuReport v5.2.0_RC1r118339 on linux.x86.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Core2_T7600_2_33GHz_h
+#define VBOX_CPUDB_Intel_Core2_T7600_2_33GHz_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Core(TM)2 CPU         T7600  @ 2.33GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Core2_T7600_2_33GHz[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000a, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000006f6, 0x00020800, 0x0000e3bd, 0xbfebfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x05b0b101, 0x005657f0, 0x00000000, 0x2cb43049, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, UINT32_MAX, 0x04000121, 0x01c0003f, 0x0000003f, 0x00000001, 0 },
+    { 0x00000004, 0x00000001, UINT32_MAX, 0x04000122, 0x01c0003f, 0x0000003f, 0x00000001, 0 },
+    { 0x00000004, 0x00000002, UINT32_MAX, 0x04004143, 0x03c0003f, 0x00000fff, 0x00000001, 0 },
+    { 0x00000004, 0x00000003, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00022220, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000001, 0x00000002, 0x00000001, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000400, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07280202, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x20100000, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x65746e49, 0x2952286c, 0x726f4320, 0x4d542865, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x43203229, 0x20205550, 0x20202020, 0x54202020, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x30303637, 0x20402020, 0x33332e32, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x10008040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003024, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Core(TM)2 CPU         T7600  @ 2.33GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Core2_T7600_2_33GHz[] =
+{
+    MFO(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr), /* value=0x12c5e80 */
+    MFO(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType), /* value=0x0 */
+    MFO(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize), /* value=0x40 */
+    MFO(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter), /* value=0x215`a3e44b5c */
+    MFX(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0x14000098548e25), 0, 0), /* value=0x140000`98548e25 */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00900), 0, UINT64_C(0xfffffffffffff7ff)),
+    MVO(0x00000021, "C2_UNK_0000_0021", 0),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0x41880000, 0, 0), /* value=0x41880000 */
+    MVO(0x0000002f, "P6_UNK_0000_002f", 0),
+    MVO(0x00000032, "P6_UNK_0000_0032", 0),
+    MVO(0x00000033, "TEST_CTL", 0),
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x5 */
+    MVO(0x0000003f, "P6_UNK_0000_003f", 0),
+    RFN(0x00000040, 0x00000043, "MSR_LASTBRANCH_n_FROM_IP", IntelLastBranchToN, ReadOnly),
+    MVO(0x0000004a, "P6_UNK_0000_004a", 0), /* value=0x0 */
+    MVO(0x0000004b, "P6_UNK_0000_004b", 0), /* value=0x0 */
+    MVO(0x0000004c, "P6_UNK_0000_004c", 0), /* value=0x0 */
+    MVO(0x0000004d, "P6_UNK_0000_004d", 0), /* value=0x3c3a9b64`1d8552bb */
+    MVO(0x0000004e, "P6_UNK_0000_004e", 0), /* value=0x3b96f62f`156143b9 */
+    MVO(0x0000004f, "P6_UNK_0000_004f", 0), /* value=0xb8 */
+    RFN(0x00000060, 0x00000063, "MSR_LASTBRANCH_n_TO_IP", IntelLastBranchFromN, ReadOnly),
+    MVO(0x0000006c, "P6_UNK_0000_006c", 0),
+    MVO(0x0000006d, "P6_UNK_0000_006d", 0),
+    MVO(0x0000006e, "P6_UNK_0000_006e", 0),
+    MVO(0x0000006f, "P6_UNK_0000_006f", 0xadb),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, IgnoreWrite),
+    MFO(0x0000008b, "BBL_CR_D3|BIOS_SIGN", Ia32BiosSignId), /* value=0xc7`00000000 */
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    MFX(0x000000a8, "C2_EMTTM_CR_TABLES_0", IntelCore2EmttmCrTablesN, ReadOnly, 0x613, 0, 0), /* value=0x613 */
+    MFX(0x000000a9, "C2_EMTTM_CR_TABLES_1", IntelCore2EmttmCrTablesN, ReadOnly, 0x613, 0, 0), /* value=0x613 */
+    MFX(0x000000aa, "C2_EMTTM_CR_TABLES_2", IntelCore2EmttmCrTablesN, ReadOnly, 0x613, 0, 0), /* value=0x613 */
+    MFX(0x000000ab, "C2_EMTTM_CR_TABLES_3", IntelCore2EmttmCrTablesN, ReadOnly, 0x613, 0, 0), /* value=0x613 */
+    MFX(0x000000ac, "C2_EMTTM_CR_TABLES_4", IntelCore2EmttmCrTablesN, ReadOnly, 0x613, 0, 0), /* value=0x613 */
+    MFX(0x000000ad, "C2_EMTTM_CR_TABLES_5", IntelCore2EmttmCrTablesN, ReadOnly, 0x613, 0, 0), /* value=0x613 */
+    RFN(0x000000c1, 0x000000c2, "IA32_PMCn", Ia32PmcN, ReadOnly),
+    MVO(0x000000c7, "P6_UNK_0000_00c7", UINT64_C(0x1e00000042000000)),
+    MFX(0x000000cd, "MSR_FSB_FREQ", IntelP6FsbFrequency, ReadOnly, 0x933, 0, 0), /* value=0x933 */
+    MVO(0x000000ce, "P6_UNK_0000_00ce", UINT64_C(0x130e253b530613)),
+    MVO(0x000000e0, "C2_UNK_0000_00e0", 0x14860f0),
+    MVO(0x000000e1, "C2_UNK_0000_00e1", UINT32_C(0xf0f00000)),
+    MFX(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl, IntelPkgCStConfigControl, 0, 0x404000, UINT64_C(0xfffffffffc001000)), /* value=0x202a01 */
+    MFO(0x000000e3, "C2_SMM_CST_MISC_INFO", IntelCore2SmmCStMiscInfo), /* value=0x8040414 */
+    MFO(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase), /* value=0x20414 */
+    MVO(0x000000e5, "C2_UNK_0000_00e5", UINT32_C(0xd0220dc8)),
+    MFO(0x000000e7, "IA32_MPERF", Ia32MPerf), /* value=0xc7`b82ef32a */
+    MFO(0x000000e8, "IA32_APERF", Ia32APerf), /* value=0x55`9818510c */
+    MFO(0x000000ee, "C1_EXT_CONFIG", IntelCore1ExtConfig), /* value=0x80b90400 */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x508, 0, 0), /* value=0x508 */
+    MVO(0x00000116, "BBL_CR_ADDR", 0),
+    MVO(0x00000118, "BBL_CR_DECC", 0xffebe),
+    MVO(0x0000011b, "P6_UNK_0000_011b", 0),
+    MVO(0x0000011c, "C2_UNK_0000_011c", UINT32_C(0xe00000cc)),
+    MFX(0x0000011e, "BBL_CR_CTL3", IntelBblCrCtl3, ReadOnly, 0x74702109, 0, 0), /* value=0x74702109 */
+    MVO(0x0000014a, "TODO_0000_014a", 0),
+    MVO(0x0000014b, "TODO_0000_014b", 0),
+    MVO(0x0000014c, "TODO_0000_014c", 0),
+    MVO(0x0000014e, "P6_UNK_0000_014e", UINT32_C(0xe4dfe927)),
+    MVO(0x0000014f, "P6_UNK_0000_014f", 0),
+    MVO(0x00000151, "P6_UNK_0000_0151", 0x3bfcb56f),
+    MFO(0x0000015f, "C1_DTS_CAL_CTRL", IntelCore1DtsCalControl), /* value=0x230613 */
+    MFO(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs), /* value=0x60 */
+    MFO(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp), /* value=0xf5a07c40 */
+    MFO(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip), /* value=0xc15af09c */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x806, 0, 0), /* value=0x806 */
+    MFO(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus), /* value=0x0 */
+    RFN(0x00000186, 0x00000187, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, ReadOnly),
+    MVO(0x00000193, "C2_UNK_0000_0193", 0),
+    MFX(0x00000194, "CLOCK_FLEX_MAX", IntelFlexRatio, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0x6130e2506040613), 0, 0), /* value=0x6130e25`06040613 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, ReadOnly, 0x613, 0, UINT64_MAX), /* Might bite. value=0x613 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, ReadOnly, 0x2, 0, 0), /* value=0x2 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, ReadOnly, 0x3, 0, 0), /* value=0x3 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, ReadOnly, UINT32_C(0x8831000c), 0, 0), /* value=0x8831000c */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0x613, 0, 0), /* value=0x613 */
+    MVO(0x0000019e, "P6_UNK_0000_019e", 0xb240000),
+    MVO(0x0000019f, "P6_UNK_0000_019f", 0),
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, UINT64_C(0x4066a52489), UINT64_C(0x52600099f6), UINT64_C(0xffffff0019004000)), /* value=0x40`66a52489 */
+    MVO(0x000001a1, "P6_UNK_0000_01a1", 0),
+    MFX(0x000001a2, "I7_MSR_TEMPERATURE_TARGET", IntelI7TemperatureTarget, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MVO(0x000001aa, "P6_PIC_SENS_CFG", 0x5ebf042f),
+    MVO(0x000001bf, "C2_UNK_0000_01bf", 0x404),
+    MFO(0x000001c9, "MSR_LASTBRANCH_TOS", IntelLastBranchTos), /* value=0x3 */
+    MVO(0x000001d3, "P6_UNK_0000_01d3", 0x8000),
+    MFO(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl), /* value=0x1 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0xc12c5d73 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xc10357d0 */
+    MFO(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp), /* value=0xc132a284 */
+    MFO(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp), /* value=0xc1329543 */
+    MVO(0x000001e0, "MSR_ROB_CR_BKUPTMPDR6", 0xff0),
+    MFO(0x000001f8, "IA32_PLATFORM_DCA_CAP", Ia32PlatformDcaCap), /* value=0x0 */
+    MFO(0x000001f9, "IA32_CPU_DCA_CAP", Ia32CpuDcaCap), /* value=0x0 */
+    MFO(0x000001fa, "IA32_DCA_0_CAP", Ia32Dca0Cap), /* value=0xc01e488 */
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xfffffffffffff2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x1b0008`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xffffff80`0d2ce6c0 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0x0 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x4700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x0 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffffff82`0dcfd000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x7fff`7c7511e0 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Core(TM)2 CPU         T7600  @ 2.33GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Core2_T7600_2_33GHz =
+{
+    /*.pszName          = */ "Intel Core2 T7600 2.33GHz",
+    /*.pszFullName      = */ "Intel(R) Core(TM)2 CPU         T7600  @ 2.33GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 15,
+    /*.uStepping        = */ 6,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_Core2_Merom,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_167MHZ,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 36,
+    /*.fMxCsrMask       = */ 0x0000ffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Core2_T7600_2_33GHz),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Core2_T7600_2_33GHz)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF,
+    /*.DefUnknownCpuId  = */ { 0x07280202, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Core2_T7600_2_33GHz)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Core2_T7600_2_33GHz),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Core2_T7600_2_33GHz_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Core2_X6800_2_93GHz.h b/src/VBox/VMM/VMMR3/cpus/Intel_Core2_X6800_2_93GHz.h
new file mode 100644
index 00000000..a9aaa030
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Core2_X6800_2_93GHz.h
@@ -0,0 +1,260 @@
+/* $Id: Intel_Core2_X6800_2_93GHz.h $ */
+/** @file
+ * CPU database entry "Intel Core2 X6800 2.93GHz".
+ * Generated at 2017-11-04T22:32:41Z by VBoxCpuReport v5.2.1r118907 on linux.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Core2_X6800_2_93GHz_h
+#define VBOX_CPUDB_Intel_Core2_X6800_2_93GHz_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Core(TM)2 CPU         X6800  @ 2.93GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Core2_X6800_2_93GHz[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000a, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000006f6, 0x00020800, 0x0000e3bd, 0xbfebfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x05b0b101, 0x005657f0, 0x00000000, 0x2cb43049, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, UINT32_MAX, 0x04000121, 0x01c0003f, 0x0000003f, 0x00000001, 0 },
+    { 0x00000004, 0x00000001, UINT32_MAX, 0x04000122, 0x01c0003f, 0x0000003f, 0x00000001, 0 },
+    { 0x00000004, 0x00000002, UINT32_MAX, 0x04004143, 0x03c0003f, 0x00000fff, 0x00000001, 0 },
+    { 0x00000004, 0x00000003, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00000020, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000001, 0x00000002, 0x00000001, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000400, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07280202, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x20100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x65746e49, 0x2952286c, 0x726f4320, 0x4d542865, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x43203229, 0x20205550, 0x20202020, 0x58202020, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x30303836, 0x20402020, 0x33392e32, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x10008040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003024, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Core(TM)2 CPU         X6800  @ 2.93GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Core2_X6800_2_93GHz[] =
+{
+    MFX(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr, Ia32P5McAddr, 0, UINT64_C(0xfffffffffffbffff), 0), /* value=0x1398780 */
+    MFI(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType), /* value=0x0 */
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x40 */
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* Villain? value=0x11d1`f468a982 */
+    MFX(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT32_C(0x88040b27), 0, 0), /* value=0x88040b27 */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00900), 0x600, UINT64_C(0xfffffff0000000ff)),
+    MVX(0x00000021, "C2_UNK_0000_0021", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xffffffe0)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, IntelEblCrPowerOn, 0x41880000, UINT64_C(0xffffffffdff7ffbe), 0), /* value=0x41880000 */
+    MVI(0x0000002f, "P6_UNK_0000_002f", 0),
+    MVX(0x00000032, "P6_UNK_0000_0032", 0, UINT64_C(0xffffffff01fe0000), 0),
+    MVX(0x00000033, "TEST_CTL", 0, UINT64_C(0xffffffff7fffffff), 0),
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x5 */
+    MVO(0x0000003f, "P6_UNK_0000_003f", 0xea),
+    RFN(0x00000040, 0x00000043, "MSR_LASTBRANCH_n_FROM_IP", IntelLastBranchToN, IntelLastBranchToN),
+    MVX(0x0000004a, "P6_UNK_0000_004a", 0, UINT64_C(0xffffff0000000000), 0), /* value=0x0 */
+    MVX(0x0000004b, "P6_UNK_0000_004b", 0, UINT64_C(0xffffff0000000000), 0), /* value=0x0 */
+    MVX(0x0000004c, "P6_UNK_0000_004c", 0, UINT64_C(0xffffff0000000000), 0), /* value=0x0 */
+    MVX(0x0000004d, "P6_UNK_0000_004d", 0, 0, 0), /* value=0xf53ed6ff`f9f9e16e */
+    MVX(0x0000004e, "P6_UNK_0000_004e", 0, 0, 0), /* value=0xf7ffbdfb`bfbfabeb */
+    MVX(0x0000004f, "P6_UNK_0000_004f", 0, UINT64_C(0xffffffffffffff00), 0), /* value=0xff */
+    RFN(0x00000060, 0x00000063, "MSR_LASTBRANCH_n_TO_IP", IntelLastBranchFromN, IntelLastBranchFromN),
+    MVX(0x0000006c, "P6_UNK_0000_006c", 0, UINT64_C(0xffffffff00000080), 0),
+    MVX(0x0000006d, "P6_UNK_0000_006d", 0, UINT64_C(0xffffffff00000080), 0),
+    MVX(0x0000006e, "P6_UNK_0000_006e", 0, UINT64_C(0xffffffff00000080), 0),
+    MVO(0x0000006f, "P6_UNK_0000_006f", 0xadb),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, IgnoreWrite),
+    MFX(0x0000008b, "BBL_CR_D3|BIOS_SIGN", Ia32BiosSignId, Ia32BiosSignId, 0, UINT32_MAX, 0), /* value=0xc6`00000000 */
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    MFX(0x000000a8, "C2_EMTTM_CR_TABLES_0", IntelCore2EmttmCrTablesN, IntelCore2EmttmCrTablesN, 0x61b, UINT64_MAX, 0), /* value=0x61b */
+    MFX(0x000000a9, "C2_EMTTM_CR_TABLES_1", IntelCore2EmttmCrTablesN, IntelCore2EmttmCrTablesN, 0x61b, UINT64_MAX, 0), /* value=0x61b */
+    MFX(0x000000aa, "C2_EMTTM_CR_TABLES_2", IntelCore2EmttmCrTablesN, IntelCore2EmttmCrTablesN, 0x61b, UINT64_MAX, 0), /* value=0x61b */
+    MFX(0x000000ab, "C2_EMTTM_CR_TABLES_3", IntelCore2EmttmCrTablesN, IntelCore2EmttmCrTablesN, 0x61b, UINT64_MAX, 0), /* value=0x61b */
+    MFX(0x000000ac, "C2_EMTTM_CR_TABLES_4", IntelCore2EmttmCrTablesN, IntelCore2EmttmCrTablesN, 0x61b, UINT64_MAX, 0), /* value=0x61b */
+    MFX(0x000000ad, "C2_EMTTM_CR_TABLES_5", IntelCore2EmttmCrTablesN, IntelCore2EmttmCrTablesN, UINT32_C(0x8000061b), UINT64_MAX, 0), /* value=0x8000061b */
+    RSN(0x000000c1, 0x000000c2, "IA32_PMCn", Ia32PmcN, Ia32PmcN, 0x0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0x000000c7, "P6_UNK_0000_00c7", UINT64_C(0x3200000058000000)),
+    MFX(0x000000cd, "MSR_FSB_FREQ", IntelP6FsbFrequency, ReadOnly, 0x800, 0, 0), /* value=0x800 */
+    MVO(0x000000ce, "P6_UNK_0000_00ce", UINT64_C(0x1b0b277f7f071b)),
+    MVO(0x000000e0, "C2_UNK_0000_00e0", 0x7820f0),
+    MVO(0x000000e1, "C2_UNK_0000_00e1", UINT32_C(0xf0f00000)),
+    MFX(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl, IntelPkgCStConfigControl, 0, UINT64_C(0xffffffff0000ffff), UINT32_C(0xff000000)), /* value=0x26b204 */
+    MFX(0x000000e3, "C2_SMM_CST_MISC_INFO", IntelCore2SmmCStMiscInfo, IntelCore2SmmCStMiscInfo, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xff800000)), /* value=0x0 */
+    MVO(0x000000e5, "C2_UNK_0000_00e5", UINT32_C(0xd00201c8)),
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf), /* value=0xa0`16e07631 */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x7e`79c4e805 */
+    MFX(0x000000ee, "C1_EXT_CONFIG", IntelCore1ExtConfig, IntelCore1ExtConfig, 0, UINT64_C(0xffffffffefc5ffff), 0), /* value=0xa8000000`c17d4300 */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x508, 0, 0), /* value=0x508 */
+    MVX(0x00000116, "BBL_CR_ADDR", 0, UINT64_C(0xffffff000000001f), 0),
+    MVX(0x00000118, "BBL_CR_DECC", 0xffdfe, UINT64_C(0xfffffffffff00000), 0),
+    MFN(0x0000011a, "BBL_CR_TRIG", WriteOnly, IgnoreWrite),
+    MVI(0x0000011b, "P6_UNK_0000_011b", 0),
+    MVX(0x0000011c, "C2_UNK_0000_011c", UINT32_C(0xe003cf6f), UINT64_C(0xffffffff07f80000), 0),
+    MFX(0x0000011e, "BBL_CR_CTL3", IntelBblCrCtl3, IntelBblCrCtl3, UINT32_C(0xbf702109), UINT64_C(0xfffffffffff3fe9f), 0), /* value=0xbf702109 */
+    MVX(0x0000014a, "TODO_0000_014a", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x0000014b, "TODO_0000_014b", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x0000014c, "TODO_0000_014c", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x0000014e, "P6_UNK_0000_014e", 0x7ab9f777, UINT64_C(0xffffffff00000080), 0),
+    MVI(0x0000014f, "P6_UNK_0000_014f", 0xf000),
+    MVX(0x00000151, "P6_UNK_0000_0151", 0x42100400, ~(uint64_t)UINT32_MAX, 0),
+    MFX(0x0000015f, "C1_DTS_CAL_CTRL", IntelCore1DtsCalControl, IntelCore1DtsCalControl, 0, UINT64_C(0xffffffffffc0ffff), 0), /* value=0x230820 */
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x10 */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0x0 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0xffffffff`81846c20 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x6, 0, 0), /* value=0x6 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    RSN(0x00000186, 0x00000187, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, ~(uint64_t)UINT32_MAX, 0),
+    MVO(0x00000193, "C2_UNK_0000_0193", 0),
+    MFX(0x00000194, "CLOCK_FLEX_MAX", IntelFlexRatio, IntelFlexRatio, 0, UINT64_C(0xfffffffffffee0c0), 0), /* value=0x0 */
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0xb270b2786320620), 0, 0), /* value=0xb270b27`86320620 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x820, 0, 0), /* Might bite. value=0x820 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0x2, UINT64_C(0xffffffffffffffe1), 0), /* value=0x2 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0x3, UINT64_C(0xffffffff00010100), UINT32_C(0xff0000e0)), /* value=0x3 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, UINT32_C(0x881c0000), UINT64_C(0xfffffffff87f017f), 0x780fc00), /* value=0x881c0000 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0x61b, 0, 0), /* value=0x61b */
+    MVX(0x0000019e, "P6_UNK_0000_019e", 0x6930000, UINT64_C(0xffffffffffff0000), 0),
+    MVI(0x0000019f, "P6_UNK_0000_019f", 0),
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, UINT64_C(0x4062972489), UINT64_C(0x52603199f6), 0), /* value=0x40`62972489 */
+    MVX(0x000001a1, "P6_UNK_0000_01a1", 0, UINT64_C(0xffff000000000000), 0),
+    MFX(0x000001a2, "I7_MSR_TEMPERATURE_TARGET", IntelI7TemperatureTarget, ReadOnly, 0x1000, 0, 0), /* value=0x1000 */
+    MVX(0x000001aa, "P6_PIC_SENS_CFG", 0x7e1f042f, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x000001bf, "C2_UNK_0000_01bf", 0x404, UINT64_C(0xffffffffffff0000), 0),
+    MFX(0x000001c9, "MSR_LASTBRANCH_TOS", IntelLastBranchTos, IntelLastBranchTos, 0, UINT64_C(0xfffffffffffffffe), 0), /* value=0x3 */
+    MVX(0x000001d3, "P6_UNK_0000_01d3", 0x8000, UINT64_C(0xffffffffffff7fff), 0),
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffffe03c)), /* value=0x1 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0xffffffff`8142d5f6 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xffffffff`810644e0 */
+    MFN(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp, P6LastIntFromIp), /* value=0xffffffff`81039669 */
+    MFN(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp, P6LastIntToIp), /* value=0xffffffff`81039020 */
+    MVO(0x000001e0, "MSR_ROB_CR_BKUPTMPDR6", 0xff0),
+    MFX(0x000001f8, "IA32_PLATFORM_DCA_CAP", Ia32PlatformDcaCap, Ia32PlatformDcaCap, 0, UINT64_C(0xfffffffffffffffe), 0), /* value=0x0 */
+    MFO(0x000001f9, "IA32_CPU_DCA_CAP", Ia32CpuDcaCap), /* value=0x0 */
+    MFX(0x000001fa, "IA32_DCA_0_CAP", Ia32Dca0Cap, Ia32Dca0Cap, 0, UINT64_C(0xfffffffffefe17ff), 0), /* value=0xc01e488 */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`80000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x80000006 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`c0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xc0000006 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`f0000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xcff00000 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`fff00800 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x1`00000006 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`e0000800 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x1`20000006 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`f0000800 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    MFX(0x00000309, "IA32_FIXED_CTR0", Ia32FixedCtrN, Ia32FixedCtrN, 0x0, UINT64_C(0xffffff0000000000), 0), /* value=0xc4e */
+    MFX(0x0000030a, "IA32_FIXED_CTR1", Ia32FixedCtrN, Ia32FixedCtrN, 0x1, UINT64_C(0xffffff0000000c00), 0), /* value=0xff`9dd0e550 */
+    MFX(0x0000030b, "IA32_FIXED_CTR2", Ia32FixedCtrN, Ia32FixedCtrN, 0x2, UINT64_C(0xffffff0000000000), 0), /* value=0x205b */
+    MFX(0x00000345, "IA32_PERF_CAPABILITIES", Ia32PerfCapabilities, ReadOnly, 0x82, 0, 0), /* value=0x82 */
+    MFX(0x0000038d, "IA32_FIXED_CTR_CTRL", Ia32FixedCtrCtrl, Ia32FixedCtrCtrl, 0, UINT64_C(0xfffffffffffff000), 0), /* value=0xf0 */
+    MFX(0x0000038e, "IA32_PERF_GLOBAL_STATUS", Ia32PerfGlobalStatus, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFN(0x0000038f, "IA32_PERF_GLOBAL_CTRL", Ia32PerfGlobalCtrl, Ia32PerfGlobalCtrl), /* value=0x7`00000003 */
+    MFI(0x00000390, "IA32_PERF_GLOBAL_OVF_CTRL", Ia32PerfGlobalOvfCtrl), /* value=0x7`00000007 */
+    MFX(0x000003f1, "IA32_PEBS_ENABLE", Ia32PebsEnable, Ia32PebsEnable, 0, UINT64_C(0xfffffffffffffffe), 0), /* value=0x0 */
+    RFN(0x00000400, 0x00000417, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MVX(0x00000478, "TODO_0000_0478", 0, 0, 0),
+    MFX(0x00000480, "IA32_VMX_BASIC", Ia32VmxBasic, ReadOnly, UINT64_C(0x1a040000000007), 0, 0), /* value=0x1a0400`00000007 */
+    MFX(0x00000481, "IA32_VMX_PINBASED_CTLS", Ia32VmxPinbasedCtls, ReadOnly, UINT64_C(0x1f00000016), 0, 0), /* value=0x1f`00000016 */
+    MFX(0x00000482, "IA32_VMX_PROCBASED_CTLS", Ia32VmxProcbasedCtls, ReadOnly, UINT64_C(0x77b9fffe0401e172), 0, 0), /* value=0x77b9fffe`0401e172 */
+    MFX(0x00000483, "IA32_VMX_EXIT_CTLS", Ia32VmxExitCtls, ReadOnly, UINT64_C(0x3efff00036dff), 0, 0), /* value=0x3efff`00036dff */
+    MFX(0x00000484, "IA32_VMX_ENTRY_CTLS", Ia32VmxEntryCtls, ReadOnly, UINT64_C(0x1fff000011ff), 0, 0), /* value=0x1fff`000011ff */
+    MFX(0x00000485, "IA32_VMX_MISC", Ia32VmxMisc, ReadOnly, 0x403c0, 0, 0), /* value=0x403c0 */
+    MFX(0x00000486, "IA32_VMX_CR0_FIXED0", Ia32VmxCr0Fixed0, ReadOnly, UINT32_C(0x80000021), 0, 0), /* value=0x80000021 */
+    MFX(0x00000487, "IA32_VMX_CR0_FIXED1", Ia32VmxCr0Fixed1, ReadOnly, UINT32_MAX, 0, 0), /* value=0xffffffff */
+    MFX(0x00000488, "IA32_VMX_CR4_FIXED0", Ia32VmxCr4Fixed0, ReadOnly, 0x2000, 0, 0), /* value=0x2000 */
+    MFX(0x00000489, "IA32_VMX_CR4_FIXED1", Ia32VmxCr4Fixed1, ReadOnly, 0x27ff, 0, 0), /* value=0x27ff */
+    MFX(0x0000048a, "IA32_VMX_VMCS_ENUM", Ia32VmxVmcsEnum, ReadOnly, 0x2c, 0, 0), /* value=0x2c */
+    MVX(0x000004f8, "C2_UNK_0000_04f8", UINT64_C(0xf5d5fc5e567f6a8e), 0, 0),
+    MVX(0x000004f9, "C2_UNK_0000_04f9", UINT64_C(0xb595ed5afff3a8ff), 0, 0),
+    MVX(0x000004fa, "C2_UNK_0000_04fa", UINT64_C(0xfddfae7f5bfb7c47), 0, 0),
+    MVX(0x000004fb, "C2_UNK_0000_04fb", UINT64_C(0xf7ffbc5f93fd6fde), 0, 0),
+    MVX(0x000004fc, "C2_UNK_0000_04fc", UINT64_C(0xb7c5c95891fb71c6), 0, 0),
+    MVX(0x000004fd, "C2_UNK_0000_04fd", UINT64_C(0xb5d5cc5c95799df6), 0, 0),
+    MVX(0x000004fe, "C2_UNK_0000_04fe", UINT64_C(0xba95c85ad1fb3973), 0, 0),
+    MVX(0x000004ff, "C2_UNK_0000_04ff", UINT64_C(0xf5bdda4f9aff3943), 0, 0),
+    MVX(0x00000590, "C2_UNK_0000_0590", 0, 0, 0),
+    MVX(0x00000591, "C2_UNK_0000_0591", 0, ~(uint64_t)UINT32_MAX, 0),
+    MFN(0x000005a0, "C2_PECI_CTL", IntelCore2PeciControl, IntelCore2PeciControl), /* Might bite. value=0x1 */
+    MVI(0x000005a1, "C2_UNK_0000_05a1", 0), /* Might bite. */
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0xffff8801`2aaeba00 */
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xfffffffffffff2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* Might bite. value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* Might bite. value=0xffffffff`81844650 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* Might bite. value=0xffffffff`81846c90 */
+    MFN(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask), /* Might bite. value=0x47700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* Might bite. value=0x7fb5`e58d0740 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* Might bite. value=0xffff8801`2fc00000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* Might bite. value=0x0 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Core(TM)2 CPU         X6800  @ 2.93GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Core2_X6800_2_93GHz =
+{
+    /*.pszName          = */ "Intel Core2 X6800 2.93GHz",
+    /*.pszFullName      = */ "Intel(R) Core(TM)2 CPU         X6800  @ 2.93GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 15,
+    /*.uStepping        = */ 6,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_Core2_Merom,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_267MHZ,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 36,
+    /*.fMxCsrMask       = */ 0x0000ffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Core2_X6800_2_93GHz),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Core2_X6800_2_93GHz)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF,
+    /*.DefUnknownCpuId  = */ { 0x07280202, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Core2_X6800_2_93GHz)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Core2_X6800_2_93GHz),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Core2_X6800_2_93GHz_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Core_Duo_T2600_2_16GHz.h b/src/VBox/VMM/VMMR3/cpus/Intel_Core_Duo_T2600_2_16GHz.h
new file mode 100644
index 00000000..8a0ea8c9
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Core_Duo_T2600_2_16GHz.h
@@ -0,0 +1,225 @@
+/* $Id: Intel_Core_Duo_T2600_2_16GHz.h $ */
+/** @file
+ * CPU database entry "Intel Core Duo T2600 2.16GHz".
+ * Generated at 2017-11-02T10:39:16Z by VBoxCpuReport v5.2.0_RC1r118339 on linux.x86.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Core_Duo_T2600_2_16GHz_h
+#define VBOX_CPUDB_Intel_Core_Duo_T2600_2_16GHz_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Core(TM) Duo CPU      T2600  @ 2.16GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Core_Duo_T2600_2_16GHz[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000a, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000006e8, 0x01020800, 0x0000c1a9, 0xbfe9fbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x02b3b001, 0x000000f0, 0x00000000, 0x2c04307d, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, UINT32_MAX, 0x04000121, 0x01c0003f, 0x0000003f, 0x00000001, 0 },
+    { 0x00000004, 0x00000001, UINT32_MAX, 0x04000122, 0x01c0003f, 0x0000003f, 0x00000001, 0 },
+    { 0x00000004, 0x00000002, UINT32_MAX, 0x04004143, 0x01c0003f, 0x00000fff, 0x00000001, 0 },
+    { 0x00000004, 0x00000003, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00022220, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000001, 0x00000002, 0x00000001, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07280201, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00100000, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x756e6547, 0x20656e69, 0x65746e49, 0x2952286c, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x55504320, 0x20202020, 0x20202020, 0x54202020, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x30303632, 0x20402020, 0x36312e32, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x08006040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00002020, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Core(TM) Duo CPU      T2600  @ 2.16GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Core_Duo_T2600_2_16GHz[] =
+{
+    MFI(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr), /* value=0xf`eeda5160 */
+    MFI(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType), /* value=0x0 */
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x40 */
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* Villain? value=0x243`e2b88071 */
+    MFX(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0x140000d80486ac), 0, 0), /* value=0x140000`d80486ac */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00900), 0x600, UINT64_C(0xfffffff0000000ff)),
+    MVX(0x00000021, "C2_UNK_0000_0021", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xfffffffe)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, IntelEblCrPowerOn, 0x41880000, UINT64_C(0xfffffffffff7fffe), 0), /* value=0x41880000 */
+    MVI(0x0000002f, "P6_UNK_0000_002f", 0),
+    MVX(0x00000032, "P6_UNK_0000_0032", 0, UINT64_C(0xffffffff01fe0000), 0),
+    MVX(0x00000033, "TEST_CTL", 0, UINT64_C(0xffffffff7fffffff), 0),
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x5 */
+    MVO(0x0000003f, "P6_UNK_0000_003f", 0),
+    RFN(0x00000040, 0x00000047, "MSR_LASTBRANCH_n_FROM_IP", IntelLastBranchToN, IntelLastBranchToN),
+    MVX(0x0000004a, "P6_UNK_0000_004a", 0, 0, 0), /* value=0x0 */
+    MVX(0x0000004b, "P6_UNK_0000_004b", 0, 0, 0), /* value=0x0 */
+    MVX(0x0000004c, "P6_UNK_0000_004c", 0, 0, 0), /* value=0x0 */
+    MVX(0x0000004d, "P6_UNK_0000_004d", 0, 0, 0), /* value=0x3392fbd9`ffbefffd */
+    MVX(0x0000004e, "P6_UNK_0000_004e", 0, 0, 0), /* value=0xa6b77ad3`7ffbffe7 */
+    MVX(0x0000004f, "P6_UNK_0000_004f", 0, UINT64_C(0xffffffffffffff00), 0), /* value=0x9d`0000009d */
+    MVX(0x0000006c, "P6_UNK_0000_006c", 0, UINT64_C(0xffffffff00000082), 0),
+    MVX(0x0000006d, "P6_UNK_0000_006d", 0, UINT64_C(0xffffffff00000082), 0),
+    MVX(0x0000006e, "P6_UNK_0000_006e", UINT32_C(0x80000000), UINT64_C(0xffffffff00000082), 0),
+    MVO(0x0000006f, "P6_UNK_0000_006f", 0xadb),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, IgnoreWrite),
+    MFX(0x0000008b, "BBL_CR_D3|BIOS_SIGN", Ia32BiosSignId, Ia32BiosSignId, 0, UINT32_MAX, 0), /* value=0x39`00000000 */
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    MFX(0x000000c1, "IA32_PMC0", Ia32PmcN, Ia32PmcN, 0x0, UINT64_C(0xffffffff00124101), 0), /* XXX: The range ended earlier than expected! */
+    MFX(0x000000c2, "IA32_PMC1", Ia32PmcN, Ia32PmcN, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MVI(0x000000c7, "P6_UNK_0000_00c7", UINT64_C(0x1f00000044000000)),
+    MFX(0x000000cd, "MSR_FSB_FREQ", IntelP6FsbFrequency, ReadOnly, 0x133, 0, 0), /* value=0x133 */
+    MVO(0x000000ce, "P6_UNK_0000_00ce", UINT64_C(0x2c130d003b538000)),
+    MVO(0x000000e0, "C2_UNK_0000_00e0", 0x14ce0f0),
+    MVO(0x000000e1, "C2_UNK_0000_00e1", UINT32_C(0xf0f00000)),
+    MFX(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl, IntelPkgCStConfigControl, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xff000000)), /* value=0x26740c */
+    MFX(0x000000e3, "C2_SMM_CST_MISC_INFO", IntelCore2SmmCStMiscInfo, IntelCore2SmmCStMiscInfo, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x8040414 */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xff800000)), /* value=0x20414 */
+    MVO(0x000000e5, "C2_UNK_0000_00e5", 0x51c20cc0),
+    MFX(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf, 0, UINT64_C(0xffffffffe0000000), 0), /* value=0x5e`dc779a5a */
+    MFX(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf, 0, UINT64_C(0xffffffffe0000000), 0), /* value=0x2b`c8585b9a */
+    MFX(0x000000ee, "C1_EXT_CONFIG", IntelCore1ExtConfig, IntelCore1ExtConfig, 0, UINT64_C(0xffffffffffc5ffff), 0), /* value=0x82b90000 */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x508, 0, 0), /* value=0x508 */
+    MVX(0x00000116, "BBL_CR_ADDR", 0, UINT64_C(0xffffff000000001f), 0),
+    MVX(0x00000118, "BBL_CR_DECC", UINT64_C(0x88000fef00030892), UINT64_C(0x4780000fff00000), 0),
+    MFN(0x0000011a, "BBL_CR_TRIG", WriteOnly, IgnoreWrite),
+    MVI(0x0000011b, "P6_UNK_0000_011b", 0),
+    MFX(0x0000011e, "BBL_CR_CTL3", IntelBblCrCtl3, IntelBblCrCtl3, 0x7874211f, UINT64_C(0xffffffffc0f3feff), 0), /* value=0x7874211f */
+    MVX(0x0000014e, "P6_UNK_0000_014e", 0x49a49f20, UINT64_C(0xffffffff0000008f), 0),
+    MVX(0x0000014f, "P6_UNK_0000_014f", UINT32_MAX, UINT64_C(0xffffffff00100000), 0),
+    MVX(0x00000151, "P6_UNK_0000_0151", 0x25febbf6, ~(uint64_t)UINT32_MAX, 0),
+    MFX(0x0000015f, "C1_DTS_CAL_CTRL", IntelCore1DtsCalControl, IntelCore1DtsCalControl, 0, UINT64_C(0xffffffffffc0ffff), 0), /* value=0x260613 */
+    MFN(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs), /* Villain? value=0x60 */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* Villain? value=0xf5a07c40 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* Villain? value=0xc15af09c */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x6, 0, 0), /* value=0x6 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    RSN(0x00000186, 0x00000187, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, ~(uint64_t)UINT32_MAX, 0),
+    MFX(0x00000194, "CLOCK_FLEX_MAX", IntelFlexRatio, IntelFlexRatio, 0, UINT64_C(0xfffffffffffee0c0), 0), /* value=0x0 */
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0x6130d2c060c0613), 0, 0), /* value=0x6130d2c`060c0613 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x613, 0, 0), /* Might bite. value=0x613 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0x2, UINT64_C(0xffffffffffffffe1), 0), /* value=0x2 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0x3, UINT64_C(0xffffffff00616100), UINT32_C(0xff0000e0)), /* value=0x3 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, UINT32_C(0x8838000c), UINT64_C(0xfffffffff87f017f), 0x780fc00), /* value=0x8838000c */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0x613, 0, 0), /* value=0x613 */
+    MVX(0x0000019e, "P6_UNK_0000_019e", 0x11b0000, UINT64_C(0xffffffffffff0000), 0),
+    MVI(0x0000019f, "P6_UNK_0000_019f", 0),
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, UINT64_C(0x264973488), 0x60319bf7, 0), /* value=0x2`64973488 */
+    MVX(0x000001a1, "P6_UNK_0000_01a1", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x000001aa, "P6_PIC_SENS_CFG", 0x263f04b7, ~(uint64_t)UINT32_MAX, 0),
+    MFX(0x000001c9, "MSR_LASTBRANCH_TOS", IntelLastBranchTos, IntelLastBranchTos, 0, UINT64_C(0xfffffffffffffffa), 0), /* value=0x8000003 */
+    MVX(0x000001d3, "P6_UNK_0000_01d3", 0x8000, UINT64_C(0xffffffffffff7fff), 0),
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xfffffffffffffe3c)), /* value=0x1 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0xc12c5d73 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xc10357d0 */
+    MFX(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp, P6LastIntFromIp, 0, UINT64_C(0xffffffffff97dc5d), 0), /* value=0xc132a284 */
+    MFX(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp, P6LastIntToIp, 0, UINT64_C(0xfffffffffffffff0), 0), /* value=0xc1329543 */
+    MVO(0x000001e0, "MSR_ROB_CR_BKUPTMPDR6", 0xff0),
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xffe00005 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`ffe00800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x6 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`80000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x7ff00000 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`fff00800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x80000001 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf0000800 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    MFX(0x00000345, "IA32_PERF_CAPABILITIES", Ia32PerfCapabilities, ReadOnly, 0, 0, 0), /* value=0x0 */
+    RFN(0x00000400, 0x00000417, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0x00000480, "IA32_VMX_BASIC", Ia32VmxBasic, ReadOnly, UINT64_C(0x1b040000000005), 0, 0), /* value=0x1b0400`00000005 */
+    MFX(0x00000481, "IA32_VMX_PINBASED_CTLS", Ia32VmxPinbasedCtls, ReadOnly, UINT64_C(0x1f00000016), 0, 0), /* value=0x1f`00000016 */
+    MFX(0x00000482, "IA32_VMX_PROCBASED_CTLS", Ia32VmxProcbasedCtls, ReadOnly, UINT64_C(0x7781fffe0401e172), 0, 0), /* value=0x7781fffe`0401e172 */
+    MFX(0x00000483, "IA32_VMX_EXIT_CTLS", Ia32VmxExitCtls, ReadOnly, UINT64_C(0x3edff00036dff), 0, 0), /* value=0x3edff`00036dff */
+    MFX(0x00000484, "IA32_VMX_ENTRY_CTLS", Ia32VmxEntryCtls, ReadOnly, UINT64_C(0x1dff000011ff), 0, 0), /* value=0x1dff`000011ff */
+    MFX(0x00000485, "IA32_VMX_MISC", Ia32VmxMisc, ReadOnly, 0x403c0, 0, 0), /* value=0x403c0 */
+    MFX(0x00000486, "IA32_VMX_CR0_FIXED0", Ia32VmxCr0Fixed0, ReadOnly, UINT32_C(0x80000021), 0, 0), /* value=0x80000021 */
+    MFX(0x00000487, "IA32_VMX_CR0_FIXED1", Ia32VmxCr0Fixed1, ReadOnly, UINT32_MAX, 0, 0), /* value=0xffffffff */
+    MFX(0x00000488, "IA32_VMX_CR4_FIXED0", Ia32VmxCr4Fixed0, ReadOnly, 0x2000, 0, 0), /* value=0x2000 */
+    MFX(0x00000489, "IA32_VMX_CR4_FIXED1", Ia32VmxCr4Fixed1, ReadOnly, 0x27ff, 0, 0), /* value=0x27ff */
+    MFX(0x0000048a, "IA32_VMX_VMCS_ENUM", Ia32VmxVmcsEnum, ReadOnly, 0x2c, 0, 0), /* value=0x2c */
+    MVX(0x000004f8, "C2_UNK_0000_04f8", UINT64_C(0x1f5e86fb9f7f6dce), 0, 0),
+    MVX(0x000004f9, "C2_UNK_0000_04f9", UINT64_C(0xafb14bb80b893244), 0, 0),
+    MVX(0x000004fa, "C2_UNK_0000_04fa", UINT64_C(0xfecd26a6e39aeefe), 0, 0),
+    MVX(0x000004fb, "C2_UNK_0000_04fb", UINT64_C(0xd5baca676b503675), 0, 0),
+    MVX(0x000004fc, "C2_UNK_0000_04fc", UINT64_C(0x2e9b76a2bdde6ed7), 0, 0),
+    MVX(0x000004fd, "C2_UNK_0000_04fd", UINT64_C(0xfdbb141e45043200), 0, 0),
+    MVX(0x000004fe, "C2_UNK_0000_04fe", UINT64_C(0x4a68f426372a837f), 0, 0),
+    MVX(0x000004ff, "C2_UNK_0000_04ff", UINT64_C(0x4104628e2e437f40), 0, 0),
+    MFX(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0x800, 0, UINT64_C(0xfffffffffffff3ff)),
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Core(TM) Duo CPU      T2600  @ 2.16GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Core_Duo_T2600_2_16GHz =
+{
+    /*.pszName          = */ "Intel Core Duo T2600 2.16GHz",
+    /*.pszFullName      = */ "Genuine Intel(R) CPU           T2600  @ 2.16GHz",
+//    /*.pszFullName      = */ "Intel(R) Core(TM) Duo CPU      T2600  @ 2.16GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 14,
+    /*.uStepping        = */ 8,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_Core_Yonah,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_167MHZ,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 32,
+    /*.fMxCsrMask       = */ 0x0000ffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Core_Duo_T2600_2_16GHz),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Core_Duo_T2600_2_16GHz)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF,
+    /*.DefUnknownCpuId  = */ { 0x07280201, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Core_Duo_T2600_2_16GHz)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Core_Duo_T2600_2_16GHz),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Core_Duo_T2600_2_16GHz_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Core_i5_3570.h b/src/VBox/VMM/VMMR3/cpus/Intel_Core_i5_3570.h
new file mode 100644
index 00000000..d45079b7
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Core_i5_3570.h
@@ -0,0 +1,339 @@
+/* $Id: Intel_Core_i5_3570.h $ */
+/** @file
+ * CPU database entry "Intel Core i5-3570".
+ * Generated at 2013-12-13T16:13:56Z by VBoxCpuReport v4.3.53r91216 on linux.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Core_i5_3570_h
+#define VBOX_CPUDB_Intel_Core_i5_3570_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Core(TM) i5-3570 CPU @ 3.40GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Core_i5_3570[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000d, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000306a9, 0x04100800, 0x7fbae3ff, 0xbfebfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x76035a01, 0x00f0b0ff, 0x00000000, 0x00ca0000, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, 0x00000000, 0x1c004121, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00001120, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000077, 0x00000002, 0x00000009, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000281, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07300803, 0x00000000, 0x00000000, 0x00000603, 0 },
+    { 0x0000000b, 0x00000000, 0x00000000, 0x00000001, 0x00000001, 0x00000100, 0x00000004, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000000, 0x00000000, 0x00000007, 0x00000340, 0x00000340, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x28100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x65746e49, 0x2952286c, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x726f4320, 0x4d542865, 0x35692029, 0x3735332d, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x50432030, 0x20402055, 0x30342e33, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x01006040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000100, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003024, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Core(TM) i5-3570 CPU @ 3.40GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Core_i5_3570[] =
+{
+    MFX(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr, Ia32P5McAddr, 0, UINT64_C(0xffffffffffffffe0), 0), /* value=0x1f */
+    MFX(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType, Ia32P5McType, 0, 0, UINT64_MAX), /* value=0x0 */
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, 0, UINT64_C(0xffffffffffff0000)), /* value=0x40 */
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x4293`b0a3f54a */
+    MFV(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0x4000000000000)),
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00c00), 0, UINT64_C(0xfffffff0000002ff)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MVX(0x0000002e, "I7_UNK_0000_002e", 0, 0x400, UINT64_C(0xfffffffffffffbff)),
+    MVX(0x00000033, "TEST_CTL", 0, 0, UINT64_C(0xffffffff7fffffff)),
+    MVO(0x00000034, "P6_UNK_0000_0034", 0x285),
+    MFO(0x00000035, "MSR_CORE_THREAD_COUNT", IntelI7CoreThreadCount), /* value=0x40004*/
+    MVO(0x00000036, "I7_UNK_0000_0036", UINT64_C(0x1000000000105df2)),
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x5 */
+    MVX(0x0000003e, "I7_UNK_0000_003e", 0x1, 0, UINT64_C(0xfffffffffffffffe)),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, IgnoreWrite),
+    MVX(0x0000008b, "BBL_CR_D3|BIOS_SIGN", UINT64_C(0x1900000000), 0x1, UINT32_C(0xfffffffe)),
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    RSN(0x000000c1, 0x000000c8, "IA32_PMCn", Ia32PmcN, Ia32PmcN, 0x0, ~(uint64_t)UINT32_MAX, 0),
+    MFO(0x000000ce, "MSR_PLATFORM_INFO", IntelPlatformInfo), /* value=0x81010'e0012200*/
+    MFX(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl, IntelPkgCStConfigControl, 0, 0, UINT64_C(0xffffffffe1ffffff)), /* value=0x1e008403 */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, 0, UINT64_C(0xfffffffffff80000)), /* value=0x10414 */
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf), /* value=0x3a`2c710584 */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x39`f97c8410 */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0xd0a, 0, 0), /* value=0xd0a */
+    MVX(0x00000102, "I7_IB_UNK_0000_0102", 0, 0, UINT64_C(0xffffffff7fff8000)),
+    MVX(0x00000103, "I7_IB_UNK_0000_0103", 0, 0, UINT64_C(0xffffffffffffff00)),
+    MVX(0x00000104, "I7_IB_UNK_0000_0104", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MFN(0x00000132, "CPUID1_FEATURE_MASK", IntelCpuId1FeatureMaskEax, IntelCpuId1FeatureMaskEax), /* value=0xffffffff`ffffffff */
+    MFN(0x00000133, "CPUIDD_01_FEATURE_MASK", IntelCpuId1FeatureMaskEcdx, IntelCpuId1FeatureMaskEcdx), /* value=0xffffffff`ffffffff */
+    MFN(0x00000134, "CPUID80000001_FEATURE_MASK", IntelCpuId80000001FeatureMaskEcdx, IntelCpuId80000001FeatureMaskEcdx), /* value=0xffffffff`ffffffff */
+    MFX(0x0000013c, "I7_SB_AES_NI_CTL", IntelI7SandyAesNiCtl, IntelI7SandyAesNiCtl, 0, 0, UINT64_C(0xfffffffffffffffc)), /* value=0x0 */
+    MVX(0x00000140, "I7_IB_UNK_0000_0140", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MVX(0x00000142, "I7_IB_UNK_0000_0142", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x10 */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0x0 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0xffffffff`8159cbe0 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0xc09, 0, 0), /* value=0xc09 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, 0, UINT64_C(0xfffffffffffffff8)), /* value=0x0 */
+    RSN(0x00000186, 0x0000018d, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, 0, UINT64_C(0xffffffff00080000)),
+    MFX(0x00000194, "CLOCK_FLEX_MAX", IntelFlexRatio, IntelFlexRatio, 0x190000, 0x1e00ff, UINT64_C(0xffffffffffe00000)),
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0x1d2400001000), 0, 0), /* value=0x1d24`00001000 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x1000, 0, 0), /* Might bite. value=0x1000 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x0 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0x1000013, 0, UINT64_C(0xfffffffffe0000e8)), /* value=0x1000013 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, UINT32_C(0x884c0000), UINT32_C(0xf87f0fff), UINT64_C(0xffffffff0780f000)), /* value=0x884c0000 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, 0x850089, 0x1080, UINT64_C(0xffffffbbff3aef72)), /* value=0x850089 */
+    MFX(0x000001a2, "I7_MSR_TEMPERATURE_TARGET", IntelI7TemperatureTarget, IntelI7TemperatureTarget, 0x691400, 0xffff00, UINT64_C(0xfffffffff00000ff)), /* value=0x691400 */
+    MVX(0x000001a4, "I7_UNK_0000_01a4", 0, 0, UINT64_C(0xfffffffffffff7f0)),
+    RSN(0x000001a6, 0x000001a7, "I7_MSR_OFFCORE_RSP_n", IntelI7MsrOffCoreResponseN, IntelI7MsrOffCoreResponseN, 0x0, 0, UINT64_C(0xffffffc000007000)),
+    MVX(0x000001a8, "I7_UNK_0000_01a8", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFX(0x000001aa, "MSR_MISC_PWR_MGMT", IntelI7MiscPwrMgmt, IntelI7MiscPwrMgmt, 0, 0, UINT64_C(0xffffffffffbffffe)), /* value=0x400000 */
+    MFX(0x000001ad, "I7_MSR_TURBO_RATIO_LIMIT", IntelI7TurboRatioLimit, ReadOnly, 0x24252626, 0, 0), /* value=0x24252626 */
+    MVX(0x000001b0, "IA32_ENERGY_PERF_BIAS", 0x6, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x000001b1, "IA32_PACKAGE_THERM_STATUS", UINT32_C(0x88490000), UINT32_C(0xf87f0fff), UINT64_C(0xffffffff0780f000)),
+    MVX(0x000001b2, "IA32_PACKAGE_THERM_INTERRUPT", 0x1000003, 0, UINT64_C(0xfffffffffe0000e8)),
+    MVO(0x000001c6, "I7_UNK_0000_01c6", 0x3),
+    MFX(0x000001c8, "MSR_LBR_SELECT", IntelI7LbrSelect, IntelI7LbrSelect, 0, 0, UINT64_C(0xfffffffffffffe00)), /* value=0x0 */
+    MFX(0x000001c9, "MSR_LASTBRANCH_TOS", IntelLastBranchTos, IntelLastBranchTos, 0, 0, UINT64_C(0xfffffffffffffff0)), /* value=0x8 */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffff803c)), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0x7fffffff`a061f4c9 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xffffffff`810473c0 */
+    MFN(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp, P6LastIntFromIp), /* value=0x0 */
+    MFN(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp, P6LastIntToIp), /* value=0x0 */
+    MFO(0x000001f0, "I7_VLW_CAPABILITY", IntelI7VirtualLegacyWireCap), /* value=0x74 */
+    MFO(0x000001f2, "IA32_SMRR_PHYSBASE", Ia32SmrrPhysBase), /* value=0xdb000006 */
+    MFO(0x000001f3, "IA32_SMRR_PHYSMASK", Ia32SmrrPhysMask), /* value=0xff800800 */
+    MFX(0x000001fc, "I7_MSR_POWER_CTL", IntelI7PowerCtl, IntelI7PowerCtl, 0, 0x20, UINT64_C(0xffffffffffc20000)), /* value=0x14005f */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xc`00000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x4`00000006 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`e0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xe0000000 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`e0000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xdc000000 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`fc000800 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xdb800000 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`ff800800 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x4`1f000000 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`ff000800 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x4`1e800000 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`ff800800 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x4`1e600000 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`ffe00800 */
+    MFX(0x00000210, "IA32_MTRR_PHYS_BASE8", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x8, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000211, "IA32_MTRR_PHYS_MASK8", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x8, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x00000212, "IA32_MTRR_PHYS_BASE9", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x9, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000213, "IA32_MTRR_PHYS_MASK9", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x9, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    RSN(0x00000280, 0x00000281, "IA32_MC0_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0x0, 0, UINT64_C(0xffffffffbfff8000)),
+    MFX(0x00000282, "IA32_MC2_CTL2", Ia32McNCtl2, Ia32McNCtl2, 0x2, 0x40007fff, UINT64_C(0xffffffffbfff8000)), /* value=0x0 */
+    MFX(0x00000283, "IA32_MC3_CTL2", Ia32McNCtl2, Ia32McNCtl2, 0x3, 0, UINT64_C(0xffffffffbfff8000)), /* value=0x40000001 */
+    MFX(0x00000284, "IA32_MC4_CTL2", Ia32McNCtl2, Ia32McNCtl2, 0x4, 0x40007fff, UINT64_C(0xffffffffbfff8000)), /* value=0x0 */
+    RSN(0x00000285, 0x00000288, "IA32_MC5_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0x5, 0, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x000002e0, "I7_SB_NO_EVICT_MODE", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFN(0x000002e6, "I7_IB_UNK_0000_02e6", WriteOnly, IgnoreWrite),
+    MVX(0x000002e7, "I7_IB_UNK_0000_02e7", 0x1, 0x1, UINT64_C(0xfffffffffffffffe)),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    MVO(0x00000305, "I7_SB_UNK_0000_0305", 0),
+    MFX(0x00000309, "IA32_FIXED_CTR0", Ia32FixedCtrN, Ia32FixedCtrN, 0x0, 0, UINT64_C(0xffff000000000000)), /* value=0x46 */
+    MFX(0x0000030a, "IA32_FIXED_CTR1", Ia32FixedCtrN, Ia32FixedCtrN, 0x1, 0x816506, UINT64_C(0xffff000000000000)), /* value=0xffff`d65aa6fb */
+    MFX(0x0000030b, "IA32_FIXED_CTR2", Ia32FixedCtrN, Ia32FixedCtrN, 0x2, 0, UINT64_C(0xffff000000000000)), /* value=0x264 */
+    MFX(0x00000345, "IA32_PERF_CAPABILITIES", Ia32PerfCapabilities, ReadOnly, 0x31c3, 0, 0), /* value=0x31c3 */
+    MFX(0x0000038d, "IA32_FIXED_CTR_CTRL", Ia32FixedCtrCtrl, Ia32FixedCtrCtrl, 0, 0, UINT64_C(0xfffffffffffff000)), /* value=0xb0 */
+    MFX(0x0000038e, "IA32_PERF_GLOBAL_STATUS", Ia32PerfGlobalStatus, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x0000038f, "IA32_PERF_GLOBAL_CTRL", Ia32PerfGlobalCtrl, Ia32PerfGlobalCtrl, 0, 0, UINT64_C(0xfffffff8ffffff00)), /* value=0x7`000000ff */
+    MFX(0x00000390, "IA32_PERF_GLOBAL_OVF_CTRL", Ia32PerfGlobalOvfCtrl, Ia32PerfGlobalOvfCtrl, 0, UINT64_C(0xe0000007000000ff), UINT64_C(0x1ffffff8ffffff00)), /* value=0x0 */
+    MFX(0x00000391, "I7_UNC_PERF_GLOBAL_CTRL", IntelI7UncPerfGlobalCtrl, IntelI7UncPerfGlobalCtrl, 0, 0, UINT64_C(0xffffffff1fffffe0)), /* value=0x2000000f */
+    MFX(0x00000392, "I7_UNC_PERF_GLOBAL_STATUS", IntelI7UncPerfGlobalStatus, IntelI7UncPerfGlobalStatus, 0, 0xf, UINT64_C(0xfffffffffffffff0)), /* value=0x0 */
+    MFX(0x00000393, "I7_UNC_PERF_GLOBAL_OVF_CTRL", IntelI7UncPerfGlobalOvfCtrl, IntelI7UncPerfGlobalOvfCtrl, 0, 0x3, UINT64_C(0xfffffffffffffffc)), /* value=0x0 */
+    MFX(0x00000394, "I7_UNC_PERF_FIXED_CTR_CTRL", IntelI7UncPerfFixedCtrCtrl, IntelI7UncPerfFixedCtrCtrl, 0, 0, UINT64_C(0xffffffffffafffff)), /* value=0x0 */
+    MFX(0x00000395, "I7_UNC_PERF_FIXED_CTR", IntelI7UncPerfFixedCtr, IntelI7UncPerfFixedCtr, 0, 0, UINT64_C(0xffff000000000000)), /* value=0x1950 */
+    MFO(0x00000396, "I7_UNC_CBO_CONFIG", IntelI7UncCBoxConfig), /* value=0x5 */
+    MVX(0x00000397, "I7_IB_UNK_0000_0397", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MFX(0x000003b0, "I7_UNC_ARB_PERF_CTR0", IntelI7UncArbPerfCtrN, IntelI7UncArbPerfCtrN, 0, 0, UINT64_C(0xfffff00000000000)), /* value=0x0 */
+    MFX(0x000003b1, "I7_UNC_ARB_PERF_CTR1", IntelI7UncArbPerfCtrN, IntelI7UncArbPerfCtrN, 0, 0, UINT64_C(0xfffff00000000000)), /* value=0x0 */
+    MFX(0x000003b2, "I7_UNC_ARB_PERF_EVT_SEL0", IntelI7UncArbPerfEvtSelN, IntelI7UncArbPerfEvtSelN, 0, 0, UINT64_C(0xffffffffc0230000)), /* value=0x0 */
+    MFX(0x000003b3, "I7_UNC_ARB_PERF_EVT_SEL1", IntelI7UncArbPerfEvtSelN, IntelI7UncArbPerfEvtSelN, 0, 0, UINT64_C(0xffffffffc0230000)), /* value=0x0 */
+    MFX(0x000003f1, "IA32_PEBS_ENABLE", Ia32PebsEnable, Ia32PebsEnable, 0, 0, UINT64_C(0x7ffffff0fffffff0)), /* value=0x0 */
+    MFX(0x000003f6, "I7_MSR_PEBS_LD_LAT", IntelI7PebsLdLat, IntelI7PebsLdLat, 0, UINT64_C(0xffffffffffff0000), 0), /* value=0xffff */
+    MFX(0x000003f8, "I7_MSR_PKG_C3_RESIDENCY", IntelI7PkgCnResidencyN, ReadOnly, 0x3, 0, UINT64_MAX), /* value=0x7`7827f19a */
+    RSN(0x000003f9, 0x000003fa, "I7_MSR_PKG_Cn_RESIDENCY", IntelI7PkgCnResidencyN, ReadOnly, 0x6, 0, UINT64_MAX),
+    MFX(0x000003fc, "I7_MSR_CORE_C3_RESIDENCY", IntelI7CoreCnResidencyN, ReadOnly, 0x3, 0, UINT64_MAX), /* value=0x1`3e604592 */
+    RSN(0x000003fd, 0x000003fe, "I7_MSR_CORE_Cn_RESIDENCY", IntelI7CoreCnResidencyN, ReadOnly, 0x6, 0, UINT64_MAX),
+    RFN(0x00000400, 0x00000423, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0x00000480, "IA32_VMX_BASIC", Ia32VmxBasic, ReadOnly, UINT64_C(0xda040000000010), 0, 0), /* value=0xda0400`00000010 */
+    MFX(0x00000481, "IA32_VMX_PINBASED_CTLS", Ia32VmxPinbasedCtls, ReadOnly, UINT64_C(0x7f00000016), 0, 0), /* value=0x7f`00000016 */
+    MFX(0x00000482, "IA32_VMX_PROCBASED_CTLS", Ia32VmxProcbasedCtls, ReadOnly, UINT64_C(0xfff9fffe0401e172), 0, 0), /* value=0xfff9fffe`0401e172 */
+    MFX(0x00000483, "IA32_VMX_EXIT_CTLS", Ia32VmxExitCtls, ReadOnly, UINT64_C(0x7fffff00036dff), 0, 0), /* value=0x7fffff`00036dff */
+    MFX(0x00000484, "IA32_VMX_ENTRY_CTLS", Ia32VmxEntryCtls, ReadOnly, UINT64_C(0xffff000011ff), 0, 0), /* value=0xffff`000011ff */
+    MFX(0x00000485, "IA32_VMX_MISC", Ia32VmxMisc, ReadOnly, 0x100401e5, 0, 0), /* value=0x100401e5 */
+    MFX(0x00000486, "IA32_VMX_CR0_FIXED0", Ia32VmxCr0Fixed0, ReadOnly, UINT32_C(0x80000021), 0, 0), /* value=0x80000021 */
+    MFX(0x00000487, "IA32_VMX_CR0_FIXED1", Ia32VmxCr0Fixed1, ReadOnly, UINT32_MAX, 0, 0), /* value=0xffffffff */
+    MFX(0x00000488, "IA32_VMX_CR4_FIXED0", Ia32VmxCr4Fixed0, ReadOnly, 0x2000, 0, 0), /* value=0x2000 */
+    MFX(0x00000489, "IA32_VMX_CR4_FIXED1", Ia32VmxCr4Fixed1, ReadOnly, 0x1767ff, 0, 0), /* value=0x1767ff */
+    MFX(0x0000048a, "IA32_VMX_VMCS_ENUM", Ia32VmxVmcsEnum, ReadOnly, 0x2a, 0, 0), /* value=0x2a */
+    MFX(0x0000048b, "IA32_VMX_PROCBASED_CTLS2", Ia32VmxProcBasedCtls2, ReadOnly, UINT64_C(0x8ff00000000), 0, 0), /* value=0x8ff`00000000 */
+    MFX(0x0000048c, "IA32_VMX_EPT_VPID_CAP", Ia32VmxEptVpidCap, ReadOnly, UINT64_C(0xf0106114141), 0, 0), /* value=0xf01`06114141 */
+    MFX(0x0000048d, "IA32_VMX_TRUE_PINBASED_CTLS", Ia32VmxTruePinbasedCtls, ReadOnly, UINT64_C(0x7f00000016), 0, 0), /* value=0x7f`00000016 */
+    MFX(0x0000048e, "IA32_VMX_TRUE_PROCBASED_CTLS", Ia32VmxTrueProcbasedCtls, ReadOnly, UINT64_C(0xfff9fffe04006172), 0, 0), /* value=0xfff9fffe`04006172 */
+    MFX(0x0000048f, "IA32_VMX_TRUE_EXIT_CTLS", Ia32VmxTrueExitCtls, ReadOnly, UINT64_C(0x7fffff00036dfb), 0, 0), /* value=0x7fffff`00036dfb */
+    MFX(0x00000490, "IA32_VMX_TRUE_ENTRY_CTLS", Ia32VmxTrueEntryCtls, ReadOnly, UINT64_C(0xffff000011fb), 0, 0), /* value=0xffff`000011fb */
+    RSN(0x000004c1, 0x000004c8, "IA32_A_PMCn", Ia32PmcN, Ia32PmcN, 0x0, 0, UINT64_C(0xffff000000000000)),
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0xffff8804`07da1cc0 */
+    MFX(0x00000601, "I7_SB_MSR_VR_CURRENT_CONFIG", IntelI7SandyVrCurrentConfig, IntelI7SandyVrCurrentConfig, 0, UINT32_C(0x80001fff), 0x7fffe000), /* value=0x18141494`80000380 */
+    MVX(0x00000602, "I7_IB_UNK_0000_0602", UINT64_C(0x1814149480000170), UINT32_C(0x80001fff), 0x7fffe000),
+    MFX(0x00000603, "I7_SB_MSR_VR_MISC_CONFIG", IntelI7SandyVrMiscConfig, IntelI7SandyVrMiscConfig, 0, UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)), /* value=0x802c2c2c */
+    MVX(0x00000604, "I7_IB_UNK_0000_0602", UINT32_C(0x80686868), UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)),
+    MFO(0x00000606, "I7_SB_MSR_RAPL_POWER_UNIT", IntelI7SandyRaplPowerUnit), /* value=0xa1003 */
+    MFX(0x0000060a, "I7_SB_MSR_PKGC3_IRTL", IntelI7SandyPkgCnIrtlN, IntelI7SandyPkgCnIrtlN, 0x3, 0, UINT64_C(0xffffffffffff6000)), /* value=0x883b */
+    RSN(0x0000060b, 0x0000060c, "I7_SB_MSR_PKGC6_IRTn", IntelI7SandyPkgCnIrtlN, IntelI7SandyPkgCnIrtlN, 0x6, 0, UINT64_C(0xffffffffffff6000)),
+    MFO(0x0000060d, "I7_SB_MSR_PKG_C2_RESIDENCY", IntelI7SandyPkgC2Residency), /* value=0x76c`bd67b914 */
+    MFX(0x00000610, "I7_SB_MSR_PKG_POWER_LIMIT", IntelI7RaplPkgPowerLimit, IntelI7RaplPkgPowerLimit, 0, UINT64_C(0x80ffffff00ffffff), UINT64_C(0x7f000000ff000000)), /* value=0x80008302`00148268 */
+    MFO(0x00000611, "I7_SB_MSR_PKG_ENERGY_STATUS", IntelI7RaplPkgEnergyStatus), /* value=0x3451b969 */
+    MFO(0x00000614, "I7_SB_MSR_PKG_POWER_INFO", IntelI7RaplPkgPowerInfo), /* value=0xd0000`01e00268 */
+    MFX(0x00000638, "I7_SB_MSR_PP0_POWER_LIMIT", IntelI7RaplPp0PowerLimit, IntelI7RaplPp0PowerLimit, 0, UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)), /* value=0x80000000 */
+    MFO(0x00000639, "I7_SB_MSR_PP0_ENERGY_STATUS", IntelI7RaplPp0EnergyStatus), /* value=0x357de52e */
+    MFX(0x0000063a, "I7_SB_MSR_PP0_POLICY", IntelI7RaplPp0Policy, IntelI7RaplPp0Policy, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x0 */
+    MFX(0x00000640, "I7_HW_MSR_PP0_POWER_LIMIT", IntelI7RaplPp1PowerLimit, IntelI7RaplPp1PowerLimit, 0, UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)), /* value=0x80000000 */
+    MFO(0x00000641, "I7_HW_MSR_PP0_ENERGY_STATUS", IntelI7RaplPp1EnergyStatus), /* value=0x6eeef */
+    MFX(0x00000642, "I7_HW_MSR_PP0_POLICY", IntelI7RaplPp1Policy, IntelI7RaplPp1Policy, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x10 */
+    MFO(0x00000648, "I7_IB_MSR_CONFIG_TDP_NOMINAL", IntelI7IvyConfigTdpNominal), /* value=0x22 */
+    MFO(0x00000649, "I7_IB_MSR_CONFIG_TDP_LEVEL1", IntelI7IvyConfigTdpLevel1), /* value=0x1e00000`00000000 */
+    MFO(0x0000064a, "I7_IB_MSR_CONFIG_TDP_LEVEL2", IntelI7IvyConfigTdpLevel2), /* value=0x1e00000`00000000 */
+    MFO(0x0000064b, "I7_IB_MSR_CONFIG_TDP_CONTROL", IntelI7IvyConfigTdpControl), /* value=0x80000000 */
+    MFX(0x0000064c, "I7_IB_MSR_TURBO_ACTIVATION_RATIO", IntelI7IvyTurboActivationRatio, IntelI7IvyTurboActivationRatio, 0, 0, UINT64_C(0xffffffff7fffff00)), /* value=0x80000000 */
+    RFN(0x00000680, 0x0000068f, "MSR_LASTBRANCH_n_FROM_IP", IntelLastBranchFromN, IntelLastBranchFromN),
+    RFN(0x000006c0, 0x000006cf, "MSR_LASTBRANCH_n_TO_IP", IntelLastBranchFromN, IntelLastBranchFromN),
+    MFX(0x000006e0, "IA32_TSC_DEADLINE", Ia32TscDeadline, Ia32TscDeadline, 0, UINT64_C(0xb280452208b), 0), /* value=0x4293`ef1535a6 */
+    MVX(0x00000700, "I7_IB_UNK_0000_0700", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000701, "I7_IB_UNK_0000_0701", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000702, "I7_IB_UNK_0000_0702", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000703, "I7_IB_UNK_0000_0703", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000704, "I7_IB_UNK_0000_0704", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000705, "I7_IB_UNK_0000_0705", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000706, "I7_IB_UNK_0000_0706", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000707, "I7_IB_UNK_0000_0707", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000708, "I7_IB_UNK_0000_0708", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000709, "I7_IB_UNK_0000_0709", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000710, "I7_IB_UNK_0000_0710", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000711, "I7_IB_UNK_0000_0711", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000712, "I7_IB_UNK_0000_0712", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000713, "I7_IB_UNK_0000_0713", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000714, "I7_IB_UNK_0000_0714", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000715, "I7_IB_UNK_0000_0715", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000716, "I7_IB_UNK_0000_0716", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000717, "I7_IB_UNK_0000_0717", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000718, "I7_IB_UNK_0000_0718", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000719, "I7_IB_UNK_0000_0719", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000720, "I7_IB_UNK_0000_0720", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000721, "I7_IB_UNK_0000_0721", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000722, "I7_IB_UNK_0000_0722", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000723, "I7_IB_UNK_0000_0723", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000724, "I7_IB_UNK_0000_0724", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000725, "I7_IB_UNK_0000_0725", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000726, "I7_IB_UNK_0000_0726", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000727, "I7_IB_UNK_0000_0727", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000728, "I7_IB_UNK_0000_0728", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000729, "I7_IB_UNK_0000_0729", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000730, "I7_IB_UNK_0000_0730", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000731, "I7_IB_UNK_0000_0731", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000732, "I7_IB_UNK_0000_0732", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000733, "I7_IB_UNK_0000_0733", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000734, "I7_IB_UNK_0000_0734", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000735, "I7_IB_UNK_0000_0735", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000736, "I7_IB_UNK_0000_0736", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000737, "I7_IB_UNK_0000_0737", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000738, "I7_IB_UNK_0000_0738", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000739, "I7_IB_UNK_0000_0739", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000740, "I7_IB_UNK_0000_0740", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000741, "I7_IB_UNK_0000_0741", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000742, "I7_IB_UNK_0000_0742", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000743, "I7_IB_UNK_0000_0743", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000744, "I7_IB_UNK_0000_0744", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000745, "I7_IB_UNK_0000_0745", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000746, "I7_IB_UNK_0000_0746", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000747, "I7_IB_UNK_0000_0747", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000748, "I7_IB_UNK_0000_0748", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000749, "I7_IB_UNK_0000_0749", 0, 0, UINT64_C(0xfffff00000000000)),
+    RFN(0x00000800, 0x000008ff, "IA32_X2APIC_n", Ia32X2ApicN, Ia32X2ApicN),
+    MFN(0x00000c80, "IA32_DEBUG_INTERFACE", Ia32DebugInterface, Ia32DebugInterface), /* value=0x0 */
+    MVX(0x00000c81, "I7_IB_UNK_0000_0c81", 0, 0, 0),
+    MVX(0x00000c82, "I7_IB_UNK_0000_0c82", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00000c83, "I7_IB_UNK_0000_0c83", 0, ~(uint64_t)UINT32_MAX, 0),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xfffffffffffff2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xffffffff`8159b620 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xffffffff`8159ce10 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x43700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x908880 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffff8804`1e200000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x0 */
+    MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, 0, ~(uint64_t)UINT32_MAX), /* value=0x0 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Core(TM) i5-3570 CPU @ 3.40GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Core_i5_3570 =
+{
+    /*.pszName          = */ "Intel Core i5-3570",
+    /*.pszFullName      = */ "Intel(R) Core(TM) i5-3570 CPU @ 3.40GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 58,
+    /*.uStepping        = */ 9,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_Core7_IvyBridge,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_100MHZ,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 36,
+    /*.fMxCsrMask       = */ 0xffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Core_i5_3570),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Core_i5_3570)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX,
+    /*.DefUnknownCpuId  = */ { 0x00000007, 0x00000340, 0x00000340, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Core_i5_3570)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Core_i5_3570),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Core_i5_3570_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_2635QM.h b/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_2635QM.h
new file mode 100644
index 00000000..66e00b6d
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_2635QM.h
@@ -0,0 +1,332 @@
+/* $Id: Intel_Core_i7_2635QM.h $ */
+/** @file
+ * CPU database entry "Intel Core i7-2635QM".
+ * Generated at 2014-02-28T18:53:09Z by VBoxCpuReport v4.3.53r92586 on darwin.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Core_i7_2635QM_h
+#define VBOX_CPUDB_Intel_Core_i7_2635QM_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Core(TM) i7-2635QM CPU @ 2.00GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Core_i7_2635QM[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000d, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000206a7, 0x04100800, 0x1fbae3bf, 0xbfebfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x76035a01, 0x00f0b2ff, 0x00000000, 0x00ca0000, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, UINT32_MAX, 0x1c004121, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000004, 0x00000001, UINT32_MAX, 0x1c004122, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000004, 0x00000002, UINT32_MAX, 0x1c004143, 0x01c0003f, 0x000001ff, 0x00000000, 0 },
+    { 0x00000004, 0x00000003, UINT32_MAX, 0x1c03c163, 0x02c0003f, 0x00001fff, 0x00000006, 0 },
+    { 0x00000004, 0x00000004, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00021120, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000077, 0x00000002, 0x00000009, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07300403, 0x00000000, 0x00000000, 0x00000603, 0 },
+    /** @todo the b entry here is WRONG!   */
+    { 0x0000000b, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000000, UINT32_MAX, 0x00000007, 0x00000340, 0x00000340, 0x00000000, 0 },
+    { 0x0000000d, 0x00000001, UINT32_MAX, 0x00000001, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000002, UINT32_MAX, 0x00000100, 0x00000240, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000003, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x28100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20202020, 0x6e492020, 0x286c6574, 0x43202952, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x2865726f, 0x20294d54, 0x322d3769, 0x51353336, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x5043204d, 0x20402055, 0x30302e32, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x01006040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000100, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003024, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Core(TM) i7-2635QM CPU @ 2.00GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Core_i7_2635QM[] =
+{
+    MFX(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr, Ia32P5McAddr, 0, UINT64_C(0xffffffffffffffe0), 0), /* value=0x1f */
+    MFX(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType, Ia32P5McType, 0, 0, UINT64_MAX), /* value=0x0 */
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, 0, UINT64_C(0xffffffffffff0000)), /* value=0x40 */
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x94d`1967512c */
+    MFX(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0x10000000000000), 0, 0), /* value=0x100000`00000000 */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0, UINT64_C(0xfffffff0000002ff)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MVX(0x0000002e, "I7_UNK_0000_002e", 0, 0x400, UINT64_C(0xfffffffffffffbff)),
+    MVX(0x00000033, "TEST_CTL", 0, 0, UINT64_C(0xffffffff7fffffff)),
+    MVO(0x00000034, "P6_UNK_0000_0034", 0x5),
+    MFO(0x00000035, "MSR_CORE_THREAD_COUNT", IntelI7CoreThreadCount), /* value=0x40008 */
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x5 */
+    MVX(0x0000003e, "I7_UNK_0000_003e", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, IgnoreWrite),
+    MFX(0x0000008b, "BBL_CR_D3|BIOS_SIGN", Ia32BiosSignId, Ia32BiosSignId, 0, 0, UINT32_C(0xfffffffe)), /* value=0x28`00000000 */
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    RSN(0x000000c1, 0x000000c4, "IA32_PMCn", Ia32PmcN, Ia32PmcN, 0x0, ~(uint64_t)UINT32_MAX, 0),
+    MFO(0x000000ce, "IA32_PLATFORM_INFO", IntelPlatformInfo), /* value=0x800`60011400 */
+    MFX(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl, IntelPkgCStConfigControl, 0, 0, UINT64_C(0xffffffffe1ffffff)), /* value=0x405 */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, 0, UINT64_C(0xfffffffffff80000)), /* value=0x20414 */
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf), /* value=0x6a`9190b14b */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x69`df4de05c */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0xd0a, 0, 0), /* value=0xd0a */
+    MFN(0x00000132, "CPUID1_FEATURE_MASK", IntelCpuId1FeatureMaskEax, IntelCpuId1FeatureMaskEax), /* value=0xffffffff`ffffffff */
+    MFN(0x00000133, "CPUIDD_01_FEATURE_MASK", IntelCpuId1FeatureMaskEcdx, IntelCpuId1FeatureMaskEcdx), /* value=0xffffffff`ffffffff */
+    MFN(0x00000134, "CPUID80000001_FEATURE_MASK", IntelCpuId80000001FeatureMaskEcdx, IntelCpuId80000001FeatureMaskEcdx), /* value=0xffffffff`ffffffff */
+    MFX(0x0000013c, "I7_SB_AES_NI_CTL", IntelI7SandyAesNiCtl, IntelI7SandyAesNiCtl, 0, 0, UINT64_C(0xfffffffffffffffc)), /* value=0x0 */
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0xb */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0xffffff80`22904080 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0xffffff80`222f3030 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0xc09, 0, 0), /* value=0xc09 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, 0, UINT64_C(0xfffffffffffffff8)), /* value=0x0 */
+    RSN(0x00000186, 0x00000189, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, 0, UINT64_C(0xffffffff00080000)),
+    MFX(0x00000194, "CLOCK_FLEX_MAX", IntelFlexRatio, IntelFlexRatio, 0, 0xe0000, UINT64_C(0xfffffffffff00000)), /* value=0x0 */
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0x1d4d00000e00), 0, 0), /* value=0x1d4d`00000e00 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x1d00, 0, 0), /* Might bite. value=0x1d00 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x0 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0, 0, UINT64_C(0xfffffffffe0000e8)), /* value=0x0 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, UINT32_C(0x883d0000), UINT32_C(0xf87f0fff), UINT64_C(0xffffffff0780f000)), /* value=0x883d0000 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, 0x850089, 0x1080, UINT64_C(0xffffffbbff3aef72)), /* value=0x850089 */
+    MFX(0x000001a2, "I7_MSR_TEMPERATURE_TARGET", IntelI7TemperatureTarget, IntelI7TemperatureTarget, 0x640e00, 0xffff00, UINT64_C(0xfffffffff00000ff)), /* value=0x640e00 */
+    MVX(0x000001a4, "I7_UNK_0000_01a4", 0, 0, UINT64_C(0xfffffffffffff7f0)),
+    RSN(0x000001a6, 0x000001a7, "I7_MSR_OFFCORE_RSP_n", IntelI7MsrOffCoreResponseN, IntelI7MsrOffCoreResponseN, 0x0, 0, UINT64_C(0xffffffc000007000)),
+    MVX(0x000001a8, "I7_UNK_0000_01a8", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFX(0x000001aa, "MSR_MISC_PWR_MGMT", IntelI7MiscPwrMgmt, IntelI7MiscPwrMgmt, 0, 0, UINT64_C(0xffffffffffbffffe)), /* value=0x400001 */
+    MFX(0x000001ad, "I7_MSR_TURBO_RATIO_LIMIT", IntelI7TurboRatioLimit, ReadOnly, 0x1a1a1c1d, 0, 0), /* value=0x1a1a1c1d */
+    MVX(0x000001b0, "IA32_ENERGY_PERF_BIAS", 0x4, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x000001b1, "IA32_PACKAGE_THERM_STATUS", UINT32_C(0x883a0000), UINT32_C(0xf87f0fff), UINT64_C(0xffffffff0780f000)),
+    MVX(0x000001b2, "IA32_PACKAGE_THERM_INTERRUPT", 0, 0, UINT64_C(0xfffffffffe0000e8)),
+    MVO(0x000001c6, "I7_UNK_0000_01c6", 0x3),
+    MFX(0x000001c8, "MSR_LBR_SELECT", IntelI7LbrSelect, IntelI7LbrSelect, 0, 0, UINT64_C(0xfffffffffffffe00)), /* value=0x0 */
+    MFX(0x000001c9, "MSR_LASTBRANCH_TOS", IntelLastBranchTos, IntelLastBranchTos, 0, 0, UINT64_C(0xfffffffffffffff0)), /* value=0xc */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffff803c)), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0x7fffff7f`a4a6e188 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xffffff80`222d5ad0 */
+    MFN(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp, P6LastIntFromIp), /* value=0x0 */
+    MFN(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp, P6LastIntToIp), /* value=0x0 */
+    MVO(0x000001e1, "I7_SB_UNK_0000_01e1", 0x2),
+    MFO(0x000001f0, "I7_VLW_CAPABILITY", IntelI7VirtualLegacyWireCap), /* value=0x74 */
+    MFO(0x000001f2, "IA32_SMRR_PHYSBASE", Ia32SmrrPhysBase), /* value=0x0 */
+    MFO(0x000001f3, "IA32_SMRR_PHYSMASK", Ia32SmrrPhysMask), /* value=0x0 */
+    MFX(0x000001fc, "I7_MSR_POWER_CTL", IntelI7PowerCtl, IntelI7PowerCtl, 0, 0x20, UINT64_C(0xfffffffffff20000)), /* value=0x4005f */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xc0000000 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`c0000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xa0000000 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`e0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x90000000 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`f0000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x8c000000 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`fc000800 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x8b800000 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`ff800800 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x00000210, "IA32_MTRR_PHYS_BASE8", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x8, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000211, "IA32_MTRR_PHYS_MASK8", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x8, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x00000212, "IA32_MTRR_PHYS_BASE9", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x9, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000213, "IA32_MTRR_PHYS_MASK9", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x9, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    RSN(0x00000280, 0x00000281, "IA32_MC0_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0x0, 0, UINT64_C(0xffffffffbfff8000)),
+    MFX(0x00000282, "IA32_MC2_CTL2", Ia32McNCtl2, Ia32McNCtl2, 0x2, 0x40007fff, UINT64_C(0xffffffffbfff8000)), /* value=0x0 */
+    MFX(0x00000283, "IA32_MC3_CTL2", Ia32McNCtl2, Ia32McNCtl2, 0x3, 0, UINT64_C(0xffffffffbfff8000)), /* value=0x0 */
+    MFX(0x00000284, "IA32_MC4_CTL2", Ia32McNCtl2, Ia32McNCtl2, 0x4, 0x40007fff, UINT64_C(0xffffffffbfff8000)), /* value=0x0 */
+    RSN(0x00000285, 0x00000288, "IA32_MC5_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0x5, 0, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x000002e0, "I7_SB_NO_EVICT_MODE", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFN(0x000002e6, "I7_IB_UNK_0000_02e6", WriteOnly, IgnoreWrite),
+    MVX(0x000002e7, "I7_IB_UNK_0000_02e7", 0x1, 0x1, UINT64_C(0xfffffffffffffffe)),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    MVO(0x00000305, "I7_SB_UNK_0000_0305", 0),
+    RSN(0x00000309, 0x0000030b, "IA32_FIXED_CTRn", Ia32FixedCtrN, Ia32FixedCtrN, 0x0, 0, UINT64_C(0xffff000000000000)),
+    MFX(0x00000345, "IA32_PERF_CAPABILITIES", Ia32PerfCapabilities, ReadOnly, 0x31c3, 0, 0), /* value=0x31c3 */
+    MFX(0x0000038d, "IA32_FIXED_CTR_CTRL", Ia32FixedCtrCtrl, Ia32FixedCtrCtrl, 0, 0, UINT64_C(0xfffffffffffff000)), /* value=0x0 */
+    MFX(0x0000038e, "IA32_PERF_GLOBAL_STATUS", Ia32PerfGlobalStatus, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x0000038f, "IA32_PERF_GLOBAL_CTRL", Ia32PerfGlobalCtrl, Ia32PerfGlobalCtrl, 0, 0, UINT64_C(0xfffffff8fffffff0)), /* value=0xf */
+    MFX(0x00000390, "IA32_PERF_GLOBAL_OVF_CTRL", Ia32PerfGlobalOvfCtrl, Ia32PerfGlobalOvfCtrl, 0, UINT64_C(0xe00000070000000f), UINT64_C(0x1ffffff8fffffff0)), /* value=0x0 */
+    MFX(0x00000391, "I7_UNC_PERF_GLOBAL_CTRL", IntelI7UncPerfGlobalCtrl, IntelI7UncPerfGlobalCtrl, 0, 0, UINT64_C(0xffffffff1fffffe0)), /* value=0x0 */
+    MFX(0x00000392, "I7_UNC_PERF_GLOBAL_STATUS", IntelI7UncPerfGlobalStatus, IntelI7UncPerfGlobalStatus, 0, 0xf, UINT64_C(0xfffffffffffffff0)), /* value=0x0 */
+    MFX(0x00000393, "I7_UNC_PERF_GLOBAL_OVF_CTRL", IntelI7UncPerfGlobalOvfCtrl, IntelI7UncPerfGlobalOvfCtrl, 0, 0x3, UINT64_C(0xfffffffffffffffc)), /* value=0x0 */
+    MFX(0x00000394, "I7_UNC_PERF_FIXED_CTR_CTRL", IntelI7UncPerfFixedCtrCtrl, IntelI7UncPerfFixedCtrCtrl, 0, 0, UINT64_C(0xffffffffffafffff)), /* value=0x0 */
+    MFX(0x00000395, "I7_UNC_PERF_FIXED_CTR", IntelI7UncPerfFixedCtr, IntelI7UncPerfFixedCtr, 0, 0, UINT64_C(0xffff000000000000)), /* value=0x0 */
+    MFO(0x00000396, "I7_UNC_CBO_CONFIG", IntelI7UncCBoxConfig), /* value=0x5 */
+    MVX(0x00000397, "I7_SB_UNK_0000_0397", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MFX(0x000003b0, "I7_UNC_ARB_PERF_CTR0", IntelI7UncArbPerfCtrN, IntelI7UncArbPerfCtrN, 0, 0, UINT64_C(0xfffff00000000000)), /* value=0x0 */
+    MFX(0x000003b1, "I7_UNC_ARB_PERF_CTR1", IntelI7UncArbPerfCtrN, IntelI7UncArbPerfCtrN, 0, 0, UINT64_C(0xfffff00000000000)), /* value=0x0 */
+    MFX(0x000003b2, "I7_UNC_ARB_PERF_EVT_SEL0", IntelI7UncArbPerfEvtSelN, IntelI7UncArbPerfEvtSelN, 0, 0, UINT64_C(0xffffffffe0230000)), /* value=0x0 */
+    MFX(0x000003b3, "I7_UNC_ARB_PERF_EVT_SEL1", IntelI7UncArbPerfEvtSelN, IntelI7UncArbPerfEvtSelN, 0, 0, UINT64_C(0xffffffffe0230000)), /* value=0x0 */
+    MFX(0x000003f1, "IA32_PEBS_ENABLE", Ia32PebsEnable, Ia32PebsEnable, 0, 0, UINT64_C(0x7ffffff0fffffff0)), /* value=0x0 */
+    MFX(0x000003f6, "I7_MSR_PEBS_LD_LAT", IntelI7PebsLdLat, IntelI7PebsLdLat, 0, UINT64_C(0xffffffffffff0000), 0), /* value=0xffff */
+    MFX(0x000003f8, "I7_MSR_PKG_C3_RESIDENCY", IntelI7PkgCnResidencyN, ReadOnly, 0x3, 0, UINT64_MAX), /* value=0x0 */
+    RSN(0x000003f9, 0x000003fa, "I7_MSR_PKG_Cn_RESIDENCY", IntelI7PkgCnResidencyN, ReadOnly, 0x6, 0, UINT64_MAX),
+    MFX(0x000003fc, "I7_MSR_CORE_C3_RESIDENCY", IntelI7CoreCnResidencyN, ReadOnly, 0x3, 0, UINT64_MAX), /* value=0x278ad50 */
+    RSN(0x000003fd, 0x000003fe, "I7_MSR_CORE_Cn_RESIDENCY", IntelI7CoreCnResidencyN, ReadOnly, 0x6, 0, UINT64_MAX),
+    RFN(0x00000400, 0x00000423, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0x00000480, "IA32_VMX_BASIC", Ia32VmxBasic, ReadOnly, UINT64_C(0xda040000000010), 0, 0), /* value=0xda0400`00000010 */
+    MFX(0x00000481, "IA32_VMX_PINBASED_CTLS", Ia32VmxPinbasedCtls, ReadOnly, UINT64_C(0x7f00000016), 0, 0), /* value=0x7f`00000016 */
+    MFX(0x00000482, "IA32_VMX_PROCBASED_CTLS", Ia32VmxProcbasedCtls, ReadOnly, UINT64_C(0xfff9fffe0401e172), 0, 0), /* value=0xfff9fffe`0401e172 */
+    MFX(0x00000483, "IA32_VMX_EXIT_CTLS", Ia32VmxExitCtls, ReadOnly, UINT64_C(0x7fffff00036dff), 0, 0), /* value=0x7fffff`00036dff */
+    MFX(0x00000484, "IA32_VMX_ENTRY_CTLS", Ia32VmxEntryCtls, ReadOnly, UINT64_C(0xffff000011ff), 0, 0), /* value=0xffff`000011ff */
+    MFX(0x00000485, "IA32_VMX_MISC", Ia32VmxMisc, ReadOnly, 0x100401e5, 0, 0), /* value=0x100401e5 */
+    MFX(0x00000486, "IA32_VMX_CR0_FIXED0", Ia32VmxCr0Fixed0, ReadOnly, UINT32_C(0x80000021), 0, 0), /* value=0x80000021 */
+    MFX(0x00000487, "IA32_VMX_CR0_FIXED1", Ia32VmxCr0Fixed1, ReadOnly, UINT32_MAX, 0, 0), /* value=0xffffffff */
+    MFX(0x00000488, "IA32_VMX_CR4_FIXED0", Ia32VmxCr4Fixed0, ReadOnly, 0x2000, 0, 0), /* value=0x2000 */
+    MFX(0x00000489, "IA32_VMX_CR4_FIXED1", Ia32VmxCr4Fixed1, ReadOnly, 0x627ff, 0, 0), /* value=0x627ff */
+    MFX(0x0000048a, "IA32_VMX_VMCS_ENUM", Ia32VmxVmcsEnum, ReadOnly, 0x2a, 0, 0), /* value=0x2a */
+    MFX(0x0000048b, "IA32_VMX_PROCBASED_CTLS2", Ia32VmxProcBasedCtls2, ReadOnly, UINT64_C(0xff00000000), 0, 0), /* value=0xff`00000000 */
+    MFX(0x0000048c, "IA32_VMX_EPT_VPID_CAP", Ia32VmxEptVpidCap, ReadOnly, UINT64_C(0xf0106114141), 0, 0), /* value=0xf01`06114141 */
+    MFX(0x0000048d, "IA32_VMX_TRUE_PINBASED_CTLS", Ia32VmxTruePinbasedCtls, ReadOnly, UINT64_C(0x7f00000016), 0, 0), /* value=0x7f`00000016 */
+    MFX(0x0000048e, "IA32_VMX_TRUE_PROCBASED_CTLS", Ia32VmxTrueProcbasedCtls, ReadOnly, UINT64_C(0xfff9fffe04006172), 0, 0), /* value=0xfff9fffe`04006172 */
+    MFX(0x0000048f, "IA32_VMX_TRUE_EXIT_CTLS", Ia32VmxTrueExitCtls, ReadOnly, UINT64_C(0x7fffff00036dfb), 0, 0), /* value=0x7fffff`00036dfb */
+    MFX(0x00000490, "IA32_VMX_TRUE_ENTRY_CTLS", Ia32VmxTrueEntryCtls, ReadOnly, UINT64_C(0xffff000011fb), 0, 0), /* value=0xffff`000011fb */
+    RSN(0x000004c1, 0x000004c4, "IA32_A_PMCn", Ia32PmcN, Ia32PmcN, 0x0, 0, UINT64_C(0xffff000000000000)),
+    MVO(0x00000502, "I7_SB_UNK_0000_0502", 0),
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0x0 */
+    MFX(0x00000601, "I7_SB_MSR_VR_CURRENT_CONFIG", IntelI7SandyVrCurrentConfig, IntelI7SandyVrCurrentConfig, 0, UINT32_C(0x80001fff), 0x7fffe000), /* value=0x18141494`8000030c */
+    MVX(0x00000602, "I7_IB_UNK_0000_0602", UINT64_C(0x1814149480000104), UINT32_C(0x80001fff), 0x7fffe000),
+    MFX(0x00000603, "I7_SB_MSR_VR_MISC_CONFIG", IntelI7SandyVrMiscConfig, IntelI7SandyVrMiscConfig, 0, UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)), /* value=0x80303030 */
+    MVX(0x00000604, "I7_IB_UNK_0000_0602", UINT32_C(0x80646464), UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)),
+    MFO(0x00000606, "I7_SB_MSR_RAPL_POWER_UNIT", IntelI7SandyRaplPowerUnit), /* value=0xa1003 */
+    MVX(0x00000609, "I7_SB_UNK_0000_0609", 0, 0, UINT64_C(0xffffffffffffff00)),
+    MFX(0x0000060a, "I7_SB_MSR_PKGC3_IRTL", IntelI7SandyPkgCnIrtlN, IntelI7SandyPkgCnIrtlN, 0x3, 0, UINT64_C(0xffffffffffff6000)), /* value=0x8c02 */
+    RSN(0x0000060b, 0x0000060c, "I7_SB_MSR_PKGC6_IRTn", IntelI7SandyPkgCnIrtlN, IntelI7SandyPkgCnIrtlN, 0x6, 0, UINT64_C(0xffffffffffff6000)),
+    MFO(0x0000060d, "I7_SB_MSR_PKG_C2_RESIDENCY", IntelI7SandyPkgC2Residency), /* value=0x11`06f311d4 */
+    MFX(0x00000610, "I7_SB_MSR_PKG_POWER_LIMIT", IntelI7RaplPkgPowerLimit, IntelI7RaplPkgPowerLimit, 0, UINT64_C(0x80ffffff00ffffff), UINT64_C(0x7f000000ff000000)), /* value=0x800001c2`00dc8168 */
+    MFO(0x00000611, "I7_SB_MSR_PKG_ENERGY_STATUS", IntelI7RaplPkgEnergyStatus), /* value=0x55a9ec99 */
+    MFO(0x00000614, "I7_SB_MSR_PKG_POWER_INFO", IntelI7RaplPkgPowerInfo), /* value=0x100240`01200168 */
+    MFX(0x00000638, "I7_SB_MSR_PP0_POWER_LIMIT", IntelI7RaplPp0PowerLimit, IntelI7RaplPp0PowerLimit, 0, UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)), /* value=0x80000000 */
+    MFO(0x00000639, "I7_SB_MSR_PP0_ENERGY_STATUS", IntelI7RaplPp0EnergyStatus), /* value=0x1dcdc9a0 */
+    MFX(0x0000063a, "I7_SB_MSR_PP0_POLICY", IntelI7RaplPp0Policy, IntelI7RaplPp0Policy, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x0 */
+    MFX(0x00000640, "I7_HW_MSR_PP0_POWER_LIMIT", IntelI7RaplPp1PowerLimit, IntelI7RaplPp1PowerLimit, 0, UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)), /* value=0x80000000 */
+    MFO(0x00000641, "I7_HW_MSR_PP0_ENERGY_STATUS", IntelI7RaplPp1EnergyStatus), /* value=0x39748b6 */
+    MFX(0x00000642, "I7_HW_MSR_PP0_POLICY", IntelI7RaplPp1Policy, IntelI7RaplPp1Policy, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x10 */
+    RFN(0x00000680, 0x0000068f, "MSR_LASTBRANCH_n_FROM_IP", IntelLastBranchFromN, IntelLastBranchFromN),
+    RFN(0x000006c0, 0x000006cf, "MSR_LASTBRANCH_n_TO_IP", IntelLastBranchToN, IntelLastBranchToN),
+    MFX(0x000006e0, "IA32_TSC_DEADLINE", Ia32TscDeadline, Ia32TscDeadline, 0, UINT64_C(0x1000000018), 0), /* value=0x94d`402e841f */
+    MVX(0x00000700, "MSR_UNC_CBO_0_PERFEVTSEL0", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000701, "MSR_UNC_CBO_0_PERFEVTSEL1", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000702, "MSR_UNC_CBO_0_PERFEVTSEL2?", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000703, "MSR_UNC_CBO_0_PERFEVTSEL3?", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000704, "MSR_UNC_CBO_0_UNK_4", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000705, "MSR_UNC_CBO_0_UNK_5", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000706, "MSR_UNC_CBO_0_PER_CTR0", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000707, "MSR_UNC_CBO_0_PER_CTR1", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000708, "MSR_UNC_CBO_0_PER_CTR2?", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000709, "MSR_UNC_CBO_0_PER_CTR3?", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000710, "MSR_UNC_CBO_1_PERFEVTSEL0", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000711, "MSR_UNC_CBO_1_PERFEVTSEL1", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000712, "MSR_UNC_CBO_1_PERFEVTSEL2?", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000713, "MSR_UNC_CBO_1_PERFEVTSEL3?", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000714, "MSR_UNC_CBO_1_UNK_4", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000715, "MSR_UNC_CBO_1_UNK_5", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000716, "MSR_UNC_CBO_1_PER_CTR0", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000717, "MSR_UNC_CBO_1_PER_CTR1", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000718, "MSR_UNC_CBO_1_PER_CTR2?", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000719, "MSR_UNC_CBO_1_PER_CTR3?", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000720, "MSR_UNC_CBO_2_PERFEVTSEL0", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000721, "MSR_UNC_CBO_2_PERFEVTSEL1", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000722, "MSR_UNC_CBO_2_PERFEVTSEL2?", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000723, "MSR_UNC_CBO_2_PERFEVTSEL3?", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000724, "MSR_UNC_CBO_2_UNK_4", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000725, "MSR_UNC_CBO_2_UNK_5", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000726, "MSR_UNC_CBO_2_PER_CTR0", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000727, "MSR_UNC_CBO_2_PER_CTR1", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000728, "MSR_UNC_CBO_2_PER_CTR2?", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000729, "MSR_UNC_CBO_2_PER_CTR3?", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000730, "MSR_UNC_CBO_3_PERFEVTSEL0", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000731, "MSR_UNC_CBO_3_PERFEVTSEL1", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000732, "MSR_UNC_CBO_3_PERFEVTSEL2?", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000733, "MSR_UNC_CBO_3_PERFEVTSEL3?", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000734, "MSR_UNC_CBO_3_UNK_4", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000735, "MSR_UNC_CBO_3_UNK_5", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000736, "MSR_UNC_CBO_3_PER_CTR0", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000737, "MSR_UNC_CBO_3_PER_CTR1", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000738, "MSR_UNC_CBO_3_PER_CTR2?", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000739, "MSR_UNC_CBO_3_PER_CTR3?", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000740, "MSR_UNC_CBO_4_PERFEVTSEL0?", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000741, "MSR_UNC_CBO_4_PERFEVTSEL1?", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000742, "MSR_UNC_CBO_4_PERFEVTSEL2?", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000743, "MSR_UNC_CBO_4_PERFEVTSEL3?", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000744, "MSR_UNC_CBO_4_UNK_4", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000745, "MSR_UNC_CBO_4_UNK_5", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000746, "MSR_UNC_CBO_4_PER_CTR0?", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000747, "MSR_UNC_CBO_4_PER_CTR1?", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000748, "MSR_UNC_CBO_4_PER_CTR2?", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000749, "MSR_UNC_CBO_4_PER_CTR3?", 0, 0, UINT64_C(0xfffff00000000000)),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xfffffffffffff2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x1b0008`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xffffff80`222f2fd0 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0x0 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x4700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x0 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffffff81`0500f000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x7fff`7b14d3f0 */
+    MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, 0, ~(uint64_t)UINT32_MAX), /* value=0x0 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Core(TM) i7-2635QM CPU @ 2.00GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Core_i7_2635QM =
+{
+    /*.pszName          = */ "Intel Core i7-2635QM",
+    /*.pszFullName      = */ "Intel(R) Core(TM) i7-2635QM CPU @ 2.00GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 42,
+    /*.uStepping        = */ 7,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_Core7_SandyBridge,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_100MHZ,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 36,
+    /*.fMxCsrMask       = */ 0xffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Core_i7_2635QM),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Core_i7_2635QM)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX,
+    /*.DefUnknownCpuId  = */ { 0x00000007, 0x00000340, 0x00000340, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Core_i7_2635QM)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Core_i7_2635QM),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Core_i7_2635QM_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_3820QM.h b/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_3820QM.h
new file mode 100644
index 00000000..c4ebd5a0
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_3820QM.h
@@ -0,0 +1,386 @@
+/* $Id: Intel_Core_i7_3820QM.h $ */
+/** @file
+ * CPU database entry "Intel Core i7-3820QM".
+ * Generated at 2013-12-04T12:54:32Z by VBoxCpuReport v4.3.51r91071 on darwin.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Core_i7_3820QM_h
+#define VBOX_CPUDB_Intel_Core_i7_3820QM_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Core(TM) i7-3820QM CPU @ 2.70GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Core_i7_3820QM[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000d, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000306a9, 0x02100800, 0x7fbae3ff, 0xbfebfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x76035a01, 0x00f0b2ff, 0x00000000, 0x00ca0000, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, 0x00000000, 0x1c004121, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00021120, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000077, 0x00000002, 0x00000009, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000281, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07300403, 0x00000000, 0x00000000, 0x00000603, 0 },
+    { 0x0000000b, 0x00000000, 0x00000000, 0x00000001, 0x00000002, 0x00000100, 0x00000002, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000000, 0x00000000, 0x00000007, 0x00000340, 0x00000340, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x28100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20202020, 0x6e492020, 0x286c6574, 0x43202952, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x2865726f, 0x20294d54, 0x332d3769, 0x51303238, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x5043204d, 0x20402055, 0x30372e32, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x01006040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000100, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003024, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Core(TM) i7-3820QM CPU @ 2.70GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Core_i7_3820QM[] =
+{
+    MFX(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr, Ia32P5McAddr, 0, UINT64_C(0xffffffffffffffe0), 0), /* value=0x1f */
+    MFX(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType, Ia32P5McType, 0, 0, UINT64_MAX), /* value=0x0 */
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, 0, UINT64_C(0xffffffffffff0000)), /* value=0x40 */
+    MFX(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter, 0, 0, 0),
+    MFV(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0x10000000000000)),
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00900), 0, UINT64_C(0xfffffff0000002ff)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MVX(0x0000002e, "I7_UNK_0000_002e", 0, 0x400, UINT64_C(0xfffffffffffffbff)),
+    MVX(0x00000033, "TEST_CTL", 0, 0, UINT64_C(0xffffffff7fffffff)),
+    MVO(0x00000034, "P6_UNK_0000_0034", 0xe),
+    MFO(0x00000035, "MSR_CORE_THREAD_COUNT", IntelI7CoreThreadCount), /* value=0x40008*/
+    MVO(0x00000036, "I7_UNK_0000_0036", 0x6c405eec),
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0xff07 */
+    MVX(0x0000003e, "I7_UNK_0000_003e", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, Ia32BiosUpdateTrigger),
+    MVX(0x0000008b, "BBL_CR_D3|BIOS_SIGN", UINT64_C(0x1500000000), 0x1, UINT32_C(0xfffffffe)),
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    MFX(0x000000c1, "IA32_PMC0", Ia32PmcN, Ia32PmcN, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x000000c2, "IA32_PMC1", Ia32PmcN, Ia32PmcN, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x000000c3, "IA32_PMC2", Ia32PmcN, Ia32PmcN, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x000000c4, "IA32_PMC3", Ia32PmcN, Ia32PmcN, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MVO(0x000000ce, "P6_UNK_0000_00ce", UINT64_C(0x80c10f0011b00)),
+    MFX(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl, IntelPkgCStConfigControl, 0, 0, UINT64_C(0xffffffffe1fffbf8)), /* value=0x8405 */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, 0, UINT64_C(0xfffffffffff80000)), /* value=0x20414 */
+    MFX(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf, 0, 0x47810, 0), /* value=0x6b`5d075e9c */
+    MFX(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf, 0, 0x1121880, 0), /* value=0x55`2bec768b */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0xd0a, 0, 0), /* value=0xd0a */
+    MVX(0x00000102, "I7_IVY_UNK_0000_0102", 0, 0, UINT64_C(0xffffffff7fff8000)),
+    MVX(0x00000103, "I7_IVY_UNK_0000_0103", 0, 0, UINT64_C(0xffffffffffffff00)),
+    MVX(0x00000104, "I7_IVY_UNK_0000_0104", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MVX(0x00000132, "I7_UNK_0000_0132", UINT64_MAX, 0, 0),
+    MVX(0x00000133, "I7_UNK_0000_0133", UINT64_MAX, 0, 0),
+    MVX(0x00000134, "I7_UNK_0000_0134", UINT64_MAX, 0, 0),
+    MVO(0x0000013c, "TODO_0000_013c", 0x1),
+    MVX(0x00000140, "I7_IVY_UNK_0000_0140", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MVX(0x00000142, "I7_IVY_UNK_0000_0142", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0xb */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0xffffff80`21af5080 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0xffffff80`214ce720 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0xc09, 0, 0), /* value=0xc09 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, 0, UINT64_C(0xfffffffffffffff8)), /* value=0x0 */
+    RSN(0x00000186, 0x00000189, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0, 0, UINT64_C(0xffffffff00080000)),
+    MVX(0x00000194, "CLOCK_FLEX_MAX", 0x180000, 0x1e00ff, UINT64_C(0xffffffffffe00000)),
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0x240700002400), 0, 0), /* value=0x2407`00002400 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x2500, 0, 0), /* Might bite. value=0x2500 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x0 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0x10, 0, UINT64_C(0xfffffffffe0000e8)), /* value=0x10 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, UINT32_C(0x88340000), UINT32_C(0xf87f0fff), UINT64_C(0xffffffff0780f000)), /* value=0x88340000 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, 0x850089, 0x1080, UINT64_C(0xffffffbbff3aef72)), /* value=0x850089 */
+    MFX(0x000001a2, "I7_MSR_TEMPERATURE_TARGET", IntelI7TemperatureTarget, IntelI7TemperatureTarget, 0x691200, 0xffff00, UINT64_C(0xfffffffff00000ff)), /* value=0x691200 */
+    MVX(0x000001a4, "I7_UNK_0000_01a4", 0, 0, UINT64_C(0xfffffffffffff7f0)),
+    RSN(0x000001a6, 0x000001a7, "I7_MSR_OFFCORE_RSP_n", IntelI7MsrOffCoreResponseN, IntelI7MsrOffCoreResponseN, 0, 0, UINT64_C(0xffffffc000007000)), /* XXX: The range ended earlier than expected! */
+    MVX(0x000001a8, "I7_UNK_0000_01a8", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFX(0x000001aa, "MSR_MISC_PWR_MGMT", IntelI7MiscPwrMgmt, IntelI7MiscPwrMgmt, 0, 0, UINT64_C(0xffffffffffbffffe)), /* value=0x400001 */
+    MVX(0x000001ad, "TODO_0000_01ad", 0x23232425, UINT32_MAX, ~(uint64_t)UINT32_MAX),
+    MVX(0x000001b0, "IA32_ENERGY_PERF_BIAS", 0x4, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x000001b1, "IA32_PACKAGE_THERM_STATUS", UINT32_C(0x88300000), UINT32_C(0xf87f0fff), UINT64_C(0xffffffff0780f000)),
+    MVX(0x000001b2, "IA32_PACKAGE_THERM_INTERRUPT", 0, 0, UINT64_C(0xfffffffffe0000e8)),
+    MVO(0x000001c6, "TODO_0000_01c6", 0x3),
+    MVX(0x000001c8, "TODO_0000_01c8", 0, 0, UINT64_C(0xfffffffffffffe00)),
+    MFX(0x000001c9, "MSR_LASTBRANCH_TOS", IntelLastBranchTos, IntelLastBranchTos, 0, 0, UINT64_C(0xfffffffffffffff0)), /* value=0x8 */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffff803c)), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0x7fffff7f`a38c2298 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xffffff80`214b24e0 */
+    MFN(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp, P6LastIntFromIp), /* value=0x0 */
+    MFN(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp, P6LastIntToIp), /* value=0x0 */
+    MVO(0x000001f0, "TODO_0000_01f0", 0x74),
+    MVO(0x000001f2, "TODO_0000_01f2", UINT32_C(0x8b000006)),
+    MVO(0x000001f3, "TODO_0000_01f3", UINT32_C(0xff800800)),
+    MVX(0x000001fc, "TODO_0000_01fc", 0x340047, 0x20, UINT64_C(0xffffffffffc20000)),
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xc0000000 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`c0000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xa0000000 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`e0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x90000000 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`f0000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x8c000000 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`fc000800 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x8b000000 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`ff000800 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x00000210, "IA32_MTRR_PHYS_BASE8", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x8, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000211, "IA32_MTRR_PHYS_MASK8", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x8, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x00000212, "IA32_MTRR_PHYS_BASE9", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x9, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000213, "IA32_MTRR_PHYS_MASK9", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x9, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MVX(0x00000280, "TODO_0000_0280", 0, 0, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x00000281, "TODO_0000_0281", 0, 0, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x00000282, "TODO_0000_0282", 0, 0x40007fff, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x00000283, "TODO_0000_0283", 0, 0, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x00000284, "TODO_0000_0284", 0, 0x40007fff, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x00000285, "TODO_0000_0285", 0, 0, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x00000286, "TODO_0000_0286", 0, 0, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x00000287, "TODO_0000_0287", 0, 0, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x00000288, "TODO_0000_0288", 0, 0, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x000002e0, "TODO_0000_02e0", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFN(0x000002e6, "TODO_0000_02e6", WriteOnly, IgnoreWrite),
+    MVX(0x000002e7, "TODO_0000_02e7", 0x1, 0x1, UINT64_C(0xfffffffffffffffe)),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    MVO(0x00000305, "TODO_0000_0305", 0),
+    MVX(0x00000309, "TODO_0000_0309", 0, 0, UINT64_C(0xffff000000000000)),
+    MVX(0x0000030a, "TODO_0000_030a", 0, 0, UINT64_C(0xffff000000000000)),
+    MVX(0x0000030b, "TODO_0000_030b", 0, 0, UINT64_C(0xffff000000000000)),
+    MVO(0x00000345, "TODO_0000_0345", 0x31c3),
+    MVX(0x0000038d, "TODO_0000_038d", 0, 0, UINT64_C(0xfffffffffffff000)),
+    MVO(0x0000038e, "TODO_0000_038e", UINT64_C(0x8000000000000000)),
+    MVX(0x0000038f, "TODO_0000_038f", 0xf, 0, UINT64_C(0xfffffff8fffffff0)),
+    MVX(0x00000390, "TODO_0000_0390", 0, UINT64_C(0xe00000070000000f), UINT64_C(0x1ffffff8fffffff0)),
+    MVX(0x00000391, "TODO_0000_0391", 0, 0, UINT64_C(0xffffffff1fffffe0)),
+    MVX(0x00000392, "TODO_0000_0392", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000393, "TODO_0000_0393", 0, 0x3, UINT64_C(0xfffffffffffffffc)),
+    MVX(0x00000394, "TODO_0000_0394", 0, 0, UINT64_C(0xffffffffffafffff)),
+    MVX(0x00000395, "TODO_0000_0395", 0, 0, UINT64_C(0xffff000000000000)),
+    MVO(0x00000396, "TODO_0000_0396", 0x5),
+    MVX(0x00000397, "TODO_0000_0397", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x000003b0, "TODO_0000_03b0", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x000003b1, "TODO_0000_03b1", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x000003b2, "TODO_0000_03b2", 0, 0, UINT64_C(0xffffffffc0230000)),
+    MVX(0x000003b3, "TODO_0000_03b3", 0, 0, UINT64_C(0xffffffffc0230000)),
+    MVX(0x000003f1, "TODO_0000_03f1", 0, 0, UINT64_C(0x7ffffff0fffffff0)),
+    MVX(0x000003f6, "TODO_0000_03f6", UINT16_MAX, UINT64_C(0xffffffffffff0000), 0),
+    MVO(0x000003f8, "TODO_0000_03f8", 0),
+    MVO(0x000003f9, "TODO_0000_03f9", UINT64_C(0x27495a818)),
+    MVO(0x000003fa, "TODO_0000_03fa", UINT64_C(0x428fa6c6207)),
+    MVO(0x000003fc, "TODO_0000_03fc", 0x389bb693),
+    MVO(0x000003fd, "TODO_0000_03fd", 0x13323393),
+    MVO(0x000003fe, "TODO_0000_03fe", UINT64_C(0x48d7ffc9bd1)),
+    RFN(0x00000400, 0x00000423, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MVO(0x00000480, "TODO_0000_0480", UINT64_C(0xda040000000010)),
+    MVO(0x00000481, "TODO_0000_0481", UINT64_C(0x7f00000016)),
+    MVO(0x00000482, "TODO_0000_0482", UINT64_C(0xfff9fffe0401e172)),
+    MVO(0x00000483, "TODO_0000_0483", UINT64_C(0x7fffff00036dff)),
+    MVO(0x00000484, "TODO_0000_0484", UINT64_C(0xffff000011ff)),
+    MVO(0x00000485, "TODO_0000_0485", 0x100401e5),
+    MVO(0x00000486, "TODO_0000_0486", UINT32_C(0x80000021)),
+    MVO(0x00000487, "TODO_0000_0487", UINT32_MAX),
+    MVO(0x00000488, "TODO_0000_0488", 0x2000),
+    MVO(0x00000489, "TODO_0000_0489", 0x1767ff),
+    MVO(0x0000048a, "TODO_0000_048a", 0x2a),
+    MVO(0x0000048b, "TODO_0000_048b", UINT64_C(0x8ff00000000)),
+    MVO(0x0000048c, "TODO_0000_048c", UINT64_C(0xf0106114141)),
+    MVO(0x0000048d, "TODO_0000_048d", UINT64_C(0x7f00000016)),
+    MVO(0x0000048e, "TODO_0000_048e", UINT64_C(0xfff9fffe04006172)),
+    MVO(0x0000048f, "TODO_0000_048f", UINT64_C(0x7fffff00036dfb)),
+    MVO(0x00000490, "TODO_0000_0490", UINT64_C(0xffff000011fb)),
+    MVX(0x000004c1, "TODO_0000_04c1", 0, 0, UINT64_C(0xffff000000000000)),
+    MVX(0x000004c2, "TODO_0000_04c2", 0, 0, UINT64_C(0xffff000000000000)),
+    MVX(0x000004c3, "TODO_0000_04c3", 0, 0, UINT64_C(0xffff000000000000)),
+    MVX(0x000004c4, "TODO_0000_04c4", 0, 0, UINT64_C(0xffff000000000000)),
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0x0 */
+    MVX(0x00000601, "TODO_0000_0601", UINT64_C(0x1814149480000380), UINT32_C(0x80001fff), 0x7fffe000),
+    MVX(0x00000602, "TODO_0000_0602", UINT64_C(0x1814149480000170), UINT32_C(0x80001fff), 0x7fffe000),
+    MVX(0x00000603, "TODO_0000_0603", UINT32_C(0x80303030), UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)),
+    MVX(0x00000604, "TODO_0000_0604", UINT32_C(0x80646464), UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)),
+    MVO(0x00000606, "TODO_0000_0606", 0xa1003),
+    MVX(0x0000060a, "TODO_0000_060a", 0x8894, 0, UINT64_C(0xffffffffffff6000)),
+    MVX(0x0000060b, "TODO_0000_060b", 0x88a9, 0, UINT64_C(0xffffffffffff6000)),
+    MVX(0x0000060c, "TODO_0000_060c", 0x88c6, 0, UINT64_C(0xffffffffffff6000)),
+    MVO(0x0000060d, "TODO_0000_060d", UINT64_C(0xd0fd23dd9)),
+    MVX(0x00000610, "TODO_0000_0610", UINT64_C(0x800083e800dd8320), UINT64_C(0x80ffffff00ffffff), UINT64_C(0x7f000000ff000000)),
+    MVO(0x00000611, "TODO_0000_0611", 0x2ed06e3b),
+    MVO(0x00000614, "TODO_0000_0614", 0x1200168),
+    MVX(0x00000638, "TODO_0000_0638", UINT32_C(0x80000000), UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)),
+    MVO(0x00000639, "TODO_0000_0639", 0x106344fd),
+    MVX(0x0000063a, "TODO_0000_063a", 0, 0, UINT64_C(0xffffffffffffffe0)),
+    MVX(0x00000640, "TODO_0000_0640", UINT32_C(0x80000000), UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)),
+    MVO(0x00000641, "TODO_0000_0641", 0xb39e93),
+    MVX(0x00000642, "TODO_0000_0642", 0x10, 0, UINT64_C(0xffffffffffffffe0)),
+    MVO(0x00000648, "TODO_0000_0648", 0x1b),
+    MVO(0x00000649, "TODO_0000_0649", UINT64_C(0x120000000000000)),
+    MVO(0x0000064a, "TODO_0000_064a", UINT64_C(0x120000000000000)),
+    MVO(0x0000064b, "TODO_0000_064b", UINT32_C(0x80000000)),
+    MVX(0x0000064c, "TODO_0000_064c", UINT32_C(0x80000000), UINT32_C(0x800000ff), UINT64_C(0xffffffff7fffff00)),
+    MVX(0x00000680, "TODO_0000_0680", 0, 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x00000681, "TODO_0000_0681", 0, 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x00000682, "TODO_0000_0682", UINT64_C(0x7fffff7fa38c2289), 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x00000683, "TODO_0000_0683", UINT64_C(0x7fffff80214b24cb), 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x00000684, "TODO_0000_0684", UINT64_C(0x7fffff7fa38c2298), 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x00000685, "TODO_0000_0685", UINT64_C(0x7fffff80214b24ee), 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x00000686, "TODO_0000_0686", UINT64_C(0x7fffff7fa38c2289), 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x00000687, "TODO_0000_0687", UINT64_C(0x7fffff80214b24cb), 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x00000688, "TODO_0000_0688", UINT64_C(0x7fffff7fa38c2298), 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x00000689, "TODO_0000_0689", 0, 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x0000068a, "TODO_0000_068a", 0, 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x0000068b, "TODO_0000_068b", 0, 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x0000068c, "TODO_0000_068c", 0, 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x0000068d, "TODO_0000_068d", 0, 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x0000068e, "TODO_0000_068e", 0, 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x0000068f, "TODO_0000_068f", 0, 0, UINT64_C(0x7fff800000000000)),
+    MVX(0x000006c0, "TODO_0000_06c0", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006c1, "TODO_0000_06c1", UINT64_C(0xffffff7fa38c227f), 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006c2, "TODO_0000_06c2", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006c3, "TODO_0000_06c3", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006c4, "TODO_0000_06c4", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006c5, "TODO_0000_06c5", UINT64_C(0xffffff7fa38c227f), 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006c6, "TODO_0000_06c6", UINT64_C(0xffffff80214b24c0), 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006c7, "TODO_0000_06c7", UINT64_C(0xffffff7fa38c228f), 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006c8, "TODO_0000_06c8", UINT64_C(0xffffff80214b24e0), 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006c9, "TODO_0000_06c9", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006ca, "TODO_0000_06ca", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006cb, "TODO_0000_06cb", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006cc, "TODO_0000_06cc", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006cd, "TODO_0000_06cd", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006ce, "TODO_0000_06ce", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006cf, "TODO_0000_06cf", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006e0, "TODO_0000_06e0", UINT64_C(0x535157ca1ca), 0x80000, 0),
+    MVX(0x00000700, "TODO_0000_0700", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000701, "TODO_0000_0701", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000702, "TODO_0000_0702", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000703, "TODO_0000_0703", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000704, "TODO_0000_0704", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000705, "TODO_0000_0705", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000706, "TODO_0000_0706", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000707, "TODO_0000_0707", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000708, "TODO_0000_0708", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000709, "TODO_0000_0709", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000710, "TODO_0000_0710", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000711, "TODO_0000_0711", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000712, "TODO_0000_0712", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000713, "TODO_0000_0713", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000714, "TODO_0000_0714", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000715, "TODO_0000_0715", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000716, "TODO_0000_0716", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000717, "TODO_0000_0717", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000718, "TODO_0000_0718", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000719, "TODO_0000_0719", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000720, "TODO_0000_0720", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000721, "TODO_0000_0721", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000722, "TODO_0000_0722", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000723, "TODO_0000_0723", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000724, "TODO_0000_0724", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000725, "TODO_0000_0725", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000726, "TODO_0000_0726", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000727, "TODO_0000_0727", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000728, "TODO_0000_0728", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000729, "TODO_0000_0729", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000730, "TODO_0000_0730", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000731, "TODO_0000_0731", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000732, "TODO_0000_0732", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000733, "TODO_0000_0733", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000734, "TODO_0000_0734", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000735, "TODO_0000_0735", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000736, "TODO_0000_0736", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000737, "TODO_0000_0737", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000738, "TODO_0000_0738", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000739, "TODO_0000_0739", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000740, "TODO_0000_0740", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000741, "TODO_0000_0741", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000742, "TODO_0000_0742", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000743, "TODO_0000_0743", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000744, "TODO_0000_0744", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000745, "TODO_0000_0745", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000746, "TODO_0000_0746", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000747, "TODO_0000_0747", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000748, "TODO_0000_0748", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000749, "TODO_0000_0749", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000c80, "TODO_0000_0c80", 0, 0, 0),
+    MVX(0x00000c81, "TODO_0000_0c81", 0, 0, 0),
+    MVX(0x00000c82, "TODO_0000_0c82", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00000c83, "TODO_0000_0c83", 0, ~(uint64_t)UINT32_MAX, 0),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xfffffffffffff2fe)),
+    MFX(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget, 0, 0, 0), /* value=0x1b0008`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xffffff80`214ce6c0 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0x0 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x4700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x0 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffffff81`e942f000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x7fff`7ccad1e0 */
+    MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, 0, ~(uint64_t)UINT32_MAX), /* value=0x0 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Core(TM) i7-3820QM CPU @ 2.70GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Core_i7_3820QM =
+{
+    /*.pszName          = */ "Intel Core i7-3820QM",
+    /*.pszFullName      = */ "Intel(R) Core(TM) i7-3820QM CPU @ 2.70GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 58,
+    /*.uStepping        = */ 9,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_Core7_IvyBridge,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 36,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Core_i7_3820QM),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Core_i7_3820QM)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX,
+    /*.DefUnknownCpuId  = */ { 0x00000007, 0x00000340, 0x00000340, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.apaMsrRanges[]   = */
+    {
+        NULL_ALONE(g_aMsrRanges_Intel_Core_i7_3820QM),
+        NULL,
+        NULL,
+        NULL,
+        NULL,
+    }
+};
+
+#endif /* !VBOX_CPUDB_Intel_Core_i7_3820QM_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_3960X.h b/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_3960X.h
new file mode 100644
index 00000000..4512358a
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_3960X.h
@@ -0,0 +1,369 @@
+/* $Id: Intel_Core_i7_3960X.h $ */
+/** @file
+ * CPU database entry "Intel Core i7-3960X".
+ * Generated at 2013-12-12T15:29:11Z by VBoxCpuReport v4.3.53r91237 on win.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Core_i7_3960X_h
+#define VBOX_CPUDB_Intel_Core_i7_3960X_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Core(TM) i7-3960X CPU @ 3.30GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Core_i7_3960X[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000d, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000206d6, 0x02200800, 0x1fbee3bf, 0xbfebfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x76035a01, 0x00f0b2ff, 0x00000000, 0x00ca0000, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, UINT32_MAX, 0x3c004121, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000004, 0x00000001, UINT32_MAX, 0x3c004122, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000004, 0x00000002, UINT32_MAX, 0x3c004143, 0x01c0003f, 0x000001ff, 0x00000000, 0 },
+    { 0x00000004, 0x00000003, UINT32_MAX, 0x3c07c163, 0x04c0003f, 0x00002fff, 0x00000006, 0 },
+    { 0x00000004, 0x00000004, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00021120, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000077, 0x00000002, 0x00000001, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000001, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07300403, 0x00000000, 0x00000000, 0x00000603, 0 },
+    { 0x0000000b, 0x00000000, UINT32_MAX, 0x00000001, 0x00000002, 0x00000100, 0x00000002, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000b, 0x00000001, UINT32_MAX, 0x00000005, 0x0000000c, 0x00000201, 0x00000002, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000b, 0x00000002, UINT32_MAX, 0x00000000, 0x00000000, 0x00000002, 0x00000002, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000000, UINT32_MAX, 0x00000007, 0x00000340, 0x00000340, 0x00000000, 0 },
+    { 0x0000000d, 0x00000001, UINT32_MAX, 0x00000001, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000002, UINT32_MAX, 0x00000100, 0x00000240, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000003, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x2c100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20202020, 0x49202020, 0x6c65746e, 0x20295228, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x65726f43, 0x294d5428, 0x2d376920, 0x30363933, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x50432058, 0x20402055, 0x30332e33, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x01006040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000100, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x0000302e, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Core(TM) i7-3960X CPU @ 3.30GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Core_i7_3960X[] =
+{
+    MFX(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr, Ia32P5McAddr, 0, UINT64_C(0xffffffffffffffe0), 0), /* value=0x1f */
+    MFX(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType, Ia32P5McType, 0, 0, UINT64_MAX), /* value=0x0 */
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, 0, UINT64_C(0xffffffffffff0000)), /* value=0x40 */
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x177ab4`48466b19 */
+    MFV(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0x8000000000000)),
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0, UINT64_C(0xffffc000000002ff)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MVX(0x0000002e, "I7_UNK_0000_002e", 0, 0x400, UINT64_C(0xfffffffffffffbff)),
+    MVX(0x00000033, "TEST_CTL", 0, 0, UINT64_C(0xffffffff7fffffff)),
+    MVO(0x00000034, "P6_UNK_0000_0034", 0x4cb),
+    MFO(0x00000035, "MSR_CORE_THREAD_COUNT", IntelI7CoreThreadCount), /* value=0x6000c*/
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x5 */
+    MVX(0x0000003e, "I7_UNK_0000_003e", 0x1, 0, UINT64_C(0xfffffffffffffffe)),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, Ia32BiosUpdateTrigger),
+    MVX(0x0000008b, "BBL_CR_D3|BIOS_SIGN", UINT64_C(0x61600000000), 0, UINT32_C(0xfffffffe)),
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    RSN(0x000000c1, 0x000000c4, "IA32_PMCn", Ia32PmcN, Ia32PmcN, 0x0, ~(uint64_t)UINT32_MAX, 0),
+    MFO(0x000000ce, "MSR_PLATFORM_INFO", IntelPlatformInfo), /* value=0xc00'70012100*/
+    MFX(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl, IntelPkgCStConfigControl, 0, 0, UINT64_C(0xffffffffe1ffffff)), /* value=0x1e008400 */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, 0, UINT64_C(0xfffffffffff80000)), /* value=0x20414 */
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf), /* value=0x2be98e4 */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x2d84ced */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0xd0a, 0, 0), /* value=0xd0a */
+    MFN(0x00000132, "CPUID1_FEATURE_MASK", IntelCpuId1FeatureMaskEax, IntelCpuId1FeatureMaskEax), /* value=0xffffffff`ffffffff */
+    MFN(0x00000133, "CPUIDD_01_FEATURE_MASK", IntelCpuId1FeatureMaskEcdx, IntelCpuId1FeatureMaskEcdx), /* value=0xffffffff`ffffffff */
+    MFN(0x00000134, "CPUID80000001_FEATURE_MASK", IntelCpuId80000001FeatureMaskEcdx, IntelCpuId80000001FeatureMaskEcdx), /* value=0xffffffff`ffffffff */
+    MFO(0x0000013c, "I7_SB_AES_NI_CTL", IntelI7SandyAesNiCtl), /* value=0x1 */
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0x0 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0x0 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0xc12, 0, 0), /* value=0xc12 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, 0, UINT64_C(0xfffffffffffffff8)), /* value=0x0 */
+    MFX(0x0000017f, "I7_SB_ERROR_CONTROL", IntelI7SandyErrorControl, IntelI7SandyErrorControl, 0, 0xc, UINT64_C(0xffffffffffffffe1)), /* value=0x0 */
+    RSN(0x00000186, 0x00000189, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, 0, UINT64_C(0xffffffff00080000)),
+    MFX(0x00000194, "CLOCK_FLEX_MAX", IntelFlexRatio, IntelFlexRatio, 0xf2100, 0xe0000, UINT64_C(0xfffffffffff00000)),
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0x288300002400), 0, 0), /* value=0x2883`00002400 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x2700, 0, 0), /* Might bite. value=0x2700 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x0 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0, 0, UINT64_C(0xfffffffffe0000e8)), /* value=0x0 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, UINT32_C(0x88380000), UINT32_C(0xf87f0fff), UINT64_C(0xffffffff0780f000)), /* value=0x88380000 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, 0x850089, 0x1080, UINT64_C(0xffffffbbff3aef72)), /* value=0x850089 */
+    MFX(0x000001a2, "I7_MSR_TEMPERATURE_TARGET", IntelI7TemperatureTarget, IntelI7TemperatureTarget, 0x5b0a00, 0xffff00, UINT64_C(0xfffffffff00000ff)), /* value=0x5b0a00 */
+    MVX(0x000001a4, "I7_UNK_0000_01a4", 0, 0, UINT64_C(0xfffffffffffff7f0)),
+    RSN(0x000001a6, 0x000001a7, "I7_MSR_OFFCORE_RSP_n", IntelI7MsrOffCoreResponseN, IntelI7MsrOffCoreResponseN, 0x0, 0, UINT64_C(0xffffffc000007000)),
+    MVX(0x000001a8, "I7_UNK_0000_01a8", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFX(0x000001aa, "MSR_MISC_PWR_MGMT", IntelI7MiscPwrMgmt, IntelI7MiscPwrMgmt, 0, 0, UINT64_C(0xffffffffffbffffe)), /* value=0x400000 */
+    MFX(0x000001ad, "I7_MSR_TURBO_RATIO_LIMIT", IntelI7TurboRatioLimit, IntelI7TurboRatioLimit, UINT64_C(0x2424242425252727), 0, 0), /* value=0x24242424`25252727 */
+    MVX(0x000001b1, "IA32_PACKAGE_THERM_STATUS", UINT32_C(0x88310000), UINT32_C(0xf87f0fff), UINT64_C(0xffffffff0780f000)),
+    MVX(0x000001b2, "IA32_PACKAGE_THERM_INTERRUPT", 0, 0, UINT64_C(0xfffffffffe0000e8)),
+    MVO(0x000001c6, "I7_UNK_0000_01c6", 0x3),
+    MFX(0x000001c8, "MSR_LBR_SELECT", IntelI7LbrSelect, IntelI7LbrSelect, 0, 0, UINT64_C(0xfffffffffffffe00)), /* value=0x0 */
+    MFX(0x000001c9, "MSR_LASTBRANCH_TOS", IntelLastBranchTos, IntelLastBranchTos, 0, 0, UINT64_C(0xfffffffffffffff0)), /* value=0xc */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffff803c)), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0x7ffff880`093814ea */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xfffff880`093a60e0 */
+    MFN(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp, P6LastIntFromIp), /* value=0x0 */
+    MFN(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp, P6LastIntToIp), /* value=0x0 */
+    MVO(0x000001e1, "I7_SB_UNK_0000_01e1", 0x2),
+    MVX(0x000001ef, "I7_SB_UNK_0000_01ef", 0xff, 0, UINT64_MAX),
+    MFO(0x000001f0, "I7_VLW_CAPABILITY", IntelI7VirtualLegacyWireCap), /* value=0x74 */
+    MFO(0x000001f2, "IA32_SMRR_PHYSBASE", Ia32SmrrPhysBase), /* value=0xad800006 */
+    MFO(0x000001f3, "IA32_SMRR_PHYSMASK", Ia32SmrrPhysMask), /* value=0xff800800 */
+    MFX(0x000001f8, "IA32_PLATFORM_DCA_CAP", Ia32PlatformDcaCap, Ia32PlatformDcaCap, 0, 0, UINT64_C(0xfffffffffffffffe)), /* value=0x1 */
+    MFO(0x000001f9, "IA32_CPU_DCA_CAP", Ia32CpuDcaCap), /* value=0x1 */
+    MFX(0x000001fa, "IA32_DCA_0_CAP", Ia32Dca0Cap, Ia32Dca0Cap, 0, 0x40007ff, UINT64_C(0xfffffffffafe1800)), /* value=0x1e489 */
+    MFX(0x000001fc, "I7_MSR_POWER_CTL", IntelI7PowerCtl, IntelI7PowerCtl, 0, 0, UINT64_C(0xffffffff00320020)), /* value=0x2500005b */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xffffc00000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xffffc000000007ff)), /* value=0x3ffc`00000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xffffc00000000ff8)), /* value=0x4`00000006 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xffffc000000007ff)), /* value=0x3fff`c0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xffffc00000000ff8)), /* value=0x4`40000006 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xffffc000000007ff)), /* value=0x3fff`f0000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xffffc00000000ff8)), /* value=0xae000000 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xffffc000000007ff)), /* value=0x3fff`fe000800 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xffffc00000000ff8)), /* value=0xb0000000 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xffffc000000007ff)), /* value=0x3fff`f0000800 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xffffc00000000ff8)), /* value=0xc0000000 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xffffc000000007ff)), /* value=0x3fff`c0000800 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xffffc00000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xffffc000000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xffffc00000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xffffc000000007ff)), /* value=0x0 */
+    MFX(0x00000210, "IA32_MTRR_PHYS_BASE8", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x8, 0, UINT64_C(0xffffc00000000ff8)), /* value=0x0 */
+    MFX(0x00000211, "IA32_MTRR_PHYS_MASK8", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x8, 0, UINT64_C(0xffffc000000007ff)), /* value=0x0 */
+    MFX(0x00000212, "IA32_MTRR_PHYS_BASE9", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x9, 0, UINT64_C(0xffffc00000000ff8)), /* value=0x0 */
+    MFX(0x00000213, "IA32_MTRR_PHYS_MASK9", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x9, 0, UINT64_C(0xffffc000000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    RSN(0x00000280, 0x00000281, "IA32_MC0_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0x0, 0, UINT64_C(0xffffffffbfff8000)),
+    MFX(0x00000282, "IA32_MC2_CTL2", Ia32McNCtl2, Ia32McNCtl2, 0x2, 0x40007fff, UINT64_C(0xffffffffbfff8000)), /* value=0x0 */
+    MFX(0x00000283, "IA32_MC3_CTL2", Ia32McNCtl2, Ia32McNCtl2, 0x3, 0, UINT64_C(0xffffffffbfff8000)), /* value=0x40000001 */
+    MFX(0x00000284, "IA32_MC4_CTL2", Ia32McNCtl2, Ia32McNCtl2, 0x4, 0x40007fff, UINT64_C(0xffffffffbfff8000)), /* value=0x0 */
+    RSN(0x00000285, 0x00000287, "IA32_MC5_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0x5, 0, UINT64_C(0xffffffffbfff8000)),
+    RSN(0x00000288, 0x0000028b, "IA32_MC8_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0x8, 0x1, UINT64_C(0xffffffffbfff8000)),
+    RSN(0x0000028c, 0x00000291, "IA32_MC12_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0xc, 0, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x000002e0, "I7_SB_NO_EVICT_MODE", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    MVO(0x00000300, "I7_SB_UNK_0000_0300", UINT32_C(0x8000ff00)),
+    MVO(0x00000305, "I7_SB_UNK_0000_0305", 0),
+    RSN(0x00000309, 0x0000030b, "IA32_FIXED_CTRn", Ia32FixedCtrN, Ia32FixedCtrN, 0x0, 0, UINT64_C(0xffff000000000000)),
+    MFX(0x00000345, "IA32_PERF_CAPABILITIES", Ia32PerfCapabilities, ReadOnly, 0x31c3, 0, 0), /* value=0x31c3 */
+    MFX(0x0000038d, "IA32_FIXED_CTR_CTRL", Ia32FixedCtrCtrl, Ia32FixedCtrCtrl, 0, 0, UINT64_C(0xfffffffffffff000)), /* value=0x0 */
+    MFX(0x0000038e, "IA32_PERF_GLOBAL_STATUS", Ia32PerfGlobalStatus, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x0000038f, "IA32_PERF_GLOBAL_CTRL", Ia32PerfGlobalCtrl, Ia32PerfGlobalCtrl, 0, 0, UINT64_C(0xfffffff8fffffff0)), /* value=0xf */
+    MFX(0x00000390, "IA32_PERF_GLOBAL_OVF_CTRL", Ia32PerfGlobalOvfCtrl, Ia32PerfGlobalOvfCtrl, 0, UINT64_C(0xe00000070000000f), UINT64_C(0x1ffffff8fffffff0)), /* value=0x0 */
+    MFX(0x0000039c, "I7_SB_MSR_PEBS_NUM_ALT", IntelI7SandyPebsNumAlt, IntelI7SandyPebsNumAlt, 0, 0, UINT64_C(0xfffffffffffffffe)), /* value=0x0 */
+    MFX(0x000003f1, "IA32_PEBS_ENABLE", Ia32PebsEnable, Ia32PebsEnable, 0, 0, UINT64_C(0x7ffffff0fffffff0)), /* value=0x0 */
+    MFX(0x000003f6, "I7_MSR_PEBS_LD_LAT", IntelI7PebsLdLat, IntelI7PebsLdLat, 0, UINT64_C(0xffffffffffff0000), 0), /* value=0xffff */
+    MFX(0x000003f8, "I7_MSR_PKG_C3_RESIDENCY", IntelI7PkgCnResidencyN, ReadOnly, 0x3, 0, UINT64_MAX), /* value=0x0 */
+    RSN(0x000003f9, 0x000003fa, "I7_MSR_PKG_Cn_RESIDENCY", IntelI7PkgCnResidencyN, ReadOnly, 0x6, 0, UINT64_MAX),
+    MFX(0x000003fc, "I7_MSR_CORE_C3_RESIDENCY", IntelI7CoreCnResidencyN, ReadOnly, 0x3, 0, UINT64_MAX), /* value=0x3f8f`5718a87c */
+    RSN(0x000003fd, 0x000003fe, "I7_MSR_CORE_Cn_RESIDENCY", IntelI7CoreCnResidencyN, ReadOnly, 0x6, 0, UINT64_MAX),
+    RFN(0x00000400, 0x00000447, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0x00000480, "IA32_VMX_BASIC", Ia32VmxBasic, ReadOnly, UINT64_C(0xda040000000010), 0, 0), /* value=0xda0400`00000010 */
+    MFX(0x00000481, "IA32_VMX_PINBASED_CTLS", Ia32VmxPinbasedCtls, ReadOnly, UINT64_C(0x7f00000016), 0, 0), /* value=0x7f`00000016 */
+    MFX(0x00000482, "IA32_VMX_PROCBASED_CTLS", Ia32VmxProcbasedCtls, ReadOnly, UINT64_C(0xfff9fffe0401e172), 0, 0), /* value=0xfff9fffe`0401e172 */
+    MFX(0x00000483, "IA32_VMX_EXIT_CTLS", Ia32VmxExitCtls, ReadOnly, UINT64_C(0x7fffff00036dff), 0, 0), /* value=0x7fffff`00036dff */
+    MFX(0x00000484, "IA32_VMX_ENTRY_CTLS", Ia32VmxEntryCtls, ReadOnly, UINT64_C(0xffff000011ff), 0, 0), /* value=0xffff`000011ff */
+    MFX(0x00000485, "IA32_VMX_MISC", Ia32VmxMisc, ReadOnly, 0x100401e5, 0, 0), /* value=0x100401e5 */
+    MFX(0x00000486, "IA32_VMX_CR0_FIXED0", Ia32VmxCr0Fixed0, ReadOnly, UINT32_C(0x80000021), 0, 0), /* value=0x80000021 */
+    MFX(0x00000487, "IA32_VMX_CR0_FIXED1", Ia32VmxCr0Fixed1, ReadOnly, UINT32_MAX, 0, 0), /* value=0xffffffff */
+    MFX(0x00000488, "IA32_VMX_CR4_FIXED0", Ia32VmxCr4Fixed0, ReadOnly, 0x2000, 0, 0), /* value=0x2000 */
+    MFX(0x00000489, "IA32_VMX_CR4_FIXED1", Ia32VmxCr4Fixed1, ReadOnly, 0x627ff, 0, 0), /* value=0x627ff */
+    MFX(0x0000048a, "IA32_VMX_VMCS_ENUM", Ia32VmxVmcsEnum, ReadOnly, 0x2a, 0, 0), /* value=0x2a */
+    MFX(0x0000048b, "IA32_VMX_PROCBASED_CTLS2", Ia32VmxProcBasedCtls2, ReadOnly, UINT64_C(0x4ff00000000), 0, 0), /* value=0x4ff`00000000 */
+    MFX(0x0000048c, "IA32_VMX_EPT_VPID_CAP", Ia32VmxEptVpidCap, ReadOnly, UINT64_C(0xf0106134141), 0, 0), /* value=0xf01`06134141 */
+    MFX(0x0000048d, "IA32_VMX_TRUE_PINBASED_CTLS", Ia32VmxTruePinbasedCtls, ReadOnly, UINT64_C(0x7f00000016), 0, 0), /* value=0x7f`00000016 */
+    MFX(0x0000048e, "IA32_VMX_TRUE_PROCBASED_CTLS", Ia32VmxTrueProcbasedCtls, ReadOnly, UINT64_C(0xfff9fffe04006172), 0, 0), /* value=0xfff9fffe`04006172 */
+    MFX(0x0000048f, "IA32_VMX_TRUE_EXIT_CTLS", Ia32VmxTrueExitCtls, ReadOnly, UINT64_C(0x7fffff00036dfb), 0, 0), /* value=0x7fffff`00036dfb */
+    MFX(0x00000490, "IA32_VMX_TRUE_ENTRY_CTLS", Ia32VmxTrueEntryCtls, ReadOnly, UINT64_C(0xffff000011fb), 0, 0), /* value=0xffff`000011fb */
+    RSN(0x000004c1, 0x000004c4, "IA32_A_PMCn", Ia32PmcN, Ia32PmcN, 0x0, 0, UINT64_C(0xffff000000000000)),
+    MVO(0x00000502, "I7_SB_UNK_0000_0502", 0),
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0x0 */
+    MFX(0x00000601, "I7_SB_MSR_VR_CURRENT_CONFIG", IntelI7SandyVrCurrentConfig, IntelI7SandyVrCurrentConfig, 0, UINT32_C(0x80001fff), 0x7fffe000), /* value=0x141494`80000640 */
+    MFX(0x00000603, "I7_SB_MSR_VR_MISC_CONFIG", IntelI7SandyVrMiscConfig, IntelI7SandyVrMiscConfig, 0, UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)), /* value=0x80151515 */
+    MFO(0x00000606, "I7_SB_MSR_RAPL_POWER_UNIT", IntelI7SandyRaplPowerUnit), /* value=0xa1003 */
+    MFX(0x0000060a, "I7_SB_MSR_PKGC3_IRTL", IntelI7SandyPkgCnIrtlN, IntelI7SandyPkgCnIrtlN, 0x3, 0, UINT64_C(0xffffffffffff6000)), /* value=0x0 */
+    RSN(0x0000060b, 0x0000060c, "I7_SB_MSR_PKGC6_IRTn", IntelI7SandyPkgCnIrtlN, IntelI7SandyPkgCnIrtlN, 0x6, 0, UINT64_C(0xffffffffffff6000)),
+    MFO(0x0000060d, "I7_SB_MSR_PKG_C2_RESIDENCY", IntelI7SandyPkgC2Residency), /* value=0x0 */
+    MFX(0x00000610, "I7_SB_MSR_PKG_POWER_LIMIT", IntelI7RaplPkgPowerLimit, IntelI7RaplPkgPowerLimit, 0, UINT64_C(0x80ffffff00ffffff), UINT64_C(0x7f000000ff000000)), /* value=0x80068960`005affff */
+    MFO(0x00000611, "I7_SB_MSR_PKG_ENERGY_STATUS", IntelI7RaplPkgEnergyStatus), /* value=0xc120ff02 */
+    MFO(0x00000613, "I7_SB_MSR_PKG_PERF_STATUS", IntelI7RaplPkgPerfStatus), /* value=0x0 */
+    MFO(0x00000614, "I7_SB_MSR_PKG_POWER_INFO", IntelI7RaplPkgPowerInfo), /* value=0x1a80410 */
+    MFX(0x00000618, "I7_SB_MSR_DRAM_POWER_LIMIT", IntelI7RaplDramPowerLimit, IntelI7RaplDramPowerLimit, 0, UINT32_C(0x80feffff), UINT64_C(0xffffffff7f010000)), /* value=0x80000000 */
+    MFO(0x00000619, "I7_SB_MSR_DRAM_ENERGY_STATUS", IntelI7RaplDramEnergyStatus), /* value=0x0 */
+    MFO(0x0000061b, "I7_SB_MSR_DRAM_PERF_STATUS", IntelI7RaplDramPerfStatus), /* value=0x0 */
+    MFO(0x0000061c, "I7_SB_MSR_DRAM_POWER_INFO", IntelI7RaplDramPowerInfo), /* value=0x280258`00780118 */
+    MFX(0x00000638, "I7_SB_MSR_PP0_POWER_LIMIT", IntelI7RaplPp0PowerLimit, IntelI7RaplPp0PowerLimit, 0, UINT32_C(0x80ffffff), UINT64_C(0xffffffff7f000000)), /* value=0x80000000 */
+    MFO(0x00000639, "I7_SB_MSR_PP0_ENERGY_STATUS", IntelI7RaplPp0EnergyStatus), /* value=0x448bc04 */
+    MFX(0x0000063a, "I7_SB_MSR_PP0_POLICY", IntelI7RaplPp0Policy, IntelI7RaplPp0Policy, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x0 */
+    MFO(0x0000063b, "I7_SB_MSR_PP0_PERF_STATUS", IntelI7RaplPp0PerfStatus), /* value=0x0 */
+    RFN(0x00000680, 0x0000068f, "MSR_LASTBRANCH_n_FROM_IP", IntelLastBranchFromN, IntelLastBranchFromN),
+    RFN(0x000006c0, 0x000006cf, "MSR_LASTBRANCH_n_TO_IP", IntelLastBranchFromN, IntelLastBranchFromN),
+    MFI(0x000006e0, "IA32_TSC_DEADLINE", Ia32TscDeadline), /* value=0x0 */
+    MVX(0x00000a00, "I7_SB_UNK_0000_0a00", 0, 0, UINT64_C(0xfffffffffffffec0)),
+    MVX(0x00000a01, "I7_SB_UNK_0000_0a01", 0x178fa000, 0, UINT64_C(0xffffffff00000f80)),
+    MVX(0x00000a02, "I7_SB_UNK_0000_0a02", 0, 0, UINT64_C(0xffffffff20002000)),
+    MVX(0x00000c00, "I7_SB_UNK_0000_0c00", 0, 0, UINT64_C(0xffffffffbfffff00)),
+    MVX(0x00000c01, "I7_SB_UNK_0000_0c01", 0, 0x9229fe7, UINT64_C(0xfffffffff6dd6018)),
+    MVO(0x00000c06, "I7_SB_UNK_0000_0c06", 0x6),
+    MVX(0x00000c08, "I7_SB_UNK_0000_0c08", 0, 0, UINT64_C(0xffffffffffafffff)),
+    MVX(0x00000c09, "I7_SB_UNK_0000_0c09", 0x301a, 0, UINT64_C(0xffff000000000000)),
+    MVX(0x00000c10, "I7_SB_UNK_0000_0c10", 0, 0x20000, UINT64_C(0xffffffffe0210000)),
+    MVX(0x00000c11, "I7_SB_UNK_0000_0c11", 0, 0x20000, UINT64_C(0xffffffffe0210000)),
+    MVX(0x00000c14, "I7_SB_UNK_0000_0c14", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000c15, "I7_SB_UNK_0000_0c15", 0, 0x3, UINT64_C(0xfffffffffffffffc)),
+    MVX(0x00000c16, "I7_SB_UNK_0000_0c16", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000c17, "I7_SB_UNK_0000_0c17", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000c24, "I7_SB_UNK_0000_0c24", 0, 0x3, UINT64_C(0xfffffffffffcfefc)),
+    MVX(0x00000c30, "I7_SB_UNK_0000_0c30", 0, 0x20000, UINT64_C(0xffffffff20013f00)),
+    MVX(0x00000c31, "I7_SB_UNK_0000_0c31", 0, 0x20000, UINT64_C(0xffffffff20013f00)),
+    MVX(0x00000c32, "I7_SB_UNK_0000_0c32", 0, 0x20000, UINT64_C(0xffffffff20013f00)),
+    MVX(0x00000c33, "I7_SB_UNK_0000_0c33", 0, 0x20000, UINT64_C(0xffffffff20013f00)),
+    MVX(0x00000c34, "I7_SB_UNK_0000_0c34", 0, 0, ~(uint64_t)UINT32_MAX),
+    MVX(0x00000c35, "I7_SB_UNK_0000_0c35", 0, 0x7f, UINT64_C(0xffffffffffffff80)),
+    MVX(0x00000c36, "I7_SB_UNK_0000_0c36", 0x203, 0, UINT64_C(0xffff000000000000)),
+    MVX(0x00000c37, "I7_SB_UNK_0000_0c37", 0x203, 0, UINT64_C(0xffff000000000000)),
+    MVX(0x00000c38, "I7_SB_UNK_0000_0c38", 0x20c, 0, UINT64_C(0xffff000000000000)),
+    MVX(0x00000c39, "I7_SB_UNK_0000_0c39", 0x203, 0, UINT64_C(0xffff000000000000)),
+    MVX(0x00000d04, "I7_SB_UNK_0000_0d04", 0, 0x3, UINT64_C(0xfffffffffffcfefc)),
+    MVX(0x00000d10, "I7_SB_UNK_0000_0d10", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d11, "I7_SB_UNK_0000_0d11", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d12, "I7_SB_UNK_0000_0d12", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d13, "I7_SB_UNK_0000_0d13", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d14, "I7_SB_UNK_0000_0d14", 0x20, 0, UINT64_C(0xffffffff00000300)),
+    MVX(0x00000d15, "I7_SB_UNK_0000_0d15", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000d16, "I7_SB_UNK_0000_0d16", 0x81c, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d17, "I7_SB_UNK_0000_0d17", 0x80c, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d18, "I7_SB_UNK_0000_0d18", 0x80c, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d19, "I7_SB_UNK_0000_0d19", 0x810, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d24, "I7_SB_UNK_0000_0d24", 0, 0x3, UINT64_C(0xfffffffffffcfefc)),
+    MVX(0x00000d30, "I7_SB_UNK_0000_0d30", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d31, "I7_SB_UNK_0000_0d31", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d32, "I7_SB_UNK_0000_0d32", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d33, "I7_SB_UNK_0000_0d33", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d34, "I7_SB_UNK_0000_0d34", 0x20, 0, UINT64_C(0xffffffff00000300)),
+    MVX(0x00000d35, "I7_SB_UNK_0000_0d35", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000d36, "I7_SB_UNK_0000_0d36", 0x864, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d37, "I7_SB_UNK_0000_0d37", 0x804, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d38, "I7_SB_UNK_0000_0d38", 0x822, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d39, "I7_SB_UNK_0000_0d39", 0x81c, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d44, "I7_SB_UNK_0000_0d44", 0, 0x3, UINT64_C(0xfffffffffffcfefc)),
+    MVX(0x00000d50, "I7_SB_UNK_0000_0d50", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d51, "I7_SB_UNK_0000_0d51", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d52, "I7_SB_UNK_0000_0d52", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d53, "I7_SB_UNK_0000_0d53", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d54, "I7_SB_UNK_0000_0d54", 0x20, 0, UINT64_C(0xffffffff00000300)),
+    MVX(0x00000d55, "I7_SB_UNK_0000_0d55", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000d56, "I7_SB_UNK_0000_0d56", 0x848, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d57, "I7_SB_UNK_0000_0d57", 0x866, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d58, "I7_SB_UNK_0000_0d58", 0x83c, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d59, "I7_SB_UNK_0000_0d59", 0x83c, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d64, "I7_SB_UNK_0000_0d64", 0, 0x3, UINT64_C(0xfffffffffffcfefc)),
+    MVX(0x00000d70, "I7_SB_UNK_0000_0d70", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d71, "I7_SB_UNK_0000_0d71", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d72, "I7_SB_UNK_0000_0d72", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d73, "I7_SB_UNK_0000_0d73", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d74, "I7_SB_UNK_0000_0d74", 0x20, 0, UINT64_C(0xffffffff00000300)),
+    MVX(0x00000d75, "I7_SB_UNK_0000_0d75", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000d76, "I7_SB_UNK_0000_0d76", 0x846, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d77, "I7_SB_UNK_0000_0d77", 0x90c, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d78, "I7_SB_UNK_0000_0d78", 0x846, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d79, "I7_SB_UNK_0000_0d79", 0x842, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d84, "I7_SB_UNK_0000_0d84", 0, 0x3, UINT64_C(0xfffffffffffcfefc)),
+    MVX(0x00000d90, "I7_SB_UNK_0000_0d90", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d91, "I7_SB_UNK_0000_0d91", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d92, "I7_SB_UNK_0000_0d92", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d93, "I7_SB_UNK_0000_0d93", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000d94, "I7_SB_UNK_0000_0d94", 0x20, 0, UINT64_C(0xffffffff00000300)),
+    MVX(0x00000d95, "I7_SB_UNK_0000_0d95", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000d96, "I7_SB_UNK_0000_0d96", 0x8c6, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d97, "I7_SB_UNK_0000_0d97", 0x840, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d98, "I7_SB_UNK_0000_0d98", 0x81a, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000d99, "I7_SB_UNK_0000_0d99", 0x910, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000da4, "I7_SB_UNK_0000_0da4", 0, 0x3, UINT64_C(0xfffffffffffcfefc)),
+    MVX(0x00000db0, "I7_SB_UNK_0000_0db0", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000db1, "I7_SB_UNK_0000_0db1", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000db2, "I7_SB_UNK_0000_0db2", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000db3, "I7_SB_UNK_0000_0db3", 0, 0x30000, UINT64_C(0xffffffff00200000)),
+    MVX(0x00000db4, "I7_SB_UNK_0000_0db4", 0x20, 0, UINT64_C(0xffffffff00000300)),
+    MVX(0x00000db5, "I7_SB_UNK_0000_0db5", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000db6, "I7_SB_UNK_0000_0db6", 0x80c, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000db7, "I7_SB_UNK_0000_0db7", 0x81e, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000db8, "I7_SB_UNK_0000_0db8", 0x810, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000db9, "I7_SB_UNK_0000_0db9", 0x80a, 0, UINT64_C(0xfffff00000000000)),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xfffffffffffff2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xfffff800`030dac00 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xfffff800`030da940 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x4700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0xfffe0000 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xfffff880`061e6000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x7ff`fffde000 */
+    MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, 0, ~(uint64_t)UINT32_MAX), /* value=0x0 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Core(TM) i7-3960X CPU @ 3.30GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Core_i7_3960X =
+{
+    /*.pszName          = */ "Intel Core i7-3960X",
+    /*.pszFullName      = */ "Intel(R) Core(TM) i7-3960X CPU @ 3.30GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 45,
+    /*.uStepping        = */ 6,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_Core7_SandyBridge,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_100MHZ,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 46,
+    /*.fMxCsrMask       = */ 0xffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Core_i7_3960X),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Core_i7_3960X)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX,
+    /*.DefUnknownCpuId  = */ { 0x00000007, 0x00000340, 0x00000340, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Core_i7_3960X)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Core_i7_3960X),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Core_i7_3960X_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_5600U.h b/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_5600U.h
new file mode 100644
index 00000000..03ccedfc
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_5600U.h
@@ -0,0 +1,368 @@
+/* $Id: Intel_Core_i7_5600U.h $ */
+/** @file
+ * CPU database entry "Intel Core i7-5600U".
+ * Generated at 2015-11-04T14:14:27Z by VBoxCpuReport v5.0.51r103906 on win.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Core_i7_5600U_h
+#define VBOX_CPUDB_Intel_Core_i7_5600U_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Core_i7_5600U[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000014, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000306d4, 0x00100800, 0x7ffafbff, 0xbfebfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x76036301, 0x00f0b5ff, 0x00000000, 0x00c30000, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, UINT32_MAX, 0x1c004121, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000004, 0x00000001, UINT32_MAX, 0x1c004122, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000004, 0x00000002, UINT32_MAX, 0x1c004143, 0x01c0003f, 0x000001ff, 0x00000000, 0 },
+    { 0x00000004, 0x00000003, UINT32_MAX, 0x1c03c163, 0x03c0003f, 0x00000fff, 0x00000006, 0 },
+    { 0x00000004, 0x00000004, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x11142120, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000077, 0x00000002, 0x00000009, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, UINT32_MAX, 0x00000000, 0x021c2fbb, 0x00000000, 0x00000000, 0 },
+    { 0x00000007, 0x00000001, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07300403, 0x00000000, 0x00000000, 0x00000603, 0 },
+    { 0x0000000b, 0x00000000, UINT32_MAX, 0x00000001, 0x00000002, 0x00000100, 0x00000000, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000b, 0x00000001, UINT32_MAX, 0x00000004, 0x00000004, 0x00000201, 0x00000000, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000b, 0x00000002, UINT32_MAX, 0x00000000, 0x00000000, 0x00000002, 0x00000000, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000c, 0x00000000, UINT32_MAX, 0x00000000, 0x00000001, 0x00000001, 0x00000000, 0 },
+    { 0x0000000c, 0x00000001, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000000, UINT32_MAX, 0x00000007, 0x00000340, 0x00000340, 0x00000000, 0 },
+    { 0x0000000d, 0x00000001, UINT32_MAX, 0x00000001, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000002, UINT32_MAX, 0x00000100, 0x00000240, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000003, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000e, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000f, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000010, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000011, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000012, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000013, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000014, 0x00000000, UINT32_MAX, 0x00000000, 0x00000001, 0x00000001, 0x00000000, 0 },
+    { 0x00000014, 0x00000001, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000121, 0x2c100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x65746e49, 0x2952286c, 0x726f4320, 0x4d542865, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x37692029, 0x3036352d, 0x43205530, 0x40205550, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x362e3220, 0x7a484730, 0x00000000, 0x00000000, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x01006040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000100, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003027, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Core_i7_5600U[] =
+{
+    MFX(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr, Ia32P5McAddr, 0, UINT64_C(0xffffffffffffff00), 0), /* value=0xff */
+    MFX(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType, Ia32P5McType, 0, 0, UINT64_MAX), /* value=0x0 */
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, 0, UINT64_C(0xffffffffffff0000)), /* value=0x40 */
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x1c93`50dd535c */
+    MFX(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0x18000000000000), 0, 0), /* value=0x180000`00000000 */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00900), 0, UINT64_C(0xffffff80000002ff)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MVX(0x0000002e, "I7_UNK_0000_002e", 0, 0x400, UINT64_C(0xfffffffffffffbff)),
+    MVX(0x00000033, "TEST_CTL", 0, 0, UINT64_C(0xffffffff7fffffff)),
+    MVO(0x00000034, "P6_UNK_0000_0034", 0x97b),
+    MFO(0x00000035, "MSR_CORE_THREAD_COUNT", IntelI7CoreThreadCount), /* value=0x20004 */
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x5 */
+    MVX(0x0000003b, "P6_UNK_0000_003b", UINT64_C(0xfffffffffffffffe), 0, 0),
+    MVX(0x0000003e, "I7_UNK_0000_003e", 0x1, 0, UINT64_C(0xfffffffffffffffe)),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, IgnoreWrite),
+    MFX(0x0000008b, "BBL_CR_D3|BIOS_SIGN", Ia32BiosSignId, Ia32BiosSignId, 0, 0, UINT32_MAX), /* value=0x1f`00000000 */
+    MVX(0x00000095, "TODO_0000_0095", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    RSN(0x000000c1, 0x000000c4, "IA32_PMCn", Ia32PmcN, Ia32PmcN, 0x0, ~(uint64_t)UINT32_MAX, 0),
+    MFO(0x000000ce, "IA32_PLATFORM_INFO", IntelPlatformInfo), /* value=0x5053b`f3011a00 */
+    MFX(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl, IntelPkgCStConfigControl, 0, 0, UINT64_C(0xffffffff01ffffff)), /* value=0x1e008408 */
+    MFX(0x000000e3, "C2_SMM_CST_MISC_INFO", IntelCore2SmmCStMiscInfo, IntelCore2SmmCStMiscInfo, 0, UINT32_C(0xffff7000), ~(uint64_t)UINT32_MAX), /* value=0x8b800000 */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, 0, UINT64_C(0xfffffffffff80000)), /* value=0x51814 */
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf), /* value=0x23c`764b31c5 */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x2af`f518152c */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0xd0a, 0, 0), /* value=0xd0a */
+    MVX(0x00000102, "I7_IB_UNK_0000_0102", 0, 0, UINT64_C(0xffffffff7fff8000)),
+    MVX(0x00000103, "I7_IB_UNK_0000_0103", 0, 0, UINT64_C(0xfffffffffffff000)),
+    MVX(0x00000104, "I7_IB_UNK_0000_0104", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MVO(0x00000110, "TODO_0000_0110", 0x3),
+    MVX(0x0000011f, "TODO_0000_011f", 0, 0, UINT64_C(0xffffffffffffff00)),
+    MVO(0x0000013a, "TODO_0000_013a", UINT64_C(0x30000007f)),
+    MFO(0x0000013c, "I7_SB_AES_NI_CTL", IntelI7SandyAesNiCtl), /* value=0x1 */
+    MVX(0x00000140, "I7_IB_UNK_0000_0140", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MVX(0x00000142, "I7_IB_UNK_0000_0142", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MVX(0x00000150, "P6_UNK_0000_0150", 0, UINT64_C(0x8000ffffffffffff), UINT64_C(0x7fff000000000000)),
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0x0 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0x0 */
+    MVX(0x00000178, "TODO_0000_0178", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x1000c07, 0, 0), /* value=0x1000c07 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, 0, UINT64_C(0xfffffffffffffff8)), /* value=0x0 */
+    RSN(0x00000186, 0x00000187, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, 0, UINT64_C(0xfffffffe00080000)), /* XXX: The range ended earlier than expected! */
+    MVX(0x00000188, "IA32_PERFEVTSEL2", 0, 0, UINT64_C(0xfffffffc00080000)),
+    MVX(0x00000189, "IA32_PERFEVTSEL3", 0, 0, UINT64_C(0xfffffffe00080000)),
+    MFX(0x00000194, "CLOCK_FLEX_MAX", IntelFlexRatio, IntelFlexRatio, 0x90000, 0xe0000, UINT64_C(0xffffffffffe00000)), /* value=0x90000 */
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0x273c00002000), 0, 0), /* value=0x273c`00002000 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x2000, 0, 0), /* Might bite. value=0x2000 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x0 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0x10, 0, UINT64_C(0xfffffffffe0000e8)), /* value=0x10 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, UINT32_C(0x88150800), UINT32_C(0xf87f07fd), UINT64_C(0xffffffff0780f000)), /* value=0x88150800 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0, 0, 0), /* value=0x0 */
+/// @todo WARNING: IA32_MISC_ENABLE probing needs hacking on this CPU!
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, 0x850089, 0x1080, UINT64_C(0xffffffbbff3aef72)), /* value=0x850089 */
+    MVO(0x000001a1, "P6_UNK_0000_01a1", 0x995),
+    MFX(0x000001a2, "I7_MSR_TEMPERATURE_TARGET", IntelI7TemperatureTarget, IntelI7TemperatureTarget, 0x5690000, 0xffff00, UINT64_C(0xffffffffc00000ff)), /* value=0x5690000 */
+    MVX(0x000001a4, "I7_UNK_0000_01a4", 0, 0, UINT64_C(0xfffffffffffff7f0)),
+    RSN(0x000001a6, 0x000001a7, "I7_MSR_OFFCORE_RSP_n", IntelI7MsrOffCoreResponseN, IntelI7MsrOffCoreResponseN, 0x0, 0, UINT64_C(0xffffffc000007000)),
+    MVX(0x000001a8, "I7_UNK_0000_01a8", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFX(0x000001aa, "MSR_MISC_PWR_MGMT", IntelI7MiscPwrMgmt, IntelI7MiscPwrMgmt, 0, 0, UINT64_C(0xffffffffffbffffe)), /* value=0x400000 */
+    MFX(0x000001ad, "I7_MSR_TURBO_RATIO_LIMIT", IntelI7TurboRatioLimit, IntelI7TurboRatioLimit, UINT64_C(0x1f1f1f1f1f20), UINT64_MAX, 0), /* value=0x1f1f`1f1f1f20 */
+    MVX(0x000001b0, "IA32_ENERGY_PERF_BIAS", 0x6, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x000001b1, "IA32_PACKAGE_THERM_STATUS", UINT32_C(0x880d0802), UINT32_C(0xf87f07fd), UINT64_C(0xffffffff0780f000)),
+    MVX(0x000001b2, "IA32_PACKAGE_THERM_INTERRUPT", 0, 0, UINT64_C(0xfffffffffe0000e8)),
+    MVO(0x000001c6, "I7_UNK_0000_01c6", 0x3),
+    MFX(0x000001c8, "MSR_LBR_SELECT", IntelI7LbrSelect, IntelI7LbrSelect, 0, 0, UINT64_C(0xfffffffffffffc00)), /* value=0x0 */
+    MFX(0x000001c9, "MSR_LASTBRANCH_TOS", IntelLastBranchTos, IntelLastBranchTos, 0, 0, UINT64_C(0xfffffffffffffff0)), /* value=0x0 */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffff003c)), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0x0 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0x0 */
+    MFX(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp, P6LastIntFromIp, 0, 0, UINT64_C(0x1fff800000000000)), /* value=0x0 */
+    MFN(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp, P6LastIntToIp), /* value=0x0 */
+    MFO(0x000001f0, "I7_VLW_CAPABILITY", IntelI7VirtualLegacyWireCap), /* value=0x74 */
+    MFO(0x000001f2, "IA32_SMRR_PHYSBASE", Ia32SmrrPhysBase), /* value=0xdc000006 */
+    MFO(0x000001f3, "IA32_SMRR_PHYSMASK", Ia32SmrrPhysMask), /* value=0xff000800 */
+    MFX(0x000001fc, "I7_MSR_POWER_CTL", IntelI7PowerCtl, IntelI7PowerCtl, 0, UINT32_C(0x80000020), UINT64_C(0xffffffff3e100000)), /* value=0x4005f */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7e`00000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x2`00000006 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`f0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x2`10000006 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`f8000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x2`18000006 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`fc000800 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x2`1c000006 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`fe000800 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x2`1e000006 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`ff800800 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xffffff8000000ff8)), /* value=0xe0000000 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`e0000800 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xffffff8000000ff8)), /* value=0xde000000 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`fe000800 */
+    MFX(0x00000210, "IA32_MTRR_PHYS_BASE8", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x8, 0, UINT64_C(0xffffff8000000ff8)), /* value=0xdd000000 */
+    MFX(0x00000211, "IA32_MTRR_PHYS_MASK8", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x8, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`ff000800 */
+    MFX(0x00000212, "IA32_MTRR_PHYS_BASE9", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x9, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x0 */
+    MFX(0x00000213, "IA32_MTRR_PHYS_MASK9", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x9, 0, UINT64_C(0xffffff80000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MFX(0x00000280, "IA32_MC0_CTL2", Ia32McNCtl2, Ia32McNCtl2, 0x0, 0x40000000, UINT64_C(0xffffffffbfff8000)), /* value=0x0 */
+    RSN(0x00000281, 0x00000283, "IA32_MC1_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0x1, 0, UINT64_C(0xffffffffbfff8000)),
+    MFX(0x00000284, "IA32_MC4_CTL2", Ia32McNCtl2, Ia32McNCtl2, 0x4, 0x40007fff, UINT64_C(0xffffffffbfff8000)), /* value=0x0 */
+    RSN(0x00000285, 0x00000286, "IA32_MC5_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0x5, 0, UINT64_C(0xffffffffbfff8000)),
+    MVX(0x000002e0, "I7_SB_NO_EVICT_MODE", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MVX(0x000002e7, "I7_IB_UNK_0000_02e7", 0x1, 0x1, UINT64_C(0xfffffffffffffffe)),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    MVO(0x00000305, "I7_SB_UNK_0000_0305", 0),
+    RSN(0x00000309, 0x0000030b, "IA32_FIXED_CTRn", Ia32FixedCtrN, Ia32FixedCtrN, 0x0, 0, UINT64_C(0xffff000000000000)),
+    MFX(0x00000345, "IA32_PERF_CAPABILITIES", Ia32PerfCapabilities, ReadOnly, 0x32c4, 0, 0), /* value=0x32c4 */
+    MFX(0x0000038d, "IA32_FIXED_CTR_CTRL", Ia32FixedCtrCtrl, Ia32FixedCtrCtrl, 0, 0, UINT64_C(0xfffffffffffff000)), /* value=0x0 */
+    MFX(0x0000038e, "IA32_PERF_GLOBAL_STATUS", Ia32PerfGlobalStatus, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x0000038f, "IA32_PERF_GLOBAL_CTRL", Ia32PerfGlobalCtrl, Ia32PerfGlobalCtrl, 0, 0, UINT64_C(0xfffffff8fffffff0)), /* value=0xf */
+    MFX(0x00000390, "IA32_PERF_GLOBAL_OVF_CTRL", Ia32PerfGlobalOvfCtrl, Ia32PerfGlobalOvfCtrl, 0, UINT64_C(0xe08000070000000f), UINT64_C(0x1f7ffff8fffffff0)), /* value=0x0 */
+    MFX(0x00000391, "I7_UNC_PERF_GLOBAL_CTRL", IntelI7UncPerfGlobalCtrl, IntelI7UncPerfGlobalCtrl, 0, 0, UINT64_C(0xffffffff1fffff80)), /* value=0x0 */
+    MFX(0x00000392, "I7_UNC_PERF_GLOBAL_STATUS", IntelI7UncPerfGlobalStatus, IntelI7UncPerfGlobalStatus, 0, 0xf, UINT64_C(0xfffffffffffffff0)), /* value=0x0 */
+    MFX(0x00000393, "I7_UNC_PERF_GLOBAL_OVF_CTRL", IntelI7UncPerfGlobalOvfCtrl, IntelI7UncPerfGlobalOvfCtrl, 0, 0x3, UINT64_C(0xfffffffffffffffc)), /* value=0x0 */
+    MFX(0x00000394, "I7_UNC_PERF_FIXED_CTR_CTRL", IntelI7UncPerfFixedCtrCtrl, IntelI7UncPerfFixedCtrCtrl, 0, 0, UINT64_C(0xffffffffffafffff)), /* value=0x0 */
+    MFX(0x00000395, "I7_UNC_PERF_FIXED_CTR", IntelI7UncPerfFixedCtr, IntelI7UncPerfFixedCtr, 0, 0, UINT64_C(0xffff000000000000)), /* value=0x0 */
+    MFO(0x00000396, "I7_UNC_CBO_CONFIG", IntelI7UncCBoxConfig), /* value=0x3 */
+    MVX(0x00000397, "I7_SB_UNK_0000_0397", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MFX(0x000003b0, "I7_UNC_ARB_PERF_CTR0", IntelI7UncArbPerfCtrN, IntelI7UncArbPerfCtrN, 0, 0, UINT64_C(0xfffff00000000000)), /* value=0x0 */
+    MFX(0x000003b1, "I7_UNC_ARB_PERF_CTR1", IntelI7UncArbPerfCtrN, IntelI7UncArbPerfCtrN, 0, 0, UINT64_C(0xfffff00000000000)), /* value=0x0 */
+    MFX(0x000003b2, "I7_UNC_ARB_PERF_EVT_SEL0", IntelI7UncArbPerfEvtSelN, IntelI7UncArbPerfEvtSelN, 0, 0, UINT64_C(0xffffffffe0230000)), /* value=0x0 */
+    MFX(0x000003b3, "I7_UNC_ARB_PERF_EVT_SEL1", IntelI7UncArbPerfEvtSelN, IntelI7UncArbPerfEvtSelN, 0, 0, UINT64_C(0xffffffffe0230000)), /* value=0x0 */
+    MVO(0x000003f0, "TODO_0000_03f0", 0),
+    MFX(0x000003f1, "IA32_PEBS_ENABLE", Ia32PebsEnable, Ia32PebsEnable, 0, 0, UINT64_C(0xfffffff0fffffff0)), /* value=0x0 */
+    MFX(0x000003f6, "I7_MSR_PEBS_LD_LAT", IntelI7PebsLdLat, IntelI7PebsLdLat, 0, UINT64_C(0xffffffffffff0000), 0), /* value=0xffff */
+    MFX(0x000003f8, "I7_MSR_PKG_C3_RESIDENCY", IntelI7PkgCnResidencyN, ReadOnly, 0x3, 0, UINT64_MAX), /* value=0x4`465710e6 */
+    RSN(0x000003f9, 0x000003fa, "I7_MSR_PKG_Cn_RESIDENCY", IntelI7PkgCnResidencyN, ReadOnly, 0x6, 0, UINT64_MAX),
+    MFX(0x000003fc, "I7_MSR_CORE_C3_RESIDENCY", IntelI7CoreCnResidencyN, ReadOnly, 0x3, 0, UINT64_MAX), /* value=0x2`3a8a1eca */
+    RSN(0x000003fd, 0x000003fe, "I7_MSR_CORE_Cn_RESIDENCY", IntelI7CoreCnResidencyN, ReadOnly, 0x6, 0, UINT64_MAX),
+    RFN(0x00000400, 0x0000041b, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0x00000480, "IA32_VMX_BASIC", Ia32VmxBasic, ReadOnly, UINT64_C(0xda040000000012), 0, 0), /* value=0xda0400`00000012 */
+    MFX(0x00000481, "IA32_VMX_PINBASED_CTLS", Ia32VmxPinbasedCtls, ReadOnly, UINT64_C(0x7f00000016), 0, 0), /* value=0x7f`00000016 */
+    MFX(0x00000482, "IA32_VMX_PROCBASED_CTLS", Ia32VmxProcbasedCtls, ReadOnly, UINT64_C(0xfff9fffe0401e172), 0, 0), /* value=0xfff9fffe`0401e172 */
+    MFX(0x00000483, "IA32_VMX_EXIT_CTLS", Ia32VmxExitCtls, ReadOnly, UINT64_C(0x7fffff00036dff), 0, 0), /* value=0x7fffff`00036dff */
+    MFX(0x00000484, "IA32_VMX_ENTRY_CTLS", Ia32VmxEntryCtls, ReadOnly, UINT64_C(0xffff000011ff), 0, 0), /* value=0xffff`000011ff */
+    MFX(0x00000485, "IA32_VMX_MISC", Ia32VmxMisc, ReadOnly, 0x300481e5, 0, 0), /* value=0x300481e5 */
+    MFX(0x00000486, "IA32_VMX_CR0_FIXED0", Ia32VmxCr0Fixed0, ReadOnly, UINT32_C(0x80000021), 0, 0), /* value=0x80000021 */
+    MFX(0x00000487, "IA32_VMX_CR0_FIXED1", Ia32VmxCr0Fixed1, ReadOnly, UINT32_MAX, 0, 0), /* value=0xffffffff */
+    MFX(0x00000488, "IA32_VMX_CR4_FIXED0", Ia32VmxCr4Fixed0, ReadOnly, 0x2000, 0, 0), /* value=0x2000 */
+    MFX(0x00000489, "IA32_VMX_CR4_FIXED1", Ia32VmxCr4Fixed1, ReadOnly, 0x3767ff, 0, 0), /* value=0x3767ff */
+    MFX(0x0000048a, "IA32_VMX_VMCS_ENUM", Ia32VmxVmcsEnum, ReadOnly, 0x2a, 0, 0), /* value=0x2a */
+    MFX(0x0000048b, "IA32_VMX_PROCBASED_CTLS2", Ia32VmxProcBasedCtls2, ReadOnly, UINT64_C(0x57cff00000000), 0, 0), /* value=0x57cff`00000000 */
+    MFX(0x0000048c, "IA32_VMX_EPT_VPID_CAP", Ia32VmxEptVpidCap, ReadOnly, UINT64_C(0xf0106334141), 0, 0), /* value=0xf01`06334141 */
+    MFX(0x0000048d, "IA32_VMX_TRUE_PINBASED_CTLS", Ia32VmxTruePinbasedCtls, ReadOnly, UINT64_C(0x7f00000016), 0, 0), /* value=0x7f`00000016 */
+    MFX(0x0000048e, "IA32_VMX_TRUE_PROCBASED_CTLS", Ia32VmxTrueProcbasedCtls, ReadOnly, UINT64_C(0xfff9fffe04006172), 0, 0), /* value=0xfff9fffe`04006172 */
+    MFX(0x0000048f, "IA32_VMX_TRUE_EXIT_CTLS", Ia32VmxTrueExitCtls, ReadOnly, UINT64_C(0x7fffff00036dfb), 0, 0), /* value=0x7fffff`00036dfb */
+    MFX(0x00000490, "IA32_VMX_TRUE_ENTRY_CTLS", Ia32VmxTrueEntryCtls, ReadOnly, UINT64_C(0xffff000011fb), 0, 0), /* value=0xffff`000011fb */
+    MFX(0x00000491, "IA32_VMX_VMFUNC", Ia32VmxVmFunc, ReadOnly, 0x1, 0, 0), /* value=0x1 */
+    RSN(0x000004c1, 0x000004c4, "IA32_A_PMCn", Ia32PmcN, Ia32PmcN, 0x0, 0, UINT64_C(0xffff000000000000)),
+    MVO(0x000004e0, "TODO_0000_04e0", 0x1),
+    MVO(0x000004e2, "TODO_0000_04e2", 0x5),
+    MVO(0x000004e3, "TODO_0000_04e3", 0xff0),
+    MVX(0x00000560, "TODO_0000_0560", 0, 0, UINT64_C(0xffffff800000007f)),
+    MVX(0x00000561, "TODO_0000_0561", 0x7f, UINT64_C(0x70000007f), UINT32_C(0xffffff80)),
+    MVX(0x00000570, "TODO_0000_0570", 0x2100, 0x2100, UINT64_C(0xffffffffffffd272)),
+    MVX(0x00000571, "TODO_0000_0571", 0, 0x7, UINT64_C(0xffffffffffffffc8)),
+    MVX(0x00000572, "TODO_0000_0572", 0, 0, UINT64_C(0xffff00000000001f)),
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0x0 */
+    MFX(0x00000601, "I7_SB_MSR_VR_CURRENT_CONFIG", IntelI7SandyVrCurrentConfig, IntelI7SandyVrCurrentConfig, 0, UINT32_C(0x80001fff), UINT64_C(0x800000007fffe000)), /* value=0x40101414`80000100 */
+    MFX(0x00000603, "I7_SB_MSR_VR_MISC_CONFIG", IntelI7SandyVrMiscConfig, IntelI7SandyVrMiscConfig, 0, 0, UINT64_C(0xff80000000000000)), /* value=0x360000`00333333 */
+    MFO(0x00000606, "I7_SB_MSR_RAPL_POWER_UNIT", IntelI7SandyRaplPowerUnit), /* value=0xa0e03 */
+    MVX(0x00000609, "I7_SB_UNK_0000_0609", 0x1a, 0xc0, UINT64_C(0xffffffffffffff00)),
+    MFX(0x0000060a, "I7_SB_MSR_PKGC3_IRTL", IntelI7SandyPkgCnIrtlN, IntelI7SandyPkgCnIrtlN, 0x3, 0, UINT64_C(0xffffffffffff6000)), /* value=0x8842 */
+    RSN(0x0000060b, 0x0000060c, "I7_SB_MSR_PKGC6_IRTn", IntelI7SandyPkgCnIrtlN, IntelI7SandyPkgCnIrtlN, 0x6, 0, UINT64_C(0xffffffffffff6000)),
+    MFO(0x0000060d, "I7_SB_MSR_PKG_C2_RESIDENCY", IntelI7SandyPkgC2Residency), /* value=0x1b`88fad668 */
+    MFX(0x00000610, "I7_SB_MSR_PKG_POWER_LIMIT", IntelI7RaplPkgPowerLimit, IntelI7RaplPkgPowerLimit, 0, UINT64_C(0x80ffffff00ffffff), UINT64_C(0x7f000000ff000000)), /* value=0x804280c8`00dd8078 */
+    MFO(0x00000611, "I7_SB_MSR_PKG_ENERGY_STATUS", IntelI7RaplPkgEnergyStatus), /* value=0x7e40b254 */
+    MFO(0x00000613, "I7_SB_MSR_PKG_PERF_STATUS", IntelI7RaplPkgPerfStatus), /* value=0xff3 */
+    MFO(0x00000614, "I7_SB_MSR_PKG_POWER_INFO", IntelI7RaplPkgPowerInfo), /* value=0x78 */
+    MVX(0x00000615, "TODO_0000_0615", 0, 0, UINT64_C(0xffffffff00010000)),
+    MFX(0x00000618, "I7_SB_MSR_DRAM_POWER_LIMIT", IntelI7RaplDramPowerLimit, IntelI7RaplDramPowerLimit, 0, UINT64_C(0x80feffff00feffff), UINT64_C(0x7f010000ff010000)), /* value=0x805400de`00000000 */
+    MFO(0x00000619, "I7_SB_MSR_DRAM_ENERGY_STATUS", IntelI7RaplDramEnergyStatus), /* value=0x9dbe152 */
+    MFO(0x0000061b, "I7_SB_MSR_DRAM_PERF_STATUS", IntelI7RaplDramPerfStatus), /* value=0x0 */
+    MVO(0x0000061d, "TODO_0000_061d", UINT64_C(0x6e231cb3da)),
+    MVX(0x00000620, "TODO_0000_0620", 0x71d, 0, UINT64_C(0xffffffffffff8080)),
+    MVO(0x00000621, "TODO_0000_0621", 0x1d),
+    MVX(0x00000622, "TODO_0000_0622", 0x1, 0, UINT64_C(0xfffffffffffffffe)),
+    MVO(0x00000623, "TODO_0000_0623", 0x1),
+    MVO(0x00000630, "TODO_0000_0630", 0),
+    MVO(0x00000631, "TODO_0000_0631", 0),
+    MVO(0x00000632, "TODO_0000_0632", 0),
+    MVX(0x00000633, "TODO_0000_0633", 0x88e4, 0, UINT64_C(0xffffffffffff6000)),
+    MVX(0x00000634, "TODO_0000_0634", 0x8945, 0, UINT64_C(0xffffffffffff6000)),
+    MVX(0x00000635, "TODO_0000_0635", 0x89ef, 0, UINT64_C(0xffffffffffff6000)),
+    MVX(0x00000636, "TODO_0000_0636", 0x6a, 0, UINT64_C(0xffffffffffff0000)),
+    MVO(0x00000637, "TODO_0000_0637", UINT64_C(0x43af89cfdf)),
+    MFX(0x00000638, "I7_SB_MSR_PP0_POWER_LIMIT", IntelI7RaplPp0PowerLimit, IntelI7RaplPp0PowerLimit, 0, 0, UINT64_C(0xffffffff7f000000)), /* value=0x0 */
+    MFO(0x00000639, "I7_SB_MSR_PP0_ENERGY_STATUS", IntelI7RaplPp0EnergyStatus), /* value=0x6f9c685f */
+    MFX(0x0000063a, "I7_SB_MSR_PP0_POLICY", IntelI7RaplPp0Policy, IntelI7RaplPp0Policy, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x7 */
+    MFX(0x00000640, "I7_HW_MSR_PP0_POWER_LIMIT", IntelI7RaplPp1PowerLimit, IntelI7RaplPp1PowerLimit, 0, 0, UINT64_C(0xffffffff7f000000)), /* value=0x0 */
+    MFO(0x00000641, "I7_HW_MSR_PP0_ENERGY_STATUS", IntelI7RaplPp1EnergyStatus), /* value=0x4d471 */
+    MFX(0x00000642, "I7_HW_MSR_PP0_POLICY", IntelI7RaplPp1Policy, IntelI7RaplPp1Policy, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0xb */
+    MFO(0x00000648, "I7_IB_MSR_CONFIG_TDP_NOMINAL", IntelI7IvyConfigTdpNominal), /* value=0x1a */
+    MFO(0x00000649, "I7_IB_MSR_CONFIG_TDP_LEVEL1", IntelI7IvyConfigTdpLevel1), /* value=0x6003c */
+    MFO(0x0000064a, "I7_IB_MSR_CONFIG_TDP_LEVEL2", IntelI7IvyConfigTdpLevel2), /* value=0x0 */
+    MFX(0x0000064b, "I7_IB_MSR_CONFIG_TDP_CONTROL", IntelI7IvyConfigTdpControl, IntelI7IvyConfigTdpControl, 0, 0, UINT64_C(0xffffffff7ffffffc)), /* value=0x80000000 */
+    MFX(0x0000064c, "I7_IB_MSR_TURBO_ACTIVATION_RATIO", IntelI7IvyTurboActivationRatio, IntelI7IvyTurboActivationRatio, 0, 0, UINT64_C(0xffffffff7fffff00)), /* value=0x80000019 */
+    RFN(0x00000680, 0x0000068f, "MSR_LASTBRANCH_n_FROM_IP", IntelLastBranchFromN, IntelLastBranchFromN),
+    MVX(0x00000690, "TODO_0000_0690", 0x1d200000, UINT32_C(0xe6dfffff), ~(uint64_t)UINT32_MAX),
+    MVX(0x000006b0, "TODO_0000_06b0", 0x1d000000, UINT32_C(0xe2ffffff), ~(uint64_t)UINT32_MAX),
+    MVX(0x000006b1, "TODO_0000_06b1", 0xd000000, UINT32_C(0xf2ffffff), ~(uint64_t)UINT32_MAX),
+    RFN(0x000006c0, 0x000006cf, "MSR_LASTBRANCH_n_TO_IP", IntelLastBranchToN, IntelLastBranchToN),
+    MFI(0x000006e0, "IA32_TSC_DEADLINE", Ia32TscDeadline), /* value=0x0 */
+    MVX(0x00000700, "TODO_0000_0700", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000701, "TODO_0000_0701", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000702, "TODO_0000_0702", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000703, "TODO_0000_0703", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000704, "TODO_0000_0704", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000705, "TODO_0000_0705", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000706, "TODO_0000_0706", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000707, "TODO_0000_0707", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000708, "TODO_0000_0708", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000709, "TODO_0000_0709", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000710, "TODO_0000_0710", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000711, "TODO_0000_0711", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000712, "TODO_0000_0712", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000713, "TODO_0000_0713", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000714, "TODO_0000_0714", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000715, "TODO_0000_0715", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000716, "TODO_0000_0716", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000717, "TODO_0000_0717", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000718, "TODO_0000_0718", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000719, "TODO_0000_0719", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000720, "TODO_0000_0720", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000721, "TODO_0000_0721", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000722, "TODO_0000_0722", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000723, "TODO_0000_0723", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000724, "TODO_0000_0724", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000725, "TODO_0000_0725", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000726, "TODO_0000_0726", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000727, "TODO_0000_0727", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000728, "TODO_0000_0728", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000729, "TODO_0000_0729", 0, 0, UINT64_C(0xfffff00000000000)),
+    MFO(0x00000c80, "IA32_DEBUG_INTERFACE", Ia32DebugInterface), /* value=0x40000000 */
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xfffffffffffff2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xfffff802`f9b59200 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xfffff802`f9b58f40 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x4700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x212000 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffffd001`83740000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x210000 */
+    MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, 0, ~(uint64_t)UINT32_MAX), /* value=0x0 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Core_i7_5600U =
+{
+    /*.pszName          = */ "Intel Core i7-5600U",
+    /*.pszFullName      = */ "Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 61,
+    /*.uStepping        = */ 4,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_Core7_Broadwell,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_100MHZ,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 39,
+    /*.fMxCsrMask       = */ 0xffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Core_i7_5600U),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Core_i7_5600U)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000001, 0x00000001, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Core_i7_5600U)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Core_i7_5600U),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Core_i7_5600U_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_6700K.h b/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_6700K.h
new file mode 100644
index 00000000..e0fa0379
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Core_i7_6700K.h
@@ -0,0 +1,510 @@
+/* $Id: Intel_Core_i7_6700K.h $ */
+/** @file
+ * CPU database entry "Intel Core i7-6700K".
+ * Generated at 2015-11-04T14:22:26Z by VBoxCpuReport v5.0.51r103906 on win.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Core_i7_6700K_h
+#define VBOX_CPUDB_Intel_Core_i7_6700K_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Core_i7_6700K[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000016, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000506e3, 0x02100800, 0x7ffafbbf, 0xbfebfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x76036301, 0x00f0b5ff, 0x00000000, 0x00c30000, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, UINT32_MAX, 0x1c004121, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000004, 0x00000001, UINT32_MAX, 0x1c004122, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000004, 0x00000002, UINT32_MAX, 0x1c004143, 0x00c0003f, 0x000003ff, 0x00000000, 0 },
+    { 0x00000004, 0x00000003, UINT32_MAX, 0x1c03c163, 0x03c0003f, 0x00001fff, 0x00000006, 0 },
+    { 0x00000004, 0x00000004, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00142120, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x000027f7, 0x00000002, 0x00000009, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, UINT32_MAX, 0x00000000, 0x029c6fbf, 0x00000000, 0x00000000, 0 },
+    { 0x00000007, 0x00000001, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07300404, 0x00000000, 0x00000000, 0x00000603, 0 },
+    { 0x0000000b, 0x00000000, UINT32_MAX, 0x00000001, 0x00000002, 0x00000100, 0x00000002, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000b, 0x00000001, UINT32_MAX, 0x00000004, 0x00000008, 0x00000201, 0x00000002, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000b, 0x00000002, UINT32_MAX, 0x00000000, 0x00000000, 0x00000002, 0x00000002, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000000, UINT32_MAX, 0x0000001f, 0x00000440, 0x00000440, 0x00000000, 0 },
+    { 0x0000000d, 0x00000001, UINT32_MAX, 0x0000000f, 0x000003c0, 0x00000100, 0x00000000, 0 },
+    { 0x0000000d, 0x00000002, UINT32_MAX, 0x00000100, 0x00000240, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000003, UINT32_MAX, 0x00000040, 0x000003c0, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000004, UINT32_MAX, 0x00000040, 0x00000400, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000005, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000006, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000007, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000008, UINT32_MAX, 0x00000080, 0x00000000, 0x00000001, 0x00000000, 0 },
+    { 0x0000000d, 0x00000009, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000e, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000f, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000010, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000011, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000012, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000013, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000014, 0x00000000, UINT32_MAX, 0x00000001, 0x0000000f, 0x00000007, 0x00000000, 0 },
+    { 0x00000014, 0x00000001, UINT32_MAX, 0x02490002, 0x003f3fff, 0x00000000, 0x00000000, 0 },
+    { 0x00000014, 0x00000002, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000015, 0x00000000, 0x00000000, 0x00000002, 0x0000014e, 0x00000000, 0x00000000, 0 },
+    { 0x00000016, 0x00000000, 0x00000000, 0x00000fa0, 0x00001068, 0x00000064, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000121, 0x2c100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x65746e49, 0x2952286c, 0x726f4320, 0x4d542865, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x37692029, 0x3037362d, 0x43204b30, 0x40205550, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x302e3420, 0x7a484730, 0x00000000, 0x00000000, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x01006040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000100, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003027, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Core_i7_6700K[] =
+{
+    MFX(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr, Ia32P5McAddr, 0, UINT64_C(0xfffffffffffff000), 0), /* value=0xfff */
+    MFX(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType, Ia32P5McType, 0, 0, UINT64_MAX), /* value=0x0 */
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, 0, UINT64_C(0xffffffffffff0000)), /* value=0x40 */
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x12fdb`64facbdf */
+    MFX(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0x4000000000000), 0, 0), /* value=0x40000`00000000 */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0, UINT64_C(0xffffff80000002ff)),
+    MVO(0x00000020, "TODO_0000_0020", UINT64_C(0xaab8e94b4b4ac1)),
+    MVO(0x00000021, "C2_UNK_0000_0021", UINT64_C(0x52d289e67f37651b)),
+    MVO(0x00000022, "TODO_0000_0022", UINT64_C(0xce7bd366cd8dc6e6)),
+    MVO(0x00000023, "TODO_0000_0023", UINT64_C(0xfd0cd1679876a91d)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MVX(0x0000002e, "I7_UNK_0000_002e", 0, 0x400, UINT64_C(0xfffffffffffffbff)),
+    MVX(0x00000033, "TEST_CTL", 0, 0, UINT64_C(0xffffffff7fffffff)),
+    MVO(0x00000034, "P6_UNK_0000_0034", 0x8),
+    MFO(0x00000035, "MSR_CORE_THREAD_COUNT", IntelI7CoreThreadCount), /* value=0x40008 */
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x5 */
+    MVX(0x0000003b, "P6_UNK_0000_003b", UINT64_C(0xfff5c5f4e22b), 0, 0),
+    MVX(0x0000003e, "I7_UNK_0000_003e", 0x1, 0, UINT64_C(0xfffffffffffffffe)),
+    MVO(0x00000059, "TODO_0000_0059", 0),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, IgnoreWrite),
+    MVX(0x0000007a, "TODO_0000_007a", 0, 0x1, UINT64_C(0xfffffffffffffffe)),
+    MVX(0x00000080, "P4_UNK_0000_0080", 0, UINT64_C(0x7ffffffffe), UINT64_C(0xffffff8000000000)),
+    MFX(0x0000008b, "BBL_CR_D3|BIOS_SIGN", Ia32BiosSignId, Ia32BiosSignId, 0, 0, UINT32_MAX), /* value=0x33`00000000 */
+    MVX(0x00000095, "TODO_0000_0095", 0x1, 0, UINT64_C(0xfffffffffffffffe)),
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    RSN(0x000000c1, 0x000000c4, "IA32_PMCn", Ia32PmcN, Ia32PmcN, 0x0, ~(uint64_t)UINT32_MAX, 0),
+    MFO(0x000000ce, "IA32_PLATFORM_INFO", IntelPlatformInfo), /* value=0x80838`f1012800 */
+    MFX(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl, IntelPkgCStConfigControl, 0, 0, UINT64_C(0xffffffff01fffbf0)), /* value=0x1e000006 */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, 0, UINT64_C(0xfffffffffff80000)), /* value=0x31814 */
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf), /* value=0x693`992a0bba */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x2d8`96416f36 */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x1d0a, 0, 0), /* value=0x1d0a */
+    MVX(0x00000102, "I7_IB_UNK_0000_0102", 0, 0, UINT64_C(0xffffffff7fff8000)),
+    MVX(0x00000103, "I7_IB_UNK_0000_0103", 0, 0, UINT64_C(0xffffffffffffff00)),
+    MVX(0x00000104, "I7_IB_UNK_0000_0104", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MVO(0x00000110, "TODO_0000_0110", 0x1),
+    MVO(0x00000118, "BBL_CR_DECC", 0),
+    MVX(0x0000011f, "TODO_0000_011f", 0, 0, UINT64_C(0xffffffffffffff00)),
+    MVO(0x00000121, "TODO_0000_0121", 0),
+    MVO(0x0000013a, "TODO_0000_013a", 0),
+    MFO(0x0000013c, "I7_SB_AES_NI_CTL", IntelI7SandyAesNiCtl), /* value=0x1 */
+    MVX(0x0000013d, "TODO_0000_013d", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MVX(0x00000140, "I7_IB_UNK_0000_0140", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MVX(0x00000150, "P6_UNK_0000_0150", 0, UINT64_C(0x8000ffffffffffff), UINT64_C(0x7fff000000000000)),
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0x0 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0x0 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0xc0a, 0, 0), /* value=0xc0a */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, 0, UINT64_C(0xfffffffffffffff0)), /* value=0x0 */
+    RSN(0x00000186, 0x00000187, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, 0, UINT64_C(0xfffffffe00080000)), /* XXX: The range ended earlier than expected! */
+    MVX(0x00000188, "IA32_PERFEVTSEL2", 0, 0, UINT64_C(0xfffffffc00080000)),
+    MVX(0x00000189, "IA32_PERFEVTSEL3", 0, 0, UINT64_C(0xfffffffe00080000)),
+    MFX(0x00000194, "CLOCK_FLEX_MAX", IntelFlexRatio, IntelFlexRatio, 0xf0000, 0xe0000, UINT64_C(0xffffffffffe00000)), /* value=0xf0000 */
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0x29eb00002a00), 0, 0), /* value=0x29eb`00002a00 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x2800, 0, 0), /* Might bite. value=0x2800 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x0 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0, 0, UINT64_C(0xfffffffffe0000e8)), /* value=0x0 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, UINT32_C(0x88430800), UINT32_C(0xf87fa7ff), UINT64_C(0xffffffff07805000)), /* value=0x88430800 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0, 0, 0), /* value=0x0 */
+/// @todo WARNING: IA32_MISC_ENABLE probing needs hacking on this CPU!
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, 0x850089, 0x1080, UINT64_C(0xffffffbbff3aef76)), /* value=0x850089 */
+    MVO(0x000001a1, "P6_UNK_0000_01a1", 0x2858),
+    MFX(0x000001a2, "I7_MSR_TEMPERATURE_TARGET", IntelI7TemperatureTarget, IntelI7TemperatureTarget, 0x641400, 0xffff00, UINT64_C(0xffffffff40000000)), /* value=0x641400 */
+    MVX(0x000001a4, "I7_UNK_0000_01a4", 0, 0, UINT64_C(0xfffffffffffff7f0)),
+    RSN(0x000001a6, 0x000001a7, "I7_MSR_OFFCORE_RSP_n", IntelI7MsrOffCoreResponseN, IntelI7MsrOffCoreResponseN, 0x0, 0, UINT64_C(0xffffffc000006000)),
+    MVX(0x000001a8, "I7_UNK_0000_01a8", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFX(0x000001aa, "MSR_MISC_PWR_MGMT", IntelI7MiscPwrMgmt, IntelI7MiscPwrMgmt, 0, 0x800, UINT64_C(0xffffffffffbff7fe)), /* value=0x401cc0 */
+    MFX(0x000001ad, "I7_MSR_TURBO_RATIO_LIMIT", IntelI7TurboRatioLimit, IntelI7TurboRatioLimit, 0x2a2a2a2a, UINT64_MAX, 0), /* value=0x2a2a2a2a */
+    MVX(0x000001b0, "IA32_ENERGY_PERF_BIAS", 0x6, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x000001b1, "IA32_PACKAGE_THERM_STATUS", UINT32_C(0x88370800), UINT32_C(0xf87f07ff), UINT64_C(0xffffffff0780f000)),
+    MVX(0x000001b2, "IA32_PACKAGE_THERM_INTERRUPT", 0, 0, UINT64_C(0xfffffffffe0000e8)),
+    MVO(0x000001c6, "I7_UNK_0000_01c6", 0x3),
+    MFX(0x000001c8, "MSR_LBR_SELECT", IntelI7LbrSelect, IntelI7LbrSelect, 0, 0, UINT64_C(0xfffffffffffffc00)), /* value=0x0 */
+    MFX(0x000001c9, "MSR_LASTBRANCH_TOS", IntelLastBranchTos, IntelLastBranchTos, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x0 */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffff003c)), /* value=0x0 */
+    MVO(0x000001da, "TODO_0000_01da", 0),
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0x0 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0x0 */
+    MFN(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp, P6LastIntFromIp), /* value=0x0 */
+    MFN(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp, P6LastIntToIp), /* value=0x0 */
+    MVX(0x000001e0, "MSR_ROB_CR_BKUPTMPDR6", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MFO(0x000001f0, "I7_VLW_CAPABILITY", IntelI7VirtualLegacyWireCap), /* value=0x74 */
+    MFO(0x000001f2, "IA32_SMRR_PHYSBASE", Ia32SmrrPhysBase), /* value=0x88400006 */
+    MFO(0x000001f3, "IA32_SMRR_PHYSMASK", Ia32SmrrPhysMask), /* value=0xffc00800 */
+    MVO(0x000001f4, "TODO_0000_01f4", UINT32_C(0x88000006)),
+    MVO(0x000001f5, "TODO_0000_01f5", UINT64_C(0x7ffff00c00)),
+    MVO(0x000001fb, "TODO_0000_01fb", 0xe1),
+    MFX(0x000001fc, "I7_MSR_POWER_CTL", IntelI7PowerCtl, IntelI7PowerCtl, 0, UINT32_C(0x80000020), UINT64_C(0xffffffff20000000)), /* value=0x3c005f */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xffffff8000000ff8)), /* value=0xc0000000 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`c0000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xffffff8000000ff8)), /* value=0xa0000000 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`e0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x90000000 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`f0000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x8c000000 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`fc000800 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x8a000000 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`fe000800 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x89000000 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`ff000800 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x88800000 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xffffff80000007ff)), /* value=0x7f`ff800800 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xffffff80000007ff)), /* value=0x0 */
+    MFX(0x00000210, "IA32_MTRR_PHYS_BASE8", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x8, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x0 */
+    MFX(0x00000211, "IA32_MTRR_PHYS_MASK8", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x8, 0, UINT64_C(0xffffff80000007ff)), /* value=0x0 */
+    MFX(0x00000212, "IA32_MTRR_PHYS_BASE9", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x9, 0, UINT64_C(0xffffff8000000ff8)), /* value=0x0 */
+    MFX(0x00000213, "IA32_MTRR_PHYS_MASK9", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x9, 0, UINT64_C(0xffffff80000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    RSN(0x00000280, 0x00000283, "IA32_MC0_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0x0, 0, UINT64_C(0xffffffffbfff8000)),
+    RSN(0x00000284, 0x00000285, "IA32_MC4_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0x4, 0x40007fff, UINT64_C(0xffffffffbfff8000)),
+    RSN(0x00000286, 0x00000289, "IA32_MC6_CTLn", Ia32McNCtl2, Ia32McNCtl2, 0x6, 0, UINT64_C(0xffffffffbfff8000)),
+    MVO(0x000002e0, "I7_SB_NO_EVICT_MODE", 0),
+    MVX(0x000002e7, "I7_IB_UNK_0000_02e7", 0x1, 0x1, UINT64_C(0xfffffffffffffffe)),
+    MVO(0x000002f4, "TODO_0000_02f4", UINT32_C(0x88000000)),
+    MVO(0x000002f5, "TODO_0000_02f5", UINT64_C(0x7ffff00c00)),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    MVX(0x00000302, "TODO_0000_0302", UINT64_C(0x1ffff020000), UINT64_C(0xfe0000fd0000), UINT64_C(0xffff00000000ffff)),
+    MVO(0x00000305, "I7_SB_UNK_0000_0305", 0),
+    RSN(0x00000309, 0x0000030b, "IA32_FIXED_CTRn", Ia32FixedCtrN, Ia32FixedCtrN, 0x0, 0, UINT64_C(0xffff000000000000)),
+    MFX(0x00000345, "IA32_PERF_CAPABILITIES", Ia32PerfCapabilities, ReadOnly, 0x33c5, 0, 0), /* value=0x33c5 */
+    MFX(0x0000038d, "IA32_FIXED_CTR_CTRL", Ia32FixedCtrCtrl, Ia32FixedCtrCtrl, 0, 0, UINT64_C(0xfffffffffffff000)), /* value=0x0 */
+    MFX(0x0000038e, "IA32_PERF_GLOBAL_STATUS", Ia32PerfGlobalStatus, ReadOnly, 0x1, 0, 0), /* value=0x1 */
+    MFX(0x0000038f, "IA32_PERF_GLOBAL_CTRL", Ia32PerfGlobalCtrl, Ia32PerfGlobalCtrl, 0, 0, UINT64_C(0xfffffff8fffffff0)), /* value=0xf */
+    MFX(0x00000390, "IA32_PERF_GLOBAL_OVF_CTRL", Ia32PerfGlobalOvfCtrl, Ia32PerfGlobalOvfCtrl, 0, UINT64_C(0xfc8000070000000f), UINT64_C(0x37ffff8fffffff0)), /* value=0x0 */
+    MFX(0x00000391, "I7_UNC_PERF_GLOBAL_CTRL", IntelI7UncPerfGlobalCtrl, IntelI7UncPerfGlobalCtrl, 0, UINT64_C(0xfc8000070000000f), UINT64_C(0x37ffff8fffffff0)), /* value=0x0 */
+    MFO(0x00000392, "I7_UNC_PERF_GLOBAL_STATUS", IntelI7UncPerfGlobalStatus), /* value=0x0 */
+    MFX(0x00000393, "I7_UNC_PERF_GLOBAL_OVF_CTRL", IntelI7UncPerfGlobalOvfCtrl, IntelI7UncPerfGlobalOvfCtrl, 0, 0x3, UINT64_C(0xfffffffffffffffc)), /* value=0x0 */
+    MFX(0x00000394, "I7_UNC_PERF_FIXED_CTR_CTRL", IntelI7UncPerfFixedCtrCtrl, IntelI7UncPerfFixedCtrCtrl, 0, 0, UINT64_C(0xffffffffffafffff)), /* value=0x0 */
+    MFX(0x00000395, "I7_UNC_PERF_FIXED_CTR", IntelI7UncPerfFixedCtr, IntelI7UncPerfFixedCtr, 0, 0, UINT64_C(0xffff000000000000)), /* value=0x0 */
+    MFO(0x00000396, "I7_UNC_CBO_CONFIG", IntelI7UncCBoxConfig), /* value=0x5 */
+    MVX(0x00000397, "I7_SB_UNK_0000_0397", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MFX(0x000003b0, "I7_UNC_ARB_PERF_CTR0", IntelI7UncArbPerfCtrN, IntelI7UncArbPerfCtrN, 0, 0, UINT64_C(0xfffff00000000000)), /* value=0x0 */
+    MFX(0x000003b1, "I7_UNC_ARB_PERF_CTR1", IntelI7UncArbPerfCtrN, IntelI7UncArbPerfCtrN, 0, 0, UINT64_C(0xfffff00000000000)), /* value=0x0 */
+    MFX(0x000003b2, "I7_UNC_ARB_PERF_EVT_SEL0", IntelI7UncArbPerfEvtSelN, IntelI7UncArbPerfEvtSelN, 0, 0, UINT64_C(0xffffffffe0230000)), /* value=0x0 */
+    MFX(0x000003b3, "I7_UNC_ARB_PERF_EVT_SEL1", IntelI7UncArbPerfEvtSelN, IntelI7UncArbPerfEvtSelN, 0, 0, UINT64_C(0xffffffffe0230000)), /* value=0x0 */
+    MFX(0x000003f1, "IA32_PEBS_ENABLE", Ia32PebsEnable, Ia32PebsEnable, 0, 0, UINT64_C(0xfffffff0fffffff0)), /* value=0x0 */
+    MFX(0x000003f6, "I7_MSR_PEBS_LD_LAT", IntelI7PebsLdLat, IntelI7PebsLdLat, 0, UINT64_C(0xffffffffffff0000), 0), /* value=0xffff */
+    MVX(0x000003f7, "I7_MSR_PEBS_LD_LAT", 0x800, 0, UINT64_C(0xffffffffff8000e8)),
+    MFX(0x000003f8, "I7_MSR_PKG_C3_RESIDENCY", IntelI7PkgCnResidencyN, ReadOnly, 0x3, 0, UINT64_MAX), /* value=0x4fd`b403a690 */
+    RSN(0x000003f9, 0x000003fa, "I7_MSR_PKG_Cn_RESIDENCY", IntelI7PkgCnResidencyN, ReadOnly, 0x6, 0, UINT64_MAX),
+    MFX(0x000003fc, "I7_MSR_CORE_C3_RESIDENCY", IntelI7CoreCnResidencyN, ReadOnly, 0x3, 0, UINT64_MAX), /* value=0x8d`96b4ea78 */
+    RSN(0x000003fd, 0x000003fe, "I7_MSR_CORE_Cn_RESIDENCY", IntelI7CoreCnResidencyN, ReadOnly, 0x6, 0, UINT64_MAX),
+    RFN(0x00000400, 0x00000427, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0x00000480, "IA32_VMX_BASIC", Ia32VmxBasic, ReadOnly, UINT64_C(0xda040000000004), 0, 0), /* value=0xda0400`00000004 */
+    MFX(0x00000481, "IA32_VMX_PINBASED_CTLS", Ia32VmxPinbasedCtls, ReadOnly, UINT64_C(0x7f00000016), 0, 0), /* value=0x7f`00000016 */
+    MFX(0x00000482, "IA32_VMX_PROCBASED_CTLS", Ia32VmxProcbasedCtls, ReadOnly, UINT64_C(0xfff9fffe0401e172), 0, 0), /* value=0xfff9fffe`0401e172 */
+    MFX(0x00000483, "IA32_VMX_EXIT_CTLS", Ia32VmxExitCtls, ReadOnly, UINT64_C(0x1ffffff00036dff), 0, 0), /* value=0x1ffffff`00036dff */
+    MFX(0x00000484, "IA32_VMX_ENTRY_CTLS", Ia32VmxEntryCtls, ReadOnly, UINT64_C(0x3ffff000011ff), 0, 0), /* value=0x3ffff`000011ff */
+    MFX(0x00000485, "IA32_VMX_MISC", Ia32VmxMisc, ReadOnly, 0x7004c1e7, 0, 0), /* value=0x7004c1e7 */
+    MFX(0x00000486, "IA32_VMX_CR0_FIXED0", Ia32VmxCr0Fixed0, ReadOnly, UINT32_C(0x80000021), 0, 0), /* value=0x80000021 */
+    MFX(0x00000487, "IA32_VMX_CR0_FIXED1", Ia32VmxCr0Fixed1, ReadOnly, UINT32_MAX, 0, 0), /* value=0xffffffff */
+    MFX(0x00000488, "IA32_VMX_CR4_FIXED0", Ia32VmxCr4Fixed0, ReadOnly, 0x2000, 0, 0), /* value=0x2000 */
+    MFX(0x00000489, "IA32_VMX_CR4_FIXED1", Ia32VmxCr4Fixed1, ReadOnly, 0x3727ff, 0, 0), /* value=0x3727ff */
+    MFX(0x0000048a, "IA32_VMX_VMCS_ENUM", Ia32VmxVmcsEnum, ReadOnly, 0x2e, 0, 0), /* value=0x2e */
+    MFX(0x0000048b, "IA32_VMX_PROCBASED_CTLS2", Ia32VmxProcBasedCtls2, ReadOnly, UINT64_C(0x1ffcff00000000), 0, 0), /* value=0x1ffcff`00000000 */
+    MFX(0x0000048c, "IA32_VMX_EPT_VPID_CAP", Ia32VmxEptVpidCap, ReadOnly, UINT64_C(0xf0106334141), 0, 0), /* value=0xf01`06334141 */
+    MFX(0x0000048d, "IA32_VMX_TRUE_PINBASED_CTLS", Ia32VmxTruePinbasedCtls, ReadOnly, UINT64_C(0x7f00000016), 0, 0), /* value=0x7f`00000016 */
+    MFX(0x0000048e, "IA32_VMX_TRUE_PROCBASED_CTLS", Ia32VmxTrueProcbasedCtls, ReadOnly, UINT64_C(0xfff9fffe04006172), 0, 0), /* value=0xfff9fffe`04006172 */
+    MFX(0x0000048f, "IA32_VMX_TRUE_EXIT_CTLS", Ia32VmxTrueExitCtls, ReadOnly, UINT64_C(0x1ffffff00036dfb), 0, 0), /* value=0x1ffffff`00036dfb */
+    MFX(0x00000490, "IA32_VMX_TRUE_ENTRY_CTLS", Ia32VmxTrueEntryCtls, ReadOnly, UINT64_C(0x3ffff000011fb), 0, 0), /* value=0x3ffff`000011fb */
+    MFX(0x00000491, "IA32_VMX_VMFUNC", Ia32VmxVmFunc, ReadOnly, 0x1, 0, 0), /* value=0x1 */
+    RSN(0x000004c1, 0x000004c4, "IA32_A_PMCn", Ia32PmcN, Ia32PmcN, 0x0, 0, UINT64_C(0xffff000000000000)),
+    MVO(0x000004e0, "TODO_0000_04e0", 0x5),
+    MVO(0x000004e2, "TODO_0000_04e2", 0x2),
+    MVO(0x000004e3, "TODO_0000_04e3", 0xf00),
+    MVO(0x00000500, "TODO_0000_0500", 0),
+    MVX(0x00000503, "TODO_0000_0503", 0, 0x2, UINT64_C(0xfffffffffffffffd)),
+    MVX(0x00000560, "TODO_0000_0560", 0, 0, UINT64_C(0xffffff800000007f)),
+    MVX(0x00000561, "TODO_0000_0561", 0x7f, UINT64_C(0xf0000007f), 0),
+    MVX(0x00000570, "TODO_0000_0570", 0, 0, UINT64_C(0xffffffccf887d070)),
+    MVX(0x00000571, "TODO_0000_0571", 0, UINT64_C(0xf00000007), UINT64_C(0xfffe0000ffffffc8)),
+    MVX(0x00000572, "TODO_0000_0572", 0, 0, 0xfff),
+    MVX(0x00000580, "TODO_0000_0580", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x00000581, "TODO_0000_0581", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x00000582, "TODO_0000_0582", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x00000583, "TODO_0000_0583", 0, 0, UINT64_C(0xffff800000000000)),
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0x0 */
+    MFX(0x00000601, "I7_SB_MSR_VR_CURRENT_CONFIG", IntelI7SandyVrCurrentConfig, IntelI7SandyVrCurrentConfig, 0, 0, UINT64_C(0x800000007fffe000)), /* value=0x0 */
+    MFX(0x00000603, "I7_SB_MSR_VR_MISC_CONFIG", IntelI7SandyVrMiscConfig, IntelI7SandyVrMiscConfig, 0, 0, UINT64_C(0xff80000000000000)), /* value=0x360000`00363636 */
+    MFO(0x00000606, "I7_SB_MSR_RAPL_POWER_UNIT", IntelI7SandyRaplPowerUnit), /* value=0xa0e03 */
+    MVX(0x00000607, "TODO_0000_0607", 0, 0, UINT64_C(0xffffffff60000000)),
+    MVX(0x00000608, "TODO_0000_0608", 0, 0, ~(uint64_t)UINT32_MAX),
+    MVX(0x00000609, "I7_SB_UNK_0000_0609", 0, 0xc0, UINT64_C(0xffffffffffffff00)),
+    MFX(0x0000060a, "I7_SB_MSR_PKGC3_IRTL", IntelI7SandyPkgCnIrtlN, IntelI7SandyPkgCnIrtlN, 0x3, 0, UINT64_C(0xffffffffffff6000)), /* value=0x884e */
+    RSN(0x0000060b, 0x0000060c, "I7_SB_MSR_PKGC6_IRTn", IntelI7SandyPkgCnIrtlN, IntelI7SandyPkgCnIrtlN, 0x6, 0, UINT64_C(0xffffffffffff6000)),
+    MFO(0x0000060d, "I7_SB_MSR_PKG_C2_RESIDENCY", IntelI7SandyPkgC2Residency), /* value=0x3c6`052d9140 */
+    MFX(0x00000610, "I7_SB_MSR_PKG_POWER_LIMIT", IntelI7RaplPkgPowerLimit, IntelI7RaplPkgPowerLimit, 0, 0x8000, UINT64_C(0x7f000000ff000000)), /* value=0x42fff8`0015fff8 */
+    MFO(0x00000611, "I7_SB_MSR_PKG_ENERGY_STATUS", IntelI7RaplPkgEnergyStatus), /* value=0x79ba094a */
+    MFO(0x00000613, "I7_SB_MSR_PKG_PERF_STATUS", IntelI7RaplPkgPerfStatus), /* value=0x1 */
+    MFO(0x00000614, "I7_SB_MSR_PKG_POWER_INFO", IntelI7RaplPkgPowerInfo), /* value=0x2f8 */
+    MVX(0x00000615, "TODO_0000_0615", UINT32_C(0x80000000), UINT32_C(0xfffeffff), UINT64_C(0xffffffff00010000)),
+    MFX(0x00000618, "I7_SB_MSR_DRAM_POWER_LIMIT", IntelI7RaplDramPowerLimit, IntelI7RaplDramPowerLimit, 0, 0, UINT64_C(0x7f010000ff010000)), /* value=0x5400de`00000000 */
+    MFO(0x00000619, "I7_SB_MSR_DRAM_ENERGY_STATUS", IntelI7RaplDramEnergyStatus), /* value=0xf282d33 */
+    MFO(0x0000061b, "I7_SB_MSR_DRAM_PERF_STATUS", IntelI7RaplDramPerfStatus), /* value=0x0 */
+    MVO(0x0000061d, "TODO_0000_061d", UINT64_C(0x7db7e4dfa38)),
+    MVX(0x00000620, "TODO_0000_0620", 0x829, 0, UINT64_C(0xffffffffffff8080)),
+    MVO(0x00000621, "TODO_0000_0621", 0x29),
+    MVX(0x00000622, "TODO_0000_0622", 0x1, 0, UINT64_C(0xfffffffffffffffe)),
+    MVO(0x00000623, "TODO_0000_0623", 0x1),
+    MVO(0x00000630, "TODO_0000_0630", 0),
+    MVO(0x00000631, "TODO_0000_0631", 0),
+    MVO(0x00000632, "TODO_0000_0632", 0),
+    MVX(0x00000633, "TODO_0000_0633", 0, 0, UINT64_C(0xffffffffffff6000)),
+    MVX(0x00000634, "TODO_0000_0634", 0, 0, UINT64_C(0xffffffffffff6000)),
+    MVX(0x00000635, "TODO_0000_0635", 0, 0, UINT64_C(0xffffffffffff6000)),
+    MVX(0x00000636, "TODO_0000_0636", 0, 0, UINT64_C(0xffffffffffff0000)),
+    MVO(0x00000637, "TODO_0000_0637", UINT64_C(0x496ce31e72)),
+    MFX(0x00000638, "I7_SB_MSR_PP0_POWER_LIMIT", IntelI7RaplPp0PowerLimit, IntelI7RaplPp0PowerLimit, 0, 0, UINT64_C(0xffffffff7f000000)), /* value=0x0 */
+    MFO(0x00000639, "I7_SB_MSR_PP0_ENERGY_STATUS", IntelI7RaplPp0EnergyStatus), /* value=0x2aa89b8a */
+    MFX(0x0000063a, "I7_SB_MSR_PP0_POLICY", IntelI7RaplPp0Policy, IntelI7RaplPp0Policy, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x10 */
+    MFX(0x00000640, "I7_HW_MSR_PP0_POWER_LIMIT", IntelI7RaplPp1PowerLimit, IntelI7RaplPp1PowerLimit, 0, 0, UINT64_C(0xffffffff7f000000)), /* value=0x0 */
+    MFO(0x00000641, "I7_HW_MSR_PP0_ENERGY_STATUS", IntelI7RaplPp1EnergyStatus), /* value=0x0 */
+    MFX(0x00000642, "I7_HW_MSR_PP0_POLICY", IntelI7RaplPp1Policy, IntelI7RaplPp1Policy, 0, 0, UINT64_C(0xffffffffffffffe0)), /* value=0x10 */
+    MFO(0x00000648, "I7_IB_MSR_CONFIG_TDP_NOMINAL", IntelI7IvyConfigTdpNominal), /* value=0x28 */
+    MFO(0x00000649, "I7_IB_MSR_CONFIG_TDP_LEVEL1", IntelI7IvyConfigTdpLevel1), /* value=0x0 */
+    MFO(0x0000064a, "I7_IB_MSR_CONFIG_TDP_LEVEL2", IntelI7IvyConfigTdpLevel2), /* value=0x0 */
+    MFO(0x0000064b, "I7_IB_MSR_CONFIG_TDP_CONTROL", IntelI7IvyConfigTdpControl), /* value=0x80000000 */
+    MFX(0x0000064c, "I7_IB_MSR_TURBO_ACTIVATION_RATIO", IntelI7IvyTurboActivationRatio, IntelI7IvyTurboActivationRatio, 0, 0, UINT64_C(0xffffffff7fffff00)), /* value=0x0 */
+    MVO(0x0000064d, "TODO_0000_064d", 0),
+    MVO(0x0000064e, "TODO_0000_064e", UINT64_C(0x1fdf361be5b)),
+    MVX(0x0000064f, "TODO_0000_064f", 0x4000000, UINT32_C(0xfbffffff), ~(uint64_t)UINT32_MAX),
+    MVX(0x00000652, "TODO_0000_0652", 0x2, 0, UINT64_C(0xfffffffff8000800)),
+    MVO(0x00000653, "TODO_0000_0653", 0),
+    MVO(0x00000655, "TODO_0000_0655", 0),
+    MVO(0x00000656, "TODO_0000_0656", 0),
+    MVO(0x00000658, "TODO_0000_0658", UINT64_C(0x296db63257a5)),
+    MVO(0x00000659, "TODO_0000_0659", UINT64_C(0x195cb5c8d10c)),
+    MVO(0x0000065a, "TODO_0000_065a", 0),
+    MVO(0x0000065b, "TODO_0000_065b", 0),
+    MVX(0x0000065c, "TODO_0000_065c", 0x1402f8, 0, UINT64_C(0x7ffe0000ff000000)),
+    RFN(0x00000680, 0x0000068f, "MSR_LASTBRANCH_n_FROM_IP", IntelLastBranchFromN, IntelLastBranchFromN),
+    MVX(0x00000690, "TODO_0000_0690", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x00000691, "TODO_0000_0691", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x00000692, "TODO_0000_0692", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x00000693, "TODO_0000_0693", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x00000694, "TODO_0000_0694", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x00000695, "TODO_0000_0695", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x00000696, "TODO_0000_0696", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x00000697, "TODO_0000_0697", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x00000698, "TODO_0000_0698", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x00000699, "TODO_0000_0699", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x0000069a, "TODO_0000_069a", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x0000069b, "TODO_0000_069b", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x0000069c, "TODO_0000_069c", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x0000069d, "TODO_0000_069d", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x0000069e, "TODO_0000_069e", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x0000069f, "TODO_0000_069f", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006b0, "TODO_0000_06b0", 0, UINT32_MAX, ~(uint64_t)UINT32_MAX),
+    MVX(0x000006b1, "TODO_0000_06b1", 0xc000000, UINT32_C(0xf3ffffff), ~(uint64_t)UINT32_MAX),
+    RFN(0x000006c0, 0x000006cf, "MSR_LASTBRANCH_n_TO_IP", IntelLastBranchToN, IntelLastBranchToN),
+    MVX(0x000006d0, "TODO_0000_06d0", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006d1, "TODO_0000_06d1", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006d2, "TODO_0000_06d2", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006d3, "TODO_0000_06d3", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006d4, "TODO_0000_06d4", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006d5, "TODO_0000_06d5", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006d6, "TODO_0000_06d6", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006d7, "TODO_0000_06d7", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006d8, "TODO_0000_06d8", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006d9, "TODO_0000_06d9", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006da, "TODO_0000_06da", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006db, "TODO_0000_06db", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006dc, "TODO_0000_06dc", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006dd, "TODO_0000_06dd", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006de, "TODO_0000_06de", 0, 0, UINT64_C(0xffff800000000000)),
+    MVX(0x000006df, "TODO_0000_06df", 0, 0, UINT64_C(0xffff800000000000)),
+    MFI(0x000006e0, "IA32_TSC_DEADLINE", Ia32TscDeadline), /* value=0x0 */
+    MVX(0x00000700, "TODO_0000_0700", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000701, "TODO_0000_0701", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000702, "TODO_0000_0702", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000703, "TODO_0000_0703", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000704, "TODO_0000_0704", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000705, "TODO_0000_0705", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000706, "TODO_0000_0706", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000707, "TODO_0000_0707", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000708, "TODO_0000_0708", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000709, "TODO_0000_0709", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000710, "TODO_0000_0710", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000711, "TODO_0000_0711", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000712, "TODO_0000_0712", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000713, "TODO_0000_0713", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000714, "TODO_0000_0714", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000715, "TODO_0000_0715", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000716, "TODO_0000_0716", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000717, "TODO_0000_0717", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000718, "TODO_0000_0718", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000719, "TODO_0000_0719", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000720, "TODO_0000_0720", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000721, "TODO_0000_0721", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000722, "TODO_0000_0722", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000723, "TODO_0000_0723", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000724, "TODO_0000_0724", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000725, "TODO_0000_0725", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000726, "TODO_0000_0726", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000727, "TODO_0000_0727", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000728, "TODO_0000_0728", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000729, "TODO_0000_0729", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000730, "TODO_0000_0730", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000731, "TODO_0000_0731", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000732, "TODO_0000_0732", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000733, "TODO_0000_0733", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000734, "TODO_0000_0734", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000735, "TODO_0000_0735", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000736, "TODO_0000_0736", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000737, "TODO_0000_0737", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000738, "TODO_0000_0738", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000739, "TODO_0000_0739", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000740, "TODO_0000_0740", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000741, "TODO_0000_0741", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000742, "TODO_0000_0742", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000743, "TODO_0000_0743", 0, 0, UINT64_C(0xffffffffe0230000)),
+    MVX(0x00000744, "TODO_0000_0744", 0, 0, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000745, "TODO_0000_0745", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MVX(0x00000746, "TODO_0000_0746", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000747, "TODO_0000_0747", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000748, "TODO_0000_0748", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000749, "TODO_0000_0749", 0, 0, UINT64_C(0xfffff00000000000)),
+    MVX(0x00000770, "TODO_0000_0770", 0x1, 0x1, UINT64_C(0xfffffffffffffffe)),
+    MVO(0x00000771, "TODO_0000_0771", 0x109282a),
+    MVX(0x00000773, "TODO_0000_0773", 0x1, 0, UINT64_C(0xfffffffffffffffc)),
+    MVX(0x00000774, "TODO_0000_0774", UINT64_C(0x19e7f2a2a02), 0, UINT64_C(0xfffffc0000000000)),
+    MVX(0x00000777, "TODO_0000_0777", 0, 0x5, UINT64_C(0xfffffffffffffffa)),
+    MFO(0x00000c80, "IA32_DEBUG_INTERFACE", Ia32DebugInterface), /* value=0x40000000 */
+    MVX(0x00000c8f, "TODO_0000_0c8f", 0, 0, UINT64_C(0xffffffffffffffe0)),
+    MVX(0x00000c90, "TODO_0000_0c90", UINT16_MAX, 0, UINT64_C(0xffffffff7fff0000)),
+    MVX(0x00000c91, "TODO_0000_0c91", 0xf, 0, UINT64_C(0xffffffff7fff0000)),
+    MVX(0x00000c92, "TODO_0000_0c92", 0x3ff, 0, UINT64_C(0xffffffff7fff0000)),
+    MVX(0x00000c93, "TODO_0000_0c93", 0xfff, 0, UINT64_C(0xffffffff7fff0000)),
+    MVX(0x00000d90, "TODO_0000_0d90", 0, 0, UINT64_C(0xffff800000000ffc)),
+    MVX(0x00000da0, "TODO_0000_0da0", 0, 0, UINT64_C(0xfffffffffffffeff)),
+    MVX(0x00000db0, "TODO_0000_0db0", 0, 0, UINT64_C(0xfffffffffffffffe)),
+    MVX(0x00000db1, "TODO_0000_0db1", 0x1, 0, UINT64_C(0xfffffffffffffffe)),
+    MVO(0x00000db2, "TODO_0000_0db2", 0),
+    MVX(0x00000dc0, "TODO_0000_0dc0", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dc1, "TODO_0000_0dc1", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dc2, "TODO_0000_0dc2", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dc3, "TODO_0000_0dc3", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dc4, "TODO_0000_0dc4", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dc5, "TODO_0000_0dc5", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dc6, "TODO_0000_0dc6", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dc7, "TODO_0000_0dc7", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dc8, "TODO_0000_0dc8", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dc9, "TODO_0000_0dc9", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dca, "TODO_0000_0dca", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dcb, "TODO_0000_0dcb", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dcc, "TODO_0000_0dcc", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dcd, "TODO_0000_0dcd", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dce, "TODO_0000_0dce", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dcf, "TODO_0000_0dcf", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dd0, "TODO_0000_0dd0", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dd1, "TODO_0000_0dd1", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dd2, "TODO_0000_0dd2", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dd3, "TODO_0000_0dd3", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dd4, "TODO_0000_0dd4", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dd5, "TODO_0000_0dd5", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dd6, "TODO_0000_0dd6", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dd7, "TODO_0000_0dd7", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dd8, "TODO_0000_0dd8", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dd9, "TODO_0000_0dd9", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dda, "TODO_0000_0dda", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000ddb, "TODO_0000_0ddb", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000ddc, "TODO_0000_0ddc", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000ddd, "TODO_0000_0ddd", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000dde, "TODO_0000_0dde", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000ddf, "TODO_0000_0ddf", 0, 0, UINT64_C(0x1fffffffffff0000)),
+    MVX(0x00000e01, "TODO_0000_0e01", 0, 0, UINT64_C(0xffffffff1fffffe0)),
+    MVX(0x00000e02, "TODO_0000_0e02", 0, 0xf, UINT64_C(0xfffffffffffffff0)),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xfffffffffffff2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xfffff801`a09745c0 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xfffff801`a0974300 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x4700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x9a90000 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffffd000`c5800000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x7ff7`09a8e000 */
+    MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, 0, ~(uint64_t)UINT32_MAX), /* value=0x2 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Core_i7_6700K =
+{
+    /*.pszName          = */ "Intel Core i7-6700K",
+    /*.pszFullName      = */ "Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 94,
+    /*.uStepping        = */ 3,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_Core7_Skylake,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_100MHZ,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 39,
+    /*.fMxCsrMask       = */ 0xffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Core_i7_6700K),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Core_i7_6700K)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF,
+    /*.DefUnknownCpuId  = */ { 0x00000fa0, 0x00001068, 0x00000064, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Core_i7_6700K)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Core_i7_6700K),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Core_i7_6700K_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Pentium_4_3_00GHz.h b/src/VBox/VMM/VMMR3/cpus/Intel_Pentium_4_3_00GHz.h
new file mode 100644
index 00000000..f3a97ef0
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Pentium_4_3_00GHz.h
@@ -0,0 +1,277 @@
+/* $Id: Intel_Pentium_4_3_00GHz.h $ */
+/** @file
+ * CPU database entry "Intel Pentium 4 3.00GHz".
+ * Generated at 2013-12-18T06:37:54Z by VBoxCpuReport v4.3.53r91376 on win.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Pentium_4_3_00GHz_h
+#define VBOX_CPUDB_Intel_Pentium_4_3_00GHz_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Pentium(R) 4 CPU 3.00GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Pentium_4_3_00GHz[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000005, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00000f43, 0x00020800, 0x0000649d, 0xbfebfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x605b5001, 0x00000000, 0x00000000, 0x007d7040, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, UINT32_MAX, 0x00004121, 0x01c0003f, 0x0000001f, 0x00000000, 0 },
+    { 0x00000004, 0x00000001, UINT32_MAX, 0x00004143, 0x01c0103f, 0x000007ff, 0x00000000, 0 },
+    { 0x00000004, 0x00000002, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x20100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x6e492020, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x286c6574, 0x50202952, 0x69746e65, 0x52286d75, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x20342029, 0x20555043, 0x30302e33, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x08006040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003024, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Pentium(R) 4 CPU 3.00GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Pentium_4_3_00GHz[] =
+{
+    MFO(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr), /* value=0xc55df88 */
+    MFO(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType), /* value=0xbe000300`1008081f */
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, UINT64_C(0xffffffffffff0000), 0), /* value=0x40 */
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x1ac`2077a134 */
+    MFV(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0x12000000000000)),
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0x600, UINT64_C(0xffffff00000000ff)),
+    MFX(0x0000002a, "P4_EBC_HARD_POWERON", IntelP4EbcHardPowerOn, IntelP4EbcHardPowerOn, 0, UINT64_MAX, 0), /* value=0x0 */
+    MFX(0x0000002b, "P4_EBC_SOFT_POWERON", IntelP4EbcSoftPowerOn, IntelP4EbcSoftPowerOn, 0x7e, UINT64_C(0xffffffffffffff80), 0), /* value=0x7e */
+    MFX(0x0000002c, "P4_EBC_FREQUENCY_ID", IntelP4EbcFrequencyId, IntelP4EbcFrequencyId, 0xf12010f, UINT64_MAX, 0), /* value=0xf12010f */
+    MVX(0x00000039, "C2_UNK_0000_0039", 0x1, 0x1f, ~(uint64_t)UINT32_MAX),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, IgnoreWrite),
+    MVX(0x00000080, "P4_UNK_0000_0080", 0, ~(uint64_t)UINT32_MAX, UINT32_MAX),
+    MFX(0x0000008b, "IA32_BIOS_SIGN_ID", Ia32BiosSignId, Ia32BiosSignId, 0, UINT32_MAX, 0), /* value=0x5`00000000 */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x508, 0, 0), /* value=0x508 */
+    MFX(0x00000119, "BBL_CR_CTL", IntelBblCrCtl, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0x0 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0x0 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x180204, 0, 0), /* value=0x180204 */
+    MFN(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus), /* value=0x0 */
+    MVX(0x00000180, "MSR_MCG_RAX", 0, 0, UINT64_MAX),
+    MVX(0x00000181, "MSR_MCG_RBX", 0, 0, UINT64_MAX),
+    MVX(0x00000182, "MSR_MCG_RCX", 0, 0, UINT64_MAX),
+    MVX(0x00000183, "MSR_MCG_RDX", 0, 0, UINT64_MAX),
+    MVX(0x00000184, "MSR_MCG_RSI", 0, 0, UINT64_MAX),
+    MVX(0x00000185, "MSR_MCG_RDI", 0, 0, UINT64_MAX),
+    MFX(0x00000186, "MSR_MCG_RBP", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0, 0, UINT64_MAX), /* value=0x0 */
+    MFX(0x00000187, "MSR_MCG_RSP", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0, 0, UINT64_MAX), /* value=0x0 */
+    MVX(0x00000188, "MSR_MCG_RFLAGS", 0, 0, UINT64_MAX),
+    MVX(0x00000189, "MSR_MCG_RIP", 0, 0, UINT64_MAX),
+    MVX(0x0000018a, "MSR_MCG_MISC", 0, 0, UINT64_MAX),
+    MVX(0x0000018b, "MSR_MCG_RESERVED1", 0, 0, UINT64_MAX),
+    MVX(0x0000018c, "MSR_MCG_RESERVED2", 0, 0, UINT64_MAX),
+    MVX(0x0000018d, "MSR_MCG_RESERVED3", 0, 0, UINT64_MAX),
+    MVX(0x0000018e, "MSR_MCG_RESERVED4", 0, 0, UINT64_MAX),
+    MVX(0x0000018f, "MSR_MCG_RESERVED5", 0, 0, UINT64_MAX),
+    MVX(0x00000190, "MSR_MCG_R8", 0, 0, UINT64_MAX),
+    MVX(0x00000191, "MSR_MCG_R9", 0, 0, UINT64_MAX),
+    MVX(0x00000192, "MSR_MCG_R10", 0, 0, UINT64_MAX),
+    MVX(0x00000193, "MSR_MCG_R11", 0, 0, UINT64_MAX),
+    MVX(0x00000194, "MSR_MCG_R12", 0, 0, UINT64_MAX),
+    MVX(0x00000195, "MSR_MCG_R13", 0, 0, UINT64_MAX),
+    MVX(0x00000196, "MSR_MCG_R14", 0, 0, UINT64_MAX),
+    MVX(0x00000197, "MSR_MCG_R15", 0, 0, UINT64_MAX),
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, Ia32PerfStatus, UINT64_C(0xf2d00000f2d), UINT64_MAX, 0), /* value=0xf2d`00000f2d */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0xf2d, 0, 0), /* Might bite. value=0xf2d */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0, UINT64_C(0xffffffffffffffe1), 0), /* value=0x0 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0, UINT64_C(0xfffffffffffffffc), 0), /* value=0x0 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, 0, UINT64_C(0xfffffffffffffff5), 0), /* value=0x0 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0xe2d, 0, 0), /* value=0xe2d */
+    MVX(0x0000019e, "P6_UNK_0000_019e", 0, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0x0000019f, "P6_UNK_0000_019f", UINT64_C(0x32050500000101), UINT64_C(0xff000000fff0c0c0), 0),
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, 0x22850089, 0x20800080, UINT64_C(0xfffffffbdc10f800)), /* value=0x22850089 */
+    MVX(0x000001a1, "MSR_PLATFORM_BRV", 0, UINT64_C(0xfffffffffffcc0c0), 0),
+    MFX(0x000001a2, "P4_UNK_0000_01a2", IntelI7TemperatureTarget, ReadOnly, 0x61048, 0, 0), /* value=0x61048 */
+    MFO(0x000001d7, "MSR_LER_FROM_LIP", P6LastIntFromIp), /* value=0x0 */
+    MFO(0x000001d8, "MSR_LER_TO_LIP", P6LastIntToIp), /* value=0x0 */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffffff80)), /* value=0x0 */
+    MFX(0x000001da, "MSR_LASTBRANCH_TOS", IntelLastBranchTos, IntelLastBranchTos, 0, UINT64_C(0xfffffffffffffff0), 0), /* value=0x0 */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xffffff00000007ff)), /* value=0xf`c0000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x3f600000 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xffffff00000007ff)), /* value=0xf`ffe00800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x3f800000 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xffffff00000007ff)), /* value=0xf`ff800800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xffffff0000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xffffff00000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    MVX(0x00000300, "P4_MSR_BPU_COUNTER0", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00000301, "P4_MSR_BPU_COUNTER1", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00000302, "P4_MSR_BPU_COUNTER2", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00000303, "P4_MSR_BPU_COUNTER3", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00000304, "P4_MSR_MS_COUNTER0", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00000305, "P4_MSR_MS_COUNTER1", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00000306, "P4_MSR_MS_COUNTER2", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00000307, "P4_MSR_MS_COUNTER3", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00000308, "P4_MSR_FLAME_COUNTER0", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00000309, "P4_MSR_FLAME_COUNTER1", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x0000030a, "P4_MSR_FLAME_COUNTER2", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x0000030b, "P4_MSR_FLAME_COUNTER3", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x0000030c, "P4_MSR_IQ_COUNTER0", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x0000030d, "P4_MSR_IQ_COUNTER1", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x0000030e, "P4_MSR_IQ_COUNTER2", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x0000030f, "P4_MSR_IQ_COUNTER3", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00000310, "P4_MSR_IQ_COUNTER4", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00000311, "P4_MSR_IQ_COUNTER5", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00000360, "P4_MSR_BPU_CCCR0", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x00000361, "P4_MSR_BPU_CCCR1", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x00000362, "P4_MSR_BPU_CCCR2", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x00000363, "P4_MSR_BPU_CCCR3", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x00000364, "P4_MSR_MS_CCCR0", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x00000365, "P4_MSR_MS_CCCR1", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x00000366, "P4_MSR_MS_CCCR2", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x00000367, "P4_MSR_MS_CCCR3", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x00000368, "P4_MSR_FLAME_CCCR0", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x00000369, "P4_MSR_FLAME_CCCR1", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x0000036a, "P4_MSR_FLAME_CCCR2", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x0000036b, "P4_MSR_FLAME_CCCR3", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x0000036c, "P4_MSR_IQ_CCCR0", 0, UINT64_C(0xffffffff000007ff), 0),
+    MVX(0x0000036d, "P4_MSR_IQ_CCCR1", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x0000036e, "P4_MSR_IQ_CCCR2", 0, UINT64_C(0xffffffff00000fff), 0),
+    MVX(0x0000036f, "P4_MSR_IQ_CCCR3", 0, UINT64_C(0xffffffff000007ff), 0),
+    MVX(0x00000370, "P4_MSR_IQ_CCCR4", 0, UINT64_C(0xffffffff000000ff), 0),
+    MVX(0x00000371, "P4_MSR_IQ_CCCR5", 0, UINT64_C(0xffffffff000000ff), 0),
+    MVX(0x000003a0, "P4_MSR_BSU_ESCR0", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003a1, "P4_MSR_BSU_ESCR1", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003a2, "P4_MSR_FSB_ESCR0", 0, UINT64_C(0xffffffff40000000), UINT32_C(0x80000000)),
+    MVX(0x000003a3, "P4_MSR_FSB_ESCR1", 0, UINT64_C(0xffffffff40000000), UINT32_C(0x80000000)),
+    MVX(0x000003a4, "P4_MSR_FIRM_ESCR0", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003a5, "P4_MSR_FIRM_ESCR1", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003a6, "P4_MSR_FLAME_ESCR0", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003a7, "P4_MSR_FLAME_ESCR1", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003a8, "P4_MSR_DAC_ESCR0", 0, UINT64_C(0xffffffff61fe01f0), UINT32_C(0x80000000)),
+    MVX(0x000003a9, "P4_MSR_DAC_ESCR1", 0, UINT64_C(0xffffffff61fe01f0), UINT32_C(0x80000000)),
+    MVX(0x000003aa, "P4_MSR_MOB_ESCR0", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003ab, "P4_MSR_MOB_ESCR1", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003ac, "P4_MSR_PMH_ESCR0", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003ad, "P4_MSR_PMH_ESCR1", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003ae, "P4_MSR_SAAT_ESCR0", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003af, "P4_MSR_SAAT_ESCR1", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003b0, "P4_MSR_U2L_ESCR0", 0, UINT64_C(0xffffffff71c001f0), UINT32_C(0x80000000)),
+    MVX(0x000003b1, "P4_MSR_U2L_ESCR1", 0, UINT64_C(0xffffffff71c001f0), UINT32_C(0x80000000)),
+    MVX(0x000003b2, "P4_MSR_BPU_ESCR0", 0, UINT64_C(0xffffffff61fc0000), UINT32_C(0x80000000)),
+    MVX(0x000003b3, "P4_MSR_BPU_ESCR1", 0, UINT64_C(0xffffffff61fc0000), UINT32_C(0x80000000)),
+    MVX(0x000003b4, "P4_MSR_IS_ESCR0", 0, UINT64_C(0xffffffff71fe01f0), UINT32_C(0x80000000)),
+    MVX(0x000003b5, "P4_MSR_IS_ESCR1", 0, UINT64_C(0xffffffff71fe01f0), UINT32_C(0x80000000)),
+    MVX(0x000003b6, "P4_MSR_ITLB_ESCR0", 0, UINT64_C(0xffffffff0ffff1e0), UINT32_C(0x80000000)),
+    MVX(0x000003b7, "P4_MSR_ITLB_ESCR1", 0, UINT64_C(0xffffffff0ffff1e0), UINT32_C(0x80000000)),
+    MVX(0x000003b8, "P4_MSR_CRU_ESCR0", 0, UINT64_C(0xffffffff71fe01f0), UINT32_C(0x80000000)),
+    MVX(0x000003b9, "P4_MSR_CRU_ESCR1", 0, UINT64_C(0xffffffff71fe01f0), UINT32_C(0x80000000)),
+    MVX(0x000003ba, "P4_MSR_IQ_ESCR0", 0, UINT64_C(0xffffffff7fffffff), UINT32_C(0x80000000)),
+    MVX(0x000003bb, "P4_MSR_IQ_ESCR1", 0, UINT64_C(0xffffffff7fffffff), UINT32_C(0x80000000)),
+    MVX(0x000003bc, "P4_MSR_RAT_ESCR0", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003bd, "P4_MSR_RAT_ESCR1", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003be, "P4_MSR_SSU_ESCR0", 0, ~(uint64_t)UINT32_MAX, UINT32_C(0x80000000)),
+    MVX(0x000003c0, "P4_MSR_MS_ESCR0", 0, UINT64_C(0xffffffff61ff81e0), UINT32_C(0x80000000)),
+    MVX(0x000003c1, "P4_MSR_MS_ESCR1", 0, UINT64_C(0xffffffff61ff81e0), UINT32_C(0x80000000)),
+    MVX(0x000003c2, "P4_MSR_TBPU_ESCR0", 0, UINT64_C(0xffffffff71fe01f0), UINT32_C(0x80000000)),
+    MVX(0x000003c3, "P4_MSR_TBPU_ESCR1", 0, UINT64_C(0xffffffff71fe01f0), UINT32_C(0x80000000)),
+    MVX(0x000003c4, "P4_MSR_TC_ESCR0", 0, UINT64_C(0xffffffff61f801f0), UINT32_C(0x80000000)),
+    MVX(0x000003c5, "P4_MSR_TC_ESCR1", 0, UINT64_C(0xffffffff61f801f0), UINT32_C(0x80000000)),
+    MVX(0x000003c8, "P4_MSR_IX_ESCR0", 0, UINT64_C(0xffffffff71fe01f0), UINT32_C(0x80000000)),
+    MVX(0x000003c9, "P4_MSR_IX_ESCR0", 0, UINT64_C(0xffffffff71fe01f0), UINT32_C(0x80000000)),
+    MVX(0x000003ca, "P4_MSR_ALF_ESCR0", 0, UINT64_C(0xffffffff700001f0), UINT32_C(0x80000000)),
+    MVX(0x000003cb, "P4_MSR_ALF_ESCR1", 0, UINT64_C(0xffffffff700001f0), UINT32_C(0x80000000)),
+    MVX(0x000003cc, "P4_MSR_CRU_ESCR2", 0, UINT64_C(0xffffffff61f001f0), UINT32_C(0x80000000)),
+    MVX(0x000003cd, "P4_MSR_CRU_ESCR3", 0, UINT64_C(0xffffffff61f001f0), UINT32_C(0x80000000)),
+    MVX(0x000003e0, "P4_MSR_CRU_ESCR4", 0, UINT64_C(0xffffffff71ff01f0), UINT32_C(0x80000000)),
+    MVX(0x000003e1, "P4_MSR_CRU_ESCR5", 0, UINT64_C(0xffffffff71ff01f0), UINT32_C(0x80000000)),
+    MVX(0x000003f0, "P4_MSR_TC_PRECISE_EVENT", 0xfc00, UINT64_C(0xfffffffffffc001f), 0),
+    MFX(0x000003f1, "IA32_PEBS_ENABLE", Ia32PebsEnable, Ia32PebsEnable, 0, UINT64_C(0xfffffffff8000000), 0), /* value=0x0 */
+    MVX(0x000003f2, "P4_MSR_PEBS_MATRIX_VERT", 0, UINT64_C(0xffffffffffffe000), 0),
+    MVX(0x000003f5, "P4_UNK_0000_03f5", 0, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0x000003f6, "P4_UNK_0000_03f6", 0, UINT64_C(0xffffffffffe00000), 0),
+    MVX(0x000003f7, "P4_UNK_0000_03f7", 0, UINT64_C(0xfffe000000000000), 0),
+    MVX(0x000003f8, "P4_UNK_0000_03f8", 0, UINT64_C(0xffffff000000003f), 0),
+    RFN(0x00000400, 0x0000040f, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0x0 */
+    RFN(0x00000680, 0x0000068f, "MSR_LASTBRANCH_n_FROM_IP", IntelLastBranchFromN, IntelLastBranchFromN),
+    RFN(0x000006c0, 0x000006cf, "MSR_LASTBRANCH_n_TO_IP", IntelLastBranchToN, IntelLastBranchToN),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xfffffffffffff2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xfffff800`654efdc0 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xfffff800`654efb00 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x4700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0xeed1e000 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xfffff880`009bf000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x7f7`eed1c000 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Pentium(R) 4 CPU 3.00GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Pentium_4_3_00GHz =
+{
+    /*.pszName          = */ "Intel Pentium 4 3.00GHz",
+    /*.pszFullName      = */ "Intel(R) Pentium(R) 4 CPU 3.00GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 15,
+    /*.uModel           = */ 4,
+    /*.uStepping        = */ 3,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_NB_Prescott2M,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 36,
+    /*.fMxCsrMask       = */ 0xffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Pentium_4_3_00GHz),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Pentium_4_3_00GHz)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF,
+    /*.DefUnknownCpuId  = */ { 0x00000040, 0x00000040, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Pentium_4_3_00GHz)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Pentium_4_3_00GHz),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Pentium_4_3_00GHz_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Pentium_M_processor_2_00GHz.h b/src/VBox/VMM/VMMR3/cpus/Intel_Pentium_M_processor_2_00GHz.h
new file mode 100644
index 00000000..f2d0931a
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Pentium_M_processor_2_00GHz.h
@@ -0,0 +1,216 @@
+/* $Id: Intel_Pentium_M_processor_2_00GHz.h $ */
+/** @file
+ * CPU database entry "Intel Pentium M processor 2.00GHz".
+ * Generated at 2013-12-09T14:18:00Z by VBoxCpuReport v4.3.51r91027 on win.x86.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Pentium_M_processor_2_00GHz_h
+#define VBOX_CPUDB_Intel_Pentium_M_processor_2_00GHz_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Pentium(R) M processor 2.00GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Pentium_M_processor_2_00GHz[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000002, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000006d6, 0x00000816, 0x00000180, 0xafe9f9bf, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x02b3b001, 0x000000f0, 0x00000000, 0x2c04307d, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000004, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x65746e49, 0x2952286c, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x6e655020, 0x6d756974, 0x20295228, 0x7270204d, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x7365636f, 0x20726f73, 0x30302e32, 0x007a4847, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Pentium(R) M processor 2.00GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Pentium_M_processor_2_00GHz[] =
+{
+    MFI(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr), /* value=0x0 */
+    MFI(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType), /* value=0x0 */
+    MFX(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x22`4d44782e */
+    MFV(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0x140000d0248a28)),
+    MVX(0x00000018, "P6_UNK_0000_0018", 0, 0, 0),
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00100), UINT64_C(0xffffffff00000600), 0xff),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, IntelEblCrPowerOn, 0x45080000, UINT64_C(0xfffffffffff7ff7e), 0), /* value=0x45080000 */
+    MVX(0x0000002f, "P6_UNK_0000_002f", 0, UINT64_C(0xfffffffffffffff5), 0),
+    MVX(0x00000032, "P6_UNK_0000_0032", 0, UINT64_C(0xfffffffffffe0000), 0),
+    MVX(0x00000033, "TEST_CTL", 0, UINT64_C(0xffffffff40000000), 0),
+    MVX(0x00000034, "P6_UNK_0000_0034", 0x77ff, ~(uint64_t)UINT32_MAX, UINT32_C(0xfff80000)),
+    MVO(0x00000035, "P6_UNK_0000_0035", 0x300008),
+    MVX(0x0000003b, "P6_UNK_0000_003b", 0, UINT64_C(0xafffffffe), UINT64_C(0xfffffff500000001)),
+    MVO(0x0000003f, "P6_UNK_0000_003f", 0x4),
+    RFN(0x00000040, 0x00000047, "MSR_LASTBRANCH_n", IntelLastBranchFromToN, ReadOnly),
+    MVX(0x0000004a, "P6_UNK_0000_004a", 0, 0, 0), /* value=0x0 */
+    MVX(0x0000004b, "P6_UNK_0000_004b", 0, 0, 0), /* value=0x0 */
+    MVX(0x0000004c, "P6_UNK_0000_004c", 0, 0, 0), /* value=0x0 */
+    MVX(0x0000004d, "P6_UNK_0000_004d", 0, 0, 0), /* value=0xeb1cffbf`8918200a */
+    MVX(0x0000004e, "P6_UNK_0000_004e", 0, 0, 0), /* value=0x8204c60a`e8009512 */
+    MVX(0x0000004f, "P6_UNK_0000_004f", 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MVI(0x00000050, "P6_UNK_0000_0050", 0), /* Villain? value=0x0 */
+    MVI(0x00000051, "P6_UNK_0000_0051", 0), /* Villain? value=0x0 */
+    MVI(0x00000052, "P6_UNK_0000_0052", 0), /* Villain? value=0x0 */
+    MVI(0x00000053, "P6_UNK_0000_0053", 0), /* Villain? value=0x0 */
+    MVI(0x00000054, "P6_UNK_0000_0054", 0), /* Villain? value=0x0 */
+    MVX(0x0000006c, "P6_UNK_0000_006c", 0, UINT64_C(0xffffffff00000082), 0),
+    MVX(0x0000006d, "P6_UNK_0000_006d", 0, UINT64_C(0xffffffff00000082), 0),
+    MVX(0x0000006e, "P6_UNK_0000_006e", 0, UINT64_C(0xffffffff00000082), 0),
+    MVO(0x0000006f, "P6_UNK_0000_006f", 0xadb),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, Ia32BiosUpdateTrigger),
+    MVX(0x00000088, "BBL_CR_D0", 0, 0, 0), /* value=0xfcaeffff`d779fd3e */
+    MVX(0x00000089, "BBL_CR_D1", 0, 0, 0), /* value=0xefffbcb7`ff77fbef */
+    MVX(0x0000008a, "BBL_CR_D2", 0, 0, 0), /* value=0xdfff3f2f`fb367d9f */
+    MVX(0x0000008b, "BBL_CR_D3|BIOS_SIGN", UINT64_C(0x1800000000), 0, 0),
+    MVX(0x0000008c, "P6_UNK_0000_008c", 0, 0, 0), /* value=0xeffff3ff`ef39bfff */
+    MVX(0x0000008d, "P6_UNK_0000_008d", 0, 0, 0), /* value=0xf773adfb`ef3ff3fc */
+    MVX(0x0000008e, "P6_UNK_0000_008e", 0, 0, 0), /* value=0xfeb7f6ff`ebbffeff */
+    MVX(0x0000008f, "P6_UNK_0000_008f", 0, 0, 0), /* value=0xd6ffb7af`ffad9e7e */
+    MVX(0x00000090, "P6_UNK_0000_0090", 0, UINT64_C(0xfffffffffffffffa), 0), /* value=0x9ebdb4b5 */
+    MVX(0x000000ae, "P6_UNK_0000_00ae", UINT64_C(0x1000000000007efc), 0, 0),
+    MFX(0x000000c1, "IA32_PMC0", Ia32PmcN, Ia32PmcN, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x000000c2, "IA32_PMC1", Ia32PmcN, Ia32PmcN, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MVI(0x000000c7, "P6_UNK_0000_00c7", UINT64_C(0x5a000000ac000000)),
+    MFX(0x000000cd, "MSR_FSB_FREQ", IntelP6FsbFrequency, ReadOnly, 0, 0, 0),
+    MVO(0x000000ce, "P6_UNK_0000_00ce", UINT64_C(0x2812140000000000)),
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x508, 0, 0), /* value=0x508 */
+    MVX(0x00000116, "BBL_CR_ADDR", UINT32_C(0xfe7efff0), UINT64_C(0xffffffff0000000f), 0),
+    MVX(0x00000118, "BBL_CR_DECC", UINT64_C(0xc0000000c1ae9fda), UINT64_C(0xfffffff00000000), 0),
+    MFX(0x00000119, "BBL_CR_CTL", IntelBblCrCtl, IntelBblCrCtl, 0x8, UINT64_C(0xffffffffc00001ff), 0), /* value=0x8 */
+    MVI(0x0000011b, "P6_UNK_0000_011b", 0),
+    MFX(0x0000011e, "BBL_CR_CTL3", IntelBblCrCtl3, IntelBblCrCtl3, 0x34272b, UINT64_C(0xfffffffffffbfc1f), 0), /* value=0x34272b */
+    MVI(0x00000131, "P6_UNK_0000_0131", 0),
+    MVX(0x0000014e, "P6_UNK_0000_014e", 0xd31f40, UINT64_C(0xfffffffff000008f), 0),
+    MVI(0x0000014f, "P6_UNK_0000_014f", 0xd31f40),
+    MVX(0x00000150, "P6_UNK_0000_0150", 0, UINT64_C(0xffffffffdfffe07f), 0x20000000),
+    MVX(0x00000151, "P6_UNK_0000_0151", 0x3c531fc6, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0x00000154, "P6_UNK_0000_0154", 0),
+    MVX(0x0000015b, "P6_UNK_0000_015b", 0, ~(uint64_t)UINT32_MAX, 0),
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x8 */
+    MFX(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0xf78af000 */
+    MFX(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x804de6f0 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x5, 0, 0), /* value=0x5 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    RSN(0x00000186, 0x00000187, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00000194, "CLOCK_FLEX_MAX", 0, UINT64_C(0xfffffffffffee0c0), 0),
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0x612142806000612), 0, 0), /* value=0x6121428`06000612 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x612, 0, 0), /* Might bite. value=0x612 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0x2, UINT64_C(0xffffffffffffffe1), 0), /* value=0x2 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0, UINT64_C(0xfffffffffffffffc), 0), /* value=0x0 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, 0, UINT64_C(0xfffffffffffffffd), 0), /* value=0x0 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, Ia32Therm2Ctl, 0x10612, UINT64_C(0xfffffffffffee0c0), 0), /* value=0x10612 */
+    MVX(0x0000019e, "P6_UNK_0000_019e", 0, UINT64_C(0xffffffffffff0000), 0),
+    MVI(0x0000019f, "P6_UNK_0000_019f", 0),
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, 0x111088, UINT64_C(0xffffffff001ffb77), 0), /* value=0x111088 */
+    MVX(0x000001a1, "P6_UNK_0000_01a1", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x000001aa, "P6_PIC_SENS_CFG", 0x3, UINT64_C(0xfffffffffffffffc), 0),
+    MVX(0x000001ae, "P6_UNK_0000_01ae", 0, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x000001af, "P6_UNK_0000_01af", 0x3ff, UINT64_C(0xfffffffffffffc00), 0),
+    MVO(0x000001c9, "TODO_0000_01c9", 0x8000000),
+    MVX(0x000001d3, "P6_UNK_0000_01d3", 0x8000, UINT64_C(0xffffffffffff7fff), 0),
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, UINT64_C(0xffffffffffffc200), 0), /* value=0x1 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0xaad05fa1 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xaad06480 */
+    MFO(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp), /* value=0x7dba1245 */
+    MFO(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp), /* value=0x806f5d54 */
+    MVO(0x000001e0, "MSR_ROB_CR_BKUPTMPDR6", 0xff0),
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000fff)), /* value=0xf`c0000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000ff8)), /* value=0x40000006 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000fff)), /* value=0xf`e0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000ff8)), /* value=0x5ff80000 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000fff)), /* value=0xf`fff80800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000fff)), /* value=0xf`00000000 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000fff)), /* value=0xf`00000000 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000fff)), /* value=0xf`00000000 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000fff)), /* value=0xf`00000000 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, UINT64_C(0xf00000000), UINT64_C(0xfffffff000000fff)), /* value=0xf`00000000 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    RFN(0x00000400, 0x00000413, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MVX(0x00001000, "P6_DEBUG_REGISTER_0", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001001, "P6_DEBUG_REGISTER_1", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001002, "P6_DEBUG_REGISTER_2", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001003, "P6_DEBUG_REGISTER_3", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001004, "P6_DEBUG_REGISTER_4", UINT32_C(0xffff0ff0), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001005, "P6_DEBUG_REGISTER_5", 0x400, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0x00001006, "P6_DEBUG_REGISTER_6", UINT32_C(0xffff0ff0)), /* Villain? */
+    MVI(0x00001007, "P6_DEBUG_REGISTER_7", 0x400), /* Villain? */
+    MVO(0x0000103f, "P6_UNK_0000_103f", 0x4),
+    MVO(0x000010cd, "P6_UNK_0000_10cd", 0),
+    MFW(0x00002000, "P6_CR0", IntelP6CrN, IntelP6CrN, UINT64_C(0xffffffff00000010)), /* value=0x8001003b */
+    MFX(0x00002002, "P6_CR2", IntelP6CrN, IntelP6CrN, 0x2, ~(uint64_t)UINT32_MAX, 0), /* value=0xc30000 */
+    MFX(0x00002003, "P6_CR3", IntelP6CrN, IntelP6CrN, 0x3, ~(uint64_t)UINT32_MAX, 0), /* value=0x29765000 */
+    MFX(0x00002004, "P6_CR4", IntelP6CrN, IntelP6CrN, 0x4, ~(uint64_t)UINT32_MAX, 0), /* value=0x6d9 */
+    MVO(0x0000203f, "P6_UNK_0000_203f", 0x4),
+    MVO(0x000020cd, "P6_UNK_0000_20cd", 0),
+    MVO(0x0000303f, "P6_UNK_0000_303f", 0x4),
+    MVO(0x000030cd, "P6_UNK_0000_30cd", 0),
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Pentium(R) M processor 2.00GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Pentium_M_processor_2_00GHz =
+{
+    /*.pszName          = */ "Intel Pentium M processor 2.00GHz",
+    /*.pszFullName      = */ "Intel(R) Pentium(R) M processor 2.00GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 13,
+    /*.uStepping        = */ 6,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_P6_M_Dothan,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 32,
+    /*.fMxCsrMask       = */ 0xffbf, ///< @todo check this
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Pentium_M_processor_2_00GHz),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Pentium_M_processor_2_00GHz)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF,
+    /*.DefUnknownCpuId  = */ { 0x02b3b001, 0x000000f0, 0x00000000, 0x2c04307d },
+    /*.fMsrMask         = */ UINT32_C(0x3fff),
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Pentium_M_processor_2_00GHz)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Pentium_M_processor_2_00GHz),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Pentium_M_processor_2_00GHz_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Pentium_N3530_2_16GHz.h b/src/VBox/VMM/VMMR3/cpus/Intel_Pentium_N3530_2_16GHz.h
new file mode 100644
index 00000000..3a019304
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Pentium_N3530_2_16GHz.h
@@ -0,0 +1,265 @@
+/* $Id: Intel_Pentium_N3530_2_16GHz.h $ */
+/** @file
+ * CPU database entry "Intel Pentium N3530 2.16GHz".
+ * Generated at 2016-04-29T13:34:27Z by VBoxCpuReport v5.0.51r106929 on win.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Pentium_N3530_2_16GHz_h
+#define VBOX_CPUDB_Intel_Pentium_N3530_2_16GHz_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Pentium(R) CPU  N3530  @ 2.16GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Pentium_N3530_2_16GHz[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000b, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00030678, 0x02100800, 0x41d8e3bf, 0xbfebfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x61b3a001, 0x0000ffc2, 0x00000000, 0x00000000, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, UINT32_MAX, 0x1c000121, 0x0140003f, 0x0000003f, 0x00000001, 0 },
+    { 0x00000004, 0x00000001, UINT32_MAX, 0x1c000122, 0x01c0003f, 0x0000003f, 0x00000001, 0 },
+    { 0x00000004, 0x00000002, UINT32_MAX, 0x1c00c143, 0x03c0003f, 0x000003ff, 0x00000001, 0 },
+    { 0x00000004, 0x00000003, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x33000020, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000007, 0x00000002, 0x00000009, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, UINT32_MAX, 0x00000000, 0x00002282, 0x00000000, 0x00000000, 0 },
+    { 0x00000007, 0x00000001, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07280203, 0x00000000, 0x00000000, 0x00004503, 0 },
+    { 0x0000000b, 0x00000000, UINT32_MAX, 0x00000001, 0x00000001, 0x00000100, 0x00000002, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000b, 0x00000001, UINT32_MAX, 0x00000004, 0x00000004, 0x00000201, 0x00000002, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000b, 0x00000002, UINT32_MAX, 0x00000000, 0x00000000, 0x00000002, 0x00000002, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000101, 0x28100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20202020, 0x6e492020, 0x286c6574, 0x50202952, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x69746e65, 0x52286d75, 0x50432029, 0x4e202055, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x30333533, 0x20402020, 0x36312e32, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x04008040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000100, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003024, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Pentium(R) CPU  N3530  @ 2.16GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Pentium_N3530_2_16GHz[] =
+{
+    MFI(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr), /* value=0x0 */
+    MFX(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType, Ia32P5McType, 0, 0, UINT64_MAX), /* value=0x0 */
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, 0, UINT64_C(0xffffffffffff0000)), /* value=0x40 */
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x4c5e`43033c62 */
+    MFX(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0xc000090341f52), 0, 0), /* value=0xc0000`90341f52 */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0, UINT64_C(0xfffffff0000006ff)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, IntelEblCrPowerOn, 0x40080000, UINT32_C(0xfff7ffff), ~(uint64_t)UINT32_MAX), /* value=0x40080000 */
+    MVX(0x00000033, "TEST_CTL", 0, 0, UINT64_C(0xffffffff7fffffff)),
+    MFO(0x00000034, "MSR_SMI_COUNT", IntelI7SmiCount), /* value=0xa */
+    MVO(0x00000039, "C2_UNK_0000_0039", 0x2),
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x5 */
+    MVX(0x0000003b, "P6_UNK_0000_003b", UINT64_C(0x4c27f41f3066), 0x800, 0),
+    RFN(0x00000040, 0x00000047, "MSR_LASTBRANCH_n_FROM_IP", IntelLastBranchToN, IntelLastBranchToN),
+    RFN(0x00000060, 0x00000067, "MSR_LASTBRANCH_n_TO_IP", IntelLastBranchFromN, IntelLastBranchFromN),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, IgnoreWrite),
+    MFX(0x0000008b, "BBL_CR_D3|BIOS_SIGN", Ia32BiosSignId, Ia32BiosSignId, 0, 0, UINT32_MAX), /* value=0x809`00000000 */
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    RSN(0x000000c1, 0x000000c2, "IA32_PMCn", Ia32PmcN, Ia32PmcN, 0x0, ~(uint64_t)UINT32_MAX, 0),
+    MFI(0x000000c7, "IA32_PMC6", Ia32PmcN), /* value=0x36c9 */
+    MFX(0x000000cd, "MSR_FSB_FREQ", IntelP6FsbFrequency, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MVO(0x000000ce, "IA32_PLATFORM_INFO", UINT64_C(0x60000001a00)),
+    MFX(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl, IntelPkgCStConfigControl, 0, 0, UINT64_C(0xffffffffffc073f0)), /* value=0x1a000f */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, 0, UINT64_C(0xffffffffff800000)), /* value=0x20000 */
+    MVO(0x000000e5, "C2_UNK_0000_00e5", UINT32_C(0x80031838)),
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf), /* value=0x1f8f8 */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x9875 */
+    MFX(0x000000ee, "C1_EXT_CONFIG", IntelCore1ExtConfig, IntelCore1ExtConfig, 0, UINT32_C(0xefc5ffff), UINT64_C(0xffffffff10000000)), /* value=0x2380002 */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0xd08, 0, 0), /* value=0xd08 */
+    MFX(0x0000011e, "BBL_CR_CTL3", IntelBblCrCtl3, IntelBblCrCtl3, 0x7e2801ff, UINT32_C(0xfe83f8ff), UINT64_C(0xffffffff00400600)), /* value=0x7e2801ff */
+    MVX(0x00000120, "SILV_UNK_0000_0120", 0x44, 0x40, UINT64_C(0xffffffffffffff33)),
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0x0 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0x0 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x806, 0, 0), /* value=0x806 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, 0, UINT64_C(0xfffffffffffffff8)), /* value=0x0 */
+    RSN(0x00000186, 0x00000187, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, 0, ~(uint64_t)UINT32_MAX),
+    MFX(0x00000194, "CLOCK_FLEX_MAX", IntelFlexRatio, IntelFlexRatio, 0, UINT32_C(0xfffec080), ~(uint64_t)UINT32_MAX), /* value=0x0 */
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0x880000001f52), 0, 0), /* value=0x8800`00001f52 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x1f52, 0, 0), /* Might bite. value=0x1f52 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0, 0, UINT64_C(0xffffffffffffffe1)), /* value=0x0 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0xcbb700, 0, UINT64_C(0xfffffffffe0000e8)), /* value=0xcbb700 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, UINT32_C(0x88420100), UINT32_C(0xfffff555), ~(uint64_t)UINT32_MAX), /* value=0x88420100 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0x623, 0, 0), /* value=0x623 */
+    MVX(0x0000019e, "P6_UNK_0000_019e", 0, UINT32_MAX, ~(uint64_t)UINT32_MAX),
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, 0x850089, 0x1080, UINT64_C(0xffffffbbff3aef76)), /* value=0x850089 */
+    MFX(0x000001a2, "I7_MSR_TEMPERATURE_TARGET", IntelI7TemperatureTarget, IntelI7TemperatureTarget, 0x690000, 0xff0000, UINT64_C(0xffffffffc000ffff)), /* value=0x690000 */
+    MFX(0x000001a6, "I7_MSR_OFFCORE_RSP_0", IntelI7MsrOffCoreResponseN, IntelI7MsrOffCoreResponseN, 0x0, 0, UINT64_C(0xffffff897ffa0000)), /* XXX: The range ended earlier than expected! */
+    MFX(0x000001a7, "I7_MSR_OFFCORE_RSP_1", IntelI7MsrOffCoreResponseN, IntelI7MsrOffCoreResponseN, 0x0, 0, UINT64_C(0xffffffc97ffa0000)), /* value=0x0 */
+    MFX(0x000001ad, "I7_MSR_TURBO_RATIO_LIMIT", IntelI7TurboRatioLimit, IntelI7TurboRatioLimit, 0, UINT64_C(0x3f3f3f3f00000000), UINT64_C(0xc0c0c0c0c0c0c0c0)), /* value=0x0 */
+    MVX(0x000001b0, "IA32_ENERGY_PERF_BIAS", 0x6, 0, UINT64_C(0xfffffffffffffff0)),
+    MVO(0x000001c6, "I7_UNK_0000_01c6", 0x3),
+    MFX(0x000001c8, "MSR_LBR_SELECT", IntelI7LbrSelect, IntelI7LbrSelect, 0, 0x200, UINT64_C(0xfffffffffffffc00)), /* value=0x0 */
+    MFX(0x000001c9, "MSR_LASTBRANCH_TOS", IntelLastBranchTos, IntelLastBranchTos, 0, 0, UINT64_C(0xfffffffffffffff8)), /* value=0x0 */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffffa03c)), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0x0 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0x0 */
+    MFN(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp, P6LastIntFromIp), /* value=0x0 */
+    MFN(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp, P6LastIntToIp), /* value=0x0 */
+    MFO(0x000001f2, "IA32_SMRR_PHYSBASE", Ia32SmrrPhysBase), /* value=0x7a000006 */
+    MFO(0x000001f3, "IA32_SMRR_PHYSMASK", Ia32SmrrPhysMask), /* value=0xff800800 */
+    MFX(0x000001fc, "I7_MSR_POWER_CTL", IntelI7PowerCtl, IntelI7PowerCtl, 0, 0, UINT64_C(0xfffffffffffffffd)), /* value=0x0 */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xffc00005 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`ffc00800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xffb80000 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`fff80800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x6 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`80000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x7c000000 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`fc000800 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x7b000000 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`ff000800 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x7ae00000 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`ffe00800 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x1`00000006 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`80000800 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MVX(0x000002e0, "I7_SB_NO_EVICT_MODE", 0, 0, UINT64_C(0xfffffffffffffffc)),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    RSN(0x00000309, 0x0000030b, "IA32_FIXED_CTRn", Ia32FixedCtrN, Ia32FixedCtrN, 0x0, 0, UINT64_C(0xffffff0000000000)),
+    MFX(0x00000345, "IA32_PERF_CAPABILITIES", Ia32PerfCapabilities, ReadOnly, 0x32c1, 0, 0), /* value=0x32c1 */
+    MFX(0x0000038d, "IA32_FIXED_CTR_CTRL", Ia32FixedCtrCtrl, Ia32FixedCtrCtrl, 0, 0, UINT64_C(0xfffffffffffff000)), /* value=0x0 */
+    MFX(0x0000038e, "IA32_PERF_GLOBAL_STATUS", Ia32PerfGlobalStatus, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x0000038f, "IA32_PERF_GLOBAL_CTRL", Ia32PerfGlobalCtrl, Ia32PerfGlobalCtrl, 0, 0, UINT64_C(0xfffffff8fffffffc)), /* value=0x3 */
+    MFX(0x00000390, "IA32_PERF_GLOBAL_OVF_CTRL", Ia32PerfGlobalOvfCtrl, Ia32PerfGlobalOvfCtrl, 0, UINT64_C(0xc000000700000003), UINT64_C(0x3ffffff8fffffffc)), /* value=0x0 */
+    MFX(0x000003f1, "IA32_PEBS_ENABLE", Ia32PebsEnable, Ia32PebsEnable, 0, 0, UINT64_C(0xfffffffffffffffe)), /* value=0x0 */
+    MFX(0x000003f8, "I7_MSR_PKG_C3_RESIDENCY", IntelI7PkgCnResidencyN, ReadOnly, 0x3, 0, UINT64_MAX), /* value=0x0 */
+    RSN(0x000003f9, 0x000003fa, "I7_MSR_PKG_Cn_RESIDENCY", IntelI7PkgCnResidencyN, ReadOnly, 0x6, 0, UINT64_MAX),
+    MFX(0x000003fc, "I7_MSR_CORE_C3_RESIDENCY", IntelI7CoreCnResidencyN, ReadOnly, 0x3, 0, UINT64_MAX), /* value=0x80000000`0000ad5b */
+    MFX(0x000003fd, "I7_MSR_CORE_C6_RESIDENCY", IntelI7CoreCnResidencyN, ReadOnly, 0x6, 0, UINT64_MAX), /* value=0x5`51eddedc */
+    RFN(0x00000400, 0x00000417, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0x00000480, "IA32_VMX_BASIC", Ia32VmxBasic, ReadOnly, UINT64_C(0xda040000000002), 0, 0), /* value=0xda0400`00000002 */
+    MFX(0x00000481, "IA32_VMX_PINBASED_CTLS", Ia32VmxPinbasedCtls, ReadOnly, UINT64_C(0x7f00000016), 0, 0), /* value=0x7f`00000016 */
+    MFX(0x00000482, "IA32_VMX_PROCBASED_CTLS", Ia32VmxProcbasedCtls, ReadOnly, UINT64_C(0xfff9fffe0401e172), 0, 0), /* value=0xfff9fffe`0401e172 */
+    MFX(0x00000483, "IA32_VMX_EXIT_CTLS", Ia32VmxExitCtls, ReadOnly, UINT64_C(0x7fffff00036dff), 0, 0), /* value=0x7fffff`00036dff */
+    MFX(0x00000484, "IA32_VMX_ENTRY_CTLS", Ia32VmxEntryCtls, ReadOnly, UINT64_C(0xffff000011ff), 0, 0), /* value=0xffff`000011ff */
+    MFX(0x00000485, "IA32_VMX_MISC", Ia32VmxMisc, ReadOnly, 0x481e6, 0, 0), /* value=0x481e6 */
+    MFX(0x00000486, "IA32_VMX_CR0_FIXED0", Ia32VmxCr0Fixed0, ReadOnly, UINT32_C(0x80000021), 0, 0), /* value=0x80000021 */
+    MFX(0x00000487, "IA32_VMX_CR0_FIXED1", Ia32VmxCr0Fixed1, ReadOnly, UINT32_MAX, 0, 0), /* value=0xffffffff */
+    MFX(0x00000488, "IA32_VMX_CR4_FIXED0", Ia32VmxCr4Fixed0, ReadOnly, 0x2000, 0, 0), /* value=0x2000 */
+    MFX(0x00000489, "IA32_VMX_CR4_FIXED1", Ia32VmxCr4Fixed1, ReadOnly, 0x1027ff, 0, 0), /* value=0x1027ff */
+    MFX(0x0000048a, "IA32_VMX_VMCS_ENUM", Ia32VmxVmcsEnum, ReadOnly, 0x2e, 0, 0), /* value=0x2e */
+    MFX(0x0000048b, "IA32_VMX_PROCBASED_CTLS2", Ia32VmxProcBasedCtls2, ReadOnly, UINT64_C(0x28ef00000000), 0, 0), /* value=0x28ef`00000000 */
+    MFX(0x0000048c, "IA32_VMX_EPT_VPID_CAP", Ia32VmxEptVpidCap, ReadOnly, UINT64_C(0xf0106114141), 0, 0), /* value=0xf01`06114141 */
+    MFX(0x0000048d, "IA32_VMX_TRUE_PINBASED_CTLS", Ia32VmxTruePinbasedCtls, ReadOnly, UINT64_C(0x7f00000016), 0, 0), /* value=0x7f`00000016 */
+    MFX(0x0000048e, "IA32_VMX_TRUE_PROCBASED_CTLS", Ia32VmxTrueProcbasedCtls, ReadOnly, UINT64_C(0xfff9fffe04006172), 0, 0), /* value=0xfff9fffe`04006172 */
+    MFX(0x0000048f, "IA32_VMX_TRUE_EXIT_CTLS", Ia32VmxTrueExitCtls, ReadOnly, UINT64_C(0x7fffff00036dfb), 0, 0), /* value=0x7fffff`00036dfb */
+    MFX(0x00000490, "IA32_VMX_TRUE_ENTRY_CTLS", Ia32VmxTrueEntryCtls, ReadOnly, UINT64_C(0xffff000011fb), 0, 0), /* value=0xffff`000011fb */
+    MFX(0x00000491, "IA32_VMX_VMFUNC", Ia32VmxVmFunc, ReadOnly, 0x1, 0, 0), /* value=0x1 */
+    RSN(0x000004c1, 0x000004c2, "IA32_A_PMCn", Ia32PmcN, Ia32PmcN, 0x0, 0, UINT64_C(0xffffff0000000000)),
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0x0 */
+    MFX(0x00000601, "I7_SB_MSR_VR_CURRENT_CONFIG", IntelI7SandyVrCurrentConfig, IntelI7SandyVrCurrentConfig, 0, UINT64_C(0xc00000007fffe000), 0), /* value=0x0 */
+    MFX(0x00000606, "I7_SB_MSR_RAPL_POWER_UNIT", IntelI7SandyRaplPowerUnit, IntelI7SandyRaplPowerUnit, 0x505, 0, UINT64_C(0xfffffffffff0e0f0)), /* value=0x505 */
+    MFN(0x0000060d, "I7_SB_MSR_PKG_C2_RESIDENCY", IntelI7SandyPkgC2Residency, IntelI7SandyPkgC2Residency), /* value=0x0 */
+    MFX(0x00000610, "I7_SB_MSR_PKG_POWER_LIMIT", IntelI7RaplPkgPowerLimit, IntelI7RaplPkgPowerLimit, 0x3880fa, 0x8000, UINT64_C(0xff000000ff000000)), /* value=0x3880fa */
+    MFX(0x00000611, "I7_SB_MSR_PKG_ENERGY_STATUS", IntelI7RaplPkgEnergyStatus, ReadOnly, 0x21823a, 0, 0), /* value=0x21823a */
+    MFX(0x00000638, "I7_SB_MSR_PP0_POWER_LIMIT", IntelI7RaplPp0PowerLimit, IntelI7RaplPp0PowerLimit, 0x20000, 0, UINT64_C(0xffffffffff000000)), /* value=0x20000 */
+    MFX(0x00000639, "I7_SB_MSR_PP0_ENERGY_STATUS", IntelI7RaplPp0EnergyStatus, ReadOnly, 0x792fa, 0, 0), /* value=0x792fa */
+    MFO(0x00000660, "SILV_CORE_C1_RESIDENCY", IntelAtSilvCoreC1Recidency), /* value=0x22`70ff1790 */
+    MVO(0x00000661, "SILV_UNK_0000_0661", 0),
+    MVO(0x00000662, "SILV_UNK_0000_0662", 0),
+    MVO(0x00000663, "SILV_UNK_0000_0663", 0),
+    MVO(0x00000664, "SILV_UNK_0000_0664", 0),
+    MVO(0x00000665, "SILV_UNK_0000_0665", 0),
+    MVO(0x00000666, "SILV_UNK_0000_0666", 0),
+    MVO(0x00000667, "SILV_UNK_0000_0667", 0x9),
+    MVX(0x00000668, "SILV_UNK_0000_0668", 0x13130f0b, 0, ~(uint64_t)UINT32_MAX),
+    MVX(0x00000669, "SILV_UNK_0000_0669", 0x1010f20, 0, ~(uint64_t)UINT32_MAX),
+    MVO(0x0000066a, "SILV_UNK_0000_066a", 0x1a0602),
+    MVO(0x0000066b, "SILV_UNK_0000_066b", 0x442323),
+    MVO(0x0000066c, "SILV_UNK_0000_066c", 0x1f1f1f1f),
+    MVO(0x0000066d, "SILV_UNK_0000_066d", 0x52525252),
+    MVX(0x0000066e, "SILV_UNK_0000_066e", 0, 0, ~(uint64_t)UINT32_MAX),
+    MVX(0x0000066f, "SILV_UNK_0000_066f", 0, 0, ~(uint64_t)UINT32_MAX),
+    MVX(0x00000670, "SILV_UNK_0000_0670", 0, 0, ~(uint64_t)UINT32_MAX),
+    MVX(0x00000671, "SILV_UNK_0000_0671", 0, 0, UINT64_C(0xffffffff000000c0)),
+    MVX(0x00000672, "SILV_UNK_0000_0672", 0, 0, UINT64_C(0xffffffffc0000000)),
+    MVX(0x00000673, "SILV_UNK_0000_0673", 0x205, 0, UINT64_C(0xffffffffffffc000)),
+    MVX(0x00000674, "SILV_UNK_0000_0674", 0x4050006, 0, UINT64_C(0xfffffffff8000000)),
+    MVX(0x00000675, "SILV_UNK_0000_0675", 0x27, 0x20, UINT64_C(0xffffffffffffffc0)),
+    MVX(0x00000676, "SILV_UNK_0000_0676", 0, UINT64_C(0x7f7f7f7f00000000), UINT64_C(0x8080808080808080)),
+    MVX(0x00000677, "SILV_UNK_0000_0677", 0, 0, ~(uint64_t)UINT32_MAX),
+    MFI(0x000006e0, "IA32_TSC_DEADLINE", Ia32TscDeadline), /* value=0x0 */
+    MVX(0x00000768, "SILV_UNK_0000_0768", 0, 0, UINT64_C(0xffffffffffff0060)),
+    MVX(0x00000769, "SILV_UNK_0000_0769", 0, 0x6, UINT64_C(0xfffffffffffffff0)),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xfffffffffffff2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xfffff802`6e9de200 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xfffff802`6e9ddf40 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x4700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x9b440000 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffffd000`20661000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x7ff7`9b43e000 */
+    MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, 0, ~(uint64_t)UINT32_MAX), /* value=0x0 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Pentium(R) CPU  N3530  @ 2.16GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Pentium_N3530_2_16GHz =
+{
+    /*.pszName          = */ "Intel Pentium N3530 2.16GHz",
+    /*.pszFullName      = */ "Intel(R) Pentium(R) CPU  N3530  @ 2.16GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 55,
+    /*.uStepping        = */ 8,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_Atom_Silvermont,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_267MHZ,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 36,
+    /*.fMxCsrMask       = */ 0xffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Pentium_N3530_2_16GHz),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Pentium_N3530_2_16GHz)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX,
+    /*.DefUnknownCpuId  = */ { 0x00000001, 0x00000001, 0x00000100, 0x00000004 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Pentium_N3530_2_16GHz)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Pentium_N3530_2_16GHz),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Pentium_N3530_2_16GHz_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Intel_Xeon_X5482_3_20GHz.h b/src/VBox/VMM/VMMR3/cpus/Intel_Xeon_X5482_3_20GHz.h
new file mode 100644
index 00000000..a42bee8c
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Intel_Xeon_X5482_3_20GHz.h
@@ -0,0 +1,248 @@
+/* $Id: Intel_Xeon_X5482_3_20GHz.h $ */
+/** @file
+ * CPU database entry "Intel Xeon X5482 3.20GHz".
+ * Generated at 2013-12-16T12:10:52Z by VBoxCpuReport v4.3.53r91299 on darwin.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Intel_Xeon_X5482_3_20GHz_h
+#define VBOX_CPUDB_Intel_Xeon_X5482_3_20GHz_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Intel(R) Xeon(R) CPU           X5482  @ 3.20GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Intel_Xeon_X5482_3_20GHz[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000a, 0x756e6547, 0x6c65746e, 0x49656e69, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00010676, 0x04040800, 0x000ce3bd, 0xbfebfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x05b0b101, 0x005657f0, 0x00000000, 0x2cb4304e, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, UINT32_MAX, 0x0c000121, 0x01c0003f, 0x0000003f, 0x00000001, 0 },
+    { 0x00000004, 0x00000001, UINT32_MAX, 0x0c000122, 0x01c0003f, 0x0000003f, 0x00000001, 0 },
+    { 0x00000004, 0x00000002, UINT32_MAX, 0x0c004143, 0x05c0003f, 0x00000fff, 0x00000001, 0 },
+    { 0x00000004, 0x00000003, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00002220, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000001, 0x00000002, 0x00000001, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000400, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07280202, 0x00000000, 0x00000000, 0x00000503, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x20100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x65746e49, 0x2952286c, 0x6f655820, 0x2952286e, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x55504320, 0x20202020, 0x20202020, 0x58202020, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x32383435, 0x20402020, 0x30322e33, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x18008040, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003026, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Intel(R) Xeon(R) CPU           X5482  @ 3.20GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Intel_Xeon_X5482_3_20GHz[] =
+{
+    MFO(0x00000000, "IA32_P5_MC_ADDR", Ia32P5McAddr), /* value=0x610010 */
+    MFX(0x00000001, "IA32_P5_MC_TYPE", Ia32P5McType, Ia32P5McType, 0, 0, UINT64_MAX), /* value=0x0 */
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, 0, UINT64_C(0xffffffffffff0000)), /* value=0x40 */
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x1358`d28c2c60 */
+    MFV(0x00000017, "IA32_PLATFORM_ID", Ia32PlatformId, ReadOnly, UINT64_C(0x18000088e40822)),
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0, UINT64_C(0xffffffc0000006ff)),
+    MVX(0x00000021, "C2_UNK_0000_0021", 0, 0, UINT64_C(0xffffffffffffffc0)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, IntelEblCrPowerOn, UINT32_C(0xc2383400), UINT64_C(0xffffffffdff7df00), 0), /* value=0xc2383400 */
+    MVX(0x00000032, "P6_UNK_0000_0032", 0, UINT64_C(0xffffffff01fe0000), 0),
+    MVX(0x00000033, "TEST_CTL", 0, UINT64_C(0xffffffff7fffffff), 0),
+    MVO(0x00000039, "C2_UNK_0000_0039", 0x7),
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x5 */
+    MVO(0x0000003f, "P6_UNK_0000_003f", 0),
+    RFN(0x00000040, 0x00000043, "MSR_LASTBRANCH_n_FROM_IP", IntelLastBranchToN, IntelLastBranchToN),
+    RFN(0x00000060, 0x00000063, "MSR_LASTBRANCH_n_TO_IP", IntelLastBranchFromN, IntelLastBranchFromN),
+    MFN(0x00000079, "IA32_BIOS_UPDT_TRIG", WriteOnly, IgnoreWrite),
+    MVX(0x0000008b, "BBL_CR_D3|BIOS_SIGN", UINT64_C(0x60b00000000), UINT32_MAX, 0),
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    MFX(0x000000a8, "C2_EMTTM_CR_TABLES_0", IntelCore2EmttmCrTablesN, IntelCore2EmttmCrTablesN, 0x612, UINT64_C(0xffffffffffff8000), 0), /* value=0x612 */
+    MFX(0x000000a9, "C2_EMTTM_CR_TABLES_1", IntelCore2EmttmCrTablesN, IntelCore2EmttmCrTablesN, 0x612, UINT64_C(0xffffffffffff8000), 0), /* value=0x612 */
+    MFX(0x000000aa, "C2_EMTTM_CR_TABLES_2", IntelCore2EmttmCrTablesN, IntelCore2EmttmCrTablesN, 0x612, UINT64_C(0xffffffffffff8000), 0), /* value=0x612 */
+    MFX(0x000000ab, "C2_EMTTM_CR_TABLES_3", IntelCore2EmttmCrTablesN, IntelCore2EmttmCrTablesN, 0x612, UINT64_C(0xffffffffffff8000), 0), /* value=0x612 */
+    MFX(0x000000ac, "C2_EMTTM_CR_TABLES_4", IntelCore2EmttmCrTablesN, IntelCore2EmttmCrTablesN, 0x612, UINT64_C(0xffffffffffff8000), 0), /* value=0x612 */
+    MFX(0x000000ad, "C2_EMTTM_CR_TABLES_5", IntelCore2EmttmCrTablesN, IntelCore2EmttmCrTablesN, 0x612, ~(uint64_t)UINT32_MAX, 0), /* value=0x612 */
+    RSN(0x000000c1, 0x000000c2, "IA32_PMCn", Ia32PmcN, Ia32PmcN, 0x0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0x000000c7, "P6_UNK_0000_00c7", UINT64_C(0x2300000052000000)),
+    MFX(0x000000cd, "P6_MSR_FSB_FREQ", IntelP6FsbFrequency, ReadOnly, 0x806, 0, 0),
+    MVO(0x000000ce, "P6_UNK_0000_00ce", UINT64_C(0x1208227f7f0710)),
+    MVO(0x000000cf, "C2_UNK_0000_00cf", 0),
+    MVO(0x000000e0, "C2_UNK_0000_00e0", 0x18820f0),
+    MVO(0x000000e1, "C2_UNK_0000_00e1", UINT32_C(0xf0f00000)),
+    MFX(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl, IntelPkgCStConfigControl, 0, 0x404000, UINT64_C(0xfffffffffc001000)), /* value=0x202a01 */
+    MFX(0x000000e3, "C2_SMM_CST_MISC_INFO", IntelCore2SmmCStMiscInfo, IntelCore2SmmCStMiscInfo, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, 0, UINT64_C(0xffffffffff800000)), /* value=0x0 */
+    MVO(0x000000e5, "C2_UNK_0000_00e5", UINT32_C(0xd00208c8)),
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf), /* value=0x40`a0a41c60 */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x3a`cc470b98 */
+    MFX(0x000000ee, "C1_EXT_CONFIG", IntelCore1ExtConfig, IntelCore1ExtConfig, 0, UINT64_C(0xffffffffefc5ffff), 0), /* value=0x4000000`877d4b01 */
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0xd08, 0, 0), /* value=0xd08 */
+    MVX(0x00000116, "BBL_CR_ADDR", 0x3fc0, UINT64_C(0xffffff000000001f), 0),
+    MVX(0x00000118, "BBL_CR_DECC", 0xa7f99, UINT64_C(0xfffffffffff00000), 0),
+    MFN(0x0000011a, "BBL_CR_TRIG", WriteOnly, IgnoreWrite),
+    MVI(0x0000011b, "P6_UNK_0000_011b", 0),
+    MVX(0x0000011c, "C2_UNK_0000_011c", UINT32_C(0xe003b94d), UINT64_C(0xffffffff07c00000), 0),
+    MFX(0x0000011e, "BBL_CR_CTL3", IntelBblCrCtl3, IntelBblCrCtl3, UINT32_C(0xbe702111), UINT64_C(0xfffffffffef3fe9f), 0), /* value=0xbe702111 */
+    MVX(0x0000014e, "P6_UNK_0000_014e", 0x70375245, UINT64_C(0xffffffff00000080), 0),
+    MVI(0x0000014f, "P6_UNK_0000_014f", UINT32_C(0xffffba7f)),
+    MVX(0x00000151, "P6_UNK_0000_0151", 0x6b929082, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x0000015e, "C2_UNK_0000_015e", 0x6, 0, UINT64_C(0xfffffffffffffff0)),
+    MFX(0x0000015f, "C1_DTS_CAL_CTRL", IntelCore1DtsCalControl, IntelCore1DtsCalControl, 0, UINT64_C(0xffffffffffc0ffff), 0), /* value=0x822 */
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0xb */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0xffffff82`0dce9190 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0xffffff80`0d2ce720 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x806, 0, 0), /* value=0x806 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    RSN(0x00000186, 0x00000187, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, 0, UINT64_C(0xffffffff00200000)),
+    MVO(0x00000193, "C2_UNK_0000_0193", 0),
+    MVX(0x00000194, "CLOCK_FLEX_MAX", 0x14822, UINT64_C(0xfffffffffffea0c0), 0),
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, ReadOnly, UINT64_C(0x822082206300622), 0, 0), /* value=0x8220822`06300622 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x822, 0, 0), /* Might bite. value=0x822 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0x2, 0, UINT64_C(0xffffffffffffffe1)), /* value=0x2 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0x10, 0, UINT64_C(0xffffffffff0000e0)), /* value=0x10 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, UINT32_C(0x883c0000), UINT32_C(0xf87f017f), UINT64_C(0xffffffff0780fc00)), /* value=0x883c0000 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0x612, 0, 0), /* value=0x612 */
+    MVX(0x0000019e, "P6_UNK_0000_019e", 0x2120000, UINT64_C(0xffffffffffff0000), 0),
+    MVI(0x0000019f, "P6_UNK_0000_019f", 0),
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, UINT64_C(0x4066a52489), UINT64_C(0x52600099f6), UINT64_C(0xffffff0019004000)), /* value=0x40`66a52489 */
+    MVX(0x000001a1, "P6_UNK_0000_01a1", 0, UINT64_C(0xffff000000000000), 0),
+    MFX(0x000001a2, "I7_MSR_TEMPERATURE_TARGET", IntelI7TemperatureTarget, ReadOnly, 0x1400, 0, 0), /* value=0x1400 */
+    MVX(0x000001aa, "P6_PIC_SENS_CFG", UINT32_C(0xfe7f042f), UINT64_C(0xffffffff7faf00af), 0),
+    MVX(0x000001bf, "C2_UNK_0000_01bf", 0x404, UINT64_C(0xffffffffffff0000), 0),
+    MFX(0x000001c9, "MSR_LASTBRANCH_TOS", IntelLastBranchTos, IntelLastBranchTos, 0, UINT64_C(0xfffffffffffffffe), 0), /* value=0x0 */
+    MVX(0x000001d3, "P6_UNK_0000_01d3", 0x8000, UINT64_C(0xffffffffffff7fff), 0),
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffffa03c)), /* value=0x1 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0xffffff7f`8f47ca6b */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xffffff80`0d2b24c0 */
+    MFN(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp, P6LastIntFromIp), /* value=0xffffff80`0d2ba20f */
+    MFN(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp, P6LastIntToIp), /* value=0xffffff80`0d2ba200 */
+    MVO(0x000001e0, "MSR_ROB_CR_BKUPTMPDR6", 0xff0),
+    MFX(0x000001f8, "IA32_PLATFORM_DCA_CAP", Ia32PlatformDcaCap, Ia32PlatformDcaCap, 0, UINT64_C(0xfffffffffffffffe), 0), /* value=0x0 */
+    MFO(0x000001f9, "IA32_CPU_DCA_CAP", Ia32CpuDcaCap), /* value=0x1 */
+    MFX(0x000001fa, "IA32_DCA_0_CAP", Ia32Dca0Cap, Ia32Dca0Cap, 0, UINT64_C(0xfffffffffefe17ff), 0), /* value=0xc01e489 */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xffffffc000000ff8)), /* value=0x80000000 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xffffffc0000007ff)), /* value=0x3f`80000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xffffffc000000ff8)), /* value=0x7fc00000 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xffffffc0000007ff)), /* value=0x3f`ffc00800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xffffffc000000ff8)), /* value=0x6 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xffffffc0000007ff)), /* value=0x30`00000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xffffffc000000ff8)), /* value=0x0 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xffffffc0000007ff)), /* value=0x0 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xffffffc000000ff8)), /* value=0x0 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xffffffc0000007ff)), /* value=0x0 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xffffffc000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xffffffc0000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xffffffc000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xffffffc0000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xffffffc000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xffffffc0000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    RSN(0x00000309, 0x0000030b, "IA32_FIXED_CTRn", Ia32FixedCtrN, Ia32FixedCtrN, 0x0, 0, UINT64_C(0xffffff0000000000)),
+    MFX(0x00000345, "IA32_PERF_CAPABILITIES", Ia32PerfCapabilities, ReadOnly, 0x10c2, 0, 0), /* value=0x10c2 */
+    MFX(0x0000038d, "IA32_FIXED_CTR_CTRL", Ia32FixedCtrCtrl, Ia32FixedCtrCtrl, 0, 0, UINT64_C(0xfffffffffffff444)), /* value=0x0 */
+    MFX(0x0000038e, "IA32_PERF_GLOBAL_STATUS", Ia32PerfGlobalStatus, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFN(0x0000038f, "IA32_PERF_GLOBAL_CTRL", Ia32PerfGlobalCtrl, Ia32PerfGlobalCtrl), /* value=0xffffffff`ffffffff */
+    MFO(0x00000390, "IA32_PERF_GLOBAL_OVF_CTRL", Ia32PerfGlobalOvfCtrl), /* value=0xffffffff`ffffffff */
+    MFX(0x000003f1, "IA32_PEBS_ENABLE", Ia32PebsEnable, Ia32PebsEnable, 0, UINT64_C(0xfffffffffffffffe), 0), /* value=0x0 */
+    RFN(0x00000400, 0x00000417, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFN(0x00000478, "CPUID1_FEATURE_MASK", IntelCpuId1FeatureMaskEcdx, IntelCpuId1FeatureMaskEcdx), /* value=0xffffffff`ffffffff */
+    MFX(0x00000480, "IA32_VMX_BASIC", Ia32VmxBasic, ReadOnly, UINT64_C(0x5a08000000000d), 0, 0), /* value=0x5a0800`0000000d */
+    MFX(0x00000481, "IA32_VMX_PINBASED_CTLS", Ia32VmxPinbasedCtls, ReadOnly, UINT64_C(0x3f00000016), 0, 0), /* value=0x3f`00000016 */
+    MFX(0x00000482, "IA32_VMX_PROCBASED_CTLS", Ia32VmxProcbasedCtls, ReadOnly, UINT64_C(0xf7f9fffe0401e172), 0, 0), /* value=0xf7f9fffe`0401e172 */
+    MFX(0x00000483, "IA32_VMX_EXIT_CTLS", Ia32VmxExitCtls, ReadOnly, UINT64_C(0x3ffff00036dff), 0, 0), /* value=0x3ffff`00036dff */
+    MFX(0x00000484, "IA32_VMX_ENTRY_CTLS", Ia32VmxEntryCtls, ReadOnly, UINT64_C(0x3fff000011ff), 0, 0), /* value=0x3fff`000011ff */
+    MFX(0x00000485, "IA32_VMX_MISC", Ia32VmxMisc, ReadOnly, 0x403c0, 0, 0), /* value=0x403c0 */
+    MFX(0x00000486, "IA32_VMX_CR0_FIXED0", Ia32VmxCr0Fixed0, ReadOnly, UINT32_C(0x80000021), 0, 0), /* value=0x80000021 */
+    MFX(0x00000487, "IA32_VMX_CR0_FIXED1", Ia32VmxCr0Fixed1, ReadOnly, UINT32_MAX, 0, 0), /* value=0xffffffff */
+    MFX(0x00000488, "IA32_VMX_CR4_FIXED0", Ia32VmxCr4Fixed0, ReadOnly, 0x2000, 0, 0), /* value=0x2000 */
+    MFX(0x00000489, "IA32_VMX_CR4_FIXED1", Ia32VmxCr4Fixed1, ReadOnly, 0x27ff, 0, 0), /* value=0x27ff */
+    MFX(0x0000048a, "IA32_VMX_VMCS_ENUM", Ia32VmxVmcsEnum, ReadOnly, 0x2c, 0, 0), /* value=0x2c */
+    MFX(0x0000048b, "IA32_VMX_PROCBASED_CTLS2", Ia32VmxProcBasedCtls2, ReadOnly, UINT64_C(0x4100000000), 0, 0), /* value=0x41`00000000 */
+    MVX(0x000004f8, "C2_UNK_0000_04f8", 0, 0, 0),
+    MVX(0x000004f9, "C2_UNK_0000_04f9", 0, 0, 0),
+    MVX(0x000004fa, "C2_UNK_0000_04fa", 0, 0, 0),
+    MVX(0x000004fb, "C2_UNK_0000_04fb", 0, 0, 0),
+    MVX(0x000004fc, "C2_UNK_0000_04fc", 0, 0, 0),
+    MVX(0x000004fd, "C2_UNK_0000_04fd", 0, 0, 0),
+    MVX(0x000004fe, "C2_UNK_0000_04fe", 0, 0, 0),
+    MVX(0x000004ff, "C2_UNK_0000_04ff", 0, 0, 0),
+    MVX(0x00000590, "C2_UNK_0000_0590", 0, 0, 0),
+    MVX(0x00000591, "C2_UNK_0000_0591", 0, ~(uint64_t)UINT32_MAX, 0),
+    MFX(0x000005a0, "C2_PECI_CTL", IntelCore2PeciControl, IntelCore2PeciControl, 0, UINT64_C(0xfffffffffffffffe), 0), /* value=0x1 */
+    MVI(0x000005a1, "C2_UNK_0000_05a1", 0x1),
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0x0 */
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xfffffffffffff2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x1b0008`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xffffff80`0d2ce6c0 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0x0 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x4700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x0 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffffff82`0dcfd000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x7fff`7c7511e0 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Intel(R) Xeon(R) CPU           X5482  @ 3.20GHz.
+ */
+static CPUMDBENTRY const g_Entry_Intel_Xeon_X5482_3_20GHz =
+{
+    /*.pszName          = */ "Intel Xeon X5482 3.20GHz",
+    /*.pszFullName      = */ "Intel(R) Xeon(R) CPU           X5482  @ 3.20GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_INTEL,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 23,
+    /*.uStepping        = */ 6,
+    /*.enmMicroarch     = */ kCpumMicroarch_Intel_Core2_Penryn,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_400MHZ,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 38,
+    /*.fMxCsrMask       = */ 0xffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Intel_Xeon_X5482_3_20GHz),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Intel_Xeon_X5482_3_20GHz)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_LAST_STD_LEAF,
+    /*.DefUnknownCpuId  = */ { 0x07280202, 0x00000000, 0x00000000, 0x00000503 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Intel_Xeon_X5482_3_20GHz)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Intel_Xeon_X5482_3_20GHz),
+};
+
+#endif /* !VBOX_CPUDB_Intel_Xeon_X5482_3_20GHz_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/Makefile.kup b/src/VBox/VMM/VMMR3/cpus/Makefile.kup
new file mode 100644
index 00000000..e69de29b
diff --git a/src/VBox/VMM/VMMR3/cpus/Quad_Core_AMD_Opteron_2384.h b/src/VBox/VMM/VMMR3/cpus/Quad_Core_AMD_Opteron_2384.h
new file mode 100644
index 00000000..b3fac766
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/Quad_Core_AMD_Opteron_2384.h
@@ -0,0 +1,270 @@
+/* $Id: Quad_Core_AMD_Opteron_2384.h $ */
+/** @file
+ * CPU database entry "Quad-Core AMD Opteron 2384".
+ * Generated at 2013-12-09T21:56:56Z by VBoxCpuReport v4.3.51r91133 on win.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_Quad_Core_AMD_Opteron_2384_h
+#define VBOX_CPUDB_Quad_Core_AMD_Opteron_2384_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for Quad-Core AMD Opteron(tm) Processor 2384.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_Quad_Core_AMD_Opteron_2384[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000005, 0x68747541, 0x444d4163, 0x69746e65, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x00100f42, 0x06040800, 0x00802009, 0x178bfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x8000001b, 0x68747541, 0x444d4163, 0x69746e65, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00100f42, 0x00000d4f, 0x000037ff, 0xefd3fbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x80000002, 0x00000000, 0x00000000, 0x64617551, 0x726f432d, 0x4d412065, 0x704f2044, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x6f726574, 0x6d74286e, 0x72502029, 0x7365636f, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x20726f73, 0x34383332, 0x00000000, 0x00000000, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0xff30ff10, 0xff30ff20, 0x40020140, 0x40020140, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x20800000, 0x42004200, 0x02008140, 0x0030b140, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x000001f9, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003030, 0x00000000, 0x00002003, 0x00000000, 0 },
+    { 0x80000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000a, 0x00000000, 0x00000000, 0x00000001, 0x00000040, 0x00000000, 0x0000000f, 0 },
+    { 0x8000000b, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000d, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000e, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000000f, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000010, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000011, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000012, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000013, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000014, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000015, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000016, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000017, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000018, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000019, 0x00000000, 0x00000000, 0xf0300000, 0x60100000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001a, 0x00000000, 0x00000000, 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x8000001b, 0x00000000, 0x00000000, 0x0000001f, 0x00000000, 0x00000000, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for Quad-Core AMD Opteron(tm) Processor 2384.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_Quad_Core_AMD_Opteron_2384[] =
+{
+    MAL(0x00000000, "IA32_P5_MC_ADDR", 0x00000402),
+    MAL(0x00000001, "IA32_P5_MC_TYPE", 0x00000401),
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0xbe`410ca9b6 */
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0, UINT64_C(0xffff0000000006ff)),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MVO(0x0000008b, "BBL_CR_D3|BIOS_SIGN", 0x1000086),
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0x508, 0, 0), /* value=0x508 */
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0x106, 0, 0), /* value=0x106 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, UINT64_C(0xfffffffffffffff8), 0), /* value=0x0 */
+    MFX(0x0000017b, "IA32_MCG_CTL", Ia32McgCtl, Ia32McgCtl, 0, UINT64_C(0xffffffffffffffc0), 0), /* value=0x3f */
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, UINT64_C(0xffffffffffffff80), 0x40), /* value=0x0 */
+    MFO(0x000001db, "P6_LAST_BRANCH_FROM_IP", P6LastBranchFromIp), /* value=0xfffff800`0245dd94 */
+    MFO(0x000001dc, "P6_LAST_BRANCH_TO_IP", P6LastBranchToIp), /* value=0xfffff800`0245e910 */
+    MFO(0x000001dd, "P6_LAST_INT_FROM_IP", P6LastIntFromIp), /* value=0x753d3416 */
+    MFO(0x000001de, "P6_LAST_INT_TO_IP", P6LastIntToIp), /* value=0x753ea130 */
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xffff000000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`80000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xffff000000000ff8)), /* value=0x80000006 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`c0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xffff000000000ff8)), /* value=0xc0000006 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xffff0000000007ff)), /* value=0xffff`f8000800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xffff000000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    RFN(0x00000400, 0x00000417, "IA32_MCi_CTL_STATUS_ADDR_MISC", Ia32McCtlStatusAddrMiscN, Ia32McCtlStatusAddrMiscN),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0x4d01, 0xfe, UINT64_C(0xffffffffffff8200)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xfffff800`0245dd00 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xfffff800`0245da80 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x14700 */
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0xfffe0000 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xfffffa60`01b8a000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x7ff`fffde000 */
+    MFX(0xc0000103, "AMD64_TSC_AUX", Amd64TscAux, Amd64TscAux, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    RSN(0xc0000408, 0xc000040a, "AMD_10H_MC4_MISCn", AmdFam10hMc4MiscN, AmdFam10hMc4MiscN, 0, UINT64_C(0xff00f000ffffffff), 0),
+    RVI(0xc000040b, 0xc000040f, "AMD_10H_MC4_MISCn", 0),
+    RSN(0xc0010000, 0xc0010003, "AMD_K8_PERF_CTL_n", AmdK8PerfCtlN, AmdK8PerfCtlN, 0x0, UINT64_C(0xfffffcf000200000), 0),
+    RSN(0xc0010004, 0xc0010007, "AMD_K8_PERF_CTR_n", AmdK8PerfCtrN, AmdK8PerfCtrN, 0x0, UINT64_C(0xffff000000000000), 0),
+    MFX(0xc0010010, "AMD_K8_SYS_CFG", AmdK8SysCfg, AmdK8SysCfg, 0x760600, UINT64_C(0xffffffffff80f8ff), 0), /* value=0x760600 */
+    MFX(0xc0010015, "AMD_K8_HW_CFG", AmdK8HwCr, AmdK8HwCr, 0x1000030, UINT64_C(0xffffffff00000020), 0), /* value=0x1000030 */
+    MFW(0xc0010016, "AMD_K8_IORR_BASE_0", AmdK8IorrBaseN, AmdK8IorrBaseN, UINT64_C(0xffff000000000fe7)), /* value=0x1`b8210000 */
+    MFW(0xc0010017, "AMD_K8_IORR_MASK_0", AmdK8IorrMaskN, AmdK8IorrMaskN, UINT64_C(0xffff0000000007ff)), /* value=0x0 */
+    MFX(0xc0010018, "AMD_K8_IORR_BASE_1", AmdK8IorrBaseN, AmdK8IorrBaseN, 0x1, UINT64_C(0xffff000000000fe7), 0), /* value=0x0 */
+    MFX(0xc0010019, "AMD_K8_IORR_MASK_1", AmdK8IorrMaskN, AmdK8IorrMaskN, 0x1, UINT64_C(0xffff0000000007ff), 0), /* value=0x0 */
+    MFW(0xc001001a, "AMD_K8_TOP_MEM", AmdK8TopOfMemN, AmdK8TopOfMemN, UINT64_C(0xffff0000007fffff)), /* value=0xc8000000 */
+    MFX(0xc001001d, "AMD_K8_TOP_MEM2", AmdK8TopOfMemN, AmdK8TopOfMemN, 0x1, UINT64_C(0xffff0000007fffff), 0), /* value=0x2`38000000 */
+    MFN(0xc001001f, "AMD_K8_NB_CFG1", AmdK8NbCfg1, AmdK8NbCfg1), /* value=0x400000`00000008 */
+    MFN(0xc0010020, "AMD_K8_PATCH_LOADER", WriteOnly, AmdK8PatchLoader),
+    MFX(0xc0010022, "AMD_K8_MC_XCPT_REDIR", AmdK8McXcptRedir, AmdK8McXcptRedir, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    RFN(0xc0010030, 0xc0010035, "AMD_K8_CPU_NAME_n", AmdK8CpuNameN, AmdK8CpuNameN),
+    MFX(0xc001003e, "AMD_K8_HTC", AmdK8HwThermalCtrl, AmdK8HwThermalCtrl, 0x327f0004, UINT64_C(0xffffffffc0008838), 0), /* value=0x327f0004 */
+    MFX(0xc001003f, "AMD_K8_STC", AmdK8SwThermalCtrl, AmdK8SwThermalCtrl, 0, UINT64_C(0xffffffffc00088c0), 0), /* value=0x30000000 */
+    MVO(0xc0010043, "AMD_K8_THERMTRIP_STATUS", 0x1830),
+    MFX(0xc0010044, "AMD_K8_MC_CTL_MASK_0", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x0, UINT64_C(0xffffffffffffff00), 0), /* value=0x80 */
+    MFX(0xc0010045, "AMD_K8_MC_CTL_MASK_1", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x1, ~(uint64_t)UINT32_MAX, 0), /* value=0x80 */
+    MFX(0xc0010046, "AMD_K8_MC_CTL_MASK_2", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x2, UINT64_C(0xfffffffffffff000), 0), /* value=0x200 */
+    MFX(0xc0010047, "AMD_K8_MC_CTL_MASK_3", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x3, UINT64_C(0xfffffffffffffffc), 0), /* value=0x0 */
+    MFX(0xc0010048, "AMD_K8_MC_CTL_MASK_4", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x4, UINT64_C(0xffffffffc0000000), 0), /* value=0x780400 */
+    MFX(0xc0010049, "AMD_K8_MC_CTL_MASK_5", AmdK8McCtlMaskN, AmdK8McCtlMaskN, 0x5, UINT64_C(0xfffffffffffffffe), 0), /* value=0x0 */
+    RFN(0xc0010050, 0xc0010053, "AMD_K8_SMI_ON_IO_TRAP_n", AmdK8SmiOnIoTrapN, AmdK8SmiOnIoTrapN),
+    MFX(0xc0010054, "AMD_K8_SMI_ON_IO_TRAP_CTL_STS", AmdK8SmiOnIoTrapCtlSts, AmdK8SmiOnIoTrapCtlSts, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0010055, "AMD_K8_INT_PENDING_MSG", AmdK8IntPendingMessage, AmdK8IntPendingMessage, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0010056, "AMD_K8_SMI_TRIGGER_IO_CYCLE", AmdK8SmiTriggerIoCycle, AmdK8SmiTriggerIoCycle, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x200242e */
+    MVX(0xc0010057, "AMD_10H_UNK_c001_0057", 0, 0, 0),
+    MFX(0xc0010058, "AMD_10H_MMIO_CFG_BASE_ADDR", AmdFam10hMmioCfgBaseAddr, AmdFam10hMmioCfgBaseAddr, 0, UINT64_C(0xffff0000000fffc0), 0), /* value=0xe0000021 */
+    MFX(0xc0010059, "AMD_10H_TRAP_CTL?", AmdFam10hTrapCtlMaybe, AmdFam10hTrapCtlMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MVX(0xc001005a, "AMD_10H_UNK_c001_005a", 0, 0, 0),
+    MVX(0xc001005b, "AMD_10H_UNK_c001_005b", 0, 0, 0),
+    MVX(0xc001005c, "AMD_10H_UNK_c001_005c", 0, 0, 0),
+    MVX(0xc001005d, "AMD_10H_UNK_c001_005d", 0, 0, 0),
+    MVO(0xc0010060, "AMD_K8_BIST_RESULT", 0),
+    MFX(0xc0010061, "AMD_10H_P_ST_CUR_LIM", AmdFam10hPStateCurLimit, ReadOnly, 0x30, 0, 0), /* value=0x30 */
+    MFX(0xc0010062, "AMD_10H_P_ST_CTL", AmdFam10hPStateControl, AmdFam10hPStateControl, 0x1, 0, UINT64_C(0xfffffffffffffff8)), /* value=0x1 */
+    MFX(0xc0010063, "AMD_10H_P_ST_STS", AmdFam10hPStateStatus, ReadOnly, 0x1, 0, 0), /* value=0x1 */
+    MFX(0xc0010064, "AMD_10H_P_ST_0", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x800001e13000300b), 0, 0), /* value=0x800001e1`3000300b */
+    MFX(0xc0010065, "AMD_10H_P_ST_1", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x800001c840004004), 0, 0), /* value=0x800001c8`40004004 */
+    MFX(0xc0010066, "AMD_10H_P_ST_2", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x800001b64000404e), 0, 0), /* value=0x800001b6`4000404e */
+    MFX(0xc0010067, "AMD_10H_P_ST_3", AmdFam10hPStateN, AmdFam10hPStateN, UINT64_C(0x8000019d40004040), 0, 0), /* value=0x8000019d`40004040 */
+    MFX(0xc0010068, "AMD_10H_P_ST_4", AmdFam10hPStateN, AmdFam10hPStateN, 0, 0, 0), /* value=0x0 */
+    MFX(0xc0010070, "AMD_10H_COFVID_CTL", AmdFam10hCofVidControl, AmdFam10hCofVidControl, 0x40014004, UINT64_C(0xffffffff01b80000), 0), /* value=0x40014004 */
+    MFX(0xc0010071, "AMD_10H_COFVID_STS", AmdFam10hCofVidStatus, AmdFam10hCofVidStatus, UINT64_C(0x38b600c340014004), UINT64_MAX, 0), /* value=0x38b600c3`40014004 */
+    MFX(0xc0010074, "AMD_10H_CPU_WD_TMR_CFG", AmdFam10hCpuWatchdogTimer, AmdFam10hCpuWatchdogTimer, 0, UINT64_C(0xffffffffffffff80), 0), /* value=0x0 */
+    MFX(0xc0010111, "AMD_K8_SMM_BASE", AmdK8SmmBase, AmdK8SmmBase, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x98e00 */
+    MFX(0xc0010112, "AMD_K8_SMM_ADDR", AmdK8SmmAddr, AmdK8SmmAddr, 0, UINT64_C(0xffff00000001ffff), 0), /* value=0x0 */
+    MFX(0xc0010113, "AMD_K8_SMM_MASK", AmdK8SmmMask, AmdK8SmmMask, 0, UINT64_C(0xffff0000000188c0), 0), /* value=0x1 */
+    MFX(0xc0010114, "AMD_K8_VM_CR", AmdK8VmCr, AmdK8VmCr, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xffffffe0)), /* value=0x8 */
+    MFX(0xc0010115, "AMD_K8_IGNNE", AmdK8IgnNe, AmdK8IgnNe, 0, ~(uint64_t)UINT32_MAX, UINT32_C(0xfffffffe)), /* value=0x0 */
+    MFX(0xc0010117, "AMD_K8_VM_HSAVE_PA", AmdK8VmHSavePa, AmdK8VmHSavePa, 0, 0, UINT64_C(0xffff000000000fff)), /* value=0x0 */
+    MFN(0xc0010118, "AMD_10H_VM_LOCK_KEY", AmdFam10hVmLockKey, AmdFam10hVmLockKey), /* value=0x0 */
+    MFN(0xc0010119, "AMD_10H_SSM_LOCK_KEY", AmdFam10hSmmLockKey, AmdFam10hSmmLockKey), /* value=0x0 */
+    MFX(0xc001011a, "AMD_10H_LOCAL_SMI_STS", AmdFam10hLocalSmiStatus, AmdFam10hLocalSmiStatus, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc0010140, "AMD_10H_OSVW_ID_LEN", AmdFam10hOsVisWrkIdLength, AmdFam10hOsVisWrkIdLength, 0x1, 0, 0), /* value=0x1 */
+    MFN(0xc0010141, "AMD_10H_OSVW_STS", AmdFam10hOsVisWrkStatus, AmdFam10hOsVisWrkStatus), /* value=0x0 */
+    MFX(0xc0011000, "AMD_K7_MCODE_CTL", AmdK7MicrocodeCtl, AmdK7MicrocodeCtl, 0, ~(uint64_t)UINT32_MAX, 0x204), /* value=0x0 */
+    MFX(0xc0011001, "AMD_K7_APIC_CLUSTER_ID", AmdK7ClusterIdMaybe, AmdK7ClusterIdMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFN(0xc0011004, "AMD_K8_CPUID_CTL_STD01", AmdK8CpuIdCtlStd01hEdcx, AmdK8CpuIdCtlStd01hEdcx), /* value=0x802009`178bfbff */
+    MFN(0xc0011005, "AMD_K8_CPUID_CTL_EXT01", AmdK8CpuIdCtlExt01hEdcx, AmdK8CpuIdCtlExt01hEdcx), /* value=0x37ff`efd3fbff */
+    MFX(0xc0011006, "AMD_K7_DEBUG_STS?", AmdK7DebugStatusMaybe, AmdK7DebugStatusMaybe, 0, UINT64_C(0xffffffff00000080), 0), /* value=0x0 */
+    MFN(0xc0011007, "AMD_K7_BH_TRACE_BASE?", AmdK7BHTraceBaseMaybe, AmdK7BHTraceBaseMaybe), /* value=0x0 */
+    MFN(0xc0011008, "AMD_K7_BH_TRACE_PTR?", AmdK7BHTracePtrMaybe, AmdK7BHTracePtrMaybe), /* value=0x0 */
+    MFN(0xc0011009, "AMD_K7_BH_TRACE_LIM?", AmdK7BHTraceLimitMaybe, AmdK7BHTraceLimitMaybe), /* value=0x0 */
+    MFX(0xc001100a, "AMD_K7_HDT_CFG?", AmdK7HardwareDebugToolCfgMaybe, AmdK7HardwareDebugToolCfgMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0xc001100b, "AMD_K7_FAST_FLUSH_COUNT?", AmdK7FastFlushCountMaybe, AmdK7FastFlushCountMaybe, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x7c0 */
+    MFX(0xc001100c, "AMD_K7_NODE_ID", AmdK7NodeId, AmdK7NodeId, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MVX(0xc001100d, "AMD_K8_LOGICAL_CPUS_NUM?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001100e, "AMD_K8_WRMSR_BP?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001100f, "AMD_K8_WRMSR_BP_MASK?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc0011010, "AMD_K8_BH_TRACE_CTL?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0xc0011011, "AMD_K8_BH_TRACE_USRD?", 0), /* value=0x0 */
+    MVI(0xc0011012, "AMD_K7_UNK_c001_1012", UINT32_MAX),
+    MVI(0xc0011013, "AMD_K7_UNK_c001_1013", UINT64_MAX),
+    MVX(0xc0011014, "AMD_K8_XCPT_BP_RIP?", 0, 0, 0),
+    MVX(0xc0011015, "AMD_K8_XCPT_BP_RIP_MASK?", 0, 0, 0),
+    MVX(0xc0011016, "AMD_K8_COND_HDT_VAL?", 0, 0, 0),
+    MVX(0xc0011017, "AMD_K8_COND_HDT_VAL_MASK?", 0, 0, 0),
+    MVX(0xc0011018, "AMD_K8_XCPT_BP_CTL?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0xc001101d, "AMD_K8_NB_BIST?", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0xc001101e, "AMD_K8_THERMTRIP_2?", 0x1830), /* Villain? */
+    MVX(0xc001101f, "AMD_K8_NB_CFG?", UINT64_C(0x40000000000008), 0, 0),
+    MFX(0xc0011020, "AMD_K7_LS_CFG", AmdK7LoadStoreCfg, AmdK7LoadStoreCfg, 0, UINT64_C(0xfffe012000000000), 0), /* value=0x10`00001000 */
+    MFW(0xc0011021, "AMD_K7_IC_CFG", AmdK7InstrCacheCfg, AmdK7InstrCacheCfg, ~(uint64_t)UINT32_MAX), /* value=0x0 */
+    MFX(0xc0011022, "AMD_K7_DC_CFG", AmdK7DataCacheCfg, AmdK7DataCacheCfg, 0, UINT64_C(0xffc0000000000000), 0), /* value=0x1c94`49000000 */
+    MFN(0xc0011023, "AMD_K7_BU_CFG", AmdK7BusUnitCfg, AmdK7BusUnitCfg), /* Villain? value=0x10200020 */
+    MFX(0xc0011024, "AMD_K7_DEBUG_CTL_2?", AmdK7DebugCtl2Maybe, AmdK7DebugCtl2Maybe, 0, UINT64_C(0xffffffffffffff00), 0), /* value=0x0 */
+    MFN(0xc0011025, "AMD_K7_DR0_DATA_MATCH?", AmdK7Dr0DataMatchMaybe, AmdK7Dr0DataMatchMaybe), /* value=0x0 */
+    MFN(0xc0011026, "AMD_K7_DR0_DATA_MATCH?", AmdK7Dr0DataMaskMaybe, AmdK7Dr0DataMaskMaybe), /* value=0x0 */
+    MFX(0xc0011027, "AMD_K7_DR0_ADDR_MASK", AmdK7DrXAddrMaskN, AmdK7DrXAddrMaskN, 0x0, UINT64_C(0xfffffffffffff000), 0), /* value=0x0 */
+    MVX(0xc0011028, "AMD_10H_UNK_c001_1028", 0, UINT64_C(0xfffffffffffffff8), 0),
+    MVX(0xc0011029, "AMD_10H_UNK_c001_1029", 0, ~(uint64_t)UINT32_MAX, 0),
+    MFX(0xc001102a, "AMD_10H_BU_CFG2", AmdFam10hBusUnitCfg2, AmdFam10hBusUnitCfg2, 0, UINT64_C(0xfff00000c0000000), 0), /* value=0x40040`050000c0 */
+    MFX(0xc0011030, "AMD_10H_IBS_FETCH_CTL", AmdFam10hIbsFetchCtl, AmdFam10hIbsFetchCtl, 0, UINT64_C(0xfdfcffff00000000), 0), /* value=0x0 */
+    MFI(0xc0011031, "AMD_10H_IBS_FETCH_LIN_ADDR", AmdFam10hIbsFetchLinAddr), /* value=0xffffff1f`6ffffec0 */
+    MFI(0xc0011032, "AMD_10H_IBS_FETCH_PHYS_ADDR", AmdFam10hIbsFetchPhysAddr), /* value=0xffffbecf`eff1fec0 */
+    MFX(0xc0011033, "AMD_10H_IBS_OP_EXEC_CTL", AmdFam10hIbsOpExecCtl, AmdFam10hIbsOpExecCtl, 0, UINT64_C(0xfffffffffff00000), 0), /* value=0x0 */
+    MFN(0xc0011034, "AMD_10H_IBS_OP_RIP", AmdFam10hIbsOpRip, AmdFam10hIbsOpRip), /* value=0xffffcf06`409f2d93 */
+    MFI(0xc0011035, "AMD_10H_IBS_OP_DATA", AmdFam10hIbsOpData), /* value=0x3b`7701fe63 */
+    MFX(0xc0011036, "AMD_10H_IBS_OP_DATA2", AmdFam10hIbsOpData2, AmdFam10hIbsOpData2, 0, UINT64_C(0xffffffffffffffc8), 0), /* value=0x0 */
+    MFI(0xc0011037, "AMD_10H_IBS_OP_DATA3", AmdFam10hIbsOpData3), /* value=0x0 */
+    MFX(0xc0011038, "AMD_10H_IBS_DC_LIN_ADDR", AmdFam10hIbsDcLinAddr, AmdFam10hIbsDcLinAddr, 0, UINT64_C(0x7fffffffffff), 0), /* value=0x0 */
+    MFI(0xc0011039, "AMD_10H_IBS_DC_PHYS_ADDR", AmdFam10hIbsDcPhysAddr), /* value=0x0 */
+    MFO(0xc001103a, "AMD_10H_IBS_CTL", AmdFam10hIbsCtl), /* value=0x100 */
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for Quad-Core AMD Opteron(tm) Processor 2384.
+ */
+static CPUMDBENTRY const g_Entry_Quad_Core_AMD_Opteron_2384 =
+{
+    /*.pszName          = */ "Quad-Core AMD Opteron 2384",
+    /*.pszFullName      = */ "Quad-Core AMD Opteron(tm) Processor 2384",
+    /*.enmVendor        = */ CPUMCPUVENDOR_AMD,
+    /*.uFamily          = */ 16,
+    /*.uModel           = */ 4,
+    /*.uStepping        = */ 2,
+    /*.enmMicroarch     = */ kCpumMicroarch_AMD_K10,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 48,
+    /*.fMxCsrMask       = */ 0x2ffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_Quad_Core_AMD_Opteron_2384),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_Quad_Core_AMD_Opteron_2384)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_Quad_Core_AMD_Opteron_2384)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_Quad_Core_AMD_Opteron_2384),
+};
+
+#endif /* !VBOX_CPUDB_Quad_Core_AMD_Opteron_2384_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/VIA_QuadCore_L4700_1_2_GHz.h b/src/VBox/VMM/VMMR3/cpus/VIA_QuadCore_L4700_1_2_GHz.h
new file mode 100644
index 00000000..8506de95
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/VIA_QuadCore_L4700_1_2_GHz.h
@@ -0,0 +1,404 @@
+/* $Id: VIA_QuadCore_L4700_1_2_GHz.h $ */
+/** @file
+ * CPU database entry "VIA QuadCore L4700 1.2+ GHz".
+ * Generated at 2013-12-20T14:40:07Z by VBoxCpuReport v4.3.53r91411 on linux.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VBOX_CPUDB_VIA_QuadCore_L4700_1_2_GHz_h
+#define VBOX_CPUDB_VIA_QuadCore_L4700_1_2_GHz_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for VIA QuadCore L4700 @ 1.2+ GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_VIA_QuadCore_L4700_1_2_GHz[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000a, 0x746e6543, 0x736c7561, 0x48727561, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000006fd, 0x06080800, 0x008863a9, 0xbfc9fbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x02b3b001, 0x00000000, 0x00000000, 0x2c04307d, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, 0x00000000, 0x1c000021, 0x03c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00022220, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000002, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x06280202, 0x00000000, 0x00000000, 0x00000503, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x20100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x20202020, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x49562020, 0x75512041, 0x6f436461, 0x4c206572, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x30303734, 0x31204020, 0x202b322e, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x00000000, 0x08800880, 0x40100140, 0x40100140, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x04008140, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003024, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0xc0000000, 0x00000000, 0x00000000, 0xc0000004, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0xc0000001, 0x00000000, 0x00000000, 0x000006fd, 0x00000000, 0x00000000, 0x1ec03dcc, 0 },
+    { 0xc0000002, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0xc0000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0xc0000004, 0x00000000, 0x00000000, 0x000fffb7, 0x08000955, 0x08530954, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for VIA QuadCore L4700 @ 1.2+ GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_VIA_QuadCore_L4700_1_2_GHz[] =
+{
+    RVI(0x00000000, 0x00000005, "ZERO_0000_0000_THRU_0000_0005", 0),
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, UINT64_C(0xffffffffffff0000), 0), /* value=0x40 */
+    RVI(0x00000007, 0x0000000f, "ZERO_0000_0007_THRU_0000_000f", 0),
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x965`912e15ac */
+    RVI(0x00000011, 0x0000001a, "ZERO_0000_0011_THRU_0000_001a", 0),
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0x600, UINT64_C(0xfffffff0000000ff)),
+    RVI(0x0000001c, 0x00000029, "ZERO_0000_001c_THRU_0000_0029", 0),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, IntelEblCrPowerOn, 0x2580000, UINT64_MAX, 0), /* value=0x2580000 */
+    RVI(0x0000002b, 0x00000039, "ZERO_0000_002b_THRU_0000_0039", 0),
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x5 */
+    RVI(0x0000003b, 0x00000078, "ZERO_0000_003b_THRU_0000_0078", 0),
+    RVI(0x0000007a, 0x0000008a, "ZERO_0000_007a_THRU_0000_008a", 0),
+    MFN(0x0000008b, "BBL_CR_D3|BIOS_SIGN", Ia32BiosSignId, Ia32BiosSignId), /* value=0xc`00000000 */
+    RVI(0x0000008c, 0x0000009a, "ZERO_0000_008c_THRU_0000_009a", 0),
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    RVI(0x0000009c, 0x000000c0, "ZERO_0000_009c_THRU_0000_00c0", 0),
+    RSN(0x000000c1, 0x000000c3, "IA32_PMCn", Ia32PmcN, Ia32PmcN, 0x0, UINT64_C(0xffffff0000000000), 0), /* XXX: The range ended earlier than expected! */
+    RVI(0x000000c4, 0x000000cc, "ZERO_0000_00c4_THRU_0000_00cc", 0),
+    MFX(0x000000cd, "MSR_FSB_FREQ", IntelP6FsbFrequency, ReadOnly, 0, 0, 0),
+    RVI(0x000000ce, 0x000000e1, "ZERO_0000_00ce_THRU_0000_00e1", 0),
+    MFI(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl), /* value=0x6a204 */
+    MFX(0x000000e3, "C2_SMM_CST_MISC_INFO", IntelCore2SmmCStMiscInfo, IntelCore2SmmCStMiscInfo, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    RVI(0x000000e5, 0x000000e6, "ZERO_0000_00e5_THRU_0000_00e6", 0),
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf), /* value=0x2f4 */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x2f2 */
+    RVI(0x000000e9, 0x000000fd, "ZERO_0000_00e9_THRU_0000_00fd", 0),
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0xd08, 0, 0), /* value=0xd08 */
+    RVI(0x000000ff, 0x0000011d, "ZERO_0000_00ff_THRU_0000_011d", 0),
+    MFX(0x0000011e, "BBL_CR_CTL3", IntelBblCrCtl3, IntelBblCrCtl3, 0, UINT64_MAX, 0), /* value=0x0 */
+    RVI(0x0000011f, 0x00000173, "ZERO_0000_011f_THRU_0000_0173", 0),
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x10 */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0x0 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0xffffffff`8166bfa0 */
+    RVI(0x00000177, 0x00000178, "ZERO_0000_0177_THRU_0000_0178", 0),
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, UINT64_C(0xfffffffffffffff8), 0), /* value=0x0 */
+    RVI(0x0000017b, 0x00000185, "ZERO_0000_017b_THRU_0000_0185", 0),
+    RSN(0x00000186, 0x00000188, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, UINT64_C(0xfffffffff8280000), 0), /* XXX: The range ended earlier than expected! */
+    RVI(0x00000189, 0x00000197, "ZERO_0000_0189_THRU_0000_0197", 0),
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, Ia32PerfStatus, UINT64_C(0x853095408000955), UINT64_MAX, 0), /* value=0x8530954`08000955 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x954, 0, 0), /* Might bite. value=0x954 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0x2, UINT64_C(0xffffffffffffffe1), 0), /* value=0x2 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0, UINT64_C(0xffffffffff0000e0), 0), /* value=0x0 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, 0x8320000, UINT64_MAX, 0), /* value=0x8320000 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0x853, 0, 0), /* value=0x853 */
+    RVI(0x0000019e, 0x0000019f, "ZERO_0000_019e_THRU_0000_019f", 0),
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, 0x173c89, UINT64_C(0xffffffb87939c176), 0), /* value=0x173c89 */
+    RVI(0x000001a1, 0x000001d8, "ZERO_0000_01a1_THRU_0000_01d8", 0),
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffffe03c)), /* value=0x1 */
+    RVI(0x000001da, 0x000001f1, "ZERO_0000_01da_THRU_0000_01f1", 0),
+    MFO(0x000001f2, "IA32_SMRR_PHYSBASE", Ia32SmrrPhysBase), /* value=0x0 */
+    MFO(0x000001f3, "IA32_SMRR_PHYSMASK", Ia32SmrrPhysMask), /* value=0x0 */
+    RVI(0x000001f4, 0x000001ff, "ZERO_0000_01f4_THRU_0000_01ff", 0),
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`80000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x70000000 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`f0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xd0000001 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`ff800800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    RVI(0x00000210, 0x0000024f, "ZERO_0000_0210_THRU_0000_024f", 0),
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    RVI(0x00000251, 0x00000257, "ZERO_0000_0251_THRU_0000_0257", 0),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    RVI(0x0000025a, 0x00000267, "ZERO_0000_025a_THRU_0000_0267", 0),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    RVI(0x00000270, 0x00000276, "ZERO_0000_0270_THRU_0000_0276", 0),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    RVI(0x00000278, 0x000002fe, "ZERO_0000_0278_THRU_0000_02fe", 0),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    RVI(0x00000300, 0x00000308, "ZERO_0000_0300_THRU_0000_0308", 0),
+    RSN(0x00000309, 0x0000030a, "IA32_FIXED_CTRn", Ia32FixedCtrN, Ia32FixedCtrN, 0x0, UINT64_C(0xffffff0000000000), 0),
+    MFX(0x0000030b, "IA32_FIXED_CTR2", Ia32FixedCtrN, Ia32FixedCtrN, 0x2, UINT64_C(0xfffff8020a068061), 0), /* value=0x2d4 */
+    RVI(0x0000030c, 0x0000038c, "ZERO_0000_030c_THRU_0000_038c", 0),
+    MFX(0x0000038d, "IA32_FIXED_CTR_CTRL", Ia32FixedCtrCtrl, Ia32FixedCtrCtrl, 0, UINT64_C(0xfffffffffffff444), 0), /* value=0x0 */
+    MFX(0x0000038e, "IA32_PERF_GLOBAL_STATUS", Ia32PerfGlobalStatus, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFN(0x0000038f, "IA32_PERF_GLOBAL_CTRL", Ia32PerfGlobalCtrl, Ia32PerfGlobalCtrl), /* value=0xffffffff`ffffffff */
+    RVI(0x00000390, 0x0000047f, "ZERO_0000_0390_THRU_0000_047f", 0),
+    MFX(0x00000480, "IA32_VMX_BASIC", Ia32VmxBasic, ReadOnly, UINT64_C(0x1a040000000007), 0, 0), /* value=0x1a0400`00000007 */
+    MFX(0x00000481, "IA32_VMX_PINBASED_CTLS", Ia32VmxPinbasedCtls, ReadOnly, UINT64_C(0x3f00000016), 0, 0), /* value=0x3f`00000016 */
+    MFX(0x00000482, "IA32_VMX_PROCBASED_CTLS", Ia32VmxProcbasedCtls, ReadOnly, UINT64_C(0x77f9fffe0401e172), 0, 0), /* value=0x77f9fffe`0401e172 */
+    MFX(0x00000483, "IA32_VMX_EXIT_CTLS", Ia32VmxExitCtls, ReadOnly, UINT64_C(0x3efff00036dff), 0, 0), /* value=0x3efff`00036dff */
+    MFX(0x00000484, "IA32_VMX_ENTRY_CTLS", Ia32VmxEntryCtls, ReadOnly, UINT64_C(0x1fff000011ff), 0, 0), /* value=0x1fff`000011ff */
+    MFX(0x00000485, "IA32_VMX_MISC", Ia32VmxMisc, ReadOnly, 0x403c0, 0, 0), /* value=0x403c0 */
+    MFX(0x00000486, "IA32_VMX_CR0_FIXED0", Ia32VmxCr0Fixed0, ReadOnly, UINT32_C(0x80000021), 0, 0), /* value=0x80000021 */
+    MFX(0x00000487, "IA32_VMX_CR0_FIXED1", Ia32VmxCr0Fixed1, ReadOnly, UINT32_MAX, 0, 0), /* value=0xffffffff */
+    MFX(0x00000488, "IA32_VMX_CR4_FIXED0", Ia32VmxCr4Fixed0, ReadOnly, 0x2000, 0, 0), /* value=0x2000 */
+    MFX(0x00000489, "IA32_VMX_CR4_FIXED1", Ia32VmxCr4Fixed1, ReadOnly, 0x27ff, 0, 0), /* value=0x27ff */
+    MFX(0x0000048a, "IA32_VMX_VMCS_ENUM", Ia32VmxVmcsEnum, ReadOnly, 0x2c, 0, 0), /* value=0x2c */
+    RVI(0x0000048b, 0x000005ff, "ZERO_0000_048b_THRU_0000_05ff", 0),
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0x0 */
+    RVI(0x00000601, 0x00001106, "ZERO_0000_0601_THRU_0000_1106", 0),
+    MVI(0x00001107, "VIA_UNK_0000_1107", 0x2),
+    RVI(0x00001108, 0x0000110e, "ZERO_0000_1108_THRU_0000_110e", 0),
+    MVI(0x0000110f, "VIA_UNK_0000_110f", 0x2),
+    RVI(0x00001110, 0x00001152, "ZERO_0000_1110_THRU_0000_1152", 0),
+    MVO(0x00001153, "VIA_UNK_0000_1153", 0),
+    RVI(0x00001154, 0x000011ff, "ZERO_0000_1154_THRU_0000_11ff", 0),
+    MVX(0x00001200, "VIA_UNK_0000_1200", UINT64_C(0x8863a9bfc9fbff), 0x40000, 0),
+    MVX(0x00001201, "VIA_UNK_0000_1201", UINT64_C(0x120100800), UINT64_C(0xfffffff000000000), 0),
+    MVX(0x00001202, "VIA_UNK_0000_1202", 0x3dcc, UINT64_C(0xffffffffffffc233), 0),
+    MVX(0x00001203, "VIA_UNK_0000_1203", 0x18, 0, 0),
+    MVX(0x00001204, "VIA_UNK_0000_1204", UINT64_C(0x6fd00000424), 0, 0),
+    MVX(0x00001205, "VIA_UNK_0000_1205", UINT64_C(0x9890000000001), 0, 0),
+    MVX(0x00001206, "VIA_ALT_VENDOR_EBX", 0, 0, 0),
+    MVX(0x00001207, "VIA_ALT_VENDOR_ECDX", 0, 0, 0),
+    MVX(0x00001208, "VIA_UNK_0000_1208", 0, 0, 0),
+    MVX(0x00001209, "VIA_UNK_0000_1209", 0, 0, 0),
+    MVX(0x0000120a, "VIA_UNK_0000_120a", 0, 0, 0),
+    MVX(0x0000120b, "VIA_UNK_0000_120b", 0, 0, 0),
+    MVX(0x0000120c, "VIA_UNK_0000_120c", 0, 0, 0),
+    MVX(0x0000120d, "VIA_UNK_0000_120d", 0, 0, 0),
+    MVI(0x0000120e, "VIA_UNK_0000_120e", UINT64_C(0x820007b100002080)), /* Villain? */
+    MVX(0x0000120f, "VIA_UNK_0000_120f", UINT64_C(0x200000001a000000), 0x18000000, 0),
+    MVI(0x00001210, "ZERO_0000_1210", 0),
+    MVX(0x00001211, "VIA_UNK_0000_1211", 0, 0, 0),
+    MVX(0x00001212, "VIA_UNK_0000_1212", 0, 0, 0),
+    MVX(0x00001213, "VIA_UNK_0000_1213", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVO(0x00001214, "VIA_UNK_0000_1214", UINT64_C(0x5dd89e10ffffffff)),
+    RVI(0x00001215, 0x0000121f, "ZERO_0000_1215_THRU_0000_121f", 0),
+    MVO(0x00001220, "VIA_UNK_0000_1220", 0),
+    MVO(0x00001221, "VIA_UNK_0000_1221", 0x4dd2e713),
+    RVI(0x00001222, 0x0000122f, "ZERO_0000_1222_THRU_0000_122f", 0),
+    MVX(0x00001230, "VIA_UNK_0000_1230", UINT64_C(0x5dd89e10ffffffff), UINT32_C(0xfffffd68), 0),
+    MVX(0x00001231, "VIA_UNK_0000_1231", UINT64_C(0x7f9110bdc740), 0x200, 0),
+    MVO(0x00001232, "VIA_UNK_0000_1232", UINT64_C(0x2603448430479888)),
+    MVI(0x00001233, "VIA_UNK_0000_1233", UINT64_C(0xb39acda158793c27)), /* Villain? */
+    MVX(0x00001234, "VIA_UNK_0000_1234", 0, 0, 0),
+    MVX(0x00001235, "VIA_UNK_0000_1235", 0, 0, 0),
+    MVX(0x00001236, "VIA_UNK_0000_1236", UINT64_C(0x5dd89e10ffffffff), UINT32_C(0xfffffd68), 0),
+    MVX(0x00001237, "VIA_UNK_0000_1237", UINT32_C(0xffc00026), UINT64_C(0xffffffff06000001), 0),
+    MVO(0x00001238, "VIA_UNK_0000_1238", 0x2),
+    MVI(0x00001239, "VIA_UNK_0000_1239", 0), /* Villain? */
+    RVI(0x0000123a, 0x0000123f, "ZERO_0000_123a_THRU_0000_123f", 0),
+    MVO(0x00001240, "VIA_UNK_0000_1240", 0),
+    MVO(0x00001241, "VIA_UNK_0000_1241", UINT64_C(0x5dd89e10ffffffff)),
+    MVI(0x00001242, "ZERO_0000_1242", 0),
+    MVX(0x00001243, "VIA_UNK_0000_1243", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0x00001244, "ZERO_0000_1244", 0),
+    MVX(0x00001245, "VIA_UNK_0000_1245", UINT64_C(0x3020400000000064), UINT64_C(0xf000000000000000), 0),
+    MVX(0x00001246, "VIA_UNK_0000_1246", UINT64_C(0x10000000000), 0, 0),
+    MVX(0x00001247, "VIA_UNK_0000_1247", 0, 0, 0),
+    MVX(0x00001248, "VIA_UNK_0000_1248", 0, 0, 0),
+    MVI(0x00001249, "VIA_UNK_0000_1249", 0), /* Villain? */
+    MVI(0x0000124a, "VIA_UNK_0000_124a", 0), /* Villain? */
+    RVI(0x0000124b, 0x00001300, "ZERO_0000_124b_THRU_0000_1300", 0),
+    MVX(0x00001301, "VIA_UNK_0000_1301", 0, 0, 0),
+    MVX(0x00001302, "VIA_UNK_0000_1302", 0, 0, 0),
+    MVX(0x00001303, "VIA_UNK_0000_1303", 0, 0, 0),
+    MVX(0x00001304, "VIA_UNK_0000_1304", 0, 0, 0),
+    MVX(0x00001305, "VIA_UNK_0000_1305", 0, 0, 0),
+    MVX(0x00001306, "VIA_UNK_0000_1306", 0, 0, 0),
+    MVX(0x00001307, "VIA_UNK_0000_1307", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00001308, "VIA_UNK_0000_1308", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001309, "VIA_UNK_0000_1309", 0, ~(uint64_t)UINT32_MAX, 0),
+    RVI(0x0000130a, 0x0000130c, "ZERO_0000_130a_THRU_0000_130c", 0),
+    MVX(0x0000130d, "VIA_UNK_0000_130d", 0, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0x0000130e, "VIA_UNK_0000_130e", UINT64_MAX, 0, 0),
+    RVI(0x0000130f, 0x00001311, "ZERO_0000_130f_THRU_0000_1311", 0),
+    MVX(0x00001312, "VIA_UNK_0000_1312", 0, 0, 0),
+    RVI(0x00001313, 0x00001314, "ZERO_0000_1313_THRU_0000_1314", 0),
+    MVX(0x00001315, "VIA_UNK_0000_1315", 0, 0, 0),
+    MVI(0x00001316, "ZERO_0000_1316", 0),
+    MVX(0x00001317, "VIA_UNK_0000_1317", 0, 0, 0),
+    MVX(0x00001318, "VIA_UNK_0000_1318", 0, 0, 0),
+    MVI(0x00001319, "ZERO_0000_1319", 0),
+    MVX(0x0000131a, "VIA_UNK_0000_131a", 0, 0, 0),
+    MVX(0x0000131b, "VIA_UNK_0000_131b", 0x3c20954, 0, 0),
+    RVI(0x0000131c, 0x00001401, "ZERO_0000_131c_THRU_0000_1401", 0),
+    MVO(0x00001402, "VIA_UNK_0000_1402", 0x148c48),
+    MVX(0x00001403, "VIA_UNK_0000_1403", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0x00001404, "VIA_UNK_0000_1404", 0), /* Villain? */
+    MVI(0x00001405, "VIA_UNK_0000_1405", UINT32_C(0x80fffffc)), /* Villain? */
+    MVX(0x00001406, "VIA_UNK_0000_1406", UINT32_C(0xc842c800), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001407, "VIA_UNK_0000_1407", UINT32_C(0x880400c0), ~(uint64_t)UINT32_MAX, 0),
+    RVI(0x00001408, 0x0000140f, "ZERO_0000_1408_THRU_0000_140f", 0),
+    MVX(0x00001410, "VIA_UNK_0000_1410", 0xfa0, UINT64_C(0xfffffffffff00000), 0),
+    MVX(0x00001411, "VIA_UNK_0000_1411", 0xa5a, UINT64_C(0xfffffffffff00000), 0),
+    MVI(0x00001412, "VIA_UNK_0000_1412", 0x4090),
+    MVI(0x00001413, "VIA_UNK_0000_1413", 0), /* Villain? */
+    MVX(0x00001414, "VIA_UNK_0000_1414", 0x5a, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x00001415, "VIA_UNK_0000_1415", 0x5a, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x00001416, "VIA_UNK_0000_1416", 0x6e, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x00001417, "VIA_UNK_0000_1417", 0x32, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x00001418, "VIA_UNK_0000_1418", 0xa, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x00001419, "VIA_UNK_0000_1419", 0x14, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000141a, "VIA_UNK_0000_141a", 0x28, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000141b, "VIA_UNK_0000_141b", 0x3c, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000141c, "VIA_UNK_0000_141c", 0x69, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000141d, "VIA_UNK_0000_141d", 0x69, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000141e, "VIA_UNK_0000_141e", 0x69, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000141f, "VIA_UNK_0000_141f", 0x32, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x00001420, "VIA_UNK_0000_1420", 0x3, UINT64_C(0xffffffffffffc000), 0),
+    MVX(0x00001421, "VIA_UNK_0000_1421", 0x1f8, UINT64_C(0xfffffffffffc0000), 0),
+    MVX(0x00001422, "VIA_UNK_0000_1422", 0x1f4, UINT64_C(0xfffffffffffc0000), 0),
+    MVI(0x00001423, "VIA_UNK_0000_1423", 0xfffb7),
+    MVI(0x00001424, "VIA_UNK_0000_1424", 0x5b6),
+    MVI(0x00001425, "VIA_UNK_0000_1425", 0x65508),
+    MVI(0x00001426, "VIA_UNK_0000_1426", 0x843b),
+    MVX(0x00001427, "VIA_UNK_0000_1427", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001428, "VIA_UNK_0000_1428", 0x1ffffff, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001429, "VIA_UNK_0000_1429", 0, UINT64_C(0xfffffffffff00000), 0),
+    MVI(0x0000142a, "VIA_UNK_0000_142a", 0x1c85d),
+    MVO(0x0000142b, "VIA_UNK_0000_142b", 0xf7e),
+    MVI(0x0000142c, "VIA_UNK_0000_142c", 0x20080), /* Villain? */
+    MVI(0x0000142d, "ZERO_0000_142d", 0),
+    MVI(0x0000142e, "VIA_UNK_0000_142e", 0x8000000), /* Villain? */
+    MVX(0x0000142f, "VIA_UNK_0000_142f", UINT64_C(0xffe57bea2ff3fdff), 0, 0),
+    RVI(0x00001430, 0x00001433, "ZERO_0000_1430_THRU_0000_1433", 0),
+    MVX(0x00001434, "VIA_UNK_0000_1434", 0x853f0e0, UINT64_C(0xffffffff7e7b0000), 0),
+    MVI(0x00001435, "VIA_UNK_0000_1435", 0x8000838), /* Villain? */
+    MVI(0x00001436, "VIA_UNK_0000_1436", 0x200004f), /* Villain? */
+    MVX(0x00001437, "VIA_UNK_0000_1437", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0x00001438, "VIA_UNK_0000_1438", 0x7004801c), /* Villain? */
+    MVI(0x00001439, "ZERO_0000_1439", 0),
+    MVX(0x0000143a, "VIA_UNK_0000_143a", 0x20000, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0x0000143b, "ZERO_0000_143b", 0),
+    MVX(0x0000143c, "VIA_UNK_0000_143c", 0, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000143d, "VIA_UNK_0000_143d", 0, UINT64_C(0xfffffffffffffe00), 0),
+    RVI(0x0000143e, 0x0000143f, "ZERO_0000_143e_THRU_0000_143f", 0),
+    MVX(0x00001440, "VIA_UNK_0000_1440", UINT32_C(0x80e00954), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001441, "VIA_UNK_0000_1441", 0xf00954, UINT64_C(0xffffffff00ff7f7f), 0),
+    MVX(0x00001442, "VIA_UNK_0000_1442", 0xf00954, UINT64_C(0xffffffff00ff7f7f), 0),
+    RVI(0x00001443, 0x00001448, "ZERO_0000_1443_THRU_0000_1448", 0),
+    MVI(0x00001449, "VIA_UNK_0000_1449", UINT64_C(0xfffff7e247)),
+    RVI(0x0000144a, 0x0000144f, "ZERO_0000_144a_THRU_0000_144f", 0),
+    MVX(0x00001450, "VIA_UNK_0000_1450", 0, UINT64_C(0xffffffffffffe000), 0),
+    MVX(0x00001451, "VIA_UNK_0000_1451", 0, UINT64_C(0xffffffffff000000), 0),
+    MVX(0x00001452, "VIA_UNK_0000_1452", 0, UINT64_C(0xffffffffff000000), 0),
+    MVI(0x00001453, "VIA_UNK_0000_1453", 0x3fffffff),
+    RVI(0x00001454, 0x0000145f, "ZERO_0000_1454_THRU_0000_145f", 0),
+    MVX(0x00001460, "VIA_UNK_0000_1460", 0, UINT64_C(0xffffffffffffffc0), 0),
+    MVX(0x00001461, "VIA_UNK_0000_1461", 0x7b, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0x00001462, "VIA_UNK_0000_1462", 0x76, UINT64_C(0xffffffffffffff00), 0),
+    MVI(0x00001463, "VIA_UNK_0000_1463", 0x4a),
+    MVI(0x00001464, "ZERO_0000_1464", 0),
+    MVI(0x00001465, "VIA_UNK_0000_1465", 0xc6),
+    MVI(0x00001466, "VIA_UNK_0000_1466", UINT64_C(0x800000053)),
+    RVI(0x00001467, 0x0000146f, "ZERO_0000_1467_THRU_0000_146f", 0),
+    MVX(0x00001470, "VIA_UNK_0000_1470", UINT64_C(0x5dd89e10ffffffff), UINT32_C(0xfffffd68), 0),
+    MVI(0x00001471, "VIA_UNK_0000_1471", 0x2a000000),
+    RVI(0x00001472, 0x0000147f, "ZERO_0000_1472_THRU_0000_147f", 0),
+    MVI(0x00001480, "VIA_UNK_0000_1480", 0x3907),
+    MVI(0x00001481, "VIA_UNK_0000_1481", 0x12c0),
+    MVI(0x00001482, "VIA_UNK_0000_1482", 0x320),
+    MVI(0x00001483, "VIA_UNK_0000_1483", 0x3),
+    MVI(0x00001484, "VIA_UNK_0000_1484", 0x1647),
+    MVI(0x00001485, "VIA_UNK_0000_1485", 0x3b7),
+    MVI(0x00001486, "VIA_UNK_0000_1486", 0x443),
+    RVI(0x00001487, 0x0000148f, "ZERO_0000_1487_THRU_0000_148f", 0),
+    MVX(0x00001490, "VIA_UNK_0000_1490", 0xf5, UINT64_C(0xffffffffffffc000), 0),
+    MVX(0x00001491, "VIA_UNK_0000_1491", 0x200, UINT64_C(0xffffffffff000000), 0),
+    MVX(0x00001492, "VIA_UNK_0000_1492", 0, UINT64_C(0xffffffffff000000), 0),
+    MVX(0x00001493, "VIA_UNK_0000_1493", 0x4, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0x00001494, "VIA_UNK_0000_1494", 0x100, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0x00001495, "VIA_UNK_0000_1495", 0x100, UINT64_C(0xffffffffff000000), 0),
+    MVX(0x00001496, "VIA_UNK_0000_1496", 0x8, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0x00001497, "VIA_UNK_0000_1497", 0, UINT64_C(0xffffffffff000000), 0),
+    MVX(0x00001498, "VIA_UNK_0000_1498", 0xffffff, UINT64_C(0xfffffffffffffe3c), 0),
+    MVI(0x00001499, "VIA_UNK_0000_1499", 0x2c5),
+    MVI(0x0000149a, "VIA_UNK_0000_149a", 0x1c1),
+    MVI(0x0000149b, "VIA_UNK_0000_149b", 0x2c5a),
+    MVI(0x0000149c, "VIA_UNK_0000_149c", 0x1c8f),
+    RVI(0x0000149d, 0x0000149e, "ZERO_0000_149d_THRU_0000_149e", 0),
+    MVI(0x0000149f, "VIA_UNK_0000_149f", 0x1c9),
+    RVI(0x000014a0, 0x00001522, "ZERO_0000_14a0_THRU_0000_1522", 0),
+    MFN(0x00001523, "VIA_UNK_0000_1523", WriteOnly, IgnoreWrite),
+    RVI(0x00001524, 0x00003179, "ZERO_0000_1524_THRU_0000_3179", 0),
+    MVO(0x0000317a, "VIA_UNK_0000_317a", UINT64_C(0x139f29749595b8)),
+    MVO(0x0000317b, "VIA_UNK_0000_317b", UINT64_C(0x5dd89e10ffffffff)),
+    MVI(0x0000317c, "ZERO_0000_317c", 0),
+    MFN(0x0000317d, "VIA_UNK_0000_317d", WriteOnly, IgnoreWrite),
+    MFN(0x0000317e, "VIA_UNK_0000_317e", WriteOnly, IgnoreWrite),
+    MVI(0x0000317f, "VIA_UNK_0000_317f", 0), /* Villain? */
+    RVI(0x00003180, 0x00003fff, "ZERO_0000_3180_THRU_0000_3fff", 0),
+    RVI(0x40000000, 0x40003fff, "ZERO_4000_0000_THRU_4000_3fff", 0),
+    RVI(0x80000000, 0x80000197, "ZERO_8000_0000_THRU_8000_0197", 0),
+    RVI(0x80000199, 0x80003fff, "ZERO_8000_0199_THRU_8000_3fff", 0),
+    RVI(0xc0000000, 0xc000007f, "ZERO_c000_0000_THRU_c000_007f", 0),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xffffffffffffd2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xffffffff`81669af0 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xffffffff`8166c1d0 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x3700 */
+    RVI(0xc0000085, 0xc00000ff, "ZERO_c000_0085_THRU_c000_00ff", 0),
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x7f91`10bdc740 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffff8800`6fd80000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x0 */
+    RVI(0xc0000104, 0xc0003fff, "ZERO_c000_0104_THRU_c000_3fff", 0),
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for VIA QuadCore L4700 @ 1.2+ GHz.
+ */
+static CPUMDBENTRY const g_Entry_VIA_QuadCore_L4700_1_2_GHz =
+{
+    /*.pszName          = */ "VIA QuadCore L4700 1.2+ GHz",
+    /*.pszFullName      = */ "VIA QuadCore L4700 @ 1.2+ GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_VIA,
+    /*.uFamily          = */ 6,
+    /*.uModel           = */ 15,
+    /*.uStepping        = */ 13,
+    /*.enmMicroarch     = */ kCpumMicroarch_VIA_Isaiah,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_267MHZ, /*??*/
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 36,
+    /*.fMxCsrMask       = */ 0xffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_VIA_QuadCore_L4700_1_2_GHz),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_VIA_QuadCore_L4700_1_2_GHz)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_VIA_QuadCore_L4700_1_2_GHz)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_VIA_QuadCore_L4700_1_2_GHz),
+};
+
+#endif /* !VBOX_CPUDB_VIA_QuadCore_L4700_1_2_GHz_h */
+
diff --git a/src/VBox/VMM/VMMR3/cpus/ZHAOXIN_KaiXian_KX_U5581_1_8GHz.h b/src/VBox/VMM/VMMR3/cpus/ZHAOXIN_KaiXian_KX_U5581_1_8GHz.h
new file mode 100644
index 00000000..d0cf0cab
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/cpus/ZHAOXIN_KaiXian_KX_U5581_1_8GHz.h
@@ -0,0 +1,417 @@
+/* $Id: ZHAOXIN_KaiXian_KX_U5581_1_8GHz.h $ */
+/** @file
+ * CPU database entry "ZHAOXIN KaiXian KX-U5581 1.8GHz"
+ * Generated at 2019-01-15T08:37:25Z by VBoxCpuReport v5.2.22r126460 on linux.amd64.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+#ifndef VBOX_CPUDB_ZHAOXIN_KaiXian_KX_U5581_1_8GHz_h
+#define VBOX_CPUDB_ZHAOXIN_KaiXian_KX_U5581_1_8GHz_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * CPUID leaves for ZHAOXIN KaiXian KX-U5581@1.8GHz.
+ */
+static CPUMCPUIDLEAF const g_aCpuIdLeaves_ZHAOXIN_KaiXian_KX_U5581_1_8GHz[] =
+{
+    { 0x00000000, 0x00000000, 0x00000000, 0x0000000d, 0x68532020, 0x20206961, 0x68676e61, 0 },
+    { 0x00000001, 0x00000000, 0x00000000, 0x000107b5, 0x07080800, 0x7eda63eb, 0xbfcbfbff, 0 | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_CONTAINS_APIC },
+    { 0x00000002, 0x00000000, 0x00000000, 0x635af001, 0x00000000, 0x00000000, 0x000000ff, 0 },
+    { 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000004, 0x00000000, UINT32_MAX, 0x1c000121, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000004, 0x00000001, UINT32_MAX, 0x1c000122, 0x01c0003f, 0x0000003f, 0x00000000, 0 },
+    { 0x00000004, 0x00000002, UINT32_MAX, 0x1c00c143, 0x03c0003f, 0x00000fff, 0x00000003, 0 },
+    { 0x00000004, 0x00000003, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000005, 0x00000000, 0x00000000, 0x00000040, 0x00000040, 0x00000003, 0x00022220, 0 },
+    { 0x00000006, 0x00000000, 0x00000000, 0x00000003, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000007, 0x00000000, UINT32_MAX, 0x00000000, 0x000c258b, 0x00000000, 0x24000000, 0 },
+    { 0x00000007, 0x00000001, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000008, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x00000009, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000a, 0x00000000, 0x00000000, 0x07300402, 0x00000000, 0x00000000, 0x00000603, 0 },
+    { 0x0000000b, 0x00000000, UINT32_MAX, 0x00000000, 0x00000001, 0x00000100, 0x00000007, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000b, 0x00000001, UINT32_MAX, 0x00000003, 0x00000008, 0x00000201, 0x00000007, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000b, 0x00000002, UINT32_MAX, 0x00000000, 0x00000000, 0x00000002, 0x00000007, 0 | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID },
+    { 0x0000000c, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000000, UINT32_MAX, 0x00000007, 0x00000340, 0x00000340, 0x00000000, 0 },
+    { 0x0000000d, 0x00000001, UINT32_MAX, 0x00000001, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000002, UINT32_MAX, 0x00000100, 0x00000240, 0x00000000, 0x00000000, 0 },
+    { 0x0000000d, 0x00000003, UINT32_MAX, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000000, 0x00000000, 0x00000000, 0x80000008, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0x80000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000121, 0x2c100800, 0 },
+    { 0x80000002, 0x00000000, 0x00000000, 0x20202020, 0x20202020, 0x20202020, 0x20202020, 0 },
+    { 0x80000003, 0x00000000, 0x00000000, 0x4f41485a, 0x204e4958, 0x5869614b, 0x206e6169, 0 },
+    { 0x80000004, 0x00000000, 0x00000000, 0x552d584b, 0x31383535, 0x382e3140, 0x007a4847, 0 },
+    { 0x80000005, 0x00000000, 0x00000000, 0x04200420, 0x06600660, 0x20080140, 0x20080140, 0 },
+    { 0x80000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x10008140, 0x00000000, 0 },
+    { 0x80000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000100, 0 },
+    { 0x80000008, 0x00000000, 0x00000000, 0x00003028, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0xc0000000, 0x00000000, 0x00000000, 0xc0000004, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0xc0000001, 0x00000000, 0x00000000, 0x000107b5, 0x00000000, 0x00000000, 0x1ec33dfc, 0 },
+    { 0xc0000002, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0xc0000003, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0 },
+    { 0xc0000004, 0x00000000, 0x00000000, 0x00000025, 0x18002463, 0x18502461, 0x00000000, 0 },
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+#ifndef CPUM_DB_STANDALONE
+/**
+ * MSR ranges for ZHAOXIN KaiXian KX-U5581@1.8GHz.
+ */
+static CPUMMSRRANGE const g_aMsrRanges_ZHAOXIN_KaiXian_KX_U5581_1_8GHz[] =
+{
+    RVI(0x00000000, 0x00000005, "ZERO_0000_0000_THRU_0000_0005", 0),
+    MFX(0x00000006, "IA32_MONITOR_FILTER_LINE_SIZE", Ia32MonitorFilterLineSize, Ia32MonitorFilterLineSize, 0, UINT64_C(0xffffffffffff0000), 0), /* value=0x40 */
+    RVI(0x00000007, 0x0000000f, "ZERO_0000_0007_THRU_0000_000f", 0),
+    MFN(0x00000010, "IA32_TIME_STAMP_COUNTER", Ia32TimestampCounter, Ia32TimestampCounter), /* value=0x965`912e15ac */
+    RVI(0x00000011, 0x0000001a, "ZERO_0000_0011_THRU_0000_001a", 0),
+    MFX(0x0000001b, "IA32_APIC_BASE", Ia32ApicBase, Ia32ApicBase, UINT32_C(0xfee00800), 0x600, UINT64_C(0xfffffff0000000ff)),
+    RVI(0x0000001c, 0x00000029, "ZERO_0000_001c_THRU_0000_0029", 0),
+    MFX(0x0000002a, "EBL_CR_POWERON", IntelEblCrPowerOn, IntelEblCrPowerOn, 0x2580000, UINT64_MAX, 0), /* value=0x2580000 */
+    RVI(0x0000002b, 0x00000039, "ZERO_0000_002b_THRU_0000_0039", 0),
+    MFO(0x0000003a, "IA32_FEATURE_CONTROL", Ia32FeatureControl), /* value=0x5 */
+    RVI(0x0000003b, 0x00000078, "ZERO_0000_003b_THRU_0000_0078", 0),
+    RVI(0x0000007a, 0x0000008a, "ZERO_0000_007a_THRU_0000_008a", 0),
+    MFN(0x0000008b, "BBL_CR_D3|BIOS_SIGN", Ia32BiosSignId, Ia32BiosSignId), /* value=0xc`00000000 */
+    RVI(0x0000008c, 0x0000009a, "ZERO_0000_008c_THRU_0000_009a", 0),
+    MFO(0x0000009b, "IA32_SMM_MONITOR_CTL", Ia32SmmMonitorCtl), /* value=0x0 */
+    RVI(0x0000009c, 0x000000c0, "ZERO_0000_009c_THRU_0000_00c0", 0),
+    RSN(0x000000c1, 0x000000c3, "IA32_PMCn", Ia32PmcN, Ia32PmcN, 0x0, UINT64_C(0xffffff0000000000), 0), /* XXX: The range ended earlier than expected! */
+    RVI(0x000000c4, 0x000000cc, "ZERO_0000_00c4_THRU_0000_00cc", 0),
+    MFX(0x000000cd, "MSR_FSB_FREQ", IntelP6FsbFrequency, ReadOnly, 0, 0, 0),
+    RVI(0x000000ce, 0x000000e1, "ZERO_0000_00ce_THRU_0000_00e1", 0),
+    MFI(0x000000e2, "MSR_PKG_CST_CONFIG_CONTROL", IntelPkgCStConfigControl), /* value=0x6a204 */
+    MFX(0x000000e3, "C2_SMM_CST_MISC_INFO", IntelCore2SmmCStMiscInfo, IntelCore2SmmCStMiscInfo, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    MFX(0x000000e4, "MSR_PMG_IO_CAPTURE_BASE", IntelPmgIoCaptureBase, IntelPmgIoCaptureBase, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x0 */
+    RVI(0x000000e5, 0x000000e6, "ZERO_0000_00e5_THRU_0000_00e6", 0),
+    MFN(0x000000e7, "IA32_MPERF", Ia32MPerf, Ia32MPerf), /* value=0x2f4 */
+    MFN(0x000000e8, "IA32_APERF", Ia32APerf, Ia32APerf), /* value=0x2f2 */
+    RVI(0x000000e9, 0x000000fd, "ZERO_0000_00e9_THRU_0000_00fd", 0),
+    MFX(0x000000fe, "IA32_MTRRCAP", Ia32MtrrCap, ReadOnly, 0xd08, 0, 0), /* value=0xd08 */
+    RVI(0x000000ff, 0x0000011d, "ZERO_0000_00ff_THRU_0000_011d", 0),
+    MFX(0x0000011e, "BBL_CR_CTL3", IntelBblCrCtl3, IntelBblCrCtl3, 0, UINT64_MAX, 0), /* value=0x0 */
+    RVI(0x0000011f, 0x00000173, "ZERO_0000_011f_THRU_0000_0173", 0),
+    MFX(0x00000174, "IA32_SYSENTER_CS", Ia32SysEnterCs, Ia32SysEnterCs, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x10 */
+    MFN(0x00000175, "IA32_SYSENTER_ESP", Ia32SysEnterEsp, Ia32SysEnterEsp), /* value=0x0 */
+    MFN(0x00000176, "IA32_SYSENTER_EIP", Ia32SysEnterEip, Ia32SysEnterEip), /* value=0xffffffff`8166bfa0 */
+    RVI(0x00000177, 0x00000178, "ZERO_0000_0177_THRU_0000_0178", 0),
+    MFX(0x00000179, "IA32_MCG_CAP", Ia32McgCap, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFX(0x0000017a, "IA32_MCG_STATUS", Ia32McgStatus, Ia32McgStatus, 0, UINT64_C(0xfffffffffffffff8), 0), /* value=0x0 */
+    RVI(0x0000017b, 0x00000185, "ZERO_0000_017b_THRU_0000_0185", 0),
+    RSN(0x00000186, 0x00000188, "IA32_PERFEVTSELn", Ia32PerfEvtSelN, Ia32PerfEvtSelN, 0x0, UINT64_C(0xfffffffff8280000), 0), /* XXX: The range ended earlier than expected! */
+    RVI(0x00000189, 0x00000197, "ZERO_0000_0189_THRU_0000_0197", 0),
+    MFX(0x00000198, "IA32_PERF_STATUS", Ia32PerfStatus, Ia32PerfStatus, UINT64_C(0x853095408000955), UINT64_MAX, 0), /* value=0x8530954`08000955 */
+    MFX(0x00000199, "IA32_PERF_CTL", Ia32PerfCtl, Ia32PerfCtl, 0x954, 0, 0), /* Might bite. value=0x954 */
+    MFX(0x0000019a, "IA32_CLOCK_MODULATION", Ia32ClockModulation, Ia32ClockModulation, 0x2, UINT64_C(0xffffffffffffffe1), 0), /* value=0x2 */
+    MFX(0x0000019b, "IA32_THERM_INTERRUPT", Ia32ThermInterrupt, Ia32ThermInterrupt, 0, UINT64_C(0xffffffffff0000e0), 0), /* value=0x0 */
+    MFX(0x0000019c, "IA32_THERM_STATUS", Ia32ThermStatus, Ia32ThermStatus, 0x8320000, UINT64_MAX, 0), /* value=0x8320000 */
+    MFX(0x0000019d, "IA32_THERM2_CTL", Ia32Therm2Ctl, ReadOnly, 0x853, 0, 0), /* value=0x853 */
+    RVI(0x0000019e, 0x0000019f, "ZERO_0000_019e_THRU_0000_019f", 0),
+    MFX(0x000001a0, "IA32_MISC_ENABLE", Ia32MiscEnable, Ia32MiscEnable, 0x173c89, UINT64_C(0xffffffb87939c176), 0), /* value=0x173c89 */
+    RVI(0x000001a1, 0x000001d8, "ZERO_0000_01a1_THRU_0000_01d8", 0),
+    MFX(0x000001d9, "IA32_DEBUGCTL", Ia32DebugCtl, Ia32DebugCtl, 0, 0, UINT64_C(0xffffffffffffe03c)), /* value=0x1 */
+    RVI(0x000001da, 0x000001f1, "ZERO_0000_01da_THRU_0000_01f1", 0),
+    MFO(0x000001f2, "IA32_SMRR_PHYSBASE", Ia32SmrrPhysBase), /* value=0x0 */
+    MFO(0x000001f3, "IA32_SMRR_PHYSMASK", Ia32SmrrPhysMask), /* value=0x0 */
+    RVI(0x000001f4, 0x000001ff, "ZERO_0000_01f4_THRU_0000_01ff", 0),
+    MFX(0x00000200, "IA32_MTRR_PHYS_BASE0", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x0, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x6 */
+    MFX(0x00000201, "IA32_MTRR_PHYS_MASK0", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x0, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`80000800 */
+    MFX(0x00000202, "IA32_MTRR_PHYS_BASE1", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x1, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x70000000 */
+    MFX(0x00000203, "IA32_MTRR_PHYS_MASK1", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x1, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`f0000800 */
+    MFX(0x00000204, "IA32_MTRR_PHYS_BASE2", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x2, 0, UINT64_C(0xfffffff000000ff8)), /* value=0xd0000001 */
+    MFX(0x00000205, "IA32_MTRR_PHYS_MASK2", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x2, 0, UINT64_C(0xfffffff0000007ff)), /* value=0xf`ff800800 */
+    MFX(0x00000206, "IA32_MTRR_PHYS_BASE3", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x3, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000207, "IA32_MTRR_PHYS_MASK3", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x3, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x00000208, "IA32_MTRR_PHYS_BASE4", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x4, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x00000209, "IA32_MTRR_PHYS_MASK4", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x4, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020a, "IA32_MTRR_PHYS_BASE5", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x5, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020b, "IA32_MTRR_PHYS_MASK5", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x5, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020c, "IA32_MTRR_PHYS_BASE6", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x6, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020d, "IA32_MTRR_PHYS_MASK6", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x6, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    MFX(0x0000020e, "IA32_MTRR_PHYS_BASE7", Ia32MtrrPhysBaseN, Ia32MtrrPhysBaseN, 0x7, 0, UINT64_C(0xfffffff000000ff8)), /* value=0x0 */
+    MFX(0x0000020f, "IA32_MTRR_PHYS_MASK7", Ia32MtrrPhysMaskN, Ia32MtrrPhysMaskN, 0x7, 0, UINT64_C(0xfffffff0000007ff)), /* value=0x0 */
+    RVI(0x00000210, 0x0000024f, "ZERO_0000_0210_THRU_0000_024f", 0),
+    MFS(0x00000250, "IA32_MTRR_FIX64K_00000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix64K_00000),
+    RVI(0x00000251, 0x00000257, "ZERO_0000_0251_THRU_0000_0257", 0),
+    MFS(0x00000258, "IA32_MTRR_FIX16K_80000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_80000),
+    MFS(0x00000259, "IA32_MTRR_FIX16K_A0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix16K_A0000),
+    RVI(0x0000025a, 0x00000267, "ZERO_0000_025a_THRU_0000_0267", 0),
+    MFS(0x00000268, "IA32_MTRR_FIX4K_C0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C0000),
+    MFS(0x00000269, "IA32_MTRR_FIX4K_C8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_C8000),
+    MFS(0x0000026a, "IA32_MTRR_FIX4K_D0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D0000),
+    MFS(0x0000026b, "IA32_MTRR_FIX4K_D8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_D8000),
+    MFS(0x0000026c, "IA32_MTRR_FIX4K_E0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E0000),
+    MFS(0x0000026d, "IA32_MTRR_FIX4K_E8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_E8000),
+    MFS(0x0000026e, "IA32_MTRR_FIX4K_F0000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F0000),
+    MFS(0x0000026f, "IA32_MTRR_FIX4K_F8000", Ia32MtrrFixed, Ia32MtrrFixed, GuestMsrs.msr.MtrrFix4K_F8000),
+    RVI(0x00000270, 0x00000276, "ZERO_0000_0270_THRU_0000_0276", 0),
+    MFS(0x00000277, "IA32_PAT", Ia32Pat, Ia32Pat, Guest.msrPAT),
+    RVI(0x00000278, 0x000002fe, "ZERO_0000_0278_THRU_0000_02fe", 0),
+    MFZ(0x000002ff, "IA32_MTRR_DEF_TYPE", Ia32MtrrDefType, Ia32MtrrDefType, GuestMsrs.msr.MtrrDefType, 0, UINT64_C(0xfffffffffffff3f8)),
+    RVI(0x00000300, 0x00000308, "ZERO_0000_0300_THRU_0000_0308", 0),
+    RSN(0x00000309, 0x0000030a, "IA32_FIXED_CTRn", Ia32FixedCtrN, Ia32FixedCtrN, 0x0, UINT64_C(0xffffff0000000000), 0),
+    MFX(0x0000030b, "IA32_FIXED_CTR2", Ia32FixedCtrN, Ia32FixedCtrN, 0x2, UINT64_C(0xfffff8020a068061), 0), /* value=0x2d4 */
+    RVI(0x0000030c, 0x0000038c, "ZERO_0000_030c_THRU_0000_038c", 0),
+    MFX(0x0000038d, "IA32_FIXED_CTR_CTRL", Ia32FixedCtrCtrl, Ia32FixedCtrCtrl, 0, UINT64_C(0xfffffffffffff444), 0), /* value=0x0 */
+    MFX(0x0000038e, "IA32_PERF_GLOBAL_STATUS", Ia32PerfGlobalStatus, ReadOnly, 0, 0, 0), /* value=0x0 */
+    MFN(0x0000038f, "IA32_PERF_GLOBAL_CTRL", Ia32PerfGlobalCtrl, Ia32PerfGlobalCtrl), /* value=0xffffffff`ffffffff */
+    RVI(0x00000390, 0x0000047f, "ZERO_0000_0390_THRU_0000_047f", 0),
+    MFX(0x00000480, "IA32_VMX_BASIC", Ia32VmxBasic, ReadOnly, UINT64_C(0x1a040000000007), 0, 0), /* value=0x1a0400`00000007 */
+    MFX(0x00000481, "IA32_VMX_PINBASED_CTLS", Ia32VmxPinbasedCtls, ReadOnly, UINT64_C(0x3f00000016), 0, 0), /* value=0x3f`00000016 */
+    MFX(0x00000482, "IA32_VMX_PROCBASED_CTLS", Ia32VmxProcbasedCtls, ReadOnly, UINT64_C(0x77f9fffe0401e172), 0, 0), /* value=0x77f9fffe`0401e172 */
+    MFX(0x00000483, "IA32_VMX_EXIT_CTLS", Ia32VmxExitCtls, ReadOnly, UINT64_C(0x3efff00036dff), 0, 0), /* value=0x3efff`00036dff */
+    MFX(0x00000484, "IA32_VMX_ENTRY_CTLS", Ia32VmxEntryCtls, ReadOnly, UINT64_C(0x1fff000011ff), 0, 0), /* value=0x1fff`000011ff */
+    MFX(0x00000485, "IA32_VMX_MISC", Ia32VmxMisc, ReadOnly, 0x403c0, 0, 0), /* value=0x403c0 */
+    MFX(0x00000486, "IA32_VMX_CR0_FIXED0", Ia32VmxCr0Fixed0, ReadOnly, UINT32_C(0x80000021), 0, 0), /* value=0x80000021 */
+    MFX(0x00000487, "IA32_VMX_CR0_FIXED1", Ia32VmxCr0Fixed1, ReadOnly, UINT32_MAX, 0, 0), /* value=0xffffffff */
+    MFX(0x00000488, "IA32_VMX_CR4_FIXED0", Ia32VmxCr4Fixed0, ReadOnly, 0x2000, 0, 0), /* value=0x2000 */
+    MFX(0x00000489, "IA32_VMX_CR4_FIXED1", Ia32VmxCr4Fixed1, ReadOnly, 0x27ff, 0, 0), /* value=0x27ff */
+    MFX(0x0000048a, "IA32_VMX_VMCS_ENUM", Ia32VmxVmcsEnum, ReadOnly, 0x2c, 0, 0), /* value=0x2c */
+    RVI(0x0000048b, 0x000005ff, "ZERO_0000_048b_THRU_0000_05ff", 0),
+    MFN(0x00000600, "IA32_DS_AREA", Ia32DsArea, Ia32DsArea), /* value=0x0 */
+    RVI(0x00000601, 0x00001106, "ZERO_0000_0601_THRU_0000_1106", 0),
+    MVI(0x00001107, "VIA_UNK_0000_1107", 0x2),
+    RVI(0x00001108, 0x0000110e, "ZERO_0000_1108_THRU_0000_110e", 0),
+    MVI(0x0000110f, "VIA_UNK_0000_110f", 0x2),
+    RVI(0x00001110, 0x00001152, "ZERO_0000_1110_THRU_0000_1152", 0),
+    MVO(0x00001153, "VIA_UNK_0000_1153", 0),
+    RVI(0x00001154, 0x000011ff, "ZERO_0000_1154_THRU_0000_11ff", 0),
+    MVX(0x00001200, "VIA_UNK_0000_1200", UINT64_C(0x8863a9bfc9fbff), 0x40000, 0),
+    MVX(0x00001201, "VIA_UNK_0000_1201", UINT64_C(0x120100800), UINT64_C(0xfffffff000000000), 0),
+    MVX(0x00001202, "VIA_UNK_0000_1202", 0x3dcc, UINT64_C(0xffffffffffffc233), 0),
+    MVX(0x00001203, "VIA_UNK_0000_1203", 0x18, 0, 0),
+    MVX(0x00001204, "VIA_UNK_0000_1204", UINT64_C(0x6fd00000424), 0, 0),
+    MVX(0x00001205, "VIA_UNK_0000_1205", UINT64_C(0x9890000000001), 0, 0),
+    MVX(0x00001206, "VIA_ALT_VENDOR_EBX", 0, 0, 0),
+    MVX(0x00001207, "VIA_ALT_VENDOR_ECDX", 0, 0, 0),
+    MVX(0x00001208, "VIA_UNK_0000_1208", 0, 0, 0),
+    MVX(0x00001209, "VIA_UNK_0000_1209", 0, 0, 0),
+    MVX(0x0000120a, "VIA_UNK_0000_120a", 0, 0, 0),
+    MVX(0x0000120b, "VIA_UNK_0000_120b", 0, 0, 0),
+    MVX(0x0000120c, "VIA_UNK_0000_120c", 0, 0, 0),
+    MVX(0x0000120d, "VIA_UNK_0000_120d", 0, 0, 0),
+    MVI(0x0000120e, "VIA_UNK_0000_120e", UINT64_C(0x820007b100002080)), /* Villain? */
+    MVX(0x0000120f, "VIA_UNK_0000_120f", UINT64_C(0x200000001a000000), 0x18000000, 0),
+    MVI(0x00001210, "ZERO_0000_1210", 0),
+    MVX(0x00001211, "VIA_UNK_0000_1211", 0, 0, 0),
+    MVX(0x00001212, "VIA_UNK_0000_1212", 0, 0, 0),
+    MVX(0x00001213, "VIA_UNK_0000_1213", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVO(0x00001214, "VIA_UNK_0000_1214", UINT64_C(0x5dd89e10ffffffff)),
+    RVI(0x00001215, 0x0000121f, "ZERO_0000_1215_THRU_0000_121f", 0),
+    MVO(0x00001220, "VIA_UNK_0000_1220", 0),
+    MVO(0x00001221, "VIA_UNK_0000_1221", 0x4dd2e713),
+    RVI(0x00001222, 0x0000122f, "ZERO_0000_1222_THRU_0000_122f", 0),
+    MVX(0x00001230, "VIA_UNK_0000_1230", UINT64_C(0x5dd89e10ffffffff), UINT32_C(0xfffffd68), 0),
+    MVX(0x00001231, "VIA_UNK_0000_1231", UINT64_C(0x7f9110bdc740), 0x200, 0),
+    MVO(0x00001232, "VIA_UNK_0000_1232", UINT64_C(0x2603448430479888)),
+    MVI(0x00001233, "VIA_UNK_0000_1233", UINT64_C(0xb39acda158793c27)), /* Villain? */
+    MVX(0x00001234, "VIA_UNK_0000_1234", 0, 0, 0),
+    MVX(0x00001235, "VIA_UNK_0000_1235", 0, 0, 0),
+    MVX(0x00001236, "VIA_UNK_0000_1236", UINT64_C(0x5dd89e10ffffffff), UINT32_C(0xfffffd68), 0),
+    MVX(0x00001237, "VIA_UNK_0000_1237", UINT32_C(0xffc00026), UINT64_C(0xffffffff06000001), 0),
+    MVO(0x00001238, "VIA_UNK_0000_1238", 0x2),
+    MVI(0x00001239, "VIA_UNK_0000_1239", 0), /* Villain? */
+    RVI(0x0000123a, 0x0000123f, "ZERO_0000_123a_THRU_0000_123f", 0),
+    MVO(0x00001240, "VIA_UNK_0000_1240", 0),
+    MVO(0x00001241, "VIA_UNK_0000_1241", UINT64_C(0x5dd89e10ffffffff)),
+    MVI(0x00001242, "ZERO_0000_1242", 0),
+    MVX(0x00001243, "VIA_UNK_0000_1243", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0x00001244, "ZERO_0000_1244", 0),
+    MVX(0x00001245, "VIA_UNK_0000_1245", UINT64_C(0x3020400000000064), UINT64_C(0xf000000000000000), 0),
+    MVX(0x00001246, "VIA_UNK_0000_1246", UINT64_C(0x10000000000), 0, 0),
+    MVX(0x00001247, "VIA_UNK_0000_1247", 0, 0, 0),
+    MVX(0x00001248, "VIA_UNK_0000_1248", 0, 0, 0),
+    MVI(0x00001249, "VIA_UNK_0000_1249", 0), /* Villain? */
+    MVI(0x0000124a, "VIA_UNK_0000_124a", 0), /* Villain? */
+    RVI(0x0000124b, 0x00001300, "ZERO_0000_124b_THRU_0000_1300", 0),
+    MVX(0x00001301, "VIA_UNK_0000_1301", 0, 0, 0),
+    MVX(0x00001302, "VIA_UNK_0000_1302", 0, 0, 0),
+    MVX(0x00001303, "VIA_UNK_0000_1303", 0, 0, 0),
+    MVX(0x00001304, "VIA_UNK_0000_1304", 0, 0, 0),
+    MVX(0x00001305, "VIA_UNK_0000_1305", 0, 0, 0),
+    MVX(0x00001306, "VIA_UNK_0000_1306", 0, 0, 0),
+    MVX(0x00001307, "VIA_UNK_0000_1307", 0, UINT64_C(0xffffff0000000000), 0),
+    MVX(0x00001308, "VIA_UNK_0000_1308", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001309, "VIA_UNK_0000_1309", 0, ~(uint64_t)UINT32_MAX, 0),
+    RVI(0x0000130a, 0x0000130c, "ZERO_0000_130a_THRU_0000_130c", 0),
+    MVX(0x0000130d, "VIA_UNK_0000_130d", 0, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0x0000130e, "VIA_UNK_0000_130e", UINT64_MAX, 0, 0),
+    RVI(0x0000130f, 0x00001311, "ZERO_0000_130f_THRU_0000_1311", 0),
+    MVX(0x00001312, "VIA_UNK_0000_1312", 0, 0, 0),
+    RVI(0x00001313, 0x00001314, "ZERO_0000_1313_THRU_0000_1314", 0),
+    MVX(0x00001315, "VIA_UNK_0000_1315", 0, 0, 0),
+    MVI(0x00001316, "ZERO_0000_1316", 0),
+    MVX(0x00001317, "VIA_UNK_0000_1317", 0, 0, 0),
+    MVX(0x00001318, "VIA_UNK_0000_1318", 0, 0, 0),
+    MVI(0x00001319, "ZERO_0000_1319", 0),
+    MVX(0x0000131a, "VIA_UNK_0000_131a", 0, 0, 0),
+    MVX(0x0000131b, "VIA_UNK_0000_131b", 0x3c20954, 0, 0),
+    RVI(0x0000131c, 0x00001401, "ZERO_0000_131c_THRU_0000_1401", 0),
+    MVO(0x00001402, "VIA_UNK_0000_1402", 0x148c48),
+    MVX(0x00001403, "VIA_UNK_0000_1403", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0x00001404, "VIA_UNK_0000_1404", 0), /* Villain? */
+    MVI(0x00001405, "VIA_UNK_0000_1405", UINT32_C(0x80fffffc)), /* Villain? */
+    MVX(0x00001406, "VIA_UNK_0000_1406", UINT32_C(0xc842c800), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001407, "VIA_UNK_0000_1407", UINT32_C(0x880400c0), ~(uint64_t)UINT32_MAX, 0),
+    RVI(0x00001408, 0x0000140f, "ZERO_0000_1408_THRU_0000_140f", 0),
+    MVX(0x00001410, "VIA_UNK_0000_1410", 0xfa0, UINT64_C(0xfffffffffff00000), 0),
+    MVX(0x00001411, "VIA_UNK_0000_1411", 0xa5a, UINT64_C(0xfffffffffff00000), 0),
+    MVI(0x00001412, "VIA_UNK_0000_1412", 0x4090),
+    MVI(0x00001413, "VIA_UNK_0000_1413", 0), /* Villain? */
+    MVX(0x00001414, "VIA_UNK_0000_1414", 0x5a, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x00001415, "VIA_UNK_0000_1415", 0x5a, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x00001416, "VIA_UNK_0000_1416", 0x6e, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x00001417, "VIA_UNK_0000_1417", 0x32, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x00001418, "VIA_UNK_0000_1418", 0xa, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x00001419, "VIA_UNK_0000_1419", 0x14, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000141a, "VIA_UNK_0000_141a", 0x28, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000141b, "VIA_UNK_0000_141b", 0x3c, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000141c, "VIA_UNK_0000_141c", 0x69, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000141d, "VIA_UNK_0000_141d", 0x69, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000141e, "VIA_UNK_0000_141e", 0x69, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000141f, "VIA_UNK_0000_141f", 0x32, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x00001420, "VIA_UNK_0000_1420", 0x3, UINT64_C(0xffffffffffffc000), 0),
+    MVX(0x00001421, "VIA_UNK_0000_1421", 0x1f8, UINT64_C(0xfffffffffffc0000), 0),
+    MVX(0x00001422, "VIA_UNK_0000_1422", 0x1f4, UINT64_C(0xfffffffffffc0000), 0),
+    MVI(0x00001423, "VIA_UNK_0000_1423", 0xfffb7),
+    MVI(0x00001424, "VIA_UNK_0000_1424", 0x5b6),
+    MVI(0x00001425, "VIA_UNK_0000_1425", 0x65508),
+    MVI(0x00001426, "VIA_UNK_0000_1426", 0x843b),
+    MVX(0x00001427, "VIA_UNK_0000_1427", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001428, "VIA_UNK_0000_1428", 0x1ffffff, ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001429, "VIA_UNK_0000_1429", 0, UINT64_C(0xfffffffffff00000), 0),
+    MVI(0x0000142a, "VIA_UNK_0000_142a", 0x1c85d),
+    MVO(0x0000142b, "VIA_UNK_0000_142b", 0xf7e),
+    MVI(0x0000142c, "VIA_UNK_0000_142c", 0x20080), /* Villain? */
+    MVI(0x0000142d, "ZERO_0000_142d", 0),
+    MVI(0x0000142e, "VIA_UNK_0000_142e", 0x8000000), /* Villain? */
+    MVX(0x0000142f, "VIA_UNK_0000_142f", UINT64_C(0xffe57bea2ff3fdff), 0, 0),
+    RVI(0x00001430, 0x00001433, "ZERO_0000_1430_THRU_0000_1433", 0),
+    MVX(0x00001434, "VIA_UNK_0000_1434", 0x853f0e0, UINT64_C(0xffffffff7e7b0000), 0),
+    MVI(0x00001435, "VIA_UNK_0000_1435", 0x8000838), /* Villain? */
+    MVI(0x00001436, "VIA_UNK_0000_1436", 0x200004f), /* Villain? */
+    MVX(0x00001437, "VIA_UNK_0000_1437", 0, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0x00001438, "VIA_UNK_0000_1438", 0x7004801c), /* Villain? */
+    MVI(0x00001439, "ZERO_0000_1439", 0),
+    MVX(0x0000143a, "VIA_UNK_0000_143a", 0x20000, ~(uint64_t)UINT32_MAX, 0),
+    MVI(0x0000143b, "ZERO_0000_143b", 0),
+    MVX(0x0000143c, "VIA_UNK_0000_143c", 0, UINT64_C(0xfffffffffffffe00), 0),
+    MVX(0x0000143d, "VIA_UNK_0000_143d", 0, UINT64_C(0xfffffffffffffe00), 0),
+    RVI(0x0000143e, 0x0000143f, "ZERO_0000_143e_THRU_0000_143f", 0),
+    MVX(0x00001440, "VIA_UNK_0000_1440", UINT32_C(0x80e00954), ~(uint64_t)UINT32_MAX, 0),
+    MVX(0x00001441, "VIA_UNK_0000_1441", 0xf00954, UINT64_C(0xffffffff00ff7f7f), 0),
+    MVX(0x00001442, "VIA_UNK_0000_1442", 0xf00954, UINT64_C(0xffffffff00ff7f7f), 0),
+    RVI(0x00001443, 0x00001448, "ZERO_0000_1443_THRU_0000_1448", 0),
+    MVI(0x00001449, "VIA_UNK_0000_1449", UINT64_C(0xfffff7e247)),
+    RVI(0x0000144a, 0x0000144f, "ZERO_0000_144a_THRU_0000_144f", 0),
+    MVX(0x00001450, "VIA_UNK_0000_1450", 0, UINT64_C(0xffffffffffffe000), 0),
+    MVX(0x00001451, "VIA_UNK_0000_1451", 0, UINT64_C(0xffffffffff000000), 0),
+    MVX(0x00001452, "VIA_UNK_0000_1452", 0, UINT64_C(0xffffffffff000000), 0),
+    MVI(0x00001453, "VIA_UNK_0000_1453", 0x3fffffff),
+    RVI(0x00001454, 0x0000145f, "ZERO_0000_1454_THRU_0000_145f", 0),
+    MVX(0x00001460, "VIA_UNK_0000_1460", 0, UINT64_C(0xffffffffffffffc0), 0),
+    MVX(0x00001461, "VIA_UNK_0000_1461", 0x7b, UINT64_C(0xffffffffffffff00), 0),
+    MVX(0x00001462, "VIA_UNK_0000_1462", 0x76, UINT64_C(0xffffffffffffff00), 0),
+    MVI(0x00001463, "VIA_UNK_0000_1463", 0x4a),
+    MVI(0x00001464, "ZERO_0000_1464", 0),
+    MVI(0x00001465, "VIA_UNK_0000_1465", 0xc6),
+    MVI(0x00001466, "VIA_UNK_0000_1466", UINT64_C(0x800000053)),
+    RVI(0x00001467, 0x0000146f, "ZERO_0000_1467_THRU_0000_146f", 0),
+    MVX(0x00001470, "VIA_UNK_0000_1470", UINT64_C(0x5dd89e10ffffffff), UINT32_C(0xfffffd68), 0),
+    MVI(0x00001471, "VIA_UNK_0000_1471", 0x2a000000),
+    RVI(0x00001472, 0x0000147f, "ZERO_0000_1472_THRU_0000_147f", 0),
+    MVI(0x00001480, "VIA_UNK_0000_1480", 0x3907),
+    MVI(0x00001481, "VIA_UNK_0000_1481", 0x12c0),
+    MVI(0x00001482, "VIA_UNK_0000_1482", 0x320),
+    MVI(0x00001483, "VIA_UNK_0000_1483", 0x3),
+    MVI(0x00001484, "VIA_UNK_0000_1484", 0x1647),
+    MVI(0x00001485, "VIA_UNK_0000_1485", 0x3b7),
+    MVI(0x00001486, "VIA_UNK_0000_1486", 0x443),
+    RVI(0x00001487, 0x0000148f, "ZERO_0000_1487_THRU_0000_148f", 0),
+    MVX(0x00001490, "VIA_UNK_0000_1490", 0xf5, UINT64_C(0xffffffffffffc000), 0),
+    MVX(0x00001491, "VIA_UNK_0000_1491", 0x200, UINT64_C(0xffffffffff000000), 0),
+    MVX(0x00001492, "VIA_UNK_0000_1492", 0, UINT64_C(0xffffffffff000000), 0),
+    MVX(0x00001493, "VIA_UNK_0000_1493", 0x4, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0x00001494, "VIA_UNK_0000_1494", 0x100, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0x00001495, "VIA_UNK_0000_1495", 0x100, UINT64_C(0xffffffffff000000), 0),
+    MVX(0x00001496, "VIA_UNK_0000_1496", 0x8, UINT64_C(0xffffffffffff0000), 0),
+    MVX(0x00001497, "VIA_UNK_0000_1497", 0, UINT64_C(0xffffffffff000000), 0),
+    MVX(0x00001498, "VIA_UNK_0000_1498", 0xffffff, UINT64_C(0xfffffffffffffe3c), 0),
+    MVI(0x00001499, "VIA_UNK_0000_1499", 0x2c5),
+    MVI(0x0000149a, "VIA_UNK_0000_149a", 0x1c1),
+    MVI(0x0000149b, "VIA_UNK_0000_149b", 0x2c5a),
+    MVI(0x0000149c, "VIA_UNK_0000_149c", 0x1c8f),
+    RVI(0x0000149d, 0x0000149e, "ZERO_0000_149d_THRU_0000_149e", 0),
+    MVI(0x0000149f, "VIA_UNK_0000_149f", 0x1c9),
+    RVI(0x000014a0, 0x00001522, "ZERO_0000_14a0_THRU_0000_1522", 0),
+    MFN(0x00001523, "VIA_UNK_0000_1523", WriteOnly, IgnoreWrite),
+    RVI(0x00001524, 0x00003179, "ZERO_0000_1524_THRU_0000_3179", 0),
+    MVO(0x0000317a, "VIA_UNK_0000_317a", UINT64_C(0x139f29749595b8)),
+    MVO(0x0000317b, "VIA_UNK_0000_317b", UINT64_C(0x5dd89e10ffffffff)),
+    MVI(0x0000317c, "ZERO_0000_317c", 0),
+    MFN(0x0000317d, "VIA_UNK_0000_317d", WriteOnly, IgnoreWrite),
+    MFN(0x0000317e, "VIA_UNK_0000_317e", WriteOnly, IgnoreWrite),
+    MVI(0x0000317f, "VIA_UNK_0000_317f", 0), /* Villain? */
+    RVI(0x00003180, 0x00003fff, "ZERO_0000_3180_THRU_0000_3fff", 0),
+    RVI(0x40000000, 0x40003fff, "ZERO_4000_0000_THRU_4000_3fff", 0),
+    RVI(0x80000000, 0x80000197, "ZERO_8000_0000_THRU_8000_0197", 0),
+    RVI(0x80000199, 0x80003fff, "ZERO_8000_0199_THRU_8000_3fff", 0),
+    RVI(0xc0000000, 0xc000007f, "ZERO_c000_0000_THRU_c000_007f", 0),
+    MFX(0xc0000080, "AMD64_EFER", Amd64Efer, Amd64Efer, 0xd01, 0x400, UINT64_C(0xffffffffffffd2fe)),
+    MFN(0xc0000081, "AMD64_STAR", Amd64SyscallTarget, Amd64SyscallTarget), /* value=0x230010`00000000 */
+    MFN(0xc0000082, "AMD64_STAR64", Amd64LongSyscallTarget, Amd64LongSyscallTarget), /* value=0xffffffff`81669af0 */
+    MFN(0xc0000083, "AMD64_STARCOMPAT", Amd64CompSyscallTarget, Amd64CompSyscallTarget), /* value=0xffffffff`8166c1d0 */
+    MFX(0xc0000084, "AMD64_SYSCALL_FLAG_MASK", Amd64SyscallFlagMask, Amd64SyscallFlagMask, 0, ~(uint64_t)UINT32_MAX, 0), /* value=0x3700 */
+    RVI(0xc0000085, 0xc00000ff, "ZERO_c000_0085_THRU_c000_00ff", 0),
+    MFN(0xc0000100, "AMD64_FS_BASE", Amd64FsBase, Amd64FsBase), /* value=0x7f91`10bdc740 */
+    MFN(0xc0000101, "AMD64_GS_BASE", Amd64GsBase, Amd64GsBase), /* value=0xffff8800`6fd80000 */
+    MFN(0xc0000102, "AMD64_KERNEL_GS_BASE", Amd64KernelGsBase, Amd64KernelGsBase), /* value=0x0 */
+    RVI(0xc0000104, 0xc0003fff, "ZERO_c000_0104_THRU_c000_3fff", 0),
+};
+#endif /* !CPUM_DB_STANDALONE */
+
+
+/**
+ * Database entry for ZHAOXIN KaiXian KX-U5581@1.8GHz.
+ */
+static CPUMDBENTRY const g_Entry_ZHAOXIN_KaiXian_KX_U5581_1_8GHz =
+{
+    /*.pszName          = */ "ZHAOXIN KaiXian KX-U5581 1.8GHz",
+    /*.pszFullName      = */ "ZHAOXIN KaiXian KX-U5581@1.8GHz",
+    /*.enmVendor        = */ CPUMCPUVENDOR_SHANGHAI,
+    /*.uFamily          = */ 7,
+    /*.uModel           = */ 11,
+    /*.uStepping        = */ 5,
+    /*.enmMicroarch     = */ kCpumMicroarch_Shanghai_Wudaokou,
+    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_UNKNOWN,
+    /*.fFlags           = */ 0,
+    /*.cMaxPhysAddrWidth= */ 40,
+    /*.fMxCsrMask       = */ 0x0000ffff,
+    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_ZHAOXIN_KaiXian_KX_U5581_1_8GHz),
+    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_ZHAOXIN_KaiXian_KX_U5581_1_8GHz)),
+    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_DEFAULTS,
+    /*.DefUnknownCpuId  = */ { 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    /*.fMsrMask         = */ UINT32_MAX /** @todo */,
+    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_ZHAOXIN_KaiXian_KX_U5581_1_8GHz)),
+    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_ZHAOXIN_KaiXian_KX_U5581_1_8GHz),
+};
+
+#endif /* !VBOX_CPUDB_ZHAOXIN_KaiXian_KX_U5581_1_8GHz_h */
+
diff --git a/src/VBox/VMM/VMMRZ/CPUMRZ.cpp b/src/VBox/VMM/VMMRZ/CPUMRZ.cpp
new file mode 100644
index 00000000..4e765df8
--- /dev/null
+++ b/src/VBox/VMM/VMMRZ/CPUMRZ.cpp
@@ -0,0 +1,144 @@
+/* $Id: CPUMRZ.cpp $ */
+/** @file
+ * CPUM - Raw-mode and ring-0 context.
+ */
+
+/*
+ * Copyright (C) 2016-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_CPUM
+#include <VBox/vmm/cpum.h>
+#include "CPUMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/vmm/hm.h>
+#include <iprt/assert.h>
+#include <iprt/x86.h>
+
+
+
+
+/**
+ * Prepares the host FPU/SSE/AVX stuff for IEM action.
+ *
+ * This will make sure the FPU/SSE/AVX guest state is _not_ loaded in the CPU.
+ * This will make sure the FPU/SSE/AVX host state is saved.
+ * Finally, it will make sure the FPU/SSE/AVX host features can be safely
+ * accessed.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMRZ_INT_DECL(void)    CPUMRZFpuStatePrepareHostCpuForUse(PVMCPUCC pVCpu)
+{
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_FPU_REM;
+    switch (pVCpu->cpum.s.fUseFlags & (CPUM_USED_FPU_GUEST | CPUM_USED_FPU_HOST))
+    {
+        case 0:
+            if (cpumRZSaveHostFPUState(&pVCpu->cpum.s) == VINF_CPUM_HOST_CR0_MODIFIED)
+                HMR0NotifyCpumModifiedHostCr0(pVCpu);
+            Log6(("CPUMRZFpuStatePrepareHostCpuForUse: #0 - %#x\n", ASMGetCR0()));
+            break;
+
+        case CPUM_USED_FPU_HOST:
+            Log6(("CPUMRZFpuStatePrepareHostCpuForUse: #1 - %#x\n", ASMGetCR0()));
+            break;
+
+        case CPUM_USED_FPU_GUEST | CPUM_USED_FPU_HOST:
+            cpumRZSaveGuestFpuState(&pVCpu->cpum.s, true /*fLeaveFpuAccessible*/);
+#ifdef IN_RING0
+            HMR0NotifyCpumUnloadedGuestFpuState(pVCpu);
+#endif
+            Log6(("CPUMRZFpuStatePrepareHostCpuForUse: #2 - %#x\n", ASMGetCR0()));
+            break;
+
+        default:
+            AssertFailed();
+    }
+
+}
+
+
+/**
+ * Makes sure the FPU/SSE/AVX guest state is saved in CPUMCPU::Guest and will be
+ * reloaded before direct use.
+ *
+ * No promisses about the FPU/SSE/AVX host features are made.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMRZ_INT_DECL(void)    CPUMRZFpuStateActualizeForChange(PVMCPUCC pVCpu)
+{
+    CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
+}
+
+
+/**
+ * Makes sure the FPU/SSE/AVX state in CPUMCPU::Guest is up to date.
+ *
+ * This will not cause CPUM_USED_FPU_GUEST to change.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMRZ_INT_DECL(void)    CPUMRZFpuStateActualizeForRead(PVMCPUCC pVCpu)
+{
+    if (pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST)
+    {
+        cpumRZSaveGuestFpuState(&pVCpu->cpum.s, false /*fLeaveFpuAccessible*/);
+        pVCpu->cpum.s.fUseFlags |= CPUM_USED_FPU_GUEST;
+        Log7(("CPUMRZFpuStateActualizeForRead\n"));
+    }
+}
+
+
+/**
+ * Makes sure the XMM0..XMM15 and MXCSR state in CPUMCPU::Guest is up to date.
+ *
+ * This will not cause CPUM_USED_FPU_GUEST to change.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMRZ_INT_DECL(void)    CPUMRZFpuStateActualizeSseForRead(PVMCPUCC pVCpu)
+{
+#if defined(VBOX_WITH_KERNEL_USING_XMM) && HC_ARCH_BITS == 64
+    NOREF(pVCpu);
+#else
+    if (pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST)
+    {
+        cpumRZSaveGuestSseRegisters(&pVCpu->cpum.s);
+        Log7(("CPUMRZFpuStateActualizeSseForRead\n"));
+    }
+#endif
+}
+
+
+/**
+ * Makes sure the YMM0..YMM15 and MXCSR state in CPUMCPU::Guest is up to date.
+ *
+ * This will not cause CPUM_USED_FPU_GUEST to change.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMRZ_INT_DECL(void)    CPUMRZFpuStateActualizeAvxForRead(PVMCPUCC pVCpu)
+{
+    if (pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST)
+    {
+        cpumRZSaveGuestAvxRegisters(&pVCpu->cpum.s);
+        Log7(("CPUMRZFpuStateActualizeAvxForRead\n"));
+    }
+}
+
diff --git a/src/VBox/VMM/VMMRZ/CPUMRZA.asm b/src/VBox/VMM/VMMRZ/CPUMRZA.asm
new file mode 100644
index 00000000..14309a9f
--- /dev/null
+++ b/src/VBox/VMM/VMMRZ/CPUMRZA.asm
@@ -0,0 +1,384 @@
+ ; $Id: CPUMRZA.asm $
+;; @file
+; CPUM - Raw-mode and Ring-0 Context Assembly Routines.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+
+;*******************************************************************************
+;* Header Files                                                                *
+;*******************************************************************************
+%define RT_ASM_WITH_SEH64
+%include "VBox/asmdefs.mac"
+%include "CPUMInternal.mac"
+%include "iprt/x86.mac"
+%include "VBox/vmm/cpum.mac"
+%include "VBox/err.mac"
+
+
+
+BEGINCODE
+
+
+;;
+; Saves the host FPU/SSE/AVX state.
+;
+; Will return with CR0.EM and CR0.TS cleared!  This is the normal state in ring-0.
+;
+; @returns  VINF_SUCCESS (0) or VINF_CPUM_HOST_CR0_MODIFIED. (EAX)
+; @param    pCpumCpu  x86:[ebp+8] gcc:rdi msc:rcx     CPUMCPU pointer
+;
+align 16
+BEGINPROC cpumRZSaveHostFPUState
+        push    xBP
+        SEH64_PUSH_xBP
+        mov     xBP, xSP
+        SEH64_SET_FRAME_xBP 0
+SEH64_END_PROLOGUE
+
+        ;
+        ; Prologue - xAX+xDX must be free for XSAVE/XRSTOR input.
+        ;
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_MSC
+        mov     r11, rcx
+ %else
+        mov     r11, rdi
+ %endif
+ %define pCpumCpu   r11
+ %define pXState    r10
+%else
+        push    ebx
+        push    esi
+        mov     ebx, dword [ebp + 8]
+ %define pCpumCpu ebx
+ %define pXState  esi
+%endif
+
+        pushf                           ; The darwin kernel can get upset or upset things if an
+        cli                             ; interrupt occurs while we're doing fxsave/fxrstor/cr0.
+
+        ;
+        ; We may have to update CR0, indirectly or directly.  We must report any
+        ; changes to the VT-x code.
+        ;
+        CPUMRZ_TOUCH_FPU_CLEAR_CR0_FPU_TRAPS_SET_RC xCX, xAX, pCpumCpu ; xCX is the return value (xAX scratch)
+
+        ;
+        ; Save the host state (xsave/fxsave will cause thread FPU state to be
+        ; loaded on systems where we are allowed to use it in ring-0.
+        ;
+        CPUMR0_SAVE_HOST
+
+        or      dword [pCpumCpu + CPUMCPU.fUseFlags], (CPUM_USED_FPU_HOST | CPUM_USED_FPU_SINCE_REM) ; Latter is not necessarily true, but normally yes.
+        popf
+
+        mov     eax, ecx                ; The return value from above.
+%ifdef RT_ARCH_X86
+        pop     esi
+        pop     ebx
+%endif
+        leave
+        ret
+%undef pCpumCpu
+%undef pXState
+ENDPROC   cpumRZSaveHostFPUState
+
+
+;;
+; Saves the guest FPU/SSE/AVX state.
+;
+; @param    pCpumCpu  x86:[ebp+8] gcc:rdi msc:rcx     CPUMCPU pointer
+; @param    fLeaveFpuAccessible  x86:[ebp+c] gcc:sil msc:dl      Whether to restore CR0 and XCR0 on
+;                                                                the way out. Only really applicable to RC.
+;
+; @remarks  64-bit Windows drivers shouldn't use AVX registers without saving+loading:
+;               https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
+;           However the compiler docs have different idea:
+;               https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
+;           We'll go with the former for now.
+;
+align 16
+BEGINPROC cpumRZSaveGuestFpuState
+        push    xBP
+        SEH64_PUSH_xBP
+        mov     xBP, xSP
+        SEH64_SET_FRAME_xBP 0
+SEH64_END_PROLOGUE
+
+        ;
+        ; Prologue - xAX+xDX must be free for XSAVE/XRSTOR input.
+        ;
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_MSC
+        mov     r11, rcx
+ %else
+        mov     r11, rdi
+ %endif
+ %define pCpumCpu   r11
+ %define pXState    r10
+%else
+        push    ebx
+        push    esi
+        mov     ebx, dword [ebp + 8]
+ %define pCpumCpu   ebx
+ %define pXState    esi
+%endif
+        pushf                           ; The darwin kernel can get upset or upset things if an
+        cli                             ; interrupt occurs while we're doing fxsave/fxrstor/cr0.
+
+ %ifdef IN_RC
+        mov     ecx, cr0                ; ecx = saved cr0
+        test    ecx, X86_CR0_TS | X86_CR0_EM
+        jz      .skip_cr0_write
+        mov     eax, ecx
+        and     eax, ~(X86_CR0_TS | X86_CR0_EM)
+        mov     cr0, eax
+.skip_cr0_write:
+ %endif
+
+ %ifndef VBOX_WITH_KERNEL_USING_XMM
+        CPUMR0_SAVE_GUEST
+ %else
+        ;
+        ; The XMM0..XMM15 registers have been saved already.  We exploit the
+        ; host state here to temporarly save the non-volatile XMM registers,
+        ; so we can load the guest ones while saving.  This is safe.
+        ;
+
+        ; Save caller's XMM registers.
+        mov     pXState, [pCpumCpu + CPUMCPU.Host.pXStateR0]
+        movdqa  [pXState + X86FXSTATE.xmm6 ], xmm6
+        movdqa  [pXState + X86FXSTATE.xmm7 ], xmm7
+        movdqa  [pXState + X86FXSTATE.xmm8 ], xmm8
+        movdqa  [pXState + X86FXSTATE.xmm9 ], xmm9
+        movdqa  [pXState + X86FXSTATE.xmm10], xmm10
+        movdqa  [pXState + X86FXSTATE.xmm11], xmm11
+        movdqa  [pXState + X86FXSTATE.xmm12], xmm12
+        movdqa  [pXState + X86FXSTATE.xmm13], xmm13
+        movdqa  [pXState + X86FXSTATE.xmm14], xmm14
+        movdqa  [pXState + X86FXSTATE.xmm15], xmm15
+        stmxcsr [pXState + X86FXSTATE.MXCSR]
+
+        ; Load the guest XMM register values we already saved in HMR0VMXStartVMWrapXMM.
+        mov     pXState, [pCpumCpu + CPUMCPU.Guest.pXStateR0]
+        movdqa  xmm0,  [pXState + X86FXSTATE.xmm0]
+        movdqa  xmm1,  [pXState + X86FXSTATE.xmm1]
+        movdqa  xmm2,  [pXState + X86FXSTATE.xmm2]
+        movdqa  xmm3,  [pXState + X86FXSTATE.xmm3]
+        movdqa  xmm4,  [pXState + X86FXSTATE.xmm4]
+        movdqa  xmm5,  [pXState + X86FXSTATE.xmm5]
+        movdqa  xmm6,  [pXState + X86FXSTATE.xmm6]
+        movdqa  xmm7,  [pXState + X86FXSTATE.xmm7]
+        movdqa  xmm8,  [pXState + X86FXSTATE.xmm8]
+        movdqa  xmm9,  [pXState + X86FXSTATE.xmm9]
+        movdqa  xmm10, [pXState + X86FXSTATE.xmm10]
+        movdqa  xmm11, [pXState + X86FXSTATE.xmm11]
+        movdqa  xmm12, [pXState + X86FXSTATE.xmm12]
+        movdqa  xmm13, [pXState + X86FXSTATE.xmm13]
+        movdqa  xmm14, [pXState + X86FXSTATE.xmm14]
+        movdqa  xmm15, [pXState + X86FXSTATE.xmm15]
+        ldmxcsr        [pXState + X86FXSTATE.MXCSR]
+
+        CPUMR0_SAVE_GUEST
+
+        ; Restore caller's XMM registers.
+        mov     pXState, [pCpumCpu + CPUMCPU.Host.pXStateR0]
+        movdqa  xmm6,  [pXState + X86FXSTATE.xmm6 ]
+        movdqa  xmm7,  [pXState + X86FXSTATE.xmm7 ]
+        movdqa  xmm8,  [pXState + X86FXSTATE.xmm8 ]
+        movdqa  xmm9,  [pXState + X86FXSTATE.xmm9 ]
+        movdqa  xmm10, [pXState + X86FXSTATE.xmm10]
+        movdqa  xmm11, [pXState + X86FXSTATE.xmm11]
+        movdqa  xmm12, [pXState + X86FXSTATE.xmm12]
+        movdqa  xmm13, [pXState + X86FXSTATE.xmm13]
+        movdqa  xmm14, [pXState + X86FXSTATE.xmm14]
+        movdqa  xmm15, [pXState + X86FXSTATE.xmm15]
+        ldmxcsr        [pXState + X86FXSTATE.MXCSR]
+
+ %endif
+
+        and     dword [pCpumCpu + CPUMCPU.fUseFlags], ~CPUM_USED_FPU_GUEST
+ %ifdef IN_RC
+        test    byte [ebp + 0ch], 1     ; fLeaveFpuAccessible
+        jz      .no_cr0_restore
+        CPUMRZ_RESTORE_CR0_IF_TS_OR_EM_SET ecx
+.no_cr0_restore:
+ %endif
+        popf
+%ifdef RT_ARCH_X86
+        pop     esi
+        pop     ebx
+%endif
+        leave
+        ret
+%undef pCpumCpu
+%undef pXState
+ENDPROC   cpumRZSaveGuestFpuState
+
+
+;;
+; Saves the guest XMM0..15 registers and MXCSR.
+;
+; The purpose is to actualize the register state for read-only use, so CR0 is
+; restored in raw-mode context (so, the FPU/SSE/AVX CPU features can be
+; inaccessible upon return).
+;
+; @param    pCpumCpu  x86:[ebp+8] gcc:rdi msc:rcx     CPUMCPU pointer
+;
+align 16
+BEGINPROC cpumRZSaveGuestSseRegisters
+        push    xBP
+        SEH64_PUSH_xBP
+        mov     xBP, xSP
+        SEH64_SET_FRAME_xBP 0
+SEH64_END_PROLOGUE
+
+%ifndef VBOX_WITH_KERNEL_USING_XMM
+        ;
+        ; Load xCX with the guest pXStateR0.
+        ;
+ %ifdef ASM_CALL64_GCC
+        mov     xCX, rdi
+ %elifdef RT_ARCH_X86
+        mov     xCX, dword [ebp + 8]
+ %endif
+ %ifdef IN_RING0
+        mov     xCX, [xCX + CPUMCPU.Guest.pXStateR0]
+ %elifdef IN_RC
+        mov     xCX, [xCX + CPUMCPU.Guest.pXStateRC]
+ %else
+  %error "Invalid context!"
+ %endif
+
+ %ifdef IN_RC
+        ; Temporarily grant access to the SSE state. xDX must be preserved until CR0 is restored!
+        mov     edx, cr0
+        test    edx, X86_CR0_TS | X86_CR0_EM
+        jz      .skip_cr0_write
+        mov     eax, edx
+        and     eax, ~(X86_CR0_TS | X86_CR0_EM)
+        mov     cr0, eax
+.skip_cr0_write:
+ %endif
+
+        ;
+        ; Do the job.
+        ;
+        stmxcsr [xCX + X86FXSTATE.MXCSR]
+        movdqa  [xCX + X86FXSTATE.xmm0 ], xmm0
+        movdqa  [xCX + X86FXSTATE.xmm1 ], xmm1
+        movdqa  [xCX + X86FXSTATE.xmm2 ], xmm2
+        movdqa  [xCX + X86FXSTATE.xmm3 ], xmm3
+        movdqa  [xCX + X86FXSTATE.xmm4 ], xmm4
+        movdqa  [xCX + X86FXSTATE.xmm5 ], xmm5
+        movdqa  [xCX + X86FXSTATE.xmm6 ], xmm6
+        movdqa  [xCX + X86FXSTATE.xmm7 ], xmm7
+ %if ARCH_BITS == 64
+        movdqa  [xCX + X86FXSTATE.xmm8 ], xmm8
+        movdqa  [xCX + X86FXSTATE.xmm9 ], xmm9
+        movdqa  [xCX + X86FXSTATE.xmm10], xmm10
+        movdqa  [xCX + X86FXSTATE.xmm11], xmm11
+        movdqa  [xCX + X86FXSTATE.xmm12], xmm12
+        movdqa  [xCX + X86FXSTATE.xmm13], xmm13
+        movdqa  [xCX + X86FXSTATE.xmm14], xmm14
+        movdqa  [xCX + X86FXSTATE.xmm15], xmm15
+ %endif
+
+ %ifdef IN_RC
+        CPUMRZ_RESTORE_CR0_IF_TS_OR_EM_SET edx  ; Restore CR0 if we changed it above.
+ %endif
+
+%endif ; !VBOX_WITH_KERNEL_USING_XMM
+
+        leave
+        ret
+ENDPROC   cpumRZSaveGuestSseRegisters
+
+;;
+; Saves the guest YMM0..15 registers.
+;
+; The purpose is to actualize the register state for read-only use, so CR0 is
+; restored in raw-mode context (so, the FPU/SSE/AVX CPU features can be
+; inaccessible upon return).
+;
+; @param    pCpumCpu  x86:[ebp+8] gcc:rdi msc:rcx     CPUMCPU pointer
+;
+align 16
+BEGINPROC cpumRZSaveGuestAvxRegisters
+        push    xBP
+        SEH64_PUSH_xBP
+        mov     xBP, xSP
+        SEH64_SET_FRAME_xBP 0
+%ifdef IN_RC
+        push    xBX
+%endif
+SEH64_END_PROLOGUE
+
+        ;
+        ; Load xCX with the guest pXStateR0.
+        ;
+%ifdef ASM_CALL64_GCC
+        mov     xCX, rdi
+%elifdef RT_ARCH_X86
+        mov     xCX, dword [ebp + 8]
+%endif
+%ifdef IN_RING0
+        mov     xCX, [xCX + CPUMCPU.Guest.pXStateR0]
+%elifdef IN_RC
+        mov     xCX, [xCX + CPUMCPU.Guest.pXStateRC]
+%else
+ %error "Invalid context!"
+%endif
+
+%ifdef IN_RC
+        ; Temporarily grant access to the SSE state. xBX must be preserved until CR0 is restored!
+        mov     ebx, cr0
+        test    ebx, X86_CR0_TS | X86_CR0_EM
+        jz      .skip_cr0_write
+        mov     eax, ebx
+        and     eax, ~(X86_CR0_TS | X86_CR0_EM)
+        mov     cr0, eax
+.skip_cr0_write:
+%endif
+
+        ;
+        ; Use XSAVE to do the job.
+        ;
+        ; Drivers shouldn't use AVX registers without saving+loading:
+        ;     https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
+        ; However the compiler docs have different idea:
+        ;     https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
+        ; We'll go with the former for now.
+        ;
+%ifdef VBOX_WITH_KERNEL_USING_XMM
+        mov     eax, XSAVE_C_YMM
+%else
+        mov     eax, XSAVE_C_YMM | XSAVE_C_SSE ; The SSE component includes MXCSR.
+%endif
+        xor     edx, edx
+%if ARCH_BITS == 64
+        o64 xsave [xCX]
+%else
+        xsave   [xCX]
+%endif
+
+%ifdef IN_RC
+        CPUMRZ_RESTORE_CR0_IF_TS_OR_EM_SET ebx  ; Restore CR0 if we changed it above.
+        pop     xBX
+%endif
+        leave
+        ret
+ENDPROC   cpumRZSaveGuestAvxRegisters
+
diff --git a/src/VBox/VMM/VMMRZ/DBGFRZ.cpp b/src/VBox/VMM/VMMRZ/DBGFRZ.cpp
new file mode 100644
index 00000000..44066fd5
--- /dev/null
+++ b/src/VBox/VMM/VMMRZ/DBGFRZ.cpp
@@ -0,0 +1,240 @@
+/* $Id: DBGFRZ.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, RZ part.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/selm.h>
+#ifdef IN_RC
+# include <VBox/vmm/trpm.h>
+#endif
+#include <VBox/log.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+
+#ifdef IN_RC
+DECLASM(void) TRPMRCHandlerAsmTrap03(void);
+#endif
+
+
+/**
+ * \#DB (Debug event) handler.
+ *
+ * @returns VBox status code.
+ *          VINF_SUCCESS means we completely handled this trap,
+ *          other codes are passed execution to host context.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pRegFrame       Pointer to the register frame for the trap.
+ * @param   uDr6            The DR6 hypervisor register value.
+ * @param   fAltStepping    Alternative stepping indicator.
+ */
+VMMRZ_INT_DECL(int) DBGFRZTrap01Handler(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, RTGCUINTREG uDr6, bool fAltStepping)
+{
+#ifdef IN_RC
+    const bool fInHyper = !(pRegFrame->ss.Sel & X86_SEL_RPL) && !pRegFrame->eflags.Bits.u1VM;
+#else
+    NOREF(pRegFrame);
+    const bool fInHyper = false;
+#endif
+
+    /** @todo Intel docs say that X86_DR6_BS has the highest priority... */
+    /*
+     * A breakpoint?
+     */
+    AssertCompile(X86_DR6_B0 == 1 && X86_DR6_B1 == 2 && X86_DR6_B2 == 4 && X86_DR6_B3 == 8);
+    if (   (uDr6 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3))
+        && pVM->dbgf.s.cEnabledHwBreakpoints > 0)
+    {
+        for (unsigned iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); iBp++)
+        {
+            if (    ((uint32_t)uDr6 & RT_BIT_32(iBp))
+                &&  pVM->dbgf.s.aHwBreakpoints[iBp].enmType == DBGFBPTYPE_REG)
+            {
+                pVCpu->dbgf.s.iActiveBp = pVM->dbgf.s.aHwBreakpoints[iBp].iBp;
+                pVCpu->dbgf.s.fSingleSteppingRaw = false;
+                LogFlow(("DBGFRZTrap03Handler: hit hw breakpoint %d at %04x:%RGv\n",
+                         pVM->dbgf.s.aHwBreakpoints[iBp].iBp, pRegFrame->cs.Sel, pRegFrame->rip));
+
+                return fInHyper ? VINF_EM_DBG_HYPER_BREAKPOINT : VINF_EM_DBG_BREAKPOINT;
+            }
+        }
+    }
+
+    /*
+     * Single step?
+     * Are we single stepping or is it the guest?
+     */
+    if (    (uDr6 & X86_DR6_BS)
+        &&  (fInHyper || pVCpu->dbgf.s.fSingleSteppingRaw || fAltStepping))
+    {
+        pVCpu->dbgf.s.fSingleSteppingRaw = false;
+        LogFlow(("DBGFRZTrap01Handler: single step at %04x:%RGv\n", pRegFrame->cs.Sel, pRegFrame->rip));
+        return fInHyper ? VINF_EM_DBG_HYPER_STEPPED : VINF_EM_DBG_STEPPED;
+    }
+
+#ifdef IN_RC
+    /*
+     * Either an ICEBP in hypervisor code or a guest related debug exception
+     * of sorts.
+     */
+    if (RT_UNLIKELY(fInHyper))
+    {
+        /*
+         * Is this a guest debug event that was delayed past a ring transition?
+         *
+         * Since we do no allow sysenter/syscall in raw-mode, the  only
+         * non-trap/fault type transitions that can occur are thru interrupt gates.
+         * Of those, only INT3 (#BP) has a DPL other than 0 with a CS.RPL of 0.
+         * See bugref:9171 and bs3-cpu-weird-1 for more details.
+         *
+         * We need to reconstruct the guest register state from the hypervisor one
+         * here, so here is the layout of the IRET frame on the stack:
+         *    20:[8] GS          (V86 only)
+         *    1C:[7] FS          (V86 only)
+         *    18:[6] DS          (V86 only)
+         *    14:[5] ES          (V86 only)
+         *    10:[4] SS
+         *    0c:[3] ESP
+         *    08:[2] EFLAGS
+         *    04:[1] CS
+         *    00:[0] EIP
+         */
+        if (pRegFrame->rip == (uintptr_t)TRPMRCHandlerAsmTrap03)
+        {
+            uint32_t const *pu32Stack = (uint32_t const *)pRegFrame->esp;
+            if (   (pu32Stack[2] & X86_EFL_VM)
+                || (pu32Stack[1] & X86_SEL_RPL))
+            {
+                LogFlow(("DBGFRZTrap01Handler: Detected guest #DB delayed past ring transition %04x:%RX32 %#x\n",
+                         pu32Stack[1] & 0xffff, pu32Stack[0], pu32Stack[2]));
+                PCPUMCTX pGstCtx = CPUMQueryGuestCtxPtr(pVCpu);
+                pGstCtx->rip      = pu32Stack[0];
+                pGstCtx->cs.Sel   = pu32Stack[1];
+                pGstCtx->eflags.u = pu32Stack[2];
+                pGstCtx->rsp      = pu32Stack[3];
+                pGstCtx->ss.Sel   = pu32Stack[4];
+                if (pu32Stack[2] & X86_EFL_VM)
+                {
+                    pGstCtx->es.Sel = pu32Stack[5];
+                    pGstCtx->ds.Sel = pu32Stack[6];
+                    pGstCtx->fs.Sel = pu32Stack[7];
+                    pGstCtx->gs.Sel = pu32Stack[8];
+                }
+                else
+                {
+                    pGstCtx->es.Sel = pRegFrame->es.Sel;
+                    pGstCtx->ds.Sel = pRegFrame->ds.Sel;
+                    pGstCtx->fs.Sel = pRegFrame->fs.Sel;
+                    pGstCtx->gs.Sel = pRegFrame->gs.Sel;
+                }
+                pGstCtx->rax      = pRegFrame->rax;
+                pGstCtx->rcx      = pRegFrame->rcx;
+                pGstCtx->rdx      = pRegFrame->rdx;
+                pGstCtx->rbx      = pRegFrame->rbx;
+                pGstCtx->rsi      = pRegFrame->rsi;
+                pGstCtx->rdi      = pRegFrame->rdi;
+                pGstCtx->rbp      = pRegFrame->rbp;
+
+                /*
+                 * We should assert a #BP followed by a #DB here, but TRPM cannot
+                 * do that.  So, we'll just assert the #BP and ignore the #DB, even
+                 * if that isn't strictly correct.
+                 */
+                TRPMResetTrap(pVCpu);
+                TRPMAssertTrap(pVCpu, X86_XCPT_BP, TRPM_SOFTWARE_INT);
+                return VINF_EM_RAW_GUEST_TRAP;
+            }
+        }
+
+        LogFlow(("DBGFRZTrap01Handler: Unknown bp at %04x:%RGv\n", pRegFrame->cs.Sel, pRegFrame->rip));
+        return VERR_DBGF_HYPER_DB_XCPT;
+    }
+#endif
+
+    LogFlow(("DBGFRZTrap01Handler: guest debug event %#x at %04x:%RGv!\n", (uint32_t)uDr6, pRegFrame->cs.Sel, pRegFrame->rip));
+    return VINF_EM_RAW_GUEST_TRAP;
+}
+
+
+/**
+ * \#BP (Breakpoint) handler.
+ *
+ * @returns VBox status code.
+ *          VINF_SUCCESS means we completely handled this trap,
+ *          other codes are passed execution to host context.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pRegFrame   Pointer to the register frame for the trap.
+ */
+VMMRZ_INT_DECL(int) DBGFRZTrap03Handler(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame)
+{
+#ifdef IN_RC
+    const bool fInHyper = !(pRegFrame->ss.Sel & X86_SEL_RPL) && !pRegFrame->eflags.Bits.u1VM;
+#else
+    const bool fInHyper = false;
+#endif
+
+    /*
+     * Get the trap address and look it up in the breakpoint table.
+     * Don't bother if we don't have any breakpoints.
+     */
+    unsigned cToSearch = pVM->dbgf.s.Int3.cToSearch;
+    if (cToSearch > 0)
+    {
+        RTGCPTR pPc;
+        int rc = SELMValidateAndConvertCSAddr(pVCpu, pRegFrame->eflags, pRegFrame->ss.Sel, pRegFrame->cs.Sel, &pRegFrame->cs,
+#ifdef IN_RC
+                                              pRegFrame->eip - 1,
+#else
+                                              pRegFrame->rip /* no -1 in R0 */,
+#endif
+                                              &pPc);
+        AssertRCReturn(rc, rc);
+
+        unsigned iBp = pVM->dbgf.s.Int3.iStartSearch;
+        while (cToSearch-- > 0)
+        {
+            if (   pVM->dbgf.s.aBreakpoints[iBp].u.GCPtr == (RTGCUINTPTR)pPc
+                && pVM->dbgf.s.aBreakpoints[iBp].enmType == DBGFBPTYPE_INT3)
+            {
+                pVM->dbgf.s.aBreakpoints[iBp].cHits++;
+                pVCpu->dbgf.s.iActiveBp = pVM->dbgf.s.aBreakpoints[iBp].iBp;
+
+                LogFlow(("DBGFRZTrap03Handler: hit breakpoint %d at %RGv (%04x:%RGv) cHits=0x%RX64\n",
+                         pVM->dbgf.s.aBreakpoints[iBp].iBp, pPc, pRegFrame->cs.Sel, pRegFrame->rip,
+                         pVM->dbgf.s.aBreakpoints[iBp].cHits));
+                return fInHyper
+                     ? VINF_EM_DBG_HYPER_BREAKPOINT
+                     : VINF_EM_DBG_BREAKPOINT;
+            }
+            iBp++;
+        }
+    }
+
+    return fInHyper
+         ? VINF_EM_DBG_HYPER_ASSERTION
+         : VINF_EM_RAW_GUEST_TRAP;
+}
+
diff --git a/src/VBox/VMM/VMMRZ/Makefile.kup b/src/VBox/VMM/VMMRZ/Makefile.kup
new file mode 100644
index 00000000..e69de29b
diff --git a/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp b/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp
new file mode 100644
index 00000000..0f4313bb
--- /dev/null
+++ b/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp
@@ -0,0 +1,2692 @@
+/* $Id: PGMRZDynMap.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor, dynamic mapping cache.
+ */
+
+/*
+ * Copyright (C) 2008-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_DYNMAP
+#include <VBox/vmm/pgm.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vm.h>
+#include "PGMInline.h"
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/sup.h>
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/assert.h>
+#ifndef IN_RC
+# include <iprt/cpuset.h>
+# include <iprt/mem.h>
+# include <iprt/memobj.h>
+# include <iprt/mp.h>
+# include <iprt/semaphore.h>
+# include <iprt/spinlock.h>
+#endif
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#ifdef IN_RING0
+/** The max size of the mapping cache (in pages). */
+# define PGMR0DYNMAP_MAX_PAGES              ((16*_1M) >> PAGE_SHIFT)
+/** The small segment size that is adopted on out-of-memory conditions with a
+ * single big segment. */
+# define PGMR0DYNMAP_SMALL_SEG_PAGES        128
+/** The number of pages we reserve per CPU. */
+# define PGMR0DYNMAP_PAGES_PER_CPU          256
+/** The minimum number of pages we reserve per CPU.
+ * This must be equal or larger than the autoset size. */
+# define PGMR0DYNMAP_PAGES_PER_CPU_MIN      64
+/** Calcs the overload threshold (safety margin).  Current set at 50%. */
+# define PGMR0DYNMAP_CALC_OVERLOAD(cPages)  ((cPages) / 2)
+/** The number of guard pages.
+ * @remarks Never do tuning of the hashing or whatnot with a strict build! */
+# if defined(VBOX_STRICT)
+#  define PGMR0DYNMAP_GUARD_PAGES           1
+# else
+#  define PGMR0DYNMAP_GUARD_PAGES           0
+# endif
+#endif /* IN_RING0 */
+/** The dummy physical address of guard pages. */
+#define PGMR0DYNMAP_GUARD_PAGE_HCPHYS       UINT32_C(0x7777feed)
+/** The dummy reference count of guard pages. (Must be non-zero.) */
+#define PGMR0DYNMAP_GUARD_PAGE_REF_COUNT    INT32_C(0x7777feed)
+#if 0
+/** Define this to just clear the present bit on guard pages.
+ * The alternative is to replace the entire PTE with an bad not-present
+ * PTE. Either way, XNU will screw us. :-/ */
+# define PGMR0DYNMAP_GUARD_NP
+#endif
+/** The dummy PTE value for a page. */
+#define PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE   X86_PTE_PG_MASK
+/** The dummy PTE value for a page. */
+#define PGMR0DYNMAP_GUARD_PAGE_PAE_PTE      UINT64_MAX /*X86_PTE_PAE_PG_MASK*/
+
+#ifdef IN_RING0 /* Note! Assertions causes panics if preemption is disabled,
+                 *       disable this to work around that. */
+/**
+ * Acquire the spinlock.
+ * This will declare a temporary variable and expands to two statements!
+ */
+# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) \
+    RTSpinlockAcquire((pThis)->hSpinlock)
+
+/**
+ * Releases the spinlock.
+ */
+# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) \
+    RTSpinlockRelease((pThis)->hSpinlock)
+
+/**
+ * Re-acquires the spinlock.
+ */
+# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) \
+    RTSpinlockAcquire((pThis)->hSpinlock)
+#else
+# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis)   do { } while (0)
+# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis)   do { } while (0)
+# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) do { } while (0)
+#endif
+
+
+/** Converts a PGMCPUM::AutoSet pointer into a PVMCPU. */
+#define PGMRZDYNMAP_SET_2_VMCPU(pSet)       (RT_FROM_MEMBER(pSet, VMCPU, pgm.s.AutoSet))
+
+/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
+#define PGMRZDYNMAP_SET_2_VM(pSet)          (PGMRZDYNMAP_SET_2_VMCPU(pSet)->CTX_SUFF(pVM))
+
+/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
+#ifdef IN_RC
+# define PGMRZDYNMAP_SET_2_DYNMAP(pSet)     (PGMRZDYNMAP_SET_2_VM(pSet)->pgm.s.pRCDynMap)
+#else
+# define PGMRZDYNMAP_SET_2_DYNMAP(pSet)     (g_pPGMR0DynMap)
+#endif
+
+/**
+ * Gets the set index of the current CPU.
+ *
+ * This always returns 0 when in raw-mode context because there is only ever
+ * one EMT in that context (at least presently).
+ */
+#ifdef IN_RC
+# define PGMRZDYNMAP_CUR_CPU()              (0)
+#else
+# define PGMRZDYNMAP_CUR_CPU()              RTMpCurSetIndex()
+#endif
+
+/** PGMRZDYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
+#define PGMRZDYNMAP_MAGIC                   UINT32_C(0x19640201)
+
+
+/** Zaps an set entry. */
+#define PGMRZDYNMAP_ZAP_ENTRY(pEntry) \
+    do \
+    { \
+        (pEntry)->iPage        = UINT16_MAX; \
+        (pEntry)->cRefs        = 0; \
+        (pEntry)->cInlinedRefs = 0; \
+        (pEntry)->cUnrefs      = 0; \
+    } while (0)
+
+
+/** @def PGMRZDYNMAP_STRICT_RELEASE
+ * Define this to force pages to be released and make non-present ASAP after
+ * use.  This should not normally be enabled as it is a bit expensive. */
+#if 0 || defined(DOXYGEN_RUNNING)
+# define PGMRZDYNMAP_STRICT_RELEASE
+#endif
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+#ifdef IN_RING0
+/**
+ * Ring-0 dynamic mapping cache segment.
+ *
+ * The dynamic mapping cache can be extended with additional segments if the
+ * load is found to be too high.  This done the next time a VM is created, under
+ * the protection of the init mutex.  The arrays is reallocated and the new
+ * segment is added to the end of these.  Nothing is rehashed of course, as the
+ * indexes / addresses must remain unchanged.
+ *
+ * This structure is only modified while owning the init mutex or during module
+ * init / term.
+ */
+typedef struct PGMR0DYNMAPSEG
+{
+    /** Pointer to the next segment. */
+    struct PGMR0DYNMAPSEG      *pNext;
+    /** The memory object for the virtual address range that we're abusing. */
+    RTR0MEMOBJ                  hMemObj;
+    /** The start page in the cache. (I.e. index into the arrays.) */
+    uint16_t                    iPage;
+    /** The number of pages this segment contributes. */
+    uint16_t                    cPages;
+    /** The number of page tables. */
+    uint16_t                    cPTs;
+    /** The memory objects for the page tables. */
+    RTR0MEMOBJ                  ahMemObjPTs[1];
+} PGMR0DYNMAPSEG;
+/** Pointer to a ring-0 dynamic mapping cache segment. */
+typedef PGMR0DYNMAPSEG *PPGMR0DYNMAPSEG;
+
+
+/**
+ * Ring-0 dynamic mapping cache entry.
+ *
+ * @sa PGMRZDYNMAPENTRY, PGMRCDYNMAPENTRY.
+ */
+typedef struct PGMR0DYNMAPENTRY
+{
+    /** The physical address of the currently mapped page.
+     * This is duplicate for three reasons: cache locality, cache policy of the PT
+     * mappings and sanity checks. */
+    RTHCPHYS                    HCPhys;
+    /** Pointer to the page. */
+    void                       *pvPage;
+    /** The number of references. */
+    int32_t volatile            cRefs;
+    /** PTE pointer union. */
+    union PGMR0DYNMAPENTRY_PPTE
+    {
+        /** PTE pointer, 32-bit legacy version. */
+        PX86PTE                 pLegacy;
+        /** PTE pointer, PAE version. */
+        PX86PTEPAE              pPae;
+        /** PTE pointer, the void version. */
+        void                   *pv;
+    } uPte;
+    /** CPUs that haven't invalidated this entry after it's last update. */
+    RTCPUSET                    PendingSet;
+} PGMR0DYNMAPENTRY;
+/** Pointer a mapping cache entry for the ring-0.
+ * @sa PPGMRZDYNMAPENTRY, PPGMRCDYNMAPENTRY, */
+typedef PGMR0DYNMAPENTRY *PPGMR0DYNMAPENTRY;
+
+
+/**
+ * Dynamic mapping cache for ring-0.
+ *
+ * This is initialized during VMMR0 module init but no segments are allocated
+ * at that time.  Segments will be added when the first VM is started and
+ * removed again when the last VM shuts down, thus avoid consuming memory while
+ * dormant. At module termination, the remaining bits will be freed up.
+ *
+ * @sa PPGMRZDYNMAP, PGMRCDYNMAP.
+ */
+typedef struct PGMR0DYNMAP
+{
+    /** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */
+    uint32_t                    u32Magic;
+    /** Spinlock serializing the normal operation of the cache. */
+    RTSPINLOCK                  hSpinlock;
+    /** Array for tracking and managing the pages. */
+    PPGMR0DYNMAPENTRY           paPages;
+    /** The cache size given as a number of pages. */
+    uint32_t                    cPages;
+    /** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
+    bool                        fLegacyMode;
+    /** The current load.
+     * This does not include guard pages. */
+    uint32_t                    cLoad;
+    /** The max load ever.
+     * This is maintained to trigger the adding of more mapping space. */
+    uint32_t                    cMaxLoad;
+    /** Initialization / termination lock. */
+    RTSEMFASTMUTEX              hInitLock;
+    /** The number of guard pages. */
+    uint32_t                    cGuardPages;
+    /** The number of users (protected by hInitLock). */
+    uint32_t                    cUsers;
+    /** Array containing a copy of the original page tables.
+     * The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */
+    void                       *pvSavedPTEs;
+    /** List of segments. */
+    PPGMR0DYNMAPSEG             pSegHead;
+    /** The paging mode. */
+    SUPPAGINGMODE               enmPgMode;
+} PGMR0DYNMAP;
+
+
+/**
+ * Paging level data.
+ */
+typedef struct PGMR0DYNMAPPGLVL
+{
+    uint32_t            cLevels;    /**< The number of levels. */
+    struct
+    {
+        RTHCPHYS        HCPhys;     /**< The address of the page for the current level,
+                                     *  i.e. what hMemObj/hMapObj is currently mapping. */
+        RTHCPHYS        fPhysMask;  /**< Mask for extracting HCPhys from uEntry. */
+        RTR0MEMOBJ      hMemObj;    /**< Memory object for HCPhys, PAGE_SIZE. */
+        RTR0MEMOBJ      hMapObj;    /**< Mapping object for hMemObj. */
+        uint32_t        fPtrShift;  /**< The pointer shift count. */
+        uint64_t        fPtrMask;   /**< The mask to apply to the shifted pointer to get the table index. */
+        uint64_t        fAndMask;   /**< And mask to check entry flags. */
+        uint64_t        fResMask;   /**< The result from applying fAndMask. */
+        union
+        {
+            void        *pv;        /**< hMapObj address. */
+            PX86PGUINT   paLegacy;  /**< Legacy table view. */
+            PX86PGPAEUINT paPae;    /**< PAE/AMD64 table view. */
+        } u;
+    } a[4];
+} PGMR0DYNMAPPGLVL;
+/** Pointer to paging level data. */
+typedef PGMR0DYNMAPPGLVL *PPGMR0DYNMAPPGLVL;
+#endif
+
+/** Mapping cache entry for the current context.
+ * @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */
+typedef CTX_MID(PGM,DYNMAPENTRY) PGMRZDYNMAPENTRY;
+/** Pointer a mapping cache entry for the current context.
+ * @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */
+typedef PGMRZDYNMAPENTRY *PPGMRZDYNMAPENTRY;
+
+/** Pointer to the mapping cache instance for the current context.
+ * @sa PGMR0DYNMAP, PGMRCDYNMAP */
+typedef CTX_MID(PGM,DYNMAP) *PPGMRZDYNMAP;
+
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#ifdef IN_RING0
+/** Pointer to the ring-0 dynamic mapping cache. */
+static PGMR0DYNMAP *g_pPGMR0DynMap;
+#endif
+/** For overflow testing. */
+static bool         g_fPGMR0DynMapTestRunning = false;
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs);
+#ifdef IN_RING0
+static int  pgmR0DynMapSetup(PPGMRZDYNMAP pThis);
+static int  pgmR0DynMapExpand(PPGMRZDYNMAP pThis);
+static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis);
+#endif
+#if 0 /*def DEBUG*/
+static int  pgmR0DynMapTest(PVM pVM);
+#endif
+
+
+/**
+ * Initializes the auto mapping sets for a VM.
+ *
+ * @returns VINF_SUCCESS on success, VERR_PGM_DYNMAP_IPE on failure.
+ * @param   pVM         The cross context VM structure.
+ */
+static int pgmRZDynMapInitAutoSetsForVM(PVM pVM)
+{
+    VMCPUID idCpu = pVM->cCpus;
+    AssertReturn(idCpu > 0 && idCpu <= VMM_MAX_CPU_COUNT, VERR_PGM_DYNMAP_IPE);
+    while (idCpu-- > 0)
+    {
+        PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
+        uint32_t j = RT_ELEMENTS(pSet->aEntries);
+        while (j-- > 0)
+        {
+            pSet->aEntries[j].pvPage        = NULL;
+            pSet->aEntries[j].HCPhys        = NIL_RTHCPHYS;
+            PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
+        }
+        pSet->cEntries = PGMMAPSET_CLOSED;
+        pSet->iSubset = UINT32_MAX;
+        pSet->iCpu = -1;
+        memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+#ifdef IN_RING0
+
+/**
+ * Initializes the ring-0 dynamic mapping cache.
+ *
+ * @returns VBox status code.
+ */
+VMMR0DECL(int) PGMR0DynMapInit(void)
+{
+    Assert(!g_pPGMR0DynMap);
+
+    /*
+     * Create and initialize the cache instance.
+     */
+    PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)RTMemAllocZ(sizeof(*pThis));
+    AssertLogRelReturn(pThis, VERR_NO_MEMORY);
+    int             rc = VINF_SUCCESS;
+    pThis->enmPgMode = SUPR0GetPagingMode();
+    switch (pThis->enmPgMode)
+    {
+        case SUPPAGINGMODE_32_BIT:
+        case SUPPAGINGMODE_32_BIT_GLOBAL:
+            pThis->fLegacyMode = false;
+            break;
+        case SUPPAGINGMODE_PAE:
+        case SUPPAGINGMODE_PAE_GLOBAL:
+        case SUPPAGINGMODE_PAE_NX:
+        case SUPPAGINGMODE_PAE_GLOBAL_NX:
+        case SUPPAGINGMODE_AMD64:
+        case SUPPAGINGMODE_AMD64_GLOBAL:
+        case SUPPAGINGMODE_AMD64_NX:
+        case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+            pThis->fLegacyMode = false;
+            break;
+        default:
+            rc = VERR_PGM_DYNMAP_IPE;
+            break;
+    }
+    if (RT_SUCCESS(rc))
+    {
+        rc = RTSemFastMutexCreate(&pThis->hInitLock);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTSpinlockCreate(&pThis->hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE, "PGMR0DynMap");
+            if (RT_SUCCESS(rc))
+            {
+                pThis->u32Magic = PGMRZDYNMAP_MAGIC;
+                g_pPGMR0DynMap = pThis;
+                return VINF_SUCCESS;
+            }
+            RTSemFastMutexDestroy(pThis->hInitLock);
+        }
+    }
+    RTMemFree(pThis);
+    return rc;
+}
+
+
+/**
+ * Terminates the ring-0 dynamic mapping cache.
+ */
+VMMR0DECL(void) PGMR0DynMapTerm(void)
+{
+    /*
+     * Destroy the cache.
+     *
+     * There is not supposed to be any races here, the loader should
+     * make sure about that. So, don't bother locking anything.
+     *
+     * The VM objects should all be destroyed by now, so there is no
+     * dangling users or anything like that to clean up. This routine
+     * is just a mirror image of PGMR0DynMapInit.
+     */
+    PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
+    if (pThis)
+    {
+        AssertPtr(pThis);
+        g_pPGMR0DynMap = NULL;
+
+        /* This should *never* happen, but in case it does try not to leak memory. */
+        AssertLogRelMsg(!pThis->cUsers && !pThis->paPages && !pThis->pvSavedPTEs && !pThis->cPages,
+                        ("cUsers=%d paPages=%p pvSavedPTEs=%p cPages=%#x\n",
+                         pThis->cUsers, pThis->paPages, pThis->pvSavedPTEs, pThis->cPages));
+        if (pThis->paPages)
+            pgmR0DynMapTearDown(pThis);
+
+        /* Free the associated resources. */
+        RTSemFastMutexDestroy(pThis->hInitLock);
+        pThis->hInitLock = NIL_RTSEMFASTMUTEX;
+        RTSpinlockDestroy(pThis->hSpinlock);
+        pThis->hSpinlock = NIL_RTSPINLOCK;
+        pThis->u32Magic = UINT32_MAX;
+        RTMemFree(pThis);
+    }
+}
+
+
+/**
+ * Initializes the dynamic mapping cache for a new VM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR0DECL(int) PGMR0DynMapInitVM(PVM pVM)
+{
+    AssertMsgReturn(!pVM->pgm.s.pvR0DynMapUsed, ("%p (pThis=%p)\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap), VERR_WRONG_ORDER);
+
+    /*
+     * Initialize the auto sets.
+     */
+    int rc = pgmRZDynMapInitAutoSetsForVM(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Do we need the cache? Skip the last bit if we don't.
+     */
+    if (VM_IS_RAW_MODE_ENABLED(pVM))
+        return VINF_SUCCESS;
+
+    /*
+     * Reference and if necessary setup or expand the cache.
+     */
+    PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
+    AssertPtrReturn(pThis, VERR_PGM_DYNMAP_IPE);
+    rc = RTSemFastMutexRequest(pThis->hInitLock);
+    AssertLogRelRCReturn(rc, rc);
+
+    pThis->cUsers++;
+    if (pThis->cUsers == 1)
+    {
+        rc = pgmR0DynMapSetup(pThis);
+#if 0 /*def DEBUG*/
+        if (RT_SUCCESS(rc))
+        {
+            rc = pgmR0DynMapTest(pVM);
+            if (RT_FAILURE(rc))
+                pgmR0DynMapTearDown(pThis);
+        }
+#endif
+    }
+    else if (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(pThis->cPages - pThis->cGuardPages))
+        rc = pgmR0DynMapExpand(pThis);
+    if (RT_SUCCESS(rc))
+        pVM->pgm.s.pvR0DynMapUsed = pThis;
+    else
+        pThis->cUsers--;
+
+    RTSemFastMutexRelease(pThis->hInitLock);
+    return rc;
+}
+
+
+/**
+ * Terminates the dynamic mapping cache usage for a VM.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+VMMR0DECL(void) PGMR0DynMapTermVM(PVM pVM)
+{
+    /*
+     * Return immediately if we're not using the cache.
+     */
+    if (!pVM->pgm.s.pvR0DynMapUsed)
+        return;
+
+    PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
+    AssertPtrReturnVoid(pThis);
+
+    int rc = RTSemFastMutexRequest(pThis->hInitLock);
+    AssertLogRelRCReturnVoid(rc);
+
+    if (pVM->pgm.s.pvR0DynMapUsed == pThis)
+    {
+        pVM->pgm.s.pvR0DynMapUsed = NULL;
+
+#ifdef VBOX_STRICT
+        PGMR0DynMapAssertIntegrity();
+#endif
+
+        /*
+         * Clean up and check the auto sets.
+         */
+        VMCPUID idCpu = pVM->cCpus;
+        while (idCpu-- > 0)
+        {
+            PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
+            uint32_t j = pSet->cEntries;
+            if (j <= RT_ELEMENTS(pSet->aEntries))
+            {
+                /*
+                 * The set is open, close it.
+                 */
+                while (j-- > 0)
+                {
+                    int32_t  cRefs = pSet->aEntries[j].cRefs;
+                    uint32_t iPage = pSet->aEntries[j].iPage;
+                    LogRel(("PGMR0DynMapTermVM: %d dangling refs to %#x\n", cRefs, iPage));
+                    if (iPage < pThis->cPages && cRefs > 0)
+                        pgmRZDynMapReleasePage(pThis, iPage, cRefs);
+                    else
+                        AssertLogRelMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages));
+
+                    PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
+                }
+                pSet->cEntries = PGMMAPSET_CLOSED;
+                pSet->iSubset = UINT32_MAX;
+                pSet->iCpu = -1;
+            }
+            else
+                AssertMsg(j == PGMMAPSET_CLOSED, ("cEntries=%#x\n", j));
+
+            j = RT_ELEMENTS(pSet->aEntries);
+            while (j-- > 0)
+            {
+                Assert(pSet->aEntries[j].iPage == UINT16_MAX);
+                Assert(!pSet->aEntries[j].cRefs);
+            }
+        }
+
+        /*
+         * Release our reference to the mapping cache.
+         */
+        Assert(pThis->cUsers > 0);
+        pThis->cUsers--;
+        if (!pThis->cUsers)
+            pgmR0DynMapTearDown(pThis);
+    }
+    else
+        AssertLogRelMsgFailed(("pvR0DynMapUsed=%p pThis=%p\n", pVM->pgm.s.pvR0DynMapUsed, pThis));
+
+    RTSemFastMutexRelease(pThis->hInitLock);
+}
+
+
+/**
+ * Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper.
+ *
+ * @param   idCpu           The current CPU.
+ * @param   pvUser1         The dynamic mapping cache instance.
+ * @param   pvUser2         Unused, NULL.
+ */
+static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    Assert(!pvUser2);
+    PPGMRZDYNMAP        pThis   = (PPGMRZDYNMAP)pvUser1;
+    Assert(pThis == g_pPGMR0DynMap);
+    PPGMRZDYNMAPENTRY   paPages = pThis->paPages;
+    uint32_t            iPage   = pThis->cPages;
+    while (iPage-- > 0)
+        ASMInvalidatePage((uintptr_t)paPages[iPage].pvPage);
+}
+
+
+/**
+ * Shoot down the TLBs for every single cache entry on all CPUs.
+ *
+ * @returns IPRT status code (RTMpOnAll).
+ * @param   pThis       The dynamic mapping cache instance.
+ */
+static int pgmR0DynMapTlbShootDown(PPGMRZDYNMAP pThis)
+{
+    int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
+    AssertRC(rc);
+    if (RT_FAILURE(rc))
+    {
+        uint32_t iPage = pThis->cPages;
+        while (iPage-- > 0)
+            ASMInvalidatePage((uintptr_t)pThis->paPages[iPage].pvPage);
+    }
+    return rc;
+}
+
+
+/**
+ * Calculate the new cache size based on cMaxLoad statistics.
+ *
+ * @returns Number of pages.
+ * @param   pThis       The dynamic mapping cache instance.
+ * @param   pcMinPages  The minimal size in pages.
+ */
+static uint32_t pgmR0DynMapCalcNewSize(PPGMRZDYNMAP pThis, uint32_t *pcMinPages)
+{
+    Assert(pThis->cPages <= PGMR0DYNMAP_MAX_PAGES);
+
+    /* cCpus * PGMR0DYNMAP_PAGES_PER_CPU(_MIN). */
+    RTCPUID     cCpus     = RTMpGetCount();
+    AssertReturn(cCpus > 0 && cCpus <= RTCPUSET_MAX_CPUS, 0);
+    uint32_t    cPages    = cCpus * PGMR0DYNMAP_PAGES_PER_CPU;
+    uint32_t    cMinPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU_MIN;
+
+    /* adjust against cMaxLoad. */
+    AssertMsg(pThis->cMaxLoad <= PGMR0DYNMAP_MAX_PAGES, ("%#x\n", pThis->cMaxLoad));
+    if (pThis->cMaxLoad > PGMR0DYNMAP_MAX_PAGES)
+        pThis->cMaxLoad = 0;
+
+    while (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(cPages))
+        cPages += PGMR0DYNMAP_PAGES_PER_CPU;
+
+    if (pThis->cMaxLoad > cMinPages)
+        cMinPages = pThis->cMaxLoad;
+
+    /* adjust against max and current size. */
+    if (cPages < pThis->cPages)
+        cPages = pThis->cPages;
+    cPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
+    if (cPages > PGMR0DYNMAP_MAX_PAGES)
+        cPages = PGMR0DYNMAP_MAX_PAGES;
+
+    if (cMinPages < pThis->cPages)
+        cMinPages = pThis->cPages;
+    cMinPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
+    if (cMinPages > PGMR0DYNMAP_MAX_PAGES)
+        cMinPages = PGMR0DYNMAP_MAX_PAGES;
+
+    Assert(cMinPages);
+    *pcMinPages = cMinPages;
+    return cPages;
+}
+
+
+/**
+ * Initializes the paging level data.
+ *
+ * @param   pThis       The dynamic mapping cache instance.
+ * @param   pPgLvl      The paging level data.
+ */
+void pgmR0DynMapPagingArrayInit(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl)
+{
+    RTCCUINTREG     cr4 = ASMGetCR4();
+    switch (pThis->enmPgMode)
+    {
+        case SUPPAGINGMODE_32_BIT:
+        case SUPPAGINGMODE_32_BIT_GLOBAL:
+            pPgLvl->cLevels = 2;
+            pPgLvl->a[0].fPhysMask = X86_CR3_PAGE_MASK;
+            pPgLvl->a[0].fAndMask  = X86_PDE_P | X86_PDE_RW | (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
+            pPgLvl->a[0].fResMask  = X86_PDE_P | X86_PDE_RW;
+            pPgLvl->a[0].fPtrMask  = X86_PD_MASK;
+            pPgLvl->a[0].fPtrShift = X86_PD_SHIFT;
+
+            pPgLvl->a[1].fPhysMask = X86_PDE_PG_MASK;
+            pPgLvl->a[1].fAndMask  = X86_PTE_P | X86_PTE_RW;
+            pPgLvl->a[1].fResMask  = X86_PTE_P | X86_PTE_RW;
+            pPgLvl->a[1].fPtrMask  = X86_PT_MASK;
+            pPgLvl->a[1].fPtrShift = X86_PT_SHIFT;
+            break;
+
+        case SUPPAGINGMODE_PAE:
+        case SUPPAGINGMODE_PAE_GLOBAL:
+        case SUPPAGINGMODE_PAE_NX:
+        case SUPPAGINGMODE_PAE_GLOBAL_NX:
+            pPgLvl->cLevels = 3;
+            pPgLvl->a[0].fPhysMask = X86_CR3_PAE_PAGE_MASK;
+            pPgLvl->a[0].fPtrMask  = X86_PDPT_MASK_PAE;
+            pPgLvl->a[0].fPtrShift = X86_PDPT_SHIFT;
+            pPgLvl->a[0].fAndMask  = X86_PDPE_P;
+            pPgLvl->a[0].fResMask  = X86_PDPE_P;
+
+            pPgLvl->a[1].fPhysMask = X86_PDPE_PG_MASK;
+            pPgLvl->a[1].fPtrMask  = X86_PD_PAE_MASK;
+            pPgLvl->a[1].fPtrShift = X86_PD_PAE_SHIFT;
+            pPgLvl->a[1].fAndMask  = X86_PDE_P | X86_PDE_RW | (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
+            pPgLvl->a[1].fResMask  = X86_PDE_P | X86_PDE_RW;
+
+            pPgLvl->a[2].fPhysMask = X86_PDE_PAE_PG_MASK;
+            pPgLvl->a[2].fPtrMask  = X86_PT_PAE_MASK;
+            pPgLvl->a[2].fPtrShift = X86_PT_PAE_SHIFT;
+            pPgLvl->a[2].fAndMask  = X86_PTE_P | X86_PTE_RW;
+            pPgLvl->a[2].fResMask  = X86_PTE_P | X86_PTE_RW;
+            break;
+
+        case SUPPAGINGMODE_AMD64:
+        case SUPPAGINGMODE_AMD64_GLOBAL:
+        case SUPPAGINGMODE_AMD64_NX:
+        case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+            pPgLvl->cLevels = 4;
+            pPgLvl->a[0].fPhysMask = X86_CR3_AMD64_PAGE_MASK;
+            pPgLvl->a[0].fPtrShift = X86_PML4_SHIFT;
+            pPgLvl->a[0].fPtrMask  = X86_PML4_MASK;
+            pPgLvl->a[0].fAndMask  = X86_PML4E_P | X86_PML4E_RW;
+            pPgLvl->a[0].fResMask  = X86_PML4E_P | X86_PML4E_RW;
+
+            pPgLvl->a[1].fPhysMask = X86_PML4E_PG_MASK;
+            pPgLvl->a[1].fPtrShift = X86_PDPT_SHIFT;
+            pPgLvl->a[1].fPtrMask  = X86_PDPT_MASK_AMD64;
+            pPgLvl->a[1].fAndMask  = X86_PDPE_P | X86_PDPE_RW /** @todo check for X86_PDPT_PS support. */;
+            pPgLvl->a[1].fResMask  = X86_PDPE_P | X86_PDPE_RW;
+
+            pPgLvl->a[2].fPhysMask = X86_PDPE_PG_MASK;
+            pPgLvl->a[2].fPtrShift = X86_PD_PAE_SHIFT;
+            pPgLvl->a[2].fPtrMask  = X86_PD_PAE_MASK;
+            pPgLvl->a[2].fAndMask  = X86_PDE_P | X86_PDE_RW | (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
+            pPgLvl->a[2].fResMask  = X86_PDE_P | X86_PDE_RW;
+
+            pPgLvl->a[3].fPhysMask = X86_PDE_PAE_PG_MASK;
+            pPgLvl->a[3].fPtrShift = X86_PT_PAE_SHIFT;
+            pPgLvl->a[3].fPtrMask  = X86_PT_PAE_MASK;
+            pPgLvl->a[3].fAndMask  = X86_PTE_P | X86_PTE_RW;
+            pPgLvl->a[3].fResMask  = X86_PTE_P | X86_PTE_RW;
+            break;
+
+        default:
+            AssertFailed();
+            pPgLvl->cLevels = 0;
+            break;
+    }
+
+    for (uint32_t i = 0; i < 4; i++) /* ASSUMING array size. */
+    {
+        pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
+        pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
+        pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
+        pPgLvl->a[i].u.pv = NULL;
+    }
+}
+
+
+/**
+ * Maps a PTE.
+ *
+ * This will update the segment structure when new PTs are mapped.
+ *
+ * It also assumes that we (for paranoid reasons) wish to establish a mapping
+ * chain from CR3 to the PT that all corresponds to the processor we're
+ * currently running on, and go about this by running with interrupts disabled
+ * and restarting from CR3 for every change.
+ *
+ * @returns VBox status code, VINF_TRY_AGAIN if we changed any mappings and had
+ *          to re-enable interrupts.
+ * @param   pThis       The dynamic mapping cache instance.
+ * @param   pPgLvl      The paging level structure.
+ * @param   pvPage      The page.
+ * @param   pSeg        The segment.
+ * @param   cMaxPTs     The max number of PTs expected in the segment.
+ * @param   ppvPTE      Where to store the PTE address.
+ */
+static int pgmR0DynMapPagingArrayMapPte(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl, void *pvPage,
+                                        PPGMR0DYNMAPSEG pSeg, uint32_t cMaxPTs, void **ppvPTE)
+{
+    Assert(!(ASMGetFlags() & X86_EFL_IF));
+    void           *pvEntry = NULL;
+    X86PGPAEUINT    uEntry = ASMGetCR3();
+    for (uint32_t i = 0; i < pPgLvl->cLevels; i++)
+    {
+        RTHCPHYS HCPhys = uEntry & pPgLvl->a[i].fPhysMask;
+        if (pPgLvl->a[i].HCPhys != HCPhys)
+        {
+            /*
+             * Need to remap this level.
+             * The final level, the PT, will not be freed since that is what it's all about.
+             */
+            ASMIntEnable();
+            if (i + 1 == pPgLvl->cLevels)
+                AssertReturn(pSeg->cPTs < cMaxPTs, VERR_PGM_DYNMAP_IPE);
+            else
+            {
+                int rc2 = RTR0MemObjFree(pPgLvl->a[i].hMemObj, true /* fFreeMappings */); AssertRC(rc2);
+                pPgLvl->a[i].hMemObj = pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
+            }
+
+            int rc = RTR0MemObjEnterPhys(&pPgLvl->a[i].hMemObj, HCPhys, PAGE_SIZE, RTMEM_CACHE_POLICY_DONT_CARE);
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTR0MemObjMapKernel(&pPgLvl->a[i].hMapObj, pPgLvl->a[i].hMemObj,
+                                         (void *)-1 /* pvFixed */, 0 /* cbAlignment */,
+                                         RTMEM_PROT_WRITE | RTMEM_PROT_READ);
+                if (RT_SUCCESS(rc))
+                {
+                    pPgLvl->a[i].u.pv   = RTR0MemObjAddress(pPgLvl->a[i].hMapObj);
+                    AssertMsg(((uintptr_t)pPgLvl->a[i].u.pv & ~(uintptr_t)PAGE_OFFSET_MASK), ("%p\n", pPgLvl->a[i].u.pv));
+                    pPgLvl->a[i].HCPhys = HCPhys;
+                    if (i + 1 == pPgLvl->cLevels)
+                        pSeg->ahMemObjPTs[pSeg->cPTs++] = pPgLvl->a[i].hMemObj;
+                    ASMIntDisable();
+                    return VINF_TRY_AGAIN;
+                }
+
+                pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
+            }
+            else
+                pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
+            pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
+            return rc;
+        }
+
+        /*
+         * The next level.
+         */
+        uint32_t iEntry = ((uint64_t)(uintptr_t)pvPage >> pPgLvl->a[i].fPtrShift) & pPgLvl->a[i].fPtrMask;
+        if (pThis->fLegacyMode)
+        {
+            pvEntry = &pPgLvl->a[i].u.paLegacy[iEntry];
+            uEntry  = pPgLvl->a[i].u.paLegacy[iEntry];
+        }
+        else
+        {
+            pvEntry = &pPgLvl->a[i].u.paPae[iEntry];
+            uEntry  = pPgLvl->a[i].u.paPae[iEntry];
+        }
+
+        if ((uEntry & pPgLvl->a[i].fAndMask) != pPgLvl->a[i].fResMask)
+        {
+            LogRel(("PGMR0DynMap: internal error - iPgLvl=%u cLevels=%u uEntry=%#llx fAnd=%#llx fRes=%#llx got=%#llx\n"
+                    "PGMR0DynMap: pv=%p pvPage=%p iEntry=%#x fLegacyMode=%RTbool\n",
+                    i, pPgLvl->cLevels, uEntry, pPgLvl->a[i].fAndMask, pPgLvl->a[i].fResMask, uEntry & pPgLvl->a[i].fAndMask,
+                    pPgLvl->a[i].u.pv, pvPage, iEntry, pThis->fLegacyMode));
+            return VERR_PGM_DYNMAP_IPE;
+        }
+        /*Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));*/
+    }
+
+    /* made it thru without needing to remap anything. */
+    *ppvPTE = pvEntry;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets up a guard page.
+ *
+ * @param   pThis       The dynamic mapping cache instance.
+ * @param   pPage       The page.
+ */
+DECLINLINE(void) pgmR0DynMapSetupGuardPage(PPGMRZDYNMAP pThis, PPGMRZDYNMAPENTRY pPage)
+{
+    memset(pPage->pvPage, 0xfd, PAGE_SIZE);
+    pPage->cRefs  = PGMR0DYNMAP_GUARD_PAGE_REF_COUNT;
+    pPage->HCPhys = PGMR0DYNMAP_GUARD_PAGE_HCPHYS;
+#ifdef PGMR0DYNMAP_GUARD_NP
+    ASMAtomicBitClear(pPage->uPte.pv, X86_PTE_BIT_P);
+#else
+    if (pThis->fLegacyMode)
+        ASMAtomicWriteU32(&pPage->uPte.pLegacy->u, PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE);
+    else
+        ASMAtomicWriteU64(&pPage->uPte.pPae->u,    PGMR0DYNMAP_GUARD_PAGE_PAE_PTE);
+#endif
+    pThis->cGuardPages++;
+}
+
+
+/**
+ * Adds a new segment of the specified size.
+ *
+ * @returns VBox status code.
+ * @param   pThis       The dynamic mapping cache instance.
+ * @param   cPages      The size of the new segment, give as a page count.
+ */
+static int pgmR0DynMapAddSeg(PPGMRZDYNMAP pThis, uint32_t cPages)
+{
+    int rc2;
+    AssertReturn(ASMGetFlags() & X86_EFL_IF, VERR_PREEMPT_DISABLED);
+
+    /*
+     * Do the array reallocations first.
+     * (The pages array has to be replaced behind the spinlock of course.)
+     */
+    void *pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * (pThis->cPages + cPages));
+    if (!pvSavedPTEs)
+        return VERR_NO_MEMORY;
+    pThis->pvSavedPTEs = pvSavedPTEs;
+
+    void *pvPages = RTMemAllocZ(sizeof(pThis->paPages[0]) * (pThis->cPages + cPages));
+    if (!pvPages)
+    {
+        pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * pThis->cPages);
+        if (pvSavedPTEs)
+            pThis->pvSavedPTEs = pvSavedPTEs;
+        return VERR_NO_MEMORY;
+    }
+
+    PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
+
+    memcpy(pvPages, pThis->paPages, sizeof(pThis->paPages[0]) * pThis->cPages);
+    void *pvToFree = pThis->paPages;
+    pThis->paPages = (PPGMRZDYNMAPENTRY)pvPages;
+
+    PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
+    RTMemFree(pvToFree);
+
+    /*
+     * Allocate the segment structure and pages of memory, then touch all the pages (paranoia).
+     */
+    uint32_t cMaxPTs = cPages / (pThis->fLegacyMode ? X86_PG_ENTRIES : X86_PG_PAE_ENTRIES) + 2;
+    PPGMR0DYNMAPSEG pSeg = (PPGMR0DYNMAPSEG)RTMemAllocZ(RT_UOFFSETOF_DYN(PGMR0DYNMAPSEG, ahMemObjPTs[cMaxPTs]));
+    if (!pSeg)
+        return VERR_NO_MEMORY;
+    pSeg->pNext  = NULL;
+    pSeg->cPages = cPages;
+    pSeg->iPage  = pThis->cPages;
+    pSeg->cPTs   = 0;
+    int rc = RTR0MemObjAllocPage(&pSeg->hMemObj, cPages << PAGE_SHIFT, false);
+    if (RT_SUCCESS(rc))
+    {
+        uint8_t            *pbPage = (uint8_t *)RTR0MemObjAddress(pSeg->hMemObj);
+        AssertMsg(VALID_PTR(pbPage) && !((uintptr_t)pbPage & PAGE_OFFSET_MASK), ("%p\n", pbPage));
+        memset(pbPage, 0xfe, cPages << PAGE_SHIFT);
+
+        /*
+         * Walk thru the pages and set them up with a mapping of their PTE and everything.
+         */
+        ASMIntDisable();
+        PGMR0DYNMAPPGLVL    PgLvl;
+        pgmR0DynMapPagingArrayInit(pThis, &PgLvl);
+        uint32_t const      iEndPage = pSeg->iPage + cPages;
+        for (uint32_t iPage = pSeg->iPage;
+             iPage < iEndPage;
+             iPage++, pbPage += PAGE_SIZE)
+        {
+            /* Initialize the page data. */
+            pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
+            pThis->paPages[iPage].pvPage = pbPage;
+            pThis->paPages[iPage].cRefs  = 0;
+            pThis->paPages[iPage].uPte.pPae = 0;
+#ifndef IN_RC
+            RTCpuSetFill(&pThis->paPages[iPage].PendingSet);
+#endif
+
+            /* Map its page table, retry until we've got a clean run (paranoia). */
+            do
+                rc = pgmR0DynMapPagingArrayMapPte(pThis, &PgLvl, pbPage, pSeg, cMaxPTs,
+                                                  &pThis->paPages[iPage].uPte.pv);
+            while (rc == VINF_TRY_AGAIN);
+            if (RT_FAILURE(rc))
+                break;
+
+            /* Save the PTE. */
+            if (pThis->fLegacyMode)
+                ((PX86PGUINT)pThis->pvSavedPTEs)[iPage]    = pThis->paPages[iPage].uPte.pLegacy->u;
+            else
+                ((PX86PGPAEUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pPae->u;
+
+#ifdef VBOX_STRICT
+            /* Check that we've got the right entry. */
+            RTHCPHYS HCPhysPage = RTR0MemObjGetPagePhysAddr(pSeg->hMemObj, iPage - pSeg->iPage);
+            RTHCPHYS HCPhysPte  = pThis->fLegacyMode
+                                ? pThis->paPages[iPage].uPte.pLegacy->u & X86_PTE_PG_MASK
+                                : pThis->paPages[iPage].uPte.pPae->u    & X86_PTE_PAE_PG_MASK;
+            if (HCPhysPage != HCPhysPte)
+            {
+                LogRel(("pgmR0DynMapAddSeg: internal error - page #%u HCPhysPage=%RHp HCPhysPte=%RHp pbPage=%p pvPte=%p\n",
+                        iPage - pSeg->iPage, HCPhysPage, HCPhysPte, pbPage, pThis->paPages[iPage].uPte.pv));
+                rc = VERR_PGM_DYNMAP_IPE;
+                break;
+            }
+#endif
+        } /* for each page */
+        ASMIntEnable();
+
+        /* cleanup non-PT mappings */
+        for (uint32_t i = 0; i < PgLvl.cLevels - 1; i++)
+            RTR0MemObjFree(PgLvl.a[i].hMemObj, true /* fFreeMappings */);
+
+        if (RT_SUCCESS(rc))
+        {
+#if PGMR0DYNMAP_GUARD_PAGES > 0
+            /*
+             * Setup guard pages.
+             * (Note: TLBs will be shot down later on.)
+             */
+            uint32_t iPage = pSeg->iPage;
+            while (iPage < iEndPage)
+            {
+                for (uint32_t iGPg = 0; iGPg < PGMR0DYNMAP_GUARD_PAGES && iPage < iEndPage; iGPg++, iPage++)
+                    pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
+                iPage++; /* the guarded page */
+            }
+
+            /* Make sure the very last page is a guard page too. */
+            iPage = iEndPage - 1;
+            if (pThis->paPages[iPage].cRefs != PGMR0DYNMAP_GUARD_PAGE_REF_COUNT)
+                pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
+#endif /* PGMR0DYNMAP_GUARD_PAGES > 0 */
+
+            /*
+             * Commit it by adding the segment to the list and updating the page count.
+             */
+            pSeg->pNext = pThis->pSegHead;
+            pThis->pSegHead = pSeg;
+            pThis->cPages += cPages;
+            return VINF_SUCCESS;
+        }
+
+        /*
+         * Bail out.
+         */
+        while (pSeg->cPTs-- > 0)
+        {
+            rc2 = RTR0MemObjFree(pSeg->ahMemObjPTs[pSeg->cPTs], true /* fFreeMappings */);
+            AssertRC(rc2);
+            pSeg->ahMemObjPTs[pSeg->cPTs] = NIL_RTR0MEMOBJ;
+        }
+
+        rc2 = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */);
+        AssertRC(rc2);
+        pSeg->hMemObj = NIL_RTR0MEMOBJ;
+    }
+    else if (rc == VERR_NO_PAGE_MEMORY || rc == VERR_NO_PHYS_MEMORY)
+        rc = VERR_NO_MEMORY;
+    RTMemFree(pSeg);
+
+    /* Don't bother resizing the arrays, but free them if we're the only user. */
+    if (!pThis->cPages)
+    {
+        RTMemFree(pThis->paPages);
+        pThis->paPages = NULL;
+        RTMemFree(pThis->pvSavedPTEs);
+        pThis->pvSavedPTEs = NULL;
+    }
+    return rc;
+}
+
+
+/**
+ * Called by PGMR0DynMapInitVM under the init lock.
+ *
+ * @returns VBox status code.
+ * @param   pThis       The dynamic mapping cache instance.
+ */
+static int pgmR0DynMapSetup(PPGMRZDYNMAP pThis)
+{
+    /*
+     * Calc the size and add a segment of that size.
+     */
+    uint32_t cMinPages;
+    uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
+    AssertReturn(cPages, VERR_PGM_DYNMAP_IPE);
+    int rc = pgmR0DynMapAddSeg(pThis, cPages);
+    if (rc == VERR_NO_MEMORY)
+    {
+        /*
+         * Try adding smaller segments.
+         */
+        do
+            rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
+        while (RT_SUCCESS(rc) && pThis->cPages < cPages);
+        if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
+            rc = VINF_SUCCESS;
+        if (rc == VERR_NO_MEMORY)
+        {
+            if (pThis->cPages)
+                pgmR0DynMapTearDown(pThis);
+            rc = VERR_PGM_DYNMAP_SETUP_ERROR;
+        }
+    }
+    Assert(ASMGetFlags() & X86_EFL_IF);
+
+#if PGMR0DYNMAP_GUARD_PAGES > 0
+    /* paranoia */
+    if (RT_SUCCESS(rc))
+        pgmR0DynMapTlbShootDown(pThis);
+#endif
+    return rc;
+}
+
+
+/**
+ * Called by PGMR0DynMapInitVM under the init lock.
+ *
+ * @returns VBox status code.
+ * @param   pThis       The dynamic mapping cache instance.
+ */
+static int pgmR0DynMapExpand(PPGMRZDYNMAP pThis)
+{
+    /*
+     * Calc the new target size and add a segment of the appropriate size.
+     */
+    uint32_t cMinPages;
+    uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
+    AssertReturn(cPages, VERR_PGM_DYNMAP_IPE);
+    if (pThis->cPages >= cPages)
+        return VINF_SUCCESS;
+
+    uint32_t cAdd = cPages - pThis->cPages;
+    int rc = pgmR0DynMapAddSeg(pThis, cAdd);
+    if (rc == VERR_NO_MEMORY)
+    {
+        /*
+         * Try adding smaller segments.
+         */
+        do
+            rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
+        while (RT_SUCCESS(rc) && pThis->cPages < cPages);
+        if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
+            rc = VINF_SUCCESS;
+        if (rc == VERR_NO_MEMORY)
+            rc = VERR_PGM_DYNMAP_EXPAND_ERROR;
+    }
+    Assert(ASMGetFlags() & X86_EFL_IF);
+
+#if PGMR0DYNMAP_GUARD_PAGES > 0
+    /* paranoia */
+    if (RT_SUCCESS(rc))
+        pgmR0DynMapTlbShootDown(pThis);
+#endif
+    return rc;
+}
+
+
+/**
+ * Called by PGMR0DynMapTermVM under the init lock.
+ *
+ * @returns VBox status code.
+ * @param   pThis       The dynamic mapping cache instance.
+ */
+static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis)
+{
+    /*
+     * Restore the original page table entries
+     */
+    PPGMRZDYNMAPENTRY   paPages = pThis->paPages;
+    uint32_t            iPage   = pThis->cPages;
+    if (pThis->fLegacyMode)
+    {
+        X86PGUINT const    *paSavedPTEs = (X86PGUINT const *)pThis->pvSavedPTEs;
+        while (iPage-- > 0)
+        {
+            X86PGUINT       uOld  = paPages[iPage].uPte.pLegacy->u;
+            X86PGUINT       uOld2 = uOld; NOREF(uOld2);
+            X86PGUINT       uNew  = paSavedPTEs[iPage];
+            while (!ASMAtomicCmpXchgExU32(&paPages[iPage].uPte.pLegacy->u, uNew, uOld, &uOld))
+                AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
+            Assert(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage]);
+        }
+    }
+    else
+    {
+        X86PGPAEUINT const *paSavedPTEs = (X86PGPAEUINT const *)pThis->pvSavedPTEs;
+        while (iPage-- > 0)
+        {
+            X86PGPAEUINT    uOld  = paPages[iPage].uPte.pPae->u;
+            X86PGPAEUINT    uOld2 = uOld; NOREF(uOld2);
+            X86PGPAEUINT    uNew  = paSavedPTEs[iPage];
+            while (!ASMAtomicCmpXchgExU64(&paPages[iPage].uPte.pPae->u, uNew, uOld, &uOld))
+                AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
+            Assert(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage]);
+        }
+    }
+
+    /*
+     * Shoot down the TLBs on all CPUs before freeing them.
+     */
+    pgmR0DynMapTlbShootDown(pThis);
+
+    /*
+     * Free the segments.
+     */
+    while (pThis->pSegHead)
+    {
+        int             rc;
+        PPGMR0DYNMAPSEG pSeg = pThis->pSegHead;
+        pThis->pSegHead = pSeg->pNext;
+
+        uint32_t iPT = pSeg->cPTs;
+        while (iPT-- > 0)
+        {
+            rc = RTR0MemObjFree(pSeg->ahMemObjPTs[iPT], true /* fFreeMappings */); AssertRC(rc);
+            pSeg->ahMemObjPTs[iPT] = NIL_RTR0MEMOBJ;
+        }
+        rc = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */); AssertRC(rc);
+        pSeg->hMemObj   = NIL_RTR0MEMOBJ;
+        pSeg->pNext     = NULL;
+        pSeg->iPage     = UINT16_MAX;
+        pSeg->cPages    = 0;
+        pSeg->cPTs      = 0;
+        RTMemFree(pSeg);
+    }
+
+    /*
+     * Free the arrays and restore the initial state.
+     * The cLoadMax value is left behind for the next setup.
+     */
+    RTMemFree(pThis->paPages);
+    pThis->paPages = NULL;
+    RTMemFree(pThis->pvSavedPTEs);
+    pThis->pvSavedPTEs = NULL;
+    pThis->cPages = 0;
+    pThis->cLoad = 0;
+    pThis->cGuardPages = 0;
+}
+
+#endif /* IN_RING0 */
+#ifdef IN_RC
+
+/**
+ * Initializes the dynamic mapping cache in raw-mode context.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMRCDECL(int) PGMRCDynMapInit(PVM pVM)
+{
+    /*
+     * Allocate and initialize the instance data and page array.
+     */
+    PPGMRZDYNMAP    pThis;
+    size_t const    cPages = MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE;
+    size_t const    cb     = RT_ALIGN_Z(sizeof(*pThis), 32)
+                           + sizeof(PGMRZDYNMAPENTRY) * cPages;
+    int rc = MMHyperAlloc(pVM, cb, 32, MM_TAG_PGM, (void **)&pThis);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    pThis->u32Magic     = PGMRZDYNMAP_MAGIC;
+    pThis->paPages      = RT_ALIGN_PT(pThis + 1, 32, PPGMRZDYNMAPENTRY);
+    pThis->cPages       = cPages;
+    pThis->cLoad        = 0;
+    pThis->cMaxLoad     = 0;
+    pThis->cGuardPages  = 0;
+    pThis->cUsers       = 1;
+
+    for (size_t iPage = 0; iPage < cPages; iPage++)
+    {
+        pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
+        pThis->paPages[iPage].pvPage = pVM->pgm.s.pbDynPageMapBaseGC + iPage * PAGE_SIZE;
+        pThis->paPages[iPage].cRefs  = 0;
+        pThis->paPages[iPage].uPte.pLegacy = &pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage];
+        pThis->paPages[iPage].uPte.pPae    = (PX86PTEPAE)&pVM->pgm.s.paDynPageMapPaePTEsGC[iPage];
+    }
+
+    pVM->pgm.s.pRCDynMap = pThis;
+
+    /*
+     * Initialize the autosets the VM.
+     */
+    rc = pgmRZDynMapInitAutoSetsForVM(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    return VINF_SUCCESS;
+}
+
+#endif /* IN_RC */
+
+/**
+ * Release references to a page, caller owns the spin lock.
+ *
+ * @param   pThis       The dynamic mapping cache instance.
+ * @param   iPage       The page.
+ * @param   cRefs       The number of references to release.
+ */
+DECLINLINE(void) pgmRZDynMapReleasePageLocked(PPGMRZDYNMAP pThis, uint32_t iPage, int32_t cRefs)
+{
+    cRefs = ASMAtomicSubS32(&pThis->paPages[iPage].cRefs, cRefs) - cRefs;
+    AssertMsg(cRefs >= 0, ("%d\n", cRefs));
+    if (!cRefs)
+    {
+        pThis->cLoad--;
+#ifdef PGMRZDYNMAP_STRICT_RELEASE
+        pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
+        ASMAtomicBitClear(pThis->paPages[iPage].uPte.pv, X86_PTE_BIT_P);
+        ASMInvalidatePage((uintptr_t)pThis->paPages[iPage].pvPage);
+#endif
+    }
+}
+
+
+/**
+ * Release references to a page, caller does not own the spin lock.
+ *
+ * @param   pThis       The dynamic mapping cache instance.
+ * @param   iPage       The page.
+ * @param   cRefs       The number of references to release.
+ */
+static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs)
+{
+    PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
+    pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
+    PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
+}
+
+
+/**
+ * pgmR0DynMapPage worker that deals with the tedious bits.
+ *
+ * @returns The page index on success, UINT32_MAX on failure.
+ * @param   pThis   The dynamic mapping cache instance.
+ * @param   HCPhys  The address of the page to be mapped.
+ * @param   iPage   The page index pgmR0DynMapPage hashed HCPhys to.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ *                  For statistics.
+ * @param   pfNew   Set to @c true if a new entry was made and @c false if
+ *                  an old entry was found and reused.
+ */
+static uint32_t pgmR0DynMapPageSlow(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage, PVMCPU pVCpu, bool *pfNew)
+{
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlow); RT_NOREF_PV(pVCpu);
+
+    /*
+     * Check if any of the first 3 pages are unreferenced since the caller
+     * already has made sure they aren't matching.
+     */
+#ifdef VBOX_WITH_STATISTICS
+    bool                fLooped = false;
+#endif
+    uint32_t const      cPages  = pThis->cPages;
+    PPGMRZDYNMAPENTRY   paPages = pThis->paPages;
+    uint32_t            iFreePage;
+    if (!paPages[iPage].cRefs)
+        iFreePage = iPage;
+    else if (!paPages[(iPage + 1) % cPages].cRefs)
+        iFreePage   = (iPage + 1) % cPages;
+    else if (!paPages[(iPage + 2) % cPages].cRefs)
+        iFreePage   = (iPage + 2) % cPages;
+    else
+    {
+        /*
+         * Search for an unused or matching entry.
+         */
+        iFreePage = (iPage + 3) % cPages;
+        for (;;)
+        {
+            if (paPages[iFreePage].HCPhys == HCPhys)
+            {
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopHits);
+                *pfNew = false;
+                return iFreePage;
+            }
+            if (!paPages[iFreePage].cRefs)
+                break;
+
+            /* advance */
+            iFreePage = (iFreePage + 1) % cPages;
+            if (RT_UNLIKELY(iFreePage == iPage))
+                return UINT32_MAX;
+        }
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopMisses);
+#ifdef VBOX_WITH_STATISTICS
+        fLooped = true;
+#endif
+    }
+    Assert(iFreePage < cPages);
+
+#if 0 //def VBOX_WITH_STATISTICS
+    /* Check for lost hits. */
+    if (!fLooped)
+        for (uint32_t iPage2 = (iPage + 3) % cPages; iPage2 != iPage; iPage2 = (iPage2 + 1) % cPages)
+            if (paPages[iPage2].HCPhys == HCPhys)
+                STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZDynMapPageSlowLostHits);
+#endif
+
+    /*
+     * Setup the new entry.
+     */
+    *pfNew = true;
+    /*Log6(("pgmR0DynMapPageSlow: old - %RHp %#x %#llx\n", paPages[iFreePage].HCPhys, paPages[iFreePage].cRefs, paPages[iFreePage].uPte.pPae->u));*/
+    paPages[iFreePage].HCPhys = HCPhys;
+#ifndef IN_RC
+    RTCpuSetFill(&paPages[iFreePage].PendingSet);
+
+    if (pThis->fLegacyMode)
+#endif
+    {
+        X86PGUINT       uOld  = paPages[iFreePage].uPte.pLegacy->u;
+        X86PGUINT       uOld2 = uOld; NOREF(uOld2);
+        X86PGUINT       uNew  = (uOld & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
+                              | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
+                              | (HCPhys & X86_PTE_PG_MASK);
+        while (!ASMAtomicCmpXchgExU32(&paPages[iFreePage].uPte.pLegacy->u, uNew, uOld, &uOld))
+            AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
+        Assert(paPages[iFreePage].uPte.pLegacy->u == uNew);
+    }
+#ifndef IN_RC
+    else
+#endif
+    {
+        X86PGPAEUINT    uOld  = paPages[iFreePage].uPte.pPae->u;
+        X86PGPAEUINT    uOld2 = uOld; NOREF(uOld2);
+        X86PGPAEUINT    uNew  = (uOld & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
+                              | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
+                              | (HCPhys & X86_PTE_PAE_PG_MASK);
+        while (!ASMAtomicCmpXchgExU64(&paPages[iFreePage].uPte.pPae->u, uNew, uOld, &uOld))
+            AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
+        Assert(paPages[iFreePage].uPte.pPae->u == uNew);
+        /*Log6(("pgmR0DynMapPageSlow: #%x - %RHp %p %#llx\n", iFreePage, HCPhys, paPages[iFreePage].pvPage, uNew));*/
+    }
+    return iFreePage;
+}
+
+
+/**
+ * Maps a page into the pool.
+ *
+ * @returns Page index on success, UINT32_MAX on failure.
+ * @param   pThis       The dynamic mapping cache instance.
+ * @param   HCPhys      The address of the page to be mapped.
+ * @param   iRealCpu    The real cpu set index. (optimization)
+ * @param   pVCpu       The cross context virtual CPU structure of the calling
+ *                      EMT.  For statistics.
+ * @param   ppvPage     Where to the page address.
+ */
+DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, int32_t iRealCpu, PVMCPU pVCpu, void **ppvPage)
+{
+    PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
+    AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPage);
+
+    /*
+     * Find an entry, if possible a matching one. The HCPhys address is hashed
+     * down to a page index, collisions are handled by linear searching.
+     * Optimized for a hit in the first 3 pages.
+     *
+     * Field easy hits here and defer the tedious searching and inserting
+     * to pgmR0DynMapPageSlow().
+     */
+    bool                fNew    = false;
+    uint32_t const      cPages  = pThis->cPages;
+    uint32_t            iPage   = (HCPhys >> PAGE_SHIFT) % cPages;
+    PPGMRZDYNMAPENTRY   paPages = pThis->paPages;
+    if (RT_LIKELY(paPages[iPage].HCPhys == HCPhys))
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits0);
+    else
+    {
+        uint32_t        iPage2 = (iPage + 1) % cPages;
+        if (RT_LIKELY(paPages[iPage2].HCPhys == HCPhys))
+        {
+            iPage = iPage2;
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits1);
+        }
+        else
+        {
+            iPage2 = (iPage + 2) % cPages;
+            if (paPages[iPage2].HCPhys == HCPhys)
+            {
+                iPage = iPage2;
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits2);
+            }
+            else
+            {
+                iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage, pVCpu, &fNew);
+                if (RT_UNLIKELY(iPage == UINT32_MAX))
+                {
+                    PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
+                    *ppvPage = NULL;
+                    return iPage;
+                }
+            }
+        }
+    }
+
+    /*
+     * Reference it, update statistics and get the return address.
+     */
+    int32_t cRefs = ASMAtomicIncS32(&paPages[iPage].cRefs);
+    if (cRefs == 1)
+    {
+        pThis->cLoad++;
+        if (pThis->cLoad > pThis->cMaxLoad)
+            pThis->cMaxLoad = pThis->cLoad;
+        AssertMsg(pThis->cLoad <= pThis->cPages - pThis->cGuardPages, ("%d/%d\n", pThis->cLoad, pThis->cPages - pThis->cGuardPages));
+    }
+    else if (RT_UNLIKELY(cRefs <= 0))
+    {
+        ASMAtomicDecS32(&paPages[iPage].cRefs);
+        PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
+        *ppvPage = NULL;
+        AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%u HCPhys=%RHp\n", cRefs, iPage, HCPhys), UINT32_MAX);
+    }
+    void *pvPage = paPages[iPage].pvPage;
+
+#ifndef IN_RC
+    /*
+     * Invalidate the entry?
+     */
+    bool fInvalidateIt = RTCpuSetIsMemberByIndex(&paPages[iPage].PendingSet, iRealCpu);
+    if (RT_UNLIKELY(fInvalidateIt))
+        RTCpuSetDelByIndex(&paPages[iPage].PendingSet, iRealCpu);
+#else
+    NOREF(iRealCpu);
+#endif
+
+    PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
+
+    /*
+     * Do the actual invalidation outside the spinlock.
+     */
+#ifdef IN_RC
+    if (RT_UNLIKELY(fNew))
+#else
+    if (RT_UNLIKELY(fInvalidateIt))
+#endif
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageInvlPg);
+        ASMInvalidatePage((uintptr_t)pvPage);
+    }
+
+    *ppvPage = pvPage;
+    return iPage;
+}
+
+
+/**
+ * Assert the integrity of the pool.
+ *
+ * @returns VBox status code.
+ */
+static int pgmRZDynMapAssertIntegrity(PPGMRZDYNMAP pThis)
+{
+    /*
+     * Basic pool stuff that doesn't require any lock, just assumes we're a user.
+     */
+    if (!pThis)
+        return VINF_SUCCESS;
+    AssertPtrReturn(pThis, VERR_INVALID_POINTER);
+    AssertReturn(pThis->u32Magic == PGMRZDYNMAP_MAGIC, VERR_INVALID_MAGIC);
+    if (!pThis->cUsers)
+        return VERR_INVALID_PARAMETER;
+
+    PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
+
+#define CHECK_RET(expr, a) \
+    do { \
+        if (RT_UNLIKELY(!(expr))) \
+        { \
+            PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); \
+            RTAssertMsg1Weak(#expr, __LINE__, __FILE__, __PRETTY_FUNCTION__); \
+            RTAssertMsg2Weak a; \
+            return VERR_PGM_DYNMAP_IPE; \
+        } \
+    } while (0)
+
+    /*
+     * Check that the PTEs are correct.
+     */
+    uint32_t            cGuard      = 0;
+    uint32_t            cLoad       = 0;
+    PPGMRZDYNMAPENTRY   paPages     = pThis->paPages;
+
+#ifndef IN_RC
+    if (pThis->fLegacyMode)
+#endif
+    {
+#ifdef IN_RING0
+        PCX86PGUINT     paSavedPTEs = (PCX86PGUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
+#endif
+        uint32_t        iPage       = pThis->cPages;
+        while (iPage-- > 0)
+        {
+            CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
+            if (    paPages[iPage].cRefs  == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
+                &&  paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
+            {
+#ifdef PGMR0DYNMAP_GUARD_NP
+                CHECK_RET(paPages[iPage].uPte.pLegacy->u == (paSavedPTEs[iPage] & ~(X86PGUINT)X86_PTE_P),
+                          ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
+#else
+                CHECK_RET(paPages[iPage].uPte.pLegacy->u == PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE,
+                          ("#%u: %#x", iPage, paPages[iPage].uPte.pLegacy->u));
+#endif
+                cGuard++;
+            }
+            else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
+            {
+                CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
+                X86PGUINT uPte = X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
+#ifdef IN_RING0
+                               | (paSavedPTEs[iPage] & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
+#endif
+                               | (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
+                CHECK_RET(paPages[iPage].uPte.pLegacy->u == uPte,
+                          ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
+                if (paPages[iPage].cRefs)
+                    cLoad++;
+            }
+#if defined(IN_RING0) && !defined(PGMRZDYNMAP_STRICT_RELEASE)
+            else
+                CHECK_RET(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage],
+                          ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
+#endif
+        }
+    }
+#ifndef IN_RC
+    else
+#endif
+    {
+#ifdef IN_RING0
+        PCX86PGPAEUINT  paSavedPTEs = (PCX86PGPAEUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
+#endif
+        uint32_t        iPage       = pThis->cPages;
+        while (iPage-- > 0)
+        {
+            CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
+            if (    paPages[iPage].cRefs  == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
+                &&  paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
+            {
+#ifdef PGMR0DYNMAP_GUARD_NP
+                CHECK_RET(paPages[iPage].uPte.pPae->u == (paSavedPTEs[iPage] & ~(X86PGPAEUINT)X86_PTE_P),
+                          ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
+#else
+                CHECK_RET(paPages[iPage].uPte.pPae->u == PGMR0DYNMAP_GUARD_PAGE_PAE_PTE,
+                          ("#%u: %#llx", iPage, paPages[iPage].uPte.pPae->u));
+#endif
+                cGuard++;
+            }
+            else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
+            {
+                CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
+                X86PGPAEUINT uPte = X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
+#ifdef IN_RING0
+                                  | (paSavedPTEs[iPage] & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
+#endif
+                                  | (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
+                CHECK_RET(paPages[iPage].uPte.pPae->u == uPte,
+                          ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
+                if (paPages[iPage].cRefs)
+                    cLoad++;
+            }
+#ifdef IN_RING0
+            else
+                CHECK_RET(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage],
+                          ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
+#endif
+        }
+    }
+
+    CHECK_RET(cLoad == pThis->cLoad, ("%u %u\n", cLoad, pThis->cLoad));
+    CHECK_RET(cGuard == pThis->cGuardPages, ("%u %u\n", cGuard, pThis->cGuardPages));
+
+#undef CHECK_RET
+    PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
+    return VINF_SUCCESS;
+}
+
+#ifdef IN_RING0
+/**
+ * Assert the integrity of the pool.
+ *
+ * @returns VBox status code.
+ */
+VMMR0DECL(int) PGMR0DynMapAssertIntegrity(void)
+{
+    return pgmRZDynMapAssertIntegrity(g_pPGMR0DynMap);
+}
+#endif /* IN_RING0 */
+
+#ifdef IN_RC
+/**
+ * Assert the integrity of the pool.
+ *
+ * @returns VBox status code.
+ */
+VMMRCDECL(int) PGMRCDynMapAssertIntegrity(PVM pVM)
+{
+    return pgmRZDynMapAssertIntegrity((PPGMRZDYNMAP)pVM->pgm.s.pRCDynMap);
+}
+#endif /* IN_RC */
+
+
+/**
+ * As a final resort for a (somewhat) full auto set or full cache, try merge
+ * duplicate entries and flush the ones we can.
+ *
+ * @param   pSet        The set.
+ */
+static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet)
+{
+    LogFlow(("pgmDynMapOptimizeAutoSet\n"));
+
+    for (uint32_t i = 0 ; i < pSet->cEntries; i++)
+    {
+        /*
+         * Try merge entries.
+         */
+        uint16_t const  iPage = pSet->aEntries[i].iPage;
+        uint32_t        j     = i + 1;
+        while (   j < pSet->cEntries
+               && (   pSet->iSubset == UINT32_MAX
+                   || pSet->iSubset < pSet->cEntries) )
+        {
+            if (pSet->aEntries[j].iPage != iPage)
+                j++;
+            else
+            {
+                uint32_t const  cHardRefs    = (uint32_t)pSet->aEntries[i].cRefs
+                                             + (uint32_t)pSet->aEntries[j].cRefs;
+                uint32_t        cInlinedRefs = (uint32_t)pSet->aEntries[i].cInlinedRefs
+                                             + (uint32_t)pSet->aEntries[j].cInlinedRefs;
+                uint32_t        cUnrefs      = (uint32_t)pSet->aEntries[i].cUnrefs
+                                             + (uint32_t)pSet->aEntries[j].cUnrefs;
+                uint32_t        cSub         = RT_MIN(cUnrefs, cInlinedRefs);
+                cInlinedRefs -= cSub;
+                cUnrefs      -= cSub;
+
+                if (    cHardRefs    < UINT16_MAX
+                    &&  cInlinedRefs < UINT16_MAX
+                    &&  cUnrefs      < UINT16_MAX)
+                {
+                    /* merge j into i removing j. */
+                    Log2(("pgmDynMapOptimizeAutoSet: Merging #%u into #%u\n", j, i));
+                    pSet->aEntries[i].cRefs        = cHardRefs;
+                    pSet->aEntries[i].cInlinedRefs = cInlinedRefs;
+                    pSet->aEntries[i].cUnrefs      = cUnrefs;
+                    pSet->cEntries--;
+                    if (j < pSet->cEntries)
+                    {
+                        pSet->aEntries[j] = pSet->aEntries[pSet->cEntries];
+                        PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]);
+                    }
+                    else
+                        PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
+                }
+#if 0 /* too complicated, skip it. */
+                else
+                {
+                    /* migrate the max number of refs from j into i and quit the inner loop. */
+                    uint32_t cMigrate = UINT16_MAX - 1 - pSet->aEntries[i].cRefs;
+                    Assert(pSet->aEntries[j].cRefs > cMigrate);
+                    pSet->aEntries[j].cRefs -= cMigrate;
+                    pSet->aEntries[i].cRefs = UINT16_MAX - 1;
+                    break;
+                }
+#endif
+            }
+        }
+
+        /*
+         * Try make use of the unused hinting (cUnrefs) to evict entries
+         * from both the set as well as the mapping cache.
+         */
+
+        uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[i].cRefs + pSet->aEntries[i].cInlinedRefs;
+        Log2(("pgmDynMapOptimizeAutoSet: #%u/%u/%u pvPage=%p iPage=%u cRefs=%u cInlinedRefs=%u cUnrefs=%u cTotalRefs=%u\n",
+              i,
+              pSet->iSubset,
+              pSet->cEntries,
+              pSet->aEntries[i].pvPage,
+              pSet->aEntries[i].iPage,
+              pSet->aEntries[i].cRefs,
+              pSet->aEntries[i].cInlinedRefs,
+              pSet->aEntries[i].cUnrefs,
+              cTotalRefs));
+        Assert(cTotalRefs >= pSet->aEntries[i].cUnrefs);
+
+        if (    cTotalRefs == pSet->aEntries[i].cUnrefs
+            &&  (   pSet->iSubset == UINT32_MAX
+                 || pSet->iSubset < pSet->cEntries)
+           )
+        {
+            Log2(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
+            //LogFlow(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
+            pgmRZDynMapReleasePage(PGMRZDYNMAP_SET_2_DYNMAP(pSet),
+                                   pSet->aEntries[i].iPage,
+                                   pSet->aEntries[i].cRefs);
+            pSet->cEntries--;
+            if (i < pSet->cEntries)
+            {
+                pSet->aEntries[i] = pSet->aEntries[pSet->cEntries];
+                PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]);
+            }
+
+            i--;
+        }
+    }
+}
+
+
+
+
+/**
+ * Signals the start of a new set of mappings.
+ *
+ * Mostly for strictness. PGMDynMapHCPage won't work unless this
+ * API is called.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(void) PGMRZDynMapStartAutoSet(PVMCPU pVCpu)
+{
+    LogFlow(("PGMRZDynMapStartAutoSet:\n"));
+    Assert(pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED);
+    Assert(pVCpu->pgm.s.AutoSet.iSubset == UINT32_MAX);
+    pVCpu->pgm.s.AutoSet.cEntries = 0;
+    pVCpu->pgm.s.AutoSet.iCpu = PGMRZDYNMAP_CUR_CPU();
+}
+
+
+#ifdef IN_RING0
+/**
+ * Starts or migrates the autoset of a virtual CPU.
+ *
+ * This is used by HMR0Enter.  When we've longjumped out of the HM
+ * execution loop with the set open, we'll migrate it when re-entering.  While
+ * under normal circumstances, we'll start it so VMXR0LoadGuestState can access
+ * guest memory.
+ *
+ * @returns @c true if started, @c false if migrated.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT
+ */
+VMMR0DECL(bool) PGMR0DynMapStartOrMigrateAutoSet(PVMCPU pVCpu)
+{
+    bool fStartIt = pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED;
+    if (fStartIt)
+        PGMRZDynMapStartAutoSet(pVCpu);
+    else
+        PGMR0DynMapMigrateAutoSet(pVCpu);
+    return fStartIt;
+}
+#endif /* IN_RING0 */
+
+
+/**
+ * Checks if the set has high load.
+ *
+ * @returns true on high load, otherwise false.
+ * @param   pSet            The set.
+ */
+DECLINLINE(bool) pgmRZDynMapHasHighLoad(PPGMMAPSET pSet)
+{
+#ifdef IN_RC
+    if (pSet->cEntries < MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE / 2)
+        return false;
+#endif
+
+    PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
+    uint32_t cUnusedPages = pThis->cPages - pThis->cLoad;
+#ifdef IN_RC
+    return cUnusedPages <= MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE * 36 / 100;
+#else
+    return cUnusedPages <= PGMR0DYNMAP_PAGES_PER_CPU_MIN;
+#endif
+}
+
+
+/**
+ * Worker that performs the actual flushing of the set.
+ *
+ * @param   pSet        The set to flush.
+ * @param   cEntries    The number of entries.
+ */
+DECLINLINE(void) pgmDynMapFlushAutoSetWorker(PPGMMAPSET pSet, uint32_t cEntries)
+{
+    /*
+     * Release any pages it's referencing.
+     */
+    if (    cEntries != 0
+        &&  RT_LIKELY(cEntries <= RT_ELEMENTS(pSet->aEntries)))
+    {
+        PPGMRZDYNMAP    pThis   = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
+        PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
+
+        uint32_t i = cEntries;
+        while (i-- > 0)
+        {
+            uint32_t iPage = pSet->aEntries[i].iPage;
+            Assert(iPage < pThis->cPages);
+            int32_t  cRefs = pSet->aEntries[i].cRefs;
+            Assert(cRefs > 0);
+            pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
+
+            PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]);
+        }
+
+        Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages);
+        PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
+    }
+}
+
+
+/**
+ * Releases the dynamic memory mappings made by PGMDynMapHCPage and associates
+ * since the PGMDynMapStartAutoSet call.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(void) PGMRZDynMapReleaseAutoSet(PVMCPU pVCpu)
+{
+    PPGMMAPSET  pSet = &pVCpu->pgm.s.AutoSet;
+
+    /*
+     * Close and flush the set.
+     */
+    uint32_t    cEntries = pSet->cEntries;
+    AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
+    pSet->cEntries = PGMMAPSET_CLOSED;
+    pSet->iSubset = UINT32_MAX;
+    pSet->iCpu = -1;
+
+#ifdef IN_RC
+    if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
+    else
+#endif
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
+    if (cEntries > RT_ELEMENTS(pSet->aEntries) * 50 / 100)
+        Log(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries));
+    else
+        LogFlow(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries));
+
+    pgmDynMapFlushAutoSetWorker(pSet, cEntries);
+}
+
+
+/**
+ * Flushes the set if it's above a certain threshold.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(void) PGMRZDynMapFlushAutoSet(PVMCPU pVCpu)
+{
+    PPGMMAPSET  pSet = &pVCpu->pgm.s.AutoSet;
+    AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
+
+    /*
+     * Only flush it if it's 45% full.
+     */
+    uint32_t cEntries = pSet->cEntries;
+    AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
+    Assert(pSet->iSubset == UINT32_MAX);
+#ifdef IN_RC
+    if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
+    else
+#endif
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
+    if (   cEntries >= RT_ELEMENTS(pSet->aEntries) * 45 / 100
+        || pgmRZDynMapHasHighLoad(pSet))
+    {
+        pSet->cEntries = 0;
+        Log(("PGMDynMapFlushAutoSet: cEntries=%d\n", pSet->cEntries));
+
+        pgmDynMapFlushAutoSetWorker(pSet, cEntries);
+        AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
+    }
+}
+
+
+#ifndef IN_RC
+/**
+ * Migrates the automatic mapping set of the current vCPU if it's active and
+ * necessary.
+ *
+ * This is called when re-entering the hardware assisted execution mode after a
+ * nip down to ring-3.  We run the risk that the CPU might have change and we
+ * will therefore make sure all the cache entries currently in the auto set will
+ * be valid on the new CPU.  If the cpu didn't change nothing will happen as all
+ * the entries will have been flagged as invalidated.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT
+ */
+VMMR0DECL(void) PGMR0DynMapMigrateAutoSet(PVMCPU pVCpu)
+{
+    LogFlow(("PGMR0DynMapMigrateAutoSet\n"));
+    PPGMMAPSET      pSet     = &pVCpu->pgm.s.AutoSet;
+    int32_t         iRealCpu = PGMRZDYNMAP_CUR_CPU();
+    if (pSet->iCpu != iRealCpu)
+    {
+        uint32_t    i        = pSet->cEntries;
+        if (i != PGMMAPSET_CLOSED)
+        {
+            AssertMsg(i <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", i, i));
+            if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries)))
+            {
+                PPGMRZDYNMAP    pThis  = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
+                PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
+
+                while (i-- > 0)
+                {
+                    Assert(pSet->aEntries[i].cRefs > 0);
+                    uint32_t iPage = pSet->aEntries[i].iPage;
+                    Assert(iPage < pThis->cPages);
+                    if (RTCpuSetIsMemberByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu))
+                    {
+                        RTCpuSetDelByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu);
+                        PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
+
+                        ASMInvalidatePage((uintptr_t)pThis->paPages[iPage].pvPage);
+                        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapMigrateInvlPg);
+
+                        PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis);
+                    }
+                }
+
+                PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
+            }
+        }
+        pSet->iCpu = iRealCpu;
+    }
+}
+#endif /* !IN_RC */
+
+
+/**
+ * Worker function that flushes the current subset.
+ *
+ * This is called when the set is popped or when the set
+ * hash a too high load. As also pointed out elsewhere, the
+ * whole subset thing is a hack for working around code that
+ * accesses too many pages. Like PGMPool.
+ *
+ * @param   pSet        The set which subset to flush.
+ */
+static void pgmDynMapFlushSubset(PPGMMAPSET pSet)
+{
+    uint32_t iSubset = pSet->iSubset;
+    uint32_t i       = pSet->cEntries;
+    Assert(i <= RT_ELEMENTS(pSet->aEntries));
+    if (    i > iSubset
+        &&  i <= RT_ELEMENTS(pSet->aEntries))
+    {
+        Log(("pgmDynMapFlushSubset: cEntries=%d iSubset=%d\n", pSet->cEntries, iSubset));
+        pSet->cEntries = iSubset;
+
+        PPGMRZDYNMAP    pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
+        PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
+
+        while (i-- > iSubset)
+        {
+            uint32_t iPage = pSet->aEntries[i].iPage;
+            Assert(iPage < pThis->cPages);
+            int32_t  cRefs = pSet->aEntries[i].cRefs;
+            Assert(cRefs > 0);
+            pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
+
+            PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]);
+        }
+
+        PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
+    }
+}
+
+
+/**
+ * Creates a subset.
+ *
+ * A subset is a hack to avoid having to rewrite code that touches a lot of
+ * pages. It prevents the mapping set from being overflowed by automatically
+ * flushing previous mappings when a certain threshold is reached.
+ *
+ * Pages mapped after calling this function are only valid until the next page
+ * is mapped.
+ *
+ * @returns The index of the previous subset. Pass this to
+ *          PGMDynMapPopAutoSubset when popping it.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(uint32_t) PGMRZDynMapPushAutoSubset(PVMCPU pVCpu)
+{
+    PPGMMAPSET      pSet = &pVCpu->pgm.s.AutoSet;
+    AssertReturn(pSet->cEntries != PGMMAPSET_CLOSED, UINT32_MAX);
+    uint32_t        iPrevSubset = pSet->iSubset;
+    LogFlow(("PGMRZDynMapPushAutoSubset: pVCpu=%p iPrevSubset=%u\n", pVCpu, iPrevSubset));
+
+    /*
+     * If it looks like we're approaching the max set size or mapping space
+     * optimize the set to drop off unused pages.
+     */
+    if (   pSet->cEntries > RT_ELEMENTS(pSet->aEntries) * 60 / 100
+        || pgmRZDynMapHasHighLoad(pSet))
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
+        pgmDynMapOptimizeAutoSet(pSet);
+    }
+
+    pSet->iSubset = pSet->cEntries;
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSubsets);
+
+    AssertMsg(iPrevSubset <= pSet->iSubset || iPrevSubset == UINT32_MAX, ("iPrevSubset=%#x iSubset=%#x\n", iPrevSubset, pSet->iSubset));
+    return iPrevSubset;
+}
+
+
+/**
+ * Pops a subset created by a previous call to PGMDynMapPushAutoSubset.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   iPrevSubset     What PGMDynMapPushAutoSubset returned.
+ */
+VMMDECL(void) PGMRZDynMapPopAutoSubset(PVMCPU pVCpu, uint32_t iPrevSubset)
+{
+    PPGMMAPSET      pSet = &pVCpu->pgm.s.AutoSet;
+    uint32_t        cEntries = pSet->cEntries;
+    LogFlow(("PGMRZDynMapPopAutoSubset: pVCpu=%p iPrevSubset=%u iSubset=%u cEntries=%u\n", pVCpu, iPrevSubset, pSet->iSubset, cEntries));
+    AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
+    AssertMsgReturnVoid(pSet->iSubset >= iPrevSubset || iPrevSubset == UINT32_MAX, ("iPrevSubset=%u iSubset=%u cEntries=%u\n", iPrevSubset, pSet->iSubset, cEntries));
+#ifdef IN_RC
+    if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
+    else
+#endif
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
+    if (    cEntries >= RT_ELEMENTS(pSet->aEntries) * 40 / 100
+        &&  cEntries != pSet->iSubset)
+    {
+        pgmDynMapFlushSubset(pSet);
+        Assert(pSet->cEntries >= iPrevSubset || iPrevSubset == UINT32_MAX);
+    }
+    pSet->iSubset = iPrevSubset;
+}
+
+
+/**
+ * Indicates that the given page is unused and its mapping can be re-used.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pvHint          The page that is now unused.  This does not have to
+ *                          point at the start of the page.  NULL is ignored.
+ */
+#ifdef LOG_ENABLED
+void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint, RT_SRC_POS_DECL)
+#else
+void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint)
+#endif
+{
+    /*
+     * Ignore NULL pointers and mask off the page offset bits.
+     */
+    if (pvHint == NULL)
+        return;
+    pvHint = (void *)((uintptr_t)pvHint & ~(uintptr_t)PAGE_OFFSET_MASK);
+
+    PPGMMAPSET  pSet    = &pVCpu->pgm.s.AutoSet;
+    uint32_t    iEntry  = pSet->cEntries;
+    AssertReturnVoid(iEntry > 0);
+
+    /*
+     * Find the entry in the usual unrolled fashion.
+     */
+    /** @todo add a hint to the set which entry was used last since it's not
+     *        always the last entry? */
+#define IS_MATCHING_ENTRY(pSet, iEntry, pvHint) \
+        (   (pSet)->aEntries[(iEntry)].pvPage == (pvHint) \
+         &&   (uint32_t)(pSet)->aEntries[(iEntry)].cRefs + (pSet)->aEntries[(iEntry)].cInlinedRefs \
+            > (pSet)->aEntries[(iEntry)].cUnrefs )
+    if (     iEntry >= 1 && IS_MATCHING_ENTRY(pSet, iEntry - 1, pvHint))
+        iEntry = iEntry - 1;
+    else if (iEntry >= 2 && IS_MATCHING_ENTRY(pSet, iEntry - 2, pvHint))
+        iEntry = iEntry - 2;
+    else if (iEntry >= 3 && IS_MATCHING_ENTRY(pSet, iEntry - 3, pvHint))
+        iEntry = iEntry - 3;
+    else if (iEntry >= 4 && IS_MATCHING_ENTRY(pSet, iEntry - 4, pvHint))
+        iEntry = iEntry - 4;
+    else if (iEntry >= 5 && IS_MATCHING_ENTRY(pSet, iEntry - 5, pvHint))
+        iEntry = iEntry - 5;
+    else if (iEntry >= 6 && IS_MATCHING_ENTRY(pSet, iEntry - 6, pvHint))
+        iEntry = iEntry - 6;
+    else if (iEntry >= 7 && IS_MATCHING_ENTRY(pSet, iEntry - 7, pvHint))
+        iEntry = iEntry - 7;
+    else
+    {
+        /*
+         * Loop till we find it.
+         */
+        bool fFound = false;
+        if (iEntry > 7)
+        {
+            iEntry -= 7;
+            while (iEntry-- > 0)
+                if (IS_MATCHING_ENTRY(pSet, iEntry, pvHint))
+                {
+                    fFound = true;
+                    break;
+                }
+        }
+        AssertMsgReturnVoid(fFound,
+                            ("pvHint=%p cEntries=%#x iSubset=%#x\n"
+                             "aEntries[0] = {%#x, %#x, %#x, %#x, %p}\n"
+                             "aEntries[1] = {%#x, %#x, %#x, %#x, %p}\n"
+                             "aEntries[2] = {%#x, %#x, %#x, %#x, %p}\n"
+                             "aEntries[3] = {%#x, %#x, %#x, %#x, %p}\n"
+                             "aEntries[4] = {%#x, %#x, %#x, %#x, %p}\n"
+                             "aEntries[5] = {%#x, %#x, %#x, %#x, %p}\n"
+                             ,
+                             pvHint, pSet->cEntries, pSet->iSubset,
+                             pSet->aEntries[0].iPage, pSet->aEntries[0].cRefs, pSet->aEntries[0].cInlinedRefs, pSet->aEntries[0].cUnrefs, pSet->aEntries[0].pvPage,
+                             pSet->aEntries[1].iPage, pSet->aEntries[1].cRefs, pSet->aEntries[1].cInlinedRefs, pSet->aEntries[1].cUnrefs, pSet->aEntries[1].pvPage,
+                             pSet->aEntries[2].iPage, pSet->aEntries[2].cRefs, pSet->aEntries[2].cInlinedRefs, pSet->aEntries[2].cUnrefs, pSet->aEntries[2].pvPage,
+                             pSet->aEntries[3].iPage, pSet->aEntries[3].cRefs, pSet->aEntries[3].cInlinedRefs, pSet->aEntries[3].cUnrefs, pSet->aEntries[3].pvPage,
+                             pSet->aEntries[4].iPage, pSet->aEntries[4].cRefs, pSet->aEntries[4].cInlinedRefs, pSet->aEntries[4].cUnrefs, pSet->aEntries[4].pvPage,
+                             pSet->aEntries[5].iPage, pSet->aEntries[5].cRefs, pSet->aEntries[5].cInlinedRefs, pSet->aEntries[5].cUnrefs, pSet->aEntries[5].pvPage));
+    }
+#undef IS_MATCHING_ENTRY
+
+    /*
+     * Update it.
+     */
+    uint32_t const  cTotalRefs = (uint32_t)pSet->aEntries[iEntry].cRefs + pSet->aEntries[iEntry].cInlinedRefs;
+    uint32_t const  cUnrefs    = pSet->aEntries[iEntry].cUnrefs;
+    LogFlow(("pgmRZDynMapUnusedHint: pvHint=%p #%u cRefs=%d cInlinedRefs=%d cUnrefs=%d (+1) cTotalRefs=%d %s(%d) %s\n",
+             pvHint, iEntry, pSet->aEntries[iEntry].cRefs, pSet->aEntries[iEntry].cInlinedRefs, cUnrefs, cTotalRefs, pszFile, iLine, pszFunction));
+    AssertReturnVoid(cTotalRefs > cUnrefs);
+
+    if (RT_LIKELY(cUnrefs < UINT16_MAX - 1))
+        pSet->aEntries[iEntry].cUnrefs++;
+    else if (pSet->aEntries[iEntry].cInlinedRefs)
+    {
+        uint32_t cSub = RT_MIN(pSet->aEntries[iEntry].cInlinedRefs, pSet->aEntries[iEntry].cUnrefs);
+        pSet->aEntries[iEntry].cInlinedRefs -= cSub;
+        pSet->aEntries[iEntry].cUnrefs      -= cSub;
+        pSet->aEntries[iEntry].cUnrefs++;
+    }
+    else
+        Log(("pgmRZDynMapUnusedHint: pvHint=%p ignored because of overflow! %s(%d) %s\n", pvHint, pszFile, iLine, pszFunction));
+
+#ifdef PGMRZDYNMAP_STRICT_RELEASE
+    /*
+     * Optimize the set to trigger the unmapping and invalidation of the page.
+     */
+    if (cUnrefs + 1 == cTotalRefs)
+        pgmDynMapOptimizeAutoSet(pSet);
+#endif
+}
+
+
+/**
+ * Common worker code for pgmRZDynMapHCPageInlined, pgmRZDynMapHCPageV2Inlined
+ * and pgmR0DynMapGCPageOffInlined.
+ *
+ * @returns VINF_SUCCESS, bails out to ring-3 on failure.
+ * @param   pSet        The set.
+ * @param   HCPhys      The physical address of the page.
+ * @param   ppv         Where to store the address of the mapping on success.
+ *
+ * @remarks This is a very hot path.
+ */
+int pgmRZDynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
+{
+    AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
+    PVMCPU pVCpu = PGMRZDYNMAP_SET_2_VMCPU(pSet);
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
+
+    /*
+     * Map it.
+     */
+    void           *pvPage;
+    PPGMRZDYNMAP    pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
+    uint32_t        iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage);
+    if (RT_UNLIKELY(iPage == UINT32_MAX))
+    {
+        /*
+         * We're out of mapping space, optimize our set to try remedy the
+         * situation.  (Only works if there are unreference hints.)
+         */
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
+        pgmDynMapOptimizeAutoSet(pSet);
+
+        iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage);
+        if (RT_UNLIKELY(iPage == UINT32_MAX))
+        {
+            RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: cLoad=%u/%u cPages=%u cGuardPages=%u\n",
+                             pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pThis->cGuardPages);
+            if (!g_fPGMR0DynMapTestRunning)
+                VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
+            *ppv = NULL;
+            STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
+            return VERR_PGM_DYNMAP_FAILED;
+        }
+    }
+
+    /*
+     * Add the page to the auto reference set.
+     *
+     * The typical usage pattern means that the same pages will be mapped
+     * several times in the same set. We can catch most of these
+     * remappings by looking a few pages back into the set. (The searching
+     * and set optimizing path will hardly ever be used when doing this.)
+     */
+    AssertCompile(RT_ELEMENTS(pSet->aEntries) >= 8);
+    int32_t i = pSet->cEntries;
+    if (i-- < 5)
+    {
+        unsigned iEntry = pSet->cEntries++;
+        pSet->aEntries[iEntry].cRefs        = 1;
+        pSet->aEntries[iEntry].cUnrefs      = 0;
+        pSet->aEntries[iEntry].cInlinedRefs = 0;
+        pSet->aEntries[iEntry].iPage        = iPage;
+        pSet->aEntries[iEntry].pvPage       = pvPage;
+        pSet->aEntries[iEntry].HCPhys       = HCPhys;
+        pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/0/0 iPage=%#x  [a] %s(%d) %s\n",
+                 pSet, HCPhys, iEntry, iEntry + 1, pvPage, 1, iPage, pszFile, iLine, pszFunction));
+    }
+    /* Any of the last 5 pages? */
+    else if (   pSet->aEntries[i - 0].iPage == iPage
+             && pSet->aEntries[i - 0].cRefs < UINT16_MAX - 1)
+    {
+        pSet->aEntries[i - 0].cRefs++;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [0] %s(%d) %s\n", pSet, HCPhys, i - 0, pSet->cEntries, pvPage, pSet->aEntries[i - 0].cRefs, pSet->aEntries[i - 0].cInlinedRefs, pSet->aEntries[i - 0].cUnrefs, iPage, pszFile, iLine, pszFunction));
+    }
+    else if (   pSet->aEntries[i - 1].iPage == iPage
+             && pSet->aEntries[i - 1].cRefs < UINT16_MAX - 1)
+    {
+        pSet->aEntries[i - 1].cRefs++;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [1] %s(%d) %s\n", pSet, HCPhys, i - 1, pSet->cEntries, pvPage, pSet->aEntries[i - 1].cRefs, pSet->aEntries[i - 1].cInlinedRefs, pSet->aEntries[i - 1].cUnrefs, iPage, pszFile, iLine, pszFunction));
+    }
+    else if (   pSet->aEntries[i - 2].iPage == iPage
+             && pSet->aEntries[i - 2].cRefs < UINT16_MAX - 1)
+    {
+        pSet->aEntries[i - 2].cRefs++;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [2] %s(%d) %s\n", pSet, HCPhys, i - 2, pSet->cEntries, pvPage, pSet->aEntries[i - 2].cRefs, pSet->aEntries[i - 2].cInlinedRefs, pSet->aEntries[i - 2].cUnrefs, iPage, pszFile, iLine, pszFunction));
+    }
+    else if (   pSet->aEntries[i - 3].iPage == iPage
+             && pSet->aEntries[i - 3].cRefs < UINT16_MAX - 1)
+    {
+        pSet->aEntries[i - 3].cRefs++;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 3, pSet->cEntries, pvPage, pSet->aEntries[i - 3].cRefs, pSet->aEntries[i - 3].cInlinedRefs, pSet->aEntries[i - 3].cUnrefs, iPage, pszFile, iLine, pszFunction));
+    }
+    else if (   pSet->aEntries[i - 4].iPage == iPage
+             && pSet->aEntries[i - 4].cRefs < UINT16_MAX - 1)
+    {
+        pSet->aEntries[i - 4].cRefs++;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 4, pSet->cEntries, pvPage, pSet->aEntries[i - 4].cRefs, pSet->aEntries[i - 4].cInlinedRefs, pSet->aEntries[i - 4].cUnrefs, iPage, pszFile, iLine, pszFunction));
+    }
+    /* Don't bother searching unless we're above a 60% load. */
+    else if (RT_LIKELY(i <= (int32_t)RT_ELEMENTS(pSet->aEntries) * 60 / 100))
+    {
+        unsigned iEntry = pSet->cEntries++;
+        pSet->aEntries[iEntry].cRefs        = 1;
+        pSet->aEntries[iEntry].cUnrefs      = 0;
+        pSet->aEntries[iEntry].cInlinedRefs = 0;
+        pSet->aEntries[iEntry].iPage        = iPage;
+        pSet->aEntries[iEntry].pvPage       = pvPage;
+        pSet->aEntries[iEntry].HCPhys       = HCPhys;
+        pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [b] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
+    }
+    else
+    {
+        /* Search the rest of the set. */
+        Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
+        i -= 4;
+        while (i-- > 0)
+            if (    pSet->aEntries[i].iPage == iPage
+                &&  pSet->aEntries[i].cRefs < UINT16_MAX - 1)
+            {
+                pSet->aEntries[i].cRefs++;
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchHits);
+                LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [c] %s(%d) %s\n", pSet, HCPhys, i, pSet->cEntries, pvPage, pSet->aEntries[i].cRefs, pSet->aEntries[i].cInlinedRefs, pSet->aEntries[i].cUnrefs, iPage, pszFile, iLine, pszFunction));
+                break;
+            }
+        if (i < 0)
+        {
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchMisses);
+#if 0 /* this is very bogus */
+            if (pSet->iSubset < pSet->cEntries)
+            {
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchFlushes);
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(pSet->cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
+                pgmDynMapFlushSubset(pSet);
+            }
+#endif
+
+            if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries)))
+            {
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
+                pgmDynMapOptimizeAutoSet(pSet);
+            }
+
+            if (RT_LIKELY(pSet->cEntries < RT_ELEMENTS(pSet->aEntries)))
+            {
+                unsigned iEntry = pSet->cEntries++;
+                pSet->aEntries[iEntry].cRefs        = 1;
+                pSet->aEntries[iEntry].cUnrefs      = 0;
+                pSet->aEntries[iEntry].cInlinedRefs = 0;
+                pSet->aEntries[iEntry].iPage        = iPage;
+                pSet->aEntries[iEntry].pvPage       = pvPage;
+                pSet->aEntries[iEntry].HCPhys       = HCPhys;
+                pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
+                LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [d] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
+            }
+            else
+            {
+                /* We're screwed. */
+                pgmRZDynMapReleasePage(pThis, iPage, 1);
+
+                RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: set is full!\n");
+                if (!g_fPGMR0DynMapTestRunning)
+                    VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
+                *ppv = NULL;
+                STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
+                return VERR_PGM_DYNMAP_FULL_SET;
+            }
+        }
+    }
+
+    *ppv = pvPage;
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
+    return VINF_SUCCESS;
+}
+
+
+#if 0 /*def DEBUG*/
+/** For pgmR0DynMapTest3PerCpu. */
+typedef struct PGMR0DYNMAPTEST
+{
+    uint32_t            u32Expect;
+    uint32_t           *pu32;
+    uint32_t volatile   cFailures;
+} PGMR0DYNMAPTEST;
+typedef PGMR0DYNMAPTEST *PPGMR0DYNMAPTEST;
+
+/**
+ * Checks that the content of the page is the same on all CPUs, i.e. that there
+ * are no CPU specific PTs or similar nasty stuff involved.
+ *
+ * @param   idCpu           The current CPU.
+ * @param   pvUser1         Pointer a PGMR0DYNMAPTEST structure.
+ * @param   pvUser2         Unused, ignored.
+ */
+static DECLCALLBACK(void) pgmR0DynMapTest3PerCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    PPGMR0DYNMAPTEST    pTest = (PPGMR0DYNMAPTEST)pvUser1;
+    ASMInvalidatePage(pTest->pu32);
+    if (*pTest->pu32 != pTest->u32Expect)
+        ASMAtomicIncU32(&pTest->cFailures);
+    NOREF(pvUser2); NOREF(idCpu);
+}
+
+
+/**
+ * Performs some basic tests in debug builds.
+ */
+static int pgmR0DynMapTest(PVM pVM)
+{
+    LogRel(("pgmR0DynMapTest: ****** START ******\n"));
+    PPGMMAPSET      pSet  = &pVM->aCpus[0].pgm.s.AutoSet;
+    PPGMRZDYNMAP    pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
+    uint32_t        i;
+
+    /*
+     * Assert internal integrity first.
+     */
+    LogRel(("Test #0\n"));
+    int rc = PGMR0DynMapAssertIntegrity();
+    if (RT_FAILURE(rc))
+        return rc;
+
+    void           *pvR0DynMapUsedSaved = pVM->pgm.s.pvR0DynMapUsed;
+    pVM->pgm.s.pvR0DynMapUsed = pThis;
+    g_fPGMR0DynMapTestRunning = true;
+
+    /*
+     * Simple test, map CR3 twice and check that we're getting the
+     * same mapping address back.
+     */
+    LogRel(("Test #1\n"));
+    ASMIntDisable();
+    PGMRZDynMapStartAutoSet(&pVM->aCpus[0]);
+
+    uint64_t cr3 = ASMGetCR3() & ~(uint64_t)PAGE_OFFSET_MASK;
+    void    *pv  = (void *)(intptr_t)-1;
+    void    *pv2 = (void *)(intptr_t)-2;
+    rc           = pgmRZDynMapHCPageCommon(pVM, cr3, &pv  RTLOG_COMMA_SRC_POS);
+    int      rc2 = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS);
+    ASMIntEnable();
+    if (    RT_SUCCESS(rc2)
+        &&  RT_SUCCESS(rc)
+        &&  pv == pv2)
+    {
+        LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
+        rc = PGMR0DynMapAssertIntegrity();
+
+        /*
+         * Check that the simple set overflow code works by filling it
+         * with more CR3 mappings.
+         */
+        LogRel(("Test #2\n"));
+        ASMIntDisable();
+        PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
+        for (i = 0 ; i < UINT16_MAX*2 - 1 && RT_SUCCESS(rc) && pv2 == pv; i++)
+        {
+            pv2 = (void *)(intptr_t)-4;
+            rc = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS);
+        }
+        ASMIntEnable();
+        if (RT_FAILURE(rc) || pv != pv2)
+        {
+            LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%p\n", __LINE__, rc, pv, pv2, i));
+            if (RT_SUCCESS(rc)) rc = VERR_PGM_DYNMAP_IPE;
+        }
+        else if (pSet->cEntries != 5)
+        {
+            LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries) / 2));
+            rc = VERR_PGM_DYNMAP_IPE;
+        }
+        else if (   pSet->aEntries[4].cRefs != UINT16_MAX - 1
+                 || pSet->aEntries[3].cRefs != UINT16_MAX - 1
+                 || pSet->aEntries[2].cRefs != 1
+                 || pSet->aEntries[1].cRefs != 1
+                 || pSet->aEntries[0].cRefs != 1)
+        {
+            LogRel(("failed(%d): bad set dist: ", __LINE__));
+            for (i = 0; i < pSet->cEntries; i++)
+                LogRel(("[%d]=%d, ", i, pSet->aEntries[i].cRefs));
+            LogRel(("\n"));
+            rc = VERR_PGM_DYNMAP_IPE;
+        }
+        if (RT_SUCCESS(rc))
+            rc = PGMR0DynMapAssertIntegrity();
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Trigger an set optimization run (exactly).
+             */
+            LogRel(("Test #3\n"));
+            ASMIntDisable();
+            PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
+            pv2 = NULL;
+            for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) - 5 && RT_SUCCESS(rc) && pv2 != pv; i++)
+            {
+                pv2 = (void *)(intptr_t)(-5 - i);
+                rc = pgmRZDynMapHCPageCommon(pVM, cr3 + PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS);
+            }
+            ASMIntEnable();
+            if (RT_FAILURE(rc) || pv == pv2)
+            {
+                LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%d\n", __LINE__, rc, pv, pv2, i));
+                if (RT_SUCCESS(rc)) rc = VERR_PGM_DYNMAP_IPE;
+            }
+            else if (pSet->cEntries != RT_ELEMENTS(pSet->aEntries))
+            {
+                LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
+                rc = VERR_PGM_DYNMAP_IPE;
+            }
+            LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
+            if (RT_SUCCESS(rc))
+                rc = PGMR0DynMapAssertIntegrity();
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Trigger an overflow error.
+                 */
+                LogRel(("Test #4\n"));
+                ASMIntDisable();
+                PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
+                for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) + 2; i++)
+                {
+                    rc = pgmRZDynMapHCPageCommon(pVM, cr3 - PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS);
+                    if (RT_SUCCESS(rc))
+                        rc = PGMR0DynMapAssertIntegrity();
+                    if (RT_FAILURE(rc))
+                        break;
+                }
+                ASMIntEnable();
+                if (rc == VERR_PGM_DYNMAP_FULL_SET)
+                {
+                    /* flush the set. */
+                    LogRel(("Test #5\n"));
+                    ASMIntDisable();
+                    PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
+                    PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]);
+                    PGMRZDynMapStartAutoSet(&pVM->aCpus[0]);
+                    ASMIntEnable();
+
+                    rc = PGMR0DynMapAssertIntegrity();
+                }
+                else
+                {
+                    LogRel(("failed(%d): rc=%Rrc, wanted %d ; pv2=%p Set=%u/%u; i=%d\n", __LINE__,
+                            rc, VERR_PGM_DYNMAP_FULL_SET, pv2, pSet->cEntries, RT_ELEMENTS(pSet->aEntries), i));
+                    if (RT_SUCCESS(rc)) rc = VERR_PGM_DYNMAP_IPE;
+                }
+            }
+        }
+    }
+    else
+    {
+        LogRel(("failed(%d): rc=%Rrc rc2=%Rrc; pv=%p pv2=%p\n", __LINE__, rc, rc2, pv, pv2));
+        if (RT_SUCCESS(rc))
+            rc = rc2;
+    }
+
+    /*
+     * Check that everyone sees the same stuff.
+     */
+    if (RT_SUCCESS(rc))
+    {
+        LogRel(("Test #5\n"));
+        ASMIntDisable();
+        PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
+        RTHCPHYS  HCPhysPT = RTR0MemObjGetPagePhysAddr(pThis->pSegHead->ahMemObjPTs[0], 0);
+        rc  = pgmRZDynMapHCPageCommon(pVM, HCPhysPT, &pv RTLOG_COMMA_SRC_POS);
+        if (RT_SUCCESS(rc))
+        {
+            PGMR0DYNMAPTEST Test;
+            uint32_t       *pu32Real = &pThis->paPages[pThis->pSegHead->iPage].uPte.pLegacy->u;
+            Test.pu32       = (uint32_t *)((uintptr_t)pv | ((uintptr_t)pu32Real & PAGE_OFFSET_MASK));
+            Test.u32Expect  = *pu32Real;
+            ASMAtomicWriteU32(&Test.cFailures, 0);
+            ASMIntEnable();
+
+            rc = RTMpOnAll(pgmR0DynMapTest3PerCpu, &Test, NULL);
+            if (RT_FAILURE(rc))
+                LogRel(("failed(%d): RTMpOnAll rc=%Rrc\n", __LINE__, rc));
+            else if (Test.cFailures)
+            {
+                LogRel(("failed(%d): cFailures=%d pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n", __LINE__,
+                        Test.cFailures, pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
+                rc = VERR_PGM_DYNMAP_IPE;
+            }
+            else
+                LogRel(("pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n",
+                        pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
+        }
+        else
+        {
+            ASMIntEnable();
+            LogRel(("failed(%d): rc=%Rrc\n", rc));
+        }
+    }
+
+    /*
+     * Clean up.
+     */
+    LogRel(("Cleanup.\n"));
+    ASMIntDisable();
+    PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
+    PGMRZDynMapFlushAutoSet(&pVM->aCpus[0]);
+    PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]);
+    ASMIntEnable();
+
+    if (RT_SUCCESS(rc))
+        rc = PGMR0DynMapAssertIntegrity();
+    else
+        PGMR0DynMapAssertIntegrity();
+
+    g_fPGMR0DynMapTestRunning = false;
+    LogRel(("Result: rc=%Rrc Load=%u/%u/%u Set=%#x/%u\n", rc,
+            pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
+    pVM->pgm.s.pvR0DynMapUsed = pvR0DynMapUsedSaved;
+    LogRel(("pgmR0DynMapTest: ****** END ******\n"));
+    return rc;
+}
+#endif /* DEBUG */
+
diff --git a/src/VBox/VMM/VMMRZ/VMMRZ.cpp b/src/VBox/VMM/VMMRZ/VMMRZ.cpp
new file mode 100644
index 00000000..fd8ccd5c
--- /dev/null
+++ b/src/VBox/VMM/VMMRZ/VMMRZ.cpp
@@ -0,0 +1,253 @@
+/* $Id: VMMRZ.cpp $ */
+/** @file
+ * VMM - Virtual Machine Monitor, Raw-mode and ring-0 context code.
+ */
+
+/*
+ * Copyright (C) 2009-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <VBox/vmm/vmm.h>
+#include "VMMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+
+#include <iprt/assert.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/string.h>
+
+
+/**
+ * Calls the ring-3 host code.
+ *
+ * @returns VBox status code of the ring-3 call.
+ * @retval  VERR_VMM_RING3_CALL_DISABLED if called at the wrong time. This must
+ *          be passed up the stack, or if that isn't possible then VMMRZCallRing3
+ *          needs to change it into an assertion.
+ *
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   enmOperation    The operation.
+ * @param   uArg            The argument to the operation.
+ */
+VMMRZDECL(int) VMMRZCallRing3(PVMCC pVM, PVMCPUCC pVCpu, VMMCALLRING3 enmOperation, uint64_t uArg)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Check if calling ring-3 has been disabled and only let let fatal calls thru.
+     */
+    if (RT_UNLIKELY(    pVCpu->vmm.s.cCallRing3Disabled != 0
+                    &&  enmOperation != VMMCALLRING3_VM_R0_ASSERTION))
+    {
+#ifndef IN_RING0
+        /*
+         * In most cases, it's sufficient to return a status code which
+         * will then be propagated up the code usually encountering several
+         * AssertRC invocations along the way. Hitting one of those is more
+         * helpful than stopping here.
+         *
+         * However, some doesn't check the status code because they are called
+         * from void functions, and for these we'll turn this into a ring-0
+         * assertion host call.
+         */
+        if (enmOperation != VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS)
+            return VERR_VMM_RING3_CALL_DISABLED;
+#endif
+#ifdef IN_RC
+        RTStrPrintf(g_szRTAssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1),
+                    "VMMRZCallRing3: enmOperation=%d uArg=%#llx idCpu=%#x cCallRing3Disabled=%#x\n",
+                    enmOperation, uArg, pVCpu->idCpu, pVCpu->vmm.s.cCallRing3Disabled);
+#endif
+        RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1),
+                    "VMMRZCallRing3: enmOperation=%d uArg=%#llx idCpu=%#x cCallRing3Disabled=%#x\n",
+                    enmOperation, uArg, pVCpu->idCpu, pVCpu->vmm.s.cCallRing3Disabled);
+        enmOperation = VMMCALLRING3_VM_R0_ASSERTION;
+    }
+
+    /*
+     * The normal path.
+     */
+/** @todo profile this! */
+    pVCpu->vmm.s.enmCallRing3Operation = enmOperation;
+    pVCpu->vmm.s.u64CallRing3Arg = uArg;
+    pVCpu->vmm.s.rcCallRing3 = VERR_VMM_RING3_CALL_NO_RC;
+#ifdef IN_RC
+    pVM->vmm.s.pfnRCToHost(VINF_VMM_CALL_HOST);
+#else
+    int rc;
+    if (pVCpu->vmm.s.pfnCallRing3CallbackR0)
+    {
+        rc = pVCpu->vmm.s.pfnCallRing3CallbackR0(pVCpu, enmOperation, pVCpu->vmm.s.pvCallRing3CallbackUserR0);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+    rc = vmmR0CallRing3LongJmp(&pVCpu->vmm.s.CallRing3JmpBufR0, VINF_VMM_CALL_HOST);
+    if (RT_FAILURE(rc))
+        return rc;
+#endif
+    return pVCpu->vmm.s.rcCallRing3;
+}
+
+
+/**
+ * Simple wrapper that adds the pVCpu argument.
+ *
+ * @returns VBox status code of the ring-3 call.
+ * @retval  VERR_VMM_RING3_CALL_DISABLED if called at the wrong time. This must
+ *          be passed up the stack, or if that isn't possible then VMMRZCallRing3
+ *          needs to change it into an assertion.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   enmOperation    The operation.
+ * @param   uArg            The argument to the operation.
+ */
+VMMRZDECL(int) VMMRZCallRing3NoCpu(PVMCC pVM, VMMCALLRING3 enmOperation, uint64_t uArg)
+{
+    return VMMRZCallRing3(pVM, VMMGetCpu(pVM), enmOperation, uArg);
+}
+
+
+/**
+ * Disables all host calls, except certain fatal ones.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT.
+ */
+VMMRZDECL(void) VMMRZCallRing3Disable(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+#if defined(LOG_ENABLED) && defined(IN_RING0)
+    RTCCUINTREG fFlags = ASMIntDisableFlags(); /* preemption consistency. */
+#endif
+
+    Assert(pVCpu->vmm.s.cCallRing3Disabled < 16);
+    if (ASMAtomicUoIncU32(&pVCpu->vmm.s.cCallRing3Disabled) == 1)
+    {
+        /** @todo it might make more sense to just disable logging here, then we
+         * won't flush away important bits... but that goes both ways really. */
+#ifdef IN_RC
+        pVCpu->pVMRC->vmm.s.fRCLoggerFlushingDisabled = true;
+#else
+# ifdef LOG_ENABLED
+        if (pVCpu->vmm.s.pR0LoggerR0)
+            pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = true;
+# endif
+        if (pVCpu->vmm.s.pR0RelLoggerR0)
+            pVCpu->vmm.s.pR0RelLoggerR0->fFlushingDisabled = true;
+#endif
+    }
+
+#if defined(LOG_ENABLED) && defined(IN_RING0)
+    ASMSetFlags(fFlags);
+#endif
+}
+
+
+/**
+ * Counters VMMRZCallRing3Disable() and re-enables host calls.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT.
+ */
+VMMRZDECL(void) VMMRZCallRing3Enable(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+#if defined(LOG_ENABLED) && defined(IN_RING0)
+    RTCCUINTREG fFlags = ASMIntDisableFlags(); /* preemption consistency. */
+#endif
+
+    Assert(pVCpu->vmm.s.cCallRing3Disabled > 0);
+    if (ASMAtomicUoDecU32(&pVCpu->vmm.s.cCallRing3Disabled) == 0)
+    {
+#ifdef IN_RC
+        pVCpu->pVMRC->vmm.s.fRCLoggerFlushingDisabled = false;
+#else
+# ifdef LOG_ENABLED
+        if (pVCpu->vmm.s.pR0LoggerR0)
+            pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = false;
+# endif
+        if (pVCpu->vmm.s.pR0RelLoggerR0)
+            pVCpu->vmm.s.pR0RelLoggerR0->fFlushingDisabled = false;
+#endif
+    }
+
+#if defined(LOG_ENABLED) && defined(IN_RING0)
+    ASMSetFlags(fFlags);
+#endif
+}
+
+
+/**
+ * Checks whether its possible to call host context or not.
+ *
+ * @returns true if it's safe, false if it isn't.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ */
+VMMRZDECL(bool) VMMRZCallRing3IsEnabled(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(pVCpu->vmm.s.cCallRing3Disabled <= 16);
+    return pVCpu->vmm.s.cCallRing3Disabled == 0;
+}
+
+
+/**
+ * Sets the ring-0 callback before doing the ring-3 call.
+ *
+ * @param   pVCpu         The cross context virtual CPU structure.
+ * @param   pfnCallback   Pointer to the callback.
+ * @param   pvUser        The user argument.
+ *
+ * @return VBox status code.
+ */
+VMMRZDECL(int) VMMRZCallRing3SetNotification(PVMCPUCC pVCpu, R0PTRTYPE(PFNVMMR0CALLRING3NOTIFICATION) pfnCallback, RTR0PTR pvUser)
+{
+    AssertPtrReturn(pVCpu, VERR_INVALID_POINTER);
+    AssertPtrReturn(pfnCallback, VERR_INVALID_POINTER);
+
+    if (pVCpu->vmm.s.pfnCallRing3CallbackR0)
+        return VERR_ALREADY_EXISTS;
+
+    pVCpu->vmm.s.pfnCallRing3CallbackR0    = pfnCallback;
+    pVCpu->vmm.s.pvCallRing3CallbackUserR0 = pvUser;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Removes the ring-0 callback.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMRZDECL(void) VMMRZCallRing3RemoveNotification(PVMCPUCC pVCpu)
+{
+    pVCpu->vmm.s.pfnCallRing3CallbackR0 = NULL;
+}
+
+
+/**
+ * Checks whether there is a ring-0 callback notification active.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @returns true if there the notification is active, false otherwise.
+ */
+VMMRZDECL(bool) VMMRZCallRing3IsNotificationSet(PVMCPUCC pVCpu)
+{
+    return pVCpu->vmm.s.pfnCallRing3CallbackR0 != NULL;
+}
+
diff --git a/src/VBox/VMM/dtrace/int-1.d b/src/VBox/VMM/dtrace/int-1.d
new file mode 100644
index 00000000..a51f0334
--- /dev/null
+++ b/src/VBox/VMM/dtrace/int-1.d
@@ -0,0 +1,125 @@
+/* $Id: int-1.d $ */
+/** @file
+ * DTracing VBox - Interrupt Experiment #1.
+ */
+
+/*
+ * Copyright (C) 2012-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#pragma D option quiet
+
+uint64_t g_aStarts[uint32_t];
+uint64_t g_cUntaggedHighs;
+uint64_t g_cUntaggedGets;
+uint64_t g_cMissedHighs;
+
+inline uint32_t kfDevIdMask = 0x3ff;
+
+
+/*
+ * Timestamp the when the device raises the IRQ.
+ */
+vboxvmm*:::pdm-irq-high,vboxvmm*:::pdm-irq-hilo
+/args[1] != 0/
+{
+    /*printf("high: tag=%#x src=%d %llx -> %llx\n", args[1], args[2], g_aStarts[args[1]], timestamp);*/
+    g_aStarts[args[1]] = timestamp;
+}
+
+vboxvmm*:::pdm-irq-high,vboxvmm*:::pdm-irq-hilo
+/args[1] == 0/
+{
+    g_cUntaggedHighs++;
+}
+
+/*
+ * Catch the CPU getting the IRQ from the (A)PIC and preparing for injection.
+ */
+vboxvmm*:::pdm-irq-get
+/g_aStarts[args[1]] == 0 && args[1] != 0/
+{
+    printf("get:  tag=%#x src=%d %llx - %llx = %llx\n", args[1], args[2], timestamp, g_aStarts[args[1]], timestamp - g_aStarts[args[1]]);
+    @g_MissedHighs[args[3], args[2] & kfDevIdMask] = count();
+    g_cMissedHighs++;
+}
+
+vboxvmm*:::pdm-irq-get
+/g_aStarts[args[1]] > 0 && args[1] != 0/
+{
+    /*printf("get:  tag=%#x src=%d %llx - %llx = %llx\n", args[1], args[2], timestamp, g_aStarts[args[1]], timestamp - g_aStarts[args[1]]);*/
+    @g_Interrupts[args[3], args[2] & kfDevIdMask] = count();
+    @g_DispAvg[   args[3], args[2] & kfDevIdMask]  = avg(timestamp - g_aStarts[args[1]]);
+    @g_DispMax[   args[3], args[2] & kfDevIdMask]  = max(timestamp - g_aStarts[args[1]]);
+    @g_DispMin[   args[3], args[2] & kfDevIdMask]  = min(timestamp - g_aStarts[args[1]]);
+    g_aStarts[args[1]] = 0;
+    g_cHits++;
+}
+
+vboxvmm*:::pdm-irq-get
+/args[1] == 0/
+{
+    @g_UntaggedGets[args[3]] = count();
+    g_cUntaggedGets++;
+}
+
+vboxvmm*:::pdm-irq-get
+/args[2] > kfDevIdMask/
+{
+    @g_Shared[args[3], args[2] & kfDevIdMask] = count();
+}
+
+/* For the time being, quit after 256 interrupts. */
+vboxvmm*:::pdm-irq-get
+/g_cHits >= 256/
+{
+    exit(0);
+}
+
+/*
+ * Catch the device clearing the IRQ.
+ */
+
+
+/*
+ * Report.
+ */
+END
+{
+    printf("\nInterrupt distribution:\n");
+    printa("    irq %3d    dev %2d    %@12u\n", @g_Interrupts);
+    printf("Interrupt sharing (devices detect pushing a line high at the same time):\n");
+    printa("    irq %3d    dev %2d    %@12u\n", @g_Shared);
+    printf("Minimum dispatch latency:\n");
+    printa("    irq %3d    dev %2d    %@12u ns\n", @g_DispMin);
+    printf("Average dispatch latency:\n");
+    printa("    irq %3d    dev %2d    %@12u ns\n", @g_DispAvg);
+    printf("Maximum dispatch latency:\n");
+    printa("    irq %3d    dev %2d    %@12u ns\n", @g_DispMax);
+}
+END
+/g_cUntaggedHighs > 0/
+{
+    printf("Untagged highs: %u\n", g_cUntaggedHighs);
+}
+END
+/g_cUntaggedGets > 0/
+{
+    printf("Untagged gets: %u\n", g_cUntaggedGets);
+    printa("    irq %3d              %@12u\n", @g_UntaggedGets);
+}
+END
+/g_cMissedHighs > 0/
+{
+    printf("Missed (or shared?) highs: %u\n", g_cMissedHighs);
+    printa("    irq %3d    dev %2d    %@12u\n", @g_MissedHighs);
+}
+
diff --git a/src/VBox/VMM/dtrace/lib/amd64/vbox-arch-types.d b/src/VBox/VMM/dtrace/lib/amd64/vbox-arch-types.d
new file mode 100644
index 00000000..fad47a3e
--- /dev/null
+++ b/src/VBox/VMM/dtrace/lib/amd64/vbox-arch-types.d
@@ -0,0 +1,39 @@
+/** @file
+ * VBox & DTrace - Types and Constants for AMD64.
+ */
+
+/*
+ * Copyright (C) 2012-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+inline unsigned HC_ARCH_BITS = 64;
+typedef uint64_t    RTR3PTR;
+typedef void       *RTR0PTR;
+typedef uint64_t    RTHCPTR;
+
+
+
+typedef union RTFLOAT80U
+{
+    uint16_t    au16[5];
+} RTFLOAT80U;
+
+typedef union RTFLOAT80U2
+{
+    uint16_t    au16[5];
+} RTFLOAT80U2;
+
+typedef struct uint128_t
+{
+    uint64_t    au64[2];
+} uint128_t;
+
diff --git a/src/VBox/VMM/dtrace/lib/vbox-types.d b/src/VBox/VMM/dtrace/lib/vbox-types.d
new file mode 100644
index 00000000..99762bd6
--- /dev/null
+++ b/src/VBox/VMM/dtrace/lib/vbox-types.d
@@ -0,0 +1,58 @@
+/** @file
+ * VBox & DTrace - Types and Constants.
+ */
+
+/*
+ * Copyright (C) 2012-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*
+ * Types used by the other D structure and type definitions.
+ *
+ * These are taken from a variation of VBox and IPRT headers.
+ */
+#pragma D depends_on library vbox-arch-types.d
+
+typedef uint64_t                RTGCPHYS;
+typedef uint64_t                RTHCPHYS;
+typedef uint16_t                RTSEL;
+typedef uint32_t                RTRCPTR;
+typedef uintptr_t               RTNATIVETHREAD;
+typedef struct RTTHREADINT     *RTTHREAD;
+typedef struct RTTRACEBUFINT   *RTTRACEBUF;
+
+
+typedef uint32_t                VMSTATE;
+typedef uint32_t                VMCPUID;
+typedef uint32_t                RTCPUID;
+typedef struct UVMCPU          *PUVMCPU;
+typedef uintptr_t               PVMR3;
+typedef uint32_t                PVMRC;
+typedef struct VM              *PVMR0;
+typedef struct SUPDRVSESSION   *PSUPDRVSESSION;
+typedef struct UVM             *PUVM;
+typedef struct CPUMCTXCORE     *PCPUMCTXCORE;
+typedef struct SVMVMCB         *PSVMVMCB;
+typedef uint32_t                VMXVDIAG;
+typedef struct VMXVVMCS        *PVMXVVMCS;
+
+typedef struct VBOXGDTR
+{
+    uint16_t    cb;
+    uint16_t    au16Addr[4];
+} VBOXGDTR, VBOXIDTR;
+
+typedef struct STAMPROFILEADV
+{
+    uint64_t            au64[5];
+} STAMPROFILEADV;
+
diff --git a/src/VBox/VMM/dtrace/lib/x86/vbox-arch-types.d b/src/VBox/VMM/dtrace/lib/x86/vbox-arch-types.d
new file mode 100644
index 00000000..ae70f4db
--- /dev/null
+++ b/src/VBox/VMM/dtrace/lib/x86/vbox-arch-types.d
@@ -0,0 +1,22 @@
+/** @file
+ * VBox & DTrace - Types and Constants for X86.
+ */
+
+/*
+ * Copyright (C) 2012-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+inline unsigned HC_ARCH_BITS = 32;
+typedef uint32_t    RTR3PTR;
+typedef uint32_t    RTHCPTR;
+typedef void       *RTR0PTR;
+
diff --git a/src/VBox/VMM/dtrace/return-to-ring-3-aggregation-1.d b/src/VBox/VMM/dtrace/return-to-ring-3-aggregation-1.d
new file mode 100644
index 00000000..a95c5abe
--- /dev/null
+++ b/src/VBox/VMM/dtrace/return-to-ring-3-aggregation-1.d
@@ -0,0 +1,36 @@
+/* $Id: return-to-ring-3-aggregation-1.d $ */
+/** @file
+ * DTracing VBox - return to ring-3 aggregation test \#1.
+ */
+
+/*
+ * Copyright (C) 2012-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#pragma D option quiet
+
+
+vboxvmm:::r0-vmm-return-to-ring3-hm
+{
+    @g_aHmRcs[args[2]] = count();
+}
+
+vboxvmm:::r0-vmm-return-to-ring3-rc
+{
+    @g_aRawRcs[args[2]] = count();
+}
+
+END
+{
+    printa(" rcHm=%04d   %@8u times\n", @g_aHmRcs);
+    printa(" rcRaw=%04d   %@8u times\n", @g_aRawRcs);
+}
+
diff --git a/src/VBox/VMM/dtrace/vmexit-reason-aggregation-1.d b/src/VBox/VMM/dtrace/vmexit-reason-aggregation-1.d
new file mode 100644
index 00000000..38b72ab2
--- /dev/null
+++ b/src/VBox/VMM/dtrace/vmexit-reason-aggregation-1.d
@@ -0,0 +1,36 @@
+/* $Id: vmexit-reason-aggregation-1.d $ */
+/** @file
+ * DTracing VBox - vmexit reason aggregation test \#1.
+ */
+
+/*
+ * Copyright (C) 2012-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#pragma D option quiet
+
+
+vboxvmm:::r0-hmsvm-vmexit
+{
+    @g_aSvmExits[args[2]] = count();
+}
+
+vboxvmm:::r0-hmvmx-vmexit-noctx
+{
+    @g_aVmxExits[args[2]] = count();
+}
+
+END
+{
+    printa(" svmexit=%#04llx   %@10u times\n", @g_aSvmExits);
+    printa(" vmxexit=%#04llx   %@10u times\n", @g_aVmxExits);
+}
+
diff --git a/src/VBox/VMM/dtrace/vmexit-rip-aggregation-1.d b/src/VBox/VMM/dtrace/vmexit-rip-aggregation-1.d
new file mode 100644
index 00000000..e0b1b6c2
--- /dev/null
+++ b/src/VBox/VMM/dtrace/vmexit-rip-aggregation-1.d
@@ -0,0 +1,32 @@
+/* $Id: vmexit-rip-aggregation-1.d $ */
+/** @file
+ * DTracing VBox - vmexit rip aggregation test \#1.
+ */
+
+/*
+ * Copyright (C) 2012-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#pragma D option quiet
+
+
+vboxvmm:::r0-hmsvm-vmexit,vboxvmm:::r0-hmvmx-vmexit
+{
+    /*printf("cs:rip=%02x:%08llx", args[1]->cs.Sel, args[1]->rip.rip);*/
+    @g_aRips[args[1]->rip.rip] = count();
+    /*@g_aRips[args[0]->cpum.s.Guest.rip.rip] = count(); - alternative access route */
+}
+
+END
+{
+    printa(" rip=%#018llx   %@4u times\n", @g_aRips);
+}
+
diff --git a/src/VBox/VMM/include/APICInternal.h b/src/VBox/VMM/include/APICInternal.h
new file mode 100644
index 00000000..7f9bf3b2
--- /dev/null
+++ b/src/VBox/VMM/include/APICInternal.h
@@ -0,0 +1,1426 @@
+/* $Id: APICInternal.h $ */
+/** @file
+ * APIC - Advanced Programmable Interrupt Controller.
+ */
+
+/*
+ * Copyright (C) 2016-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_APICInternal_h
+#define VMM_INCLUDED_SRC_include_APICInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/sup.h>
+#include <VBox/vmm/pdmdev.h> /* before apic.h! */
+#include <VBox/vmm/apic.h>
+
+/** @defgroup grp_apic_int       Internal
+ * @ingroup grp_apic
+ * @internal
+ * @{
+ */
+
+/** The APIC hardware version number for Pentium 4. */
+#define XAPIC_HARDWARE_VERSION_P4            UINT8_C(0x14)
+/** Maximum number of LVT entries for Pentium 4. */
+#define XAPIC_MAX_LVT_ENTRIES_P4             UINT8_C(6)
+/** Size of the APIC ID bits for Pentium 4. */
+#define XAPIC_APIC_ID_BIT_COUNT_P4           UINT8_C(8)
+
+/** The APIC hardware version number for Pentium 6. */
+#define XAPIC_HARDWARE_VERSION_P6            UINT8_C(0x10)
+/** Maximum number of LVT entries for Pentium 6. */
+#define XAPIC_MAX_LVT_ENTRIES_P6             UINT8_C(4)
+/** Size of the APIC ID bits for Pentium 6. */
+#define XAPIC_APIC_ID_BIT_COUNT_P6           UINT8_C(4)
+
+/** The APIC hardware version we are emulating. */
+#define XAPIC_HARDWARE_VERSION               XAPIC_HARDWARE_VERSION_P4
+
+#define VMCPU_TO_XAPICPAGE(a_pVCpu)          ((PXAPICPAGE)(CTX_SUFF((a_pVCpu)->apic.s.pvApicPage)))
+#define VMCPU_TO_CXAPICPAGE(a_pVCpu)         ((PCXAPICPAGE)(CTX_SUFF((a_pVCpu)->apic.s.pvApicPage)))
+
+#define VMCPU_TO_X2APICPAGE(a_pVCpu)         ((PX2APICPAGE)(CTX_SUFF((a_pVCpu)->apic.s.pvApicPage)))
+#define VMCPU_TO_CX2APICPAGE(a_pVCpu)        ((PCX2APICPAGE)(CTX_SUFF((a_pVCpu)->apic.s.pvApicPage)))
+
+#define VMCPU_TO_APICCPU(a_pVCpu)            (&(a_pVCpu)->apic.s)
+#define VM_TO_APIC(a_pVM)                    (&(a_pVM)->apic.s)
+#define VM_TO_APICDEV(a_pVM)                 CTX_SUFF(VM_TO_APIC(a_pVM)->pApicDev)
+#ifdef IN_RING3
+# define VMCPU_TO_DEVINS(a_pVCpu)           ((a_pVCpu)->pVMR3->apic.s.pDevInsR3)
+#elif defined(IN_RING0)
+# define VMCPU_TO_DEVINS(a_pVCpu)           ((a_pVCpu)->pGVM->apicr0.s.pDevInsR0)
+#endif
+
+#define APICCPU_TO_XAPICPAGE(a_ApicCpu)      ((PXAPICPAGE)(CTX_SUFF((a_ApicCpu)->pvApicPage)))
+#define APICCPU_TO_CXAPICPAGE(a_ApicCpu)     ((PCXAPICPAGE)(CTX_SUFF((a_ApicCpu)->pvApicPage)))
+
+/** Whether the APIC is in X2APIC mode or not. */
+#define XAPIC_IN_X2APIC_MODE(a_pVCpu)        (   (  ((a_pVCpu)->apic.s.uApicBaseMsr) \
+                                                  & (MSR_IA32_APICBASE_EN | MSR_IA32_APICBASE_EXTD)) \
+                                              ==    (MSR_IA32_APICBASE_EN | MSR_IA32_APICBASE_EXTD) )
+
+/** Get an xAPIC page offset for an x2APIC MSR value. */
+#define X2APIC_GET_XAPIC_OFF(a_uMsr)         ((((a_uMsr) - MSR_IA32_X2APIC_START) << 4) & UINT32_C(0xff0))
+/** Get an x2APIC MSR for an xAPIC page offset. */
+#define XAPIC_GET_X2APIC_MSR(a_offReg)       ((((a_offReg) & UINT32_C(0xff0)) >> 4) | MSR_IA32_X2APIC_START)
+
+/** Illegal APIC vector value start. */
+#define XAPIC_ILLEGAL_VECTOR_START           UINT8_C(0)
+/** Illegal APIC vector value end (inclusive). */
+#define XAPIC_ILLEGAL_VECTOR_END             UINT8_C(15)
+/** Reserved APIC vector value start. */
+#define XAPIC_RSVD_VECTOR_START              UINT8_C(16)
+/** Reserved APIC vector value end (inclusive). */
+#define XAPIC_RSVD_VECTOR_END                UINT8_C(31)
+
+/** Vector offset in an APIC 256-bit sparse register. */
+#define XAPIC_REG256_VECTOR_OFF(a_Vector)    (((a_Vector) & UINT32_C(0xe0)) >> 1)
+/** Bit position at offset in an APIC 256-bit sparse register. */
+#define XAPIC_REG256_VECTOR_BIT(a_Vector)    ((a_Vector) & UINT32_C(0x1f))
+
+/** Maximum valid offset for a register (16-byte aligned, 4 byte wide access). */
+#define XAPIC_OFF_MAX_VALID                  (sizeof(XAPICPAGE) - 4 * sizeof(uint32_t))
+
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P6
+/** ESR - Send checksum error. */
+# define XAPIC_ESR_SEND_CHKSUM_ERROR         RT_BIT(0)
+/** ESR - Send accept error. */
+# define XAPIC_ESR_RECV_CHKSUM_ERROR         RT_BIT(1)
+/** ESR - Send accept error. */
+# define XAPIC_ESR_SEND_ACCEPT_ERROR         RT_BIT(2)
+/** ESR - Receive accept error. */
+# define XAPIC_ESR_RECV_ACCEPT_ERROR         RT_BIT(3)
+#endif
+/** ESR - Redirectable IPI. */
+#define XAPIC_ESR_REDIRECTABLE_IPI           RT_BIT(4)
+/** ESR - Send accept error. */
+#define XAPIC_ESR_SEND_ILLEGAL_VECTOR        RT_BIT(5)
+/** ESR - Send accept error. */
+#define XAPIC_ESR_RECV_ILLEGAL_VECTOR        RT_BIT(6)
+/** ESR - Send accept error. */
+#define XAPIC_ESR_ILLEGAL_REG_ADDRESS        RT_BIT(7)
+/** ESR - Valid write-only bits. */
+#define XAPIC_ESR_WO_VALID                   UINT32_C(0x0)
+
+/** TPR - Valid bits. */
+#define XAPIC_TPR_VALID                      UINT32_C(0xff)
+/** TPR - Task-priority class. */
+#define XAPIC_TPR_TP                         UINT32_C(0xf0)
+/** TPR - Task-priority subclass. */
+#define XAPIC_TPR_TP_SUBCLASS                UINT32_C(0x0f)
+/** TPR - Gets the task-priority class. */
+#define XAPIC_TPR_GET_TP(a_Tpr)              ((a_Tpr) & XAPIC_TPR_TP)
+/** TPR - Gets the task-priority subclass. */
+#define XAPIC_TPR_GET_TP_SUBCLASS(a_Tpr)     ((a_Tpr) & XAPIC_TPR_TP_SUBCLASS)
+
+/** PPR - Valid bits. */
+#define XAPIC_PPR_VALID                      UINT32_C(0xff)
+/** PPR - Processor-priority class. */
+#define XAPIC_PPR_PP                         UINT32_C(0xf0)
+/** PPR - Processor-priority subclass. */
+#define XAPIC_PPR_PP_SUBCLASS                UINT32_C(0x0f)
+/** PPR - Get the processor-priority class. */
+#define XAPIC_PPR_GET_PP(a_Ppr)              ((a_Ppr) & XAPIC_PPR_PP)
+/** PPR - Get the processor-priority subclass. */
+#define XAPIC_PPR_GET_PP_SUBCLASS(a_Ppr)     ((a_Ppr) & XAPIC_PPR_PP_SUBCLASS)
+
+/** Timer mode - One-shot. */
+#define XAPIC_TIMER_MODE_ONESHOT             UINT32_C(0)
+/** Timer mode - Periodic. */
+#define XAPIC_TIMER_MODE_PERIODIC            UINT32_C(1)
+/** Timer mode - TSC deadline. */
+#define XAPIC_TIMER_MODE_TSC_DEADLINE        UINT32_C(2)
+
+/** LVT - The vector. */
+#define XAPIC_LVT_VECTOR                     UINT32_C(0xff)
+/** LVT - Gets the vector from an LVT entry. */
+#define XAPIC_LVT_GET_VECTOR(a_Lvt)          ((a_Lvt) & XAPIC_LVT_VECTOR)
+/** LVT - The mask. */
+#define XAPIC_LVT_MASK                       RT_BIT(16)
+/** LVT - Is the LVT masked? */
+#define XAPIC_LVT_IS_MASKED(a_Lvt)           RT_BOOL((a_Lvt) & XAPIC_LVT_MASK)
+/** LVT - Timer mode. */
+#define XAPIC_LVT_TIMER_MODE                 RT_BIT(17)
+/** LVT - Timer TSC-deadline timer mode. */
+#define XAPIC_LVT_TIMER_TSCDEADLINE          RT_BIT(18)
+/** LVT - Gets the timer mode. */
+#define XAPIC_LVT_GET_TIMER_MODE(a_Lvt)      (XAPICTIMERMODE)(((a_Lvt) >> 17) & UINT32_C(3))
+/** LVT - Delivery mode. */
+#define XAPIC_LVT_DELIVERY_MODE              (RT_BIT(8) | RT_BIT(9) | RT_BIT(10))
+/** LVT - Gets the delivery mode. */
+#define XAPIC_LVT_GET_DELIVERY_MODE(a_Lvt)   (XAPICDELIVERYMODE)(((a_Lvt) >> 8) & UINT32_C(7))
+/** LVT - Delivery status. */
+#define XAPIC_LVT_DELIVERY_STATUS            RT_BIT(12)
+/** LVT - Trigger mode. */
+#define XAPIC_LVT_TRIGGER_MODE               RT_BIT(15)
+/** LVT - Gets the trigger mode. */
+#define XAPIC_LVT_GET_TRIGGER_MODE(a_Lvt)    (XAPICTRIGGERMODE)(((a_Lvt) >> 15) & UINT32_C(1))
+/** LVT - Remote IRR. */
+#define XAPIC_LVT_REMOTE_IRR                 RT_BIT(14)
+/** LVT - Gets the Remote IRR. */
+#define XAPIC_LVT_GET_REMOTE_IRR(a_Lvt)      (((a_Lvt) >> 14) & 1)
+/** LVT - Interrupt Input Pin Polarity. */
+#define XAPIC_LVT_POLARITY                   RT_BIT(13)
+/** LVT - Gets the Interrupt Input Pin Polarity. */
+#define XAPIC_LVT_GET_POLARITY(a_Lvt)        (((a_Lvt) >> 13) & 1)
+/** LVT - Valid bits common to all LVTs. */
+#define XAPIC_LVT_COMMON_VALID               (XAPIC_LVT_VECTOR | XAPIC_LVT_DELIVERY_STATUS | XAPIC_LVT_MASK)
+/** LVT CMCI - Valid bits. */
+#define XAPIC_LVT_CMCI_VALID                 (XAPIC_LVT_COMMON_VALID | XAPIC_LVT_DELIVERY_MODE)
+/** LVT Timer - Valid bits. */
+#define XAPIC_LVT_TIMER_VALID                (XAPIC_LVT_COMMON_VALID | XAPIC_LVT_TIMER_MODE | XAPIC_LVT_TIMER_TSCDEADLINE)
+/** LVT Thermal - Valid bits. */
+#define XAPIC_LVT_THERMAL_VALID              (XAPIC_LVT_COMMON_VALID | XAPIC_LVT_DELIVERY_MODE)
+/** LVT Perf - Valid bits. */
+#define XAPIC_LVT_PERF_VALID                 (XAPIC_LVT_COMMON_VALID | XAPIC_LVT_DELIVERY_MODE)
+/** LVT LINTx - Valid bits. */
+#define XAPIC_LVT_LINT_VALID                 (  XAPIC_LVT_COMMON_VALID | XAPIC_LVT_DELIVERY_MODE | XAPIC_LVT_DELIVERY_STATUS \
+                                              | XAPIC_LVT_POLARITY | XAPIC_LVT_REMOTE_IRR | XAPIC_LVT_TRIGGER_MODE)
+/** LVT Error - Valid bits. */
+#define XAPIC_LVT_ERROR_VALID                (XAPIC_LVT_COMMON_VALID)
+
+/** SVR - The vector. */
+#define XAPIC_SVR_VECTOR                     UINT32_C(0xff)
+/** SVR - APIC Software enable. */
+#define XAPIC_SVR_SOFTWARE_ENABLE            RT_BIT(8)
+/** SVR - Supress EOI broadcast. */
+#define XAPIC_SVR_SUPRESS_EOI_BROADCAST      RT_BIT(12)
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+/** SVR - Valid bits. */
+# define XAPIC_SVR_VALID                     (XAPIC_SVR_VECTOR | XAPIC_SVR_SOFTWARE_ENABLE)
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+
+/** DFR - Valid bits. */
+#define XAPIC_DFR_VALID                      UINT32_C(0xf0000000)
+/** DFR - Reserved bits that must always remain set. */
+#define XAPIC_DFR_RSVD_MB1                   UINT32_C(0x0fffffff)
+/** DFR - The model. */
+#define XAPIC_DFR_MODEL                      UINT32_C(0xf)
+/** DFR - Gets the destination model. */
+#define XAPIC_DFR_GET_MODEL(a_uReg)          (((a_uReg) >> 28) & XAPIC_DFR_MODEL)
+
+/** LDR - Valid bits. */
+#define XAPIC_LDR_VALID                      UINT32_C(0xff000000)
+/** LDR - Cluster ID mask (x2APIC). */
+#define X2APIC_LDR_CLUSTER_ID                UINT32_C(0xffff0000)
+/** LDR - Mask of the LDR cluster ID (x2APIC). */
+#define X2APIC_LDR_GET_CLUSTER_ID(a_uReg)    ((a_uReg) & X2APIC_LDR_CLUSTER_ID)
+/** LDR - Mask of the LDR logical ID (x2APIC). */
+#define X2APIC_LDR_LOGICAL_ID                UINT32_C(0x0000ffff)
+
+/** LDR - Flat mode logical ID mask. */
+#define XAPIC_LDR_FLAT_LOGICAL_ID            UINT32_C(0xff)
+/** LDR - Clustered mode cluster ID mask. */
+#define XAPIC_LDR_CLUSTERED_CLUSTER_ID       UINT32_C(0xf0)
+/** LDR - Clustered mode logical ID mask. */
+#define XAPIC_LDR_CLUSTERED_LOGICAL_ID       UINT32_C(0x0f)
+/** LDR - Gets the clustered mode cluster ID. */
+#define XAPIC_LDR_CLUSTERED_GET_CLUSTER_ID(a_uReg)   ((a_uReg) & XAPIC_LDR_CLUSTERED_CLUSTER_ID)
+
+
+/** EOI - Valid write-only bits. */
+#define XAPIC_EOI_WO_VALID                   UINT32_C(0x0)
+/** Timer ICR - Valid bits. */
+#define XAPIC_TIMER_ICR_VALID                UINT32_C(0xffffffff)
+/** Timer DCR - Valid bits. */
+#define XAPIC_TIMER_DCR_VALID                (RT_BIT(0) | RT_BIT(1) | RT_BIT(3))
+
+/** Self IPI - Valid bits. */
+#define XAPIC_SELF_IPI_VALID                 UINT32_C(0xff)
+/** Self IPI - The vector. */
+#define XAPIC_SELF_IPI_VECTOR                UINT32_C(0xff)
+/** Self IPI - Gets the vector. */
+#define XAPIC_SELF_IPI_GET_VECTOR(a_uReg)    ((a_uReg) & XAPIC_SELF_IPI_VECTOR)
+
+/** ICR Low - The Vector. */
+#define XAPIC_ICR_LO_VECTOR                  UINT32_C(0xff)
+/** ICR Low - Gets the vector. */
+#define XAPIC_ICR_LO_GET_VECTOR(a_uIcr)      ((a_uIcr) & XAPIC_ICR_LO_VECTOR)
+/** ICR Low - The delivery mode. */
+#define XAPIC_ICR_LO_DELIVERY_MODE           (RT_BIT(8) | RT_BIT(9) | RT_BIT(10))
+/** ICR Low - The destination mode. */
+#define XAPIC_ICR_LO_DEST_MODE               RT_BIT(11)
+/** ICR Low - The delivery status. */
+#define XAPIC_ICR_LO_DELIVERY_STATUS         RT_BIT(12)
+/** ICR Low - The level. */
+#define XAPIC_ICR_LO_LEVEL                   RT_BIT(14)
+/** ICR Low - The trigger mode. */
+#define XAPIC_ICR_TRIGGER_MODE               RT_BIT(15)
+/** ICR Low - The destination shorthand. */
+#define XAPIC_ICR_LO_DEST_SHORTHAND          (RT_BIT(18) | RT_BIT(19))
+/** ICR Low - Valid write bits. */
+#define XAPIC_ICR_LO_WR_VALID                (  XAPIC_ICR_LO_VECTOR | XAPIC_ICR_LO_DELIVERY_MODE | XAPIC_ICR_LO_DEST_MODE \
+                                              | XAPIC_ICR_LO_LEVEL | XAPIC_ICR_TRIGGER_MODE | XAPIC_ICR_LO_DEST_SHORTHAND)
+
+/** ICR High - The destination field. */
+#define XAPIC_ICR_HI_DEST                    UINT32_C(0xff000000)
+/** ICR High - Get the destination field. */
+#define XAPIC_ICR_HI_GET_DEST(a_u32IcrHi)    (((a_u32IcrHi) >> 24) & XAPIC_ICR_HI_DEST)
+/** ICR High - Valid write bits in xAPIC mode. */
+#define XAPIC_ICR_HI_WR_VALID                XAPIC_ICR_HI_DEST
+
+/** APIC ID broadcast mask - x2APIC mode. */
+#define X2APIC_ID_BROADCAST_MASK             UINT32_C(0xffffffff)
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+/** APIC ID broadcast mask - xAPIC mode. */
+# define XAPIC_ID_BROADCAST_MASK             UINT32_C(0xff)
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+
+/**
+ * The xAPIC sparse 256-bit register.
+ */
+typedef union XAPIC256BITREG
+{
+    /** The sparse-bitmap view. */
+    struct
+    {
+        uint32_t    u32Reg;
+        uint32_t    uReserved0[3];
+    } u[8];
+    /** The 32-bit view. */
+    uint32_t        au32[32];
+} XAPIC256BITREG;
+/** Pointer to an xAPIC sparse bitmap register. */
+typedef XAPIC256BITREG *PXAPIC256BITREG;
+/** Pointer to a const xAPIC sparse bitmap register. */
+typedef XAPIC256BITREG const *PCXAPIC256BITREG;
+AssertCompileSize(XAPIC256BITREG, 128);
+
+/**
+ * The xAPIC memory layout as per Intel/AMD specs.
+ */
+typedef struct XAPICPAGE
+{
+    /* 0x00 - Reserved. */
+    uint32_t                    uReserved0[8];
+    /* 0x20 - APIC ID. */
+    struct
+    {
+        uint8_t                 u8Reserved0[3];
+        uint8_t                 u8ApicId;
+        uint32_t                u32Reserved0[3];
+    } id;
+    /* 0x30 - APIC version register. */
+    union
+    {
+        struct
+        {
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+            uint8_t             u8Version;
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+            uint8_t             uReserved0;
+            uint8_t             u8MaxLvtEntry;
+            uint8_t             fEoiBroadcastSupression : 1;
+            uint8_t             u7Reserved1   : 7;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32Version;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } version;
+    /* 0x40 - Reserved. */
+    uint32_t                    uReserved1[16];
+    /* 0x80 - Task Priority Register (TPR). */
+    struct
+    {
+        uint8_t                 u8Tpr;
+        uint8_t                 u8Reserved0[3];
+        uint32_t                u32Reserved0[3];
+    } tpr;
+    /* 0x90 - Arbitration Priority Register (APR). */
+    struct
+    {
+        uint8_t                 u8Apr;
+        uint8_t                 u8Reserved0[3];
+        uint32_t                u32Reserved0[3];
+    } apr;
+    /* 0xA0 - Processor Priority Register (PPR). */
+    struct
+    {
+        uint8_t                 u8Ppr;
+        uint8_t                 u8Reserved0[3];
+        uint32_t                u32Reserved0[3];
+    } ppr;
+    /* 0xB0 - End Of Interrupt Register (EOI). */
+    struct
+    {
+        uint32_t                u32Eoi;
+        uint32_t                u32Reserved0[3];
+    } eoi;
+    /* 0xC0 - Remote Read Register (RRD). */
+    struct
+    {
+        uint32_t                u32Rrd;
+        uint32_t                u32Reserved0[3];
+    } rrd;
+    /* 0xD0 - Logical Destination Register (LDR). */
+    union
+    {
+        struct
+        {
+            uint8_t             u8Reserved0[3];
+            uint8_t             u8LogicalApicId;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32Ldr;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } ldr;
+    /* 0xE0 - Destination Format Register (DFR). */
+    union
+    {
+        struct
+        {
+            uint32_t            u28ReservedMb1 : 28;    /* MB1 */
+            uint32_t            u4Model        : 4;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32Dfr;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } dfr;
+    /* 0xF0 - Spurious-Interrupt Vector Register (SVR). */
+    union
+    {
+        struct
+        {
+            uint32_t            u8SpuriousVector     : 8;
+            uint32_t            fApicSoftwareEnable  : 1;
+            uint32_t            u3Reserved0          : 3;
+            uint32_t            fSupressEoiBroadcast : 1;
+            uint32_t            u19Reserved1         : 19;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32Svr;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } svr;
+    /* 0x100 - In-service Register (ISR). */
+    XAPIC256BITREG              isr;
+    /* 0x180 - Trigger Mode Register (TMR). */
+    XAPIC256BITREG              tmr;
+    /* 0x200 - Interrupt Request Register (IRR). */
+    XAPIC256BITREG              irr;
+    /* 0x280 - Error Status Register (ESR). */
+    union
+    {
+        struct
+        {
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+            uint32_t            u4Reserved0        : 4;
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+            uint32_t            fRedirectableIpi   : 1;
+            uint32_t            fSendIllegalVector : 1;
+            uint32_t            fRcvdIllegalVector : 1;
+            uint32_t            fIllegalRegAddr    : 1;
+            uint32_t            u24Reserved1       : 24;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32Errors;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } esr;
+    /* 0x290 - Reserved. */
+    uint32_t                    uReserved2[28];
+    /* 0x300 - Interrupt Command Register (ICR) - Low. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u3DeliveryMode   : 3;
+            uint32_t            u1DestMode       : 1;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            fReserved0       : 1;
+            uint32_t            u1Level          : 1;
+            uint32_t            u1TriggerMode    : 1;
+            uint32_t            u2Reserved1      : 2;
+            uint32_t            u2DestShorthand  : 2;
+            uint32_t            u12Reserved2     : 12;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32IcrLo;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } icr_lo;
+    /* 0x310 - Interrupt Comannd Register (ICR) - High. */
+    union
+    {
+        struct
+        {
+            uint32_t            u24Reserved0 : 24;
+            uint32_t            u8Dest       : 8;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32IcrHi;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } icr_hi;
+    /* 0x320 - Local Vector Table (LVT) Timer Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u4Reserved0      : 4;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            u3Reserved1      : 3;
+            uint32_t            u1Mask           : 1;
+            uint32_t            u2TimerMode      : 2;
+            uint32_t            u13Reserved2     : 13;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32LvtTimer;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } lvt_timer;
+    /* 0x330 - Local Vector Table (LVT) Thermal Sensor Register. */
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u3DeliveryMode   : 3;
+            uint32_t            u1Reserved0      : 1;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            u3Reserved1      : 3;
+            uint32_t            u1Mask           : 1;
+            uint32_t            u15Reserved2     : 15;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32LvtThermal;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } lvt_thermal;
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+    /* 0x340 - Local Vector Table (LVT) Performance Monitor Counter (PMC) Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u3DeliveryMode   : 3;
+            uint32_t            u1Reserved0      : 1;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            u3Reserved1      : 3;
+            uint32_t            u1Mask           : 1;
+            uint32_t            u15Reserved2     : 15;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32LvtPerf;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } lvt_perf;
+    /* 0x350 - Local Vector Table (LVT) LINT0 Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u3DeliveryMode   : 3;
+            uint32_t            u1Reserved0      : 1;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            u1IntrPolarity   : 1;
+            uint32_t            u1RemoteIrr      : 1;
+            uint32_t            u1TriggerMode    : 1;
+            uint32_t            u1Mask           : 1;
+            uint32_t            u15Reserved2     : 15;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32LvtLint0;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } lvt_lint0;
+    /* 0x360 - Local Vector Table (LVT) LINT1 Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u3DeliveryMode   : 3;
+            uint32_t            u1Reserved0      : 1;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            u1IntrPolarity   : 1;
+            uint32_t            u1RemoteIrr      : 1;
+            uint32_t            u1TriggerMode    : 1;
+            uint32_t            u1Mask           : 1;
+            uint32_t            u15Reserved2     : 15;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32LvtLint1;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } lvt_lint1;
+    /* 0x370 - Local Vector Table (LVT) Error Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u4Reserved0      : 4;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            u3Reserved1      : 3;
+            uint32_t            u1Mask           : 1;
+            uint32_t            u15Reserved2     : 15;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32LvtError;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } lvt_error;
+    /* 0x380 - Timer Initial Counter Register. */
+    struct
+    {
+        uint32_t                u32InitialCount;
+        uint32_t                u32Reserved0[3];
+    } timer_icr;
+    /* 0x390 - Timer Current Counter Register. */
+    struct
+    {
+        uint32_t                u32CurrentCount;
+        uint32_t                u32Reserved0[3];
+    } timer_ccr;
+    /* 0x3A0 - Reserved. */
+    uint32_t                    u32Reserved3[16];
+    /* 0x3E0 - Timer Divide Configuration Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u2DivideValue0 : 2;
+            uint32_t            u1Reserved0    : 1;
+            uint32_t            u1DivideValue1 : 1;
+            uint32_t            u28Reserved1   : 28;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32DivideValue;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } timer_dcr;
+    /* 0x3F0 - Reserved. */
+    uint8_t                     u8Reserved0[3088];
+} XAPICPAGE;
+/** Pointer to a XAPICPAGE struct. */
+typedef XAPICPAGE *PXAPICPAGE;
+/** Pointer to a const XAPICPAGE struct. */
+typedef const XAPICPAGE *PCXAPICPAGE;
+AssertCompileSize(XAPICPAGE, 4096);
+AssertCompileMemberOffset(XAPICPAGE, id,          XAPIC_OFF_ID);
+AssertCompileMemberOffset(XAPICPAGE, version,     XAPIC_OFF_VERSION);
+AssertCompileMemberOffset(XAPICPAGE, tpr,         XAPIC_OFF_TPR);
+AssertCompileMemberOffset(XAPICPAGE, apr,         XAPIC_OFF_APR);
+AssertCompileMemberOffset(XAPICPAGE, ppr,         XAPIC_OFF_PPR);
+AssertCompileMemberOffset(XAPICPAGE, eoi,         XAPIC_OFF_EOI);
+AssertCompileMemberOffset(XAPICPAGE, rrd,         XAPIC_OFF_RRD);
+AssertCompileMemberOffset(XAPICPAGE, ldr,         XAPIC_OFF_LDR);
+AssertCompileMemberOffset(XAPICPAGE, dfr,         XAPIC_OFF_DFR);
+AssertCompileMemberOffset(XAPICPAGE, svr,         XAPIC_OFF_SVR);
+AssertCompileMemberOffset(XAPICPAGE, isr,         XAPIC_OFF_ISR0);
+AssertCompileMemberOffset(XAPICPAGE, tmr,         XAPIC_OFF_TMR0);
+AssertCompileMemberOffset(XAPICPAGE, irr,         XAPIC_OFF_IRR0);
+AssertCompileMemberOffset(XAPICPAGE, esr,         XAPIC_OFF_ESR);
+AssertCompileMemberOffset(XAPICPAGE, icr_lo,      XAPIC_OFF_ICR_LO);
+AssertCompileMemberOffset(XAPICPAGE, icr_hi,      XAPIC_OFF_ICR_HI);
+AssertCompileMemberOffset(XAPICPAGE, lvt_timer,   XAPIC_OFF_LVT_TIMER);
+AssertCompileMemberOffset(XAPICPAGE, lvt_thermal, XAPIC_OFF_LVT_THERMAL);
+AssertCompileMemberOffset(XAPICPAGE, lvt_perf,    XAPIC_OFF_LVT_PERF);
+AssertCompileMemberOffset(XAPICPAGE, lvt_lint0,   XAPIC_OFF_LVT_LINT0);
+AssertCompileMemberOffset(XAPICPAGE, lvt_lint1,   XAPIC_OFF_LVT_LINT1);
+AssertCompileMemberOffset(XAPICPAGE, lvt_error,   XAPIC_OFF_LVT_ERROR);
+AssertCompileMemberOffset(XAPICPAGE, timer_icr,   XAPIC_OFF_TIMER_ICR);
+AssertCompileMemberOffset(XAPICPAGE, timer_ccr,   XAPIC_OFF_TIMER_CCR);
+AssertCompileMemberOffset(XAPICPAGE, timer_dcr,   XAPIC_OFF_TIMER_DCR);
+
+/**
+ * The x2APIC memory layout as per Intel/AMD specs.
+ */
+typedef struct X2APICPAGE
+{
+    /* 0x00 - Reserved. */
+    uint32_t                    uReserved0[8];
+    /* 0x20 - APIC ID. */
+    struct
+    {
+        uint32_t                u32ApicId;
+        uint32_t                u32Reserved0[3];
+    } id;
+    /* 0x30 - APIC version register. */
+    union
+    {
+        struct
+        {
+            uint8_t             u8Version;
+            uint8_t             u8Reserved0;
+            uint8_t             u8MaxLvtEntry;
+            uint8_t             fEoiBroadcastSupression : 1;
+            uint8_t             u7Reserved1   : 7;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32Version;
+            uint32_t            u32Reserved2[3];
+        } all;
+    } version;
+    /* 0x40 - Reserved. */
+    uint32_t                    uReserved1[16];
+    /* 0x80 - Task Priority Register (TPR). */
+    struct
+    {
+        uint8_t                 u8Tpr;
+        uint8_t                 u8Reserved0[3];
+        uint32_t                u32Reserved0[3];
+    } tpr;
+    /* 0x90 - Reserved. */
+    uint32_t                    uReserved2[4];
+    /* 0xA0 - Processor Priority Register (PPR). */
+    struct
+    {
+        uint8_t                 u8Ppr;
+        uint8_t                 u8Reserved0[3];
+        uint32_t                u32Reserved0[3];
+    } ppr;
+    /* 0xB0 - End Of Interrupt Register (EOI). */
+    struct
+    {
+        uint32_t                u32Eoi;
+        uint32_t                u32Reserved0[3];
+    } eoi;
+    /* 0xC0 - Remote Read Register (RRD). */
+    struct
+    {
+        uint32_t                u32Rrd;
+        uint32_t                u32Reserved0[3];
+    } rrd;
+    /* 0xD0 - Logical Destination Register (LDR). */
+    struct
+    {
+        uint32_t                u32LogicalApicId;
+        uint32_t                u32Reserved1[3];
+    } ldr;
+    /* 0xE0 - Reserved. */
+    uint32_t                    uReserved3[4];
+    /* 0xF0 - Spurious-Interrupt Vector Register (SVR). */
+    union
+    {
+        struct
+        {
+            uint32_t            u8SpuriousVector     : 8;
+            uint32_t            fApicSoftwareEnable  : 1;
+            uint32_t            u3Reserved0          : 3;
+            uint32_t            fSupressEoiBroadcast : 1;
+            uint32_t            u19Reserved1         : 19;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32Svr;
+            uint32_t            uReserved0[3];
+        } all;
+    } svr;
+    /* 0x100 - In-service Register (ISR). */
+    XAPIC256BITREG              isr;
+    /* 0x180 - Trigger Mode Register (TMR). */
+    XAPIC256BITREG              tmr;
+    /* 0x200 - Interrupt Request Register (IRR). */
+    XAPIC256BITREG              irr;
+    /* 0x280 - Error Status Register (ESR). */
+    union
+    {
+        struct
+        {
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+            uint32_t            u4Reserved0        : 4;
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+            uint32_t            fRedirectableIpi   : 1;
+            uint32_t            fSendIllegalVector : 1;
+            uint32_t            fRcvdIllegalVector : 1;
+            uint32_t            fIllegalRegAddr    : 1;
+            uint32_t            u24Reserved1       : 24;
+            uint32_t            uReserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32Errors;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } esr;
+    /* 0x290 - Reserved. */
+    uint32_t                    uReserved4[28];
+    /* 0x300 - Interrupt Command Register (ICR) - Low. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u3DeliveryMode   : 3;
+            uint32_t            u1DestMode       : 1;
+            uint32_t            u2Reserved0      : 2;
+            uint32_t            u1Level          : 1;
+            uint32_t            u1TriggerMode    : 1;
+            uint32_t            u2Reserved1      : 2;
+            uint32_t            u2DestShorthand  : 2;
+            uint32_t            u12Reserved2     : 12;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32IcrLo;
+            uint32_t            u32Reserved3[3];
+        } all;
+    } icr_lo;
+    /* 0x310 - Interrupt Comannd Register (ICR) - High. */
+    struct
+    {
+        uint32_t                u32IcrHi;
+        uint32_t                uReserved1[3];
+    } icr_hi;
+    /* 0x320 - Local Vector Table (LVT) Timer Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u4Reserved0      : 4;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            u3Reserved1      : 3;
+            uint32_t            u1Mask           : 1;
+            uint32_t            u2TimerMode      : 2;
+            uint32_t            u13Reserved2     : 13;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32LvtTimer;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } lvt_timer;
+    /* 0x330 - Local Vector Table (LVT) Thermal Sensor Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u3DeliveryMode   : 3;
+            uint32_t            u1Reserved0      : 1;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            u3Reserved1      : 3;
+            uint32_t            u1Mask           : 1;
+            uint32_t            u15Reserved2     : 15;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32LvtThermal;
+            uint32_t            uReserved0[3];
+        } all;
+    } lvt_thermal;
+    /* 0x340 - Local Vector Table (LVT) Performance Monitor Counter (PMC) Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u3DeliveryMode   : 3;
+            uint32_t            u1Reserved0      : 1;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            u3Reserved1      : 3;
+            uint32_t            u1Mask           : 1;
+            uint32_t            u15Reserved2     : 15;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32LvtPerf;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } lvt_perf;
+    /* 0x350 - Local Vector Table (LVT) LINT0 Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u3DeliveryMode   : 3;
+            uint32_t            u1Reserved0      : 1;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            u1IntrPolarity   : 1;
+            uint32_t            u1RemoteIrr      : 1;
+            uint32_t            u1TriggerMode    : 1;
+            uint32_t            u1Mask           : 1;
+            uint32_t            u15Reserved2     : 15;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32LvtLint0;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } lvt_lint0;
+    /* 0x360 - Local Vector Table (LVT) LINT1 Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u3DeliveryMode   : 3;
+            uint32_t            u1Reserved0      : 1;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            u1IntrPolarity   : 1;
+            uint32_t            u1RemoteIrr      : 1;
+            uint32_t            u1TriggerMode    : 1;
+            uint32_t            u1Mask           : 1;
+            uint32_t            u15Reserved2     : 15;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32LvtLint1;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } lvt_lint1;
+    /* 0x370 - Local Vector Table (LVT) Error Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u8Vector         : 8;
+            uint32_t            u4Reserved0      : 4;
+            uint32_t            u1DeliveryStatus : 1;
+            uint32_t            u3Reserved1      : 3;
+            uint32_t            u1Mask           : 1;
+            uint32_t            u15Reserved2     : 15;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32LvtError;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } lvt_error;
+    /* 0x380 - Timer Initial Counter Register. */
+    struct
+    {
+        uint32_t                u32InitialCount;
+        uint32_t                u32Reserved0[3];
+    } timer_icr;
+    /* 0x390 - Timer Current Counter Register. */
+    struct
+    {
+        uint32_t                u32CurrentCount;
+        uint32_t                u32Reserved0[3];
+    } timer_ccr;
+    /* 0x3A0 - Reserved. */
+    uint32_t                    uReserved5[16];
+    /* 0x3E0 - Timer Divide Configuration Register. */
+    union
+    {
+        struct
+        {
+            uint32_t            u2DivideValue0 : 2;
+            uint32_t            u1Reserved0    : 1;
+            uint32_t            u1DivideValue1 : 1;
+            uint32_t            u28Reserved1   : 28;
+            uint32_t            u32Reserved0[3];
+        } u;
+        struct
+        {
+            uint32_t            u32DivideValue;
+            uint32_t            u32Reserved0[3];
+        } all;
+    } timer_dcr;
+    /* 0x3F0 - Self IPI Register. */
+    struct
+    {
+        uint32_t                u8Vector     : 8;
+        uint32_t                u24Reserved0 : 24;
+        uint32_t                u32Reserved0[3];
+    } self_ipi;
+    /* 0x400 - Reserved. */
+    uint8_t                     u8Reserved0[3072];
+} X2APICPAGE;
+/** Pointer to a X2APICPAGE struct. */
+typedef X2APICPAGE *PX2APICPAGE;
+/** Pointer to a const X2APICPAGE struct. */
+typedef const X2APICPAGE *PCX2APICPAGE;
+AssertCompileSize(X2APICPAGE, 4096);
+AssertCompileSize(X2APICPAGE, sizeof(XAPICPAGE));
+AssertCompileMemberOffset(X2APICPAGE, id,          XAPIC_OFF_ID);
+AssertCompileMemberOffset(X2APICPAGE, version,     XAPIC_OFF_VERSION);
+AssertCompileMemberOffset(X2APICPAGE, tpr,         XAPIC_OFF_TPR);
+AssertCompileMemberOffset(X2APICPAGE, ppr,         XAPIC_OFF_PPR);
+AssertCompileMemberOffset(X2APICPAGE, eoi,         XAPIC_OFF_EOI);
+AssertCompileMemberOffset(X2APICPAGE, rrd,         XAPIC_OFF_RRD);
+AssertCompileMemberOffset(X2APICPAGE, ldr,         XAPIC_OFF_LDR);
+AssertCompileMemberOffset(X2APICPAGE, svr,         XAPIC_OFF_SVR);
+AssertCompileMemberOffset(X2APICPAGE, isr,         XAPIC_OFF_ISR0);
+AssertCompileMemberOffset(X2APICPAGE, tmr,         XAPIC_OFF_TMR0);
+AssertCompileMemberOffset(X2APICPAGE, irr,         XAPIC_OFF_IRR0);
+AssertCompileMemberOffset(X2APICPAGE, esr,         XAPIC_OFF_ESR);
+AssertCompileMemberOffset(X2APICPAGE, icr_lo,      XAPIC_OFF_ICR_LO);
+AssertCompileMemberOffset(X2APICPAGE, icr_hi,      XAPIC_OFF_ICR_HI);
+AssertCompileMemberOffset(X2APICPAGE, lvt_timer,   XAPIC_OFF_LVT_TIMER);
+AssertCompileMemberOffset(X2APICPAGE, lvt_thermal, XAPIC_OFF_LVT_THERMAL);
+AssertCompileMemberOffset(X2APICPAGE, lvt_perf,    XAPIC_OFF_LVT_PERF);
+AssertCompileMemberOffset(X2APICPAGE, lvt_lint0,   XAPIC_OFF_LVT_LINT0);
+AssertCompileMemberOffset(X2APICPAGE, lvt_lint1,   XAPIC_OFF_LVT_LINT1);
+AssertCompileMemberOffset(X2APICPAGE, lvt_error,   XAPIC_OFF_LVT_ERROR);
+AssertCompileMemberOffset(X2APICPAGE, timer_icr,   XAPIC_OFF_TIMER_ICR);
+AssertCompileMemberOffset(X2APICPAGE, timer_ccr,   XAPIC_OFF_TIMER_CCR);
+AssertCompileMemberOffset(X2APICPAGE, timer_dcr,   XAPIC_OFF_TIMER_DCR);
+AssertCompileMemberOffset(X2APICPAGE, self_ipi,    X2APIC_OFF_SELF_IPI);
+
+/**
+ * APIC MSR access error.
+ * @note The values must match the array indices in apicMsrAccessError().
+ */
+typedef enum APICMSRACCESS
+{
+    /** MSR read while not in x2APIC. */
+    APICMSRACCESS_INVALID_READ_MODE = 0,
+    /** MSR write while not in x2APIC. */
+    APICMSRACCESS_INVALID_WRITE_MODE,
+    /** MSR read for a reserved/unknown/invalid MSR. */
+    APICMSRACCESS_READ_RSVD_OR_UNKNOWN,
+    /** MSR write for a reserved/unknown/invalid MSR. */
+    APICMSRACCESS_WRITE_RSVD_OR_UNKNOWN,
+    /** MSR read for a write-only MSR. */
+    APICMSRACCESS_READ_WRITE_ONLY,
+    /** MSR write for a read-only MSR. */
+    APICMSRACCESS_WRITE_READ_ONLY,
+    /** MSR read to reserved bits. */
+    APICMSRACCESS_READ_RSVD_BITS,
+    /** MSR write to reserved bits. */
+    APICMSRACCESS_WRITE_RSVD_BITS,
+    /** MSR write with invalid value. */
+    APICMSRACCESS_WRITE_INVALID,
+    /** MSR write disallowed due to incompatible config. */
+    APICMSRACCESS_WRITE_DISALLOWED_CONFIG,
+    /** MSR read disallowed due to incompatible config. */
+    APICMSRACCESS_READ_DISALLOWED_CONFIG,
+    /** Count of enum members (don't use). */
+    APICMSRACCESS_COUNT
+} APICMSRACCESS;
+
+/** @name xAPIC Destination Format Register bits.
+ * See Intel spec. 10.6.2.2 "Logical Destination Mode".
+ * @{ */
+typedef enum XAPICDESTFORMAT
+{
+    XAPICDESTFORMAT_FLAT    = 0xf,
+    XAPICDESTFORMAT_CLUSTER = 0
+} XAPICDESTFORMAT;
+/** @} */
+
+/** @name xAPIC Timer Mode bits.
+ * See Intel spec. 10.5.1 "Local Vector Table".
+ * @{ */
+typedef enum XAPICTIMERMODE
+{
+    XAPICTIMERMODE_ONESHOT      = XAPIC_TIMER_MODE_ONESHOT,
+    XAPICTIMERMODE_PERIODIC     = XAPIC_TIMER_MODE_PERIODIC,
+    XAPICTIMERMODE_TSC_DEADLINE = XAPIC_TIMER_MODE_TSC_DEADLINE
+} XAPICTIMERMODE;
+/** @} */
+
+/** @name xAPIC Interrupt Command Register bits.
+ * See Intel spec. 10.6.1 "Interrupt Command Register (ICR)".
+ * See Intel spec. 10.5.1 "Local Vector Table".
+ * @{ */
+/**
+ * xAPIC destination shorthand.
+ */
+typedef enum XAPICDESTSHORTHAND
+{
+    XAPICDESTSHORTHAND_NONE = 0,
+    XAPICDESTSHORTHAND_SELF,
+    XAPIDDESTSHORTHAND_ALL_INCL_SELF,
+    XAPICDESTSHORTHAND_ALL_EXCL_SELF
+} XAPICDESTSHORTHAND;
+
+/**
+ * xAPIC INIT level de-assert delivery mode.
+ */
+typedef enum XAPICINITLEVEL
+{
+    XAPICINITLEVEL_DEASSERT = 0,
+    XAPICINITLEVEL_ASSERT
+} XAPICLEVEL;
+
+/**
+ * xAPIC destination mode.
+ */
+typedef enum XAPICDESTMODE
+{
+    XAPICDESTMODE_PHYSICAL = 0,
+    XAPICDESTMODE_LOGICAL
+} XAPICDESTMODE;
+
+/**
+ * xAPIC delivery mode type.
+ */
+typedef enum XAPICDELIVERYMODE
+{
+    XAPICDELIVERYMODE_FIXED               = 0,
+    XAPICDELIVERYMODE_LOWEST_PRIO         = 1,
+    XAPICDELIVERYMODE_SMI                 = 2,
+    XAPICDELIVERYMODE_NMI                 = 4,
+    XAPICDELIVERYMODE_INIT                = 5,
+    XAPICDELIVERYMODE_STARTUP             = 6,
+    XAPICDELIVERYMODE_EXTINT              = 7
+} XAPICDELIVERYMODE;
+/** @} */
+
+/** @def APIC_CACHE_LINE_SIZE
+ * Padding (in bytes) for aligning data in different cache lines. Present
+ * generation x86 CPUs use 64-byte cache lines[1]. However, Intel NetBurst
+ * architecture supposedly uses 128-byte cache lines[2]. Since 128 is a
+ * multiple of 64, we use the larger one here.
+ *
+ * [1] - Intel spec "Table 11-1. Characteristics of the Caches, TLBs, Store
+ *       Buffer, and Write Combining Buffer in Intel 64 and IA-32 Processors"
+ * [2] - Intel spec. 8.10.6.7 "Place Locks and Semaphores in Aligned, 128-Byte
+ *       Blocks of Memory".
+ */
+#define APIC_CACHE_LINE_SIZE              128
+
+/**
+ * APIC Pending-Interrupt Bitmap (PIB).
+ */
+typedef struct APICPIB
+{
+    uint64_t volatile au64VectorBitmap[4];
+    uint32_t volatile fOutstandingNotification;
+    uint8_t           au8Reserved[APIC_CACHE_LINE_SIZE - sizeof(uint32_t) - (sizeof(uint64_t) * 4)];
+} APICPIB;
+AssertCompileMemberOffset(APICPIB, fOutstandingNotification, 256 / 8);
+AssertCompileSize(APICPIB, APIC_CACHE_LINE_SIZE);
+/** Pointer to a pending-interrupt bitmap. */
+typedef APICPIB *PAPICPIB;
+/** Pointer to a const pending-interrupt bitmap. */
+typedef const APICPIB *PCAPICPIB;
+
+/**
+ * APIC PDM instance data (per-VM).
+ */
+typedef struct APICDEV
+{
+    /** The MMIO handle. */
+    IOMMMIOHANDLE               hMmio;
+} APICDEV;
+/** Pointer to an APIC device. */
+typedef APICDEV *PAPICDEV;
+/** Pointer to a const APIC device. */
+typedef APICDEV const *PCAPICDEV;
+
+
+/**
+ * The APIC GVM instance data.
+ */
+typedef struct APICR0PERVM
+{
+    /** The ring-0 device instance. */
+    PPDMDEVINSR0                pDevInsR0;
+} APICR0PERVM;
+
+
+/**
+ * APIC VM Instance data.
+ */
+typedef struct APIC
+{
+    /** The ring-3 device instance. */
+    PPDMDEVINSR3                pDevInsR3;
+
+    /** @name The APIC pending-interrupt bitmap (PIB).
+     * @{ */
+    /** The host-context physical address of the PIB. */
+    RTHCPHYS                    HCPhysApicPib;
+    /** The ring-0 memory object of the PIB. */
+    RTR0MEMOBJ                  hMemObjApicPibR0;
+    /** The ring-3 mapping of the memory object of the PIB. */
+    RTR0MEMOBJ                  hMapObjApicPibR0;
+    /** The APIC PIB virtual address - R0 ptr. */
+    R0PTRTYPE(void *)           pvApicPibR0;
+    /** The APIC PIB virtual address - R3 ptr. */
+    R3PTRTYPE(void *)           pvApicPibR3;
+    /** The size of the page in bytes. */
+    uint32_t                    cbApicPib;
+    /** @} */
+
+    /** @name Other miscellaneous data.
+     * @{ */
+    /** Whether full APIC register virtualization is enabled. */
+    bool                        fVirtApicRegsEnabled;
+    /** Whether posted-interrupt processing is enabled. */
+    bool                        fPostedIntrsEnabled;
+    /** Whether TSC-deadline timer mode is supported for the guest. */
+    bool                        fSupportsTscDeadline;
+    /** Whether this VM has an IO-APIC. */
+    bool                        fIoApicPresent;
+    /** Whether R0 is enabled or not (applies to MSR handling as well). */
+    bool                        fR0Enabled;
+    /** Whether RC is enabled or not (applies to MSR handling as well). */
+    bool                        fRCEnabled;
+    /** Whether Hyper-V x2APIC compatibility mode is enabled. */
+    bool                        fHyperVCompatMode;
+    /** Alignment padding. */
+    bool                        afAlignment[1];
+    /** The max supported APIC mode from CFGM.  */
+    PDMAPICMODE                 enmMaxMode;
+    /** @} */
+} APIC;
+/** Pointer to APIC VM instance data. */
+typedef APIC *PAPIC;
+/** Pointer to const APIC VM instance data. */
+typedef APIC const *PCAPIC;
+AssertCompileMemberAlignment(APIC, cbApicPib, 8);
+AssertCompileSizeAlignment(APIC, 8);
+
+/**
+ * APIC VMCPU Instance data.
+ */
+typedef struct APICCPU
+{
+    /** @name The APIC page.
+     * @{ */
+    /** The host-context physical address of the page. */
+    RTHCPHYS                    HCPhysApicPage;
+    /** The ring-0 memory object of the page. */
+    RTR0MEMOBJ                  hMemObjApicPageR0;
+    /** The ring-3 mapping of the memory object of the page. */
+    RTR0MEMOBJ                  hMapObjApicPageR0;
+    /** The APIC page virtual address - R0 ptr. */
+    R0PTRTYPE(void *)           pvApicPageR0;
+    /** The APIC page virtual address - R3 ptr. */
+    R3PTRTYPE(void *)           pvApicPageR3;
+    /** The size of the page in bytes. */
+    uint32_t                    cbApicPage;
+    /** @} */
+
+    /** @name Auxiliary state.
+     * @{ */
+    /** The error status register's internal state. */
+    uint32_t                    uEsrInternal;
+    /** The APIC base MSR.*/
+    uint64_t volatile           uApicBaseMsr;
+    /** @} */
+
+    /** @name The pending-interrupt bitmaps (PIB).
+     * @{ */
+    /** The host-context physical address of the page. */
+    RTHCPHYS                    HCPhysApicPib;
+    /** The APIC PIB virtual address - R0 ptr. */
+    R0PTRTYPE(void *)           pvApicPibR0;
+    /** The APIC PIB virtual address - R3 ptr. */
+    R3PTRTYPE(void *)           pvApicPibR3;
+    /** The APIC PIB for level-sensitive interrupts. */
+    APICPIB                     ApicPibLevel;
+    /** @} */
+
+    /** @name Other miscellaneous data.
+     * @{ */
+    /** Whether the LINT0 interrupt line is active. */
+    bool volatile               fActiveLint0;
+    /** Whether the LINT1 interrupt line is active. */
+    bool volatile               fActiveLint1;
+    /** Alignment padding. */
+    uint8_t                     auAlignment2[6];
+    /** The source tags corresponding to each interrupt vector (debugging). */
+    uint32_t                    auSrcTags[256];
+    /** @} */
+
+    /** @name The APIC timer.
+     * @{ */
+    /** The timer. */
+    TMTIMERHANDLE               hTimer;
+    /** The time stamp when the timer was initialized.
+     * @note Access protected by the timer critsect.  */
+    uint64_t                    u64TimerInitial;
+    /** Cache of timer initial count of the frequency hint to TM. */
+    uint32_t                    uHintedTimerInitialCount;
+    /** Cache of timer shift of the frequency hint to TM. */
+    uint32_t                    uHintedTimerShift;
+    /** The timer description. */
+    char                        szTimerDesc[16];
+    /** @} */
+
+    /** @name Log Max counters
+     * @{ */
+    uint32_t                    cLogMaxAccessError;
+    uint32_t                    cLogMaxSetApicBaseAddr;
+    uint32_t                    cLogMaxGetApicBaseAddr;
+    uint32_t                    uAlignment4;
+    /** @} */
+
+#ifdef VBOX_WITH_STATISTICS
+    /** @name APIC statistics.
+     * @{ */
+    /** Number of MMIO reads in RZ. */
+    STAMCOUNTER                 StatMmioReadRZ;
+    /** Number of MMIO reads in R3. */
+    STAMCOUNTER                 StatMmioReadR3;
+
+    /** Number of MMIO writes in RZ. */
+    STAMCOUNTER                 StatMmioWriteRZ;
+    /** Number of MMIO writes in R3. */
+    STAMCOUNTER                 StatMmioWriteR3;
+
+    /** Number of MSR reads in RZ. */
+    STAMCOUNTER                 StatMsrReadRZ;
+    /** Number of MSR reads in R3. */
+    STAMCOUNTER                 StatMsrReadR3;
+
+    /** Number of MSR writes in RZ. */
+    STAMCOUNTER                 StatMsrWriteRZ;
+    /** Number of MSR writes in R3. */
+    STAMCOUNTER                 StatMsrWriteR3;
+
+    /** Profiling of APICUpdatePendingInterrupts().  */
+    STAMPROFILE                 StatUpdatePendingIntrs;
+    /** Profiling of apicPostInterrupt().  */
+    STAMPROFILE                 StatPostIntr;
+    /** Number of times an interrupt is already pending in
+     *  apicPostInterrupts().*/
+    STAMCOUNTER                 StatPostIntrAlreadyPending;
+    /** Number of times the timer callback is invoked. */
+    STAMCOUNTER                 StatTimerCallback;
+    /** Number of times the TPR is written. */
+    STAMCOUNTER                 StatTprWrite;
+    /** Number of times the TPR is read. */
+    STAMCOUNTER                 StatTprRead;
+    /** Number of times the EOI is written. */
+    STAMCOUNTER                 StatEoiWrite;
+    /** Number of times TPR masks an interrupt in apicGetInterrupt(). */
+    STAMCOUNTER                 StatMaskedByTpr;
+    /** Number of times PPR masks an interrupt in apicGetInterrupt(). */
+    STAMCOUNTER                 StatMaskedByPpr;
+    /** Number of times the timer ICR is written. */
+    STAMCOUNTER                 StatTimerIcrWrite;
+    /** Number of times the ICR Lo (send IPI) is written. */
+    STAMCOUNTER                 StatIcrLoWrite;
+    /** Number of times the ICR Hi is written. */
+    STAMCOUNTER                 StatIcrHiWrite;
+    /** Number of times the full ICR (x2APIC send IPI) is written. */
+    STAMCOUNTER                 StatIcrFullWrite;
+    /** Number of times the APIC-ID MSR is read. */
+    STAMCOUNTER                 StatIdMsrRead;
+    /** @} */
+#endif
+} APICCPU;
+/** Pointer to APIC VMCPU instance data. */
+typedef APICCPU *PAPICCPU;
+/** Pointer to a const APIC VMCPU instance data. */
+typedef APICCPU const *PCAPICCPU;
+AssertCompileMemberAlignment(APICCPU, uApicBaseMsr, 8);
+
+/**
+ * APIC operating modes as returned by apicGetMode().
+ *
+ * The values match hardware states.
+ * See Intel spec. 10.12.1 "Detecting and Enabling x2APIC Mode".
+ */
+typedef enum APICMODE
+{
+    APICMODE_DISABLED = 0,
+    APICMODE_INVALID,
+    APICMODE_XAPIC,
+    APICMODE_X2APIC
+} APICMODE;
+
+/**
+ * Gets the timer shift value.
+ *
+ * @returns The timer shift value.
+ * @param   pXApicPage      The xAPIC page.
+ */
+DECLINLINE(uint8_t) apicGetTimerShift(PCXAPICPAGE pXApicPage)
+{
+    /* See Intel spec. 10.5.4 "APIC Timer". */
+    uint32_t uShift = pXApicPage->timer_dcr.u.u2DivideValue0 | (pXApicPage->timer_dcr.u.u1DivideValue1 << 2);
+    return (uShift + 1) & 7;
+}
+
+
+const char                   *apicGetModeName(APICMODE enmMode);
+const char                   *apicGetDestFormatName(XAPICDESTFORMAT enmDestFormat);
+const char                   *apicGetDeliveryModeName(XAPICDELIVERYMODE enmDeliveryMode);
+const char                   *apicGetDestModeName(XAPICDESTMODE enmDestMode);
+const char                   *apicGetTriggerModeName(XAPICTRIGGERMODE enmTriggerMode);
+const char                   *apicGetDestShorthandName(XAPICDESTSHORTHAND enmDestShorthand);
+const char                   *apicGetTimerModeName(XAPICTIMERMODE enmTimerMode);
+void                          apicHintTimerFreq(PPDMDEVINS pDevIns, PAPICCPU pApicCpu, uint32_t uInitialCount, uint8_t uTimerShift);
+APICMODE                      apicGetMode(uint64_t uApicBaseMsr);
+
+DECLCALLBACK(VBOXSTRICTRC)    apicReadMmio(PPDMDEVINS pDevIns, void *pvUser, RTGCPHYS off, void *pv, unsigned cb);
+DECLCALLBACK(VBOXSTRICTRC)    apicWriteMmio(PPDMDEVINS pDevIns, void *pvUser, RTGCPHYS off, void const *pv, unsigned cb);
+
+bool                          apicPostInterrupt(PVMCPUCC pVCpu, uint8_t uVector, XAPICTRIGGERMODE enmTriggerMode, uint32_t uSrcTag);
+void                          apicStartTimer(PVMCPUCC pVCpu, uint32_t uInitialCount);
+void                          apicClearInterruptFF(PVMCPUCC pVCpu, PDMAPICIRQ enmType);
+void                          apicInitIpi(PVMCPUCC pVCpu);
+void                          apicResetCpu(PVMCPUCC pVCpu, bool fResetApicBaseMsr);
+
+DECLCALLBACK(int)             apicR3Construct(PPDMDEVINS pDevIns, int iInstance, PCFGMNODE pCfg);
+DECLCALLBACK(int)             apicR3Destruct(PPDMDEVINS pDevIns);
+DECLCALLBACK(void)            apicR3Relocate(PPDMDEVINS pDevIns, RTGCINTPTR offDelta);
+DECLCALLBACK(void)            apicR3Reset(PPDMDEVINS pDevIns);
+DECLCALLBACK(int)             apicR3InitComplete(PPDMDEVINS pDevIns);
+
+/** @} */
+
+#endif /* !VMM_INCLUDED_SRC_include_APICInternal_h */
+
diff --git a/src/VBox/VMM/include/CFGMInternal.h b/src/VBox/VMM/include/CFGMInternal.h
new file mode 100644
index 00000000..054082e9
--- /dev/null
+++ b/src/VBox/VMM/include/CFGMInternal.h
@@ -0,0 +1,134 @@
+/* $Id: CFGMInternal.h $ */
+/** @file
+ * CFGM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_CFGMInternal_h
+#define VMM_INCLUDED_SRC_include_CFGMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+
+
+/** @defgroup grp_cfgm_int Internals.
+ * @ingroup grp_cfgm
+ * @{
+ */
+
+
+/**
+ * Configuration manager propertype value.
+ */
+typedef union CFGMVALUE
+{
+    /** Integer value. */
+    struct CFGMVALUE_INTEGER
+    {
+        /** The integer represented as 64-bit unsigned. */
+        uint64_t        u64;
+    } Integer;
+
+    /** String value. (UTF-8 of course) */
+    struct CFGMVALUE_STRING
+    {
+        /** Length of string. (In bytes, including the terminator.) */
+        size_t          cb;
+        /** Pointer to the string. */
+        char           *psz;
+    } String;
+
+    /** Byte string value. */
+    struct CFGMVALUE_BYTES
+    {
+        /** Length of byte string. (in bytes) */
+        size_t          cb;
+        /** Pointer to the byte string. */
+        uint8_t        *pau8;
+    } Bytes;
+} CFGMVALUE;
+/** Pointer to configuration manager property value. */
+typedef CFGMVALUE *PCFGMVALUE;
+
+
+/**
+ * Configuration manager tree node.
+ */
+typedef struct CFGMLEAF
+{
+    /** Pointer to the next leaf. */
+    PCFGMLEAF       pNext;
+    /** Pointer to the previous leaf. */
+    PCFGMLEAF       pPrev;
+
+    /** Property type. */
+    CFGMVALUETYPE   enmType;
+    /** Property value. */
+    CFGMVALUE       Value;
+
+    /** Name length. (exclusive) */
+    size_t          cchName;
+    /** Name. */
+    char            szName[1];
+} CFGMLEAF;
+
+
+/**
+ * Configuration manager tree node.
+ */
+typedef struct CFGMNODE
+{
+    /** Pointer to the next node (on this level). */
+    PCFGMNODE       pNext;
+    /** Pointer to the previous node (on this level). */
+    PCFGMNODE       pPrev;
+    /** Pointer Parent node. */
+    PCFGMNODE       pParent;
+    /** Pointer to first child node. */
+    PCFGMNODE       pFirstChild;
+    /** Pointer to first property leaf. */
+    PCFGMLEAF       pFirstLeaf;
+
+    /** Pointer to the VM owning this node. */
+    PVM             pVM;
+
+    /** The root of a 'restricted' subtree, i.e. the parent is
+     * invisible to non-trusted users.
+     */
+    bool            fRestrictedRoot;
+
+    /** Name length. (exclusive) */
+    size_t          cchName;
+    /** Name. */
+    char            szName[1];
+} CFGMNODE;
+
+
+
+/**
+ * CFGM VM Instance data.
+ * Changes to this must checked against the padding of the cfgm union in VM!
+ */
+typedef struct CFGM
+{
+    /** Pointer to root node. */
+    R3PTRTYPE(PCFGMNODE)    pRoot;
+} CFGM;
+
+/** @} */
+
+#endif /* !VMM_INCLUDED_SRC_include_CFGMInternal_h */
diff --git a/src/VBox/VMM/include/CPUMInternal.h b/src/VBox/VMM/include/CPUMInternal.h
new file mode 100644
index 00000000..2ed78863
--- /dev/null
+++ b/src/VBox/VMM/include/CPUMInternal.h
@@ -0,0 +1,534 @@
+/* $Id: CPUMInternal.h $ */
+/** @file
+ * CPUM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_CPUMInternal_h
+#define VMM_INCLUDED_SRC_include_CPUMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#ifndef VBOX_FOR_DTRACE_LIB
+# include <VBox/cdefs.h>
+# include <VBox/types.h>
+# include <VBox/vmm/stam.h>
+# include <iprt/x86.h>
+# include <VBox/vmm/pgm.h>
+#else
+# pragma D depends_on library x86.d
+# pragma D depends_on library cpumctx.d
+# pragma D depends_on library cpum.d
+
+/* Some fudging. */
+typedef uint64_t STAMCOUNTER;
+#endif
+
+
+
+
+/** @defgroup grp_cpum_int   Internals
+ * @ingroup grp_cpum
+ * @internal
+ * @{
+ */
+
+/** Flags and types for CPUM fault handlers
+ * @{ */
+/** Type: Load DS */
+#define CPUM_HANDLER_DS                 1
+/** Type: Load ES */
+#define CPUM_HANDLER_ES                 2
+/** Type: Load FS */
+#define CPUM_HANDLER_FS                 3
+/** Type: Load GS */
+#define CPUM_HANDLER_GS                 4
+/** Type: IRET */
+#define CPUM_HANDLER_IRET               5
+/** Type mask. */
+#define CPUM_HANDLER_TYPEMASK           0xff
+/** If set EBP points to the CPUMCTXCORE that's being used. */
+#define CPUM_HANDLER_CTXCORE_IN_EBP     RT_BIT(31)
+/** @} */
+
+
+/** Use flags (CPUM::fUseFlags).
+ * (Don't forget to sync this with CPUMInternal.mac !)
+ * @note Part of saved state.
+ * @{ */
+/** Indicates that we've saved the host FPU, SSE, whatever state and that it
+ * needs to be restored. */
+#define CPUM_USED_FPU_HOST              RT_BIT(0)
+/** Indicates that we've loaded the guest FPU, SSE, whatever state and that it
+ * needs to be saved. */
+#define CPUM_USED_FPU_GUEST             RT_BIT(10)
+/** Used the guest FPU, SSE or such stuff since last we were in REM.
+ * REM syncing is clearing this, lazy FPU is setting it. */
+#define CPUM_USED_FPU_SINCE_REM         RT_BIT(1)
+/** The XMM state was manually restored. (AMD only) */
+#define CPUM_USED_MANUAL_XMM_RESTORE    RT_BIT(2)
+
+/** Host OS is using SYSENTER and we must NULL the CS. */
+#define CPUM_USE_SYSENTER               RT_BIT(3)
+/** Host OS is using SYSENTER and we must NULL the CS. */
+#define CPUM_USE_SYSCALL                RT_BIT(4)
+
+/** Debug registers are used by host and that DR7 and DR6 must be saved and
+ *  disabled when switching to raw-mode. */
+#define CPUM_USE_DEBUG_REGS_HOST        RT_BIT(5)
+/** Records that we've saved the host DRx registers.
+ * In ring-0 this means all (DR0-7), while in raw-mode context this means DR0-3
+ * since DR6 and DR7 are covered by CPUM_USE_DEBUG_REGS_HOST. */
+#define CPUM_USED_DEBUG_REGS_HOST       RT_BIT(6)
+/** Set to indicate that we should save host DR0-7 and load the hypervisor debug
+ * registers in the raw-mode world switchers. (See CPUMRecalcHyperDRx.) */
+#define CPUM_USE_DEBUG_REGS_HYPER       RT_BIT(7)
+/** Used in ring-0 to indicate that we have loaded the hypervisor debug
+ * registers. */
+#define CPUM_USED_DEBUG_REGS_HYPER      RT_BIT(8)
+/** Used in ring-0 to indicate that we have loaded the guest debug
+ * registers (DR0-3 and maybe DR6) for direct use by the guest.
+ * DR7 (and AMD-V DR6) are handled via the VMCB. */
+#define CPUM_USED_DEBUG_REGS_GUEST      RT_BIT(9)
+
+/** Sync the FPU state on next entry (32->64 switcher only). */
+#define CPUM_SYNC_FPU_STATE             RT_BIT(16)
+/** Sync the debug state on next entry (32->64 switcher only). */
+#define CPUM_SYNC_DEBUG_REGS_GUEST      RT_BIT(17)
+/** Sync the debug state on next entry (32->64 switcher only).
+ * Almost the same as CPUM_USE_DEBUG_REGS_HYPER in the raw-mode switchers. */
+#define CPUM_SYNC_DEBUG_REGS_HYPER      RT_BIT(18)
+/** Host CPU requires fxsave/fxrstor leaky bit handling. */
+#define CPUM_USE_FFXSR_LEAKY            RT_BIT(19)
+/** Set if the VM supports long-mode. */
+#define CPUM_USE_SUPPORTS_LONGMODE      RT_BIT(20)
+/** @} */
+
+
+/** @name CPUM Saved State Version.
+ * @{ */
+/** The current saved state version. */
+#define CPUM_SAVED_STATE_VERSION                CPUM_SAVED_STATE_VERSION_HWVIRT_VMX_IEM
+/** The saved state version including VMX hardware virtualization state (IEM only
+ *  execution). */
+#define CPUM_SAVED_STATE_VERSION_HWVIRT_VMX_IEM 19
+/** The saved state version including SVM hardware virtualization state. */
+#define CPUM_SAVED_STATE_VERSION_HWVIRT_SVM     18
+/** The saved state version including XSAVE state. */
+#define CPUM_SAVED_STATE_VERSION_XSAVE          17
+/** The saved state version with good CPUID leaf count. */
+#define CPUM_SAVED_STATE_VERSION_GOOD_CPUID_COUNT 16
+/** CPUID changes with explode forgetting to update the leaf count on
+ * restore, resulting in garbage being saved restoring+saving old states). */
+#define CPUM_SAVED_STATE_VERSION_BAD_CPUID_COUNT 15
+/** The saved state version before the CPUIDs changes. */
+#define CPUM_SAVED_STATE_VERSION_PUT_STRUCT     14
+/** The saved state version before using SSMR3PutStruct. */
+#define CPUM_SAVED_STATE_VERSION_MEM            13
+/** The saved state version before introducing the MSR size field. */
+#define CPUM_SAVED_STATE_VERSION_NO_MSR_SIZE    12
+/** The saved state version of 3.2, 3.1 and 3.3 trunk before the hidden
+ * selector register change (CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID). */
+#define CPUM_SAVED_STATE_VERSION_VER3_2         11
+/** The saved state version of 3.0 and 3.1 trunk before the teleportation
+ * changes. */
+#define CPUM_SAVED_STATE_VERSION_VER3_0         10
+/** The saved state version for the 2.1 trunk before the MSR changes. */
+#define CPUM_SAVED_STATE_VERSION_VER2_1_NOMSR   9
+/** The saved state version of 2.0, used for backwards compatibility. */
+#define CPUM_SAVED_STATE_VERSION_VER2_0         8
+/** The saved state version of 1.6, used for backwards compatibility. */
+#define CPUM_SAVED_STATE_VERSION_VER1_6         6
+/** @} */
+
+
+/**
+ * CPU info
+ */
+typedef struct CPUMINFO
+{
+    /** The number of MSR ranges (CPUMMSRRANGE) in the array pointed to below. */
+    uint32_t                    cMsrRanges;
+    /** Mask applied to ECX before looking up the MSR for a RDMSR/WRMSR
+     * instruction.  Older hardware has been observed to ignore higher bits. */
+    uint32_t                    fMsrMask;
+
+    /** MXCSR mask. */
+    uint32_t                    fMxCsrMask;
+
+    /** The number of CPUID leaves (CPUMCPUIDLEAF) in the array pointed to below. */
+    uint32_t                    cCpuIdLeaves;
+    /** The index of the first extended CPUID leaf in the array.
+     *  Set to cCpuIdLeaves if none present. */
+    uint32_t                    iFirstExtCpuIdLeaf;
+    /** How to handle unknown CPUID leaves. */
+    CPUMUNKNOWNCPUID            enmUnknownCpuIdMethod;
+    /** For use with CPUMUNKNOWNCPUID_DEFAULTS (DB & VM),
+     * CPUMUNKNOWNCPUID_LAST_STD_LEAF (VM) and CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX (VM). */
+    CPUMCPUID                   DefCpuId;
+
+    /** Scalable bus frequency used for reporting other frequencies. */
+    uint64_t                    uScalableBusFreq;
+
+    /** Pointer to the MSR ranges (ring-0 pointer). */
+    R0PTRTYPE(PCPUMMSRRANGE)    paMsrRangesR0;
+    /** Pointer to the CPUID leaves (ring-0 pointer). */
+    R0PTRTYPE(PCPUMCPUIDLEAF)   paCpuIdLeavesR0;
+
+    /** Pointer to the MSR ranges (ring-3 pointer). */
+    R3PTRTYPE(PCPUMMSRRANGE)    paMsrRangesR3;
+    /** Pointer to the CPUID leaves (ring-3 pointer). */
+    R3PTRTYPE(PCPUMCPUIDLEAF)   paCpuIdLeavesR3;
+} CPUMINFO;
+/** Pointer to a CPU info structure. */
+typedef CPUMINFO *PCPUMINFO;
+/** Pointer to a const CPU info structure. */
+typedef CPUMINFO const *CPCPUMINFO;
+
+
+/**
+ * The saved host CPU state.
+ */
+typedef struct CPUMHOSTCTX
+{
+    /** General purpose register, selectors, flags and more
+     * @{ */
+    /** General purpose register ++
+     * { */
+    /*uint64_t        rax; - scratch*/
+    uint64_t        rbx;
+    /*uint64_t        rcx; - scratch*/
+    /*uint64_t        rdx; - scratch*/
+    uint64_t        rdi;
+    uint64_t        rsi;
+    uint64_t        rbp;
+    uint64_t        rsp;
+    /*uint64_t        r8; - scratch*/
+    /*uint64_t        r9; - scratch*/
+    uint64_t        r10;
+    uint64_t        r11;
+    uint64_t        r12;
+    uint64_t        r13;
+    uint64_t        r14;
+    uint64_t        r15;
+    /*uint64_t        rip; - scratch*/
+    uint64_t        rflags;
+    /** @} */
+
+    /** Selector registers
+     * @{ */
+    RTSEL           ss;
+    RTSEL           ssPadding;
+    RTSEL           gs;
+    RTSEL           gsPadding;
+    RTSEL           fs;
+    RTSEL           fsPadding;
+    RTSEL           es;
+    RTSEL           esPadding;
+    RTSEL           ds;
+    RTSEL           dsPadding;
+    RTSEL           cs;
+    RTSEL           csPadding;
+    /** @} */
+
+    /** Control registers.
+     * @{ */
+    /** The CR0 FPU state in HM mode.  */
+    uint64_t        cr0;
+    /*uint64_t        cr2; - scratch*/
+    uint64_t        cr3;
+    uint64_t        cr4;
+    uint64_t        cr8;
+    /** @} */
+
+    /** Debug registers.
+     * @{ */
+    uint64_t        dr0;
+    uint64_t        dr1;
+    uint64_t        dr2;
+    uint64_t        dr3;
+    uint64_t        dr6;
+    uint64_t        dr7;
+    /** @} */
+
+    /** Global Descriptor Table register. */
+    X86XDTR64       gdtr;
+    uint16_t        gdtrPadding;
+    /** Interrupt Descriptor Table register. */
+    X86XDTR64       idtr;
+    uint16_t        idtrPadding;
+    /** The task register. */
+    RTSEL           ldtr;
+    RTSEL           ldtrPadding;
+    /** The task register. */
+    RTSEL           tr;
+    RTSEL           trPadding;
+
+    /** MSRs
+     * @{ */
+    CPUMSYSENTER    SysEnter;
+    uint64_t        FSbase;
+    uint64_t        GSbase;
+    uint64_t        efer;
+    /** @} */
+
+    /* padding to get 64byte aligned size */
+    uint8_t         auPadding[8];
+
+#if HC_ARCH_BITS != 64
+# error HC_ARCH_BITS not defined or unsupported
+#endif
+
+    /** Pointer to the FPU/SSE/AVX/XXXX state ring-0 mapping. */
+    R0PTRTYPE(PX86XSAVEAREA)    pXStateR0;
+    /** Pointer to the FPU/SSE/AVX/XXXX state ring-3 mapping. */
+    R3PTRTYPE(PX86XSAVEAREA)    pXStateR3;
+    /** The XCR0 register. */
+    uint64_t                    xcr0;
+    /** The mask to pass to XSAVE/XRSTOR in EDX:EAX.  If zero we use
+     *  FXSAVE/FXRSTOR (since bit 0 will always be set, we only need to test it). */
+    uint64_t                    fXStateMask;
+} CPUMHOSTCTX;
+#ifndef VBOX_FOR_DTRACE_LIB
+AssertCompileSizeAlignment(CPUMHOSTCTX, 64);
+#endif
+/** Pointer to the saved host CPU state. */
+typedef CPUMHOSTCTX *PCPUMHOSTCTX;
+
+
+/**
+ * The hypervisor context CPU state (just DRx left now).
+ */
+typedef struct CPUMHYPERCTX
+{
+    /** Debug registers.
+     * @remarks DR4 and DR5 should not be used since they are aliases for
+     *          DR6 and DR7 respectively on both AMD and Intel CPUs.
+     * @remarks DR8-15 are currently not supported by AMD or Intel, so
+     *          neither do we.
+     */
+    uint64_t        dr[8];
+    /** @todo eliminiate the rest.   */
+    uint64_t        cr3;
+    uint64_t        au64Padding[7];
+} CPUMHYPERCTX;
+#ifndef VBOX_FOR_DTRACE_LIB
+AssertCompileSizeAlignment(CPUMHYPERCTX, 64);
+#endif
+/** Pointer to the hypervisor context CPU state. */
+typedef CPUMHYPERCTX *PCPUMHYPERCTX;
+
+
+/**
+ * CPUM Data (part of VM)
+ */
+typedef struct CPUM
+{
+    /** Use flags.
+     * These flags indicates which CPU features the host uses.
+     */
+    uint32_t                fHostUseFlags;
+
+    /** CR4 mask */
+    struct
+    {
+        uint32_t            AndMask; /**< @todo Move these to the per-CPU structure and fix the switchers. Saves a register! */
+        uint32_t            OrMask;
+    } CR4;
+
+    /** The (more) portable CPUID level. */
+    uint8_t                 u8PortableCpuIdLevel;
+    /** Indicates that a state restore is pending.
+     * This is used to verify load order dependencies (PGM). */
+    bool                    fPendingRestore;
+    uint8_t                 abPadding0[2];
+
+    /** XSAVE/XRTOR components we can expose to the guest mask. */
+    uint64_t                fXStateGuestMask;
+    /** XSAVE/XRSTOR host mask.  Only state components in this mask can be exposed
+     * to the guest.  This is 0 if no XSAVE/XRSTOR bits can be exposed. */
+    uint64_t                fXStateHostMask;
+
+    /** The host MXCSR mask (determined at init). */
+    uint32_t                fHostMxCsrMask;
+    /** Nested VMX: Whether to expose VMX-preemption timer to the guest. */
+    bool                    fNestedVmxPreemptTimer;
+    uint8_t                 abPadding1[3];
+
+    /** Align to 64-byte boundary. */
+    uint8_t                 abPadding2[20+4];
+
+    /** Host CPU feature information.
+     * Externaly visible via the VM structure, aligned on 64-byte boundrary. */
+    CPUMFEATURES            HostFeatures;
+    /** Guest CPU feature information.
+     * Externaly visible via that VM structure, aligned with HostFeatures. */
+    CPUMFEATURES            GuestFeatures;
+    /** Guest CPU info. */
+    CPUMINFO                GuestInfo;
+
+    /** The standard set of CpuId leaves. */
+    CPUMCPUID               aGuestCpuIdPatmStd[6];
+    /** The extended set of CpuId leaves. */
+    CPUMCPUID               aGuestCpuIdPatmExt[10];
+    /** The centaur set of CpuId leaves. */
+    CPUMCPUID               aGuestCpuIdPatmCentaur[4];
+
+    /** @name MSR statistics.
+     * @{ */
+    STAMCOUNTER             cMsrWrites;
+    STAMCOUNTER             cMsrWritesToIgnoredBits;
+    STAMCOUNTER             cMsrWritesRaiseGp;
+    STAMCOUNTER             cMsrWritesUnknown;
+    STAMCOUNTER             cMsrReads;
+    STAMCOUNTER             cMsrReadsRaiseGp;
+    STAMCOUNTER             cMsrReadsUnknown;
+    /** @} */
+} CPUM;
+#ifndef VBOX_FOR_DTRACE_LIB
+AssertCompileMemberOffset(CPUM, HostFeatures, 64);
+AssertCompileMemberOffset(CPUM, GuestFeatures, 112);
+#endif
+/** Pointer to the CPUM instance data residing in the shared VM structure. */
+typedef CPUM *PCPUM;
+
+/**
+ * CPUM Data (part of VMCPU)
+ */
+typedef struct CPUMCPU
+{
+    /**
+     * Guest context.
+     * Aligned on a 64-byte boundary.
+     */
+    CPUMCTX                 Guest;
+
+    /**
+     * Guest context - misc MSRs
+     * Aligned on a 64-byte boundary.
+     */
+    CPUMCTXMSRS             GuestMsrs;
+
+    /** Nested VMX: VMX-preemption timer - R0 ptr. */
+    PTMTIMERR0              pNestedVmxPreemptTimerR0;
+    /** Nested VMX: VMX-preemption timer - R3 ptr. */
+    PTMTIMERR3              pNestedVmxPreemptTimerR3;
+
+    /** Use flags.
+     * These flags indicates both what is to be used and what has been used.
+     */
+    uint32_t                fUseFlags;
+
+    /** Changed flags.
+     * These flags indicates to REM (and others) which important guest
+     * registers which has been changed since last time the flags were cleared.
+     * See the CPUM_CHANGED_* defines for what we keep track of.
+     */
+    uint32_t                fChanged;
+
+    /** Temporary storage for the return code of the function called in the
+     * 32-64 switcher. */
+    uint32_t                u32RetCode;
+
+#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+    /** Used by the world switcher code to store which vectors needs restoring on
+     * the way back. */
+    uint32_t                fApicDisVectors;
+    /** The address of the APIC mapping, NULL if no APIC.
+     * Call CPUMR0SetLApic to update this before doing a world switch. */
+    RTHCPTR                 pvApicBase;
+    /** Set if the CPU has the X2APIC mode enabled.
+     * Call CPUMR0SetLApic to update this before doing a world switch. */
+    bool                    fX2Apic;
+#else
+    uint8_t                 abPadding3[4 + sizeof(RTHCPTR) + 1];
+#endif
+
+    /** Have we entered the recompiler? */
+    bool                    fRemEntered;
+    /** Whether the X86_CPUID_FEATURE_EDX_APIC and X86_CPUID_AMD_FEATURE_EDX_APIC
+     *  (?) bits are visible or not.  (The APIC is responsible for setting this
+     *  when loading state, so we won't save it.) */
+    bool                    fCpuIdApicFeatureVisible;
+
+    /** Align the next member on a 64-byte boundary. */
+    uint8_t                 abPadding2[64 - (16 + 12 + 4 + 8 + 1 + 2)];
+
+    /** Saved host context.  Only valid while inside RC or HM contexts.
+     * Must be aligned on a 64-byte boundary. */
+    CPUMHOSTCTX             Host;
+    /** Old hypervisor context, only used for combined DRx values now.
+     * Must be aligned on a 64-byte boundary. */
+    CPUMHYPERCTX            Hyper;
+
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    uint8_t                 aMagic[56];
+    uint64_t                uMagic;
+#endif
+} CPUMCPU;
+/** Pointer to the CPUMCPU instance data residing in the shared VMCPU structure. */
+typedef CPUMCPU *PCPUMCPU;
+
+#ifndef VBOX_FOR_DTRACE_LIB
+RT_C_DECLS_BEGIN
+
+PCPUMCPUIDLEAF      cpumCpuIdGetLeaf(PVM pVM, uint32_t uLeaf);
+PCPUMCPUIDLEAF      cpumCpuIdGetLeafEx(PVM pVM, uint32_t uLeaf, uint32_t uSubLeaf, bool *pfExactSubLeafHit);
+
+# ifdef IN_RING3
+int                 cpumR3DbgInit(PVM pVM);
+int                 cpumR3CpuIdExplodeFeatures(PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, PCCPUMMSRS pMsrs, PCPUMFEATURES pFeatures);
+int                 cpumR3InitCpuIdAndMsrs(PVM pVM, PCCPUMMSRS pHostMsrs);
+void                cpumR3InitVmxGuestFeaturesAndMsrs(PVM pVM, PCVMXMSRS pHostVmxMsrs, PVMXMSRS pGuestVmxMsrs);
+void                cpumR3SaveCpuId(PVM pVM, PSSMHANDLE pSSM);
+int                 cpumR3LoadCpuId(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, PCCPUMMSRS pGuestMsrs);
+int                 cpumR3LoadCpuIdPre32(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion);
+DECLCALLBACK(void)  cpumR3CpuIdInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+int                 cpumR3DbGetCpuInfo(const char *pszName, PCPUMINFO pInfo);
+int                 cpumR3MsrRangesInsert(PVM pVM, PCPUMMSRRANGE *ppaMsrRanges, uint32_t *pcMsrRanges, PCCPUMMSRRANGE pNewRange);
+int                 cpumR3MsrReconcileWithCpuId(PVM pVM);
+int                 cpumR3MsrApplyFudge(PVM pVM);
+int                 cpumR3MsrRegStats(PVM pVM);
+int                 cpumR3MsrStrictInitChecks(void);
+PCPUMMSRRANGE       cpumLookupMsrRange(PVM pVM, uint32_t idMsr);
+# endif
+
+# ifdef IN_RC
+DECLASM(int)        cpumHandleLazyFPUAsm(PCPUMCPU pCPUM);
+# endif
+
+# ifdef IN_RING0
+DECLASM(int)        cpumR0SaveHostRestoreGuestFPUState(PCPUMCPU pCPUM);
+DECLASM(void)       cpumR0SaveGuestRestoreHostFPUState(PCPUMCPU pCPUM);
+#  if ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
+DECLASM(void)       cpumR0RestoreHostFPUState(PCPUMCPU pCPUM);
+#  endif
+# endif
+
+# if defined(IN_RC) || defined(IN_RING0)
+DECLASM(int)        cpumRZSaveHostFPUState(PCPUMCPU pCPUM);
+DECLASM(void)       cpumRZSaveGuestFpuState(PCPUMCPU pCPUM, bool fLeaveFpuAccessible);
+DECLASM(void)       cpumRZSaveGuestSseRegisters(PCPUMCPU pCPUM);
+DECLASM(void)       cpumRZSaveGuestAvxRegisters(PCPUMCPU pCPUM);
+# endif
+
+RT_C_DECLS_END
+#endif /* !VBOX_FOR_DTRACE_LIB */
+
+/** @} */
+
+#endif /* !VMM_INCLUDED_SRC_include_CPUMInternal_h */
+
diff --git a/src/VBox/VMM/include/CPUMInternal.mac b/src/VBox/VMM/include/CPUMInternal.mac
new file mode 100644
index 00000000..4b571409
--- /dev/null
+++ b/src/VBox/VMM/include/CPUMInternal.mac
@@ -0,0 +1,709 @@
+; $Id: CPUMInternal.mac $
+;; @file
+; CPUM - Internal header file (asm).
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%include "VBox/asmdefs.mac"
+%include "VBox/vmm/cpum.mac"
+
+;; Check sanity.
+%ifdef VBOX_WITH_KERNEL_USING_XMM
+ %ifndef IN_RING0
+  %error "What? We've got code assuming VBOX_WITH_KERNEL_USING_XMM is only defined in ring-0!"
+ %endif
+%endif
+
+;; For numeric expressions
+%ifdef RT_ARCH_AMD64
+ %define CPUM_IS_AMD64      1
+%else
+ %define CPUM_IS_AMD64      0
+%endif
+
+
+;;
+; CPU info
+struc CPUMINFO
+    .cMsrRanges             resd 1                  ; uint32_t
+    .fMsrMask               resd 1                  ; uint32_t
+    .fMxCsrMask             resd 1                  ; uint32_t
+    .cCpuIdLeaves           resd 1                  ; uint32_t
+    .iFirstExtCpuIdLeaf     resd 1                  ; uint32_t
+    .enmUnknownCpuIdMethod  resd 1                  ; CPUMUNKNOWNCPUID
+    .DefCpuId               resb CPUMCPUID_size     ; CPUMCPUID
+    .uScalableBusFreq       resq 1                  ; uint64_t
+    .paMsrRangesR0          RTR0PTR_RES 1           ; R0PTRTYPE(PCPUMMSRRANGE)
+    .paCpuIdLeavesR0        RTR0PTR_RES 1           ; R0PTRTYPE(PCPUMCPUIDLEAF)
+    .paMsrRangesR3          RTR3PTR_RES 1           ; R3PTRTYPE(PCPUMMSRRANGE)
+    .paCpuIdLeavesR3        RTR3PTR_RES 1           ; R3PTRTYPE(PCPUMCPUIDLEAF)
+endstruc
+
+
+%define CPUM_USED_FPU_HOST              RT_BIT(0)
+%define CPUM_USED_FPU_GUEST             RT_BIT(10)
+%define CPUM_USED_FPU_SINCE_REM         RT_BIT(1)
+%define CPUM_USED_MANUAL_XMM_RESTORE    RT_BIT(2)
+%define CPUM_USE_SYSENTER               RT_BIT(3)
+%define CPUM_USE_SYSCALL                RT_BIT(4)
+%define CPUM_USE_DEBUG_REGS_HOST        RT_BIT(5)
+%define CPUM_USED_DEBUG_REGS_HOST       RT_BIT(6)
+%define CPUM_USE_DEBUG_REGS_HYPER       RT_BIT(7)
+%define CPUM_USED_DEBUG_REGS_HYPER      RT_BIT(8)
+%define CPUM_USED_DEBUG_REGS_GUEST      RT_BIT(9)
+%define CPUM_SYNC_FPU_STATE             RT_BIT(16)
+%define CPUM_SYNC_DEBUG_REGS_GUEST      RT_BIT(17)
+%define CPUM_SYNC_DEBUG_REGS_HYPER      RT_BIT(18)
+%define CPUM_USE_FFXSR_LEAKY            RT_BIT(19)
+%define CPUM_USE_SUPPORTS_LONGMODE      RT_BIT(20)
+
+%define CPUM_HANDLER_DS                 1
+%define CPUM_HANDLER_ES                 2
+%define CPUM_HANDLER_FS                 3
+%define CPUM_HANDLER_GS                 4
+%define CPUM_HANDLER_IRET               5
+%define CPUM_HANDLER_TYPEMASK           0ffh
+%define CPUM_HANDLER_CTXCORE_IN_EBP     RT_BIT(31)
+
+
+struc CPUM
+    ;...
+    .fHostUseFlags              resd    1
+
+    ; CR4 masks
+    .CR4.AndMask                resd    1
+    .CR4.OrMask                 resd    1
+    .u8PortableCpuIdLevel       resb    1
+    .fPendingRestore            resb    1
+
+    alignb 8
+    .fXStateGuestMask           resq    1
+    .fXStateHostMask            resq    1
+
+    alignb 64
+    .HostFeatures               resb    48
+    .GuestFeatures              resb    48
+    .GuestInfo                  resb    RTHCPTR_CB*4 + 4*12
+
+    ; Patch manager saved state compatability CPUID leaf arrays
+    .aGuestCpuIdPatmStd         resb    16*6
+    .aGuestCpuIdPatmExt         resb    16*10
+    .aGuestCpuIdPatmCentaur     resb    16*4
+
+    alignb 8
+    .cMsrWrites                 resq    1
+    .cMsrWritesToIgnoredBits    resq    1
+    .cMsrWritesRaiseGp          resq    1
+    .cMsrWritesUnknown          resq    1
+    .cMsrReads                  resq    1
+    .cMsrReadsRaiseGp           resq    1
+    .cMsrReadsUnknown           resq    1
+endstruc
+
+struc CPUMCPU
+    ;
+    ; Guest context state
+    ; (Identical to the .Hyper chunk below and to CPUMCTX in cpum.mac.)
+    ;
+    .Guest                    resq    0
+    .Guest.eax                resq    1
+    .Guest.ecx                resq    1
+    .Guest.edx                resq    1
+    .Guest.ebx                resq    1
+    .Guest.esp                resq    1
+    .Guest.ebp                resq    1
+    .Guest.esi                resq    1
+    .Guest.edi                resq    1
+    .Guest.r8                 resq    1
+    .Guest.r9                 resq    1
+    .Guest.r10                resq    1
+    .Guest.r11                resq    1
+    .Guest.r12                resq    1
+    .Guest.r13                resq    1
+    .Guest.r14                resq    1
+    .Guest.r15                resq    1
+    .Guest.es.Sel             resw    1
+    .Guest.es.PaddingSel      resw    1
+    .Guest.es.ValidSel        resw    1
+    .Guest.es.fFlags          resw    1
+    .Guest.es.u64Base         resq    1
+    .Guest.es.u32Limit        resd    1
+    .Guest.es.Attr            resd    1
+    .Guest.cs.Sel             resw    1
+    .Guest.cs.PaddingSel      resw    1
+    .Guest.cs.ValidSel        resw    1
+    .Guest.cs.fFlags          resw    1
+    .Guest.cs.u64Base         resq    1
+    .Guest.cs.u32Limit        resd    1
+    .Guest.cs.Attr            resd    1
+    .Guest.ss.Sel             resw    1
+    .Guest.ss.PaddingSel      resw    1
+    .Guest.ss.ValidSel        resw    1
+    .Guest.ss.fFlags          resw    1
+    .Guest.ss.u64Base         resq    1
+    .Guest.ss.u32Limit        resd    1
+    .Guest.ss.Attr            resd    1
+    .Guest.ds.Sel             resw    1
+    .Guest.ds.PaddingSel      resw    1
+    .Guest.ds.ValidSel        resw    1
+    .Guest.ds.fFlags          resw    1
+    .Guest.ds.u64Base         resq    1
+    .Guest.ds.u32Limit        resd    1
+    .Guest.ds.Attr            resd    1
+    .Guest.fs.Sel             resw    1
+    .Guest.fs.PaddingSel      resw    1
+    .Guest.fs.ValidSel        resw    1
+    .Guest.fs.fFlags          resw    1
+    .Guest.fs.u64Base         resq    1
+    .Guest.fs.u32Limit        resd    1
+    .Guest.fs.Attr            resd    1
+    .Guest.gs.Sel             resw    1
+    .Guest.gs.PaddingSel      resw    1
+    .Guest.gs.ValidSel        resw    1
+    .Guest.gs.fFlags          resw    1
+    .Guest.gs.u64Base         resq    1
+    .Guest.gs.u32Limit        resd    1
+    .Guest.gs.Attr            resd    1
+    .Guest.eip                resq    1
+    .Guest.eflags             resq    1
+    .Guest.cr0                resq    1
+    .Guest.cr2                resq    1
+    .Guest.cr3                resq    1
+    .Guest.cr4                resq    1
+    .Guest.dr                 resq    8
+    .Guest.gdtrPadding        resw    3
+    .Guest.gdtr               resw    0
+    .Guest.gdtr.cbGdt         resw    1
+    .Guest.gdtr.pGdt          resq    1
+    .Guest.idtrPadding        resw    3
+    .Guest.idtr               resw    0
+    .Guest.idtr.cbIdt         resw    1
+    .Guest.idtr.pIdt          resq    1
+    .Guest.ldtr.Sel           resw    1
+    .Guest.ldtr.PaddingSel    resw    1
+    .Guest.ldtr.ValidSel      resw    1
+    .Guest.ldtr.fFlags        resw    1
+    .Guest.ldtr.u64Base       resq    1
+    .Guest.ldtr.u32Limit      resd    1
+    .Guest.ldtr.Attr          resd    1
+    .Guest.tr.Sel             resw    1
+    .Guest.tr.PaddingSel      resw    1
+    .Guest.tr.ValidSel        resw    1
+    .Guest.tr.fFlags          resw    1
+    .Guest.tr.u64Base         resq    1
+    .Guest.tr.u32Limit        resd    1
+    .Guest.tr.Attr            resd    1
+    .Guest.SysEnter.cs        resb    8
+    .Guest.SysEnter.eip       resb    8
+    .Guest.SysEnter.esp       resb    8
+    .Guest.msrEFER            resb    8
+    .Guest.msrSTAR            resb    8
+    .Guest.msrPAT             resb    8
+    .Guest.msrLSTAR           resb    8
+    .Guest.msrCSTAR           resb    8
+    .Guest.msrSFMASK          resb    8
+    .Guest.msrKERNELGSBASE    resb    8
+    .Guest.uMsrPadding0       resb    8
+    alignb 8
+    .Guest.aXcr               resq    2
+    .Guest.fXStateMask        resq    1
+    .Guest.pXStateR0      RTR0PTR_RES 1
+    alignb 8
+    .Guest.pXStateR3      RTR3PTR_RES 1
+    alignb 8
+    .Guest.aoffXState         resw    64
+    .Guest.fWorldSwitcher     resd    1
+    alignb 8
+    .Guest.fExtrn             resq    1
+    alignb 8
+    .Guest.hwvirt.svm.uMsrHSavePa            resq         1
+    .Guest.hwvirt.svm.GCPhysVmcb             resq         1
+    .Guest.hwvirt.svm.pVmcbR0                RTR0PTR_RES  1
+    alignb 8
+    .Guest.hwvirt.svm.pVmcbR3                RTR3PTR_RES  1
+    alignb 8
+    .Guest.hwvirt.svm.HostState              resb         184
+    .Guest.hwvirt.svm.uPrevPauseTick         resq         1
+    .Guest.hwvirt.svm.cPauseFilter           resw         1
+    .Guest.hwvirt.svm.cPauseFilterThreshold  resw         1
+    .Guest.hwvirt.svm.fInterceptEvents       resb         1
+    alignb 8
+    .Guest.hwvirt.svm.pvMsrBitmapR0          RTR0PTR_RES  1
+    alignb 8
+    .Guest.hwvirt.svm.pvMsrBitmapR3          RTR3PTR_RES  1
+    alignb 8
+    .Guest.hwvirt.svm.pvIoBitmapR0           RTR0PTR_RES  1
+    alignb 8
+    .Guest.hwvirt.svm.pvIoBitmapR3           RTR3PTR_RES  1
+    alignb 8
+    .Guest.hwvirt.svm.HCPhysVmcb             RTHCPHYS_RES 1
+    .Guest.hwvirt.svm.abPadding0             resb       272
+    .Guest.hwvirt.enmHwvirt                  resd         1
+    .Guest.hwvirt.fGif                       resb         1
+    alignb 8
+    .Guest.hwvirt.fLocalForcedActions        resd         1
+    alignb 64
+
+    .GuestMsrs                resq    0
+    .GuestMsrs.au64           resq    64
+
+    ;
+    ; Other stuff.
+    ;
+    .pNestedVmxPreemptTimerR0   RTR0PTR_RES    1
+    .pNestedVmxPreemptTimerR3   RTR3PTR_RES    1
+
+    .fUseFlags            resd    1
+    .fChanged             resd    1
+    .u32RetCode           resd    1
+
+%ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+    .fApicDisVectors      resd    1
+    .pvApicBase           RTR0PTR_RES 1
+    .fX2Apic              resb    1
+%else
+    .abPadding3           resb    (4 + RTR0PTR_CB + 1)
+%endif
+
+    .fRemEntered                resb    1
+    .fCpuIdApicFeatureVisible   resb    1
+
+    .abPadding2           resb    (64 - (RTR0PTR_CB + RTR3PTR_CB + 12 + 4 + RTR0PTR_CB + 1 + 2))
+
+    ;
+    ; Host context state
+    ;
+    alignb 64
+    .Host                resb    0
+    ;.Host.rax            resq    1 - scratch
+    .Host.rbx            resq    1
+    ;.Host.rcx            resq    1 - scratch
+    ;.Host.rdx            resq    1 - scratch
+    .Host.rdi            resq    1
+    .Host.rsi            resq    1
+    .Host.rbp            resq    1
+    .Host.rsp            resq    1
+    ;.Host.r8             resq    1 - scratch
+    ;.Host.r9             resq    1 - scratch
+    .Host.r10            resq    1
+    .Host.r11            resq    1
+    .Host.r12            resq    1
+    .Host.r13            resq    1
+    .Host.r14            resq    1
+    .Host.r15            resq    1
+    ;.Host.rip            resd    1 - scratch
+    .Host.rflags         resq    1
+    .Host.ss             resw    1
+    .Host.ssPadding      resw    1
+    .Host.gs             resw    1
+    .Host.gsPadding      resw    1
+    .Host.fs             resw    1
+    .Host.fsPadding      resw    1
+    .Host.es             resw    1
+    .Host.esPadding      resw    1
+    .Host.ds             resw    1
+    .Host.dsPadding      resw    1
+    .Host.cs             resw    1
+    .Host.csPadding      resw    1
+
+    .Host.cr0Fpu:
+    .Host.cr0            resq    1
+    ;.Host.cr2            resq    1 - scratch
+    .Host.cr3            resq    1
+    .Host.cr4            resq    1
+    .Host.cr8            resq    1
+
+    .Host.dr0            resq    1
+    .Host.dr1            resq    1
+    .Host.dr2            resq    1
+    .Host.dr3            resq    1
+    .Host.dr6            resq    1
+    .Host.dr7            resq    1
+
+    .Host.gdtr           resb    10         ; GDT limit + linear address
+    .Host.gdtrPadding    resw    1
+    .Host.idtr           resb    10         ; IDT limit + linear address
+    .Host.idtrPadding    resw    1
+    .Host.ldtr           resw    1
+    .Host.ldtrPadding    resw    1
+    .Host.tr             resw    1
+    .Host.trPadding      resw    1
+
+    .Host.SysEnter.cs    resq    1
+    .Host.SysEnter.eip   resq    1
+    .Host.SysEnter.esp   resq    1
+    .Host.FSbase         resq    1
+    .Host.GSbase         resq    1
+    .Host.efer           resq    1
+    .Host.auPadding      resb    4
+    alignb RTR0PTR_CB
+    .Host.pXStateR0 RTR0PTR_RES  1
+    .Host.pXStateR3 RTR3PTR_RES  1
+    alignb 8
+    .Host.xcr0           resq    1
+    .Host.fXStateMask    resq    1
+
+    ;
+    ; Hypervisor Context.
+    ;
+    alignb 64
+    .Hyper                    resq    0
+    .Hyper.dr                 resq    8
+    .Hyper.cr3                resq    1
+    alignb 64
+
+%ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    .aMagic              resb    56
+    .uMagic              resq    1
+%endif
+endstruc
+
+
+
+%if 0 ; Currently not used anywhere.
+;;
+; Macro for FXSAVE/FXRSTOR leaky behaviour on AMD CPUs, see cpumR3CheckLeakyFpu().
+;
+; Cleans the FPU state, if necessary, before restoring the FPU.
+;
+; This macro ASSUMES CR0.TS is not set!
+;
+; @param    xDX     Pointer to CPUMCPU.
+; @uses     xAX, EFLAGS
+;
+; Changes here should also be reflected in CPUMRCA.asm's copy!
+;
+%macro CLEANFPU 0
+        test    dword [xDX + CPUMCPU.fUseFlags], CPUM_USE_FFXSR_LEAKY
+        jz      .nothing_to_clean
+
+        xor     eax, eax
+        fnstsw  ax                      ; FSW -> AX.
+        test    eax, RT_BIT(7)          ; If FSW.ES (bit 7) is set, clear it to not cause FPU exceptions
+                                        ; while clearing & loading the FPU bits in 'clean_fpu' below.
+        jz      .clean_fpu
+        fnclex
+
+.clean_fpu:
+        ffree   st7                     ; Clear FPU stack register(7)'s tag entry to prevent overflow if a wraparound occurs.
+                                        ; for the upcoming push (load)
+        fild    dword [g_r32_Zero xWrtRIP] ; Explicit FPU load to overwrite FIP, FOP, FDP registers in the FPU.
+.nothing_to_clean:
+%endmacro
+%endif ; Unused.
+
+
+;;
+; Makes sure we don't trap (#NM) accessing the FPU.
+;
+; In ring-0 this is a bit of work since we may have try convince the host kernel
+; to do the work for us, also, we must report any CR0 changes back to HMR0VMX
+; via the VINF_CPUM_HOST_CR0_MODIFIED status code.
+;
+; If we end up clearing CR0.TS/EM ourselves in ring-0, we'll save the original
+; value in CPUMCPU.Host.cr0Fpu.  If we don't, we'll store zero there.  (See also
+; CPUMRZ_RESTORE_CR0_IF_TS_OR_EM_SET.)
+;
+; In raw-mode we will always have to clear TS and it will be recalculated
+; elsewhere and thus needs no saving.
+;
+; @param    %1          Register to return the return status code in.
+; @param    %2          Temporary scratch register.
+; @param    %3          Ring-0 only, register pointing to the CPUMCPU structure
+;                       of the EMT we're on.
+; @uses     EFLAGS, CR0, %1, %2
+;
+%macro CPUMRZ_TOUCH_FPU_CLEAR_CR0_FPU_TRAPS_SET_RC 3
+        ;
+        ; ring-0 - slightly complicated (than old raw-mode).
+        ;
+        xor     %1, %1                          ; 0 / VINF_SUCCESS. Wishing for no CR0 changes.
+        mov     [%3 + CPUMCPU.Host.cr0Fpu], %1
+
+        mov     %2, cr0
+        test    %2, X86_CR0_TS | X86_CR0_EM ; Make sure its safe to access the FPU state.
+        jz      %%no_cr0_change
+
+ %ifdef VMM_R0_TOUCH_FPU
+        ; Touch the state and check that the kernel updated CR0 for us.
+        movdqa  xmm0, xmm0
+        mov     %2, cr0
+        test    %2, X86_CR0_TS | X86_CR0_EM
+        jz      %%cr0_changed
+ %endif
+
+        ; Save CR0 and clear them flags ourselves.
+        mov     [%3 + CPUMCPU.Host.cr0Fpu], %2
+        and     %2, ~(X86_CR0_TS | X86_CR0_EM)
+        mov     cr0, %2
+
+%%cr0_changed:
+        mov     %1,  VINF_CPUM_HOST_CR0_MODIFIED
+%%no_cr0_change:
+%endmacro
+
+
+;;
+; Restore CR0 if CR0.TS or CR0.EM were non-zero in the original state.
+;
+; @param    %1      The original state to restore (or zero).
+;
+%macro CPUMRZ_RESTORE_CR0_IF_TS_OR_EM_SET 1
+        test    %1, X86_CR0_TS | X86_CR0_EM
+        jz      %%skip_cr0_restore
+        mov     cr0, %1
+%%skip_cr0_restore:
+%endmacro
+
+
+;;
+; Saves the host state.
+;
+; @uses     rax, rdx
+; @param    pCpumCpu    Define for the register containing the CPUMCPU pointer.
+; @param    pXState     Define for the register containing the extended state pointer.
+;
+%macro CPUMR0_SAVE_HOST 0
+        ;
+        ; Load a couple of registers we'll use later in all branches.
+        ;
+ %ifdef IN_RING0
+        mov     pXState, [pCpumCpu + CPUMCPU.Host.pXStateR0]
+ %else
+  %error "Unsupported context!"
+ %endif
+        mov     eax, [pCpumCpu + CPUMCPU.Host.fXStateMask]
+
+        ;
+        ; XSAVE or FXSAVE?
+        ;
+        or      eax, eax
+        jz      %%host_fxsave
+
+        ; XSAVE
+        mov     edx, [pCpumCpu + CPUMCPU.Host.fXStateMask + 4]
+ %ifdef RT_ARCH_AMD64
+        o64 xsave [pXState]
+ %else
+        xsave   [pXState]
+ %endif
+        jmp     %%host_done
+
+        ; FXSAVE
+%%host_fxsave:
+ %ifdef RT_ARCH_AMD64
+        o64 fxsave [pXState]            ; Use explicit REX prefix. See @bugref{6398}.
+ %else
+        fxsave  [pXState]
+ %endif
+
+%%host_done:
+%endmacro ; CPUMR0_SAVE_HOST
+
+
+;;
+; Loads the host state.
+;
+; @uses     rax, rdx
+; @param    pCpumCpu    Define for the register containing the CPUMCPU pointer.
+; @param    pXState     Define for the register containing the extended state pointer.
+;
+%macro CPUMR0_LOAD_HOST 0
+        ;
+        ; Load a couple of registers we'll use later in all branches.
+        ;
+ %ifdef IN_RING0
+        mov     pXState, [pCpumCpu + CPUMCPU.Host.pXStateR0]
+ %else
+  %error "Unsupported context!"
+ %endif
+        mov     eax, [pCpumCpu + CPUMCPU.Host.fXStateMask]
+
+        ;
+        ; XRSTOR or FXRSTOR?
+        ;
+        or      eax, eax
+        jz      %%host_fxrstor
+
+        ; XRSTOR
+        mov     edx, [pCpumCpu + CPUMCPU.Host.fXStateMask + 4]
+ %ifdef RT_ARCH_AMD64
+        o64 xrstor [pXState]
+ %else
+        xrstor  [pXState]
+ %endif
+        jmp     %%host_done
+
+        ; FXRSTOR
+%%host_fxrstor:
+ %ifdef RT_ARCH_AMD64
+        o64 fxrstor [pXState]           ; Use explicit REX prefix. See @bugref{6398}.
+ %else
+        fxrstor [pXState]
+ %endif
+
+%%host_done:
+%endmacro ; CPUMR0_LOAD_HOST
+
+
+
+;; Macro for XSAVE/FXSAVE for the guest FPU but tries to figure out whether to
+;  save the 32-bit FPU state or 64-bit FPU state.
+;
+; @param    %1      Pointer to CPUMCPU.
+; @param    %2      Pointer to XState.
+; @param    %3      Force AMD64
+; @param    %4      The instruction to use (xsave or fxsave)
+; @uses     xAX, xDX, EFLAGS, 20h of stack.
+;
+%macro SAVE_32_OR_64_FPU 4
+%if CPUM_IS_AMD64 || %3
+        ; Save the guest FPU (32-bit or 64-bit), preserves existing broken state. See @bugref{7138}.
+        test    dword [pCpumCpu + CPUMCPU.fUseFlags], CPUM_USE_SUPPORTS_LONGMODE
+        jnz     short %%save_long_mode_guest
+%endif
+        %4      [pXState]
+%if CPUM_IS_AMD64 || %3
+        jmp     %%save_done_32bit_cs_ds
+
+%%save_long_mode_guest:
+        o64 %4  [pXState]
+
+        xor     edx, edx
+        cmp     dword [pXState + X86FXSTATE.FPUCS], 0
+        jne     short %%save_done
+
+        sub     rsp, 20h                ; Only need 1ch bytes but keep stack aligned otherwise we #GP(0).
+        fnstenv [rsp]
+        movzx   eax, word [rsp + 10h]
+        mov     [pXState + X86FXSTATE.FPUCS], eax
+        movzx   eax, word [rsp + 18h]
+        add     rsp, 20h
+        mov     [pXState + X86FXSTATE.FPUDS], eax
+%endif
+%%save_done_32bit_cs_ds:
+        mov     edx, X86_FXSTATE_RSVD_32BIT_MAGIC
+%%save_done:
+        mov     dword [pXState + X86_OFF_FXSTATE_RSVD], edx
+%endmacro ; SAVE_32_OR_64_FPU
+
+
+;;
+; Save the guest state.
+;
+; @uses     rax, rdx
+; @param    pCpumCpu    Define for the register containing the CPUMCPU pointer.
+; @param    pXState     Define for the register containing the extended state pointer.
+;
+%macro CPUMR0_SAVE_GUEST 0
+        ;
+        ; Load a couple of registers we'll use later in all branches.
+        ;
+ %ifdef IN_RING0
+        mov     pXState, [pCpumCpu + CPUMCPU.Guest.pXStateR0]
+ %else
+  %error "Unsupported context!"
+ %endif
+        mov     eax, [pCpumCpu + CPUMCPU.Guest.fXStateMask]
+
+        ;
+        ; XSAVE or FXSAVE?
+        ;
+        or      eax, eax
+        jz      %%guest_fxsave
+
+        ; XSAVE
+        mov     edx, [pCpumCpu + CPUMCPU.Guest.fXStateMask + 4]
+ %ifdef VBOX_WITH_KERNEL_USING_XMM
+        and     eax, ~CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS ; Already saved in HMR0A.asm.
+ %endif
+        SAVE_32_OR_64_FPU pCpumCpu, pXState, 0, xsave
+        jmp     %%guest_done
+
+        ; FXSAVE
+%%guest_fxsave:
+        SAVE_32_OR_64_FPU pCpumCpu, pXState, 0, fxsave
+
+%%guest_done:
+%endmacro ; CPUMR0_SAVE_GUEST
+
+
+;;
+; Wrapper for selecting 32-bit or 64-bit XRSTOR/FXRSTOR according to what SAVE_32_OR_64_FPU did.
+;
+; @param    %1      Pointer to CPUMCPU.
+; @param    %2      Pointer to XState.
+; @param    %3      Force AMD64.
+; @param    %4      The instruction to use (xrstor or fxrstor).
+; @uses     xAX, xDX, EFLAGS
+;
+%macro RESTORE_32_OR_64_FPU 4
+%if CPUM_IS_AMD64 || %3
+        ; Restore the guest FPU (32-bit or 64-bit), preserves existing broken state. See @bugref{7138}.
+        test    dword [pCpumCpu + CPUMCPU.fUseFlags], CPUM_USE_SUPPORTS_LONGMODE
+        jz      %%restore_32bit_fpu
+        cmp     dword [pXState + X86_OFF_FXSTATE_RSVD], X86_FXSTATE_RSVD_32BIT_MAGIC
+        jne     short %%restore_64bit_fpu
+%%restore_32bit_fpu:
+%endif
+        %4      [pXState]
+%if CPUM_IS_AMD64 || %3
+        ; TODO: Restore XMM8-XMM15!
+        jmp     short %%restore_fpu_done
+%%restore_64bit_fpu:
+        o64 %4  [pXState]
+%%restore_fpu_done:
+%endif
+%endmacro ; RESTORE_32_OR_64_FPU
+
+
+;;
+; Loads the guest state.
+;
+; @uses     rax, rdx
+; @param    pCpumCpu    Define for the register containing the CPUMCPU pointer.
+; @param    pXState     Define for the register containing the extended state pointer.
+;
+%macro CPUMR0_LOAD_GUEST 0
+        ;
+        ; Load a couple of registers we'll use later in all branches.
+        ;
+ %ifdef IN_RING0
+        mov     pXState, [pCpumCpu + CPUMCPU.Guest.pXStateR0]
+ %else
+  %error "Unsupported context!"
+ %endif
+        mov     eax, [pCpumCpu + CPUMCPU.Guest.fXStateMask]
+
+        ;
+        ; XRSTOR or FXRSTOR?
+        ;
+        or      eax, eax
+        jz      %%guest_fxrstor
+
+        ; XRSTOR
+        mov     edx, [pCpumCpu + CPUMCPU.Guest.fXStateMask + 4]
+ %ifdef VBOX_WITH_KERNEL_USING_XMM
+        and     eax, ~CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS ; Will be loaded by HMR0A.asm.
+ %endif
+        RESTORE_32_OR_64_FPU pCpumCpu, pXState, 0, xrstor
+        jmp     %%guest_done
+
+        ; FXRSTOR
+%%guest_fxrstor:
+        RESTORE_32_OR_64_FPU pCpumCpu, pXState, 0, fxrstor
+
+%%guest_done:
+%endmacro ; CPUMR0_LOAD_GUEST
+
diff --git a/src/VBox/VMM/include/DBGFInternal.h b/src/VBox/VMM/include/DBGFInternal.h
new file mode 100644
index 00000000..1cf2dc4d
--- /dev/null
+++ b/src/VBox/VMM/include/DBGFInternal.h
@@ -0,0 +1,605 @@
+/* $Id: DBGFInternal.h $ */
+/** @file
+ * DBGF - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_DBGFInternal_h
+#define VMM_INCLUDED_SRC_include_DBGFInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#ifdef IN_RING3
+# include <VBox/dis.h>
+#endif
+#include <VBox/types.h>
+#include <iprt/semaphore.h>
+#include <iprt/critsect.h>
+#include <iprt/string.h>
+#include <iprt/avl.h>
+#include <iprt/dbg.h>
+#include <VBox/vmm/dbgf.h>
+
+
+
+/** @defgroup grp_dbgf_int   Internals
+ * @ingroup grp_dbgf
+ * @internal
+ * @{
+ */
+
+
+/** VMM Debugger Command. */
+typedef enum DBGFCMD
+{
+    /** No command.
+     * This is assigned to the field by the emulation thread after
+     * a command has been completed. */
+    DBGFCMD_NO_COMMAND = 0,
+    /** Halt the VM. */
+    DBGFCMD_HALT,
+    /** Resume execution. */
+    DBGFCMD_GO,
+    /** Single step execution - stepping into calls. */
+    DBGFCMD_SINGLE_STEP,
+    /** Detaches the debugger.
+     * Disabling all breakpoints, watch points and the like. */
+    DBGFCMD_DETACH_DEBUGGER,
+    /** Detached the debugger.
+     * The isn't a command as such, it's just that it's necessary for the
+     * detaching protocol to be racefree. */
+    DBGFCMD_DETACHED_DEBUGGER
+} DBGFCMD;
+
+/**
+ * VMM Debugger Command.
+ */
+typedef union DBGFCMDDATA
+{
+    uint32_t    uDummy;
+} DBGFCMDDATA;
+/** Pointer to DBGF Command Data. */
+typedef DBGFCMDDATA *PDBGFCMDDATA;
+
+/**
+ * Info type.
+ */
+typedef enum DBGFINFOTYPE
+{
+    /** Invalid. */
+    DBGFINFOTYPE_INVALID = 0,
+    /** Device owner. */
+    DBGFINFOTYPE_DEV,
+    /** Driver owner. */
+    DBGFINFOTYPE_DRV,
+    /** Internal owner. */
+    DBGFINFOTYPE_INT,
+    /** External owner. */
+    DBGFINFOTYPE_EXT,
+    /** Device owner. */
+    DBGFINFOTYPE_DEV_ARGV,
+    /** Driver owner. */
+    DBGFINFOTYPE_DRV_ARGV,
+    /** USB device  owner. */
+    DBGFINFOTYPE_USB_ARGV,
+    /** Internal owner, argv. */
+    DBGFINFOTYPE_INT_ARGV,
+    /** External owner. */
+    DBGFINFOTYPE_EXT_ARGV
+} DBGFINFOTYPE;
+
+
+/** Pointer to info structure. */
+typedef struct DBGFINFO *PDBGFINFO;
+
+#ifdef IN_RING3
+/**
+ * Info structure.
+ */
+typedef struct DBGFINFO
+{
+    /** The flags. */
+    uint32_t        fFlags;
+    /** Owner type. */
+    DBGFINFOTYPE    enmType;
+    /** Per type data. */
+    union
+    {
+        /** DBGFINFOTYPE_DEV */
+        struct
+        {
+            /** Device info handler function. */
+            PFNDBGFHANDLERDEV   pfnHandler;
+            /** The device instance. */
+            PPDMDEVINS          pDevIns;
+        } Dev;
+
+        /** DBGFINFOTYPE_DRV */
+        struct
+        {
+            /** Driver info handler function. */
+            PFNDBGFHANDLERDRV   pfnHandler;
+            /** The driver instance. */
+            PPDMDRVINS          pDrvIns;
+        } Drv;
+
+        /** DBGFINFOTYPE_INT */
+        struct
+        {
+            /** Internal info handler function. */
+            PFNDBGFHANDLERINT   pfnHandler;
+        } Int;
+
+        /** DBGFINFOTYPE_EXT */
+        struct
+        {
+            /** External info handler function. */
+            PFNDBGFHANDLEREXT   pfnHandler;
+            /** The user argument. */
+            void               *pvUser;
+        } Ext;
+
+        /** DBGFINFOTYPE_DEV_ARGV */
+        struct
+        {
+            /** Device info handler function. */
+            PFNDBGFINFOARGVDEV  pfnHandler;
+            /** The device instance. */
+            PPDMDEVINS          pDevIns;
+        } DevArgv;
+
+        /** DBGFINFOTYPE_DRV_ARGV */
+        struct
+        {
+            /** Driver info handler function. */
+            PFNDBGFINFOARGVDRV  pfnHandler;
+            /** The driver instance. */
+            PPDMDRVINS          pDrvIns;
+        } DrvArgv;
+
+        /** DBGFINFOTYPE_USB_ARGV */
+        struct
+        {
+            /** Driver info handler function. */
+            PFNDBGFINFOARGVUSB  pfnHandler;
+            /** The driver instance. */
+            PPDMUSBINS          pUsbIns;
+        } UsbArgv;
+
+        /** DBGFINFOTYPE_INT_ARGV */
+        struct
+        {
+            /** Internal info handler function. */
+            PFNDBGFINFOARGVINT  pfnHandler;
+        } IntArgv;
+
+        /** DBGFINFOTYPE_EXT_ARGV */
+        struct
+        {
+            /** External info handler function. */
+            PFNDBGFINFOARGVEXT  pfnHandler;
+            /** The user argument. */
+            void               *pvUser;
+        } ExtArgv;
+    } u;
+
+    /** Pointer to the description. */
+    const char     *pszDesc;
+    /** Pointer to the next info structure. */
+    PDBGFINFO       pNext;
+    /** The identifier name length. */
+    size_t          cchName;
+    /** The identifier name. (Extends 'beyond' the struct as usual.) */
+    char            szName[1];
+} DBGFINFO;
+#endif /* IN_RING3 */
+
+
+#ifdef IN_RING3
+/**
+ * Guest OS digger instance.
+ */
+typedef struct DBGFOS
+{
+    /** Pointer to the registration record. */
+    PCDBGFOSREG                 pReg;
+    /** Pointer to the next OS we've registered. */
+    struct DBGFOS              *pNext;
+    /** List of EMT interface wrappers. */
+    struct DBGFOSEMTWRAPPER    *pWrapperHead;
+    /** The instance data (variable size). */
+    uint8_t                     abData[16];
+} DBGFOS;
+#endif
+/** Pointer to guest OS digger instance. */
+typedef struct DBGFOS *PDBGFOS;
+/** Pointer to const guest OS digger instance. */
+typedef struct DBGFOS const *PCDBGFOS;
+
+
+/**
+ * Breakpoint search optimization.
+ */
+typedef struct DBGFBPSEARCHOPT
+{
+    /** Where to start searching for hits.
+     * (First enabled is #DBGF::aBreakpoints[iStartSearch]). */
+    uint32_t volatile       iStartSearch;
+    /** The number of aBreakpoints entries to search.
+     * (Last enabled is #DBGF::aBreakpoints[iStartSearch + cToSearch - 1])  */
+    uint32_t volatile       cToSearch;
+} DBGFBPSEARCHOPT;
+/** Pointer to a breakpoint search optimziation structure. */
+typedef DBGFBPSEARCHOPT *PDBGFBPSEARCHOPT;
+
+
+
+/**
+ * DBGF Data (part of VM)
+ */
+typedef struct DBGF
+{
+    /** Bitmap of enabled hardware interrupt breakpoints. */
+    uint32_t                    bmHardIntBreakpoints[256 / 32];
+    /** Bitmap of enabled software interrupt breakpoints. */
+    uint32_t                    bmSoftIntBreakpoints[256 / 32];
+    /** Bitmap of selected events.
+     * This includes non-selectable events too for simplicity, we maintain the
+     * state for some of these, as it may come in handy. */
+    uint64_t                    bmSelectedEvents[(DBGFEVENT_END + 63) / 64];
+
+    /** Enabled hardware interrupt breakpoints. */
+    uint32_t                    cHardIntBreakpoints;
+    /** Enabled software interrupt breakpoints. */
+    uint32_t                    cSoftIntBreakpoints;
+
+    /** The number of selected events. */
+    uint32_t                    cSelectedEvents;
+
+    /** The number of enabled hardware breakpoints. */
+    uint8_t                     cEnabledHwBreakpoints;
+    /** The number of enabled hardware I/O breakpoints. */
+    uint8_t                     cEnabledHwIoBreakpoints;
+    /** The number of enabled INT3 breakpoints. */
+    uint8_t                     cEnabledInt3Breakpoints;
+    uint8_t                     abPadding; /**< Unused padding space up for grabs. */
+    uint32_t                    uPadding;
+
+    /** Debugger Attached flag.
+     * Set if a debugger is attached, elsewise it's clear.
+     */
+    bool volatile               fAttached;
+
+    /** Stopped in the Hypervisor.
+     * Set if we're stopped on a trace, breakpoint or assertion inside
+     * the hypervisor and have to restrict the available operations.
+     */
+    bool volatile               fStoppedInHyper;
+
+    /**
+     * Ping-Pong construct where the Ping side is the VMM and the Pong side
+     * the Debugger.
+     */
+    RTPINGPONG                  PingPong;
+    RTHCUINTPTR                 uPtrPadding; /**< Alignment padding. */
+
+    /** The Event to the debugger.
+     * The VMM will ping the debugger when the event is ready. The event is
+     * either a response to a command or to a break/watch point issued
+     * previously.
+     */
+    DBGFEVENT                   DbgEvent;
+
+    /** The Command to the VMM.
+     * Operated in an atomic fashion since the VMM will poll on this.
+     * This means that a the command data must be written before this member
+     * is set. The VMM will reset this member to the no-command state
+     * when it have processed it.
+     */
+    DBGFCMD volatile            enmVMMCmd;
+    /** The Command data.
+     * Not all commands take data. */
+    DBGFCMDDATA                 VMMCmdData;
+
+    /** Stepping filtering. */
+    struct
+    {
+        /** The CPU doing the stepping.
+         * Set to NIL_VMCPUID when filtering is inactive */
+        VMCPUID                 idCpu;
+        /** The specified flags. */
+        uint32_t                fFlags;
+        /** The effective PC address to stop at, if given. */
+        RTGCPTR                 AddrPc;
+        /** The lowest effective stack address to stop at.
+         * Together with cbStackPop, this forms a range of effective stack pointer
+         * addresses that we stop for.   */
+        RTGCPTR                 AddrStackPop;
+        /** The size of the stack stop area starting at AddrStackPop. */
+        RTGCPTR                 cbStackPop;
+        /** Maximum number of steps. */
+        uint32_t                cMaxSteps;
+
+        /** Number of steps made thus far. */
+        uint32_t                cSteps;
+        /** Current call counting balance for step-over handling. */
+        uint32_t                uCallDepth;
+
+        uint32_t                u32Padding; /**< Alignment padding. */
+
+    } SteppingFilter;
+
+    uint32_t                    u32Padding[2]; /**< Alignment padding. */
+
+    /** Array of hardware breakpoints. (0..3)
+     * This is shared among all the CPUs because life is much simpler that way. */
+    DBGFBP                      aHwBreakpoints[4];
+    /** Array of int 3 and REM breakpoints. (4..)
+     * @remark This is currently a fixed size array for reasons of simplicity. */
+    DBGFBP                      aBreakpoints[32];
+
+    /** MMIO breakpoint search optimizations. */
+    DBGFBPSEARCHOPT             Mmio;
+    /** I/O port breakpoint search optimizations. */
+    DBGFBPSEARCHOPT             PortIo;
+    /** INT3 breakpoint search optimizations. */
+    DBGFBPSEARCHOPT             Int3;
+
+    /**
+     * Bug check data.
+     * @note This will not be reset on reset.
+     */
+    struct
+    {
+        /** The ID of the CPU reporting it. */
+        VMCPUID                 idCpu;
+        /** The event associated with the bug check (gives source).
+         * This is set to DBGFEVENT_END if no BSOD data here. */
+        DBGFEVENTTYPE           enmEvent;
+        /** The total reset count at the time (VMGetResetCount). */
+        uint32_t                uResetNo;
+        /** Explicit padding. */
+        uint32_t                uPadding;
+        /** When it was reported (TMVirtualGet). */
+        uint64_t                uTimestamp;
+        /** The bug check number.
+         * @note This is really just 32-bit wide, see KeBugCheckEx.  */
+        uint64_t                uBugCheck;
+        /** The bug check parameters. */
+        uint64_t                auParameters[4];
+    } BugCheck;
+} DBGF;
+AssertCompileMemberAlignment(DBGF, DbgEvent, 8);
+AssertCompileMemberAlignment(DBGF, aHwBreakpoints, 8);
+AssertCompileMemberAlignment(DBGF, bmHardIntBreakpoints, 8);
+/** Pointer to DBGF Data. */
+typedef DBGF *PDBGF;
+
+
+/**
+ * Event state (for DBGFCPU::aEvents).
+ */
+typedef enum DBGFEVENTSTATE
+{
+    /** Invalid event stack entry. */
+    DBGFEVENTSTATE_INVALID = 0,
+    /** The current event stack entry. */
+    DBGFEVENTSTATE_CURRENT,
+    /** Event that should be ignored but hasn't yet actually been ignored. */
+    DBGFEVENTSTATE_IGNORE,
+    /** Event that has been ignored but may be restored to IGNORE should another
+     * debug event fire before the instruction is completed. */
+    DBGFEVENTSTATE_RESTORABLE,
+    /** End of valid events.   */
+    DBGFEVENTSTATE_END,
+    /** Make sure we've got a 32-bit type. */
+    DBGFEVENTSTATE_32BIT_HACK = 0x7fffffff
+} DBGFEVENTSTATE;
+
+
+/** Converts a DBGFCPU pointer into a VM pointer. */
+#define DBGFCPU_2_VM(pDbgfCpu) ((PVM)((uint8_t *)(pDbgfCpu) + (pDbgfCpu)->offVM))
+
+/**
+ * The per CPU data for DBGF.
+ */
+typedef struct DBGFCPU
+{
+    /** The offset into the VM structure.
+     * @see DBGFCPU_2_VM(). */
+    uint32_t                offVM;
+
+    /** Current active breakpoint (id).
+     * This is ~0U if not active. It is set when a execution engine
+     * encounters a breakpoint and returns VINF_EM_DBG_BREAKPOINT. This is
+     * currently not used for REM breakpoints because of the lazy coupling
+     * between VBox and REM.
+     *
+     * @todo drop this in favor of aEvents!  */
+    uint32_t                iActiveBp;
+    /** Set if we're singlestepping in raw mode.
+     * This is checked and cleared in the \#DB handler. */
+    bool                    fSingleSteppingRaw;
+
+    /** Alignment padding. */
+    bool                    afPadding[3];
+
+    /** The number of events on the stack (aEvents).
+     * The pending event is the last one (aEvents[cEvents - 1]), but only when
+     * enmState is DBGFEVENTSTATE_CURRENT. */
+    uint32_t                cEvents;
+    /** Events - current, ignoring and ignored.
+     *
+     * We maintain a stack of events in order to try avoid ending up in an infinit
+     * loop when resuming after an event fired.  There are cases where we may end
+     * generating additional events before the instruction can be executed
+     * successfully.  Like for instance an XCHG on MMIO with separate read and write
+     * breakpoints, or a MOVSB instruction working on breakpointed MMIO as both
+     * source and destination.
+     *
+     * So, when resuming after dropping into the debugger for an event, we convert
+     * the DBGFEVENTSTATE_CURRENT event into a DBGFEVENTSTATE_IGNORE event, leaving
+     * cEvents unchanged.  If the event is reported again, we will ignore it and
+     * tell the reporter to continue executing.  The event change to the
+     * DBGFEVENTSTATE_RESTORABLE state.
+     *
+     * Currently, the event reporter has to figure out that it is a nested event and
+     * tell DBGF to restore DBGFEVENTSTATE_RESTORABLE events (and keep
+     * DBGFEVENTSTATE_IGNORE, should they happen out of order for some weird
+     * reason).
+     */
+    struct
+    {
+        /** The event details. */
+        DBGFEVENT           Event;
+        /** The RIP at which this happend (for validating ignoring). */
+        uint64_t            rip;
+        /** The event state. */
+        DBGFEVENTSTATE      enmState;
+        /** Alignment padding. */
+        uint32_t            u32Alignment;
+    } aEvents[3];
+} DBGFCPU;
+AssertCompileMemberAlignment(DBGFCPU, aEvents, 8);
+AssertCompileMemberSizeAlignment(DBGFCPU, aEvents[0], 8);
+/** Pointer to DBGFCPU data. */
+typedef DBGFCPU *PDBGFCPU;
+
+struct DBGFOSEMTWRAPPER;
+
+/**
+ * The DBGF data kept in the UVM.
+ */
+typedef struct DBGFUSERPERVM
+{
+    /** The address space database lock. */
+    RTSEMRW                     hAsDbLock;
+    /** The address space handle database.      (Protected by hAsDbLock.) */
+    R3PTRTYPE(AVLPVTREE)        AsHandleTree;
+    /** The address space process id database.  (Protected by hAsDbLock.) */
+    R3PTRTYPE(AVLU32TREE)       AsPidTree;
+    /** The address space name database.        (Protected by hAsDbLock.) */
+    R3PTRTYPE(RTSTRSPACE)       AsNameSpace;
+    /** Special address space aliases.          (Protected by hAsDbLock.) */
+    RTDBGAS volatile            ahAsAliases[DBGF_AS_COUNT];
+    /** For lazily populating the aliased address spaces. */
+    bool volatile               afAsAliasPopuplated[DBGF_AS_COUNT];
+    /** Alignment padding. */
+    bool                        afAlignment1[2];
+    /** Debug configuration. */
+    R3PTRTYPE(RTDBGCFG)         hDbgCfg;
+
+    /** The register database lock. */
+    RTSEMRW                     hRegDbLock;
+    /** String space for looking up registers.  (Protected by hRegDbLock.) */
+    R3PTRTYPE(RTSTRSPACE)       RegSpace;
+    /** String space holding the register sets. (Protected by hRegDbLock.)  */
+    R3PTRTYPE(RTSTRSPACE)       RegSetSpace;
+    /** The number of registers (aliases, sub-fields and the special CPU
+     * register aliases (eg AH) are not counted). */
+    uint32_t                    cRegs;
+    /** For early initialization by . */
+    bool volatile               fRegDbInitialized;
+    /** Alignment padding. */
+    bool                        afAlignment2[3];
+
+    /** Critical section protecting the Guest OS Digger data, the info handlers
+     * and the plugins.  These share to give the best possible plugin unload
+     * race protection. */
+    RTCRITSECTRW                CritSect;
+    /** Head of the LIFO of loaded DBGF plugins. */
+    R3PTRTYPE(struct DBGFPLUGIN *) pPlugInHead;
+    /** The current Guest OS digger. */
+    R3PTRTYPE(PDBGFOS)          pCurOS;
+    /** The head of the Guest OS digger instances. */
+    R3PTRTYPE(PDBGFOS)          pOSHead;
+    /** List of registered info handlers. */
+    R3PTRTYPE(PDBGFINFO)        pInfoFirst;
+
+    /** The type database lock. */
+    RTSEMRW                     hTypeDbLock;
+    /** String space for looking up types.  (Protected by hTypeDbLock.) */
+    R3PTRTYPE(RTSTRSPACE)       TypeSpace;
+    /** For early initialization by . */
+    bool volatile               fTypeDbInitialized;
+    /** Alignment padding. */
+    bool                        afAlignment3[3];
+
+} DBGFUSERPERVM;
+typedef DBGFUSERPERVM *PDBGFUSERPERVM;
+typedef DBGFUSERPERVM const *PCDBGFUSERPERVM;
+
+/**
+ * The per-CPU DBGF data kept in the UVM.
+ */
+typedef struct DBGFUSERPERVMCPU
+{
+    /** The guest register set for this CPU.  Can be NULL. */
+    R3PTRTYPE(struct DBGFREGSET *) pGuestRegSet;
+    /** The hypervisor register set for this CPU.  Can be NULL. */
+    R3PTRTYPE(struct DBGFREGSET *) pHyperRegSet;
+} DBGFUSERPERVMCPU;
+
+
+#ifdef IN_RING3
+int  dbgfR3AsInit(PUVM pUVM);
+void dbgfR3AsTerm(PUVM pUVM);
+void dbgfR3AsRelocate(PUVM pUVM, RTGCUINTPTR offDelta);
+int  dbgfR3BpInit(PVM pVM);
+int  dbgfR3InfoInit(PUVM pUVM);
+int  dbgfR3InfoTerm(PUVM pUVM);
+int  dbgfR3OSInit(PUVM pUVM);
+void dbgfR3OSTermPart1(PUVM pUVM);
+void dbgfR3OSTermPart2(PUVM pUVM);
+int  dbgfR3OSStackUnwindAssist(PUVM pUVM, VMCPUID idCpu, PDBGFSTACKFRAME pFrame, PRTDBGUNWINDSTATE pState,
+                               PCCPUMCTX pInitialCtx, RTDBGAS hAs, uint64_t *puScratch);
+int  dbgfR3RegInit(PUVM pUVM);
+void dbgfR3RegTerm(PUVM pUVM);
+int  dbgfR3TraceInit(PVM pVM);
+void dbgfR3TraceRelocate(PVM pVM);
+void dbgfR3TraceTerm(PVM pVM);
+DECLHIDDEN(int)  dbgfR3TypeInit(PUVM pUVM);
+DECLHIDDEN(void) dbgfR3TypeTerm(PUVM pUVM);
+int  dbgfR3PlugInInit(PUVM pUVM);
+void dbgfR3PlugInTerm(PUVM pUVM);
+int  dbgfR3BugCheckInit(PVM pVM);
+
+/**
+ * DBGF disassembler state (substate of DISSTATE).
+ */
+typedef struct DBGFDISSTATE
+{
+    /** Pointer to the current instruction. */
+    PCDISOPCODE     pCurInstr;
+    /** Size of the instruction in bytes. */
+    uint32_t        cbInstr;
+    /** Parameters.  */
+    DISOPPARAM      Param1;
+    DISOPPARAM      Param2;
+    DISOPPARAM      Param3;
+    DISOPPARAM      Param4;
+} DBGFDISSTATE;
+/** Pointer to a DBGF disassembler state. */
+typedef DBGFDISSTATE *PDBGFDISSTATE;
+
+DECLHIDDEN(int) dbgfR3DisasInstrStateEx(PUVM pUVM, VMCPUID idCpu, PDBGFADDRESS pAddr, uint32_t fFlags,
+                                        char *pszOutput, uint32_t cbOutput, PDBGFDISSTATE pDisState);
+
+#endif /* IN_RING3 */
+
+/** @} */
+
+#endif /* !VMM_INCLUDED_SRC_include_DBGFInternal_h */
diff --git a/src/VBox/VMM/include/EMHandleRCTmpl.h b/src/VBox/VMM/include/EMHandleRCTmpl.h
new file mode 100644
index 00000000..758dfad4
--- /dev/null
+++ b/src/VBox/VMM/include/EMHandleRCTmpl.h
@@ -0,0 +1,261 @@
+/* $Id: EMHandleRCTmpl.h $ */
+/** @file
+ * EM - emR3[Raw|Hm|Nem]HandleRC template.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_EMHandleRCTmpl_h
+#define VMM_INCLUDED_SRC_include_EMHandleRCTmpl_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#if defined(EMHANDLERC_WITH_PATM) + defined(EMHANDLERC_WITH_HM) + defined(EMHANDLERC_WITH_NEM) != 1
+# error "Exactly one of these must be defined: EMHANDLERC_WITH_PATM, EMHANDLERC_WITH_HM, EMHANDLERC_WITH_NEM"
+#endif
+
+
+/**
+ * Process a subset of the raw-mode, HM and NEM return codes.
+ *
+ * Since we have to share this with raw-mode single stepping, this inline
+ * function has been created to avoid code duplication.
+ *
+ * @returns VINF_SUCCESS if it's ok to continue raw mode.
+ * @returns VBox status code to return to the EM main loop.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   rc      The return code.
+ */
+#if defined(EMHANDLERC_WITH_HM) || defined(DOXYGEN_RUNNING)
+int emR3HmHandleRC(PVM pVM, PVMCPU pVCpu, int rc)
+#elif defined(EMHANDLERC_WITH_NEM)
+int emR3NemHandleRC(PVM pVM, PVMCPU pVCpu, int rc)
+#endif
+{
+    switch (rc)
+    {
+        /*
+         * Common & simple ones.
+         */
+        case VINF_SUCCESS:
+            break;
+        case VINF_EM_RESCHEDULE_RAW:
+        case VINF_EM_RESCHEDULE_HM:
+        case VINF_EM_RAW_INTERRUPT:
+        case VINF_EM_RAW_TO_R3:
+        case VINF_EM_RAW_TIMER_PENDING:
+        case VINF_EM_PENDING_REQUEST:
+            rc = VINF_SUCCESS;
+            break;
+
+#ifndef EMHANDLERC_WITH_NEM
+        /*
+         * Conflict or out of page tables.
+         *
+         * VM_FF_PGM_SYNC_CR3 is set by the hypervisor and all we need to
+         * do here is to execute the pending forced actions.
+         */
+        case VINF_PGM_SYNC_CR3:
+            AssertMsg(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL),
+                      ("VINF_PGM_SYNC_CR3 and no VMCPU_FF_PGM_SYNC_CR3*!\n"));
+            rc = VINF_SUCCESS;
+            break;
+
+        /*
+         * PGM pool flush pending (guest SMP only).
+         */
+        /** @todo jumping back and forth between ring 0 and 3 can burn a lot of cycles
+         * if the EMT thread that's supposed to handle the flush is currently not active
+         * (e.g. waiting to be scheduled) -> fix this properly!
+         *
+         * bird: Since the clearing is global and done via a rendezvous any CPU can do
+         *       it. They would have to choose who to call VMMR3EmtRendezvous and send
+         *       the rest to VMMR3EmtRendezvousFF ... Hmm ... that's not going to work
+         *       all that well since the latter will race the setup done by the
+         *       first.  Guess that means we need some new magic in that area for
+         *       handling this case. :/
+         */
+        case VINF_PGM_POOL_FLUSH_PENDING:
+            rc = VINF_SUCCESS;
+            break;
+
+        /*
+         * Paging mode change.
+         */
+        case VINF_PGM_CHANGE_MODE:
+            CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);
+            rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER);
+            if (rc == VINF_SUCCESS)
+                rc = VINF_EM_RESCHEDULE;
+            AssertMsg(RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST), ("%Rrc\n", rc));
+            break;
+#endif /* !EMHANDLERC_WITH_NEM */
+
+        /*
+         * I/O Port access - emulate the instruction.
+         */
+        case VINF_IOM_R3_IOPORT_READ:
+        case VINF_IOM_R3_IOPORT_WRITE:
+        case VINF_EM_RESUME_R3_HISTORY_EXEC: /* Resume EMHistoryExec after VMCPU_FF_IOM. */
+            rc = emR3ExecuteIOInstruction(pVM, pVCpu);
+            break;
+
+        /*
+         * Execute pending I/O Port access.
+         */
+        case VINF_EM_PENDING_R3_IOPORT_WRITE:
+            rc = VBOXSTRICTRC_TODO(emR3ExecutePendingIoPortWrite(pVM, pVCpu));
+            break;
+        case VINF_EM_PENDING_R3_IOPORT_READ:
+            rc = VBOXSTRICTRC_TODO(emR3ExecutePendingIoPortRead(pVM, pVCpu));
+            break;
+
+        /*
+         * Memory mapped I/O access - emulate the instruction.
+         */
+        case VINF_IOM_R3_MMIO_READ:
+        case VINF_IOM_R3_MMIO_WRITE:
+        case VINF_IOM_R3_MMIO_READ_WRITE:
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "MMIO");
+            break;
+
+        /*
+         * Machine specific register access - emulate the instruction.
+         */
+        case VINF_CPUM_R3_MSR_READ:
+        case VINF_CPUM_R3_MSR_WRITE:
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "MSR");
+            break;
+
+        /*
+         * GIM hypercall.
+         */
+        case VINF_GIM_R3_HYPERCALL:
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "Hypercall");
+            break;
+
+#ifdef EMHANDLERC_WITH_HM
+        case VINF_EM_HM_PATCH_TPR_INSTR:
+            rc = HMR3PatchTprInstr(pVM, pVCpu);
+            break;
+#endif
+
+        case VINF_EM_RAW_GUEST_TRAP:
+        case VINF_EM_RAW_EMULATE_INSTR:
+            Assert(!TRPMHasTrap(pVCpu)); /* We're directly executing instructions below without respecting any pending traps! */
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "EMUL: ");
+            break;
+
+        case VINF_EM_RAW_INJECT_TRPM_EVENT:
+            CPUM_IMPORT_EXTRN_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+            rc = VBOXSTRICTRC_VAL(IEMInjectTrpmEvent(pVCpu));
+            /* The following condition should be removed when IEM_IMPLEMENTS_TASKSWITCH becomes true. */
+            if (rc == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
+                rc = emR3ExecuteInstruction(pVM, pVCpu, "EVENT: ");
+            break;
+
+
+        /*
+         * Up a level.
+         */
+        case VINF_EM_TERMINATE:
+        case VINF_EM_OFF:
+        case VINF_EM_RESET:
+        case VINF_EM_SUSPEND:
+        case VINF_EM_HALT:
+        case VINF_EM_RESUME:
+        case VINF_EM_NO_MEMORY:
+        case VINF_EM_RESCHEDULE:
+        case VINF_EM_RESCHEDULE_REM:
+        case VINF_EM_WAIT_SIPI:
+            break;
+
+        /*
+         * Up a level and invoke the debugger.
+         */
+        case VINF_EM_DBG_STEPPED:
+        case VINF_EM_DBG_BREAKPOINT:
+        case VINF_EM_DBG_STEP:
+        case VINF_EM_DBG_HYPER_BREAKPOINT:
+        case VINF_EM_DBG_HYPER_STEPPED:
+        case VINF_EM_DBG_HYPER_ASSERTION:
+        case VINF_EM_DBG_STOP:
+        case VINF_EM_DBG_EVENT:
+            break;
+
+        /*
+         * Up a level, dump and debug.
+         */
+        case VERR_TRPM_DONT_PANIC:
+        case VERR_TRPM_PANIC:
+        case VERR_VMM_RING0_ASSERTION:
+        case VINF_EM_TRIPLE_FAULT:
+        case VERR_VMM_HYPER_CR3_MISMATCH:
+        case VERR_VMM_RING3_CALL_DISABLED:
+        case VERR_IEM_INSTR_NOT_IMPLEMENTED:
+        case VERR_IEM_ASPECT_NOT_IMPLEMENTED:
+        case VERR_EM_GUEST_CPU_HANG:
+            break;
+
+#ifdef EMHANDLERC_WITH_HM
+        /*
+         * Up a level, after Hm have done some release logging.
+         */
+        case VERR_VMX_INVALID_VMCS_FIELD:
+        case VERR_VMX_INVALID_VMCS_PTR:
+        case VERR_VMX_INVALID_VMXON_PTR:
+        case VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE:
+        case VERR_VMX_UNEXPECTED_EXCEPTION:
+        case VERR_VMX_UNEXPECTED_EXIT:
+        case VERR_VMX_INVALID_GUEST_STATE:
+        case VERR_VMX_UNABLE_TO_START_VM:
+        case VERR_SVM_UNKNOWN_EXIT:
+        case VERR_SVM_UNEXPECTED_EXIT:
+        case VERR_SVM_UNEXPECTED_PATCH_TYPE:
+        case VERR_SVM_UNEXPECTED_XCPT_EXIT:
+            HMR3CheckError(pVM, rc);
+            break;
+
+        /* Up a level; fatal */
+        case VERR_VMX_IN_VMX_ROOT_MODE:
+        case VERR_SVM_IN_USE:
+        case VERR_SVM_UNABLE_TO_START_VM:
+            break;
+#endif
+
+        /*
+         * These two should be handled via the force flag already, but just in
+         * case they end up here deal with it.
+         */
+        case VINF_IOM_R3_IOPORT_COMMIT_WRITE:
+        case VINF_IOM_R3_MMIO_COMMIT_WRITE:
+            AssertFailed();
+            rc = VBOXSTRICTRC_TODO(IOMR3ProcessForceFlag(pVM, pVCpu, rc));
+            break;
+
+        /*
+         * Anything which is not known to us means an internal error
+         * and the termination of the VM!
+         */
+        default:
+            AssertMsgFailed(("Unknown GC return code: %Rra\n", rc));
+            break;
+    }
+    return rc;
+}
+
+#endif /* !VMM_INCLUDED_SRC_include_EMHandleRCTmpl_h */
+
diff --git a/src/VBox/VMM/include/EMInternal.h b/src/VBox/VMM/include/EMInternal.h
new file mode 100644
index 00000000..c4d6d0ac
--- /dev/null
+++ b/src/VBox/VMM/include/EMInternal.h
@@ -0,0 +1,368 @@
+/* $Id: EMInternal.h $ */
+/** @file
+ * EM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_EMInternal_h
+#define VMM_INCLUDED_SRC_include_EMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/dis.h>
+#include <VBox/vmm/pdmcritsect.h>
+#include <iprt/avl.h>
+#include <setjmp.h>
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_em_int       Internal
+ * @ingroup grp_em
+ * @internal
+ * @{
+ */
+
+/** The saved state version. */
+#define EM_SAVED_STATE_VERSION                          5
+#define EM_SAVED_STATE_VERSION_PRE_IEM                  4
+#define EM_SAVED_STATE_VERSION_PRE_MWAIT                3
+#define EM_SAVED_STATE_VERSION_PRE_SMP                  2
+
+
+/** @name MWait state flags.
+ * @{
+ */
+/** MWait activated. */
+#define EMMWAIT_FLAG_ACTIVE             RT_BIT(0)
+/** MWait will continue when an interrupt is pending even when IF=0. */
+#define EMMWAIT_FLAG_BREAKIRQIF0        RT_BIT(1)
+/** Monitor instruction was executed previously. */
+#define EMMWAIT_FLAG_MONITOR_ACTIVE     RT_BIT(2)
+/** @} */
+
+/** EM time slice in ms; used for capping execution time. */
+#define EM_TIME_SLICE                   100
+
+/**
+ * Cli node structure
+ */
+typedef struct CLISTAT
+{
+    /** The key is the cli address. */
+    AVLGCPTRNODECORE        Core;
+#if HC_ARCH_BITS == 32 && !defined(RT_OS_WINDOWS)
+    /** Padding. */
+    uint32_t                u32Padding;
+#endif
+    /** Occurrences. */
+    STAMCOUNTER             Counter;
+} CLISTAT, *PCLISTAT;
+#ifdef IN_RING3
+AssertCompileMemberAlignment(CLISTAT, Counter, 8);
+#endif
+
+
+/**
+ * Excessive (used to be) EM statistics.
+ */
+typedef struct EMSTATS
+{
+#if 1 /* rawmode only? */
+    /** @name Privileged Instructions Ending Up In HC.
+     * @{ */
+    STAMCOUNTER             StatIoRestarted;
+    STAMCOUNTER             StatIoIem;
+    STAMCOUNTER             StatCli;
+    STAMCOUNTER             StatSti;
+    STAMCOUNTER             StatInvlpg;
+    STAMCOUNTER             StatHlt;
+    STAMCOUNTER             StatMovReadCR[DISCREG_CR4 + 1];
+    STAMCOUNTER             StatMovWriteCR[DISCREG_CR4 + 1];
+    STAMCOUNTER             StatMovDRx;
+    STAMCOUNTER             StatIret;
+    STAMCOUNTER             StatMovLgdt;
+    STAMCOUNTER             StatMovLldt;
+    STAMCOUNTER             StatMovLidt;
+    STAMCOUNTER             StatMisc;
+    STAMCOUNTER             StatSysEnter;
+    STAMCOUNTER             StatSysExit;
+    STAMCOUNTER             StatSysCall;
+    STAMCOUNTER             StatSysRet;
+    /** @} */
+#endif
+} EMSTATS;
+/** Pointer to the excessive EM statistics. */
+typedef EMSTATS *PEMSTATS;
+
+
+/**
+ * Exit history entry.
+ *
+ * @remarks We could perhaps trim this down a little bit by assuming uFlatPC
+ *          only needs 48 bits (currently true but will change) and stuffing
+ *          the flags+type in the available 16 bits made available.  The
+ *          timestamp could likewise be shortened to accomodate the index, or
+ *          we might skip the index entirely.  However, since we will have to
+ *          deal with 56-bit wide PC address before long, there's not point.
+ *
+ *          On the upside, there are unused bits in both uFlagsAndType and the
+ *          idxSlot fields if needed for anything.
+ */
+typedef struct EMEXITENTRY
+{
+    /** The flat PC (CS:EIP/RIP) address of the exit.
+     * UINT64_MAX if not available.  */
+    uint64_t        uFlatPC;
+    /** The EMEXIT_MAKE_FLAGS_AND_TYPE */
+    uint32_t        uFlagsAndType;
+    /** The index into the exit slot hash table.
+     * UINT32_MAX if too many collisions and not entered into it. */
+    uint32_t        idxSlot;
+    /** The TSC timestamp of the exit.
+     * This is 0 if not timestamped. */
+    uint64_t        uTimestamp;
+} EMEXITENTRY;
+/** Pointer to an exit history entry. */
+typedef EMEXITENTRY *PEMEXITENTRY;
+/** Pointer to a const exit history entry. */
+typedef EMEXITENTRY const *PCEMEXITENTRY;
+
+
+/**
+ * EM VM Instance data.
+ */
+typedef struct EM
+{
+    /** Whether IEM executes everything. */
+    bool                    fIemExecutesAll;
+    /** Whether a triple fault triggers a guru. */
+    bool                    fGuruOnTripleFault;
+    /** Alignment padding. */
+    bool                    afPadding[2];
+
+    /** Id of the VCPU that last executed code in the recompiler. */
+    VMCPUID                 idLastRemCpu;
+} EM;
+/** Pointer to EM VM instance data. */
+typedef EM *PEM;
+
+
+/**
+ * EM VMCPU Instance data.
+ */
+typedef struct EMCPU
+{
+    /** Execution Manager State. */
+    EMSTATE volatile        enmState;
+
+    /** The state prior to the suspending of the VM. */
+    EMSTATE                 enmPrevState;
+
+    /** Set if hypercall instruction VMMCALL (AMD) & VMCALL (Intel) are enabled.
+     * GIM sets this and the execution managers queries it.  Not saved, as GIM
+     * takes care of that bit too.  */
+    bool                    fHypercallEnabled;
+
+    /** Explicit padding. */
+    uint8_t                 abPadding0[3];
+
+    /** The number of instructions we've executed in IEM since switching to the
+     *  EMSTATE_IEM_THEN_REM state. */
+    uint32_t                cIemThenRemInstructions;
+
+    /** Inhibit interrupts for this instruction. Valid only when VM_FF_INHIBIT_INTERRUPTS is set. */
+    RTGCUINTPTR             GCPtrInhibitInterrupts;
+
+    /** Start of the current time slice in ms. */
+    uint64_t                u64TimeSliceStart;
+    /** Start of the current time slice in thread execution time (ms). */
+    uint64_t                u64TimeSliceStartExec;
+    /** Current time slice value. */
+    uint64_t                u64TimeSliceExec;
+
+    /** Pending ring-3 I/O port access (VINF_EM_PENDING_R3_IOPORT_READ / VINF_EM_PENDING_R3_IOPORT_WRITE). */
+    struct
+    {
+        RTIOPORT            uPort;          /**< The I/O port number.*/
+        uint8_t             cbValue;        /**< The value size in bytes.  Zero when not pending. */
+        uint8_t             cbInstr;        /**< The instruction length. */
+        uint32_t            uValue;         /**< The value to write. */
+    } PendingIoPortAccess;
+
+    /** MWait halt state. */
+    struct
+    {
+        uint32_t            fWait;          /**< Type of mwait; see EMMWAIT_FLAG_*. */
+        uint32_t            u32Padding;
+        RTGCPTR             uMWaitRAX;      /**< MWAIT hints. */
+        RTGCPTR             uMWaitRCX;      /**< MWAIT extensions. */
+        RTGCPTR             uMonitorRAX;    /**< Monitored address. */
+        RTGCPTR             uMonitorRCX;    /**< Monitor extension. */
+        RTGCPTR             uMonitorRDX;    /**< Monitor hint. */
+    } MWait;
+
+    /** Make sure the jmp_buf is at a 32-byte boundrary. */
+    uint64_t                au64Padding1[3];
+    union
+    {
+        /** Padding used in the other rings.
+         * This must be larger than jmp_buf on any supported platform. */
+        char                achPaddingFatalLongJump[256];
+#ifdef IN_RING3
+        /** Long buffer jump for fatal VM errors.
+         * It will jump to before the outer EM loop is entered. */
+        jmp_buf             FatalLongJump;
+#endif
+    } u;
+
+    /** For saving stack space, the disassembler state is allocated here instead of
+     * on the stack. */
+    DISCPUSTATE             DisState;
+
+    /** @name Execution profiling.
+     * @{ */
+    STAMPROFILE             StatForcedActions;
+    STAMPROFILE             StatHalted;
+    STAMPROFILEADV          StatCapped;
+    STAMPROFILEADV          StatHMEntry;
+    STAMPROFILE             StatHMExec;
+    STAMPROFILE             StatIEMEmu;
+    STAMPROFILE             StatIEMThenREM;
+    STAMPROFILEADV          StatNEMEntry;
+    STAMPROFILE             StatNEMExec;
+    STAMPROFILE             StatREMEmu;
+    STAMPROFILE             StatREMExec;
+    STAMPROFILE             StatREMSync;
+    STAMPROFILEADV          StatREMTotal;
+    STAMPROFILE             StatRAWExec;
+    STAMPROFILEADV          StatRAWEntry;
+    STAMPROFILEADV          StatRAWTail;
+    STAMPROFILEADV          StatRAWTotal;
+    STAMPROFILEADV          StatTotal;
+    /** @} */
+
+    /** R3: Profiling of emR3RawExecuteIOInstruction. */
+    STAMPROFILE             StatIOEmu;
+    /** R3: Profiling of emR3RawPrivileged. */
+    STAMPROFILE             StatPrivEmu;
+    /** R3: Number of times emR3HmExecute is called. */
+    STAMCOUNTER             StatHMExecuteCalled;
+    /** R3: Number of times emR3NEMExecute is called. */
+    STAMCOUNTER             StatNEMExecuteCalled;
+
+    /** More statistics (R3). */
+    R3PTRTYPE(PEMSTATS)     pStatsR3;
+    /** More statistics (R0). */
+    R0PTRTYPE(PEMSTATS)     pStatsR0;
+
+    /** Tree for keeping track of cli occurrences (debug only). */
+    R3PTRTYPE(PAVLGCPTRNODECORE) pCliStatTree;
+    STAMCOUNTER             StatTotalClis;
+    /** Align the next member at a 16-byte boundrary. */
+    uint64_t                au64Padding2[1];
+
+    /** Exit history table (6KB). */
+    EMEXITENTRY             aExitHistory[256];
+    /** Where to store the next exit history entry.
+     * Since aExitHistory is 256 items longs, we'll just increment this and
+     * mask it when using it.  That help the readers detect whether we've
+     * wrapped around or not.  */
+    uint64_t                iNextExit;
+
+    /** Index into aExitRecords set by EMHistoryExec when returning to ring-3.
+     * This is UINT16_MAX if not armed.  */
+    uint16_t volatile       idxContinueExitRec;
+    /** Whether exit optimizations are enabled or not (in general). */
+    bool                    fExitOptimizationEnabled : 1;
+    /** Whether exit optimizations are enabled for ring-0 (in general). */
+    bool                    fExitOptimizationEnabledR0 : 1;
+    /** Whether exit optimizations are enabled for ring-0 when preemption is disabled. */
+    bool                    fExitOptimizationEnabledR0PreemptDisabled : 1;
+    /** Explicit padding. */
+    bool                    fPadding2;
+    /** Max number of instructions to execute. */
+    uint16_t                cHistoryExecMaxInstructions;
+    /** Min number of instructions to execute while probing. */
+    uint16_t                cHistoryProbeMinInstructions;
+    /** Max number of instructions to execute without an exit before giving up probe. */
+    uint16_t                cHistoryProbeMaxInstructionsWithoutExit;
+    uint16_t                uPadding3;
+    /** Number of exit records in use. */
+    uint32_t                cExitRecordUsed;
+    /** Profiling the EMHistoryExec when executing (not probing). */
+    STAMPROFILE             StatHistoryExec;
+    /** Number of saved exits. */
+    STAMCOUNTER             StatHistoryExecSavedExits;
+    /** Number of instructions executed by EMHistoryExec. */
+    STAMCOUNTER             StatHistoryExecInstructions;
+    uint64_t                uPadding4;
+    /** Number of instructions executed by EMHistoryExec when probing. */
+    STAMCOUNTER             StatHistoryProbeInstructions;
+    /** Number of times probing resulted in EMEXITACTION_NORMAL_PROBED. */
+    STAMCOUNTER             StatHistoryProbedNormal;
+    /** Number of times probing resulted in EMEXITACTION_EXEC_WITH_MAX. */
+    STAMCOUNTER             StatHistoryProbedExecWithMax;
+    /** Number of times probing resulted in ring-3 continuation. */
+    STAMCOUNTER             StatHistoryProbedToRing3;
+    /** Profiling the EMHistoryExec when probing.*/
+    STAMPROFILE             StatHistoryProbe;
+    /** Hit statistics for each lookup step. */
+    STAMCOUNTER             aStatHistoryRecHits[16];
+    /** Type change statistics for each lookup step. */
+    STAMCOUNTER             aStatHistoryRecTypeChanged[16];
+    /** Replacement statistics for each lookup step. */
+    STAMCOUNTER             aStatHistoryRecReplaced[16];
+    /** New record statistics for each lookup step. */
+    STAMCOUNTER             aStatHistoryRecNew[16];
+
+    /** Exit records (32KB). (Aligned on 32 byte boundrary.) */
+    EMEXITREC               aExitRecords[1024];
+} EMCPU;
+/** Pointer to EM VM instance data. */
+typedef EMCPU *PEMCPU;
+
+/** @} */
+
+int             emR3InitDbg(PVM pVM);
+
+int             emR3HmExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
+VBOXSTRICTRC    emR3NemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
+int             emR3RawExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
+
+EMSTATE         emR3Reschedule(PVM pVM, PVMCPU pVCpu);
+int             emR3ForcedActions(PVM pVM, PVMCPU pVCpu, int rc);
+VBOXSTRICTRC    emR3HighPriorityPostForcedActions(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc);
+
+int             emR3RawResumeHyper(PVM pVM, PVMCPU pVCpu);
+int             emR3RawStep(PVM pVM, PVMCPU pVCpu);
+
+VBOXSTRICTRC    emR3NemSingleInstruction(PVM pVM, PVMCPU pVCpu, uint32_t fFlags);
+
+int             emR3SingleStepExecRem(PVM pVM, PVMCPU pVCpu, uint32_t cIterations);
+
+bool            emR3IsExecutionAllowed(PVM pVM, PVMCPU pVCpu);
+
+VBOXSTRICTRC    emR3ExecutePendingIoPortWrite(PVM pVM, PVMCPU pVCpu);
+VBOXSTRICTRC    emR3ExecutePendingIoPortRead(PVM pVM, PVMCPU pVCpu);
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_EMInternal_h */
+
diff --git a/src/VBox/VMM/include/GIMHvInternal.h b/src/VBox/VMM/include/GIMHvInternal.h
new file mode 100644
index 00000000..9dd86e54
--- /dev/null
+++ b/src/VBox/VMM/include/GIMHvInternal.h
@@ -0,0 +1,1375 @@
+/* $Id: GIMHvInternal.h $ */
+/** @file
+ * GIM - Hyper-V, Internal header file.
+ */
+
+/*
+ * Copyright (C) 2014-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_GIMHvInternal_h
+#define VMM_INCLUDED_SRC_include_GIMHvInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/cpum.h>
+
+#include <iprt/net.h>
+
+/** @name Hyper-V base feature identification.
+ * Features based on current partition privileges (per-VM).
+ * @{
+ */
+/** Virtual processor runtime MSR available. */
+#define GIM_HV_BASE_FEAT_VP_RUNTIME_MSR                     RT_BIT(0)
+/** Partition reference counter MSR available. */
+#define GIM_HV_BASE_FEAT_PART_TIME_REF_COUNT_MSR            RT_BIT(1)
+/** Basic Synthetic Interrupt Controller MSRs available. */
+#define GIM_HV_BASE_FEAT_BASIC_SYNIC_MSRS                   RT_BIT(2)
+/** Synthetic Timer MSRs available. */
+#define GIM_HV_BASE_FEAT_STIMER_MSRS                        RT_BIT(3)
+/** APIC access MSRs (EOI, ICR, TPR) available. */
+#define GIM_HV_BASE_FEAT_APIC_ACCESS_MSRS                   RT_BIT(4)
+/** Hypercall MSRs available. */
+#define GIM_HV_BASE_FEAT_HYPERCALL_MSRS                     RT_BIT(5)
+/** Access to VCPU index MSR available. */
+#define GIM_HV_BASE_FEAT_VP_ID_MSR                          RT_BIT(6)
+/** Virtual system reset MSR available. */
+#define GIM_HV_BASE_FEAT_VIRT_SYS_RESET_MSR                 RT_BIT(7)
+/** Statistic pages MSRs available. */
+#define GIM_HV_BASE_FEAT_STAT_PAGES_MSR                     RT_BIT(8)
+/** Paritition reference TSC MSR available. */
+#define GIM_HV_BASE_FEAT_PART_REF_TSC_MSR                   RT_BIT(9)
+/** Virtual guest idle state MSR available. */
+#define GIM_HV_BASE_FEAT_GUEST_IDLE_STATE_MSR               RT_BIT(10)
+/** Timer frequency MSRs (TSC and APIC) available. */
+#define GIM_HV_BASE_FEAT_TIMER_FREQ_MSRS                    RT_BIT(11)
+/** Debug MSRs available. */
+#define GIM_HV_BASE_FEAT_DEBUG_MSRS                         RT_BIT(12)
+/** @}  */
+
+/** @name Hyper-V partition-creation feature identification.
+ * Indicates flags specified during partition creation.
+ * @{
+ */
+/** Create partitions. */
+#define GIM_HV_PART_FLAGS_CREATE_PART                       RT_BIT(0)
+/** Access partition Id. */
+#define GIM_HV_PART_FLAGS_ACCESS_PART_ID                    RT_BIT(1)
+/** Access memory pool. */
+#define GIM_HV_PART_FLAGS_ACCESS_MEMORY_POOL                RT_BIT(2)
+/** Adjust message buffers. */
+#define GIM_HV_PART_FLAGS_ADJUST_MSG_BUFFERS                RT_BIT(3)
+/** Post messages. */
+#define GIM_HV_PART_FLAGS_POST_MSGS                         RT_BIT(4)
+/** Signal events. */
+#define GIM_HV_PART_FLAGS_SIGNAL_EVENTS                     RT_BIT(5)
+/** Create port. */
+#define GIM_HV_PART_FLAGS_CREATE_PORT                       RT_BIT(6)
+/** Connect port. */
+#define GIM_HV_PART_FLAGS_CONNECT_PORT                      RT_BIT(7)
+/** Access statistics. */
+#define GIM_HV_PART_FLAGS_ACCESS_STATS                      RT_BIT(8)
+/** Debugging.*/
+#define GIM_HV_PART_FLAGS_DEBUGGING                         RT_BIT(11)
+/** CPU management. */
+#define GIM_HV_PART_FLAGS_CPU_MGMT                          RT_BIT(12)
+/** CPU profiler. */
+#define GIM_HV_PART_FLAGS_CPU_PROFILER                      RT_BIT(13)
+/** Enable expanded stack walking. */
+#define GIM_HV_PART_FLAGS_EXPANDED_STACK_WALK               RT_BIT(14)
+/** Access VSM. */
+#define GIM_HV_PART_FLAGS_ACCESS_VSM                        RT_BIT(16)
+/** Access VP registers. */
+#define GIM_HV_PART_FLAGS_ACCESS_VP_REGS                    RT_BIT(17)
+/** Enable extended hypercalls. */
+#define GIM_HV_PART_FLAGS_EXTENDED_HYPERCALLS               RT_BIT(20)
+/** Start virtual processor. */
+#define GIM_HV_PART_FLAGS_START_VP                          RT_BIT(21)
+/** @}  */
+
+/** @name Hyper-V power management feature identification.
+ * @{
+ */
+/** Maximum CPU power state C0. */
+#define GIM_HV_PM_MAX_CPU_POWER_STATE_C0                    RT_BIT(0)
+/** Maximum CPU power state C1. */
+#define GIM_HV_PM_MAX_CPU_POWER_STATE_C1                    RT_BIT(1)
+/** Maximum CPU power state C2. */
+#define GIM_HV_PM_MAX_CPU_POWER_STATE_C2                    RT_BIT(2)
+/** Maximum CPU power state C3. */
+#define GIM_HV_PM_MAX_CPU_POWER_STATE_C3                    RT_BIT(3)
+/** HPET is required to enter C3 power state. */
+#define GIM_HV_PM_HPET_REQD_FOR_C3                          RT_BIT(4)
+/** @}  */
+
+/** @name Hyper-V miscellaneous feature identification.
+ * Miscellaneous features available for the current partition.
+ * @{
+ */
+/** MWAIT instruction available. */
+#define GIM_HV_MISC_FEAT_MWAIT                              RT_BIT(0)
+/** Guest debugging support available. */
+#define GIM_HV_MISC_FEAT_GUEST_DEBUGGING                    RT_BIT(1)
+/** Performance monitor support is available. */
+#define GIM_HV_MISC_FEAT_PERF_MON                           RT_BIT(2)
+/** Support for physical CPU dynamic partitioning events. */
+#define GIM_HV_MISC_FEAT_PCPU_DYN_PART_EVENT                RT_BIT(3)
+/** Support for passing hypercall input parameter block via XMM registers. */
+#define GIM_HV_MISC_FEAT_XMM_HYPERCALL_INPUT                RT_BIT(4)
+/** Support for virtual guest idle state. */
+#define GIM_HV_MISC_FEAT_GUEST_IDLE_STATE                   RT_BIT(5)
+/** Support for hypervisor sleep state. */
+#define GIM_HV_MISC_FEAT_HYPERVISOR_SLEEP_STATE             RT_BIT(6)
+/** Support for querying NUMA distances. */
+#define GIM_HV_MISC_FEAT_QUERY_NUMA_DISTANCE                RT_BIT(7)
+/** Support for determining timer frequencies. */
+#define GIM_HV_MISC_FEAT_TIMER_FREQ                         RT_BIT(8)
+/** Support for injecting synthetic machine checks. */
+#define GIM_HV_MISC_FEAT_INJECT_SYNMC_XCPT                  RT_BIT(9)
+/** Support for guest crash MSRs. */
+#define GIM_HV_MISC_FEAT_GUEST_CRASH_MSRS                   RT_BIT(10)
+/** Support for debug MSRs. */
+#define GIM_HV_MISC_FEAT_DEBUG_MSRS                         RT_BIT(11)
+/** Npiep1 Available */ /** @todo What the heck is this? */
+#define GIM_HV_MISC_FEAT_NPIEP1                             RT_BIT(12)
+/** Disable hypervisor available. */
+#define GIM_HV_MISC_FEAT_DISABLE_HYPERVISOR                 RT_BIT(13)
+/** Extended GVA ranges for FlushVirtualAddressList available. */
+#define GIM_HV_MISC_FEAT_EXT_GVA_RANGE_FOR_FLUSH_VA_LIST    RT_BIT(14)
+/** Support for returning hypercall output via XMM registers. */
+#define GIM_HV_MISC_FEAT_HYPERCALL_OUTPUT_XMM               RT_BIT(15)
+/** Synthetic interrupt source polling mode available. */
+#define GIM_HV_MISC_FEAT_SINT_POLLING_MODE                  RT_BIT(17)
+/** Hypercall MSR lock available. */
+#define GIM_HV_MISC_FEAT_HYPERCALL_MSR_LOCK                 RT_BIT(18)
+/** Use direct synthetic MSRs. */
+#define GIM_HV_MISC_FEAT_USE_DIRECT_SYNTH_MSRS              RT_BIT(19)
+/** @}  */
+
+/** @name Hyper-V implementation recommendations.
+ * Recommendations from the hypervisor for the guest for optimal performance.
+ * @{
+ */
+/** Use hypercall for address space switches rather than MOV CR3. */
+#define GIM_HV_HINT_HYPERCALL_FOR_PROCESS_SWITCH            RT_BIT(0)
+/** Use hypercall for local TLB flushes rather than INVLPG/MOV CR3. */
+#define GIM_HV_HINT_HYPERCALL_FOR_TLB_FLUSH                 RT_BIT(1)
+/** Use hypercall for inter-CPU TLB flushes rather than IPIs. */
+#define GIM_HV_HINT_HYPERCALL_FOR_TLB_SHOOTDOWN             RT_BIT(2)
+/** Use MSRs for APIC access (EOI, ICR, TPR) rather than MMIO. */
+#define GIM_HV_HINT_MSR_FOR_APIC_ACCESS                     RT_BIT(3)
+/** Use hypervisor provided MSR for a system reset. */
+#define GIM_HV_HINT_MSR_FOR_SYS_RESET                       RT_BIT(4)
+/** Relax timer-related checks (watchdogs/deadman timeouts) that rely on
+ *  timely deliver of external interrupts. */
+#define GIM_HV_HINT_RELAX_TIME_CHECKS                       RT_BIT(5)
+/** Recommend using DMA remapping. */
+#define GIM_HV_HINT_DMA_REMAPPING                           RT_BIT(6)
+/** Recommend using interrupt remapping. */
+#define GIM_HV_HINT_INTERRUPT_REMAPPING                     RT_BIT(7)
+/** Recommend using X2APIC MSRs rather than MMIO. */
+#define GIM_HV_HINT_X2APIC_MSRS                             RT_BIT(8)
+/** Recommend deprecating Auto EOI (end of interrupt). */
+#define GIM_HV_HINT_DEPRECATE_AUTO_EOI                      RT_BIT(9)
+/** Recommend using SyntheticClusterIpi hypercall. */
+#define GIM_HV_HINT_SYNTH_CLUSTER_IPI_HYPERCALL             RT_BIT(10)
+/** Recommend using newer ExProcessMasks interface. */
+#define GIM_HV_HINT_EX_PROC_MASKS_INTERFACE                 RT_BIT(11)
+/** Indicate that Hyper-V is nested within a Hyper-V partition. */
+#define GIM_HV_HINT_NESTED_HYPERV                           RT_BIT(12)
+/** Recommend using INT for MBEC system calls. */
+#define GIM_HV_HINT_INT_FOR_MBEC_SYSCALLS                   RT_BIT(13)
+/** Recommend using enlightened VMCS interfacea and nested enlightenments. */
+#define GIM_HV_HINT_NESTED_ENLIGHTENED_VMCS_INTERFACE       RT_BIT(14)
+/** @}  */
+
+
+/** @name Hyper-V implementation hardware features.
+ * Which hardware features are in use by the hypervisor.
+ * @{
+ */
+/** APIC overlay is used. */
+#define GIM_HV_HOST_FEAT_AVIC                               RT_BIT(0)
+/** MSR bitmaps is used. */
+#define GIM_HV_HOST_FEAT_MSR_BITMAP                         RT_BIT(1)
+/** Architectural performance counter supported. */
+#define GIM_HV_HOST_FEAT_PERF_COUNTER                       RT_BIT(2)
+/** Nested paging is used. */
+#define GIM_HV_HOST_FEAT_NESTED_PAGING                      RT_BIT(3)
+/** DMA remapping is used. */
+#define GIM_HV_HOST_FEAT_DMA_REMAPPING                      RT_BIT(4)
+/** Interrupt remapping is used. */
+#define GIM_HV_HOST_FEAT_INTERRUPT_REMAPPING                RT_BIT(5)
+/** Memory patrol scrubber is present. */
+#define GIM_HV_HOST_FEAT_MEM_PATROL_SCRUBBER                RT_BIT(6)
+/** DMA protection is in use. */
+#define GIM_HV_HOST_FEAT_DMA_PROT_IN_USE                    RT_BIT(7)
+/** HPET is requested. */
+#define GIM_HV_HOST_FEAT_HPET_REQUESTED                     RT_BIT(8)
+/** Synthetic timers are volatile. */
+#define GIM_HV_HOST_FEAT_STIMER_VOLATILE                    RT_BIT(9)
+/** @}  */
+
+
+/** @name Hyper-V MSRs.
+ * @{
+ */
+/** Start of range 0. */
+#define MSR_GIM_HV_RANGE0_FIRST                   UINT32_C(0x40000000)
+/** Guest OS identification (R/W) */
+#define MSR_GIM_HV_GUEST_OS_ID                    UINT32_C(0x40000000)
+/** Enable hypercall interface (R/W) */
+#define MSR_GIM_HV_HYPERCALL                      UINT32_C(0x40000001)
+/** Virtual processor's (VCPU) index (R) */
+#define MSR_GIM_HV_VP_INDEX                       UINT32_C(0x40000002)
+/** Reset operation (R/W) */
+#define MSR_GIM_HV_RESET                          UINT32_C(0x40000003)
+/** End of range 0. */
+#define MSR_GIM_HV_RANGE0_LAST                    MSR_GIM_HV_RESET
+
+/** Start of range 1. */
+#define MSR_GIM_HV_RANGE1_FIRST                   UINT32_C(0x40000010)
+/** Virtual processor's (VCPU) runtime (R) */
+#define MSR_GIM_HV_VP_RUNTIME                     UINT32_C(0x40000010)
+/** End of range 1. */
+#define MSR_GIM_HV_RANGE1_LAST                    MSR_GIM_HV_VP_RUNTIME
+
+/** Start of range 2. */
+#define MSR_GIM_HV_RANGE2_FIRST                   UINT32_C(0x40000020)
+/** Per-VM reference counter (R) */
+#define MSR_GIM_HV_TIME_REF_COUNT                 UINT32_C(0x40000020)
+/** Per-VM TSC page (R/W) */
+#define MSR_GIM_HV_REF_TSC                        UINT32_C(0x40000021)
+/** Frequency of TSC in Hz as reported by the hypervisor (R) */
+#define MSR_GIM_HV_TSC_FREQ                       UINT32_C(0x40000022)
+/** Frequency of LAPIC in Hz as reported by the hypervisor (R) */
+#define MSR_GIM_HV_APIC_FREQ                      UINT32_C(0x40000023)
+/** End of range 2. */
+#define MSR_GIM_HV_RANGE2_LAST                    MSR_GIM_HV_APIC_FREQ
+
+/** Start of range 3. */
+#define MSR_GIM_HV_RANGE3_FIRST                   UINT32_C(0x40000070)
+/** Access to APIC EOI (End-Of-Interrupt) register (W) */
+#define MSR_GIM_HV_EOI                            UINT32_C(0x40000070)
+/** Access to APIC ICR (Interrupt Command) register (R/W) */
+#define MSR_GIM_HV_ICR                            UINT32_C(0x40000071)
+/** Access to APIC TPR (Task Priority) register (R/W) */
+#define MSR_GIM_HV_TPR                            UINT32_C(0x40000072)
+/** Enables lazy EOI processing (R/W) */
+#define MSR_GIM_HV_APIC_ASSIST_PAGE               UINT32_C(0x40000073)
+/** End of range 3. */
+#define MSR_GIM_HV_RANGE3_LAST                    MSR_GIM_HV_APIC_ASSIST_PAGE
+
+/** Start of range 4. */
+#define MSR_GIM_HV_RANGE4_FIRST                   UINT32_C(0x40000080)
+/** Control behaviour of synthetic interrupt controller (R/W) */
+#define MSR_GIM_HV_SCONTROL                       UINT32_C(0x40000080)
+/** Synthetic interrupt controller version (R) */
+#define MSR_GIM_HV_SVERSION                       UINT32_C(0x40000081)
+/** Base address of synthetic interrupt event flag (R/W) */
+#define MSR_GIM_HV_SIEFP                          UINT32_C(0x40000082)
+/** Base address of synthetic interrupt message page (R/W) */
+#define MSR_GIM_HV_SIMP                           UINT32_C(0x40000083)
+/** End-Of-Message in synthetic interrupt parameter page (W) */
+#define MSR_GIM_HV_EOM                            UINT32_C(0x40000084)
+/** End of range 4. */
+#define MSR_GIM_HV_RANGE4_LAST                    MSR_GIM_HV_EOM
+
+/** Start of range 5. */
+#define MSR_GIM_HV_RANGE5_FIRST                   UINT32_C(0x40000090)
+/** Configures synthetic interrupt source 0 (R/W) */
+#define MSR_GIM_HV_SINT0                          UINT32_C(0x40000090)
+/** Configures synthetic interrupt source 1 (R/W) */
+#define MSR_GIM_HV_SINT1                          UINT32_C(0x40000091)
+/** Configures synthetic interrupt source 2 (R/W) */
+#define MSR_GIM_HV_SINT2                          UINT32_C(0x40000092)
+/** Configures synthetic interrupt source 3 (R/W) */
+#define MSR_GIM_HV_SINT3                          UINT32_C(0x40000093)
+/** Configures synthetic interrupt source 4 (R/W) */
+#define MSR_GIM_HV_SINT4                          UINT32_C(0x40000094)
+/** Configures synthetic interrupt source 5 (R/W) */
+#define MSR_GIM_HV_SINT5                          UINT32_C(0x40000095)
+/** Configures synthetic interrupt source 6 (R/W) */
+#define MSR_GIM_HV_SINT6                          UINT32_C(0x40000096)
+/** Configures synthetic interrupt source 7 (R/W) */
+#define MSR_GIM_HV_SINT7                          UINT32_C(0x40000097)
+/** Configures synthetic interrupt source 8 (R/W) */
+#define MSR_GIM_HV_SINT8                          UINT32_C(0x40000098)
+/** Configures synthetic interrupt source 9 (R/W) */
+#define MSR_GIM_HV_SINT9                          UINT32_C(0x40000099)
+/** Configures synthetic interrupt source 10 (R/W) */
+#define MSR_GIM_HV_SINT10                         UINT32_C(0x4000009A)
+/** Configures synthetic interrupt source 11 (R/W) */
+#define MSR_GIM_HV_SINT11                         UINT32_C(0x4000009B)
+/** Configures synthetic interrupt source 12 (R/W) */
+#define MSR_GIM_HV_SINT12                         UINT32_C(0x4000009C)
+/** Configures synthetic interrupt source 13 (R/W) */
+#define MSR_GIM_HV_SINT13                         UINT32_C(0x4000009D)
+/** Configures synthetic interrupt source 14 (R/W) */
+#define MSR_GIM_HV_SINT14                         UINT32_C(0x4000009E)
+/** Configures synthetic interrupt source 15 (R/W) */
+#define MSR_GIM_HV_SINT15                         UINT32_C(0x4000009F)
+/** End of range 5. */
+#define MSR_GIM_HV_RANGE5_LAST                    MSR_GIM_HV_SINT15
+
+/** Start of range 6. */
+#define MSR_GIM_HV_RANGE6_FIRST                   UINT32_C(0x400000B0)
+/** Configures register for synthetic timer 0 (R/W) */
+#define MSR_GIM_HV_STIMER0_CONFIG                 UINT32_C(0x400000B0)
+/** Expiration time or period for synthetic timer 0 (R/W) */
+#define MSR_GIM_HV_STIMER0_COUNT                  UINT32_C(0x400000B1)
+/** Configures register for synthetic timer 1 (R/W) */
+#define MSR_GIM_HV_STIMER1_CONFIG                 UINT32_C(0x400000B2)
+/** Expiration time or period for synthetic timer 1 (R/W) */
+#define MSR_GIM_HV_STIMER1_COUNT                  UINT32_C(0x400000B3)
+/** Configures register for synthetic timer 2 (R/W) */
+#define MSR_GIM_HV_STIMER2_CONFIG                 UINT32_C(0x400000B4)
+/** Expiration time or period for synthetic timer 2 (R/W) */
+#define MSR_GIM_HV_STIMER2_COUNT                  UINT32_C(0x400000B5)
+/** Configures register for synthetic timer 3 (R/W) */
+#define MSR_GIM_HV_STIMER3_CONFIG                 UINT32_C(0x400000B6)
+/** Expiration time or period for synthetic timer 3 (R/W) */
+#define MSR_GIM_HV_STIMER3_COUNT                  UINT32_C(0x400000B7)
+/** End of range 6. */
+#define MSR_GIM_HV_RANGE6_LAST                    MSR_GIM_HV_STIMER3_COUNT
+
+/** Start of range 7. */
+#define MSR_GIM_HV_RANGE7_FIRST                   UINT32_C(0x400000C1)
+/** Trigger to transition to power state C1 (R) */
+#define MSR_GIM_HV_POWER_STATE_TRIGGER_C1         UINT32_C(0x400000C1)
+/** Trigger to transition to power state C2 (R) */
+#define MSR_GIM_HV_POWER_STATE_TRIGGER_C2         UINT32_C(0x400000C2)
+/** Trigger to transition to power state C3 (R) */
+#define MSR_GIM_HV_POWER_STATE_TRIGGER_C3         UINT32_C(0x400000C3)
+/** End of range 7. */
+#define MSR_GIM_HV_RANGE7_LAST                    MSR_GIM_HV_POWER_STATE_TRIGGER_C3
+
+/** Start of range 8. */
+#define MSR_GIM_HV_RANGE8_FIRST                   UINT32_C(0x400000D1)
+/** Configure the recipe for power state transitions to C1 (R/W) */
+#define MSR_GIM_HV_POWER_STATE_CONFIG_C1          UINT32_C(0x400000D1)
+/** Configure the recipe for power state transitions to C2 (R/W) */
+#define MSR_GIM_HV_POWER_STATE_CONFIG_C2          UINT32_C(0x400000D2)
+/** Configure the recipe for power state transitions to C3 (R/W) */
+#define MSR_GIM_HV_POWER_STATE_CONFIG_C3          UINT32_C(0x400000D3)
+/** End of range 8. */
+#define MSR_GIM_HV_RANGE8_LAST                    MSR_GIM_HV_POWER_STATE_CONFIG_C3
+
+/** Start of range 9. */
+#define MSR_GIM_HV_RANGE9_FIRST                   UINT32_C(0x400000E0)
+/** Map the guest's retail partition stats page (R/W) */
+#define MSR_GIM_HV_STATS_PART_RETAIL_PAGE         UINT32_C(0x400000E0)
+/** Map the guest's internal partition stats page (R/W) */
+#define MSR_GIM_HV_STATS_PART_INTERNAL_PAGE       UINT32_C(0x400000E1)
+/** Map the guest's retail VP stats page (R/W) */
+#define MSR_GIM_HV_STATS_VP_RETAIL_PAGE           UINT32_C(0x400000E2)
+/** Map the guest's internal VP stats page (R/W) */
+#define MSR_GIM_HV_STATS_VP_INTERNAL_PAGE         UINT32_C(0x400000E3)
+/** End of range 9. */
+#define MSR_GIM_HV_RANGE9_LAST                    MSR_GIM_HV_STATS_VP_INTERNAL_PAGE
+
+/** Start of range 10. */
+#define MSR_GIM_HV_RANGE10_FIRST                  UINT32_C(0x400000F0)
+/** Trigger the guest's transition to idle power state (R) */
+#define MSR_GIM_HV_GUEST_IDLE                     UINT32_C(0x400000F0)
+/** Synthetic debug control. */
+#define MSR_GIM_HV_SYNTH_DEBUG_CONTROL            UINT32_C(0x400000F1)
+/** Synthetic debug status. */
+#define MSR_GIM_HV_SYNTH_DEBUG_STATUS             UINT32_C(0x400000F2)
+/** Synthetic debug send buffer. */
+#define MSR_GIM_HV_SYNTH_DEBUG_SEND_BUFFER        UINT32_C(0x400000F3)
+/** Synthetic debug receive buffer. */
+#define MSR_GIM_HV_SYNTH_DEBUG_RECEIVE_BUFFER     UINT32_C(0x400000F4)
+/** Synthetic debug pending buffer. */
+#define MSR_GIM_HV_SYNTH_DEBUG_PENDING_BUFFER     UINT32_C(0x400000F5)
+/** End of range 10. */
+#define MSR_GIM_HV_RANGE10_LAST                   MSR_GIM_HV_SYNTH_DEBUG_PENDING_BUFFER
+
+/** Start of range 11. */
+#define MSR_GIM_HV_RANGE11_FIRST                  UINT32_C(0x400000FF)
+/** Undocumented debug options MSR. */
+#define MSR_GIM_HV_DEBUG_OPTIONS_MSR              UINT32_C(0x400000FF)
+/** End of range 11. */
+#define MSR_GIM_HV_RANGE11_LAST                   MSR_GIM_HV_DEBUG_OPTIONS_MSR
+
+/** Start of range 12. */
+#define MSR_GIM_HV_RANGE12_FIRST                  UINT32_C(0x40000100)
+/** Guest crash MSR 0. */
+#define MSR_GIM_HV_CRASH_P0                       UINT32_C(0x40000100)
+/** Guest crash MSR 1. */
+#define MSR_GIM_HV_CRASH_P1                       UINT32_C(0x40000101)
+/** Guest crash MSR 2. */
+#define MSR_GIM_HV_CRASH_P2                       UINT32_C(0x40000102)
+/** Guest crash MSR 3. */
+#define MSR_GIM_HV_CRASH_P3                       UINT32_C(0x40000103)
+/** Guest crash MSR 4. */
+#define MSR_GIM_HV_CRASH_P4                       UINT32_C(0x40000104)
+/** Guest crash control. */
+#define MSR_GIM_HV_CRASH_CTL                      UINT32_C(0x40000105)
+/** End of range 12. */
+#define MSR_GIM_HV_RANGE12_LAST                   MSR_GIM_HV_CRASH_CTL
+/** @} */
+
+AssertCompile(MSR_GIM_HV_RANGE0_FIRST  <= MSR_GIM_HV_RANGE0_LAST);
+AssertCompile(MSR_GIM_HV_RANGE1_FIRST  <= MSR_GIM_HV_RANGE1_LAST);
+AssertCompile(MSR_GIM_HV_RANGE2_FIRST  <= MSR_GIM_HV_RANGE2_LAST);
+AssertCompile(MSR_GIM_HV_RANGE3_FIRST  <= MSR_GIM_HV_RANGE3_LAST);
+AssertCompile(MSR_GIM_HV_RANGE4_FIRST  <= MSR_GIM_HV_RANGE4_LAST);
+AssertCompile(MSR_GIM_HV_RANGE5_FIRST  <= MSR_GIM_HV_RANGE5_LAST);
+AssertCompile(MSR_GIM_HV_RANGE6_FIRST  <= MSR_GIM_HV_RANGE6_LAST);
+AssertCompile(MSR_GIM_HV_RANGE7_FIRST  <= MSR_GIM_HV_RANGE7_LAST);
+AssertCompile(MSR_GIM_HV_RANGE8_FIRST  <= MSR_GIM_HV_RANGE8_LAST);
+AssertCompile(MSR_GIM_HV_RANGE9_FIRST  <= MSR_GIM_HV_RANGE9_LAST);
+AssertCompile(MSR_GIM_HV_RANGE10_FIRST <= MSR_GIM_HV_RANGE10_LAST);
+AssertCompile(MSR_GIM_HV_RANGE11_FIRST <= MSR_GIM_HV_RANGE11_LAST);
+
+/** @name Hyper-V MSR - Reset (MSR_GIM_HV_RESET).
+ * @{
+ */
+/** The reset enable mask. */
+#define MSR_GIM_HV_RESET_ENABLE                       RT_BIT_64(0)
+/** Whether the reset MSR is enabled. */
+#define MSR_GIM_HV_RESET_IS_ENABLED(a)                RT_BOOL((a) & MSR_GIM_HV_RESET_ENABLE)
+/** @} */
+
+/** @name Hyper-V MSR - Hypercall (MSR_GIM_HV_HYPERCALL).
+ * @{
+ */
+/** Guest-physical page frame number of the hypercall-page. */
+#define MSR_GIM_HV_HYPERCALL_GUEST_PFN(a)         ((a) >> 12)
+/** The hypercall enable mask. */
+#define MSR_GIM_HV_HYPERCALL_PAGE_ENABLE          RT_BIT_64(0)
+/** Whether the hypercall-page is enabled or not. */
+#define MSR_GIM_HV_HYPERCALL_PAGE_IS_ENABLED(a)   RT_BOOL((a) & MSR_GIM_HV_HYPERCALL_PAGE_ENABLE)
+/** @} */
+
+/** @name Hyper-V MSR - Reference TSC (MSR_GIM_HV_REF_TSC).
+ * @{
+ */
+/** Guest-physical page frame number of the TSC-page. */
+#define MSR_GIM_HV_REF_TSC_GUEST_PFN(a)           ((a) >> 12)
+/** The TSC-page enable mask. */
+#define MSR_GIM_HV_REF_TSC_ENABLE                 RT_BIT_64(0)
+/** Whether the TSC-page is enabled or not. */
+#define MSR_GIM_HV_REF_TSC_IS_ENABLED(a)          RT_BOOL((a) & MSR_GIM_HV_REF_TSC_ENABLE)
+/** @} */
+
+/** @name Hyper-V MSR - Guest crash control (MSR_GIM_HV_CRASH_CTL).
+ * @{
+ */
+/** The Crash Control notify mask. */
+#define MSR_GIM_HV_CRASH_CTL_NOTIFY               RT_BIT_64(63)
+/** @} */
+
+/** @name Hyper-V MSR - Guest OS ID (MSR_GIM_HV_GUEST_OS_ID).
+ * @{
+ */
+/** An open-source operating system. */
+#define MSR_GIM_HV_GUEST_OS_ID_IS_OPENSOURCE(a)   RT_BOOL((a) & RT_BIT_64(63))
+/** Vendor ID. */
+#define MSR_GIM_HV_GUEST_OS_ID_VENDOR(a)          (uint32_t)(((a) >> 48) & 0xfff)
+/** Guest OS variant, depending on the vendor ID.  */
+#define MSR_GIM_HV_GUEST_OS_ID_OS_VARIANT(a)      (uint32_t)(((a) >> 40) & 0xff)
+/** Guest OS major version. */
+#define MSR_GIM_HV_GUEST_OS_ID_MAJOR_VERSION(a)   (uint32_t)(((a) >> 32) & 0xff)
+/** Guest OS minor version. */
+#define MSR_GIM_HV_GUEST_OS_ID_MINOR_VERSION(a)   (uint32_t)(((a) >> 24) & 0xff)
+/** Guest OS service version (e.g. service pack number in case of Windows). */
+#define MSR_GIM_HV_GUEST_OS_ID_SERVICE_VERSION(a) (uint32_t)(((a) >> 16) & 0xff)
+/** Guest OS build number. */
+#define MSR_GIM_HV_GUEST_OS_ID_BUILD(a)           (uint32_t)((a) & 0xffff)
+/** @} */
+
+/** @name Hyper-V MSR - APIC-assist page (MSR_GIM_HV_APIC_ASSIST_PAGE).
+ * @{
+ */
+/** Guest-physical page frame number of the APIC-assist page. */
+#define MSR_GIM_HV_APICASSIST_GUEST_PFN(a)        ((a) >> 12)
+/** The APIC-assist page enable mask. */
+#define MSR_GIM_HV_APICASSIST_PAGE_ENABLE         RT_BIT_64(0)
+/** Whether the APIC-assist page is enabled or not. */
+#define MSR_GIM_HV_APICASSIST_PAGE_IS_ENABLED(a)  RT_BOOL((a) & MSR_GIM_HV_APICASSIST_PAGE_ENABLE)
+/** @} */
+
+/** @name Hyper-V MSR - Synthetic Interrupt Event Flags page
+ *        (MSR_GIM_HV_SIEFP).
+ * @{
+ */
+/** Guest-physical page frame number of the APIC-assist page. */
+#define MSR_GIM_HV_SIEF_GUEST_PFN(a)              ((a) >> 12)
+/** The SIEF enable mask. */
+#define MSR_GIM_HV_SIEF_PAGE_ENABLE               RT_BIT_64(0)
+/** Whether the SIEF page is enabled or not. */
+#define MSR_GIM_HV_SIEF_PAGE_IS_ENABLED(a)        RT_BOOL((a) & MSR_GIM_HV_SIEF_PAGE_ENABLE)
+/** @} */
+
+/** @name Hyper-V MSR - Synthetic Interrupt Control (MSR_GIM_HV_CONTROL).
+ * @{
+ */
+/** The SControl enable mask. */
+#define MSR_GIM_HV_SCONTROL_ENABLE                RT_BIT_64(0)
+/** Whether SControl is enabled or not. */
+#define MSR_GIM_HV_SCONTROL_IS_ENABLED(a)         RT_BOOL((a) & MSR_GIM_HV_SCONTROL_ENABLE)
+/** @} */
+
+/** @name Hyper-V MSR - Synthetic Timer Config (MSR_GIM_HV_STIMER_CONFIG).
+ * @{
+ */
+/** The Stimer enable mask. */
+#define MSR_GIM_HV_STIMER_ENABLE                  RT_BIT_64(0)
+/** Whether Stimer is enabled or not. */
+#define MSR_GIM_HV_STIMER_IS_ENABLED(a)           RT_BOOL((a) & MSR_GIM_HV_STIMER_ENABLE)
+/** The Stimer periodic mask. */
+#define MSR_GIM_HV_STIMER_PERIODIC                RT_BIT_64(1)
+/** Whether Stimer is enabled or not. */
+#define MSR_GIM_HV_STIMER_IS_PERIODIC(a)          RT_BOOL((a) & MSR_GIM_HV_STIMER_PERIODIC)
+/** The Stimer lazy mask. */
+#define MSR_GIM_HV_STIMER_LAZY                    RT_BIT_64(2)
+/** Whether Stimer is enabled or not. */
+#define MSR_GIM_HV_STIMER_IS_LAZY(a)              RT_BOOL((a) & MSR_GIM_HV_STIMER_LAZY)
+/** The Stimer auto-enable mask. */
+#define MSR_GIM_HV_STIMER_AUTO_ENABLE             RT_BIT_64(3)
+/** Whether Stimer is enabled or not. */
+#define MSR_GIM_HV_STIMER_IS_AUTO_ENABLED(a)      RT_BOOL((a) & MSR_GIM_HV_STIMER_AUTO_ENABLE)
+/** The Stimer SINTx mask (bits 16:19). */
+#define MSR_GIM_HV_STIMER_SINTX                   UINT64_C(0xf0000)
+/** Gets the Stimer synthetic interrupt source. */
+#define MSR_GIM_HV_STIMER_GET_SINTX(a)            (((a) >> 16) & 0xf)
+/** The Stimer valid read/write mask. */
+#define MSR_GIM_HV_STIMER_RW_VALID                (  MSR_GIM_HV_STIMER_ENABLE | MSR_GIM_HV_STIMER_PERIODIC    \
+                                                   | MSR_GIM_HV_STIMER_LAZY   | MSR_GIM_HV_STIMER_AUTO_ENABLE \
+                                                   | MSR_GIM_HV_STIMER_SINTX)
+/** @} */
+
+/**
+ * Hyper-V APIC-assist (HV_REFERENCE_TSC_PAGE) structure placed in the TSC
+ * reference page.
+ */
+typedef struct GIMHVAPICASSIST
+{
+    uint32_t fNoEoiRequired : 1;
+    uint32_t u31Reserved0   : 31;
+} GIMHVAPICASSIST;
+/** Pointer to Hyper-V reference TSC. */
+typedef GIMHVAPICASSIST *PGIMHVAPICASSIST;
+/** Pointer to a const Hyper-V reference TSC. */
+typedef GIMHVAPICASSIST const *PCGIMHVAPICASSIST;
+AssertCompileSize(GIMHVAPICASSIST, 4);
+
+/**
+ * Hypercall parameter type.
+ */
+typedef enum GIMHVHYPERCALLPARAM
+{
+    GIMHVHYPERCALLPARAM_IN = 0,
+    GIMHVHYPERCALLPARAM_OUT
+} GIMHVHYPERCALLPARAM;
+
+
+/** @name Hyper-V hypercall op codes.
+ * @{
+ */
+/** Post message to hypervisor or VMs. */
+#define GIM_HV_HYPERCALL_OP_POST_MESSAGE          0x5C
+/** Post debug data to hypervisor. */
+#define GIM_HV_HYPERCALL_OP_POST_DEBUG_DATA       0x69
+/** Retreive debug data from hypervisor. */
+#define GIM_HV_HYPERCALL_OP_RETREIVE_DEBUG_DATA   0x6A
+/** Reset debug session. */
+#define GIM_HV_HYPERCALL_OP_RESET_DEBUG_SESSION   0x6B
+/** @} */
+
+/** @name Hyper-V extended hypercall op codes.
+ * @{
+ */
+/** Query extended hypercall capabilities. */
+#define GIM_HV_EXT_HYPERCALL_OP_QUERY_CAP                0x8001
+/** Query guest physical address range that has zero'd filled memory. */
+#define GIM_HV_EXT_HYPERCALL_OP_GET_BOOT_ZEROED_MEM      0x8002
+/** @} */
+
+
+/** @name Hyper-V Extended hypercall - HvExtCallQueryCapabilities.
+ * @{
+ */
+/** Boot time zeroed pages. */
+#define GIM_HV_EXT_HYPERCALL_CAP_ZERO_MEM                       RT_BIT_64(0)
+/** Whether boot time zeroed pages capability is enabled. */
+#define GIM_HV_EXT_HYPERCALL_CAP_IS_ZERO_MEM_ENABLED(a)         RT_BOOL((a) & GIM_HV_EXT_HYPERCALL_CAP_ZERO_MEM)
+/** @} */
+
+
+/** @name Hyper-V hypercall inputs.
+ * @{
+ */
+/** The hypercall call operation code. */
+#define GIM_HV_HYPERCALL_IN_CALL_CODE(a)         ((a) & UINT64_C(0xffff))
+/** Whether it's a fast (register based) hypercall or not (memory-based). */
+#define GIM_HV_HYPERCALL_IN_IS_FAST(a)           RT_BOOL((a) & RT_BIT_64(16))
+/** Total number of reps for a rep hypercall. */
+#define GIM_HV_HYPERCALL_IN_REP_COUNT(a)         (((a) << 32) & UINT64_C(0xfff))
+/** Rep start index for a rep hypercall. */
+#define GIM_HV_HYPERCALL_IN_REP_START_IDX(a)     (((a) << 48) & UINT64_C(0xfff))
+/** Reserved bits range 1. */
+#define GIM_HV_HYPERCALL_IN_RSVD_1(a)            (((a) << 17) & UINT64_C(0x7fff))
+/** Reserved bits range 2. */
+#define GIM_HV_HYPERCALL_IN_RSVD_2(a)            (((a) << 44) & UINT64_C(0xf))
+/** Reserved bits range 3. */
+#define GIM_HV_HYPERCALL_IN_RSVD_3(a)            (((a) << 60) & UINT64_C(0x7))
+/** @} */
+
+
+/** @name Hyper-V hypercall status codes.
+ * @{
+ */
+/** Success. */
+#define GIM_HV_STATUS_SUCCESS                                        0x00
+/** Unrecognized hypercall. */
+#define GIM_HV_STATUS_INVALID_HYPERCALL_CODE                         0x02
+/** Invalid hypercall input (rep count, rsvd bits). */
+#define GIM_HV_STATUS_INVALID_HYPERCALL_INPUT                        0x03
+/** Hypercall guest-physical address not 8-byte aligned or crosses page boundary. */
+#define GIM_HV_STATUS_INVALID_ALIGNMENT                              0x04
+/** Invalid hypercall parameters. */
+#define GIM_HV_STATUS_INVALID_PARAMETER                              0x05
+/** Access denied. */
+#define GIM_HV_STATUS_ACCESS_DENIED                                  0x06
+/** The partition state not valid for specified op. */
+#define GIM_HV_STATUS_INVALID_PARTITION_STATE                        0x07
+/** The hypercall operation could not be performed. */
+#define GIM_HV_STATUS_OPERATION_DENIED                               0x08
+/** Specified partition property ID not recognized. */
+#define GIM_HV_STATUS_UNKNOWN_PROPERTY                               0x09
+/** Specified partition property value not within range. */
+#define GIM_HV_STATUS_PROPERTY_VALUE_OUT_OF_RANGE                    0x0a
+/** Insufficient memory for performing the hypercall. */
+#define GIM_HV_STATUS_INSUFFICIENT_MEMORY                            0x0b
+/** Maximum partition depth has been exceeded for the partition hierarchy. */
+#define GIM_HV_STATUS_PARTITION_TOO_DEEP                             0x0c
+/** The specified partition ID is not valid. */
+#define GIM_HV_STATUS_INVALID_PARTITION_ID                           0x0d
+/** The specified virtual processor index in invalid. */
+#define GIM_HV_STATUS_INVALID_VP_INDEX                               0x0e
+/** The specified port ID is not unique or doesn't exist. */
+#define GIM_HV_STATUS_INVALID_PORT_ID                                0x11
+/** The specified connection ID is not unique or doesn't exist. */
+#define GIM_HV_STATUS_INVALID_CONNECTION_ID                          0x12
+/** The target port doesn't have sufficient buffers for the caller to post a message. */
+#define GIM_HV_STATUS_INSUFFICIENT_BUFFERS                           0x13
+/** External interrupt not acknowledged.*/
+#define GIM_HV_STATUS_NOT_ACKNOWLEDGED                               0x14
+/** External interrupt acknowledged. */
+#define GIM_HV_STATUS_ACKNOWLEDGED                                   0x16
+/** Invalid state due to misordering Hv[Save|Restore]PartitionState. */
+#define GIM_HV_STATUS_INVALID_SAVE_RESTORE_STATE                     0x17
+/** Operation not perform due to a required feature of SynIc was disabled. */
+#define GIM_HV_STATUS_INVALID_SYNIC_STATE                            0x18
+/** Object or value already in use. */
+#define GIM_HV_STATUS_OBJECT_IN_USE                                  0x19
+/** Invalid proximity domain information. */
+#define GIM_HV_STATUS_INVALID_PROXIMITY_DOMAIN_INFO                  0x1A
+/** Attempt to retrieve data failed. */
+#define GIM_HV_STATUS_NO_DATA                                        0x1B
+/** Debug connection has not recieved any new data since the last time. */
+#define GIM_HV_STATUS_INACTIVE                                       0x1C
+/** A resource is unavailable for allocation. */
+#define GIM_HV_STATUS_NO_RESOURCES                                   0x1D
+/** A hypervisor feature is not available to the caller. */
+#define GIM_HV_STATUS_FEATURE_UNAVAILABLE                            0x1E
+/** The debug packet returned is partial due to an I/O error. */
+#define GIM_HV_STATUS_PARTIAL_PACKET                                 0x1F
+/** Processor feature SSE3 unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_SSE3_NOT_SUPPORTED                   0x20
+/** Processor feature LAHSAHF unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_LAHSAHF_NOT_SUPPORTED                0x21
+/** Processor feature SSSE3 unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_SSSE3_NOT_SUPPORTED                  0x22
+/** Processor feature SSE4.1 unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_SSE4_1_NOT_SUPPORTED                 0x23
+/** Processor feature SSE4.2 unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_SSE4_2_NOT_SUPPORTED                 0x24
+/** Processor feature SSE4A unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_SSE4A_NOT_SUPPORTED                  0x25
+/** Processor feature XOP unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_XOP_NOT_SUPPORTED                    0x26
+/** Processor feature POPCNT unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_POPCNT_NOT_SUPPORTED                 0x27
+/** Processor feature CMPXCHG16B unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_CMPXCHG16B_NOT_SUPPORTED             0x28
+/** Processor feature ALTMOVCR8 unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_ALTMOVCR8_NOT_SUPPORTED              0x29
+/** Processor feature LZCNT unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_LZCNT_NOT_SUPPORTED                  0x2A
+/** Processor feature misaligned SSE unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_MISALIGNED_SSE_NOT_SUPPORTED         0x2B
+/** Processor feature MMX extensions unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_MMX_EXT_NOT_SUPPORTED                0x2C
+/** Processor feature 3DNow! unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_3DNOW_NOT_SUPPORTED                  0x2D
+/** Processor feature Extended 3DNow! unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_EXTENDED_3DNOW_NOT_SUPPORTED         0x2E
+/** Processor feature 1GB large page unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_PAGE_1GB_NOT_SUPPORTED               0x2F
+/** Processor cache line flush size incompatible. */
+#define GIM_HV_STATUS_PROC_CACHE_LINE_FLUSH_SIZE_INCOMPATIBLE        0x30
+/** Processor feature XSAVE unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_XSAVE_NOT_SUPPORTED                  0x31
+/** Processor feature XSAVEOPT unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_XSAVEOPT_NOT_SUPPORTED               0x32
+/** The specified buffer was too small for all requested data. */
+#define GIM_HV_STATUS_INSUFFICIENT_BUFFER                            0x33
+/** Processor feature XSAVEOPT unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_XSAVE_AVX_NOT_SUPPORTED              0x34
+/** Processor feature XSAVEOPT unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_XSAVE_FEAT_NOT_SUPPORTED             0x35   /** Huh, isn't this same as 0x31? */
+/** Processor feature XSAVEOPT unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_PAGE_XSAVE_SAVE_AREA_INCOMPATIBLE    0x36
+/** Processor architecture unsupoorted. */
+#define GIM_HV_STATUS_INCOMPATIBLE_PROCESSOR                         0x37
+/** Max. domains for platform I/O remapping reached. */
+#define GIM_HV_STATUS_INSUFFICIENT_DEVICE_DOMAINS                    0x38
+/** Processor feature AES unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_AES_NOT_SUPPORTED                    0x39
+/** Processor feature PCMULQDQ unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_PCMULQDQ_NOT_SUPPORTED               0x3A
+/** Processor feature XSAVE features unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_XSAVE_FEATURES_INCOMPATIBLE          0x3B
+/** Generic CPUID validation error. */
+#define GIM_HV_STATUS_CPUID_FEAT_VALIDATION_ERROR                    0x3C
+/** XSAVE CPUID validation error. */
+#define GIM_HV_STATUS_CPUID_XSAVE_FEAT_VALIDATION_ERROR              0x3D
+/** Processor startup timed out. */
+#define GIM_HV_STATUS_PROCESSOR_STARTUP_TIMEOUT                      0x3E
+/** SMX enabled by the BIOS. */
+#define GIM_HV_STATUS_SMX_ENABLED                                    0x3F
+/** Processor feature PCID unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_PCID_NOT_SUPPORTED                   0x40
+/** Invalid LP index. */
+#define GIM_HV_STATUS_INVALID_LP_INDEX                               0x41
+/** Processor feature PCID unsupported. */
+#define GIM_HV_STATUS_FEAT_FMA4_NOT_SUPPORTED                        0x42
+/** Processor feature PCID unsupported. */
+#define GIM_HV_STATUS_FEAT_F16C_NOT_SUPPORTED                        0x43
+/** Processor feature PCID unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_RDRAND_NOT_SUPPORTED                 0x44
+/** Processor feature RDWRFSGS unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_RDWRFSGS_NOT_SUPPORTED               0x45
+/** Processor feature SMEP unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_SMEP_NOT_SUPPORTED                   0x46
+/** Processor feature enhanced fast string unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_ENHANCED_FAST_STRING_NOT_SUPPORTED   0x47
+/** Processor feature MOVBE unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_MOVBE_NOT_SUPPORTED                  0x48
+/** Processor feature BMI1 unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_BMI1_NOT_SUPPORTED                   0x49
+/** Processor feature BMI2 unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_BMI2_NOT_SUPPORTED                   0x4A
+/** Processor feature HLE unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_HLE_NOT_SUPPORTED                    0x4B
+/** Processor feature RTM unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_RTM_NOT_SUPPORTED                    0x4C
+/** Processor feature XSAVE FMA unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_XSAVE_FMA_NOT_SUPPORTED              0x4D
+/** Processor feature XSAVE AVX2 unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_XSAVE_AVX2_NOT_SUPPORTED             0x4E
+/** Processor feature NPIEP1 unsupported. */
+#define GIM_HV_STATUS_PROC_FEAT_NPIEP1_NOT_SUPPORTED                 0x4F
+/** @} */
+
+
+/** @name Hyper-V MSR - Debug control (MSR_GIM_HV_SYNTH_DEBUG_CONTROL).
+ * @{
+ */
+/** Perform debug write. */
+#define MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_WRITE(a)     RT_BOOL((a) & RT_BIT_64(0))
+/** Perform debug read. */
+#define MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_READ(a)      RT_BOOL((a) & RT_BIT_64(1))
+/** Returns length of the debug write buffer. */
+#define MSR_GIM_HV_SYNTH_DEBUG_CONTROL_W_LEN(a)        (((a) & UINT64_C(0xffff0000)) >> 16)
+/** @} */
+
+
+/** @name Hyper-V MSR - Debug status (MSR_GIM_HV_SYNTH_DEBUG_STATUS).
+ * @{
+ */
+/** Debug send buffer operation success. */
+#define MSR_GIM_HV_SYNTH_DEBUG_STATUS_W_SUCCESS        RT_BIT_64(0)
+/** Debug receive buffer operation success. */
+#define MSR_GIM_HV_SYNTH_DEBUG_STATUS_R_SUCCESS        RT_BIT_64(2)
+/** Debug connection was reset. */
+#define MSR_GIM_HV_SYNTH_DEBUG_STATUS_CONN_RESET       RT_BIT_64(3)
+/** @} */
+
+
+/** @name Hyper-V MSR - synthetic interrupt (MSR_GIM_HV_SINTx).
+ * @{
+ */
+/** The interrupt masked mask. */
+#define MSR_GIM_HV_SINT_MASKED                         RT_BIT_64(16)
+/** Whether the interrupt source is masked. */
+#define MSR_GIM_HV_SINT_IS_MASKED(a)                   RT_BOOL((a) & MSR_GIM_HV_SINT_MASKED)
+/** Gets the interrupt vector. */
+#define MSR_GIM_HV_SINT_GET_VECTOR(a)                  ((a) & UINT64_C(0xff))
+/** The AutoEoi mask. */
+#define MSR_GIM_HV_SINT_AUTOEOI                        RT_BIT_64(17)
+/** Gets whether AutoEoi is enabled for the synthetic interrupt. */
+#define MSR_GIM_HV_SINT_IS_AUTOEOI(a)                  RT_BOOL((a) & MSR_GIM_HV_SINT_AUTOEOI)
+/** @} */
+
+
+/** @name Hyper-V MSR - synthetic interrupt message page (MSR_GIM_HV_SIMP).
+ * @{
+ */
+/** The SIMP enable mask. */
+#define MSR_GIM_HV_SIMP_ENABLE                         RT_BIT_64(0)
+/** Whether the SIMP is enabled. */
+#define MSR_GIM_HV_SIMP_IS_ENABLED(a)                  RT_BOOL((a) & MSR_GIM_HV_SIMP_ENABLE)
+/** The SIMP guest-physical address. */
+#define MSR_GIM_HV_SIMP_GPA(a)                         ((a) & UINT64_C(0xfffffffffffff000))
+/** @} */
+
+
+/** @name Hyper-V hypercall debug options.
+ * @{ */
+/** Maximum debug data payload size in bytes. */
+#define GIM_HV_DEBUG_MAX_DATA_SIZE                4088
+
+/** The undocumented bit for MSR_GIM_HV_DEBUG_OPTIONS_MSR that makes it all
+ *  work. */
+#define GIM_HV_DEBUG_OPTIONS_USE_HYPERCALLS       RT_BIT(2)
+
+/** Guest will perform the HvPostDebugData hypercall until completion. */
+#define GIM_HV_DEBUG_POST_LOOP                    RT_BIT_32(0)
+/** Mask of valid HvPostDebugData options. */
+#define GIM_HV_DEBUG_POST_OPTIONS_MASK            RT_BIT_32(0)
+
+/** Guest will perform the HvRetrieveDebugData hypercall until completion. */
+#define GIM_HV_DEBUG_RETREIVE_LOOP                RT_BIT_32(0)
+/** Guest checks if any global debug session is active. */
+#define GIM_HV_DEBUG_RETREIVE_TEST_ACTIVITY       RT_BIT_32(1)
+/** Mask of valid HvRetrieveDebugData options. */
+#define GIM_HV_DEBUG_RETREIVE_OPTIONS_MASK        RT_BIT_32(0) | RT_BIT_32(1)
+
+/** Guest requests purging of incoming debug data. */
+#define GIM_HV_DEBUG_PURGE_INCOMING_DATA          RT_BIT_32(0)
+/** Guest requests purging of outgoing debug data. */
+#define GIM_HV_DEBUG_PURGE_OUTGOING_DATA          RT_BIT_32(1)
+/** @} */
+
+
+/** @name VMBus.
+ *  These are just arbitrary definitions made up by Microsoft without
+ *  any publicly available specification behind it.
+ * @{ */
+/** VMBus connection ID. */
+#define GIM_HV_VMBUS_MSG_CONNECTION_ID            1
+/** VMBus synthetic interrupt source (see VMBUS_MESSAGE_SINT in linux
+ *  sources). */
+#define GIM_HV_VMBUS_MSG_SINT                     2
+/** @} */
+
+/** @name SynIC.
+ *  Synthetic Interrupt Controller definitions.
+ * @{ */
+/** SynIC version register. */
+#define GIM_HV_SVERSION                           1
+/** Number of synthetic interrupt sources (warning, fixed in saved-states!). */
+#define GIM_HV_SINT_COUNT                         16
+/** Lowest valid vector for synthetic interrupt. */
+#define GIM_HV_SINT_VECTOR_VALID_MIN              16
+/** Highest valid vector for synthetic interrupt. */
+#define GIM_HV_SINT_VECTOR_VALID_MAX              255
+/** Number of synthetic timers. */
+#define GIM_HV_STIMER_COUNT                       4
+/** @} */
+
+/** @name Hyper-V synthetic interrupt message type.
+ * See 14.8.2 "SynIC Message Types"
+ * @{
+ */
+typedef enum GIMHVMSGTYPE
+{
+    GIMHVMSGTYPE_NONE                 = 0,              /* Common messages */
+    GIMHVMSGTYPE_VMBUS                = 1,              /* Guest messages */
+    GIMHVMSGTYPE_UNMAPPEDGPA          = 0x80000000,     /* Hypervisor messages */
+    GIMHVMSGTYPE_GPAINTERCEPT         = 0x80000001,
+    GIMHVMSGTYPE_TIMEREXPIRED         = 0x80000010,
+    GIMHVMSGTYPE_INVALIDVPREGVAL      = 0x80000020,
+    GIMHVMSGTYPE_UNRECOVERABLEXCPT    = 0x80000021,
+    GIMHVMSGTYPE_UNSUPPORTEDFEAT      = 0x80000022,
+    GIMHVMSGTYPE_APICEOI              = 0x80000030,
+    GIMHVMSGTYPE_X64LEGACYFPERROR     = 0x80000031,
+    GIMHVMSGTYPE_EVENTLOGBUFSCOMPLETE = 0x80000040,
+    GIMHVMSGTYPE_X64IOPORTINTERCEPT   = 0x80010000,
+    GIMHVMSGTYPE_X64MSRINTERCEPT      = 0x80010001,
+    GIMHVMSGTYPE_X64CPUIDINTERCEPT    = 0x80010002,
+    GIMHVMSGTYPE_X64XCPTINTERCEPT     = 0x80010003
+} GIMHVMSGTYPE;
+AssertCompileSize(GIMHVMSGTYPE, 4);
+/** @} */
+
+
+/** @name Hyper-V synthetic interrupt message format.
+ * @{ */
+#define GIM_HV_MSG_SIZE                           256
+#define GIM_HV_MSG_MAX_PAYLOAD_SIZE               240
+#define GIM_HV_MSG_MAX_PAYLOAD_UNITS               30
+
+/**
+ * Synthetic interrupt message flags.
+ */
+typedef union GIMHVMSGFLAGS
+{
+    struct
+    {
+        uint8_t  u1Pending  : 1;
+        uint8_t  u7Reserved : 7;
+    } n;
+    uint8_t u;
+} GIMHVMSGFLAGS;
+AssertCompileSize(GIMHVMSGFLAGS, sizeof(uint8_t));
+
+/**
+ * Synthetic interrupt message header.
+ *
+ * @remarks The layout of this structure differs from
+ *          the Hyper-V spec. Aug 8, 2013 v4.0a. Layout
+ *          in accordance w/ VMBus client expectations.
+ */
+typedef struct GIMHVMSGHDR
+{
+    GIMHVMSGTYPE    enmMessageType;
+    uint8_t         cbPayload;
+    GIMHVMSGFLAGS   MessageFlags;
+    uint16_t        uRsvd;
+    union
+    {
+        uint64_t    uOriginatorId;
+        uint64_t    uPartitionId;
+        uint64_t    uPortId;
+    } msgid;
+} GIMHVMSGHDR;
+/** Pointer to a synthetic interrupt message header. */
+typedef GIMHVMSGHDR *PGIMHVMSGHDR;
+AssertCompileMemberOffset(GIMHVMSGHDR, cbPayload,    4);
+AssertCompileMemberOffset(GIMHVMSGHDR, MessageFlags, 5);
+AssertCompileMemberOffset(GIMHVMSGHDR, msgid,        8);
+AssertCompileSize(GIMHVMSGHDR, GIM_HV_MSG_SIZE - GIM_HV_MSG_MAX_PAYLOAD_SIZE);
+
+/**
+ * Synthetic interrupt message.
+ */
+typedef struct GIMHVMSG
+{
+    GIMHVMSGHDR     MsgHdr;
+    uint64_t        aPayload[GIM_HV_MSG_MAX_PAYLOAD_UNITS];
+} GIMHVMSG;
+/** Pointer to a synthetic interrupt message. */
+typedef GIMHVMSG *PGIMHVMSG;
+AssertCompileSize(GIMHVMSG, GIM_HV_MSG_SIZE);
+/** @} */
+
+
+/** @name Hyper-V hypercall parameters.
+ * @{ */
+/**
+ * HvPostMessage hypercall input.
+ */
+typedef struct GIMHVPOSTMESSAGEIN
+{
+    uint32_t      uConnectionId;
+    uint32_t      uPadding;
+    GIMHVMSGTYPE  enmMessageType;
+    uint32_t      cbPayload;
+} GIMHVPOSTMESSAGEIN;
+/** Pointer to a HvPostMessage input struct. */
+typedef GIMHVPOSTMESSAGEIN *PGIMHVPOSTMESSAGEIN;
+AssertCompileSize(GIMHVPOSTMESSAGEIN, 16);
+
+/**
+ * HvResetDebugData hypercall input.
+ */
+typedef struct GIMHVDEBUGRESETIN
+{
+    uint32_t fFlags;
+    uint32_t uPadding;
+} GIMHVDEBUGRESETIN;
+/** Pointer to a HvResetDebugData input struct. */
+typedef GIMHVDEBUGRESETIN *PGIMHVDEBUGRESETIN;
+AssertCompileSize(GIMHVDEBUGRESETIN, 8);
+
+/**
+ * HvPostDebugData hypercall input.
+ */
+typedef struct GIMHVDEBUGPOSTIN
+{
+    uint32_t cbWrite;
+    uint32_t fFlags;
+} GIMHVDEBUGPOSTIN;
+/** Pointer to a HvPostDebugData input struct. */
+typedef GIMHVDEBUGPOSTIN *PGIMHVDEBUGPOSTIN;
+AssertCompileSize(GIMHVDEBUGPOSTIN, 8);
+
+/**
+ * HvPostDebugData hypercall output.
+ */
+typedef struct GIMHVDEBUGPOSTOUT
+{
+    uint32_t cbPending;
+    uint32_t uPadding;
+} GIMHVDEBUGPOSTOUT;
+/** Pointer to a HvPostDebugData output struct. */
+typedef GIMHVDEBUGPOSTOUT *PGIMHVDEBUGPOSTOUT;
+AssertCompileSize(GIMHVDEBUGPOSTOUT, 8);
+
+/**
+ * HvRetrieveDebugData hypercall input.
+ */
+typedef struct GIMHVDEBUGRETRIEVEIN
+{
+    uint32_t cbRead;
+    uint32_t fFlags;
+    uint64_t u64Timeout;
+} GIMHVDEBUGRETRIEVEIN;
+/** Pointer to a HvRetrieveDebugData input struct. */
+typedef GIMHVDEBUGRETRIEVEIN *PGIMHVDEBUGRETRIEVEIN;
+AssertCompileSize(GIMHVDEBUGRETRIEVEIN, 16);
+
+/**
+ * HvRetriveDebugData hypercall output.
+ */
+typedef struct GIMHVDEBUGRETRIEVEOUT
+{
+    uint32_t cbRead;
+    uint32_t cbRemaining;
+} GIMHVDEBUGRETRIEVEOUT;
+/** Pointer to a HvRetrieveDebugData output struct. */
+typedef GIMHVDEBUGRETRIEVEOUT *PGIMHVDEBUGRETRIEVEOUT;
+AssertCompileSize(GIMHVDEBUGRETRIEVEOUT, 8);
+
+/**
+ * HvExtCallQueryCapabilities hypercall output.
+ */
+typedef struct GIMHVEXTQUERYCAP
+{
+    uint64_t fCapabilities;
+} GIMHVEXTQUERYCAP;
+/** Pointer to a HvExtCallQueryCapabilities output struct. */
+typedef GIMHVEXTQUERYCAP *PGIMHVEXTQUERYCAP;
+AssertCompileSize(GIMHVEXTQUERYCAP, 8);
+
+/**
+ * HvExtCallGetBootZeroedMemory hypercall output.
+ */
+typedef struct GIMHVEXTGETBOOTZEROMEM
+{
+    RTGCPHYS GCPhysStart;
+    uint64_t cPages;
+} GIMHVEXTGETBOOTZEROMEM;
+/** Pointer to a HvExtCallGetBootZeroedMemory output struct. */
+typedef GIMHVEXTGETBOOTZEROMEM *PGIMHVEXTGETBOOTZEROMEM;
+AssertCompileSize(GIMHVEXTGETBOOTZEROMEM, 16);
+/** @} */
+
+
+/** Hyper-V page size.  */
+#define GIM_HV_PAGE_SIZE                          4096
+/** Hyper-V page shift. */
+#define GIM_HV_PAGE_SHIFT                         12
+
+/** Microsoft Hyper-V vendor signature. */
+#define GIM_HV_VENDOR_MICROSOFT                   "Microsoft Hv"
+
+/**
+ * MMIO2 region indices.
+ */
+/** The hypercall page region. */
+#define GIM_HV_HYPERCALL_PAGE_REGION_IDX          UINT8_C(0)
+/** The TSC page region. */
+#define GIM_HV_REF_TSC_PAGE_REGION_IDX            UINT8_C(1)
+/** The maximum region index (must be <= UINT8_MAX). */
+#define GIM_HV_REGION_IDX_MAX                     GIM_HV_REF_TSC_PAGE_REGION_IDX
+
+/**
+ * Hyper-V TSC (HV_REFERENCE_TSC_PAGE) structure placed in the TSC reference
+ * page.
+ */
+typedef struct GIMHVREFTSC
+{
+    uint32_t u32TscSequence;
+    uint32_t uReserved0;
+    uint64_t u64TscScale;
+    int64_t  i64TscOffset;
+} GIMHVTSCPAGE;
+/** Pointer to Hyper-V reference TSC. */
+typedef GIMHVREFTSC *PGIMHVREFTSC;
+/** Pointer to a const Hyper-V reference TSC. */
+typedef GIMHVREFTSC const *PCGIMHVREFTSC;
+
+/**
+ * Type of the next reply to be sent to the debug connection of the guest.
+ *
+ * @remarks This is saved as part of saved-state, so don't re-order or
+ *          alter the size!
+ */
+typedef enum GIMHVDEBUGREPLY
+{
+    /** Send UDP packet. */
+    GIMHVDEBUGREPLY_UDP = 0,
+    /** Send DHCP offer for DHCP discover. */
+    GIMHVDEBUGREPLY_DHCP_OFFER,
+    /** DHCP offer sent. */
+    GIMHVDEBUGREPLY_DHCP_OFFER_SENT,
+    /** Send DHCP acknowledgement for DHCP request. */
+    GIMHVDEBUGREPLY_DHCP_ACK,
+    /** DHCP acknowledgement sent.  */
+    GIMHVDEBUGREPLY_DHCP_ACK_SENT,
+    /** Sent ARP reply. */
+    GIMHVDEBUGREPLY_ARP_REPLY,
+    /** ARP reply sent. */
+    GIMHVDEBUGREPLY_ARP_REPLY_SENT,
+    /** Customary 32-bit type hack. */
+    GIMHVDEBUGREPLY_32BIT_HACK = 0x7fffffff
+} GIMHVDEBUGREPLY;
+AssertCompileSize(GIMHVDEBUGREPLY, sizeof(uint32_t));
+
+/**
+ * GIM Hyper-V VM instance data.
+ * Changes to this must checked against the padding of the gim union in VM!
+ */
+typedef struct GIMHV
+{
+    /** @name Primary MSRs.
+     * @{ */
+    /** Guest OS identity MSR. */
+    uint64_t                    u64GuestOsIdMsr;
+    /** Hypercall MSR. */
+    uint64_t                    u64HypercallMsr;
+    /** Reference TSC page MSR. */
+    uint64_t                    u64TscPageMsr;
+    /** @}  */
+
+    /** @name CPUID features.
+     * @{ */
+    /** Basic features. */
+    uint32_t                    uBaseFeat;
+    /** Partition flags. */
+    uint32_t                    uPartFlags;
+    /** Power management. */
+    uint32_t                    uPowMgmtFeat;
+    /** Miscellaneous. */
+    uint32_t                    uMiscFeat;
+    /** Hypervisor hints to the guest. */
+    uint32_t                    uHyperHints;
+    /** Hypervisor capabilities. */
+    uint32_t                    uHyperCaps;
+    /** @} */
+
+    /** @name Guest Crash MSRs.
+     * @{
+     */
+    /** Guest crash control MSR. */
+    uint64_t                    uCrashCtlMsr;
+    /** Guest crash parameter 0 MSR. */
+    uint64_t                    uCrashP0Msr;
+    /** Guest crash parameter 1 MSR. */
+    uint64_t                    uCrashP1Msr;
+    /** Guest crash parameter 2 MSR. */
+    uint64_t                    uCrashP2Msr;
+    /** Guest crash parameter 3 MSR. */
+    uint64_t                    uCrashP3Msr;
+    /** Guest crash parameter 4 MSR. */
+    uint64_t                    uCrashP4Msr;
+    /** @} */
+
+    /** @name Time management.
+     * @{ */
+    /** Per-VM R0 Spinlock for protecting EMT writes to the TSC page. */
+    RTSPINLOCK                  hSpinlockR0;
+    /** The TSC frequency (in HZ) reported to the guest. */
+    uint64_t                    cTscTicksPerSecond;
+    /** @} */
+
+    /** @name Hypercalls.
+     * @{ */
+    /** Guest address of the hypercall input parameter page. */
+    RTGCPHYS                    GCPhysHypercallIn;
+    /** Guest address of the hypercall output parameter page. */
+    RTGCPHYS                    GCPhysHypercallOut;
+    /** Pointer to the hypercall input parameter page - R3. */
+    R3PTRTYPE(uint8_t *)        pbHypercallIn;
+    /** Pointer to the hypercall output parameter page - R3. */
+    R3PTRTYPE(uint8_t *)        pbHypercallOut;
+    /** @} */
+
+    /** @name Guest debugging.
+     * @{ */
+    /** Whether we're posing as the Microsoft vendor. */
+    bool                        fIsVendorMsHv;
+    /** Whether we're posing as the Microsoft virtualization service. */
+    bool                        fIsInterfaceVs;
+    /** Whether debugging support is enabled. */
+    bool                        fDbgEnabled;
+    /** Whether we should suggest a hypercall-based debug interface to the guest. */
+    bool                        fDbgHypercallInterface;
+    bool                        afAlignment0[4];
+    /** The action to take while sending replies. */
+    GIMHVDEBUGREPLY             enmDbgReply;
+    /** The IP address chosen by/assigned to the guest. */
+    RTNETADDRIPV4               DbgGuestIp4Addr;
+    /** Transaction ID for the BOOTP+DHCP sequence. */
+    uint32_t                    uDbgBootpXId;
+    /** The source UDP port used by the guest while sending debug packets. */
+    uint16_t                    uUdpGuestSrcPort;
+    /** The destination UDP port used by the guest while sending debug packets. */
+    uint16_t                    uUdpGuestDstPort;
+    /** Debug send buffer MSR. */
+    uint64_t                    uDbgSendBufferMsr;
+    /** Debug receive buffer MSR. */
+    uint64_t                    uDbgRecvBufferMsr;
+    /** Debug pending buffer MSR. */
+    uint64_t                    uDbgPendingBufferMsr;
+    /** Debug status MSR. */
+    uint64_t                    uDbgStatusMsr;
+    /** Intermediate debug I/O buffer. */
+    R3PTRTYPE(void *)           pvDbgBuffer;
+    R3PTRTYPE(void *)           pvAlignment0;
+    /** @} */
+
+    /** Array of MMIO2 regions. */
+    GIMMMIO2REGION              aMmio2Regions[GIM_HV_REGION_IDX_MAX + 1];
+} GIMHV;
+/** Pointer to per-VM GIM Hyper-V instance data. */
+typedef GIMHV *PGIMHV;
+/** Pointer to const per-VM GIM Hyper-V instance data. */
+typedef GIMHV const *PCGIMHV;
+AssertCompileMemberAlignment(GIMHV, aMmio2Regions, 8);
+AssertCompileMemberAlignment(GIMHV, hSpinlockR0, sizeof(uintptr_t));
+
+/**
+ * Hyper-V per-VCPU synthetic timer.
+ */
+typedef struct GIMHVSTIMER
+{
+    /** Synthetic timer object - R0 ptr. */
+    PTMTIMERR0                  pTimerR0;
+    /** Synthetic timer object - R3 ptr. */
+    PTMTIMERR3                  pTimerR3;
+    /** Virtual CPU ID this timer belongs to (for reverse mapping). */
+    VMCPUID                     idCpu;
+    /** The index of this timer in the auStimers array (for reverse mapping). */
+    uint32_t                    idxStimer;
+    /** Synthetic timer config MSR. */
+    uint64_t                    uStimerConfigMsr;
+    /** Synthetic timer count MSR. */
+    uint64_t                    uStimerCountMsr;
+    /** Timer description. */
+    char                        szTimerDesc[24];
+
+} GIMHVSTIMER;
+/** Pointer to per-VCPU Hyper-V synthetic timer. */
+typedef GIMHVSTIMER *PGIMHVSTIMER;
+/** Pointer to a const per-VCPU Hyper-V synthetic timer. */
+typedef GIMHVSTIMER const *PCGIMHVSTIMER;
+AssertCompileSizeAlignment(GIMHVSTIMER, 8);
+
+/**
+ * Hyper-V VCPU instance data.
+ * Changes to this must checked against the padding of the gim union in VMCPU!
+ */
+typedef struct GIMHVCPU
+{
+    /** @name Synthetic interrupt MSRs.
+     * @{ */
+    /** Synthetic interrupt message page MSR. */
+    uint64_t                    uSimpMsr;
+    /** Interrupt source MSRs. */
+    uint64_t                    auSintMsrs[GIM_HV_SINT_COUNT];
+    /** Synethtic interrupt events flag page MSR. */
+    uint64_t                    uSiefpMsr;
+    /** APIC-assist page MSR. */
+    uint64_t                    uApicAssistPageMsr;
+    /** Synthetic interrupt control MSR. */
+    uint64_t                    uSControlMsr;
+    /** Synthetic timers. */
+    GIMHVSTIMER                 aStimers[GIM_HV_STIMER_COUNT];
+    /** @} */
+
+    /** @name Statistics.
+     * @{ */
+    STAMCOUNTER                 aStatStimerFired[GIM_HV_STIMER_COUNT];
+    /** @} */
+} GIMHVCPU;
+/** Pointer to per-VCPU GIM Hyper-V instance data. */
+typedef GIMHVCPU *PGIMHVCPU;
+/** Pointer to const per-VCPU GIM Hyper-V instance data. */
+typedef GIMHVCPU const *PCGIMHVCPU;
+
+
+RT_C_DECLS_BEGIN
+
+#ifdef IN_RING0
+VMMR0_INT_DECL(int)             gimR0HvInitVM(PVMCC pVM);
+VMMR0_INT_DECL(int)             gimR0HvTermVM(PVMCC pVM);
+VMMR0_INT_DECL(int)             gimR0HvUpdateParavirtTsc(PVMCC pVM, uint64_t u64Offset);
+#endif /* IN_RING0 */
+
+#ifdef IN_RING3
+VMMR3_INT_DECL(int)             gimR3HvInit(PVM pVM, PCFGMNODE pGimCfg);
+VMMR3_INT_DECL(int)             gimR3HvInitCompleted(PVM pVM);
+VMMR3_INT_DECL(int)             gimR3HvTerm(PVM pVM);
+VMMR3_INT_DECL(void)            gimR3HvRelocate(PVM pVM, RTGCINTPTR offDelta);
+VMMR3_INT_DECL(void)            gimR3HvReset(PVM pVM);
+VMMR3_INT_DECL(int)             gimR3HvSave(PVM pVM, PSSMHANDLE pSSM);
+VMMR3_INT_DECL(int)             gimR3HvLoad(PVM pVM, PSSMHANDLE pSSM);
+VMMR3_INT_DECL(int)             gimR3HvLoadDone(PVM pVM, PSSMHANDLE pSSM);
+VMMR3_INT_DECL(int)             gimR3HvGetDebugSetup(PVM pVM, PGIMDEBUGSETUP pDbgSetup);
+
+VMMR3_INT_DECL(int)             gimR3HvDisableSiefPage(PVMCPU pVCpu);
+VMMR3_INT_DECL(int)             gimR3HvEnableSiefPage(PVMCPU pVCpu, RTGCPHYS GCPhysSiefPage);
+VMMR3_INT_DECL(int)             gimR3HvEnableSimPage(PVMCPU pVCpu, RTGCPHYS GCPhysSimPage);
+VMMR3_INT_DECL(int)             gimR3HvDisableSimPage(PVMCPU pVCpu);
+VMMR3_INT_DECL(int)             gimR3HvDisableApicAssistPage(PVMCPU pVCpu);
+VMMR3_INT_DECL(int)             gimR3HvEnableApicAssistPage(PVMCPU pVCpu, RTGCPHYS GCPhysTscPage);
+VMMR3_INT_DECL(int)             gimR3HvDisableTscPage(PVM pVM);
+VMMR3_INT_DECL(int)             gimR3HvEnableTscPage(PVM pVM, RTGCPHYS GCPhysTscPage, bool fUseThisTscSeq, uint32_t uTscSeq);
+VMMR3_INT_DECL(int)             gimR3HvDisableHypercallPage(PVM pVM);
+VMMR3_INT_DECL(int)             gimR3HvEnableHypercallPage(PVM pVM, RTGCPHYS GCPhysHypercallPage);
+
+VMMR3_INT_DECL(int)             gimR3HvHypercallPostDebugData(PVM pVM, int *prcHv);
+VMMR3_INT_DECL(int)             gimR3HvHypercallRetrieveDebugData(PVM pVM, int *prcHv);
+VMMR3_INT_DECL(int)             gimR3HvDebugWrite(PVM pVM, void *pvData, uint32_t cbWrite, uint32_t *pcbWritten, bool fUdpPkt);
+VMMR3_INT_DECL(int)             gimR3HvDebugRead(PVM pVM, void *pvBuf, uint32_t cbBuf, uint32_t cbRead, uint32_t *pcbRead,
+                                                 uint32_t cMsTimeout, bool fUdpPkt);
+VMMR3_INT_DECL(int)             gimR3HvHypercallExtQueryCap(PVM pVM, int *prcHv);
+VMMR3_INT_DECL(int)             gimR3HvHypercallExtGetBootZeroedMem(PVM pVM, int *prcHv);
+
+#endif /* IN_RING3 */
+
+VMM_INT_DECL(PGIMMMIO2REGION)   gimHvGetMmio2Regions(PVM pVM, uint32_t *pcRegions);
+VMM_INT_DECL(bool)              gimHvIsParavirtTscEnabled(PVM pVM);
+VMM_INT_DECL(bool)              gimHvAreHypercallsEnabled(PCVM pVM);
+VMM_INT_DECL(bool)              gimHvShouldTrapXcptUD(PVMCPU pVCpu);
+VMM_INT_DECL(VBOXSTRICTRC)      gimHvXcptUD(PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr);
+VMM_INT_DECL(VBOXSTRICTRC)      gimHvHypercall(PVMCPUCC pVCpu, PCPUMCTX pCtx);
+VMM_INT_DECL(VBOXSTRICTRC)      gimHvHypercallEx(PVMCPUCC pVCpu, PCPUMCTX pCtx, unsigned uDisOpcode, uint8_t cbInstr);
+VMM_INT_DECL(VBOXSTRICTRC)      gimHvReadMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue);
+VMM_INT_DECL(VBOXSTRICTRC)      gimHvWriteMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uRawValue);
+
+VMM_INT_DECL(void)              gimHvStartStimer(PVMCPUCC pVCpu, PCGIMHVSTIMER pHvStimer);
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_GIMHvInternal_h */
+
diff --git a/src/VBox/VMM/include/GIMInternal.h b/src/VBox/VMM/include/GIMInternal.h
new file mode 100644
index 00000000..cdd4fa1a
--- /dev/null
+++ b/src/VBox/VMM/include/GIMInternal.h
@@ -0,0 +1,123 @@
+/* $Id: GIMInternal.h $ */
+/** @file
+ * GIM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2014-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_GIMInternal_h
+#define VMM_INCLUDED_SRC_include_GIMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/pgm.h>
+#include "GIMHvInternal.h"
+#include "GIMKvmInternal.h"
+#include "GIMMinimalInternal.h"
+
+RT_C_DECLS_BEGIN
+
+/** @defgroup grp_gim_int       Internal
+ * @ingroup grp_gim
+ * @internal
+ * @{
+ */
+
+/** The saved state version. */
+#define GIM_SAVED_STATE_VERSION         1
+
+/**
+ * GIM VM Instance data.
+ */
+typedef struct GIM
+{
+    /** The provider that is active for this VM. */
+    GIMPROVIDERID                    enmProviderId;
+    /** The interface implementation version. */
+    uint32_t                         u32Version;
+
+    /** Physical access handler type for semi-read-only MMIO2 memory. Lazy creation. */
+    PGMPHYSHANDLERTYPE              hSemiReadOnlyMmio2Handler;
+    /** Alignment padding. */
+    uint32_t                        u32Padding;
+
+    /** Pointer to the GIM device - R3 ptr. */
+    R3PTRTYPE(PPDMDEVINS)            pDevInsR3;
+    /** The debug struct - R3 ptr. */
+    R3PTRTYPE(PGIMDEBUG)             pDbgR3;
+
+    /** The provider specific data. */
+    union
+    {
+        GIMHV  Hv;
+        GIMKVM Kvm;
+    } u;
+
+    /** Number of hypercalls initiated. */
+    STAMCOUNTER                      StatHypercalls;
+    /** Debug packets sent. */
+    STAMCOUNTER                      StatDbgXmit;
+    /** Debug bytes sent. */
+    STAMCOUNTER                      StatDbgXmitBytes;
+    /** Debug packets received. */
+    STAMCOUNTER                      StatDbgRecv;
+    /** Debug bytes received. */
+    STAMCOUNTER                      StatDbgRecvBytes;
+} GIM;
+/** Pointer to GIM VM instance data. */
+typedef GIM *PGIM;
+
+/**
+ * GIM VMCPU Instance data.
+ */
+typedef struct GIMCPU
+{
+    union
+    {
+        GIMKVMCPU KvmCpu;
+        GIMHVCPU  HvCpu;
+    } u;
+} GIMCPU;
+/** Pointer to GIM VMCPU instance data. */
+typedef GIMCPU *PGIMCPU;
+
+/**
+ * Callback when a debug buffer read has completed and before signalling the next
+ * read.
+ *
+ * @param   pVM             The cross context VM structure.
+ */
+typedef DECLCALLBACK(void) FNGIMDEBUGBUFREADCOMPLETED(PVM pVM);
+/** Pointer to GIM debug buffer read completion callback. */
+typedef FNGIMDEBUGBUFREADCOMPLETED *PFNGIMDEBUGBUFREADCOMPLETED;
+
+#ifdef IN_RING3
+#if 0
+VMMR3_INT_DECL(int)           gimR3Mmio2Unmap(PVM pVM, PGIMMMIO2REGION pRegion);
+VMMR3_INT_DECL(int)           gimR3Mmio2Map(PVM pVM, PGIMMMIO2REGION pRegion, RTGCPHYS GCPhysRegion);
+VMMR3_INT_DECL(int)           gimR3Mmio2HandlerPhysicalRegister(PVM pVM, PGIMMMIO2REGION pRegion);
+VMMR3_INT_DECL(int)           gimR3Mmio2HandlerPhysicalDeregister(PVM pVM, PGIMMMIO2REGION pRegion);
+#endif
+
+VMMR3_INT_DECL(int)           gimR3DebugRead(PVM pVM, void *pvRead, size_t *pcbRead, PFNGIMDEBUGBUFREADCOMPLETED pfnReadComplete);
+VMMR3_INT_DECL(int)           gimR3DebugWrite(PVM pVM, void *pvWrite, size_t *pcbWrite);
+#endif /* IN_RING3 */
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_GIMInternal_h */
+
diff --git a/src/VBox/VMM/include/GIMKvmInternal.h b/src/VBox/VMM/include/GIMKvmInternal.h
new file mode 100644
index 00000000..4d68b1d2
--- /dev/null
+++ b/src/VBox/VMM/include/GIMKvmInternal.h
@@ -0,0 +1,272 @@
+/* $Id: GIMKvmInternal.h $ */
+/** @file
+ * GIM - KVM, Internal header file.
+ */
+
+/*
+ * Copyright (C) 2015-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_GIMKvmInternal_h
+#define VMM_INCLUDED_SRC_include_GIMKvmInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/cpum.h>
+
+
+/** @name KVM base features.
+ * @{
+ */
+/** Old, deprecated clock source available. */
+#define GIM_KVM_BASE_FEAT_CLOCK_OLD                RT_BIT(0)
+/** No need for artifical delays on IO operations. */
+#define GIM_KVM_BASE_FEAT_NOP_IO_DELAY             RT_BIT(1)
+/** MMU op supported (deprecated, unused). */
+#define GIM_KVM_BASE_FEAT_MMU_OP                   RT_BIT(2)
+/** Clock source available. */
+#define GIM_KVM_BASE_FEAT_CLOCK                    RT_BIT(3)
+/** Asynchronous page faults supported. */
+#define GIM_KVM_BASE_FEAT_ASYNC_PF                 RT_BIT(4)
+/** Steal time (VCPU not executing guest code time in ns) available. */
+#define GIM_KVM_BASE_FEAT_STEAL_TIME               RT_BIT(5)
+/** Paravirtualized EOI (end-of-interrupt) supported. */
+#define GIM_KVM_BASE_FEAT_PV_EOI                   RT_BIT(6)
+/** Paravirtualized spinlock (unhalting VCPU) supported. */
+#define GIM_KVM_BASE_FEAT_PV_UNHALT                RT_BIT(7)
+/** The TSC is stable (fixed rate, monotonic). */
+#define GIM_KVM_BASE_FEAT_TSC_STABLE               RT_BIT(24)
+/** @}  */
+
+
+/** @name KVM MSRs.
+ * @{
+ */
+/** Start of range 0. */
+#define MSR_GIM_KVM_RANGE0_FIRST                   UINT32_C(0x11)
+/** Old, deprecated wall clock. */
+#define MSR_GIM_KVM_WALL_CLOCK_OLD                 UINT32_C(0x11)
+/** Old, deprecated System time. */
+#define MSR_GIM_KVM_SYSTEM_TIME_OLD                UINT32_C(0x12)
+/** End of range 0. */
+#define MSR_GIM_KVM_RANGE0_LAST                    MSR_GIM_KVM_SYSTEM_TIME_OLD
+
+/** Start of range 1. */
+#define MSR_GIM_KVM_RANGE1_FIRST                   UINT32_C(0x4b564d00)
+/** Wall clock. */
+#define MSR_GIM_KVM_WALL_CLOCK                     UINT32_C(0x4b564d00)
+/** System time. */
+#define MSR_GIM_KVM_SYSTEM_TIME                    UINT32_C(0x4b564d01)
+/** Asynchronous page fault. */
+#define MSR_GIM_KVM_ASYNC_PF                       UINT32_C(0x4b564d02)
+/** Steal time. */
+#define MSR_GIM_KVM_STEAL_TIME                     UINT32_C(0x4b564d03)
+/** Paravirtualized EOI (end-of-interrupt). */
+#define MSR_GIM_KVM_EOI                            UINT32_C(0x4b564d04)
+/** End of range 1. */
+#define MSR_GIM_KVM_RANGE1_LAST                    MSR_GIM_KVM_EOI
+
+AssertCompile(MSR_GIM_KVM_RANGE0_FIRST <= MSR_GIM_KVM_RANGE0_LAST);
+AssertCompile(MSR_GIM_KVM_RANGE1_FIRST <= MSR_GIM_KVM_RANGE1_LAST);
+/** @} */
+
+/** KVM page size.  */
+#define GIM_KVM_PAGE_SIZE                          0x1000
+
+/**
+ * MMIO2 region indices.
+ */
+/** The system time page(s) region. */
+#define GIM_KVM_SYSTEM_TIME_PAGE_REGION_IDX        UINT8_C(0)
+/** The steal time page(s) region. */
+#define GIM_KVM_STEAL_TIME_PAGE_REGION_IDX         UINT8_C(1)
+/** The maximum region index (must be <= UINT8_MAX). */
+#define GIM_KVM_REGION_IDX_MAX                     GIM_KVM_STEAL_TIME_PAGE_REGION_IDX
+
+/**
+ * KVM system-time structure (GIM_KVM_SYSTEM_TIME_FLAGS_XXX) flags.
+ * See "Documentation/virtual/kvm/api.txt".
+ */
+/** The TSC is stable (monotonic). */
+#define GIM_KVM_SYSTEM_TIME_FLAGS_TSC_STABLE       RT_BIT(0)
+/** The guest VCPU has been paused by the hypervisor. */
+#define GIM_KVM_SYSTEM_TIME_FLAGS_GUEST_PAUSED     RT_BIT(1)
+/** */
+
+/** @name KVM MSR - System time (MSR_GIM_KVM_SYSTEM_TIME and
+ * MSR_GIM_KVM_SYSTEM_TIME_OLD).
+ * @{
+ */
+/** The system-time enable bit. */
+#define MSR_GIM_KVM_SYSTEM_TIME_ENABLE_BIT        RT_BIT_64(0)
+/** Whether the system-time struct. is enabled or not. */
+#define MSR_GIM_KVM_SYSTEM_TIME_IS_ENABLED(a)     RT_BOOL((a) & MSR_GIM_KVM_SYSTEM_TIME_ENABLE_BIT)
+/** Guest-physical address of the system-time struct. */
+#define MSR_GIM_KVM_SYSTEM_TIME_GUEST_GPA(a)      ((a) & ~MSR_GIM_KVM_SYSTEM_TIME_ENABLE_BIT)
+/** @} */
+
+/** @name KVM MSR - Wall clock (MSR_GIM_KVM_WALL_CLOCK and
+ * MSR_GIM_KVM_WALL_CLOCK_OLD).
+ * @{
+ */
+/** Guest-physical address of the wall-clock struct. */
+#define MSR_GIM_KVM_WALL_CLOCK_GUEST_GPA(a)        (a)
+/** @} */
+
+
+/** @name KVM Hypercall operations.
+ *  @{ */
+#define KVM_HYPERCALL_OP_VAPIC_POLL_IRQ            1
+#define KVM_HYPERCALL_OP_MMU                       2
+#define KVM_HYPERCALL_OP_FEATURES                  3
+#define KVM_HYPERCALL_OP_KICK_CPU                  5
+/** @} */
+
+/** @name KVM Hypercall return values.
+ *  @{ */
+/* Return values for hypercalls */
+#define KVM_HYPERCALL_RET_SUCCESS                  0
+#define KVM_HYPERCALL_RET_ENOSYS                   (uint64_t)(-1000)
+#define KVM_HYPERCALL_RET_EFAULT                   (uint64_t)(-14)
+#define KVM_HYPERCALL_RET_E2BIG                    (uint64_t)(-7)
+#define KVM_HYPERCALL_RET_EPERM                    (uint64_t)(-1)
+/** @} */
+
+/**
+ * KVM per-VCPU system-time structure.
+ */
+typedef struct GIMKVMSYSTEMTIME
+{
+    /** Version (sequence number). */
+    uint32_t        u32Version;
+    /** Alignment padding. */
+    uint32_t        u32Padding0;
+    /** TSC time stamp.  */
+    uint64_t        u64Tsc;
+    /** System time in nanoseconds. */
+    uint64_t        u64NanoTS;
+    /** TSC to system time scale factor. */
+    uint32_t        u32TscScale;
+    /** TSC frequency shift.  */
+    int8_t          i8TscShift;
+    /** Clock source (GIM_KVM_SYSTEM_TIME_FLAGS_XXX) flags. */
+    uint8_t         fFlags;
+    /** Alignment padding. */
+    uint8_t         abPadding0[2];
+} GIMKVMSYSTEMTIME;
+/** Pointer to KVM system-time struct. */
+typedef GIMKVMSYSTEMTIME *PGIMKVMSYSTEMTIME;
+/** Pointer to a const KVM system-time struct. */
+typedef GIMKVMSYSTEMTIME const *PCGIMKVMSYSTEMTIME;
+AssertCompileSize(GIMKVMSYSTEMTIME, 32);
+
+
+/**
+ * KVM per-VM wall-clock structure.
+ */
+typedef struct GIMKVMWALLCLOCK
+{
+    /** Version (sequence number). */
+    uint32_t        u32Version;
+    /** Number of seconds since boot. */
+    uint32_t        u32Sec;
+    /** Number of nanoseconds since boot. */
+    uint32_t        u32Nano;
+} GIMKVMWALLCLOCK;
+/** Pointer to KVM wall-clock struct. */
+typedef GIMKVMWALLCLOCK *PGIMKVMWALLCLOCK;
+/** Pointer to a const KVM wall-clock struct. */
+typedef GIMKVMWALLCLOCK const *PCGIMKVMWALLCLOCK;
+AssertCompileSize(GIMKVMWALLCLOCK, 12);
+
+
+/**
+ * GIM KVM VM instance data.
+ * Changes to this must checked against the padding of the gim union in VM!
+ */
+typedef struct GIMKVM
+{
+    /** Wall-clock MSR. */
+    uint64_t                    u64WallClockMsr;
+    /**  CPUID features: Basic. */
+    uint32_t                    uBaseFeat;
+    /** Whether GIM needs to trap \#UD exceptions. */
+    bool                        fTrapXcptUD;
+    /** Disassembler opcode of hypercall instruction native for this host CPU. */
+    uint16_t                    uOpcodeNative;
+    /** Native hypercall opcode bytes.  Use for replacing. */
+    uint8_t                     abOpcodeNative[3];
+    /** Alignment padding. */
+    uint8_t                     abPadding[5];
+    /** The TSC frequency (in HZ) reported to the guest. */
+    uint64_t                    cTscTicksPerSecond;
+} GIMKVM;
+/** Pointer to per-VM GIM KVM instance data. */
+typedef GIMKVM *PGIMKVM;
+/** Pointer to const per-VM GIM KVM instance data. */
+typedef GIMKVM const *PCGIMKVM;
+
+/**
+ * GIM KVMV VCPU instance data.
+ * Changes to this must checked against the padding of the gim union in VMCPU!
+ */
+typedef struct GIMKVMCPU
+{
+    /** System-time MSR. */
+    uint64_t                    u64SystemTimeMsr;
+    /** The guest-physical address of the system-time struct. */
+    RTGCPHYS                    GCPhysSystemTime;
+    /** The version (sequence number) of the system-time struct. */
+    uint32_t                    u32SystemTimeVersion;
+    /** The guest TSC value while enabling the system-time MSR. */
+    uint64_t                    uTsc;
+    /** The guest virtual time while enabling the system-time MSR. */
+    uint64_t                    uVirtNanoTS;
+    /** The flags of the system-time struct. */
+    uint8_t                     fSystemTimeFlags;
+} GIMKVMCPU;
+/** Pointer to per-VCPU GIM KVM instance data. */
+typedef GIMKVMCPU *PGIMKVMCPU;
+/** Pointer to const per-VCPU GIM KVM instance data. */
+typedef GIMKVMCPU const *PCGIMKVMCPU;
+
+
+RT_C_DECLS_BEGIN
+
+#ifdef IN_RING3
+VMMR3_INT_DECL(int)             gimR3KvmInit(PVM pVM);
+VMMR3_INT_DECL(int)             gimR3KvmInitCompleted(PVM pVM);
+VMMR3_INT_DECL(int)             gimR3KvmTerm(PVM pVM);
+VMMR3_INT_DECL(void)            gimR3KvmRelocate(PVM pVM, RTGCINTPTR offDelta);
+VMMR3_INT_DECL(void)            gimR3KvmReset(PVM pVM);
+VMMR3_INT_DECL(int)             gimR3KvmSave(PVM pVM, PSSMHANDLE pSSM);
+VMMR3_INT_DECL(int)             gimR3KvmLoad(PVM pVM, PSSMHANDLE pSSM);
+
+VMMR3_INT_DECL(int)             gimR3KvmDisableSystemTime(PVM pVM);
+VMMR3_INT_DECL(int)             gimR3KvmEnableSystemTime(PVM pVM, PVMCPU pVCpu, uint64_t uMsrSystemTime);
+VMMR3_INT_DECL(int)             gimR3KvmEnableWallClock(PVM pVM, RTGCPHYS GCPhysSysTime);
+#endif /* IN_RING3 */
+
+VMM_INT_DECL(bool)              gimKvmIsParavirtTscEnabled(PVMCC pVM);
+VMM_INT_DECL(bool)              gimKvmAreHypercallsEnabled(PVMCPU pVCpu);
+VMM_INT_DECL(VBOXSTRICTRC)      gimKvmHypercall(PVMCPUCC pVCpu, PCPUMCTX pCtx);
+VMM_INT_DECL(VBOXSTRICTRC)      gimKvmReadMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue);
+VMM_INT_DECL(VBOXSTRICTRC)      gimKvmWriteMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uRawValue);
+VMM_INT_DECL(bool)              gimKvmShouldTrapXcptUD(PVM pVM);
+VMM_INT_DECL(VBOXSTRICTRC)      gimKvmXcptUD(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr);
+VMM_INT_DECL(VBOXSTRICTRC)      gimKvmHypercallEx(PVMCPUCC pVCpu, PCPUMCTX pCtx, unsigned uDisOpcode, uint8_t cbInstr);
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_GIMKvmInternal_h */
+
diff --git a/src/VBox/VMM/include/GIMMinimalInternal.h b/src/VBox/VMM/include/GIMMinimalInternal.h
new file mode 100644
index 00000000..91ed3a84
--- /dev/null
+++ b/src/VBox/VMM/include/GIMMinimalInternal.h
@@ -0,0 +1,38 @@
+/* $Id: GIMMinimalInternal.h $ */
+/** @file
+ * GIM - Minimal, Internal header file.
+ */
+
+/*
+ * Copyright (C) 2014-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_GIMMinimalInternal_h
+#define VMM_INCLUDED_SRC_include_GIMMinimalInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <iprt/cdefs.h>
+#include <VBox/types.h>
+
+RT_C_DECLS_BEGIN
+
+#ifdef IN_RING3
+VMMR3_INT_DECL(int)         gimR3MinimalInit(PVM pVM);
+VMMR3_INT_DECL(int)         gimR3MinimalInitCompleted(PVM pVM);
+VMMR3_INT_DECL(void)        gimR3MinimalRelocate(PVM pVM, RTGCINTPTR offDelta);
+#endif /* IN_RING3 */
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_GIMMinimalInternal_h */
+
diff --git a/src/VBox/VMM/include/HMInternal.h b/src/VBox/VMM/include/HMInternal.h
new file mode 100644
index 00000000..450b0baf
--- /dev/null
+++ b/src/VBox/VMM/include/HMInternal.h
@@ -0,0 +1,1239 @@
+/* $Id: HMInternal.h $ */
+/** @file
+ * HM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_HMInternal_h
+#define VMM_INCLUDED_SRC_include_HMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/dis.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/hm_vmx.h>
+#include <VBox/vmm/hm_svm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/trpm.h>
+#include <iprt/memobj.h>
+#include <iprt/cpuset.h>
+#include <iprt/mp.h>
+#include <iprt/avl.h>
+#include <iprt/string.h>
+
+#if HC_ARCH_BITS == 32
+# error "32-bit hosts are no longer supported. Go back to 6.0 or earlier!"
+#endif
+
+/** @def HM_PROFILE_EXIT_DISPATCH
+ * Enables profiling of the VM exit handler dispatching. */
+#if 0 || defined(DOXYGEN_RUNNING)
+# define HM_PROFILE_EXIT_DISPATCH
+#endif
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_hm_int       Internal
+ * @ingroup grp_hm
+ * @internal
+ * @{
+ */
+
+/** @name HM_CHANGED_XXX
+ * HM CPU-context changed flags.
+ *
+ * These flags are used to keep track of which registers and state has been
+ * modified since they were imported back into the guest-CPU context.
+ *
+ * @{
+ */
+#define HM_CHANGED_HOST_CONTEXT                  UINT64_C(0x0000000000000001)
+#define HM_CHANGED_GUEST_RIP                     UINT64_C(0x0000000000000004)
+#define HM_CHANGED_GUEST_RFLAGS                  UINT64_C(0x0000000000000008)
+
+#define HM_CHANGED_GUEST_RAX                     UINT64_C(0x0000000000000010)
+#define HM_CHANGED_GUEST_RCX                     UINT64_C(0x0000000000000020)
+#define HM_CHANGED_GUEST_RDX                     UINT64_C(0x0000000000000040)
+#define HM_CHANGED_GUEST_RBX                     UINT64_C(0x0000000000000080)
+#define HM_CHANGED_GUEST_RSP                     UINT64_C(0x0000000000000100)
+#define HM_CHANGED_GUEST_RBP                     UINT64_C(0x0000000000000200)
+#define HM_CHANGED_GUEST_RSI                     UINT64_C(0x0000000000000400)
+#define HM_CHANGED_GUEST_RDI                     UINT64_C(0x0000000000000800)
+#define HM_CHANGED_GUEST_R8_R15                  UINT64_C(0x0000000000001000)
+#define HM_CHANGED_GUEST_GPRS_MASK               UINT64_C(0x0000000000001ff0)
+
+#define HM_CHANGED_GUEST_ES                      UINT64_C(0x0000000000002000)
+#define HM_CHANGED_GUEST_CS                      UINT64_C(0x0000000000004000)
+#define HM_CHANGED_GUEST_SS                      UINT64_C(0x0000000000008000)
+#define HM_CHANGED_GUEST_DS                      UINT64_C(0x0000000000010000)
+#define HM_CHANGED_GUEST_FS                      UINT64_C(0x0000000000020000)
+#define HM_CHANGED_GUEST_GS                      UINT64_C(0x0000000000040000)
+#define HM_CHANGED_GUEST_SREG_MASK               UINT64_C(0x000000000007e000)
+
+#define HM_CHANGED_GUEST_GDTR                    UINT64_C(0x0000000000080000)
+#define HM_CHANGED_GUEST_IDTR                    UINT64_C(0x0000000000100000)
+#define HM_CHANGED_GUEST_LDTR                    UINT64_C(0x0000000000200000)
+#define HM_CHANGED_GUEST_TR                      UINT64_C(0x0000000000400000)
+#define HM_CHANGED_GUEST_TABLE_MASK              UINT64_C(0x0000000000780000)
+
+#define HM_CHANGED_GUEST_CR0                     UINT64_C(0x0000000000800000)
+#define HM_CHANGED_GUEST_CR2                     UINT64_C(0x0000000001000000)
+#define HM_CHANGED_GUEST_CR3                     UINT64_C(0x0000000002000000)
+#define HM_CHANGED_GUEST_CR4                     UINT64_C(0x0000000004000000)
+#define HM_CHANGED_GUEST_CR_MASK                 UINT64_C(0x0000000007800000)
+
+#define HM_CHANGED_GUEST_APIC_TPR                UINT64_C(0x0000000008000000)
+#define HM_CHANGED_GUEST_EFER_MSR                UINT64_C(0x0000000010000000)
+
+#define HM_CHANGED_GUEST_DR0_DR3                 UINT64_C(0x0000000020000000)
+#define HM_CHANGED_GUEST_DR6                     UINT64_C(0x0000000040000000)
+#define HM_CHANGED_GUEST_DR7                     UINT64_C(0x0000000080000000)
+#define HM_CHANGED_GUEST_DR_MASK                 UINT64_C(0x00000000e0000000)
+
+#define HM_CHANGED_GUEST_X87                     UINT64_C(0x0000000100000000)
+#define HM_CHANGED_GUEST_SSE_AVX                 UINT64_C(0x0000000200000000)
+#define HM_CHANGED_GUEST_OTHER_XSAVE             UINT64_C(0x0000000400000000)
+#define HM_CHANGED_GUEST_XCRx                    UINT64_C(0x0000000800000000)
+
+#define HM_CHANGED_GUEST_KERNEL_GS_BASE          UINT64_C(0x0000001000000000)
+#define HM_CHANGED_GUEST_SYSCALL_MSRS            UINT64_C(0x0000002000000000)
+#define HM_CHANGED_GUEST_SYSENTER_CS_MSR         UINT64_C(0x0000004000000000)
+#define HM_CHANGED_GUEST_SYSENTER_EIP_MSR        UINT64_C(0x0000008000000000)
+#define HM_CHANGED_GUEST_SYSENTER_ESP_MSR        UINT64_C(0x0000010000000000)
+#define HM_CHANGED_GUEST_SYSENTER_MSR_MASK       UINT64_C(0x000001c000000000)
+#define HM_CHANGED_GUEST_TSC_AUX                 UINT64_C(0x0000020000000000)
+#define HM_CHANGED_GUEST_OTHER_MSRS              UINT64_C(0x0000040000000000)
+#define HM_CHANGED_GUEST_ALL_MSRS                (  HM_CHANGED_GUEST_EFER              \
+                                                  | HM_CHANGED_GUEST_KERNEL_GS_BASE    \
+                                                  | HM_CHANGED_GUEST_SYSCALL_MSRS      \
+                                                  | HM_CHANGED_GUEST_SYSENTER_MSR_MASK \
+                                                  | HM_CHANGED_GUEST_TSC_AUX           \
+                                                  | HM_CHANGED_GUEST_OTHER_MSRS)
+
+#define HM_CHANGED_GUEST_HWVIRT                  UINT64_C(0x0000080000000000)
+#define HM_CHANGED_GUEST_MASK                    UINT64_C(0x00000ffffffffffc)
+
+#define HM_CHANGED_KEEPER_STATE_MASK             UINT64_C(0xffff000000000000)
+
+#define HM_CHANGED_VMX_XCPT_INTERCEPTS           UINT64_C(0x0001000000000000)
+#define HM_CHANGED_VMX_GUEST_AUTO_MSRS           UINT64_C(0x0002000000000000)
+#define HM_CHANGED_VMX_GUEST_LAZY_MSRS           UINT64_C(0x0004000000000000)
+#define HM_CHANGED_VMX_ENTRY_EXIT_CTLS           UINT64_C(0x0008000000000000)
+#define HM_CHANGED_VMX_MASK                      UINT64_C(0x000f000000000000)
+#define HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE   (  HM_CHANGED_GUEST_DR_MASK \
+                                                  | HM_CHANGED_VMX_GUEST_LAZY_MSRS)
+
+#define HM_CHANGED_SVM_XCPT_INTERCEPTS           UINT64_C(0x0001000000000000)
+#define HM_CHANGED_SVM_MASK                      UINT64_C(0x0001000000000000)
+#define HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE   HM_CHANGED_GUEST_DR_MASK
+
+#define HM_CHANGED_ALL_GUEST                     (  HM_CHANGED_GUEST_MASK \
+                                                  | HM_CHANGED_KEEPER_STATE_MASK)
+
+/** Mask of what state might have changed when IEM raised an exception.
+ *  This is a based on IEM_CPUMCTX_EXTRN_XCPT_MASK. */
+#define HM_CHANGED_RAISED_XCPT_MASK              (  HM_CHANGED_GUEST_GPRS_MASK  \
+                                                  | HM_CHANGED_GUEST_RIP        \
+                                                  | HM_CHANGED_GUEST_RFLAGS     \
+                                                  | HM_CHANGED_GUEST_SS         \
+                                                  | HM_CHANGED_GUEST_CS         \
+                                                  | HM_CHANGED_GUEST_CR0        \
+                                                  | HM_CHANGED_GUEST_CR3        \
+                                                  | HM_CHANGED_GUEST_CR4        \
+                                                  | HM_CHANGED_GUEST_APIC_TPR   \
+                                                  | HM_CHANGED_GUEST_EFER_MSR   \
+                                                  | HM_CHANGED_GUEST_DR7        \
+                                                  | HM_CHANGED_GUEST_CR2        \
+                                                  | HM_CHANGED_GUEST_SREG_MASK  \
+                                                  | HM_CHANGED_GUEST_TABLE_MASK)
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+/** Mask of what state might have changed when \#VMEXIT is emulated. */
+# define HM_CHANGED_SVM_VMEXIT_MASK              (  HM_CHANGED_GUEST_RSP         \
+                                                  | HM_CHANGED_GUEST_RAX         \
+                                                  | HM_CHANGED_GUEST_RIP         \
+                                                  | HM_CHANGED_GUEST_RFLAGS      \
+                                                  | HM_CHANGED_GUEST_CS          \
+                                                  | HM_CHANGED_GUEST_SS          \
+                                                  | HM_CHANGED_GUEST_DS          \
+                                                  | HM_CHANGED_GUEST_ES          \
+                                                  | HM_CHANGED_GUEST_GDTR        \
+                                                  | HM_CHANGED_GUEST_IDTR        \
+                                                  | HM_CHANGED_GUEST_CR_MASK     \
+                                                  | HM_CHANGED_GUEST_EFER_MSR    \
+                                                  | HM_CHANGED_GUEST_DR6         \
+                                                  | HM_CHANGED_GUEST_DR7         \
+                                                  | HM_CHANGED_GUEST_OTHER_MSRS  \
+                                                  | HM_CHANGED_GUEST_HWVIRT      \
+                                                  | HM_CHANGED_SVM_MASK          \
+                                                  | HM_CHANGED_GUEST_APIC_TPR)
+
+/** Mask of what state might have changed when VMRUN is emulated. */
+# define HM_CHANGED_SVM_VMRUN_MASK               HM_CHANGED_SVM_VMEXIT_MASK
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/** Mask of what state might have changed when VM-exit is emulated.
+ *
+ *  This is currently unused, but keeping it here in case we can get away a bit more
+ *  fine-grained state handling.
+ *
+ *  @note Update IEM_CPUMCTX_EXTRN_VMX_VMEXIT_MASK when this changes. */
+# define HM_CHANGED_VMX_VMEXIT_MASK             (  HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_CR3 | HM_CHANGED_GUEST_CR4 \
+                                                 | HM_CHANGED_GUEST_DR7 | HM_CHANGED_GUEST_DR6 \
+                                                 | HM_CHANGED_GUEST_EFER_MSR \
+                                                 | HM_CHANGED_GUEST_SYSENTER_MSR_MASK \
+                                                 | HM_CHANGED_GUEST_OTHER_MSRS    /* for PAT MSR */ \
+                                                 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS \
+                                                 | HM_CHANGED_GUEST_SREG_MASK \
+                                                 | HM_CHANGED_GUEST_TR \
+                                                 | HM_CHANGED_GUEST_LDTR | HM_CHANGED_GUEST_GDTR | HM_CHANGED_GUEST_IDTR \
+                                                 | HM_CHANGED_GUEST_HWVIRT )
+#endif
+/** @} */
+
+/** Maximum number of exit reason statistics counters. */
+#define MAX_EXITREASON_STAT                        0x100
+#define MASK_EXITREASON_STAT                       0xff
+#define MASK_INJECT_IRQ_STAT                       0xff
+
+/** Size for the EPT identity page table (1024 4 MB pages to cover the entire address space). */
+#define HM_EPT_IDENTITY_PG_TABLE_SIZE               PAGE_SIZE
+/** Size of the TSS structure + 2 pages for the IO bitmap + end byte. */
+#define HM_VTX_TSS_SIZE                             (sizeof(VBOXTSS) + 2 * PAGE_SIZE + 1)
+/** Total guest mapped memory needed. */
+#define HM_VTX_TOTAL_DEVHEAP_MEM                    (HM_EPT_IDENTITY_PG_TABLE_SIZE + HM_VTX_TSS_SIZE)
+
+
+/** @name Macros for enabling and disabling preemption.
+ * These are really just for hiding the RTTHREADPREEMPTSTATE and asserting that
+ * preemption has already been disabled when there is no context hook.
+ * @{ */
+#ifdef VBOX_STRICT
+# define HM_DISABLE_PREEMPT(a_pVCpu) \
+    RTTHREADPREEMPTSTATE PreemptStateInternal = RTTHREADPREEMPTSTATE_INITIALIZER; \
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD) || VMMR0ThreadCtxHookIsEnabled((a_pVCpu))); \
+    RTThreadPreemptDisable(&PreemptStateInternal)
+#else
+# define HM_DISABLE_PREEMPT(a_pVCpu) \
+    RTTHREADPREEMPTSTATE PreemptStateInternal = RTTHREADPREEMPTSTATE_INITIALIZER; \
+    RTThreadPreemptDisable(&PreemptStateInternal)
+#endif /* VBOX_STRICT */
+#define HM_RESTORE_PREEMPT()    do { RTThreadPreemptRestore(&PreemptStateInternal); } while(0)
+/** @} */
+
+
+/** @name HM saved state versions.
+ * @{
+ */
+#define HM_SAVED_STATE_VERSION                         HM_SAVED_STATE_VERSION_SVM_NESTED_HWVIRT
+#define HM_SAVED_STATE_VERSION_SVM_NESTED_HWVIRT       6
+#define HM_SAVED_STATE_VERSION_TPR_PATCHING            5
+#define HM_SAVED_STATE_VERSION_NO_TPR_PATCHING         4
+#define HM_SAVED_STATE_VERSION_2_0_X                   3
+/** @} */
+
+
+/**
+ * HM physical (host) CPU information.
+ */
+typedef struct HMPHYSCPU
+{
+    /** The CPU ID. */
+    RTCPUID             idCpu;
+    /** The VM_HSAVE_AREA (AMD-V) / VMXON region (Intel) memory backing. */
+    RTR0MEMOBJ          hMemObj;
+    /** The physical address of the first page in hMemObj (it's a
+     *  physcially contigous allocation if it spans multiple pages). */
+    RTHCPHYS            HCPhysMemObj;
+    /** The address of the memory (for pfnEnable). */
+    void               *pvMemObj;
+    /** Current ASID (AMD-V) / VPID (Intel). */
+    uint32_t            uCurrentAsid;
+    /** TLB flush count. */
+    uint32_t            cTlbFlushes;
+    /** Whether to flush each new ASID/VPID before use. */
+    bool                fFlushAsidBeforeUse;
+    /** Configured for VT-x or AMD-V. */
+    bool                fConfigured;
+    /** Set if the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE hack is active. */
+    bool                fIgnoreAMDVInUseError;
+    /** Whether CR4.VMXE was already enabled prior to us enabling it. */
+    bool                fVmxeAlreadyEnabled;
+    /** In use by our code. (for power suspend) */
+    bool volatile       fInUse;
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    /** Nested-guest union (put data common to SVM/VMX outside the union). */
+    union
+    {
+        /** Nested-guest SVM data. */
+        struct
+        {
+            /** The active nested-guest MSR permission bitmap memory backing. */
+            RTR0MEMOBJ          hNstGstMsrpm;
+            /** The physical address of the first page in hNstGstMsrpm (physcially
+             *  contiguous allocation). */
+            RTHCPHYS            HCPhysNstGstMsrpm;
+            /** The address of the active nested-guest MSRPM. */
+            void               *pvNstGstMsrpm;
+        } svm;
+        /** @todo Nested-VMX. */
+    } n;
+#endif
+} HMPHYSCPU;
+/** Pointer to HMPHYSCPU struct. */
+typedef HMPHYSCPU *PHMPHYSCPU;
+/** Pointer to a const HMPHYSCPU struct. */
+typedef const HMPHYSCPU *PCHMPHYSCPU;
+
+/**
+ * TPR-instruction type.
+ */
+typedef enum
+{
+    HMTPRINSTR_INVALID,
+    HMTPRINSTR_READ,
+    HMTPRINSTR_READ_SHR4,
+    HMTPRINSTR_WRITE_REG,
+    HMTPRINSTR_WRITE_IMM,
+    HMTPRINSTR_JUMP_REPLACEMENT,
+    /** The usual 32-bit paranoia. */
+    HMTPRINSTR_32BIT_HACK   = 0x7fffffff
+} HMTPRINSTR;
+
+/**
+ * TPR patch information.
+ */
+typedef struct
+{
+    /** The key is the address of patched instruction. (32 bits GC ptr) */
+    AVLOU32NODECORE         Core;
+    /** Original opcode. */
+    uint8_t                 aOpcode[16];
+    /** Instruction size. */
+    uint32_t                cbOp;
+    /** Replacement opcode. */
+    uint8_t                 aNewOpcode[16];
+    /** Replacement instruction size. */
+    uint32_t                cbNewOp;
+    /** Instruction type. */
+    HMTPRINSTR              enmType;
+    /** Source operand. */
+    uint32_t                uSrcOperand;
+    /** Destination operand. */
+    uint32_t                uDstOperand;
+    /** Number of times the instruction caused a fault. */
+    uint32_t                cFaults;
+    /** Patch address of the jump replacement. */
+    RTGCPTR32               pJumpTarget;
+} HMTPRPATCH;
+/** Pointer to HMTPRPATCH. */
+typedef HMTPRPATCH *PHMTPRPATCH;
+/** Pointer to a const HMTPRPATCH. */
+typedef const HMTPRPATCH *PCHMTPRPATCH;
+
+
+/**
+ * Makes a HMEXITSTAT::uKey value from a program counter and an exit code.
+ *
+ * @returns 64-bit key
+ * @param   a_uPC           The RIP + CS.BASE value of the exit.
+ * @param   a_uExit         The exit code.
+ * @todo    Add CPL?
+ */
+#define HMEXITSTAT_MAKE_KEY(a_uPC, a_uExit) (((a_uPC) & UINT64_C(0x0000ffffffffffff)) | (uint64_t)(a_uExit) << 48)
+
+typedef struct HMEXITINFO
+{
+    /** See HMEXITSTAT_MAKE_KEY(). */
+    uint64_t                uKey;
+    /** Number of recent hits (depreciates with time). */
+    uint32_t volatile       cHits;
+    /** The age + lock. */
+    uint16_t volatile       uAge;
+    /** Action or action table index. */
+    uint16_t                iAction;
+} HMEXITINFO;
+AssertCompileSize(HMEXITINFO, 16); /* Lots of these guys, so don't add any unnecessary stuff! */
+
+typedef struct HMEXITHISTORY
+{
+    /** The exit timestamp. */
+    uint64_t                uTscExit;
+    /** The index of the corresponding HMEXITINFO entry.
+     * UINT32_MAX if none (too many collisions, race, whatever). */
+    uint32_t                iExitInfo;
+    /** Figure out later, needed for padding now. */
+    uint32_t                uSomeClueOrSomething;
+} HMEXITHISTORY;
+
+/**
+ * Switcher function, HC to the special 64-bit RC.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   offCpumVCpu     Offset from pVM->cpum to pVM->aCpus[idCpu].cpum.
+ * @returns Return code indicating the action to take.
+ */
+typedef DECLCALLBACK(int) FNHMSWITCHERHC(PVM pVM, uint32_t offCpumVCpu);
+/** Pointer to switcher function. */
+typedef FNHMSWITCHERHC *PFNHMSWITCHERHC;
+
+/** @def HM_UNION_NM
+ * For compilers (like DTrace) that does not grok nameless unions, we have a
+ * little hack to make them palatable.
+ */
+/** @def HM_STRUCT_NM
+ * For compilers (like DTrace) that does not grok nameless structs (it is
+ * non-standard C++), we have a little hack to make them palatable.
+ */
+#ifdef VBOX_FOR_DTRACE_LIB
+# define HM_UNION_NM(a_Nm)  a_Nm
+# define HM_STRUCT_NM(a_Nm) a_Nm
+#elif defined(IPRT_WITHOUT_NAMED_UNIONS_AND_STRUCTS)
+# define HM_UNION_NM(a_Nm)  a_Nm
+# define HM_STRUCT_NM(a_Nm) a_Nm
+#else
+# define HM_UNION_NM(a_Nm)
+# define HM_STRUCT_NM(a_Nm)
+#endif
+
+/**
+ * HM event.
+ *
+ * VT-x and AMD-V common event injection structure.
+ */
+typedef struct HMEVENT
+{
+    /** Whether the event is pending. */
+    uint32_t        fPending;
+    /** The error-code associated with the event. */
+    uint32_t        u32ErrCode;
+    /** The length of the instruction in bytes (only relevant for software
+     *  interrupts or software exceptions). */
+    uint32_t        cbInstr;
+    /** Alignment. */
+    uint32_t        u32Padding;
+    /** The encoded event (VM-entry interruption-information for VT-x or EVENTINJ
+     *  for SVM). */
+    uint64_t        u64IntInfo;
+    /** Guest virtual address if this is a page-fault event. */
+    RTGCUINTPTR     GCPtrFaultAddress;
+} HMEVENT;
+/** Pointer to a HMEVENT struct. */
+typedef HMEVENT *PHMEVENT;
+/** Pointer to a const HMEVENT struct. */
+typedef const HMEVENT *PCHMEVENT;
+AssertCompileSizeAlignment(HMEVENT, 8);
+
+/**
+ * HM VM Instance data.
+ * Changes to this must checked against the padding of the hm union in VM!
+ */
+typedef struct HM
+{
+    /** Set if nested paging is enabled. */
+    bool                        fNestedPaging;
+    /** Set when we've initialized VMX or SVM. */
+    bool                        fInitialized;
+    /** Set if nested paging is allowed. */
+    bool                        fAllowNestedPaging;
+    /** Set if large pages are enabled (requires nested paging). */
+    bool                        fLargePages;
+    /** Set if we can support 64-bit guests or not. */
+    bool                        fAllow64BitGuests;
+    /** Set when TPR patching is allowed. */
+    bool                        fTprPatchingAllowed;
+    /** Set when we initialize VT-x or AMD-V once for all CPUs. */
+    bool                        fGlobalInit;
+    /** Set when TPR patching is active. */
+    bool                        fTPRPatchingActive;
+    /** Set when the debug facility has breakpoints/events enabled that requires
+     *  us to use the debug execution loop in ring-0. */
+    bool                        fUseDebugLoop;
+    /** Set if hardware APIC virtualization is enabled. */
+    bool                        fVirtApicRegs;
+    /** Set if posted interrupt processing is enabled. */
+    bool                        fPostedIntrs;
+    /** Set if indirect branch prediction barrier on VM exit. */
+    bool                        fIbpbOnVmExit;
+    /** Set if indirect branch prediction barrier on VM entry. */
+    bool                        fIbpbOnVmEntry;
+    /** Set if level 1 data cache should be flushed on VM entry. */
+    bool                        fL1dFlushOnVmEntry;
+    /** Set if level 1 data cache should be flushed on EMT scheduling. */
+    bool                        fL1dFlushOnSched;
+    /** Set if host manages speculation control settings. */
+    bool                        fSpecCtrlByHost;
+    /** Set if MDS related buffers should be cleared on VM entry. */
+    bool                        fMdsClearOnVmEntry;
+    /** Set if MDS related buffers should be cleared on EMT scheduling. */
+    bool                        fMdsClearOnSched;
+    /** Alignment padding. */
+    bool                        afPaddingMinus1[6];
+
+    /** Maximum ASID allowed. */
+    uint32_t                    uMaxAsid;
+    /** The maximum number of resumes loops allowed in ring-0 (safety precaution).
+     * This number is set much higher when RTThreadPreemptIsPending is reliable. */
+    uint32_t                    cMaxResumeLoops;
+
+    /** Host kernel flags that HM might need to know (SUPKERNELFEATURES_XXX). */
+    uint32_t                    fHostKernelFeatures;
+
+    /** Size of the guest patch memory block. */
+    uint32_t                    cbGuestPatchMem;
+    /** Guest allocated memory for patching purposes. */
+    RTGCPTR                     pGuestPatchMem;
+    /** Current free pointer inside the patch block. */
+    RTGCPTR                     pFreeGuestPatchMem;
+
+    struct
+    {
+        /** Set by the ring-0 side of HM to indicate VMX is supported by the
+         *  CPU. */
+        bool                        fSupported;
+        /** Set when we've enabled VMX. */
+        bool                        fEnabled;
+        /** Set if VPID is supported. */
+        bool                        fVpid;
+        /** Set if VT-x VPID is allowed. */
+        bool                        fAllowVpid;
+        /** Set if unrestricted guest execution is in use (real and protected mode
+         *  without paging). */
+        bool                        fUnrestrictedGuest;
+        /** Set if unrestricted guest execution is allowed to be used. */
+        bool                        fAllowUnrestricted;
+        /** Set if the preemption timer is in use or not. */
+        bool                        fUsePreemptTimer;
+        /** The shift mask employed by the VMX-Preemption timer. */
+        uint8_t                     cPreemptTimerShift;
+
+        /** Virtual address of the APIC-access page. */
+        R0PTRTYPE(uint8_t *)        pbApicAccess;
+        /** Pointer to the VMREAD bitmap. */
+        R0PTRTYPE(void *)           pvVmreadBitmap;
+        /** Pointer to the VMWRITE bitmap. */
+        R0PTRTYPE(void *)           pvVmwriteBitmap;
+
+        /** Pointer to the shadow VMCS read-only fields array. */
+        R0PTRTYPE(uint32_t *)       paShadowVmcsRoFields;
+        /** Pointer to the shadow VMCS read/write fields array. */
+        R0PTRTYPE(uint32_t *)       paShadowVmcsFields;
+        /** Number of elements in the shadow VMCS read-only fields array. */
+        uint32_t                    cShadowVmcsRoFields;
+        /** Number of elements in the shadow VMCS read-write fields array. */
+        uint32_t                    cShadowVmcsFields;
+
+        /** Tagged-TLB flush type. */
+        VMXTLBFLUSHTYPE             enmTlbFlushType;
+        /** Flush type to use for INVEPT. */
+        VMXTLBFLUSHEPT              enmTlbFlushEpt;
+        /** Flush type to use for INVVPID. */
+        VMXTLBFLUSHVPID             enmTlbFlushVpid;
+
+        /** Pause-loop exiting (PLE) gap in ticks. */
+        uint32_t                    cPleGapTicks;
+        /** Pause-loop exiting (PLE) window in ticks. */
+        uint32_t                    cPleWindowTicks;
+        uint32_t                    u32Alignment0;
+
+        /** Host CR4 value (set by ring-0 VMX init) */
+        uint64_t                    u64HostCr4;
+        /** Host SMM monitor control (set by ring-0 VMX init) */
+        uint64_t                    u64HostSmmMonitorCtl;
+        /** Host EFER value (set by ring-0 VMX init) */
+        uint64_t                    u64HostMsrEfer;
+        /** Whether the CPU supports VMCS fields for swapping EFER. */
+        bool                        fSupportsVmcsEfer;
+        /** Whether to use VMCS shadowing. */
+        bool                        fUseVmcsShadowing;
+        /** Set if Last Branch Record (LBR) is enabled. */
+        bool                        fLbr;
+        uint8_t                     u8Alignment2[5];
+
+        /** The first valid host LBR branch-from-IP stack range. */
+        uint32_t                    idLbrFromIpMsrFirst;
+        /** The last valid host LBR branch-from-IP stack range. */
+        uint32_t                    idLbrFromIpMsrLast;
+
+        /** The first valid host LBR branch-to-IP stack range. */
+        uint32_t                    idLbrToIpMsrFirst;
+        /** The last valid host LBR branch-to-IP stack range. */
+        uint32_t                    idLbrToIpMsrLast;
+
+        /** The host LBR TOS (top-of-stack) MSR id. */
+        uint32_t                    idLbrTosMsr;
+        /** Padding. */
+        uint32_t                    u32Alignment1;
+
+        /** VMX MSR values. */
+        VMXMSRS                     Msrs;
+
+        /** Host-physical address for a failing VMXON instruction. */
+        RTHCPHYS                    HCPhysVmxEnableError;
+        /** Host-physical address of the APIC-access page. */
+        RTHCPHYS                    HCPhysApicAccess;
+        /** Host-physical address of the VMREAD bitmap. */
+        RTHCPHYS                    HCPhysVmreadBitmap;
+        /** Host-physical address of the VMWRITE bitmap. */
+        RTHCPHYS                    HCPhysVmwriteBitmap;
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+        /** Host-physical address of the crash-dump scratch area. */
+        RTHCPHYS                    HCPhysScratch;
+#endif
+
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+        /** Pointer to the crash-dump scratch bitmap. */
+        R0PTRTYPE(uint8_t *)        pbScratch;
+#endif
+        /** Virtual address of the TSS page used for real mode emulation. */
+        R3PTRTYPE(PVBOXTSS)         pRealModeTSS;
+        /** Virtual address of the identity page table used for real mode and protected
+         *  mode without paging emulation in EPT mode. */
+        R3PTRTYPE(PX86PD)           pNonPagingModeEPTPageTable;
+
+        /** Ring-0 memory object for per-VM VMX structures. */
+        RTR0MEMOBJ                  hMemObj;
+    } vmx;
+
+    struct
+    {
+        /** Set by the ring-0 side of HM to indicate SVM is supported by the
+         *  CPU. */
+        bool                        fSupported;
+        /** Set when we've enabled SVM. */
+        bool                        fEnabled;
+        /** Set if erratum 170 affects the AMD cpu. */
+        bool                        fAlwaysFlushTLB;
+        /** Set when the hack to ignore VERR_SVM_IN_USE is active. */
+        bool                        fIgnoreInUseError;
+        /** Whether to use virtualized VMSAVE/VMLOAD feature. */
+        bool                        fVirtVmsaveVmload;
+        /** Whether to use virtual GIF feature. */
+        bool                        fVGif;
+        /** Whether to use LBR virtualization feature. */
+        bool                        fLbrVirt;
+        uint8_t                     u8Alignment0[1];
+
+        /** Physical address of the IO bitmap (12kb). */
+        RTHCPHYS                    HCPhysIOBitmap;
+        /** R0 memory object for the IO bitmap (12kb). */
+        RTR0MEMOBJ                  hMemObjIOBitmap;
+        /** Virtual address of the IO bitmap. */
+        R0PTRTYPE(void *)           pvIOBitmap;
+
+        /* HWCR MSR (for diagnostics) */
+        uint64_t                    u64MsrHwcr;
+
+        /** SVM revision. */
+        uint32_t                    u32Rev;
+        /** SVM feature bits from cpuid 0x8000000a */
+        uint32_t                    u32Features;
+
+        /** Pause filter counter. */
+        uint16_t                    cPauseFilter;
+        /** Pause filter treshold in ticks. */
+        uint16_t                    cPauseFilterThresholdTicks;
+        uint32_t                    u32Alignment0;
+    } svm;
+
+    /**
+     * AVL tree with all patches (active or disabled) sorted by guest instruction
+     * address.
+     */
+    AVLOU32TREE                 PatchTree;
+    uint32_t                    cPatches;
+    HMTPRPATCH                  aPatches[64];
+
+    /** Last recorded error code during HM ring-0 init. */
+    int32_t                     rcInit;
+
+    /** HMR0Init was run */
+    bool                        fHMR0Init;
+    bool                        u8Alignment1[3];
+
+    STAMCOUNTER                 StatTprPatchSuccess;
+    STAMCOUNTER                 StatTprPatchFailure;
+    STAMCOUNTER                 StatTprReplaceSuccessCr8;
+    STAMCOUNTER                 StatTprReplaceSuccessVmc;
+    STAMCOUNTER                 StatTprReplaceFailure;
+} HM;
+/** Pointer to HM VM instance data. */
+typedef HM *PHM;
+AssertCompileMemberAlignment(HM, StatTprPatchSuccess, 8);
+AssertCompileMemberAlignment(HM, vmx,                 8);
+AssertCompileMemberAlignment(HM, svm,                 8);
+
+
+/**
+ * VMX StartVM function.
+ *
+ * @returns VBox status code (no informational stuff).
+ * @param   fResume     Whether to use VMRESUME (true) or VMLAUNCH (false).
+ * @param   pCtx        The CPU register context.
+ * @param   pvUnused    Unused argument.
+ * @param   pVM         Pointer to the cross context VM structure.
+ * @param   pVCpu       Pointer to the cross context per-CPU structure.
+ */
+typedef DECLCALLBACK(int) FNHMVMXSTARTVM(RTHCUINT fResume, PCPUMCTX pCtx, void *pvUnused, PVMCC pVM, PVMCPUCC pVCpu);
+/** Pointer to a VMX StartVM function. */
+typedef R0PTRTYPE(FNHMVMXSTARTVM *) PFNHMVMXSTARTVM;
+
+/** SVM VMRun function. */
+typedef DECLCALLBACK(int) FNHMSVMVMRUN(RTHCPHYS pVmcbHostPhys, RTHCPHYS pVmcbPhys, PCPUMCTX pCtx, PVMCC pVM, PVMCPUCC pVCpu);
+/** Pointer to a SVM VMRun function. */
+typedef R0PTRTYPE(FNHMSVMVMRUN *) PFNHMSVMVMRUN;
+
+/**
+ * VMX VMCS information.
+ *
+ * This structure provides information maintained for and during the executing of a
+ * guest (or nested-guest) VMCS (VM control structure) using hardware-assisted VMX.
+ *
+ * Note! The members here are ordered and aligned based on estimated frequency of
+ * usage and grouped to fit within a cache line in hot code paths. Even subtle
+ * changes here have a noticeable effect in the bootsector benchmarks. Modify with
+ * care.
+ */
+typedef struct VMXVMCSINFO
+{
+    /** @name Auxiliary information.
+     * @{ */
+    /** Ring-0 pointer to the hardware-assisted VMX execution function. */
+    PFNHMVMXSTARTVM             pfnStartVM;
+    /** Host-physical address of the EPTP. */
+    RTHCPHYS                    HCPhysEPTP;
+    /** The VMCS launch state, see VMX_V_VMCS_LAUNCH_STATE_XXX. */
+    uint32_t                    fVmcsState;
+    /** The VMCS launch state of the shadow VMCS, see VMX_V_VMCS_LAUNCH_STATE_XXX. */
+    uint32_t                    fShadowVmcsState;
+    /** The host CPU for which its state has been exported to this VMCS. */
+    RTCPUID                     idHostCpuState;
+    /** The host CPU on which we last executed this VMCS. */
+    RTCPUID                     idHostCpuExec;
+    /** Number of guest MSRs in the VM-entry MSR-load area. */
+    uint32_t                    cEntryMsrLoad;
+    /** Number of guest MSRs in the VM-exit MSR-store area. */
+    uint32_t                    cExitMsrStore;
+    /** Number of host MSRs in the VM-exit MSR-load area. */
+    uint32_t                    cExitMsrLoad;
+    /** @} */
+
+    /** @name Cache of execution related VMCS fields.
+     *  @{ */
+    /** Pin-based VM-execution controls. */
+    uint32_t                    u32PinCtls;
+    /** Processor-based VM-execution controls. */
+    uint32_t                    u32ProcCtls;
+    /** Secondary processor-based VM-execution controls. */
+    uint32_t                    u32ProcCtls2;
+    /** VM-entry controls. */
+    uint32_t                    u32EntryCtls;
+    /** VM-exit controls. */
+    uint32_t                    u32ExitCtls;
+    /** Exception bitmap. */
+    uint32_t                    u32XcptBitmap;
+    /** Page-fault exception error-code mask. */
+    uint32_t                    u32XcptPFMask;
+    /** Page-fault exception error-code match. */
+    uint32_t                    u32XcptPFMatch;
+    /** Padding. */
+    uint32_t                    u32Alignment0;
+    /** TSC offset. */
+    uint64_t                    u64TscOffset;
+    /** VMCS link pointer. */
+    uint64_t                    u64VmcsLinkPtr;
+    /** CR0 guest/host mask. */
+    uint64_t                    u64Cr0Mask;
+    /** CR4 guest/host mask. */
+    uint64_t                    u64Cr4Mask;
+    /** @} */
+
+    /** @name Host-virtual address of VMCS and related data structures.
+     *  @{ */
+    /** The VMCS. */
+    R0PTRTYPE(void *)           pvVmcs;
+    /** The shadow VMCS. */
+    R0PTRTYPE(void *)           pvShadowVmcs;
+    /** The virtual-APIC page. */
+    R0PTRTYPE(uint8_t *)        pbVirtApic;
+    /** The MSR bitmap. */
+    R0PTRTYPE(void *)           pvMsrBitmap;
+    /** The VM-entry MSR-load area. */
+    R0PTRTYPE(void *)           pvGuestMsrLoad;
+    /** The VM-exit MSR-store area. */
+    R0PTRTYPE(void *)           pvGuestMsrStore;
+    /** The VM-exit MSR-load area. */
+    R0PTRTYPE(void *)           pvHostMsrLoad;
+    /** @} */
+
+    /** @name Real-mode emulation state.
+     * @{ */
+    /** Set if guest was executing in real mode (extra checks). */
+    bool                        fWasInRealMode;
+    /** Set if the guest switched to 64-bit mode on a 32-bit host. */
+    bool                        fSwitchedTo64on32Obsolete;
+    /** Padding. */
+    bool                        afPadding0[6];
+    struct
+    {
+        X86DESCATTR             AttrCS;
+        X86DESCATTR             AttrDS;
+        X86DESCATTR             AttrES;
+        X86DESCATTR             AttrFS;
+        X86DESCATTR             AttrGS;
+        X86DESCATTR             AttrSS;
+        X86EFLAGS               Eflags;
+        bool                    fRealOnV86Active;
+        bool                    afPadding1[3];
+    } RealMode;
+    /** @} */
+
+    /** @name Host-physical address of VMCS and related data structures.
+     *  @{ */
+    /** The VMCS. */
+    RTHCPHYS                    HCPhysVmcs;
+    /** The shadow VMCS. */
+    RTHCPHYS                    HCPhysShadowVmcs;
+    /** The virtual APIC page. */
+    RTHCPHYS                    HCPhysVirtApic;
+    /** The MSR bitmap. */
+    RTHCPHYS                    HCPhysMsrBitmap;
+    /** The VM-entry MSR-load area. */
+    RTHCPHYS                    HCPhysGuestMsrLoad;
+    /** The VM-exit MSR-store area. */
+    RTHCPHYS                    HCPhysGuestMsrStore;
+    /** The VM-exit MSR-load area. */
+    RTHCPHYS                    HCPhysHostMsrLoad;
+    /** @} */
+
+    /** @name R0-memory objects address for VMCS and related data structures.
+     *  @{ */
+    /** R0-memory object for VMCS and related data structures. */
+    RTR0MEMOBJ                  hMemObj;
+    /** @} */
+
+    /** @name LBR MSR data.
+     *  @{ */
+    /** List of LastBranch-From-IP MSRs. */
+    uint64_t                    au64LbrFromIpMsr[32];
+    /** List of LastBranch-To-IP MSRs. */
+    uint64_t                    au64LbrToIpMsr[32];
+    /** The MSR containing the index to the most recent branch record.  */
+    uint64_t                    u64LbrTosMsr;
+    /** @} */
+} VMXVMCSINFO;
+/** Pointer to a VMXVMCSINFO struct. */
+typedef VMXVMCSINFO *PVMXVMCSINFO;
+/** Pointer to a const VMXVMCSINFO struct. */
+typedef const VMXVMCSINFO *PCVMXVMCSINFO;
+AssertCompileSizeAlignment(VMXVMCSINFO, 8);
+AssertCompileMemberAlignment(VMXVMCSINFO, pfnStartVM,      8);
+AssertCompileMemberAlignment(VMXVMCSINFO, u32PinCtls,      4);
+AssertCompileMemberAlignment(VMXVMCSINFO, u64VmcsLinkPtr,  8);
+AssertCompileMemberAlignment(VMXVMCSINFO, pvVmcs,          8);
+AssertCompileMemberAlignment(VMXVMCSINFO, pvShadowVmcs,    8);
+AssertCompileMemberAlignment(VMXVMCSINFO, pbVirtApic,      8);
+AssertCompileMemberAlignment(VMXVMCSINFO, pvMsrBitmap,     8);
+AssertCompileMemberAlignment(VMXVMCSINFO, pvGuestMsrLoad,  8);
+AssertCompileMemberAlignment(VMXVMCSINFO, pvGuestMsrStore, 8);
+AssertCompileMemberAlignment(VMXVMCSINFO, pvHostMsrLoad,   8);
+AssertCompileMemberAlignment(VMXVMCSINFO, HCPhysVmcs,      8);
+AssertCompileMemberAlignment(VMXVMCSINFO, hMemObj,         8);
+
+/**
+ * HM VMCPU Instance data.
+ *
+ * Note! If you change members of this struct, make sure to check if the
+ * assembly counterpart in HMInternal.mac needs to be updated as well.
+ *
+ * Note! The members here are ordered and aligned based on estimated frequency of
+ * usage and grouped to fit within a cache line in hot code paths. Even subtle
+ * changes here have a noticeable effect in the bootsector benchmarks. Modify with
+ * care.
+ */
+typedef struct HMCPU
+{
+    /** Set when the TLB has been checked until we return from the world switch. */
+    bool volatile               fCheckedTLBFlush;
+    /** Set when we're using VT-x or AMD-V at that moment. */
+    bool                        fActive;
+    /** Whether we've completed the inner HM leave function. */
+    bool                        fLeaveDone;
+    /** Whether we're using the hyper DR7 or guest DR7. */
+    bool                        fUsingHyperDR7;
+
+    /** Set if we need to flush the TLB during the world switch. */
+    bool                        fForceTLBFlush;
+    /** Whether we should use the debug loop because of single stepping or special
+     *  debug breakpoints / events are armed. */
+    bool                        fUseDebugLoop;
+    /** Whether we are currently executing in the debug loop.
+     *  Mainly for assertions. */
+    bool                        fUsingDebugLoop;
+    /** Set if we using the debug loop and wish to intercept RDTSC. */
+    bool                        fDebugWantRdTscExit;
+
+    /** Set if XCR0 needs to be saved/restored when entering/exiting guest code
+     *  execution. */
+    bool                        fLoadSaveGuestXcr0;
+    /** Whether \#UD needs to be intercepted (required by certain GIM providers). */
+    bool                        fGIMTrapXcptUD;
+    /** Whether \#GP needs to be intercept for mesa driver workaround. */
+    bool                        fTrapXcptGpForLovelyMesaDrv;
+    /** Whether we're executing a single instruction. */
+    bool                        fSingleInstruction;
+
+    /** Set if we need to clear the trap flag because of single stepping. */
+    bool                        fClearTrapFlag;
+    bool                        afAlignment0[3];
+
+    /** World switch exit counter. */
+    uint32_t volatile           cWorldSwitchExits;
+    /** The last CPU we were executing code on (NIL_RTCPUID for the first time). */
+    RTCPUID                     idLastCpu;
+    /** TLB flush count. */
+    uint32_t                    cTlbFlushes;
+    /** Current ASID in use by the VM. */
+    uint32_t                    uCurrentAsid;
+    /** An additional error code used for some gurus. */
+    uint32_t                    u32HMError;
+    /** The last exit-to-ring-3 reason. */
+    int32_t                     rcLastExitToR3;
+    /** CPU-context changed flags (see HM_CHANGED_xxx). */
+    uint64_t                    fCtxChanged;
+
+    union /* no tag! */
+    {
+        /** VT-x data.   */
+        struct
+        {
+            /** @name Guest information.
+             * @{ */
+            /** Guest VMCS information. */
+            VMXVMCSINFO                 VmcsInfo;
+            /** Nested-guest VMCS information. */
+            VMXVMCSINFO                 VmcsInfoNstGst;
+            /** Whether the nested-guest VMCS was the last current VMCS. */
+            bool                        fSwitchedToNstGstVmcs;
+            /** Whether the static guest VMCS controls has been merged with the
+             *  nested-guest VMCS controls. */
+            bool                        fMergedNstGstCtls;
+            /** Whether the nested-guest VMCS has been copied to the shadow VMCS. */
+            bool                        fCopiedNstGstToShadowVmcs;
+            /** Whether flushing the TLB is required due to switching to/from the
+             *  nested-guest. */
+            bool                        fSwitchedNstGstFlushTlb;
+            /** Alignment. */
+            bool                        afAlignment0[4];
+            /** Cached guest APIC-base MSR for identifying when to map the APIC-access page. */
+            uint64_t                    u64GstMsrApicBase;
+            /** @} */
+
+            /** @name Host information.
+             * @{ */
+            /** Host LSTAR MSR to restore lazily while leaving VT-x. */
+            uint64_t                    u64HostMsrLStar;
+            /** Host STAR MSR to restore lazily while leaving VT-x. */
+            uint64_t                    u64HostMsrStar;
+            /** Host SF_MASK MSR to restore lazily while leaving VT-x. */
+            uint64_t                    u64HostMsrSfMask;
+            /** Host KernelGS-Base MSR to restore lazily while leaving VT-x. */
+            uint64_t                    u64HostMsrKernelGsBase;
+            /** The mask of lazy MSRs swap/restore state, see VMX_LAZY_MSRS_XXX. */
+            uint32_t                    fLazyMsrs;
+            /** Whether the host MSR values are up-to-date in the auto-load/store MSR area. */
+            bool                        fUpdatedHostAutoMsrs;
+            /** Alignment. */
+            uint8_t                     au8Alignment0[3];
+            /** Which host-state bits to restore before being preempted. */
+            uint32_t                    fRestoreHostFlags;
+            /** Alignment. */
+            uint32_t                    u32Alignment0;
+            /** The host-state restoration structure. */
+            VMXRESTOREHOST              RestoreHost;
+            /** @} */
+
+            /** @name Error reporting and diagnostics.
+             * @{ */
+            /** VT-x error-reporting (mainly for ring-3 propagation). */
+            struct
+            {
+                RTCPUID                 idCurrentCpu;
+                RTCPUID                 idEnteredCpu;
+                RTHCPHYS                HCPhysCurrentVmcs;
+                uint32_t                u32VmcsRev;
+                uint32_t                u32InstrError;
+                uint32_t                u32ExitReason;
+                uint32_t                u32GuestIntrState;
+            } LastError;
+            /** @} */
+        } vmx;
+
+        /** SVM data. */
+        struct
+        {
+            /** Ring 0 handlers for VT-x. */
+            PFNHMSVMVMRUN               pfnVMRun;
+
+            /** Physical address of the host VMCB which holds additional host-state. */
+            RTHCPHYS                    HCPhysVmcbHost;
+            /** R0 memory object for the host VMCB which holds additional host-state. */
+            RTR0MEMOBJ                  hMemObjVmcbHost;
+            /** Padding. */
+            R0PTRTYPE(void *)           pvPadding;
+
+            /** Physical address of the guest VMCB. */
+            RTHCPHYS                    HCPhysVmcb;
+            /** R0 memory object for the guest VMCB. */
+            RTR0MEMOBJ                  hMemObjVmcb;
+            /** Pointer to the guest VMCB. */
+            R0PTRTYPE(PSVMVMCB)         pVmcb;
+
+            /** Physical address of the MSR bitmap (8 KB). */
+            RTHCPHYS                    HCPhysMsrBitmap;
+            /** R0 memory object for the MSR bitmap (8 KB). */
+            RTR0MEMOBJ                  hMemObjMsrBitmap;
+            /** Pointer to the MSR bitmap. */
+            R0PTRTYPE(void *)           pvMsrBitmap;
+
+            /** Whether VTPR with V_INTR_MASKING set is in effect, indicating
+             *  we should check if the VTPR changed on every VM-exit. */
+            bool                        fSyncVTpr;
+            uint8_t                     au8Alignment0[7];
+
+            /** Host's TSC_AUX MSR (used when RDTSCP doesn't cause VM-exits). */
+            uint64_t                    u64HostTscAux;
+
+            /** Cache of the nested-guest's VMCB fields that we modify in order to run the
+             *  nested-guest using AMD-V. This will be restored on \#VMEXIT. */
+            SVMNESTEDVMCBCACHE          NstGstVmcbCache;
+        } svm;
+    } HM_UNION_NM(u);
+
+    /** Event injection state. */
+    HMEVENT                 Event;
+
+    /** The CPU ID of the CPU currently owning the VMCS. Set in
+     * HMR0Enter and cleared in HMR0Leave. */
+    RTCPUID                 idEnteredCpu;
+
+    /** Current shadow paging mode for updating CR4. */
+    PGMMODE                 enmShadowMode;
+
+    /** The PAE PDPEs used with Nested Paging (only valid when
+     *  VMCPU_FF_HM_UPDATE_PAE_PDPES is set). */
+    X86PDPE                 aPdpes[4];
+
+    /** For saving stack space, the disassembler state is allocated here instead of
+     * on the stack. */
+    DISCPUSTATE             DisState;
+
+    STAMPROFILEADV          StatEntry;
+    STAMPROFILEADV          StatPreExit;
+    STAMPROFILEADV          StatExitHandling;
+    STAMPROFILEADV          StatExitIO;
+    STAMPROFILEADV          StatExitMovCRx;
+    STAMPROFILEADV          StatExitXcptNmi;
+    STAMPROFILEADV          StatExitVmentry;
+    STAMPROFILEADV          StatImportGuestState;
+    STAMPROFILEADV          StatExportGuestState;
+    STAMPROFILEADV          StatLoadGuestFpuState;
+    STAMPROFILEADV          StatInGC;
+    STAMPROFILEADV          StatPoke;
+    STAMPROFILEADV          StatSpinPoke;
+    STAMPROFILEADV          StatSpinPokeFailed;
+
+    STAMCOUNTER             StatInjectInterrupt;
+    STAMCOUNTER             StatInjectXcpt;
+    STAMCOUNTER             StatInjectReflect;
+    STAMCOUNTER             StatInjectConvertDF;
+    STAMCOUNTER             StatInjectInterpret;
+    STAMCOUNTER             StatInjectReflectNPF;
+
+    STAMCOUNTER             StatExitAll;
+    STAMCOUNTER             StatNestedExitAll;
+    STAMCOUNTER             StatExitShadowNM;
+    STAMCOUNTER             StatExitGuestNM;
+    STAMCOUNTER             StatExitShadowPF;       /**< Misleading, currently used for MMIO \#PFs as well. */
+    STAMCOUNTER             StatExitShadowPFEM;
+    STAMCOUNTER             StatExitGuestPF;
+    STAMCOUNTER             StatExitGuestUD;
+    STAMCOUNTER             StatExitGuestSS;
+    STAMCOUNTER             StatExitGuestNP;
+    STAMCOUNTER             StatExitGuestTS;
+    STAMCOUNTER             StatExitGuestOF;
+    STAMCOUNTER             StatExitGuestGP;
+    STAMCOUNTER             StatExitGuestDE;
+    STAMCOUNTER             StatExitGuestDF;
+    STAMCOUNTER             StatExitGuestBR;
+    STAMCOUNTER             StatExitGuestAC;
+    STAMCOUNTER             StatExitGuestDB;
+    STAMCOUNTER             StatExitGuestMF;
+    STAMCOUNTER             StatExitGuestBP;
+    STAMCOUNTER             StatExitGuestXF;
+    STAMCOUNTER             StatExitGuestXcpUnk;
+    STAMCOUNTER             StatExitDRxWrite;
+    STAMCOUNTER             StatExitDRxRead;
+    STAMCOUNTER             StatExitCR0Read;
+    STAMCOUNTER             StatExitCR2Read;
+    STAMCOUNTER             StatExitCR3Read;
+    STAMCOUNTER             StatExitCR4Read;
+    STAMCOUNTER             StatExitCR8Read;
+    STAMCOUNTER             StatExitCR0Write;
+    STAMCOUNTER             StatExitCR2Write;
+    STAMCOUNTER             StatExitCR3Write;
+    STAMCOUNTER             StatExitCR4Write;
+    STAMCOUNTER             StatExitCR8Write;
+    STAMCOUNTER             StatExitRdmsr;
+    STAMCOUNTER             StatExitWrmsr;
+    STAMCOUNTER             StatExitClts;
+    STAMCOUNTER             StatExitXdtrAccess;
+    STAMCOUNTER             StatExitLmsw;
+    STAMCOUNTER             StatExitIOWrite;
+    STAMCOUNTER             StatExitIORead;
+    STAMCOUNTER             StatExitIOStringWrite;
+    STAMCOUNTER             StatExitIOStringRead;
+    STAMCOUNTER             StatExitIntWindow;
+    STAMCOUNTER             StatExitExtInt;
+    STAMCOUNTER             StatExitHostNmiInGC;
+    STAMCOUNTER             StatExitHostNmiInGCIpi;
+    STAMCOUNTER             StatExitPreemptTimer;
+    STAMCOUNTER             StatExitTprBelowThreshold;
+    STAMCOUNTER             StatExitTaskSwitch;
+    STAMCOUNTER             StatExitApicAccess;
+    STAMCOUNTER             StatExitReasonNpf;
+
+    STAMCOUNTER             StatNestedExitReasonNpf;
+
+    STAMCOUNTER             StatFlushPage;
+    STAMCOUNTER             StatFlushPageManual;
+    STAMCOUNTER             StatFlushPhysPageManual;
+    STAMCOUNTER             StatFlushTlb;
+    STAMCOUNTER             StatFlushTlbNstGst;
+    STAMCOUNTER             StatFlushTlbManual;
+    STAMCOUNTER             StatFlushTlbWorldSwitch;
+    STAMCOUNTER             StatNoFlushTlbWorldSwitch;
+    STAMCOUNTER             StatFlushEntire;
+    STAMCOUNTER             StatFlushAsid;
+    STAMCOUNTER             StatFlushNestedPaging;
+    STAMCOUNTER             StatFlushTlbInvlpgVirt;
+    STAMCOUNTER             StatFlushTlbInvlpgPhys;
+    STAMCOUNTER             StatTlbShootdown;
+    STAMCOUNTER             StatTlbShootdownFlush;
+
+    STAMCOUNTER             StatSwitchPendingHostIrq;
+    STAMCOUNTER             StatSwitchTprMaskedIrq;
+    STAMCOUNTER             StatSwitchGuestIrq;
+    STAMCOUNTER             StatSwitchHmToR3FF;
+    STAMCOUNTER             StatSwitchVmReq;
+    STAMCOUNTER             StatSwitchPgmPoolFlush;
+    STAMCOUNTER             StatSwitchDma;
+    STAMCOUNTER             StatSwitchExitToR3;
+    STAMCOUNTER             StatSwitchLongJmpToR3;
+    STAMCOUNTER             StatSwitchMaxResumeLoops;
+    STAMCOUNTER             StatSwitchHltToR3;
+    STAMCOUNTER             StatSwitchApicAccessToR3;
+    STAMCOUNTER             StatSwitchPreempt;
+    STAMCOUNTER             StatSwitchNstGstVmexit;
+
+    STAMCOUNTER             StatTscParavirt;
+    STAMCOUNTER             StatTscOffset;
+    STAMCOUNTER             StatTscIntercept;
+
+    STAMCOUNTER             StatDRxArmed;
+    STAMCOUNTER             StatDRxContextSwitch;
+    STAMCOUNTER             StatDRxIoCheck;
+
+    STAMCOUNTER             StatExportMinimal;
+    STAMCOUNTER             StatExportFull;
+    STAMCOUNTER             StatLoadGuestFpu;
+    STAMCOUNTER             StatExportHostState;
+
+    STAMCOUNTER             StatVmxCheckBadRmSelBase;
+    STAMCOUNTER             StatVmxCheckBadRmSelLimit;
+    STAMCOUNTER             StatVmxCheckBadRmSelAttr;
+    STAMCOUNTER             StatVmxCheckBadV86SelBase;
+    STAMCOUNTER             StatVmxCheckBadV86SelLimit;
+    STAMCOUNTER             StatVmxCheckBadV86SelAttr;
+    STAMCOUNTER             StatVmxCheckRmOk;
+    STAMCOUNTER             StatVmxCheckBadSel;
+    STAMCOUNTER             StatVmxCheckBadRpl;
+    STAMCOUNTER             StatVmxCheckPmOk;
+
+#ifdef VBOX_WITH_STATISTICS
+    R3PTRTYPE(PSTAMCOUNTER) paStatExitReason;
+    R0PTRTYPE(PSTAMCOUNTER) paStatExitReasonR0;
+    R3PTRTYPE(PSTAMCOUNTER) paStatInjectedIrqs;
+    R0PTRTYPE(PSTAMCOUNTER) paStatInjectedIrqsR0;
+    R3PTRTYPE(PSTAMCOUNTER) paStatInjectedXcpts;
+    R0PTRTYPE(PSTAMCOUNTER) paStatInjectedXcptsR0;
+    R3PTRTYPE(PSTAMCOUNTER) paStatNestedExitReason;
+    R0PTRTYPE(PSTAMCOUNTER) paStatNestedExitReasonR0;
+#endif
+#ifdef HM_PROFILE_EXIT_DISPATCH
+    STAMPROFILEADV          StatExitDispatch;
+#endif
+} HMCPU;
+/** Pointer to HM VMCPU instance data. */
+typedef HMCPU *PHMCPU;
+AssertCompileMemberAlignment(HMCPU, fCheckedTLBFlush,  4);
+AssertCompileMemberAlignment(HMCPU, fForceTLBFlush,    4);
+AssertCompileMemberAlignment(HMCPU, cWorldSwitchExits, 4);
+AssertCompileMemberAlignment(HMCPU, fCtxChanged,       8);
+AssertCompileMemberAlignment(HMCPU, HM_UNION_NM(u.) vmx, 8);
+AssertCompileMemberAlignment(HMCPU, HM_UNION_NM(u.) vmx.VmcsInfo,       8);
+AssertCompileMemberAlignment(HMCPU, HM_UNION_NM(u.) vmx.VmcsInfoNstGst, 8);
+AssertCompileMemberAlignment(HMCPU, HM_UNION_NM(u.) vmx.RestoreHost,    8);
+AssertCompileMemberAlignment(HMCPU, HM_UNION_NM(u.) svm, 8);
+AssertCompileMemberAlignment(HMCPU, Event, 8);
+
+#ifdef IN_RING0
+VMMR0_INT_DECL(PHMPHYSCPU)  hmR0GetCurrentCpu(void);
+VMMR0_INT_DECL(int)         hmR0EnterCpu(PVMCPUCC pVCpu);
+
+# ifdef VBOX_STRICT
+#  define HM_DUMP_REG_FLAGS_GPRS      RT_BIT(0)
+#  define HM_DUMP_REG_FLAGS_FPU       RT_BIT(1)
+#  define HM_DUMP_REG_FLAGS_MSRS      RT_BIT(2)
+#  define HM_DUMP_REG_FLAGS_ALL       (HM_DUMP_REG_FLAGS_GPRS | HM_DUMP_REG_FLAGS_FPU | HM_DUMP_REG_FLAGS_MSRS)
+
+VMMR0_INT_DECL(void)        hmR0DumpRegs(PVMCPUCC pVCpu, uint32_t fFlags);
+VMMR0_INT_DECL(void)        hmR0DumpDescriptor(PCX86DESCHC pDesc, RTSEL Sel, const char *pszMsg);
+# endif
+
+# ifdef VBOX_WITH_KERNEL_USING_XMM
+DECLASM(int)                hmR0VMXStartVMWrapXMM(RTHCUINT fResume, PCPUMCTX pCtx, void *pvUnused, PVMCC pVM, PVMCPUCC pVCpu,
+                                                  PFNHMVMXSTARTVM pfnStartVM);
+DECLASM(int)                hmR0SVMRunWrapXMM(RTHCPHYS pVmcbHostPhys, RTHCPHYS pVmcbPhys, PCPUMCTX pCtx, PVMCC pVM, PVMCPUCC pVCpu,
+                                              PFNHMSVMVMRUN pfnVMRun);
+# endif
+DECLASM(void)               hmR0MdsClear(void);
+#endif /* IN_RING0 */
+
+VMM_INT_DECL(int)           hmEmulateSvmMovTpr(PVMCC pVM, PVMCPUCC pVCpu);
+
+VMM_INT_DECL(PVMXVMCSINFO)  hmGetVmxActiveVmcsInfo(PVMCPU pVCpu);
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_HMInternal_h */
+
diff --git a/src/VBox/VMM/include/HMInternal.mac b/src/VBox/VMM/include/HMInternal.mac
new file mode 100644
index 00000000..84ca1d6c
--- /dev/null
+++ b/src/VBox/VMM/include/HMInternal.mac
@@ -0,0 +1,45 @@
+;$Id: HMInternal.mac $
+;; @file
+; HM - Internal header file.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+struc HMCPU
+    .fCheckedTLBFlush               resb    1
+    .fActive                        resb    1
+    .fLeaveDone                     resb    1
+    .fUsingHyperDR7                 resb    1
+    .fForceTLBFlush                 resb    1
+    .fUseDebugLoop                  resb    1
+    .fUsingDebugLoop                resb    1
+    .fDebugWantRdTscExit            resb    1
+
+    .fLoadSaveGuestXcr0             resb    1
+    .fGIMTrapXcptUD                 resb    1
+    .fTrapXcptGpForLovelyMesaDrv    resb    1
+    .fSingleInstruction             resb    1
+    .fClearTrapFlag                 resb    1
+    alignb 8
+
+    .cWorldSwitchExits              resd    1
+    .idLastCpu                      resd    1
+    .cTlbFlushes                    resd    1
+    .uCurrentAsid                   resd    1
+    .u32HMError                     resd    1
+    .rcLastExitToR3                 resd    1
+    .fCtxChanged                    resq    1
+
+    ; incomplete to save unnecessary pain...
+endstruc
+
diff --git a/src/VBox/VMM/include/IEMInternal.h b/src/VBox/VMM/include/IEMInternal.h
new file mode 100644
index 00000000..fc622d7f
--- /dev/null
+++ b/src/VBox/VMM/include/IEMInternal.h
@@ -0,0 +1,1902 @@
+/* $Id: IEMInternal.h $ */
+/** @file
+ * IEM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_IEMInternal_h
+#define VMM_INCLUDED_SRC_include_IEMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/param.h>
+
+#include <setjmp.h>
+
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_iem_int       Internals
+ * @ingroup grp_iem
+ * @internal
+ * @{
+ */
+
+/** For expanding symbol in slickedit and other products tagging and
+ *  crossreferencing IEM symbols. */
+#ifndef IEM_STATIC
+# define IEM_STATIC static
+#endif
+
+/** @def IEM_WITH_3DNOW
+ * Includes the 3DNow decoding.  */
+#define IEM_WITH_3DNOW
+
+/** @def IEM_WITH_THREE_0F_38
+ * Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
+#define IEM_WITH_THREE_0F_38
+
+/** @def IEM_WITH_THREE_0F_3A
+ * Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
+#define IEM_WITH_THREE_0F_3A
+
+/** @def IEM_WITH_VEX
+ * Includes the VEX decoding. */
+#define IEM_WITH_VEX
+
+/** @def IEM_CFG_TARGET_CPU
+ * The minimum target CPU for the IEM emulation (IEMTARGETCPU_XXX value).
+ *
+ * By default we allow this to be configured by the user via the
+ * CPUM/GuestCpuName config string, but this comes at a slight cost during
+ * decoding.  So, for applications of this code where there is no need to
+ * be dynamic wrt target CPU, just modify this define.
+ */
+#if !defined(IEM_CFG_TARGET_CPU) || defined(DOXYGEN_RUNNING)
+# define IEM_CFG_TARGET_CPU     IEMTARGETCPU_DYNAMIC
+#endif
+
+
+//#define IEM_WITH_CODE_TLB// - work in progress
+
+
+#if !defined(IN_TSTVMSTRUCT) && !defined(DOXYGEN_RUNNING)
+/** Instruction statistics.   */
+typedef struct IEMINSTRSTATS
+{
+# define IEM_DO_INSTR_STAT(a_Name, a_szDesc) uint32_t a_Name;
+# include "IEMInstructionStatisticsTmpl.h"
+# undef IEM_DO_INSTR_STAT
+} IEMINSTRSTATS;
+#else
+struct IEMINSTRSTATS;
+typedef struct IEMINSTRSTATS IEMINSTRSTATS;
+#endif
+/** Pointer to IEM instruction statistics. */
+typedef IEMINSTRSTATS *PIEMINSTRSTATS;
+
+/** Finish and move to types.h */
+typedef union
+{
+    uint32_t u32;
+} RTFLOAT32U;
+typedef RTFLOAT32U *PRTFLOAT32U;
+typedef RTFLOAT32U const *PCRTFLOAT32U;
+
+
+/**
+ * Extended operand mode that includes a representation of 8-bit.
+ *
+ * This is used for packing down modes when invoking some C instruction
+ * implementations.
+ */
+typedef enum IEMMODEX
+{
+    IEMMODEX_16BIT = IEMMODE_16BIT,
+    IEMMODEX_32BIT = IEMMODE_32BIT,
+    IEMMODEX_64BIT = IEMMODE_64BIT,
+    IEMMODEX_8BIT
+} IEMMODEX;
+AssertCompileSize(IEMMODEX, 4);
+
+
+/**
+ * Branch types.
+ */
+typedef enum IEMBRANCH
+{
+    IEMBRANCH_JUMP = 1,
+    IEMBRANCH_CALL,
+    IEMBRANCH_TRAP,
+    IEMBRANCH_SOFTWARE_INT,
+    IEMBRANCH_HARDWARE_INT
+} IEMBRANCH;
+AssertCompileSize(IEMBRANCH, 4);
+
+
+/**
+ * INT instruction types.
+ */
+typedef enum IEMINT
+{
+    /** INT n instruction (opcode 0xcd imm). */
+    IEMINT_INTN  = 0,
+    /** Single byte INT3 instruction (opcode 0xcc). */
+    IEMINT_INT3  = IEM_XCPT_FLAGS_BP_INSTR,
+    /** Single byte INTO instruction (opcode 0xce). */
+    IEMINT_INTO  = IEM_XCPT_FLAGS_OF_INSTR,
+    /** Single byte INT1 (ICEBP) instruction (opcode 0xf1). */
+    IEMINT_INT1 = IEM_XCPT_FLAGS_ICEBP_INSTR
+} IEMINT;
+AssertCompileSize(IEMINT, 4);
+
+
+/**
+ * A FPU result.
+ */
+typedef struct IEMFPURESULT
+{
+    /** The output value. */
+    RTFLOAT80U      r80Result;
+    /** The output status. */
+    uint16_t        FSW;
+} IEMFPURESULT;
+AssertCompileMemberOffset(IEMFPURESULT, FSW, 10);
+/** Pointer to a FPU result. */
+typedef IEMFPURESULT *PIEMFPURESULT;
+/** Pointer to a const FPU result. */
+typedef IEMFPURESULT const *PCIEMFPURESULT;
+
+
+/**
+ * A FPU result consisting of two output values and FSW.
+ */
+typedef struct IEMFPURESULTTWO
+{
+    /** The first output value. */
+    RTFLOAT80U      r80Result1;
+    /** The output status. */
+    uint16_t        FSW;
+    /** The second output value. */
+    RTFLOAT80U      r80Result2;
+} IEMFPURESULTTWO;
+AssertCompileMemberOffset(IEMFPURESULTTWO, FSW, 10);
+AssertCompileMemberOffset(IEMFPURESULTTWO, r80Result2, 12);
+/** Pointer to a FPU result consisting of two output values and FSW. */
+typedef IEMFPURESULTTWO *PIEMFPURESULTTWO;
+/** Pointer to a const FPU result consisting of two output values and FSW. */
+typedef IEMFPURESULTTWO const *PCIEMFPURESULTTWO;
+
+
+/**
+ * IEM TLB entry.
+ *
+ * Lookup assembly:
+ * @code{.asm}
+        ; Calculate tag.
+        mov     rax, [VA]
+        shl     rax, 16
+        shr     rax, 16 + X86_PAGE_SHIFT
+        or      rax, [uTlbRevision]
+
+        ; Do indexing.
+        movzx   ecx, al
+        lea     rcx, [pTlbEntries + rcx]
+
+        ; Check tag.
+        cmp     [rcx + IEMTLBENTRY.uTag], rax
+        jne     .TlbMiss
+
+        ; Check access.
+        movsx   rax, ACCESS_FLAGS | MAPPING_R3_NOT_VALID | 0xffffff00
+        and     rax, [rcx + IEMTLBENTRY.fFlagsAndPhysRev]
+        cmp     rax, [uTlbPhysRev]
+        jne     .TlbMiss
+
+        ; Calc address and we're done.
+        mov     eax, X86_PAGE_OFFSET_MASK
+        and     eax, [VA]
+        or      rax, [rcx + IEMTLBENTRY.pMappingR3]
+    %ifdef VBOX_WITH_STATISTICS
+        inc     qword [cTlbHits]
+    %endif
+        jmp     .Done
+
+    .TlbMiss:
+        mov     r8d, ACCESS_FLAGS
+        mov     rdx, [VA]
+        mov     rcx, [pVCpu]
+        call    iemTlbTypeMiss
+    .Done:
+
+   @endcode
+ *
+ */
+typedef struct IEMTLBENTRY
+{
+    /** The TLB entry tag.
+     * Bits 35 thru 0 are made up of the virtual address shifted right 12 bits.
+     * Bits 63 thru 36 are made up of the TLB revision (zero means invalid).
+     *
+     * The TLB lookup code uses the current TLB revision, which won't ever be zero,
+     * enabling an extremely cheap TLB invalidation most of the time.  When the TLB
+     * revision wraps around though, the tags needs to be zeroed.
+     *
+     * @note    Try use SHRD instruction?  After seeing
+     *          https://gmplib.org/~tege/x86-timing.pdf, maybe not.
+     */
+    uint64_t                uTag;
+    /** Access flags and physical TLB revision.
+     *
+     * - Bit  0 - page tables   - not executable (X86_PTE_PAE_NX).
+     * - Bit  1 - page tables   - not writable (complemented X86_PTE_RW).
+     * - Bit  2 - page tables   - not user (complemented X86_PTE_US).
+     * - Bit  3 - pgm phys/virt - not directly writable.
+     * - Bit  4 - pgm phys page - not directly readable.
+     * - Bit  5 - currently unused.
+     * - Bit  6 - page tables   - not dirty (complemented X86_PTE_D).
+     * - Bit  7 - tlb entry     - pMappingR3 member not valid.
+     * - Bits 63 thru 8 are used for the physical TLB revision number.
+     *
+     * We're using complemented bit meanings here because it makes it easy to check
+     * whether special action is required.  For instance a user mode write access
+     * would do a "TEST fFlags, (X86_PTE_RW | X86_PTE_US | X86_PTE_D)" and a
+     * non-zero result would mean special handling needed because either it wasn't
+     * writable, or it wasn't user, or the page wasn't dirty.  A user mode read
+     * access would do "TEST fFlags, X86_PTE_US"; and a kernel mode read wouldn't
+     * need to check any PTE flag.
+     */
+    uint64_t                fFlagsAndPhysRev;
+    /** The guest physical page address. */
+    uint64_t                GCPhys;
+    /** Pointer to the ring-3 mapping (possibly also valid in ring-0). */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(uint8_t *)    pbMappingR3;
+#else
+    R3R0PTRTYPE(uint8_t *)  pbMappingR3;
+#endif
+#if HC_ARCH_BITS == 32
+    uint32_t                u32Padding1;
+#endif
+} IEMTLBENTRY;
+AssertCompileSize(IEMTLBENTRY, 32);
+/** Pointer to an IEM TLB entry. */
+typedef IEMTLBENTRY *PIEMTLBENTRY;
+
+/** @name IEMTLBE_F_XXX - TLB entry flags (IEMTLBENTRY::fFlagsAndPhysRev)
+ * @{  */
+#define IEMTLBE_F_PT_NO_EXEC        RT_BIT_64(0) /**< Page tables: Not executable. */
+#define IEMTLBE_F_PT_NO_WRITE       RT_BIT_64(1) /**< Page tables: Not writable. */
+#define IEMTLBE_F_PT_NO_USER        RT_BIT_64(2) /**< Page tables: Not user accessible (supervisor only). */
+#define IEMTLBE_F_PG_NO_WRITE       RT_BIT_64(3) /**< Phys page:   Not writable (access handler, ROM, whatever). */
+#define IEMTLBE_F_PG_NO_READ        RT_BIT_64(4) /**< Phys page:   Not readable (MMIO / access handler, ROM) */
+#define IEMTLBE_F_PT_NO_DIRTY       RT_BIT_64(5) /**< Page tables: Not dirty (needs to be made dirty on write). */
+#define IEMTLBE_F_NO_MAPPINGR3      RT_BIT_64(6) /**< TLB entry:   The IEMTLBENTRY::pMappingR3 member is invalid. */
+#define IEMTLBE_F_PHYS_REV          UINT64_C(0xffffffffffffff00) /**< Physical revision mask. */
+/** @} */
+
+
+/**
+ * An IEM TLB.
+ *
+ * We've got two of these, one for data and one for instructions.
+ */
+typedef struct IEMTLB
+{
+    /** The TLB entries.
+     * We've choosen 256 because that way we can obtain the result directly from a
+     * 8-bit register without an additional AND instruction. */
+    IEMTLBENTRY         aEntries[256];
+    /** The TLB revision.
+     * This is actually only 28 bits wide (see IEMTLBENTRY::uTag) and is incremented
+     * by adding RT_BIT_64(36) to it.  When it wraps around and becomes zero, all
+     * the tags in the TLB must be zeroed and the revision set to RT_BIT_64(36).
+     * (The revision zero indicates an invalid TLB entry.)
+     *
+     * The initial value is choosen to cause an early wraparound. */
+    uint64_t            uTlbRevision;
+    /** The TLB physical address revision - shadow of PGM variable.
+     *
+     * This is actually only 56 bits wide (see IEMTLBENTRY::fFlagsAndPhysRev) and is
+     * incremented by adding RT_BIT_64(8).  When it wraps around and becomes zero,
+     * a rendezvous is called and each CPU wipe the IEMTLBENTRY::pMappingR3 as well
+     * as IEMTLBENTRY::fFlagsAndPhysRev bits 63 thru 8, 4, and 3.
+     *
+     * The initial value is choosen to cause an early wraparound. */
+    uint64_t volatile   uTlbPhysRev;
+
+    /* Statistics: */
+
+    /** TLB hits (VBOX_WITH_STATISTICS only). */
+    uint64_t            cTlbHits;
+    /** TLB misses. */
+    uint32_t            cTlbMisses;
+    /** Slow read path.  */
+    uint32_t            cTlbSlowReadPath;
+#if 0
+    /** TLB misses because of tag mismatch. */
+    uint32_t            cTlbMissesTag;
+    /** TLB misses because of virtual access violation. */
+    uint32_t            cTlbMissesVirtAccess;
+    /** TLB misses because of dirty bit. */
+    uint32_t            cTlbMissesDirty;
+    /** TLB misses because of MMIO */
+    uint32_t            cTlbMissesMmio;
+    /** TLB misses because of write access handlers. */
+    uint32_t            cTlbMissesWriteHandler;
+    /** TLB misses because no r3(/r0) mapping. */
+    uint32_t            cTlbMissesMapping;
+#endif
+    /** Alignment padding. */
+    uint32_t            au32Padding[3+5];
+} IEMTLB;
+AssertCompileSizeAlignment(IEMTLB, 64);
+/** IEMTLB::uTlbRevision increment.  */
+#define IEMTLB_REVISION_INCR    RT_BIT_64(36)
+/** IEMTLB::uTlbPhysRev increment.  */
+#define IEMTLB_PHYS_REV_INCR    RT_BIT_64(8)
+
+
+/**
+ * The per-CPU IEM state.
+ */
+typedef struct IEMCPU
+{
+    /** Info status code that needs to be propagated to the IEM caller.
+     * This cannot be passed internally, as it would complicate all success
+     * checks within the interpreter making the code larger and almost impossible
+     * to get right.  Instead, we'll store status codes to pass on here.  Each
+     * source of these codes will perform appropriate sanity checks. */
+    int32_t                 rcPassUp;                                                                       /* 0x00 */
+
+    /** The current CPU execution mode (CS). */
+    IEMMODE                 enmCpuMode;                                                                     /* 0x04 */
+    /** The CPL. */
+    uint8_t                 uCpl;                                                                           /* 0x05 */
+
+    /** Whether to bypass access handlers or not. */
+    bool                    fBypassHandlers;                                                                /* 0x06 */
+    bool                    fUnusedWasInPatchCode;                                                          /* 0x07 */
+
+    /** @name Decoder state.
+     * @{ */
+#ifdef IEM_WITH_CODE_TLB
+    /** The offset of the next instruction byte. */
+    uint32_t                offInstrNextByte;                                                               /* 0x08 */
+    /** The number of bytes available at pbInstrBuf for the current instruction.
+     * This takes the max opcode length into account so that doesn't need to be
+     * checked separately. */
+    uint32_t                cbInstrBuf;                                                                     /* 0x0c */
+    /** Pointer to the page containing RIP, user specified buffer or abOpcode.
+     * This can be NULL if the page isn't mappable for some reason, in which
+     * case we'll do fallback stuff.
+     *
+     * If we're executing an instruction from a user specified buffer,
+     * IEMExecOneWithPrefetchedByPC and friends, this is not necessarily a page
+     * aligned pointer but pointer to the user data.
+     *
+     * For instructions crossing pages, this will start on the first page and be
+     * advanced to the next page by the time we've decoded the instruction.  This
+     * therefore precludes stuff like <tt>pbInstrBuf[offInstrNextByte + cbInstrBuf - cbCurInstr]</tt>
+     */
+    uint8_t const          *pbInstrBuf;                                                                     /* 0x10 */
+# if ARCH_BITS == 32
+    uint32_t                uInstrBufHigh; /** The high dword of the host context pbInstrBuf member. */
+# endif
+    /** The program counter corresponding to pbInstrBuf.
+     * This is set to a non-canonical address when we need to invalidate it. */
+    uint64_t                uInstrBufPc;                                                                    /* 0x18 */
+    /** The number of bytes available at pbInstrBuf in total (for IEMExecLots).
+     * This takes the CS segment limit into account. */
+    uint16_t                cbInstrBufTotal;                                                                /* 0x20 */
+    /** Offset into pbInstrBuf of the first byte of the current instruction.
+     * Can be negative to efficiently handle cross page instructions. */
+    int16_t                 offCurInstrStart;                                                               /* 0x22 */
+
+    /** The prefix mask (IEM_OP_PRF_XXX). */
+    uint32_t                fPrefixes;                                                                      /* 0x24 */
+    /** The extra REX ModR/M register field bit (REX.R << 3). */
+    uint8_t                 uRexReg;                                                                        /* 0x28 */
+    /** The extra REX ModR/M r/m field, SIB base and opcode reg bit
+     * (REX.B << 3). */
+    uint8_t                 uRexB;                                                                          /* 0x29 */
+    /** The extra REX SIB index field bit (REX.X << 3). */
+    uint8_t                 uRexIndex;                                                                      /* 0x2a */
+
+    /** The effective segment register (X86_SREG_XXX). */
+    uint8_t                 iEffSeg;                                                                        /* 0x2b */
+
+    /** The offset of the ModR/M byte relative to the start of the instruction. */
+    uint8_t                 offModRm;                                                                       /* 0x2c */
+#else
+    /** The size of what has currently been fetched into abOpcode. */
+    uint8_t                 cbOpcode;                                                                       /*       0x08 */
+    /** The current offset into abOpcode. */
+    uint8_t                 offOpcode;                                                                      /*       0x09 */
+    /** The offset of the ModR/M byte relative to the start of the instruction. */
+    uint8_t                 offModRm;                                                                       /*       0x0a */
+
+    /** The effective segment register (X86_SREG_XXX). */
+    uint8_t                 iEffSeg;                                                                        /*       0x0b */
+
+    /** The prefix mask (IEM_OP_PRF_XXX). */
+    uint32_t                fPrefixes;                                                                      /*       0x0c */
+    /** The extra REX ModR/M register field bit (REX.R << 3). */
+    uint8_t                 uRexReg;                                                                        /*       0x10 */
+    /** The extra REX ModR/M r/m field, SIB base and opcode reg bit
+     * (REX.B << 3). */
+    uint8_t                 uRexB;                                                                          /*       0x11 */
+    /** The extra REX SIB index field bit (REX.X << 3). */
+    uint8_t                 uRexIndex;                                                                      /*       0x12 */
+
+#endif
+
+    /** The effective operand mode. */
+    IEMMODE                 enmEffOpSize;                                                                   /* 0x2d, 0x13 */
+    /** The default addressing mode. */
+    IEMMODE                 enmDefAddrMode;                                                                 /* 0x2e, 0x14 */
+    /** The effective addressing mode. */
+    IEMMODE                 enmEffAddrMode;                                                                 /* 0x2f, 0x15 */
+    /** The default operand mode. */
+    IEMMODE                 enmDefOpSize;                                                                   /* 0x30, 0x16 */
+
+    /** Prefix index (VEX.pp) for two byte and three byte tables. */
+    uint8_t                 idxPrefix;                                                                      /* 0x31, 0x17 */
+    /** 3rd VEX/EVEX/XOP register.
+     * Please use IEM_GET_EFFECTIVE_VVVV to access.  */
+    uint8_t                 uVex3rdReg;                                                                     /* 0x32, 0x18 */
+    /** The VEX/EVEX/XOP length field. */
+    uint8_t                 uVexLength;                                                                     /* 0x33, 0x19 */
+    /** Additional EVEX stuff. */
+    uint8_t                 fEvexStuff;                                                                     /* 0x34, 0x1a */
+
+    /** Explicit alignment padding. */
+    uint8_t                 abAlignment2a[1];                                                               /* 0x35, 0x1b */
+    /** The FPU opcode (FOP). */
+    uint16_t                uFpuOpcode;                                                                     /* 0x36, 0x1c */
+#ifndef IEM_WITH_CODE_TLB
+    /** Explicit alignment padding. */
+    uint8_t                 abAlignment2b[2];                                                               /*       0x1e */
+#endif
+
+    /** The opcode bytes. */
+    uint8_t                 abOpcode[15];                                                                   /* 0x48, 0x20 */
+    /** Explicit alignment padding. */
+#ifdef IEM_WITH_CODE_TLB
+    uint8_t                 abAlignment2c[0x48 - 0x47];                                                     /* 0x37 */
+#else
+    uint8_t                 abAlignment2c[0x48 - 0x2f];                                                     /*       0x2f */
+#endif
+    /** @} */
+
+
+    /** The flags of the current exception / interrupt. */
+    uint32_t                fCurXcpt;                                                                       /* 0x48, 0x48 */
+    /** The current exception / interrupt. */
+    uint8_t                 uCurXcpt;
+    /** Exception / interrupt recursion depth. */
+    int8_t                  cXcptRecursions;
+
+    /** The number of active guest memory mappings. */
+    uint8_t                 cActiveMappings;
+    /** The next unused mapping index. */
+    uint8_t                 iNextMapping;
+    /** Records for tracking guest memory mappings. */
+    struct
+    {
+        /** The address of the mapped bytes. */
+        void               *pv;
+        /** The access flags (IEM_ACCESS_XXX).
+         * IEM_ACCESS_INVALID if the entry is unused. */
+        uint32_t            fAccess;
+#if HC_ARCH_BITS == 64
+        uint32_t            u32Alignment4; /**< Alignment padding. */
+#endif
+    } aMemMappings[3];
+
+    /** Locking records for the mapped memory. */
+    union
+    {
+        PGMPAGEMAPLOCK      Lock;
+        uint64_t            au64Padding[2];
+    } aMemMappingLocks[3];
+
+    /** Bounce buffer info.
+     * This runs in parallel to aMemMappings. */
+    struct
+    {
+        /** The physical address of the first byte. */
+        RTGCPHYS            GCPhysFirst;
+        /** The physical address of the second page. */
+        RTGCPHYS            GCPhysSecond;
+        /** The number of bytes in the first page. */
+        uint16_t            cbFirst;
+        /** The number of bytes in the second page. */
+        uint16_t            cbSecond;
+        /** Whether it's unassigned memory. */
+        bool                fUnassigned;
+        /** Explicit alignment padding. */
+        bool                afAlignment5[3];
+    } aMemBbMappings[3];
+
+    /** Bounce buffer storage.
+     * This runs in parallel to aMemMappings and aMemBbMappings. */
+    struct
+    {
+        uint8_t             ab[512];
+    } aBounceBuffers[3];
+
+
+    /** Pointer set jump buffer - ring-3 context. */
+    R3PTRTYPE(jmp_buf *)    pJmpBufR3;
+    /** Pointer set jump buffer - ring-0 context. */
+    R0PTRTYPE(jmp_buf *)    pJmpBufR0;
+
+    /** @todo Should move this near @a fCurXcpt later. */
+    /** The CR2 for the current exception / interrupt. */
+    uint64_t                uCurXcptCr2;
+    /** The error code for the current exception / interrupt. */
+    uint32_t                uCurXcptErr;
+    /** The VMX APIC-access page handler type. */
+    PGMPHYSHANDLERTYPE      hVmxApicAccessPage;
+
+    /** @name Statistics
+     * @{  */
+    /** The number of instructions we've executed. */
+    uint32_t                cInstructions;
+    /** The number of potential exits. */
+    uint32_t                cPotentialExits;
+    /** The number of bytes data or stack written (mostly for IEMExecOneEx).
+     * This may contain uncommitted writes.  */
+    uint32_t                cbWritten;
+    /** Counts the VERR_IEM_INSTR_NOT_IMPLEMENTED returns. */
+    uint32_t                cRetInstrNotImplemented;
+    /** Counts the VERR_IEM_ASPECT_NOT_IMPLEMENTED returns. */
+    uint32_t                cRetAspectNotImplemented;
+    /** Counts informational statuses returned (other than VINF_SUCCESS). */
+    uint32_t                cRetInfStatuses;
+    /** Counts other error statuses returned. */
+    uint32_t                cRetErrStatuses;
+    /** Number of times rcPassUp has been used. */
+    uint32_t                cRetPassUpStatus;
+    /** Number of times RZ left with instruction commit pending for ring-3. */
+    uint32_t                cPendingCommit;
+    /** Number of long jumps. */
+    uint32_t                cLongJumps;
+    /** @} */
+
+    /** @name Target CPU information.
+     * @{ */
+#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
+    /** The target CPU. */
+    uint32_t                uTargetCpu;
+#else
+    uint32_t                u32TargetCpuPadding;
+#endif
+    /** The CPU vendor. */
+    CPUMCPUVENDOR           enmCpuVendor;
+    /** @} */
+
+    /** @name Host CPU information.
+     * @{ */
+    /** The CPU vendor. */
+    CPUMCPUVENDOR           enmHostCpuVendor;
+    /** @} */
+
+    /** Counts RDMSR \#GP(0) LogRel(). */
+    uint8_t                 cLogRelRdMsr;
+    /** Counts WRMSR \#GP(0) LogRel(). */
+    uint8_t                 cLogRelWrMsr;
+    /** Alignment padding. */
+    uint8_t                 abAlignment8[50];
+
+    /** Data TLB.
+     * @remarks Must be 64-byte aligned. */
+    IEMTLB                  DataTlb;
+    /** Instruction TLB.
+     * @remarks Must be 64-byte aligned. */
+    IEMTLB                  CodeTlb;
+
+    /** Pointer to instruction statistics for ring-0 context. */
+    R0PTRTYPE(PIEMINSTRSTATS) pStatsR0;
+    /** Ring-3 pointer to instruction statistics for non-ring-3 code. */
+    R3PTRTYPE(PIEMINSTRSTATS) pStatsCCR3;
+    /** Pointer to instruction statistics for ring-3 context. */
+    R3PTRTYPE(PIEMINSTRSTATS) pStatsR3;
+} IEMCPU;
+AssertCompileMemberOffset(IEMCPU, fCurXcpt, 0x48);
+AssertCompileMemberAlignment(IEMCPU, DataTlb, 64);
+AssertCompileMemberAlignment(IEMCPU, CodeTlb, 64);
+/** Pointer to the per-CPU IEM state. */
+typedef IEMCPU *PIEMCPU;
+/** Pointer to the const per-CPU IEM state. */
+typedef IEMCPU const *PCIEMCPU;
+
+
+/** @def IEM_GET_CTX
+ * Gets the guest CPU context for the calling EMT.
+ * @returns PCPUMCTX
+ * @param   a_pVCpu The cross context virtual CPU structure of the calling thread.
+ */
+#define IEM_GET_CTX(a_pVCpu)                    (&(a_pVCpu)->cpum.GstCtx)
+
+/** @def IEM_CTX_ASSERT
+ * Asserts that the @a a_fExtrnMbz is present in the CPU context.
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling thread.
+ * @param   a_fExtrnMbz     The mask of CPUMCTX_EXTRN_XXX flags that must be zero.
+ */
+#define IEM_CTX_ASSERT(a_pVCpu, a_fExtrnMbz)    AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
+                                                          ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", (a_pVCpu)->cpum.GstCtx.fExtrn, \
+                                                          (a_fExtrnMbz)))
+
+/** @def IEM_CTX_IMPORT_RET
+ * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
+ *
+ * Will call the keep to import the bits as needed.
+ *
+ * Returns on import failure.
+ *
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling thread.
+ * @param   a_fExtrnImport  The mask of CPUMCTX_EXTRN_XXX flags to import.
+ */
+#define IEM_CTX_IMPORT_RET(a_pVCpu, a_fExtrnImport) \
+    do { \
+        if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
+        { /* likely */ } \
+        else \
+        { \
+            int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
+            AssertRCReturn(rcCtxImport, rcCtxImport); \
+        } \
+    } while (0)
+
+/** @def IEM_CTX_IMPORT_NORET
+ * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
+ *
+ * Will call the keep to import the bits as needed.
+ *
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling thread.
+ * @param   a_fExtrnImport  The mask of CPUMCTX_EXTRN_XXX flags to import.
+ */
+#define IEM_CTX_IMPORT_NORET(a_pVCpu, a_fExtrnImport) \
+    do { \
+        if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
+        { /* likely */ } \
+        else \
+        { \
+            int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
+            AssertLogRelRC(rcCtxImport); \
+        } \
+    } while (0)
+
+/** @def IEM_CTX_IMPORT_JMP
+ * Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
+ *
+ * Will call the keep to import the bits as needed.
+ *
+ * Jumps on import failure.
+ *
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling thread.
+ * @param   a_fExtrnImport  The mask of CPUMCTX_EXTRN_XXX flags to import.
+ */
+#define IEM_CTX_IMPORT_JMP(a_pVCpu, a_fExtrnImport) \
+    do { \
+        if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
+        { /* likely */ } \
+        else \
+        { \
+            int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
+            AssertRCStmt(rcCtxImport, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), rcCtxImport)); \
+        } \
+    } while (0)
+
+
+
+/** Gets the current IEMTARGETCPU value.
+ * @returns IEMTARGETCPU value.
+ * @param   a_pVCpu The cross context virtual CPU structure of the calling thread.
+ */
+#if IEM_CFG_TARGET_CPU != IEMTARGETCPU_DYNAMIC
+# define IEM_GET_TARGET_CPU(a_pVCpu)    (IEM_CFG_TARGET_CPU)
+#else
+# define IEM_GET_TARGET_CPU(a_pVCpu)    ((a_pVCpu)->iem.s.uTargetCpu)
+#endif
+
+/** @def Gets the instruction length. */
+#ifdef IEM_WITH_CODE_TLB
+# define IEM_GET_INSTR_LEN(a_pVCpu)     ((a_pVCpu)->iem.s.offInstrNextByte - (uint32_t)(int32_t)(a_pVCpu)->iem.s.offCurInstrStart)
+#else
+# define IEM_GET_INSTR_LEN(a_pVCpu)     ((a_pVCpu)->iem.s.offOpcode)
+#endif
+
+
+/** @name IEM_ACCESS_XXX - Access details.
+ * @{ */
+#define IEM_ACCESS_INVALID              UINT32_C(0x000000ff)
+#define IEM_ACCESS_TYPE_READ            UINT32_C(0x00000001)
+#define IEM_ACCESS_TYPE_WRITE           UINT32_C(0x00000002)
+#define IEM_ACCESS_TYPE_EXEC            UINT32_C(0x00000004)
+#define IEM_ACCESS_TYPE_MASK            UINT32_C(0x00000007)
+#define IEM_ACCESS_WHAT_CODE            UINT32_C(0x00000010)
+#define IEM_ACCESS_WHAT_DATA            UINT32_C(0x00000020)
+#define IEM_ACCESS_WHAT_STACK           UINT32_C(0x00000030)
+#define IEM_ACCESS_WHAT_SYS             UINT32_C(0x00000040)
+#define IEM_ACCESS_WHAT_MASK            UINT32_C(0x00000070)
+/** The writes are partial, so if initialize the bounce buffer with the
+ * orignal RAM content. */
+#define IEM_ACCESS_PARTIAL_WRITE        UINT32_C(0x00000100)
+/** Used in aMemMappings to indicate that the entry is bounce buffered. */
+#define IEM_ACCESS_BOUNCE_BUFFERED      UINT32_C(0x00000200)
+/** Bounce buffer with ring-3 write pending, first page. */
+#define IEM_ACCESS_PENDING_R3_WRITE_1ST UINT32_C(0x00000400)
+/** Bounce buffer with ring-3 write pending, second page. */
+#define IEM_ACCESS_PENDING_R3_WRITE_2ND UINT32_C(0x00000800)
+/** Valid bit mask. */
+#define IEM_ACCESS_VALID_MASK           UINT32_C(0x00000fff)
+/** Read+write data alias. */
+#define IEM_ACCESS_DATA_RW              (IEM_ACCESS_TYPE_READ  | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA)
+/** Write data alias. */
+#define IEM_ACCESS_DATA_W               (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA)
+/** Read data alias. */
+#define IEM_ACCESS_DATA_R               (IEM_ACCESS_TYPE_READ  | IEM_ACCESS_WHAT_DATA)
+/** Instruction fetch alias. */
+#define IEM_ACCESS_INSTRUCTION          (IEM_ACCESS_TYPE_EXEC  | IEM_ACCESS_WHAT_CODE)
+/** Stack write alias. */
+#define IEM_ACCESS_STACK_W              (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_STACK)
+/** Stack read alias. */
+#define IEM_ACCESS_STACK_R              (IEM_ACCESS_TYPE_READ  | IEM_ACCESS_WHAT_STACK)
+/** Stack read+write alias. */
+#define IEM_ACCESS_STACK_RW             (IEM_ACCESS_TYPE_READ  | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_STACK)
+/** Read system table alias. */
+#define IEM_ACCESS_SYS_R                (IEM_ACCESS_TYPE_READ  | IEM_ACCESS_WHAT_SYS)
+/** Read+write system table alias. */
+#define IEM_ACCESS_SYS_RW               (IEM_ACCESS_TYPE_READ  | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_SYS)
+/** @} */
+
+/** @name Prefix constants (IEMCPU::fPrefixes)
+ * @{ */
+#define IEM_OP_PRF_SEG_CS               RT_BIT_32(0)  /**< CS segment prefix (0x2e). */
+#define IEM_OP_PRF_SEG_SS               RT_BIT_32(1)  /**< SS segment prefix (0x36). */
+#define IEM_OP_PRF_SEG_DS               RT_BIT_32(2)  /**< DS segment prefix (0x3e). */
+#define IEM_OP_PRF_SEG_ES               RT_BIT_32(3)  /**< ES segment prefix (0x26). */
+#define IEM_OP_PRF_SEG_FS               RT_BIT_32(4)  /**< FS segment prefix (0x64). */
+#define IEM_OP_PRF_SEG_GS               RT_BIT_32(5)  /**< GS segment prefix (0x65). */
+#define IEM_OP_PRF_SEG_MASK             UINT32_C(0x3f)
+
+#define IEM_OP_PRF_SIZE_OP              RT_BIT_32(8)  /**< Operand size prefix (0x66). */
+#define IEM_OP_PRF_SIZE_REX_W           RT_BIT_32(9)  /**< REX.W prefix (0x48-0x4f). */
+#define IEM_OP_PRF_SIZE_ADDR            RT_BIT_32(10) /**< Address size prefix (0x67). */
+
+#define IEM_OP_PRF_LOCK                 RT_BIT_32(16) /**< Lock prefix (0xf0). */
+#define IEM_OP_PRF_REPNZ                RT_BIT_32(17) /**< Repeat-not-zero prefix (0xf2). */
+#define IEM_OP_PRF_REPZ                 RT_BIT_32(18) /**< Repeat-if-zero prefix (0xf3). */
+
+#define IEM_OP_PRF_REX                  RT_BIT_32(24) /**< Any REX prefix (0x40-0x4f). */
+#define IEM_OP_PRF_REX_R                RT_BIT_32(25) /**< REX.R prefix (0x44,0x45,0x46,0x47,0x4c,0x4d,0x4e,0x4f). */
+#define IEM_OP_PRF_REX_B                RT_BIT_32(26) /**< REX.B prefix (0x41,0x43,0x45,0x47,0x49,0x4b,0x4d,0x4f). */
+#define IEM_OP_PRF_REX_X                RT_BIT_32(27) /**< REX.X prefix (0x42,0x43,0x46,0x47,0x4a,0x4b,0x4e,0x4f). */
+/** Mask with all the REX prefix flags.
+ * This is generally for use when needing to undo the REX prefixes when they
+ * are followed legacy prefixes and therefore does not immediately preceed
+ * the first opcode byte.
+ * For testing whether any REX prefix is present, use  IEM_OP_PRF_REX instead. */
+#define IEM_OP_PRF_REX_MASK  (IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W )
+
+#define IEM_OP_PRF_VEX                  RT_BIT_32(28) /**< Indiciates VEX prefix. */
+#define IEM_OP_PRF_EVEX                 RT_BIT_32(29) /**< Indiciates EVEX prefix. */
+#define IEM_OP_PRF_XOP                  RT_BIT_32(30) /**< Indiciates XOP prefix. */
+/** @} */
+
+/** @name IEMOPFORM_XXX - Opcode forms
+ * @note These are ORed together with IEMOPHINT_XXX.
+ * @{ */
+/** ModR/M: reg, r/m */
+#define IEMOPFORM_RM            0
+/** ModR/M: reg, r/m (register) */
+#define IEMOPFORM_RM_REG        (IEMOPFORM_RM | IEMOPFORM_MOD3)
+/** ModR/M: reg, r/m (memory)   */
+#define IEMOPFORM_RM_MEM        (IEMOPFORM_RM | IEMOPFORM_NOT_MOD3)
+/** ModR/M: r/m, reg */
+#define IEMOPFORM_MR            1
+/** ModR/M: r/m (register), reg */
+#define IEMOPFORM_MR_REG        (IEMOPFORM_MR | IEMOPFORM_MOD3)
+/** ModR/M: r/m (memory), reg */
+#define IEMOPFORM_MR_MEM        (IEMOPFORM_MR | IEMOPFORM_NOT_MOD3)
+/** ModR/M: r/m only */
+#define IEMOPFORM_M             2
+/** ModR/M: r/m only (register). */
+#define IEMOPFORM_M_REG         (IEMOPFORM_M | IEMOPFORM_MOD3)
+/** ModR/M: r/m only (memory). */
+#define IEMOPFORM_M_MEM         (IEMOPFORM_M | IEMOPFORM_NOT_MOD3)
+/** ModR/M: reg only */
+#define IEMOPFORM_R             3
+
+/** VEX+ModR/M: reg, r/m */
+#define IEMOPFORM_VEX_RM        4
+/** VEX+ModR/M: reg, r/m (register) */
+#define IEMOPFORM_VEX_RM_REG    (IEMOPFORM_VEX_RM | IEMOPFORM_MOD3)
+/** VEX+ModR/M: reg, r/m (memory)   */
+#define IEMOPFORM_VEX_RM_MEM    (IEMOPFORM_VEX_RM | IEMOPFORM_NOT_MOD3)
+/** VEX+ModR/M: r/m, reg */
+#define IEMOPFORM_VEX_MR        5
+/** VEX+ModR/M: r/m (register), reg */
+#define IEMOPFORM_VEX_MR_REG    (IEMOPFORM_VEX_MR | IEMOPFORM_MOD3)
+/** VEX+ModR/M: r/m (memory), reg */
+#define IEMOPFORM_VEX_MR_MEM    (IEMOPFORM_VEX_MR | IEMOPFORM_NOT_MOD3)
+/** VEX+ModR/M: r/m only */
+#define IEMOPFORM_VEX_M         6
+/** VEX+ModR/M: r/m only (register). */
+#define IEMOPFORM_VEX_M_REG     (IEMOPFORM_VEX_M | IEMOPFORM_MOD3)
+/** VEX+ModR/M: r/m only (memory). */
+#define IEMOPFORM_VEX_M_MEM     (IEMOPFORM_VEX_M | IEMOPFORM_NOT_MOD3)
+/** VEX+ModR/M: reg only */
+#define IEMOPFORM_VEX_R         7
+/** VEX+ModR/M: reg, vvvv, r/m */
+#define IEMOPFORM_VEX_RVM       8
+/** VEX+ModR/M: reg, vvvv, r/m (register). */
+#define IEMOPFORM_VEX_RVM_REG   (IEMOPFORM_VEX_RVM | IEMOPFORM_MOD3)
+/** VEX+ModR/M: reg, vvvv, r/m (memory). */
+#define IEMOPFORM_VEX_RVM_MEM   (IEMOPFORM_VEX_RVM | IEMOPFORM_NOT_MOD3)
+/** VEX+ModR/M: r/m, vvvv, reg */
+#define IEMOPFORM_VEX_MVR       9
+/** VEX+ModR/M: r/m, vvvv, reg (register) */
+#define IEMOPFORM_VEX_MVR_REG   (IEMOPFORM_VEX_MVR | IEMOPFORM_MOD3)
+/** VEX+ModR/M: r/m, vvvv, reg (memory) */
+#define IEMOPFORM_VEX_MVR_MEM   (IEMOPFORM_VEX_MVR | IEMOPFORM_NOT_MOD3)
+
+/** Fixed register instruction, no R/M. */
+#define IEMOPFORM_FIXED         16
+
+/** The r/m is a register. */
+#define IEMOPFORM_MOD3          RT_BIT_32(8)
+/** The r/m is a memory access. */
+#define IEMOPFORM_NOT_MOD3      RT_BIT_32(9)
+/** @} */
+
+/** @name IEMOPHINT_XXX - Additional Opcode Hints
+ * @note These are ORed together with IEMOPFORM_XXX.
+ * @{ */
+/** Ignores the operand size prefix (66h). */
+#define IEMOPHINT_IGNORES_OZ_PFX    RT_BIT_32(10)
+/** Ignores REX.W (aka WIG). */
+#define IEMOPHINT_IGNORES_REXW      RT_BIT_32(11)
+/** Both the operand size prefixes (66h + REX.W) are ignored. */
+#define IEMOPHINT_IGNORES_OP_SIZES  (IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_IGNORES_REXW)
+/** Allowed with the lock prefix. */
+#define IEMOPHINT_LOCK_ALLOWED      RT_BIT_32(11)
+/** The VEX.L value is ignored (aka LIG). */
+#define IEMOPHINT_VEX_L_IGNORED     RT_BIT_32(12)
+/** The VEX.L value must be zero (i.e. 128-bit width only). */
+#define IEMOPHINT_VEX_L_ZERO        RT_BIT_32(13)
+
+/** Hint to IEMAllInstructionPython.py that this macro should be skipped.  */
+#define IEMOPHINT_SKIP_PYTHON       RT_BIT_32(31)
+/** @} */
+
+/**
+ * Possible hardware task switch sources.
+ */
+typedef enum IEMTASKSWITCH
+{
+    /** Task switch caused by an interrupt/exception. */
+    IEMTASKSWITCH_INT_XCPT = 1,
+    /** Task switch caused by a far CALL. */
+    IEMTASKSWITCH_CALL,
+    /** Task switch caused by a far JMP. */
+    IEMTASKSWITCH_JUMP,
+    /** Task switch caused by an IRET. */
+    IEMTASKSWITCH_IRET
+} IEMTASKSWITCH;
+AssertCompileSize(IEMTASKSWITCH, 4);
+
+/**
+ * Possible CrX load (write) sources.
+ */
+typedef enum IEMACCESSCRX
+{
+    /** CrX access caused by 'mov crX' instruction. */
+    IEMACCESSCRX_MOV_CRX,
+    /** CrX (CR0) write caused by 'lmsw' instruction. */
+    IEMACCESSCRX_LMSW,
+    /** CrX (CR0) write caused by 'clts' instruction. */
+    IEMACCESSCRX_CLTS,
+    /** CrX (CR0) read caused by 'smsw' instruction. */
+    IEMACCESSCRX_SMSW
+} IEMACCESSCRX;
+
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) iemVmxApicAccessPageHandler;
+# endif
+
+/**
+ * Indicates to the verifier that the given flag set is undefined.
+ *
+ * Can be invoked again to add more flags.
+ *
+ * This is a NOOP if the verifier isn't compiled in.
+ *
+ * @note We're temporarily keeping this until code is converted to new
+ *       disassembler style opcode handling.
+ */
+#define IEMOP_VERIFICATION_UNDEFINED_EFLAGS(a_fEfl) do { } while (0)
+
+
+/** @def IEM_DECL_IMPL_TYPE
+ * For typedef'ing an instruction implementation function.
+ *
+ * @param   a_RetType           The return type.
+ * @param   a_Name              The name of the type.
+ * @param   a_ArgList           The argument list enclosed in parentheses.
+ */
+
+/** @def IEM_DECL_IMPL_DEF
+ * For defining an instruction implementation function.
+ *
+ * @param   a_RetType           The return type.
+ * @param   a_Name              The name of the type.
+ * @param   a_ArgList           The argument list enclosed in parentheses.
+ */
+
+#if defined(__GNUC__) && defined(RT_ARCH_X86)
+# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
+    __attribute__((__fastcall__)) a_RetType (a_Name) a_ArgList
+# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
+    __attribute__((__fastcall__, __nothrow__)) a_RetType a_Name a_ArgList
+
+#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
+# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
+    a_RetType (__fastcall a_Name) a_ArgList
+# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
+    a_RetType __fastcall a_Name a_ArgList
+
+#else
+# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
+    a_RetType (VBOXCALL a_Name) a_ArgList
+# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
+    a_RetType VBOXCALL a_Name a_ArgList
+
+#endif
+
+/** @name Arithmetic assignment operations on bytes (binary).
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU8,  (uint8_t  *pu8Dst,  uint8_t  u8Src,  uint32_t *pEFlags));
+typedef FNIEMAIMPLBINU8  *PFNIEMAIMPLBINU8;
+FNIEMAIMPLBINU8 iemAImpl_add_u8, iemAImpl_add_u8_locked;
+FNIEMAIMPLBINU8 iemAImpl_adc_u8, iemAImpl_adc_u8_locked;
+FNIEMAIMPLBINU8 iemAImpl_sub_u8, iemAImpl_sub_u8_locked;
+FNIEMAIMPLBINU8 iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked;
+FNIEMAIMPLBINU8  iemAImpl_or_u8,  iemAImpl_or_u8_locked;
+FNIEMAIMPLBINU8 iemAImpl_xor_u8, iemAImpl_xor_u8_locked;
+FNIEMAIMPLBINU8 iemAImpl_and_u8, iemAImpl_and_u8_locked;
+/** @} */
+
+/** @name Arithmetic assignment operations on words (binary).
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU16,  (uint16_t *pu16Dst, uint16_t u16Src, uint32_t *pEFlags));
+typedef FNIEMAIMPLBINU16  *PFNIEMAIMPLBINU16;
+FNIEMAIMPLBINU16 iemAImpl_add_u16, iemAImpl_add_u16_locked;
+FNIEMAIMPLBINU16 iemAImpl_adc_u16, iemAImpl_adc_u16_locked;
+FNIEMAIMPLBINU16 iemAImpl_sub_u16, iemAImpl_sub_u16_locked;
+FNIEMAIMPLBINU16 iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked;
+FNIEMAIMPLBINU16  iemAImpl_or_u16,  iemAImpl_or_u16_locked;
+FNIEMAIMPLBINU16 iemAImpl_xor_u16, iemAImpl_xor_u16_locked;
+FNIEMAIMPLBINU16 iemAImpl_and_u16, iemAImpl_and_u16_locked;
+/** @}  */
+
+/** @name Arithmetic assignment operations on double words (binary).
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU32, (uint32_t *pu32Dst, uint32_t u32Src, uint32_t *pEFlags));
+typedef FNIEMAIMPLBINU32 *PFNIEMAIMPLBINU32;
+FNIEMAIMPLBINU32 iemAImpl_add_u32, iemAImpl_add_u32_locked;
+FNIEMAIMPLBINU32 iemAImpl_adc_u32, iemAImpl_adc_u32_locked;
+FNIEMAIMPLBINU32 iemAImpl_sub_u32, iemAImpl_sub_u32_locked;
+FNIEMAIMPLBINU32 iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked;
+FNIEMAIMPLBINU32  iemAImpl_or_u32,  iemAImpl_or_u32_locked;
+FNIEMAIMPLBINU32 iemAImpl_xor_u32, iemAImpl_xor_u32_locked;
+FNIEMAIMPLBINU32 iemAImpl_and_u32, iemAImpl_and_u32_locked;
+/** @}  */
+
+/** @name Arithmetic assignment operations on quad words (binary).
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU64, (uint64_t *pu64Dst, uint64_t u64Src, uint32_t *pEFlags));
+typedef FNIEMAIMPLBINU64 *PFNIEMAIMPLBINU64;
+FNIEMAIMPLBINU64 iemAImpl_add_u64, iemAImpl_add_u64_locked;
+FNIEMAIMPLBINU64 iemAImpl_adc_u64, iemAImpl_adc_u64_locked;
+FNIEMAIMPLBINU64 iemAImpl_sub_u64, iemAImpl_sub_u64_locked;
+FNIEMAIMPLBINU64 iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked;
+FNIEMAIMPLBINU64  iemAImpl_or_u64,  iemAImpl_or_u64_locked;
+FNIEMAIMPLBINU64 iemAImpl_xor_u64, iemAImpl_xor_u64_locked;
+FNIEMAIMPLBINU64 iemAImpl_and_u64, iemAImpl_and_u64_locked;
+/** @}  */
+
+/** @name Compare operations (thrown in with the binary ops).
+ * @{ */
+FNIEMAIMPLBINU8  iemAImpl_cmp_u8;
+FNIEMAIMPLBINU16 iemAImpl_cmp_u16;
+FNIEMAIMPLBINU32 iemAImpl_cmp_u32;
+FNIEMAIMPLBINU64 iemAImpl_cmp_u64;
+/** @}  */
+
+/** @name Test operations (thrown in with the binary ops).
+ * @{ */
+FNIEMAIMPLBINU8  iemAImpl_test_u8;
+FNIEMAIMPLBINU16 iemAImpl_test_u16;
+FNIEMAIMPLBINU32 iemAImpl_test_u32;
+FNIEMAIMPLBINU64 iemAImpl_test_u64;
+/** @}  */
+
+/** @name Bit operations operations (thrown in with the binary ops).
+ * @{ */
+FNIEMAIMPLBINU16 iemAImpl_bt_u16,  iemAImpl_bt_u16_locked;
+FNIEMAIMPLBINU32 iemAImpl_bt_u32,  iemAImpl_bt_u32_locked;
+FNIEMAIMPLBINU64 iemAImpl_bt_u64,  iemAImpl_bt_u64_locked;
+FNIEMAIMPLBINU16 iemAImpl_btc_u16, iemAImpl_btc_u16_locked;
+FNIEMAIMPLBINU32 iemAImpl_btc_u32, iemAImpl_btc_u32_locked;
+FNIEMAIMPLBINU64 iemAImpl_btc_u64, iemAImpl_btc_u64_locked;
+FNIEMAIMPLBINU16 iemAImpl_btr_u16, iemAImpl_btr_u16_locked;
+FNIEMAIMPLBINU32 iemAImpl_btr_u32, iemAImpl_btr_u32_locked;
+FNIEMAIMPLBINU64 iemAImpl_btr_u64, iemAImpl_btr_u64_locked;
+FNIEMAIMPLBINU16 iemAImpl_bts_u16, iemAImpl_bts_u16_locked;
+FNIEMAIMPLBINU32 iemAImpl_bts_u32, iemAImpl_bts_u32_locked;
+FNIEMAIMPLBINU64 iemAImpl_bts_u64, iemAImpl_bts_u64_locked;
+/** @}  */
+
+/** @name Exchange memory with register operations.
+ * @{ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8, (uint8_t  *pu8Mem,  uint8_t  *pu8Reg));
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16,(uint16_t *pu16Mem, uint16_t *pu16Reg));
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32,(uint32_t *pu32Mem, uint32_t *pu32Reg));
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64,(uint64_t *pu64Mem, uint64_t *pu64Reg));
+/** @}  */
+
+/** @name Exchange and add operations.
+ * @{ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u8, (uint8_t  *pu8Dst,  uint8_t  *pu8Reg,  uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u16,(uint16_t *pu16Dst, uint16_t *pu16Reg, uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u32,(uint32_t *pu32Dst, uint32_t *pu32Reg, uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u64,(uint64_t *pu64Dst, uint64_t *pu64Reg, uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u8_locked, (uint8_t  *pu8Dst,  uint8_t  *pu8Reg,  uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u16_locked,(uint16_t *pu16Dst, uint16_t *pu16Reg, uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u32_locked,(uint32_t *pu32Dst, uint32_t *pu32Reg, uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u64_locked,(uint64_t *pu64Dst, uint64_t *pu64Reg, uint32_t *pEFlags));
+/** @}  */
+
+/** @name Compare and exchange.
+ * @{ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8,        (uint8_t  *pu8Dst,  uint8_t  *puAl,  uint8_t  uSrcReg, uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8_locked, (uint8_t  *pu8Dst,  uint8_t  *puAl,  uint8_t  uSrcReg, uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16,       (uint16_t *pu16Dst, uint16_t *puAx,  uint16_t uSrcReg, uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16_locked,(uint16_t *pu16Dst, uint16_t *puAx,  uint16_t uSrcReg, uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32,       (uint32_t *pu32Dst, uint32_t *puEax, uint32_t uSrcReg, uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32_locked,(uint32_t *pu32Dst, uint32_t *puEax, uint32_t uSrcReg, uint32_t *pEFlags));
+#ifdef RT_ARCH_X86
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64,       (uint64_t *pu64Dst, uint64_t *puRax, uint64_t *puSrcReg, uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t *pu64Dst, uint64_t *puRax, uint64_t *puSrcReg, uint32_t *pEFlags));
+#else
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64,       (uint64_t *pu64Dst, uint64_t *puRax, uint64_t uSrcReg, uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t *pu64Dst, uint64_t *puRax, uint64_t uSrcReg, uint32_t *pEFlags));
+#endif
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
+                                            uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b_locked,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
+                                                   uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
+                                             uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_locked,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
+                                                    uint32_t *pEFlags));
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_fallback,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx,
+                                                      PRTUINT128U pu128RbxRcx, uint32_t *pEFlags));
+/** @} */
+
+/** @name Memory ordering
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEMFENCE,(void));
+typedef FNIEMAIMPLMEMFENCE *PFNIEMAIMPLMEMFENCE;
+IEM_DECL_IMPL_DEF(void, iemAImpl_mfence,(void));
+IEM_DECL_IMPL_DEF(void, iemAImpl_sfence,(void));
+IEM_DECL_IMPL_DEF(void, iemAImpl_lfence,(void));
+IEM_DECL_IMPL_DEF(void, iemAImpl_alt_mem_fence,(void));
+/** @} */
+
+/** @name Double precision shifts
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU16,(uint16_t *pu16Dst, uint16_t u16Src, uint8_t cShift, uint32_t *pEFlags));
+typedef FNIEMAIMPLSHIFTDBLU16  *PFNIEMAIMPLSHIFTDBLU16;
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU32,(uint32_t *pu32Dst, uint32_t u32Src, uint8_t cShift, uint32_t *pEFlags));
+typedef FNIEMAIMPLSHIFTDBLU32  *PFNIEMAIMPLSHIFTDBLU32;
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU64,(uint64_t *pu64Dst, uint64_t u64Src, uint8_t cShift, uint32_t *pEFlags));
+typedef FNIEMAIMPLSHIFTDBLU64  *PFNIEMAIMPLSHIFTDBLU64;
+FNIEMAIMPLSHIFTDBLU16 iemAImpl_shld_u16;
+FNIEMAIMPLSHIFTDBLU32 iemAImpl_shld_u32;
+FNIEMAIMPLSHIFTDBLU64 iemAImpl_shld_u64;
+FNIEMAIMPLSHIFTDBLU16 iemAImpl_shrd_u16;
+FNIEMAIMPLSHIFTDBLU32 iemAImpl_shrd_u32;
+FNIEMAIMPLSHIFTDBLU64 iemAImpl_shrd_u64;
+/** @}  */
+
+
+/** @name Bit search operations (thrown in with the binary ops).
+ * @{ */
+FNIEMAIMPLBINU16 iemAImpl_bsf_u16;
+FNIEMAIMPLBINU32 iemAImpl_bsf_u32;
+FNIEMAIMPLBINU64 iemAImpl_bsf_u64;
+FNIEMAIMPLBINU16 iemAImpl_bsr_u16;
+FNIEMAIMPLBINU32 iemAImpl_bsr_u32;
+FNIEMAIMPLBINU64 iemAImpl_bsr_u64;
+/** @}  */
+
+/** @name Signed multiplication operations (thrown in with the binary ops).
+ * @{ */
+FNIEMAIMPLBINU16 iemAImpl_imul_two_u16;
+FNIEMAIMPLBINU32 iemAImpl_imul_two_u32;
+FNIEMAIMPLBINU64 iemAImpl_imul_two_u64;
+/** @}  */
+
+/** @name Arithmetic assignment operations on bytes (unary).
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU8,  (uint8_t  *pu8Dst,  uint32_t *pEFlags));
+typedef FNIEMAIMPLUNARYU8  *PFNIEMAIMPLUNARYU8;
+FNIEMAIMPLUNARYU8 iemAImpl_inc_u8, iemAImpl_inc_u8_locked;
+FNIEMAIMPLUNARYU8 iemAImpl_dec_u8, iemAImpl_dec_u8_locked;
+FNIEMAIMPLUNARYU8 iemAImpl_not_u8, iemAImpl_not_u8_locked;
+FNIEMAIMPLUNARYU8 iemAImpl_neg_u8, iemAImpl_neg_u8_locked;
+/** @} */
+
+/** @name Arithmetic assignment operations on words (unary).
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU16,  (uint16_t  *pu16Dst,  uint32_t *pEFlags));
+typedef FNIEMAIMPLUNARYU16  *PFNIEMAIMPLUNARYU16;
+FNIEMAIMPLUNARYU16 iemAImpl_inc_u16, iemAImpl_inc_u16_locked;
+FNIEMAIMPLUNARYU16 iemAImpl_dec_u16, iemAImpl_dec_u16_locked;
+FNIEMAIMPLUNARYU16 iemAImpl_not_u16, iemAImpl_not_u16_locked;
+FNIEMAIMPLUNARYU16 iemAImpl_neg_u16, iemAImpl_neg_u16_locked;
+/** @} */
+
+/** @name Arithmetic assignment operations on double words (unary).
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU32,  (uint32_t  *pu32Dst,  uint32_t *pEFlags));
+typedef FNIEMAIMPLUNARYU32  *PFNIEMAIMPLUNARYU32;
+FNIEMAIMPLUNARYU32 iemAImpl_inc_u32, iemAImpl_inc_u32_locked;
+FNIEMAIMPLUNARYU32 iemAImpl_dec_u32, iemAImpl_dec_u32_locked;
+FNIEMAIMPLUNARYU32 iemAImpl_not_u32, iemAImpl_not_u32_locked;
+FNIEMAIMPLUNARYU32 iemAImpl_neg_u32, iemAImpl_neg_u32_locked;
+/** @} */
+
+/** @name Arithmetic assignment operations on quad words (unary).
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU64,  (uint64_t  *pu64Dst,  uint32_t *pEFlags));
+typedef FNIEMAIMPLUNARYU64  *PFNIEMAIMPLUNARYU64;
+FNIEMAIMPLUNARYU64 iemAImpl_inc_u64, iemAImpl_inc_u64_locked;
+FNIEMAIMPLUNARYU64 iemAImpl_dec_u64, iemAImpl_dec_u64_locked;
+FNIEMAIMPLUNARYU64 iemAImpl_not_u64, iemAImpl_not_u64_locked;
+FNIEMAIMPLUNARYU64 iemAImpl_neg_u64, iemAImpl_neg_u64_locked;
+/** @} */
+
+
+/** @name Shift operations on bytes (Group 2).
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU8,(uint8_t *pu8Dst, uint8_t cShift, uint32_t *pEFlags));
+typedef FNIEMAIMPLSHIFTU8  *PFNIEMAIMPLSHIFTU8;
+FNIEMAIMPLSHIFTU8 iemAImpl_rol_u8;
+FNIEMAIMPLSHIFTU8 iemAImpl_ror_u8;
+FNIEMAIMPLSHIFTU8 iemAImpl_rcl_u8;
+FNIEMAIMPLSHIFTU8 iemAImpl_rcr_u8;
+FNIEMAIMPLSHIFTU8 iemAImpl_shl_u8;
+FNIEMAIMPLSHIFTU8 iemAImpl_shr_u8;
+FNIEMAIMPLSHIFTU8 iemAImpl_sar_u8;
+/** @} */
+
+/** @name Shift operations on words (Group 2).
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU16,(uint16_t *pu16Dst, uint8_t cShift, uint32_t *pEFlags));
+typedef FNIEMAIMPLSHIFTU16  *PFNIEMAIMPLSHIFTU16;
+FNIEMAIMPLSHIFTU16 iemAImpl_rol_u16;
+FNIEMAIMPLSHIFTU16 iemAImpl_ror_u16;
+FNIEMAIMPLSHIFTU16 iemAImpl_rcl_u16;
+FNIEMAIMPLSHIFTU16 iemAImpl_rcr_u16;
+FNIEMAIMPLSHIFTU16 iemAImpl_shl_u16;
+FNIEMAIMPLSHIFTU16 iemAImpl_shr_u16;
+FNIEMAIMPLSHIFTU16 iemAImpl_sar_u16;
+/** @} */
+
+/** @name Shift operations on double words (Group 2).
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU32,(uint32_t *pu32Dst, uint8_t cShift, uint32_t *pEFlags));
+typedef FNIEMAIMPLSHIFTU32  *PFNIEMAIMPLSHIFTU32;
+FNIEMAIMPLSHIFTU32 iemAImpl_rol_u32;
+FNIEMAIMPLSHIFTU32 iemAImpl_ror_u32;
+FNIEMAIMPLSHIFTU32 iemAImpl_rcl_u32;
+FNIEMAIMPLSHIFTU32 iemAImpl_rcr_u32;
+FNIEMAIMPLSHIFTU32 iemAImpl_shl_u32;
+FNIEMAIMPLSHIFTU32 iemAImpl_shr_u32;
+FNIEMAIMPLSHIFTU32 iemAImpl_sar_u32;
+/** @} */
+
+/** @name Shift operations on words (Group 2).
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU64,(uint64_t *pu64Dst, uint8_t cShift, uint32_t *pEFlags));
+typedef FNIEMAIMPLSHIFTU64  *PFNIEMAIMPLSHIFTU64;
+FNIEMAIMPLSHIFTU64 iemAImpl_rol_u64;
+FNIEMAIMPLSHIFTU64 iemAImpl_ror_u64;
+FNIEMAIMPLSHIFTU64 iemAImpl_rcl_u64;
+FNIEMAIMPLSHIFTU64 iemAImpl_rcr_u64;
+FNIEMAIMPLSHIFTU64 iemAImpl_shl_u64;
+FNIEMAIMPLSHIFTU64 iemAImpl_shr_u64;
+FNIEMAIMPLSHIFTU64 iemAImpl_sar_u64;
+/** @} */
+
+/** @name Multiplication and division operations.
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU8,(uint16_t *pu16AX, uint8_t u8FactorDivisor, uint32_t *pEFlags));
+typedef FNIEMAIMPLMULDIVU8  *PFNIEMAIMPLMULDIVU8;
+FNIEMAIMPLMULDIVU8 iemAImpl_mul_u8, iemAImpl_imul_u8;
+FNIEMAIMPLMULDIVU8 iemAImpl_div_u8, iemAImpl_idiv_u8;
+
+typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU16,(uint16_t *pu16AX, uint16_t *pu16DX, uint16_t u16FactorDivisor, uint32_t *pEFlags));
+typedef FNIEMAIMPLMULDIVU16  *PFNIEMAIMPLMULDIVU16;
+FNIEMAIMPLMULDIVU16 iemAImpl_mul_u16, iemAImpl_imul_u16;
+FNIEMAIMPLMULDIVU16 iemAImpl_div_u16, iemAImpl_idiv_u16;
+
+typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU32,(uint32_t *pu32EAX, uint32_t *pu32EDX, uint32_t u32FactorDivisor, uint32_t *pEFlags));
+typedef FNIEMAIMPLMULDIVU32  *PFNIEMAIMPLMULDIVU32;
+FNIEMAIMPLMULDIVU32 iemAImpl_mul_u32, iemAImpl_imul_u32;
+FNIEMAIMPLMULDIVU32 iemAImpl_div_u32, iemAImpl_idiv_u32;
+
+typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU64,(uint64_t *pu64RAX, uint64_t *pu64RDX, uint64_t u64FactorDivisor, uint32_t *pEFlags));
+typedef FNIEMAIMPLMULDIVU64  *PFNIEMAIMPLMULDIVU64;
+FNIEMAIMPLMULDIVU64 iemAImpl_mul_u64, iemAImpl_imul_u64;
+FNIEMAIMPLMULDIVU64 iemAImpl_div_u64, iemAImpl_idiv_u64;
+/** @} */
+
+/** @name Byte Swap.
+ * @{  */
+IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u16,(uint32_t *pu32Dst)); /* Yes, 32-bit register access. */
+IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u32,(uint32_t *pu32Dst));
+IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u64,(uint64_t *pu64Dst));
+/** @}  */
+
+/** @name Misc.
+ * @{ */
+FNIEMAIMPLBINU16 iemAImpl_arpl;
+/** @} */
+
+
+/** @name FPU operations taking a 32-bit float argument
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR32FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
+                                                      PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2));
+typedef FNIEMAIMPLFPUR32FSW *PFNIEMAIMPLFPUR32FSW;
+
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                   PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2));
+typedef FNIEMAIMPLFPUR32    *PFNIEMAIMPLFPUR32;
+
+FNIEMAIMPLFPUR32FSW iemAImpl_fcom_r80_by_r32;
+FNIEMAIMPLFPUR32    iemAImpl_fadd_r80_by_r32;
+FNIEMAIMPLFPUR32    iemAImpl_fmul_r80_by_r32;
+FNIEMAIMPLFPUR32    iemAImpl_fsub_r80_by_r32;
+FNIEMAIMPLFPUR32    iemAImpl_fsubr_r80_by_r32;
+FNIEMAIMPLFPUR32    iemAImpl_fdiv_r80_by_r32;
+FNIEMAIMPLFPUR32    iemAImpl_fdivr_r80_by_r32;
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r32_to_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT32U pr32Val));
+IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                 PRTFLOAT32U pr32Val, PCRTFLOAT80U pr80Val));
+/** @} */
+
+/** @name FPU operations taking a 64-bit float argument
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                   PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2));
+typedef FNIEMAIMPLFPUR64   *PFNIEMAIMPLFPUR64;
+
+FNIEMAIMPLFPUR64  iemAImpl_fadd_r80_by_r64;
+FNIEMAIMPLFPUR64  iemAImpl_fmul_r80_by_r64;
+FNIEMAIMPLFPUR64  iemAImpl_fsub_r80_by_r64;
+FNIEMAIMPLFPUR64  iemAImpl_fsubr_r80_by_r64;
+FNIEMAIMPLFPUR64  iemAImpl_fdiv_r80_by_r64;
+FNIEMAIMPLFPUR64  iemAImpl_fdivr_r80_by_r64;
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r64,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
+                                                  PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2));
+IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r64_to_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT64U pr64Val));
+IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                 PRTFLOAT64U pr32Val, PCRTFLOAT80U pr80Val));
+/** @} */
+
+/** @name FPU operations taking a 80-bit float argument
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                   PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
+typedef FNIEMAIMPLFPUR80    *PFNIEMAIMPLFPUR80;
+FNIEMAIMPLFPUR80            iemAImpl_fadd_r80_by_r80;
+FNIEMAIMPLFPUR80            iemAImpl_fmul_r80_by_r80;
+FNIEMAIMPLFPUR80            iemAImpl_fsub_r80_by_r80;
+FNIEMAIMPLFPUR80            iemAImpl_fsubr_r80_by_r80;
+FNIEMAIMPLFPUR80            iemAImpl_fdiv_r80_by_r80;
+FNIEMAIMPLFPUR80            iemAImpl_fdivr_r80_by_r80;
+FNIEMAIMPLFPUR80            iemAImpl_fprem_r80_by_r80;
+FNIEMAIMPLFPUR80            iemAImpl_fprem1_r80_by_r80;
+FNIEMAIMPLFPUR80            iemAImpl_fscale_r80_by_r80;
+
+FNIEMAIMPLFPUR80            iemAImpl_fpatan_r80_by_r80;
+FNIEMAIMPLFPUR80            iemAImpl_fyl2x_r80_by_r80;
+FNIEMAIMPLFPUR80            iemAImpl_fyl2xp1_r80_by_r80;
+
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
+                                                      PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
+typedef FNIEMAIMPLFPUR80FSW *PFNIEMAIMPLFPUR80FSW;
+FNIEMAIMPLFPUR80FSW         iemAImpl_fcom_r80_by_r80;
+FNIEMAIMPLFPUR80FSW         iemAImpl_fucom_r80_by_r80;
+
+typedef IEM_DECL_IMPL_TYPE(uint32_t, FNIEMAIMPLFPUR80EFL,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
+                                                          PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
+typedef FNIEMAIMPLFPUR80EFL *PFNIEMAIMPLFPUR80EFL;
+FNIEMAIMPLFPUR80EFL         iemAImpl_fcomi_r80_by_r80;
+FNIEMAIMPLFPUR80EFL         iemAImpl_fucomi_r80_by_r80;
+
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARY,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val));
+typedef FNIEMAIMPLFPUR80UNARY *PFNIEMAIMPLFPUR80UNARY;
+FNIEMAIMPLFPUR80UNARY       iemAImpl_fabs_r80;
+FNIEMAIMPLFPUR80UNARY       iemAImpl_fchs_r80;
+FNIEMAIMPLFPUR80UNARY       iemAImpl_f2xm1_r80;
+FNIEMAIMPLFPUR80UNARY       iemAImpl_fsqrt_r80;
+FNIEMAIMPLFPUR80UNARY       iemAImpl_frndint_r80;
+FNIEMAIMPLFPUR80UNARY       iemAImpl_fsin_r80;
+FNIEMAIMPLFPUR80UNARY       iemAImpl_fcos_r80;
+
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARYFSW,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val));
+typedef FNIEMAIMPLFPUR80UNARYFSW *PFNIEMAIMPLFPUR80UNARYFSW;
+FNIEMAIMPLFPUR80UNARYFSW    iemAImpl_ftst_r80;
+FNIEMAIMPLFPUR80UNARYFSW    iemAImpl_fxam_r80;
+
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80LDCONST,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes));
+typedef FNIEMAIMPLFPUR80LDCONST *PFNIEMAIMPLFPUR80LDCONST;
+FNIEMAIMPLFPUR80LDCONST     iemAImpl_fld1;
+FNIEMAIMPLFPUR80LDCONST     iemAImpl_fldl2t;
+FNIEMAIMPLFPUR80LDCONST     iemAImpl_fldl2e;
+FNIEMAIMPLFPUR80LDCONST     iemAImpl_fldpi;
+FNIEMAIMPLFPUR80LDCONST     iemAImpl_fldlg2;
+FNIEMAIMPLFPUR80LDCONST     iemAImpl_fldln2;
+FNIEMAIMPLFPUR80LDCONST     iemAImpl_fldz;
+
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARYTWO,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo,
+                                                           PCRTFLOAT80U pr80Val));
+typedef FNIEMAIMPLFPUR80UNARYTWO *PFNIEMAIMPLFPUR80UNARYTWO;
+FNIEMAIMPLFPUR80UNARYTWO    iemAImpl_fptan_r80_r80;
+FNIEMAIMPLFPUR80UNARYTWO    iemAImpl_fxtract_r80_r80;
+FNIEMAIMPLFPUR80UNARYTWO    iemAImpl_fsincos_r80_r80;
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val));
+IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                 PRTFLOAT80U pr80Dst, PCRTFLOAT80U pr80Src));
+
+/** @} */
+
+/** @name FPU operations taking a 16-bit signed integer argument
+ * @{  */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                   PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2));
+typedef FNIEMAIMPLFPUI16    *PFNIEMAIMPLFPUI16;
+
+FNIEMAIMPLFPUI16    iemAImpl_fiadd_r80_by_i16;
+FNIEMAIMPLFPUI16    iemAImpl_fimul_r80_by_i16;
+FNIEMAIMPLFPUI16    iemAImpl_fisub_r80_by_i16;
+FNIEMAIMPLFPUI16    iemAImpl_fisubr_r80_by_i16;
+FNIEMAIMPLFPUI16    iemAImpl_fidiv_r80_by_i16;
+FNIEMAIMPLFPUI16    iemAImpl_fidivr_r80_by_i16;
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_ficom_r80_by_i16,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
+                                                   PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2));
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fild_i16_to_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int16_t const *pi16Val));
+IEM_DECL_IMPL_DEF(void, iemAImpl_fist_r80_to_i16,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                  int16_t *pi16Val, PCRTFLOAT80U pr80Val));
+IEM_DECL_IMPL_DEF(void, iemAImpl_fistt_r80_to_i16,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                   int16_t *pi16Val, PCRTFLOAT80U pr80Val));
+/** @}  */
+
+/** @name FPU operations taking a 32-bit signed integer argument
+ * @{  */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                   PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2));
+typedef FNIEMAIMPLFPUI32    *PFNIEMAIMPLFPUI32;
+
+FNIEMAIMPLFPUI32    iemAImpl_fiadd_r80_by_i32;
+FNIEMAIMPLFPUI32    iemAImpl_fimul_r80_by_i32;
+FNIEMAIMPLFPUI32    iemAImpl_fisub_r80_by_i32;
+FNIEMAIMPLFPUI32    iemAImpl_fisubr_r80_by_i32;
+FNIEMAIMPLFPUI32    iemAImpl_fidiv_r80_by_i32;
+FNIEMAIMPLFPUI32    iemAImpl_fidivr_r80_by_i32;
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_ficom_r80_by_i32,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
+                                                   PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2));
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fild_i32_to_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int32_t const *pi32Val));
+IEM_DECL_IMPL_DEF(void, iemAImpl_fist_r80_to_i32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                  int32_t *pi32Val, PCRTFLOAT80U pr80Val));
+IEM_DECL_IMPL_DEF(void, iemAImpl_fistt_r80_to_i32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                   int32_t *pi32Val, PCRTFLOAT80U pr80Val));
+/** @}  */
+
+/** @name FPU operations taking a 64-bit signed integer argument
+ * @{  */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                   PCRTFLOAT80U pr80Val1, int64_t const *pi64Val2));
+typedef FNIEMAIMPLFPUI64    *PFNIEMAIMPLFPUI64;
+
+FNIEMAIMPLFPUI64    iemAImpl_fiadd_r80_by_i64;
+FNIEMAIMPLFPUI64    iemAImpl_fimul_r80_by_i64;
+FNIEMAIMPLFPUI64    iemAImpl_fisub_r80_by_i64;
+FNIEMAIMPLFPUI64    iemAImpl_fisubr_r80_by_i64;
+FNIEMAIMPLFPUI64    iemAImpl_fidiv_r80_by_i64;
+FNIEMAIMPLFPUI64    iemAImpl_fidivr_r80_by_i64;
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_ficom_r80_by_i64,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
+                                                   PCRTFLOAT80U pr80Val1, int64_t const *pi64Val2));
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fild_i64_to_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int64_t const *pi64Val));
+IEM_DECL_IMPL_DEF(void, iemAImpl_fist_r80_to_i64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                  int64_t *pi64Val, PCRTFLOAT80U pr80Val));
+IEM_DECL_IMPL_DEF(void, iemAImpl_fistt_r80_to_i64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                   int64_t *pi32Val, PCRTFLOAT80U pr80Val));
+/** @} */
+
+
+/** Temporary type representing a 256-bit vector register. */
+typedef struct {uint64_t au64[4]; } IEMVMM256;
+/** Temporary type pointing to a 256-bit vector register. */
+typedef IEMVMM256 *PIEMVMM256;
+/** Temporary type pointing to a const 256-bit vector register. */
+typedef IEMVMM256 *PCIEMVMM256;
+
+
+/** @name Media (SSE/MMX/AVX) operations: full1 + full2 -> full1.
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF2U64,(PCX86FXSTATE pFpuState, uint64_t *pu64Dst, uint64_t const *pu64Src));
+typedef FNIEMAIMPLMEDIAF2U64   *PFNIEMAIMPLMEDIAF2U64;
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF2U128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src));
+typedef FNIEMAIMPLMEDIAF2U128  *PFNIEMAIMPLMEDIAF2U128;
+FNIEMAIMPLMEDIAF2U64  iemAImpl_pxor_u64,  iemAImpl_pcmpeqb_u64,  iemAImpl_pcmpeqw_u64,  iemAImpl_pcmpeqd_u64;
+FNIEMAIMPLMEDIAF2U128 iemAImpl_pxor_u128, iemAImpl_pcmpeqb_u128, iemAImpl_pcmpeqw_u128, iemAImpl_pcmpeqd_u128;
+/** @} */
+
+/** @name Media (SSE/MMX/AVX) operations: lowhalf1 + lowhalf1 -> full1.
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF1L1U64,(PCX86FXSTATE pFpuState, uint64_t *pu64Dst, uint32_t const *pu32Src));
+typedef FNIEMAIMPLMEDIAF1L1U64   *PFNIEMAIMPLMEDIAF1L1U64;
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF1L1U128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, uint64_t const *pu64Src));
+typedef FNIEMAIMPLMEDIAF1L1U128  *PFNIEMAIMPLMEDIAF1L1U128;
+FNIEMAIMPLMEDIAF1L1U64  iemAImpl_punpcklbw_u64,  iemAImpl_punpcklwd_u64,  iemAImpl_punpckldq_u64;
+FNIEMAIMPLMEDIAF1L1U128 iemAImpl_punpcklbw_u128, iemAImpl_punpcklwd_u128, iemAImpl_punpckldq_u128, iemAImpl_punpcklqdq_u128;
+/** @} */
+
+/** @name Media (SSE/MMX/AVX) operations: hihalf1 + hihalf2 -> full1.
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF1H1U64,(PCX86FXSTATE pFpuState, uint64_t *pu64Dst, uint64_t const *pu64Src));
+typedef FNIEMAIMPLMEDIAF2U64   *PFNIEMAIMPLMEDIAF1H1U64;
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF1H1U128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src));
+typedef FNIEMAIMPLMEDIAF2U128  *PFNIEMAIMPLMEDIAF1H1U128;
+FNIEMAIMPLMEDIAF1H1U64  iemAImpl_punpckhbw_u64,  iemAImpl_punpckhwd_u64,  iemAImpl_punpckhdq_u64;
+FNIEMAIMPLMEDIAF1H1U128 iemAImpl_punpckhbw_u128, iemAImpl_punpckhwd_u128, iemAImpl_punpckhdq_u128, iemAImpl_punpckhqdq_u128;
+/** @} */
+
+/** @name Media (SSE/MMX/AVX) operation: Packed Shuffle Stuff (evil)
+ * @{ */
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHUF,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst,
+                                                       PCRTUINT128U pu128Src, uint8_t bEvil));
+typedef FNIEMAIMPLMEDIAPSHUF *PFNIEMAIMPLMEDIAPSHUF;
+FNIEMAIMPLMEDIAPSHUF iemAImpl_pshufhw, iemAImpl_pshuflw, iemAImpl_pshufd;
+IEM_DECL_IMPL_DEF(void, iemAImpl_pshufw,(PCX86FXSTATE pFpuState, uint64_t *pu64Dst, uint64_t const *pu64Src, uint8_t bEvil));
+/** @} */
+
+/** @name Media (SSE/MMX/AVX) operation: Move Byte Mask
+ * @{ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u64,(PCX86FXSTATE pFpuState, uint64_t *pu64Dst, uint64_t const *pu64Src));
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u128,(PCX86FXSTATE pFpuState, uint64_t *pu64Dst, PCRTUINT128U pu128Src));
+/** @} */
+
+/** @name Media (SSE/MMX/AVX) operation: Sort this later
+ * @{ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_movsldup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc));
+IEM_DECL_IMPL_DEF(void, iemAImpl_movshdup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc));
+IEM_DECL_IMPL_DEF(void, iemAImpl_movddup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, uint64_t uSrc));
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
+
+/** @} */
+
+
+/** @name Function tables.
+ * @{
+ */
+
+/**
+ * Function table for a binary operator providing implementation based on
+ * operand size.
+ */
+typedef struct IEMOPBINSIZES
+{
+    PFNIEMAIMPLBINU8  pfnNormalU8,    pfnLockedU8;
+    PFNIEMAIMPLBINU16 pfnNormalU16,   pfnLockedU16;
+    PFNIEMAIMPLBINU32 pfnNormalU32,   pfnLockedU32;
+    PFNIEMAIMPLBINU64 pfnNormalU64,   pfnLockedU64;
+} IEMOPBINSIZES;
+/** Pointer to a binary operator function table. */
+typedef IEMOPBINSIZES const *PCIEMOPBINSIZES;
+
+
+/**
+ * Function table for a unary operator providing implementation based on
+ * operand size.
+ */
+typedef struct IEMOPUNARYSIZES
+{
+    PFNIEMAIMPLUNARYU8  pfnNormalU8,    pfnLockedU8;
+    PFNIEMAIMPLUNARYU16 pfnNormalU16,   pfnLockedU16;
+    PFNIEMAIMPLUNARYU32 pfnNormalU32,   pfnLockedU32;
+    PFNIEMAIMPLUNARYU64 pfnNormalU64,   pfnLockedU64;
+} IEMOPUNARYSIZES;
+/** Pointer to a unary operator function table. */
+typedef IEMOPUNARYSIZES const *PCIEMOPUNARYSIZES;
+
+
+/**
+ * Function table for a shift operator providing implementation based on
+ * operand size.
+ */
+typedef struct IEMOPSHIFTSIZES
+{
+    PFNIEMAIMPLSHIFTU8  pfnNormalU8;
+    PFNIEMAIMPLSHIFTU16 pfnNormalU16;
+    PFNIEMAIMPLSHIFTU32 pfnNormalU32;
+    PFNIEMAIMPLSHIFTU64 pfnNormalU64;
+} IEMOPSHIFTSIZES;
+/** Pointer to a shift operator function table. */
+typedef IEMOPSHIFTSIZES const *PCIEMOPSHIFTSIZES;
+
+
+/**
+ * Function table for a multiplication or division operation.
+ */
+typedef struct IEMOPMULDIVSIZES
+{
+    PFNIEMAIMPLMULDIVU8  pfnU8;
+    PFNIEMAIMPLMULDIVU16 pfnU16;
+    PFNIEMAIMPLMULDIVU32 pfnU32;
+    PFNIEMAIMPLMULDIVU64 pfnU64;
+} IEMOPMULDIVSIZES;
+/** Pointer to a multiplication or division operation function table. */
+typedef IEMOPMULDIVSIZES const *PCIEMOPMULDIVSIZES;
+
+
+/**
+ * Function table for a double precision shift operator providing implementation
+ * based on operand size.
+ */
+typedef struct IEMOPSHIFTDBLSIZES
+{
+    PFNIEMAIMPLSHIFTDBLU16 pfnNormalU16;
+    PFNIEMAIMPLSHIFTDBLU32 pfnNormalU32;
+    PFNIEMAIMPLSHIFTDBLU64 pfnNormalU64;
+} IEMOPSHIFTDBLSIZES;
+/** Pointer to a double precision shift function table. */
+typedef IEMOPSHIFTDBLSIZES const *PCIEMOPSHIFTDBLSIZES;
+
+
+/**
+ * Function table for media instruction taking two full sized media registers,
+ * optionally the 2nd being a memory reference (only modifying the first op.)
+ */
+typedef struct IEMOPMEDIAF2
+{
+    PFNIEMAIMPLMEDIAF2U64  pfnU64;
+    PFNIEMAIMPLMEDIAF2U128 pfnU128;
+} IEMOPMEDIAF2;
+/** Pointer to a media operation function table for full sized ops. */
+typedef IEMOPMEDIAF2 const *PCIEMOPMEDIAF2;
+
+/**
+ * Function table for media instruction taking taking one full and one lower
+ * half media register.
+ */
+typedef struct IEMOPMEDIAF1L1
+{
+    PFNIEMAIMPLMEDIAF1L1U64  pfnU64;
+    PFNIEMAIMPLMEDIAF1L1U128 pfnU128;
+} IEMOPMEDIAF1L1;
+/** Pointer to a media operation function table for lowhalf+lowhalf -> full. */
+typedef IEMOPMEDIAF1L1 const *PCIEMOPMEDIAF1L1;
+
+/**
+ * Function table for media instruction taking taking one full and one high half
+ * media register.
+ */
+typedef struct IEMOPMEDIAF1H1
+{
+    PFNIEMAIMPLMEDIAF1H1U64  pfnU64;
+    PFNIEMAIMPLMEDIAF1H1U128 pfnU128;
+} IEMOPMEDIAF1H1;
+/** Pointer to a media operation function table for hihalf+hihalf -> full. */
+typedef IEMOPMEDIAF1H1 const *PCIEMOPMEDIAF1H1;
+
+
+/** @} */
+
+
+/** @name C instruction implementations for anything slightly complicated.
+ * @{ */
+
+/**
+ * For typedef'ing or declaring a C instruction implementation function taking
+ * no extra arguments.
+ *
+ * @param   a_Name              The name of the type.
+ */
+# define IEM_CIMPL_DECL_TYPE_0(a_Name) \
+    IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
+/**
+ * For defining a C instruction implementation function taking no extra
+ * arguments.
+ *
+ * @param   a_Name              The name of the function
+ */
+# define IEM_CIMPL_DEF_0(a_Name) \
+    IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
+/**
+ * For calling a C instruction implementation function taking no extra
+ * arguments.
+ *
+ * This special call macro adds default arguments to the call and allow us to
+ * change these later.
+ *
+ * @param   a_fn                The name of the function.
+ */
+# define IEM_CIMPL_CALL_0(a_fn)            a_fn(pVCpu, cbInstr)
+
+/**
+ * For typedef'ing or declaring a C instruction implementation function taking
+ * one extra argument.
+ *
+ * @param   a_Name              The name of the type.
+ * @param   a_Type0             The argument type.
+ * @param   a_Arg0              The argument name.
+ */
+# define IEM_CIMPL_DECL_TYPE_1(a_Name, a_Type0, a_Arg0) \
+    IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
+/**
+ * For defining a C instruction implementation function taking one extra
+ * argument.
+ *
+ * @param   a_Name              The name of the function
+ * @param   a_Type0             The argument type.
+ * @param   a_Arg0              The argument name.
+ */
+# define IEM_CIMPL_DEF_1(a_Name, a_Type0, a_Arg0) \
+    IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
+/**
+ * For calling a C instruction implementation function taking one extra
+ * argument.
+ *
+ * This special call macro adds default arguments to the call and allow us to
+ * change these later.
+ *
+ * @param   a_fn                The name of the function.
+ * @param   a0                  The name of the 1st argument.
+ */
+# define IEM_CIMPL_CALL_1(a_fn, a0)        a_fn(pVCpu, cbInstr, (a0))
+
+/**
+ * For typedef'ing or declaring a C instruction implementation function taking
+ * two extra arguments.
+ *
+ * @param   a_Name              The name of the type.
+ * @param   a_Type0             The type of the 1st argument
+ * @param   a_Arg0              The name of the 1st argument.
+ * @param   a_Type1             The type of the 2nd argument.
+ * @param   a_Arg1              The name of the 2nd argument.
+ */
+# define IEM_CIMPL_DECL_TYPE_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
+    IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
+/**
+ * For defining a C instruction implementation function taking two extra
+ * arguments.
+ *
+ * @param   a_Name              The name of the function.
+ * @param   a_Type0             The type of the 1st argument
+ * @param   a_Arg0              The name of the 1st argument.
+ * @param   a_Type1             The type of the 2nd argument.
+ * @param   a_Arg1              The name of the 2nd argument.
+ */
+# define IEM_CIMPL_DEF_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
+    IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
+/**
+ * For calling a C instruction implementation function taking two extra
+ * arguments.
+ *
+ * This special call macro adds default arguments to the call and allow us to
+ * change these later.
+ *
+ * @param   a_fn                The name of the function.
+ * @param   a0                  The name of the 1st argument.
+ * @param   a1                  The name of the 2nd argument.
+ */
+# define IEM_CIMPL_CALL_2(a_fn, a0, a1)    a_fn(pVCpu, cbInstr, (a0), (a1))
+
+/**
+ * For typedef'ing or declaring a C instruction implementation function taking
+ * three extra arguments.
+ *
+ * @param   a_Name              The name of the type.
+ * @param   a_Type0             The type of the 1st argument
+ * @param   a_Arg0              The name of the 1st argument.
+ * @param   a_Type1             The type of the 2nd argument.
+ * @param   a_Arg1              The name of the 2nd argument.
+ * @param   a_Type2             The type of the 3rd argument.
+ * @param   a_Arg2              The name of the 3rd argument.
+ */
+# define IEM_CIMPL_DECL_TYPE_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
+    IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
+/**
+ * For defining a C instruction implementation function taking three extra
+ * arguments.
+ *
+ * @param   a_Name              The name of the function.
+ * @param   a_Type0             The type of the 1st argument
+ * @param   a_Arg0              The name of the 1st argument.
+ * @param   a_Type1             The type of the 2nd argument.
+ * @param   a_Arg1              The name of the 2nd argument.
+ * @param   a_Type2             The type of the 3rd argument.
+ * @param   a_Arg2              The name of the 3rd argument.
+ */
+# define IEM_CIMPL_DEF_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
+    IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
+/**
+ * For calling a C instruction implementation function taking three extra
+ * arguments.
+ *
+ * This special call macro adds default arguments to the call and allow us to
+ * change these later.
+ *
+ * @param   a_fn                The name of the function.
+ * @param   a0                  The name of the 1st argument.
+ * @param   a1                  The name of the 2nd argument.
+ * @param   a2                  The name of the 3rd argument.
+ */
+# define IEM_CIMPL_CALL_3(a_fn, a0, a1, a2) a_fn(pVCpu, cbInstr, (a0), (a1), (a2))
+
+
+/**
+ * For typedef'ing or declaring a C instruction implementation function taking
+ * four extra arguments.
+ *
+ * @param   a_Name              The name of the type.
+ * @param   a_Type0             The type of the 1st argument
+ * @param   a_Arg0              The name of the 1st argument.
+ * @param   a_Type1             The type of the 2nd argument.
+ * @param   a_Arg1              The name of the 2nd argument.
+ * @param   a_Type2             The type of the 3rd argument.
+ * @param   a_Arg2              The name of the 3rd argument.
+ * @param   a_Type3             The type of the 4th argument.
+ * @param   a_Arg3              The name of the 4th argument.
+ */
+# define IEM_CIMPL_DECL_TYPE_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
+    IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, a_Type3 a_Arg3))
+/**
+ * For defining a C instruction implementation function taking four extra
+ * arguments.
+ *
+ * @param   a_Name              The name of the function.
+ * @param   a_Type0             The type of the 1st argument
+ * @param   a_Arg0              The name of the 1st argument.
+ * @param   a_Type1             The type of the 2nd argument.
+ * @param   a_Arg1              The name of the 2nd argument.
+ * @param   a_Type2             The type of the 3rd argument.
+ * @param   a_Arg2              The name of the 3rd argument.
+ * @param   a_Type3             The type of the 4th argument.
+ * @param   a_Arg3              The name of the 4th argument.
+ */
+# define IEM_CIMPL_DEF_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
+    IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
+                                             a_Type2 a_Arg2, a_Type3 a_Arg3))
+/**
+ * For calling a C instruction implementation function taking four extra
+ * arguments.
+ *
+ * This special call macro adds default arguments to the call and allow us to
+ * change these later.
+ *
+ * @param   a_fn                The name of the function.
+ * @param   a0                  The name of the 1st argument.
+ * @param   a1                  The name of the 2nd argument.
+ * @param   a2                  The name of the 3rd argument.
+ * @param   a3                  The name of the 4th argument.
+ */
+# define IEM_CIMPL_CALL_4(a_fn, a0, a1, a2, a3) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3))
+
+
+/**
+ * For typedef'ing or declaring a C instruction implementation function taking
+ * five extra arguments.
+ *
+ * @param   a_Name              The name of the type.
+ * @param   a_Type0             The type of the 1st argument
+ * @param   a_Arg0              The name of the 1st argument.
+ * @param   a_Type1             The type of the 2nd argument.
+ * @param   a_Arg1              The name of the 2nd argument.
+ * @param   a_Type2             The type of the 3rd argument.
+ * @param   a_Arg2              The name of the 3rd argument.
+ * @param   a_Type3             The type of the 4th argument.
+ * @param   a_Arg3              The name of the 4th argument.
+ * @param   a_Type4             The type of the 5th argument.
+ * @param   a_Arg4              The name of the 5th argument.
+ */
+# define IEM_CIMPL_DECL_TYPE_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
+    IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, \
+                                               a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, \
+                                               a_Type3 a_Arg3, a_Type4 a_Arg4))
+/**
+ * For defining a C instruction implementation function taking five extra
+ * arguments.
+ *
+ * @param   a_Name              The name of the function.
+ * @param   a_Type0             The type of the 1st argument
+ * @param   a_Arg0              The name of the 1st argument.
+ * @param   a_Type1             The type of the 2nd argument.
+ * @param   a_Arg1              The name of the 2nd argument.
+ * @param   a_Type2             The type of the 3rd argument.
+ * @param   a_Arg2              The name of the 3rd argument.
+ * @param   a_Type3             The type of the 4th argument.
+ * @param   a_Arg3              The name of the 4th argument.
+ * @param   a_Type4             The type of the 5th argument.
+ * @param   a_Arg4              The name of the 5th argument.
+ */
+# define IEM_CIMPL_DEF_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
+    IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, \
+                                             a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, \
+                                             a_Type3 a_Arg3, a_Type4 a_Arg4))
+/**
+ * For calling a C instruction implementation function taking five extra
+ * arguments.
+ *
+ * This special call macro adds default arguments to the call and allow us to
+ * change these later.
+ *
+ * @param   a_fn                The name of the function.
+ * @param   a0                  The name of the 1st argument.
+ * @param   a1                  The name of the 2nd argument.
+ * @param   a2                  The name of the 3rd argument.
+ * @param   a3                  The name of the 4th argument.
+ * @param   a4                  The name of the 5th argument.
+ */
+# define IEM_CIMPL_CALL_5(a_fn, a0, a1, a2, a3, a4) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3), (a4))
+
+/** @}  */
+
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_IEMInternal_h */
+
diff --git a/src/VBox/VMM/include/IOMInline.h b/src/VBox/VMM/include/IOMInline.h
new file mode 100644
index 00000000..a0a9cd6e
--- /dev/null
+++ b/src/VBox/VMM/include/IOMInline.h
@@ -0,0 +1,260 @@
+/* $Id: IOMInline.h $ */
+/** @file
+ * IOM - Inlined functions.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_IOMInline_h
+#define VMM_INCLUDED_SRC_include_IOMInline_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <iprt/errcore.h>
+
+/** @addtogroup grp_iom_int   Internals
+ * @internal
+ * @{
+ */
+
+
+/**
+ * Gets the I/O port entry for the specified I/O port in the current context.
+ *
+ * @returns Pointer to I/O port entry.
+ * @returns NULL if no port registered.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   uPort           The I/O port to lookup.
+ * @param   poffPort        Where to return the port offset relative to the
+ *                          start of the I/O port range.
+ * @param   pidxLastHint    Pointer to IOMCPU::idxIoPortLastRead or
+ *                          IOMCPU::idxIoPortLastWrite.
+ *
+ * @note    In ring-0 it is possible to get an uninitialized entry (pDevIns is
+ *          NULL, cPorts is 0), in which case there should be ring-3 handlers
+ *          for the entry.  Use IOMIOPORTENTRYR0::idxSelf to get the ring-3
+ *          entry.
+ *
+ * @note    This code is almost identical to iomMmioGetEntry, so keep in sync.
+ */
+DECLINLINE(CTX_SUFF(PIOMIOPORTENTRY)) iomIoPortGetEntry(PVMCC pVM, RTIOPORT uPort, PRTIOPORT poffPort, uint16_t *pidxLastHint)
+{
+    Assert(IOM_IS_SHARED_LOCK_OWNER(pVM));
+
+#ifdef IN_RING0
+    uint32_t               iEnd      = RT_MIN(pVM->iom.s.cIoPortLookupEntries, pVM->iomr0.s.cIoPortAlloc);
+    PCIOMIOPORTLOOKUPENTRY paLookup  = pVM->iomr0.s.paIoPortLookup;
+#else
+    uint32_t               iEnd      = pVM->iom.s.cIoPortLookupEntries;
+    PCIOMIOPORTLOOKUPENTRY paLookup  = pVM->iom.s.paIoPortLookup;
+#endif
+    if (iEnd > 0)
+    {
+        uint32_t iFirst = 0;
+        uint32_t i      = *pidxLastHint;
+        if (i < iEnd)
+        { /* likely */ }
+        else
+            i = iEnd / 2;
+        for (;;)
+        {
+            PCIOMIOPORTLOOKUPENTRY pCur = &paLookup[i];
+            if (pCur->uFirstPort > uPort)
+            {
+                if (i > iFirst)
+                    iEnd = i;
+                else
+                    break;
+            }
+            else if (pCur->uLastPort < uPort)
+            {
+                i += 1;
+                if (i < iEnd)
+                    iFirst = i;
+                else
+                    break;
+            }
+            else
+            {
+                *pidxLastHint = (uint16_t)i;
+                *poffPort     = uPort - pCur->uFirstPort;
+
+                /*
+                 * Translate the 'idx' member into a pointer.
+                 */
+                size_t const idx = pCur->idx;
+#ifdef IN_RING0
+                AssertMsg(idx < pVM->iom.s.cIoPortRegs && idx < pVM->iomr0.s.cIoPortAlloc,
+                          ("%#zx vs %#x/%x (port %#x)\n", idx, pVM->iom.s.cIoPortRegs, pVM->iomr0.s.cIoPortMax, uPort));
+                if (idx < pVM->iomr0.s.cIoPortAlloc)
+                    return &pVM->iomr0.s.paIoPortRegs[idx];
+#else
+                if (idx < pVM->iom.s.cIoPortRegs)
+                    return &pVM->iom.s.paIoPortRegs[idx];
+                AssertMsgFailed(("%#zx vs %#x (port %#x)\n", idx, pVM->iom.s.cIoPortRegs, uPort));
+#endif
+                break;
+            }
+
+            i = iFirst + (iEnd - iFirst) / 2;
+        }
+    }
+    *poffPort = 0;
+    return NULL;
+}
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * Gets the statistics entry for an I/O port.
+ *
+ * @returns Pointer to stats.  Instead of NULL, a pointer to IoPortDummyStats is
+ *          returned, so the caller does not need to check for NULL.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pRegEntry   The I/O port entry to get stats for.
+ * @param   offPort     The offset of the  port relative to the start of the
+ *                      registration entry.
+ */
+DECLINLINE(PIOMIOPORTSTATSENTRY) iomIoPortGetStats(PVMCC pVM, CTX_SUFF(PIOMIOPORTENTRY) pRegEntry, uint16_t offPort)
+{
+    size_t idxStats = pRegEntry->idxStats;
+    idxStats += offPort;
+# ifdef IN_RING0
+    if (idxStats < pVM->iomr0.s.cIoPortStatsAllocation)
+        return &pVM->iomr0.s.paIoPortStats[idxStats];
+# else
+    if (idxStats < pVM->iom.s.cIoPortStats)
+        return &pVM->iom.s.paIoPortStats[idxStats];
+# endif
+    return &pVM->iom.s.IoPortDummyStats;
+}
+#endif
+
+
+/**
+ * Gets the MMIO region entry for the specified address in the current context.
+ *
+ * @returns Pointer to MMIO region entry.
+ * @returns NULL if no MMIO region registered for the given address.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The address to lookup.
+ * @param   poffRegion      Where to return the byte offset into the MMIO
+ *                          region that corresponds to @a GCPhys.
+ * @param   pidxLastHint    Pointer to IOMCPU::idxMmioLastRead,
+ *                          IOMCPU::idxMmioLastWrite, or similar.
+ *
+ * @note    In ring-0 it is possible to get an uninitialized entry (pDevIns is
+ *          NULL, cbRegion is 0), in which case there should be ring-3 handlers
+ *          for the entry.  Use IOMMMIOENTRYR0::idxSelf to get the ring-3 entry.
+ *
+ * @note    This code is almost identical to iomIoPortGetEntry, so keep in sync.
+ */
+DECLINLINE(CTX_SUFF(PIOMMMIOENTRY)) iomMmioGetEntry(PVMCC pVM, RTGCPHYS GCPhys, PRTGCPHYS poffRegion, uint16_t *pidxLastHint)
+{
+    Assert(IOM_IS_SHARED_LOCK_OWNER(pVM));
+
+#ifdef IN_RING0
+    uint32_t               iEnd      = RT_MIN(pVM->iom.s.cMmioLookupEntries, pVM->iomr0.s.cMmioAlloc);
+    PCIOMMMIOLOOKUPENTRY   paLookup  = pVM->iomr0.s.paMmioLookup;
+#else
+    uint32_t               iEnd      = pVM->iom.s.cMmioLookupEntries;
+    PCIOMMMIOLOOKUPENTRY   paLookup  = pVM->iom.s.paMmioLookup;
+#endif
+    if (iEnd > 0)
+    {
+        uint32_t iFirst = 0;
+        uint32_t i      = *pidxLastHint;
+        if (i < iEnd)
+        { /* likely */ }
+        else
+            i = iEnd / 2;
+        for (;;)
+        {
+            PCIOMMMIOLOOKUPENTRY pCur = &paLookup[i];
+            if (pCur->GCPhysFirst > GCPhys)
+            {
+                if (i > iFirst)
+                    iEnd = i;
+                else
+                    break;
+            }
+            else if (pCur->GCPhysLast < GCPhys)
+            {
+                i += 1;
+                if (i < iEnd)
+                    iFirst = i;
+                else
+                    break;
+            }
+            else
+            {
+                *pidxLastHint = (uint16_t)i;
+                *poffRegion   = GCPhys - pCur->GCPhysFirst;
+
+                /*
+                 * Translate the 'idx' member into a pointer.
+                 */
+                size_t const idx = pCur->idx;
+#ifdef IN_RING0
+                AssertMsg(idx < pVM->iom.s.cMmioRegs && idx < pVM->iomr0.s.cMmioAlloc,
+                          ("%#zx vs %#x/%x (GCPhys=%RGp)\n", idx, pVM->iom.s.cMmioRegs, pVM->iomr0.s.cMmioMax, GCPhys));
+                if (idx < pVM->iomr0.s.cMmioAlloc)
+                    return &pVM->iomr0.s.paMmioRegs[idx];
+#else
+                if (idx < pVM->iom.s.cMmioRegs)
+                    return &pVM->iom.s.paMmioRegs[idx];
+                AssertMsgFailed(("%#zx vs %#x (GCPhys=%RGp)\n", idx, pVM->iom.s.cMmioRegs, GCPhys));
+#endif
+                break;
+            }
+
+            i = iFirst + (iEnd - iFirst) / 2;
+        }
+    }
+    *poffRegion = 0;
+    return NULL;
+}
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * Gets the statistics entry for an MMIO region.
+ *
+ * @returns Pointer to stats.  Instead of NULL, a pointer to MmioDummyStats is
+ *          returned, so the caller does not need to check for NULL.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pRegEntry   The I/O port entry to get stats for.
+ */
+DECLINLINE(PIOMMMIOSTATSENTRY) iomMmioGetStats(PVMCC pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
+{
+    size_t idxStats = pRegEntry->idxStats;
+# ifdef IN_RING0
+    if (idxStats < pVM->iomr0.s.cMmioStatsAllocation)
+        return &pVM->iomr0.s.paMmioStats[idxStats];
+# else
+    if (idxStats < pVM->iom.s.cMmioStats)
+        return &pVM->iom.s.paMmioStats[idxStats];
+# endif
+    return &pVM->iom.s.MmioDummyStats;
+}
+#endif
+
+/** @}  */
+
+#endif /* !VMM_INCLUDED_SRC_include_IOMInline_h */
+
diff --git a/src/VBox/VMM/include/IOMInternal.h b/src/VBox/VMM/include/IOMInternal.h
new file mode 100644
index 00000000..6154a9de
--- /dev/null
+++ b/src/VBox/VMM/include/IOMInternal.h
@@ -0,0 +1,610 @@
+/* $Id: IOMInternal.h $ */
+/** @file
+ * IOM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_IOMInternal_h
+#define VMM_INCLUDED_SRC_include_IOMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#define IOM_WITH_CRIT_SECT_RW
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/pdmcritsect.h>
+#ifdef IOM_WITH_CRIT_SECT_RW
+# include <VBox/vmm/pdmcritsectrw.h>
+#endif
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <iprt/avl.h>
+
+
+
+/** @defgroup grp_iom_int   Internals
+ * @ingroup grp_iom
+ * @internal
+ * @{
+ */
+
+/**
+ * I/O port lookup table entry.
+ */
+typedef struct IOMIOPORTLOOKUPENTRY
+{
+    /** The first port in the range. */
+    RTIOPORT                    uFirstPort;
+    /** The last port in the range (inclusive). */
+    RTIOPORT                    uLastPort;
+    /** The registration handle/index. */
+    uint16_t                    idx;
+} IOMIOPORTLOOKUPENTRY;
+/** Pointer to an I/O port lookup table entry. */
+typedef IOMIOPORTLOOKUPENTRY *PIOMIOPORTLOOKUPENTRY;
+/** Pointer to a const I/O port lookup table entry. */
+typedef IOMIOPORTLOOKUPENTRY const *PCIOMIOPORTLOOKUPENTRY;
+
+/**
+ * Ring-0 I/O port handle table entry.
+ */
+typedef struct IOMIOPORTENTRYR0
+{
+    /** Pointer to user argument. */
+    RTR0PTR                             pvUser;
+    /** Pointer to the associated device instance, NULL if entry not used. */
+    R0PTRTYPE(PPDMDEVINS)               pDevIns;
+    /** Pointer to OUT callback function. */
+    R0PTRTYPE(PFNIOMIOPORTNEWOUT)       pfnOutCallback;
+    /** Pointer to IN callback function. */
+    R0PTRTYPE(PFNIOMIOPORTNEWIN)        pfnInCallback;
+    /** Pointer to string OUT callback function. */
+    R0PTRTYPE(PFNIOMIOPORTNEWOUTSTRING) pfnOutStrCallback;
+    /** Pointer to string IN callback function. */
+    R0PTRTYPE(PFNIOMIOPORTNEWINSTRING)  pfnInStrCallback;
+    /** The entry of the first statistics entry, UINT16_MAX if no stats. */
+    uint16_t                            idxStats;
+    /** The number of ports covered by this entry, 0 if entry not used. */
+    RTIOPORT                            cPorts;
+    /** Same as the handle index. */
+    uint16_t                            idxSelf;
+    /** IOM_IOPORT_F_XXX (copied from ring-3). */
+    uint16_t                            fFlags;
+} IOMIOPORTENTRYR0;
+/** Pointer to a ring-0 I/O port handle table entry. */
+typedef IOMIOPORTENTRYR0 *PIOMIOPORTENTRYR0;
+/** Pointer to a const ring-0 I/O port handle table entry. */
+typedef IOMIOPORTENTRYR0 const *PCIOMIOPORTENTRYR0;
+
+/**
+ * Ring-3 I/O port handle table entry.
+ */
+typedef struct IOMIOPORTENTRYR3
+{
+    /** Pointer to user argument. */
+    RTR3PTR                             pvUser;
+    /** Pointer to the associated device instance. */
+    R3PTRTYPE(PPDMDEVINS)               pDevIns;
+    /** Pointer to OUT callback function. */
+    R3PTRTYPE(PFNIOMIOPORTNEWOUT)       pfnOutCallback;
+    /** Pointer to IN callback function. */
+    R3PTRTYPE(PFNIOMIOPORTNEWIN)        pfnInCallback;
+    /** Pointer to string OUT callback function. */
+    R3PTRTYPE(PFNIOMIOPORTNEWOUTSTRING) pfnOutStrCallback;
+    /** Pointer to string IN callback function. */
+    R3PTRTYPE(PFNIOMIOPORTNEWINSTRING)  pfnInStrCallback;
+    /** Description / Name. For easing debugging. */
+    R3PTRTYPE(const char *)             pszDesc;
+    /** Extended port description table, optional. */
+    R3PTRTYPE(PCIOMIOPORTDESC)          paExtDescs;
+    /** PCI device the registration is associated with. */
+    R3PTRTYPE(PPDMPCIDEV)               pPciDev;
+    /** The PCI device region (high 16-bit word) and subregion (low word),
+     *  UINT32_MAX if not applicable. */
+    uint32_t                            iPciRegion;
+    /** The number of ports covered by this entry. */
+    RTIOPORT                            cPorts;
+    /** The current port mapping (duplicates lookup table). */
+    RTIOPORT                            uPort;
+    /** The entry of the first statistics entry, UINT16_MAX if no stats. */
+    uint16_t                            idxStats;
+    /** Set if mapped, clear if not.
+     * Only updated when critsect is held exclusively.   */
+    bool                                fMapped;
+    /** Set if there is an ring-0 entry too. */
+    bool                                fRing0;
+    /** Set if there is an raw-mode entry too. */
+    bool                                fRawMode;
+    /** IOM_IOPORT_F_XXX */
+    uint8_t                             fFlags;
+    /** Same as the handle index. */
+    uint16_t                            idxSelf;
+} IOMIOPORTENTRYR3;
+AssertCompileSize(IOMIOPORTENTRYR3, 9 * sizeof(RTR3PTR) + 16);
+/** Pointer to a ring-3 I/O port handle table entry. */
+typedef IOMIOPORTENTRYR3 *PIOMIOPORTENTRYR3;
+/** Pointer to a const ring-3 I/O port handle table entry. */
+typedef IOMIOPORTENTRYR3 const *PCIOMIOPORTENTRYR3;
+
+/**
+ * I/O port statistics entry (one I/O port).
+ */
+typedef struct IOMIOPORTSTATSENTRY
+{
+    /** All accesses (only updated for the first port in a range). */
+    STAMCOUNTER                 Total;
+
+    /** Number of INs to this port from R3. */
+    STAMCOUNTER                 InR3;
+    /** Profiling IN handler overhead in R3. */
+    STAMPROFILE                 ProfInR3;
+    /** Number of OUTs to this port from R3. */
+    STAMCOUNTER                 OutR3;
+    /** Profiling OUT handler overhead in R3. */
+    STAMPROFILE                 ProfOutR3;
+
+    /** Number of INs to this port from R0/RC. */
+    STAMCOUNTER                 InRZ;
+    /** Profiling IN handler overhead in R0/RC. */
+    STAMPROFILE                 ProfInRZ;
+    /** Number of INs to this port from R0/RC which was serviced in R3. */
+    STAMCOUNTER                 InRZToR3;
+
+    /** Number of OUTs to this port from R0/RC. */
+    STAMCOUNTER                 OutRZ;
+    /** Profiling OUT handler overhead in R0/RC. */
+    STAMPROFILE                 ProfOutRZ;
+    /** Number of OUTs to this port from R0/RC which was serviced in R3. */
+    STAMCOUNTER                 OutRZToR3;
+} IOMIOPORTSTATSENTRY;
+/** Pointer to I/O port statistics entry. */
+typedef IOMIOPORTSTATSENTRY *PIOMIOPORTSTATSENTRY;
+
+
+
+/**
+ * MMIO lookup table entry.
+ */
+typedef struct IOMMMIOLOOKUPENTRY
+{
+    /** The first port in the range. */
+    RTGCPHYS                    GCPhysFirst;
+    /** The last port in the range (inclusive). */
+    RTGCPHYS                    GCPhysLast;
+    /** The registration handle/index.
+     * @todo bake this into the lower/upper bits of GCPhysFirst & GCPhysLast. */
+    uint16_t                    idx;
+    uint16_t                    abPadding[3];
+} IOMMMIOLOOKUPENTRY;
+/** Pointer to an MMIO lookup table entry. */
+typedef IOMMMIOLOOKUPENTRY *PIOMMMIOLOOKUPENTRY;
+/** Pointer to a const MMIO lookup table entry. */
+typedef IOMMMIOLOOKUPENTRY const *PCIOMMMIOLOOKUPENTRY;
+
+/**
+ * Ring-0 MMIO handle table entry.
+ */
+typedef struct IOMMMIOENTRYR0
+{
+    /** The number of bytes covered by this entry, 0 if entry not used. */
+    RTGCPHYS                            cbRegion;
+    /** Pointer to user argument. */
+    RTR0PTR                             pvUser;
+    /** Pointer to the associated device instance, NULL if entry not used. */
+    R0PTRTYPE(PPDMDEVINS)               pDevIns;
+    /** Pointer to the write callback function. */
+    R0PTRTYPE(PFNIOMMMIONEWWRITE)       pfnWriteCallback;
+    /** Pointer to the read callback function. */
+    R0PTRTYPE(PFNIOMMMIONEWREAD)        pfnReadCallback;
+    /** Pointer to the fill callback function. */
+    R0PTRTYPE(PFNIOMMMIONEWFILL)        pfnFillCallback;
+    /** The entry of the first statistics entry, UINT16_MAX if no stats.
+     * @note For simplicity, this is always copied from ring-3 for all entries at
+     *       the end of VM creation. */
+    uint16_t                            idxStats;
+    /** Same as the handle index. */
+    uint16_t                            idxSelf;
+    /** IOM_MMIO_F_XXX (copied from ring-3). */
+    uint32_t                            fFlags;
+} IOMMMIOENTRYR0;
+/** Pointer to a ring-0 MMIO handle table entry. */
+typedef IOMMMIOENTRYR0 *PIOMMMIOENTRYR0;
+/** Pointer to a const ring-0 MMIO handle table entry. */
+typedef IOMMMIOENTRYR0 const *PCIOMMMIOENTRYR0;
+
+/**
+ * Ring-3 MMIO handle table entry.
+ */
+typedef struct IOMMMIOENTRYR3
+{
+    /** The number of bytes covered by this entry. */
+    RTGCPHYS                            cbRegion;
+    /** The current mapping address (duplicates lookup table).
+     * This is set to NIL_RTGCPHYS if not mapped (exclusive lock + atomic). */
+    RTGCPHYS volatile                   GCPhysMapping;
+    /** Pointer to user argument. */
+    RTR3PTR                             pvUser;
+    /** Pointer to the associated device instance. */
+    R3PTRTYPE(PPDMDEVINS)               pDevIns;
+    /** Pointer to the write callback function. */
+    R3PTRTYPE(PFNIOMMMIONEWWRITE)       pfnWriteCallback;
+    /** Pointer to the read callback function. */
+    R3PTRTYPE(PFNIOMMMIONEWREAD)        pfnReadCallback;
+    /** Pointer to the fill callback function. */
+    R3PTRTYPE(PFNIOMMMIONEWFILL)        pfnFillCallback;
+    /** Description / Name. For easing debugging. */
+    R3PTRTYPE(const char *)             pszDesc;
+    /** PCI device the registration is associated with. */
+    R3PTRTYPE(PPDMPCIDEV)               pPciDev;
+    /** The PCI device region (high 16-bit word) and subregion (low word),
+     *  UINT32_MAX if not applicable. */
+    uint32_t                            iPciRegion;
+    /** IOM_MMIO_F_XXX */
+    uint32_t                            fFlags;
+    /** The entry of the first statistics entry, UINT16_MAX if no stats. */
+    uint16_t                            idxStats;
+    /** Set if mapped, clear if not.
+     * Only updated when critsect is held exclusively.
+     * @todo remove as GCPhysMapping != NIL_RTGCPHYS serves the same purpose. */
+    bool volatile                       fMapped;
+    /** Set if there is an ring-0 entry too. */
+    bool                                fRing0;
+    /** Set if there is an raw-mode entry too. */
+    bool                                fRawMode;
+    uint8_t                             bPadding;
+    /** Same as the handle index. */
+    uint16_t                            idxSelf;
+} IOMMMIOENTRYR3;
+AssertCompileSize(IOMMMIOENTRYR3, sizeof(RTGCPHYS) * 2 + 7 * sizeof(RTR3PTR) + 16);
+/** Pointer to a ring-3 MMIO handle table entry. */
+typedef IOMMMIOENTRYR3 *PIOMMMIOENTRYR3;
+/** Pointer to a const ring-3 MMIO handle table entry. */
+typedef IOMMMIOENTRYR3 const *PCIOMMMIOENTRYR3;
+
+/**
+ * MMIO statistics entry (one MMIO).
+ */
+typedef struct IOMMMIOSTATSENTRY
+{
+    /** Counting and profiling reads in R0/RC. */
+    STAMPROFILE                 ProfReadRZ;
+    /** Number of successful read accesses. */
+    STAMCOUNTER                 Reads;
+    /** Number of reads to this address from R0/RC which was serviced in R3. */
+    STAMCOUNTER                 ReadRZToR3;
+    /** Number of complicated reads. */
+    STAMCOUNTER                 ComplicatedReads;
+    /** Number of reads of 0xff or 0x00. */
+    STAMCOUNTER                 FFor00Reads;
+    /** Profiling read handler overhead in R3. */
+    STAMPROFILE                 ProfReadR3;
+
+    /** Counting and profiling writes in R0/RC. */
+    STAMPROFILE                 ProfWriteRZ;
+    /** Number of successful read accesses. */
+    STAMCOUNTER                 Writes;
+    /** Number of writes to this address from R0/RC which was serviced in R3. */
+    STAMCOUNTER                 WriteRZToR3;
+    /** Number of writes to this address from R0/RC which was committed in R3. */
+    STAMCOUNTER                 CommitRZToR3;
+    /** Number of complicated writes. */
+    STAMCOUNTER                 ComplicatedWrites;
+    /** Profiling write handler overhead in R3. */
+    STAMPROFILE                 ProfWriteR3;
+} IOMMMIOSTATSENTRY;
+/** Pointer to MMIO statistics entry. */
+typedef IOMMMIOSTATSENTRY *PIOMMMIOSTATSENTRY;
+
+
+/**
+ * IOM per virtual CPU instance data.
+ */
+typedef struct IOMCPU
+{
+    /** For saving stack space, the disassembler state is allocated here instead of
+     * on the stack. */
+    DISCPUSTATE                     DisState;
+
+    /**
+     * Pending I/O port write commit (VINF_IOM_R3_IOPORT_COMMIT_WRITE).
+     *
+     * This is a converted VINF_IOM_R3_IOPORT_WRITE handler return that lets the
+     * execution engine commit the instruction and then return to ring-3 to complete
+     * the I/O port write there.  This avoids having to decode the instruction again
+     * in ring-3.
+     */
+    struct
+    {
+        /** The value size (0 if not pending). */
+        uint16_t                        cbValue;
+        /** The I/O port. */
+        RTIOPORT                        IOPort;
+        /** The value. */
+        uint32_t                        u32Value;
+    } PendingIOPortWrite;
+
+    /**
+     * Pending MMIO write commit (VINF_IOM_R3_MMIO_COMMIT_WRITE).
+     *
+     * This is a converted VINF_IOM_R3_MMIO_WRITE handler return that lets the
+     * execution engine commit the instruction, stop any more REPs, and return to
+     * ring-3 to complete the MMIO write there.  The avoid the tedious decoding of
+     * the instruction again once we're in ring-3, more importantly it allows us to
+     * correctly deal with read-modify-write instructions like XCHG, OR, and XOR.
+     */
+    struct
+    {
+        /** Guest physical MMIO address. */
+        RTGCPHYS                        GCPhys;
+        /** The number of bytes to write (0 if nothing pending). */
+        uint32_t                        cbValue;
+        /** Hint. */
+        uint32_t                        idxMmioRegionHint;
+        /** The value to write. */
+        uint8_t                         abValue[128];
+    } PendingMmioWrite;
+
+    /** @name Caching of I/O Port and MMIO ranges and statistics.
+     * (Saves quite some time in rep outs/ins instruction emulation.)
+     * @{ */
+    /** I/O port registration index for the last read operation. */
+    uint16_t                            idxIoPortLastRead;
+    /** I/O port registration index for the last write operation. */
+    uint16_t                            idxIoPortLastWrite;
+    /** I/O port registration index for the last read string operation. */
+    uint16_t                            idxIoPortLastReadStr;
+    /** I/O port registration index for the last write string operation. */
+    uint16_t                            idxIoPortLastWriteStr;
+
+    /** MMIO port registration index for the last IOMR3MmioPhysHandler call.
+     * @note pretty static as only used by APIC on AMD-V.  */
+    uint16_t                            idxMmioLastPhysHandler;
+    uint16_t                            au16Padding[3];
+    /** @} */
+} IOMCPU;
+/** Pointer to IOM per virtual CPU instance data. */
+typedef IOMCPU *PIOMCPU;
+
+
+/**
+ * IOM Data (part of VM)
+ */
+typedef struct IOM
+{
+    /** @name I/O ports
+     * @note The updating of these variables is done exclusively from EMT(0).
+     * @{ */
+    /** Number of I/O port registrations. */
+    uint32_t                        cIoPortRegs;
+    /** The size of the paIoPortRegs allocation (in entries). */
+    uint32_t                        cIoPortAlloc;
+    /** I/O port registration table for ring-3.
+     * There is a parallel table in ring-0, IOMR0PERVM::paIoPortRegs. */
+    R3PTRTYPE(PIOMIOPORTENTRYR3)    paIoPortRegs;
+    /** Number of entries in the lookup table. */
+    uint32_t                        cIoPortLookupEntries;
+    uint32_t                        u32Padding1;
+    /** I/O port lookup table. */
+    R3PTRTYPE(PIOMIOPORTLOOKUPENTRY) paIoPortLookup;
+
+    /** The number of valid entries in paioPortStats. */
+    uint32_t                        cIoPortStats;
+    /** The size of the paIoPortStats allocation (in entries). */
+    uint32_t                        cIoPortStatsAllocation;
+    /** I/O port lookup table.   */
+    R3PTRTYPE(PIOMIOPORTSTATSENTRY) paIoPortStats;
+    /** Dummy stats entry so we don't need to check for NULL pointers so much. */
+    IOMIOPORTSTATSENTRY             IoPortDummyStats;
+    /** @} */
+
+    /** @name MMIO ports
+     * @note The updating of these variables is done exclusively from EMT(0).
+     * @{ */
+    /** MMIO physical access handler type, new style.   */
+    PGMPHYSHANDLERTYPE              hNewMmioHandlerType;
+    /** Number of MMIO registrations. */
+    uint32_t                        cMmioRegs;
+    /** The size of the paMmioRegs allocation (in entries). */
+    uint32_t                        cMmioAlloc;
+    /** MMIO registration table for ring-3.
+     * There is a parallel table in ring-0, IOMR0PERVM::paMmioRegs. */
+    R3PTRTYPE(PIOMMMIOENTRYR3)      paMmioRegs;
+    /** Number of entries in the lookup table. */
+    uint32_t                        cMmioLookupEntries;
+    uint32_t                        u32Padding2;
+    /** MMIO lookup table. */
+    R3PTRTYPE(PIOMMMIOLOOKUPENTRY)  paMmioLookup;
+
+    /** The number of valid entries in paioPortStats. */
+    uint32_t                        cMmioStats;
+    /** The size of the paMmioStats allocation (in entries). */
+    uint32_t                        cMmioStatsAllocation;
+    /** MMIO lookup table.   */
+    R3PTRTYPE(PIOMMMIOSTATSENTRY)   paMmioStats;
+    /** Dummy stats entry so we don't need to check for NULL pointers so much. */
+    IOMMMIOSTATSENTRY               MmioDummyStats;
+    /** @} */
+
+
+    /** Lock serializing EMT access to IOM. */
+#ifdef IOM_WITH_CRIT_SECT_RW
+    PDMCRITSECTRW                   CritSect;
+#else
+    PDMCRITSECT                     CritSect;
+#endif
+
+    /** @name I/O Port statistics.
+     * @{ */
+    STAMCOUNTER                     StatIoPortIn;
+    STAMCOUNTER                     StatIoPortOut;
+    STAMCOUNTER                     StatIoPortInS;
+    STAMCOUNTER                     StatIoPortOutS;
+    STAMCOUNTER                     StatIoPortCommits;
+    /** @} */
+
+    /** @name MMIO statistics.
+     * @{ */
+    STAMPROFILE                     StatMmioPfHandler;
+    STAMPROFILE                     StatMmioPhysHandler;
+    STAMCOUNTER                     StatMmioHandlerR3;
+    STAMCOUNTER                     StatMmioHandlerR0;
+    STAMCOUNTER                     StatMmioReadsR0ToR3;
+    STAMCOUNTER                     StatMmioWritesR0ToR3;
+    STAMCOUNTER                     StatMmioCommitsR0ToR3;
+    STAMCOUNTER                     StatMmioCommitsDirect;
+    STAMCOUNTER                     StatMmioCommitsPgm;
+    STAMCOUNTER                     StatMmioStaleMappings;
+    STAMCOUNTER                     StatMmioDevLockContentionR0;
+    /** @} */
+} IOM;
+/** Pointer to IOM instance data. */
+typedef IOM *PIOM;
+
+
+/**
+ * IOM data kept in the ring-0 GVM.
+ */
+typedef struct IOMR0PERVM
+{
+    /** @name I/O ports
+     * @{ */
+    /** The higest ring-0 I/O port registration plus one. */
+    uint32_t                        cIoPortMax;
+    /** The size of the paIoPortRegs allocation (in entries). */
+    uint32_t                        cIoPortAlloc;
+    /** I/O port registration table for ring-0.
+     * There is a parallel table for ring-3, paIoPortRing3Regs. */
+    R0PTRTYPE(PIOMIOPORTENTRYR0)    paIoPortRegs;
+    /** I/O port lookup table. */
+    R0PTRTYPE(PIOMIOPORTLOOKUPENTRY) paIoPortLookup;
+    /** I/O port registration table for ring-3.
+     * Also mapped to ring-3 as IOM::paIoPortRegs. */
+    R0PTRTYPE(PIOMIOPORTENTRYR3)    paIoPortRing3Regs;
+    /** Handle to the allocation backing both the ring-0 and ring-3 registration
+     * tables as well as the lookup table. */
+    RTR0MEMOBJ                      hIoPortMemObj;
+    /** Handle to the ring-3 mapping of the lookup and ring-3 registration table. */
+    RTR0MEMOBJ                      hIoPortMapObj;
+#ifdef VBOX_WITH_STATISTICS
+    /** The size of the paIoPortStats allocation (in entries). */
+    uint32_t                        cIoPortStatsAllocation;
+    /** Prevents paIoPortStats from growing, set by IOMR0IoPortSyncStatisticsIndices(). */
+    bool                            fIoPortStatsFrozen;
+    /** I/O port lookup table.   */
+    R0PTRTYPE(PIOMIOPORTSTATSENTRY) paIoPortStats;
+    /** Handle to the allocation backing the I/O port statistics. */
+    RTR0MEMOBJ                      hIoPortStatsMemObj;
+    /** Handle to the ring-3 mapping of the I/O port statistics. */
+    RTR0MEMOBJ                      hIoPortStatsMapObj;
+#endif
+    /** @} */
+
+    /** @name MMIO
+     * @{ */
+    /** The higest ring-0 MMIO registration plus one. */
+    uint32_t                        cMmioMax;
+    /** The size of the paMmioRegs allocation (in entries). */
+    uint32_t                        cMmioAlloc;
+    /** MMIO registration table for ring-0.
+     * There is a parallel table for ring-3, paMmioRing3Regs. */
+    R0PTRTYPE(PIOMMMIOENTRYR0)      paMmioRegs;
+    /** MMIO lookup table. */
+    R0PTRTYPE(PIOMMMIOLOOKUPENTRY)  paMmioLookup;
+    /** MMIO registration table for ring-3.
+     * Also mapped to ring-3 as IOM::paMmioRegs. */
+    R0PTRTYPE(PIOMMMIOENTRYR3)      paMmioRing3Regs;
+    /** Handle to the allocation backing both the ring-0 and ring-3 registration
+     * tables as well as the lookup table. */
+    RTR0MEMOBJ                      hMmioMemObj;
+    /** Handle to the ring-3 mapping of the lookup and ring-3 registration table. */
+    RTR0MEMOBJ                      hMmioMapObj;
+#ifdef VBOX_WITH_STATISTICS
+    /** The size of the paMmioStats allocation (in entries). */
+    uint32_t                        cMmioStatsAllocation;
+    /* Prevents paMmioStats from growing, set by IOMR0MmioSyncStatisticsIndices(). */
+    bool                            fMmioStatsFrozen;
+    /** MMIO lookup table.   */
+    R0PTRTYPE(PIOMMMIOSTATSENTRY)   paMmioStats;
+    /** Handle to the allocation backing the MMIO statistics. */
+    RTR0MEMOBJ                      hMmioStatsMemObj;
+    /** Handle to the ring-3 mapping of the MMIO statistics. */
+    RTR0MEMOBJ                      hMmioStatsMapObj;
+#endif
+    /** @} */
+
+} IOMR0PERVM;
+
+
+RT_C_DECLS_BEGIN
+
+#ifdef IN_RING3
+DECLCALLBACK(void)  iomR3IoPortInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+void                iomR3IoPortRegStats(PVM pVM, PIOMIOPORTENTRYR3 pRegEntry);
+DECLCALLBACK(void)  iomR3MmioInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+void                iomR3MmioRegStats(PVM pVM, PIOMMMIOENTRYR3 pRegEntry);
+VBOXSTRICTRC        iomR3MmioCommitWorker(PVM pVM, PVMCPU pVCpu, PIOMMMIOENTRYR3 pRegEntry, RTGCPHYS offRegion); /* IOMAllMmioNew.cpp */
+#endif /* IN_RING3 */
+#ifdef IN_RING0
+void                iomR0IoPortCleanupVM(PGVM pGVM);
+void                iomR0IoPortInitPerVMData(PGVM pGVM);
+void                iomR0MmioCleanupVM(PGVM pGVM);
+void                iomR0MmioInitPerVMData(PGVM pGVM);
+#endif
+
+#ifndef IN_RING3
+DECLEXPORT(FNPGMRZPHYSPFHANDLER)    iomMmioPfHandlerNew;
+#endif
+PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) iomMmioHandlerNew;
+
+/* IOM locking helpers. */
+#ifdef IOM_WITH_CRIT_SECT_RW
+# define IOM_LOCK_EXCL(a_pVM)                   PDMCritSectRwEnterExcl(&(a_pVM)->iom.s.CritSect, VERR_SEM_BUSY)
+# define IOM_UNLOCK_EXCL(a_pVM)                 do { PDMCritSectRwLeaveExcl(&(a_pVM)->iom.s.CritSect); } while (0)
+# if 0 /* (in case needed for debugging) */
+# define IOM_LOCK_SHARED_EX(a_pVM, a_rcBusy)    PDMCritSectRwEnterExcl(&(a_pVM)->iom.s.CritSect, (a_rcBusy))
+# define IOM_UNLOCK_SHARED(a_pVM)               do { PDMCritSectRwLeaveExcl(&(a_pVM)->iom.s.CritSect); } while (0)
+# define IOM_IS_SHARED_LOCK_OWNER(a_pVM)        PDMCritSectRwIsWriteOwner(&(a_pVM)->iom.s.CritSect)
+# else
+# define IOM_LOCK_SHARED_EX(a_pVM, a_rcBusy)    PDMCritSectRwEnterShared(&(a_pVM)->iom.s.CritSect, (a_rcBusy))
+# define IOM_UNLOCK_SHARED(a_pVM)               do { PDMCritSectRwLeaveShared(&(a_pVM)->iom.s.CritSect); } while (0)
+# define IOM_IS_SHARED_LOCK_OWNER(a_pVM)        PDMCritSectRwIsReadOwner(&(a_pVM)->iom.s.CritSect, true)
+# endif
+# define IOM_IS_EXCL_LOCK_OWNER(a_pVM)          PDMCritSectRwIsWriteOwner(&(a_pVM)->iom.s.CritSect)
+#else
+# define IOM_LOCK_EXCL(a_pVM)                   PDMCritSectEnter(&(a_pVM)->iom.s.CritSect, VERR_SEM_BUSY)
+# define IOM_UNLOCK_EXCL(a_pVM)                 do { PDMCritSectLeave(&(a_pVM)->iom.s.CritSect); } while (0)
+# define IOM_LOCK_SHARED_EX(a_pVM, a_rcBusy)    PDMCritSectEnter(&(a_pVM)->iom.s.CritSect, (a_rcBusy))
+# define IOM_UNLOCK_SHARED(a_pVM)               do { PDMCritSectLeave(&(a_pVM)->iom.s.CritSect); } while (0)
+# define IOM_IS_SHARED_LOCK_OWNER(a_pVM)        PDMCritSectIsOwner(&(a_pVM)->iom.s.CritSect)
+# define IOM_IS_EXCL_LOCK_OWNER(a_pVM)          PDMCritSectIsOwner(&(a_pVM)->iom.s.CritSect)
+#endif
+#define IOM_LOCK_SHARED(a_pVM)                  IOM_LOCK_SHARED_EX(a_pVM, VERR_SEM_BUSY)
+
+
+RT_C_DECLS_END
+
+
+#ifdef IN_RING3
+
+#endif
+
+/** @} */
+
+#endif /* !VMM_INCLUDED_SRC_include_IOMInternal_h */
+
diff --git a/src/VBox/VMM/include/MMInternal.h b/src/VBox/VMM/include/MMInternal.h
new file mode 100644
index 00000000..e29cb156
--- /dev/null
+++ b/src/VBox/VMM/include/MMInternal.h
@@ -0,0 +1,661 @@
+/* $Id: MMInternal.h $ */
+/** @file
+ * MM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_MMInternal_h
+#define VMM_INCLUDED_SRC_include_MMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/pdmcritsect.h>
+#include <iprt/assert.h>
+#include <iprt/avl.h>
+#include <iprt/critsect.h>
+
+
+
+/** @defgroup grp_mm_int   Internals
+ * @ingroup grp_mm
+ * @internal
+ * @{
+ */
+
+
+/** @name MMR3Heap - VM Ring-3 Heap Internals
+ * @{
+ */
+
+/** @def MMR3HEAP_SIZE_ALIGNMENT
+ * The allocation size alignment of the MMR3Heap.
+ */
+#define MMR3HEAP_SIZE_ALIGNMENT     16
+
+/** @def MMR3HEAP_WITH_STATISTICS
+ * Enable MMR3Heap statistics.
+ */
+#if !defined(MMR3HEAP_WITH_STATISTICS) && defined(VBOX_WITH_STATISTICS)
+# define MMR3HEAP_WITH_STATISTICS
+#endif
+
+/**
+ * Heap statistics record.
+ * There is one global and one per allocation tag.
+ */
+typedef struct MMHEAPSTAT
+{
+    /** Core avl node, key is the tag. */
+    AVLULNODECORE           Core;
+    /** Pointer to the heap the memory belongs to. */
+    struct MMHEAP          *pHeap;
+#ifdef MMR3HEAP_WITH_STATISTICS
+    /** Number of bytes currently allocated. */
+    size_t                  cbCurAllocated;
+    /** Number of allocation. */
+    uint64_t                cAllocations;
+    /** Number of reallocations. */
+    uint64_t                cReallocations;
+    /** Number of frees. */
+    uint64_t                cFrees;
+    /** Failures. */
+    uint64_t                cFailures;
+    /** Number of bytes allocated (sum). */
+    uint64_t                cbAllocated;
+    /** Number of bytes freed. */
+    uint64_t                cbFreed;
+#endif
+} MMHEAPSTAT;
+#if defined(MMR3HEAP_WITH_STATISTICS) && defined(IN_RING3)
+AssertCompileMemberAlignment(MMHEAPSTAT, cAllocations, 8);
+AssertCompileSizeAlignment(MMHEAPSTAT, 8);
+#endif
+/** Pointer to heap statistics record. */
+typedef MMHEAPSTAT *PMMHEAPSTAT;
+
+
+
+
+/**
+ * Additional heap block header for relating allocations to the VM.
+ */
+typedef struct MMHEAPHDR
+{
+    /** Pointer to the next record. */
+    struct MMHEAPHDR       *pNext;
+    /** Pointer to the previous record. */
+    struct MMHEAPHDR       *pPrev;
+    /** Pointer to the heap statistics record.
+     * (Where the a PVM can be found.) */
+    PMMHEAPSTAT             pStat;
+    /** Size of the allocation (including this header). */
+    size_t                  cbSize;
+} MMHEAPHDR;
+/** Pointer to MM heap header. */
+typedef MMHEAPHDR *PMMHEAPHDR;
+
+
+/** MM Heap structure. */
+typedef struct MMHEAP
+{
+    /** Lock protecting the heap. */
+    RTCRITSECT              Lock;
+    /** Heap block list head. */
+    PMMHEAPHDR              pHead;
+    /** Heap block list tail. */
+    PMMHEAPHDR              pTail;
+    /** Heap per tag statistics tree. */
+    PAVLULNODECORE          pStatTree;
+    /** The VM handle. */
+    PUVM                    pUVM;
+    /** Heap global statistics. */
+    MMHEAPSTAT              Stat;
+} MMHEAP;
+/** Pointer to MM Heap structure. */
+typedef MMHEAP *PMMHEAP;
+
+/** @} */
+
+
+/** @name MMUkHeap - VM User-kernel Heap Internals
+ * @{
+ */
+
+/** @def MMUKHEAP_SIZE_ALIGNMENT
+ * The allocation size alignment of the MMR3UkHeap.
+ */
+#define MMUKHEAP_SIZE_ALIGNMENT   16
+
+/** @def MMUKHEAP_WITH_STATISTICS
+ * Enable MMUkHeap statistics.
+ */
+#if !defined(MMUKHEAP_WITH_STATISTICS) && defined(VBOX_WITH_STATISTICS)
+# define MMUKHEAP_WITH_STATISTICS
+#endif
+
+
+/**
+ * Heap statistics record.
+ * There is one global and one per allocation tag.
+ */
+typedef struct MMUKHEAPSTAT
+{
+    /** Core avl node, key is the tag. */
+    AVLULNODECORE           Core;
+    /** Number of allocation. */
+    uint64_t                cAllocations;
+    /** Number of reallocations. */
+    uint64_t                cReallocations;
+    /** Number of frees. */
+    uint64_t                cFrees;
+    /** Failures. */
+    uint64_t                cFailures;
+    /** Number of bytes allocated (sum). */
+    uint64_t                cbAllocated;
+    /** Number of bytes freed. */
+    uint64_t                cbFreed;
+    /** Number of bytes currently allocated. */
+    size_t                  cbCurAllocated;
+} MMUKHEAPSTAT;
+#ifdef IN_RING3
+AssertCompileMemberAlignment(MMUKHEAPSTAT, cAllocations, 8);
+#endif
+/** Pointer to heap statistics record. */
+typedef MMUKHEAPSTAT *PMMUKHEAPSTAT;
+
+/**
+ * Sub heap tracking record.
+ */
+typedef struct MMUKHEAPSUB
+{
+    /** Pointer to the next sub-heap. */
+    struct MMUKHEAPSUB     *pNext;
+    /** The base address of the sub-heap. */
+    void                   *pv;
+    /** The size of the sub-heap.  */
+    size_t                  cb;
+    /** The handle of the simple block pointer. */
+    RTHEAPSIMPLE            hSimple;
+    /** The ring-0 address corresponding to MMUKHEAPSUB::pv. */
+    RTR0PTR                 pvR0;
+} MMUKHEAPSUB;
+/** Pointer to a sub-heap tracking record. */
+typedef MMUKHEAPSUB *PMMUKHEAPSUB;
+
+
+/** MM User-kernel Heap structure. */
+typedef struct MMUKHEAP
+{
+    /** Lock protecting the heap. */
+    RTCRITSECT              Lock;
+    /** Head of the sub-heap LIFO. */
+    PMMUKHEAPSUB            pSubHeapHead;
+    /** Heap per tag statistics tree. */
+    PAVLULNODECORE          pStatTree;
+    /** The VM handle. */
+    PUVM                    pUVM;
+#if HC_ARCH_BITS == 32
+    /** Aligning the statistics on an 8 byte boundary (for uint64_t and STAM). */
+    void                   *pvAlignment;
+#endif
+    /** Heap global statistics. */
+    MMUKHEAPSTAT            Stat;
+} MMUKHEAP;
+#ifdef IN_RING3
+AssertCompileMemberAlignment(MMUKHEAP, Stat, 8);
+#endif
+/** Pointer to MM Heap structure. */
+typedef MMUKHEAP *PMMUKHEAP;
+
+/** @} */
+
+
+
+/** @name Hypervisor Heap Internals
+ * @{
+ */
+
+/** @def MMHYPER_HEAP_FREE_DELAY
+ * If defined, it indicates the number of frees that should be delayed.
+ */
+#if defined(DOXYGEN_RUNNING)
+# define MMHYPER_HEAP_FREE_DELAY            64
+#endif
+
+/** @def MMHYPER_HEAP_FREE_POISON
+ * If defined, it indicates that freed memory should be poisoned
+ * with the value it has.
+ */
+#if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING)
+# define MMHYPER_HEAP_FREE_POISON           0xcb
+#endif
+
+/** @def MMHYPER_HEAP_STRICT
+ * Enables a bunch of assertions in the heap code. */
+#if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING)
+# define MMHYPER_HEAP_STRICT 1
+# if 0 || defined(DOXYGEN_RUNNING)
+/** @def MMHYPER_HEAP_STRICT_FENCE
+ * Enables tail fence. */
+#  define MMHYPER_HEAP_STRICT_FENCE
+/** @def MMHYPER_HEAP_STRICT_FENCE_SIZE
+ * The fence size in bytes. */
+#  define MMHYPER_HEAP_STRICT_FENCE_SIZE    256
+/** @def MMHYPER_HEAP_STRICT_FENCE_U32
+ * The fence filler. */
+#  define MMHYPER_HEAP_STRICT_FENCE_U32     UINT32_C(0xdeadbeef)
+# endif
+#endif
+
+/**
+ * Hypervisor heap statistics record.
+ * There is one global and one per allocation tag.
+ */
+typedef struct MMHYPERSTAT
+{
+    /** Core avl node, key is the tag.
+     * @todo The type is wrong! Get your lazy a$$ over and create that offsetted uint32_t version we need here!  */
+    AVLOGCPHYSNODECORE      Core;
+    /** Aligning the 64-bit fields on a 64-bit line. */
+    uint32_t                u32Padding0;
+    /** Indicator for whether these statistics are registered with STAM or not. */
+    bool                    fRegistered;
+    /** Number of allocation. */
+    uint64_t                cAllocations;
+    /** Number of frees. */
+    uint64_t                cFrees;
+    /** Failures. */
+    uint64_t                cFailures;
+    /** Number of bytes allocated (sum). */
+    uint64_t                cbAllocated;
+    /** Number of bytes freed (sum). */
+    uint64_t                cbFreed;
+    /** Number of bytes currently allocated. */
+    uint32_t                cbCurAllocated;
+    /** Max number of bytes allocated. */
+    uint32_t                cbMaxAllocated;
+} MMHYPERSTAT;
+AssertCompileMemberAlignment(MMHYPERSTAT, cAllocations, 8);
+/** Pointer to hypervisor heap statistics record. */
+typedef MMHYPERSTAT *PMMHYPERSTAT;
+
+/**
+ * Hypervisor heap chunk.
+ */
+typedef struct MMHYPERCHUNK
+{
+    /** Previous block in the list of all blocks.
+     * This is relative to the start of the heap. */
+    uint32_t                offNext;
+    /** Offset to the previous block relative to this one. */
+    int32_t                 offPrev;
+    /** The statistics record this allocation belongs to (self relative). */
+    int32_t                 offStat;
+    /** Offset to the heap block (self relative). */
+    int32_t                 offHeap;
+} MMHYPERCHUNK;
+/** Pointer to a hypervisor heap chunk. */
+typedef MMHYPERCHUNK *PMMHYPERCHUNK;
+
+
+/**
+ * Hypervisor heap chunk.
+ */
+typedef struct MMHYPERCHUNKFREE
+{
+    /** Main list. */
+    MMHYPERCHUNK            core;
+    /** Offset of the next chunk in the list of free nodes. */
+    uint32_t                offNext;
+    /** Offset of the previous chunk in the list of free nodes. */
+    int32_t                 offPrev;
+    /** Size of the block. */
+    uint32_t                cb;
+} MMHYPERCHUNKFREE;
+/** Pointer to a free hypervisor heap chunk. */
+typedef MMHYPERCHUNKFREE *PMMHYPERCHUNKFREE;
+
+
+/**
+ * The hypervisor heap.
+ */
+typedef struct MMHYPERHEAP
+{
+    /** The typical magic (MMHYPERHEAP_MAGIC). */
+    uint32_t                u32Magic;
+    /** The heap size. (This structure is not included!) */
+    uint32_t                cbHeap;
+    /** Lock protecting the heap. */
+    PDMCRITSECT             Lock;
+    /** The HC ring-3 address of the heap. */
+    R3PTRTYPE(uint8_t *)    pbHeapR3;
+    /** The HC ring-3 address of the shared VM structure. */
+    PVMR3                   pVMR3;
+    /** The HC ring-0 address of the heap. */
+    R0PTRTYPE(uint8_t *)    pbHeapR0;
+    /** The HC ring-0 address of the shared VM structure. */
+    PVMR0                   pVMR0;
+    /** The RC address of the heap. */
+    RCPTRTYPE(uint8_t *)    pbHeapRC;
+    /** The RC address of the shared VM structure. */
+    PVMRC                   pVMRC;
+    /** The amount of free memory in the heap. */
+    uint32_t                cbFree;
+    /** Offset of the first free chunk in the heap.
+     * The offset is relative to the start of the heap. */
+    uint32_t                offFreeHead;
+    /** Offset of the last free chunk in the heap.
+     * The offset is relative to the start of the heap. */
+    uint32_t                offFreeTail;
+    /** Offset of the first page aligned block in the heap.
+     * The offset is equal to cbHeap initially. */
+    uint32_t                offPageAligned;
+    /** Tree of hypervisor heap statistics. */
+    AVLOGCPHYSTREE          HyperHeapStatTree;
+#ifdef MMHYPER_HEAP_FREE_DELAY
+    /** Where to insert the next free. */
+    uint32_t                iDelayedFree;
+    /** Array of delayed frees. Circular. Offsets relative to this structure. */
+    struct
+    {
+        /** The free caller address. */
+        RTUINTPTR           uCaller;
+        /** The offset of the freed chunk. */
+        uint32_t            offChunk;
+    } aDelayedFrees[MMHYPER_HEAP_FREE_DELAY];
+#else
+    /** Padding the structure to a 64-bit aligned size. */
+    uint32_t                u32Padding0;
+#endif
+    /** The heap physical pages. */
+    R3PTRTYPE(PSUPPAGE)     paPages;
+#if HC_ARCH_BITS == 32
+    /** Padding the structure to a 64-bit aligned size. */
+    uint32_t                u32Padding1;
+#endif
+} MMHYPERHEAP;
+/** Pointer to the hypervisor heap. */
+typedef MMHYPERHEAP *PMMHYPERHEAP;
+
+/** Magic value for MMHYPERHEAP. (C. S. Lewis) */
+#define MMHYPERHEAP_MAGIC               UINT32_C(0x18981129)
+
+
+/**
+ * Hypervisor heap minimum alignment (16 bytes).
+ */
+#define MMHYPER_HEAP_ALIGN_MIN          16
+
+/**
+ * The aligned size of the MMHYPERHEAP structure.
+ */
+#define MMYPERHEAP_HDR_SIZE             RT_ALIGN_Z(sizeof(MMHYPERHEAP), MMHYPER_HEAP_ALIGN_MIN * 4)
+
+/** @name Hypervisor heap chunk flags.
+ * The flags are put in the first bits of the MMHYPERCHUNK::offPrev member.
+ * These bits aren't used anyway because of the chunk minimal alignment (16 bytes).
+ * @{ */
+/** The chunk is free. (The code ASSUMES this is 0!) */
+#define MMHYPERCHUNK_FLAGS_FREE         0x0
+/** The chunk is in use. */
+#define MMHYPERCHUNK_FLAGS_USED         0x1
+/** The type mask. */
+#define MMHYPERCHUNK_FLAGS_TYPE_MASK    0x1
+/** The flag mask */
+#define MMHYPERCHUNK_FLAGS_MASK         0x1
+
+/** Checks if the chunk is free. */
+#define MMHYPERCHUNK_ISFREE(pChunk)                 ( (((pChunk)->offPrev) & MMHYPERCHUNK_FLAGS_TYPE_MASK) == MMHYPERCHUNK_FLAGS_FREE )
+/** Checks if the chunk is used. */
+#define MMHYPERCHUNK_ISUSED(pChunk)                 ( (((pChunk)->offPrev) & MMHYPERCHUNK_FLAGS_TYPE_MASK) == MMHYPERCHUNK_FLAGS_USED )
+/** Toggles FREE/USED flag of a chunk. */
+#define MMHYPERCHUNK_SET_TYPE(pChunk, type)         do { (pChunk)->offPrev = ((pChunk)->offPrev & ~MMHYPERCHUNK_FLAGS_TYPE_MASK) | ((type) & MMHYPERCHUNK_FLAGS_TYPE_MASK); } while (0)
+
+/** Gets the prev offset without the flags. */
+#define MMHYPERCHUNK_GET_OFFPREV(pChunk)            ((int32_t)((pChunk)->offPrev & ~MMHYPERCHUNK_FLAGS_MASK))
+/** Sets the prev offset without changing the flags. */
+#define MMHYPERCHUNK_SET_OFFPREV(pChunk, off)       do { (pChunk)->offPrev = (off) | ((pChunk)->offPrev & MMHYPERCHUNK_FLAGS_MASK); } while (0)
+#if 0
+/** Clears one or more flags. */
+#define MMHYPERCHUNK_FLAGS_OP_CLEAR(pChunk, fFlags) do { ((pChunk)->offPrev) &= ~((fFlags) & MMHYPERCHUNK_FLAGS_MASK); } while (0)
+/** Sets one or more flags. */
+#define MMHYPERCHUNK_FLAGS_OP_SET(pChunk, fFlags)   do { ((pChunk)->offPrev) |=  ((fFlags) & MMHYPERCHUNK_FLAGS_MASK); } while (0)
+/** Checks if one is set. */
+#define MMHYPERCHUNK_FLAGS_OP_ISSET(pChunk, fFlag)  (!!(((pChunk)->offPrev) & ((fFlag) & MMHYPERCHUNK_FLAGS_MASK)))
+#endif
+/** @} */
+
+/** @} */
+
+/**
+ * Hypervisor memory mapping type.
+ */
+typedef enum MMLOOKUPHYPERTYPE
+{
+    /** Invalid record. This is used for record which are incomplete. */
+    MMLOOKUPHYPERTYPE_INVALID = 0,
+    /** Mapping of locked memory. */
+    MMLOOKUPHYPERTYPE_LOCKED,
+    /** Mapping of contiguous HC physical memory. */
+    MMLOOKUPHYPERTYPE_HCPHYS,
+    /** Mapping of contiguous GC physical memory. */
+    MMLOOKUPHYPERTYPE_GCPHYS,
+    /** Mapping of MMIO2 memory. */
+    MMLOOKUPHYPERTYPE_MMIO2,
+    /** Dynamic mapping area (MMR3HyperReserve).
+     * A conversion will require to check what's in the page table for the pages. */
+    MMLOOKUPHYPERTYPE_DYNAMIC
+} MMLOOKUPHYPERTYPE;
+
+/**
+ * Lookup record for the hypervisor memory area.
+ */
+typedef struct MMLOOKUPHYPER
+{
+    /** Byte offset from the start of this record to the next.
+     * If the value is NIL_OFFSET the chain is terminated. */
+    int32_t                 offNext;
+    /** Offset into the hypervisor memory area. */
+    uint32_t                off;
+    /** Size of this part. */
+    uint32_t                cb;
+    /** Locking type. */
+    MMLOOKUPHYPERTYPE       enmType;
+    /** Type specific data */
+    union
+    {
+        /** Locked memory. */
+        struct
+        {
+            /** Host context ring-3 pointer. */
+            R3PTRTYPE(void *)       pvR3;
+            /** Host context ring-0 pointer. Optional. */
+            RTR0PTR                 pvR0;
+            /** Pointer to an array containing the physical address of each page. */
+            R3PTRTYPE(PRTHCPHYS)    paHCPhysPages;
+        } Locked;
+
+        /** Contiguous physical memory. */
+        struct
+        {
+            /** Host context ring-3 pointer. */
+            R3PTRTYPE(void *)       pvR3;
+            /** Host context ring-0 pointer. Optional. */
+            RTR0PTR                 pvR0;
+            /** HC physical address corresponding to pvR3/pvR0. */
+            RTHCPHYS                HCPhys;
+        } HCPhys;
+
+        /** Contiguous guest physical memory. */
+        struct
+        {
+            /** The memory address (Guest Context). */
+            RTGCPHYS                GCPhys;
+        } GCPhys;
+
+        /** MMIO2 memory. */
+        struct
+        {
+            /** The device instance owning the MMIO2 region. */
+            PPDMDEVINSR3            pDevIns;
+            /** The sub-device number. */
+            uint32_t                iSubDev;
+            /** The region number. */
+            uint32_t                iRegion;
+#if HC_ARCH_BITS == 32
+            /** Alignment padding. */
+            uint32_t                uPadding;
+#endif
+            /** The offset into the MMIO2 region. */
+            RTGCPHYS                off;
+        } MMIO2;
+    } u;
+    /** Description. */
+    R3PTRTYPE(const char *) pszDesc;
+} MMLOOKUPHYPER;
+/** Pointer to a hypervisor memory lookup record. */
+typedef MMLOOKUPHYPER *PMMLOOKUPHYPER;
+
+
+/**
+ * Converts a MM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the MM is part of.
+ * @param   pMM   Pointer to MM instance data.
+ */
+#define MM2VM(pMM)  ( (PVM)((uint8_t *)pMM - pMM->offVM) )
+
+
+/**
+ * MM Data (part of VM)
+ */
+typedef struct MM
+{
+    /** Offset to the VM structure.
+     * See MM2VM(). */
+    RTINT                       offVM;
+
+    /** Set if MMR3InitPaging has been called. */
+    bool                        fDoneMMR3InitPaging;
+    /** Set if PGM has been initialized and we can safely call PGMR3Map(). */
+    bool                        fPGMInitialized;
+#if GC_ARCH_BITS == 64 || HC_ARCH_BITS == 64
+    uint32_t                    u32Padding1; /**< alignment padding. */
+#endif
+
+    /** Lookup list for the Hypervisor Memory Area.
+     * The offset is relative to the start of the heap.
+     * Use pHyperHeapR3, pHyperHeapR0 or pHypeRHeapRC to calculate the address.
+     */
+    RTUINT                      offLookupHyper;
+
+    /** The offset of the next static mapping in the Hypervisor Memory Area. */
+    RTUINT                      offHyperNextStatic;
+    /** The size of the HMA.
+     * Starts at 12MB and will be fixed late in the init process. */
+    RTUINT                      cbHyperArea;
+
+    /** Guest address of the Hypervisor Memory Area.
+     * @remarks It's still a bit open whether this should be change to RTRCPTR or
+     *          remain a RTGCPTR. */
+    RTGCPTR                     pvHyperAreaGC;
+
+    /** The hypervisor heap (GC Ptr). */
+    RCPTRTYPE(PMMHYPERHEAP)     pHyperHeapRC;
+#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 64
+    uint32_t                    u32Padding2;
+#endif
+
+    /** The hypervisor heap (R0 Ptr). */
+    R0PTRTYPE(PMMHYPERHEAP)     pHyperHeapR0;
+
+    /** The hypervisor heap (R3 Ptr). */
+    R3PTRTYPE(PMMHYPERHEAP)     pHyperHeapR3;
+
+    /** Pointer to the dummy page.
+     * The dummy page is a paranoia thingy used for instance for pure MMIO RAM ranges
+     * to make sure any bugs will not harm whatever the system stores in the first
+     * physical page. */
+    R3PTRTYPE(void *)           pvDummyPage;
+    /** Physical address of the dummy page. */
+    RTHCPHYS                    HCPhysDummyPage;
+
+    /** Size of the base RAM in bytes. (The CFGM RamSize value.) */
+    uint64_t                    cbRamBase;
+    /** Number of bytes of RAM above 4GB, starting at address 4GB.  */
+    uint64_t                    cbRamAbove4GB;
+    /** Size of the below 4GB RAM hole. */
+    uint32_t                    cbRamHole;
+    /** Number of bytes of RAM below 4GB, starting at address 0.  */
+    uint32_t                    cbRamBelow4GB;
+    /** The number of base RAM pages that PGM has reserved (GMM).
+     * @remarks Shadow ROMs will be counted twice (RAM+ROM), so it won't be 1:1 with
+     *          what the guest sees. */
+    uint64_t                    cBasePages;
+    /** The number of handy pages that PGM has reserved (GMM).
+     * These are kept out of cBasePages and thus out of the saved state. */
+    uint32_t                    cHandyPages;
+    /** The number of shadow pages PGM has reserved (GMM). */
+    uint32_t                    cShadowPages;
+    /** The number of fixed pages we've reserved (GMM). */
+    uint32_t                    cFixedPages;
+    /** Padding. */
+    uint32_t                    u32Padding0;
+} MM;
+/** Pointer to MM Data (part of VM). */
+typedef MM *PMM;
+
+
+/**
+ * MM data kept in the UVM.
+ */
+typedef struct MMUSERPERVM
+{
+    /** Pointer to the MM R3 Heap. */
+    R3PTRTYPE(PMMHEAP)          pHeap;
+    /** Pointer to the MM Uk Heap. */
+    R3PTRTYPE(PMMUKHEAP)        pUkHeap;
+} MMUSERPERVM;
+/** Pointer to the MM data kept in the UVM. */
+typedef MMUSERPERVM *PMMUSERPERVM;
+
+
+RT_C_DECLS_BEGIN
+
+
+int  mmR3UpdateReservation(PVM pVM);
+
+int  mmR3HeapCreateU(PUVM pUVM, PMMHEAP *ppHeap);
+void mmR3HeapDestroy(PMMHEAP pHeap);
+
+void mmR3UkHeapDestroy(PMMUKHEAP pHeap);
+int  mmR3UkHeapCreateU(PUVM pUVM, PMMUKHEAP *ppHeap);
+
+
+int  mmR3HyperInit(PVM pVM);
+int  mmR3HyperTerm(PVM pVM);
+int  mmR3HyperInitPaging(PVM pVM);
+
+const char *mmGetTagName(MMTAG enmTag);
+
+RT_C_DECLS_END
+
+/** @} */
+
+#endif /* !VMM_INCLUDED_SRC_include_MMInternal_h */
+
diff --git a/src/VBox/VMM/include/NEMInternal.h b/src/VBox/VMM/include/NEMInternal.h
new file mode 100644
index 00000000..523a39eb
--- /dev/null
+++ b/src/VBox/VMM/include/NEMInternal.h
@@ -0,0 +1,453 @@
+/* $Id: NEMInternal.h $ */
+/** @file
+ * NEM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2018-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_NEMInternal_h
+#define VMM_INCLUDED_SRC_include_NEMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/cpum.h> /* For CPUMCPUVENDOR. */
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/vmapi.h>
+#ifdef RT_OS_WINDOWS
+#include <iprt/nt/hyperv.h>
+#include <iprt/critsect.h>
+#endif
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_nem_int      Internal
+ * @ingroup grp_nem
+ * @internal
+ * @{
+ */
+
+
+#ifdef RT_OS_WINDOWS
+/*
+ * Windows: Code configuration.
+ */
+# define NEM_WIN_USE_HYPERCALLS_FOR_PAGES
+//# define NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS /**< Applies to ring-3 code only. Useful for testing VID API. */
+//# define NEM_WIN_USE_OUR_OWN_RUN_API          /**< Applies to ring-3 code only. Useful for testing VID API. */
+//# define NEM_WIN_WITH_RING0_RUNLOOP             /**< Enables the ring-0 runloop. */
+//# define NEM_WIN_USE_RING0_RUNLOOP_BY_DEFAULT   /**< For quickly testing ring-3 API without messing with CFGM. */
+# if defined(NEM_WIN_USE_OUR_OWN_RUN_API) && !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
+#  error "NEM_WIN_USE_OUR_OWN_RUN_API requires NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS"
+# endif
+# if defined(NEM_WIN_USE_OUR_OWN_RUN_API) && !defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES)
+#  error "NEM_WIN_USE_OUR_OWN_RUN_API requires NEM_WIN_USE_HYPERCALLS_FOR_PAGES"
+# endif
+# if defined(NEM_WIN_WITH_RING0_RUNLOOP) && !defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES)
+#  error "NEM_WIN_WITH_RING0_RUNLOOP requires NEM_WIN_USE_HYPERCALLS_FOR_PAGES"
+# endif
+
+/**
+ * Windows VID I/O control information.
+ */
+typedef struct NEMWINIOCTL
+{
+    /** The I/O control function number. */
+    uint32_t    uFunction;
+    uint32_t    cbInput;
+    uint32_t    cbOutput;
+} NEMWINIOCTL;
+
+/** @name Windows: Our two-bit physical page state for PGMPAGE
+ * @{ */
+# define NEM_WIN_PAGE_STATE_NOT_SET     0
+# define NEM_WIN_PAGE_STATE_UNMAPPED    1
+# define NEM_WIN_PAGE_STATE_READABLE    2
+# define NEM_WIN_PAGE_STATE_WRITABLE    3
+/** @} */
+
+/** Windows: Checks if a_GCPhys is subject to the limited A20 gate emulation. */
+# define NEM_WIN_IS_SUBJECT_TO_A20(a_GCPhys)    ((RTGCPHYS)((a_GCPhys) - _1M) < (RTGCPHYS)_64K)
+/** Windows: Checks if a_GCPhys is relevant to the limited A20 gate emulation. */
+# define NEM_WIN_IS_RELEVANT_TO_A20(a_GCPhys)    \
+    ( ((RTGCPHYS)((a_GCPhys) - _1M) < (RTGCPHYS)_64K) || ((RTGCPHYS)(a_GCPhys) < (RTGCPHYS)_64K) )
+
+/** The CPUMCTX_EXTRN_XXX mask for IEM. */
+# define NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM      (  IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \
+                                                  | CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI )
+/** The CPUMCTX_EXTRN_XXX mask for IEM when raising exceptions. */
+# define NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT (IEM_CPUMCTX_EXTRN_XCPT_MASK | NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM)
+
+/** @name Windows: Interrupt window flags (NEM_WIN_INTW_F_XXX).
+ * @{  */
+# define NEM_WIN_INTW_F_NMI             UINT8_C(0x01)
+# define NEM_WIN_INTW_F_REGULAR         UINT8_C(0x02)
+# define NEM_WIN_INTW_F_PRIO_MASK       UINT8_C(0x3c)
+# define NEM_WIN_INTW_F_PRIO_SHIFT      2
+/** @} */
+
+#endif /* RT_OS_WINDOWS */
+
+
+/** Trick to make slickedit see the static functions in the template. */
+#ifndef IN_SLICKEDIT
+# define NEM_TMPL_STATIC static
+#else
+# define NEM_TMPL_STATIC
+#endif
+
+
+/**
+ * Generic NEM exit type enumeration for use with EMHistoryAddExit.
+ *
+ * On windows we've got two different set of exit types and they are both jumping
+ * around the place value wise, so EM can use their values.
+ *
+ * @note We only have exit types for exits not covered by EM here.
+ */
+typedef enum NEMEXITTYPE
+{
+    /* windows: */
+    NEMEXITTYPE_UNRECOVERABLE_EXCEPTION = 1,
+    NEMEXITTYPE_INVALID_VP_REGISTER_VALUE,
+    NEMEXITTYPE_INTTERRUPT_WINDOW,
+    NEMEXITTYPE_HALT,
+    NEMEXITTYPE_XCPT_UD,
+    NEMEXITTYPE_XCPT_DB,
+    NEMEXITTYPE_XCPT_BP,
+    NEMEXITTYPE_CANCELED,
+    NEMEXITTYPE_MEMORY_ACCESS
+} NEMEXITTYPE;
+
+
+/**
+ * NEM VM Instance data.
+ */
+typedef struct NEM
+{
+    /** NEM_MAGIC. */
+    uint32_t                    u32Magic;
+
+    /** Set if enabled. */
+    bool                        fEnabled;
+    /** Set if long mode guests are allowed. */
+    bool                        fAllow64BitGuests;
+#ifdef RT_OS_WINDOWS
+    /** Set if we've created the EMTs. */
+    bool                        fCreatedEmts : 1;
+    /** WHvRunVpExitReasonX64Cpuid is supported. */
+    bool                        fExtendedMsrExit : 1;
+    /** WHvRunVpExitReasonX64MsrAccess is supported. */
+    bool                        fExtendedCpuIdExit : 1;
+    /** WHvRunVpExitReasonException is supported. */
+    bool                        fExtendedXcptExit : 1;
+    /** Set if we're using the ring-0 API to do the work. */
+    bool                        fUseRing0Runloop : 1;
+    /** Set if we've started more than one CPU and cannot mess with A20. */
+    bool                        fA20Fixed : 1;
+    /** Set if A20 is enabled. */
+    bool                        fA20Enabled : 1;
+    /** The reported CPU vendor.   */
+    CPUMCPUVENDOR               enmCpuVendor;
+    /** Cache line flush size as a power of two. */
+    uint8_t                     cCacheLineFlushShift;
+    /** The result of WHvCapabilityCodeProcessorFeatures. */
+    union
+    {
+        /** 64-bit view. */
+        uint64_t                u64;
+# ifdef _WINHVAPIDEFS_H_
+        /** Interpreed features. */
+        WHV_PROCESSOR_FEATURES  u;
+# endif
+    } uCpuFeatures;
+
+    /** The partition handle. */
+# ifdef _WINHVAPIDEFS_H_
+    WHV_PARTITION_HANDLE
+# else
+    RTHCUINTPTR
+# endif
+                                hPartition;
+    /** The device handle for the partition, for use with Vid APIs or direct I/O
+     * controls. */
+    RTR3PTR                     hPartitionDevice;
+    /** The Hyper-V partition ID.   */
+    uint64_t                    idHvPartition;
+
+    /** Number of currently mapped pages. */
+    uint32_t volatile           cMappedPages;
+
+    /** Info about the VidGetHvPartitionId I/O control interface. */
+    NEMWINIOCTL                 IoCtlGetHvPartitionId;
+    /** Info about the VidStartVirtualProcessor I/O control interface. */
+    NEMWINIOCTL                 IoCtlStartVirtualProcessor;
+    /** Info about the VidStopVirtualProcessor I/O control interface. */
+    NEMWINIOCTL                 IoCtlStopVirtualProcessor;
+    /** Info about the VidStopVirtualProcessor I/O control interface. */
+    NEMWINIOCTL                 IoCtlMessageSlotHandleAndGetNext;
+
+    /** Statistics updated by NEMR0UpdateStatistics. */
+    struct
+    {
+        uint64_t                cPagesAvailable;
+        uint64_t                cPagesInUse;
+    } R0Stats;
+#endif /* RT_OS_WINDOWS */
+} NEM;
+/** Pointer to NEM VM instance data. */
+typedef NEM *PNEM;
+
+/** NEM::u32Magic value. */
+#define NEM_MAGIC               UINT32_C(0x004d454e)
+/** NEM::u32Magic value after termination. */
+#define NEM_MAGIC_DEAD          UINT32_C(0xdead1111)
+
+
+/**
+ * NEM VMCPU Instance data.
+ */
+typedef struct NEMCPU
+{
+    /** NEMCPU_MAGIC. */
+    uint32_t                    u32Magic;
+    /** Whether \#UD needs to be intercepted and presented to GIM. */
+    bool                        fGIMTrapXcptUD : 1;
+    /** Whether \#GP needs to be intercept for mesa driver workaround. */
+    bool                        fTrapXcptGpForLovelyMesaDrv: 1;
+#ifdef RT_OS_WINDOWS
+    /** The current state of the interrupt windows (NEM_WIN_INTW_F_XXX). */
+    uint8_t                     fCurrentInterruptWindows;
+    /** The desired state of the interrupt windows (NEM_WIN_INTW_F_XXX). */
+    uint8_t                     fDesiredInterruptWindows;
+    /** Last copy of HV_X64_VP_EXECUTION_STATE::InterruptShadow. */
+    bool                        fLastInterruptShadow : 1;
+# ifdef NEM_WIN_WITH_RING0_RUNLOOP
+    /** Pending VINF_NEM_FLUSH_TLB. */
+    int32_t                     rcPending;
+# else
+    uint32_t                    uPadding;
+# endif
+    /** The VID_MSHAGN_F_XXX flags.
+     * Either VID_MSHAGN_F_HANDLE_MESSAGE | VID_MSHAGN_F_GET_NEXT_MESSAGE or zero. */
+    uint32_t                    fHandleAndGetFlags;
+    /** What VidMessageSlotMap returns and is used for passing exit info. */
+    RTR3PTR                     pvMsgSlotMapping;
+    /** The windows thread handle. */
+    RTR3PTR                     hNativeThreadHandle;
+    /** Parameters for making Hyper-V hypercalls. */
+    union
+    {
+        uint8_t                 ab[64];
+        /** Arguments for NEMR0MapPages (HvCallMapGpaPages). */
+        struct
+        {
+            RTGCPHYS            GCPhysSrc;
+            RTGCPHYS            GCPhysDst; /**< Same as GCPhysSrc except maybe when the A20 gate is disabled. */
+            uint32_t            cPages;
+            HV_MAP_GPA_FLAGS    fFlags;
+        }                       MapPages;
+        /** Arguments for NEMR0UnmapPages (HvCallUnmapGpaPages). */
+        struct
+        {
+            RTGCPHYS            GCPhys;
+            uint32_t            cPages;
+        }                       UnmapPages;
+        /** Result from NEMR0QueryCpuTick. */
+        struct
+        {
+            uint64_t            cTicks;
+            uint32_t            uAux;
+        }                       QueryCpuTick;
+        /** Input and output for NEMR0DoExperiment. */
+        struct
+        {
+            uint32_t            uItem;
+            bool                fSuccess;
+            uint64_t            uStatus;
+            uint64_t            uLoValue;
+            uint64_t            uHiValue;
+        }                       Experiment;
+    } Hypercall;
+    /** I/O control buffer, we always use this for I/O controls. */
+    union
+    {
+        uint8_t                 ab[64];
+        HV_PARTITION_ID         idPartition;
+        HV_VP_INDEX             idCpu;
+# ifdef VID_MSHAGN_F_GET_NEXT_MESSAGE
+        VID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT MsgSlotHandleAndGetNext;
+# endif
+    } uIoCtlBuf;
+
+    /** @name Statistics
+     * @{ */
+    STAMCOUNTER                 StatExitPortIo;
+    STAMCOUNTER                 StatExitMemUnmapped;
+    STAMCOUNTER                 StatExitMemIntercept;
+    STAMCOUNTER                 StatExitHalt;
+    STAMCOUNTER                 StatExitInterruptWindow;
+    STAMCOUNTER                 StatExitCpuId;
+    STAMCOUNTER                 StatExitMsr;
+    STAMCOUNTER                 StatExitException;
+    STAMCOUNTER                 StatExitExceptionBp;
+    STAMCOUNTER                 StatExitExceptionDb;
+    STAMCOUNTER                 StatExitExceptionGp;
+    STAMCOUNTER                 StatExitExceptionGpMesa;
+    STAMCOUNTER                 StatExitExceptionUd;
+    STAMCOUNTER                 StatExitExceptionUdHandled;
+    STAMCOUNTER                 StatExitUnrecoverable;
+    STAMCOUNTER                 StatGetMsgTimeout;
+    STAMCOUNTER                 StatStopCpuSuccess;
+    STAMCOUNTER                 StatStopCpuPending;
+    STAMCOUNTER                 StatStopCpuPendingAlerts;
+    STAMCOUNTER                 StatStopCpuPendingOdd;
+    STAMCOUNTER                 StatCancelChangedState;
+    STAMCOUNTER                 StatCancelAlertedThread;
+    STAMCOUNTER                 StatBreakOnCancel;
+    STAMCOUNTER                 StatBreakOnFFPre;
+    STAMCOUNTER                 StatBreakOnFFPost;
+    STAMCOUNTER                 StatBreakOnStatus;
+    STAMCOUNTER                 StatImportOnDemand;
+    STAMCOUNTER                 StatImportOnReturn;
+    STAMCOUNTER                 StatImportOnReturnSkipped;
+    STAMCOUNTER                 StatQueryCpuTick;
+    /** @} */
+#endif /* RT_OS_WINDOWS */
+} NEMCPU;
+/** Pointer to NEM VMCPU instance data. */
+typedef NEMCPU *PNEMCPU;
+
+/** NEMCPU::u32Magic value. */
+#define NEMCPU_MAGIC            UINT32_C(0x4d454e20)
+/** NEMCPU::u32Magic value after termination. */
+#define NEMCPU_MAGIC_DEAD       UINT32_C(0xdead2222)
+
+
+#ifdef IN_RING0
+# ifdef RT_OS_WINDOWS
+/**
+ * Windows: Hypercall input/ouput page info.
+ */
+typedef struct NEMR0HYPERCALLDATA
+{
+    /** Host physical address of the hypercall input/output page. */
+    RTHCPHYS                    HCPhysPage;
+    /** Pointer to the hypercall input/output page. */
+    uint8_t                    *pbPage;
+    /** Handle to the memory object of the hypercall input/output page. */
+    RTR0MEMOBJ                  hMemObj;
+} NEMR0HYPERCALLDATA;
+/** Pointer to a Windows hypercall input/output page info. */
+typedef NEMR0HYPERCALLDATA *PNEMR0HYPERCALLDATA;
+# endif /* RT_OS_WINDOWS */
+
+/**
+ * NEM GVMCPU instance data.
+ */
+typedef struct NEMR0PERVCPU
+{
+# ifdef RT_OS_WINDOWS
+    /** Hypercall input/ouput page. */
+    NEMR0HYPERCALLDATA          HypercallData;
+    /** Delta to add to convert a ring-0 pointer to a ring-3 one.   */
+    uintptr_t                   offRing3ConversionDelta;
+# else
+    uint32_t                    uDummy;
+# endif
+} NEMR0PERVCPU;
+
+/**
+ * NEM GVM instance data.
+ */
+typedef struct NEMR0PERVM
+{
+# ifdef RT_OS_WINDOWS
+    /** The partition ID. */
+    uint64_t                    idHvPartition;
+    /** I/O control context. */
+    PSUPR0IOCTLCTX              pIoCtlCtx;
+    /** Info about the VidGetHvPartitionId I/O control interface. */
+    NEMWINIOCTL                 IoCtlGetHvPartitionId;
+    /** Info about the VidStartVirtualProcessor I/O control interface. */
+    NEMWINIOCTL                 IoCtlStartVirtualProcessor;
+    /** Info about the VidStopVirtualProcessor I/O control interface. */
+    NEMWINIOCTL                 IoCtlStopVirtualProcessor;
+    /** Info about the VidStopVirtualProcessor I/O control interface. */
+    NEMWINIOCTL                 IoCtlMessageSlotHandleAndGetNext;
+    /** Whether we may use the ring-0 runloop or not. */
+    bool                        fMayUseRing0Runloop;
+
+    /** Hypercall input/ouput page for non-EMT. */
+    NEMR0HYPERCALLDATA          HypercallData;
+    /** Critical section protecting use of HypercallData. */
+    RTCRITSECT                  HypercallDataCritSect;
+
+# else
+    uint32_t                    uDummy;
+# endif
+} NEMR0PERVM;
+
+#endif /* IN_RING*/
+
+
+#ifdef IN_RING3
+int     nemR3NativeInit(PVM pVM, bool fFallback, bool fForced);
+int     nemR3NativeInitAfterCPUM(PVM pVM);
+int     nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat);
+int     nemR3NativeTerm(PVM pVM);
+void    nemR3NativeReset(PVM pVM);
+void    nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi);
+VBOXSTRICTRC    nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu);
+bool            nemR3NativeCanExecuteGuest(PVM pVM, PVMCPU pVCpu);
+bool            nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable);
+void            nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags);
+
+int     nemR3NativeNotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb);
+int     nemR3NativeNotifyPhysMmioExMap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvMmio2);
+int     nemR3NativeNotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags);
+int     nemR3NativeNotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags);
+int     nemR3NativeNotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags);
+void    nemR3NativeNotifySetA20(PVMCPU pVCpu, bool fEnabled);
+#endif
+
+void    nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb);
+void    nemHCNativeNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
+                                                   int fRestoreAsRAM, bool fRestoreAsRAM2);
+void    nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
+                                               RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM);
+int     nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
+                                           PGMPAGETYPE enmType, uint8_t *pu2State);
+void    nemHCNativeNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
+                                             PGMPAGETYPE enmType, uint8_t *pu2State);
+void    nemHCNativeNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew, uint32_t fPageProt,
+                                         PGMPAGETYPE enmType, uint8_t *pu2State);
+
+
+#ifdef RT_OS_WINDOWS
+/** Maximum number of pages we can map in a single NEMR0MapPages call. */
+# define NEM_MAX_MAP_PAGES      ((PAGE_SIZE - RT_UOFFSETOF(HV_INPUT_MAP_GPA_PAGES, PageList)) / sizeof(HV_SPA_PAGE_NUMBER))
+/** Maximum number of pages we can unmap in a single NEMR0UnmapPages call. */
+# define NEM_MAX_UNMAP_PAGES    4095
+
+#endif
+/** @} */
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_NEMInternal_h */
+
diff --git a/src/VBox/VMM/include/PDMAsyncCompletionFileInternal.h b/src/VBox/VMM/include/PDMAsyncCompletionFileInternal.h
new file mode 100644
index 00000000..bf3d2958
--- /dev/null
+++ b/src/VBox/VMM/include/PDMAsyncCompletionFileInternal.h
@@ -0,0 +1,566 @@
+/* $Id: PDMAsyncCompletionFileInternal.h $ */
+/** @file
+ * PDM Async I/O - Transport data asynchronous in R3 using EMT.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_PDMAsyncCompletionFileInternal_h
+#define VMM_INCLUDED_SRC_include_PDMAsyncCompletionFileInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/tm.h>
+#include <iprt/types.h>
+#include <iprt/file.h>
+#include <iprt/thread.h>
+#include <iprt/semaphore.h>
+#include <iprt/critsect.h>
+#include <iprt/avl.h>
+#include <iprt/list.h>
+#include <iprt/spinlock.h>
+#include <iprt/memcache.h>
+
+#include "PDMAsyncCompletionInternal.h"
+
+/** @todo: Revise the caching of tasks. We have currently four caches:
+ *  Per endpoint task cache
+ *  Per class cache
+ *  Per endpoint task segment cache
+ *  Per class task segment cache
+ *
+ *  We could use the RT heap for this probably or extend MMR3Heap (uses RTMemAlloc
+ *  instead of managing larger blocks) to have this global for the whole VM.
+ */
+
+/** Enable for delay injection from the debugger. */
+#if 0
+# define PDM_ASYNC_COMPLETION_FILE_WITH_DELAY
+#endif
+
+RT_C_DECLS_BEGIN
+
+/**
+ * A few forward declarations.
+ */
+typedef struct PDMASYNCCOMPLETIONENDPOINTFILE *PPDMASYNCCOMPLETIONENDPOINTFILE;
+/** Pointer to a request segment. */
+typedef struct PDMACTASKFILE *PPDMACTASKFILE;
+/** Pointer to the endpoint class data. */
+typedef struct PDMASYNCCOMPLETIONTASKFILE *PPDMASYNCCOMPLETIONTASKFILE;
+/** Pointer to a cache LRU list. */
+typedef struct PDMACFILELRULIST *PPDMACFILELRULIST;
+/** Pointer to the global cache structure. */
+typedef struct PDMACFILECACHEGLOBAL *PPDMACFILECACHEGLOBAL;
+/** Pointer to a task segment. */
+typedef struct PDMACFILETASKSEG *PPDMACFILETASKSEG;
+
+/**
+ * Blocking event types.
+ */
+typedef enum PDMACEPFILEAIOMGRBLOCKINGEVENT
+{
+    /** Invalid tye */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID = 0,
+    /** An endpoint is added to the manager. */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT,
+    /** An endpoint is removed from the manager. */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT,
+    /** An endpoint is about to be closed. */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT,
+    /** The manager is requested to terminate */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN,
+    /** The manager is requested to suspend */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND,
+    /** The manager is requested to resume */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME,
+    /** 32bit hack */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_32BIT_HACK = 0x7fffffff
+} PDMACEPFILEAIOMGRBLOCKINGEVENT;
+
+/**
+ * I/O manager type.
+ */
+typedef enum PDMACEPFILEMGRTYPE
+{
+    /** Simple aka failsafe */
+    PDMACEPFILEMGRTYPE_SIMPLE = 0,
+    /** Async I/O with host cache enabled. */
+    PDMACEPFILEMGRTYPE_ASYNC,
+    /** 32bit hack */
+    PDMACEPFILEMGRTYPE_32BIT_HACK = 0x7fffffff
+} PDMACEPFILEMGRTYPE;
+/** Pointer to a I/O manager type */
+typedef PDMACEPFILEMGRTYPE *PPDMACEPFILEMGRTYPE;
+
+/**
+ * States of the I/O manager.
+ */
+typedef enum PDMACEPFILEMGRSTATE
+{
+    /** Invalid state. */
+    PDMACEPFILEMGRSTATE_INVALID = 0,
+    /** Normal running state accepting new requests
+     * and processing them.
+     */
+    PDMACEPFILEMGRSTATE_RUNNING,
+    /** Fault state - not accepting new tasks for endpoints but waiting for
+     * remaining ones to finish.
+     */
+    PDMACEPFILEMGRSTATE_FAULT,
+    /** Suspending state - not accepting new tasks for endpoints but waiting
+     * for remaining ones to finish.
+     */
+    PDMACEPFILEMGRSTATE_SUSPENDING,
+    /** Shutdown state - not accepting new tasks for endpoints but waiting
+     * for remaining ones to finish.
+     */
+    PDMACEPFILEMGRSTATE_SHUTDOWN,
+    /** The I/O manager waits for all active requests to complete and doesn't queue
+     * new ones because it needs to grow to handle more requests.
+     */
+    PDMACEPFILEMGRSTATE_GROWING,
+    /** 32bit hack */
+    PDMACEPFILEMGRSTATE_32BIT_HACK = 0x7fffffff
+} PDMACEPFILEMGRSTATE;
+
+/**
+ * State of a async I/O manager.
+ */
+typedef struct PDMACEPFILEMGR
+{
+    /** Next Aio manager in the list. */
+    R3PTRTYPE(struct PDMACEPFILEMGR *)     pNext;
+    /** Previous Aio manager in the list. */
+    R3PTRTYPE(struct PDMACEPFILEMGR *)     pPrev;
+    /** Manager type */
+    PDMACEPFILEMGRTYPE                     enmMgrType;
+    /** Current state of the manager. */
+    PDMACEPFILEMGRSTATE                    enmState;
+    /** Event semaphore the manager sleeps on when waiting for new requests. */
+    RTSEMEVENT                             EventSem;
+    /** Flag whether the thread waits in the event semaphore. */
+    volatile bool                          fWaitingEventSem;
+    /** Thread data */
+    RTTHREAD                               Thread;
+    /** The async I/O context for this manager. */
+    RTFILEAIOCTX                           hAioCtx;
+    /** Flag whether the I/O manager was woken up. */
+    volatile bool                          fWokenUp;
+    /** List of endpoints assigned to this manager. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINTFILE) pEndpointsHead;
+    /** Number of endpoints assigned to the manager. */
+    unsigned                               cEndpoints;
+    /** Number of requests active currently. */
+    unsigned                               cRequestsActive;
+    /** Number of maximum requests active. */
+    uint32_t                               cRequestsActiveMax;
+    /** Pointer to an array of free async I/O request handles. */
+    RTFILEAIOREQ                          *pahReqsFree;
+    /** Index of the next free entry in the cache. */
+    uint32_t                               iFreeEntry;
+    /** Size of the array. */
+    unsigned                               cReqEntries;
+    /** Memory cache for file range locks. */
+    RTMEMCACHE                             hMemCacheRangeLocks;
+    /** Number of milliseconds to wait until the bandwidth is refreshed for at least
+     * one endpoint and it is possible to process more requests. */
+    RTMSINTERVAL                           msBwLimitExpired;
+    /** Critical section protecting the blocking event handling. */
+    RTCRITSECT                             CritSectBlockingEvent;
+    /** Event semaphore for blocking external events.
+     * The caller waits on it until the async I/O manager
+     * finished processing the event. */
+    RTSEMEVENT                             EventSemBlock;
+    /** Flag whether a blocking event is pending and needs
+     * processing by the I/O manager. */
+    volatile bool                          fBlockingEventPending;
+    /** Blocking event type */
+    volatile PDMACEPFILEAIOMGRBLOCKINGEVENT enmBlockingEvent;
+    /** Event type data */
+    union
+    {
+        /** Add endpoint event. */
+        struct
+        {
+            /** The endpoint to be added */
+            volatile PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
+        } AddEndpoint;
+        /** Remove endpoint event. */
+        struct
+        {
+            /** The endpoint to be removed */
+            volatile PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
+        } RemoveEndpoint;
+        /** Close endpoint event. */
+        struct
+        {
+            /** The endpoint to be closed */
+            volatile PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
+        } CloseEndpoint;
+    } BlockingEventData;
+} PDMACEPFILEMGR;
+/** Pointer to a async I/O manager state. */
+typedef PDMACEPFILEMGR *PPDMACEPFILEMGR;
+/** Pointer to a async I/O manager state pointer. */
+typedef PPDMACEPFILEMGR *PPPDMACEPFILEMGR;
+
+/**
+ * A file access range lock.
+ */
+typedef struct PDMACFILERANGELOCK
+{
+    /** AVL node in the locked range tree of the endpoint. */
+    AVLRFOFFNODECORE            Core;
+    /** How many tasks have locked this range. */
+    uint32_t                    cRefs;
+    /** Flag whether this is a read or write lock. */
+    bool                        fReadLock;
+    /** List of tasks which are waiting that the range gets unlocked. */
+    PPDMACTASKFILE              pWaitingTasksHead;
+    /** List of tasks which are waiting that the range gets unlocked. */
+    PPDMACTASKFILE              pWaitingTasksTail;
+} PDMACFILERANGELOCK, *PPDMACFILERANGELOCK;
+
+/**
+ * Backend type for the endpoint.
+ */
+typedef enum PDMACFILEEPBACKEND
+{
+    /** Non buffered. */
+    PDMACFILEEPBACKEND_NON_BUFFERED = 0,
+    /** Buffered (i.e host cache enabled) */
+    PDMACFILEEPBACKEND_BUFFERED,
+    /** 32bit hack */
+    PDMACFILEEPBACKEND_32BIT_HACK = 0x7fffffff
+} PDMACFILEEPBACKEND;
+/** Pointer to a backend type. */
+typedef PDMACFILEEPBACKEND *PPDMACFILEEPBACKEND;
+
+/**
+ * Global data for the file endpoint class.
+ */
+typedef struct PDMASYNCCOMPLETIONEPCLASSFILE
+{
+    /** Common data. */
+    PDMASYNCCOMPLETIONEPCLASS           Core;
+    /** Override I/O manager type - set to SIMPLE after failure. */
+    PDMACEPFILEMGRTYPE                  enmMgrTypeOverride;
+    /** Default backend type for the endpoint. */
+    PDMACFILEEPBACKEND                  enmEpBackendDefault;
+    RTCRITSECT                          CritSect;
+    /** Pointer to the head of the async I/O managers. */
+    R3PTRTYPE(PPDMACEPFILEMGR)          pAioMgrHead;
+    /** Number of async I/O managers currently running. */
+    unsigned                            cAioMgrs;
+    /** Maximum number of segments to cache per endpoint */
+    unsigned                            cTasksCacheMax;
+    /** Maximum number of simultaneous outstandingrequests. */
+    uint32_t                            cReqsOutstandingMax;
+    /** Bitmask for checking the alignment of a buffer. */
+    RTR3UINTPTR                         uBitmaskAlignment;
+    /** Flag whether the out of resources warning was printed already. */
+    bool                                fOutOfResourcesWarningPrinted;
+#ifdef PDM_ASYNC_COMPLETION_FILE_WITH_DELAY
+    /** Timer for delayed request completion. */
+    PTMTIMERR3                          pTimer;
+    /** Milliseconds until the next delay expires. */
+    volatile uint64_t                   cMilliesNext;
+#endif
+} PDMASYNCCOMPLETIONEPCLASSFILE;
+/** Pointer to the endpoint class data. */
+typedef PDMASYNCCOMPLETIONEPCLASSFILE *PPDMASYNCCOMPLETIONEPCLASSFILE;
+
+typedef enum PDMACEPFILEBLOCKINGEVENT
+{
+    /** The invalid event type */
+    PDMACEPFILEBLOCKINGEVENT_INVALID = 0,
+    /** A task is about to be canceled */
+    PDMACEPFILEBLOCKINGEVENT_CANCEL,
+    /** Usual 32bit hack */
+    PDMACEPFILEBLOCKINGEVENT_32BIT_HACK = 0x7fffffff
+} PDMACEPFILEBLOCKINGEVENT;
+
+/**
+ * States of the endpoint.
+ */
+typedef enum PDMASYNCCOMPLETIONENDPOINTFILESTATE
+{
+    /** Invalid state. */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE_INVALID = 0,
+    /** Normal running state accepting new requests
+     * and processing them.
+     */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
+    /** The endpoint is about to be closed - not accepting new tasks for endpoints but waiting for
+     *  remaining ones to finish.
+     */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING,
+    /** Removing from current I/O manager state - not processing new tasks for endpoints but waiting
+     * for remaining ones to finish.
+     */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING,
+    /** The current endpoint will be migrated to another I/O manager. */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE_MIGRATING,
+    /** 32bit hack */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE_32BIT_HACK = 0x7fffffff
+} PDMASYNCCOMPLETIONENDPOINTFILESTATE;
+
+typedef enum PDMACFILEREQTYPEDELAY
+{
+    PDMACFILEREQTYPEDELAY_ANY = 0,
+    PDMACFILEREQTYPEDELAY_READ,
+    PDMACFILEREQTYPEDELAY_WRITE,
+    PDMACFILEREQTYPEDELAY_FLUSH,
+    PDMACFILEREQTYPEDELAY_32BIT_HACK = 0x7fffffff
+} PDMACFILEREQTYPEDELAY;
+
+/**
+ * Data for the file endpoint.
+ */
+typedef struct PDMASYNCCOMPLETIONENDPOINTFILE
+{
+    /** Common data. */
+    PDMASYNCCOMPLETIONENDPOINT             Core;
+    /** Current state of the endpoint. */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE    enmState;
+    /** The backend to use for this endpoint. */
+    PDMACFILEEPBACKEND                     enmBackendType;
+    /** async I/O manager this endpoint is assigned to. */
+    R3PTRTYPE(volatile PPDMACEPFILEMGR)    pAioMgr;
+    /** Flags for opening the file. */
+    unsigned                               fFlags;
+    /** File handle. */
+    RTFILE                                 hFile;
+    /** Real size of the file. Only updated if data is appended. */
+    volatile uint64_t                      cbFile;
+    /** List of new tasks. */
+    R3PTRTYPE(volatile PPDMACTASKFILE)     pTasksNewHead;
+
+    /** Head of the small cache for allocated task segments for exclusive
+     * use by this endpoint. */
+    R3PTRTYPE(volatile PPDMACTASKFILE)     pTasksFreeHead;
+    /** Tail of the small cache for allocated task segments for exclusive
+     * use by this endpoint. */
+    R3PTRTYPE(volatile PPDMACTASKFILE)     pTasksFreeTail;
+    /** Number of elements in the cache. */
+    volatile uint32_t                      cTasksCached;
+
+    /** Flag whether a flush request is currently active */
+    PPDMACTASKFILE                         pFlushReq;
+
+#ifdef VBOX_WITH_STATISTICS
+    /** Time spend in a read. */
+    STAMPROFILEADV                         StatRead;
+    /** Time spend in a write. */
+    STAMPROFILEADV                         StatWrite;
+#endif
+
+    /** Event semaphore for blocking external events.
+     * The caller waits on it until the async I/O manager
+     * finished processing the event. */
+    RTSEMEVENT                             EventSemBlock;
+    /** Flag whether caching is enabled for this file. */
+    bool                                   fCaching;
+    /** Flag whether the file was opened readonly. */
+    bool                                   fReadonly;
+    /** Flag whether the host supports the async flush API. */
+    bool                                   fAsyncFlushSupported;
+#ifdef VBOX_WITH_DEBUGGER
+    /** Status code to inject for the next complete read. */
+    volatile int                           rcReqRead;
+    /** Status code to inject for the next complete write. */
+    volatile int                           rcReqWrite;
+#endif
+#ifdef PDM_ASYNC_COMPLETION_FILE_WITH_DELAY
+    /** Request delay. */
+    volatile uint32_t                      msDelay;
+    /** Request delay jitter. */
+    volatile uint32_t                      msJitter;
+    /** Number of requests to delay. */
+    volatile uint32_t                      cReqsDelay;
+    /** Task type to delay. */
+    PDMACFILEREQTYPEDELAY                  enmTypeDelay;
+    /** The current task which gets delayed. */
+    PPDMASYNCCOMPLETIONTASKFILE            pDelayedHead;
+#endif
+    /** Flag whether a blocking event is pending and needs
+     * processing by the I/O manager. */
+    bool                                   fBlockingEventPending;
+    /** Blocking event type */
+    PDMACEPFILEBLOCKINGEVENT               enmBlockingEvent;
+
+    /** Additional data needed for the event types. */
+    union
+    {
+        /** Cancelation event. */
+        struct
+        {
+            /** The task to cancel. */
+            PPDMACTASKFILE                 pTask;
+        } Cancel;
+    } BlockingEventData;
+    /** Data for exclusive use by the assigned async I/O manager. */
+    struct
+    {
+        /** Pointer to the next endpoint assigned to the manager. */
+        R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINTFILE) pEndpointNext;
+        /** Pointer to the previous endpoint assigned to the manager. */
+        R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINTFILE) pEndpointPrev;
+        /** List of pending requests (not submitted due to usage restrictions
+         *  or a pending flush request) */
+        R3PTRTYPE(PPDMACTASKFILE)                  pReqsPendingHead;
+        /** Tail of pending requests. */
+        R3PTRTYPE(PPDMACTASKFILE)                  pReqsPendingTail;
+        /** Tree of currently locked ranges.
+         * If a write task is enqueued the range gets locked and any other
+         * task writing to that range has to wait until the task completes.
+         */
+        PAVLRFOFFTREE                              pTreeRangesLocked;
+        /** Number of requests with a range lock active. */
+        unsigned                                   cLockedReqsActive;
+        /** Number of requests currently being processed for this endpoint
+         * (excluded flush requests). */
+        unsigned                                   cRequestsActive;
+        /** Number of requests processed during the last second. */
+        unsigned                                   cReqsPerSec;
+        /** Current number of processed requests for the current update period. */
+        unsigned                                   cReqsProcessed;
+        /** Flag whether the endpoint is about to be moved to another manager. */
+        bool                                       fMoving;
+        /** Destination I/O manager. */
+        PPDMACEPFILEMGR                            pAioMgrDst;
+    } AioMgr;
+} PDMASYNCCOMPLETIONENDPOINTFILE;
+/** Pointer to the endpoint class data. */
+typedef PDMASYNCCOMPLETIONENDPOINTFILE *PPDMASYNCCOMPLETIONENDPOINTFILE;
+#ifdef VBOX_WITH_STATISTICS
+AssertCompileMemberAlignment(PDMASYNCCOMPLETIONENDPOINTFILE, StatRead, sizeof(uint64_t));
+#endif
+
+/** Request completion function */
+typedef DECLCALLBACK(void)   FNPDMACTASKCOMPLETED(PPDMACTASKFILE pTask, void *pvUser, int rc);
+/** Pointer to a request completion function. */
+typedef FNPDMACTASKCOMPLETED *PFNPDMACTASKCOMPLETED;
+
+/**
+ * Transfer type.
+ */
+typedef enum PDMACTASKFILETRANSFER
+{
+    /** Invalid. */
+    PDMACTASKFILETRANSFER_INVALID = 0,
+    /** Read transfer. */
+    PDMACTASKFILETRANSFER_READ,
+    /** Write transfer. */
+    PDMACTASKFILETRANSFER_WRITE,
+    /** Flush transfer. */
+    PDMACTASKFILETRANSFER_FLUSH
+} PDMACTASKFILETRANSFER;
+
+/**
+ * Data of a request.
+ */
+typedef struct PDMACTASKFILE
+{
+    /** Pointer to the range lock we are waiting for */
+    PPDMACFILERANGELOCK                  pRangeLock;
+    /** Next task in the list. (Depending on the state) */
+    struct PDMACTASKFILE                *pNext;
+    /** Endpoint */
+    PPDMASYNCCOMPLETIONENDPOINTFILE      pEndpoint;
+    /** Transfer type. */
+    PDMACTASKFILETRANSFER                enmTransferType;
+    /** Start offset */
+    RTFOFF                               Off;
+    /** Amount of data transfered so far. */
+    size_t                               cbTransfered;
+    /** Data segment. */
+    RTSGSEG                              DataSeg;
+    /** When non-zero the segment uses a bounce buffer because the provided buffer
+     * doesn't meet host requirements. */
+    size_t                               cbBounceBuffer;
+    /** Pointer to the used bounce buffer if any. */
+    void                                *pvBounceBuffer;
+    /** Start offset in the bounce buffer to copy from. */
+    uint32_t                             offBounceBuffer;
+    /** Flag whether this is a prefetch request. */
+    bool                                 fPrefetch;
+    /** Already prepared native I/O request.
+     * Used if the request is prepared already but
+     * was not queued because the host has not enough
+     * resources. */
+    RTFILEAIOREQ                         hReq;
+    /** Completion function to call on completion. */
+    PFNPDMACTASKCOMPLETED                pfnCompleted;
+    /** User data */
+    void                                *pvUser;
+} PDMACTASKFILE;
+
+/**
+ * Per task data.
+ */
+typedef struct PDMASYNCCOMPLETIONTASKFILE
+{
+    /** Common data. */
+    PDMASYNCCOMPLETIONTASK Core;
+    /** Number of bytes to transfer until this task completes. */
+    volatile int32_t      cbTransferLeft;
+    /** Flag whether the task completed. */
+    volatile bool         fCompleted;
+    /** Return code. */
+    volatile int          rc;
+#ifdef PDM_ASYNC_COMPLETION_FILE_WITH_DELAY
+    volatile PPDMASYNCCOMPLETIONTASKFILE pDelayedNext;
+    /** Timestamp when the delay expires. */
+    uint64_t                             tsDelayEnd;
+#endif
+} PDMASYNCCOMPLETIONTASKFILE;
+
+DECLCALLBACK(int) pdmacFileAioMgrFailsafe(RTTHREAD hThreadSelf, void *pvUser);
+DECLCALLBACK(int) pdmacFileAioMgrNormal(RTTHREAD hThreadSelf, void *pvUser);
+
+int pdmacFileAioMgrNormalInit(PPDMACEPFILEMGR pAioMgr);
+void pdmacFileAioMgrNormalDestroy(PPDMACEPFILEMGR pAioMgr);
+
+int pdmacFileAioMgrCreate(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass, PPPDMACEPFILEMGR ppAioMgr, PDMACEPFILEMGRTYPE enmMgrType);
+
+int pdmacFileAioMgrAddEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+
+PPDMACTASKFILE pdmacFileEpGetNewTasks(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+PPDMACTASKFILE pdmacFileTaskAlloc(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+void pdmacFileTaskFree(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                       PPDMACTASKFILE pTask);
+
+int pdmacFileEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask);
+
+int pdmacFileCacheInit(PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile, PCFGMNODE pCfgNode);
+void pdmacFileCacheDestroy(PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile);
+int pdmacFileEpCacheInit(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile);
+void pdmacFileEpCacheDestroy(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+
+int pdmacFileEpCacheRead(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONTASKFILE pTask,
+                         RTFOFF off, PCRTSGSEG paSegments, size_t cSegments,
+                         size_t cbRead);
+int pdmacFileEpCacheWrite(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONTASKFILE pTask,
+                          RTFOFF off, PCRTSGSEG paSegments, size_t cSegments,
+                          size_t cbWrite);
+int pdmacFileEpCacheFlush(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_PDMAsyncCompletionFileInternal_h */
+
diff --git a/src/VBox/VMM/include/PDMAsyncCompletionInternal.h b/src/VBox/VMM/include/PDMAsyncCompletionInternal.h
new file mode 100644
index 00000000..2e2669bf
--- /dev/null
+++ b/src/VBox/VMM/include/PDMAsyncCompletionInternal.h
@@ -0,0 +1,281 @@
+/* $Id: PDMAsyncCompletionInternal.h $ */
+/** @file
+ * PDM - Pluggable Device Manager, Async I/O Completion internal header.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_PDMAsyncCompletionInternal_h
+#define VMM_INCLUDED_SRC_include_PDMAsyncCompletionInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <iprt/critsect.h>
+#include <iprt/memcache.h>
+#include <iprt/sg.h>
+#include <VBox/types.h>
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/pdmasynccompletion.h>
+#include "PDMInternal.h"
+
+RT_C_DECLS_BEGIN
+
+
+/**
+ * PDM Async completion endpoint operations.
+ */
+typedef struct PDMASYNCCOMPLETIONEPCLASSOPS
+{
+    /** Version identifier. */
+    uint32_t                      u32Version;
+    /** Name of the endpoint class. */
+    const char                   *pszName;
+    /** Class type. */
+    PDMASYNCCOMPLETIONEPCLASSTYPE enmClassType;
+    /** Size of the global endpoint class data in bytes. */
+    size_t                        cbEndpointClassGlobal;
+    /** Size of an endpoint in bytes. */
+    size_t                        cbEndpoint;
+    /** size of a task in bytes. */
+    size_t                        cbTask;
+
+    /**
+     * Initializes the global data for a endpoint class.
+     *
+     * @returns VBox status code.
+     * @param   pClassGlobals    Pointer to the uninitialized globals data.
+     * @param   pCfgNode         Node for querying configuration data.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnInitialize, (PPDMASYNCCOMPLETIONEPCLASS pClassGlobals, PCFGMNODE pCfgNode));
+
+    /**
+     * Frees all allocated resources which were allocated during init.
+     *
+     * @returns VBox status code.
+     * @param   pClassGlobals    Pointer to the globals data.
+     */
+    DECLR3CALLBACKMEMBER(void, pfnTerminate, (PPDMASYNCCOMPLETIONEPCLASS pClassGlobals));
+
+    /**
+     * Initializes a given endpoint.
+     *
+     * @returns VBox status code.
+     * @param   pEndpoint     Pointer to the uninitialized endpoint.
+     * @param   pszUri        Pointer to the string containing the endpoint
+     *                        destination (filename, IP address, ...)
+     * @param   fFlags        Creation flags.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpInitialize, (PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                                const char *pszUri, uint32_t fFlags));
+
+    /**
+     * Closes a endpoint finishing all tasks.
+     *
+     * @returns VBox status code.
+     * @param   pEndpoint     Pointer to the endpoint to be closed.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpClose, (PPDMASYNCCOMPLETIONENDPOINT pEndpoint));
+
+    /**
+     * Initiates a read request from the given endpoint.
+     *
+     * @returns VBox status code.
+     * @param   pTask         Pointer to the task object associated with the request.
+     * @param   pEndpoint     Endpoint the request is for.
+     * @param   off           Where to start reading from.
+     * @param   paSegments    Scatter gather list to store the data in.
+     * @param   cSegments     Number of segments in the list.
+     * @param   cbRead        The overall number of bytes to read.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpRead, (PPDMASYNCCOMPLETIONTASK pTask,
+                                          PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                                          PCRTSGSEG paSegments, size_t cSegments,
+                                         size_t cbRead));
+
+    /**
+     * Initiates a write request to the given endpoint.
+     *
+     * @returns VBox status code.
+     * @param   pTask         Pointer to the task object associated with the request.
+     * @param   pEndpoint     Endpoint the request is for.
+     * @param   off           Where to start writing to.
+     * @param   paSegments    Scatter gather list to store the data in.
+     * @param   cSegments     Number of segments in the list.
+     * @param   cbRead        The overall number of bytes to write.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpWrite, (PPDMASYNCCOMPLETIONTASK pTask,
+                                           PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                                           PCRTSGSEG paSegments, size_t cSegments,
+                                           size_t cbWrite));
+
+    /**
+     * Initiates a flush request on the given endpoint.
+     *
+     * @returns VBox status code.
+     * @param   pTask         Pointer to the task object associated with the request.
+     * @param   pEndpoint     Endpoint the request is for.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpFlush, (PPDMASYNCCOMPLETIONTASK pTask,
+                                           PPDMASYNCCOMPLETIONENDPOINT pEndpoint));
+
+    /**
+     * Queries the size of the endpoint. Optional.
+     *
+     * @returns VBox status code.
+     * @param   pEndpoint     Endpoint the request is for.
+     * @param   pcbSize       Where to store the size of the endpoint.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpGetSize, (PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                             uint64_t *pcbSize));
+
+    /**
+     * Sets the size of the endpoint. Optional.
+     * This is a synchronous operation.
+     *
+     *
+     * @returns VBox status code.
+     * @param   pEndpoint     Endpoint the request is for.
+     * @param   cbSize        New size for the endpoint.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpSetSize, (PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                             uint64_t cbSize));
+
+    /** Initialization safety marker. */
+    uint32_t    u32VersionEnd;
+} PDMASYNCCOMPLETIONEPCLASSOPS;
+/** Pointer to a async completion endpoint class operation table. */
+typedef PDMASYNCCOMPLETIONEPCLASSOPS *PPDMASYNCCOMPLETIONEPCLASSOPS;
+/** Const pointer to a async completion endpoint class operation table. */
+typedef const PDMASYNCCOMPLETIONEPCLASSOPS *PCPDMASYNCCOMPLETIONEPCLASSOPS;
+
+/** Version for the endpoint class operations structure. */
+#define PDMAC_EPCLASS_OPS_VERSION 0x00000001
+
+/** Pointer to a bandwidth control manager. */
+typedef struct PDMACBWMGR *PPDMACBWMGR;
+
+/**
+ * PDM Async completion endpoint class.
+ * Common data.
+ */
+typedef struct PDMASYNCCOMPLETIONEPCLASS
+{
+    /** Pointer to the VM. */
+    PVM                                         pVM;
+    /** Critical section protecting the lists below. */
+    RTCRITSECT                                  CritSect;
+    /** Number of endpoints in the list. */
+    volatile unsigned                           cEndpoints;
+    /** Head of endpoints with this class. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINT)      pEndpointsHead;
+    /** Head of the bandwidth managers for this class. */
+    R3PTRTYPE(PPDMACBWMGR)                      pBwMgrsHead;
+    /** Pointer to the callback table. */
+    R3PTRTYPE(PCPDMASYNCCOMPLETIONEPCLASSOPS)   pEndpointOps;
+    /** Task cache. */
+    RTMEMCACHE                                  hMemCacheTasks;
+    /** Flag whether to gather advanced statistics about requests. */
+    bool                                        fGatherAdvancedStatistics;
+} PDMASYNCCOMPLETIONEPCLASS;
+/** Pointer to the PDM async completion endpoint class data. */
+typedef PDMASYNCCOMPLETIONEPCLASS *PPDMASYNCCOMPLETIONEPCLASS;
+
+/**
+ * A PDM Async completion endpoint.
+ * Common data.
+ */
+typedef struct PDMASYNCCOMPLETIONENDPOINT
+{
+    /** Next endpoint in the list. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINT)      pNext;
+    /** Previous endpoint in the list. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINT)      pPrev;
+    /** Pointer to the class this endpoint belongs to. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONEPCLASS)       pEpClass;
+    /** Template associated with this endpoint. */
+    PPDMASYNCCOMPLETIONTEMPLATE                 pTemplate;
+    /** Statistics ID for endpoints having a similar URI (same filename for example)
+     * to avoid assertions. */
+    unsigned                                    iStatId;
+    /** URI describing the endpoint */
+    char                                       *pszUri;
+    /** Pointer to the assigned bandwidth manager. */
+    volatile PPDMACBWMGR                        pBwMgr;
+    /** Aligns following statistic counters on a 8 byte boundary. */
+    uint32_t                                    u32Alignment;
+    /** @name Request size statistics.
+     * @{ */
+    STAMCOUNTER                                 StatReqSizeSmaller512;
+    STAMCOUNTER                                 StatReqSize512To1K;
+    STAMCOUNTER                                 StatReqSize1KTo2K;
+    STAMCOUNTER                                 StatReqSize2KTo4K;
+    STAMCOUNTER                                 StatReqSize4KTo8K;
+    STAMCOUNTER                                 StatReqSize8KTo16K;
+    STAMCOUNTER                                 StatReqSize16KTo32K;
+    STAMCOUNTER                                 StatReqSize32KTo64K;
+    STAMCOUNTER                                 StatReqSize64KTo128K;
+    STAMCOUNTER                                 StatReqSize128KTo256K;
+    STAMCOUNTER                                 StatReqSize256KTo512K;
+    STAMCOUNTER                                 StatReqSizeOver512K;
+    STAMCOUNTER                                 StatReqsUnaligned512;
+    STAMCOUNTER                                 StatReqsUnaligned4K;
+    STAMCOUNTER                                 StatReqsUnaligned8K;
+    /** @} */
+    /** @name Request completion time statistics.
+     * @{ */
+    STAMCOUNTER                                 StatTaskRunTimesNs[10];
+    STAMCOUNTER                                 StatTaskRunTimesUs[10];
+    STAMCOUNTER                                 StatTaskRunTimesMs[10];
+    STAMCOUNTER                                 StatTaskRunTimesSec[10];
+    STAMCOUNTER                                 StatTaskRunOver100Sec;
+    STAMCOUNTER                                 StatIoOpsPerSec;
+    STAMCOUNTER                                 StatIoOpsStarted;
+    STAMCOUNTER                                 StatIoOpsCompleted;
+    uint64_t                                    tsIntervalStartMs;
+    uint64_t                                    cIoOpsCompleted;
+    /** @} */
+} PDMASYNCCOMPLETIONENDPOINT;
+AssertCompileMemberAlignment(PDMASYNCCOMPLETIONENDPOINT, StatReqSizeSmaller512, sizeof(uint64_t));
+AssertCompileMemberAlignment(PDMASYNCCOMPLETIONENDPOINT, StatTaskRunTimesNs, sizeof(uint64_t));
+
+/**
+ * A PDM async completion task handle.
+ * Common data.
+ */
+typedef struct PDMASYNCCOMPLETIONTASK
+{
+    /** Next task in the list
+     * (for free and assigned tasks). */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONTASK)      pNext;
+    /** Previous task in the list
+     * (for free and assigned tasks). */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONTASK)      pPrev;
+    /** Endpoint this task is assigned to. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINT)  pEndpoint;
+    /** Opaque user data for this task. */
+    void                                   *pvUser;
+    /** Start timestamp. */
+    uint64_t                                tsNsStart;
+} PDMASYNCCOMPLETIONTASK;
+
+void pdmR3AsyncCompletionCompleteTask(PPDMASYNCCOMPLETIONTASK pTask, int rc, bool fCallCompletionHandler);
+bool pdmacEpIsTransferAllowed(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint32_t cbTransfer, RTMSINTERVAL *pmsWhenNext);
+
+RT_C_DECLS_END
+
+extern const PDMASYNCCOMPLETIONEPCLASSOPS g_PDMAsyncCompletionEndpointClassFile;
+
+#endif /* !VMM_INCLUDED_SRC_include_PDMAsyncCompletionInternal_h */
+
diff --git a/src/VBox/VMM/include/PDMBlkCacheInternal.h b/src/VBox/VMM/include/PDMBlkCacheInternal.h
new file mode 100644
index 00000000..6dcbbbfd
--- /dev/null
+++ b/src/VBox/VMM/include/PDMBlkCacheInternal.h
@@ -0,0 +1,334 @@
+/* $Id: PDMBlkCacheInternal.h $ */
+/** @file
+ * PDM Block Cache.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_PDMBlkCacheInternal_h
+#define VMM_INCLUDED_SRC_include_PDMBlkCacheInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/pdmblkcache.h>
+#include <iprt/types.h>
+#include <iprt/file.h>
+#include <iprt/thread.h>
+#include <iprt/semaphore.h>
+#include <iprt/critsect.h>
+#include <iprt/avl.h>
+#include <iprt/list.h>
+#include <iprt/spinlock.h>
+#include <iprt/memcache.h>
+
+RT_C_DECLS_BEGIN
+
+/**
+ * A few forward declarations.
+ */
+/** Pointer to a cache LRU list. */
+typedef struct PDMBLKLRULIST *PPDMBLKLRULIST;
+/** Pointer to the global cache structure. */
+typedef struct PDMBLKCACHEGLOBAL *PPDMBLKCACHEGLOBAL;
+/** Pointer to a cache entry waiter structure. */
+typedef struct PDMBLKCACHEWAITER *PPDMBLKCACHEWAITER;
+
+/**
+ * A cache entry
+ */
+typedef struct PDMBLKCACHEENTRY
+{
+    /** The AVL entry data. */
+    AVLRU64NODECORE                 Core;
+    /** Pointer to the previous element. Used in one of the LRU lists.*/
+    struct PDMBLKCACHEENTRY        *pPrev;
+    /** Pointer to the next element. Used in one of the LRU lists.*/
+    struct PDMBLKCACHEENTRY        *pNext;
+    /** Pointer to the list the entry is in. */
+    PPDMBLKLRULIST                  pList;
+    /** Cache the entry belongs to. */
+    PPDMBLKCACHE                    pBlkCache;
+    /** Flags for this entry. Combinations of PDMACFILECACHE_* \#defines */
+    volatile uint32_t               fFlags;
+    /** Reference counter. Prevents eviction of the entry if > 0. */
+    volatile uint32_t               cRefs;
+    /** Size of the entry. */
+    uint32_t                        cbData;
+    /** Pointer to the memory containing the data. */
+    uint8_t                        *pbData;
+    /** Head of list of tasks waiting for this one to finish. */
+    PPDMBLKCACHEWAITER              pWaitingHead;
+    /** Tail of list of tasks waiting for this one to finish. */
+    PPDMBLKCACHEWAITER              pWaitingTail;
+    /** Node for dirty but not yet committed entries list per endpoint. */
+    RTLISTNODE                      NodeNotCommitted;
+} PDMBLKCACHEENTRY, *PPDMBLKCACHEENTRY;
+/** I/O is still in progress for this entry. This entry is not evictable. */
+#define PDMBLKCACHE_ENTRY_IO_IN_PROGRESS RT_BIT(0)
+/** Entry is locked and thus not evictable. */
+#define PDMBLKCACHE_ENTRY_LOCKED         RT_BIT(1)
+/** Entry is dirty */
+#define PDMBLKCACHE_ENTRY_IS_DIRTY       RT_BIT(2)
+/** Entry is not evictable. */
+#define PDMBLKCACHE_NOT_EVICTABLE  (PDMBLKCACHE_ENTRY_LOCKED | PDMBLKCACHE_ENTRY_IO_IN_PROGRESS | PDMBLKCACHE_ENTRY_IS_DIRTY)
+
+/**
+ * LRU list data
+ */
+typedef struct PDMBLKLRULIST
+{
+    /** Head of the list. */
+    PPDMBLKCACHEENTRY pHead;
+    /** Tail of the list. */
+    PPDMBLKCACHEENTRY pTail;
+    /** Number of bytes cached in the list. */
+    uint32_t          cbCached;
+} PDMBLKLRULIST;
+
+/**
+ * Global cache data.
+ */
+typedef struct PDMBLKCACHEGLOBAL
+{
+    /** Pointer to the owning VM instance. */
+    PVM                 pVM;
+    /** Maximum size of the cache in bytes. */
+    uint32_t            cbMax;
+    /** Current size of the cache in bytes. */
+    uint32_t            cbCached;
+    /** Critical section protecting the cache. */
+    RTCRITSECT          CritSect;
+    /** Maximum number of bytes cached. */
+    uint32_t            cbRecentlyUsedInMax;
+    /** Maximum number of bytes in the paged out list .*/
+    uint32_t            cbRecentlyUsedOutMax;
+    /** Recently used cache entries list */
+    PDMBLKLRULIST       LruRecentlyUsedIn;
+    /** Scorecard cache entry list. */
+    PDMBLKLRULIST       LruRecentlyUsedOut;
+    /** List of frequently used cache entries */
+    PDMBLKLRULIST       LruFrequentlyUsed;
+    /** Commit timeout in milli seconds */
+    uint32_t            u32CommitTimeoutMs;
+    /** Number of dirty bytes needed to start a commit of the data to the disk. */
+    uint32_t            cbCommitDirtyThreshold;
+    /** Current number of dirty bytes in the cache. */
+    volatile uint32_t   cbDirty;
+    /** Flag whether the VM was suspended becaus of an I/O error. */
+    volatile bool       fIoErrorVmSuspended;
+    /** Flag whether a commit is currently in progress. */
+    volatile bool       fCommitInProgress;
+    /** Commit interval timer */
+    PTMTIMERR3          pTimerCommit;
+    /** Number of endpoints using the cache. */
+    uint32_t            cRefs;
+    /** List of all users of this cache. */
+    RTLISTANCHOR        ListUsers;
+#ifdef VBOX_WITH_STATISTICS
+    /** Hit counter. */
+    STAMCOUNTER         cHits;
+    /** Partial hit counter. */
+    STAMCOUNTER         cPartialHits;
+    /** Miss counter. */
+    STAMCOUNTER         cMisses;
+    /** Bytes read from cache. */
+    STAMCOUNTER         StatRead;
+    /** Bytes written to the cache. */
+    STAMCOUNTER         StatWritten;
+    /** Time spend to get an entry in the AVL tree. */
+    STAMPROFILEADV      StatTreeGet;
+    /** Time spend to insert an entry in the AVL tree. */
+    STAMPROFILEADV      StatTreeInsert;
+    /** Time spend to remove an entry in the AVL tree. */
+    STAMPROFILEADV      StatTreeRemove;
+    /** Number of times a buffer could be reused. */
+    STAMCOUNTER         StatBuffersReused;
+#endif
+} PDMBLKCACHEGLOBAL;
+#ifdef VBOX_WITH_STATISTICS
+AssertCompileMemberAlignment(PDMBLKCACHEGLOBAL, cHits, sizeof(uint64_t));
+#endif
+
+/**
+ * Block cache type.
+ */
+typedef enum PDMBLKCACHETYPE
+{
+    /** Device . */
+    PDMBLKCACHETYPE_DEV = 1,
+    /** Driver consumer. */
+    PDMBLKCACHETYPE_DRV,
+    /** Internal consumer. */
+    PDMBLKCACHETYPE_INTERNAL,
+    /** Usb consumer. */
+    PDMBLKCACHETYPE_USB
+} PDMBLKCACHETYPE;
+
+/**
+ * Per user cache data.
+ */
+typedef struct PDMBLKCACHE
+{
+    /** Pointer to the id for the cache. */
+    char                         *pszId;
+    /** AVL tree managing cache entries. */
+    PAVLRU64TREE                  pTree;
+    /** R/W semaphore protecting cached entries for this endpoint. */
+    RTSEMRW                       SemRWEntries;
+    /** Pointer to the gobal cache data */
+    PPDMBLKCACHEGLOBAL            pCache;
+    /** Lock protecting the dirty entries list. */
+    RTSPINLOCK                    LockList;
+    /** List of dirty but not committed entries for this endpoint. */
+    RTLISTANCHOR                  ListDirtyNotCommitted;
+    /** Node of the cache user list. */
+    RTLISTNODE                    NodeCacheUser;
+    /** Block cache type. */
+    PDMBLKCACHETYPE               enmType;
+    /** Type specific data. */
+    union
+    {
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_DEV */
+        struct
+        {
+            /** Pointer to the device instance owning the block cache. */
+            R3PTRTYPE(PPDMDEVINS)                            pDevIns;
+            /** Complete callback to the user. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERCOMPLETEDEV)         pfnXferComplete;
+            /** I/O enqueue callback. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEDEV)          pfnXferEnqueue;
+            /** Discard enqueue callback. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEDISCARDDEV)   pfnXferEnqueueDiscard;
+        } Dev;
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_DRV */
+        struct
+        {
+            /** Pointer to the driver instance owning the block cache. */
+            R3PTRTYPE(PPDMDRVINS)                            pDrvIns;
+            /** Complete callback to the user. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERCOMPLETEDRV)         pfnXferComplete;
+            /** I/O enqueue callback. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEDRV)          pfnXferEnqueue;
+            /** Discard enqueue callback. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEDISCARDDRV)   pfnXferEnqueueDiscard;
+        } Drv;
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL */
+        struct
+        {
+            /** Pointer to user data. */
+            R3PTRTYPE(void *)                                pvUser;
+            /** Complete callback to the user. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERCOMPLETEINT)         pfnXferComplete;
+            /** I/O enqueue callback. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEINT)          pfnXferEnqueue;
+            /** Discard enqueue callback. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEDISCARDINT)   pfnXferEnqueueDiscard;
+        } Int;
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_USB */
+        struct
+        {
+            /** Pointer to the usb instance owning the template. */
+            R3PTRTYPE(PPDMUSBINS)                            pUsbIns;
+            /** Complete callback to the user. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERCOMPLETEUSB)         pfnXferComplete;
+            /** I/O enqueue callback. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEUSB)          pfnXferEnqueue;
+            /** Discard enqueue callback. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEDISCARDUSB)   pfnXferEnqueueDiscard;
+        } Usb;
+    } u;
+
+#ifdef VBOX_WITH_STATISTICS
+
+#if HC_ARCH_BITS == 64
+    uint32_t                      u32Alignment;
+#endif
+    /** Number of times a write was deferred because the cache entry was still in progress */
+    STAMCOUNTER                   StatWriteDeferred;
+    /** Number appended cache entries. */
+    STAMCOUNTER                   StatAppendedWrites;
+#endif
+
+    /** Flag whether the cache was suspended. */
+    volatile bool                 fSuspended;
+    /** Number of outstanding I/O transfers. */
+    volatile uint32_t             cIoXfersActive;
+
+} PDMBLKCACHE, *PPDMBLKCACHE;
+#ifdef VBOX_WITH_STATISTICS
+AssertCompileMemberAlignment(PDMBLKCACHE, StatWriteDeferred, sizeof(uint64_t));
+#endif
+
+/**
+ * I/O task.
+ */
+typedef struct PDMBLKCACHEREQ
+{
+    /** Opaque user data returned on completion. */
+    void             *pvUser;
+    /** Number of pending transfers (waiting for a cache entry and passed through). */
+    volatile uint32_t cXfersPending;
+    /** Status code. */
+    volatile int      rcReq;
+} PDMBLKCACHEREQ, *PPDMBLKCACHEREQ;
+
+/**
+ * I/O transfer from the cache to the underlying medium.
+ */
+typedef struct PDMBLKCACHEIOXFER
+{
+    /** Flag whether the I/O xfer updates a cache entry or updates the request directly. */
+    bool                  fIoCache;
+    /** Type dependent data. */
+    union
+    {
+        /** Pointer to the entry the transfer updates. */
+        PPDMBLKCACHEENTRY pEntry;
+        /** Pointer to the request the transfer updates. */
+        PPDMBLKCACHEREQ   pReq;
+    };
+    /** Transfer direction. */
+    PDMBLKCACHEXFERDIR    enmXferDir;
+    /** Segment used if a cache entry is updated. */
+    RTSGSEG               SgSeg;
+    /** S/G buffer. */
+    RTSGBUF               SgBuf;
+} PDMBLKCACHEIOXFER;
+
+/**
+ * Cache waiter
+ */
+typedef struct PDMBLKCACHEWAITER
+{
+    /* Next waiter in the list. */
+    struct PDMBLKCACHEWAITER *pNext;
+    /** S/G buffer holding or receiving data. */
+    RTSGBUF                   SgBuf;
+    /** Offset into the cache entry to start the transfer. */
+    uint32_t                  offCacheEntry;
+    /** How many bytes to transfer. */
+    size_t                    cbTransfer;
+    /** Flag whether the task wants to read or write into the entry. */
+    bool                      fWrite;
+    /** Task the waiter is for. */
+    PPDMBLKCACHEREQ           pReq;
+} PDMBLKCACHEWAITER;
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_PDMBlkCacheInternal_h */
+
diff --git a/src/VBox/VMM/include/PDMInline.h b/src/VBox/VMM/include/PDMInline.h
new file mode 100644
index 00000000..abf1b6b8
--- /dev/null
+++ b/src/VBox/VMM/include/PDMInline.h
@@ -0,0 +1,42 @@
+/* $Id: PDMInline.h $ */
+/** @file
+ * PDM - Internal header file containing the inlined functions.
+ */
+
+/*
+ * Copyright (C) 2012-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_PDMInline_h
+#define VMM_INCLUDED_SRC_include_PDMInline_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+/**
+ * Calculates the next IRQ tag.
+ *
+ * @returns IRQ tag.
+ * @param   pVM                 The cross context VM structure.
+ * @param   idTracer            The ID of the source device.
+ */
+DECLINLINE(uint32_t) pdmCalcIrqTag(PVM pVM, uint32_t idTracer)
+{
+    uint32_t uTag = (pVM->pdm.s.uIrqTag + 1) & 0x3ff; /* {0..1023} */
+    if (!uTag)
+        uTag++;
+    pVM->pdm.s.uIrqTag = uTag |= (idTracer << 16);
+    return uTag;
+}
+
+#endif /* !VMM_INCLUDED_SRC_include_PDMInline_h */
+
diff --git a/src/VBox/VMM/include/PDMInternal.h b/src/VBox/VMM/include/PDMInternal.h
new file mode 100644
index 00000000..25350b82
--- /dev/null
+++ b/src/VBox/VMM/include/PDMInternal.h
@@ -0,0 +1,1538 @@
+/* $Id: PDMInternal.h $ */
+/** @file
+ * PDM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_PDMInternal_h
+#define VMM_INCLUDED_SRC_include_PDMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/types.h>
+#include <VBox/param.h>
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vusb.h>
+#include <VBox/vmm/pdmasynccompletion.h>
+#ifdef VBOX_WITH_NETSHAPER
+# include <VBox/vmm/pdmnetshaper.h>
+#endif
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+# include <VBox/vmm/pdmasynccompletion.h>
+#endif
+#include <VBox/vmm/pdmblkcache.h>
+#include <VBox/vmm/pdmcommon.h>
+#include <VBox/vmm/pdmtask.h>
+#include <VBox/sup.h>
+#include <iprt/assert.h>
+#include <iprt/critsect.h>
+#ifdef IN_RING3
+# include <iprt/thread.h>
+#endif
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_pdm_int       Internal
+ * @ingroup grp_pdm
+ * @internal
+ * @{
+ */
+
+/** @def PDM_WITH_R3R0_CRIT_SECT
+ * Enables or disabled ring-3/ring-0 critical sections. */
+#if defined(DOXYGEN_RUNNING) || 1
+# define PDM_WITH_R3R0_CRIT_SECT
+#endif
+
+/** @def PDMCRITSECT_STRICT
+ * Enables/disables PDM critsect strictness like deadlock detection. */
+#if (defined(RT_LOCK_STRICT) && defined(IN_RING3) && !defined(PDMCRITSECT_STRICT)) \
+  || defined(DOXYGEN_RUNNING)
+# define PDMCRITSECT_STRICT
+#endif
+
+/** @def PDMCRITSECT_STRICT
+ * Enables/disables PDM read/write critsect strictness like deadlock
+ * detection. */
+#if (defined(RT_LOCK_STRICT) && defined(IN_RING3) && !defined(PDMCRITSECTRW_STRICT)) \
+  || defined(DOXYGEN_RUNNING)
+# define PDMCRITSECTRW_STRICT
+#endif
+
+/** The maximum device instance (total) size, ring-0/raw-mode capable devices. */
+#define PDM_MAX_DEVICE_INSTANCE_SIZE    _4M
+/** The maximum device instance (total) size, ring-3 only devices. */
+#define PDM_MAX_DEVICE_INSTANCE_SIZE_R3 _8M
+
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+
+/** Pointer to a PDM Device. */
+typedef struct PDMDEV *PPDMDEV;
+/** Pointer to a pointer to a PDM Device. */
+typedef PPDMDEV *PPPDMDEV;
+
+/** Pointer to a PDM USB Device. */
+typedef struct PDMUSB *PPDMUSB;
+/** Pointer to a pointer to a PDM USB Device. */
+typedef PPDMUSB *PPPDMUSB;
+
+/** Pointer to a PDM Driver. */
+typedef struct PDMDRV *PPDMDRV;
+/** Pointer to a pointer to a PDM Driver. */
+typedef PPDMDRV *PPPDMDRV;
+
+/** Pointer to a PDM Logical Unit. */
+typedef struct PDMLUN *PPDMLUN;
+/** Pointer to a pointer to a PDM Logical Unit. */
+typedef PPDMLUN *PPPDMLUN;
+
+/** Pointer to a PDM PCI Bus instance. */
+typedef struct PDMPCIBUS *PPDMPCIBUS;
+/** Pointer to a DMAC instance. */
+typedef struct PDMDMAC *PPDMDMAC;
+/** Pointer to a RTC instance. */
+typedef struct PDMRTC *PPDMRTC;
+
+/** Pointer to an USB HUB registration record. */
+typedef struct PDMUSBHUB *PPDMUSBHUB;
+
+/**
+ * Supported asynchronous completion endpoint classes.
+ */
+typedef enum PDMASYNCCOMPLETIONEPCLASSTYPE
+{
+    /** File class. */
+    PDMASYNCCOMPLETIONEPCLASSTYPE_FILE = 0,
+    /** Number of supported classes. */
+    PDMASYNCCOMPLETIONEPCLASSTYPE_MAX,
+    /** 32bit hack. */
+    PDMASYNCCOMPLETIONEPCLASSTYPE_32BIT_HACK = 0x7fffffff
+} PDMASYNCCOMPLETIONEPCLASSTYPE;
+
+/**
+ * Private device instance data, ring-3.
+ */
+typedef struct PDMDEVINSINTR3
+{
+    /** Pointer to the next instance.
+     * (Head is pointed to by PDM::pDevInstances.) */
+    R3PTRTYPE(PPDMDEVINS)           pNextR3;
+    /** Pointer to the next per device instance.
+     * (Head is pointed to by PDMDEV::pInstances.) */
+    R3PTRTYPE(PPDMDEVINS)           pPerDeviceNextR3;
+    /** Pointer to device structure. */
+    R3PTRTYPE(PPDMDEV)              pDevR3;
+    /** Pointer to the list of logical units associated with the device. (FIFO) */
+    R3PTRTYPE(PPDMLUN)              pLunsR3;
+    /** Pointer to the asynchronous notification callback set while in
+     * FNPDMDEVSUSPEND or FNPDMDEVPOWEROFF. */
+    R3PTRTYPE(PFNPDMDEVASYNCNOTIFY) pfnAsyncNotify;
+    /** Configuration handle to the instance node. */
+    R3PTRTYPE(PCFGMNODE)            pCfgHandle;
+
+    /** R3 pointer to the VM this instance was created for. */
+    PVMR3                           pVMR3;
+
+    /** Flags, see PDMDEVINSINT_FLAGS_XXX. */
+    uint32_t                        fIntFlags;
+    /** The last IRQ tag (for tracing it thru clearing). */
+    uint32_t                        uLastIrqTag;
+    /** The ring-0 device index (for making ring-0 calls). */
+    uint32_t                        idxR0Device;
+} PDMDEVINSINTR3;
+
+
+/**
+ * Private device instance data, ring-0.
+ */
+typedef struct PDMDEVINSINTR0
+{
+    /** Pointer to the VM this instance was created for. */
+    R0PTRTYPE(PGVM)                 pGVM;
+    /** Pointer to device structure. */
+    R0PTRTYPE(struct PDMDEVREGR0 const *) pRegR0;
+    /** The ring-0 module reference. */
+    RTR0PTR                         hMod;
+    /** Pointer to the ring-0 mapping of the ring-3 internal data (for uLastIrqTag). */
+    R0PTRTYPE(PDMDEVINSINTR3 *)     pIntR3R0;
+    /** Pointer to the ring-0 mapping of the ring-3 instance (for idTracing). */
+    R0PTRTYPE(struct PDMDEVINSR3 *) pInsR3R0;
+    /** The device instance memory. */
+    RTR0MEMOBJ                      hMemObj;
+    /** The ring-3 mapping object. */
+    RTR0MEMOBJ                      hMapObj;
+    /** Index into PDMR0PERVM::apDevInstances. */
+    uint32_t                        idxR0Device;
+} PDMDEVINSINTR0;
+
+
+/**
+ * Private device instance data, raw-mode
+ */
+typedef struct PDMDEVINSINTRC
+{
+    /** Pointer to the VM this instance was created for. */
+    RGPTRTYPE(PVM)                  pVMRC;
+} PDMDEVINSINTRC;
+
+
+/**
+ * Private device instance data.
+ */
+typedef struct PDMDEVINSINT
+{
+    /** Pointer to the next instance (HC Ptr).
+     * (Head is pointed to by PDM::pDevInstances.) */
+    R3PTRTYPE(PPDMDEVINS)           pNextR3;
+    /** Pointer to the next per device instance (HC Ptr).
+     * (Head is pointed to by PDMDEV::pInstances.) */
+    R3PTRTYPE(PPDMDEVINS)           pPerDeviceNextR3;
+    /** Pointer to device structure - HC Ptr. */
+    R3PTRTYPE(PPDMDEV)              pDevR3;
+    /** Pointer to the list of logical units associated with the device. (FIFO) */
+    R3PTRTYPE(PPDMLUN)              pLunsR3;
+    /** Pointer to the asynchronous notification callback set while in
+     * FNPDMDEVSUSPEND or FNPDMDEVPOWEROFF. */
+    R3PTRTYPE(PFNPDMDEVASYNCNOTIFY) pfnAsyncNotify;
+    /** Configuration handle to the instance node. */
+    R3PTRTYPE(PCFGMNODE)            pCfgHandle;
+
+    /** R3 pointer to the VM this instance was created for. */
+    PVMR3                           pVMR3;
+
+    /** R0 pointer to the VM this instance was created for. */
+    R0PTRTYPE(PVMCC)                pVMR0;
+
+    /** RC pointer to the VM this instance was created for. */
+    PVMRC                           pVMRC;
+
+    /** Flags, see PDMDEVINSINT_FLAGS_XXX. */
+    uint32_t                        fIntFlags;
+    /** The last IRQ tag (for tracing it thru clearing). */
+    uint32_t                        uLastIrqTag;
+} PDMDEVINSINT;
+
+/** @name PDMDEVINSINT::fIntFlags
+ * @{ */
+/** Used by pdmR3Load to mark device instances it found in the saved state. */
+#define PDMDEVINSINT_FLAGS_FOUND            RT_BIT_32(0)
+/** Indicates that the device hasn't been powered on or resumed.
+ * This is used by PDMR3PowerOn, PDMR3Resume, PDMR3Suspend and PDMR3PowerOff
+ * to make sure each device gets exactly one notification for each of those
+ * events.  PDMR3Resume and PDMR3PowerOn also makes use of it to bail out on
+ * a failure (already resumed/powered-on devices are suspended).
+ * PDMR3PowerOff resets this flag once before going through the devices to make sure
+ * every device gets the power off notification even if it was suspended before with
+ * PDMR3Suspend.
+ */
+#define PDMDEVINSINT_FLAGS_SUSPENDED        RT_BIT_32(1)
+/** Indicates that the device has been reset already.  Used by PDMR3Reset. */
+#define PDMDEVINSINT_FLAGS_RESET            RT_BIT_32(2)
+#define PDMDEVINSINT_FLAGS_R0_ENABLED       RT_BIT_32(3)
+#define PDMDEVINSINT_FLAGS_RC_ENABLED       RT_BIT_32(4)
+/** Set if we've called the ring-0 constructor. */
+#define PDMDEVINSINT_FLAGS_R0_CONTRUCT      RT_BIT_32(5)
+/** Set if using non-default critical section.  */
+#define PDMDEVINSINT_FLAGS_CHANGED_CRITSECT RT_BIT_32(6)
+/** @} */
+
+
+/**
+ * Private USB device instance data.
+ */
+typedef struct PDMUSBINSINT
+{
+    /** The UUID of this instance. */
+    RTUUID                          Uuid;
+    /** Pointer to the next instance.
+     * (Head is pointed to by PDM::pUsbInstances.) */
+    R3PTRTYPE(PPDMUSBINS)           pNext;
+    /** Pointer to the next per USB device instance.
+     * (Head is pointed to by PDMUSB::pInstances.) */
+    R3PTRTYPE(PPDMUSBINS)           pPerDeviceNext;
+
+    /** Pointer to device structure. */
+    R3PTRTYPE(PPDMUSB)              pUsbDev;
+
+    /** Pointer to the VM this instance was created for. */
+    PVMR3                           pVM;
+    /** Pointer to the list of logical units associated with the device. (FIFO) */
+    R3PTRTYPE(PPDMLUN)              pLuns;
+    /** The per instance device configuration. */
+    R3PTRTYPE(PCFGMNODE)            pCfg;
+    /** Same as pCfg if the configuration should be deleted when detaching the device. */
+    R3PTRTYPE(PCFGMNODE)            pCfgDelete;
+    /** The global device configuration. */
+    R3PTRTYPE(PCFGMNODE)            pCfgGlobal;
+
+    /** Pointer to the USB hub this device is attached to.
+     * This is NULL if the device isn't connected to any HUB. */
+    R3PTRTYPE(PPDMUSBHUB)           pHub;
+    /** The port number that we're connected to. */
+    uint32_t                        iPort;
+    /** Indicates that the USB device hasn't been powered on or resumed.
+     * See PDMDEVINSINT_FLAGS_SUSPENDED. */
+    bool                            fVMSuspended;
+    /** Indicates that the USB device has been reset. */
+    bool                            fVMReset;
+    /** Pointer to the asynchronous notification callback set while in
+     * FNPDMDEVSUSPEND or FNPDMDEVPOWEROFF. */
+    R3PTRTYPE(PFNPDMUSBASYNCNOTIFY) pfnAsyncNotify;
+} PDMUSBINSINT;
+
+
+/**
+ * Private driver instance data.
+ */
+typedef struct PDMDRVINSINT
+{
+    /** Pointer to the driver instance above.
+     * This is NULL for the topmost drive. */
+    R3PTRTYPE(PPDMDRVINS)           pUp;
+    /** Pointer to the driver instance below.
+     * This is NULL for the bottommost driver. */
+    R3PTRTYPE(PPDMDRVINS)           pDown;
+    /** Pointer to the logical unit this driver chained on. */
+    R3PTRTYPE(PPDMLUN)              pLun;
+    /** Pointer to driver structure from which this was instantiated. */
+    R3PTRTYPE(PPDMDRV)              pDrv;
+    /** Pointer to the VM this instance was created for, ring-3 context. */
+    PVMR3                           pVMR3;
+    /** Pointer to the VM this instance was created for, ring-0 context. */
+    R0PTRTYPE(PVMCC)                pVMR0;
+    /** Pointer to the VM this instance was created for, raw-mode context. */
+    PVMRC                           pVMRC;
+    /** Flag indicating that the driver is being detached and destroyed.
+     * (Helps detect potential recursive detaching.) */
+    bool                            fDetaching;
+    /** Indicates that the driver hasn't been powered on or resumed.
+     * See PDMDEVINSINT_FLAGS_SUSPENDED. */
+    bool                            fVMSuspended;
+    /** Indicates that the driver has been reset already. */
+    bool                            fVMReset;
+    /** Set if allocated on the hyper heap, false if on the ring-3 heap. */
+    bool                            fHyperHeap;
+    /** Pointer to the asynchronous notification callback set while in
+     * PDMUSBREG::pfnVMSuspend or PDMUSBREG::pfnVMPowerOff. */
+    R3PTRTYPE(PFNPDMDRVASYNCNOTIFY) pfnAsyncNotify;
+    /** Configuration handle to the instance node. */
+    R3PTRTYPE(PCFGMNODE)            pCfgHandle;
+    /** Pointer to the ring-0 request handler function. */
+    PFNPDMDRVREQHANDLERR0           pfnReqHandlerR0;
+} PDMDRVINSINT;
+
+
+/**
+ * Private critical section data.
+ */
+typedef struct PDMCRITSECTINT
+{
+    /** The critical section core which is shared with IPRT.
+     * @note The semaphore is a SUPSEMEVENT.  */
+    RTCRITSECT                      Core;
+    /** Pointer to the next critical section.
+     * This chain is used for relocating pVMRC and device cleanup. */
+    R3PTRTYPE(struct PDMCRITSECTINT *) pNext;
+    /** Owner identifier.
+     * This is pDevIns if the owner is a device. Similarly for a driver or service.
+     * PDMR3CritSectInit() sets this to point to the critsect itself. */
+    RTR3PTR                         pvKey;
+    /** Pointer to the VM - R3Ptr. */
+    PVMR3                           pVMR3;
+    /** Pointer to the VM - R0Ptr. */
+    R0PTRTYPE(PVMCC)                pVMR0;
+    /** Pointer to the VM - GCPtr. */
+    PVMRC                           pVMRC;
+    /** Set if this critical section is the automatically created default
+     * section of a device. */
+    bool                            fAutomaticDefaultCritsect;
+    /** Set if the critical section is used by a timer or similar.
+     * See PDMR3DevGetCritSect.  */
+    bool                            fUsedByTimerOrSimilar;
+    /** Alignment padding. */
+    bool                            afPadding[2];
+    /** Support driver event semaphore that is scheduled to be signaled upon leaving
+     * the critical section. This is only for Ring-3 and Ring-0. */
+    SUPSEMEVENT                     hEventToSignal;
+    /** The lock name. */
+    R3PTRTYPE(const char *)         pszName;
+    /** R0/RC lock contention. */
+    STAMCOUNTER                     StatContentionRZLock;
+    /** R0/RC unlock contention. */
+    STAMCOUNTER                     StatContentionRZUnlock;
+    /** R3 lock contention. */
+    STAMCOUNTER                     StatContentionR3;
+    /** Profiling the time the section is locked. */
+    STAMPROFILEADV                  StatLocked;
+} PDMCRITSECTINT;
+AssertCompileMemberAlignment(PDMCRITSECTINT, StatContentionRZLock, 8);
+/** Pointer to private critical section data. */
+typedef PDMCRITSECTINT *PPDMCRITSECTINT;
+
+/** Indicates that the critical section is queued for unlock.
+ * PDMCritSectIsOwner and PDMCritSectIsOwned optimizations. */
+#define PDMCRITSECT_FLAGS_PENDING_UNLOCK    RT_BIT_32(17)
+
+
+/**
+ * Private critical section data.
+ */
+typedef struct PDMCRITSECTRWINT
+{
+    /** The read/write critical section core which is shared with IPRT.
+     * @note The semaphores are SUPSEMEVENT and SUPSEMEVENTMULTI.  */
+    RTCRITSECTRW                        Core;
+
+    /** Pointer to the next critical section.
+     * This chain is used for relocating pVMRC and device cleanup. */
+    R3PTRTYPE(struct PDMCRITSECTRWINT *) pNext;
+    /** Owner identifier.
+     * This is pDevIns if the owner is a device. Similarly for a driver or service.
+     * PDMR3CritSectInit() sets this to point to the critsect itself. */
+    RTR3PTR                             pvKey;
+    /** Pointer to the VM - R3Ptr. */
+    PVMR3                               pVMR3;
+    /** Pointer to the VM - R0Ptr. */
+    R0PTRTYPE(PVMCC)                    pVMR0;
+    /** Pointer to the VM - GCPtr. */
+    PVMRC                               pVMRC;
+#if HC_ARCH_BITS == 64
+    /** Alignment padding. */
+    RTRCPTR                             RCPtrPadding;
+#endif
+    /** The lock name. */
+    R3PTRTYPE(const char *)             pszName;
+    /** R0/RC write lock contention. */
+    STAMCOUNTER                         StatContentionRZEnterExcl;
+    /** R0/RC write unlock contention. */
+    STAMCOUNTER                         StatContentionRZLeaveExcl;
+    /** R0/RC read lock contention. */
+    STAMCOUNTER                         StatContentionRZEnterShared;
+    /** R0/RC read unlock contention. */
+    STAMCOUNTER                         StatContentionRZLeaveShared;
+    /** R0/RC writes. */
+    STAMCOUNTER                         StatRZEnterExcl;
+    /** R0/RC reads. */
+    STAMCOUNTER                         StatRZEnterShared;
+    /** R3 write lock contention. */
+    STAMCOUNTER                         StatContentionR3EnterExcl;
+    /** R3 read lock contention. */
+    STAMCOUNTER                         StatContentionR3EnterShared;
+    /** R3 writes. */
+    STAMCOUNTER                         StatR3EnterExcl;
+    /** R3 reads. */
+    STAMCOUNTER                         StatR3EnterShared;
+    /** Profiling the time the section is write locked. */
+    STAMPROFILEADV                      StatWriteLocked;
+} PDMCRITSECTRWINT;
+AssertCompileMemberAlignment(PDMCRITSECTRWINT, StatContentionRZEnterExcl, 8);
+AssertCompileMemberAlignment(PDMCRITSECTRWINT, Core.u64State, 8);
+/** Pointer to private critical section data. */
+typedef PDMCRITSECTRWINT *PPDMCRITSECTRWINT;
+
+
+
+/**
+ * The usual device/driver/internal/external stuff.
+ */
+typedef enum
+{
+    /** The usual invalid entry. */
+    PDMTHREADTYPE_INVALID = 0,
+    /** Device type. */
+    PDMTHREADTYPE_DEVICE,
+    /** USB Device type. */
+    PDMTHREADTYPE_USB,
+    /** Driver type. */
+    PDMTHREADTYPE_DRIVER,
+    /** Internal type. */
+    PDMTHREADTYPE_INTERNAL,
+    /** External type. */
+    PDMTHREADTYPE_EXTERNAL,
+    /** The usual 32-bit hack. */
+    PDMTHREADTYPE_32BIT_HACK = 0x7fffffff
+} PDMTHREADTYPE;
+
+
+/**
+ * The internal structure for the thread.
+ */
+typedef struct PDMTHREADINT
+{
+    /** The VM pointer. */
+    PVMR3                           pVM;
+    /** The event semaphore the thread blocks on when not running. */
+    RTSEMEVENTMULTI                 BlockEvent;
+    /** The event semaphore the thread sleeps on while running. */
+    RTSEMEVENTMULTI                 SleepEvent;
+    /** Pointer to the next thread. */
+    R3PTRTYPE(struct PDMTHREAD *)   pNext;
+    /** The thread type. */
+    PDMTHREADTYPE                   enmType;
+} PDMTHREADINT;
+
+
+
+/* Must be included after PDMDEVINSINT is defined. */
+#define PDMDEVINSINT_DECLARED
+#define PDMUSBINSINT_DECLARED
+#define PDMDRVINSINT_DECLARED
+#define PDMCRITSECTINT_DECLARED
+#define PDMCRITSECTRWINT_DECLARED
+#define PDMTHREADINT_DECLARED
+#ifdef ___VBox_pdm_h
+# error "Invalid header PDM order. Include PDMInternal.h before VBox/vmm/pdm.h!"
+#endif
+RT_C_DECLS_END
+#include <VBox/vmm/pdm.h>
+RT_C_DECLS_BEGIN
+
+/**
+ * PDM Logical Unit.
+ *
+ * This typically the representation of a physical port on a
+ * device, like for instance the PS/2 keyboard port on the
+ * keyboard controller device. The LUNs are chained on the
+ * device they belong to (PDMDEVINSINT::pLunsR3).
+ */
+typedef struct PDMLUN
+{
+    /** The LUN - The Logical Unit Number. */
+    RTUINT                          iLun;
+    /** Pointer to the next LUN. */
+    PPDMLUN                         pNext;
+    /** Pointer to the top driver in the driver chain. */
+    PPDMDRVINS                      pTop;
+    /** Pointer to the bottom driver in the driver chain. */
+    PPDMDRVINS                      pBottom;
+    /** Pointer to the device instance which the LUN belongs to.
+     * Either this is set or pUsbIns is set. Both is never set at the same time. */
+    PPDMDEVINS                      pDevIns;
+    /** Pointer to the USB device instance which the LUN belongs to. */
+    PPDMUSBINS                      pUsbIns;
+    /** Pointer to the device base interface. */
+    PPDMIBASE                       pBase;
+    /** Description of this LUN. */
+    const char                     *pszDesc;
+} PDMLUN;
+
+
+/**
+ * PDM Device, ring-3.
+ */
+typedef struct PDMDEV
+{
+    /** Pointer to the next device (R3 Ptr). */
+    R3PTRTYPE(PPDMDEV)              pNext;
+    /** Device name length. (search optimization) */
+    uint32_t                        cchName;
+    /** Registration structure. */
+    R3PTRTYPE(const struct PDMDEVREGR3 *) pReg;
+    /** Number of instances. */
+    uint32_t                        cInstances;
+    /** Pointer to chain of instances (R3 Ptr). */
+    PPDMDEVINSR3                    pInstances;
+    /** The search path for raw-mode context modules (';' as separator). */
+    char                           *pszRCSearchPath;
+    /** The search path for ring-0 context modules (';' as separator). */
+    char                           *pszR0SearchPath;
+} PDMDEV;
+
+
+#if 0
+/**
+ * PDM Device, ring-0.
+ */
+typedef struct PDMDEVR0
+{
+    /** Pointer to the next device. */
+    R0PTRTYPE(PPDMDEVR0)            pNext;
+    /** Device name length. (search optimization) */
+    uint32_t                        cchName;
+    /** Registration structure. */
+    R3PTRTYPE(const struct PDMDEVREGR0 *) pReg;
+    /** Number of instances. */
+    uint32_t                        cInstances;
+    /** Pointer to chain of instances. */
+    PPDMDEVINSR0                    pInstances;
+} PDMDEVR0;
+#endif
+
+
+/**
+ * PDM USB Device.
+ */
+typedef struct PDMUSB
+{
+    /** Pointer to the next device (R3 Ptr). */
+    R3PTRTYPE(PPDMUSB)              pNext;
+    /** Device name length. (search optimization) */
+    RTUINT                          cchName;
+    /** Registration structure. */
+    R3PTRTYPE(const struct PDMUSBREG *) pReg;
+    /** Next instance number. */
+    uint32_t                        iNextInstance;
+    /** Pointer to chain of instances (R3 Ptr). */
+    R3PTRTYPE(PPDMUSBINS)           pInstances;
+} PDMUSB;
+
+
+/**
+ * PDM Driver.
+ */
+typedef struct PDMDRV
+{
+    /** Pointer to the next device. */
+    PPDMDRV                         pNext;
+    /** Registration structure. */
+    const struct PDMDRVREG *        pReg;
+    /** Current number of instances. */
+    uint32_t                        cInstances;
+    /** The next instance number. */
+    uint32_t                        iNextInstance;
+    /** The search path for raw-mode context modules (';' as separator). */
+    char                           *pszRCSearchPath;
+    /** The search path for ring-0 context modules (';' as separator). */
+    char                           *pszR0SearchPath;
+} PDMDRV;
+
+
+/**
+ * PDM registered PIC device.
+ */
+typedef struct PDMPIC
+{
+    /** Pointer to the PIC device instance - R3. */
+    PPDMDEVINSR3                    pDevInsR3;
+    /** @copydoc PDMPICREG::pfnSetIrq */
+    DECLR3CALLBACKMEMBER(void,      pfnSetIrqR3,(PPDMDEVINS pDevIns, int iIrq, int iLevel, uint32_t uTagSrc));
+    /** @copydoc PDMPICREG::pfnGetInterrupt */
+    DECLR3CALLBACKMEMBER(int,       pfnGetInterruptR3,(PPDMDEVINS pDevIns, uint32_t *puTagSrc));
+
+    /** Pointer to the PIC device instance - R0. */
+    PPDMDEVINSR0                    pDevInsR0;
+    /** @copydoc PDMPICREG::pfnSetIrq */
+    DECLR0CALLBACKMEMBER(void,      pfnSetIrqR0,(PPDMDEVINS pDevIns, int iIrq, int iLevel, uint32_t uTagSrc));
+    /** @copydoc PDMPICREG::pfnGetInterrupt */
+    DECLR0CALLBACKMEMBER(int,       pfnGetInterruptR0,(PPDMDEVINS pDevIns, uint32_t *puTagSrc));
+
+    /** Pointer to the PIC device instance - RC. */
+    PPDMDEVINSRC                    pDevInsRC;
+    /** @copydoc PDMPICREG::pfnSetIrq */
+    DECLRCCALLBACKMEMBER(void,      pfnSetIrqRC,(PPDMDEVINS pDevIns, int iIrq, int iLevel, uint32_t uTagSrc));
+    /** @copydoc PDMPICREG::pfnGetInterrupt */
+    DECLRCCALLBACKMEMBER(int,       pfnGetInterruptRC,(PPDMDEVINS pDevIns, uint32_t *puTagSrc));
+    /** Alignment padding. */
+    RTRCPTR                         RCPtrPadding;
+} PDMPIC;
+
+
+/**
+ * PDM registered APIC device.
+ */
+typedef struct PDMAPIC
+{
+    /** Pointer to the APIC device instance - R3 Ptr. */
+    PPDMDEVINSR3                       pDevInsR3;
+    /** Pointer to the APIC device instance - R0 Ptr. */
+    PPDMDEVINSR0                       pDevInsR0;
+    /** Pointer to the APIC device instance - RC Ptr. */
+    PPDMDEVINSRC                       pDevInsRC;
+    uint8_t                            Alignment[4];
+} PDMAPIC;
+
+
+/**
+ * PDM registered I/O APIC device.
+ */
+typedef struct PDMIOAPIC
+{
+    /** Pointer to the APIC device instance - R3 Ptr. */
+    PPDMDEVINSR3                    pDevInsR3;
+    /** @copydoc PDMIOAPICREG::pfnSetIrq */
+    DECLR3CALLBACKMEMBER(void,      pfnSetIrqR3,(PPDMDEVINS pDevIns, int iIrq, int iLevel, uint32_t uTagSrc));
+    /** @copydoc PDMIOAPICREG::pfnSendMsi */
+    DECLR3CALLBACKMEMBER(void,      pfnSendMsiR3,(PPDMDEVINS pDevIns, RTGCPHYS GCAddr, uint32_t uValue, uint32_t uTagSrc));
+    /** @copydoc PDMIOAPICREG::pfnSetEoi */
+    DECLR3CALLBACKMEMBER(VBOXSTRICTRC, pfnSetEoiR3,(PPDMDEVINS pDevIns, uint8_t u8Vector));
+
+    /** Pointer to the PIC device instance - R0. */
+    PPDMDEVINSR0                    pDevInsR0;
+    /** @copydoc PDMIOAPICREG::pfnSetIrq */
+    DECLR0CALLBACKMEMBER(void,      pfnSetIrqR0,(PPDMDEVINS pDevIns, int iIrq, int iLevel, uint32_t uTagSrc));
+    /** @copydoc PDMIOAPICREG::pfnSendMsi */
+    DECLR0CALLBACKMEMBER(void,      pfnSendMsiR0,(PPDMDEVINS pDevIns, RTGCPHYS GCAddr, uint32_t uValue, uint32_t uTagSrc));
+    /** @copydoc PDMIOAPICREG::pfnSetEoi */
+    DECLR0CALLBACKMEMBER(VBOXSTRICTRC, pfnSetEoiR0,(PPDMDEVINS pDevIns, uint8_t u8Vector));
+
+    /** Pointer to the APIC device instance - RC Ptr. */
+    PPDMDEVINSRC                    pDevInsRC;
+    /** @copydoc PDMIOAPICREG::pfnSetIrq */
+    DECLRCCALLBACKMEMBER(void,      pfnSetIrqRC,(PPDMDEVINS pDevIns, int iIrq, int iLevel, uint32_t uTagSrc));
+     /** @copydoc PDMIOAPICREG::pfnSendMsi */
+    DECLRCCALLBACKMEMBER(void,      pfnSendMsiRC,(PPDMDEVINS pDevIns, RTGCPHYS GCAddr, uint32_t uValue, uint32_t uTagSrc));
+     /** @copydoc PDMIOAPICREG::pfnSendMsi */
+    DECLRCCALLBACKMEMBER(VBOXSTRICTRC, pfnSetEoiRC,(PPDMDEVINS pDevIns, uint8_t u8Vector));
+} PDMIOAPIC;
+
+/** Maximum number of PCI busses for a VM. */
+#define PDM_PCI_BUSSES_MAX 8
+
+
+#ifdef IN_RING3
+/**
+ * PDM registered firmware device.
+ */
+typedef struct PDMFW
+{
+    /** Pointer to the firmware device instance. */
+    PPDMDEVINSR3                    pDevIns;
+    /** Copy of the registration structure. */
+    PDMFWREG                        Reg;
+} PDMFW;
+/** Pointer to a firmware instance. */
+typedef PDMFW *PPDMFW;
+#endif
+
+
+/**
+ * PDM PCI bus instance.
+ */
+typedef struct PDMPCIBUS
+{
+    /** PCI bus number. */
+    uint32_t                   iBus;
+    uint32_t                   uPadding0; /**< Alignment padding.*/
+
+    /** Pointer to PCI bus device instance. */
+    PPDMDEVINSR3               pDevInsR3;
+    /** @copydoc PDMPCIBUSREGR3::pfnSetIrqR3 */
+    DECLR3CALLBACKMEMBER(void, pfnSetIrqR3,(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, int iIrq, int iLevel, uint32_t uTagSrc));
+
+    /** @copydoc PDMPCIBUSREGR3::pfnRegisterR3 */
+    DECLR3CALLBACKMEMBER(int,  pfnRegister,(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, uint32_t fFlags,
+                                            uint8_t uPciDevNo, uint8_t uPciFunNo, const char *pszName));
+    /** @copydoc PDMPCIBUSREGR3::pfnRegisterMsiR3 */
+    DECLR3CALLBACKMEMBER(int,  pfnRegisterMsi,(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, PPDMMSIREG pMsiReg));
+    /** @copydoc PDMPCIBUSREGR3::pfnIORegionRegisterR3 */
+    DECLR3CALLBACKMEMBER(int,  pfnIORegionRegister,(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, uint32_t iRegion,
+                                                    RTGCPHYS cbRegion, PCIADDRESSSPACE enmType, uint32_t fFlags,
+                                                    uint64_t hHandle, PFNPCIIOREGIONMAP pfnCallback));
+    /** @copydoc PDMPCIBUSREGR3::pfnInterceptConfigAccesses */
+    DECLR3CALLBACKMEMBER(void, pfnInterceptConfigAccesses,(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
+                                                           PFNPCICONFIGREAD pfnRead, PFNPCICONFIGWRITE pfnWrite));
+    /** @copydoc PDMPCIBUSREGR3::pfnConfigWrite */
+    DECLR3CALLBACKMEMBER(VBOXSTRICTRC, pfnConfigWrite,(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
+                                                       uint32_t uAddress, unsigned cb, uint32_t u32Value));
+    /** @copydoc PDMPCIBUSREGR3::pfnConfigRead */
+    DECLR3CALLBACKMEMBER(VBOXSTRICTRC, pfnConfigRead,(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
+                                                      uint32_t uAddress, unsigned cb, uint32_t *pu32Value));
+} PDMPCIBUS;
+
+
+/**
+ * Ring-0 PDM PCI bus instance data.
+ */
+typedef struct PDMPCIBUSR0
+{
+    /** PCI bus number. */
+    uint32_t                   iBus;
+    uint32_t                   uPadding0; /**< Alignment padding.*/
+    /** Pointer to PCI bus device instance. */
+    PPDMDEVINSR0               pDevInsR0;
+    /** @copydoc PDMPCIBUSREGR0::pfnSetIrq */
+    DECLR0CALLBACKMEMBER(void, pfnSetIrqR0,(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, int iIrq, int iLevel, uint32_t uTagSrc));
+} PDMPCIBUSR0;
+/** Pointer to the ring-0 PCI bus data. */
+typedef PDMPCIBUSR0 *PPDMPCIBUSR0;
+
+#ifdef IN_RING3
+/**
+ * PDM registered DMAC (DMA Controller) device.
+ */
+typedef struct PDMDMAC
+{
+    /** Pointer to the DMAC device instance. */
+    PPDMDEVINSR3                    pDevIns;
+    /** Copy of the registration structure. */
+    PDMDMACREG                      Reg;
+} PDMDMAC;
+
+
+/**
+ * PDM registered RTC (Real Time Clock) device.
+ */
+typedef struct PDMRTC
+{
+    /** Pointer to the RTC device instance. */
+    PPDMDEVINSR3                    pDevIns;
+    /** Copy of the registration structure. */
+    PDMRTCREG                       Reg;
+} PDMRTC;
+
+#endif /* IN_RING3 */
+
+/**
+ * Module type.
+ */
+typedef enum PDMMODTYPE
+{
+    /** Raw-mode (RC) context module. */
+    PDMMOD_TYPE_RC,
+    /** Ring-0 (host) context module. */
+    PDMMOD_TYPE_R0,
+    /** Ring-3 (host) context module. */
+    PDMMOD_TYPE_R3
+} PDMMODTYPE;
+
+
+/** The module name length including the terminator. */
+#define PDMMOD_NAME_LEN             32
+
+/**
+ * Loaded module instance.
+ */
+typedef struct PDMMOD
+{
+    /** Module name. This is used for referring to
+     * the module internally, sort of like a handle. */
+    char                            szName[PDMMOD_NAME_LEN];
+    /** Module type. */
+    PDMMODTYPE                      eType;
+    /** Loader module handle. Not used for R0 modules. */
+    RTLDRMOD                        hLdrMod;
+    /** Loaded address.
+     * This is the 'handle' for R0 modules. */
+    RTUINTPTR                       ImageBase;
+    /** Old loaded address.
+     * This is used during relocation of GC modules. Not used for R0 modules. */
+    RTUINTPTR                       OldImageBase;
+    /** Where the R3 HC bits are stored.
+     * This can be equal to ImageBase but doesn't have to. Not used for R0 modules. */
+    void                           *pvBits;
+
+    /** Pointer to next module. */
+    struct PDMMOD                  *pNext;
+    /** Module filename. */
+    char                            szFilename[1];
+} PDMMOD;
+/** Pointer to loaded module instance. */
+typedef PDMMOD *PPDMMOD;
+
+
+
+/** Extra space in the free array. */
+#define PDMQUEUE_FREE_SLACK         16
+
+/**
+ * Queue type.
+ */
+typedef enum PDMQUEUETYPE
+{
+    /** Device consumer. */
+    PDMQUEUETYPE_DEV = 1,
+    /** Driver consumer. */
+    PDMQUEUETYPE_DRV,
+    /** Internal consumer. */
+    PDMQUEUETYPE_INTERNAL,
+    /** External consumer. */
+    PDMQUEUETYPE_EXTERNAL
+} PDMQUEUETYPE;
+
+/** Pointer to a PDM Queue. */
+typedef struct PDMQUEUE *PPDMQUEUE;
+
+/**
+ * PDM Queue.
+ */
+typedef struct PDMQUEUE
+{
+    /** Pointer to the next queue in the list. */
+    R3PTRTYPE(PPDMQUEUE)            pNext;
+    /** Type specific data. */
+    union
+    {
+        /** PDMQUEUETYPE_DEV */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMQUEUEDEV)   pfnCallback;
+            /** Pointer to the device instance owning the queue. */
+            R3PTRTYPE(PPDMDEVINS)       pDevIns;
+        } Dev;
+        /** PDMQUEUETYPE_DRV */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMQUEUEDRV)   pfnCallback;
+            /** Pointer to the driver instance owning the queue. */
+            R3PTRTYPE(PPDMDRVINS)       pDrvIns;
+        } Drv;
+        /** PDMQUEUETYPE_INTERNAL */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMQUEUEINT)   pfnCallback;
+        } Int;
+        /** PDMQUEUETYPE_EXTERNAL */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMQUEUEEXT)   pfnCallback;
+            /** Pointer to user argument. */
+            R3PTRTYPE(void *)           pvUser;
+        } Ext;
+    } u;
+    /** Queue type. */
+    PDMQUEUETYPE                    enmType;
+    /** The interval between checking the queue for events.
+     * The realtime timer below is used to do the waiting.
+     * If 0, the queue will use the VM_FF_PDM_QUEUE forced action. */
+    uint32_t                        cMilliesInterval;
+    /** Interval timer. Only used if cMilliesInterval is non-zero. */
+    PTMTIMERR3                      pTimer;
+    /** Pointer to the VM - R3. */
+    PVMR3                           pVMR3;
+    /** LIFO of pending items - R3. */
+    R3PTRTYPE(PPDMQUEUEITEMCORE) volatile pPendingR3;
+    /** Pointer to the VM - R0. */
+    PVMR0                           pVMR0;
+    /** LIFO of pending items - R0. */
+    R0PTRTYPE(PPDMQUEUEITEMCORE) volatile pPendingR0;
+    /** Pointer to the GC VM and indicator for GC enabled queue.
+     * If this is NULL, the queue cannot be used in GC.
+     */
+    PVMRC                           pVMRC;
+    /** LIFO of pending items - GC. */
+    RCPTRTYPE(PPDMQUEUEITEMCORE) volatile pPendingRC;
+
+    /** Item size (bytes). */
+    uint32_t                        cbItem;
+    /** Number of items in the queue. */
+    uint32_t                        cItems;
+    /** Index to the free head (where we insert). */
+    uint32_t volatile               iFreeHead;
+    /** Index to the free tail (where we remove). */
+    uint32_t volatile               iFreeTail;
+
+    /** Unique queue name. */
+    R3PTRTYPE(const char *)         pszName;
+#if HC_ARCH_BITS == 32
+    RTR3PTR                         Alignment1;
+#endif
+    /** Stat: Times PDMQueueAlloc fails. */
+    STAMCOUNTER                     StatAllocFailures;
+    /** Stat: PDMQueueInsert calls. */
+    STAMCOUNTER                     StatInsert;
+    /** Stat: Queue flushes. */
+    STAMCOUNTER                     StatFlush;
+    /** Stat: Queue flushes with pending items left over. */
+    STAMCOUNTER                     StatFlushLeftovers;
+#ifdef VBOX_WITH_STATISTICS
+    /** State: Profiling the flushing. */
+    STAMPROFILE                     StatFlushPrf;
+    /** State: Pending items. */
+    uint32_t volatile               cStatPending;
+    uint32_t volatile               cAlignment;
+#endif
+
+    /** Array of pointers to free items. Variable size. */
+    struct PDMQUEUEFREEITEM
+    {
+        /** Pointer to the free item - HC Ptr. */
+        R3PTRTYPE(PPDMQUEUEITEMCORE) volatile   pItemR3;
+        /** Pointer to the free item - HC Ptr. */
+        R0PTRTYPE(PPDMQUEUEITEMCORE) volatile   pItemR0;
+        /** Pointer to the free item - GC Ptr. */
+        RCPTRTYPE(PPDMQUEUEITEMCORE) volatile   pItemRC;
+#if HC_ARCH_BITS == 64
+        RTRCPTR                                 Alignment0;
+#endif
+    }                               aFreeItems[1];
+} PDMQUEUE;
+
+/** @name PDM::fQueueFlushing
+ * @{ */
+/** Used to make sure only one EMT will flush the queues.
+ * Set when an EMT is flushing queues, clear otherwise.  */
+#define PDM_QUEUE_FLUSH_FLAG_ACTIVE_BIT     0
+/** Indicating there are queues with items pending.
+ * This is make sure we don't miss inserts happening during flushing.  The FF
+ * cannot be used for this since it has to be cleared immediately to prevent
+ * other EMTs from spinning. */
+#define PDM_QUEUE_FLUSH_FLAG_PENDING_BIT    1
+/** @}  */
+
+
+/** @name PDM task structures.
+ * @{ */
+
+/**
+ * A asynchronous user mode task.
+ */
+typedef struct PDMTASK
+{
+    /** Task owner type. */
+    PDMTASKTYPE volatile        enmType;
+    /** Queue flags. */
+    uint32_t volatile           fFlags;
+    /** User argument for the callback. */
+    R3PTRTYPE(void *) volatile  pvUser;
+    /** The callback (will be cast according to enmType before callout). */
+    R3PTRTYPE(PFNRT) volatile   pfnCallback;
+    /** The owner identifier. */
+    R3PTRTYPE(void *) volatile  pvOwner;
+    /** Task name. */
+    R3PTRTYPE(const char *)     pszName;
+    /** Number of times already triggered when PDMTaskTrigger was called. */
+    uint32_t volatile           cAlreadyTrigged;
+    /** Number of runs. */
+    uint32_t                    cRuns;
+} PDMTASK;
+/** Pointer to a PDM task. */
+typedef PDMTASK *PPDMTASK;
+
+/**
+ * A task set.
+ *
+ * This is served by one task executor thread.
+ */
+typedef struct PDMTASKSET
+{
+    /** Magic value (PDMTASKSET_MAGIC). */
+    uint32_t                u32Magic;
+    /** Set if this task set works for ring-0 and raw-mode. */
+    bool                    fRZEnabled;
+    /** Number of allocated taks. */
+    uint8_t volatile        cAllocated;
+    /** Base handle value for this set. */
+    uint16_t                uHandleBase;
+    /** The task executor thread. */
+    R3PTRTYPE(RTTHREAD)     hThread;
+    /** Event semaphore for waking up the thread when fRZEnabled is set. */
+    SUPSEMEVENT             hEventR0;
+    /** Event semaphore for waking up the thread when fRZEnabled is clear. */
+    R3PTRTYPE(RTSEMEVENT)   hEventR3;
+    /** The VM pointer.   */
+    PVM                     pVM;
+    /** Padding so fTriggered is in its own cacheline. */
+    uint64_t                au64Padding2[3];
+
+    /** Bitmask of triggered tasks. */
+    uint64_t volatile       fTriggered;
+    /** Shutdown thread indicator. */
+    bool volatile           fShutdown;
+    /** Padding.   */
+    bool volatile           afPadding3[3];
+    /** Task currently running, UINT32_MAX if idle. */
+    uint32_t volatile       idxRunning;
+    /** Padding so fTriggered and fShutdown are in their own cacheline. */
+    uint64_t volatile       au64Padding3[6];
+
+    /** The individual tasks.  (Unallocated tasks have NULL pvOwner.)  */
+    PDMTASK                 aTasks[64];
+} PDMTASKSET;
+AssertCompileMemberAlignment(PDMTASKSET, fTriggered, 64);
+AssertCompileMemberAlignment(PDMTASKSET, aTasks, 64);
+/** Magic value for PDMTASKSET::u32Magic. */
+#define PDMTASKSET_MAGIC        UINT32_C(0x19320314)
+/** Pointer to a task set. */
+typedef PDMTASKSET *PPDMTASKSET;
+
+/** @} */
+
+
+/**
+ * Queue device helper task operation.
+ */
+typedef enum PDMDEVHLPTASKOP
+{
+    /** The usual invalid 0 entry. */
+    PDMDEVHLPTASKOP_INVALID = 0,
+    /** ISASetIrq */
+    PDMDEVHLPTASKOP_ISA_SET_IRQ,
+    /** PCISetIrq */
+    PDMDEVHLPTASKOP_PCI_SET_IRQ,
+    /** PCISetIrq */
+    PDMDEVHLPTASKOP_IOAPIC_SET_IRQ,
+    /** The usual 32-bit hack. */
+    PDMDEVHLPTASKOP_32BIT_HACK = 0x7fffffff
+} PDMDEVHLPTASKOP;
+
+/**
+ * Queued Device Helper Task.
+ */
+typedef struct PDMDEVHLPTASK
+{
+    /** The queue item core (don't touch). */
+    PDMQUEUEITEMCORE                Core;
+    /** Pointer to the device instance (R3 Ptr). */
+    PPDMDEVINSR3                    pDevInsR3;
+    /** This operation to perform. */
+    PDMDEVHLPTASKOP                 enmOp;
+#if HC_ARCH_BITS == 64
+    uint32_t                        Alignment0;
+#endif
+    /** Parameters to the operation. */
+    union PDMDEVHLPTASKPARAMS
+    {
+        /**
+         * PDMDEVHLPTASKOP_ISA_SET_IRQ and PDMDEVHLPTASKOP_IOAPIC_SET_IRQ.
+         */
+        struct PDMDEVHLPTASKISASETIRQ
+        {
+            /** The IRQ */
+            int                     iIrq;
+            /** The new level. */
+            int                     iLevel;
+            /** The IRQ tag and source. */
+            uint32_t                uTagSrc;
+        } IsaSetIRQ, IoApicSetIRQ;
+
+        /**
+         * PDMDEVHLPTASKOP_PCI_SET_IRQ
+         */
+        struct PDMDEVHLPTASKPCISETIRQ
+        {
+            /** Pointer to the PCI device (R3 Ptr). */
+            R3PTRTYPE(PPDMPCIDEV)   pPciDevR3;
+            /** The IRQ */
+            int                     iIrq;
+            /** The new level. */
+            int                     iLevel;
+            /** The IRQ tag and source. */
+            uint32_t                uTagSrc;
+        } PciSetIRQ;
+
+        /** Expanding the structure. */
+        uint64_t    au64[3];
+    } u;
+} PDMDEVHLPTASK;
+/** Pointer to a queued Device Helper Task. */
+typedef PDMDEVHLPTASK *PPDMDEVHLPTASK;
+/** Pointer to a const queued Device Helper Task. */
+typedef const PDMDEVHLPTASK *PCPDMDEVHLPTASK;
+
+
+
+/**
+ * An USB hub registration record.
+ */
+typedef struct PDMUSBHUB
+{
+    /** The USB versions this hub support.
+     * Note that 1.1 hubs can take on 2.0 devices. */
+    uint32_t                        fVersions;
+    /** The number of ports on the hub. */
+    uint32_t                        cPorts;
+    /** The number of available ports (0..cPorts). */
+    uint32_t                        cAvailablePorts;
+    /** The driver instance of the hub. */
+    PPDMDRVINS                      pDrvIns;
+    /** Copy of the to the registration structure. */
+    PDMUSBHUBREG                    Reg;
+
+    /** Pointer to the next hub in the list. */
+    struct PDMUSBHUB               *pNext;
+} PDMUSBHUB;
+
+/** Pointer to a const USB HUB registration record. */
+typedef const PDMUSBHUB *PCPDMUSBHUB;
+
+/** Pointer to a PDM Async I/O template. */
+typedef struct PDMASYNCCOMPLETIONTEMPLATE *PPDMASYNCCOMPLETIONTEMPLATE;
+
+/** Pointer to the main PDM Async completion endpoint class. */
+typedef struct PDMASYNCCOMPLETIONEPCLASS *PPDMASYNCCOMPLETIONEPCLASS;
+
+/** Pointer to the global block cache structure. */
+typedef struct PDMBLKCACHEGLOBAL *PPDMBLKCACHEGLOBAL;
+
+/**
+ * PDM VMCPU Instance data.
+ * Changes to this must checked against the padding of the pdm union in VMCPU!
+ */
+typedef struct PDMCPU
+{
+    /** The number of entries in the apQueuedCritSectsLeaves table that's currently
+     * in use. */
+    uint32_t                        cQueuedCritSectLeaves;
+    uint32_t                        uPadding0; /**< Alignment padding.*/
+    /** Critical sections queued in RC/R0 because of contention preventing leave to
+     * complete. (R3 Ptrs)
+     * We will return to Ring-3 ASAP, so this queue doesn't have to be very long. */
+    R3PTRTYPE(PPDMCRITSECT)         apQueuedCritSectLeaves[8];
+
+    /** The number of entries in the apQueuedCritSectRwExclLeaves table that's
+     * currently in use. */
+    uint32_t                        cQueuedCritSectRwExclLeaves;
+    uint32_t                        uPadding1; /**< Alignment padding.*/
+    /** Read/write critical sections queued in RC/R0 because of contention
+     * preventing exclusive leave to complete. (R3 Ptrs)
+     * We will return to Ring-3 ASAP, so this queue doesn't have to be very long. */
+    R3PTRTYPE(PPDMCRITSECTRW)       apQueuedCritSectRwExclLeaves[8];
+
+    /** The number of entries in the apQueuedCritSectsRwShrdLeaves table that's
+     * currently in use. */
+    uint32_t                        cQueuedCritSectRwShrdLeaves;
+    uint32_t                        uPadding2; /**< Alignment padding.*/
+    /** Read/write critical sections queued in RC/R0 because of contention
+     * preventing shared leave to complete. (R3 Ptrs)
+     * We will return to Ring-3 ASAP, so this queue doesn't have to be very long. */
+    R3PTRTYPE(PPDMCRITSECTRW)       apQueuedCritSectRwShrdLeaves[8];
+} PDMCPU;
+
+
+/**
+ * PDM VM Instance data.
+ * Changes to this must checked against the padding of the cfgm union in VM!
+ */
+typedef struct PDM
+{
+    /** The PDM lock.
+     * This is used to protect everything that deals with interrupts, i.e.
+     * the PIC, APIC, IOAPIC and PCI devices plus some PDM functions. */
+    PDMCRITSECT                     CritSect;
+    /** The NOP critical section.
+     * This is a dummy critical section that will not do any thread
+     * serialization but instead let all threads enter immediately and
+     * concurrently. */
+    PDMCRITSECT                     NopCritSect;
+
+    /** The ring-0 capable task sets (max 128). */
+    PDMTASKSET                      aTaskSets[2];
+    /** Pointer to task sets (max 512). */
+    R3PTRTYPE(PPDMTASKSET)          apTaskSets[8];
+
+    /** PCI Buses. */
+    PDMPCIBUS                       aPciBuses[PDM_PCI_BUSSES_MAX];
+    /** The register PIC device. */
+    PDMPIC                          Pic;
+    /** The registered APIC device. */
+    PDMAPIC                         Apic;
+    /** The registered I/O APIC device. */
+    PDMIOAPIC                       IoApic;
+    /** The registered HPET device. */
+    PPDMDEVINSR3                    pHpet;
+
+    /** List of registered devices. (FIFO) */
+    R3PTRTYPE(PPDMDEV)              pDevs;
+    /** List of devices instances. (FIFO) */
+    R3PTRTYPE(PPDMDEVINS)           pDevInstances;
+    /** List of registered USB devices. (FIFO) */
+    R3PTRTYPE(PPDMUSB)              pUsbDevs;
+    /** List of USB devices instances. (FIFO) */
+    R3PTRTYPE(PPDMUSBINS)           pUsbInstances;
+    /** List of registered drivers. (FIFO) */
+    R3PTRTYPE(PPDMDRV)              pDrvs;
+    /** The registered firmware device (can be NULL). */
+    R3PTRTYPE(PPDMFW)               pFirmware;
+    /** The registered DMAC device. */
+    R3PTRTYPE(PPDMDMAC)             pDmac;
+    /** The registered RTC device. */
+    R3PTRTYPE(PPDMRTC)              pRtc;
+    /** The registered USB HUBs. (FIFO) */
+    R3PTRTYPE(PPDMUSBHUB)           pUsbHubs;
+
+    /** @name Queues
+     * @{ */
+    /** Queue in which devhlp tasks are queued for R3 execution - R3 Ptr. */
+    R3PTRTYPE(PPDMQUEUE)            pDevHlpQueueR3;
+    /** Queue in which devhlp tasks are queued for R3 execution - R0 Ptr. */
+    R0PTRTYPE(PPDMQUEUE)            pDevHlpQueueR0;
+    /** Queue in which devhlp tasks are queued for R3 execution - RC Ptr. */
+    RCPTRTYPE(PPDMQUEUE)            pDevHlpQueueRC;
+    /** Pointer to the queue which should be manually flushed - RC Ptr.
+     * Only touched by EMT. */
+    RCPTRTYPE(struct PDMQUEUE *)    pQueueFlushRC;
+    /** Pointer to the queue which should be manually flushed - R0 Ptr.
+     * Only touched by EMT. */
+    R0PTRTYPE(struct PDMQUEUE *)    pQueueFlushR0;
+    /** Bitmask controlling the queue flushing.
+     * See PDM_QUEUE_FLUSH_FLAG_ACTIVE and PDM_QUEUE_FLUSH_FLAG_PENDING. */
+    uint32_t volatile               fQueueFlushing;
+    /** @} */
+
+    /** The current IRQ tag (tracing purposes). */
+    uint32_t volatile               uIrqTag;
+
+    /** Pending reset flags (PDMVMRESET_F_XXX). */
+    uint32_t volatile               fResetFlags;
+
+    /** Set by pdmR3LoadExec for use in assertions. */
+    bool                            fStateLoaded;
+    /** Alignment padding. */
+    bool                            afPadding[3];
+
+    /** The tracing ID of the next device instance.
+     *
+     * @remarks We keep the device tracing ID seperate from the rest as these are
+     *          then more likely to end up with the same ID from one run to
+     *          another, making analysis somewhat easier.  Drivers and USB devices
+     *          are more volatile and can be changed at runtime, thus these are much
+     *          less likely to remain stable, so just heap them all together. */
+    uint32_t                        idTracingDev;
+    /** The tracing ID of the next driver instance, USB device instance or other
+     * PDM entity requiring an ID. */
+    uint32_t                        idTracingOther;
+
+    /** @name   VMM device heap
+     * @{ */
+    /** The heap size. */
+    uint32_t                        cbVMMDevHeap;
+    /** Free space. */
+    uint32_t                        cbVMMDevHeapLeft;
+    /** Pointer to the heap base (MMIO2 ring-3 mapping). NULL if not registered. */
+    RTR3PTR                         pvVMMDevHeap;
+    /** Ring-3 mapping/unmapping notification callback for the user. */
+    PFNPDMVMMDEVHEAPNOTIFY          pfnVMMDevHeapNotify;
+    /** The current mapping. NIL_RTGCPHYS if not mapped or registered. */
+    RTGCPHYS                        GCPhysVMMDevHeap;
+    /** @} */
+
+    /** Number of times a critical section leave request needed to be queued for ring-3 execution. */
+    STAMCOUNTER                     StatQueuedCritSectLeaves;
+} PDM;
+AssertCompileMemberAlignment(PDM, CritSect, 8);
+AssertCompileMemberAlignment(PDM, aTaskSets, 64);
+AssertCompileMemberAlignment(PDM, StatQueuedCritSectLeaves, 8);
+AssertCompileMemberAlignment(PDM, GCPhysVMMDevHeap, sizeof(RTGCPHYS));
+/** Pointer to PDM VM instance data. */
+typedef PDM *PPDM;
+
+
+/**
+ * PDM data kept in the ring-0 GVM.
+ */
+typedef struct PDMR0PERVM
+{
+    /** PCI Buses, ring-0 data. */
+    PDMPCIBUSR0                     aPciBuses[PDM_PCI_BUSSES_MAX];
+    /** Number of valid ring-0 device instances (apDevInstances). */
+    uint32_t                        cDevInstances;
+    uint32_t                        u32Padding;
+    /** Pointer to ring-0 device instances. */
+    R0PTRTYPE(struct PDMDEVINSR0 *) apDevInstances[190];
+} PDMR0PERVM;
+
+
+/**
+ * PDM data kept in the UVM.
+ */
+typedef struct PDMUSERPERVM
+{
+    /** @todo move more stuff over here. */
+
+    /** Linked list of timer driven PDM queues.
+     * Currently serialized by PDM::CritSect.  */
+    R3PTRTYPE(struct PDMQUEUE *)    pQueuesTimer;
+    /** Linked list of force action driven PDM queues.
+     * Currently serialized by PDM::CritSect. */
+    R3PTRTYPE(struct PDMQUEUE *)    pQueuesForced;
+
+    /** Lock protecting the lists below it. */
+    RTCRITSECT                      ListCritSect;
+    /** Pointer to list of loaded modules. */
+    PPDMMOD                         pModules;
+    /** List of initialized critical sections. (LIFO) */
+    R3PTRTYPE(PPDMCRITSECTINT)      pCritSects;
+    /** List of initialized read/write critical sections. (LIFO) */
+    R3PTRTYPE(PPDMCRITSECTRWINT)    pRwCritSects;
+    /** Head of the PDM Thread list. (singly linked) */
+    R3PTRTYPE(PPDMTHREAD)           pThreads;
+    /** Tail of the PDM Thread list. (singly linked) */
+    R3PTRTYPE(PPDMTHREAD)           pThreadsTail;
+
+    /** @name   PDM Async Completion
+     * @{ */
+    /** Pointer to the array of supported endpoint classes. */
+    PPDMASYNCCOMPLETIONEPCLASS      apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_MAX];
+    /** Head of the templates. Singly linked, protected by ListCritSect. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONTEMPLATE) pAsyncCompletionTemplates;
+    /** @} */
+
+    /** Global block cache data. */
+    R3PTRTYPE(PPDMBLKCACHEGLOBAL)   pBlkCacheGlobal;
+#ifdef VBOX_WITH_NETSHAPER
+    /** Pointer to network shaper instance. */
+    R3PTRTYPE(PPDMNETSHAPER)        pNetShaper;
+#endif /* VBOX_WITH_NETSHAPER */
+
+} PDMUSERPERVM;
+/** Pointer to the PDM data kept in the UVM. */
+typedef PDMUSERPERVM *PPDMUSERPERVM;
+
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+#ifdef IN_RING3
+extern const PDMDRVHLPR3    g_pdmR3DrvHlp;
+extern const PDMDEVHLPR3    g_pdmR3DevHlpTrusted;
+extern const PDMDEVHLPR3    g_pdmR3DevHlpUnTrusted;
+extern const PDMPICHLP      g_pdmR3DevPicHlp;
+extern const PDMIOAPICHLP   g_pdmR3DevIoApicHlp;
+extern const PDMFWHLPR3     g_pdmR3DevFirmwareHlp;
+extern const PDMPCIHLPR3    g_pdmR3DevPciHlp;
+extern const PDMDMACHLP     g_pdmR3DevDmacHlp;
+extern const PDMRTCHLP      g_pdmR3DevRtcHlp;
+extern const PDMHPETHLPR3   g_pdmR3DevHpetHlp;
+extern const PDMPCIRAWHLPR3 g_pdmR3DevPciRawHlp;
+#endif
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** @def PDMDEV_ASSERT_DEVINS
+ * Asserts the validity of the device instance.
+ */
+#ifdef VBOX_STRICT
+# define PDMDEV_ASSERT_DEVINS(pDevIns)   \
+    do { \
+        AssertPtr(pDevIns); \
+        Assert(pDevIns->u32Version == PDM_DEVINS_VERSION); \
+        Assert(pDevIns->CTX_SUFF(pvInstanceDataFor) == (void *)&pDevIns->achInstanceData[0]); \
+    } while (0)
+#else
+# define PDMDEV_ASSERT_DEVINS(pDevIns)   do { } while (0)
+#endif
+
+/** @def PDMDRV_ASSERT_DRVINS
+ * Asserts the validity of the driver instance.
+ */
+#ifdef VBOX_STRICT
+# define PDMDRV_ASSERT_DRVINS(pDrvIns) \
+    do { \
+        AssertPtr(pDrvIns); \
+        Assert(pDrvIns->u32Version == PDM_DRVINS_VERSION); \
+        Assert(pDrvIns->CTX_SUFF(pvInstanceData) == (void *)&pDrvIns->achInstanceData[0]); \
+    } while (0)
+#else
+# define PDMDRV_ASSERT_DRVINS(pDrvIns)   do { } while (0)
+#endif
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+#ifdef IN_RING3
+bool        pdmR3IsValidName(const char *pszName);
+
+int         pdmR3CritSectBothInitStats(PVM pVM);
+void        pdmR3CritSectBothRelocate(PVM pVM);
+int         pdmR3CritSectBothDeleteDevice(PVM pVM, PPDMDEVINS pDevIns);
+int         pdmR3CritSectBothDeleteDriver(PVM pVM, PPDMDRVINS pDrvIns);
+int         pdmR3CritSectInitDevice(        PVM pVM, PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
+                                            const char *pszNameFmt, va_list va);
+int         pdmR3CritSectInitDeviceAuto(    PVM pVM, PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
+                                            const char *pszNameFmt, ...);
+int         pdmR3CritSectInitDriver(        PVM pVM, PPDMDRVINS pDrvIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
+                                            const char *pszNameFmt, ...);
+int         pdmR3CritSectRwInitDevice(      PVM pVM, PPDMDEVINS pDevIns, PPDMCRITSECTRW pCritSect, RT_SRC_POS_DECL,
+                                            const char *pszNameFmt, va_list va);
+int         pdmR3CritSectRwInitDeviceAuto(  PVM pVM, PPDMDEVINS pDevIns, PPDMCRITSECTRW pCritSect, RT_SRC_POS_DECL,
+                                            const char *pszNameFmt, ...);
+int         pdmR3CritSectRwInitDriver(      PVM pVM, PPDMDRVINS pDrvIns, PPDMCRITSECTRW pCritSect, RT_SRC_POS_DECL,
+                                            const char *pszNameFmt, ...);
+
+int         pdmR3DevInit(PVM pVM);
+int         pdmR3DevInitComplete(PVM pVM);
+PPDMDEV     pdmR3DevLookup(PVM pVM, const char *pszName);
+int         pdmR3DevFindLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPDMLUN *ppLun);
+DECLCALLBACK(bool) pdmR3DevHlpQueueConsumer(PVM pVM, PPDMQUEUEITEMCORE pItem);
+
+int         pdmR3UsbLoadModules(PVM pVM);
+int         pdmR3UsbInstantiateDevices(PVM pVM);
+PPDMUSB     pdmR3UsbLookup(PVM pVM, const char *pszName);
+int         pdmR3UsbRegisterHub(PVM pVM, PPDMDRVINS pDrvIns, uint32_t fVersions, uint32_t cPorts, PCPDMUSBHUBREG pUsbHubReg, PPCPDMUSBHUBHLP ppUsbHubHlp);
+int         pdmR3UsbVMInitComplete(PVM pVM);
+
+int         pdmR3DrvInit(PVM pVM);
+int         pdmR3DrvInstantiate(PVM pVM, PCFGMNODE pNode, PPDMIBASE pBaseInterface, PPDMDRVINS pDrvAbove,
+                                PPDMLUN pLun, PPDMIBASE *ppBaseInterface);
+int         pdmR3DrvDetach(PPDMDRVINS pDrvIns, uint32_t fFlags);
+void        pdmR3DrvDestroyChain(PPDMDRVINS pDrvIns, uint32_t fFlags);
+PPDMDRV     pdmR3DrvLookup(PVM pVM, const char *pszName);
+
+int         pdmR3LdrInitU(PUVM pUVM);
+void        pdmR3LdrTermU(PUVM pUVM);
+char       *pdmR3FileR3(const char *pszFile, bool fShared);
+int         pdmR3LoadR3U(PUVM pUVM, const char *pszFilename, const char *pszName);
+
+void        pdmR3QueueRelocate(PVM pVM, RTGCINTPTR offDelta);
+
+int         pdmR3TaskInit(PVM pVM);
+void        pdmR3TaskTerm(PVM pVM);
+
+int         pdmR3ThreadCreateDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDEV pfnThread,
+                                    PFNPDMTHREADWAKEUPDEV pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName);
+int         pdmR3ThreadCreateUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADUSB pfnThread,
+                                 PFNPDMTHREADWAKEUPUSB pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName);
+int         pdmR3ThreadCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDRV pfnThread,
+                                    PFNPDMTHREADWAKEUPDRV pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName);
+int         pdmR3ThreadDestroyDevice(PVM pVM, PPDMDEVINS pDevIns);
+int         pdmR3ThreadDestroyUsb(PVM pVM, PPDMUSBINS pUsbIns);
+int         pdmR3ThreadDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns);
+void        pdmR3ThreadDestroyAll(PVM pVM);
+int         pdmR3ThreadResumeAll(PVM pVM);
+int         pdmR3ThreadSuspendAll(PVM pVM);
+
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+int         pdmR3AsyncCompletionInit(PVM pVM);
+int         pdmR3AsyncCompletionTerm(PVM pVM);
+void        pdmR3AsyncCompletionResume(PVM pVM);
+int         pdmR3AsyncCompletionTemplateCreateDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, PFNPDMASYNCCOMPLETEDEV pfnCompleted, const char *pszDesc);
+int         pdmR3AsyncCompletionTemplateCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate,
+                                                     PFNPDMASYNCCOMPLETEDRV pfnCompleted, void *pvTemplateUser, const char *pszDesc);
+int         pdmR3AsyncCompletionTemplateCreateUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, PFNPDMASYNCCOMPLETEUSB pfnCompleted, const char *pszDesc);
+int         pdmR3AsyncCompletionTemplateDestroyDevice(PVM pVM, PPDMDEVINS pDevIns);
+int         pdmR3AsyncCompletionTemplateDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns);
+int         pdmR3AsyncCompletionTemplateDestroyUsb(PVM pVM, PPDMUSBINS pUsbIns);
+#endif
+
+#ifdef VBOX_WITH_NETSHAPER
+int         pdmR3NetShaperInit(PVM pVM);
+int         pdmR3NetShaperTerm(PVM pVM);
+#endif
+
+int         pdmR3BlkCacheInit(PVM pVM);
+void        pdmR3BlkCacheTerm(PVM pVM);
+int         pdmR3BlkCacheResume(PVM pVM);
+
+#endif /* IN_RING3 */
+
+void        pdmLock(PVMCC pVM);
+int         pdmLockEx(PVMCC pVM, int rc);
+void        pdmUnlock(PVMCC pVM);
+
+#if defined(IN_RING3) || defined(IN_RING0)
+void        pdmCritSectRwLeaveSharedQueued(PPDMCRITSECTRW pThis);
+void        pdmCritSectRwLeaveExclQueued(PPDMCRITSECTRW pThis);
+#endif
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_PDMInternal_h */
+
diff --git a/src/VBox/VMM/include/PDMNetShaperInternal.h b/src/VBox/VMM/include/PDMNetShaperInternal.h
new file mode 100644
index 00000000..3f5531dd
--- /dev/null
+++ b/src/VBox/VMM/include/PDMNetShaperInternal.h
@@ -0,0 +1,54 @@
+/* $Id: PDMNetShaperInternal.h $ */
+/** @file
+ * PDM Network Shaper - Internal data structures and functions common for both R0 and R3 parts.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_PDMNetShaperInternal_h
+#define VMM_INCLUDED_SRC_include_PDMNetShaperInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+/**
+ * Bandwidth group instance data
+ */
+typedef struct PDMNSBWGROUP
+{
+    /** Pointer to the next group in the list. */
+    R3PTRTYPE(struct PDMNSBWGROUP *)            pNextR3;
+    /** Pointer to the shared UVM structure. */
+    R3PTRTYPE(struct PDMNETSHAPER *)            pShaperR3;
+    /** Critical section protecting all members below. */
+    PDMCRITSECT                                 Lock;
+    /** Pointer to the first filter attached to this group. */
+    R3PTRTYPE(struct PDMNSFILTER *)             pFiltersHeadR3;
+    /** Bandwidth group name. */
+    R3PTRTYPE(char *)                           pszNameR3;
+    /** Maximum number of bytes filters are allowed to transfer. */
+    volatile uint64_t                           cbPerSecMax;
+    /** Number of bytes we are allowed to transfer in one burst. */
+    volatile uint32_t                           cbBucket;
+    /** Number of bytes we were allowed to transfer at the last update. */
+    volatile uint32_t                           cbTokensLast;
+    /** Timestamp of the last update */
+    volatile uint64_t                           tsUpdatedLast;
+    /** Reference counter - How many filters are associated with this group. */
+    volatile uint32_t                           cRefs;
+} PDMNSBWGROUP;
+/** Pointer to a bandwidth group. */
+typedef PDMNSBWGROUP *PPDMNSBWGROUP;
+
+#endif /* !VMM_INCLUDED_SRC_include_PDMNetShaperInternal_h */
+
diff --git a/src/VBox/VMM/include/PGMGstDefs.h b/src/VBox/VMM/include/PGMGstDefs.h
new file mode 100644
index 00000000..85c4cd52
--- /dev/null
+++ b/src/VBox/VMM/include/PGMGstDefs.h
@@ -0,0 +1,231 @@
+/* $Id: PGMGstDefs.h $ */
+/** @file
+ * VBox - Page Manager, Guest Paging Template - All context code.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+#undef GSTPT
+#undef PGSTPT
+#undef GSTPTE
+#undef PGSTPTE
+#undef GSTPD
+#undef PGSTPD
+#undef GSTPDE
+#undef PGSTPDE
+#undef GSTPTWALK
+#undef PGSTPTWALK
+#undef PCGSTPTWALK
+#undef GST_BIG_PAGE_SIZE
+#undef GST_BIG_PAGE_OFFSET_MASK
+#undef GST_PDE_PG_MASK
+#undef GST_PDE_BIG_PG_MASK
+#undef GST_PD_SHIFT
+#undef GST_PD_MASK
+#undef GST_PTE_PG_MASK
+#undef GST_GET_PTE_SHW_FLAGS
+#undef GST_PT_SHIFT
+#undef GST_PT_MASK
+#undef GST_TOTAL_PD_ENTRIES
+#undef GST_CR3_PAGE_MASK
+#undef GST_PDPE_ENTRIES
+#undef GST_PDPT_SHIFT
+#undef GST_PDPT_MASK
+#undef GST_PDPE_PG_MASK
+#undef GST_GET_PTE_GCPHYS
+#undef GST_GET_PDE_GCPHYS
+#undef GST_GET_BIG_PDE_GCPHYS
+#undef GST_GET_PDE_SHW_FLAGS
+#undef GST_GET_BIG_PDE_SHW_FLAGS
+#undef GST_GET_BIG_PDE_SHW_FLAGS_4_PTE
+#undef GST_IS_PTE_VALID
+#undef GST_IS_PDE_VALID
+#undef GST_IS_BIG_PDE_VALID
+#undef GST_IS_PDPE_VALID
+#undef GST_IS_BIG_PDPE_VALID
+#undef GST_IS_PML4E_VALID
+#undef GST_IS_PSE_ACTIVE
+#undef GST_IS_NX_ACTIVE
+#undef BTH_IS_NP_ACTIVE
+
+#if PGM_GST_TYPE == PGM_TYPE_REAL \
+ || PGM_GST_TYPE == PGM_TYPE_PROT
+
+# if PGM_SHW_TYPE == PGM_TYPE_EPT
+#  define GSTPT                                 X86PTPAE
+#  define PGSTPT                                PX86PTPAE
+#  define GSTPTE                                X86PTEPAE
+#  define PGSTPTE                               PX86PTEPAE
+#  define GSTPD                                 X86PDPAE
+#  define PGSTPD                                PX86PDPAE
+#  define GSTPDE                                X86PDEPAE
+#  define PGSTPDE                               PX86PDEPAE
+#  define GST_PTE_PG_MASK                       X86_PTE_PAE_PG_MASK
+#  define GST_IS_NX_ACTIVE(pVCpu)               (true && This_should_perhaps_not_be_used_in_this_context)
+#  define BTH_IS_NP_ACTIVE(pVM)                 (true)
+# else
+#  if PGM_SHW_TYPE == PGM_TYPE_32BIT /* Same as shadow paging, but no PGMSHWPTEPAE. */
+#   define GSTPT                                X86PT
+#   define PGSTPT                               PX86PT
+#   define GSTPTE                               X86PTE
+#   define PGSTPTE                              PX86PTE
+#   define GSTPD                                X86PD
+#   define PGSTPD                               PX86PD
+#   define GSTPDE                               X86PDE
+#   define PGSTPDE                              PX86PDE
+#   define GST_PTE_PG_MASK                      X86_PTE_PG_MASK
+#  else
+#   define GSTPT                                X86PTPAE
+#   define PGSTPT                               PX86PTPAE
+#   define GSTPTE                               X86PTEPAE
+#   define PGSTPTE                              PX86PTEPAE
+#   define GSTPD                                X86PDPAE
+#   define PGSTPD                               PX86PDPAE
+#   define GSTPDE                               X86PDEPAE
+#   define PGSTPDE                              PX86PDEPAE
+#   define GST_PTE_PG_MASK                      X86_PTE_PAE_PG_MASK
+#  endif
+#  define GST_IS_NX_ACTIVE(pVCpu)               (pgmGstIsNoExecuteActive(pVCpu))
+#  if PGM_GST_TYPE == PGM_TYPE_PROT             /* (comment at top of PGMAllBth.h) */
+#   define BTH_IS_NP_ACTIVE(pVM)                (pVM->pgm.s.fNestedPaging)
+#  else
+#   define BTH_IS_NP_ACTIVE(pVM)                (false)
+#  endif
+# endif
+# define GST_GET_PTE_GCPHYS(Pte)                PGM_A20_APPLY(pVCpu, ((Pte).u & GST_PTE_PG_MASK))
+# define GST_GET_PDE_GCPHYS(Pde)                (true && This_should_perhaps_not_be_used_in_this_context) //??
+# define GST_GET_BIG_PDE_GCPHYS(Pde)            (true && This_should_perhaps_not_be_used_in_this_context) //??
+# define GST_GET_PTE_SHW_FLAGS(pVCpu, Pte)      ((Pte).u & (X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G)) /**< @todo Could return P|RW|US|A|D here without consulting the PTE. */
+# define GST_GET_PDE_SHW_FLAGS(pVCpu, Pde)      (true && This_should_perhaps_not_be_used_in_this_context) //??
+# define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde)  (true && This_should_perhaps_not_be_used_in_this_context) //??
+# define GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, Pde) (true && This_should_perhaps_not_be_used_in_this_context) //??
+# define GST_IS_PTE_VALID(pVCpu, Pte)           (true)
+# define GST_IS_PDE_VALID(pVCpu, Pde)           (true)
+# define GST_IS_BIG_PDE_VALID(pVCpu, Pde)       (true)
+# define GST_IS_PDPE_VALID(pVCpu, Pdpe)         (true)
+# define GST_IS_BIG_PDPE_VALID(pVCpu, Pdpe)     (true)
+# define GST_IS_PML4E_VALID(pVCpu, Pml4e)       (true)
+# define GST_IS_PSE_ACTIVE(pVCpu)               (false && This_should_not_be_used_in_this_context)
+
+#elif PGM_GST_TYPE == PGM_TYPE_32BIT
+# define GSTPT                                  X86PT
+# define PGSTPT                                 PX86PT
+# define GSTPTE                                 X86PTE
+# define PGSTPTE                                PX86PTE
+# define GSTPD                                  X86PD
+# define PGSTPD                                 PX86PD
+# define GSTPDE                                 X86PDE
+# define PGSTPDE                                PX86PDE
+# define GSTPTWALK                              PGMPTWALKGST32BIT
+# define PGSTPTWALK                             PPGMPTWALKGST32BIT
+# define PCGSTPTWALK                            PCPGMPTWALKGST32BIT
+# define GST_BIG_PAGE_SIZE                      X86_PAGE_4M_SIZE
+# define GST_BIG_PAGE_OFFSET_MASK               X86_PAGE_4M_OFFSET_MASK
+# define GST_PDE_PG_MASK                        X86_PDE_PG_MASK
+# define GST_PDE_BIG_PG_MASK                    X86_PDE4M_PG_MASK
+# define GST_GET_PTE_GCPHYS(Pte)                PGM_A20_APPLY(pVCpu, ((Pte).u & GST_PDE_PG_MASK))
+# define GST_GET_PDE_GCPHYS(Pde)                PGM_A20_APPLY(pVCpu, ((Pde).u & GST_PDE_PG_MASK))
+# define GST_GET_BIG_PDE_GCPHYS(pVM, Pde)       PGM_A20_APPLY(pVCpu, pgmGstGet4MBPhysPage((pVM), Pde))
+# define GST_GET_PDE_SHW_FLAGS(pVCpu, Pde)      ((Pde).u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A))
+# define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde) \
+    ( ((Pde).u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A)) | PGM_PDFLAGS_BIG_PAGE )
+# define GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, Pde) \
+    ((Pde).u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A | X86_PDE4M_D | X86_PDE4M_G))
+# define GST_PD_SHIFT                           X86_PD_SHIFT
+# define GST_PD_MASK                            X86_PD_MASK
+# define GST_TOTAL_PD_ENTRIES                   X86_PG_ENTRIES
+# define GST_PTE_PG_MASK                        X86_PTE_PG_MASK
+# define GST_GET_PTE_SHW_FLAGS(pVCpu, Pte)      ((Pte).u & (X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G))
+# define GST_PT_SHIFT                           X86_PT_SHIFT
+# define GST_PT_MASK                            X86_PT_MASK
+# define GST_CR3_PAGE_MASK                      X86_CR3_PAGE_MASK
+# define GST_IS_PTE_VALID(pVCpu, Pte)           (true)
+# define GST_IS_PDE_VALID(pVCpu, Pde)           (true)
+# define GST_IS_BIG_PDE_VALID(pVCpu, Pde)       (!( (Pde).u & (pVCpu)->pgm.s.fGst32BitMbzBigPdeMask ))
+//# define GST_IS_PDPE_VALID(pVCpu, Pdpe)         (false)
+//# define GST_IS_BIG_PDPE_VALID(pVCpu, Pdpe)     (false)
+//# define GST_IS_PML4E_VALID(pVCpu, Pml4e)       (false)
+# define GST_IS_PSE_ACTIVE(pVCpu)               pgmGst32BitIsPageSizeExtActive(pVCpu)
+# define GST_IS_NX_ACTIVE(pVCpu)                (false)
+# define BTH_IS_NP_ACTIVE(pVM)                  (false)
+
+#elif   PGM_GST_TYPE == PGM_TYPE_PAE \
+     || PGM_GST_TYPE == PGM_TYPE_AMD64
+# define GSTPT                                  X86PTPAE
+# define PGSTPT                                 PX86PTPAE
+# define GSTPTE                                 X86PTEPAE
+# define PGSTPTE                                PX86PTEPAE
+# define GSTPD                                  X86PDPAE
+# define PGSTPD                                 PX86PDPAE
+# define GSTPDE                                 X86PDEPAE
+# define PGSTPDE                                PX86PDEPAE
+# define GST_BIG_PAGE_SIZE                      X86_PAGE_2M_SIZE
+# define GST_BIG_PAGE_OFFSET_MASK               X86_PAGE_2M_OFFSET_MASK
+# define GST_PDE_PG_MASK                        X86_PDE_PAE_PG_MASK
+# define GST_PDE_BIG_PG_MASK                    X86_PDE2M_PAE_PG_MASK
+# define GST_GET_PTE_GCPHYS(Pte)                PGM_A20_APPLY(pVCpu, ((Pte).u & GST_PTE_PG_MASK))
+# define GST_GET_PDE_GCPHYS(Pde)                PGM_A20_APPLY(pVCpu, ((Pde).u & GST_PDE_PG_MASK))
+# define GST_GET_BIG_PDE_GCPHYS(pVM, Pde)       PGM_A20_APPLY(pVCpu, ((Pde).u & GST_PDE_BIG_PG_MASK))
+# define GST_GET_PTE_SHW_FLAGS(pVCpu, Pte)      ((Pte).u & (pVCpu)->pgm.s.fGst64ShadowedPteMask )
+# define GST_GET_PDE_SHW_FLAGS(pVCpu, Pde)      ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedPdeMask )
+# define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde)  ( ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedBigPdeMask ) | PGM_PDFLAGS_BIG_PAGE)
+# define GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, Pde)  ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedBigPde4PteMask )
+
+# define GST_PD_SHIFT                           X86_PD_PAE_SHIFT
+# define GST_PD_MASK                            X86_PD_PAE_MASK
+# if PGM_GST_TYPE == PGM_TYPE_PAE
+#  define GSTPTWALK                             PGMPTWALKGSTPAE
+#  define PGSTPTWALK                            PPGMPTWALKGSTPAE
+#  define PCGSTPTWALK                           PCPGMPTWALKGSTPAE
+#  define GST_TOTAL_PD_ENTRIES                  (X86_PG_PAE_ENTRIES * X86_PG_PAE_PDPE_ENTRIES)
+#  define GST_PDPE_ENTRIES                      X86_PG_PAE_PDPE_ENTRIES
+#  define GST_PDPE_PG_MASK                      X86_PDPE_PG_MASK
+#  define GST_PDPT_SHIFT                        X86_PDPT_SHIFT
+#  define GST_PDPT_MASK                         X86_PDPT_MASK_PAE
+#  define GST_PTE_PG_MASK                       X86_PTE_PAE_PG_MASK
+#  define GST_CR3_PAGE_MASK                     X86_CR3_PAE_PAGE_MASK
+#  define GST_IS_PTE_VALID(pVCpu, Pte)          (!( (Pte).u   & (pVCpu)->pgm.s.fGstPaeMbzPteMask ))
+#  define GST_IS_PDE_VALID(pVCpu, Pde)          (!( (Pde).u   & (pVCpu)->pgm.s.fGstPaeMbzPdeMask ))
+#  define GST_IS_BIG_PDE_VALID(pVCpu, Pde)      (!( (Pde).u   & (pVCpu)->pgm.s.fGstPaeMbzBigPdeMask ))
+#  define GST_IS_PDPE_VALID(pVCpu, Pdpe)        (!( (Pdpe).u  & (pVCpu)->pgm.s.fGstPaeMbzPdpeMask ))
+//# define GST_IS_BIG_PDPE_VALID(pVCpu, Pdpe)    (false)
+//# define GST_IS_PML4E_VALID(pVCpu, Pml4e)      (false)
+# else
+#  define GSTPTWALK                             PGMPTWALKGSTAMD64
+#  define PGSTPTWALK                            PPGMPTWALKGSTAMD64
+#  define PCGSTPTWALK                           PCPGMPTWALKGSTAMD64
+#  define GST_TOTAL_PD_ENTRIES                  (X86_PG_AMD64_ENTRIES * X86_PG_AMD64_PDPE_ENTRIES)
+#  define GST_PDPE_ENTRIES                      X86_PG_AMD64_PDPE_ENTRIES
+#  define GST_PDPT_SHIFT                        X86_PDPT_SHIFT
+#  define GST_PDPE_PG_MASK                      X86_PDPE_PG_MASK
+#  define GST_PDPT_MASK                         X86_PDPT_MASK_AMD64
+#  define GST_PTE_PG_MASK                       X86_PTE_PAE_PG_MASK
+#  define GST_CR3_PAGE_MASK                     X86_CR3_AMD64_PAGE_MASK
+#  define GST_IS_PTE_VALID(pVCpu, Pte)          (!( (Pte).u   & (pVCpu)->pgm.s.fGstAmd64MbzPteMask ))
+#  define GST_IS_PDE_VALID(pVCpu, Pde)          (!( (Pde).u   & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask ))
+#  define GST_IS_BIG_PDE_VALID(pVCpu, Pde)      (!( (Pde).u   & (pVCpu)->pgm.s.fGstAmd64MbzBigPdeMask ))
+#  define GST_IS_PDPE_VALID(pVCpu, Pdpe)        (!( (Pdpe).u  & (pVCpu)->pgm.s.fGstAmd64MbzPdpeMask ))
+#  define GST_IS_BIG_PDPE_VALID(pVCpu, Pdpe)    (!( (Pdpe).u  & (pVCpu)->pgm.s.fGstAmd64MbzBigPdpeMask ))
+#  define GST_IS_PML4E_VALID(pVCpu, Pml4e)      (!( (Pml4e).u & (pVCpu)->pgm.s.fGstAmd64MbzPml4eMask ))
+# endif
+# define GST_PT_SHIFT                           X86_PT_PAE_SHIFT
+# define GST_PT_MASK                            X86_PT_PAE_MASK
+# define GST_IS_PSE_ACTIVE(pVCpu)               (true)
+# define GST_IS_NX_ACTIVE(pVCpu)                (pgmGstIsNoExecuteActive(pVCpu))
+# define BTH_IS_NP_ACTIVE(pVM)                  (false)
+#endif
+
diff --git a/src/VBox/VMM/include/PGMInline.h b/src/VBox/VMM/include/PGMInline.h
new file mode 100644
index 00000000..4018ab0d
--- /dev/null
+++ b/src/VBox/VMM/include/PGMInline.h
@@ -0,0 +1,1435 @@
+/* $Id: PGMInline.h $ */
+/** @file
+ * PGM - Inlined functions.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_PGMInline_h
+#define VMM_INCLUDED_SRC_include_PGMInline_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/err.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/param.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/dis.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/log.h>
+#include <VBox/vmm/gmm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/nem.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/avl.h>
+#include <iprt/critsect.h>
+#include <iprt/sha.h>
+
+
+
+/** @addtogroup grp_pgm_int   Internals
+ * @internal
+ * @{
+ */
+
+/**
+ * Gets the PGMRAMRANGE structure for a guest page.
+ *
+ * @returns Pointer to the RAM range on success.
+ * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address.
+ */
+DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    if (!pRam || GCPhys - pRam->GCPhys >= pRam->cb)
+        return pgmPhysGetRangeSlow(pVM, GCPhys);
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
+    return pRam;
+}
+
+
+/**
+ * Gets the PGMRAMRANGE structure for a guest page, if unassigned get the ram
+ * range above it.
+ *
+ * @returns Pointer to the RAM range on success.
+ * @returns NULL if the address is located after the last range.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address.
+ */
+DECLINLINE(PPGMRAMRANGE) pgmPhysGetRangeAtOrAbove(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    if (   !pRam
+        || (GCPhys - pRam->GCPhys) >= pRam->cb)
+        return pgmPhysGetRangeAtOrAboveSlow(pVM, GCPhys);
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
+    return pRam;
+}
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page.
+ *
+ * @returns Pointer to the page on success.
+ * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address.
+ */
+DECLINLINE(PPGMPAGE) pgmPhysGetPage(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    RTGCPHYS off;
+    if (   !pRam
+        || (off = GCPhys - pRam->GCPhys) >= pRam->cb)
+        return pgmPhysGetPageSlow(pVM, GCPhys);
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
+    return &pRam->aPages[off >> PAGE_SHIFT];
+}
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page.
+ *
+ * Old Phys code: Will make sure the page is present.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS and a valid *ppPage on success.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address.
+ * @param   ppPage      Where to store the page pointer on success.
+ */
+DECLINLINE(int) pgmPhysGetPageEx(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
+{
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    RTGCPHYS off;
+    if (   !pRam
+        || (off = GCPhys - pRam->GCPhys) >= pRam->cb)
+        return pgmPhysGetPageExSlow(pVM, GCPhys, ppPage);
+    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page.
+ *
+ * Old Phys code: Will make sure the page is present.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS and a valid *ppPage on success.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address.
+ * @param   ppPage      Where to store the page pointer on success.
+ * @param   ppRamHint   Where to read and store the ram list hint.
+ *                      The caller initializes this to NULL before the call.
+ */
+DECLINLINE(int) pgmPhysGetPageWithHintEx(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
+{
+    RTGCPHYS off;
+    PPGMRAMRANGE pRam = *ppRamHint;
+    if (    !pRam
+        ||  RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
+    {
+        pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+        if (   !pRam
+            || (off = GCPhys - pRam->GCPhys) >= pRam->cb)
+            return pgmPhysGetPageAndRangeExSlow(pVM, GCPhys, ppPage, ppRamHint);
+
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
+        *ppRamHint = pRam;
+    }
+    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
+ *
+ * @returns Pointer to the page on success.
+ * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address.
+ * @param   ppPage      Where to store the pointer to the PGMPAGE structure.
+ * @param   ppRam       Where to store the pointer to the PGMRAMRANGE structure.
+ */
+DECLINLINE(int) pgmPhysGetPageAndRangeEx(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
+{
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    RTGCPHYS off;
+    if (   !pRam
+        || (off = GCPhys - pRam->GCPhys) >= pRam->cb)
+        return pgmPhysGetPageAndRangeExSlow(pVM, GCPhys, ppPage, ppRam);
+
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
+    *ppRam = pRam;
+    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Convert GC Phys to HC Phys.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address.
+ * @param   pHCPhys     Where to store the corresponding HC physical address.
+ *
+ * @deprecated  Doesn't deal with zero, shared or write monitored pages.
+ *              Avoid when writing new code!
+ */
+DECLINLINE(int) pgmRamGCPhys2HCPhys(PVMCC pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
+{
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+    if (RT_FAILURE(rc))
+        return rc;
+    *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) | (GCPhys & PAGE_OFFSET_MASK);
+    return VINF_SUCCESS;
+}
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+
+/**
+ * Inlined version of the ring-0 version of the host page mapping code
+ * that optimizes access to pages already in the set.
+ *
+ * @returns VINF_SUCCESS. Will bail out to ring-3 on failure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   HCPhys      The physical address of the page.
+ * @param   ppv         Where to store the mapping address.
+ * @param   SRC_POS     The source location of the caller.
+ */
+DECLINLINE(int) pgmRZDynMapHCPageInlined(PVMCPUCC pVCpu, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
+{
+    PPGMMAPSET  pSet    = &pVCpu->pgm.s.AutoSet;
+
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInl, a);
+    Assert(!(HCPhys & PAGE_OFFSET_MASK));
+    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
+
+    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
+    unsigned    iEntry  = pSet->aiHashTable[iHash];
+    if (    iEntry < pSet->cEntries
+        &&  pSet->aEntries[iEntry].HCPhys == HCPhys
+        &&  pSet->aEntries[iEntry].cInlinedRefs < UINT16_MAX - 1)
+    {
+        pSet->aEntries[iEntry].cInlinedRefs++;
+        *ppv = pSet->aEntries[iEntry].pvPage;
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInlHits);
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInlMisses);
+        pgmRZDynMapHCPageCommon(pSet, HCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
+    }
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInl, a);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inlined version of the guest page mapping code that optimizes access to pages
+ * already in the set.
+ *
+ * @returns VBox status code, see pgmRZDynMapGCPageCommon for details.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPhys      The guest physical address of the page.
+ * @param   ppv         Where to store the mapping address.
+ * @param   SRC_POS     The source location of the caller.
+ */
+DECLINLINE(int) pgmRZDynMapGCPageV2Inlined(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
+{
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInl, a);
+    AssertMsg(!(GCPhys & PAGE_OFFSET_MASK), ("%RGp\n", GCPhys));
+
+    /*
+     * Get the ram range.
+     */
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    RTGCPHYS off;
+    if (   !pRam
+        || (off = GCPhys - pRam->GCPhys) >= pRam->cb
+        /** @todo   || page state stuff */
+       )
+    {
+        /* This case is not counted into StatRZDynMapGCPageInl. */
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamMisses);
+        return pgmRZDynMapGCPageCommon(pVM, pVCpu, GCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
+    }
+
+    RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamHits);
+
+    /*
+     * pgmRZDynMapHCPageInlined with out stats.
+     */
+    PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
+    Assert(!(HCPhys & PAGE_OFFSET_MASK));
+    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
+
+    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
+    unsigned    iEntry  = pSet->aiHashTable[iHash];
+    if (    iEntry < pSet->cEntries
+        &&  pSet->aEntries[iEntry].HCPhys == HCPhys
+        &&  pSet->aEntries[iEntry].cInlinedRefs < UINT16_MAX - 1)
+    {
+        pSet->aEntries[iEntry].cInlinedRefs++;
+        *ppv = pSet->aEntries[iEntry].pvPage;
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlHits);
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlMisses);
+        pgmRZDynMapHCPageCommon(pSet, HCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
+    }
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInl, a);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inlined version of the ring-0 version of guest page mapping that optimizes
+ * access to pages already in the set.
+ *
+ * @returns VBox status code, see pgmRZDynMapGCPageCommon for details.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPhys      The guest physical address of the page.
+ * @param   ppv         Where to store the mapping address.
+ * @param   SRC_POS     The source location of the caller.
+ */
+DECLINLINE(int) pgmRZDynMapGCPageInlined(PVMCPUCC pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
+{
+    return pgmRZDynMapGCPageV2Inlined(pVCpu->CTX_SUFF(pVM), pVCpu, GCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
+}
+
+
+/**
+ * Inlined version of the ring-0 version of the guest byte mapping code
+ * that optimizes access to pages already in the set.
+ *
+ * @returns VBox status code, see pgmRZDynMapGCPageCommon for details.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPhys      The guest physical address of the page.
+ * @param   ppv         Where to store the mapping address. The offset is
+ *                      preserved.
+ * @param   SRC_POS     The source location of the caller.
+ */
+DECLINLINE(int) pgmRZDynMapGCPageOffInlined(PVMCPUCC pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
+{
+    STAM_PROFILE_START(&pVCpu->pgm.s.StatRZDynMapGCPageInl, a);
+
+    /*
+     * Get the ram range.
+     */
+    PVMCC             pVM  = pVCpu->CTX_SUFF(pVM);
+    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    RTGCPHYS        off;
+    if (   !pRam
+        || (off = GCPhys - pRam->GCPhys) >= pRam->cb
+        /** @todo   || page state stuff */
+       )
+    {
+        /* This case is not counted into StatRZDynMapGCPageInl. */
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamMisses);
+        return pgmRZDynMapGCPageCommon(pVM, pVCpu, GCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
+    }
+
+    RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamHits);
+
+    /*
+     * pgmRZDynMapHCPageInlined with out stats.
+     */
+    PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
+    Assert(!(HCPhys & PAGE_OFFSET_MASK));
+    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
+
+    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
+    unsigned    iEntry  = pSet->aiHashTable[iHash];
+    if (    iEntry < pSet->cEntries
+        &&  pSet->aEntries[iEntry].HCPhys == HCPhys
+        &&  pSet->aEntries[iEntry].cInlinedRefs < UINT16_MAX - 1)
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlHits);
+        pSet->aEntries[iEntry].cInlinedRefs++;
+        *ppv = (void *)((uintptr_t)pSet->aEntries[iEntry].pvPage | (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlMisses);
+        pgmRZDynMapHCPageCommon(pSet, HCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
+        *ppv = (void *)((uintptr_t)*ppv | (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
+    }
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInl, a);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Maps the page into current context (RC and maybe R0).
+ *
+ * @returns pointer to the mapping.
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The page.
+ * @param   SRC_POS     The source location of the caller.
+ */
+DECLINLINE(void *) pgmPoolMapPageInlined(PVMCC pVM, PPGMPOOLPAGE pPage RTLOG_COMMA_SRC_POS_DECL)
+{
+    if (pPage->idx >= PGMPOOL_IDX_FIRST)
+    {
+        Assert(pPage->idx < pVM->pgm.s.CTX_SUFF(pPool)->cCurPages);
+        void *pv;
+        pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), pPage->Core.Key, &pv RTLOG_COMMA_SRC_POS_ARGS);
+        return pv;
+    }
+    AssertFatalMsgFailed(("pgmPoolMapPageInlined invalid page index %x\n", pPage->idx));
+}
+
+
+/**
+ * Maps the page into current context (RC and maybe R0).
+ *
+ * @returns pointer to the mapping.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pPage       The page.
+ * @param   SRC_POS     The source location of the caller.
+ */
+DECLINLINE(void *) pgmPoolMapPageV2Inlined(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOLPAGE pPage RTLOG_COMMA_SRC_POS_DECL)
+{
+    if (pPage->idx >= PGMPOOL_IDX_FIRST)
+    {
+        Assert(pPage->idx < pVM->pgm.s.CTX_SUFF(pPool)->cCurPages);
+        void *pv;
+        Assert(pVCpu == VMMGetCpu(pVM)); RT_NOREF_PV(pVM);
+        pgmRZDynMapHCPageInlined(pVCpu, pPage->Core.Key, &pv RTLOG_COMMA_SRC_POS_ARGS);
+        return pv;
+    }
+    AssertFatalMsgFailed(("pgmPoolMapPageV2Inlined invalid page index %x\n", pPage->idx));
+}
+
+#endif /*  VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+
+/**
+ * Queries the Physical TLB entry for a physical guest page,
+ * attempting to load the TLB entry if necessary.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The address of the guest page.
+ * @param   ppTlbe      Where to store the pointer to the TLB entry.
+ */
+DECLINLINE(int) pgmPhysPageQueryTlbe(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
+{
+    int rc;
+    PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTX_SUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
+    if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbHits));
+        rc = VINF_SUCCESS;
+    }
+    else
+        rc = pgmPhysPageLoadIntoTlb(pVM, GCPhys);
+    *ppTlbe = pTlbe;
+    return rc;
+}
+
+
+/**
+ * Queries the Physical TLB entry for a physical guest page,
+ * attempting to load the TLB entry if necessary.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       Pointer to the PGMPAGE structure corresponding to
+ *                      GCPhys.
+ * @param   GCPhys      The address of the guest page.
+ * @param   ppTlbe      Where to store the pointer to the TLB entry.
+ */
+DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
+{
+    int rc;
+    PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTX_SUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
+    if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbHits));
+        rc = VINF_SUCCESS;
+#if 0 //def VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+# ifdef IN_RING3
+        if (pTlbe->pv == (void *)pVM->pgm.s.pvZeroPgR0)
+# else
+        if (pTlbe->pv == (void *)pVM->pgm.s.pvZeroPgR3)
+# endif
+            pTlbe->pv = pVM->pgm.s.CTX_SUFF(pvZeroPg);
+#endif
+        AssertPtr(pTlbe->pv);
+#if defined(IN_RING3) || (!defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) && !defined(VBOX_WITH_RAM_IN_KERNEL))
+        Assert(!pTlbe->pMap || RT_VALID_PTR(pTlbe->pMap->pv));
+#endif
+    }
+    else
+        rc = pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
+    *ppTlbe = pTlbe;
+    return rc;
+}
+
+
+/**
+ * Calculates NEM page protection flags.
+ */
+DECL_FORCE_INLINE(uint32_t) pgmPhysPageCalcNemProtection(PPGMPAGE pPage, PGMPAGETYPE enmType)
+{
+    /*
+     * Deal with potentially writable pages first.
+     */
+    if (PGMPAGETYPE_IS_RWX(enmType))
+    {
+        if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+        {
+            if (PGM_PAGE_IS_ALLOCATED(pPage))
+                return NEM_PAGE_PROT_READ | NEM_PAGE_PROT_EXECUTE | NEM_PAGE_PROT_WRITE;
+            return NEM_PAGE_PROT_READ | NEM_PAGE_PROT_EXECUTE;
+        }
+        if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+            return NEM_PAGE_PROT_READ | NEM_PAGE_PROT_EXECUTE;
+    }
+    /*
+     * Potentially readable & executable pages.
+     */
+    else if (   PGMPAGETYPE_IS_ROX(enmType)
+             && !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+        return NEM_PAGE_PROT_READ | NEM_PAGE_PROT_EXECUTE;
+
+    /*
+     * The rest is needs special access handling.
+     */
+    return NEM_PAGE_PROT_NONE;
+}
+
+
+/**
+ * Enables write monitoring for an allocated page.
+ *
+ * The caller is responsible for updating the shadow page tables.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The page to write monitor.
+ * @param   GCPhysPage  The address of the page.
+ */
+DECLINLINE(void) pgmPhysPageWriteMonitor(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage)
+{
+    Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_WRITE_MONITORED);
+    pVM->pgm.s.cMonitoredPages++;
+
+    /* Large pages must disabled. */
+    if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
+    {
+        PPGMPAGE pFirstPage = pgmPhysGetPage(pVM, GCPhysPage & X86_PDE2M_PAE_PG_MASK);
+        AssertFatal(pFirstPage);
+        if (PGM_PAGE_GET_PDE_TYPE(pFirstPage) == PGM_PAGE_PDE_TYPE_PDE)
+        {
+            PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
+            pVM->pgm.s.cLargePagesDisabled++;
+        }
+        else
+            Assert(PGM_PAGE_GET_PDE_TYPE(pFirstPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
+    }
+
+    /* Tell NEM. */
+    if (VM_IS_NEM_ENABLED(pVM))
+    {
+        uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+        PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+        NEMHCNotifyPhysPageProtChanged(pVM, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage),
+                                       pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+        PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+    }
+}
+
+
+/**
+ * Checks if the no-execute (NX) feature is active (EFER.NXE=1).
+ *
+ * Only used when the guest is in PAE or long mode.  This is inlined so that we
+ * can perform consistency checks in debug builds.
+ *
+ * @returns true if it is, false if it isn't.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECL_FORCE_INLINE(bool) pgmGstIsNoExecuteActive(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu->pgm.s.fNoExecuteEnabled == CPUMIsGuestNXEnabled(pVCpu));
+    Assert(CPUMIsGuestInPAEMode(pVCpu) || CPUMIsGuestInLongMode(pVCpu));
+    return pVCpu->pgm.s.fNoExecuteEnabled;
+}
+
+
+/**
+ * Checks if the page size extension (PSE) is currently enabled (CR4.PSE=1).
+ *
+ * Only used when the guest is in paged 32-bit mode.  This is inlined so that
+ * we can perform consistency checks in debug builds.
+ *
+ * @returns true if it is, false if it isn't.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECL_FORCE_INLINE(bool) pgmGst32BitIsPageSizeExtActive(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu->pgm.s.fGst32BitPageSizeExtension == CPUMIsGuestPageSizeExtEnabled(pVCpu));
+    Assert(!CPUMIsGuestInPAEMode(pVCpu));
+    Assert(!CPUMIsGuestInLongMode(pVCpu));
+    return pVCpu->pgm.s.fGst32BitPageSizeExtension;
+}
+
+
+/**
+ * Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode.
+ * Takes PSE-36 into account.
+ *
+ * @returns guest physical address
+ * @param   pVM         The cross context VM structure.
+ * @param   Pde         Guest Pde
+ */
+DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PVMCC pVM, X86PDE Pde)
+{
+    RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
+    GCPhys |= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
+
+    return GCPhys & pVM->pgm.s.GCPhys4MBPSEMask;
+}
+
+
+/**
+ * Gets the address the guest page directory (32-bit paging).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   ppPd        Where to return the mapping. This is always set.
+ */
+DECLINLINE(int) pgmGstGet32bitPDPtrEx(PVMCPUCC pVCpu, PX86PD *ppPd)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)ppPd RTLOG_COMMA_SRC_POS);
+    if (RT_FAILURE(rc))
+    {
+        *ppPd = NULL;
+        return rc;
+    }
+#else
+    *ppPd = pVCpu->pgm.s.CTX_SUFF(pGst32BitPd);
+    if (RT_UNLIKELY(!*ppPd))
+        return pgmGstLazyMap32BitPD(pVCpu, ppPd);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the address the guest page directory (32-bit paging).
+ *
+ * @returns Pointer to the page directory entry in question.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PVMCPUCC pVCpu)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PD pGuestPD = NULL;
+    int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)&pGuestPD RTLOG_COMMA_SRC_POS);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+        return NULL;
+    }
+#else
+    PX86PD pGuestPD = pVCpu->pgm.s.CTX_SUFF(pGst32BitPd);
+    if (RT_UNLIKELY(!pGuestPD))
+    {
+        int rc = pgmGstLazyMap32BitPD(pVCpu, &pGuestPD);
+        if (RT_FAILURE(rc))
+            return NULL;
+    }
+#endif
+    return pGuestPD;
+}
+
+
+/**
+ * Gets the guest page directory pointer table.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   ppPdpt      Where to return the mapping.  This is always set.
+ */
+DECLINLINE(int) pgmGstGetPaePDPTPtrEx(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    int rc = pgmRZDynMapGCPageOffInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)ppPdpt RTLOG_COMMA_SRC_POS);
+    if (RT_FAILURE(rc))
+    {
+        *ppPdpt = NULL;
+        return rc;
+    }
+#else
+    *ppPdpt = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
+    if (RT_UNLIKELY(!*ppPdpt))
+        return pgmGstLazyMapPaePDPT(pVCpu, ppPdpt);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the guest page directory pointer table.
+ *
+ * @returns Pointer to the page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PVMCPUCC pVCpu)
+{
+    PX86PDPT pGuestPdpt;
+    int rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pGuestPdpt);
+    AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc);
+    return pGuestPdpt;
+}
+
+
+/**
+ * Gets the guest page directory pointer table entry for the specified address.
+ *
+ * @returns Pointer to the page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr)
+{
+    AssertGCPtr32(GCPtr);
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PDPT pGuestPDPT = NULL;
+    int rc = pgmRZDynMapGCPageOffInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)&pGuestPDPT RTLOG_COMMA_SRC_POS);
+    AssertRCReturn(rc, NULL);
+#else
+    PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
+    if (RT_UNLIKELY(!pGuestPDPT))
+    {
+        int rc = pgmGstLazyMapPaePDPT(pVCpu, &pGuestPDPT);
+        if (RT_FAILURE(rc))
+            return NULL;
+    }
+#endif
+    return &pGuestPDPT->a[(uint32_t)GCPtr >> X86_PDPT_SHIFT];
+}
+
+
+/**
+ * Gets the page directory entry for the specified address.
+ *
+ * @returns The page directory entry in question.
+ * @returns A non-present entry if the page directory is not present or on an invalid page.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PVMCPUCC pVCpu, RTGCPTR GCPtr)
+{
+    AssertGCPtr32(GCPtr);
+    PX86PDPT    pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
+    if (RT_LIKELY(pGuestPDPT))
+    {
+        const unsigned iPdpt = (uint32_t)GCPtr >> X86_PDPT_SHIFT;
+        if (    pGuestPDPT->a[iPdpt].n.u1Present
+            &&  !(pGuestPDPT->a[iPdpt].u & pVCpu->pgm.s.fGstPaeMbzPdpeMask) )
+        {
+            const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+            PX86PDPAE   pGuestPD = NULL;
+            int rc = pgmRZDynMapGCPageInlined(pVCpu,
+                                              pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK,
+                                              (void **)&pGuestPD
+                                              RTLOG_COMMA_SRC_POS);
+            if (RT_SUCCESS(rc))
+                return pGuestPD->a[iPD];
+            AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+#else
+            PX86PDPAE   pGuestPD = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt];
+            if (    !pGuestPD
+                ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt])
+                pgmGstLazyMapPaePD(pVCpu, iPdpt, &pGuestPD);
+            if (pGuestPD)
+                return pGuestPD->a[iPD];
+#endif
+        }
+    }
+
+    X86PDEPAE ZeroPde = {0};
+    return ZeroPde;
+}
+
+
+/**
+ * Gets the page directory pointer table entry for the specified address
+ * and returns the index into the page directory
+ *
+ * @returns Pointer to the page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ * @param   piPD        Receives the index into the returned page directory
+ * @param   pPdpe       Receives the page directory pointer entry. Optional.
+ */
+DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe)
+{
+    AssertGCPtr32(GCPtr);
+
+    /* The PDPE. */
+    PX86PDPT        pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
+    if (RT_UNLIKELY(!pGuestPDPT))
+        return NULL;
+    const unsigned  iPdpt = (uint32_t)GCPtr >> X86_PDPT_SHIFT;
+    if (pPdpe)
+        *pPdpe = pGuestPDPT->a[iPdpt];
+    if (!pGuestPDPT->a[iPdpt].n.u1Present)
+        return NULL;
+    if (RT_UNLIKELY(pVCpu->pgm.s.fGstPaeMbzPdpeMask & pGuestPDPT->a[iPdpt].u))
+        return NULL;
+
+    /* The PDE. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PDPAE   pGuestPD = NULL;
+    int rc = pgmRZDynMapGCPageInlined(pVCpu,
+                                      pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK,
+                                      (void **)&pGuestPD
+                                      RTLOG_COMMA_SRC_POS);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+        return NULL;
+    }
+#else
+    PX86PDPAE   pGuestPD = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt];
+    if (    !pGuestPD
+        ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt])
+        pgmGstLazyMapPaePD(pVCpu, iPdpt, &pGuestPD);
+#endif
+
+    *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+    return pGuestPD;
+}
+
+
+/**
+ * Gets the page map level-4 pointer for the guest.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   ppPml4      Where to return the mapping.  Always set.
+ */
+DECLINLINE(int) pgmGstGetLongModePML4PtrEx(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)ppPml4 RTLOG_COMMA_SRC_POS);
+    if (RT_FAILURE(rc))
+    {
+        *ppPml4 = NULL;
+        return rc;
+    }
+#else
+    *ppPml4 = pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4);
+    if (RT_UNLIKELY(!*ppPml4))
+        return pgmGstLazyMapPml4(pVCpu, ppPml4);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the page map level-4 pointer for the guest.
+ *
+ * @returns Pointer to the PML4 page.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PVMCPUCC pVCpu)
+{
+    PX86PML4 pGuestPml4;
+    int rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pGuestPml4);
+    AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc);
+    return pGuestPml4;
+}
+
+
+/**
+ * Gets the pointer to a page map level-4 entry.
+ *
+ * @returns Pointer to the PML4 entry.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iPml4       The index.
+ * @remarks Only used by AssertCR3.
+ */
+DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PVMCPUCC pVCpu, unsigned int iPml4)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PML4 pGuestPml4;
+    int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)&pGuestPml4 RTLOG_COMMA_SRC_POS);
+    AssertRCReturn(rc, NULL);
+#else
+    PX86PML4 pGuestPml4 = pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4);
+    if (RT_UNLIKELY(!pGuestPml4))
+    {
+         int rc = pgmGstLazyMapPml4(pVCpu, &pGuestPml4);
+         AssertRCReturn(rc, NULL);
+    }
+#endif
+    return &pGuestPml4->a[iPml4];
+}
+
+
+/**
+ * Gets the page directory entry for the specified address.
+ *
+ * @returns The page directory entry in question.
+ * @returns A non-present entry if the page directory is not present or on an invalid page.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PVMCPUCC pVCpu, RTGCPTR64 GCPtr)
+{
+    /*
+     * Note! To keep things simple, ASSUME invalid physical addresses will
+     *       cause X86_TRAP_PF_RSVD.  This isn't a problem until we start
+     *       supporting 52-bit wide physical guest addresses.
+     */
+    PCX86PML4       pGuestPml4 = pgmGstGetLongModePML4Ptr(pVCpu);
+    const unsigned  iPml4      = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    if (    RT_LIKELY(pGuestPml4)
+        &&  pGuestPml4->a[iPml4].n.u1Present
+        &&  !(pGuestPml4->a[iPml4].u & pVCpu->pgm.s.fGstAmd64MbzPml4eMask) )
+    {
+        PCX86PDPT   pPdptTemp;
+        int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
+        if (RT_SUCCESS(rc))
+        {
+            const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+            if (    pPdptTemp->a[iPdpt].n.u1Present
+                &&  !(pPdptTemp->a[iPdpt].u & pVCpu->pgm.s.fGstAmd64MbzPdpeMask) )
+            {
+                PCX86PDPAE pPD;
+                rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
+                if (RT_SUCCESS(rc))
+                {
+                    const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+                    return pPD->a[iPD];
+                }
+            }
+        }
+        AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+    }
+
+    X86PDEPAE ZeroPde = {0};
+    return ZeroPde;
+}
+
+
+/**
+ * Gets the GUEST page directory pointer for the specified address.
+ *
+ * @returns The page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ * @param   ppPml4e     Page Map Level-4 Entry (out)
+ * @param   pPdpe       Page directory pointer table entry (out)
+ * @param   piPD        Receives the index into the returned page directory
+ */
+DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe, unsigned *piPD)
+{
+    /* The PMLE4. */
+    PX86PML4        pGuestPml4 = pgmGstGetLongModePML4Ptr(pVCpu);
+    if (RT_UNLIKELY(!pGuestPml4))
+        return NULL;
+    const unsigned  iPml4      = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    PCX86PML4E      pPml4e     = *ppPml4e = &pGuestPml4->a[iPml4];
+    if (!pPml4e->n.u1Present)
+        return NULL;
+    if (RT_UNLIKELY(pPml4e->u & pVCpu->pgm.s.fGstAmd64MbzPml4eMask))
+        return NULL;
+
+    /* The PDPE. */
+    PCX86PDPT       pPdptTemp;
+    int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+        return NULL;
+    }
+    const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    *pPdpe = pPdptTemp->a[iPdpt];
+    if (!pPdpe->n.u1Present)
+        return NULL;
+    if (RT_UNLIKELY(pPdpe->u & pVCpu->pgm.s.fGstAmd64MbzPdpeMask))
+        return NULL;
+
+    /* The PDE. */
+    PX86PDPAE pPD;
+    rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+        return NULL;
+    }
+
+    *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+    return pPD;
+}
+
+
+/**
+ * Gets the shadow page directory, 32-bit.
+ *
+ * @returns Pointer to the shadow 32-bit PD.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PVMCPUCC pVCpu)
+{
+    return (PX86PD)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+}
+
+
+/**
+ * Gets the shadow page directory entry for the specified address, 32-bit.
+ *
+ * @returns Shadow 32-bit PDE.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDE) pgmShwGet32BitPDE(PVMCPUCC pVCpu, RTGCPTR GCPtr)
+{
+    PX86PD pShwPde = pgmShwGet32BitPDPtr(pVCpu);
+    if (!pShwPde)
+    {
+        X86PDE ZeroPde = {0};
+        return ZeroPde;
+    }
+    return pShwPde->a[(uint32_t)GCPtr >> X86_PD_SHIFT];
+}
+
+
+/**
+ * Gets the pointer to the shadow page directory entry for the specified
+ * address, 32-bit.
+ *
+ * @returns Pointer to the shadow 32-bit PDE.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr)
+{
+    PX86PD pPde = pgmShwGet32BitPDPtr(pVCpu);
+    AssertReturn(pPde, NULL);
+    return &pPde->a[(uint32_t)GCPtr >> X86_PD_SHIFT];
+}
+
+
+/**
+ * Gets the shadow page pointer table, PAE.
+ *
+ * @returns Pointer to the shadow PAE PDPT.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PVMCPUCC pVCpu)
+{
+    return (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+}
+
+
+/**
+ * Gets the shadow page directory for the specified address, PAE.
+ *
+ * @returns Pointer to the shadow PD.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr)
+{
+    const unsigned  iPdpt = (uint32_t)GCPtr >> X86_PDPT_SHIFT;
+    PX86PDPT        pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
+
+    if (!pPdpt->a[iPdpt].n.u1Present)
+        return NULL;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
+    AssertReturn(pShwPde, NULL);
+
+    return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
+}
+
+
+/**
+ * Gets the shadow page directory for the specified address, PAE.
+ *
+ * @returns Pointer to the shadow PD.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pPdpt       Pointer to the page directory pointer table.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PVMCPUCC pVCpu, PX86PDPT pPdpt, RTGCPTR GCPtr)
+{
+    const unsigned  iPdpt = (uint32_t)GCPtr >> X86_PDPT_SHIFT;
+
+    if (!pPdpt->a[iPdpt].n.u1Present)
+        return NULL;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PVMCC             pVM     = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOLPAGE    pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
+    AssertReturn(pShwPde, NULL);
+
+    return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
+}
+
+
+/**
+ * Gets the shadow page directory entry, PAE.
+ *
+ * @returns PDE.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PVMCPUCC pVCpu, RTGCPTR GCPtr)
+{
+    const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+
+    PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pVCpu, GCPtr);
+    if (!pShwPde)
+    {
+        X86PDEPAE ZeroPde = {0};
+        return ZeroPde;
+    }
+    return pShwPde->a[iPd];
+}
+
+
+/**
+ * Gets the pointer to the shadow page directory entry for an address, PAE.
+ *
+ * @returns Pointer to the PDE.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ * @remarks Only used by AssertCR3.
+ */
+DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr)
+{
+    const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+
+    PX86PDPAE pPde = pgmShwGetPaePDPtr(pVCpu, GCPtr);
+    AssertReturn(pPde, NULL);
+    return &pPde->a[iPd];
+}
+
+
+/**
+ * Gets the shadow page map level-4 pointer.
+ *
+ * @returns Pointer to the shadow PML4.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PVMCPUCC pVCpu)
+{
+    return (PX86PML4)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+}
+
+
+/**
+ * Gets the shadow page map level-4 entry for the specified address.
+ *
+ * @returns The entry.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PVMCPUCC pVCpu, RTGCPTR GCPtr)
+{
+    const unsigned  iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    PX86PML4        pShwPml4 = pgmShwGetLongModePML4Ptr(pVCpu);
+
+    if (!pShwPml4)
+    {
+        X86PML4E ZeroPml4e = {0};
+        return ZeroPml4e;
+    }
+    return pShwPml4->a[iPml4];
+}
+
+
+/**
+ * Gets the pointer to the specified shadow page map level-4 entry.
+ *
+ * @returns The entry.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iPml4       The PML4 index.
+ */
+DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PVMCPUCC pVCpu, unsigned int iPml4)
+{
+    PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pVCpu);
+    if (!pShwPml4)
+        return NULL;
+    return &pShwPml4->a[iPml4];
+}
+
+
+/**
+ * Cached physical handler lookup.
+ *
+ * @returns Physical handler covering @a GCPhys.
+ * @param   pVM                 The cross context VM structure.
+ * @param   GCPhys              The lookup address.
+ */
+DECLINLINE(PPGMPHYSHANDLER) pgmHandlerPhysicalLookup(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PPGMPHYSHANDLER pHandler = pVM->pgm.s.CTX_SUFF(pLastPhysHandler);
+    if (   pHandler
+        && GCPhys >= pHandler->Core.Key
+        && GCPhys < pHandler->Core.KeyLast)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysHandlerLookupHits));
+        return pHandler;
+    }
+
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysHandlerLookupMisses));
+    pHandler = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
+    if (pHandler)
+        pVM->pgm.s.CTX_SUFF(pLastPhysHandler) = pHandler;
+    return pHandler;
+}
+
+
+/**
+ * Internal worker for finding a 'in-use' shadow page give by it's physical address.
+ *
+ * @returns Pointer to the shadow page structure.
+ * @param   pPool       The pool.
+ * @param   idx         The pool page index.
+ */
+DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx)
+{
+    AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx));
+    return &pPool->aPages[idx];
+}
+
+
+/**
+ * Clear references to guest physical memory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPoolPage   The pool page.
+ * @param   pPhysPage   The physical guest page tracking structure.
+ * @param   iPte        Shadow PTE index
+ */
+DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage, uint16_t iPte)
+{
+    /*
+     * Just deal with the simple case here.
+     */
+#ifdef VBOX_STRICT
+    PVMCC pVM = pPool->CTX_SUFF(pVM); NOREF(pVM);
+#endif
+#ifdef LOG_ENABLED
+    const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage);
+#endif
+    const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
+    if (cRefs == 1)
+    {
+        Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage));
+        Assert(iPte == PGM_PAGE_GET_PTE_INDEX(pPhysPage));
+        /* Invalidate the tracking data. */
+        PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
+    }
+    else
+        pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage, iPte);
+    Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage ));
+}
+
+
+/**
+ * Moves the page to the head of the age list.
+ *
+ * This is done when the cached page is used in one way or another.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ */
+DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
+
+    /*
+     * Move to the head of the age list.
+     */
+    if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
+    {
+        /* unlink */
+        pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
+        if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
+            pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
+        else
+            pPool->iAgeTail = pPage->iAgePrev;
+
+        /* insert at head */
+        pPage->iAgePrev = NIL_PGMPOOL_IDX;
+        pPage->iAgeNext = pPool->iAgeHead;
+        Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */
+        pPool->iAgeHead = pPage->idx;
+        pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx;
+    }
+}
+
+
+/**
+ * Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages).
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       PGM pool page
+ */
+DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM)); NOREF(pPool);
+    ASMAtomicIncU32(&pPage->cLocked);
+}
+
+
+/**
+ * Unlocks a page to allow flushing again
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       PGM pool page
+ */
+DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM)); NOREF(pPool);
+    Assert(pPage->cLocked);
+    ASMAtomicDecU32(&pPage->cLocked);
+}
+
+
+/**
+ * Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT)
+ *
+ * @returns VBox status code.
+ * @param   pPage       PGM pool page
+ */
+DECLINLINE(bool) pgmPoolIsPageLocked(PPGMPOOLPAGE pPage)
+{
+    if (pPage->cLocked)
+    {
+        LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind));
+        if (pPage->cModifications)
+            pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Check if the specified page is dirty (not write monitored)
+ *
+ * @return dirty or not
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Guest physical address
+ */
+DECLINLINE(bool) pgmPoolIsDirtyPage(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    if (!pPool->cDirtyPages)
+        return false;
+    return pgmPoolIsDirtyPageSlow(pVM, GCPhys);
+}
+
+
+/**
+ * Tells if mappings are to be put into the shadow page table or not.
+ *
+ * @returns boolean result
+ * @param   pVM         The cross context VM structure.
+ */
+DECL_FORCE_INLINE(bool) pgmMapAreMappingsEnabled(PVMCC pVM)
+{
+#ifdef PGM_WITHOUT_MAPPINGS
+    /* Only raw-mode has mappings. */
+    Assert(!VM_IS_RAW_MODE_ENABLED(pVM)); NOREF(pVM);
+    return false;
+#else
+    Assert(pVM->cCpus == 1 || !VM_IS_RAW_MODE_ENABLED(pVM));
+    return VM_IS_RAW_MODE_ENABLED(pVM);
+#endif
+}
+
+
+/**
+ * Checks if the mappings are floating and enabled.
+ *
+ * @returns true / false.
+ * @param   pVM         The cross context VM structure.
+ */
+DECL_FORCE_INLINE(bool) pgmMapAreMappingsFloating(PVMCC pVM)
+{
+#ifdef PGM_WITHOUT_MAPPINGS
+    /* Only raw-mode has mappings. */
+    Assert(!VM_IS_RAW_MODE_ENABLED(pVM)); NOREF(pVM);
+    return false;
+#else
+    return !pVM->pgm.s.fMappingsFixed
+        && pgmMapAreMappingsEnabled(pVM);
+#endif
+}
+
+/** @} */
+
+#endif /* !VMM_INCLUDED_SRC_include_PGMInline_h */
+
diff --git a/src/VBox/VMM/include/PGMInternal.h b/src/VBox/VMM/include/PGMInternal.h
new file mode 100644
index 00000000..1a757db6
--- /dev/null
+++ b/src/VBox/VMM/include/PGMInternal.h
@@ -0,0 +1,4073 @@
+/* $Id: PGMInternal.h $ */
+/** @file
+ * PGM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_PGMInternal_h
+#define VMM_INCLUDED_SRC_include_PGMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/err.h>
+#include <VBox/dbg.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/param.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/dis.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/log.h>
+#include <VBox/vmm/gmm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/hm_vmx.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/avl.h>
+#include <iprt/critsect.h>
+#include <iprt/list-off32.h>
+#include <iprt/sha.h>
+
+
+
+/** @defgroup grp_pgm_int   Internals
+ * @ingroup grp_pgm
+ * @internal
+ * @{
+ */
+
+
+/** @name PGM Compile Time Config
+ * @{
+ */
+
+/**
+ * Indicates that there are no guest mappings in the shadow tables.
+ *
+ * Note! In ring-3 the macro is also used to exclude the managment of the
+ * intermediate context page tables.  On 32-bit systems we use the intermediate
+ * context to support 64-bit guest execution.  Thus, we cannot fully make it
+ * without mappings there even when VBOX_WITH_RAW_MODE is not defined.
+ *
+ * In raw-mode context there are by design always guest mappings (the code is
+ * executed from one), while in ring-0 there are none at all.  Neither context
+ * manages the page tables for intermediate switcher context, that's all done in
+ * ring-3.
+ *
+ * Update 6.1: It is always defined now, in pgm.h
+ */
+#if defined(IN_RING0) \
+  || (   !defined(VBOX_WITH_RAW_MODE) \
+      && (   HC_ARCH_BITS != 32 \
+          || !defined(VBOX_WITH_64_BITS_GUESTS) \
+         ) \
+     )
+# undef  PGM_WITHOUT_MAPPINGS
+# define PGM_WITHOUT_MAPPINGS
+#endif
+
+/**
+ * Check and skip global PDEs for non-global flushes
+ */
+#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
+
+/**
+ * Optimization for PAE page tables that are modified often
+ */
+//#if 0 /* disabled again while debugging */
+#define PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+//#endif
+
+/**
+ * Large page support enabled only on 64 bits hosts; applies to nested paging only.
+ */
+#define PGM_WITH_LARGE_PAGES
+
+/**
+ * Enables optimizations for MMIO handlers that exploits X86_TRAP_PF_RSVD and
+ * VMX_EXIT_EPT_MISCONFIG.
+ */
+#define PGM_WITH_MMIO_OPTIMIZATIONS
+
+/**
+ * Sync N pages instead of a whole page table
+ */
+#define PGM_SYNC_N_PAGES
+
+/**
+ * Number of pages to sync during a page fault
+ *
+ * When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here
+ * causes a lot of unnecessary extents and also is slower than taking more \#PFs.
+ *
+ * Note that \#PFs are much more expensive in the VT-x/AMD-V case due to
+ * world switch overhead, so let's sync more.
+ */
+# ifdef IN_RING0
+/* Chose 32 based on the compile test in @bugref{4219}; 64 shows worse stats.
+ * 32 again shows better results than 16; slightly more overhead in the \#PF handler,
+ * but ~5% fewer faults.
+ */
+# define PGM_SYNC_NR_PAGES              32
+#else
+# define PGM_SYNC_NR_PAGES              8
+#endif
+
+/**
+ * Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t))
+ */
+#define PGM_MAX_PHYSCACHE_ENTRIES       64
+#define PGM_MAX_PHYSCACHE_ENTRIES_MASK  (PGM_MAX_PHYSCACHE_ENTRIES-1)
+
+
+/** @def PGMPOOL_CFG_MAX_GROW
+ * The maximum number of pages to add to the pool in one go.
+ */
+#define PGMPOOL_CFG_MAX_GROW            (_2M >> PAGE_SHIFT)
+
+/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
+ * Enables some extra assertions for virtual handlers (mainly phys2virt related).
+ */
+#ifdef VBOX_STRICT
+# define VBOX_STRICT_PGM_HANDLER_VIRTUAL
+#endif
+
+/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+ * Enables the experimental lazy page allocation code. */
+#ifdef DOXYGEN_RUNNING
+# define VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+#endif
+
+/** @def VBOX_WITH_REAL_WRITE_MONITORED_PAGES
+ * Enables real write monitoring of pages, i.e. mapping them read-only and
+ * only making them writable when getting a write access \#PF. */
+#define VBOX_WITH_REAL_WRITE_MONITORED_PAGES
+
+/** @} */
+
+
+/** @name PDPT and PML4 flags.
+ * These are placed in the three bits available for system programs in
+ * the PDPT and PML4 entries.
+ * @{ */
+/** The entry is a permanent one and it's must always be present.
+ * Never free such an entry. */
+#define PGM_PLXFLAGS_PERMANENT          RT_BIT_64(10)
+/** Mapping (hypervisor allocated pagetable). */
+#define PGM_PLXFLAGS_MAPPING            RT_BIT_64(11)
+/** @} */
+
+/** @name Page directory flags.
+ * These are placed in the three bits available for system programs in
+ * the page directory entries.
+ * @{ */
+/** Indicates the original entry was a big page.
+ * @remarks This is currently only used for statistics and can be recycled.  */
+#define PGM_PDFLAGS_BIG_PAGE            RT_BIT_64(9)
+/** Mapping (hypervisor allocated pagetable). */
+#define PGM_PDFLAGS_MAPPING             RT_BIT_64(10)
+/** Made read-only to facilitate dirty bit tracking. */
+#define PGM_PDFLAGS_TRACK_DIRTY         RT_BIT_64(11)
+/** @} */
+
+/** @name Page flags.
+ * These are placed in the three bits available for system programs in
+ * the page entries.
+ * @{ */
+/** Made read-only to facilitate dirty bit tracking. */
+#define PGM_PTFLAGS_TRACK_DIRTY         RT_BIT_64(9)
+
+#ifndef PGM_PTFLAGS_CSAM_VALIDATED
+/** Scanned and approved by CSAM (tm).
+ * NOTE: Must be identical to the one defined in CSAMInternal.h!!
+ * @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/vmm/pgm.h. */
+#define PGM_PTFLAGS_CSAM_VALIDATED      RT_BIT_64(11)
+#endif
+
+/** @} */
+
+/** @name Defines used to indicate the shadow and guest paging in the templates.
+ * @{ */
+#define PGM_TYPE_REAL                   1
+#define PGM_TYPE_PROT                   2
+#define PGM_TYPE_32BIT                  3
+#define PGM_TYPE_PAE                    4
+#define PGM_TYPE_AMD64                  5
+#define PGM_TYPE_NESTED_32BIT           6
+#define PGM_TYPE_NESTED_PAE             7
+#define PGM_TYPE_NESTED_AMD64           8
+#define PGM_TYPE_EPT                    9
+#define PGM_TYPE_NONE                   10 /**< Dummy shadow paging mode for NEM. */
+#define PGM_TYPE_END                    (PGM_TYPE_NONE + 1)
+#define PGM_TYPE_FIRST_SHADOW           PGM_TYPE_32BIT /**< The first type used by shadow paging. */
+/** @} */
+
+/** Macro for checking if the guest is using paging.
+ * @param   uGstType   PGM_TYPE_*
+ * @param   uShwType   PGM_TYPE_*
+ * @remark  ASSUMES certain order of the PGM_TYPE_* values.
+ */
+#define PGM_WITH_PAGING(uGstType, uShwType)  \
+    (   (uGstType) >= PGM_TYPE_32BIT \
+     && (uShwType) < PGM_TYPE_NESTED_32BIT)
+
+/** Macro for checking if the guest supports the NX bit.
+ * @param   uGstType   PGM_TYPE_*
+ * @param   uShwType   PGM_TYPE_*
+ * @remark  ASSUMES certain order of the PGM_TYPE_* values.
+ */
+#define PGM_WITH_NX(uGstType, uShwType)  \
+    (   (uGstType) >= PGM_TYPE_PAE \
+     && (uShwType) < PGM_TYPE_NESTED_32BIT)
+
+/** Macro for checking for nested or EPT.
+ * @param   uType   PGM_TYPE_*
+ */
+#define PGM_TYPE_IS_NESTED(uType) \
+     (   (uType) == PGM_TYPE_NESTED_32BIT \
+      || (uType) == PGM_TYPE_NESTED_PAE \
+      || (uType) == PGM_TYPE_NESTED_AMD64)
+
+/** Macro for checking for nested or EPT.
+ * @param   uType   PGM_TYPE_*
+ */
+#define PGM_TYPE_IS_NESTED_OR_EPT(uType) \
+      (   (uType) == PGM_TYPE_NESTED_32BIT \
+       || (uType) == PGM_TYPE_NESTED_PAE \
+       || (uType) == PGM_TYPE_NESTED_AMD64 \
+       || (uType) == PGM_TYPE_EPT)
+
+
+
+/** @def PGM_HCPHYS_2_PTR
+ * Maps a HC physical page pool address to a virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   HCPhys      The HC physical address to map to a virtual one.
+ * @param   ppv         Where to store the virtual address. No need to cast
+ *                      this.
+ *
+ * @remark  There is no need to assert on the result.
+ */
+#define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) pgmPoolHCPhys2Ptr(pVM, HCPhys, (void **)(ppv))
+
+/** @def PGM_GCPHYS_2_PTR_V2
+ * Maps a GC physical page address to a virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPhys  The GC physical address to map to a virtual one.
+ * @param   ppv     Where to store the virtual address. No need to cast this.
+ *
+ * @remark  Use with care as we don't have so much dynamic mapping space in
+ *          ring-0 on 32-bit darwin and in RC.
+ * @remark  There is no need to assert on the result.
+ */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+# define PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, ppv) \
+     pgmRZDynMapGCPageV2Inlined(pVM, pVCpu, GCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
+#else
+# define PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, ppv) \
+     pgmPhysGCPhys2R3Ptr(pVM, GCPhys, (PRTR3PTR)(ppv)) /** @todo this isn't asserting! */
+#endif
+
+/** @def PGM_GCPHYS_2_PTR
+ * Maps a GC physical page address to a virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPhys  The GC physical address to map to a virtual one.
+ * @param   ppv     Where to store the virtual address. No need to cast this.
+ *
+ * @remark  Use with care as we don't have so much dynamic mapping space in
+ *          ring-0 on 32-bit darwin and in RC.
+ * @remark  There is no need to assert on the result.
+ */
+#define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2(pVM, VMMGetCpu(pVM), GCPhys, ppv)
+
+/** @def PGM_GCPHYS_2_PTR_BY_VMCPU
+ * Maps a GC physical page address to a virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPhys  The GC physical address to map to a virtual one.
+ * @param   ppv     Where to store the virtual address. No need to cast this.
+ *
+ * @remark  Use with care as we don't have so much dynamic mapping space in
+ *          ring-0 on 32-bit darwin and in RC.
+ * @remark  There is no need to assert on the result.
+ */
+#define PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2((pVCpu)->CTX_SUFF(pVM), pVCpu, GCPhys, ppv)
+
+/** @def PGM_GCPHYS_2_PTR_EX
+ * Maps a unaligned GC physical page address to a virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPhys  The GC physical address to map to a virtual one.
+ * @param   ppv     Where to store the virtual address. No need to cast this.
+ *
+ * @remark  Use with care as we don't have so much dynamic mapping space in
+ *          ring-0 on 32-bit darwin and in RC.
+ * @remark  There is no need to assert on the result.
+ */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
+     pgmRZDynMapGCPageOffInlined(VMMGetCpu(pVM), GCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
+#else
+# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
+     pgmPhysGCPhys2R3Ptr(pVM, GCPhys, (PRTR3PTR)(ppv)) /** @todo this isn't asserting! */
+#endif
+
+/** @def PGM_DYNMAP_UNUSED_HINT
+ * Hints to the dynamic mapping code in RC and R0/darwin that the specified page
+ * is no longer used.
+ *
+ * For best effect only apply this to the page that was mapped most recently.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   pvPage  The pool page.
+ */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+# ifdef LOG_ENABLED
+#  define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage)  pgmRZDynMapUnusedHint(pVCpu, pvPage, RT_SRC_POS)
+# else
+#  define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage)  pgmRZDynMapUnusedHint(pVCpu, pvPage)
+# endif
+#else
+# define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage)  do {} while (0)
+#endif
+
+/** @def PGM_DYNMAP_UNUSED_HINT_VM
+ * Hints to the dynamic mapping code in RC and R0/darwin that the specified page
+ * is no longer used.
+ *
+ * For best effect only apply this to the page that was mapped most recently.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   pvPage  The pool page.
+ */
+#define PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvPage)  PGM_DYNMAP_UNUSED_HINT(VMMGetCpu(pVM), pvPage)
+
+
+/** @def PGM_INVL_PG
+ * Invalidates a page.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCVirt      The virtual address of the page to invalidate.
+ */
+#ifdef IN_RING0
+# define PGM_INVL_PG(pVCpu, GCVirt)             HMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
+#elif defined(IN_RING3)
+# define PGM_INVL_PG(pVCpu, GCVirt)             HMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
+#else
+# error "Not IN_RING0 or IN_RING3!"
+#endif
+
+/** @def PGM_INVL_PG_ALL_VCPU
+ * Invalidates a page on all VCPUs
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCVirt      The virtual address of the page to invalidate.
+ */
+#ifdef IN_RING0
+# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt)      HMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
+#else
+# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt)      HMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
+#endif
+
+/** @def PGM_INVL_BIG_PG
+ * Invalidates a 4MB page directory entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCVirt      The virtual address within the page directory to invalidate.
+ */
+#ifdef IN_RING0
+# define PGM_INVL_BIG_PG(pVCpu, GCVirt)         HMFlushTlb(pVCpu)
+#else
+# define PGM_INVL_BIG_PG(pVCpu, GCVirt)         HMFlushTlb(pVCpu)
+#endif
+
+/** @def PGM_INVL_VCPU_TLBS()
+ * Invalidates the TLBs of the specified VCPU
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+#ifdef IN_RING0
+# define PGM_INVL_VCPU_TLBS(pVCpu)             HMFlushTlb(pVCpu)
+#else
+# define PGM_INVL_VCPU_TLBS(pVCpu)             HMFlushTlb(pVCpu)
+#endif
+
+/** @def PGM_INVL_ALL_VCPU_TLBS()
+ * Invalidates the TLBs of all VCPUs
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+#ifdef IN_RING0
+# define PGM_INVL_ALL_VCPU_TLBS(pVM)            HMFlushTlbOnAllVCpus(pVM)
+#else
+# define PGM_INVL_ALL_VCPU_TLBS(pVM)            HMFlushTlbOnAllVCpus(pVM)
+#endif
+
+
+/** @name Safer Shadow PAE PT/PTE
+ * For helping avoid misinterpreting invalid PAE/AMD64 page table entries as
+ * present.
+ *
+ * @{
+ */
+#if 1
+/**
+ * For making sure that u1Present and X86_PTE_P checks doesn't mistake
+ * invalid entries for present.
+ * @sa X86PTEPAE.
+ */
+typedef union PGMSHWPTEPAE
+{
+    /** Unsigned integer view */
+    X86PGPAEUINT    uCareful;
+    /* Not other views. */
+} PGMSHWPTEPAE;
+
+# define PGMSHWPTEPAE_IS_P(Pte)                 ( ((Pte).uCareful & (X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX)) == X86_PTE_P )
+# define PGMSHWPTEPAE_IS_RW(Pte)                ( !!((Pte).uCareful & X86_PTE_RW))
+# define PGMSHWPTEPAE_IS_US(Pte)                ( !!((Pte).uCareful & X86_PTE_US))
+# define PGMSHWPTEPAE_IS_A(Pte)                 ( !!((Pte).uCareful & X86_PTE_A))
+# define PGMSHWPTEPAE_IS_D(Pte)                 ( !!((Pte).uCareful & X86_PTE_D))
+# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte)       ( !!((Pte).uCareful & PGM_PTFLAGS_TRACK_DIRTY) )
+# define PGMSHWPTEPAE_IS_P_RW(Pte)              ( ((Pte).uCareful & (X86_PTE_P | X86_PTE_RW | X86_PTE_PAE_MBZ_MASK_NX)) == (X86_PTE_P | X86_PTE_RW) )
+# define PGMSHWPTEPAE_GET_LOG(Pte)              ( (Pte).uCareful )
+# define PGMSHWPTEPAE_GET_HCPHYS(Pte)           ( (Pte).uCareful & X86_PTE_PAE_PG_MASK )
+# define PGMSHWPTEPAE_GET_U(Pte)                ( (Pte).uCareful ) /**< Use with care. */
+# define PGMSHWPTEPAE_SET(Pte, uVal)            do { (Pte).uCareful = (uVal); } while (0)
+# define PGMSHWPTEPAE_SET2(Pte, Pte2)           do { (Pte).uCareful = (Pte2).uCareful; } while (0)
+# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal)     do { ASMAtomicWriteU64(&(Pte).uCareful, (uVal)); } while (0)
+# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2)    do { ASMAtomicWriteU64(&(Pte).uCareful, (Pte2).uCareful); } while (0)
+# define PGMSHWPTEPAE_SET_RO(Pte)               do { (Pte).uCareful &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0)
+# define PGMSHWPTEPAE_SET_RW(Pte)               do { (Pte).uCareful |= X86_PTE_RW; } while (0)
+
+/**
+ * For making sure that u1Present and X86_PTE_P checks doesn't mistake
+ * invalid entries for present.
+ * @sa X86PTPAE.
+ */
+typedef struct PGMSHWPTPAE
+{
+    PGMSHWPTEPAE  a[X86_PG_PAE_ENTRIES];
+} PGMSHWPTPAE;
+
+#else
+typedef X86PTEPAE           PGMSHWPTEPAE;
+typedef X86PTPAE            PGMSHWPTPAE;
+# define PGMSHWPTEPAE_IS_P(Pte)                 ( (Pte).n.u1Present )
+# define PGMSHWPTEPAE_IS_RW(Pte)                ( (Pte).n.u1Write )
+# define PGMSHWPTEPAE_IS_US(Pte)                ( (Pte).n.u1User )
+# define PGMSHWPTEPAE_IS_A(Pte)                 ( (Pte).n.u1Accessed )
+# define PGMSHWPTEPAE_IS_D(Pte)                 ( (Pte).n.u1Dirty )
+# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte)       ( !!((Pte).u & PGM_PTFLAGS_TRACK_DIRTY) )
+# define PGMSHWPTEPAE_IS_P_RW(Pte)              ( ((Pte).u & (X86_PTE_P | X86_PTE_RW)) == (X86_PTE_P | X86_PTE_RW) )
+# define PGMSHWPTEPAE_GET_LOG(Pte)              ( (Pte).u )
+# define PGMSHWPTEPAE_GET_HCPHYS(Pte)           ( (Pte).u & X86_PTE_PAE_PG_MASK )
+# define PGMSHWPTEPAE_GET_U(Pte)                ( (Pte).u ) /**< Use with care. */
+# define PGMSHWPTEPAE_SET(Pte, uVal)            do { (Pte).u = (uVal); } while (0)
+# define PGMSHWPTEPAE_SET2(Pte, Pte2)           do { (Pte).u = (Pte2).u; } while (0)
+# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal)     do { ASMAtomicWriteU64(&(Pte).u, (uVal)); } while (0)
+# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2)    do { ASMAtomicWriteU64(&(Pte).u, (Pte2).u); } while (0)
+# define PGMSHWPTEPAE_SET_RO(Pte)               do { (Pte).u &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0)
+# define PGMSHWPTEPAE_SET_RW(Pte)               do { (Pte).u |= X86_PTE_RW; } while (0)
+
+#endif
+
+/** Pointer to a shadow PAE PTE. */
+typedef PGMSHWPTEPAE       *PPGMSHWPTEPAE;
+/** Pointer to a const shadow PAE PTE. */
+typedef PGMSHWPTEPAE const *PCPGMSHWPTEPAE;
+
+/** Pointer to a shadow PAE page table. */
+typedef PGMSHWPTPAE        *PPGMSHWPTPAE;
+/** Pointer to a const shadow PAE page table. */
+typedef PGMSHWPTPAE const  *PCPGMSHWPTPAE;
+/** @} */
+
+#ifndef PGM_WITHOUT_MAPPINGS
+
+/** Size of the GCPtrConflict array in PGMMAPPING.
+ * @remarks Must be a power of two. */
+# define PGMMAPPING_CONFLICT_MAX         8
+
+/**
+ * Structure for tracking GC Mappings.
+ *
+ * This structure is used by linked list in both GC and HC.
+ */
+typedef struct PGMMAPPING
+{
+    /** Pointer to next entry. */
+    R3PTRTYPE(struct PGMMAPPING *)      pNextR3;
+    /** Pointer to next entry. */
+    R0PTRTYPE(struct PGMMAPPING *)      pNextR0;
+    /** Indicate whether this entry is finalized. */
+    bool                                fFinalized;
+    bool                                afPadding[7];
+    /** Start Virtual address. */
+    RTGCPTR                             GCPtr;
+    /** Last Virtual address (inclusive). */
+    RTGCPTR                             GCPtrLast;
+    /** Range size (bytes). */
+    RTGCPTR                             cb;
+    /** Pointer to relocation callback function. */
+    R3PTRTYPE(PFNPGMRELOCATE)           pfnRelocate;
+    /** User argument to the callback. */
+    R3PTRTYPE(void *)                   pvUser;
+    /** Mapping description / name. For easing debugging. */
+    R3PTRTYPE(const char *)             pszDesc;
+    /** Last 8 addresses that caused conflicts. */
+    RTGCPTR                             aGCPtrConflicts[PGMMAPPING_CONFLICT_MAX];
+    /** Number of conflicts for this hypervisor mapping. */
+    uint32_t                            cConflicts;
+    /** Number of page tables. */
+    uint32_t                            cPTs;
+
+    /** Array of page table mapping data. Each entry
+     * describes one page table. The array can be longer
+     * than the declared length.
+     */
+    struct
+    {
+        /** The HC physical address of the page table. */
+        RTHCPHYS                        HCPhysPT;
+        /** The HC physical address of the first PAE page table. */
+        RTHCPHYS                        HCPhysPaePT0;
+        /** The HC physical address of the second PAE page table. */
+        RTHCPHYS                        HCPhysPaePT1;
+        /** The HC virtual address of the 32-bit page table. */
+        R3PTRTYPE(PX86PT)               pPTR3;
+        /** The HC virtual address of the two PAE page table. (i.e 1024 entries instead of 512) */
+        R3PTRTYPE(PPGMSHWPTPAE)         paPaePTsR3;
+        /** The R0 virtual address of the 32-bit page table. */
+        R0PTRTYPE(PX86PT)               pPTR0;
+        /** The R0 virtual address of the two PAE page table. */
+        R0PTRTYPE(PPGMSHWPTPAE)         paPaePTsR0;
+    } aPTs[1];
+} PGMMAPPING;
+/** Pointer to structure for tracking GC Mappings. */
+typedef struct PGMMAPPING *PPGMMAPPING;
+
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
+
+/**
+ * Physical page access handler type registration.
+ */
+typedef struct PGMPHYSHANDLERTYPEINT
+{
+    /** Number of references.   */
+    uint32_t volatile                   cRefs;
+    /** Magic number (PGMPHYSHANDLERTYPEINT_MAGIC). */
+    uint32_t                            u32Magic;
+    /** Link of handler types anchored in PGMTREES::HeadPhysHandlerTypes.   */
+    RTLISTOFF32NODE                     ListNode;
+    /** The kind of accesses we're handling. */
+    PGMPHYSHANDLERKIND                  enmKind;
+    /** The PGM_PAGE_HNDL_PHYS_STATE_XXX value corresponding to enmKind. */
+    uint32_t                            uState;
+    /** Pointer to R3 callback function. */
+    R3PTRTYPE(PFNPGMPHYSHANDLER)        pfnHandlerR3;
+    /** Pointer to R0 callback function. */
+    R0PTRTYPE(PFNPGMPHYSHANDLER)        pfnHandlerR0;
+    /** Pointer to R0 callback function for \#PFs. */
+    R0PTRTYPE(PFNPGMRZPHYSPFHANDLER)    pfnPfHandlerR0;
+    /** Description / Name. For easing debugging. */
+    R3PTRTYPE(const char *)             pszDesc;
+} PGMPHYSHANDLERTYPEINT;
+/** Pointer to a physical access handler type registration. */
+typedef PGMPHYSHANDLERTYPEINT *PPGMPHYSHANDLERTYPEINT;
+/** Magic value for the physical handler callbacks (Robert A. Heinlein). */
+#define PGMPHYSHANDLERTYPEINT_MAGIC        UINT32_C(0x19070707)
+/** Magic value for the physical handler callbacks. */
+#define PGMPHYSHANDLERTYPEINT_MAGIC_DEAD   UINT32_C(0x19880508)
+
+/**
+ * Converts a handle to a pointer.
+ * @returns PPGMPHYSHANDLERTYPEINT
+ * @param   a_pVM           The cross context VM structure.
+ * @param   a_hType         Physical access handler type handle.
+ */
+#define PGMPHYSHANDLERTYPEINT_FROM_HANDLE(a_pVM, a_hType) ((PPGMPHYSHANDLERTYPEINT)MMHyperHeapOffsetToPtr(a_pVM, a_hType))
+
+
+/**
+ * Physical page access handler structure.
+ *
+ * This is used to keep track of physical address ranges
+ * which are being monitored in some kind of way.
+ */
+typedef struct PGMPHYSHANDLER
+{
+    AVLROGCPHYSNODECORE                 Core;
+    /** Number of pages to update. */
+    uint32_t                            cPages;
+    /** Set if we have pages that have been aliased. */
+    uint32_t                            cAliasedPages;
+    /** Set if we have pages that have temporarily been disabled. */
+    uint32_t                            cTmpOffPages;
+    /** Registered handler type handle (heap offset). */
+    PGMPHYSHANDLERTYPE                  hType;
+    /** User argument for R3 handlers. */
+    R3PTRTYPE(void *)                   pvUserR3;
+    /** User argument for R0 handlers. */
+    R0PTRTYPE(void *)                   pvUserR0;
+    /** Description / Name. For easing debugging. */
+    R3PTRTYPE(const char *)             pszDesc;
+#ifdef VBOX_WITH_STATISTICS
+    /** Profiling of this handler. */
+    STAMPROFILE                         Stat;
+#endif
+} PGMPHYSHANDLER;
+/** Pointer to a physical page access handler structure. */
+typedef PGMPHYSHANDLER *PPGMPHYSHANDLER;
+
+/**
+ * Gets the type record for a physical handler (no reference added).
+ * @returns PPGMPHYSHANDLERTYPEINT
+ * @param   a_pVM           The cross context VM structure.
+ * @param   a_pPhysHandler  Pointer to the physical handler structure
+ *                          (PGMPHYSHANDLER).
+ */
+#define PGMPHYSHANDLER_GET_TYPE(a_pVM, a_pPhysHandler) PGMPHYSHANDLERTYPEINT_FROM_HANDLE(a_pVM, (a_pPhysHandler)->hType)
+
+
+/**
+ * A Physical Guest Page tracking structure.
+ *
+ * The format of this structure is complicated because we have to fit a lot
+ * of information into as few bits as possible. The format is also subject
+ * to change (there is one coming up soon). Which means that for we'll be
+ * using PGM_PAGE_GET_*, PGM_PAGE_IS_ and PGM_PAGE_SET_* macros for *all*
+ * accesses to the structure.
+ */
+typedef union PGMPAGE
+{
+    /** Structured view. */
+    struct
+    {
+        /** 1:0   - The physical handler state (PGM_PAGE_HNDL_PHYS_STATE_*). */
+        uint64_t    u2HandlerPhysStateY : 2;
+        /** 3:2   - Paging structure needed to map the page
+         * (PGM_PAGE_PDE_TYPE_*). */
+        uint64_t    u2PDETypeY          : 2;
+        /** 4     - Unused (was used by FTE for dirty tracking). */
+        uint64_t    fUnused1            : 1;
+        /** 5     - Flag indicating that a write monitored page was written to
+         *  when set. */
+        uint64_t    fWrittenToY         : 1;
+        /** 7:6   - Unused. */
+        uint64_t    u2Unused0           : 2;
+        /** 9:8   - Unused (was used by PGM_PAGE_HNDL_VIRT_STATE_*). */
+        uint64_t    u2Unused1           : 2;
+        /** 11:10 - NEM state bits. */
+        uint64_t    u2NemStateY         : 2;
+        /** 12:48 - The host physical frame number (shift left to get the
+         *  address). */
+        uint64_t    HCPhysFN            : 36;
+        /** 50:48 - The page state. */
+        uint64_t    uStateY             : 3;
+        /** 51:53 - The page type (PGMPAGETYPE). */
+        uint64_t    uTypeY              : 3;
+        /** 63:54 - PTE index for usage tracking (page pool). */
+        uint64_t    u10PteIdx           : 10;
+
+        /** The GMM page ID.
+         * @remarks In the current implementation, MMIO2 and pages aliased to
+         *          MMIO2 pages will be exploiting this field to calculate the
+         *          ring-3 mapping address corresponding to the page.
+         *          Later we may consider including MMIO2 management into GMM. */
+        uint32_t    idPage;
+        /** Usage tracking (page pool). */
+        uint16_t    u16TrackingY;
+        /** The number of read locks on this page. */
+        uint8_t     cReadLocksY;
+        /** The number of write locks on this page. */
+        uint8_t     cWriteLocksY;
+    } s;
+
+    /** 64-bit integer view. */
+    uint64_t    au64[2];
+    /** 16-bit view. */
+    uint32_t    au32[4];
+    /** 16-bit view. */
+    uint16_t    au16[8];
+    /** 8-bit view. */
+    uint8_t     au8[16];
+} PGMPAGE;
+AssertCompileSize(PGMPAGE, 16);
+/** Pointer to a physical guest page. */
+typedef PGMPAGE *PPGMPAGE;
+/** Pointer to a const physical guest page. */
+typedef const PGMPAGE *PCPGMPAGE;
+/** Pointer to a physical guest page pointer. */
+typedef PPGMPAGE *PPPGMPAGE;
+
+
+/**
+ * Clears the page structure.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_CLEAR(a_pPage) \
+    do { \
+        (a_pPage)->au64[0] = 0; \
+        (a_pPage)->au64[1] = 0; \
+    } while (0)
+
+/**
+ * Initializes the page structure.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ * @param   a_HCPhys    The host physical address of the page.
+ * @param   a_idPage    The (GMM) page ID of the page.
+ * @param   a_uType     The page type (PGMPAGETYPE).
+ * @param   a_uState    The page state (PGM_PAGE_STATE_XXX).
+ */
+#define PGM_PAGE_INIT(a_pPage, a_HCPhys, a_idPage, a_uType, a_uState) \
+    do { \
+        RTHCPHYS SetHCPhysTmp = (a_HCPhys); \
+        AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
+        (a_pPage)->au64[0]           = SetHCPhysTmp; \
+        (a_pPage)->au64[1]           = 0; \
+        (a_pPage)->s.idPage          = (a_idPage); \
+        (a_pPage)->s.uStateY         = (a_uState); \
+        (a_pPage)->s.uTypeY          = (a_uType); \
+    } while (0)
+
+/**
+ * Initializes the page structure of a ZERO page.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ * @param   a_pVM       The VM handle (for getting the zero page address).
+ * @param   a_uType     The page type (PGMPAGETYPE).
+ */
+#define PGM_PAGE_INIT_ZERO(a_pPage, a_pVM, a_uType)  \
+    PGM_PAGE_INIT((a_pPage), (a_pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (a_uType), PGM_PAGE_STATE_ZERO)
+
+
+/** @name The Page state, PGMPAGE::uStateY.
+ * @{ */
+/** The zero page.
+ * This is a per-VM page that's never ever mapped writable. */
+#define PGM_PAGE_STATE_ZERO             0U
+/** A allocated page.
+ * This is a per-VM page allocated from the page pool (or wherever
+ * we get MMIO2 pages from if the type is MMIO2).
+ */
+#define PGM_PAGE_STATE_ALLOCATED        1U
+/** A allocated page that's being monitored for writes.
+ * The shadow page table mappings are read-only. When a write occurs, the
+ * fWrittenTo member is set, the page remapped as read-write and the state
+ * moved back to allocated. */
+#define PGM_PAGE_STATE_WRITE_MONITORED  2U
+/** The page is shared, aka. copy-on-write.
+ * This is a page that's shared with other VMs. */
+#define PGM_PAGE_STATE_SHARED           3U
+/** The page is ballooned, so no longer available for this VM. */
+#define PGM_PAGE_STATE_BALLOONED        4U
+/** @} */
+
+
+/** Asserts lock ownership in some of the PGM_PAGE_XXX macros. */
+#if defined(VBOX_STRICT) && 0 /** @todo triggers in pgmRZDynMapGCPageV2Inlined */
+# define PGM_PAGE_ASSERT_LOCK(a_pVM)  PGM_LOCK_ASSERT_OWNER(a_pVM)
+#else
+# define PGM_PAGE_ASSERT_LOCK(a_pVM)  do { } while (0)
+#endif
+
+/**
+ * Gets the page state.
+ * @returns page state (PGM_PAGE_STATE_*).
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ *
+ * @remarks See PGM_PAGE_GET_HCPHYS_NA for remarks about GCC and strict
+ *          builds.
+ */
+#define PGM_PAGE_GET_STATE_NA(a_pPage)          ( (a_pPage)->s.uStateY )
+#if defined(__GNUC__) && defined(VBOX_STRICT)
+# define PGM_PAGE_GET_STATE(a_pPage)        __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_STATE_NA(a_pPage); })
+#else
+# define PGM_PAGE_GET_STATE                     PGM_PAGE_GET_STATE_NA
+#endif
+
+/**
+ * Sets the page state.
+ * @param   a_pVM       The VM handle, only used for lock ownership assertions.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ * @param   a_uState    The new page state.
+ */
+#define PGM_PAGE_SET_STATE(a_pVM, a_pPage, a_uState) \
+    do { (a_pPage)->s.uStateY = (a_uState); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
+
+
+/**
+ * Gets the host physical address of the guest page.
+ * @returns host physical address (RTHCPHYS).
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ *
+ * @remarks In strict builds on gcc platforms, this macro will make some ugly
+ *          assumption about a valid pVM variable/parameter being in the
+ *          current context.  It will use this pVM variable to assert that the
+ *          PGM lock is held.  Use the PGM_PAGE_GET_HCPHYS_NA in contexts where
+ *          pVM is not around.
+ */
+#if 0
+# define PGM_PAGE_GET_HCPHYS_NA(a_pPage)        ( (a_pPage)->s.HCPhysFN << 12 )
+# define PGM_PAGE_GET_HCPHYS                    PGM_PAGE_GET_HCPHYS_NA
+#else
+# define PGM_PAGE_GET_HCPHYS_NA(a_pPage)        ( (a_pPage)->au64[0] & UINT64_C(0x0000fffffffff000) )
+# if defined(__GNUC__) && defined(VBOX_STRICT)
+#  define PGM_PAGE_GET_HCPHYS(a_pPage)      __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_HCPHYS_NA(a_pPage); })
+# else
+#  define PGM_PAGE_GET_HCPHYS                   PGM_PAGE_GET_HCPHYS_NA
+# endif
+#endif
+
+/**
+ * Sets the host physical address of the guest page.
+ *
+ * @param   a_pVM       The VM handle, only used for lock ownership assertions.
+ * @param   a_pPage      Pointer to the physical guest page tracking structure.
+ * @param   a_HCPhys     The new host physical address.
+ */
+#define PGM_PAGE_SET_HCPHYS(a_pVM, a_pPage, a_HCPhys) \
+    do { \
+        RTHCPHYS const SetHCPhysTmp = (a_HCPhys); \
+        AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
+        (a_pPage)->s.HCPhysFN = SetHCPhysTmp >> 12; \
+        PGM_PAGE_ASSERT_LOCK(a_pVM); \
+    } while (0)
+
+/**
+ * Get the Page ID.
+ * @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_PAGEID(a_pPage)            (  (uint32_t)(a_pPage)->s.idPage )
+
+/**
+ * Sets the Page ID.
+ * @param   a_pVM       The VM handle, only used for lock ownership assertions.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ * @param   a_idPage    The new page ID.
+ */
+#define PGM_PAGE_SET_PAGEID(a_pVM, a_pPage, a_idPage) \
+    do { \
+        (a_pPage)->s.idPage = (a_idPage); \
+        PGM_PAGE_ASSERT_LOCK(a_pVM); \
+    } while (0)
+
+/**
+ * Get the Chunk ID.
+ * @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_CHUNKID(a_pPage)           ( PGM_PAGE_GET_PAGEID(a_pPage) >> GMM_CHUNKID_SHIFT )
+
+/**
+ * Get the index of the page within the allocation chunk.
+ * @returns The page index.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_PAGE_IN_CHUNK(a_pPage)     ( PGM_PAGE_GET_PAGEID(a_pPage) & GMM_PAGEID_IDX_MASK )
+
+/**
+ * Gets the page type.
+ * @returns The page type.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ *
+ * @remarks See PGM_PAGE_GET_HCPHYS_NA for remarks about GCC and strict
+ *          builds.
+ */
+#define PGM_PAGE_GET_TYPE_NA(a_pPage)           ( (a_pPage)->s.uTypeY )
+#if defined(__GNUC__) && defined(VBOX_STRICT)
+# define PGM_PAGE_GET_TYPE(a_pPage)         __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_TYPE_NA(a_pPage); })
+#else
+# define PGM_PAGE_GET_TYPE                      PGM_PAGE_GET_TYPE_NA
+#endif
+
+/**
+ * Sets the page type.
+ *
+ * @param   a_pVM       The VM handle, only used for lock ownership assertions.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ * @param   a_enmType   The new page type (PGMPAGETYPE).
+ */
+#define PGM_PAGE_SET_TYPE(a_pVM, a_pPage, a_enmType) \
+    do { (a_pPage)->s.uTypeY = (a_enmType); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
+
+/**
+ * Gets the page table index
+ * @returns The page table index.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_PTE_INDEX(a_pPage)         ( (a_pPage)->s.u10PteIdx )
+
+/**
+ * Sets the page table index.
+ * @param   a_pVM       The VM handle, only used for lock ownership assertions.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ * @param   a_iPte      New page table index.
+ */
+#define PGM_PAGE_SET_PTE_INDEX(a_pVM, a_pPage, a_iPte) \
+    do { (a_pPage)->s.u10PteIdx = (a_iPte); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
+
+/**
+ * Checks if the page is marked for MMIO, no MMIO2 aliasing.
+ * @returns true/false.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_MMIO(a_pPage)               ( (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO )
+
+/**
+ * Checks if the page is marked for MMIO, including both aliases.
+ * @returns true/false.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_MMIO_OR_ALIAS(a_pPage)      (   (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO \
+                                                 || (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO2_ALIAS_MMIO \
+                                                 || (a_pPage)->s.uTypeY == PGMPAGETYPE_SPECIAL_ALIAS_MMIO \
+                                                 )
+
+/**
+ * Checks if the page is marked for MMIO, including special aliases.
+ * @returns true/false.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(a_pPage) (   (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO \
+                                                    || (a_pPage)->s.uTypeY == PGMPAGETYPE_SPECIAL_ALIAS_MMIO )
+
+/**
+ * Checks if the page is a special aliased MMIO page.
+ * @returns true/false.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(a_pPage) ( (a_pPage)->s.uTypeY == PGMPAGETYPE_SPECIAL_ALIAS_MMIO )
+
+/**
+ * Checks if the page is backed by the ZERO page.
+ * @returns true/false.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_ZERO(a_pPage)               ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_ZERO )
+
+/**
+ * Checks if the page is backed by a SHARED page.
+ * @returns true/false.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_SHARED(a_pPage)             ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_SHARED )
+
+/**
+ * Checks if the page is ballooned.
+ * @returns true/false.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_BALLOONED(a_pPage)          ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_BALLOONED )
+
+/**
+ * Checks if the page is allocated.
+ * @returns true/false.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_ALLOCATED(a_pPage)          ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_ALLOCATED )
+
+/**
+ * Marks the page as written to (for GMM change monitoring).
+ * @param   a_pVM       The VM handle, only used for lock ownership assertions.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_SET_WRITTEN_TO(a_pVM, a_pPage) \
+    do { (a_pPage)->s.fWrittenToY = 1; PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
+
+/**
+ * Clears the written-to indicator.
+ * @param   a_pVM       The VM handle, only used for lock ownership assertions.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_CLEAR_WRITTEN_TO(a_pVM, a_pPage) \
+    do { (a_pPage)->s.fWrittenToY = 0; PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
+
+/**
+ * Checks if the page was marked as written-to.
+ * @returns true/false.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_WRITTEN_TO(a_pPage)         ( (a_pPage)->s.fWrittenToY )
+
+
+/** @name PT usage values (PGMPAGE::u2PDEType).
+ *
+ * @{ */
+/** Either as a PT or PDE. */
+#define PGM_PAGE_PDE_TYPE_DONTCARE              0
+/** Must use a page table to map the range. */
+#define PGM_PAGE_PDE_TYPE_PT                    1
+/** Can use a page directory entry to map the continuous range. */
+#define PGM_PAGE_PDE_TYPE_PDE                   2
+/** Can use a page directory entry to map the continuous range - temporarily disabled (by page monitoring). */
+#define PGM_PAGE_PDE_TYPE_PDE_DISABLED          3
+/** @} */
+
+/**
+ * Set the PDE type of the page
+ * @param   a_pVM       The VM handle, only used for lock ownership assertions.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ * @param   a_uType     PGM_PAGE_PDE_TYPE_*.
+ */
+#define PGM_PAGE_SET_PDE_TYPE(a_pVM, a_pPage, a_uType) \
+    do { (a_pPage)->s.u2PDETypeY = (a_uType); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
+
+/**
+ * Checks if the page was marked being part of a large page
+ * @returns true/false.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_PDE_TYPE(a_pPage)          ( (a_pPage)->s.u2PDETypeY )
+
+/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateY).
+ *
+ * @remarks The values are assigned in order of priority, so we can calculate
+ *          the correct state for a page with different handlers installed.
+ * @{ */
+/** No handler installed. */
+#define PGM_PAGE_HNDL_PHYS_STATE_NONE           0
+/** Monitoring is temporarily disabled. */
+#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED       1
+/** Write access is monitored. */
+#define PGM_PAGE_HNDL_PHYS_STATE_WRITE          2
+/** All access is monitored. */
+#define PGM_PAGE_HNDL_PHYS_STATE_ALL            3
+/** @} */
+
+/**
+ * Gets the physical access handler state of a page.
+ * @returns PGM_PAGE_HNDL_PHYS_STATE_* value.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage)   ( (a_pPage)->s.u2HandlerPhysStateY )
+
+/**
+ * Sets the physical access handler state of a page.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ * @param   a_uState    The new state value.
+ */
+#define PGM_PAGE_SET_HNDL_PHYS_STATE(a_pPage, a_uState) \
+    do { (a_pPage)->s.u2HandlerPhysStateY = (a_uState); } while (0)
+
+/**
+ * Checks if the page has any physical access handlers, including temporarily disabled ones.
+ * @returns true/false
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(a_pPage) \
+    ( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE )
+
+/**
+ * Checks if the page has any active physical access handlers.
+ * @returns true/false
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(a_pPage) \
+    ( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
+
+/**
+ * Checks if the page has any access handlers, including temporarily disabled ones.
+ * @returns true/false
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_HAS_ANY_HANDLERS(a_pPage) \
+    ( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE )
+
+/**
+ * Checks if the page has any active access handlers.
+ * @returns true/false
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_HAS_ACTIVE_HANDLERS(a_pPage) \
+     (PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
+
+/**
+ * Checks if the page has any active access handlers catching all accesses.
+ * @returns true/false
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(a_pPage) \
+    ( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL )
+
+
+/** @def PGM_PAGE_GET_TRACKING
+ * Gets the packed shadow page pool tracking data associated with a guest page.
+ * @returns uint16_t containing the data.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_TRACKING_NA(a_pPage)       ( (a_pPage)->s.u16TrackingY )
+#if defined(__GNUC__) && defined(VBOX_STRICT)
+# define PGM_PAGE_GET_TRACKING(a_pPage)     __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_TRACKING_NA(a_pPage); })
+#else
+# define PGM_PAGE_GET_TRACKING                  PGM_PAGE_GET_TRACKING_NA
+#endif
+
+/** @def PGM_PAGE_SET_TRACKING
+ * Sets the packed shadow page pool tracking data associated with a guest page.
+ * @param   a_pVM       The VM handle, only used for lock ownership assertions.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ * @param   a_u16TrackingData   The tracking data to store.
+ */
+#define PGM_PAGE_SET_TRACKING(a_pVM, a_pPage, a_u16TrackingData) \
+    do { (a_pPage)->s.u16TrackingY = (a_u16TrackingData); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
+
+/** @def PGM_PAGE_GET_TD_CREFS
+ * Gets the @a cRefs tracking data member.
+ * @returns cRefs.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_TD_CREFS(a_pPage) \
+    ((PGM_PAGE_GET_TRACKING(a_pPage) >> PGMPOOL_TD_CREFS_SHIFT)    & PGMPOOL_TD_CREFS_MASK)
+#define PGM_PAGE_GET_TD_CREFS_NA(a_pPage) \
+    ((PGM_PAGE_GET_TRACKING_NA(a_pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
+
+/** @def PGM_PAGE_GET_TD_IDX
+ * Gets the @a idx tracking data member.
+ * @returns idx.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_TD_IDX(a_pPage) \
+    ((PGM_PAGE_GET_TRACKING(a_pPage) >> PGMPOOL_TD_IDX_SHIFT)      & PGMPOOL_TD_IDX_MASK)
+#define PGM_PAGE_GET_TD_IDX_NA(a_pPage) \
+    ((PGM_PAGE_GET_TRACKING_NA(a_pPage) >> PGMPOOL_TD_IDX_SHIFT)   & PGMPOOL_TD_IDX_MASK)
+
+
+/** Max number of locks on a page. */
+#define PGM_PAGE_MAX_LOCKS                      UINT8_C(254)
+
+/** Get the read lock count.
+ * @returns count.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_READ_LOCKS(a_pPage)        ( (a_pPage)->s.cReadLocksY )
+
+/** Get the write lock count.
+ * @returns count.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_WRITE_LOCKS(a_pPage)       ( (a_pPage)->s.cWriteLocksY )
+
+/** Decrement the read lock counter.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_DEC_READ_LOCKS(a_pPage)        do { --(a_pPage)->s.cReadLocksY; } while (0)
+
+/** Decrement the write lock counter.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_DEC_WRITE_LOCKS(a_pPage)       do { --(a_pPage)->s.cWriteLocksY; } while (0)
+
+/** Increment the read lock counter.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_INC_READ_LOCKS(a_pPage)        do { ++(a_pPage)->s.cReadLocksY; } while (0)
+
+/** Increment the write lock counter.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_INC_WRITE_LOCKS(a_pPage)       do { ++(a_pPage)->s.cWriteLocksY; } while (0)
+
+
+/** Gets the NEM state.
+ * @returns NEM state value (two bits).
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_NEM_STATE(a_pPage)         ((a_pPage)->s.u2NemStateY)
+
+/** Sets the NEM state.
+ * @param   a_pPage     Pointer to the physical guest page tracking structure.
+ * @param   a_u2State   The NEM state value (specific to NEM impl.).
+ */
+#define PGM_PAGE_SET_NEM_STATE(a_pPage, a_u2State) \
+    do { Assert((a_u2State) < 4); (a_pPage)->s.u2NemStateY = (a_u2State); } while (0)
+
+
+#if 0
+/** Enables sanity checking of write monitoring using CRC-32. */
+# define PGMLIVESAVERAMPAGE_WITH_CRC32
+#endif
+
+/**
+ * Per page live save tracking data.
+ */
+typedef struct PGMLIVESAVERAMPAGE
+{
+    /** Number of times it has been dirtied. */
+    uint32_t    cDirtied : 24;
+    /** Whether it is currently dirty. */
+    uint32_t    fDirty : 1;
+    /** Ignore the page.
+     *  This is used for pages that has been MMIO, MMIO2 or ROM pages once.  We will
+     *  deal with these after pausing the VM and DevPCI have said it bit about
+     *  remappings. */
+    uint32_t    fIgnore : 1;
+    /** Was a ZERO page last time around. */
+    uint32_t    fZero : 1;
+    /** Was a SHARED page last time around. */
+    uint32_t    fShared : 1;
+    /** Whether the page is/was write monitored in a previous pass. */
+    uint32_t    fWriteMonitored : 1;
+    /** Whether the page is/was write monitored earlier in this pass. */
+    uint32_t    fWriteMonitoredJustNow : 1;
+    /** Bits reserved for future use. */
+    uint32_t    u2Reserved : 2;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+    /** CRC-32 for the page. This is for internal consistency checks. */
+    uint32_t    u32Crc;
+#endif
+} PGMLIVESAVERAMPAGE;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+AssertCompileSize(PGMLIVESAVERAMPAGE, 8);
+#else
+AssertCompileSize(PGMLIVESAVERAMPAGE, 4);
+#endif
+/** Pointer to the per page live save tracking data. */
+typedef PGMLIVESAVERAMPAGE *PPGMLIVESAVERAMPAGE;
+
+/** The max value of PGMLIVESAVERAMPAGE::cDirtied. */
+#define PGMLIVSAVEPAGE_MAX_DIRTIED 0x00fffff0
+
+
+/**
+ * RAM range for GC Phys to HC Phys conversion.
+ *
+ * Can be used for HC Virt to GC Phys and HC Virt to HC Phys
+ * conversions too, but we'll let MM handle that for now.
+ *
+ * This structure is used by linked lists in both GC and HC.
+ */
+typedef struct PGMRAMRANGE
+{
+    /** Start of the range. Page aligned. */
+    RTGCPHYS                            GCPhys;
+    /** Size of the range. (Page aligned of course). */
+    RTGCPHYS                            cb;
+    /** Pointer to the next RAM range - for R3. */
+    R3PTRTYPE(struct PGMRAMRANGE *)     pNextR3;
+    /** Pointer to the next RAM range - for R0. */
+    R0PTRTYPE(struct PGMRAMRANGE *)     pNextR0;
+    /** PGM_RAM_RANGE_FLAGS_* flags. */
+    uint32_t                            fFlags;
+    uint32_t                            fPadding1;
+    /** Last address in the range (inclusive). Page aligned (-1). */
+    RTGCPHYS                            GCPhysLast;
+    /** Start of the HC mapping of the range. This is only used for MMIO2. */
+    R3PTRTYPE(void *)                   pvR3;
+    /** Live save per page tracking data. */
+    R3PTRTYPE(PPGMLIVESAVERAMPAGE)      paLSPages;
+    /** The range description. */
+    R3PTRTYPE(const char *)             pszDesc;
+    /** Pointer to self - R0 pointer. */
+    R0PTRTYPE(struct PGMRAMRANGE *)     pSelfR0;
+
+    /** Pointer to the left search three node - ring-3 context. */
+    R3PTRTYPE(struct PGMRAMRANGE *)     pLeftR3;
+    /** Pointer to the right search three node - ring-3 context. */
+    R3PTRTYPE(struct PGMRAMRANGE *)     pRightR3;
+    /** Pointer to the left search three node - ring-0 context. */
+    R0PTRTYPE(struct PGMRAMRANGE *)     pLeftR0;
+    /** Pointer to the right search three node - ring-0 context. */
+    R0PTRTYPE(struct PGMRAMRANGE *)     pRightR0;
+
+    /** Padding to make aPage aligned on sizeof(PGMPAGE). */
+#if HC_ARCH_BITS == 32
+    uint32_t                            au32Alignment2[HC_ARCH_BITS == 32 ? 2 : 0];
+#endif
+    /** Array of physical guest page tracking structures. */
+    PGMPAGE                             aPages[1];
+} PGMRAMRANGE;
+/** Pointer to RAM range for GC Phys to HC Phys conversion. */
+typedef PGMRAMRANGE *PPGMRAMRANGE;
+
+/** @name PGMRAMRANGE::fFlags
+ * @{ */
+/** The RAM range is floating around as an independent guest mapping. */
+#define PGM_RAM_RANGE_FLAGS_FLOATING        RT_BIT(20)
+/** Ad hoc RAM range for an ROM mapping. */
+#define PGM_RAM_RANGE_FLAGS_AD_HOC_ROM      RT_BIT(21)
+/** Ad hoc RAM range for an MMIO mapping. */
+#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO     RT_BIT(22)
+/** Ad hoc RAM range for an MMIO2 or pre-registered MMIO mapping. */
+#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX  RT_BIT(23)
+/** @} */
+
+/** Tests if a RAM range is an ad hoc one or not.
+ * @returns true/false.
+ * @param   pRam    The RAM range.
+ */
+#define PGM_RAM_RANGE_IS_AD_HOC(pRam) \
+    (!!( (pRam)->fFlags & (PGM_RAM_RANGE_FLAGS_AD_HOC_ROM | PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO | PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX) ) )
+
+/** The number of entries in the RAM range TLBs (there is one for each
+ *  context).  Must be a power of two. */
+#define PGM_RAMRANGE_TLB_ENTRIES            8
+
+/**
+ * Calculates the RAM range TLB index for the physical address.
+ *
+ * @returns RAM range TLB index.
+ * @param   a_GCPhys    The guest physical address.
+ */
+#define PGM_RAMRANGE_TLB_IDX(a_GCPhys)      ( ((a_GCPhys) >> 20) & (PGM_RAMRANGE_TLB_ENTRIES - 1) )
+
+
+
+/**
+ * Per page tracking structure for ROM image.
+ *
+ * A ROM image may have a shadow page, in which case we may have two pages
+ * backing it.  This structure contains the PGMPAGE for both while
+ * PGMRAMRANGE have a copy of the active one.  It is important that these
+ * aren't out of sync in any regard other than page pool tracking data.
+ */
+typedef struct PGMROMPAGE
+{
+    /** The page structure for the virgin ROM page. */
+    PGMPAGE     Virgin;
+    /** The page structure for the shadow RAM page. */
+    PGMPAGE     Shadow;
+    /** The current protection setting. */
+    PGMROMPROT  enmProt;
+    /** Live save status information. Makes use of unused alignment space. */
+    struct
+    {
+        /** The previous protection value. */
+        uint8_t u8Prot;
+        /** Written to flag set by the handler. */
+        bool    fWrittenTo;
+        /** Whether the shadow page is dirty or not. */
+        bool    fDirty;
+        /** Whether it was dirtied in the recently. */
+        bool    fDirtiedRecently;
+    } LiveSave;
+} PGMROMPAGE;
+AssertCompileSizeAlignment(PGMROMPAGE, 8);
+/** Pointer to a ROM page tracking structure. */
+typedef PGMROMPAGE *PPGMROMPAGE;
+
+
+/**
+ * A registered ROM image.
+ *
+ * This is needed to keep track of ROM image since they generally intrude
+ * into a PGMRAMRANGE.  It also keeps track of additional info like the
+ * two page sets (read-only virgin and read-write shadow), the current
+ * state of each page.
+ *
+ * Because access handlers cannot easily be executed in a different
+ * context, the ROM ranges needs to be accessible and in all contexts.
+ */
+typedef struct PGMROMRANGE
+{
+    /** Pointer to the next range - R3. */
+    R3PTRTYPE(struct PGMROMRANGE *)     pNextR3;
+    /** Pointer to the next range - R0. */
+    R0PTRTYPE(struct PGMROMRANGE *)     pNextR0;
+    /** Address of the range. */
+    RTGCPHYS                            GCPhys;
+    /** Address of the last byte in the range. */
+    RTGCPHYS                            GCPhysLast;
+    /** Size of the range. */
+    RTGCPHYS                            cb;
+    /** The flags (PGMPHYS_ROM_FLAGS_*). */
+    uint32_t                            fFlags;
+    /** The saved state range ID. */
+    uint8_t                             idSavedState;
+    /** Alignment padding. */
+    uint8_t                             au8Alignment[3];
+    /** Alignment padding ensuring that aPages is sizeof(PGMROMPAGE) aligned. */
+    uint32_t                            au32Alignemnt[HC_ARCH_BITS == 32 ? 5 : 1];
+    /** The size bits pvOriginal points to. */
+    uint32_t                            cbOriginal;
+    /** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified.
+     * This is used for strictness checks. */
+    R3PTRTYPE(const void *)             pvOriginal;
+    /** The ROM description. */
+    R3PTRTYPE(const char *)             pszDesc;
+    /** The per page tracking structures. */
+    PGMROMPAGE                          aPages[1];
+} PGMROMRANGE;
+/** Pointer to a ROM range. */
+typedef PGMROMRANGE *PPGMROMRANGE;
+
+
+/**
+ * Live save per page data for an MMIO2 page.
+ *
+ * Not using PGMLIVESAVERAMPAGE here because we cannot use normal write monitoring
+ * of MMIO2 pages.  The current approach is using some optimistic SHA-1 +
+ * CRC-32 for detecting changes as well as special handling of zero pages.  This
+ * is a TEMPORARY measure which isn't perfect, but hopefully it is good enough
+ * for speeding things up.  (We're using SHA-1 and not SHA-256 or SHA-512
+ * because of speed (2.5x and 6x slower).)
+ *
+ * @todo Implement dirty MMIO2 page reporting that can be enabled during live
+ *       save but normally is disabled.  Since we can write monitor guest
+ *       accesses on our own, we only need this for host accesses.  Shouldn't be
+ *       too difficult for DevVGA, VMMDev might be doable, the planned
+ *       networking fun will be fun since it involves ring-0.
+ */
+typedef struct PGMLIVESAVEMMIO2PAGE
+{
+    /** Set if the page is considered dirty. */
+    bool        fDirty;
+    /** The number of scans this page has remained unchanged for.
+     * Only updated for dirty pages. */
+    uint8_t     cUnchangedScans;
+    /** Whether this page was zero at the last scan. */
+    bool        fZero;
+    /** Alignment padding. */
+    bool        fReserved;
+    /** CRC-32 for the first half of the page.
+     * This is used together with u32CrcH2 to quickly detect changes in the page
+     * during the non-final passes. */
+    uint32_t    u32CrcH1;
+    /** CRC-32 for the second half of the page. */
+    uint32_t    u32CrcH2;
+    /** SHA-1 for the saved page.
+     * This is used in the final pass to skip pages without changes. */
+    uint8_t     abSha1Saved[RTSHA1_HASH_SIZE];
+} PGMLIVESAVEMMIO2PAGE;
+/** Pointer to a live save status data for an MMIO2 page. */
+typedef PGMLIVESAVEMMIO2PAGE *PPGMLIVESAVEMMIO2PAGE;
+
+/**
+ * A registered MMIO2 (= Device RAM) range.
+ *
+ * There are a few reason why we need to keep track of these registrations.  One
+ * of them is the deregistration & cleanup stuff, while another is that the
+ * PGMRAMRANGE associated with such a region may have to be removed from the ram
+ * range list.
+ *
+ * Overlapping with a RAM range has to be 100% or none at all.  The pages in the
+ * existing RAM range must not be ROM nor MMIO.  A guru meditation will be
+ * raised if a partial overlap or an overlap of ROM pages is encountered.  On an
+ * overlap we will free all the existing RAM pages and put in the ram range
+ * pages instead.
+ */
+typedef struct PGMREGMMIO2RANGE
+{
+    /** The owner of the range. (a device) */
+    PPDMDEVINSR3                        pDevInsR3;
+    /** Pointer to the ring-3 mapping of the allocation. */
+    RTR3PTR                             pvR3;
+#if defined(VBOX_WITH_RAM_IN_KERNEL) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+    /** Pointer to the ring-0 mapping of the allocation. */
+    RTR0PTR                             pvR0;
+#endif
+    /** Pointer to the next range - R3. */
+    R3PTRTYPE(struct PGMREGMMIO2RANGE *) pNextR3;
+    /** Flags (PGMREGMMIO2RANGE_F_XXX). */
+    uint16_t                            fFlags;
+    /** The sub device number (internal PCI config (CFGM) number). */
+    uint8_t                             iSubDev;
+    /** The PCI region number. */
+    uint8_t                             iRegion;
+    /** The saved state range ID. */
+    uint8_t                             idSavedState;
+    /** MMIO2 range identifier, for page IDs (PGMPAGE::s.idPage). */
+    uint8_t                             idMmio2;
+    /** Alignment padding for putting the ram range on a PGMPAGE alignment boundary. */
+#if defined(VBOX_WITH_RAM_IN_KERNEL) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+    uint8_t                             abAlignment[HC_ARCH_BITS == 32 ? 6 + 4 : 2];
+#else
+    uint8_t                             abAlignment[HC_ARCH_BITS == 32 ? 6 + 8 : 2 + 8];
+#endif
+    /** The real size.
+     * This may be larger than indicated by RamRange.cb if the range has been
+     * reduced during saved state loading. */
+    RTGCPHYS                            cbReal;
+    /** Pointer to the physical handler for MMIO. */
+    R3PTRTYPE(PPGMPHYSHANDLER)          pPhysHandlerR3;
+    /** Live save per page tracking data for MMIO2. */
+    R3PTRTYPE(PPGMLIVESAVEMMIO2PAGE)    paLSPages;
+    /** The associated RAM range. */
+    PGMRAMRANGE                         RamRange;
+} PGMREGMMIO2RANGE;
+AssertCompileMemberAlignment(PGMREGMMIO2RANGE, RamRange, 16);
+/** Pointer to a MMIO2 or pre-registered MMIO range. */
+typedef PGMREGMMIO2RANGE *PPGMREGMMIO2RANGE;
+
+/** @name PGMREGMMIO2RANGE_F_XXX - Registered MMIO2 range flags.
+ * @{ */
+/** Set if it's an MMIO2 range.
+ * @note Historical.  For a while we did some of the MMIO this way too.  */
+#define PGMREGMMIO2RANGE_F_MMIO2            UINT16_C(0x0001)
+/** Set if this is the first chunk in the MMIO2 range. */
+#define PGMREGMMIO2RANGE_F_FIRST_CHUNK      UINT16_C(0x0002)
+/** Set if this is the last chunk in the MMIO2 range. */
+#define PGMREGMMIO2RANGE_F_LAST_CHUNK       UINT16_C(0x0004)
+/** Set if the whole range is mapped. */
+#define PGMREGMMIO2RANGE_F_MAPPED           UINT16_C(0x0008)
+/** Set if it's overlapping, clear if not. */
+#define PGMREGMMIO2RANGE_F_OVERLAPPING      UINT16_C(0x0010)
+/** @} */
+
+
+/** @name Internal MMIO2 constants.
+ * @{ */
+/** The maximum number of MMIO2 ranges. */
+#define PGM_MMIO2_MAX_RANGES                        32
+/** The maximum number of pages in a MMIO2 range. */
+#define PGM_MMIO2_MAX_PAGE_COUNT                    UINT32_C(0x01000000)
+/** Makes a MMIO2 page ID out of a MMIO2 range ID and page index number. */
+#define PGM_MMIO2_PAGEID_MAKE(a_idMmio2, a_iPage)   ( ((uint32_t)(a_idMmio2) << 24) | (uint32_t)(a_iPage) )
+/** Gets the MMIO2 range ID from an MMIO2 page ID. */
+#define PGM_MMIO2_PAGEID_GET_MMIO2_ID(a_idPage)     ( (uint8_t)((a_idPage) >> 24) )
+/** Gets the MMIO2 page index from an MMIO2 page ID. */
+#define PGM_MMIO2_PAGEID_GET_IDX(a_idPage)          ( ((a_idPage) & UINT32_C(0x00ffffff)) )
+/** @} */
+
+
+
+/**
+ * PGMPhysRead/Write cache entry
+ */
+typedef struct PGMPHYSCACHEENTRY
+{
+    /** R3 pointer to physical page. */
+    R3PTRTYPE(uint8_t *)                pbR3;
+    /** GC Physical address for cache entry */
+    RTGCPHYS                            GCPhys;
+#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
+    RTGCPHYS                            u32Padding0; /**< alignment padding. */
+#endif
+} PGMPHYSCACHEENTRY;
+
+/**
+ * PGMPhysRead/Write cache to reduce REM memory access overhead
+ */
+typedef struct PGMPHYSCACHE
+{
+    /** Bitmap of valid cache entries */
+    uint64_t                            aEntries;
+    /** Cache entries */
+    PGMPHYSCACHEENTRY                   Entry[PGM_MAX_PHYSCACHE_ENTRIES];
+} PGMPHYSCACHE;
+
+
+/** @name Ring-3 page mapping TLBs
+ * @{  */
+
+/** Pointer to an allocation chunk ring-3 mapping. */
+typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP;
+/** Pointer to an allocation chunk ring-3 mapping pointer. */
+typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP;
+
+/**
+ * Ring-3 tracking structure for an allocation chunk ring-3 mapping.
+ *
+ * The primary tree (Core) uses the chunk id as key.
+ */
+typedef struct PGMCHUNKR3MAP
+{
+    /** The key is the chunk id. */
+    AVLU32NODECORE                      Core;
+    /** The time (ChunkR3Map.iNow) this chunk was last used.  Used for unmap
+     *  selection. */
+    uint32_t                            iLastUsed;
+    /** The current reference count. */
+    uint32_t volatile                   cRefs;
+    /** The current permanent reference count. */
+    uint32_t volatile                   cPermRefs;
+    /** The mapping address. */
+    void                               *pv;
+} PGMCHUNKR3MAP;
+
+/**
+ * Allocation chunk ring-3 mapping TLB entry.
+ */
+typedef struct PGMCHUNKR3MAPTLBE
+{
+    /** The chunk id. */
+    uint32_t volatile                   idChunk;
+#if HC_ARCH_BITS == 64
+    uint32_t                            u32Padding; /**< alignment padding. */
+#endif
+    /** The chunk map. */
+#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE) || defined(VBOX_WITH_RAM_IN_KERNEL)
+    R3PTRTYPE(PPGMCHUNKR3MAP) volatile  pChunk;
+#else
+    R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile  pChunk;
+#endif
+} PGMCHUNKR3MAPTLBE;
+/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */
+typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE;
+
+/** The number of TLB entries in PGMCHUNKR3MAPTLB.
+ * @remark Must be a power of two value. */
+#define PGM_CHUNKR3MAPTLB_ENTRIES   64
+
+/**
+ * Allocation chunk ring-3 mapping TLB.
+ *
+ * @remarks We use a TLB to speed up lookups by avoiding walking the AVL.
+ *          At first glance this might look kinda odd since AVL trees are
+ *          supposed to give the most optimal lookup times of all trees
+ *          due to their balancing. However, take a tree with 1023 nodes
+ *          in it, that's 10 levels, meaning that most searches has to go
+ *          down 9 levels before they find what they want. This isn't fast
+ *          compared to a TLB hit. There is the factor of cache misses,
+ *          and of course the problem with trees and branch prediction.
+ *          This is why we use TLBs in front of most of the trees.
+ *
+ * @todo    Generalize this TLB + AVL stuff, shouldn't be all that
+ *          difficult when we switch to the new inlined AVL trees (from kStuff).
+ */
+typedef struct PGMCHUNKR3MAPTLB
+{
+    /** The TLB entries. */
+    PGMCHUNKR3MAPTLBE   aEntries[PGM_CHUNKR3MAPTLB_ENTRIES];
+} PGMCHUNKR3MAPTLB;
+
+/**
+ * Calculates the index of a guest page in the Ring-3 Chunk TLB.
+ * @returns Chunk TLB index.
+ * @param   idChunk         The Chunk ID.
+ */
+#define PGM_CHUNKR3MAPTLB_IDX(idChunk)     ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) )
+
+
+/**
+ * Ring-3 guest page mapping TLB entry.
+ * @remarks used in ring-0 as well at the moment.
+ */
+typedef struct PGMPAGER3MAPTLBE
+{
+    /** Address of the page. */
+    RTGCPHYS volatile                   GCPhys;
+    /** The guest page. */
+#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE) || defined(VBOX_WITH_RAM_IN_KERNEL)
+    R3PTRTYPE(PPGMPAGE) volatile        pPage;
+#else
+    R3R0PTRTYPE(PPGMPAGE) volatile      pPage;
+#endif
+    /** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */
+#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE) || defined(VBOX_WITH_RAM_IN_KERNEL)
+    R3PTRTYPE(PPGMCHUNKR3MAP) volatile  pMap;
+#else
+    R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
+#endif
+    /** The address */
+#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE) || defined(VBOX_WITH_RAM_IN_KERNEL)
+    R3PTRTYPE(void *) volatile          pv;
+#else
+    R3R0PTRTYPE(void *) volatile        pv;
+#endif
+#if HC_ARCH_BITS == 32
+    uint32_t                            u32Padding; /**< alignment padding. */
+#endif
+} PGMPAGER3MAPTLBE;
+/** Pointer to an entry in the HC physical TLB. */
+typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE;
+
+
+/** The number of entries in the ring-3 guest page mapping TLB.
+ * @remarks The value must be a power of two. */
+#define PGM_PAGER3MAPTLB_ENTRIES 256
+
+/**
+ * Ring-3 guest page mapping TLB.
+ * @remarks used in ring-0 as well at the moment.
+ */
+typedef struct PGMPAGER3MAPTLB
+{
+    /** The TLB entries. */
+    PGMPAGER3MAPTLBE            aEntries[PGM_PAGER3MAPTLB_ENTRIES];
+} PGMPAGER3MAPTLB;
+/** Pointer to the ring-3 guest page mapping TLB. */
+typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB;
+
+/**
+ * Calculates the index of the TLB entry for the specified guest page.
+ * @returns Physical TLB index.
+ * @param   GCPhys      The guest physical address.
+ */
+#define PGM_PAGER3MAPTLB_IDX(GCPhys)    ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) )
+
+/** @} */
+
+#if defined(VBOX_WITH_RAM_IN_KERNEL) || defined(DOXYGEN_RUNNING)
+/** @name Ring-0 page mapping TLB
+ * @{  */
+/**
+ * Ring-0 guest page mapping TLB entry.
+ */
+typedef struct PGMPAGER0MAPTLBE
+{
+    /** Address of the page. */
+    RTGCPHYS volatile                   GCPhys;
+    /** The guest page. */
+    R0PTRTYPE(PPGMPAGE) volatile        pPage;
+    /** The address */
+    R0PTRTYPE(void *) volatile          pv;
+} PGMPAGER0MAPTLBE;
+/** Pointer to an entry in the HC physical TLB. */
+typedef PGMPAGER0MAPTLBE *PPGMPAGER0MAPTLBE;
+
+
+/** The number of entries in the ring-3 guest page mapping TLB.
+ * @remarks The value must be a power of two. */
+#define PGM_PAGER0MAPTLB_ENTRIES 256
+
+/**
+ * Ring-3 guest page mapping TLB.
+ * @remarks used in ring-0 as well at the moment.
+ */
+typedef struct PGMPAGER0MAPTLB
+{
+    /** The TLB entries. */
+    PGMPAGER0MAPTLBE            aEntries[PGM_PAGER0MAPTLB_ENTRIES];
+} PGMPAGER0MAPTLB;
+/** Pointer to the ring-3 guest page mapping TLB. */
+typedef PGMPAGER0MAPTLB *PPGMPAGER0MAPTLB;
+
+/**
+ * Calculates the index of the TLB entry for the specified guest page.
+ * @returns Physical TLB index.
+ * @param   GCPhys      The guest physical address.
+ */
+#define PGM_PAGER0MAPTLB_IDX(GCPhys)    ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER0MAPTLB_ENTRIES - 1) )
+/** @} */
+#endif /* VBOX_WITH_RAM_IN_KERNEL || DOXYGEN_RUNNING */
+
+/**
+ * Raw-mode context dynamic mapping cache entry.
+ *
+ * Because of raw-mode context being reloctable and all relocations are applied
+ * in ring-3, this has to be defined here and be RC specific.
+ *
+ * @sa PGMRZDYNMAPENTRY, PGMR0DYNMAPENTRY.
+ */
+typedef struct PGMRCDYNMAPENTRY
+{
+    /** The physical address of the currently mapped page.
+     * This is duplicate for three reasons: cache locality, cache policy of the PT
+     * mappings and sanity checks. */
+    RTHCPHYS                    HCPhys;
+    /** Pointer to the page. */
+    RTRCPTR                     pvPage;
+    /** The number of references. */
+    int32_t volatile            cRefs;
+    /** PTE pointer union. */
+    struct PGMRCDYNMAPENTRY_PPTE
+    {
+        /** PTE pointer, 32-bit legacy version. */
+        RCPTRTYPE(PX86PTE)      pLegacy;
+        /** PTE pointer, PAE version. */
+        RCPTRTYPE(PX86PTEPAE)   pPae;
+    } uPte;
+} PGMRCDYNMAPENTRY;
+/** Pointer to a dynamic mapping cache entry for the raw-mode context. */
+typedef PGMRCDYNMAPENTRY *PPGMRCDYNMAPENTRY;
+
+
+/**
+ * Dynamic mapping cache for the raw-mode context.
+ *
+ * This is initialized during VMMRC init based upon the pbDynPageMapBaseGC and
+ * paDynPageMap* PGM members.  However, it has to be defined in PGMInternal.h
+ * so that we can perform relocations from PGMR3Relocate.  This has the
+ * consequence that we must have separate ring-0 and raw-mode context versions
+ * of this struct even if they share the basic elements.
+ *
+ * @sa PPGMRZDYNMAP, PGMR0DYNMAP.
+ */
+typedef struct PGMRCDYNMAP
+{
+    /** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */
+    uint32_t                        u32Magic;
+    /** Array for tracking and managing the pages. */
+    RCPTRTYPE(PPGMRCDYNMAPENTRY)    paPages;
+    /** The cache size given as a number of pages. */
+    uint32_t                        cPages;
+    /** The current load.
+     * This does not include guard pages. */
+    uint32_t                        cLoad;
+    /** The max load ever.
+     * This is maintained to get trigger adding of more mapping space. */
+    uint32_t                        cMaxLoad;
+    /** The number of guard pages. */
+    uint32_t                        cGuardPages;
+    /** The number of users (protected by hInitLock). */
+    uint32_t                        cUsers;
+} PGMRCDYNMAP;
+/** Pointer to the dynamic cache for the raw-mode context. */
+typedef PGMRCDYNMAP *PPGMRCDYNMAP;
+
+
+/**
+ * Mapping cache usage set entry.
+ *
+ * @remarks 16-bit ints was chosen as the set is not expected to be used beyond
+ *          the dynamic ring-0 and (to some extent) raw-mode context mapping
+ *          cache.  If it's extended to include ring-3, well, then something
+ *          will have be changed here...
+ */
+typedef struct PGMMAPSETENTRY
+{
+    /** Pointer to the page. */
+    RTR0PTR                     pvPage;
+    /** The mapping cache index. */
+    uint16_t                    iPage;
+    /** The number of references.
+     * The max is UINT16_MAX - 1. */
+    uint16_t                    cRefs;
+    /** The number inlined references.
+     * The max is UINT16_MAX - 1. */
+    uint16_t                    cInlinedRefs;
+    /** Unreferences. */
+    uint16_t                    cUnrefs;
+
+#if HC_ARCH_BITS == 32
+    uint32_t                    u32Alignment1;
+#endif
+    /** The physical address for this entry. */
+    RTHCPHYS                    HCPhys;
+} PGMMAPSETENTRY;
+AssertCompileMemberOffset(PGMMAPSETENTRY, iPage, RT_MAX(sizeof(RTR0PTR), sizeof(RTRCPTR)));
+AssertCompileMemberAlignment(PGMMAPSETENTRY, HCPhys, sizeof(RTHCPHYS));
+/** Pointer to a mapping cache usage set entry. */
+typedef PGMMAPSETENTRY *PPGMMAPSETENTRY;
+
+/**
+ * Mapping cache usage set.
+ *
+ * This is used in ring-0 and the raw-mode context to track dynamic mappings
+ * done during exits / traps.  The set is
+ */
+typedef struct PGMMAPSET
+{
+    /** The number of occupied entries.
+     * This is PGMMAPSET_CLOSED if the set is closed and we're not supposed to do
+     * dynamic mappings. */
+    uint32_t                    cEntries;
+    /** The start of the current subset.
+     * This is UINT32_MAX if no subset is currently open. */
+    uint32_t                    iSubset;
+    /** The index of the current CPU, only valid if the set is open. */
+    int32_t                     iCpu;
+    uint32_t                    alignment;
+    /** The entries. */
+    PGMMAPSETENTRY              aEntries[64];
+    /** HCPhys -> iEntry fast lookup table.
+     * Use PGMMAPSET_HASH for hashing.
+     * The entries may or may not be valid, check against cEntries. */
+    uint8_t                     aiHashTable[128];
+} PGMMAPSET;
+AssertCompileSizeAlignment(PGMMAPSET, 8);
+/** Pointer to the mapping cache set. */
+typedef PGMMAPSET *PPGMMAPSET;
+
+/** PGMMAPSET::cEntries value for a closed set. */
+#define PGMMAPSET_CLOSED            UINT32_C(0xdeadc0fe)
+
+/** Hash function for aiHashTable. */
+#define PGMMAPSET_HASH(HCPhys)      (((HCPhys) >> PAGE_SHIFT) & 127)
+
+
+/** @name Context neutral page mapper TLB.
+ *
+ * Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr
+ * code is writting in a kind of context neutral way. Time will show whether
+ * this actually makes sense or not...
+ *
+ * @todo this needs to be reconsidered and dropped/redone since the ring-0
+ *       context ends up using a global mapping cache on some platforms
+ *       (darwin).
+ *
+ * @{ */
+/** @typedef PPGMPAGEMAPTLB
+ * The page mapper TLB pointer type for the current context. */
+/** @typedef PPGMPAGEMAPTLB
+ * The page mapper TLB entry pointer type for the current context. */
+/** @typedef PPGMPAGEMAPTLB
+ * The page mapper TLB entry pointer pointer type for the current context. */
+/** @def PGM_PAGEMAPTLB_ENTRIES
+ * The number of TLB entries in the page mapper TLB for the current context. */
+/** @def PGM_PAGEMAPTLB_IDX
+ * Calculate the TLB index for a guest physical address.
+ * @returns The TLB index.
+ * @param   GCPhys      The guest physical address. */
+/** @typedef PPGMPAGEMAP
+ * Pointer to a page mapper unit for current context. */
+/** @typedef PPPGMPAGEMAP
+ * Pointer to a page mapper unit pointer for current context. */
+#if defined(IN_RING0) && defined(VBOX_WITH_RAM_IN_KERNEL)
+typedef PPGMPAGER0MAPTLB                PPGMPAGEMAPTLB;
+typedef PPGMPAGER0MAPTLBE               PPGMPAGEMAPTLBE;
+typedef PPGMPAGER0MAPTLBE              *PPPGMPAGEMAPTLBE;
+# define PGM_PAGEMAPTLB_ENTRIES         PGM_PAGER0MAPTLB_ENTRIES
+# define PGM_PAGEMAPTLB_IDX(GCPhys)     PGM_PAGER0MAPTLB_IDX(GCPhys)
+typedef struct PGMCHUNKR0MAP           *PPGMPAGEMAP;
+typedef struct PGMCHUNKR0MAP          **PPPGMPAGEMAP;
+#else
+typedef PPGMPAGER3MAPTLB                PPGMPAGEMAPTLB;
+typedef PPGMPAGER3MAPTLBE               PPGMPAGEMAPTLBE;
+typedef PPGMPAGER3MAPTLBE              *PPPGMPAGEMAPTLBE;
+# define PGM_PAGEMAPTLB_ENTRIES         PGM_PAGER3MAPTLB_ENTRIES
+# define PGM_PAGEMAPTLB_IDX(GCPhys)     PGM_PAGER3MAPTLB_IDX(GCPhys)
+typedef PPGMCHUNKR3MAP                  PPGMPAGEMAP;
+typedef PPPGMCHUNKR3MAP                 PPPGMPAGEMAP;
+#endif
+/** @} */
+
+
+/** @name PGM Pool Indexes.
+ * Aka. the unique shadow page identifier.
+ * @{ */
+/** NIL page pool IDX. */
+#define NIL_PGMPOOL_IDX                 0
+/** The first normal index.  There used to be 5 fictive pages up front, now
+ * there is only the NIL page. */
+#define PGMPOOL_IDX_FIRST               1
+/** The last valid index. (inclusive, 14 bits) */
+#define PGMPOOL_IDX_LAST                0x3fff
+/** @} */
+
+/** The NIL index for the parent chain. */
+#define NIL_PGMPOOL_USER_INDEX          ((uint16_t)0xffff)
+#define NIL_PGMPOOL_PRESENT_INDEX       ((uint16_t)0xffff)
+
+/**
+ * Node in the chain linking a shadowed page to it's parent (user).
+ */
+#pragma pack(1)
+typedef struct PGMPOOLUSER
+{
+    /** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */
+    uint16_t            iNext;
+    /** The user page index. */
+    uint16_t            iUser;
+    /** Index into the user table. */
+    uint32_t            iUserTable;
+} PGMPOOLUSER, *PPGMPOOLUSER;
+typedef const PGMPOOLUSER *PCPGMPOOLUSER;
+#pragma pack()
+
+
+/** The NIL index for the phys ext chain. */
+#define NIL_PGMPOOL_PHYSEXT_INDEX       ((uint16_t)0xffff)
+/** The NIL pte index for a phys ext chain slot. */
+#define NIL_PGMPOOL_PHYSEXT_IDX_PTE     ((uint16_t)0xffff)
+
+/**
+ * Node in the chain of physical cross reference extents.
+ * @todo Calling this an 'extent' is not quite right, find a better name.
+ * @todo find out the optimal size of the aidx array
+ */
+#pragma pack(1)
+typedef struct PGMPOOLPHYSEXT
+{
+    /** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */
+    uint16_t            iNext;
+    /** Alignment. */
+    uint16_t            u16Align;
+    /** The user page index. */
+    uint16_t            aidx[3];
+    /** The page table index or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown. */
+    uint16_t            apte[3];
+} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT;
+typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT;
+#pragma pack()
+
+
+/**
+ * The kind of page that's being shadowed.
+ */
+typedef enum PGMPOOLKIND
+{
+    /** The virtual invalid 0 entry. */
+    PGMPOOLKIND_INVALID = 0,
+    /** The entry is free (=unused). */
+    PGMPOOLKIND_FREE,
+
+    /** Shw: 32-bit page table;     Gst: no paging. */
+    PGMPOOLKIND_32BIT_PT_FOR_PHYS,
+    /** Shw: 32-bit page table;     Gst: 32-bit page table. */
+    PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT,
+    /** Shw: 32-bit page table;     Gst: 4MB page. */
+    PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB,
+    /** Shw: PAE page table;        Gst: no paging. */
+    PGMPOOLKIND_PAE_PT_FOR_PHYS,
+    /** Shw: PAE page table;        Gst: 32-bit page table. */
+    PGMPOOLKIND_PAE_PT_FOR_32BIT_PT,
+    /** Shw: PAE page table;        Gst: Half of a 4MB page. */
+    PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB,
+    /** Shw: PAE page table;        Gst: PAE page table. */
+    PGMPOOLKIND_PAE_PT_FOR_PAE_PT,
+    /** Shw: PAE page table;        Gst: 2MB page. */
+    PGMPOOLKIND_PAE_PT_FOR_PAE_2MB,
+
+    /** Shw: 32-bit page directory. Gst: 32-bit page directory. */
+    PGMPOOLKIND_32BIT_PD,
+    /** Shw: 32-bit page directory. Gst: no paging. */
+    PGMPOOLKIND_32BIT_PD_PHYS,
+    /** Shw: PAE page directory 0;  Gst: 32-bit page directory. */
+    PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD,
+    /** Shw: PAE page directory 1;  Gst: 32-bit page directory. */
+    PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD,
+    /** Shw: PAE page directory 2;  Gst: 32-bit page directory. */
+    PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD,
+    /** Shw: PAE page directory 3;  Gst: 32-bit page directory. */
+    PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
+    /** Shw: PAE page directory;    Gst: PAE page directory. */
+    PGMPOOLKIND_PAE_PD_FOR_PAE_PD,
+    /** Shw: PAE page directory;    Gst: no paging.             Note: +NP. */
+    PGMPOOLKIND_PAE_PD_PHYS,
+
+    /** Shw: PAE page directory pointer table (legacy, 4 entries);  Gst 32 bits paging. */
+    PGMPOOLKIND_PAE_PDPT_FOR_32BIT,
+    /** Shw: PAE page directory pointer table (legacy, 4 entries);  Gst PAE PDPT. */
+    PGMPOOLKIND_PAE_PDPT,
+    /** Shw: PAE page directory pointer table (legacy, 4 entries);  Gst: no paging. */
+    PGMPOOLKIND_PAE_PDPT_PHYS,
+
+    /** Shw: 64-bit page directory pointer table;   Gst: 64-bit page directory pointer table. */
+    PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT,
+    /** Shw: 64-bit page directory pointer table;   Gst: no paging. */
+    PGMPOOLKIND_64BIT_PDPT_FOR_PHYS,
+    /** Shw: 64-bit page directory table;           Gst: 64-bit page directory table. */
+    PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD,
+    /** Shw: 64-bit page directory table;           Gst: no paging. */
+    PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 24 */
+
+    /** Shw: 64-bit PML4;                           Gst: 64-bit PML4. */
+    PGMPOOLKIND_64BIT_PML4,
+
+    /** Shw: EPT page directory pointer table;      Gst: no paging. */
+    PGMPOOLKIND_EPT_PDPT_FOR_PHYS,
+    /** Shw: EPT page directory table;              Gst: no paging. */
+    PGMPOOLKIND_EPT_PD_FOR_PHYS,
+    /** Shw: EPT page table;                        Gst: no paging. */
+    PGMPOOLKIND_EPT_PT_FOR_PHYS,
+
+    /** Shw: Root Nested paging table. */
+    PGMPOOLKIND_ROOT_NESTED,
+
+    /** The last valid entry. */
+    PGMPOOLKIND_LAST = PGMPOOLKIND_ROOT_NESTED
+} PGMPOOLKIND;
+
+/**
+ * The access attributes of the page; only applies to big pages.
+ */
+typedef enum
+{
+    PGMPOOLACCESS_DONTCARE = 0,
+    PGMPOOLACCESS_USER_RW,
+    PGMPOOLACCESS_USER_R,
+    PGMPOOLACCESS_USER_RW_NX,
+    PGMPOOLACCESS_USER_R_NX,
+    PGMPOOLACCESS_SUPERVISOR_RW,
+    PGMPOOLACCESS_SUPERVISOR_R,
+    PGMPOOLACCESS_SUPERVISOR_RW_NX,
+    PGMPOOLACCESS_SUPERVISOR_R_NX
+} PGMPOOLACCESS;
+
+/**
+ * The tracking data for a page in the pool.
+ */
+typedef struct PGMPOOLPAGE
+{
+    /** AVL node code with the (HC) physical address of this page. */
+    AVLOHCPHYSNODECORE  Core;
+    /** Pointer to the R3 mapping of the page. */
+    R3PTRTYPE(void *)   pvPageR3;
+    /** Pointer to the R0 mapping of the page. */
+    R0PTRTYPE(void *)   pvPageR0;
+    /** The guest physical address. */
+    RTGCPHYS            GCPhys;
+    /** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */
+    uint8_t             enmKind;
+    /** The subkind of page we're shadowing. (This is really a PGMPOOLACCESS enum.) */
+    uint8_t             enmAccess;
+    /** This supplements enmKind and enmAccess */
+    bool                fA20Enabled : 1;
+
+    /** Used to indicate that the page is zeroed. */
+    bool                fZeroed : 1;
+    /** Used to indicate that a PT has non-global entries. */
+    bool                fSeenNonGlobal : 1;
+    /** Used to indicate that we're monitoring writes to the guest page. */
+    bool                fMonitored : 1;
+    /** Used to indicate that the page is in the cache (e.g. in the GCPhys hash).
+     * (All pages are in the age list.) */
+    bool                fCached : 1;
+    /** This is used by the R3 access handlers when invoked by an async thread.
+     * It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */
+    bool volatile       fReusedFlushPending : 1;
+    /** Used to mark the page as dirty (write monitoring is temporarily
+     *  off). */
+    bool                fDirty : 1;
+    bool                fPadding1 : 1;
+    bool                fPadding2;
+
+    /** The index of this page. */
+    uint16_t            idx;
+    /** The next entry in the list this page currently resides in.
+     * It's either in the free list or in the GCPhys hash. */
+    uint16_t            iNext;
+    /** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */
+    uint16_t            iUserHead;
+    /** The number of present entries. */
+    uint16_t            cPresent;
+    /** The first entry in the table which is present. */
+    uint16_t            iFirstPresent;
+    /** The number of modifications to the monitored page. */
+    uint16_t            cModifications;
+    /** The next modified page. NIL_PGMPOOL_IDX if tail. */
+    uint16_t            iModifiedNext;
+    /** The previous modified page. NIL_PGMPOOL_IDX if head. */
+    uint16_t            iModifiedPrev;
+    /** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */
+    uint16_t            iMonitoredNext;
+    /** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */
+    uint16_t            iMonitoredPrev;
+    /** The next page in the age list. */
+    uint16_t            iAgeNext;
+    /** The previous page in the age list. */
+    uint16_t            iAgePrev;
+    /** Index into PGMPOOL::aDirtyPages if fDirty is set. */
+    uint8_t             idxDirtyEntry;
+
+    /** @name Access handler statistics to determine whether the guest is
+     *        (re)initializing a page table.
+     * @{ */
+    RTGCPTR             GCPtrLastAccessHandlerRip;
+    RTGCPTR             GCPtrLastAccessHandlerFault;
+    uint64_t            cLastAccessHandler;
+    /** @} */
+    /** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages. */
+    uint32_t volatile   cLocked;
+#if GC_ARCH_BITS == 64
+    uint32_t            u32Alignment3;
+#endif
+# ifdef VBOX_STRICT
+    RTGCPTR             GCPtrDirtyFault;
+# endif
+} PGMPOOLPAGE;
+/** Pointer to a pool page. */
+typedef PGMPOOLPAGE *PPGMPOOLPAGE;
+/** Pointer to a const pool page. */
+typedef PGMPOOLPAGE const *PCPGMPOOLPAGE;
+/** Pointer to a pool page pointer. */
+typedef PGMPOOLPAGE **PPPGMPOOLPAGE;
+
+
+/** The hash table size. */
+# define PGMPOOL_HASH_SIZE      0x40
+/** The hash function. */
+# define PGMPOOL_HASH(GCPhys)   ( ((GCPhys) >> PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) )
+
+
+/**
+ * The shadow page pool instance data.
+ *
+ * It's all one big allocation made at init time, except for the
+ * pages that is. The user nodes follows immediately after the
+ * page structures.
+ */
+typedef struct PGMPOOL
+{
+    /** The VM handle - R3 Ptr. */
+    PVMR3                       pVMR3;
+    /** The VM handle - R0 Ptr. */
+    R0PTRTYPE(PVMCC)            pVMR0;
+    /** The max pool size. This includes the special IDs. */
+    uint16_t                    cMaxPages;
+    /** The current pool size. */
+    uint16_t                    cCurPages;
+    /** The head of the free page list. */
+    uint16_t                    iFreeHead;
+    /* Padding. */
+    uint16_t                    u16Padding;
+    /** Head of the chain of free user nodes. */
+    uint16_t                    iUserFreeHead;
+    /** The number of user nodes we've allocated. */
+    uint16_t                    cMaxUsers;
+    /** The number of present page table entries in the entire pool. */
+    uint32_t                    cPresent;
+    /** Pointer to the array of user nodes - R3 pointer. */
+    R3PTRTYPE(PPGMPOOLUSER)     paUsersR3;
+    /** Pointer to the array of user nodes - R0 pointer. */
+    R0PTRTYPE(PPGMPOOLUSER)     paUsersR0;
+    /** Head of the chain of free phys ext nodes. */
+    uint16_t                    iPhysExtFreeHead;
+    /** The number of user nodes we've allocated. */
+    uint16_t                    cMaxPhysExts;
+    uint32_t                    u32Padding0b;
+    /** Pointer to the array of physical xref extent nodes - R3 pointer. */
+    R3PTRTYPE(PPGMPOOLPHYSEXT)  paPhysExtsR3;
+    /** Pointer to the array of physical xref extent nodes - R0 pointer. */
+    R0PTRTYPE(PPGMPOOLPHYSEXT)  paPhysExtsR0;
+    /** Hash table for GCPhys addresses. */
+    uint16_t                    aiHash[PGMPOOL_HASH_SIZE];
+    /** The head of the age list. */
+    uint16_t                    iAgeHead;
+    /** The tail of the age list. */
+    uint16_t                    iAgeTail;
+    /** Set if the cache is enabled. */
+    bool                        fCacheEnabled;
+    /** Alignment padding. */
+    bool                        afPadding1[3];
+    /** Head of the list of modified pages. */
+    uint16_t                    iModifiedHead;
+    /** The current number of modified pages. */
+    uint16_t                    cModifiedPages;
+    /** Physical access handler type registration handle. */
+    PGMPHYSHANDLERTYPE          hAccessHandlerType;
+    /** Next available slot (in aDirtyPages). */
+    uint32_t                    idxFreeDirtyPage;
+    /** Number of active dirty pages. */
+    uint32_t                    cDirtyPages;
+    /** Array of current dirty pgm pool page indices. */
+    uint16_t                    aidxDirtyPages[16];
+    /** Array running in parallel to aidxDirtyPages with the page data. */
+    struct
+    {
+        uint64_t                aPage[512];
+    } aDirtyPages[16];
+
+    /** The number of pages currently in use. */
+    uint16_t                    cUsedPages;
+#ifdef VBOX_WITH_STATISTICS
+    /** The high water mark for cUsedPages. */
+    uint16_t                    cUsedPagesHigh;
+    uint32_t                    Alignment1;         /**< Align the next member on a 64-bit boundary. */
+    /** Profiling pgmPoolAlloc(). */
+    STAMPROFILEADV              StatAlloc;
+    /** Profiling pgmR3PoolClearDoIt(). */
+    STAMPROFILE                 StatClearAll;
+    /** Profiling pgmR3PoolReset(). */
+    STAMPROFILE                 StatR3Reset;
+    /** Profiling pgmPoolFlushPage(). */
+    STAMPROFILE                 StatFlushPage;
+    /** Profiling pgmPoolFree(). */
+    STAMPROFILE                 StatFree;
+    /** Counting explicit flushes by PGMPoolFlushPage(). */
+    STAMCOUNTER                 StatForceFlushPage;
+    /** Counting explicit flushes of dirty pages by PGMPoolFlushPage(). */
+    STAMCOUNTER                 StatForceFlushDirtyPage;
+    /** Counting flushes for reused pages. */
+    STAMCOUNTER                 StatForceFlushReused;
+    /** Profiling time spent zeroing pages. */
+    STAMPROFILE                 StatZeroPage;
+    /** Profiling of pgmPoolTrackDeref. */
+    STAMPROFILE                 StatTrackDeref;
+    /** Profiling pgmTrackFlushGCPhysPT. */
+    STAMPROFILE                 StatTrackFlushGCPhysPT;
+    /** Profiling pgmTrackFlushGCPhysPTs. */
+    STAMPROFILE                 StatTrackFlushGCPhysPTs;
+    /** Profiling pgmTrackFlushGCPhysPTsSlow. */
+    STAMPROFILE                 StatTrackFlushGCPhysPTsSlow;
+    /** Number of times we've been out of user records. */
+    STAMCOUNTER                 StatTrackFreeUpOneUser;
+    /** Nr of flushed entries. */
+    STAMCOUNTER                 StatTrackFlushEntry;
+    /** Nr of updated entries. */
+    STAMCOUNTER                 StatTrackFlushEntryKeep;
+    /** Profiling deref activity related tracking GC physical pages. */
+    STAMPROFILE                 StatTrackDerefGCPhys;
+    /** Number of linear searches for a HCPhys in the ram ranges. */
+    STAMCOUNTER                 StatTrackLinearRamSearches;
+    /** The number of failing pgmPoolTrackPhysExtAlloc calls. */
+    STAMCOUNTER                 StamTrackPhysExtAllocFailures;
+
+    /** Profiling the RC/R0 \#PF access handler. */
+    STAMPROFILE                 StatMonitorPfRZ;
+    /** Profiling the RC/R0 access we've handled (except REP STOSD). */
+    STAMPROFILE                 StatMonitorPfRZHandled;
+    /** Times we've failed interpreting the instruction. */
+    STAMCOUNTER                 StatMonitorPfRZEmulateInstr;
+    /** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
+    STAMPROFILE                 StatMonitorPfRZFlushPage;
+    /** Times we've detected a page table reinit. */
+    STAMCOUNTER                 StatMonitorPfRZFlushReinit;
+    /** Counting flushes for pages that are modified too often. */
+    STAMCOUNTER                 StatMonitorPfRZFlushModOverflow;
+    /** Times we've detected fork(). */
+    STAMCOUNTER                 StatMonitorPfRZFork;
+    /** Times we've failed interpreting a patch code instruction. */
+    STAMCOUNTER                 StatMonitorPfRZIntrFailPatch1;
+    /** Times we've failed interpreting a patch code instruction during flushing. */
+    STAMCOUNTER                 StatMonitorPfRZIntrFailPatch2;
+    /** The number of times we've seen rep prefixes we can't handle. */
+    STAMCOUNTER                 StatMonitorPfRZRepPrefix;
+    /** Profiling the REP STOSD cases we've handled. */
+    STAMPROFILE                 StatMonitorPfRZRepStosd;
+
+    /** Profiling the R0/RC regular access handler. */
+    STAMPROFILE                 StatMonitorRZ;
+    /** Profiling the pgmPoolFlushPage calls made from the regular access handler in R0/RC. */
+    STAMPROFILE                 StatMonitorRZFlushPage;
+    /** Per access size counts indexed by size minus 1, last for larger. */
+    STAMCOUNTER                 aStatMonitorRZSizes[16+3];
+    /** Missaligned access counts indexed by offset - 1. */
+    STAMCOUNTER                 aStatMonitorRZMisaligned[7];
+
+    /** Nr of handled PT faults. */
+    STAMCOUNTER                 StatMonitorRZFaultPT;
+    /** Nr of handled PD faults. */
+    STAMCOUNTER                 StatMonitorRZFaultPD;
+    /** Nr of handled PDPT faults. */
+    STAMCOUNTER                 StatMonitorRZFaultPDPT;
+    /** Nr of handled PML4 faults. */
+    STAMCOUNTER                 StatMonitorRZFaultPML4;
+
+    /** Profiling the R3 access handler. */
+    STAMPROFILE                 StatMonitorR3;
+    /** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
+    STAMPROFILE                 StatMonitorR3FlushPage;
+    /** Per access size counts indexed by size minus 1, last for larger. */
+    STAMCOUNTER                 aStatMonitorR3Sizes[16+3];
+    /** Missaligned access counts indexed by offset - 1. */
+    STAMCOUNTER                 aStatMonitorR3Misaligned[7];
+    /** Nr of handled PT faults. */
+    STAMCOUNTER                 StatMonitorR3FaultPT;
+    /** Nr of handled PD faults. */
+    STAMCOUNTER                 StatMonitorR3FaultPD;
+    /** Nr of handled PDPT faults. */
+    STAMCOUNTER                 StatMonitorR3FaultPDPT;
+    /** Nr of handled PML4 faults. */
+    STAMCOUNTER                 StatMonitorR3FaultPML4;
+
+    /** Times we've called pgmPoolResetDirtyPages (and there were dirty page). */
+    STAMCOUNTER                 StatResetDirtyPages;
+    /** Times we've called pgmPoolAddDirtyPage. */
+    STAMCOUNTER                 StatDirtyPage;
+    /** Times we've had to flush duplicates for dirty page management. */
+    STAMCOUNTER                 StatDirtyPageDupFlush;
+    /** Times we've had to flush because of overflow. */
+    STAMCOUNTER                 StatDirtyPageOverFlowFlush;
+
+    /** The high water mark for cModifiedPages. */
+    uint16_t                    cModifiedPagesHigh;
+    uint16_t                    Alignment2[3];      /**< Align the next member on a 64-bit boundary. */
+
+    /** The number of cache hits. */
+    STAMCOUNTER                 StatCacheHits;
+    /** The number of cache misses. */
+    STAMCOUNTER                 StatCacheMisses;
+    /** The number of times we've got a conflict of 'kind' in the cache. */
+    STAMCOUNTER                 StatCacheKindMismatches;
+    /** Number of times we've been out of pages. */
+    STAMCOUNTER                 StatCacheFreeUpOne;
+    /** The number of cacheable allocations. */
+    STAMCOUNTER                 StatCacheCacheable;
+    /** The number of uncacheable allocations. */
+    STAMCOUNTER                 StatCacheUncacheable;
+#else
+    uint32_t                    Alignment3;         /**< Align the next member on a 64-bit boundary. */
+#endif
+    /** Profiling PGMR0PoolGrow(). */
+    STAMPROFILE                 StatGrow;
+    /** The AVL tree for looking up a page by its HC physical address. */
+    AVLOHCPHYSTREE              HCPhysTree;
+    uint32_t                    Alignment4;         /**< Align the next member on a 64-bit boundary. */
+    /** Array of pages. (cMaxPages in length)
+     * The Id is the index into thist array.
+     */
+    PGMPOOLPAGE                 aPages[PGMPOOL_IDX_FIRST];
+} PGMPOOL, *PPGMPOOL, **PPPGMPOOL;
+AssertCompileMemberAlignment(PGMPOOL, iModifiedHead, 8);
+AssertCompileMemberAlignment(PGMPOOL, aDirtyPages, 8);
+AssertCompileMemberAlignment(PGMPOOL, cUsedPages, 8);
+#ifdef VBOX_WITH_STATISTICS
+AssertCompileMemberAlignment(PGMPOOL, StatAlloc, 8);
+#endif
+AssertCompileMemberAlignment(PGMPOOL, aPages, 8);
+
+
+/** @def PGMPOOL_PAGE_2_PTR
+ * Maps a pool page pool into the current context.
+ *
+ * @returns VBox status code.
+ * @param   a_pVM       Pointer to the VM.
+ * @param   a_pPage     The pool page.
+ *
+ * @remark  In RC this uses PGMGCDynMapHCPage(), so it will consume of the
+ *          small page window employeed by that function. Be careful.
+ * @remark  There is no need to assert on the result.
+ */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+# define PGMPOOL_PAGE_2_PTR(a_pVM, a_pPage)     pgmPoolMapPageInlined((a_pVM), (a_pPage) RTLOG_COMMA_SRC_POS)
+#elif defined(VBOX_STRICT) || 1 /* temporarily going strict here */
+# define PGMPOOL_PAGE_2_PTR(a_pVM, a_pPage)     pgmPoolMapPageStrict(a_pPage, __FUNCTION__)
+DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE a_pPage, const char *pszCaller)
+{
+    RT_NOREF(pszCaller);
+    AssertPtr(a_pPage);
+    AssertMsg(RT_VALID_PTR(a_pPage->CTX_SUFF(pvPage)),
+              ("enmKind=%d idx=%#x HCPhys=%RHp GCPhys=%RGp pvPageR3=%p pvPageR0=%p caller=%s\n",
+               a_pPage->enmKind, a_pPage->idx, a_pPage->Core.Key, a_pPage->GCPhys, a_pPage->pvPageR3, a_pPage->pvPageR0, pszCaller));
+    return a_pPage->CTX_SUFF(pvPage);
+}
+#else
+# define PGMPOOL_PAGE_2_PTR(pVM, a_pPage)       ((a_pPage)->CTX_SUFF(pvPage))
+#endif
+
+
+/** @def PGMPOOL_PAGE_2_PTR_V2
+ * Maps a pool page pool into the current context, taking both VM and VMCPU.
+ *
+ * @returns VBox status code.
+ * @param   a_pVM       Pointer to the VM.
+ * @param   a_pVCpu     The current CPU.
+ * @param   a_pPage     The pool page.
+ *
+ * @remark  In RC this uses PGMGCDynMapHCPage(), so it will consume of the
+ *          small page window employeed by that function. Be careful.
+ * @remark  There is no need to assert on the result.
+ */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+# define PGMPOOL_PAGE_2_PTR_V2(a_pVM, a_pVCpu, a_pPage)     pgmPoolMapPageV2Inlined((a_pVM), (a_pVCpu), (a_pPage) RTLOG_COMMA_SRC_POS)
+#else
+# define PGMPOOL_PAGE_2_PTR_V2(a_pVM, a_pVCpu, a_pPage)     PGMPOOL_PAGE_2_PTR((a_pVM), (a_pPage))
+#endif
+
+
+/** @name Per guest page tracking data.
+ * This is currently as a 16-bit word in the PGMPAGE structure, the idea though
+ * is to use more bits for it and split it up later on. But for now we'll play
+ * safe and change as little as possible.
+ *
+ * The 16-bit word has two parts:
+ *
+ * The first 14-bit forms the @a idx field. It is either the index of a page in
+ * the shadow page pool, or and index into the extent list.
+ *
+ * The 2 topmost bits makes up the @a cRefs field, which counts the number of
+ * shadow page pool references to the page. If cRefs equals
+ * PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent
+ * (misnomer) table and not the shadow page pool.
+ *
+ * See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set
+ * the 16-bit word.
+ *
+ * @{ */
+/** The shift count for getting to the cRefs part. */
+#define PGMPOOL_TD_CREFS_SHIFT          14
+/** The mask applied after shifting the tracking data down by
+ * PGMPOOL_TD_CREFS_SHIFT. */
+#define PGMPOOL_TD_CREFS_MASK           0x3
+/** The cRefs value used to indicate that the idx is the head of a
+ * physical cross reference list. */
+#define PGMPOOL_TD_CREFS_PHYSEXT        PGMPOOL_TD_CREFS_MASK
+/** The shift used to get idx. */
+#define PGMPOOL_TD_IDX_SHIFT            0
+/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */
+#define PGMPOOL_TD_IDX_MASK             0x3fff
+/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are
+ * simply too many mappings of this page. */
+#define PGMPOOL_TD_IDX_OVERFLOWED       PGMPOOL_TD_IDX_MASK
+
+/** @def PGMPOOL_TD_MAKE
+ * Makes a 16-bit tracking data word.
+ *
+ * @returns tracking data.
+ * @param   cRefs       The @a cRefs field. Must be within bounds!
+ * @param   idx         The @a idx field. Must also be within bounds! */
+#define PGMPOOL_TD_MAKE(cRefs, idx)     ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) | (idx) )
+
+/** @def PGMPOOL_TD_GET_CREFS
+ * Get the @a cRefs field from a tracking data word.
+ *
+ * @returns The @a cRefs field
+ * @param   u16         The tracking data word.
+ * @remarks This will only return 1 or PGMPOOL_TD_CREFS_PHYSEXT for a
+ *          non-zero @a u16. */
+#define PGMPOOL_TD_GET_CREFS(u16)       ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK )
+
+/** @def PGMPOOL_TD_GET_IDX
+ * Get the @a idx field from a tracking data word.
+ *
+ * @returns The @a idx field
+ * @param   u16         The tracking data word. */
+#define PGMPOOL_TD_GET_IDX(u16)         ( ((u16) >> PGMPOOL_TD_IDX_SHIFT)   & PGMPOOL_TD_IDX_MASK   )
+/** @} */
+
+
+
+/** @name A20 gate macros
+ * @{ */
+#define PGM_WITH_A20
+#ifdef PGM_WITH_A20
+# define PGM_A20_IS_ENABLED(a_pVCpu)                        ((a_pVCpu)->pgm.s.fA20Enabled)
+# define PGM_A20_APPLY(a_pVCpu, a_GCPhys)                   ((a_GCPhys) & (a_pVCpu)->pgm.s.GCPhysA20Mask)
+# define PGM_A20_APPLY_TO_VAR(a_pVCpu, a_GCPhysVar)         \
+    do { a_GCPhysVar &= (a_pVCpu)->pgm.s.GCPhysA20Mask; } while (0)
+# define PGM_A20_ASSERT_MASKED(pVCpu, a_GCPhys)             Assert(PGM_A20_APPLY(pVCpu, a_GCPhys) == (a_GCPhys))
+#else
+# define PGM_A20_IS_ENABLED(a_pVCpu)                        (true)
+# define PGM_A20_APPLY(a_pVCpu, a_GCPhys)                   (a_GCPhys)
+# define PGM_A20_APPLY_TO_VAR(a_pVCpu, a_GCPhysVar)         do { } while (0)
+# define PGM_A20_ASSERT_MASKED(pVCpu, a_GCPhys)             do { } while (0)
+#endif
+/** @} */
+
+
+/**
+ * Roots and anchors for trees and list employing self relative offsets as
+ * pointers.
+ *
+ * When using self-relative offsets instead of pointers, the offsets needs to be
+ * the same in all offsets.  Thus the roots and anchors needs to live on the
+ * hyper heap just like the nodes.
+ */
+typedef struct PGMTREES
+{
+    /** List of physical access handler types (offset pointers) of type
+     * PGMPHYSHANDLERTYPEINT.  This is needed for relocations. */
+    RTLISTOFF32ANCHOR               HeadPhysHandlerTypes;
+    /** Physical access handlers (AVL range+offsetptr tree). */
+    AVLROGCPHYSTREE                 PhysHandlers;
+} PGMTREES;
+/** Pointer to PGM trees. */
+typedef PGMTREES *PPGMTREES;
+
+
+/**
+ * Page fault guest state for the AMD64 paging mode.
+ */
+typedef struct PGMPTWALKCORE
+{
+    /** The guest virtual address that is being resolved by the walk
+     *  (input). */
+    RTGCPTR         GCPtr;
+
+    /** The guest physical address that is the result of the walk.
+     * @remarks only valid if fSucceeded is set. */
+    RTGCPHYS        GCPhys;
+
+    /** Set if the walk succeeded, i.d. GCPhys is valid. */
+    bool            fSucceeded;
+    /** The level problem arrised at.
+     * PTE is level 1, PDE is level 2, PDPE is level 3, PML4 is level 4, CR3 is
+     * level 8.  This is 0 on success. */
+    uint8_t         uLevel;
+    /** Set if the page isn't present. */
+    bool            fNotPresent;
+    /** Encountered a bad physical address. */
+    bool            fBadPhysAddr;
+    /** Set if there was reserved bit violations. */
+    bool            fRsvdError;
+    /** Set if it involves a big page (2/4 MB). */
+    bool            fBigPage;
+    /** Set if it involves a gigantic page (1 GB). */
+    bool            fGigantPage;
+    /** The effective X86_PTE_US flag for the address. */
+    bool            fEffectiveUS;
+    /** The effective X86_PTE_RW flag for the address. */
+    bool            fEffectiveRW;
+    /** The effective X86_PTE_NX flag for the address. */
+    bool            fEffectiveNX;
+    bool            afPadding1[2];
+    /** Effective flags thus far: RW, US, PWT, PCD, A, ~NX >> 63.
+     * The NX bit is inverted and shifted down 63 places to bit 0. */
+    uint32_t        fEffective;
+} PGMPTWALKCORE;
+
+/** @name PGMPTWALKCORE::fEffective bits.
+ * @{ */
+/** Effective execute bit (!NX).   */
+#define PGMPTWALK_EFF_X     UINT32_C(1)
+/** Effective write access bit. */
+#define PGMPTWALK_EFF_RW    X86_PTE_RW
+/** Effective user-mode access bit. */
+#define PGMPTWALK_EFF_US    X86_PTE_US
+/** Effective write through cache bit. */
+#define PGMPTWALK_EFF_PWT   X86_PTE_PWT
+/** Effective cache disabled bit. */
+#define PGMPTWALK_EFF_PCD   X86_PTE_PCD
+/** Effective accessed bit. */
+#define PGMPTWALK_EFF_A     X86_PTE_A
+/** The dirty bit of the final entry. */
+#define PGMPTWALK_EFF_D     X86_PTE_D
+/** The PAT bit of the final entry. */
+#define PGMPTWALK_EFF_PAT   X86_PTE_PAT
+/** The global bit of the final entry. */
+#define PGMPTWALK_EFF_G     X86_PTE_G
+/** @} */
+
+
+/**
+ * Guest page table walk for the AMD64 mode.
+ */
+typedef struct PGMPTWALKGSTAMD64
+{
+    /** The common core. */
+    PGMPTWALKCORE   Core;
+
+    PX86PML4        pPml4;
+    PX86PML4E       pPml4e;
+    X86PML4E        Pml4e;
+
+    PX86PDPT        pPdpt;
+    PX86PDPE        pPdpe;
+    X86PDPE         Pdpe;
+
+    PX86PDPAE       pPd;
+    PX86PDEPAE      pPde;
+    X86PDEPAE       Pde;
+
+    PX86PTPAE       pPt;
+    PX86PTEPAE      pPte;
+    X86PTEPAE       Pte;
+} PGMPTWALKGSTAMD64;
+/** Pointer to a AMD64 guest page table walk. */
+typedef PGMPTWALKGSTAMD64 *PPGMPTWALKGSTAMD64;
+/** Pointer to a const AMD64 guest page table walk. */
+typedef PGMPTWALKGSTAMD64 const *PCPGMPTWALKGSTAMD64;
+
+/**
+ * Guest page table walk for the PAE mode.
+ */
+typedef struct PGMPTWALKGSTPAE
+{
+    /** The common core. */
+    PGMPTWALKCORE   Core;
+
+    PX86PDPT        pPdpt;
+    PX86PDPE        pPdpe;
+    X86PDPE         Pdpe;
+
+    PX86PDPAE       pPd;
+    PX86PDEPAE      pPde;
+    X86PDEPAE       Pde;
+
+    PX86PTPAE       pPt;
+    PX86PTEPAE      pPte;
+    X86PTEPAE       Pte;
+} PGMPTWALKGSTPAE;
+/** Pointer to a PAE guest page table walk. */
+typedef PGMPTWALKGSTPAE *PPGMPTWALKGSTPAE;
+/** Pointer to a const AMD64 guest page table walk. */
+typedef PGMPTWALKGSTPAE const *PCPGMPTWALKGSTPAE;
+
+/**
+ * Guest page table walk for the 32-bit mode.
+ */
+typedef struct PGMPTWALKGST32BIT
+{
+    /** The common core. */
+    PGMPTWALKCORE   Core;
+
+    PX86PD          pPd;
+    PX86PDE         pPde;
+    X86PDE          Pde;
+
+    PX86PT          pPt;
+    PX86PTE         pPte;
+    X86PTE          Pte;
+} PGMPTWALKGST32BIT;
+/** Pointer to a 32-bit guest page table walk. */
+typedef PGMPTWALKGST32BIT *PPGMPTWALKGST32BIT;
+/** Pointer to a const 32-bit guest page table walk. */
+typedef PGMPTWALKGST32BIT const *PCPGMPTWALKGST32BIT;
+
+/**
+ * Which part of PGMPTWALKGST that is valid.
+ */
+typedef enum PGMPTWALKGSTTYPE
+{
+    /** Customary invalid 0 value. */
+    PGMPTWALKGSTTYPE_INVALID = 0,
+    /**  PGMPTWALKGST::u.Amd64 is valid. */
+    PGMPTWALKGSTTYPE_AMD64,
+    /**  PGMPTWALKGST::u.Pae is valid. */
+    PGMPTWALKGSTTYPE_PAE,
+    /**  PGMPTWALKGST::u.Legacy is valid. */
+    PGMPTWALKGSTTYPE_32BIT,
+    /** Customary 32-bit type hack. */
+    PGMPTWALKGSTTYPE_32BIT_HACK = 0x7fff0000
+} PGMPTWALKGSTTYPE;
+
+/**
+ * Combined guest page table walk result.
+ */
+typedef struct PGMPTWALKGST
+{
+    union
+    {
+        /** The page walker core - always valid. */
+        PGMPTWALKCORE       Core;
+        /** The page walker for AMD64. */
+        PGMPTWALKGSTAMD64   Amd64;
+        /** The page walker for PAE (32-bit). */
+        PGMPTWALKGSTPAE     Pae;
+        /** The page walker for 32-bit paging (called legacy due to C naming
+         * convension). */
+        PGMPTWALKGST32BIT   Legacy;
+    } u;
+    /** Indicates which part of the union is valid. */
+    PGMPTWALKGSTTYPE        enmType;
+} PGMPTWALKGST;
+/** Pointer to a combined guest page table walk result. */
+typedef PGMPTWALKGST *PPGMPTWALKGST;
+/** Pointer to a read-only combined guest page table walk result. */
+typedef PGMPTWALKGST const *PCPGMPTWALKGST;
+
+
+/** @name Paging mode macros
+ * @{
+ */
+#ifdef IN_RING3
+# define PGM_CTX(a,b)                   a##R3##b
+# define PGM_CTX_STR(a,b)               a "R3" b
+# define PGM_CTX_DECL(type)             DECLCALLBACK(type)
+#elif defined(IN_RING0)
+# define PGM_CTX(a,b)                   a##R0##b
+# define PGM_CTX_STR(a,b)               a "R0" b
+# define PGM_CTX_DECL(type)             VMMDECL(type)
+#else
+# error "Not IN_RING3 or IN_RING0!"
+#endif
+
+#define PGM_GST_NAME_REAL(name)                         PGM_CTX(pgm,GstReal##name)
+#define PGM_GST_NAME_RC_REAL_STR(name)                  "pgmRCGstReal" #name
+#define PGM_GST_NAME_R0_REAL_STR(name)                  "pgmR0GstReal" #name
+#define PGM_GST_NAME_PROT(name)                         PGM_CTX(pgm,GstProt##name)
+#define PGM_GST_NAME_RC_PROT_STR(name)                  "pgmRCGstProt" #name
+#define PGM_GST_NAME_R0_PROT_STR(name)                  "pgmR0GstProt" #name
+#define PGM_GST_NAME_32BIT(name)                        PGM_CTX(pgm,Gst32Bit##name)
+#define PGM_GST_NAME_RC_32BIT_STR(name)                 "pgmRCGst32Bit" #name
+#define PGM_GST_NAME_R0_32BIT_STR(name)                 "pgmR0Gst32Bit" #name
+#define PGM_GST_NAME_PAE(name)                          PGM_CTX(pgm,GstPAE##name)
+#define PGM_GST_NAME_RC_PAE_STR(name)                   "pgmRCGstPAE" #name
+#define PGM_GST_NAME_R0_PAE_STR(name)                   "pgmR0GstPAE" #name
+#define PGM_GST_NAME_AMD64(name)                        PGM_CTX(pgm,GstAMD64##name)
+#define PGM_GST_NAME_RC_AMD64_STR(name)                 "pgmRCGstAMD64" #name
+#define PGM_GST_NAME_R0_AMD64_STR(name)                 "pgmR0GstAMD64" #name
+#define PGM_GST_DECL(type, name)                        PGM_CTX_DECL(type) PGM_GST_NAME(name)
+
+#define PGM_SHW_NAME_32BIT(name)                        PGM_CTX(pgm,Shw32Bit##name)
+#define PGM_SHW_NAME_RC_32BIT_STR(name)                 "pgmRCShw32Bit" #name
+#define PGM_SHW_NAME_R0_32BIT_STR(name)                 "pgmR0Shw32Bit" #name
+#define PGM_SHW_NAME_PAE(name)                          PGM_CTX(pgm,ShwPAE##name)
+#define PGM_SHW_NAME_RC_PAE_STR(name)                   "pgmRCShwPAE" #name
+#define PGM_SHW_NAME_R0_PAE_STR(name)                   "pgmR0ShwPAE" #name
+#define PGM_SHW_NAME_AMD64(name)                        PGM_CTX(pgm,ShwAMD64##name)
+#define PGM_SHW_NAME_RC_AMD64_STR(name)                 "pgmRCShwAMD64" #name
+#define PGM_SHW_NAME_R0_AMD64_STR(name)                 "pgmR0ShwAMD64" #name
+#define PGM_SHW_NAME_NESTED_32BIT(name)                 PGM_CTX(pgm,ShwNested32Bit##name)
+#define PGM_SHW_NAME_RC_NESTED_32BIT_STR(name)          "pgmRCShwNested32Bit" #name
+#define PGM_SHW_NAME_R0_NESTED_32BIT_STR(name)          "pgmR0ShwNested32Bit" #name
+#define PGM_SHW_NAME_NESTED_PAE(name)                   PGM_CTX(pgm,ShwNestedPAE##name)
+#define PGM_SHW_NAME_RC_NESTED_PAE_STR(name)            "pgmRCShwNestedPAE" #name
+#define PGM_SHW_NAME_R0_NESTED_PAE_STR(name)            "pgmR0ShwNestedPAE" #name
+#define PGM_SHW_NAME_NESTED_AMD64(name)                 PGM_CTX(pgm,ShwNestedAMD64##name)
+#define PGM_SHW_NAME_RC_NESTED_AMD64_STR(name)          "pgmRCShwNestedAMD64" #name
+#define PGM_SHW_NAME_R0_NESTED_AMD64_STR(name)          "pgmR0ShwNestedAMD64" #name
+#define PGM_SHW_NAME_EPT(name)                          PGM_CTX(pgm,ShwEPT##name)
+#define PGM_SHW_NAME_RC_EPT_STR(name)                   "pgmRCShwEPT" #name
+#define PGM_SHW_NAME_R0_EPT_STR(name)                   "pgmR0ShwEPT" #name
+#define PGM_SHW_NAME_NONE(name)                         PGM_CTX(pgm,ShwNone##name)
+#define PGM_SHW_NAME_RC_NONE_STR(name)                  "pgmRCShwNone" #name
+#define PGM_SHW_NAME_R0_NONE_STR(name)                  "pgmR0ShwNone" #name
+#define PGM_SHW_DECL(type, name)                        PGM_CTX_DECL(type) PGM_SHW_NAME(name)
+
+/*                   Shw_Gst */
+#define PGM_BTH_NAME_32BIT_REAL(name)                   PGM_CTX(pgm,Bth32BitReal##name)
+#define PGM_BTH_NAME_32BIT_PROT(name)                   PGM_CTX(pgm,Bth32BitProt##name)
+#define PGM_BTH_NAME_32BIT_32BIT(name)                  PGM_CTX(pgm,Bth32Bit32Bit##name)
+#define PGM_BTH_NAME_PAE_REAL(name)                     PGM_CTX(pgm,BthPAEReal##name)
+#define PGM_BTH_NAME_PAE_PROT(name)                     PGM_CTX(pgm,BthPAEProt##name)
+#define PGM_BTH_NAME_PAE_32BIT(name)                    PGM_CTX(pgm,BthPAE32Bit##name)
+#define PGM_BTH_NAME_PAE_PAE(name)                      PGM_CTX(pgm,BthPAEPAE##name)
+#define PGM_BTH_NAME_AMD64_PROT(name)                   PGM_CTX(pgm,BthAMD64Prot##name)
+#define PGM_BTH_NAME_AMD64_AMD64(name)                  PGM_CTX(pgm,BthAMD64AMD64##name)
+#define PGM_BTH_NAME_NESTED_32BIT_REAL(name)            PGM_CTX(pgm,BthNested32BitReal##name)
+#define PGM_BTH_NAME_NESTED_32BIT_PROT(name)            PGM_CTX(pgm,BthNested32BitProt##name)
+#define PGM_BTH_NAME_NESTED_32BIT_32BIT(name)           PGM_CTX(pgm,BthNested32Bit32Bit##name)
+#define PGM_BTH_NAME_NESTED_32BIT_PAE(name)             PGM_CTX(pgm,BthNested32BitPAE##name)
+#define PGM_BTH_NAME_NESTED_32BIT_AMD64(name)           PGM_CTX(pgm,BthNested32BitAMD64##name)
+#define PGM_BTH_NAME_NESTED_PAE_REAL(name)              PGM_CTX(pgm,BthNestedPAEReal##name)
+#define PGM_BTH_NAME_NESTED_PAE_PROT(name)              PGM_CTX(pgm,BthNestedPAEProt##name)
+#define PGM_BTH_NAME_NESTED_PAE_32BIT(name)             PGM_CTX(pgm,BthNestedPAE32Bit##name)
+#define PGM_BTH_NAME_NESTED_PAE_PAE(name)               PGM_CTX(pgm,BthNestedPAEPAE##name)
+#define PGM_BTH_NAME_NESTED_PAE_AMD64(name)             PGM_CTX(pgm,BthNestedPAEAMD64##name)
+#define PGM_BTH_NAME_NESTED_AMD64_REAL(name)            PGM_CTX(pgm,BthNestedAMD64Real##name)
+#define PGM_BTH_NAME_NESTED_AMD64_PROT(name)            PGM_CTX(pgm,BthNestedAMD64Prot##name)
+#define PGM_BTH_NAME_NESTED_AMD64_32BIT(name)           PGM_CTX(pgm,BthNestedAMD6432Bit##name)
+#define PGM_BTH_NAME_NESTED_AMD64_PAE(name)             PGM_CTX(pgm,BthNestedAMD64PAE##name)
+#define PGM_BTH_NAME_NESTED_AMD64_AMD64(name)           PGM_CTX(pgm,BthNestedAMD64AMD64##name)
+#define PGM_BTH_NAME_EPT_REAL(name)                     PGM_CTX(pgm,BthEPTReal##name)
+#define PGM_BTH_NAME_EPT_PROT(name)                     PGM_CTX(pgm,BthEPTProt##name)
+#define PGM_BTH_NAME_EPT_32BIT(name)                    PGM_CTX(pgm,BthEPT32Bit##name)
+#define PGM_BTH_NAME_EPT_PAE(name)                      PGM_CTX(pgm,BthEPTPAE##name)
+#define PGM_BTH_NAME_EPT_AMD64(name)                    PGM_CTX(pgm,BthEPTAMD64##name)
+#define PGM_BTH_NAME_NONE_REAL(name)                    PGM_CTX(pgm,BthNoneReal##name)
+#define PGM_BTH_NAME_NONE_PROT(name)                    PGM_CTX(pgm,BthNoneProt##name)
+#define PGM_BTH_NAME_NONE_32BIT(name)                   PGM_CTX(pgm,BthNone32Bit##name)
+#define PGM_BTH_NAME_NONE_PAE(name)                     PGM_CTX(pgm,BthNonePAE##name)
+#define PGM_BTH_NAME_NONE_AMD64(name)                   PGM_CTX(pgm,BthNoneAMD64##name)
+
+#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name)            "pgmRCBth32BitReal" #name
+#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name)            "pgmRCBth32BitProt" #name
+#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name)           "pgmRCBth32Bit32Bit" #name
+#define PGM_BTH_NAME_RC_PAE_REAL_STR(name)              "pgmRCBthPAEReal" #name
+#define PGM_BTH_NAME_RC_PAE_PROT_STR(name)              "pgmRCBthPAEProt" #name
+#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name)             "pgmRCBthPAE32Bit" #name
+#define PGM_BTH_NAME_RC_PAE_PAE_STR(name)               "pgmRCBthPAEPAE" #name
+#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name)           "pgmRCBthAMD64AMD64" #name
+#define PGM_BTH_NAME_RC_NESTED_32BIT_REAL_STR(name)     "pgmRCBthNested32BitReal" #name
+#define PGM_BTH_NAME_RC_NESTED_32BIT_PROT_STR(name)     "pgmRCBthNested32BitProt" #name
+#define PGM_BTH_NAME_RC_NESTED_32BIT_32BIT_STR(name)    "pgmRCBthNested32Bit32Bit" #name
+#define PGM_BTH_NAME_RC_NESTED_32BIT_PAE_STR(name)      "pgmRCBthNested32BitPAE" #name
+#define PGM_BTH_NAME_RC_NESTED_32BIT_AMD64_STR(name)    "pgmRCBthNested32BitAMD64" #name
+#define PGM_BTH_NAME_RC_NESTED_PAE_REAL_STR(name)       "pgmRCBthNestedPAEReal" #name
+#define PGM_BTH_NAME_RC_NESTED_PAE_PROT_STR(name)       "pgmRCBthNestedPAEProt" #name
+#define PGM_BTH_NAME_RC_NESTED_PAE_32BIT_STR(name)      "pgmRCBthNestedPAE32Bit" #name
+#define PGM_BTH_NAME_RC_NESTED_PAE_PAE_STR(name)        "pgmRCBthNestedPAEPAE" #name
+#define PGM_BTH_NAME_RC_NESTED_PAE_AMD64_STR(name)      "pgmRCBthNestedPAEAMD64" #name
+#define PGM_BTH_NAME_RC_NESTED_AMD64_REAL_STR(name)     "pgmRCBthNestedAMD64Real" #name
+#define PGM_BTH_NAME_RC_NESTED_AMD64_PROT_STR(name)     "pgmRCBthNestedAMD64Prot" #name
+#define PGM_BTH_NAME_RC_NESTED_AMD64_32BIT_STR(name)    "pgmRCBthNestedAMD6432Bit" #name
+#define PGM_BTH_NAME_RC_NESTED_AMD64_PAE_STR(name)      "pgmRCBthNestedAMD64PAE" #name
+#define PGM_BTH_NAME_RC_NESTED_AMD64_AMD64_STR(name)    "pgmRCBthNestedAMD64AMD64" #name
+#define PGM_BTH_NAME_RC_EPT_REAL_STR(name)              "pgmRCBthEPTReal" #name
+#define PGM_BTH_NAME_RC_EPT_PROT_STR(name)              "pgmRCBthEPTProt" #name
+#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name)             "pgmRCBthEPT32Bit" #name
+#define PGM_BTH_NAME_RC_EPT_PAE_STR(name)               "pgmRCBthEPTPAE" #name
+#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name)             "pgmRCBthEPTAMD64" #name
+
+#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name)            "pgmR0Bth32BitReal" #name
+#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name)            "pgmR0Bth32BitProt" #name
+#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name)           "pgmR0Bth32Bit32Bit" #name
+#define PGM_BTH_NAME_R0_PAE_REAL_STR(name)              "pgmR0BthPAEReal" #name
+#define PGM_BTH_NAME_R0_PAE_PROT_STR(name)              "pgmR0BthPAEProt" #name
+#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name)             "pgmR0BthPAE32Bit" #name
+#define PGM_BTH_NAME_R0_PAE_PAE_STR(name)               "pgmR0BthPAEPAE" #name
+#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name)            "pgmR0BthAMD64Prot" #name
+#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name)           "pgmR0BthAMD64AMD64" #name
+#define PGM_BTH_NAME_R0_NESTED_32BIT_REAL_STR(name)     "pgmR0BthNested32BitReal" #name
+#define PGM_BTH_NAME_R0_NESTED_32BIT_PROT_STR(name)     "pgmR0BthNested32BitProt" #name
+#define PGM_BTH_NAME_R0_NESTED_32BIT_32BIT_STR(name)    "pgmR0BthNested32Bit32Bit" #name
+#define PGM_BTH_NAME_R0_NESTED_32BIT_PAE_STR(name)      "pgmR0BthNested32BitPAE" #name
+#define PGM_BTH_NAME_R0_NESTED_32BIT_AMD64_STR(name)    "pgmR0BthNested32BitAMD64" #name
+#define PGM_BTH_NAME_R0_NESTED_PAE_REAL_STR(name)       "pgmR0BthNestedPAEReal" #name
+#define PGM_BTH_NAME_R0_NESTED_PAE_PROT_STR(name)       "pgmR0BthNestedPAEProt" #name
+#define PGM_BTH_NAME_R0_NESTED_PAE_32BIT_STR(name)      "pgmR0BthNestedPAE32Bit" #name
+#define PGM_BTH_NAME_R0_NESTED_PAE_PAE_STR(name)        "pgmR0BthNestedPAEPAE" #name
+#define PGM_BTH_NAME_R0_NESTED_PAE_AMD64_STR(name)      "pgmR0BthNestedPAEAMD64" #name
+#define PGM_BTH_NAME_R0_NESTED_AMD64_REAL_STR(name)     "pgmR0BthNestedAMD64Real" #name
+#define PGM_BTH_NAME_R0_NESTED_AMD64_PROT_STR(name)     "pgmR0BthNestedAMD64Prot" #name
+#define PGM_BTH_NAME_R0_NESTED_AMD64_32BIT_STR(name)    "pgmR0BthNestedAMD6432Bit" #name
+#define PGM_BTH_NAME_R0_NESTED_AMD64_PAE_STR(name)      "pgmR0BthNestedAMD64PAE" #name
+#define PGM_BTH_NAME_R0_NESTED_AMD64_AMD64_STR(name)    "pgmR0BthNestedAMD64AMD64" #name
+#define PGM_BTH_NAME_R0_EPT_REAL_STR(name)              "pgmR0BthEPTReal" #name
+#define PGM_BTH_NAME_R0_EPT_PROT_STR(name)              "pgmR0BthEPTProt" #name
+#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name)             "pgmR0BthEPT32Bit" #name
+#define PGM_BTH_NAME_R0_EPT_PAE_STR(name)               "pgmR0BthEPTPAE" #name
+#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name)             "pgmR0BthEPTAMD64" #name
+
+#define PGM_BTH_DECL(type, name)        PGM_CTX_DECL(type) PGM_BTH_NAME(name)
+/** @} */
+
+
+/**
+ * Function pointers for guest paging.
+ */
+typedef struct PGMMODEDATAGST
+{
+    /** The guest mode type. */
+    uint32_t                        uType;
+    DECLCALLBACKMEMBER(int,         pfnGetPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys);
+    DECLCALLBACKMEMBER(int,         pfnModifyPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask);
+    DECLCALLBACKMEMBER(int,         pfnGetPDE)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde);
+    DECLCALLBACKMEMBER(int,         pfnEnter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
+    DECLCALLBACKMEMBER(int,         pfnExit)(PVMCPUCC pVCpu);
+#ifdef IN_RING3
+    DECLCALLBACKMEMBER(int,         pfnRelocate)(PVMCPUCC pVCpu, RTGCPTR offDelta); /**< Only in ring-3. */
+#endif
+} PGMMODEDATAGST;
+
+/** The length of g_aPgmGuestModeData. */
+#ifdef VBOX_WITH_64_BITS_GUESTS
+# define PGM_GUEST_MODE_DATA_ARRAY_SIZE     (PGM_TYPE_AMD64 + 1)
+#else
+# define PGM_GUEST_MODE_DATA_ARRAY_SIZE     (PGM_TYPE_PAE + 1)
+#endif
+/** The guest mode data array. */
+extern PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE];
+
+
+/**
+ * Function pointers for shadow paging.
+ */
+typedef struct PGMMODEDATASHW
+{
+    /** The shadow mode type. */
+    uint32_t                        uType;
+    DECLCALLBACKMEMBER(int,         pfnGetPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys);
+    DECLCALLBACKMEMBER(int,         pfnModifyPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags,
+                                                   uint64_t fMask, uint32_t fOpFlags);
+    DECLCALLBACKMEMBER(int,         pfnEnter)(PVMCPUCC pVCpu, bool fIs64BitsPagingMode);
+    DECLCALLBACKMEMBER(int,         pfnExit)(PVMCPUCC pVCpu);
+#ifdef IN_RING3
+    DECLCALLBACKMEMBER(int,         pfnRelocate)(PVMCPUCC pVCpu, RTGCPTR offDelta); /**< Only in ring-3. */
+#endif
+} PGMMODEDATASHW;
+
+/** The length of g_aPgmShadowModeData. */
+#define PGM_SHADOW_MODE_DATA_ARRAY_SIZE     PGM_TYPE_END
+/** The shadow mode data array. */
+extern PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE];
+
+
+/**
+ * Function pointers for guest+shadow paging.
+ */
+typedef struct PGMMODEDATABTH
+{
+    /** The shadow mode type. */
+    uint32_t                        uShwType;
+    /** The guest mode type. */
+    uint32_t                        uGstType;
+
+    DECLCALLBACKMEMBER(int,         pfnInvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
+    DECLCALLBACKMEMBER(int,         pfnSyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
+    DECLCALLBACKMEMBER(int,         pfnPrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
+    DECLCALLBACKMEMBER(int,         pfnVerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError);
+    DECLCALLBACKMEMBER(int,         pfnMapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
+    DECLCALLBACKMEMBER(int,         pfnUnmapCR3)(PVMCPUCC pVCpu);
+    DECLCALLBACKMEMBER(int,         pfnEnter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
+#ifndef IN_RING3
+    DECLCALLBACKMEMBER(int,         pfnTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
+#endif
+#ifdef VBOX_STRICT
+    DECLCALLBACKMEMBER(unsigned,    pfnAssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb);
+#endif
+} PGMMODEDATABTH;
+
+/** The length of g_aPgmBothModeData. */
+#define PGM_BOTH_MODE_DATA_ARRAY_SIZE       ((PGM_TYPE_END     - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END)
+/** The guest+shadow mode data array. */
+extern PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE];
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * PGM statistics.
+ *
+ * These lives on the heap when compiled in as they would otherwise waste
+ * unnecessary space in release builds.
+ */
+typedef struct PGMSTATS
+{
+    /* R3 only: */
+    STAMCOUNTER StatR3DetectedConflicts;            /**< R3: Number of times PGMR3MapHasConflicts() detected a conflict. */
+    STAMPROFILE StatR3ResolveConflict;              /**< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). */
+
+    /* R3+RZ */
+    STAMCOUNTER StatRZChunkR3MapTlbHits;            /**< RC/R0: Ring-3/0 chunk mapper TLB hits. */
+    STAMCOUNTER StatRZChunkR3MapTlbMisses;          /**< RC/R0: Ring-3/0 chunk mapper TLB misses. */
+    STAMCOUNTER StatRZPageMapTlbHits;               /**< RC/R0: Ring-3/0 page mapper TLB hits. */
+    STAMCOUNTER StatRZPageMapTlbMisses;             /**< RC/R0: Ring-3/0 page mapper TLB misses. */
+    STAMCOUNTER StatPageMapTlbFlushes;              /**< ALL: Ring-3/0 page mapper TLB flushes. */
+    STAMCOUNTER StatPageMapTlbFlushEntry;           /**< ALL: Ring-3/0 page mapper TLB flushes. */
+    STAMCOUNTER StatR3ChunkR3MapTlbHits;            /**< R3: Ring-3/0 chunk mapper TLB hits. */
+    STAMCOUNTER StatR3ChunkR3MapTlbMisses;          /**< R3: Ring-3/0 chunk mapper TLB misses. */
+    STAMCOUNTER StatR3PageMapTlbHits;               /**< R3: Ring-3/0 page mapper TLB hits. */
+    STAMCOUNTER StatR3PageMapTlbMisses;             /**< R3: Ring-3/0 page mapper TLB misses. */
+    STAMCOUNTER StatRZRamRangeTlbHits;              /**< RC/R0: RAM range TLB hits. */
+    STAMCOUNTER StatRZRamRangeTlbMisses;            /**< RC/R0: RAM range TLB misses. */
+    STAMCOUNTER StatR3RamRangeTlbHits;              /**< R3: RAM range TLB hits. */
+    STAMCOUNTER StatR3RamRangeTlbMisses;            /**< R3: RAM range TLB misses. */
+    STAMCOUNTER StatR3PhysHandlerReset;             /**< R3: The number of times PGMHandlerPhysicalReset is called. */
+    STAMCOUNTER StatRZPhysHandlerReset;             /**< RC/R0: The number of times PGMHandlerPhysicalReset is called. */
+    STAMCOUNTER StatR3PhysHandlerLookupHits;        /**< R3: Number of cache hits when looking up physical handlers. */
+    STAMCOUNTER StatR3PhysHandlerLookupMisses;      /**< R3: Number of cache misses when looking up physical handlers. */
+    STAMCOUNTER StatRZPhysHandlerLookupHits;        /**< RC/R0: Number of cache hits when lookup up physical handlers. */
+    STAMCOUNTER StatRZPhysHandlerLookupMisses;      /**< RC/R0: Number of cache misses when looking up physical handlers */
+    STAMCOUNTER StatRZPageReplaceShared;            /**< RC/R0: Times a shared page has been replaced by a private one. */
+    STAMCOUNTER StatRZPageReplaceZero;              /**< RC/R0: Times the zero page has been replaced by a private one. */
+/// @todo    STAMCOUNTER StatRZPageHandyAllocs;              /**< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. */
+    STAMCOUNTER StatR3PageReplaceShared;            /**< R3: Times a shared page has been replaced by a private one. */
+    STAMCOUNTER StatR3PageReplaceZero;              /**< R3: Times the zero page has been replaced by a private one. */
+/// @todo    STAMCOUNTER StatR3PageHandyAllocs;              /**< R3: The number of times we've executed GMMR3AllocateHandyPages. */
+
+    /* RC only: */
+    STAMCOUNTER StatRCInvlPgConflict;               /**< RC: Number of times PGMInvalidatePage() detected a mapping conflict. */
+    STAMCOUNTER StatRCInvlPgSyncMonCR3;             /**< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. */
+
+    STAMCOUNTER StatRZPhysRead;
+    STAMCOUNTER StatRZPhysReadBytes;
+    STAMCOUNTER StatRZPhysWrite;
+    STAMCOUNTER StatRZPhysWriteBytes;
+    STAMCOUNTER StatR3PhysRead;
+    STAMCOUNTER StatR3PhysReadBytes;
+    STAMCOUNTER StatR3PhysWrite;
+    STAMCOUNTER StatR3PhysWriteBytes;
+    STAMCOUNTER StatRCPhysRead;
+    STAMCOUNTER StatRCPhysReadBytes;
+    STAMCOUNTER StatRCPhysWrite;
+    STAMCOUNTER StatRCPhysWriteBytes;
+
+    STAMCOUNTER StatRZPhysSimpleRead;
+    STAMCOUNTER StatRZPhysSimpleReadBytes;
+    STAMCOUNTER StatRZPhysSimpleWrite;
+    STAMCOUNTER StatRZPhysSimpleWriteBytes;
+    STAMCOUNTER StatR3PhysSimpleRead;
+    STAMCOUNTER StatR3PhysSimpleReadBytes;
+    STAMCOUNTER StatR3PhysSimpleWrite;
+    STAMCOUNTER StatR3PhysSimpleWriteBytes;
+    STAMCOUNTER StatRCPhysSimpleRead;
+    STAMCOUNTER StatRCPhysSimpleReadBytes;
+    STAMCOUNTER StatRCPhysSimpleWrite;
+    STAMCOUNTER StatRCPhysSimpleWriteBytes;
+
+    STAMCOUNTER StatTrackVirgin;                    /**< The number of first time shadowings. */
+    STAMCOUNTER StatTrackAliased;                   /**< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. */
+    STAMCOUNTER StatTrackAliasedMany;               /**< The number of times we're tracking using cRef2. */
+    STAMCOUNTER StatTrackAliasedLots;               /**< The number of times we're hitting pages which has overflowed cRef2. */
+    STAMCOUNTER StatTrackNoExtentsLeft;             /**< The number of times the extent list was exhausted. */
+    STAMCOUNTER StatTrackOverflows;                 /**< The number of times the extent list grows to long. */
+    STAMPROFILE StatTrackDeref;                     /**< Profiling of SyncPageWorkerTrackDeref (expensive). */
+
+    /** Time spent by the host OS for large page allocation. */
+    STAMPROFILE                 StatAllocLargePage;
+    /** Time spent clearing the newly allocated large pages. */
+    STAMPROFILE                 StatClearLargePage;
+    /** The number of times allocating a large pages takes more than the allowed period. */
+    STAMCOUNTER                 StatLargePageOverflow;
+    /** pgmPhysIsValidLargePage profiling - R3 */
+    STAMPROFILE                 StatR3IsValidLargePage;
+    /** pgmPhysIsValidLargePage profiling - RZ*/
+    STAMPROFILE                 StatRZIsValidLargePage;
+
+    STAMPROFILE                 StatChunkAging;
+    STAMPROFILE                 StatChunkFindCandidate;
+    STAMPROFILE                 StatChunkUnmap;
+    STAMPROFILE                 StatChunkMap;
+} PGMSTATS;
+#endif /* VBOX_WITH_STATISTICS */
+
+
+/**
+ * Converts a PGM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the PGM is part of.
+ * @param   pPGM   Pointer to PGM instance data.
+ */
+#define PGM2VM(pPGM)  ( (PVM)((char*)pPGM - pPGM->offVM) )
+
+/**
+ * PGM Data (part of VM)
+ */
+typedef struct PGM
+{
+    /** Offset to the VM structure. */
+    int32_t                         offVM;
+    /** Offset of the PGMCPU structure relative to VMCPU. */
+    int32_t                         offVCpuPGM;
+
+    /** @cfgm{/RamPreAlloc, boolean, false}
+     * Indicates whether the base RAM should all be allocated before starting
+     * the VM (default), or if it should be allocated when first written to.
+     */
+    bool                            fRamPreAlloc;
+    /** Indicates whether write monitoring is currently in use.
+     * This is used to prevent conflicts between live saving and page sharing
+     * detection. */
+    bool                            fPhysWriteMonitoringEngaged;
+    /** Set if the CPU has less than 52-bit physical address width.
+     * This is used */
+    bool                            fLessThan52PhysicalAddressBits;
+    /** Set when nested paging is active.
+     * This is meant to save calls to HMIsNestedPagingActive and let the
+     * compilers optimize the code better.  Whether we use nested paging or
+     * not is something we find out during VMM initialization and we won't
+     * change this later on. */
+    bool                            fNestedPaging;
+    /** The host paging mode. (This is what SUPLib reports.) */
+    SUPPAGINGMODE                   enmHostMode;
+    /** We're not in a state which permits writes to guest memory.
+     * (Only used in strict builds.) */
+    bool                            fNoMorePhysWrites;
+    /** @cfgm{/PageFusionAllowed, boolean, false}
+     * Whether page fusion is allowed. */
+    bool                            fPageFusionAllowed;
+    /** @cfgm{/PGM/PciPassThrough, boolean, false}
+     * Whether PCI passthrough is enabled. */
+    bool                            fPciPassthrough;
+    /** The number of MMIO2 regions (serves as the next MMIO2 ID). */
+    uint8_t                         cMmio2Regions;
+    /** Restore original ROM page content when resetting after loading state.
+     * The flag is set by pgmR3LoadRomRanges and cleared at reset.  This
+     * enables the VM to start using an updated ROM without requiring powering
+     * down the VM, just rebooting or resetting it. */
+    bool                            fRestoreRomPagesOnReset;
+    /** Whether to automatically clear all RAM pages on reset. */
+    bool                            fZeroRamPagesOnReset;
+    /** Alignment padding. */
+    bool                            afAlignment3[7];
+
+    /** Indicates that PGMR3FinalizeMappings has been called and that further
+     * PGMR3MapIntermediate calls will be rejected. */
+    bool                            fFinalizedMappings;
+    /** If set no conflict checks are required. */
+    bool                            fMappingsFixed;
+    /** If set if restored as fixed but we were unable to re-fixate at the old
+     *  location because of room or address incompatibilities. */
+    bool                            fMappingsFixedRestored;
+    /** Size of fixed mapping.
+     * This is valid if either fMappingsFixed or fMappingsFixedRestored is set. */
+    uint32_t                        cbMappingFixed;
+    /** Generation ID for the RAM ranges. This member is incremented everytime
+     * a RAM range is linked or unlinked. */
+    uint32_t volatile               idRamRangesGen;
+
+    /** Base address (GC) of fixed mapping.
+     * This is valid if either fMappingsFixed or fMappingsFixedRestored is set. */
+    RTGCPTR                         GCPtrMappingFixed;
+    /** The address of the previous RAM range mapping. */
+    RTGCPTR                         GCPtrPrevRamRangeMapping;
+
+    /** Physical access handler type for ROM protection. */
+    PGMPHYSHANDLERTYPE              hRomPhysHandlerType;
+    /** Alignment padding.   */
+    uint32_t                        u32Padding;
+
+    /** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */
+    RTGCPHYS                        GCPhys4MBPSEMask;
+    /** Mask containing the invalid bits of a guest physical address.
+     * @remarks this does not stop at bit 52. */
+    RTGCPHYS                        GCPhysInvAddrMask;
+
+
+    /** RAM range TLB for R3. */
+    R3PTRTYPE(PPGMRAMRANGE)         apRamRangesTlbR3[PGM_RAMRANGE_TLB_ENTRIES];
+    /** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
+     * This is sorted by physical address and contains no overlapping ranges. */
+    R3PTRTYPE(PPGMRAMRANGE)         pRamRangesXR3;
+    /** Root of the RAM range search tree for ring-3. */
+    R3PTRTYPE(PPGMRAMRANGE)         pRamRangeTreeR3;
+    /** PGM offset based trees - R3 Ptr. */
+    R3PTRTYPE(PPGMTREES)            pTreesR3;
+    /** Caching the last physical handler we looked up in R3. */
+    R3PTRTYPE(PPGMPHYSHANDLER)      pLastPhysHandlerR3;
+    /** Shadow Page Pool - R3 Ptr. */
+    R3PTRTYPE(PPGMPOOL)             pPoolR3;
+#ifndef PGM_WITHOUT_MAPPINGS
+    /** Linked list of GC mappings - for HC.
+     * The list is sorted ascending on address. */
+    R3PTRTYPE(PPGMMAPPING)          pMappingsR3;
+#endif
+    /** Pointer to the list of ROM ranges - for R3.
+     * This is sorted by physical address and contains no overlapping ranges. */
+    R3PTRTYPE(PPGMROMRANGE)         pRomRangesR3;
+    /** Pointer to the list of MMIO2 ranges - for R3.
+     * Registration order. */
+    R3PTRTYPE(PPGMREGMMIO2RANGE)    pRegMmioRangesR3;
+    /** MMIO2 lookup array for ring-3.  Indexed by idMmio2 minus 1. */
+    R3PTRTYPE(PPGMREGMMIO2RANGE)    apMmio2RangesR3[PGM_MMIO2_MAX_RANGES];
+
+    /** RAM range TLB for R0. */
+    R0PTRTYPE(PPGMRAMRANGE)         apRamRangesTlbR0[PGM_RAMRANGE_TLB_ENTRIES];
+    /** R0 pointer corresponding to PGM::pRamRangesXR3. */
+    R0PTRTYPE(PPGMRAMRANGE)         pRamRangesXR0;
+    /** Root of the RAM range search tree for ring-0. */
+    R0PTRTYPE(PPGMRAMRANGE)         pRamRangeTreeR0;
+    /** PGM offset based trees - R0 Ptr. */
+    R0PTRTYPE(PPGMTREES)            pTreesR0;
+    /** Caching the last physical handler we looked up in R0. */
+    R0PTRTYPE(PPGMPHYSHANDLER)      pLastPhysHandlerR0;
+    /** Shadow Page Pool - R0 Ptr. */
+    R0PTRTYPE(PPGMPOOL)             pPoolR0;
+#ifndef PGM_WITHOUT_MAPPINGS
+    /** Linked list of GC mappings - for R0.
+     * The list is sorted ascending on address. */
+    R0PTRTYPE(PPGMMAPPING)          pMappingsR0;
+    RTR0PTR                         R0PtrAlignment0;
+#endif
+    /** R0 pointer corresponding to PGM::pRomRangesR3. */
+    R0PTRTYPE(PPGMROMRANGE)         pRomRangesR0;
+    /** MMIO2 lookup array for ring-0.  Indexed by idMmio2 minus 1. */
+    R0PTRTYPE(PPGMREGMMIO2RANGE)    apMmio2RangesR0[PGM_MMIO2_MAX_RANGES];
+
+#ifndef PGM_WITHOUT_MAPPINGS
+    /** Pointer to the 5 page CR3 content mapping.
+     * The first page is always the CR3 (in some form) while the 4 other pages
+     * are used for the PDs in PAE mode. */
+    RTGCPTR                         GCPtrCR3Mapping;
+
+    /** @name Intermediate Context
+     * @{ */
+    /** Pointer to the intermediate page directory - Normal. */
+    R3PTRTYPE(PX86PD)               pInterPD;
+    /** Pointer to the intermediate page tables - Normal.
+     * There are two page tables, one for the identity mapping and one for
+     * the host context mapping (of the core code). */
+    R3PTRTYPE(PX86PT)               apInterPTs[2];
+    /** Pointer to the intermediate page tables - PAE. */
+    R3PTRTYPE(PX86PTPAE)            apInterPaePTs[2];
+    /** Pointer to the intermediate page directory - PAE. */
+    R3PTRTYPE(PX86PDPAE)            apInterPaePDs[4];
+    /** Pointer to the intermediate page directory - PAE. */
+    R3PTRTYPE(PX86PDPT)             pInterPaePDPT;
+    /** Pointer to the intermediate page-map level 4 - AMD64. */
+    R3PTRTYPE(PX86PML4)             pInterPaePML4;
+    /** Pointer to the intermediate page directory - AMD64. */
+    R3PTRTYPE(PX86PDPT)             pInterPaePDPT64;
+    /** The Physical Address (HC) of the intermediate Page Directory - Normal. */
+    RTHCPHYS                        HCPhysInterPD;
+    /** The Physical Address (HC) of the intermediate Page Directory Pointer Table - PAE. */
+    RTHCPHYS                        HCPhysInterPaePDPT;
+    /** The Physical Address (HC) of the intermediate Page Map Level 4 table - AMD64. */
+    RTHCPHYS                        HCPhysInterPaePML4;
+    /** @} */
+#endif
+
+#ifndef PGM_WITHOUT_MAPPINGS
+    /** Base address of the dynamic page mapping area.
+     * The array is MM_HYPER_DYNAMIC_SIZE bytes big.
+     *
+     * @todo The plan of keeping PGMRCDYNMAP private to PGMRZDynMap.cpp didn't
+     *       work out.  Some cleaning up of the initialization that would
+     *       remove this memory is yet to be done...
+     */
+    RCPTRTYPE(uint8_t *)            pbDynPageMapBaseGC;
+    /** The address of the raw-mode context mapping cache. */
+    RCPTRTYPE(PPGMRCDYNMAP)         pRCDynMap;
+    /** The address of the ring-0 mapping cache if we're making use of it. */
+    RTR0PTR                         pvR0DynMapUsed;
+#endif
+
+    /** Hack: Number of deprecated page mapping locks taken by the current lock
+     *  owner via pgmPhysGCPhys2CCPtrInternalDepr. */
+    uint32_t                        cDeprecatedPageLocks;
+    /** Alignment padding. */
+    uint32_t                        au32Alignment2[1];
+
+
+    /** PGM critical section.
+     * This protects the physical, ram ranges, and the page flag updating (some of
+     * it anyway).
+     */
+    PDMCRITSECT                     CritSectX;
+
+    /**
+     * Data associated with managing the ring-3 mappings of the allocation chunks.
+     */
+    struct
+    {
+        /** The chunk mapping TLB. */
+        PGMCHUNKR3MAPTLB            Tlb;
+        /** The chunk tree, ordered by chunk id. */
+#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE) || defined(VBOX_WITH_RAM_IN_KERNEL)
+        R3PTRTYPE(PAVLU32NODECORE)  pTree;
+#else
+        R3R0PTRTYPE(PAVLU32NODECORE) pTree;
+#endif
+#if HC_ARCH_BITS == 32
+        uint32_t                    u32Alignment0;
+#endif
+        /** The number of mapped chunks. */
+        uint32_t                    c;
+        /** @cfgm{/PGM/MaxRing3Chunks, uint32_t, host dependent}
+         * The maximum number of mapped chunks.  On 64-bit this is unlimited by default,
+         * on 32-bit it defaults to 1 or 3 GB depending on the host. */
+        uint32_t                    cMax;
+        /** The current time.  This is incremented whenever a chunk is inserted. */
+        uint32_t                    iNow;
+        /** Alignment padding. */
+        uint32_t                    au32Alignment1[3];
+    } ChunkR3Map;
+
+    /** The page mapping TLB for ring-3. */
+    PGMPAGER3MAPTLB                 PhysTlbR3;
+#ifdef VBOX_WITH_RAM_IN_KERNEL
+    /** The page mapping TLB for ring-0. */
+    PGMPAGER0MAPTLB                 PhysTlbR0;
+#else
+    /** The page mapping TLB for ring-0 (still using ring-3 mappings). */
+    PGMPAGER3MAPTLB                 PhysTlbR0;
+#endif
+
+    /** @name   The zero page.
+     * @{ */
+    /** The host physical address of the zero page. */
+    RTHCPHYS                        HCPhysZeroPg;
+    /** The ring-3 mapping of the zero page. */
+    RTR3PTR                         pvZeroPgR3;
+    /** The ring-0 mapping of the zero page. */
+    RTR0PTR                         pvZeroPgR0;
+    /** The GC mapping of the zero page. */
+    RTRCPTR                         pvZeroPgRC;
+    RTRCPTR                         RCPtrAlignment3;
+    /** @}*/
+
+    /** @name   The Invalid MMIO page.
+     * This page is filled with 0xfeedface.
+     * @{ */
+    /** The host physical address of the invalid MMIO page. */
+    RTHCPHYS                        HCPhysMmioPg;
+    /** The host pysical address of the invalid MMIO page plus all invalid
+     * physical address bits set.  This is used to trigger X86_TRAP_PF_RSVD.
+     * @remarks Check fLessThan52PhysicalAddressBits before use. */
+    RTHCPHYS                        HCPhysInvMmioPg;
+    /** The ring-3 mapping of the invalid MMIO page. */
+    RTR3PTR                         pvMmioPgR3;
+#if HC_ARCH_BITS == 32
+    RTR3PTR                         R3PtrAlignment4;
+#endif
+    /** @} */
+
+
+    /** The number of handy pages. */
+    uint32_t                        cHandyPages;
+
+    /** The number of large handy pages. */
+    uint32_t                        cLargeHandyPages;
+
+    /**
+     * Array of handy pages.
+     *
+     * This array is used in a two way communication between pgmPhysAllocPage
+     * and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as
+     * an intermediary.
+     *
+     * The size of this array is important, see pgmPhysEnsureHandyPage for details.
+     * (The current size of 32 pages, means 128 KB of handy memory.)
+     */
+    GMMPAGEDESC                     aHandyPages[PGM_HANDY_PAGES];
+
+    /**
+     * Array of large handy pages. (currently size 1)
+     *
+     * This array is used in a two way communication between pgmPhysAllocLargePage
+     * and GMMR0AllocateLargePage, with PGMR3PhysAllocateLargePage serving as
+     * an intermediary.
+     */
+    GMMPAGEDESC                     aLargeHandyPage[1];
+
+    /**
+     * Live save data.
+     */
+    struct
+    {
+        /** Per type statistics. */
+        struct
+        {
+            /** The number of ready pages. */
+            uint32_t                cReadyPages;
+            /** The number of dirty pages. */
+            uint32_t                cDirtyPages;
+            /** The number of ready zero pages. */
+            uint32_t                cZeroPages;
+            /** The number of write monitored pages. */
+            uint32_t                cMonitoredPages;
+        }                           Rom,
+                                    Mmio2,
+                                    Ram;
+        /** The number of ignored pages in the RAM ranges (i.e. MMIO, MMIO2 and ROM). */
+        uint32_t                    cIgnoredPages;
+        /** Indicates that a live save operation is active. */
+        bool                        fActive;
+        /** Padding. */
+        bool                        afReserved[2];
+        /** The next history index. */
+        uint8_t                     iDirtyPagesHistory;
+        /** History of the total amount of dirty pages. */
+        uint32_t                    acDirtyPagesHistory[64];
+        /** Short term dirty page average. */
+        uint32_t                    cDirtyPagesShort;
+        /** Long term dirty page average. */
+        uint32_t                    cDirtyPagesLong;
+        /** The number of saved pages.  This is used to get some kind of estimate of the
+         * link speed so we can decide when we're done.  It is reset after the first
+         * 7 passes so the speed estimate doesn't get inflated by the initial set of
+         * zero pages. */
+        uint64_t                    cSavedPages;
+        /** The nanosecond timestamp when cSavedPages was 0. */
+        uint64_t                    uSaveStartNS;
+        /** Pages per second (for statistics). */
+        uint32_t                    cPagesPerSecond;
+        uint32_t                    cAlignment;
+    } LiveSave;
+
+    /** @name   Error injection.
+     * @{ */
+    /** Inject handy page allocation errors pretending we're completely out of
+     * memory. */
+    bool volatile                   fErrInjHandyPages;
+    /** Padding. */
+    bool                            afReserved[3];
+    /** @} */
+
+    /** @name Release Statistics
+     * @{ */
+    uint32_t                        cAllPages;              /**< The total number of pages. (Should be Private + Shared + Zero + Pure MMIO.) */
+    uint32_t                        cPrivatePages;          /**< The number of private pages. */
+    uint32_t                        cSharedPages;           /**< The number of shared pages. */
+    uint32_t                        cReusedSharedPages;     /**< The number of reused shared pages. */
+    uint32_t                        cZeroPages;             /**< The number of zero backed pages. */
+    uint32_t                        cPureMmioPages;         /**< The number of pure MMIO pages. */
+    uint32_t                        cMonitoredPages;        /**< The number of write monitored pages. */
+    uint32_t                        cWrittenToPages;        /**< The number of previously write monitored pages. */
+    uint32_t                        cWriteLockedPages;      /**< The number of write locked pages. */
+    uint32_t                        cReadLockedPages;       /**< The number of read locked pages. */
+    uint32_t                        cBalloonedPages;        /**< The number of ballooned pages. */
+    uint32_t                        cMappedChunks;          /**< Number of times we mapped a chunk. */
+    uint32_t                        cUnmappedChunks;        /**< Number of times we unmapped a chunk. */
+    uint32_t                        cLargePages;            /**< The number of large pages. */
+    uint32_t                        cLargePagesDisabled;    /**< The number of disabled large pages. */
+/*    uint32_t                        aAlignment4[1]; */
+
+    /** The number of times we were forced to change the hypervisor region location. */
+    STAMCOUNTER                     cRelocations;
+
+    STAMCOUNTER                     StatLargePageReused;    /**< The number of large pages we've reused.*/
+    STAMCOUNTER                     StatLargePageRefused;   /**< The number of times we couldn't use a large page.*/
+    STAMCOUNTER                     StatLargePageRecheck;   /**< The number of times we rechecked a disabled large page.*/
+
+    STAMPROFILE                     StatShModCheck;         /**< Profiles shared module checks. */
+    /** @} */
+
+#ifdef VBOX_WITH_STATISTICS
+    /** @name Statistics on the heap.
+     * @{ */
+    R3PTRTYPE(PGMSTATS *)           pStatsR3;
+    R0PTRTYPE(PGMSTATS *)           pStatsR0;
+    /** @} */
+#endif
+} PGM;
+#ifndef IN_TSTVMSTRUCTGC /* HACK */
+# ifndef PGM_WITHOUT_MAPPINGS
+AssertCompileMemberAlignment(PGM, paDynPageMap32BitPTEsGC, 8);
+# endif
+AssertCompileMemberAlignment(PGM, GCPtrMappingFixed, sizeof(RTGCPTR));
+# ifndef PGM_WITHOUT_MAPPINGS
+AssertCompileMemberAlignment(PGM, HCPhysInterPD, 8);
+# endif
+AssertCompileMemberAlignment(PGM, CritSectX, 8);
+AssertCompileMemberAlignment(PGM, ChunkR3Map, 16);
+AssertCompileMemberAlignment(PGM, PhysTlbR3, 32); /** @todo 32 byte alignment! */
+AssertCompileMemberAlignment(PGM, PhysTlbR0, 32);
+AssertCompileMemberAlignment(PGM, HCPhysZeroPg, 8);
+AssertCompileMemberAlignment(PGM, aHandyPages, 8);
+AssertCompileMemberAlignment(PGM, cRelocations, 8);
+#endif /* !IN_TSTVMSTRUCTGC */
+/** Pointer to the PGM instance data. */
+typedef PGM *PPGM;
+
+
+
+typedef struct PGMCPUSTATS
+{
+    /* Common */
+    STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES];       /**< SyncPT - PD distribution. */
+    STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES];     /**< SyncPage - PD distribution. */
+
+    /* R0 only: */
+    STAMPROFILE StatR0NpMiscfg;                     /**< R0: PGMR0Trap0eHandlerNPMisconfig() profiling. */
+    STAMCOUNTER StatR0NpMiscfgSyncPage;             /**< R0: SyncPage calls from PGMR0Trap0eHandlerNPMisconfig(). */
+
+    /* RZ only: */
+    STAMPROFILE StatRZTrap0e;                       /**< RC/R0: PGMTrap0eHandler() profiling. */
+    STAMPROFILE StatRZTrap0eTime2Ballooned;         /**< RC/R0: Profiling of the Trap0eHandler body when the cause is read access to a ballooned page. */
+    STAMPROFILE StatRZTrap0eTime2CSAM;              /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. */
+    STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed;  /**< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. */
+    STAMPROFILE StatRZTrap0eTime2GuestTrap;         /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. */
+    STAMPROFILE StatRZTrap0eTime2HndPhys;           /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. */
+    STAMPROFILE StatRZTrap0eTime2HndUnhandled;      /**< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. */
+    STAMPROFILE StatRZTrap0eTime2InvalidPhys;       /**< RC/R0: Profiling of the Trap0eHandler body when the cause is access to an invalid physical guest address. */
+    STAMPROFILE StatRZTrap0eTime2MakeWritable;      /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a page that needed to be made writable. */
+    STAMPROFILE StatRZTrap0eTime2Mapping;           /**< RC/R0: Profiling of the Trap0eHandler body when the cause is the guest mappings. */
+    STAMPROFILE StatRZTrap0eTime2Misc;              /**< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. */
+    STAMPROFILE StatRZTrap0eTime2OutOfSync;         /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. */
+    STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys;  /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. */
+    STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs;   /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. */
+    STAMPROFILE StatRZTrap0eTime2SyncPT;            /**< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. */
+    STAMPROFILE StatRZTrap0eTime2WPEmulation;       /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP emulation. */
+    STAMPROFILE StatRZTrap0eTime2Wp0RoUsHack;       /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP and netware hack to be enabled. */
+    STAMPROFILE StatRZTrap0eTime2Wp0RoUsUnhack;     /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP and netware hack to be disabled. */
+    STAMCOUNTER StatRZTrap0eConflicts;              /**< RC/R0: The number of times \#PF was caused by an undetected conflict. */
+    STAMCOUNTER StatRZTrap0eHandlersMapping;        /**< RC/R0: Number of traps due to access handlers in mappings. */
+    STAMCOUNTER StatRZTrap0eHandlersOutOfSync;      /**< RC/R0: Number of out-of-sync handled pages. */
+    STAMCOUNTER StatRZTrap0eHandlersPhysAll;        /**< RC/R0: Number of traps due to physical all-access handlers. */
+    STAMCOUNTER StatRZTrap0eHandlersPhysAllOpt;     /**< RC/R0: Number of the physical all-access handler traps using the optimization. */
+    STAMCOUNTER StatRZTrap0eHandlersPhysWrite;      /**< RC/R0: Number of traps due to write-physical access handlers. */
+    STAMCOUNTER StatRZTrap0eHandlersUnhandled;      /**< RC/R0: Number of traps due to access outside range of monitored page(s). */
+    STAMCOUNTER StatRZTrap0eHandlersInvalid;        /**< RC/R0: Number of traps due to access to invalid physical memory. */
+    STAMCOUNTER StatRZTrap0eUSNotPresentRead;       /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eUSNotPresentWrite;      /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eUSWrite;                /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eUSReserved;             /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eUSNXE;                  /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eUSRead;                 /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eSVNotPresentRead;       /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eSVNotPresentWrite;      /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eSVWrite;                /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eSVReserved;             /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eSNXE;                   /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eGuestPF;                /**< RC/R0: Real guest \#PFs. */
+    STAMCOUNTER StatRZTrap0eGuestPFMapping;         /**< RC/R0: Real guest \#PF to HMA or other mapping. */
+    STAMCOUNTER StatRZTrap0eWPEmulInRZ;             /**< RC/R0: WP=0 virtualization trap, handled. */
+    STAMCOUNTER StatRZTrap0eWPEmulToR3;             /**< RC/R0: WP=0 virtualization trap, chickened out. */
+    STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES];     /**< RC/R0: PD distribution of the \#PFs. */
+    STAMCOUNTER StatRZGuestCR3WriteHandled;         /**< RC/R0: The number of times WriteHandlerCR3() was successfully called. */
+    STAMCOUNTER StatRZGuestCR3WriteUnhandled;       /**< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. */
+    STAMCOUNTER StatRZGuestCR3WriteConflict;        /**< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. */
+    STAMCOUNTER StatRZGuestROMWriteHandled;         /**< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. */
+    STAMCOUNTER StatRZGuestROMWriteUnhandled;       /**< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler */
+    STAMCOUNTER StatRZDynMapMigrateInvlPg;          /**< RZ: invlpg in PGMR0DynMapMigrateAutoSet. */
+    STAMPROFILE StatRZDynMapGCPageInl;              /**< RZ: Calls to pgmRZDynMapGCPageInlined. */
+    STAMCOUNTER StatRZDynMapGCPageInlHits;          /**< RZ: Hash table lookup hits. */
+    STAMCOUNTER StatRZDynMapGCPageInlMisses;        /**< RZ: Misses that falls back to the code common. */
+    STAMCOUNTER StatRZDynMapGCPageInlRamHits;       /**< RZ: 1st ram range hits. */
+    STAMCOUNTER StatRZDynMapGCPageInlRamMisses;     /**< RZ: 1st ram range misses, takes slow path. */
+    STAMPROFILE StatRZDynMapHCPageInl;              /**< RZ: Calls to pgmRZDynMapHCPageInlined. */
+    STAMCOUNTER StatRZDynMapHCPageInlHits;          /**< RZ: Hash table lookup hits. */
+    STAMCOUNTER StatRZDynMapHCPageInlMisses;        /**< RZ: Misses that falls back to the code common. */
+    STAMPROFILE StatRZDynMapHCPage;                 /**< RZ: Calls to pgmRZDynMapHCPageCommon. */
+    STAMCOUNTER StatRZDynMapSetOptimize;            /**< RZ: Calls to pgmRZDynMapOptimizeAutoSet. */
+    STAMCOUNTER StatRZDynMapSetSearchFlushes;       /**< RZ: Set search restoring to subset flushes. */
+    STAMCOUNTER StatRZDynMapSetSearchHits;          /**< RZ: Set search hits. */
+    STAMCOUNTER StatRZDynMapSetSearchMisses;        /**< RZ: Set search misses. */
+    STAMCOUNTER StatRZDynMapPage;                   /**< RZ: Calls to pgmR0DynMapPage. */
+    STAMCOUNTER StatRZDynMapPageHits0;              /**< RZ: Hits at iPage+0. */
+    STAMCOUNTER StatRZDynMapPageHits1;              /**< RZ: Hits at iPage+1. */
+    STAMCOUNTER StatRZDynMapPageHits2;              /**< RZ: Hits at iPage+2. */
+    STAMCOUNTER StatRZDynMapPageInvlPg;             /**< RZ: invlpg. */
+    STAMCOUNTER StatRZDynMapPageSlow;               /**< RZ: Calls to pgmR0DynMapPageSlow. */
+    STAMCOUNTER StatRZDynMapPageSlowLoopHits;       /**< RZ: Hits in the pgmR0DynMapPageSlow search loop. */
+    STAMCOUNTER StatRZDynMapPageSlowLoopMisses;     /**< RZ: Misses in the pgmR0DynMapPageSlow search loop. */
+    //STAMCOUNTER StatRZDynMapPageSlowLostHits;       /**< RZ: Lost hits. */
+    STAMCOUNTER StatRZDynMapSubsets;                /**< RZ: Times PGMDynMapPushAutoSubset was called. */
+    STAMCOUNTER StatRZDynMapPopFlushes;             /**< RZ: Times PGMDynMapPopAutoSubset flushes the subset. */
+    STAMCOUNTER aStatRZDynMapSetFilledPct[11];      /**< RZ: Set fill distribution, percent. */
+
+    /* HC - R3 and (maybe) R0: */
+
+    /* RZ & R3: */
+    STAMPROFILE StatRZSyncCR3;                      /**< RC/R0: PGMSyncCR3() profiling. */
+    STAMPROFILE StatRZSyncCR3Handlers;              /**< RC/R0: Profiling of the PGMSyncCR3() update handler section. */
+    STAMCOUNTER StatRZSyncCR3Global;                /**< RC/R0: The number of global CR3 syncs. */
+    STAMCOUNTER StatRZSyncCR3NotGlobal;             /**< RC/R0: The number of non-global CR3 syncs. */
+    STAMCOUNTER StatRZSyncCR3DstCacheHit;           /**< RC/R0: The number of times we got some kind of cache hit on a page table. */
+    STAMCOUNTER StatRZSyncCR3DstFreed;              /**< RC/R0: The number of times we've had to free a shadow entry. */
+    STAMCOUNTER StatRZSyncCR3DstFreedSrcNP;         /**< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. */
+    STAMCOUNTER StatRZSyncCR3DstNotPresent;         /**< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. */
+    STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD;    /**< RC/R0: The number of times a global page directory wasn't flushed. */
+    STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT;    /**< RC/R0: The number of times a page table with only global entries wasn't flushed. */
+    STAMPROFILE StatRZSyncPT;                       /**< RC/R0: PGMSyncPT() profiling. */
+    STAMCOUNTER StatRZSyncPTFailed;                 /**< RC/R0: The number of times PGMSyncPT() failed. */
+    STAMCOUNTER StatRZSyncPT4K;                     /**< RC/R0: Number of 4KB syncs. */
+    STAMCOUNTER StatRZSyncPT4M;                     /**< RC/R0: Number of 4MB syncs. */
+    STAMCOUNTER StatRZSyncPagePDNAs;                /**< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. */
+    STAMCOUNTER StatRZSyncPagePDOutOfSync;          /**< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. */
+    STAMCOUNTER StatRZAccessedPage;                 /**< RC/R0: The number of pages marked not present for accessed bit emulation. */
+    STAMPROFILE StatRZDirtyBitTracking;             /**< RC/R0: Profiling the dirty bit tracking in CheckPageFault(). */
+    STAMCOUNTER StatRZDirtyPage;                    /**< RC/R0: The number of pages marked read-only for dirty bit tracking. */
+    STAMCOUNTER StatRZDirtyPageBig;                 /**< RC/R0: The number of pages marked read-only for dirty bit tracking. */
+    STAMCOUNTER StatRZDirtyPageSkipped;             /**< RC/R0: The number of pages already dirty or readonly. */
+    STAMCOUNTER StatRZDirtyPageTrap;                /**< RC/R0: The number of traps generated for dirty bit tracking. */
+    STAMCOUNTER StatRZDirtyPageStale;               /**< RC/R0: The number of traps generated for dirty bit tracking. (stale tlb entries) */
+    STAMCOUNTER StatRZDirtyTrackRealPF;             /**< RC/R0: The number of real pages faults during dirty bit tracking. */
+    STAMCOUNTER StatRZDirtiedPage;                  /**< RC/R0: The number of pages marked dirty because of write accesses. */
+    STAMCOUNTER StatRZPageAlreadyDirty;             /**< RC/R0: The number of pages already marked dirty because of write accesses. */
+    STAMPROFILE StatRZInvalidatePage;               /**< RC/R0: PGMInvalidatePage() profiling. */
+    STAMCOUNTER StatRZInvalidatePage4KBPages;       /**< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. */
+    STAMCOUNTER StatRZInvalidatePage4MBPages;       /**< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. */
+    STAMCOUNTER StatRZInvalidatePage4MBPagesSkip;   /**< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. */
+    STAMCOUNTER StatRZInvalidatePagePDMappings;     /**< RC/R0: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). */
+    STAMCOUNTER StatRZInvalidatePagePDNAs;          /**< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. */
+    STAMCOUNTER StatRZInvalidatePagePDNPs;          /**< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. */
+    STAMCOUNTER StatRZInvalidatePagePDOutOfSync;    /**< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. */
+    STAMCOUNTER StatRZInvalidatePageSizeChanges ;   /**< RC/R0: The number of times PGMInvalidatePage() was called on a page size change (4KB <-> 2/4MB). */
+    STAMCOUNTER StatRZInvalidatePageSkipped;        /**< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. */
+    STAMCOUNTER StatRZPageOutOfSyncUser;            /**< RC/R0: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. */
+    STAMCOUNTER StatRZPageOutOfSyncSupervisor;      /**< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. */
+    STAMCOUNTER StatRZPageOutOfSyncUserWrite;       /**< RC/R0: The number of times user page is out of sync was detected in \#PF. */
+    STAMCOUNTER StatRZPageOutOfSyncSupervisorWrite; /**< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF. */
+    STAMCOUNTER StatRZPageOutOfSyncBallloon;        /**< RC/R0: The number of times a ballooned page was accessed (read). */
+    STAMPROFILE StatRZPrefetch;                     /**< RC/R0: PGMPrefetchPage. */
+    STAMPROFILE StatRZFlushTLB;                     /**< RC/R0: Profiling of the PGMFlushTLB() body. */
+    STAMCOUNTER StatRZFlushTLBNewCR3;               /**< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) */
+    STAMCOUNTER StatRZFlushTLBNewCR3Global;         /**< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) */
+    STAMCOUNTER StatRZFlushTLBSameCR3;              /**< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) */
+    STAMCOUNTER StatRZFlushTLBSameCR3Global;        /**< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) */
+    STAMPROFILE StatRZGstModifyPage;                /**< RC/R0: Profiling of the PGMGstModifyPage() body */
+
+    STAMPROFILE StatR3SyncCR3;                      /**< R3: PGMSyncCR3() profiling. */
+    STAMPROFILE StatR3SyncCR3Handlers;              /**< R3: Profiling of the PGMSyncCR3() update handler section. */
+    STAMCOUNTER StatR3SyncCR3Global;                /**< R3: The number of global CR3 syncs. */
+    STAMCOUNTER StatR3SyncCR3NotGlobal;             /**< R3: The number of non-global CR3 syncs. */
+    STAMCOUNTER StatR3SyncCR3DstFreed;              /**< R3: The number of times we've had to free a shadow entry. */
+    STAMCOUNTER StatR3SyncCR3DstFreedSrcNP;         /**< R3: The number of times we've had to free a shadow entry for which the source entry was not present. */
+    STAMCOUNTER StatR3SyncCR3DstNotPresent;         /**< R3: The number of times we've encountered a not present shadow entry for a present guest entry. */
+    STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD;    /**< R3: The number of times a global page directory wasn't flushed. */
+    STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT;    /**< R3: The number of times a page table with only global entries wasn't flushed. */
+    STAMCOUNTER StatR3SyncCR3DstCacheHit;           /**< R3: The number of times we got some kind of cache hit on a page table. */
+    STAMPROFILE StatR3SyncPT;                       /**< R3: PGMSyncPT() profiling. */
+    STAMCOUNTER StatR3SyncPTFailed;                 /**< R3: The number of times PGMSyncPT() failed. */
+    STAMCOUNTER StatR3SyncPT4K;                     /**< R3: Number of 4KB syncs. */
+    STAMCOUNTER StatR3SyncPT4M;                     /**< R3: Number of 4MB syncs. */
+    STAMCOUNTER StatR3SyncPagePDNAs;                /**< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. */
+    STAMCOUNTER StatR3SyncPagePDOutOfSync;          /**< R3: The number of time we've encountered an out-of-sync PD in SyncPage. */
+    STAMCOUNTER StatR3AccessedPage;                 /**< R3: The number of pages marked not present for accessed bit emulation. */
+    STAMPROFILE StatR3DirtyBitTracking;             /**< R3: Profiling the dirty bit tracking in CheckPageFault(). */
+    STAMCOUNTER StatR3DirtyPage;                    /**< R3: The number of pages marked read-only for dirty bit tracking. */
+    STAMCOUNTER StatR3DirtyPageBig;                 /**< R3: The number of pages marked read-only for dirty bit tracking. */
+    STAMCOUNTER StatR3DirtyPageSkipped;             /**< R3: The number of pages already dirty or readonly. */
+    STAMCOUNTER StatR3DirtyPageTrap;                /**< R3: The number of traps generated for dirty bit tracking. */
+    STAMCOUNTER StatR3DirtyTrackRealPF;             /**< R3: The number of real pages faults during dirty bit tracking. */
+    STAMCOUNTER StatR3DirtiedPage;                  /**< R3: The number of pages marked dirty because of write accesses. */
+    STAMCOUNTER StatR3PageAlreadyDirty;             /**< R3: The number of pages already marked dirty because of write accesses. */
+    STAMPROFILE StatR3InvalidatePage;               /**< R3: PGMInvalidatePage() profiling. */
+    STAMCOUNTER StatR3InvalidatePage4KBPages;       /**< R3: The number of times PGMInvalidatePage() was called for a 4KB page. */
+    STAMCOUNTER StatR3InvalidatePage4MBPages;       /**< R3: The number of times PGMInvalidatePage() was called for a 4MB page. */
+    STAMCOUNTER StatR3InvalidatePage4MBPagesSkip;   /**< R3: The number of times PGMInvalidatePage() skipped a 4MB page. */
+    STAMCOUNTER StatR3InvalidatePagePDNAs;          /**< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. */
+    STAMCOUNTER StatR3InvalidatePagePDNPs;          /**< R3: The number of times PGMInvalidatePage() was called for a not present page directory. */
+    STAMCOUNTER StatR3InvalidatePagePDMappings;     /**< R3: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). */
+    STAMCOUNTER StatR3InvalidatePagePDOutOfSync;    /**< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. */
+    STAMCOUNTER StatR3InvalidatePageSizeChanges ;   /**< R3: The number of times PGMInvalidatePage() was called on a page size change (4KB <-> 2/4MB). */
+    STAMCOUNTER StatR3InvalidatePageSkipped;        /**< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. */
+    STAMCOUNTER StatR3PageOutOfSyncUser;            /**< R3: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. */
+    STAMCOUNTER StatR3PageOutOfSyncSupervisor;      /**< R3: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. */
+    STAMCOUNTER StatR3PageOutOfSyncUserWrite;       /**< R3: The number of times user page is out of sync was detected in \#PF. */
+    STAMCOUNTER StatR3PageOutOfSyncSupervisorWrite; /**< R3: The number of times supervisor page is out of sync was detected in in \#PF. */
+    STAMCOUNTER StatR3PageOutOfSyncBallloon;        /**< R3: The number of times a ballooned page was accessed (read). */
+    STAMPROFILE StatR3Prefetch;                     /**< R3: PGMPrefetchPage. */
+    STAMPROFILE StatR3FlushTLB;                     /**< R3: Profiling of the PGMFlushTLB() body. */
+    STAMCOUNTER StatR3FlushTLBNewCR3;               /**< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) */
+    STAMCOUNTER StatR3FlushTLBNewCR3Global;         /**< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) */
+    STAMCOUNTER StatR3FlushTLBSameCR3;              /**< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) */
+    STAMCOUNTER StatR3FlushTLBSameCR3Global;        /**< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) */
+    STAMPROFILE StatR3GstModifyPage;                /**< R3: Profiling of the PGMGstModifyPage() body */
+} PGMCPUSTATS;
+
+
+/**
+ * Converts a PGMCPU pointer into a VM pointer.
+ * @returns Pointer to the VM structure the PGM is part of.
+ * @param   pPGM    Pointer to PGMCPU instance data.
+ */
+#define PGMCPU2VM(pPGM)         ( (PVM)((char*)(pPGM) - (pPGM)->offVM) )
+
+/**
+ * Converts a PGMCPU pointer into a PGM pointer.
+ * @returns Pointer to the VM structure the PGM is part of.
+ * @param   pPGMCpu Pointer to PGMCPU instance data.
+ */
+#define PGMCPU2PGM(pPGMCpu)     ( (PPGM)((char *)(pPGMCpu) - (pPGMCpu)->offPGM) )
+
+/**
+ * PGMCPU Data (part of VMCPU).
+ */
+typedef struct PGMCPU
+{
+    /** Offset to the VM structure. */
+    int32_t                         offVM;
+    /** Offset to the VMCPU structure. */
+    int32_t                         offVCpu;
+    /** Offset of the PGM structure relative to VMCPU. */
+    int32_t                         offPGM;
+    uint32_t                        uPadding0;      /**< structure size alignment. */
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    /** Automatically tracked physical memory mapping set.
+     * Ring-0 and strict raw-mode builds. */
+    PGMMAPSET                       AutoSet;
+#endif
+
+    /** A20 gate mask.
+     * Our current approach to A20 emulation is to let REM do it and don't bother
+     * anywhere else. The interesting Guests will be operating with it enabled anyway.
+     * But whould need arrise, we'll subject physical addresses to this mask. */
+    RTGCPHYS                        GCPhysA20Mask;
+    /** A20 gate state - boolean! */
+    bool                            fA20Enabled;
+    /** Mirror of the EFER.NXE bit.  Managed by PGMNotifyNxeChanged. */
+    bool                            fNoExecuteEnabled;
+    /** Unused bits. */
+    bool                            afUnused[2];
+
+    /** What needs syncing (PGM_SYNC_*).
+     * This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
+     * PGMFlushTLB, and PGMR3Load. */
+    uint32_t                        fSyncFlags;
+
+    /** The shadow paging mode. */
+    PGMMODE                         enmShadowMode;
+    /** The guest paging mode. */
+    PGMMODE                         enmGuestMode;
+    /** Guest mode data table index (PGM_TYPE_XXX). */
+    uint8_t volatile                idxGuestModeData;
+    /** Shadow mode data table index (PGM_TYPE_XXX). */
+    uint8_t volatile                idxShadowModeData;
+    /** Both mode data table index (complicated). */
+    uint8_t volatile                idxBothModeData;
+    /** Alignment padding. */
+    uint8_t                         abPadding[5];
+
+    /** The current physical address represented in the guest CR3 register. */
+    RTGCPHYS                        GCPhysCR3;
+
+    /** @name 32-bit Guest Paging.
+     * @{ */
+    /** The guest's page directory, R3 pointer. */
+    R3PTRTYPE(PX86PD)               pGst32BitPdR3;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    /** The guest's page directory, R0 pointer. */
+    R0PTRTYPE(PX86PD)               pGst32BitPdR0;
+#endif
+    /** Mask containing the MBZ bits of a big page PDE. */
+    uint32_t                        fGst32BitMbzBigPdeMask;
+    /** Set if the page size extension (PSE) is enabled. */
+    bool                            fGst32BitPageSizeExtension;
+    /** Alignment padding. */
+    bool                            afAlignment2[3];
+    /** @} */
+
+    /** @name PAE Guest Paging.
+     * @{ */
+    /** The guest's page directory pointer table, R3 pointer. */
+    R3PTRTYPE(PX86PDPT)             pGstPaePdptR3;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    /** The guest's page directory pointer table, R0 pointer. */
+    R0PTRTYPE(PX86PDPT)             pGstPaePdptR0;
+#endif
+
+    /** The guest's page directories, R3 pointers.
+     * These are individual pointers and don't have to be adjacent.
+     * These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
+    R3PTRTYPE(PX86PDPAE)            apGstPaePDsR3[4];
+    /** The guest's page directories, R0 pointers.
+     * Same restrictions as apGstPaePDsR3. */
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R0PTRTYPE(PX86PDPAE)            apGstPaePDsR0[4];
+#endif
+    /** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC.
+     * @todo Remove this and use aGstPaePdpeRegs instead? */
+    RTGCPHYS                        aGCPhysGstPaePDs[4];
+    /** The values of the 4 PDPE CPU registers (PAE). */
+    X86PDPE                         aGstPaePdpeRegs[4];
+    /** The physical addresses of the monitored guest page directories (PAE). */
+    RTGCPHYS                        aGCPhysGstPaePDsMonitored[4];
+    /** Mask containing the MBZ PTE bits. */
+    uint64_t                        fGstPaeMbzPteMask;
+    /** Mask containing the MBZ PDE bits. */
+    uint64_t                        fGstPaeMbzPdeMask;
+    /** Mask containing the MBZ big page PDE bits. */
+    uint64_t                        fGstPaeMbzBigPdeMask;
+    /** Mask containing the MBZ PDPE bits. */
+    uint64_t                        fGstPaeMbzPdpeMask;
+    /** @} */
+
+    /** @name AMD64 Guest Paging.
+     * @{ */
+    /** The guest's page directory pointer table, R3 pointer. */
+    R3PTRTYPE(PX86PML4)             pGstAmd64Pml4R3;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    /** The guest's page directory pointer table, R0 pointer. */
+    R0PTRTYPE(PX86PML4)             pGstAmd64Pml4R0;
+#else
+    RTR0PTR                         alignment6b; /**< alignment equalizer. */
+#endif
+    /** Mask containing the MBZ PTE bits. */
+    uint64_t                        fGstAmd64MbzPteMask;
+    /** Mask containing the MBZ PDE bits. */
+    uint64_t                        fGstAmd64MbzPdeMask;
+    /** Mask containing the MBZ big page PDE bits. */
+    uint64_t                        fGstAmd64MbzBigPdeMask;
+    /** Mask containing the MBZ PDPE bits. */
+    uint64_t                        fGstAmd64MbzPdpeMask;
+    /** Mask containing the MBZ big page PDPE bits. */
+    uint64_t                        fGstAmd64MbzBigPdpeMask;
+    /** Mask containing the MBZ PML4E bits. */
+    uint64_t                        fGstAmd64MbzPml4eMask;
+    /** Mask containing the PDPE bits that we shadow. */
+    uint64_t                        fGstAmd64ShadowedPdpeMask;
+    /** Mask containing the PML4E bits that we shadow. */
+    uint64_t                        fGstAmd64ShadowedPml4eMask;
+    /** @} */
+
+    /** @name PAE and AMD64 Guest Paging.
+     * @{ */
+    /** Mask containing the PTE bits that we shadow. */
+    uint64_t                        fGst64ShadowedPteMask;
+    /** Mask containing the PDE bits that we shadow. */
+    uint64_t                        fGst64ShadowedPdeMask;
+    /** Mask containing the big page PDE bits that we shadow in the PDE. */
+    uint64_t                        fGst64ShadowedBigPdeMask;
+    /** Mask containing the big page PDE bits that we shadow in the PTE. */
+    uint64_t                        fGst64ShadowedBigPde4PteMask;
+    /** @} */
+
+    /** Pointer to the page of the current active CR3 - R3 Ptr. */
+    R3PTRTYPE(PPGMPOOLPAGE)         pShwPageCR3R3;
+    /** Pointer to the page of the current active CR3 - R0 Ptr. */
+    R0PTRTYPE(PPGMPOOLPAGE)         pShwPageCR3R0;
+
+    /** For saving stack space, the disassembler state is allocated here instead of
+     * on the stack. */
+    DISCPUSTATE                     DisState;
+
+    /** Counts the number of times the netware WP0+RO+US hack has been applied. */
+    uint64_t                        cNetwareWp0Hacks;
+
+    /** Count the number of pgm pool access handler calls. */
+    uint64_t                        cPoolAccessHandler;
+
+    /** @name Release Statistics
+     * @{ */
+    /** The number of times the guest has switched mode since last reset or statistics reset. */
+    STAMCOUNTER                     cGuestModeChanges;
+    /** The number of times the guest has switched mode since last reset or statistics reset. */
+    STAMCOUNTER                     cA20Changes;
+    /** @} */
+
+#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
+    /** @name Statistics
+     * @{ */
+    /** R0: Pointer to the statistics. */
+    R0PTRTYPE(PGMCPUSTATS *)        pStatsR0;
+    /** R0: Which statistic this \#PF should be attributed to. */
+    R0PTRTYPE(PSTAMPROFILE)         pStatTrap0eAttributionR0;
+    /** R3: Pointer to the statistics. */
+    R3PTRTYPE(PGMCPUSTATS *)        pStatsR3;
+    /** Alignment padding. */
+    RTR3PTR                         pPaddingR3;
+    /** @} */
+#endif /* VBOX_WITH_STATISTICS */
+} PGMCPU;
+/** Pointer to the per-cpu PGM data. */
+typedef PGMCPU *PPGMCPU;
+
+
+/** @name PGM::fSyncFlags Flags
+ * @note Was part of saved state a long time ago.
+ * @{
+ */
+/* 0 used to be PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL */
+/** Always sync CR3. */
+#define PGM_SYNC_ALWAYS                         RT_BIT(1)
+/** Check monitoring on next CR3 (re)load and invalidate page.
+ * @todo This is obsolete now. Remove after 2.2.0 is branched off. */
+#define PGM_SYNC_MONITOR_CR3                    RT_BIT(2)
+/** Check guest mapping in SyncCR3. */
+#define PGM_SYNC_MAP_CR3                        RT_BIT(3)
+/** Clear the page pool (a light weight flush). */
+#define PGM_SYNC_CLEAR_PGM_POOL_BIT             8
+#define PGM_SYNC_CLEAR_PGM_POOL                 RT_BIT(PGM_SYNC_CLEAR_PGM_POOL_BIT)
+/** @} */
+
+
+/**
+ * PGM GVM instance data.
+ */
+typedef struct PGMR0PERVM
+{
+    /** @name PGM Pool related stuff.
+     * @{ */
+    /** Critical section for serializing pool growth. */
+    RTCRITSECT  PoolGrowCritSect;
+    /** The memory objects for the pool pages. */
+    RTR0MEMOBJ  ahPoolMemObjs[(PGMPOOL_IDX_LAST + PGMPOOL_CFG_MAX_GROW - 1) / PGMPOOL_CFG_MAX_GROW];
+    /** The ring-3 mapping objects for the pool pages. */
+    RTR0MEMOBJ  ahPoolMapObjs[(PGMPOOL_IDX_LAST + PGMPOOL_CFG_MAX_GROW - 1) / PGMPOOL_CFG_MAX_GROW];
+    /** @} */
+} PGMR0PERVM;
+
+RT_C_DECLS_BEGIN
+
+#if defined(VBOX_STRICT) && defined(IN_RING3)
+int             pgmLockDebug(PVMCC pVM, RT_SRC_POS_DECL);
+# define pgmLock(a_pVM) pgmLockDebug(a_pVM, RT_SRC_POS)
+#else
+int             pgmLock(PVMCC pVM);
+#endif
+void            pgmUnlock(PVM pVM);
+/**
+ * Asserts that the caller owns the PDM lock.
+ * This is the internal variant of PGMIsLockOwner.
+ * @param   a_pVM           Pointer to the VM.
+ */
+#define PGM_LOCK_ASSERT_OWNER(a_pVM)    Assert(PDMCritSectIsOwner(&(a_pVM)->pgm.s.CritSectX))
+/**
+ * Asserts that the caller owns the PDM lock.
+ * This is the internal variant of PGMIsLockOwner.
+ * @param   a_pVM           Pointer to the VM.
+ * @param   a_pVCpu         The current CPU handle.
+ */
+#define PGM_LOCK_ASSERT_OWNER_EX(a_pVM, a_pVCpu)  Assert(PDMCritSectIsOwnerEx(&(a_pVM)->pgm.s.CritSectX, a_pVCpu))
+
+#ifndef PGM_WITHOUT_MAPPINGS
+int             pgmR3MappingsFixInternal(PVM pVM, RTGCPTR GCPtrBase, uint32_t cb);
+int             pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping);
+int             pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping);
+int             pgmMapResolveConflicts(PVM pVM);
+PPGMMAPPING     pgmGetMapping(PVM pVM, RTGCPTR GCPtr);
+DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+#endif /* !PGM_WITHOUT_MAPPINGS */
+
+int             pgmHandlerPhysicalExCreate(PVMCC pVM, PGMPHYSHANDLERTYPE hType, RTR3PTR pvUserR3, RTR0PTR pvUserR0,
+                                           RTRCPTR pvUserRC, R3PTRTYPE(const char *) pszDesc, PPGMPHYSHANDLER *ppPhysHandler);
+int             pgmHandlerPhysicalExDup(PVMCC pVM, PPGMPHYSHANDLER pPhysHandlerSrc, PPGMPHYSHANDLER *ppPhysHandler);
+int             pgmHandlerPhysicalExRegister(PVMCC pVM, PPGMPHYSHANDLER pPhysHandler, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast);
+int             pgmHandlerPhysicalExDeregister(PVMCC pVM, PPGMPHYSHANDLER pPhysHandler, int fRestoreAsRAM);
+int             pgmHandlerPhysicalExDestroy(PVMCC pVM, PPGMPHYSHANDLER pHandler);
+void            pgmR3HandlerPhysicalUpdateAll(PVM pVM);
+bool            pgmHandlerPhysicalIsAll(PVMCC pVM, RTGCPHYS GCPhys);
+void            pgmHandlerPhysicalResetAliasedPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage, bool fDoAccounting);
+DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+int             pgmR3InitSavedState(PVM pVM, uint64_t cbRam);
+
+int             pgmPhysAllocPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
+int             pgmPhysAllocLargePage(PVMCC pVM, RTGCPHYS GCPhys);
+int             pgmPhysRecheckLargePage(PVMCC pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage);
+int             pgmPhysPageLoadIntoTlb(PVMCC pVM, RTGCPHYS GCPhys);
+int             pgmPhysPageLoadIntoTlbWithPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
+void            pgmPhysPageMakeWriteMonitoredWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
+int             pgmPhysPageMakeWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
+int             pgmPhysPageMakeWritableAndMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
+int             pgmPhysPageMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
+int             pgmPhysPageMapReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv);
+int             pgmPhysPageMapByPageID(PVMCC pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv);
+int             pgmPhysGCPhys2R3Ptr(PVMCC pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr);
+int             pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr);
+int             pgmPhysGCPhys2CCPtrInternalDepr(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
+int             pgmPhysGCPhys2CCPtrInternal(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock);
+int             pgmPhysGCPhys2CCPtrInternalReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock);
+void            pgmPhysReleaseInternalPageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock);
+PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) pgmPhysRomWriteHandler;
+#ifndef IN_RING3
+DECLEXPORT(FNPGMPHYSHANDLER)        pgmPhysHandlerRedirectToHC;
+DECLEXPORT(FNPGMRZPHYSPFHANDLER)    pgmPhysPfHandlerRedirectToHC;
+DECLEXPORT(FNPGMRZPHYSPFHANDLER)    pgmPhysRomWritePfHandler;
+#endif
+int             pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys,
+                                PGMPAGETYPE enmNewType);
+void            pgmPhysInvalidRamRangeTlbs(PVMCC pVM);
+void            pgmPhysInvalidatePageMapTLB(PVMCC pVM);
+void            pgmPhysInvalidatePageMapTLBEntry(PVM pVM, RTGCPHYS GCPhys);
+PPGMRAMRANGE    pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys);
+PPGMRAMRANGE    pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys);
+PPGMPAGE        pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys);
+int             pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage);
+int             pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam);
+
+#ifdef IN_RING3
+void            pgmR3PhysRelinkRamRanges(PVM pVM);
+int             pgmR3PhysRamPreAllocate(PVM pVM);
+int             pgmR3PhysRamReset(PVM pVM);
+int             pgmR3PhysRomReset(PVM pVM);
+int             pgmR3PhysRamZeroAll(PVM pVM);
+int             pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk);
+int             pgmR3PhysRamTerm(PVM pVM);
+void            pgmR3PhysRomTerm(PVM pVM);
+void            pgmR3PhysAssertSharedPageChecksums(PVM pVM);
+
+int             pgmR3PoolInit(PVM pVM);
+void            pgmR3PoolRelocate(PVM pVM);
+void            pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu);
+void            pgmR3PoolReset(PVM pVM);
+void            pgmR3PoolClearAll(PVM pVM, bool fFlushRemTlb);
+DECLCALLBACK(VBOXSTRICTRC) pgmR3PoolClearAllRendezvous(PVM pVM, PVMCPU pVCpu, void *fpvFlushRemTbl);
+void            pgmR3PoolWriteProtectPages(PVM pVM);
+
+#endif /* IN_RING3 */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+int             pgmRZDynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL);
+int             pgmRZDynMapGCPageCommon(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL);
+# ifdef LOG_ENABLED
+void            pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint, RT_SRC_POS_DECL);
+# else
+void            pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint);
+# endif
+#endif
+int             pgmPoolAlloc(PVMCC pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
+                             uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage);
+void            pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable);
+void            pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
+int             pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush = true /* DO NOT USE false UNLESS YOU KNOWN WHAT YOU'RE DOING!! */);
+void            pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys);
+PPGMPOOLPAGE    pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys);
+PPGMPOOLPAGE    pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys);
+int             pgmPoolHCPhys2Ptr(PVM pVM, RTHCPHYS HCPhys, void **ppv);
+int             pgmPoolSyncCR3(PVMCPUCC pVCpu);
+bool            pgmPoolIsDirtyPageSlow(PVM pVM, RTGCPHYS GCPhys);
+void            pgmPoolInvalidateDirtyPage(PVMCC pVM, RTGCPHYS GCPhysPT);
+int             pgmPoolTrackUpdateGCPhys(PVMCC pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs);
+void            pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte);
+uint16_t        pgmPoolTrackPhysExtAddref(PVMCC pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte);
+void            pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage, uint16_t iPte);
+void            pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
+void            pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
+PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) pgmPoolAccessHandler;
+#ifndef IN_RING3
+DECLEXPORT(FNPGMRZPHYSPFHANDLER)    pgmRZPoolAccessPfHandler;
+#endif
+
+void            pgmPoolAddDirtyPage(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage);
+void            pgmPoolResetDirtyPages(PVMCC pVM);
+void            pgmPoolResetDirtyPage(PVM pVM, RTGCPTR GCPtrPage);
+
+int             pgmR3ExitShadowModeBeforePoolFlush(PVMCPU pVCpu);
+int             pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu);
+void            pgmR3RefreshShadowModeAfterA20Change(PVMCPU pVCpu);
+
+#ifndef PGM_WITHOUT_MAPPINGS
+void            pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
+void            pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3);
+int             pgmMapActivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
+int             pgmMapDeactivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
+#endif
+
+int             pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags);
+int             pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
+int             pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, uint32_t cPages, PGMMODE enmShwPagingMode);
+
+int             pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd);
+int             pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt);
+int             pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd);
+int             pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4);
+int             pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk);
+int             pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk);
+
+# if defined(VBOX_STRICT) && HC_ARCH_BITS == 64 && defined(IN_RING3)
+FNDBGCCMD       pgmR3CmdCheckDuplicatePages;
+FNDBGCCMD       pgmR3CmdShowSharedModules;
+# endif
+
+void            pgmLogState(PVM pVM);
+
+RT_C_DECLS_END
+
+/** @} */
+
+#endif /* !VMM_INCLUDED_SRC_include_PGMInternal_h */
+
diff --git a/src/VBox/VMM/include/SELMInternal.h b/src/VBox/VMM/include/SELMInternal.h
new file mode 100644
index 00000000..f74a5019
--- /dev/null
+++ b/src/VBox/VMM/include/SELMInternal.h
@@ -0,0 +1,62 @@
+/* $Id: SELMInternal.h $ */
+/** @file
+ * SELM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_SELMInternal_h
+#define VMM_INCLUDED_SRC_include_SELMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/log.h>
+#include <iprt/x86.h>
+
+
+
+/** @defgroup grp_selm_int   Internals
+ * @ingroup grp_selm
+ * @internal
+ * @{
+ */
+
+/** The number of GDTS allocated for our GDT. (full size) */
+#define SELM_GDT_ELEMENTS                   8192
+
+
+/**
+ * SELM Data (part of VM)
+ *
+ * @note This is a very marginal component after kicking raw-mode.
+ */
+typedef struct SELM
+{
+#ifdef VBOX_WITH_STATISTICS
+    STAMCOUNTER             StatLoadHidSelGst;
+    STAMCOUNTER             StatLoadHidSelShw;
+#endif
+    STAMCOUNTER             StatLoadHidSelReadErrors;
+    STAMCOUNTER             StatLoadHidSelGstNoGood;
+} SELM, *PSELM;
+
+
+/** @} */
+
+#endif /* !VMM_INCLUDED_SRC_include_SELMInternal_h */
diff --git a/src/VBox/VMM/include/SSMInternal.h b/src/VBox/VMM/include/SSMInternal.h
new file mode 100644
index 00000000..d27bf340
--- /dev/null
+++ b/src/VBox/VMM/include/SSMInternal.h
@@ -0,0 +1,331 @@
+/* $Id: SSMInternal.h $ */
+/** @file
+ * SSM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_SSMInternal_h
+#define VMM_INCLUDED_SRC_include_SSMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/vmm/ssm.h>
+#include <iprt/critsect.h>
+
+RT_C_DECLS_BEGIN
+
+/** @defgroup grp_ssm_int       Internals
+ * @ingroup grp_ssm
+ * @internal
+ * @{
+ */
+
+
+/**
+ * Data unit callback type.
+ */
+typedef enum SSMUNITTYPE
+{
+    /** PDM Device . */
+    SSMUNITTYPE_DEV = 1,
+    /** PDM Driver. */
+    SSMUNITTYPE_DRV,
+    /** PDM USB device. */
+    SSMUNITTYPE_USB,
+    /** VM Internal. */
+    SSMUNITTYPE_INTERNAL,
+    /** External Wrapper. */
+    SSMUNITTYPE_EXTERNAL
+} SSMUNITTYPE;
+
+/** Pointer to a data unit descriptor. */
+typedef struct SSMUNIT *PSSMUNIT;
+
+/**
+ * Data unit descriptor.
+ */
+typedef struct SSMUNIT
+{
+    /** Pointer ot the next one in the list. */
+    PSSMUNIT                pNext;
+
+    /** Called in this save/load operation.
+     * The flag is used to determine whether there is need for a call to
+     * done or not. */
+    bool                    fCalled;
+    /** Finished its live part.
+     * This is used to handle VERR_SSM_VOTE_FOR_GIVING_UP.  */
+    bool                    fDoneLive;
+    /** Callback interface type. */
+    SSMUNITTYPE             enmType;
+    /** Type specific data. */
+    union
+    {
+        /** SSMUNITTYPE_DEV. */
+        struct
+        {
+            /** Prepare live save. */
+            PFNSSMDEVLIVEPREP   pfnLivePrep;
+            /** Execute live save. */
+            PFNSSMDEVLIVEEXEC   pfnLiveExec;
+            /** Vote live save complete. */
+            PFNSSMDEVLIVEVOTE   pfnLiveVote;
+            /** Prepare save. */
+            PFNSSMDEVSAVEPREP   pfnSavePrep;
+            /** Execute save. */
+            PFNSSMDEVSAVEEXEC   pfnSaveExec;
+            /** Done save. */
+            PFNSSMDEVSAVEDONE   pfnSaveDone;
+            /** Prepare load. */
+            PFNSSMDEVLOADPREP   pfnLoadPrep;
+            /** Execute load. */
+            PFNSSMDEVLOADEXEC   pfnLoadExec;
+            /** Done load. */
+            PFNSSMDEVLOADDONE   pfnLoadDone;
+            /** Device instance. */
+            PPDMDEVINS          pDevIns;
+        } Dev;
+
+        /** SSMUNITTYPE_DRV. */
+        struct
+        {
+            /** Prepare live save. */
+            PFNSSMDRVLIVEPREP   pfnLivePrep;
+            /** Execute live save. */
+            PFNSSMDRVLIVEEXEC   pfnLiveExec;
+            /** Vote live save complete. */
+            PFNSSMDRVLIVEVOTE   pfnLiveVote;
+            /** Prepare save. */
+            PFNSSMDRVSAVEPREP   pfnSavePrep;
+            /** Execute save. */
+            PFNSSMDRVSAVEEXEC   pfnSaveExec;
+            /** Done save. */
+            PFNSSMDRVSAVEDONE   pfnSaveDone;
+            /** Prepare load. */
+            PFNSSMDRVLOADPREP   pfnLoadPrep;
+            /** Execute load. */
+            PFNSSMDRVLOADEXEC   pfnLoadExec;
+            /** Done load. */
+            PFNSSMDRVLOADDONE   pfnLoadDone;
+            /** Driver instance. */
+            PPDMDRVINS          pDrvIns;
+        } Drv;
+
+        /** SSMUNITTYPE_USB. */
+        struct
+        {
+            /** Prepare live save. */
+            PFNSSMUSBLIVEPREP   pfnLivePrep;
+            /** Execute live save. */
+            PFNSSMUSBLIVEEXEC   pfnLiveExec;
+            /** Vote live save complete. */
+            PFNSSMUSBLIVEVOTE   pfnLiveVote;
+            /** Prepare save. */
+            PFNSSMUSBSAVEPREP   pfnSavePrep;
+            /** Execute save. */
+            PFNSSMUSBSAVEEXEC   pfnSaveExec;
+            /** Done save. */
+            PFNSSMUSBSAVEDONE   pfnSaveDone;
+            /** Prepare load. */
+            PFNSSMUSBLOADPREP   pfnLoadPrep;
+            /** Execute load. */
+            PFNSSMUSBLOADEXEC   pfnLoadExec;
+            /** Done load. */
+            PFNSSMUSBLOADDONE   pfnLoadDone;
+            /** USB instance. */
+            PPDMUSBINS          pUsbIns;
+        } Usb;
+
+        /** SSMUNITTYPE_INTERNAL. */
+        struct
+        {
+            /** Prepare live save. */
+            PFNSSMINTLIVEPREP   pfnLivePrep;
+            /** Execute live save. */
+            PFNSSMINTLIVEEXEC   pfnLiveExec;
+            /** Vote live save complete. */
+            PFNSSMINTLIVEVOTE   pfnLiveVote;
+            /** Prepare save. */
+            PFNSSMINTSAVEPREP   pfnSavePrep;
+            /** Execute save. */
+            PFNSSMINTSAVEEXEC   pfnSaveExec;
+            /** Done save. */
+            PFNSSMINTSAVEDONE   pfnSaveDone;
+            /** Prepare load. */
+            PFNSSMINTLOADPREP   pfnLoadPrep;
+            /** Execute load. */
+            PFNSSMINTLOADEXEC   pfnLoadExec;
+            /** Done load. */
+            PFNSSMINTLOADDONE   pfnLoadDone;
+        } Internal;
+
+        /** SSMUNITTYPE_EXTERNAL. */
+        struct
+        {
+            /** Prepare live save. */
+            PFNSSMEXTLIVEPREP   pfnLivePrep;
+            /** Execute live save. */
+            PFNSSMEXTLIVEEXEC   pfnLiveExec;
+            /** Vote live save complete. */
+            PFNSSMEXTLIVEVOTE   pfnLiveVote;
+            /** Prepare save. */
+            PFNSSMEXTSAVEPREP   pfnSavePrep;
+            /** Execute save. */
+            PFNSSMEXTSAVEEXEC   pfnSaveExec;
+            /** Done save. */
+            PFNSSMEXTSAVEDONE   pfnSaveDone;
+            /** Prepare load. */
+            PFNSSMEXTLOADPREP   pfnLoadPrep;
+            /** Execute load. */
+            PFNSSMEXTLOADEXEC   pfnLoadExec;
+            /** Done load. */
+            PFNSSMEXTLOADDONE   pfnLoadDone;
+            /** User data. */
+            void               *pvUser;
+        } External;
+
+        struct
+        {
+            /** Prepare live save. */
+            PFNRT               pfnLivePrep;
+            /** Execute live save. */
+            PFNRT               pfnLiveExec;
+            /** Vote live save complete. */
+            PFNRT               pfnLiveVote;
+            /** Prepare save. */
+            PFNRT               pfnSavePrep;
+            /** Execute save. */
+            PFNRT               pfnSaveExec;
+            /** Done save. */
+            PFNRT               pfnSaveDone;
+            /** Prepare load. */
+            PFNRT               pfnLoadPrep;
+            /** Execute load. */
+            PFNRT               pfnLoadExec;
+            /** Done load. */
+            PFNRT               pfnLoadDone;
+            /** User data. */
+            void               *pvKey;
+        } Common;
+    } u;
+    /** Data layout version. */
+    uint32_t                u32Version;
+    /** Instance number. */
+    uint32_t                u32Instance;
+    /** The offset of the final data unit.
+     * This is used for constructing the directory. */
+    RTFOFF                  offStream;
+    /** Critical section to be taken before working any of the callbacks. */
+    PPDMCRITSECT            pCritSect;
+    /** The guessed size of the data unit - used only for progress indication. */
+    size_t                  cbGuess;
+    /** Name size. (bytes) */
+    size_t                  cchName;
+    /** Name of this unit. (extends beyond the defined size) */
+    char                    szName[1];
+} SSMUNIT;
+
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.Dev.pfnLivePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.Dev.pfnLiveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.Dev.pfnLiveVote);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.Dev.pfnSavePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.Dev.pfnSaveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.Dev.pfnSaveDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.Dev.pfnLoadPrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.Dev.pfnLoadExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.Dev.pfnLoadDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pvKey,       u.Dev.pDevIns);
+
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.Drv.pfnLivePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.Drv.pfnLiveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.Drv.pfnLiveVote);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.Drv.pfnSavePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.Drv.pfnSaveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.Drv.pfnSaveDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.Drv.pfnLoadPrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.Drv.pfnLoadExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.Drv.pfnLoadDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pvKey,       u.Drv.pDrvIns);
+
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.Usb.pfnLivePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.Usb.pfnLiveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.Usb.pfnLiveVote);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.Usb.pfnSavePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.Usb.pfnSaveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.Usb.pfnSaveDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.Usb.pfnLoadPrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.Usb.pfnLoadExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.Usb.pfnLoadDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pvKey,       u.Usb.pUsbIns);
+
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.Internal.pfnLivePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.Internal.pfnLiveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.Internal.pfnLiveVote);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.Internal.pfnSavePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.Internal.pfnSaveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.Internal.pfnSaveDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.Internal.pfnLoadPrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.Internal.pfnLoadExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.Internal.pfnLoadDone);
+
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.External.pfnLivePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.External.pfnLiveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.External.pfnLiveVote);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.External.pfnSavePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.External.pfnSaveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.External.pfnSaveDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.External.pfnLoadPrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.External.pfnLoadExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.External.pfnLoadDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pvKey,       u.External.pvUser);
+
+
+/**
+ * SSM VM Instance data.
+ * Changes to this must checked against the padding of the cfgm union in VM!
+ *
+ * @todo Move this to UVM.
+ */
+typedef struct SSM
+{
+    /** Critical section for serializing cancellation (pSSM). */
+    RTCRITSECT              CancelCritSect;
+    /** The handle of the current save or load operation.
+     * This is used by SSMR3Cancel.  */
+    PSSMHANDLE volatile     pSSM;
+
+    /** FIFO of data entity descriptors. */
+    R3PTRTYPE(PSSMUNIT)     pHead;
+    /** The number of register units. */
+    uint32_t                cUnits;
+    /** For lazy init. */
+    bool                    fInitialized;
+    /** Current pass (for STAM). */
+    uint32_t                uPass;
+    uint32_t                u32Alignment;
+} SSM;
+/** Pointer to SSM VM instance data. */
+typedef SSM *PSSM;
+
+
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_SSMInternal_h */
+
diff --git a/src/VBox/VMM/include/STAMInternal.h b/src/VBox/VMM/include/STAMInternal.h
new file mode 100644
index 00000000..53d73a32
--- /dev/null
+++ b/src/VBox/VMM/include/STAMInternal.h
@@ -0,0 +1,177 @@
+/* $Id: STAMInternal.h $ */
+/** @file
+ * STAM Internal Header.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_STAMInternal_h
+#define VMM_INCLUDED_SRC_include_STAMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/gvmm.h>
+#include <VBox/vmm/gmm.h>
+#include <iprt/list.h>
+#include <iprt/semaphore.h>
+
+
+
+RT_C_DECLS_BEGIN
+
+/** @defgroup grp_stam_int   Internals
+ * @ingroup grp_stam
+ * @internal
+ * @{
+ */
+
+/** Pointer to sample descriptor. */
+typedef struct STAMDESC    *PSTAMDESC;
+/** Pointer to a sample lookup node. */
+typedef struct STAMLOOKUP  *PSTAMLOOKUP;
+
+/**
+ * Sample lookup node.
+ */
+typedef struct STAMLOOKUP
+{
+    /** The parent lookup record. This is NULL for the root node. */
+    PSTAMLOOKUP         pParent;
+    /** Array of children (using array for binary searching). */
+    PSTAMLOOKUP        *papChildren;
+    /** Pointer to the description node, if any. */
+    PSTAMDESC           pDesc;
+    /** Number of decentants with descriptors. (Use for freeing up sub-trees.) */
+    uint32_t            cDescsInTree;
+    /** The number of children. */
+    uint16_t            cChildren;
+    /** The index in the parent paChildren array. UINT16_MAX for the root node. */
+    uint16_t            iParent;
+    /** The path offset. */
+    uint16_t            off;
+    /** The size of the path component. */
+    uint16_t            cch;
+    /** The name (variable size). */
+    char                szName[1];
+} STAMLOOKUP;
+
+
+/**
+ * Sample descriptor.
+ */
+typedef struct STAMDESC
+{
+    /** Our entry in the big linear list. */
+    RTLISTNODE          ListEntry;
+    /** Pointer to our lookup node. */
+    PSTAMLOOKUP         pLookup;
+    /** Sample name. */
+    const char         *pszName;
+    /** Sample type. */
+    STAMTYPE            enmType;
+    /** Visibility type. */
+    STAMVISIBILITY      enmVisibility;
+    /** Pointer to the sample data. */
+    union STAMDESCSAMPLEDATA
+    {
+        /** Counter. */
+        PSTAMCOUNTER    pCounter;
+        /** Profile. */
+        PSTAMPROFILE    pProfile;
+        /** Advanced profile. */
+        PSTAMPROFILEADV pProfileAdv;
+        /** Ratio, unsigned 32-bit. */
+        PSTAMRATIOU32   pRatioU32;
+        /** unsigned 8-bit. */
+        uint8_t        *pu8;
+        /** unsigned 16-bit. */
+        uint16_t       *pu16;
+        /** unsigned 32-bit. */
+        uint32_t       *pu32;
+        /** unsigned 64-bit. */
+        uint64_t       *pu64;
+        /** Simple void pointer. */
+        void           *pv;
+        /** Boolean. */
+        bool           *pf;
+        /** */
+        struct STAMDESCSAMPLEDATACALLBACKS
+        {
+            /** The same pointer. */
+            void                   *pvSample;
+            /** Pointer to the reset callback. */
+            PFNSTAMR3CALLBACKRESET  pfnReset;
+            /** Pointer to the print callback. */
+            PFNSTAMR3CALLBACKPRINT  pfnPrint;
+        }               Callback;
+    }                   u;
+    /** Unit. */
+    STAMUNIT            enmUnit;
+    /** The refresh group number (STAM_REFRESH_GRP_XXX). */
+    uint8_t             iRefreshGroup;
+    /** Description. */
+    const char         *pszDesc;
+} STAMDESC;
+
+
+/**
+ * STAM data kept in the UVM.
+ */
+typedef struct STAMUSERPERVM
+{
+    /** List of samples. */
+    RTLISTANCHOR            List;
+    /** Root of the lookup tree. */
+    PSTAMLOOKUP             pRoot;
+
+    /** RW Lock for the list and tree. */
+    RTSEMRW                 RWSem;
+
+    /** The copy of the GVMM statistics. */
+    GVMMSTATS               GVMMStats;
+    /** The number of registered host CPU leaves. */
+    uint32_t                cRegisteredHostCpus;
+
+    /** Explicit alignment padding. */
+    uint32_t                uAlignment;
+    /** The copy of the GMM statistics. */
+    GMMSTATS                GMMStats;
+} STAMUSERPERVM;
+#ifdef IN_RING3
+AssertCompileMemberAlignment(STAMUSERPERVM, GMMStats, 8);
+#endif
+
+/** Pointer to the STAM data kept in the UVM. */
+typedef STAMUSERPERVM *PSTAMUSERPERVM;
+
+
+/** Locks the sample descriptors for reading. */
+#define STAM_LOCK_RD(pUVM)      do { int rcSem = RTSemRWRequestRead(pUVM->stam.s.RWSem, RT_INDEFINITE_WAIT);  AssertRC(rcSem); } while (0)
+/** Locks the sample descriptors for writing. */
+#define STAM_LOCK_WR(pUVM)      do { int rcSem = RTSemRWRequestWrite(pUVM->stam.s.RWSem, RT_INDEFINITE_WAIT); AssertRC(rcSem); } while (0)
+/** UnLocks the sample descriptors after reading. */
+#define STAM_UNLOCK_RD(pUVM)    do { int rcSem = RTSemRWReleaseRead(pUVM->stam.s.RWSem);  AssertRC(rcSem); } while (0)
+/** UnLocks the sample descriptors after writing. */
+#define STAM_UNLOCK_WR(pUVM)    do { int rcSem = RTSemRWReleaseWrite(pUVM->stam.s.RWSem); AssertRC(rcSem); } while (0)
+/** Lazy initialization */
+#define STAM_LAZY_INIT(pUVM)    do { } while (0)
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_STAMInternal_h */
diff --git a/src/VBox/VMM/include/TMInline.h b/src/VBox/VMM/include/TMInline.h
new file mode 100644
index 00000000..6d5951a8
--- /dev/null
+++ b/src/VBox/VMM/include/TMInline.h
@@ -0,0 +1,59 @@
+/* $Id: TMInline.h $ */
+/** @file
+ * TM - Common Inlined functions.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_TMInline_h
+#define VMM_INCLUDED_SRC_include_TMInline_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+/**
+ * Used to unlink a timer from the active list.
+ *
+ * @param   pQueue      The timer queue.
+ * @param   pTimer      The timer that needs linking.
+ *
+ * @remarks Called while owning the relevant queue lock.
+ */
+DECL_FORCE_INLINE(void) tmTimerQueueUnlinkActive(PTMTIMERQUEUE pQueue, PTMTIMER pTimer)
+{
+#ifdef VBOX_STRICT
+    TMTIMERSTATE const enmState = pTimer->enmState;
+    Assert(  pTimer->enmClock == TMCLOCK_VIRTUAL_SYNC
+           ? enmState == TMTIMERSTATE_ACTIVE
+           : enmState == TMTIMERSTATE_PENDING_SCHEDULE || enmState == TMTIMERSTATE_PENDING_STOP_SCHEDULE);
+#endif
+
+    const PTMTIMER pPrev = TMTIMER_GET_PREV(pTimer);
+    const PTMTIMER pNext = TMTIMER_GET_NEXT(pTimer);
+    if (pPrev)
+        TMTIMER_SET_NEXT(pPrev, pNext);
+    else
+    {
+        TMTIMER_SET_HEAD(pQueue, pNext);
+        pQueue->u64Expire = pNext ? pNext->u64Expire : INT64_MAX;
+        DBGFTRACE_U64_TAG(pTimer->CTX_SUFF(pVM), pQueue->u64Expire, "tmTimerQueueUnlinkActive");
+    }
+    if (pNext)
+        TMTIMER_SET_PREV(pNext, pPrev);
+    pTimer->offNext = 0;
+    pTimer->offPrev = 0;
+}
+
+#endif /* !VMM_INCLUDED_SRC_include_TMInline_h */
+
diff --git a/src/VBox/VMM/include/TMInternal.h b/src/VBox/VMM/include/TMInternal.h
new file mode 100644
index 00000000..b04ed38a
--- /dev/null
+++ b/src/VBox/VMM/include/TMInternal.h
@@ -0,0 +1,843 @@
+/* $Id: TMInternal.h $ */
+/** @file
+ * TM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_TMInternal_h
+#define VMM_INCLUDED_SRC_include_TMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <iprt/time.h>
+#include <iprt/timer.h>
+#include <iprt/assert.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/pdmcritsect.h>
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_tm_int       Internal
+ * @ingroup grp_tm
+ * @internal
+ * @{
+ */
+
+/** Frequency of the real clock. */
+#define TMCLOCK_FREQ_REAL       UINT32_C(1000)
+/** Frequency of the virtual clock. */
+#define TMCLOCK_FREQ_VIRTUAL    UINT32_C(1000000000)
+
+
+/**
+ * Timer type.
+ */
+typedef enum TMTIMERTYPE
+{
+    /** Device timer. */
+    TMTIMERTYPE_DEV = 1,
+    /** USB device timer. */
+    TMTIMERTYPE_USB,
+    /** Driver timer. */
+    TMTIMERTYPE_DRV,
+    /** Internal timer . */
+    TMTIMERTYPE_INTERNAL,
+    /** External timer. */
+    TMTIMERTYPE_EXTERNAL
+} TMTIMERTYPE;
+
+/**
+ * Timer state
+ */
+typedef enum TMTIMERSTATE
+{
+    /** Timer is stopped. */
+    TMTIMERSTATE_STOPPED = 1,
+    /** Timer is active. */
+    TMTIMERSTATE_ACTIVE,
+    /** Timer is expired, getting expire and unlinking. */
+    TMTIMERSTATE_EXPIRED_GET_UNLINK,
+    /** Timer is expired and is being delivered. */
+    TMTIMERSTATE_EXPIRED_DELIVER,
+
+    /** Timer is stopped but still in the active list.
+     * Currently in the ScheduleTimers list. */
+    TMTIMERSTATE_PENDING_STOP,
+    /** Timer is stopped but needs unlinking from the ScheduleTimers list.
+     * Currently in the ScheduleTimers list. */
+    TMTIMERSTATE_PENDING_STOP_SCHEDULE,
+    /** Timer is being modified and will soon be pending scheduling.
+     * Currently in the ScheduleTimers list. */
+    TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE,
+    /** Timer is pending scheduling.
+     * Currently in the ScheduleTimers list. */
+    TMTIMERSTATE_PENDING_SCHEDULE,
+    /** Timer is being modified and will soon be pending rescheduling.
+     * Currently in the ScheduleTimers list and the active list. */
+    TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE,
+    /** Timer is modified and is now pending rescheduling.
+     * Currently in the ScheduleTimers list and the active list. */
+    TMTIMERSTATE_PENDING_RESCHEDULE,
+    /** Timer is being destroyed. */
+    TMTIMERSTATE_DESTROY,
+    /** Timer is free. */
+    TMTIMERSTATE_FREE
+} TMTIMERSTATE;
+
+/** Predicate that returns true if the give state is pending scheduling or
+ *  rescheduling of any kind. Will reference the argument more than once! */
+#define TMTIMERSTATE_IS_PENDING_SCHEDULING(enmState) \
+    (   (enmState) <= TMTIMERSTATE_PENDING_RESCHEDULE \
+     && (enmState) >= TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE)
+
+
+/**
+ * Internal representation of a timer.
+ *
+ * For correct serialization (without the use of semaphores and
+ * other blocking/slow constructs) certain rules applies to updating
+ * this structure:
+ *      - For thread other than EMT only u64Expire, enmState and pScheduleNext*
+ *        are changeable. Everything else is out of bounds.
+ *      - Updating of u64Expire timer can only happen in the TMTIMERSTATE_STOPPED
+ *        and TMTIMERSTATE_PENDING_RESCHEDULING_SET_EXPIRE states.
+ *      - Timers in the TMTIMERSTATE_EXPIRED state are only accessible from EMT.
+ *      - Actual destruction of a timer can only be done at scheduling time.
+ */
+typedef struct TMTIMER
+{
+    /** Expire time. */
+    volatile uint64_t       u64Expire;
+    /** Clock to apply to u64Expire. */
+    TMCLOCK                 enmClock;
+    /** Timer callback type. */
+    TMTIMERTYPE             enmType;
+    /** Type specific data. */
+    union
+    {
+        /** TMTIMERTYPE_DEV. */
+        struct
+        {
+            /** Callback. */
+            R3PTRTYPE(PFNTMTIMERDEV)    pfnTimer;
+            /** Device instance. */
+            PPDMDEVINSR3                pDevIns;
+        } Dev;
+
+        /** TMTIMERTYPE_DEV. */
+        struct
+        {
+            /** Callback. */
+            R3PTRTYPE(PFNTMTIMERUSB)    pfnTimer;
+            /** USB device instance. */
+            PPDMUSBINS                  pUsbIns;
+        } Usb;
+
+        /** TMTIMERTYPE_DRV. */
+        struct
+        {
+            /** Callback. */
+            R3PTRTYPE(PFNTMTIMERDRV)    pfnTimer;
+            /** Device instance. */
+            R3PTRTYPE(PPDMDRVINS)       pDrvIns;
+        } Drv;
+
+        /** TMTIMERTYPE_INTERNAL. */
+        struct
+        {
+            /** Callback. */
+            R3PTRTYPE(PFNTMTIMERINT)    pfnTimer;
+        } Internal;
+
+        /** TMTIMERTYPE_EXTERNAL. */
+        struct
+        {
+            /** Callback. */
+            R3PTRTYPE(PFNTMTIMEREXT)    pfnTimer;
+        } External;
+    } u;
+
+    /** Timer state. */
+    volatile TMTIMERSTATE   enmState;
+    /** Timer relative offset to the next timer in the schedule list. */
+    int32_t volatile        offScheduleNext;
+
+    /** Timer relative offset to the next timer in the chain. */
+    int32_t                 offNext;
+    /** Timer relative offset to the previous timer in the chain. */
+    int32_t                 offPrev;
+
+    /** Pointer to the VM the timer belongs to - R3 Ptr. */
+    PVMR3                   pVMR3;
+    /** Pointer to the VM the timer belongs to - R0 Ptr. */
+    R0PTRTYPE(PVMCC)        pVMR0;
+    /** Pointer to the VM the timer belongs to - RC Ptr. */
+    PVMRC                   pVMRC;
+    /** The timer frequency hint.  This is 0 if not hint was given. */
+    uint32_t volatile       uHzHint;
+
+    /** User argument. */
+    RTR3PTR                 pvUser;
+    /** The critical section associated with the lock. */
+    R3PTRTYPE(PPDMCRITSECT) pCritSect;
+
+    /** Pointer to the next timer in the list of created or free timers. (TM::pTimers or TM::pFree) */
+    PTMTIMERR3              pBigNext;
+    /** Pointer to the previous timer in the list of all created timers. (TM::pTimers) */
+    PTMTIMERR3              pBigPrev;
+    /** Pointer to the timer description. */
+    R3PTRTYPE(const char *) pszDesc;
+#if HC_ARCH_BITS == 32
+    uint32_t                padding0; /**< pad structure to multiple of 8 bytes. */
+#endif
+#ifdef VBOX_WITH_STATISTICS
+    STAMPROFILE             StatTimer;
+    STAMPROFILE             StatCritSectEnter;
+    STAMCOUNTER             StatGet;
+    STAMCOUNTER             StatSetAbsolute;
+    STAMCOUNTER             StatSetRelative;
+    STAMCOUNTER             StatStop;
+#endif
+} TMTIMER;
+AssertCompileMemberSize(TMTIMER, enmState, sizeof(uint32_t));
+
+
+/**
+ * Updates a timer state in the correct atomic manner.
+ */
+#if 1
+# define TM_SET_STATE(pTimer, state) \
+    ASMAtomicWriteU32((uint32_t volatile *)&(pTimer)->enmState, state)
+#else
+# define TM_SET_STATE(pTimer, state) \
+    do { \
+        uint32_t uOld1 = (pTimer)->enmState; \
+        Log(("%s: %p: %d -> %d\n", __FUNCTION__, (pTimer), (pTimer)->enmState, state)); \
+        uint32_t uOld2 = ASMAtomicXchgU32((uint32_t volatile *)&(pTimer)->enmState, state); \
+        Assert(uOld1 == uOld2); \
+    } while (0)
+#endif
+
+/**
+ * Tries to updates a timer state in the correct atomic manner.
+ */
+#if 1
+# define TM_TRY_SET_STATE(pTimer, StateNew, StateOld, fRc) \
+    (fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)&(pTimer)->enmState, StateNew, StateOld)
+#else
+# define TM_TRY_SET_STATE(pTimer, StateNew, StateOld, fRc) \
+    do { (fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)&(pTimer)->enmState, StateNew, StateOld); \
+         Log(("%s: %p: %d -> %d %RTbool\n", __FUNCTION__, (pTimer), StateOld, StateNew, fRc)); \
+    } while (0)
+#endif
+
+/** Get the previous timer. */
+#define TMTIMER_GET_PREV(pTimer) ((PTMTIMER)((pTimer)->offPrev ? (intptr_t)(pTimer) + (pTimer)->offPrev : 0))
+/** Get the next timer. */
+#define TMTIMER_GET_NEXT(pTimer) ((PTMTIMER)((pTimer)->offNext ? (intptr_t)(pTimer) + (pTimer)->offNext : 0))
+/** Set the previous timer link. */
+#define TMTIMER_SET_PREV(pTimer, pPrev) ((pTimer)->offPrev = (pPrev) ? (intptr_t)(pPrev) - (intptr_t)(pTimer) : 0)
+/** Set the next timer link. */
+#define TMTIMER_SET_NEXT(pTimer, pNext) ((pTimer)->offNext = (pNext) ? (intptr_t)(pNext) - (intptr_t)(pTimer) : 0)
+
+
+/**
+ * A timer queue.
+ *
+ * This is allocated on the hyper heap.
+ */
+typedef struct TMTIMERQUEUE
+{
+    /** The cached expire time for this queue.
+     * Updated by EMT when scheduling the queue or modifying the head timer.
+     * Assigned UINT64_MAX when there is no head timer. */
+    uint64_t                u64Expire;
+    /** Doubly linked list of active timers.
+     *
+     * When no scheduling is pending, this list is will be ordered by expire time (ascending).
+     * Access is serialized by only letting the emulation thread (EMT) do changes.
+     *
+     * The offset is relative to the queue structure.
+     */
+    int32_t                 offActive;
+    /** List of timers pending scheduling of some kind.
+     *
+     * Timer stats allowed in the list are TMTIMERSTATE_PENDING_STOPPING,
+     * TMTIMERSTATE_PENDING_DESTRUCTION, TMTIMERSTATE_PENDING_STOPPING_DESTRUCTION,
+     * TMTIMERSTATE_PENDING_RESCHEDULING and TMTIMERSTATE_PENDING_SCHEDULE.
+     *
+     * The offset is relative to the queue structure.
+     */
+    int32_t volatile        offSchedule;
+    /** The clock for this queue. */
+    TMCLOCK                 enmClock;
+    /** Pad the structure up to 32 bytes. */
+    uint32_t                au32Padding[3];
+} TMTIMERQUEUE;
+
+/** Pointer to a timer queue. */
+typedef TMTIMERQUEUE *PTMTIMERQUEUE;
+
+/** Get the head of the active timer list. */
+#define TMTIMER_GET_HEAD(pQueue)        ((PTMTIMER)((pQueue)->offActive ? (intptr_t)(pQueue) + (pQueue)->offActive : 0))
+/** Set the head of the active timer list. */
+#define TMTIMER_SET_HEAD(pQueue, pHead) ((pQueue)->offActive = pHead ? (intptr_t)pHead - (intptr_t)(pQueue) : 0)
+
+
+/**
+ * CPU load data set.
+ * Mainly used by tmR3CpuLoadTimer.
+ */
+typedef struct TMCPULOADSTATE
+{
+    /** The percent of the period spent executing guest code. */
+    uint8_t                 cPctExecuting;
+    /** The percent of the period spent halted. */
+    uint8_t                 cPctHalted;
+    /** The percent of the period spent on other things. */
+    uint8_t                 cPctOther;
+    /** Explicit alignment padding */
+    uint8_t                 au8Alignment[1];
+    /** Index into aHistory of the current entry. */
+    uint16_t volatile       idxHistory;
+    /** Number of valid history entries before idxHistory. */
+    uint16_t volatile       cHistoryEntries;
+
+    /** Previous cNsTotal value. */
+    uint64_t                cNsPrevTotal;
+    /** Previous cNsExecuting value. */
+    uint64_t                cNsPrevExecuting;
+    /** Previous cNsHalted value. */
+    uint64_t                cNsPrevHalted;
+    /** Data for the last 30 min (given an interval of 1 second). */
+    struct
+    {
+        uint8_t             cPctExecuting;
+        /** The percent of the period spent halted. */
+        uint8_t             cPctHalted;
+        /** The percent of the period spent on other things. */
+        uint8_t             cPctOther;
+    }                       aHistory[30*60];
+} TMCPULOADSTATE;
+AssertCompileSizeAlignment(TMCPULOADSTATE, 8);
+AssertCompileMemberAlignment(TMCPULOADSTATE, cNsPrevTotal, 8);
+/** Pointer to a CPU load data set. */
+typedef TMCPULOADSTATE *PTMCPULOADSTATE;
+
+
+/**
+ * TSC mode.
+ *
+ * The main modes of how TM implements the TSC clock (TMCLOCK_TSC).
+ */
+typedef enum TMTSCMODE
+{
+    /** The guest TSC is an emulated, virtual TSC. */
+    TMTSCMODE_VIRT_TSC_EMULATED = 1,
+    /** The guest TSC is an offset of the real TSC. */
+    TMTSCMODE_REAL_TSC_OFFSET,
+    /** The guest TSC is dynamically derived through emulating or offsetting. */
+    TMTSCMODE_DYNAMIC,
+    /** The native API provides it. */
+    TMTSCMODE_NATIVE_API
+} TMTSCMODE;
+AssertCompileSize(TMTSCMODE, sizeof(uint32_t));
+
+
+/**
+ * Converts a TM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the TM is part of.
+ * @param   pTM   Pointer to TM instance data.
+ */
+#define TM2VM(pTM)  ( (PVM)((char*)pTM - pTM->offVM) )
+
+
+/**
+ * TM VM Instance data.
+ * Changes to this must checked against the padding of the cfgm union in VM!
+ */
+typedef struct TM
+{
+    /** Offset to the VM structure.
+     * See TM2VM(). */
+    RTUINT                      offVM;
+
+    /** The current TSC mode of the VM.
+     *  Config variable: Mode (string). */
+    TMTSCMODE                   enmTSCMode;
+    /** The original TSC mode of the VM. */
+    TMTSCMODE                   enmOriginalTSCMode;
+    /** Alignment padding. */
+    uint32_t                    u32Alignment0;
+    /** Whether the TSC is tied to the execution of code.
+     * Config variable: TSCTiedToExecution (bool) */
+    bool                        fTSCTiedToExecution;
+    /** Modifier for fTSCTiedToExecution which pauses the TSC while halting if true.
+     * Config variable: TSCNotTiedToHalt (bool) */
+    bool                        fTSCNotTiedToHalt;
+    /** Whether TM TSC mode switching is allowed at runtime. */
+    bool                        fTSCModeSwitchAllowed;
+    /** Whether the guest has enabled use of paravirtualized TSC. */
+    bool                        fParavirtTscEnabled;
+    /** The ID of the virtual CPU that normally runs the timers. */
+    VMCPUID                     idTimerCpu;
+
+    /** The number of CPU clock ticks per second (TMCLOCK_TSC).
+     * Config variable: TSCTicksPerSecond (64-bit unsigned int)
+     * The config variable implies @c enmTSCMode would be
+     * TMTSCMODE_VIRT_TSC_EMULATED. */
+    uint64_t                    cTSCTicksPerSecond;
+    /** The TSC difference introduced by pausing the VM. */
+    uint64_t                    offTSCPause;
+    /** The TSC value when the last TSC was paused. */
+    uint64_t                    u64LastPausedTSC;
+    /** CPU TSCs ticking indicator (one for each VCPU). */
+    uint32_t volatile           cTSCsTicking;
+
+    /** Virtual time ticking enabled indicator (counter for each VCPU). (TMCLOCK_VIRTUAL) */
+    uint32_t volatile           cVirtualTicking;
+    /** Virtual time is not running at 100%. */
+    bool                        fVirtualWarpDrive;
+    /** Virtual timer synchronous time ticking enabled indicator (bool). (TMCLOCK_VIRTUAL_SYNC) */
+    bool volatile               fVirtualSyncTicking;
+    /** Virtual timer synchronous time catch-up active. */
+    bool volatile               fVirtualSyncCatchUp;
+    /** Alignment padding. */
+    bool                        afAlignment1[1];
+    /** WarpDrive percentage.
+     * 100% is normal (fVirtualSyncNormal == true). When other than 100% we apply
+     * this percentage to the raw time source for the period it's been valid in,
+     * i.e. since u64VirtualWarpDriveStart. */
+    uint32_t                    u32VirtualWarpDrivePercentage;
+
+    /** The offset of the virtual clock relative to it's timesource.
+     * Only valid if fVirtualTicking is set. */
+    uint64_t                    u64VirtualOffset;
+    /** The guest virtual time when fVirtualTicking is cleared. */
+    uint64_t                    u64Virtual;
+    /** When the Warp drive was started or last adjusted.
+     * Only valid when fVirtualWarpDrive is set. */
+    uint64_t                    u64VirtualWarpDriveStart;
+    /** The previously returned nano TS.
+     * This handles TSC drift on SMP systems and expired interval.
+     * This is a valid range u64NanoTS to u64NanoTS + 1000000000 (ie. 1sec). */
+    uint64_t volatile           u64VirtualRawPrev;
+    /** The ring-3 data structure for the RTTimeNanoTS workers used by tmVirtualGetRawNanoTS. */
+    RTTIMENANOTSDATAR3          VirtualGetRawDataR3;
+    /** The ring-0 data structure for the RTTimeNanoTS workers used by tmVirtualGetRawNanoTS. */
+    RTTIMENANOTSDATAR0          VirtualGetRawDataR0;
+    /** The ring-0 data structure for the RTTimeNanoTS workers used by tmVirtualGetRawNanoTS. */
+    RTTIMENANOTSDATARC          VirtualGetRawDataRC;
+    /** Pointer to the ring-3 tmVirtualGetRawNanoTS worker function. */
+    R3PTRTYPE(PFNTIMENANOTSINTERNAL) pfnVirtualGetRawR3;
+    /** Pointer to the ring-0 tmVirtualGetRawNanoTS worker function. */
+    R0PTRTYPE(PFNTIMENANOTSINTERNAL) pfnVirtualGetRawR0;
+    /** Pointer to the raw-mode tmVirtualGetRawNanoTS worker function. */
+    RCPTRTYPE(PFNTIMENANOTSINTERNAL) pfnVirtualGetRawRC;
+    /** Alignment. */
+    RTRCPTR                     AlignmentRCPtr;
+    /** The guest virtual timer synchronous time when fVirtualSyncTicking is cleared.
+     * When fVirtualSyncTicking is set it holds the last time returned to
+     * the guest (while the lock was held). */
+    uint64_t volatile           u64VirtualSync;
+    /** The offset of the timer synchronous virtual clock (TMCLOCK_VIRTUAL_SYNC) relative
+     * to the virtual clock (TMCLOCK_VIRTUAL).
+     * (This is accessed by the timer thread and must be updated atomically.) */
+    uint64_t volatile           offVirtualSync;
+    /** The offset into offVirtualSync that's been irrevocably given up by failed catch-up attempts.
+     * Thus the current lag is offVirtualSync - offVirtualSyncGivenUp. */
+    uint64_t                    offVirtualSyncGivenUp;
+    /** The TMCLOCK_VIRTUAL at the previous TMVirtualGetSync call when catch-up is active. */
+    uint64_t volatile           u64VirtualSyncCatchUpPrev;
+    /** The current catch-up percentage. */
+    uint32_t volatile           u32VirtualSyncCatchUpPercentage;
+    /** How much slack when processing timers. */
+    uint32_t                    u32VirtualSyncScheduleSlack;
+    /** When to stop catch-up. */
+    uint64_t                    u64VirtualSyncCatchUpStopThreshold;
+    /** When to give up catch-up. */
+    uint64_t                    u64VirtualSyncCatchUpGiveUpThreshold;
+/** @def TM_MAX_CATCHUP_PERIODS
+ * The number of catchup rates. */
+#define TM_MAX_CATCHUP_PERIODS  10
+    /** The aggressiveness of the catch-up relative to how far we've lagged behind.
+     * The idea is to have increasing catch-up percentage as the lag increases. */
+    struct TMCATCHUPPERIOD
+    {
+        uint64_t                u64Start;       /**< When this period starts. (u64VirtualSyncOffset). */
+        uint32_t                u32Percentage;  /**< The catch-up percent to apply. */
+        uint32_t                u32Alignment;   /**< Structure alignment */
+    }                           aVirtualSyncCatchUpPeriods[TM_MAX_CATCHUP_PERIODS];
+
+    /** The current max timer Hz hint. */
+    uint32_t volatile           uMaxHzHint;
+    /** Whether to recalulate the HzHint next time its queried. */
+    bool volatile               fHzHintNeedsUpdating;
+    /** Alignment */
+    bool                        afAlignment2[3];
+    /** @cfgm{/TM/HostHzMax, uint32_t, Hz, 0, UINT32_MAX, 20000}
+     * The max host Hz frequency hint returned by TMCalcHostTimerFrequency.  */
+    uint32_t                    cHostHzMax;
+    /** @cfgm{/TM/HostHzFudgeFactorTimerCpu, uint32_t, Hz, 0, UINT32_MAX, 111}
+     * The number of Hz TMCalcHostTimerFrequency adds for the timer CPU.  */
+    uint32_t                    cPctHostHzFudgeFactorTimerCpu;
+    /** @cfgm{/TM/HostHzFudgeFactorOtherCpu, uint32_t, Hz, 0, UINT32_MAX, 110}
+     * The number of Hz TMCalcHostTimerFrequency adds for the other CPUs. */
+    uint32_t                    cPctHostHzFudgeFactorOtherCpu;
+    /** @cfgm{/TM/HostHzFudgeFactorCatchUp100, uint32_t, Hz, 0, UINT32_MAX, 300}
+     *  The fudge factor (expressed in percent) that catch-up percentages below
+     * 100% is multiplied by. */
+    uint32_t                    cPctHostHzFudgeFactorCatchUp100;
+    /** @cfgm{/TM/HostHzFudgeFactorCatchUp200, uint32_t, Hz, 0, UINT32_MAX, 250}
+     * The fudge factor (expressed in percent) that catch-up percentages
+     * 100%-199% is multiplied by. */
+    uint32_t                    cPctHostHzFudgeFactorCatchUp200;
+    /** @cfgm{/TM/HostHzFudgeFactorCatchUp400, uint32_t, Hz, 0, UINT32_MAX, 200}
+     * The fudge factor (expressed in percent) that catch-up percentages
+     * 200%-399% is multiplied by. */
+    uint32_t                    cPctHostHzFudgeFactorCatchUp400;
+
+    /** The UTC offset in ns.
+     * This is *NOT* for converting UTC to local time. It is for converting real
+     * world UTC time to VM UTC time. This feature is indented for doing date
+     * testing of software and similar.
+     * @todo Implement warpdrive on UTC. */
+    int64_t                     offUTC;
+    /** The last value TMR3UtcNow returned. */
+    int64_t volatile            nsLastUtcNow;
+    /** File to touch on UTC jump. */
+    R3PTRTYPE(char *)           pszUtcTouchFileOnJump;
+    /** Just to avoid dealing with 32-bit alignment trouble. */
+    R3PTRTYPE(char *)           pszAlignment2b;
+
+    /** Timer queues for the different clock types - R3 Ptr */
+    R3PTRTYPE(PTMTIMERQUEUE)    paTimerQueuesR3;
+    /** Timer queues for the different clock types - R0 Ptr */
+    R0PTRTYPE(PTMTIMERQUEUE)    paTimerQueuesR0;
+    /** Timer queues for the different clock types - RC Ptr */
+    RCPTRTYPE(PTMTIMERQUEUE)    paTimerQueuesRC;
+
+    /** Pointer to our RC mapping of the GIP. */
+    RCPTRTYPE(void *)           pvGIPRC;
+    /** Pointer to our R3 mapping of the GIP. */
+    R3PTRTYPE(void *)           pvGIPR3;
+
+    /** Pointer to a singly linked list of free timers.
+     * This chain is using the TMTIMER::pBigNext members.
+     * Only accessible from the emulation thread. */
+    PTMTIMERR3                  pFree;
+
+    /** Pointer to a doubly linked list of created timers.
+     * This chain is using the TMTIMER::pBigNext and TMTIMER::pBigPrev members.
+     * Only accessible from the emulation thread. */
+    PTMTIMERR3                  pCreated;
+
+    /** The schedule timer timer handle (runtime timer).
+     * This timer will do frequent check on pending queue schedules and
+     * raise VM_FF_TIMER to pull EMTs attention to them.
+     */
+    R3PTRTYPE(PRTTIMER)         pTimer;
+    /** Interval in milliseconds of the pTimer timer. */
+    uint32_t                    u32TimerMillies;
+
+    /** Indicates that queues are being run. */
+    bool volatile               fRunningQueues;
+    /** Indicates that the virtual sync queue is being run. */
+    bool volatile               fRunningVirtualSyncQueue;
+    /** Alignment */
+    bool                        afAlignment3[2];
+
+    /** Lock serializing access to the timer lists. */
+    PDMCRITSECT                 TimerCritSect;
+    /** Lock serializing access to the VirtualSync clock and the associated
+     * timer queue. */
+    PDMCRITSECT                 VirtualSyncLock;
+
+    /** CPU load state for all the virtual CPUs (tmR3CpuLoadTimer). */
+    TMCPULOADSTATE              CpuLoad;
+
+    /** TMR3TimerQueuesDo
+     * @{ */
+    STAMPROFILE                 StatDoQueues;
+    STAMPROFILEADV              aStatDoQueues[TMCLOCK_MAX];
+    /** @} */
+    /** tmSchedule
+     * @{ */
+    STAMPROFILE                 StatScheduleOneRZ;
+    STAMPROFILE                 StatScheduleOneR3;
+    STAMCOUNTER                 StatScheduleSetFF;
+    STAMCOUNTER                 StatPostponedR3;
+    STAMCOUNTER                 StatPostponedRZ;
+    /** @} */
+    /** Read the time
+     * @{ */
+    STAMCOUNTER                 StatVirtualGet;
+    STAMCOUNTER                 StatVirtualGetSetFF;
+    STAMCOUNTER                 StatVirtualSyncGet;
+    STAMCOUNTER                 StatVirtualSyncGetAdjLast;
+    STAMCOUNTER                 StatVirtualSyncGetELoop;
+    STAMCOUNTER                 StatVirtualSyncGetExpired;
+    STAMCOUNTER                 StatVirtualSyncGetLockless;
+    STAMCOUNTER                 StatVirtualSyncGetLocked;
+    STAMCOUNTER                 StatVirtualSyncGetSetFF;
+    STAMCOUNTER                 StatVirtualPause;
+    STAMCOUNTER                 StatVirtualResume;
+    /** @} */
+    /** TMTimerPoll
+     * @{ */
+    STAMCOUNTER                 StatPoll;
+    STAMCOUNTER                 StatPollAlreadySet;
+    STAMCOUNTER                 StatPollELoop;
+    STAMCOUNTER                 StatPollMiss;
+    STAMCOUNTER                 StatPollRunning;
+    STAMCOUNTER                 StatPollSimple;
+    STAMCOUNTER                 StatPollVirtual;
+    STAMCOUNTER                 StatPollVirtualSync;
+    /** @} */
+    /** TMTimerSet sans virtual sync timers.
+     * @{ */
+    STAMCOUNTER                 StatTimerSet;
+    STAMCOUNTER                 StatTimerSetOpt;
+    STAMPROFILE                 StatTimerSetRZ;
+    STAMPROFILE                 StatTimerSetR3;
+    STAMCOUNTER                 StatTimerSetStStopped;
+    STAMCOUNTER                 StatTimerSetStExpDeliver;
+    STAMCOUNTER                 StatTimerSetStActive;
+    STAMCOUNTER                 StatTimerSetStPendStop;
+    STAMCOUNTER                 StatTimerSetStPendStopSched;
+    STAMCOUNTER                 StatTimerSetStPendSched;
+    STAMCOUNTER                 StatTimerSetStPendResched;
+    STAMCOUNTER                 StatTimerSetStOther;
+    /** @}  */
+    /** TMTimerSet on virtual sync timers.
+     * @{ */
+    STAMCOUNTER                 StatTimerSetVs;
+    STAMPROFILE                 StatTimerSetVsRZ;
+    STAMPROFILE                 StatTimerSetVsR3;
+    STAMCOUNTER                 StatTimerSetVsStStopped;
+    STAMCOUNTER                 StatTimerSetVsStExpDeliver;
+    STAMCOUNTER                 StatTimerSetVsStActive;
+    /** @} */
+    /** TMTimerSetRelative sans virtual sync timers
+     * @{ */
+    STAMCOUNTER                 StatTimerSetRelative;
+    STAMPROFILE                 StatTimerSetRelativeRZ;
+    STAMPROFILE                 StatTimerSetRelativeR3;
+    STAMCOUNTER                 StatTimerSetRelativeOpt;
+    STAMCOUNTER                 StatTimerSetRelativeStStopped;
+    STAMCOUNTER                 StatTimerSetRelativeStExpDeliver;
+    STAMCOUNTER                 StatTimerSetRelativeStActive;
+    STAMCOUNTER                 StatTimerSetRelativeStPendStop;
+    STAMCOUNTER                 StatTimerSetRelativeStPendStopSched;
+    STAMCOUNTER                 StatTimerSetRelativeStPendSched;
+    STAMCOUNTER                 StatTimerSetRelativeStPendResched;
+    STAMCOUNTER                 StatTimerSetRelativeStOther;
+    /** @} */
+    /** TMTimerSetRelative on virtual sync timers.
+     * @{ */
+    STAMCOUNTER                 StatTimerSetRelativeVs;
+    STAMPROFILE                 StatTimerSetRelativeVsRZ;
+    STAMPROFILE                 StatTimerSetRelativeVsR3;
+    STAMCOUNTER                 StatTimerSetRelativeVsStStopped;
+    STAMCOUNTER                 StatTimerSetRelativeVsStExpDeliver;
+    STAMCOUNTER                 StatTimerSetRelativeVsStActive;
+    /** @} */
+    /** TMTimerStop sans virtual sync.
+     * @{ */
+    STAMPROFILE                 StatTimerStopRZ;
+    STAMPROFILE                 StatTimerStopR3;
+    /** @} */
+    /** TMTimerStop on virtual sync timers.
+     * @{ */
+    STAMPROFILE                 StatTimerStopVsRZ;
+    STAMPROFILE                 StatTimerStopVsR3;
+    /** @} */
+    /** VirtualSync - Running and Catching Up
+     * @{ */
+    STAMCOUNTER                 StatVirtualSyncRun;
+    STAMCOUNTER                 StatVirtualSyncRunRestart;
+    STAMPROFILE                 StatVirtualSyncRunSlack;
+    STAMCOUNTER                 StatVirtualSyncRunStop;
+    STAMCOUNTER                 StatVirtualSyncRunStoppedAlready;
+    STAMCOUNTER                 StatVirtualSyncGiveUp;
+    STAMCOUNTER                 StatVirtualSyncGiveUpBeforeStarting;
+    STAMPROFILEADV              StatVirtualSyncCatchup;
+    STAMCOUNTER                 aStatVirtualSyncCatchupInitial[TM_MAX_CATCHUP_PERIODS];
+    STAMCOUNTER                 aStatVirtualSyncCatchupAdjust[TM_MAX_CATCHUP_PERIODS];
+    /** @} */
+    /** TMR3VirtualSyncFF (non dedicated EMT). */
+    STAMPROFILE                 StatVirtualSyncFF;
+    /** The timer callback. */
+    STAMCOUNTER                 StatTimerCallbackSetFF;
+    STAMCOUNTER                 StatTimerCallback;
+
+    /** Calls to TMCpuTickSet. */
+    STAMCOUNTER                 StatTSCSet;
+
+    /** TSC starts and stops. */
+    STAMCOUNTER                 StatTSCPause;
+    STAMCOUNTER                 StatTSCResume;
+
+    /** @name Reasons for refusing TSC offsetting in TMCpuTickCanUseRealTSC.
+     * @{ */
+    STAMCOUNTER                 StatTSCNotFixed;
+    STAMCOUNTER                 StatTSCNotTicking;
+    STAMCOUNTER                 StatTSCCatchupLE010;
+    STAMCOUNTER                 StatTSCCatchupLE025;
+    STAMCOUNTER                 StatTSCCatchupLE100;
+    STAMCOUNTER                 StatTSCCatchupOther;
+    STAMCOUNTER                 StatTSCWarp;
+    STAMCOUNTER                 StatTSCUnderflow;
+    STAMCOUNTER                 StatTSCSyncNotTicking;
+    /** @} */
+} TM;
+/** Pointer to TM VM instance data. */
+typedef TM *PTM;
+
+/**
+ * TM VMCPU Instance data.
+ * Changes to this must checked against the padding of the tm union in VM!
+ */
+typedef struct TMCPU
+{
+    /** Offset to the VMCPU structure.
+     * See TMCPU2VM(). */
+    RTUINT                      offVMCPU;
+
+    /** CPU timestamp ticking enabled indicator (bool). (RDTSC) */
+    bool                        fTSCTicking;
+    bool                        afAlignment0[3]; /**< alignment padding */
+
+    /** The offset between the host tick (TSC/virtual depending on the TSC mode) and
+     *  the guest tick. */
+    uint64_t                    offTSCRawSrc;
+
+    /** The guest TSC when fTicking is cleared. */
+    uint64_t                    u64TSC;
+
+    /** The last seen TSC by the guest. */
+    uint64_t                    u64TSCLastSeen;
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    /** The nanosecond timestamp of the CPU start or resume.
+     * This is recalculated when the VM is started so that
+     * cNsTotal = RTTimeNanoTS() - u64NsTsStartCpu. */
+    uint64_t                    u64NsTsStartTotal;
+    /** The nanosecond timestamp of the last start-execute notification. */
+    uint64_t                    u64NsTsStartExecuting;
+    /** The nanosecond timestamp of the last start-halt notification. */
+    uint64_t                    u64NsTsStartHalting;
+    /** The cNsXXX generation. */
+    uint32_t volatile           uTimesGen;
+    /** Explicit alignment padding.  */
+    uint32_t                    u32Alignment;
+    /** The number of nanoseconds total run time.
+     * @remarks This is updated when cNsExecuting and cNsHalted are updated. */
+    uint64_t                    cNsTotal;
+    /** The number of nanoseconds spent executing. */
+    uint64_t                    cNsExecuting;
+    /** The number of nanoseconds being halted. */
+    uint64_t                    cNsHalted;
+    /** The number of nanoseconds spent on other things.
+     * @remarks This is updated when cNsExecuting and cNsHalted are updated. */
+    uint64_t                    cNsOther;
+    /** The number of halts. */
+    uint64_t                    cPeriodsHalted;
+    /** The number of guest execution runs. */
+    uint64_t                    cPeriodsExecuting;
+# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS)
+    /** Resettable version of cNsTotal. */
+    STAMCOUNTER                 StatNsTotal;
+    /** Resettable version of cNsExecuting. */
+    STAMPROFILE                 StatNsExecuting;
+    /** Long execution intervals. */
+    STAMPROFILE                 StatNsExecLong;
+    /** Short execution intervals . */
+    STAMPROFILE                 StatNsExecShort;
+    /** Tiny execution intervals . */
+    STAMPROFILE                 StatNsExecTiny;
+    /** Resettable version of cNsHalted. */
+    STAMPROFILE                 StatNsHalted;
+    /** Resettable version of cNsOther. */
+    STAMPROFILE                 StatNsOther;
+# endif
+
+    /** CPU load state for this virtual CPU (tmR3CpuLoadTimer). */
+    TMCPULOADSTATE              CpuLoad;
+#endif
+} TMCPU;
+/** Pointer to TM VMCPU instance data. */
+typedef TMCPU *PTMCPU;
+
+const char             *tmTimerState(TMTIMERSTATE enmState);
+void                    tmTimerQueueSchedule(PVM pVM, PTMTIMERQUEUE pQueue);
+#ifdef VBOX_STRICT
+void                    tmTimerQueuesSanityChecks(PVM pVM, const char *pszWhere);
+#endif
+
+uint64_t                tmR3CpuTickGetRawVirtualNoCheck(PVM pVM);
+int                     tmCpuTickPause(PVMCPUCC pVCpu);
+int                     tmCpuTickPauseLocked(PVMCC pVM, PVMCPUCC pVCpu);
+int                     tmCpuTickResume(PVMCC pVM, PVMCPUCC pVCpu);
+int                     tmCpuTickResumeLocked(PVMCC pVM, PVMCPUCC pVCpu);
+
+int                     tmVirtualPauseLocked(PVMCC pVM);
+int                     tmVirtualResumeLocked(PVMCC pVM);
+DECLCALLBACK(DECLEXPORT(void))      tmVirtualNanoTSBad(PRTTIMENANOTSDATA pData, uint64_t u64NanoTS,
+                                                       uint64_t u64DeltaPrev, uint64_t u64PrevNanoTS);
+DECLCALLBACK(DECLEXPORT(uint64_t))  tmVirtualNanoTSRediscover(PRTTIMENANOTSDATA pData);
+DECLCALLBACK(DECLEXPORT(uint64_t))  tmVirtualNanoTSBadCpuIndex(PRTTIMENANOTSDATA pData, uint16_t idApic,
+                                                               uint16_t iCpuSet, uint16_t iGipCpu);
+
+/**
+ * Try take the timer lock, wait in ring-3 return VERR_SEM_BUSY in R0/RC.
+ *
+ * @retval  VINF_SUCCESS on success (always in ring-3).
+ * @retval  VERR_SEM_BUSY in RC and R0 if the semaphore is busy.
+ *
+ * @param   a_pVM       Pointer to the VM.
+ *
+ * @remarks The virtual sync timer queue requires the virtual sync lock.
+ */
+#define TM_LOCK_TIMERS(a_pVM)       PDMCritSectEnter(&(a_pVM)->tm.s.TimerCritSect, VERR_SEM_BUSY)
+
+/**
+ * Try take the timer lock, no waiting.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if busy.
+ *
+ * @param   a_pVM       Pointer to the VM.
+ *
+ * @remarks The virtual sync timer queue requires the virtual sync lock.
+ */
+#define TM_TRY_LOCK_TIMERS(a_pVM)   PDMCritSectTryEnter(&(a_pVM)->tm.s.TimerCritSect)
+
+/** Lock the timers (sans the virtual sync queue). */
+#define TM_UNLOCK_TIMERS(a_pVM)     do { PDMCritSectLeave(&(a_pVM)->tm.s.TimerCritSect); } while (0)
+
+/** Checks that the caller owns the timer lock.  */
+#define TM_ASSERT_TIMER_LOCK_OWNERSHIP(a_pVM) \
+    Assert(PDMCritSectIsOwner(&(a_pVM)->tm.s.TimerCritSect))
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_TMInternal_h */
+
diff --git a/src/VBox/VMM/include/TRPMInternal.h b/src/VBox/VMM/include/TRPMInternal.h
new file mode 100644
index 00000000..ad7c0282
--- /dev/null
+++ b/src/VBox/VMM/include/TRPMInternal.h
@@ -0,0 +1,96 @@
+/* $Id: TRPMInternal.h $ */
+/** @file
+ * TRPM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_TRPMInternal_h
+#define VMM_INCLUDED_SRC_include_TRPMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/pgm.h>
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_trpm_int   Internals
+ * @ingroup grp_trpm
+ * @internal
+ * @{
+ */
+
+/**
+ * TRPM Data (part of VM)
+ *
+ * @note This used to be a big deal when we had raw-mode, now it's a dud. :-)
+ */
+typedef struct TRPM
+{
+#ifdef VBOX_WITH_STATISTICS
+    /** Statistics for interrupt handlers (allocated on the hypervisor heap) - R3
+     * pointer. */
+    R3PTRTYPE(PSTAMCOUNTER) paStatForwardedIRQR3;
+#endif
+    uint64_t                u64Dummy;
+} TRPM;
+
+/** Pointer to TRPM Data. */
+typedef TRPM *PTRPM;
+
+
+/**
+ * Per CPU data for TRPM.
+ */
+typedef struct TRPMCPU
+{
+    /** Active Interrupt or trap vector number.
+     * If not UINT32_MAX this indicates that we're currently processing a
+     * interrupt, trap, fault, abort, whatever which have arrived at that
+     * vector number.
+     */
+    uint32_t                uActiveVector;
+
+    /** Active trap type. */
+    TRPMEVENT               enmActiveType;
+
+    /** Errorcode for the active interrupt/trap. */
+    uint32_t                uActiveErrorCode;
+
+    /** Instruction length for software interrupts and software exceptions
+     * (\#BP, \#OF) */
+    uint8_t                 cbInstr;
+
+    /** Whether this \#DB trap is caused due to INT1/ICEBP. */
+    bool                    fIcebp;
+
+    /** CR2 at the time of the active exception. */
+    RTGCUINTPTR             uActiveCR2;
+} TRPMCPU;
+
+/** Pointer to TRPMCPU Data. */
+typedef TRPMCPU *PTRPMCPU;
+/** Pointer to const TRPMCPU Data. */
+typedef const TRPMCPU *PCTRPMCPU;
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_include_TRPMInternal_h */
diff --git a/src/VBox/VMM/include/VMInternal.h b/src/VBox/VMM/include/VMInternal.h
new file mode 100644
index 00000000..4dc9c063
--- /dev/null
+++ b/src/VBox/VMM/include/VMInternal.h
@@ -0,0 +1,485 @@
+/* $Id: VMInternal.h $ */
+/** @file
+ * VM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_VMInternal_h
+#define VMM_INCLUDED_SRC_include_VMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/vmm/vmapi.h>
+#include <iprt/assert.h>
+#include <iprt/critsect.h>
+#include <setjmp.h>
+
+
+
+/** @defgroup grp_vm_int   Internals
+ * @ingroup grp_vm
+ * @internal
+ * @{
+ */
+
+
+/**
+ * VM state change callback.
+ */
+typedef struct VMATSTATE
+{
+    /** Pointer to the next one. */
+    struct VMATSTATE               *pNext;
+    /** Pointer to the callback. */
+    PFNVMATSTATE                    pfnAtState;
+    /** The user argument. */
+    void                           *pvUser;
+} VMATSTATE;
+/** Pointer to a VM state change callback. */
+typedef VMATSTATE *PVMATSTATE;
+
+
+/**
+ * VM error callback.
+ */
+typedef struct VMATERROR
+{
+    /** Pointer to the next one. */
+    struct VMATERROR               *pNext;
+    /** Pointer to the callback. */
+    PFNVMATERROR                    pfnAtError;
+    /** The user argument. */
+    void                           *pvUser;
+} VMATERROR;
+/** Pointer to a VM error callback. */
+typedef VMATERROR *PVMATERROR;
+
+
+/**
+ * Chunk of memory allocated off the hypervisor heap in which
+ * we copy the error details.
+ */
+typedef struct VMERROR
+{
+    /** The size of the chunk. */
+    uint32_t                        cbAllocated;
+    /** The current offset into the chunk.
+     * We start by putting the filename and function immediately
+     * after the end of the buffer. */
+    uint32_t                        off;
+    /** Offset from the start of this structure to the file name. */
+    uint32_t                        offFile;
+    /** The line number. */
+    uint32_t                        iLine;
+    /** Offset from the start of this structure to the function name. */
+    uint32_t                        offFunction;
+    /** Offset from the start of this structure to the formatted message text. */
+    uint32_t                        offMessage;
+    /** The VBox status code. */
+    int32_t                         rc;
+} VMERROR, *PVMERROR;
+
+
+/**
+ * VM runtime error callback.
+ */
+typedef struct VMATRUNTIMEERROR
+{
+    /** Pointer to the next one. */
+    struct VMATRUNTIMEERROR         *pNext;
+    /** Pointer to the callback. */
+    PFNVMATRUNTIMEERROR              pfnAtRuntimeError;
+    /** The user argument. */
+    void                            *pvUser;
+} VMATRUNTIMEERROR;
+/** Pointer to a VM error callback. */
+typedef VMATRUNTIMEERROR *PVMATRUNTIMEERROR;
+
+
+/**
+ * Chunk of memory allocated off the hypervisor heap in which
+ * we copy the runtime error details.
+ */
+typedef struct VMRUNTIMEERROR
+{
+    /** The size of the chunk. */
+    uint32_t                        cbAllocated;
+    /** The current offset into the chunk.
+     * We start by putting the error ID immediately
+     * after the end of the buffer. */
+    uint32_t                        off;
+    /** Offset from the start of this structure to the error ID. */
+    uint32_t                        offErrorId;
+    /** Offset from the start of this structure to the formatted message text. */
+    uint32_t                        offMessage;
+    /** Error flags. */
+    uint32_t                        fFlags;
+} VMRUNTIMEERROR, *PVMRUNTIMEERROR;
+
+/** The halt method. */
+typedef enum
+{
+    /** The usual invalid value. */
+    VMHALTMETHOD_INVALID = 0,
+    /** Use the method used during bootstrapping. */
+    VMHALTMETHOD_BOOTSTRAP,
+    /** Use the default method. */
+    VMHALTMETHOD_DEFAULT,
+    /** The old spin/yield/block method. */
+    VMHALTMETHOD_OLD,
+    /** The first go at a block/spin method. */
+    VMHALTMETHOD_1,
+    /** The first go at a more global approach. */
+    VMHALTMETHOD_GLOBAL_1,
+    /** The end of valid methods. (not inclusive of course) */
+    VMHALTMETHOD_END,
+    /** The usual 32-bit max value. */
+    VMHALTMETHOD_32BIT_HACK = 0x7fffffff
+} VMHALTMETHOD;
+
+
+/**
+ * VM Internal Data (part of the VM structure).
+ *
+ * @todo Move this and all related things to VMM. The VM component was, to some
+ *       extent at least, a bad ad hoc design which should all have been put in
+ *       VMM. @see pg_vm.
+ */
+typedef struct VMINT
+{
+    /** VM Error Message. */
+    R3PTRTYPE(PVMERROR)             pErrorR3;
+    /** VM Runtime Error Message. */
+    R3PTRTYPE(PVMRUNTIMEERROR)      pRuntimeErrorR3;
+    /** The VM was/is-being teleported and has not yet been fully resumed. */
+    bool                            fTeleportedAndNotFullyResumedYet;
+    /** The VM should power off instead of reset. */
+    bool                            fPowerOffInsteadOfReset;
+    /** Reset counter (soft + hard). */
+    uint32_t                        cResets;
+    /** Hard reset counter. */
+    uint32_t                        cHardResets;
+    /** Soft reset counter. */
+    uint32_t                        cSoftResets;
+} VMINT;
+/** Pointer to the VM Internal Data (part of the VM structure). */
+typedef VMINT *PVMINT;
+
+
+#ifdef IN_RING3
+
+/**
+ * VM internal data kept in the UVM.
+ */
+typedef struct VMINTUSERPERVM
+{
+    /** Head of the standard request queue. Atomic. */
+    volatile PVMREQ                 pNormalReqs;
+    /** Head of the priority request queue. Atomic. */
+    volatile PVMREQ                 pPriorityReqs;
+    /** The last index used during alloc/free. */
+    volatile uint32_t               iReqFree;
+    /** Number of free request packets. */
+    volatile uint32_t               cReqFree;
+    /** Array of pointers to lists of free request packets. Atomic. */
+    volatile PVMREQ                 apReqFree[16 - (HC_ARCH_BITS == 32 ? 5 : 4)];
+
+    /** The reference count of the UVM handle. */
+    volatile uint32_t               cUvmRefs;
+
+    /** Number of active EMTs. */
+    volatile uint32_t               cActiveEmts;
+
+# ifdef VBOX_WITH_STATISTICS
+#  if HC_ARCH_BITS == 32
+    uint32_t                        uPadding;
+#  endif
+    /** Number of VMR3ReqAlloc returning a new packet. */
+    STAMCOUNTER                     StatReqAllocNew;
+    /** Number of VMR3ReqAlloc causing races. */
+    STAMCOUNTER                     StatReqAllocRaces;
+    /** Number of VMR3ReqAlloc returning a recycled packet. */
+    STAMCOUNTER                     StatReqAllocRecycled;
+    /** Number of VMR3ReqFree calls. */
+    STAMCOUNTER                     StatReqFree;
+    /** Number of times the request was actually freed. */
+    STAMCOUNTER                     StatReqFreeOverflow;
+    /** Number of requests served. */
+    STAMCOUNTER                     StatReqProcessed;
+    /** Number of times there are more than one request and the others needed to be
+     * pushed back onto the list. */
+    STAMCOUNTER                     StatReqMoreThan1;
+    /** Number of times we've raced someone when pushing the other requests back
+     * onto the list. */
+    STAMCOUNTER                     StatReqPushBackRaces;
+# endif
+
+    /** Pointer to the support library session.
+     * Mainly for creation and destruction. */
+    PSUPDRVSESSION                  pSession;
+
+    /** Force EMT to terminate. */
+    bool volatile                   fTerminateEMT;
+
+    /** Critical section for pAtState and enmPrevVMState. */
+    RTCRITSECT                      AtStateCritSect;
+    /** List of registered state change callbacks. */
+    PVMATSTATE                      pAtState;
+    /** List of registered state change callbacks. */
+    PVMATSTATE                     *ppAtStateNext;
+    /** The previous VM state.
+     * This is mainly used for the 'Resetting' state, but may come in handy later
+     * and when debugging. */
+    VMSTATE                         enmPrevVMState;
+
+    /** Reason for the most recent suspend operation. */
+    VMSUSPENDREASON                 enmSuspendReason;
+    /** Reason for the most recent operation. */
+    VMRESUMEREASON                  enmResumeReason;
+
+    /** Critical section for pAtError and pAtRuntimeError. */
+    RTCRITSECT                      AtErrorCritSect;
+
+    /** List of registered error callbacks. */
+    PVMATERROR                      pAtError;
+    /** List of registered error callbacks. */
+    PVMATERROR                     *ppAtErrorNext;
+    /** The error message count.
+     * This is incremented every time an error is raised.  */
+    uint32_t volatile               cErrors;
+
+    /** The runtime error message count.
+     * This is incremented every time a runtime error is raised.  */
+    uint32_t volatile               cRuntimeErrors;
+    /** List of registered error callbacks. */
+    PVMATRUNTIMEERROR               pAtRuntimeError;
+    /** List of registered error callbacks. */
+    PVMATRUNTIMEERROR              *ppAtRuntimeErrorNext;
+
+    /** @name Generic Halt data
+     * @{
+     */
+    /** The current halt method.
+     * Can be selected by CFGM option 'VM/HaltMethod'. */
+    VMHALTMETHOD                    enmHaltMethod;
+    /** The index into g_aHaltMethods of the current halt method. */
+    uint32_t volatile               iHaltMethod;
+    /** @} */
+
+    /** @todo Do NOT add new members here or reuse the current, we need to store the config for
+     *  each halt method separately because we're racing on SMP guest rigs. */
+    union
+    {
+       /**
+        * Method 1 & 2 - Block whenever possible, and when lagging behind
+        * switch to spinning with regular blocking every 5-200ms (defaults)
+        * depending on the accumulated lag. The blocking interval is adjusted
+        * with the average oversleeping of the last 64 times.
+        *
+        * The difference between 1 and 2 is that we use native absolute
+        * time APIs for the blocking instead of the millisecond based IPRT
+        * interface.
+        */
+        struct
+        {
+            /** The max interval without blocking (when spinning). */
+            uint32_t                u32MinBlockIntervalCfg;
+            /** The minimum interval between blocking (when spinning). */
+            uint32_t                u32MaxBlockIntervalCfg;
+            /** The value to divide the current lag by to get the raw blocking interval (when spinning). */
+            uint32_t                u32LagBlockIntervalDivisorCfg;
+            /** When to start spinning (lag / nano secs). */
+            uint32_t                u32StartSpinningCfg;
+            /** When to stop spinning (lag / nano secs). */
+            uint32_t                u32StopSpinningCfg;
+        }                           Method12;
+
+       /**
+        * The GVMM manages halted and waiting EMTs.
+        */
+        struct
+        {
+            /** The threshold between spinning and blocking. */
+            uint32_t                cNsSpinBlockThresholdCfg;
+        }                           Global1;
+    }                               Halt;
+
+    /** Pointer to the DBGC instance data. */
+    void                           *pvDBGC;
+
+    /** TLS index for the VMINTUSERPERVMCPU pointer. */
+    RTTLS                           idxTLS;
+
+    /** The VM name. (Set after the config constructure has been called.) */
+    char                           *pszName;
+    /** The VM UUID. (Set after the config constructure has been called.) */
+    RTUUID                          Uuid;
+} VMINTUSERPERVM;
+# ifdef VBOX_WITH_STATISTICS
+AssertCompileMemberAlignment(VMINTUSERPERVM, StatReqAllocNew, 8);
+# endif
+
+/** Pointer to the VM internal data kept in the UVM. */
+typedef VMINTUSERPERVM *PVMINTUSERPERVM;
+
+
+/**
+ * VMCPU internal data kept in the UVM.
+ *
+ * Almost a copy of VMINTUSERPERVM. Separate data properly later on.
+ */
+typedef struct VMINTUSERPERVMCPU
+{
+    /** Head of the normal request queue. Atomic. */
+    volatile PVMREQ                 pNormalReqs;
+    /** Head of the priority request queue. Atomic. */
+    volatile PVMREQ                 pPriorityReqs;
+
+    /** The handle to the EMT thread. */
+    RTTHREAD                        ThreadEMT;
+    /** The native of the EMT thread. */
+    RTNATIVETHREAD                  NativeThreadEMT;
+    /** Wait event semaphore. */
+    RTSEMEVENT                      EventSemWait;
+    /** Wait/Idle indicator. */
+    bool volatile                   fWait;
+    /** Set if we've been thru vmR3Destroy and decremented the active EMT count
+     *  already. */
+    bool volatile                   fBeenThruVmDestroy;
+    /** Align the next bit. */
+    bool                            afAlignment[HC_ARCH_BITS == 32 ? 2 : 6];
+
+    /** @name Generic Halt data
+     * @{
+     */
+    /** The average time (ns) between two halts in the last second. (updated once per second) */
+    uint32_t                        HaltInterval;
+    /** The average halt frequency for the last second. (updated once per second) */
+    uint32_t                        HaltFrequency;
+    /** The number of halts in the current period. */
+    uint32_t                        cHalts;
+    uint32_t                        padding; /**< alignment padding. */
+    /** When we started counting halts in cHalts (RTTimeNanoTS). */
+    uint64_t                        u64HaltsStartTS;
+    /** @} */
+
+    /** Union containing data and config for the different halt algorithms. */
+    union
+    {
+       /**
+        * Method 1 & 2 - Block whenever possible, and when lagging behind
+        * switch to spinning with regular blocking every 5-200ms (defaults)
+        * depending on the accumulated lag. The blocking interval is adjusted
+        * with the average oversleeping of the last 64 times.
+        *
+        * The difference between 1 and 2 is that we use native absolute
+        * time APIs for the blocking instead of the millisecond based IPRT
+        * interface.
+        */
+        struct
+        {
+            /** How many times we've blocked while cBlockedNS and cBlockedTooLongNS has been accumulating. */
+            uint32_t                cBlocks;
+            /** Align the next member. */
+            uint32_t                u32Alignment;
+            /** Avg. time spend oversleeping when blocking. (Re-calculated every so often.) */
+            uint64_t                cNSBlockedTooLongAvg;
+            /** Total time spend oversleeping when blocking. */
+            uint64_t                cNSBlockedTooLong;
+            /** Total time spent blocking. */
+            uint64_t                cNSBlocked;
+            /** The timestamp (RTTimeNanoTS) of the last block. */
+            uint64_t                u64LastBlockTS;
+
+            /** When we started spinning relentlessly in order to catch up some of the oversleeping.
+             * This is 0 when we're not spinning. */
+            uint64_t                u64StartSpinTS;
+        }                           Method12;
+
+# if 0
+       /**
+        * Method 3 & 4 - Same as method 1 & 2 respectivly, except that we
+        * sprinkle it with yields.
+        */
+       struct
+       {
+           /** How many times we've blocked while cBlockedNS and cBlockedTooLongNS has been accumulating. */
+           uint32_t                 cBlocks;
+           /** Avg. time spend oversleeping when blocking. (Re-calculated every so often.) */
+           uint64_t                 cBlockedTooLongNSAvg;
+           /** Total time spend oversleeping when blocking. */
+           uint64_t                 cBlockedTooLongNS;
+           /** Total time spent blocking. */
+           uint64_t                 cBlockedNS;
+           /** The timestamp (RTTimeNanoTS) of the last block. */
+           uint64_t                 u64LastBlockTS;
+
+           /** How many times we've yielded while cBlockedNS and cBlockedTooLongNS has been accumulating. */
+           uint32_t                 cYields;
+           /** Avg. time spend oversleeping when yielding. */
+           uint32_t                 cYieldTooLongNSAvg;
+           /** Total time spend oversleeping when yielding. */
+           uint64_t                 cYieldTooLongNS;
+           /** Total time spent yielding. */
+           uint64_t                 cYieldedNS;
+           /** The timestamp (RTTimeNanoTS) of the last block. */
+           uint64_t                 u64LastYieldTS;
+
+           /** When we started spinning relentlessly in order to catch up some of the oversleeping. */
+           uint64_t                 u64StartSpinTS;
+       }                            Method34;
+# endif
+    }                               Halt;
+
+    /** Profiling the halted state; yielding vs blocking.
+     * @{ */
+    STAMPROFILE                     StatHaltYield;
+    STAMPROFILE                     StatHaltBlock;
+    STAMPROFILE                     StatHaltBlockOverslept;
+    STAMPROFILE                     StatHaltBlockInsomnia;
+    STAMPROFILE                     StatHaltBlockOnTime;
+    STAMPROFILE                     StatHaltTimers;
+    STAMPROFILE                     StatHaltPoll;
+    /** @} */
+} VMINTUSERPERVMCPU;
+AssertCompileMemberAlignment(VMINTUSERPERVMCPU, u64HaltsStartTS, 8);
+AssertCompileMemberAlignment(VMINTUSERPERVMCPU, Halt.Method12.cNSBlockedTooLongAvg, 8);
+AssertCompileMemberAlignment(VMINTUSERPERVMCPU, StatHaltYield, 8);
+
+/** Pointer to the VM internal data kept in the UVM. */
+typedef VMINTUSERPERVMCPU *PVMINTUSERPERVMCPU;
+
+#endif /* IN_RING3 */
+
+RT_C_DECLS_BEGIN
+
+DECLCALLBACK(int)   vmR3EmulationThread(RTTHREAD ThreadSelf, void *pvArg);
+int                 vmR3SetHaltMethodU(PUVM pUVM, VMHALTMETHOD enmHaltMethod);
+DECLCALLBACK(int)   vmR3Destroy(PVM pVM);
+DECLCALLBACK(void)  vmR3SetErrorUV(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list *args);
+void                vmSetErrorCopy(PVM pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list args);
+DECLCALLBACK(int)   vmR3SetRuntimeError(PVM pVM, uint32_t fFlags, const char *pszErrorId, char *pszMessage);
+DECLCALLBACK(int)   vmR3SetRuntimeErrorV(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list *pVa);
+void                vmSetRuntimeErrorCopy(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va);
+void                vmR3SetTerminated(PVM pVM);
+
+RT_C_DECLS_END
+
+
+/** @} */
+
+#endif /* !VMM_INCLUDED_SRC_include_VMInternal_h */
+
diff --git a/src/VBox/VMM/include/VMMInternal.h b/src/VBox/VMM/include/VMMInternal.h
new file mode 100644
index 00000000..aa3ee80c
--- /dev/null
+++ b/src/VBox/VMM/include/VMMInternal.h
@@ -0,0 +1,589 @@
+/* $Id: VMMInternal.h $ */
+/** @file
+ * VMM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_VMMInternal_h
+#define VMM_INCLUDED_SRC_include_VMMInternal_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/log.h>
+#include <iprt/critsect.h>
+
+#if !defined(IN_VMM_R3) && !defined(IN_VMM_R0) && !defined(IN_VMM_RC)
+# error "Not in VMM! This is an internal header!"
+#endif
+#if HC_ARCH_BITS == 32
+# error "32-bit hosts are no longer supported. Go back to 6.0 or earlier!"
+#endif
+
+
+
+/** @defgroup grp_vmm_int   Internals
+ * @ingroup grp_vmm
+ * @internal
+ * @{
+ */
+
+/** @def VBOX_WITH_RC_RELEASE_LOGGING
+ * Enables RC release logging. */
+#define VBOX_WITH_RC_RELEASE_LOGGING
+
+/** @def VBOX_WITH_R0_LOGGING
+ * Enables Ring-0 logging (non-release).
+ *
+ * Ring-0 logging isn't 100% safe yet (thread id reuse / process exit cleanup),
+ * so you have to sign up here by adding your defined(DEBUG_<userid>) to the
+ * \#if, or by adding VBOX_WITH_R0_LOGGING to your LocalConfig.kmk.
+ */
+#if defined(DEBUG_sandervl) || defined(DEBUG_frank) || defined(DEBUG_ramshankar) || defined(DOXYGEN_RUNNING)
+# define VBOX_WITH_R0_LOGGING
+#endif
+
+/** @def VBOX_STRICT_VMM_STACK
+ * Enables VMM stack guard pages to catch stack over- and underruns. */
+#if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING)
+# define VBOX_STRICT_VMM_STACK
+#endif
+
+
+/**
+ * The ring-0 logger instance wrapper.
+ *
+ * We need to be able to find the VM handle from the logger instance, so we wrap
+ * it in this structure.
+ */
+typedef struct VMMR0LOGGER
+{
+    /** Pointer to Pointer to the VM. */
+    R0PTRTYPE(PVMCC)            pVM;
+    /** Size of the allocated logger instance (Logger). */
+    uint32_t                    cbLogger;
+    /** Flag indicating whether we've create the logger Ring-0 instance yet. */
+    bool                        fCreated;
+    /** Flag indicating whether we've disabled flushing (world switch) or not. */
+    bool                        fFlushingDisabled;
+    /** Flag indicating whether we've registered the instance already. */
+    bool                        fRegistered;
+    bool                        a8Alignment;
+    /** The CPU ID. */
+    VMCPUID                     idCpu;
+#if HC_ARCH_BITS == 64
+    uint32_t                    u32Alignment;
+#endif
+    /** The ring-0 logger instance. This extends beyond the size.  */
+    RTLOGGER                    Logger;
+} VMMR0LOGGER;
+/** Pointer to a ring-0 logger instance wrapper. */
+typedef VMMR0LOGGER *PVMMR0LOGGER;
+
+
+/**
+ * Jump buffer for the setjmp/longjmp like constructs used to
+ * quickly 'call' back into Ring-3.
+ */
+typedef struct VMMR0JMPBUF
+{
+    /** Traditional jmp_buf stuff
+     * @{ */
+#if HC_ARCH_BITS == 32
+    uint32_t                    ebx;
+    uint32_t                    esi;
+    uint32_t                    edi;
+    uint32_t                    ebp;
+    uint32_t                    esp;
+    uint32_t                    eip;
+    uint32_t                    eflags;
+#endif
+#if HC_ARCH_BITS == 64
+    uint64_t                    rbx;
+# ifdef RT_OS_WINDOWS
+    uint64_t                    rsi;
+    uint64_t                    rdi;
+# endif
+    uint64_t                    rbp;
+    uint64_t                    r12;
+    uint64_t                    r13;
+    uint64_t                    r14;
+    uint64_t                    r15;
+    uint64_t                    rsp;
+    uint64_t                    rip;
+# ifdef RT_OS_WINDOWS
+    uint128_t                   xmm6;
+    uint128_t                   xmm7;
+    uint128_t                   xmm8;
+    uint128_t                   xmm9;
+    uint128_t                   xmm10;
+    uint128_t                   xmm11;
+    uint128_t                   xmm12;
+    uint128_t                   xmm13;
+    uint128_t                   xmm14;
+    uint128_t                   xmm15;
+# endif
+    uint64_t                    rflags;
+#endif
+    /** @} */
+
+    /** Flag that indicates that we've done a ring-3 call. */
+    bool                        fInRing3Call;
+    /** The number of bytes we've saved. */
+    uint32_t                    cbSavedStack;
+    /** Pointer to the buffer used to save the stack.
+     * This is assumed to be 8KB. */
+    RTR0PTR                     pvSavedStack;
+    /** Esp we we match against esp on resume to make sure the stack wasn't relocated. */
+    RTHCUINTREG                 SpCheck;
+    /** The esp we should resume execution with after the restore. */
+    RTHCUINTREG                 SpResume;
+    /** ESP/RSP at the time of the jump to ring 3. */
+    RTHCUINTREG                 SavedEsp;
+    /** EBP/RBP at the time of the jump to ring 3. */
+    RTHCUINTREG                 SavedEbp;
+    /** EIP/RIP within vmmR0CallRing3LongJmp for assisting unwinding. */
+    RTHCUINTREG                 SavedEipForUnwind;
+    /** Unwind: The vmmR0CallRing3SetJmp return address value. */
+    RTHCUINTREG                 UnwindRetPcValue;
+    /** Unwind: The vmmR0CallRing3SetJmp return address stack location. */
+    RTHCUINTREG                 UnwindRetPcLocation;
+#if HC_ARCH_BITS == 32
+    /** Alignment padding. */
+    uint32_t                    uPadding;
+#endif
+
+    /** Stats: Max amount of stack used. */
+    uint32_t                    cbUsedMax;
+    /** Stats: Average stack usage. (Avg = cbUsedTotal / cUsedTotal) */
+    uint32_t                    cbUsedAvg;
+    /** Stats: Total amount of stack used. */
+    uint64_t                    cbUsedTotal;
+    /** Stats: Number of stack usages. */
+    uint64_t                    cUsedTotal;
+} VMMR0JMPBUF;
+/** Pointer to a ring-0 jump buffer. */
+typedef VMMR0JMPBUF *PVMMR0JMPBUF;
+
+
+/**
+ * VMM Data (part of VM)
+ */
+typedef struct VMM
+{
+    /** Whether we should use the periodic preemption timers. */
+    bool                        fUsePeriodicPreemptionTimers;
+    /** Alignment padding. */
+    bool                        afPadding0[7];
+
+    /** The EMT yield timer. */
+    PTMTIMERR3                  pYieldTimer;
+    /** The period to the next timeout when suspended or stopped.
+     * This is 0 when running. */
+    uint32_t                    cYieldResumeMillies;
+    /** The EMT yield timer interval (milliseconds). */
+    uint32_t                    cYieldEveryMillies;
+    /** The timestamp of the previous yield. (nano) */
+    uint64_t                    u64LastYield;
+
+    /** @name EMT Rendezvous
+     * @{ */
+    /** Semaphore to wait on upon entering ordered execution. */
+    R3PTRTYPE(PRTSEMEVENT)      pahEvtRendezvousEnterOrdered;
+    /** Semaphore to wait on upon entering for one-by-one execution. */
+    RTSEMEVENT                  hEvtRendezvousEnterOneByOne;
+    /** Semaphore to wait on upon entering for all-at-once execution. */
+    RTSEMEVENTMULTI             hEvtMulRendezvousEnterAllAtOnce;
+    /** Semaphore to wait on when done. */
+    RTSEMEVENTMULTI             hEvtMulRendezvousDone;
+    /** Semaphore the VMMR3EmtRendezvous caller waits on at the end. */
+    RTSEMEVENT                  hEvtRendezvousDoneCaller;
+    /** Semaphore to wait on upon recursing. */
+    RTSEMEVENTMULTI             hEvtMulRendezvousRecursionPush;
+    /** Semaphore to wait on after done with recursion (caller restoring state). */
+    RTSEMEVENTMULTI             hEvtMulRendezvousRecursionPop;
+    /** Semaphore the initiator waits on while the EMTs are getting into position
+     *  on hEvtMulRendezvousRecursionPush. */
+    RTSEMEVENT                  hEvtRendezvousRecursionPushCaller;
+    /** Semaphore the initiator waits on while the EMTs sitting on
+     *  hEvtMulRendezvousRecursionPop wakes up and leave. */
+    RTSEMEVENT                  hEvtRendezvousRecursionPopCaller;
+    /** Callback. */
+    R3PTRTYPE(PFNVMMEMTRENDEZVOUS) volatile pfnRendezvous;
+    /** The user argument for the callback. */
+    RTR3PTR volatile            pvRendezvousUser;
+    /** Flags. */
+    volatile uint32_t           fRendezvousFlags;
+    /** The number of EMTs that has entered. */
+    volatile uint32_t           cRendezvousEmtsEntered;
+    /** The number of EMTs that has done their job. */
+    volatile uint32_t           cRendezvousEmtsDone;
+    /** The number of EMTs that has returned. */
+    volatile uint32_t           cRendezvousEmtsReturned;
+    /** The status code. */
+    volatile int32_t            i32RendezvousStatus;
+    /** Spin lock. */
+    volatile uint32_t           u32RendezvousLock;
+    /** The recursion depth. */
+    volatile uint32_t           cRendezvousRecursions;
+    /** The number of EMTs that have entered the recursion routine. */
+    volatile uint32_t           cRendezvousEmtsRecursingPush;
+    /** The number of EMTs that have leaft the recursion routine. */
+    volatile uint32_t           cRendezvousEmtsRecursingPop;
+    /** Triggers rendezvous recursion in the other threads. */
+    volatile bool               fRendezvousRecursion;
+
+    /** @} */
+
+    /** RTThreadPreemptIsPendingTrusty() result, set by vmmR0InitVM() for
+     * release logging purposes. */
+    bool                        fIsPreemptPendingApiTrusty : 1;
+    /** The RTThreadPreemptIsPossible() result,  set by vmmR0InitVM() for
+     * release logging purposes.  */
+    bool                        fIsPreemptPossible : 1;
+
+    bool                        afAlignment2[2]; /**< Alignment padding. */
+
+    /** Buffer for storing the standard assertion message for a ring-0 assertion.
+     * Used for saving the assertion message text for the release log and guru
+     * meditation dump. */
+    char                        szRing0AssertMsg1[512];
+    /** Buffer for storing the custom message for a ring-0 assertion. */
+    char                        szRing0AssertMsg2[256];
+
+    /** Number of VMMR0_DO_HM_RUN or VMMR0_DO_NEM_RUN calls. */
+    STAMCOUNTER                 StatRunGC;
+
+    /** Statistics for each of the RC/R0 return codes.
+     * @{ */
+    STAMCOUNTER                 StatRZRetNormal;
+    STAMCOUNTER                 StatRZRetInterrupt;
+    STAMCOUNTER                 StatRZRetInterruptHyper;
+    STAMCOUNTER                 StatRZRetGuestTrap;
+    STAMCOUNTER                 StatRZRetRingSwitch;
+    STAMCOUNTER                 StatRZRetRingSwitchInt;
+    STAMCOUNTER                 StatRZRetStaleSelector;
+    STAMCOUNTER                 StatRZRetIRETTrap;
+    STAMCOUNTER                 StatRZRetEmulate;
+    STAMCOUNTER                 StatRZRetPatchEmulate;
+    STAMCOUNTER                 StatRZRetIORead;
+    STAMCOUNTER                 StatRZRetIOWrite;
+    STAMCOUNTER                 StatRZRetIOCommitWrite;
+    STAMCOUNTER                 StatRZRetMMIORead;
+    STAMCOUNTER                 StatRZRetMMIOWrite;
+    STAMCOUNTER                 StatRZRetMMIOCommitWrite;
+    STAMCOUNTER                 StatRZRetMMIOPatchRead;
+    STAMCOUNTER                 StatRZRetMMIOPatchWrite;
+    STAMCOUNTER                 StatRZRetMMIOReadWrite;
+    STAMCOUNTER                 StatRZRetMSRRead;
+    STAMCOUNTER                 StatRZRetMSRWrite;
+    STAMCOUNTER                 StatRZRetLDTFault;
+    STAMCOUNTER                 StatRZRetGDTFault;
+    STAMCOUNTER                 StatRZRetIDTFault;
+    STAMCOUNTER                 StatRZRetTSSFault;
+    STAMCOUNTER                 StatRZRetCSAMTask;
+    STAMCOUNTER                 StatRZRetSyncCR3;
+    STAMCOUNTER                 StatRZRetMisc;
+    STAMCOUNTER                 StatRZRetPatchInt3;
+    STAMCOUNTER                 StatRZRetPatchPF;
+    STAMCOUNTER                 StatRZRetPatchGP;
+    STAMCOUNTER                 StatRZRetPatchIretIRQ;
+    STAMCOUNTER                 StatRZRetRescheduleREM;
+    STAMCOUNTER                 StatRZRetToR3Total;
+    STAMCOUNTER                 StatRZRetToR3FF;
+    STAMCOUNTER                 StatRZRetToR3Unknown;
+    STAMCOUNTER                 StatRZRetToR3TMVirt;
+    STAMCOUNTER                 StatRZRetToR3HandyPages;
+    STAMCOUNTER                 StatRZRetToR3PDMQueues;
+    STAMCOUNTER                 StatRZRetToR3Rendezvous;
+    STAMCOUNTER                 StatRZRetToR3Timer;
+    STAMCOUNTER                 StatRZRetToR3DMA;
+    STAMCOUNTER                 StatRZRetToR3CritSect;
+    STAMCOUNTER                 StatRZRetToR3Iem;
+    STAMCOUNTER                 StatRZRetToR3Iom;
+    STAMCOUNTER                 StatRZRetTimerPending;
+    STAMCOUNTER                 StatRZRetInterruptPending;
+    STAMCOUNTER                 StatRZRetCallRing3;
+    STAMCOUNTER                 StatRZRetPATMDuplicateFn;
+    STAMCOUNTER                 StatRZRetPGMChangeMode;
+    STAMCOUNTER                 StatRZRetPendingRequest;
+    STAMCOUNTER                 StatRZRetPGMFlushPending;
+    STAMCOUNTER                 StatRZRetPatchTPR;
+    STAMCOUNTER                 StatRZCallPDMCritSectEnter;
+    STAMCOUNTER                 StatRZCallPDMLock;
+    STAMCOUNTER                 StatRZCallLogFlush;
+    STAMCOUNTER                 StatRZCallPGMPoolGrow;
+    STAMCOUNTER                 StatRZCallPGMMapChunk;
+    STAMCOUNTER                 StatRZCallPGMAllocHandy;
+    STAMCOUNTER                 StatRZCallVMSetError;
+    STAMCOUNTER                 StatRZCallVMSetRuntimeError;
+    STAMCOUNTER                 StatRZCallPGMLock;
+    /** @} */
+} VMM;
+/** Pointer to VMM. */
+typedef VMM *PVMM;
+
+
+/**
+ * VMMCPU Data (part of VMCPU)
+ */
+typedef struct VMMCPU
+{
+    /** The last RC/R0 return code. */
+    int32_t                     iLastGZRc;
+    /** Alignment padding. */
+    uint32_t                    u32Padding0;
+
+    /** VMM stack, pointer to the top of the stack in R3.
+     * Stack is allocated from the hypervisor heap and is page aligned
+     * and always writable in RC. */
+    R3PTRTYPE(uint8_t *)        pbEMTStackR3;
+
+    /** Pointer to the R0 logger instance - R3 Ptr.
+     * This is NULL if logging is disabled. */
+    R3PTRTYPE(PVMMR0LOGGER)     pR0LoggerR3;
+    /** Pointer to the R0 logger instance - R0 Ptr.
+     * This is NULL if logging is disabled. */
+    R0PTRTYPE(PVMMR0LOGGER)     pR0LoggerR0;
+
+    /** Pointer to the R0 release logger instance - R3 Ptr.
+     * This is NULL if logging is disabled. */
+    R3PTRTYPE(PVMMR0LOGGER)     pR0RelLoggerR3;
+    /** Pointer to the R0 release instance - R0 Ptr.
+     * This is NULL if logging is disabled. */
+    R0PTRTYPE(PVMMR0LOGGER)     pR0RelLoggerR0;
+
+    /** Thread context switching hook (ring-0). */
+    RTTHREADCTXHOOK             hCtxHook;
+
+    /** @name Rendezvous
+     * @{ */
+    /** Whether the EMT is executing a rendezvous right now. For detecting
+     *  attempts at recursive rendezvous. */
+    bool volatile               fInRendezvous;
+    bool                        afPadding1[10];
+    /** @} */
+
+    /** Whether we can HLT in VMMR0 rather than having to return to EM.
+     * Updated by vmR3SetHaltMethodU(). */
+    bool                        fMayHaltInRing0;
+    /** The minimum delta for which we can HLT in ring-0 for.
+     * The deadlines we can calculate are  from TM, so, if it's too close
+     * we should just return to ring-3 and run the timer wheel, no point
+     * in spinning in ring-0.
+     * Updated by vmR3SetHaltMethodU(). */
+    uint32_t                    cNsSpinBlockThreshold;
+    /** Number of ring-0 halts (used for depreciating following values). */
+    uint32_t                    cR0Halts;
+    /** Number of ring-0 halts succeeding (VINF_SUCCESS) recently. */
+    uint32_t                    cR0HaltsSucceeded;
+    /** Number of ring-0 halts failing (VINF_EM_HALT) recently. */
+    uint32_t                    cR0HaltsToRing3;
+    /** Padding   */
+    uint32_t                    u32Padding2;
+
+    /** @name Raw-mode context tracing data.
+     * @{ */
+    SUPDRVTRACERUSRCTX          TracerCtx;
+    /** @} */
+
+    /** Alignment padding, making sure u64CallRing3Arg and CallRing3JmpBufR0 are nicely aligned. */
+    uint32_t                    au32Padding3[1];
+
+    /** @name Call Ring-3
+     * Formerly known as host calls.
+     * @{ */
+    /** The disable counter. */
+    uint32_t                    cCallRing3Disabled;
+    /** The pending operation. */
+    VMMCALLRING3                enmCallRing3Operation;
+    /** The result of the last operation. */
+    int32_t                     rcCallRing3;
+    /** The argument to the operation. */
+    uint64_t                    u64CallRing3Arg;
+    /** The Ring-0 notification callback. */
+    R0PTRTYPE(PFNVMMR0CALLRING3NOTIFICATION)   pfnCallRing3CallbackR0;
+    /** The Ring-0 notification callback user argument. */
+    R0PTRTYPE(void *)           pvCallRing3CallbackUserR0;
+    /** The Ring-0 jmp buffer.
+     * @remarks The size of this type isn't stable in assembly, so don't put
+     *          anything that needs to be accessed from assembly after it. */
+    VMMR0JMPBUF                 CallRing3JmpBufR0;
+    /** @} */
+
+    STAMPROFILE                 StatR0HaltBlock;
+    STAMPROFILE                 StatR0HaltBlockOnTime;
+    STAMPROFILE                 StatR0HaltBlockOverslept;
+    STAMPROFILE                 StatR0HaltBlockInsomnia;
+    STAMCOUNTER                 StatR0HaltExec;
+    STAMCOUNTER                 StatR0HaltExecFromBlock;
+    STAMCOUNTER                 StatR0HaltExecFromSpin;
+    STAMCOUNTER                 StatR0HaltToR3;
+    STAMCOUNTER                 StatR0HaltToR3FromSpin;
+    STAMCOUNTER                 StatR0HaltToR3Other;
+    STAMCOUNTER                 StatR0HaltToR3PendingFF;
+    STAMCOUNTER                 StatR0HaltToR3SmallDelta;
+    STAMCOUNTER                 StatR0HaltToR3PostNoInt;
+    STAMCOUNTER                 StatR0HaltToR3PostPendingFF;
+} VMMCPU;
+AssertCompileMemberAlignment(VMMCPU, TracerCtx, 8);
+/** Pointer to VMMCPU. */
+typedef VMMCPU *PVMMCPU;
+
+
+
+RT_C_DECLS_BEGIN
+
+int  vmmInitFormatTypes(void);
+void vmmTermFormatTypes(void);
+uint32_t vmmGetBuildType(void);
+
+#ifdef IN_RING3
+int  vmmR3SwitcherInit(PVM pVM);
+void vmmR3SwitcherRelocate(PVM pVM, RTGCINTPTR offDelta);
+#endif /* IN_RING3 */
+
+#ifdef IN_RING0
+
+/**
+ * World switcher assembly routine.
+ * It will call VMMRCEntry().
+ *
+ * @returns return code from VMMRCEntry().
+ * @param   pVM         The cross context VM structure.
+ * @param   uArg        See VMMRCEntry().
+ * @internal
+ */
+DECLASM(int)    vmmR0WorldSwitch(PVM pVM, unsigned uArg);
+
+/**
+ * Callback function for vmmR0CallRing3SetJmp.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ */
+typedef DECLCALLBACK(int) FNVMMR0SETJMP(PVMCC pVM, PVMCPUCC pVCpu);
+/** Pointer to FNVMMR0SETJMP(). */
+typedef FNVMMR0SETJMP *PFNVMMR0SETJMP;
+
+/**
+ * The setjmp variant used for calling Ring-3.
+ *
+ * This differs from the normal setjmp in that it will resume VMMRZCallRing3 if we're
+ * in the middle of a ring-3 call. Another differences is the function pointer and
+ * argument. This has to do with resuming code and the stack frame of the caller.
+ *
+ * @returns VINF_SUCCESS on success or whatever is passed to vmmR0CallRing3LongJmp.
+ * @param   pJmpBuf     The jmp_buf to set.
+ * @param   pfn         The function to be called when not resuming.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+DECLASM(int)    vmmR0CallRing3SetJmp(PVMMR0JMPBUF pJmpBuf, PFNVMMR0SETJMP pfn, PVM pVM, PVMCPU pVCpu);
+
+
+/**
+ * Callback function for vmmR0CallRing3SetJmp2.
+ *
+ * @returns VBox status code.
+ * @param   pvUser      The user argument.
+ */
+typedef DECLCALLBACK(int) FNVMMR0SETJMP2(PGVM pGVM, VMCPUID idCpu);
+/** Pointer to FNVMMR0SETJMP2(). */
+typedef FNVMMR0SETJMP2 *PFNVMMR0SETJMP2;
+
+/**
+ * Same as vmmR0CallRing3SetJmp except for the function signature.
+ *
+ * @returns VINF_SUCCESS on success or whatever is passed to vmmR0CallRing3LongJmp.
+ * @param   pJmpBuf     The jmp_buf to set.
+ * @param   pfn         The function to be called when not resuming.
+ * @param   pGVM        The ring-0 VM structure.
+ * @param   idCpu       The ID of the calling EMT.
+ */
+DECLASM(int)    vmmR0CallRing3SetJmp2(PVMMR0JMPBUF pJmpBuf, PFNVMMR0SETJMP2 pfn, PGVM pGVM, VMCPUID idCpu);
+
+
+/**
+ * Callback function for vmmR0CallRing3SetJmpEx.
+ *
+ * @returns VBox status code.
+ * @param   pvUser      The user argument.
+ */
+typedef DECLCALLBACK(int) FNVMMR0SETJMPEX(void *pvUser);
+/** Pointer to FNVMMR0SETJMPEX(). */
+typedef FNVMMR0SETJMPEX *PFNVMMR0SETJMPEX;
+
+/**
+ * Same as vmmR0CallRing3SetJmp except for the function signature.
+ *
+ * @returns VINF_SUCCESS on success or whatever is passed to vmmR0CallRing3LongJmp.
+ * @param   pJmpBuf     The jmp_buf to set.
+ * @param   pfn         The function to be called when not resuming.
+ * @param   pvUser      The argument of that function.
+ */
+DECLASM(int)    vmmR0CallRing3SetJmpEx(PVMMR0JMPBUF pJmpBuf, PFNVMMR0SETJMPEX pfn, void *pvUser);
+
+
+/**
+ * Worker for VMMRZCallRing3.
+ * This will save the stack and registers.
+ *
+ * @returns rc.
+ * @param   pJmpBuf     Pointer to the jump buffer.
+ * @param   rc          The return code.
+ */
+DECLASM(int)    vmmR0CallRing3LongJmp(PVMMR0JMPBUF pJmpBuf, int rc);
+
+/**
+ * Internal R0 logger worker: Logger wrapper.
+ */
+VMMR0DECL(void) vmmR0LoggerWrapper(const char *pszFormat, ...);
+
+/**
+ * Internal R0 logger worker: Flush logger.
+ *
+ * @param   pLogger     The logger instance to flush.
+ * @remark  This function must be exported!
+ */
+VMMR0DECL(void) vmmR0LoggerFlush(PRTLOGGER pLogger);
+
+/**
+ * Internal R0 logger worker: Custom prefix.
+ *
+ * @returns Number of chars written.
+ *
+ * @param   pLogger     The logger instance.
+ * @param   pchBuf      The output buffer.
+ * @param   cchBuf      The size of the buffer.
+ * @param   pvUser      User argument (ignored).
+ */
+VMMR0DECL(size_t) vmmR0LoggerPrefix(PRTLOGGER pLogger, char *pchBuf, size_t cchBuf, void *pvUser);
+
+# ifdef VBOX_WITH_TRIPLE_FAULT_HACK
+int  vmmR0TripleFaultHackInit(void);
+void vmmR0TripleFaultHackTerm(void);
+# endif
+
+#endif /* IN_RING0 */
+
+RT_C_DECLS_END
+
+/** @} */
+
+#endif /* !VMM_INCLUDED_SRC_include_VMMInternal_h */
diff --git a/src/VBox/VMM/include/VMMInternal.mac b/src/VBox/VMM/include/VMMInternal.mac
new file mode 100644
index 00000000..10b64c78
--- /dev/null
+++ b/src/VBox/VMM/include/VMMInternal.mac
@@ -0,0 +1,141 @@
+; $Id: VMMInternal.mac $
+;; @file
+; VMM - Internal header file.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%include "VBox/asmdefs.mac"
+%include "VBox/sup.mac"
+
+;
+; Determine the default stack switching unless specified explicitly.
+;
+%ifndef VMM_R0_SWITCH_STACK
+ %ifndef VMM_R0_NO_SWITCH_STACK
+  %ifdef RT_OS_DARWIN
+   %define VMM_R0_SWITCH_STACK
+  %endif
+ %endif
+%endif
+
+
+struc VMMR0JMPBUF
+%ifdef RT_ARCH_X86
+    ; traditional jmp_buf
+    .ebx            resd 1
+    .esi            resd 1
+    .edi            resd 1
+    .ebp            resd 1
+    .esp            resd 1
+    .eip            resd 1
+    .eflags         resd 1
+
+    ; additional state and stack info.
+    .fInRing3Call   resd 1
+    .cbSavedStack   resd 1
+    .pvSavedStack   resd 1
+    .SpCheck        resd 1
+    .SpResume       resd 1
+    .SavedEsp       resd 1
+    .SavedEbp       resd 1
+    .SavedEipForUnwind      resd 1
+    .UnwindRetPcValue       resd 1
+    .UnwindRetPcLocation    resd 1
+%endif
+%ifdef RT_ARCH_AMD64
+    ; traditional jmp_buf
+    .rbx            resq 1
+ %ifdef RT_OS_WINDOWS
+    .rsi            resq 1
+    .rdi            resq 1
+ %endif
+    .rbp            resq 1
+    .r12            resq 1
+    .r13            resq 1
+    .r14            resq 1
+    .r15            resq 1
+    .rsp            resq 1
+    .rip            resq 1
+ %ifdef RT_OS_WINDOWS
+    .xmm6           resq 2
+    .xmm7           resq 2
+    .xmm8           resq 2
+    .xmm9           resq 2
+    .xmm10          resq 2
+    .xmm11          resq 2
+    .xmm12          resq 2
+    .xmm13          resq 2
+    .xmm14          resq 2
+    .xmm15          resq 2
+ %endif
+    .rflags         resq 1
+
+    ; additional state and stack info.
+    .fInRing3Call   resd 1
+    .cbSavedStack   resd 1
+    .pvSavedStack   resq 1
+    .SpCheck        resq 1
+    .SpResume       resq 1
+    .SavedEsp       resq 1
+    .SavedEbp       resq 1
+    .SavedEipForUnwind      resq 1
+    .UnwindRetPcValue       resq 1
+    .UnwindRetPcLocation    resq 1
+%endif
+
+    ; Statistics
+    alignb 8
+    .cbUsedMax      resd 1
+    .cbUsedAvg      resd 1
+    .cbUsedTotal    resq 1
+    .cUsedTotal     resq 1
+endstruc
+
+
+struc VMMCPU
+
+        .iLastGZRc              resd 1
+        alignb 8
+        .pbEMTStackR3           RTR3PTR_RES 1
+
+        .pR0LoggerR3            RTR3PTR_RES 1
+        .pR0LoggerR0            RTR0PTR_RES 1
+        .pR0RelLoggerR3         RTR3PTR_RES 1
+        .pR0RelLoggerR0         RTR0PTR_RES 1
+
+        .hCtxHook               RTR0PTR_RES 1
+
+        .fInRendezvous          resb 1
+        .afPadding1             resb 10
+        .fMayHaltInRing0        resb 1
+        .cNsSpinBlockThreshold  resd 1
+        .cR0Halts               resd 1
+        .cR0HaltsSucceeded      resd 1
+        .cR0HaltsToRing3        resd 1
+
+        alignb 8
+        .TracerCtx              resb SUPDRVTRACERUSRCTX64_size
+
+        .au32Padding3           resd 1
+
+        .cCallRing3Disabled     resd 1
+        .enmCallRing3Operation  resd 1
+        .rcCallRing3            resd 1
+        alignb 8
+        .u64CallRing3Arg        resq 1
+        .pfnCallRing3CallbackR0         RTR0PTR_RES 1
+        .pvCallRing3CallbackUserR0      RTR0PTR_RES 1
+        ; .CallRing3JmpBufR0    resb no-can-do
+endstruc
+
diff --git a/src/VBox/VMM/include/VMMTracing.h b/src/VBox/VMM/include/VMMTracing.h
new file mode 100644
index 00000000..f07b4f89
--- /dev/null
+++ b/src/VBox/VMM/include/VMMTracing.h
@@ -0,0 +1,126 @@
+/* $Id: VMMTracing.h $ */
+/** @file
+ * VBoxVMM - Trace point macros for the VMM.
+ */
+
+/*
+ * Copyright (C) 2012-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_include_VMMTracing_h
+#define VMM_INCLUDED_SRC_include_VMMTracing_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#ifdef DOXYGEN_RUNNING
+# undef VBOX_WITH_DTRACE
+# undef VBOX_WITH_DTRACE_R3
+# undef VBOX_WITH_DTRACE_R0
+# undef VBOX_WITH_DTRACE_RC
+# define DBGFTRACE_ENABLED
+#endif
+#include <VBox/vmm/dbgftrace.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** Gets the trace buffer handle from a VMCPU pointer. */
+#define VMCPU_TO_HTB(a_pVCpu)     ((a_pVCpu)->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf))
+
+/** Gets the trace buffer handle from a VMCPU pointer. */
+#define VM_TO_HTB(a_pVM)          ((a_pVM)->CTX_SUFF(hTraceBuf))
+
+/** Macro wrapper for trace points that are disabled by default. */
+#define TP_COND_VMCPU(a_pVCpu, a_GrpSuff, a_TraceStmt) \
+    do { \
+        if (RT_UNLIKELY( (a_pVCpu)->fTraceGroups & VMMTPGROUP_##a_GrpSuff )) \
+        { \
+            RTTRACEBUF const hTB = (a_pVCpu)->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf); \
+            a_TraceStmt; \
+        } \
+    } while (0)
+
+/** @name VMM Trace Point Groups.
+ * @{ */
+#define VMMTPGROUP_EM       RT_BIT(0)
+#define VMMTPGROUP_HM       RT_BIT(1)
+#define VMMTPGROUP_TM       RT_BIT(2)
+/** @}  */
+
+
+
+/** @name Ring-3 trace points.
+ * @{
+ */
+#ifdef IN_RING3
+# ifdef VBOX_WITH_DTRACE_R3
+#  include "dtrace/VBoxVMM.h"
+
+# elif defined(DBGFTRACE_ENABLED)
+#  define VBOXVMM_EM_STATE_CHANGED(a_pVCpu, a_enmOldState, a_enmNewState, a_rc) \
+        TP_COND_VMCPU(a_pVCpu, EM, RTTraceBufAddMsgF(hTB, "em-state-changed %d -> %d (rc=%d)", a_enmOldState, a_enmNewState, a_rc))
+#  define VBOXVMM_EM_STATE_UNCHANGED(a_pVCpu, a_enmState, a_rc) \
+        TP_COND_VMCPU(a_pVCpu, EM, RTTraceBufAddMsgF(hTB, "em-state-unchanged %d (rc=%d)", a_enmState, a_rc))
+#   define VBOXVMM_EM_RAW_RUN_PRE(a_pVCpu, a_pCtx) \
+        TP_COND_VMCPU(a_pVCpu, EM, RTTraceBufAddMsgF(hTB, "em-raw-pre %04x:%08llx", (a_pCtx)->cs, (a_pCtx)->rip))
+#   define VBOXVMM_EM_RAW_RUN_RET(a_pVCpu, a_pCtx, a_rc) \
+        TP_COND_VMCPU(a_pVCpu, EM, RTTraceBufAddMsgF(hTB, "em-raw-ret %04x:%08llx rc=%d", (a_pCtx)->cs, (a_pCtx)->rip, (a_rc)))
+#   define VBOXVMM_EM_FF_HIGH(a_pVCpu, a_fGlobal, a_fLocal, a_rc) \
+        TP_COND_VMCPU(a_pVCpu, EM, RTTraceBufAddMsgF(hTB, "em-ff-high vm=%#x cpu=%#x rc=%d", (a_fGlobal), (a_fLocal), (a_rc)))
+#   define VBOXVMM_EM_FF_ALL(a_pVCpu, a_fGlobal, a_fLocal, a_rc) \
+        TP_COND_VMCPU(a_pVCpu, EM, RTTraceBufAddMsgF(hTB, "em-ff-all vm=%#x cpu=%#x rc=%d", (a_fGlobal), (a_fLocal), (a_rc)))
+#   define VBOXVMM_EM_FF_ALL_RET(a_pVCpu, a_rc) \
+        TP_COND_VMCPU(a_pVCpu, EM, RTTraceBufAddMsgF(hTB, "em-ff-all-ret %d", (a_rc)))
+#   define VBOXVMM_EM_FF_RAW(a_pVCpu, a_fGlobal, a_fLocal) \
+        TP_COND_VMCPU(a_pVCpu, EM, RTTraceBufAddMsgF(hTB, "em-ff-raw vm=%#x cpu=%#x", (a_fGlobal), (a_fLocal)))
+#   define VBOXVMM_EM_FF_RAW_RET(a_pVCpu, a_rc) \
+        TP_COND_VMCPU(a_pVCpu, EM, RTTraceBufAddMsgF(hTB, "em-ff-raw-ret %d", (a_rc)))
+
+# else
+#   define VBOXVMM_EM_STATE_CHANGED(a_pVCpu, a_enmOldState, a_enmNewState, a_rc) do { } while (0)
+#   define VBOXVMM_EM_STATE_UNCHANGED(a_pVCpu, a_enmState, a_rc) do { } while (0)
+#   define VBOXVMM_EM_RAW_RUN_PRE(a_pVCpu, a_pCtx) do { } while (0)
+#   define VBOXVMM_EM_RAW_RUN_RET(a_pVCpu, a_pCtx, a_rc) do { } while (0)
+#   define VBOXVMM_EM_FF_HIGH(a_pVCpu, a_fGlobal, a_fLocal, a_rc) do { } while (0)
+#   define VBOXVMM_EM_FF_ALL(a_pVCpu, a_fGlobal, a_fLocal, a_rc) do { } while (0)
+#   define VBOXVMM_EM_FF_ALL_RET(a_pVCpu, a_rc) do { } while (0)
+#   define VBOXVMM_EM_FF_RAW(a_pVCpu, a_fGlobal, a_fLocal) do { } while (0)
+#   define VBOXVMM_EM_FF_RAW_RET(a_pVCpu, a_rc) do { } while (0)
+
+# endif
+#endif  /* IN_RING3 */
+/** @} */
+
+
+/** @name Ring-0 trace points.
+ * @{
+ */
+#ifdef IN_RING0
+# ifdef VBOX_WITH_DTRACE_R0
+#  include "VBoxVMMR0-dtrace.h"
+
+# elif defined(DBGFTRACE_ENABLED)
+
+# else
+
+# endif
+#endif /* IN_RING0 */
+/** @} */
+
+
+#endif /* !VMM_INCLUDED_SRC_include_VMMTracing_h */
+
diff --git a/src/VBox/VMM/pure_test.sh b/src/VBox/VMM/pure_test.sh
new file mode 100755
index 00000000..336102a6
--- /dev/null
+++ b/src/VBox/VMM/pure_test.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+# $Id: pure_test.sh $
+## @file
+# pure_test.sh - test the effect of __attribute__((pure)) on a set of
+#                functions.
+#
+# Mark the functions with EXPERIMENT_PURE where the attribute normally would,
+# go update this script so it points to the right header and execute it.  At
+# the end you'll get a pt-report.txt showing the fluctuations in the text size.
+#
+
+#
+# Copyright (C) 2010-2020 Oracle Corporation
+#
+# This file is part of VirtualBox Open Source Edition (OSE), as
+# available from http://www.virtualbox.org. This file is free software;
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License (GPL) as published by the Free Software
+# Foundation, in version 2 as it comes in the "COPYING" file of the
+# VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+# hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+#
+
+set -e
+set -x
+
+BINDIR="../../../out/linux.amd64/release/bin/"
+DLLEXT="so"
+HEADER="../../../include/VBox/cpum.h"
+REPORT="pt-report.txt"
+
+test -e ${HEADER}.bak || kmk_cp $HEADER ${HEADER}.bak
+NAMES=`kmk_sed -e '/EXPERIMENT_PURE/!d' -e '/^#/d' -e 's/^[^()]*([^()]*)[[:space:]]*\([^() ]*\)(.*$/\1/' ${HEADER}.bak `
+echo NAMES=$NAMES
+
+
+#
+# baseline
+#
+kmk_sed -e 's/EXPERIMENT_PURE//' ${HEADER}.bak --output ${HEADER}
+kmk KBUILD_TYPE=release VBoxVMM VMMR0 VMMGC
+size ${BINDIR}/VMMR0.r0 ${BINDIR}/VMMGC.gc ${BINDIR}/VBoxVMM.${DLLEXT} > pt-baseline.txt
+
+exec < "pt-baseline.txt"
+read buf                                # ignore
+read buf; baseline_r0=`echo $buf | kmk_sed -e 's/^[[:space:]]*\([^[[:space:]]*\).*$/\1/' `
+read buf; baseline_rc=`echo $buf | kmk_sed -e 's/^[[:space:]]*\([^[[:space:]]*\).*$/\1/' `
+read buf; baseline_r3=`echo $buf | kmk_sed -e 's/^[[:space:]]*\([^[[:space:]]*\).*$/\1/' `
+
+kmk_cp -f "pt-baseline.txt" "${REPORT}"
+kmk_printf -- "\n" >> "${REPORT}"
+kmk_printf -- "%7s  %7s  %7s  Name\n" "VMMR0" "VMMGC" "VBoxVMM" >> "${REPORT}"
+kmk_printf -- "-------------------------------\n" >> "${REPORT}"
+kmk_printf -- "%7d  %7d  %7d  baseline\n" ${baseline_r0}  ${baseline_rc} ${baseline_r3} >> "${REPORT}"
+
+#
+# Now, do each of the names.
+#
+for name in $NAMES;
+do
+    kmk_sed \
+        -e '/'"${name}"'/s/EXPERIMENT_PURE/__attribute__((pure))/' \
+        -e 's/EXPERIMENT_PURE//' \
+        ${HEADER}.bak --output ${HEADER}
+    kmk KBUILD_TYPE=release VBoxVMM VMMR0 VMMGC
+    size ${BINDIR}/VMMR0.r0 ${BINDIR}/VMMGC.gc ${BINDIR}/VBoxVMM.${DLLEXT} > "pt-${name}.txt"
+
+    exec < "pt-${name}.txt"
+    read buf                                # ignore
+    read buf; cur_r0=`echo $buf | kmk_sed -e 's/^[[:space:]]*\([^[[:space:]]*\).*$/\1/' `
+    read buf; cur_rc=`echo $buf | kmk_sed -e 's/^[[:space:]]*\([^[[:space:]]*\).*$/\1/' `
+    read buf; cur_r3=`echo $buf | kmk_sed -e 's/^[[:space:]]*\([^[[:space:]]*\).*$/\1/' `
+    kmk_printf -- "%7d  %7d  %7d  ${name}\n"  \
+             `kmk_expr ${baseline_r0} - ${cur_r0} ` \
+             `kmk_expr ${baseline_rc} - ${cur_rc} ` \
+             `kmk_expr ${baseline_r3} - ${cur_r3} ` \
+        >> "${REPORT}"
+done
+
+# clean up
+kmk_mv -f ${HEADER}.bak ${HEADER}
+
+
+
diff --git a/src/VBox/VMM/testcase/Instructions/InstructionTestGen.py b/src/VBox/VMM/testcase/Instructions/InstructionTestGen.py
new file mode 100755
index 00000000..16c57a67
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/InstructionTestGen.py
@@ -0,0 +1,2239 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id: InstructionTestGen.py $
+
+"""
+Instruction Test Generator.
+"""
+
+from __future__ import print_function;
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2020 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+"""
+__version__ = "$Revision: 135976 $";
+
+
+# pylint: disable=C0103,R0913
+
+
+# Standard python imports.
+import io;
+import os;
+from optparse import OptionParser
+import random;
+import sys;
+
+
+## @name Exit codes
+## @{
+RTEXITCODE_SUCCESS = 0;
+RTEXITCODE_SYNTAX  = 2;
+## @}
+
+## @name Various C macros we're used to.
+## @{
+UINT8_MAX  = 0xff
+UINT16_MAX = 0xffff
+UINT32_MAX = 0xffffffff
+UINT64_MAX = 0xffffffffffffffff
+def RT_BIT_32(iBit): # pylint: disable=C0103
+    """ 32-bit one bit mask. """
+    return 1 << iBit;
+def RT_BIT_64(iBit): # pylint: disable=C0103
+    """ 64-bit one bit mask. """
+    return 1 << iBit;
+## @}
+
+
+## @name ModR/M
+## @{
+X86_MODRM_RM_MASK   = 0x07;
+X86_MODRM_REG_MASK  = 0x38;
+X86_MODRM_REG_SMASK = 0x07;
+X86_MODRM_REG_SHIFT = 3;
+X86_MODRM_MOD_MASK  = 0xc0;
+X86_MODRM_MOD_SMASK = 0x03;
+X86_MODRM_MOD_SHIFT = 6;
+## @}
+
+## @name SIB
+## @{
+X86_SIB_BASE_MASK   = 0x07;
+X86_SIB_INDEX_MASK  = 0x38;
+X86_SIB_INDEX_SMASK = 0x07;
+X86_SIB_INDEX_SHIFT = 3;
+X86_SIB_SCALE_MASK  = 0xc0;
+X86_SIB_SCALE_SMASK = 0x03;
+X86_SIB_SCALE_SHIFT = 6;
+## @}
+
+## @name Prefixes
+## @
+X86_OP_PRF_CS       = 0x2e;
+X86_OP_PRF_SS       = 0x36;
+X86_OP_PRF_DS       = 0x3e;
+X86_OP_PRF_ES       = 0x26;
+X86_OP_PRF_FS       = 0x64;
+X86_OP_PRF_GS       = 0x65;
+X86_OP_PRF_SIZE_OP  = 0x66;
+X86_OP_PRF_SIZE_ADDR = 0x67;
+X86_OP_PRF_LOCK     = 0xf0;
+X86_OP_PRF_REPNZ    = 0xf2;
+X86_OP_PRF_REPZ     = 0xf3;
+X86_OP_REX_B        = 0x41;
+X86_OP_REX_X        = 0x42;
+X86_OP_REX_R        = 0x44;
+X86_OP_REX_W        = 0x48;
+## @}
+
+
+## @name General registers
+## @
+X86_GREG_xAX        = 0
+X86_GREG_xCX        = 1
+X86_GREG_xDX        = 2
+X86_GREG_xBX        = 3
+X86_GREG_xSP        = 4
+X86_GREG_xBP        = 5
+X86_GREG_xSI        = 6
+X86_GREG_xDI        = 7
+X86_GREG_x8         = 8
+X86_GREG_x9         = 9
+X86_GREG_x10        = 10
+X86_GREG_x11        = 11
+X86_GREG_x12        = 12
+X86_GREG_x13        = 13
+X86_GREG_x14        = 14
+X86_GREG_x15        = 15
+## @}
+
+
+## @name Register names.
+## @{
+g_asGRegs64NoSp = ('rax', 'rcx', 'rdx', 'rbx', None,  'rbp', 'rsi', 'rdi', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15');
+g_asGRegs64     = ('rax', 'rcx', 'rdx', 'rbx', 'rsp', 'rbp', 'rsi', 'rdi', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15');
+g_asGRegs32NoSp = ('eax', 'ecx', 'edx', 'ebx', None,  'ebp', 'esi', 'edi',
+                   'r8d', 'r9d', 'r10d', 'r11d', 'r12d', 'r13d', 'r14d', 'r15d');
+g_asGRegs32     = ('eax', 'ecx', 'edx', 'ebx', 'esp', 'ebp', 'esi', 'edi',
+                   'r8d', 'r9d', 'r10d', 'r11d', 'r12d', 'r13d', 'r14d', 'r15d');
+g_asGRegs16NoSp = ('ax',  'cx',  'dx',  'bx',  None,  'bp',  'si',  'di',
+                   'r8w', 'r9w', 'r10w', 'r11w', 'r12w', 'r13w', 'r14w', 'r15w');
+g_asGRegs16     = ('ax',  'cx',  'dx',  'bx',  'sp',  'bp',  'si',  'di',
+                   'r8w', 'r9w', 'r10w', 'r11w', 'r12w', 'r13w', 'r14w', 'r15w');
+g_asGRegs8      = ('al',  'cl',  'dl',  'bl',  'ah',  'ch',  'dh',  'bh');
+g_asGRegs8Rex   = ('al',  'cl',  'dl',  'bl',  'spl', 'bpl', 'sil',  'dil',
+                   'r8b', 'r9b', 'r10b', 'r11b', 'r12b', 'r13b', 'r14b', 'r15b',
+                   'ah',  'ch',  'dh',  'bh');
+## @}
+
+## @name EFLAGS/RFLAGS/EFLAGS
+## @{
+X86_EFL_CF              = RT_BIT_32(0);
+X86_EFL_CF_BIT          = 0;
+X86_EFL_1               = RT_BIT_32(1);
+X86_EFL_PF              = RT_BIT_32(2);
+X86_EFL_AF              = RT_BIT_32(4);
+X86_EFL_AF_BIT          = 4;
+X86_EFL_ZF              = RT_BIT_32(6);
+X86_EFL_ZF_BIT          = 6;
+X86_EFL_SF              = RT_BIT_32(7);
+X86_EFL_SF_BIT          = 7;
+X86_EFL_TF              = RT_BIT_32(8);
+X86_EFL_IF              = RT_BIT_32(9);
+X86_EFL_DF              = RT_BIT_32(10);
+X86_EFL_OF              = RT_BIT_32(11);
+X86_EFL_OF_BIT          = 11;
+X86_EFL_IOPL            = (RT_BIT_32(12) | RT_BIT_32(13));
+X86_EFL_NT              = RT_BIT_32(14);
+X86_EFL_RF              = RT_BIT_32(16);
+X86_EFL_VM              = RT_BIT_32(17);
+X86_EFL_AC              = RT_BIT_32(18);
+X86_EFL_VIF             = RT_BIT_32(19);
+X86_EFL_VIP             = RT_BIT_32(20);
+X86_EFL_ID              = RT_BIT_32(21);
+X86_EFL_LIVE_MASK       = 0x003f7fd5;
+X86_EFL_RA1_MASK        = RT_BIT_32(1);
+X86_EFL_IOPL_SHIFT      = 12;
+X86_EFL_STATUS_BITS     = ( X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF | X86_EFL_OF );
+## @}
+
+## @name Random
+## @{
+g_iMyRandSeed = int((os.urandom(4)).encode('hex'), 16);
+#g_iMyRandSeed = 286523426;
+#g_iMyRandSeed = 1994382324;
+g_oMyRand = random.Random(g_iMyRandSeed);
+#g_oMyRand = random.SystemRandom();
+
+def randU8():
+    """ Unsigned 8-bit random number. """
+    return g_oMyRand.getrandbits(8);
+
+def randU16():
+    """ Unsigned 16-bit random number. """
+    return g_oMyRand.getrandbits(16);
+
+def randU32():
+    """ Unsigned 32-bit random number. """
+    return g_oMyRand.getrandbits(32);
+
+def randU64():
+    """ Unsigned 64-bit random number. """
+    return g_oMyRand.getrandbits(64);
+
+def randUxx(cBits):
+    """ Unsigned 8-, 16-, 32-, or 64-bit random number. """
+    return g_oMyRand.getrandbits(cBits);
+
+def randSxx(cBits):
+    """ Signed 8-, 16-, 32-, or 64-bit random number. """
+    uVal = randUxx(cBits);
+    iRet = uVal & ((1 << (cBits - 1)) - 1);
+    if iRet != uVal:
+        iRet = -iRet;
+    return iRet;
+
+def randUxxList(cBits, cElements):
+    """ List of unsigned 8-, 16-, 32-, or 64-bit random numbers. """
+    return [randUxx(cBits) for _ in range(cElements)];
+## @}
+
+
+
+
+## @name Instruction Emitter Helpers
+## @{
+
+def calcRexPrefixForTwoModRmRegs(iReg, iRm, bOtherRexPrefixes = 0):
+    """
+    Calculates a rex prefix if neccessary given the two registers
+    and optional rex size prefixes.
+    Returns an empty array if not necessary.
+    """
+    bRex = bOtherRexPrefixes;
+    if iReg >= 8:
+        bRex |= X86_OP_REX_R;
+    if iRm >= 8:
+        bRex |= X86_OP_REX_B;
+    if bRex == 0:
+        return [];
+    return [bRex,];
+
+def calcModRmForTwoRegs(iReg, iRm):
+    """
+    Calculate the RM byte for two registers.
+    Returns an array with one byte in it.
+    """
+    bRm = (0x3   << X86_MODRM_MOD_SHIFT) \
+        | ((iReg << X86_MODRM_REG_SHIFT) & X86_MODRM_REG_MASK) \
+        | (iRm   &  X86_MODRM_RM_MASK);
+    return [bRm,];
+
+## @}
+
+
+## @name Misc
+## @{
+
+def convU32ToSigned(u32):
+    """ Converts a 32-bit unsigned value to 32-bit signed. """
+    if u32 < 0x80000000:
+        return u32;
+    return u32 - UINT32_MAX - 1;
+
+def rotateLeftUxx(cBits, uVal, cShift):
+    """ Rotate a xx-bit wide unsigned number to the left. """
+    assert cShift < cBits;
+
+    if cBits == 16:
+        uMask = UINT16_MAX;
+    elif cBits == 32:
+        uMask = UINT32_MAX;
+    elif cBits == 64:
+        uMask = UINT64_MAX;
+    else:
+        assert cBits == 8;
+        uMask = UINT8_MAX;
+
+    uVal &= uMask;
+    uRet = (uVal << cShift) & uMask;
+    uRet |= (uVal >> (cBits - cShift));
+    return uRet;
+
+def rotateRightUxx(cBits, uVal, cShift):
+    """ Rotate a xx-bit wide unsigned number to the right. """
+    assert cShift < cBits;
+
+    if cBits == 16:
+        uMask = UINT16_MAX;
+    elif cBits == 32:
+        uMask = UINT32_MAX;
+    elif cBits == 64:
+        uMask = UINT64_MAX;
+    else:
+        assert cBits == 8;
+        uMask = UINT8_MAX;
+
+    uVal &= uMask;
+    uRet = (uVal >> cShift);
+    uRet |= (uVal << (cBits - cShift)) & uMask;
+    return uRet;
+
+def gregName(iReg, cBits, fRexByteRegs = True):
+    """ Gets the name of a general register by index and width. """
+    if cBits == 64:
+        return g_asGRegs64[iReg];
+    if cBits == 32:
+        return g_asGRegs32[iReg];
+    if cBits == 16:
+        return g_asGRegs16[iReg];
+    assert cBits == 8;
+    if fRexByteRegs:
+        return g_asGRegs8Rex[iReg];
+    return g_asGRegs8[iReg];
+
+## @}
+
+
+class TargetEnv(object):
+    """
+    Target Runtime Environment.
+    """
+
+    ## @name CPU Modes
+    ## @{
+    ksCpuMode_Real      = 'real';
+    ksCpuMode_Protect   = 'prot';
+    ksCpuMode_Paged     = 'paged';
+    ksCpuMode_Long      = 'long';
+    ksCpuMode_V86       = 'v86';
+    ## @}
+
+    ## @name Instruction set.
+    ## @{
+    ksInstrSet_16       = '16';
+    ksInstrSet_32       = '32';
+    ksInstrSet_64       = '64';
+    ## @}
+
+    def __init__(self, sName,
+                 sInstrSet = ksInstrSet_32,
+                 sCpuMode = ksCpuMode_Paged,
+                 iRing = 3,
+                 ):
+        self.sName          = sName;
+        self.sInstrSet      = sInstrSet;
+        self.sCpuMode       = sCpuMode;
+        self.iRing          = iRing;
+        self.asGRegs        = g_asGRegs64     if self.is64Bit() else g_asGRegs32;
+        self.asGRegsNoSp    = g_asGRegs64NoSp if self.is64Bit() else g_asGRegs32NoSp;
+
+    def isUsingIprt(self):
+        """ Whether it's an IPRT environment or not. """
+        return self.sName.startswith('iprt');
+
+    def is64Bit(self):
+        """ Whether it's a 64-bit environment or not. """
+        return self.sInstrSet == self.ksInstrSet_64;
+
+    def getDefOpBits(self):
+        """ Get the default operand size as a bit count. """
+        if self.sInstrSet == self.ksInstrSet_16:
+            return 16;
+        return 32;
+
+    def getDefOpBytes(self):
+        """ Get the default operand size as a byte count. """
+        return self.getDefOpBits() / 8;
+
+    def getMaxOpBits(self):
+        """ Get the max operand size as a bit count. """
+        if self.sInstrSet == self.ksInstrSet_64:
+            return 64;
+        return 32;
+
+    def getMaxOpBytes(self):
+        """ Get the max operand size as a byte count. """
+        return self.getMaxOpBits() / 8;
+
+    def getDefAddrBits(self):
+        """ Get the default address size as a bit count. """
+        if self.sInstrSet == self.ksInstrSet_16:
+            return 16;
+        if self.sInstrSet == self.ksInstrSet_32:
+            return 32;
+        return 64;
+
+    def getDefAddrBytes(self):
+        """ Get the default address size as a byte count. """
+        return self.getDefAddrBits() / 8;
+
+    def getGRegCount(self, cbEffBytes = 4):
+        """ Get the number of general registers. """
+        if self.sInstrSet == self.ksInstrSet_64:
+            if cbEffBytes == 1:
+                return 16 + 4;
+            return 16;
+        return 8;
+
+    def randGRegNoSp(self, cbEffBytes = 4):
+        """ Returns a random general register number, excluding the SP register. """
+        iReg = randU16() % self.getGRegCount(cbEffBytes);
+        while iReg == X86_GREG_xSP:
+            iReg = randU16() % self.getGRegCount(cbEffBytes);
+        return iReg;
+
+    def randGRegNoSpList(self, cItems, cbEffBytes = 4):
+        """ List of randGRegNoSp values. """
+        aiRegs = [];
+        for _ in range(cItems):
+            aiRegs.append(self.randGRegNoSp(cbEffBytes));
+        return aiRegs;
+
+    def getAddrModes(self):
+        """ Gets a list of addressing mode (16, 32, or/and 64). """
+        if self.sInstrSet == self.ksInstrSet_16:
+            return [16, 32];
+        if self.sInstrSet == self.ksInstrSet_32:
+            return [32, 16];
+        return [64, 32];
+
+    def is8BitHighGReg(self, cbEffOp, iGReg):
+        """ Checks if the given register is a high 8-bit general register (AH, CH, DH or BH). """
+        assert cbEffOp in [1, 2, 4, 8];
+        if cbEffOp == 1:
+            if iGReg >= 16:
+                return True;
+            if iGReg >= 4 and not self.is64Bit():
+                return True;
+        return False;
+
+    def gregNameBits(self, iReg, cBits):
+        """ Gets the name of the given register for the specified width (bits). """
+        return gregName(iReg, cBits, self.is64Bit());
+
+    def gregNameBytes(self, iReg, cbWidth):
+        """ Gets the name of the given register for the specified with (in bytes). """
+        return gregName(iReg, cbWidth * 8, self.is64Bit());
+
+
+
+
+## Target environments.
+g_dTargetEnvs = {
+    'iprt-r3-32':       TargetEnv('iprt-r3-32',     TargetEnv.ksInstrSet_32, TargetEnv.ksCpuMode_Protect, 3),
+    'iprt-r3-64':       TargetEnv('iprt-r3-64',     TargetEnv.ksInstrSet_64, TargetEnv.ksCpuMode_Long,    3),
+    'bs2-r0-64':        TargetEnv('bs2-r0-64',      TargetEnv.ksInstrSet_64, TargetEnv.ksCpuMode_Long,    0),
+    'bs2-r0-64-big':    TargetEnv('bs2-r0-64-big',  TargetEnv.ksInstrSet_64, TargetEnv.ksCpuMode_Long,    0),
+    'bs2-r0-32-big':    TargetEnv('bs2-r0-32-big',  TargetEnv.ksInstrSet_32, TargetEnv.ksCpuMode_Protect, 0),
+};
+
+
+class InstrTestBase(object):
+    """
+    Base class for testing one instruction.
+    """
+
+    def __init__(self, sName, sInstr = None):
+        self.sName = sName;
+        self.sInstr = sInstr if sInstr else sName.split()[0];
+
+    def isApplicable(self, oGen):
+        """
+        Tests if the instruction test is applicable to the selected environment.
+        """
+        _ = oGen;
+        return True;
+
+    def generateTest(self, oGen, sTestFnName):
+        """
+        Emits the test assembly code.
+        """
+        oGen.write(';; @todo not implemented. This is for the linter: %s, %s\n' % (oGen, sTestFnName));
+        return True;
+
+    def generateInputs(self, cbEffOp, cbMaxOp, oGen, fLong = False):
+        """ Generate a list of inputs. """
+        if fLong:
+            #
+            # Try do extremes as well as different ranges of random numbers.
+            #
+            auRet = [0, 1, ];
+            if cbMaxOp >= 1:
+                auRet += [ UINT8_MAX  / 2, UINT8_MAX  / 2 + 1, UINT8_MAX  ];
+            if cbMaxOp >= 2:
+                auRet += [ UINT16_MAX / 2, UINT16_MAX / 2 + 1, UINT16_MAX ];
+            if cbMaxOp >= 4:
+                auRet += [ UINT32_MAX / 2, UINT32_MAX / 2 + 1, UINT32_MAX ];
+            if cbMaxOp >= 8:
+                auRet += [ UINT64_MAX / 2, UINT64_MAX / 2 + 1, UINT64_MAX ];
+
+            if oGen.oOptions.sTestSize == InstructionTestGen.ksTestSize_Tiny:
+                for cBits, cValues in ( (8, 4), (16, 4), (32, 8), (64, 8) ):
+                    if cBits < cbMaxOp * 8:
+                        auRet += randUxxList(cBits, cValues);
+                cWanted = 16;
+            elif oGen.oOptions.sTestSize == InstructionTestGen.ksTestSize_Medium:
+                for cBits, cValues in ( (8, 8), (16, 8), (24, 2), (32, 16), (40, 1), (48, 1), (56, 1), (64, 16) ):
+                    if cBits < cbMaxOp * 8:
+                        auRet += randUxxList(cBits, cValues);
+                cWanted = 64;
+            else:
+                for cBits, cValues in ( (8, 16), (16, 16), (24, 4), (32, 64), (40, 4), (48, 4), (56, 4), (64, 64) ):
+                    if cBits < cbMaxOp * 8:
+                        auRet += randUxxList(cBits, cValues);
+                cWanted = 168;
+            if len(auRet) < cWanted:
+                auRet += randUxxList(cbEffOp * 8, cWanted - len(auRet));
+        else:
+            #
+            # Short list, just do some random numbers.
+            #
+            auRet = [];
+            if oGen.oOptions.sTestSize == InstructionTestGen.ksTestSize_Tiny:
+                auRet += randUxxList(cbMaxOp, 1);
+            elif oGen.oOptions.sTestSize == InstructionTestGen.ksTestSize_Medium:
+                auRet += randUxxList(cbMaxOp, 2);
+            else:
+                auRet = [];
+                for cBits in (8, 16, 32, 64):
+                    if cBits < cbMaxOp * 8:
+                        auRet += randUxxList(cBits, 1);
+        return auRet;
+
+
+class InstrTest_MemOrGreg_2_Greg(InstrTestBase):
+    """
+    Instruction reading memory or general register and writing the result to a
+    general register.
+    """
+
+    def __init__(self, sName, fnCalcResult, sInstr = None, acbOpVars = None):
+        InstrTestBase.__init__(self, sName, sInstr);
+        self.fnCalcResult = fnCalcResult;
+        self.acbOpVars    = [ 1, 2, 4, 8 ] if not acbOpVars else list(acbOpVars);
+        self.fTestRegForm = True;
+        self.fTestMemForm = True;
+
+    ## @name Test Instruction Writers
+    ## @{
+
+    def writeInstrGregGreg(self, cbEffOp, iOp1, iOp2, oGen):
+        """ Writes the instruction with two general registers as operands. """
+        oGen.write('        %s %s, %s\n'
+                   % ( self.sInstr, oGen.gregNameBytes(iOp1, cbEffOp), oGen.gregNameBytes(iOp2, cbEffOp),));
+        return True;
+
+    def writeInstrGregPureRM(self, cbEffOp, iOp1, cAddrBits, iOp2, iMod, offDisp, oGen):
+        """ Writes the instruction with two general registers as operands. """
+        oGen.write('        ');
+        if iOp2 == 13 and iMod == 0 and cAddrBits == 64:
+            oGen.write('altrexb '); # Alternative encoding for rip relative addressing.
+        oGen.write('%s %s, [' % (self.sInstr, oGen.gregNameBytes(iOp1, cbEffOp),));
+        if (iOp2 == 5 or iOp2 == 13) and iMod == 0:
+            oGen.write('VBINSTST_NAME(g_u%sData)' % (cbEffOp * 8,))
+            if oGen.oTarget.is64Bit():
+                oGen.write(' wrt rip');
+        else:
+            if iMod == 1:
+                oGen.write('byte %d + ' % (offDisp,));
+            elif iMod == 2:
+                oGen.write('dword %d + ' % (offDisp,));
+            else:
+                assert iMod == 0;
+
+            if cAddrBits == 64:
+                oGen.write(g_asGRegs64[iOp2]);
+            elif cAddrBits == 32:
+                oGen.write(g_asGRegs32[iOp2]);
+            elif cAddrBits == 16:
+                assert False; ## @todo implement 16-bit addressing.
+            else:
+                assert False, str(cAddrBits);
+
+        oGen.write(']\n');
+        return True;
+
+    def writeInstrGregSibLabel(self, cbEffOp, iOp1, cAddrBits, iBaseReg, iIndexReg, iScale, offDisp, oGen):
+        """ Writes the instruction taking a register and a label (base only w/o reg), SIB form. """
+        assert offDisp is None; assert iBaseReg in [5, 13]; assert iIndexReg == 4; assert cAddrBits != 16;
+        if cAddrBits == 64:
+            # Note! Cannot test this in 64-bit mode in any sensible way because the disp is 32-bit
+            #       and we cannot (yet) make assumtions about where we're loaded.
+            ## @todo Enable testing this in environments where we can make assumptions (boot sector).
+            oGen.write('        %s %s, [VBINSTST_NAME(g_u%sData) xWrtRIP]\n'
+                       % ( self.sInstr, oGen.gregNameBytes(iOp1, cbEffOp), cbEffOp * 8,));
+        else:
+            oGen.write('        altsibx%u %s %s, [VBINSTST_NAME(g_u%sData) xWrtRIP] ; iOp1=%s cbEffOp=%s\n'
+                       % ( iScale, self.sInstr, oGen.gregNameBytes(iOp1, cbEffOp), cbEffOp * 8, iOp1, cbEffOp));
+        return True;
+
+    def writeInstrGregSibScaledReg(self, cbEffOp, iOp1, cAddrBits, iBaseReg, iIndexReg, iScale, offDisp, oGen):
+        """ Writes the instruction taking a register and disp+scaled register (no base reg), SIB form. """
+        assert iBaseReg in [5, 13]; assert iIndexReg != 4;  assert cAddrBits != 16;
+        # Note! Using altsibxN to force scaled encoding. This is only really a
+        #       necessity for iScale=1, but doesn't hurt for the rest.
+        oGen.write('        altsibx%u %s %s, [%s * %#x'
+                   % (iScale, self.sInstr, oGen.gregNameBytes(iOp1, cbEffOp), oGen.gregNameBits(iIndexReg, cAddrBits), iScale,));
+        if offDisp is not None:
+            oGen.write(' + %#x' % (offDisp,));
+        oGen.write(']\n');
+        _ = iBaseReg;
+        return True;
+
+    def writeInstrGregSibBase(self, cbEffOp, iOp1, cAddrBits, iBaseReg, iIndexReg, iScale, offDisp, oGen):
+        """ Writes the instruction taking a register and base only (with reg), SIB form. """
+        oGen.write('        altsibx%u %s %s, [%s'
+                   % (iScale, self.sInstr, oGen.gregNameBytes(iOp1, cbEffOp), oGen.gregNameBits(iBaseReg, cAddrBits),));
+        if offDisp is not None:
+            oGen.write(' + %#x' % (offDisp,));
+        oGen.write(']\n');
+        _ = iIndexReg;
+        return True;
+
+    def writeInstrGregSibBaseAndScaledReg(self, cbEffOp, iOp1, cAddrBits, iBaseReg, iIndexReg, iScale, offDisp, oGen):
+        """ Writes tinstruction taking a register and full featured SIB form address. """
+        # Note! From the looks of things, yasm will encode the following instructions the same way:
+        #           mov eax, [rsi*1 + rbx]
+        #           mov eax, [rbx + rsi*1]
+        #       So, when there are two registers involved, the '*1' selects
+        #       which is index and which is base.
+        oGen.write('        %s %s, [%s + %s * %u'
+                   % ( self.sInstr, oGen.gregNameBytes(iOp1, cbEffOp),
+                       oGen.gregNameBits(iBaseReg, cAddrBits), oGen.gregNameBits(iIndexReg, cAddrBits), iScale,));
+        if offDisp is not None:
+            oGen.write(' + %#x' % (offDisp,));
+        oGen.write(']\n');
+        return True;
+
+    ## @}
+
+
+    ## @name Memory setups
+    ## @{
+
+    def generateMemSetupReadByLabel(self, oGen, cbEffOp, uInput):
+        """ Sets up memory for a memory read. """
+        oGen.pushConst(uInput);
+        oGen.write('        call    VBINSTST_NAME(Common_SetupMemReadU%u)\n' % (cbEffOp*8,));
+        return True;
+
+    def generateMemSetupReadByReg(self, oGen, cAddrBits, cbEffOp, iReg1, uInput, offDisp = None):
+        """ Sets up memory for a memory read indirectly addressed thru one register and optional displacement. """
+        oGen.pushConst(uInput);
+        oGen.write('        call    VBINSTST_NAME(%s)\n'
+                   % (oGen.needGRegMemSetup(cAddrBits, cbEffOp, iBaseReg = iReg1, offDisp = offDisp),));
+        oGen.write('        push    %s\n' % (oGen.oTarget.asGRegs[iReg1],));
+        return True;
+
+    def generateMemSetupReadByScaledReg(self, oGen, cAddrBits, cbEffOp, iIndexReg, iScale, uInput, offDisp = None):
+        """ Sets up memory for a memory read indirectly addressed thru one register and optional displacement. """
+        oGen.pushConst(uInput);
+        oGen.write('        call    VBINSTST_NAME(%s)\n'
+                   % (oGen.needGRegMemSetup(cAddrBits, cbEffOp, offDisp = offDisp, iIndexReg = iIndexReg, iScale = iScale),));
+        oGen.write('        push    %s\n' % (oGen.oTarget.asGRegs[iIndexReg],));
+        return True;
+
+    def generateMemSetupReadByBaseAndScaledReg(self, oGen, cAddrBits, cbEffOp, iBaseReg, iIndexReg, iScale, uInput, offDisp):
+        """ Sets up memory for a memory read indirectly addressed thru two registers with optional displacement. """
+        oGen.pushConst(uInput);
+        oGen.write('        call    VBINSTST_NAME(%s)\n'
+                   % (oGen.needGRegMemSetup(cAddrBits, cbEffOp, iBaseReg = iBaseReg, offDisp = offDisp,
+                                            iIndexReg = iIndexReg, iScale = iScale),));
+        oGen.write('        push    %s\n' % (oGen.oTarget.asGRegs[iIndexReg],));
+        oGen.write('        push    %s\n' % (oGen.oTarget.asGRegs[iBaseReg],));
+        return True;
+
+    def generateMemSetupPureRM(self, oGen, cAddrBits, cbEffOp, iOp2, iMod, uInput, offDisp = None):
+        """ Sets up memory for a pure R/M addressed read, iOp2 being the R/M value. """
+        oGen.pushConst(uInput);
+        assert offDisp is None or iMod != 0;
+        if (iOp2 != 5 and iOp2 != 13) or iMod != 0:
+            oGen.write('        call    VBINSTST_NAME(%s)\n'
+                       % (oGen.needGRegMemSetup(cAddrBits, cbEffOp, iOp2, offDisp),));
+        else:
+            oGen.write('        call    VBINSTST_NAME(Common_SetupMemReadU%u)\n' % (cbEffOp*8,));
+        oGen.write('        push    %s\n' % (oGen.oTarget.asGRegs[iOp2],));
+        return True;
+
+    ## @}
+
+    def generateOneStdTestGregGreg(self, oGen, cbEffOp, cbMaxOp, iOp1, iOp1X, iOp2, iOp2X, uInput, uResult):
+        """ Generate one standard instr greg,greg test. """
+        oGen.write('        call VBINSTST_NAME(Common_LoadKnownValues)\n');
+        oGen.write('        mov     %s, 0x%x\n' % (oGen.oTarget.asGRegs[iOp2X], uInput,));
+        if iOp1X != iOp2X:
+            oGen.write('        push    %s\n' % (oGen.oTarget.asGRegs[iOp2X],));
+        self.writeInstrGregGreg(cbEffOp, iOp1, iOp2, oGen);
+        oGen.pushConst(uResult);
+        oGen.write('        call VBINSTST_NAME(%s)\n' % (oGen.needGRegChecker(iOp1X, iOp2X if iOp1X != iOp2X else None),));
+        _ = cbMaxOp;
+        return True;
+
+    def generateOneStdTestGregGreg8BitHighPain(self, oGen, cbEffOp, cbMaxOp, iOp1, iOp2, uInput):
+        """ High 8-bit registers are a real pain! """
+        assert oGen.oTarget.is8BitHighGReg(cbEffOp, iOp1) or oGen.oTarget.is8BitHighGReg(cbEffOp, iOp2);
+        # Figure out the register indexes of the max op sized regs involved.
+        iOp1X = iOp1 & 3;
+        iOp2X = iOp2 & 3;
+        oGen.write('        ; iOp1=%u iOp1X=%u iOp2=%u iOp2X=%u\n' % (iOp1, iOp1X, iOp2, iOp2X,));
+
+        # Calculate unshifted result.
+        if iOp1X != iOp2X:
+            uCur = oGen.auRegValues[iOp1X];
+            if oGen.oTarget.is8BitHighGReg(cbEffOp, iOp1):
+                uCur = rotateRightUxx(cbMaxOp * 8, uCur, 8);
+        else:
+            uCur = uInput;
+            if oGen.oTarget.is8BitHighGReg(cbEffOp, iOp1) != oGen.oTarget.is8BitHighGReg(cbEffOp, iOp2):
+                if oGen.oTarget.is8BitHighGReg(cbEffOp, iOp1):
+                    uCur = rotateRightUxx(cbMaxOp * 8, uCur, 8);
+                else:
+                    uCur = rotateLeftUxx(cbMaxOp * 8, uCur, 8);
+        uResult = self.fnCalcResult(cbEffOp, uInput, uCur, oGen);
+
+
+        # Rotate the input and/or result to match their max-op-sized registers.
+        if oGen.oTarget.is8BitHighGReg(cbEffOp, iOp2):
+            uInput = rotateLeftUxx(cbMaxOp * 8, uInput, 8);
+        if oGen.oTarget.is8BitHighGReg(cbEffOp, iOp1):
+            uResult = rotateLeftUxx(cbMaxOp * 8, uResult, 8);
+
+        # Hand it over to an overridable worker method.
+        return self.generateOneStdTestGregGreg(oGen, cbEffOp, cbMaxOp, iOp1, iOp1X, iOp2, iOp2X, uInput, uResult);
+
+
+    def generateOneStdTestGregMemNoSib(self, oGen, cAddrBits, cbEffOp, cbMaxOp, iOp1, iOp2, uInput, uResult):
+        """ Generate mode 0, 1 and 2 test for the R/M=iOp2. """
+        if cAddrBits == 16:
+            _ = cbMaxOp;
+        else:
+            iMod = 0; # No disp, except for i=5.
+            oGen.write('        call    VBINSTST_NAME(Common_LoadKnownValues)\n');
+            self.generateMemSetupPureRM(oGen, cAddrBits, cbEffOp, iOp2, iMod, uInput);
+            self.writeInstrGregPureRM(cbEffOp, iOp1, cAddrBits, iOp2, iMod, None, oGen);
+            oGen.pushConst(uResult);
+            oGen.write('        call    VBINSTST_NAME(%s)\n' % (oGen.needGRegChecker(iOp1, iOp2),));
+
+            if iOp2 != 5 and iOp2 != 13:
+                iMod = 1;
+                for offDisp in oGen.getDispForMod(iMod):
+                    oGen.write('        call    VBINSTST_NAME(Common_LoadKnownValues)\n');
+                    self.generateMemSetupPureRM(oGen, cAddrBits, cbEffOp, iOp2, iMod, uInput, offDisp);
+                    self.writeInstrGregPureRM(cbEffOp, iOp1, cAddrBits, iOp2, iMod, offDisp, oGen);
+                    oGen.pushConst(uResult);
+                    oGen.write('        call    VBINSTST_NAME(%s)\n' % (oGen.needGRegChecker(iOp1, iOp2),));
+
+                iMod = 2;
+                for offDisp in oGen.getDispForMod(iMod):
+                    oGen.write('        call    VBINSTST_NAME(Common_LoadKnownValues)\n');
+                    self.generateMemSetupPureRM(oGen, cAddrBits, cbEffOp, iOp2, iMod, uInput, offDisp);
+                    self.writeInstrGregPureRM(cbEffOp, iOp1, cAddrBits, iOp2, iMod, offDisp, oGen);
+                    oGen.pushConst(uResult);
+                    oGen.write('        call    VBINSTST_NAME(%s)\n' % (oGen.needGRegChecker(iOp1, iOp2),));
+
+        return True;
+
+    def generateOneStdTestGregMemSib(self, oGen, cAddrBits, cbEffOp, cbMaxOp, iOp1, iMod, # pylint: disable=R0913
+                                     iBaseReg, iIndexReg, iScale, uInput, uResult):
+        """ Generate one SIB variations. """
+        for offDisp in oGen.getDispForMod(iMod, cbEffOp):
+            if ((iBaseReg == 5 or iBaseReg == 13) and iMod == 0):
+                if iIndexReg == 4:
+                    if cAddrBits == 64:
+                        continue; # skipping.
+                    oGen.write('        call    VBINSTST_NAME(Common_LoadKnownValues)\n');
+                    self.generateMemSetupReadByLabel(oGen, cbEffOp, uInput);
+                    self.writeInstrGregSibLabel(cbEffOp, iOp1, cAddrBits, iBaseReg, iIndexReg, iScale, offDisp, oGen);
+                    sChecker = oGen.needGRegChecker(iOp1);
+                else:
+                    oGen.write('        call    VBINSTST_NAME(Common_LoadKnownValues)\n');
+                    self.generateMemSetupReadByScaledReg(oGen, cAddrBits, cbEffOp, iIndexReg, iScale, uInput, offDisp);
+                    self.writeInstrGregSibScaledReg(cbEffOp, iOp1, cAddrBits, iBaseReg, iIndexReg, iScale, offDisp, oGen);
+                    sChecker = oGen.needGRegChecker(iOp1, iIndexReg);
+            else:
+                oGen.write('        call    VBINSTST_NAME(Common_LoadKnownValues)\n');
+                if iIndexReg == 4:
+                    self.generateMemSetupReadByReg(oGen, cAddrBits, cbEffOp, iBaseReg, uInput, offDisp);
+                    self.writeInstrGregSibBase(cbEffOp, iOp1, cAddrBits, iBaseReg, iIndexReg, iScale, offDisp, oGen);
+                    sChecker = oGen.needGRegChecker(iOp1, iBaseReg);
+                else:
+                    if iIndexReg == iBaseReg and iScale == 1 and offDisp is not None and (offDisp & 1):
+                        if offDisp < 0: offDisp += 1;
+                        else:           offDisp -= 1;
+                    self.generateMemSetupReadByBaseAndScaledReg(oGen, cAddrBits, cbEffOp, iBaseReg,
+                                                                iIndexReg, iScale, uInput, offDisp);
+                    self.writeInstrGregSibBaseAndScaledReg(cbEffOp, iOp1, cAddrBits, iBaseReg, iIndexReg, iScale, offDisp, oGen);
+                    sChecker = oGen.needGRegChecker(iOp1, iBaseReg, iIndexReg);
+            oGen.pushConst(uResult);
+            oGen.write('        call    VBINSTST_NAME(%s)\n' % (sChecker,));
+        _ = cbMaxOp;
+        return True;
+
+    def generateStdTestGregMemSib(self, oGen, cAddrBits, cbEffOp, cbMaxOp, iOp1, auInputs):
+        """ Generate all SIB variations for the given iOp1 (reg) value. """
+        assert cAddrBits in [32, 64];
+        i = oGen.cSibBasePerRun;
+        while i > 0:
+            oGen.iSibBaseReg = (oGen.iSibBaseReg + 1) % oGen.oTarget.getGRegCount(cAddrBits / 8);
+            if oGen.iSibBaseReg == X86_GREG_xSP: # no RSP testing atm.
+                continue;
+
+            j = oGen.getSibIndexPerRun();
+            while j > 0:
+                oGen.iSibIndexReg = (oGen.iSibIndexReg + 1) % oGen.oTarget.getGRegCount(cAddrBits / 8);
+                if oGen.iSibIndexReg == iOp1 and oGen.iSibIndexReg != 4 and cAddrBits != cbMaxOp:
+                    continue; # Don't know the high bit of the address ending up the result - skip it for now.
+
+                for iMod in [0, 1, 2]:
+                    if oGen.iSibBaseReg == iOp1 \
+                      and ((oGen.iSibBaseReg != 5 and oGen.iSibBaseReg != 13) or iMod != 0) \
+                      and cAddrBits != cbMaxOp:
+                        continue; # Don't know the high bit of the address ending up the result - skip it for now.
+
+                    for _ in oGen.oSibScaleRange:
+                        oGen.iSibScale *= 2;
+                        if oGen.iSibScale > 8:
+                            oGen.iSibScale = 1;
+
+                        for uInput in auInputs:
+                            oGen.newSubTest();
+                            uResult = self.fnCalcResult(cbEffOp, uInput, oGen.auRegValues[iOp1], oGen);
+                            self.generateOneStdTestGregMemSib(oGen, cAddrBits, cbEffOp, cbMaxOp, iOp1, iMod,
+                                                              oGen.iSibBaseReg, oGen.iSibIndexReg, oGen.iSibScale,
+                                                              uInput, uResult);
+                j -= 1;
+            i -= 1;
+
+        return True;
+
+
+    def generateStandardTests(self, oGen):
+        """ Generate standard tests. """
+
+        # Parameters.
+        cbDefOp       = oGen.oTarget.getDefOpBytes();
+        cbMaxOp       = oGen.oTarget.getMaxOpBytes();
+        auShortInputs = self.generateInputs(cbDefOp, cbMaxOp, oGen);
+        auLongInputs  = self.generateInputs(cbDefOp, cbMaxOp, oGen, fLong = True);
+        iLongOp1      = oGen.oTarget.randGRegNoSp();
+        iLongOp2      = oGen.oTarget.randGRegNoSp();
+
+        # Register tests
+        if self.fTestRegForm:
+            for cbEffOp in self.acbOpVars:
+                if cbEffOp > cbMaxOp:
+                    continue;
+                oOp2Range = range(oGen.oTarget.getGRegCount(cbEffOp));
+                if oGen.oOptions.sTestSize == InstructionTestGen.ksTestSize_Tiny:
+                    oOp2Range = [iLongOp2,];
+                oGen.write('; cbEffOp=%u\n' % (cbEffOp,));
+
+                for iOp1 in range(oGen.oTarget.getGRegCount(cbEffOp)):
+                    if iOp1 == X86_GREG_xSP:
+                        continue; # Cannot test xSP atm.
+                    for iOp2 in oOp2Range:
+                        if   (iOp2 >= 16 and iOp1 in range(4, 16)) \
+                          or (iOp1 >= 16 and iOp2 in range(4, 16)):
+                            continue; # Any REX encoding turns AH,CH,DH,BH regs into SPL,BPL,SIL,DIL.
+                        if iOp2 == X86_GREG_xSP:
+                            continue; # Cannot test xSP atm.
+
+                        oGen.write('; iOp2=%u cbEffOp=%u\n' % (iOp2, cbEffOp));
+                        for uInput in (auLongInputs if iOp1 == iLongOp1 and iOp2 == iLongOp2 else auShortInputs):
+                            oGen.newSubTest();
+                            if not oGen.oTarget.is8BitHighGReg(cbEffOp, iOp1) and not oGen.oTarget.is8BitHighGReg(cbEffOp, iOp2):
+                                uCur = oGen.auRegValues[iOp1 & 15] if iOp1 != iOp2 else uInput;
+                                uResult = self.fnCalcResult(cbEffOp, uInput, uCur, oGen);
+                                self.generateOneStdTestGregGreg(oGen, cbEffOp, cbMaxOp, iOp1, iOp1 & 15, iOp2, iOp2 & 15,
+                                                                uInput, uResult);
+                            else:
+                                self.generateOneStdTestGregGreg8BitHighPain(oGen, cbEffOp, cbMaxOp, iOp1, iOp2, uInput);
+
+        # Memory test.
+        if self.fTestMemForm:
+            for cAddrBits in oGen.oTarget.getAddrModes():
+                for cbEffOp in self.acbOpVars:
+                    if cbEffOp > cbMaxOp:
+                        continue;
+
+                    for _ in oGen.getModRegRange(cbEffOp):
+                        oGen.iModReg = (oGen.iModReg + 1) % oGen.oTarget.getGRegCount(cbEffOp);
+                        if oGen.iModReg == X86_GREG_xSP:
+                            continue; # Cannot test xSP atm.
+                        if oGen.iModReg > 15:
+                            continue; ## TODO AH,CH,DH,BH
+
+                        auInputs = auLongInputs if oGen.iModReg == iLongOp1 else auShortInputs;
+                        for _ in oGen.oModRmRange:
+                            oGen.iModRm = (oGen.iModRm + 1) % oGen.oTarget.getGRegCount(cAddrBits * 8);
+                            if oGen.iModRm != 4 or cAddrBits == 16:
+                                for uInput in auInputs:
+                                    oGen.newSubTest();
+                                    if oGen.iModReg == oGen.iModRm and oGen.iModRm != 5 \
+                                       and oGen.iModRm != 13 and cbEffOp != cbMaxOp:
+                                        continue; # Don't know the high bit of the address ending up the result - skip it for now.
+                                    uResult = self.fnCalcResult(cbEffOp, uInput, oGen.auRegValues[oGen.iModReg & 15], oGen);
+                                    self.generateOneStdTestGregMemNoSib(oGen, cAddrBits, cbEffOp, cbMaxOp,
+                                                                        oGen.iModReg, oGen.iModRm, uInput, uResult);
+                            else:
+                                # SIB - currently only short list of inputs or things may get seriously out of hand.
+                                self.generateStdTestGregMemSib(oGen, cAddrBits, cbEffOp, cbMaxOp, oGen.iModReg, auShortInputs);
+        return True;
+
+    def generateTest(self, oGen, sTestFnName):
+        oGen.write('VBINSTST_BEGINPROC %s\n' % (sTestFnName,));
+
+        self.generateStandardTests(oGen);
+
+        oGen.write('        ret\n');
+        oGen.write('VBINSTST_ENDPROC   %s\n' % (sTestFnName,));
+        return True;
+
+
+
+class InstrTest_Mov_Gv_Ev(InstrTest_MemOrGreg_2_Greg):
+    """
+    Tests MOV Gv,Ev.
+    """
+    def __init__(self):
+        InstrTest_MemOrGreg_2_Greg.__init__(self, 'mov Gv,Ev', self.calc_mov);
+
+    @staticmethod
+    def calc_mov(cbEffOp, uInput, uCur, oGen):
+        """ Calculates the result of a mov instruction."""
+        if cbEffOp == 8:
+            return uInput & UINT64_MAX;
+        if cbEffOp == 4:
+            return uInput & UINT32_MAX;
+        if cbEffOp == 2:
+            return (uCur & 0xffffffffffff0000) | (uInput & UINT16_MAX);
+        assert cbEffOp == 1; _ = oGen;
+        return (uCur & 0xffffffffffffff00) | (uInput & UINT8_MAX);
+
+
+class InstrTest_MovSxD_Gv_Ev(InstrTest_MemOrGreg_2_Greg):
+    """
+    Tests MOVSXD Gv,Ev.
+    """
+    def __init__(self):
+        InstrTest_MemOrGreg_2_Greg.__init__(self, 'movsxd Gv,Ev', self.calc_movsxd, acbOpVars = [ 8, 4, 2, ]);
+        self.fTestMemForm = False; # drop this...
+
+    def writeInstrGregGreg(self, cbEffOp, iOp1, iOp2, oGen):
+        """ Writes the instruction with two general registers as operands. """
+        if cbEffOp == 8:
+            oGen.write('        movsxd %s, %s\n'
+                       % ( oGen.gregNameBytes(iOp1, cbEffOp), oGen.gregNameBytes(iOp2, cbEffOp / 2),));
+        else:
+            oGen.write('        oddmovsxd %s, %s\n'
+                       % ( oGen.gregNameBytes(iOp1, cbEffOp), oGen.gregNameBytes(iOp2, cbEffOp),));
+        return True;
+
+    def isApplicable(self, oGen):
+        return oGen.oTarget.is64Bit();
+
+    @staticmethod
+    def calc_movsxd(cbEffOp, uInput, uCur, oGen):
+        """
+        Calculates the result of a movxsd instruction.
+        Returns the result value (cbMaxOp sized).
+        """
+        _ = oGen;
+        if cbEffOp == 8 and (uInput & RT_BIT_32(31)):
+            return (UINT32_MAX << 32) | (uInput & UINT32_MAX);
+        if cbEffOp == 2:
+            return (uCur & 0xffffffffffff0000) | (uInput & 0xffff);
+        return uInput & UINT32_MAX;
+
+
+class InstrTest_DivIDiv(InstrTestBase):
+    """
+    Tests IDIV and DIV instructions.
+    """
+
+    def __init__(self, fIsIDiv):
+        if not fIsIDiv:
+            InstrTestBase.__init__(self, 'div Gv,Ev', 'div');
+        else:
+            InstrTestBase.__init__(self, 'idiv Gv,Ev', 'idiv');
+        self.fIsIDiv = fIsIDiv;
+
+    def generateInputEdgeCases(self, cbEffOp, fLong, fXcpt):
+        """ Generate edge case inputs for cbEffOp. Returns a list of pairs, dividen + divisor. """
+        # Test params.
+        uStep = 1 << (cbEffOp * 8);
+        if self.fIsIDiv:
+            uStep /= 2;
+
+        # edge tests
+        auRet = [];
+
+        uDivisor  = 1 if fLong else 3;
+        uDividend = uStep * uDivisor - 1;
+        for i in range(5 if fLong else 3):
+            auRet.append([uDividend + fXcpt, uDivisor]);
+            if self.fIsIDiv:
+                auRet.append([-uDividend - fXcpt, -uDivisor]);
+                auRet.append([-(uDividend + uDivisor + fXcpt), uDivisor]);
+                auRet.append([ (uDividend + uDivisor + fXcpt), -uDivisor]);
+            if i <= 3 and fLong:
+                auRet.append([uDividend - 1 + fXcpt*3, uDivisor]);
+                if self.fIsIDiv:
+                    auRet.append([-(uDividend - 1 + fXcpt*3), -uDivisor]);
+            uDivisor += 1;
+            uDividend += uStep;
+
+        uDivisor  = uStep - 1;
+        uDividend = uStep * uDivisor - 1;
+        for _ in range(3 if fLong else 1):
+            auRet.append([uDividend + fXcpt, uDivisor]);
+            if self.fIsIDiv:
+                auRet.append([-uDividend - fXcpt, -uDivisor]);
+            uDivisor  -= 1;
+            uDividend -= uStep;
+
+        if self.fIsIDiv:
+            uDivisor = -uStep;
+            for _ in range(3 if fLong else 1):
+                auRet.append([uDivisor * (-uStep - 1) - (not fXcpt), uDivisor]);
+                uDivisor += 1
+            uDivisor = uStep - 1;
+            for _ in range(3 if fLong else 1):
+                auRet.append([-(uDivisor * (uStep + 1) - (not fXcpt)), uDivisor]);
+                uDivisor -= 1
+
+        return auRet;
+
+    def generateInputsNoXcpt(self, cbEffOp, fLong = False):
+        """ Generate inputs for cbEffOp. Returns a list of pairs, dividen + divisor. """
+        # Test params.
+        uStep = 1 << (cbEffOp * 8);
+        if self.fIsIDiv:
+            uStep /= 2;
+
+        # edge tests
+        auRet = self.generateInputEdgeCases(cbEffOp, fLong, False)
+
+        # random tests.
+        if self.fIsIDiv:
+            for _ in range(6 if fLong else 2):
+                while True:
+                    uDivisor  = randSxx(cbEffOp * 8);
+                    if uDivisor == 0 or uDivisor >= uStep or uDivisor < -uStep:
+                        continue;
+                    uDividend = randSxx(cbEffOp * 16);
+                    uResult = uDividend / uDivisor;
+                    if uResult >= uStep or uResult <= -uStep: # exclude difficulties
+                        continue;
+                    break;
+                auRet.append([uDividend, uDivisor]);
+        else:
+            for _ in range(6 if fLong else 2):
+                while True:
+                    uDivisor  = randUxx(cbEffOp * 8);
+                    if uDivisor == 0 or uDivisor >= uStep:
+                        continue;
+                    uDividend = randUxx(cbEffOp * 16);
+                    uResult = uDividend / uDivisor;
+                    if uResult >= uStep:
+                        continue;
+                    break;
+                auRet.append([uDividend, uDivisor]);
+
+        return auRet;
+
+    def generateOneStdTestGreg(self, oGen, cbEffOp, iOp2, iDividend, iDivisor):
+        """ Generate code of one '[I]DIV rDX:rAX,<GREG>' test. """
+        cbMaxOp   = oGen.oTarget.getMaxOpBytes();
+        fEffOp    = ((1 << (cbEffOp *8) ) - 1);
+        fMaxOp    = UINT64_MAX if cbMaxOp == 8 else UINT32_MAX; assert cbMaxOp in [8, 4];
+        fTopOp    = fMaxOp - fEffOp;
+        fFullOp1  = ((1 << (cbEffOp*16)) - 1);
+
+        uAX       = iDividend & fFullOp1;       # full with unsigned
+        uDX       = uAX >> (cbEffOp*8);
+        uAX      &= fEffOp;
+        uOp2Val   = iDivisor & fEffOp;
+
+        iQuotient = iDividend / iDivisor;
+        iReminder = iDividend % iDivisor;
+        if iReminder != 0 and iQuotient < 0:    # python has different rounding rules for negative division.
+            iQuotient += 1;
+            iReminder -= iDivisor;
+        uAXResult = iQuotient & fEffOp;
+        uDXResult = iReminder & fEffOp;
+
+        if cbEffOp < cbMaxOp:
+            uAX     |= randUxx(cbMaxOp * 8) & fTopOp;
+            uDX     |= randUxx(cbMaxOp * 8) & fTopOp;
+            uOp2Val |= randUxx(cbMaxOp * 8) & fTopOp;
+            if cbEffOp < 4:
+                uAXResult |= uAX & fTopOp;
+                uDXResult |= uDX & fTopOp;
+        oGen.write('        ; iDividend=%#x (%d) iDivisor=%#x (%d)\n'
+                   '        ; iQuotient=%#x (%d) iReminder=%#x (%d)\n'
+                   % ( iDividend & fFullOp1, iDividend, iDivisor & fEffOp, iDivisor,
+                       iQuotient & fEffOp, iQuotient, iReminder & fEffOp, iReminder, ));
+
+        oGen.write('        call VBINSTST_NAME(Common_LoadKnownValues)\n');
+        oGen.write('        mov     %s, 0x%x\n' % (oGen.oTarget.asGRegs[X86_GREG_xDX], uDX,));
+        oGen.write('        mov     %s, 0x%x\n' % (oGen.oTarget.asGRegs[X86_GREG_xAX], uAX,));
+        oGen.write('        mov     %s, 0x%x\n' % (oGen.oTarget.asGRegs[iOp2],         uOp2Val,));
+
+        oGen.write('        push    %s\n' % (oGen.oTarget.asGRegs[iOp2],));
+        oGen.pushConst(uDXResult);
+        oGen.pushConst(uAXResult);
+
+        oGen.write('        %-4s    %s\n' % (self.sInstr, oGen.gregNameBytes(iOp2, cbEffOp),));
+        oGen.write('        call VBINSTST_NAME(%s)\n' % (oGen.needGRegChecker(X86_GREG_xAX, X86_GREG_xDX, iOp2),));
+        return True;
+
+    def generateOneStdTestGreg8Bit(self, oGen, cbEffOp, iOp2, iDividend, iDivisor):
+        """ Generate code of one '[I]DIV AX,<GREG>' test (8-bit). """
+        cbMaxOp   = oGen.oTarget.getMaxOpBytes();
+        fMaxOp    = UINT64_MAX if cbMaxOp == 8 else UINT32_MAX; assert cbMaxOp in [8, 4];
+        iOp2X     = (iOp2 & 3) if oGen.oTarget.is8BitHighGReg(cbEffOp, iOp2) else iOp2;
+        assert iOp2X != X86_GREG_xAX;
+
+        uAX       = iDividend & UINT16_MAX;     # full with unsigned
+        uOp2Val   = iDivisor  & UINT8_MAX;
+
+        iQuotient = iDividend / iDivisor;
+        iReminder = iDividend % iDivisor;
+        if iReminder != 0 and iQuotient < 0:    # python has different rounding rules for negative division.
+            iQuotient += 1;
+            iReminder -= iDivisor;
+        uAXResult = (iQuotient & UINT8_MAX) | ((iReminder & UINT8_MAX) << 8);
+
+        uAX       |= randUxx(cbMaxOp * 8) & (fMaxOp - UINT16_MAX);
+        uAXResult |= uAX & (fMaxOp - UINT16_MAX);
+        uOp2Val   |= randUxx(cbMaxOp * 8) & (fMaxOp - UINT8_MAX);
+        if iOp2X != iOp2:
+            uOp2Val = rotateLeftUxx(cbMaxOp * 8, uOp2Val, 8);
+        oGen.write('        ; iDividend=%#x (%d) iDivisor=%#x (%d)\n'
+                   '        ; iQuotient=%#x (%d) iReminder=%#x (%d)\n'
+                   % ( iDividend & UINT16_MAX, iDividend, iDivisor & UINT8_MAX, iDivisor,
+                       iQuotient & UINT8_MAX, iQuotient, iReminder & UINT8_MAX, iReminder, ));
+
+        oGen.write('        call VBINSTST_NAME(Common_LoadKnownValues)\n');
+        oGen.write('        mov     %s, 0x%x\n' % (oGen.oTarget.asGRegs[X86_GREG_xAX], uAX,));
+        oGen.write('        mov     %s, 0x%x\n' % (oGen.oTarget.asGRegs[iOp2X], uOp2Val,));
+        oGen.write('        push    %s\n'       % (oGen.oTarget.asGRegs[iOp2X],));
+        oGen.pushConst(uAXResult);
+
+        oGen.write('        %-4s    %s\n' % (self.sInstr, oGen.gregNameBytes(iOp2, cbEffOp),));
+        oGen.write('        call VBINSTST_NAME(%s)\n' % (oGen.needGRegChecker(X86_GREG_xAX, iOp2X),));
+        return;
+
+
+    def generateStandardTests(self, oGen):
+        """ Generates test that causes no exceptions. """
+
+        # Parameters.
+        iLongOp2       = oGen.oTarget.randGRegNoSp();
+
+        # Register tests
+        if True:
+            for cbEffOp in  ( 8, 4, 2, 1 ):
+                if cbEffOp > oGen.oTarget.getMaxOpBytes():
+                    continue;
+                oGen.write('; cbEffOp=%u\n' % (cbEffOp,));
+                oOp2Range = range(oGen.oTarget.getGRegCount(cbEffOp));
+                if oGen.oOptions.sTestSize == InstructionTestGen.ksTestSize_Tiny:
+                    oOp2Range = [iLongOp2,];
+                for iOp2 in oOp2Range:
+                    if iOp2 == X86_GREG_xSP:
+                        continue; # Cannot test xSP atm.
+                    if iOp2 == X86_GREG_xAX or (cbEffOp > 1 and iOp2 == X86_GREG_xDX):
+                        continue; # Will overflow or be too complicated to get right.
+                    if cbEffOp == 1 and iOp2 == (16 if oGen.oTarget.is64Bit() else 4):
+                        continue; # Avoid dividing by AH, same reasons as above.
+
+                    for iDividend, iDivisor in self.generateInputsNoXcpt(cbEffOp, iOp2 == iLongOp2):
+                        oGen.newSubTest();
+                        if cbEffOp > 1:
+                            self.generateOneStdTestGreg(oGen, cbEffOp, iOp2, iDividend, iDivisor);
+                        else:
+                            self.generateOneStdTestGreg8Bit(oGen, cbEffOp, iOp2, iDividend, iDivisor);
+
+        ## Memory test.
+        #if False:
+        #    for cAddrBits in oGen.oTarget.getAddrModes():
+        #        for cbEffOp in self.acbOpVars:
+        #            if cbEffOp > cbMaxOp:
+        #                continue;
+        #
+        #            auInputs = auLongInputs if oGen.iModReg == iLongOp1 else auShortInputs;
+        #            for _ in oGen.oModRmRange:
+        #                oGen.iModRm = (oGen.iModRm + 1) % oGen.oTarget.getGRegCount(cAddrBits * 8);
+        #                if oGen.iModRm != 4 or cAddrBits == 16:
+        #                    for uInput in auInputs:
+        #                        oGen.newSubTest();
+        #                        if oGen.iModReg == oGen.iModRm and oGen.iModRm != 5 and oGen.iModRm != 13 and cbEffOp != cbMaxOp:
+        #                            continue; # Don't know the high bit of the address ending up the result - skip it for now.
+        #                        uResult = self.fnCalcResult(cbEffOp, uInput, oGen.auRegValues[oGen.iModReg & 15], oGen);
+        #                        self.generateOneStdTestGregMemNoSib(oGen, cAddrBits, cbEffOp, cbMaxOp,
+        #                                                            oGen.iModReg, oGen.iModRm, uInput, uResult);
+        #                else:
+        #                    # SIB - currently only short list of inputs or things may get seriously out of hand.
+        #                    self.generateStdTestGregMemSib(oGen, cAddrBits, cbEffOp, cbMaxOp, oGen.iModReg, auShortInputs);
+        #
+        return True;
+
+    def generateInputsXcpt(self, cbEffOp, fLong = False):
+        """
+        Generate inputs for cbEffOp that will overflow or underflow.
+        Returns a list of pairs, dividen + divisor.
+        """
+        # Test params.
+        uStep = 1 << (cbEffOp * 8);
+        if self.fIsIDiv:
+            uStep /= 2;
+
+        # edge tests
+        auRet = self.generateInputEdgeCases(cbEffOp, fLong, True);
+        auRet.extend([[0, 0], [1, 0], [ uStep * uStep / 2 - 1, 0]]);
+
+        # random tests.
+        if self.fIsIDiv:
+            for _ in range(6 if fLong else 2):
+                while True:
+                    uDivisor  = randSxx(cbEffOp * 8);
+                    uDividend = randSxx(cbEffOp * 16);
+                    if uDivisor >= uStep or uDivisor < -uStep:
+                        continue;
+                    if uDivisor != 0:
+                        uResult = uDividend / uDivisor;
+                        if (uResult <= uStep and uResult >= 0) or (uResult >= -uStep and uResult < 0):
+                            continue;  # exclude difficulties
+                    break;
+                auRet.append([uDividend, uDivisor]);
+        else:
+            for _ in range(6 if fLong else 2):
+                while True:
+                    uDivisor  = randUxx(cbEffOp * 8);
+                    uDividend = randUxx(cbEffOp * 16);
+                    if uDivisor >= uStep:
+                        continue;
+                    if uDivisor != 0:
+                        uResult = uDividend / uDivisor;
+                        if uResult < uStep:
+                            continue;
+                    break;
+                auRet.append([uDividend, uDivisor]);
+
+        return auRet;
+
+    def generateOneDivideErrorTestGreg(self, oGen, cbEffOp, iOp2, iDividend, iDivisor):
+        """ Generate code of one '[I]DIV rDX:rAX,<GREG>' test that causes #DE. """
+        cbMaxOp   = oGen.oTarget.getMaxOpBytes();
+        fEffOp    = ((1 << (cbEffOp *8) ) - 1);
+        fMaxOp    = UINT64_MAX if cbMaxOp == 8 else UINT32_MAX; assert cbMaxOp in [8, 4];
+        fTopOp    = fMaxOp - fEffOp;
+        fFullOp1  = ((1 << (cbEffOp*16)) - 1);
+
+        uAX       = iDividend & fFullOp1;       # full with unsigned
+        uDX       = uAX >> (cbEffOp*8);
+        uAX      &= fEffOp;
+        uOp2Val   = iDivisor & fEffOp;
+
+        if cbEffOp < cbMaxOp:
+            uAX     |= randUxx(cbMaxOp * 8) & fTopOp;
+            uDX     |= randUxx(cbMaxOp * 8) & fTopOp;
+            uOp2Val |= randUxx(cbMaxOp * 8) & fTopOp;
+        oGen.write('        ; iDividend=%#x (%d) iDivisor=%#x (%d)\n'
+                   % ( iDividend & fFullOp1, iDividend, iDivisor & fEffOp, iDivisor,));
+        oGen.write('        call VBINSTST_NAME(Common_LoadKnownValues)\n');
+        oGen.write('        mov     %s, 0x%x\n' % (oGen.oTarget.asGRegs[X86_GREG_xDX], uDX,));
+        oGen.write('        mov     %s, 0x%x\n' % (oGen.oTarget.asGRegs[X86_GREG_xAX], uAX,));
+        oGen.write('        mov     %s, 0x%x\n' % (oGen.oTarget.asGRegs[iOp2],         uOp2Val,));
+        oGen.write('        push    %s\n' % (oGen.oTarget.asGRegs[iOp2],));
+        oGen.write('        push    %s\n' % (oGen.oTarget.asGRegs[X86_GREG_xDX],));
+        oGen.write('        push    %s\n' % (oGen.oTarget.asGRegs[X86_GREG_xAX],));
+        oGen.write('        VBINSTST_TRAP_INSTR X86_XCPT_DE, 0, %-4s    %s\n'
+                   % (self.sInstr, oGen.gregNameBytes(iOp2, cbEffOp),));
+        oGen.write('        call VBINSTST_NAME(%s)\n' % (oGen.needGRegChecker(X86_GREG_xAX, X86_GREG_xDX, iOp2),));
+        return True;
+
+    def generateOneDivideErrorTestGreg8Bit(self, oGen, cbEffOp, iOp2, iDividend, iDivisor):
+        """ Generate code of one '[I]DIV AX,<GREG>' test that causes #DE (8-bit). """
+        if not oGen.oTarget.is64Bit() and iOp2 == 4: # Avoid AH.
+            iOp2 = 5;
+
+        cbMaxOp   = oGen.oTarget.getMaxOpBytes();
+        fMaxOp    = UINT64_MAX if cbMaxOp == 8 else UINT32_MAX; assert cbMaxOp in [8, 4];
+        iOp2X     = (iOp2 & 3) if oGen.oTarget.is8BitHighGReg(cbEffOp, iOp2) else iOp2;
+        assert iOp2X != X86_GREG_xAX;
+
+        uAX       = iDividend & UINT16_MAX;     # full with unsigned
+        uOp2Val   = iDivisor  & UINT8_MAX;
+
+        uAX       |= randUxx(cbMaxOp * 8) & (fMaxOp - UINT16_MAX);
+        uOp2Val   |= randUxx(cbMaxOp * 8) & (fMaxOp - UINT8_MAX);
+        if iOp2X != iOp2:
+            uOp2Val = rotateLeftUxx(cbMaxOp * 8, uOp2Val, 8);
+        oGen.write('        ; iDividend=%#x (%d) iDivisor=%#x (%d)\n'
+                   % ( iDividend & UINT16_MAX, iDividend, iDivisor & UINT8_MAX, iDivisor,));
+        oGen.write('        call VBINSTST_NAME(Common_LoadKnownValues)\n');
+        oGen.write('        mov     %s, 0x%x\n' % (oGen.oTarget.asGRegs[X86_GREG_xAX], uAX,));
+        oGen.write('        mov     %s, 0x%x\n' % (oGen.oTarget.asGRegs[iOp2X], uOp2Val,));
+        oGen.write('        push    %s\n'       % (oGen.oTarget.asGRegs[iOp2X],));
+        oGen.write('        push    sAX\n');
+        oGen.write('        VBINSTST_TRAP_INSTR X86_XCPT_DE, 0, %-4s    %s\n'
+                   % (self.sInstr, oGen.gregNameBytes(iOp2, cbEffOp),));
+        oGen.write('        call VBINSTST_NAME(%s)\n' % (oGen.needGRegChecker(X86_GREG_xAX, iOp2X),));
+        return;
+
+    def generateDivideErrorTests(self, oGen):
+        """ Generate divide error tests (raises X86_XCPT_DE). """
+        oGen.write('%ifdef VBINSTST_CAN_DO_TRAPS\n');
+
+        # We do one register variation here, assuming the standard test has got them covered.
+        # Register tests
+        if True:
+            iOp2 = oGen.oTarget.randGRegNoSp();
+            while iOp2 == X86_GREG_xAX or iOp2 == X86_GREG_xDX:
+                iOp2 = oGen.oTarget.randGRegNoSp();
+
+            for cbEffOp in ( 8, 4, 2, 1 ):
+                if cbEffOp > oGen.oTarget.getMaxOpBytes():
+                    continue;
+                oGen.write('; cbEffOp=%u iOp2=%u\n' % (cbEffOp, iOp2,));
+
+                for iDividend, iDivisor in self.generateInputsXcpt(cbEffOp, fLong = not oGen.isTiny()):
+                    oGen.newSubTest();
+                    if cbEffOp > 1:
+                        self.generateOneDivideErrorTestGreg(oGen, cbEffOp, iOp2, iDividend, iDivisor);
+                    else:
+                        self.generateOneDivideErrorTestGreg8Bit(oGen, cbEffOp, iOp2, iDividend, iDivisor);
+
+        oGen.write('%endif ; VBINSTST_CAN_DO_TRAPS\n');
+        return True;
+
+
+    def generateTest(self, oGen, sTestFnName):
+        oGen.write('VBINSTST_BEGINPROC %s\n' % (sTestFnName,));
+        #oGen.write('        int3\n');
+
+        self.generateStandardTests(oGen);
+        self.generateDivideErrorTests(oGen);
+
+        #oGen.write('        int3\n');
+        oGen.write('        ret\n');
+        oGen.write('VBINSTST_ENDPROC   %s\n' % (sTestFnName,));
+        return True;
+
+
+
+class InstrTest_DaaDas(InstrTestBase):
+    """ Tests the DAA and DAS instructions. """
+
+    def __init__(self, fIsDas):
+        InstrTestBase.__init__(self, 'das' if fIsDas else 'daa');
+        self.fIsDas = fIsDas;
+
+    def isApplicable(self, oGen):
+        return not oGen.oTarget.is64Bit();
+
+    def generateTest(self, oGen, sTestFnName):
+        if self.fIsDas: from itgTableDas import g_aItgDasResults as aItgResults;
+        else:           from itgTableDaa import g_aItgDaaResults as aItgResults;
+        cMax = len(aItgResults);
+        if oGen.isTiny():
+            cMax = 64;
+
+        oGen.write('VBINSTST_BEGINPROC %s\n' % (sTestFnName,));
+        oGen.write('        xor     ebx, ebx\n');
+        oGen.write('.das_loop:\n');
+        # Save the loop variable so we can load known values.
+        oGen.write('        push    ebx\n');
+        oGen.newSubTestEx('ebx');
+
+        # Push the results.
+        oGen.write('        movzx   eax, byte [.abAlResults + ebx]\n');
+        oGen.write('        or      eax, %#x\n' % (oGen.au32Regs[X86_GREG_xAX] & ~0xff,));
+        oGen.write('        push    eax\n');
+        oGen.write('        movzx   eax, byte [.aFlagsResults + ebx]\n');
+        oGen.write('        push    eax\n');
+        # Calc and push the inputs.
+        oGen.write('        mov     eax, ebx\n');
+        oGen.write('        shr     eax, 2\n');
+        oGen.write('        and     eax, 0ffh\n');
+        oGen.write('        or      eax, %#x\n' % (oGen.au32Regs[X86_GREG_xAX] & ~0xff,));
+        oGen.write('        push    eax\n');
+
+        oGen.write('        pushfd\n')
+        oGen.write('        and     dword [xSP], ~(X86_EFL_CF | X86_EFL_AF)\n');
+        oGen.write('        mov     al, bl\n');
+        oGen.write('        and     al, 2\n');
+        oGen.write('        shl     al, X86_EFL_AF_BIT - 1\n');
+        oGen.write('        or      [xSP], al\n');
+        oGen.write('        mov     al, bl\n');
+        oGen.write('        and     al, X86_EFL_CF\n');
+        oGen.write('        or      [xSP], al\n');
+
+        # Load register values and do the test.
+        oGen.write('        call VBINSTST_NAME(Common_LoadKnownValues)\n');
+        oGen.write('        popfd\n');
+        oGen.write('        pop     eax\n');
+        if self.fIsDas:
+            oGen.write('        das\n');
+        else:
+            oGen.write('        daa\n');
+
+        # Verify the results.
+        fFlagsToCheck = X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_ZF;
+        oGen.write('        call VBINSTST_NAME(%s)\n' % (oGen.needFlagsGRegChecker(fFlagsToCheck, X86_GREG_xAX),));
+
+        # Restore the loop variable and advance.
+        oGen.write('        pop     ebx\n');
+        oGen.write('        inc     ebx\n');
+        oGen.write('        cmp     ebx, %#x\n' % (cMax,));
+        oGen.write('        jb      .das_loop\n');
+
+        oGen.write('        ret\n');
+
+        oGen.write('.abAlResults:\n');
+        for i in range(cMax):
+            oGen.write('        db %#x\n' % (aItgResults[i][0],));
+
+        oGen.write('.aFlagsResults:\n');
+        for i in range(cMax):
+            oGen.write('        db %#x\n' % (aItgResults[i][1],));
+
+        oGen.write('VBINSTST_ENDPROC   %s\n' % (sTestFnName,));
+        return True;
+
+
+##
+# Instruction Tests.
+#
+g_aoInstructionTests = [
+    InstrTest_Mov_Gv_Ev(),
+    InstrTest_MovSxD_Gv_Ev(),
+    InstrTest_DivIDiv(fIsIDiv = False),
+    InstrTest_DivIDiv(fIsIDiv = True),
+    InstrTest_DaaDas(fIsDas = False),
+    InstrTest_DaaDas(fIsDas = True),
+];
+
+
+
+
+
+class InstructionTestGen(object): # pylint: disable=R0902
+    """
+    Instruction Test Generator.
+    """
+
+    ## @name Test size
+    ## @{
+    ksTestSize_Large  = 'large';
+    ksTestSize_Medium = 'medium';
+    ksTestSize_Tiny   = 'tiny';
+    ## @}
+    kasTestSizes = ( ksTestSize_Large, ksTestSize_Medium, ksTestSize_Tiny );
+
+    ## The prefix for the checker functions.
+    ksCheckerPrefix   = 'Common_Check_'
+
+
+    def __init__(self, oOptions):
+        self.oOptions = oOptions;
+        self.oTarget  = g_dTargetEnvs[oOptions.sTargetEnv];
+
+        # Calculate the number of output files.
+        self.cFiles  = 1;
+        if len(g_aoInstructionTests) > self.oOptions.cInstrPerFile:
+            self.cFiles = len(g_aoInstructionTests) / self.oOptions.cInstrPerFile;
+            if self.cFiles * self.oOptions.cInstrPerFile < len(g_aoInstructionTests):
+                self.cFiles += 1;
+
+        # Fix the known register values.
+        self.au64Regs       = randUxxList(64, 16);
+        self.au32Regs       = [(self.au64Regs[i] & UINT32_MAX) for i in range(8)];
+        self.au16Regs       = [(self.au64Regs[i] & UINT16_MAX) for i in range(8)];
+        self.auRegValues    = self.au64Regs if self.oTarget.is64Bit() else self.au32Regs;
+
+        # Declare state variables used while generating.
+        self.oFile          = sys.stderr;
+        self.iFile          = -1;
+        self.sFile          = '';
+        self._dCheckFns     = dict();
+        self._dMemSetupFns  = dict();
+        self._d64BitConsts  = dict();
+
+        # State variables used while generating test convenientely placed here (lazy bird)...
+        self.iModReg        = 0;
+        self.iModRm         = 0;
+        self.iSibBaseReg    = 0;
+        self.iSibIndexReg   = 0;
+        self.iSibScale      = 1;
+        if self.oOptions.sTestSize == InstructionTestGen.ksTestSize_Tiny:
+            self._oModRegRange     = range(2);
+            self._oModRegRange8    = range(2);
+            self.oModRmRange       = range(2);
+            self.cSibBasePerRun    = 1;
+            self._cSibIndexPerRun  = 2;
+            self.oSibScaleRange    = range(1);
+        elif self.oOptions.sTestSize == InstructionTestGen.ksTestSize_Medium:
+            self._oModRegRange     = range( 5 if self.oTarget.is64Bit() else 4);
+            self._oModRegRange8    = range( 6 if self.oTarget.is64Bit() else 4);
+            self.oModRmRange       = range(5);
+            self.cSibBasePerRun    = 5;
+            self._cSibIndexPerRun  = 4
+            self.oSibScaleRange    = range(2);
+        else:
+            self._oModRegRange     = range(16 if self.oTarget.is64Bit() else 8);
+            self._oModRegRange8    = range(20 if self.oTarget.is64Bit() else 8);
+            self.oModRmRange       = range(16 if self.oTarget.is64Bit() else 8);
+            self.cSibBasePerRun    = 8;
+            self._cSibIndexPerRun  = 9;
+            self.oSibScaleRange    = range(4);
+        self.iSibIndexRange = 0;
+
+
+    #
+    # Methods used by instruction tests.
+    #
+
+    def write(self, sText):
+        """ Writes to the current output file. """
+        return self.oFile.write(unicode(sText));
+
+    def writeln(self, sText):
+        """ Writes a line to the current output file. """
+        self.write(sText);
+        return self.write('\n');
+
+    def writeInstrBytes(self, abInstr):
+        """
+        Emits an instruction given as a sequence of bytes values.
+        """
+        self.write('        db %#04x' % (abInstr[0],));
+        for i in range(1, len(abInstr)):
+            self.write(', %#04x' % (abInstr[i],));
+        return self.write('\n');
+
+    def newSubTest(self):
+        """
+        Indicates that a new subtest has started.
+        """
+        self.write('        mov     dword [VBINSTST_NAME(g_uVBInsTstSubTestIndicator) xWrtRIP], __LINE__\n');
+        return True;
+
+    def newSubTestEx(self, sIndicator):
+        """
+        Indicates that a new subtest has started.
+        """
+        self.write('        mov     dword [VBINSTST_NAME(g_uVBInsTstSubTestIndicator) xWrtRIP], %s\n' % (sIndicator, ));
+        return True;
+
+    def needGRegChecker(self, iReg1, iReg2 = None, iReg3 = None):
+        """
+        Records the need for a given register checker function, returning its label.
+        """
+        if iReg2 is not None:
+            if iReg3 is not None:
+                sName = '%s_%s_%s' % (self.oTarget.asGRegs[iReg1], self.oTarget.asGRegs[iReg2], self.oTarget.asGRegs[iReg3],);
+            else:
+                sName = '%s_%s' % (self.oTarget.asGRegs[iReg1], self.oTarget.asGRegs[iReg2],);
+        else:
+            sName = '%s' % (self.oTarget.asGRegs[iReg1],);
+            assert iReg3 is None;
+
+        if sName in self._dCheckFns:
+            self._dCheckFns[sName] += 1;
+        else:
+            self._dCheckFns[sName]  = 1;
+
+        return self.ksCheckerPrefix + sName;
+
+    def needFlagsGRegChecker(self, fFlagsToCheck, iReg1, iReg2 = None, iReg3 = None):
+        """
+        Records the need for a given rFLAGS + register checker function, returning its label.
+        """
+        sWorkerName = self.needGRegChecker(iReg1, iReg2, iReg3);
+
+        sName = 'eflags_%#x_%s' % (fFlagsToCheck, sWorkerName[len(self.ksCheckerPrefix):]);
+        if sName in self._dCheckFns:
+            self._dCheckFns[sName] += 1;
+        else:
+            self._dCheckFns[sName]  = 1;
+
+        return self.ksCheckerPrefix + sName;
+
+    def needGRegMemSetup(self, cAddrBits, cbEffOp, iBaseReg = None, offDisp = None, iIndexReg = None, iScale = 1):
+        """
+        Records the need for a given register checker function, returning its label.
+        """
+        assert cAddrBits in [64, 32, 16];
+        assert cbEffOp   in [8, 4, 2, 1];
+        assert iScale    in [1, 2, 4, 8];
+
+        sName = '%ubit_U%u' % (cAddrBits, cbEffOp * 8,);
+        if iBaseReg is not None:
+            sName += '_%s' % (gregName(iBaseReg, cAddrBits),);
+        sName += '_x%u' % (iScale,);
+        if iIndexReg is not None:
+            sName += '_%s' % (gregName(iIndexReg, cAddrBits),);
+        if offDisp is not None:
+            sName += '_%#010x' % (offDisp & UINT32_MAX, );
+        if sName in self._dMemSetupFns:
+            self._dMemSetupFns[sName] += 1;
+        else:
+            self._dMemSetupFns[sName]  = 1;
+        return 'Common_MemSetup_' + sName;
+
+    def need64BitConstant(self, uVal):
+        """
+        Records the need for a 64-bit constant, returning its label.
+        These constants are pooled to attempt reduce the size of the whole thing.
+        """
+        assert uVal >= 0 and uVal <= UINT64_MAX;
+        if uVal in self._d64BitConsts:
+            self._d64BitConsts[uVal] += 1;
+        else:
+            self._d64BitConsts[uVal]  = 1;
+        return 'g_u64Const_0x%016x' % (uVal, );
+
+    def pushConst(self, uResult):
+        """
+        Emits a push constant value, taking care of high values on 64-bit hosts.
+        """
+        if self.oTarget.is64Bit() and uResult >= 0x80000000:
+            self.write('        push    qword [%s wrt rip]\n' % (self.need64BitConstant(uResult),));
+        else:
+            self.write('        push    dword 0x%x\n' % (uResult,));
+        return True;
+
+    def getDispForMod(self, iMod, cbAlignment = 1):
+        """
+        Get a set of address dispositions for a given addressing mode.
+        The alignment restriction is for SIB scaling.
+        """
+        assert cbAlignment in [1, 2, 4, 8];
+        if iMod == 0:
+            aoffDisp = [ None, ];
+        elif iMod == 1:
+            aoffDisp = [ 127 & ~(cbAlignment - 1), -128 ];
+        elif iMod == 2:
+            aoffDisp = [ 2147483647 & ~(cbAlignment - 1), -2147483648 ];
+        else: assert False;
+        return aoffDisp;
+
+    def getModRegRange(self, cbEffOp):
+        """
+        The Mod R/M register range varies with the effective operand size, for
+        8-bit registers we have 4 more.
+        """
+        if cbEffOp == 1:
+            return self._oModRegRange8;
+        return self._oModRegRange;
+
+    def getSibIndexPerRun(self):
+        """
+        We vary the SIB index test range a little to try cover more operand
+        combinations and avoid repeating the same ones.
+        """
+        self.iSibIndexRange += 1;
+        self.iSibIndexRange %= 3;
+        if self.iSibIndexRange == 0:
+            return self._cSibIndexPerRun - 1;
+        return self._cSibIndexPerRun;
+
+    def isTiny(self):
+        """ Checks if we're in tiny mode."""
+        return self.oOptions.sTestSize == InstructionTestGen.ksTestSize_Tiny;
+
+    def isMedium(self):
+        """ Checks if we're in medium mode."""
+        return self.oOptions.sTestSize == InstructionTestGen.ksTestSize_Medium;
+
+
+    #
+    # Forwarding calls for oTarget to shorted typing and lessen the attacks
+    # on the right margin.
+    #
+
+    def gregNameBits(self, iReg, cBitsWide):
+        """ Target: Get the name of a general register for the given size (in bits). """
+        return self.oTarget.gregNameBits(iReg, cBitsWide);
+
+    def gregNameBytes(self, iReg, cbWide):
+        """ Target: Get the name of a general register for the given size (in bytes). """
+        return self.oTarget.gregNameBytes(iReg, cbWide);
+
+    def is64Bit(self):
+        """ Target: Is the target 64-bit? """
+        return self.oTarget.is64Bit();
+
+
+    #
+    # Internal machinery.
+    #
+
+    def _randInitIndexes(self):
+        """
+        Initializes the Mod R/M and SIB state index with random numbers prior
+        to generating a test.
+
+        Note! As with all other randomness and variations we do, we cannot
+              test all combinations for each and every instruction so we try
+              get coverage over time.
+        """
+        self.iModReg        = randU8();
+        self.iModRm         = randU8();
+        self.iSibBaseReg    = randU8();
+        self.iSibIndexReg   = randU8();
+        self.iSibScale      = 1 << (randU8() & 3);
+        self.iSibIndexRange = randU8();
+        return True;
+
+    def _calcTestFunctionName(self, oInstrTest, iInstrTest):
+        """
+        Calc a test function name for the given instruction test.
+        """
+        sName = 'TestInstr%03u_%s' % (iInstrTest, oInstrTest.sName);
+        return sName.replace(',', '_').replace(' ', '_').replace('%', '_');
+
+    def _generateFileHeader(self, ):
+        """
+        Writes the file header.
+        Raises exception on trouble.
+        """
+        self.write('; $Id: InstructionTestGen.py $\n'
+                   ';; @file %s\n'
+                   '; Autogenerate by %s %s. DO NOT EDIT\n'
+                   ';\n'
+                   '\n'
+                   ';\n'
+                   '; Headers\n'
+                   ';\n'
+                   '%%include "env-%s.mac"\n'
+                   % ( os.path.basename(self.sFile),
+                       os.path.basename(__file__), __version__[11:-1],
+                       self.oTarget.sName,
+                   ) );
+        # Target environment specific init stuff.
+
+        #
+        # Global variables.
+        #
+        self.write('\n\n'
+                   ';\n'
+                   '; Globals\n'
+                   ';\n');
+        self.write('VBINSTST_BEGINDATA\n'
+                   'VBINSTST_GLOBALNAME_EX g_pvLow16Mem4K, data hidden\n'
+                   '        dq      0\n'
+                   'VBINSTST_GLOBALNAME_EX g_pvLow32Mem4K, data hidden\n'
+                   '        dq      0\n'
+                   'VBINSTST_GLOBALNAME_EX g_pvMem4K, data hidden\n'
+                   '        dq      0\n'
+                   'VBINSTST_GLOBALNAME_EX g_uVBInsTstSubTestIndicator, data hidden\n'
+                   '        dd      0\n'
+                   '%ifdef VBINSTST_CAN_DO_TRAPS\n'
+                   'VBINSTST_TRAP_RECS_BEGIN\n'
+                   '%endif\n'
+                   'VBINSTST_BEGINCODE\n'
+                   );
+        self.write('%ifdef RT_ARCH_AMD64\n');
+        for i in range(len(g_asGRegs64)):
+            self.write('g_u64KnownValue_%s: dq 0x%x\n' % (g_asGRegs64[i], self.au64Regs[i]));
+        self.write('%endif\n\n')
+
+        #
+        # Common functions.
+        #
+
+        # Loading common values.
+        self.write('\n\n'
+                   'VBINSTST_BEGINPROC Common_LoadKnownValues\n'
+                   '%ifdef RT_ARCH_AMD64\n');
+        for i in range(len(g_asGRegs64NoSp)):
+            if g_asGRegs64NoSp[i]:
+                self.write('        mov     %s, 0x%x\n' % (g_asGRegs64NoSp[i], self.au64Regs[i],));
+        self.write('%else\n');
+        for i in range(8):
+            if g_asGRegs32NoSp[i]:
+                self.write('        mov     %s, 0x%x\n' % (g_asGRegs32NoSp[i], self.au32Regs[i],));
+        self.write('%endif\n'
+                   '        ret\n'
+                   'VBINSTST_ENDPROC   Common_LoadKnownValues\n'
+                   '\n');
+
+        self.write('VBINSTST_BEGINPROC Common_CheckKnownValues\n'
+                   '%ifdef RT_ARCH_AMD64\n');
+        for i in range(len(g_asGRegs64NoSp)):
+            if g_asGRegs64NoSp[i]:
+                self.write('        cmp     %s, [g_u64KnownValue_%s wrt rip]\n'
+                           '        je      .ok_%u\n'
+                           '        push    %u         ; register number\n'
+                           '        push    %s         ; actual\n'
+                           '        push    qword [g_u64KnownValue_%s wrt rip] ; expected\n'
+                           '        call    VBINSTST_NAME(Common_BadValue)\n'
+                           '.ok_%u:\n'
+                           % ( g_asGRegs64NoSp[i], g_asGRegs64NoSp[i], i, i, g_asGRegs64NoSp[i], g_asGRegs64NoSp[i], i,));
+        self.write('%else\n');
+        for i in range(8):
+            if g_asGRegs32NoSp[i]:
+                self.write('        cmp     %s, 0x%x\n'
+                           '        je      .ok_%u\n'
+                           '        push    %u         ; register number\n'
+                           '        push    %s         ; actual\n'
+                           '        push    dword 0x%x ; expected\n'
+                           '        call    VBINSTST_NAME(Common_BadValue)\n'
+                           '.ok_%u:\n'
+                           % ( g_asGRegs32NoSp[i], self.au32Regs[i], i, i, g_asGRegs32NoSp[i], self.au32Regs[i], i,));
+        self.write('%endif\n'
+                   '        ret\n'
+                   'VBINSTST_ENDPROC   Common_CheckKnownValues\n'
+                   '\n');
+
+        return True;
+
+    def _generateMemSetupFunctions(self): # pylint: disable=R0915
+        """
+        Generates the memory setup functions.
+        """
+        cDefAddrBits = self.oTarget.getDefAddrBits();
+        for sName in self._dMemSetupFns:
+            # Unpack it.
+            asParams = sName.split('_');
+            cAddrBits  = int(asParams[0][:-3]); assert asParams[0][-3:] == 'bit';
+            cEffOpBits = int(asParams[1][1:]);  assert asParams[1][0]   == 'U';
+            if cAddrBits == 64:   asAddrGRegs = g_asGRegs64;
+            elif cAddrBits == 32: asAddrGRegs = g_asGRegs32;
+            else:                 asAddrGRegs = g_asGRegs16;
+
+            i = 2;
+            iBaseReg = None;
+            sBaseReg = None;
+            if i < len(asParams) and asParams[i] in asAddrGRegs:
+                sBaseReg = asParams[i];
+                iBaseReg = asAddrGRegs.index(sBaseReg);
+                i += 1
+
+            assert i < len(asParams); assert asParams[i][0] == 'x';
+            iScale = iScale = int(asParams[i][1:]); assert iScale in [1, 2, 4, 8], '%u %s' % (iScale, sName);
+            i += 1;
+
+            sIndexReg = None;
+            iIndexReg = None;
+            if i < len(asParams) and asParams[i] in asAddrGRegs:
+                sIndexReg = asParams[i];
+                iIndexReg  = asAddrGRegs.index(sIndexReg);
+                i += 1;
+
+            u32Disp = None;
+            if i < len(asParams) and len(asParams[i]) == 10:
+                u32Disp = long(asParams[i], 16);
+                i += 1;
+
+            assert i == len(asParams), 'i=%d len=%d len[i]=%d (%s)' % (i, len(asParams), len(asParams[i]), asParams[i],);
+            assert iScale == 1 or iIndexReg is not None;
+
+            # Find a temporary register.
+            iTmpReg1 = X86_GREG_xCX;
+            while iTmpReg1 in [iBaseReg, iIndexReg]:
+                iTmpReg1 += 1;
+
+            # Prologue.
+            self.write('\n\n'
+                       '; cAddrBits=%s cEffOpBits=%s iBaseReg=%s u32Disp=%s iIndexReg=%s iScale=%s\n'
+                       'VBINSTST_BEGINPROC Common_MemSetup_%s\n'
+                       '        MY_PUSH_FLAGS\n'
+                       '        push    %s\n'
+                       % ( cAddrBits, cEffOpBits, iBaseReg, u32Disp, iIndexReg, iScale,
+                           sName, self.oTarget.asGRegs[iTmpReg1], ));
+
+            # Figure out what to use.
+            if cEffOpBits == 64:
+                sTmpReg1 = g_asGRegs64[iTmpReg1];
+                sDataVar = 'VBINSTST_NAME(g_u64Data)';
+            elif cEffOpBits == 32:
+                sTmpReg1 = g_asGRegs32[iTmpReg1];
+                sDataVar = 'VBINSTST_NAME(g_u32Data)';
+            elif cEffOpBits == 16:
+                sTmpReg1 = g_asGRegs16[iTmpReg1];
+                sDataVar = 'VBINSTST_NAME(g_u16Data)';
+            else:
+                assert cEffOpBits == 8; assert iTmpReg1 < 4;
+                sTmpReg1 = g_asGRegs8Rex[iTmpReg1];
+                sDataVar = 'VBINSTST_NAME(g_u8Data)';
+
+            # Special case: reg + reg * [2,4,8]
+            if iBaseReg == iIndexReg and iBaseReg is not None and iScale != 1:
+                iTmpReg2 = X86_GREG_xBP;
+                while iTmpReg2 in [iBaseReg, iIndexReg, iTmpReg1]:
+                    iTmpReg2 += 1;
+                sTmpReg2 = self.gregNameBits(iTmpReg2, cAddrBits);
+                self.write('        push    sAX\n'
+                           '        push    %s\n'
+                           '        push    sDX\n'
+                           % (self.oTarget.asGRegs[iTmpReg2],));
+                if cAddrBits == 16:
+                    self.write('        mov     %s, [VBINSTST_NAME(g_pvLow16Mem4K) xWrtRIP]\n' % (sTmpReg2,));
+                else:
+                    self.write('        mov     %s, [VBINSTST_NAME(g_pvLow32Mem4K) xWrtRIP]\n' % (sTmpReg2,));
+                self.write('        add     %s, 0x200\n' % (sTmpReg2,));
+                self.write('        mov     %s, %s\n' % (self.gregNameBits(X86_GREG_xAX, cAddrBits), sTmpReg2,));
+                if u32Disp is not None:
+                    self.write('        sub     %s, %d\n'
+                               % ( self.gregNameBits(X86_GREG_xAX, cAddrBits), convU32ToSigned(u32Disp), ));
+                self.write('        xor     edx, edx\n'
+                           '%if xCB == 2\n'
+                           '        push    0\n'
+                           '%endif\n');
+                self.write('        push    %u\n' % (iScale + 1,));
+                self.write('        div     %s [xSP]\n' % ('qword' if cAddrBits == 64 else 'dword',));
+                self.write('        sub     %s, %s\n' % (sTmpReg2, self.gregNameBits(X86_GREG_xDX, cAddrBits),));
+                self.write('        pop     sDX\n'
+                           '        pop     sDX\n'); # sTmpReg2 is eff address; sAX is sIndexReg value.
+                # Note! sTmpReg1 can be xDX and that's no problem now.
+                self.write('        mov     %s, [xSP + sCB*3 + MY_PUSH_FLAGS_SIZE + xCB]\n' % (sTmpReg1,));
+                self.write('        mov     [%s], %s\n' % (sTmpReg2, sTmpReg1,)); # Value in place.
+                self.write('        pop     %s\n' % (self.oTarget.asGRegs[iTmpReg2],));
+                if iBaseReg == X86_GREG_xAX:
+                    self.write('        pop     %s\n' % (self.oTarget.asGRegs[iTmpReg1],));
+                else:
+                    self.write('        mov     %s, %s\n' % (sBaseReg, self.gregNameBits(X86_GREG_xAX, cAddrBits),));
+                    self.write('        pop     sAX\n');
+
+            else:
+                # Load the value and mem address, storing the value there.
+                # Note! ASSUMES that the scale and disposition works fine together.
+                sAddrReg = sBaseReg if sBaseReg is not None else sIndexReg;
+                self.write('        mov     %s, [xSP + sCB + MY_PUSH_FLAGS_SIZE + xCB]\n' % (sTmpReg1,));
+                if cAddrBits >= cDefAddrBits:
+                    self.write('        mov     [%s xWrtRIP], %s\n' % (sDataVar, sTmpReg1,));
+                    self.write('        lea     %s, [%s xWrtRIP]\n' % (sAddrReg, sDataVar,));
+                else:
+                    if cAddrBits == 16:
+                        self.write('        mov     %s, [VBINSTST_NAME(g_pvLow16Mem4K) xWrtRIP]\n' % (sAddrReg,));
+                    else:
+                        self.write('        mov     %s, [VBINSTST_NAME(g_pvLow32Mem4K) xWrtRIP]\n' % (sAddrReg,));
+                    self.write('        add     %s, %s\n' % (sAddrReg, (randU16() << cEffOpBits) & 0xfff, ));
+                    self.write('        mov     [%s], %s\n' % (sAddrReg, sTmpReg1, ));
+
+                # Adjust for disposition and scaling.
+                if u32Disp is not None:
+                    self.write('        sub     %s, %d\n' % ( sAddrReg, convU32ToSigned(u32Disp), ));
+                if iIndexReg is not None:
+                    if iBaseReg == iIndexReg:
+                        assert iScale == 1;
+                        assert u32Disp is None or (u32Disp & 1) == 0;
+                        self.write('        shr     %s, 1\n' % (sIndexReg,));
+                    elif sBaseReg is not None:
+                        uIdxRegVal = randUxx(cAddrBits);
+                        if cAddrBits == 64:
+                            self.write('        mov     %s, %u\n'
+                                       '        sub     %s, %s\n'
+                                       '        mov     %s, %u\n'
+                                       % ( sIndexReg, (uIdxRegVal * iScale) & UINT64_MAX,
+                                           sBaseReg, sIndexReg,
+                                           sIndexReg, uIdxRegVal, ));
+                        else:
+                            assert cAddrBits == 32;
+                            self.write('        mov     %s, %u\n'
+                                       '        sub     %s, %#06x\n'
+                                       % ( sIndexReg, uIdxRegVal, sBaseReg, (uIdxRegVal * iScale) & UINT32_MAX, ));
+                    elif iScale == 2:
+                        assert u32Disp is None or (u32Disp & 1) == 0;
+                        self.write('        shr     %s, 1\n' % (sIndexReg,));
+                    elif iScale == 4:
+                        assert u32Disp is None or (u32Disp & 3) == 0;
+                        self.write('        shr     %s, 2\n' % (sIndexReg,));
+                    elif iScale == 8:
+                        assert u32Disp is None or (u32Disp & 7) == 0;
+                        self.write('        shr     %s, 3\n' % (sIndexReg,));
+                    else:
+                        assert iScale == 1;
+
+            # Set upper bits that's supposed to be unused.
+            if cDefAddrBits > cAddrBits or cAddrBits == 16:
+                if cDefAddrBits == 64:
+                    assert cAddrBits == 32;
+                    if iBaseReg is not None:
+                        self.write('        mov     %s, %#018x\n'
+                                   '        or      %s, %s\n'
+                                   % ( g_asGRegs64[iTmpReg1], randU64() & 0xffffffff00000000,
+                                       g_asGRegs64[iBaseReg], g_asGRegs64[iTmpReg1],));
+                    if iIndexReg is not None and iIndexReg != iBaseReg:
+                        self.write('        mov     %s, %#018x\n'
+                                   '        or      %s, %s\n'
+                                   % ( g_asGRegs64[iTmpReg1], randU64() & 0xffffffff00000000,
+                                       g_asGRegs64[iIndexReg], g_asGRegs64[iTmpReg1],));
+                else:
+                    assert cDefAddrBits == 32; assert cAddrBits == 16; assert iIndexReg is None;
+                    if iBaseReg is not None:
+                        self.write('        or      %s, %#010x\n'
+                                   % ( g_asGRegs32[iBaseReg], randU32() & 0xffff0000, ));
+
+            # Epilogue.
+            self.write('        pop     %s\n'
+                       '        MY_POP_FLAGS\n'
+                       '        ret sCB\n'
+                       'VBINSTST_ENDPROC   Common_MemSetup_%s\n'
+                       % ( self.oTarget.asGRegs[iTmpReg1], sName,));
+
+
+    def _generateFileFooter(self):
+        """
+        Generates file footer.
+        """
+
+        # Terminate the trap records.
+        self.write('\n\n'
+                   ';\n'
+                   '; Terminate the trap records\n'
+                   ';\n'
+                   'VBINSTST_BEGINDATA\n'
+                   '%ifdef VBINSTST_CAN_DO_TRAPS\n'
+                   'VBINSTST_TRAP_RECS_END\n'
+                   '%endif\n'
+                   'VBINSTST_BEGINCODE\n');
+
+        # Register checking functions.
+        for sName in self._dCheckFns:
+            asRegs = sName.split('_');
+            sPushSize = 'dword';
+
+            # Do we check eflags first.
+            if asRegs[0] == 'eflags':
+                asRegs.pop(0);
+                sFlagsToCheck = asRegs.pop(0);
+                self.write('\n\n'
+                           '; Check flags and then defers to the register-only checker\n'
+                           '; To save space, the callee cleans up the stack.'
+                           '; Ref count: %u\n'
+                           'VBINSTST_BEGINPROC %s%s\n'
+                           '        MY_PUSH_FLAGS\n'
+                           '        push    sAX\n'
+                           '        mov     sAX, [xSP + sCB]\n'
+                           '        and     sAX, %s\n'
+                           '        cmp     sAX, [xSP + xCB + sCB*2]\n'
+                           '        je      .equal\n'
+                           % ( self._dCheckFns[sName], self.ksCheckerPrefix, sName,
+                               sFlagsToCheck,));
+                self.write('        push    dword 0xef ; register number\n'
+                           '        push    sAX        ; actual\n'
+                           '        mov     sAX, [xSP + xCB + sCB*4]\n'
+                           '        push    sAX        ; expected\n'
+                           '        call    VBINSTST_NAME(Common_BadValue)\n');
+                self.write('.equal:\n'
+                           '        mov     xAX, [xSP + sCB*2]\n'  # Remove the expected eflags value from the stack frame.
+                           '        mov     [xSP + sCB*2  + xCB + sCB - xCB], xAX\n'
+                           '        pop     sAX\n'
+                           '        MY_POP_FLAGS\n'
+                           '        lea     xSP, [xSP + sCB]\n'
+                           '        jmp     VBINSTST_NAME(Common_Check_%s)\n'
+                           'VBINSTST_ENDPROC   %s%s\n'
+                           % ( '_'.join(asRegs),
+                               self.ksCheckerPrefix, sName,) );
+            else:
+                # Prologue
+                self.write('\n\n'
+                           '; Checks 1 or more register values, expected values pushed on the stack.\n'
+                           '; To save space, the callee cleans up the stack.'
+                           '; Ref count: %u\n'
+                           'VBINSTST_BEGINPROC %s%s\n'
+                           '        MY_PUSH_FLAGS\n'
+                           % ( self._dCheckFns[sName], self.ksCheckerPrefix, sName, ) );
+
+                # Register checks.
+                for i in range(len(asRegs)):
+                    sReg = asRegs[i];
+                    iReg = self.oTarget.asGRegs.index(sReg);
+                    if i == asRegs.index(sReg): # Only check once, i.e. input = output reg.
+                        self.write('        cmp     %s, [xSP + MY_PUSH_FLAGS_SIZE + xCB + sCB * %u]\n'
+                                   '        je      .equal%u\n'
+                                   '        push    %s %u      ; register number\n'
+                                   '        push    %s         ; actual\n'
+                                   '        mov     %s, [xSP + sCB*2 + MY_PUSH_FLAGS_SIZE + xCB + sCB * %u]\n'
+                                   '        push    %s         ; expected\n'
+                                   '        call    VBINSTST_NAME(Common_BadValue)\n'
+                                   '.equal%u:\n'
+                               % ( sReg, i, i, sPushSize, iReg, sReg, sReg, i, sReg, i, ) );
+
+
+                # Restore known register values and check the other registers.
+                for sReg in asRegs:
+                    if self.oTarget.is64Bit():
+                        self.write('        mov     %s, [g_u64KnownValue_%s wrt rip]\n' % (sReg, sReg,));
+                    else:
+                        iReg = self.oTarget.asGRegs.index(sReg)
+                        self.write('        mov     %s, 0x%x\n' % (sReg, self.au32Regs[iReg],));
+                self.write('        MY_POP_FLAGS\n'
+                           '        call    VBINSTST_NAME(Common_CheckKnownValues)\n'
+                           '        ret     sCB*%u\n'
+                           'VBINSTST_ENDPROC   %s%s\n'
+                           % (len(asRegs), self.ksCheckerPrefix, sName,));
+
+        # memory setup functions
+        self._generateMemSetupFunctions();
+
+        # 64-bit constants.
+        if len(self._d64BitConsts) > 0:
+            self.write('\n\n'
+                       ';\n'
+                       '; 64-bit constants\n'
+                       ';\n');
+            for uVal in self._d64BitConsts:
+                self.write('g_u64Const_0x%016x: dq 0x%016x ; Ref count: %d\n' % (uVal, uVal, self._d64BitConsts[uVal], ) );
+
+        return True;
+
+    def _generateTests(self):
+        """
+        Generate the test cases.
+        """
+        for self.iFile in range(self.cFiles):
+            if self.cFiles == 1:
+                self.sFile = '%s.asm' % (self.oOptions.sOutputBase,)
+            else:
+                self.sFile = '%s-%u.asm' % (self.oOptions.sOutputBase, self.iFile)
+            self.oFile = sys.stdout;
+            if self.oOptions.sOutputBase != '-':
+                self.oFile = io.open(self.sFile, 'w', buffering = 65536, encoding = 'utf-8');
+
+            self._generateFileHeader();
+
+            # Calc the range.
+            iInstrTestStart = self.iFile * self.oOptions.cInstrPerFile;
+            iInstrTestEnd   = iInstrTestStart + self.oOptions.cInstrPerFile;
+            if iInstrTestEnd > len(g_aoInstructionTests):
+                iInstrTestEnd = len(g_aoInstructionTests);
+
+            # Generate the instruction tests.
+            for iInstrTest in range(iInstrTestStart, iInstrTestEnd):
+                oInstrTest = g_aoInstructionTests[iInstrTest];
+                if oInstrTest.isApplicable(self):
+                    self.write('\n'
+                               '\n'
+                               ';\n'
+                               '; %s\n'
+                               ';\n'
+                               % (oInstrTest.sName,));
+                    self._randInitIndexes();
+                    oInstrTest.generateTest(self, self._calcTestFunctionName(oInstrTest, iInstrTest));
+
+            # Generate the main function.
+            self.write('\n\n'
+                       'VBINSTST_BEGINPROC TestInstrMain\n'
+                       '        MY_PUSH_ALL\n'
+                       '        sub xSP, 40h\n'
+                       '%ifdef VBINSTST_CAN_DO_TRAPS\n'
+                       '        VBINSTST_TRAP_RECS_INSTALL\n'
+                       '%endif\n'
+                       '\n');
+
+            for iInstrTest in range(iInstrTestStart, iInstrTestEnd):
+                oInstrTest = g_aoInstructionTests[iInstrTest];
+                if oInstrTest.isApplicable(self):
+                    self.write('%%ifdef ASM_CALL64_GCC\n'
+                               '        lea  rdi, [.szInstr%03u wrt rip]\n'
+                               '%%elifdef ASM_CALL64_MSC\n'
+                               '        lea  rcx, [.szInstr%03u wrt rip]\n'
+                               '%%else\n'
+                               '        mov  xAX, .szInstr%03u\n'
+                               '        mov  [xSP], xAX\n'
+                               '%%endif\n'
+                               '        VBINSTST_CALL_FN_SUB_TEST\n'
+                               '        call VBINSTST_NAME(%s)\n'
+                               % ( iInstrTest, iInstrTest, iInstrTest, self._calcTestFunctionName(oInstrTest, iInstrTest)));
+
+            self.write('\n'
+                       '%ifdef VBINSTST_CAN_DO_TRAPS\n'
+                       '        VBINSTST_TRAP_RECS_UNINSTALL\n'
+                       '%endif\n'
+                       '        add  xSP, 40h\n'
+                       '        MY_POP_ALL\n'
+                       '        ret\n\n');
+            for iInstrTest in range(iInstrTestStart, iInstrTestEnd):
+                self.write('.szInstr%03u: db \'%s\', 0\n' % (iInstrTest, g_aoInstructionTests[iInstrTest].sName,));
+            self.write('VBINSTST_ENDPROC TestInstrMain\n\n');
+
+            self._generateFileFooter();
+            if self.oOptions.sOutputBase != '-':
+                self.oFile.close();
+            self.oFile = None;
+        self.sFile = '';
+
+        return RTEXITCODE_SUCCESS;
+
+    def _runMakefileMode(self):
+        """
+        Generate a list of output files on standard output.
+        """
+        if self.cFiles == 1:
+            print('%s.asm' % (self.oOptions.sOutputBase,));
+        else:
+            print(' '.join('%s-%s.asm' % (self.oOptions.sOutputBase, i) for i in range(self.cFiles)));
+        return RTEXITCODE_SUCCESS;
+
+    def run(self):
+        """
+        Generates the tests or whatever is required.
+        """
+        if self.oOptions.fMakefileMode:
+            return self._runMakefileMode();
+        sys.stderr.write('InstructionTestGen.py: Seed = %s\n' % (g_iMyRandSeed,));
+        return self._generateTests();
+
+    @staticmethod
+    def main():
+        """
+        Main function a la C/C++. Returns exit code.
+        """
+
+        #
+        # Parse the command line.
+        #
+        oParser = OptionParser(version = __version__[11:-1].strip());
+        oParser.add_option('--makefile-mode', dest = 'fMakefileMode', action = 'store_true', default = False,
+                           help = 'Special mode for use to output a list of output files for the benefit of '
+                                  'the make program (kmk).');
+        oParser.add_option('--split', dest = 'cInstrPerFile', metavar = '<instr-per-file>', type = 'int', default = 9999999,
+                           help = 'Number of instruction to test per output file.');
+        oParser.add_option('--output-base', dest = 'sOutputBase', metavar = '<file>', default = None,
+                           help = 'The output file base name, no suffix please. Required.');
+        oParser.add_option('--target', dest = 'sTargetEnv', metavar = '<target>',
+                           default = 'iprt-r3-32',
+                           choices = g_dTargetEnvs.keys(),
+                           help = 'The target environment. Choices: %s'
+                                % (', '.join(sorted(g_dTargetEnvs.keys())),));
+        oParser.add_option('--test-size', dest = 'sTestSize', default = InstructionTestGen.ksTestSize_Medium,
+                           choices = InstructionTestGen.kasTestSizes,
+                           help = 'Selects the test size.');
+
+        (oOptions, asArgs) = oParser.parse_args();
+        if len(asArgs) > 0:
+            oParser.print_help();
+            return RTEXITCODE_SYNTAX
+        if oOptions.sOutputBase is None:
+            print('syntax error: Missing required option --output-base.', file = sys.stderr);
+            return RTEXITCODE_SYNTAX
+
+        #
+        # Instantiate the program class and run it.
+        #
+        oProgram = InstructionTestGen(oOptions);
+        return oProgram.run();
+
+
+if __name__ == '__main__':
+    sys.exit(InstructionTestGen.main());
+
diff --git a/src/VBox/VMM/testcase/Instructions/Makefile.kmk b/src/VBox/VMM/testcase/Instructions/Makefile.kmk
new file mode 100644
index 00000000..8b34f8cd
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/Makefile.kmk
@@ -0,0 +1,69 @@
+# $Id: Makefile.kmk $
+## @file
+# Sub-Makefile for the X86 and AMD64 Instruction Tests.
+#
+
+#
+# Copyright (C) 2006-2020 Oracle Corporation
+#
+# This file is part of VirtualBox Open Source Edition (OSE), as
+# available from http://www.virtualbox.org. This file is free software;
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License (GPL) as published by the Free Software
+# Foundation, in version 2 as it comes in the "COPYING" file of the
+# VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+# hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+#
+
+SUB_DEPTH = ../../../../..
+include $(KBUILD_PATH)/subheader.kmk
+
+#
+# Python linting (can't live without pylint!).
+#
+ifdef VBOX_WITH_PYLINT
+TESTING +=
+endif
+BLDDIRS += $(PATH_TARGET)/pylint
+
+define def_vbox_instructions_py_check
+$(eval name:=$(basename $(notdir $(py))))
+
+pylint:    $(name)-py-phony.o
+$(name).o: $(name)-py-phony.o
+$(PATH_TARGET)/pylint/$(name).o $(name)-py-phony.o:: $(py) | $(PATH_TARGET)/pylint/
+ifdef VBOX_WITH_PYLINT
+	$(QUIET2)$(call MSG_L1,Subjecting $(py) to pylint...)
+	$(QUIET)$(REDIRECT_EXT) -E LC_ALL=C -E PYTHONPATH="$(dir $(py))" -C $(dir $(py)) \
+		-- $$(VBOX_PYLINT) $$(VBOX_PYLINT_FLAGS) $$($(py)_VBOX_PYLINT_FLAGS) ./$(notdir $(py))
+endif
+	$(QUIET)$(APPEND) -t "$(PATH_TARGET)/pylint/$(name).o"
+
+TESTING += $(name)-py-phony.o
+endef # def_vbox_instructions_py_check
+
+
+$(foreach py, $(addprefix $(PATH_SUB_CURRENT)/, InstructionTestGen.py ) , $(eval $(def_vbox_instructions_py_check)))
+
+
+
+#
+# Ring-3 test program based on IPRT.
+#
+PROGRAMS += tstVBInsTstR3
+tstVBInsTstR3_TEMPLATE = VBOXR3TSTEXE
+tstVBInsTstR3_INCS = .
+tstVBInsTstR3_SOURCES = \
+	tstVBInsTstR3.cpp \
+       $(tstVBInsTstR3_0_OUTDIR)/tstVBInsTstR3A.asm
+tstVBInsTstR3_CLEAN = \
+       $(tstVBInsTstR3_0_OUTDIR)/tstVBInsTstR3A.asm
+
+$$(tstVBInsTstR3_0_OUTDIR)/tstVBInsTstR3A.asm: $(PATH_SUB_CURRENT)/InstructionTestGen.py
+	$(VBOX_BLD_PYTHON) $(PATH_SUB_CURRENT)/InstructionTestGen.py \
+		--target iprt-r3-$(if-expr $(intersects $(KBUILD_TARGET_ARCH), $(KBUILD_ARCHES_64)),64,32) \
+		--output-base $(basename $@)
+
+
+include $(FILE_KBUILD_SUB_FOOTER)
+
diff --git a/src/VBox/VMM/testcase/Instructions/env-bs2-r0-32-big.mac b/src/VBox/VMM/testcase/Instructions/env-bs2-r0-32-big.mac
new file mode 100644
index 00000000..7cc3a416
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/env-bs2-r0-32-big.mac
@@ -0,0 +1,35 @@
+; $Id: env-bs2-r0-32-big.mac $
+;; @file
+; Instruction Test Environment - Big Boot Sector Type 2, Ring-0, 64-Bit.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%undef  RT_ARCH_AMD64
+%undef  RT_ARCH_X86
+%undef  RT_ARCH_X86_32
+%undef  RT_ARCH_X86_16
+%undef  ASM_CALL64_MSC
+%undef  ASM_CALL64_GCC
+%undef  ASM_CALL64_BS2
+%undef  ARCH_BITS
+%undef  xWrtRIP
+
+%define ARCH_BITS       32
+%define RT_ARCH_X86
+%define ASM_CALL32_BS2
+%define xWrtRIP
+%define RTCCPTR_PRE     dword
+
+%include "env-bs2-r0-big.mac"
+
diff --git a/src/VBox/VMM/testcase/Instructions/env-bs2-r0-64-big.mac b/src/VBox/VMM/testcase/Instructions/env-bs2-r0-64-big.mac
new file mode 100644
index 00000000..c2f4275c
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/env-bs2-r0-64-big.mac
@@ -0,0 +1,35 @@
+; $Id: env-bs2-r0-64-big.mac $
+;; @file
+; Instruction Test Environment - Big Boot Sector Type 2, Ring-0, 64-Bit.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%undef  RT_ARCH_AMD64
+%undef  RT_ARCH_X86
+%undef  RT_ARCH_X86_32
+%undef  RT_ARCH_X86_16
+%undef  ASM_CALL64_MSC
+%undef  ASM_CALL64_GCC
+%undef  ASM_CALL64_BS2
+%undef  ARCH_BITS
+%undef  xWrtRIP
+
+%define ARCH_BITS       64
+%define RT_ARCH_AMD64
+%define ASM_CALL64_BS2
+%define xWrtRIP         wrt rip
+%define RTCCPTR_PRE     qword
+
+%include "env-bs2-r0-big.mac"
+
diff --git a/src/VBox/VMM/testcase/Instructions/env-bs2-r0-64.mac b/src/VBox/VMM/testcase/Instructions/env-bs2-r0-64.mac
new file mode 100644
index 00000000..ca54c801
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/env-bs2-r0-64.mac
@@ -0,0 +1,35 @@
+; $Id: env-bs2-r0-64.mac $
+;; @file
+; Instruction Test Environment - Boot Sector Type 2, Ring-0, 64-Bit.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%undef  RT_ARCH_AMD64
+%undef  RT_ARCH_X86
+%undef  RT_ARCH_X86_32
+%undef  RT_ARCH_X86_16
+%undef  ASM_CALL64_MSC
+%undef  ASM_CALL64_GCC
+%undef  ASM_CALL64_BS2
+%undef  ARCH_BITS
+%undef  xWrtRIP
+
+%define ARCH_BITS       64
+%define RT_ARCH_AMD64
+%define ASM_CALL64_BS2
+%define xWrtRIP         wrt rip
+%define RTCCPTR_PRE     qword
+
+%include "env-bs2-r0.mac"
+
diff --git a/src/VBox/VMM/testcase/Instructions/env-bs2-r0-big.mac b/src/VBox/VMM/testcase/Instructions/env-bs2-r0-big.mac
new file mode 100644
index 00000000..caa3b8d5
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/env-bs2-r0-big.mac
@@ -0,0 +1,57 @@
+; $Id: env-bs2-r0-big.mac $
+;; @file
+; Instruction Test Environment - Big Boot Sector Type 2, Ring-0.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%ifndef ___env_bs2_r0_big_mac
+%define ___env_bs2_r0_big_mac
+
+;
+; Include the BS2 API for BIG images.
+;
+%include "bootsector2-api.mac"
+
+
+;; Call RTTestISub like function.
+%define VBINSTST_CALL_FN_SUB_TEST   call [TMPL_NM_CMN(g_pfnTestSub) xWrtRIP]
+
+;; Call RTTestIFailure like function with simple message.
+%define VBINSTST_CALL_FN_FAILURE    call [TMPL_NM_CMN(g_pfnTestFailedF) xWrtRIP]
+
+;; Call RTTestIFailure like function with format message + 1 arg.
+%define VBINSTST_CALL_FN_FAILURE_1  call [TMPL_NM_CMN(g_pfnTestFailedF) xWrtRIP]
+
+;; Call RTTestIFailure like function with format message + 2 args.
+%define VBINSTST_CALL_FN_FAILURE_2  call [TMPL_NM_CMN(g_pfnTestFailedF) xWrtRIP]
+
+;; Call RTTestIFailure like function with format message + 3 args.
+%define VBINSTST_CALL_FN_FAILURE_3  call [TMPL_NM_CMN(g_pfnTestFailedF) xWrtRIP]
+
+;; Call RTTestIFailure like function with format message + 4 args.
+%define VBINSTST_CALL_FN_FAILURE_4  call [TMPL_NM_CMN(g_pfnTestFailedF) xWrtRIP]
+
+;; The image base label (used by the trap macros).
+%define VBINSTST_IMAGE_BASE_LABLE bs2_big_image_start
+
+;; Wrapper for calling TestInstallTrapRecs (used by the trap macros).
+%define VBINSTST_CALL_TEST_INSTALL_TRAP_RECS  call [TMPL_NM_CMN(g_pfnTestInstallTrapRecs) xWrtRIP]
+
+;
+; Include the common bits (contains code using above macros)
+;
+%include "env-bs2-r0-common.mac"
+
+%endif
+
diff --git a/src/VBox/VMM/testcase/Instructions/env-bs2-r0-common.mac b/src/VBox/VMM/testcase/Instructions/env-bs2-r0-common.mac
new file mode 100644
index 00000000..3b3bcf2d
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/env-bs2-r0-common.mac
@@ -0,0 +1,115 @@
+; $Id: env-bs2-r0-common.mac $
+;; @file
+; Instruction Test Environment - Boot Sector Type 2, Ring-0.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%ifndef ___env_bs2_r0_common_mac
+%define ___env_bs2_r0_common_mac
+
+
+;; Same as BEGINPROC in asmdefs.mac.
+%macro VBINSTST_BEGINPROC 1
+VBINSTST_GLOBALNAME_EX %1, function hidden
+%endm
+
+;; Same as ENDPROC in asmdefs.mac.
+%macro VBINSTST_ENDPROC 1,
+VBINSTST_GLOBALNAME_EX %1 %+ _EndProc, function hidden
+%endm
+
+;; Same as NAME in asmdefs.mac.
+%define VBINSTST_NAME(a_Name)   TMPL_NM(a_Name)
+
+;; Same as GLOBALNAME_EX in asmdefs.mac.
+%macro VBINSTST_GLOBALNAME_EX 2,
+VBINSTST_NAME(%1):
+%endmacro
+
+;; Same as BEGINCODE in asmdefs.mac.
+%macro VBINSTST_BEGINCODE 0,
+BEGINCODE
+%endmacro
+
+;; Same as BEGINDATA in asmdefs.mac.
+%macro VBINSTST_BEGINDATA 0,
+BEGINDATA
+%endmacro
+
+
+;
+; Trap related macros.
+;
+%define VBINSTST_CAN_DO_TRAPS 1
+
+%macro VBINSTST_TRAP_INSTR 3+,
+        section .traprecs
+    istruc BS2TRAPREC
+        at BS2TRAPREC.offWhere,           dd (%%trapinstr - VBINSTST_IMAGE_BASE_LABLE)
+        at BS2TRAPREC.offResumeAddend,    db (%%resume - %%trapinstr)
+        at BS2TRAPREC.u8TrapNo,           db %1
+        at BS2TRAPREC.u16ErrCd,           dw %2
+    iend
+        VBINSTST_BEGINCODE
+ %if %1 != X86_XCPT_BP
+  %%trapinstr:
+        %3
+ %else
+        %3
+  %%trapinstr:
+ %endif
+        call    VBINSTST_NAME(Common_MissingTrap_ %+ %1)
+ %%resume:
+%endmacro
+
+%macro VBINSTST_TRAP_RECS_BEGIN 0,
+        VBINSTST_BEGINDATA
+        section .traprecs progbits valign=8 vfollows=.data align=8 follows=.data
+        dq 0ffffffffeeeeeeeeh
+        dq 0ddddddddcccccccch
+VBINSTST_GLOBALNAME_EX g_aTrapRecs, hidden
+        VBINSTST_BEGINCODE
+%endmacro
+
+%macro VBINSTST_TRAP_RECS_END 0,
+        section .traprecs
+VBINSTST_GLOBALNAME_EX g_aTrapRecsEnd, hidden
+        dq 0ddddddddcccccccch
+        dq 0ffffffffeeeeeeeeh
+        VBINSTST_BEGINCODE
+%endmacro
+
+%macro VBINSTST_TRAP_RECS_INSTALL 0,
+        mov     sAX, VBINSTST_NAME(g_aTrapRecs)
+        mov     edx, VBINSTST_NAME(g_aTrapRecsEnd) - VBINSTST_NAME(g_aTrapRecs)
+        shr     edx, BS2TRAPREC_SIZE_SHIFT
+        mov     sCX, VBINSTST_IMAGE_BASE_LABLE
+        VBINSTST_CALL_TEST_INSTALL_TRAP_RECS
+%endmacro
+
+%macro VBINSTST_TRAP_RECS_UNINSTALL 0,
+        xor     sAX, sAX
+        xor     edx, edx
+        xor     sCX, sCX
+        VBINSTST_CALL_TEST_INSTALL_TRAP_RECS
+%endmacro
+
+
+;
+; Include the common bits (contains code using above macros)
+;
+%include "env-common.mac"
+
+%endif
+
diff --git a/src/VBox/VMM/testcase/Instructions/env-bs2-r0.mac b/src/VBox/VMM/testcase/Instructions/env-bs2-r0.mac
new file mode 100644
index 00000000..202754d0
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/env-bs2-r0.mac
@@ -0,0 +1,53 @@
+; $Id: env-bs2-r0.mac $
+;; @file
+; Instruction Test Environment - Boot Sector Type 2, Ring-0.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%ifndef ___env_bs2_r0_mac
+%define ___env_bs2_r0_mac
+
+
+;; Call RTTestISub like function.
+%define VBINSTST_CALL_FN_SUB_TEST   call TMPL_NM_CMN(TestSub)
+
+;; Call RTTestIFailure like function with simple message.
+%define VBINSTST_CALL_FN_FAILURE    call TMPL_NM_CMN(TestFailedF)
+
+;; Call RTTestIFailure like function with format message + 1 arg.
+%define VBINSTST_CALL_FN_FAILURE_1  call TMPL_NM_CMN(TestFailedF)
+
+;; Call RTTestIFailure like function with format message + 2 args.
+%define VBINSTST_CALL_FN_FAILURE_2  call TMPL_NM_CMN(TestFailedF)
+
+;; Call RTTestIFailure like function with format message + 3 args.
+%define VBINSTST_CALL_FN_FAILURE_3  call TMPL_NM_CMN(TestFailedF)
+
+;; Call RTTestIFailure like function with format message + 4 args.
+%define VBINSTST_CALL_FN_FAILURE_4  call TMPL_NM_CMN(TestFailedF)
+
+;; The image base label (used by the trap macros).
+%define VBINSTST_IMAGE_BASE_LABLE start
+
+;; Wrapper for calling TestInstallTrapRecs (used by the trap macros).
+%define VBINSTST_CALL_TEST_INSTALL_TRAP_RECS  call TMPL_NM_CMN(TestInstallTrapRecs)
+
+
+;
+; Include the common bits (contains code using above macros)
+;
+%include "env-bs2-r0-common.mac"
+
+%endif
+
diff --git a/src/VBox/VMM/testcase/Instructions/env-common.mac b/src/VBox/VMM/testcase/Instructions/env-common.mac
new file mode 100644
index 00000000..12879f58
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/env-common.mac
@@ -0,0 +1,346 @@
+; $Id: env-common.mac $
+;; @file
+; Instruction Test Environment - Common Bits.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%ifndef ___env_common_mac
+%define ___env_common_mac
+
+%include "iprt/x86.mac"
+
+;*******************************************************************************
+;*  Defined Constants And Macros                                               *
+;*******************************************************************************
+%ifdef RT_ARCH_AMD64
+ %define MY_PUSH_FLAGS      pushfq
+ %define MY_POP_FLAGS       popfq
+ %define MY_PUSH_FLAGS_SIZE 8
+
+ %macro MY_PUSH_ALL 0
+        push rbp
+        mov  rbp, rsp
+        push rax
+        push rbx
+        push rcx
+        push rdx
+        push rsi
+        push rdi
+        push r8
+        push r9
+        push r10
+        push r11
+        push r12
+        push r13
+        push r14
+        push r15
+        pushfq
+ %endm
+ %macro MY_POP_ALL 0
+        popfq
+        pop  r15
+        pop  r14
+        pop  r13
+        pop  r12
+        pop  r11
+        pop  r10
+        pop  r9
+        pop  r8
+        pop  rdi
+        pop  rsi
+        pop  rdx
+        pop  rcx
+        pop  rbx
+        pop  rax
+        pop  rbp
+ %endm
+
+%else
+ %define MY_PUSH_FLAGS      pushfd
+ %define MY_POP_FLAGS       popfd
+ %define MY_PUSH_FLAGS_SIZE 4
+
+ %macro MY_PUSH_ALL 0
+        push eBP
+        mov  xBP, xSP
+        push eax
+        push ebx
+        push ecx
+        push edx
+        push esi
+        push edi
+        pushfd
+ %endm
+ %macro MY_POP_ALL 0
+        popfd
+        pop  edi
+        pop  esi
+        pop  edx
+        pop  ecx
+        pop  ebx
+        pop  eax
+        pop  ebp
+ %endm
+%endif
+
+
+
+;*******************************************************************************
+;*  Internal Functions                                                         *
+;*******************************************************************************
+
+VBINSTST_BEGINCODE
+
+;;
+; Report bad register value.
+;
+; Primary purpose is save all registers and convert from our stack-based to
+; the correct calling convention for the environment.
+;
+; This function will clean up the stack upon return (to save space in the caller).
+;
+; @param    uExpected
+; @param    uActual
+; @param    uRegisterNo
+;
+VBINSTST_BEGINPROC Common_BadValue
+        MY_PUSH_ALL
+        mov     xAX, xSP                ; 16-byte align the stack and reserve space for arguments and stuff.
+        sub     xSP, 40h
+        and     xSP, ~15
+        mov     [xSP + 38h], xAX
+
+%ifdef ASM_CALL64_GCC
+        mov     r8d, [VBINSTST_NAME(g_uVBInsTstSubTestIndicator) wrt rip]
+        mov     rcx, [rbp + 10h] ; expected
+        mov     rdx, [rbp + 18h] ; actual
+        mov     rsi, [rbp + 20h] ; reg#
+        lea     rdi, [.szFmt wrt rip]
+        VBINSTST_CALL_FN_FAILURE_4
+
+%elifdef ASM_CALL64_MSC
+        mov     r10d, [VBINSTST_NAME(g_uVBInsTstSubTestIndicator) wrt rip]
+        mov     [rsp + 20h], r10
+        mov     r9,  [rbp + 10h] ; expected
+        mov     r8,  [rbp + 18h] ; actual
+        mov     rdx, [rbp + 20h] ; reg#
+        lea     rcx, [.szFmt wrt rip]
+        VBINSTST_CALL_FN_FAILURE_4
+
+%elifdef ASM_CALL64_BS2
+        mov     sBX, [VBINSTST_NAME(g_uVBInsTstSubTestIndicator) xWrtRIP]
+        mov     sCX, [xBP + xCB + xCB]     ; expected
+        mov     sAX, [xBP + xCB + xCB + sCB*1] ; actual
+        mov     sDX, [xBP + xCB + xCB + sCB*2] ; reg#
+        lea     sSI, [.szFmt xWrtRIP]
+        mov     qword [xSP + xCB + 3*sCB], sBX
+        mov     qword [xSP + xCB + 2*sCB], sCX
+        mov     qword [xSP + xCB + 1*sCB], sAX
+        mov     qword [xSP + xCB], sDX
+        mov     [xSP], sSI
+        VBINSTST_CALL_FN_FAILURE_4
+
+%else
+        mov     sBX, [VBINSTST_NAME(g_uVBInsTstSubTestIndicator)]
+        mov     sCX, [xBP + xCB + xCB]     ; expected
+        mov     sAX, [xBP + xCB + xCB + sCB*1] ; actual
+        mov     sDX, [xBP + xCB + xCB + sCB*2] ; reg#
+        mov     [xSP + xCB + 3*sCB], sBX
+        mov     [xSP + xCB + 2*sCB], sCX
+        mov     [xSP + xCB + 1*sCB], sAX
+        mov     [xSP + xCB], sDX
+        mov     [xSP], RTCCPTR_PRE .szFmt
+        VBINSTST_CALL_FN_FAILURE_4
+%endif
+
+        mov     xSP, [xSP + 38h]
+        MY_POP_ALL
+        ret     3*sCB
+%if ARCH_BITS == 64
+.szFmt: db 'Bad register 0x%RX32 value 0x%RX64, expected 0x%RX64 (line %RU64)', 13, 0
+%else
+.szFmt: db 'Bad register 0x%RX32 value 0x%RX32, expected 0x%RX32 (line %RU32)', 13, 0
+%endif
+VBINSTST_ENDPROC   Common_BadValue
+
+
+%ifdef VBINSTST_CAN_DO_TRAPS
+
+;;
+; Report a missing TRAP.
+;
+; Primary purpose is save all registers and convert from our stack-based to
+; the correct calling convention for the environment.
+;
+; This function will clean up the stack upon return (to save space in the caller).
+;
+; @param    uExpected
+;
+VBINSTST_BEGINPROC Common_MissingTrap
+        MY_PUSH_ALL
+        mov     xAX, xSP                ; 16-byte align the stack and reserve space for arguments and stuff.
+        sub     xSP, 40h
+        and     xSP, ~15
+        mov     [xSP + 38h], xAX
+
+ %ifdef ASM_CALL64_GCC
+        mov     rdx, [VBINSTST_NAME(g_uVBInsTstSubTestIndicator) wrt rip]
+        movzx   rsi, byte [rbp + 10h] ; expected
+        lea     rdi, [.szFmt wrt rip]
+        VBINSTST_CALL_FN_FAILURE_2
+
+ %elifdef ASM_CALL64_MSC
+        mov     r8d, [VBINSTST_NAME(g_uVBInsTstSubTestIndicator) wrt rip]
+        movzx   rdx, byte [rbp + 10h] ; expected
+        lea     rcx, [.szFmt wrt rip]
+        VBINSTST_CALL_FN_FAILURE_2
+
+ %elifdef ASM_CALL64_BS2
+        mov     sBX, [VBINSTST_NAME(g_uVBInsTstSubTestIndicator) xWrtRIP]
+        mov     sDX, [xBP + xCB + xCB]     ; expected
+        lea     sSI, [.szFmt xWrtRIP]
+        mov     qword [xSP + xCB + 1*sCB], sBX
+        mov     qword [xSP + xCB], sDX
+        mov     [xSP], sSI
+        VBINSTST_CALL_FN_FAILURE_2
+
+ %else
+        mov     sBX, [VBINSTST_NAME(g_uVBInsTstSubTestIndicator)]
+        mov     sDX, [xBP + xCB + xCB]     ; expected
+        mov     [xSP + xCB + 1*sCB], sBX
+        mov     [xSP + xCB], sDX
+        mov     [xSP], RTCCPTR_PRE .szFmt
+        VBINSTST_CALL_FN_FAILURE_2
+ %endif
+
+        mov     xSP, [xSP + 38h]
+        MY_POP_ALL
+        ret     1*sCB
+ %if ARCH_BITS == 64
+.szFmt: db 'Missing trap %RX8 (line %RU64)', 13, 0
+ %else
+.szFmt: db 'Missing trap %RX8 (line %RU32)', 13, 0
+ %endif
+VBINSTST_ENDPROC   Common_MissingTrap
+
+ %macro Common_MissingTrapTemplate 1
+    VBINSTST_BEGINPROC Common_MissingTrap_%1
+        push    %1
+        call    VBINSTST_NAME(Common_MissingTrap)
+        ret
+    VBINSTST_ENDPROC   Common_MissingTrap_%1
+ %endmacro
+ Common_MissingTrapTemplate X86_XCPT_DE
+ Common_MissingTrapTemplate X86_XCPT_DB
+ Common_MissingTrapTemplate X86_XCPT_NMI
+ Common_MissingTrapTemplate X86_XCPT_BP
+ Common_MissingTrapTemplate X86_XCPT_OF
+ Common_MissingTrapTemplate X86_XCPT_BR
+ Common_MissingTrapTemplate X86_XCPT_UD
+ Common_MissingTrapTemplate X86_XCPT_NM
+ ;Common_MissingTrapTemplate X86_XCPT_DF
+ ;Common_MissingTrapTemplate X86_XCPT_CO_SEG_OVERRUN
+ Common_MissingTrapTemplate X86_XCPT_TS
+ Common_MissingTrapTemplate X86_XCPT_NP
+ Common_MissingTrapTemplate X86_XCPT_SS
+ Common_MissingTrapTemplate X86_XCPT_GP
+ Common_MissingTrapTemplate X86_XCPT_PF
+ Common_MissingTrapTemplate X86_XCPT_MF
+ Common_MissingTrapTemplate X86_XCPT_AC
+ ;Common_MissingTrapTemplate X86_XCPT_MC
+ Common_MissingTrapTemplate X86_XCPT_XF
+
+%endif ; VBINSTST_CAN_DO_TRAPS
+
+
+;
+; Global data variables used by Common_SetupMemReadUxx.
+; For address calculation reasons, these must be qword aligned.
+;
+VBINSTST_BEGINDATA
+        align 64
+        dd      09d8af498h, 09ab3e5f8h
+VBINSTST_GLOBALNAME_EX g_u64Data, data hidden
+        dq      0
+        dd      07d7af797h, 096b36562h
+VBINSTST_GLOBALNAME_EX g_u32Data, data hidden
+        dd      0
+        dd      012305987h
+VBINSTST_GLOBALNAME_EX g_u16Data, data hidden
+        dw      0
+        dw      05865h
+        dw      03863h
+        dw      02679h
+VBINSTST_GLOBALNAME_EX g_u8Data, data hidden
+        db      0
+        db      90h
+        dw      0865ah
+        dd      058daffe2h
+
+VBINSTST_BEGINCODE
+
+;;
+; Sets up g_u8Data.
+; @param    uValue
+VBINSTST_BEGINPROC Common_SetupMemReadU8
+        push    sAX
+        mov     ax, [xSP + sCB + xCB]
+        mov     [VBINSTST_NAME(g_u8Data) xWrtRIP], ax
+        pop     sAX
+        ret sCB
+VBINSTST_ENDPROC   Common_SetupMemReadU8
+
+;;
+; Sets up g_u16Data.
+; @param    uValue
+VBINSTST_BEGINPROC Common_SetupMemReadU16
+        push    sAX
+        mov     ax, [xSP + sCB + xCB]
+        mov     [VBINSTST_NAME(g_u16Data) xWrtRIP], ax
+        pop     sAX
+        ret sCB
+VBINSTST_ENDPROC   Common_SetupMemReadU16
+
+;;
+; Sets up g_u32Data.
+; @param    uValue
+VBINSTST_BEGINPROC Common_SetupMemReadU32
+        push    sAX
+        mov     eax, [xSP + sCB + xCB]
+        mov     [VBINSTST_NAME(g_u32Data) xWrtRIP], eax
+        pop     sAX
+        ret sCB
+VBINSTST_ENDPROC   Common_SetupMemReadU32
+
+;;
+; Sets up g_u64Data.
+; @param    uValue
+VBINSTST_BEGINPROC Common_SetupMemReadU64
+        push    sAX
+%ifdef RT_ARCH_AMD64
+        mov     rax, [xSP + sCB + xCB]
+        mov     [VBINSTST_NAME(g_u64Data) xWrtRIP], rax
+%else
+        mov     eax, [xSP + sCB + xCB]
+        mov     [VBINSTST_NAME(g_u64Data) xWrtRIP], eax
+        mov     eax, [xSP + sCB + xCB + 4]
+        mov     [VBINSTST_NAME(g_u64Data) + 4 xWrtRIP], eax
+%endif
+        pop     sAX
+        ret sCB
+VBINSTST_ENDPROC   Common_SetupMemReadU64
+
+
+%endif
+
diff --git a/src/VBox/VMM/testcase/Instructions/env-iprt-r3-32.mac b/src/VBox/VMM/testcase/Instructions/env-iprt-r3-32.mac
new file mode 100644
index 00000000..94afc032
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/env-iprt-r3-32.mac
@@ -0,0 +1,19 @@
+; $Id: env-iprt-r3-32.mac $
+;; @file
+; Instruction Test Environment - IPRT, Ring-3, 32-Bit.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%include "env-iprt-r3.mac"
+
diff --git a/src/VBox/VMM/testcase/Instructions/env-iprt-r3-64.mac b/src/VBox/VMM/testcase/Instructions/env-iprt-r3-64.mac
new file mode 100644
index 00000000..11f1b351
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/env-iprt-r3-64.mac
@@ -0,0 +1,19 @@
+; $Id: env-iprt-r3-64.mac $
+;; @file
+; Instruction Test Environment - IPRT, Ring-3, 64-Bit.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%include "env-iprt-r3.mac"
+
diff --git a/src/VBox/VMM/testcase/Instructions/env-iprt-r3.mac b/src/VBox/VMM/testcase/Instructions/env-iprt-r3.mac
new file mode 100644
index 00000000..80b42b93
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/env-iprt-r3.mac
@@ -0,0 +1,99 @@
+; $Id: env-iprt-r3.mac $
+;; @file
+; Instruction Test Environment - IPRT, Ring-3, 32-bit and 64-bit.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%ifndef ___env_iprt_r3_mac
+%define ___env_iprt_r3_mac
+
+;*******************************************************************************
+;*  Header Files                                                               *
+;*******************************************************************************
+%include "iprt/asmdefs.mac"
+
+
+;*******************************************************************************
+;*  Defined Constants And Macros                                               *
+;*******************************************************************************
+%define sAX xAX
+%define sBX xBX
+%define sCX xCX
+%define sDX xDX
+%define sSP xSP
+%define sBP xBP
+%define sSI xSI
+%define sDI xDI
+%define sCB xCB
+
+
+;; Same as BEGINPROC in asmdefs.mac.
+%macro VBINSTST_BEGINPROC 1
+BEGINPROC %1
+%endm
+
+;; Same as ENDPROC in asmdefs.mac.
+%macro VBINSTST_ENDPROC   1
+ENDPROC %1
+%endm
+
+;; Same as NAME in asmdefs.mac.
+%define VBINSTST_NAME(a_Name)   NAME(a_Name)
+
+;; Same as GLOBALNAME_EX in asmdefs.mac.
+%define VBINSTST_GLOBALNAME_EX  GLOBALNAME_EX
+
+;; Same as BEGINCODE in asmdefs.mac.
+%define VBINSTST_BEGINCODE      BEGINCODE
+
+;; Same as BEGINDATA in asmdefs.mac.
+%define VBINSTST_BEGINDATA      BEGINDATA
+
+
+;; Call RTTestISub like function.
+%define VBINSTST_CALL_FN_SUB_TEST   call IMP2(RTTestISub)
+EXTERN_IMP2 RTTestISub
+
+;; Call RTTestIFailure like function with simple message.
+%define VBINSTST_CALL_FN_FAILURE    call NAME(VBInsTstFailure)
+extern NAME(VBInsTstFailure)
+
+;; Call RTTestIFailure like function with format message + 1 arg.
+%define VBINSTST_CALL_FN_FAILURE_1  call NAME(VBInsTstFailure1)
+extern NAME(VBInsTstFailure1)
+
+;; Call RTTestIFailure like function with format message + 2 args.
+%define VBINSTST_CALL_FN_FAILURE_2  call NAME(VBInsTstFailure2)
+extern NAME(VBInsTstFailure2)
+
+;; Call RTTestIFailure like function with format message + 3 args.
+%define VBINSTST_CALL_FN_FAILURE_3  call NAME(VBInsTstFailure3)
+extern NAME(VBInsTstFailure3)
+
+;; Call RTTestIFailure like function with format message + 4 args.
+%define VBINSTST_CALL_FN_FAILURE_4  call NAME(VBInsTstFailure4)
+extern NAME(VBInsTstFailure4)
+
+
+;; Cannot do traps yet.
+%undef VBINSTST_CAN_DO_TRAPS
+
+
+;
+; Include the common bits (contains code using above macros)
+;
+%include "env-common.mac"
+
+%endif
+
diff --git a/src/VBox/VMM/testcase/Instructions/itgTableDaa.py b/src/VBox/VMM/testcase/Instructions/itgTableDaa.py
new file mode 100644
index 00000000..3aab0f10
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/itgTableDaa.py
@@ -0,0 +1,1105 @@
+# -*- coding: utf-8 -*-
+# $Id: itgTableDaa.py $
+
+"""
+DAA (instruction) result table.
+"""
+
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2020 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+"""
+__version__ = "$Revision: 135976 $";
+
+
+## The 32-bit GCC (C99) program that produced the table below.
+g_sItgCProgramDaa = \
+"""
+#include <stdio.h>
+
+int main()
+{
+    for (unsigned uInputAL = 0; uInputAL < 256; uInputAL++)
+        for (unsigned fAux = 0; fAux < 2; fAux++)
+            for (unsigned fCarry = 0; fCarry < 2; fCarry++)
+            {
+                unsigned uInputEFlags = fCarry | (fAux << 4);
+                unsigned uResultAL;
+                unsigned uResultEFlags;
+                __asm__ __volatile__("pushl %1\\n"
+                                     "popfl\\n"
+                                     "daa\\n"
+                                     "pushf\\n"
+                                     "pop   %1\\n"
+                                     : "=a" (uResultAL),
+                                       "=r" (uResultEFlags)
+                                     : "0" (uInputAL),
+                                       "1" (uInputEFlags)
+                                     : "memory"
+                                     );
+                printf("    ( 0x%02x, 0x%02x ), # AL=0x%02x, AF=%u CF=%u\\n",
+                       uResultAL, uResultEFlags & 0xd5, uInputAL, fAux, fCarry);
+                       /* 0xd5 = CF, PF, AF, ZF, SF */
+            }
+    return 0;
+}
+""";
+
+
+#
+# Compile and run the above program if requested to do so.
+#
+if __name__ == '__main__':
+    import sys;
+    if len(sys.argv) > 1 and sys.argv[1] == 'gen':
+        import subprocess;
+        oProc = subprocess.Popen(['gcc', '-x', 'c', '-std=gnu99', '-m32', '-o', './itgTableDaa', '-'], stdin = subprocess.PIPE);
+        oProc.communicate(g_sItgCProgramDaa);
+        oProc.wait();
+        oProc = subprocess.Popen(['./itgTableDaa',]).wait();
+        sys.exit(0);
+
+
+
+##
+# The DAA results.
+#
+# The index / input relation is: index = (AL << 2) | (CF << 1) | AF
+#
+g_aItgDaaResults = \
+[
+    ( 0x00, 0x44 ), # AL=0x00, AF=0 CF=0
+    ( 0x60, 0x05 ), # AL=0x00, AF=0 CF=1
+    ( 0x06, 0x14 ), # AL=0x00, AF=1 CF=0
+    ( 0x66, 0x15 ), # AL=0x00, AF=1 CF=1
+    ( 0x01, 0x00 ), # AL=0x01, AF=0 CF=0
+    ( 0x61, 0x01 ), # AL=0x01, AF=0 CF=1
+    ( 0x07, 0x10 ), # AL=0x01, AF=1 CF=0
+    ( 0x67, 0x11 ), # AL=0x01, AF=1 CF=1
+    ( 0x02, 0x00 ), # AL=0x02, AF=0 CF=0
+    ( 0x62, 0x01 ), # AL=0x02, AF=0 CF=1
+    ( 0x08, 0x10 ), # AL=0x02, AF=1 CF=0
+    ( 0x68, 0x11 ), # AL=0x02, AF=1 CF=1
+    ( 0x03, 0x04 ), # AL=0x03, AF=0 CF=0
+    ( 0x63, 0x05 ), # AL=0x03, AF=0 CF=1
+    ( 0x09, 0x14 ), # AL=0x03, AF=1 CF=0
+    ( 0x69, 0x15 ), # AL=0x03, AF=1 CF=1
+    ( 0x04, 0x00 ), # AL=0x04, AF=0 CF=0
+    ( 0x64, 0x01 ), # AL=0x04, AF=0 CF=1
+    ( 0x0a, 0x14 ), # AL=0x04, AF=1 CF=0
+    ( 0x6a, 0x15 ), # AL=0x04, AF=1 CF=1
+    ( 0x05, 0x04 ), # AL=0x05, AF=0 CF=0
+    ( 0x65, 0x05 ), # AL=0x05, AF=0 CF=1
+    ( 0x0b, 0x10 ), # AL=0x05, AF=1 CF=0
+    ( 0x6b, 0x11 ), # AL=0x05, AF=1 CF=1
+    ( 0x06, 0x04 ), # AL=0x06, AF=0 CF=0
+    ( 0x66, 0x05 ), # AL=0x06, AF=0 CF=1
+    ( 0x0c, 0x14 ), # AL=0x06, AF=1 CF=0
+    ( 0x6c, 0x15 ), # AL=0x06, AF=1 CF=1
+    ( 0x07, 0x00 ), # AL=0x07, AF=0 CF=0
+    ( 0x67, 0x01 ), # AL=0x07, AF=0 CF=1
+    ( 0x0d, 0x10 ), # AL=0x07, AF=1 CF=0
+    ( 0x6d, 0x11 ), # AL=0x07, AF=1 CF=1
+    ( 0x08, 0x00 ), # AL=0x08, AF=0 CF=0
+    ( 0x68, 0x01 ), # AL=0x08, AF=0 CF=1
+    ( 0x0e, 0x10 ), # AL=0x08, AF=1 CF=0
+    ( 0x6e, 0x11 ), # AL=0x08, AF=1 CF=1
+    ( 0x09, 0x04 ), # AL=0x09, AF=0 CF=0
+    ( 0x69, 0x05 ), # AL=0x09, AF=0 CF=1
+    ( 0x0f, 0x14 ), # AL=0x09, AF=1 CF=0
+    ( 0x6f, 0x15 ), # AL=0x09, AF=1 CF=1
+    ( 0x10, 0x10 ), # AL=0x0a, AF=0 CF=0
+    ( 0x70, 0x11 ), # AL=0x0a, AF=0 CF=1
+    ( 0x10, 0x10 ), # AL=0x0a, AF=1 CF=0
+    ( 0x70, 0x11 ), # AL=0x0a, AF=1 CF=1
+    ( 0x11, 0x14 ), # AL=0x0b, AF=0 CF=0
+    ( 0x71, 0x15 ), # AL=0x0b, AF=0 CF=1
+    ( 0x11, 0x14 ), # AL=0x0b, AF=1 CF=0
+    ( 0x71, 0x15 ), # AL=0x0b, AF=1 CF=1
+    ( 0x12, 0x14 ), # AL=0x0c, AF=0 CF=0
+    ( 0x72, 0x15 ), # AL=0x0c, AF=0 CF=1
+    ( 0x12, 0x14 ), # AL=0x0c, AF=1 CF=0
+    ( 0x72, 0x15 ), # AL=0x0c, AF=1 CF=1
+    ( 0x13, 0x10 ), # AL=0x0d, AF=0 CF=0
+    ( 0x73, 0x11 ), # AL=0x0d, AF=0 CF=1
+    ( 0x13, 0x10 ), # AL=0x0d, AF=1 CF=0
+    ( 0x73, 0x11 ), # AL=0x0d, AF=1 CF=1
+    ( 0x14, 0x14 ), # AL=0x0e, AF=0 CF=0
+    ( 0x74, 0x15 ), # AL=0x0e, AF=0 CF=1
+    ( 0x14, 0x14 ), # AL=0x0e, AF=1 CF=0
+    ( 0x74, 0x15 ), # AL=0x0e, AF=1 CF=1
+    ( 0x15, 0x10 ), # AL=0x0f, AF=0 CF=0
+    ( 0x75, 0x11 ), # AL=0x0f, AF=0 CF=1
+    ( 0x15, 0x10 ), # AL=0x0f, AF=1 CF=0
+    ( 0x75, 0x11 ), # AL=0x0f, AF=1 CF=1
+    ( 0x10, 0x00 ), # AL=0x10, AF=0 CF=0
+    ( 0x70, 0x01 ), # AL=0x10, AF=0 CF=1
+    ( 0x16, 0x10 ), # AL=0x10, AF=1 CF=0
+    ( 0x76, 0x11 ), # AL=0x10, AF=1 CF=1
+    ( 0x11, 0x04 ), # AL=0x11, AF=0 CF=0
+    ( 0x71, 0x05 ), # AL=0x11, AF=0 CF=1
+    ( 0x17, 0x14 ), # AL=0x11, AF=1 CF=0
+    ( 0x77, 0x15 ), # AL=0x11, AF=1 CF=1
+    ( 0x12, 0x04 ), # AL=0x12, AF=0 CF=0
+    ( 0x72, 0x05 ), # AL=0x12, AF=0 CF=1
+    ( 0x18, 0x14 ), # AL=0x12, AF=1 CF=0
+    ( 0x78, 0x15 ), # AL=0x12, AF=1 CF=1
+    ( 0x13, 0x00 ), # AL=0x13, AF=0 CF=0
+    ( 0x73, 0x01 ), # AL=0x13, AF=0 CF=1
+    ( 0x19, 0x10 ), # AL=0x13, AF=1 CF=0
+    ( 0x79, 0x11 ), # AL=0x13, AF=1 CF=1
+    ( 0x14, 0x04 ), # AL=0x14, AF=0 CF=0
+    ( 0x74, 0x05 ), # AL=0x14, AF=0 CF=1
+    ( 0x1a, 0x10 ), # AL=0x14, AF=1 CF=0
+    ( 0x7a, 0x11 ), # AL=0x14, AF=1 CF=1
+    ( 0x15, 0x00 ), # AL=0x15, AF=0 CF=0
+    ( 0x75, 0x01 ), # AL=0x15, AF=0 CF=1
+    ( 0x1b, 0x14 ), # AL=0x15, AF=1 CF=0
+    ( 0x7b, 0x15 ), # AL=0x15, AF=1 CF=1
+    ( 0x16, 0x00 ), # AL=0x16, AF=0 CF=0
+    ( 0x76, 0x01 ), # AL=0x16, AF=0 CF=1
+    ( 0x1c, 0x10 ), # AL=0x16, AF=1 CF=0
+    ( 0x7c, 0x11 ), # AL=0x16, AF=1 CF=1
+    ( 0x17, 0x04 ), # AL=0x17, AF=0 CF=0
+    ( 0x77, 0x05 ), # AL=0x17, AF=0 CF=1
+    ( 0x1d, 0x14 ), # AL=0x17, AF=1 CF=0
+    ( 0x7d, 0x15 ), # AL=0x17, AF=1 CF=1
+    ( 0x18, 0x04 ), # AL=0x18, AF=0 CF=0
+    ( 0x78, 0x05 ), # AL=0x18, AF=0 CF=1
+    ( 0x1e, 0x14 ), # AL=0x18, AF=1 CF=0
+    ( 0x7e, 0x15 ), # AL=0x18, AF=1 CF=1
+    ( 0x19, 0x00 ), # AL=0x19, AF=0 CF=0
+    ( 0x79, 0x01 ), # AL=0x19, AF=0 CF=1
+    ( 0x1f, 0x10 ), # AL=0x19, AF=1 CF=0
+    ( 0x7f, 0x11 ), # AL=0x19, AF=1 CF=1
+    ( 0x20, 0x10 ), # AL=0x1a, AF=0 CF=0
+    ( 0x80, 0x91 ), # AL=0x1a, AF=0 CF=1
+    ( 0x20, 0x10 ), # AL=0x1a, AF=1 CF=0
+    ( 0x80, 0x91 ), # AL=0x1a, AF=1 CF=1
+    ( 0x21, 0x14 ), # AL=0x1b, AF=0 CF=0
+    ( 0x81, 0x95 ), # AL=0x1b, AF=0 CF=1
+    ( 0x21, 0x14 ), # AL=0x1b, AF=1 CF=0
+    ( 0x81, 0x95 ), # AL=0x1b, AF=1 CF=1
+    ( 0x22, 0x14 ), # AL=0x1c, AF=0 CF=0
+    ( 0x82, 0x95 ), # AL=0x1c, AF=0 CF=1
+    ( 0x22, 0x14 ), # AL=0x1c, AF=1 CF=0
+    ( 0x82, 0x95 ), # AL=0x1c, AF=1 CF=1
+    ( 0x23, 0x10 ), # AL=0x1d, AF=0 CF=0
+    ( 0x83, 0x91 ), # AL=0x1d, AF=0 CF=1
+    ( 0x23, 0x10 ), # AL=0x1d, AF=1 CF=0
+    ( 0x83, 0x91 ), # AL=0x1d, AF=1 CF=1
+    ( 0x24, 0x14 ), # AL=0x1e, AF=0 CF=0
+    ( 0x84, 0x95 ), # AL=0x1e, AF=0 CF=1
+    ( 0x24, 0x14 ), # AL=0x1e, AF=1 CF=0
+    ( 0x84, 0x95 ), # AL=0x1e, AF=1 CF=1
+    ( 0x25, 0x10 ), # AL=0x1f, AF=0 CF=0
+    ( 0x85, 0x91 ), # AL=0x1f, AF=0 CF=1
+    ( 0x25, 0x10 ), # AL=0x1f, AF=1 CF=0
+    ( 0x85, 0x91 ), # AL=0x1f, AF=1 CF=1
+    ( 0x20, 0x00 ), # AL=0x20, AF=0 CF=0
+    ( 0x80, 0x81 ), # AL=0x20, AF=0 CF=1
+    ( 0x26, 0x10 ), # AL=0x20, AF=1 CF=0
+    ( 0x86, 0x91 ), # AL=0x20, AF=1 CF=1
+    ( 0x21, 0x04 ), # AL=0x21, AF=0 CF=0
+    ( 0x81, 0x85 ), # AL=0x21, AF=0 CF=1
+    ( 0x27, 0x14 ), # AL=0x21, AF=1 CF=0
+    ( 0x87, 0x95 ), # AL=0x21, AF=1 CF=1
+    ( 0x22, 0x04 ), # AL=0x22, AF=0 CF=0
+    ( 0x82, 0x85 ), # AL=0x22, AF=0 CF=1
+    ( 0x28, 0x14 ), # AL=0x22, AF=1 CF=0
+    ( 0x88, 0x95 ), # AL=0x22, AF=1 CF=1
+    ( 0x23, 0x00 ), # AL=0x23, AF=0 CF=0
+    ( 0x83, 0x81 ), # AL=0x23, AF=0 CF=1
+    ( 0x29, 0x10 ), # AL=0x23, AF=1 CF=0
+    ( 0x89, 0x91 ), # AL=0x23, AF=1 CF=1
+    ( 0x24, 0x04 ), # AL=0x24, AF=0 CF=0
+    ( 0x84, 0x85 ), # AL=0x24, AF=0 CF=1
+    ( 0x2a, 0x10 ), # AL=0x24, AF=1 CF=0
+    ( 0x8a, 0x91 ), # AL=0x24, AF=1 CF=1
+    ( 0x25, 0x00 ), # AL=0x25, AF=0 CF=0
+    ( 0x85, 0x81 ), # AL=0x25, AF=0 CF=1
+    ( 0x2b, 0x14 ), # AL=0x25, AF=1 CF=0
+    ( 0x8b, 0x95 ), # AL=0x25, AF=1 CF=1
+    ( 0x26, 0x00 ), # AL=0x26, AF=0 CF=0
+    ( 0x86, 0x81 ), # AL=0x26, AF=0 CF=1
+    ( 0x2c, 0x10 ), # AL=0x26, AF=1 CF=0
+    ( 0x8c, 0x91 ), # AL=0x26, AF=1 CF=1
+    ( 0x27, 0x04 ), # AL=0x27, AF=0 CF=0
+    ( 0x87, 0x85 ), # AL=0x27, AF=0 CF=1
+    ( 0x2d, 0x14 ), # AL=0x27, AF=1 CF=0
+    ( 0x8d, 0x95 ), # AL=0x27, AF=1 CF=1
+    ( 0x28, 0x04 ), # AL=0x28, AF=0 CF=0
+    ( 0x88, 0x85 ), # AL=0x28, AF=0 CF=1
+    ( 0x2e, 0x14 ), # AL=0x28, AF=1 CF=0
+    ( 0x8e, 0x95 ), # AL=0x28, AF=1 CF=1
+    ( 0x29, 0x00 ), # AL=0x29, AF=0 CF=0
+    ( 0x89, 0x81 ), # AL=0x29, AF=0 CF=1
+    ( 0x2f, 0x10 ), # AL=0x29, AF=1 CF=0
+    ( 0x8f, 0x91 ), # AL=0x29, AF=1 CF=1
+    ( 0x30, 0x14 ), # AL=0x2a, AF=0 CF=0
+    ( 0x90, 0x95 ), # AL=0x2a, AF=0 CF=1
+    ( 0x30, 0x14 ), # AL=0x2a, AF=1 CF=0
+    ( 0x90, 0x95 ), # AL=0x2a, AF=1 CF=1
+    ( 0x31, 0x10 ), # AL=0x2b, AF=0 CF=0
+    ( 0x91, 0x91 ), # AL=0x2b, AF=0 CF=1
+    ( 0x31, 0x10 ), # AL=0x2b, AF=1 CF=0
+    ( 0x91, 0x91 ), # AL=0x2b, AF=1 CF=1
+    ( 0x32, 0x10 ), # AL=0x2c, AF=0 CF=0
+    ( 0x92, 0x91 ), # AL=0x2c, AF=0 CF=1
+    ( 0x32, 0x10 ), # AL=0x2c, AF=1 CF=0
+    ( 0x92, 0x91 ), # AL=0x2c, AF=1 CF=1
+    ( 0x33, 0x14 ), # AL=0x2d, AF=0 CF=0
+    ( 0x93, 0x95 ), # AL=0x2d, AF=0 CF=1
+    ( 0x33, 0x14 ), # AL=0x2d, AF=1 CF=0
+    ( 0x93, 0x95 ), # AL=0x2d, AF=1 CF=1
+    ( 0x34, 0x10 ), # AL=0x2e, AF=0 CF=0
+    ( 0x94, 0x91 ), # AL=0x2e, AF=0 CF=1
+    ( 0x34, 0x10 ), # AL=0x2e, AF=1 CF=0
+    ( 0x94, 0x91 ), # AL=0x2e, AF=1 CF=1
+    ( 0x35, 0x14 ), # AL=0x2f, AF=0 CF=0
+    ( 0x95, 0x95 ), # AL=0x2f, AF=0 CF=1
+    ( 0x35, 0x14 ), # AL=0x2f, AF=1 CF=0
+    ( 0x95, 0x95 ), # AL=0x2f, AF=1 CF=1
+    ( 0x30, 0x04 ), # AL=0x30, AF=0 CF=0
+    ( 0x90, 0x85 ), # AL=0x30, AF=0 CF=1
+    ( 0x36, 0x14 ), # AL=0x30, AF=1 CF=0
+    ( 0x96, 0x95 ), # AL=0x30, AF=1 CF=1
+    ( 0x31, 0x00 ), # AL=0x31, AF=0 CF=0
+    ( 0x91, 0x81 ), # AL=0x31, AF=0 CF=1
+    ( 0x37, 0x10 ), # AL=0x31, AF=1 CF=0
+    ( 0x97, 0x91 ), # AL=0x31, AF=1 CF=1
+    ( 0x32, 0x00 ), # AL=0x32, AF=0 CF=0
+    ( 0x92, 0x81 ), # AL=0x32, AF=0 CF=1
+    ( 0x38, 0x10 ), # AL=0x32, AF=1 CF=0
+    ( 0x98, 0x91 ), # AL=0x32, AF=1 CF=1
+    ( 0x33, 0x04 ), # AL=0x33, AF=0 CF=0
+    ( 0x93, 0x85 ), # AL=0x33, AF=0 CF=1
+    ( 0x39, 0x14 ), # AL=0x33, AF=1 CF=0
+    ( 0x99, 0x95 ), # AL=0x33, AF=1 CF=1
+    ( 0x34, 0x00 ), # AL=0x34, AF=0 CF=0
+    ( 0x94, 0x81 ), # AL=0x34, AF=0 CF=1
+    ( 0x3a, 0x14 ), # AL=0x34, AF=1 CF=0
+    ( 0x9a, 0x95 ), # AL=0x34, AF=1 CF=1
+    ( 0x35, 0x04 ), # AL=0x35, AF=0 CF=0
+    ( 0x95, 0x85 ), # AL=0x35, AF=0 CF=1
+    ( 0x3b, 0x10 ), # AL=0x35, AF=1 CF=0
+    ( 0x9b, 0x91 ), # AL=0x35, AF=1 CF=1
+    ( 0x36, 0x04 ), # AL=0x36, AF=0 CF=0
+    ( 0x96, 0x85 ), # AL=0x36, AF=0 CF=1
+    ( 0x3c, 0x14 ), # AL=0x36, AF=1 CF=0
+    ( 0x9c, 0x95 ), # AL=0x36, AF=1 CF=1
+    ( 0x37, 0x00 ), # AL=0x37, AF=0 CF=0
+    ( 0x97, 0x81 ), # AL=0x37, AF=0 CF=1
+    ( 0x3d, 0x10 ), # AL=0x37, AF=1 CF=0
+    ( 0x9d, 0x91 ), # AL=0x37, AF=1 CF=1
+    ( 0x38, 0x00 ), # AL=0x38, AF=0 CF=0
+    ( 0x98, 0x81 ), # AL=0x38, AF=0 CF=1
+    ( 0x3e, 0x10 ), # AL=0x38, AF=1 CF=0
+    ( 0x9e, 0x91 ), # AL=0x38, AF=1 CF=1
+    ( 0x39, 0x04 ), # AL=0x39, AF=0 CF=0
+    ( 0x99, 0x85 ), # AL=0x39, AF=0 CF=1
+    ( 0x3f, 0x14 ), # AL=0x39, AF=1 CF=0
+    ( 0x9f, 0x95 ), # AL=0x39, AF=1 CF=1
+    ( 0x40, 0x10 ), # AL=0x3a, AF=0 CF=0
+    ( 0xa0, 0x95 ), # AL=0x3a, AF=0 CF=1
+    ( 0x40, 0x10 ), # AL=0x3a, AF=1 CF=0
+    ( 0xa0, 0x95 ), # AL=0x3a, AF=1 CF=1
+    ( 0x41, 0x14 ), # AL=0x3b, AF=0 CF=0
+    ( 0xa1, 0x91 ), # AL=0x3b, AF=0 CF=1
+    ( 0x41, 0x14 ), # AL=0x3b, AF=1 CF=0
+    ( 0xa1, 0x91 ), # AL=0x3b, AF=1 CF=1
+    ( 0x42, 0x14 ), # AL=0x3c, AF=0 CF=0
+    ( 0xa2, 0x91 ), # AL=0x3c, AF=0 CF=1
+    ( 0x42, 0x14 ), # AL=0x3c, AF=1 CF=0
+    ( 0xa2, 0x91 ), # AL=0x3c, AF=1 CF=1
+    ( 0x43, 0x10 ), # AL=0x3d, AF=0 CF=0
+    ( 0xa3, 0x95 ), # AL=0x3d, AF=0 CF=1
+    ( 0x43, 0x10 ), # AL=0x3d, AF=1 CF=0
+    ( 0xa3, 0x95 ), # AL=0x3d, AF=1 CF=1
+    ( 0x44, 0x14 ), # AL=0x3e, AF=0 CF=0
+    ( 0xa4, 0x91 ), # AL=0x3e, AF=0 CF=1
+    ( 0x44, 0x14 ), # AL=0x3e, AF=1 CF=0
+    ( 0xa4, 0x91 ), # AL=0x3e, AF=1 CF=1
+    ( 0x45, 0x10 ), # AL=0x3f, AF=0 CF=0
+    ( 0xa5, 0x95 ), # AL=0x3f, AF=0 CF=1
+    ( 0x45, 0x10 ), # AL=0x3f, AF=1 CF=0
+    ( 0xa5, 0x95 ), # AL=0x3f, AF=1 CF=1
+    ( 0x40, 0x00 ), # AL=0x40, AF=0 CF=0
+    ( 0xa0, 0x85 ), # AL=0x40, AF=0 CF=1
+    ( 0x46, 0x10 ), # AL=0x40, AF=1 CF=0
+    ( 0xa6, 0x95 ), # AL=0x40, AF=1 CF=1
+    ( 0x41, 0x04 ), # AL=0x41, AF=0 CF=0
+    ( 0xa1, 0x81 ), # AL=0x41, AF=0 CF=1
+    ( 0x47, 0x14 ), # AL=0x41, AF=1 CF=0
+    ( 0xa7, 0x91 ), # AL=0x41, AF=1 CF=1
+    ( 0x42, 0x04 ), # AL=0x42, AF=0 CF=0
+    ( 0xa2, 0x81 ), # AL=0x42, AF=0 CF=1
+    ( 0x48, 0x14 ), # AL=0x42, AF=1 CF=0
+    ( 0xa8, 0x91 ), # AL=0x42, AF=1 CF=1
+    ( 0x43, 0x00 ), # AL=0x43, AF=0 CF=0
+    ( 0xa3, 0x85 ), # AL=0x43, AF=0 CF=1
+    ( 0x49, 0x10 ), # AL=0x43, AF=1 CF=0
+    ( 0xa9, 0x95 ), # AL=0x43, AF=1 CF=1
+    ( 0x44, 0x04 ), # AL=0x44, AF=0 CF=0
+    ( 0xa4, 0x81 ), # AL=0x44, AF=0 CF=1
+    ( 0x4a, 0x10 ), # AL=0x44, AF=1 CF=0
+    ( 0xaa, 0x95 ), # AL=0x44, AF=1 CF=1
+    ( 0x45, 0x00 ), # AL=0x45, AF=0 CF=0
+    ( 0xa5, 0x85 ), # AL=0x45, AF=0 CF=1
+    ( 0x4b, 0x14 ), # AL=0x45, AF=1 CF=0
+    ( 0xab, 0x91 ), # AL=0x45, AF=1 CF=1
+    ( 0x46, 0x00 ), # AL=0x46, AF=0 CF=0
+    ( 0xa6, 0x85 ), # AL=0x46, AF=0 CF=1
+    ( 0x4c, 0x10 ), # AL=0x46, AF=1 CF=0
+    ( 0xac, 0x95 ), # AL=0x46, AF=1 CF=1
+    ( 0x47, 0x04 ), # AL=0x47, AF=0 CF=0
+    ( 0xa7, 0x81 ), # AL=0x47, AF=0 CF=1
+    ( 0x4d, 0x14 ), # AL=0x47, AF=1 CF=0
+    ( 0xad, 0x91 ), # AL=0x47, AF=1 CF=1
+    ( 0x48, 0x04 ), # AL=0x48, AF=0 CF=0
+    ( 0xa8, 0x81 ), # AL=0x48, AF=0 CF=1
+    ( 0x4e, 0x14 ), # AL=0x48, AF=1 CF=0
+    ( 0xae, 0x91 ), # AL=0x48, AF=1 CF=1
+    ( 0x49, 0x00 ), # AL=0x49, AF=0 CF=0
+    ( 0xa9, 0x85 ), # AL=0x49, AF=0 CF=1
+    ( 0x4f, 0x10 ), # AL=0x49, AF=1 CF=0
+    ( 0xaf, 0x95 ), # AL=0x49, AF=1 CF=1
+    ( 0x50, 0x14 ), # AL=0x4a, AF=0 CF=0
+    ( 0xb0, 0x91 ), # AL=0x4a, AF=0 CF=1
+    ( 0x50, 0x14 ), # AL=0x4a, AF=1 CF=0
+    ( 0xb0, 0x91 ), # AL=0x4a, AF=1 CF=1
+    ( 0x51, 0x10 ), # AL=0x4b, AF=0 CF=0
+    ( 0xb1, 0x95 ), # AL=0x4b, AF=0 CF=1
+    ( 0x51, 0x10 ), # AL=0x4b, AF=1 CF=0
+    ( 0xb1, 0x95 ), # AL=0x4b, AF=1 CF=1
+    ( 0x52, 0x10 ), # AL=0x4c, AF=0 CF=0
+    ( 0xb2, 0x95 ), # AL=0x4c, AF=0 CF=1
+    ( 0x52, 0x10 ), # AL=0x4c, AF=1 CF=0
+    ( 0xb2, 0x95 ), # AL=0x4c, AF=1 CF=1
+    ( 0x53, 0x14 ), # AL=0x4d, AF=0 CF=0
+    ( 0xb3, 0x91 ), # AL=0x4d, AF=0 CF=1
+    ( 0x53, 0x14 ), # AL=0x4d, AF=1 CF=0
+    ( 0xb3, 0x91 ), # AL=0x4d, AF=1 CF=1
+    ( 0x54, 0x10 ), # AL=0x4e, AF=0 CF=0
+    ( 0xb4, 0x95 ), # AL=0x4e, AF=0 CF=1
+    ( 0x54, 0x10 ), # AL=0x4e, AF=1 CF=0
+    ( 0xb4, 0x95 ), # AL=0x4e, AF=1 CF=1
+    ( 0x55, 0x14 ), # AL=0x4f, AF=0 CF=0
+    ( 0xb5, 0x91 ), # AL=0x4f, AF=0 CF=1
+    ( 0x55, 0x14 ), # AL=0x4f, AF=1 CF=0
+    ( 0xb5, 0x91 ), # AL=0x4f, AF=1 CF=1
+    ( 0x50, 0x04 ), # AL=0x50, AF=0 CF=0
+    ( 0xb0, 0x81 ), # AL=0x50, AF=0 CF=1
+    ( 0x56, 0x14 ), # AL=0x50, AF=1 CF=0
+    ( 0xb6, 0x91 ), # AL=0x50, AF=1 CF=1
+    ( 0x51, 0x00 ), # AL=0x51, AF=0 CF=0
+    ( 0xb1, 0x85 ), # AL=0x51, AF=0 CF=1
+    ( 0x57, 0x10 ), # AL=0x51, AF=1 CF=0
+    ( 0xb7, 0x95 ), # AL=0x51, AF=1 CF=1
+    ( 0x52, 0x00 ), # AL=0x52, AF=0 CF=0
+    ( 0xb2, 0x85 ), # AL=0x52, AF=0 CF=1
+    ( 0x58, 0x10 ), # AL=0x52, AF=1 CF=0
+    ( 0xb8, 0x95 ), # AL=0x52, AF=1 CF=1
+    ( 0x53, 0x04 ), # AL=0x53, AF=0 CF=0
+    ( 0xb3, 0x81 ), # AL=0x53, AF=0 CF=1
+    ( 0x59, 0x14 ), # AL=0x53, AF=1 CF=0
+    ( 0xb9, 0x91 ), # AL=0x53, AF=1 CF=1
+    ( 0x54, 0x00 ), # AL=0x54, AF=0 CF=0
+    ( 0xb4, 0x85 ), # AL=0x54, AF=0 CF=1
+    ( 0x5a, 0x14 ), # AL=0x54, AF=1 CF=0
+    ( 0xba, 0x91 ), # AL=0x54, AF=1 CF=1
+    ( 0x55, 0x04 ), # AL=0x55, AF=0 CF=0
+    ( 0xb5, 0x81 ), # AL=0x55, AF=0 CF=1
+    ( 0x5b, 0x10 ), # AL=0x55, AF=1 CF=0
+    ( 0xbb, 0x95 ), # AL=0x55, AF=1 CF=1
+    ( 0x56, 0x04 ), # AL=0x56, AF=0 CF=0
+    ( 0xb6, 0x81 ), # AL=0x56, AF=0 CF=1
+    ( 0x5c, 0x14 ), # AL=0x56, AF=1 CF=0
+    ( 0xbc, 0x91 ), # AL=0x56, AF=1 CF=1
+    ( 0x57, 0x00 ), # AL=0x57, AF=0 CF=0
+    ( 0xb7, 0x85 ), # AL=0x57, AF=0 CF=1
+    ( 0x5d, 0x10 ), # AL=0x57, AF=1 CF=0
+    ( 0xbd, 0x95 ), # AL=0x57, AF=1 CF=1
+    ( 0x58, 0x00 ), # AL=0x58, AF=0 CF=0
+    ( 0xb8, 0x85 ), # AL=0x58, AF=0 CF=1
+    ( 0x5e, 0x10 ), # AL=0x58, AF=1 CF=0
+    ( 0xbe, 0x95 ), # AL=0x58, AF=1 CF=1
+    ( 0x59, 0x04 ), # AL=0x59, AF=0 CF=0
+    ( 0xb9, 0x81 ), # AL=0x59, AF=0 CF=1
+    ( 0x5f, 0x14 ), # AL=0x59, AF=1 CF=0
+    ( 0xbf, 0x91 ), # AL=0x59, AF=1 CF=1
+    ( 0x60, 0x14 ), # AL=0x5a, AF=0 CF=0
+    ( 0xc0, 0x95 ), # AL=0x5a, AF=0 CF=1
+    ( 0x60, 0x14 ), # AL=0x5a, AF=1 CF=0
+    ( 0xc0, 0x95 ), # AL=0x5a, AF=1 CF=1
+    ( 0x61, 0x10 ), # AL=0x5b, AF=0 CF=0
+    ( 0xc1, 0x91 ), # AL=0x5b, AF=0 CF=1
+    ( 0x61, 0x10 ), # AL=0x5b, AF=1 CF=0
+    ( 0xc1, 0x91 ), # AL=0x5b, AF=1 CF=1
+    ( 0x62, 0x10 ), # AL=0x5c, AF=0 CF=0
+    ( 0xc2, 0x91 ), # AL=0x5c, AF=0 CF=1
+    ( 0x62, 0x10 ), # AL=0x5c, AF=1 CF=0
+    ( 0xc2, 0x91 ), # AL=0x5c, AF=1 CF=1
+    ( 0x63, 0x14 ), # AL=0x5d, AF=0 CF=0
+    ( 0xc3, 0x95 ), # AL=0x5d, AF=0 CF=1
+    ( 0x63, 0x14 ), # AL=0x5d, AF=1 CF=0
+    ( 0xc3, 0x95 ), # AL=0x5d, AF=1 CF=1
+    ( 0x64, 0x10 ), # AL=0x5e, AF=0 CF=0
+    ( 0xc4, 0x91 ), # AL=0x5e, AF=0 CF=1
+    ( 0x64, 0x10 ), # AL=0x5e, AF=1 CF=0
+    ( 0xc4, 0x91 ), # AL=0x5e, AF=1 CF=1
+    ( 0x65, 0x14 ), # AL=0x5f, AF=0 CF=0
+    ( 0xc5, 0x95 ), # AL=0x5f, AF=0 CF=1
+    ( 0x65, 0x14 ), # AL=0x5f, AF=1 CF=0
+    ( 0xc5, 0x95 ), # AL=0x5f, AF=1 CF=1
+    ( 0x60, 0x04 ), # AL=0x60, AF=0 CF=0
+    ( 0xc0, 0x85 ), # AL=0x60, AF=0 CF=1
+    ( 0x66, 0x14 ), # AL=0x60, AF=1 CF=0
+    ( 0xc6, 0x95 ), # AL=0x60, AF=1 CF=1
+    ( 0x61, 0x00 ), # AL=0x61, AF=0 CF=0
+    ( 0xc1, 0x81 ), # AL=0x61, AF=0 CF=1
+    ( 0x67, 0x10 ), # AL=0x61, AF=1 CF=0
+    ( 0xc7, 0x91 ), # AL=0x61, AF=1 CF=1
+    ( 0x62, 0x00 ), # AL=0x62, AF=0 CF=0
+    ( 0xc2, 0x81 ), # AL=0x62, AF=0 CF=1
+    ( 0x68, 0x10 ), # AL=0x62, AF=1 CF=0
+    ( 0xc8, 0x91 ), # AL=0x62, AF=1 CF=1
+    ( 0x63, 0x04 ), # AL=0x63, AF=0 CF=0
+    ( 0xc3, 0x85 ), # AL=0x63, AF=0 CF=1
+    ( 0x69, 0x14 ), # AL=0x63, AF=1 CF=0
+    ( 0xc9, 0x95 ), # AL=0x63, AF=1 CF=1
+    ( 0x64, 0x00 ), # AL=0x64, AF=0 CF=0
+    ( 0xc4, 0x81 ), # AL=0x64, AF=0 CF=1
+    ( 0x6a, 0x14 ), # AL=0x64, AF=1 CF=0
+    ( 0xca, 0x95 ), # AL=0x64, AF=1 CF=1
+    ( 0x65, 0x04 ), # AL=0x65, AF=0 CF=0
+    ( 0xc5, 0x85 ), # AL=0x65, AF=0 CF=1
+    ( 0x6b, 0x10 ), # AL=0x65, AF=1 CF=0
+    ( 0xcb, 0x91 ), # AL=0x65, AF=1 CF=1
+    ( 0x66, 0x04 ), # AL=0x66, AF=0 CF=0
+    ( 0xc6, 0x85 ), # AL=0x66, AF=0 CF=1
+    ( 0x6c, 0x14 ), # AL=0x66, AF=1 CF=0
+    ( 0xcc, 0x95 ), # AL=0x66, AF=1 CF=1
+    ( 0x67, 0x00 ), # AL=0x67, AF=0 CF=0
+    ( 0xc7, 0x81 ), # AL=0x67, AF=0 CF=1
+    ( 0x6d, 0x10 ), # AL=0x67, AF=1 CF=0
+    ( 0xcd, 0x91 ), # AL=0x67, AF=1 CF=1
+    ( 0x68, 0x00 ), # AL=0x68, AF=0 CF=0
+    ( 0xc8, 0x81 ), # AL=0x68, AF=0 CF=1
+    ( 0x6e, 0x10 ), # AL=0x68, AF=1 CF=0
+    ( 0xce, 0x91 ), # AL=0x68, AF=1 CF=1
+    ( 0x69, 0x04 ), # AL=0x69, AF=0 CF=0
+    ( 0xc9, 0x85 ), # AL=0x69, AF=0 CF=1
+    ( 0x6f, 0x14 ), # AL=0x69, AF=1 CF=0
+    ( 0xcf, 0x95 ), # AL=0x69, AF=1 CF=1
+    ( 0x70, 0x10 ), # AL=0x6a, AF=0 CF=0
+    ( 0xd0, 0x91 ), # AL=0x6a, AF=0 CF=1
+    ( 0x70, 0x10 ), # AL=0x6a, AF=1 CF=0
+    ( 0xd0, 0x91 ), # AL=0x6a, AF=1 CF=1
+    ( 0x71, 0x14 ), # AL=0x6b, AF=0 CF=0
+    ( 0xd1, 0x95 ), # AL=0x6b, AF=0 CF=1
+    ( 0x71, 0x14 ), # AL=0x6b, AF=1 CF=0
+    ( 0xd1, 0x95 ), # AL=0x6b, AF=1 CF=1
+    ( 0x72, 0x14 ), # AL=0x6c, AF=0 CF=0
+    ( 0xd2, 0x95 ), # AL=0x6c, AF=0 CF=1
+    ( 0x72, 0x14 ), # AL=0x6c, AF=1 CF=0
+    ( 0xd2, 0x95 ), # AL=0x6c, AF=1 CF=1
+    ( 0x73, 0x10 ), # AL=0x6d, AF=0 CF=0
+    ( 0xd3, 0x91 ), # AL=0x6d, AF=0 CF=1
+    ( 0x73, 0x10 ), # AL=0x6d, AF=1 CF=0
+    ( 0xd3, 0x91 ), # AL=0x6d, AF=1 CF=1
+    ( 0x74, 0x14 ), # AL=0x6e, AF=0 CF=0
+    ( 0xd4, 0x95 ), # AL=0x6e, AF=0 CF=1
+    ( 0x74, 0x14 ), # AL=0x6e, AF=1 CF=0
+    ( 0xd4, 0x95 ), # AL=0x6e, AF=1 CF=1
+    ( 0x75, 0x10 ), # AL=0x6f, AF=0 CF=0
+    ( 0xd5, 0x91 ), # AL=0x6f, AF=0 CF=1
+    ( 0x75, 0x10 ), # AL=0x6f, AF=1 CF=0
+    ( 0xd5, 0x91 ), # AL=0x6f, AF=1 CF=1
+    ( 0x70, 0x00 ), # AL=0x70, AF=0 CF=0
+    ( 0xd0, 0x81 ), # AL=0x70, AF=0 CF=1
+    ( 0x76, 0x10 ), # AL=0x70, AF=1 CF=0
+    ( 0xd6, 0x91 ), # AL=0x70, AF=1 CF=1
+    ( 0x71, 0x04 ), # AL=0x71, AF=0 CF=0
+    ( 0xd1, 0x85 ), # AL=0x71, AF=0 CF=1
+    ( 0x77, 0x14 ), # AL=0x71, AF=1 CF=0
+    ( 0xd7, 0x95 ), # AL=0x71, AF=1 CF=1
+    ( 0x72, 0x04 ), # AL=0x72, AF=0 CF=0
+    ( 0xd2, 0x85 ), # AL=0x72, AF=0 CF=1
+    ( 0x78, 0x14 ), # AL=0x72, AF=1 CF=0
+    ( 0xd8, 0x95 ), # AL=0x72, AF=1 CF=1
+    ( 0x73, 0x00 ), # AL=0x73, AF=0 CF=0
+    ( 0xd3, 0x81 ), # AL=0x73, AF=0 CF=1
+    ( 0x79, 0x10 ), # AL=0x73, AF=1 CF=0
+    ( 0xd9, 0x91 ), # AL=0x73, AF=1 CF=1
+    ( 0x74, 0x04 ), # AL=0x74, AF=0 CF=0
+    ( 0xd4, 0x85 ), # AL=0x74, AF=0 CF=1
+    ( 0x7a, 0x10 ), # AL=0x74, AF=1 CF=0
+    ( 0xda, 0x91 ), # AL=0x74, AF=1 CF=1
+    ( 0x75, 0x00 ), # AL=0x75, AF=0 CF=0
+    ( 0xd5, 0x81 ), # AL=0x75, AF=0 CF=1
+    ( 0x7b, 0x14 ), # AL=0x75, AF=1 CF=0
+    ( 0xdb, 0x95 ), # AL=0x75, AF=1 CF=1
+    ( 0x76, 0x00 ), # AL=0x76, AF=0 CF=0
+    ( 0xd6, 0x81 ), # AL=0x76, AF=0 CF=1
+    ( 0x7c, 0x10 ), # AL=0x76, AF=1 CF=0
+    ( 0xdc, 0x91 ), # AL=0x76, AF=1 CF=1
+    ( 0x77, 0x04 ), # AL=0x77, AF=0 CF=0
+    ( 0xd7, 0x85 ), # AL=0x77, AF=0 CF=1
+    ( 0x7d, 0x14 ), # AL=0x77, AF=1 CF=0
+    ( 0xdd, 0x95 ), # AL=0x77, AF=1 CF=1
+    ( 0x78, 0x04 ), # AL=0x78, AF=0 CF=0
+    ( 0xd8, 0x85 ), # AL=0x78, AF=0 CF=1
+    ( 0x7e, 0x14 ), # AL=0x78, AF=1 CF=0
+    ( 0xde, 0x95 ), # AL=0x78, AF=1 CF=1
+    ( 0x79, 0x00 ), # AL=0x79, AF=0 CF=0
+    ( 0xd9, 0x81 ), # AL=0x79, AF=0 CF=1
+    ( 0x7f, 0x10 ), # AL=0x79, AF=1 CF=0
+    ( 0xdf, 0x91 ), # AL=0x79, AF=1 CF=1
+    ( 0x80, 0x90 ), # AL=0x7a, AF=0 CF=0
+    ( 0xe0, 0x91 ), # AL=0x7a, AF=0 CF=1
+    ( 0x80, 0x90 ), # AL=0x7a, AF=1 CF=0
+    ( 0xe0, 0x91 ), # AL=0x7a, AF=1 CF=1
+    ( 0x81, 0x94 ), # AL=0x7b, AF=0 CF=0
+    ( 0xe1, 0x95 ), # AL=0x7b, AF=0 CF=1
+    ( 0x81, 0x94 ), # AL=0x7b, AF=1 CF=0
+    ( 0xe1, 0x95 ), # AL=0x7b, AF=1 CF=1
+    ( 0x82, 0x94 ), # AL=0x7c, AF=0 CF=0
+    ( 0xe2, 0x95 ), # AL=0x7c, AF=0 CF=1
+    ( 0x82, 0x94 ), # AL=0x7c, AF=1 CF=0
+    ( 0xe2, 0x95 ), # AL=0x7c, AF=1 CF=1
+    ( 0x83, 0x90 ), # AL=0x7d, AF=0 CF=0
+    ( 0xe3, 0x91 ), # AL=0x7d, AF=0 CF=1
+    ( 0x83, 0x90 ), # AL=0x7d, AF=1 CF=0
+    ( 0xe3, 0x91 ), # AL=0x7d, AF=1 CF=1
+    ( 0x84, 0x94 ), # AL=0x7e, AF=0 CF=0
+    ( 0xe4, 0x95 ), # AL=0x7e, AF=0 CF=1
+    ( 0x84, 0x94 ), # AL=0x7e, AF=1 CF=0
+    ( 0xe4, 0x95 ), # AL=0x7e, AF=1 CF=1
+    ( 0x85, 0x90 ), # AL=0x7f, AF=0 CF=0
+    ( 0xe5, 0x91 ), # AL=0x7f, AF=0 CF=1
+    ( 0x85, 0x90 ), # AL=0x7f, AF=1 CF=0
+    ( 0xe5, 0x91 ), # AL=0x7f, AF=1 CF=1
+    ( 0x80, 0x80 ), # AL=0x80, AF=0 CF=0
+    ( 0xe0, 0x81 ), # AL=0x80, AF=0 CF=1
+    ( 0x86, 0x90 ), # AL=0x80, AF=1 CF=0
+    ( 0xe6, 0x91 ), # AL=0x80, AF=1 CF=1
+    ( 0x81, 0x84 ), # AL=0x81, AF=0 CF=0
+    ( 0xe1, 0x85 ), # AL=0x81, AF=0 CF=1
+    ( 0x87, 0x94 ), # AL=0x81, AF=1 CF=0
+    ( 0xe7, 0x95 ), # AL=0x81, AF=1 CF=1
+    ( 0x82, 0x84 ), # AL=0x82, AF=0 CF=0
+    ( 0xe2, 0x85 ), # AL=0x82, AF=0 CF=1
+    ( 0x88, 0x94 ), # AL=0x82, AF=1 CF=0
+    ( 0xe8, 0x95 ), # AL=0x82, AF=1 CF=1
+    ( 0x83, 0x80 ), # AL=0x83, AF=0 CF=0
+    ( 0xe3, 0x81 ), # AL=0x83, AF=0 CF=1
+    ( 0x89, 0x90 ), # AL=0x83, AF=1 CF=0
+    ( 0xe9, 0x91 ), # AL=0x83, AF=1 CF=1
+    ( 0x84, 0x84 ), # AL=0x84, AF=0 CF=0
+    ( 0xe4, 0x85 ), # AL=0x84, AF=0 CF=1
+    ( 0x8a, 0x90 ), # AL=0x84, AF=1 CF=0
+    ( 0xea, 0x91 ), # AL=0x84, AF=1 CF=1
+    ( 0x85, 0x80 ), # AL=0x85, AF=0 CF=0
+    ( 0xe5, 0x81 ), # AL=0x85, AF=0 CF=1
+    ( 0x8b, 0x94 ), # AL=0x85, AF=1 CF=0
+    ( 0xeb, 0x95 ), # AL=0x85, AF=1 CF=1
+    ( 0x86, 0x80 ), # AL=0x86, AF=0 CF=0
+    ( 0xe6, 0x81 ), # AL=0x86, AF=0 CF=1
+    ( 0x8c, 0x90 ), # AL=0x86, AF=1 CF=0
+    ( 0xec, 0x91 ), # AL=0x86, AF=1 CF=1
+    ( 0x87, 0x84 ), # AL=0x87, AF=0 CF=0
+    ( 0xe7, 0x85 ), # AL=0x87, AF=0 CF=1
+    ( 0x8d, 0x94 ), # AL=0x87, AF=1 CF=0
+    ( 0xed, 0x95 ), # AL=0x87, AF=1 CF=1
+    ( 0x88, 0x84 ), # AL=0x88, AF=0 CF=0
+    ( 0xe8, 0x85 ), # AL=0x88, AF=0 CF=1
+    ( 0x8e, 0x94 ), # AL=0x88, AF=1 CF=0
+    ( 0xee, 0x95 ), # AL=0x88, AF=1 CF=1
+    ( 0x89, 0x80 ), # AL=0x89, AF=0 CF=0
+    ( 0xe9, 0x81 ), # AL=0x89, AF=0 CF=1
+    ( 0x8f, 0x90 ), # AL=0x89, AF=1 CF=0
+    ( 0xef, 0x91 ), # AL=0x89, AF=1 CF=1
+    ( 0x90, 0x94 ), # AL=0x8a, AF=0 CF=0
+    ( 0xf0, 0x95 ), # AL=0x8a, AF=0 CF=1
+    ( 0x90, 0x94 ), # AL=0x8a, AF=1 CF=0
+    ( 0xf0, 0x95 ), # AL=0x8a, AF=1 CF=1
+    ( 0x91, 0x90 ), # AL=0x8b, AF=0 CF=0
+    ( 0xf1, 0x91 ), # AL=0x8b, AF=0 CF=1
+    ( 0x91, 0x90 ), # AL=0x8b, AF=1 CF=0
+    ( 0xf1, 0x91 ), # AL=0x8b, AF=1 CF=1
+    ( 0x92, 0x90 ), # AL=0x8c, AF=0 CF=0
+    ( 0xf2, 0x91 ), # AL=0x8c, AF=0 CF=1
+    ( 0x92, 0x90 ), # AL=0x8c, AF=1 CF=0
+    ( 0xf2, 0x91 ), # AL=0x8c, AF=1 CF=1
+    ( 0x93, 0x94 ), # AL=0x8d, AF=0 CF=0
+    ( 0xf3, 0x95 ), # AL=0x8d, AF=0 CF=1
+    ( 0x93, 0x94 ), # AL=0x8d, AF=1 CF=0
+    ( 0xf3, 0x95 ), # AL=0x8d, AF=1 CF=1
+    ( 0x94, 0x90 ), # AL=0x8e, AF=0 CF=0
+    ( 0xf4, 0x91 ), # AL=0x8e, AF=0 CF=1
+    ( 0x94, 0x90 ), # AL=0x8e, AF=1 CF=0
+    ( 0xf4, 0x91 ), # AL=0x8e, AF=1 CF=1
+    ( 0x95, 0x94 ), # AL=0x8f, AF=0 CF=0
+    ( 0xf5, 0x95 ), # AL=0x8f, AF=0 CF=1
+    ( 0x95, 0x94 ), # AL=0x8f, AF=1 CF=0
+    ( 0xf5, 0x95 ), # AL=0x8f, AF=1 CF=1
+    ( 0x90, 0x84 ), # AL=0x90, AF=0 CF=0
+    ( 0xf0, 0x85 ), # AL=0x90, AF=0 CF=1
+    ( 0x96, 0x94 ), # AL=0x90, AF=1 CF=0
+    ( 0xf6, 0x95 ), # AL=0x90, AF=1 CF=1
+    ( 0x91, 0x80 ), # AL=0x91, AF=0 CF=0
+    ( 0xf1, 0x81 ), # AL=0x91, AF=0 CF=1
+    ( 0x97, 0x90 ), # AL=0x91, AF=1 CF=0
+    ( 0xf7, 0x91 ), # AL=0x91, AF=1 CF=1
+    ( 0x92, 0x80 ), # AL=0x92, AF=0 CF=0
+    ( 0xf2, 0x81 ), # AL=0x92, AF=0 CF=1
+    ( 0x98, 0x90 ), # AL=0x92, AF=1 CF=0
+    ( 0xf8, 0x91 ), # AL=0x92, AF=1 CF=1
+    ( 0x93, 0x84 ), # AL=0x93, AF=0 CF=0
+    ( 0xf3, 0x85 ), # AL=0x93, AF=0 CF=1
+    ( 0x99, 0x94 ), # AL=0x93, AF=1 CF=0
+    ( 0xf9, 0x95 ), # AL=0x93, AF=1 CF=1
+    ( 0x94, 0x80 ), # AL=0x94, AF=0 CF=0
+    ( 0xf4, 0x81 ), # AL=0x94, AF=0 CF=1
+    ( 0x9a, 0x94 ), # AL=0x94, AF=1 CF=0
+    ( 0xfa, 0x95 ), # AL=0x94, AF=1 CF=1
+    ( 0x95, 0x84 ), # AL=0x95, AF=0 CF=0
+    ( 0xf5, 0x85 ), # AL=0x95, AF=0 CF=1
+    ( 0x9b, 0x90 ), # AL=0x95, AF=1 CF=0
+    ( 0xfb, 0x91 ), # AL=0x95, AF=1 CF=1
+    ( 0x96, 0x84 ), # AL=0x96, AF=0 CF=0
+    ( 0xf6, 0x85 ), # AL=0x96, AF=0 CF=1
+    ( 0x9c, 0x94 ), # AL=0x96, AF=1 CF=0
+    ( 0xfc, 0x95 ), # AL=0x96, AF=1 CF=1
+    ( 0x97, 0x80 ), # AL=0x97, AF=0 CF=0
+    ( 0xf7, 0x81 ), # AL=0x97, AF=0 CF=1
+    ( 0x9d, 0x90 ), # AL=0x97, AF=1 CF=0
+    ( 0xfd, 0x91 ), # AL=0x97, AF=1 CF=1
+    ( 0x98, 0x80 ), # AL=0x98, AF=0 CF=0
+    ( 0xf8, 0x81 ), # AL=0x98, AF=0 CF=1
+    ( 0x9e, 0x90 ), # AL=0x98, AF=1 CF=0
+    ( 0xfe, 0x91 ), # AL=0x98, AF=1 CF=1
+    ( 0x99, 0x84 ), # AL=0x99, AF=0 CF=0
+    ( 0xf9, 0x85 ), # AL=0x99, AF=0 CF=1
+    ( 0x9f, 0x94 ), # AL=0x99, AF=1 CF=0
+    ( 0xff, 0x95 ), # AL=0x99, AF=1 CF=1
+    ( 0x00, 0x55 ), # AL=0x9a, AF=0 CF=0
+    ( 0x00, 0x55 ), # AL=0x9a, AF=0 CF=1
+    ( 0x00, 0x55 ), # AL=0x9a, AF=1 CF=0
+    ( 0x00, 0x55 ), # AL=0x9a, AF=1 CF=1
+    ( 0x01, 0x11 ), # AL=0x9b, AF=0 CF=0
+    ( 0x01, 0x11 ), # AL=0x9b, AF=0 CF=1
+    ( 0x01, 0x11 ), # AL=0x9b, AF=1 CF=0
+    ( 0x01, 0x11 ), # AL=0x9b, AF=1 CF=1
+    ( 0x02, 0x11 ), # AL=0x9c, AF=0 CF=0
+    ( 0x02, 0x11 ), # AL=0x9c, AF=0 CF=1
+    ( 0x02, 0x11 ), # AL=0x9c, AF=1 CF=0
+    ( 0x02, 0x11 ), # AL=0x9c, AF=1 CF=1
+    ( 0x03, 0x15 ), # AL=0x9d, AF=0 CF=0
+    ( 0x03, 0x15 ), # AL=0x9d, AF=0 CF=1
+    ( 0x03, 0x15 ), # AL=0x9d, AF=1 CF=0
+    ( 0x03, 0x15 ), # AL=0x9d, AF=1 CF=1
+    ( 0x04, 0x11 ), # AL=0x9e, AF=0 CF=0
+    ( 0x04, 0x11 ), # AL=0x9e, AF=0 CF=1
+    ( 0x04, 0x11 ), # AL=0x9e, AF=1 CF=0
+    ( 0x04, 0x11 ), # AL=0x9e, AF=1 CF=1
+    ( 0x05, 0x15 ), # AL=0x9f, AF=0 CF=0
+    ( 0x05, 0x15 ), # AL=0x9f, AF=0 CF=1
+    ( 0x05, 0x15 ), # AL=0x9f, AF=1 CF=0
+    ( 0x05, 0x15 ), # AL=0x9f, AF=1 CF=1
+    ( 0x00, 0x45 ), # AL=0xa0, AF=0 CF=0
+    ( 0x00, 0x45 ), # AL=0xa0, AF=0 CF=1
+    ( 0x06, 0x15 ), # AL=0xa0, AF=1 CF=0
+    ( 0x06, 0x15 ), # AL=0xa0, AF=1 CF=1
+    ( 0x01, 0x01 ), # AL=0xa1, AF=0 CF=0
+    ( 0x01, 0x01 ), # AL=0xa1, AF=0 CF=1
+    ( 0x07, 0x11 ), # AL=0xa1, AF=1 CF=0
+    ( 0x07, 0x11 ), # AL=0xa1, AF=1 CF=1
+    ( 0x02, 0x01 ), # AL=0xa2, AF=0 CF=0
+    ( 0x02, 0x01 ), # AL=0xa2, AF=0 CF=1
+    ( 0x08, 0x11 ), # AL=0xa2, AF=1 CF=0
+    ( 0x08, 0x11 ), # AL=0xa2, AF=1 CF=1
+    ( 0x03, 0x05 ), # AL=0xa3, AF=0 CF=0
+    ( 0x03, 0x05 ), # AL=0xa3, AF=0 CF=1
+    ( 0x09, 0x15 ), # AL=0xa3, AF=1 CF=0
+    ( 0x09, 0x15 ), # AL=0xa3, AF=1 CF=1
+    ( 0x04, 0x01 ), # AL=0xa4, AF=0 CF=0
+    ( 0x04, 0x01 ), # AL=0xa4, AF=0 CF=1
+    ( 0x0a, 0x15 ), # AL=0xa4, AF=1 CF=0
+    ( 0x0a, 0x15 ), # AL=0xa4, AF=1 CF=1
+    ( 0x05, 0x05 ), # AL=0xa5, AF=0 CF=0
+    ( 0x05, 0x05 ), # AL=0xa5, AF=0 CF=1
+    ( 0x0b, 0x11 ), # AL=0xa5, AF=1 CF=0
+    ( 0x0b, 0x11 ), # AL=0xa5, AF=1 CF=1
+    ( 0x06, 0x05 ), # AL=0xa6, AF=0 CF=0
+    ( 0x06, 0x05 ), # AL=0xa6, AF=0 CF=1
+    ( 0x0c, 0x15 ), # AL=0xa6, AF=1 CF=0
+    ( 0x0c, 0x15 ), # AL=0xa6, AF=1 CF=1
+    ( 0x07, 0x01 ), # AL=0xa7, AF=0 CF=0
+    ( 0x07, 0x01 ), # AL=0xa7, AF=0 CF=1
+    ( 0x0d, 0x11 ), # AL=0xa7, AF=1 CF=0
+    ( 0x0d, 0x11 ), # AL=0xa7, AF=1 CF=1
+    ( 0x08, 0x01 ), # AL=0xa8, AF=0 CF=0
+    ( 0x08, 0x01 ), # AL=0xa8, AF=0 CF=1
+    ( 0x0e, 0x11 ), # AL=0xa8, AF=1 CF=0
+    ( 0x0e, 0x11 ), # AL=0xa8, AF=1 CF=1
+    ( 0x09, 0x05 ), # AL=0xa9, AF=0 CF=0
+    ( 0x09, 0x05 ), # AL=0xa9, AF=0 CF=1
+    ( 0x0f, 0x15 ), # AL=0xa9, AF=1 CF=0
+    ( 0x0f, 0x15 ), # AL=0xa9, AF=1 CF=1
+    ( 0x10, 0x11 ), # AL=0xaa, AF=0 CF=0
+    ( 0x10, 0x11 ), # AL=0xaa, AF=0 CF=1
+    ( 0x10, 0x11 ), # AL=0xaa, AF=1 CF=0
+    ( 0x10, 0x11 ), # AL=0xaa, AF=1 CF=1
+    ( 0x11, 0x15 ), # AL=0xab, AF=0 CF=0
+    ( 0x11, 0x15 ), # AL=0xab, AF=0 CF=1
+    ( 0x11, 0x15 ), # AL=0xab, AF=1 CF=0
+    ( 0x11, 0x15 ), # AL=0xab, AF=1 CF=1
+    ( 0x12, 0x15 ), # AL=0xac, AF=0 CF=0
+    ( 0x12, 0x15 ), # AL=0xac, AF=0 CF=1
+    ( 0x12, 0x15 ), # AL=0xac, AF=1 CF=0
+    ( 0x12, 0x15 ), # AL=0xac, AF=1 CF=1
+    ( 0x13, 0x11 ), # AL=0xad, AF=0 CF=0
+    ( 0x13, 0x11 ), # AL=0xad, AF=0 CF=1
+    ( 0x13, 0x11 ), # AL=0xad, AF=1 CF=0
+    ( 0x13, 0x11 ), # AL=0xad, AF=1 CF=1
+    ( 0x14, 0x15 ), # AL=0xae, AF=0 CF=0
+    ( 0x14, 0x15 ), # AL=0xae, AF=0 CF=1
+    ( 0x14, 0x15 ), # AL=0xae, AF=1 CF=0
+    ( 0x14, 0x15 ), # AL=0xae, AF=1 CF=1
+    ( 0x15, 0x11 ), # AL=0xaf, AF=0 CF=0
+    ( 0x15, 0x11 ), # AL=0xaf, AF=0 CF=1
+    ( 0x15, 0x11 ), # AL=0xaf, AF=1 CF=0
+    ( 0x15, 0x11 ), # AL=0xaf, AF=1 CF=1
+    ( 0x10, 0x01 ), # AL=0xb0, AF=0 CF=0
+    ( 0x10, 0x01 ), # AL=0xb0, AF=0 CF=1
+    ( 0x16, 0x11 ), # AL=0xb0, AF=1 CF=0
+    ( 0x16, 0x11 ), # AL=0xb0, AF=1 CF=1
+    ( 0x11, 0x05 ), # AL=0xb1, AF=0 CF=0
+    ( 0x11, 0x05 ), # AL=0xb1, AF=0 CF=1
+    ( 0x17, 0x15 ), # AL=0xb1, AF=1 CF=0
+    ( 0x17, 0x15 ), # AL=0xb1, AF=1 CF=1
+    ( 0x12, 0x05 ), # AL=0xb2, AF=0 CF=0
+    ( 0x12, 0x05 ), # AL=0xb2, AF=0 CF=1
+    ( 0x18, 0x15 ), # AL=0xb2, AF=1 CF=0
+    ( 0x18, 0x15 ), # AL=0xb2, AF=1 CF=1
+    ( 0x13, 0x01 ), # AL=0xb3, AF=0 CF=0
+    ( 0x13, 0x01 ), # AL=0xb3, AF=0 CF=1
+    ( 0x19, 0x11 ), # AL=0xb3, AF=1 CF=0
+    ( 0x19, 0x11 ), # AL=0xb3, AF=1 CF=1
+    ( 0x14, 0x05 ), # AL=0xb4, AF=0 CF=0
+    ( 0x14, 0x05 ), # AL=0xb4, AF=0 CF=1
+    ( 0x1a, 0x11 ), # AL=0xb4, AF=1 CF=0
+    ( 0x1a, 0x11 ), # AL=0xb4, AF=1 CF=1
+    ( 0x15, 0x01 ), # AL=0xb5, AF=0 CF=0
+    ( 0x15, 0x01 ), # AL=0xb5, AF=0 CF=1
+    ( 0x1b, 0x15 ), # AL=0xb5, AF=1 CF=0
+    ( 0x1b, 0x15 ), # AL=0xb5, AF=1 CF=1
+    ( 0x16, 0x01 ), # AL=0xb6, AF=0 CF=0
+    ( 0x16, 0x01 ), # AL=0xb6, AF=0 CF=1
+    ( 0x1c, 0x11 ), # AL=0xb6, AF=1 CF=0
+    ( 0x1c, 0x11 ), # AL=0xb6, AF=1 CF=1
+    ( 0x17, 0x05 ), # AL=0xb7, AF=0 CF=0
+    ( 0x17, 0x05 ), # AL=0xb7, AF=0 CF=1
+    ( 0x1d, 0x15 ), # AL=0xb7, AF=1 CF=0
+    ( 0x1d, 0x15 ), # AL=0xb7, AF=1 CF=1
+    ( 0x18, 0x05 ), # AL=0xb8, AF=0 CF=0
+    ( 0x18, 0x05 ), # AL=0xb8, AF=0 CF=1
+    ( 0x1e, 0x15 ), # AL=0xb8, AF=1 CF=0
+    ( 0x1e, 0x15 ), # AL=0xb8, AF=1 CF=1
+    ( 0x19, 0x01 ), # AL=0xb9, AF=0 CF=0
+    ( 0x19, 0x01 ), # AL=0xb9, AF=0 CF=1
+    ( 0x1f, 0x11 ), # AL=0xb9, AF=1 CF=0
+    ( 0x1f, 0x11 ), # AL=0xb9, AF=1 CF=1
+    ( 0x20, 0x11 ), # AL=0xba, AF=0 CF=0
+    ( 0x20, 0x11 ), # AL=0xba, AF=0 CF=1
+    ( 0x20, 0x11 ), # AL=0xba, AF=1 CF=0
+    ( 0x20, 0x11 ), # AL=0xba, AF=1 CF=1
+    ( 0x21, 0x15 ), # AL=0xbb, AF=0 CF=0
+    ( 0x21, 0x15 ), # AL=0xbb, AF=0 CF=1
+    ( 0x21, 0x15 ), # AL=0xbb, AF=1 CF=0
+    ( 0x21, 0x15 ), # AL=0xbb, AF=1 CF=1
+    ( 0x22, 0x15 ), # AL=0xbc, AF=0 CF=0
+    ( 0x22, 0x15 ), # AL=0xbc, AF=0 CF=1
+    ( 0x22, 0x15 ), # AL=0xbc, AF=1 CF=0
+    ( 0x22, 0x15 ), # AL=0xbc, AF=1 CF=1
+    ( 0x23, 0x11 ), # AL=0xbd, AF=0 CF=0
+    ( 0x23, 0x11 ), # AL=0xbd, AF=0 CF=1
+    ( 0x23, 0x11 ), # AL=0xbd, AF=1 CF=0
+    ( 0x23, 0x11 ), # AL=0xbd, AF=1 CF=1
+    ( 0x24, 0x15 ), # AL=0xbe, AF=0 CF=0
+    ( 0x24, 0x15 ), # AL=0xbe, AF=0 CF=1
+    ( 0x24, 0x15 ), # AL=0xbe, AF=1 CF=0
+    ( 0x24, 0x15 ), # AL=0xbe, AF=1 CF=1
+    ( 0x25, 0x11 ), # AL=0xbf, AF=0 CF=0
+    ( 0x25, 0x11 ), # AL=0xbf, AF=0 CF=1
+    ( 0x25, 0x11 ), # AL=0xbf, AF=1 CF=0
+    ( 0x25, 0x11 ), # AL=0xbf, AF=1 CF=1
+    ( 0x20, 0x01 ), # AL=0xc0, AF=0 CF=0
+    ( 0x20, 0x01 ), # AL=0xc0, AF=0 CF=1
+    ( 0x26, 0x11 ), # AL=0xc0, AF=1 CF=0
+    ( 0x26, 0x11 ), # AL=0xc0, AF=1 CF=1
+    ( 0x21, 0x05 ), # AL=0xc1, AF=0 CF=0
+    ( 0x21, 0x05 ), # AL=0xc1, AF=0 CF=1
+    ( 0x27, 0x15 ), # AL=0xc1, AF=1 CF=0
+    ( 0x27, 0x15 ), # AL=0xc1, AF=1 CF=1
+    ( 0x22, 0x05 ), # AL=0xc2, AF=0 CF=0
+    ( 0x22, 0x05 ), # AL=0xc2, AF=0 CF=1
+    ( 0x28, 0x15 ), # AL=0xc2, AF=1 CF=0
+    ( 0x28, 0x15 ), # AL=0xc2, AF=1 CF=1
+    ( 0x23, 0x01 ), # AL=0xc3, AF=0 CF=0
+    ( 0x23, 0x01 ), # AL=0xc3, AF=0 CF=1
+    ( 0x29, 0x11 ), # AL=0xc3, AF=1 CF=0
+    ( 0x29, 0x11 ), # AL=0xc3, AF=1 CF=1
+    ( 0x24, 0x05 ), # AL=0xc4, AF=0 CF=0
+    ( 0x24, 0x05 ), # AL=0xc4, AF=0 CF=1
+    ( 0x2a, 0x11 ), # AL=0xc4, AF=1 CF=0
+    ( 0x2a, 0x11 ), # AL=0xc4, AF=1 CF=1
+    ( 0x25, 0x01 ), # AL=0xc5, AF=0 CF=0
+    ( 0x25, 0x01 ), # AL=0xc5, AF=0 CF=1
+    ( 0x2b, 0x15 ), # AL=0xc5, AF=1 CF=0
+    ( 0x2b, 0x15 ), # AL=0xc5, AF=1 CF=1
+    ( 0x26, 0x01 ), # AL=0xc6, AF=0 CF=0
+    ( 0x26, 0x01 ), # AL=0xc6, AF=0 CF=1
+    ( 0x2c, 0x11 ), # AL=0xc6, AF=1 CF=0
+    ( 0x2c, 0x11 ), # AL=0xc6, AF=1 CF=1
+    ( 0x27, 0x05 ), # AL=0xc7, AF=0 CF=0
+    ( 0x27, 0x05 ), # AL=0xc7, AF=0 CF=1
+    ( 0x2d, 0x15 ), # AL=0xc7, AF=1 CF=0
+    ( 0x2d, 0x15 ), # AL=0xc7, AF=1 CF=1
+    ( 0x28, 0x05 ), # AL=0xc8, AF=0 CF=0
+    ( 0x28, 0x05 ), # AL=0xc8, AF=0 CF=1
+    ( 0x2e, 0x15 ), # AL=0xc8, AF=1 CF=0
+    ( 0x2e, 0x15 ), # AL=0xc8, AF=1 CF=1
+    ( 0x29, 0x01 ), # AL=0xc9, AF=0 CF=0
+    ( 0x29, 0x01 ), # AL=0xc9, AF=0 CF=1
+    ( 0x2f, 0x11 ), # AL=0xc9, AF=1 CF=0
+    ( 0x2f, 0x11 ), # AL=0xc9, AF=1 CF=1
+    ( 0x30, 0x15 ), # AL=0xca, AF=0 CF=0
+    ( 0x30, 0x15 ), # AL=0xca, AF=0 CF=1
+    ( 0x30, 0x15 ), # AL=0xca, AF=1 CF=0
+    ( 0x30, 0x15 ), # AL=0xca, AF=1 CF=1
+    ( 0x31, 0x11 ), # AL=0xcb, AF=0 CF=0
+    ( 0x31, 0x11 ), # AL=0xcb, AF=0 CF=1
+    ( 0x31, 0x11 ), # AL=0xcb, AF=1 CF=0
+    ( 0x31, 0x11 ), # AL=0xcb, AF=1 CF=1
+    ( 0x32, 0x11 ), # AL=0xcc, AF=0 CF=0
+    ( 0x32, 0x11 ), # AL=0xcc, AF=0 CF=1
+    ( 0x32, 0x11 ), # AL=0xcc, AF=1 CF=0
+    ( 0x32, 0x11 ), # AL=0xcc, AF=1 CF=1
+    ( 0x33, 0x15 ), # AL=0xcd, AF=0 CF=0
+    ( 0x33, 0x15 ), # AL=0xcd, AF=0 CF=1
+    ( 0x33, 0x15 ), # AL=0xcd, AF=1 CF=0
+    ( 0x33, 0x15 ), # AL=0xcd, AF=1 CF=1
+    ( 0x34, 0x11 ), # AL=0xce, AF=0 CF=0
+    ( 0x34, 0x11 ), # AL=0xce, AF=0 CF=1
+    ( 0x34, 0x11 ), # AL=0xce, AF=1 CF=0
+    ( 0x34, 0x11 ), # AL=0xce, AF=1 CF=1
+    ( 0x35, 0x15 ), # AL=0xcf, AF=0 CF=0
+    ( 0x35, 0x15 ), # AL=0xcf, AF=0 CF=1
+    ( 0x35, 0x15 ), # AL=0xcf, AF=1 CF=0
+    ( 0x35, 0x15 ), # AL=0xcf, AF=1 CF=1
+    ( 0x30, 0x05 ), # AL=0xd0, AF=0 CF=0
+    ( 0x30, 0x05 ), # AL=0xd0, AF=0 CF=1
+    ( 0x36, 0x15 ), # AL=0xd0, AF=1 CF=0
+    ( 0x36, 0x15 ), # AL=0xd0, AF=1 CF=1
+    ( 0x31, 0x01 ), # AL=0xd1, AF=0 CF=0
+    ( 0x31, 0x01 ), # AL=0xd1, AF=0 CF=1
+    ( 0x37, 0x11 ), # AL=0xd1, AF=1 CF=0
+    ( 0x37, 0x11 ), # AL=0xd1, AF=1 CF=1
+    ( 0x32, 0x01 ), # AL=0xd2, AF=0 CF=0
+    ( 0x32, 0x01 ), # AL=0xd2, AF=0 CF=1
+    ( 0x38, 0x11 ), # AL=0xd2, AF=1 CF=0
+    ( 0x38, 0x11 ), # AL=0xd2, AF=1 CF=1
+    ( 0x33, 0x05 ), # AL=0xd3, AF=0 CF=0
+    ( 0x33, 0x05 ), # AL=0xd3, AF=0 CF=1
+    ( 0x39, 0x15 ), # AL=0xd3, AF=1 CF=0
+    ( 0x39, 0x15 ), # AL=0xd3, AF=1 CF=1
+    ( 0x34, 0x01 ), # AL=0xd4, AF=0 CF=0
+    ( 0x34, 0x01 ), # AL=0xd4, AF=0 CF=1
+    ( 0x3a, 0x15 ), # AL=0xd4, AF=1 CF=0
+    ( 0x3a, 0x15 ), # AL=0xd4, AF=1 CF=1
+    ( 0x35, 0x05 ), # AL=0xd5, AF=0 CF=0
+    ( 0x35, 0x05 ), # AL=0xd5, AF=0 CF=1
+    ( 0x3b, 0x11 ), # AL=0xd5, AF=1 CF=0
+    ( 0x3b, 0x11 ), # AL=0xd5, AF=1 CF=1
+    ( 0x36, 0x05 ), # AL=0xd6, AF=0 CF=0
+    ( 0x36, 0x05 ), # AL=0xd6, AF=0 CF=1
+    ( 0x3c, 0x15 ), # AL=0xd6, AF=1 CF=0
+    ( 0x3c, 0x15 ), # AL=0xd6, AF=1 CF=1
+    ( 0x37, 0x01 ), # AL=0xd7, AF=0 CF=0
+    ( 0x37, 0x01 ), # AL=0xd7, AF=0 CF=1
+    ( 0x3d, 0x11 ), # AL=0xd7, AF=1 CF=0
+    ( 0x3d, 0x11 ), # AL=0xd7, AF=1 CF=1
+    ( 0x38, 0x01 ), # AL=0xd8, AF=0 CF=0
+    ( 0x38, 0x01 ), # AL=0xd8, AF=0 CF=1
+    ( 0x3e, 0x11 ), # AL=0xd8, AF=1 CF=0
+    ( 0x3e, 0x11 ), # AL=0xd8, AF=1 CF=1
+    ( 0x39, 0x05 ), # AL=0xd9, AF=0 CF=0
+    ( 0x39, 0x05 ), # AL=0xd9, AF=0 CF=1
+    ( 0x3f, 0x15 ), # AL=0xd9, AF=1 CF=0
+    ( 0x3f, 0x15 ), # AL=0xd9, AF=1 CF=1
+    ( 0x40, 0x11 ), # AL=0xda, AF=0 CF=0
+    ( 0x40, 0x11 ), # AL=0xda, AF=0 CF=1
+    ( 0x40, 0x11 ), # AL=0xda, AF=1 CF=0
+    ( 0x40, 0x11 ), # AL=0xda, AF=1 CF=1
+    ( 0x41, 0x15 ), # AL=0xdb, AF=0 CF=0
+    ( 0x41, 0x15 ), # AL=0xdb, AF=0 CF=1
+    ( 0x41, 0x15 ), # AL=0xdb, AF=1 CF=0
+    ( 0x41, 0x15 ), # AL=0xdb, AF=1 CF=1
+    ( 0x42, 0x15 ), # AL=0xdc, AF=0 CF=0
+    ( 0x42, 0x15 ), # AL=0xdc, AF=0 CF=1
+    ( 0x42, 0x15 ), # AL=0xdc, AF=1 CF=0
+    ( 0x42, 0x15 ), # AL=0xdc, AF=1 CF=1
+    ( 0x43, 0x11 ), # AL=0xdd, AF=0 CF=0
+    ( 0x43, 0x11 ), # AL=0xdd, AF=0 CF=1
+    ( 0x43, 0x11 ), # AL=0xdd, AF=1 CF=0
+    ( 0x43, 0x11 ), # AL=0xdd, AF=1 CF=1
+    ( 0x44, 0x15 ), # AL=0xde, AF=0 CF=0
+    ( 0x44, 0x15 ), # AL=0xde, AF=0 CF=1
+    ( 0x44, 0x15 ), # AL=0xde, AF=1 CF=0
+    ( 0x44, 0x15 ), # AL=0xde, AF=1 CF=1
+    ( 0x45, 0x11 ), # AL=0xdf, AF=0 CF=0
+    ( 0x45, 0x11 ), # AL=0xdf, AF=0 CF=1
+    ( 0x45, 0x11 ), # AL=0xdf, AF=1 CF=0
+    ( 0x45, 0x11 ), # AL=0xdf, AF=1 CF=1
+    ( 0x40, 0x01 ), # AL=0xe0, AF=0 CF=0
+    ( 0x40, 0x01 ), # AL=0xe0, AF=0 CF=1
+    ( 0x46, 0x11 ), # AL=0xe0, AF=1 CF=0
+    ( 0x46, 0x11 ), # AL=0xe0, AF=1 CF=1
+    ( 0x41, 0x05 ), # AL=0xe1, AF=0 CF=0
+    ( 0x41, 0x05 ), # AL=0xe1, AF=0 CF=1
+    ( 0x47, 0x15 ), # AL=0xe1, AF=1 CF=0
+    ( 0x47, 0x15 ), # AL=0xe1, AF=1 CF=1
+    ( 0x42, 0x05 ), # AL=0xe2, AF=0 CF=0
+    ( 0x42, 0x05 ), # AL=0xe2, AF=0 CF=1
+    ( 0x48, 0x15 ), # AL=0xe2, AF=1 CF=0
+    ( 0x48, 0x15 ), # AL=0xe2, AF=1 CF=1
+    ( 0x43, 0x01 ), # AL=0xe3, AF=0 CF=0
+    ( 0x43, 0x01 ), # AL=0xe3, AF=0 CF=1
+    ( 0x49, 0x11 ), # AL=0xe3, AF=1 CF=0
+    ( 0x49, 0x11 ), # AL=0xe3, AF=1 CF=1
+    ( 0x44, 0x05 ), # AL=0xe4, AF=0 CF=0
+    ( 0x44, 0x05 ), # AL=0xe4, AF=0 CF=1
+    ( 0x4a, 0x11 ), # AL=0xe4, AF=1 CF=0
+    ( 0x4a, 0x11 ), # AL=0xe4, AF=1 CF=1
+    ( 0x45, 0x01 ), # AL=0xe5, AF=0 CF=0
+    ( 0x45, 0x01 ), # AL=0xe5, AF=0 CF=1
+    ( 0x4b, 0x15 ), # AL=0xe5, AF=1 CF=0
+    ( 0x4b, 0x15 ), # AL=0xe5, AF=1 CF=1
+    ( 0x46, 0x01 ), # AL=0xe6, AF=0 CF=0
+    ( 0x46, 0x01 ), # AL=0xe6, AF=0 CF=1
+    ( 0x4c, 0x11 ), # AL=0xe6, AF=1 CF=0
+    ( 0x4c, 0x11 ), # AL=0xe6, AF=1 CF=1
+    ( 0x47, 0x05 ), # AL=0xe7, AF=0 CF=0
+    ( 0x47, 0x05 ), # AL=0xe7, AF=0 CF=1
+    ( 0x4d, 0x15 ), # AL=0xe7, AF=1 CF=0
+    ( 0x4d, 0x15 ), # AL=0xe7, AF=1 CF=1
+    ( 0x48, 0x05 ), # AL=0xe8, AF=0 CF=0
+    ( 0x48, 0x05 ), # AL=0xe8, AF=0 CF=1
+    ( 0x4e, 0x15 ), # AL=0xe8, AF=1 CF=0
+    ( 0x4e, 0x15 ), # AL=0xe8, AF=1 CF=1
+    ( 0x49, 0x01 ), # AL=0xe9, AF=0 CF=0
+    ( 0x49, 0x01 ), # AL=0xe9, AF=0 CF=1
+    ( 0x4f, 0x11 ), # AL=0xe9, AF=1 CF=0
+    ( 0x4f, 0x11 ), # AL=0xe9, AF=1 CF=1
+    ( 0x50, 0x15 ), # AL=0xea, AF=0 CF=0
+    ( 0x50, 0x15 ), # AL=0xea, AF=0 CF=1
+    ( 0x50, 0x15 ), # AL=0xea, AF=1 CF=0
+    ( 0x50, 0x15 ), # AL=0xea, AF=1 CF=1
+    ( 0x51, 0x11 ), # AL=0xeb, AF=0 CF=0
+    ( 0x51, 0x11 ), # AL=0xeb, AF=0 CF=1
+    ( 0x51, 0x11 ), # AL=0xeb, AF=1 CF=0
+    ( 0x51, 0x11 ), # AL=0xeb, AF=1 CF=1
+    ( 0x52, 0x11 ), # AL=0xec, AF=0 CF=0
+    ( 0x52, 0x11 ), # AL=0xec, AF=0 CF=1
+    ( 0x52, 0x11 ), # AL=0xec, AF=1 CF=0
+    ( 0x52, 0x11 ), # AL=0xec, AF=1 CF=1
+    ( 0x53, 0x15 ), # AL=0xed, AF=0 CF=0
+    ( 0x53, 0x15 ), # AL=0xed, AF=0 CF=1
+    ( 0x53, 0x15 ), # AL=0xed, AF=1 CF=0
+    ( 0x53, 0x15 ), # AL=0xed, AF=1 CF=1
+    ( 0x54, 0x11 ), # AL=0xee, AF=0 CF=0
+    ( 0x54, 0x11 ), # AL=0xee, AF=0 CF=1
+    ( 0x54, 0x11 ), # AL=0xee, AF=1 CF=0
+    ( 0x54, 0x11 ), # AL=0xee, AF=1 CF=1
+    ( 0x55, 0x15 ), # AL=0xef, AF=0 CF=0
+    ( 0x55, 0x15 ), # AL=0xef, AF=0 CF=1
+    ( 0x55, 0x15 ), # AL=0xef, AF=1 CF=0
+    ( 0x55, 0x15 ), # AL=0xef, AF=1 CF=1
+    ( 0x50, 0x05 ), # AL=0xf0, AF=0 CF=0
+    ( 0x50, 0x05 ), # AL=0xf0, AF=0 CF=1
+    ( 0x56, 0x15 ), # AL=0xf0, AF=1 CF=0
+    ( 0x56, 0x15 ), # AL=0xf0, AF=1 CF=1
+    ( 0x51, 0x01 ), # AL=0xf1, AF=0 CF=0
+    ( 0x51, 0x01 ), # AL=0xf1, AF=0 CF=1
+    ( 0x57, 0x11 ), # AL=0xf1, AF=1 CF=0
+    ( 0x57, 0x11 ), # AL=0xf1, AF=1 CF=1
+    ( 0x52, 0x01 ), # AL=0xf2, AF=0 CF=0
+    ( 0x52, 0x01 ), # AL=0xf2, AF=0 CF=1
+    ( 0x58, 0x11 ), # AL=0xf2, AF=1 CF=0
+    ( 0x58, 0x11 ), # AL=0xf2, AF=1 CF=1
+    ( 0x53, 0x05 ), # AL=0xf3, AF=0 CF=0
+    ( 0x53, 0x05 ), # AL=0xf3, AF=0 CF=1
+    ( 0x59, 0x15 ), # AL=0xf3, AF=1 CF=0
+    ( 0x59, 0x15 ), # AL=0xf3, AF=1 CF=1
+    ( 0x54, 0x01 ), # AL=0xf4, AF=0 CF=0
+    ( 0x54, 0x01 ), # AL=0xf4, AF=0 CF=1
+    ( 0x5a, 0x15 ), # AL=0xf4, AF=1 CF=0
+    ( 0x5a, 0x15 ), # AL=0xf4, AF=1 CF=1
+    ( 0x55, 0x05 ), # AL=0xf5, AF=0 CF=0
+    ( 0x55, 0x05 ), # AL=0xf5, AF=0 CF=1
+    ( 0x5b, 0x11 ), # AL=0xf5, AF=1 CF=0
+    ( 0x5b, 0x11 ), # AL=0xf5, AF=1 CF=1
+    ( 0x56, 0x05 ), # AL=0xf6, AF=0 CF=0
+    ( 0x56, 0x05 ), # AL=0xf6, AF=0 CF=1
+    ( 0x5c, 0x15 ), # AL=0xf6, AF=1 CF=0
+    ( 0x5c, 0x15 ), # AL=0xf6, AF=1 CF=1
+    ( 0x57, 0x01 ), # AL=0xf7, AF=0 CF=0
+    ( 0x57, 0x01 ), # AL=0xf7, AF=0 CF=1
+    ( 0x5d, 0x11 ), # AL=0xf7, AF=1 CF=0
+    ( 0x5d, 0x11 ), # AL=0xf7, AF=1 CF=1
+    ( 0x58, 0x01 ), # AL=0xf8, AF=0 CF=0
+    ( 0x58, 0x01 ), # AL=0xf8, AF=0 CF=1
+    ( 0x5e, 0x11 ), # AL=0xf8, AF=1 CF=0
+    ( 0x5e, 0x11 ), # AL=0xf8, AF=1 CF=1
+    ( 0x59, 0x05 ), # AL=0xf9, AF=0 CF=0
+    ( 0x59, 0x05 ), # AL=0xf9, AF=0 CF=1
+    ( 0x5f, 0x15 ), # AL=0xf9, AF=1 CF=0
+    ( 0x5f, 0x15 ), # AL=0xf9, AF=1 CF=1
+    ( 0x60, 0x15 ), # AL=0xfa, AF=0 CF=0
+    ( 0x60, 0x15 ), # AL=0xfa, AF=0 CF=1
+    ( 0x60, 0x15 ), # AL=0xfa, AF=1 CF=0
+    ( 0x60, 0x15 ), # AL=0xfa, AF=1 CF=1
+    ( 0x61, 0x11 ), # AL=0xfb, AF=0 CF=0
+    ( 0x61, 0x11 ), # AL=0xfb, AF=0 CF=1
+    ( 0x61, 0x11 ), # AL=0xfb, AF=1 CF=0
+    ( 0x61, 0x11 ), # AL=0xfb, AF=1 CF=1
+    ( 0x62, 0x11 ), # AL=0xfc, AF=0 CF=0
+    ( 0x62, 0x11 ), # AL=0xfc, AF=0 CF=1
+    ( 0x62, 0x11 ), # AL=0xfc, AF=1 CF=0
+    ( 0x62, 0x11 ), # AL=0xfc, AF=1 CF=1
+    ( 0x63, 0x15 ), # AL=0xfd, AF=0 CF=0
+    ( 0x63, 0x15 ), # AL=0xfd, AF=0 CF=1
+    ( 0x63, 0x15 ), # AL=0xfd, AF=1 CF=0
+    ( 0x63, 0x15 ), # AL=0xfd, AF=1 CF=1
+    ( 0x64, 0x11 ), # AL=0xfe, AF=0 CF=0
+    ( 0x64, 0x11 ), # AL=0xfe, AF=0 CF=1
+    ( 0x64, 0x11 ), # AL=0xfe, AF=1 CF=0
+    ( 0x64, 0x11 ), # AL=0xfe, AF=1 CF=1
+    ( 0x65, 0x15 ), # AL=0xff, AF=0 CF=0
+    ( 0x65, 0x15 ), # AL=0xff, AF=0 CF=1
+    ( 0x65, 0x15 ), # AL=0xff, AF=1 CF=0
+    ( 0x65, 0x15 ), # AL=0xff, AF=1 CF=1
+];
+
diff --git a/src/VBox/VMM/testcase/Instructions/itgTableDas.py b/src/VBox/VMM/testcase/Instructions/itgTableDas.py
new file mode 100644
index 00000000..d3603bdc
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/itgTableDas.py
@@ -0,0 +1,1105 @@
+# -*- coding: utf-8 -*-
+# $Id: itgTableDas.py $
+
+"""
+DAS (instruction) result table.
+"""
+
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2020 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+"""
+__version__ = "$Revision: 135976 $";
+
+
+## The 32-bit GCC (C99) program that produced the table below.
+g_sItgCProgramDas = \
+"""
+#include <stdio.h>
+
+int main()
+{
+    for (unsigned uInputAL = 0; uInputAL < 256; uInputAL++)
+        for (unsigned fAux = 0; fAux < 2; fAux++)
+            for (unsigned fCarry = 0; fCarry < 2; fCarry++)
+            {
+                unsigned uInputEFlags = fCarry | (fAux << 4);
+                unsigned uResultAL;
+                unsigned uResultEFlags;
+                __asm__ __volatile__("pushl %1\\n"
+                                     "popfl\\n"
+                                     "das\\n"
+                                     "pushf\\n"
+                                     "pop   %1\\n"
+                                     : "=a" (uResultAL),
+                                       "=r" (uResultEFlags)
+                                     : "0" (uInputAL),
+                                       "1" (uInputEFlags)
+                                     : "memory"
+                                     );
+                printf("    ( 0x%02x, 0x%02x ), # AL=0x%02x, AF=%u CF=%u\\n",
+                       uResultAL, uResultEFlags & 0xd5, uInputAL, fAux, fCarry);
+                       /* 0xd5 = CF, PF, AF, ZF, SF */
+            }
+    return 0;
+}
+""";
+
+
+#
+# Compile and run the above program if requested to do so.
+#
+if __name__ == '__main__':
+    import sys;
+    if len(sys.argv) > 1 and sys.argv[1] == 'gen':
+        import subprocess;
+        oProc = subprocess.Popen(['gcc', '-x', 'c', '-std=gnu99', '-m32', '-o', './itgTableDas', '-'], stdin = subprocess.PIPE);
+        oProc.communicate(g_sItgCProgramDas);
+        oProc.wait();
+        oProc = subprocess.Popen(['./itgTableDas',]).wait();
+        sys.exit(0);
+
+
+
+##
+# The DAS results.
+#
+# The index / input relation is: index = (AL << 2) | (CF << 1) | AF
+#
+g_aItgDasResults = \
+[
+    ( 0x00, 0x44 ), # AL=0x00, AF=0 CF=0
+    ( 0xa0, 0x85 ), # AL=0x00, AF=0 CF=1
+    ( 0xfa, 0x95 ), # AL=0x00, AF=1 CF=0
+    ( 0x9a, 0x95 ), # AL=0x00, AF=1 CF=1
+    ( 0x01, 0x00 ), # AL=0x01, AF=0 CF=0
+    ( 0xa1, 0x81 ), # AL=0x01, AF=0 CF=1
+    ( 0xfb, 0x91 ), # AL=0x01, AF=1 CF=0
+    ( 0x9b, 0x91 ), # AL=0x01, AF=1 CF=1
+    ( 0x02, 0x00 ), # AL=0x02, AF=0 CF=0
+    ( 0xa2, 0x81 ), # AL=0x02, AF=0 CF=1
+    ( 0xfc, 0x95 ), # AL=0x02, AF=1 CF=0
+    ( 0x9c, 0x95 ), # AL=0x02, AF=1 CF=1
+    ( 0x03, 0x04 ), # AL=0x03, AF=0 CF=0
+    ( 0xa3, 0x85 ), # AL=0x03, AF=0 CF=1
+    ( 0xfd, 0x91 ), # AL=0x03, AF=1 CF=0
+    ( 0x9d, 0x91 ), # AL=0x03, AF=1 CF=1
+    ( 0x04, 0x00 ), # AL=0x04, AF=0 CF=0
+    ( 0xa4, 0x81 ), # AL=0x04, AF=0 CF=1
+    ( 0xfe, 0x91 ), # AL=0x04, AF=1 CF=0
+    ( 0x9e, 0x91 ), # AL=0x04, AF=1 CF=1
+    ( 0x05, 0x04 ), # AL=0x05, AF=0 CF=0
+    ( 0xa5, 0x85 ), # AL=0x05, AF=0 CF=1
+    ( 0xff, 0x95 ), # AL=0x05, AF=1 CF=0
+    ( 0x9f, 0x95 ), # AL=0x05, AF=1 CF=1
+    ( 0x06, 0x04 ), # AL=0x06, AF=0 CF=0
+    ( 0xa6, 0x85 ), # AL=0x06, AF=0 CF=1
+    ( 0x00, 0x54 ), # AL=0x06, AF=1 CF=0
+    ( 0xa0, 0x95 ), # AL=0x06, AF=1 CF=1
+    ( 0x07, 0x00 ), # AL=0x07, AF=0 CF=0
+    ( 0xa7, 0x81 ), # AL=0x07, AF=0 CF=1
+    ( 0x01, 0x10 ), # AL=0x07, AF=1 CF=0
+    ( 0xa1, 0x91 ), # AL=0x07, AF=1 CF=1
+    ( 0x08, 0x00 ), # AL=0x08, AF=0 CF=0
+    ( 0xa8, 0x81 ), # AL=0x08, AF=0 CF=1
+    ( 0x02, 0x10 ), # AL=0x08, AF=1 CF=0
+    ( 0xa2, 0x91 ), # AL=0x08, AF=1 CF=1
+    ( 0x09, 0x04 ), # AL=0x09, AF=0 CF=0
+    ( 0xa9, 0x85 ), # AL=0x09, AF=0 CF=1
+    ( 0x03, 0x14 ), # AL=0x09, AF=1 CF=0
+    ( 0xa3, 0x95 ), # AL=0x09, AF=1 CF=1
+    ( 0x04, 0x10 ), # AL=0x0a, AF=0 CF=0
+    ( 0xa4, 0x91 ), # AL=0x0a, AF=0 CF=1
+    ( 0x04, 0x10 ), # AL=0x0a, AF=1 CF=0
+    ( 0xa4, 0x91 ), # AL=0x0a, AF=1 CF=1
+    ( 0x05, 0x14 ), # AL=0x0b, AF=0 CF=0
+    ( 0xa5, 0x95 ), # AL=0x0b, AF=0 CF=1
+    ( 0x05, 0x14 ), # AL=0x0b, AF=1 CF=0
+    ( 0xa5, 0x95 ), # AL=0x0b, AF=1 CF=1
+    ( 0x06, 0x14 ), # AL=0x0c, AF=0 CF=0
+    ( 0xa6, 0x95 ), # AL=0x0c, AF=0 CF=1
+    ( 0x06, 0x14 ), # AL=0x0c, AF=1 CF=0
+    ( 0xa6, 0x95 ), # AL=0x0c, AF=1 CF=1
+    ( 0x07, 0x10 ), # AL=0x0d, AF=0 CF=0
+    ( 0xa7, 0x91 ), # AL=0x0d, AF=0 CF=1
+    ( 0x07, 0x10 ), # AL=0x0d, AF=1 CF=0
+    ( 0xa7, 0x91 ), # AL=0x0d, AF=1 CF=1
+    ( 0x08, 0x10 ), # AL=0x0e, AF=0 CF=0
+    ( 0xa8, 0x91 ), # AL=0x0e, AF=0 CF=1
+    ( 0x08, 0x10 ), # AL=0x0e, AF=1 CF=0
+    ( 0xa8, 0x91 ), # AL=0x0e, AF=1 CF=1
+    ( 0x09, 0x14 ), # AL=0x0f, AF=0 CF=0
+    ( 0xa9, 0x95 ), # AL=0x0f, AF=0 CF=1
+    ( 0x09, 0x14 ), # AL=0x0f, AF=1 CF=0
+    ( 0xa9, 0x95 ), # AL=0x0f, AF=1 CF=1
+    ( 0x10, 0x00 ), # AL=0x10, AF=0 CF=0
+    ( 0xb0, 0x81 ), # AL=0x10, AF=0 CF=1
+    ( 0x0a, 0x14 ), # AL=0x10, AF=1 CF=0
+    ( 0xaa, 0x95 ), # AL=0x10, AF=1 CF=1
+    ( 0x11, 0x04 ), # AL=0x11, AF=0 CF=0
+    ( 0xb1, 0x85 ), # AL=0x11, AF=0 CF=1
+    ( 0x0b, 0x10 ), # AL=0x11, AF=1 CF=0
+    ( 0xab, 0x91 ), # AL=0x11, AF=1 CF=1
+    ( 0x12, 0x04 ), # AL=0x12, AF=0 CF=0
+    ( 0xb2, 0x85 ), # AL=0x12, AF=0 CF=1
+    ( 0x0c, 0x14 ), # AL=0x12, AF=1 CF=0
+    ( 0xac, 0x95 ), # AL=0x12, AF=1 CF=1
+    ( 0x13, 0x00 ), # AL=0x13, AF=0 CF=0
+    ( 0xb3, 0x81 ), # AL=0x13, AF=0 CF=1
+    ( 0x0d, 0x10 ), # AL=0x13, AF=1 CF=0
+    ( 0xad, 0x91 ), # AL=0x13, AF=1 CF=1
+    ( 0x14, 0x04 ), # AL=0x14, AF=0 CF=0
+    ( 0xb4, 0x85 ), # AL=0x14, AF=0 CF=1
+    ( 0x0e, 0x10 ), # AL=0x14, AF=1 CF=0
+    ( 0xae, 0x91 ), # AL=0x14, AF=1 CF=1
+    ( 0x15, 0x00 ), # AL=0x15, AF=0 CF=0
+    ( 0xb5, 0x81 ), # AL=0x15, AF=0 CF=1
+    ( 0x0f, 0x14 ), # AL=0x15, AF=1 CF=0
+    ( 0xaf, 0x95 ), # AL=0x15, AF=1 CF=1
+    ( 0x16, 0x00 ), # AL=0x16, AF=0 CF=0
+    ( 0xb6, 0x81 ), # AL=0x16, AF=0 CF=1
+    ( 0x10, 0x10 ), # AL=0x16, AF=1 CF=0
+    ( 0xb0, 0x91 ), # AL=0x16, AF=1 CF=1
+    ( 0x17, 0x04 ), # AL=0x17, AF=0 CF=0
+    ( 0xb7, 0x85 ), # AL=0x17, AF=0 CF=1
+    ( 0x11, 0x14 ), # AL=0x17, AF=1 CF=0
+    ( 0xb1, 0x95 ), # AL=0x17, AF=1 CF=1
+    ( 0x18, 0x04 ), # AL=0x18, AF=0 CF=0
+    ( 0xb8, 0x85 ), # AL=0x18, AF=0 CF=1
+    ( 0x12, 0x14 ), # AL=0x18, AF=1 CF=0
+    ( 0xb2, 0x95 ), # AL=0x18, AF=1 CF=1
+    ( 0x19, 0x00 ), # AL=0x19, AF=0 CF=0
+    ( 0xb9, 0x81 ), # AL=0x19, AF=0 CF=1
+    ( 0x13, 0x10 ), # AL=0x19, AF=1 CF=0
+    ( 0xb3, 0x91 ), # AL=0x19, AF=1 CF=1
+    ( 0x14, 0x14 ), # AL=0x1a, AF=0 CF=0
+    ( 0xb4, 0x95 ), # AL=0x1a, AF=0 CF=1
+    ( 0x14, 0x14 ), # AL=0x1a, AF=1 CF=0
+    ( 0xb4, 0x95 ), # AL=0x1a, AF=1 CF=1
+    ( 0x15, 0x10 ), # AL=0x1b, AF=0 CF=0
+    ( 0xb5, 0x91 ), # AL=0x1b, AF=0 CF=1
+    ( 0x15, 0x10 ), # AL=0x1b, AF=1 CF=0
+    ( 0xb5, 0x91 ), # AL=0x1b, AF=1 CF=1
+    ( 0x16, 0x10 ), # AL=0x1c, AF=0 CF=0
+    ( 0xb6, 0x91 ), # AL=0x1c, AF=0 CF=1
+    ( 0x16, 0x10 ), # AL=0x1c, AF=1 CF=0
+    ( 0xb6, 0x91 ), # AL=0x1c, AF=1 CF=1
+    ( 0x17, 0x14 ), # AL=0x1d, AF=0 CF=0
+    ( 0xb7, 0x95 ), # AL=0x1d, AF=0 CF=1
+    ( 0x17, 0x14 ), # AL=0x1d, AF=1 CF=0
+    ( 0xb7, 0x95 ), # AL=0x1d, AF=1 CF=1
+    ( 0x18, 0x14 ), # AL=0x1e, AF=0 CF=0
+    ( 0xb8, 0x95 ), # AL=0x1e, AF=0 CF=1
+    ( 0x18, 0x14 ), # AL=0x1e, AF=1 CF=0
+    ( 0xb8, 0x95 ), # AL=0x1e, AF=1 CF=1
+    ( 0x19, 0x10 ), # AL=0x1f, AF=0 CF=0
+    ( 0xb9, 0x91 ), # AL=0x1f, AF=0 CF=1
+    ( 0x19, 0x10 ), # AL=0x1f, AF=1 CF=0
+    ( 0xb9, 0x91 ), # AL=0x1f, AF=1 CF=1
+    ( 0x20, 0x00 ), # AL=0x20, AF=0 CF=0
+    ( 0xc0, 0x85 ), # AL=0x20, AF=0 CF=1
+    ( 0x1a, 0x10 ), # AL=0x20, AF=1 CF=0
+    ( 0xba, 0x91 ), # AL=0x20, AF=1 CF=1
+    ( 0x21, 0x04 ), # AL=0x21, AF=0 CF=0
+    ( 0xc1, 0x81 ), # AL=0x21, AF=0 CF=1
+    ( 0x1b, 0x14 ), # AL=0x21, AF=1 CF=0
+    ( 0xbb, 0x95 ), # AL=0x21, AF=1 CF=1
+    ( 0x22, 0x04 ), # AL=0x22, AF=0 CF=0
+    ( 0xc2, 0x81 ), # AL=0x22, AF=0 CF=1
+    ( 0x1c, 0x10 ), # AL=0x22, AF=1 CF=0
+    ( 0xbc, 0x91 ), # AL=0x22, AF=1 CF=1
+    ( 0x23, 0x00 ), # AL=0x23, AF=0 CF=0
+    ( 0xc3, 0x85 ), # AL=0x23, AF=0 CF=1
+    ( 0x1d, 0x14 ), # AL=0x23, AF=1 CF=0
+    ( 0xbd, 0x95 ), # AL=0x23, AF=1 CF=1
+    ( 0x24, 0x04 ), # AL=0x24, AF=0 CF=0
+    ( 0xc4, 0x81 ), # AL=0x24, AF=0 CF=1
+    ( 0x1e, 0x14 ), # AL=0x24, AF=1 CF=0
+    ( 0xbe, 0x95 ), # AL=0x24, AF=1 CF=1
+    ( 0x25, 0x00 ), # AL=0x25, AF=0 CF=0
+    ( 0xc5, 0x85 ), # AL=0x25, AF=0 CF=1
+    ( 0x1f, 0x10 ), # AL=0x25, AF=1 CF=0
+    ( 0xbf, 0x91 ), # AL=0x25, AF=1 CF=1
+    ( 0x26, 0x00 ), # AL=0x26, AF=0 CF=0
+    ( 0xc6, 0x85 ), # AL=0x26, AF=0 CF=1
+    ( 0x20, 0x10 ), # AL=0x26, AF=1 CF=0
+    ( 0xc0, 0x95 ), # AL=0x26, AF=1 CF=1
+    ( 0x27, 0x04 ), # AL=0x27, AF=0 CF=0
+    ( 0xc7, 0x81 ), # AL=0x27, AF=0 CF=1
+    ( 0x21, 0x14 ), # AL=0x27, AF=1 CF=0
+    ( 0xc1, 0x91 ), # AL=0x27, AF=1 CF=1
+    ( 0x28, 0x04 ), # AL=0x28, AF=0 CF=0
+    ( 0xc8, 0x81 ), # AL=0x28, AF=0 CF=1
+    ( 0x22, 0x14 ), # AL=0x28, AF=1 CF=0
+    ( 0xc2, 0x91 ), # AL=0x28, AF=1 CF=1
+    ( 0x29, 0x00 ), # AL=0x29, AF=0 CF=0
+    ( 0xc9, 0x85 ), # AL=0x29, AF=0 CF=1
+    ( 0x23, 0x10 ), # AL=0x29, AF=1 CF=0
+    ( 0xc3, 0x95 ), # AL=0x29, AF=1 CF=1
+    ( 0x24, 0x14 ), # AL=0x2a, AF=0 CF=0
+    ( 0xc4, 0x91 ), # AL=0x2a, AF=0 CF=1
+    ( 0x24, 0x14 ), # AL=0x2a, AF=1 CF=0
+    ( 0xc4, 0x91 ), # AL=0x2a, AF=1 CF=1
+    ( 0x25, 0x10 ), # AL=0x2b, AF=0 CF=0
+    ( 0xc5, 0x95 ), # AL=0x2b, AF=0 CF=1
+    ( 0x25, 0x10 ), # AL=0x2b, AF=1 CF=0
+    ( 0xc5, 0x95 ), # AL=0x2b, AF=1 CF=1
+    ( 0x26, 0x10 ), # AL=0x2c, AF=0 CF=0
+    ( 0xc6, 0x95 ), # AL=0x2c, AF=0 CF=1
+    ( 0x26, 0x10 ), # AL=0x2c, AF=1 CF=0
+    ( 0xc6, 0x95 ), # AL=0x2c, AF=1 CF=1
+    ( 0x27, 0x14 ), # AL=0x2d, AF=0 CF=0
+    ( 0xc7, 0x91 ), # AL=0x2d, AF=0 CF=1
+    ( 0x27, 0x14 ), # AL=0x2d, AF=1 CF=0
+    ( 0xc7, 0x91 ), # AL=0x2d, AF=1 CF=1
+    ( 0x28, 0x14 ), # AL=0x2e, AF=0 CF=0
+    ( 0xc8, 0x91 ), # AL=0x2e, AF=0 CF=1
+    ( 0x28, 0x14 ), # AL=0x2e, AF=1 CF=0
+    ( 0xc8, 0x91 ), # AL=0x2e, AF=1 CF=1
+    ( 0x29, 0x10 ), # AL=0x2f, AF=0 CF=0
+    ( 0xc9, 0x95 ), # AL=0x2f, AF=0 CF=1
+    ( 0x29, 0x10 ), # AL=0x2f, AF=1 CF=0
+    ( 0xc9, 0x95 ), # AL=0x2f, AF=1 CF=1
+    ( 0x30, 0x04 ), # AL=0x30, AF=0 CF=0
+    ( 0xd0, 0x81 ), # AL=0x30, AF=0 CF=1
+    ( 0x2a, 0x10 ), # AL=0x30, AF=1 CF=0
+    ( 0xca, 0x95 ), # AL=0x30, AF=1 CF=1
+    ( 0x31, 0x00 ), # AL=0x31, AF=0 CF=0
+    ( 0xd1, 0x85 ), # AL=0x31, AF=0 CF=1
+    ( 0x2b, 0x14 ), # AL=0x31, AF=1 CF=0
+    ( 0xcb, 0x91 ), # AL=0x31, AF=1 CF=1
+    ( 0x32, 0x00 ), # AL=0x32, AF=0 CF=0
+    ( 0xd2, 0x85 ), # AL=0x32, AF=0 CF=1
+    ( 0x2c, 0x10 ), # AL=0x32, AF=1 CF=0
+    ( 0xcc, 0x95 ), # AL=0x32, AF=1 CF=1
+    ( 0x33, 0x04 ), # AL=0x33, AF=0 CF=0
+    ( 0xd3, 0x81 ), # AL=0x33, AF=0 CF=1
+    ( 0x2d, 0x14 ), # AL=0x33, AF=1 CF=0
+    ( 0xcd, 0x91 ), # AL=0x33, AF=1 CF=1
+    ( 0x34, 0x00 ), # AL=0x34, AF=0 CF=0
+    ( 0xd4, 0x85 ), # AL=0x34, AF=0 CF=1
+    ( 0x2e, 0x14 ), # AL=0x34, AF=1 CF=0
+    ( 0xce, 0x91 ), # AL=0x34, AF=1 CF=1
+    ( 0x35, 0x04 ), # AL=0x35, AF=0 CF=0
+    ( 0xd5, 0x81 ), # AL=0x35, AF=0 CF=1
+    ( 0x2f, 0x10 ), # AL=0x35, AF=1 CF=0
+    ( 0xcf, 0x95 ), # AL=0x35, AF=1 CF=1
+    ( 0x36, 0x04 ), # AL=0x36, AF=0 CF=0
+    ( 0xd6, 0x81 ), # AL=0x36, AF=0 CF=1
+    ( 0x30, 0x14 ), # AL=0x36, AF=1 CF=0
+    ( 0xd0, 0x91 ), # AL=0x36, AF=1 CF=1
+    ( 0x37, 0x00 ), # AL=0x37, AF=0 CF=0
+    ( 0xd7, 0x85 ), # AL=0x37, AF=0 CF=1
+    ( 0x31, 0x10 ), # AL=0x37, AF=1 CF=0
+    ( 0xd1, 0x95 ), # AL=0x37, AF=1 CF=1
+    ( 0x38, 0x00 ), # AL=0x38, AF=0 CF=0
+    ( 0xd8, 0x85 ), # AL=0x38, AF=0 CF=1
+    ( 0x32, 0x10 ), # AL=0x38, AF=1 CF=0
+    ( 0xd2, 0x95 ), # AL=0x38, AF=1 CF=1
+    ( 0x39, 0x04 ), # AL=0x39, AF=0 CF=0
+    ( 0xd9, 0x81 ), # AL=0x39, AF=0 CF=1
+    ( 0x33, 0x14 ), # AL=0x39, AF=1 CF=0
+    ( 0xd3, 0x91 ), # AL=0x39, AF=1 CF=1
+    ( 0x34, 0x10 ), # AL=0x3a, AF=0 CF=0
+    ( 0xd4, 0x95 ), # AL=0x3a, AF=0 CF=1
+    ( 0x34, 0x10 ), # AL=0x3a, AF=1 CF=0
+    ( 0xd4, 0x95 ), # AL=0x3a, AF=1 CF=1
+    ( 0x35, 0x14 ), # AL=0x3b, AF=0 CF=0
+    ( 0xd5, 0x91 ), # AL=0x3b, AF=0 CF=1
+    ( 0x35, 0x14 ), # AL=0x3b, AF=1 CF=0
+    ( 0xd5, 0x91 ), # AL=0x3b, AF=1 CF=1
+    ( 0x36, 0x14 ), # AL=0x3c, AF=0 CF=0
+    ( 0xd6, 0x91 ), # AL=0x3c, AF=0 CF=1
+    ( 0x36, 0x14 ), # AL=0x3c, AF=1 CF=0
+    ( 0xd6, 0x91 ), # AL=0x3c, AF=1 CF=1
+    ( 0x37, 0x10 ), # AL=0x3d, AF=0 CF=0
+    ( 0xd7, 0x95 ), # AL=0x3d, AF=0 CF=1
+    ( 0x37, 0x10 ), # AL=0x3d, AF=1 CF=0
+    ( 0xd7, 0x95 ), # AL=0x3d, AF=1 CF=1
+    ( 0x38, 0x10 ), # AL=0x3e, AF=0 CF=0
+    ( 0xd8, 0x95 ), # AL=0x3e, AF=0 CF=1
+    ( 0x38, 0x10 ), # AL=0x3e, AF=1 CF=0
+    ( 0xd8, 0x95 ), # AL=0x3e, AF=1 CF=1
+    ( 0x39, 0x14 ), # AL=0x3f, AF=0 CF=0
+    ( 0xd9, 0x91 ), # AL=0x3f, AF=0 CF=1
+    ( 0x39, 0x14 ), # AL=0x3f, AF=1 CF=0
+    ( 0xd9, 0x91 ), # AL=0x3f, AF=1 CF=1
+    ( 0x40, 0x00 ), # AL=0x40, AF=0 CF=0
+    ( 0xe0, 0x81 ), # AL=0x40, AF=0 CF=1
+    ( 0x3a, 0x14 ), # AL=0x40, AF=1 CF=0
+    ( 0xda, 0x91 ), # AL=0x40, AF=1 CF=1
+    ( 0x41, 0x04 ), # AL=0x41, AF=0 CF=0
+    ( 0xe1, 0x85 ), # AL=0x41, AF=0 CF=1
+    ( 0x3b, 0x10 ), # AL=0x41, AF=1 CF=0
+    ( 0xdb, 0x95 ), # AL=0x41, AF=1 CF=1
+    ( 0x42, 0x04 ), # AL=0x42, AF=0 CF=0
+    ( 0xe2, 0x85 ), # AL=0x42, AF=0 CF=1
+    ( 0x3c, 0x14 ), # AL=0x42, AF=1 CF=0
+    ( 0xdc, 0x91 ), # AL=0x42, AF=1 CF=1
+    ( 0x43, 0x00 ), # AL=0x43, AF=0 CF=0
+    ( 0xe3, 0x81 ), # AL=0x43, AF=0 CF=1
+    ( 0x3d, 0x10 ), # AL=0x43, AF=1 CF=0
+    ( 0xdd, 0x95 ), # AL=0x43, AF=1 CF=1
+    ( 0x44, 0x04 ), # AL=0x44, AF=0 CF=0
+    ( 0xe4, 0x85 ), # AL=0x44, AF=0 CF=1
+    ( 0x3e, 0x10 ), # AL=0x44, AF=1 CF=0
+    ( 0xde, 0x95 ), # AL=0x44, AF=1 CF=1
+    ( 0x45, 0x00 ), # AL=0x45, AF=0 CF=0
+    ( 0xe5, 0x81 ), # AL=0x45, AF=0 CF=1
+    ( 0x3f, 0x14 ), # AL=0x45, AF=1 CF=0
+    ( 0xdf, 0x91 ), # AL=0x45, AF=1 CF=1
+    ( 0x46, 0x00 ), # AL=0x46, AF=0 CF=0
+    ( 0xe6, 0x81 ), # AL=0x46, AF=0 CF=1
+    ( 0x40, 0x10 ), # AL=0x46, AF=1 CF=0
+    ( 0xe0, 0x91 ), # AL=0x46, AF=1 CF=1
+    ( 0x47, 0x04 ), # AL=0x47, AF=0 CF=0
+    ( 0xe7, 0x85 ), # AL=0x47, AF=0 CF=1
+    ( 0x41, 0x14 ), # AL=0x47, AF=1 CF=0
+    ( 0xe1, 0x95 ), # AL=0x47, AF=1 CF=1
+    ( 0x48, 0x04 ), # AL=0x48, AF=0 CF=0
+    ( 0xe8, 0x85 ), # AL=0x48, AF=0 CF=1
+    ( 0x42, 0x14 ), # AL=0x48, AF=1 CF=0
+    ( 0xe2, 0x95 ), # AL=0x48, AF=1 CF=1
+    ( 0x49, 0x00 ), # AL=0x49, AF=0 CF=0
+    ( 0xe9, 0x81 ), # AL=0x49, AF=0 CF=1
+    ( 0x43, 0x10 ), # AL=0x49, AF=1 CF=0
+    ( 0xe3, 0x91 ), # AL=0x49, AF=1 CF=1
+    ( 0x44, 0x14 ), # AL=0x4a, AF=0 CF=0
+    ( 0xe4, 0x95 ), # AL=0x4a, AF=0 CF=1
+    ( 0x44, 0x14 ), # AL=0x4a, AF=1 CF=0
+    ( 0xe4, 0x95 ), # AL=0x4a, AF=1 CF=1
+    ( 0x45, 0x10 ), # AL=0x4b, AF=0 CF=0
+    ( 0xe5, 0x91 ), # AL=0x4b, AF=0 CF=1
+    ( 0x45, 0x10 ), # AL=0x4b, AF=1 CF=0
+    ( 0xe5, 0x91 ), # AL=0x4b, AF=1 CF=1
+    ( 0x46, 0x10 ), # AL=0x4c, AF=0 CF=0
+    ( 0xe6, 0x91 ), # AL=0x4c, AF=0 CF=1
+    ( 0x46, 0x10 ), # AL=0x4c, AF=1 CF=0
+    ( 0xe6, 0x91 ), # AL=0x4c, AF=1 CF=1
+    ( 0x47, 0x14 ), # AL=0x4d, AF=0 CF=0
+    ( 0xe7, 0x95 ), # AL=0x4d, AF=0 CF=1
+    ( 0x47, 0x14 ), # AL=0x4d, AF=1 CF=0
+    ( 0xe7, 0x95 ), # AL=0x4d, AF=1 CF=1
+    ( 0x48, 0x14 ), # AL=0x4e, AF=0 CF=0
+    ( 0xe8, 0x95 ), # AL=0x4e, AF=0 CF=1
+    ( 0x48, 0x14 ), # AL=0x4e, AF=1 CF=0
+    ( 0xe8, 0x95 ), # AL=0x4e, AF=1 CF=1
+    ( 0x49, 0x10 ), # AL=0x4f, AF=0 CF=0
+    ( 0xe9, 0x91 ), # AL=0x4f, AF=0 CF=1
+    ( 0x49, 0x10 ), # AL=0x4f, AF=1 CF=0
+    ( 0xe9, 0x91 ), # AL=0x4f, AF=1 CF=1
+    ( 0x50, 0x04 ), # AL=0x50, AF=0 CF=0
+    ( 0xf0, 0x85 ), # AL=0x50, AF=0 CF=1
+    ( 0x4a, 0x10 ), # AL=0x50, AF=1 CF=0
+    ( 0xea, 0x91 ), # AL=0x50, AF=1 CF=1
+    ( 0x51, 0x00 ), # AL=0x51, AF=0 CF=0
+    ( 0xf1, 0x81 ), # AL=0x51, AF=0 CF=1
+    ( 0x4b, 0x14 ), # AL=0x51, AF=1 CF=0
+    ( 0xeb, 0x95 ), # AL=0x51, AF=1 CF=1
+    ( 0x52, 0x00 ), # AL=0x52, AF=0 CF=0
+    ( 0xf2, 0x81 ), # AL=0x52, AF=0 CF=1
+    ( 0x4c, 0x10 ), # AL=0x52, AF=1 CF=0
+    ( 0xec, 0x91 ), # AL=0x52, AF=1 CF=1
+    ( 0x53, 0x04 ), # AL=0x53, AF=0 CF=0
+    ( 0xf3, 0x85 ), # AL=0x53, AF=0 CF=1
+    ( 0x4d, 0x14 ), # AL=0x53, AF=1 CF=0
+    ( 0xed, 0x95 ), # AL=0x53, AF=1 CF=1
+    ( 0x54, 0x00 ), # AL=0x54, AF=0 CF=0
+    ( 0xf4, 0x81 ), # AL=0x54, AF=0 CF=1
+    ( 0x4e, 0x14 ), # AL=0x54, AF=1 CF=0
+    ( 0xee, 0x95 ), # AL=0x54, AF=1 CF=1
+    ( 0x55, 0x04 ), # AL=0x55, AF=0 CF=0
+    ( 0xf5, 0x85 ), # AL=0x55, AF=0 CF=1
+    ( 0x4f, 0x10 ), # AL=0x55, AF=1 CF=0
+    ( 0xef, 0x91 ), # AL=0x55, AF=1 CF=1
+    ( 0x56, 0x04 ), # AL=0x56, AF=0 CF=0
+    ( 0xf6, 0x85 ), # AL=0x56, AF=0 CF=1
+    ( 0x50, 0x14 ), # AL=0x56, AF=1 CF=0
+    ( 0xf0, 0x95 ), # AL=0x56, AF=1 CF=1
+    ( 0x57, 0x00 ), # AL=0x57, AF=0 CF=0
+    ( 0xf7, 0x81 ), # AL=0x57, AF=0 CF=1
+    ( 0x51, 0x10 ), # AL=0x57, AF=1 CF=0
+    ( 0xf1, 0x91 ), # AL=0x57, AF=1 CF=1
+    ( 0x58, 0x00 ), # AL=0x58, AF=0 CF=0
+    ( 0xf8, 0x81 ), # AL=0x58, AF=0 CF=1
+    ( 0x52, 0x10 ), # AL=0x58, AF=1 CF=0
+    ( 0xf2, 0x91 ), # AL=0x58, AF=1 CF=1
+    ( 0x59, 0x04 ), # AL=0x59, AF=0 CF=0
+    ( 0xf9, 0x85 ), # AL=0x59, AF=0 CF=1
+    ( 0x53, 0x14 ), # AL=0x59, AF=1 CF=0
+    ( 0xf3, 0x95 ), # AL=0x59, AF=1 CF=1
+    ( 0x54, 0x10 ), # AL=0x5a, AF=0 CF=0
+    ( 0xf4, 0x91 ), # AL=0x5a, AF=0 CF=1
+    ( 0x54, 0x10 ), # AL=0x5a, AF=1 CF=0
+    ( 0xf4, 0x91 ), # AL=0x5a, AF=1 CF=1
+    ( 0x55, 0x14 ), # AL=0x5b, AF=0 CF=0
+    ( 0xf5, 0x95 ), # AL=0x5b, AF=0 CF=1
+    ( 0x55, 0x14 ), # AL=0x5b, AF=1 CF=0
+    ( 0xf5, 0x95 ), # AL=0x5b, AF=1 CF=1
+    ( 0x56, 0x14 ), # AL=0x5c, AF=0 CF=0
+    ( 0xf6, 0x95 ), # AL=0x5c, AF=0 CF=1
+    ( 0x56, 0x14 ), # AL=0x5c, AF=1 CF=0
+    ( 0xf6, 0x95 ), # AL=0x5c, AF=1 CF=1
+    ( 0x57, 0x10 ), # AL=0x5d, AF=0 CF=0
+    ( 0xf7, 0x91 ), # AL=0x5d, AF=0 CF=1
+    ( 0x57, 0x10 ), # AL=0x5d, AF=1 CF=0
+    ( 0xf7, 0x91 ), # AL=0x5d, AF=1 CF=1
+    ( 0x58, 0x10 ), # AL=0x5e, AF=0 CF=0
+    ( 0xf8, 0x91 ), # AL=0x5e, AF=0 CF=1
+    ( 0x58, 0x10 ), # AL=0x5e, AF=1 CF=0
+    ( 0xf8, 0x91 ), # AL=0x5e, AF=1 CF=1
+    ( 0x59, 0x14 ), # AL=0x5f, AF=0 CF=0
+    ( 0xf9, 0x95 ), # AL=0x5f, AF=0 CF=1
+    ( 0x59, 0x14 ), # AL=0x5f, AF=1 CF=0
+    ( 0xf9, 0x95 ), # AL=0x5f, AF=1 CF=1
+    ( 0x60, 0x04 ), # AL=0x60, AF=0 CF=0
+    ( 0x00, 0x45 ), # AL=0x60, AF=0 CF=1
+    ( 0x5a, 0x14 ), # AL=0x60, AF=1 CF=0
+    ( 0xfa, 0x95 ), # AL=0x60, AF=1 CF=1
+    ( 0x61, 0x00 ), # AL=0x61, AF=0 CF=0
+    ( 0x01, 0x01 ), # AL=0x61, AF=0 CF=1
+    ( 0x5b, 0x10 ), # AL=0x61, AF=1 CF=0
+    ( 0xfb, 0x91 ), # AL=0x61, AF=1 CF=1
+    ( 0x62, 0x00 ), # AL=0x62, AF=0 CF=0
+    ( 0x02, 0x01 ), # AL=0x62, AF=0 CF=1
+    ( 0x5c, 0x14 ), # AL=0x62, AF=1 CF=0
+    ( 0xfc, 0x95 ), # AL=0x62, AF=1 CF=1
+    ( 0x63, 0x04 ), # AL=0x63, AF=0 CF=0
+    ( 0x03, 0x05 ), # AL=0x63, AF=0 CF=1
+    ( 0x5d, 0x10 ), # AL=0x63, AF=1 CF=0
+    ( 0xfd, 0x91 ), # AL=0x63, AF=1 CF=1
+    ( 0x64, 0x00 ), # AL=0x64, AF=0 CF=0
+    ( 0x04, 0x01 ), # AL=0x64, AF=0 CF=1
+    ( 0x5e, 0x10 ), # AL=0x64, AF=1 CF=0
+    ( 0xfe, 0x91 ), # AL=0x64, AF=1 CF=1
+    ( 0x65, 0x04 ), # AL=0x65, AF=0 CF=0
+    ( 0x05, 0x05 ), # AL=0x65, AF=0 CF=1
+    ( 0x5f, 0x14 ), # AL=0x65, AF=1 CF=0
+    ( 0xff, 0x95 ), # AL=0x65, AF=1 CF=1
+    ( 0x66, 0x04 ), # AL=0x66, AF=0 CF=0
+    ( 0x06, 0x05 ), # AL=0x66, AF=0 CF=1
+    ( 0x60, 0x14 ), # AL=0x66, AF=1 CF=0
+    ( 0x00, 0x55 ), # AL=0x66, AF=1 CF=1
+    ( 0x67, 0x00 ), # AL=0x67, AF=0 CF=0
+    ( 0x07, 0x01 ), # AL=0x67, AF=0 CF=1
+    ( 0x61, 0x10 ), # AL=0x67, AF=1 CF=0
+    ( 0x01, 0x11 ), # AL=0x67, AF=1 CF=1
+    ( 0x68, 0x00 ), # AL=0x68, AF=0 CF=0
+    ( 0x08, 0x01 ), # AL=0x68, AF=0 CF=1
+    ( 0x62, 0x10 ), # AL=0x68, AF=1 CF=0
+    ( 0x02, 0x11 ), # AL=0x68, AF=1 CF=1
+    ( 0x69, 0x04 ), # AL=0x69, AF=0 CF=0
+    ( 0x09, 0x05 ), # AL=0x69, AF=0 CF=1
+    ( 0x63, 0x14 ), # AL=0x69, AF=1 CF=0
+    ( 0x03, 0x15 ), # AL=0x69, AF=1 CF=1
+    ( 0x64, 0x10 ), # AL=0x6a, AF=0 CF=0
+    ( 0x04, 0x11 ), # AL=0x6a, AF=0 CF=1
+    ( 0x64, 0x10 ), # AL=0x6a, AF=1 CF=0
+    ( 0x04, 0x11 ), # AL=0x6a, AF=1 CF=1
+    ( 0x65, 0x14 ), # AL=0x6b, AF=0 CF=0
+    ( 0x05, 0x15 ), # AL=0x6b, AF=0 CF=1
+    ( 0x65, 0x14 ), # AL=0x6b, AF=1 CF=0
+    ( 0x05, 0x15 ), # AL=0x6b, AF=1 CF=1
+    ( 0x66, 0x14 ), # AL=0x6c, AF=0 CF=0
+    ( 0x06, 0x15 ), # AL=0x6c, AF=0 CF=1
+    ( 0x66, 0x14 ), # AL=0x6c, AF=1 CF=0
+    ( 0x06, 0x15 ), # AL=0x6c, AF=1 CF=1
+    ( 0x67, 0x10 ), # AL=0x6d, AF=0 CF=0
+    ( 0x07, 0x11 ), # AL=0x6d, AF=0 CF=1
+    ( 0x67, 0x10 ), # AL=0x6d, AF=1 CF=0
+    ( 0x07, 0x11 ), # AL=0x6d, AF=1 CF=1
+    ( 0x68, 0x10 ), # AL=0x6e, AF=0 CF=0
+    ( 0x08, 0x11 ), # AL=0x6e, AF=0 CF=1
+    ( 0x68, 0x10 ), # AL=0x6e, AF=1 CF=0
+    ( 0x08, 0x11 ), # AL=0x6e, AF=1 CF=1
+    ( 0x69, 0x14 ), # AL=0x6f, AF=0 CF=0
+    ( 0x09, 0x15 ), # AL=0x6f, AF=0 CF=1
+    ( 0x69, 0x14 ), # AL=0x6f, AF=1 CF=0
+    ( 0x09, 0x15 ), # AL=0x6f, AF=1 CF=1
+    ( 0x70, 0x00 ), # AL=0x70, AF=0 CF=0
+    ( 0x10, 0x01 ), # AL=0x70, AF=0 CF=1
+    ( 0x6a, 0x14 ), # AL=0x70, AF=1 CF=0
+    ( 0x0a, 0x15 ), # AL=0x70, AF=1 CF=1
+    ( 0x71, 0x04 ), # AL=0x71, AF=0 CF=0
+    ( 0x11, 0x05 ), # AL=0x71, AF=0 CF=1
+    ( 0x6b, 0x10 ), # AL=0x71, AF=1 CF=0
+    ( 0x0b, 0x11 ), # AL=0x71, AF=1 CF=1
+    ( 0x72, 0x04 ), # AL=0x72, AF=0 CF=0
+    ( 0x12, 0x05 ), # AL=0x72, AF=0 CF=1
+    ( 0x6c, 0x14 ), # AL=0x72, AF=1 CF=0
+    ( 0x0c, 0x15 ), # AL=0x72, AF=1 CF=1
+    ( 0x73, 0x00 ), # AL=0x73, AF=0 CF=0
+    ( 0x13, 0x01 ), # AL=0x73, AF=0 CF=1
+    ( 0x6d, 0x10 ), # AL=0x73, AF=1 CF=0
+    ( 0x0d, 0x11 ), # AL=0x73, AF=1 CF=1
+    ( 0x74, 0x04 ), # AL=0x74, AF=0 CF=0
+    ( 0x14, 0x05 ), # AL=0x74, AF=0 CF=1
+    ( 0x6e, 0x10 ), # AL=0x74, AF=1 CF=0
+    ( 0x0e, 0x11 ), # AL=0x74, AF=1 CF=1
+    ( 0x75, 0x00 ), # AL=0x75, AF=0 CF=0
+    ( 0x15, 0x01 ), # AL=0x75, AF=0 CF=1
+    ( 0x6f, 0x14 ), # AL=0x75, AF=1 CF=0
+    ( 0x0f, 0x15 ), # AL=0x75, AF=1 CF=1
+    ( 0x76, 0x00 ), # AL=0x76, AF=0 CF=0
+    ( 0x16, 0x01 ), # AL=0x76, AF=0 CF=1
+    ( 0x70, 0x10 ), # AL=0x76, AF=1 CF=0
+    ( 0x10, 0x11 ), # AL=0x76, AF=1 CF=1
+    ( 0x77, 0x04 ), # AL=0x77, AF=0 CF=0
+    ( 0x17, 0x05 ), # AL=0x77, AF=0 CF=1
+    ( 0x71, 0x14 ), # AL=0x77, AF=1 CF=0
+    ( 0x11, 0x15 ), # AL=0x77, AF=1 CF=1
+    ( 0x78, 0x04 ), # AL=0x78, AF=0 CF=0
+    ( 0x18, 0x05 ), # AL=0x78, AF=0 CF=1
+    ( 0x72, 0x14 ), # AL=0x78, AF=1 CF=0
+    ( 0x12, 0x15 ), # AL=0x78, AF=1 CF=1
+    ( 0x79, 0x00 ), # AL=0x79, AF=0 CF=0
+    ( 0x19, 0x01 ), # AL=0x79, AF=0 CF=1
+    ( 0x73, 0x10 ), # AL=0x79, AF=1 CF=0
+    ( 0x13, 0x11 ), # AL=0x79, AF=1 CF=1
+    ( 0x74, 0x14 ), # AL=0x7a, AF=0 CF=0
+    ( 0x14, 0x15 ), # AL=0x7a, AF=0 CF=1
+    ( 0x74, 0x14 ), # AL=0x7a, AF=1 CF=0
+    ( 0x14, 0x15 ), # AL=0x7a, AF=1 CF=1
+    ( 0x75, 0x10 ), # AL=0x7b, AF=0 CF=0
+    ( 0x15, 0x11 ), # AL=0x7b, AF=0 CF=1
+    ( 0x75, 0x10 ), # AL=0x7b, AF=1 CF=0
+    ( 0x15, 0x11 ), # AL=0x7b, AF=1 CF=1
+    ( 0x76, 0x10 ), # AL=0x7c, AF=0 CF=0
+    ( 0x16, 0x11 ), # AL=0x7c, AF=0 CF=1
+    ( 0x76, 0x10 ), # AL=0x7c, AF=1 CF=0
+    ( 0x16, 0x11 ), # AL=0x7c, AF=1 CF=1
+    ( 0x77, 0x14 ), # AL=0x7d, AF=0 CF=0
+    ( 0x17, 0x15 ), # AL=0x7d, AF=0 CF=1
+    ( 0x77, 0x14 ), # AL=0x7d, AF=1 CF=0
+    ( 0x17, 0x15 ), # AL=0x7d, AF=1 CF=1
+    ( 0x78, 0x14 ), # AL=0x7e, AF=0 CF=0
+    ( 0x18, 0x15 ), # AL=0x7e, AF=0 CF=1
+    ( 0x78, 0x14 ), # AL=0x7e, AF=1 CF=0
+    ( 0x18, 0x15 ), # AL=0x7e, AF=1 CF=1
+    ( 0x79, 0x10 ), # AL=0x7f, AF=0 CF=0
+    ( 0x19, 0x11 ), # AL=0x7f, AF=0 CF=1
+    ( 0x79, 0x10 ), # AL=0x7f, AF=1 CF=0
+    ( 0x19, 0x11 ), # AL=0x7f, AF=1 CF=1
+    ( 0x80, 0x80 ), # AL=0x80, AF=0 CF=0
+    ( 0x20, 0x01 ), # AL=0x80, AF=0 CF=1
+    ( 0x7a, 0x10 ), # AL=0x80, AF=1 CF=0
+    ( 0x1a, 0x11 ), # AL=0x80, AF=1 CF=1
+    ( 0x81, 0x84 ), # AL=0x81, AF=0 CF=0
+    ( 0x21, 0x05 ), # AL=0x81, AF=0 CF=1
+    ( 0x7b, 0x14 ), # AL=0x81, AF=1 CF=0
+    ( 0x1b, 0x15 ), # AL=0x81, AF=1 CF=1
+    ( 0x82, 0x84 ), # AL=0x82, AF=0 CF=0
+    ( 0x22, 0x05 ), # AL=0x82, AF=0 CF=1
+    ( 0x7c, 0x10 ), # AL=0x82, AF=1 CF=0
+    ( 0x1c, 0x11 ), # AL=0x82, AF=1 CF=1
+    ( 0x83, 0x80 ), # AL=0x83, AF=0 CF=0
+    ( 0x23, 0x01 ), # AL=0x83, AF=0 CF=1
+    ( 0x7d, 0x14 ), # AL=0x83, AF=1 CF=0
+    ( 0x1d, 0x15 ), # AL=0x83, AF=1 CF=1
+    ( 0x84, 0x84 ), # AL=0x84, AF=0 CF=0
+    ( 0x24, 0x05 ), # AL=0x84, AF=0 CF=1
+    ( 0x7e, 0x14 ), # AL=0x84, AF=1 CF=0
+    ( 0x1e, 0x15 ), # AL=0x84, AF=1 CF=1
+    ( 0x85, 0x80 ), # AL=0x85, AF=0 CF=0
+    ( 0x25, 0x01 ), # AL=0x85, AF=0 CF=1
+    ( 0x7f, 0x10 ), # AL=0x85, AF=1 CF=0
+    ( 0x1f, 0x11 ), # AL=0x85, AF=1 CF=1
+    ( 0x86, 0x80 ), # AL=0x86, AF=0 CF=0
+    ( 0x26, 0x01 ), # AL=0x86, AF=0 CF=1
+    ( 0x80, 0x90 ), # AL=0x86, AF=1 CF=0
+    ( 0x20, 0x11 ), # AL=0x86, AF=1 CF=1
+    ( 0x87, 0x84 ), # AL=0x87, AF=0 CF=0
+    ( 0x27, 0x05 ), # AL=0x87, AF=0 CF=1
+    ( 0x81, 0x94 ), # AL=0x87, AF=1 CF=0
+    ( 0x21, 0x15 ), # AL=0x87, AF=1 CF=1
+    ( 0x88, 0x84 ), # AL=0x88, AF=0 CF=0
+    ( 0x28, 0x05 ), # AL=0x88, AF=0 CF=1
+    ( 0x82, 0x94 ), # AL=0x88, AF=1 CF=0
+    ( 0x22, 0x15 ), # AL=0x88, AF=1 CF=1
+    ( 0x89, 0x80 ), # AL=0x89, AF=0 CF=0
+    ( 0x29, 0x01 ), # AL=0x89, AF=0 CF=1
+    ( 0x83, 0x90 ), # AL=0x89, AF=1 CF=0
+    ( 0x23, 0x11 ), # AL=0x89, AF=1 CF=1
+    ( 0x84, 0x94 ), # AL=0x8a, AF=0 CF=0
+    ( 0x24, 0x15 ), # AL=0x8a, AF=0 CF=1
+    ( 0x84, 0x94 ), # AL=0x8a, AF=1 CF=0
+    ( 0x24, 0x15 ), # AL=0x8a, AF=1 CF=1
+    ( 0x85, 0x90 ), # AL=0x8b, AF=0 CF=0
+    ( 0x25, 0x11 ), # AL=0x8b, AF=0 CF=1
+    ( 0x85, 0x90 ), # AL=0x8b, AF=1 CF=0
+    ( 0x25, 0x11 ), # AL=0x8b, AF=1 CF=1
+    ( 0x86, 0x90 ), # AL=0x8c, AF=0 CF=0
+    ( 0x26, 0x11 ), # AL=0x8c, AF=0 CF=1
+    ( 0x86, 0x90 ), # AL=0x8c, AF=1 CF=0
+    ( 0x26, 0x11 ), # AL=0x8c, AF=1 CF=1
+    ( 0x87, 0x94 ), # AL=0x8d, AF=0 CF=0
+    ( 0x27, 0x15 ), # AL=0x8d, AF=0 CF=1
+    ( 0x87, 0x94 ), # AL=0x8d, AF=1 CF=0
+    ( 0x27, 0x15 ), # AL=0x8d, AF=1 CF=1
+    ( 0x88, 0x94 ), # AL=0x8e, AF=0 CF=0
+    ( 0x28, 0x15 ), # AL=0x8e, AF=0 CF=1
+    ( 0x88, 0x94 ), # AL=0x8e, AF=1 CF=0
+    ( 0x28, 0x15 ), # AL=0x8e, AF=1 CF=1
+    ( 0x89, 0x90 ), # AL=0x8f, AF=0 CF=0
+    ( 0x29, 0x11 ), # AL=0x8f, AF=0 CF=1
+    ( 0x89, 0x90 ), # AL=0x8f, AF=1 CF=0
+    ( 0x29, 0x11 ), # AL=0x8f, AF=1 CF=1
+    ( 0x90, 0x84 ), # AL=0x90, AF=0 CF=0
+    ( 0x30, 0x05 ), # AL=0x90, AF=0 CF=1
+    ( 0x8a, 0x90 ), # AL=0x90, AF=1 CF=0
+    ( 0x2a, 0x11 ), # AL=0x90, AF=1 CF=1
+    ( 0x91, 0x80 ), # AL=0x91, AF=0 CF=0
+    ( 0x31, 0x01 ), # AL=0x91, AF=0 CF=1
+    ( 0x8b, 0x94 ), # AL=0x91, AF=1 CF=0
+    ( 0x2b, 0x15 ), # AL=0x91, AF=1 CF=1
+    ( 0x92, 0x80 ), # AL=0x92, AF=0 CF=0
+    ( 0x32, 0x01 ), # AL=0x92, AF=0 CF=1
+    ( 0x8c, 0x90 ), # AL=0x92, AF=1 CF=0
+    ( 0x2c, 0x11 ), # AL=0x92, AF=1 CF=1
+    ( 0x93, 0x84 ), # AL=0x93, AF=0 CF=0
+    ( 0x33, 0x05 ), # AL=0x93, AF=0 CF=1
+    ( 0x8d, 0x94 ), # AL=0x93, AF=1 CF=0
+    ( 0x2d, 0x15 ), # AL=0x93, AF=1 CF=1
+    ( 0x94, 0x80 ), # AL=0x94, AF=0 CF=0
+    ( 0x34, 0x01 ), # AL=0x94, AF=0 CF=1
+    ( 0x8e, 0x94 ), # AL=0x94, AF=1 CF=0
+    ( 0x2e, 0x15 ), # AL=0x94, AF=1 CF=1
+    ( 0x95, 0x84 ), # AL=0x95, AF=0 CF=0
+    ( 0x35, 0x05 ), # AL=0x95, AF=0 CF=1
+    ( 0x8f, 0x90 ), # AL=0x95, AF=1 CF=0
+    ( 0x2f, 0x11 ), # AL=0x95, AF=1 CF=1
+    ( 0x96, 0x84 ), # AL=0x96, AF=0 CF=0
+    ( 0x36, 0x05 ), # AL=0x96, AF=0 CF=1
+    ( 0x90, 0x94 ), # AL=0x96, AF=1 CF=0
+    ( 0x30, 0x15 ), # AL=0x96, AF=1 CF=1
+    ( 0x97, 0x80 ), # AL=0x97, AF=0 CF=0
+    ( 0x37, 0x01 ), # AL=0x97, AF=0 CF=1
+    ( 0x91, 0x90 ), # AL=0x97, AF=1 CF=0
+    ( 0x31, 0x11 ), # AL=0x97, AF=1 CF=1
+    ( 0x98, 0x80 ), # AL=0x98, AF=0 CF=0
+    ( 0x38, 0x01 ), # AL=0x98, AF=0 CF=1
+    ( 0x92, 0x90 ), # AL=0x98, AF=1 CF=0
+    ( 0x32, 0x11 ), # AL=0x98, AF=1 CF=1
+    ( 0x99, 0x84 ), # AL=0x99, AF=0 CF=0
+    ( 0x39, 0x05 ), # AL=0x99, AF=0 CF=1
+    ( 0x93, 0x94 ), # AL=0x99, AF=1 CF=0
+    ( 0x33, 0x15 ), # AL=0x99, AF=1 CF=1
+    ( 0x34, 0x11 ), # AL=0x9a, AF=0 CF=0
+    ( 0x34, 0x11 ), # AL=0x9a, AF=0 CF=1
+    ( 0x34, 0x11 ), # AL=0x9a, AF=1 CF=0
+    ( 0x34, 0x11 ), # AL=0x9a, AF=1 CF=1
+    ( 0x35, 0x15 ), # AL=0x9b, AF=0 CF=0
+    ( 0x35, 0x15 ), # AL=0x9b, AF=0 CF=1
+    ( 0x35, 0x15 ), # AL=0x9b, AF=1 CF=0
+    ( 0x35, 0x15 ), # AL=0x9b, AF=1 CF=1
+    ( 0x36, 0x15 ), # AL=0x9c, AF=0 CF=0
+    ( 0x36, 0x15 ), # AL=0x9c, AF=0 CF=1
+    ( 0x36, 0x15 ), # AL=0x9c, AF=1 CF=0
+    ( 0x36, 0x15 ), # AL=0x9c, AF=1 CF=1
+    ( 0x37, 0x11 ), # AL=0x9d, AF=0 CF=0
+    ( 0x37, 0x11 ), # AL=0x9d, AF=0 CF=1
+    ( 0x37, 0x11 ), # AL=0x9d, AF=1 CF=0
+    ( 0x37, 0x11 ), # AL=0x9d, AF=1 CF=1
+    ( 0x38, 0x11 ), # AL=0x9e, AF=0 CF=0
+    ( 0x38, 0x11 ), # AL=0x9e, AF=0 CF=1
+    ( 0x38, 0x11 ), # AL=0x9e, AF=1 CF=0
+    ( 0x38, 0x11 ), # AL=0x9e, AF=1 CF=1
+    ( 0x39, 0x15 ), # AL=0x9f, AF=0 CF=0
+    ( 0x39, 0x15 ), # AL=0x9f, AF=0 CF=1
+    ( 0x39, 0x15 ), # AL=0x9f, AF=1 CF=0
+    ( 0x39, 0x15 ), # AL=0x9f, AF=1 CF=1
+    ( 0x40, 0x01 ), # AL=0xa0, AF=0 CF=0
+    ( 0x40, 0x01 ), # AL=0xa0, AF=0 CF=1
+    ( 0x3a, 0x15 ), # AL=0xa0, AF=1 CF=0
+    ( 0x3a, 0x15 ), # AL=0xa0, AF=1 CF=1
+    ( 0x41, 0x05 ), # AL=0xa1, AF=0 CF=0
+    ( 0x41, 0x05 ), # AL=0xa1, AF=0 CF=1
+    ( 0x3b, 0x11 ), # AL=0xa1, AF=1 CF=0
+    ( 0x3b, 0x11 ), # AL=0xa1, AF=1 CF=1
+    ( 0x42, 0x05 ), # AL=0xa2, AF=0 CF=0
+    ( 0x42, 0x05 ), # AL=0xa2, AF=0 CF=1
+    ( 0x3c, 0x15 ), # AL=0xa2, AF=1 CF=0
+    ( 0x3c, 0x15 ), # AL=0xa2, AF=1 CF=1
+    ( 0x43, 0x01 ), # AL=0xa3, AF=0 CF=0
+    ( 0x43, 0x01 ), # AL=0xa3, AF=0 CF=1
+    ( 0x3d, 0x11 ), # AL=0xa3, AF=1 CF=0
+    ( 0x3d, 0x11 ), # AL=0xa3, AF=1 CF=1
+    ( 0x44, 0x05 ), # AL=0xa4, AF=0 CF=0
+    ( 0x44, 0x05 ), # AL=0xa4, AF=0 CF=1
+    ( 0x3e, 0x11 ), # AL=0xa4, AF=1 CF=0
+    ( 0x3e, 0x11 ), # AL=0xa4, AF=1 CF=1
+    ( 0x45, 0x01 ), # AL=0xa5, AF=0 CF=0
+    ( 0x45, 0x01 ), # AL=0xa5, AF=0 CF=1
+    ( 0x3f, 0x15 ), # AL=0xa5, AF=1 CF=0
+    ( 0x3f, 0x15 ), # AL=0xa5, AF=1 CF=1
+    ( 0x46, 0x01 ), # AL=0xa6, AF=0 CF=0
+    ( 0x46, 0x01 ), # AL=0xa6, AF=0 CF=1
+    ( 0x40, 0x11 ), # AL=0xa6, AF=1 CF=0
+    ( 0x40, 0x11 ), # AL=0xa6, AF=1 CF=1
+    ( 0x47, 0x05 ), # AL=0xa7, AF=0 CF=0
+    ( 0x47, 0x05 ), # AL=0xa7, AF=0 CF=1
+    ( 0x41, 0x15 ), # AL=0xa7, AF=1 CF=0
+    ( 0x41, 0x15 ), # AL=0xa7, AF=1 CF=1
+    ( 0x48, 0x05 ), # AL=0xa8, AF=0 CF=0
+    ( 0x48, 0x05 ), # AL=0xa8, AF=0 CF=1
+    ( 0x42, 0x15 ), # AL=0xa8, AF=1 CF=0
+    ( 0x42, 0x15 ), # AL=0xa8, AF=1 CF=1
+    ( 0x49, 0x01 ), # AL=0xa9, AF=0 CF=0
+    ( 0x49, 0x01 ), # AL=0xa9, AF=0 CF=1
+    ( 0x43, 0x11 ), # AL=0xa9, AF=1 CF=0
+    ( 0x43, 0x11 ), # AL=0xa9, AF=1 CF=1
+    ( 0x44, 0x15 ), # AL=0xaa, AF=0 CF=0
+    ( 0x44, 0x15 ), # AL=0xaa, AF=0 CF=1
+    ( 0x44, 0x15 ), # AL=0xaa, AF=1 CF=0
+    ( 0x44, 0x15 ), # AL=0xaa, AF=1 CF=1
+    ( 0x45, 0x11 ), # AL=0xab, AF=0 CF=0
+    ( 0x45, 0x11 ), # AL=0xab, AF=0 CF=1
+    ( 0x45, 0x11 ), # AL=0xab, AF=1 CF=0
+    ( 0x45, 0x11 ), # AL=0xab, AF=1 CF=1
+    ( 0x46, 0x11 ), # AL=0xac, AF=0 CF=0
+    ( 0x46, 0x11 ), # AL=0xac, AF=0 CF=1
+    ( 0x46, 0x11 ), # AL=0xac, AF=1 CF=0
+    ( 0x46, 0x11 ), # AL=0xac, AF=1 CF=1
+    ( 0x47, 0x15 ), # AL=0xad, AF=0 CF=0
+    ( 0x47, 0x15 ), # AL=0xad, AF=0 CF=1
+    ( 0x47, 0x15 ), # AL=0xad, AF=1 CF=0
+    ( 0x47, 0x15 ), # AL=0xad, AF=1 CF=1
+    ( 0x48, 0x15 ), # AL=0xae, AF=0 CF=0
+    ( 0x48, 0x15 ), # AL=0xae, AF=0 CF=1
+    ( 0x48, 0x15 ), # AL=0xae, AF=1 CF=0
+    ( 0x48, 0x15 ), # AL=0xae, AF=1 CF=1
+    ( 0x49, 0x11 ), # AL=0xaf, AF=0 CF=0
+    ( 0x49, 0x11 ), # AL=0xaf, AF=0 CF=1
+    ( 0x49, 0x11 ), # AL=0xaf, AF=1 CF=0
+    ( 0x49, 0x11 ), # AL=0xaf, AF=1 CF=1
+    ( 0x50, 0x05 ), # AL=0xb0, AF=0 CF=0
+    ( 0x50, 0x05 ), # AL=0xb0, AF=0 CF=1
+    ( 0x4a, 0x11 ), # AL=0xb0, AF=1 CF=0
+    ( 0x4a, 0x11 ), # AL=0xb0, AF=1 CF=1
+    ( 0x51, 0x01 ), # AL=0xb1, AF=0 CF=0
+    ( 0x51, 0x01 ), # AL=0xb1, AF=0 CF=1
+    ( 0x4b, 0x15 ), # AL=0xb1, AF=1 CF=0
+    ( 0x4b, 0x15 ), # AL=0xb1, AF=1 CF=1
+    ( 0x52, 0x01 ), # AL=0xb2, AF=0 CF=0
+    ( 0x52, 0x01 ), # AL=0xb2, AF=0 CF=1
+    ( 0x4c, 0x11 ), # AL=0xb2, AF=1 CF=0
+    ( 0x4c, 0x11 ), # AL=0xb2, AF=1 CF=1
+    ( 0x53, 0x05 ), # AL=0xb3, AF=0 CF=0
+    ( 0x53, 0x05 ), # AL=0xb3, AF=0 CF=1
+    ( 0x4d, 0x15 ), # AL=0xb3, AF=1 CF=0
+    ( 0x4d, 0x15 ), # AL=0xb3, AF=1 CF=1
+    ( 0x54, 0x01 ), # AL=0xb4, AF=0 CF=0
+    ( 0x54, 0x01 ), # AL=0xb4, AF=0 CF=1
+    ( 0x4e, 0x15 ), # AL=0xb4, AF=1 CF=0
+    ( 0x4e, 0x15 ), # AL=0xb4, AF=1 CF=1
+    ( 0x55, 0x05 ), # AL=0xb5, AF=0 CF=0
+    ( 0x55, 0x05 ), # AL=0xb5, AF=0 CF=1
+    ( 0x4f, 0x11 ), # AL=0xb5, AF=1 CF=0
+    ( 0x4f, 0x11 ), # AL=0xb5, AF=1 CF=1
+    ( 0x56, 0x05 ), # AL=0xb6, AF=0 CF=0
+    ( 0x56, 0x05 ), # AL=0xb6, AF=0 CF=1
+    ( 0x50, 0x15 ), # AL=0xb6, AF=1 CF=0
+    ( 0x50, 0x15 ), # AL=0xb6, AF=1 CF=1
+    ( 0x57, 0x01 ), # AL=0xb7, AF=0 CF=0
+    ( 0x57, 0x01 ), # AL=0xb7, AF=0 CF=1
+    ( 0x51, 0x11 ), # AL=0xb7, AF=1 CF=0
+    ( 0x51, 0x11 ), # AL=0xb7, AF=1 CF=1
+    ( 0x58, 0x01 ), # AL=0xb8, AF=0 CF=0
+    ( 0x58, 0x01 ), # AL=0xb8, AF=0 CF=1
+    ( 0x52, 0x11 ), # AL=0xb8, AF=1 CF=0
+    ( 0x52, 0x11 ), # AL=0xb8, AF=1 CF=1
+    ( 0x59, 0x05 ), # AL=0xb9, AF=0 CF=0
+    ( 0x59, 0x05 ), # AL=0xb9, AF=0 CF=1
+    ( 0x53, 0x15 ), # AL=0xb9, AF=1 CF=0
+    ( 0x53, 0x15 ), # AL=0xb9, AF=1 CF=1
+    ( 0x54, 0x11 ), # AL=0xba, AF=0 CF=0
+    ( 0x54, 0x11 ), # AL=0xba, AF=0 CF=1
+    ( 0x54, 0x11 ), # AL=0xba, AF=1 CF=0
+    ( 0x54, 0x11 ), # AL=0xba, AF=1 CF=1
+    ( 0x55, 0x15 ), # AL=0xbb, AF=0 CF=0
+    ( 0x55, 0x15 ), # AL=0xbb, AF=0 CF=1
+    ( 0x55, 0x15 ), # AL=0xbb, AF=1 CF=0
+    ( 0x55, 0x15 ), # AL=0xbb, AF=1 CF=1
+    ( 0x56, 0x15 ), # AL=0xbc, AF=0 CF=0
+    ( 0x56, 0x15 ), # AL=0xbc, AF=0 CF=1
+    ( 0x56, 0x15 ), # AL=0xbc, AF=1 CF=0
+    ( 0x56, 0x15 ), # AL=0xbc, AF=1 CF=1
+    ( 0x57, 0x11 ), # AL=0xbd, AF=0 CF=0
+    ( 0x57, 0x11 ), # AL=0xbd, AF=0 CF=1
+    ( 0x57, 0x11 ), # AL=0xbd, AF=1 CF=0
+    ( 0x57, 0x11 ), # AL=0xbd, AF=1 CF=1
+    ( 0x58, 0x11 ), # AL=0xbe, AF=0 CF=0
+    ( 0x58, 0x11 ), # AL=0xbe, AF=0 CF=1
+    ( 0x58, 0x11 ), # AL=0xbe, AF=1 CF=0
+    ( 0x58, 0x11 ), # AL=0xbe, AF=1 CF=1
+    ( 0x59, 0x15 ), # AL=0xbf, AF=0 CF=0
+    ( 0x59, 0x15 ), # AL=0xbf, AF=0 CF=1
+    ( 0x59, 0x15 ), # AL=0xbf, AF=1 CF=0
+    ( 0x59, 0x15 ), # AL=0xbf, AF=1 CF=1
+    ( 0x60, 0x05 ), # AL=0xc0, AF=0 CF=0
+    ( 0x60, 0x05 ), # AL=0xc0, AF=0 CF=1
+    ( 0x5a, 0x15 ), # AL=0xc0, AF=1 CF=0
+    ( 0x5a, 0x15 ), # AL=0xc0, AF=1 CF=1
+    ( 0x61, 0x01 ), # AL=0xc1, AF=0 CF=0
+    ( 0x61, 0x01 ), # AL=0xc1, AF=0 CF=1
+    ( 0x5b, 0x11 ), # AL=0xc1, AF=1 CF=0
+    ( 0x5b, 0x11 ), # AL=0xc1, AF=1 CF=1
+    ( 0x62, 0x01 ), # AL=0xc2, AF=0 CF=0
+    ( 0x62, 0x01 ), # AL=0xc2, AF=0 CF=1
+    ( 0x5c, 0x15 ), # AL=0xc2, AF=1 CF=0
+    ( 0x5c, 0x15 ), # AL=0xc2, AF=1 CF=1
+    ( 0x63, 0x05 ), # AL=0xc3, AF=0 CF=0
+    ( 0x63, 0x05 ), # AL=0xc3, AF=0 CF=1
+    ( 0x5d, 0x11 ), # AL=0xc3, AF=1 CF=0
+    ( 0x5d, 0x11 ), # AL=0xc3, AF=1 CF=1
+    ( 0x64, 0x01 ), # AL=0xc4, AF=0 CF=0
+    ( 0x64, 0x01 ), # AL=0xc4, AF=0 CF=1
+    ( 0x5e, 0x11 ), # AL=0xc4, AF=1 CF=0
+    ( 0x5e, 0x11 ), # AL=0xc4, AF=1 CF=1
+    ( 0x65, 0x05 ), # AL=0xc5, AF=0 CF=0
+    ( 0x65, 0x05 ), # AL=0xc5, AF=0 CF=1
+    ( 0x5f, 0x15 ), # AL=0xc5, AF=1 CF=0
+    ( 0x5f, 0x15 ), # AL=0xc5, AF=1 CF=1
+    ( 0x66, 0x05 ), # AL=0xc6, AF=0 CF=0
+    ( 0x66, 0x05 ), # AL=0xc6, AF=0 CF=1
+    ( 0x60, 0x15 ), # AL=0xc6, AF=1 CF=0
+    ( 0x60, 0x15 ), # AL=0xc6, AF=1 CF=1
+    ( 0x67, 0x01 ), # AL=0xc7, AF=0 CF=0
+    ( 0x67, 0x01 ), # AL=0xc7, AF=0 CF=1
+    ( 0x61, 0x11 ), # AL=0xc7, AF=1 CF=0
+    ( 0x61, 0x11 ), # AL=0xc7, AF=1 CF=1
+    ( 0x68, 0x01 ), # AL=0xc8, AF=0 CF=0
+    ( 0x68, 0x01 ), # AL=0xc8, AF=0 CF=1
+    ( 0x62, 0x11 ), # AL=0xc8, AF=1 CF=0
+    ( 0x62, 0x11 ), # AL=0xc8, AF=1 CF=1
+    ( 0x69, 0x05 ), # AL=0xc9, AF=0 CF=0
+    ( 0x69, 0x05 ), # AL=0xc9, AF=0 CF=1
+    ( 0x63, 0x15 ), # AL=0xc9, AF=1 CF=0
+    ( 0x63, 0x15 ), # AL=0xc9, AF=1 CF=1
+    ( 0x64, 0x11 ), # AL=0xca, AF=0 CF=0
+    ( 0x64, 0x11 ), # AL=0xca, AF=0 CF=1
+    ( 0x64, 0x11 ), # AL=0xca, AF=1 CF=0
+    ( 0x64, 0x11 ), # AL=0xca, AF=1 CF=1
+    ( 0x65, 0x15 ), # AL=0xcb, AF=0 CF=0
+    ( 0x65, 0x15 ), # AL=0xcb, AF=0 CF=1
+    ( 0x65, 0x15 ), # AL=0xcb, AF=1 CF=0
+    ( 0x65, 0x15 ), # AL=0xcb, AF=1 CF=1
+    ( 0x66, 0x15 ), # AL=0xcc, AF=0 CF=0
+    ( 0x66, 0x15 ), # AL=0xcc, AF=0 CF=1
+    ( 0x66, 0x15 ), # AL=0xcc, AF=1 CF=0
+    ( 0x66, 0x15 ), # AL=0xcc, AF=1 CF=1
+    ( 0x67, 0x11 ), # AL=0xcd, AF=0 CF=0
+    ( 0x67, 0x11 ), # AL=0xcd, AF=0 CF=1
+    ( 0x67, 0x11 ), # AL=0xcd, AF=1 CF=0
+    ( 0x67, 0x11 ), # AL=0xcd, AF=1 CF=1
+    ( 0x68, 0x11 ), # AL=0xce, AF=0 CF=0
+    ( 0x68, 0x11 ), # AL=0xce, AF=0 CF=1
+    ( 0x68, 0x11 ), # AL=0xce, AF=1 CF=0
+    ( 0x68, 0x11 ), # AL=0xce, AF=1 CF=1
+    ( 0x69, 0x15 ), # AL=0xcf, AF=0 CF=0
+    ( 0x69, 0x15 ), # AL=0xcf, AF=0 CF=1
+    ( 0x69, 0x15 ), # AL=0xcf, AF=1 CF=0
+    ( 0x69, 0x15 ), # AL=0xcf, AF=1 CF=1
+    ( 0x70, 0x01 ), # AL=0xd0, AF=0 CF=0
+    ( 0x70, 0x01 ), # AL=0xd0, AF=0 CF=1
+    ( 0x6a, 0x15 ), # AL=0xd0, AF=1 CF=0
+    ( 0x6a, 0x15 ), # AL=0xd0, AF=1 CF=1
+    ( 0x71, 0x05 ), # AL=0xd1, AF=0 CF=0
+    ( 0x71, 0x05 ), # AL=0xd1, AF=0 CF=1
+    ( 0x6b, 0x11 ), # AL=0xd1, AF=1 CF=0
+    ( 0x6b, 0x11 ), # AL=0xd1, AF=1 CF=1
+    ( 0x72, 0x05 ), # AL=0xd2, AF=0 CF=0
+    ( 0x72, 0x05 ), # AL=0xd2, AF=0 CF=1
+    ( 0x6c, 0x15 ), # AL=0xd2, AF=1 CF=0
+    ( 0x6c, 0x15 ), # AL=0xd2, AF=1 CF=1
+    ( 0x73, 0x01 ), # AL=0xd3, AF=0 CF=0
+    ( 0x73, 0x01 ), # AL=0xd3, AF=0 CF=1
+    ( 0x6d, 0x11 ), # AL=0xd3, AF=1 CF=0
+    ( 0x6d, 0x11 ), # AL=0xd3, AF=1 CF=1
+    ( 0x74, 0x05 ), # AL=0xd4, AF=0 CF=0
+    ( 0x74, 0x05 ), # AL=0xd4, AF=0 CF=1
+    ( 0x6e, 0x11 ), # AL=0xd4, AF=1 CF=0
+    ( 0x6e, 0x11 ), # AL=0xd4, AF=1 CF=1
+    ( 0x75, 0x01 ), # AL=0xd5, AF=0 CF=0
+    ( 0x75, 0x01 ), # AL=0xd5, AF=0 CF=1
+    ( 0x6f, 0x15 ), # AL=0xd5, AF=1 CF=0
+    ( 0x6f, 0x15 ), # AL=0xd5, AF=1 CF=1
+    ( 0x76, 0x01 ), # AL=0xd6, AF=0 CF=0
+    ( 0x76, 0x01 ), # AL=0xd6, AF=0 CF=1
+    ( 0x70, 0x11 ), # AL=0xd6, AF=1 CF=0
+    ( 0x70, 0x11 ), # AL=0xd6, AF=1 CF=1
+    ( 0x77, 0x05 ), # AL=0xd7, AF=0 CF=0
+    ( 0x77, 0x05 ), # AL=0xd7, AF=0 CF=1
+    ( 0x71, 0x15 ), # AL=0xd7, AF=1 CF=0
+    ( 0x71, 0x15 ), # AL=0xd7, AF=1 CF=1
+    ( 0x78, 0x05 ), # AL=0xd8, AF=0 CF=0
+    ( 0x78, 0x05 ), # AL=0xd8, AF=0 CF=1
+    ( 0x72, 0x15 ), # AL=0xd8, AF=1 CF=0
+    ( 0x72, 0x15 ), # AL=0xd8, AF=1 CF=1
+    ( 0x79, 0x01 ), # AL=0xd9, AF=0 CF=0
+    ( 0x79, 0x01 ), # AL=0xd9, AF=0 CF=1
+    ( 0x73, 0x11 ), # AL=0xd9, AF=1 CF=0
+    ( 0x73, 0x11 ), # AL=0xd9, AF=1 CF=1
+    ( 0x74, 0x15 ), # AL=0xda, AF=0 CF=0
+    ( 0x74, 0x15 ), # AL=0xda, AF=0 CF=1
+    ( 0x74, 0x15 ), # AL=0xda, AF=1 CF=0
+    ( 0x74, 0x15 ), # AL=0xda, AF=1 CF=1
+    ( 0x75, 0x11 ), # AL=0xdb, AF=0 CF=0
+    ( 0x75, 0x11 ), # AL=0xdb, AF=0 CF=1
+    ( 0x75, 0x11 ), # AL=0xdb, AF=1 CF=0
+    ( 0x75, 0x11 ), # AL=0xdb, AF=1 CF=1
+    ( 0x76, 0x11 ), # AL=0xdc, AF=0 CF=0
+    ( 0x76, 0x11 ), # AL=0xdc, AF=0 CF=1
+    ( 0x76, 0x11 ), # AL=0xdc, AF=1 CF=0
+    ( 0x76, 0x11 ), # AL=0xdc, AF=1 CF=1
+    ( 0x77, 0x15 ), # AL=0xdd, AF=0 CF=0
+    ( 0x77, 0x15 ), # AL=0xdd, AF=0 CF=1
+    ( 0x77, 0x15 ), # AL=0xdd, AF=1 CF=0
+    ( 0x77, 0x15 ), # AL=0xdd, AF=1 CF=1
+    ( 0x78, 0x15 ), # AL=0xde, AF=0 CF=0
+    ( 0x78, 0x15 ), # AL=0xde, AF=0 CF=1
+    ( 0x78, 0x15 ), # AL=0xde, AF=1 CF=0
+    ( 0x78, 0x15 ), # AL=0xde, AF=1 CF=1
+    ( 0x79, 0x11 ), # AL=0xdf, AF=0 CF=0
+    ( 0x79, 0x11 ), # AL=0xdf, AF=0 CF=1
+    ( 0x79, 0x11 ), # AL=0xdf, AF=1 CF=0
+    ( 0x79, 0x11 ), # AL=0xdf, AF=1 CF=1
+    ( 0x80, 0x81 ), # AL=0xe0, AF=0 CF=0
+    ( 0x80, 0x81 ), # AL=0xe0, AF=0 CF=1
+    ( 0x7a, 0x11 ), # AL=0xe0, AF=1 CF=0
+    ( 0x7a, 0x11 ), # AL=0xe0, AF=1 CF=1
+    ( 0x81, 0x85 ), # AL=0xe1, AF=0 CF=0
+    ( 0x81, 0x85 ), # AL=0xe1, AF=0 CF=1
+    ( 0x7b, 0x15 ), # AL=0xe1, AF=1 CF=0
+    ( 0x7b, 0x15 ), # AL=0xe1, AF=1 CF=1
+    ( 0x82, 0x85 ), # AL=0xe2, AF=0 CF=0
+    ( 0x82, 0x85 ), # AL=0xe2, AF=0 CF=1
+    ( 0x7c, 0x11 ), # AL=0xe2, AF=1 CF=0
+    ( 0x7c, 0x11 ), # AL=0xe2, AF=1 CF=1
+    ( 0x83, 0x81 ), # AL=0xe3, AF=0 CF=0
+    ( 0x83, 0x81 ), # AL=0xe3, AF=0 CF=1
+    ( 0x7d, 0x15 ), # AL=0xe3, AF=1 CF=0
+    ( 0x7d, 0x15 ), # AL=0xe3, AF=1 CF=1
+    ( 0x84, 0x85 ), # AL=0xe4, AF=0 CF=0
+    ( 0x84, 0x85 ), # AL=0xe4, AF=0 CF=1
+    ( 0x7e, 0x15 ), # AL=0xe4, AF=1 CF=0
+    ( 0x7e, 0x15 ), # AL=0xe4, AF=1 CF=1
+    ( 0x85, 0x81 ), # AL=0xe5, AF=0 CF=0
+    ( 0x85, 0x81 ), # AL=0xe5, AF=0 CF=1
+    ( 0x7f, 0x11 ), # AL=0xe5, AF=1 CF=0
+    ( 0x7f, 0x11 ), # AL=0xe5, AF=1 CF=1
+    ( 0x86, 0x81 ), # AL=0xe6, AF=0 CF=0
+    ( 0x86, 0x81 ), # AL=0xe6, AF=0 CF=1
+    ( 0x80, 0x91 ), # AL=0xe6, AF=1 CF=0
+    ( 0x80, 0x91 ), # AL=0xe6, AF=1 CF=1
+    ( 0x87, 0x85 ), # AL=0xe7, AF=0 CF=0
+    ( 0x87, 0x85 ), # AL=0xe7, AF=0 CF=1
+    ( 0x81, 0x95 ), # AL=0xe7, AF=1 CF=0
+    ( 0x81, 0x95 ), # AL=0xe7, AF=1 CF=1
+    ( 0x88, 0x85 ), # AL=0xe8, AF=0 CF=0
+    ( 0x88, 0x85 ), # AL=0xe8, AF=0 CF=1
+    ( 0x82, 0x95 ), # AL=0xe8, AF=1 CF=0
+    ( 0x82, 0x95 ), # AL=0xe8, AF=1 CF=1
+    ( 0x89, 0x81 ), # AL=0xe9, AF=0 CF=0
+    ( 0x89, 0x81 ), # AL=0xe9, AF=0 CF=1
+    ( 0x83, 0x91 ), # AL=0xe9, AF=1 CF=0
+    ( 0x83, 0x91 ), # AL=0xe9, AF=1 CF=1
+    ( 0x84, 0x95 ), # AL=0xea, AF=0 CF=0
+    ( 0x84, 0x95 ), # AL=0xea, AF=0 CF=1
+    ( 0x84, 0x95 ), # AL=0xea, AF=1 CF=0
+    ( 0x84, 0x95 ), # AL=0xea, AF=1 CF=1
+    ( 0x85, 0x91 ), # AL=0xeb, AF=0 CF=0
+    ( 0x85, 0x91 ), # AL=0xeb, AF=0 CF=1
+    ( 0x85, 0x91 ), # AL=0xeb, AF=1 CF=0
+    ( 0x85, 0x91 ), # AL=0xeb, AF=1 CF=1
+    ( 0x86, 0x91 ), # AL=0xec, AF=0 CF=0
+    ( 0x86, 0x91 ), # AL=0xec, AF=0 CF=1
+    ( 0x86, 0x91 ), # AL=0xec, AF=1 CF=0
+    ( 0x86, 0x91 ), # AL=0xec, AF=1 CF=1
+    ( 0x87, 0x95 ), # AL=0xed, AF=0 CF=0
+    ( 0x87, 0x95 ), # AL=0xed, AF=0 CF=1
+    ( 0x87, 0x95 ), # AL=0xed, AF=1 CF=0
+    ( 0x87, 0x95 ), # AL=0xed, AF=1 CF=1
+    ( 0x88, 0x95 ), # AL=0xee, AF=0 CF=0
+    ( 0x88, 0x95 ), # AL=0xee, AF=0 CF=1
+    ( 0x88, 0x95 ), # AL=0xee, AF=1 CF=0
+    ( 0x88, 0x95 ), # AL=0xee, AF=1 CF=1
+    ( 0x89, 0x91 ), # AL=0xef, AF=0 CF=0
+    ( 0x89, 0x91 ), # AL=0xef, AF=0 CF=1
+    ( 0x89, 0x91 ), # AL=0xef, AF=1 CF=0
+    ( 0x89, 0x91 ), # AL=0xef, AF=1 CF=1
+    ( 0x90, 0x85 ), # AL=0xf0, AF=0 CF=0
+    ( 0x90, 0x85 ), # AL=0xf0, AF=0 CF=1
+    ( 0x8a, 0x91 ), # AL=0xf0, AF=1 CF=0
+    ( 0x8a, 0x91 ), # AL=0xf0, AF=1 CF=1
+    ( 0x91, 0x81 ), # AL=0xf1, AF=0 CF=0
+    ( 0x91, 0x81 ), # AL=0xf1, AF=0 CF=1
+    ( 0x8b, 0x95 ), # AL=0xf1, AF=1 CF=0
+    ( 0x8b, 0x95 ), # AL=0xf1, AF=1 CF=1
+    ( 0x92, 0x81 ), # AL=0xf2, AF=0 CF=0
+    ( 0x92, 0x81 ), # AL=0xf2, AF=0 CF=1
+    ( 0x8c, 0x91 ), # AL=0xf2, AF=1 CF=0
+    ( 0x8c, 0x91 ), # AL=0xf2, AF=1 CF=1
+    ( 0x93, 0x85 ), # AL=0xf3, AF=0 CF=0
+    ( 0x93, 0x85 ), # AL=0xf3, AF=0 CF=1
+    ( 0x8d, 0x95 ), # AL=0xf3, AF=1 CF=0
+    ( 0x8d, 0x95 ), # AL=0xf3, AF=1 CF=1
+    ( 0x94, 0x81 ), # AL=0xf4, AF=0 CF=0
+    ( 0x94, 0x81 ), # AL=0xf4, AF=0 CF=1
+    ( 0x8e, 0x95 ), # AL=0xf4, AF=1 CF=0
+    ( 0x8e, 0x95 ), # AL=0xf4, AF=1 CF=1
+    ( 0x95, 0x85 ), # AL=0xf5, AF=0 CF=0
+    ( 0x95, 0x85 ), # AL=0xf5, AF=0 CF=1
+    ( 0x8f, 0x91 ), # AL=0xf5, AF=1 CF=0
+    ( 0x8f, 0x91 ), # AL=0xf5, AF=1 CF=1
+    ( 0x96, 0x85 ), # AL=0xf6, AF=0 CF=0
+    ( 0x96, 0x85 ), # AL=0xf6, AF=0 CF=1
+    ( 0x90, 0x95 ), # AL=0xf6, AF=1 CF=0
+    ( 0x90, 0x95 ), # AL=0xf6, AF=1 CF=1
+    ( 0x97, 0x81 ), # AL=0xf7, AF=0 CF=0
+    ( 0x97, 0x81 ), # AL=0xf7, AF=0 CF=1
+    ( 0x91, 0x91 ), # AL=0xf7, AF=1 CF=0
+    ( 0x91, 0x91 ), # AL=0xf7, AF=1 CF=1
+    ( 0x98, 0x81 ), # AL=0xf8, AF=0 CF=0
+    ( 0x98, 0x81 ), # AL=0xf8, AF=0 CF=1
+    ( 0x92, 0x91 ), # AL=0xf8, AF=1 CF=0
+    ( 0x92, 0x91 ), # AL=0xf8, AF=1 CF=1
+    ( 0x99, 0x85 ), # AL=0xf9, AF=0 CF=0
+    ( 0x99, 0x85 ), # AL=0xf9, AF=0 CF=1
+    ( 0x93, 0x95 ), # AL=0xf9, AF=1 CF=0
+    ( 0x93, 0x95 ), # AL=0xf9, AF=1 CF=1
+    ( 0x94, 0x91 ), # AL=0xfa, AF=0 CF=0
+    ( 0x94, 0x91 ), # AL=0xfa, AF=0 CF=1
+    ( 0x94, 0x91 ), # AL=0xfa, AF=1 CF=0
+    ( 0x94, 0x91 ), # AL=0xfa, AF=1 CF=1
+    ( 0x95, 0x95 ), # AL=0xfb, AF=0 CF=0
+    ( 0x95, 0x95 ), # AL=0xfb, AF=0 CF=1
+    ( 0x95, 0x95 ), # AL=0xfb, AF=1 CF=0
+    ( 0x95, 0x95 ), # AL=0xfb, AF=1 CF=1
+    ( 0x96, 0x95 ), # AL=0xfc, AF=0 CF=0
+    ( 0x96, 0x95 ), # AL=0xfc, AF=0 CF=1
+    ( 0x96, 0x95 ), # AL=0xfc, AF=1 CF=0
+    ( 0x96, 0x95 ), # AL=0xfc, AF=1 CF=1
+    ( 0x97, 0x91 ), # AL=0xfd, AF=0 CF=0
+    ( 0x97, 0x91 ), # AL=0xfd, AF=0 CF=1
+    ( 0x97, 0x91 ), # AL=0xfd, AF=1 CF=0
+    ( 0x97, 0x91 ), # AL=0xfd, AF=1 CF=1
+    ( 0x98, 0x91 ), # AL=0xfe, AF=0 CF=0
+    ( 0x98, 0x91 ), # AL=0xfe, AF=0 CF=1
+    ( 0x98, 0x91 ), # AL=0xfe, AF=1 CF=0
+    ( 0x98, 0x91 ), # AL=0xfe, AF=1 CF=1
+    ( 0x99, 0x95 ), # AL=0xff, AF=0 CF=0
+    ( 0x99, 0x95 ), # AL=0xff, AF=0 CF=1
+    ( 0x99, 0x95 ), # AL=0xff, AF=1 CF=0
+    ( 0x99, 0x95 ), # AL=0xff, AF=1 CF=1
+];
+
diff --git a/src/VBox/VMM/testcase/Instructions/tstVBInsTstR3.cpp b/src/VBox/VMM/testcase/Instructions/tstVBInsTstR3.cpp
new file mode 100644
index 00000000..014bfbda
--- /dev/null
+++ b/src/VBox/VMM/testcase/Instructions/tstVBInsTstR3.cpp
@@ -0,0 +1,120 @@
+/* $Id: tstVBInsTstR3.cpp $ */
+/** @file
+ * Instruction Test Environment - IPRT ring-3 driver.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <iprt/mem.h>
+#include <iprt/string.h>
+#include <iprt/test.h>
+
+#ifdef RT_OS_WINDOWS
+# define NO_LOW_MEM
+#elif defined(RT_OS_OS2) || defined(RT_OS_HAIKU)
+# define NO_LOW_MEM
+#else
+# include <sys/mman.h>
+#endif
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+#if HC_ARCH_BITS == 64
+typedef uint64_t VBINSTSTREG;
+#else
+typedef uint32_t VBINSTSTREG;
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+RTTEST g_hTest;
+
+
+RT_C_DECLS_BEGIN
+extern void *g_pvLow16Mem4K;
+extern void *g_pvLow32Mem4K;
+DECLASM(void)    TestInstrMain(void);
+
+DECLEXPORT(void) VBInsTstFailure(const char *pszMessage);
+DECLEXPORT(void) VBInsTstFailure1(const char *pszFmt, VBINSTSTREG uArg1);
+DECLEXPORT(void) VBInsTstFailure2(const char *pszFmt, VBINSTSTREG uArg1, VBINSTSTREG uArg2);
+DECLEXPORT(void) VBInsTstFailure3(const char *pszFmt, VBINSTSTREG uArg1, VBINSTSTREG uArg2, VBINSTSTREG uArg3);
+DECLEXPORT(void) VBInsTstFailure4(const char *pszFmt, VBINSTSTREG uArg1, VBINSTSTREG uArg2, VBINSTSTREG uArg3, VBINSTSTREG uArg4);
+RT_C_DECLS_END
+
+
+DECLEXPORT(void) VBInsTstFailure(const char *pszMessage)
+{
+    RTTestFailed(g_hTest, "%s", pszMessage);
+}
+
+DECLEXPORT(void) VBInsTstFailure1(const char *pszFmt, VBINSTSTREG uArg1)
+{
+    RTTestFailed(g_hTest, pszFmt, uArg1);
+}
+
+
+DECLEXPORT(void) VBInsTstFailure2(const char *pszFmt, VBINSTSTREG uArg1, VBINSTSTREG uArg2)
+{
+    RTTestFailed(g_hTest, pszFmt, uArg1, uArg2);
+}
+
+
+DECLEXPORT(void) VBInsTstFailure3(const char *pszFmt, VBINSTSTREG uArg1, VBINSTSTREG uArg2, VBINSTSTREG uArg3)
+{
+    RTTestFailed(g_hTest, pszFmt, uArg1, uArg2, uArg3);
+}
+
+
+DECLEXPORT(void) VBInsTstFailure4(const char *pszFmt, VBINSTSTREG uArg1, VBINSTSTREG uArg2, VBINSTSTREG uArg3, VBINSTSTREG uArg4)
+{
+    RTTestFailed(g_hTest, pszFmt, uArg1, uArg2, uArg3, uArg4);
+}
+
+
+
+
+int main()
+{
+    RTEXITCODE rcExit = RTTestInitAndCreate("VBInsTstR3", &g_hTest);
+    if (rcExit != RTEXITCODE_SUCCESS)
+        return rcExit;
+    RTTestBanner(g_hTest);
+
+    int rc = RTMemAllocEx(_4K, 0, RTMEMALLOCEX_FLAGS_16BIT_REACH, &g_pvLow16Mem4K);
+    if (RT_FAILURE(rc))
+    {
+        RTTestPrintf(g_hTest, RTTESTLVL_ALWAYS, "Could not allocate low 16-bit memory (%Rrc)\n", rc);
+        g_pvLow16Mem4K = NULL;
+    }
+
+    rc = RTMemAllocEx(_4K, 0, RTMEMALLOCEX_FLAGS_32BIT_REACH, &g_pvLow32Mem4K);
+    if (RT_FAILURE(rc))
+    {
+        RTTestPrintf(g_hTest, RTTESTLVL_ALWAYS, "Could not allocate low 32-bit memory (%Rrc)\n", rc);
+        g_pvLow32Mem4K = NULL;
+    }
+
+    TestInstrMain();
+
+    return RTTestSummaryAndDestroy(g_hTest);
+}
+
diff --git a/src/VBox/VMM/testcase/Makefile.kmk b/src/VBox/VMM/testcase/Makefile.kmk
new file mode 100644
index 00000000..d7a95af9
--- /dev/null
+++ b/src/VBox/VMM/testcase/Makefile.kmk
@@ -0,0 +1,653 @@
+# $Id: Makefile.kmk $
+## @file
+# Sub-Makefile for the VMM testcases.
+#
+
+#
+# Copyright (C) 2006-2020 Oracle Corporation
+#
+# This file is part of VirtualBox Open Source Edition (OSE), as
+# available from http://www.virtualbox.org. This file is free software;
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License (GPL) as published by the Free Software
+# Foundation, in version 2 as it comes in the "COPYING" file of the
+# VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+# hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+#
+
+SUB_DEPTH = ../../../..
+include $(KBUILD_PATH)/subheader.kmk
+
+#
+# Include sub-makefiles.
+#
+if 0 # Not ready for general consumption yet.
+ include $(PATH_SUB_CURRENT)/Instructions/Makefile.kmk
+endif
+
+#
+# Target lists.
+#
+PROGRAMS   += tstVMStructSize tstAsmStructs
+ifdef VBOX_WITH_RAW_MODE
+ PROGRAMS  += tstVMStructRC tstAsmStructsRC
+endif
+if !defined(VBOX_ONLY_EXTPACKS) \
+ && (   defined(VBOX_WITH_DTRACE_R3) \
+     || defined(VBOX_WITH_DTRACE_R0) \
+     || defined(VBOX_WITH_DTRACE_RC))
+PROGRAMS += tstVMStructDTrace
+INSTALLS += VMMLibDTraceStructTest
+endif
+ifndef VBOX_ONLY_EXTPACKS_USE_IMPLIBS
+ if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+PROGRAMS += tstGlobalConfigHardened
+DLL      += tstGlobalConfig
+ else
+PROGRAMS += tstGlobalConfig
+ endif
+
+ ifdef VBOX_WITH_RAW_MODE
+  if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+PROGRAMS += tstVMMHardened
+DLLS     += tstVMM
+  else
+PROGRAMS += tstVMM tstVMM-HM
+  endif
+  ifneq ($(KBUILD_TARGET),win)
+PROGRAMS += tstVMMFork
+  endif
+ endif
+ ifdef VBOX_WITH_TESTCASES
+  if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+PROGRAMS += tstCFGMHardened tstVMREQHardened tstMMHyperHeapHardened tstAnimateHardened
+DLLS     += tstCFGM tstVMREQ tstMMHyperHeap tstAnimate
+  else
+PROGRAMS += tstCFGM tstVMREQ tstMMHyperHeap tstAnimate
+  endif
+PROGRAMS += \
+	tstCompressionBenchmark \
+	tstIEMCheckMc \
+	tstSSM \
+	tstVMMR0CallHost-1 \
+	tstVMMR0CallHost-2 \
+	tstX86-FpuSaveRestore
+  ifn1of ($(KBUILD_TARGET).$(KBUILD_TARGET_ARCH), solaris.x86 solaris.amd64 win.amd64 ) ## TODO: Fix the code.
+PROGRAMS += tstX86-1
+  endif
+  ifdef VBOX_WITH_RAW_MODE
+   if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+PROGRAMS += tstMicroHardened
+DLLS     += tstMicro
+   else
+PROGRAMS += tstMicro
+   endif
+SYSMODS   += tstMicroRC
+  endif
+  ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+   if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+PROGRAMS += tstPDMAsyncCompletionHardened tstPDMAsyncCompletionStressHardened
+DLLS     += tstPDMAsyncCompletion tstPDMAsyncCompletionStress
+   else
+PROGRAMS += tstPDMAsyncCompletion tstPDMAsyncCompletionStress
+   endif
+  endif
+ endif # VBOX_WITH_TESTCASES
+endif # !VBOX_ONLY_EXTPACKS_USE_IMPLIBS
+
+# Where we put our temporary files (just for simplicity)
+VBOX_VMM_TESTCASE_OUT_DIR := $(PATH_TARGET)/VMM
+BLDDIRS += $(VBOX_VMM_TESTCASE_OUT_DIR)
+
+#
+# We setup two 'other' targets for executing the two structure & alignment
+# validation testcases. Perhaps a bit hackish, but extremely useful.
+#
+ifeq ($(KBUILD_TARGET),$(KBUILD_HOST))
+ ifeq ($(filter-out x86.x86 amd64.amd64 x86.amd64, $(KBUILD_TARGET_ARCH).$(KBUILD_HOST_ARCH)),)
+OTHERS += \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructs.run \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstVMStructSize.run
+ endif
+endif
+
+# The normal testing pass.
+TESTING += \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructs.run \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstVMStructSize.run
+
+OTHER_CLEAN += \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructs.run \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstVMStructSize.run \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsAsm.o \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsAsm.o.dep \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsAsm.mac \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsAsm.mac.o \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsAsm.mac.lst \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsRC.h \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsHC.h \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)/tstVMStructRC.h
+
+#
+# Globals
+#
+VBOX_PATH_VMM_SRC = $(PATH_ROOT)/src/VBox/VMM
+
+#
+# Targets
+#
+ifdef VBOX_WITH_RAW_MODE
+tstVMStructRC_TEMPLATE  = VBoxRcExe
+tstVMStructRC_DEFS      = VBOX_IN_VMM IN_VMM_RC IN_DIS IN_RT_RC VBOX_WITH_RAW_MODE $(VMM_COMMON_DEFS)
+ ifdef VBOX_WITH_R0_LOGGING
+tstVMStructRC_DEFS     += VBOX_WITH_R0_LOGGING
+ endif
+ ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+tstVMStructRC_DEFS     += VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+ endif
+tstVMStructRC_SOURCES   = tstVMStructRC.cpp
+tstVMStructRC_INCS      = \
+	$(VBOX_PATH_VMM_SRC)/include \
+	$(VBOX_PATH_VMM_SRC)/PATM
+endif
+
+tstVMStructSize_TEMPLATE= VBOXR3AUTOTST
+ifneq ($(KBUILD_TARGET),win)
+tstVMStructSize_CXXFLAGS += $(VBOX_GCC_Wno-invalid-offsetof)
+endif
+tstVMStructSize_DEFS    = VBOX_IN_VMM IN_VMM_R3 IN_DIS $(VMM_COMMON_DEFS)
+ifdef VBOX_WITH_RAW_MODE
+tstVMStructSize_DEFS   += VBOX_WITH_RAW_MODE
+endif
+tstVMStructSize_INCS    = \
+	$(VBOX_PATH_VMM_SRC)/include \
+	$(VBOX_PATH_VMM_SRC)/PATM \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)
+tstVMStructSize_SOURCES = tstVMStructSize.cpp
+ifdef VBOX_WITH_RAW_MODE
+tstVMStructSize.cpp_DEPS= $(VBOX_VMM_TESTCASE_OUT_DIR)/tstVMStructRC.h
+endif
+ifdef VBOX_WITH_R0_LOGGING
+tstVMStructSize_DEFS   += VBOX_WITH_R0_LOGGING
+endif
+ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+tstVMStructSize_DEFS   += VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+endif
+
+tstAsmStructs_TEMPLATE  = VBOXR3AUTOTST
+ifneq ($(KBUILD_TARGET),win)
+tstAsmStructSize_CXXFLAGS += $(VBOX_GCC_Wno-invalid-offsetof)
+endif
+tstAsmStructs_DEFS      = VBOX_IN_VMM IN_VMM_R3 IN_DIS $(VMM_COMMON_DEFS)
+ifdef VBOX_WITH_RAW_MODE
+tstAsmStructs_DEFS     += VBOX_WITH_RAW_MODE
+endif
+ifdef VBOX_WITH_R0_LOGGING
+tstAsmStructs_DEFS     += VBOX_WITH_R0_LOGGING
+endif
+ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+tstAsmStructs_DEFS     += VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+endif
+tstAsmStructs_INCS      = \
+	$(VBOX_PATH_VMM_SRC)/include \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)
+tstAsmStructs_SOURCES   = tstAsmStructs.cpp
+tstAsmStructs.cpp_DEPS  = $(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsHC.h
+
+ifdef VBOX_WITH_RAW_MODE
+tstAsmStructsRC_TEMPLATE= VBoxRcExe
+tstAsmStructsRC_DEFS    = VBOX_IN_VMM IN_VMM_RC IN_DIS IN_RT_RC VBOX_WITH_RAW_MODE $(VMM_COMMON_DEFS)
+ ifdef VBOX_WITH_R0_LOGGING
+tstAsmStructsRC_DEFS   += VBOX_WITH_R0_LOGGING
+ endif
+ ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+tstAsmStructsRC_DEFS   += VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+ endif
+tstAsmStructsRC_INCS    = \
+	$(VBOX_PATH_VMM_SRC)/include \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)
+tstAsmStructsRC_SOURCES = tstAsmStructs.cpp
+tstAsmStructs.cpp_DEPS += $(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsRC.h
+endif # VBOX_WITH_RAW_MODE
+
+
+#
+# Glboal config tool.
+#
+if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+tstGlobalConfigHardened_TEMPLATE = VBoxR3HardenedTstExe
+tstGlobalConfigHardened_NAME     = tstGlobalConfig
+tstGlobalConfigHardened_DEFS     = PROGRAM_NAME_STR=\"tstGlobalConfig\"
+tstGlobalConfigHardened_SOURCES  = ../../HostDrivers/Support/SUPR3HardenedMainTemplateTestcase.cpp
+tstGlobalConfig_TEMPLATE = VBoxR3HardenedTstDll
+else
+tstGlobalConfig_TEMPLATE = VBOXR3TSTEXE
+endif
+tstGlobalConfig_SOURCES  = tstGlobalConfig.cpp
+tstGlobalConfig_LIBS     = $(LIB_RUNTIME)
+
+#
+# Testcase for checking the repurposing of the IEM instruction code.
+#
+tstIEMCheckMc_TEMPLATE  = VBOXR3TSTEXE
+tstIEMCheckMc_SOURCES   = tstIEMCheckMc.cpp
+tstIEMCheckMc_DEFS      = $(VMM_COMMON_DEFS)
+tstIEMCheckMc_LIBS      = $(LIB_RUNTIME)
+ifeq ($(KBUILD_TARGET),win)
+tstIEMCheckMc_CXXFLAGS  = $(VBOX_C_CXX_FLAGS_NO_UNUSED_PARAMETERS) -wd4189 # local variable is initialized but not used.
+else
+tstIEMCheckMc_CXXFLAGS  = $(VBOX_C_CXX_FLAGS_NO_UNUSED_PARAMETERS) -Wno-unused-value -Wno-unused-variable
+endif
+
+#
+# VMM heap testcase.
+#
+if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+tstMMHyperHeapHardened_TEMPLATE = VBoxR3HardenedTstExe
+tstMMHyperHeapHardened_NAME     = tstMMHyperHeap
+tstMMHyperHeapHardened_DEFS     = PROGRAM_NAME_STR=\"tstMMHyperHeap\"
+tstMMHyperHeapHardened_SOURCES  = ../../HostDrivers/Support/SUPR3HardenedMainTemplateTestcase.cpp
+tstMMHyperHeap_TEMPLATE = VBoxR3HardenedTstDll
+else
+tstMMHyperHeap_TEMPLATE = VBOXR3TSTEXE
+endif
+tstMMHyperHeap_DEFS     = $(VMM_COMMON_DEFS)
+tstMMHyperHeap_SOURCES  = tstMMHyperHeap.cpp
+tstMMHyperHeap_LIBS     = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
+
+#
+# Saved state manager testcase.
+#
+tstSSM_TEMPLATE         = VBOXR3TSTEXE
+tstSSM_INCS             = $(VBOX_PATH_VMM_SRC)/include
+tstSSM_DEFS             = $(VMM_COMMON_DEFS)
+tstSSM_SOURCES          = tstSSM.cpp
+tstSSM_LIBS             = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
+
+#
+# VMM configuration manager tests.
+#
+if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+tstCFGMHardened_TEMPLATE = VBoxR3HardenedTstExe
+tstCFGMHardened_NAME     = tstCFGM
+tstCFGMHardened_DEFS     = PROGRAM_NAME_STR=\"tstCFGM\"
+tstCFGMHardened_SOURCES  = ../../HostDrivers/Support/SUPR3HardenedMainTemplateTestcase.cpp
+tstCFGM_TEMPLATE         = VBoxR3HardenedTstDll
+else
+tstCFGM_TEMPLATE         = VBOXR3TSTEXE
+endif
+tstCFGM_DEFS             = $(VMM_COMMON_DEFS)
+tstCFGM_SOURCES          = tstCFGM.cpp
+tstCFGM_LIBS             = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
+
+#
+# Comparing some compression algorithms considered for SSM usage.
+#
+tstCompressionBenchmark_TEMPLATE = VBOXR3TSTEXE
+tstCompressionBenchmark_SOURCES  = tstCompressionBenchmark.cpp
+
+#
+# Two testcases for checking the ring-3 "long jump" code.
+#
+tstVMMR0CallHost-1_TEMPLATE = VBOXR3TSTEXE
+tstVMMR0CallHost-1_DEFS = VMM_R0_NO_SWITCH_STACK
+tstVMMR0CallHost-1_INCS = $(VBOX_PATH_VMM_SRC)/include
+tstVMMR0CallHost-1_SOURCES = \
+	tstVMMR0CallHost-1.cpp
+tstVMMR0CallHost-1_SOURCES.amd64 = \
+	$(VBOX_PATH_VMM_SRC)/VMMR0/VMMR0JmpA-amd64.asm
+tstVMMR0CallHost-1_SOURCES.x86 = \
+	$(VBOX_PATH_VMM_SRC)/VMMR0/VMMR0JmpA-x86.asm
+
+tstVMMR0CallHost-2_EXTENDS = tstVMMR0CallHost-1
+tstVMMR0CallHost-2_DEFS = VMM_R0_SWITCH_STACK
+
+#
+# For testing the VM request queue code.
+#
+if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+tstVMREQHardened_TEMPLATE = VBOXR3HARDENEDEXE
+tstVMREQHardened_NAME     = tstVMREQ
+tstVMREQHardened_DEFS     = PROGRAM_NAME_STR=\"tstVMREQ\"
+tstVMREQHardened_SOURCES  = ../../HostDrivers/Support/SUPR3HardenedMainTemplate.cpp
+tstVMREQ_TEMPLATE         = VBOXR3
+else
+tstVMREQ_TEMPLATE         = VBOXR3EXE
+endif
+tstVMREQ_DEFS             = $(VMM_COMMON_DEFS)
+tstVMREQ_SOURCES          = tstVMREQ.cpp
+tstVMREQ_LIBS             = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
+
+#
+# Tool for reanimate things like OS/2 dumps.
+#
+if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+tstAnimateHardened_TEMPLATE = VBOXR3HARDENEDEXE
+tstAnimateHardened_NAME     = tstAnimate
+tstAnimateHardened_DEFS     = PROGRAM_NAME_STR=\"tstAnimate\"
+tstAnimateHardened_SOURCES  = ../../HostDrivers/Support/SUPR3HardenedMainTemplate.cpp
+tstAnimate_TEMPLATE         = VBOXR3
+else
+tstAnimate_TEMPLATE         = VBOXR3EXE
+endif
+tstAnimate_DEFS             = $(VMM_COMMON_DEFS)
+tstAnimate_SOURCES          = tstAnimate.cpp
+tstAnimate_LIBS             = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
+
+tstX86-1_TEMPLATE      = VBOXR3TSTEXE
+tstX86-1_SOURCES       = tstX86-1.cpp tstX86-1A.asm
+tstX86-1_LIBS          = $(LIB_RUNTIME)
+tstX86-1_LDFLAGS.linux = $(VBOX_GCC_no-pie)
+
+tstX86-FpuSaveRestore_TEMPLATE = VBOXR3TSTEXE
+tstX86-FpuSaveRestore_SOURCES  = tstX86-FpuSaveRestore.cpp tstX86-FpuSaveRestoreA.asm
+tstX86-FpuSaveRestore_LIBS     = $(LIB_RUNTIME)
+
+ifdef VBOX_WITH_RAW_MODE
+
+ #
+ # Raw-mode VMM testcase.
+ #
+ if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+tstVMMHardened_TEMPLATE = VBOXR3HARDENEDEXE
+tstVMMHardened_NAME     = tstVMM
+tstVMMHardened_DEFS     = PROGRAM_NAME_STR=\"tstVMM\"
+tstVMMHardened_SOURCES  = ../../HostDrivers/Support/SUPR3HardenedMainTemplate.cpp
+tstVMM_TEMPLATE         = VBOXR3
+ else
+tstVMM_TEMPLATE         = VBOXR3EXE
+ endif
+tstVMM_SOURCES          = tstVMM.cpp
+tstVMM_LIBS             = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
+
+#
+# HM VMM testcase.
+#
+tstVMM-HM_TEMPLATE     = VBOXR3EXE
+tstVMM-HM_SOURCES      = tstVMM-HM.cpp
+tstVMM-HM_LIBS         = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
+
+#
+# VMM host process fork test case (memory ++).
+#
+tstVMMFork_TEMPLATE    = VBOXR3EXE
+tstVMMFork_SOURCES     = tstVMMFork.cpp
+tstVMMFork_LIBS        = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
+
+#
+# Raw-mode micro benchmark.
+#
+ if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+tstMicroHardened_TEMPLATE = VBOXR3HARDENEDEXE
+tstMicroHardened_NAME     = tstMicro
+tstMicroHardened_DEFS     = PROGRAM_NAME_STR=\"tstMicro\"
+tstMicroHardened_SOURCES  = ../../HostDrivers/Support/SUPR3HardenedMainTemplate.cpp
+tstMicro_TEMPLATE         = VBOXR3
+ else
+tstMicro_TEMPLATE         = VBOXR3EXE
+ endif
+tstMicro_SOURCES          = tstMicro.cpp
+tstMicro_LIBS             = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
+tstMicro_DEFS             = $(if $(VBOX_WITH_RAW_MODE),VBOX_WITH_RAW_MODE,)
+
+tstMicroRC_TEMPLATE       = VBoxRc
+tstMicroRC_SOURCES        = tstMicroRC.cpp tstMicroRCA.asm
+tstMicroRC_DEFS           = $(if $(VBOX_WITH_RAW_MODE),VBOX_WITH_RAW_MODE,)
+tstMicroRC_INCS           = $(VBOX_PATH_VMM_SRC)/testcase
+ ifeq ($(VBOX_LDR_FMT32),pe)
+tstMicroRC_LDFLAGS        = -Entry:tstMicroRC
+ endif
+tstMicroRC_SYSSUFF        = .gc
+tstMicroRC_LIBS           = \
+	$(PATH_STAGE_LIB)/DisasmRC$(VBOX_SUFF_LIB) \
+	$(PATH_STAGE_LIB)/RuntimeRC$(VBOX_SUFF_LIB)
+ ifeq ($(filter-out pe lx,$(VBOX_LDR_FMT32)),)
+tstMicroRC_LIBS          += \
+	$(PATH_STAGE_LIB)/VMMRCBuiltin$(VBOX_SUFF_LIB) \
+	$(LIB_VMMRC)
+ endif
+tstMicroRC_SOURCES.win = tstMicroRC.def
+
+endif # VBOX_WITH_RAW_MODE
+
+
+if !defined(VBOX_ONLY_EXTPACKS_USE_IMPLIBS)
+#
+# Special NEM host testcase.
+#
+ if ("$(KBUILD_TARGET_ARCH).$(KBUILD_TARGET_ARCH)" == "darwin.amd64" && (defined(VBOX_WITH_NATIVE_NEM) || "$(USERNAME)" == "bird")) \
+ || ("$(KBUILD_TARGET_ARCH).$(KBUILD_TARGET_ARCH)" == "linux.amd64"  && (defined(VBOX_WITH_NATIVE_NEM) || "$(USERNAME)" == "bird")) \
+ || ("$(KBUILD_TARGET_ARCH).$(KBUILD_TARGET_ARCH)" == "win.amd64"    &&  defined(VBOX_WITH_NATIVE_NEM))
+PROGRAMS += NemRawBench-1
+NemRawBench-1_TEMPLATE = VBoxR3Static
+NemRawBench-1_SOURCES = NemRawBench-1.cpp
+NemRawBench-1_BLD_TYPE = release
+NemRawBench-1_INCS.win = \
+	$(KBUILD_DEVTOOLS)/win.x86/sdk/v10.0.17134.0/include/10.0.17134.0/um \
+	$(KBUILD_DEVTOOLS)/win.x86/sdk/v10.0.17134.0/include/10.0.17134.0/shared
+NemRawBench-1_CXXFLAGS.darwin = \
+	-F/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform//Developer/SDKs/MacOSX10.13.sdk/System/Library/Frameworks
+#NemRawBench-1_LDFLAGS.darwin =  \
+#	-F/System/Library/Frameworks \
+#	-framework Hypervisor
+NemRawBench-1_LDFLAGS.darwin = \
+	/System/Library/Frameworks/Hypervisor.framework/Hypervisor
+ endif
+endif
+
+
+ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+#
+# PDM asynchronous completation test.
+#
+ if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+tstPDMAsyncCompletionHardened_TEMPLATE = VBOXR3HARDENEDEXE
+tstPDMAsyncCompletionHardened_NAME     = tstPDMAsyncCompletion
+tstPDMAsyncCompletionHardened_DEFS     = PROGRAM_NAME_STR=\"tstPDMAsyncCompletion\"
+tstPDMAsyncCompletionHardened_SOURCES  = ../../HostDrivers/Support/SUPR3HardenedMainTemplate.cpp
+tstPDMAsyncCompletion_TEMPLATE         = VBOXR3
+ else
+tstPDMAsyncCompletion_TEMPLATE         = VBOXR3EXE
+ endif
+tstPDMAsyncCompletion_DEFS             = $(VMM_COMMON_DEFS)
+tstPDMAsyncCompletion_INCS             = $(VBOX_PATH_VMM_SRC)/include
+tstPDMAsyncCompletion_SOURCES          = tstPDMAsyncCompletion.cpp
+tstPDMAsyncCompletion_LIBS             = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
+
+#
+# PDM asynchronous completation stress test.
+#
+ if defined(VBOX_WITH_HARDENING) && "$(KBUILD_TARGET)" == "win"
+tstPDMAsyncCompletionStressHardened_TEMPLATE = VBOXR3HARDENEDEXE
+tstPDMAsyncCompletionStressHardened_NAME     = tstPDMAsyncCompletionStress
+tstPDMAsyncCompletionStressHardened_DEFS     = PROGRAM_NAME_STR=\"tstPDMAsyncCompletionStress\"
+tstPDMAsyncCompletionStressHardened_SOURCES  = ../../HostDrivers/Support/SUPR3HardenedMainTemplate.cpp
+tstPDMAsyncCompletionStress_TEMPLATE   = VBOXR3
+ else
+tstPDMAsyncCompletionStress_TEMPLATE   = VBOXR3EXE
+ endif
+tstPDMAsyncCompletionStress_DEFS       = $(VMM_COMMON_DEFS)
+tstPDMAsyncCompletionStress_INCS       = $(VBOX_PATH_VMM_SRC)/include
+tstPDMAsyncCompletionStress_SOURCES    = tstPDMAsyncCompletionStress.cpp
+tstPDMAsyncCompletionStress_LIBS       = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
+endif
+
+ifndef VBOX_ONLY_EXTPACKS
+PROGRAMS += tstSSM-2
+tstSSM-2_TEMPLATE       = VBOXR3TSTEXE
+tstSSM-2_DEFS           = IN_VMM_STATIC
+tstSSM-2_SOURCES        = tstSSM-2.cpp
+tstSSM-2_LIBS           = $(PATH_STAGE_LIB)/SSMStandalone$(VBOX_SUFF_LIB)
+endif
+
+#
+# Generate VM structure tests.
+#
+if !defined(VBOX_ONLY_EXTPACKS) \
+ && (   defined(VBOX_WITH_DTRACE_R3) \
+     || defined(VBOX_WITH_DTRACE_R0) \
+     || defined(VBOX_WITH_DTRACE_RC))
+tstVMStructDTrace_TEMPLATE = VBOXR3AUTOTST
+tstVMStructDTrace_DEFS     = VBOX_IN_VMM IN_VMM_R3 IN_DIS $(VMM_COMMON_DEFS)
+ ifdef VBOX_WITH_RAW_MODE
+tstVMStructDTrace_DEFS    += VBOX_WITH_RAW_MODE
+ endif
+tstVMStructDTrace_INCS     = \
+	$(VBOX_PATH_VMM_SRC)/include \
+	$(VBOX_PATH_VMM_SRC)/PATM \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)
+tstVMStructDTrace_SOURCES  = tstVMStructDTrace.cpp
+ ifdef VBOX_WITH_R0_LOGGING
+tstVMStructDTrace_DEFS    += VBOX_WITH_R0_LOGGING
+ endif
+ ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+tstVMStructDTrace_DEFS    += VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+ endif
+
+
+VMMLibDTraceStructTest_INST = $(VBOX_INST_DTRACE_TST)$(KBUILD_TARGET_ARCH)/
+VMMLibDTraceStructTest_SOURCES = \
+	$(tstVMStructDTrace_0_OUTDIR)/vbox-vm-struct-test.d
+VMMLibDTraceStructTest_CLEAN = \
+	$(tstVMStructDTrace_0_OUTDIR)/vbox-vm-struct-test.d
+
+$$(tstVMStructDTrace_0_OUTDIR)/vbox-vm-struct-test.d: \
+		$$(tstVMStructDTrace_1_STAGE_TARGET) | $$(dir $$@)
+	$(QUIET)$(RM) -f $@
+	$< > $@
+
+endif
+
+
+include $(FILE_KBUILD_SUB_FOOTER)
+
+
+#
+# Some handcrafted support targets for tstAsmStructs.
+#
+MY_ASA_ASM_STUFF = \
+		$(addprefix -D, \
+			$(DEFS) \
+			$(DEFS.$(KBUILD_TYPE)) \
+			$(DEFS.$(KBUILD_TARGET)) \
+			IN_RING3 $(ARCH_BITS_DEFS) \
+			$(DEFS.$(KBUILD_TARGET_ARCH)) \
+			$(DEFS.$(KBUILD_TARGET).$(KBUILD_TARGET_ARCH)) \
+			$(if $(VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI),VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI,) \
+                       $(VMM_COMMON_DEFS) \
+		) \
+		-f $(if $(eq $(KBUILD_TARGET),darwin),macho,elf) \
+		$(foreach inc,$(INCS) $(VBOX_PATH_VMM_SRC)/testcase $(VBOX_PATH_VMM_SRC)/include $(VBOX_VMM_TESTCASE_OUT_DIR)\
+			,-I$(inc)/)
+
+# 1a. make a header file which makes all the structures+members globals.
+$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsAsm.mac: \
+		$(VBOX_PATH_VMM_SRC)/testcase/tstAsmStructsAsm.asm \
+		$(VBOX_PATH_VMM_SRC)/testcase/tstAsmStructsAsm-lst.sed \
+		$(DEPTH)/include/iprt/asmdefs.mac \
+		$(DEPTH)/include/VBox/vmm/cpum.mac \
+		$(DEPTH)/include/VBox/vmm/vm.mac \
+		$(DEPTH)/include/VBox/sup.mac \
+		$(DEPTH)/include/iprt/x86.mac \
+		$(VBOX_PATH_VMM_SRC)/include/CPUMInternal.mac \
+		$(VBOX_PATH_VMM_SRC)/include/HMInternal.mac \
+		$(VBOX_PATH_VMM_SRC)/include/VMMInternal.mac \
+		$(VBOX_PATH_VMM_SRC)/testcase/Makefile.kmk \
+		$(PATH_ROOT)/Config.kmk $(LOCALCFG) $(AUTOCFG) \
+		| $$(dir $$@)
+	$(call MSG_GENERATE,tstVMStructSize,$@,$<)
+ifndef DONT_USE_YASM
+	$(QUIET)$(TOOL_YASM_AS) $(MY_ASA_ASM_STUFF) -o $@.o -l $@.lst $<
+	$(SED) -f $(VBOX_PATH_VMM_SRC)/testcase/tstAsmStructsAsm-lst.sed --output $@ $@.lst
+else
+	$(QUIET)$(TOOL_NASM_AS) -g $(MY_ASA_ASM_STUFF) -o $@.o -l $@.lst $<
+	$(VBOX_NM) $@.o | $(SED) \
+			-e '/[0-9a-fA-F][0-9a-fA-F]* [^a] /d' \
+			-e 's/[0-9a-fA-F][0-9a-fA-F]* a \([^ ]*\)/global \1/' \
+			> $@
+endif
+
+# 1b. make an elf/macho object containing the offsets.
+includedep $(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsAsm.o.dep
+$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsAsm.o: \
+		$(VBOX_PATH_VMM_SRC)/testcase/tstAsmStructsAsm.asm \
+		$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsAsm.mac \
+		$(DEPTH)/include/iprt/asmdefs.mac \
+		$(DEPTH)/include/VBox/vmm/cpum.mac \
+		$(DEPTH)/include/VBox/vmm/hm_vmx.mac \
+		$(DEPTH)/include/VBox/vmm/stam.mac \
+		$(DEPTH)/include/VBox/vmm/vm.mac \
+		$(DEPTH)/include/VBox/sup.mac \
+		$(DEPTH)/include/iprt/x86.mac \
+		$(VBOX_PATH_VMM_SRC)/include/CPUMInternal.mac \
+		$(VBOX_PATH_VMM_SRC)/include/HMInternal.mac \
+		$(VBOX_PATH_VMM_SRC)/include/VMMInternal.mac \
+		$(VBOX_PATH_VMM_SRC)/testcase/Makefile.kmk \
+		$$(if $$(eq $$(tstAsmStructsAsmDep_STUFF),$$(MY_ASA_ASM_STUFF)),,FORCE) \
+		| $$(dir $$@)
+	$(call MSG_COMPILE,tstAsmStructsasm,$<,$@,AS)
+ifndef DONT_USE_YASM
+	$(QUIET)$(TOOL_YASM_AS) $(MY_ASA_ASM_STUFF) -DDO_GLOBALS -o $@ $<
+else
+	$(QUIET)$(TOOL_NASM_AS) $(MY_ASA_ASM_STUFF) -DDO_GLOBALS -o $@ $<
+endif
+	%$(QUIET2)$(RM) -f -- $(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsAsm.o.dep
+	%$(QUIET2)$(APPEND) '$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsAsm.o.dep' 'tstAsmStructsAsmDep_STUFF=$(MY_ASA_ASM_STUFF)'
+
+# 2. use nm and sed to transform this into the header we want.
+$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsHC.h: $(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsAsm.o
+	$(call MSG_GENERATE,tstVMStructSize,$@,$<)
+	$(QUIET)$(RM) -f $@ $@.dump $@.tmp
+	$(QUIET)$(REDIRECT) -wo $@.dump -- $(VBOX_NM) $<
+	$(QUIET)$(SED) \
+		-e '/STAMPROFILEADV/d' \
+		\
+		-e '/^\(0x\)\{0,1\}00[0-9a-fA-F]* [aAnN] [^_.]*\./!d' \
+		-e 's/^\(0x\)\{0,1\}\(00[0-9a-fA-F]*\) [aAnN] \([^.]*\)\.\(.*$$\)/    CHECK_OFF(\3, 0x0\2, \4);/' \
+		--output $@.tmp $@.dump
+	$(QUIET)$(SED) \
+		-e '/VM_size$$/d' \
+		-e '/VMCPU_size$$/d' \
+		-e '/VMMCPU_size$$/d' \
+		-e '/SUPDRVTRACERUSRCTX32_size$$/d' \
+		-e '/HMCPU_size$$/d' \
+		\
+		-e '/^\(0x\)\{0,1\}00[0-9a-fA-F]* [aAnN] [^_.]*_size$$/!d' \
+		-e 's/^\(0x\)\{0,1\}\(00[0-9a-fA-F]*\) [aAnN] \([^_.]*\)_size/    CHECK_SIZE(\3, 0x0\2);/' \
+		--append $@.tmp $@.dump
+	$(QUIET)$(MV) -f $@.tmp $@
+	$(QUIET)$(RM) -f $@.dump
+
+# 3. run it.
+$(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructs.run: \
+		$$(tstAsmStructs_1_STAGE_TARGET) \
+		$(if-expr defined(VBOX_WITH_RAW_MODE),$$(tstAsmStructsRC_1_STAGE_TARGET),)
+	$(QUIET)$(RM) -f $@
+	$(tstAsmStructs_1_STAGE_TARGET)
+ifdef VBOX_WITH_RAW_MODE
+	$(tstAsmStructsRC_1_STAGE_TARGET)
+endif
+	$(QUIET)$(APPEND) "$@" "done"
+
+
+
+#
+# Run rule for tstVMStructSize.
+#
+
+ifdef VBOX_WITH_RAW_MODE
+# 1. Manually dump selected structures and members.
+$(VBOX_VMM_TESTCASE_OUT_DIR)/tstVMStructRC.h: $$(tstVMStructRC_1_STAGE_TARGET) | $$(dir $$@)
+	$(call MSG_GENERATE,tstVMStructSize,$@)
+	$(QUIET)$(REDIRECT) -wo $@ -- $<
+endif # VBOX_WITH_RAW_MODE
+
+# 2. run it.
+$(VBOX_VMM_TESTCASE_OUT_DIR)/tstVMStructSize.run: $$(tstVMStructSize_1_STAGE_TARGET) | $$(dir $$@)
+	$(QUIET)$(RM) -f $@
+	$<
+	$(QUIET)$(APPEND) "$@" "done"
+
+# alias for the two struct tests.
+run-struct-tests: $(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructs.run $(VBOX_VMM_TESTCASE_OUT_DIR)/tstVMStructSize.run
+
diff --git a/src/VBox/VMM/testcase/NemRawBench-1.cpp b/src/VBox/VMM/testcase/NemRawBench-1.cpp
new file mode 100644
index 00000000..944b372e
--- /dev/null
+++ b/src/VBox/VMM/testcase/NemRawBench-1.cpp
@@ -0,0 +1,1346 @@
+/* $Id: NemRawBench-1.cpp $ */
+/** @file
+ * NEM Benchmark.
+ */
+
+/*
+ * Copyright (C) 2018-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#ifdef RT_OS_WINDOWS
+# include <iprt/win/windows.h>
+# include <WinHvPlatform.h>
+# if !defined(_INTPTR) && defined(_M_AMD64) /* void pedantic stdint.h warnings  */
+#  define _INTPTR 2
+# endif
+
+#elif defined(RT_OS_LINUX)
+# include <linux/kvm.h>
+# include <errno.h>
+# include <sys/fcntl.h>
+# include <sys/ioctl.h>
+# include <sys/mman.h>
+# include <unistd.h>
+# include <time.h>
+
+#elif defined(RT_OS_DARWIN)
+# include <Availability.h>
+# if 1 /* header mix hack */
+#  undef __OSX_AVAILABLE_STARTING
+#  define __OSX_AVAILABLE_STARTING(_osx, _ios)
+# endif
+# include <Hypervisor/hv.h>
+# include <Hypervisor/hv_arch_x86.h>
+# include <Hypervisor/hv_arch_vmx.h>
+# include <Hypervisor/hv_vmx.h>
+# include <mach/mach_time.h>
+# include <mach/kern_return.h>
+# include <sys/time.h>
+# include <time.h>
+# include <sys/mman.h>
+# include <errno.h>
+
+#else
+# error "port me"
+#endif
+
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The base mapping address of the g_pbMem. */
+#define MY_MEM_BASE             0x1000
+/** No-op MMIO access address.   */
+#define MY_NOP_MMIO             0x0808
+/** The RIP which the testcode starts. */
+#define MY_TEST_RIP             0x2000
+
+/** The test termination port number. */
+#define MY_TERM_PORT            0x01
+/** The no-op test port number. */
+#define MY_NOP_PORT             0x7f
+
+#define MY_TEST_F_NOP_IO        (1U<<0)
+#define MY_TEST_F_CPUID         (1U<<1)
+#define MY_TEST_F_NOP_MMIO      (1U<<2)
+
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** Chunk of memory mapped at address 0x1000 (MY_MEM_BASE). */
+static unsigned char           *g_pbMem;
+/** Amount of RAM at address 0x1000 (MY_MEM_BASE). */
+static size_t                   g_cbMem;
+#ifdef RT_OS_WINDOWS
+static WHV_PARTITION_HANDLE     g_hPartition = NULL;
+
+/** @name APIs imported from WinHvPlatform.dll
+ * @{ */
+static decltype(WHvCreatePartition)                *g_pfnWHvCreatePartition;
+static decltype(WHvSetupPartition)                 *g_pfnWHvSetupPartition;
+static decltype(WHvGetPartitionProperty)           *g_pfnWHvGetPartitionProperty;
+static decltype(WHvSetPartitionProperty)           *g_pfnWHvSetPartitionProperty;
+static decltype(WHvMapGpaRange)                    *g_pfnWHvMapGpaRange;
+static decltype(WHvCreateVirtualProcessor)         *g_pfnWHvCreateVirtualProcessor;
+static decltype(WHvRunVirtualProcessor)            *g_pfnWHvRunVirtualProcessor;
+static decltype(WHvGetVirtualProcessorRegisters)   *g_pfnWHvGetVirtualProcessorRegisters;
+static decltype(WHvSetVirtualProcessorRegisters)   *g_pfnWHvSetVirtualProcessorRegisters;
+/** @} */
+static uint64_t (WINAPI                            *g_pfnRtlGetSystemTimePrecise)(void);
+
+#elif defined(RT_OS_LINUX)
+/** The VM handle.   */
+static int                      g_fdVm;
+/** The VCPU handle.   */
+static int                      g_fdVCpu;
+/** The kvm_run structure for the VCpu. */
+static struct kvm_run          *g_pVCpuRun;
+/** The size of the g_pVCpuRun mapping. */
+static ssize_t                  g_cbVCpuRun;
+
+#elif defined(RT_OS_DARWIN)
+/** The VCpu ID. */
+static hv_vcpuid_t              g_idVCpu;
+#endif
+
+
+static int error(const char *pszFormat, ...)
+{
+    fprintf(stderr, "error: ");
+    va_list va;
+    va_start(va, pszFormat);
+    vfprintf(stderr, pszFormat, va);
+    va_end(va);
+    return 1;
+}
+
+
+static uint64_t getNanoTS(void)
+{
+#ifdef RT_OS_WINDOWS
+    return g_pfnRtlGetSystemTimePrecise() * 100;
+
+#elif defined(RT_OS_LINUX)
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    return (uint64_t)ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec;
+
+#elif defined(RT_OS_DARWIN)
+    static struct mach_timebase_info    s_Info = { 0, 0 };
+    static double                       s_rdFactor = 0.0;
+    /* Lazy init. */
+    if (s_Info.denom != 0)
+    { /* likely */ }
+    else if (mach_timebase_info(&s_Info) == KERN_SUCCESS)
+        s_rdFactor = (double)s_Info.numer / (double)s_Info.denom;
+    else
+    {
+        error("mach_timebase_info(&Info) failed\n");
+        exit(1);
+    }
+    if (s_Info.denom == 1 && s_Info.numer == 1) /* special case: absolute time is in nanoseconds */
+        return mach_absolute_time();
+    return mach_absolute_time() * s_rdFactor;
+#else
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    return (uint64_t)tv.tv_sec * UINT64_C(1000000000)
+         + (tv.tv_usec * UINT32_C(1000));
+#endif
+}
+
+
+char *formatNum(uint64_t uNum, unsigned cchWidth, char *pszDst, size_t cbDst)
+{
+    char szTmp[64 + 22];
+#ifdef _MSC_VER
+    size_t cchTmp = _snprintf(szTmp, sizeof(szTmp) - 22, "%I64u", uNum);
+#else
+    size_t cchTmp = snprintf(szTmp, sizeof(szTmp) - 22, "%llu", (unsigned long long)uNum);
+#endif
+    size_t cSeps  = (cchTmp - 1) / 3;
+    size_t const cchTotal = cchTmp + cSeps;
+    if (cSeps)
+    {
+        szTmp[cchTotal] = '\0';
+        for (size_t iSrc = cchTmp, iDst = cchTotal; cSeps > 0; cSeps--)
+        {
+            szTmp[--iDst] = szTmp[--iSrc];
+            szTmp[--iDst] = szTmp[--iSrc];
+            szTmp[--iDst] = szTmp[--iSrc];
+            szTmp[--iDst] = ' ';
+        }
+    }
+
+    size_t offDst = 0;
+    while (cchWidth-- > cchTotal && offDst < cbDst)
+        pszDst[offDst++] = ' ';
+    size_t offSrc = 0;
+    while (offSrc < cchTotal && offDst < cbDst)
+        pszDst[offDst++] = szTmp[offSrc++];
+    pszDst[offDst] = '\0';
+    return pszDst;
+}
+
+
+int reportResult(const char *pszInstruction, uint32_t cInstructions, uint64_t nsElapsed, uint32_t cExits)
+{
+    uint64_t const cInstrPerSec = nsElapsed ? (uint64_t)cInstructions * 1000000000 / nsElapsed : 0;
+    char szTmp1[64], szTmp2[64], szTmp3[64];
+    printf("%s %7s instructions per second (%s exits in %s ns)\n",
+           formatNum(cInstrPerSec, 10, szTmp1, sizeof(szTmp1)), pszInstruction,
+           formatNum(cExits, 0, szTmp2, sizeof(szTmp2)),
+           formatNum(nsElapsed, 0, szTmp3, sizeof(szTmp3)));
+    return 0;
+}
+
+
+
+#ifdef RT_OS_WINDOWS
+
+/*
+ * Windows - Hyper-V Platform API.
+ */
+
+static int createVM(void)
+{
+    /*
+     * Resolve APIs.
+     */
+    HMODULE hmod = LoadLibraryW(L"WinHvPlatform.dll");
+    if (hmod == NULL)
+        return error("Error loading WinHvPlatform.dll: %u\n", GetLastError());
+    static struct { const char *pszFunction; FARPROC *ppfn; } const s_aImports[] =
+    {
+# define IMPORT_ENTRY(a_Name) { #a_Name, (FARPROC *)&g_pfn##a_Name }
+        IMPORT_ENTRY(WHvCreatePartition),
+        IMPORT_ENTRY(WHvSetupPartition),
+        IMPORT_ENTRY(WHvGetPartitionProperty),
+        IMPORT_ENTRY(WHvSetPartitionProperty),
+        IMPORT_ENTRY(WHvMapGpaRange),
+        IMPORT_ENTRY(WHvCreateVirtualProcessor),
+        IMPORT_ENTRY(WHvRunVirtualProcessor),
+        IMPORT_ENTRY(WHvGetVirtualProcessorRegisters),
+        IMPORT_ENTRY(WHvSetVirtualProcessorRegisters),
+# undef IMPORT_ENTRY
+    };
+    FARPROC pfn;
+    for (size_t i = 0; i < sizeof(s_aImports) / sizeof(s_aImports[0]); i++)
+    {
+        *s_aImports[i].ppfn = pfn = GetProcAddress(hmod, s_aImports[i].pszFunction);
+        if (!pfn)
+            return error("Error resolving WinHvPlatform.dll!%s: %u\n", s_aImports[i].pszFunction, GetLastError());
+    }
+# ifndef IN_SLICKEDIT
+#  define WHvCreatePartition                         g_pfnWHvCreatePartition
+#  define WHvSetupPartition                          g_pfnWHvSetupPartition
+#  define WHvGetPartitionProperty                    g_pfnWHvGetPartitionProperty
+#  define WHvSetPartitionProperty                    g_pfnWHvSetPartitionProperty
+#  define WHvMapGpaRange                             g_pfnWHvMapGpaRange
+#  define WHvCreateVirtualProcessor                  g_pfnWHvCreateVirtualProcessor
+#  define WHvRunVirtualProcessor                     g_pfnWHvRunVirtualProcessor
+#  define WHvGetVirtualProcessorRegisters            g_pfnWHvGetVirtualProcessorRegisters
+#  define WHvSetVirtualProcessorRegisters            g_pfnWHvSetVirtualProcessorRegisters
+# endif
+    /* Need a precise time function. */
+    *(FARPROC *)&g_pfnRtlGetSystemTimePrecise = pfn = GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "RtlGetSystemTimePrecise");
+    if (pfn == NULL)
+        return error("Error resolving ntdll.dll!RtlGetSystemTimePrecise: %u\n", GetLastError());
+
+    /*
+     * Create the partition with 1 CPU and the specfied amount of memory.
+     */
+    WHV_PARTITION_HANDLE hPartition;
+    HRESULT hrc = WHvCreatePartition(&hPartition);
+    if (!SUCCEEDED(hrc))
+        return error("WHvCreatePartition failed: %#x\n", hrc);
+    g_hPartition = hPartition;
+
+    WHV_PARTITION_PROPERTY Property;
+    memset(&Property, 0, sizeof(Property));
+    Property.ProcessorCount = 1;
+    hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeProcessorCount, &Property, sizeof(Property));
+    if (!SUCCEEDED(hrc))
+        return error("WHvSetPartitionProperty/WHvPartitionPropertyCodeProcessorCount failed: %#x\n", hrc);
+
+    memset(&Property, 0, sizeof(Property));
+    Property.ExtendedVmExits.X64CpuidExit  = 1;
+    Property.ExtendedVmExits.X64MsrExit    = 1;
+    hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeExtendedVmExits, &Property, sizeof(Property));
+    if (!SUCCEEDED(hrc))
+        return error("WHvSetPartitionProperty/WHvPartitionPropertyCodeExtendedVmExits failed: %#x\n", hrc);
+
+    hrc = WHvSetupPartition(hPartition);
+    if (!SUCCEEDED(hrc))
+        return error("WHvSetupPartition failed: %#x\n", hrc);
+
+    hrc = WHvCreateVirtualProcessor(hPartition, 0 /*idVCpu*/, 0 /*fFlags*/);
+    if (!SUCCEEDED(hrc))
+        return error("WHvCreateVirtualProcessor failed: %#x\n", hrc);
+
+    g_pbMem = (unsigned char *)VirtualAlloc(NULL, g_cbMem, MEM_COMMIT, PAGE_READWRITE);
+    if (!g_pbMem)
+        return error("VirtualAlloc failed: %u\n", GetLastError());
+    memset(g_pbMem, 0xcc, g_cbMem);
+
+    hrc = WHvMapGpaRange(hPartition, g_pbMem, MY_MEM_BASE /*GCPhys*/, g_cbMem,
+                         WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagWrite | WHvMapGpaRangeFlagExecute);
+    if (!SUCCEEDED(hrc))
+        return error("WHvMapGpaRange failed: %#x\n", hrc);
+
+    WHV_RUN_VP_EXIT_CONTEXT ExitInfo;
+    memset(&ExitInfo, 0, sizeof(ExitInfo));
+    WHvRunVirtualProcessor(g_hPartition, 0 /*idCpu*/, &ExitInfo, sizeof(ExitInfo));
+
+    return 0;
+}
+
+
+static int runtimeError(const char *pszFormat, ...)
+{
+    fprintf(stderr, "runtime error: ");
+    va_list va;
+    va_start(va, pszFormat);
+    vfprintf(stderr, pszFormat, va);
+    va_end(va);
+
+    static struct { const char *pszName; WHV_REGISTER_NAME enmName; unsigned uType; } const s_aRegs[] =
+    {
+        { "rip",    WHvX64RegisterRip, 64 },
+        { "cs",     WHvX64RegisterCs, 1 },
+        { "rflags", WHvX64RegisterRflags, 32 },
+        { "rax",    WHvX64RegisterRax, 64 },
+        { "rcx",    WHvX64RegisterRcx, 64 },
+        { "rdx",    WHvX64RegisterRdx, 64 },
+        { "rbx",    WHvX64RegisterRbx, 64 },
+        { "rsp",    WHvX64RegisterRsp, 64 },
+        { "ss",     WHvX64RegisterSs, 1 },
+        { "rbp",    WHvX64RegisterRbp, 64 },
+        { "rsi",    WHvX64RegisterRsi, 64 },
+        { "rdi",    WHvX64RegisterRdi, 64 },
+        { "ds",     WHvX64RegisterDs, 1 },
+        { "es",     WHvX64RegisterEs, 1 },
+        { "fs",     WHvX64RegisterFs, 1 },
+        { "gs",     WHvX64RegisterGs, 1 },
+        { "cr0",    WHvX64RegisterCr0, 64 },
+        { "cr2",    WHvX64RegisterCr2, 64 },
+        { "cr3",    WHvX64RegisterCr3, 64 },
+        { "cr4",    WHvX64RegisterCr4, 64 },
+    };
+    for (unsigned i = 0; i < sizeof(s_aRegs) / sizeof(s_aRegs[0]); i++)
+    {
+        WHV_REGISTER_VALUE Value;
+        WHV_REGISTER_NAME  enmName = s_aRegs[i].enmName;
+        HRESULT hrc = WHvGetVirtualProcessorRegisters(g_hPartition, 0 /*idCpu*/, &enmName, 1, &Value);
+        if (SUCCEEDED(hrc))
+        {
+            if (s_aRegs[i].uType == 32)
+                fprintf(stderr, "%8s=%08x\n", s_aRegs[i].pszName, Value.Reg32);
+            else if (s_aRegs[i].uType == 64)
+                fprintf(stderr, "%8s=%08x'%08x\n", s_aRegs[i].pszName, (unsigned)(Value.Reg64 >> 32), Value.Reg32);
+            else if (s_aRegs[i].uType == 1)
+                fprintf(stderr, "%8s=%04x  base=%08x'%08x  limit=%08x attr=%04x\n", s_aRegs[i].pszName,
+                        Value.Segment.Selector, (unsigned)(Value.Segment.Base >> 32), (unsigned)Value.Segment.Base,
+                        Value.Segment.Limit, Value.Segment.Attributes);
+        }
+        else
+            fprintf(stderr, "%8s=<WHvGetVirtualProcessorRegisters failed %#x>\n", s_aRegs[i].pszName, hrc);
+    }
+
+    return 1;
+}
+
+
+static int runRealModeTest(unsigned cInstructions, const char *pszInstruction, unsigned fTest,
+                           unsigned uEax, unsigned uEcx, unsigned uEdx, unsigned uEbx,
+                           unsigned uEsp, unsigned uEbp, unsigned uEsi, unsigned uEdi)
+{
+    (void)fTest;
+
+    /*
+     * Initialize the real mode context.
+     */
+# define ADD_REG64(a_enmName, a_uValue) do { \
+            aenmNames[iReg]      = (a_enmName); \
+            aValues[iReg].Reg128.High64 = 0; \
+            aValues[iReg].Reg64  = (a_uValue); \
+            iReg++; \
+        } while (0)
+# define ADD_SEG(a_enmName, a_Base, a_Limit, a_Sel, a_fCode) \
+        do { \
+            aenmNames[iReg]                  = a_enmName; \
+            aValues[iReg].Segment.Base       = (a_Base); \
+            aValues[iReg].Segment.Limit      = (a_Limit); \
+            aValues[iReg].Segment.Selector   = (a_Sel); \
+            aValues[iReg].Segment.Attributes = a_fCode ? 0x9b : 0x93; \
+            iReg++; \
+        } while (0)
+    WHV_REGISTER_NAME  aenmNames[80];
+    WHV_REGISTER_VALUE aValues[80];
+    unsigned iReg = 0;
+    ADD_REG64(WHvX64RegisterRax, uEax);
+    ADD_REG64(WHvX64RegisterRcx, uEcx);
+    ADD_REG64(WHvX64RegisterRdx, uEdx);
+    ADD_REG64(WHvX64RegisterRbx, uEbx);
+    ADD_REG64(WHvX64RegisterRsp, uEsp);
+    ADD_REG64(WHvX64RegisterRbp, uEbp);
+    ADD_REG64(WHvX64RegisterRsi, uEsi);
+    ADD_REG64(WHvX64RegisterRdi, uEdi);
+    ADD_REG64(WHvX64RegisterRip, MY_TEST_RIP);
+    ADD_REG64(WHvX64RegisterRflags, 2);
+    ADD_SEG(WHvX64RegisterEs, 0x00000, 0xffff, 0x0000, 0);
+    ADD_SEG(WHvX64RegisterCs, 0x00000, 0xffff, 0x0000, 1);
+    ADD_SEG(WHvX64RegisterSs, 0x00000, 0xffff, 0x0000, 0);
+    ADD_SEG(WHvX64RegisterDs, 0x00000, 0xffff, 0x0000, 0);
+    ADD_SEG(WHvX64RegisterFs, 0x00000, 0xffff, 0x0000, 0);
+    ADD_SEG(WHvX64RegisterGs, 0x00000, 0xffff, 0x0000, 0);
+    ADD_REG64(WHvX64RegisterCr0, 0x10010 /*WP+ET*/);
+    ADD_REG64(WHvX64RegisterCr2, 0);
+    ADD_REG64(WHvX64RegisterCr3, 0);
+    ADD_REG64(WHvX64RegisterCr4, 0);
+    HRESULT hrc = WHvSetVirtualProcessorRegisters(g_hPartition, 0 /*idCpu*/, aenmNames, iReg, aValues);
+    if (!SUCCEEDED(hrc))
+        return error("WHvSetVirtualProcessorRegisters failed (for %s): %#x\n", pszInstruction, hrc);
+# undef ADD_REG64
+# undef ADD_SEG
+
+    /*
+     * Run the test.
+     */
+    uint32_t cExits = 0;
+    uint64_t const nsStart = getNanoTS();
+    for (;;)
+    {
+        WHV_RUN_VP_EXIT_CONTEXT ExitInfo;
+        memset(&ExitInfo, 0, sizeof(ExitInfo));
+        hrc = WHvRunVirtualProcessor(g_hPartition, 0 /*idCpu*/, &ExitInfo, sizeof(ExitInfo));
+        if (SUCCEEDED(hrc))
+        {
+            cExits++;
+            if (ExitInfo.ExitReason == WHvRunVpExitReasonX64IoPortAccess)
+            {
+                if (ExitInfo.IoPortAccess.PortNumber == MY_NOP_PORT)
+                { /* likely: nop instruction */ }
+                else if (ExitInfo.IoPortAccess.PortNumber == MY_TERM_PORT)
+                    break;
+                else
+                    return runtimeError("Unexpected I/O port access (for %s): %#x\n", pszInstruction, ExitInfo.IoPortAccess.PortNumber);
+
+                /* Advance. */
+                if (ExitInfo.VpContext.InstructionLength)
+                {
+                    aenmNames[0] = WHvX64RegisterRip;
+                    aValues[0].Reg64 = ExitInfo.VpContext.Rip + ExitInfo.VpContext.InstructionLength;
+                    hrc = WHvSetVirtualProcessorRegisters(g_hPartition, 0 /*idCpu*/, aenmNames, 1, aValues);
+                    if (SUCCEEDED(hrc))
+                    { /* likely */ }
+                    else
+                        return runtimeError("Error advancing RIP (for %s): %#x\n", pszInstruction, hrc);
+                }
+                else
+                    return runtimeError("VpContext.InstructionLength is zero (for %s)\n", pszInstruction);
+            }
+            else if (ExitInfo.ExitReason == WHvRunVpExitReasonX64Cpuid)
+            {
+                /* Advance RIP and set default results. */
+                if (ExitInfo.VpContext.InstructionLength)
+                {
+                    aenmNames[0] = WHvX64RegisterRip;
+                    aValues[0].Reg64 = ExitInfo.VpContext.Rip + ExitInfo.VpContext.InstructionLength;
+                    aenmNames[1] = WHvX64RegisterRax;
+                    aValues[1].Reg64 = ExitInfo.CpuidAccess.DefaultResultRax;
+                    aenmNames[2] = WHvX64RegisterRcx;
+                    aValues[2].Reg64 = ExitInfo.CpuidAccess.DefaultResultRcx;
+                    aenmNames[3] = WHvX64RegisterRdx;
+                    aValues[3].Reg64 = ExitInfo.CpuidAccess.DefaultResultRdx;
+                    aenmNames[4] = WHvX64RegisterRbx;
+                    aValues[4].Reg64 = ExitInfo.CpuidAccess.DefaultResultRbx;
+                    hrc = WHvSetVirtualProcessorRegisters(g_hPartition, 0 /*idCpu*/, aenmNames, 5, aValues);
+                    if (SUCCEEDED(hrc))
+                    { /* likely */ }
+                    else
+                        return runtimeError("Error advancing RIP (for %s): %#x\n", pszInstruction, hrc);
+                }
+                else
+                    return runtimeError("VpContext.InstructionLength is zero (for %s)\n", pszInstruction);
+            }
+            else if (ExitInfo.ExitReason == WHvRunVpExitReasonMemoryAccess)
+            {
+                if (ExitInfo.MemoryAccess.Gpa == MY_NOP_MMIO)
+                { /* likely: nop address */ }
+                else
+                    return runtimeError("Unexpected memory access (for %s): %#x\n", pszInstruction, ExitInfo.MemoryAccess.Gpa);
+
+                /* Advance and set return register (assuming RAX and two byte instruction). */
+                aenmNames[0] = WHvX64RegisterRip;
+                if (ExitInfo.VpContext.InstructionLength)
+                    aValues[0].Reg64 = ExitInfo.VpContext.Rip + ExitInfo.VpContext.InstructionLength;
+                else
+                    aValues[0].Reg64 = ExitInfo.VpContext.Rip + 2;
+                aenmNames[1] = WHvX64RegisterRax;
+                aValues[1].Reg64 = 42;
+                hrc = WHvSetVirtualProcessorRegisters(g_hPartition, 0 /*idCpu*/, aenmNames, 2, aValues);
+                if (SUCCEEDED(hrc))
+                { /* likely */ }
+                else
+                    return runtimeError("Error advancing RIP (for %s): %#x\n", pszInstruction, hrc);
+            }
+            else
+                return runtimeError("Unexpected exit (for %s): %#x\n", pszInstruction, ExitInfo.ExitReason);
+        }
+        else
+            return runtimeError("WHvRunVirtualProcessor failed (for %s): %#x\n", pszInstruction, hrc);
+    }
+    uint64_t const nsElapsed = getNanoTS() - nsStart;
+    return reportResult(pszInstruction, cInstructions, nsElapsed, cExits);
+}
+
+
+
+#elif defined(RT_OS_LINUX)
+
+/*
+ * GNU/linux - KVM
+ */
+
+static int createVM(void)
+{
+    int fd = open("/dev/kvm", O_RDWR);
+    if (fd < 0)
+        return error("Error opening /dev/kvm: %d\n", errno);
+
+    g_fdVm = ioctl(fd, KVM_CREATE_VM, (uintptr_t)0);
+    if (g_fdVm < 0)
+        return error("KVM_CREATE_VM failed: %d\n", errno);
+
+    /* Create the VCpu. */
+    g_cbVCpuRun = ioctl(fd, KVM_GET_VCPU_MMAP_SIZE, (uintptr_t)0);
+    if (g_cbVCpuRun <= 0x1000 || (g_cbVCpuRun & 0xfff))
+        return error("Failed to get KVM_GET_VCPU_MMAP_SIZE: %#xz errno=%d\n", g_cbVCpuRun, errno);
+
+    g_fdVCpu = ioctl(g_fdVm, KVM_CREATE_VCPU, (uintptr_t)0);
+    if (g_fdVCpu < 0)
+        return error("KVM_CREATE_VCPU failed: %d\n", errno);
+
+    g_pVCpuRun = (struct kvm_run *)mmap(NULL, g_cbVCpuRun, PROT_READ | PROT_WRITE, MAP_PRIVATE, g_fdVCpu, 0);
+    if ((void *)g_pVCpuRun == MAP_FAILED)
+        return error("mmap kvm_run failed: %d\n", errno);
+
+    /* Memory. */
+    g_pbMem = (unsigned char *)mmap(NULL, g_cbMem, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if ((void *)g_pbMem == MAP_FAILED)
+        return error("mmap RAM failed: %d\n", errno);
+
+    struct kvm_userspace_memory_region MemReg;
+    MemReg.slot            = 0;
+    MemReg.flags           = 0;
+    MemReg.guest_phys_addr = MY_MEM_BASE;
+    MemReg.memory_size     = g_cbMem;
+    MemReg.userspace_addr  = (uintptr_t)g_pbMem;
+    int rc = ioctl(g_fdVm, KVM_SET_USER_MEMORY_REGION, &MemReg);
+    if (rc != 0)
+        return error("KVM_SET_USER_MEMORY_REGION failed: %d (%d)\n", errno, rc);
+
+    close(fd);
+    return 0;
+}
+
+
+static void printSReg(const char *pszName, struct kvm_segment const *pSReg)
+{
+    fprintf(stderr, "     %5s=%04x  base=%016llx  limit=%08x type=%#x p=%d dpl=%d db=%d s=%d l=%d g=%d avl=%d un=%d\n",
+            pszName, pSReg->selector, pSReg->base, pSReg->limit, pSReg->type, pSReg->present, pSReg->dpl,
+            pSReg->db, pSReg->s, pSReg->l, pSReg->g, pSReg->avl, pSReg->unusable);
+}
+
+
+static int runtimeError(const char *pszFormat, ...)
+{
+    fprintf(stderr, "runtime error: ");
+    va_list va;
+    va_start(va, pszFormat);
+    vfprintf(stderr, pszFormat, va);
+    va_end(va);
+
+    fprintf(stderr, "                  exit_reason=%#010x\n", g_pVCpuRun->exit_reason);
+    fprintf(stderr, "ready_for_interrupt_injection=%#x\n", g_pVCpuRun->ready_for_interrupt_injection);
+    fprintf(stderr, "                      if_flag=%#x\n", g_pVCpuRun->if_flag);
+    fprintf(stderr, "                        flags=%#x\n", g_pVCpuRun->flags);
+    fprintf(stderr, "               kvm_valid_regs=%#018llx\n", g_pVCpuRun->kvm_valid_regs);
+    fprintf(stderr, "               kvm_dirty_regs=%#018llx\n", g_pVCpuRun->kvm_dirty_regs);
+
+    struct kvm_regs Regs;
+    memset(&Regs, 0, sizeof(Regs));
+    struct kvm_sregs SRegs;
+    memset(&SRegs, 0, sizeof(SRegs));
+    if (   ioctl(g_fdVCpu, KVM_GET_REGS, &Regs) != -1
+        && ioctl(g_fdVCpu, KVM_GET_SREGS, &SRegs) != -1)
+    {
+        fprintf(stderr, "       rip=%016llx\n", Regs.rip);
+        printSReg("cs", &SRegs.cs);
+        fprintf(stderr, "    rflags=%08llx\n", Regs.rflags);
+        fprintf(stderr, "       rax=%016llx\n", Regs.rax);
+        fprintf(stderr, "       rbx=%016llx\n", Regs.rcx);
+        fprintf(stderr, "       rdx=%016llx\n", Regs.rdx);
+        fprintf(stderr, "       rcx=%016llx\n", Regs.rbx);
+        fprintf(stderr, "       rsp=%016llx\n", Regs.rsp);
+        fprintf(stderr, "       rbp=%016llx\n", Regs.rbp);
+        fprintf(stderr, "       rsi=%016llx\n", Regs.rsi);
+        fprintf(stderr, "       rdi=%016llx\n", Regs.rdi);
+        printSReg("ss", &SRegs.ss);
+        printSReg("ds", &SRegs.ds);
+        printSReg("es", &SRegs.es);
+        printSReg("fs", &SRegs.fs);
+        printSReg("gs", &SRegs.gs);
+        printSReg("tr", &SRegs.tr);
+        printSReg("ldtr", &SRegs.ldt);
+
+        uint64_t const offMem = Regs.rip + SRegs.cs.base - MY_MEM_BASE;
+        if (offMem < g_cbMem - 10)
+            fprintf(stderr, "  bytes at PC (%#zx): %02x %02x %02x %02x %02x %02x %02x %02x\n", (size_t)(offMem + MY_MEM_BASE),
+                    g_pbMem[offMem    ], g_pbMem[offMem + 1], g_pbMem[offMem + 2], g_pbMem[offMem + 3],
+                    g_pbMem[offMem + 4], g_pbMem[offMem + 5], g_pbMem[offMem + 6], g_pbMem[offMem + 7]);
+    }
+
+    return 1;
+}
+
+static int runRealModeTest(unsigned cInstructions, const char *pszInstruction, unsigned fTest,
+                           unsigned uEax, unsigned uEcx, unsigned uEdx, unsigned uEbx,
+                           unsigned uEsp, unsigned uEbp, unsigned uEsi, unsigned uEdi)
+{
+    (void)fTest;
+
+    /*
+     * Setup real mode context.
+     */
+#define SET_SEG(a_SReg, a_Base, a_Limit, a_Sel, a_fCode) \
+        do { \
+            a_SReg.base     = (a_Base); \
+            a_SReg.limit    = (a_Limit); \
+            a_SReg.selector = (a_Sel); \
+            a_SReg.type     = (a_fCode) ? 10 : 3; \
+            a_SReg.present  = 1; \
+            a_SReg.dpl      = 0; \
+            a_SReg.db       = 0; \
+            a_SReg.s        = 1; \
+            a_SReg.l        = 0; \
+            a_SReg.g        = 0; \
+            a_SReg.avl      = 0; \
+            a_SReg.unusable = 0; \
+            a_SReg.padding  = 0; \
+        } while (0)
+    struct kvm_regs Regs;
+    memset(&Regs, 0, sizeof(Regs));
+    Regs.rax = uEax;
+    Regs.rcx = uEcx;
+    Regs.rdx = uEdx;
+    Regs.rbx = uEbx;
+    Regs.rsp = uEsp;
+    Regs.rbp = uEbp;
+    Regs.rsi = uEsi;
+    Regs.rdi = uEdi;
+    Regs.rip = MY_TEST_RIP;
+    Regs.rflags = 2;
+    int rc = ioctl(g_fdVCpu, KVM_SET_REGS, &Regs);
+    if (rc != 0)
+        return error("KVM_SET_REGS failed: %d (rc=%d)\n", errno, rc);
+
+    struct kvm_sregs SRegs;
+    memset(&SRegs, 0, sizeof(SRegs));
+    rc = ioctl(g_fdVCpu, KVM_GET_SREGS, &SRegs);
+    if (rc != 0)
+        return error("KVM_GET_SREGS failed: %d (rc=%d)\n", errno, rc);
+    SET_SEG(SRegs.es, 0x00000, 0xffff, 0x0000, 0);
+    SET_SEG(SRegs.cs, 0x00000, 0xffff, 0x0000, 1);
+    SET_SEG(SRegs.ss, 0x00000, 0xffff, 0x0000, 0);
+    SET_SEG(SRegs.ds, 0x00000, 0xffff, 0x0000, 0);
+    SET_SEG(SRegs.fs, 0x00000, 0xffff, 0x0000, 0);
+    SET_SEG(SRegs.gs, 0x00000, 0xffff, 0x0000, 0);
+    //SRegs.cr0 = 0x10010 /*WP+ET*/;
+    SRegs.cr2 = 0;
+    //SRegs.cr3 = 0;
+    //SRegs.cr4 = 0;
+    rc = ioctl(g_fdVCpu, KVM_SET_SREGS, &SRegs);
+    if (rc != 0)
+        return error("KVM_SET_SREGS failed: %d (rc=%d)\n", errno, rc);
+
+    /*
+     * Run the test.
+     */
+    uint32_t cExits = 0;
+    uint64_t const nsStart = getNanoTS();
+    for (;;)
+    {
+        rc = ioctl(g_fdVCpu, KVM_RUN, (uintptr_t)0);
+        if (rc == 0)
+        {
+            cExits++;
+            if (g_pVCpuRun->exit_reason == KVM_EXIT_IO)
+            {
+                if (g_pVCpuRun->io.port == MY_NOP_PORT)
+                { /* likely: nop instruction */ }
+                else if (g_pVCpuRun->io.port == MY_TERM_PORT)
+                    break;
+                else
+                    return runtimeError("Unexpected I/O port access (for %s): %#x\n", pszInstruction, g_pVCpuRun->io.port);
+            }
+            else if (g_pVCpuRun->exit_reason == KVM_EXIT_MMIO)
+            {
+                if (g_pVCpuRun->mmio.phys_addr == MY_NOP_MMIO)
+                { /* likely: nop address */ }
+                else
+                    return runtimeError("Unexpected memory access (for %s): %#llx\n", pszInstruction, g_pVCpuRun->mmio.phys_addr);
+            }
+            else
+                return runtimeError("Unexpected exit (for %s): %d\n", pszInstruction, g_pVCpuRun->exit_reason);
+        }
+        else
+            return runtimeError("KVM_RUN failed (for %s): %#x (ret %d)\n", pszInstruction, errno, rc);
+    }
+    uint64_t const nsElapsed = getNanoTS() - nsStart;
+    return reportResult(pszInstruction, cInstructions, nsElapsed, cExits);
+}
+
+
+#elif defined(RT_OS_DARWIN)
+
+/*
+ * Mac OS X - Hypervisor API.
+ */
+
+static int createVM(void)
+{
+    /* VM and VCpu */
+    hv_return_t rcHv = hv_vm_create(HV_VM_DEFAULT);
+    if (rcHv != HV_SUCCESS)
+        return error("hv_vm_create failed: %#x\n", rcHv);
+
+    g_idVCpu = -1;
+    rcHv = hv_vcpu_create(&g_idVCpu, HV_VCPU_DEFAULT);
+    if (rcHv != HV_SUCCESS)
+        return error("hv_vcpu_create failed: %#x\n", rcHv);
+
+    /* Memory. */
+    g_pbMem = (unsigned char *)mmap(NULL, g_cbMem, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON, -1, 0);
+    if ((void *)g_pbMem == MAP_FAILED)
+        return error("mmap RAM failed: %d\n", errno);
+    memset(g_pbMem, 0xf4, g_cbMem);
+
+    rcHv = hv_vm_map(g_pbMem, MY_MEM_BASE, g_cbMem, HV_MEMORY_READ | HV_MEMORY_WRITE | HV_MEMORY_EXEC);
+    if (rcHv != HV_SUCCESS)
+        return error("hv_vm_map failed: %#x\n", rcHv);
+
+    rcHv = hv_vm_protect(0x2000, 0x1000, HV_MEMORY_READ | HV_MEMORY_WRITE | HV_MEMORY_EXEC);
+    if (rcHv != HV_SUCCESS)
+        return error("hv_vm_protect failed: %#x\n", rcHv);
+    return 0;
+}
+
+
+static int runtimeError(const char *pszFormat, ...)
+{
+    fprintf(stderr, "runtime error: ");
+    va_list va;
+    va_start(va, pszFormat);
+    vfprintf(stderr, pszFormat, va);
+    va_end(va);
+
+    static struct { const char *pszName; uint32_t uField; uint32_t uFmt : 31; uint32_t fIsReg : 1; } const s_aFields[] =
+    {
+        { "VMCS_RO_EXIT_REASON",        VMCS_RO_EXIT_REASON,        64, 0 },
+        { "VMCS_RO_EXIT_QUALIFIC",      VMCS_RO_EXIT_QUALIFIC,      64, 0 },
+        { "VMCS_RO_INSTR_ERROR",        VMCS_RO_INSTR_ERROR,        64, 0 },
+        { "VMCS_RO_VMEXIT_IRQ_INFO",    VMCS_RO_VMEXIT_IRQ_INFO,    64, 0 },
+        { "VMCS_RO_VMEXIT_IRQ_ERROR",   VMCS_RO_VMEXIT_IRQ_ERROR,   64, 0 },
+        { "VMCS_RO_VMEXIT_INSTR_LEN",   VMCS_RO_VMEXIT_INSTR_LEN,   64, 0 },
+        { "VMCS_RO_VMX_INSTR_INFO",     VMCS_RO_VMX_INSTR_INFO,     64, 0 },
+        { "VMCS_RO_GUEST_LIN_ADDR",     VMCS_RO_GUEST_LIN_ADDR,     64, 0 },
+        { "VMCS_GUEST_PHYSICAL_ADDRESS",VMCS_GUEST_PHYSICAL_ADDRESS,64, 0 },
+        { "VMCS_RO_IO_RCX",             VMCS_RO_IO_RCX,             64, 0 },
+        { "VMCS_RO_IO_RSI",             VMCS_RO_IO_RSI,             64, 0 },
+        { "VMCS_RO_IO_RDI",             VMCS_RO_IO_RDI,             64, 0 },
+        { "VMCS_RO_IO_RIP",             VMCS_RO_IO_RIP,             64, 0 },
+        { "rip",                        HV_X86_RIP,                 64, 1 },
+        { "rip (vmcs)",                 VMCS_GUEST_RIP,             64, 0 },
+        { "cs",                         HV_X86_CS,                  16, 1 },
+        { "cs (vmcs)",                  VMCS_GUEST_CS,              16, 0 },
+        { "cs.base",                    VMCS_GUEST_CS_BASE,         64, 0 },
+        { "cs.limit",                   VMCS_GUEST_CS_LIMIT,        32, 0 },
+        { "cs.attr",                    VMCS_GUEST_CS_AR,           32, 0 },
+        { "rflags",                     HV_X86_RFLAGS,              32, 1 },
+        { "rax",                        HV_X86_RAX,                 64, 1 },
+        { "rcx",                        HV_X86_RCX,                 64, 1 },
+        { "rdx",                        HV_X86_RDX,                 64, 1 },
+        { "rbx",                        HV_X86_RBX,                 64, 1 },
+        { "rsp",                        HV_X86_RSP,                 64, 1 },
+        { "rsp (vmcs)",                 VMCS_GUEST_RSP,             64, 0 },
+        { "ss",                         HV_X86_SS,                  16, 1 },
+        { "ss (vmcs)",                  VMCS_GUEST_SS,              16, 0 },
+        { "ss.base",                    VMCS_GUEST_SS_BASE,         64, 0 },
+        { "ss.limit",                   VMCS_GUEST_SS_LIMIT,        32, 0 },
+        { "ss.attr",                    VMCS_GUEST_SS_AR,           32, 0 },
+        { "rbp",                        HV_X86_RBP,                 64, 1 },
+        { "rsi",                        HV_X86_RSI,                 64, 1 },
+        { "rdi",                        HV_X86_RDI,                 64, 1 },
+        { "ds",                         HV_X86_DS,                  16, 1 },
+        { "ds (vmcs)",                  VMCS_GUEST_DS,              16, 0 },
+        { "ds.base",                    VMCS_GUEST_DS_BASE,         64, 0 },
+        { "ds.limit",                   VMCS_GUEST_DS_LIMIT,        32, 0 },
+        { "ds.attr",                    VMCS_GUEST_DS_AR,           32, 0 },
+        { "es",                         HV_X86_ES,                  16, 1 },
+        { "es (vmcs)",                  VMCS_GUEST_ES,              16, 0 },
+        { "es.base",                    VMCS_GUEST_ES_BASE,         64, 0 },
+        { "es.limit",                   VMCS_GUEST_ES_LIMIT,        32, 0 },
+        { "es.attr",                    VMCS_GUEST_ES_AR,           32, 0 },
+        { "fs",                         HV_X86_FS,                  16, 1 },
+        { "fs (vmcs)",                  VMCS_GUEST_FS,              16, 0 },
+        { "fs.base",                    VMCS_GUEST_FS_BASE,         64, 0 },
+        { "fs.limit",                   VMCS_GUEST_FS_LIMIT,        32, 0 },
+        { "fs.attr",                    VMCS_GUEST_FS_AR,           32, 0 },
+        { "gs",                         HV_X86_GS,                  16, 1 },
+        { "gs (vmcs)",                  VMCS_GUEST_GS,              16, 0 },
+        { "gs.base",                    VMCS_GUEST_GS_BASE,         64, 0 },
+        { "gs.limit",                   VMCS_GUEST_GS_LIMIT,        32, 0 },
+        { "gs.attr",                    VMCS_GUEST_GS_AR,           32, 0 },
+        { "cr0",                        HV_X86_CR0,                 64, 1 },
+        { "cr0 (vmcs)",                 VMCS_GUEST_CR0,             64, 0 },
+        { "cr2",                        HV_X86_CR2,                 64, 1 },
+        { "cr3",                        HV_X86_CR3,                 64, 1 },
+        { "cr3 (vmcs)",                 VMCS_GUEST_CR3,             64, 0 },
+        { "cr4",                        HV_X86_CR4,                 64, 1 },
+        { "cr4 (vmcs)",                 VMCS_GUEST_CR4,             64, 0 },
+        { "idtr.base",                  VMCS_GUEST_IDTR_BASE,       64, 0 },
+        { "idtr.limit",                 VMCS_GUEST_IDTR_LIMIT,      32, 0 },
+        { "gdtr.base",                  VMCS_GUEST_GDTR_BASE,       64, 0 },
+        { "gdtr.limit",                 VMCS_GUEST_GDTR_LIMIT,      32, 0 },
+
+        { "VMCS_CTRL_PIN_BASED",        VMCS_CTRL_PIN_BASED,        64, 0 },
+        { "VMCS_CTRL_CPU_BASED",        VMCS_CTRL_CPU_BASED,        64, 0 },
+        { "VMCS_CTRL_CPU_BASED2",       VMCS_CTRL_CPU_BASED2,       64, 0 },
+        { "VMCS_CTRL_VMENTRY_CONTROLS", VMCS_CTRL_VMENTRY_CONTROLS, 64, 0 },
+        { "VMCS_CTRL_VMEXIT_CONTROLS",  VMCS_CTRL_VMEXIT_CONTROLS,  64, 0 },
+        { "VMCS_CTRL_EXC_BITMAP",       VMCS_CTRL_EXC_BITMAP,       64, 0 },
+        { "VMCS_CTRL_CR0_MASK",         VMCS_CTRL_CR0_MASK,         64, 0 },
+        { "VMCS_CTRL_CR0_SHADOW",       VMCS_CTRL_CR0_SHADOW,       64, 0 },
+        { "VMCS_CTRL_CR4_MASK",         VMCS_CTRL_CR4_MASK,         64, 0 },
+        { "VMCS_CTRL_CR4_SHADOW",       VMCS_CTRL_CR4_SHADOW,       64, 0 },
+    };
+    for (unsigned i = 0; i < sizeof(s_aFields) / sizeof(s_aFields[0]); i++)
+    {
+        uint64_t uValue = UINT64_MAX;
+        hv_return_t rcHv;
+        if (s_aFields[i].fIsReg)
+            rcHv = hv_vcpu_read_register(g_idVCpu, (hv_x86_reg_t)s_aFields[i].uField, &uValue);
+        else
+            rcHv = hv_vmx_vcpu_read_vmcs(g_idVCpu, s_aFields[i].uField, &uValue);
+        if (rcHv == HV_SUCCESS)
+        {
+            if (s_aFields[i].uFmt == 16)
+                fprintf(stderr, "%28s=%04llx\n", s_aFields[i].pszName, uValue);
+            else if (s_aFields[i].uFmt == 32)
+                fprintf(stderr, "%28s=%08llx\n", s_aFields[i].pszName, uValue);
+            else
+                fprintf(stderr, "%28s=%08x'%08x\n", s_aFields[i].pszName, (uint32_t)(uValue >> 32), (uint32_t)uValue);
+        }
+        else
+            fprintf(stderr, "%28s=<%s failed %#x>\n", s_aFields[i].pszName,
+                    s_aFields[i].fIsReg ? "hv_vcpu_read_register" : "hv_vmx_vcpu_read_vmcs", rcHv);
+    }
+    return 1;
+}
+
+
+static int runRealModeTest(unsigned cInstructions, const char *pszInstruction, unsigned fTest,
+                           unsigned uEax, unsigned uEcx, unsigned uEdx, unsigned uEbx,
+                           unsigned uEsp, unsigned uEbp, unsigned uEsi, unsigned uEdi)
+{
+    /*
+     * Setup real mode context.
+     */
+#define WRITE_REG_RET(a_enmReg, a_uValue) \
+        do { \
+            hv_return_t rcHvX = hv_vcpu_write_register(g_idVCpu, a_enmReg, a_uValue); \
+            if (rcHvX == HV_SUCCESS) { /* likely */ } \
+            else return error("hv_vcpu_write_register(%#x, %s, %#llx) -> %#x\n", g_idVCpu, #a_enmReg, (uint64_t)(a_uValue), rcHvX); \
+        } while (0)
+#define READ_REG_RET(a_enmReg, a_puValue) \
+        do { \
+            hv_return_t rcHvX = hv_vcpu_read_register(g_idVCpu, a_enmReg, a_puValue); \
+            if (rcHvX == HV_SUCCESS) { /* likely */ } \
+            else return error("hv_vcpu_read_register(%#x, %s,) -> %#x\n", g_idVCpu, #a_enmReg, rcHvX); \
+        } while (0)
+#define WRITE_VMCS_RET(a_enmField, a_uValue) \
+        do { \
+            hv_return_t rcHvX = hv_vmx_vcpu_write_vmcs(g_idVCpu, a_enmField, a_uValue); \
+            if (rcHvX == HV_SUCCESS) { /* likely */ } \
+            else return error("hv_vmx_vcpu_write_vmcs(%#x, %s, %#llx) -> %#x\n", g_idVCpu, #a_enmField, (uint64_t)(a_uValue), rcHvX); \
+        } while (0)
+#define READ_VMCS_RET(a_enmField, a_puValue) \
+        do { \
+            hv_return_t rcHvX = hv_vmx_vcpu_read_vmcs(g_idVCpu, a_enmField, a_puValue); \
+            if (rcHvX == HV_SUCCESS) { /* likely */ } \
+            else return error("hv_vmx_vcpu_read_vmcs(%#x, %s,) -> %#x\n", g_idVCpu, #a_enmField, rcHvX); \
+        } while (0)
+#define READ_CAP_RET(a_enmCap, a_puValue) \
+        do { \
+            hv_return_t rcHvX = hv_vmx_read_capability(a_enmCap, a_puValue); \
+            if (rcHvX == HV_SUCCESS) { /* likely */ } \
+            else return error("hv_vmx_read_capability(%s) -> %#x\n", #a_enmCap); \
+        } while (0)
+#define CAP_2_CTRL(a_uCap, a_fWanted) ( ((a_fWanted) | (uint32_t)(a_uCap)) & (uint32_t)((a_uCap) >> 32) )
+#if 1
+    uint64_t uCap;
+    READ_CAP_RET(HV_VMX_CAP_PINBASED, &uCap);
+    WRITE_VMCS_RET(VMCS_CTRL_PIN_BASED, CAP_2_CTRL(uCap, PIN_BASED_INTR | PIN_BASED_NMI | PIN_BASED_VIRTUAL_NMI));
+    READ_CAP_RET(HV_VMX_CAP_PROCBASED, &uCap);
+    WRITE_VMCS_RET(VMCS_CTRL_CPU_BASED, CAP_2_CTRL(uCap, CPU_BASED_HLT
+                                                       | CPU_BASED_INVLPG
+                                                       | CPU_BASED_MWAIT
+                                                       | CPU_BASED_RDPMC
+                                                       | CPU_BASED_RDTSC
+                                                       | CPU_BASED_CR3_LOAD
+                                                       | CPU_BASED_CR3_STORE
+                                                       | CPU_BASED_CR8_LOAD
+                                                       | CPU_BASED_CR8_STORE
+                                                       | CPU_BASED_MOV_DR
+                                                       | CPU_BASED_UNCOND_IO
+                                                       | CPU_BASED_MONITOR
+                                                       | CPU_BASED_PAUSE
+                                                       ));
+    READ_CAP_RET(HV_VMX_CAP_PROCBASED2, &uCap);
+    WRITE_VMCS_RET(VMCS_CTRL_CPU_BASED2, CAP_2_CTRL(uCap, 0));
+    READ_CAP_RET(HV_VMX_CAP_ENTRY, &uCap);
+    WRITE_VMCS_RET(VMCS_CTRL_VMENTRY_CONTROLS, CAP_2_CTRL(uCap, 0));
+#endif
+    WRITE_VMCS_RET(VMCS_CTRL_EXC_BITMAP, UINT32_MAX);
+    WRITE_VMCS_RET(VMCS_CTRL_CR0_MASK,   0x60000000);
+    WRITE_VMCS_RET(VMCS_CTRL_CR0_SHADOW, 0x00000000);
+    WRITE_VMCS_RET(VMCS_CTRL_CR4_MASK,   0x00000000);
+    WRITE_VMCS_RET(VMCS_CTRL_CR4_SHADOW, 0x00000000);
+
+    WRITE_REG_RET(HV_X86_RAX, uEax);
+    WRITE_REG_RET(HV_X86_RCX, uEcx);
+    WRITE_REG_RET(HV_X86_RDX, uEdx);
+    WRITE_REG_RET(HV_X86_RBX, uEbx);
+    WRITE_REG_RET(HV_X86_RSP, uEsp);
+    WRITE_REG_RET(HV_X86_RBP, uEbp);
+    WRITE_REG_RET(HV_X86_RSI, uEsi);
+    WRITE_REG_RET(HV_X86_RDI, uEdi);
+    WRITE_REG_RET(HV_X86_RIP, MY_TEST_RIP);
+    WRITE_REG_RET(HV_X86_RFLAGS, 2);
+    WRITE_REG_RET(HV_X86_ES, 0x0000);
+    WRITE_VMCS_RET(VMCS_GUEST_ES_BASE, 0x0000000);
+    WRITE_VMCS_RET(VMCS_GUEST_ES_LIMIT,   0xffff);
+    WRITE_VMCS_RET(VMCS_GUEST_ES_AR,        0x93);
+    WRITE_REG_RET(HV_X86_CS, 0x0000);
+    WRITE_VMCS_RET(VMCS_GUEST_CS_BASE, 0x0000000);
+    WRITE_VMCS_RET(VMCS_GUEST_CS_LIMIT,   0xffff);
+    WRITE_VMCS_RET(VMCS_GUEST_CS_AR,        0x9b);
+    WRITE_REG_RET(HV_X86_SS, 0x0000);
+    WRITE_VMCS_RET(VMCS_GUEST_SS_BASE, 0x0000000);
+    WRITE_VMCS_RET(VMCS_GUEST_SS_LIMIT,   0xffff);
+    WRITE_VMCS_RET(VMCS_GUEST_SS_AR,        0x93);
+    WRITE_REG_RET(HV_X86_DS, 0x0000);
+    WRITE_VMCS_RET(VMCS_GUEST_DS_BASE, 0x0000000);
+    WRITE_VMCS_RET(VMCS_GUEST_DS_LIMIT,  0xffff);
+    WRITE_VMCS_RET(VMCS_GUEST_DS_AR,        0x93);
+    WRITE_REG_RET(HV_X86_FS, 0x0000);
+    WRITE_VMCS_RET(VMCS_GUEST_FS_BASE, 0x0000000);
+    WRITE_VMCS_RET(VMCS_GUEST_FS_LIMIT,   0xffff);
+    WRITE_VMCS_RET(VMCS_GUEST_FS_AR,        0x93);
+    WRITE_REG_RET(HV_X86_GS, 0x0000);
+    WRITE_VMCS_RET(VMCS_GUEST_GS_BASE, 0x0000000);
+    WRITE_VMCS_RET(VMCS_GUEST_GS_LIMIT,   0xffff);
+    WRITE_VMCS_RET(VMCS_GUEST_GS_AR,        0x93);
+    //WRITE_REG_RET(HV_X86_CR0, 0x10030 /*WP+NE+ET*/);
+    WRITE_VMCS_RET(VMCS_GUEST_CR0,        0x10030 /*WP+NE+ET*/);
+    //WRITE_REG_RET(HV_X86_CR2, 0);
+    //WRITE_REG_RET(HV_X86_CR3, 0);
+    WRITE_VMCS_RET(VMCS_GUEST_CR3,             0);
+    //WRITE_REG_RET(HV_X86_CR4, 0x2000);
+    WRITE_VMCS_RET(VMCS_GUEST_CR4,        0x2000);
+    WRITE_VMCS_RET(VMCS_GUEST_LDTR,          0x0000);
+    WRITE_VMCS_RET(VMCS_GUEST_LDTR_BASE, 0x00000000);
+    WRITE_VMCS_RET(VMCS_GUEST_LDTR_LIMIT,    0x0000);
+    WRITE_VMCS_RET(VMCS_GUEST_LDTR_AR,      0x10000);
+    WRITE_VMCS_RET(VMCS_GUEST_TR,            0x0000);
+    WRITE_VMCS_RET(VMCS_GUEST_TR_BASE,   0x00000000);
+    WRITE_VMCS_RET(VMCS_GUEST_TR_LIMIT,      0x0000);
+    WRITE_VMCS_RET(VMCS_GUEST_TR_AR,        0x00083);
+    hv_vcpu_flush(g_idVCpu);
+    hv_vcpu_invalidate_tlb(g_idVCpu);
+
+    /*
+     * Run the test.
+     */
+    uint32_t cExits = 0;
+    uint64_t const nsStart = getNanoTS();
+    for (;;)
+    {
+        hv_return_t rcHv = hv_vcpu_run(g_idVCpu);
+        if (rcHv == HV_SUCCESS)
+        {
+            cExits++;
+            uint64_t uExitReason = UINT64_MAX;
+            READ_VMCS_RET(VMCS_RO_EXIT_REASON, &uExitReason);
+            if (!(uExitReason & UINT64_C(0x80000000)))
+            {
+                if (uExitReason == VMX_REASON_IO)
+                {
+                    uint64_t uIoQual = UINT64_MAX;
+                    READ_VMCS_RET(VMCS_RO_EXIT_QUALIFIC, &uIoQual);
+                    if ((uint16_t)(uIoQual >> 16) == MY_NOP_PORT && (fTest & MY_TEST_F_NOP_IO))
+                    { /* likely: nop instruction */ }
+                    else if ((uint16_t)(uIoQual >> 16) == MY_TERM_PORT)
+                        break;
+                    else
+                        return runtimeError("Unexpected I/O port access (for %s): %#x\n", pszInstruction, (uint16_t)(uIoQual >> 16));
+
+                    /* Advance RIP. */
+                    uint64_t cbInstr = UINT64_MAX;
+                    READ_VMCS_RET(VMCS_RO_VMEXIT_INSTR_LEN, &cbInstr);
+                    if (cbInstr < 1 || cbInstr > 15)
+                        return runtimeError("Bad instr len: %#llx\n", cbInstr);
+                    uint64_t uRip = UINT64_MAX;
+                    READ_REG_RET(HV_X86_RIP, &uRip);
+                    WRITE_REG_RET(HV_X86_RIP, uRip + cbInstr);
+                }
+                else if (uExitReason == VMX_REASON_CPUID && (fTest & MY_TEST_F_CPUID))
+                {
+                    /* Set registers and advance RIP. */
+                    WRITE_REG_RET(HV_X86_RAX, 0x42424242);
+                    WRITE_REG_RET(HV_X86_RCX, 0x04242424);
+                    WRITE_REG_RET(HV_X86_RDX, 0x00424242);
+                    WRITE_REG_RET(HV_X86_RBX, 0x00024242);
+
+                    uint64_t cbInstr = UINT64_MAX;
+                    READ_VMCS_RET(VMCS_RO_VMEXIT_INSTR_LEN, &cbInstr);
+                    if (cbInstr < 1 || cbInstr > 15)
+                        return runtimeError("Bad instr len: %#llx\n", cbInstr);
+                    uint64_t uRip = UINT64_MAX;
+                    READ_REG_RET(HV_X86_RIP, &uRip);
+                    WRITE_REG_RET(HV_X86_RIP, uRip + cbInstr);
+                }
+                else if (uExitReason == VMX_REASON_EPT_VIOLATION)
+                {
+                    uint64_t uEptQual = UINT64_MAX;
+                    READ_VMCS_RET(VMCS_RO_EXIT_QUALIFIC, &uEptQual);
+                    uint64_t GCPhys = UINT64_MAX;
+                    READ_VMCS_RET(VMCS_GUEST_PHYSICAL_ADDRESS, &GCPhys);
+                    if (GCPhys == MY_NOP_MMIO && (fTest & MY_TEST_F_NOP_MMIO))
+                    { /* likely */ }
+                    else if (GCPhys == MY_TEST_RIP)
+                        continue; /* dunno why we get this, but restarting it works */
+                    else
+                        return runtimeError("Unexpected EPT viotaion at %#llx\n", GCPhys);
+
+                    /* Set RAX and advance RIP. */
+                    WRITE_REG_RET(HV_X86_RAX, 42);
+
+                    uint64_t cbInstr = UINT64_MAX;
+                    READ_VMCS_RET(VMCS_RO_VMEXIT_INSTR_LEN, &cbInstr);
+                    if (cbInstr < 1 || cbInstr > 15)
+                        return runtimeError("Bad instr len: %#llx\n", cbInstr);
+                    uint64_t uRip = UINT64_MAX;
+                    READ_REG_RET(HV_X86_RIP, &uRip);
+                    WRITE_REG_RET(HV_X86_RIP, uRip + cbInstr);
+                }
+                else if (uExitReason == VMX_REASON_IRQ)
+                { /* ignore */ }
+                else
+                    return runtimeError("Unexpected exit reason: %#x\n", uExitReason);
+            }
+            else
+                return runtimeError("VM entry failure: %#x\n", uExitReason);
+        }
+        else
+            return runtimeError("hv_vcpu_run failed (for %s): %#x\n", pszInstruction, rcHv);
+    }
+    uint64_t const nsElapsed = getNanoTS() - nsStart;
+    return reportResult(pszInstruction, cInstructions, nsElapsed, cExits);
+}
+
+#else
+# error "port me"
+#endif
+
+void dumpCode(uint8_t const *pb, uint8_t *pbEnd)
+{
+    printf("testing:");
+    for (; pb != pbEnd; pb++)
+        printf(" %02x", *pb);
+    printf("\n");
+}
+
+
+int ioportTest(unsigned cFactor)
+{
+    /*
+     * Produce realmode code
+     */
+    unsigned char *pb = &g_pbMem[MY_TEST_RIP - MY_MEM_BASE];
+    unsigned char * const pbStart = pb;
+    /* OUT DX, AL - 10 times */
+    for (unsigned i = 0; i < 10; i++)
+        *pb++ = 0xee;
+    /* DEC ECX */
+    *pb++ = 0x66;
+    *pb++ = 0x48 + 1;
+    /* JNZ MY_TEST_RIP */
+    *pb++ = 0x75;
+    *pb   = (signed char)(pbStart - pb - 1);
+    pb++;
+    /* OUT 1, AL -  Temination port call. */
+    *pb++ = 0xe6;
+    *pb++ = MY_TERM_PORT;
+    /* JMP to previous instruction */
+    *pb++ = 0xeb;
+    *pb++ = 0xfc;
+    dumpCode(pbStart, pb);
+
+    return runRealModeTest(100000 * cFactor, "OUT", MY_TEST_F_NOP_IO,
+                           42 /*eax*/, 10000 * cFactor /*ecx*/, MY_NOP_PORT /*edx*/, 0 /*ebx*/,
+                           0 /*esp*/, 0 /*ebp*/, 0 /*esi*/, 0 /*uEdi*/);
+}
+
+
+int cpuidTest(unsigned cFactor)
+{
+    /*
+     * Produce realmode code
+     */
+    unsigned char *pb = &g_pbMem[MY_TEST_RIP - MY_MEM_BASE];
+    unsigned char * const pbStart = pb;
+    for (unsigned i = 0; i < 10; i++)
+    {
+        /* XOR EAX,EAX */
+        *pb++ = 0x66;
+        *pb++ = 0x33;
+        *pb++ = 0xc0;
+
+        /* CPUID */
+        *pb++ = 0x0f;
+        *pb++ = 0xa2;
+    }
+    /* DEC ESI */
+    *pb++ = 0x66;
+    *pb++ = 0x48 + 6;
+    /* JNZ MY_TEST_RIP */
+    *pb++ = 0x75;
+    *pb   = (signed char)(pbStart - pb - 1);
+    pb++;
+    /* OUT 1, AL -  Temination port call. */
+    *pb++ = 0xe6;
+    *pb++ = MY_TERM_PORT;
+    /* JMP to previous instruction */
+    *pb++ = 0xeb;
+    *pb++ = 0xfc;
+    dumpCode(pbStart, pb);
+
+    return runRealModeTest(100000 * cFactor, "CPUID", MY_TEST_F_CPUID,
+                           0 /*eax*/, 0 /*ecx*/, 0 /*edx*/, 0 /*ebx*/,
+                           0 /*esp*/, 0 /*ebp*/, 10000 * cFactor /*esi*/, 0 /*uEdi*/);
+}
+
+
+int mmioTest(unsigned cFactor)
+{
+    /*
+     * Produce realmode code accessing MY_MMIO_NOP address assuming it's low.
+     */
+    unsigned char *pb = &g_pbMem[MY_TEST_RIP - MY_MEM_BASE];
+    unsigned char * const pbStart = pb;
+    for (unsigned i = 0; i < 10; i++)
+    {
+        /* MOV AL,DS:[BX] */
+        *pb++ = 0x8a;
+        *pb++ = 0x07;
+    }
+    /* DEC ESI */
+    *pb++ = 0x66;
+    *pb++ = 0x48 + 6;
+    /* JNZ MY_TEST_RIP */
+    *pb++ = 0x75;
+    *pb   = (signed char)(pbStart - pb - 1);
+    pb++;
+    /* OUT 1, AL -  Temination port call. */
+    *pb++ = 0xe6;
+    *pb++ = MY_TERM_PORT;
+    /* JMP to previous instruction */
+    *pb++ = 0xeb;
+    *pb++ = 0xfc;
+    dumpCode(pbStart, pb);
+
+    return runRealModeTest(100000 * cFactor, "MMIO/r1", MY_TEST_F_NOP_MMIO,
+                           0 /*eax*/, 0 /*ecx*/, 0 /*edx*/, MY_NOP_MMIO /*ebx*/,
+                           0 /*esp*/, 0 /*ebp*/, 10000 * cFactor /*esi*/, 0 /*uEdi*/);
+}
+
+
+
+int main(int argc, char **argv)
+{
+    /*
+     * Do some parameter parsing.
+     */
+#ifdef RT_OS_WINDOWS
+    unsigned const  cFactorDefault = 4;
+#elif RT_OS_DARWIN
+    unsigned const  cFactorDefault = 32;
+#else
+    unsigned const  cFactorDefault = 24;
+#endif
+    unsigned        cFactor = cFactorDefault;
+    for (int i = 1; i < argc; i++)
+    {
+        const char *pszArg = argv[i];
+        if (   strcmp(pszArg, "--help") == 0
+            || strcmp(pszArg, "/help") == 0
+            || strcmp(pszArg, "-h") == 0
+            || strcmp(pszArg, "-?") == 0
+            || strcmp(pszArg, "/?") == 0)
+        {
+            printf("Does some benchmarking of the native NEM engine.\n"
+                   "\n"
+                   "Usage: NemRawBench-1 --factor <factor>\n"
+                   "\n"
+                   "Options\n"
+                   "  --factor <factor>\n"
+                   "        Iteration count factor.  Default is %u.\n"
+                   "        Lower it if execution is slow, increase if quick.\n",
+                   cFactorDefault);
+            return 0;
+        }
+        if (strcmp(pszArg, "--factor") == 0)
+        {
+            i++;
+            if (i < argc)
+                cFactor = atoi(argv[i]);
+            else
+            {
+                fprintf(stderr, "syntax error: Option %s is takes a value!\n", pszArg);
+                return 2;
+            }
+        }
+        else
+        {
+            fprintf(stderr, "syntax error: Unknown option: %s\n", pszArg);
+            return 2;
+        }
+    }
+
+    /*
+     * Create the VM
+     */
+    g_cbMem = 128*1024 - MY_MEM_BASE;
+    int rcExit = createVM();
+    if (rcExit == 0)
+    {
+        printf("tstNemBench-1: Successfully created test VM...\n");
+
+        /*
+         * Do the benchmarking.
+         */
+        ioportTest(cFactor);
+        cpuidTest(cFactor);
+        mmioTest(cFactor);
+
+        printf("tstNemBench-1: done\n");
+    }
+    return rcExit;
+}
+
+/*
+ * Results:
+ *
+ * - Darwin/xnu 10.12.6/16.7.0; 3.1GHz Intel Core i7-7920HQ (Kaby Lake):
+ *    925 845     OUT instructions per second (3 200 307 exits in 3 456 301 621 ns)
+ *    949 278   CPUID instructions per second (3 200 222 exits in 3 370 980 173 ns)
+ *    871 499 MMIO/r1 instructions per second (3 200 223 exits in 3 671 834 221 ns)
+ *
+ * - Linux 4.15.0 / ubuntu 18.04.1 Desktop LiveCD; 3.1GHz Intel Core i7-7920HQ (Kaby Lake):
+ *    829 775     OUT instructions per second (3 200 001 exits in 3 856 466 567 ns)
+ *  2 212 038   CPUID instructions per second (1 exits in 1 446 629 591 ns)             [1]
+ *    477 962 MMIO/r1 instructions per second (3 200 001 exits in 6 695 090 600 ns)
+ *
+ * - Linux 4.15.0 / ubuntu 18.04.1 Desktop LiveCD; 3.4GHz Core i5-3570 (Ivy Bridge):
+ *    717 216     OUT instructions per second (2 400 001 exits in 3 346 271 640 ns)
+ *  1 675 983   CPUID instructions per second (1 exits in 1 431 995 135 ns)             [1]
+ *    402 621 MMIO/r1 instructions per second (2 400 001 exits in 5 960 930 854 ns)
+ *
+ * - Linux 4.18.0-1-amd64 (debian); 3.4GHz AMD Threadripper 1950X:
+ *    455 727     OUT instructions per second (2 400 001 exits in 5 266 300 471 ns)
+ *  1 745 014   CPUID instructions per second (1 exits in 1 375 346 658 ns)             [1]
+ *    351 767 MMIO/r1 instructions per second (2 400 001 exits in 6 822 684 544 ns)
+ *
+ * - Windows 1803 updated as per 2018-10-01; 3.4GHz Core i5-3570 (Ivy Bridge):
+ *     67 778     OUT instructions per second (400 001 exits in 5 901 560 700 ns)
+ *     66 113   CPUID instructions per second (400 001 exits in 6 050 208 000 ns)
+ *     62 939 MMIO/r1 instructions per second (400 001 exits in 6 355 302 900 ns)
+ *
+ * - Windows 1803 updated as per 2018-09-28; 3.4GHz AMD Threadripper 1950X:
+ *     34 485     OUT instructions per second (400 001 exits in 11 598 918 200 ns)
+ *     34 043   CPUID instructions per second (400 001 exits in 11 749 753 200 ns)
+ *     33 124 MMIO/r1 instructions per second (400 001 exits in 12 075 617 000 ns)
+ *
+ * - Windows build 17763; 3.4GHz AMD Threadripper 1950X:
+ *     65 633     OUT instructions per second (400 001 exits in 6 094 409 100 ns)
+ *     65 245   CPUID instructions per second (400 001 exits in 6 130 720 600 ns)
+ *     61 642 MMIO/r1 instructions per second (400 001 exits in 6 489 013 700 ns)
+ *
+ *
+ * [1] CPUID causes no return to ring-3 with KVM.
+ *
+ *
+ * For reference we can compare with similar tests in bs2-test1 running VirtualBox:
+ *
+ * - Linux 4.18.0-1-amd64 (debian); 3.4GHz AMD Threadripper 1950X; trunk/r125404:
+ *      real mode, 32-bit OUT            :        1 338 471 ins/sec
+ *      real mode, 32-bit OUT-to-ring-3  :          500 337 ins/sec
+ *      real mode, CPUID                 :        1 566 343 ins/sec
+ *      real mode, 32-bit write          :          870 671 ins/sec
+ *      real mode, 32-bit write-to-ring-3:          391 014 ins/sec
+ *
+ * - Darwin/xnu 10.12.6/16.7.0; 3.1GHz Intel Core i7-7920HQ (Kaby Lake); trunk/r125404:
+ *      real mode, 32-bit OUT            :          790 117 ins/sec
+ *      real mode, 32-bit OUT-to-ring-3  :          157 205 ins/sec
+ *      real mode, CPUID                 :        1 001 087 ins/sec
+ *      real mode, 32-bit write          :          651 257 ins/sec
+ *      real mode, 32-bit write-to-ring-3:          157 773 ins/sec
+ *
+ * - Linux 4.15.0 / ubuntu 18.04.1 Desktop LiveCD; 3.1GHz Intel Core i7-7920HQ (Kaby Lake); trunk/r125450:
+ *      real mode, 32-bit OUT            :        1 229 245 ins/sec
+ *      real mode, 32-bit OUT-to-ring-3  :          284 848 ins/sec
+ *      real mode, CPUID                 :        1 429 760 ins/sec
+ *      real mode, 32-bit write          :          820 679 ins/sec
+ *      real mode, 32-bit write-to-ring-3:          245 159 ins/sec
+ *
+ * - Windows 1803 updated as per 2018-10-01; 3.4GHz Core i5-3570 (Ivy Bridge); trunk/r15442:
+ *      real mode, 32-bit OUT            :          961 939 ins/sec
+ *      real mode, 32-bit OUT-to-ring-3  :          189 458 ins/sec
+ *      real mode, CPUID                 :        1 060 582 ins/sec
+ *      real mode, 32-bit write          :          637 967 ins/sec
+ *      real mode, 32-bit write-to-ring-3:          148 573 ins/sec
+ *
+ */
diff --git a/src/VBox/VMM/testcase/dev.tar.gz b/src/VBox/VMM/testcase/dev.tar.gz
new file mode 100644
index 00000000..95d3a358
Binary files /dev/null and b/src/VBox/VMM/testcase/dev.tar.gz differ
diff --git a/src/VBox/VMM/testcase/mkdsk.sh b/src/VBox/VMM/testcase/mkdsk.sh
new file mode 100755
index 00000000..f6115cc0
--- /dev/null
+++ b/src/VBox/VMM/testcase/mkdsk.sh
@@ -0,0 +1,76 @@
+#!/bin/sh
+## @file
+# Obsolete?
+#
+
+#
+# Copyright (C) 2006-2020 Oracle Corporation
+#
+# This file is part of VirtualBox Open Source Edition (OSE), as
+# available from http://www.virtualbox.org. This file is free software;
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License (GPL) as published by the Free Software
+# Foundation, in version 2 as it comes in the "COPYING" file of the
+# VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+# hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+#
+
+if [ "x$3" == "x" ]; then
+
+    echo "syntax error"
+    echo "syntax: $0 imagename <size-in-KBs> <init prog> [tar files]"
+    echo ""
+    echo "Simples qemu boot image is archived by only specifying an statically"
+    echo "linked init program and using the dev.tar.gz file to create devices."
+    echo "The boot linux in qemu specifying the image as -hda. Use the -kernel"
+    echo "option to specify a bzImage kernel image to use, and specify"
+    echo "-append root=/dev/hda so the kernel will mount /dev/hda and look"
+    echo "for /sbin/init there."
+    echo ""
+    echo "Example:"
+    echo "  sh ./mkdsk.sh foo.img 2048 ~/VBox/Tree/out/linux/debug/bin/tstProg1 dev.tar.gz"
+    echo "  qemu -hda foo.img -m 32  -kernel ~/qemutest/linux-test/bzImage-2.4.21 -append root=/dev/hda"
+    exit 1
+fi
+
+image=$1
+size=$2
+init=$3
+
+sizebytes=`expr $size '*' 1024`
+cyls=`expr 8225280 / $sizebytes`
+echo $cyls
+
+echo "* Creating $image of $size kb...."
+rm -f $image
+dd if=/dev/zero of=$image count=$size bs=1024 || exit 1
+
+echo "* Formatting with ext2..."
+/sbin/mkfs.ext2 $image || exit 1
+
+echo "* Mounting temporarily at ./tmpmnt..."
+mkdir -p tmpmnt
+sudo mount $image ./tmpmnt -t ext2 -o loop=/dev/loop7 || exit 1
+
+# init
+echo "* Copying $init to sbin/init..."
+mkdir tmpmnt/sbin
+sudo cp $init tmpmnt/sbin/init
+sudo chmod 755 tmpmnt/sbin/init
+
+shift
+shift
+shift
+while [ "x$1" != "x" ];
+do
+    echo "* Untarring $1 to disk..."
+    sudo tar -xzv -C tmpmnt -f $1
+    shift
+done
+
+echo "* Unmounting    tmpmnt..."
+sudo umount tmpmnt
+rmdir tmpmnt
+echo "* Done! (Perhaps even successfully so...)"
+echo "  'root=/dev/hda' remember :-)"
+exit 0
diff --git a/src/VBox/VMM/testcase/tstAnimate.cpp b/src/VBox/VMM/testcase/tstAnimate.cpp
new file mode 100644
index 00000000..32a1ef44
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstAnimate.cpp
@@ -0,0 +1,943 @@
+/* $Id: tstAnimate.cpp $ */
+/** @file
+ * VBox Animation Testcase / Tool.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/ssm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/err.h>
+#include <VBox/vmm/pdmifs.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/initterm.h>
+#include <iprt/semaphore.h>
+#include <iprt/string.h>
+#include <iprt/stream.h>
+#include <iprt/file.h>
+#include <iprt/thread.h>
+#include <iprt/ctype.h>
+#include <iprt/uuid.h>
+
+#include <signal.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+static volatile bool g_fSignaled = false;
+
+
+static void SigInterrupt(int iSignal)
+{
+    NOREF(iSignal);
+    signal(SIGINT, SigInterrupt);
+    g_fSignaled = true;
+    RTPrintf("caught SIGINT\n");
+}
+
+typedef DECLCALLBACK(int) FNSETGUESTGPR(PVM, uint32_t);
+typedef FNSETGUESTGPR *PFNSETGUESTGPR;
+static int scriptGPReg(PVM pVM, char *pszVar, char *pszValue, void *pvUser)
+{
+    NOREF(pszVar);
+    uint32_t u32;
+    int rc = RTStrToUInt32Ex(pszValue, NULL, 16, &u32);
+    if (RT_FAILURE(rc))
+        return rc;
+    return ((PFNSETGUESTGPR)(uintptr_t)pvUser)(pVM, u32);
+}
+
+typedef DECLCALLBACK(int) FNSETGUESTSEL(PVM, uint16_t);
+typedef FNSETGUESTSEL *PFNSETGUESTSEL;
+static int scriptSelReg(PVM pVM, char *pszVar, char *pszValue, void *pvUser)
+{
+    NOREF(pszVar);
+    uint16_t u16;
+    int rc = RTStrToUInt16Ex(pszValue, NULL, 16, &u16);
+    if (RT_FAILURE(rc))
+        return rc;
+    return ((PFNSETGUESTSEL)(uintptr_t)pvUser)(pVM, u16);
+}
+
+typedef DECLCALLBACK(int) FNSETGUESTSYS(PVM, uint32_t);
+typedef FNSETGUESTSYS *PFNSETGUESTSYS;
+static int scriptSysReg(PVM pVM, char *pszVar, char *pszValue, void *pvUser)
+{
+    NOREF(pszVar);
+    uint32_t u32;
+    int rc = RTStrToUInt32Ex(pszValue, NULL, 16, &u32);
+    if (RT_FAILURE(rc))
+        return rc;
+    return ((PFNSETGUESTSYS)(uintptr_t)pvUser)(pVM, u32);
+}
+
+
+typedef DECLCALLBACK(int) FNSETGUESTDTR(PVM, uint32_t, uint16_t);
+typedef FNSETGUESTDTR *PFNSETGUESTDTR;
+static int scriptDtrReg(PVM pVM, char *pszVar, char *pszValue, void *pvUser)
+{
+    NOREF(pszVar);
+    char *pszPart2 = strchr(pszValue, ':');
+    if (!pszPart2)
+        return -1;
+    *pszPart2++ = '\0';
+    pszPart2 = RTStrStripL(pszPart2);
+    pszValue = RTStrStripR(pszValue);
+
+    uint32_t u32;
+    int rc = RTStrToUInt32Ex(pszValue, NULL, 16, &u32);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    uint16_t u16;
+    rc = RTStrToUInt16Ex(pszPart2, NULL, 16, &u16);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    return ((PFNSETGUESTDTR)(uintptr_t)pvUser)(pVM, u32, u16);
+}
+
+
+
+
+/* variables - putting in global scope to avoid MSC warning C4640.  */
+static struct
+{
+    const char *pszVar;
+    int (*pfnHandler)(PVM pVM, char *pszVar, char *pszValue, void *pvUser);
+    PFNRT pvUser;
+} g_aVars[] =
+{
+    { "eax", scriptGPReg,  (PFNRT)CPUMSetGuestEAX },
+    { "ebx", scriptGPReg,  (PFNRT)CPUMSetGuestEBX },
+    { "ecx", scriptGPReg,  (PFNRT)CPUMSetGuestECX },
+    { "edx", scriptGPReg,  (PFNRT)CPUMSetGuestEDX },
+    { "esp", scriptGPReg,  (PFNRT)CPUMSetGuestESP },
+    { "ebp", scriptGPReg,  (PFNRT)CPUMSetGuestEBP },
+    { "esi", scriptGPReg,  (PFNRT)CPUMSetGuestESI },
+    { "edi", scriptGPReg,  (PFNRT)CPUMSetGuestEDI },
+    { "efl", scriptGPReg,  (PFNRT)CPUMSetGuestEFlags },
+    { "eip", scriptGPReg,  (PFNRT)CPUMSetGuestEIP },
+    { "ss",  scriptSelReg, (PFNRT)CPUMSetGuestSS },
+    { "cs",  scriptSelReg, (PFNRT)CPUMSetGuestCS },
+    { "ds",  scriptSelReg, (PFNRT)CPUMSetGuestDS },
+    { "es",  scriptSelReg, (PFNRT)CPUMSetGuestES },
+    { "fs",  scriptSelReg, (PFNRT)CPUMSetGuestFS },
+    { "gs",  scriptSelReg, (PFNRT)CPUMSetGuestGS },
+    { "cr0", scriptSysReg, (PFNRT)CPUMSetGuestCR0 },
+    { "cr2", scriptSysReg, (PFNRT)CPUMSetGuestCR2 },
+    { "cr3", scriptSysReg, (PFNRT)CPUMSetGuestCR3 },
+    { "cr4", scriptSysReg, (PFNRT)CPUMSetGuestCR4 },
+    { "ldtr",scriptSelReg, (PFNRT)CPUMSetGuestLDTR },
+    { "tr",  scriptSelReg, (PFNRT)CPUMSetGuestTR },
+    { "idtr",scriptDtrReg, (PFNRT)CPUMSetGuestIDTR },
+    { "gdtr",scriptDtrReg, (PFNRT)CPUMSetGuestGDTR }
+};
+
+
+static int scriptCommand(PVM pVM, const char *pszIn, size_t cch)
+{
+    NOREF(cch);
+    int rc = VINF_SUCCESS;
+    char *psz = RTStrDup(pszIn);
+    char *pszEqual = strchr(psz, '=');
+    if (pszEqual)
+    {
+        /*
+         * var = value
+         */
+        *pszEqual = '\0';
+        RTStrStripR(psz);
+        char *pszValue = RTStrStrip(pszEqual + 1);
+
+        rc = -1;
+        for (unsigned i = 0; i < RT_ELEMENTS(g_aVars); i++)
+        {
+            if (!strcmp(psz, g_aVars[i].pszVar))
+            {
+                rc = g_aVars[i].pfnHandler(pVM, psz, pszValue, (void *)(uintptr_t)g_aVars[i].pvUser);
+                break;
+            }
+        }
+    }
+
+    RTStrFree(psz);
+    return rc;
+}
+
+static DECLCALLBACK(int) scriptRun(PVM pVM, RTFILE File)
+{
+    RTPrintf("info: running script...\n");
+    uint64_t cb;
+    int rc = RTFileQuerySize(File, &cb);
+    if (RT_SUCCESS(rc))
+    {
+        if (cb == 0)
+            return VINF_SUCCESS;
+        if (cb < _1M)
+        {
+            char *pszBuf = (char *)RTMemAllocZ(cb + 1);
+            if (pszBuf)
+            {
+                rc = RTFileRead(File, pszBuf, cb, NULL);
+                if (RT_SUCCESS(rc))
+                {
+                    pszBuf[cb] = '\0';
+
+                    /*
+                     * Now process what's in the buffer.
+                     */
+                    char *psz = pszBuf;
+                    while (psz && *psz)
+                    {
+                        /* skip blanks. */
+                        while (RT_C_IS_SPACE(*psz))
+                            psz++;
+                        if (!*psz)
+                            break;
+
+                        /* end of line */
+                        char *pszNext;
+                        char *pszEnd = strchr(psz, '\n');
+                        if (!pszEnd)
+                            pszEnd = strchr(psz, '\r');
+                        if (!pszEnd)
+                            pszNext = pszEnd = strchr(psz, '\0');
+                        else
+                            pszNext = pszEnd + 1;
+
+                        if (*psz != ';' && *psz != '#' && *psz != '/')
+                        {
+                            /* strip end */
+                            *pszEnd = '\0';
+                            while (pszEnd > psz && RT_C_IS_SPACE(pszEnd[-1]))
+                                *--pszEnd = '\0';
+
+                            /* process the line */
+                            RTPrintf("debug: executing script line '%s'\n",  psz);
+                            rc = scriptCommand(pVM, psz, pszEnd - psz);
+                            if (RT_FAILURE(rc))
+                            {
+                                RTPrintf("error: '%s' failed: %Rrc\n", psz, rc);
+                                break;
+                            }
+                        }
+                        /* else comment line */
+
+                        /* next */
+                        psz = pszNext;
+                    }
+
+                }
+                else
+                    RTPrintf("error: failed to read script file: %Rrc\n", rc);
+                RTMemFree(pszBuf);
+            }
+            else
+            {
+                RTPrintf("error: Out of memory. (%d bytes)\n", cb + 1);
+                rc = VERR_NO_MEMORY;
+            }
+        }
+        else
+            RTPrintf("error: script file is too large (0x%llx bytes)\n", cb);
+    }
+    else
+        RTPrintf("error: couldn't get size of script file: %Rrc\n", rc);
+
+    return rc;
+}
+
+
+static DECLCALLBACK(int) loadMem(PVM pVM, RTFILE File, uint64_t *poff)
+{
+    uint64_t off = *poff;
+    RTPrintf("info: loading memory...\n");
+
+    int rc = RTFileSeek(File, off, RTFILE_SEEK_BEGIN, NULL);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCPHYS GCPhys = 0;
+        for (;;)
+        {
+            if (!(GCPhys % (PAGE_SIZE * 0x1000)))
+                RTPrintf("info: %RGp...\n", GCPhys);
+
+            /* read a page from the file */
+            size_t cbRead = 0;
+            uint8_t au8Page[PAGE_SIZE * 16];
+            rc = RTFileRead(File, &au8Page, sizeof(au8Page), &cbRead);
+            if (RT_SUCCESS(rc) && !cbRead)
+                rc = RTFileRead(File, &au8Page, sizeof(au8Page), &cbRead);
+            if (RT_SUCCESS(rc) && !cbRead)
+                rc = VERR_EOF;
+            if (RT_FAILURE(rc) || rc == VINF_EOF)
+            {
+                if (rc == VERR_EOF)
+                    rc = VINF_SUCCESS;
+                else
+                    RTPrintf("error: Read error %Rrc while reading the raw memory file.\n", rc);
+                break;
+            }
+
+            /* Write that page to the guest - skip known rom areas for now. */
+            if (GCPhys < 0xa0000 || GCPhys >= 0x100000) /* ASSUME size of a8Page is a power of 2. */
+                PGMPhysWrite(pVM, GCPhys, &au8Page, cbRead, PGMACCESSORIGIN_DEBUGGER);
+            GCPhys += cbRead;
+        }
+    }
+    else
+        RTPrintf("error: Failed to seek to 0x%llx in the raw memory file. rc=%Rrc\n", off, rc);
+
+    return rc;
+}
+
+
+/**
+ * Creates the default configuration.
+ * This assumes an empty tree.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to the VM.
+ */
+static DECLCALLBACK(int) cfgmR3CreateDefault(PUVM pUVM, PVM pVM, void *pvUser)
+{
+    RT_NOREF1(pUVM);
+    uint64_t cbMem = *(uint64_t *)pvUser;
+    int rc;
+    int rcAll = VINF_SUCCESS;
+    bool fIOAPIC = false;
+#define UPDATERC() do { if (RT_FAILURE(rc) && RT_SUCCESS(rcAll)) rcAll = rc; } while (0)
+
+    /*
+     * Create VM default values.
+     */
+    PCFGMNODE pRoot = CFGMR3GetRoot(pVM);
+    rc = CFGMR3InsertString(pRoot,  "Name",                 "Default VM");
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "RamSize",              cbMem);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "TimerMillies",         10);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "RawR3Enabled",         0);
+    UPDATERC();
+    /** @todo CFGM Defaults: RawR0, PATMEnabled and CASMEnabled needs attention later. */
+    rc = CFGMR3InsertInteger(pRoot, "RawR0Enabled",         0);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "PATMEnabled",          0);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "CSAMEnabled",          0);
+    UPDATERC();
+
+    /*
+     * PDM.
+     */
+    PCFGMNODE pPdm;
+    rc = CFGMR3InsertNode(pRoot, "PDM", &pPdm);
+    UPDATERC();
+    PCFGMNODE pDevices = NULL;
+    rc = CFGMR3InsertNode(pPdm, "Devices", &pDevices);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pDevices, "LoadBuiltin",       1); /* boolean */
+    UPDATERC();
+    PCFGMNODE pDrivers = NULL;
+    rc = CFGMR3InsertNode(pPdm, "Drivers", &pDrivers);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pDrivers, "LoadBuiltin",       1); /* boolean */
+    UPDATERC();
+
+
+    /*
+     * Devices
+     */
+    pDevices = NULL;
+    rc = CFGMR3InsertNode(pRoot, "Devices", &pDevices);
+    UPDATERC();
+    /* device */
+    PCFGMNODE pDev = NULL;
+    PCFGMNODE pInst = NULL;
+    PCFGMNODE pCfg = NULL;
+#if 0
+    PCFGMNODE pLunL0 = NULL;
+    PCFGMNODE pLunL1 = NULL;
+#endif
+
+    /*
+     * PC Arch.
+     */
+    rc = CFGMR3InsertNode(pDevices, "pcarch", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * PC Bios.
+     */
+    rc = CFGMR3InsertNode(pDevices, "pcbios", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "BootDevice0",          "IDE");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "BootDevice1",          "NONE");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "BootDevice2",          "NONE");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "BootDevice3",          "NONE");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "HardDiskDevice",       "piix3ide");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "FloppyDevice",         "i82078");
+    rc = CFGMR3InsertInteger(pCfg,  "IOAPIC", fIOAPIC);                         UPDATERC();
+    RTUUID Uuid;
+    RTUuidClear(&Uuid);
+    rc = CFGMR3InsertBytes(pCfg,    "UUID", &Uuid, sizeof(Uuid));               UPDATERC();
+    /* Bios logo. */
+    rc = CFGMR3InsertInteger(pCfg,  "FadeIn",               0);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "FadeOut",              0);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "LogoTime",             0);
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "LogoFile",             "");
+    UPDATERC();
+
+    /*
+     * ACPI
+     */
+    rc = CFGMR3InsertNode(pDevices, "acpi", &pDev);                             UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);                               UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted", 1);              /* boolean */   UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);                           UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "IOAPIC", fIOAPIC);                         UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "PCIDeviceNo",          7);                 UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "PCIFunctionNo",        0);                 UPDATERC();
+
+    /*
+     * DMA
+     */
+    rc = CFGMR3InsertNode(pDevices, "8237A", &pDev);                            UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);                               UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted", 1);             /* boolean */    UPDATERC();
+
+    /*
+     * PCI bus.
+     */
+    rc = CFGMR3InsertNode(pDevices, "pci", &pDev); /* piix3 */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "IOAPIC", fIOAPIC);                         UPDATERC();
+
+    /*
+     * PS/2 keyboard & mouse
+     */
+    rc = CFGMR3InsertNode(pDevices, "pckbd", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted", 1); /* boolean */                UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * Floppy
+     */
+    rc = CFGMR3InsertNode(pDevices, "i82078",    &pDev);                        UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0",         &pInst);                       UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",   1);                            UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config",    &pCfg);                        UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "IRQ",       6);                            UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "DMA",       2);                            UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "MemMapped", 0 );                           UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "IOBase",    0x3f0);                        UPDATERC();
+
+    /*
+     * i8254 Programmable Interval Timer And Dummy Speaker
+     */
+    rc = CFGMR3InsertNode(pDevices, "i8254", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * i8259 Programmable Interrupt Controller.
+     */
+    rc = CFGMR3InsertNode(pDevices, "i8259", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * APIC.
+     */
+    rc = CFGMR3InsertNode(pDevices, "apic", &pDev);                                 UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);                                   UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);     /* boolean */   UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);                               UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "IOAPIC", fIOAPIC);                             UPDATERC();
+
+    if (fIOAPIC)
+    {
+        /*
+         * I/O Advanced Programmable Interrupt Controller.
+         */
+        rc = CFGMR3InsertNode(pDevices, "ioapic", &pDev);                           UPDATERC();
+        rc = CFGMR3InsertNode(pDev,     "0", &pInst);                               UPDATERC();
+        rc = CFGMR3InsertInteger(pInst, "Trusted",          1);     /* boolean */   UPDATERC();
+        rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);                           UPDATERC();
+    }
+
+
+    /*
+     * RTC MC146818.
+     */
+    rc = CFGMR3InsertNode(pDevices, "mc146818", &pDev);                             UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);                                   UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);                               UPDATERC();
+
+    /*
+     * VGA.
+     */
+    rc = CFGMR3InsertNode(pDevices, "vga", &pDev);                                  UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);                                   UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);     /* boolean */   UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "PCIDeviceNo",          2);                     UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "PCIFunctionNo",        0);                     UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);                               UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "VRamSize",             8 * _1M);               UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "CustomVideoModes",     0);
+    rc = CFGMR3InsertInteger(pCfg,  "HeightReduction",      0);                     UPDATERC();
+    //rc = CFGMR3InsertInteger(pCfg,  "MonitorCount",         1);                     UPDATERC();
+
+    /*
+     * IDE controller.
+     */
+    rc = CFGMR3InsertNode(pDevices, "piix3ide", &pDev); /* piix3 */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);                               UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "PCIDeviceNo",          1);                     UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "PCIFunctionNo",        1);                     UPDATERC();
+
+    /*
+     * Network card.
+     */
+    rc = CFGMR3InsertNode(pDevices, "pcnet", &pDev);                                UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);                                   UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);      /* boolean */  UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "PCIDeviceNo",          3);                     UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "PCIFunctionNo",        0);                     UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);                               UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "Am79C973",             1);                     UPDATERC();
+    RTMAC Mac;
+    Mac.au16[0] = 0x0080;
+    Mac.au16[2] = Mac.au16[1] = 0x8086;
+    rc = CFGMR3InsertBytes(pCfg,    "MAC", &Mac, sizeof(Mac));                      UPDATERC();
+
+    /*
+     * VMM Device
+     */
+    rc = CFGMR3InsertNode(pDevices, "VMMDev", &pDev);                               UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);                                   UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);                               UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);     /* boolean */   UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "PCIDeviceNo",          4);                     UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "PCIFunctionNo",        0);                     UPDATERC();
+
+    /*
+     * ...
+     */
+
+#undef UPDATERC
+    return rcAll;
+}
+
+static void syntax(void)
+{
+    RTPrintf("Syntax: tstAnimate < -r <raw-mem-file> | -z <saved-state> > \n"
+             "              [-o <rawmem offset>]\n"
+             "              [-s <script file>]\n"
+             "              [-m <memory size>]\n"
+             "              [-w <warp drive percent>]\n"
+             "              [-p]\n"
+             "\n"
+             "The script is on the form:\n"
+             "<reg>=<value>\n");
+}
+
+
+/**
+ *  Entry point.
+ */
+extern "C" DECLEXPORT(int) TrustedMain(int argc, char **argv, char **envp)
+{
+    RT_NOREF1(envp);
+    int rcRet = 1;
+    int rc;
+    RTR3InitExe(argc, &argv, RTR3INIT_FLAGS_SUPLIB);
+
+    /*
+     * Parse input.
+     */
+    if (argc <= 1)
+    {
+        syntax();
+        return 1;
+    }
+
+    bool        fPowerOn = false;
+    uint32_t    u32WarpDrive = 100; /* % */
+    uint64_t    cbMem = ~0ULL;
+    const char *pszSavedState = NULL;
+    const char *pszRawMem = NULL;
+    uint64_t    offRawMem = 0;
+    const char *pszScript = NULL;
+    for (int i = 1; i < argc; i++)
+    {
+        if (argv[i][0] == '-')
+        {
+            /* check that it's on short form */
+            if (argv[i][2])
+            {
+                if (    strcmp(argv[i], "--help")
+                    &&  strcmp(argv[i], "-help"))
+                    RTPrintf("tstAnimate: Syntax error: Unknown argument '%s'.\n", argv[i]);
+                else
+                    syntax();
+                return 1;
+            }
+
+            /* check for 2nd argument */
+            switch (argv[i][1])
+            {
+                case 'r':
+                case 'o':
+                case 'c':
+                case 'm':
+                case 'w':
+                case 'z':
+                    if (i + 1 < argc)
+                        break;
+                    RTPrintf("tstAnimate: Syntax error: '%s' takes a 2nd argument.\n", argv[i]);
+                    return 1;
+            }
+
+            /* process argument */
+            switch (argv[i][1])
+            {
+                case 'r':
+                    pszRawMem = argv[++i];
+                    break;
+
+                case 'z':
+                    pszSavedState = argv[++i];
+                    break;
+
+                case 'o':
+                {
+                    rc = RTStrToUInt64Ex(argv[++i], NULL, 0, &offRawMem);
+                    if (RT_FAILURE(rc))
+                    {
+                        RTPrintf("tstAnimate: Syntax error: Invalid offset given to -o.\n");
+                        return 1;
+                    }
+                    break;
+                }
+
+                case 'm':
+                {
+                    char *pszNext;
+                    rc = RTStrToUInt64Ex(argv[++i], &pszNext, 0, &cbMem);
+                    if (RT_FAILURE(rc))
+                    {
+                        RTPrintf("tstAnimate: Syntax error: Invalid memory size given to -m.\n");
+                        return 1;
+                    }
+                    switch (*pszNext)
+                    {
+                        case 'G':   cbMem *= _1G; pszNext++; break;
+                        case 'M':   cbMem *= _1M; pszNext++; break;
+                        case 'K':   cbMem *= _1K; pszNext++; break;
+                        case '\0':  break;
+                        default:
+                            RTPrintf("tstAnimate: Syntax error: Invalid memory size given to -m.\n");
+                            return 1;
+                    }
+                    if (*pszNext)
+                    {
+                        RTPrintf("tstAnimate: Syntax error: Invalid memory size given to -m.\n");
+                        return 1;
+                    }
+                    break;
+                }
+
+                case 's':
+                    pszScript = argv[++i];
+                    break;
+
+                case 'p':
+                    fPowerOn = true;
+                    break;
+
+                case 'w':
+                {
+                    rc = RTStrToUInt32Ex(argv[++i], NULL, 0, &u32WarpDrive);
+                    if (RT_FAILURE(rc))
+                    {
+                        RTPrintf("tstAnimate: Syntax error: Invalid number given to -w.\n");
+                        return 1;
+                    }
+                    break;
+                }
+
+                case 'h':
+                case 'H':
+                case '?':
+                    syntax();
+                    return 1;
+
+                default:
+                    RTPrintf("tstAnimate: Syntax error: Unknown argument '%s'.\n", argv[i]);
+                    return 1;
+            }
+        }
+        else
+        {
+            RTPrintf("tstAnimate: Syntax error at arg no. %d '%s'.\n", i, argv[i]);
+            syntax();
+            return 1;
+        }
+    }
+
+    /*
+     * Check that the basic requirements are met.
+     */
+    if (pszRawMem && pszSavedState)
+    {
+        RTPrintf("tstAnimate: Syntax error: Either -z or -r, not both.\n");
+        return 1;
+    }
+    if (!pszRawMem && !pszSavedState)
+    {
+        RTPrintf("tstAnimate: Syntax error: The -r argument is compulsory.\n");
+        return 1;
+    }
+
+    /*
+     * Open the files.
+     */
+    RTFILE FileRawMem = NIL_RTFILE;
+    if (pszRawMem)
+    {
+        rc = RTFileOpen(&FileRawMem, pszRawMem, RTFILE_O_READ | RTFILE_O_OPEN | RTFILE_O_DENY_WRITE);
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf("tstAnimate: error: Failed to open '%s': %Rrc\n", pszRawMem, rc);
+            return 1;
+        }
+    }
+    RTFILE FileScript = NIL_RTFILE;
+    if (pszScript)
+    {
+        rc = RTFileOpen(&FileScript, pszScript, RTFILE_O_READ | RTFILE_O_OPEN | RTFILE_O_DENY_WRITE);
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf("tstAnimate: error: Failed to open '%s': %Rrc\n", pszScript, rc);
+            return 1;
+        }
+    }
+
+    /*
+     * Figure the memsize if not specified.
+     */
+    if (cbMem == ~0ULL)
+    {
+        if (FileRawMem != NIL_RTFILE)
+        {
+            rc = RTFileQuerySize(FileRawMem, &cbMem);
+            AssertReleaseRC(rc);
+            cbMem -= offRawMem;
+            cbMem &= ~(PAGE_SIZE - 1);
+        }
+        else
+        {
+            RTPrintf("tstAnimate: error: too lazy to figure out the memsize in a saved state.\n");
+            return 1;
+        }
+    }
+    RTPrintf("tstAnimate: info: cbMem=0x%llx bytes\n", cbMem);
+
+    /*
+     * Open a release log.
+     */
+    static const char * const s_apszGroups[] = VBOX_LOGGROUP_NAMES;
+    PRTLOGGER pRelLogger;
+    rc = RTLogCreate(&pRelLogger, RTLOGFLAGS_PREFIX_TIME_PROG, "all", "VBOX_RELEASE_LOG",
+                     RT_ELEMENTS(s_apszGroups), s_apszGroups, RTLOGDEST_FILE, "./tstAnimate.log");
+    if (RT_SUCCESS(rc))
+        RTLogRelSetDefaultInstance(pRelLogger);
+    else
+        RTPrintf("tstAnimate: rtLogCreateEx failed - %Rrc\n", rc);
+
+    /*
+     * Create empty VM.
+     */
+    PVM pVM;
+    PUVM pUVM;
+    rc = VMR3Create(1, NULL, NULL, NULL, cfgmR3CreateDefault, &cbMem, &pVM, &pUVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Load memory.
+         */
+        if (FileRawMem != NIL_RTFILE)
+            rc = VMR3ReqCallWaitU(pUVM, VMCPUID_ANY, (PFNRT)loadMem, 3, pVM, FileRawMem, &offRawMem);
+        else
+            rc = VMR3ReqCallWaitU(pUVM, VMCPUID_ANY, (PFNRT)SSMR3Load,
+                                  7, pVM, pszSavedState, (uintptr_t)NULL /*pStreamOps*/, (uintptr_t)NULL /*pvUser*/,
+                                  SSMAFTER_DEBUG_IT, (uintptr_t)NULL /*pfnProgress*/, (uintptr_t)NULL /*pvProgressUser*/);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Load register script.
+             */
+            if (FileScript != NIL_RTFILE)
+                rc = VMR3ReqCallWaitU(pUVM, VMCPUID_ANY, (PFNRT)scriptRun, 2, pVM, FileScript);
+            if (RT_SUCCESS(rc))
+            {
+                if (fPowerOn)
+                {
+                    /*
+                     * Adjust warpspeed?
+                     */
+                    if (u32WarpDrive != 100)
+                    {
+                        rc = TMR3SetWarpDrive(pUVM, u32WarpDrive);
+                        if (RT_FAILURE(rc))
+                            RTPrintf("warning: TMVirtualSetWarpDrive(,%u) -> %Rrc\n", u32WarpDrive, rc);
+                    }
+
+                    /*
+                     * Start the thing with single stepping and stuff enabled.
+                     * (Try make sure we don't execute anything in raw mode.)
+                     */
+                    RTPrintf("info: powering on the VM...\n");
+                    RTLogGroupSettings(NULL, "+REM_DISAS.e.l.f");
+                    rc = VERR_NOT_IMPLEMENTED; /** @todo need some EM single-step indicator (was REMR3DisasEnableStepping) */
+                    if (RT_SUCCESS(rc))
+                    {
+                        rc = EMR3SetExecutionPolicy(pUVM, EMEXECPOLICY_RECOMPILE_RING0, true); AssertReleaseRC(rc);
+                        rc = EMR3SetExecutionPolicy(pUVM, EMEXECPOLICY_RECOMPILE_RING3, true); AssertReleaseRC(rc);
+                        DBGFR3Info(pUVM, "cpumguest", "verbose", NULL);
+                        if (fPowerOn)
+                            rc = VMR3PowerOn(pUVM);
+                        if (RT_SUCCESS(rc))
+                        {
+                            RTPrintf("info: VM is running\n");
+                            signal(SIGINT, SigInterrupt);
+                            while (!g_fSignaled)
+                                RTThreadSleep(1000);
+                        }
+                        else
+                            RTPrintf("error: Failed to power on the VM: %Rrc\n", rc);
+                    }
+                    else
+                        RTPrintf("error: Failed to enabled singlestepping: %Rrc\n", rc);
+                }
+                else
+                {
+                    /*
+                     * Don't start it, just enter the debugger.
+                     */
+                    RTPrintf("info: entering debugger...\n");
+                    DBGFR3Info(pUVM, "cpumguest", "verbose", NULL);
+                    signal(SIGINT, SigInterrupt);
+                    while (!g_fSignaled)
+                        RTThreadSleep(1000);
+                }
+                RTPrintf("info: shutting down the VM...\n");
+            }
+            /* execScript complains */
+        }
+        else if (FileRawMem == NIL_RTFILE) /* loadMem complains, SSMR3Load doesn't */
+            RTPrintf("tstAnimate: error: SSMR3Load failed: rc=%Rrc\n", rc);
+        rcRet = RT_SUCCESS(rc) ? 0 : 1;
+
+        /*
+         * Cleanup.
+         */
+        rc = VMR3Destroy(pUVM);
+        if (!RT_SUCCESS(rc))
+        {
+            RTPrintf("tstAnimate: error: failed to destroy vm! rc=%Rrc\n", rc);
+            rcRet++;
+        }
+
+        VMR3ReleaseUVM(pUVM);
+    }
+    else
+    {
+        RTPrintf("tstAnimate: fatal error: failed to create vm! rc=%Rrc\n", rc);
+        rcRet++;
+    }
+
+    return rcRet;
+}
+
+
+#if !defined(VBOX_WITH_HARDENING) || !defined(RT_OS_WINDOWS)
+/**
+ * Main entry point.
+ */
+int main(int argc, char **argv, char **envp)
+{
+    return TrustedMain(argc, argv, envp);
+}
+#endif
+
diff --git a/src/VBox/VMM/testcase/tstAsmStructs.cpp b/src/VBox/VMM/testcase/tstAsmStructs.cpp
new file mode 100644
index 00000000..f3d80f65
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstAsmStructs.cpp
@@ -0,0 +1,54 @@
+/* $Id: tstAsmStructs.cpp $ */
+/** @file
+ * Testcase for checking offsets in the assembly structures shared with C/C++.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/vmm/cpum.h>
+#include "CPUMInternal.h"
+#include "HMInternal.h"
+#include "VMMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/hm_vmx.h>
+
+#include "tstHelp.h"
+#include <stdio.h>
+
+
+/* For sup.mac simplifications. */
+#define SUPDRVTRACERUSRCTX32    SUPDRVTRACERUSRCTX
+#define SUPDRVTRACERUSRCTX64    SUPDRVTRACERUSRCTX
+
+
+int main()
+{
+    int rc = 0;
+    printf("tstAsmStructs: TESTING\n");
+
+#ifdef IN_RING3
+# include "tstAsmStructsHC.h"
+#else
+# include "tstAsmStructsRC.h"
+#endif
+
+    if (rc)
+        printf("tstAsmStructs: FAILURE - %d errors \n", rc);
+    else
+        printf("tstAsmStructs: SUCCESS\n");
+    return rc;
+}
diff --git a/src/VBox/VMM/testcase/tstAsmStructsAsm-lst.sed b/src/VBox/VMM/testcase/tstAsmStructsAsm-lst.sed
new file mode 100644
index 00000000..1d416585
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstAsmStructsAsm-lst.sed
@@ -0,0 +1,105 @@
+# $Id: tstAsmStructsAsm-lst.sed $
+## @file
+# For testing assembly struct when using yasm.
+#
+
+#
+# Copyright (C) 2006-2020 Oracle Corporation
+#
+# This file is part of VirtualBox Open Source Edition (OSE), as
+# available from http://www.virtualbox.org. This file is free software;
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License (GPL) as published by the Free Software
+# Foundation, in version 2 as it comes in the "COPYING" file of the
+# VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+# hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+#
+
+#
+# Strip stuff lines and spaces we don't care about.
+#
+/ %line /d
+/\[section /d
+/\[bits /d
+/\[absolute /d
+/ times /d
+s/ *[[:digit:]]* //
+/^ *$/d
+s/ *$//g
+s/^ *//g
+/^\.text$/d
+/^\.data$/d
+/^\.bss$/d
+s/[[:space:]][[:space:]]*/ /g
+
+#
+# Figure which type of line this is and process it accordingly.
+#
+/^[[:alpha:]_][[:alnum:]_]*:/b struct
+/^[[:alpha:]_][[:alnum:]_]*_size EQU \$ - .*$/b struct_equ
+/<gap>/b member
+/^\.[[:alpha:]_][[:alnum:]_.:]* res.*$/b member_two
+/^\.[[:alpha:]_][[:alnum:]_.:]*:$/b member_alias
+b error
+b member_two
+
+
+#
+# Struct start / end.
+#
+:struct_equ
+s/_size EQU.*$/_size/
+:struct
+s/:$//
+h
+s/^/global /
+s/$/ ; struct/
+b end
+
+
+#
+# Struct member
+# Note: the 't' command doesn't seem to be working right with 's'.
+#
+:member
+s/[[:xdigit:]]* *//
+s/<gap> *//
+/^\.[[:alnum:]_.]*[:]* .*$/!t error
+s/\(\.[[:alnum:]_]*\)[:]* .*$/\1 /
+G
+s/^\([^ ]*\) \(.*\)$/global \2\1 ; member/
+s/\n//m
+
+b end
+
+
+#
+# Struct member, no address. yasm r1842 and later.
+#
+:member_two
+s/[:]*  *res[bwdtq] .*$//
+s/$/ /
+/^\.[[:alnum:]_.]* *$/!t error
+G
+s/^\([^ ]*\) \(.*\)$/global \2\1 ; member2/
+s/\n//m
+
+b end
+
+#
+# Alias member like Host.cr0Fpu in 64-bit.  Drop it.
+#
+:member_alias
+d
+b end
+
+:error
+s/^/\nSed script logic error!\nBuffer: /
+s/$/\nHold: /
+G
+q 1
+b end
+
+
+:end
+
diff --git a/src/VBox/VMM/testcase/tstAsmStructsAsm.asm b/src/VBox/VMM/testcase/tstAsmStructsAsm.asm
new file mode 100644
index 00000000..5465c587
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstAsmStructsAsm.asm
@@ -0,0 +1,39 @@
+; $Id: tstAsmStructsAsm.asm $
+;; @file
+; Assembly / C structure layout testcase.
+;
+; Make yasm/nasm create absolute symbols for the structure definition
+; which we can parse and make code from using objdump and sed.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%ifdef RT_ARCH_AMD64
+BITS 64
+%endif
+
+%include "CPUMInternal.mac"
+%include "HMInternal.mac"
+%include "VMMInternal.mac"
+%include "VBox/vmm/cpum.mac"
+%include "VBox/vmm/vm.mac"
+%include "VBox/vmm/hm_vmx.mac"
+%include "VBox/sup.mac"
+%ifdef DO_GLOBALS
+ %include "tstAsmStructsAsm.mac"
+%endif
+
+.text
+.data
+.bss
+
diff --git a/src/VBox/VMM/testcase/tstCFGM.cpp b/src/VBox/VMM/testcase/tstCFGM.cpp
new file mode 100644
index 00000000..85c0c404
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstCFGM.cpp
@@ -0,0 +1,171 @@
+/* $Id: tstCFGM.cpp $ */
+/** @file
+ * Testcase for CFGM.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/sup.h>
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <iprt/initterm.h>
+#include <iprt/stream.h>
+#include <iprt/mem.h>
+#include <iprt/string.h>
+
+#include <iprt/test.h>
+
+
+static void doGeneralTests(PCFGMNODE pRoot)
+{
+    /* test multilevel node creation */
+    PCFGMNODE pChild = NULL;
+    RTTESTI_CHECK_RC_RETV(CFGMR3InsertNode(pRoot, "First/Second/Third//Final", &pChild), VINF_SUCCESS);
+    RTTESTI_CHECK_RETV(RT_VALID_PTR(pChild));
+    RTTESTI_CHECK(CFGMR3GetChild(pRoot, "First/Second/Third/Final") == pChild);
+
+    /*
+     * Boolean queries.
+     */
+    RTTESTI_CHECK_RC(CFGMR3InsertInteger(pChild, "BoolValue", 1), VINF_SUCCESS);
+    bool f = false;
+    RTTESTI_CHECK_RC(CFGMR3QueryBool(pChild, "BoolValue", &f), VINF_SUCCESS);
+    RTTESTI_CHECK(f == true);
+
+    RTTESTI_CHECK_RC(CFGMR3QueryBool(pRoot, "BoolValue", &f), VERR_CFGM_VALUE_NOT_FOUND);
+    RTTESTI_CHECK_RC(CFGMR3QueryBool(NULL, "BoolValue", &f), VERR_CFGM_NO_PARENT);
+
+    RTTESTI_CHECK_RC(CFGMR3QueryBoolDef(pChild, "ValueNotFound", &f, true), VINF_SUCCESS);
+    RTTESTI_CHECK(f == true);
+    RTTESTI_CHECK_RC(CFGMR3QueryBoolDef(pChild, "ValueNotFound", &f, false), VINF_SUCCESS);
+    RTTESTI_CHECK(f == false);
+
+    RTTESTI_CHECK_RC(CFGMR3QueryBoolDef(NULL, "BoolValue", &f, true), VINF_SUCCESS);
+    RTTESTI_CHECK(f == true);
+    RTTESTI_CHECK_RC(CFGMR3QueryBoolDef(NULL, "BoolValue", &f, false), VINF_SUCCESS);
+    RTTESTI_CHECK(f == false);
+
+}
+
+
+
+static void doTestsOnDefaultValues(PCFGMNODE pRoot)
+{
+    /* integer */
+    uint64_t u64;
+    RTTESTI_CHECK_RC(CFGMR3QueryU64(pRoot, "RamSize", &u64), VINF_SUCCESS);
+
+    size_t cb = 0;
+    RTTESTI_CHECK_RC(CFGMR3QuerySize(pRoot, "RamSize", &cb), VINF_SUCCESS);
+    RTTESTI_CHECK(cb == sizeof(uint64_t));
+
+    /* string */
+    char *pszName = NULL;
+    RTTESTI_CHECK_RC(CFGMR3QueryStringAlloc(pRoot, "Name", &pszName), VINF_SUCCESS);
+    RTTESTI_CHECK_RC(CFGMR3QuerySize(pRoot, "Name", &cb), VINF_SUCCESS);
+    RTTESTI_CHECK(cb == strlen(pszName) + 1);
+    MMR3HeapFree(pszName);
+}
+
+
+static void doInVmmTests(RTTEST hTest)
+{
+    /*
+     * Create empty VM structure and init SSM.
+     */
+    int rc = SUPR3Init(NULL);
+    if (RT_FAILURE(rc))
+    {
+        RTTestSkipped(hTest, "SUPR3Init failed with rc=%Rrc",  rc);
+        return;
+    }
+
+    PVM pVM;
+    RTTESTI_CHECK_RC_RETV(SUPR3PageAlloc(RT_ALIGN_Z(sizeof(*pVM), PAGE_SIZE) >> PAGE_SHIFT, (void **)&pVM), VINF_SUCCESS);
+
+
+    PUVM pUVM = (PUVM)RTMemPageAlloc(sizeof(*pUVM));
+    pUVM->u32Magic = UVM_MAGIC;
+    pUVM->pVM = pVM;
+    pVM->pUVM = pUVM;
+
+    /*
+     * Do the testing.
+     */
+    RTTESTI_CHECK_RC_RETV(STAMR3InitUVM(pUVM), VINF_SUCCESS);
+    RTTESTI_CHECK_RC_RETV(MMR3InitUVM(pUVM), VINF_SUCCESS);
+    RTTESTI_CHECK_RC_RETV(CFGMR3Init(pVM, NULL, NULL), VINF_SUCCESS);
+    RTTESTI_CHECK_RETV(CFGMR3GetRoot(pVM) != NULL);
+
+    doTestsOnDefaultValues(CFGMR3GetRoot(pVM));
+    doGeneralTests(CFGMR3GetRoot(pVM));
+
+
+    /* done */
+    RTTESTI_CHECK_RC_RETV(CFGMR3Term(pVM), VINF_SUCCESS);
+}
+
+
+static void doStandaloneTests(void)
+{
+    RTTestISub("Standalone");
+    PCFGMNODE pRoot;;
+    RTTESTI_CHECK_RETV((pRoot = CFGMR3CreateTree(NULL)) != NULL);
+    doGeneralTests(pRoot);
+    CFGMR3DestroyTree(pRoot);
+}
+
+
+/**
+ *  Entry point.
+ */
+extern "C" DECLEXPORT(int) TrustedMain(int argc, char **argv, char **envp)
+{
+    RT_NOREF3(argc, argv, envp);
+
+    /*
+     * Init runtime.
+     */
+    RTTEST hTest;
+    RTR3InitExeNoArguments(RTR3INIT_FLAGS_SUPLIB);
+    RTEXITCODE rcExit = RTTestInitAndCreate("tstCFGM", &hTest);
+    if (rcExit != RTEXITCODE_SUCCESS)
+        return rcExit;
+
+    doInVmmTests(hTest);
+    doStandaloneTests();
+
+    return RTTestSummaryAndDestroy(hTest);
+}
+
+
+#if !defined(VBOX_WITH_HARDENING) || !defined(RT_OS_WINDOWS)
+/**
+ * Main entry point.
+ */
+int main(int argc, char **argv, char **envp)
+{
+    return TrustedMain(argc, argv, envp);
+}
+#endif
+
diff --git a/src/VBox/VMM/testcase/tstCompressionBenchmark.cpp b/src/VBox/VMM/testcase/tstCompressionBenchmark.cpp
new file mode 100644
index 00000000..3a04992a
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstCompressionBenchmark.cpp
@@ -0,0 +1,642 @@
+/* $Id: tstCompressionBenchmark.cpp $ */
+/** @file
+ * Compression Benchmark for SSM and PGM.
+ */
+
+/*
+ * Copyright (C) 2009-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/buildconfig.h>
+#include <iprt/crc.h>
+#include <iprt/ctype.h>
+#include <iprt/err.h>
+#include <iprt/file.h>
+#include <iprt/getopt.h>
+#include <iprt/initterm.h>
+#include <iprt/md5.h>
+#include <iprt/sha.h>
+#include <iprt/mem.h>
+#include <iprt/param.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+#include <iprt/time.h>
+#include <iprt/zip.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+static size_t   g_cPages = 20*_1M / PAGE_SIZE;
+static size_t   g_cbPages;
+static uint8_t *g_pabSrc;
+
+/** Buffer for the decompressed data (g_cbPages). */
+static uint8_t *g_pabDecompr;
+
+/** Buffer for the compressed data (g_cbComprAlloc). */
+static uint8_t *g_pabCompr;
+/** The current size of the compressed data, ComprOutCallback */
+static size_t   g_cbCompr;
+/** The current offset into the compressed data, DecomprInCallback. */
+static size_t   g_offComprIn;
+/** The amount of space allocated for compressed data. */
+static size_t   g_cbComprAlloc;
+
+
+/**
+ * Store compressed data in the g_pabCompr buffer.
+ */
+static DECLCALLBACK(int) ComprOutCallback(void *pvUser, const void *pvBuf, size_t cbBuf)
+{
+    NOREF(pvUser);
+    AssertReturn(g_cbCompr + cbBuf <= g_cbComprAlloc, VERR_BUFFER_OVERFLOW);
+    memcpy(&g_pabCompr[g_cbCompr], pvBuf, cbBuf);
+    g_cbCompr += cbBuf;
+    return VINF_SUCCESS;
+}
+
+/**
+ * Read compressed data from g_pabComrp.
+ */
+static DECLCALLBACK(int) DecomprInCallback(void *pvUser, void *pvBuf, size_t cbBuf, size_t *pcbBuf)
+{
+    NOREF(pvUser);
+    size_t cb = RT_MIN(cbBuf, g_cbCompr - g_offComprIn);
+    if (pcbBuf)
+        *pcbBuf = cb;
+//    AssertReturn(cb > 0, VERR_EOF);
+    memcpy(pvBuf, &g_pabCompr[g_offComprIn], cb);
+    g_offComprIn += cb;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Benchmark RTCrc routines potentially relevant for SSM or PGM - All in one go.
+ *
+ * @param  pabSrc   Pointer to the test data.
+ * @param  cbSrc    The size of the test data.
+ */
+static void tstBenchmarkCRCsAllInOne(uint8_t const *pabSrc, size_t cbSrc)
+{
+    RTPrintf("Algorithm     Speed                  Time      Digest\n"
+             "------------------------------------------------------------------------------\n");
+
+    uint64_t NanoTS = RTTimeNanoTS();
+    uint32_t u32Crc = RTCrc32(pabSrc, cbSrc);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    unsigned uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTPrintf("CRC-32    %'9u KB/s  %'15llu ns - %08x\n", uSpeed, NanoTS, u32Crc);
+
+
+    NanoTS = RTTimeNanoTS();
+    uint64_t u64Crc = RTCrc64(pabSrc, cbSrc);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTPrintf("CRC-64    %'9u KB/s  %'15llu ns - %016llx\n", uSpeed, NanoTS, u64Crc);
+
+    NanoTS = RTTimeNanoTS();
+    u32Crc = RTCrcAdler32(pabSrc, cbSrc);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTPrintf("Adler-32  %'9u KB/s  %'15llu ns - %08x\n", uSpeed, NanoTS, u32Crc);
+
+    NanoTS = RTTimeNanoTS();
+    uint8_t abMd5Hash[RTMD5HASHSIZE];
+    RTMd5(pabSrc, cbSrc, abMd5Hash);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    char szDigest[257];
+    RTMd5ToString(abMd5Hash, szDigest, sizeof(szDigest));
+    RTPrintf("MD5       %'9u KB/s  %'15llu ns - %s\n", uSpeed, NanoTS, szDigest);
+
+    NanoTS = RTTimeNanoTS();
+    uint8_t abSha1Hash[RTSHA1_HASH_SIZE];
+    RTSha1(pabSrc, cbSrc, abSha1Hash);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTSha1ToString(abSha1Hash, szDigest, sizeof(szDigest));
+    RTPrintf("SHA-1     %'9u KB/s  %'15llu ns - %s\n", uSpeed, NanoTS, szDigest);
+
+    NanoTS = RTTimeNanoTS();
+    uint8_t abSha256Hash[RTSHA256_HASH_SIZE];
+    RTSha256(pabSrc, cbSrc, abSha256Hash);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTSha256ToString(abSha256Hash, szDigest, sizeof(szDigest));
+    RTPrintf("SHA-256   %'9u KB/s  %'15llu ns - %s\n", uSpeed, NanoTS, szDigest);
+
+    NanoTS = RTTimeNanoTS();
+    uint8_t abSha512Hash[RTSHA512_HASH_SIZE];
+    RTSha512(pabSrc, cbSrc, abSha512Hash);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTSha512ToString(abSha512Hash, szDigest, sizeof(szDigest));
+    RTPrintf("SHA-512   %'9u KB/s  %'15llu ns - %s\n", uSpeed, NanoTS, szDigest);
+}
+
+
+/**
+ * Benchmark RTCrc routines potentially relevant for SSM or PGM - Page by page.
+ *
+ * @param  pabSrc   Pointer to the test data.
+ * @param  cbSrc    The size of the test data.
+ */
+static void tstBenchmarkCRCsPageByPage(uint8_t const *pabSrc, size_t cbSrc)
+{
+    RTPrintf("Algorithm     Speed                  Time     \n"
+             "----------------------------------------------\n");
+
+    size_t const cPages = cbSrc / PAGE_SIZE;
+
+    uint64_t NanoTS = RTTimeNanoTS();
+    for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        RTCrc32(&pabSrc[iPage * PAGE_SIZE], PAGE_SIZE);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    unsigned uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTPrintf("CRC-32    %'9u KB/s  %'15llu ns\n", uSpeed, NanoTS);
+
+
+    NanoTS = RTTimeNanoTS();
+    for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        RTCrc64(&pabSrc[iPage * PAGE_SIZE], PAGE_SIZE);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTPrintf("CRC-64    %'9u KB/s  %'15llu ns\n", uSpeed, NanoTS);
+
+    NanoTS = RTTimeNanoTS();
+    for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        RTCrcAdler32(&pabSrc[iPage * PAGE_SIZE], PAGE_SIZE);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTPrintf("Adler-32  %'9u KB/s  %'15llu ns\n", uSpeed, NanoTS);
+
+    NanoTS = RTTimeNanoTS();
+    uint8_t abMd5Hash[RTMD5HASHSIZE];
+    for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        RTMd5(&pabSrc[iPage * PAGE_SIZE], PAGE_SIZE, abMd5Hash);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTPrintf("MD5       %'9u KB/s  %'15llu ns\n", uSpeed, NanoTS);
+
+    NanoTS = RTTimeNanoTS();
+    uint8_t abSha1Hash[RTSHA1_HASH_SIZE];
+    for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        RTSha1(&pabSrc[iPage * PAGE_SIZE], PAGE_SIZE, abSha1Hash);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTPrintf("SHA-1     %'9u KB/s  %'15llu ns\n", uSpeed, NanoTS);
+
+    NanoTS = RTTimeNanoTS();
+    uint8_t abSha256Hash[RTSHA256_HASH_SIZE];
+    for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        RTSha256(&pabSrc[iPage * PAGE_SIZE], PAGE_SIZE, abSha256Hash);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTPrintf("SHA-256   %'9u KB/s  %'15llu ns\n", uSpeed, NanoTS);
+
+    NanoTS = RTTimeNanoTS();
+    uint8_t abSha512Hash[RTSHA512_HASH_SIZE];
+    for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        RTSha512(&pabSrc[iPage * PAGE_SIZE], PAGE_SIZE, abSha512Hash);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+    uSpeed = (unsigned)(cbSrc / (long double)NanoTS * 1000000000.0 / 1024);
+    RTPrintf("SHA-512   %'9u KB/s  %'15llu ns\n", uSpeed, NanoTS);
+}
+
+
+/** Prints an error message and returns 1 for quick return from main use. */
+static int Error(const char *pszMsgFmt, ...)
+{
+    RTStrmPrintf(g_pStdErr, "\nerror: ");
+    va_list va;
+    va_start(va, pszMsgFmt);
+    RTStrmPrintfV(g_pStdErr, pszMsgFmt, va);
+    va_end(va);
+    return 1;
+}
+
+
+int main(int argc, char **argv)
+{
+    RTR3InitExe(argc, &argv, 0);
+
+    /*
+     * Parse arguments.
+     */
+    static const RTGETOPTDEF    s_aOptions[] =
+    {
+        { "--iterations",     'i', RTGETOPT_REQ_UINT32 },
+        { "--num-pages",      'n', RTGETOPT_REQ_UINT32 },
+        { "--page-at-a-time", 'c', RTGETOPT_REQ_UINT32 },
+        { "--page-file",      'f', RTGETOPT_REQ_STRING },
+        { "--offset",         'o', RTGETOPT_REQ_UINT64 },
+    };
+
+    const char     *pszPageFile = NULL;
+    uint64_t        offPageFile = 0;
+    uint32_t        cIterations = 1;
+    uint32_t        cPagesAtATime = 1;
+    RTGETOPTUNION   Val;
+    RTGETOPTSTATE   State;
+    int rc = RTGetOptInit(&State, argc, argv, &s_aOptions[0], RT_ELEMENTS(s_aOptions), 1, 0);
+    AssertRCReturn(rc, 1);
+
+    while ((rc = RTGetOpt(&State, &Val)))
+    {
+        switch (rc)
+        {
+            case 'n':
+                g_cPages = Val.u32;
+                if (g_cPages * PAGE_SIZE * 4 / (PAGE_SIZE * 4) != g_cPages)
+                    return Error("The specified page count is too high: %#x (%#llx bytes)\n", g_cPages, (uint64_t)g_cPages * PAGE_SHIFT);
+                if (g_cPages < 1)
+                    return Error("The specified page count is too low: %#x\n", g_cPages);
+                break;
+
+            case 'i':
+                cIterations = Val.u32;
+                if (cIterations < 1)
+                    return Error("The number of iterations must be 1 or higher\n");
+                break;
+
+            case 'c':
+                cPagesAtATime = Val.u32;
+                if (cPagesAtATime < 1 || cPagesAtATime > 10240)
+                    return Error("The specified pages-at-a-time count is out of range: %#x\n", cPagesAtATime);
+                break;
+
+            case 'f':
+                pszPageFile = Val.psz;
+                break;
+
+            case 'o':
+                offPageFile = Val.u64;
+                break;
+
+            case 'O':
+                offPageFile = Val.u64 * PAGE_SIZE;
+                break;
+
+            case 'h':
+                RTPrintf("syntax: tstCompressionBenchmark [options]\n"
+                         "\n"
+                         "Options:\n"
+                         "  -h, --help\n"
+                         "    Show this help page\n"
+                         "  -i, --iterations <num>\n"
+                         "    The number of iterations.\n"
+                         "  -n, --num-pages <pages>\n"
+                         "    The number of pages.\n"
+                         "  -c, --pages-at-a-time <pages>\n"
+                         "    Number of pages at a time.\n"
+                         "  -f, --page-file <filename>\n"
+                         "    File or device to read the page from. The default\n"
+                         "    is to generate some garbage.\n"
+                         "  -o, --offset <file-offset>\n"
+                         "    Offset into the page file to start reading at.\n");
+                return 0;
+
+            case 'V':
+                RTPrintf("%sr%s\n", RTBldCfgVersion(), RTBldCfgRevisionStr());
+                return 0;
+
+            default:
+                return RTGetOptPrintError(rc, &Val);
+        }
+    }
+
+    g_cbPages = g_cPages * PAGE_SIZE;
+    uint64_t cbTotal = (uint64_t)g_cPages * PAGE_SIZE * cIterations;
+    uint64_t cbTotalKB = cbTotal / _1K;
+    if (cbTotal / cIterations != g_cbPages)
+        return Error("cPages * cIterations -> overflow\n");
+
+    /*
+     * Gather the test memory.
+     */
+    if (pszPageFile)
+    {
+        size_t cbFile;
+        rc = RTFileReadAllEx(pszPageFile, offPageFile, g_cbPages, RTFILE_RDALL_O_DENY_NONE, (void **)&g_pabSrc, &cbFile);
+        if (RT_FAILURE(rc))
+            return Error("Error reading %zu bytes from %s at %llu: %Rrc\n", g_cbPages, pszPageFile, offPageFile, rc);
+        if (cbFile != g_cbPages)
+            return Error("Error reading %zu bytes from %s at %llu: got %zu bytes\n", g_cbPages, pszPageFile, offPageFile, cbFile);
+    }
+    else
+    {
+        g_pabSrc = (uint8_t *)RTMemAlloc(g_cbPages);
+        if (g_pabSrc)
+        {
+            /* Just fill it with something - warn about the low quality of the something. */
+            RTPrintf("tstCompressionBenchmark: WARNING! No input file was specified so the source\n"
+                     "buffer will be filled with generated data of questionable quality.\n");
+#ifdef RT_OS_LINUX
+            RTPrintf("To get real RAM on linux: sudo dd if=/dev/mem ... \n");
+#endif
+            uint8_t *pb    = g_pabSrc;
+            uint8_t *pbEnd = &g_pabSrc[g_cbPages];
+            for (; pb != pbEnd; pb += 16)
+            {
+                char szTmp[17];
+                RTStrPrintf(szTmp, sizeof(szTmp), "aaaa%08Xzzzz", (uint32_t)(uintptr_t)pb);
+                memcpy(pb, szTmp, 16);
+            }
+        }
+    }
+
+    g_pabDecompr = (uint8_t *)RTMemAlloc(g_cbPages);
+    g_cbComprAlloc = RT_MAX(g_cbPages * 2, 256 * PAGE_SIZE);
+    g_pabCompr   = (uint8_t *)RTMemAlloc(g_cbComprAlloc);
+    if (!g_pabSrc || !g_pabDecompr || !g_pabCompr)
+        return Error("failed to allocate memory buffers (g_cPages=%#x)\n", g_cPages);
+
+    /*
+     * Double loop compressing and uncompressing the data, where the outer does
+     * the specified number of iterations while the inner applies the different
+     * compression algorithms.
+     */
+    struct
+    {
+        /** The time spent decompressing. */
+        uint64_t    cNanoDecompr;
+        /** The time spent compressing. */
+        uint64_t    cNanoCompr;
+        /** The size of the compressed data. */
+        uint64_t    cbCompr;
+        /** First error. */
+        int         rc;
+        /** The compression style: block or stream. */
+        bool        fBlock;
+        /** Compression type.  */
+        RTZIPTYPE   enmType;
+        /** Compression level.  */
+        RTZIPLEVEL  enmLevel;
+        /** Method name. */
+        const char *pszName;
+    } aTests[] =
+    {
+        { 0, 0, 0, VINF_SUCCESS, false, RTZIPTYPE_STORE, RTZIPLEVEL_DEFAULT, "RTZip/Store"      },
+        { 0, 0, 0, VINF_SUCCESS, false, RTZIPTYPE_LZF,   RTZIPLEVEL_DEFAULT, "RTZip/LZF"        },
+/*      { 0, 0, 0, VINF_SUCCESS, false, RTZIPTYPE_ZLIB,  RTZIPLEVEL_DEFAULT, "RTZip/zlib"       }, - slow plus it randomly hits VERR_GENERAL_FAILURE atm. */
+        { 0, 0, 0, VINF_SUCCESS, true,  RTZIPTYPE_STORE, RTZIPLEVEL_DEFAULT, "RTZipBlock/Store" },
+        { 0, 0, 0, VINF_SUCCESS, true,  RTZIPTYPE_LZF,   RTZIPLEVEL_DEFAULT, "RTZipBlock/LZF"   },
+        { 0, 0, 0, VINF_SUCCESS, true,  RTZIPTYPE_LZJB,  RTZIPLEVEL_DEFAULT, "RTZipBlock/LZJB"  },
+        { 0, 0, 0, VINF_SUCCESS, true,  RTZIPTYPE_LZO,   RTZIPLEVEL_DEFAULT, "RTZipBlock/LZO"   },
+    };
+    RTPrintf("tstCompressionBenchmark: TESTING..");
+    for (uint32_t i = 0; i < cIterations; i++)
+    {
+        for (uint32_t j = 0; j < RT_ELEMENTS(aTests); j++)
+        {
+            if (RT_FAILURE(aTests[j].rc))
+                continue;
+            memset(g_pabCompr,   0xaa, g_cbComprAlloc);
+            memset(g_pabDecompr, 0xcc, g_cbPages);
+            g_cbCompr = 0;
+            g_offComprIn = 0;
+            RTPrintf("."); RTStrmFlush(g_pStdOut);
+
+            /*
+             * Compress it.
+             */
+            uint64_t NanoTS = RTTimeNanoTS();
+            if (aTests[j].fBlock)
+            {
+                size_t          cbLeft    = g_cbComprAlloc;
+                uint8_t const  *pbSrcPage = g_pabSrc;
+                uint8_t        *pbDstPage = g_pabCompr;
+                for (size_t iPage = 0; iPage < g_cPages; iPage += cPagesAtATime)
+                {
+                    AssertBreakStmt(cbLeft > PAGE_SIZE * 4, aTests[j].rc = rc = VERR_BUFFER_OVERFLOW);
+                    uint32_t *pcb = (uint32_t *)pbDstPage;
+                    pbDstPage    += sizeof(uint32_t);
+                    cbLeft       -= sizeof(uint32_t);
+                    size_t  cbSrc = RT_MIN(g_cPages - iPage, cPagesAtATime) * PAGE_SIZE;
+                    size_t  cbDst;
+                    rc = RTZipBlockCompress(aTests[j].enmType, aTests[j].enmLevel, 0 /*fFlags*/,
+                                            pbSrcPage, cbSrc,
+                                            pbDstPage, cbLeft, &cbDst);
+                    if (RT_FAILURE(rc))
+                    {
+                        Error("RTZipBlockCompress failed for '%s' (#%u): %Rrc\n", aTests[j].pszName, j, rc);
+                        aTests[j].rc = rc;
+                        break;
+                    }
+                    *pcb       = (uint32_t)cbDst;
+                    cbLeft    -= cbDst;
+                    pbDstPage += cbDst;
+                    pbSrcPage += cbSrc;
+                }
+                if (RT_FAILURE(rc))
+                    continue;
+                g_cbCompr = pbDstPage - g_pabCompr;
+            }
+            else
+            {
+                PRTZIPCOMP pZipComp;
+                rc = RTZipCompCreate(&pZipComp, NULL, ComprOutCallback, aTests[j].enmType, aTests[j].enmLevel);
+                if (RT_FAILURE(rc))
+                {
+                    Error("Failed to create the compressor for '%s' (#%u): %Rrc\n", aTests[j].pszName, j, rc);
+                    aTests[j].rc = rc;
+                    continue;
+                }
+
+                uint8_t const  *pbSrcPage = g_pabSrc;
+                for (size_t iPage = 0; iPage < g_cPages; iPage += cPagesAtATime)
+                {
+                    size_t cb = RT_MIN(g_cPages - iPage, cPagesAtATime) * PAGE_SIZE;
+                    rc = RTZipCompress(pZipComp, pbSrcPage, cb);
+                    if (RT_FAILURE(rc))
+                    {
+                        Error("RTZipCompress failed for '%s' (#%u): %Rrc\n", aTests[j].pszName, j, rc);
+                        aTests[j].rc = rc;
+                        break;
+                    }
+                    pbSrcPage += cb;
+                }
+                if (RT_FAILURE(rc))
+                    continue;
+                rc = RTZipCompFinish(pZipComp);
+                if (RT_FAILURE(rc))
+                {
+                    Error("RTZipCompFinish failed for '%s' (#%u): %Rrc\n", aTests[j].pszName, j, rc);
+                    aTests[j].rc = rc;
+                    break;
+                }
+                RTZipCompDestroy(pZipComp);
+            }
+            NanoTS = RTTimeNanoTS() - NanoTS;
+            aTests[j].cbCompr    += g_cbCompr;
+            aTests[j].cNanoCompr += NanoTS;
+
+            /*
+             * Decompress it.
+             */
+            NanoTS = RTTimeNanoTS();
+            if (aTests[j].fBlock)
+            {
+                uint8_t const  *pbSrcPage = g_pabCompr;
+                size_t          cbLeft    = g_cbCompr;
+                uint8_t        *pbDstPage = g_pabDecompr;
+                for (size_t iPage = 0; iPage < g_cPages; iPage += cPagesAtATime)
+                {
+                    size_t   cbDst = RT_MIN(g_cPages - iPage, cPagesAtATime) * PAGE_SIZE;
+                    size_t   cbSrc = *(uint32_t *)pbSrcPage;
+                    pbSrcPage     += sizeof(uint32_t);
+                    cbLeft        -= sizeof(uint32_t);
+                    rc = RTZipBlockDecompress(aTests[j].enmType, 0 /*fFlags*/,
+                                              pbSrcPage, cbSrc, &cbSrc,
+                                              pbDstPage, cbDst, &cbDst);
+                    if (RT_FAILURE(rc))
+                    {
+                        Error("RTZipBlockDecompress failed for '%s' (#%u): %Rrc\n", aTests[j].pszName, j, rc);
+                        aTests[j].rc = rc;
+                        break;
+                    }
+                    pbDstPage += cbDst;
+                    cbLeft    -= cbSrc;
+                    pbSrcPage += cbSrc;
+                }
+                if (RT_FAILURE(rc))
+                    continue;
+            }
+            else
+            {
+                PRTZIPDECOMP pZipDecomp;
+                rc = RTZipDecompCreate(&pZipDecomp, NULL, DecomprInCallback);
+                if (RT_FAILURE(rc))
+                {
+                    Error("Failed to create the decompressor for '%s' (#%u): %Rrc\n", aTests[j].pszName, j, rc);
+                    aTests[j].rc = rc;
+                    continue;
+                }
+
+                uint8_t *pbDstPage = g_pabDecompr;
+                for (size_t iPage = 0; iPage < g_cPages; iPage += cPagesAtATime)
+                {
+                    size_t cb = RT_MIN(g_cPages - iPage, cPagesAtATime) * PAGE_SIZE;
+                    rc = RTZipDecompress(pZipDecomp, pbDstPage, cb, NULL);
+                    if (RT_FAILURE(rc))
+                    {
+                        Error("RTZipDecompress failed for '%s' (#%u): %Rrc\n", aTests[j].pszName, j, rc);
+                        aTests[j].rc = rc;
+                        break;
+                    }
+                    pbDstPage += cb;
+                }
+                RTZipDecompDestroy(pZipDecomp);
+                if (RT_FAILURE(rc))
+                    continue;
+            }
+            NanoTS = RTTimeNanoTS() - NanoTS;
+            aTests[j].cNanoDecompr += NanoTS;
+
+            if (memcmp(g_pabDecompr, g_pabSrc, g_cbPages))
+            {
+                Error("The compressed data doesn't match the source for '%s' (%#u)\n", aTests[j].pszName, j);
+                aTests[j].rc = VERR_BAD_EXE_FORMAT;
+                continue;
+            }
+        }
+    }
+    if (RT_SUCCESS(rc))
+        RTPrintf("\n");
+
+    /*
+     * Report the results.
+     */
+    rc = 0;
+    RTPrintf("tstCompressionBenchmark: BEGIN RESULTS\n");
+    RTPrintf("%-20s           Compression                                             Decompression\n", "");
+    RTPrintf("%-20s        In             Out      Ratio         Size                In             Out\n", "Method");
+    RTPrintf("%.20s-----------------------------------------------------------------------------------------\n", "---------------------------------------------");
+    for (uint32_t j = 0; j < RT_ELEMENTS(aTests); j++)
+    {
+        if (RT_SUCCESS(aTests[j].rc))
+        {
+            unsigned uComprSpeedIn    = (unsigned)(cbTotalKB         / (long double)aTests[j].cNanoCompr   * 1000000000.0);
+            unsigned uComprSpeedOut   = (unsigned)(aTests[j].cbCompr / (long double)aTests[j].cNanoCompr   * 1000000000.0 / 1024);
+            unsigned uRatio           = (unsigned)(aTests[j].cbCompr / cIterations * 100 / g_cbPages);
+            unsigned uDecomprSpeedIn  = (unsigned)(aTests[j].cbCompr / (long double)aTests[j].cNanoDecompr * 1000000000.0 / 1024);
+            unsigned uDecomprSpeedOut = (unsigned)(cbTotalKB         / (long double)aTests[j].cNanoDecompr * 1000000000.0);
+            RTPrintf("%-20s %'9u KB/s  %'9u KB/s  %3u%%  %'11llu bytes   %'9u KB/s  %'9u KB/s",
+                     aTests[j].pszName,
+                     uComprSpeedIn,   uComprSpeedOut, uRatio, aTests[j].cbCompr / cIterations,
+                     uDecomprSpeedIn, uDecomprSpeedOut);
+#if 0
+            RTPrintf("  [%'14llu / %'14llu ns]\n",
+                     aTests[j].cNanoCompr / cIterations,
+                     aTests[j].cNanoDecompr / cIterations);
+#else
+            RTPrintf("\n");
+#endif
+        }
+        else
+        {
+            RTPrintf("%-20s: %Rrc\n", aTests[j].pszName, aTests[j].rc);
+            rc = 1;
+        }
+    }
+    if (pszPageFile)
+        RTPrintf("Input: %'10zu pages from '%s' starting at offset %'lld (%#llx)\n"
+                 "                                                           %'11zu bytes\n",
+                 g_cPages, pszPageFile, offPageFile, offPageFile, g_cbPages);
+    else
+        RTPrintf("Input: %'10zu pages of generated rubbish               %'11zu bytes\n",
+                 g_cPages, g_cbPages);
+
+    /*
+     * Count zero pages in the data set.
+     */
+    size_t cZeroPages = 0;
+    for (size_t iPage = 0; iPage < g_cPages; iPage++)
+    {
+        if (ASMMemIsZeroPage(&g_pabSrc[iPage * PAGE_SIZE]))
+            cZeroPages++;
+    }
+    RTPrintf("       %'10zu zero pages (%u %%)\n", cZeroPages, cZeroPages * 100 / g_cPages);
+
+    /*
+     * A little extension to the test, benchmark relevant CRCs.
+     */
+    RTPrintf("\n"
+             "tstCompressionBenchmark: Hash/CRC - All In One\n");
+    tstBenchmarkCRCsAllInOne(g_pabSrc, g_cbPages);
+
+    RTPrintf("\n"
+             "tstCompressionBenchmark: Hash/CRC - Page by Page\n");
+    tstBenchmarkCRCsPageByPage(g_pabSrc, g_cbPages);
+
+    RTPrintf("\n"
+             "tstCompressionBenchmark: Hash/CRC - Zero Page Digest\n");
+    static uint8_t s_abZeroPg[PAGE_SIZE];
+    RT_ZERO(s_abZeroPg);
+    tstBenchmarkCRCsAllInOne(s_abZeroPg, PAGE_SIZE);
+
+    RTPrintf("\n"
+             "tstCompressionBenchmark: Hash/CRC - Zero Half Page Digest\n");
+    tstBenchmarkCRCsAllInOne(s_abZeroPg, PAGE_SIZE / 2);
+
+    RTPrintf("tstCompressionBenchmark: END RESULTS\n");
+
+    return rc;
+}
+
diff --git a/src/VBox/VMM/testcase/tstGlobalConfig.cpp b/src/VBox/VMM/testcase/tstGlobalConfig.cpp
new file mode 100644
index 00000000..efddf841
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstGlobalConfig.cpp
@@ -0,0 +1,138 @@
+/* $Id: tstGlobalConfig.cpp $ */
+/** @file
+ * Ring-3 Management program for the GCFGM mock-up.
+ */
+
+/*
+ * Copyright (C) 2007-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/vmm/vmm.h>
+#include <iprt/errcore.h>
+#include <iprt/assert.h>
+#include <iprt/initterm.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+
+
+/**
+ * Prints the usage and returns 1.
+ * @return 1
+ */
+static int Usage(void)
+{
+    RTPrintf("usage: tstGlobalConfig <value-name> [new value]\n");
+    return 1;
+}
+
+
+/**
+ *  Entry point.
+ */
+extern "C" DECLEXPORT(int) TrustedMain(int argc, char **argv, char **envp)
+{
+    RT_NOREF1(envp);
+    RTR3InitExe(argc, &argv, 0);
+
+    /*
+     * Parse args, building the request as we do so.
+     */
+    if (argc <= 1)
+        return Usage();
+    if (argc > 3)
+    {
+        RTPrintf("syntax error: too many arguments\n");
+        Usage();
+        return 1;
+    }
+
+    VMMR0OPERATION enmOp = VMMR0_DO_GCFGM_QUERY_VALUE;
+    GCFGMVALUEREQ Req;
+    memset(&Req, 0, sizeof(Req));
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq = sizeof(Req);
+
+    /* arg[1] = szName */
+    size_t cch = strlen(argv[1]);
+    if (cch < 2 || argv[1][0] != '/')
+    {
+        RTPrintf("syntax error: malformed name '%s'\n", argv[1]);
+        return 1;
+    }
+    if (cch >= sizeof(Req.szName))
+    {
+        RTPrintf("syntax error: the name '%s' is too long. (max %zu chars)\n", argv[1], sizeof(Req.szName) - 1);
+        return 1;
+    }
+    memcpy(&Req.szName[0], argv[1], cch + 1);
+
+    /* argv[2] = u64SetValue; optional */
+    if (argc == 3)
+    {
+        char *pszNext = NULL;
+        int rc = RTStrToUInt64Ex(argv[2], &pszNext, 0, &Req.u64Value);
+        if (RT_FAILURE(rc) || *pszNext)
+        {
+            RTPrintf("syntax error: '%s' didn't convert successfully to a number. (%Rrc,'%s')\n", argv[2], rc, pszNext);
+            return 1;
+        }
+        enmOp = VMMR0_DO_GCFGM_SET_VALUE;
+    }
+
+    /*
+     * Open the session, load ring-0 and issue the request.
+     */
+    PSUPDRVSESSION pSession;
+    int rc = SUPR3Init(&pSession);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("tstGlobalConfig: SUPR3Init -> %Rrc\n", rc);
+        return 1;
+    }
+
+    rc = SUPR3LoadVMM("./VMMR0.r0", NULL /*pErrInfo*/);
+    if (RT_SUCCESS(rc))
+    {
+        Req.pSession = pSession;
+        rc = SUPR3CallVMMR0Ex(NIL_RTR0PTR, NIL_VMCPUID, enmOp, 0, &Req.Hdr);
+        if (RT_SUCCESS(rc))
+        {
+            if (enmOp == VMMR0_DO_GCFGM_QUERY_VALUE)
+                RTPrintf("%s = %RU64 (%#RX64)\n", Req.szName, Req.u64Value, Req.u64Value);
+            else
+                RTPrintf("Successfully set %s = %RU64 (%#RX64)\n", Req.szName, Req.u64Value, Req.u64Value);
+        }
+        else if (enmOp == VMMR0_DO_GCFGM_QUERY_VALUE)
+            RTPrintf("error: Failed to query '%s', rc=%Rrc\n", Req.szName, rc);
+        else
+            RTPrintf("error: Failed to set '%s' to %RU64, rc=%Rrc\n", Req.szName, Req.u64Value, rc);
+
+    }
+    SUPR3Term(false /*fForced*/);
+
+    return RT_FAILURE(rc) ? 1 : 0;
+}
+
+
+#if !defined(VBOX_WITH_HARDENING) || !defined(RT_OS_WINDOWS)
+/**
+ * Main entry point.
+ */
+int main(int argc, char **argv, char **envp)
+{
+    return TrustedMain(argc, argv, envp);
+}
+#endif
+
diff --git a/src/VBox/VMM/testcase/tstHelp.h b/src/VBox/VMM/testcase/tstHelp.h
new file mode 100644
index 00000000..618c5a05
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstHelp.h
@@ -0,0 +1,169 @@
+/* $Id: tstHelp.h $ */
+/** @file
+ * VMM testcase - Helper stuff.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_testcase_tstHelp_h
+#define VMM_INCLUDED_SRC_testcase_tstHelp_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/cdefs.h>
+#include <VBox/vmm/cpum.h>
+
+RT_C_DECLS_BEGIN
+void tstDumpCtx(PCPUMCTX pCtx, const char *pszComment);
+RT_C_DECLS_END
+
+
+/**
+ * Checks the offset of a data member.
+ * @param   type    Type.
+ * @param   off     Correct offset.
+ * @param   m       Member name.
+ */
+#define CHECK_OFF(type, off, m) \
+    do { \
+        if (off != RT_OFFSETOF(type, m)) \
+        { \
+            printf("error! %#010x %s  Off by %d!! (expected off=%#x)\n", \
+                   RT_OFFSETOF(type, m), #type "." #m, off - RT_OFFSETOF(type, m), (int)off); \
+            rc++; \
+        } \
+        /*else */ \
+            /*printf("%#08x %s\n", RT_OFFSETOF(type, m), #m);*/ \
+    } while (0)
+
+/**
+ * Checks the size of type.
+ * @param   type    Type.
+ * @param   size    Correct size.
+ */
+#define CHECK_SIZE(type, size) \
+    do { \
+        if (size != sizeof(type)) \
+        { \
+            printf("error! sizeof(%s): %#x (%d)  Off by %d!! (expected %#x)\n", \
+                   #type, (int)sizeof(type), (int)sizeof(type), (int)sizeof(type) - (int)size, (int)size); \
+            rc++; \
+        } \
+        else \
+            printf("info: sizeof(%s): %#x (%d)\n", #type, (int)sizeof(type), (int)sizeof(type)); \
+    } while (0)
+
+/**
+ * Checks the alignment of a struct member.
+ */
+#define CHECK_MEMBER_ALIGNMENT(strct, member, align) \
+    do \
+    { \
+        if (RT_UOFFSETOF(strct, member) & ((align) - 1) ) \
+        { \
+            printf("error! %s::%s offset=%#x (%u) expected alignment %#x, meaning %#x (%u) off\n", \
+                   #strct, #member, \
+                   (unsigned)RT_OFFSETOF(strct, member), \
+                   (unsigned)RT_OFFSETOF(strct, member), \
+                   (unsigned)(align), \
+                   (unsigned)(((align) - RT_OFFSETOF(strct, member)) & ((align) - 1)), \
+                   (unsigned)(((align) - RT_OFFSETOF(strct, member)) & ((align) - 1)) ); \
+            rc++; \
+        } \
+    } while (0)
+
+/**
+ * Checks that the size of a type is aligned correctly.
+ */
+#define CHECK_SIZE_ALIGNMENT(type, align) \
+    do { \
+        if (RT_ALIGN_Z(sizeof(type), (align)) != sizeof(type)) \
+        { \
+            printf("error! %s size=%#x (%u), align=%#x %#x (%u) bytes off\n", \
+                   #type, \
+                   (unsigned)sizeof(type), \
+                   (unsigned)sizeof(type), \
+                   (align), \
+                   (unsigned)RT_ALIGN_Z(sizeof(type), align) - (unsigned)sizeof(type), \
+                   (unsigned)RT_ALIGN_Z(sizeof(type), align) - (unsigned)sizeof(type)); \
+            rc++; \
+        } \
+    } while (0)
+
+/**
+ * Checks that a internal struct padding is big enough.
+ */
+#define CHECK_PADDING(strct, member, align) \
+    do \
+    { \
+        strct *p = NULL; NOREF(p); \
+        if (sizeof(p->member.s) > sizeof(p->member.padding)) \
+        { \
+            printf("error! padding of %s::%s is too small, padding=%d struct=%d correct=%d\n", #strct, #member, \
+                   (int)sizeof(p->member.padding), (int)sizeof(p->member.s), (int)RT_ALIGN_Z(sizeof(p->member.s), (align))); \
+            rc++; \
+        } \
+        else if (RT_ALIGN_Z(sizeof(p->member.padding), (align)) != sizeof(p->member.padding)) \
+        { \
+            printf("error! padding of %s::%s is misaligned, padding=%d correct=%d\n", #strct, #member, \
+                   (int)sizeof(p->member.padding), (int)RT_ALIGN_Z(sizeof(p->member.s), (align))); \
+            rc++; \
+        } \
+    } while (0)
+
+/**
+ * Checks that a internal struct padding is big enough.
+ */
+#define CHECK_PADDING2(strct) \
+    do \
+    { \
+        strct *p = NULL; NOREF(p); \
+        if (sizeof(p->s) > sizeof(p->padding)) \
+        { \
+            printf("error! padding of %s is too small, padding=%d struct=%d correct=%d\n", #strct, \
+                   (int)sizeof(p->padding), (int)sizeof(p->s), (int)RT_ALIGN_Z(sizeof(p->s), 64)); \
+            rc++; \
+        } \
+    } while (0)
+
+/**
+ * Checks that a internal struct padding is big enough.
+ */
+#define CHECK_PADDING3(strct, member, pad_member) \
+    do \
+    { \
+        strct *p = NULL; NOREF(p); \
+        if (sizeof(p->member) > sizeof(p->pad_member)) \
+        { \
+            printf("error! padding of %s::%s is too small, padding=%d struct=%d\n", #strct, #member, \
+                   (int)sizeof(p->pad_member), (int)sizeof(p->member)); \
+            rc++; \
+        } \
+    } while (0)
+
+/**
+ * Checks that an expression is true.
+ */
+#define CHECK_EXPR(expr) \
+    do \
+    { \
+        if (!(expr)) \
+        { \
+            printf("error! '%s' failed! (line %d)\n", #expr, __LINE__); \
+            rc++; \
+        } \
+    } while (0)
+
+
+#endif /* !VMM_INCLUDED_SRC_testcase_tstHelp_h */
diff --git a/src/VBox/VMM/testcase/tstIEMCheckMc.cpp b/src/VBox/VMM/testcase/tstIEMCheckMc.cpp
new file mode 100644
index 00000000..71c4faa6
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstIEMCheckMc.cpp
@@ -0,0 +1,769 @@
+/* $Id: tstIEMCheckMc.cpp $ */
+/** @file
+ * IEM Testcase - Check the "Microcode".
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <iprt/assert.h>
+#include <iprt/rand.h>
+#include <iprt/test.h>
+
+#include <VBox/types.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#define IN_TSTVMSTRUCT 1
+#include "../include/IEMInternal.h"
+#include <VBox/vmm/vm.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+bool volatile       g_fRandom;
+uint8_t volatile    g_bRandom;
+RTUINT128U          g_u128Zero;
+
+
+/** For hacks.  */
+#define TST_IEM_CHECK_MC
+
+#define CHK_TYPE(a_ExpectedType, a_Param) \
+    do { a_ExpectedType const * pCheckType = &(a_Param); NOREF(pCheckType); } while (0)
+#define CHK_PTYPE(a_ExpectedType, a_Param) \
+    do { a_ExpectedType pCheckType = (a_Param); NOREF(pCheckType); } while (0)
+
+#define CHK_CONST(a_ExpectedType, a_Const) \
+    do { \
+        AssertCompile(((a_Const) >> 1) == ((a_Const) >> 1)); \
+        AssertCompile((a_ExpectedType)(a_Const) == (a_Const)); \
+    } while (0)
+
+#define CHK_SINGLE_BIT(a_ExpectedType, a_fBitMask) \
+    do { \
+        CHK_CONST(a_ExpectedType, a_fBitMask); \
+        AssertCompile(RT_IS_POWER_OF_TWO(a_fBitMask)); \
+    } while (0)
+
+#define CHK_GCPTR(a_EffAddr) \
+    CHK_TYPE(RTGCPTR, a_EffAddr)
+
+#define CHK_SEG_IDX(a_iSeg) \
+    do { \
+        uint8_t iMySeg = (a_iSeg); NOREF(iMySeg); /** @todo const or variable. grr. */ \
+    } while (0)
+
+#define CHK_CALL_ARG(a_Name, a_iArg) \
+    do { RT_CONCAT3(iArgCheck_,a_iArg,a_Name) = 1; } while (0)
+
+
+/** @name Other stubs.
+ * @{   */
+
+typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPU pVCpu);
+#define FNIEMOP_DEF(a_Name) \
+    static VBOXSTRICTRC a_Name(PVMCPU pVCpu) RT_NO_THROW_DEF
+#define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
+    static VBOXSTRICTRC a_Name(PVMCPU pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
+#define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
+    static VBOXSTRICTRC a_Name(PVMCPU pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
+
+typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPU pVCpu, uint8_t bRm);
+#define FNIEMOPRM_DEF(a_Name) \
+    static VBOXSTRICTRC a_Name(PVMCPU pVCpu, uint8_t bRm) RT_NO_THROW_DEF
+
+#define IEM_NOT_REACHED_DEFAULT_CASE_RET()                  default: return VERR_IPE_NOT_REACHED_DEFAULT_CASE
+#define IEM_RETURN_ASPECT_NOT_IMPLEMENTED()                 return IEM_RETURN_ASPECT_NOT_IMPLEMENTED
+#define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) return IEM_RETURN_ASPECT_NOT_IMPLEMENTED
+
+
+#define IEM_OPCODE_GET_NEXT_RM(a_pu8)                       do { *(a_pu8)  = g_bRandom; CHK_PTYPE(uint8_t  *, a_pu8);  } while (0)
+#define IEM_OPCODE_GET_NEXT_U8(a_pu8)                       do { *(a_pu8)  = g_bRandom; CHK_PTYPE(uint8_t  *, a_pu8);  } while (0)
+#define IEM_OPCODE_GET_NEXT_S8(a_pi8)                       do { *(a_pi8)  = g_bRandom; CHK_PTYPE(int8_t   *, a_pi8);  } while (0)
+#define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16)               do { *(a_pu16) = g_bRandom; CHK_PTYPE(uint16_t *, a_pu16); } while (0)
+#define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32)               do { *(a_pu32) = g_bRandom; CHK_PTYPE(uint32_t *, a_pu32); } while (0)
+#define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64)               do { *(a_pu64) = g_bRandom; CHK_PTYPE(uint64_t *, a_pu64); } while (0)
+#define IEM_OPCODE_GET_NEXT_U16(a_pu16)                     do { *(a_pu16) = g_bRandom; CHK_PTYPE(uint16_t *, a_pu16); } while (0)
+#define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32)              do { *(a_pu32) = g_bRandom; CHK_PTYPE(uint32_t *, a_pu32); } while (0)
+#define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64)              do { *(a_pu64) = g_bRandom; CHK_PTYPE(uint64_t *, a_pu64); } while (0)
+#define IEM_OPCODE_GET_NEXT_S16(a_pi16)                     do { *(a_pi16) = g_bRandom; CHK_PTYPE(int16_t  *, a_pi16); } while (0)
+#define IEM_OPCODE_GET_NEXT_U32(a_pu32)                     do { *(a_pu32) = g_bRandom; CHK_PTYPE(uint32_t *, a_pu32); } while (0)
+#define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64)              do { *(a_pu64) = g_bRandom; CHK_PTYPE(uint64_t *, a_pu64); } while (0)
+#define IEM_OPCODE_GET_NEXT_S32(a_pi32)                     do { *(a_pi32) = g_bRandom; CHK_PTYPE(int32_t  *, a_pi32); } while (0)
+#define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64)              do { *(a_pu64) = g_bRandom; CHK_PTYPE(uint64_t *, a_pu64); } while (0)
+#define IEM_OPCODE_GET_NEXT_U64(a_pu64)                     do { *(a_pu64) = g_bRandom; CHK_PTYPE(uint64_t *, a_pu64); } while (0)
+#define IEMOP_HLP_MIN_186()                                 do { } while (0)
+#define IEMOP_HLP_MIN_286()                                 do { } while (0)
+#define IEMOP_HLP_MIN_386()                                 do { } while (0)
+#define IEMOP_HLP_MIN_386_EX(a_fTrue)                       do { } while (0)
+#define IEMOP_HLP_MIN_486()                                 do { } while (0)
+#define IEMOP_HLP_MIN_586()                                 do { } while (0)
+#define IEMOP_HLP_MIN_686()                                 do { } while (0)
+#define IEMOP_HLP_NO_REAL_OR_V86_MODE()                     do { } while (0)
+#define IEMOP_HLP_NO_64BIT()                                do { } while (0)
+#define IEMOP_HLP_ONLY_64BIT()                              do { } while (0)
+#define IEMOP_HLP_64BIT_OP_SIZE()                           do { } while (0)
+#define IEMOP_HLP_DEFAULT_64BIT_OP_SIZE()                   do { } while (0)
+#define IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE(a_szPrf)      do { } while (0)
+#define IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX()            do { } while (0)
+#define IEMOP_HLP_DONE_VEX_DECODING()                       do { } while (0)
+#define IEMOP_HLP_DONE_VEX_DECODING_L0()                    do { } while (0)
+#define IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV()               do { } while (0)
+#define IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV()        do { } while (0)
+#define IEMOP_HLP_DONE_DECODING_NO_LOCK_REPZ_OR_REPNZ_PREFIXES()                                    do { } while (0)
+#define IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES()                                 do { } while (0)
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+# define IEMOP_HLP_VMX_INSTR(a_szInstr, a_InsDiagPrefix)                                            do { } while (0)
+# define IEMOP_HLP_IN_VMX_OPERATION(a_szInstr, a_InsDiagPrefix)                                     do { } while (0)
+#endif
+
+
+#define IEMOP_HLP_DONE_DECODING()                           do { } while (0)
+
+#define IEMOP_HLP_DECODED_NL_1(a_uDisOpNo, a_fIemOpFlags, a_uDisParam0, a_fDisOpType)               do { } while (0)
+#define IEMOP_HLP_DECODED_NL_2(a_uDisOpNo, a_fIemOpFlags, a_uDisParam0, a_uDisParam1, a_fDisOpType) do { } while (0)
+#define IEMOP_RAISE_DIVIDE_ERROR()                          VERR_TRPM_ACTIVE_TRAP
+#define IEMOP_RAISE_INVALID_OPCODE()                        VERR_TRPM_ACTIVE_TRAP
+#define IEMOP_RAISE_INVALID_LOCK_PREFIX()                   VERR_TRPM_ACTIVE_TRAP
+#define IEMOP_MNEMONIC(a_Stats, a_szMnemonic)               do { } while (0)
+#define IEMOP_MNEMONIC0EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) do { } while (0)
+#define IEMOP_MNEMONIC1EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) do { } while (0)
+#define IEMOP_MNEMONIC2EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) do { } while (0)
+#define IEMOP_MNEMONIC3EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) do { } while (0)
+#define IEMOP_MNEMONIC4EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) do { } while (0)
+#define IEMOP_MNEMONIC0(a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints)                         do { } while (0)
+#define IEMOP_MNEMONIC1(a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints)                  do { } while (0)
+#define IEMOP_MNEMONIC2(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints)           do { } while (0)
+#define IEMOP_MNEMONIC3(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints)    do { } while (0)
+#define IEMOP_MNEMONIC4(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints)    do { } while (0)
+#define IEMOP_BITCH_ABOUT_STUB()                            do { } while (0)
+#define FNIEMOP_STUB(a_Name) \
+    FNIEMOP_DEF(a_Name) { return VERR_NOT_IMPLEMENTED; } \
+    typedef int ignore_semicolon
+#define FNIEMOP_STUB_1(a_Name, a_Type0, a_Name0) \
+    FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) { return VERR_NOT_IMPLEMENTED; } \
+    typedef int ignore_semicolon
+
+#define FNIEMOP_UD_STUB(a_Name) \
+    FNIEMOP_DEF(a_Name) { return IEMOP_RAISE_INVALID_OPCODE(); } \
+    typedef int ignore_semicolon
+#define FNIEMOP_UD_STUB_1(a_Name, a_Type0, a_Name0) \
+    FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) { return IEMOP_RAISE_INVALID_OPCODE(); } \
+    typedef int ignore_semicolon
+
+
+#define FNIEMOP_CALL(a_pfn)                                 (a_pfn)(pVCpu)
+#define FNIEMOP_CALL_1(a_pfn, a0)                           (a_pfn)(pVCpu, a0)
+#define FNIEMOP_CALL_2(a_pfn, a0, a1)                       (a_pfn)(pVCpu, a0, a1)
+
+#define IEM_IS_REAL_OR_V86_MODE(a_pVCpu)                    (g_fRandom)
+#define IEM_IS_LONG_MODE(a_pVCpu)                           (g_fRandom)
+#define IEM_IS_REAL_MODE(a_pVCpu)                           (g_fRandom)
+#define IEM_IS_GUEST_CPU_AMD(a_pVCpu)                       (g_fRandom)
+#define IEM_IS_GUEST_CPU_INTEL(a_pVCpu)                     (g_fRandom)
+#define IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)                 ((PCCPUMFEATURES)(uintptr_t)42)
+#define IEM_GET_HOST_CPU_FEATURES(a_pVCpu)                  ((PCCPUMFEATURES)(uintptr_t)88)
+
+#define iemRecalEffOpSize(a_pVCpu)                          do { } while (0)
+
+IEMOPBINSIZES g_iemAImpl_add;
+IEMOPBINSIZES g_iemAImpl_adc;
+IEMOPBINSIZES g_iemAImpl_sub;
+IEMOPBINSIZES g_iemAImpl_sbb;
+IEMOPBINSIZES g_iemAImpl_or;
+IEMOPBINSIZES g_iemAImpl_xor;
+IEMOPBINSIZES g_iemAImpl_and;
+IEMOPBINSIZES g_iemAImpl_cmp;
+IEMOPBINSIZES g_iemAImpl_test;
+IEMOPBINSIZES g_iemAImpl_bt;
+IEMOPBINSIZES g_iemAImpl_btc;
+IEMOPBINSIZES g_iemAImpl_btr;
+IEMOPBINSIZES g_iemAImpl_bts;
+IEMOPBINSIZES g_iemAImpl_bsf;
+IEMOPBINSIZES g_iemAImpl_bsr;
+IEMOPBINSIZES g_iemAImpl_imul_two;
+PCIEMOPBINSIZES g_apIemImplGrp1[8];
+IEMOPUNARYSIZES g_iemAImpl_inc;
+IEMOPUNARYSIZES g_iemAImpl_dec;
+IEMOPUNARYSIZES g_iemAImpl_neg;
+IEMOPUNARYSIZES g_iemAImpl_not;
+IEMOPSHIFTSIZES g_iemAImpl_rol;
+IEMOPSHIFTSIZES g_iemAImpl_ror;
+IEMOPSHIFTSIZES g_iemAImpl_rcl;
+IEMOPSHIFTSIZES g_iemAImpl_rcr;
+IEMOPSHIFTSIZES g_iemAImpl_shl;
+IEMOPSHIFTSIZES g_iemAImpl_shr;
+IEMOPSHIFTSIZES g_iemAImpl_sar;
+IEMOPMULDIVSIZES g_iemAImpl_mul;
+IEMOPMULDIVSIZES g_iemAImpl_imul;
+IEMOPMULDIVSIZES g_iemAImpl_div;
+IEMOPMULDIVSIZES g_iemAImpl_idiv;
+IEMOPSHIFTDBLSIZES g_iemAImpl_shld;
+IEMOPSHIFTDBLSIZES g_iemAImpl_shrd;
+IEMOPMEDIAF1L1 g_iemAImpl_punpcklbw;
+IEMOPMEDIAF1L1 g_iemAImpl_punpcklwd;
+IEMOPMEDIAF1L1 g_iemAImpl_punpckldq;
+IEMOPMEDIAF1L1 g_iemAImpl_punpcklqdq;
+IEMOPMEDIAF1H1 g_iemAImpl_punpckhbw;
+IEMOPMEDIAF1H1 g_iemAImpl_punpckhwd;
+IEMOPMEDIAF1H1 g_iemAImpl_punpckhdq;
+IEMOPMEDIAF1H1 g_iemAImpl_punpckhqdq;
+IEMOPMEDIAF2 g_iemAImpl_pxor;
+IEMOPMEDIAF2 g_iemAImpl_pcmpeqb;
+IEMOPMEDIAF2 g_iemAImpl_pcmpeqw;
+IEMOPMEDIAF2 g_iemAImpl_pcmpeqd;
+
+
+#define iemAImpl_idiv_u8    ((PFNIEMAIMPLMULDIVU8)0)
+#define iemAImpl_div_u8     ((PFNIEMAIMPLMULDIVU8)0)
+#define iemAImpl_imul_u8    ((PFNIEMAIMPLMULDIVU8)0)
+#define iemAImpl_mul_u8     ((PFNIEMAIMPLMULDIVU8)0)
+
+#define iemAImpl_fpu_r32_to_r80         NULL
+#define iemAImpl_fcom_r80_by_r32        NULL
+#define iemAImpl_fadd_r80_by_r32        NULL
+#define iemAImpl_fmul_r80_by_r32        NULL
+#define iemAImpl_fsub_r80_by_r32        NULL
+#define iemAImpl_fsubr_r80_by_r32       NULL
+#define iemAImpl_fdiv_r80_by_r32        NULL
+#define iemAImpl_fdivr_r80_by_r32       NULL
+
+#define iemAImpl_fpu_r64_to_r80         NULL
+#define iemAImpl_fadd_r80_by_r64        NULL
+#define iemAImpl_fmul_r80_by_r64        NULL
+#define iemAImpl_fcom_r80_by_r64        NULL
+#define iemAImpl_fsub_r80_by_r64        NULL
+#define iemAImpl_fsubr_r80_by_r64       NULL
+#define iemAImpl_fdiv_r80_by_r64        NULL
+#define iemAImpl_fdivr_r80_by_r64       NULL
+
+#define iemAImpl_fadd_r80_by_r80        NULL
+#define iemAImpl_fmul_r80_by_r80        NULL
+#define iemAImpl_fsub_r80_by_r80        NULL
+#define iemAImpl_fsubr_r80_by_r80       NULL
+#define iemAImpl_fdiv_r80_by_r80        NULL
+#define iemAImpl_fdivr_r80_by_r80       NULL
+#define iemAImpl_fprem_r80_by_r80       NULL
+#define iemAImpl_fprem1_r80_by_r80      NULL
+#define iemAImpl_fscale_r80_by_r80      NULL
+
+#define iemAImpl_fpatan_r80_by_r80      NULL
+#define iemAImpl_fyl2x_r80_by_r80       NULL
+#define iemAImpl_fyl2xp1_r80_by_r80     NULL
+
+#define iemAImpl_fcom_r80_by_r80        NULL
+#define iemAImpl_fucom_r80_by_r80       NULL
+#define iemAImpl_fabs_r80               NULL
+#define iemAImpl_fchs_r80               NULL
+#define iemAImpl_ftst_r80               NULL
+#define iemAImpl_fxam_r80               NULL
+#define iemAImpl_f2xm1_r80              NULL
+#define iemAImpl_fsqrt_r80              NULL
+#define iemAImpl_frndint_r80            NULL
+#define iemAImpl_fsin_r80               NULL
+#define iemAImpl_fcos_r80               NULL
+
+#define iemAImpl_fld1                   NULL
+#define iemAImpl_fldl2t                 NULL
+#define iemAImpl_fldl2e                 NULL
+#define iemAImpl_fldpi                  NULL
+#define iemAImpl_fldlg2                 NULL
+#define iemAImpl_fldln2                 NULL
+#define iemAImpl_fldz                   NULL
+
+#define iemAImpl_fptan_r80_r80          NULL
+#define iemAImpl_fxtract_r80_r80        NULL
+#define iemAImpl_fsincos_r80_r80        NULL
+
+#define iemAImpl_fiadd_r80_by_i16       NULL
+#define iemAImpl_fimul_r80_by_i16       NULL
+#define iemAImpl_fisub_r80_by_i16       NULL
+#define iemAImpl_fisubr_r80_by_i16      NULL
+#define iemAImpl_fidiv_r80_by_i16       NULL
+#define iemAImpl_fidivr_r80_by_i16      NULL
+
+#define iemAImpl_fiadd_r80_by_i32       NULL
+#define iemAImpl_fimul_r80_by_i32       NULL
+#define iemAImpl_fisub_r80_by_i32       NULL
+#define iemAImpl_fisubr_r80_by_i32      NULL
+#define iemAImpl_fidiv_r80_by_i32       NULL
+#define iemAImpl_fidivr_r80_by_i32      NULL
+
+#define iemCImpl_callf                  NULL
+#define iemCImpl_FarJmp                 NULL
+
+#define iemAImpl_pshufhw                NULL
+#define iemAImpl_pshuflw                NULL
+#define iemAImpl_pshufd                 NULL
+
+/** @}  */
+
+
+#define IEM_REPEAT_0(a_Callback, a_User)    do { } while (0)
+#define IEM_REPEAT_1(a_Callback, a_User)                                      a_Callback##_CALLBACK(0, a_User)
+#define IEM_REPEAT_2(a_Callback, a_User)    IEM_REPEAT_1(a_Callback, a_User); a_Callback##_CALLBACK(1, a_User)
+#define IEM_REPEAT_3(a_Callback, a_User)    IEM_REPEAT_2(a_Callback, a_User); a_Callback##_CALLBACK(2, a_User)
+#define IEM_REPEAT_4(a_Callback, a_User)    IEM_REPEAT_3(a_Callback, a_User); a_Callback##_CALLBACK(3, a_User)
+#define IEM_REPEAT_5(a_Callback, a_User)    IEM_REPEAT_4(a_Callback, a_User); a_Callback##_CALLBACK(4, a_User)
+#define IEM_REPEAT_6(a_Callback, a_User)    IEM_REPEAT_5(a_Callback, a_User); a_Callback##_CALLBACK(5, a_User)
+#define IEM_REPEAT_7(a_Callback, a_User)    IEM_REPEAT_6(a_Callback, a_User); a_Callback##_CALLBACK(6, a_User)
+#define IEM_REPEAT_8(a_Callback, a_User)    IEM_REPEAT_7(a_Callback, a_User); a_Callback##_CALLBACK(7, a_User)
+#define IEM_REPEAT_9(a_Callback, a_User)    IEM_REPEAT_8(a_Callback, a_User); a_Callback##_CALLBACK(8, a_User)
+#define IEM_REPEAT(a_cTimes, a_Callback, a_User) RT_CONCAT(IEM_REPEAT_,a_cTimes)(a_Callback, a_User)
+
+
+
+/** @name Microcode test stubs
+ * @{  */
+
+#define IEM_ARG_CHECK_CALLBACK(a_idx, a_User) int RT_CONCAT(iArgCheck_,a_idx); NOREF(RT_CONCAT(iArgCheck_,a_idx))
+#define IEM_MC_BEGIN(a_cArgs, a_cLocals) \
+    { \
+        const uint8_t cArgs   = (a_cArgs); NOREF(cArgs); \
+        const uint8_t cLocals = (a_cLocals); NOREF(cLocals); \
+        const uint8_t fMcBegin = (a_cArgs) + (a_cLocals); \
+        IEM_REPEAT(a_cArgs, IEM_ARG_CHECK, 0); \
+
+#define IEM_MC_END() \
+    }
+
+#define IEM_MC_PAUSE()                                  do { (void)fMcBegin; } while (0)
+#define IEM_MC_CONTINUE()                               do { (void)fMcBegin; } while (0)
+#define IEM_MC_ADVANCE_RIP()                            do { (void)fMcBegin; } while (0)
+#define IEM_MC_REL_JMP_S8(a_i8)                         do { (void)fMcBegin; CHK_TYPE(int8_t, a_i8); } while (0)
+#define IEM_MC_REL_JMP_S16(a_i16)                       do { (void)fMcBegin; CHK_TYPE(int16_t, a_i16); } while (0)
+#define IEM_MC_REL_JMP_S32(a_i32)                       do { (void)fMcBegin; CHK_TYPE(int32_t, a_i32); } while (0)
+#define IEM_MC_SET_RIP_U16(a_u16NewIP)                  do { (void)fMcBegin; CHK_TYPE(uint16_t, a_u16NewIP); } while (0)
+#define IEM_MC_SET_RIP_U32(a_u32NewIP)                  do { (void)fMcBegin; CHK_TYPE(uint32_t, a_u32NewIP); } while (0)
+#define IEM_MC_SET_RIP_U64(a_u64NewIP)                  do { (void)fMcBegin; CHK_TYPE(uint64_t, a_u64NewIP); } while (0)
+#define IEM_MC_RAISE_DIVIDE_ERROR()                     do { (void)fMcBegin; return VERR_TRPM_ACTIVE_TRAP; } while (0)
+#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE()       do { (void)fMcBegin; } while (0)
+#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE()  do { (void)fMcBegin; } while (0)
+#define IEM_MC_MAYBE_RAISE_FPU_XCPT()                   do { (void)fMcBegin; } while (0)
+#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT()           do { (void)fMcBegin; } while (0)
+#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT() do { (void)fMcBegin; } while (0)
+#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT()           do { (void)fMcBegin; } while (0)
+#define IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT()          do { (void)fMcBegin; } while (0)
+#define IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT()          do { (void)fMcBegin; } while (0)
+#define IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT()         do { (void)fMcBegin; } while (0)
+#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT()           do { (void)fMcBegin; } while (0)
+#define IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT()          do { (void)fMcBegin; } while (0)
+#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO()              do { (void)fMcBegin; } while (0)
+#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
+    do { (void)fMcBegin; AssertCompile(RT_IS_POWER_OF_TWO(a_cbAlign)); CHK_TYPE(RTGCPTR,  a_EffAddr); } while (0)
+#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT()              do { (void)fMcBegin; } while (0)
+#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) do { (void)fMcBegin; } while (0)
+
+#define IEM_MC_LOCAL(a_Type, a_Name) (void)fMcBegin; \
+    a_Type a_Name; NOREF(a_Name); (void)fMcBegin
+#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) (void)fMcBegin; \
+    a_Type const a_Name = (a_Value); \
+    NOREF(a_Name)
+#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (void)fMcBegin; \
+    (a_pRefArg) = &(a_Local)
+
+#define IEM_MC_ARG(a_Type, a_Name, a_iArg) (void)fMcBegin; \
+    RT_CONCAT(iArgCheck_,a_iArg) = 1; NOREF(RT_CONCAT(iArgCheck_,a_iArg)); \
+    int RT_CONCAT3(iArgCheck_,a_iArg,a_Name); NOREF(RT_CONCAT3(iArgCheck_,a_iArg,a_Name)); \
+    AssertCompile((a_iArg) < cArgs); \
+    a_Type a_Name; \
+    NOREF(a_Name)
+#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) (void)fMcBegin; \
+    RT_CONCAT(iArgCheck_, a_iArg) = 1; NOREF(RT_CONCAT(iArgCheck_,a_iArg)); \
+    int RT_CONCAT3(iArgCheck_,a_iArg,a_Name); NOREF(RT_CONCAT3(iArgCheck_,a_iArg,a_Name)); \
+    AssertCompile((a_iArg) < cArgs); \
+    a_Type const a_Name = (a_Value); \
+    NOREF(a_Name)
+#define IEM_MC_ARG_XSTATE(a_Name, a_iArg) \
+    IEM_MC_ARG_CONST(PX86XSAVEAREA, a_Name, NULL, a_iArg)
+
+#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) (void)fMcBegin; \
+    RT_CONCAT(iArgCheck_, a_iArg) = 1; NOREF(RT_CONCAT(iArgCheck_,a_iArg)); \
+    int RT_CONCAT3(iArgCheck_,a_iArg,a_Name); NOREF(RT_CONCAT3(iArgCheck_,a_iArg,a_Name)); \
+    AssertCompile((a_iArg) < cArgs); \
+    a_Type const a_Name = &(a_Local); \
+    NOREF(a_Name)
+#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) (void)fMcBegin; \
+    RT_CONCAT(iArgCheck_, a_iArg) = 1; NOREF(RT_CONCAT(iArgCheck_,a_iArg)); \
+    int RT_CONCAT3(iArgCheck_,a_iArg,a_pName); NOREF(RT_CONCAT3(iArgCheck_,a_iArg,a_pName)); \
+    AssertCompile((a_iArg) < cArgs); \
+    uint32_t a_Name; \
+    uint32_t *a_pName = &a_Name; \
+    NOREF(a_pName)
+
+#define IEM_MC_COMMIT_EFLAGS(a_EFlags)                  do { CHK_TYPE(uint32_t, a_EFlags); (void)fMcBegin; } while (0)
+#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst)   do { (a_VarOrArg) = (0); (void)fMcBegin; } while (0)
+#define IEM_MC_ASSIGN_TO_SMALLER                        IEM_MC_ASSIGN
+
+#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg)          do { (a_u8Dst)  = 0; CHK_TYPE(uint8_t,  a_u8Dst);  (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg)  do { (a_u16Dst) = 0; CHK_TYPE(uint16_t, a_u16Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg)  do { (a_u32Dst) = 0; CHK_TYPE(uint32_t, a_u32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg)  do { (a_u64Dst) = 0; CHK_TYPE(uint64_t, a_u64Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg)  do { (a_u16Dst) = 0; CHK_TYPE(uint16_t, a_u16Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg)  do { (a_u32Dst) = 0; CHK_TYPE(uint32_t, a_u32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg)  do { (a_u64Dst) = 0; CHK_TYPE(uint64_t, a_u64Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg)        do { (a_u16Dst) = 0; CHK_TYPE(uint16_t, a_u16Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) do { (a_u32Dst) = 0; CHK_TYPE(uint32_t, a_u32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) do { (a_u64Dst) = 0; CHK_TYPE(uint64_t, a_u64Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) do { (a_u32Dst) = 0; CHK_TYPE(uint32_t, a_u32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) do { (a_u64Dst) = 0; CHK_TYPE(uint64_t, a_u64Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg)        do { (a_u32Dst) = 0; CHK_TYPE(uint32_t, a_u32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) do { (a_u64Dst) = 0; CHK_TYPE(uint64_t, a_u64Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) do { (a_u64Dst) = 0; CHK_TYPE(uint64_t, a_u64Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg)        do { (a_u64Dst) = 0; CHK_TYPE(uint64_t, a_u64Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_GREG_U64_ZX_U64                    IEM_MC_FETCH_GREG_U64
+#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg)        do { (a_u16Dst) = 0; CHK_TYPE(uint16_t, a_u16Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg)     do { (a_u32Dst) = 0; CHK_TYPE(uint32_t, a_u32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg)     do { (a_u64Dst) = 0; CHK_TYPE(uint64_t, a_u64Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg)   do { (a_u64Dst) = 0; CHK_TYPE(uint64_t, a_u64Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg)   do { (a_u32Dst) = 0; CHK_TYPE(uint32_t, a_u32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_EFLAGS(a_EFlags)                   do { (a_EFlags) = 0; CHK_TYPE(uint32_t, a_EFlags); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags)                do { (a_EFlags) = 0; CHK_TYPE(uint8_t,  a_EFlags); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_FSW(a_u16Fsw)                      do { (a_u16Fsw) = 0; CHK_TYPE(uint16_t, a_u16Fsw); (void)fFpuRead; (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_FCW(a_u16Fcw)                      do { (a_u16Fcw) = 0; CHK_TYPE(uint16_t, a_u16Fcw); (void)fFpuRead; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value)        do { CHK_TYPE(uint8_t, a_u8Value); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value)      do { CHK_TYPE(uint16_t, a_u16Value); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value)      do { (void)fMcBegin;  } while (0)
+#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value)      do { (void)fMcBegin;  } while (0)
+#define IEM_MC_STORE_GREG_U8_CONST(a_iGReg, a_u8C)      do { AssertCompile((uint8_t )(a_u8C)  == (a_u8C) ); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_GREG_U16_CONST(a_iGReg, a_u16C)    do { AssertCompile((uint16_t)(a_u16C) == (a_u16C)); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_GREG_U32_CONST(a_iGReg, a_u32C)    do { AssertCompile((uint32_t)(a_u32C) == (a_u32C)); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_GREG_U64_CONST(a_iGReg, a_u64C)    do { AssertCompile((uint64_t)(a_u64C) == (a_u64C)); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) do { CHK_PTYPE(PCRTFLOAT80U, a_pr80Src); Assert((a_iSt) < 8); (void)fMcBegin; } while (0)
+#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg)             do { (void)fMcBegin;  } while (0)
+#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst)    do { CHK_PTYPE(uint32_t *, a_pu32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_SREG_BASE_U64(a_iSeg, a_u64Value)  do { (void)fMcBegin;  } while (0)
+#define IEM_MC_STORE_SREG_BASE_U32(a_iSeg, a_u32Value)  do { (void)fMcBegin;  } while (0)
+#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg)           do { (a_pu8Dst)  = (uint8_t  *)((uintptr_t)0); CHK_PTYPE(uint8_t  *, a_pu8Dst);  (void)fMcBegin; } while (0)
+#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg)         do { (a_pu16Dst) = (uint16_t *)((uintptr_t)0); CHK_PTYPE(uint16_t *, a_pu16Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg)         do { (a_pu32Dst) = (uint32_t *)((uintptr_t)0); CHK_PTYPE(uint32_t *, a_pu32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg)         do { (a_pu64Dst) = (uint64_t *)((uintptr_t)0); CHK_PTYPE(uint64_t *, a_pu64Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_REF_EFLAGS(a_pEFlags)                    do { (a_pEFlags) = (uint32_t *)((uintptr_t)0); CHK_PTYPE(uint32_t *, a_pEFlags); (void)fMcBegin; } while (0)
+
+#define IEM_MC_ADD_GREG_U8(a_iGReg, a_u8Value)          do { CHK_CONST(uint8_t,  a_u8Value);  (void)fMcBegin; } while (0)
+#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value)        do { CHK_CONST(uint16_t, a_u16Value); (void)fMcBegin; } while (0)
+#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value)        do { CHK_CONST(uint32_t, a_u32Value); (void)fMcBegin; } while (0)
+#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value)        do { CHK_CONST(uint64_t, a_u64Value); (void)fMcBegin; } while (0)
+#define IEM_MC_SUB_GREG_U8(a_iGReg,  a_u8Value)         do { CHK_CONST(uint8_t,  a_u8Value);  (void)fMcBegin; } while (0)
+#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u16Value)        do { CHK_CONST(uint16_t, a_u16Value); (void)fMcBegin; } while (0)
+#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u32Value)        do { CHK_CONST(uint32_t, a_u32Value); (void)fMcBegin; } while (0)
+#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u64Value)        do { CHK_CONST(uint64_t, a_u64Value); (void)fMcBegin; } while (0)
+#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const)    do { CHK_CONST(uint16_t, a_u16Const); (void)fMcBegin; } while (0)
+
+#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value)          do { CHK_CONST(uint8_t,  a_u8Value);  (void)fMcBegin; } while (0)
+#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value)        do { CHK_CONST(uint16_t, a_u16Value); (void)fMcBegin; } while (0)
+#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value)        do { CHK_CONST(uint32_t, a_u32Value); (void)fMcBegin; } while (0)
+#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value)        do { CHK_CONST(uint64_t, a_u64Value); (void)fMcBegin; } while (0)
+#define IEM_MC_OR_GREG_U8(a_iGReg,  a_u8Value)          do { CHK_CONST(uint8_t,  a_u8Value);  (void)fMcBegin; } while (0)
+#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value)         do { CHK_CONST(uint16_t, a_u16Value); (void)fMcBegin; } while (0)
+#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value)         do { CHK_CONST(uint32_t, a_u32Value); (void)fMcBegin; } while (0)
+#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value)         do { CHK_CONST(uint64_t, a_u64Value); (void)fMcBegin; } while (0)
+
+#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u16Value, a_iGReg)   do { (a_u8Value)  += 1; CHK_TYPE(uint8_t,  a_u8Value);  (void)fMcBegin; } while (0)
+#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg)  do { (a_u16Value) += 1; CHK_TYPE(uint16_t, a_u16Value); (void)fMcBegin; } while (0)
+#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg)  do { (a_u32Value) += 1; CHK_TYPE(uint32_t, a_u32Value); (void)fMcBegin; } while (0)
+#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg)  do { (a_u64Value) += 1; CHK_TYPE(uint64_t, a_u64Value); (void)fMcBegin; } while (0)
+#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); CHK_GCPTR(a_EffAddr); (void)fMcBegin; } while (0)
+#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); CHK_GCPTR(a_EffAddr); (void)fMcBegin; } while (0)
+#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); CHK_GCPTR(a_EffAddr); (void)fMcBegin; } while (0)
+#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask)        do { (a_u8Local)  &= (a_u8Mask);  CHK_TYPE(uint8_t,  a_u8Local);  CHK_CONST(uint8_t,  a_u8Mask);  (void)fMcBegin; } while (0)
+#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask)     do { (a_u16Local) &= (a_u16Mask); CHK_TYPE(uint16_t, a_u16Local); CHK_CONST(uint16_t, a_u16Mask); (void)fMcBegin; } while (0)
+#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask)     do { (a_u32Local) &= (a_u32Mask); CHK_TYPE(uint32_t, a_u32Local); CHK_CONST(uint32_t, a_u32Mask); (void)fMcBegin; } while (0)
+#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask)     do { (a_u64Local) &= (a_u64Mask); CHK_TYPE(uint64_t, a_u64Local); CHK_CONST(uint64_t, a_u64Mask); (void)fMcBegin; } while (0)
+#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask)         do { (a_u16Arg)   &= (a_u16Mask); CHK_TYPE(uint16_t, a_u16Arg);   CHK_CONST(uint16_t, a_u16Mask); (void)fMcBegin; } while (0)
+#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask)         do { (a_u32Arg)   &= (a_u32Mask); CHK_TYPE(uint32_t, a_u32Arg);   CHK_CONST(uint32_t, a_u32Mask); (void)fMcBegin; } while (0)
+#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask)         do { (a_u64Arg)   &= (a_u64Mask); CHK_TYPE(uint64_t, a_u64Arg);   CHK_CONST(uint64_t, a_u64Mask); (void)fMcBegin; } while (0)
+#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask)         do { (a_u8Local)  |= (a_u8Mask);  CHK_TYPE(uint8_t,  a_u8Local);  CHK_CONST(uint8_t,  a_u8Mask);  (void)fMcBegin; } while (0)
+#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask)      do { (a_u16Local) |= (a_u16Mask); CHK_TYPE(uint16_t, a_u16Local); CHK_CONST(uint16_t, a_u16Mask); (void)fMcBegin; } while (0)
+#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask)      do { (a_u32Local) |= (a_u32Mask); CHK_TYPE(uint32_t, a_u32Local); CHK_CONST(uint32_t, a_u32Mask); (void)fMcBegin; } while (0)
+#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift)      do { (a_i16Local) >>= (a_cShift); CHK_TYPE(int16_t, a_i16Local);  CHK_CONST(uint8_t,  a_cShift);  (void)fMcBegin; } while (0)
+#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift)      do { (a_i32Local) >>= (a_cShift); CHK_TYPE(int32_t, a_i32Local);  CHK_CONST(uint8_t,  a_cShift);  (void)fMcBegin; } while (0)
+#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift)      do { (a_i64Local) >>= (a_cShift); CHK_TYPE(int64_t, a_i64Local);  CHK_CONST(uint8_t,  a_cShift);  (void)fMcBegin; } while (0)
+#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift)      do { (a_i16Local) <<= (a_cShift); CHK_TYPE(int16_t, a_i16Local);  CHK_CONST(uint8_t,  a_cShift);  (void)fMcBegin; } while (0)
+#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift)      do { (a_i32Local) <<= (a_cShift); CHK_TYPE(int32_t, a_i32Local);  CHK_CONST(uint8_t,  a_cShift);  (void)fMcBegin; } while (0)
+#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift)      do { (a_i64Local) <<= (a_cShift); CHK_TYPE(int64_t, a_i64Local);  CHK_CONST(uint8_t,  a_cShift);  (void)fMcBegin; } while (0)
+#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask)     do { (a_u32Local) &= (a_u32Mask); CHK_TYPE(uint32_t, a_u32Local); (void)fMcBegin; } while (0)
+#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask)      do { (a_u32Local) |= (a_u32Mask); CHK_TYPE(uint32_t, a_u32Local); (void)fMcBegin; } while (0)
+#define IEM_MC_SET_EFL_BIT(a_fBit)                      do { CHK_SINGLE_BIT(uint32_t, a_fBit); (void)fMcBegin; } while (0)
+#define IEM_MC_CLEAR_EFL_BIT(a_fBit)                    do { CHK_SINGLE_BIT(uint32_t, a_fBit); (void)fMcBegin; } while (0)
+#define IEM_MC_FLIP_EFL_BIT(a_fBit)                     do { CHK_SINGLE_BIT(uint32_t, a_fBit); (void)fMcBegin; } while (0)
+#define IEM_MC_CLEAR_FSW_EX()                           do { (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_TO_MMX_MODE()                        do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_FROM_MMX_MODE()                      do { (void)fMcBegin; } while (0)
+
+#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg)          do { (a_u64Value) = 0; CHK_TYPE(uint64_t, a_u64Value); (void)fFpuRead; (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg)          do { (a_u32Value) = 0; CHK_TYPE(uint32_t, a_u32Value); (void)fFpuRead; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value)          do { CHK_TYPE(uint64_t, a_u64Value); (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value)   do { CHK_TYPE(uint32_t, a_u32Value); (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg)             do { (a_pu64Dst) = (uint64_t *)((uintptr_t)0); CHK_PTYPE(uint64_t *, a_pu64Dst);             (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg)       do { (a_pu64Dst) = (uint64_t const *)((uintptr_t)0); CHK_PTYPE(uint64_t const *, a_pu64Dst); (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg)       do { (a_pu32Dst) = (uint32_t const *)((uintptr_t)0); CHK_PTYPE(uint32_t const *, a_pu32Dst); (void)fFpuWrite; (void)fMcBegin; } while (0)
+
+#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg)        do { (a_u128Value) = g_u128Zero; CHK_TYPE(RTUINT128U, a_u128Value); (void)fSseRead;  (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg)          do { (a_u64Value) = 0; CHK_TYPE(uint64_t, a_u64Value); (void)fSseRead; (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg)          do { (a_u32Value) = 0; CHK_TYPE(uint32_t, a_u32Value); (void)fSseRead; (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_XREG_HI_U64(a_u64Value, a_iXReg)       do { (a_u64Value) = 0; CHK_TYPE(uint64_t, a_u64Value); (void)fSseRead; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value)        do { CHK_TYPE(RTUINT128U, a_u128Value); (void)fSseWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_XREG_U64(a_iXReg, a_u64Value)          do { CHK_TYPE(uint64_t,  a_u64Value);  (void)fSseWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value)  do { CHK_TYPE(uint64_t,  a_u64Value);  (void)fSseWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_XREG_U32(a_iXReg, a_u32Value)          do { CHK_TYPE(uint32_t,  a_u32Value);  (void)fSseWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value)  do { CHK_TYPE(uint32_t,  a_u32Value);  (void)fSseWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_XREG_HI_U64(a_iXReg, a_u64Value)       do { CHK_TYPE(uint64_t,  a_u64Value);  (void)fSseWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg)           do { (a_pu128Dst) = (PRTUINT128U)((uintptr_t)0);        CHK_PTYPE(PRTUINT128U, a_pu128Dst);     (void)fSseWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg)     do { (a_pu128Dst) = (PCRTUINT128U)((uintptr_t)0);       CHK_PTYPE(PCRTUINT128U, a_pu128Dst);    (void)fSseWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg)       do { (a_pu64Dst)  = (uint64_t const *)((uintptr_t)0);   CHK_PTYPE(uint64_t const *, a_pu64Dst); (void)fSseWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc)       do { (void)fSseWrite; (void)fMcBegin; } while (0)
+
+#define IEM_MC_FETCH_YREG_U256(a_u256Value, a_iYRegSrc)           do { (a_u256Value).au64[0] = (a_u256Value).au64[1] = (a_u256Value).au64[2] = (a_u256Value).au64[3] = 0; CHK_TYPE(RTUINT256U, a_u256Value); (void)fAvxRead; (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_YREG_U128(a_u128Value, a_iYRegSrc)           do { (a_u128Value).au64[0] = (a_u128Value).au64[1] = 0; CHK_TYPE(RTUINT128U, a_u128Value); (void)fAvxRead; (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_YREG_U64(a_u64Value, a_iYRegSrc)             do { (a_u64Value) = UINT64_MAX; CHK_TYPE(uint64_t, a_u64Value); (void)fAvxRead; (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_YREG_U32(a_u32Value, a_iYRegSrc)             do { (a_u32Value) = UINT32_MAX; CHK_TYPE(uint32_t, a_u32Value); (void)fAvxRead; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Value)    do { CHK_TYPE(uint32_t, a_u32Value); (void)fAvxWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Value)    do { CHK_TYPE(uint64_t, a_u64Value); (void)fAvxWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Value)  do { CHK_TYPE(RTUINT128U, a_u128Value); (void)fAvxWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Value)  do { CHK_TYPE(RTUINT256U, a_u256Value); (void)fAvxWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg)                 do { (a_pu128Dst) = (PRTUINT128U)((uintptr_t)0);      CHK_PTYPE(PRTUINT128U, a_pu128Dst);       (void)fAvxWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg)           do { (a_pu128Dst) = (PCRTUINT128U)((uintptr_t)0);     CHK_PTYPE(PCRTUINT128U, a_pu128Dst);      (void)fAvxWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg)             do { (a_pu64Dst)  = (uint64_t const *)((uintptr_t)0); CHK_PTYPE(uint64_t const *, a_pu64Dst);   (void)fAvxWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg)                         do { (void)fAvxWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc)    do { (void)fAvxWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc)    do { (void)fAvxWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc)     do { (void)fAvxWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx)      do { (void)fAvxWrite; (void)fAvxRead; (void)fMcBegin; } while (0)
+#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx)      do { (void)fAvxWrite; (void)fAvxRead; (void)fMcBegin; } while (0)
+#define IEM_MC_MERGE_YREG_U64HI_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx)    do { (void)fAvxWrite; (void)fAvxRead; (void)fMcBegin; } while (0)
+#define IEM_MC_MERGE_YREG_U64LOCAL_U64_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx)   do { (void)fAvxWrite; (void)fAvxRead; (void)fMcBegin; } while (0)
+
+#define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem)                do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16)            do { CHK_TYPE(uint16_t, a_GCPtrMem16); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32)            do { CHK_TYPE(uint32_t, a_GCPtrMem32); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem)              do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem)              do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(int16_t, a_i16Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem)              do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem)              do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(int32_t, a_i32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem)       do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem)              do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem)   do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem)              do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(int64_t, a_i64Dst); (void)fMcBegin; } while (0)
+
+#define IEM_MC_FETCH_MEM_U8_DISP(a_u8Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
+    do { CHK_GCPTR(a_GCPtrMem); CHK_CONST(uint8_t, a_offDisp); CHK_TYPE(uint8_t, a_u8Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
+    do { CHK_GCPTR(a_GCPtrMem); CHK_CONST(uint8_t, a_offDisp); CHK_TYPE(uint16_t, a_u16Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
+    do { CHK_GCPTR(a_GCPtrMem); CHK_CONST(uint8_t, a_offDisp); CHK_TYPE(uint32_t, a_u32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
+    do { CHK_GCPTR(a_GCPtrMem); CHK_CONST(uint8_t, a_offDisp); CHK_TYPE(uint64_t, a_u64Dst); (void)fMcBegin; } while (0)
+
+#define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem)        do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem)        do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem)        do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem)       do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem)       do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem)       do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem)        do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem)        do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem)        do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem)       do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem)       do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem)       do { CHK_GCPTR(a_GCPtrMem); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem)              do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(RTFLOAT32U, a_r32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem)              do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(RTFLOAT64U, a_r64Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem)              do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(RTFLOAT80U, a_r80Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem)            do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(RTUINT128U, a_u128Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem)  do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(RTUINT128U, a_u128Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem)            do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(RTUINT256U, a_u256Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem)  do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(RTUINT256U, a_u256Dst); (void)fMcBegin; } while (0)
+
+#define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value)              do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(uint8_t,  a_u8Value); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value)            do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(uint16_t, a_u16Value);      (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value)            do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(uint32_t, a_u32Value);      (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value)            do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(uint64_t, a_u64Value);      (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C)            do { CHK_GCPTR(a_GCPtrMem); CHK_CONST(uint8_t,  a_u8C);          (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C)          do { CHK_GCPTR(a_GCPtrMem); CHK_CONST(uint16_t, a_u16C);         (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C)          do { CHK_GCPTR(a_GCPtrMem); CHK_CONST(uint32_t, a_u32C);         (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C)          do { CHK_GCPTR(a_GCPtrMem); CHK_CONST(uint64_t, a_u64C);         (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst,  a_i8C)             do { CHK_TYPE(int8_t *,  a_pi8Dst);  CHK_CONST(int8_t,  a_i8C);  (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C)            do { CHK_TYPE(int16_t *, a_pi16Dst); CHK_CONST(int16_t, a_i16C); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C)            do { CHK_TYPE(int32_t *, a_pi32Dst); CHK_CONST(int32_t, a_i32C); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C)            do { CHK_TYPE(int64_t *, a_pi64Dst); CHK_CONST(int64_t, a_i64C); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst)                 do { CHK_TYPE(PRTFLOAT32U, a_pr32Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst)                 do { CHK_TYPE(PRTFLOAT64U, a_pr64Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst)                 do { CHK_TYPE(PRTFLOAT80U, a_pr80Dst); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Src)            do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(RTUINT128U, a_u128Src); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Src)  do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(RTUINT128U, a_u128Src); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Src)            do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(RTUINT256U, a_u256Src); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Src)  do { CHK_GCPTR(a_GCPtrMem); CHK_TYPE(RTUINT256U, a_u256Src); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+
+#define IEM_MC_PUSH_U16(a_u16Value)                                     do { (void)fMcBegin; } while (0)
+#define IEM_MC_PUSH_U32(a_u32Value)                                     do { (void)fMcBegin; } while (0)
+#define IEM_MC_PUSH_U32_SREG(a_u32Value)                                do { (void)fMcBegin; } while (0)
+#define IEM_MC_PUSH_U64(a_u64Value)                                     do { (void)fMcBegin; } while (0)
+#define IEM_MC_POP_U16(a_pu16Value)                                     do { (void)fMcBegin; } while (0)
+#define IEM_MC_POP_U32(a_pu32Value)                                     do { (void)fMcBegin; } while (0)
+#define IEM_MC_POP_U64(a_pu64Value)                                     do { (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg)   do { (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_iArg)  do { (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess)                             do { (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW)     do { (void)fMcBegin; } while (0)
+#define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm, cbImm)                 do { (a_GCPtrEff) = 0; CHK_GCPTR(a_GCPtrEff); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn)                                 do { (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) \
+    do { CHK_CALL_ARG(a0, 0); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) \
+    do { CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) \
+    do { CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) \
+    do { CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2); CHK_CALL_ARG(a3, 3); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) \
+    do { CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2);  (a_rc) = VINF_SUCCESS; (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) \
+    do { CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2); CHK_CALL_ARG(a3, 3);  (a_rc) = VINF_SUCCESS; (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_CIMPL_0(a_pfnCImpl)                                 do { (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_CIMPL_1(a_pfnCImpl, a0) \
+    do { CHK_CALL_ARG(a0, 0);  (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_CIMPL_2(a_pfnCImpl, a0, a1) \
+    do { CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_CIMPL_3(a_pfnCImpl, a0, a1, a2) \
+    do { CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_CIMPL_4(a_pfnCImpl, a0, a1, a2, a3) \
+    do { CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2); CHK_CALL_ARG(a3, 3); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_CIMPL_5(a_pfnCImpl, a0, a1, a2, a3, a4) \
+    do { CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2); CHK_CALL_ARG(a3, 3); CHK_CALL_ARG(a4, 4); (void)fMcBegin; } while (0)
+#define IEM_MC_DEFER_TO_CIMPL_0(a_pfnCImpl)                             (VINF_SUCCESS)
+#define IEM_MC_DEFER_TO_CIMPL_1(a_pfnCImpl, a0)                         (VINF_SUCCESS)
+#define IEM_MC_DEFER_TO_CIMPL_2(a_pfnCImpl, a0, a1)                     (VINF_SUCCESS)
+#define IEM_MC_DEFER_TO_CIMPL_3(a_pfnCImpl, a0, a1, a2)                 (VINF_SUCCESS)
+
+#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
+    do { (void)fFpuHost; (void)fFpuWrite; CHK_CALL_ARG(a0, 0);  (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
+    do { (void)fFpuHost; (void)fFpuWrite; CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
+    do { (void)fFpuHost; (void)fFpuWrite; CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2); (void)fMcBegin; } while (0)
+#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value)            do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_PUSH_FPU_RESULT(a_FpuData)                               do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff) do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo)                        do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg)                    do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg)           do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)              do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_STORE_FPU_RESULT_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)     do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStReg)                                                    do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStReg, a_iEffSeg, a_GCPtrEff)                      do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStReg)                                           do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStReg, a_iEffSeg, a_GCPtrEff)             do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP()                                               do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW()                                                       do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO()                                                   do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_STACK_PUSH_OVERFLOW()                                                        do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff)                            do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_UPDATE_FPU_OPCODE_IP()                                                           do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_STACK_DEC_TOP()                                                              do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_STACK_INC_TOP()                                                              do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_FPU_STACK_FREE(a_iStReg)                                                         do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_UPDATE_FSW(a_u16FSW)                                                             do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW)                                                       do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff)                          do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW)                                                    do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff)                 do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW)                                                do { (void)fFpuWrite; (void)fMcBegin; } while (0)
+#define IEM_MC_PREPARE_FPU_USAGE() (void)fMcBegin; \
+    const int fFpuRead = 1, fFpuWrite = 1, fFpuHost = 1, fSseRead = 1, fSseWrite = 1, fSseHost = 1, fAvxRead = 1, fAvxWrite = 1, fAvxHost = 1
+#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ()   (void)fMcBegin; const int fFpuRead = 1, fSseRead = 1
+#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() (void)fMcBegin; const int fFpuRead = 1, fFpuWrite = 1, fSseRead = 1, fSseWrite = 1
+#define IEM_MC_PREPARE_SSE_USAGE()              (void)fMcBegin; const int fSseRead = 1, fSseWrite = 1, fSseHost = 1
+#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ()   (void)fMcBegin; const int fSseRead = 1
+#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() (void)fMcBegin; const int fSseRead = 1, fSseWrite = 1
+#define IEM_MC_PREPARE_AVX_USAGE()              (void)fMcBegin; const int fAvxRead = 1, fAvxWrite = 1, fAvxHost = 1, fSseRead = 1, fSseWrite = 1, fSseHost = 1
+#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ()   (void)fMcBegin; const int fAvxRead = 1, fSseRead = 1
+#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() (void)fMcBegin; const int fAvxRead = 1, fAvxWrite = 1, fSseRead = 1, fSseWrite = 1
+
+#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
+    do { (void)fFpuHost; (void)fFpuWrite; CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
+    do { (void)fFpuHost; (void)fFpuWrite; CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
+    do { (void)fSseHost; (void)fSseWrite; CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
+    do { (void)fSseHost; (void)fSseWrite; CHK_CALL_ARG(a0, 0); CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2); (void)fMcBegin; } while (0)
+#define IEM_MC_IMPLICIT_AVX_AIMPL_ARGS() do { IEM_MC_ARG_CONST(PX86XSAVEAREA, pXState, pVCpu->cpum.GstCtx.CTX_SUFF(pXState), 0); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a1, a2) \
+    do { (void)fAvxHost; (void)fAvxWrite; CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a1, a2, a3) \
+    do { (void)fAvxHost; (void)fAvxWrite; CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2); CHK_CALL_ARG(a3, 3); (void)fMcBegin; } while (0)
+#define IEM_MC_CALL_AVX_AIMPL_4(a_pfnAImpl, a1, a2, a3, a4) \
+    do { (void)fAvxHost; (void)fAvxWrite; CHK_CALL_ARG(a1, 1); CHK_CALL_ARG(a2, 2); CHK_CALL_ARG(a3, 3); CHK_CALL_ARG(a4, 4); (void)fMcBegin; } while (0)
+
+#define IEM_MC_IF_EFL_BIT_SET(a_fBit)                                   (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit)                               (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits)                             (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits)                              (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2)                         (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2)                         (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2)      (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_CX_IS_NZ()                                            (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_ECX_IS_NZ()                                           (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_RCX_IS_NZ()                                           (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit)                      (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit)                     (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit)                     (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit)                  (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit)                 (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit)                 (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_LOCAL_IS_Z(a_Local)                                   (void)fMcBegin; if ((a_Local) == 0) {
+#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo)                       (void)fMcBegin; if (g_fRandom) {
+#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt)                               (void)fMcBegin; if (g_fRandom != fFpuRead) {
+#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt)                                (void)fMcBegin; if (g_fRandom != fFpuRead) {
+#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) (void)fMcBegin; \
+    a_pr80Dst = NULL; \
+    if (g_fRandom != fFpuRead) {
+#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(p0, i0, p1, i1) (void)fMcBegin; \
+    p0 = NULL; \
+    p1 = NULL; \
+    if (g_fRandom != fFpuRead) {
+#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(p0, i0, i1) (void)fMcBegin; \
+    p0 = NULL; \
+    if (g_fRandom != fFpuRead) {
+#define IEM_MC_IF_FCW_IM()                                              (void)fMcBegin; if (g_fRandom != fFpuRead) {
+#define IEM_MC_ELSE()                                                   } else {
+#define IEM_MC_ENDIF()                                                  } do { (void)fMcBegin; } while (0)
+
+/** @}  */
+
+#include "../VMMAll/IEMAllInstructions.cpp.h"
+
+
+
+/**
+ * Formalities...
+ */
+int main()
+{
+    RTTEST hTest;
+    RTEXITCODE rcExit = RTTestInitAndCreate("tstIEMCheckMc", &hTest);
+    if (rcExit == RTEXITCODE_SUCCESS)
+    {
+        RTTestBanner(hTest);
+        RTTestPrintf(hTest, RTTESTLVL_ALWAYS, "(this is only a compile test.)");
+        rcExit = RTTestSummaryAndDestroy(hTest);
+    }
+    return rcExit;
+}
diff --git a/src/VBox/VMM/testcase/tstMMHyperHeap.cpp b/src/VBox/VMM/testcase/tstMMHyperHeap.cpp
new file mode 100644
index 00000000..46f818e2
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstMMHyperHeap.cpp
@@ -0,0 +1,284 @@
+/* $Id: tstMMHyperHeap.cpp $ */
+/** @file
+ * MM Hypervisor Heap testcase.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/gvm.h>
+#include <VBox/sup.h>
+#include <VBox/param.h>
+#include <iprt/errcore.h>
+
+#include <VBox/log.h>
+#include <iprt/initterm.h>
+#include <iprt/mem.h>
+#include <iprt/assert.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+
+#define NUM_CPUS  16
+
+#define OUTPUT(a) do { Log(a); RTPrintf a; } while (0)
+
+/**
+ *  Entry point.
+ */
+extern "C" DECLEXPORT(int) TrustedMain(int argc, char **argv, char **envp)
+{
+    RT_NOREF1(envp);
+
+    /*
+     * Init runtime.
+     */
+    int rc = RTR3InitExe(argc, &argv, 0);
+    AssertRCReturn(rc, RTEXITCODE_INIT);
+
+    /*
+     * Create empty VM structure and call MMR3Init().
+     */
+    void       *pvVM = NULL;
+    RTR0PTR     pvR0 = NIL_RTR0PTR;
+    SUPPAGE     aPages[(sizeof(GVM) + NUM_CPUS * sizeof(GVMCPU)) >> PAGE_SHIFT];
+    rc = SUPR3Init(NULL);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Fatal error: SUP failure! rc=%Rrc\n", rc);
+        return RTEXITCODE_FAILURE;
+    }
+    rc = SUPR3PageAllocEx(RT_ELEMENTS(aPages), 0, &pvVM, &pvR0, &aPages[0]);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Fatal error: Allocation failure! rc=%Rrc\n", rc);
+        return RTEXITCODE_FAILURE;
+    }
+    RT_BZERO(pvVM, RT_ELEMENTS(aPages) * PAGE_SIZE); /* SUPR3PageAllocEx doesn't necessarily zero the memory. */
+    PVM  pVM = (PVM)pvVM;
+    pVM->paVMPagesR3 = aPages;
+    pVM->pVMR0ForCall = pvR0;
+
+    PUVM pUVM = (PUVM)RTMemPageAllocZ(RT_ALIGN_Z(sizeof(*pUVM), PAGE_SIZE));
+    if (!pUVM)
+    {
+        RTPrintf("Fatal error: RTMEmPageAllocZ failed\n");
+        return RTEXITCODE_FAILURE;
+    }
+    pUVM->u32Magic = UVM_MAGIC;
+    pUVM->pVM = pVM;
+    pVM->pUVM = pUVM;
+
+    pVM->cCpus = NUM_CPUS;
+    pVM->cbSelf = sizeof(VM);
+    pVM->cbVCpu = sizeof(VMCPU);
+    PVMCPU pVCpu = (PVMCPU)((uintptr_t)pVM + sizeof(GVM));
+    for (VMCPUID idCpu = 0; idCpu < NUM_CPUS; idCpu++)
+    {
+        pVM->apCpusR3[idCpu] = pVCpu;
+        pVCpu = (PVMCPU)((uintptr_t)pVCpu + sizeof(GVMCPU));
+    }
+
+    rc = STAMR3InitUVM(pUVM);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("FAILURE: STAMR3Init failed. rc=%Rrc\n", rc);
+        return 1;
+    }
+
+    rc = MMR3InitUVM(pUVM);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("FAILURE: STAMR3Init failed. rc=%Rrc\n", rc);
+        return 1;
+    }
+
+    rc = CFGMR3Init(pVM, NULL, NULL);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("FAILURE: CFGMR3Init failed. rc=%Rrc\n", rc);
+        return 1;
+    }
+
+    rc = MMR3Init(pVM);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Fatal error: MMR3Init failed! rc=%Rrc\n", rc);
+        return 1;
+    }
+
+    /*
+     * Try allocate.
+     */
+    static struct
+    {
+        size_t      cb;
+        unsigned    uAlignment;
+        void       *pvAlloc;
+        unsigned    iFreeOrder;
+    } aOps[] =
+    {
+        {        16,          0,    NULL,  0 },
+        {        16,          4,    NULL,  1 },
+        {        16,          8,    NULL,  2 },
+        {        16,         16,    NULL,  5 },
+        {        16,         32,    NULL,  4 },
+        {        32,          0,    NULL,  3 },
+        {        31,          0,    NULL,  6 },
+        {      1024,          0,    NULL,  8 },
+        {      1024,         32,    NULL, 10 },
+        {      1024,         32,    NULL, 12 },
+        { PAGE_SIZE,  PAGE_SIZE,    NULL, 13 },
+        {      1024,         32,    NULL,  9 },
+        { PAGE_SIZE,         32,    NULL, 11 },
+        { PAGE_SIZE,  PAGE_SIZE,    NULL, 14 },
+        {        16,          0,    NULL, 15 },
+        {        9,           0,    NULL,  7 },
+        {        16,          0,    NULL,  7 },
+        {        36,          0,    NULL,  7 },
+        {        16,          0,    NULL,  7 },
+        {     12344,          0,    NULL,  7 },
+        {        50,          0,    NULL,  7 },
+        {        16,          0,    NULL,  7 },
+    };
+    unsigned i;
+#ifdef DEBUG
+    MMHyperHeapDump(pVM);
+#endif
+    size_t cbBefore = MMHyperHeapGetFreeSize(pVM);
+    static char szFill[] = "01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+    /* allocate */
+    for (i = 0; i < RT_ELEMENTS(aOps); i++)
+    {
+        rc = MMHyperAlloc(pVM, aOps[i].cb, aOps[i].uAlignment, MM_TAG_VM, &aOps[i].pvAlloc);
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf("Failure: MMHyperAlloc(, %#x, %#x,) -> %d i=%d\n", aOps[i].cb, aOps[i].uAlignment, rc, i);
+            return 1;
+        }
+        memset(aOps[i].pvAlloc, szFill[i], aOps[i].cb);
+        if (RT_ALIGN_P(aOps[i].pvAlloc, (aOps[i].uAlignment ? aOps[i].uAlignment : 8)) != aOps[i].pvAlloc)
+        {
+            RTPrintf("Failure: MMHyperAlloc(, %#x, %#x,) -> %p, invalid alignment!\n", aOps[i].cb, aOps[i].uAlignment, aOps[i].pvAlloc);
+            return 1;
+        }
+    }
+
+    /* free and allocate the same node again. */
+#ifdef DEBUG
+    MMHyperHeapDump(pVM);
+#endif
+    for (i = 0; i < RT_ELEMENTS(aOps); i++)
+    {
+        if (    !aOps[i].pvAlloc
+            ||  aOps[i].uAlignment == PAGE_SIZE)
+            continue;
+        size_t cbBeforeSub = MMHyperHeapGetFreeSize(pVM);
+        rc = MMHyperFree(pVM, aOps[i].pvAlloc);
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf("Failure: MMHyperFree(, %p,) -> %d i=%d\n", aOps[i].pvAlloc, rc, i);
+            return 1;
+        }
+        size_t const cbFreed = MMHyperHeapGetFreeSize(pVM);
+        void *pv;
+        rc = MMHyperAlloc(pVM, aOps[i].cb, aOps[i].uAlignment, MM_TAG_VM_REQ, &pv);
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf("Failure: MMHyperAlloc(, %#x, %#x,) -> %d i=%d\n", aOps[i].cb, aOps[i].uAlignment, rc, i);
+            return 1;
+        }
+        if (pv != aOps[i].pvAlloc)
+        {
+            RTPrintf("Failure: Free+Alloc returned different address. new=%p old=%p i=%d (doesn't work with delayed free)\n", pv, aOps[i].pvAlloc, i);
+            //return 1;
+        }
+        aOps[i].pvAlloc = pv;
+        OUTPUT(("debug: i=%02d cbBeforeSub=%d cbFreed=%d now=%d\n", i, cbBeforeSub, cbFreed, MMHyperHeapGetFreeSize(pVM)));
+#if 0 /* won't work :/ */
+        size_t cbAfterSub = MMHyperHeapGetFreeSize(pVM);
+        if (cbBeforeSub != cbAfterSub)
+        {
+            RTPrintf("Failure: cbBeforeSub=%d cbAfterSub=%d. i=%d\n", cbBeforeSub, cbAfterSub, i);
+            return 1;
+        }
+#endif
+    }
+
+    /* free it in a specific order. */
+    int cFreed = 0;
+    for (i = 0; i < RT_ELEMENTS(aOps); i++)
+    {
+        unsigned j;
+        for (j = 0; j < RT_ELEMENTS(aOps); j++)
+        {
+            if (    aOps[j].iFreeOrder != i
+                ||  !aOps[j].pvAlloc)
+                continue;
+            OUTPUT(("j=%02d i=%02d free=%d cb=%5u pv=%p\n", j, i, MMHyperHeapGetFreeSize(pVM), aOps[j].cb, aOps[j].pvAlloc));
+            if (aOps[j].uAlignment == PAGE_SIZE)
+                cbBefore -= aOps[j].cb;
+            else
+            {
+                rc = MMHyperFree(pVM, aOps[j].pvAlloc);
+                if (RT_FAILURE(rc))
+                {
+                    RTPrintf("Failure: MMHyperFree(, %p,) -> %d j=%d i=%d\n", aOps[j].pvAlloc, rc, i, j);
+                    return 1;
+                }
+            }
+            aOps[j].pvAlloc = NULL;
+            cFreed++;
+        }
+    }
+    Assert(cFreed == RT_ELEMENTS(aOps));
+    OUTPUT(("i=done free=%d\n", MMHyperHeapGetFreeSize(pVM)));
+
+    /* check that we're back at the right amount of free memory. */
+    size_t cbAfter = MMHyperHeapGetFreeSize(pVM);
+    if (cbBefore != cbAfter)
+    {
+        OUTPUT(("Warning: Either we've split out an alignment chunk at the start, or we've got\n"
+                "         an alloc/free accounting bug: cbBefore=%d cbAfter=%d\n", cbBefore, cbAfter));
+#ifdef DEBUG
+        MMHyperHeapDump(pVM);
+#endif
+    }
+
+    RTPrintf("tstMMHyperHeap: Success\n");
+#ifdef LOG_ENABLED
+    RTLogFlush(NULL);
+#endif
+    SUPR3PageFreeEx(pVM, RT_ELEMENTS(aPages));
+    RTMemPageFree(pUVM, RT_ALIGN_Z(sizeof(*pUVM), PAGE_SIZE));
+    return 0;
+}
+
+
+#if !defined(VBOX_WITH_HARDENING) || !defined(RT_OS_WINDOWS)
+/**
+ * Main entry point.
+ */
+int main(int argc, char **argv, char **envp)
+{
+    return TrustedMain(argc, argv, envp);
+}
+#endif
+
diff --git a/src/VBox/VMM/testcase/tstMicro.h b/src/VBox/VMM/testcase/tstMicro.h
new file mode 100644
index 00000000..e8bd8a0a
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstMicro.h
@@ -0,0 +1,146 @@
+/* $Id: tstMicro.h $ */
+/** @file
+ * Micro Testcase, profiling special CPU operations.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_testcase_tstMicro_h
+#define VMM_INCLUDED_SRC_testcase_tstMicro_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+/**
+ * The testcase identifier.
+ */
+typedef enum TSTMICROTEST
+{
+    TSTMICROTEST_OVERHEAD = 0,
+    TSTMICROTEST_INVLPG_0,
+    TSTMICROTEST_INVLPG_EIP,
+    TSTMICROTEST_INVLPG_ESP,
+    TSTMICROTEST_CR3_RELOAD,
+    TSTMICROTEST_WP_DISABLE,
+    TSTMICROTEST_WP_ENABLE,
+
+    TSTMICROTEST_TRAP_FIRST,
+    TSTMICROTEST_PF_R0 = TSTMICROTEST_TRAP_FIRST,
+    TSTMICROTEST_PF_R1,
+    TSTMICROTEST_PF_R2,
+    TSTMICROTEST_PF_R3,
+
+    /** The max testcase. */
+    TSTMICROTEST_MAX
+} TSTMICROTEST;
+
+
+/**
+ *
+ */
+typedef struct TSTMICRORESULT
+{
+    /** The total number of ticks spent executing the testcase.
+     * This may include extra overhead stuff if we're weird stuff during trap handler. */
+    uint64_t    cTotalTicks;
+    /** Number of ticks spent getting into Rx from R0.
+     * This will include time spent setting up the testcase in R3. */
+    uint64_t    cToRxFirstTicks;
+    /** Number of ticks spent executing the trap.
+     * I.e. from right before trapping instruction to the start of  the trap handler.
+     * This does not apply to testcases which doesn't trap. */
+    uint64_t    cTrapTicks;
+    /** Number of ticks spent resuming Rx executing after a trap.
+    * This does not apply to testcases which doesn't trap. */
+    uint64_t    cToRxTrapTicks;
+    /** Number of ticks to get to back to r0 after resuming the trapped code.
+     * This does not apply to testcases which doesn't trap. */
+    uint64_t    cToR0Ticks;
+} TSTMICRORESULT, *PTSTMICRORESULT;
+
+/**
+ * Micro profiling testcase
+ */
+typedef struct TSTMICRO
+{
+    /** The RC address of this structure. */
+    RTRCPTR     RCPtr;
+    /** Just for proper alignment. */
+    RTRCPTR     RCPtrStack;
+
+    /** TSC sampled right before leaving R0. */
+    uint64_t    u64TSCR0Start;
+    /** TSC sampled right before the exception. */
+    uint64_t    u64TSCRxStart;
+    /** TSC sampled right after entering the trap handler. */
+    uint64_t    u64TSCR0Enter;
+    /** TSC sampled right before exitting the trap handler. */
+    uint64_t    u64TSCR0Exit;
+    /** TSC sampled right after resuming guest trap. */
+    uint64_t    u64TSCRxEnd;
+    /** TSC sampled right after re-entering R0. */
+    uint64_t    u64TSCR0End;
+    /** Number of times entered (should be one). */
+    uint32_t    cHits;
+    /** Advance EIP. */
+    int32_t     offEIPAdd;
+    /** The last CR3 code. */
+    uint32_t    u32CR2;
+    /** The last error code. */
+    uint32_t    u32ErrCd;
+    /** The last trap eip. */
+    uint32_t    u32EIP;
+    /** The original IDT address and limit. */
+    VBOXIDTR    OriginalIDTR;
+    /** Our IDT. */
+    VBOXIDTE    aIDT[256];
+
+    /** The overhead for the rdtsc + 2 xchg instr. */
+    uint64_t    u64Overhead;
+
+    /** The testresults. */
+    TSTMICRORESULT  aResults[TSTMICROTEST_MAX];
+    /** Ring-3 stack. */
+    uint8_t     au8Stack[4096];
+
+} TSTMICRO, *PTSTMICRO;
+
+
+RT_C_DECLS_BEGIN
+
+DECLASM(void) idtOnly42(PTSTMICRO pTst);
+
+
+DECLASM(void) tstOverhead(PTSTMICRO pTst);
+DECLASM(void) tstInvlpg0(PTSTMICRO pTst);
+DECLASM(void) tstInvlpgEIP(PTSTMICRO pTst);
+DECLASM(void) tstInvlpgESP(PTSTMICRO pTst);
+DECLASM(void) tstCR3Reload(PTSTMICRO pTst);
+DECLASM(void) tstWPEnable(PTSTMICRO pTst);
+DECLASM(void) tstWPDisable(PTSTMICRO pTst);
+
+
+DECLASM(int)  tstPFR0(PTSTMICRO pTst);
+DECLASM(int)  tstPFR1(PTSTMICRO pTst);
+DECLASM(int)  tstPFR2(PTSTMICRO pTst);
+DECLASM(int)  tstPFR3(PTSTMICRO pTst);
+
+
+
+DECLASM(void) tstTrapHandlerNoErr(void);
+DECLASM(void) tstTrapHandler(void);
+DECLASM(void) tstInterrupt42(void);
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_testcase_tstMicro_h */
diff --git a/src/VBox/VMM/testcase/tstMicro.mac b/src/VBox/VMM/testcase/tstMicro.mac
new file mode 100644
index 00000000..f89ab7bc
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstMicro.mac
@@ -0,0 +1,40 @@
+; $Id: tstMicro.mac $
+;; @file
+; Micro Testcase, profiling special CPU operations.
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%ifndef __tstMicro_mac__
+%define __tstMicro_mac__
+
+
+struc TSTMICRO
+    .RCPtr                   resd 1
+    .RCPtrStack              resd 1
+    .u64TSCR0Start           resq 1
+    .u64TSCRxStart           resq 1
+    .u64TSCR0Enter           resq 1
+    .u64TSCR0Exit            resq 1
+    .u64TSCRxEnd             resq 1
+    .u64TSCR0End             resq 1
+    .cHits                   resd 1
+    .offEIPAdd               resd 1
+    .u32CR2                  resd 1
+    .u32ErrCd                resd 1
+    .u32EIP                  resd 1
+    .OriginalIDTR            resb 6
+endstruc
+
+
+%endif
diff --git a/src/VBox/VMM/testcase/tstMicroRC.cpp b/src/VBox/VMM/testcase/tstMicroRC.cpp
new file mode 100644
index 00000000..7540b510
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstMicroRC.cpp
@@ -0,0 +1,258 @@
+/* $Id: tstMicroRC.cpp $ */
+/** @file
+ * Micro Testcase, profiling special CPU operations - GC Code (hacks).
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/selm.h>
+#include "tstMicro.h"
+
+#include <iprt/errcore.h>
+#include <iprt/asm-amd64-x86.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+DECLEXPORT(int) tstMicroRC(PTSTMICRO pTst, unsigned uTestcase);
+RT_C_DECLS_END
+
+
+/**
+ * Save and load our IDT.
+ *
+ * @param   pTst    Pointer to the instance data.
+ * @param   iIDT    The index of the IDT entry which should be hooked.
+ */
+void idtInstall(PTSTMICRO pTst, int iIDT)
+{
+    RTIDTR Idtr;
+    ASMGetIDTR(&Idtr);
+    if (Idtr.pIdt == (uintptr_t)&pTst->aIDT[0])
+        return;
+    pTst->OriginalIDTR.cbIdt = Idtr.cbIdt;
+    pTst->OriginalIDTR.pIdt  = Idtr.pIdt;
+
+    /*
+     * Copy the IDT.
+     */
+    if (Idtr.cbIdt >= sizeof(pTst->aIDT))
+        Idtr.cbIdt = sizeof(pTst->aIDT) - 1;
+    memcpy(&pTst->aIDT[0], (void *)Idtr.pIdt, Idtr.cbIdt + 1);
+
+
+    /* Hook up IDT entry. */
+    if (iIDT >= 0)
+    {
+        uintptr_t uHandler = (uintptr_t)tstTrapHandlerNoErr;
+        if (    iIDT == 8
+            ||  iIDT == 0xa
+            ||  iIDT == 0xb
+            ||  iIDT == 0xc
+            ||  iIDT == 0xd
+            ||  iIDT == 0xe
+            ||  iIDT == 0x11)
+            uHandler = (uintptr_t)tstTrapHandler;
+        pTst->aIDT[iIDT].Int.u16OffsetHigh  = uHandler >> 16;
+        pTst->aIDT[iIDT].Int.u16OffsetLow   = uHandler & 0xffff;
+        pTst->aIDT[iIDT].Int.u16SegSel      = SELMGetHyperCS(&g_VM);
+        pTst->aIDT[iIDT].Int.u2DPL          = 3;
+        pTst->aIDT[iIDT].Int.u1Present      = 1;
+        pTst->aIDT[iIDT].Int.u1Fixed0       = 0;
+        pTst->aIDT[iIDT].Int.u1Fixed1       = 0;
+        pTst->aIDT[iIDT].Int.u1Fixed2       = 0;
+        pTst->aIDT[iIDT].Int.u1Fixed3       = 0;
+        pTst->aIDT[iIDT].Int.u1Fixed4       = 1;
+        pTst->aIDT[iIDT].Int.u1Fixed5       = 1;
+        pTst->aIDT[iIDT].Int.u132BitGate    = 1;
+        pTst->aIDT[iIDT].Int.u1Fixed6       = 0;
+        pTst->aIDT[iIDT].Int.u5Reserved2    = 0;
+    }
+
+    /* Install int 42h, R3 gate */
+    pTst->aIDT[0x42].Int.u16OffsetHigh  = (uintptr_t)tstInterrupt42 >> 16;
+    pTst->aIDT[0x42].Int.u16OffsetLow   = (uintptr_t)tstInterrupt42 & 0xffff;
+    pTst->aIDT[0x42].Int.u16SegSel      = SELMGetHyperCS(&g_VM);
+    pTst->aIDT[0x42].Int.u2DPL          = 3;
+    pTst->aIDT[0x42].Int.u1Present      = 1;
+    pTst->aIDT[0x42].Int.u1Fixed0       = 0;
+    pTst->aIDT[0x42].Int.u1Fixed1       = 0;
+    pTst->aIDT[0x42].Int.u1Fixed2       = 0;
+    pTst->aIDT[0x42].Int.u1Fixed3       = 0;
+    pTst->aIDT[0x42].Int.u1Fixed4       = 1;
+    pTst->aIDT[0x42].Int.u1Fixed5       = 1;
+    pTst->aIDT[0x42].Int.u132BitGate    = 1;
+    pTst->aIDT[0x42].Int.u1Fixed6       = 0;
+    pTst->aIDT[0x42].Int.u5Reserved2    = 0;
+
+    /*
+     * Load our IDT.
+     */
+    Idtr.pIdt = (uintptr_t)&pTst->aIDT[0];
+    ASMSetIDTR(&Idtr);
+
+    RTIDTR Idtr2;
+    ASMGetIDTR(&Idtr2);
+    Assert(Idtr2.pIdt == (uintptr_t)&pTst->aIDT[0]);
+}
+
+
+/**
+ * Removes all trap overrides except for gate 42.
+ */
+DECLASM(void) idtOnly42(PTSTMICRO pTst)
+{
+    if (pTst->OriginalIDTR.pIdt)
+        memcpy(&pTst->aIDT[0], (void *)(uintptr_t)pTst->OriginalIDTR.pIdt, sizeof(VBOXIDTE) * 32);
+}
+
+
+
+DECLEXPORT(int) tstMicroRC(PTSTMICRO pTst, unsigned uTestcase)
+{
+    RTLogPrintf("pTst=%p uTestcase=%d\n", pTst, uTestcase);
+
+    /*
+     * Validate input.
+     */
+    if (uTestcase >= TSTMICROTEST_MAX)
+        return VERR_INVALID_PARAMETER;
+
+    /*
+     * Clear the results.
+     */
+    pTst->u64TSCR0Start = 0;
+    pTst->u64TSCRxStart = 0;
+    pTst->u64TSCR0Enter = 0;
+    pTst->u64TSCR0Exit = 0;
+    pTst->u64TSCRxEnd = 0;
+    pTst->u64TSCR0End = 0;
+    pTst->cHits = 0;
+    pTst->offEIPAdd = 0;
+    pTst->u32CR2 = 0;
+    pTst->u32EIP = 0;
+    pTst->u32ErrCd = 0;
+    PTSTMICRORESULT pRes = &pTst->aResults[uTestcase];
+    memset(&pTst->aResults[uTestcase], 0, sizeof(pTst->aResults[uTestcase]));
+
+
+    /*
+     * Do the testcase.
+     */
+    int rc = VINF_SUCCESS;
+    switch (uTestcase)
+    {
+        case TSTMICROTEST_OVERHEAD:
+        {
+            tstOverhead(pTst);
+            break;
+        }
+
+        case TSTMICROTEST_INVLPG_0:
+        {
+            tstInvlpg0(pTst);
+            break;
+        }
+
+        case TSTMICROTEST_INVLPG_EIP:
+        {
+            tstInvlpgEIP(pTst);
+            break;
+        }
+
+        case TSTMICROTEST_INVLPG_ESP:
+        {
+            tstInvlpgESP(pTst);
+            break;
+        }
+
+        case TSTMICROTEST_CR3_RELOAD:
+        {
+            tstCR3Reload(pTst);
+            break;
+        }
+
+        case TSTMICROTEST_WP_DISABLE:
+        {
+            tstWPDisable(pTst);
+            break;
+        }
+
+        case TSTMICROTEST_WP_ENABLE:
+        {
+            tstWPEnable(pTst);
+            break;
+        }
+
+        case TSTMICROTEST_PF_R0:
+        {
+            idtInstall(pTst, 0xe);
+            pTst->offEIPAdd = 2;
+            rc = tstPFR0(pTst);
+            break;
+        }
+
+        case TSTMICROTEST_PF_R1:
+        {
+            idtInstall(pTst, 0xe);
+            pTst->offEIPAdd = 2;
+            rc = tstPFR1(pTst);
+            break;
+        }
+
+        case TSTMICROTEST_PF_R2:
+        {
+            idtInstall(pTst, 0xe);
+            pTst->offEIPAdd = 2;
+            rc = tstPFR2(pTst);
+            break;
+        }
+
+        case TSTMICROTEST_PF_R3:
+        {
+            idtInstall(pTst, 0xe);
+            pTst->offEIPAdd = 2;
+            rc = tstPFR3(pTst);
+            break;
+        }
+
+    }
+
+    /*
+     * Compute the results.
+     */
+    if (pTst->u64TSCR0End   && pTst->u64TSCR0Start)
+        pRes->cTotalTicks       = pTst->u64TSCR0End     - pTst->u64TSCR0Start   - pTst->u64Overhead;
+    if (pTst->u64TSCRxStart && pTst->u64TSCR0Start)
+        pRes->cToRxFirstTicks   = pTst->u64TSCRxStart   - pTst->u64TSCR0Start   - pTst->u64Overhead;
+    if (pTst->u64TSCR0Enter && pTst->u64TSCRxStart)
+        pRes->cTrapTicks        = pTst->u64TSCR0Enter   - pTst->u64TSCRxStart   - pTst->u64Overhead;
+    if (pTst->u64TSCRxEnd   && pTst->u64TSCR0Exit)
+        pRes->cToRxTrapTicks    = pTst->u64TSCRxEnd     - pTst->u64TSCR0Exit    - pTst->u64Overhead;
+    if (pTst->u64TSCR0End   && pTst->u64TSCRxEnd)
+        pRes->cToR0Ticks        = pTst->u64TSCR0End     - pTst->u64TSCRxEnd     - pTst->u64Overhead;
+
+    return rc;
+}
+
diff --git a/src/VBox/VMM/testcase/tstMicroRC.def b/src/VBox/VMM/testcase/tstMicroRC.def
new file mode 100644
index 00000000..11b73ce9
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstMicroRC.def
@@ -0,0 +1,28 @@
+;; @file
+;
+; VMM Guest Context Micro Benchmark - Definition file.
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+LIBRARY tstMicroRC.gc
+EXPORTS
+    ; data
+
+    ; code
+    tstMicroRCAsmStart
+    tstMicroRCAsmEnd
+    tstPFR1
+    tstPFR2
+    tstPFR3
+    tstTrapHandlerNoErr
+
diff --git a/src/VBox/VMM/testcase/tstMicroRCA.asm b/src/VBox/VMM/testcase/tstMicroRCA.asm
new file mode 100644
index 00000000..19c9a9b7
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstMicroRCA.asm
@@ -0,0 +1,558 @@
+; $Id: tstMicroRCA.asm $
+;; @file
+; tstMicroRCA
+;
+
+;
+; Copyright (C) 2006-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+;*******************************************************************************
+;*      Header Files                                                           *
+;*******************************************************************************
+%include "VBox/asmdefs.mac"
+%include "iprt/x86.mac"
+%include "VBox/vmm/cpum.mac"
+%include "VBox/err.mac"
+%include "VBox/vmm/vm.mac"
+%include "tstMicro.mac"
+
+
+;*******************************************************************************
+;*      Defined Constants And Macros                                           *
+;*******************************************************************************
+;;
+; Function prolog which saves everything and loads the first parameter into ebx.
+%macro PROLOG 0
+    push    ebp
+    mov     ebp, esp
+    push    ebx
+    push    esi
+    push    edi
+    mov     ebx, [ebp + 8]                      ; pTst
+%endm
+
+;;
+; Function epilog which saves everything and loads the first parameter into ebx.
+%macro EPILOG 0
+    pop     edi
+    pop     esi
+    pop     ebx
+    leave
+%endm
+
+;;
+; Does an rdtsc (trashing edx:eax) and move the result to edi:esi.
+%macro RDTSC_EDI_ESI 0
+    rdtsc
+    xchg    eax, esi
+    xchg    edx, edi
+%endm
+
+;;
+; Does an rdtsc (trashing edx:eax) and move the result to ecx:ebp.
+%macro RDTSC_ECX_EBP 0
+    rdtsc
+    xchg    eax, ebp
+    xchg    edx, ecx
+%endm
+
+;;
+; Saves the result of an instruction profiling operation.
+;
+; Input is in edi:esi (start) and [ebp + 8] points to TSTMICRO.
+; Trashes ebx.
+%macro STORE_INSTR_PRF_RESULT 0
+    mov     ebx, [ebp + 8]
+    mov     [ebx + TSTMICRO.u64TSCR0Start    ], esi
+    mov     [ebx + TSTMICRO.u64TSCR0Start + 4], edi
+    mov     [ebx + TSTMICRO.u64TSCR0End      ], eax
+    mov     [ebx + TSTMICRO.u64TSCR0End   + 4], edx
+%endm
+
+;;
+; Samples the end time of an instruction profiling operation and
+; Saves the result of an instruction profiling operation.
+;
+; Input is in edi:esi (start) and [ebp + 8] points to TSTMICRO.
+; Trashes ebx.
+%macro RDTSC_STORE_INSTR_PRF_RESULT 0
+    rdtsc
+    STORE_INSTR_PRF_RESULT
+%endm
+
+
+;;
+; copies the stack to gabStackCopy and saves ESP and EBP in gpESP and gpEBP.
+;
+; @param    %1      The resume label.
+; @param    ebx     TSTMICRO pointer.
+; @uses     ecx, edi, esi, flags.
+%macro COPY_STACK_ESP_EBP_RESUME 1
+    mov     [gpTst], ebx
+    mov     [gESPResume], esp
+    mov     [gEBPResume], ebp
+    mov     dword [gEIPResume], %1
+
+    mov     esi, esp
+    and     esi, ~0fffh
+    mov     edi, gabStackCopy
+    mov     ecx, 01000h / 4
+    rep     movsd
+
+%endm
+
+
+;*******************************************************************************
+;*  Global Variables                                                           *
+;*******************************************************************************
+BEGINDATA
+gpTst       dd 0
+
+gESPResume  dd 0
+gEBPResume  dd 0
+gEIPResume  dd 0
+
+BEGINBSS
+gabStackCopy resb  4096
+
+extern NAME(idtOnly42)
+extern IMPNAME(g_VM)
+
+BEGINCODE
+EXPORTEDNAME tstMicroRCAsmStart
+
+
+;;
+; Check the overhead of doing rdtsc + two xchg operations.
+;
+BEGINPROC tstOverhead
+    PROLOG
+
+    RDTSC_EDI_ESI
+    RDTSC_STORE_INSTR_PRF_RESULT
+
+    EPILOG
+    ret
+ENDPROC tstOverhead
+
+
+;;
+; Invalidate page 0.
+;
+BEGINPROC tstInvlpg0
+    PROLOG
+
+    RDTSC_EDI_ESI
+    invlpg  [0]
+    RDTSC_STORE_INSTR_PRF_RESULT
+
+    EPILOG
+    ret
+ENDPROC tstInvlpg0
+
+;;
+; Invalidate the current code page.
+;
+BEGINPROC tstInvlpgEIP
+    PROLOG
+
+    RDTSC_EDI_ESI
+    invlpg  [NAME(tstInvlpgEIP)]
+    RDTSC_STORE_INSTR_PRF_RESULT
+
+    EPILOG
+    ret
+ENDPROC tstInvlpgEIP
+
+
+;;
+; Invalidate page 0.
+;
+BEGINPROC tstInvlpgESP
+    PROLOG
+
+    RDTSC_EDI_ESI
+    invlpg  [esp]
+    RDTSC_STORE_INSTR_PRF_RESULT
+
+    EPILOG
+    ret
+ENDPROC tstInvlpgESP
+
+
+;;
+; cr3 reload sequence.
+;
+BEGINPROC tstCR3Reload
+    PROLOG
+
+    RDTSC_EDI_ESI
+    mov     ebx, cr3
+    mov     cr3, ebx
+    RDTSC_STORE_INSTR_PRF_RESULT
+
+    EPILOG
+    ret
+ENDPROC tstCR3Reload
+
+
+;;
+; Enable WP sequence.
+;
+BEGINPROC tstWPEnable
+    PROLOG
+
+    RDTSC_EDI_ESI
+    mov     ebx, cr0
+    or      ebx, X86_CR0_WRITE_PROTECT
+    mov     cr0, ebx
+    rdtsc
+    ; disabled it now or we'll die...
+    and     ebx, ~X86_CR0_WRITE_PROTECT
+    mov     cr0, ebx
+    STORE_INSTR_PRF_RESULT
+
+    EPILOG
+    ret
+ENDPROC tstWPEnable
+
+
+;;
+; Disable WP sequence.
+;
+BEGINPROC tstWPDisable
+    PROLOG
+
+    ;
+    mov     ebx, cr0
+    or      ebx, X86_CR0_WRITE_PROTECT
+    mov     cr0, ebx
+    ; just wast a bit of space and time to try avoid the enable bit tainting the results of the disable.
+    xor     ebx, ebx
+    rdtsc
+    add     ebx, eax
+    rdtsc
+    add     ebx, edx
+    rdtsc
+    sub     ebx, eax
+
+    RDTSC_EDI_ESI
+    mov     ebx, cr0
+    and     ebx, ~X86_CR0_WRITE_PROTECT
+    mov     cr0, ebx
+    RDTSC_STORE_INSTR_PRF_RESULT
+
+    EPILOG
+    ret
+ENDPROC tstWPDisable
+
+
+
+
+;;
+; Generate a #PF accessing page 0 in
+;
+BEGINPROC tstPFR0
+    PROLOG
+
+    COPY_STACK_ESP_EBP_RESUME tstPFR0_Resume
+
+    rdtsc
+    mov     [ebx + TSTMICRO.u64TSCR0Start    ], eax
+    mov     [ebx + TSTMICRO.u64TSCR0Start + 4], edx
+    xor     ebx, ebx                    ; The NULL pointer.
+    xor     ecx, ecx
+    xor     ebp, ebp                    ; ebp:ecx - Rx enter time (0:0).
+    RDTSC_EDI_ESI                       ; edi:esi - Before trap.
+    mov     [ebx], ebx                  ; traps - 2 bytes
+
+    RDTSC_EDI_ESI                       ; edi:esi - Rx entry time.
+    int     42h                         ; we're done.
+
+tstPFR0_Resume:
+    EPILOG
+    ret
+ENDPROC tstPFR0
+
+
+
+;;
+; Generate a #PF accessing page 0 in ring-1
+;
+BEGINPROC_EXPORTED tstPFR1
+    PROLOG
+
+    COPY_STACK_ESP_EBP_RESUME tstPFR1_Resume
+
+    ; Setup iret to execute r1 code.
+    mov     eax, 02069h                 ; load ds and es with R1 selectors.
+    mov     es, eax
+    mov     ds, eax
+    push    dword 01069h                ; ss
+    push    dword [ebx + TSTMICRO.RCPtrStack] ; esp
+    push    dword 0000h                 ; eflags
+    push    dword 01061h                ; cs
+    push    tstPTR1_R1                  ; eip
+
+    rdtsc
+    mov     [ebx + TSTMICRO.u64TSCR0Start    ], eax
+    mov     [ebx + TSTMICRO.u64TSCR0Start + 4], edx
+    iret
+
+    ; R1 code
+tstPTR1_R1:
+    RDTSC_ECX_EBP                       ; ebp:ecx - Rx enter time (0:0).
+    xor     ebx, ebx
+    RDTSC_EDI_ESI                       ; edi:esi - Before trap.
+    mov     [ebx], ebx                  ; traps - 2 bytes
+
+    RDTSC_EDI_ESI                       ; edi:esi - Rx entry time.
+    int     42h                         ; we're done.
+
+    ; Resume in R0
+tstPFR1_Resume:
+    EPILOG
+    ret
+ENDPROC tstPFR1
+
+
+;;
+; Generate a #PF accessing page 0 in ring-2
+;
+BEGINPROC_EXPORTED tstPFR2
+    PROLOG
+
+    COPY_STACK_ESP_EBP_RESUME tstPFR2_Resume
+
+    ; Setup iret to execute r2 code.
+    mov     eax, 0206ah                 ; load ds and es with R2 selectors.
+    mov     es, eax
+    mov     ds, eax
+    push    0206ah                      ; ss
+    push    dword [ebx + TSTMICRO.RCPtrStack] ; esp
+    push    dword 0000h                 ; eflags
+    push    02062h                      ; cs
+    push    tstPTR2_R2                  ; eip
+
+    rdtsc
+    mov     [ebx + TSTMICRO.u64TSCR0Start    ], eax
+    mov     [ebx + TSTMICRO.u64TSCR0Start + 4], edx
+    iret
+
+    ; R2 code
+tstPTR2_R2:
+    RDTSC_ECX_EBP                       ; ebp:ecx - Rx enter time (0:0).
+    xor     ebx, ebx
+    RDTSC_EDI_ESI                       ; edi:esi - Before trap.
+    mov     [ebx], ebx                  ; traps - 2 bytes
+
+    RDTSC_EDI_ESI                       ; edi:esi - Rx entry time.
+    int     42h                         ; we're done.
+
+    ; Resume in R0
+tstPFR2_Resume:
+    EPILOG
+    ret
+ENDPROC tstPFR2
+
+
+;;
+; Generate a #PF accessing page 0 in ring-3
+;
+BEGINPROC_EXPORTED tstPFR3
+    PROLOG
+
+    COPY_STACK_ESP_EBP_RESUME tstPFR3_Resume
+
+    ; Setup iret to execute r3 code.
+    mov     eax, 0306bh                 ; load ds and es with R3 selectors.
+    mov     es, eax
+    mov     ds, eax
+    push    0306bh                      ; ss
+    push    dword [ebx + TSTMICRO.RCPtrStack] ; esp
+    push    dword 0000h                 ; eflags
+    push    03063h                      ; cs
+    push    tstPTR3_R3                  ; eip
+
+    rdtsc
+    mov     [ebx + TSTMICRO.u64TSCR0Start    ], eax
+    mov     [ebx + TSTMICRO.u64TSCR0Start + 4], edx
+    iret
+
+    ; R3 code
+tstPTR3_R3:
+    RDTSC_ECX_EBP                       ; ebp:ecx - Rx enter time (0:0).
+    xor     ebx, ebx
+    RDTSC_EDI_ESI                       ; edi:esi - Before trap.
+    mov     [ebx], ebx                  ; traps - 2 bytes
+
+    RDTSC_EDI_ESI                       ; edi:esi - Rx entry time.
+    int     42h                         ; we're done.
+
+    ; Resume in R0
+tstPFR3_Resume:
+    EPILOG
+    ret
+ENDPROC tstPFR3
+
+
+
+;;
+; Trap handler with error code - share code with tstTrapHandler.
+align 8
+BEGINPROC_EXPORTED tstTrapHandlerNoErr
+    rdtsc
+    push    0ffffffffh
+    jmp     tstTrapHandler_Common
+
+;;
+; Trap handler with error code.
+;           14  SS          (only if ring transition.)
+;           10  ESP         (only if ring transition.)
+;            c  EFLAGS
+;            8  CS
+;            4  EIP
+;            0  Error code. (~0 for vectors which don't take an error code.)
+;; @todo This is a bit of a mess - clean up!
+align 8
+BEGINPROC tstTrapHandler
+    ; get the time
+    rdtsc
+
+tstTrapHandler_Common:
+    xchg    ecx, eax
+    mov     eax, -1                         ; return code
+
+    ; disable WP
+    mov     ebx, cr0
+    and     ebx, ~X86_CR0_WRITE_PROTECT
+    mov     cr0, ebx
+
+    ; first hit, or final hit?
+    mov     ebx, [gpTst]
+    inc     dword [ebx + TSTMICRO.cHits]
+    cmp     dword [ebx + TSTMICRO.cHits], byte 1
+    jne     near tstTrapHandler_Fault
+
+    ; save the results - edx:ecx == r0 enter time, edi:esi == before trap, ecx:ebp == Rx enter time.
+
+    mov     [ebx + TSTMICRO.u64TSCR0Enter    ], ecx
+    mov     [ebx + TSTMICRO.u64TSCR0Enter + 4], edx
+
+    ;mov     [ebx + TSTMICRO.u64TSCRxStart    ], ecx
+    ;mov     [ebx + TSTMICRO.u64TSCRxStart + 4], ebp
+
+    mov     [ebx + TSTMICRO.u64TSCRxStart    ], esi
+    mov     [ebx + TSTMICRO.u64TSCRxStart + 4], edi
+
+    mov     eax, cr2
+    mov     [ebx + TSTMICRO.u32CR2], eax
+    mov     eax, [esp + 0]
+    mov     [ebx + TSTMICRO.u32ErrCd], eax
+    mov     eax, [esp + 4]
+    mov     [ebx + TSTMICRO.u32EIP], eax
+
+    ;
+    ; Advance the EIP and resume.
+    ;
+    mov     ecx, [ebx + TSTMICRO.offEIPAdd]
+    add     [esp + 4], ecx              ; return eip + offEIPAdd
+
+    add     esp, byte 4                 ; skip the err code
+
+    ; take the timestamp before resuming.
+    rdtsc
+    mov     [ebx + TSTMICRO.u64TSCR0Exit    ], eax
+    mov     [ebx + TSTMICRO.u64TSCR0Exit + 4], edx
+    iret
+
+
+tstTrapHandler_Fault:
+    cld
+
+%if 0 ; this has been broken for quite some time
+    ;
+    ; Setup CPUMCTXCORE frame
+    ;
+    push    dword [esp +  4h +  0h]     ; 3ch - eip
+    push    dword [esp + 0ch +  4h]     ; 38h - eflags
+    ;;;;push    dword [esp + 08h +  8h]     ; 34h - cs
+    push    cs;want disasm
+    push    ds                ; c       ; 30h
+    push    es                ;10       ; 2ch
+    push    fs                ;14       ; 28h
+    push    gs                ;18       ; 24h
+    push    dword [esp + 14h + 1ch]     ; 20h - ss
+    push    dword [esp + 10h + 20h]     ; 1ch - esp
+    push    ecx               ;24       ; 18h
+    push    edx               ;28       ; 14h
+    push    ebx               ;2c       ; 10h
+    push    eax               ;30       ;  ch
+    push    ebp               ;34       ;  8h
+    push    esi               ;38       ;  4h
+    push    edi               ;3c       ;  0h
+                              ;40
+%endif
+
+    test    byte [esp + 0ch +  4h], 3h ; check CPL of the cs selector
+    jmp short tstTrapHandler_Fault_Hyper ;; @todo
+    jz short tstTrapHandler_Fault_Hyper
+tstTrapHandler_Fault_Guest:
+    mov     ecx, esp
+    mov     edx, IMP(g_VM)
+    mov     eax, VERR_TRPM_DONT_PANIC
+    call    [edx + VM.pfnVMMRCToHostAsm]
+    jmp short tstTrapHandler_Fault_Guest
+
+tstTrapHandler_Fault_Hyper:
+    ; fix ss:esp.
+    lea     ebx, [esp + 14h + 040h]     ; calc esp at trap
+    mov     [esp + CPUMCTXCORE.esp], ebx; update esp in register frame
+    mov     [esp + CPUMCTXCORE.ss.Sel], ss  ; update ss in register frame
+
+    mov     ecx, esp
+    mov     edx, IMP(g_VM)
+    mov     eax, VERR_TRPM_DONT_PANIC
+    call    [edx + VM.pfnVMMRCToHostAsm]
+    jmp short tstTrapHandler_Fault_Hyper
+
+BEGINPROC tstInterrupt42
+    rdtsc
+    push    byte 0
+    mov     ecx, eax                    ; low ts
+    xor     eax, eax                    ; return code.
+
+    ; save the results - edx:ecx == r0 end time, edi:esi == Rx end time.
+    mov     [ebx + TSTMICRO.u64TSCR0End    ], ecx
+    mov     [ebx + TSTMICRO.u64TSCR0End + 4], edx
+
+    mov     [ebx + TSTMICRO.u64TSCRxEnd    ], esi
+    mov     [ebx + TSTMICRO.u64TSCRxEnd + 4], edi
+
+    ;
+    ; Restore the IDT and stack, and resume the testcase code.
+    ;
+    lidt    [ebx + TSTMICRO.OriginalIDTR]
+
+    mov     edi, esp
+    and     edi, ~0fffh
+    mov     esi, gabStackCopy
+    mov     ecx, 01000h / 4
+    mov     esp, [gESPResume]
+    mov     ebp, [gEBPResume]
+    rep movsd
+
+    jmp     [gEIPResume]
+
+ENDPROC tstTrapHandler
+
+EXPORTEDNAME tstMicroRCAsmEnd
diff --git a/src/VBox/VMM/testcase/tstPDMAsyncCompletion.cpp b/src/VBox/VMM/testcase/tstPDMAsyncCompletion.cpp
new file mode 100644
index 00000000..4ef4a33f
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstPDMAsyncCompletion.cpp
@@ -0,0 +1,274 @@
+/* $Id: tstPDMAsyncCompletion.cpp $ */
+/** @file
+ * PDM Asynchronous Completion Testcase.
+ *
+ * This testcase is for testing the async completion interface.
+ * It implements a file copy program which uses the interface to copy the data.
+ *
+ * Use: ./tstPDMAsyncCompletion <source> <destination>
+ */
+
+/*
+ * Copyright (C) 2008-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
+
+#include "VMInternal.h" /* UVM */
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/pdmasynccompletion.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/cpum.h>
+#include <iprt/errcore.h>
+#include <VBox/log.h>
+#include <VBox/vmm/pdmapi.h>
+#include <iprt/alloc.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/file.h>
+#include <iprt/initterm.h>
+#include <iprt/semaphore.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+#define TESTCASE "tstPDMAsyncCompletion"
+
+/*
+ * Number of simultaneous active tasks.
+ */
+#define NR_TASKS      80
+#define BUFFER_SIZE (64*_1K)
+
+/* Buffers to store data in .*/
+uint8_t                *g_AsyncCompletionTasksBuffer[NR_TASKS];
+PPDMASYNCCOMPLETIONTASK g_AsyncCompletionTasks[NR_TASKS];
+volatile uint32_t       g_cTasksLeft;
+RTSEMEVENT              g_FinishedEventSem;
+
+static DECLCALLBACK(void) AsyncTaskCompleted(PVM pVM, void *pvUser, void *pvUser2, int rc)
+{
+    RT_NOREF4(pVM, pvUser, pvUser2, rc);
+    LogFlow((TESTCASE ": %s: pVM=%p pvUser=%p pvUser2=%p\n", __FUNCTION__, pVM, pvUser, pvUser2));
+
+    uint32_t cTasksStillLeft = ASMAtomicDecU32(&g_cTasksLeft);
+
+    if (!cTasksStillLeft)
+    {
+        /* All tasks processed. Wakeup main. */
+        RTSemEventSignal(g_FinishedEventSem);
+    }
+}
+
+/**
+ *  Entry point.
+ */
+extern "C" DECLEXPORT(int) TrustedMain(int argc, char **argv, char **envp)
+{
+    RT_NOREF1(envp);
+    int rcRet = 0; /* error count */
+    PPDMASYNCCOMPLETIONENDPOINT pEndpointSrc, pEndpointDst;
+
+    RTR3InitExe(argc, &argv, RTR3INIT_FLAGS_SUPLIB);
+
+    if (argc != 3)
+    {
+        RTPrintf(TESTCASE ": Usage is ./tstPDMAsyncCompletion <source> <dest>\n");
+        return 1;
+    }
+
+    PVM pVM;
+    PUVM pUVM;
+    int rc = VMR3Create(1, NULL, NULL, NULL, NULL, NULL, &pVM, &pUVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Little hack to avoid the VM_ASSERT_EMT assertion.
+         */
+        RTTlsSet(pVM->pUVM->vm.s.idxTLS, &pVM->pUVM->aCpus[0]);
+        pVM->pUVM->aCpus[0].pUVM = pVM->pUVM;
+        pVM->pUVM->aCpus[0].vm.s.NativeThreadEMT = RTThreadNativeSelf();
+
+        /*
+         * Create the template.
+         */
+        PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
+        rc = PDMR3AsyncCompletionTemplateCreateInternal(pVM, &pTemplate, AsyncTaskCompleted, NULL, "Test");
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf(TESTCASE ": Error while creating the template!! rc=%d\n", rc);
+            return 1;
+        }
+
+        /*
+         * Create event semaphore.
+         */
+        rc = RTSemEventCreate(&g_FinishedEventSem);
+        AssertRC(rc);
+
+        /*
+         * Create the temporary buffers.
+         */
+        for (unsigned i=0; i < NR_TASKS; i++)
+        {
+            g_AsyncCompletionTasksBuffer[i] = (uint8_t *)RTMemAllocZ(BUFFER_SIZE);
+            if (!g_AsyncCompletionTasksBuffer[i])
+            {
+                RTPrintf(TESTCASE ": out of memory!\n");
+                return ++rcRet;
+            }
+        }
+
+        /* Create the destination as the async completion API can't do this. */
+        RTFILE FileTmp;
+        rc = RTFileOpen(&FileTmp, argv[2], RTFILE_O_READWRITE | RTFILE_O_OPEN_CREATE | RTFILE_O_DENY_NONE);
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf(TESTCASE ": Error while creating the destination!! rc=%d\n", rc);
+            return ++rcRet;
+        }
+        RTFileClose(FileTmp);
+
+        /* Create our file endpoint */
+        rc = PDMR3AsyncCompletionEpCreateForFile(&pEndpointSrc, argv[1], 0, pTemplate);
+        if (RT_SUCCESS(rc))
+        {
+            rc = PDMR3AsyncCompletionEpCreateForFile(&pEndpointDst, argv[2], 0, pTemplate);
+            if (RT_SUCCESS(rc))
+            {
+                PDMR3PowerOn(pVM);
+
+                /* Wait for all threads to finish initialization. */
+                RTThreadSleep(100);
+
+                int fReadPass = true;
+                uint64_t cbSrc;
+                size_t   offSrc = 0;
+                size_t   offDst = 0;
+                uint32_t cTasksUsed = 0;
+
+                rc = PDMR3AsyncCompletionEpGetSize(pEndpointSrc, &cbSrc);
+                if (RT_SUCCESS(rc))
+                {
+                    /* Copy the data. */
+                    for (;;)
+                    {
+                        if (fReadPass)
+                        {
+                            cTasksUsed = (BUFFER_SIZE * NR_TASKS) <= (cbSrc - offSrc)
+                                         ? NR_TASKS
+                                         :   ((cbSrc - offSrc) / BUFFER_SIZE)
+                                           + ((cbSrc - offSrc) % BUFFER_SIZE) > 0
+                                         ? 1
+                                         : 0;
+
+                            g_cTasksLeft = cTasksUsed;
+
+                            for (uint32_t i = 0; i < cTasksUsed; i++)
+                            {
+                                size_t cbRead = ((size_t)offSrc + BUFFER_SIZE) <= cbSrc ? BUFFER_SIZE : cbSrc - offSrc;
+                                RTSGSEG DataSeg;
+
+                                DataSeg.pvSeg = g_AsyncCompletionTasksBuffer[i];
+                                DataSeg.cbSeg = cbRead;
+
+                                rc = PDMR3AsyncCompletionEpRead(pEndpointSrc, offSrc, &DataSeg, 1, cbRead, NULL,
+                                                                &g_AsyncCompletionTasks[i]);
+                                AssertRC(rc);
+                                offSrc += cbRead;
+                                if (offSrc == cbSrc)
+                                    break;
+                            }
+                        }
+                        else
+                        {
+                            g_cTasksLeft = cTasksUsed;
+
+                            for (uint32_t i = 0; i < cTasksUsed; i++)
+                            {
+                                size_t cbWrite = (offDst + BUFFER_SIZE) <= cbSrc ? BUFFER_SIZE : cbSrc - offDst;
+                                RTSGSEG DataSeg;
+
+                                DataSeg.pvSeg = g_AsyncCompletionTasksBuffer[i];
+                                DataSeg.cbSeg = cbWrite;
+
+                                rc = PDMR3AsyncCompletionEpWrite(pEndpointDst, offDst, &DataSeg, 1, cbWrite, NULL,
+                                                                 &g_AsyncCompletionTasks[i]);
+                                AssertRC(rc);
+                                offDst += cbWrite;
+                                if (offDst == cbSrc)
+                                    break;
+                            }
+                        }
+
+                        rc = RTSemEventWait(g_FinishedEventSem, RT_INDEFINITE_WAIT);
+                        AssertRC(rc);
+
+                        if (!fReadPass && (offDst == cbSrc))
+                            break;
+                        else if (fReadPass)
+                            fReadPass = false;
+                        else
+                        {
+                            cTasksUsed = 0;
+                            fReadPass = true;
+                        }
+                    }
+                }
+                else
+                {
+                    RTPrintf(TESTCASE ": Error querying size of the endpoint!! rc=%d\n", rc);
+                    rcRet++;
+                }
+
+                PDMR3PowerOff(pVM);
+                PDMR3AsyncCompletionEpClose(pEndpointDst);
+            }
+            PDMR3AsyncCompletionEpClose(pEndpointSrc);
+        }
+
+        rc = VMR3Destroy(pUVM);
+        AssertMsg(rc == VINF_SUCCESS, ("%s: Destroying VM failed rc=%Rrc!!\n", __FUNCTION__, rc));
+        VMR3ReleaseUVM(pUVM);
+
+        /*
+         * Clean up.
+         */
+        for (uint32_t i = 0; i < NR_TASKS; i++)
+        {
+            RTMemFree(g_AsyncCompletionTasksBuffer[i]);
+        }
+    }
+    else
+    {
+        RTPrintf(TESTCASE ": failed to create VM!! rc=%Rrc\n", rc);
+        rcRet++;
+    }
+
+    return rcRet;
+}
+
+
+#if !defined(VBOX_WITH_HARDENING) || !defined(RT_OS_WINDOWS)
+/**
+ * Main entry point.
+ */
+int main(int argc, char **argv, char **envp)
+{
+    return TrustedMain(argc, argv, envp);
+}
+#endif
+
diff --git a/src/VBox/VMM/testcase/tstPDMAsyncCompletionStress.cpp b/src/VBox/VMM/testcase/tstPDMAsyncCompletionStress.cpp
new file mode 100644
index 00000000..3233d33b
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstPDMAsyncCompletionStress.cpp
@@ -0,0 +1,648 @@
+/* $Id: tstPDMAsyncCompletionStress.cpp $ */
+/** @file
+ * PDM Asynchronous Completion Stresstest.
+ *
+ * This testcase is for stress testing the async completion interface.
+ */
+
+/*
+ * Copyright (C) 2008-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
+
+#include "VMInternal.h" /* UVM */
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/pdmasynccompletion.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pdmthread.h>
+#include <iprt/alloc.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/file.h>
+#include <iprt/initterm.h>
+#include <iprt/semaphore.h>
+#include <iprt/rand.h>
+#include <iprt/string.h>
+#include <iprt/path.h>
+#include <iprt/stream.h>
+#include <iprt/thread.h>
+#include <iprt/param.h>
+#include <iprt/message.h>
+
+#define TESTCASE "tstPDMAsyncCompletionStress"
+
+#if 0
+/** Number of simultaneous open endpoints for reading and writing. */
+#define NR_OPEN_ENDPOINTS 10
+/** Test pattern size. */
+#define TEST_PATTERN_SIZE (100*_1M)
+/** Minimum file size. */
+#define FILE_SIZE_MIN (100 * _1M)
+/** Maximum file size. */
+#define FILE_SIZE_MAX (10000UL * _1M)
+/** Minimum segment size. */
+#define SEGMENT_SIZE_MIN (512)
+/** Maximum segment size. */
+#define SEGMENT_SIZE_MAX (TEST_PATTERN_SIZE)
+/** Maximum number of active tasks. */
+#define TASK_ACTIVE_MAX (1024)
+/** Maximum size of a transfer. */
+#define TASK_TRANSFER_SIZE_MAX (10*_1M)
+#else
+/** Number of simultaneous open endpoints for reading and writing. */
+#define NR_OPEN_ENDPOINTS 5
+/** Test pattern size. */
+#define TEST_PATTERN_SIZE (10*_1M)
+/** Minimum file size. */
+#define FILE_SIZE_MIN (100 * _1M)
+/** Maximum file size. */
+#define FILE_SIZE_MAX (1000UL * _1M)
+/** Minimum segment size. */
+#define SEGMENT_SIZE_MIN (512)
+/** Maximum segment size. */
+#define SEGMENT_SIZE_MAX (TEST_PATTERN_SIZE)
+/** Maximum number of active tasks. */
+#define TASK_ACTIVE_MAX (1)
+/** Maximum size of a transfer. */
+#define TASK_TRANSFER_SIZE_MAX (_1M)
+#endif
+
+/**
+ * Structure defining a file segment.
+ */
+typedef struct PDMACTESTFILESEG
+{
+    /** Start offset in the file. */
+    RTFOFF                     off;
+    /** Size of the segment. */
+    size_t                     cbSegment;
+    /** Pointer to the start of the data in the test pattern used for the segment. */
+    uint8_t                   *pbData;
+} PDMACTESTFILESEG, *PPDMACTESTFILESEG;
+
+/**
+ * Structure defining a I/O task.
+ */
+typedef struct PDMACTESTFILETASK
+{
+    /** Flag whether the task is currently active. */
+    bool                        fActive;
+    /** Flag whether this is a write. */
+    bool                        fWrite;
+    /** Start offset. */
+    RTFOFF                      off;
+    /** Data segment */
+    RTSGSEG                     DataSeg;
+    /** Task handle. */
+    PPDMASYNCCOMPLETIONTASK     hTask;
+} PDMACTESTFILETASK, *PPDMACTESTFILETASK;
+
+/**
+ * Structure defining a test file.
+ */
+typedef struct PDMACTESTFILE
+{
+    /** The PDM async completion endpoint handle. */
+    PPDMASYNCCOMPLETIONENDPOINT hEndpoint;
+    /** Template used for this file. */
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
+    /** Maximum size of the file. */
+    uint64_t                   cbFileMax;
+    /** Current size of the file. */
+    uint64_t                   cbFileCurr;
+    /** Size of a file segment. */
+    size_t                     cbFileSegment;
+    /** Maximum number of segments. */
+    size_t                     cSegments;
+    /** Pointer to the array describing how the file is assembled
+     * of the test pattern. Used for comparing read data to ensure
+     * that no corruption occurred.
+     */
+    PPDMACTESTFILESEG          paSegs;
+    /** Maximum number of active tasks for this endpoint. */
+    uint32_t                   cTasksActiveMax;
+    /** Number of current active tasks. */
+    volatile uint32_t          cTasksActiveCurr;
+    /** Pointer to the array of task. */
+    PPDMACTESTFILETASK         paTasks;
+    /** I/O thread handle. */
+    PPDMTHREAD                 hThread;
+    /** Flag whether the thread should terminate. */
+    bool                       fRunning;
+} PDMACTESTFILE, *PPDMACTESTFILE;
+
+/** Buffer storing the random test pattern. */
+uint8_t *g_pbTestPattern = NULL;
+/** Size of the test pattern. */
+size_t   g_cbTestPattern;
+/** Array holding test files. */
+PDMACTESTFILE g_aTestFiles[NR_OPEN_ENDPOINTS];
+
+static DECLCALLBACK(void) tstPDMACStressTestFileTaskCompleted(PVM pVM, void *pvUser, void *pvUser2, int rcReq);
+
+static void tstPDMACStressTestFileVerify(PPDMACTESTFILE pTestFile, PPDMACTESTFILETASK pTestTask)
+{
+    size_t   cbLeft = pTestTask->DataSeg.cbSeg;
+    RTFOFF   off    = pTestTask->off;
+    uint8_t *pbBuf  = (uint8_t *)pTestTask->DataSeg.pvSeg;
+
+    while (cbLeft)
+    {
+        size_t cbCompare;
+        size_t iSeg = off / pTestFile->cbFileSegment;
+        PPDMACTESTFILESEG pSeg = &pTestFile->paSegs[iSeg];
+        uint8_t *pbTestPattern;
+        unsigned offSeg = off - pSeg->off;
+
+        cbCompare = RT_MIN(cbLeft, pSeg->cbSegment - offSeg);
+        pbTestPattern = pSeg->pbData + offSeg;
+
+        if (memcmp(pbBuf, pbTestPattern, cbCompare))
+        {
+            unsigned idx = 0;
+
+            while (   (idx < cbCompare)
+                   && (pbBuf[idx] == pbTestPattern[idx]))
+                idx++;
+
+            RTMsgError("Unexpected data for off=%RTfoff size=%u\n"
+                       "Expected %c got %c\n",
+                       pTestTask->off + idx, pTestTask->DataSeg.cbSeg,
+                       pbTestPattern[idx], pbBuf[idx]);
+            RTAssertDebugBreak();
+        }
+
+        pbBuf  += cbCompare;
+        off    += cbCompare;
+        cbLeft -= cbCompare;
+    }
+}
+
+static void tstPDMACStressTestFileFillBuffer(PPDMACTESTFILE pTestFile, PPDMACTESTFILETASK pTestTask)
+{
+    uint8_t *pbBuf = (uint8_t *)pTestTask->DataSeg.pvSeg;
+    size_t  cbLeft = pTestTask->DataSeg.cbSeg;
+    RTFOFF  off    = pTestTask->off;
+
+    Assert(pTestTask->fWrite && pTestTask->fActive);
+
+    while (cbLeft)
+    {
+        size_t cbFill;
+        size_t iSeg = off / pTestFile->cbFileSegment;
+        PPDMACTESTFILESEG pSeg = &pTestFile->paSegs[iSeg];
+        uint8_t *pbTestPattern;
+        unsigned offSeg = off - pSeg->off;
+
+        cbFill = RT_MIN(cbLeft, pSeg->cbSegment - offSeg);
+        pbTestPattern = pSeg->pbData + offSeg;
+
+        memcpy(pbBuf, pbTestPattern, cbFill);
+
+        pbBuf  += cbFill;
+        off    += cbFill;
+        cbLeft -= cbFill;
+    }
+}
+
+static int tstPDMACStressTestFileWrite(PPDMACTESTFILE pTestFile, PPDMACTESTFILETASK pTestTask)
+{
+    int rc = VINF_SUCCESS;
+
+    Assert(!pTestTask->fActive);
+
+    pTestTask->fActive       = true;
+    pTestTask->fWrite        = true;
+    pTestTask->DataSeg.cbSeg = RTRandU32Ex(512, TASK_TRANSFER_SIZE_MAX) & ~511;
+
+    uint64_t offMax;
+
+    /* Did we reached the maximum file size */
+    if (pTestFile->cbFileCurr < pTestFile->cbFileMax)
+    {
+        offMax =   (pTestFile->cbFileMax - pTestFile->cbFileCurr) < pTestTask->DataSeg.cbSeg
+                 ? pTestFile->cbFileMax - pTestTask->DataSeg.cbSeg
+                 : pTestFile->cbFileCurr;
+    }
+    else
+        offMax = pTestFile->cbFileMax - pTestTask->DataSeg.cbSeg;
+
+    uint64_t offMin;
+
+    /*
+     * If we reached the maximum file size write in the whole file
+     * otherwise we will enforce the range for random offsets to let it grow
+     * more quickly.
+     */
+    if (pTestFile->cbFileCurr == pTestFile->cbFileMax)
+        offMin = 0;
+    else
+        offMin = RT_MIN(pTestFile->cbFileCurr, offMax);
+
+
+    pTestTask->off = RTRandU64Ex(offMin, offMax) & ~511;
+
+    /* Set new file size of required */
+    if ((uint64_t)pTestTask->off + pTestTask->DataSeg.cbSeg > pTestFile->cbFileCurr)
+        pTestFile->cbFileCurr = pTestTask->off + pTestTask->DataSeg.cbSeg;
+
+    AssertMsg(pTestFile->cbFileCurr <= pTestFile->cbFileMax,
+              ("Current file size (%llu) exceeds final size (%llu)\n",
+              pTestFile->cbFileCurr, pTestFile->cbFileMax));
+
+    /* Allocate data buffer. */
+    pTestTask->DataSeg.pvSeg = RTMemAlloc(pTestTask->DataSeg.cbSeg);
+    if (!pTestTask->DataSeg.pvSeg)
+        return VERR_NO_MEMORY;
+
+    /* Fill data into buffer. */
+    tstPDMACStressTestFileFillBuffer(pTestFile, pTestTask);
+
+    /* Engage */
+    rc = PDMR3AsyncCompletionEpWrite(pTestFile->hEndpoint, pTestTask->off,
+                                     &pTestTask->DataSeg, 1,
+                                     pTestTask->DataSeg.cbSeg,
+                                     pTestTask,
+                                     &pTestTask->hTask);
+
+    return rc;
+}
+
+static int tstPDMACStressTestFileRead(PPDMACTESTFILE pTestFile, PPDMACTESTFILETASK pTestTask)
+{
+    int rc = VINF_SUCCESS;
+
+    Assert(!pTestTask->fActive);
+
+    pTestTask->fActive       = true;
+    pTestTask->fWrite        = false;
+    pTestTask->DataSeg.cbSeg = RTRandU32Ex(1, RT_MIN(pTestFile->cbFileCurr, TASK_TRANSFER_SIZE_MAX));
+
+    AssertMsg(pTestFile->cbFileCurr >= pTestTask->DataSeg.cbSeg, ("Impossible\n"));
+    pTestTask->off = RTRandU64Ex(0, pTestFile->cbFileCurr - pTestTask->DataSeg.cbSeg);
+
+    /* Allocate data buffer. */
+    pTestTask->DataSeg.pvSeg = RTMemAlloc(pTestTask->DataSeg.cbSeg);
+    if (!pTestTask->DataSeg.pvSeg)
+        return VERR_NO_MEMORY;
+
+    /* Engage */
+    rc = PDMR3AsyncCompletionEpRead(pTestFile->hEndpoint, pTestTask->off,
+                                     &pTestTask->DataSeg, 1,
+                                     pTestTask->DataSeg.cbSeg,
+                                     pTestTask,
+                                     &pTestTask->hTask);
+
+    return rc;
+}
+
+/**
+ * Returns true with the given chance in percent.
+ *
+ * @returns true or false
+ * @param   iPercentage   The percentage of the chance to return true.
+ */
+static bool tstPDMACTestIsTrue(int iPercentage)
+{
+    int uRnd = RTRandU32Ex(0, 100);
+
+    return (uRnd <= iPercentage); /* This should be enough for our purpose */
+}
+
+static DECLCALLBACK(int) tstPDMACTestFileThread(PVM pVM, PPDMTHREAD pThread)
+{
+    PPDMACTESTFILE pTestFile = (PPDMACTESTFILE)pThread->pvUser;
+    int iWriteChance = 100; /* Chance to get a write task in percent. */
+    uint32_t cTasksStarted = 0;
+    int rc = VINF_SUCCESS;
+
+    if (pThread->enmState == PDMTHREADSTATE_INITIALIZING)
+        return VINF_SUCCESS;
+
+    while (pTestFile->fRunning)
+    {
+        unsigned iTaskCurr = 0;
+
+
+        /* Fill all tasks */
+        while (   (pTestFile->cTasksActiveCurr < pTestFile->cTasksActiveMax)
+               && (iTaskCurr < pTestFile->cTasksActiveMax))
+        {
+            PPDMACTESTFILETASK pTask = &pTestFile->paTasks[iTaskCurr];
+
+            if (!pTask->fActive)
+            {
+                /* Read or write task? */
+                bool fWrite = tstPDMACTestIsTrue(iWriteChance);
+
+                ASMAtomicIncU32(&pTestFile->cTasksActiveCurr);
+
+                if (fWrite)
+                    rc = tstPDMACStressTestFileWrite(pTestFile, pTask);
+                else
+                    rc = tstPDMACStressTestFileRead(pTestFile, pTask);
+
+                if (rc != VINF_AIO_TASK_PENDING)
+                    tstPDMACStressTestFileTaskCompleted(pVM, pTask, pTestFile, rc);
+
+                cTasksStarted++;
+            }
+
+            iTaskCurr++;
+        }
+
+        /*
+         * Recalc write chance. The bigger the file the lower the chance to have a write.
+         * The minimum chance is 33 percent.
+         */
+        iWriteChance = 100 - (int)(((float)100.0 / pTestFile->cbFileMax) * (float)pTestFile->cbFileCurr);
+        iWriteChance = RT_MAX(33, iWriteChance);
+
+        /* Wait a random amount of time. (1ms - 100ms) */
+        RTThreadSleep(RTRandU32Ex(1, 100));
+    }
+
+    /* Wait for the rest to complete. */
+    while (pTestFile->cTasksActiveCurr)
+        RTThreadSleep(250);
+
+    RTPrintf("Thread exiting: processed %u tasks\n", cTasksStarted);
+    return rc;
+}
+
+static DECLCALLBACK(void) tstPDMACStressTestFileTaskCompleted(PVM pVM, void *pvUser, void *pvUser2, int rcReq)
+{
+    PPDMACTESTFILE pTestFile = (PPDMACTESTFILE)pvUser2;
+    PPDMACTESTFILETASK pTestTask = (PPDMACTESTFILETASK)pvUser;
+    NOREF(pVM); NOREF(rcReq);
+
+    if (pTestTask->fWrite)
+    {
+        /** @todo Do something sensible here. */
+    }
+    else
+    {
+        tstPDMACStressTestFileVerify(pTestFile, pTestTask); /* Will assert if it fails */
+    }
+
+    RTMemFree(pTestTask->DataSeg.pvSeg);
+    pTestTask->fActive = false;
+    AssertMsg(pTestFile->cTasksActiveCurr > 0, ("Trying to complete a non active task\n"));
+    ASMAtomicDecU32(&pTestFile->cTasksActiveCurr);
+}
+
+/**
+ * Sets up a test file creating the I/O thread.
+ *
+ * @returns VBox status code.
+ * @param   pVM          Pointer to the shared VM instance structure.
+ * @param   pTestFile    Pointer to the uninitialized test file structure.
+ * @param   iTestId      Unique test id.
+ */
+static int tstPDMACStressTestFileOpen(PVM pVM, PPDMACTESTFILE pTestFile, unsigned iTestId)
+{
+    int rc = VERR_NO_MEMORY;
+
+    /* Size is a multiple of 512 */
+    pTestFile->cbFileMax     = RTRandU64Ex(FILE_SIZE_MIN, FILE_SIZE_MAX) & ~(511UL);
+    pTestFile->cbFileCurr    = 0;
+    pTestFile->cbFileSegment = RTRandU32Ex(SEGMENT_SIZE_MIN, RT_MIN(pTestFile->cbFileMax, SEGMENT_SIZE_MAX)) & ~((size_t)511);
+
+    Assert(pTestFile->cbFileMax >= pTestFile->cbFileSegment);
+
+    /* Set up the segments array. */
+    pTestFile->cSegments  = pTestFile->cbFileMax / pTestFile->cbFileSegment;
+    pTestFile->cSegments += ((pTestFile->cbFileMax % pTestFile->cbFileSegment) > 0) ? 1 : 0;
+
+    pTestFile->paSegs = (PPDMACTESTFILESEG)RTMemAllocZ(pTestFile->cSegments * sizeof(PDMACTESTFILESEG));
+    if (pTestFile->paSegs)
+    {
+        /* Init the segments */
+        for (unsigned i = 0; i < pTestFile->cSegments; i++)
+        {
+            PPDMACTESTFILESEG pSeg = &pTestFile->paSegs[i];
+
+            pSeg->off       = (RTFOFF)i * pTestFile->cbFileSegment;
+            pSeg->cbSegment = pTestFile->cbFileSegment;
+
+            /* Let the buffer point to a random position in the test pattern. */
+            uint32_t offTestPattern = RTRandU64Ex(0, g_cbTestPattern - pSeg->cbSegment);
+
+            pSeg->pbData = g_pbTestPattern + offTestPattern;
+        }
+
+        /* Init task array. */
+        pTestFile->cTasksActiveMax = RTRandU32Ex(1, TASK_ACTIVE_MAX);
+        pTestFile->paTasks         = (PPDMACTESTFILETASK)RTMemAllocZ(pTestFile->cTasksActiveMax * sizeof(PDMACTESTFILETASK));
+        if (pTestFile->paTasks)
+        {
+            /* Create the template */
+            char szDesc[256];
+
+            RTStrPrintf(szDesc, sizeof(szDesc), "Template-%d", iTestId);
+            rc = PDMR3AsyncCompletionTemplateCreateInternal(pVM, &pTestFile->pTemplate, tstPDMACStressTestFileTaskCompleted,
+                                                            pTestFile, szDesc);
+            if (RT_SUCCESS(rc))
+            {
+                /* Open the endpoint now. Because async completion endpoints cannot create files we have to do it before. */
+                char szFile[RTPATH_MAX];
+
+                RTStrPrintf(szFile, sizeof(szFile), "tstPDMAsyncCompletionStress-%d.tmp", iTestId);
+
+                RTFILE FileTmp;
+                rc = RTFileOpen(&FileTmp, szFile, RTFILE_O_READWRITE | RTFILE_O_CREATE | RTFILE_O_DENY_NONE);
+                if (RT_SUCCESS(rc))
+                {
+                    RTFileClose(FileTmp);
+
+                    rc = PDMR3AsyncCompletionEpCreateForFile(&pTestFile->hEndpoint, szFile, 0, pTestFile->pTemplate);
+                    if (RT_SUCCESS(rc))
+                    {
+                        char szThreadDesc[256];
+
+                        pTestFile->fRunning = true;
+
+                        /* Create the thread creating the I/O for the given file. */
+                        RTStrPrintf(szThreadDesc, sizeof(szThreadDesc), "PDMACThread-%d", iTestId);
+                        rc = PDMR3ThreadCreate(pVM, &pTestFile->hThread, pTestFile, tstPDMACTestFileThread,
+                                               NULL, 0, RTTHREADTYPE_IO, szThreadDesc);
+                        if (RT_SUCCESS(rc))
+                        {
+                            rc = PDMR3ThreadResume(pTestFile->hThread);
+                            AssertRC(rc);
+
+                            RTPrintf(TESTCASE ": Created test file %s cbFileMax=%llu cbFileSegment=%u cSegments=%u cTasksActiveMax=%u\n",
+                                     szFile, pTestFile->cbFileMax, pTestFile->cbFileSegment, pTestFile->cSegments, pTestFile->cTasksActiveMax);
+                            return VINF_SUCCESS;
+                        }
+
+                        PDMR3AsyncCompletionEpClose(pTestFile->hEndpoint);
+                    }
+
+                    RTFileDelete(szFile);
+                }
+
+                PDMR3AsyncCompletionTemplateDestroy(pTestFile->pTemplate);
+            }
+
+            RTMemFree(pTestFile->paTasks);
+        }
+        else
+            rc = VERR_NO_MEMORY;
+
+        RTMemFree(pTestFile->paSegs);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    RTPrintf(TESTCASE ": Opening test file with id %d failed rc=%Rrc\n", iTestId, rc);
+
+    return rc;
+}
+
+/**
+ * Closes a test file.
+ *
+ * @returns nothing.
+ * @param pTestFile    Pointer to the test file.
+ */
+static void tstPDMACStressTestFileClose(PPDMACTESTFILE pTestFile)
+{
+    int rcThread;
+    int rc;
+
+    RTPrintf("Terminating I/O thread, please wait...\n");
+
+    /* Let the thread know that it should terminate. */
+    pTestFile->fRunning = false;
+
+    /* Wait for the thread to terminate. */
+    rc = PDMR3ThreadDestroy(pTestFile->hThread, &rcThread);
+
+    RTPrintf("Thread terminated with status code rc=%Rrc\n", rcThread);
+
+    /* Free resources */
+    RTMemFree(pTestFile->paTasks);
+    RTMemFree(pTestFile->paSegs);
+    PDMR3AsyncCompletionEpClose(pTestFile->hEndpoint);
+    PDMR3AsyncCompletionTemplateDestroy(pTestFile->pTemplate);
+}
+
+/**
+ * Inits the test pattern.
+ *
+ * @returns VBox status code.
+ */
+static int tstPDMACStressTestPatternInit(void)
+{
+    RTPrintf(TESTCASE ": Creating test pattern. Please wait...\n");
+    g_cbTestPattern = TEST_PATTERN_SIZE;
+    g_pbTestPattern = (uint8_t *)RTMemAlloc(g_cbTestPattern);
+    if (!g_pbTestPattern)
+        return VERR_NO_MEMORY;
+
+    RTRandBytes(g_pbTestPattern, g_cbTestPattern);
+    return VINF_SUCCESS;
+}
+
+static void tstPDMACStressTestPatternDestroy(void)
+{
+    RTPrintf(TESTCASE ": Destroying test pattern\n");
+    RTMemFree(g_pbTestPattern);
+}
+
+/**
+ *  Entry point.
+ */
+extern "C" DECLEXPORT(int) TrustedMain(int argc, char **argv, char **envp)
+{
+    RT_NOREF1(envp);
+    int rcRet = 0; /* error count */
+
+    RTR3InitExe(argc, &argv, RTR3INIT_FLAGS_SUPLIB);
+
+    PVM pVM;
+    PUVM pUVM;
+    int rc = VMR3Create(1, NULL, NULL, NULL, NULL, NULL, &pVM, &pUVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Little hack to avoid the VM_ASSERT_EMT assertion.
+         */
+        RTTlsSet(pVM->pUVM->vm.s.idxTLS, &pVM->pUVM->aCpus[0]);
+        pVM->pUVM->aCpus[0].pUVM = pVM->pUVM;
+        pVM->pUVM->aCpus[0].vm.s.NativeThreadEMT = RTThreadNativeSelf();
+
+        rc = tstPDMACStressTestPatternInit();
+        if (RT_SUCCESS(rc))
+        {
+            unsigned cFilesOpened = 0;
+
+            /* Open the endpoints. */
+            for (cFilesOpened = 0; cFilesOpened < NR_OPEN_ENDPOINTS; cFilesOpened++)
+            {
+                rc = tstPDMACStressTestFileOpen(pVM, &g_aTestFiles[cFilesOpened], cFilesOpened);
+                if (RT_FAILURE(rc))
+                    break;
+            }
+
+            if (RT_SUCCESS(rc))
+            {
+                /* Tests are running now. */
+                RTPrintf(TESTCASE ": Successfully opened all files. Running tests forever now or until an error is hit :)\n");
+                RTThreadSleep(RT_INDEFINITE_WAIT);
+            }
+
+            /* Close opened endpoints. */
+            for (unsigned i = 0; i < cFilesOpened; i++)
+                tstPDMACStressTestFileClose(&g_aTestFiles[i]);
+
+            tstPDMACStressTestPatternDestroy();
+        }
+        else
+        {
+            RTPrintf(TESTCASE ": failed to init test pattern!! rc=%Rrc\n", rc);
+            rcRet++;
+        }
+
+        rc = VMR3Destroy(pUVM);
+        AssertMsg(rc == VINF_SUCCESS, ("%s: Destroying VM failed rc=%Rrc!!\n", __FUNCTION__, rc));
+    }
+    else
+    {
+        RTPrintf(TESTCASE ": failed to create VM!! rc=%Rrc\n", rc);
+        rcRet++;
+    }
+
+    return rcRet;
+}
+
+
+#if !defined(VBOX_WITH_HARDENING) || !defined(RT_OS_WINDOWS)
+/**
+ * Main entry point.
+ */
+int main(int argc, char **argv, char **envp)
+{
+    return TrustedMain(argc, argv, envp);
+}
+#endif
+
diff --git a/src/VBox/VMM/testcase/tstSSM-2.cpp b/src/VBox/VMM/testcase/tstSSM-2.cpp
new file mode 100644
index 00000000..7b28e85d
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstSSM-2.cpp
@@ -0,0 +1,86 @@
+/* $Id: tstSSM-2.cpp $ */
+/** @file
+ * Saved State Manager Testcase: Extract the content of a saved state.
+ */
+
+/*
+ * Copyright (C) 2015-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/vmm/ssm.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/getopt.h>
+#include <iprt/errcore.h>
+#include <iprt/file.h>
+#include <iprt/path.h>
+#include <iprt/stream.h>
+#include <iprt/initterm.h>
+
+static RTEXITCODE extractUnit(const char *pszFilename, const char *pszUnitname, const char *pszOutputFilename)
+{
+    PSSMHANDLE pSSM;
+    int rc = SSMR3Open(pszFilename, 0, &pSSM);
+    RTEXITCODE rcExit = RTEXITCODE_FAILURE;
+    if (RT_SUCCESS(rc))
+    {
+        RTFILE hFile;
+        rc = RTFileOpen(&hFile, pszOutputFilename, RTFILE_O_DENY_NONE | RTFILE_O_WRITE | RTFILE_O_CREATE);
+        if (RT_SUCCESS(rc))
+        {
+            uint32_t version = 0;
+            rc = SSMR3Seek(pSSM, pszUnitname, 0 /* iInstance */, &version);
+            size_t cbUnit = 0;
+            if (RT_SUCCESS(rc))
+            {
+                for (;;)
+                {
+                    uint8_t u8;
+                    rc = SSMR3GetU8(pSSM, &u8);
+                    if (RT_FAILURE(rc))
+                        break;
+                    size_t cbWritten;
+                    rc = RTFileWrite(hFile, &u8, sizeof(u8), &cbWritten);
+                    cbUnit++;
+                }
+                RTPrintf("Unit size %zu bytes, version %d\n", cbUnit, version);
+            }
+            else
+                RTPrintf("Cannot find unit '%s' (%Rrc)\n", pszUnitname, rc);
+            RTFileClose(hFile);
+        }
+        else
+            RTPrintf("Cannot open output file '%s' (%Rrc)\n", pszOutputFilename, rc);
+        SSMR3Close(pSSM);
+    }
+    else
+        RTPrintf("Cannot open SSM file '%s' (%Rrc)\n", pszFilename, rc);
+    return rcExit;
+}
+
+int main(int argc, char **argv)
+{
+    int rc = RTR3InitExe(argc, &argv, 0);
+    AssertRCReturn(rc, RTEXITCODE_INIT);
+
+    if (argc != 4)
+    {
+        RTPrintf("Usage: %s <SSM filename> <SSM unitname> <outfile>\n", RTPathFilename(argv[0]));
+        /* don't fail by default */
+        return RTEXITCODE_SUCCESS;
+    }
+    return extractUnit(argv[1], argv[2], argv[3]);
+}
diff --git a/src/VBox/VMM/testcase/tstSSM.cpp b/src/VBox/VMM/testcase/tstSSM.cpp
new file mode 100644
index 00000000..6f157aaf
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstSSM.cpp
@@ -0,0 +1,935 @@
+/* $Id: tstSSM.cpp $ */
+/** @file
+ * Saved State Manager Testcase.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/vmm/ssm.h>
+#include "VMInternal.h" /* createFakeVM */
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/stam.h>
+
+#include <VBox/log.h>
+#include <VBox/sup.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <iprt/file.h>
+#include <iprt/initterm.h>
+#include <iprt/mem.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+#include <iprt/time.h>
+#include <iprt/thread.h>
+#include <iprt/path.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#define TSTSSM_BIG_CONFIG   1
+
+#ifdef TSTSSM_BIG_CONFIG
+# define TSTSSM_ITEM_SIZE    (512*_1M)
+#else
+# define TSTSSM_ITEM_SIZE    (5*_1M)
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+const uint8_t   gabPage[PAGE_SIZE] = {0};
+const char      gachMem1[] = "sdfg\1asdfa\177hjkl;sdfghjkl;dfghjkl;dfghjkl;\0\0asdf;kjasdf;lkjasd;flkjasd;lfkjasd\0;lfk";
+#ifdef TSTSSM_BIG_CONFIG
+uint8_t         gabBigMem[_1M];
+#else
+uint8_t         gabBigMem[8*_1M];
+#endif
+
+
+/** initializes gabBigMem with some non zero stuff. */
+void initBigMem(void)
+{
+#if 0
+    uint32_t *puch = (uint32_t *)&gabBigMem[0];
+    uint32_t *puchEnd = (uint32_t *)&gabBigMem[sizeof(gabBigMem)];
+    uint32_t  u32 = 0xdeadbeef;
+    for (; puch < puchEnd; puch++)
+    {
+        *puch = u32;
+        u32 += 19;
+        u32 = (u32 << 1) | (u32 >> 31);
+    }
+#else
+    uint8_t *pb = &gabBigMem[0];
+    uint8_t *pbEnd = &gabBigMem[sizeof(gabBigMem)];
+    for (; pb < pbEnd; pb += 16)
+    {
+        char szTmp[17];
+        RTStrPrintf(szTmp, sizeof(szTmp), "aaaa%08Xzzzz", (uint32_t)(uintptr_t)pb);
+        memcpy(pb, szTmp, 16);
+    }
+
+    /* add some zero pages */
+    memset(&gabBigMem[sizeof(gabBigMem) / 4],     0, PAGE_SIZE * 4);
+    memset(&gabBigMem[sizeof(gabBigMem) / 4 * 3], 0, PAGE_SIZE * 4);
+#endif
+}
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM handle.
+ * @param   pSSM            SSM operation handle.
+ */
+DECLCALLBACK(int) Item01Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    uint64_t u64Start = RTTimeNanoTS();
+    NOREF(pVM);
+
+    /*
+     * Test writing some memory block.
+     */
+    int rc = SSMR3PutMem(pSSM, gachMem1, sizeof(gachMem1));
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item01: #1 - SSMR3PutMem -> %Rrc\n", rc);
+        return rc;
+    }
+
+    /*
+     * Test writing a zeroterminated string.
+     */
+    rc = SSMR3PutStrZ(pSSM, "String");
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item01: #1 - SSMR3PutMem -> %Rrc\n", rc);
+        return rc;
+    }
+
+
+    /*
+     * Test the individual integer put functions to see that they all work.
+     * (Testcases are also known as "The Land of The Ugly Code"...)
+     */
+#define ITEM(suff,bits, val) \
+    rc = SSMR3Put##suff(pSSM, val); \
+    if (RT_FAILURE(rc)) \
+    { \
+        RTPrintf("Item01: #" #suff " - SSMR3Put" #suff "(," #val ") -> %Rrc\n", rc); \
+        return rc; \
+    }
+    /* copy & past with the load one! */
+    ITEM(U8,  uint8_t,  0xff);
+    ITEM(U8,  uint8_t,  0x0);
+    ITEM(U8,  uint8_t,  1);
+    ITEM(U8,  uint8_t,  42);
+    ITEM(U8,  uint8_t,  230);
+    ITEM(S8,   int8_t,  -128);
+    ITEM(S8,   int8_t,  127);
+    ITEM(S8,   int8_t,  12);
+    ITEM(S8,   int8_t,  -76);
+    ITEM(U16, uint16_t, 0xffff);
+    ITEM(U16, uint16_t, 0x0);
+    ITEM(S16,  int16_t, 32767);
+    ITEM(S16,  int16_t, -32768);
+    ITEM(U32, uint32_t, 4294967295U);
+    ITEM(U32, uint32_t, 0);
+    ITEM(U32, uint32_t, 42);
+    ITEM(U32, uint32_t, 2342342344U);
+    ITEM(S32,  int32_t, -2147483647-1);
+    ITEM(S32,  int32_t, 2147483647);
+    ITEM(S32,  int32_t, 42);
+    ITEM(S32,  int32_t, 568459834);
+    ITEM(S32,  int32_t, -58758999);
+    ITEM(U64, uint64_t, 18446744073709551615ULL);
+    ITEM(U64, uint64_t, 0);
+    ITEM(U64, uint64_t, 42);
+    ITEM(U64, uint64_t, 593023944758394234ULL);
+    ITEM(S64,  int64_t, 9223372036854775807LL);
+    ITEM(S64,  int64_t, -9223372036854775807LL - 1);
+    ITEM(S64,  int64_t, 42);
+    ITEM(S64,  int64_t, 21398723459873LL);
+    ITEM(S64,  int64_t, -5848594593453453245LL);
+#undef ITEM
+
+    uint64_t u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Saved 1st item in %'RI64 ns\n", u64Elapsed);
+    return 0;
+}
+
+/**
+ * Prepare state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        The data layout version.
+ * @param   uPass           The data pass.
+ */
+DECLCALLBACK(int) Item01Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    NOREF(pVM); NOREF(uPass);
+    if (uVersion != 0)
+    {
+        RTPrintf("Item01: uVersion=%#x, expected 0\n", uVersion);
+        return VERR_GENERAL_FAILURE;
+    }
+
+    /*
+     * Load the memory block.
+     */
+    char achTmp[sizeof(gachMem1)];
+    int rc = SSMR3GetMem(pSSM, achTmp, sizeof(gachMem1));
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item01: #1 - SSMR3GetMem -> %Rrc\n", rc);
+        return rc;
+    }
+
+    /*
+     * Load the string.
+     */
+    rc = SSMR3GetStrZ(pSSM, achTmp, sizeof(achTmp));
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item01: #2 - SSMR3GetStrZ -> %Rrc\n", rc);
+        return rc;
+    }
+
+    /*
+     * Test the individual integer put functions to see that they all work.
+     * (Testcases are also known as "The Land of The Ugly Code"...)
+     */
+#define ITEM(suff, type, val) \
+    do { \
+        type var = {0}; \
+        rc = SSMR3Get##suff(pSSM, &var); \
+        if (RT_FAILURE(rc)) \
+        { \
+            RTPrintf("Item01: #" #suff " - SSMR3Get" #suff "(," #val ") -> %Rrc\n", rc); \
+            return rc; \
+        } \
+        if (var != val) \
+        { \
+            RTPrintf("Item01: #" #suff " - SSMR3Get" #suff "(," #val ") -> %d returned wrong value!\n", rc); \
+            return VERR_GENERAL_FAILURE; \
+        } \
+    } while (0)
+    /* copy & past with the load one! */
+    ITEM(U8,  uint8_t,  0xff);
+    ITEM(U8,  uint8_t,  0x0);
+    ITEM(U8,  uint8_t,  1);
+    ITEM(U8,  uint8_t,  42);
+    ITEM(U8,  uint8_t,  230);
+    ITEM(S8,   int8_t,  -128);
+    ITEM(S8,   int8_t,  127);
+    ITEM(S8,   int8_t,  12);
+    ITEM(S8,   int8_t,  -76);
+    ITEM(U16, uint16_t, 0xffff);
+    ITEM(U16, uint16_t, 0x0);
+    ITEM(S16,  int16_t, 32767);
+    ITEM(S16,  int16_t, -32768);
+    ITEM(U32, uint32_t, 4294967295U);
+    ITEM(U32, uint32_t, 0);
+    ITEM(U32, uint32_t, 42);
+    ITEM(U32, uint32_t, 2342342344U);
+    ITEM(S32,  int32_t, -2147483647-1);
+    ITEM(S32,  int32_t, 2147483647);
+    ITEM(S32,  int32_t, 42);
+    ITEM(S32,  int32_t, 568459834);
+    ITEM(S32,  int32_t, -58758999);
+    ITEM(U64, uint64_t, 18446744073709551615ULL);
+    ITEM(U64, uint64_t, 0);
+    ITEM(U64, uint64_t, 42);
+    ITEM(U64, uint64_t, 593023944758394234ULL);
+    ITEM(S64,  int64_t, 9223372036854775807LL);
+    ITEM(S64,  int64_t, -9223372036854775807LL - 1);
+    ITEM(S64,  int64_t, 42);
+    ITEM(S64,  int64_t, 21398723459873LL);
+    ITEM(S64,  int64_t, -5848594593453453245LL);
+#undef ITEM
+
+    return 0;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM handle.
+ * @param   pSSM            SSM operation handle.
+ */
+DECLCALLBACK(int) Item02Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    NOREF(pVM);
+    uint64_t u64Start = RTTimeNanoTS();
+
+    /*
+     * Put the size.
+     */
+    uint32_t cb = sizeof(gabBigMem);
+    int rc = SSMR3PutU32(pSSM, cb);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item02: PutU32 -> %Rrc\n", rc);
+        return rc;
+    }
+
+    /*
+     * Put 8MB of memory to the file in 3 chunks.
+     */
+    uint8_t *pbMem = &gabBigMem[0];
+    uint32_t cbChunk = cb / 47;
+    rc = SSMR3PutMem(pSSM, pbMem, cbChunk);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item02: PutMem(,%p,%#x) -> %Rrc\n", pbMem, cbChunk, rc);
+        return rc;
+    }
+    cb -= cbChunk;
+    pbMem += cbChunk;
+
+    /* next piece. */
+    cbChunk *= 19;
+    rc = SSMR3PutMem(pSSM, pbMem, cbChunk);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item02: PutMem(,%p,%#x) -> %Rrc\n", pbMem, cbChunk, rc);
+        return rc;
+    }
+    cb -= cbChunk;
+    pbMem += cbChunk;
+
+    /* last piece. */
+    cbChunk = cb;
+    rc = SSMR3PutMem(pSSM, pbMem, cbChunk);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item02: PutMem(,%p,%#x) -> %Rrc\n", pbMem, cbChunk, rc);
+        return rc;
+    }
+
+    uint64_t u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Saved 2nd item in %'RI64 ns\n", u64Elapsed);
+    return 0;
+}
+
+/**
+ * Prepare state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        The data layout version.
+ * @param   uPass           The data pass.
+ */
+DECLCALLBACK(int) Item02Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    NOREF(pVM); NOREF(uPass);
+    if (uVersion != 0)
+    {
+        RTPrintf("Item02: uVersion=%#x, expected 0\n", uVersion);
+        return VERR_GENERAL_FAILURE;
+    }
+
+    /*
+     * Load the size.
+     */
+    uint32_t cb;
+    int rc = SSMR3GetU32(pSSM, &cb);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item02: SSMR3GetU32 -> %Rrc\n", rc);
+        return rc;
+    }
+    if (cb != sizeof(gabBigMem))
+    {
+        RTPrintf("Item02: loaded size doesn't match the real thing. %#x != %#x\n", cb, sizeof(gabBigMem));
+        return VERR_GENERAL_FAILURE;
+    }
+
+    /*
+     * Load the memory chunk by chunk.
+     */
+    uint8_t    *pbMem = &gabBigMem[0];
+    char        achTmp[16383];
+    uint32_t    cbChunk = sizeof(achTmp);
+    while (cb > 0)
+    {
+        cbChunk -= 7;
+        if (cbChunk < 64)
+            cbChunk = sizeof(achTmp) - (cbChunk % 47);
+        if (cbChunk > cb)
+            cbChunk = cb;
+        rc = SSMR3GetMem(pSSM, &achTmp[0], cbChunk);
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf("Item02: SSMR3GetMem(,,%#x) -> %d offset %#x\n", cbChunk, rc, pbMem - &gabBigMem[0]);
+            return rc;
+        }
+        if (memcmp(achTmp, pbMem, cbChunk))
+        {
+            RTPrintf("Item02: compare failed. mem offset=%#x cbChunk=%#x\n", pbMem - &gabBigMem[0], cbChunk);
+            return VERR_GENERAL_FAILURE;
+        }
+
+        /* next */
+        pbMem += cbChunk;
+        cb -= cbChunk;
+    }
+
+    return 0;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM handle.
+ * @param   pSSM            SSM operation handle.
+ */
+DECLCALLBACK(int) Item03Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    NOREF(pVM);
+    uint64_t u64Start = RTTimeNanoTS();
+
+    /*
+     * Put the size.
+     */
+    uint32_t cb = TSTSSM_ITEM_SIZE;
+    int rc = SSMR3PutU32(pSSM, cb);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item03: PutU32 -> %Rrc\n", rc);
+        return rc;
+    }
+
+    /*
+     * Put 512 MB page by page.
+     */
+    const uint8_t *pu8Org = &gabBigMem[0];
+    while (cb > 0)
+    {
+        rc = SSMR3PutMem(pSSM, pu8Org, PAGE_SIZE);
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf("Item03: PutMem(,%p,%#x) -> %Rrc\n", pu8Org, PAGE_SIZE, rc);
+            return rc;
+        }
+
+        /* next */
+        cb -= PAGE_SIZE;
+        pu8Org += PAGE_SIZE;
+        if (pu8Org >= &gabBigMem[sizeof(gabBigMem)])
+            pu8Org = &gabBigMem[0];
+    }
+
+    uint64_t u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Saved 3rd item in %'RI64 ns\n", u64Elapsed);
+    return 0;
+}
+
+/**
+ * Prepare state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        The data layout version.
+ * @param   uPass           The data pass.
+ */
+DECLCALLBACK(int) Item03Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    NOREF(pVM); NOREF(uPass);
+    if (uVersion != 123)
+    {
+        RTPrintf("Item03: uVersion=%#x, expected 123\n", uVersion);
+        return VERR_GENERAL_FAILURE;
+    }
+
+    /*
+     * Load the size.
+     */
+    uint32_t cb;
+    int rc = SSMR3GetU32(pSSM, &cb);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item03: SSMR3GetU32 -> %Rrc\n", rc);
+        return rc;
+    }
+    if (cb != TSTSSM_ITEM_SIZE)
+    {
+        RTPrintf("Item03: loaded size doesn't match the real thing. %#x != %#x\n", cb, TSTSSM_ITEM_SIZE);
+        return VERR_GENERAL_FAILURE;
+    }
+
+    /*
+     * Load the memory page by page.
+     */
+    const uint8_t *pu8Org = &gabBigMem[0];
+    while (cb > 0)
+    {
+        char achPage[PAGE_SIZE];
+        rc = SSMR3GetMem(pSSM, &achPage[0], PAGE_SIZE);
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf("Item03: SSMR3GetMem(,,%#x) -> %Rrc offset %#x\n", PAGE_SIZE, rc, TSTSSM_ITEM_SIZE - cb);
+            return rc;
+        }
+        if (memcmp(achPage, pu8Org, PAGE_SIZE))
+        {
+            RTPrintf("Item03: compare failed. mem offset=%#x\n", TSTSSM_ITEM_SIZE - cb);
+            return VERR_GENERAL_FAILURE;
+        }
+
+        /* next */
+        cb -= PAGE_SIZE;
+        pu8Org += PAGE_SIZE;
+        if (pu8Org >= &gabBigMem[sizeof(gabBigMem)])
+            pu8Org = &gabBigMem[0];
+    }
+
+    return 0;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM handle.
+ * @param   pSSM            SSM operation handle.
+ */
+DECLCALLBACK(int) Item04Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    NOREF(pVM);
+    uint64_t u64Start = RTTimeNanoTS();
+
+    /*
+     * Put the size.
+     */
+    uint32_t cb = 512*_1M;
+    int rc = SSMR3PutU32(pSSM, cb);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item04: PutU32 -> %Rrc\n", rc);
+        return rc;
+    }
+
+    /*
+     * Put 512 MB page by page.
+     */
+    while (cb > 0)
+    {
+        rc = SSMR3PutMem(pSSM, gabPage, PAGE_SIZE);
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf("Item04: PutMem(,%p,%#x) -> %Rrc\n", gabPage, PAGE_SIZE, rc);
+            return rc;
+        }
+
+        /* next */
+        cb -= PAGE_SIZE;
+    }
+
+    uint64_t u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Saved 4th item in %'RI64 ns\n", u64Elapsed);
+    return 0;
+}
+
+/**
+ * Prepare state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        The data layout version.
+ * @param   uPass           The data pass.
+ */
+DECLCALLBACK(int) Item04Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    NOREF(pVM); NOREF(uPass);
+    if (uVersion != 42)
+    {
+        RTPrintf("Item04: uVersion=%#x, expected 42\n", uVersion);
+        return VERR_GENERAL_FAILURE;
+    }
+
+    /*
+     * Load the size.
+     */
+    uint32_t cb;
+    int rc = SSMR3GetU32(pSSM, &cb);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item04: SSMR3GetU32 -> %Rrc\n", rc);
+        return rc;
+    }
+    if (cb != 512*_1M)
+    {
+        RTPrintf("Item04: loaded size doesn't match the real thing. %#x != %#x\n", cb, 512*_1M);
+        return VERR_GENERAL_FAILURE;
+    }
+
+    /*
+     * Load the memory page by page.
+     */
+    while (cb > 0)
+    {
+        char achPage[PAGE_SIZE];
+        rc = SSMR3GetMem(pSSM, &achPage[0], PAGE_SIZE);
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf("Item04: SSMR3GetMem(,,%#x) -> %Rrc offset %#x\n", PAGE_SIZE, rc, 512*_1M - cb);
+            return rc;
+        }
+        if (memcmp(achPage, gabPage, PAGE_SIZE))
+        {
+            RTPrintf("Item04: compare failed. mem offset=%#x\n", 512*_1M - cb);
+            return VERR_GENERAL_FAILURE;
+        }
+
+        /* next */
+        cb -= PAGE_SIZE;
+    }
+
+    return 0;
+}
+
+
+/**
+ * Creates a mockup VM structure for testing SSM.
+ *
+ * @returns 0 on success, 1 on failure.
+ * @param   ppVM    Where to store Pointer to the VM.
+ *
+ * @todo    Move this to VMM/VM since it's stuff done by several testcases.
+ */
+static int createFakeVM(PVM *ppVM)
+{
+    /*
+     * Allocate and init the UVM structure.
+     */
+    PUVM pUVM = (PUVM)RTMemPageAllocZ(sizeof(*pUVM));
+    AssertReturn(pUVM, 1);
+    pUVM->u32Magic = UVM_MAGIC;
+    pUVM->vm.s.idxTLS = RTTlsAlloc();
+    int rc = RTTlsSet(pUVM->vm.s.idxTLS, &pUVM->aCpus[0]);
+    if (RT_SUCCESS(rc))
+    {
+        pUVM->aCpus[0].pUVM = pUVM;
+        pUVM->aCpus[0].vm.s.NativeThreadEMT = RTThreadNativeSelf();
+
+        rc = STAMR3InitUVM(pUVM);
+        if (RT_SUCCESS(rc))
+        {
+            rc = MMR3InitUVM(pUVM);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Allocate and init the VM structure.
+                 */
+                PVM pVM = (PVM)RTMemPageAllocZ(sizeof(VM) + sizeof(VMCPU));
+                rc = pVM ? VINF_SUCCESS : VERR_NO_PAGE_MEMORY;
+                if (RT_SUCCESS(rc))
+                {
+                    pVM->enmVMState = VMSTATE_CREATED;
+                    pVM->pVMR3 = pVM;
+                    pVM->pUVM = pUVM;
+                    pVM->cCpus = 1;
+
+                    PVMCPU pVCpu = (PVMCPU)(pVM + 1);
+                    pVCpu->pVMR3 = pVM;
+                    pVCpu->hNativeThread = RTThreadNativeSelf();
+                    pVM->apCpusR3[0] = pVCpu;
+
+                    pUVM->pVM = pVM;
+                    *ppVM = pVM;
+                    return 0;
+                }
+
+                RTPrintf("Fatal error: failed to allocated pages for the VM structure, rc=%Rrc\n", rc);
+            }
+            else
+                RTPrintf("Fatal error: MMR3InitUVM failed, rc=%Rrc\n", rc);
+        }
+        else
+            RTPrintf("Fatal error: SSMR3InitUVM failed, rc=%Rrc\n", rc);
+    }
+    else
+        RTPrintf("Fatal error: RTTlsSet failed, rc=%Rrc\n", rc);
+
+    *ppVM = NULL;
+    return 1;
+}
+
+
+/**
+ * Destroy the VM structure.
+ *
+ * @param   pVM     Pointer to the VM.
+ *
+ * @todo    Move this to VMM/VM since it's stuff done by several testcases.
+ */
+static void destroyFakeVM(PVM pVM)
+{
+    STAMR3TermUVM(pVM->pUVM);
+    MMR3TermUVM(pVM->pUVM);
+}
+
+
+/**
+ *  Entry point.
+ */
+int main(int argc, char **argv)
+{
+    /*
+     * Init runtime and static data.
+     */
+    int rc = RTR3InitExe(argc, &argv, 0);
+    AssertRCReturn(rc, RTEXITCODE_INIT);
+    RTPrintf("tstSSM: TESTING...\n");
+    initBigMem();
+    const char *pszFilename = "SSMTestSave#1";
+
+    /*
+     * Create an fake VM structure and init SSM.
+     */
+    PVM pVM;
+    if (createFakeVM(&pVM))
+        return 1;
+
+    /*
+     * Register a few callbacks.
+     */
+    rc = SSMR3RegisterInternal(pVM, "SSM Testcase Data Item no.1 (all types)", 1, 0, 256,
+                               NULL, NULL, NULL,
+                               NULL, Item01Save, NULL,
+                               NULL, Item01Load, NULL);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3Register #1 -> %Rrc\n", rc);
+        return 1;
+    }
+
+    rc = SSMR3RegisterInternal(pVM, "SSM Testcase Data Item no.2 (rand mem)", 2, 0, _1M * 8,
+                               NULL, NULL, NULL,
+                               NULL, Item02Save, NULL,
+                               NULL, Item02Load, NULL);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3Register #2 -> %Rrc\n", rc);
+        return 1;
+    }
+
+    rc = SSMR3RegisterInternal(pVM, "SSM Testcase Data Item no.3 (big mem)", 0, 123, 512*_1M,
+                               NULL, NULL, NULL,
+                               NULL, Item03Save, NULL,
+                               NULL, Item03Load, NULL);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3Register #3 -> %Rrc\n", rc);
+        return 1;
+    }
+
+    rc = SSMR3RegisterInternal(pVM, "SSM Testcase Data Item no.4 (big zero mem)", 0, 42, 512*_1M,
+                               NULL, NULL, NULL,
+                               NULL, Item04Save, NULL,
+                               NULL, Item04Load, NULL);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3Register #4 -> %Rrc\n", rc);
+        return 1;
+    }
+
+    /*
+     * Attempt a save.
+     */
+    uint64_t u64Start = RTTimeNanoTS();
+    rc = SSMR3Save(pVM, pszFilename, NULL, NULL, SSMAFTER_DESTROY, NULL, NULL);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3Save #1 -> %Rrc\n", rc);
+        return 1;
+    }
+    uint64_t u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Saved in %'RI64 ns\n", u64Elapsed);
+
+    RTFSOBJINFO Info;
+    rc = RTPathQueryInfo(pszFilename, &Info, RTFSOBJATTRADD_NOTHING);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("tstSSM: failed to query file size: %Rrc\n", rc);
+        return 1;
+    }
+    RTPrintf("tstSSM: file size %'RI64 bytes\n", Info.cbObject);
+
+    /*
+     * Attempt a load.
+     */
+    u64Start = RTTimeNanoTS();
+    rc = SSMR3Load(pVM, pszFilename, NULL /*pStreamOps*/, NULL /*pStreamOpsUser*/,
+                   SSMAFTER_RESUME, NULL /*pfnProgress*/, NULL /*pvProgressUser*/);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3Load #1 -> %Rrc\n", rc);
+        return 1;
+    }
+    u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Loaded in %'RI64 ns\n", u64Elapsed);
+
+    /*
+     * Validate it.
+     */
+    u64Start = RTTimeNanoTS();
+    rc = SSMR3ValidateFile(pszFilename, false /* fChecksumIt*/ );
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3ValidateFile #1 -> %Rrc\n", rc);
+        return 1;
+    }
+    u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Validated without checksumming in %'RI64 ns\n", u64Elapsed);
+
+    u64Start = RTTimeNanoTS();
+    rc = SSMR3ValidateFile(pszFilename, true /* fChecksumIt */);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3ValidateFile #1 -> %Rrc\n", rc);
+        return 1;
+    }
+    u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Validated and checksummed in %'RI64 ns\n", u64Elapsed);
+
+    /*
+     * Open it and read.
+     */
+    u64Start = RTTimeNanoTS();
+    PSSMHANDLE pSSM;
+    rc = SSMR3Open(pszFilename, 0, &pSSM);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3Open #1 -> %Rrc\n", rc);
+        return 1;
+    }
+    u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Opened in %'RI64 ns\n", u64Elapsed);
+
+    /* negative */
+    u64Start = RTTimeNanoTS();
+    rc = SSMR3Seek(pSSM, "some unit that doesn't exist", 0, NULL);
+    if (rc != VERR_SSM_UNIT_NOT_FOUND)
+    {
+        RTPrintf("SSMR3Seek #1 negative -> %Rrc\n", rc);
+        return 1;
+    }
+    u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Failed seek in %'RI64 ns\n", u64Elapsed);
+
+    /* another negative, now only the instance number isn't matching. */
+    rc = SSMR3Seek(pSSM, "SSM Testcase Data Item no.2 (rand mem)", 0, NULL);
+    if (rc != VERR_SSM_UNIT_NOT_FOUND)
+    {
+        RTPrintf("SSMR3Seek #1 unit 2 -> %Rrc\n", rc);
+        return 1;
+    }
+
+    /* 2nd unit */
+    rc = SSMR3Seek(pSSM, "SSM Testcase Data Item no.2 (rand mem)", 2, NULL);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3Seek #1 unit 2 -> %Rrc [2]\n", rc);
+        return 1;
+    }
+    uint32_t uVersion = 0xbadc0ded;
+    rc = SSMR3Seek(pSSM, "SSM Testcase Data Item no.2 (rand mem)", 2, &uVersion);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3Seek #1 unit 2 -> %Rrc [3]\n", rc);
+        return 1;
+    }
+    u64Start = RTTimeNanoTS();
+    rc = Item02Load(NULL, pSSM, uVersion, SSM_PASS_FINAL);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item02Load #1 -> %Rrc\n", rc);
+        return 1;
+    }
+    u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Loaded 2nd item in %'RI64 ns\n", u64Elapsed);
+
+    /* 1st unit */
+    uVersion = 0xbadc0ded;
+    rc = SSMR3Seek(pSSM, "SSM Testcase Data Item no.1 (all types)", 1, &uVersion);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3Seek #1 unit 1 -> %Rrc\n", rc);
+        return 1;
+    }
+    u64Start = RTTimeNanoTS();
+    rc = Item01Load(NULL, pSSM, uVersion, SSM_PASS_FINAL);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item01Load #1 -> %Rrc\n", rc);
+        return 1;
+    }
+    u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Loaded 1st item in %'RI64 ns\n", u64Elapsed);
+
+    /* 3st unit */
+    uVersion = 0xbadc0ded;
+    rc = SSMR3Seek(pSSM, "SSM Testcase Data Item no.3 (big mem)", 0, &uVersion);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3Seek #3 unit 1 -> %Rrc\n", rc);
+        return 1;
+    }
+    u64Start = RTTimeNanoTS();
+    rc = Item03Load(NULL, pSSM, uVersion, SSM_PASS_FINAL);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("Item01Load #3 -> %Rrc\n", rc);
+        return 1;
+    }
+    u64Elapsed = RTTimeNanoTS() - u64Start;
+    RTPrintf("tstSSM: Loaded 3rd item in %'RI64 ns\n", u64Elapsed);
+
+    /* close */
+    rc = SSMR3Close(pSSM);
+    if (RT_FAILURE(rc))
+    {
+        RTPrintf("SSMR3Close #1 -> %Rrc\n", rc);
+        return 1;
+    }
+
+    destroyFakeVM(pVM);
+
+    /* delete */
+    RTFileDelete(pszFilename);
+
+    RTPrintf("tstSSM: SUCCESS\n");
+    return 0;
+}
+
diff --git a/src/VBox/VMM/testcase/tstVMM-HM.cpp b/src/VBox/VMM/testcase/tstVMM-HM.cpp
new file mode 100644
index 00000000..2d96f815
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstVMM-HM.cpp
@@ -0,0 +1,121 @@
+/* $Id: tstVMM-HM.cpp $ */
+/** @file
+ * VMM Testcase.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/cpum.h>
+#include <iprt/errcore.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/initterm.h>
+#include <iprt/semaphore.h>
+#include <iprt/stream.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#define TESTCASE    "tstVMM-Hm"
+
+VMMR3DECL(int) VMMDoHmTest(PVM pVM);
+
+
+#if 0
+static DECLCALLBACK(int) tstVmmHmConfigConstructor(PUVM pUVM, PVM pVM, void *pvUser)
+{
+    RT_NOREF2(pUVM, pvUser);
+
+    /*
+     * Get root node first.
+     * This is the only node in the tree.
+     */
+    PCFGMNODE pRoot = CFGMR3GetRoot(pVM);
+    int rc = CFGMR3InsertInteger(pRoot, "RamSize", 32*1024*1024);
+    AssertRC(rc);
+
+    /* rc = CFGMR3InsertInteger(pRoot, "EnableNestedPaging", false);
+    AssertRC(rc); */
+
+    PCFGMNODE pHWVirtExt;
+    rc = CFGMR3InsertNode(pRoot, "HWVirtExt", &pHWVirtExt);
+    AssertRC(rc);
+    rc = CFGMR3InsertInteger(pHWVirtExt, "Enabled", 1);
+    AssertRC(rc);
+
+    return VINF_SUCCESS;
+}
+#endif
+
+int main(int argc, char **argv)
+{
+    RTR3InitExe(argc, &argv, RTR3INIT_FLAGS_SUPLIB);
+
+    /*
+     * Doesn't work and I'm sick of rebooting the machine to try figure out
+     * what the heck is going wrong. (Linux sucks at this)
+     */
+    RTPrintf(TESTCASE ": This testcase hits a bunch of breakpoint assertions which\n"
+             TESTCASE ": causes kernel panics on linux regardless of what\n"
+             TESTCASE ": RTAssertDoBreakpoint returns. Only checked AMD-V on linux.\n");
+#if 1
+    /** @todo Make tstVMM-Hm to cause kernel panics. */
+    return 1;
+#else
+    int     rcRet = 0;                  /* error count. */
+
+    /*
+     * Create empty VM.
+     */
+    RTPrintf(TESTCASE ": Initializing...\n");
+    PVM pVM;
+    PUVM pUVM;
+    int rc = VMR3Create(1, NULL, NULL, NULL, tstVmmHmConfigConstructor, NULL, &pVM, &pUVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Do testing.
+         */
+        RTPrintf(TESTCASE ": Testing...\n");
+        rc = VMR3ReqCallWaitU(pUVM, VMCPUID_ANY, (PFNRT)VMMDoHmTest, 1, pVM);
+        AssertRC(rc);
+
+        STAMR3Dump(pUVM, "*");
+
+        /*
+         * Cleanup.
+         */
+        rc = VMR3Destroy(pUVM);
+        if (RT_FAILURE(rc))
+        {
+            RTPrintf(TESTCASE ": error: failed to destroy vm! rc=%d\n", rc);
+            rcRet++;
+        }
+        VMR3ReleaseUVM(pUVM);
+    }
+    else
+    {
+        RTPrintf(TESTCASE ": fatal error: failed to create vm! rc=%d\n", rc);
+        rcRet++;
+    }
+
+    return rcRet;
+#endif
+}
diff --git a/src/VBox/VMM/testcase/tstVMMFork.cpp b/src/VBox/VMM/testcase/tstVMMFork.cpp
new file mode 100644
index 00000000..1473e820
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstVMMFork.cpp
@@ -0,0 +1,170 @@
+/* $Id: tstVMMFork.cpp $ */
+/** @file
+ * VMM Fork Test.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/vmm.h>
+#include <iprt/errcore.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/initterm.h>
+#include <iprt/stream.h>
+
+#include <errno.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#define TESTCASE        "tstVMMFork"
+#define AUTO_TEST_ARGS  1
+
+VMMR3DECL(int) VMMDoTest(PVM pVM);
+
+
+int main(int argc, char* argv[])
+{
+    int rcErrors = 0;
+
+    /*
+     * Initialize the runtime.
+     */
+    RTR3InitExe(argc, &argv, RTR3INIT_FLAGS_SUPLIB);
+
+#ifndef AUTO_TEST_ARGS
+    if (argc < 2)
+    {
+        RTPrintf("syntax: %s command [args]\n"
+                    "\n"
+                    "command    Command to run under child process in fork.\n"
+                    "[args]     Arguments to command.\n", argv[0]);
+        return 1;
+    }
+#endif
+
+    /*
+     * Create empty VM.
+     */
+    RTPrintf(TESTCASE ": Initializing...\n");
+    PVM pVM;
+    PUVM pUVM;
+    int rc = VMR3Create(1, NULL, NULL, NULL, NULL, NULL, &pVM, &pUVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Do testing.
+         */
+        int iCowTester = 0;
+        char cCowTester = 'a';
+
+#ifndef AUTO_TEST_ARGS
+        int cArgs = argc - 1;
+        char **ppszArgs = &argv[1];
+#else
+        int cArgs = 2;
+        char *ppszArgs[3];
+        ppszArgs[0] = (char *)"/bin/sleep";
+        ppszArgs[1] = (char *)"3";
+        ppszArgs[2] = NULL;
+#endif
+
+        RTPrintf(TESTCASE ": forking current process...\n");
+        pid_t pid = fork();
+        if (pid < 0)
+        {
+            /* Bad. fork() failed! */
+            RTPrintf(TESTCASE ": error: fork() failed.\n");
+            rcErrors++;
+        }
+        else if (pid == 0)
+        {
+            /*
+             * The child process.
+             * Write to some local variables to trigger copy-on-write if it's used.
+             */
+            RTPrintf(TESTCASE ": running child process...\n");
+            RTPrintf(TESTCASE ": writing local variables...\n");
+            iCowTester = 2;
+            cCowTester = 'z';
+
+            RTPrintf(TESTCASE ": calling execv() with command-line:\n");
+            for (int i = 0; i < cArgs; i++)
+                RTPrintf(TESTCASE ": ppszArgs[%d]=%s\n", i, ppszArgs[i]);
+            execv(ppszArgs[0], ppszArgs);
+            RTPrintf(TESTCASE ": error: execv() returned to caller. errno=%d.\n", errno);
+            _exit(-1);
+        }
+        else
+        {
+            /*
+             * The parent process.
+             * Wait for child & run VMM test to ensure things are fine.
+             */
+            int result;
+            while (waitpid(pid, &result, 0) < 0)
+                ;
+            if (!WIFEXITED(result) || WEXITSTATUS(result) != 0)
+            {
+                RTPrintf(TESTCASE ": error: failed to run child process. errno=%d\n", errno);
+                rcErrors++;
+            }
+
+            if (rcErrors == 0)
+            {
+                RTPrintf(TESTCASE ": fork() returned fine.\n");
+                RTPrintf(TESTCASE ": testing VM after fork.\n");
+                VMR3ReqCallWaitU(pUVM, VMCPUID_ANY, (PFNRT)VMMDoTest, 1, pVM);
+
+                STAMR3Dump(pUVM, "*");
+            }
+        }
+
+        if (rcErrors > 0)
+            RTPrintf(TESTCASE ": error: %d error(s) during fork(). Cannot proceed to test the VM.\n", rcErrors);
+        else
+            RTPrintf(TESTCASE ": fork() and VM test, SUCCESS.\n");
+
+        /*
+         * Cleanup.
+         */
+        rc = VMR3PowerOff(pUVM);
+        if (!RT_SUCCESS(rc))
+        {
+            RTPrintf(TESTCASE ": error: failed to power off vm! rc=%Rrc\n", rc);
+            rcErrors++;
+        }
+        rc = VMR3Destroy(pUVM);
+        if (!RT_SUCCESS(rc))
+        {
+            RTPrintf(TESTCASE ": error: failed to destroy vm! rc=%Rrc\n", rc);
+            rcErrors++;
+        }
+        VMR3ReleaseUVM(pUVM);
+    }
+    else
+    {
+        RTPrintf(TESTCASE ": fatal error: failed to create vm! rc=%Rrc\n", rc);
+        rcErrors++;
+    }
+
+    return rcErrors;
+}
diff --git a/src/VBox/VMM/testcase/tstVMMR0CallHost-1.cpp b/src/VBox/VMM/testcase/tstVMMR0CallHost-1.cpp
new file mode 100644
index 00000000..ea871db4
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstVMMR0CallHost-1.cpp
@@ -0,0 +1,181 @@
+/* $Id: tstVMMR0CallHost-1.cpp $ */
+/** @file
+ * Testcase for the VMMR0JMPBUF operations.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <iprt/errcore.h>
+#include <VBox/param.h>
+#include <iprt/alloca.h>
+#include <iprt/initterm.h>
+#include <iprt/rand.h>
+#include <iprt/string.h>
+#include <iprt/stream.h>
+#include <iprt/test.h>
+
+#define IN_VMM_R0
+#define IN_RING0 /* pretent we're in Ring-0 to get the prototypes. */
+#include <VBox/vmm/vmm.h>
+#include "VMMInternal.h"
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#if !defined(VMM_R0_SWITCH_STACK) && !defined(VMM_R0_NO_SWITCH_STACK)
+# error "VMM_R0_SWITCH_STACK or VMM_R0_NO_SWITCH_STACK has to be defined."
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** The jump buffer. */
+static VMMR0JMPBUF          g_Jmp;
+/** The number of jumps we've done. */
+static unsigned volatile    g_cJmps;
+/** Number of bytes allocated last time we called foo(). */
+static size_t volatile      g_cbFoo;
+/** Number of bytes used last time we called foo(). */
+static intptr_t volatile    g_cbFooUsed;
+
+
+int foo(int i, int iZero, int iMinusOne)
+{
+    NOREF(iZero);
+
+    /* allocate a buffer which we fill up to the end. */
+    size_t cb = (i % 1555) + 32;
+    g_cbFoo = cb;
+    char  *pv = (char *)alloca(cb);
+    RTStrPrintf(pv, cb, "i=%d%*s\n", i, cb, "");
+#ifdef VMM_R0_SWITCH_STACK
+    g_cbFooUsed = VMM_STACK_SIZE - ((uintptr_t)pv - (uintptr_t)g_Jmp.pvSavedStack);
+    RTTESTI_CHECK_MSG_RET(g_cbFooUsed < (intptr_t)VMM_STACK_SIZE - 128, ("%#x - (%p - %p) -> %#x; cb=%#x i=%d\n", VMM_STACK_SIZE, pv, g_Jmp.pvSavedStack, g_cbFooUsed, cb, i), -15);
+#elif defined(RT_ARCH_AMD64)
+    g_cbFooUsed = (uintptr_t)g_Jmp.rsp - (uintptr_t)pv;
+    RTTESTI_CHECK_MSG_RET(g_cbFooUsed < VMM_STACK_SIZE - 128, ("%p - %p -> %#x; cb=%#x i=%d\n", g_Jmp.rsp, pv, g_cbFooUsed, cb, i), -15);
+#elif defined(RT_ARCH_X86)
+    g_cbFooUsed = (uintptr_t)g_Jmp.esp - (uintptr_t)pv;
+    RTTESTI_CHECK_MSG_RET(g_cbFooUsed < (intptr_t)VMM_STACK_SIZE - 128, ("%p - %p -> %#x; cb=%#x i=%d\n", g_Jmp.esp, pv, g_cbFooUsed, cb, i), -15);
+#endif
+
+    /* Twice in a row, every 7th time. */
+    if ((i % 7) <= 1)
+    {
+        g_cJmps++;
+        int rc = vmmR0CallRing3LongJmp(&g_Jmp, 42);
+        if (!rc)
+            return i + 10000;
+        return -1;
+    }
+    NOREF(iMinusOne);
+    return i;
+}
+
+
+DECLCALLBACK(int) tst2(intptr_t i, intptr_t i2)
+{
+    RTTESTI_CHECK_MSG_RET(i >= 0 && i <= 8192, ("i=%d is out of range [0..8192]\n", i),      1);
+    RTTESTI_CHECK_MSG_RET(i2 == 0,             ("i2=%d is out of range [0]\n", i2),          1);
+    int iExpect = (i % 7) <= 1 ? i + 10000 : i;
+    int rc = foo(i, 0, -1);
+    RTTESTI_CHECK_MSG_RET(rc == iExpect,       ("i=%d rc=%d expected=%d\n", i, rc, iExpect), 1);
+    return 0;
+}
+
+
+DECLCALLBACK(DECL_NO_INLINE(RT_NOTHING, int)) stackRandom(PVMMR0JMPBUF pJmpBuf, PFNVMMR0SETJMP pfn, PVM pVM, PVMCPU pVCpu)
+{
+#ifdef RT_ARCH_AMD64
+    uint32_t            cbRand  = RTRandU32Ex(1, 96);
+#else
+    uint32_t            cbRand  = 1;
+#endif
+    uint8_t volatile   *pabFuzz = (uint8_t volatile *)alloca(cbRand);
+    memset((void *)pabFuzz, 0xfa, cbRand);
+    int rc = vmmR0CallRing3SetJmp(pJmpBuf, pfn, pVM, pVCpu);
+    memset((void *)pabFuzz, 0xaf, cbRand);
+    return rc;
+}
+
+
+void tst(int iFrom, int iTo, int iInc)
+{
+#ifdef VMM_R0_SWITCH_STACK
+    int const cIterations = iFrom > iTo ? iFrom - iTo : iTo - iFrom;
+    void   *pvPrev = alloca(1);
+#endif
+
+    RTR0PTR R0PtrSaved = g_Jmp.pvSavedStack;
+    RT_ZERO(g_Jmp);
+    g_Jmp.pvSavedStack = R0PtrSaved;
+    memset((void *)g_Jmp.pvSavedStack, '\0', VMM_STACK_SIZE);
+    g_cbFoo = 0;
+    g_cJmps = 0;
+    g_cbFooUsed = 0;
+
+    for (int i = iFrom, iItr = 0; i != iTo; i += iInc, iItr++)
+    {
+        int rc = stackRandom(&g_Jmp, (PFNVMMR0SETJMP)(uintptr_t)tst2, (PVM)(uintptr_t)i, 0);
+        RTTESTI_CHECK_MSG_RETV(rc == 0 || rc == 42, ("i=%d rc=%d setjmp; cbFoo=%#x cbFooUsed=%#x\n", i, rc, g_cbFoo, g_cbFooUsed));
+
+#ifdef VMM_R0_SWITCH_STACK
+        /* Make the stack pointer slide for the second half of the calls. */
+        if (iItr >= cIterations / 2)
+        {
+            /* Note! gcc does funny rounding up of alloca(). */
+            void  *pv2 = alloca((i % 63) | 1);
+            size_t cb2 = (uintptr_t)pvPrev - (uintptr_t)pv2;
+            RTTESTI_CHECK_MSG(cb2 >= 16 && cb2 <= 128, ("cb2=%zu pv2=%p pvPrev=%p iAlloca=%d\n", cb2, pv2, pvPrev, iItr));
+            memset(pv2, 0xff, cb2);
+            memset(pvPrev, 0xee, 1);
+            pvPrev = pv2;
+        }
+#endif
+    }
+    RTTESTI_CHECK_MSG_RETV(g_cJmps, ("No jumps!"));
+    if (g_Jmp.cbUsedAvg || g_Jmp.cUsedTotal)
+        RTTestIPrintf(RTTESTLVL_ALWAYS, "cbUsedAvg=%#x cbUsedMax=%#x cUsedTotal=%#llx\n",
+                      g_Jmp.cbUsedAvg, g_Jmp.cbUsedMax, g_Jmp.cUsedTotal);
+}
+
+
+int main()
+{
+    /*
+     * Init.
+     */
+    RTTEST hTest;
+    RTEXITCODE rcExit = RTTestInitAndCreate("tstVMMR0CallHost-1", &hTest);
+    if (rcExit != RTEXITCODE_SUCCESS)
+        return rcExit;
+    RTTestBanner(hTest);
+
+    g_Jmp.pvSavedStack = (RTR0PTR)RTTestGuardedAllocTail(hTest, VMM_STACK_SIZE);
+
+    /*
+     * Run two test with about 1000 long jumps each.
+     */
+    RTTestSub(hTest, "Increasing stack usage");
+    tst(0, 7000, 1);
+    RTTestSub(hTest, "Decreasing stack usage");
+    tst(7599, 0, -1);
+
+    return RTTestSummaryAndDestroy(hTest);
+}
diff --git a/src/VBox/VMM/testcase/tstVMREQ.cpp b/src/VBox/VMM/testcase/tstVMREQ.cpp
new file mode 100644
index 00000000..6b91af05
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstVMREQ.cpp
@@ -0,0 +1,344 @@
+/* $Id: tstVMREQ.cpp $ */
+/** @file
+ * VMM Testcase.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/initterm.h>
+#include <iprt/semaphore.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/time.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#define TESTCASE    "tstVMREQ"
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** the error count. */
+static int g_cErrors = 0;
+
+
+/**
+ * Testings va_list passing in VMSetRuntimeError.
+ */
+static DECLCALLBACK(void) MyAtRuntimeError(PUVM pUVM, void *pvUser, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
+{
+    NOREF(pUVM);
+    if (strcmp((const char *)pvUser, "user argument"))
+    {
+        RTPrintf(TESTCASE ": pvUser=%p:{%s}!\n", pvUser, (const char *)pvUser);
+        g_cErrors++;
+    }
+    if (fFlags)
+    {
+        RTPrintf(TESTCASE ": fFlags=%#x!\n", fFlags);
+        g_cErrors++;
+    }
+    if (strcmp(pszErrorId, "enum"))
+    {
+        RTPrintf(TESTCASE ": pszErrorId=%p:{%s}!\n", pszErrorId, pszErrorId);
+        g_cErrors++;
+    }
+    if (strcmp(pszFormat, "some %s string"))
+    {
+        RTPrintf(TESTCASE ": pszFormat=%p:{%s}!\n", pszFormat, pszFormat);
+        g_cErrors++;
+    }
+
+    char szBuf[1024];
+    RTStrPrintfV(szBuf, sizeof(szBuf), pszFormat, va);
+    if (strcmp(szBuf, "some error string"))
+    {
+        RTPrintf(TESTCASE ": RTStrPrintfV -> '%s'!\n", szBuf);
+        g_cErrors++;
+    }
+}
+
+
+/**
+ * The function PassVA and PassVA2 calls.
+ */
+static DECLCALLBACK(int) PassVACallback(PUVM pUVM, unsigned u4K, unsigned u1G, const char *pszFormat, va_list *pva)
+{
+    NOREF(pUVM);
+    if (u4K != _4K)
+    {
+        RTPrintf(TESTCASE ": u4K=%#x!\n", u4K);
+        g_cErrors++;
+    }
+    if (u1G != _1G)
+    {
+        RTPrintf(TESTCASE ": u1G=%#x!\n", u1G);
+        g_cErrors++;
+    }
+
+    if (strcmp(pszFormat, "hello %s"))
+    {
+        RTPrintf(TESTCASE ": pszFormat=%p:{%s}!\n", pszFormat, pszFormat);
+        g_cErrors++;
+    }
+
+    char szBuf[1024];
+    RTStrPrintfV(szBuf, sizeof(szBuf), pszFormat, *pva);
+    if (strcmp(szBuf, "hello world"))
+    {
+        RTPrintf(TESTCASE ": RTStrPrintfV -> '%s'!\n", szBuf);
+        g_cErrors++;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Functions that tests passing a va_list * argument in a request,
+ * similar to VMSetRuntimeError.
+ */
+static void PassVA2(PUVM pUVM, const char *pszFormat, va_list va)
+{
+#if 0 /** @todo test if this is a GCC problem only or also happens with AMD64+VCC80... */
+    void *pvVA = &va;
+#else
+    va_list va2;
+    va_copy(va2, va);
+    void *pvVA = &va2;
+#endif
+
+    int rc = VMR3ReqCallWaitU(pUVM, VMCPUID_ANY, (PFNRT)PassVACallback, 5, pUVM, _4K, _1G, pszFormat, pvVA);
+    NOREF(rc);
+
+#if 1
+    va_end(va2);
+#endif
+}
+
+
+/**
+ * Functions that tests passing a va_list * argument in a request,
+ * similar to VMSetRuntimeError.
+ */
+static void PassVA(PUVM pUVM, const char *pszFormat, ...)
+{
+    /* 1st test */
+    va_list va1;
+    va_start(va1, pszFormat);
+    int rc = VMR3ReqCallWaitU(pUVM, VMCPUID_ANY, (PFNRT)PassVACallback, 5, pUVM, _4K, _1G, pszFormat, &va1);
+    va_end(va1);
+    NOREF(rc);
+
+    /* 2nd test */
+    va_list va2;
+    va_start(va2, pszFormat);
+    PassVA2(pUVM, pszFormat, va2);
+    va_end(va2);
+}
+
+
+/**
+ * Thread function which allocates and frees requests like wildfire.
+ */
+static DECLCALLBACK(int) Thread(RTTHREAD hThreadSelf, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    PUVM pUVM = (PUVM)pvUser;
+    NOREF(hThreadSelf);
+
+    for (unsigned i = 0; i < 100000; i++)
+    {
+        PVMREQ          apReq[17];
+        const unsigned  cReqs = i % RT_ELEMENTS(apReq);
+        unsigned        iReq;
+        for (iReq = 0; iReq < cReqs; iReq++)
+        {
+            rc = VMR3ReqAlloc(pUVM, &apReq[iReq], VMREQTYPE_INTERNAL, VMCPUID_ANY);
+            if (RT_FAILURE(rc))
+            {
+                RTPrintf(TESTCASE ": i=%d iReq=%d cReqs=%d rc=%Rrc (alloc)\n", i, iReq, cReqs, rc);
+                return rc;
+            }
+            apReq[iReq]->iStatus = iReq + i;
+        }
+
+        for (iReq = 0; iReq < cReqs; iReq++)
+        {
+            if (apReq[iReq]->iStatus != (int)(iReq + i))
+            {
+                RTPrintf(TESTCASE ": i=%d iReq=%d cReqs=%d: iStatus=%d != %d\n", i, iReq, cReqs, apReq[iReq]->iStatus, iReq + i);
+                return VERR_GENERAL_FAILURE;
+            }
+            rc = VMR3ReqFree(apReq[iReq]);
+            if (RT_FAILURE(rc))
+            {
+                RTPrintf(TESTCASE ": i=%d iReq=%d cReqs=%d rc=%Rrc (free)\n", i, iReq, cReqs, rc);
+                return rc;
+            }
+        }
+        //if (!(i % 10000))
+        //    RTPrintf(TESTCASE ": i=%d\n", i);
+    }
+
+    return VINF_SUCCESS;
+}
+
+static DECLCALLBACK(int)
+tstVMREQConfigConstructor(PUVM pUVM, PVM pVM, void *pvUser)
+{
+    RT_NOREF2(pUVM, pvUser);
+    return CFGMR3ConstructDefaultTree(pVM);
+}
+
+/**
+ *  Entry point.
+ */
+extern "C" DECLEXPORT(int) TrustedMain(int argc, char **argv, char **envp)
+{
+    RT_NOREF1(envp);
+    RTR3InitExe(argc, &argv, RTR3INIT_FLAGS_SUPLIB);
+    RTPrintf(TESTCASE ": TESTING...\n");
+    RTStrmFlush(g_pStdOut);
+
+    /*
+     * Create empty VM.
+     */
+    PUVM pUVM;
+    int rc = VMR3Create(1, NULL, NULL, NULL, tstVMREQConfigConstructor, NULL, NULL, &pUVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Do testing.
+         */
+        uint64_t u64StartTS = RTTimeNanoTS();
+        RTTHREAD Thread0;
+        rc = RTThreadCreate(&Thread0, Thread, pUVM, 0, RTTHREADTYPE_DEFAULT, RTTHREADFLAGS_WAITABLE, "REQ1");
+        if (RT_SUCCESS(rc))
+        {
+            RTTHREAD Thread1;
+            rc = RTThreadCreate(&Thread1, Thread, pUVM, 0, RTTHREADTYPE_DEFAULT, RTTHREADFLAGS_WAITABLE, "REQ1");
+            if (RT_SUCCESS(rc))
+            {
+                int rcThread1;
+                rc = RTThreadWait(Thread1, RT_INDEFINITE_WAIT, &rcThread1);
+                if (RT_FAILURE(rc))
+                {
+                    RTPrintf(TESTCASE ": RTThreadWait(Thread1,,) failed, rc=%Rrc\n", rc);
+                    g_cErrors++;
+                }
+                if (RT_FAILURE(rcThread1))
+                    g_cErrors++;
+            }
+            else
+            {
+                RTPrintf(TESTCASE ": RTThreadCreate(&Thread1,,,,) failed, rc=%Rrc\n", rc);
+                g_cErrors++;
+            }
+
+            int rcThread0;
+            rc = RTThreadWait(Thread0, RT_INDEFINITE_WAIT, &rcThread0);
+            if (RT_FAILURE(rc))
+            {
+                RTPrintf(TESTCASE ": RTThreadWait(Thread1,,) failed, rc=%Rrc\n", rc);
+                g_cErrors++;
+            }
+            if (RT_FAILURE(rcThread0))
+                g_cErrors++;
+        }
+        else
+        {
+            RTPrintf(TESTCASE ": RTThreadCreate(&Thread0,,,,) failed, rc=%Rrc\n", rc);
+            g_cErrors++;
+        }
+        uint64_t u64ElapsedTS = RTTimeNanoTS() - u64StartTS;
+        RTPrintf(TESTCASE  ": %llu ns elapsed\n", u64ElapsedTS);
+        RTStrmFlush(g_pStdOut);
+
+        /*
+         * Print stats.
+         */
+        STAMR3Print(pUVM, "/VM/Req/*");
+
+        /*
+         * Testing va_list fun.
+         */
+        RTPrintf(TESTCASE ": va_list argument test...\n"); RTStrmFlush(g_pStdOut);
+        PassVA(pUVM, "hello %s", "world");
+        VMR3AtRuntimeErrorRegister(pUVM, MyAtRuntimeError, (void *)"user argument");
+        VMSetRuntimeError(VMR3GetVM(pUVM), 0 /*fFlags*/, "enum", "some %s string", "error");
+
+        /*
+         * Cleanup.
+         */
+        rc = VMR3PowerOff(pUVM);
+        if (!RT_SUCCESS(rc))
+        {
+            RTPrintf(TESTCASE ": error: failed to power off vm! rc=%Rrc\n", rc);
+            g_cErrors++;
+        }
+        rc = VMR3Destroy(pUVM);
+        if (!RT_SUCCESS(rc))
+        {
+            RTPrintf(TESTCASE ": error: failed to destroy vm! rc=%Rrc\n", rc);
+            g_cErrors++;
+        }
+        VMR3ReleaseUVM(pUVM);
+    }
+    else if (rc == VERR_SVM_NO_SVM || rc == VERR_VMX_NO_VMX)
+    {
+        RTPrintf(TESTCASE ": Skipped: %Rrc\n", rc);
+        return RTEXITCODE_SKIPPED;
+    }
+    else
+    {
+        RTPrintf(TESTCASE ": fatal error: failed to create vm! rc=%Rrc\n", rc);
+        g_cErrors++;
+    }
+
+    /*
+     * Summary and return.
+     */
+    if (!g_cErrors)
+        RTPrintf(TESTCASE ": SUCCESS\n");
+    else
+        RTPrintf(TESTCASE ": FAILURE - %d errors\n", g_cErrors);
+
+    return !!g_cErrors;
+}
+
+
+#if !defined(VBOX_WITH_HARDENING) || !defined(RT_OS_WINDOWS)
+/**
+ * Main entry point.
+ */
+int main(int argc, char **argv, char **envp)
+{
+    return TrustedMain(argc, argv, envp);
+}
+#endif
+
diff --git a/src/VBox/VMM/testcase/tstVMStruct.h b/src/VBox/VMM/testcase/tstVMStruct.h
new file mode 100644
index 00000000..60920488
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstVMStruct.h
@@ -0,0 +1,1494 @@
+/* $Id: tstVMStruct.h $ */
+/** @file
+ * tstVMMStruct - Statements for generating VM and VMCPU offset and size tests.
+ *
+ * This is used by tstVMStructRC.cpp and tstVMStructDTrace.cpp.  Tests that
+ * are not yet available in DTrace are blocked by VBOX_FOR_DTRACE_LIB.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+#ifndef VBOX_FOR_DTRACE_LIB
+    GEN_CHECK_SIZE(CFGM);
+#endif
+
+    GEN_CHECK_SIZE(X86CPUIDFEATECX);
+
+    GEN_CHECK_SIZE(CPUM);    // has .mac
+    GEN_CHECK_OFF(CPUM, fHostUseFlags);
+    GEN_CHECK_OFF(CPUM, CR4);
+#ifndef VBOX_FOR_DTRACE_LIB
+    GEN_CHECK_OFF(CPUM, u8PortableCpuIdLevel);
+    GEN_CHECK_OFF(CPUM, fPendingRestore);
+#endif
+    GEN_CHECK_OFF(CPUM, aGuestCpuIdPatmStd);
+    GEN_CHECK_OFF(CPUM, aGuestCpuIdPatmExt);
+    GEN_CHECK_OFF(CPUM, aGuestCpuIdPatmCentaur);
+
+    GEN_CHECK_SIZE(CPUMCPU); // has .mac
+    GEN_CHECK_OFF(CPUMCPU, Hyper);
+    GEN_CHECK_OFF(CPUMCPU, Host);
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    GEN_CHECK_OFF(CPUMCPU, aMagic);
+    GEN_CHECK_OFF(CPUMCPU, uMagic);
+#endif
+    GEN_CHECK_OFF(CPUMCPU, Guest);
+    GEN_CHECK_OFF(CPUMCPU, GuestMsrs);
+    GEN_CHECK_OFF(CPUMCPU, fUseFlags);
+    GEN_CHECK_OFF(CPUMCPU, fChanged);
+    GEN_CHECK_OFF(CPUMCPU, u32RetCode);
+#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+    GEN_CHECK_OFF(CPUMCPU, pvApicBase);
+    GEN_CHECK_OFF(CPUMCPU, fApicDisVectors);
+    GEN_CHECK_OFF(CPUMCPU, fX2Apic);
+#endif
+    GEN_CHECK_OFF(CPUMCPU, fRemEntered);
+
+    GEN_CHECK_SIZE(CPUMHOSTCTX);
+    GEN_CHECK_OFF(CPUMHOSTCTX, pXStateR3);
+    GEN_CHECK_OFF(CPUMHOSTCTX, pXStateR0);
+#if HC_ARCH_BITS == 64
+    GEN_CHECK_OFF(CPUMHOSTCTX, rbx);
+    GEN_CHECK_OFF(CPUMHOSTCTX, rdi);
+    GEN_CHECK_OFF(CPUMHOSTCTX, rsi);
+    GEN_CHECK_OFF(CPUMHOSTCTX, rbp);
+    GEN_CHECK_OFF(CPUMHOSTCTX, rsp);
+    GEN_CHECK_OFF(CPUMHOSTCTX, r10);
+    GEN_CHECK_OFF(CPUMHOSTCTX, r11);
+    GEN_CHECK_OFF(CPUMHOSTCTX, r12);
+    GEN_CHECK_OFF(CPUMHOSTCTX, r13);
+    GEN_CHECK_OFF(CPUMHOSTCTX, r14);
+    GEN_CHECK_OFF(CPUMHOSTCTX, r15);
+    GEN_CHECK_OFF(CPUMHOSTCTX, rflags);
+#endif
+#if HC_ARCH_BITS == 32
+    GEN_CHECK_OFF(CPUMHOSTCTX, ebx);
+    GEN_CHECK_OFF(CPUMHOSTCTX, edi);
+    GEN_CHECK_OFF(CPUMHOSTCTX, esi);
+    GEN_CHECK_OFF(CPUMHOSTCTX, ebp);
+    GEN_CHECK_OFF(CPUMHOSTCTX, eflags);
+    GEN_CHECK_OFF(CPUMHOSTCTX, esp);
+#endif
+    GEN_CHECK_OFF(CPUMHOSTCTX, ss);
+    GEN_CHECK_OFF(CPUMHOSTCTX, gs);
+    GEN_CHECK_OFF(CPUMHOSTCTX, fs);
+    GEN_CHECK_OFF(CPUMHOSTCTX, es);
+    GEN_CHECK_OFF(CPUMHOSTCTX, ds);
+    GEN_CHECK_OFF(CPUMHOSTCTX, cs);
+#if HC_ARCH_BITS == 32
+    GEN_CHECK_OFF(CPUMHOSTCTX, cr0);
+    GEN_CHECK_OFF(CPUMHOSTCTX, cr3);
+    GEN_CHECK_OFF(CPUMHOSTCTX, cr4);
+    GEN_CHECK_OFF(CPUMHOSTCTX, dr0);
+    GEN_CHECK_OFF(CPUMHOSTCTX, dr1);
+    GEN_CHECK_OFF(CPUMHOSTCTX, dr2);
+    GEN_CHECK_OFF(CPUMHOSTCTX, dr3);
+    GEN_CHECK_OFF(CPUMHOSTCTX, dr6);
+    GEN_CHECK_OFF(CPUMHOSTCTX, dr7);
+    GEN_CHECK_OFF(CPUMHOSTCTX, gdtr);
+    GEN_CHECK_OFF(CPUMHOSTCTX, idtr);
+    GEN_CHECK_OFF(CPUMHOSTCTX, ldtr);
+    GEN_CHECK_OFF(CPUMHOSTCTX, tr);
+    GEN_CHECK_OFF(CPUMHOSTCTX, SysEnter);
+    GEN_CHECK_OFF(CPUMHOSTCTX, efer);
+#elif HC_ARCH_BITS == 64
+    GEN_CHECK_OFF(CPUMHOSTCTX, cr0);
+    GEN_CHECK_OFF(CPUMHOSTCTX, cr3);
+    GEN_CHECK_OFF(CPUMHOSTCTX, cr4);
+    GEN_CHECK_OFF(CPUMHOSTCTX, cr8);
+    GEN_CHECK_OFF(CPUMHOSTCTX, dr0);
+    GEN_CHECK_OFF(CPUMHOSTCTX, dr1);
+    GEN_CHECK_OFF(CPUMHOSTCTX, dr2);
+    GEN_CHECK_OFF(CPUMHOSTCTX, dr3);
+    GEN_CHECK_OFF(CPUMHOSTCTX, dr6);
+    GEN_CHECK_OFF(CPUMHOSTCTX, dr7);
+    GEN_CHECK_OFF(CPUMHOSTCTX, gdtr);
+    GEN_CHECK_OFF(CPUMHOSTCTX, idtr);
+    GEN_CHECK_OFF(CPUMHOSTCTX, ldtr);
+    GEN_CHECK_OFF(CPUMHOSTCTX, tr);
+    GEN_CHECK_OFF(CPUMHOSTCTX, SysEnter);
+    GEN_CHECK_OFF(CPUMHOSTCTX, FSbase);
+    GEN_CHECK_OFF(CPUMHOSTCTX, GSbase);
+    GEN_CHECK_OFF(CPUMHOSTCTX, efer);
+#else
+# error HC_ARCH_BITS not defined
+#endif
+
+    GEN_CHECK_SIZE(CPUMCTX);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.uMsrHSavePa);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.GCPhysVmcb);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.pVmcbR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.pVmcbR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.HostState);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.uPrevPauseTick);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.cPauseFilter);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.cPauseFilterThreshold);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.fInterceptEvents);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.pvMsrBitmapR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.pvMsrBitmapR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.pvIoBitmapR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.pvIoBitmapR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.svm.HCPhysVmcb);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.GCPhysVmxon);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.GCPhysVmcs);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.enmDiag);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.enmAbort);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.uDiagAux);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.uAbortAux);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.fInVmxRootMode);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.fInVmxNonRootMode);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.fInterceptEvents);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.fNmiUnblockingIret);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pVmcsR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pVmcsR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pShadowVmcsR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pShadowVmcsR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pvVirtApicPageR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pvVirtApicPageR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pvVmreadBitmapR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pvVmreadBitmapR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pvVmwriteBitmapR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pvVmwriteBitmapR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pEntryMsrLoadAreaR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pEntryMsrLoadAreaR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pExitMsrStoreAreaR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pExitMsrStoreAreaR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pExitMsrLoadAreaR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pExitMsrLoadAreaR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pvMsrBitmapR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pvMsrBitmapR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pvIoBitmapR0);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.pvIoBitmapR3);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.uFirstPauseLoopTick);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.uPrevPauseTick);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.uEntryTick);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.offVirtApicWrite);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.fVirtNmiBlocking);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.Msrs);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.HCPhysVmcs);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.HCPhysShadowVmcs);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.HCPhysVmreadBitmap);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.HCPhysVmwriteBitmap);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.HCPhysEntryMsrLoadArea);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.HCPhysExitMsrStoreArea);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.HCPhysExitMsrLoadArea);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.HCPhysMsrBitmap);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.vmx.HCPhysIoBitmap);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.enmHwvirt);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.fGif);
+    GEN_CHECK_OFF(CPUMCTX, hwvirt.fLocalForcedActions);
+    /** @todo NSTVMX: add rest of hwvirt fields when code is more
+     *        finalized. */
+    GEN_CHECK_OFF(CPUMCTX, pXStateR0);
+    GEN_CHECK_OFF(CPUMCTX, pXStateR3);
+    GEN_CHECK_OFF(CPUMCTX, rdi);
+    GEN_CHECK_OFF(CPUMCTX, rsi);
+    GEN_CHECK_OFF(CPUMCTX, rbp);
+    GEN_CHECK_OFF(CPUMCTX, rax);
+    GEN_CHECK_OFF(CPUMCTX, rbx);
+    GEN_CHECK_OFF(CPUMCTX, rdx);
+    GEN_CHECK_OFF(CPUMCTX, rcx);
+    GEN_CHECK_OFF(CPUMCTX, rsp);
+    GEN_CHECK_OFF(CPUMCTX, es);
+#ifndef VBOX_FOR_DTRACE_LIB
+    GEN_CHECK_OFF(CPUMCTX, es.Sel);
+    GEN_CHECK_OFF(CPUMCTX, es.ValidSel);
+    GEN_CHECK_OFF(CPUMCTX, es.fFlags);
+    GEN_CHECK_OFF(CPUMCTX, es.u64Base);
+    GEN_CHECK_OFF(CPUMCTX, es.u32Limit);
+    GEN_CHECK_OFF(CPUMCTX, es.Attr);
+#endif
+    GEN_CHECK_OFF(CPUMCTX, cs);
+    GEN_CHECK_OFF(CPUMCTX, ss);
+    GEN_CHECK_OFF(CPUMCTX, ds);
+    GEN_CHECK_OFF(CPUMCTX, fs);
+    GEN_CHECK_OFF(CPUMCTX, gs);
+    GEN_CHECK_OFF(CPUMCTX, rflags);
+    GEN_CHECK_OFF(CPUMCTX, rip);
+    GEN_CHECK_OFF(CPUMCTX, r8);
+    GEN_CHECK_OFF(CPUMCTX, r9);
+    GEN_CHECK_OFF(CPUMCTX, r10);
+    GEN_CHECK_OFF(CPUMCTX, r11);
+    GEN_CHECK_OFF(CPUMCTX, r12);
+    GEN_CHECK_OFF(CPUMCTX, r13);
+    GEN_CHECK_OFF(CPUMCTX, r14);
+    GEN_CHECK_OFF(CPUMCTX, r15);
+    GEN_CHECK_OFF(CPUMCTX, cr0);
+    GEN_CHECK_OFF(CPUMCTX, cr2);
+    GEN_CHECK_OFF(CPUMCTX, cr3);
+    GEN_CHECK_OFF(CPUMCTX, cr4);
+    GEN_CHECK_OFF(CPUMCTX, dr);
+    GEN_CHECK_OFF(CPUMCTX, gdtr);
+    GEN_CHECK_OFF(CPUMCTX, idtr);
+    GEN_CHECK_OFF(CPUMCTX, ldtr);
+    GEN_CHECK_OFF(CPUMCTX, tr);
+    GEN_CHECK_OFF(CPUMCTX, SysEnter);
+    GEN_CHECK_OFF(CPUMCTX, msrEFER);
+    GEN_CHECK_OFF(CPUMCTX, msrSTAR);
+    GEN_CHECK_OFF(CPUMCTX, msrPAT);
+    GEN_CHECK_OFF(CPUMCTX, msrLSTAR);
+    GEN_CHECK_OFF(CPUMCTX, msrCSTAR);
+    GEN_CHECK_OFF(CPUMCTX, msrSFMASK);
+    GEN_CHECK_OFF(CPUMCTX, msrKERNELGSBASE);
+    GEN_CHECK_OFF(CPUMCTX, ldtr);
+    GEN_CHECK_OFF(CPUMCTX, tr);
+#ifndef VBOX_FOR_DTRACE_LIB
+    GEN_CHECK_OFF(CPUMCTX, tr.Sel);
+    GEN_CHECK_OFF(CPUMCTX, tr.ValidSel);
+    GEN_CHECK_OFF(CPUMCTX, tr.fFlags);
+    GEN_CHECK_OFF(CPUMCTX, tr.u64Base);
+    GEN_CHECK_OFF(CPUMCTX, tr.u32Limit);
+    GEN_CHECK_OFF(CPUMCTX, tr.Attr);
+#endif
+
+    GEN_CHECK_SIZE(CPUMCTXMSRS);
+    GEN_CHECK_SIZE(CPUMCTXCORE);
+
+#ifndef VBOX_FOR_DTRACE_LIB
+    GEN_CHECK_SIZE(STAMRATIOU32);
+    GEN_CHECK_SIZE(AVLOHCPHYSNODECORE);
+    GEN_CHECK_SIZE(AVLOGCPHYSNODECORE);
+    GEN_CHECK_SIZE(AVLROGCPHYSNODECORE);
+    GEN_CHECK_SIZE(AVLOGCPTRNODECORE);
+    GEN_CHECK_SIZE(AVLROGCPTRNODECORE);
+    GEN_CHECK_SIZE(AVLOIOPORTNODECORE);
+    GEN_CHECK_SIZE(AVLROIOPORTNODECORE);
+
+    GEN_CHECK_SIZE(DBGF);
+    GEN_CHECK_OFF(DBGF, bmHardIntBreakpoints);
+    GEN_CHECK_OFF(DBGF, bmSoftIntBreakpoints);
+    GEN_CHECK_OFF(DBGF, bmSelectedEvents);
+    GEN_CHECK_OFF(DBGF, cHardIntBreakpoints);
+    GEN_CHECK_OFF(DBGF, cSoftIntBreakpoints);
+    GEN_CHECK_OFF(DBGF, cSelectedEvents);
+    GEN_CHECK_OFF(DBGF, fAttached);
+    GEN_CHECK_OFF(DBGF, fStoppedInHyper);
+    GEN_CHECK_OFF(DBGF, PingPong);
+    GEN_CHECK_OFF(DBGF, DbgEvent);
+    GEN_CHECK_OFF(DBGF, enmVMMCmd);
+    GEN_CHECK_OFF(DBGF, VMMCmdData);
+    //GEN_CHECK_OFF(DBGF, pInfoFirst);
+    //GEN_CHECK_OFF(DBGF, InfoCritSect);
+    GEN_CHECK_OFF(DBGF, cEnabledHwBreakpoints);
+    GEN_CHECK_OFF(DBGF, cEnabledHwIoBreakpoints);
+    GEN_CHECK_OFF(DBGF, aHwBreakpoints);
+    GEN_CHECK_OFF(DBGF, aBreakpoints);
+    GEN_CHECK_OFF(DBGF, Mmio);
+    GEN_CHECK_OFF(DBGF, PortIo);
+    GEN_CHECK_OFF(DBGF, Int3);
+    //GEN_CHECK_OFF(DBGF, hAsDbLock);
+    //GEN_CHECK_OFF(DBGF, hRegDbLock);
+    //GEN_CHECK_OFF(DBGF, RegSetSpace);
+    //GEN_CHECK_OFF(DBGF, pCurOS);
+    GEN_CHECK_SIZE(DBGFEVENT);
+
+    GEN_CHECK_SIZE(DBGFCPU);
+    GEN_CHECK_OFF(DBGFCPU, iActiveBp);
+    GEN_CHECK_OFF(DBGFCPU, fSingleSteppingRaw);
+    GEN_CHECK_OFF(DBGFCPU, cEvents);
+    GEN_CHECK_OFF(DBGFCPU, aEvents);
+    GEN_CHECK_OFF(DBGFCPU, aEvents[1]);
+    GEN_CHECK_OFF(DBGFCPU, aEvents[1].Event);
+    GEN_CHECK_OFF(DBGFCPU, aEvents[1].Event.enmCtx);
+    GEN_CHECK_OFF(DBGFCPU, aEvents[1].Event.enmType);
+    GEN_CHECK_OFF(DBGFCPU, aEvents[1].Event.u.Bp.iBp);
+    GEN_CHECK_OFF(DBGFCPU, aEvents[1].rip);
+    GEN_CHECK_OFF(DBGFCPU, aEvents[1].enmState);
+    //GEN_CHECK_OFF(DBGFCPU, pGuestRegSet);
+    //GEN_CHECK_OFF(DBGFCPU, pHyperRegSet);
+
+    GEN_CHECK_SIZE(EM);
+    GEN_CHECK_OFF(EM, offVM);
+    GEN_CHECK_OFF(EMCPU, enmState);
+    GEN_CHECK_OFF_DOT(EMCPU, u.achPaddingFatalLongJump);
+    GEN_CHECK_OFF(EMCPU, DisState);
+    GEN_CHECK_OFF(EMCPU, StatForcedActions);
+    GEN_CHECK_OFF(EMCPU, StatTotalClis);
+    GEN_CHECK_OFF(EMCPU, pStatsR3);
+    GEN_CHECK_OFF(EMCPU, pStatsR0);
+    GEN_CHECK_OFF(EMCPU, pStatsRC);
+    GEN_CHECK_OFF(EMCPU, pCliStatTree);
+    GEN_CHECK_OFF(EMCPU, PendingIoPortAccess);
+    GEN_CHECK_OFF_DOT(EMCPU, PendingIoPortAccess.uPort);
+    GEN_CHECK_OFF_DOT(EMCPU, PendingIoPortAccess.cbValue);
+    GEN_CHECK_OFF_DOT(EMCPU, PendingIoPortAccess.uValue);
+    GEN_CHECK_OFF(EMCPU, MWait);
+    GEN_CHECK_OFF_DOT(EMCPU, MWait.fWait);
+    GEN_CHECK_OFF_DOT(EMCPU, MWait.uMWaitRAX);
+    GEN_CHECK_OFF_DOT(EMCPU, MWait.uMWaitRCX);
+    GEN_CHECK_OFF_DOT(EMCPU, MWait.uMonitorRAX);
+    GEN_CHECK_OFF_DOT(EMCPU, MWait.uMonitorRCX);
+    GEN_CHECK_OFF_DOT(EMCPU, MWait.uMonitorRDX);
+
+    GEN_CHECK_SIZE(IEMCPU);
+    GEN_CHECK_OFF(IEMCPU, enmCpuMode);
+    GEN_CHECK_OFF(IEMCPU, fPrefixes);
+    GEN_CHECK_OFF(IEMCPU, abOpcode);
+    GEN_CHECK_OFF(IEMCPU, cActiveMappings);
+    GEN_CHECK_OFF(IEMCPU, iNextMapping);
+    GEN_CHECK_OFF(IEMCPU, aMemMappings);
+    GEN_CHECK_OFF(IEMCPU, aMemMappings[1]);
+    GEN_CHECK_OFF(IEMCPU, aBounceBuffers);
+    GEN_CHECK_OFF(IEMCPU, aBounceBuffers[1]);
+    GEN_CHECK_OFF(IEMCPU, aMemBbMappings);
+    GEN_CHECK_OFF(IEMCPU, aMemBbMappings[1]);
+    GEN_CHECK_OFF(IEMCPU, cLogRelRdMsr);
+    GEN_CHECK_OFF(IEMCPU, cLogRelWrMsr);
+    GEN_CHECK_OFF(IEMCPU, DataTlb);
+    GEN_CHECK_OFF(IEMCPU, CodeTlb);
+
+    GEN_CHECK_SIZE(IOM);
+    GEN_CHECK_OFF(IOM, pTreesRC);
+    GEN_CHECK_OFF(IOM, pTreesR3);
+    GEN_CHECK_OFF(IOM, pTreesR0);
+
+    GEN_CHECK_SIZE(IOMCPU);
+    GEN_CHECK_OFF(IOMCPU, DisState);
+    GEN_CHECK_OFF(IOMCPU, PendingIOPortWrite);
+    GEN_CHECK_OFF(IOMCPU, PendingIOPortWrite.IOPort);
+    GEN_CHECK_OFF(IOMCPU, PendingIOPortWrite.u32Value);
+    GEN_CHECK_OFF(IOMCPU, PendingIOPortWrite.cbValue);
+    GEN_CHECK_OFF(IOMCPU, PendingMmioWrite);
+    GEN_CHECK_OFF(IOMCPU, PendingMmioWrite.GCPhys);
+    GEN_CHECK_OFF(IOMCPU, PendingMmioWrite.abValue);
+    GEN_CHECK_OFF(IOMCPU, PendingMmioWrite.cbValue);
+    GEN_CHECK_OFF(IOMCPU, pMMIORangeLastR3);
+    GEN_CHECK_OFF(IOMCPU, pMMIOStatsLastR3);
+    GEN_CHECK_OFF(IOMCPU, pMMIORangeLastR0);
+    GEN_CHECK_OFF(IOMCPU, pMMIOStatsLastR0);
+    GEN_CHECK_OFF(IOMCPU, pMMIORangeLastRC);
+    GEN_CHECK_OFF(IOMCPU, pMMIOStatsLastRC);
+    GEN_CHECK_OFF(IOMCPU, pRangeLastReadR0);
+    GEN_CHECK_OFF(IOMCPU, pRangeLastReadRC);
+
+    GEN_CHECK_SIZE(IOMMMIORANGE);
+    GEN_CHECK_OFF(IOMMMIORANGE, GCPhys);
+    GEN_CHECK_OFF(IOMMMIORANGE, cb);
+    GEN_CHECK_OFF(IOMMMIORANGE, cRefs);
+    GEN_CHECK_OFF(IOMMMIORANGE, fFlags);
+    GEN_CHECK_OFF(IOMMMIORANGE, pszDesc);
+    GEN_CHECK_OFF(IOMMMIORANGE, pvUserR3);
+    GEN_CHECK_OFF(IOMMMIORANGE, pDevInsR3);
+    GEN_CHECK_OFF(IOMMMIORANGE, pfnWriteCallbackR3);
+    GEN_CHECK_OFF(IOMMMIORANGE, pfnReadCallbackR3);
+    GEN_CHECK_OFF(IOMMMIORANGE, pfnFillCallbackR3);
+    GEN_CHECK_OFF(IOMMMIORANGE, pvUserR0);
+    GEN_CHECK_OFF(IOMMMIORANGE, pDevInsR0);
+    GEN_CHECK_OFF(IOMMMIORANGE, pfnWriteCallbackR0);
+    GEN_CHECK_OFF(IOMMMIORANGE, pfnReadCallbackR0);
+    GEN_CHECK_OFF(IOMMMIORANGE, pfnFillCallbackR0);
+    GEN_CHECK_OFF(IOMMMIORANGE, pvUserRC);
+    GEN_CHECK_OFF(IOMMMIORANGE, pDevInsRC);
+    GEN_CHECK_OFF(IOMMMIORANGE, pfnWriteCallbackRC);
+    GEN_CHECK_OFF(IOMMMIORANGE, pfnReadCallbackRC);
+    GEN_CHECK_OFF(IOMMMIORANGE, pfnFillCallbackRC);
+
+    GEN_CHECK_SIZE(IOMMMIOSTATS);
+    GEN_CHECK_OFF(IOMMMIOSTATS, Accesses);
+    GEN_CHECK_OFF(IOMMMIOSTATS, WriteRZToR3);
+
+    GEN_CHECK_SIZE(IOMIOPORTRANGER0);
+    GEN_CHECK_OFF(IOMIOPORTRANGER0, Port);
+    GEN_CHECK_OFF(IOMIOPORTRANGER0, cPorts);
+    GEN_CHECK_OFF(IOMIOPORTRANGER0, pvUser);
+    GEN_CHECK_OFF(IOMIOPORTRANGER0, pDevIns);
+    GEN_CHECK_OFF(IOMIOPORTRANGER0, pszDesc);
+
+    GEN_CHECK_SIZE(IOMIOPORTRANGERC);
+    GEN_CHECK_OFF(IOMIOPORTRANGERC, Port);
+    GEN_CHECK_OFF(IOMIOPORTRANGERC, cPorts);
+    GEN_CHECK_OFF(IOMIOPORTRANGERC, pvUser);
+    GEN_CHECK_OFF(IOMIOPORTRANGERC, pDevIns);
+    GEN_CHECK_OFF(IOMIOPORTRANGERC, pszDesc);
+
+    GEN_CHECK_SIZE(IOMIOPORTSTATS);
+    GEN_CHECK_OFF(IOMIOPORTSTATS, InR3);
+
+    GEN_CHECK_SIZE(IOMTREES);
+    GEN_CHECK_OFF(IOMTREES, IOPortTreeR3);
+    GEN_CHECK_OFF(IOMTREES, IOPortTreeR0);
+    GEN_CHECK_OFF(IOMTREES, IOPortTreeRC);
+    GEN_CHECK_OFF(IOMTREES, MMIOTree);
+    GEN_CHECK_OFF(IOMTREES, IOPortStatTree);
+    GEN_CHECK_OFF(IOMTREES, MmioStatTree);
+
+    GEN_CHECK_SIZE(MM);
+    GEN_CHECK_OFF(MM, offVM);
+    GEN_CHECK_OFF(MM, offHyperNextStatic);
+    GEN_CHECK_OFF(MM, cbHyperArea);
+    GEN_CHECK_OFF(MM, fDoneMMR3InitPaging);
+    GEN_CHECK_OFF(MM, fPGMInitialized);
+    GEN_CHECK_OFF(MM, offLookupHyper);
+    GEN_CHECK_OFF(MM, pHyperHeapRC);
+    GEN_CHECK_OFF(MM, pHyperHeapR3);
+    GEN_CHECK_OFF(MM, pHyperHeapR0);
+    GEN_CHECK_OFF(MM, pPagePoolR3);
+    GEN_CHECK_OFF(MM, pPagePoolLowR3);
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    GEN_CHECK_OFF(MM, pPagePoolR0);
+    GEN_CHECK_OFF(MM, pPagePoolLowR0);
+#endif
+    GEN_CHECK_OFF(MM, pvDummyPage);
+    GEN_CHECK_OFF(MM, HCPhysDummyPage);
+    GEN_CHECK_OFF(MM, cbRamBase);
+    GEN_CHECK_OFF(MM, cBasePages);
+    GEN_CHECK_OFF(MM, cHandyPages);
+    GEN_CHECK_OFF(MM, cShadowPages);
+    GEN_CHECK_OFF(MM, cFixedPages);
+    GEN_CHECK_SIZE(MMHYPERSTAT);
+    GEN_CHECK_SIZE(MMHYPERCHUNK);
+    GEN_CHECK_SIZE(MMHYPERCHUNKFREE);
+    GEN_CHECK_SIZE(MMHYPERHEAP);
+    GEN_CHECK_OFF(MMHYPERHEAP, u32Magic);
+    GEN_CHECK_OFF(MMHYPERHEAP, cbHeap);
+    GEN_CHECK_OFF(MMHYPERHEAP, pbHeapR3);
+    GEN_CHECK_OFF(MMHYPERHEAP, pVMR3);
+    GEN_CHECK_OFF(MMHYPERHEAP, pbHeapR0);
+    GEN_CHECK_OFF(MMHYPERHEAP, pVMR0);
+    GEN_CHECK_OFF(MMHYPERHEAP, pbHeapRC);
+    GEN_CHECK_OFF(MMHYPERHEAP, pVMRC);
+    GEN_CHECK_OFF(MMHYPERHEAP, cbFree);
+    GEN_CHECK_OFF(MMHYPERHEAP, offFreeHead);
+    GEN_CHECK_OFF(MMHYPERHEAP, offFreeTail);
+    GEN_CHECK_OFF(MMHYPERHEAP, offPageAligned);
+    GEN_CHECK_OFF(MMHYPERHEAP, HyperHeapStatTree);
+    GEN_CHECK_SIZE(MMLOOKUPHYPER);
+    GEN_CHECK_OFF(MMLOOKUPHYPER, offNext);
+    GEN_CHECK_OFF(MMLOOKUPHYPER, off);
+    GEN_CHECK_OFF(MMLOOKUPHYPER, cb);
+    GEN_CHECK_OFF(MMLOOKUPHYPER, enmType);
+    GEN_CHECK_OFF_DOT(MMLOOKUPHYPER, u.Locked.pvR3);
+    GEN_CHECK_OFF_DOT(MMLOOKUPHYPER, u.Locked.pvR0);
+    GEN_CHECK_OFF_DOT(MMLOOKUPHYPER, u.Locked.paHCPhysPages);
+    GEN_CHECK_OFF_DOT(MMLOOKUPHYPER, u.HCPhys.pvR3);
+    GEN_CHECK_OFF_DOT(MMLOOKUPHYPER, u.HCPhys.HCPhys);
+    GEN_CHECK_OFF_DOT(MMLOOKUPHYPER, u.GCPhys.GCPhys);
+    GEN_CHECK_OFF_DOT(MMLOOKUPHYPER, u.MMIO2.pDevIns);
+    GEN_CHECK_OFF_DOT(MMLOOKUPHYPER, u.MMIO2.iSubDev);
+    GEN_CHECK_OFF_DOT(MMLOOKUPHYPER, u.MMIO2.iRegion);
+    GEN_CHECK_OFF_DOT(MMLOOKUPHYPER, u.MMIO2.off);
+    GEN_CHECK_OFF(MMLOOKUPHYPER, pszDesc);
+
+    GEN_CHECK_SIZE(NEM);
+    GEN_CHECK_SIZE(NEMCPU);
+
+    GEN_CHECK_SIZE(PDM);
+    GEN_CHECK_OFF(PDM, CritSect);
+    GEN_CHECK_OFF(PDM, NopCritSect);
+    GEN_CHECK_OFF(PDM, pDevs);
+    GEN_CHECK_OFF(PDM, pDevInstances);
+    GEN_CHECK_OFF(PDM, pUsbDevs);
+    GEN_CHECK_OFF(PDM, pUsbInstances);
+    GEN_CHECK_OFF(PDM, pDrvs);
+    GEN_CHECK_OFF(PDM, aPciBuses);
+    GEN_CHECK_OFF_DOT(PDM, aPciBuses[0].iBus);
+    GEN_CHECK_OFF_DOT(PDM, aPciBuses[0].pDevInsR3);
+    GEN_CHECK_OFF_DOT(PDM, aPciBuses[0].pfnSetIrqR3);
+    GEN_CHECK_OFF_DOT(PDM, aPciBuses[0].pfnRegisterR3);
+    GEN_CHECK_OFF_DOT(PDM, aPciBuses[0].pfnIORegionRegisterR3);
+    GEN_CHECK_OFF_DOT(PDM, aPciBuses[0].pDevInsR0);
+    GEN_CHECK_OFF_DOT(PDM, aPciBuses[0].pfnSetIrqR0);
+    GEN_CHECK_OFF_DOT(PDM, aPciBuses[0].pDevInsRC);
+    GEN_CHECK_OFF_DOT(PDM, aPciBuses[0].pfnSetIrqRC);
+    GEN_CHECK_OFF(PDM, Pic);
+    GEN_CHECK_OFF_DOT(PDM, Pic.pDevInsR3);
+    GEN_CHECK_OFF_DOT(PDM, Pic.pfnSetIrqR3);
+    GEN_CHECK_OFF_DOT(PDM, Pic.pfnGetInterruptR3);
+    GEN_CHECK_OFF_DOT(PDM, Pic.pDevInsR0);
+    GEN_CHECK_OFF_DOT(PDM, Pic.pfnSetIrqR0);
+    GEN_CHECK_OFF_DOT(PDM, Pic.pfnGetInterruptR0);
+    GEN_CHECK_OFF_DOT(PDM, Pic.pDevInsRC);
+    GEN_CHECK_OFF_DOT(PDM, Pic.pfnSetIrqRC);
+    GEN_CHECK_OFF_DOT(PDM, Pic.pfnGetInterruptRC);
+    GEN_CHECK_OFF(PDM, Apic);
+    GEN_CHECK_OFF_DOT(PDM, Apic.pDevInsR3);
+    GEN_CHECK_OFF_DOT(PDM, Apic.pDevInsR0);
+    GEN_CHECK_OFF_DOT(PDM, Apic.pDevInsRC);
+    GEN_CHECK_OFF(PDM, IoApic);
+    GEN_CHECK_OFF_DOT(PDM, IoApic.pDevInsR3);
+    GEN_CHECK_OFF_DOT(PDM, IoApic.pfnSetIrqR3);
+    GEN_CHECK_OFF_DOT(PDM, IoApic.pDevInsR0);
+    GEN_CHECK_OFF_DOT(PDM, IoApic.pfnSetIrqR0);
+    GEN_CHECK_OFF_DOT(PDM, IoApic.pDevInsRC);
+    GEN_CHECK_OFF_DOT(PDM, IoApic.pfnSetIrqRC);
+    GEN_CHECK_OFF(PDM, pDmac);
+    GEN_CHECK_OFF(PDM, pRtc);
+    GEN_CHECK_OFF(PDM, pUsbHubs);
+    GEN_CHECK_OFF(PDM, pDevHlpQueueR3);
+    GEN_CHECK_OFF(PDM, pDevHlpQueueR0);
+    GEN_CHECK_OFF(PDM, pDevHlpQueueRC);
+    GEN_CHECK_OFF(PDMCPU, cQueuedCritSectLeaves);
+    GEN_CHECK_OFF(PDMCPU, apQueuedCritSectLeaves);
+    GEN_CHECK_OFF(PDMCPU, cQueuedCritSectRwExclLeaves);
+    GEN_CHECK_OFF(PDMCPU, apQueuedCritSectRwExclLeaves);
+    GEN_CHECK_OFF(PDMCPU, cQueuedCritSectRwShrdLeaves);
+    GEN_CHECK_OFF(PDMCPU, apQueuedCritSectRwShrdLeaves);
+    GEN_CHECK_OFF(PDM, pQueueFlushR0);
+    GEN_CHECK_OFF(PDM, pQueueFlushRC);
+    GEN_CHECK_OFF(PDM, StatQueuedCritSectLeaves);
+
+    GEN_CHECK_SIZE(PDMDEVINSINT);
+    GEN_CHECK_OFF(PDMDEVINSINT, pNextR3);
+    GEN_CHECK_OFF(PDMDEVINSINT, pPerDeviceNextR3);
+    GEN_CHECK_OFF(PDMDEVINSINT, pDevR3);
+    GEN_CHECK_OFF(PDMDEVINSINT, pLunsR3);
+    GEN_CHECK_OFF(PDMDEVINSINT, pfnAsyncNotify);
+    GEN_CHECK_OFF(PDMDEVINSINT, pCfgHandle);
+    GEN_CHECK_OFF(PDMDEVINSINT, pVMR3);
+    GEN_CHECK_OFF(PDMDEVINSINT, pVMR0);
+    GEN_CHECK_OFF(PDMDEVINSINT, pVMRC);
+    GEN_CHECK_OFF(PDMDEVINSINT, pHeadPciDevR3);
+    GEN_CHECK_OFF(PDMDEVINSINT, pHeadPciDevR0);
+    GEN_CHECK_OFF(PDMDEVINSINT, pHeadPciDevRC);
+    GEN_CHECK_OFF(PDMDEVINSINT, fIntFlags);
+    GEN_CHECK_OFF(PDMDEVINSINT, uLastIrqTag);
+    GEN_CHECK_OFF(PDMDEVINS, u32Version);
+    GEN_CHECK_OFF(PDMDEVINS, iInstance);
+    GEN_CHECK_OFF(PDMDEVINS, pHlpRC);
+    GEN_CHECK_OFF(PDMDEVINS, pvInstanceDataRC);
+    GEN_CHECK_OFF(PDMDEVINS, pHlpR0);
+    GEN_CHECK_OFF(PDMDEVINS, pvInstanceDataR0);
+    GEN_CHECK_OFF(PDMDEVINS, pHlpR3);
+    GEN_CHECK_OFF(PDMDEVINS, pvInstanceDataR3);
+    GEN_CHECK_OFF(PDMDEVINS, pReg);
+    GEN_CHECK_OFF(PDMDEVINS, pCfg);
+    GEN_CHECK_OFF(PDMDEVINS, IBase);
+    GEN_CHECK_OFF(PDMDEVINS, Internal);
+    GEN_CHECK_OFF(PDMDEVINS, achInstanceData);
+
+    GEN_CHECK_SIZE(PDMDRVINSINT);
+    GEN_CHECK_OFF(PDMDRVINSINT, pUp);
+    GEN_CHECK_OFF(PDMDRVINSINT, pDown);
+    GEN_CHECK_OFF(PDMDRVINSINT, pLun);
+    GEN_CHECK_OFF(PDMDRVINSINT, pDrv);
+    GEN_CHECK_OFF(PDMDRVINSINT, pVMR3);
+    GEN_CHECK_OFF(PDMDRVINSINT, pVMR0);
+    GEN_CHECK_OFF(PDMDRVINSINT, pVMRC);
+    GEN_CHECK_OFF(PDMDRVINSINT, fDetaching);
+    GEN_CHECK_OFF(PDMDRVINSINT, fVMSuspended);
+    GEN_CHECK_OFF(PDMDRVINSINT, fVMReset);
+    GEN_CHECK_OFF(PDMDRVINSINT, pfnAsyncNotify);
+    GEN_CHECK_OFF(PDMDRVINSINT, pCfgHandle);
+    GEN_CHECK_OFF(PDMDRVINS, u32Version);
+    GEN_CHECK_OFF(PDMDRVINS, iInstance);
+    GEN_CHECK_OFF(PDMDRVINS, pHlpRC);
+    GEN_CHECK_OFF(PDMDRVINS, pvInstanceDataRC);
+    GEN_CHECK_OFF(PDMDRVINS, pHlpR0);
+    GEN_CHECK_OFF(PDMDRVINS, pvInstanceDataR0);
+    GEN_CHECK_OFF(PDMDRVINS, pHlpR3);
+    GEN_CHECK_OFF(PDMDRVINS, pvInstanceDataR3);
+    GEN_CHECK_OFF(PDMDRVINS, pReg);
+    GEN_CHECK_OFF(PDMDRVINS, pCfg);
+    GEN_CHECK_OFF(PDMDRVINS, IBase);
+    GEN_CHECK_OFF(PDMDRVINS, Internal);
+    GEN_CHECK_OFF(PDMDRVINS, achInstanceData);
+
+    GEN_CHECK_SIZE(PDMCRITSECTINT);
+    GEN_CHECK_OFF(PDMCRITSECTINT, Core);
+    GEN_CHECK_OFF(PDMCRITSECTINT, pNext);
+    GEN_CHECK_OFF(PDMCRITSECTINT, pvKey);
+    GEN_CHECK_OFF(PDMCRITSECTINT, pVMR3);
+    GEN_CHECK_OFF(PDMCRITSECTINT, pVMR0);
+    GEN_CHECK_OFF(PDMCRITSECTINT, pVMRC);
+    GEN_CHECK_OFF(PDMCRITSECTINT, StatContentionRZLock);
+    GEN_CHECK_OFF(PDMCRITSECTINT, StatContentionRZUnlock);
+    GEN_CHECK_OFF(PDMCRITSECTINT, StatContentionR3);
+    GEN_CHECK_OFF(PDMCRITSECTINT, StatLocked);
+    GEN_CHECK_SIZE(PDMCRITSECT);
+    GEN_CHECK_SIZE(PDMCRITSECTRWINT);
+    GEN_CHECK_OFF(PDMCRITSECTRWINT, Core);
+    GEN_CHECK_OFF(PDMCRITSECTRWINT, pNext);
+    GEN_CHECK_OFF(PDMCRITSECTRWINT, pvKey);
+    GEN_CHECK_OFF(PDMCRITSECTRWINT, pVMR3);
+    GEN_CHECK_OFF(PDMCRITSECTRWINT, pVMR0);
+    GEN_CHECK_OFF(PDMCRITSECTRWINT, pVMRC);
+    GEN_CHECK_OFF(PDMCRITSECTRWINT, pszName);
+    GEN_CHECK_OFF(PDMCRITSECTRWINT, StatContentionRZEnterExcl);
+    GEN_CHECK_OFF(PDMCRITSECTRWINT, StatWriteLocked);
+    GEN_CHECK_SIZE(PDMCRITSECTRW);
+    GEN_CHECK_SIZE(PDMQUEUE);
+    GEN_CHECK_OFF(PDMQUEUE, pNext);
+    GEN_CHECK_OFF(PDMQUEUE, enmType);
+    GEN_CHECK_OFF(PDMQUEUE, u);
+    GEN_CHECK_OFF_DOT(PDMQUEUE, u.Dev.pfnCallback);
+    GEN_CHECK_OFF_DOT(PDMQUEUE, u.Dev.pDevIns);
+    GEN_CHECK_OFF_DOT(PDMQUEUE, u.Drv.pfnCallback);
+    GEN_CHECK_OFF_DOT(PDMQUEUE, u.Drv.pDrvIns);
+    GEN_CHECK_OFF_DOT(PDMQUEUE, u.Int.pfnCallback);
+    GEN_CHECK_OFF_DOT(PDMQUEUE, u.Ext.pfnCallback);
+    GEN_CHECK_OFF_DOT(PDMQUEUE, u.Ext.pvUser);
+    GEN_CHECK_OFF(PDMQUEUE, pVMR3);
+    GEN_CHECK_OFF(PDMQUEUE, pVMR0);
+    GEN_CHECK_OFF(PDMQUEUE, pVMRC);
+    GEN_CHECK_OFF(PDMQUEUE, cMilliesInterval);
+    GEN_CHECK_OFF(PDMQUEUE, pTimer);
+    GEN_CHECK_OFF(PDMQUEUE, cbItem);
+    GEN_CHECK_OFF(PDMQUEUE, cItems);
+    GEN_CHECK_OFF(PDMQUEUE, pPendingR3);
+    GEN_CHECK_OFF(PDMQUEUE, pPendingR0);
+    GEN_CHECK_OFF(PDMQUEUE, pPendingRC);
+    GEN_CHECK_OFF(PDMQUEUE, iFreeHead);
+    GEN_CHECK_OFF(PDMQUEUE, iFreeTail);
+    GEN_CHECK_OFF(PDMQUEUE, pszName);
+    GEN_CHECK_OFF(PDMQUEUE, StatAllocFailures);
+    GEN_CHECK_OFF(PDMQUEUE, StatInsert);
+    GEN_CHECK_OFF(PDMQUEUE, StatFlush);
+    GEN_CHECK_OFF(PDMQUEUE, StatFlushLeftovers);
+    GEN_CHECK_OFF(PDMQUEUE, aFreeItems);
+    GEN_CHECK_OFF(PDMQUEUE, aFreeItems[1]);
+    GEN_CHECK_OFF_DOT(PDMQUEUE, aFreeItems[0].pItemR3);
+    GEN_CHECK_OFF_DOT(PDMQUEUE, aFreeItems[0].pItemR0);
+    GEN_CHECK_OFF_DOT(PDMQUEUE, aFreeItems[1].pItemRC);
+    GEN_CHECK_SIZE(PDMDEVHLPTASK);
+    GEN_CHECK_OFF(PDMDEVHLPTASK, Core);
+    GEN_CHECK_OFF(PDMDEVHLPTASK, pDevInsR3);
+    GEN_CHECK_OFF(PDMDEVHLPTASK, enmOp);
+    GEN_CHECK_OFF(PDMDEVHLPTASK, u);
+    GEN_CHECK_OFF_DOT(PDMDEVHLPTASK, u.IsaSetIRQ.iIrq);
+    GEN_CHECK_OFF_DOT(PDMDEVHLPTASK, u.IsaSetIRQ.iLevel);
+    GEN_CHECK_OFF_DOT(PDMDEVHLPTASK, u.IsaSetIRQ.uTagSrc);
+    GEN_CHECK_OFF_DOT(PDMDEVHLPTASK, u.IoApicSetIRQ.iIrq);
+    GEN_CHECK_OFF_DOT(PDMDEVHLPTASK, u.IoApicSetIRQ.iLevel);
+    GEN_CHECK_OFF_DOT(PDMDEVHLPTASK, u.IoApicSetIRQ.uTagSrc);
+    GEN_CHECK_OFF_DOT(PDMDEVHLPTASK, u.PciSetIRQ.pPciDevR3);
+    GEN_CHECK_OFF_DOT(PDMDEVHLPTASK, u.PciSetIRQ.iIrq);
+    GEN_CHECK_OFF_DOT(PDMDEVHLPTASK, u.PciSetIRQ.iLevel);
+    GEN_CHECK_OFF_DOT(PDMDEVHLPTASK, u.PciSetIRQ.uTagSrc);
+
+    GEN_CHECK_SIZE(PGM);
+    GEN_CHECK_OFF(PGM, offVM);
+    GEN_CHECK_OFF(PGM, fRamPreAlloc);
+    GEN_CHECK_OFF(PGM, paDynPageMap32BitPTEsGC);
+    GEN_CHECK_OFF(PGM, paDynPageMapPaePTEsGC);
+    GEN_CHECK_OFF(PGM, enmHostMode);
+    GEN_CHECK_OFF(PGMCPU, offVM);
+    GEN_CHECK_OFF(PGMCPU, offVCpu);
+    GEN_CHECK_OFF(PGMCPU, offPGM);
+#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE) || defined(VBOX_WITH_RAW_MODE)
+    GEN_CHECK_OFF(PGMCPU, AutoSet);
+#endif
+    GEN_CHECK_OFF(PGMCPU, GCPhysA20Mask);
+    GEN_CHECK_OFF(PGMCPU, fA20Enabled);
+    GEN_CHECK_OFF(PGMCPU, fSyncFlags);
+    GEN_CHECK_OFF(PGMCPU, enmShadowMode);
+    GEN_CHECK_OFF(PGMCPU, enmGuestMode);
+    GEN_CHECK_OFF(PGMCPU, GCPhysCR3);
+    GEN_CHECK_OFF(PGM, GCPtrCR3Mapping);
+    GEN_CHECK_OFF(PGMCPU, pGst32BitPdR3);
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    GEN_CHECK_OFF(PGMCPU, pGst32BitPdR0);
+#endif
+    GEN_CHECK_OFF(PGMCPU, pGst32BitPdRC);
+    GEN_CHECK_OFF(PGMCPU, pGstPaePdptR3);
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    GEN_CHECK_OFF(PGMCPU, pGstPaePdptR0);
+#endif
+    GEN_CHECK_OFF(PGMCPU, pGstPaePdptRC);
+    GEN_CHECK_OFF(PGMCPU, apGstPaePDsR3);
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    GEN_CHECK_OFF(PGMCPU, apGstPaePDsR0);
+#endif
+    GEN_CHECK_OFF(PGMCPU, apGstPaePDsRC);
+    GEN_CHECK_OFF(PGMCPU, aGCPhysGstPaePDs);
+    GEN_CHECK_OFF(PGMCPU, aGCPhysGstPaePDsMonitored);
+    GEN_CHECK_OFF(PGMCPU, pShwPageCR3R3);
+    GEN_CHECK_OFF(PGMCPU, pShwPageCR3R0);
+    GEN_CHECK_OFF(PGMCPU, pShwPageCR3RC);
+    GEN_CHECK_OFF(PGMCPU, DisState);
+    GEN_CHECK_OFF(PGMCPU, cGuestModeChanges);
+#ifdef VBOX_WITH_STATISTICS
+    GEN_CHECK_OFF(PGMCPU, pStatsR0);
+    GEN_CHECK_OFF(PGMCPU, pStatTrap0eAttributionR0);
+    GEN_CHECK_OFF(PGMCPU, pStatsRC);
+    GEN_CHECK_OFF(PGMCPU, pStatTrap0eAttributionRC);
+    GEN_CHECK_OFF(PGMCPU, pStatsR3);
+#endif
+    GEN_CHECK_OFF(PGM, offVM);
+    GEN_CHECK_OFF(PGM, offVCpuPGM);
+    GEN_CHECK_OFF(PGM, fRamPreAlloc);
+    GEN_CHECK_OFF(PGM, paDynPageMap32BitPTEsGC);
+    GEN_CHECK_OFF(PGM, paDynPageMapPaePTEsGC);
+    GEN_CHECK_OFF(PGM, enmHostMode);
+    GEN_CHECK_OFF(PGM, fRestoreRomPagesOnReset);
+    GEN_CHECK_OFF(PGM, fZeroRamPagesOnReset);
+    GEN_CHECK_OFF(PGM, GCPhys4MBPSEMask);
+    GEN_CHECK_OFF(PGM, pRamRangesXR3);
+    GEN_CHECK_OFF(PGM, pRamRangesXR0);
+    GEN_CHECK_OFF(PGM, pRamRangesXRC);
+    GEN_CHECK_OFF(PGM, pRomRangesR3);
+    GEN_CHECK_OFF(PGM, pRomRangesR0);
+    GEN_CHECK_OFF(PGM, pRomRangesRC);
+    GEN_CHECK_OFF(PGM, pTreesR3);
+    GEN_CHECK_OFF(PGM, pTreesR0);
+    GEN_CHECK_OFF(PGM, pTreesRC);
+    GEN_CHECK_OFF(PGM, pMappingsR3);
+    GEN_CHECK_OFF(PGM, pMappingsRC);
+    GEN_CHECK_OFF(PGM, pMappingsR0);
+    GEN_CHECK_OFF(PGM, fFinalizedMappings);
+    GEN_CHECK_OFF(PGM, fMappingsFixed);
+    GEN_CHECK_OFF(PGM, fMappingsFixedRestored);
+    GEN_CHECK_OFF(PGM, GCPtrMappingFixed);
+    GEN_CHECK_OFF(PGM, cbMappingFixed);
+    GEN_CHECK_OFF(PGM, pInterPD);
+    GEN_CHECK_OFF(PGM, apInterPTs);
+    GEN_CHECK_OFF(PGM, apInterPaePTs);
+    GEN_CHECK_OFF(PGM, apInterPaePDs);
+    GEN_CHECK_OFF(PGM, pInterPaePDPT);
+    GEN_CHECK_OFF(PGM, pInterPaePDPT64);
+    GEN_CHECK_OFF(PGM, pInterPaePML4);
+    GEN_CHECK_OFF(PGM, HCPhysInterPD);
+    GEN_CHECK_OFF(PGM, HCPhysInterPaePDPT);
+    GEN_CHECK_OFF(PGM, HCPhysInterPaePML4);
+    GEN_CHECK_OFF(PGM, pbDynPageMapBaseGC);
+    GEN_CHECK_OFF(PGM, pRCDynMap);
+    GEN_CHECK_OFF(PGM, pvR0DynMapUsed);
+    GEN_CHECK_OFF(PGM, GCPhys4MBPSEMask);
+    GEN_CHECK_OFF(PGMCPU, GCPhysA20Mask);
+    GEN_CHECK_OFF(PGMCPU, fA20Enabled);
+    GEN_CHECK_OFF(PGMCPU, fSyncFlags);
+    GEN_CHECK_OFF(PGM, CritSectX);
+    GEN_CHECK_OFF(PGM, pPoolR3);
+    GEN_CHECK_OFF(PGM, pPoolR0);
+    GEN_CHECK_OFF(PGM, pPoolRC);
+    GEN_CHECK_OFF(PGM, fNoMorePhysWrites);
+    GEN_CHECK_OFF(PGM, ChunkR3Map);
+    GEN_CHECK_OFF_DOT(PGM, ChunkR3Map.pTree);
+    GEN_CHECK_OFF_DOT(PGM, ChunkR3Map.Tlb);
+    GEN_CHECK_OFF_DOT(PGM, ChunkR3Map.c);
+    GEN_CHECK_OFF_DOT(PGM, ChunkR3Map.cMax);
+    GEN_CHECK_OFF_DOT(PGM, ChunkR3Map.iNow);
+    GEN_CHECK_OFF(PGM, PhysTlbHC);
+    GEN_CHECK_OFF_DOT(PGM, PhysTlbHC.aEntries[0]);
+    GEN_CHECK_OFF_DOT(PGM, PhysTlbHC.aEntries[1]);
+    GEN_CHECK_OFF_DOT(PGM, PhysTlbHC.aEntries[1].GCPhys);
+    GEN_CHECK_OFF_DOT(PGM, PhysTlbHC.aEntries[1].pMap);
+    GEN_CHECK_OFF_DOT(PGM, PhysTlbHC.aEntries[1].pPage);
+    GEN_CHECK_OFF_DOT(PGM, PhysTlbHC.aEntries[1].pv);
+    GEN_CHECK_OFF(PGM, HCPhysZeroPg);
+    GEN_CHECK_OFF(PGM, pvZeroPgR3);
+    GEN_CHECK_OFF(PGM, pvZeroPgR0);
+    GEN_CHECK_OFF(PGM, pvZeroPgRC);
+    GEN_CHECK_OFF(PGM, cHandyPages);
+    GEN_CHECK_OFF(PGM, aHandyPages);
+    GEN_CHECK_OFF_DOT(PGM, aHandyPages[1]);
+    GEN_CHECK_OFF_DOT(PGM, aHandyPages[1].HCPhysGCPhys);
+    GEN_CHECK_OFF_DOT(PGM, aHandyPages[1].idPage);
+    GEN_CHECK_OFF_DOT(PGM, aHandyPages[1].idSharedPage);
+    GEN_CHECK_OFF(PGM, cAllPages);
+    GEN_CHECK_OFF(PGM, cPrivatePages);
+    GEN_CHECK_OFF(PGM, cSharedPages);
+    GEN_CHECK_OFF(PGM, cZeroPages);
+    GEN_CHECK_OFF(PGM, cPureMmioPages);
+    GEN_CHECK_OFF(PGM, cMonitoredPages);
+    GEN_CHECK_OFF(PGM, cWrittenToPages);
+    GEN_CHECK_OFF(PGM, cWriteLockedPages);
+    GEN_CHECK_OFF(PGM, cReadLockedPages);
+    GEN_CHECK_OFF(PGM, cRelocations);
+#ifdef VBOX_WITH_STATISTICS
+    GEN_CHECK_OFF(PGMCPU, pStatsR0);
+    GEN_CHECK_OFF(PGMCPU, pStatsRC);
+    GEN_CHECK_OFF(PGMCPU, pStatsR3);
+#endif
+
+    GEN_CHECK_SIZE(PGMMAPPING);
+    GEN_CHECK_OFF(PGMMAPPING, pNextR3);
+    GEN_CHECK_OFF(PGMMAPPING, pNextRC);
+    GEN_CHECK_OFF(PGMMAPPING, pNextR0);
+    GEN_CHECK_OFF(PGMMAPPING, GCPtr);
+    GEN_CHECK_OFF(PGMMAPPING, GCPtrLast);
+    GEN_CHECK_OFF(PGMMAPPING, cb);
+    GEN_CHECK_OFF(PGMMAPPING, pfnRelocate);
+    GEN_CHECK_OFF(PGMMAPPING, pvUser);
+    GEN_CHECK_OFF(PGMMAPPING, pszDesc);
+    GEN_CHECK_OFF(PGMMAPPING, cPTs);
+    GEN_CHECK_OFF_DOT(PGMMAPPING, aPTs[1].HCPhysPT);
+    GEN_CHECK_OFF_DOT(PGMMAPPING, aPTs[1].pPTR3);
+    GEN_CHECK_OFF_DOT(PGMMAPPING, aPTs[1].pPTR0);
+    GEN_CHECK_OFF_DOT(PGMMAPPING, aPTs[1].pPTRC);
+    GEN_CHECK_OFF_DOT(PGMMAPPING, aPTs[1].HCPhysPaePT0);
+    GEN_CHECK_OFF_DOT(PGMMAPPING, aPTs[1].HCPhysPaePT1);
+    GEN_CHECK_OFF_DOT(PGMMAPPING, aPTs[1].paPaePTsR3);
+    GEN_CHECK_OFF_DOT(PGMMAPPING, aPTs[1].paPaePTsRC);
+    GEN_CHECK_OFF_DOT(PGMMAPPING, aPTs[1].paPaePTsR0);
+    GEN_CHECK_SIZE(PGMPHYSHANDLER);
+    GEN_CHECK_OFF(PGMPHYSHANDLER, Core);
+    GEN_CHECK_SIZE(((PPGMPHYSHANDLER)0)->Core);
+    GEN_CHECK_OFF(PGMPHYSHANDLER, cPages);
+    GEN_CHECK_OFF(PGMPHYSHANDLER, cAliasedPages);
+    GEN_CHECK_OFF(PGMPHYSHANDLER, cTmpOffPages);
+    GEN_CHECK_OFF(PGMPHYSHANDLER, hType);
+    GEN_CHECK_OFF(PGMPHYSHANDLER, pvUserR3);
+    GEN_CHECK_OFF(PGMPHYSHANDLER, pvUserR0);
+    GEN_CHECK_OFF(PGMPHYSHANDLER, pvUserRC);
+    //GEN_CHECK_OFF(PGMPHYSHANDLER, pszDesc);
+    GEN_CHECK_SIZE(PGMPHYSHANDLERTYPEINT);
+    GEN_CHECK_OFF(PGMPHYSHANDLERTYPEINT, u32Magic);
+    GEN_CHECK_OFF(PGMPHYSHANDLERTYPEINT, cRefs);
+    GEN_CHECK_OFF(PGMPHYSHANDLERTYPEINT, ListNode);
+    GEN_CHECK_OFF(PGMPHYSHANDLERTYPEINT, enmKind);
+    GEN_CHECK_OFF(PGMPHYSHANDLERTYPEINT, uState);
+    GEN_CHECK_OFF(PGMPHYSHANDLERTYPEINT, pfnHandlerR3);
+    GEN_CHECK_OFF(PGMPHYSHANDLERTYPEINT, pfnHandlerR0);
+    GEN_CHECK_OFF(PGMPHYSHANDLERTYPEINT, pfnPfHandlerR0);
+    GEN_CHECK_OFF(PGMPHYSHANDLERTYPEINT, pfnHandlerRC);
+    GEN_CHECK_OFF(PGMPHYSHANDLERTYPEINT, pfnPfHandlerRC);
+    GEN_CHECK_OFF(PGMPHYSHANDLERTYPEINT, pszDesc);
+    GEN_CHECK_SIZE(PGMPHYS2VIRTHANDLER);
+    GEN_CHECK_OFF(PGMPHYS2VIRTHANDLER, Core);
+    GEN_CHECK_OFF(PGMPHYS2VIRTHANDLER, offVirtHandler);
+    GEN_CHECK_SIZE(PGMVIRTHANDLER);
+    GEN_CHECK_OFF(PGMVIRTHANDLER, Core);
+    GEN_CHECK_OFF(PGMVIRTHANDLER, hType);
+    GEN_CHECK_OFF(PGMVIRTHANDLER, cb);
+    GEN_CHECK_OFF(PGMVIRTHANDLER, cPages);
+    GEN_CHECK_OFF(PGMVIRTHANDLER, pszDesc);
+    GEN_CHECK_OFF(PGMVIRTHANDLER, aPhysToVirt);
+    GEN_CHECK_SIZE(PGMVIRTHANDLERTYPEINT);
+    GEN_CHECK_OFF(PGMVIRTHANDLERTYPEINT, u32Magic);
+    GEN_CHECK_OFF(PGMVIRTHANDLERTYPEINT, cRefs);
+    GEN_CHECK_OFF(PGMVIRTHANDLERTYPEINT, ListNode);
+    GEN_CHECK_OFF(PGMVIRTHANDLERTYPEINT, enmKind);
+    GEN_CHECK_OFF(PGMVIRTHANDLERTYPEINT, uState);
+    GEN_CHECK_OFF(PGMVIRTHANDLERTYPEINT, fRelocUserRC);
+    GEN_CHECK_OFF(PGMVIRTHANDLERTYPEINT, pfnHandlerRC);
+    GEN_CHECK_OFF(PGMVIRTHANDLERTYPEINT, pfnPfHandlerRC);
+    GEN_CHECK_OFF(PGMVIRTHANDLERTYPEINT, pfnInvalidateR3);
+    GEN_CHECK_OFF(PGMVIRTHANDLERTYPEINT, pfnHandlerR3);
+    GEN_CHECK_OFF(PGMVIRTHANDLERTYPEINT, pszDesc);
+    GEN_CHECK_SIZE(PGMPAGE);
+    GEN_CHECK_OFF_DOT(PGMPAGE, s.cReadLocksY);
+    GEN_CHECK_OFF_DOT(PGMPAGE, s.cWriteLocksY);
+    GEN_CHECK_OFF_DOT(PGMPAGE, s.u16TrackingY);
+    GEN_CHECK_SIZE(PGMRAMRANGE);
+    GEN_CHECK_OFF(PGMRAMRANGE, pNextR3);
+    GEN_CHECK_OFF(PGMRAMRANGE, pNextR0);
+    GEN_CHECK_OFF(PGMRAMRANGE, pNextRC);
+    GEN_CHECK_OFF(PGMRAMRANGE, GCPhys);
+    GEN_CHECK_OFF(PGMRAMRANGE, GCPhysLast);
+    GEN_CHECK_OFF(PGMRAMRANGE, cb);
+    GEN_CHECK_OFF(PGMRAMRANGE, fFlags);
+    GEN_CHECK_OFF(PGMRAMRANGE, pvR3);
+    GEN_CHECK_OFF(PGMRAMRANGE, pszDesc);
+    GEN_CHECK_OFF(PGMRAMRANGE, aPages);
+    GEN_CHECK_OFF(PGMRAMRANGE, aPages[1]);
+    GEN_CHECK_SIZE(PGMROMPAGE);
+    GEN_CHECK_OFF(PGMROMPAGE, Virgin);
+    GEN_CHECK_OFF(PGMROMPAGE, Shadow);
+    GEN_CHECK_OFF(PGMROMPAGE, enmProt);
+    GEN_CHECK_SIZE(PGMROMRANGE);
+    GEN_CHECK_OFF(PGMROMRANGE, pNextR3);
+    GEN_CHECK_OFF(PGMROMRANGE, pNextR0);
+    GEN_CHECK_OFF(PGMROMRANGE, pNextRC);
+    GEN_CHECK_OFF(PGMROMRANGE, GCPhys);
+    GEN_CHECK_OFF(PGMROMRANGE, GCPhysLast);
+    GEN_CHECK_OFF(PGMROMRANGE, cb);
+    GEN_CHECK_OFF(PGMROMRANGE, fFlags);
+    GEN_CHECK_OFF(PGMROMRANGE, cbOriginal);
+    GEN_CHECK_OFF(PGMROMRANGE, pvOriginal);
+    GEN_CHECK_OFF(PGMROMRANGE, pszDesc);
+    GEN_CHECK_OFF(PGMROMRANGE, aPages);
+    GEN_CHECK_OFF(PGMROMRANGE, aPages[1]);
+    GEN_CHECK_SIZE(PGMREGMMIORANGE);
+    GEN_CHECK_OFF(PGMREGMMIORANGE, pDevInsR3);
+    GEN_CHECK_OFF(PGMREGMMIORANGE, pNextR3);
+    GEN_CHECK_OFF(PGMREGMMIORANGE, fFlags);
+    GEN_CHECK_OFF(PGMREGMMIORANGE, iRegion);
+    GEN_CHECK_OFF(PGMREGMMIORANGE, pPhysHandlerR3);
+    GEN_CHECK_OFF(PGMREGMMIORANGE, RamRange);
+    GEN_CHECK_SIZE(PGMTREES);
+    GEN_CHECK_OFF(PGMTREES, PhysHandlers);
+    GEN_CHECK_OFF(PGMTREES, HeadPhysHandlerTypes);
+#ifdef VBOX_WITH_RAW_MODE
+    GEN_CHECK_OFF(PGMTREES, VirtHandlers);
+    GEN_CHECK_OFF(PGMTREES, PhysToVirtHandlers);
+    GEN_CHECK_OFF(PGMTREES, HyperVirtHandlers);
+    GEN_CHECK_OFF(PGMTREES, HeadVirtHandlerTypes);
+#endif
+    GEN_CHECK_SIZE(PGMPOOLPAGE);
+    GEN_CHECK_OFF(PGMPOOLPAGE, Core);
+    GEN_CHECK_OFF(PGMPOOLPAGE, GCPhys);
+    GEN_CHECK_OFF(PGMPOOLPAGE, pvPageR3);
+    GEN_CHECK_OFF(PGMPOOLPAGE, enmKind);
+    GEN_CHECK_OFF(PGMPOOLPAGE, enmAccess);
+    //GEN_CHECK_OFF(PGMPOOLPAGE, fA20Enabled);
+    //GEN_CHECK_OFF(PGMPOOLPAGE, fSeenNonGlobal);
+    //GEN_CHECK_OFF(PGMPOOLPAGE, fMonitored);
+    //GEN_CHECK_OFF(PGMPOOLPAGE, fCached);
+    //GEN_CHECK_OFF(PGMPOOLPAGE, fReusedFlushPending);
+    GEN_CHECK_OFF(PGMPOOLPAGE, idx);
+    GEN_CHECK_OFF(PGMPOOLPAGE, iNext);
+    GEN_CHECK_OFF(PGMPOOLPAGE, iUserHead);
+    GEN_CHECK_OFF(PGMPOOLPAGE, cPresent);
+    GEN_CHECK_OFF(PGMPOOLPAGE, iFirstPresent);
+    GEN_CHECK_OFF(PGMPOOLPAGE, cModifications);
+    GEN_CHECK_OFF(PGMPOOLPAGE, iModifiedNext);
+    GEN_CHECK_OFF(PGMPOOLPAGE, iModifiedPrev);
+    GEN_CHECK_OFF(PGMPOOLPAGE, iMonitoredNext);
+    GEN_CHECK_OFF(PGMPOOLPAGE, iMonitoredPrev);
+    GEN_CHECK_OFF(PGMPOOLPAGE, iAgeNext);
+    GEN_CHECK_OFF(PGMPOOLPAGE, iAgePrev);
+    GEN_CHECK_OFF(PGMPOOLPAGE, idxDirtyEntry);
+    GEN_CHECK_OFF(PGMPOOLPAGE, GCPtrLastAccessHandlerRip);
+    GEN_CHECK_OFF(PGMPOOLPAGE, GCPtrLastAccessHandlerFault);
+    GEN_CHECK_OFF(PGMPOOLPAGE, cLastAccessHandler);
+    GEN_CHECK_OFF(PGMPOOLPAGE, cLocked);
+#ifdef VBOX_STRICT
+    GEN_CHECK_OFF(PGMPOOLPAGE, GCPtrDirtyFault);
+#endif
+    GEN_CHECK_SIZE(PGMPOOL);
+    GEN_CHECK_OFF(PGMPOOL, pVMR3);
+    GEN_CHECK_OFF(PGMPOOL, pVMR0);
+    GEN_CHECK_OFF(PGMPOOL, pVMRC);
+    GEN_CHECK_OFF(PGMPOOL, cMaxPages);
+    GEN_CHECK_OFF(PGMPOOL, cCurPages);
+    GEN_CHECK_OFF(PGMPOOL, iFreeHead);
+    GEN_CHECK_OFF(PGMPOOL, u16Padding);
+#ifdef PGMPOOL_WITH_USER_TRACKING
+    GEN_CHECK_OFF(PGMPOOL, iUserFreeHead);
+    GEN_CHECK_OFF(PGMPOOL, cMaxUsers);
+    GEN_CHECK_OFF(PGMPOOL, cPresent);
+    GEN_CHECK_OFF(PGMPOOL, paUsersR3);
+    GEN_CHECK_OFF(PGMPOOL, paUsersR0);
+    GEN_CHECK_OFF(PGMPOOL, paUsersRC);
+#endif /* PGMPOOL_WITH_USER_TRACKING */
+#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
+    GEN_CHECK_OFF(PGMPOOL, iPhysExtFreeHead);
+    GEN_CHECK_OFF(PGMPOOL, cMaxPhysExts);
+    GEN_CHECK_OFF(PGMPOOL, paPhysExtsR3);
+    GEN_CHECK_OFF(PGMPOOL, paPhysExtsR0);
+    GEN_CHECK_OFF(PGMPOOL, paPhysExtsRC);
+#endif
+#ifdef PGMPOOL_WITH_CACHE
+    GEN_CHECK_OFF(PGMPOOL, aiHash);
+    GEN_CHECK_OFF(PGMPOOL, iAgeHead);
+    GEN_CHECK_OFF(PGMPOOL, iAgeTail);
+    GEN_CHECK_OFF(PGMPOOL, fCacheEnabled);
+#endif
+#ifdef PGMPOOL_WITH_MONITORING
+    GEN_CHECK_OFF(PGMPOOL, pfnAccessHandlerRC);
+    GEN_CHECK_OFF(PGMPOOL, pfnAccessHandlerR0);
+    GEN_CHECK_OFF(PGMPOOL, pfnAccessHandlerR3);
+    GEN_CHECK_OFF(PGMPOOL, pszAccessHandler);
+    GEN_CHECK_OFF(PGMPOOL, iModifiedHead);
+    GEN_CHECK_OFF(PGMPOOL, cModifiedPages);
+#endif
+    GEN_CHECK_OFF(PGMPOOL, cUsedPages);
+#ifdef VBOX_WITH_STATISTICS
+    GEN_CHECK_OFF(PGMPOOL, cUsedPagesHigh);
+    GEN_CHECK_OFF(PGMPOOL, StatAlloc);
+    GEN_CHECK_OFF(PGMPOOL, StatClearAll);
+#endif
+    GEN_CHECK_OFF(PGMPOOL, HCPhysTree);
+    GEN_CHECK_OFF(PGMPOOL, aPages);
+    GEN_CHECK_OFF(PGMPOOL, aPages[1]);
+    GEN_CHECK_OFF(PGMPOOL, aPages[PGMPOOL_IDX_FIRST - 1]);
+    GEN_CHECK_SIZE(PGMRCDYNMAP);
+    GEN_CHECK_OFF(PGMRCDYNMAP, u32Magic);
+    GEN_CHECK_OFF(PGMRCDYNMAP, paPages);
+    GEN_CHECK_OFF(PGMRCDYNMAP, cPages);
+    GEN_CHECK_OFF(PGMRCDYNMAP, cLoad);
+    GEN_CHECK_OFF(PGMRCDYNMAP, cMaxLoad);
+    GEN_CHECK_OFF(PGMRCDYNMAP, cGuardPages);
+    GEN_CHECK_OFF(PGMRCDYNMAP, cUsers);
+    GEN_CHECK_SIZE(PGMRCDYNMAPENTRY);
+    GEN_CHECK_OFF(PGMRCDYNMAPENTRY, HCPhys);
+    GEN_CHECK_OFF(PGMRCDYNMAPENTRY, pvPage);
+    GEN_CHECK_OFF(PGMRCDYNMAPENTRY, cRefs);
+    GEN_CHECK_OFF_DOT(PGMRCDYNMAPENTRY, uPte.pLegacy);
+    GEN_CHECK_OFF_DOT(PGMRCDYNMAPENTRY, uPte.pPae);
+    GEN_CHECK_OFF(PGMMAPSETENTRY, pvPage);
+    GEN_CHECK_OFF(PGMMAPSETENTRY, iPage);
+    GEN_CHECK_OFF(PGMMAPSETENTRY, cRefs);
+    GEN_CHECK_OFF(PGMMAPSETENTRY, cInlinedRefs);
+    GEN_CHECK_OFF(PGMMAPSETENTRY, cUnrefs);
+    GEN_CHECK_OFF(PGMMAPSETENTRY, HCPhys);
+
+    GEN_CHECK_SIZE(REM);
+    GEN_CHECK_OFF(REM, pCtx);
+    GEN_CHECK_OFF(REM, cCanExecuteRaw);
+    GEN_CHECK_OFF(REM, aGCPtrInvalidatedPages);
+    GEN_CHECK_OFF(REM, idxPendingList);
+    GEN_CHECK_OFF(REM, aHandlerNotifications);
+    GEN_CHECK_OFF(REM, idxFreeList);
+    GEN_CHECK_OFF(REM, CritSectRegister);
+    GEN_CHECK_OFF(REM, rc);
+    GEN_CHECK_OFF(REM, StatsInQEMU);
+    GEN_CHECK_OFF(REM, Env);
+
+    GEN_CHECK_SIZE(REMHANDLERNOTIFICATION);
+    GEN_CHECK_OFF(REMHANDLERNOTIFICATION, enmKind);
+    GEN_CHECK_OFF(REMHANDLERNOTIFICATION, u);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalRegister.GCPhys);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalRegister.cb);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalRegister.enmKind);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalRegister.fHasHCHandler);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalDeregister.GCPhys);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalDeregister.cb);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalDeregister.enmKind);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalDeregister.fHasHCHandler);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalDeregister.fRestoreAsRAM);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalModify.GCPhysOld);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalModify.GCPhysNew);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalModify.cb);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalModify.enmKind);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalModify.fHasHCHandler);
+    GEN_CHECK_OFF_DOT(REMHANDLERNOTIFICATION, u.PhysicalModify.fRestoreAsRAM);
+    GEN_CHECK_OFF(REMHANDLERNOTIFICATION, idxSelf);
+    GEN_CHECK_OFF(REMHANDLERNOTIFICATION, idxNext);
+
+    GEN_CHECK_SIZE(SELM);
+    GEN_CHECK_OFF(SELM, offVM);
+    GEN_CHECK_OFF(SELM, aHyperSel[SELM_HYPER_SEL_CS]);
+    GEN_CHECK_OFF(SELM, aHyperSel[SELM_HYPER_SEL_DS]);
+    GEN_CHECK_OFF(SELM, aHyperSel[SELM_HYPER_SEL_CS64]);
+    GEN_CHECK_OFF(SELM, aHyperSel[SELM_HYPER_SEL_TSS]);
+    GEN_CHECK_OFF(SELM, aHyperSel[SELM_HYPER_SEL_TSS_TRAP08]);
+    GEN_CHECK_OFF(SELM, hShadowGdtWriteHandlerType);
+    GEN_CHECK_OFF(SELM, hGuestGdtWriteHandlerType);
+    GEN_CHECK_OFF(SELM, paGdtR3);
+    GEN_CHECK_OFF(SELM, paGdtRC);
+    GEN_CHECK_OFF(SELM, GuestGdtr);
+    GEN_CHECK_OFF(SELM, cbEffGuestGdtLimit);
+    GEN_CHECK_OFF(SELM, hShadowLdtWriteHandlerType);
+    GEN_CHECK_OFF(SELM, hGuestLdtWriteHandlerType);
+    GEN_CHECK_OFF(SELM, pvLdtR3);
+    GEN_CHECK_OFF(SELM, pvLdtRC);
+    GEN_CHECK_OFF(SELM, GCPtrGuestLdt);
+    GEN_CHECK_OFF(SELM, cbLdtLimit);
+    GEN_CHECK_OFF(SELM, offLdtHyper);
+    GEN_CHECK_OFF(SELM, Tss);
+    GEN_CHECK_OFF(SELM, TssTrap08);
+    GEN_CHECK_OFF(SELM, hShadowTssWriteHandlerType);
+    GEN_CHECK_OFF(SELM, hGuestTssWriteHandlerType);
+    GEN_CHECK_OFF(SELM, pvMonShwTssRC);
+    GEN_CHECK_OFF(SELM, GCPtrGuestTss);
+    GEN_CHECK_OFF(SELM, cbGuestTss);
+    GEN_CHECK_OFF(SELM, fGuestTss32Bit);
+    GEN_CHECK_OFF(SELM, cbMonitoredGuestTss);
+    GEN_CHECK_OFF(SELM, GCSelTss);
+    GEN_CHECK_OFF(SELM, fGDTRangeRegistered);
+    GEN_CHECK_OFF(SELM, StatUpdateFromCPUM);
+    GEN_CHECK_OFF(SELM, StatStaleToUnstaleSReg);
+    GEN_CHECK_OFF(SELM, StatLoadHidSelGstNoGood);
+
+    GEN_CHECK_SIZE(TM);
+    GEN_CHECK_OFF(TM, offVM);
+    GEN_CHECK_OFF(TM, pvGIPR3);
+    //GEN_CHECK_OFF(TM, pvGIPR0);
+    GEN_CHECK_OFF(TM, pvGIPRC);
+    GEN_CHECK_OFF(TMCPU, fTSCTicking);
+    GEN_CHECK_OFF(TM, enmTSCMode);
+    GEN_CHECK_OFF(TM, fTSCTiedToExecution);
+    GEN_CHECK_OFF(TMCPU, offTSCRawSrc);
+    GEN_CHECK_OFF(TMCPU, u64TSC);
+    GEN_CHECK_OFF(TM, cTSCTicksPerSecond);
+    GEN_CHECK_OFF(TM, cVirtualTicking);
+    GEN_CHECK_OFF(TM, fVirtualWarpDrive);
+    GEN_CHECK_OFF(TM, fVirtualSyncTicking);
+    GEN_CHECK_OFF(TM, fVirtualSyncCatchUp);
+    GEN_CHECK_OFF(TM, u32VirtualWarpDrivePercentage);
+    GEN_CHECK_OFF(TM, u64VirtualOffset);
+    GEN_CHECK_OFF(TM, u64Virtual);
+    GEN_CHECK_OFF(TM, u64VirtualRawPrev);
+    GEN_CHECK_OFF(TM, VirtualGetRawDataR3);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR3.pu64Prev);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR3.pfnBad);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR3.pfnRediscover);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR3.c1nsSteps);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR3.cBadPrev);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR3.cExpired);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR3.cUpdateRaces);
+    GEN_CHECK_OFF(TM, VirtualGetRawDataR0);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR0.pu64Prev);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR0.pfnBad);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR0.pfnRediscover);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR0.c1nsSteps);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR0.cBadPrev);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR0.cExpired);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataR0.cUpdateRaces);
+    GEN_CHECK_OFF(TM, VirtualGetRawDataRC);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataRC.pu64Prev);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataRC.pfnBad);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataRC.pfnRediscover);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataRC.c1nsSteps);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataRC.cBadPrev);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataRC.cExpired);
+    GEN_CHECK_OFF_DOT(TM, VirtualGetRawDataRC.cUpdateRaces);
+    GEN_CHECK_OFF(TM, pfnVirtualGetRawR3);
+    GEN_CHECK_OFF(TM, pfnVirtualGetRawR0);
+    GEN_CHECK_OFF(TM, pfnVirtualGetRawRC);
+    GEN_CHECK_OFF(TM, u64VirtualWarpDriveStart);
+    GEN_CHECK_OFF(TM, u64VirtualSync);
+    GEN_CHECK_OFF(TM, offVirtualSync);
+    GEN_CHECK_OFF(TM, offVirtualSyncGivenUp);
+    GEN_CHECK_OFF(TM, u64VirtualSyncCatchUpPrev);
+    GEN_CHECK_OFF(TM, u32VirtualSyncCatchUpPercentage);
+    GEN_CHECK_OFF(TM, u32VirtualSyncScheduleSlack);
+    GEN_CHECK_OFF(TM, u64VirtualSyncCatchUpStopThreshold);
+    GEN_CHECK_OFF(TM, u64VirtualSyncCatchUpGiveUpThreshold);
+    GEN_CHECK_OFF(TM, aVirtualSyncCatchUpPeriods);
+    GEN_CHECK_OFF_DOT(TM, aVirtualSyncCatchUpPeriods[0].u64Start);
+    GEN_CHECK_OFF_DOT(TM, aVirtualSyncCatchUpPeriods[0].u32Percentage);
+    GEN_CHECK_OFF_DOT(TM, aVirtualSyncCatchUpPeriods[1].u64Start);
+    GEN_CHECK_OFF_DOT(TM, aVirtualSyncCatchUpPeriods[1].u32Percentage);
+    GEN_CHECK_OFF(TM, pTimer);
+    GEN_CHECK_OFF(TM, u32TimerMillies);
+    GEN_CHECK_OFF(TM, pFree);
+    GEN_CHECK_OFF(TM, pCreated);
+    GEN_CHECK_OFF(TM, paTimerQueuesR3);
+    GEN_CHECK_OFF(TM, paTimerQueuesR0);
+    GEN_CHECK_OFF(TM, paTimerQueuesRC);
+    GEN_CHECK_OFF(TM, TimerCritSect);
+    GEN_CHECK_OFF(TM, VirtualSyncLock);
+    GEN_CHECK_OFF(TM, StatDoQueues);
+    GEN_CHECK_OFF(TM, StatTimerCallbackSetFF);
+    GEN_CHECK_SIZE(TMTIMER);
+    GEN_CHECK_OFF(TMTIMER, u64Expire);
+    GEN_CHECK_OFF(TMTIMER, enmClock);
+    GEN_CHECK_OFF(TMTIMER, enmType);
+    GEN_CHECK_OFF_DOT(TMTIMER, u.Dev.pfnTimer);
+    GEN_CHECK_OFF_DOT(TMTIMER, u.Dev.pDevIns);
+    GEN_CHECK_OFF_DOT(TMTIMER, u.Drv.pfnTimer);
+    GEN_CHECK_OFF_DOT(TMTIMER, u.Drv.pDrvIns);
+    GEN_CHECK_OFF_DOT(TMTIMER, u.Internal.pfnTimer);
+    GEN_CHECK_OFF_DOT(TMTIMER, u.External.pfnTimer);
+    GEN_CHECK_OFF(TMTIMER, enmState);
+    GEN_CHECK_OFF(TMTIMER, offScheduleNext);
+    GEN_CHECK_OFF(TMTIMER, offNext);
+    GEN_CHECK_OFF(TMTIMER, offPrev);
+    GEN_CHECK_OFF(TMTIMER, pVMR0);
+    GEN_CHECK_OFF(TMTIMER, pVMR3);
+    GEN_CHECK_OFF(TMTIMER, pVMRC);
+    GEN_CHECK_OFF(TMTIMER, uHzHint);
+    GEN_CHECK_OFF(TMTIMER, pvUser);
+    GEN_CHECK_OFF(TMTIMER, pCritSect);
+    GEN_CHECK_OFF(TMTIMER, pBigNext);
+    GEN_CHECK_OFF(TMTIMER, pBigPrev);
+    GEN_CHECK_OFF(TMTIMER, pszDesc);
+    GEN_CHECK_SIZE(TMTIMERQUEUE);
+    GEN_CHECK_OFF(TMTIMERQUEUE, offActive);
+    GEN_CHECK_OFF(TMTIMERQUEUE, offSchedule);
+    GEN_CHECK_OFF(TMTIMERQUEUE, enmClock);
+
+    GEN_CHECK_SIZE(TRPM);
+    GEN_CHECK_SIZE(TRPMCPU);
+    GEN_CHECK_SIZE(VM);  // has .mac
+    GEN_CHECK_SIZE(VMM);
+    GEN_CHECK_OFF(VMM, offVM);
+    GEN_CHECK_OFF(VMM, cbCoreCode);
+    GEN_CHECK_OFF(VMM, HCPhysCoreCode);
+    GEN_CHECK_OFF(VMM, pvCoreCodeR3);
+    GEN_CHECK_OFF(VMM, pvCoreCodeR0);
+    GEN_CHECK_OFF(VMM, pvCoreCodeRC);
+    GEN_CHECK_OFF(VMM, enmSwitcher);
+    GEN_CHECK_OFF(VMM, aoffSwitchers);
+    GEN_CHECK_OFF_DOT(VMM, aoffSwitchers[1]);
+    GEN_CHECK_OFF(VMM, pfnR0ToRawMode);
+    GEN_CHECK_OFF(VMM, pfnRCToHost);
+    GEN_CHECK_OFF(VMM, pfnCallTrampolineRC);
+    GEN_CHECK_OFF(VMM, pfnCPUMRCResumeGuest);
+    GEN_CHECK_OFF(VMM, pfnCPUMRCResumeGuestV86);
+    GEN_CHECK_OFF(VMM, pRCLoggerRC);
+    GEN_CHECK_OFF(VMM, pRCLoggerR3);
+    GEN_CHECK_OFF(VMM, cbRCLogger);
+    GEN_CHECK_OFF(VMM, fRCLoggerFlushingDisabled);
+    GEN_CHECK_OFF(VMM, fStackGuardsStationed);
+    GEN_CHECK_OFF(VMM, fUsePeriodicPreemptionTimers);
+    GEN_CHECK_OFF(VMM, pYieldTimer);
+    GEN_CHECK_OFF(VMM, cYieldResumeMillies);
+    GEN_CHECK_OFF(VMM, cYieldEveryMillies);
+    GEN_CHECK_OFF(VMM, pahEvtRendezvousEnterOrdered);
+    GEN_CHECK_OFF(VMM, hEvtRendezvousEnterOneByOne);
+    GEN_CHECK_OFF(VMM, hEvtMulRendezvousEnterAllAtOnce);
+    GEN_CHECK_OFF(VMM, hEvtMulRendezvousDone);
+    GEN_CHECK_OFF(VMM, hEvtRendezvousDoneCaller);
+    GEN_CHECK_OFF(VMM, pfnRendezvous);
+    GEN_CHECK_OFF(VMM, pvRendezvousUser);
+    GEN_CHECK_OFF(VMM, fRendezvousFlags);
+    GEN_CHECK_OFF(VMM, cRendezvousEmtsEntered);
+    GEN_CHECK_OFF(VMM, cRendezvousEmtsDone);
+    GEN_CHECK_OFF(VMM, cRendezvousEmtsReturned);
+    GEN_CHECK_OFF(VMM, i32RendezvousStatus);
+    GEN_CHECK_OFF(VMM, u32RendezvousLock);
+    GEN_CHECK_OFF(VMM, szRing0AssertMsg1);
+    GEN_CHECK_OFF(VMM, szRing0AssertMsg2);
+    GEN_CHECK_OFF(VMM, StatRunRC);
+    GEN_CHECK_OFF(VMM, StatRZCallPGMLock);
+    GEN_CHECK_OFF(VMMCPU, iLastGZRc);
+    GEN_CHECK_OFF(VMMCPU, pbEMTStackR3);
+    GEN_CHECK_OFF(VMMCPU, pbEMTStackRC);
+    GEN_CHECK_OFF(VMMCPU, pbEMTStackBottomRC);
+#ifdef LOG_ENABLED
+    GEN_CHECK_OFF(VMMCPU, pR0LoggerR0);
+    GEN_CHECK_OFF(VMMCPU, pR0LoggerR3);
+#endif
+    GEN_CHECK_OFF(VMMCPU, cCallRing3Disabled);
+    GEN_CHECK_OFF(VMMCPU, enmCallRing3Operation);
+    GEN_CHECK_OFF(VMMCPU, rcCallRing3);
+    GEN_CHECK_OFF(VMMCPU, u64CallRing3Arg);
+    GEN_CHECK_OFF(VMMCPU, CallRing3JmpBufR0);
+    GEN_CHECK_OFF_DOT(VMMCPU, CallRing3JmpBufR0.SpCheck);
+    GEN_CHECK_OFF_DOT(VMMCPU, CallRing3JmpBufR0.SpResume);
+
+    GEN_CHECK_SIZE(RTPINGPONG);
+    GEN_CHECK_SIZE(RTCRITSECT);
+    GEN_CHECK_OFF(RTCRITSECT, u32Magic);
+    GEN_CHECK_OFF(RTCRITSECT, cLockers);
+    GEN_CHECK_OFF(RTCRITSECT, NativeThreadOwner);
+    GEN_CHECK_OFF(RTCRITSECT, cNestings);
+    GEN_CHECK_OFF(RTCRITSECT, fFlags);
+    GEN_CHECK_OFF(RTCRITSECT, EventSem);
+    GEN_CHECK_OFF(RTCRITSECT, pValidatorRec);
+
+    GEN_CHECK_SIZE(CSAM);
+    GEN_CHECK_OFF(CSAM, offVM);
+    GEN_CHECK_OFF(CSAM, pPageTree);
+    GEN_CHECK_OFF(CSAM, aDangerousInstr);
+    GEN_CHECK_OFF(CSAM, aDangerousInstr[1]);
+    GEN_CHECK_OFF(CSAM, aDangerousInstr[CSAM_MAX_DANGR_INSTR - 1]);
+    GEN_CHECK_OFF(CSAM, cDangerousInstr);
+    GEN_CHECK_OFF(CSAM, iDangerousInstr);
+    GEN_CHECK_OFF(CSAM, pPDBitmapGC);
+    GEN_CHECK_OFF(CSAM, pPDHCBitmapGC);
+    GEN_CHECK_OFF(CSAM, pPDBitmapHC);
+    GEN_CHECK_OFF(CSAM, pPDGCBitmapHC);
+    GEN_CHECK_OFF(CSAM, savedstate);
+    GEN_CHECK_OFF_DOT(CSAM, savedstate.pSSM);
+    GEN_CHECK_OFF_DOT(CSAM, savedstate.cPageRecords);
+    GEN_CHECK_OFF_DOT(CSAM, savedstate.cPatchPageRecords);
+    GEN_CHECK_OFF(CSAM, cDirtyPages);
+    GEN_CHECK_OFF(CSAM, pvDirtyBasePage);
+    GEN_CHECK_OFF_DOT(CSAM, pvDirtyBasePage[1]);
+    GEN_CHECK_OFF_DOT(CSAM, pvDirtyBasePage[CSAM_MAX_DIRTY_PAGES - 1]);
+    GEN_CHECK_OFF(CSAM, pvDirtyFaultPage);
+    GEN_CHECK_OFF_DOT(CSAM, pvDirtyFaultPage[1]);
+    GEN_CHECK_OFF_DOT(CSAM, pvDirtyFaultPage[CSAM_MAX_DIRTY_PAGES - 1]);
+    GEN_CHECK_OFF(CSAM, pvCallInstruction);
+    GEN_CHECK_OFF(CSAM, iCallInstruction);
+    GEN_CHECK_OFF(CSAM, fScanningStarted);
+    GEN_CHECK_OFF(CSAM, fGatesChecked);
+    GEN_CHECK_OFF(CSAM, StatNrTraps);
+    GEN_CHECK_OFF(CSAM, StatNrPages);
+
+    GEN_CHECK_SIZE(PATM);
+    GEN_CHECK_OFF(PATM, offVM);
+    GEN_CHECK_OFF(PATM, pPatchMemGC);
+    GEN_CHECK_OFF(PATM, pPatchMemHC);
+    GEN_CHECK_OFF(PATM, cbPatchMem);
+    GEN_CHECK_OFF(PATM, offPatchMem);
+    GEN_CHECK_OFF(PATM, fOutOfMemory);
+    GEN_CHECK_OFF(PATM, deltaReloc);
+    GEN_CHECK_OFF(PATM, pGCStateGC);
+    GEN_CHECK_OFF(PATM, pGCStateHC);
+    GEN_CHECK_OFF(PATM, pGCStackGC);
+    GEN_CHECK_OFF(PATM, pGCStackHC);
+    GEN_CHECK_OFF(PATM, pCPUMCtxGC);
+    GEN_CHECK_OFF(PATM, pStatsGC);
+    GEN_CHECK_OFF(PATM, pStatsHC);
+    GEN_CHECK_OFF(PATM, uCurrentPatchIdx);
+    GEN_CHECK_OFF(PATM, ulCallDepth);
+    GEN_CHECK_OFF(PATM, cPageRecords);
+    GEN_CHECK_OFF(PATM, pPatchedInstrGCLowest);
+    GEN_CHECK_OFF(PATM, pPatchedInstrGCHighest);
+    GEN_CHECK_OFF(PATM, PatchLookupTreeHC);
+    GEN_CHECK_OFF(PATM, PatchLookupTreeGC);
+    GEN_CHECK_OFF(PATM, pfnHelperCallGC);
+    GEN_CHECK_OFF(PATM, pfnHelperRetGC);
+    GEN_CHECK_OFF(PATM, pfnHelperJumpGC);
+    GEN_CHECK_OFF(PATM, pfnHelperIretGC);
+    GEN_CHECK_OFF(PATM, pGlobalPatchRec);
+    GEN_CHECK_OFF(PATM, pfnSysEnterGC);
+    GEN_CHECK_OFF(PATM, pfnSysEnterPatchGC);
+    GEN_CHECK_OFF(PATM, uSysEnterPatchIdx);
+    GEN_CHECK_OFF(PATM, pvFaultMonitor);
+    GEN_CHECK_OFF(PATM, mmio);
+    GEN_CHECK_OFF_DOT(PATM, mmio.GCPhys);
+    GEN_CHECK_OFF_DOT(PATM, mmio.pCachedData);
+    GEN_CHECK_OFF(PATM, savedstate);
+    GEN_CHECK_OFF_DOT(PATM, savedstate.pSSM);
+    GEN_CHECK_OFF_DOT(PATM, savedstate.cPatches);
+    GEN_CHECK_OFF(PATM, StatNrOpcodeRead);
+    GEN_CHECK_OFF(PATM, StatU32FunctionMaxSlotsUsed);
+
+    GEN_CHECK_SIZE(PATMGCSTATE);
+    GEN_CHECK_OFF(PATMGCSTATE, uVMFlags);
+    GEN_CHECK_OFF(PATMGCSTATE, uPendingAction);
+    GEN_CHECK_OFF(PATMGCSTATE, uPatchCalls);
+    GEN_CHECK_OFF(PATMGCSTATE, uScratch);
+    GEN_CHECK_OFF(PATMGCSTATE, uIretEFlags);
+    GEN_CHECK_OFF(PATMGCSTATE, uIretCS);
+    GEN_CHECK_OFF(PATMGCSTATE, uIretEIP);
+    GEN_CHECK_OFF(PATMGCSTATE, Psp);
+    GEN_CHECK_OFF(PATMGCSTATE, fPIF);
+    GEN_CHECK_OFF(PATMGCSTATE, GCPtrInhibitInterrupts);
+    GEN_CHECK_OFF(PATMGCSTATE, Restore);
+    GEN_CHECK_OFF_DOT(PATMGCSTATE, Restore.uEAX);
+    GEN_CHECK_OFF_DOT(PATMGCSTATE, Restore.uECX);
+    GEN_CHECK_OFF_DOT(PATMGCSTATE, Restore.uEDI);
+    GEN_CHECK_OFF_DOT(PATMGCSTATE, Restore.eFlags);
+    GEN_CHECK_OFF_DOT(PATMGCSTATE, Restore.uFlags);
+    GEN_CHECK_SIZE(PATMTREES);
+    GEN_CHECK_OFF(PATMTREES, PatchTree);
+    GEN_CHECK_OFF(PATMTREES, PatchTreeByPatchAddr);
+    GEN_CHECK_OFF(PATMTREES, PatchTreeByPage);
+    GEN_CHECK_SIZE(PATMPATCHREC);
+    GEN_CHECK_OFF(PATMPATCHREC, Core);
+    GEN_CHECK_OFF(PATMPATCHREC, CoreOffset);
+    GEN_CHECK_OFF(PATMPATCHREC, patch);
+    GEN_CHECK_SIZE(PATCHINFO);
+    GEN_CHECK_OFF(PATCHINFO, uState);
+    GEN_CHECK_OFF(PATCHINFO, uOldState);
+    GEN_CHECK_OFF(PATCHINFO, uOpMode);
+    GEN_CHECK_OFF(PATCHINFO, unusedHC);
+    GEN_CHECK_OFF(PATCHINFO, pPrivInstrGC);
+    GEN_CHECK_OFF(PATCHINFO, aPrivInstr);
+    GEN_CHECK_OFF(PATCHINFO, aPrivInstr[1]);
+    GEN_CHECK_OFF(PATCHINFO, aPrivInstr[MAX_INSTR_SIZE - 1]);
+    GEN_CHECK_OFF(PATCHINFO, cbPrivInstr);
+    GEN_CHECK_OFF(PATCHINFO, opcode);
+    GEN_CHECK_OFF(PATCHINFO, cbPatchJump);
+    GEN_CHECK_OFF(PATCHINFO, pPatchJumpDestGC);
+    GEN_CHECK_OFF(PATCHINFO, pPatchBlockOffset);
+    GEN_CHECK_OFF(PATCHINFO, cbPatchBlockSize);
+    GEN_CHECK_OFF(PATCHINFO, uCurPatchOffset);
+    GEN_CHECK_OFF(PATCHINFO, flags);
+    GEN_CHECK_OFF(PATCHINFO, pInstrGCLowest);
+    GEN_CHECK_OFF(PATCHINFO, pInstrGCHighest);
+    GEN_CHECK_OFF(PATCHINFO, FixupTree);
+    GEN_CHECK_OFF(PATCHINFO, nrFixups);
+    GEN_CHECK_OFF(PATCHINFO, JumpTree);
+    GEN_CHECK_OFF(PATCHINFO, nrJumpRecs);
+    GEN_CHECK_OFF(PATCHINFO, Patch2GuestAddrTree);
+    GEN_CHECK_OFF(PATCHINFO, Guest2PatchAddrTree);
+    GEN_CHECK_OFF(PATCHINFO, nrPatch2GuestRecs);
+    GEN_CHECK_OFF(PATCHINFO, unused);
+    GEN_CHECK_OFF_DOT(PATCHINFO, unused.pPatchLocStartHC);
+    GEN_CHECK_OFF_DOT(PATCHINFO, unused.pPatchLocEndHC);
+    GEN_CHECK_OFF_DOT(PATCHINFO, unused.pGuestLoc);
+    GEN_CHECK_OFF_DOT(PATCHINFO, unused.opsize);
+    GEN_CHECK_OFF(PATCHINFO, pTempInfo);
+    GEN_CHECK_OFF(PATCHINFO, pTrampolinePatchesHead);
+    GEN_CHECK_OFF(PATCHINFO, cCodeWrites);
+    GEN_CHECK_OFF(PATCHINFO, cTraps);
+    GEN_CHECK_OFF(PATCHINFO, cInvalidWrites);
+    GEN_CHECK_OFF(PATCHINFO, uPatchIdx);
+    GEN_CHECK_OFF(PATCHINFO, bDirtyOpcode);
+    GEN_CHECK_SIZE(PATMPATCHPAGE);
+    GEN_CHECK_OFF(PATMPATCHPAGE, Core);
+    GEN_CHECK_OFF(PATMPATCHPAGE, pLowestAddrGC);
+    GEN_CHECK_OFF(PATMPATCHPAGE, pHighestAddrGC);
+    GEN_CHECK_OFF(PATMPATCHPAGE, cCount);
+    GEN_CHECK_OFF(PATMPATCHPAGE, cMaxPatches);
+    GEN_CHECK_OFF(PATMPATCHPAGE, papPatch);
+#endif
+
+    GEN_CHECK_SIZE(APIC);
+    GEN_CHECK_OFF(APIC, pvApicPibR0);
+    GEN_CHECK_OFF(APIC, pvApicPibR3);
+    GEN_CHECK_OFF(APIC, cbApicPib);
+    GEN_CHECK_OFF(APIC, enmMaxMode);
+    GEN_CHECK_OFF(APICCPU, pvApicPageR0);
+    GEN_CHECK_OFF(APICCPU, pvApicPageR3);
+    GEN_CHECK_OFF(APICCPU, cbApicPage);
+    GEN_CHECK_OFF(APICCPU, pvApicPibR0);
+    GEN_CHECK_OFF(APICCPU, pvApicPibR3);
+    GEN_CHECK_OFF(APICCPU, ApicPibLevel);
+    GEN_CHECK_OFF(APICCPU, hTimer);
+
+    GEN_CHECK_SIZE(VM);
+    GEN_CHECK_OFF(VM, enmVMState);
+    GEN_CHECK_OFF(VM, fGlobalForcedActions);
+    GEN_CHECK_OFF(VM, paVMPagesR3);
+    GEN_CHECK_OFF(VM, pSession);
+    GEN_CHECK_OFF(VM, pUVM);
+    GEN_CHECK_OFF(VM, pVMR3);
+    GEN_CHECK_OFF(VM, pVMR0ForCall);
+    GEN_CHECK_OFF(VM, pVMRC);
+#ifdef IN_RING0
+    GEN_CHECK_OFF(VM, hSelfUnsafe);
+    GEN_CHECK_OFF(VM, cCpusUnsafe);
+#else
+    GEN_CHECK_OFF(VM, hSelf);
+    GEN_CHECK_OFF(VM, cCpus);
+#endif
+    GEN_CHECK_OFF(VM, uCpuExecutionCap);
+    GEN_CHECK_OFF(VM, cbSelf);
+    GEN_CHECK_OFF(VM, bMainExecutionEngine);
+    GEN_CHECK_OFF(VM, fHMEnabled);
+    GEN_CHECK_OFF(VM, fUseLargePages);
+    GEN_CHECK_OFF(VM, hTraceBufR3);
+    GEN_CHECK_OFF(VM, hTraceBufR0);
+    GEN_CHECK_OFF(VM, cpum);
+    GEN_CHECK_OFF(VM, vmm);
+    GEN_CHECK_OFF(VM, pgm);
+    GEN_CHECK_OFF(VM, hm);
+    GEN_CHECK_OFF(VM, trpm);
+    GEN_CHECK_OFF(VM, selm);
+    GEN_CHECK_OFF(VM, mm);
+    GEN_CHECK_OFF(VM, pdm);
+    GEN_CHECK_OFF(VM, iom);
+#ifdef VBOX_WITH_RAW_MODE
+    GEN_CHECK_OFF(VM, patm);
+    GEN_CHECK_OFF(VM, csam);
+#endif
+    GEN_CHECK_OFF(VM, em);
+    GEN_CHECK_OFF(VM, tm);
+    GEN_CHECK_OFF(VM, dbgf);
+    GEN_CHECK_OFF(VM, ssm);
+    GEN_CHECK_OFF(VM, gim);
+    GEN_CHECK_OFF(VM, vm);
+    GEN_CHECK_OFF(VM, cfgm);
+    GEN_CHECK_OFF(VM, apic);
+
+
+    GEN_CHECK_SIZE(VMCPU);
+    GEN_CHECK_OFF(VMCPU, fLocalForcedActions);
+    GEN_CHECK_OFF(VMCPU, enmState);
+    GEN_CHECK_OFF(VMCPU, pUVCpu);
+    GEN_CHECK_OFF(VMCPU, pVMR3);
+    GEN_CHECK_OFF(VMCPU, pVCpuR0ForVtg);
+    GEN_CHECK_OFF(VMCPU, pVMRC);
+    GEN_CHECK_OFF(VMCPU, idCpu);
+    GEN_CHECK_OFF(VMCPU, hNativeThread);
+    GEN_CHECK_OFF(VMCPU, hNativeThreadR0);
+    GEN_CHECK_OFF(VMCPU, idHostCpu);
+    GEN_CHECK_OFF(VMCPU, fTraceGroups);
+    GEN_CHECK_OFF(VMCPU, uAdHoc);
+    GEN_CHECK_OFF(VMCPU, aStatAdHoc);
+    GEN_CHECK_OFF(VMCPU, hm);
+    GEN_CHECK_OFF(VMCPU, em);
+    GEN_CHECK_OFF(VMCPU, iem);
+    GEN_CHECK_OFF(VMCPU, trpm);
+    GEN_CHECK_OFF(VMCPU, tm);
+    GEN_CHECK_OFF(VMCPU, vmm);
+    GEN_CHECK_OFF(VMCPU, pdm);
+    GEN_CHECK_OFF(VMCPU, iom);
+    GEN_CHECK_OFF(VMCPU, dbgf);
+    GEN_CHECK_OFF(VMCPU, gim);
+    GEN_CHECK_OFF(VMCPU, apic);
+    GEN_CHECK_OFF(VMCPU, pgm);
+    GEN_CHECK_OFF(VMCPU, cpum);
+
+#ifndef VBOX_FOR_DTRACE_LIB
+    GEN_CHECK_SIZE(DISCPUSTATE);
+    GEN_CHECK_OFF(DISCPUSTATE, Param1);
+    GEN_CHECK_OFF(DISCPUSTATE, Param2);
+    GEN_CHECK_OFF(DISCPUSTATE, Param3);
+    GEN_CHECK_OFF(DISCPUSTATE, i32SibDisp);
+    GEN_CHECK_OFF(DISCPUSTATE, fFilter);
+    GEN_CHECK_OFF(DISCPUSTATE, uInstrAddr);
+    GEN_CHECK_OFF(DISCPUSTATE, abInstr);
+    GEN_CHECK_OFF(DISCPUSTATE, pvUser);
+#endif
diff --git a/src/VBox/VMM/testcase/tstVMStructDTrace.cpp b/src/VBox/VMM/testcase/tstVMStructDTrace.cpp
new file mode 100644
index 00000000..496e8537
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstVMStructDTrace.cpp
@@ -0,0 +1,144 @@
+/* $Id: tstVMStructDTrace.cpp $ */
+/** @file
+ * tstVMMStructDTrace - Generates the DTrace test scripts for check that C/C++
+ *                      and DTrace has the same understand of the VM, VMCPU and
+ *                      other structures.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define IN_TSTVMSTRUCT 1
+#define IN_TSTVMSTRUCTGC 1
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/stam.h>
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include "CFGMInternal.h"
+#include "CPUMInternal.h"
+#include "MMInternal.h"
+#include "PGMInternal.h"
+#include "SELMInternal.h"
+#include "TRPMInternal.h"
+#include "TMInternal.h"
+#include "IOMInternal.h"
+#include "HMInternal.h"
+#include "APICInternal.h"
+#include "VMMInternal.h"
+#include "DBGFInternal.h"
+#include "GIMInternal.h"
+#include "STAMInternal.h"
+#include "EMInternal.h"
+#include "IEMInternal.h"
+#include "NEMInternal.h"
+#ifdef VBOX_WITH_RAW_MODE
+# include "CSAMInternal.h"
+# include "PATMInternal.h"
+#endif
+#include <VBox/vmm/vm.h>
+#include <VBox/param.h>
+#include <iprt/x86.h>
+#include <iprt/assert.h>
+
+/* we don't use iprt here because we wish to run without trouble. */
+#include <stdio.h>
+
+
+int main()
+{
+    /*
+     * File header and pragmas.
+     */
+    printf("#pragma D option quiet\n");
+//    printf("#pragma D depends_on library x86.d\n");
+//    printf("#pragma D depends_on library cpumctx.d\n");
+//    printf("#pragma D depends_on library CPUMInternal.d\n");
+//    printf("#pragma D depends_on library vm.d\n");
+
+    printf("int g_cErrors;\n"
+           "\n"
+           "dtrace:::BEGIN\n"
+           "{\n"
+           "    g_cErrors = 0;\n"
+           "}\n"
+           "\n"
+           );
+
+    /*
+     * Test generator macros.
+     */
+#define GEN_CHECK_SIZE(s) \
+    printf("dtrace:::BEGIN\n" \
+           "/sizeof(" #s ") != %u/\n" \
+           "{\n" \
+           "    printf(\"error: sizeof(" #s ") should be %u, not %%u\\n\", sizeof(" #s "));\n" \
+           "    g_cErrors++;\n" \
+           "}\n" \
+           "\n", \
+           (unsigned)sizeof(s), (unsigned)sizeof(s))
+
+#if 1
+# define GEN_CHECK_OFF(s, m) \
+   printf("dtrace:::BEGIN\n" \
+          "/offsetof(" #s ", " #m ") != %u/\n" \
+          "{\n" \
+          "    printf(\"error: offsetof(" #s ", " #m ") should be %u, not %%u\\n\", offsetof(" #s ", " #m "));\n" \
+          "    g_cErrors++;\n" \
+          "}\n" \
+          "\n", \
+          (unsigned)RT_OFFSETOF(s, m), (unsigned)RT_OFFSETOF(s, m))
+
+#else
+# define GEN_CHECK_OFF(s, m) do { } while (0)
+#endif
+
+#define GEN_CHECK_OFF_DOT(s, m)  do { } while (0)
+
+
+    /*
+     *  Body.
+     */
+#define VBOX_FOR_DTRACE_LIB
+#include "tstVMStruct.h"
+
+    /*
+     * Footer.
+     */
+    printf("dtrace:::BEGIN\n"
+           "/g_cErrors != 0/\n"
+           "{\n"
+           "    printf(\"%%u errors!\\n\", g_cErrors);\n"
+           "    exit(1);\n"
+           "}\n"
+           "\n"
+           "dtrace:::BEGIN\n"
+           "{\n"
+           "    printf(\"Success!\\n\");\n"
+           "    exit(0);\n"
+           "}\n"
+           "\n"
+           );
+
+
+    return (0);
+}
+
diff --git a/src/VBox/VMM/testcase/tstVMStructRC.cpp b/src/VBox/VMM/testcase/tstVMStructRC.cpp
new file mode 100644
index 00000000..8c065fff
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstVMStructRC.cpp
@@ -0,0 +1,99 @@
+/* $Id: tstVMStructRC.cpp $ */
+/** @file
+ * tstVMMStructRC - Generate structure member and size checks from the
+ *                  RC perspective.
+ *
+ * This is built using the VBOXRC template but linked into a host
+ * ring-3 executable, rather hacky.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*
+ * Sanity checks.
+ */
+#ifndef IN_RC
+# error Incorrect template!
+#endif
+#if defined(IN_RING3) || defined(IN_RING0)
+# error Incorrect template!
+#endif
+
+#include <VBox/types.h>
+#include <iprt/assert.h>
+AssertCompileSize(uint8_t,  1);
+AssertCompileSize(uint16_t, 2);
+AssertCompileSize(uint32_t, 4);
+AssertCompileSize(uint64_t, 8);
+AssertCompileSize(RTRCPTR,  4);
+#ifdef VBOX_WITH_64_BITS_GUESTS
+AssertCompileSize(RTGCPTR,  8);
+#else
+AssertCompileSize(RTGCPTR,  4);
+#endif
+AssertCompileSize(RTGCPHYS, 8);
+AssertCompileSize(RTHCPHYS, 8);
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define IN_TSTVMSTRUCT 1
+#define IN_TSTVMSTRUCTGC 1
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/stam.h>
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include "CFGMInternal.h"
+#include "CPUMInternal.h"
+#include "MMInternal.h"
+#include "PGMInternal.h"
+#include "SELMInternal.h"
+#include "TMInternal.h"
+#include "IOMInternal.h"
+#include "HMInternal.h"
+#include "APICInternal.h"
+#include "PATMInternal.h"
+#include "VMMInternal.h"
+#include "DBGFInternal.h"
+#include "GIMInternal.h"
+#include "STAMInternal.h"
+#include "CSAMInternal.h"
+#include "EMInternal.h"
+#include "IEMInternal.h"
+#include "NEMInternal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/hm_vmx.h>
+#include <VBox/param.h>
+#include <iprt/x86.h>
+#include <iprt/assert.h>
+
+/* we don't use iprt here because we're pretending to be in GC! */
+#include <stdio.h>
+
+
+int main()
+{
+#define GEN_CHECK_SIZE(s)       printf("    CHECK_SIZE(%s, %u);\n",    #s, (unsigned)sizeof(s))
+#define GEN_CHECK_OFF(s, m)     printf("    CHECK_OFF(%s, %u, %s);\n", #s, (unsigned)RT_OFFSETOF(s, m), #m)
+#define GEN_CHECK_OFF_DOT(s, m) printf("    CHECK_OFF(%s, %u, %s);\n", #s, (unsigned)RT_OFFSETOF(s, m), #m)
+#include "tstVMStruct.h"
+    return (0);
+}
+
diff --git a/src/VBox/VMM/testcase/tstVMStructSize.cpp b/src/VBox/VMM/testcase/tstVMStructSize.cpp
new file mode 100644
index 00000000..5c140f33
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstVMStructSize.cpp
@@ -0,0 +1,464 @@
+/* $Id: tstVMStructSize.cpp $ */
+/** @file
+ * tstVMStructSize - testcase for check structure sizes/alignment
+ *                   and to verify that HC and GC uses the same
+ *                   representation of the structures.
+ */
+
+/*
+ * Copyright (C) 2006-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define IN_TSTVMSTRUCT 1
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/stam.h>
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include "CFGMInternal.h"
+#include "CPUMInternal.h"
+#include "MMInternal.h"
+#include "PGMInternal.h"
+#include "SELMInternal.h"
+#include "TRPMInternal.h"
+#include "TMInternal.h"
+#include "IOMInternal.h"
+#include "SSMInternal.h"
+#include "HMInternal.h"
+#include "VMMInternal.h"
+#include "DBGFInternal.h"
+#include "GIMInternal.h"
+#include "APICInternal.h"
+#include "STAMInternal.h"
+#include "VMInternal.h"
+#include "EMInternal.h"
+#include "IEMInternal.h"
+#include "NEMInternal.h"
+#include "../VMMR0/GMMR0Internal.h"
+#include "../VMMR0/GVMMR0Internal.h"
+#include <VBox/vmm/vm.h>
+#include <VBox/vmm/uvm.h>
+#include <VBox/vmm/gvm.h>
+#include <VBox/param.h>
+#include <VBox/dis.h>
+#include <iprt/x86.h>
+
+#include "tstHelp.h"
+#include <stdio.h>
+
+
+
+int main()
+{
+    int rc = 0;
+    printf("tstVMStructSize: TESTING\n");
+
+    printf("info: struct VM: %d bytes\n", (int)sizeof(VM));
+
+#define CHECK_PADDING_VM(align, member) \
+    do \
+    { \
+        CHECK_PADDING(VM, member, align); \
+        CHECK_MEMBER_ALIGNMENT(VM, member, align); \
+        VM *p = NULL; NOREF(p); \
+        if (sizeof(p->member.padding) >= (ssize_t)sizeof(p->member.s) + 128 + sizeof(p->member.s) / 20) \
+            printf("warning: VM::%-8s: padding=%-5d s=%-5d -> %-4d  suggest=%-5u\n", \
+                   #member, (int)sizeof(p->member.padding), (int)sizeof(p->member.s), \
+                   (int)sizeof(p->member.padding) - (int)sizeof(p->member.s), \
+                   (int)RT_ALIGN_Z(sizeof(p->member.s), (align))); \
+    } while (0)
+
+
+#define CHECK_PADDING_VMCPU(align, member) \
+    do \
+    { \
+        CHECK_PADDING(VMCPU, member, align); \
+        CHECK_MEMBER_ALIGNMENT(VMCPU, member, align); \
+        VMCPU *p = NULL; NOREF(p); \
+        if (sizeof(p->member.padding) >= (ssize_t)sizeof(p->member.s) + 128 + sizeof(p->member.s) / 20) \
+            printf("warning: VMCPU::%-8s: padding=%-5d s=%-5d -> %-4d  suggest=%-5u\n", \
+                   #member, (int)sizeof(p->member.padding), (int)sizeof(p->member.s), \
+                   (int)sizeof(p->member.padding) - (int)sizeof(p->member.s), \
+                   (int)RT_ALIGN_Z(sizeof(p->member.s), (align))); \
+    } while (0)
+
+#define CHECK_CPUMCTXCORE(member) \
+    do { \
+        unsigned off1 = RT_OFFSETOF(CPUMCTX, member) - RT_OFFSETOF(CPUMCTX, rax); \
+        unsigned off2 = RT_OFFSETOF(CPUMCTXCORE, member); \
+        if (off1 != off2) \
+        { \
+            printf("error! CPUMCTX/CORE:: %s! (%#x vs %#x (ctx))\n", #member, off1, off2); \
+            rc++; \
+        } \
+    } while (0)
+
+#define CHECK_PADDING_UVM(align, member) \
+    do \
+    { \
+        CHECK_PADDING(UVM, member, align); \
+        CHECK_MEMBER_ALIGNMENT(UVM, member, align); \
+        UVM *p = NULL; NOREF(p); \
+        if (sizeof(p->member.padding) >= (ssize_t)sizeof(p->member.s) + 128 + sizeof(p->member.s) / 20) \
+            printf("warning: UVM::%-8s: padding=%-5d s=%-5d -> %-4d  suggest=%-5u\n", \
+                   #member, (int)sizeof(p->member.padding), (int)sizeof(p->member.s), \
+                   (int)sizeof(p->member.padding) - (int)sizeof(p->member.s), \
+                   (int)RT_ALIGN_Z(sizeof(p->member.s), (align))); \
+    } while (0)
+
+#define CHECK_PADDING_UVMCPU(align, member) \
+    do \
+    { \
+        CHECK_PADDING(UVMCPU, member, align); \
+        CHECK_MEMBER_ALIGNMENT(UVMCPU, member, align); \
+        UVMCPU *p = NULL; NOREF(p); \
+        if (sizeof(p->member.padding) >= (ssize_t)sizeof(p->member.s) + 128 + sizeof(p->member.s) / 20) \
+            printf("warning: UVMCPU::%-8s: padding=%-5d s=%-5d -> %-4d  suggest=%-5u\n", \
+                   #member, (int)sizeof(p->member.padding), (int)sizeof(p->member.s), \
+                   (int)sizeof(p->member.padding) - (int)sizeof(p->member.s), \
+                   (int)RT_ALIGN_Z(sizeof(p->member.s), (align))); \
+    } while (0)
+
+#define CHECK_PADDING_GVM(align, member) \
+    do \
+    { \
+        CHECK_PADDING(GVM, member, align); \
+        CHECK_MEMBER_ALIGNMENT(GVM, member, align); \
+        GVM *p = NULL; NOREF(p); \
+        if (sizeof(p->member.padding) >= (ssize_t)sizeof(p->member.s) + 128 + sizeof(p->member.s) / 20) \
+            printf("warning: GVM::%-8s: padding=%-5d s=%-5d -> %-4d  suggest=%-5u\n", \
+                   #member, (int)sizeof(p->member.padding), (int)sizeof(p->member.s), \
+                   (int)sizeof(p->member.padding) - (int)sizeof(p->member.s), \
+                   (int)RT_ALIGN_Z(sizeof(p->member.s), (align))); \
+    } while (0)
+
+#define CHECK_PADDING_GVMCPU(align, member) \
+    do \
+    { \
+        CHECK_PADDING(GVMCPU, member, align); \
+        CHECK_MEMBER_ALIGNMENT(GVMCPU, member, align); \
+        GVMCPU *p = NULL; NOREF(p); \
+        if (sizeof(p->member.padding) >= (ssize_t)sizeof(p->member.s) + 128 + sizeof(p->member.s) / 20) \
+            printf("warning: GVMCPU::%-8s: padding=%-5d s=%-5d -> %-4d  suggest=%-5u\n", \
+                   #member, (int)sizeof(p->member.padding), (int)sizeof(p->member.s), \
+                   (int)sizeof(p->member.padding) - (int)sizeof(p->member.s), \
+                   (int)RT_ALIGN_Z(sizeof(p->member.s), (align))); \
+    } while (0)
+
+#define PRINT_OFFSET(strct, member) \
+    do \
+    { \
+        printf("info: %10s::%-24s offset %#6x (%6d) sizeof %4d\n",  #strct, #member, (int)RT_OFFSETOF(strct, member), (int)RT_OFFSETOF(strct, member), (int)RT_SIZEOFMEMB(strct, member)); \
+    } while (0)
+
+
+    CHECK_SIZE(uint128_t, 128/8);
+    CHECK_SIZE(int128_t, 128/8);
+    CHECK_SIZE(uint64_t, 64/8);
+    CHECK_SIZE(int64_t, 64/8);
+    CHECK_SIZE(uint32_t, 32/8);
+    CHECK_SIZE(int32_t, 32/8);
+    CHECK_SIZE(uint16_t, 16/8);
+    CHECK_SIZE(int16_t, 16/8);
+    CHECK_SIZE(uint8_t, 8/8);
+    CHECK_SIZE(int8_t, 8/8);
+
+    CHECK_SIZE(X86DESC, 8);
+    CHECK_SIZE(X86DESC64, 16);
+    CHECK_SIZE(VBOXIDTE, 8);
+    CHECK_SIZE(VBOXIDTR, 10);
+    CHECK_SIZE(VBOXGDTR, 10);
+    CHECK_SIZE(VBOXTSS, 136);
+    CHECK_SIZE(X86FXSTATE, 512);
+    CHECK_SIZE(RTUUID, 16);
+    CHECK_SIZE(X86PTE, 4);
+    CHECK_SIZE(X86PD, PAGE_SIZE);
+    CHECK_SIZE(X86PDE, 4);
+    CHECK_SIZE(X86PT, PAGE_SIZE);
+    CHECK_SIZE(X86PTEPAE, 8);
+    CHECK_SIZE(X86PTPAE, PAGE_SIZE);
+    CHECK_SIZE(X86PDEPAE, 8);
+    CHECK_SIZE(X86PDPAE, PAGE_SIZE);
+    CHECK_SIZE(X86PDPE, 8);
+    CHECK_SIZE(X86PDPT, PAGE_SIZE);
+    CHECK_SIZE(X86PML4E, 8);
+    CHECK_SIZE(X86PML4, PAGE_SIZE);
+
+    PRINT_OFFSET(VM, cpum);
+    CHECK_PADDING_VM(64, cpum);
+    CHECK_PADDING_VM(64, vmm);
+    PRINT_OFFSET(VM, pgm);
+    PRINT_OFFSET(VM, pgm.s.CritSectX);
+    CHECK_PADDING_VM(64, pgm);
+    PRINT_OFFSET(VM, hm);
+    CHECK_PADDING_VM(64, hm);
+    CHECK_PADDING_VM(64, trpm);
+    CHECK_PADDING_VM(64, selm);
+    CHECK_PADDING_VM(64, mm);
+    CHECK_PADDING_VM(64, pdm);
+    PRINT_OFFSET(VM, pdm.s.CritSect);
+    CHECK_PADDING_VM(64, iom);
+    CHECK_PADDING_VM(64, em);
+    /*CHECK_PADDING_VM(64, iem);*/
+    CHECK_PADDING_VM(64, nem);
+    CHECK_PADDING_VM(64, tm);
+    PRINT_OFFSET(VM, tm.s.VirtualSyncLock);
+    CHECK_PADDING_VM(64, dbgf);
+    CHECK_PADDING_VM(64, gim);
+    CHECK_PADDING_VM(64, ssm);
+    CHECK_PADDING_VM(8, vm);
+    CHECK_PADDING_VM(8, cfgm);
+    CHECK_PADDING_VM(8, apic);
+    PRINT_OFFSET(VM, cfgm);
+    PRINT_OFFSET(VM, apCpusR3);
+
+    PRINT_OFFSET(VMCPU, cpum);
+    CHECK_PADDING_VMCPU(64, iem);
+    CHECK_PADDING_VMCPU(64, hm);
+    CHECK_PADDING_VMCPU(64, em);
+    CHECK_PADDING_VMCPU(64, nem);
+    CHECK_PADDING_VMCPU(64, trpm);
+    CHECK_PADDING_VMCPU(64, tm);
+    CHECK_PADDING_VMCPU(64, vmm);
+    CHECK_PADDING_VMCPU(64, pdm);
+    CHECK_PADDING_VMCPU(64, iom);
+    CHECK_PADDING_VMCPU(64, dbgf);
+    CHECK_PADDING_VMCPU(64, gim);
+    CHECK_PADDING_VMCPU(64, apic);
+
+    PRINT_OFFSET(VMCPU, pgm);
+    CHECK_PADDING_VMCPU(4096, pgm);
+    CHECK_PADDING_VMCPU(4096, cpum);
+
+    PVM pVM = NULL; NOREF(pVM);
+
+    CHECK_MEMBER_ALIGNMENT(VMCPU, vmm.s.u64CallRing3Arg, 8);
+#if defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
+    CHECK_MEMBER_ALIGNMENT(VMCPU, vmm.s.CallRing3JmpBufR0, 16);
+    CHECK_MEMBER_ALIGNMENT(VMCPU, vmm.s.CallRing3JmpBufR0.xmm6, 16);
+#endif
+    CHECK_MEMBER_ALIGNMENT(VM, vmm.s.u64LastYield, 8);
+
+    /* the VMCPUs are page aligned TLB hit reasons. */
+    CHECK_SIZE_ALIGNMENT(VMCPU, 4096);
+
+    /* cpumctx */
+    CHECK_MEMBER_ALIGNMENT(CPUMCTX, rax, 32);
+    CHECK_MEMBER_ALIGNMENT(CPUMCTX, idtr.pIdt, 8);
+    CHECK_MEMBER_ALIGNMENT(CPUMCTX, gdtr.pGdt, 8);
+    CHECK_MEMBER_ALIGNMENT(CPUMCTX, SysEnter, 8);
+    CHECK_MEMBER_ALIGNMENT(CPUMCTX, hwvirt, 8);
+    CHECK_CPUMCTXCORE(rax);
+    CHECK_CPUMCTXCORE(rcx);
+    CHECK_CPUMCTXCORE(rdx);
+    CHECK_CPUMCTXCORE(rbx);
+    CHECK_CPUMCTXCORE(rsp);
+    CHECK_CPUMCTXCORE(rbp);
+    CHECK_CPUMCTXCORE(rsi);
+    CHECK_CPUMCTXCORE(rdi);
+    CHECK_CPUMCTXCORE(r8);
+    CHECK_CPUMCTXCORE(r9);
+    CHECK_CPUMCTXCORE(r10);
+    CHECK_CPUMCTXCORE(r11);
+    CHECK_CPUMCTXCORE(r12);
+    CHECK_CPUMCTXCORE(r13);
+    CHECK_CPUMCTXCORE(r14);
+    CHECK_CPUMCTXCORE(r15);
+    CHECK_CPUMCTXCORE(es);
+    CHECK_CPUMCTXCORE(ss);
+    CHECK_CPUMCTXCORE(cs);
+    CHECK_CPUMCTXCORE(ds);
+    CHECK_CPUMCTXCORE(fs);
+    CHECK_CPUMCTXCORE(gs);
+    CHECK_CPUMCTXCORE(rip);
+    CHECK_CPUMCTXCORE(rflags);
+
+#if HC_ARCH_BITS == 32
+    /* CPUMHOSTCTX - lss pair */
+    if (RT_UOFFSETOF(CPUMHOSTCTX, esp) + 4 != RT_UOFFSETOF(CPUMHOSTCTX, ss))
+    {
+        printf("error! CPUMHOSTCTX lss has been split up!\n");
+        rc++;
+    }
+#endif
+    CHECK_SIZE_ALIGNMENT(CPUMCTX, 64);
+    CHECK_SIZE_ALIGNMENT(CPUMHOSTCTX, 64);
+    CHECK_SIZE_ALIGNMENT(CPUMCTXMSRS, 64);
+
+    /* pdm */
+    PRINT_OFFSET(PDMDEVINSR3, Internal);
+    PRINT_OFFSET(PDMDEVINSR3, achInstanceData);
+    CHECK_MEMBER_ALIGNMENT(PDMDEVINSR3, achInstanceData, 64);
+    CHECK_PADDING(PDMDEVINSR3, Internal, 1);
+
+    PRINT_OFFSET(PDMDEVINSR0, Internal);
+    PRINT_OFFSET(PDMDEVINSR0, achInstanceData);
+    CHECK_MEMBER_ALIGNMENT(PDMDEVINSR0, achInstanceData, 64);
+    CHECK_PADDING(PDMDEVINSR0, Internal, 1);
+
+    PRINT_OFFSET(PDMDEVINSRC, Internal);
+    PRINT_OFFSET(PDMDEVINSRC, achInstanceData);
+    CHECK_MEMBER_ALIGNMENT(PDMDEVINSRC, achInstanceData, 64);
+    CHECK_PADDING(PDMDEVINSRC, Internal, 1);
+
+    PRINT_OFFSET(PDMUSBINS, Internal);
+    PRINT_OFFSET(PDMUSBINS, achInstanceData);
+    CHECK_MEMBER_ALIGNMENT(PDMUSBINS, achInstanceData, 32);
+    CHECK_PADDING(PDMUSBINS, Internal, 1);
+
+    PRINT_OFFSET(PDMDRVINS, Internal);
+    PRINT_OFFSET(PDMDRVINS, achInstanceData);
+    CHECK_MEMBER_ALIGNMENT(PDMDRVINS, achInstanceData, 32);
+    CHECK_PADDING(PDMDRVINS, Internal, 1);
+
+    CHECK_PADDING2(PDMCRITSECT);
+    CHECK_PADDING2(PDMCRITSECTRW);
+
+    /* pgm */
+#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE)
+    CHECK_MEMBER_ALIGNMENT(PGMCPU, AutoSet, 8);
+#endif
+    CHECK_MEMBER_ALIGNMENT(PGMCPU, GCPhysCR3, sizeof(RTGCPHYS));
+    CHECK_MEMBER_ALIGNMENT(PGMCPU, aGCPhysGstPaePDs, sizeof(RTGCPHYS));
+    CHECK_MEMBER_ALIGNMENT(PGMCPU, DisState, 8);
+    CHECK_MEMBER_ALIGNMENT(PGMCPU, cPoolAccessHandler, 8);
+    CHECK_MEMBER_ALIGNMENT(PGMPOOLPAGE, idx, sizeof(uint16_t));
+    CHECK_MEMBER_ALIGNMENT(PGMPOOLPAGE, pvPageR3, sizeof(RTHCPTR));
+    CHECK_MEMBER_ALIGNMENT(PGMPOOLPAGE, GCPhys, sizeof(RTGCPHYS));
+    CHECK_SIZE(PGMPAGE, 16);
+    CHECK_MEMBER_ALIGNMENT(PGMRAMRANGE, aPages, 16);
+    CHECK_MEMBER_ALIGNMENT(PGMREGMMIO2RANGE, RamRange, 16);
+
+    /* TM */
+    CHECK_MEMBER_ALIGNMENT(TM, TimerCritSect, sizeof(uintptr_t));
+    CHECK_MEMBER_ALIGNMENT(TM, VirtualSyncLock, sizeof(uintptr_t));
+
+    /* misc */
+    CHECK_PADDING3(EMCPU, u.FatalLongJump, u.achPaddingFatalLongJump);
+    CHECK_SIZE_ALIGNMENT(VMMR0JMPBUF, 8);
+#if 0
+    PRINT_OFFSET(VM, fForcedActions);
+    PRINT_OFFSET(VM, StatQemuToGC);
+    PRINT_OFFSET(VM, StatGCToQemu);
+#endif
+
+    CHECK_MEMBER_ALIGNMENT(IOM, CritSect, sizeof(uintptr_t));
+    CHECK_MEMBER_ALIGNMENT(EMCPU, u.achPaddingFatalLongJump, 32);
+    CHECK_MEMBER_ALIGNMENT(EMCPU, aExitRecords, sizeof(EMEXITREC));
+    CHECK_MEMBER_ALIGNMENT(PGM, CritSectX, sizeof(uintptr_t));
+    CHECK_MEMBER_ALIGNMENT(PDM, CritSect, sizeof(uintptr_t));
+    CHECK_MEMBER_ALIGNMENT(MMHYPERHEAP, Lock, sizeof(uintptr_t));
+
+    /* hm - 32-bit gcc won't align uint64_t naturally, so check. */
+    CHECK_MEMBER_ALIGNMENT(HM, uMaxAsid, 8);
+    CHECK_MEMBER_ALIGNMENT(HM, vmx, 8);
+    CHECK_MEMBER_ALIGNMENT(HM, vmx.Msrs, 8);
+    CHECK_MEMBER_ALIGNMENT(HM, svm, 8);
+    CHECK_MEMBER_ALIGNMENT(HM, PatchTree, 8);
+    CHECK_MEMBER_ALIGNMENT(HM, aPatches, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, vmx, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, vmx.VmcsInfo, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, vmx.VmcsInfo.pfnStartVM, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, vmx.VmcsInfoNstGst, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, vmx.VmcsInfoNstGst.pfnStartVM, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, vmx.RestoreHost, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, vmx.LastError, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, svm, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, svm.pfnVMRun, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, Event, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, Event.u64IntInfo, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, DisState, 8);
+    CHECK_MEMBER_ALIGNMENT(HMCPU, StatEntry, 8);
+
+    /* Make sure the set is large enough and has the correct size. */
+    CHECK_SIZE(VMCPUSET, 32);
+    if (sizeof(VMCPUSET) * 8 < VMM_MAX_CPU_COUNT)
+    {
+        printf("error! VMCPUSET is too small for VMM_MAX_CPU_COUNT=%u!\n", VMM_MAX_CPU_COUNT);
+        rc++;
+    }
+
+    printf("info: struct UVM: %d bytes\n", (int)sizeof(UVM));
+
+    CHECK_PADDING_UVM(32, vm);
+    CHECK_PADDING_UVM(32, mm);
+    CHECK_PADDING_UVM(32, pdm);
+    CHECK_PADDING_UVM(32, stam);
+
+    printf("info: struct UVMCPU: %d bytes\n", (int)sizeof(UVMCPU));
+    CHECK_PADDING_UVMCPU(32, vm);
+
+    CHECK_PADDING_GVM(4, gvmm);
+    CHECK_PADDING_GVM(4, gmm);
+    CHECK_PADDING_GVMCPU(4, gvmm);
+
+    /*
+     * Check that the optimized access macros for PGMPAGE works correctly (kind of
+     * obsolete after dropping raw-mode).
+     */
+    PGMPAGE Page;
+    PGM_PAGE_CLEAR(&Page);
+
+    CHECK_EXPR(PGM_PAGE_GET_HNDL_PHYS_STATE(&Page) == PGM_PAGE_HNDL_PHYS_STATE_NONE);
+    CHECK_EXPR(PGM_PAGE_HAS_ANY_HANDLERS(&Page) == false);
+    CHECK_EXPR(PGM_PAGE_HAS_ACTIVE_HANDLERS(&Page) == false);
+    CHECK_EXPR(PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(&Page) == false);
+
+    PGM_PAGE_SET_HNDL_PHYS_STATE(&Page, PGM_PAGE_HNDL_PHYS_STATE_ALL);
+    CHECK_EXPR(PGM_PAGE_GET_HNDL_PHYS_STATE(&Page) == PGM_PAGE_HNDL_PHYS_STATE_ALL);
+    CHECK_EXPR(PGM_PAGE_HAS_ANY_HANDLERS(&Page) == true);
+    CHECK_EXPR(PGM_PAGE_HAS_ACTIVE_HANDLERS(&Page) == true);
+    CHECK_EXPR(PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(&Page) == true);
+
+    PGM_PAGE_SET_HNDL_PHYS_STATE(&Page, PGM_PAGE_HNDL_PHYS_STATE_WRITE);
+    CHECK_EXPR(PGM_PAGE_GET_HNDL_PHYS_STATE(&Page) == PGM_PAGE_HNDL_PHYS_STATE_WRITE);
+    CHECK_EXPR(PGM_PAGE_HAS_ANY_HANDLERS(&Page) == true);
+    CHECK_EXPR(PGM_PAGE_HAS_ACTIVE_HANDLERS(&Page) == true);
+    CHECK_EXPR(PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(&Page) == false);
+
+#undef AssertFatal
+#define AssertFatal(expr) do { } while (0)
+#undef Assert
+#define Assert(expr)      do { } while (0)
+
+    PGM_PAGE_CLEAR(&Page);
+    CHECK_EXPR(PGM_PAGE_GET_HCPHYS_NA(&Page) == 0);
+    PGM_PAGE_SET_HCPHYS(NULL, &Page, UINT64_C(0x0000fffeff1ff000));
+    CHECK_EXPR(PGM_PAGE_GET_HCPHYS_NA(&Page) == UINT64_C(0x0000fffeff1ff000));
+    PGM_PAGE_SET_HCPHYS(NULL, &Page, UINT64_C(0x0000000000001000));
+    CHECK_EXPR(PGM_PAGE_GET_HCPHYS_NA(&Page) == UINT64_C(0x0000000000001000));
+
+    PGM_PAGE_INIT(&Page, UINT64_C(0x0000feedfacef000), UINT32_C(0x12345678), PGMPAGETYPE_RAM, PGM_PAGE_STATE_ALLOCATED);
+    CHECK_EXPR(PGM_PAGE_GET_HCPHYS_NA(&Page) == UINT64_C(0x0000feedfacef000));
+    CHECK_EXPR(PGM_PAGE_GET_PAGEID(&Page) == UINT32_C(0x12345678));
+    CHECK_EXPR(PGM_PAGE_GET_TYPE_NA(&Page)   == PGMPAGETYPE_RAM);
+    CHECK_EXPR(PGM_PAGE_GET_STATE_NA(&Page)  == PGM_PAGE_STATE_ALLOCATED);
+
+
+    /*
+     * Report result.
+     */
+    if (rc)
+        printf("tstVMStructSize: FAILURE - %d errors\n", rc);
+    else
+        printf("tstVMStructSize: SUCCESS\n");
+    return rc;
+}
+
diff --git a/src/VBox/VMM/testcase/tstX86-1.cpp b/src/VBox/VMM/testcase/tstX86-1.cpp
new file mode 100644
index 00000000..8e38e437
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstX86-1.cpp
@@ -0,0 +1,270 @@
+/* $Id: tstX86-1.cpp $ */
+/** @file
+ * X86 instruction set exploration/testcase #1.
+ */
+
+/*
+ * Copyright (C) 2011-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <iprt/test.h>
+#include <iprt/param.h>
+#include <iprt/mem.h>
+#include <iprt/errcore.h>
+#include <iprt/assert.h>
+#include <iprt/x86.h>
+
+#ifdef RT_OS_WINDOWS
+# include <iprt/win/windows.h>
+#else
+# ifdef RT_OS_DARWIN
+#  define _XOPEN_SOURCE
+# endif
+# include <signal.h>
+# include <ucontext.h>
+# define USE_SIGNAL
+#endif
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+typedef struct TRAPINFO
+{
+    uintptr_t   uTrapPC;
+    uintptr_t   uResumePC;
+    uint8_t     u8Trap;
+    uint8_t     cbInstr;
+    uint8_t     auAlignment[sizeof(uintptr_t) * 2 - 2];
+} TRAPINFO;
+typedef TRAPINFO const *PCTRAPINFO;
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+uint8_t *g_pbEfPage = NULL;
+uint8_t *g_pbEfExecPage = NULL;
+extern TRAPINFO g_aTrapInfo[];
+RT_C_DECLS_END
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+DECLASM(int32_t) x861_Test1(void);
+DECLASM(int32_t) x861_Test2(void);
+DECLASM(int32_t) x861_Test3(void);
+DECLASM(int32_t) x861_Test4(void);
+DECLASM(int32_t) x861_Test5(void);
+DECLASM(int32_t) x861_Test6(void);
+DECLASM(int32_t) x861_Test7(void);
+DECLASM(int32_t) x861_TestFPUInstr1(void);
+
+
+
+static PCTRAPINFO findTrapInfo(uintptr_t uTrapPC, uintptr_t uTrapSP)
+{
+    /* Search by trap program counter. */
+    for (unsigned i = 0; g_aTrapInfo[i].uTrapPC; i++)
+        if (g_aTrapInfo[i].uTrapPC == uTrapPC)
+            return &g_aTrapInfo[i];
+
+    /* Search by return address. */
+    uintptr_t uReturn = *(uintptr_t *)uTrapSP;
+    for (unsigned i = 0; g_aTrapInfo[i].uTrapPC; i++)
+        if (g_aTrapInfo[i].uTrapPC + g_aTrapInfo[i].cbInstr == uReturn)
+            return &g_aTrapInfo[i];
+
+    return NULL;
+}
+
+#ifdef USE_SIGNAL
+static void sigHandler(int iSig, siginfo_t *pSigInfo, void *pvSigCtx)
+{
+    ucontext_t *pCtx = (ucontext_t *)pvSigCtx;
+    NOREF(pSigInfo);
+
+# if defined(RT_ARCH_AMD64) && defined(RT_OS_DARWIN)
+    uintptr_t  *puPC    = (uintptr_t *)&pCtx->uc_mcontext->__ss.__rip;
+    uintptr_t  *puSP    = (uintptr_t *)&pCtx->uc_mcontext->__ss.__rsp;
+    uintptr_t   uTrapNo = pCtx->uc_mcontext->__es.__trapno;
+    uintptr_t   uErr    = pCtx->uc_mcontext->__es.__err;
+    uintptr_t   uCr2    = pCtx->uc_mcontext->__es.__faultvaddr;
+
+# elif defined(RT_ARCH_AMD64) && defined(RT_OS_FREEBSD)
+    uintptr_t  *puPC    = (uintptr_t *)&pCtx->uc_mcontext.mc_rip;
+    uintptr_t  *puSP    = (uintptr_t *)&pCtx->uc_mcontext.mc_rsp;
+    uintptr_t   uTrapNo = ~(uintptr_t)0;
+    uintptr_t   uErr    = ~(uintptr_t)0;
+    uintptr_t   uCr2    = ~(uintptr_t)0;
+
+# elif defined(RT_ARCH_AMD64)
+    uintptr_t  *puPC    = (uintptr_t *)&pCtx->uc_mcontext.gregs[REG_RIP];
+    uintptr_t  *puSP    = (uintptr_t *)&pCtx->uc_mcontext.gregs[REG_RSP];
+    uintptr_t   uTrapNo = pCtx->uc_mcontext.gregs[REG_TRAPNO];
+    uintptr_t   uErr    = pCtx->uc_mcontext.gregs[REG_ERR];
+    uintptr_t   uCr2    = pCtx->uc_mcontext.gregs[REG_CR2];
+
+# elif defined(RT_ARCH_X86) && defined(RT_OS_DARWIN)
+    uintptr_t  *puPC    = (uintptr_t *)&pCtx->uc_mcontext->__ss.__eip;
+    uintptr_t  *puSP    = (uintptr_t *)&pCtx->uc_mcontext->__ss.__esp;
+    uintptr_t   uTrapNo = pCtx->uc_mcontext->__es.__trapno;
+    uintptr_t   uErr    = pCtx->uc_mcontext->__es.__err;
+    uintptr_t   uCr2    = pCtx->uc_mcontext->__es.__faultvaddr;
+
+# elif defined(RT_ARCH_X86) && defined(RT_OS_FREEBSD)
+    uintptr_t  *puPC    = (uintptr_t *)&pCtx->uc_mcontext.mc_eip;
+    uintptr_t  *puSP    = (uintptr_t *)&pCtx->uc_mcontext.mc_esp;
+    uintptr_t   uTrapNo = ~(uintptr_t)0;
+    uintptr_t   uErr    = ~(uintptr_t)0;
+    uintptr_t   uCr2    = ~(uintptr_t)0;
+
+# elif defined(RT_ARCH_X86)
+    uintptr_t  *puPC    = (uintptr_t *)&pCtx->uc_mcontext.gregs[REG_EIP];
+    uintptr_t  *puSP    = (uintptr_t *)&pCtx->uc_mcontext.gregs[REG_ESP];
+    uintptr_t   uTrapNo = pCtx->uc_mcontext.gregs[REG_TRAPNO];
+    uintptr_t   uErr    = pCtx->uc_mcontext.gregs[REG_ERR];
+#  ifdef REG_CR2 /** @todo ... */
+    uintptr_t   uCr2    = pCtx->uc_mcontext.gregs[REG_CR2];
+#  else
+    uintptr_t   uCr2    = ~(uintptr_t)0;
+#  endif
+
+# else
+    uintptr_t  *puPC    = NULL;
+    uintptr_t  *puSP    = NULL;
+    uintptr_t   uTrapNo = ~(uintptr_t)0;
+    uintptr_t   uErr    = ~(uintptr_t)0;
+    uintptr_t   uCr2    = ~(uintptr_t)0;
+# endif
+    if (uTrapNo == X86_XCPT_PF)
+        RTAssertMsg2("tstX86-1: Trap #%#04x err=%#06x at %p / %p\n", uTrapNo, uErr, *puPC, uCr2);
+    else
+        RTAssertMsg2("tstX86-1: Trap #%#04x err=%#06x at %p\n", uTrapNo, uErr, *puPC);
+
+    PCTRAPINFO pTrapInfo = findTrapInfo(*puPC, *puSP);
+    if (pTrapInfo)
+    {
+        if (pTrapInfo->u8Trap != uTrapNo && uTrapNo != ~(uintptr_t)0)
+            RTAssertMsg2("tstX86-1: Expected #%#04x, got #%#04x\n", pTrapInfo->u8Trap, uTrapNo);
+        else
+        {
+            if (*puPC != pTrapInfo->uTrapPC)
+                *puSP += sizeof(uintptr_t);
+            *puPC = pTrapInfo->uResumePC;
+            return;
+        }
+    }
+    else
+        RTAssertMsg2("tstX86-1: Unexpected trap!\n");
+
+    /* die */
+    signal(iSig, SIG_IGN);
+}
+#else
+
+#endif
+
+
+
+int main()
+{
+    /*
+     * Set up the test environment.
+     */
+    RTTEST hTest;
+    RTEXITCODE rcExit = RTTestInitAndCreate("tstX86-1", &hTest);
+    if (rcExit != RTEXITCODE_SUCCESS)
+        return rcExit;
+    RTTestBanner(hTest);
+
+    g_pbEfPage = (uint8_t *)RTTestGuardedAllocTail(hTest, PAGE_SIZE);
+    RTTESTI_CHECK(g_pbEfPage != NULL);
+
+    g_pbEfExecPage = (uint8_t *)RTMemExecAlloc(PAGE_SIZE*2);
+    RTTESTI_CHECK(g_pbEfExecPage != NULL);
+    RTTESTI_CHECK(!((uintptr_t)g_pbEfExecPage & PAGE_OFFSET_MASK));
+    RTTESTI_CHECK_RC(RTMemProtect(g_pbEfExecPage + PAGE_SIZE, PAGE_SIZE, RTMEM_PROT_NONE), VINF_SUCCESS);
+
+#ifdef USE_SIGNAL
+    static int const s_aiSigs[] = { SIGBUS, SIGSEGV, SIGFPE, SIGILL };
+    for (unsigned i = 0; i < RT_ELEMENTS(s_aiSigs); i++)
+    {
+        struct sigaction SigAct;
+        RTTESTI_CHECK_BREAK(sigaction(s_aiSigs[i], NULL, &SigAct) == 0);
+        SigAct.sa_sigaction = sigHandler;
+        SigAct.sa_flags    |= SA_SIGINFO;
+        RTTESTI_CHECK(sigaction(s_aiSigs[i], &SigAct, NULL) == 0);
+    }
+#else
+    /** @todo implement me. */
+#endif
+
+
+    if (!RTTestErrorCount(hTest))
+    {
+        /*
+         * Do the testing.
+         */
+        int32_t rc;
+#if 0
+        RTTestSub(hTest, "Misc 1");
+        rc = x861_Test1();
+        if (rc != 0)
+            RTTestFailed(hTest, "x861_Test1 -> %d", rc);
+
+        RTTestSub(hTest, "Prefixes and groups");
+        rc = x861_Test2();
+        if (rc != 0)
+            RTTestFailed(hTest, "x861_Test2 -> %d", rc);
+
+        RTTestSub(hTest, "fxsave / fxrstor and #PFs");
+        rc = x861_Test3();
+        if (rc != 0)
+            RTTestFailed(hTest, "x861_Test3 -> %d", rc);
+
+        RTTestSub(hTest, "Multibyte NOPs");
+        rc = x861_Test4();
+        if (rc != 0)
+            RTTestFailed(hTest, "x861_Test4 -> %d", rc);
+//#endif
+
+        RTTestSub(hTest, "Odd encodings and odd ends");
+        rc = x861_Test5();
+        if (rc != 0)
+            RTTestFailed(hTest, "x861_Test5 -> %d", rc);
+
+//#if 0
+        RTTestSub(hTest, "Odd floating point encodings");
+        rc = x861_Test6();
+        if (rc != 0)
+            RTTestFailed(hTest, "x861_Test5 -> %d", rc);
+
+        RTTestSub(hTest, "Floating point exceptions ++");
+        rc = x861_Test7();
+        if (rc != 0)
+            RTTestFailed(hTest, "x861_Test6 -> %d", rc);
+#endif
+
+        rc = x861_TestFPUInstr1();
+        if (rc != 0)
+            RTTestFailed(hTest, "x861_TestFPUInstr1 -> %d", rc);
+    }
+
+    return RTTestSummaryAndDestroy(hTest);
+}
+
diff --git a/src/VBox/VMM/testcase/tstX86-1A.asm b/src/VBox/VMM/testcase/tstX86-1A.asm
new file mode 100644
index 00000000..d2b74a06
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstX86-1A.asm
@@ -0,0 +1,3443 @@
+; $Id: tstX86-1A.asm $
+;; @file
+; X86 instruction set exploration/testcase #1.
+;
+
+;
+; Copyright (C) 2011-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;   Header Files                                                              ;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%include "iprt/asmdefs.mac"
+%include "iprt/x86.mac"
+
+;; @todo Move this to a header?
+struc TRAPINFO
+        .uTrapPC        RTCCPTR_RES 1
+        .uResumePC      RTCCPTR_RES 1
+        .u8TrapNo       resb 1
+        .cbInstr        resb 1
+        .au8Padding     resb (RTCCPTR_CB*2 - 2)
+endstruc
+
+
+%ifdef RT_ARCH_AMD64
+ %define arch_fxsave    o64 fxsave
+ %define arch_fxrstor   o64 fxrstor
+%else
+ %define arch_fxsave    fxsave
+ %define arch_fxrstor   fxrstor
+%endif
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;   Global Variables                                                          ;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+BEGINDATA
+extern NAME(g_pbEfPage)
+extern NAME(g_pbEfExecPage)
+
+GLOBALNAME g_szAlpha
+        db      "abcdefghijklmnopqrstuvwxyz", 0
+g_szAlpha_end:
+%define g_cchAlpha (g_szAlpha_end - NAME(g_szAlpha))
+        db      0, 0, 0,
+
+;; @name Floating point constants.
+; @{
+g_r32_0dot1:    dd 0.1
+g_r32_3dot2:    dd 3.2
+g_r32_Zero:     dd 0.0
+g_r32_One:      dd 1.0
+g_r32_Two:      dd 2.0
+g_r32_Three:    dd 3.0
+g_r32_Ten:      dd 10.0
+g_r32_Eleven:   dd 11.0
+g_r32_ThirtyTwo:dd 32.0
+g_r32_Min:      dd 000800000h
+g_r32_Max:      dd 07f7fffffh
+g_r32_Inf:      dd 07f800000h
+g_r32_SNaN:     dd 07f800001h
+g_r32_SNaNMax:  dd 07fbfffffh
+g_r32_QNaN:     dd 07fc00000h
+g_r32_QNaNMax:  dd 07fffffffh
+g_r32_NegQNaN:  dd 0ffc00000h
+
+g_r64_0dot1:    dq 0.1
+g_r64_6dot9:    dq 6.9
+g_r64_Zero:     dq 0.0
+g_r64_One:      dq 1.0
+g_r64_Two:      dq 2.0
+g_r64_Three:    dq 3.0
+g_r64_Ten:      dq 10.0
+g_r64_Eleven:   dq 11.0
+g_r64_ThirtyTwo:dq 32.0
+g_r64_Min:      dq 00010000000000000h
+g_r64_Max:      dq 07fefffffffffffffh
+g_r64_Inf:      dq 07ff0000000000000h
+g_r64_SNaN:     dq 07ff0000000000001h
+g_r64_SNaNMax:  dq 07ff7ffffffffffffh
+g_r64_NegQNaN:  dq 0fff8000000000000h
+g_r64_QNaN:     dq 07ff8000000000000h
+g_r64_QNaNMax:  dq 07fffffffffffffffh
+g_r64_DnMin:    dq 00000000000000001h
+g_r64_DnMax:    dq 0000fffffffffffffh
+
+
+g_r80_0dot1:    dt 0.1
+g_r80_3dot2:    dt 3.2
+g_r80_Zero:     dt 0.0
+g_r80_One:      dt 1.0
+g_r80_Two:      dt 2.0
+g_r80_Three:    dt 3.0
+g_r80_Ten:      dt 10.0
+g_r80_Eleven:   dt 11.0
+g_r80_ThirtyTwo:dt 32.0
+g_r80_Min:      dt 000018000000000000000h
+g_r80_Max:      dt 07ffeffffffffffffffffh
+g_r80_Inf:      dt 07fff8000000000000000h
+g_r80_QNaN:     dt 07fffc000000000000000h
+g_r80_QNaNMax:  dt 07fffffffffffffffffffh
+g_r80_NegQNaN:  dt 0ffffc000000000000000h
+g_r80_SNaN:     dt 07fff8000000000000001h
+g_r80_SNaNMax:  dt 07fffbfffffffffffffffh
+g_r80_DnMin:    dt 000000000000000000001h
+g_r80_DnMax:    dt 000007fffffffffffffffh
+
+g_r32V1:        dd 3.2
+g_r32V2:        dd -1.9
+g_r64V1:        dq 6.4
+g_r80V1:        dt 8.0
+
+; Denormal numbers.
+g_r32D0:        dd 000200000h
+;; @}
+
+;; @name Upconverted Floating point constants
+; @{
+;g_r80_r32_0dot1:        dt 0.1
+g_r80_r32_3dot2:        dt 04000cccccd0000000000h
+;g_r80_r32_Zero:         dt 0.0
+;g_r80_r32_One:          dt 1.0
+;g_r80_r32_Two:          dt 2.0
+;g_r80_r32_Three:        dt 3.0
+;g_r80_r32_Ten:          dt 10.0
+;g_r80_r32_Eleven:       dt 11.0
+;g_r80_r32_ThirtyTwo:    dt 32.0
+;; @}
+
+;; @name Decimal constants.
+; @{
+g_u64Zero:      dd 0
+g_u32Zero:      dw 0
+g_u64Two:       dd 2
+g_u32Two:       dw 2
+;; @}
+
+
+;;
+; The last global data item. We build this as we write the code.
+        align   8
+GLOBALNAME g_aTrapInfo
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;   Defined Constants And Macros                                              ;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Reference a variable
+%ifdef RT_ARCH_AMD64
+ %define REF(a_Name)     a_Name wrt rip
+%else
+ %define REF(a_Name)     a_Name
+%endif
+
+;; Reference a global variable
+%ifdef RT_ARCH_AMD64
+ %define REF_EXTERN(a_Name)     NAME(a_Name) wrt rip
+%else
+ %define REF_EXTERN(a_Name)     NAME(a_Name)
+%endif
+
+
+;;
+; Macro for checking a memory value.
+;
+; @param        1       The size (byte, word, dword, etc)
+; @param        2       The memory address expression.
+; @param        3       The valued expected at the location.
+%macro CheckMemoryValue 3
+        cmp     %1 [%2], %3
+        je      %%ok
+        mov     eax, __LINE__
+        jmp     .return
+%%ok:
+%endmacro
+
+
+;;
+; Checks if a 32-bit floating point memory value is the same as the specified
+; constant (also memory).
+;
+; @uses         eax
+; @param        1       Address expression for the 32-bit floating point value
+;                       to be checked.
+; @param        2       The address expression of the constant.
+;
+%macro CheckMemoryR32ValueConst 2
+        mov     eax, [%2]
+        cmp     dword [%1], eax
+        je      %%ok
+%%bad:
+        mov     eax, 90000000 + __LINE__
+        jmp     .return
+%%ok:
+%endmacro
+
+
+;;
+; Checks if a 80-bit floating point memory value is the same as the specified
+; constant (also memory).
+;
+; @uses         eax
+; @param        1       Address expression for the FXSAVE image.
+; @param        2       The address expression of the constant.
+;
+%macro CheckMemoryR80ValueConst 2
+        mov     eax, [%2]
+        cmp     dword [%1], eax
+        je      %%ok1
+%%bad:
+        mov     eax, 92000000 + __LINE__
+        jmp     .return
+%%ok1:
+        mov     eax, [4 + %2]
+        cmp     dword [%1 + 4], eax
+        jne     %%bad
+        mov     ax, [8 + %2]
+        cmp     word  [%1 + 8], ax
+        jne     %%bad
+%endmacro
+
+
+;;
+; Macro for recording a trapping instruction (simple).
+;
+; @param        1       The trap number.
+; @param        2+      The instruction which should trap.
+%macro ShouldTrap 2+
+%%trap:
+        %2
+%%trap_end:
+        mov     eax, __LINE__
+        jmp     .return
+BEGINDATA
+%%trapinfo: istruc TRAPINFO
+        at TRAPINFO.uTrapPC,    RTCCPTR_DEF     %%trap
+        at TRAPINFO.uResumePC,  RTCCPTR_DEF     %%resume
+        at TRAPINFO.u8TrapNo,   db              %1
+        at TRAPINFO.cbInstr,    db              (%%trap_end - %%trap)
+iend
+BEGINCODE
+%%resume:
+%endmacro
+
+;;
+; Macro for recording a trapping instruction in the exec page.
+;
+; @uses     xAX, xDX
+; @param        1       The trap number.
+; @param        2       The offset into the exec page.
+%macro ShouldTrapExecPage 2
+        lea     xDX, [REF(NAME(g_aTrapInfoExecPage))]
+        lea     xAX, [REF(%%resume)]
+        mov     byte [xDX + TRAPINFO.cbInstr],  PAGE_SIZE - (%2)
+        mov     byte [xDX + TRAPINFO.u8TrapNo], %1
+        mov     [xDX + TRAPINFO.uResumePC],     xAX
+        mov     xAX, [REF_EXTERN(g_pbEfExecPage)]
+        lea     xAX, [xAX + (%2)]
+        mov     [xDX + TRAPINFO.uTrapPC],       xAX
+        jmp     xAX
+%%resume:
+%endmacro
+
+
+;;
+; Macro for recording a FPU instruction trapping on a following fwait.
+;
+; Uses stack.
+;
+; @param        1       The status flags that are expected to be set afterwards.
+; @param        2       C0..C3 to mask out in case undefined.
+; @param        3+      The instruction which should trap.
+; @uses         eax
+;
+%macro FpuShouldTrap 3+
+        fnclex
+        %3
+%%trap:
+        fwait
+%%trap_end:
+        mov     eax, __LINE__
+        jmp     .return
+BEGINDATA
+%%trapinfo: istruc TRAPINFO
+        at TRAPINFO.uTrapPC,    RTCCPTR_DEF     %%trap
+        at TRAPINFO.uResumePC,  RTCCPTR_DEF     %%resume
+        at TRAPINFO.u8TrapNo,   db              X86_XCPT_MF
+        at TRAPINFO.cbInstr,    db              (%%trap_end - %%trap)
+iend
+BEGINCODE
+%%resume:
+        FpuCheckFSW ((%1) | X86_FSW_ES | X86_FSW_B), %2
+        fnclex
+%endmacro
+
+;;
+; Macro for recording checking the FSW value.
+;
+; Uses stack.
+;
+; @param        1       The status flags that are expected to be set afterwards.
+; @param        2       C0..C3 to mask out in case undefined.
+; @uses         eax
+;
+%macro FpuCheckFSW 2
+%%resume:
+        fnstsw  ax
+        and     eax, ~X86_FSW_TOP_MASK & ~(%2)
+        cmp     eax, (%1)
+        je      %%ok
+        ;int3
+        lea     eax, [eax + __LINE__ * 100000]
+        jmp     .return
+%%ok:
+%endmacro
+
+
+;;
+; Checks that ST0 has a certain value
+;
+; @uses tword at [xSP]
+;
+%macro CheckSt0Value 3
+        fstp    tword [xSP]
+        fld     tword [xSP]
+        cmp     dword [xSP], %1
+        je      %%ok1
+%%bad:
+        mov     eax, __LINE__
+        jmp     .return
+%%ok1:
+        cmp     dword [xSP + 4], %2
+        jne     %%bad
+        cmp     word  [xSP + 8], %3
+        jne     %%bad
+%endmacro
+
+;; Checks that ST0 contains QNaN.
+%define CheckSt0Value_QNaN              CheckSt0Value 0x00000000, 0xc0000000, 0xffff
+;; Checks that ST0 contains +Inf.
+%define CheckSt0Value_PlusInf           CheckSt0Value 0x00000000, 0x80000000, 0x7fff
+;; Checks that ST0 contains 3 & 1/3.
+%define CheckSt0Value_3_and_a_3rd       CheckSt0Value 0x55555555, 0xd5555555, 0x4000
+;; Checks that ST0 contains 3 & 1/3.
+%define CheckSt0Value_3_and_two_3rds    CheckSt0Value 0xaaaaaaab, 0xeaaaaaaa, 0x4000
+;; Checks that ST0 contains 8.0.
+%define CheckSt0Value_Eight             CheckSt0Value 0x00000000, 0x80000000, 0x4002
+
+
+;;
+; Macro for recording checking the FSW value of a FXSAVE image.
+;
+; Uses stack.
+;
+; @param        1       Address expression for the FXSAVE image.
+; @param        2       The status flags that are expected to be set afterwards.
+; @param        3       C0..C3 to mask out in case undefined.
+; @uses         eax
+; @sa   FpuCheckFSW
+;
+%macro FxSaveCheckFSW 3
+%%resume:
+        movzx   eax, word [%1 + X86FXSTATE.FSW]
+        and     eax, ~X86_FSW_TOP_MASK & ~(%3)
+        cmp     eax, (%2)
+        je      %%ok
+        mov     eax, 100000000 + __LINE__
+        jmp     .return
+%%ok:
+%endmacro
+
+
+;;
+; Checks that ST0 is empty in an FXSAVE image.
+;
+; @uses         eax
+; @param        1       Address expression for the FXSAVE image.
+;
+%macro FxSaveCheckSt0Empty 1
+        movzx   eax, word [%1 + X86FXSTATE.FSW]
+        and     eax, X86_FSW_TOP_MASK
+        shr     eax, X86_FSW_TOP_SHIFT
+        bt      [%1 + X86FXSTATE.FTW], eax
+        jnc     %%ok
+        mov     eax, 200000000 + __LINE__
+        jmp     .return
+%%ok:
+%endmacro
+
+
+;;
+; Checks that ST0 is not-empty in an FXSAVE image.
+;
+; @uses         eax
+; @param        1       Address expression for the FXSAVE image.
+;
+%macro FxSaveCheckSt0NonEmpty 1
+        movzx   eax, word [%1 + X86FXSTATE.FSW]
+        and     eax, X86_FSW_TOP_MASK
+        shr     eax, X86_FSW_TOP_SHIFT
+        bt      [%1 + X86FXSTATE.FTW], eax
+        jc      %%ok
+        mov     eax, 30000000 + __LINE__
+        jmp     .return
+%%ok:
+%endmacro
+
+;;
+; Checks that STn in a FXSAVE image has a certain value (empty or not
+; is ignored).
+;
+; @uses         eax
+; @param        1       Address expression for the FXSAVE image.
+; @param        2       The register number.
+; @param        3       First dword of value.
+; @param        4       Second dword of value.
+; @param        5       Final word of value.
+;
+%macro FxSaveCheckStNValueEx 5
+        cmp     dword [%1 + X86FXSTATE.st0 + %2 * 16], %3
+        je      %%ok1
+%%bad:
+        mov     eax, 40000000 + __LINE__
+        jmp     .return
+%%ok1:
+        cmp     dword [%1 + X86FXSTATE.st0 + %2 * 16 + 4], %4
+        jne     %%bad
+        cmp     word  [%1 + X86FXSTATE.st0 + %2 * 16 + 8], %5
+        jne     %%bad
+%endmacro
+
+
+;;
+; Checks if STn in a FXSAVE image has the same value as the specified
+; floating point (80-bit) constant.
+;
+; @uses         eax, xDX
+; @param        1       Address expression for the FXSAVE image.
+; @param        2       The register number.
+; @param        3       The address expression of the constant.
+;
+%macro FxSaveCheckStNValueConstEx 3
+        mov     eax, [%3]
+        cmp     dword [%1 + X86FXSTATE.st0 + %2 * 16], eax
+        je      %%ok1
+%%bad:
+        mov     eax, 40000000 + __LINE__
+        jmp     .return
+%%ok1:
+        mov     eax, [4 + %3]
+        cmp     dword [%1 + X86FXSTATE.st0 + %2 * 16 + 4], eax
+        jne     %%bad
+        mov     ax, [8 + %3]
+        cmp     word  [%1 + X86FXSTATE.st0 + %2 * 16 + 8], ax
+        jne     %%bad
+%endmacro
+
+
+;;
+; Checks that ST0 in a FXSAVE image has a certain value.
+;
+; @uses         eax
+; @param        1       Address expression for the FXSAVE image.
+; @param        2       First dword of value.
+; @param        3       Second dword of value.
+; @param        4       Final word of value.
+;
+%macro FxSaveCheckSt0Value 4
+        FxSaveCheckSt0NonEmpty %1
+        FxSaveCheckStNValueEx %1, 0, %2, %3, %4
+%endmacro
+
+
+;;
+; Checks that ST0 in a FXSAVE image is empty and that the value stored is the
+; init value set by FpuInitWithCW.
+;
+; @uses         eax
+; @param        1       Address expression for the FXSAVE image.
+;
+%macro FxSaveCheckSt0EmptyInitValue 1
+        FxSaveCheckSt0Empty %1
+        FxSaveCheckStNValueEx %1, 0, 0x40404040, 0x40404040, 0xffff
+%endmacro
+
+;;
+; Checks that ST0 in a FXSAVE image is non-empty and has the same value as the
+; specified constant (80-bit).
+;
+; @uses         eax, xDX
+; @param        1       Address expression for the FXSAVE image.
+; @param        2       The address expression of the constant.
+%macro FxSaveCheckSt0ValueConst 2
+        FxSaveCheckSt0NonEmpty %1
+        FxSaveCheckStNValueConstEx %1, 0, %2
+%endmacro
+
+;; Checks that ST0 contains QNaN.
+%define FxSaveCheckSt0Value_QNaN(p)              FxSaveCheckSt0Value p, 0x00000000, 0xc0000000, 0xffff
+;; Checks that ST0 contains +Inf.
+%define FxSaveCheckSt0Value_PlusInf(p)           FxSaveCheckSt0Value p, 0x00000000, 0x80000000, 0x7fff
+;; Checks that ST0 contains 3 & 1/3.
+%define FxSaveCheckSt0Value_3_and_a_3rd(p)       FxSaveCheckSt0Value p, 0x55555555, 0xd5555555, 0x4000
+;; Checks that ST0 contains 3 & 1/3.
+%define FxSaveCheckSt0Value_3_and_two_3rds(p)    FxSaveCheckSt0Value p, 0xaaaaaaab, 0xeaaaaaaa, 0x4000
+
+
+
+;;
+; Checks that STn is empty in an FXSAVE image.
+;
+; @uses         eax
+; @param        1       Address expression for the FXSAVE image.
+; @param        2       The register number.
+;
+%macro FxSaveCheckStNEmpty 2
+        movzx   eax, word [%1 + X86FXSTATE.FSW]
+        and     eax, X86_FSW_TOP_MASK
+        shr     eax, X86_FSW_TOP_SHIFT
+        add     eax, %2
+        and     eax, X86_FSW_TOP_SMASK
+        bt      [%1 + X86FXSTATE.FTW], eax
+        jnc     %%ok
+        mov     eax, 20000000 + __LINE__
+        jmp     .return
+%%ok:
+%endmacro
+
+
+;;
+; Checks that STn is not-empty in an FXSAVE image.
+;
+; @uses         eax
+; @param        1       Address expression for the FXSAVE image.
+; @param        2       The register number.
+;
+%macro FxSaveCheckStNNonEmpty 2
+        movzx   eax, word [%1 + X86FXSTATE.FSW]
+        and     eax, X86_FSW_TOP_MASK
+        shr     eax, X86_FSW_TOP_SHIFT
+        add     eax, %2
+        and     eax, X86_FSW_TOP_SMASK
+        bt      [%1 + X86FXSTATE.FTW], eax
+        jc      %%ok
+        mov     eax, 30000000 + __LINE__
+        jmp     .return
+%%ok:
+%endmacro
+
+
+;;
+; Checks that STn in a FXSAVE image has a certain value.
+;
+; @uses         eax
+; @param        1       Address expression for the FXSAVE image.
+; @param        2       The register number.
+; @param        3       First dword of value.
+; @param        4       Second dword of value.
+; @param        5       Final word of value.
+;
+%macro FxSaveCheckStNValue 5
+        FxSaveCheckStNNonEmpty %1, %2
+        FxSaveCheckStNValueEx %1, %2, %3, %4, %5
+%endmacro
+
+;;
+; Checks that ST0 in a FXSAVE image is non-empty and has the same value as the
+; specified constant (80-bit).
+;
+; @uses         eax, xDX
+; @param        1       Address expression for the FXSAVE image.
+; @param        2       The register number.
+; @param        3       The address expression of the constant.
+%macro FxSaveCheckStNValueConst 3
+        FxSaveCheckStNNonEmpty %1, %2
+        FxSaveCheckStNValueConstEx %1, %2, %3
+%endmacro
+
+;; Checks that ST0 contains QNaN.
+%define FxSaveCheckStNValue_QNaN(p, iSt)            FxSaveCheckStNValue p, iSt, 0x00000000, 0xc0000000, 0xffff
+;; Checks that ST0 contains +Inf.
+%define FxSaveCheckStNValue_PlusInf(p, iSt)         FxSaveCheckStNValue p, iSt, 0x00000000, 0x80000000, 0x7fff
+;; Checks that ST0 contains 3 & 1/3.
+%define FxSaveCheckStNValue_3_and_a_3rd(p, iSt)     FxSaveCheckStNValue p, iSt, 0x55555555, 0xd5555555, 0x4000
+;; Checks that ST0 contains 3 & 1/3.
+%define FxSaveCheckStNValue_3_and_two_3rds(p, iSt)  FxSaveCheckStNValue p, iSt, 0xaaaaaaab, 0xeaaaaaaa, 0x4000
+
+
+;;
+; Function prologue saving all registers except EAX and aligns the stack
+; on a 16-byte boundrary.
+;
+%macro SAVE_ALL_PROLOGUE 0
+        push    xBP
+        mov     xBP, xSP
+        pushf
+        push    xBX
+        push    xCX
+        push    xDX
+        push    xSI
+        push    xDI
+%ifdef RT_ARCH_AMD64
+        push    r8
+        push    r9
+        push    r10
+        push    r11
+        push    r12
+        push    r13
+        push    r14
+        push    r15
+%endif
+        and     xSP, ~0fh;
+%endmacro
+
+
+;;
+; Function epilogue restoring all regisers except EAX.
+;
+%macro SAVE_ALL_EPILOGUE 0
+%ifdef RT_ARCH_AMD64
+        lea     rsp, [rbp - 14 * 8]
+        pop     r15
+        pop     r14
+        pop     r13
+        pop     r12
+        pop     r11
+        pop     r10
+        pop     r9
+        pop     r8
+%else
+        lea     esp, [ebp - 6 * 4]
+%endif
+        pop     xDI
+        pop     xSI
+        pop     xDX
+        pop     xCX
+        pop     xBX
+        popf
+        leave
+%endmacro
+
+
+
+
+BEGINCODE
+
+;;
+; Loads all general registers except xBP and xSP with unique values.
+;
+x861_LoadUniqueRegValues:
+%ifdef RT_ARCH_AMD64
+        mov     rax, 00000000000000000h
+        mov     rcx, 01111111111111111h
+        mov     rdx, 02222222222222222h
+        mov     rbx, 03333333333333333h
+        mov     rsi, 06666666666666666h
+        mov     rdi, 07777777777777777h
+        mov     r8,  08888888888888888h
+        mov     r9,  09999999999999999h
+        mov     r10, 0aaaaaaaaaaaaaaaah
+        mov     r11, 0bbbbbbbbbbbbbbbbh
+        mov     r12, 0cccccccccccccccch
+        mov     r13, 0ddddddddddddddddh
+        mov     r14, 0eeeeeeeeeeeeeeeeh
+        mov     r15, 0ffffffffffffffffh
+%else
+        mov     eax, 000000000h
+        mov     ecx, 011111111h
+        mov     edx, 022222222h
+        mov     ebx, 033333333h
+        mov     esi, 066666666h
+        mov     edi, 077777777h
+%endif
+        ret
+; end x861_LoadUniqueRegValues
+
+
+;;
+; Clears all general registers except xBP and xSP.
+;
+x861_ClearRegisters:
+        xor     eax, eax
+        xor     ebx, ebx
+        xor     ecx, ecx
+        xor     edx, edx
+        xor     esi, esi
+        xor     edi, edi
+%ifdef RT_ARCH_AMD64
+        xor     r8,  r8
+        xor     r9,  r9
+        xor     r10, r10
+        xor     r11, r11
+        xor     r12, r12
+        xor     r13, r13
+        xor     r14, r14
+        xor     r15, r15
+%endif
+        ret
+; x861_ClearRegisters
+
+
+;;
+; Loads all MMX and SSE registers except xBP and xSP with unique values.
+;
+x861_LoadUniqueRegValuesSSE:
+        fninit
+        movq    mm0, [REF(._mm0)]
+        movq    mm1, [REF(._mm1)]
+        movq    mm2, [REF(._mm2)]
+        movq    mm3, [REF(._mm3)]
+        movq    mm4, [REF(._mm4)]
+        movq    mm5, [REF(._mm5)]
+        movq    mm6, [REF(._mm6)]
+        movq    mm7, [REF(._mm7)]
+        movdqu  xmm0, [REF(._xmm0)]
+        movdqu  xmm1, [REF(._xmm1)]
+        movdqu  xmm2, [REF(._xmm2)]
+        movdqu  xmm3, [REF(._xmm3)]
+        movdqu  xmm4, [REF(._xmm4)]
+        movdqu  xmm5, [REF(._xmm5)]
+        movdqu  xmm6, [REF(._xmm6)]
+        movdqu  xmm7, [REF(._xmm7)]
+%ifdef RT_ARCH_AMD64
+        movdqu  xmm8,  [REF(._xmm8)]
+        movdqu  xmm9,  [REF(._xmm9)]
+        movdqu  xmm10, [REF(._xmm10)]
+        movdqu  xmm11, [REF(._xmm11)]
+        movdqu  xmm12, [REF(._xmm12)]
+        movdqu  xmm13, [REF(._xmm13)]
+        movdqu  xmm14, [REF(._xmm14)]
+        movdqu  xmm15, [REF(._xmm15)]
+%endif
+        ret
+._mm0:   times 8  db 040h
+._mm1:   times 8  db 041h
+._mm2:   times 8  db 042h
+._mm3:   times 8  db 043h
+._mm4:   times 8  db 044h
+._mm5:   times 8  db 045h
+._mm6:   times 8  db 046h
+._mm7:   times 8  db 047h
+._xmm0:  times 16 db 080h
+._xmm1:  times 16 db 081h
+._xmm2:  times 16 db 082h
+._xmm3:  times 16 db 083h
+._xmm4:  times 16 db 084h
+._xmm5:  times 16 db 085h
+._xmm6:  times 16 db 086h
+._xmm7:  times 16 db 087h
+%ifdef RT_ARCH_AMD64
+._xmm8:  times 16 db 088h
+._xmm9:  times 16 db 089h
+._xmm10: times 16 db 08ah
+._xmm11: times 16 db 08bh
+._xmm12: times 16 db 08ch
+._xmm13: times 16 db 08dh
+._xmm14: times 16 db 08eh
+._xmm15: times 16 db 08fh
+%endif
+; end x861_LoadUniqueRegValuesSSE
+
+
+;;
+; Clears all MMX and SSE registers.
+;
+x861_ClearRegistersSSE:
+        fninit
+        movq    mm0,   [REF(.zero)]
+        movq    mm1,   [REF(.zero)]
+        movq    mm2,   [REF(.zero)]
+        movq    mm3,   [REF(.zero)]
+        movq    mm4,   [REF(.zero)]
+        movq    mm5,   [REF(.zero)]
+        movq    mm6,   [REF(.zero)]
+        movq    mm7,   [REF(.zero)]
+        movdqu  xmm0,  [REF(.zero)]
+        movdqu  xmm1,  [REF(.zero)]
+        movdqu  xmm2,  [REF(.zero)]
+        movdqu  xmm3,  [REF(.zero)]
+        movdqu  xmm4,  [REF(.zero)]
+        movdqu  xmm5,  [REF(.zero)]
+        movdqu  xmm6,  [REF(.zero)]
+        movdqu  xmm7,  [REF(.zero)]
+%ifdef RT_ARCH_AMD64
+        movdqu  xmm8,  [REF(.zero)]
+        movdqu  xmm9,  [REF(.zero)]
+        movdqu  xmm10, [REF(.zero)]
+        movdqu  xmm11, [REF(.zero)]
+        movdqu  xmm12, [REF(.zero)]
+        movdqu  xmm13, [REF(.zero)]
+        movdqu  xmm14, [REF(.zero)]
+        movdqu  xmm15, [REF(.zero)]
+%endif
+        ret
+
+        ret
+.zero   times 16 db 000h
+; x861_ClearRegistersSSE
+
+
+;;
+; Loads all general, MMX and SSE registers except xBP and xSP with unique values.
+;
+x861_LoadUniqueRegValuesSSEAndGRegs:
+        call    x861_LoadUniqueRegValuesSSE
+        call    x861_LoadUniqueRegValues
+        ret
+
+;;
+; Clears all general, MMX and SSE registers except xBP and xSP.
+;
+x861_ClearRegistersSSEAndGRegs:
+        call    x861_ClearRegistersSSE
+        call    x861_ClearRegisters
+        ret
+
+BEGINPROC x861_Test1
+        push    xBP
+        mov     xBP, xSP
+        pushf
+        push    xBX
+        push    xCX
+        push    xDX
+        push    xSI
+        push    xDI
+%ifdef RT_ARCH_AMD64
+        push    r8
+        push    r9
+        push    r10
+        push    r11
+        push    r12
+        push    r13
+        push    r14
+        push    r15
+%endif
+
+        ;
+        ; Odd push behavior
+        ;
+%if 0 ; Seems to be so on AMD only
+%ifdef RT_ARCH_X86
+        ; upper word of a 'push cs' is cleared.
+        mov     eax, __LINE__
+        mov     dword [esp - 4], 0f0f0f0fh
+        push    cs
+        pop     ecx
+        mov     bx, cs
+        and     ebx, 0000ffffh
+        cmp     ecx, ebx
+        jne     .failed
+
+        ; upper word of a 'push ds' is cleared.
+        mov     eax, __LINE__
+        mov     dword [esp - 4], 0f0f0f0fh
+        push    ds
+        pop     ecx
+        mov     bx, ds
+        and     ebx, 0000ffffh
+        cmp     ecx, ebx
+        jne     .failed
+
+        ; upper word of a 'push es' is cleared.
+        mov     eax, __LINE__
+        mov     dword [esp - 4], 0f0f0f0fh
+        push    es
+        pop     ecx
+        mov     bx, es
+        and     ebx, 0000ffffh
+        cmp     ecx, ebx
+        jne     .failed
+%endif ; RT_ARCH_X86
+
+        ; The upper part of a 'push fs' is cleared.
+        mov     eax, __LINE__
+        xor     ecx, ecx
+        not     xCX
+        push    xCX
+        pop     xCX
+        push    fs
+        pop     xCX
+        mov     bx, fs
+        and     ebx, 0000ffffh
+        cmp     xCX, xBX
+        jne     .failed
+
+        ; The upper part of a 'push gs' is cleared.
+        mov     eax, __LINE__
+        xor     ecx, ecx
+        not     xCX
+        push    xCX
+        pop     xCX
+        push    gs
+        pop     xCX
+        mov     bx, gs
+        and     ebx, 0000ffffh
+        cmp     xCX, xBX
+        jne     .failed
+%endif
+
+%ifdef RT_ARCH_AMD64
+        ; REX.B works with 'push r64'.
+        call    x861_LoadUniqueRegValues
+        mov     eax, __LINE__
+        push    rcx
+        pop     rdx
+        cmp     rdx, rcx
+        jne     .failed
+
+        call    x861_LoadUniqueRegValues
+        mov     eax, __LINE__
+        db 041h                         ; REX.B
+        push    rcx
+        pop     rdx
+        cmp     rdx, r9
+        jne     .failed
+
+        call    x861_LoadUniqueRegValues
+        mov     eax, __LINE__
+        db 042h                         ; REX.X
+        push    rcx
+        pop     rdx
+        cmp     rdx, rcx
+        jne     .failed
+
+        call    x861_LoadUniqueRegValues
+        mov     eax, __LINE__
+        db 044h                         ; REX.R
+        push    rcx
+        pop     rdx
+        cmp     rdx, rcx
+        jne     .failed
+
+        call    x861_LoadUniqueRegValues
+        mov     eax, __LINE__
+        db 048h                         ; REX.W
+        push    rcx
+        pop     rdx
+        cmp     rdx, rcx
+        jne     .failed
+
+        call    x861_LoadUniqueRegValues
+        mov     eax, __LINE__
+        db 04fh                         ; REX.*
+        push    rcx
+        pop     rdx
+        cmp     rdx, r9
+        jne     .failed
+%endif
+
+        ;
+        ; Zero extening when moving from a segreg as well as memory access sizes.
+        ;
+        call    x861_LoadUniqueRegValues
+        mov     eax, __LINE__
+        mov     ecx, ds
+        shr     xCX, 16
+        cmp     xCX, 0
+        jnz     .failed
+
+%ifdef RT_ARCH_AMD64
+        call    x861_LoadUniqueRegValues
+        mov     eax, __LINE__
+        mov     rcx, ds
+        shr     rcx, 16
+        cmp     rcx, 0
+        jnz     .failed
+%endif
+
+        call    x861_LoadUniqueRegValues
+        mov     eax, __LINE__
+        mov     xDX, xCX
+        mov     cx, ds
+        shr     xCX, 16
+        shr     xDX, 16
+        cmp     xCX, xDX
+        jnz     .failed
+
+        ; Loading is always a word access.
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfPage)]
+        lea     xDI, [xDI + 0x1000 - 2]
+        mov     xDX, es
+        mov     [xDI], dx
+        mov     es, [xDI]               ; should not crash
+
+        ; Saving is always a word access.
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfPage)]
+        mov     dword [xDI + 0x1000 - 4], -1
+        mov     [xDI + 0x1000 - 2], ss ; Should not crash.
+        mov     bx, ss
+        mov     cx, [xDI + 0x1000 - 2]
+        cmp     cx, bx
+        jne     .failed
+
+%ifdef RT_ARCH_AMD64
+        ; Check that the rex.R and rex.W bits don't have any influence over a memory write.
+        call    x861_ClearRegisters
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfPage)]
+        mov     dword [xDI + 0x1000 - 4], -1
+        db 04ah
+        mov     [xDI + 0x1000 - 2], ss ; Should not crash.
+        mov     bx, ss
+        mov     cx, [xDI + 0x1000 - 2]
+        cmp     cx, bx
+        jne     .failed
+%endif
+
+
+        ;
+        ; Check what happens when both string prefixes are used.
+        ;
+        cld
+        mov     dx, ds
+        mov     es, dx
+
+        ; check that repne scasb (al=0) behaves like expected.
+        lea     xDI, [REF(NAME(g_szAlpha))]
+        xor     eax, eax                ; find the end
+        mov     ecx, g_cchAlpha + 1
+        repne scasb
+        cmp     ecx, 1
+        mov     eax, __LINE__
+        jne     .failed
+
+        ; check that repe scasb (al=0) behaves like expected.
+        lea     xDI, [REF(NAME(g_szAlpha))]
+        xor     eax, eax                ; find the end
+        mov     ecx, g_cchAlpha + 1
+        repe scasb
+        cmp     ecx, g_cchAlpha
+        mov     eax, __LINE__
+        jne     .failed
+
+        ; repne is last, it wins.
+        lea     xDI, [REF(NAME(g_szAlpha))]
+        xor     eax, eax                ; find the end
+        mov     ecx, g_cchAlpha + 1
+        db 0f3h                         ; repe  - ignored
+        db 0f2h                         ; repne
+        scasb
+        cmp     ecx, 1
+        mov     eax, __LINE__
+        jne     .failed
+
+        ; repe is last, it wins.
+        lea     xDI, [REF(NAME(g_szAlpha))]
+        xor     eax, eax                ; find the end
+        mov     ecx, g_cchAlpha + 1
+        db 0f2h                         ; repne - ignored
+        db 0f3h                         ; repe
+        scasb
+        cmp     ecx, g_cchAlpha
+        mov     eax, __LINE__
+        jne     .failed
+
+        ;
+        ; Check if stosb works with both prefixes.
+        ;
+        cld
+        mov     dx, ds
+        mov     es, dx
+        mov     xDI, [REF_EXTERN(g_pbEfPage)]
+        xor     eax, eax
+        mov     ecx, 01000h
+        rep stosb
+
+        mov     xDI, [REF_EXTERN(g_pbEfPage)]
+        mov     ecx, 4
+        mov     eax, 0ffh
+        db 0f2h                         ; repne
+        stosb
+        mov     eax, __LINE__
+        cmp     ecx, 0
+        jne     .failed
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfPage)]
+        cmp     dword [xDI], 0ffffffffh
+        jne     .failed
+        cmp     dword [xDI+4], 0
+        jne     .failed
+
+        mov     xDI, [REF_EXTERN(g_pbEfPage)]
+        mov     ecx, 4
+        mov     eax, 0feh
+        db 0f3h                         ; repe
+        stosb
+        mov     eax, __LINE__
+        cmp     ecx, 0
+        jne     .failed
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfPage)]
+        cmp     dword [xDI], 0fefefefeh
+        jne     .failed
+        cmp     dword [xDI+4], 0
+        jne     .failed
+
+        ;
+        ; String operations shouldn't crash because of an invalid address if rCX is 0.
+        ;
+        mov     eax, __LINE__
+        cld
+        mov     dx, ds
+        mov     es, dx
+        mov     xDI, [REF_EXTERN(g_pbEfPage)]
+        xor     xCX, xCX
+        rep stosb                       ; no trap
+
+        ;
+        ; INS/OUTS will trap in ring-3 even when rCX is 0. (ASSUMES IOPL < 3)
+        ;
+        mov     eax, __LINE__
+        cld
+        mov     dx, ss
+        mov     ss, dx
+        mov     xDI, xSP
+        xor     xCX, xCX
+        ShouldTrap X86_XCPT_GP, rep insb
+
+        ;
+        ; SMSW can get to the whole of CR0.
+        ;
+        mov     eax, __LINE__
+        xor     xBX, xBX
+        smsw    xBX
+        test    ebx, X86_CR0_PG
+        jz      .failed
+        test    ebx, X86_CR0_PE
+        jz      .failed
+
+        ;
+        ; Will the CPU decode the whole r/m+sib stuff before signalling a lock
+        ; prefix error?  Use the EF exec page and a LOCK ADD CL,[rDI + disp32]
+        ; instruction at the very end of it.
+        ;
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        add     xDI, 1000h - 8h
+        mov     byte [xDI+0], 0f0h
+        mov     byte [xDI+1], 002h
+        mov     byte [xDI+2], 08fh
+        mov     dword [xDI+3], 000000000h
+        mov     byte [xDI+7], 0cch
+        ShouldTrap X86_XCPT_UD, call xDI
+
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        add     xDI, 1000h - 7h
+        mov     byte [xDI+0], 0f0h
+        mov     byte [xDI+1], 002h
+        mov     byte [xDI+2], 08Fh
+        mov     dword [xDI+3], 000000000h
+        ShouldTrap X86_XCPT_UD, call xDI
+
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        add     xDI, 1000h - 4h
+        mov     byte [xDI+0], 0f0h
+        mov     byte [xDI+1], 002h
+        mov     byte [xDI+2], 08Fh
+        mov     byte [xDI+3], 000h
+        ShouldTrap X86_XCPT_PF, call xDI
+
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        add     xDI, 1000h - 6h
+        mov     byte [xDI+0], 0f0h
+        mov     byte [xDI+1], 002h
+        mov     byte [xDI+2], 08Fh
+        mov     byte [xDI+3], 00h
+        mov     byte [xDI+4], 00h
+        mov     byte [xDI+5], 00h
+        ShouldTrap X86_XCPT_PF, call xDI
+
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        add     xDI, 1000h - 5h
+        mov     byte [xDI+0], 0f0h
+        mov     byte [xDI+1], 002h
+        mov     byte [xDI+2], 08Fh
+        mov     byte [xDI+3], 00h
+        mov     byte [xDI+4], 00h
+        ShouldTrap X86_XCPT_PF, call xDI
+
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        add     xDI, 1000h - 4h
+        mov     byte [xDI+0], 0f0h
+        mov     byte [xDI+1], 002h
+        mov     byte [xDI+2], 08Fh
+        mov     byte [xDI+3], 00h
+        ShouldTrap X86_XCPT_PF, call xDI
+
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        add     xDI, 1000h - 3h
+        mov     byte [xDI+0], 0f0h
+        mov     byte [xDI+1], 002h
+        mov     byte [xDI+2], 08Fh
+        ShouldTrap X86_XCPT_PF, call xDI
+
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        add     xDI, 1000h - 2h
+        mov     byte [xDI+0], 0f0h
+        mov     byte [xDI+1], 002h
+        ShouldTrap X86_XCPT_PF, call xDI
+
+        mov     eax, __LINE__
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        add     xDI, 1000h - 1h
+        mov     byte [xDI+0], 0f0h
+        ShouldTrap X86_XCPT_PF, call xDI
+
+
+
+.success:
+        xor     eax, eax
+.return:
+%ifdef RT_ARCH_AMD64
+        pop     r15
+        pop     r14
+        pop     r13
+        pop     r12
+        pop     r11
+        pop     r10
+        pop     r9
+        pop     r8
+%endif
+        pop     xDI
+        pop     xSI
+        pop     xDX
+        pop     xCX
+        pop     xBX
+        popf
+        leave
+        ret
+
+.failed2:
+        mov     eax, -1
+.failed:
+        jmp     .return
+ENDPROC   x861_Test1
+
+
+
+;;
+; Tests the effect of prefix order in group 14.
+;
+BEGINPROC   x861_Test2
+        SAVE_ALL_PROLOGUE
+
+        ; Check testcase preconditions.
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db       00Fh, 073h, 0D0h, 080h  ;    psrlq   mm0, 128
+        call    .check_mm0_zero_and_xmm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 00Fh, 073h, 0D0h, 080h  ;    psrlq   xmm0, 128
+        call    .check_xmm0_zero_and_mm0_nz
+
+
+        ;
+        ; Real test - Inject other prefixes before the 066h and see what
+        ;             happens.
+        ;
+
+        ; General checks that order does not matter, etc.
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 026h, 066h, 00Fh, 073h, 0D0h, 080h
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 026h, 00Fh, 073h, 0D0h, 080h
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 067h, 00Fh, 073h, 0D0h, 080h
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 067h, 066h, 00Fh, 073h, 0D0h, 080h
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 067h, 066h, 065h, 00Fh, 073h, 0D0h, 080h
+        call    .check_xmm0_zero_and_mm0_nz
+
+%ifdef RT_ARCH_AMD64
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 048h, 066h, 00Fh, 073h, 0D0h, 080h ; REX.W
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 044h, 066h, 00Fh, 073h, 0D0h, 080h ; REX.R
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 042h, 066h, 00Fh, 073h, 0D0h, 080h ; REX.X
+        call    .check_xmm0_zero_and_mm0_nz
+
+        ; Actually for REX, order does matter if the prefix is used.
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 041h, 066h, 00Fh, 073h, 0D0h, 080h ; REX.B
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 041h, 00Fh, 073h, 0D0h, 080h ; REX.B
+        call    .check_xmm8_zero_and_xmm0_nz
+%endif
+
+        ; Check all ignored prefixes (repeates some of the above).
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 026h, 00Fh, 073h, 0D0h, 080h ; es
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 065h, 00Fh, 073h, 0D0h, 080h ; gs
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 064h, 00Fh, 073h, 0D0h, 080h ; fs
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 02eh, 00Fh, 073h, 0D0h, 080h ; cs
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 036h, 00Fh, 073h, 0D0h, 080h ; ss
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 03eh, 00Fh, 073h, 0D0h, 080h ; ds
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 067h, 00Fh, 073h, 0D0h, 080h ; addr size
+        call    .check_xmm0_zero_and_mm0_nz
+
+%ifdef RT_ARCH_AMD64
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 048h, 00Fh, 073h, 0D0h, 080h ; REX.W
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 044h, 00Fh, 073h, 0D0h, 080h ; REX.R
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 042h, 00Fh, 073h, 0D0h, 080h ; REX.X
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 041h, 00Fh, 073h, 0D0h, 080h ; REX.B - has actual effect on the instruction.
+        call    .check_xmm8_zero_and_xmm0_nz
+%endif
+
+        ; Repeated prefix until we hit the max opcode limit.
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 066h, 00Fh, 073h, 0D0h, 080h
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 066h, 066h, 00Fh, 073h, 0D0h, 080h
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 066h, 066h, 066h, 066h, 066h, 066h, 066h, 00Fh, 073h, 0D0h, 080h
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 066h, 066h, 066h, 066h, 066h, 066h, 066h, 066h, 066h, 066h, 00Fh, 073h, 0D0h, 080h
+        call    .check_xmm0_zero_and_mm0_nz
+
+        ShouldTrap X86_XCPT_GP, db 066h, 066h, 066h, 066h, 066h, 066h, 066h, 066h, 066h, 066h, 066h, 066h, 00Fh, 073h, 0D0h, 080h
+
+%ifdef RT_ARCH_AMD64
+        ; Repeated REX is parsed, but only the last byte matters.
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 041h, 048h, 00Fh, 073h, 0D0h, 080h ; REX.B, REX.W
+        call    .check_xmm0_zero_and_mm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 048h, 041h, 00Fh, 073h, 0D0h, 080h ; REX.B, REX.W
+        call    .check_xmm8_zero_and_xmm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 048h, 044h, 042h, 048h, 044h, 042h, 048h, 044h, 042h, 041h, 00Fh, 073h, 0D0h, 080h
+        call    .check_xmm8_zero_and_xmm0_nz
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     eax, __LINE__
+        db 066h, 041h, 041h, 041h, 041h, 041h, 041h, 041h, 041h, 041h, 04eh, 00Fh, 073h, 0D0h, 080h
+        call    .check_xmm0_zero_and_mm0_nz
+%endif
+
+        ; Undefined sequences with prefixes that counts.
+        ShouldTrap X86_XCPT_UD, db 0f0h, 066h, 00Fh, 073h, 0D0h, 080h ; LOCK
+        ShouldTrap X86_XCPT_UD, db 0f2h, 066h, 00Fh, 073h, 0D0h, 080h ; REPNZ
+        ShouldTrap X86_XCPT_UD, db 0f3h, 066h, 00Fh, 073h, 0D0h, 080h ; REPZ
+        ShouldTrap X86_XCPT_UD, db 066h, 0f2h, 00Fh, 073h, 0D0h, 080h
+        ShouldTrap X86_XCPT_UD, db 066h, 0f3h, 00Fh, 073h, 0D0h, 080h
+        ShouldTrap X86_XCPT_UD, db 066h, 0f3h, 0f2h, 00Fh, 073h, 0D0h, 080h
+        ShouldTrap X86_XCPT_UD, db 066h, 0f2h, 0f3h, 00Fh, 073h, 0D0h, 080h
+        ShouldTrap X86_XCPT_UD, db 0f2h, 066h, 0f3h, 00Fh, 073h, 0D0h, 080h
+        ShouldTrap X86_XCPT_UD, db 0f3h, 066h, 0f2h, 00Fh, 073h, 0D0h, 080h
+        ShouldTrap X86_XCPT_UD, db 0f3h, 0f2h, 066h, 00Fh, 073h, 0D0h, 080h
+        ShouldTrap X86_XCPT_UD, db 0f2h, 0f3h, 066h, 00Fh, 073h, 0D0h, 080h
+        ShouldTrap X86_XCPT_UD, db 0f0h, 0f2h, 066h, 0f3h, 00Fh, 073h, 0D0h, 080h
+        ShouldTrap X86_XCPT_UD, db 0f0h, 0f3h, 066h, 0f2h, 00Fh, 073h, 0D0h, 080h
+        ShouldTrap X86_XCPT_UD, db 0f0h, 0f3h, 0f2h, 066h, 00Fh, 073h, 0D0h, 080h
+        ShouldTrap X86_XCPT_UD, db 0f0h, 0f2h, 0f3h, 066h, 00Fh, 073h, 0D0h, 080h
+
+.success:
+        xor     eax, eax
+.return:
+        SAVE_ALL_EPILOGUE
+        ret
+
+.check_xmm0_zero_and_mm0_nz:
+        sub     xSP, 20h
+        movdqu  [xSP], xmm0
+        cmp     dword [xSP], 0
+        jne     .failed3
+        cmp     dword [xSP + 4], 0
+        jne     .failed3
+        cmp     dword [xSP + 8], 0
+        jne     .failed3
+        cmp     dword [xSP + 12], 0
+        jne     .failed3
+        movq    [xSP], mm0
+        cmp     dword [xSP], 0
+        je      .failed3
+        cmp     dword [xSP + 4], 0
+        je      .failed3
+        add     xSP, 20h
+        ret
+
+.check_mm0_zero_and_xmm0_nz:
+        sub     xSP, 20h
+        movq    [xSP], mm0
+        cmp     dword [xSP], 0
+        jne     .failed3
+        cmp     dword [xSP + 4], 0
+        jne     .failed3
+        movdqu  [xSP], xmm0
+        cmp     dword [xSP], 0
+        je      .failed3
+        cmp     dword [xSP + 4], 0
+        je      .failed3
+        cmp     dword [xSP + 8], 0
+        je      .failed3
+        cmp     dword [xSP + 12], 0
+        je      .failed3
+        add     xSP, 20h
+        ret
+
+%ifdef RT_ARCH_AMD64
+.check_xmm8_zero_and_xmm0_nz:
+        sub     xSP, 20h
+        movdqu  [xSP], xmm8
+        cmp     dword [xSP], 0
+        jne     .failed3
+        cmp     dword [xSP + 4], 0
+        jne     .failed3
+        cmp     dword [xSP + 8], 0
+        jne     .failed3
+        cmp     dword [xSP + 12], 0
+        jne     .failed3
+        movdqu  [xSP], xmm0
+        cmp     dword [xSP], 0
+        je      .failed3
+        cmp     dword [xSP + 4], 0
+        je      .failed3
+        cmp     dword [xSP + 8], 0
+        je      .failed3
+        cmp     dword [xSP + 12], 0
+        je      .failed3
+        add     xSP, 20h
+        ret
+%endif
+
+.failed3:
+        add     xSP, 20h + xCB
+        jmp     .return
+
+
+ENDPROC     x861_Test2
+
+
+;;
+; Tests how much fxsave and fxrstor actually accesses of their 512 memory
+; operand.
+;
+BEGINPROC   x861_Test3
+        SAVE_ALL_PROLOGUE
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+
+        ; Check testcase preconditions.
+        fxsave   [xDI]
+        fxrstor  [xDI]
+
+        add     xDI, PAGE_SIZE - 512
+        mov     xSI, xDI
+        fxsave  [xDI]
+        fxrstor [xDI]
+
+        ; 464:511 are available to software use.  Check that they are left
+        ; untouched by fxsave.
+        mov     eax, 0aabbccddh
+        mov     ecx, 512 / 4
+        cld
+        rep stosd
+        mov     xDI, xSI
+        fxsave  [xDI]
+
+        mov     ebx, 512
+.chech_software_area_loop:
+        cmp     [xDI + xBX - 4], eax
+        jne     .chech_software_area_done
+        sub     ebx, 4
+        jmp     .chech_software_area_loop
+.chech_software_area_done:
+        cmp     ebx, 464
+        mov     eax, __LINE__
+        ja      .return
+
+        ; Check that a save + restore + save cycle yield the same results.
+        mov     xBX, [REF_EXTERN(g_pbEfExecPage)]
+        mov     xDI, xBX
+        mov     eax, 066778899h
+        mov     ecx, 512 * 2 / 4
+        cld
+        rep stosd
+        fxsave  [xBX]
+
+        call    x861_ClearRegistersSSEAndGRegs
+        mov     xBX, [REF_EXTERN(g_pbEfExecPage)]
+        fxrstor [xBX]
+
+        fxsave  [xBX + 512]
+        mov     xSI, xBX
+        lea     xDI, [xBX + 512]
+        mov     ecx, 512
+        cld
+        repe cmpsb
+        mov     eax, __LINE__
+        jnz     .return
+
+
+        ; 464:511 are available to software use.  Let see how carefully access
+        ; to the full 512 bytes are checked...
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        add     xDI, PAGE_SIZE - 512
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 16]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 32]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 48]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 64]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 80]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 96]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 128]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 144]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 160]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 176]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 192]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 208]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 224]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 240]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 256]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 384]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 432]
+        ShouldTrap X86_XCPT_PF, fxsave  [xDI + 496]
+
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 16]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 32]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 48]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 64]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 80]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 96]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 128]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 144]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 160]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 176]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 192]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 208]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 224]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 240]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 256]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 384]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 432]
+        ShouldTrap X86_XCPT_PF, fxrstor [xDI + 496]
+
+        ; Unaligned accesses will cause #GP(0). This takes precedence over #PF.
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 1]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 2]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 3]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 4]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 5]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 6]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 7]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 8]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 9]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 10]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 11]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 12]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 13]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 14]
+        ShouldTrap X86_XCPT_GP, fxsave  [xDI + 15]
+
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 1]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 2]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 3]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 4]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 5]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 6]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 7]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 8]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 9]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 10]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 11]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 12]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 13]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 14]
+        ShouldTrap X86_XCPT_GP, fxrstor [xDI + 15]
+
+        ; Lets check what a FP in fxsave changes ... nothing on intel.
+        mov     ebx, 16
+.fxsave_pf_effect_loop:
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        add     xDI, PAGE_SIZE - 512 * 2
+        mov     xSI, xDI
+        mov     eax, 066778899h
+        mov     ecx, 512 * 2 / 4
+        cld
+        rep stosd
+
+        ShouldTrap X86_XCPT_PF, fxsave  [xSI + PAGE_SIZE - 512 + xBX]
+
+        mov     ecx, 512 / 4
+        lea     xDI, [xSI + 512]
+        cld
+        repz cmpsd
+        lea     xAX, [xBX + 20000]
+        jnz     .return
+
+        add     ebx, 16
+        cmp     ebx, 512
+        jbe     .fxsave_pf_effect_loop
+
+        ; Lets check that a FP in fxrstor does not have any effect on the FPU or SSE state.
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        mov     ecx, PAGE_SIZE / 4
+        mov     eax, 0ffaa33cch
+        cld
+        rep stosd
+
+        call    x861_LoadUniqueRegValuesSSEAndGRegs
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        fxsave  [xDI]
+
+        call    x861_ClearRegistersSSEAndGRegs
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        fxsave  [xDI + 512]
+
+        mov     ebx, 16
+.fxrstor_pf_effect_loop:
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        mov     xSI, xDI
+        lea     xDI, [xDI + PAGE_SIZE - 512 + xBX]
+        mov     ecx, 512
+        sub     ecx, ebx
+        cld
+        rep movsb                       ; copy unique state to end of page.
+
+        push    xBX
+        call    x861_ClearRegistersSSEAndGRegs
+        pop     xBX
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        ShouldTrap X86_XCPT_PF, fxrstor  [xDI + PAGE_SIZE - 512 + xBX] ; try load unique state
+
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        lea     xSI, [xDI + 512]        ; point it to the clean state, which is what we expect.
+        lea     xDI, [xDI + 1024]
+        fxsave  [xDI]                   ; save whatever the fpu state currently is.
+        mov     ecx, 512 / 4
+        cld
+        repe cmpsd
+        lea     xAX, [xBX + 40000]
+        jnz     .return                 ; it shouldn't be modified by faulting fxrstor, i.e. a clean state.
+
+        add     ebx, 16
+        cmp     ebx, 512
+        jbe     .fxrstor_pf_effect_loop
+
+.success:
+        xor     eax, eax
+.return:
+        SAVE_ALL_EPILOGUE
+        ret
+ENDPROC     x861_Test3
+
+
+;;
+; Tests various multibyte NOP sequences.
+;
+BEGINPROC   x861_Test4
+        SAVE_ALL_PROLOGUE
+        call    x861_ClearRegisters
+
+        ; Intel recommended sequences.
+        nop
+        db 066h, 090h
+        db 00fh, 01fh, 000h
+        db 00fh, 01fh, 040h, 000h
+        db 00fh, 01fh, 044h, 000h, 000h
+        db 066h, 00fh, 01fh, 044h, 000h, 000h
+        db 00fh, 01fh, 080h, 000h, 000h, 000h, 000h
+        db 00fh, 01fh, 084h, 000h, 000h, 000h, 000h, 000h
+        db 066h, 00fh, 01fh, 084h, 000h, 000h, 000h, 000h, 000h
+
+        ; Check that the NOPs are allergic to lock prefixing.
+        ShouldTrap X86_XCPT_UD, db 0f0h, 090h               ; lock prefixed NOP.
+        ShouldTrap X86_XCPT_UD, db 0f0h, 066h, 090h         ; lock prefixed two byte NOP.
+        ShouldTrap X86_XCPT_UD, db 0f0h, 00fh, 01fh, 000h   ; lock prefixed three byte NOP.
+
+        ; Check the range of instructions that AMD marks as NOPs.
+%macro TST_NOP 1
+        db 00fh, %1, 000h
+        db 00fh, %1, 040h, 000h
+        db 00fh, %1, 044h, 000h, 000h
+        db 066h, 00fh, %1, 044h, 000h, 000h
+        db 00fh, %1, 080h, 000h, 000h, 000h, 000h
+        db 00fh, %1, 084h, 000h, 000h, 000h, 000h, 000h
+        db 066h, 00fh, %1, 084h, 000h, 000h, 000h, 000h, 000h
+        ShouldTrap X86_XCPT_UD, db 0f0h, 00fh, %1, 000h
+%endmacro
+        TST_NOP 019h
+        TST_NOP 01ah
+        TST_NOP 01bh
+        TST_NOP 01ch
+        TST_NOP 01dh
+        TST_NOP 01eh
+        TST_NOP 01fh
+
+        ; The AMD P group, intel marks this as a NOP.
+        TST_NOP 00dh
+
+.success:
+        xor     eax, eax
+.return:
+        SAVE_ALL_EPILOGUE
+        ret
+ENDPROC     x861_Test4
+
+
+;;
+; Tests various odd/weird/bad encodings.
+;
+BEGINPROC   x861_Test5
+        SAVE_ALL_PROLOGUE
+        call    x861_ClearRegisters
+
+%if 0
+        ; callf eax...
+        ShouldTrap X86_XCPT_UD, db 0xff, 11011000b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11011001b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11011010b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11011011b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11011100b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11011101b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11011110b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11011111b
+
+        ; jmpf eax...
+        ShouldTrap X86_XCPT_UD, db 0xff, 11101000b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11101001b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11101010b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11101011b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11101100b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11101101b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11101110b
+        ShouldTrap X86_XCPT_UD, db 0xff, 11101111b
+
+        ; #GP(0) vs #UD.
+        ShouldTrap X86_XCPT_GP, mov xAX, cr0
+        ShouldTrap X86_XCPT_UD, lock mov xAX, cr0
+        ShouldTrap X86_XCPT_GP, mov cr0, xAX
+        ShouldTrap X86_XCPT_UD, lock mov cr0, xAX
+        ShouldTrap X86_XCPT_UD, db 0x0f, 0x20,11001000b ; mov xAX, cr1
+        ShouldTrap X86_XCPT_UD, db 0x0f, 0x20,11101000b ; mov xAX, cr5
+        ShouldTrap X86_XCPT_UD, db 0x0f, 0x20,11110000b ; mov xAX, cr6
+        ShouldTrap X86_XCPT_UD, db 0x0f, 0x20,11111000b ; mov xAX, cr7
+        ShouldTrap X86_XCPT_GP, mov xAX, dr7
+        ShouldTrap X86_XCPT_UD, lock mov xAX, dr7
+
+        ; The MOD is ignored by MOV CRx,GReg and MOV GReg,CRx
+        ShouldTrap X86_XCPT_GP, db 0x0f, 0x20,00000000b ; mov xAX, cr0
+        ShouldTrap X86_XCPT_GP, db 0x0f, 0x20,01000000b ; mov xAX, cr0
+        ShouldTrap X86_XCPT_GP, db 0x0f, 0x20,10000000b ; mov xAX, cr0
+        ShouldTrap X86_XCPT_GP, db 0x0f, 0x20,11000000b ; mov xAX, cr0
+        ShouldTrap X86_XCPT_GP, db 0x0f, 0x22,00000000b ; mov cr0, xAX
+        ShouldTrap X86_XCPT_GP, db 0x0f, 0x22,01000000b ; mov cr0, xAX
+        ShouldTrap X86_XCPT_GP, db 0x0f, 0x22,10000000b ; mov cr0, xAX
+        ShouldTrap X86_XCPT_GP, db 0x0f, 0x22,11000000b ; mov cr0, xAX
+%endif
+
+        ; mov eax, tr0, 0x0f 0x24
+        ShouldTrap X86_XCPT_UD, db 0x0f, 0x24, 0xc0     ; mov xAX, tr1
+
+        mov     xAX, [REF_EXTERN(g_pbEfExecPage)]
+        add     xAX, PAGE_SIZE - 3
+        mov     byte [xAX    ], 0x0f
+        mov     byte [xAX + 1], 0x24
+        mov     byte [xAX + 2], 0xc0
+        ShouldTrapExecPage X86_XCPT_UD, PAGE_SIZE - 3
+
+        mov     xAX, [REF_EXTERN(g_pbEfExecPage)]
+        add     xAX, PAGE_SIZE - 2
+        mov     byte [xAX    ], 0x0f
+        mov     byte [xAX + 1], 0x24
+        ShouldTrapExecPage X86_XCPT_UD, PAGE_SIZE - 2
+
+.success:
+        xor     eax, eax
+.return:
+        SAVE_ALL_EPILOGUE
+        ret
+ENDPROC     x861_Test5
+
+
+;;
+; Tests an reserved FPU encoding, checking that it does not affect the FPU or
+; CPU state in any way.
+;
+; @uses stack
+%macro FpuNopEncoding 1+
+        fnclex
+        call    SetFSW_C0_thru_C3
+
+        push    xBP
+        mov     xBP, xSP
+        sub     xSP, 1024
+        and     xSP, ~0fh
+        call    SaveFPUAndGRegsToStack
+        %1
+        call    CompareFPUAndGRegsOnStackIgnoreOpAndIp
+        leave
+
+        jz      %%ok
+        add     eax, __LINE__
+        jmp     .return
+%%ok:
+%endmacro
+
+;;
+; Used for marking encodings which has a meaning other than FNOP and
+; needs investigating.
+%macro FpuReservedEncoding 2
+        fnclex
+        call    SetFSW_C0_thru_C3
+
+        push    xBP
+        mov     xBP, xSP
+        sub     xSP, 2048
+        and     xSP, ~0fh
+        mov     dword [xSP + 1024 + X86FXSTATE.FPUIP], 0
+        mov     dword [xSP + 1024 + X86FXSTATE.FPUCS], 0
+        mov     dword [xSP + 1024 + X86FXSTATE.FPUDP], 0
+        mov     dword [xSP + 1024 + X86FXSTATE.FPUDS], 0
+        arch_fxsave [xSP + 1024]
+        %1
+        call    SaveFPUAndGRegsToStack
+
+        arch_fxrstor [xSP + 1024]
+        %2
+        call    CompareFPUAndGRegsOnStackIgnoreOpAndIp
+        ;arch_fxrstor [xSP + 1024]
+        leave
+
+        jz      %%ok
+        add     eax, __LINE__
+        jmp     .return
+%%ok:
+%endmacro
+
+
+;;
+; Saves the FPU and general registers to the stack area right next to the
+; return address.
+;
+; The required area size is 512 + 80h = 640.
+;
+; @uses Nothing, except stack.
+;
+SaveFPUAndGRegsToStack:
+        ; Must clear the FXSAVE area.
+        pushf
+        push    xCX
+        push    xAX
+        push    xDI
+
+        lea     xDI, [xSP + xCB * 5]
+        mov     xCX, 512 / 4
+        mov     eax, 0cccccccch
+        cld
+        rep stosd
+
+        pop     xDI
+        pop     xAX
+        pop     xCX
+        popf
+
+        ; Save the FPU state.
+        mov     dword [xSP + xCB + X86FXSTATE.FPUIP], 0
+        mov     dword [xSP + xCB + X86FXSTATE.FPUCS], 0
+        mov     dword [xSP + xCB + X86FXSTATE.FPUDP], 0
+        mov     dword [xSP + xCB + X86FXSTATE.FPUDS], 0
+        arch_fxsave [xSP + xCB]
+
+        ; Save GRegs (80h bytes).
+%ifdef RT_ARCH_AMD64
+        mov     [xSP + 512 + xCB + 000h], xAX
+        mov     [xSP + 512 + xCB + 008h], xBX
+        mov     [xSP + 512 + xCB + 010h], xCX
+        mov     [xSP + 512 + xCB + 018h], xDX
+        mov     [xSP + 512 + xCB + 020h], xDI
+        mov     [xSP + 512 + xCB + 028h], xSI
+        mov     [xSP + 512 + xCB + 030h], xBP
+        mov     [xSP + 512 + xCB + 038h], r8
+        mov     [xSP + 512 + xCB + 040h], r9
+        mov     [xSP + 512 + xCB + 048h], r10
+        mov     [xSP + 512 + xCB + 050h], r11
+        mov     [xSP + 512 + xCB + 058h], r12
+        mov     [xSP + 512 + xCB + 060h], r13
+        mov     [xSP + 512 + xCB + 068h], r14
+        mov     [xSP + 512 + xCB + 070h], r15
+        pushf
+        pop     rax
+        mov     [xSP + 512 + xCB + 078h], rax
+        mov     rax, [xSP + 512 + xCB + 000h]
+%else
+        mov     [xSP + 512 + xCB + 000h], eax
+        mov     [xSP + 512 + xCB + 004h], eax
+        mov     [xSP + 512 + xCB + 008h], ebx
+        mov     [xSP + 512 + xCB + 00ch], ebx
+        mov     [xSP + 512 + xCB + 010h], ecx
+        mov     [xSP + 512 + xCB + 014h], ecx
+        mov     [xSP + 512 + xCB + 018h], edx
+        mov     [xSP + 512 + xCB + 01ch], edx
+        mov     [xSP + 512 + xCB + 020h], edi
+        mov     [xSP + 512 + xCB + 024h], edi
+        mov     [xSP + 512 + xCB + 028h], esi
+        mov     [xSP + 512 + xCB + 02ch], esi
+        mov     [xSP + 512 + xCB + 030h], ebp
+        mov     [xSP + 512 + xCB + 034h], ebp
+        mov     [xSP + 512 + xCB + 038h], eax
+        mov     [xSP + 512 + xCB + 03ch], eax
+        mov     [xSP + 512 + xCB + 040h], eax
+        mov     [xSP + 512 + xCB + 044h], eax
+        mov     [xSP + 512 + xCB + 048h], eax
+        mov     [xSP + 512 + xCB + 04ch], eax
+        mov     [xSP + 512 + xCB + 050h], eax
+        mov     [xSP + 512 + xCB + 054h], eax
+        mov     [xSP + 512 + xCB + 058h], eax
+        mov     [xSP + 512 + xCB + 05ch], eax
+        mov     [xSP + 512 + xCB + 060h], eax
+        mov     [xSP + 512 + xCB + 064h], eax
+        mov     [xSP + 512 + xCB + 068h], eax
+        mov     [xSP + 512 + xCB + 06ch], eax
+        mov     [xSP + 512 + xCB + 070h], eax
+        mov     [xSP + 512 + xCB + 074h], eax
+        pushf
+        pop     eax
+        mov     [xSP + 512 + xCB + 078h], eax
+        mov     [xSP + 512 + xCB + 07ch], eax
+        mov     eax, [xSP + 512 + xCB + 000h]
+%endif
+        ret
+
+;;
+; Compares the current FPU and general registers to that found in the stack
+; area prior to the return address.
+;
+; @uses     Stack, flags and eax/rax.
+; @returns  eax is zero on success, eax is 1000000 * offset on failure.
+;           ZF reflects the eax value to save a couple of instructions...
+;
+CompareFPUAndGRegsOnStack:
+        lea     xSP, [xSP - (1024 - xCB)]
+        call    SaveFPUAndGRegsToStack
+
+        push    xSI
+        push    xDI
+        push    xCX
+
+        mov     xCX, 640
+        lea     xSI, [xSP + xCB*3]
+        lea     xDI, [xSI + 1024]
+
+        cld
+        repe cmpsb
+        je      .ok
+
+        ;int3
+        lea     xAX, [xSP + xCB*3]
+        xchg    xAX, xSI
+        sub     xAX, xSI
+
+        push    xDX
+        mov     xDX, 1000000
+        mul     xDX
+        pop     xDX
+        jmp     .return
+.ok:
+        xor     eax, eax
+.return:
+        pop     xCX
+        pop     xDI
+        pop     xSI
+        lea     xSP, [xSP + (1024 - xCB)]
+        or      eax, eax
+        ret
+
+;;
+; Same as CompareFPUAndGRegsOnStack, except that it ignores the FOP and FPUIP
+; registers.
+;
+; @uses     Stack, flags and eax/rax.
+; @returns  eax is zero on success, eax is 1000000 * offset on failure.
+;           ZF reflects the eax value to save a couple of instructions...
+;
+CompareFPUAndGRegsOnStackIgnoreOpAndIp:
+        lea     xSP, [xSP - (1024 - xCB)]
+        call    SaveFPUAndGRegsToStack
+
+        push    xSI
+        push    xDI
+        push    xCX
+
+        mov     xCX, 640
+        lea     xSI, [xSP + xCB*3]
+        lea     xDI, [xSI + 1024]
+
+        mov     word [xSI + X86FXSTATE.FOP], 0          ; ignore
+        mov     word [xDI + X86FXSTATE.FOP], 0          ; ignore
+        mov     dword [xSI + X86FXSTATE.FPUIP], 0       ; ignore
+        mov     dword [xDI + X86FXSTATE.FPUIP], 0       ; ignore
+
+        cld
+        repe cmpsb
+        je      .ok
+
+        ;int3
+        lea     xAX, [xSP + xCB*3]
+        xchg    xAX, xSI
+        sub     xAX, xSI
+
+        push    xDX
+        mov     xDX, 1000000
+        mul     xDX
+        pop     xDX
+        jmp     .return
+.ok:
+        xor     eax, eax
+.return:
+        pop     xCX
+        pop     xDI
+        pop     xSI
+        lea     xSP, [xSP + (1024 - xCB)]
+        or      eax, eax
+        ret
+
+
+SetFSW_C0_thru_C3:
+        sub     xSP, 20h
+        fstenv  [xSP]
+        or      word [xSP + 4], X86_FSW_C0 | X86_FSW_C1 | X86_FSW_C2 | X86_FSW_C3
+        fldenv  [xSP]
+        add     xSP, 20h
+        ret
+
+
+;;
+; Tests some odd floating point instruction encodings.
+;
+BEGINPROC   x861_Test6
+        SAVE_ALL_PROLOGUE
+
+        ; standard stuff...
+        fld dword [REF(g_r32V1)]
+        fld qword [REF(g_r64V1)]
+        fld tword [REF(g_r80V1)]
+        fld qword [REF(g_r64V1)]
+        fld dword [REF(g_r32V2)]
+        fld dword [REF(g_r32V1)]
+
+        ; Test the nop check.
+        FpuNopEncoding fnop
+
+
+        ; the 0xd9 block
+        ShouldTrap X86_XCPT_UD, db 0d9h, 008h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 009h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 00ah
+        ShouldTrap X86_XCPT_UD, db 0d9h, 00bh
+        ShouldTrap X86_XCPT_UD, db 0d9h, 00ch
+        ShouldTrap X86_XCPT_UD, db 0d9h, 00dh
+        ShouldTrap X86_XCPT_UD, db 0d9h, 00eh
+        ShouldTrap X86_XCPT_UD, db 0d9h, 00fh
+
+        ShouldTrap X86_XCPT_UD, db 0d9h, 0d1h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 0d2h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 0d3h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 0d4h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 0d5h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 0d6h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 0d7h
+        FpuReservedEncoding {db 0d9h, 0d8h}, { fstp st0 }
+        FpuReservedEncoding {db 0d9h, 0d9h}, { fstp st1 }
+        FpuReservedEncoding {db 0d9h, 0dah}, { fstp st2 }
+        FpuReservedEncoding {db 0d9h, 0dbh}, { fstp st3 }
+        FpuReservedEncoding {db 0d9h, 0dch}, { fstp st4 }
+        FpuReservedEncoding {db 0d9h, 0ddh}, { fstp st5 }
+        FpuReservedEncoding {db 0d9h, 0deh}, { fstp st6 }
+        ;FpuReservedEncoding {db 0d9h, 0dfh}, { fstp st7 } ; This variant seems to ignore empty ST(0) values!
+        ShouldTrap X86_XCPT_UD, db 0d9h, 0e2h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 0e3h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 0e6h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 0e7h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 0efh
+        ShouldTrap X86_XCPT_UD, db 0d9h, 008h
+        ShouldTrap X86_XCPT_UD, db 0d9h, 00fh
+
+        ; the 0xda block
+        ShouldTrap X86_XCPT_UD, db 0dah, 0e0h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0e1h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0e2h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0e3h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0e4h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0e5h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0e6h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0e7h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0e8h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0eah
+        ShouldTrap X86_XCPT_UD, db 0dah, 0ebh
+        ShouldTrap X86_XCPT_UD, db 0dah, 0ech
+        ShouldTrap X86_XCPT_UD, db 0dah, 0edh
+        ShouldTrap X86_XCPT_UD, db 0dah, 0eeh
+        ShouldTrap X86_XCPT_UD, db 0dah, 0efh
+        ShouldTrap X86_XCPT_UD, db 0dah, 0f0h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0f1h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0f2h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0f3h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0f4h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0f5h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0f6h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0f7h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0f8h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0f9h
+        ShouldTrap X86_XCPT_UD, db 0dah, 0fah
+        ShouldTrap X86_XCPT_UD, db 0dah, 0fbh
+        ShouldTrap X86_XCPT_UD, db 0dah, 0fch
+        ShouldTrap X86_XCPT_UD, db 0dah, 0fdh
+        ShouldTrap X86_XCPT_UD, db 0dah, 0feh
+        ShouldTrap X86_XCPT_UD, db 0dah, 0ffh
+
+        ; the 0xdb block
+        FpuNopEncoding db 0dbh, 0e0h ; fneni
+        FpuNopEncoding db 0dbh, 0e1h ; fndisi
+        FpuNopEncoding db 0dbh, 0e4h ; fnsetpm
+        ShouldTrap X86_XCPT_UD, db 0dbh, 0e5h
+        ShouldTrap X86_XCPT_UD, db 0dbh, 0e6h
+        ShouldTrap X86_XCPT_UD, db 0dbh, 0e7h
+        ShouldTrap X86_XCPT_UD, db 0dbh, 0f8h
+        ShouldTrap X86_XCPT_UD, db 0dbh, 0f9h
+        ShouldTrap X86_XCPT_UD, db 0dbh, 0fah
+        ShouldTrap X86_XCPT_UD, db 0dbh, 0fbh
+        ShouldTrap X86_XCPT_UD, db 0dbh, 0fch
+        ShouldTrap X86_XCPT_UD, db 0dbh, 0fdh
+        ShouldTrap X86_XCPT_UD, db 0dbh, 0feh
+        ShouldTrap X86_XCPT_UD, db 0dbh, 0ffh
+        ShouldTrap X86_XCPT_UD, db 0dbh, 020h
+        ShouldTrap X86_XCPT_UD, db 0dbh, 023h
+        ShouldTrap X86_XCPT_UD, db 0dbh, 030h
+        ShouldTrap X86_XCPT_UD, db 0dbh, 032h
+
+        ; the 0xdc block
+        FpuReservedEncoding {db 0dch, 0d0h}, { fcom st0 }
+        FpuReservedEncoding {db 0dch, 0d1h}, { fcom st1 }
+        FpuReservedEncoding {db 0dch, 0d2h}, { fcom st2 }
+        FpuReservedEncoding {db 0dch, 0d3h}, { fcom st3 }
+        FpuReservedEncoding {db 0dch, 0d4h}, { fcom st4 }
+        FpuReservedEncoding {db 0dch, 0d5h}, { fcom st5 }
+        FpuReservedEncoding {db 0dch, 0d6h}, { fcom st6 }
+        FpuReservedEncoding {db 0dch, 0d7h}, { fcom st7 }
+        FpuReservedEncoding {db 0dch, 0d8h}, { fcomp st0 }
+        FpuReservedEncoding {db 0dch, 0d9h}, { fcomp st1 }
+        FpuReservedEncoding {db 0dch, 0dah}, { fcomp st2 }
+        FpuReservedEncoding {db 0dch, 0dbh}, { fcomp st3 }
+        FpuReservedEncoding {db 0dch, 0dch}, { fcomp st4 }
+        FpuReservedEncoding {db 0dch, 0ddh}, { fcomp st5 }
+        FpuReservedEncoding {db 0dch, 0deh}, { fcomp st6 }
+        FpuReservedEncoding {db 0dch, 0dfh}, { fcomp st7 }
+
+        ; the 0xdd block
+        FpuReservedEncoding {db 0ddh, 0c8h}, { fxch st0 }
+        FpuReservedEncoding {db 0ddh, 0c9h}, { fxch st1 }
+        FpuReservedEncoding {db 0ddh, 0cah}, { fxch st2 }
+        FpuReservedEncoding {db 0ddh, 0cbh}, { fxch st3 }
+        FpuReservedEncoding {db 0ddh, 0cch}, { fxch st4 }
+        FpuReservedEncoding {db 0ddh, 0cdh}, { fxch st5 }
+        FpuReservedEncoding {db 0ddh, 0ceh}, { fxch st6 }
+        FpuReservedEncoding {db 0ddh, 0cfh}, { fxch st7 }
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0f0h
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0f1h
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0f2h
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0f3h
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0f4h
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0f5h
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0f6h
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0f7h
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0f8h
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0f9h
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0fah
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0fbh
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0fch
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0fdh
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0feh
+        ShouldTrap X86_XCPT_UD, db 0ddh, 0ffh
+        ShouldTrap X86_XCPT_UD, db 0ddh, 028h
+        ShouldTrap X86_XCPT_UD, db 0ddh, 02fh
+
+        ; the 0xde block
+        FpuReservedEncoding {db 0deh, 0d0h}, { fcomp st0 }
+        FpuReservedEncoding {db 0deh, 0d1h}, { fcomp st1 }
+        FpuReservedEncoding {db 0deh, 0d2h}, { fcomp st2 }
+        FpuReservedEncoding {db 0deh, 0d3h}, { fcomp st3 }
+        FpuReservedEncoding {db 0deh, 0d4h}, { fcomp st4 }
+        FpuReservedEncoding {db 0deh, 0d5h}, { fcomp st5 }
+        FpuReservedEncoding {db 0deh, 0d6h}, { fcomp st6 }
+        FpuReservedEncoding {db 0deh, 0d7h}, { fcomp st7 }
+        ShouldTrap X86_XCPT_UD, db 0deh, 0d8h
+        ShouldTrap X86_XCPT_UD, db 0deh, 0dah
+        ShouldTrap X86_XCPT_UD, db 0deh, 0dbh
+        ShouldTrap X86_XCPT_UD, db 0deh, 0dch
+        ShouldTrap X86_XCPT_UD, db 0deh, 0ddh
+        ShouldTrap X86_XCPT_UD, db 0deh, 0deh
+        ShouldTrap X86_XCPT_UD, db 0deh, 0dfh
+
+        ; the 0xdf block
+        FpuReservedEncoding {db 0dfh, 0c8h}, { fxch st0 }
+        FpuReservedEncoding {db 0dfh, 0c9h}, { fxch st1 }
+        FpuReservedEncoding {db 0dfh, 0cah}, { fxch st2 }
+        FpuReservedEncoding {db 0dfh, 0cbh}, { fxch st3 }
+        FpuReservedEncoding {db 0dfh, 0cch}, { fxch st4 }
+        FpuReservedEncoding {db 0dfh, 0cdh}, { fxch st5 }
+        FpuReservedEncoding {db 0dfh, 0ceh}, { fxch st6 }
+        FpuReservedEncoding {db 0dfh, 0cfh}, { fxch st7 }
+        FpuReservedEncoding {db 0dfh, 0d0h}, { fstp st0 }
+        FpuReservedEncoding {db 0dfh, 0d1h}, { fstp st1 }
+        FpuReservedEncoding {db 0dfh, 0d2h}, { fstp st2 }
+        FpuReservedEncoding {db 0dfh, 0d3h}, { fstp st3 }
+        FpuReservedEncoding {db 0dfh, 0d4h}, { fstp st4 }
+        FpuReservedEncoding {db 0dfh, 0d5h}, { fstp st5 }
+        FpuReservedEncoding {db 0dfh, 0d6h}, { fstp st6 }
+        FpuReservedEncoding {db 0dfh, 0d7h}, { fstp st7 }
+        FpuReservedEncoding {db 0dfh, 0d8h}, { fstp st0 }
+        FpuReservedEncoding {db 0dfh, 0d9h}, { fstp st1 }
+        FpuReservedEncoding {db 0dfh, 0dah}, { fstp st2 }
+        FpuReservedEncoding {db 0dfh, 0dbh}, { fstp st3 }
+        FpuReservedEncoding {db 0dfh, 0dch}, { fstp st4 }
+        FpuReservedEncoding {db 0dfh, 0ddh}, { fstp st5 }
+        FpuReservedEncoding {db 0dfh, 0deh}, { fstp st6 }
+        FpuReservedEncoding {db 0dfh, 0dfh}, { fstp st7 }
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0e1h
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0e2h
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0e3h
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0e4h
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0e5h
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0e6h
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0e7h
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0f8h
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0f9h
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0fah
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0fbh
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0fch
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0fdh
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0feh
+        ShouldTrap X86_XCPT_UD, db 0dfh, 0ffh
+
+
+.success:
+        xor     eax, eax
+.return:
+        SAVE_ALL_EPILOGUE
+        ret
+
+ENDPROC     x861_Test6
+
+
+;;
+; Tests some floating point exceptions and such.
+;
+;
+;
+BEGINPROC   x861_Test7
+        SAVE_ALL_PROLOGUE
+        sub     xSP, 2048
+
+        ; Load some pointers.
+        lea     xSI, [REF(g_r32V1)]
+        mov     xDI, [REF_EXTERN(g_pbEfExecPage)]
+        add     xDI, PAGE_SIZE          ; invalid page.
+
+        ;
+        ; Check denormal numbers.
+        ; Turns out the number is loaded onto the stack even if an exception is triggered.
+        ;
+        fninit
+        mov     dword [xSP], X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fldcw   [xSP]
+        FpuShouldTrap  X86_FSW_DE, 0, fld dword [REF(g_r32D0)]
+        CheckSt0Value 0x00000000, 0x80000000, 0x3f7f
+
+        mov     dword [xSP], X86_FCW_PC_64 | X86_FCW_RC_NEAREST | X86_FCW_DM
+        fldcw   [xSP]
+        fld     dword [REF(g_r32D0)]
+        fwait
+        FpuCheckFSW X86_FSW_DE, 0
+        CheckSt0Value 0x00000000, 0x80000000, 0x3f7f
+
+        ;
+        ; stack overflow
+        ;
+        fninit
+        mov     dword [xSP], X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fldcw   [xSP]
+        fld     qword [REF(g_r64V1)]
+        fld     dword [xSI]
+        fld     dword [xSI]
+        fld     dword [xSI]
+        fld     dword [xSI]
+        fld     dword [xSI]
+        fld     dword [xSI]
+        fld     tword [REF(g_r80V1)]
+        fwait
+
+        FpuShouldTrap  X86_FSW_IE | X86_FSW_SF | X86_FSW_C1, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3, \
+                fld     dword [xSI]
+        CheckSt0Value_Eight
+
+        FpuShouldTrap  X86_FSW_IE | X86_FSW_SF | X86_FSW_C1, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3, \
+                fld     dword [xSI]
+        CheckSt0Value_Eight
+
+        ; stack overflow vs #PF.
+        ShouldTrap X86_XCPT_PF, fld     dword [xDI]
+        fwait
+
+        ; stack overflow vs denormal number
+        FpuShouldTrap  X86_FSW_IE | X86_FSW_SF | X86_FSW_C1, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3, \
+                fld     dword [xSI]
+        CheckSt0Value_Eight
+
+        ;
+        ; Mask the overflow exception. We should get QNaN now regardless of
+        ; what we try to push (provided the memory is valid).
+        ;
+        mov     dword [xSP], X86_FCW_PC_64 | X86_FCW_RC_NEAREST | X86_FCW_IM
+        fldcw   [xSP]
+
+        fld     dword [xSI]
+        FpuCheckFSW X86_FSW_IE | X86_FSW_SF | X86_FSW_C1, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        fnclex
+        CheckSt0Value 0x00000000, 0xc0000000, 0xffff
+
+        fld     qword [REF(g_r64V1)]
+        FpuCheckFSW X86_FSW_IE | X86_FSW_SF | X86_FSW_C1, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        fnclex
+        CheckSt0Value 0x00000000, 0xc0000000, 0xffff
+
+        ; This is includes denormal values.
+        fld     dword [REF(g_r32D0)]
+        fwait
+        FpuCheckFSW X86_FSW_IE | X86_FSW_SF | X86_FSW_C1, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckSt0Value 0x00000000, 0xc0000000, 0xffff
+        fnclex
+
+        ;
+        ; #PF vs previous stack overflow. I.e. whether pending FPU exception
+        ; is checked before fetching memory operands.
+        ;
+        mov     dword [xSP], X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fldcw   [xSP]
+        fld     qword [REF(g_r64V1)]
+        ShouldTrap X86_XCPT_MF, fld     dword [xDI]
+        fnclex
+
+        ;
+        ; What happens when we unmask an exception and fwait?
+        ;
+        mov     dword [xSP], X86_FCW_PC_64 | X86_FCW_RC_NEAREST | X86_FCW_IM
+        fldcw   [xSP]
+        fld     dword [xSI]
+        fwait
+        FpuCheckFSW X86_FSW_IE | X86_FSW_SF | X86_FSW_C1, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        mov     dword [xSP], X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fldcw   [xSP]
+        FpuCheckFSW X86_FSW_ES | X86_FSW_B | X86_FSW_IE | X86_FSW_SF | X86_FSW_C1, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+
+        ShouldTrap X86_XCPT_MF, fwait
+        ShouldTrap X86_XCPT_MF, fwait
+        ShouldTrap X86_XCPT_MF, fwait
+        fnclex
+
+
+.success:
+        xor     eax, eax
+.return:
+        add     xSP, 2048
+        SAVE_ALL_EPILOGUE
+        ret
+ENDPROC     x861_Test7
+
+
+extern NAME(RTTestISub)
+
+;;
+; Sets the current subtest.
+%macro SetSubTest 1
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_GCC
+        lea     rdi, [%%s_szName wrt rip]
+ %else
+        lea     rcx, [%%s_szName wrt rip]
+ %endif
+        call    NAME(RTTestISub)
+%else
+ %ifdef RT_OS_DARWIN
+        sub     esp, 12
+        push    %%s_szName
+        call    NAME(RTTestISub)
+        add     esp, 16
+ %else
+        push    %%s_szName
+        call    NAME(RTTestISub)
+        add     esp, 4
+ %endif
+%endif
+        jmp     %%done
+%%s_szName:
+        db %1, 0
+%%done:
+%endmacro
+
+
+;;
+; Checks the opcode and CS:IP FPU.
+;
+; @returns ZF=1 on success, ZF=0 on failure.
+; @param    xSP + xCB    fxsave image followed by fnstenv.
+; @param    xCX         Opcode address (no prefixes).
+;
+CheckOpcodeCsIp:
+        push    xBP
+        mov     xBP, xSP
+        push    xAX
+
+        ; Check the IP.
+%ifdef RT_ARCH_AMD64
+        cmp     rcx, [xBP + xCB*2 + X86FXSTATE.FPUIP]
+%else
+        cmp     ecx, [xBP + xCB*2 + X86FXSTATE.FPUIP]
+%endif
+        jne     .failure1
+
+.check_fpucs:
+        mov     ax, cs
+        cmp     ax, [xBP + xCB*2 + 512 + X86FSTENV32P.FPUCS]
+        jne     .failure2
+
+        ; Check the opcode.  This may be disabled.
+        mov     ah, [xCX]
+        mov     al, [xCX + 1]
+        and     ax, 07ffh
+
+        cmp     ax, [xBP + xCB*2 + X86FXSTATE.FOP]
+        je      .success
+        cmp     ax, [xBP + xCB*2 + 512 + X86FSTENV32P.FOP]
+        je      .success
+
+;        xor     ax, ax
+;        cmp     ax, [xBP + xCB*2 + X86FXSTATE.FOP]
+;        jne     .failure3
+
+.success:
+        xor     eax, eax                ; clear Z
+.return:
+        pop     xAX
+        leave
+        ret
+
+.failure1:
+        ; AMD64 doesn't seem to store anything at IP and DP, so use the
+        ; fnstenv image instead even if that only contains the lower 32-bit.
+        xor     eax, eax
+        cmp     xAX, [xBP + xCB*2 + X86FXSTATE.FPUIP]
+        jne     .failure1_for_real
+        cmp     xAX, [xBP + xCB*2 + X86FXSTATE.FPUDP]
+        jne     .failure1_for_real
+        cmp     ecx, [xBP + xCB*2 + 512 + X86FSTENV32P.FPUIP]
+        je      .check_fpucs
+.failure1_for_real:
+        mov     eax, 10000000
+        jmp     .failure
+.failure2:
+        mov     eax, 20000000
+        jmp     .failure
+.failure3:
+        mov     eax, 30000000
+        jmp     .failure
+.failure:
+        or      eax, eax
+        leave
+        ret
+
+;;
+; Checks a FPU instruction, no memory operand.
+;
+; @uses xCX, xAX, Stack.
+;
+%macro FpuCheckOpcodeCsIp 1
+        mov     dword [xSP + X86FXSTATE.FPUIP], 0
+        mov     dword [xSP + X86FXSTATE.FPUCS], 0
+        mov     dword [xSP + X86FXSTATE.FPUDP], 0
+        mov     dword [xSP + X86FXSTATE.FPUDS], 0
+%%instruction:
+        %1
+        arch_fxsave  [xSP]
+        fnstenv [xSP + 512]             ; for the selectors (64-bit)
+        arch_fxrstor [xSP]              ; fnstenv screws up the ES bit.
+        lea     xCX, [REF(%%instruction)]
+        call    CheckOpcodeCsIp
+        jz      %%ok
+        lea     xAX, [xAX + __LINE__]
+        jmp     .return
+%%ok:
+%endmacro
+
+
+;;
+; Checks a trapping FPU instruction, no memory operand.
+;
+; Upon return, there is are two FXSAVE image on the stack at xSP.
+;
+; @uses xCX, xAX, Stack.
+;
+; @param    %1  The instruction.
+;
+%macro FpuTrapOpcodeCsIp 1
+        mov     dword [xSP + 1024 + 512 + X86FXSTATE.FPUIP], 0
+        mov     dword [xSP + 1024 + 512 + X86FXSTATE.FPUCS], 0
+        mov     dword [xSP + 1024 + 512 + X86FXSTATE.FPUDP], 0
+        mov     dword [xSP + 1024 + 512 + X86FXSTATE.FPUDS], 0
+        mov     dword [xSP + X86FXSTATE.FPUIP], 0
+        mov     dword [xSP + X86FXSTATE.FPUCS], 0
+        mov     dword [xSP + X86FXSTATE.FPUDP], 0
+        mov     dword [xSP + X86FXSTATE.FPUDS], 0
+%%instruction:
+        %1
+        fxsave [xSP + 1024 +512]        ; FPUDS and FPUCS for 64-bit hosts.
+                                        ; WEIRD: When saved after FWAIT they are ZEROed! (64-bit Intel)
+        arch_fxsave  [xSP]
+        fnstenv [xSP + 512]
+        arch_fxrstor [xSP]
+%%trap:
+        fwait
+%%trap_end:
+        mov     eax, __LINE__
+        jmp     .return
+BEGINDATA
+%%trapinfo: istruc TRAPINFO
+        at TRAPINFO.uTrapPC,    RTCCPTR_DEF     %%trap
+        at TRAPINFO.uResumePC,  RTCCPTR_DEF     %%resume
+        at TRAPINFO.u8TrapNo,   db              X86_XCPT_MF
+        at TRAPINFO.cbInstr,    db              (%%trap_end - %%trap)
+iend
+BEGINCODE
+%%resume:
+        lea     xCX, [REF(%%instruction)]
+        call    CheckOpcodeCsIp
+        jz      %%ok
+        lea     xAX, [xAX + __LINE__]
+        jmp     .return
+%%ok:
+%endmacro
+
+
+
+
+;;
+; Checks the opcode, CS:IP and DS:DP of the FPU.
+;
+; @returns ZF=1 on success, ZF=0+EAX on failure.
+; @param    xSP + xCB    fxsave image followed by fnstenv.
+; @param    xCX         Opcode address (no prefixes).
+; @param    xDX         Memory address (DS relative).
+;
+CheckOpcodeCsIpDsDp:
+        push    xBP
+        mov     xBP, xSP
+        push    xAX
+
+        ; Check the memory operand.
+%ifdef RT_ARCH_AMD64
+        cmp     rdx, [xBP + xCB*2 + X86FXSTATE.FPUDP]
+%else
+        cmp     edx, [xBP + xCB*2 + X86FXSTATE.FPUDP]
+%endif
+        jne     .failure1
+
+.check_fpuds:
+        mov     ax, ds
+        cmp     ax, [xBP + xCB*2 + 512 + X86FSTENV32P.FPUDS]
+        jne     .failure2
+
+.success:
+        pop     xAX
+        leave
+        ; Let CheckOpcodeCsIp to the rest.
+        jmp     CheckOpcodeCsIp
+
+.failure1:
+        ; AMD may leave all fields as ZERO in the FXSAVE image - figure
+        ; if there is a flag controlling this anywhere...
+        xor        eax, eax
+        cmp     xAX, [xBP + xCB*2 + X86FXSTATE.FPUDP]
+        jne     .failure1_for_real
+        cmp     xAX, [xBP + xCB*2 + X86FXSTATE.FPUIP]
+        jne     .failure1_for_real
+        cmp     edx, [xBP + xCB*2 + 512 + X86FSTENV32P.FPUDP]
+        je      .check_fpuds
+.failure1_for_real:
+        mov     eax, 60000000
+        jmp     .failure
+.failure2:
+        mov     eax, 80000000
+.failure:
+        or      eax, eax
+        leave
+        ret
+
+
+;;
+; Checks a FPU instruction taking a memory operand.
+;
+; @uses xCX, xDX, xAX, Stack.
+;
+%macro FpuCheckOpcodeCsIpDsDp 2
+        mov     dword [xSP + X86FXSTATE.FPUIP], 0
+        mov     dword [xSP + X86FXSTATE.FPUCS], 0
+        mov     dword [xSP + X86FXSTATE.FPUDP], 0
+        mov     dword [xSP + X86FXSTATE.FPUDS], 0
+%%instruction:
+        %1
+        arch_fxsave  [xSP]
+        fnstenv [xSP + 512]             ; for the selectors (64-bit)
+        arch_fxrstor [xSP]              ; fnstenv screws up the ES bit.
+        lea     xDX, %2
+        lea     xCX, [REF(%%instruction)]
+        call    CheckOpcodeCsIpDsDp
+        jz      %%ok
+        lea     xAX, [xAX + __LINE__]
+        jmp     .return
+%%ok:
+%endmacro
+
+
+;;
+; Checks a trapping FPU instruction taking a memory operand.
+;
+; Upon return, there is are two FXSAVE image on the stack at xSP.
+;
+; @uses xCX, xDX, xAX, Stack.
+;
+; @param    %1  The instruction.
+; @param    %2  Operand memory address (DS relative).
+;
+%macro FpuTrapOpcodeCsIpDsDp 2
+        mov     dword [xSP + X86FXSTATE.FPUIP], 0
+        mov     dword [xSP + X86FXSTATE.FPUCS], 0
+        mov     dword [xSP + X86FXSTATE.FPUDP], 0
+        mov     dword [xSP + X86FXSTATE.FPUDS], 0
+%%instruction:
+        %1
+        fxsave [xSP + 1024 +512]        ; FPUDS and FPUCS for 64-bit hosts.
+                                        ; WEIRD: When saved after FWAIT they are ZEROed! (64-bit Intel)
+        arch_fxsave  [xSP]
+        fnstenv [xSP + 512]
+        arch_fxrstor [xSP]
+%%trap:
+        fwait
+%%trap_end:
+        mov     eax, __LINE__
+        jmp     .return
+BEGINDATA
+%%trapinfo: istruc TRAPINFO
+        at TRAPINFO.uTrapPC,    RTCCPTR_DEF     %%trap
+        at TRAPINFO.uResumePC,  RTCCPTR_DEF     %%resume
+        at TRAPINFO.u8TrapNo,   db              X86_XCPT_MF
+        at TRAPINFO.cbInstr,    db              (%%trap_end - %%trap)
+iend
+BEGINCODE
+%%resume:
+        lea     xDX, %2
+        lea     xCX, [REF(%%instruction)]
+        call    CheckOpcodeCsIpDsDp
+        jz      %%ok
+        lea     xAX, [xAX + __LINE__]
+        jmp     .return
+%%ok:
+%endmacro
+
+
+;;
+; Checks that the FPU and GReg state is completely unchanged after an instruction
+; resulting in a CPU trap.
+;
+; @param        1       The trap number.
+; @param        2+      The instruction which should trap.
+;
+%macro FpuCheckCpuTrapUnchangedState 2+
+        call    SaveFPUAndGRegsToStack
+        ShouldTrap %1, %2
+        call    CompareFPUAndGRegsOnStack
+        jz      %%ok
+        lea     xAX, [xAX + __LINE__]
+        jmp     .return
+%%ok:
+%endmacro
+
+
+;;
+; Initialize the FPU and set CW to %1.
+;
+; @uses dword at [xSP].
+;
+%macro FpuInitWithCW 1
+        call    x861_LoadUniqueRegValuesSSE
+        fninit
+        mov     dword [xSP], %1
+        fldcw   [xSP]
+%endmacro
+
+
+;;
+; First bunch of FPU instruction tests.
+;
+;
+BEGINPROC   x861_TestFPUInstr1
+        SAVE_ALL_PROLOGUE
+        sub     xSP, 2048
+%if 0
+        ;
+        ; FDIV with 64-bit floating point memory operand.
+        ;
+        SetSubTest "FDIV m64r"
+
+        ; ## Normal operation. ##
+
+        fninit
+        FpuCheckOpcodeCsIpDsDp { fld  dword [REF(g_r32_3dot2)]     }, [REF(g_r32_3dot2)]
+        CheckSt0Value 0x00000000, 0xcccccd00, 0x4000
+        FpuCheckOpcodeCsIpDsDp { fdiv qword [REF(g_r64_One)]       }, [REF(g_r64_One)]
+        FpuCheckFSW 0, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckSt0Value 0x00000000, 0xcccccd00, 0x4000
+
+
+        ; ## Masked exceptions. ##
+
+        ; Masked stack underflow.
+        fninit
+        FpuCheckOpcodeCsIpDsDp { fdiv    qword [REF(g_r64_One)]    }, [REF(g_r64_One)]
+        FpuCheckFSW X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckSt0Value_QNaN
+
+        ; Masked zero divide.
+        fninit
+        FpuCheckOpcodeCsIpDsDp { fld     dword [REF(g_r32_3dot2)]  }, [REF(g_r32_3dot2)]
+        FpuCheckOpcodeCsIpDsDp { fdiv    qword [REF(g_r64_Zero)]   }, [REF(g_r64_Zero)]
+        FpuCheckFSW X86_FSW_ZE, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckSt0Value_PlusInf
+
+        ; Masked Inf/Inf.
+        fninit
+        FpuCheckOpcodeCsIpDsDp { fld     dword [REF(g_r32_Inf)]    }, [REF(g_r32_Inf)]
+        FpuCheckOpcodeCsIpDsDp { fdiv    qword [REF(g_r64_Inf)]    }, [REF(g_r64_Inf)]
+        FpuCheckFSW X86_FSW_IE, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckSt0Value_QNaN
+
+        ; Masked 0/0.
+        fninit
+        FpuCheckOpcodeCsIpDsDp { fld     dword [REF(g_r32_Zero)]   }, [REF(g_r32_Zero)]
+        FpuCheckOpcodeCsIpDsDp { fdiv    qword [REF(g_r64_Zero)]   }, [REF(g_r64_Zero)]
+        FpuCheckFSW X86_FSW_IE, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckSt0Value_QNaN
+
+        ; Masked precision exception, rounded down.
+        fninit
+        FpuCheckOpcodeCsIpDsDp { fld     dword [REF(g_r32_Ten)]    }, [REF(g_r32_Ten)]
+        FpuCheckOpcodeCsIpDsDp { fdiv    qword [REF(g_r64_Three)]  }, [REF(g_r64_Three)]
+        FpuCheckFSW X86_FSW_PE, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckSt0Value_3_and_a_3rd
+
+        ; Masked precision exception, rounded up.
+        fninit
+        FpuCheckOpcodeCsIpDsDp { fld     dword [REF(g_r32_Eleven)] }, [REF(g_r32_Eleven)]
+        FpuCheckOpcodeCsIpDsDp { fdiv    qword [REF(g_r64_Three)]  }, [REF(g_r64_Three)]
+        FpuCheckFSW X86_FSW_PE | X86_FSW_C1, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckSt0Value_3_and_two_3rds
+
+        ; Masked overflow exception.
+        fninit
+        FpuCheckOpcodeCsIpDsDp { fld     tword [REF(g_r80_Max)]    }, [REF(g_r80_Max)]
+        FpuCheckOpcodeCsIpDsDp { fdiv    qword [REF(g_r64_0dot1)]  }, [REF(g_r64_0dot1)]
+        FpuCheckFSW X86_FSW_PE | X86_FSW_OE | X86_FSW_C1, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckSt0Value_PlusInf
+
+        ; Masked underflow exception.
+        fninit
+        FpuCheckOpcodeCsIpDsDp { fld     tword [REF(g_r80_Min)]    }, [REF(g_r80_Min)]
+        FpuCheckOpcodeCsIpDsDp { fdiv    qword [REF(g_r64_Ten)]    }, [REF(g_r64_Ten)]
+        FpuCheckFSW X86_FSW_PE | X86_FSW_UE | X86_FSW_C1, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckSt0Value 0xcccccccd, 0x0ccccccc, 0x0000
+
+        ; Denormal operand.
+        fninit
+        FpuCheckOpcodeCsIpDsDp { fld     tword [REF(g_r80_One)]    }, [REF(g_r80_One)]
+        FpuCheckOpcodeCsIpDsDp { fdiv    qword [REF(g_r64_DnMax)]  }, [REF(g_r64_DnMax)]
+        FxSaveCheckFSW xSP, X86_FSW_DE | X86_FSW_PE, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0Value xSP, 0x00000800, 0x80000000, 0x43fd
+
+        ; ## Unmasked exceptions. ##
+
+        ; Stack underflow - TOP and ST0 unmodified.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        FpuTrapOpcodeCsIpDsDp  { fdiv    qword [REF(g_r64_One)]    }, [REF(g_r64_One)]
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF | X86_FSW_B | X86_FSW_ES, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0EmptyInitValue xSP
+
+        ; Zero divide - Unmodified ST0.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        FpuCheckOpcodeCsIpDsDp { fld     dword [REF(g_r32_3dot2)]  }, [REF(g_r32_3dot2)]
+        FpuTrapOpcodeCsIpDsDp  { fdiv    qword [REF(g_r64_Zero)]   }, [REF(g_r64_Zero)]
+        FxSaveCheckFSW xSP, X86_FSW_ZE | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0ValueConst xSP, REF(g_r80_r32_3dot2)
+
+        ; Invalid Operand (Inf/Inf) - Unmodified ST0.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        FpuCheckOpcodeCsIpDsDp { fld     dword [REF(g_r32_Inf)]    }, [REF(g_r32_Inf)]
+        FpuTrapOpcodeCsIpDsDp  { fdiv    qword [REF(g_r64_Inf)]    }, [REF(g_r64_Inf)]
+        FpuCheckFSW X86_FSW_IE | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0ValueConst xSP, REF(g_r80_Inf)
+
+        ; Invalid Operand (0/0) - Unmodified ST0.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        FpuCheckOpcodeCsIpDsDp { fld     dword [REF(g_r32_Zero)]   }, [REF(g_r32_Zero)]
+        FpuTrapOpcodeCsIpDsDp  { fdiv    qword [REF(g_r64_Zero)]   }, [REF(g_r64_Zero)]
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0ValueConst xSP, REF(g_r80_Zero)
+
+        ; Precision exception, rounded down.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        FpuCheckOpcodeCsIpDsDp { fld     dword [REF(g_r32_Ten)]    }, [REF(g_r32_Ten)]
+        FpuTrapOpcodeCsIpDsDp  { fdiv    qword [REF(g_r64_Three)]  }, [REF(g_r64_Three)]
+        FxSaveCheckFSW xSP, X86_FSW_PE | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0Value_3_and_a_3rd(xSP)
+
+        ; Precision exception, rounded up.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        FpuCheckOpcodeCsIpDsDp { fld     dword [REF(g_r32_Eleven)] }, [REF(g_r32_Eleven)]
+        FpuTrapOpcodeCsIpDsDp  { fdiv    qword [REF(g_r64_Three)]  }, [REF(g_r64_Three)]
+        FxSaveCheckFSW xSP, X86_FSW_PE | X86_FSW_C1 | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0Value_3_and_two_3rds(xSP)
+
+        ; Overflow exception.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        FpuCheckOpcodeCsIpDsDp { fld     tword [REF(g_r80_Max)]    }, [REF(g_r80_Max)]
+        FpuTrapOpcodeCsIpDsDp  { fdiv    qword [REF(g_r64_0dot1)]  }, [REF(g_r64_0dot1)]
+        FxSaveCheckFSW xSP, X86_FSW_PE | X86_FSW_OE | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0Value xSP, 0xfffffd7f, 0x9fffffff, 0x2002
+
+        ; Underflow exception.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        FpuCheckOpcodeCsIpDsDp { fld     tword [REF(g_r80_Min)]    }, [REF(g_r80_Min)]
+        FpuTrapOpcodeCsIpDsDp  { fdiv    qword [REF(g_r64_Ten)]    }, [REF(g_r64_Ten)]
+        FxSaveCheckFSW xSP, X86_FSW_PE | X86_FSW_UE | X86_FSW_C1 | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0Value xSP, 0xcccccccd, 0xcccccccc, 0x5ffd
+
+        ; Denormal operand - Unmodified ST0.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        FpuCheckOpcodeCsIpDsDp { fld     tword [REF(g_r80_One)]    }, [REF(g_r80_One)]
+        FpuTrapOpcodeCsIpDsDp  { fdiv    qword [REF(g_r64_DnMax)]  }, [REF(g_r64_DnMax)]
+        FxSaveCheckFSW xSP, X86_FSW_DE | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0ValueConst xSP, REF(g_r80_One)
+
+        ;;; @todo exception priority checks.
+
+
+
+        ; ## A couple of variations on the #PF theme. ##
+
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        mov     xBX, [REF_EXTERN(g_pbEfExecPage)]
+        FpuCheckCpuTrapUnchangedState X86_XCPT_PF, fdiv qword [xBX + PAGE_SIZE]
+
+        ; Check that a pending FPU exception takes precedence over a #PF.
+        fninit
+        fdiv    qword [REF(g_r64_One)]
+        fstcw   [xSP]
+        and      word [xSP], ~(X86_FCW_IM)
+        fldcw   [xSP]
+        mov     xBX, [REF_EXTERN(g_pbEfExecPage)]
+        ShouldTrap X86_XCPT_MF, fdiv qword [xBX + PAGE_SIZE]
+
+        ;
+        ; FSUBRP STn, ST0
+        ;
+        SetSubTest "FSUBRP STn, ST0"
+
+        ; ## Normal operation. ##
+        fninit
+        FpuCheckOpcodeCsIpDsDp { fld  dword [REF(g_r32_3dot2)]     }, [REF(g_r32_3dot2)]
+        FpuCheckOpcodeCsIpDsDp { fld  dword [REF(g_r32_3dot2)]     }, [REF(g_r32_3dot2)]
+        FpuCheckOpcodeCsIp     { fsubrp st1, st0 }
+        FxSaveCheckFSW xSP, 0, 0
+        FxSaveCheckSt0ValueConst xSP, REF(g_r80_Zero)
+
+        ; ## Masked exceptions. ##
+
+        ; Masked stack underflow, both operands.
+        fninit
+        FpuCheckOpcodeCsIp     { fsubrp st1, st0 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0Value_QNaN(xSP)
+
+        ; Masked stack underflow, one operand.
+        fninit
+        FpuCheckOpcodeCsIpDsDp { fld  dword [REF(g_r32_3dot2)]     }, [REF(g_r32_3dot2)]
+        FpuCheckOpcodeCsIp     { fsubrp st1, st0 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0Value_QNaN(xSP)
+
+        ; Denormal operand.
+        fninit
+        fld    tword [REF(g_r80_DnMax)]
+        fld    tword [REF(g_r80_DnMin)]
+        FpuCheckOpcodeCsIp     { fsubrp st1, st0 }
+        FxSaveCheckFSW xSP, X86_FSW_DE, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0Value xSP, 0xfffffffe, 0x7fffffff, 0x8000
+
+        ; ## Unmasked exceptions. ##
+
+        ; Stack underflow, both operands - no pop or change.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        FpuTrapOpcodeCsIp      { fsubrp st1, st0 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0EmptyInitValue xSP
+
+        ; Stack underflow, one operand - no pop or change.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        FpuCheckOpcodeCsIpDsDp { fld  dword [REF(g_r32_3dot2)]     }, [REF(g_r32_3dot2)]
+        FpuTrapOpcodeCsIp      { fsubrp st1, st0 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0ValueConst xSP, REF(g_r80_r32_3dot2)
+
+        ; Denormal operand - no pop.
+        fninit
+        fld    tword [REF(g_r80_DnMax)]
+        fld    tword [REF(g_r80_DnMin)]
+        fnclex
+        mov     dword [xSP], X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fldcw   [xSP]
+        FpuTrapOpcodeCsIp      { fsubrp st1, st0 }
+        FxSaveCheckFSW xSP, X86_FSW_DE | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_DnMax)
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_DnMin)
+
+        ;
+        ; FSTP ST0, STn
+        ;
+        SetSubTest "FSTP ST0, STn"
+
+        ; ## Normal operation. ##
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_0dot1)]
+        fld     tword [REF(g_r80_3dot2)]
+        FpuCheckOpcodeCsIp     { fstp st2 }
+        FxSaveCheckFSW xSP, 0, 0
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_0dot1)
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_3dot2)
+
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_Max)]
+        fld     tword [REF(g_r80_Inf)]
+        FpuCheckOpcodeCsIp     { fstp st3 }
+        FxSaveCheckFSW xSP, 0, 0
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_Max)
+        FxSaveCheckStNValueConst xSP, 2, REF(g_r80_Inf)
+
+        ; Denormal register values doesn't matter get reasserted.
+        fninit
+        fld     tword [REF(g_r80_DnMin)]
+        fld     tword [REF(g_r80_DnMax)]
+        fnclex
+        mov     dword [xSP], X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fldcw   [xSP]
+        FpuCheckOpcodeCsIp     { fstp st2 }
+        FxSaveCheckFSW xSP, 0, 0
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_DnMin)
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_DnMax)
+
+        ; Signaled NaN doesn't matter.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_SNaN)]
+        fld     tword [REF(g_r80_SNaN)]
+        fnclex
+        FpuCheckOpcodeCsIp     { fstp st3 }
+        FxSaveCheckFSW xSP, 0, 0
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_SNaN)
+        FxSaveCheckStNValueConst xSP, 2, REF(g_r80_SNaN)
+
+        ; Quiet NaN doesn't matter either
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_QNaN)]
+        fld     tword [REF(g_r80_QNaN)]
+        fnclex
+        FpuCheckOpcodeCsIp     { fstp st4 }
+        FxSaveCheckFSW xSP, 0, 0
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_QNaN)
+        FxSaveCheckStNValueConst xSP, 3, REF(g_r80_QNaN)
+
+        ; There is no overflow signalled.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_SNaNMax)]
+        fld     tword [REF(g_r80_SNaNMax)]
+        fnclex
+        FpuCheckOpcodeCsIp     { fstp st1 }
+        FxSaveCheckFSW xSP, 0, 0
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_SNaNMax)
+
+        ; ## Masked exceptions. ##
+
+        ; Masked stack underflow.
+        fninit
+        FpuCheckOpcodeCsIp     { fstp st1 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0Value_QNaN(xSP)
+
+        fninit
+        FpuCheckOpcodeCsIp     { fstp st0 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0Empty xSP
+
+        ; ## Unmasked exceptions. ##
+
+        ; Stack underflow - no pop or change.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_0dot1)]
+        fld     tword [REF(g_r80_3dot2)]
+        fld     tword [REF(g_r80_Ten)]
+        ffree   st0
+        FpuTrapOpcodeCsIp      { fstp st1 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0Empty xSP
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_3dot2)
+        FxSaveCheckStNValueConst xSP, 2, REF(g_r80_0dot1)
+%endif
+
+        ;
+        ; FSTP M32R, ST0
+        ;
+        SetSubTest "FSTP M32R, ST0"
+
+        mov     xBX, [REF_EXTERN(g_pbEfExecPage)]
+        lea     xBX, [xBX + PAGE_SIZE - 4]
+
+        ; ## Normal operation. ##
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     dword [REF(g_r32_Ten)]
+        FpuCheckOpcodeCsIp     { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, 0, 0
+        FxSaveCheckSt0Empty xSP
+        CheckMemoryR32ValueConst xBX, REF(g_r32_Ten)
+
+        ; ## Masked exceptions. ##
+
+        ; Masked stack underflow.
+        fninit
+        FpuCheckOpcodeCsIp     { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryR32ValueConst xBX, REF(g_r32_NegQNaN)
+
+        fninit
+        fld     tword [REF(g_r80_0dot1)]
+        fld     tword [REF(g_r80_3dot2)]
+        fld     tword [REF(g_r80_Ten)]
+        ffree   st0
+        FpuCheckOpcodeCsIp     { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryR32ValueConst xBX, REF(g_r32_NegQNaN)
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_3dot2)
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_0dot1)
+
+        ; Masked #IA caused by SNaN.
+        fninit
+        fld     tword [REF(g_r80_SNaN)]
+        FpuCheckOpcodeCsIp     { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_IE, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryR32ValueConst xBX, REF(g_r32_QNaN)
+
+        ; Masked #U caused by a denormal value.
+        fninit
+        fld     tword [REF(g_r80_DnMin)]
+        FpuCheckOpcodeCsIp     { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_UE | X86_FSW_PE, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryR32ValueConst xBX, REF(g_r32_Zero)
+
+        ; Masked #P caused by a decimal value.
+        fninit
+        fld     tword [REF(g_r80_3dot2)]
+        FpuCheckOpcodeCsIp     { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_C1 | X86_FSW_PE, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryR32ValueConst xBX, REF(g_r32_3dot2)
+
+        ; ## Unmasked exceptions. ##
+
+        ; Stack underflow - nothing stored or popped.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        mov     dword [xBX], 0xffeeddcc
+        FpuTrapOpcodeCsIp      { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryValue dword, xBX, 0xffeeddcc
+
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_0dot1)]
+        fld     tword [REF(g_r80_3dot2)]
+        fld     tword [REF(g_r80_Ten)]
+        ffree   st0
+        mov     dword [xBX], 0xffeeddcc
+        FpuTrapOpcodeCsIp      { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryValue dword, xBX, 0xffeeddcc
+        FxSaveCheckStNEmpty      xSP, 0
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_3dot2)
+        FxSaveCheckStNValueConst xSP, 2, REF(g_r80_0dot1)
+
+        ; #IA caused by SNaN.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_SNaN)]
+        mov     dword [xBX], 0xffeeddcc
+        FpuTrapOpcodeCsIp      { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryValue dword, xBX, 0xffeeddcc
+
+        ; #U caused by a denormal value - nothing written
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_DnMin)]
+        mov     dword [xBX], 0xffeeddcc
+        FpuTrapOpcodeCsIp      { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_UE | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryValue dword, xBX, 0xffeeddcc
+
+        ; #U caused by a small value - nothing written
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_Min)]
+        mov     dword [xBX], 0xffeeddcc
+        FpuTrapOpcodeCsIp      { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_UE | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryValue dword, xBX, 0xffeeddcc
+
+        ; #O caused by a small value - nothing written
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_Max)]
+        mov     dword [xBX], 0xffeeddcc
+        FpuTrapOpcodeCsIp      { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_OE | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryValue dword, xBX, 0xffeeddcc
+
+        ; #P caused by a decimal value - rounded value is written just like if it was masked.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_3dot2)]
+        mov     dword [xBX], 0xffeeddcc
+        FpuTrapOpcodeCsIp      { fstp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_C1 | X86_FSW_PE | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryR32ValueConst xBX, REF(g_r32_3dot2)
+
+%if 0 ;; @todo implement me
+        ;
+        ; FISTP M32I, ST0
+        ;
+        SetSubTest "FISTP M32I, ST0"
+
+        mov     xBX, [REF_EXTERN(g_pbEfExecPage)]
+        lea     xBX, [xBX + PAGE_SIZE - 4]
+
+        ; ## Normal operation. ##
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_Ten)]
+        FpuCheckOpcodeCsIp     { fistp dword [xBX] }
+        FxSaveCheckFSW xSP, 0, 0
+        FxSaveCheckSt0Empty xSP
+        CheckMemoryValue dword, xBX, 10
+
+        ; ## Masked exceptions. ##
+
+        ; Masked stack underflow.
+        fninit
+        FpuCheckOpcodeCsIp     { fistp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryValue dword, xBX, 0x80000000
+
+        fninit
+        fld     tword [REF(g_r80_0dot1)]
+        fld     tword [REF(g_r80_3dot2)]
+        fld     tword [REF(g_r80_Ten)]
+        ffree   st0
+        FpuCheckOpcodeCsIp     { fistp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        CheckMemoryValue dword, xBX, 0x80000000
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_3dot2)
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_0dot1)
+
+        ; ## Unmasked exceptions. ##
+
+        ; Stack underflow - no pop or change.
+        FpuInitWithCW X86_FCW_PC_64 | X86_FCW_RC_NEAREST
+        fld     tword [REF(g_r80_0dot1)]
+        fld     tword [REF(g_r80_3dot2)]
+        fld     tword [REF(g_r80_Ten)]
+        ffree   st0
+        mov     dword [xBX], 0xffeeddcc
+        FpuTrapOpcodeCsIp      { fistp dword [xBX] }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckSt0Empty xSP
+        CheckMemoryValue dword, xBX, 0xffeeddcc
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_3dot2)
+        FxSaveCheckStNValueConst xSP, 2, REF(g_r80_0dot1)
+%endif
+%if 0
+        ;
+        ; FPTAN - calc, store ST0, push 1.0.
+        ;
+        SetSubTest "FPTAN"
+
+        ; ## Normal operation. ##
+        fninit
+        fldpi
+        FpuCheckOpcodeCsIp     { fptan }
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_One)
+        FxSaveCheckStNValue xSP, 1, 0x00000000, 0x80000000, 0x3fbf ; should be zero, so, this might fail due to precision later.
+
+        ; Masked stack underflow - two QNaNs.
+        fninit
+        FpuCheckOpcodeCsIp     { fptan }
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_NegQNaN)
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_NegQNaN)
+
+        ; Masked stack overflow - two QNaNs
+        fninit
+        fldpi
+        fldpi
+        fldpi
+        fldpi
+        fldpi
+        fldpi
+        fldpi
+        fldpi
+        FpuCheckOpcodeCsIp     { fptan }
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_NegQNaN)
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_NegQNaN)
+
+        ;; @todo Finish FPTAN testcase.
+
+        ;
+        ; FCMOVB - move if CF=1.
+        ;
+        SetSubTest "FCMOVB ST0,STn"
+
+        ; ## Normal operation. ##
+        fninit
+        fldz
+        fldpi
+        call    SetFSW_C0_thru_C3
+        stc
+        FpuCheckOpcodeCsIp     { fcmovb st0,st1 }
+        FxSaveCheckFSW xSP, X86_FSW_C0 | X86_FSW_C1 | X86_FSW_C2 | X86_FSW_C3, 0 ; seems to be preserved...
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_Zero)
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_Zero)
+
+        fninit
+        fldz
+        fld1
+        call    SetFSW_C0_thru_C3
+        clc
+        FpuCheckOpcodeCsIp     { fcmovb st0,st1 }
+        FxSaveCheckFSW xSP, X86_FSW_C0 | X86_FSW_C1 | X86_FSW_C2 | X86_FSW_C3, 0 ; seems to be preserved...
+        FxSaveCheckStNValueConst xSP, 0, REF(g_r80_One)
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_Zero)
+
+        ; ## Masked exceptions. ##
+
+        ; Masked stack underflow - both.
+        ; Note! #IE triggers regardless of the test result!
+        fninit
+        stc
+        FpuCheckOpcodeCsIp     { fcmovb st0,st1 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckStNValue_QNaN(xSP, 0)
+        FxSaveCheckStNEmpty      xSP, 1
+
+        fninit
+        clc
+        FpuCheckOpcodeCsIp     { fcmovb st0,st1 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckStNValue_QNaN(xSP, 0)
+        FxSaveCheckStNEmpty      xSP, 1
+
+        ; Masked stack underflow - source.
+        fninit
+        fldz
+        stc
+        FpuCheckOpcodeCsIp     { fcmovb st0,st1 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckStNValue_QNaN(xSP, 0)
+        FxSaveCheckStNEmpty      xSP, 1
+
+        fninit
+        fldz
+        stc
+        FpuCheckOpcodeCsIp     { fcmovb st0,st1 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckStNValue_QNaN(xSP, 0)
+        FxSaveCheckStNEmpty      xSP, 1
+
+        ; Masked stack underflow - destination.
+        fninit
+        fldz
+        fldpi
+        ffree st0
+        stc
+        FpuCheckOpcodeCsIp     { fcmovb st0,st1 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckStNValue_QNaN(xSP, 0)
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_Zero)
+
+        fninit
+        fldz
+        fldpi
+        ffree st0
+        clc
+        FpuCheckOpcodeCsIp     { fcmovb st0,st1 }
+        FxSaveCheckFSW xSP, X86_FSW_IE | X86_FSW_SF, X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3
+        FxSaveCheckStNValue_QNaN(xSP, 0)
+        FxSaveCheckStNValueConst xSP, 1, REF(g_r80_Zero)
+
+        ;; @todo Finish FCMOVB testcase.
+%endif
+
+
+.success:
+        xor     eax, eax
+.return:
+        add     xSP, 2048
+        SAVE_ALL_EPILOGUE
+        ret
+
+ENDPROC     x861_TestFPUInstr1
+
+
+
+
+;;
+; Terminate the trap info array with a NIL entry.
+BEGINDATA
+GLOBALNAME g_aTrapInfoExecPage
+istruc TRAPINFO
+        at TRAPINFO.uTrapPC,    RTCCPTR_DEF     1
+        at TRAPINFO.uResumePC,  RTCCPTR_DEF     1
+        at TRAPINFO.u8TrapNo,   db              16
+        at TRAPINFO.cbInstr,    db              3
+iend
+GLOBALNAME g_aTrapInfoEnd
+istruc TRAPINFO
+        at TRAPINFO.uTrapPC,    RTCCPTR_DEF     0
+        at TRAPINFO.uResumePC,  RTCCPTR_DEF     0
+        at TRAPINFO.u8TrapNo,   db              0
+        at TRAPINFO.cbInstr,    db              0
+iend
+
diff --git a/src/VBox/VMM/testcase/tstX86-FpuSaveRestore.cpp b/src/VBox/VMM/testcase/tstX86-FpuSaveRestore.cpp
new file mode 100644
index 00000000..d9f44346
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstX86-FpuSaveRestore.cpp
@@ -0,0 +1,116 @@
+/* $Id: tstX86-FpuSaveRestore.cpp $ */
+/** @file
+ * tstX86-FpuSaveRestore - Experimenting with saving and restoring FPU.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <iprt/initterm.h>
+#include <iprt/message.h>
+#include <iprt/string.h>
+#include <iprt/test.h>
+#include <iprt/x86.h>
+
+DECLASM(void) MyFpuPrepXcpt(void);
+DECLASM(void) MyFpuSave(PX86FXSTATE pState);
+DECLASM(void) MyFpuStoreEnv(PX86FSTENV32P pEnv);
+DECLASM(void) MyFpuRestore(PX86FXSTATE pState);
+DECLASM(void) MyFpuLoadEnv(PX86FSTENV32P pEnv);
+
+int main()
+{
+    RTTEST hTest;
+    int rc = RTTestInitAndCreate("tstX86-FpuSaveRestore", &hTest);
+    if (RT_FAILURE(rc))
+        return RTEXITCODE_FAILURE;
+    RTTestBanner(hTest);
+
+    RTTestSub(hTest, "CS/DS Selector");
+
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "Initial state (0x20 will be subtracted from IP):\n");
+    /* Trigger an exception to make sure we've got something to look at. */
+    MyFpuPrepXcpt();
+    static X86FXSTATE FxState;
+    MyFpuSave(&FxState);
+    static X86FSTENV32P FpuEnv;
+    MyFpuStoreEnv(&FpuEnv);
+#ifdef RT_ARCH_AMD64
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "  FxState IP=%#06x%04x%08x\n",  FxState.Rsrvd1, FxState.CS, FxState.FPUIP);
+#else
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "  FxState CS:IP=%#06x:%#010x\n",  FxState.CS, FxState.FPUIP);
+#endif
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "  FpuEnv  CS:IP=%#06x:%#010x\n",  FpuEnv.FPUCS, FpuEnv.FPUIP);
+
+    /* Modify the state a little so we can tell the difference. */
+    static X86FXSTATE FxState2;
+    FxState2 = FxState;
+    FxState2.FPUIP -= 0x20;
+    static X86FSTENV32P FpuEnv2;
+    FpuEnv2 = FpuEnv;
+    FpuEnv2.FPUIP  -= 0x20;
+
+    /* Just do FXRSTOR. */
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "Just FXRSTOR:\n");
+    MyFpuRestore(&FxState2);
+
+    static X86FXSTATE FxStateJustRestore;
+    MyFpuSave(&FxStateJustRestore);
+    static X86FSTENV32P FpuEnvJustRestore;
+    MyFpuStoreEnv(&FpuEnvJustRestore);
+#ifdef RT_ARCH_AMD64
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "  FxState IP=%#06x%04x%08x\n",  FxStateJustRestore.Rsrvd1, FxStateJustRestore.CS, FxStateJustRestore.FPUIP);
+#else
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "  FxState CS:IP=%#06x:%#010x\n",  FxStateJustRestore.CS, FxStateJustRestore.FPUIP);
+#endif
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "  FpuEnv  CS:IP=%#06x:%#010x\n",  FpuEnvJustRestore.FPUCS, FpuEnvJustRestore.FPUIP);
+
+
+    /* FXRSTORE + FLDENV */
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "FXRSTOR first, then FLDENV:\n");
+    MyFpuRestore(&FxState2);
+    MyFpuLoadEnv(&FpuEnv2);
+
+    static X86FXSTATE FxStateRestoreLoad;
+    MyFpuSave(&FxStateRestoreLoad);
+    static X86FSTENV32P FpuEnvRestoreLoad;
+    MyFpuStoreEnv(&FpuEnvRestoreLoad);
+#ifdef RT_ARCH_AMD64
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "  FxState IP=%#06x%04x%08x\n",  FxStateRestoreLoad.Rsrvd1, FxStateRestoreLoad.CS, FxStateRestoreLoad.FPUIP);
+#else
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "  FxState CS:IP=%#06x:%#010x\n",  FxStateRestoreLoad.CS, FxStateRestoreLoad.FPUIP);
+#endif
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "  FpuEnv  CS:IP=%#06x:%#010x\n",  FpuEnvRestoreLoad.FPUCS, FpuEnvRestoreLoad.FPUIP);
+
+    /* Reverse the order (FLDENV + FXRSTORE). */
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "FLDENV first, then FXRSTOR:\n");
+    MyFpuLoadEnv(&FpuEnv2);
+    MyFpuRestore(&FxState2);
+
+    static X86FXSTATE FxStateLoadRestore;
+    MyFpuSave(&FxStateLoadRestore);
+    static X86FSTENV32P FpuEnvLoadRestore;
+    MyFpuStoreEnv(&FpuEnvLoadRestore);
+#ifdef RT_ARCH_AMD64
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "  FxState IP=%#06x%04x%08x\n",  FxStateLoadRestore.Rsrvd1, FxStateLoadRestore.CS, FxStateLoadRestore.FPUIP);
+#else
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "  FxState CS:IP=%#06x:%#010x\n",  FxStateLoadRestore.CS, FxStateLoadRestore.FPUIP);
+#endif
+    RTTestIPrintf(RTTESTLVL_ALWAYS,  "  FpuEnv  CS:IP=%#06x:%#010x\n",  FpuEnvLoadRestore.FPUCS, FpuEnvLoadRestore.FPUIP);
+
+
+    return RTTestSummaryAndDestroy(hTest);
+}
diff --git a/src/VBox/VMM/testcase/tstX86-FpuSaveRestoreA.asm b/src/VBox/VMM/testcase/tstX86-FpuSaveRestoreA.asm
new file mode 100644
index 00000000..d4f989b9
--- /dev/null
+++ b/src/VBox/VMM/testcase/tstX86-FpuSaveRestoreA.asm
@@ -0,0 +1,117 @@
+; $Id: tstX86-FpuSaveRestoreA.asm $
+;; @file
+; tstX86-FpuSaveRestore - Experimenting with saving and restoring FPU, assembly bits.
+;
+
+;
+; Copyright (C) 2013-2020 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+
+
+;*******************************************************************************
+;*  Header Files                                                               *
+;*******************************************************************************
+%include "iprt/asmdefs.mac"
+%include "iprt/x86.mac"
+
+
+;*******************************************************************************
+;*  Global Variables                                                           *
+;*******************************************************************************
+BEGINCODE
+g_r80_Zero:     dt 0.0
+g_r80_One:      dt 1.0
+
+
+BEGINCODE
+
+;; Prepares a FPU exception.
+BEGINPROC MyFpuPrepXcpt
+        fld tword [g_r80_One  xWrtRIP]
+        fld tword [g_r80_Zero xWrtRIP]
+        fdiv    st0
+        ret
+ENDPROC   MyFpuPrepXcpt
+
+
+;; Same as above, just different address.
+BEGINPROC MyFpuPrepXcpt2
+        fld tword [g_r80_One  xWrtRIP]
+        fld tword [g_r80_Zero xWrtRIP]
+        fdiv    st0
+        ret
+ENDPROC   MyFpuPrepXcpt2
+
+
+BEGINPROC MyFpuSave
+%ifdef ASM_CALL64_MSC
+        o64 fxsave  [rcx]
+%elifdef ASM_CALL64_GCC
+        o64 fxsave  [rdi]
+%elifdef RT_ARCH_X86
+        mov         ecx, [esp + 4]
+        fxsave      [ecx]
+%else
+ %error "Unsupported architecture."
+        bad arch
+%endif
+        ret
+ENDPROC   MyFpuSave
+
+
+BEGINPROC MyFpuStoreEnv
+%ifdef ASM_CALL64_MSC
+        fstenv  [rcx]
+%elifdef ASM_CALL64_GCC
+        fstenv  [rdi]
+%elifdef RT_ARCH_X86
+        mov     ecx, [esp + 4]
+        fstenv  [ecx]
+%else
+ %error "Unsupported architecture."
+        bad arch
+%endif
+        ret
+ENDPROC   MyFpuStoreEnv
+
+
+BEGINPROC MyFpuRestore
+%ifdef ASM_CALL64_MSC
+        o64 fxrstor [rcx]
+%elifdef ASM_CALL64_GCC
+        o64 fxrstor [rdi]
+%elifdef RT_ARCH_X86
+        mov         ecx, [esp + 4]
+        fxrstor     [ecx]
+%else
+ %error "Unsupported architecture."
+        bad arch
+%endif
+        ret
+ENDPROC   MyFpuRestore
+
+
+BEGINPROC MyFpuLoadEnv
+%ifdef ASM_CALL64_MSC
+        fldenv  [rcx]
+%elifdef ASM_CALL64_GCC
+        fldenv  [rdi]
+%elifdef RT_ARCH_X86
+        mov     ecx, [esp + 4]
+        fldenv  [ecx]
+%else
+ %error "Unsupported architecture."
+        bad arch
+%endif
+        ret
+ENDPROC   MyFpuLoadEnv
+
diff --git a/src/VBox/VMM/tools/Makefile.kmk b/src/VBox/VMM/tools/Makefile.kmk
new file mode 100644
index 00000000..8fecc54d
--- /dev/null
+++ b/src/VBox/VMM/tools/Makefile.kmk
@@ -0,0 +1,76 @@
+# $Id: Makefile.kmk $
+## @file
+# Sub-Makefile for VMM tools.
+#
+
+#
+# Copyright (C) 2006-2020 Oracle Corporation
+#
+# This file is part of VirtualBox Open Source Edition (OSE), as
+# available from http://www.virtualbox.org. This file is free software;
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License (GPL) as published by the Free Software
+# Foundation, in version 2 as it comes in the "COPYING" file of the
+# VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+# hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+#
+
+SUB_DEPTH = ../../../..
+include $(KBUILD_PATH)/subheader.kmk
+
+
+#
+# Targets.
+#
+ifdef VBOX_WITH_HARDENING
+PROGRAMS += VBoxVMMPreloadHardened
+DLLS     += VBoxVMMPreload
+else
+PROGRAMS += VBoxVMMPreload
+endif
+
+
+#
+# Hardened stub.
+#
+VBoxVMMPreloadHardened_TEMPLATE = VBOXR3HARDENEDEXE
+VBoxVMMPreloadHardened_SOURCES  = VBoxVMMPreloadHardened.cpp
+VBoxVMMPreloadHardened_NAME     = VBoxVMMPreload
+
+
+#
+# The real thing
+#
+VBoxVMMPreload_TEMPLATE  := $(if $(VBOX_WITH_HARDENING),VBoxR3Dll,VBOXR3EXE)
+VBoxVMMPreload_SOURCES    = VBoxVMMPreload.cpp
+VBoxVMMPreload_LIBS = $(LIB_RUNTIME)
+ifdef VBOX_WITH_HARDENING
+VBoxVMMPreload_LDFLAGS.darwin += -install_name $(VBOX_DYLD_EXECUTABLE_PATH)/VBoxVMMPreload.dylib
+endif
+
+
+#
+# CPU report program (CPUM DB).
+#
+PROGRAMS += VBoxCpuReport
+VBoxCpuReport_TEMPLATE := VBoxR3Static
+VBoxCpuReport_DEFS      = VBOX_IN_VMM IN_VMM_R3 IN_VBOX_CPU_REPORT $(VMM_COMMON_DEFS)
+VBoxCpuReport_INCS      = ../include
+VBoxCpuReport_SOURCES   = \
+	VBoxCpuReport.cpp \
+	VBoxCpuReportMsrSup.cpp \
+	../VMMR3/CPUMR3CpuId.cpp
+VBoxCpuReport_SOURCES.linux = \
+	VBoxCpuReportMsrLinux.cpp
+VBoxCpuReport_DEFS.linux    = VBCR_HAVE_PLATFORM_MSR_PROBER
+VBoxCpuReport_LIBS      = \
+	$(PATH_STAGE_LIB)/SUPR3Static$(VBOX_SUFF_LIB) \
+       $(VBOX_LIB_RUNTIME_STATIC)
+VBoxCpuReport_LIBS.solaris = kstat contract
+
+VBoxCpuReport_LDFLAGS.darwin = \
+	-framework IOKit -framework CoreFoundation -framework CoreServices
+
+
+include $(FILE_KBUILD_SUB_FOOTER)
+
diff --git a/src/VBox/VMM/tools/VBoxCpuReport.cpp b/src/VBox/VMM/tools/VBoxCpuReport.cpp
new file mode 100644
index 00000000..b549495b
--- /dev/null
+++ b/src/VBox/VMM/tools/VBoxCpuReport.cpp
@@ -0,0 +1,4989 @@
+/* $Id: VBoxCpuReport.cpp $ */
+/** @file
+ * VBoxCpuReport - Produces the basis for a CPU DB entry.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/buildconfig.h>
+#include <iprt/ctype.h>
+#include <iprt/file.h>
+#include <iprt/getopt.h>
+#include <iprt/initterm.h>
+#include <iprt/message.h>
+#include <iprt/mem.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+#include <iprt/stream.h>
+#include <iprt/symlink.h>
+#include <iprt/thread.h>
+#include <iprt/time.h>
+
+#include <VBox/err.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/sup.h>
+
+#include "VBoxCpuReport.h"
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/** Write only register. */
+#define VBCPUREPMSR_F_WRITE_ONLY      RT_BIT(0)
+
+typedef struct VBCPUREPMSR
+{
+    /** The first MSR register number. */
+    uint32_t        uMsr;
+    /** Flags (MSRREPORT_F_XXX). */
+    uint32_t        fFlags;
+    /** The value we read, unless write-only.  */
+    uint64_t        uValue;
+} VBCPUREPMSR;
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** The CPU vendor.  Used by the MSR code. */
+static CPUMCPUVENDOR    g_enmVendor = CPUMCPUVENDOR_INVALID;
+/** The CPU microarchitecture.  Used by the MSR code. */
+static CPUMMICROARCH    g_enmMicroarch = kCpumMicroarch_Invalid;
+/** Set if g_enmMicroarch indicates an Intel NetBurst CPU. */
+static bool             g_fIntelNetBurst = false;
+/** The alternative report stream. */
+static PRTSTREAM        g_pReportOut;
+/** The alternative debug stream. */
+static PRTSTREAM        g_pDebugOut;
+/** Whether to skip MSR collection.   */
+static bool             g_fNoMsrs = false;
+
+/** Snooping info storage for vbCpuRepGuessScalableBusFrequencyName. */
+static uint64_t         g_uMsrIntelP6FsbFrequency = UINT64_MAX;
+
+/** The MSR accessors interface. */
+static VBCPUREPMSRACCESSORS g_MsrAcc;
+
+
+
+void vbCpuRepDebug(const char *pszMsg, ...)
+{
+    va_list va;
+
+    /* Always print a copy of the report to standard error. */
+    va_start(va, pszMsg);
+    RTStrmPrintfV(g_pStdErr, pszMsg, va);
+    va_end(va);
+    RTStrmFlush(g_pStdErr);
+
+    /* Alternatively, also print to a log file. */
+    if (g_pDebugOut)
+    {
+        va_start(va, pszMsg);
+        RTStrmPrintfV(g_pDebugOut, pszMsg, va);
+        va_end(va);
+        RTStrmFlush(g_pDebugOut);
+    }
+
+    /* Give the output device a chance to write / display it. */
+    RTThreadSleep(1);
+}
+
+
+void vbCpuRepPrintf(const char *pszMsg, ...)
+{
+    va_list va;
+
+    /* Output to report file, if requested. */
+    if (g_pReportOut)
+    {
+        va_start(va, pszMsg);
+        RTStrmPrintfV(g_pReportOut, pszMsg, va);
+        va_end(va);
+        RTStrmFlush(g_pReportOut);
+    }
+
+    /* Always print a copy of the report to standard out. */
+    va_start(va, pszMsg);
+    RTStrmPrintfV(g_pStdOut, pszMsg, va);
+    va_end(va);
+    RTStrmFlush(g_pStdOut);
+}
+
+
+
+static int vbCpuRepMsrsAddOne(VBCPUREPMSR **ppaMsrs, uint32_t *pcMsrs,
+                              uint32_t uMsr, uint64_t uValue, uint32_t fFlags)
+{
+    /*
+     * Grow the array?
+     */
+    uint32_t cMsrs = *pcMsrs;
+    if ((cMsrs % 64) == 0)
+    {
+        void *pvNew = RTMemRealloc(*ppaMsrs, (cMsrs + 64) * sizeof(**ppaMsrs));
+        if (!pvNew)
+        {
+            RTMemFree(*ppaMsrs);
+            *ppaMsrs = NULL;
+            *pcMsrs  = 0;
+            return VERR_NO_MEMORY;
+        }
+        *ppaMsrs = (VBCPUREPMSR *)pvNew;
+    }
+
+    /*
+     * Add it.
+     */
+    VBCPUREPMSR *pEntry = *ppaMsrs + cMsrs;
+    pEntry->uMsr   = uMsr;
+    pEntry->fFlags = fFlags;
+    pEntry->uValue = uValue;
+    *pcMsrs = cMsrs + 1;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Returns the max physical address width as a number of bits.
+ *
+ * @returns Bit count.
+ */
+static uint8_t vbCpuRepGetPhysAddrWidth(void)
+{
+    uint8_t  cMaxWidth;
+    if (!ASMHasCpuId())
+        cMaxWidth = 32;
+    else
+    {
+        uint32_t cMaxExt = ASMCpuId_EAX(0x80000000);
+        if (ASMIsValidExtRange(cMaxExt)&& cMaxExt >= 0x80000008)
+            cMaxWidth = ASMCpuId_EAX(0x80000008) & 0xff;
+        else if (   ASMIsValidStdRange(ASMCpuId_EAX(0))
+                 && (ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_PSE36))
+            cMaxWidth = 36;
+        else
+            cMaxWidth = 32;
+    }
+    return cMaxWidth;
+}
+
+
+static bool vbCpuRepSupportsPae(void)
+{
+    return ASMHasCpuId()
+        && ASMIsValidStdRange(ASMCpuId_EAX(0))
+        && (ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_PAE);
+}
+
+
+static bool vbCpuRepSupportsLongMode(void)
+{
+    return ASMHasCpuId()
+        && ASMIsValidExtRange(ASMCpuId_EAX(0x80000000))
+        && (ASMCpuId_EDX(0x80000001) & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE);
+}
+
+
+static bool vbCpuRepSupportsNX(void)
+{
+    return ASMHasCpuId()
+        && ASMIsValidExtRange(ASMCpuId_EAX(0x80000000))
+        && (ASMCpuId_EDX(0x80000001) & X86_CPUID_EXT_FEATURE_EDX_NX);
+}
+
+
+static bool vbCpuRepSupportsX2Apic(void)
+{
+    return ASMHasCpuId()
+        && ASMIsValidStdRange(ASMCpuId_EAX(0))
+        && (ASMCpuId_ECX(1) & X86_CPUID_FEATURE_ECX_X2APIC);
+}
+
+
+
+#if 0 /* unused */
+static bool msrProberWrite(uint32_t uMsr, uint64_t uValue)
+{
+    bool fGp;
+    int rc = g_MsrAcc.pfnMsrWrite(uMsr, NIL_RTCPUID, uValue, &fGp);
+    AssertRC(rc);
+    return RT_SUCCESS(rc) && !fGp;
+}
+#endif
+
+
+static bool msrProberRead(uint32_t uMsr, uint64_t *puValue)
+{
+    *puValue = 0;
+    bool fGp;
+    int rc = g_MsrAcc.pfnMsrProberRead(uMsr, NIL_RTCPUID, puValue, &fGp);
+    AssertRC(rc);
+    return RT_SUCCESS(rc) && !fGp;
+}
+
+
+/** Tries to modify the register by writing the original value to it. */
+static bool msrProberModifyNoChange(uint32_t uMsr)
+{
+    SUPMSRPROBERMODIFYRESULT Result;
+    int rc = g_MsrAcc.pfnMsrProberModify(uMsr, NIL_RTCPUID, UINT64_MAX, 0, &Result);
+    return RT_SUCCESS(rc)
+        && !Result.fBeforeGp
+        && !Result.fModifyGp
+        && !Result.fAfterGp
+        && !Result.fRestoreGp;
+}
+
+
+/** Tries to modify the register by writing zero to it. */
+static bool msrProberModifyZero(uint32_t uMsr)
+{
+    SUPMSRPROBERMODIFYRESULT Result;
+    int rc = g_MsrAcc.pfnMsrProberModify(uMsr, NIL_RTCPUID, 0, 0, &Result);
+    return RT_SUCCESS(rc)
+        && !Result.fBeforeGp
+        && !Result.fModifyGp
+        && !Result.fAfterGp
+        && !Result.fRestoreGp;
+}
+
+
+/**
+ * Tries to modify each bit in the MSR and see if we can make it change.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR.
+ * @param   pfIgnMask           The ignore mask to update.
+ * @param   pfGpMask            The GP mask to update.
+ * @param   fSkipMask           Mask of bits to skip.
+ */
+static int msrProberModifyBitChanges(uint32_t uMsr, uint64_t *pfIgnMask, uint64_t *pfGpMask, uint64_t fSkipMask)
+{
+    for (unsigned iBit = 0; iBit < 64; iBit++)
+    {
+        uint64_t fBitMask = RT_BIT_64(iBit);
+        if (fBitMask & fSkipMask)
+            continue;
+
+        /* Set it. */
+        SUPMSRPROBERMODIFYRESULT ResultSet;
+        int rc = g_MsrAcc.pfnMsrProberModify(uMsr, NIL_RTCPUID, ~fBitMask, fBitMask, &ResultSet);
+        if (RT_FAILURE(rc))
+            return RTMsgErrorRc(rc, "pfnMsrProberModify(%#x,,%#llx,%#llx,): %Rrc", uMsr, ~fBitMask, fBitMask, rc);
+
+        /* Clear it. */
+        SUPMSRPROBERMODIFYRESULT ResultClear;
+        rc = g_MsrAcc.pfnMsrProberModify(uMsr, NIL_RTCPUID, ~fBitMask, 0, &ResultClear);
+        if (RT_FAILURE(rc))
+            return RTMsgErrorRc(rc, "pfnMsrProberModify(%#x,,%#llx,%#llx,): %Rrc", uMsr, ~fBitMask, 0, rc);
+
+        if (ResultSet.fModifyGp || ResultClear.fModifyGp)
+            *pfGpMask |= fBitMask;
+        else if (   (   ((ResultSet.uBefore   ^ ResultSet.uAfter)   & fBitMask) == 0
+                     && !ResultSet.fBeforeGp
+                     && !ResultSet.fAfterGp)
+                 && (   ((ResultClear.uBefore ^ ResultClear.uAfter) & fBitMask) == 0
+                     && !ResultClear.fBeforeGp
+                     && !ResultClear.fAfterGp) )
+            *pfIgnMask |= fBitMask;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+#if 0 /* currently unused */
+/**
+ * Tries to modify one bit.
+ *
+ * @retval  -2 on API error.
+ * @retval  -1 on \#GP.
+ * @retval  0 if ignored.
+ * @retval  1 if it changed.
+ *
+ * @param   uMsr                The MSR.
+ * @param   iBit                The bit to try modify.
+ */
+static int msrProberModifyBit(uint32_t uMsr, unsigned iBit)
+{
+    uint64_t fBitMask = RT_BIT_64(iBit);
+
+    /* Set it. */
+    SUPMSRPROBERMODIFYRESULT ResultSet;
+    int rc = g_MsrAcc.pfnMsrProberModify(uMsr, NIL_RTCPUID, ~fBitMask, fBitMask, &ResultSet);
+    if (RT_FAILURE(rc))
+        return RTMsgErrorRc(-2, "pfnMsrProberModify(%#x,,%#llx,%#llx,): %Rrc", uMsr, ~fBitMask, fBitMask, rc);
+
+    /* Clear it. */
+    SUPMSRPROBERMODIFYRESULT ResultClear;
+    rc = g_MsrAcc.pfnMsrProberModify(uMsr, NIL_RTCPUID, ~fBitMask, 0, &ResultClear);
+    if (RT_FAILURE(rc))
+        return RTMsgErrorRc(-2, "pfnMsrProberModify(%#x,,%#llx,%#llx,): %Rrc", uMsr, ~fBitMask, 0, rc);
+
+    if (ResultSet.fModifyGp || ResultClear.fModifyGp)
+        return -1;
+
+    if (   (   ((ResultSet.uBefore   ^ ResultSet.uAfter)   & fBitMask) != 0
+            && !ResultSet.fBeforeGp
+            && !ResultSet.fAfterGp)
+        || (   ((ResultClear.uBefore ^ ResultClear.uAfter) & fBitMask) != 0
+            && !ResultClear.fBeforeGp
+            && !ResultClear.fAfterGp) )
+        return 1;
+
+    return 0;
+}
+#endif
+
+
+/**
+ * Tries to do a simple AND+OR change and see if we \#GP or not.
+ *
+ * @retval  @c true if successfully modified.
+ * @retval  @c false if \#GP or other error.
+ *
+ * @param   uMsr                The MSR.
+ * @param   fAndMask            The AND mask.
+ * @param   fOrMask             The OR mask.
+ */
+static bool msrProberModifySimpleGp(uint32_t uMsr, uint64_t fAndMask, uint64_t fOrMask)
+{
+    SUPMSRPROBERMODIFYRESULT Result;
+    int rc = g_MsrAcc.pfnMsrProberModify(uMsr, NIL_RTCPUID, fAndMask, fOrMask, &Result);
+    if (RT_FAILURE(rc))
+    {
+        RTMsgError("g_MsrAcc.pfnMsrProberModify(%#x,,%#llx,%#llx,): %Rrc", uMsr, fAndMask, fOrMask, rc);
+        return false;
+    }
+    return !Result.fBeforeGp
+        && !Result.fModifyGp
+        && !Result.fAfterGp
+        && !Result.fRestoreGp;
+}
+
+
+
+
+/**
+ * Combination of the basic tests.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR.
+ * @param   fSkipMask           Mask of bits to skip.
+ * @param   pfReadOnly          Where to return read-only status.
+ * @param   pfIgnMask           Where to return the write ignore mask.  Need not
+ *                              be initialized.
+ * @param   pfGpMask            Where to return the write GP mask.  Need not
+ *                              be initialized.
+ */
+static int msrProberModifyBasicTests(uint32_t uMsr, uint64_t fSkipMask, bool *pfReadOnly, uint64_t *pfIgnMask, uint64_t *pfGpMask)
+{
+    if (msrProberModifyNoChange(uMsr))
+    {
+        *pfReadOnly = false;
+        *pfIgnMask  = 0;
+        *pfGpMask  = 0;
+        return msrProberModifyBitChanges(uMsr, pfIgnMask, pfGpMask, fSkipMask);
+    }
+
+    *pfReadOnly = true;
+    *pfIgnMask  = 0;
+    *pfGpMask   = UINT64_MAX;
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Determines for the MSR AND mask.
+ *
+ * Older CPUs doesn't necessiarly implement all bits of the MSR register number.
+ * So, we have to approximate how many are used so we don't get an overly large
+ * and confusing set of MSRs when probing.
+ *
+ * @returns The mask.
+ */
+static uint32_t determineMsrAndMask(void)
+{
+#define VBCPUREP_MASK_TEST_MSRS     7
+    static uint32_t const s_aMsrs[VBCPUREP_MASK_TEST_MSRS] =
+    {
+        /* Try a bunch of mostly read only registers: */
+        MSR_P5_MC_TYPE, MSR_IA32_PLATFORM_ID, MSR_IA32_MTRR_CAP, MSR_IA32_MCG_CAP, MSR_IA32_CR_PAT,
+        /* Then some which aren't supposed to be present on any CPU: */
+        0x00000015, 0x00000019,
+    };
+
+    /* Get the base values. */
+    uint64_t auBaseValues[VBCPUREP_MASK_TEST_MSRS];
+    for (unsigned i = 0; i < RT_ELEMENTS(s_aMsrs); i++)
+    {
+        if (!msrProberRead(s_aMsrs[i], &auBaseValues[i]))
+            auBaseValues[i] = UINT64_MAX;
+        //vbCpuRepDebug("Base: %#x -> %#llx\n", s_aMsrs[i], auBaseValues[i]);
+    }
+
+    /* Do the probing. */
+    unsigned iBit;
+    for (iBit = 31; iBit > 8; iBit--)
+    {
+        uint64_t fMsrOrMask = RT_BIT_64(iBit);
+        for (unsigned iTest = 0; iTest <= 64 && fMsrOrMask < UINT32_MAX; iTest++)
+        {
+            for (unsigned i = 0; i < RT_ELEMENTS(s_aMsrs); i++)
+            {
+                uint64_t uValue;
+                if (!msrProberRead(s_aMsrs[i] | fMsrOrMask, &uValue))
+                    uValue = UINT64_MAX;
+                if (uValue != auBaseValues[i])
+                {
+                    uint32_t fMsrMask = iBit >= 31 ? UINT32_MAX : RT_BIT_32(iBit + 1) - 1;
+                    vbCpuRepDebug("MSR AND mask: quit on iBit=%u uMsr=%#x (%#x) %llx != %llx => fMsrMask=%#x\n",
+                                  iBit, s_aMsrs[i] | (uint32_t)fMsrOrMask, s_aMsrs[i], uValue, auBaseValues[i], fMsrMask);
+                    return fMsrMask;
+                }
+            }
+
+            /* Advance. */
+            if (iBit <= 6)
+                fMsrOrMask += RT_BIT_64(iBit);
+            else if (iBit <= 11)
+                fMsrOrMask += RT_BIT_64(iBit) * 33;
+            else if (iBit <= 16)
+                fMsrOrMask += RT_BIT_64(iBit) * 1025;
+            else if (iBit <= 22)
+                fMsrOrMask += RT_BIT_64(iBit) * 65537;
+            else
+                fMsrOrMask += RT_BIT_64(iBit) * 262145;
+        }
+    }
+
+    uint32_t fMsrMask = RT_BIT_32(iBit + 1) - 1;
+    vbCpuRepDebug("MSR AND mask: less that %u bits that matters?!? => fMsrMask=%#x\n", iBit + 1, fMsrMask);
+    return fMsrMask;
+}
+
+
+static int findMsrs(VBCPUREPMSR **ppaMsrs, uint32_t *pcMsrs, uint32_t fMsrMask)
+{
+    /*
+     * Gather them.
+     */
+    static struct { uint32_t uFirst, cMsrs; } const s_aRanges[] =
+    {
+        { 0x00000000, 0x00042000 },
+        { 0x10000000, 0x00001000 },
+        { 0x20000000, 0x00001000 },
+        { 0x40000000, 0x00012000 },
+        { 0x80000000, 0x00012000 },
+        { 0xc0000000, 0x00022000 }, /* Had some trouble here on solaris with the tstVMM setup. */
+    };
+
+    *pcMsrs  = 0;
+    *ppaMsrs = NULL;
+
+    for (unsigned i = 0; i < RT_ELEMENTS(s_aRanges); i++)
+    {
+        uint32_t uMsr  = s_aRanges[i].uFirst;
+        if ((uMsr & fMsrMask) != uMsr)
+            continue;
+        uint32_t cLeft = s_aRanges[i].cMsrs;
+        while (cLeft-- > 0 && (uMsr & fMsrMask) == uMsr)
+        {
+            if ((uMsr & 0xfff) == 0)
+            {
+                vbCpuRepDebug("testing %#x...\n", uMsr);
+                RTThreadSleep(22);
+            }
+#if 0
+            else if (uMsr >= 0x00003170 && uMsr <= 0xc0000090)
+            {
+                vbCpuRepDebug("testing %#x...\n", uMsr);
+                RTThreadSleep(250);
+            }
+#endif
+            /* Skip 0xc0011012..13 as it seems to be bad for our health (Phenom II X6 1100T). */
+            /* Ditto for 0x0000002a (EBL_CR_POWERON) and 0x00000277 (MSR_IA32_CR_PAT) on Intel (Atom 330). */
+            /* And more of the same for 0x280 on Intel Pentium III. */
+            if (   ((uMsr >= 0xc0011012 && uMsr <= 0xc0011013) && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+                || (   (uMsr == 0x2a || uMsr == 0x277)
+                    && g_enmVendor == CPUMCPUVENDOR_INTEL
+                    && g_enmMicroarch == kCpumMicroarch_Intel_Atom_Bonnell)
+                || (   (uMsr == 0x280)
+                    && g_enmMicroarch == kCpumMicroarch_Intel_P6_III))
+                vbCpuRepDebug("Skipping %#x\n", uMsr);
+            else
+            {
+                /* Read probing normally does it. */
+                uint64_t uValue = 0;
+                bool     fGp    = true;
+                int rc = g_MsrAcc.pfnMsrProberRead(uMsr, NIL_RTCPUID, &uValue, &fGp);
+                if (RT_FAILURE(rc))
+                {
+                    RTMemFree(*ppaMsrs);
+                    *ppaMsrs = NULL;
+                    return RTMsgErrorRc(rc, "pfnMsrProberRead failed on %#x: %Rrc\n", uMsr, rc);
+                }
+
+                uint32_t fFlags;
+                if (!fGp)
+                    fFlags = 0;
+                /* VIA/Shanghai HACK - writing to 0x0000317e on a quad core make the core unresponsive. */
+                else if (uMsr == 0x0000317e && (g_enmVendor == CPUMCPUVENDOR_VIA || g_enmVendor == CPUMCPUVENDOR_SHANGHAI))
+                {
+                    uValue = 0;
+                    fFlags = VBCPUREPMSR_F_WRITE_ONLY;
+                    fGp    = *pcMsrs == 0
+                          || (*ppaMsrs)[*pcMsrs - 1].uMsr != 0x0000317d
+                          || (*ppaMsrs)[*pcMsrs - 1].fFlags != VBCPUREPMSR_F_WRITE_ONLY;
+                }
+                else
+                {
+                    /* Is it a write only register? */
+#if 0
+                    if (uMsr >= 0x00003170 && uMsr <= 0xc0000090)
+                    {
+                        vbCpuRepDebug("test writing %#x...\n", uMsr);
+                        RTThreadSleep(250);
+                    }
+#endif
+                    fGp = true;
+                    rc = g_MsrAcc.pfnMsrProberWrite(uMsr, NIL_RTCPUID, 0, &fGp);
+                    if (RT_FAILURE(rc))
+                    {
+                        RTMemFree(*ppaMsrs);
+                        *ppaMsrs = NULL;
+                        return RTMsgErrorRc(rc, "pfnMsrProberWrite failed on %#x: %Rrc\n", uMsr, rc);
+                    }
+                    uValue = 0;
+                    fFlags = VBCPUREPMSR_F_WRITE_ONLY;
+
+                    /*
+                     * Tweaks.  On Intel CPUs we've got trouble detecting
+                     * IA32_BIOS_UPDT_TRIG (0x00000079), so we have to add it manually here.
+                     * Ditto on AMD with PATCH_LOADER (0xc0010020).
+                     */
+                    if (   uMsr == 0x00000079
+                        && fGp
+                        && g_enmMicroarch >= kCpumMicroarch_Intel_P6_Core_Atom_First
+                        && g_enmMicroarch <= kCpumMicroarch_Intel_End)
+                        fGp = false;
+                    if (   uMsr == 0xc0010020
+                        && fGp
+                        && g_enmMicroarch >= kCpumMicroarch_AMD_K8_First
+                        && g_enmMicroarch <= kCpumMicroarch_AMD_End)
+                        fGp = false;
+                }
+
+                if (!fGp)
+                {
+                    /* Add it. */
+                    rc = vbCpuRepMsrsAddOne(ppaMsrs, pcMsrs, uMsr, uValue, fFlags);
+                    if (RT_FAILURE(rc))
+                        return RTMsgErrorRc(rc, "Out of memory (uMsr=%#x).\n", uMsr);
+                    if (   (g_enmVendor != CPUMCPUVENDOR_VIA && g_enmVendor != CPUMCPUVENDOR_SHANGHAI)
+                        || uValue
+                        || fFlags)
+                        vbCpuRepDebug("%#010x: uValue=%#llx fFlags=%#x\n", uMsr, uValue, fFlags);
+                }
+            }
+
+            uMsr++;
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Get the name of the specified MSR, if we know it and can handle it.
+ *
+ * Do _NOT_ add any new names here without ALSO at the SAME TIME making sure it
+ * is handled correctly by the PROBING CODE and REPORTED correctly!!
+ *
+ * @returns Pointer to name if handled, NULL if not yet explored.
+ * @param   uMsr                The MSR in question.
+ */
+static const char *getMsrNameHandled(uint32_t uMsr)
+{
+    /** @todo figure out where NCU_EVENT_CORE_MASK might be... */
+    switch (uMsr)
+    {
+        case 0x00000000: return "IA32_P5_MC_ADDR";
+        case 0x00000001: return "IA32_P5_MC_TYPE";
+        case 0x00000006:
+            if (g_enmMicroarch >= kCpumMicroarch_Intel_First && g_enmMicroarch <= kCpumMicroarch_Intel_P6_Core_Atom_First)
+                return NULL; /* TR4 / cache tag on Pentium, but that's for later. */
+            return "IA32_MONITOR_FILTER_LINE_SIZE";
+        //case 0x0000000e: return "P?_TR12"; /* K6-III docs */
+        case 0x00000010: return "IA32_TIME_STAMP_COUNTER";
+        case 0x00000017: return "IA32_PLATFORM_ID";
+        case 0x00000018: return "P6_UNK_0000_0018"; /* P6_M_Dothan. */
+        case 0x0000001b: return "IA32_APIC_BASE";
+        case 0x00000021: return "C2_UNK_0000_0021"; /* Core2_Penryn */
+        case 0x0000002a: return g_fIntelNetBurst ? "P4_EBC_HARD_POWERON" : "EBL_CR_POWERON";
+        case 0x0000002b: return g_fIntelNetBurst ? "P4_EBC_SOFT_POWERON" : NULL;
+        case 0x0000002c: return g_fIntelNetBurst ? "P4_EBC_FREQUENCY_ID" : NULL;
+        case 0x0000002e: return "I7_UNK_0000_002e"; /* SandyBridge, IvyBridge. */
+        case 0x0000002f: return "P6_UNK_0000_002f"; /* P6_M_Dothan. */
+        case 0x00000032: return "P6_UNK_0000_0032"; /* P6_M_Dothan. */
+        case 0x00000033: return "TEST_CTL";
+        case 0x00000034: return CPUMMICROARCH_IS_INTEL_CORE7(g_enmMicroarch)
+                             || CPUMMICROARCH_IS_INTEL_SILVERMONT_PLUS(g_enmMicroarch)
+                              ? "MSR_SMI_COUNT" : "P6_UNK_0000_0034"; /* P6_M_Dothan. */
+        case 0x00000035: return CPUMMICROARCH_IS_INTEL_CORE7(g_enmMicroarch) ? "MSR_CORE_THREAD_COUNT" : "P6_UNK_0000_0035"; /* P6_M_Dothan. */
+        case 0x00000036: return "I7_UNK_0000_0036"; /* SandyBridge, IvyBridge. */
+        case 0x00000039: return "C2_UNK_0000_0039"; /* Core2_Penryn */
+        case 0x0000003a: return "IA32_FEATURE_CONTROL";
+        case 0x0000003b: return "P6_UNK_0000_003b"; /* P6_M_Dothan. */
+        case 0x0000003e: return "I7_UNK_0000_003e"; /* SandyBridge, IvyBridge. */
+        case 0x0000003f: return "P6_UNK_0000_003f"; /* P6_M_Dothan. */
+        case 0x00000040: return g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah ? "MSR_LASTBRANCH_0_FROM_IP" : "MSR_LASTBRANCH_0";
+        case 0x00000041: return g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah ? "MSR_LASTBRANCH_1_FROM_IP" : "MSR_LASTBRANCH_1";
+        case 0x00000042: return g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah ? "MSR_LASTBRANCH_2_FROM_IP" : "MSR_LASTBRANCH_2";
+        case 0x00000043: return g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah ? "MSR_LASTBRANCH_3_FROM_IP" : "MSR_LASTBRANCH_3";
+        case 0x00000044: return g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah ? "MSR_LASTBRANCH_4_FROM_IP" : "MSR_LASTBRANCH_4";
+        case 0x00000045: return g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah ? "MSR_LASTBRANCH_5_FROM_IP" : "MSR_LASTBRANCH_5";
+        case 0x00000046: return g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah ? "MSR_LASTBRANCH_6_FROM_IP" : "MSR_LASTBRANCH_6";
+        case 0x00000047: return g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah ? "MSR_LASTBRANCH_7_FROM_IP" : "MSR_LASTBRANCH_7";
+        case 0x00000048: return "MSR_LASTBRANCH_8"; /*??*/
+        case 0x00000049: return "MSR_LASTBRANCH_9"; /*??*/
+        case 0x0000004a: return "P6_UNK_0000_004a"; /* P6_M_Dothan. */
+        case 0x0000004b: return "P6_UNK_0000_004b"; /* P6_M_Dothan. */
+        case 0x0000004c: return "P6_UNK_0000_004c"; /* P6_M_Dothan. */
+        case 0x0000004d: return "P6_UNK_0000_004d"; /* P6_M_Dothan. */
+        case 0x0000004e: return "P6_UNK_0000_004e"; /* P6_M_Dothan. */
+        case 0x0000004f: return "P6_UNK_0000_004f"; /* P6_M_Dothan. */
+        case 0x00000050: return "P6_UNK_0000_0050"; /* P6_M_Dothan. */
+        case 0x00000051: return "P6_UNK_0000_0051"; /* P6_M_Dothan. */
+        case 0x00000052: return "P6_UNK_0000_0052"; /* P6_M_Dothan. */
+        case 0x00000053: return "P6_UNK_0000_0053"; /* P6_M_Dothan. */
+        case 0x00000054: return "P6_UNK_0000_0054"; /* P6_M_Dothan. */
+        case 0x00000060: return "MSR_LASTBRANCH_0_TO_IP"; /* Core2_Penryn */
+        case 0x00000061: return "MSR_LASTBRANCH_1_TO_IP"; /* Core2_Penryn */
+        case 0x00000062: return "MSR_LASTBRANCH_2_TO_IP"; /* Core2_Penryn */
+        case 0x00000063: return "MSR_LASTBRANCH_3_TO_IP"; /* Core2_Penryn */
+        case 0x00000064: return "MSR_LASTBRANCH_4_TO_IP"; /* Atom? */
+        case 0x00000065: return "MSR_LASTBRANCH_5_TO_IP";
+        case 0x00000066: return "MSR_LASTBRANCH_6_TO_IP";
+        case 0x00000067: return "MSR_LASTBRANCH_7_TO_IP";
+        case 0x0000006c: return "P6_UNK_0000_006c"; /* P6_M_Dothan. */
+        case 0x0000006d: return "P6_UNK_0000_006d"; /* P6_M_Dothan. */
+        case 0x0000006e: return "P6_UNK_0000_006e"; /* P6_M_Dothan. */
+        case 0x0000006f: return "P6_UNK_0000_006f"; /* P6_M_Dothan. */
+        case 0x00000079: return "IA32_BIOS_UPDT_TRIG";
+        case 0x00000080: return "P4_UNK_0000_0080";
+        case 0x00000088: return "BBL_CR_D0";
+        case 0x00000089: return "BBL_CR_D1";
+        case 0x0000008a: return "BBL_CR_D2";
+        case 0x0000008b: return g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON ? "AMD_K8_PATCH_LEVEL"
+                              : g_fIntelNetBurst ? "IA32_BIOS_SIGN_ID" : "BBL_CR_D3|BIOS_SIGN";
+        case 0x0000008c: return "P6_UNK_0000_008c"; /* P6_M_Dothan. */
+        case 0x0000008d: return "P6_UNK_0000_008d"; /* P6_M_Dothan. */
+        case 0x0000008e: return "P6_UNK_0000_008e"; /* P6_M_Dothan. */
+        case 0x0000008f: return "P6_UNK_0000_008f"; /* P6_M_Dothan. */
+        case 0x00000090: return "P6_UNK_0000_0090"; /* P6_M_Dothan. */
+        case 0x0000009b: return "IA32_SMM_MONITOR_CTL";
+        case 0x000000a8: return "C2_EMTTM_CR_TABLES_0";
+        case 0x000000a9: return "C2_EMTTM_CR_TABLES_1";
+        case 0x000000aa: return "C2_EMTTM_CR_TABLES_2";
+        case 0x000000ab: return "C2_EMTTM_CR_TABLES_3";
+        case 0x000000ac: return "C2_EMTTM_CR_TABLES_4";
+        case 0x000000ad: return "C2_EMTTM_CR_TABLES_5";
+        case 0x000000ae: return "P6_UNK_0000_00ae"; /* P6_M_Dothan. */
+        case 0x000000c1: return "IA32_PMC0";
+        case 0x000000c2: return "IA32_PMC1";
+        case 0x000000c3: return "IA32_PMC2";
+        case 0x000000c4: return "IA32_PMC3";
+        /* PMC4+ first seen on SandyBridge. The earlier cut off is just to be
+           on the safe side as we must avoid P6_M_Dothan and possibly others. */
+        case 0x000000c5: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_First ? "IA32_PMC4" : NULL;
+        case 0x000000c6: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_First ? "IA32_PMC5" : NULL;
+        case 0x000000c7: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_First ? "IA32_PMC6" : "P6_UNK_0000_00c7"; /* P6_M_Dothan. */
+        case 0x000000c8: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_First ? "IA32_PMC7" : NULL;
+        case 0x000000cd: return "MSR_FSB_FREQ"; /* P6_M_Dothan. */
+        case 0x000000ce: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_First ? "IA32_PLATFORM_INFO" : "P6_UNK_0000_00ce"; /* P6_M_Dothan. */
+        case 0x000000cf: return "C2_UNK_0000_00cf"; /* Core2_Penryn. */
+        case 0x000000e0: return "C2_UNK_0000_00e0"; /* Core2_Penryn. */
+        case 0x000000e1: return "C2_UNK_0000_00e1"; /* Core2_Penryn. */
+        case 0x000000e2: return "MSR_PKG_CST_CONFIG_CONTROL";
+        case 0x000000e3: return "C2_SMM_CST_MISC_INFO"; /* Core2_Penryn. */
+        case 0x000000e4: return "MSR_PMG_IO_CAPTURE_BASE";
+        case 0x000000e5: return "C2_UNK_0000_00e5"; /* Core2_Penryn. */
+        case 0x000000e7: return "IA32_MPERF";
+        case 0x000000e8: return "IA32_APERF";
+        case 0x000000ee: return "C1_EXT_CONFIG"; /* Core2_Penryn. msrtool lists it for Core1 as well. */
+        case 0x000000fe: return "IA32_MTRRCAP";
+        case 0x00000102: return "I7_IB_UNK_0000_0102"; /* IvyBridge. */
+        case 0x00000103: return "I7_IB_UNK_0000_0103"; /* IvyBridge. */
+        case 0x00000104: return "I7_IB_UNK_0000_0104"; /* IvyBridge. */
+        case 0x00000116: return "BBL_CR_ADDR";
+        case 0x00000118: return "BBL_CR_DECC";
+        case 0x00000119: return "BBL_CR_CTL";
+        case 0x0000011a: return "BBL_CR_TRIG";
+        case 0x0000011b: return "P6_UNK_0000_011b"; /* P6_M_Dothan. */
+        case 0x0000011c: return "C2_UNK_0000_011c"; /* Core2_Penryn. */
+        case 0x0000011e: return "BBL_CR_CTL3";
+        case 0x00000120: return "SILV_UNK_0000_0120"; /* Silvermont */
+        case 0x00000130: return g_enmMicroarch == kCpumMicroarch_Intel_Core7_Westmere
+                             || g_enmMicroarch == kCpumMicroarch_Intel_Core7_Nehalem
+                              ? "CPUID1_FEATURE_MASK" : NULL;
+        case 0x00000131: return g_enmMicroarch == kCpumMicroarch_Intel_Core7_Westmere
+                             || g_enmMicroarch == kCpumMicroarch_Intel_Core7_Nehalem
+                              ? "CPUID80000001_FEATURE_MASK" : "P6_UNK_0000_0131" /* P6_M_Dothan. */;
+        case 0x00000132: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_SandyBridge
+                              ? "CPUID1_FEATURE_MASK" : NULL;
+        case 0x00000133: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_SandyBridge
+                              ? "CPUIDD_01_FEATURE_MASK" : NULL;
+        case 0x00000134: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_SandyBridge
+                              ? "CPUID80000001_FEATURE_MASK" : NULL;
+        case 0x0000013c: return "I7_SB_AES_NI_CTL"; /* SandyBridge. Bit 0 is lock bit, bit 1 disables AES-NI. */
+        case 0x00000140: return "I7_IB_UNK_0000_0140"; /* IvyBridge. */
+        case 0x00000142: return "I7_IB_UNK_0000_0142"; /* IvyBridge. */
+        case 0x0000014e: return "P6_UNK_0000_014e"; /* P6_M_Dothan. */
+        case 0x0000014f: return "P6_UNK_0000_014f"; /* P6_M_Dothan. */
+        case 0x00000150: return "P6_UNK_0000_0150"; /* P6_M_Dothan. */
+        case 0x00000151: return "P6_UNK_0000_0151"; /* P6_M_Dothan. */
+        case 0x00000154: return "P6_UNK_0000_0154"; /* P6_M_Dothan. */
+        case 0x0000015b: return "P6_UNK_0000_015b"; /* P6_M_Dothan. */
+        case 0x0000015e: return "C2_UNK_0000_015e"; /* Core2_Penryn. */
+        case 0x0000015f: return "C1_DTS_CAL_CTRL"; /* Core2_Penryn. msrtool only docs this for core1! */
+        case 0x00000174: return "IA32_SYSENTER_CS";
+        case 0x00000175: return "IA32_SYSENTER_ESP";
+        case 0x00000176: return "IA32_SYSENTER_EIP";
+        case 0x00000179: return "IA32_MCG_CAP";
+        case 0x0000017a: return "IA32_MCG_STATUS";
+        case 0x0000017b: return "IA32_MCG_CTL";
+        case 0x0000017f: return "I7_SB_ERROR_CONTROL"; /* SandyBridge. */
+        case 0x00000180: return g_fIntelNetBurst ? "MSR_MCG_RAX"       : NULL;
+        case 0x00000181: return g_fIntelNetBurst ? "MSR_MCG_RBX"       : NULL;
+        case 0x00000182: return g_fIntelNetBurst ? "MSR_MCG_RCX"       : NULL;
+        case 0x00000183: return g_fIntelNetBurst ? "MSR_MCG_RDX"       : NULL;
+        case 0x00000184: return g_fIntelNetBurst ? "MSR_MCG_RSI"       : NULL;
+        case 0x00000185: return g_fIntelNetBurst ? "MSR_MCG_RDI"       : NULL;
+        case 0x00000186: return g_fIntelNetBurst ? "MSR_MCG_RBP"       : "IA32_PERFEVTSEL0";
+        case 0x00000187: return g_fIntelNetBurst ? "MSR_MCG_RSP"       : "IA32_PERFEVTSEL1";
+        case 0x00000188: return g_fIntelNetBurst ? "MSR_MCG_RFLAGS"    : "IA32_PERFEVTSEL2";
+        case 0x00000189: return g_fIntelNetBurst ? "MSR_MCG_RIP"       : "IA32_PERFEVTSEL3";
+        case 0x0000018a: return g_fIntelNetBurst ? "MSR_MCG_MISC"      : "IA32_PERFEVTSEL4";
+        case 0x0000018b: return g_fIntelNetBurst ? "MSR_MCG_RESERVED1" : "IA32_PERFEVTSEL5";
+        case 0x0000018c: return g_fIntelNetBurst ? "MSR_MCG_RESERVED2" : "IA32_PERFEVTSEL6";
+        case 0x0000018d: return g_fIntelNetBurst ? "MSR_MCG_RESERVED3" : "IA32_PERFEVTSEL7";
+        case 0x0000018e: return g_fIntelNetBurst ? "MSR_MCG_RESERVED4" : "IA32_PERFEVTSEL8";
+        case 0x0000018f: return g_fIntelNetBurst ? "MSR_MCG_RESERVED5" : "IA32_PERFEVTSEL9";
+        case 0x00000190: return g_fIntelNetBurst ? "MSR_MCG_R8"        : NULL;
+        case 0x00000191: return g_fIntelNetBurst ? "MSR_MCG_R9"        : NULL;
+        case 0x00000192: return g_fIntelNetBurst ? "MSR_MCG_R10"       : NULL;
+        case 0x00000193: return g_fIntelNetBurst ? "MSR_MCG_R11"       : "C2_UNK_0000_0193";
+        case 0x00000194: return g_fIntelNetBurst ? "MSR_MCG_R12"       : "CLOCK_FLEX_MAX";
+        case 0x00000195: return g_fIntelNetBurst ? "MSR_MCG_R13"       : NULL;
+        case 0x00000196: return g_fIntelNetBurst ? "MSR_MCG_R14"       : NULL;
+        case 0x00000197: return g_fIntelNetBurst ? "MSR_MCG_R15"       : NULL;
+        case 0x00000198: return "IA32_PERF_STATUS";
+        case 0x00000199: return "IA32_PERF_CTL";
+        case 0x0000019a: return "IA32_CLOCK_MODULATION";
+        case 0x0000019b: return "IA32_THERM_INTERRUPT";
+        case 0x0000019c: return "IA32_THERM_STATUS";
+        case 0x0000019d: return "IA32_THERM2_CTL";
+        case 0x0000019e: return "P6_UNK_0000_019e"; /* P6_M_Dothan. */
+        case 0x0000019f: return "P6_UNK_0000_019f"; /* P6_M_Dothan. */
+        case 0x000001a0: return "IA32_MISC_ENABLE";
+        case 0x000001a1: return g_fIntelNetBurst ? "MSR_PLATFORM_BRV" : "P6_UNK_0000_01a1" /* P6_M_Dothan. */;
+        case 0x000001a2: return g_fIntelNetBurst ? "P4_UNK_0000_01a2" : "I7_MSR_TEMPERATURE_TARGET" /* SandyBridge, IvyBridge. */;
+        case 0x000001a4: return "I7_UNK_0000_01a4"; /* SandyBridge, IvyBridge. */
+        case 0x000001a6: return "I7_MSR_OFFCORE_RSP_0";
+        case 0x000001a7: return "I7_MSR_OFFCORE_RSP_1";
+        case 0x000001a8: return "I7_UNK_0000_01a8"; /* SandyBridge, IvyBridge. */
+        case 0x000001aa: return CPUMMICROARCH_IS_INTEL_CORE7(g_enmMicroarch) ? "MSR_MISC_PWR_MGMT" : "P6_PIC_SENS_CFG" /* Pentium M. */;
+        case 0x000001ad: return "I7_MSR_TURBO_RATIO_LIMIT"; /* SandyBridge+, Silvermount+ */
+        case 0x000001ae: return "P6_UNK_0000_01ae"; /* P6_M_Dothan. */
+        case 0x000001af: return "P6_UNK_0000_01af"; /* P6_M_Dothan. */
+        case 0x000001b0: return "IA32_ENERGY_PERF_BIAS";
+        case 0x000001b1: return "IA32_PACKAGE_THERM_STATUS";
+        case 0x000001b2: return "IA32_PACKAGE_THERM_INTERRUPT";
+        case 0x000001bf: return "C2_UNK_0000_01bf"; /* Core2_Penryn. */
+        case 0x000001c6: return "I7_UNK_0000_01c6"; /* SandyBridge*/
+        case 0x000001c8: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_Nehalem ? "MSR_LBR_SELECT" : NULL;
+        case 0x000001c9: return    g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah
+                                && g_enmMicroarch <= kCpumMicroarch_Intel_P6_Core_Atom_End
+                              ? "MSR_LASTBRANCH_TOS" : NULL /* Pentium M Dothan seems to have something else here. */;
+        case 0x000001d3: return "P6_UNK_0000_01d3"; /* P6_M_Dothan. */
+        case 0x000001d7: return g_fIntelNetBurst ? "MSR_LER_FROM_LIP" : NULL;
+        case 0x000001d8: return g_fIntelNetBurst ? "MSR_LER_TO_LIP"   : NULL;
+        case 0x000001d9: return "IA32_DEBUGCTL";
+        case 0x000001da: return g_fIntelNetBurst ? "MSR_LASTBRANCH_TOS" : NULL;
+        case 0x000001db: return g_fIntelNetBurst ? "P6_LASTBRANCH_0" : "P6_LAST_BRANCH_FROM_IP"; /* Not exclusive to P6, also AMD. */
+        case 0x000001dc: return g_fIntelNetBurst ? "P6_LASTBRANCH_1" : "P6_LAST_BRANCH_TO_IP";
+        case 0x000001dd: return g_fIntelNetBurst ? "P6_LASTBRANCH_2" : "P6_LAST_INT_FROM_IP";
+        case 0x000001de: return g_fIntelNetBurst ? "P6_LASTBRANCH_3" : "P6_LAST_INT_TO_IP";
+        case 0x000001e0: return "MSR_ROB_CR_BKUPTMPDR6";
+        case 0x000001e1: return "I7_SB_UNK_0000_01e1";
+        case 0x000001ef: return "I7_SB_UNK_0000_01ef";
+        case 0x000001f0: return "I7_VLW_CAPABILITY"; /* SandyBridge.  Bit 1 is A20M and was implemented incorrectly (AAJ49). */
+        case 0x000001f2: return "IA32_SMRR_PHYSBASE";
+        case 0x000001f3: return "IA32_SMRR_PHYSMASK";
+        case 0x000001f8: return "IA32_PLATFORM_DCA_CAP";
+        case 0x000001f9: return "IA32_CPU_DCA_CAP";
+        case 0x000001fa: return "IA32_DCA_0_CAP";
+        case 0x000001fc: return "I7_MSR_POWER_CTL";
+
+        case 0x00000200: return "IA32_MTRR_PHYS_BASE0";
+        case 0x00000202: return "IA32_MTRR_PHYS_BASE1";
+        case 0x00000204: return "IA32_MTRR_PHYS_BASE2";
+        case 0x00000206: return "IA32_MTRR_PHYS_BASE3";
+        case 0x00000208: return "IA32_MTRR_PHYS_BASE4";
+        case 0x0000020a: return "IA32_MTRR_PHYS_BASE5";
+        case 0x0000020c: return "IA32_MTRR_PHYS_BASE6";
+        case 0x0000020e: return "IA32_MTRR_PHYS_BASE7";
+        case 0x00000210: return "IA32_MTRR_PHYS_BASE8";
+        case 0x00000212: return "IA32_MTRR_PHYS_BASE9";
+        case 0x00000214: return "IA32_MTRR_PHYS_BASE10";
+        case 0x00000216: return "IA32_MTRR_PHYS_BASE11";
+        case 0x00000218: return "IA32_MTRR_PHYS_BASE12";
+        case 0x0000021a: return "IA32_MTRR_PHYS_BASE13";
+        case 0x0000021c: return "IA32_MTRR_PHYS_BASE14";
+        case 0x0000021e: return "IA32_MTRR_PHYS_BASE15";
+
+        case 0x00000201: return "IA32_MTRR_PHYS_MASK0";
+        case 0x00000203: return "IA32_MTRR_PHYS_MASK1";
+        case 0x00000205: return "IA32_MTRR_PHYS_MASK2";
+        case 0x00000207: return "IA32_MTRR_PHYS_MASK3";
+        case 0x00000209: return "IA32_MTRR_PHYS_MASK4";
+        case 0x0000020b: return "IA32_MTRR_PHYS_MASK5";
+        case 0x0000020d: return "IA32_MTRR_PHYS_MASK6";
+        case 0x0000020f: return "IA32_MTRR_PHYS_MASK7";
+        case 0x00000211: return "IA32_MTRR_PHYS_MASK8";
+        case 0x00000213: return "IA32_MTRR_PHYS_MASK9";
+        case 0x00000215: return "IA32_MTRR_PHYS_MASK10";
+        case 0x00000217: return "IA32_MTRR_PHYS_MASK11";
+        case 0x00000219: return "IA32_MTRR_PHYS_MASK12";
+        case 0x0000021b: return "IA32_MTRR_PHYS_MASK13";
+        case 0x0000021d: return "IA32_MTRR_PHYS_MASK14";
+        case 0x0000021f: return "IA32_MTRR_PHYS_MASK15";
+
+        case 0x00000250: return "IA32_MTRR_FIX64K_00000";
+        case 0x00000258: return "IA32_MTRR_FIX16K_80000";
+        case 0x00000259: return "IA32_MTRR_FIX16K_A0000";
+        case 0x00000268: return "IA32_MTRR_FIX4K_C0000";
+        case 0x00000269: return "IA32_MTRR_FIX4K_C8000";
+        case 0x0000026a: return "IA32_MTRR_FIX4K_D0000";
+        case 0x0000026b: return "IA32_MTRR_FIX4K_D8000";
+        case 0x0000026c: return "IA32_MTRR_FIX4K_E0000";
+        case 0x0000026d: return "IA32_MTRR_FIX4K_E8000";
+        case 0x0000026e: return "IA32_MTRR_FIX4K_F0000";
+        case 0x0000026f: return "IA32_MTRR_FIX4K_F8000";
+        case 0x00000277: return "IA32_PAT";
+        case 0x00000280: return "IA32_MC0_CTL2";
+        case 0x00000281: return "IA32_MC1_CTL2";
+        case 0x00000282: return "IA32_MC2_CTL2";
+        case 0x00000283: return "IA32_MC3_CTL2";
+        case 0x00000284: return "IA32_MC4_CTL2";
+        case 0x00000285: return "IA32_MC5_CTL2";
+        case 0x00000286: return "IA32_MC6_CTL2";
+        case 0x00000287: return "IA32_MC7_CTL2";
+        case 0x00000288: return "IA32_MC8_CTL2";
+        case 0x00000289: return "IA32_MC9_CTL2";
+        case 0x0000028a: return "IA32_MC10_CTL2";
+        case 0x0000028b: return "IA32_MC11_CTL2";
+        case 0x0000028c: return "IA32_MC12_CTL2";
+        case 0x0000028d: return "IA32_MC13_CTL2";
+        case 0x0000028e: return "IA32_MC14_CTL2";
+        case 0x0000028f: return "IA32_MC15_CTL2";
+        case 0x00000290: return "IA32_MC16_CTL2";
+        case 0x00000291: return "IA32_MC17_CTL2";
+        case 0x00000292: return "IA32_MC18_CTL2";
+        case 0x00000293: return "IA32_MC19_CTL2";
+        case 0x00000294: return "IA32_MC20_CTL2";
+        case 0x00000295: return "IA32_MC21_CTL2";
+        //case 0x00000296: return "IA32_MC22_CTL2";
+        //case 0x00000297: return "IA32_MC23_CTL2";
+        //case 0x00000298: return "IA32_MC24_CTL2";
+        //case 0x00000299: return "IA32_MC25_CTL2";
+        //case 0x0000029a: return "IA32_MC26_CTL2";
+        //case 0x0000029b: return "IA32_MC27_CTL2";
+        //case 0x0000029c: return "IA32_MC28_CTL2";
+        //case 0x0000029d: return "IA32_MC29_CTL2";
+        //case 0x0000029e: return "IA32_MC30_CTL2";
+        //case 0x0000029f: return "IA32_MC31_CTL2";
+        case 0x000002e0: return "I7_SB_NO_EVICT_MODE"; /* (Bits 1 & 0 are said to have something to do with no-evict cache mode used during early boot.) */
+        case 0x000002e6: return "I7_IB_UNK_0000_02e6"; /* IvyBridge */
+        case 0x000002e7: return "I7_IB_UNK_0000_02e7"; /* IvyBridge */
+        case 0x000002ff: return "IA32_MTRR_DEF_TYPE";
+        case 0x00000300: return g_fIntelNetBurst ? "P4_MSR_BPU_COUNTER0"   : "I7_SB_UNK_0000_0300" /* SandyBridge */;
+        case 0x00000301: return g_fIntelNetBurst ? "P4_MSR_BPU_COUNTER1"   : NULL;
+        case 0x00000302: return g_fIntelNetBurst ? "P4_MSR_BPU_COUNTER2"   : NULL;
+        case 0x00000303: return g_fIntelNetBurst ? "P4_MSR_BPU_COUNTER3"   : NULL;
+        case 0x00000304: return g_fIntelNetBurst ? "P4_MSR_MS_COUNTER0"    : NULL;
+        case 0x00000305: return g_fIntelNetBurst ? "P4_MSR_MS_COUNTER1"    : "I7_SB_UNK_0000_0305" /* SandyBridge, IvyBridge */;
+        case 0x00000306: return g_fIntelNetBurst ? "P4_MSR_MS_COUNTER2"    : NULL;
+        case 0x00000307: return g_fIntelNetBurst ? "P4_MSR_MS_COUNTER3"    : NULL;
+        case 0x00000308: return g_fIntelNetBurst ? "P4_MSR_FLAME_COUNTER0" : NULL;
+        case 0x00000309: return g_fIntelNetBurst ? "P4_MSR_FLAME_COUNTER1" : "IA32_FIXED_CTR0";
+        case 0x0000030a: return g_fIntelNetBurst ? "P4_MSR_FLAME_COUNTER2" : "IA32_FIXED_CTR1";
+        case 0x0000030b: return g_fIntelNetBurst ? "P4_MSR_FLAME_COUNTER3" : "IA32_FIXED_CTR2";
+        case 0x0000030c: return g_fIntelNetBurst ? "P4_MSR_IQ_COUNTER0" : NULL;
+        case 0x0000030d: return g_fIntelNetBurst ? "P4_MSR_IQ_COUNTER1" : NULL;
+        case 0x0000030e: return g_fIntelNetBurst ? "P4_MSR_IQ_COUNTER2" : NULL;
+        case 0x0000030f: return g_fIntelNetBurst ? "P4_MSR_IQ_COUNTER3" : NULL;
+        case 0x00000310: return g_fIntelNetBurst ? "P4_MSR_IQ_COUNTER4" : NULL;
+        case 0x00000311: return g_fIntelNetBurst ? "P4_MSR_IQ_COUNTER5" : NULL;
+        case 0x00000345: return "IA32_PERF_CAPABILITIES";
+        case 0x00000360: return g_fIntelNetBurst ? "P4_MSR_BPU_CCCR0"   : NULL;
+        case 0x00000361: return g_fIntelNetBurst ? "P4_MSR_BPU_CCCR1"   : NULL;
+        case 0x00000362: return g_fIntelNetBurst ? "P4_MSR_BPU_CCCR2"   : NULL;
+        case 0x00000363: return g_fIntelNetBurst ? "P4_MSR_BPU_CCCR3"   : NULL;
+        case 0x00000364: return g_fIntelNetBurst ? "P4_MSR_MS_CCCR0"    : NULL;
+        case 0x00000365: return g_fIntelNetBurst ? "P4_MSR_MS_CCCR1"    : NULL;
+        case 0x00000366: return g_fIntelNetBurst ? "P4_MSR_MS_CCCR2"    : NULL;
+        case 0x00000367: return g_fIntelNetBurst ? "P4_MSR_MS_CCCR3"    : NULL;
+        case 0x00000368: return g_fIntelNetBurst ? "P4_MSR_FLAME_CCCR0" : NULL;
+        case 0x00000369: return g_fIntelNetBurst ? "P4_MSR_FLAME_CCCR1" : NULL;
+        case 0x0000036a: return g_fIntelNetBurst ? "P4_MSR_FLAME_CCCR2" : NULL;
+        case 0x0000036b: return g_fIntelNetBurst ? "P4_MSR_FLAME_CCCR3" : NULL;
+        case 0x0000036c: return g_fIntelNetBurst ? "P4_MSR_IQ_CCCR0"    : NULL;
+        case 0x0000036d: return g_fIntelNetBurst ? "P4_MSR_IQ_CCCR1"    : NULL;
+        case 0x0000036e: return g_fIntelNetBurst ? "P4_MSR_IQ_CCCR2"    : NULL;
+        case 0x0000036f: return g_fIntelNetBurst ? "P4_MSR_IQ_CCCR3"    : NULL;
+        case 0x00000370: return g_fIntelNetBurst ? "P4_MSR_IQ_CCCR4"    : NULL;
+        case 0x00000371: return g_fIntelNetBurst ? "P4_MSR_IQ_CCCR5"    : NULL;
+        case 0x0000038d: return "IA32_FIXED_CTR_CTRL";
+        case 0x0000038e: return "IA32_PERF_GLOBAL_STATUS";
+        case 0x0000038f: return "IA32_PERF_GLOBAL_CTRL";
+        case 0x00000390: return "IA32_PERF_GLOBAL_OVF_CTRL";
+        case 0x00000391: return "I7_UNC_PERF_GLOBAL_CTRL";             /* S,H,X */
+        case 0x00000392: return "I7_UNC_PERF_GLOBAL_STATUS";           /* S,H,X */
+        case 0x00000393: return "I7_UNC_PERF_GLOBAL_OVF_CTRL";         /* X. ASSUMING this is the same on sandybridge and later. */
+        case 0x00000394: return g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "I7_UNC_PERF_FIXED_CTR"  /* X */    : "I7_UNC_PERF_FIXED_CTR_CTRL"; /* >= S,H */
+        case 0x00000395: return g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "I7_UNC_PERF_FIXED_CTR_CTRL" /* X*/ : "I7_UNC_PERF_FIXED_CTR";      /* >= S,H */
+        case 0x00000396: return g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "I7_UNC_ADDR_OPCODE_MATCH" /* X */  : "I7_UNC_CBO_CONFIG";          /* >= S,H */
+        case 0x00000397: return g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? NULL                                : "I7_SB_UNK_0000_0397";
+        case 0x0000039c: return "I7_SB_MSR_PEBS_NUM_ALT";
+        case 0x000003a0: return g_fIntelNetBurst ? "P4_MSR_BSU_ESCR0"   : NULL;
+        case 0x000003a1: return g_fIntelNetBurst ? "P4_MSR_BSU_ESCR1"   : NULL;
+        case 0x000003a2: return g_fIntelNetBurst ? "P4_MSR_FSB_ESCR0"   : NULL;
+        case 0x000003a3: return g_fIntelNetBurst ? "P4_MSR_FSB_ESCR1"   : NULL;
+        case 0x000003a4: return g_fIntelNetBurst ? "P4_MSR_FIRM_ESCR0"  : NULL;
+        case 0x000003a5: return g_fIntelNetBurst ? "P4_MSR_FIRM_ESCR1"  : NULL;
+        case 0x000003a6: return g_fIntelNetBurst ? "P4_MSR_FLAME_ESCR0" : NULL;
+        case 0x000003a7: return g_fIntelNetBurst ? "P4_MSR_FLAME_ESCR1" : NULL;
+        case 0x000003a8: return g_fIntelNetBurst ? "P4_MSR_DAC_ESCR0"   : NULL;
+        case 0x000003a9: return g_fIntelNetBurst ? "P4_MSR_DAC_ESCR1"   : NULL;
+        case 0x000003aa: return g_fIntelNetBurst ? "P4_MSR_MOB_ESCR0"   : NULL;
+        case 0x000003ab: return g_fIntelNetBurst ? "P4_MSR_MOB_ESCR1"   : NULL;
+        case 0x000003ac: return g_fIntelNetBurst ? "P4_MSR_PMH_ESCR0"   : NULL;
+        case 0x000003ad: return g_fIntelNetBurst ? "P4_MSR_PMH_ESCR1"   : NULL;
+        case 0x000003ae: return g_fIntelNetBurst ? "P4_MSR_SAAT_ESCR0"  : NULL;
+        case 0x000003af: return g_fIntelNetBurst ? "P4_MSR_SAAT_ESCR1"  : NULL;
+        case 0x000003b0: return g_fIntelNetBurst ? "P4_MSR_U2L_ESCR0" : g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "I7_UNC_PMC0" /* X */               : "I7_UNC_ARB_PERF_CTR0";       /* >= S,H */
+        case 0x000003b1: return g_fIntelNetBurst ? "P4_MSR_U2L_ESCR1" : g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "I7_UNC_PMC1" /* X */               : "I7_UNC_ARB_PERF_CTR1";       /* >= S,H */
+        case 0x000003b2: return g_fIntelNetBurst ? "P4_MSR_BPU_ESCR0" : g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "I7_UNC_PMC2" /* X */               : "I7_UNC_ARB_PERF_EVT_SEL0";   /* >= S,H */
+        case 0x000003b3: return g_fIntelNetBurst ? "P4_MSR_BPU_ESCR1" : g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "I7_UNC_PMC3" /* X */               : "I7_UNC_ARB_PERF_EVT_SEL1";   /* >= S,H */
+        case 0x000003b4: return g_fIntelNetBurst ? "P4_MSR_IS_ESCR0"    : "I7_UNC_PMC4";
+        case 0x000003b5: return g_fIntelNetBurst ? "P4_MSR_IS_ESCR1"    : "I7_UNC_PMC5";
+        case 0x000003b6: return g_fIntelNetBurst ? "P4_MSR_ITLB_ESCR0"  : "I7_UNC_PMC6";
+        case 0x000003b7: return g_fIntelNetBurst ? "P4_MSR_ITLB_ESCR1"  : "I7_UNC_PMC7";
+        case 0x000003b8: return g_fIntelNetBurst ? "P4_MSR_CRU_ESCR0"   : NULL;
+        case 0x000003b9: return g_fIntelNetBurst ? "P4_MSR_CRU_ESCR1"   : NULL;
+        case 0x000003ba: return g_fIntelNetBurst ? "P4_MSR_IQ_ESCR0"    : NULL;
+        case 0x000003bb: return g_fIntelNetBurst ? "P4_MSR_IQ_ESCR1"    : NULL;
+        case 0x000003bc: return g_fIntelNetBurst ? "P4_MSR_RAT_ESCR0"   : NULL;
+        case 0x000003bd: return g_fIntelNetBurst ? "P4_MSR_RAT_ESCR1"   : NULL;
+        case 0x000003be: return g_fIntelNetBurst ? "P4_MSR_SSU_ESCR0"   : NULL;
+        case 0x000003c0: return g_fIntelNetBurst ? "P4_MSR_MS_ESCR0"    : "I7_UNC_PERF_EVT_SEL0";
+        case 0x000003c1: return g_fIntelNetBurst ? "P4_MSR_MS_ESCR1"    : "I7_UNC_PERF_EVT_SEL1";
+        case 0x000003c2: return g_fIntelNetBurst ? "P4_MSR_TBPU_ESCR0"  : "I7_UNC_PERF_EVT_SEL2";
+        case 0x000003c3: return g_fIntelNetBurst ? "P4_MSR_TBPU_ESCR1"  : "I7_UNC_PERF_EVT_SEL3";
+        case 0x000003c4: return g_fIntelNetBurst ? "P4_MSR_TC_ESCR0"    : "I7_UNC_PERF_EVT_SEL4";
+        case 0x000003c5: return g_fIntelNetBurst ? "P4_MSR_TC_ESCR1"    : "I7_UNC_PERF_EVT_SEL5";
+        case 0x000003c6: return g_fIntelNetBurst ? NULL                 : "I7_UNC_PERF_EVT_SEL6";
+        case 0x000003c7: return g_fIntelNetBurst ? NULL                 : "I7_UNC_PERF_EVT_SEL7";
+        case 0x000003c8: return g_fIntelNetBurst ? "P4_MSR_IX_ESCR0"    : NULL;
+        case 0x000003c9: return g_fIntelNetBurst ? "P4_MSR_IX_ESCR0"    : NULL;
+        case 0x000003ca: return g_fIntelNetBurst ? "P4_MSR_ALF_ESCR0"   : NULL;
+        case 0x000003cb: return g_fIntelNetBurst ? "P4_MSR_ALF_ESCR1"   : NULL;
+        case 0x000003cc: return g_fIntelNetBurst ? "P4_MSR_CRU_ESCR2"   : NULL;
+        case 0x000003cd: return g_fIntelNetBurst ? "P4_MSR_CRU_ESCR3"   : NULL;
+        case 0x000003e0: return g_fIntelNetBurst ? "P4_MSR_CRU_ESCR4"   : NULL;
+        case 0x000003e1: return g_fIntelNetBurst ? "P4_MSR_CRU_ESCR5"   : NULL;
+        case 0x000003f0: return g_fIntelNetBurst ? "P4_MSR_TC_PRECISE_EVENT" : NULL;
+        case 0x000003f1: return "IA32_PEBS_ENABLE";
+        case 0x000003f2: return g_fIntelNetBurst ? "P4_MSR_PEBS_MATRIX_VERT" : "IA32_PEBS_ENABLE";
+        case 0x000003f3: return g_fIntelNetBurst ? "P4_UNK_0000_03f3" : NULL;
+        case 0x000003f4: return g_fIntelNetBurst ? "P4_UNK_0000_03f4" : NULL;
+        case 0x000003f5: return g_fIntelNetBurst ? "P4_UNK_0000_03f5" : NULL;
+        case 0x000003f6: return g_fIntelNetBurst ? "P4_UNK_0000_03f6" : "I7_MSR_PEBS_LD_LAT";
+        case 0x000003f7: return g_fIntelNetBurst ? "P4_UNK_0000_03f7" : "I7_MSR_PEBS_LD_LAT";
+        case 0x000003f8: return g_fIntelNetBurst ? "P4_UNK_0000_03f8" : "I7_MSR_PKG_C3_RESIDENCY";
+        case 0x000003f9: return "I7_MSR_PKG_C6_RESIDENCY";
+        case 0x000003fa: return "I7_MSR_PKG_C7_RESIDENCY";
+        case 0x000003fc: return "I7_MSR_CORE_C3_RESIDENCY";
+        case 0x000003fd: return "I7_MSR_CORE_C6_RESIDENCY";
+        case 0x000003fe: return "I7_MSR_CORE_C7_RESIDENCY";
+        case 0x00000478: return g_enmMicroarch == kCpumMicroarch_Intel_Core2_Penryn ? "CPUID1_FEATURE_MASK" : NULL;
+        case 0x00000480: return "IA32_VMX_BASIC";
+        case 0x00000481: return "IA32_VMX_PINBASED_CTLS";
+        case 0x00000482: return "IA32_VMX_PROCBASED_CTLS";
+        case 0x00000483: return "IA32_VMX_EXIT_CTLS";
+        case 0x00000484: return "IA32_VMX_ENTRY_CTLS";
+        case 0x00000485: return "IA32_VMX_MISC";
+        case 0x00000486: return "IA32_VMX_CR0_FIXED0";
+        case 0x00000487: return "IA32_VMX_CR0_FIXED1";
+        case 0x00000488: return "IA32_VMX_CR4_FIXED0";
+        case 0x00000489: return "IA32_VMX_CR4_FIXED1";
+        case 0x0000048a: return "IA32_VMX_VMCS_ENUM";
+        case 0x0000048b: return "IA32_VMX_PROCBASED_CTLS2";
+        case 0x0000048c: return "IA32_VMX_EPT_VPID_CAP";
+        case 0x0000048d: return "IA32_VMX_TRUE_PINBASED_CTLS";
+        case 0x0000048e: return "IA32_VMX_TRUE_PROCBASED_CTLS";
+        case 0x0000048f: return "IA32_VMX_TRUE_EXIT_CTLS";
+        case 0x00000490: return "IA32_VMX_TRUE_ENTRY_CTLS";
+        case 0x00000491: return "IA32_VMX_VMFUNC";
+        case 0x000004c1: return "IA32_A_PMC0";
+        case 0x000004c2: return "IA32_A_PMC1";
+        case 0x000004c3: return "IA32_A_PMC2";
+        case 0x000004c4: return "IA32_A_PMC3";
+        case 0x000004c5: return "IA32_A_PMC4";
+        case 0x000004c6: return "IA32_A_PMC5";
+        case 0x000004c7: return "IA32_A_PMC6";
+        case 0x000004c8: return "IA32_A_PMC7";
+        case 0x000004f8: return "C2_UNK_0000_04f8"; /* Core2_Penryn. */
+        case 0x000004f9: return "C2_UNK_0000_04f9"; /* Core2_Penryn. */
+        case 0x000004fa: return "C2_UNK_0000_04fa"; /* Core2_Penryn. */
+        case 0x000004fb: return "C2_UNK_0000_04fb"; /* Core2_Penryn. */
+        case 0x000004fc: return "C2_UNK_0000_04fc"; /* Core2_Penryn. */
+        case 0x000004fd: return "C2_UNK_0000_04fd"; /* Core2_Penryn. */
+        case 0x000004fe: return "C2_UNK_0000_04fe"; /* Core2_Penryn. */
+        case 0x000004ff: return "C2_UNK_0000_04ff"; /* Core2_Penryn. */
+        case 0x00000502: return "I7_SB_UNK_0000_0502";
+        case 0x00000590: return "C2_UNK_0000_0590"; /* Core2_Penryn. */
+        case 0x00000591: return "C2_UNK_0000_0591"; /* Core2_Penryn. */
+        case 0x000005a0: return "C2_PECI_CTL"; /* Core2_Penryn. */
+        case 0x000005a1: return "C2_UNK_0000_05a1"; /* Core2_Penryn. */
+        case 0x00000600: return "IA32_DS_AREA";
+        case 0x00000601: return "I7_SB_MSR_VR_CURRENT_CONFIG"; /* SandyBridge, IvyBridge. */
+        case 0x00000602: return "I7_IB_UNK_0000_0602";
+        case 0x00000603: return "I7_SB_MSR_VR_MISC_CONFIG"; /* SandyBridge, IvyBridge. */
+        case 0x00000604: return "I7_IB_UNK_0000_0602";
+        case 0x00000606: return "I7_SB_MSR_RAPL_POWER_UNIT"; /* SandyBridge, IvyBridge. */
+        case 0x00000609: return "I7_SB_UNK_0000_0609";  /* SandyBridge (non EP). */
+        case 0x0000060a: return "I7_SB_MSR_PKGC3_IRTL"; /* SandyBridge, IvyBridge. */
+        case 0x0000060b: return "I7_SB_MSR_PKGC6_IRTL"; /* SandyBridge, IvyBridge. */
+        case 0x0000060c: return "I7_SB_MSR_PKGC7_IRTL"; /* SandyBridge, IvyBridge. */
+        case 0x0000060d: return "I7_SB_MSR_PKG_C2_RESIDENCY"; /* SandyBridge, IvyBridge. */
+        case 0x00000610: return "I7_SB_MSR_PKG_POWER_LIMIT";
+        case 0x00000611: return "I7_SB_MSR_PKG_ENERGY_STATUS";
+        case 0x00000613: return "I7_SB_MSR_PKG_PERF_STATUS";
+        case 0x00000614: return "I7_SB_MSR_PKG_POWER_INFO";
+        case 0x00000618: return "I7_SB_MSR_DRAM_POWER_LIMIT";
+        case 0x00000619: return "I7_SB_MSR_DRAM_ENERGY_STATUS";
+        case 0x0000061b: return "I7_SB_MSR_DRAM_PERF_STATUS";
+        case 0x0000061c: return "I7_SB_MSR_DRAM_POWER_INFO";
+        case 0x00000638: return "I7_SB_MSR_PP0_POWER_LIMIT";
+        case 0x00000639: return "I7_SB_MSR_PP0_ENERGY_STATUS";
+        case 0x0000063a: return "I7_SB_MSR_PP0_POLICY";
+        case 0x0000063b: return "I7_SB_MSR_PP0_PERF_STATUS";
+        case 0x00000640: return "I7_HW_MSR_PP0_POWER_LIMIT";
+        case 0x00000641: return "I7_HW_MSR_PP0_ENERGY_STATUS";
+        case 0x00000642: return "I7_HW_MSR_PP0_POLICY";
+        case 0x00000648: return "I7_IB_MSR_CONFIG_TDP_NOMINAL";
+        case 0x00000649: return "I7_IB_MSR_CONFIG_TDP_LEVEL1";
+        case 0x0000064a: return "I7_IB_MSR_CONFIG_TDP_LEVEL2";
+        case 0x0000064b: return "I7_IB_MSR_CONFIG_TDP_CONTROL";
+        case 0x0000064c: return "I7_IB_MSR_TURBO_ACTIVATION_RATIO";
+        case 0x00000660: return "SILV_CORE_C1_RESIDENCY";
+        case 0x00000661: return "SILV_UNK_0000_0661";
+        case 0x00000662: return "SILV_UNK_0000_0662";
+        case 0x00000663: return "SILV_UNK_0000_0663";
+        case 0x00000664: return "SILV_UNK_0000_0664";
+        case 0x00000665: return "SILV_UNK_0000_0665";
+        case 0x00000666: return "SILV_UNK_0000_0666";
+        case 0x00000667: return "SILV_UNK_0000_0667";
+        case 0x00000668: return "SILV_UNK_0000_0668";
+        case 0x00000669: return "SILV_UNK_0000_0669";
+        case 0x0000066a: return "SILV_UNK_0000_066a";
+        case 0x0000066b: return "SILV_UNK_0000_066b";
+        case 0x0000066c: return "SILV_UNK_0000_066c";
+        case 0x0000066d: return "SILV_UNK_0000_066d";
+        case 0x0000066e: return "SILV_UNK_0000_066e";
+        case 0x0000066f: return "SILV_UNK_0000_066f";
+        case 0x00000670: return "SILV_UNK_0000_0670";
+        case 0x00000671: return "SILV_UNK_0000_0671";
+        case 0x00000672: return "SILV_UNK_0000_0672";
+        case 0x00000673: return "SILV_UNK_0000_0673";
+        case 0x00000674: return "SILV_UNK_0000_0674";
+        case 0x00000675: return "SILV_UNK_0000_0675";
+        case 0x00000676: return "SILV_UNK_0000_0676";
+        case 0x00000677: return "SILV_UNK_0000_0677";
+
+        case 0x00000680: return "MSR_LASTBRANCH_0_FROM_IP";
+        case 0x00000681: return "MSR_LASTBRANCH_1_FROM_IP";
+        case 0x00000682: return "MSR_LASTBRANCH_2_FROM_IP";
+        case 0x00000683: return "MSR_LASTBRANCH_3_FROM_IP";
+        case 0x00000684: return "MSR_LASTBRANCH_4_FROM_IP";
+        case 0x00000685: return "MSR_LASTBRANCH_5_FROM_IP";
+        case 0x00000686: return "MSR_LASTBRANCH_6_FROM_IP";
+        case 0x00000687: return "MSR_LASTBRANCH_7_FROM_IP";
+        case 0x00000688: return "MSR_LASTBRANCH_8_FROM_IP";
+        case 0x00000689: return "MSR_LASTBRANCH_9_FROM_IP";
+        case 0x0000068a: return "MSR_LASTBRANCH_10_FROM_IP";
+        case 0x0000068b: return "MSR_LASTBRANCH_11_FROM_IP";
+        case 0x0000068c: return "MSR_LASTBRANCH_12_FROM_IP";
+        case 0x0000068d: return "MSR_LASTBRANCH_13_FROM_IP";
+        case 0x0000068e: return "MSR_LASTBRANCH_14_FROM_IP";
+        case 0x0000068f: return "MSR_LASTBRANCH_15_FROM_IP";
+        case 0x000006c0: return "MSR_LASTBRANCH_0_TO_IP";
+        case 0x000006c1: return "MSR_LASTBRANCH_1_TO_IP";
+        case 0x000006c2: return "MSR_LASTBRANCH_2_TO_IP";
+        case 0x000006c3: return "MSR_LASTBRANCH_3_TO_IP";
+        case 0x000006c4: return "MSR_LASTBRANCH_4_TO_IP";
+        case 0x000006c5: return "MSR_LASTBRANCH_5_TO_IP";
+        case 0x000006c6: return "MSR_LASTBRANCH_6_TO_IP";
+        case 0x000006c7: return "MSR_LASTBRANCH_7_TO_IP";
+        case 0x000006c8: return "MSR_LASTBRANCH_8_TO_IP";
+        case 0x000006c9: return "MSR_LASTBRANCH_9_TO_IP";
+        case 0x000006ca: return "MSR_LASTBRANCH_10_TO_IP";
+        case 0x000006cb: return "MSR_LASTBRANCH_11_TO_IP";
+        case 0x000006cc: return "MSR_LASTBRANCH_12_TO_IP";
+        case 0x000006cd: return "MSR_LASTBRANCH_13_TO_IP";
+        case 0x000006ce: return "MSR_LASTBRANCH_14_TO_IP";
+        case 0x000006cf: return "MSR_LASTBRANCH_15_TO_IP";
+        case 0x000006e0: return "IA32_TSC_DEADLINE";
+
+        case 0x00000768: return "SILV_UNK_0000_0768";
+        case 0x00000769: return "SILV_UNK_0000_0769";
+        case 0x0000076a: return "SILV_UNK_0000_076a";
+        case 0x0000076b: return "SILV_UNK_0000_076b";
+        case 0x0000076c: return "SILV_UNK_0000_076c";
+        case 0x0000076d: return "SILV_UNK_0000_076d";
+        case 0x0000076e: return "SILV_UNK_0000_076e";
+
+        case 0x00000c80: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_IvyBridge ? "IA32_DEBUG_INTERFACE" : NULL; /* Mentioned in an intel dataskit called 4th-gen-core-family-desktop-vol-1-datasheet.pdf. */
+        case 0x00000c81: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_IvyBridge ? "I7_IB_UNK_0000_0c81"  : NULL; /* Probably related to IA32_DEBUG_INTERFACE... */
+        case 0x00000c82: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_IvyBridge ? "I7_IB_UNK_0000_0c82"  : NULL; /* Probably related to IA32_DEBUG_INTERFACE... */
+        case 0x00000c83: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_IvyBridge ? "I7_IB_UNK_0000_0c83"  : NULL; /* Probably related to IA32_DEBUG_INTERFACE... */
+
+        /* 0x1000..0x1004 seems to have been used by IBM 386 and 486 clones too. */
+        case 0x00001000: return "P6_DEBUG_REGISTER_0";
+        case 0x00001001: return "P6_DEBUG_REGISTER_1";
+        case 0x00001002: return "P6_DEBUG_REGISTER_2";
+        case 0x00001003: return "P6_DEBUG_REGISTER_3";
+        case 0x00001004: return "P6_DEBUG_REGISTER_4";
+        case 0x00001005: return "P6_DEBUG_REGISTER_5";
+        case 0x00001006: return "P6_DEBUG_REGISTER_6";
+        case 0x00001007: return "P6_DEBUG_REGISTER_7";
+        case 0x0000103f: return "P6_UNK_0000_103f"; /* P6_M_Dothan. */
+        case 0x000010cd: return "P6_UNK_0000_10cd"; /* P6_M_Dothan. */
+
+        case 0x00001107: return "VIA_UNK_0000_1107";
+        case 0x0000110f: return "VIA_UNK_0000_110f";
+        case 0x00001153: return "VIA_UNK_0000_1153";
+        case 0x00001200: return "VIA_UNK_0000_1200";
+        case 0x00001201: return "VIA_UNK_0000_1201";
+        case 0x00001202: return "VIA_UNK_0000_1202";
+        case 0x00001203: return "VIA_UNK_0000_1203";
+        case 0x00001204: return "VIA_UNK_0000_1204";
+        case 0x00001205: return "VIA_UNK_0000_1205";
+        case 0x00001206: return "VIA_ALT_VENDOR_EBX";
+        case 0x00001207: return "VIA_ALT_VENDOR_ECDX";
+        case 0x00001208: return "VIA_UNK_0000_1208";
+        case 0x00001209: return "VIA_UNK_0000_1209";
+        case 0x0000120a: return "VIA_UNK_0000_120a";
+        case 0x0000120b: return "VIA_UNK_0000_120b";
+        case 0x0000120c: return "VIA_UNK_0000_120c";
+        case 0x0000120d: return "VIA_UNK_0000_120d";
+        case 0x0000120e: return "VIA_UNK_0000_120e";
+        case 0x0000120f: return "VIA_UNK_0000_120f";
+        case 0x00001210: return "VIA_UNK_0000_1210";
+        case 0x00001211: return "VIA_UNK_0000_1211";
+        case 0x00001212: return "VIA_UNK_0000_1212";
+        case 0x00001213: return "VIA_UNK_0000_1213";
+        case 0x00001214: return "VIA_UNK_0000_1214";
+        case 0x00001220: return "VIA_UNK_0000_1220";
+        case 0x00001221: return "VIA_UNK_0000_1221";
+        case 0x00001230: return "VIA_UNK_0000_1230";
+        case 0x00001231: return "VIA_UNK_0000_1231";
+        case 0x00001232: return "VIA_UNK_0000_1232";
+        case 0x00001233: return "VIA_UNK_0000_1233";
+        case 0x00001234: return "VIA_UNK_0000_1234";
+        case 0x00001235: return "VIA_UNK_0000_1235";
+        case 0x00001236: return "VIA_UNK_0000_1236";
+        case 0x00001237: return "VIA_UNK_0000_1237";
+        case 0x00001238: return "VIA_UNK_0000_1238";
+        case 0x00001239: return "VIA_UNK_0000_1239";
+        case 0x00001240: return "VIA_UNK_0000_1240";
+        case 0x00001241: return "VIA_UNK_0000_1241";
+        case 0x00001243: return "VIA_UNK_0000_1243";
+        case 0x00001245: return "VIA_UNK_0000_1245";
+        case 0x00001246: return "VIA_UNK_0000_1246";
+        case 0x00001247: return "VIA_UNK_0000_1247";
+        case 0x00001248: return "VIA_UNK_0000_1248";
+        case 0x00001249: return "VIA_UNK_0000_1249";
+        case 0x0000124a: return "VIA_UNK_0000_124a";
+
+        case 0x00001301: return "VIA_UNK_0000_1301";
+        case 0x00001302: return "VIA_UNK_0000_1302";
+        case 0x00001303: return "VIA_UNK_0000_1303";
+        case 0x00001304: return "VIA_UNK_0000_1304";
+        case 0x00001305: return "VIA_UNK_0000_1305";
+        case 0x00001306: return "VIA_UNK_0000_1306";
+        case 0x00001307: return "VIA_UNK_0000_1307";
+        case 0x00001308: return "VIA_UNK_0000_1308";
+        case 0x00001309: return "VIA_UNK_0000_1309";
+        case 0x0000130d: return "VIA_UNK_0000_130d";
+        case 0x0000130e: return "VIA_UNK_0000_130e";
+        case 0x00001312: return "VIA_UNK_0000_1312";
+        case 0x00001315: return "VIA_UNK_0000_1315";
+        case 0x00001317: return "VIA_UNK_0000_1317";
+        case 0x00001318: return "VIA_UNK_0000_1318";
+        case 0x0000131a: return "VIA_UNK_0000_131a";
+        case 0x0000131b: return "VIA_UNK_0000_131b";
+        case 0x00001402: return "VIA_UNK_0000_1402";
+        case 0x00001403: return "VIA_UNK_0000_1403";
+        case 0x00001404: return "VIA_UNK_0000_1404";
+        case 0x00001405: return "VIA_UNK_0000_1405";
+        case 0x00001406: return "VIA_UNK_0000_1406";
+        case 0x00001407: return "VIA_UNK_0000_1407";
+        case 0x00001410: return "VIA_UNK_0000_1410";
+        case 0x00001411: return "VIA_UNK_0000_1411";
+        case 0x00001412: return "VIA_UNK_0000_1412";
+        case 0x00001413: return "VIA_UNK_0000_1413";
+        case 0x00001414: return "VIA_UNK_0000_1414";
+        case 0x00001415: return "VIA_UNK_0000_1415";
+        case 0x00001416: return "VIA_UNK_0000_1416";
+        case 0x00001417: return "VIA_UNK_0000_1417";
+        case 0x00001418: return "VIA_UNK_0000_1418";
+        case 0x00001419: return "VIA_UNK_0000_1419";
+        case 0x0000141a: return "VIA_UNK_0000_141a";
+        case 0x0000141b: return "VIA_UNK_0000_141b";
+        case 0x0000141c: return "VIA_UNK_0000_141c";
+        case 0x0000141d: return "VIA_UNK_0000_141d";
+        case 0x0000141e: return "VIA_UNK_0000_141e";
+        case 0x0000141f: return "VIA_UNK_0000_141f";
+        case 0x00001420: return "VIA_UNK_0000_1420";
+        case 0x00001421: return "VIA_UNK_0000_1421";
+        case 0x00001422: return "VIA_UNK_0000_1422";
+        case 0x00001423: return "VIA_UNK_0000_1423";
+        case 0x00001424: return "VIA_UNK_0000_1424";
+        case 0x00001425: return "VIA_UNK_0000_1425";
+        case 0x00001426: return "VIA_UNK_0000_1426";
+        case 0x00001427: return "VIA_UNK_0000_1427";
+        case 0x00001428: return "VIA_UNK_0000_1428";
+        case 0x00001429: return "VIA_UNK_0000_1429";
+        case 0x0000142a: return "VIA_UNK_0000_142a";
+        case 0x0000142b: return "VIA_UNK_0000_142b";
+        case 0x0000142c: return "VIA_UNK_0000_142c";
+        case 0x0000142d: return "VIA_UNK_0000_142d";
+        case 0x0000142e: return "VIA_UNK_0000_142e";
+        case 0x0000142f: return "VIA_UNK_0000_142f";
+        case 0x00001434: return "VIA_UNK_0000_1434";
+        case 0x00001435: return "VIA_UNK_0000_1435";
+        case 0x00001436: return "VIA_UNK_0000_1436";
+        case 0x00001437: return "VIA_UNK_0000_1437";
+        case 0x00001438: return "VIA_UNK_0000_1438";
+        case 0x0000143a: return "VIA_UNK_0000_143a";
+        case 0x0000143c: return "VIA_UNK_0000_143c";
+        case 0x0000143d: return "VIA_UNK_0000_143d";
+        case 0x00001440: return "VIA_UNK_0000_1440";
+        case 0x00001441: return "VIA_UNK_0000_1441";
+        case 0x00001442: return "VIA_UNK_0000_1442";
+        case 0x00001449: return "VIA_UNK_0000_1449";
+        case 0x00001450: return "VIA_UNK_0000_1450";
+        case 0x00001451: return "VIA_UNK_0000_1451";
+        case 0x00001452: return "VIA_UNK_0000_1452";
+        case 0x00001453: return "VIA_UNK_0000_1453";
+        case 0x00001460: return "VIA_UNK_0000_1460";
+        case 0x00001461: return "VIA_UNK_0000_1461";
+        case 0x00001462: return "VIA_UNK_0000_1462";
+        case 0x00001463: return "VIA_UNK_0000_1463";
+        case 0x00001465: return "VIA_UNK_0000_1465";
+        case 0x00001466: return "VIA_UNK_0000_1466";
+        case 0x00001470: return "VIA_UNK_0000_1470";
+        case 0x00001471: return "VIA_UNK_0000_1471";
+        case 0x00001480: return "VIA_UNK_0000_1480";
+        case 0x00001481: return "VIA_UNK_0000_1481";
+        case 0x00001482: return "VIA_UNK_0000_1482";
+        case 0x00001483: return "VIA_UNK_0000_1483";
+        case 0x00001484: return "VIA_UNK_0000_1484";
+        case 0x00001485: return "VIA_UNK_0000_1485";
+        case 0x00001486: return "VIA_UNK_0000_1486";
+        case 0x00001490: return "VIA_UNK_0000_1490";
+        case 0x00001491: return "VIA_UNK_0000_1491";
+        case 0x00001492: return "VIA_UNK_0000_1492";
+        case 0x00001493: return "VIA_UNK_0000_1493";
+        case 0x00001494: return "VIA_UNK_0000_1494";
+        case 0x00001495: return "VIA_UNK_0000_1495";
+        case 0x00001496: return "VIA_UNK_0000_1496";
+        case 0x00001497: return "VIA_UNK_0000_1497";
+        case 0x00001498: return "VIA_UNK_0000_1498";
+        case 0x00001499: return "VIA_UNK_0000_1499";
+        case 0x0000149a: return "VIA_UNK_0000_149a";
+        case 0x0000149b: return "VIA_UNK_0000_149b";
+        case 0x0000149c: return "VIA_UNK_0000_149c";
+        case 0x0000149f: return "VIA_UNK_0000_149f";
+        case 0x00001523: return "VIA_UNK_0000_1523";
+
+        case 0x00002000: return g_enmVendor == CPUMCPUVENDOR_INTEL ? "P6_CR0" : NULL;
+        case 0x00002002: return g_enmVendor == CPUMCPUVENDOR_INTEL ? "P6_CR2" : NULL;
+        case 0x00002003: return g_enmVendor == CPUMCPUVENDOR_INTEL ? "P6_CR3" : NULL;
+        case 0x00002004: return g_enmVendor == CPUMCPUVENDOR_INTEL ? "P6_CR4" : NULL;
+        case 0x0000203f: return g_enmVendor == CPUMCPUVENDOR_INTEL ? "P6_UNK_0000_203f" /* P6_M_Dothan. */ : NULL;
+        case 0x000020cd: return g_enmVendor == CPUMCPUVENDOR_INTEL ? "P6_UNK_0000_20cd" /* P6_M_Dothan. */ : NULL;
+        case 0x0000303f: return g_enmVendor == CPUMCPUVENDOR_INTEL ? "P6_UNK_0000_303f" /* P6_M_Dothan. */ : NULL;
+        case 0x000030cd: return g_enmVendor == CPUMCPUVENDOR_INTEL ? "P6_UNK_0000_30cd" /* P6_M_Dothan. */ : NULL;
+
+        case 0x0000317a: return "VIA_UNK_0000_317a";
+        case 0x0000317b: return "VIA_UNK_0000_317b";
+        case 0x0000317d: return "VIA_UNK_0000_317d";
+        case 0x0000317e: return "VIA_UNK_0000_317e";
+        case 0x0000317f: return "VIA_UNK_0000_317f";
+        case 0x80000198: return "VIA_UNK_8000_0198";
+
+        case 0xc0000080: return "AMD64_EFER";
+        case 0xc0000081: return "AMD64_STAR";
+        case 0xc0000082: return "AMD64_STAR64";
+        case 0xc0000083: return "AMD64_STARCOMPAT";
+        case 0xc0000084: return "AMD64_SYSCALL_FLAG_MASK";
+        case 0xc0000100: return "AMD64_FS_BASE";
+        case 0xc0000101: return "AMD64_GS_BASE";
+        case 0xc0000102: return "AMD64_KERNEL_GS_BASE";
+        case 0xc0000103: return "AMD64_TSC_AUX";
+        case 0xc0000104: return "AMD_15H_TSC_RATE";
+        case 0xc0000105: return "AMD_15H_LWP_CFG";      /* Only Family 15h? */
+        case 0xc0000106: return "AMD_15H_LWP_CBADDR";   /* Only Family 15h? */
+        case 0xc0000408: return "AMD_10H_MC4_MISC1";
+        case 0xc0000409: return "AMD_10H_MC4_MISC2";
+        case 0xc000040a: return "AMD_10H_MC4_MISC3";
+        case 0xc000040b: return "AMD_10H_MC4_MISC4";
+        case 0xc000040c: return "AMD_10H_MC4_MISC5";
+        case 0xc000040d: return "AMD_10H_MC4_MISC6";
+        case 0xc000040e: return "AMD_10H_MC4_MISC7";
+        case 0xc000040f: return "AMD_10H_MC4_MISC8";
+        case 0xc0010000: return "AMD_K8_PERF_CTL_0";
+        case 0xc0010001: return "AMD_K8_PERF_CTL_1";
+        case 0xc0010002: return "AMD_K8_PERF_CTL_2";
+        case 0xc0010003: return "AMD_K8_PERF_CTL_3";
+        case 0xc0010004: return "AMD_K8_PERF_CTR_0";
+        case 0xc0010005: return "AMD_K8_PERF_CTR_1";
+        case 0xc0010006: return "AMD_K8_PERF_CTR_2";
+        case 0xc0010007: return "AMD_K8_PERF_CTR_3";
+        case 0xc0010010: return "AMD_K8_SYS_CFG";
+        case 0xc0010015: return "AMD_K8_HW_CFG";
+        case 0xc0010016: return "AMD_K8_IORR_BASE_0";
+        case 0xc0010017: return "AMD_K8_IORR_MASK_0";
+        case 0xc0010018: return "AMD_K8_IORR_BASE_1";
+        case 0xc0010019: return "AMD_K8_IORR_MASK_1";
+        case 0xc001001a: return "AMD_K8_TOP_MEM";
+        case 0xc001001d: return "AMD_K8_TOP_MEM2";
+        case 0xc001001e: return "AMD_K8_MANID";
+        case 0xc001001f: return "AMD_K8_NB_CFG1";
+        case 0xc0010020: return "AMD_K8_PATCH_LOADER";
+        case 0xc0010021: return "AMD_K8_UNK_c001_0021";
+        case 0xc0010022: return "AMD_K8_MC_XCPT_REDIR";
+        case 0xc0010028: return "AMD_K8_UNK_c001_0028";
+        case 0xc0010029: return "AMD_K8_UNK_c001_0029";
+        case 0xc001002a: return "AMD_K8_UNK_c001_002a";
+        case 0xc001002b: return "AMD_K8_UNK_c001_002b";
+        case 0xc001002c: return "AMD_K8_UNK_c001_002c";
+        case 0xc001002d: return "AMD_K8_UNK_c001_002d";
+        case 0xc0010030: return "AMD_K8_CPU_NAME_0";
+        case 0xc0010031: return "AMD_K8_CPU_NAME_1";
+        case 0xc0010032: return "AMD_K8_CPU_NAME_2";
+        case 0xc0010033: return "AMD_K8_CPU_NAME_3";
+        case 0xc0010034: return "AMD_K8_CPU_NAME_4";
+        case 0xc0010035: return "AMD_K8_CPU_NAME_5";
+        case 0xc001003e: return "AMD_K8_HTC";
+        case 0xc001003f: return "AMD_K8_STC";
+        case 0xc0010041: return "AMD_K8_FIDVID_CTL";
+        case 0xc0010042: return "AMD_K8_FIDVID_STATUS";
+        case 0xc0010043: return "AMD_K8_THERMTRIP_STATUS"; /* BDKG says it was removed in K8 revision C.*/
+        case 0xc0010044: return "AMD_K8_MC_CTL_MASK_0";
+        case 0xc0010045: return "AMD_K8_MC_CTL_MASK_1";
+        case 0xc0010046: return "AMD_K8_MC_CTL_MASK_2";
+        case 0xc0010047: return "AMD_K8_MC_CTL_MASK_3";
+        case 0xc0010048: return "AMD_K8_MC_CTL_MASK_4";
+        case 0xc0010049: return "AMD_K8_MC_CTL_MASK_5";
+        case 0xc001004a: return "AMD_K8_MC_CTL_MASK_6";
+        //case 0xc001004b: return "AMD_K8_MC_CTL_MASK_7";
+        case 0xc0010050: return "AMD_K8_SMI_ON_IO_TRAP_0";
+        case 0xc0010051: return "AMD_K8_SMI_ON_IO_TRAP_1";
+        case 0xc0010052: return "AMD_K8_SMI_ON_IO_TRAP_2";
+        case 0xc0010053: return "AMD_K8_SMI_ON_IO_TRAP_3";
+        case 0xc0010054: return "AMD_K8_SMI_ON_IO_TRAP_CTL_STS";
+        case 0xc0010055: return "AMD_K8_INT_PENDING_MSG";
+        case 0xc0010056: return "AMD_K8_SMI_TRIGGER_IO_CYCLE";
+        case 0xc0010057: return "AMD_10H_UNK_c001_0057";
+        case 0xc0010058: return "AMD_10H_MMIO_CFG_BASE_ADDR";
+        case 0xc0010059: return "AMD_10H_TRAP_CTL?"; /* Undocumented, only one google hit. */
+        case 0xc001005a: return "AMD_10H_UNK_c001_005a";
+        case 0xc001005b: return "AMD_10H_UNK_c001_005b";
+        case 0xc001005c: return "AMD_10H_UNK_c001_005c";
+        case 0xc001005d: return "AMD_10H_UNK_c001_005d";
+        case 0xc0010060: return "AMD_K8_BIST_RESULT";   /* BDKG says it as introduced with revision F. */
+        case 0xc0010061: return "AMD_10H_P_ST_CUR_LIM";
+        case 0xc0010062: return "AMD_10H_P_ST_CTL";
+        case 0xc0010063: return "AMD_10H_P_ST_STS";
+        case 0xc0010064: return "AMD_10H_P_ST_0";
+        case 0xc0010065: return "AMD_10H_P_ST_1";
+        case 0xc0010066: return "AMD_10H_P_ST_2";
+        case 0xc0010067: return "AMD_10H_P_ST_3";
+        case 0xc0010068: return "AMD_10H_P_ST_4";
+        case 0xc0010069: return "AMD_10H_P_ST_5";
+        case 0xc001006a: return "AMD_10H_P_ST_6";
+        case 0xc001006b: return "AMD_10H_P_ST_7";
+        case 0xc0010070: return "AMD_10H_COFVID_CTL";
+        case 0xc0010071: return "AMD_10H_COFVID_STS";
+        case 0xc0010073: return "AMD_10H_C_ST_IO_BASE_ADDR";
+        case 0xc0010074: return "AMD_10H_CPU_WD_TMR_CFG";
+        // case 0xc0010075: return "AMD_15H_APML_TDP_LIM";
+        // case 0xc0010077: return "AMD_15H_CPU_PWR_IN_TDP";
+        // case 0xc0010078: return "AMD_15H_PWR_AVG_PERIOD";
+        // case 0xc0010079: return "AMD_15H_DRAM_CTR_CMD_THR";
+        // case 0xc0010080: return "AMD_16H_FSFM_ACT_CNT_0";
+        // case 0xc0010081: return "AMD_16H_FSFM_REF_CNT_0";
+        case 0xc0010111: return "AMD_K8_SMM_BASE";
+        case 0xc0010112: return "AMD_K8_SMM_ADDR";
+        case 0xc0010113: return "AMD_K8_SMM_MASK";
+        case 0xc0010114: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm_AMDV ? "AMD_K8_VM_CR"          : "AMD_K8_UNK_c001_0114";
+        case 0xc0010115: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm      ? "AMD_K8_IGNNE"          : "AMD_K8_UNK_c001_0115";
+        case 0xc0010116: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm      ? "AMD_K8_SMM_CTL"        : "AMD_K8_UNK_c001_0116";
+        case 0xc0010117: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm_AMDV ? "AMD_K8_VM_HSAVE_PA"    : "AMD_K8_UNK_c001_0117";
+        case 0xc0010118: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm_AMDV ? "AMD_10H_VM_LOCK_KEY"   : "AMD_K8_UNK_c001_0118";
+        case 0xc0010119: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm      ? "AMD_10H_SSM_LOCK_KEY"  : "AMD_K8_UNK_c001_0119";
+        case 0xc001011a: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm      ? "AMD_10H_LOCAL_SMI_STS" : "AMD_K8_UNK_c001_011a";
+        case 0xc001011b: return "AMD_K8_UNK_c001_011b";
+        case 0xc001011c: return "AMD_K8_UNK_c001_011c";
+        case 0xc0010140: return "AMD_10H_OSVW_ID_LEN";
+        case 0xc0010141: return "AMD_10H_OSVW_STS";
+        case 0xc0010200: return "AMD_K8_PERF_CTL_0";
+        case 0xc0010202: return "AMD_K8_PERF_CTL_1";
+        case 0xc0010204: return "AMD_K8_PERF_CTL_2";
+        case 0xc0010206: return "AMD_K8_PERF_CTL_3";
+        case 0xc0010208: return "AMD_K8_PERF_CTL_4";
+        case 0xc001020a: return "AMD_K8_PERF_CTL_5";
+        //case 0xc001020c: return "AMD_K8_PERF_CTL_6";
+        //case 0xc001020e: return "AMD_K8_PERF_CTL_7";
+        case 0xc0010201: return "AMD_K8_PERF_CTR_0";
+        case 0xc0010203: return "AMD_K8_PERF_CTR_1";
+        case 0xc0010205: return "AMD_K8_PERF_CTR_2";
+        case 0xc0010207: return "AMD_K8_PERF_CTR_3";
+        case 0xc0010209: return "AMD_K8_PERF_CTR_4";
+        case 0xc001020b: return "AMD_K8_PERF_CTR_5";
+        //case 0xc001020d: return "AMD_K8_PERF_CTR_6";
+        //case 0xc001020f: return "AMD_K8_PERF_CTR_7";
+        case 0xc0010230: return "AMD_16H_L2I_PERF_CTL_0";
+        case 0xc0010232: return "AMD_16H_L2I_PERF_CTL_1";
+        case 0xc0010234: return "AMD_16H_L2I_PERF_CTL_2";
+        case 0xc0010236: return "AMD_16H_L2I_PERF_CTL_3";
+        //case 0xc0010238: return "AMD_16H_L2I_PERF_CTL_4";
+        //case 0xc001023a: return "AMD_16H_L2I_PERF_CTL_5";
+        //case 0xc001030c: return "AMD_16H_L2I_PERF_CTL_6";
+        //case 0xc001023e: return "AMD_16H_L2I_PERF_CTL_7";
+        case 0xc0010231: return "AMD_16H_L2I_PERF_CTR_0";
+        case 0xc0010233: return "AMD_16H_L2I_PERF_CTR_1";
+        case 0xc0010235: return "AMD_16H_L2I_PERF_CTR_2";
+        case 0xc0010237: return "AMD_16H_L2I_PERF_CTR_3";
+        //case 0xc0010239: return "AMD_16H_L2I_PERF_CTR_4";
+        //case 0xc001023b: return "AMD_16H_L2I_PERF_CTR_5";
+        //case 0xc001023d: return "AMD_16H_L2I_PERF_CTR_6";
+        //case 0xc001023f: return "AMD_16H_L2I_PERF_CTR_7";
+        case 0xc0010240: return "AMD_15H_NB_PERF_CTL_0";
+        case 0xc0010242: return "AMD_15H_NB_PERF_CTL_1";
+        case 0xc0010244: return "AMD_15H_NB_PERF_CTL_2";
+        case 0xc0010246: return "AMD_15H_NB_PERF_CTL_3";
+        //case 0xc0010248: return "AMD_15H_NB_PERF_CTL_4";
+        //case 0xc001024a: return "AMD_15H_NB_PERF_CTL_5";
+        //case 0xc001024c: return "AMD_15H_NB_PERF_CTL_6";
+        //case 0xc001024e: return "AMD_15H_NB_PERF_CTL_7";
+        case 0xc0010241: return "AMD_15H_NB_PERF_CTR_0";
+        case 0xc0010243: return "AMD_15H_NB_PERF_CTR_1";
+        case 0xc0010245: return "AMD_15H_NB_PERF_CTR_2";
+        case 0xc0010247: return "AMD_15H_NB_PERF_CTR_3";
+        //case 0xc0010249: return "AMD_15H_NB_PERF_CTR_4";
+        //case 0xc001024b: return "AMD_15H_NB_PERF_CTR_5";
+        //case 0xc001024d: return "AMD_15H_NB_PERF_CTR_6";
+        //case 0xc001024f: return "AMD_15H_NB_PERF_CTR_7";
+        case 0xc0011000: return "AMD_K7_MCODE_CTL";
+        case 0xc0011001: return "AMD_K7_APIC_CLUSTER_ID"; /* Mentioned in BKDG (r3.00) for fam16h when describing EBL_CR_POWERON. */
+        case 0xc0011002: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AMD_K8_CPUID_CTL_STD07" : NULL;
+        case 0xc0011003: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AMD_K8_CPUID_CTL_STD06" : NULL;
+        case 0xc0011004: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AMD_K8_CPUID_CTL_STD01" : NULL;
+        case 0xc0011005: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AMD_K8_CPUID_CTL_EXT01" : NULL;
+        case 0xc0011006: return "AMD_K7_DEBUG_STS?";
+        case 0xc0011007: return "AMD_K7_BH_TRACE_BASE?";
+        case 0xc0011008: return "AMD_K7_BH_TRACE_PTR?";
+        case 0xc0011009: return "AMD_K7_BH_TRACE_LIM?";
+        case 0xc001100a: return "AMD_K7_HDT_CFG?";
+        case 0xc001100b: return "AMD_K7_FAST_FLUSH_COUNT?";
+        case 0xc001100c: return "AMD_K7_NODE_ID";
+        case 0xc001100d: return "AMD_K8_LOGICAL_CPUS_NUM?";
+        case 0xc001100e: return "AMD_K8_WRMSR_BP?";
+        case 0xc001100f: return "AMD_K8_WRMSR_BP_MASK?";
+        case 0xc0011010: return "AMD_K8_BH_TRACE_CTL?";
+        case 0xc0011011: return "AMD_K8_BH_TRACE_USRD?";
+        case 0xc0011012: return "AMD_K7_UNK_c001_1012";
+        case 0xc0011013: return "AMD_K7_UNK_c001_1013";
+        case 0xc0011014: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AMD_K8_XCPT_BP_RIP?" : "AMD_K7_MOBIL_DEBUG?";
+        case 0xc0011015: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AMD_K8_XCPT_BP_RIP_MASK?" : NULL;
+        case 0xc0011016: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AMD_K8_COND_HDT_VAL?" : NULL;
+        case 0xc0011017: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AMD_K8_COND_HDT_VAL_MASK?" : NULL;
+        case 0xc0011018: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AMD_K8_XCPT_BP_CTL?" : NULL;
+        case 0xc0011019: return g_enmMicroarch >= kCpumMicroarch_AMD_15h_Piledriver ? "AMD_16H_DR1_ADDR_MASK" : NULL;
+        case 0xc001101a: return g_enmMicroarch >= kCpumMicroarch_AMD_15h_Piledriver ? "AMD_16H_DR2_ADDR_MASK" : NULL;
+        case 0xc001101b: return g_enmMicroarch >= kCpumMicroarch_AMD_15h_Piledriver ? "AMD_16H_DR3_ADDR_MASK" : NULL;
+        case 0xc001101d: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AMD_K8_NB_BIST?" : NULL;
+        case 0xc001101e: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AMD_K8_THERMTRIP_2?" : NULL;
+        case 0xc001101f: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AMD_K8_NB_CFG?" : NULL;
+        case 0xc0011020: return "AMD_K7_LS_CFG";
+        case 0xc0011021: return "AMD_K7_IC_CFG";
+        case 0xc0011022: return "AMD_K7_DC_CFG";
+        case 0xc0011023: return CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch) ? "AMD_15H_CU_CFG" : "AMD_K7_BU_CFG";
+        case 0xc0011024: return "AMD_K7_DEBUG_CTL_2?";
+        case 0xc0011025: return "AMD_K7_DR0_DATA_MATCH?";
+        case 0xc0011026: return "AMD_K7_DR0_DATA_MATCH?";
+        case 0xc0011027: return "AMD_K7_DR0_ADDR_MASK";
+        case 0xc0011028: return g_enmMicroarch >= kCpumMicroarch_AMD_15h_First ? "AMD_15H_FP_CFG"
+                              : CPUMMICROARCH_IS_AMD_FAM_10H(g_enmMicroarch)   ? "AMD_10H_UNK_c001_1028"
+                              : NULL;
+        case 0xc0011029: return g_enmMicroarch >= kCpumMicroarch_AMD_15h_First ? "AMD_15H_DC_CFG"
+                              : CPUMMICROARCH_IS_AMD_FAM_10H(g_enmMicroarch)   ? "AMD_10H_UNK_c001_1029"
+                              : NULL;
+        case 0xc001102a: return CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch)   ? "AMD_15H_CU_CFG2"
+                              : CPUMMICROARCH_IS_AMD_FAM_10H(g_enmMicroarch) || g_enmMicroarch > kCpumMicroarch_AMD_15h_End
+                              ? "AMD_10H_BU_CFG2" /* 10h & 16h */ : NULL;
+        case 0xc001102b: return CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch)   ? "AMD_15H_CU_CFG3" : NULL;
+        case 0xc001102c: return CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch)   ? "AMD_15H_EX_CFG" : NULL;
+        case 0xc001102d: return CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch)   ? "AMD_15H_LS_CFG2" : NULL;
+        case 0xc0011030: return "AMD_10H_IBS_FETCH_CTL";
+        case 0xc0011031: return "AMD_10H_IBS_FETCH_LIN_ADDR";
+        case 0xc0011032: return "AMD_10H_IBS_FETCH_PHYS_ADDR";
+        case 0xc0011033: return "AMD_10H_IBS_OP_EXEC_CTL";
+        case 0xc0011034: return "AMD_10H_IBS_OP_RIP";
+        case 0xc0011035: return "AMD_10H_IBS_OP_DATA";
+        case 0xc0011036: return "AMD_10H_IBS_OP_DATA2";
+        case 0xc0011037: return "AMD_10H_IBS_OP_DATA3";
+        case 0xc0011038: return "AMD_10H_IBS_DC_LIN_ADDR";
+        case 0xc0011039: return "AMD_10H_IBS_DC_PHYS_ADDR";
+        case 0xc001103a: return "AMD_10H_IBS_CTL";
+        case 0xc001103b: return "AMD_14H_IBS_BR_TARGET";
+
+        case 0xc0011040: return "AMD_15H_UNK_c001_1040";
+        case 0xc0011041: return "AMD_15H_UNK_c001_1041";
+        case 0xc0011042: return "AMD_15H_UNK_c001_1042";
+        case 0xc0011043: return "AMD_15H_UNK_c001_1043";
+        case 0xc0011044: return "AMD_15H_UNK_c001_1044";
+        case 0xc0011045: return "AMD_15H_UNK_c001_1045";
+        case 0xc0011046: return "AMD_15H_UNK_c001_1046";
+        case 0xc0011047: return "AMD_15H_UNK_c001_1047";
+        case 0xc0011048: return "AMD_15H_UNK_c001_1048";
+        case 0xc0011049: return "AMD_15H_UNK_c001_1049";
+        case 0xc001104a: return "AMD_15H_UNK_c001_104a";
+        case 0xc001104b: return "AMD_15H_UNK_c001_104b";
+        case 0xc001104c: return "AMD_15H_UNK_c001_104c";
+        case 0xc001104d: return "AMD_15H_UNK_c001_104d";
+        case 0xc001104e: return "AMD_15H_UNK_c001_104e";
+        case 0xc001104f: return "AMD_15H_UNK_c001_104f";
+        case 0xc0011050: return "AMD_15H_UNK_c001_1050";
+        case 0xc0011051: return "AMD_15H_UNK_c001_1051";
+        case 0xc0011052: return "AMD_15H_UNK_c001_1052";
+        case 0xc0011053: return "AMD_15H_UNK_c001_1053";
+        case 0xc0011054: return "AMD_15H_UNK_c001_1054";
+        case 0xc0011055: return "AMD_15H_UNK_c001_1055";
+        case 0xc0011056: return "AMD_15H_UNK_c001_1056";
+        case 0xc0011057: return "AMD_15H_UNK_c001_1057";
+        case 0xc0011058: return "AMD_15H_UNK_c001_1058";
+        case 0xc0011059: return "AMD_15H_UNK_c001_1059";
+        case 0xc001105a: return "AMD_15H_UNK_c001_105a";
+        case 0xc001105b: return "AMD_15H_UNK_c001_105b";
+        case 0xc001105c: return "AMD_15H_UNK_c001_105c";
+        case 0xc001105d: return "AMD_15H_UNK_c001_105d";
+        case 0xc001105e: return "AMD_15H_UNK_c001_105e";
+        case 0xc001105f: return "AMD_15H_UNK_c001_105f";
+        case 0xc0011060: return "AMD_15H_UNK_c001_1060";
+        case 0xc0011061: return "AMD_15H_UNK_c001_1061";
+        case 0xc0011062: return "AMD_15H_UNK_c001_1062";
+        case 0xc0011063: return "AMD_15H_UNK_c001_1063";
+        case 0xc0011064: return "AMD_15H_UNK_c001_1064";
+        case 0xc0011065: return "AMD_15H_UNK_c001_1065";
+        case 0xc0011066: return "AMD_15H_UNK_c001_1066";
+        case 0xc0011067: return "AMD_15H_UNK_c001_1067";
+        case 0xc0011068: return "AMD_15H_UNK_c001_1068";
+        case 0xc0011069: return "AMD_15H_UNK_c001_1069";
+        case 0xc001106a: return "AMD_15H_UNK_c001_106a";
+        case 0xc001106b: return "AMD_15H_UNK_c001_106b";
+        case 0xc001106c: return "AMD_15H_UNK_c001_106c";
+        case 0xc001106d: return "AMD_15H_UNK_c001_106d";
+        case 0xc001106e: return "AMD_15H_UNK_c001_106e";
+        case 0xc001106f: return "AMD_15H_UNK_c001_106f";
+        case 0xc0011070: return "AMD_15H_UNK_c001_1070"; /* coreboot defines this, but with a numerical name. */
+        case 0xc0011071: return "AMD_15H_UNK_c001_1071";
+        case 0xc0011072: return "AMD_15H_UNK_c001_1072";
+        case 0xc0011073: return "AMD_15H_UNK_c001_1073";
+        case 0xc0011080: return "AMD_15H_UNK_c001_1080";
+    }
+
+    /*
+     * Uncore stuff on Sandy. Putting it here to avoid ugly microarch checks for each register.
+     * Note! These are found on model 42 (2a) but not 45 (2d), the latter is the EP variant.
+     */
+    if (g_enmMicroarch == kCpumMicroarch_Intel_Core7_SandyBridge)
+        switch (uMsr)
+        {
+            case 0x00000700: return "MSR_UNC_CBO_0_PERFEVTSEL0";
+            case 0x00000701: return "MSR_UNC_CBO_0_PERFEVTSEL1";
+            case 0x00000702: return "MSR_UNC_CBO_0_PERFEVTSEL2?";
+            case 0x00000703: return "MSR_UNC_CBO_0_PERFEVTSEL3?";
+            case 0x00000704: return "MSR_UNC_CBO_0_UNK_4";
+            case 0x00000705: return "MSR_UNC_CBO_0_UNK_5";
+            case 0x00000706: return "MSR_UNC_CBO_0_PER_CTR0";
+            case 0x00000707: return "MSR_UNC_CBO_0_PER_CTR1";
+            case 0x00000708: return "MSR_UNC_CBO_0_PER_CTR2?";
+            case 0x00000709: return "MSR_UNC_CBO_0_PER_CTR3?";
+            case 0x00000710: return "MSR_UNC_CBO_1_PERFEVTSEL0";
+            case 0x00000711: return "MSR_UNC_CBO_1_PERFEVTSEL1";
+            case 0x00000712: return "MSR_UNC_CBO_1_PERFEVTSEL2?";
+            case 0x00000713: return "MSR_UNC_CBO_1_PERFEVTSEL3?";
+            case 0x00000714: return "MSR_UNC_CBO_1_UNK_4";
+            case 0x00000715: return "MSR_UNC_CBO_1_UNK_5";
+            case 0x00000716: return "MSR_UNC_CBO_1_PER_CTR0";
+            case 0x00000717: return "MSR_UNC_CBO_1_PER_CTR1";
+            case 0x00000718: return "MSR_UNC_CBO_1_PER_CTR2?";
+            case 0x00000719: return "MSR_UNC_CBO_1_PER_CTR3?";
+            case 0x00000720: return "MSR_UNC_CBO_2_PERFEVTSEL0";
+            case 0x00000721: return "MSR_UNC_CBO_2_PERFEVTSEL1";
+            case 0x00000722: return "MSR_UNC_CBO_2_PERFEVTSEL2?";
+            case 0x00000723: return "MSR_UNC_CBO_2_PERFEVTSEL3?";
+            case 0x00000724: return "MSR_UNC_CBO_2_UNK_4";
+            case 0x00000725: return "MSR_UNC_CBO_2_UNK_5";
+            case 0x00000726: return "MSR_UNC_CBO_2_PER_CTR0";
+            case 0x00000727: return "MSR_UNC_CBO_2_PER_CTR1";
+            case 0x00000728: return "MSR_UNC_CBO_2_PER_CTR2?";
+            case 0x00000729: return "MSR_UNC_CBO_2_PER_CTR3?";
+            case 0x00000730: return "MSR_UNC_CBO_3_PERFEVTSEL0";
+            case 0x00000731: return "MSR_UNC_CBO_3_PERFEVTSEL1";
+            case 0x00000732: return "MSR_UNC_CBO_3_PERFEVTSEL2?";
+            case 0x00000733: return "MSR_UNC_CBO_3_PERFEVTSEL3?";
+            case 0x00000734: return "MSR_UNC_CBO_3_UNK_4";
+            case 0x00000735: return "MSR_UNC_CBO_3_UNK_5";
+            case 0x00000736: return "MSR_UNC_CBO_3_PER_CTR0";
+            case 0x00000737: return "MSR_UNC_CBO_3_PER_CTR1";
+            case 0x00000738: return "MSR_UNC_CBO_3_PER_CTR2?";
+            case 0x00000739: return "MSR_UNC_CBO_3_PER_CTR3?";
+            case 0x00000740: return "MSR_UNC_CBO_4_PERFEVTSEL0?";
+            case 0x00000741: return "MSR_UNC_CBO_4_PERFEVTSEL1?";
+            case 0x00000742: return "MSR_UNC_CBO_4_PERFEVTSEL2?";
+            case 0x00000743: return "MSR_UNC_CBO_4_PERFEVTSEL3?";
+            case 0x00000744: return "MSR_UNC_CBO_4_UNK_4";
+            case 0x00000745: return "MSR_UNC_CBO_4_UNK_5";
+            case 0x00000746: return "MSR_UNC_CBO_4_PER_CTR0?";
+            case 0x00000747: return "MSR_UNC_CBO_4_PER_CTR1?";
+            case 0x00000748: return "MSR_UNC_CBO_4_PER_CTR2?";
+            case 0x00000749: return "MSR_UNC_CBO_4_PER_CTR3?";
+
+        }
+
+    /*
+     * Bunch of unknown sandy bridge registers.  They might seem like the
+     * nehalem based xeon stuff, but the layout doesn't match.  I bet it's the
+     * same kind of registes though (i.e. uncore (UNC)).
+     *
+     * Kudos to Intel for keeping these a secret!  Many thanks guys!!
+     */
+    if (g_enmMicroarch == kCpumMicroarch_Intel_Core7_SandyBridge)
+        switch (uMsr)
+        {
+            case 0x00000a00: return "I7_SB_UNK_0000_0a00"; case 0x00000a01: return "I7_SB_UNK_0000_0a01";
+            case 0x00000a02: return "I7_SB_UNK_0000_0a02";
+            case 0x00000c00: return "I7_SB_UNK_0000_0c00"; case 0x00000c01: return "I7_SB_UNK_0000_0c01";
+            case 0x00000c06: return "I7_SB_UNK_0000_0c06"; case 0x00000c08: return "I7_SB_UNK_0000_0c08";
+            case 0x00000c09: return "I7_SB_UNK_0000_0c09"; case 0x00000c10: return "I7_SB_UNK_0000_0c10";
+            case 0x00000c11: return "I7_SB_UNK_0000_0c11"; case 0x00000c14: return "I7_SB_UNK_0000_0c14";
+            case 0x00000c15: return "I7_SB_UNK_0000_0c15"; case 0x00000c16: return "I7_SB_UNK_0000_0c16";
+            case 0x00000c17: return "I7_SB_UNK_0000_0c17"; case 0x00000c24: return "I7_SB_UNK_0000_0c24";
+            case 0x00000c30: return "I7_SB_UNK_0000_0c30"; case 0x00000c31: return "I7_SB_UNK_0000_0c31";
+            case 0x00000c32: return "I7_SB_UNK_0000_0c32"; case 0x00000c33: return "I7_SB_UNK_0000_0c33";
+            case 0x00000c34: return "I7_SB_UNK_0000_0c34"; case 0x00000c35: return "I7_SB_UNK_0000_0c35";
+            case 0x00000c36: return "I7_SB_UNK_0000_0c36"; case 0x00000c37: return "I7_SB_UNK_0000_0c37";
+            case 0x00000c38: return "I7_SB_UNK_0000_0c38"; case 0x00000c39: return "I7_SB_UNK_0000_0c39";
+            case 0x00000d04: return "I7_SB_UNK_0000_0d04";
+            case 0x00000d10: return "I7_SB_UNK_0000_0d10"; case 0x00000d11: return "I7_SB_UNK_0000_0d11";
+            case 0x00000d12: return "I7_SB_UNK_0000_0d12"; case 0x00000d13: return "I7_SB_UNK_0000_0d13";
+            case 0x00000d14: return "I7_SB_UNK_0000_0d14"; case 0x00000d15: return "I7_SB_UNK_0000_0d15";
+            case 0x00000d16: return "I7_SB_UNK_0000_0d16"; case 0x00000d17: return "I7_SB_UNK_0000_0d17";
+            case 0x00000d18: return "I7_SB_UNK_0000_0d18"; case 0x00000d19: return "I7_SB_UNK_0000_0d19";
+            case 0x00000d24: return "I7_SB_UNK_0000_0d24";
+            case 0x00000d30: return "I7_SB_UNK_0000_0d30"; case 0x00000d31: return "I7_SB_UNK_0000_0d31";
+            case 0x00000d32: return "I7_SB_UNK_0000_0d32"; case 0x00000d33: return "I7_SB_UNK_0000_0d33";
+            case 0x00000d34: return "I7_SB_UNK_0000_0d34"; case 0x00000d35: return "I7_SB_UNK_0000_0d35";
+            case 0x00000d36: return "I7_SB_UNK_0000_0d36"; case 0x00000d37: return "I7_SB_UNK_0000_0d37";
+            case 0x00000d38: return "I7_SB_UNK_0000_0d38"; case 0x00000d39: return "I7_SB_UNK_0000_0d39";
+            case 0x00000d44: return "I7_SB_UNK_0000_0d44";
+            case 0x00000d50: return "I7_SB_UNK_0000_0d50"; case 0x00000d51: return "I7_SB_UNK_0000_0d51";
+            case 0x00000d52: return "I7_SB_UNK_0000_0d52"; case 0x00000d53: return "I7_SB_UNK_0000_0d53";
+            case 0x00000d54: return "I7_SB_UNK_0000_0d54"; case 0x00000d55: return "I7_SB_UNK_0000_0d55";
+            case 0x00000d56: return "I7_SB_UNK_0000_0d56"; case 0x00000d57: return "I7_SB_UNK_0000_0d57";
+            case 0x00000d58: return "I7_SB_UNK_0000_0d58"; case 0x00000d59: return "I7_SB_UNK_0000_0d59";
+            case 0x00000d64: return "I7_SB_UNK_0000_0d64";
+            case 0x00000d70: return "I7_SB_UNK_0000_0d70"; case 0x00000d71: return "I7_SB_UNK_0000_0d71";
+            case 0x00000d72: return "I7_SB_UNK_0000_0d72"; case 0x00000d73: return "I7_SB_UNK_0000_0d73";
+            case 0x00000d74: return "I7_SB_UNK_0000_0d74"; case 0x00000d75: return "I7_SB_UNK_0000_0d75";
+            case 0x00000d76: return "I7_SB_UNK_0000_0d76"; case 0x00000d77: return "I7_SB_UNK_0000_0d77";
+            case 0x00000d78: return "I7_SB_UNK_0000_0d78"; case 0x00000d79: return "I7_SB_UNK_0000_0d79";
+            case 0x00000d84: return "I7_SB_UNK_0000_0d84";
+            case 0x00000d90: return "I7_SB_UNK_0000_0d90"; case 0x00000d91: return "I7_SB_UNK_0000_0d91";
+            case 0x00000d92: return "I7_SB_UNK_0000_0d92"; case 0x00000d93: return "I7_SB_UNK_0000_0d93";
+            case 0x00000d94: return "I7_SB_UNK_0000_0d94"; case 0x00000d95: return "I7_SB_UNK_0000_0d95";
+            case 0x00000d96: return "I7_SB_UNK_0000_0d96"; case 0x00000d97: return "I7_SB_UNK_0000_0d97";
+            case 0x00000d98: return "I7_SB_UNK_0000_0d98"; case 0x00000d99: return "I7_SB_UNK_0000_0d99";
+            case 0x00000da4: return "I7_SB_UNK_0000_0da4";
+            case 0x00000db0: return "I7_SB_UNK_0000_0db0"; case 0x00000db1: return "I7_SB_UNK_0000_0db1";
+            case 0x00000db2: return "I7_SB_UNK_0000_0db2"; case 0x00000db3: return "I7_SB_UNK_0000_0db3";
+            case 0x00000db4: return "I7_SB_UNK_0000_0db4"; case 0x00000db5: return "I7_SB_UNK_0000_0db5";
+            case 0x00000db6: return "I7_SB_UNK_0000_0db6"; case 0x00000db7: return "I7_SB_UNK_0000_0db7";
+            case 0x00000db8: return "I7_SB_UNK_0000_0db8"; case 0x00000db9: return "I7_SB_UNK_0000_0db9";
+        }
+
+    /*
+     * Ditto for ivy bridge (observed on the i5-3570).  There are some haswell
+     * and sandybridge related docs on registers in this ares, but either
+     * things are different for ivy or they're very incomplete.  Again, kudos
+     * to intel!
+     */
+    if (g_enmMicroarch == kCpumMicroarch_Intel_Core7_IvyBridge)
+        switch (uMsr)
+        {
+            case 0x00000700: return "I7_IB_UNK_0000_0700"; case 0x00000701: return "I7_IB_UNK_0000_0701";
+            case 0x00000702: return "I7_IB_UNK_0000_0702"; case 0x00000703: return "I7_IB_UNK_0000_0703";
+            case 0x00000704: return "I7_IB_UNK_0000_0704"; case 0x00000705: return "I7_IB_UNK_0000_0705";
+            case 0x00000706: return "I7_IB_UNK_0000_0706"; case 0x00000707: return "I7_IB_UNK_0000_0707";
+            case 0x00000708: return "I7_IB_UNK_0000_0708"; case 0x00000709: return "I7_IB_UNK_0000_0709";
+            case 0x00000710: return "I7_IB_UNK_0000_0710"; case 0x00000711: return "I7_IB_UNK_0000_0711";
+            case 0x00000712: return "I7_IB_UNK_0000_0712"; case 0x00000713: return "I7_IB_UNK_0000_0713";
+            case 0x00000714: return "I7_IB_UNK_0000_0714"; case 0x00000715: return "I7_IB_UNK_0000_0715";
+            case 0x00000716: return "I7_IB_UNK_0000_0716"; case 0x00000717: return "I7_IB_UNK_0000_0717";
+            case 0x00000718: return "I7_IB_UNK_0000_0718"; case 0x00000719: return "I7_IB_UNK_0000_0719";
+            case 0x00000720: return "I7_IB_UNK_0000_0720"; case 0x00000721: return "I7_IB_UNK_0000_0721";
+            case 0x00000722: return "I7_IB_UNK_0000_0722"; case 0x00000723: return "I7_IB_UNK_0000_0723";
+            case 0x00000724: return "I7_IB_UNK_0000_0724"; case 0x00000725: return "I7_IB_UNK_0000_0725";
+            case 0x00000726: return "I7_IB_UNK_0000_0726"; case 0x00000727: return "I7_IB_UNK_0000_0727";
+            case 0x00000728: return "I7_IB_UNK_0000_0728"; case 0x00000729: return "I7_IB_UNK_0000_0729";
+            case 0x00000730: return "I7_IB_UNK_0000_0730"; case 0x00000731: return "I7_IB_UNK_0000_0731";
+            case 0x00000732: return "I7_IB_UNK_0000_0732"; case 0x00000733: return "I7_IB_UNK_0000_0733";
+            case 0x00000734: return "I7_IB_UNK_0000_0734"; case 0x00000735: return "I7_IB_UNK_0000_0735";
+            case 0x00000736: return "I7_IB_UNK_0000_0736"; case 0x00000737: return "I7_IB_UNK_0000_0737";
+            case 0x00000738: return "I7_IB_UNK_0000_0738"; case 0x00000739: return "I7_IB_UNK_0000_0739";
+            case 0x00000740: return "I7_IB_UNK_0000_0740"; case 0x00000741: return "I7_IB_UNK_0000_0741";
+            case 0x00000742: return "I7_IB_UNK_0000_0742"; case 0x00000743: return "I7_IB_UNK_0000_0743";
+            case 0x00000744: return "I7_IB_UNK_0000_0744"; case 0x00000745: return "I7_IB_UNK_0000_0745";
+            case 0x00000746: return "I7_IB_UNK_0000_0746"; case 0x00000747: return "I7_IB_UNK_0000_0747";
+            case 0x00000748: return "I7_IB_UNK_0000_0748"; case 0x00000749: return "I7_IB_UNK_0000_0749";
+
+        }
+    return NULL;
+}
+
+
+/**
+ * Gets the name of an MSR.
+ *
+ * This may return a static buffer, so the content should only be considered
+ * valid until the next time this function is called!.
+ *
+ * @returns MSR name.
+ * @param   uMsr                The MSR in question.
+ */
+static const char *getMsrName(uint32_t uMsr)
+{
+    const char *pszReadOnly = getMsrNameHandled(uMsr);
+    if (pszReadOnly)
+        return pszReadOnly;
+
+    /*
+     * This MSR needs looking into, return a TODO_XXXX_XXXX name.
+     */
+    static char s_szBuf[32];
+    RTStrPrintf(s_szBuf, sizeof(s_szBuf), "TODO_%04x_%04x", RT_HI_U16(uMsr), RT_LO_U16(uMsr));
+    return s_szBuf;
+}
+
+
+
+/**
+ * Gets the name of an MSR range.
+ *
+ * This may return a static buffer, so the content should only be considered
+ * valid until the next time this function is called!.
+ *
+ * @returns MSR name.
+ * @param   uMsr                The first MSR in the range.
+ */
+static const char *getMsrRangeName(uint32_t uMsr)
+{
+    switch (uMsr)
+    {
+        case 0x00000040:
+            return g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah ? "MSR_LASTBRANCH_n_FROM_IP" : "MSR_LASTBRANCH_n";
+        case 0x00000060:
+            if (g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah)
+                return "MSR_LASTBRANCH_n_TO_IP";
+            break;
+
+        case 0x000003f8:
+        case 0x000003f9:
+        case 0x000003fa:
+            return "I7_MSR_PKG_Cn_RESIDENCY";
+        case 0x000003fc:
+        case 0x000003fd:
+        case 0x000003fe:
+            return "I7_MSR_CORE_Cn_RESIDENCY";
+
+        case 0x00000400:
+            return "IA32_MCi_CTL_STATUS_ADDR_MISC";
+
+        case 0x00000680:
+            return "MSR_LASTBRANCH_n_FROM_IP";
+        case 0x000006c0:
+            return "MSR_LASTBRANCH_n_TO_IP";
+
+        case 0x00000800: case 0x00000801: case 0x00000802: case 0x00000803:
+        case 0x00000804: case 0x00000805: case 0x00000806: case 0x00000807:
+        case 0x00000808: case 0x00000809: case 0x0000080a: case 0x0000080b:
+        case 0x0000080c: case 0x0000080d: case 0x0000080e: case 0x0000080f:
+            return "IA32_X2APIC_n";
+    }
+
+    static char s_szBuf[96];
+    const char *pszReadOnly = getMsrNameHandled(uMsr);
+    if (pszReadOnly)
+    {
+        /*
+         * Replace the last char with 'n'.
+         */
+        RTStrCopy(s_szBuf, sizeof(s_szBuf), pszReadOnly);
+        size_t off = strlen(s_szBuf);
+        if (off > 0)
+            off--;
+        if (off + 1 < sizeof(s_szBuf))
+        {
+            s_szBuf[off] = 'n';
+            s_szBuf[off + 1] = '\0';
+        }
+    }
+    else
+    {
+        /*
+         * This MSR needs looking into, return a TODO_XXXX_XXXX_n name.
+         */
+        RTStrPrintf(s_szBuf, sizeof(s_szBuf), "TODO_%04x_%04x_n", RT_HI_U16(uMsr), RT_LO_U16(uMsr));
+    }
+    return s_szBuf;
+}
+
+
+/**
+ * Returns the function name for MSRs that have one or two.
+ *
+ * @returns Function name if applicable, NULL if not.
+ * @param   uMsr            The MSR in question.
+ * @param   pfTakesValue    Whether this MSR function takes a value or not.
+ *                          Optional.
+ */
+static const char *getMsrFnName(uint32_t uMsr, bool *pfTakesValue)
+{
+    bool fTmp;
+    if (!pfTakesValue)
+        pfTakesValue = &fTmp;
+
+    *pfTakesValue = false;
+
+    switch (uMsr)
+    {
+        case 0x00000000: return "Ia32P5McAddr";
+        case 0x00000001: return "Ia32P5McType";
+        case 0x00000006:
+            if (g_enmMicroarch >= kCpumMicroarch_Intel_First && g_enmMicroarch <= kCpumMicroarch_Intel_P6_Core_Atom_First)
+                return NULL; /* TR4 / cache tag on Pentium, but that's for later. */
+            return "Ia32MonitorFilterLineSize";
+        case 0x00000010: return "Ia32TimestampCounter";
+        case 0x00000017: *pfTakesValue = true; return "Ia32PlatformId";
+        case 0x0000001b: return "Ia32ApicBase";
+        case 0x0000002a: *pfTakesValue = true; return g_fIntelNetBurst ? "IntelP4EbcHardPowerOn" : "IntelEblCrPowerOn";
+        case 0x0000002b: *pfTakesValue = true; return g_fIntelNetBurst ? "IntelP4EbcSoftPowerOn" : NULL;
+        case 0x0000002c: *pfTakesValue = true; return g_fIntelNetBurst ? "IntelP4EbcFrequencyId" : NULL;
+        //case 0x00000033: return "IntelTestCtl";
+        case 0x00000034: return CPUMMICROARCH_IS_INTEL_CORE7(g_enmMicroarch)
+                             || CPUMMICROARCH_IS_INTEL_SILVERMONT_PLUS(g_enmMicroarch)
+                              ? "IntelI7SmiCount" : NULL;
+        case 0x00000035: return CPUMMICROARCH_IS_INTEL_CORE7(g_enmMicroarch) ? "IntelI7CoreThreadCount" : NULL;
+        case 0x0000003a: return "Ia32FeatureControl";
+
+        case 0x00000040:
+        case 0x00000041:
+        case 0x00000042:
+        case 0x00000043:
+        case 0x00000044:
+        case 0x00000045:
+        case 0x00000046:
+        case 0x00000047:
+            return "IntelLastBranchFromToN";
+
+        case 0x0000008b: return g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON
+                              ? "AmdK8PatchLevel" : "Ia32BiosSignId";
+        case 0x0000009b: return "Ia32SmmMonitorCtl";
+
+        case 0x000000a8:
+        case 0x000000a9:
+        case 0x000000aa:
+        case 0x000000ab:
+        case 0x000000ac:
+        case 0x000000ad:
+            *pfTakesValue = true;
+            return "IntelCore2EmttmCrTablesN";
+
+        case 0x000000c1:
+        case 0x000000c2:
+        case 0x000000c3:
+        case 0x000000c4:
+            return "Ia32PmcN";
+        case 0x000000c5:
+        case 0x000000c6:
+        case 0x000000c7:
+        case 0x000000c8:
+            if (g_enmMicroarch >= kCpumMicroarch_Intel_Core7_First)
+                return "Ia32PmcN";
+            return NULL;
+
+        case 0x000000cd: *pfTakesValue = true; return "IntelP6FsbFrequency";
+        case 0x000000ce: return CPUMMICROARCH_IS_INTEL_CORE7(g_enmMicroarch)  ? "IntelPlatformInfo" : NULL;
+        case 0x000000e2: return "IntelPkgCStConfigControl";
+        case 0x000000e3: return "IntelCore2SmmCStMiscInfo";
+        case 0x000000e4: return "IntelPmgIoCaptureBase";
+        case 0x000000e7: return "Ia32MPerf";
+        case 0x000000e8: return "Ia32APerf";
+        case 0x000000ee: return "IntelCore1ExtConfig";
+        case 0x000000fe: *pfTakesValue = true; return "Ia32MtrrCap";
+        case 0x00000119: *pfTakesValue = true; return "IntelBblCrCtl";
+        case 0x0000011e: *pfTakesValue = true; return "IntelBblCrCtl3";
+
+        case 0x00000130: return g_enmMicroarch == kCpumMicroarch_Intel_Core7_Westmere
+                             || g_enmMicroarch == kCpumMicroarch_Intel_Core7_Nehalem
+                              ? "IntelCpuId1FeatureMaskEcdx" : NULL;
+        case 0x00000131: return g_enmMicroarch == kCpumMicroarch_Intel_Core7_Westmere
+                             || g_enmMicroarch == kCpumMicroarch_Intel_Core7_Nehalem
+                              ? "IntelCpuId80000001FeatureMaskEcdx" : NULL;
+        case 0x00000132: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_SandyBridge
+                              ? "IntelCpuId1FeatureMaskEax" : NULL;
+        case 0x00000133: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_SandyBridge
+                              ? "IntelCpuId1FeatureMaskEcdx" : NULL;
+        case 0x00000134: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_SandyBridge
+                              ? "IntelCpuId80000001FeatureMaskEcdx" : NULL;
+        case 0x0000013c: return "IntelI7SandyAesNiCtl";
+        case 0x0000015f: return "IntelCore1DtsCalControl";
+        case 0x00000174: return "Ia32SysEnterCs";
+        case 0x00000175: return "Ia32SysEnterEsp";
+        case 0x00000176: return "Ia32SysEnterEip";
+        case 0x00000179: *pfTakesValue = true; return "Ia32McgCap";
+        case 0x0000017a: return "Ia32McgStatus";
+        case 0x0000017b: return "Ia32McgCtl";
+        case 0x0000017f: return "IntelI7SandyErrorControl"; /* SandyBridge. */
+        case 0x00000186: return "Ia32PerfEvtSelN";
+        case 0x00000187: return "Ia32PerfEvtSelN";
+        case 0x00000193: return /*g_fIntelNetBurst ? NULL :*/ NULL /* Core2_Penryn. */;
+        case 0x00000194: if (g_fIntelNetBurst) break;   *pfTakesValue = true; return "IntelFlexRatio";
+        case 0x00000198: *pfTakesValue = true; return "Ia32PerfStatus";
+        case 0x00000199: *pfTakesValue = true; return "Ia32PerfCtl";
+        case 0x0000019a: *pfTakesValue = true; return "Ia32ClockModulation";
+        case 0x0000019b: *pfTakesValue = true; return "Ia32ThermInterrupt";
+        case 0x0000019c: *pfTakesValue = true; return "Ia32ThermStatus";
+        case 0x0000019d: *pfTakesValue = true; return "Ia32Therm2Ctl";
+        case 0x000001a0: *pfTakesValue = true; return "Ia32MiscEnable";
+        case 0x000001a2: *pfTakesValue = true; return "IntelI7TemperatureTarget";
+        case 0x000001a6: return "IntelI7MsrOffCoreResponseN";
+        case 0x000001a7: return "IntelI7MsrOffCoreResponseN";
+        case 0x000001aa: return CPUMMICROARCH_IS_INTEL_CORE7(g_enmMicroarch) ? "IntelI7MiscPwrMgmt" : NULL /*"P6PicSensCfg"*/;
+        case 0x000001ad: *pfTakesValue = true; return "IntelI7TurboRatioLimit"; /* SandyBridge+, Silvermount+ */
+        case 0x000001c8: return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_Nehalem ? "IntelI7LbrSelect" : NULL;
+        case 0x000001c9: return    g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah
+                                && g_enmMicroarch <= kCpumMicroarch_Intel_P6_Core_Atom_End
+                              ? "IntelLastBranchTos" : NULL /* Pentium M Dothan seems to have something else here. */;
+        case 0x000001d7: return g_fIntelNetBurst ? "P6LastIntFromIp" : NULL;
+        case 0x000001d8: return g_fIntelNetBurst ? "P6LastIntToIp"   : NULL;
+        case 0x000001d9: return "Ia32DebugCtl";
+        case 0x000001da: return g_fIntelNetBurst ? "IntelLastBranchTos" : NULL;
+        case 0x000001db: return g_fIntelNetBurst ? "IntelLastBranchFromToN" : "P6LastBranchFromIp";
+        case 0x000001dc: return g_fIntelNetBurst ? "IntelLastBranchFromToN" : "P6LastBranchToIp";
+        case 0x000001dd: return g_fIntelNetBurst ? "IntelLastBranchFromToN" : "P6LastIntFromIp";
+        case 0x000001de: return g_fIntelNetBurst ? "IntelLastBranchFromToN" : "P6LastIntToIp";
+        case 0x000001f0: return "IntelI7VirtualLegacyWireCap"; /* SandyBridge. */
+        case 0x000001f2: return "Ia32SmrrPhysBase";
+        case 0x000001f3: return "Ia32SmrrPhysMask";
+        case 0x000001f8: return "Ia32PlatformDcaCap";
+        case 0x000001f9: return "Ia32CpuDcaCap";
+        case 0x000001fa: return "Ia32Dca0Cap";
+        case 0x000001fc: return "IntelI7PowerCtl";
+
+        case 0x00000200: case 0x00000202: case 0x00000204: case 0x00000206:
+        case 0x00000208: case 0x0000020a: case 0x0000020c: case 0x0000020e:
+        case 0x00000210: case 0x00000212: case 0x00000214: case 0x00000216:
+        case 0x00000218: case 0x0000021a: case 0x0000021c: case 0x0000021e:
+            return "Ia32MtrrPhysBaseN";
+        case 0x00000201: case 0x00000203: case 0x00000205: case 0x00000207:
+        case 0x00000209: case 0x0000020b: case 0x0000020d: case 0x0000020f:
+        case 0x00000211: case 0x00000213: case 0x00000215: case 0x00000217:
+        case 0x00000219: case 0x0000021b: case 0x0000021d: case 0x0000021f:
+            return "Ia32MtrrPhysMaskN";
+        case 0x00000250:
+        case 0x00000258: case 0x00000259:
+        case 0x00000268: case 0x00000269: case 0x0000026a: case 0x0000026b:
+        case 0x0000026c: case 0x0000026d: case 0x0000026e: case 0x0000026f:
+            return "Ia32MtrrFixed";
+        case 0x00000277: *pfTakesValue = true; return "Ia32Pat";
+
+        case 0x00000280: case 0x00000281:  case 0x00000282: case 0x00000283:
+        case 0x00000284: case 0x00000285:  case 0x00000286: case 0x00000287:
+        case 0x00000288: case 0x00000289:  case 0x0000028a: case 0x0000028b:
+        case 0x0000028c: case 0x0000028d:  case 0x0000028e: case 0x0000028f:
+        case 0x00000290: case 0x00000291:  case 0x00000292: case 0x00000293:
+        case 0x00000294: case 0x00000295:  //case 0x00000296: case 0x00000297:
+        //case 0x00000298: case 0x00000299:  case 0x0000029a: case 0x0000029b:
+        //case 0x0000029c: case 0x0000029d:  case 0x0000029e: case 0x0000029f:
+            return "Ia32McNCtl2";
+
+        case 0x000002ff: return "Ia32MtrrDefType";
+        //case 0x00000305: return g_fIntelNetBurst ? TODO : NULL;
+        case 0x00000309: return g_fIntelNetBurst ? NULL /** @todo P4 */ : "Ia32FixedCtrN";
+        case 0x0000030a: return g_fIntelNetBurst ? NULL /** @todo P4 */ : "Ia32FixedCtrN";
+        case 0x0000030b: return g_fIntelNetBurst ? NULL /** @todo P4 */ : "Ia32FixedCtrN";
+        case 0x00000345: *pfTakesValue = true; return "Ia32PerfCapabilities";
+        /* Note! Lots of P4 MSR 0x00000360..0x00000371. */
+        case 0x0000038d: return "Ia32FixedCtrCtrl";
+        case 0x0000038e: *pfTakesValue = true; return "Ia32PerfGlobalStatus";
+        case 0x0000038f: return "Ia32PerfGlobalCtrl";
+        case 0x00000390: return "Ia32PerfGlobalOvfCtrl";
+        case 0x00000391: return "IntelI7UncPerfGlobalCtrl";             /* S,H,X */
+        case 0x00000392: return "IntelI7UncPerfGlobalStatus";           /* S,H,X */
+        case 0x00000393: return "IntelI7UncPerfGlobalOvfCtrl";          /* X. ASSUMING this is the same on sandybridge and later. */
+        case 0x00000394: return g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "IntelI7UncPerfFixedCtr"  /* X */   : "IntelI7UncPerfFixedCtrCtrl"; /* >= S,H */
+        case 0x00000395: return g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "IntelI7UncPerfFixedCtrCtrl" /* X*/ : "IntelI7UncPerfFixedCtr";     /* >= S,H */
+        case 0x00000396: return g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "IntelI7UncAddrOpcodeMatch" /* X */ : "IntelI7UncCBoxConfig";       /* >= S,H */
+        case 0x0000039c: return "IntelI7SandyPebsNumAlt";
+         /* Note! Lots of P4 MSR 0x000003a0..0x000003e1. */
+        case 0x000003b0: return g_fIntelNetBurst ? NULL : g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "IntelI7UncPmcN" /* X */            : "IntelI7UncArbPerfCtrN";      /* >= S,H */
+        case 0x000003b1: return g_fIntelNetBurst ? NULL : g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "IntelI7UncPmcN" /* X */            : "IntelI7UncArbPerfCtrN";      /* >= S,H */
+        case 0x000003b2: return g_fIntelNetBurst ? NULL : g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "IntelI7UncPmcN" /* X */            : "IntelI7UncArbPerfEvtSelN";   /* >= S,H */
+        case 0x000003b3: return g_fIntelNetBurst ? NULL : g_enmMicroarch < kCpumMicroarch_Intel_Core7_SandyBridge ? "IntelI7UncPmcN" /* X */            : "IntelI7UncArbPerfEvtSelN";   /* >= S,H */
+        case 0x000003b4: case 0x000003b5: case 0x000003b6: case 0x000003b7:
+            return g_fIntelNetBurst ? NULL : "IntelI7UncPmcN";
+        case 0x000003c0: case 0x000003c1: case 0x000003c2: case 0x000003c3:
+        case 0x000003c4: case 0x000003c5: case 0x000003c6: case 0x000003c7:
+            return g_fIntelNetBurst ? NULL : "IntelI7UncPerfEvtSelN";
+        case 0x000003f1: return "Ia32PebsEnable";
+        case 0x000003f6: return g_fIntelNetBurst ? NULL /*??*/ : "IntelI7PebsLdLat";
+        case 0x000003f8: return g_fIntelNetBurst ? NULL : "IntelI7PkgCnResidencyN";
+        case 0x000003f9: return "IntelI7PkgCnResidencyN";
+        case 0x000003fa: return "IntelI7PkgCnResidencyN";
+        case 0x000003fc: return "IntelI7CoreCnResidencyN";
+        case 0x000003fd: return "IntelI7CoreCnResidencyN";
+        case 0x000003fe: return "IntelI7CoreCnResidencyN";
+
+        case 0x00000478: return g_enmMicroarch == kCpumMicroarch_Intel_Core2_Penryn ? "IntelCpuId1FeatureMaskEcdx" : NULL;
+        case 0x00000480: *pfTakesValue = true; return "Ia32VmxBasic";
+        case 0x00000481: *pfTakesValue = true; return "Ia32VmxPinbasedCtls";
+        case 0x00000482: *pfTakesValue = true; return "Ia32VmxProcbasedCtls";
+        case 0x00000483: *pfTakesValue = true; return "Ia32VmxExitCtls";
+        case 0x00000484: *pfTakesValue = true; return "Ia32VmxEntryCtls";
+        case 0x00000485: *pfTakesValue = true; return "Ia32VmxMisc";
+        case 0x00000486: *pfTakesValue = true; return "Ia32VmxCr0Fixed0";
+        case 0x00000487: *pfTakesValue = true; return "Ia32VmxCr0Fixed1";
+        case 0x00000488: *pfTakesValue = true; return "Ia32VmxCr4Fixed0";
+        case 0x00000489: *pfTakesValue = true; return "Ia32VmxCr4Fixed1";
+        case 0x0000048a: *pfTakesValue = true; return "Ia32VmxVmcsEnum";
+        case 0x0000048b: *pfTakesValue = true; return "Ia32VmxProcBasedCtls2";
+        case 0x0000048c: *pfTakesValue = true; return "Ia32VmxEptVpidCap";
+        case 0x0000048d: *pfTakesValue = true; return "Ia32VmxTruePinbasedCtls";
+        case 0x0000048e: *pfTakesValue = true; return "Ia32VmxTrueProcbasedCtls";
+        case 0x0000048f: *pfTakesValue = true; return "Ia32VmxTrueExitCtls";
+        case 0x00000490: *pfTakesValue = true; return "Ia32VmxTrueEntryCtls";
+        case 0x00000491: *pfTakesValue = true; return "Ia32VmxVmFunc";
+
+        case 0x000004c1:
+        case 0x000004c2:
+        case 0x000004c3:
+        case 0x000004c4:
+        case 0x000004c5:
+        case 0x000004c6:
+        case 0x000004c7:
+        case 0x000004c8:
+            return "Ia32PmcN";
+
+        case 0x000005a0: return "IntelCore2PeciControl"; /* Core2_Penryn. */
+
+        case 0x00000600: return "Ia32DsArea";
+        case 0x00000601: *pfTakesValue = true; return "IntelI7SandyVrCurrentConfig";
+        case 0x00000603: *pfTakesValue = true; return "IntelI7SandyVrMiscConfig";
+        case 0x00000606: *pfTakesValue = true; return "IntelI7SandyRaplPowerUnit";
+        case 0x0000060a: *pfTakesValue = true; return "IntelI7SandyPkgCnIrtlN";
+        case 0x0000060b: *pfTakesValue = true; return "IntelI7SandyPkgCnIrtlN";
+        case 0x0000060c: *pfTakesValue = true; return "IntelI7SandyPkgCnIrtlN";
+        case 0x0000060d: *pfTakesValue = true; return "IntelI7SandyPkgC2Residency";
+
+        case 0x00000610: *pfTakesValue = true; return "IntelI7RaplPkgPowerLimit";
+        case 0x00000611: *pfTakesValue = true; return "IntelI7RaplPkgEnergyStatus";
+        case 0x00000613: *pfTakesValue = true; return "IntelI7RaplPkgPerfStatus";
+        case 0x00000614: *pfTakesValue = true; return "IntelI7RaplPkgPowerInfo";
+        case 0x00000618: *pfTakesValue = true; return "IntelI7RaplDramPowerLimit";
+        case 0x00000619: *pfTakesValue = true; return "IntelI7RaplDramEnergyStatus";
+        case 0x0000061b: *pfTakesValue = true; return "IntelI7RaplDramPerfStatus";
+        case 0x0000061c: *pfTakesValue = true; return "IntelI7RaplDramPowerInfo";
+        case 0x00000638: *pfTakesValue = true; return "IntelI7RaplPp0PowerLimit";
+        case 0x00000639: *pfTakesValue = true; return "IntelI7RaplPp0EnergyStatus";
+        case 0x0000063a: *pfTakesValue = true; return "IntelI7RaplPp0Policy";
+        case 0x0000063b: *pfTakesValue = true; return "IntelI7RaplPp0PerfStatus";
+        case 0x00000640: *pfTakesValue = true; return "IntelI7RaplPp1PowerLimit";
+        case 0x00000641: *pfTakesValue = true; return "IntelI7RaplPp1EnergyStatus";
+        case 0x00000642: *pfTakesValue = true; return "IntelI7RaplPp1Policy";
+        case 0x00000648: *pfTakesValue = true; return "IntelI7IvyConfigTdpNominal";
+        case 0x00000649: *pfTakesValue = true; return "IntelI7IvyConfigTdpLevel1";
+        case 0x0000064a: *pfTakesValue = true; return "IntelI7IvyConfigTdpLevel2";
+        case 0x0000064b: return "IntelI7IvyConfigTdpControl";
+        case 0x0000064c: return "IntelI7IvyTurboActivationRatio";
+
+        case 0x00000660: return "IntelAtSilvCoreC1Recidency";
+
+        case 0x00000680: case 0x00000681: case 0x00000682: case 0x00000683:
+        case 0x00000684: case 0x00000685: case 0x00000686: case 0x00000687:
+        case 0x00000688: case 0x00000689: case 0x0000068a: case 0x0000068b:
+        case 0x0000068c: case 0x0000068d: case 0x0000068e: case 0x0000068f:
+        //case 0x00000690: case 0x00000691: case 0x00000692: case 0x00000693:
+        //case 0x00000694: case 0x00000695: case 0x00000696: case 0x00000697:
+        //case 0x00000698: case 0x00000699: case 0x0000069a: case 0x0000069b:
+        //case 0x0000069c: case 0x0000069d: case 0x0000069e: case 0x0000069f:
+            return "IntelLastBranchFromN";
+        case 0x000006c0: case 0x000006c1: case 0x000006c2: case 0x000006c3:
+        case 0x000006c4: case 0x000006c5: case 0x000006c6: case 0x000006c7:
+        case 0x000006c8: case 0x000006c9: case 0x000006ca: case 0x000006cb:
+        case 0x000006cc: case 0x000006cd: case 0x000006ce: case 0x000006cf:
+        //case 0x000006d0: case 0x000006d1: case 0x000006d2: case 0x000006d3:
+        //case 0x000006d4: case 0x000006d5: case 0x000006d6: case 0x000006d7:
+        //case 0x000006d8: case 0x000006d9: case 0x000006da: case 0x000006db:
+        //case 0x000006dc: case 0x000006dd: case 0x000006de: case 0x000006df:
+            return "IntelLastBranchToN";
+        case 0x000006e0: return "Ia32TscDeadline"; /** @todo detect this correctly! */
+
+        case 0x00000c80: return g_enmMicroarch > kCpumMicroarch_Intel_Core7_Nehalem ? "Ia32DebugInterface" : NULL;
+
+        case 0xc0000080: return "Amd64Efer";
+        case 0xc0000081: return "Amd64SyscallTarget";
+        case 0xc0000082: return "Amd64LongSyscallTarget";
+        case 0xc0000083: return "Amd64CompSyscallTarget";
+        case 0xc0000084: return "Amd64SyscallFlagMask";
+        case 0xc0000100: return "Amd64FsBase";
+        case 0xc0000101: return "Amd64GsBase";
+        case 0xc0000102: return "Amd64KernelGsBase";
+        case 0xc0000103: return "Amd64TscAux";
+        case 0xc0000104: return "AmdFam15hTscRate";
+        case 0xc0000105: return "AmdFam15hLwpCfg";
+        case 0xc0000106: return "AmdFam15hLwpCbAddr";
+        case 0xc0000408: return "AmdFam10hMc4MiscN";
+        case 0xc0000409: return "AmdFam10hMc4MiscN";
+        case 0xc000040a: return "AmdFam10hMc4MiscN";
+        case 0xc000040b: return "AmdFam10hMc4MiscN";
+        case 0xc000040c: return "AmdFam10hMc4MiscN";
+        case 0xc000040d: return "AmdFam10hMc4MiscN";
+        case 0xc000040e: return "AmdFam10hMc4MiscN";
+        case 0xc000040f: return "AmdFam10hMc4MiscN";
+        case 0xc0010000: return "AmdK8PerfCtlN";
+        case 0xc0010001: return "AmdK8PerfCtlN";
+        case 0xc0010002: return "AmdK8PerfCtlN";
+        case 0xc0010003: return "AmdK8PerfCtlN";
+        case 0xc0010004: return "AmdK8PerfCtrN";
+        case 0xc0010005: return "AmdK8PerfCtrN";
+        case 0xc0010006: return "AmdK8PerfCtrN";
+        case 0xc0010007: return "AmdK8PerfCtrN";
+        case 0xc0010010: *pfTakesValue = true; return "AmdK8SysCfg";
+        case 0xc0010015: return "AmdK8HwCr";
+        case 0xc0010016: case 0xc0010018: return "AmdK8IorrBaseN";
+        case 0xc0010017: case 0xc0010019: return "AmdK8IorrMaskN";
+        case 0xc001001a: case 0xc001001d: return "AmdK8TopOfMemN";
+        case 0xc001001f: return "AmdK8NbCfg1";
+        case 0xc0010020: return "AmdK8PatchLoader";
+        case 0xc0010022: return "AmdK8McXcptRedir";
+        case 0xc0010030: case 0xc0010031: case 0xc0010032:
+        case 0xc0010033: case 0xc0010034: case 0xc0010035:
+            return "AmdK8CpuNameN";
+        case 0xc001003e: *pfTakesValue = true; return "AmdK8HwThermalCtrl";
+        case 0xc001003f: return "AmdK8SwThermalCtrl";
+        case 0xc0010041: *pfTakesValue = true; return "AmdK8FidVidControl";
+        case 0xc0010042: *pfTakesValue = true; return "AmdK8FidVidStatus";
+        case 0xc0010044: case 0xc0010045: case 0xc0010046: case 0xc0010047:
+        case 0xc0010048: case 0xc0010049: case 0xc001004a: //case 0xc001004b:
+            return "AmdK8McCtlMaskN";
+        case 0xc0010050: case 0xc0010051: case 0xc0010052: case 0xc0010053:
+            return "AmdK8SmiOnIoTrapN";
+        case 0xc0010054: return "AmdK8SmiOnIoTrapCtlSts";
+        case 0xc0010055: return "AmdK8IntPendingMessage";
+        case 0xc0010056: return "AmdK8SmiTriggerIoCycle";
+        case 0xc0010058: return "AmdFam10hMmioCfgBaseAddr";
+        case 0xc0010059: return "AmdFam10hTrapCtlMaybe";
+        case 0xc0010061: *pfTakesValue = true; return "AmdFam10hPStateCurLimit";
+        case 0xc0010062: *pfTakesValue = true; return "AmdFam10hPStateControl";
+        case 0xc0010063: *pfTakesValue = true; return "AmdFam10hPStateStatus";
+        case 0xc0010064: case 0xc0010065: case 0xc0010066: case 0xc0010067:
+        case 0xc0010068: case 0xc0010069: case 0xc001006a: case 0xc001006b:
+            *pfTakesValue = true; return "AmdFam10hPStateN";
+        case 0xc0010070: *pfTakesValue = true; return "AmdFam10hCofVidControl";
+        case 0xc0010071: *pfTakesValue = true; return "AmdFam10hCofVidStatus";
+        case 0xc0010073: return "AmdFam10hCStateIoBaseAddr";
+        case 0xc0010074: return "AmdFam10hCpuWatchdogTimer";
+        // case 0xc0010075: return "AmdFam15hApmlTdpLimit";
+        // case 0xc0010077: return "AmdFam15hCpuPowerInTdp";
+        // case 0xc0010078: return "AmdFam15hPowerAveragingPeriod";
+        // case 0xc0010079: return "AmdFam15hDramCtrlCmdThrottle";
+        // case 0xc0010080: return "AmdFam16hFreqSensFeedbackMonActCnt0";
+        // case 0xc0010081: return "AmdFam16hFreqSensFeedbackMonRefCnt0";
+        case 0xc0010111: return "AmdK8SmmBase";     /** @todo probably misdetected ign/gp due to locking */
+        case 0xc0010112: return "AmdK8SmmAddr";     /** @todo probably misdetected ign/gp due to locking */
+        case 0xc0010113: return "AmdK8SmmMask";     /** @todo probably misdetected ign/gp due to locking */
+        case 0xc0010114: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm_AMDV ? "AmdK8VmCr" : NULL;        /** @todo probably misdetected due to locking */
+        case 0xc0010115: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm      ? "AmdK8IgnNe" : NULL;
+        case 0xc0010116: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm      ? "AmdK8SmmCtl" : NULL;
+        case 0xc0010117: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm_AMDV ? "AmdK8VmHSavePa" : NULL;   /** @todo probably misdetected due to locking */
+        case 0xc0010118: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm_AMDV ? "AmdFam10hVmLockKey" : NULL;
+        case 0xc0010119: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm      ? "AmdFam10hSmmLockKey" : NULL; /* Not documented by BKDG, found in netbsd patch. */
+        case 0xc001011a: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm      ? "AmdFam10hLocalSmiStatus" : NULL;
+        case 0xc0010140: *pfTakesValue = true; return "AmdFam10hOsVisWrkIdLength";
+        case 0xc0010141: *pfTakesValue = true; return "AmdFam10hOsVisWrkStatus";
+        case 0xc0010200: case 0xc0010202: case 0xc0010204: case 0xc0010206:
+        case 0xc0010208: case 0xc001020a: //case 0xc001020c: case 0xc001020e:
+            return "AmdK8PerfCtlN";
+        case 0xc0010201: case 0xc0010203: case 0xc0010205: case 0xc0010207:
+        case 0xc0010209: case 0xc001020b: //case 0xc001020d: case 0xc001020f:
+            return "AmdK8PerfCtrN";
+        case 0xc0010230: case 0xc0010232: case 0xc0010234: case 0xc0010236:
+        //case 0xc0010238: case 0xc001023a: case 0xc001030c: case 0xc001023e:
+            return "AmdFam16hL2IPerfCtlN";
+        case 0xc0010231: case 0xc0010233: case 0xc0010235: case 0xc0010237:
+        //case 0xc0010239: case 0xc001023b: case 0xc001023d: case 0xc001023f:
+            return "AmdFam16hL2IPerfCtrN";
+        case 0xc0010240: case 0xc0010242: case 0xc0010244: case 0xc0010246:
+        //case 0xc0010248: case 0xc001024a: case 0xc001024c: case 0xc001024e:
+            return "AmdFam15hNorthbridgePerfCtlN";
+        case 0xc0010241: case 0xc0010243: case 0xc0010245: case 0xc0010247:
+        //case 0xc0010249: case 0xc001024b: case 0xc001024d: case 0xc001024f:
+            return "AmdFam15hNorthbridgePerfCtrN";
+        case 0xc0011000: *pfTakesValue = true; return "AmdK7MicrocodeCtl";
+        case 0xc0011001: *pfTakesValue = true; return "AmdK7ClusterIdMaybe";
+        case 0xc0011002: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AmdK8CpuIdCtlStd07hEbax" : NULL;
+        case 0xc0011003: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AmdK8CpuIdCtlStd06hEcx"  : NULL;
+        case 0xc0011004: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AmdK8CpuIdCtlStd01hEdcx" : NULL;
+        case 0xc0011005: return g_enmMicroarch >= kCpumMicroarch_AMD_K8_First ? "AmdK8CpuIdCtlExt01hEdcx" : NULL;
+        case 0xc0011006: return "AmdK7DebugStatusMaybe";
+        case 0xc0011007: return "AmdK7BHTraceBaseMaybe";
+        case 0xc0011008: return "AmdK7BHTracePtrMaybe";
+        case 0xc0011009: return "AmdK7BHTraceLimitMaybe";
+        case 0xc001100a: return "AmdK7HardwareDebugToolCfgMaybe";
+        case 0xc001100b: return "AmdK7FastFlushCountMaybe";
+        case 0xc001100c: return "AmdK7NodeId"; /** @todo dunno if this was there is K7 already. Kinda doubt it. */
+        case 0xc0011019: return g_enmMicroarch >= kCpumMicroarch_AMD_15h_Piledriver ? "AmdK7DrXAddrMaskN" : NULL;
+        case 0xc001101a: return g_enmMicroarch >= kCpumMicroarch_AMD_15h_Piledriver ? "AmdK7DrXAddrMaskN" : NULL;
+        case 0xc001101b: return g_enmMicroarch >= kCpumMicroarch_AMD_15h_Piledriver ? "AmdK7DrXAddrMaskN" : NULL;
+        case 0xc0011020: return "AmdK7LoadStoreCfg";
+        case 0xc0011021: return "AmdK7InstrCacheCfg";
+        case 0xc0011022: return "AmdK7DataCacheCfg";
+        case 0xc0011023: return CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch) ? "AmdFam15hCombUnitCfg" : "AmdK7BusUnitCfg";
+        case 0xc0011024: return "AmdK7DebugCtl2Maybe";
+        case 0xc0011025: return "AmdK7Dr0DataMatchMaybe";
+        case 0xc0011026: return "AmdK7Dr0DataMaskMaybe";
+        case 0xc0011027: return "AmdK7DrXAddrMaskN";
+        case 0xc0011028: return g_enmMicroarch >= kCpumMicroarch_AMD_15h_First ? "AmdFam15hFpuCfg" : NULL;
+        case 0xc0011029: return g_enmMicroarch >= kCpumMicroarch_AMD_15h_First ? "AmdFam15hDecoderCfg" : NULL;
+        case 0xc001102a: return CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch)   ? "AmdFam15hCombUnitCfg2"
+                              : CPUMMICROARCH_IS_AMD_FAM_10H(g_enmMicroarch) || g_enmMicroarch > kCpumMicroarch_AMD_15h_End
+                              ? "AmdFam10hBusUnitCfg2" /* 10h & 16h */ : NULL;
+        case 0xc001102b: return CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch)   ? "AmdFam15hCombUnitCfg3" : NULL;
+        case 0xc001102c: return CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch)   ? "AmdFam15hExecUnitCfg" : NULL;
+        case 0xc001102d: return CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch)   ? "AmdFam15hLoadStoreCfg2" : NULL;
+        case 0xc0011030: return "AmdFam10hIbsFetchCtl";
+        case 0xc0011031: return "AmdFam10hIbsFetchLinAddr";
+        case 0xc0011032: return "AmdFam10hIbsFetchPhysAddr";
+        case 0xc0011033: return "AmdFam10hIbsOpExecCtl";
+        case 0xc0011034: return "AmdFam10hIbsOpRip";
+        case 0xc0011035: return "AmdFam10hIbsOpData";
+        case 0xc0011036: return "AmdFam10hIbsOpData2";
+        case 0xc0011037: return "AmdFam10hIbsOpData3";
+        case 0xc0011038: return "AmdFam10hIbsDcLinAddr";
+        case 0xc0011039: return "AmdFam10hIbsDcPhysAddr";
+        case 0xc001103a: return "AmdFam10hIbsCtl";
+        case 0xc001103b: return "AmdFam14hIbsBrTarget";
+    }
+    return NULL;
+}
+
+
+/**
+ * Names CPUMCPU variables that MSRs corresponds to.
+ *
+ * @returns The variable name @a uMsr corresponds to, NULL if no variable.
+ * @param   uMsr                The MSR in question.
+ */
+static const char *getMsrCpumCpuVarName(uint32_t uMsr)
+{
+    switch (uMsr)
+    {
+        case 0x00000250: return "GuestMsrs.msr.MtrrFix64K_00000";
+        case 0x00000258: return "GuestMsrs.msr.MtrrFix16K_80000";
+        case 0x00000259: return "GuestMsrs.msr.MtrrFix16K_A0000";
+        case 0x00000268: return "GuestMsrs.msr.MtrrFix4K_C0000";
+        case 0x00000269: return "GuestMsrs.msr.MtrrFix4K_C8000";
+        case 0x0000026a: return "GuestMsrs.msr.MtrrFix4K_D0000";
+        case 0x0000026b: return "GuestMsrs.msr.MtrrFix4K_D8000";
+        case 0x0000026c: return "GuestMsrs.msr.MtrrFix4K_E0000";
+        case 0x0000026d: return "GuestMsrs.msr.MtrrFix4K_E8000";
+        case 0x0000026e: return "GuestMsrs.msr.MtrrFix4K_F0000";
+        case 0x0000026f: return "GuestMsrs.msr.MtrrFix4K_F8000";
+        case 0x00000277: return "Guest.msrPAT";
+        case 0x000002ff: return "GuestMsrs.msr.MtrrDefType";
+    }
+    return NULL;
+}
+
+
+/**
+ * Checks whether the MSR should read as zero for some reason.
+ *
+ * @returns true if the register should read as zero, false if not.
+ * @param   uMsr                The MSR.
+ */
+static bool doesMsrReadAsZero(uint32_t uMsr)
+{
+    switch (uMsr)
+    {
+        case 0x00000088: return true; // "BBL_CR_D0" - RAZ until understood/needed.
+        case 0x00000089: return true; // "BBL_CR_D1" - RAZ until understood/needed.
+        case 0x0000008a: return true; // "BBL_CR_D2" - RAZ until understood/needed.
+
+        /* Non-zero, but unknown register. */
+        case 0x0000004a:
+        case 0x0000004b:
+        case 0x0000004c:
+        case 0x0000004d:
+        case 0x0000004e:
+        case 0x0000004f:
+        case 0x00000050:
+        case 0x00000051:
+        case 0x00000052:
+        case 0x00000053:
+        case 0x00000054:
+        case 0x0000008c:
+        case 0x0000008d:
+        case 0x0000008e:
+        case 0x0000008f:
+        case 0x00000090:
+        case 0xc0011011:
+            return true;
+    }
+
+    return false;
+}
+
+
+/**
+ * Gets the skip mask for the given MSR.
+ *
+ * @returns Skip mask (0 means skipping nothing).
+ * @param   uMsr                The MSR.
+ */
+static uint64_t getGenericSkipMask(uint32_t uMsr)
+{
+    switch (uMsr)
+    {
+        case 0x0000013c: return 3; /* AES-NI lock bit ++. */
+
+        case 0x000001f2: return UINT64_C(0xfffff00f); /* Ia32SmrrPhysBase - Only writable in SMM. */
+        case 0x000001f3: return UINT64_C(0xfffff800); /* Ia32SmrrPhysMask - Only writable in SMM. */
+
+        /* these two have lock bits. */
+        case 0x0000064b: return UINT64_C(0x80000003);
+        case 0x0000064c: return UINT64_C(0x800000ff);
+
+        case 0xc0010015: return 1; /* SmmLock bit */
+
+        /* SmmLock effect: */
+        case 0xc0010111: return UINT32_MAX;
+        case 0xc0010112: return UINT64_C(0xfffe0000) | ((RT_BIT_64(vbCpuRepGetPhysAddrWidth()) - 1) & ~(uint64_t)UINT32_MAX);
+        case 0xc0010113: return UINT64_C(0xfffe773f) | ((RT_BIT_64(vbCpuRepGetPhysAddrWidth()) - 1) & ~(uint64_t)UINT32_MAX);
+        case 0xc0010116: return 0x1f;
+
+        case 0xc0010114: return RT_BIT_64(3) /* SVM lock */ | RT_BIT_64(4) /* SvmeDisable */;
+
+        /* Canonical */
+        case 0xc0011034:
+        case 0xc0011038:
+        case 0xc001103b:
+            return UINT64_C(0xffff800000000000);
+
+        case 0x00000060: case 0x00000061: case 0x00000062: case 0x00000063:
+        case 0x00000064: case 0x00000065: case 0x00000066: case 0x00000067:
+        case 0x00000040: case 0x00000041: case 0x00000042: case 0x00000043:
+        case 0x00000044: case 0x00000045: case 0x00000046: case 0x00000047:
+        case 0x00000600:
+            if (g_enmMicroarch >= kCpumMicroarch_Intel_Core2_First)
+                return UINT64_C(0xffff800000000000);
+            break;
+
+
+        /* Write only bits. */
+        case 0xc0010041: return RT_BIT_64(16); /* FIDVID_CTL.InitFidVid */
+
+        /* Time counters - fudge them to avoid incorrect ignore masks. */
+        case 0x00000010:
+        case 0x000000e7:
+        case 0x000000e8:
+            return RT_BIT_32(29) - 1;
+    }
+    return 0;
+}
+
+
+
+
+/** queryMsrWriteBadness return values. */
+typedef enum
+{
+    /** . */
+    VBCPUREPBADNESS_MOSTLY_HARMLESS = 0,
+    /** Not a problem if accessed with care. */
+    VBCPUREPBADNESS_MIGHT_BITE,
+    /** Worse than a bad james bond villain. */
+    VBCPUREPBADNESS_BOND_VILLAIN
+} VBCPUREPBADNESS;
+
+
+/**
+ * Backlisting and graylisting of MSRs which may cause tripple faults.
+ *
+ * @returns Badness factor.
+ * @param   uMsr                The MSR in question.
+ */
+static VBCPUREPBADNESS queryMsrWriteBadness(uint32_t uMsr)
+{
+    /** @todo Having trouble in the 0xc0010247,0xc0011006,?? region on Bulldozer. */
+    /** @todo Having trouble in the 0xc001100f,0xc001100d,?? region on Opteron
+     *        2384. */
+
+    switch (uMsr)
+    {
+        case 0x00000050:
+        case 0x00000051:
+        case 0x00000052:
+        case 0x00000053:
+        case 0x00000054:
+
+        case 0x00001006:
+        case 0x00001007:
+            return VBCPUREPBADNESS_BOND_VILLAIN;
+
+        case 0x0000120e:
+        case 0x00001233:
+        case 0x00001239:
+        case 0x00001249:
+        case 0x0000124a:
+        case 0x00001404:
+        case 0x00001405:
+        case 0x00001413:
+        case 0x0000142c: /* Caused rip to be set to 297 or some such weirdness... */
+        case 0x0000142e:
+        case 0x00001435:
+        case 0x00001436:
+        case 0x00001438:
+        case 0x0000317f:
+            if (g_enmVendor == CPUMCPUVENDOR_VIA || g_enmVendor == CPUMCPUVENDOR_SHANGHAI)
+                return VBCPUREPBADNESS_BOND_VILLAIN;
+            break;
+
+        case 0xc0010010:
+        case 0xc0010016:
+        case 0xc0010017:
+        case 0xc0010018:
+        case 0xc0010019:
+        case 0xc001001a:
+        case 0xc001001d:
+        case 0xc0010064: /* P-state fequency, voltage, ++. */
+        case 0xc0010065: /* P-state fequency, voltage, ++. */
+        case 0xc0010066: /* P-state fequency, voltage, ++. */
+        case 0xc0010067: /* P-state fequency, voltage, ++. */
+        case 0xc0010068: /* P-state fequency, voltage, ++. */
+        case 0xc0010069: /* P-state fequency, voltage, ++. */
+        case 0xc001006a: /* P-state fequency, voltage, ++. */
+        case 0xc001006b: /* P-state fequency, voltage, ++. */
+        case 0xc0010070: /* COFVID Control. */
+        case 0xc001101e:
+        case 0xc0011021: /* IC_CFG (instruction cache configuration) */
+        case 0xc0011023: /* CU_CFG (combined unit configuration) */
+        case 0xc001102c: /* EX_CFG (execution unit configuration) */
+            return VBCPUREPBADNESS_BOND_VILLAIN;
+
+        case 0xc0011012:
+            if (CPUMMICROARCH_IS_AMD_FAM_0FH(g_enmMicroarch))
+                return VBCPUREPBADNESS_MIGHT_BITE;
+            break;
+
+        /* KVM MSRs that are unsafe to touch. */
+        case 0x00000011: /* KVM */
+        case 0x00000012: /* KVM */
+            return VBCPUREPBADNESS_BOND_VILLAIN;
+
+        /*
+         * The TSC is tricky -- writing it isn't a problem, but if we put back the original
+         * value, we'll throw it out of whack. If we're on an SMP OS that uses the TSC for timing,
+         * we'll likely kill it, especially if we can't do the modification very quickly.
+         */
+        case 0x00000010: /* IA32_TIME_STAMP_COUNTER */
+            if (!g_MsrAcc.fAtomic)
+                return VBCPUREPBADNESS_BOND_VILLAIN;
+            break;
+
+        /*
+         * The following MSRs are not safe to modify in a typical OS if we can't do it atomically,
+         * i.e. read/modify/restore without allowing any other code to execute. Everything related
+         * to syscalls will blow up in our face if we go back to userland with modified MSRs.
+         */
+//        case 0x0000001b: /* IA32_APIC_BASE */
+        case 0xc0000081: /* MSR_K6_STAR */
+        case 0xc0000082: /* AMD64_STAR64 */
+        case 0xc0000083: /* AMD64_STARCOMPAT */
+        case 0xc0000084: /* AMD64_SYSCALL_FLAG_MASK */
+        case 0xc0000100: /* AMD64_FS_BASE */
+        case 0xc0000101: /* AMD64_GS_BASE */
+        case 0xc0000102: /* AMD64_KERNEL_GS_BASE */
+            if (!g_MsrAcc.fAtomic)
+                return VBCPUREPBADNESS_MIGHT_BITE;
+            break;
+
+        case 0x000001a0: /* IA32_MISC_ENABLE */
+        case 0x00000199: /* IA32_PERF_CTL */
+            return VBCPUREPBADNESS_MIGHT_BITE;
+
+        case 0x000005a0: /* C2_PECI_CTL */
+        case 0x000005a1: /* C2_UNK_0000_05a1 */
+            if (g_enmVendor == CPUMCPUVENDOR_INTEL)
+                return VBCPUREPBADNESS_MIGHT_BITE;
+            break;
+
+        case 0x00002000: /* P6_CR0. */
+        case 0x00002003: /* P6_CR3. */
+        case 0x00002004: /* P6_CR4. */
+            if (g_enmVendor == CPUMCPUVENDOR_INTEL)
+                return VBCPUREPBADNESS_MIGHT_BITE;
+            break;
+        case 0xc0000080: /* MSR_K6_EFER */
+            return VBCPUREPBADNESS_MIGHT_BITE;
+    }
+    return VBCPUREPBADNESS_MOSTLY_HARMLESS;
+}
+
+
+/**
+ * Checks if this might be a VIA/Shanghai dummy register.
+ *
+ * @returns true if it's a dummy, false if it isn't.
+ * @param   uMsr                The MSR.
+ * @param   uValue              The value.
+ * @param   fFlags              The flags.
+ */
+static bool isMsrViaShanghaiDummy(uint32_t uMsr, uint64_t uValue, uint32_t fFlags)
+{
+    if (g_enmVendor != CPUMCPUVENDOR_VIA && g_enmVendor != CPUMCPUVENDOR_SHANGHAI)
+        return false;
+
+    if (uValue)
+        return false;
+
+    if (fFlags)
+        return false;
+
+    switch (uMsr)
+    {
+        case 0x00000010:
+        case 0x0000001b:
+        case 0x000000c1:
+        case 0x000000c2:
+        case 0x0000011e:
+        case 0x00000186:
+        case 0x00000187:
+        //case 0x00000200 ... (mtrrs will be detected)
+            return false;
+
+        case 0xc0000080:
+        case 0xc0000081:
+        case 0xc0000082:
+        case 0xc0000083:
+            if (vbCpuRepSupportsLongMode())
+                return false;
+            break;
+    }
+
+    if (uMsr >= 0x00001200 && uMsr <= 0x00003fff && queryMsrWriteBadness(uMsr) != VBCPUREPBADNESS_MOSTLY_HARMLESS)
+        return false;
+
+    if (   !msrProberModifyNoChange(uMsr)
+        && !msrProberModifyZero(uMsr))
+        return false;
+
+    uint64_t fIgnMask  = 0;
+    uint64_t fGpMask   = 0;
+    int rc = msrProberModifyBitChanges(uMsr, &fIgnMask, &fGpMask, 0);
+    if (RT_FAILURE(rc))
+        return false;
+
+    if (fIgnMask != UINT64_MAX)
+        return false;
+    if (fGpMask != 0)
+        return false;
+
+    return true;
+}
+
+
+/**
+ * Adjusts the ignore and GP masks for MSRs which contains canonical addresses.
+ *
+ * @param   uMsr                The MSR.
+ * @param   pfIgn               Pointer to the ignore mask.
+ * @param   pfGp                Pointer to the GP mask.
+ */
+static void adjustCanonicalIgnAndGpMasks(uint32_t uMsr, uint64_t *pfIgn, uint64_t *pfGp)
+{
+    RT_NOREF1(pfIgn);
+    if (!vbCpuRepSupportsLongMode())
+        return;
+    switch (uMsr)
+    {
+        case 0x00000175:
+        case 0x00000176:
+        case 0x000001da:
+        case 0x000001db:
+        case 0x000001dc:
+        case 0x000001de:
+        case 0x00000600:
+            if (*pfGp == UINT64_C(0xffff800000000000))
+                *pfGp = 0;
+            break;
+        case 0x000001dd:
+            if (*pfGp == UINT64_C(0x7fff800000000000) || *pfGp == UINT64_C(0xffff800000000000)) /* why is the top bit writable? */
+                *pfGp = 0;
+            break;
+
+        case 0xc0000082:
+        case 0xc0000083:
+        case 0xc0000100:
+        case 0xc0000101:
+        case 0xc0000102:
+            *pfGp = 0;
+            break;
+    }
+}
+
+
+
+/**
+ * Prints a 64-bit value in the best way.
+ *
+ * @param   uValue              The value.
+ */
+static void printMsrValueU64(uint64_t uValue)
+{
+    if (uValue == 0)
+        vbCpuRepPrintf(", 0");
+    else if (uValue == UINT16_MAX)
+        vbCpuRepPrintf(", UINT16_MAX");
+    else if (uValue == UINT32_MAX)
+        vbCpuRepPrintf(", UINT32_MAX");
+    else if (uValue == UINT64_MAX)
+        vbCpuRepPrintf(", UINT64_MAX");
+    else if (uValue == UINT64_C(0xffffffff00000000))
+        vbCpuRepPrintf(", ~(uint64_t)UINT32_MAX");
+    else if (uValue <= (UINT32_MAX >> 1))
+        vbCpuRepPrintf(", %#llx", uValue);
+    else if (uValue <= UINT32_MAX)
+        vbCpuRepPrintf(", UINT32_C(%#llx)", uValue);
+    else
+        vbCpuRepPrintf(", UINT64_C(%#llx)", uValue);
+}
+
+
+/**
+ * Prints the newline after an MSR line has been printed.
+ *
+ * This is used as a hook to slow down the output and make sure the remote
+ * terminal or/and output file has received the last update before we go and
+ * crash probing the next MSR.
+ */
+static void printMsrNewLine(void)
+{
+    vbCpuRepPrintf("\n");
+#if 1
+    RTThreadSleep(8);
+#endif
+}
+
+static int printMsrWriteOnly(uint32_t uMsr, const char *pszWrFnName, const char *pszAnnotation)
+{
+    if (!pszWrFnName)
+        pszWrFnName = "IgnoreWrite";
+    vbCpuRepPrintf(pszAnnotation
+                   ? "    MFN(%#010x, \"%s\", WriteOnly, %s), /* %s */"
+                   : "    MFN(%#010x, \"%s\", WriteOnly, %s),",
+                   uMsr, getMsrName(uMsr), pszWrFnName, pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrValueReadOnly(uint32_t uMsr, uint64_t uValue, const char *pszAnnotation)
+{
+    vbCpuRepPrintf("    MVO(%#010x, \"%s\"", uMsr, getMsrName(uMsr));
+    printMsrValueU64(uValue);
+    vbCpuRepPrintf("),");
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+
+static int printMsrValueIgnoreWritesNamed(uint32_t uMsr, uint64_t uValue, const char *pszName, const char *pszAnnotation)
+{
+    vbCpuRepPrintf("    MVI(%#010x, \"%s\"", uMsr, pszName);
+    printMsrValueU64(uValue);
+    vbCpuRepPrintf("),");
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrValueIgnoreWrites(uint32_t uMsr, uint64_t uValue, const char *pszAnnotation)
+{
+    return printMsrValueIgnoreWritesNamed(uMsr, uValue, getMsrName(uMsr), pszAnnotation);
+}
+
+
+static int printMsrValueExtended(uint32_t uMsr, uint64_t uValue, uint64_t fIgnMask, uint64_t fGpMask,
+                                  const char *pszAnnotation)
+{
+    vbCpuRepPrintf("    MVX(%#010x, \"%s\"", uMsr, getMsrName(uMsr));
+    printMsrValueU64(uValue);
+    printMsrValueU64(fIgnMask);
+    printMsrValueU64(fGpMask);
+    vbCpuRepPrintf("),");
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrRangeValueReadOnly(uint32_t uMsr, uint32_t uLast, uint64_t uValue, const char *pszAnnotation)
+{
+    vbCpuRepPrintf("    RVO(%#010x, %#010x, \"%s\"", uMsr, uLast, getMsrRangeName(uMsr));
+    printMsrValueU64(uValue);
+    vbCpuRepPrintf("),");
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrRangeValueIgnoreWritesNamed(uint32_t uMsr, uint32_t uLast, uint64_t uValue, const char *pszName, const char *pszAnnotation)
+{
+    vbCpuRepPrintf("    RVI(%#010x, %#010x, \"%s\"", uMsr, uLast, pszName);
+    printMsrValueU64(uValue);
+    vbCpuRepPrintf("),");
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrRangeValueIgnoreWrites(uint32_t uMsr, uint32_t uLast, uint64_t uValue, const char *pszAnnotation)
+{
+    return printMsrRangeValueIgnoreWritesNamed(uMsr, uLast, uValue, getMsrRangeName(uMsr), pszAnnotation);
+}
+
+
+static int printMsrFunction(uint32_t uMsr, const char *pszRdFnName, const char *pszWrFnName, const char *pszAnnotation)
+{
+    if (!pszRdFnName)
+        pszRdFnName = getMsrFnName(uMsr, NULL);
+    if (!pszWrFnName)
+        pszWrFnName = pszRdFnName;
+    vbCpuRepPrintf("    MFN(%#010x, \"%s\", %s, %s),", uMsr, getMsrName(uMsr), pszRdFnName, pszWrFnName);
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrFunctionReadOnly(uint32_t uMsr, const char *pszRdFnName, const char *pszAnnotation)
+{
+    if (!pszRdFnName)
+        pszRdFnName = getMsrFnName(uMsr, NULL);
+    vbCpuRepPrintf("    MFO(%#010x, \"%s\", %s),", uMsr, getMsrName(uMsr), pszRdFnName);
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrFunctionIgnoreWrites(uint32_t uMsr, const char *pszRdFnName, const char *pszAnnotation)
+{
+    if (!pszRdFnName)
+        pszRdFnName = getMsrFnName(uMsr, NULL);
+    vbCpuRepPrintf("    MFI(%#010x, \"%s\", %s),", uMsr, getMsrName(uMsr), pszRdFnName);
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrFunctionIgnoreMask(uint32_t uMsr, const char *pszRdFnName, const char *pszWrFnName,
+                                      uint64_t fIgnMask, const char *pszAnnotation)
+{
+    if (!pszRdFnName)
+        pszRdFnName = getMsrFnName(uMsr, NULL);
+    if (!pszWrFnName)
+        pszWrFnName = pszRdFnName;
+    vbCpuRepPrintf("    MFW(%#010x, \"%s\", %s, %s", uMsr, getMsrName(uMsr), pszRdFnName, pszWrFnName);
+    printMsrValueU64(fIgnMask);
+    vbCpuRepPrintf("),");
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrFunctionExtended(uint32_t uMsr, const char *pszRdFnName, const char *pszWrFnName, uint64_t uValue,
+                                    uint64_t fIgnMask, uint64_t fGpMask, const char *pszAnnotation)
+{
+    if (!pszRdFnName)
+        pszRdFnName = getMsrFnName(uMsr, NULL);
+    if (!pszWrFnName)
+        pszWrFnName = pszRdFnName;
+    vbCpuRepPrintf("    MFX(%#010x, \"%s\", %s, %s", uMsr, getMsrName(uMsr), pszRdFnName, pszWrFnName);
+    printMsrValueU64(uValue);
+    printMsrValueU64(fIgnMask);
+    printMsrValueU64(fGpMask);
+    vbCpuRepPrintf("),");
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrFunctionExtendedIdxVal(uint32_t uMsr, const char *pszRdFnName, const char *pszWrFnName, uint64_t uValue,
+                                          uint64_t fIgnMask, uint64_t fGpMask, const char *pszAnnotation)
+{
+    if (!pszRdFnName)
+        pszRdFnName = getMsrFnName(uMsr, NULL);
+    if (!pszWrFnName)
+        pszWrFnName = pszRdFnName;
+    vbCpuRepPrintf("    MFX(%#010x, \"%s\", %s, %s, %#x", uMsr, getMsrName(uMsr), pszRdFnName, pszWrFnName, uValue);
+    printMsrValueU64(fIgnMask);
+    printMsrValueU64(fGpMask);
+    vbCpuRepPrintf("),");
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrFunctionCpumCpu(uint32_t uMsr, const char *pszRdFnName, const char *pszWrFnName,
+                                   const char *pszCpumCpuStorage, const char *pszAnnotation)
+{
+    if (!pszRdFnName)
+        pszRdFnName = getMsrFnName(uMsr, NULL);
+    if (!pszWrFnName)
+        pszWrFnName = pszRdFnName;
+    if (!pszCpumCpuStorage)
+        pszCpumCpuStorage = getMsrCpumCpuVarName(uMsr);
+    if (!pszCpumCpuStorage)
+        return RTMsgErrorRc(VERR_NOT_FOUND, "Missing CPUMCPU member for %#s (%#x)\n", getMsrName(uMsr), uMsr);
+    vbCpuRepPrintf("    MFS(%#010x, \"%s\", %s, %s, %s),", uMsr, getMsrName(uMsr), pszRdFnName, pszWrFnName, pszCpumCpuStorage);
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrFunctionCpumCpuEx(uint32_t uMsr, const char *pszRdFnName, const char *pszWrFnName,
+                                     const char *pszCpumCpuStorage, uint64_t fIgnMask, uint64_t fGpMask,
+                                     const char *pszAnnotation)
+{
+    if (!pszRdFnName)
+        pszRdFnName = getMsrFnName(uMsr, NULL);
+    if (!pszWrFnName)
+        pszWrFnName = pszRdFnName;
+    if (!pszCpumCpuStorage)
+        pszCpumCpuStorage = getMsrCpumCpuVarName(uMsr);
+    if (!pszCpumCpuStorage)
+        return RTMsgErrorRc(VERR_NOT_FOUND, "Missing CPUMCPU member for %#s (%#x)\n", getMsrName(uMsr), uMsr);
+    vbCpuRepPrintf("    MFZ(%#010x, \"%s\", %s, %s, %s", uMsr, getMsrName(uMsr), pszRdFnName, pszWrFnName, pszCpumCpuStorage);
+    printMsrValueU64(fIgnMask);
+    printMsrValueU64(fGpMask);
+    vbCpuRepPrintf("),");
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrRangeFunction(uint32_t uMsr, uint32_t uLast, const char *pszRdFnName, const char *pszWrFnName,
+                                 const char *pszAnnotation)
+{
+    if (!pszRdFnName)
+        pszRdFnName = getMsrFnName(uMsr, NULL);
+    if (!pszWrFnName)
+        pszWrFnName = pszRdFnName;
+    vbCpuRepPrintf("    RFN(%#010x, %#010x, \"%s\", %s, %s),", uMsr, uLast, getMsrRangeName(uMsr), pszRdFnName, pszWrFnName);
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrRangeFunctionEx(uint32_t uMsr, uint32_t uLast, const char *pszRdFnName, const char *pszWrFnName,
+                                   uint64_t uValue, uint64_t fIgnMask, uint64_t fGpMask, const char *pszAnnotation)
+{
+    if (!pszRdFnName)
+        pszRdFnName = getMsrFnName(uMsr, NULL);
+    if (!pszWrFnName)
+        pszWrFnName = pszRdFnName;
+    vbCpuRepPrintf("    RSN(%#010x, %#010x, \"%s\", %s, %s", uMsr, uLast, getMsrRangeName(uMsr), pszRdFnName, pszWrFnName);
+    printMsrValueU64(uValue);
+    printMsrValueU64(fIgnMask);
+    printMsrValueU64(fGpMask);
+    vbCpuRepPrintf("),");
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrRangeFunctionExIdxVal(uint32_t uMsr, uint32_t uLast, const char *pszRdFnName, const char *pszWrFnName,
+                                         uint64_t uValue, uint64_t fIgnMask, uint64_t fGpMask, const char *pszAnnotation)
+{
+    if (!pszRdFnName)
+        pszRdFnName = getMsrFnName(uMsr, NULL);
+    if (!pszWrFnName)
+        pszWrFnName = pszRdFnName;
+    vbCpuRepPrintf("    RSN(%#010x, %#010x, \"%s\", %s, %s, %#x",
+                   uMsr, uLast, getMsrRangeName(uMsr), pszRdFnName, pszWrFnName, uValue);
+    printMsrValueU64(fIgnMask);
+    printMsrValueU64(fGpMask);
+    vbCpuRepPrintf("),");
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+static int printMsrAlias(uint32_t uMsr, uint32_t uTarget, const char *pszAnnotation)
+{
+    vbCpuRepPrintf("    MAL(%#010x, \"%s\", %#010x),", uMsr, getMsrName(uMsr), uTarget);
+    if (pszAnnotation)
+        vbCpuRepPrintf(" /* %s */", pszAnnotation);
+    printMsrNewLine();
+    return VINF_SUCCESS;
+}
+
+
+
+static const char *annotateValue(uint64_t uValue)
+{
+    static char s_szBuf[40];
+    if (uValue <= UINT32_MAX)
+        RTStrPrintf(s_szBuf, sizeof(s_szBuf), "value=%#llx", uValue);
+    else
+        RTStrPrintf(s_szBuf, sizeof(s_szBuf), "value=%#x`%08x", RT_HI_U32(uValue), RT_LO_U32(uValue));
+    return s_szBuf;
+}
+
+
+static const char *annotateValueExtra(const char *pszExtra, uint64_t uValue)
+{
+    static char s_szBuf[40];
+    if (uValue <= UINT32_MAX)
+        RTStrPrintf(s_szBuf, sizeof(s_szBuf), "%s value=%#llx", pszExtra, uValue);
+    else
+        RTStrPrintf(s_szBuf, sizeof(s_szBuf), "%s value=%#x`%08x", pszExtra, RT_HI_U32(uValue), RT_LO_U32(uValue));
+    return s_szBuf;
+}
+
+
+static const char *annotateIfMissingBits(uint64_t uValue, uint64_t fBits)
+{
+    static char s_szBuf[80];
+    if ((uValue & fBits) == fBits)
+        return annotateValue(uValue);
+    RTStrPrintf(s_szBuf, sizeof(s_szBuf), "XXX: Unexpected value %#llx - wanted bits %#llx to be set.", uValue, fBits);
+    return s_szBuf;
+}
+
+
+static int reportMsr_Generic(uint32_t uMsr, uint32_t fFlags, uint64_t uValue)
+{
+    int         rc;
+    bool        fTakesValue = false;
+    const char *pszFnName   = getMsrFnName(uMsr, &fTakesValue);
+
+    if (fFlags & VBCPUREPMSR_F_WRITE_ONLY)
+        rc = printMsrWriteOnly(uMsr, pszFnName, NULL);
+    else
+    {
+        bool    fReadAsZero = doesMsrReadAsZero(uMsr);
+        fTakesValue = fTakesValue && !fReadAsZero;
+
+
+        switch (queryMsrWriteBadness(uMsr))
+        {
+            /* This is what we're here for... */
+            case VBCPUREPBADNESS_MOSTLY_HARMLESS:
+            {
+                if (   msrProberModifyNoChange(uMsr)
+                    || msrProberModifyZero(uMsr))
+                {
+                    uint64_t fSkipMask = getGenericSkipMask(uMsr);
+                    uint64_t fIgnMask  = 0;
+                    uint64_t fGpMask   = 0;
+                    rc = msrProberModifyBitChanges(uMsr, &fIgnMask, &fGpMask, fSkipMask);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                    adjustCanonicalIgnAndGpMasks(uMsr, &fIgnMask, &fGpMask);
+
+                    if (pszFnName)
+                    {
+                        if (fGpMask == 0 && fIgnMask == UINT64_MAX && !fTakesValue)
+                            rc = printMsrFunctionIgnoreWrites(uMsr, pszFnName, annotateValue(uValue));
+                        else if (fGpMask == 0 && fIgnMask == 0 && (!fTakesValue || uValue == 0))
+                            rc = printMsrFunction(uMsr, pszFnName, pszFnName, annotateValue(uValue));
+                        else
+                            rc = printMsrFunctionExtended(uMsr, pszFnName, pszFnName, fTakesValue ? uValue : 0,
+                                                          fIgnMask, fGpMask, annotateValue(uValue));
+                    }
+                    else if (fGpMask == 0 && fIgnMask == UINT64_MAX)
+                        rc = printMsrValueIgnoreWrites(uMsr, fReadAsZero ? 0 : uValue, fReadAsZero ? annotateValue(uValue) : NULL);
+                    else
+                        rc = printMsrValueExtended(uMsr, fReadAsZero ? 0 : uValue, fIgnMask, fGpMask,
+                                                   fReadAsZero ? annotateValue(uValue) : NULL);
+                }
+                /* Most likely read-only. */
+                else if (pszFnName && !fTakesValue)
+                    rc = printMsrFunctionReadOnly(uMsr, pszFnName, annotateValue(uValue));
+                else if (pszFnName)
+                    rc = printMsrFunctionExtended(uMsr, pszFnName, "ReadOnly", uValue, 0, 0, annotateValue(uValue));
+                else if (fReadAsZero)
+                    rc = printMsrValueReadOnly(uMsr, 0, annotateValue(uValue));
+                else
+                    rc = printMsrValueReadOnly(uMsr, uValue, NULL);
+                break;
+            }
+
+            /* These should have special handling, so just do a simple
+               write back same value check to see if it's writable. */
+            case VBCPUREPBADNESS_MIGHT_BITE:
+                if (msrProberModifyNoChange(uMsr))
+                {
+                    if (pszFnName && !fTakesValue)
+                        rc = printMsrFunction(uMsr, pszFnName, pszFnName, annotateValueExtra("Might bite.", uValue));
+                    else if (pszFnName)
+                        rc = printMsrFunctionExtended(uMsr, pszFnName, pszFnName, uValue, 0, 0,
+                                                      annotateValueExtra("Might bite.", uValue));
+                    else if (fReadAsZero)
+                        rc = printMsrValueIgnoreWrites(uMsr, 0, annotateValueExtra("Might bite.", uValue));
+                    else
+                        rc = printMsrValueIgnoreWrites(uMsr, uValue, "Might bite.");
+                }
+                else if (pszFnName && !fTakesValue)
+                    rc = printMsrFunctionReadOnly(uMsr, pszFnName, annotateValueExtra("Might bite.", uValue));
+                else if (pszFnName)
+                    rc = printMsrFunctionExtended(uMsr, pszFnName, "ReadOnly", uValue, 0, UINT64_MAX,
+                                                  annotateValueExtra("Might bite.", uValue));
+                else if (fReadAsZero)
+                    rc = printMsrValueReadOnly(uMsr, 0, annotateValueExtra("Might bite.", uValue));
+                else
+                    rc = printMsrValueReadOnly(uMsr, uValue, "Might bite.");
+                break;
+
+
+            /* Don't try anything with these guys. */
+            case VBCPUREPBADNESS_BOND_VILLAIN:
+            default:
+                if (pszFnName && !fTakesValue)
+                    rc = printMsrFunction(uMsr, pszFnName, pszFnName, annotateValueExtra("Villain?", uValue));
+                else if (pszFnName)
+                    rc = printMsrFunctionExtended(uMsr, pszFnName, pszFnName, uValue, 0, 0,
+                                                  annotateValueExtra("Villain?", uValue));
+                else if (fReadAsZero)
+                    rc = printMsrValueIgnoreWrites(uMsr, 0, annotateValueExtra("Villain?", uValue));
+                else
+                    rc = printMsrValueIgnoreWrites(uMsr, uValue, "Villain?");
+                break;
+        }
+    }
+
+    return rc;
+}
+
+
+static int reportMsr_GenRangeFunctionEx(VBCPUREPMSR const *paMsrs, uint32_t cMsrs, uint32_t cMax, const char *pszRdWrFnName,
+                                        uint32_t uMsrBase, bool fEarlyEndOk, bool fNoIgnMask, uint64_t fSkipMask, uint32_t *pidxLoop)
+{
+    uint32_t uMsr   = paMsrs[0].uMsr;
+    uint32_t iRange = uMsr - uMsrBase;
+    Assert(cMax > iRange);
+    cMax -= iRange;
+
+    /* Resolve default function name. */
+    if (!pszRdWrFnName)
+    {
+        pszRdWrFnName = getMsrFnName(uMsr, NULL);
+        if (!pszRdWrFnName)
+            return RTMsgErrorRc(VERR_INVALID_PARAMETER, "uMsr=%#x no function name\n", uMsr);
+    }
+
+    /* Figure the possible register count. */
+    if (cMax > cMsrs)
+        cMax = cMsrs;
+    uint32_t cRegs = 1;
+    while (   cRegs < cMax
+           && paMsrs[cRegs].uMsr == uMsr + cRegs)
+        cRegs++;
+
+    /* Probe the first register and check that the others exhibit
+       the same characteristics. */
+    bool     fReadOnly0;
+    uint64_t fIgnMask0, fGpMask0;
+    int rc = msrProberModifyBasicTests(uMsr, fSkipMask, &fReadOnly0, &fIgnMask0, &fGpMask0);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    const char *pszAnnotation = NULL;
+    for (uint32_t i = 1; i < cRegs; i++)
+    {
+        bool     fReadOnlyN;
+        uint64_t fIgnMaskN, fGpMaskN;
+        rc = msrProberModifyBasicTests(paMsrs[i].uMsr, fSkipMask, &fReadOnlyN, &fIgnMaskN, &fGpMaskN);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (   fReadOnlyN != fReadOnly0
+            || (fIgnMaskN != fIgnMask0 && !fNoIgnMask)
+            || fGpMaskN   != fGpMask0)
+        {
+            if (!fEarlyEndOk && !isMsrViaShanghaiDummy(uMsr, paMsrs[i].uValue, paMsrs[i].fFlags))
+            {
+                vbCpuRepDebug("MSR %s (%#x) range ended unexpectedly early on %#x: ro=%d ign=%#llx/%#llx gp=%#llx/%#llx [N/0]\n",
+                              getMsrNameHandled(uMsr), uMsr, paMsrs[i].uMsr,
+                              fReadOnlyN, fReadOnly0, fIgnMaskN, fIgnMask0, fGpMaskN, fGpMask0);
+                pszAnnotation = "XXX: The range ended earlier than expected!";
+            }
+            cRegs = i;
+            break;
+        }
+    }
+
+    /*
+     * Report the range (or single MSR as it might be).
+     */
+    *pidxLoop += cRegs - 1;
+
+    if (fNoIgnMask)
+        fIgnMask0 = 0;
+    bool fSimple = fIgnMask0 == 0
+                && (fGpMask0 == 0 || (fGpMask0 == UINT64_MAX && fReadOnly0))
+                && iRange == 0;
+    if (cRegs == 1)
+        return printMsrFunctionExtendedIdxVal(uMsr, pszRdWrFnName, fReadOnly0 ? "ReadOnly" : pszRdWrFnName,
+                                              iRange, fIgnMask0, fGpMask0,
+                                              pszAnnotation ? pszAnnotation : annotateValue(paMsrs[0].uValue));
+    if (fSimple)
+        return printMsrRangeFunction(uMsr, uMsr + cRegs - 1,
+                                     pszRdWrFnName, fReadOnly0 ? "ReadOnly" : pszRdWrFnName, pszAnnotation);
+
+    return printMsrRangeFunctionExIdxVal(uMsr, uMsr + cRegs - 1, pszRdWrFnName, fReadOnly0 ? "ReadOnly" : pszRdWrFnName,
+                                         iRange /*uValue*/, fIgnMask0, fGpMask0, pszAnnotation);
+}
+
+
+static int reportMsr_GenRangeFunction(VBCPUREPMSR const *paMsrs, uint32_t cMsrs, uint32_t cMax, const char *pszRdWrFnName,
+                                      uint32_t *pidxLoop)
+{
+    return reportMsr_GenRangeFunctionEx(paMsrs, cMsrs, cMax, pszRdWrFnName, paMsrs[0].uMsr, false /*fEarlyEndOk*/, false /*fNoIgnMask*/,
+                                        getGenericSkipMask(paMsrs[0].uMsr), pidxLoop);
+}
+
+
+/**
+ * Generic report for an MSR implemented by functions, extended version.
+ *
+ * @returns VBox status code.
+ * @param   uMsr            The MSR.
+ * @param   pszRdWrFnName   The read/write function name, optional.
+ * @param   uValue          The MSR range value.
+ * @param   fSkipMask       Mask of bits to skip.
+ * @param   fNoGpMask       Mask of bits to remove from the GP mask after
+ *                          probing
+ * @param   pszAnnotate     Annotation.
+ */
+static int reportMsr_GenFunctionEx(uint32_t uMsr, const char *pszRdWrFnName, uint32_t uValue,
+                                   uint64_t fSkipMask, uint64_t fNoGpMask, const char *pszAnnotate)
+{
+    /* Resolve default function name. */
+    if (!pszRdWrFnName)
+    {
+        pszRdWrFnName = getMsrFnName(uMsr, NULL);
+        if (!pszRdWrFnName)
+            return RTMsgErrorRc(VERR_INVALID_PARAMETER, "uMsr=%#x no function name\n", uMsr);
+    }
+
+    /* Probe the register and report. */
+    uint64_t fIgnMask = 0;
+    uint64_t fGpMask  = 0;
+    int rc = msrProberModifyBitChanges(uMsr, &fIgnMask, &fGpMask, fSkipMask);
+    if (RT_SUCCESS(rc))
+    {
+        fGpMask &= ~fNoGpMask;
+
+        if (fGpMask == UINT64_MAX && uValue == 0 && !msrProberModifyZero(uMsr))
+            rc = printMsrFunctionReadOnly(uMsr, pszRdWrFnName, pszAnnotate);
+        else if (fIgnMask == UINT64_MAX && fGpMask == 0 && uValue == 0)
+            rc = printMsrFunctionIgnoreWrites(uMsr, pszRdWrFnName, pszAnnotate);
+        else if (fIgnMask != 0 && fGpMask == 0 && uValue == 0)
+            rc = printMsrFunctionIgnoreMask(uMsr, pszRdWrFnName, NULL, fIgnMask, pszAnnotate);
+        else if (fIgnMask == 0 && fGpMask == 0 && uValue == 0)
+            rc = printMsrFunction(uMsr, pszRdWrFnName, NULL, pszAnnotate);
+        else
+            rc = printMsrFunctionExtended(uMsr, pszRdWrFnName, NULL, uValue, fIgnMask, fGpMask, pszAnnotate);
+    }
+    return rc;
+}
+
+
+/**
+ * Reports a VIA/Shanghai dummy range.
+ *
+ * @returns VBox status code.
+ * @param   paMsrs              Pointer to the first MSR.
+ * @param   cMsrs               The number of MSRs in the array @a paMsr.
+ * @param   pidxLoop            Index variable that should be advanced to the
+ *                              last MSR entry in the range.
+ */
+static int reportMsr_ViaShanghaiDummyRange(VBCPUREPMSR const *paMsrs, uint32_t cMsrs, uint32_t *pidxLoop)
+{
+    /* Figure how many. */
+    uint32_t uMsr  = paMsrs[0].uMsr;
+    uint32_t cRegs = 1;
+    while (   cRegs < cMsrs
+           && paMsrs[cRegs].uMsr == uMsr + cRegs
+           && isMsrViaShanghaiDummy(paMsrs[cRegs].uMsr, paMsrs[cRegs].uValue, paMsrs[cRegs].fFlags))
+    {
+        cRegs++;
+        if (!(cRegs % 0x80))
+            vbCpuRepDebug("VIA dummy detection %#llx..%#llx (%#x regs)...\n", uMsr, uMsr + cRegs - 1, cRegs);
+    }
+
+    /* Advance. */
+    *pidxLoop += cRegs - 1;
+
+    /* Report it/them. */
+    char szName[80];
+    if (cRegs == 1)
+    {
+        RTStrPrintf(szName, sizeof(szName), "ZERO_%04x_%04x", RT_HI_U16(uMsr), RT_LO_U16(uMsr));
+        return printMsrValueIgnoreWritesNamed(uMsr, 0, szName, NULL);
+    }
+
+    uint32_t uMsrLast = uMsr +  cRegs - 1;
+    RTStrPrintf(szName, sizeof(szName), "ZERO_%04x_%04x_THRU_%04x_%04x",
+                RT_HI_U16(uMsr), RT_LO_U16(uMsr), RT_HI_U16(uMsrLast), RT_LO_U16(uMsrLast));
+    return printMsrRangeValueIgnoreWritesNamed(uMsr, uMsrLast, 0, szName, NULL);
+}
+
+
+/**
+ * Special function for reporting the IA32_APIC_BASE register, as it seems to be
+ * causing trouble on newer systems.
+ *
+ * @returns
+ * @param   uMsr                The MSR number.
+ * @param   uValue              The value.
+ */
+static int reportMsr_Ia32ApicBase(uint32_t uMsr, uint64_t uValue)
+{
+    /* Trouble with the generic treatment of both the "APIC Global Enable" and
+       "Enable x2APIC mode" bits on an i7-3820QM running OS X 10.8.5.  */
+    uint64_t fSkipMask = RT_BIT_64(11);
+    if (vbCpuRepSupportsX2Apic())
+        fSkipMask |= RT_BIT_64(10);
+    /* For some reason, twiddling this bit kills a Tualatin PIII-S. */
+    if (g_enmMicroarch == kCpumMicroarch_Intel_P6_III)
+        fSkipMask |= RT_BIT(9);
+
+    /* If the OS uses the APIC, we have to be super careful. */
+    if (!g_MsrAcc.fAtomic)
+        fSkipMask |= UINT64_C(0x0000000ffffff000);
+
+    return reportMsr_GenFunctionEx(uMsr, "Ia32ApicBase", uValue, fSkipMask, 0, NULL);
+}
+
+
+/**
+ * Special function for reporting the IA32_MISC_ENABLE register, as it seems to
+ * be causing trouble on newer systems.
+ *
+ * @returns
+ * @param   uMsr                The MSR number.
+ * @param   uValue              The value.
+ */
+static int reportMsr_Ia32MiscEnable(uint32_t uMsr, uint64_t uValue)
+{
+    uint64_t fSkipMask = 0;
+
+    if (   (   g_enmMicroarch >= kCpumMicroarch_Intel_Core7_Broadwell
+            && g_enmMicroarch <= kCpumMicroarch_Intel_Core7_End)
+        || (   g_enmMicroarch >= kCpumMicroarch_Intel_Atom_Airmount
+            && g_enmMicroarch <= kCpumMicroarch_Intel_Atom_End)
+       )
+    {
+        vbCpuRepPrintf("WARNING: IA32_MISC_ENABLE probing needs hacking on this CPU!\n");
+        RTThreadSleep(128);
+    }
+
+    /* If the OS is using MONITOR/MWAIT we'd better not disable it! */
+    if (!g_MsrAcc.fAtomic)
+        fSkipMask |= RT_BIT(18);
+
+    /* The no execute related flag is deadly if clear.  */
+    if (   !(uValue & MSR_IA32_MISC_ENABLE_XD_DISABLE)
+        && (   g_enmMicroarch <  kCpumMicroarch_Intel_First
+            || g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah
+            || vbCpuRepSupportsNX() ) )
+        fSkipMask |= MSR_IA32_MISC_ENABLE_XD_DISABLE;
+
+    uint64_t fIgnMask = 0;
+    uint64_t fGpMask  = 0;
+    int rc = msrProberModifyBitChanges(uMsr, &fIgnMask, &fGpMask, fSkipMask);
+    if (RT_SUCCESS(rc))
+        rc = printMsrFunctionExtended(uMsr, "Ia32MiscEnable", "Ia32MiscEnable", uValue,
+                                      fIgnMask, fGpMask, annotateValue(uValue));
+    return rc;
+}
+
+
+/**
+ * Verifies that MTRR type field works correctly in the given MSR.
+ *
+ * @returns VBox status code (failure if bad MSR behavior).
+ * @param   uMsr                The MSR.
+ * @param   iBit                The first bit of the type field (8-bit wide).
+ * @param   cExpected           The number of types expected - PAT=8, MTRR=7.
+ */
+static int msrVerifyMtrrTypeGPs(uint32_t uMsr, uint32_t iBit, uint32_t cExpected)
+{
+    uint32_t uEndTypes = 0;
+    while (uEndTypes < 255)
+    {
+        bool fGp = !msrProberModifySimpleGp(uMsr, ~(UINT64_C(0xff) << iBit), (uint64_t)uEndTypes << iBit);
+        if (!fGp && (uEndTypes == 2 || uEndTypes == 3))
+            return RTMsgErrorRc(VERR_INVALID_PARAMETER, "MTRR types %u does not cause a GP as it should. (msr %#x)\n",
+                                uEndTypes, uMsr);
+        if (fGp && uEndTypes != 2 && uEndTypes != 3)
+            break;
+        uEndTypes++;
+    }
+    if (uEndTypes != cExpected)
+        return RTMsgErrorRc(VERR_INVALID_PARAMETER, "MTRR types detected to be %#x (msr %#x). Expected %#x.\n",
+                            uEndTypes, uMsr, cExpected);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deals with the variable MTRR MSRs.
+ *
+ * @returns VBox status code.
+ * @param   paMsrs              Pointer to the first variable MTRR MSR (200h).
+ * @param   cMsrs               The number of MSRs in the array @a paMsr.
+ * @param   pidxLoop            Index variable that should be advanced to the
+ *                              last MTRR MSR entry.
+ */
+static int reportMsr_Ia32MtrrPhysBaseMaskN(VBCPUREPMSR const *paMsrs, uint32_t cMsrs, uint32_t *pidxLoop)
+{
+    uint32_t uMsr = paMsrs[0].uMsr;
+
+    /* Count them. */
+    uint32_t cRegs = 1;
+    while (   cRegs < cMsrs
+           && paMsrs[cRegs].uMsr == uMsr + cRegs
+           && !isMsrViaShanghaiDummy(paMsrs[cRegs].uMsr, paMsrs[cRegs].uValue, paMsrs[cRegs].fFlags) )
+        cRegs++;
+    if (cRegs & 1)
+        return RTMsgErrorRc(VERR_INVALID_PARAMETER, "MTRR variable MSR range is odd: cRegs=%#x\n", cRegs);
+    if (cRegs > 0x20)
+        return RTMsgErrorRc(VERR_INVALID_PARAMETER, "MTRR variable MSR range is too large: cRegs=%#x\n", cRegs);
+
+    /* Find a disabled register that we can play around with. */
+    uint32_t iGuineaPig;
+    for (iGuineaPig = 0; iGuineaPig < cRegs; iGuineaPig += 2)
+        if (!(paMsrs[iGuineaPig + 1].uValue & RT_BIT_32(11)))
+            break;
+    if (iGuineaPig >= cRegs)
+        iGuineaPig = cRegs - 2;
+    vbCpuRepDebug("iGuineaPig=%#x -> %#x\n", iGuineaPig, uMsr + iGuineaPig);
+
+    /* Probe the base.  */
+    uint64_t fIgnBase = 0;
+    uint64_t fGpBase  = 0;
+    int rc = msrProberModifyBitChanges(uMsr + iGuineaPig, &fIgnBase, &fGpBase, 0);
+    if (RT_FAILURE(rc))
+        return rc;
+    rc = msrVerifyMtrrTypeGPs(uMsr + iGuineaPig, 0, 7);
+    if (RT_FAILURE(rc))
+        return rc;
+    vbCpuRepDebug("fIgnBase=%#llx fGpBase=%#llx\n", fIgnBase, fGpBase);
+
+    /* Probing the mask is relatively straight forward. */
+    uint64_t fIgnMask = 0;
+    uint64_t fGpMask  = 0;
+    rc = msrProberModifyBitChanges(uMsr + iGuineaPig + 1, &fIgnMask, &fGpMask, 0x800); /* enabling it may cause trouble */
+    if (RT_FAILURE(rc))
+        return rc;
+    vbCpuRepDebug("fIgnMask=%#llx fGpMask=%#llx\n", fIgnMask, fGpMask);
+
+    /* Validate that the whole range subscribes to the apprimately same GP rules. */
+    for (uint32_t i = 0; i < cRegs; i += 2)
+    {
+        uint64_t fSkipBase = ~fGpBase;
+        uint64_t fSkipMask = ~fGpMask;
+        if (!(paMsrs[i + 1].uValue & RT_BIT_32(11)))
+            fSkipBase = fSkipMask = 0;
+        fSkipBase |= 0x7;           /* Always skip the type. */
+        fSkipMask |= RT_BIT_32(11); /* Always skip the enable bit. */
+
+        vbCpuRepDebug("i=%#x fSkipBase=%#llx fSkipMask=%#llx\n", i, fSkipBase, fSkipMask);
+
+        if (!(paMsrs[i + 1].uValue & RT_BIT_32(11)))
+        {
+            rc = msrVerifyMtrrTypeGPs(uMsr + iGuineaPig, 0, 7);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+
+        uint64_t fIgnBaseN = 0;
+        uint64_t fGpBaseN  = 0;
+        rc = msrProberModifyBitChanges(uMsr + i, &fIgnBaseN, &fGpBaseN, fSkipBase);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        if (   fIgnBaseN != (fIgnBase & ~fSkipBase)
+            || fGpBaseN  != (fGpBase  & ~fSkipBase) )
+            return RTMsgErrorRc(VERR_INVALID_PARAMETER,
+                                "MTRR PHYS BASE register %#x behaves differently from %#x: ign=%#llx/%#llx gp=%#llx/%#llx (fSkipBase=%#llx)\n",
+                                uMsr + i, uMsr + iGuineaPig,
+                                fIgnBaseN, fIgnBase & ~fSkipBase, fGpBaseN, fGpBase & ~fSkipBase, fSkipBase);
+
+        uint64_t fIgnMaskN = 0;
+        uint64_t fGpMaskN  = 0;
+        rc = msrProberModifyBitChanges(uMsr + i + 1, &fIgnMaskN, &fGpMaskN, fSkipMask);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (   fIgnMaskN != (fIgnMask & ~fSkipMask)
+            || fGpMaskN  != (fGpMask  & ~fSkipMask) )
+            return RTMsgErrorRc(VERR_INVALID_PARAMETER,
+                                "MTRR PHYS MASK register %#x behaves differently from %#x: ign=%#llx/%#llx gp=%#llx/%#llx (fSkipMask=%#llx)\n",
+                                uMsr + i + 1, uMsr + iGuineaPig + 1,
+                                fIgnMaskN, fIgnMask & ~fSkipMask, fGpMaskN, fGpMask & ~fSkipMask, fSkipMask);
+    }
+
+    /* Print the whole range. */
+    fGpBase &= ~(uint64_t)0x7; /* Valid type bits, see msrVerifyMtrrTypeGPs(). */
+    for (uint32_t i = 0; i < cRegs; i += 2)
+    {
+        printMsrFunctionExtendedIdxVal(uMsr + i,     "Ia32MtrrPhysBaseN", NULL, i / 2, fIgnBase, fGpBase,
+                                       annotateValue(paMsrs[i].uValue));
+        printMsrFunctionExtendedIdxVal(uMsr + i + 1, "Ia32MtrrPhysMaskN", NULL, i / 2, fIgnMask, fGpMask,
+                                       annotateValue(paMsrs[i + 1].uValue));
+    }
+
+    *pidxLoop += cRegs - 1;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deals with fixed MTRR and PAT MSRs, checking the 8 memory type fields.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR.
+ */
+static int reportMsr_Ia32MtrrFixedOrPat(uint32_t uMsr)
+{
+    /* Had a spot of trouble on an old macbook pro with core2 duo T9900 (penryn)
+       running 64-bit win81pe. Not giving PAT such a scrutiny fixes it. */
+    if (   uMsr != 0x00000277
+        || (  g_enmVendor == CPUMCPUVENDOR_INTEL
+            ? g_enmMicroarch >= kCpumMicroarch_Intel_Core7_First
+            : g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON
+            ? g_enmMicroarch != kCpumMicroarch_AMD_K8_90nm_AMDV
+            : true) )
+    {
+        /* Every 8 bytes is a type, check the type ranges one by one. */
+        for (uint32_t iBit = 0; iBit < 64; iBit += 8)
+        {
+            int rc = msrVerifyMtrrTypeGPs(uMsr, iBit, 7 + (uMsr == 0x00000277));
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+    }
+
+    return printMsrFunctionCpumCpu(uMsr, NULL, NULL, NULL, NULL);
+}
+
+
+/**
+ * Deals with IA32_MTRR_DEF_TYPE.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR.
+ */
+static int reportMsr_Ia32MtrrDefType(uint32_t uMsr)
+{
+    uint64_t fGpMask  = 0;
+    uint64_t fIgnMask = 0;
+    if (g_enmMicroarch == kCpumMicroarch_AMD_K8_90nm_AMDV)
+    {
+        /* Problematic CPU! Fake it for now. */
+        fGpMask = ~(uint64_t)0xc07;
+        fIgnMask = 0;
+    }
+    else
+    {
+        int rc = msrVerifyMtrrTypeGPs(uMsr, 0, 7);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        rc = msrProberModifyBitChanges(uMsr, &fIgnMask, &fGpMask, 0x7);
+        if (RT_FAILURE(rc))
+            return rc;
+        Assert(!(fGpMask & 7)); Assert(!(fIgnMask & 7));
+    }
+
+    return printMsrFunctionCpumCpuEx(uMsr, NULL, NULL, NULL, fIgnMask, fGpMask, NULL);
+}
+
+
+/**
+ * Deals with the Machine Check (MC) MSRs in the 400h+ area.
+ *
+ * @returns VBox status code.
+ * @param   paMsrs              Pointer to the first MC MSR (400h).
+ * @param   cMsrs               The number of MSRs in the array @a paMsr.
+ * @param   pidxLoop            Index variable that should be advanced to the
+ *                              last MC MSR entry.
+ */
+static int reportMsr_Ia32McCtlStatusAddrMiscN(VBCPUREPMSR const *paMsrs, uint32_t cMsrs, uint32_t *pidxLoop)
+{
+    uint32_t uMsr = paMsrs[0].uMsr;
+
+    /* Count them. */
+    uint32_t cRegs = 1;
+    uint32_t cDetectedRegs = 1;
+    while (   cDetectedRegs < cMsrs
+           && (   paMsrs[cDetectedRegs].uMsr == uMsr + cRegs
+               || (cRegs & 3) == 2 /* ADDR may or may not be there, depends on STATUS and CPU. */
+               || (cRegs & 3) == 3 /* MISC may or may not be there, depends on STATUS and CPU. */
+               || cRegs == 0x13 /* MC4_MISC may not be there, depends on CPU. */
+               || cRegs == 0x14 /* MC5_CTL may not be there, depends on CPU. */)
+           && cRegs < 0x7f )
+    {
+        if (paMsrs[cDetectedRegs].uMsr == uMsr + cRegs)
+            cDetectedRegs++;
+        cRegs++;
+    }
+    if (cRegs & 3)
+        return RTMsgErrorRc(VERR_INVALID_PARAMETER, "MC MSR range is odd: cRegs=%#x\n", cRegs);
+
+    /* Just report them.  We don't bother probing here as the CTL format
+       and such seems to be a lot of work to test correctly and changes between
+       cpu generations.  */
+    *pidxLoop += cDetectedRegs - 1;
+    return printMsrRangeFunction(uMsr, uMsr + cRegs - 1, "Ia32McCtlStatusAddrMiscN", NULL, NULL);
+}
+
+
+
+/**
+ * Deals with the X2APIC msrs.
+ *
+ * @returns VBox status code.
+ * @param   paMsrs              Pointer to the first X2APIC MSR.
+ * @param   cMsrs               The number of MSRs in the array @a paMsr.
+ * @param   pidxLoop            Index variable that should be advanced to the
+ *                              last X2APIC MSR entry.
+ */
+static int reportMsr_GenX2Apic(VBCPUREPMSR const *paMsrs, uint32_t cMsrs, uint32_t *pidxLoop)
+{
+    /* Advance. */
+    uint32_t cRegs = 1;
+    while (   cRegs < cMsrs
+           && paMsrs[cRegs].uMsr <= 0x8ff)
+        cRegs++;
+    *pidxLoop += cRegs - 1;
+
+    /* Just emit an X2APIC range. */
+    return printMsrRangeFunction(0x800, 0x8ff, "Ia32X2ApicN", NULL, NULL);
+}
+
+
+/**
+ * Deals carefully with the EFER register.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR number.
+ * @param   uValue              The current value.
+ */
+static int reportMsr_Amd64Efer(uint32_t uMsr, uint64_t uValue)
+{
+    uint64_t fSkipMask = 0;
+    if (vbCpuRepSupportsLongMode())
+    {
+        fSkipMask |= MSR_K6_EFER_LME;
+        if (!g_MsrAcc.fAtomic && (uValue & MSR_K6_EFER_SCE))
+            fSkipMask |= MSR_K6_EFER_SCE;
+    }
+    if (   (uValue & MSR_K6_EFER_NXE)
+        || vbCpuRepSupportsNX())
+        fSkipMask |= MSR_K6_EFER_NXE;
+
+    /* NetBurst prescott 2MB (model 4) hung or triple faulted here.  The extra
+       sleep or something seemed to help for some screwed up reason. */
+    if (g_fIntelNetBurst)
+    {
+        // This doesn't matter:
+        //fSkipMask |= MSR_K6_EFER_SCE;
+        //if (vbCpuRepSupportsLongMode())
+        //    fSkipMask |= MSR_K6_EFER_LMA;
+        //vbCpuRepDebug("EFER - netburst workaround - ignore SCE & LMA (fSkipMask=%#llx)\n", fSkipMask);
+
+        vbCpuRepDebug("EFER - netburst sleep fudge - fSkipMask=%#llx\n", fSkipMask);
+        RTThreadSleep(1000);
+    }
+
+    return reportMsr_GenFunctionEx(uMsr, NULL, uValue, fSkipMask, MSR_K6_EFER_LMA, NULL);
+}
+
+
+/**
+ * Deals with the MC4_MISCn (n >= 1) range and the following reserved MSRs.
+ *
+ * @returns VBox status code.
+ * @param   paMsrs              Pointer to the first MSR.
+ * @param   cMsrs               The number of MSRs in the array @a paMsr.
+ * @param   pidxLoop            Index variable that should be advanced to the
+ *                              last MSR entry in the range.
+ */
+static int reportMsr_AmdFam10hMc4MiscN(VBCPUREPMSR const *paMsrs, uint32_t cMsrs, uint32_t *pidxLoop)
+{
+    /* Count registers. */
+    uint32_t cRegs = 1;
+    while (   cRegs < cMsrs
+           && cRegs < 8
+           && paMsrs[cRegs].uMsr == paMsrs[0].uMsr + cRegs)
+        cRegs++;
+
+    /* Probe & report used MSRs. */
+    uint64_t fIgnMask = 0;
+    uint64_t fGpMask  = 0;
+    uint32_t cUsed    = 0;
+    while (cUsed < cRegs)
+    {
+        uint64_t fIgnMaskN = 0;
+        uint64_t fGpMaskN  = 0;
+        int rc = msrProberModifyBitChanges(paMsrs[cUsed].uMsr, &fIgnMaskN, &fGpMaskN, 0);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (fIgnMaskN == UINT64_MAX || fGpMaskN == UINT64_MAX)
+            break;
+        if (cUsed == 0)
+        {
+            fIgnMask = fIgnMaskN;
+            fGpMask  = fGpMaskN;
+        }
+        else if (   fIgnMaskN != fIgnMask
+                 || fGpMaskN  != fGpMask)
+            return RTMsgErrorRc(VERR_NOT_EQUAL, "AmdFam16hMc4MiscN mismatch: fIgn=%#llx/%#llx fGp=%#llx/%#llx uMsr=%#x\n",
+                                fIgnMaskN, fIgnMask, fGpMaskN, fGpMask, paMsrs[cUsed].uMsr);
+        cUsed++;
+    }
+    if (cUsed > 0)
+        printMsrRangeFunctionEx(paMsrs[0].uMsr, paMsrs[cUsed - 1].uMsr, "AmdFam10hMc4MiscN", NULL, 0, fIgnMask, fGpMask, NULL);
+
+    /* Probe & report reserved MSRs. */
+    uint32_t cReserved = 0;
+    while (cUsed + cReserved < cRegs)
+    {
+        fIgnMask = fGpMask = 0;
+        int rc = msrProberModifyBitChanges(paMsrs[cUsed + cReserved].uMsr, &fIgnMask, &fGpMask, 0);
+        if (RT_FAILURE(rc))
+            return rc;
+        if ((fIgnMask != UINT64_MAX && fGpMask != UINT64_MAX) || paMsrs[cUsed + cReserved].uValue)
+            return RTMsgErrorRc(VERR_NOT_EQUAL,
+                                "Unexpected reserved AmdFam16hMc4MiscN: fIgn=%#llx fGp=%#llx uMsr=%#x uValue=%#llx\n",
+                                fIgnMask, fGpMask, paMsrs[cUsed + cReserved].uMsr, paMsrs[cUsed + cReserved].uValue);
+        cReserved++;
+    }
+    if (cReserved > 0 && fIgnMask == UINT64_MAX)
+        printMsrRangeValueIgnoreWrites(paMsrs[cUsed].uMsr, paMsrs[cUsed + cReserved - 1].uMsr, 0, NULL);
+    else if (cReserved > 0 && fGpMask == UINT64_MAX)
+        printMsrRangeValueReadOnly(paMsrs[cUsed].uMsr, paMsrs[cUsed + cReserved - 1].uMsr, 0, NULL);
+
+    *pidxLoop += cRegs - 1;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deals with the AMD PERF_CTL range.
+ *
+ * @returns VBox status code.
+ * @param   paMsrs              Pointer to the first MSR.
+ * @param   cMsrs               The number of MSRs in the array @a paMsr.
+ * @param   pidxLoop            Index variable that should be advanced to the
+ *                              last MSR entry in the range.
+ */
+static int reportMsr_AmdK8PerfCtlN(VBCPUREPMSR const *paMsrs, uint32_t cMsrs, uint32_t *pidxLoop)
+{
+    uint32_t uMsr = paMsrs[0].uMsr;
+    Assert(uMsr == 0xc0010000);
+
+    /* Family 15h (bulldozer +) aliases these registers sparsely onto c001020x. */
+    if (CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch))
+    {
+        for (uint32_t i = 0; i < 4; i++)
+            printMsrAlias(uMsr + i, 0xc0010200 + i * 2, NULL);
+        *pidxLoop += 3;
+    }
+    else
+        return reportMsr_GenRangeFunction(paMsrs, cMsrs, 4, "AmdK8PerfCtlN", pidxLoop);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deals with the AMD PERF_CTR range.
+ *
+ * @returns VBox status code.
+ * @param   paMsrs              Pointer to the first MSR.
+ * @param   cMsrs               The number of MSRs in the array @a paMsr.
+ * @param   pidxLoop            Index variable that should be advanced to the
+ *                              last MSR entry in the range.
+ */
+static int reportMsr_AmdK8PerfCtrN(VBCPUREPMSR const *paMsrs, uint32_t cMsrs, uint32_t *pidxLoop)
+{
+    uint32_t uMsr = paMsrs[0].uMsr;
+    Assert(uMsr == 0xc0010004);
+
+    /* Family 15h (bulldozer +) aliases these registers sparsely onto c001020x. */
+    if (CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch))
+    {
+        for (uint32_t i = 0; i < 4; i++)
+            printMsrAlias(uMsr + i, 0xc0010201 + i * 2, NULL);
+        *pidxLoop += 3;
+    }
+    else
+        return reportMsr_GenRangeFunction(paMsrs, cMsrs, 4, "AmdK8PerfCtrN", pidxLoop);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deals carefully with the SYS_CFG register.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR number.
+ * @param   uValue              The current value.
+ */
+static int reportMsr_AmdK8SysCfg(uint32_t uMsr, uint64_t uValue)
+{
+    uint64_t fSkipMask = 0;
+
+    /* Bit 21 (MtrrTom2En) is marked reserved in family 0fh, while in family
+       10h BKDG this changes (as does the document style).  Testing this bit
+       causes bulldozer running win64 to restart, thus this special treatment. */
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_K10)
+        fSkipMask |= RT_BIT(21);
+
+    /* Turns out there are more killer bits here, at least on Opteron 2384.
+       Skipping all known bits. */
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_K8_90nm_AMDV /* Not sure when introduced - harmless? */)
+        fSkipMask |= RT_BIT(22); /* Tom2ForceMemTypeWB */
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_K8_First)
+        fSkipMask |= RT_BIT(21); /* MtrrTom2En */
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_K8_First)
+        fSkipMask |= RT_BIT(20); /* MtrrVarDramEn*/
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_K8_First)
+        fSkipMask |= RT_BIT(19); /* MtrrFixDramModEn */
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_K8_First)
+        fSkipMask |= RT_BIT(18); /* MtrrFixDramEn */
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_K8_First)
+        fSkipMask |= RT_BIT(17); /* SysUcLockEn */
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_K8_First)
+        fSkipMask |= RT_BIT(16); /* ChgToDirtyDis */
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_K8_First && g_enmMicroarch < kCpumMicroarch_AMD_15h_First)
+        fSkipMask |= RT_BIT(10); /* SetDirtyEnO */
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_K8_First && g_enmMicroarch < kCpumMicroarch_AMD_15h_First)
+        fSkipMask |= RT_BIT(9);  /* SetDirtyEnS */
+    if (   CPUMMICROARCH_IS_AMD_FAM_0FH(g_enmMicroarch)
+        || CPUMMICROARCH_IS_AMD_FAM_10H(g_enmMicroarch))
+        fSkipMask |= RT_BIT(8);  /* SetDirtyEnE */
+    if (   CPUMMICROARCH_IS_AMD_FAM_0FH(g_enmMicroarch)
+        || CPUMMICROARCH_IS_AMD_FAM_11H(g_enmMicroarch) )
+        fSkipMask |= RT_BIT(7)   /* SysVicLimit */
+                  |  RT_BIT(6)   /* SysVicLimit */
+                  |  RT_BIT(5)   /* SysVicLimit */
+                  |  RT_BIT(4)   /* SysAckLimit */
+                  |  RT_BIT(3)   /* SysAckLimit */
+                  |  RT_BIT(2)   /* SysAckLimit */
+                  |  RT_BIT(1)   /* SysAckLimit */
+                  |  RT_BIT(0)   /* SysAckLimit */;
+
+    return reportMsr_GenFunctionEx(uMsr, NULL, uValue, fSkipMask, 0, annotateValue(uValue));
+}
+
+
+/**
+ * Deals carefully with the HWCR register.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR number.
+ * @param   uValue              The current value.
+ */
+static int reportMsr_AmdK8HwCr(uint32_t uMsr, uint64_t uValue)
+{
+    uint64_t fSkipMask = 0;
+
+    /* Trouble on Opteron 2384, skip some of the known bits. */
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_K10 && !CPUMMICROARCH_IS_AMD_FAM_11H(g_enmMicroarch))
+        fSkipMask |= /*RT_BIT(10)*/ 0  /* MonMwaitUserEn */
+                  |  RT_BIT(9);  /* MonMwaitDis */
+    fSkipMask |= RT_BIT(8);      /* #IGNNE port emulation */
+    if (   CPUMMICROARCH_IS_AMD_FAM_0FH(g_enmMicroarch)
+        || CPUMMICROARCH_IS_AMD_FAM_11H(g_enmMicroarch) )
+        fSkipMask |= RT_BIT(7)   /* DisLock */
+                  |  RT_BIT(6);  /* FFDis (TLB flush filter) */
+    fSkipMask |= RT_BIT(4);      /* INVD to WBINVD */
+    fSkipMask |= RT_BIT(3);      /* TLBCACHEDIS */
+    if (   CPUMMICROARCH_IS_AMD_FAM_0FH(g_enmMicroarch)
+        || CPUMMICROARCH_IS_AMD_FAM_10H(g_enmMicroarch)
+        || CPUMMICROARCH_IS_AMD_FAM_11H(g_enmMicroarch) )
+        fSkipMask |= RT_BIT(1);  /* SLOWFENCE */
+    fSkipMask |= RT_BIT(0);      /* SMMLOCK */
+
+    return reportMsr_GenFunctionEx(uMsr, NULL, uValue, fSkipMask, 0, annotateValue(uValue));
+}
+
+
+/**
+ * Deals carefully with a IORRBasei register.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR number.
+ * @param   uValue              The current value.
+ */
+static int reportMsr_AmdK8IorrBaseN(uint32_t uMsr, uint64_t uValue)
+{
+    /* Skip know bits here, as harm seems to come from messing with them. */
+    uint64_t fSkipMask = RT_BIT(4) | RT_BIT(3);
+    fSkipMask |= (RT_BIT_64(vbCpuRepGetPhysAddrWidth()) - 1) & X86_PAGE_4K_BASE_MASK;
+    return reportMsr_GenFunctionEx(uMsr, NULL, (uMsr - 0xc0010016) / 2, fSkipMask, 0, annotateValue(uValue));
+}
+
+
+/**
+ * Deals carefully with a IORRMaski register.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR number.
+ * @param   uValue              The current value.
+ */
+static int reportMsr_AmdK8IorrMaskN(uint32_t uMsr, uint64_t uValue)
+{
+    /* Skip know bits here, as harm seems to come from messing with them. */
+    uint64_t fSkipMask = RT_BIT(11);
+    fSkipMask |= (RT_BIT_64(vbCpuRepGetPhysAddrWidth()) - 1) & X86_PAGE_4K_BASE_MASK;
+    return reportMsr_GenFunctionEx(uMsr, NULL, (uMsr - 0xc0010017) / 2, fSkipMask, 0, annotateValue(uValue));
+}
+
+
+/**
+ * Deals carefully with a IORRMaski register.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR number.
+ * @param   uValue              The current value.
+ */
+static int reportMsr_AmdK8TopMemN(uint32_t uMsr, uint64_t uValue)
+{
+    /* Skip know bits here, as harm seems to come from messing with them. */
+    uint64_t fSkipMask = (RT_BIT_64(vbCpuRepGetPhysAddrWidth()) - 1) & ~(RT_BIT_64(23) - 1);
+    return reportMsr_GenFunctionEx(uMsr, NULL, uMsr == 0xc001001d, fSkipMask, 0, annotateValue(uValue));
+}
+
+
+/**
+ * Deals with the AMD P-state config range.
+ *
+ * @returns VBox status code.
+ * @param   paMsrs              Pointer to the first MSR.
+ * @param   cMsrs               The number of MSRs in the array @a paMsr.
+ * @param   pidxLoop            Index variable that should be advanced to the
+ *                              last MSR entry in the range.
+ */
+static int reportMsr_AmdFam10hPStateN(VBCPUREPMSR const *paMsrs, uint32_t cMsrs, uint32_t *pidxLoop)
+{
+    uint32_t uMsr = paMsrs[0].uMsr;
+    AssertRelease(uMsr == 0xc0010064);
+
+    /* Count them. */
+    uint32_t cRegs = 1;
+    while (   cRegs < 8
+           && cRegs < cMsrs
+           && paMsrs[cRegs].uMsr == uMsr + cRegs)
+        cRegs++;
+
+    /* Figure out which bits we should skip when probing.  This is based on
+       specs and may need adjusting for real life when handy. */
+    uint64_t fSkipMask = RT_BIT_64(63);             /* PstateEn */
+    fSkipMask |= RT_BIT_64(41) | RT_BIT_64(40);     /* IddDiv */
+    fSkipMask |= UINT64_C(0x000000ff00000000);      /* IddValue */
+    if (CPUMMICROARCH_IS_AMD_FAM_10H(g_enmMicroarch))
+        fSkipMask |= UINT32_C(0xfe000000);          /* NbVid - Northbridge VID */
+    if (   CPUMMICROARCH_IS_AMD_FAM_10H(g_enmMicroarch)
+        || CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch))
+        fSkipMask |= RT_BIT_32(22);                 /* NbDid or NbPstate. */
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_15h_Piledriver) /* ?? - listed in 10-1Fh model BDKG as well asFam16h */
+        fSkipMask |= RT_BIT_32(16);                 /* CpuVid[7] */
+    fSkipMask |= UINT32_C(0x0000fe00);              /* CpuVid[6:0] */
+    fSkipMask |= UINT32_C(0x000001c0);              /* CpuDid */
+    fSkipMask |= UINT32_C(0x0000003f);              /* CpuFid */
+
+    /* Probe and report them one by one since we're passing values instead of
+       register indexes to the functions. */
+    for (uint32_t i = 0; i < cRegs; i++)
+    {
+        uint64_t fIgnMask = 0;
+        uint64_t fGpMask = 0;
+        int rc = msrProberModifyBitChanges(uMsr + i, &fIgnMask, &fGpMask, fSkipMask);
+        if (RT_FAILURE(rc))
+            return rc;
+        printMsrFunctionExtended(uMsr + i, "AmdFam10hPStateN", NULL, paMsrs[i].uValue, fIgnMask, fGpMask,
+                                 annotateValue(paMsrs[i].uValue));
+    }
+
+    /* Advance. */
+    *pidxLoop += cRegs - 1;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deals carefully with a COFVID control register.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR number.
+ * @param   uValue              The current value.
+ */
+static int reportMsr_AmdFam10hCofVidControl(uint32_t uMsr, uint64_t uValue)
+{
+    /* Skip know bits here, as harm seems to come from messing with them. */
+    uint64_t fSkipMask = 0;
+    if (CPUMMICROARCH_IS_AMD_FAM_10H(g_enmMicroarch))
+        fSkipMask |= UINT32_C(0xfe000000);          /* NbVid - Northbridge VID */
+    else if (g_enmMicroarch >= kCpumMicroarch_AMD_15h_First) /* Listed in preliminary Fam16h BDKG. */
+        fSkipMask |= UINT32_C(0xff000000);          /* NbVid - Northbridge VID - includes bit 24 for Fam15h and Fam16h. Odd... */
+    if (   CPUMMICROARCH_IS_AMD_FAM_10H(g_enmMicroarch)
+        || g_enmMicroarch >= kCpumMicroarch_AMD_15h_First) /* Listed in preliminary Fam16h BDKG. */
+        fSkipMask |= RT_BIT_32(22);                 /* NbDid or NbPstate. */
+    if (g_enmMicroarch >= kCpumMicroarch_AMD_15h_Piledriver) /* ?? - listed in 10-1Fh model BDKG as well asFam16h */
+        fSkipMask |= RT_BIT_32(20);                 /* CpuVid[7] */
+    fSkipMask |= UINT32_C(0x00070000);              /* PstatId */
+    fSkipMask |= UINT32_C(0x0000fe00);              /* CpuVid[6:0] */
+    fSkipMask |= UINT32_C(0x000001c0);              /* CpuDid */
+    fSkipMask |= UINT32_C(0x0000003f);              /* CpuFid */
+
+    return reportMsr_GenFunctionEx(uMsr, NULL, uValue, fSkipMask, 0, annotateValue(uValue));
+}
+
+
+/**
+ * Deals with the AMD [|L2I_|NB_]PERF_CT[LR] mixed ranges.
+ *
+ * Mixed here refers to the control and counter being in mixed in pairs as
+ * opposed to them being two separate parallel arrays like in the 0xc0010000
+ * area.
+ *
+ * @returns VBox status code.
+ * @param   paMsrs              Pointer to the first MSR.
+ * @param   cMsrs               The number of MSRs in the array @a paMsr.
+ * @param   cMax                The max number of MSRs (not counters).
+ * @param   pidxLoop            Index variable that should be advanced to the
+ *                              last MSR entry in the range.
+ */
+static int reportMsr_AmdGenPerfMixedRange(VBCPUREPMSR const *paMsrs, uint32_t cMsrs, uint32_t cMax, uint32_t *pidxLoop)
+{
+    uint32_t uMsr = paMsrs[0].uMsr;
+
+    /* Count them. */
+    uint32_t cRegs = 1;
+    while (   cRegs < cMax
+           && cRegs < cMsrs
+           && paMsrs[cRegs].uMsr == uMsr + cRegs)
+        cRegs++;
+    if (cRegs & 1)
+        return RTMsgErrorRc(VERR_INVALID_PARAMETER, "PERF range at %#x is odd: cRegs=%#x\n", uMsr, cRegs);
+
+    /* Report them as individual entries, using default names and such. */
+    for (uint32_t i = 0; i < cRegs; i++)
+    {
+        uint64_t fIgnMask = 0;
+        uint64_t fGpMask  = 0;
+        int rc = msrProberModifyBitChanges(uMsr + i, &fIgnMask, &fGpMask, 0);
+        if (RT_FAILURE(rc))
+            return rc;
+        printMsrFunctionExtendedIdxVal(uMsr + i, NULL, NULL, i / 2, fIgnMask, fGpMask, annotateValue(paMsrs[i].uValue));
+    }
+
+    /* Advance. */
+    *pidxLoop += cRegs - 1;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deals carefully with a LS_CFG register.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR number.
+ * @param   uValue              The current value.
+ */
+static int reportMsr_AmdK7InstrCacheCfg(uint32_t uMsr, uint64_t uValue)
+{
+    /* Skip know bits here, as harm seems to come from messing with them. */
+    uint64_t fSkipMask = RT_BIT_64(9) /* DIS_SPEC_TLB_RLD */;
+    if (CPUMMICROARCH_IS_AMD_FAM_10H(g_enmMicroarch))
+        fSkipMask |= RT_BIT_64(14); /* DIS_IND */
+    if (CPUMMICROARCH_IS_AMD_FAM_16H(g_enmMicroarch))
+        fSkipMask |= RT_BIT_64(26); /* DIS_WIDEREAD_PWR_SAVE */
+    if (CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch))
+    {
+        fSkipMask |= 0x1e;          /* DisIcWayFilter */
+        fSkipMask |= RT_BIT_64(39); /* DisLoopPredictor */
+        fSkipMask |= RT_BIT_64(27); /* Unknown killer bit, possibly applicable to other microarchs. */
+        fSkipMask |= RT_BIT_64(28); /* Unknown killer bit, possibly applicable to other microarchs. */
+    }
+    return reportMsr_GenFunctionEx(uMsr, NULL, uValue, fSkipMask, 0, annotateValue(uValue));
+}
+
+
+/**
+ * Deals carefully with a CU_CFG register.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR number.
+ * @param   uValue              The current value.
+ */
+static int reportMsr_AmdFam15hCombUnitCfg(uint32_t uMsr, uint64_t uValue)
+{
+    /* Skip know bits here, as harm seems to come from messing with them. */
+    uint64_t fSkipMask = RT_BIT_64(23) /* L2WayLock  */
+                       | RT_BIT_64(22) /* L2FirstLockWay */
+                       | RT_BIT_64(21) /* L2FirstLockWay */
+                       | RT_BIT_64(20) /* L2FirstLockWay */
+                       | RT_BIT_64(19) /* L2FirstLockWay */
+                       | RT_BIT_64(10) /* DcacheAggressivePriority */;
+    fSkipMask |= RT_BIT_64(46) | RT_BIT_64(45); /* Killer field. Seen bit 46 set, 45 clear. Messing with either means reboot/BSOD. */
+    return reportMsr_GenFunctionEx(uMsr, NULL, uValue, fSkipMask, 0, annotateValue(uValue));
+}
+
+
+/**
+ * Deals carefully with a EX_CFG register.
+ *
+ * @returns VBox status code.
+ * @param   uMsr                The MSR number.
+ * @param   uValue              The current value.
+ */
+static int reportMsr_AmdFam15hExecUnitCfg(uint32_t uMsr, uint64_t uValue)
+{
+    /* Skip know bits here, as harm seems to come from messing with them. */
+    uint64_t fSkipMask = RT_BIT_64(54) /* LateSbzResync  */;
+    fSkipMask |= RT_BIT_64(35); /* Undocumented killer bit. */
+    return reportMsr_GenFunctionEx(uMsr, NULL, uValue, fSkipMask, 0, annotateValue(uValue));
+}
+
+
+
+static int produceMsrReport(VBCPUREPMSR *paMsrs, uint32_t cMsrs)
+{
+    vbCpuRepDebug("produceMsrReport\n");
+    RTThreadSleep(500);
+
+    for (uint32_t i = 0; i < cMsrs; i++)
+    {
+        uint32_t    uMsr       = paMsrs[i].uMsr;
+        uint32_t    fFlags     = paMsrs[i].fFlags;
+        uint64_t    uValue     = paMsrs[i].uValue;
+        int         rc;
+#if 0
+        //if (uMsr < 0x00000000)
+        //    continue;
+        if (uMsr >= 0x00000277)
+        {
+            vbCpuRepDebug("produceMsrReport: uMsr=%#x (%s)...\n", uMsr, getMsrNameHandled(uMsr));
+            RTThreadSleep(1000);
+        }
+#endif
+        /*
+         * Deal with write only regs first to avoid having to avoid them all the time.
+         */
+        if (fFlags & VBCPUREPMSR_F_WRITE_ONLY)
+        {
+            if (uMsr == 0x00000079)
+                rc = printMsrWriteOnly(uMsr, NULL, NULL);
+            else
+                rc = reportMsr_Generic(uMsr, fFlags, uValue);
+        }
+        /*
+         * VIA implement MSRs in a interesting way, so we have to select what we
+         * want to handle there to avoid making the code below unreadable.
+         */
+        /** @todo r=klaus check if Shanghai CPUs really are behaving the same */
+        else if (isMsrViaShanghaiDummy(uMsr, uValue, fFlags))
+            rc = reportMsr_ViaShanghaiDummyRange(&paMsrs[i], cMsrs - i, &i);
+        /*
+         * This shall be sorted by uMsr as much as possible.
+         */
+        else if (uMsr == 0x00000000 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON) && g_enmMicroarch >= kCpumMicroarch_AMD_K8_First)
+            rc = printMsrAlias(uMsr, 0x00000402, NULL);
+        else if (uMsr == 0x00000001 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON) && g_enmMicroarch >= kCpumMicroarch_AMD_K8_First)
+            rc = printMsrAlias(uMsr, 0x00000401, NULL); /** @todo not 101% correct on Fam15h and later, 0xc0010015[McstatusWrEn] effect differs. */
+        else if (uMsr == 0x0000001b)
+            rc = reportMsr_Ia32ApicBase(uMsr, uValue);
+        else if (uMsr == 0x00000040 && g_enmMicroarch <= kCpumMicroarch_Intel_P6_M_Dothan)
+            rc = reportMsr_GenRangeFunction(&paMsrs[i], cMsrs - i, 8 /*cMax*/, "IntelLastBranchFromToN", &i);
+        else if (uMsr == 0x00000040)
+            rc = reportMsr_GenRangeFunctionEx(&paMsrs[i], cMsrs - i, 8 /*cMax*/, "IntelLastBranchToN", uMsr, false,
+                                              true, getGenericSkipMask(uMsr), &i);
+        else if (uMsr == 0x00000060 && g_enmMicroarch >= kCpumMicroarch_Intel_Core_Yonah)
+            rc = reportMsr_GenRangeFunctionEx(&paMsrs[i], cMsrs - i, 8 /*cMax*/, "IntelLastBranchFromN", uMsr, false,
+                                              true, getGenericSkipMask(uMsr), &i);
+        else if (uMsr == 0x000000c1)
+            rc = reportMsr_GenRangeFunction(&paMsrs[i], cMsrs - i,
+                                            g_enmMicroarch >= kCpumMicroarch_Intel_Core7_First ? 8 : 4 /*cMax*/,
+                                            NULL, &i);
+        else if (uMsr == 0x00000186 && !g_fIntelNetBurst)
+            rc = reportMsr_GenRangeFunction(&paMsrs[i], cMsrs - i, 8 /*cMax*/, "Ia32PerfEvtSelN", &i);
+        else if (uMsr == 0x000001a0)
+            rc = reportMsr_Ia32MiscEnable(uMsr, uValue);
+        else if (uMsr >= 0x000001a6 && uMsr <= 0x000001a7)
+            rc = reportMsr_GenRangeFunction(&paMsrs[i], cMsrs - i, 2 /*cMax*/, "IntelI7MsrOffCoreResponseN", &i);
+        else if (uMsr == 0x000001db && g_fIntelNetBurst)
+            rc = reportMsr_GenRangeFunction(&paMsrs[i], cMsrs - i, 4 /*cMax*/, "IntelLastBranchFromToN", &i);
+        else if (uMsr == 0x00000200)
+            rc = reportMsr_Ia32MtrrPhysBaseMaskN(&paMsrs[i], cMsrs - i, &i);
+        else if (uMsr >= 0x00000250 && uMsr <= 0x00000279)
+            rc = reportMsr_Ia32MtrrFixedOrPat(uMsr);
+        else if (uMsr >= 0x00000280 && uMsr <= 0x00000295)
+            rc = reportMsr_GenRangeFunctionEx(&paMsrs[i], cMsrs - i, 22 /*cMax*/, NULL, 0x00000280, true /*fEarlyEndOk*/, false, 0, &i);
+        else if (uMsr == 0x000002ff)
+            rc = reportMsr_Ia32MtrrDefType(uMsr);
+        else if (uMsr >= 0x00000309 && uMsr <= 0x0000030b && !g_fIntelNetBurst)
+            rc = reportMsr_GenRangeFunctionEx(&paMsrs[i], cMsrs - i, 3 /*cMax*/, NULL, 0x00000309, true /*fEarlyEndOk*/, false, 0, &i);
+        else if ((uMsr == 0x000003f8 || uMsr == 0x000003fc || uMsr == 0x0000060a) && !g_fIntelNetBurst)
+            rc = reportMsr_GenRangeFunctionEx(&paMsrs[i], cMsrs - i, 4, NULL, uMsr - 3, true, false, 0, &i);
+        else if ((uMsr == 0x000003f9 || uMsr == 0x000003fd || uMsr == 0x0000060b) && !g_fIntelNetBurst)
+            rc = reportMsr_GenRangeFunctionEx(&paMsrs[i], cMsrs - i, 8, NULL, uMsr - 6, true, false, 0, &i);
+        else if ((uMsr == 0x000003fa || uMsr == 0x000003fe || uMsr == 0x0000060c) && !g_fIntelNetBurst)
+            rc = reportMsr_GenRangeFunctionEx(&paMsrs[i], cMsrs - i, 8, NULL, uMsr - 7, true, false, 0, &i);
+        else if (uMsr >= 0x00000400 && uMsr <= 0x00000477)
+            rc = reportMsr_Ia32McCtlStatusAddrMiscN(&paMsrs[i], cMsrs - i, &i);
+        else if (uMsr == 0x000004c1)
+            rc = reportMsr_GenRangeFunction(&paMsrs[i], cMsrs - i, 8, NULL, &i);
+        else if (uMsr == 0x00000680 || uMsr == 0x000006c0)
+            rc = reportMsr_GenRangeFunctionEx(&paMsrs[i], cMsrs - i, 16, NULL, uMsr, false, false,
+                                              g_fIntelNetBurst
+                                              ? UINT64_C(0xffffffffffffff00) /* kludge */
+                                              : UINT64_C(0xffff800000000000), &i);
+        else if (uMsr >= 0x00000800 && uMsr <= 0x000008ff)
+            rc = reportMsr_GenX2Apic(&paMsrs[i], cMsrs - i, &i);
+        else if (uMsr == 0x00002000 && g_enmVendor == CPUMCPUVENDOR_INTEL)
+            rc = reportMsr_GenFunctionEx(uMsr, "IntelP6CrN", 0, X86_CR0_PE | X86_CR0_PG, 0,
+                                         annotateIfMissingBits(uValue, X86_CR0_PE | X86_CR0_PE | X86_CR0_ET));
+        else if (uMsr == 0x00002002 && g_enmVendor == CPUMCPUVENDOR_INTEL)
+            rc = reportMsr_GenFunctionEx(uMsr, "IntelP6CrN", 2, 0, 0, annotateValue(uValue));
+        else if (uMsr == 0x00002003 && g_enmVendor == CPUMCPUVENDOR_INTEL)
+        {
+            uint64_t fCr3Mask = (RT_BIT_64(vbCpuRepGetPhysAddrWidth()) - 1) & (X86_CR3_PAE_PAGE_MASK | X86_CR3_AMD64_PAGE_MASK);
+            if (!vbCpuRepSupportsPae())
+                fCr3Mask &= X86_CR3_PAGE_MASK | X86_CR3_AMD64_PAGE_MASK;
+            rc = reportMsr_GenFunctionEx(uMsr, "IntelP6CrN", 3, fCr3Mask, 0, annotateValue(uValue));
+        }
+        else if (uMsr == 0x00002004 && g_enmVendor == CPUMCPUVENDOR_INTEL)
+            rc = reportMsr_GenFunctionEx(uMsr, "IntelP6CrN", 4,
+                                         X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE | X86_CR4_SMXE, 0,
+                                         annotateValue(uValue));
+        else if (uMsr == 0xc0000080)
+            rc = reportMsr_Amd64Efer(uMsr, uValue);
+        else if (uMsr >= 0xc0000408 && uMsr <= 0xc000040f)
+            rc = reportMsr_AmdFam10hMc4MiscN(&paMsrs[i], cMsrs - i, &i);
+        else if (uMsr == 0xc0010000 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdK8PerfCtlN(&paMsrs[i], cMsrs - i, &i);
+        else if (uMsr == 0xc0010004 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdK8PerfCtrN(&paMsrs[i], cMsrs - i, &i);
+        else if (uMsr == 0xc0010010 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdK8SysCfg(uMsr, uValue);
+        else if (uMsr == 0xc0010015 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdK8HwCr(uMsr, uValue);
+        else if ((uMsr == 0xc0010016 || uMsr == 0xc0010018) && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdK8IorrBaseN(uMsr, uValue);
+        else if ((uMsr == 0xc0010017 || uMsr == 0xc0010019) && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdK8IorrMaskN(uMsr, uValue);
+        else if ((uMsr == 0xc001001a || uMsr == 0xc001001d) && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdK8TopMemN(uMsr, uValue);
+        else if (uMsr == 0xc0010030 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_GenRangeFunction(&paMsrs[i], cMsrs - i, 6, "AmdK8CpuNameN", &i);
+        else if (uMsr >= 0xc0010044 && uMsr <= 0xc001004a && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_GenRangeFunctionEx(&paMsrs[i], cMsrs - i, 7, "AmdK8McCtlMaskN", 0xc0010044, true /*fEarlyEndOk*/, false, 0, &i);
+        else if (uMsr == 0xc0010050 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_GenRangeFunction(&paMsrs[i], cMsrs - i, 4, "AmdK8SmiOnIoTrapN", &i);
+        else if (uMsr == 0xc0010064 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdFam10hPStateN(&paMsrs[i], cMsrs - i, &i);
+        else if (uMsr == 0xc0010070 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdFam10hCofVidControl(uMsr, uValue);
+        else if ((uMsr == 0xc0010118 || uMsr == 0xc0010119) && getMsrFnName(uMsr, NULL) && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = printMsrFunction(uMsr, NULL, NULL, annotateValue(uValue)); /* RAZ, write key. */
+        else if (uMsr == 0xc0010200 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdGenPerfMixedRange(&paMsrs[i], cMsrs - i, 12, &i);
+        else if (uMsr == 0xc0010230 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdGenPerfMixedRange(&paMsrs[i], cMsrs - i, 8, &i);
+        else if (uMsr == 0xc0010240 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdGenPerfMixedRange(&paMsrs[i], cMsrs - i, 8, &i);
+        else if (uMsr == 0xc0011019 && g_enmMicroarch >= kCpumMicroarch_AMD_15h_Piledriver && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_GenRangeFunctionEx(&paMsrs[i], cMsrs - i, 3, "AmdK7DrXAddrMaskN", 0xc0011019 - 1,
+                                              false /*fEarlyEndOk*/, false /*fNoIgnMask*/, 0, &i);
+        else if (uMsr == 0xc0011021 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_AmdK7InstrCacheCfg(uMsr, uValue);
+        else if (uMsr == 0xc0011023 && CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch))
+            rc = reportMsr_AmdFam15hCombUnitCfg(uMsr, uValue);
+        else if (uMsr == 0xc0011027 && (g_enmVendor == CPUMCPUVENDOR_AMD || g_enmVendor == CPUMCPUVENDOR_HYGON))
+            rc = reportMsr_GenRangeFunctionEx(&paMsrs[i], cMsrs - i, 1, "AmdK7DrXAddrMaskN", 0xc0011027,
+                                              false /*fEarlyEndOk*/, false /*fNoIgnMask*/, 0, &i);
+        else if (uMsr == 0xc001102c && CPUMMICROARCH_IS_AMD_FAM_15H(g_enmMicroarch))
+            rc = reportMsr_AmdFam15hExecUnitCfg(uMsr, uValue);
+        /* generic handling. */
+        else
+            rc = reportMsr_Generic(uMsr, fFlags, uValue);
+
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /*
+         *  A little ugly snooping.
+         */
+        if (uMsr == 0x000000cd && !(fFlags & VBCPUREPMSR_F_WRITE_ONLY))
+            g_uMsrIntelP6FsbFrequency = uValue;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Custom MSR hacking & probing.
+ *
+ * Called when the '-d' option is given.
+ *
+ * @returns VBox status code.
+ */
+static int hackingMsrs(void)
+{
+#if 0
+    vbCpuRepDebug("\nhackingMsrs:\n"); RTStrmFlush(g_pDebugOut); RTThreadSleep(2000);
+
+    uint32_t uMsr = 0xc0000081;
+    vbCpuRepDebug("%#x: msrProberModifyNoChange -> %RTbool\n", uMsr, msrProberModifyNoChange(uMsr));
+    RTThreadSleep(3000);
+
+    vbCpuRepDebug("%#x: msrProberModifyBit 30 -> %d\n", uMsr, msrProberModifyBit(uMsr, 30));
+    RTThreadSleep(3000);
+
+    vbCpuRepDebug("%#x: msrProberModifyZero -> %RTbool\n", uMsr, msrProberModifyZero(uMsr));
+    RTThreadSleep(3000);
+
+    for (uint32_t i = 0; i < 63; i++)
+    {
+        vbCpuRepDebug("%#x: bit=%02u -> %d\n", msrProberModifyBit(uMsr, i));
+        RTThreadSleep(500);
+    }
+#else
+
+    uint32_t uMsr = 0xc0010010;
+    uint64_t uValue = 0;
+    msrProberRead(uMsr, &uValue);
+    reportMsr_AmdK8SysCfg(uMsr, uValue);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+static int probeMsrs(bool fHacking, const char *pszNameC, const char *pszCpuDesc,
+                     char *pszMsrMask, size_t cbMsrMask)
+{
+    /* Initialize the mask. */
+    if (pszMsrMask && cbMsrMask)
+        RTStrCopy(pszMsrMask, cbMsrMask, "UINT32_MAX /** @todo */");
+
+    /*
+     * Are MSRs supported by the CPU?
+     */
+    if (   !ASMIsValidStdRange(ASMCpuId_EAX(0))
+        || !(ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_MSR) )
+    {
+        vbCpuRepDebug("Skipping MSR probing, CPUID indicates there isn't any MSR support.\n");
+        return VINF_SUCCESS;
+    }
+    if (g_fNoMsrs)
+    {
+        vbCpuRepDebug("Skipping MSR probing (--no-msr).\n");
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * First try the the support library (also checks if we can really read MSRs).
+     */
+    int rc = VbCpuRepMsrProberInitSupDrv(&g_MsrAcc);
+    if (RT_FAILURE(rc))
+    {
+#ifdef VBCR_HAVE_PLATFORM_MSR_PROBER
+        /* Next try a platform-specific interface. */
+        rc = VbCpuRepMsrProberInitPlatform(&g_MsrAcc);
+#endif
+        if (RT_FAILURE(rc))
+        {
+            vbCpuRepDebug("warning: Unable to initialize any MSR access interface (%Rrc), skipping MSR detection.\n", rc);
+            return VINF_SUCCESS;
+        }
+    }
+
+    uint64_t uValue;
+    bool     fGp;
+    rc = g_MsrAcc.pfnMsrProberRead(MSR_IA32_TSC, NIL_RTCPUID, &uValue, &fGp);
+    if (RT_FAILURE(rc))
+    {
+        vbCpuRepDebug("warning: MSR probing not supported by the support driver (%Rrc), skipping MSR detection.\n", rc);
+        return VINF_SUCCESS;
+    }
+    vbCpuRepDebug("MSR_IA32_TSC: %#llx fGp=%RTbool\n", uValue, fGp);
+    rc = g_MsrAcc.pfnMsrProberRead(0xdeadface, NIL_RTCPUID, &uValue, &fGp);
+    vbCpuRepDebug("0xdeadface: %#llx fGp=%RTbool rc=%Rrc\n", uValue, fGp, rc);
+
+    /*
+     * Initialize globals we use.
+     */
+    uint32_t uEax, uEbx, uEcx, uEdx;
+    ASMCpuIdExSlow(0, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
+    if (!ASMIsValidStdRange(uEax))
+        return RTMsgErrorRc(VERR_NOT_SUPPORTED, "Invalid std CPUID range: %#x\n", uEax);
+    g_enmVendor = CPUMR3CpuIdDetectVendorEx(uEax, uEbx, uEcx, uEdx);
+
+    ASMCpuIdExSlow(1, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
+    g_enmMicroarch = CPUMR3CpuIdDetermineMicroarchEx(g_enmVendor,
+                                                     ASMGetCpuFamily(uEax),
+                                                     ASMGetCpuModel(uEax, g_enmVendor == CPUMCPUVENDOR_INTEL),
+                                                     ASMGetCpuStepping(uEax));
+    g_fIntelNetBurst = CPUMMICROARCH_IS_INTEL_NETBURST(g_enmMicroarch);
+
+    /*
+     * Do the probing.
+     */
+    if (fHacking)
+        rc = hackingMsrs();
+    else
+    {
+        /* Determine the MSR mask. */
+        uint32_t fMsrMask = determineMsrAndMask();
+        if (fMsrMask == UINT32_MAX)
+            RTStrCopy(pszMsrMask, cbMsrMask, "UINT32_MAX");
+        else
+            RTStrPrintf(pszMsrMask, cbMsrMask, "UINT32_C(%#x)", fMsrMask);
+
+        /* Detect MSR. */
+        VBCPUREPMSR    *paMsrs;
+        uint32_t        cMsrs;
+        rc = findMsrs(&paMsrs, &cMsrs, fMsrMask);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* Probe the MSRs and spit out the database table. */
+        vbCpuRepPrintf("\n"
+                       "#ifndef CPUM_DB_STANDALONE\n"
+                       "/**\n"
+                       " * MSR ranges for %s.\n"
+                       " */\n"
+                       "static CPUMMSRRANGE const g_aMsrRanges_%s[] =\n{\n",
+                       pszCpuDesc,
+                       pszNameC);
+        rc = produceMsrReport(paMsrs, cMsrs);
+        vbCpuRepPrintf("};\n"
+                       "#endif /* !CPUM_DB_STANDALONE */\n"
+                       "\n"
+                       );
+
+        RTMemFree(paMsrs);
+        paMsrs = NULL;
+    }
+    if (g_MsrAcc.pfnTerm)
+        g_MsrAcc.pfnTerm();
+    RT_ZERO(g_MsrAcc);
+    return rc;
+}
+
+
+static int produceCpuIdArray(const char *pszNameC, const char *pszCpuDesc)
+{
+    /*
+     * Collect the data.
+     */
+    PCPUMCPUIDLEAF  paLeaves;
+    uint32_t        cLeaves;
+    int rc = CPUMR3CpuIdCollectLeaves(&paLeaves, &cLeaves);
+    if (RT_FAILURE(rc))
+        return RTMsgErrorRc(rc, "CPUMR3CollectCpuIdInfo failed: %Rrc\n", rc);
+
+    /*
+     * Dump the array.
+     */
+    vbCpuRepPrintf("\n"
+                   "#ifndef CPUM_DB_STANDALONE\n"
+                   "/**\n"
+                   " * CPUID leaves for %s.\n"
+                   " */\n"
+                   "static CPUMCPUIDLEAF const g_aCpuIdLeaves_%s[] = \n{\n",
+                   pszCpuDesc,
+                   pszNameC);
+    for (uint32_t i = 0; i < cLeaves; i++)
+    {
+        vbCpuRepPrintf("    { %#010x, %#010x, ", paLeaves[i].uLeaf, paLeaves[i].uSubLeaf);
+        if (paLeaves[i].fSubLeafMask == UINT32_MAX)
+            vbCpuRepPrintf("UINT32_MAX, ");
+        else
+            vbCpuRepPrintf("%#010x, ", paLeaves[i].fSubLeafMask);
+        vbCpuRepPrintf("%#010x, %#010x, %#010x, %#010x, ",
+                     paLeaves[i].uEax, paLeaves[i].uEbx, paLeaves[i].uEcx, paLeaves[i].uEdx);
+        if (paLeaves[i].fFlags == 0)
+            vbCpuRepPrintf("0 },\n");
+        else
+        {
+            vbCpuRepPrintf("0");
+            uint32_t fFlags = paLeaves[i].fFlags;
+            if (paLeaves[i].fFlags & CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES)
+            {
+                vbCpuRepPrintf(" | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES");
+                fFlags &= ~CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES;
+            }
+            if (paLeaves[i].fFlags & CPUMCPUIDLEAF_F_CONTAINS_APIC_ID)
+            {
+                vbCpuRepPrintf(" | CPUMCPUIDLEAF_F_CONTAINS_APIC_ID");
+                fFlags &= ~CPUMCPUIDLEAF_F_CONTAINS_APIC_ID;
+            }
+            if (paLeaves[i].fFlags & CPUMCPUIDLEAF_F_CONTAINS_APIC)
+            {
+                vbCpuRepPrintf(" | CPUMCPUIDLEAF_F_CONTAINS_APIC");
+                fFlags &= ~CPUMCPUIDLEAF_F_CONTAINS_APIC;
+            }
+            if (fFlags)
+            {
+                RTMemFree(paLeaves);
+                return RTMsgErrorRc(rc, "Unknown CPUID flags %#x\n", fFlags);
+            }
+            vbCpuRepPrintf(" },\n");
+        }
+    }
+    vbCpuRepPrintf("};\n"
+                   "#endif /* !CPUM_DB_STANDALONE */\n"
+                   "\n");
+    RTMemFree(paLeaves);
+    return VINF_SUCCESS;
+}
+
+
+static const char *cpuVendorToString(CPUMCPUVENDOR enmCpuVendor)
+{
+    switch (enmCpuVendor)
+    {
+        case CPUMCPUVENDOR_INTEL:       return "Intel";
+        case CPUMCPUVENDOR_AMD:         return "AMD";
+        case CPUMCPUVENDOR_VIA:         return "VIA";
+        case CPUMCPUVENDOR_CYRIX:       return "Cyrix";
+        case CPUMCPUVENDOR_SHANGHAI:    return "Shanghai";
+        case CPUMCPUVENDOR_HYGON:       return "Hygon";
+        case CPUMCPUVENDOR_INVALID:
+        case CPUMCPUVENDOR_UNKNOWN:
+        case CPUMCPUVENDOR_32BIT_HACK:
+            break;
+    }
+    return "invalid-cpu-vendor";
+}
+
+
+/**
+ * Takes a shot a the bus frequency name (last part).
+ *
+ * @returns Name suffix.
+ */
+static const char *vbCpuRepGuessScalableBusFrequencyName(void)
+{
+    if (CPUMMICROARCH_IS_INTEL_CORE7(g_enmMicroarch))
+        return g_enmMicroarch >= kCpumMicroarch_Intel_Core7_SandyBridge ? "100MHZ" : "133MHZ";
+
+    if (g_uMsrIntelP6FsbFrequency != UINT64_MAX)
+        switch (g_uMsrIntelP6FsbFrequency & 0x7)
+        {
+            case 5: return "100MHZ";
+            case 1: return "133MHZ";
+            case 3: return "167MHZ";
+            case 2: return "200MHZ";
+            case 0: return "267MHZ";
+            case 4: return "333MHZ";
+            case 6: return "400MHZ";
+        }
+
+    return "UNKNOWN";
+}
+
+
+static int produceCpuReport(void)
+{
+    /*
+     * Figure the cpu vendor.
+     */
+    if (!ASMHasCpuId())
+        return RTMsgErrorRc(VERR_NOT_SUPPORTED, "No CPUID support.\n");
+    uint32_t uEax, uEbx, uEcx, uEdx;
+    ASMCpuIdExSlow(0, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
+    if (!ASMIsValidStdRange(uEax))
+        return RTMsgErrorRc(VERR_NOT_SUPPORTED, "Invalid std CPUID range: %#x\n", uEax);
+
+    CPUMCPUVENDOR enmVendor = CPUMR3CpuIdDetectVendorEx(uEax, uEbx, uEcx, uEdx);
+    if (enmVendor == CPUMCPUVENDOR_UNKNOWN)
+        return RTMsgErrorRc(VERR_NOT_IMPLEMENTED, "Unknown CPU vendor: %.4s%.4s%.4s\n", &uEbx, &uEdx, &uEcx);
+    vbCpuRepDebug("CPU Vendor: %s - %.4s%.4s%.4s\n", CPUMR3CpuVendorName(enmVendor), &uEbx, &uEdx, &uEcx);
+
+    /*
+     * Determine the micro arch.
+     */
+    ASMCpuIdExSlow(1, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
+    CPUMMICROARCH enmMicroarch = CPUMR3CpuIdDetermineMicroarchEx(enmVendor,
+                                                                 ASMGetCpuFamily(uEax),
+                                                                 ASMGetCpuModel(uEax, enmVendor == CPUMCPUVENDOR_INTEL),
+                                                                 ASMGetCpuStepping(uEax));
+
+    /*
+     * Generate a name.
+     */
+    char  szName[16*3+1];
+    char  szNameC[16*3+1];
+    char  szNameRaw[16*3+1];
+    char *pszName    = szName;
+    char *pszCpuDesc = (char *)"";
+
+    ASMCpuIdExSlow(0x80000000, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
+    if (ASMIsValidExtRange(uEax) && uEax >= UINT32_C(0x80000004))
+    {
+        /* Get the raw name and strip leading spaces. */
+        ASMCpuIdExSlow(0x80000002, 0, 0, 0, &szNameRaw[0 +  0], &szNameRaw[4 +  0], &szNameRaw[8 +  0], &szNameRaw[12 +  0]);
+        ASMCpuIdExSlow(0x80000003, 0, 0, 0, &szNameRaw[0 + 16], &szNameRaw[4 + 16], &szNameRaw[8 + 16], &szNameRaw[12 + 16]);
+        ASMCpuIdExSlow(0x80000004, 0, 0, 0, &szNameRaw[0 + 32], &szNameRaw[4 + 32], &szNameRaw[8 + 32], &szNameRaw[12 + 32]);
+        szNameRaw[48] = '\0';
+        pszCpuDesc = RTStrStrip(szNameRaw);
+        vbCpuRepDebug("Name2: %s\n", pszCpuDesc);
+
+        /* Reduce the name. */
+        pszName = strcpy(szName, pszCpuDesc);
+
+        static const char * const s_apszSuffixes[] =
+        {
+            "CPU @",
+        };
+        for (uint32_t i = 0; i < RT_ELEMENTS(s_apszSuffixes); i++)
+        {
+            char *pszHit = strstr(pszName, s_apszSuffixes[i]);
+            if (pszHit)
+                RT_BZERO(pszHit, strlen(pszHit));
+        }
+
+        static const char * const s_apszWords[] =
+        {
+            "(TM)", "(tm)", "(R)", "(r)", "Processor", "CPU", "@",
+        };
+        for (uint32_t i = 0; i < RT_ELEMENTS(s_apszWords); i++)
+        {
+            const char *pszWord = s_apszWords[i];
+            size_t      cchWord = strlen(pszWord);
+            char       *pszHit;
+            while ((pszHit = strstr(pszName, pszWord)) != NULL)
+                memset(pszHit, ' ', cchWord);
+        }
+
+        RTStrStripR(pszName);
+        for (char *psz = pszName; *psz; psz++)
+            if (RT_C_IS_BLANK(*psz))
+            {
+                size_t cchBlanks = 1;
+                while (RT_C_IS_BLANK(psz[cchBlanks]))
+                    cchBlanks++;
+                *psz = ' ';
+                if (cchBlanks > 1)
+                    memmove(psz + 1, psz + cchBlanks, strlen(psz + cchBlanks) + 1);
+            }
+        pszName = RTStrStripL(pszName);
+        vbCpuRepDebug("Name: %s\n", pszName);
+
+        /* Make it C/C++ acceptable. */
+        strcpy(szNameC, pszName);
+        unsigned offDst = 0;
+        for (unsigned offSrc = 0; ; offSrc++)
+        {
+            char ch = szNameC[offSrc];
+            if (!RT_C_IS_ALNUM(ch) && ch != '_' && ch != '\0')
+                ch = '_';
+            if (ch == '_' && offDst > 0 && szNameC[offDst - 1] == '_')
+                offDst--;
+            szNameC[offDst++] = ch;
+            if (!ch)
+                break;
+        }
+        while (offDst > 1 && szNameC[offDst - 1] == '_')
+            szNameC[--offDst] = '\0';
+
+        vbCpuRepDebug("NameC: %s\n", szNameC);
+    }
+    else
+    {
+        ASMCpuIdExSlow(1, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
+        RTStrPrintf(szNameC, sizeof(szNameC), "%s_%u_%u_%u", cpuVendorToString(enmVendor), ASMGetCpuFamily(uEax),
+                    ASMGetCpuModel(uEax, enmVendor == CPUMCPUVENDOR_INTEL), ASMGetCpuStepping(uEax));
+        pszCpuDesc = pszName = szNameC;
+        vbCpuRepDebug("Name/NameC: %s\n", szNameC);
+    }
+
+    /*
+     * Print a file header, if we're not outputting to stdout (assumption being
+     * that stdout is used while hacking the reporter and too much output is
+     * unwanted).
+     */
+    if (g_pReportOut)
+    {
+        RTTIMESPEC Now;
+        char       szNow[64];
+        RTTimeSpecToString(RTTimeNow(&Now), szNow, sizeof(szNow));
+        char *pchDot = strchr(szNow, '.');
+        if (pchDot)
+            strcpy(pchDot, "Z");
+
+        vbCpuRepPrintf("/* $" "Id" "$ */\n"
+                       "/** @file\n"
+                       " * CPU database entry \"%s\".\n"
+                       " * Generated at %s by VBoxCpuReport v%sr%s on %s.%s.\n"
+                       " */\n"
+                       "\n"
+                       "/*\n"
+                       " * Copyright (C) 2013-2020 Oracle Corporation\n"
+                       " *\n"
+                       " * This file is part of VirtualBox Open Source Edition (OSE), as\n"
+                       " * available from http://www.virtualbox.org. This file is free software;\n"
+                       " * you can redistribute it and/or modify it under the terms of the GNU\n"
+                       " * General Public License (GPL) as published by the Free Software\n"
+                       " * Foundation, in version 2 as it comes in the \"COPYING\" file of the\n"
+                       " * VirtualBox OSE distribution. VirtualBox OSE is distributed in the\n"
+                       " * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.\n"
+                       " */\n"
+                       "\n"
+                       "#ifndef VBOX_CPUDB_%s_h\n"
+                       "#define VBOX_CPUDB_%s_h\n"
+                       "#ifndef RT_WITHOUT_PRAGMA_ONCE\n"
+                       "# pragma once\n"
+                       "#endif\n"
+                       "\n",
+                       pszName,
+                       szNow, RTBldCfgVersion(), RTBldCfgRevisionStr(), RTBldCfgTarget(), RTBldCfgTargetArch(),
+                       szNameC, szNameC);
+    }
+
+    /*
+     * Extract CPUID based data.
+     */
+    int rc = produceCpuIdArray(szNameC, pszCpuDesc);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    CPUMUNKNOWNCPUID enmUnknownMethod;
+    CPUMCPUID       DefUnknown;
+    rc = CPUMR3CpuIdDetectUnknownLeafMethod(&enmUnknownMethod, &DefUnknown);
+    if (RT_FAILURE(rc))
+        return RTMsgErrorRc(rc, "CPUMR3DetectCpuIdUnknownMethod failed: %Rrc\n", rc);
+    vbCpuRepDebug("enmUnknownMethod=%s\n", CPUMR3CpuIdUnknownLeafMethodName(enmUnknownMethod));
+
+    /*
+     * Do the MSRs, if we can.
+     */
+    char szMsrMask[64];
+    probeMsrs(false /*fHacking*/, szNameC, pszCpuDesc, szMsrMask, sizeof(szMsrMask));
+
+    /*
+     * Emit the CPUMDBENTRY record.
+     */
+    ASMCpuIdExSlow(1, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
+    vbCpuRepPrintf("\n"
+                   "/**\n"
+                   " * Database entry for %s.\n"
+                   " */\n"
+                   "static CPUMDBENTRY const g_Entry_%s = \n"
+                   "{\n"
+                   "    /*.pszName          = */ \"%s\",\n"
+                   "    /*.pszFullName      = */ \"%s\",\n"
+                   "    /*.enmVendor        = */ CPUMCPUVENDOR_%s,\n"
+                   "    /*.uFamily          = */ %u,\n"
+                   "    /*.uModel           = */ %u,\n"
+                   "    /*.uStepping        = */ %u,\n"
+                   "    /*.enmMicroarch     = */ kCpumMicroarch_%s,\n"
+                   "    /*.uScalableBusFreq = */ CPUM_SBUSFREQ_%s,\n"
+                   "    /*.fFlags           = */ 0,\n"
+                   "    /*.cMaxPhysAddrWidth= */ %u,\n"
+                   "    /*.fMxCsrMask       = */ %#010x,\n"
+                   "    /*.paCpuIdLeaves    = */ NULL_ALONE(g_aCpuIdLeaves_%s),\n"
+                   "    /*.cCpuIdLeaves     = */ ZERO_ALONE(RT_ELEMENTS(g_aCpuIdLeaves_%s)),\n"
+                   "    /*.enmUnknownCpuId  = */ CPUMUNKNOWNCPUID_%s,\n"
+                   "    /*.DefUnknownCpuId  = */ { %#010x, %#010x, %#010x, %#010x },\n"
+                   "    /*.fMsrMask         = */ %s,\n"
+                   "    /*.cMsrRanges       = */ ZERO_ALONE(RT_ELEMENTS(g_aMsrRanges_%s)),\n"
+                   "    /*.paMsrRanges      = */ NULL_ALONE(g_aMsrRanges_%s),\n"
+                   "};\n"
+                   "\n"
+                   "#endif /* !VBOX_CPUDB_%s_h */\n"
+                   "\n",
+                   pszCpuDesc,
+                   szNameC,
+                   pszName,
+                   pszCpuDesc,
+                   CPUMR3CpuVendorName(enmVendor),
+                   ASMGetCpuFamily(uEax),
+                   ASMGetCpuModel(uEax, enmVendor == CPUMCPUVENDOR_INTEL),
+                   ASMGetCpuStepping(uEax),
+                   CPUMR3MicroarchName(enmMicroarch),
+                   vbCpuRepGuessScalableBusFrequencyName(),
+                   vbCpuRepGetPhysAddrWidth(),
+                   CPUMR3DeterminHostMxCsrMask(),
+                   szNameC,
+                   szNameC,
+                   CPUMR3CpuIdUnknownLeafMethodName(enmUnknownMethod),
+                   DefUnknown.uEax,
+                   DefUnknown.uEbx,
+                   DefUnknown.uEcx,
+                   DefUnknown.uEdx,
+                   szMsrMask,
+                   szNameC,
+                   szNameC,
+                   szNameC
+                   );
+
+    return VINF_SUCCESS;
+}
+
+
+int main(int argc, char **argv)
+{
+    int rc = RTR3InitExe(argc, &argv, 0 /*fFlags*/);
+    if (RT_FAILURE(rc))
+        return RTMsgInitFailure(rc);
+
+    /*
+     * Argument parsing?
+     */
+    static const RTGETOPTDEF s_aOptions[] =
+    {
+        { "--msrs-only", 'm', RTGETOPT_REQ_NOTHING },
+        { "--msrs-dev",  'd', RTGETOPT_REQ_NOTHING },
+        { "--no-msrs",   'n', RTGETOPT_REQ_NOTHING },
+        { "--output",    'o', RTGETOPT_REQ_STRING  },
+        { "--log",       'l', RTGETOPT_REQ_STRING  },
+    };
+    RTGETOPTSTATE State;
+    RTGetOptInit(&State, argc, argv, &s_aOptions[0], RT_ELEMENTS(s_aOptions), 1, RTGETOPTINIT_FLAGS_OPTS_FIRST);
+
+    enum
+    {
+        kCpuReportOp_Normal,
+        kCpuReportOp_MsrsOnly,
+        kCpuReportOp_MsrsHacking
+    } enmOp = kCpuReportOp_Normal;
+    g_pReportOut = NULL;
+    g_pDebugOut  = NULL;
+    const char *pszOutput   = NULL;
+    const char *pszDebugOut = NULL;
+
+    int iOpt;
+    RTGETOPTUNION ValueUnion;
+    while ((iOpt = RTGetOpt(&State, &ValueUnion)) != 0)
+    {
+        switch (iOpt)
+        {
+            case 'm':
+                enmOp = kCpuReportOp_MsrsOnly;
+                break;
+
+            case 'd':
+                enmOp = kCpuReportOp_MsrsHacking;
+                break;
+
+            case 'n':
+                g_fNoMsrs = true;
+                break;
+
+            case 'o':
+                pszOutput = ValueUnion.psz;
+                break;
+
+            case 'l':
+                pszDebugOut = ValueUnion.psz;
+                break;
+
+            case 'h':
+                RTPrintf("Usage: VBoxCpuReport [-m|--msrs-only] [-d|--msrs-dev] [-n|--no-msrs] [-h|--help] [-V|--version] [-o filename.h] [-l debug.log]\n");
+                RTPrintf("Internal tool for gathering information to the VMM CPU database.\n");
+                return RTEXITCODE_SUCCESS;
+            case 'V':
+                RTPrintf("%sr%s\n", RTBldCfgVersion(), RTBldCfgRevisionStr());
+                return RTEXITCODE_SUCCESS;
+            default:
+                return RTGetOptPrintError(iOpt, &ValueUnion);
+        }
+    }
+
+    /*
+     * Open the alternative debug log stream.
+     */
+    if (pszDebugOut)
+    {
+        if (RTFileExists(pszDebugOut) && !RTSymlinkExists(pszDebugOut))
+        {
+            char szOld[RTPATH_MAX];
+            rc = RTStrCopy(szOld, sizeof(szOld), pszDebugOut);
+            if (RT_SUCCESS(rc))
+                rc = RTStrCat(szOld, sizeof(szOld), ".old");
+            if (RT_SUCCESS(rc))
+                RTFileRename(pszDebugOut, szOld, RTFILEMOVE_FLAGS_REPLACE);
+        }
+        rc = RTStrmOpen(pszDebugOut, "w", &g_pDebugOut);
+        if (RT_FAILURE(rc))
+        {
+            RTMsgError("Error opening '%s': %Rrc", pszDebugOut, rc);
+            g_pDebugOut = NULL;
+        }
+    }
+
+    /*
+     * Do the requested job.
+     */
+    rc = VERR_INTERNAL_ERROR;
+    switch (enmOp)
+    {
+        case kCpuReportOp_Normal:
+            /* switch output file. */
+            if (pszOutput)
+            {
+                if (RTFileExists(pszOutput) && !RTSymlinkExists(pszOutput))
+                {
+                    char szOld[RTPATH_MAX];
+                    rc = RTStrCopy(szOld, sizeof(szOld), pszOutput);
+                    if (RT_SUCCESS(rc))
+                        rc = RTStrCat(szOld, sizeof(szOld), ".old");
+                    if (RT_SUCCESS(rc))
+                        RTFileRename(pszOutput, szOld, RTFILEMOVE_FLAGS_REPLACE);
+                }
+                rc = RTStrmOpen(pszOutput, "w", &g_pReportOut);
+                if (RT_FAILURE(rc))
+                {
+                    RTMsgError("Error opening '%s': %Rrc", pszOutput, rc);
+                    break;
+                }
+            }
+            rc = produceCpuReport();
+            break;
+        case kCpuReportOp_MsrsOnly:
+        case kCpuReportOp_MsrsHacking:
+            rc = probeMsrs(enmOp == kCpuReportOp_MsrsHacking, NULL, NULL, NULL, 0);
+            break;
+    }
+
+    /*
+     * Close the output files.
+     */
+    if (g_pReportOut)
+    {
+        RTStrmClose(g_pReportOut);
+        g_pReportOut = NULL;
+    }
+
+    if (g_pDebugOut)
+    {
+        RTStrmClose(g_pDebugOut);
+        g_pDebugOut = NULL;
+    }
+
+    return RT_SUCCESS(rc) ? RTEXITCODE_SUCCESS : RTEXITCODE_FAILURE;
+}
+
diff --git a/src/VBox/VMM/tools/VBoxCpuReport.h b/src/VBox/VMM/tools/VBoxCpuReport.h
new file mode 100644
index 00000000..6f1aa8c5
--- /dev/null
+++ b/src/VBox/VMM/tools/VBoxCpuReport.h
@@ -0,0 +1,53 @@
+/* $Id: VBoxCpuReport.h $ */
+/** @file
+ * VBoxCpuReport internal header file.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef VMM_INCLUDED_SRC_tools_VBoxCpuReport_h
+#define VMM_INCLUDED_SRC_tools_VBoxCpuReport_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <VBox/sup.h>
+
+RT_C_DECLS_BEGIN
+
+typedef struct VBCPUREPMSRACCESSORS
+{
+    /** Wheter MSR prober can read/modify/restore MSRs more or less
+     *  atomically, without allowing other code to be executed. */
+    bool                    fAtomic;
+    /** @copydoc SUPR3MsrProberRead  */
+    DECLCALLBACKMEMBER(int, pfnMsrProberRead)(uint32_t uMsr, RTCPUID idCpu, uint64_t *puValue, bool *pfGp);
+    /** @copydoc SUPR3MsrProberWrite  */
+    DECLCALLBACKMEMBER(int, pfnMsrProberWrite)(uint32_t uMsr, RTCPUID idCpu, uint64_t uValue, bool *pfGp);
+    /** @copydoc SUPR3MsrProberModify */
+    DECLCALLBACKMEMBER(int, pfnMsrProberModify)(uint32_t uMsr, RTCPUID idCpu, uint64_t fAndMask, uint64_t fOrMask,
+                                                PSUPMSRPROBERMODIFYRESULT pResult);
+    /** Termination callback, optional. */
+    DECLCALLBACKMEMBER(void, pfnTerm)(void);
+} VBCPUREPMSRACCESSORS;
+typedef VBCPUREPMSRACCESSORS *PVBCPUREPMSRACCESSORS;
+
+extern void vbCpuRepDebug(const char *pszMsg, ...);
+extern void vbCpuRepPrintf(const char *pszMsg, ...);
+extern int  VbCpuRepMsrProberInitSupDrv(PVBCPUREPMSRACCESSORS pMsrAccessors);
+extern int  VbCpuRepMsrProberInitPlatform(PVBCPUREPMSRACCESSORS pMsrAccessors);
+
+RT_C_DECLS_END
+
+#endif /* !VMM_INCLUDED_SRC_tools_VBoxCpuReport_h */
+
diff --git a/src/VBox/VMM/tools/VBoxCpuReportMsrLinux.cpp b/src/VBox/VMM/tools/VBoxCpuReportMsrLinux.cpp
new file mode 100644
index 00000000..52368498
--- /dev/null
+++ b/src/VBox/VMM/tools/VBoxCpuReportMsrLinux.cpp
@@ -0,0 +1,170 @@
+/* $Id: VBoxCpuReportMsrLinux.cpp $ */
+/** @file
+ * MsrLinux - Linux-specific MSR access.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include "VBoxCpuReport.h"
+
+#include <iprt/err.h>
+#include <iprt/file.h>
+#include <iprt/thread.h>
+
+#ifndef RT_OS_WINDOWS
+# include <unistd.h>
+#else /* RT_OS_WINDOWS: for test compiling this file on windows */
+# include <io.h>
+int pread(int, void *, size_t, off_t);
+int pwrite(int, void const *, size_t, off_t);
+#endif
+#include <fcntl.h>
+#include <errno.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#define MSR_DEV_NAME    "/dev/cpu/0/msr"
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** The /dev/xxx/msr file descriptor. */
+static int              g_fdMsr;
+
+
+/**
+ * @interface_method_impl{VBCPUREPMSRACCESSORS,pfnMsrProberRead}
+ */
+static DECLCALLBACK(int) linuxMsrProberRead(uint32_t uMsr, RTCPUID idCpu, uint64_t *puValue, bool *pfGp)
+{
+    int  rc = VINF_SUCCESS;
+
+    if (idCpu != NIL_RTCPUID)
+        return VERR_INVALID_PARAMETER;
+
+    if (g_fdMsr < 0)
+        return VERR_INVALID_STATE;
+
+    *pfGp = true;
+    if (pread(g_fdMsr, puValue, sizeof(*puValue), uMsr) != sizeof(*puValue))
+        rc = VERR_READ_ERROR;
+    else
+        *pfGp = false;
+
+    return RT_SUCCESS(rc) && !pfGp;
+}
+
+
+/**
+ * @interface_method_impl{VBCPUREPMSRACCESSORS,pfnMsrProberWrite}
+ */
+static DECLCALLBACK(int) linuxMsrProberWrite(uint32_t uMsr, RTCPUID idCpu, uint64_t uValue, bool *pfGp)
+{
+    int  rc = VINF_SUCCESS;
+
+    if (idCpu != NIL_RTCPUID)
+        return VERR_INVALID_PARAMETER;
+
+    if (g_fdMsr < 0)
+        return VERR_INVALID_STATE;
+
+    *pfGp = true;
+    if (pwrite(g_fdMsr, &uValue, sizeof(uValue), uMsr) != sizeof(uValue))
+        rc = VERR_WRITE_ERROR;
+    else
+        *pfGp = false;
+
+    return RT_SUCCESS(rc) && !pfGp;
+}
+
+/**
+ * @interface_method_impl{VBCPUREPMSRACCESSORS,pfnMsrProberModify}
+ */
+static DECLCALLBACK(int) linuxMsrProberModify(uint32_t uMsr, RTCPUID idCpu, uint64_t fAndMask, uint64_t fOrMask,
+                                              PSUPMSRPROBERMODIFYRESULT pResult)
+{
+    int         rc = VINF_SUCCESS;
+    uint64_t    uBefore, uWrite, uAfter;
+    int         rcBefore, rcWrite, rcAfter, rcRestore;
+
+    if (idCpu != NIL_RTCPUID)
+        return VERR_INVALID_PARAMETER;
+
+    if (g_fdMsr < 0)
+        return VERR_INVALID_STATE;
+
+#if 0
+    vbCpuRepDebug("MSR %#x\n", uMsr);
+    RTThreadSleep(10);
+#endif
+    rcBefore = pread(g_fdMsr, &uBefore, sizeof(uBefore), uMsr);
+    uWrite = (uBefore & fAndMask) | fOrMask;
+    rcWrite = pwrite(g_fdMsr, &uWrite, sizeof(uWrite), uMsr);
+    rcAfter = pread(g_fdMsr, &uAfter, sizeof(uAfter), uMsr);
+    rcRestore = pwrite(g_fdMsr, &uBefore, sizeof(uBefore), uMsr);
+
+#if 0
+    vbCpuRepDebug("MSR: %#x, %#llx -> %#llx -> %#llx (%d/%d/%d/%d)\n",
+                  uMsr, uBefore, uWrite, uAfter,
+                  rcBefore, rcWrite != sizeof(uWrite), rcAfter, rcRestore);
+#endif
+    pResult->uBefore    = uBefore;
+    pResult->uWritten   = uWrite;
+    pResult->uAfter     = uAfter;
+    pResult->fBeforeGp  = rcBefore  != sizeof(uBefore);
+    pResult->fModifyGp  = rcWrite   != sizeof(uWrite);
+    pResult->fAfterGp   = rcAfter   != sizeof(uAfter);
+    pResult->fRestoreGp = rcRestore != sizeof(uBefore);
+
+    return rc;
+}
+
+
+/**
+ * @interface_method_impl{VBCPUREPMSRACCESSORS,pfnTerm}
+ */
+static DECLCALLBACK(void) linuxMsrProberTerm(void)
+{
+    if (g_fdMsr >= 0)
+    {
+        close(g_fdMsr);
+        g_fdMsr = -1;
+    }
+}
+
+int VbCpuRepMsrProberInitPlatform(PVBCPUREPMSRACCESSORS pMsrAccessors)
+{
+    RTFILE hFile;
+    int rc = RTFileOpen(&hFile, MSR_DEV_NAME, RTFILE_O_READWRITE | RTFILE_O_DENY_NONE | RTFILE_O_OPEN);
+    if (RT_SUCCESS(rc))
+    {
+        g_fdMsr = RTFileToNative(hFile);
+        Assert(g_fdMsr != -1);
+
+        pMsrAccessors->fAtomic             = false; /* Can't modify/restore MSRs without trip to R3. */
+        pMsrAccessors->pfnMsrProberRead    = linuxMsrProberRead;
+        pMsrAccessors->pfnMsrProberWrite   = linuxMsrProberWrite;
+        pMsrAccessors->pfnMsrProberModify  = linuxMsrProberModify;
+        pMsrAccessors->pfnTerm             = linuxMsrProberTerm;
+        return VINF_SUCCESS;
+    }
+    vbCpuRepDebug("warning: Failed to open " MSR_DEV_NAME ": %Rrc\n", rc);
+    return rc;
+}
diff --git a/src/VBox/VMM/tools/VBoxCpuReportMsrSup.cpp b/src/VBox/VMM/tools/VBoxCpuReportMsrSup.cpp
new file mode 100644
index 00000000..60c87690
--- /dev/null
+++ b/src/VBox/VMM/tools/VBoxCpuReportMsrSup.cpp
@@ -0,0 +1,54 @@
+/* $Id: VBoxCpuReportMsrSup.cpp $ */
+/** @file
+ * MsrSup - SupDrv-specific MSR access.
+ */
+
+/*
+ * Copyright (C) 2013-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include "VBoxCpuReport.h"
+#include <iprt/errcore.h>
+#include <iprt/x86.h>
+
+
+int VbCpuRepMsrProberInitSupDrv(PVBCPUREPMSRACCESSORS pMsrFunctions)
+{
+    int rc = SUPR3Init(NULL);
+    if (RT_SUCCESS(rc))
+    {
+        /* Test if the MSR prober is available, since the interface is optional. The TSC MSR will exist on any supported CPU. */
+        uint64_t uValue;
+        bool     fGp;
+        rc = SUPR3MsrProberRead(MSR_IA32_TSC, NIL_RTCPUID, &uValue, &fGp);
+        if (   rc != VERR_NOT_IMPLEMENTED
+            && rc != VERR_INVALID_FUNCTION)
+        {
+            pMsrFunctions->fAtomic            = true;
+            pMsrFunctions->pfnMsrProberRead   = SUPR3MsrProberRead;
+            pMsrFunctions->pfnMsrProberWrite  = SUPR3MsrProberWrite;
+            pMsrFunctions->pfnMsrProberModify = SUPR3MsrProberModify;
+
+            pMsrFunctions->pfnTerm            = NULL;
+            return VINF_SUCCESS;
+
+        }
+        vbCpuRepDebug("warning: MSR probing not supported by the support driver (%Rrc).\n", rc);
+    }
+    else
+        vbCpuRepDebug("warning: Unable to initialize the support library (%Rrc).\n", rc);
+    return rc;
+}
+
diff --git a/src/VBox/VMM/tools/VBoxVMMPreload.cpp b/src/VBox/VMM/tools/VBoxVMMPreload.cpp
new file mode 100644
index 00000000..1b9d93c8
--- /dev/null
+++ b/src/VBox/VMM/tools/VBoxVMMPreload.cpp
@@ -0,0 +1,224 @@
+/* $Id: VBoxVMMPreload.cpp $ */
+/** @file
+ * VBoxVMMPreload - Preload VBox the ring-0 modules.
+ */
+
+/*
+ * Copyright (C) 2012-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <iprt/buildconfig.h>
+#include <iprt/getopt.h>
+#include <iprt/initterm.h>
+#include <iprt/message.h>
+#include <iprt/path.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+#include <VBox/sup.h>
+#include <VBox/version.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/**
+ * Known modules and their associated data (there are only known modules!).
+ */
+static struct
+{
+    const char     *pszName;
+    bool            fPreload;
+    void           *pvImageBase;
+} g_aModules[] =
+{
+    { "VMMR0.r0",       true,  NULL },
+    { "VBoxDDR0.r0",    true,  NULL },
+};
+
+static uint32_t     g_cVerbose = 1;
+static bool         g_fLockDown = false;
+
+
+/**
+ * Parses the options.
+ *
+ * @returns RTEXITCODE_SUCCESS on success.
+ * @param   argc                Argument count .
+ * @param   argv                Argument vector.
+ * @param   pfExit              Set to @c true if we should exit.
+ */
+static RTEXITCODE ParseOptions(int argc, char **argv, bool *pfExit)
+{
+    /*
+     * Parse arguments.
+     */
+    static const RTGETOPTDEF s_aOptions[] =
+    {
+        { "--only",     'o', RTGETOPT_REQ_STRING  },
+        { "--quiet",    'q', RTGETOPT_REQ_NOTHING },
+        { "--lock" ,    'l', RTGETOPT_REQ_NOTHING },
+        { "--verbose",  'v', RTGETOPT_REQ_NOTHING },
+    };
+
+    bool            fAll     = true;
+
+    int             ch;
+    RTGETOPTUNION   ValueUnion;
+    RTGETOPTSTATE   GetState;
+    RTGetOptInit(&GetState, argc, argv, s_aOptions, RT_ELEMENTS(s_aOptions), 1, 0 /* fFlags */);
+    while ((ch = RTGetOpt(&GetState, &ValueUnion)))
+    {
+        switch(ch)
+        {
+            case 'o':
+            {
+                uint32_t i;
+
+                if (fAll)
+                {
+                    fAll = false;
+                    for (i = 0; i < RT_ELEMENTS(g_aModules); i++)
+                        g_aModules[i].fPreload = false;
+                }
+
+                i = RT_ELEMENTS(g_aModules);
+                while (i-- > 0)
+                    if (!strcmp(ValueUnion.psz, g_aModules[i].pszName))
+                    {
+                        g_aModules[i].fPreload = true;
+                        break;
+                    }
+                if (i > RT_ELEMENTS(g_aModules))
+                    return RTMsgErrorExit(RTEXITCODE_FAILURE, "No known module '%s'", ValueUnion.psz);
+                break;
+            }
+
+            case 'v':
+                g_cVerbose++;
+                break;
+
+            case 'q':
+                g_cVerbose = 0;
+                break;
+
+            case 'l':
+                g_fLockDown = true;
+                break;
+
+            case 'h':
+                RTPrintf(VBOX_PRODUCT " VMM ring-0 Module Preloader Version " VBOX_VERSION_STRING
+                         "(C) 2005-" VBOX_C_YEAR " " VBOX_VENDOR "\n"
+                         "All rights reserved.\n"
+                         "\n"
+                         "Usage: VBoxVMMPreload [-hlqvV] [-o|--only <mod>]\n"
+                         "\n");
+                *pfExit = true;
+                return RTEXITCODE_SUCCESS;
+
+            case 'V':
+                RTPrintf("%sr%s\n", RTBldCfgVersion(), RTBldCfgRevisionStr());
+                *pfExit = true;
+                return RTEXITCODE_SUCCESS;
+
+            default:
+                return RTGetOptPrintError(ch, &ValueUnion);
+        }
+    }
+    return RTEXITCODE_SUCCESS;
+}
+
+
+/**
+ * Loads the modules.
+ *
+ * @returns RTEXITCODE_SUCCESS on success.
+ */
+static RTEXITCODE LoadModules(void)
+{
+    RTERRINFOSTATIC ErrInfo;
+
+    for (uint32_t i = 0; i < RT_ELEMENTS(g_aModules); i++)
+    {
+        if (g_aModules[i].fPreload)
+        {
+            char            szPath[RTPATH_MAX];
+            int rc = RTPathAppPrivateArch(szPath, sizeof(szPath));
+            if (RT_SUCCESS(rc))
+                rc = RTPathAppend(szPath, sizeof(szPath), g_aModules[i].pszName);
+            if (RT_FAILURE(rc))
+                return RTMsgErrorExit(RTEXITCODE_FAILURE, "RTPathAppPrivateArch or RTPathAppend returned %Rrc", rc);
+
+            RTErrInfoInitStatic(&ErrInfo);
+            rc = SUPR3LoadModule(szPath, g_aModules[i].pszName, &g_aModules[i].pvImageBase, &ErrInfo.Core);
+            if (RT_FAILURE(rc))
+                return RTMsgErrorExit(RTEXITCODE_FAILURE, "SUPR3LoadModule failed for %s (%s): %s (rc=%Rrc)",
+                                      g_aModules[i].pszName, szPath, ErrInfo.Core.pszMsg, rc);
+            if (g_cVerbose >= 1)
+                RTMsgInfo("Loaded '%s' ('%s') at %p\n", szPath, g_aModules[i].pszName, g_aModules[i].pvImageBase);
+        }
+    }
+
+    if (g_fLockDown)
+    {
+        RTErrInfoInitStatic(&ErrInfo);
+        int rc = SUPR3LockDownLoader(&ErrInfo.Core);
+        if (RT_FAILURE(rc))
+            return RTMsgErrorExit(RTEXITCODE_FAILURE, "SUPR3LockDownLoader failed: %s (rc=%Rrc)",
+                                  ErrInfo.Core.pszMsg, rc);
+        if (g_cVerbose >= 1)
+            RTMsgInfo("Locked down module loader interface!\n");
+    }
+
+    RTStrmFlush(g_pStdOut);
+    return RTEXITCODE_SUCCESS;
+}
+
+
+/**
+ *  Entry point.
+ */
+extern "C" DECLEXPORT(int) TrustedMain(int argc, char **argv, char **envp)
+{
+    RT_NOREF1(envp);
+    bool fExit = false;
+    RTEXITCODE rcExit = ParseOptions(argc, argv, &fExit);
+    if (rcExit == RTEXITCODE_SUCCESS && !fExit)
+    {
+        rcExit = LoadModules();
+        if (rcExit == RTEXITCODE_SUCCESS)
+        {
+            for (;;)
+                RTThreadSleep(RT_INDEFINITE_WAIT);
+        }
+    }
+    return rcExit;
+}
+
+
+#ifndef VBOX_WITH_HARDENING
+/**
+ * Main entry point.
+ */
+int main(int argc, char **argv, char **envp)
+{
+    int rc = RTR3InitExe(argc, &argv, RTR3INIT_FLAGS_SUPLIB);
+    if (RT_FAILURE(rc))
+        return RTMsgInitFailure(rc);
+    return TrustedMain(argc, argv, envp);
+}
+#endif /* !VBOX_WITH_HARDENING */
+
diff --git a/src/VBox/VMM/tools/VBoxVMMPreloadHardened.cpp b/src/VBox/VMM/tools/VBoxVMMPreloadHardened.cpp
new file mode 100644
index 00000000..9c33f874
--- /dev/null
+++ b/src/VBox/VMM/tools/VBoxVMMPreloadHardened.cpp
@@ -0,0 +1,25 @@
+/* $Id: VBoxVMMPreloadHardened.cpp $ */
+/** @file
+ * VBoxVMMPreload - Hardened main().
+ */
+
+/*
+ * Copyright (C) 2012-2020 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#include <VBox/sup.h>
+
+
+int main(int argc, char **argv, char **envp)
+{
+    return SUPR3HardenedMain("VBoxVMMPreload", 0, argc, argv, envp);
+}
+
-- 
cgit v1.2.3